From 4443bc775442d568357f72d96e2fbdae2ea58c3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 17 Jun 2016 05:11:29 +0200
Subject: make_preload: Save some memory by making preloaded code 'const'

Mark the preloaded code 'const' to allow the compiler to put it into
the 'text' segment instead of into the 'data' segment. Since the
'text' segment is shared among all instances of the Erlang virtual
machine, this change could potentially reduce memory consumption
(slightly).

Before the change:

$ size bin/x86_64-unknown-linux-gnu/beam.smp
   text	   data	    bss	    dec	    hex	filename
2920246	 352273	 158472	3430991	 345a4f	bin/x86_64-unknown-linux-gnu/beam.smp

After the change:

$ size bin/x86_64-unknown-linux-gnu/beam.smp
   text	   data	    bss	    dec	    hex	filename
3081046	 191473	 158472	3430991	 345a4f	bin/x86_64-unknown-linux-gnu/beam.smp

Roughly speaking, this change cuts the size of the data segment in half.
---
 erts/emulator/utils/make_preload | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/make_preload b/erts/emulator/utils/make_preload
index f489bc2a39..8b629d9517 100755
--- a/erts/emulator/utils/make_preload
+++ b/erts/emulator/utils/make_preload
@@ -94,8 +94,8 @@ foreach $file (@ARGV) {
 	close(FILE);
 	
 	push(@modules, "  {\"$module\", " . length($_) . ", preloaded_$module},\n");
-	print "unsigned preloaded_size_$module = ", length($_), ";\n";
-	print "unsigned char preloaded_$module", "[] = {\n";
+	print "const unsigned preloaded_size_$module = ", length($_), ";\n";
+	print "const unsigned char preloaded_$module", "[] = {\n";
 	for ($i = 0; $i < length($_); $i++) {
 	    if ($i % 8 == 0 && $comment ne '') {
 		$comment =~ s@/\*@..@g; # Comment start -- avoid warning.
@@ -125,10 +125,10 @@ if ($gen_rc) {
     print @modules;
     print "END\n";
 } elsif ($gen_old) {
-    print "struct {\n";
+    print "const struct {\n";
     print "   char* name;\n";
     print "   int size;\n";
-    print "   unsigned char* code;\n";
+    print "   const unsigned char* code;\n";
     print "} pre_loaded[] = {\n";
     foreach (@modules) {
 	print;
-- 
cgit v1.2.3


From 09b73b39dde3f2c8924947fc5512281b83c939ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 20 Jun 2016 12:41:16 +0200
Subject: make_tables: Remove broken automatic BIF aliasing

The make_tables script still contains a broken implementation
of a mechanism to automatically give a BIF an additional
(for example, was used for erlang:'++'/2 and erlang:append/2).
When that featured broke, it was worked around by adding
additional entries to bif.tab. There is therefore no reason
to mend the feature.
---
 erts/emulator/Makefile.in       |  3 +--
 erts/emulator/utils/make_tables | 29 ++++-------------------------
 2 files changed, 5 insertions(+), 27 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 2212aed5e0..7c9b29b146 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -564,7 +564,6 @@ $(TARGET)/erl_bif_wrap.c \
 $(TARGET)/erl_bif_list.h \
 $(TARGET)/erl_atom_table.c \
 $(TARGET)/erl_atom_table.h \
-$(TARGET)/erl_pbifs.c \
 	: $(TARGET)/TABLES-GENERATED
 $(TARGET)/TABLES-GENERATED: $(ATOMS) $(BIFS) utils/make_tables
 	$(gen_verbose)LANG=C $(PERL) utils/make_tables -src $(TARGET) -include $(TARGET)\
@@ -744,7 +743,7 @@ EMU_OBJS = \
 	$(OBJDIR)/beam_ranges.o
 
 RUN_OBJS = \
-	$(OBJDIR)/erl_pbifs.o		$(OBJDIR)/benchmark.o \
+	$(OBJDIR)/benchmark.o \
 	$(OBJDIR)/erl_alloc.o		$(OBJDIR)/erl_mtrace.o \
 	$(OBJDIR)/erl_alloc_util.o	$(OBJDIR)/erl_goodfit_alloc.o \
 	$(OBJDIR)/erl_bestfit_alloc.o	$(OBJDIR)/erl_afit_alloc.o \
diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables
index c158778f43..82c8c299f4 100755
--- a/erts/emulator/utils/make_tables
+++ b/erts/emulator/utils/make_tables
@@ -36,7 +36,6 @@ use File::Basename;
 #    <-src>/erl_am.c
 #    <-src>/erl_bif_table.c
 #    <-src>/erl_bif_wrap.c
-#    <-src>/erl_pbifs.c
 #    <-include>/erl_atom_table.h
 #    <-include>/erl_bif_table.h
 #
@@ -54,8 +53,6 @@ my %aliases;
 my $auto_alias_num = 0;
 
 my @bif;
-my @implementation;
-my @pbif;
 
 while (@ARGV && $ARGV[0] =~ /^-(\w+)/) {
     my $opt = shift;
@@ -80,7 +77,10 @@ while (<>) {
     if ($type eq 'atom') {
 	save_atoms(@args);
     } elsif ($type eq 'bif' or $type eq 'ubif') {
-	my($bif,$alias,$alias2) = (@args);
+	if (@args > 2) {
+	    error("$type only allows two arguments");
+	}
+	my($bif,$alias) = (@args);
 	$bif =~ m@^([a-z_.'0-9]+):(.*)/(\d)$@ or error("invalid BIF");
 	my($mod,$name,$arity) = ($1,$2,$3);
 	save_atoms($mod, $name);
@@ -94,8 +94,6 @@ while (<>) {
 	$wrapper = $alias if $type eq 'ubif';
 	push(@bif, ["am_$atom_alias{$mod}","am_$atom_alias{$name}",$arity,
 		    $alias,$wrapper]);
-        push(@pbif, $bif =~ m/^'/ && $alias =~ m/^ebif_/);
-        push(@implementation, $alias2);
     } else {
 	error("invalid line");
     }
@@ -228,25 +226,6 @@ for ($i = 0; $i < @bif; $i++) {
 # Generate the package bif file.
 #
 
-open_file("$src/erl_pbifs.c");
-my $i;
-includes("export.h", "sys.h", "erl_vm.h", "global.h", "erl_process.h", "bif.h",
-	 "erl_bif_table.h", "erl_atom_table.h");
-for ($i = 0; $i < @bif; $i++) {
-    my $arity = $bif[$i]->[2];
-    my $func = $bif[$i]->[3];
-    my $arg;
-    next unless $pbif[$i];
-    next unless $func =~ m/^ebif_(.*)/;
-    my $orig_func = $1;
-    $orig_func = $implementation[$i] if $implementation[$i];
-    print "Eterm\n";
-    print "$func(Process* p, Eterm* BIF__ARGS)\n";
-    print "{\n";
-    print "    return $orig_func(p, BIF__ARGS);\n";
-    print "}\n\n";
-}
-
 sub open_file {			# or die
     my($name) = @_;
 
-- 
cgit v1.2.3


From 6a6c72801e6fb021851b4f2591b03ed136a27363 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 17 Jun 2016 12:54:26 +0200
Subject: beam_makeops: Separate static information from counters

The counters are only used in the special 'icount' emulator.

We will save some memory by including the counters in the
OpEntry. It will also make it possible to make opc 'const'.
---
 erts/emulator/beam/erl_bif_info.c |  2 +-
 erts/emulator/beam/erl_vm.h       |  3 ++-
 erts/emulator/utils/beam_makeops  | 13 +++++++++++--
 3 files changed, 14 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 3fb866733c..e5d3f38ce4 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2394,7 +2394,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 							     ERTS_ATOM_ENC_LATIN1,
 							     1),
 					       erts_bld_uint(hpp, hszp,
-							     opc[i].count)),
+							     erts_instr_count[i])),
 				res);
 	}
 
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index f97716d030..22c8e8ee12 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -146,12 +146,13 @@ typedef struct op_entry {
    int sz;			/* Number of loaded words. */
    char* pack;			/* Instructions for packing engine. */
    char* sign;			/* Signature string. */
-   unsigned count;		/* Number of times executed. */
 } OpEntry;
 
 extern OpEntry opc[];		/* Description of all instructions. */
 extern int num_instructions;	/* Number of instruction in opc[]. */
 
+extern Uint erts_instr_count[];
+
 /* some constants for various table sizes etc */
 
 #define ATOM_TEXT_SIZE  32768	/* Increment for allocating atom text space */
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 4407f7e289..acc0084145 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -566,7 +566,7 @@ sub emulator_output {
 		$sep = ",";
 	    }
 	    $init .= "}";
-	    init_item($print_name, $init, $involves_r, $size, $pack, $sign, 0);
+	    init_item($print_name, $init, $involves_r, $size, $pack, $sign);
 	    $op_to_name[$spec_opnum] = $instr;
 	    $spec_opnum++;
 	}
@@ -574,6 +574,15 @@ sub emulator_output {
     print "};\n\n";
     print "int num_instructions = $spec_opnum;\n\n";
 
+    #
+    # Print the array for instruction counts.
+    #
+
+    print "#ifdef ERTS_OPCODE_COUNTER_SUPPORT\n";
+    print "Uint erts_instr_count[$spec_opnum];\n";
+    print "#endif\n";
+    print "\n";
+
     #
     # Generate transformations.
     #
@@ -708,7 +717,7 @@ sub emulator_output {
     print "#define DEFINE_COUNTING_LABELS";
     for ($i = 0; $i < @op_to_name; $i++) {
 	my($name) = $op_to_name[$i];
-	print " \\\nCountCase($name): opc[$i].count++; goto lb_$name;";
+	print " \\\nCountCase($name): erts_instr_count[$i]++; goto lb_$name;";
     }
     print "\n\n";
 
-- 
cgit v1.2.3


From 644cc778e9770b847700dca41a0c3ebe20da64df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 17 Jun 2016 05:23:08 +0200
Subject: beam_makeops: Save some memory by making loader tables 'const'

Before:

$ size bin/x86_64-unknown-linux-gnu/beam.smp
   text	   data	    bss	    dec	    hex	filename
3080982	 188369	 158472	3427823	 344def	bin/x86_64-unknown-linux-gnu/beam.smp

After:

$ size bin/x86_64-unknown-linux-gnu/beam.smp
   text	   data	    bss	    dec	    hex	filename
3164694	 104657	 158472	3427823	 344def	bin/x86_64-unknown-linux-gnu/beam.smp
---
 erts/emulator/beam/beam_load.c   |  4 ++--
 erts/emulator/beam/beam_load.h   |  2 +-
 erts/emulator/beam/erl_vm.h      |  4 ++--
 erts/emulator/utils/beam_makeops | 14 +++++++-------
 4 files changed, 12 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 0c2743beb2..4ca2c6a13f 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4860,7 +4860,7 @@ transform_engine(LoaderState* st)
 {
     Uint op;
     int ap;			/* Current argument. */
-    Uint* restart;		/* Where to restart if current match fails. */
+    const Uint* restart; /* Where to restart if current match fails. */
     GenOpArg var[TE_MAX_VARS];	/* Buffer for variables. */
     GenOpArg* rest_args = NULL;
     int num_rest_args = 0;
@@ -4869,7 +4869,7 @@ transform_engine(LoaderState* st)
     GenOp* instr;
     GenOp* first = st->genop;
     GenOp* keep = NULL;
-    Uint* pc;
+    const Uint* pc;
     static Uint restart_fail[1] = {TOP_fail};
 
     ASSERT(gen_opc[first->op].transform != -1);
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index fd2dd97fee..92a9abddbf 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -35,7 +35,7 @@ typedef struct gen_op_entry {
    int transform;
 } GenOpEntry;
 
-extern GenOpEntry gen_opc[];
+extern const GenOpEntry gen_opc[];
 
 #ifdef NO_JUMP_TABLE 
 #define BeamOp(Op) (Op)
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 22c8e8ee12..60c2349f36 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -148,8 +148,8 @@ typedef struct op_entry {
    char* sign;			/* Signature string. */
 } OpEntry;
 
-extern OpEntry opc[];		/* Description of all instructions. */
-extern int num_instructions;	/* Number of instruction in opc[]. */
+extern const OpEntry opc[];	/* Description of all instructions. */
+extern const int num_instructions; /* Number of instruction in opc[]. */
 
 extern Uint erts_instr_count[];
 
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index acc0084145..9813142585 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -475,7 +475,7 @@ sub emulator_output {
     print '#include "beam_load.h"', "\n";
     print "\n";
 
-    print "char tag_to_letter[] = {\n  ";
+    print "const char tag_to_letter[] = {\n  ";
     for ($i = 0; $i < length($genop_types); $i++) {
 	print "'$tag_type[$i]', ";
     }
@@ -489,7 +489,7 @@ sub emulator_output {
     # Generate code for specific ops.
     #
     my($spec_opnum) = 0;
-    print "OpEntry opc[] = {\n";
+    print "const OpEntry opc[] = {\n";
     foreach $key (sort keys %specific_op) {
 	$gen_to_spec{$key} = $spec_opnum;
 	$num_specific{$key} = @{$specific_op{$key}};
@@ -572,7 +572,7 @@ sub emulator_output {
 	}
     }
     print "};\n\n";
-    print "int num_instructions = $spec_opnum;\n\n";
+    print "const int num_instructions = $spec_opnum;\n\n";
 
     #
     # Print the array for instruction counts.
@@ -593,7 +593,7 @@ sub emulator_output {
     # Print the generic instruction table.
     #
 
-    print "GenOpEntry gen_opc[] = {\n";
+    print "const GenOpEntry gen_opc[] = {\n";
     for ($i = 0; $i < @gen_opname; $i++) {
 	if ($i == $num_file_opcodes) {
 	    print "\n/*\n * Internal generic instructions.\n */\n\n";
@@ -687,8 +687,8 @@ sub emulator_output {
     print "#define TE_MAX_VARS $te_max_vars\n";
     print "\n";
 
-    print "extern char tag_to_letter[];\n";
-    print "extern Uint op_transform[];\n";
+    print "extern const char tag_to_letter[];\n";
+    print "extern const Uint op_transform[];\n";
     print "\n";
 
     for ($i = 0; $i < @op_to_name; $i++) {
@@ -1426,7 +1426,7 @@ sub tr_gen {
     # Print the generated transformation engine.
     #
     my($offset) = 0;
-    print "Uint op_transform[] = {\n";
+    print "const Uint op_transform[] = {\n";
     foreach $key (sort keys %gen_transform) {
 	$gen_transform_offset{$key} = $offset;
 	my @instr = @{$gen_transform{$key}};
-- 
cgit v1.2.3


From f0f4e72c8ec5c08993ff84d4eac5c48897a09657 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 20 Jun 2016 13:02:59 +0200
Subject: Simplify creation of new GC BIFs

Add the BIF type "gcbif" in bif.tab for defining GC BIFs. That will
eliminate some of the hand-written administrative code for handling
GC BIFs, saving the developer's time.
---
 erts/emulator/Makefile.in       |   2 +
 erts/emulator/beam/beam_emu.c   |  31 +++--------
 erts/emulator/beam/beam_load.c  | 111 ++++++++++++----------------------------
 erts/emulator/beam/bif.tab      |  38 +++++++-------
 erts/emulator/beam/global.h     |  12 -----
 erts/emulator/utils/make_tables |  38 ++++++++++++--
 6 files changed, 97 insertions(+), 135 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 7c9b29b146..6cbf22066c 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -562,6 +562,7 @@ $(TARGET)/erl_bif_table.c \
 $(TARGET)/erl_bif_table.h \
 $(TARGET)/erl_bif_wrap.c \
 $(TARGET)/erl_bif_list.h \
+$(TARGET)/erl_gc_bifs.c \
 $(TARGET)/erl_atom_table.c \
 $(TARGET)/erl_atom_table.h \
 	: $(TARGET)/TABLES-GENERATED
@@ -754,6 +755,7 @@ RUN_OBJS = \
 	$(OBJDIR)/erl_bif_os.o		$(OBJDIR)/erl_bif_lists.o \
 	$(OBJDIR)/erl_bif_trace.o	$(OBJDIR)/erl_bif_unique.o \
 	$(OBJDIR)/erl_bif_wrap.o \
+	$(OBJDIR)/erl_gc_bifs.o \
 	$(OBJDIR)/erl_trace.o		$(OBJDIR)/copy.o \
 	$(OBJDIR)/utils.o		$(OBJDIR)/bif.o \
 	$(OBJDIR)/io.o 			$(OBJDIR)/erl_printf_term.o\
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 4716460a6b..c9fb5a3022 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5428,31 +5428,14 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 static BifFunction
 translate_gc_bif(void* gcf)
 {
-    if (gcf == erts_gc_length_1) {
-	return length_1;
-    } else if (gcf == erts_gc_size_1) {
-	return size_1;
-    } else if (gcf == erts_gc_bit_size_1) {
-	return bit_size_1;
-    } else if (gcf == erts_gc_byte_size_1) {
-	return byte_size_1;
-    } else if (gcf == erts_gc_map_size_1) {
-	return map_size_1;
-    } else if (gcf == erts_gc_abs_1) {
-	return abs_1;
-    } else if (gcf == erts_gc_float_1) {
-	return float_1;
-    } else if (gcf == erts_gc_round_1) {
-	return round_1;
-    } else if (gcf == erts_gc_trunc_1) {
-	return round_1;
-    } else if (gcf == erts_gc_binary_part_2) {
-	return binary_part_2;
-    } else if (gcf == erts_gc_binary_part_3) {
-	return binary_part_3;
-    } else {
-	erts_exit(ERTS_ERROR_EXIT, "bad gc bif");
+    const ErtsGcBif* p;
+
+    for (p = erts_gc_bifs; p->bif != 0; p++) {
+	if (p->gc_bif == gcf) {
+	    return p->bif;
+	}
     }
+    erts_exit(ERTS_ERROR_EXIT, "bad gc bif");
 }
 
 /*
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 0c2743beb2..897b0f7e6a 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4013,60 +4013,52 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx)
     op->next = NULL;
     return op;
 }
+
+static GenOp*
+translate_gc_bif(LoaderState* stp, GenOp* op, GenOpArg Bif)
+{
+    const ErtsGcBif* p;
+    BifFunction bf;
+
+    bf = stp->import[Bif.val].bf;
+    for (p = erts_gc_bifs; p->bif != 0; p++) {
+	if (p->bif == bf) {
+	    op->a[1].type = TAG_u;
+	    op->a[1].val = (BeamInstr) p->gc_bif;
+	    return op;
+	}
+    }
+
+    op->op = genop_unsupported_guard_bif_3;
+    op->arity = 3;
+    op->a[0].type = TAG_a;
+    op->a[0].val = stp->import[Bif.val].module;
+    op->a[1].type = TAG_a;
+    op->a[1].val = stp->import[Bif.val].function;
+    op->a[2].type = TAG_u;
+    op->a[2].val = stp->import[Bif.val].arity;
+    return op;
+}
+
 /*
- * Rewrite gc_bifs with one parameter (the common case). Utilized
- * in ops.tab to rewrite instructions calling bif's in guards
- * to use a garbage collecting implementation.
+ * Rewrite gc_bifs with one parameter (the common case).
  */
 static GenOp*
 gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
 	      GenOpArg Src, GenOpArg Dst)
 {
     GenOp* op;
-    BifFunction bf;
 
     NEW_GENOP(stp, op);
     op->next = NULL;
-    bf = stp->import[Bif.val].bf;
-    /* The translations here need to have a reverse counterpart in
-       beam_emu.c:translate_gc_bif for error handling to work properly. */
-    if (bf == length_1) {
-	op->a[1].val = (BeamInstr) (void *) erts_gc_length_1;
-    } else if (bf == size_1) {
-	op->a[1].val = (BeamInstr) (void *) erts_gc_size_1;
-    } else if (bf == bit_size_1) {
-	op->a[1].val = (BeamInstr) (void *) erts_gc_bit_size_1;
-    } else if (bf == byte_size_1) {
-	op->a[1].val = (BeamInstr) (void *) erts_gc_byte_size_1;
-    } else if (bf == map_size_1) {
-	op->a[1].val = (BeamInstr) (void *) erts_gc_map_size_1;
-    } else if (bf == abs_1) {
-	op->a[1].val = (BeamInstr) (void *) erts_gc_abs_1;
-    } else if (bf == float_1) {
-	op->a[1].val = (BeamInstr) (void *) erts_gc_float_1;
-    } else if (bf == round_1) {
-	op->a[1].val = (BeamInstr) (void *) erts_gc_round_1;
-    } else if (bf == trunc_1) {
-	op->a[1].val = (BeamInstr) (void *) erts_gc_trunc_1;
-    } else {
-	op->op = genop_unsupported_guard_bif_3;
-	op->arity = 3;
-	op->a[0].type = TAG_a;
-	op->a[0].val = stp->import[Bif.val].module;
-	op->a[1].type = TAG_a;
-	op->a[1].val = stp->import[Bif.val].function;
-	op->a[2].type = TAG_u;
-	op->a[2].val = stp->import[Bif.val].arity;
-	return op;
-    }
     op->op = genop_i_gc_bif1_5;
     op->arity = 5;
     op->a[0] = Fail;
-    op->a[1].type = TAG_u;
+    /* op->a[1] is set by translate_gc_bif() */
     op->a[2] = Src;
     op->a[3] = Live;
     op->a[4] = Dst;
-    return op;
+    return translate_gc_bif(stp, op, Bif);
 }
 
 /*
@@ -4077,35 +4069,18 @@ gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
 	      GenOpArg S1, GenOpArg S2, GenOpArg Dst)
 {
     GenOp* op;
-    BifFunction bf;
 
     NEW_GENOP(stp, op);
     op->next = NULL;
-    bf = stp->import[Bif.val].bf;
-    /* The translations here need to have a reverse counterpart in
-       beam_emu.c:translate_gc_bif for error handling to work properly. */
-    if (bf == binary_part_2) {
-	op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_2;
-    } else {
-	op->op = genop_unsupported_guard_bif_3;
-	op->arity = 3;
-	op->a[0].type = TAG_a;
-	op->a[0].val = stp->import[Bif.val].module;
-	op->a[1].type = TAG_a;
-	op->a[1].val = stp->import[Bif.val].function;
-	op->a[2].type = TAG_u;
-	op->a[2].val = stp->import[Bif.val].arity;
-	return op;
-    }
     op->op = genop_i_gc_bif2_6;
     op->arity = 6;
     op->a[0] = Fail;
-    op->a[1].type = TAG_u;
+    /* op->a[1] is set by translate_gc_bif() */
     op->a[2] = Live;
     op->a[3] = S1;
     op->a[4] = S2;
     op->a[5] = Dst;
-    return op;
+    return translate_gc_bif(stp, op, Bif);
 }
 
 /*
@@ -4116,37 +4091,19 @@ gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
 	      GenOpArg S1, GenOpArg S2, GenOpArg S3, GenOpArg Dst)
 {
     GenOp* op;
-    BifFunction bf;
 
     NEW_GENOP(stp, op);
     op->next = NULL;
-    bf = stp->import[Bif.val].bf;
-    /* The translations here need to have a reverse counterpart in
-       beam_emu.c:translate_gc_bif for error handling to work properly. */
-    if (bf == binary_part_3) {
-	op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_3;
-    } else {
-	op->op = genop_unsupported_guard_bif_3;
-	op->arity = 3;
-	op->a[0].type = TAG_a;
-	op->a[0].val = stp->import[Bif.val].module;
-	op->a[1].type = TAG_a;
-	op->a[1].val = stp->import[Bif.val].function;
-	op->a[2].type = TAG_u;
-	op->a[2].val = stp->import[Bif.val].arity;
-	return op;
-    }
     op->op = genop_ii_gc_bif3_7;
     op->arity = 7;
     op->a[0] = Fail;
-    op->a[1].type = TAG_u;
+    /* op->a[1] is set by translate_gc_bif() */
     op->a[2] = Live;
     op->a[3] = S1;
     op->a[4] = S2;
     op->a[5] = S3;
     op->a[6] = Dst;
-    op->next = NULL;
-    return op;
+    return translate_gc_bif(stp, op, Bif);
 }
 
 static GenOp*
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 065018514a..978bcd4bba 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -23,22 +23,24 @@
 #
 # Lines starting with '#' are ignored.
 #
-# <bif-decl> ::= "bif" <bif> <C-name>* | "ubif" <bif> <C-name>*
+# <bif-decl> ::= "bif" <bif> <C-name>* |
+#                "ubif" <bif> <C-name>* |
+#                "gcbif" <bif> <C-name>*
 # <bif> ::= <module> ":" <name> "/" <arity>
 #
-# "ubif" is an unwrapped bif, i.e. a bif without a trace wrapper,
-# or rather; the trace entry point in the export entry is the same
-# as the normal entry point, and no trace wrapper is generated.
+# ubif:   Use for operators and guard BIFs that never build anything
+#         on the heap (such as tuple_size/1) and operators.
 #
-# Important: Use "ubif" for guard BIFs and operators; use "bif" for ordinary BIFs.
+# gcbif:  Use for guard BIFs that may build on the heap (such as abs/1).
+#
+# bif:    Use for all other BIFs.
 #
 # Add new BIFs to the end of the file.
 #
-# Note: Guards BIFs require special support in the compiler (to be able to actually
-# call them from within a guard).
+# Note: Guards BIFs usually require special support in the compiler.
 #
 
-ubif erlang:abs/1
+gcbif erlang:abs/1
 bif erlang:adler32/1
 bif erlang:adler32/2
 bif erlang:adler32_combine/3
@@ -62,7 +64,7 @@ bif erlang:exit/1
 bif erlang:exit/2
 bif erlang:external_size/1
 bif erlang:external_size/2
-ubif erlang:float/1
+gcbif erlang:float/1
 bif erlang:float_to_list/1
 bif erlang:float_to_list/2
 bif erlang:fun_info/2
@@ -79,7 +81,7 @@ bif erlang:phash2/2
 ubif erlang:hd/1
 bif erlang:integer_to_list/1
 bif erlang:is_alive/0
-ubif erlang:length/1
+gcbif erlang:length/1
 bif erlang:link/1
 bif erlang:list_to_atom/1
 bif erlang:list_to_binary/1
@@ -126,10 +128,10 @@ bif erlang:processes/0
 bif erlang:put/2
 bif erlang:register/2
 bif erlang:registered/0
-ubif erlang:round/1
+gcbif erlang:round/1
 ubif erlang:self/0
 bif erlang:setelement/3
-ubif erlang:size/1
+gcbif erlang:size/1
 bif erlang:spawn/3
 bif erlang:spawn_link/3
 bif erlang:split_binary/2
@@ -139,7 +141,7 @@ bif erlang:term_to_binary/2
 bif erlang:throw/1
 bif erlang:time/0
 ubif erlang:tl/1
-ubif erlang:trunc/1
+gcbif erlang:trunc/1
 bif erlang:tuple_to_list/1
 bif erlang:universaltime/0
 bif erlang:universaltime_to_localtime/1
@@ -461,8 +463,8 @@ bif erlang:list_to_existing_atom/1
 #
 ubif erlang:is_bitstring/1
 ubif erlang:tuple_size/1
-ubif erlang:byte_size/1
-ubif erlang:bit_size/1
+gcbif erlang:byte_size/1
+gcbif erlang:bit_size/1
 bif erlang:list_to_bitstring/1
 bif erlang:bitstring_to_list/1
 
@@ -514,8 +516,8 @@ bif erlang:binary_to_term/2
 #
 # The searching/splitting/substituting thingies
 #
-ubif erlang:binary_part/2
-ubif erlang:binary_part/3
+gcbif erlang:binary_part/2
+gcbif erlang:binary_part/3
 
 bif binary:compile_pattern/1
 bif binary:match/2
@@ -607,7 +609,7 @@ bif os:unsetenv/1
 bif re:inspect/2
 
 ubif erlang:is_map/1
-ubif erlang:map_size/1
+gcbif erlang:map_size/1
 bif maps:to_list/1
 bif maps:find/2
 bif maps:get/2
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index f3d4ac56cd..961260041f 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1427,18 +1427,6 @@ Eterm erts_gc_bor(Process* p, Eterm* reg, Uint live);
 Eterm erts_gc_bxor(Process* p, Eterm* reg, Uint live);
 Eterm erts_gc_bnot(Process* p, Eterm* reg, Uint live);
 
-Eterm erts_gc_length_1(Process* p, Eterm* reg, Uint live);
-Eterm erts_gc_size_1(Process* p, Eterm* reg, Uint live);
-Eterm erts_gc_bit_size_1(Process* p, Eterm* reg, Uint live);
-Eterm erts_gc_byte_size_1(Process* p, Eterm* reg, Uint live);
-Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live);
-Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live);
-Eterm erts_gc_float_1(Process* p, Eterm* reg, Uint live);
-Eterm erts_gc_round_1(Process* p, Eterm* reg, Uint live);
-Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live);
-Eterm erts_gc_binary_part_3(Process* p, Eterm* reg, Uint live);
-Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live);
-
 Uint erts_current_reductions(Process* current, Process *p);
 
 int erts_print_system_version(int to, void *arg, Process *c_p);
diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables
index 82c8c299f4..fc0500d77f 100755
--- a/erts/emulator/utils/make_tables
+++ b/erts/emulator/utils/make_tables
@@ -53,6 +53,7 @@ my %aliases;
 my $auto_alias_num = 0;
 
 my @bif;
+my @bif_type;
 
 while (@ARGV && $ARGV[0] =~ /^-(\w+)/) {
     my $opt = shift;
@@ -76,7 +77,7 @@ while (<>) {
     my($type, @args) = split;
     if ($type eq 'atom') {
 	save_atoms(@args);
-    } elsif ($type eq 'bif' or $type eq 'ubif') {
+    } elsif ($type eq 'bif' or $type eq 'ubif' or $type eq 'gcbif') {
 	if (@args > 2) {
 	    error("$type only allows two arguments");
 	}
@@ -94,6 +95,7 @@ while (<>) {
 	$wrapper = $alias if $type eq 'ubif';
 	push(@bif, ["am_$atom_alias{$mod}","am_$atom_alias{$name}",$arity,
 		    $alias,$wrapper]);
+	push(@bif_type, $type);
     } else {
 	error("invalid line");
     }
@@ -164,8 +166,14 @@ typedef struct bif_entry {
     BifFunction traced;
 } BifEntry;
 
+typedef struct erts_gc_bif {
+    BifFunction bif;
+    BifFunction gc_bif;
+} ErtsGcBif;
+
 extern BifEntry bif_table[];
 extern Export* bif_export[];
+extern const ErtsGcBif erts_gc_bifs[];
 
 #define BIF_SIZE $bif_size
 
@@ -180,8 +188,12 @@ print "\n";
 
 for ($i = 0; $i < @bif; $i++) {
     my $args = join(', ', 'Process*', 'Eterm*');
-    print "Eterm $bif[$i]->[3]($args);\n";
-    print "Eterm wrap_$bif[$i]->[3]($args, UWord *I);\n";
+    my $name = $bif[$i]->[3];
+    print "Eterm $name($args);\n";
+    print "Eterm wrap_$name($args, UWord *I);\n";
+    print "Eterm erts_gc_$name(Process* p, Eterm* reg, Uint live);\n"
+	if $bif_type[$i] eq 'gcbif';
+    print "\n";
 }
 print "#endif\n";
 
@@ -223,7 +235,25 @@ for ($i = 0; $i < @bif; $i++) {
 }
 
 #
-# Generate the package bif file.
+# Generate erl_gc_bifs.c.
+#
+
+open_file("$src/erl_gc_bifs.c");
+my $i;
+includes("export.h", "sys.h", "erl_vm.h", "global.h", "erl_process.h", "bif.h",
+	 "erl_bif_table.h");
+print "const ErtsGcBif erts_gc_bifs[] = {\n";
+for ($i = 0; $i < @bif; $i++) {
+    next unless $bif_type[$i] eq 'gcbif';
+    my $arity = $bif[$i]->[2];
+    my $func = $bif[$i]->[3];
+    print "  {$func, erts_gc_$func},\n";
+}
+print "  {0, 0}\n";
+print "};\n";
+
+#
+# Utilities follow.
 #
 
 sub open_file {			# or die
-- 
cgit v1.2.3


From eb9766ca5d507d8cb8e1448780f96ba1cb45ff36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 22 Jun 2016 12:57:56 +0200
Subject: beam_debug: Improve the disassembly of gc_bif instructions

Using the translation table in erts_gc_bifs[], we can now print
out the name of GC BIFs, instead of just the pointer value.
---
 erts/emulator/beam/beam_debug.c | 55 ++++++++++++++++++++++++++++++-----------
 1 file changed, 41 insertions(+), 14 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index a4ad3e7886..8f47ed4d9d 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -51,6 +51,7 @@ void dbg_bt(Process* p, Eterm* sp);
 void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg);
 
 static int print_op(int to, void *to_arg, int op, int size, BeamInstr* addr);
+static void print_bif_name(int to, void* to_arg, BifFunction bif);
 
 BIF_RETTYPE
 erts_debug_same_2(BIF_ALIST_2)
@@ -520,7 +521,27 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	    break;
 	case 'I':		/* Untagged integer. */
 	case 't':
-	    erts_print(to, to_arg, "%d", *ap);
+	    switch (op) {
+	    case op_i_gc_bif1_jIsId:
+	    case op_i_gc_bif2_jIIssd:
+	    case op_i_gc_bif3_jIIssd:
+		{
+		    const ErtsGcBif* p;
+		    BifFunction gcf = (BifFunction) *ap;
+		    for (p = erts_gc_bifs; p->bif != 0; p++) {
+			if (p->gc_bif == gcf) {
+			    print_bif_name(to, to_arg, p->bif);
+			    break;
+			}
+		    }
+		    if (p->bif == 0) {
+			erts_print(to, to_arg, "%d", (Uint)gcf);
+		    }
+		    break;
+		}
+	    default:
+		erts_print(to, to_arg, "%d", *ap);
+	    }
 	    ap++;
 	    break;
 	case 'f':		/* Destination label */
@@ -560,19 +581,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	case 'F':		/* Function definition */
 	    break;
 	case 'b':
-	    for (i = 0; i < BIF_SIZE; i++) {
-		BifFunction bif = (BifFunction) *ap;
-		if (bif == bif_table[i].f) {
-		    break;
-		}
-	    }
-	    if (i == BIF_SIZE) {
-		erts_print(to, to_arg, "b(%d)", (Uint) *ap);
-	    } else {
-		Eterm name = bif_table[i].name;
-		unsigned arity = bif_table[i].arity;
-		erts_print(to, to_arg, "%T/%u", name, arity);
-	    }
+	    print_bif_name(to, to_arg, (BifFunction) *ap);
 	    ap++;
 	    break;
 	case 'P':	/* Byte offset into tuple (see beam_load.c) */
@@ -731,3 +740,21 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 
     return size;
 }
+
+static void print_bif_name(int to, void* to_arg, BifFunction bif)
+{
+    int i;
+
+    for (i = 0; i < BIF_SIZE; i++) {
+	if (bif == bif_table[i].f) {
+	    break;
+	}
+    }
+    if (i == BIF_SIZE) {
+	erts_print(to, to_arg, "b(%d)", (Uint) bif);
+    } else {
+	Eterm name = bif_table[i].name;
+	unsigned arity = bif_table[i].arity;
+	erts_print(to, to_arg, "%T/%u", name, arity);
+    }
+}
-- 
cgit v1.2.3


From cdc483461c1c82cbefd333d65fd86ac0d78058c8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 1 Jul 2016 15:54:11 +0200
Subject: erts: Fix gcbif trace wrappers

Broken on master by f0f4e72c8ec5c08993ff.
---
 erts/emulator/utils/make_tables | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables
index fc0500d77f..27f9dcc878 100755
--- a/erts/emulator/utils/make_tables
+++ b/erts/emulator/utils/make_tables
@@ -91,8 +91,11 @@ while (<>) {
 	    $alias .= "${name}_$arity";
 	}
 	my $wrapper;
-	$wrapper = "wrap_$alias" if $type eq 'bif';
-	$wrapper = $alias if $type eq 'ubif';
+	if ($type eq 'bif') {
+	    $wrapper = "wrap_$alias";
+	} else {
+	    $wrapper = $alias;
+	}
 	push(@bif, ["am_$atom_alias{$mod}","am_$atom_alias{$name}",$arity,
 		    $alias,$wrapper]);
 	push(@bif_type, $type);
-- 
cgit v1.2.3


From 69b8d1ab13473f851f8957e2bb6f5486f64612ad Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Sat, 23 Jul 2016 19:10:10 +0100
Subject: zlib: support extraction of inflation dictionary

---
 erts/emulator/drivers/common/zlib_drv.c | 51 +++++++++++++++++++++++----------
 1 file changed, 36 insertions(+), 15 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/zlib_drv.c b/erts/emulator/drivers/common/zlib_drv.c
index 440ba956d8..acbe675c41 100644
--- a/erts/emulator/drivers/common/zlib_drv.c
+++ b/erts/emulator/drivers/common/zlib_drv.c
@@ -44,30 +44,34 @@
 #define INFLATE_INIT    8
 #define INFLATE_INIT2   9
 #define INFLATE_SETDICT 10
-#define INFLATE_SYNC    11
-#define INFLATE_RESET   12
-#define INFLATE_END     13
-#define INFLATE         14
+#define INFLATE_GETDICT 11
+#define INFLATE_SYNC    12
+#define INFLATE_RESET   13
+#define INFLATE_END     14
+#define INFLATE         15
 
-#define CRC32_0         15
-#define CRC32_1         16
-#define CRC32_2         17
+#define CRC32_0         16
+#define CRC32_1         17
+#define CRC32_2         18
 
-#define SET_BUFSZ       18
-#define GET_BUFSZ       19
-#define GET_QSIZE       20
+#define SET_BUFSZ       19
+#define GET_BUFSZ       20
+#define GET_QSIZE       21
 
-#define ADLER32_1       21
-#define ADLER32_2       22
+#define ADLER32_1       22
+#define ADLER32_2       23
 
-#define CRC32_COMBINE   23
-#define ADLER32_COMBINE 24
+#define CRC32_COMBINE   24
+#define ADLER32_COMBINE 25
 
-#define INFLATE_CHUNK   25
+#define INFLATE_CHUNK   26
 
 
 #define DEFAULT_BUFSZ   4000
 
+/* According to zlib documentation, it can never exceed this */
+#define INFL_DICT_SZ    32768
+
 /* This flag is used in the same places, where zlib return codes
  * (Z_OK, Z_STREAM_END, Z_NEED_DICT) are. So, we need to set it to
  * relatively large value to avoid possible value clashes in future.
@@ -248,6 +252,18 @@ static int zlib_output(ZLibData* d)
     return zlib_output_init(d);
 }
 
+static int zlib_inflate_get_dictionary(ZLibData* d)
+{
+    ErlDrvBinary* dbin = driver_alloc_binary(INFL_DICT_SZ);
+    uInt dlen = 0;
+    int res = inflateGetDictionary(&d->s, (unsigned char*)dbin->orig_bytes, &dlen);
+    if ((res == Z_OK) && (driver_output_binary(d->port, NULL, 0, dbin, 0, dlen) < 0)) {
+        res = Z_ERRNO;
+    }
+    driver_free_binary(dbin);
+    return res;
+}
+
 static int zlib_inflate(ZLibData* d, int flush)
 {
     int res = Z_OK;
@@ -586,6 +602,11 @@ static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *bu
 	res = inflateSetDictionary(&d->s, (unsigned char*)buf, len);
 	return zlib_return(res, rbuf, rlen);
 
+    case INFLATE_GETDICT:
+        if (d->state != ST_INFLATE) goto badarg;
+        res = zlib_inflate_get_dictionary(d);
+        return zlib_return(res, rbuf, rlen);
+
     case INFLATE_SYNC:
 	if (d->state != ST_INFLATE) goto badarg;
 	if (len != 0) goto badarg;
-- 
cgit v1.2.3


From a2359927ca4e7c315e9849deec3375b1f2ae4170 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 26 Jul 2016 10:19:31 +0200
Subject: erts: Remove the need for copying of literals

* Literals are not copied between processes for messages or spawn

Increases performance of message sent and processes spawned when
literals are involved in messages or arguments.
---
 erts/emulator/beam/copy.c        | 80 ++++++++++++++++++++++++++++------------
 erts/emulator/beam/erl_gc.c      |  8 ----
 erts/emulator/beam/erl_message.c | 19 ++++++----
 erts/emulator/beam/erl_process.c | 10 +++--
 erts/emulator/beam/global.h      | 20 ++++++----
 5 files changed, 88 insertions(+), 49 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index ccc4cbad43..7d3777f37e 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -40,7 +40,8 @@ static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*, int);
 /*
  *  Copy object "obj" to process p.
  */
-Eterm copy_object_x(Eterm obj, Process* to, Uint extra) {
+Eterm copy_object_x(Eterm obj, Process* to, Uint extra)
+{
     if (!is_immed(obj)) {
         Uint size = size_object(obj);
         Eterm* hp = HAllocX(to, size, extra);
@@ -70,7 +71,12 @@ Eterm copy_object_x(Eterm obj, Process* to, Uint extra) {
  * Return the "flat" size of the object.
  */
 
-Uint size_object(Eterm obj)
+#define in_literal_purge_area(PTR)                   \
+    (lit_purge_ptr && (                              \
+        (lit_purge_ptr <= (PTR) &&                   \
+        (PTR) < (lit_purge_ptr + lit_purge_sz))))
+
+Uint size_object_x(Eterm obj, Eterm *lit_purge_ptr, Uint lit_purge_sz, Uint litopt)
 {
     Uint sum = 0;
     Eterm* ptr;
@@ -78,7 +84,6 @@ Uint size_object(Eterm obj)
 #ifdef DEBUG
     Eterm mypid = erts_get_current_pid();
 #endif
-
     DECLARE_ESTACK(s);
 
     VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size_object %p\n", mypid, obj));
@@ -86,17 +91,25 @@ Uint size_object(Eterm obj)
     for (;;) {
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_LIST:
-	    sum += 2;
 	    ptr = list_val(obj);
+            if (litopt && erts_is_literal(obj,ptr) && !in_literal_purge_area(ptr)) {
+                goto pop_next;
+            }
+	    sum += 2;
 	    obj = *ptr++;
 	    if (!IS_CONST(obj)) {
 		ESTACK_PUSH(s, obj);
-	    }	    
+	    }
 	    obj = *ptr;
 	    break;
 	case TAG_PRIMARY_BOXED:
 	    {
-		Eterm hdr = *boxed_val(obj);
+		Eterm hdr;
+                ptr = boxed_val(obj);
+                if (litopt && erts_is_literal(obj,ptr) && !in_literal_purge_area(ptr)) {
+                    goto pop_next;
+                }
+                hdr = *ptr;
 		ASSERT(is_header(hdr));
 		switch (hdr & _TAG_HEADER_MASK) {
 		case ARITYVAL_SUBTAG:
@@ -279,10 +292,6 @@ do {                                                                    \
 
 #define COUNT_OFF_HEAP (0)
 
-#define IN_LITERAL_PURGE_AREA(info, ptr)                 \
-    ((info)->range_ptr && (                              \
-        (info)->range_ptr <= (ptr) &&                    \
-        (ptr) < ((info)->range_ptr + (info)->range_sz)))
 /*
  *  Return the real size of an object and find sharing information
  *  This currently returns the same as erts_debug:size/1.
@@ -599,7 +608,8 @@ cleanup:
 /*
  *  Copy a structure to a heap.
  */
-Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz)
+Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz,
+                    Eterm *lit_purge_ptr, Uint lit_purge_sz, Uint litopt)
 {
     char* hstart;
     Uint hsize;
@@ -651,7 +661,6 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
  L_copy:
     while (hp != htop) {
 	obj = *hp;
-
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_IMMED1:
 	    hp++;
@@ -667,6 +676,10 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 
 	L_copy_list:
 	    tailp = argp;
+            if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) {
+                *tailp = obj;
+                goto L_copy;
+            }
 	    for (;;) {
 		tp = tailp;
 		elem = CAR(objp);
@@ -674,18 +687,23 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 		    hbot -= 2;
 		    CAR(hbot) = elem;
 		    tailp = &CDR(hbot);
-		}
-		else {
+		} else {
 		    CAR(htop) = elem;
 		    tailp = &CDR(htop);
 		    htop += 2;
 		}
 		*tp = make_list(tailp - 1);
 		obj = CDR(objp);
+
 		if (!is_list(obj)) {
 		    break;
 		}
 		objp = list_val(obj);
+
+                if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) {
+                    *tailp = obj;
+                    goto L_copy;
+                }
 	    }
 	    switch (primary_tag(obj)) {
 	    case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy;
@@ -695,7 +713,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 			 "%s, line %d: Internal error in copy_struct: 0x%08x\n",
 			 __FILE__, __LINE__,obj);
 	    }
-	    
+
 	case TAG_PRIMARY_BOXED:
 	    if (ErtsInArea(boxed_val(obj),hstart,hsize)) {
 		hp++;
@@ -705,6 +723,10 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 
 	L_copy_boxed:
 	    objp = boxed_val(obj);
+            if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) {
+                *argp = obj;
+                break;
+            }
 	    hdr = *objp;
 	    switch (hdr & _TAG_HEADER_MASK) {
 	    case ARITYVAL_SUBTAG:
@@ -765,7 +787,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 			extra_bytes = 1;
 		    } else {
 			extra_bytes = 0;
-		    } 
+		    }
 		    real_size = size+extra_bytes;
 		    objp = binary_val(real_bin);
 		    if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) {
@@ -780,7 +802,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 		    } else {
 			ProcBin* from = (ProcBin *) objp;
 			ProcBin* to;
-			
+
 			ASSERT(thing_subtag(*objp) == REFC_BINARY_SUBTAG);
 			if (from->flags) {
 			    erts_emasculate_writable_binary(from);
@@ -900,6 +922,12 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
         *bsz = hend - hbot;
     } else {
 #ifdef DEBUG
+        if (!eq(org_obj, res)) {
+            erts_exit(ERTS_ABORT_EXIT,
+                    "Internal error in copy_struct() when copying %T:"
+                    " not equal to copy %T\n",
+                    org_obj, res);
+        }
         if (htop != hbot)
             erts_exit(ERTS_ABORT_EXIT,
                     "Internal error in copy_struct() when copying %T:"
@@ -1036,6 +1064,8 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info)
     Uint e;
     unsigned sz;
     Eterm* ptr;
+    Eterm *lit_purge_ptr = info->lit_purge_ptr;
+    Uint lit_purge_sz = info->lit_purge_sz;
 #ifdef DEBUG
     Eterm mypid = erts_get_current_pid();
 #endif
@@ -1081,7 +1111,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info)
 	    /* off heap list pointers are copied verbatim */
 	    if (erts_is_literal(obj,ptr)) {
 		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj));
-                if (IN_LITERAL_PURGE_AREA(info,ptr))
+                if (in_literal_purge_area(ptr))
                     info->literal_size += size_object(obj);
 		goto pop_next;
 	    }
@@ -1132,7 +1162,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info)
 	    /* off heap pointers to boxes are copied verbatim */
 	    if (erts_is_literal(obj,ptr)) {
 		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj));
-                if (IN_LITERAL_PURGE_AREA(info,ptr))
+                if (in_literal_purge_area(ptr))
                     info->literal_size += size_object(obj);
 		goto pop_next;
 	    }
@@ -1298,6 +1328,8 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
     Eterm* resp;
     Eterm *hbot, *hend;
     unsigned remaining;
+    Eterm *lit_purge_ptr = info->lit_purge_ptr;
+    Uint lit_purge_sz = info->lit_purge_sz;
 #ifdef DEBUG
     Eterm mypid = erts_get_current_pid();
     Eterm saved_obj = obj;
@@ -1347,11 +1379,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 	    ptr = list_val(obj);
 	    /* off heap list pointers are copied verbatim */
 	    if (erts_is_literal(obj,ptr)) {
-                if (!IN_LITERAL_PURGE_AREA(info,ptr)) {
+                if (!in_literal_purge_area(ptr)) {
                     *resp = obj;
                 } else {
                     Uint bsz = 0;
-                    *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz);
+                    *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL, 0, 0); /* copy literal */
                     hbot -= bsz;
                 }
 		goto cleanup_next;
@@ -1415,11 +1447,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 	    ptr = boxed_val(obj);
 	    /* off heap pointers to boxes are copied verbatim */
 	    if (erts_is_literal(obj,ptr)) {
-                if (!IN_LITERAL_PURGE_AREA(info,ptr)) {
+                if (!in_literal_purge_area(ptr)) {
                     *resp = obj;
                 } else {
                     Uint bsz = 0;
-                    *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz);
+                    *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL, 0, 0); /* copy literal */
                     hbot -= bsz;
                 }
 		goto cleanup_next;
@@ -1923,7 +1955,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite
 	if (is_header(val)) {
 	    struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp;
 	    ASSERT(ptr + header_arity(val) < end);
-	    MOVE_BOXED(ptr, val, hp, &dummy_ref);	    
+	    MOVE_BOXED(ptr, val, hp, &dummy_ref);
 	    switch (val & _HEADER_SUBTAG_MASK) {
 	    case REFC_BINARY_SUBTAG:
 	    case FUN_SUBTAG:
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index a224383493..22a7d430c8 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2155,26 +2155,18 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap,
 	    *hp++ = val;
 	    break;
 	case TAG_PRIMARY_LIST:
-#ifdef SHCOPY_SEND
             if (erts_is_literal(val,list_val(val))) {
                 *hp++ = val;
             } else {
                 *hp++ = offset_ptr(val, offs);
             }
-#else
-            *hp++ = offset_ptr(val, offs);
-#endif
             break;
 	case TAG_PRIMARY_BOXED:
-#ifdef SHCOPY_SEND
             if (erts_is_literal(val,boxed_val(val))) {
                 *hp++ = val;
             } else {
                 *hp++ = offset_ptr(val, offs);
             }
-#else
-            *hp++ = offset_ptr(val, offs);
-#endif
 	    break;
 	case TAG_PRIMARY_HEADER:
 	    *hp++ = val;
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 91e06cde2d..14ee07a304 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -696,6 +696,9 @@ erts_send_message(Process* sender,
     erts_aint32_t receiver_state;
 #ifdef SHCOPY_SEND
     erts_shcopy_t info;
+#else
+    Eterm *lit_purge_ptr = erts_clrange.ptr;
+    Uint lit_purge_sz = erts_clrange.sz;
 #endif
 
 #ifdef USE_VM_PROBES
@@ -725,7 +728,7 @@ erts_send_message(Process* sender,
          */
         if (have_seqtrace(stoken)) {
 	    seq_trace_update_send(sender);
-	    seq_trace_output(stoken, message, SEQ_TRACE_SEND, 
+	    seq_trace_output(stoken, message, SEQ_TRACE_SEND,
 			     receiver->common.id, sender);
 	    seq_trace_size = 6; /* TUPLE5 */
 	}
@@ -741,7 +744,7 @@ erts_send_message(Process* sender,
         INITIALIZE_SHCOPY(info);
         msize = copy_shared_calculate(message, &info);
 #else
-        msize = size_object(message);
+        msize = size_object_litopt(message, lit_purge_ptr, lit_purge_sz);
 #endif
         mp = erts_alloc_message_heap_state(receiver,
                                            &receiver_state,
@@ -760,7 +763,8 @@ erts_send_message(Process* sender,
         DESTROY_SHCOPY(info);
 #else
 	if (is_not_immed(message))
-            message = copy_struct(message, msize, &hp, ohp);
+            message = copy_struct_litopt(message, msize, &hp, ohp,
+                                         lit_purge_ptr, lit_purge_sz);
 #endif
 	if (is_immed(stoken))
 	    token = stoken;
@@ -796,7 +800,7 @@ erts_send_message(Process* sender,
             INITIALIZE_SHCOPY(info);
             msize = copy_shared_calculate(message, &info);
 #else
-            msize = size_object(message);
+            msize = size_object_litopt(message, lit_purge_ptr, lit_purge_sz);
 #endif
 	    mp = erts_alloc_message_heap_state(receiver,
 					       &receiver_state,
@@ -810,7 +814,8 @@ erts_send_message(Process* sender,
             DESTROY_SHCOPY(info);
 #else
             if (is_not_immed(message))
-                message = copy_struct(message, msize, &hp, ohp);
+                message = copy_struct_litopt(message, msize, &hp, ohp,
+                                             lit_purge_ptr, lit_purge_sz);
 #endif
 	}
 #ifdef USE_VM_PROBES
@@ -998,8 +1003,8 @@ erts_move_messages_off_heap(Process *c_p)
 	hp = hfrag->mem;
 	if (is_not_immed(ERL_MESSAGE_TERM(mp)))
 	    ERL_MESSAGE_TERM(mp) = copy_struct(ERL_MESSAGE_TERM(mp),
-					       msg_sz, &hp,
-					       &hfrag->off_heap);
+                                               msg_sz, &hp,
+                                               &hfrag->off_heap);
 	if (is_not_immed(ERL_MESSAGE_TOKEN(mp)))
 	    ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp),
 						token_sz, &hp,
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b5d8c5bc75..d6037e9f5d 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11099,6 +11099,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 #ifdef SHCOPY_SPAWN
     erts_shcopy_t info;
     INITIALIZE_SHCOPY(info);
+#else
+    Eterm *lit_purge_ptr = erts_clrange.ptr;
+    Uint lit_purge_sz = erts_clrange.sz;
 #endif
 
     erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR);
@@ -11157,7 +11160,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 #ifdef SHCOPY_SPAWN
     arg_size = copy_shared_calculate(args, &info);
 #else
-    arg_size = size_object(args);
+    arg_size = size_object_litopt(args, lit_purge_ptr, lit_purge_sz);
 #endif
     heap_need = arg_size;
 
@@ -11181,7 +11184,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     }
     p->schedule_count = 0;
     ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0));
-    
+
     p->u.initial[INITIAL_MOD] = mod;
     p->u.initial[INITIAL_FUN] = func;
     p->u.initial[INITIAL_ARI] = (Uint) arity;
@@ -11239,7 +11242,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap);
     DESTROY_SHCOPY(info);
 #else
-    p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap);
+    p->arg_reg[2] = copy_struct_litopt(args, arg_size, &p->htop, &p->off_heap,
+                                       lit_purge_ptr, lit_purge_sz);
 #endif
     p->arity = 3;
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index bf2e50e52f..67ce0d0723 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1085,8 +1085,8 @@ typedef struct {
     Eterm* shtable_start;
     ErtsAlcType_t shtable_alloc_type;
     Uint literal_size;
-    Eterm *range_ptr;
-    Uint  range_sz;
+    Eterm *lit_purge_ptr;
+    Uint lit_purge_sz;
 } erts_shcopy_t;
 
 #define INITIALIZE_SHCOPY(info)                         \
@@ -1095,8 +1095,8 @@ do {                                                    \
     info.bitstore_start = info.bitstore_default;        \
     info.shtable_start = info.shtable_default;          \
     info.literal_size = 0;                              \
-    info.range_ptr = erts_clrange.ptr;                  \
-    info.range_sz  = erts_clrange.sz;                   \
+    info.lit_purge_ptr = erts_clrange.ptr;              \
+    info.lit_purge_sz  = erts_clrange.sz;               \
 } while(0)
 
 #define DESTROY_SHCOPY(info)                                            \
@@ -1116,15 +1116,21 @@ do {                                                                    \
 Eterm copy_object_x(Eterm, Process*, Uint);
 #define copy_object(Term, Proc) copy_object_x(Term,Proc,0)
 
-Uint size_object(Eterm);
+Uint size_object_x(Eterm,Eterm*,Uint,Uint);
+#define size_object(Term) size_object_x(Term,NULL,0,0)
+#define size_object_litopt(Term,LitPtr,LitSz) size_object_x(Term,LitPtr,LitSz,1)
+
 Uint copy_shared_calculate(Eterm, erts_shcopy_t*);
 Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*);
 
 Uint size_shared(Eterm);
 
-Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz);
+Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz, Eterm *lit_ptr, Uint lit_sz, Uint litopt);
 #define copy_struct(Obj,Sz,HPP,OH) \
-    copy_struct_x(Obj,Sz,HPP,OH,NULL)
+    copy_struct_x(Obj,Sz,HPP,OH,NULL,NULL,0,0)
+#define copy_struct_litopt(Obj,Sz,HPP,OH,LitPtr,LitSz) \
+    copy_struct_x(Obj,Sz,HPP,OH,NULL,LitPtr,LitSz,1)
+
 Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*);
 
 void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first,
-- 
cgit v1.2.3


From a9368b3c1ee78a3c83fcff83590a3d94ea4ddbe0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 27 Jul 2016 17:00:32 +0200
Subject: erts: Fix literal size calculation in check_process_code

We want to know the total size of literals of all heap fragmens.
---
 erts/emulator/beam/beam_bif_load.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 15e878ba65..2a165bed00 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -873,12 +873,12 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls
 	    continue;
         {
             ErlHeapFragment *hf;
-            Uint lit_sz;
+            Uint lit_sz = 0;
             for (hf=hfrag; hf; hf = hf->next) {
                 if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size))
                     return am_true;
-                lit_sz = hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size],
-                                            literals, lit_bsize);
+                lit_sz += hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size],
+                                             literals, lit_bsize);
             }
             if (lit_sz > 0) {
                 ErlHeapFragment *bp = new_message_buffer(lit_sz);
-- 
cgit v1.2.3


From 13ec43880b8565a5ddd7da590c36a520d4ff4b26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 29 Jul 2016 12:19:11 +0200
Subject: erts: Don't copy literals in enif_send

---
 erts/emulator/beam/erl_nif.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index c6127a4967..e1944fff29 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -627,7 +627,9 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
             MBUF(&menv->phony_proc) = NULL;
         }
     } else {
-        Uint sz = size_object(msg);
+        Eterm *lit_purge_ptr = erts_clrange.ptr;
+        Uint lit_purge_sz = erts_clrange.sz;
+        Uint sz = size_object_litopt(msg, lit_purge_ptr, lit_purge_sz);
 	ErlOffHeap *ohp;
         Eterm *hp;
 	if (env && !env->tracee) {
@@ -649,7 +651,8 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 		ohp = &bp->off_heap;
 	    }
 	}
-        msg = copy_struct(msg, sz, &hp, ohp);
+        msg = copy_struct_litopt(msg, sz, &hp, ohp,
+                                 lit_purge_ptr, lit_purge_sz);
     }
 
     ERL_MESSAGE_TERM(mp) = msg;
-- 
cgit v1.2.3


From dbea5c4023ec6ac59bb1988711532682a032536d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 29 Jul 2016 15:59:16 +0200
Subject: erts: No need to literal opt message token

---
 erts/emulator/beam/erl_gc.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 22a7d430c8..cddcc847a3 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2252,10 +2252,12 @@ move_msgq_to_heap(Process *p)
 		    ASSERT(mp->data.dist_ext->heap_size >= 0);
 		    if (is_not_nil(ERL_MESSAGE_TOKEN(mp))) {
 			bp = erts_dist_ext_trailer(mp->data.dist_ext);
+                        /* Tokens does not use literal optimization */
 			ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp),
 							    bp->used_size,
-							    &factory.hp,
-							    factory.off_heap);
+                                                            &factory.hp,
+                                                            factory.off_heap);
+
 			erts_cleanup_offheap(&bp->off_heap);
 		    }
 		    ERL_MESSAGE_TERM(mp) = erts_decode_dist_ext(&factory,
-- 
cgit v1.2.3


From e52fa24451306080be39a35444cf52210982b41a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 29 Jul 2016 17:16:53 +0200
Subject: erts: Refactor literal purge area arguments

---
 erts/emulator/beam/copy.c        | 14 ++++++++------
 erts/emulator/beam/erl_message.c | 14 ++++++--------
 erts/emulator/beam/erl_nif.c     | 10 +++++-----
 erts/emulator/beam/erl_process.c |  9 ++++-----
 erts/emulator/beam/global.h      | 25 ++++++++++++++++++-------
 5 files changed, 41 insertions(+), 31 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 7d3777f37e..c4dcd6a3cc 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -76,16 +76,17 @@ Eterm copy_object_x(Eterm obj, Process* to, Uint extra)
         (lit_purge_ptr <= (PTR) &&                   \
         (PTR) < (lit_purge_ptr + lit_purge_sz))))
 
-Uint size_object_x(Eterm obj, Eterm *lit_purge_ptr, Uint lit_purge_sz, Uint litopt)
+Uint size_object_x(Eterm obj, erts_literal_area_t *litopt)
 {
     Uint sum = 0;
     Eterm* ptr;
     int arity;
+    Eterm *lit_purge_ptr = litopt ? litopt->lit_purge_ptr : NULL;
+    Uint   lit_purge_sz  = litopt ? litopt->lit_purge_sz  : 0;
 #ifdef DEBUG
     Eterm mypid = erts_get_current_pid();
 #endif
     DECLARE_ESTACK(s);
-
     VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size_object %p\n", mypid, obj));
 
     for (;;) {
@@ -608,8 +609,7 @@ cleanup:
 /*
  *  Copy a structure to a heap.
  */
-Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz,
-                    Eterm *lit_purge_ptr, Uint lit_purge_sz, Uint litopt)
+Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz, erts_literal_area_t *litopt)
 {
     char* hstart;
     Uint hsize;
@@ -626,6 +626,8 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
     Eterm hdr;
     Eterm *hend;
     int i;
+    Eterm *lit_purge_ptr = litopt ? litopt->lit_purge_ptr : NULL;
+    Uint   lit_purge_sz  = litopt ? litopt->lit_purge_sz  : 0;
 #ifdef DEBUG
     Eterm org_obj = obj;
     Uint org_sz = sz;
@@ -1383,7 +1385,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
                     *resp = obj;
                 } else {
                     Uint bsz = 0;
-                    *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL, 0, 0); /* copy literal */
+                    *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL); /* copy literal */
                     hbot -= bsz;
                 }
 		goto cleanup_next;
@@ -1451,7 +1453,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
                     *resp = obj;
                 } else {
                     Uint bsz = 0;
-                    *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL, 0, 0); /* copy literal */
+                    *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL); /* copy literal */
                     hbot -= bsz;
                 }
 		goto cleanup_next;
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 14ee07a304..e9f0586edd 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -697,8 +697,8 @@ erts_send_message(Process* sender,
 #ifdef SHCOPY_SEND
     erts_shcopy_t info;
 #else
-    Eterm *lit_purge_ptr = erts_clrange.ptr;
-    Uint lit_purge_sz = erts_clrange.sz;
+    erts_literal_area_t litarea;
+    INITIALIZE_LITERAL_PURGE_AREA(litarea);
 #endif
 
 #ifdef USE_VM_PROBES
@@ -744,7 +744,7 @@ erts_send_message(Process* sender,
         INITIALIZE_SHCOPY(info);
         msize = copy_shared_calculate(message, &info);
 #else
-        msize = size_object_litopt(message, lit_purge_ptr, lit_purge_sz);
+        msize = size_object_litopt(message, &litarea);
 #endif
         mp = erts_alloc_message_heap_state(receiver,
                                            &receiver_state,
@@ -763,8 +763,7 @@ erts_send_message(Process* sender,
         DESTROY_SHCOPY(info);
 #else
 	if (is_not_immed(message))
-            message = copy_struct_litopt(message, msize, &hp, ohp,
-                                         lit_purge_ptr, lit_purge_sz);
+            message = copy_struct_litopt(message, msize, &hp, ohp, &litarea);
 #endif
 	if (is_immed(stoken))
 	    token = stoken;
@@ -800,7 +799,7 @@ erts_send_message(Process* sender,
             INITIALIZE_SHCOPY(info);
             msize = copy_shared_calculate(message, &info);
 #else
-            msize = size_object_litopt(message, lit_purge_ptr, lit_purge_sz);
+            msize = size_object_litopt(message, &litarea);
 #endif
 	    mp = erts_alloc_message_heap_state(receiver,
 					       &receiver_state,
@@ -814,8 +813,7 @@ erts_send_message(Process* sender,
             DESTROY_SHCOPY(info);
 #else
             if (is_not_immed(message))
-                message = copy_struct_litopt(message, msize, &hp, ohp,
-                                             lit_purge_ptr, lit_purge_sz);
+                message = copy_struct_litopt(message, msize, &hp, ohp, &litarea);
 #endif
 	}
 #ifdef USE_VM_PROBES
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index e1944fff29..6bd27d11f4 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -627,11 +627,12 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
             MBUF(&menv->phony_proc) = NULL;
         }
     } else {
-        Eterm *lit_purge_ptr = erts_clrange.ptr;
-        Uint lit_purge_sz = erts_clrange.sz;
-        Uint sz = size_object_litopt(msg, lit_purge_ptr, lit_purge_sz);
+        erts_literal_area_t litarea;
 	ErlOffHeap *ohp;
         Eterm *hp;
+        Uint sz;
+        INITIALIZE_LITERAL_PURGE_AREA(litarea);
+        sz = size_object_litopt(msg, &litarea);
 	if (env && !env->tracee) {
 	    flush_env(env);
 	    mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
@@ -651,8 +652,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 		ohp = &bp->off_heap;
 	    }
 	}
-        msg = copy_struct_litopt(msg, sz, &hp, ohp,
-                                 lit_purge_ptr, lit_purge_sz);
+        msg = copy_struct_litopt(msg, sz, &hp, ohp, &litarea);
     }
 
     ERL_MESSAGE_TERM(mp) = msg;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index d6037e9f5d..85c37e08be 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11100,8 +11100,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     erts_shcopy_t info;
     INITIALIZE_SHCOPY(info);
 #else
-    Eterm *lit_purge_ptr = erts_clrange.ptr;
-    Uint lit_purge_sz = erts_clrange.sz;
+    erts_literal_area_t litarea;
+    INITIALIZE_LITERAL_PURGE_AREA(litarea);
 #endif
 
     erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR);
@@ -11160,7 +11160,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 #ifdef SHCOPY_SPAWN
     arg_size = copy_shared_calculate(args, &info);
 #else
-    arg_size = size_object_litopt(args, lit_purge_ptr, lit_purge_sz);
+    arg_size = size_object_litopt(args, &litarea);
 #endif
     heap_need = arg_size;
 
@@ -11242,8 +11242,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap);
     DESTROY_SHCOPY(info);
 #else
-    p->arg_reg[2] = copy_struct_litopt(args, arg_size, &p->htop, &p->off_heap,
-                                       lit_purge_ptr, lit_purge_sz);
+    p->arg_reg[2] = copy_struct_litopt(args, arg_size, &p->htop, &p->off_heap, &litarea);
 #endif
     p->arity = 3;
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 67ce0d0723..1423973739 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1113,23 +1113,34 @@ do {                                                                    \
 } while(0)
 
 /* copy.c */
+typedef struct {
+    Eterm *lit_purge_ptr;
+    Uint lit_purge_sz;
+} erts_literal_area_t;
+
+#define INITIALIZE_LITERAL_PURGE_AREA(Area)      \
+    do {                                         \
+        (Area).lit_purge_ptr = erts_clrange.ptr; \
+        (Area).lit_purge_sz  = erts_clrange.sz;  \
+    } while(0)
+
 Eterm copy_object_x(Eterm, Process*, Uint);
 #define copy_object(Term, Proc) copy_object_x(Term,Proc,0)
 
-Uint size_object_x(Eterm,Eterm*,Uint,Uint);
-#define size_object(Term) size_object_x(Term,NULL,0,0)
-#define size_object_litopt(Term,LitPtr,LitSz) size_object_x(Term,LitPtr,LitSz,1)
+Uint size_object_x(Eterm, erts_literal_area_t*);
+#define size_object(Term) size_object_x(Term,NULL)
+#define size_object_litopt(Term,LitArea) size_object_x(Term,LitArea)
 
 Uint copy_shared_calculate(Eterm, erts_shcopy_t*);
 Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*);
 
 Uint size_shared(Eterm);
 
-Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz, Eterm *lit_ptr, Uint lit_sz, Uint litopt);
+Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint*, erts_literal_area_t*);
 #define copy_struct(Obj,Sz,HPP,OH) \
-    copy_struct_x(Obj,Sz,HPP,OH,NULL,NULL,0,0)
-#define copy_struct_litopt(Obj,Sz,HPP,OH,LitPtr,LitSz) \
-    copy_struct_x(Obj,Sz,HPP,OH,NULL,LitPtr,LitSz,1)
+    copy_struct_x(Obj,Sz,HPP,OH,NULL,NULL)
+#define copy_struct_litopt(Obj,Sz,HPP,OH,LitArea) \
+    copy_struct_x(Obj,Sz,HPP,OH,NULL,LitArea)
 
 Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*);
 
-- 
cgit v1.2.3


From 58d8a3ac4d491c8cd962ac0839a56cd1a0e339f9 Mon Sep 17 00:00:00 2001
From: Dmytro Lytovchenko <dmytro.lytovchenko@gmail.com>
Date: Mon, 13 Jun 2016 12:05:55 +0200
Subject: Option to erlang:garbage_collect to request minor (generational) GC

Note: Minor GC option is a hint, and GC may still decide to run fullsweep.
Test case for major and minor gc on self
Test case for major and minor gs on some other process + async gc test check

docs fix
---
 erts/emulator/beam/atom.names    |   2 +
 erts/emulator/beam/bif.c         |  19 +++++++-
 erts/emulator/beam/bif.tab       |   1 +
 erts/emulator/beam/erl_process.c |  23 ++++++---
 erts/emulator/beam/ops.tab       |   4 ++
 erts/emulator/test/gc_SUITE.erl  | 103 ++++++++++++++++++++++++++++++++++++++-
 6 files changed, 142 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index badd69856e..f88aeba49f 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -361,6 +361,7 @@ atom long_schedule
 atom low
 atom Lt='<'
 atom machine
+atom major
 atom match
 atom match_limit
 atom match_limit_recursion
@@ -388,6 +389,7 @@ atom microstate_accounting
 atom milli_seconds
 atom min_heap_size
 atom min_bin_vheap_size
+atom minor
 atom minor_version
 atom Minus='-'
 atom module
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index d9048065c8..84eab5f651 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3869,7 +3869,24 @@ BIF_RETTYPE garbage_collect_0(BIF_ALIST_0)
 {
     FLAGS(BIF_P) |= F_NEED_FULLSWEEP;
     erts_garbage_collect(BIF_P, 0, NULL, 0);
-    BIF_RET(am_true);
+    return am_true;
+}
+
+/*
+ * Pass atom 'minor' for relaxed generational GC run. This is only
+ * recommendation, major run may still be chosen by VM.
+ * Pass atom 'major' for default behaviour - major GC run (fullsweep)
+ */
+BIF_RETTYPE
+erts_internal_garbage_collect_1(BIF_ALIST_1)
+{
+    switch (BIF_ARG_1) {
+    case am_minor:  break;
+    case am_major:  FLAGS(BIF_P) |= F_NEED_FULLSWEEP; break;
+    default:        BIF_ERROR(BIF_P, BADARG);
+    }
+    erts_garbage_collect(BIF_P, 0, NULL, 0);
+    return am_true;
 }
 
 /**********************************************************************/
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 978bcd4bba..5bb7db5d47 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -69,6 +69,7 @@ bif erlang:float_to_list/1
 bif erlang:float_to_list/2
 bif erlang:fun_info/2
 bif erlang:garbage_collect/0
+bif erts_internal:garbage_collect/1
 bif erlang:get/0
 bif erlang:get/1
 bif erlang:get_keys/1
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 85c37e08be..a0c5e247d4 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -446,7 +446,8 @@ int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP;
 #endif
 
 typedef enum {
-    ERTS_PSTT_GC,	/* Garbage Collect */
+    ERTS_PSTT_GC_MAJOR,	/* Garbage Collect: Fullsweep */
+    ERTS_PSTT_GC_MINOR,	/* Garbage Collect: Generational */
     ERTS_PSTT_CPC,	/* Check Process Code */
     ERTS_PSTT_COHMQ,    /* Change off heap message queue */
     ERTS_PSTT_FTMQ      /* Flush trace msg queue */
@@ -10358,7 +10359,8 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	    break;
 
 	switch (st->type) {
-	case ERTS_PSTT_GC:
+        case ERTS_PSTT_GC_MAJOR:
+        case ERTS_PSTT_GC_MINOR:
 	    if (c_p->flags & F_DISABLE_GC) {
 		save_gc_task(c_p, st, st_prio);
 		st = NULL;
@@ -10366,7 +10368,9 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	    }
 	    else {
 		if (!garbage_collected) {
-		    FLAGS(c_p) |= F_NEED_FULLSWEEP;
+                    if (st->type == ERTS_PSTT_GC_MAJOR) {
+                        FLAGS(c_p) |= F_NEED_FULLSWEEP;
+                    }
 		    reds -= scheduler_gc_proc(c_p, reds);
 		    garbage_collected = 1;
 		}
@@ -10442,7 +10446,8 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
 	    break;
 
 	switch (st->type) {
-	case ERTS_PSTT_GC:
+        case ERTS_PSTT_GC_MAJOR:
+        case ERTS_PSTT_GC_MINOR:
 	    st_res = am_false;
 	    break;
 	case ERTS_PSTT_CPC:
@@ -10555,9 +10560,13 @@ erts_internal_request_system_task_3(BIF_ALIST_3)
     switch (req_type) {
 
     case am_garbage_collect:
-	st->type = ERTS_PSTT_GC;
-	noproc_res = am_false;
-	if (!rp)
+        switch (st->arg[0]) {
+        case am_minor:  st->type = ERTS_PSTT_GC_MINOR; break;
+        case am_major:  st->type = ERTS_PSTT_GC_MAJOR; break;
+        default: goto badarg;
+        }
+        noproc_res = am_false;
+        if (!rp)
 	    goto noproc;
 	break;
 
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 879daaca0a..801d43e957 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -811,6 +811,10 @@ call_ext u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext Bif
 call_ext_last u==0 Bif=u$bif:erlang:garbage_collect/0 D => i_call_ext_last Bif D
 call_ext_only u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext_only Bif
 
+call_ext u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext Bif
+call_ext_last u==1 Bif=u$bif:erts_internal:garbage_collect/1 D => i_call_ext_last Bif D
+call_ext_only u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext_only Bif
+
 #
 # put/2 and erase/1 must be able to do garbage collection, so we must call
 # them like functions.
diff --git a/erts/emulator/test/gc_SUITE.erl b/erts/emulator/test/gc_SUITE.erl
index 8a600b7d9f..35dd147550 100644
--- a/erts/emulator/test/gc_SUITE.erl
+++ b/erts/emulator/test/gc_SUITE.erl
@@ -23,15 +23,26 @@
 -module(gc_SUITE).
 
 -include_lib("common_test/include/ct.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
 -export([all/0, suite/0]).
 
--export([grow_heap/1, grow_stack/1, grow_stack_heap/1, max_heap_size/1]).
+-export([
+    grow_heap/1,
+    grow_stack/1,
+    grow_stack_heap/1,
+    max_heap_size/1,
+    minor_major_gc_option_async/1,
+    minor_major_gc_option_self/1
+]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]}].
 
 all() -> 
-    [grow_heap, grow_stack, grow_stack_heap, max_heap_size].
+    [grow_heap, grow_stack, grow_stack_heap, max_heap_size,
+    minor_major_gc_option_self,
+    minor_major_gc_option_async].
 
 
 %% Produce a growing list of elements,
@@ -190,3 +201,91 @@ long_receive() ->
     after 10000 ->
             ok
     end.
+
+minor_major_gc_option_self(_Config) ->
+    Endless = fun Endless() ->
+                receive
+                    {gc, Type} -> erlang:garbage_collect(self(), [{type, Type}])
+                    after 100 -> ok end,
+                Endless()
+              end,
+
+    %% Try as major, a test process will self-trigger GC
+    P1 = spawn(Endless),
+    erlang:garbage_collect(P1, []),
+    erlang:trace(P1, true, [garbage_collection]),
+    P1 ! {gc, major},
+    expect_trace_messages(P1, [gc_major_start, gc_major_end]),
+    erlang:trace(P1, false, [garbage_collection]),
+    erlang:exit(P1, kill),
+
+    %% Try as minor, a test process will self-trigger GC
+    P2 = spawn(Endless),
+    erlang:garbage_collect(P2, []),
+    erlang:trace(P2, true, [garbage_collection]),
+    P2 ! {gc, minor},
+    expect_trace_messages(P2, [gc_minor_start, gc_minor_end]),
+    erlang:trace(P2, false, [garbage_collection]),
+    erlang:exit(P2, kill).
+
+minor_major_gc_option_async(_Config) ->
+    Endless = fun Endless() ->
+                  receive after 100 -> ok end,
+                  Endless()
+              end,
+
+    %% Try with default option, must be major gc
+    P1 = spawn(Endless),
+    erlang:garbage_collect(P1, []),
+    erlang:trace(P1, true, [garbage_collection]),
+    erlang:garbage_collect(P1, []),
+    expect_trace_messages(P1, [gc_major_start, gc_major_end]),
+    erlang:trace(P1, false, [garbage_collection]),
+    erlang:exit(P1, kill),
+
+    %% Try with the 'major' type
+    P2 = spawn(Endless),
+    erlang:garbage_collect(P2, []),
+    erlang:trace(P2, true, [garbage_collection]),
+    erlang:garbage_collect(P2, [{type, major}]),
+    expect_trace_messages(P2, [gc_major_start, gc_major_end]),
+    erlang:trace(P2, false, [garbage_collection]),
+    erlang:exit(P2, kill),
+
+    %% Try with 'minor' option, once
+    P3 = spawn(Endless),
+    erlang:garbage_collect(P3, []),
+    erlang:trace(P3, true, [garbage_collection]),
+    erlang:garbage_collect(P3, [{type, minor}]),
+    expect_trace_messages(P3, [gc_minor_start, gc_minor_end]),
+    erlang:trace(P3, false, [garbage_collection]),
+    erlang:exit(P3, kill),
+
+    %% Try with 'minor' option, once, async
+    P4 = spawn(Endless),
+    Ref = erlang:make_ref(),
+    erlang:garbage_collect(P4, []),
+    erlang:trace(P4, true, [garbage_collection]),
+    ?assertEqual(async,
+                 erlang:garbage_collect(P4, [{type, minor}, {async, Ref}])),
+    expect_trace_messages(P4, [gc_minor_start, gc_minor_end]),
+    erlang:trace(P4, false, [garbage_collection]),
+    receive {garbage_collect, Ref, true} -> ok;
+        Other4 -> ct:pal("Unexpected message: ~p~n"
+                         ++ "while waiting for async gc result", [Other4])
+    after 2000 -> ?assert(false)
+    end,
+    erlang:exit(P4, kill).
+
+%% Given a list of atoms, trace tags - receives messages and checks if they are
+%% trace events, and if the tag matches. Else will crash failing the test.
+expect_trace_messages(_Pid, []) -> ok;
+expect_trace_messages(Pid, [Tag | TraceTags]) ->
+    receive
+        {trace, Pid, Tag, _Data} -> ok;
+        AnythingElse ->
+            ct:pal("Unexpected message: ~p~nWhile expected {trace, _, ~p, _}",
+                   [AnythingElse, Tag]),
+            ?assert(false)
+    end,
+    expect_trace_messages(Pid, TraceTags).
-- 
cgit v1.2.3


From 640c988fd41f9709b494554b2e5ef1f06f06957e Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Sat, 20 Aug 2016 20:59:20 +0100
Subject: zlib: Only link inflateGetDictionary if available

Which at the moment means zlib versions >= 1.2.8.
---
 erts/emulator/drivers/common/zlib_drv.c | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/zlib_drv.c b/erts/emulator/drivers/common/zlib_drv.c
index acbe675c41..066cf87c9d 100644
--- a/erts/emulator/drivers/common/zlib_drv.c
+++ b/erts/emulator/drivers/common/zlib_drv.c
@@ -252,6 +252,7 @@ static int zlib_output(ZLibData* d)
     return zlib_output_init(d);
 }
 
+#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY
 static int zlib_inflate_get_dictionary(ZLibData* d)
 {
     ErlDrvBinary* dbin = driver_alloc_binary(INFL_DICT_SZ);
@@ -263,6 +264,7 @@ static int zlib_inflate_get_dictionary(ZLibData* d)
     driver_free_binary(dbin);
     return res;
 }
+#endif
 
 static int zlib_inflate(ZLibData* d, int flush)
 {
@@ -603,9 +605,14 @@ static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *bu
 	return zlib_return(res, rbuf, rlen);
 
     case INFLATE_GETDICT:
+#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY
         if (d->state != ST_INFLATE) goto badarg;
         res = zlib_inflate_get_dictionary(d);
         return zlib_return(res, rbuf, rlen);
+#else
+        errno = ENOTSUP;
+        return zlib_return(Z_ERRNO, rbuf, rlen);
+#endif
 
     case INFLATE_SYNC:
 	if (d->state != ST_INFLATE) goto badarg;
-- 
cgit v1.2.3


From 6cba7d35d3322db8acc25c45889c2b03f1b2f4c2 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 19 Aug 2016 20:16:20 +0200
Subject: erts: Add ErtsContainerStruct_ for array members

that otherwise may produce warning from compilers
that think T* and T[] are incompatible types (?).
---
 erts/emulator/beam/sys.h | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index dfe82cab44..2d0628f70e 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -99,6 +99,10 @@
 #define ErtsContainerStruct(ptr, type, member) \
     ((type *)((char *)(1 ? (ptr) : &((type *)0)->member) - offsetof(type, member)))
 
+/* Use this variant when the member is an array */
+#define ErtsContainerStruct_(ptr, type, memberv) \
+    ((type *)((char *)(1 ? (ptr) : ((type *)0)->memberv) - offsetof(type, memberv)))
+
 #if defined (__WIN32__)
 #  include "erl_win_sys.h"
 #else
-- 
cgit v1.2.3


From 1b4a59c405e6bd3532921d5c534e2264bb05b2eb Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 29 Aug 2016 19:21:53 +0200
Subject: Remove old purge strategy

---
 erts/emulator/Makefile.in            |   1 -
 erts/emulator/beam/beam_bif_load.c   | 324 ++---------------------------------
 erts/emulator/beam/bif.tab           |   2 +-
 erts/emulator/beam/erl_bif_info.c    |  21 ---
 erts/emulator/beam/erl_init.c        |   2 -
 erts/emulator/beam/erl_lock_check.c  |   2 -
 erts/emulator/beam/erl_process.c     |  11 --
 erts/emulator/beam/global.h          |   8 +-
 erts/emulator/beam/ops.tab           |   8 +-
 erts/emulator/hipe/hipe_bif_list.m4  |   2 +-
 erts/emulator/hipe/hipe_native_bif.c |   8 +-
 erts/emulator/test/code_SUITE.erl    | 168 +-----------------
 12 files changed, 24 insertions(+), 533 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 43d1914b42..21bcbbab27 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -51,7 +51,6 @@ ARFLAGS=rc
 OMIT_OMIT_FP=no
 
 DIRTY_SCHEDULER_SUPPORT=@DIRTY_SCHEDULER_SUPPORT@
-NEW_PURGE_STRATEGY=@NEW_PURGE_STRATEGY@
 
 ifeq ($(TYPE),debug)
 PURIFY =
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 5c28fb1a98..10759c1f57 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -61,7 +61,7 @@ ErtsLiteralArea *erts_copy_literal_area = NULL;
 #ifdef ERTS_DIRTY_SCHEDULERS
 Process *erts_dirty_process_code_checker;
 #endif
-#ifdef ERTS_NEW_PURGE_STRATEGY
+
 Process *erts_literal_area_collector = NULL;
 
 typedef struct ErtsLiteralAreaRef_ ErtsLiteralAreaRef;
@@ -75,10 +75,9 @@ struct {
     ErtsLiteralAreaRef *first;
     ErtsLiteralAreaRef *last;
 } release_literal_areas;
-#endif
 
 static void set_default_trace_pattern(Eterm module);
-static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls);
+static Eterm check_process_code(Process* rp, Module* modp, int *redsp, int fcalls);
 static void delete_code(Module* modp);
 static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
 static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
@@ -107,11 +106,9 @@ init_purge_state(void)
 void
 erts_beam_bif_load_init(void)
 {
-#ifdef ERTS_NEW_PURGE_STRATEGY
     erts_smp_mtx_init(&release_literal_areas.mtx, "release_literal_areas");
     release_literal_areas.first = NULL;
     release_literal_areas.last = NULL;
-#endif
 
     init_purge_state();
 }
@@ -539,7 +536,7 @@ check_old_code_1(BIF_ALIST_1)
 }
 
 Eterm
-erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls)
+erts_check_process_code(Process *c_p, Eterm module, int *redsp, int fcalls)
 {
     Module* modp;
     Eterm res;
@@ -554,31 +551,23 @@ erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int
     if (!modp)
 	return am_false;
     erts_rlock_old_code(code_ix);
-    res = (!modp->old.code_hdr ? am_false :
-           check_process_code(c_p, modp, flags, redsp, fcalls));
+    res = (!modp->old.code_hdr
+	   ? am_false
+	   : check_process_code(c_p, modp, redsp, fcalls));
     erts_runlock_old_code(code_ix);
 
     return res;
 }
 
-BIF_RETTYPE erts_internal_check_process_code_2(BIF_ALIST_2)
+BIF_RETTYPE erts_internal_check_process_code_1(BIF_ALIST_1)
 {
     int reds = 0;
-    Uint flags;
     Eterm res;
 
     if (is_not_atom(BIF_ARG_1))
 	goto badarg;
 
-    if (is_not_small(BIF_ARG_2))
-        goto badarg;
-
-    flags = unsigned_val(BIF_ARG_2);
-    if (flags & ~ERTS_CPC_ALL) {
-        goto badarg;
-    }
-
-    res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds, BIF_P->fcalls);
+    res = erts_check_process_code(BIF_P, BIF_ARG_1, &reds, BIF_P->fcalls);
 
     ASSERT(is_value(res));
 
@@ -614,7 +603,7 @@ BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2)
     if (!rp)
 	BIF_RET(am_false);
 	
-    res = erts_check_process_code(rp, BIF_ARG_2, 0, &reds, BIF_P->fcalls);
+    res = erts_check_process_code(rp, BIF_ARG_2, &reds, BIF_P->fcalls);
 
     if (BIF_P != rp)
 	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
@@ -857,32 +846,12 @@ set_default_trace_pattern(Eterm module)
     }
 }
 
-#ifndef ERTS_NEW_PURGE_STRATEGY
-
-static ERTS_INLINE int
-check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size)
-{
-    struct erl_off_heap_header* oh;
-    for (oh = off_heap->first; oh; oh = oh->next) {
-	if (thing_subtag(oh->thing_word) == FUN_SUBTAG) {
-	    ErlFunThing* funp = (ErlFunThing*) oh;
-	    if (ErtsInArea(funp->fe->address, area, area_size))
-		return !0;
-	}
-    }
-    return 0;
-}
-
-#endif
-
 static Uint hfrag_literal_size(Eterm* start, Eterm* end,
                                char* lit_start, Uint lit_size);
 static void hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp,
                                Eterm *start, Eterm *end,
                                char *lit_start, Uint lit_size);
 
-#ifdef ERTS_NEW_PURGE_STRATEGY
-
 Eterm
 erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed)
 {
@@ -1044,7 +1013,7 @@ literal_gc:
 }
 
 static Eterm
-check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls)
+check_process_code(Process* rp, Module* modp, int *redsp, int fcalls)
 {
     BeamInstr* start;
     char* mod_start;
@@ -1109,253 +1078,6 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls
     return am_false;
 }
 
-#else /* !ERTS_NEW_PURGE_STRATEGY, i.e, old style purge... */
-
-static Eterm
-check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls)
-{
-    BeamInstr* start;
-    char* literals;
-    Uint lit_bsize;
-    char* mod_start;
-    Uint mod_size;
-    Eterm* sp;
-    int done_gc = 0;
-    int need_gc = 0;
-    ErtsMessage *msgp;
-    ErlHeapFragment *hfrag;
-
-#define ERTS_ORDINARY_GC__ (1 << 0)
-#define ERTS_LITERAL_GC__  (1 << 1)
-
-    /*
-     * Pick up limits for the module.
-     */
-    start = (BeamInstr*) modp->old.code_hdr;
-    mod_start = (char *) start;
-    mod_size = modp->old.code_length;
-
-    /*
-     * Check if current instruction or continuation pointer points into module.
-     */
-    if (ErtsInArea(rp->i, mod_start, mod_size)
-	|| ErtsInArea(rp->cp, mod_start, mod_size)) {
-	return am_true;
-    }
-
-    /*
-     * Check all continuation pointers stored on the stack.
-     */
-    for (sp = rp->stop; sp < STACK_START(rp); sp++) {
-	if (is_CP(*sp) && ErtsInArea(cp_val(*sp), mod_start, mod_size)) {
-	    return am_true;
-	}
-    }
-
-    /* 
-     * Check all continuation pointers stored in stackdump
-     * and clear exception stackdump if there is a pointer
-     * to the module.
-     */
-    if (rp->ftrace != NIL) {
-	struct StackTrace *s;
-	ASSERT(is_list(rp->ftrace));
-	s = (struct StackTrace *) big_val(CDR(list_val(rp->ftrace)));
-	if ((s->pc && ErtsInArea(s->pc, mod_start, mod_size)) ||
-	    (s->current && ErtsInArea(s->current, mod_start, mod_size))) {
-	    rp->freason = EXC_NULL;
-	    rp->fvalue = NIL;
-	    rp->ftrace = NIL;
-	} else {
-	    int i;
-	    for (i = 0;  i < s->depth;  i++) {
-		if (ErtsInArea(s->trace[i], mod_start, mod_size)) {
-		    rp->freason = EXC_NULL;
-		    rp->fvalue = NIL;
-		    rp->ftrace = NIL;
-		    break;
-		}
-	    }
-	}
-    }
-
-    if (rp->flags & F_DISABLE_GC) {
-	/*
-	 * Cannot proceed. Process has disabled gc in order to
-	 * safely leave inconsistent data on the heap and/or
-	 * off heap lists. Need to wait for gc to be enabled
-	 * again.
-	 */ 
-	return THE_NON_VALUE;
-    }
-
-    /*
-     * Message queue can contains funs, and may contain
-     * literals. If we got references to this module from the message
-     * queue.
-     *
-     * If a literal is in the message queue we maka an explicit copy of
-     * and attach it to the heap fragment. Each message needs to be
-     * self contained, we cannot save the literal in the old_heap or
-     * any other heap than the message it self.
-     */
-
-    erts_smp_proc_lock(rp, ERTS_PROC_LOCK_MSGQ);
-    ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp);
-    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ);
-
-    if (modp->old.code_hdr->literal_area) {
-	literals = (char*) modp->old.code_hdr->literal_area->start;
-	lit_bsize = (char*) modp->old.code_hdr->literal_area->end - literals;
-    }
-    else {
-	literals = NULL;
-	lit_bsize = 0;
-    }
-
-    for (msgp = rp->msg.first; msgp; msgp = msgp->next) {
-	if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG)
-	    hfrag = &msgp->hfrag;
-	else if (is_value(ERL_MESSAGE_TERM(msgp)) && msgp->data.heap_frag)
-	    hfrag = msgp->data.heap_frag;
-	else
-	    continue;
-        {
-            ErlHeapFragment *hf;
-            Uint lit_sz = 0;
-            for (hf=hfrag; hf; hf = hf->next) {
-                if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size))
-                    return am_true;
-                lit_sz += hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size],
-                                             literals, lit_bsize);
-            }
-            if (lit_sz > 0) {
-                ErlHeapFragment *bp = new_message_buffer(lit_sz);
-                Eterm *hp = bp->mem;
-
-                for (hf=hfrag; hf; hf = hf->next) {
-                    hfrag_literal_copy(&hp, &bp->off_heap,
-                                       &hf->mem[0], &hf->mem[hf->used_size],
-                                       literals, lit_bsize);
-                    hfrag=hf;
-                }
-                /* link new hfrag last */
-                ASSERT(hfrag->next == NULL);
-                hfrag->next = bp;
-                bp->next = NULL;
-            }
-        }
-    }
-
-    while (1) {
-
-	/* Check heap, stack etc... */
-	if (check_mod_funs(rp, &rp->off_heap, mod_start, mod_size))
-	    goto try_gc;
-	if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, literals, lit_bsize)) {
-	    rp->freason = EXC_NULL;
-	    rp->fvalue = NIL;
-	    rp->ftrace = NIL;
-	}
-	if (any_heap_ref_ptrs(rp->stop, rp->hend, literals, lit_bsize))
-	    goto try_literal_gc;
-#ifdef HIPE
-	if (nstack_any_heap_ref_ptrs(rp, literals, lit_bsize))
-	    goto try_literal_gc;
-#endif
-	if (any_heap_refs(rp->heap, rp->htop, literals, lit_bsize))
-	    goto try_literal_gc;
-	if (any_heap_refs(rp->old_heap, rp->old_htop, literals, lit_bsize))
-	    goto try_literal_gc;
-
-	/* Check dictionary */
-	if (rp->dictionary) {
-	    Eterm* start = ERTS_PD_START(rp->dictionary);
-	    Eterm* end = start + ERTS_PD_SIZE(rp->dictionary);
-
-	    if (any_heap_ref_ptrs(start, end, literals, lit_bsize))
-		goto try_literal_gc;
-	}
-
-	/* Check heap fragments */
-	for (hfrag = rp->mbuf; hfrag; hfrag = hfrag->next) {
-	    Eterm *hp, *hp_end;
-	    /* Off heap lists should already have been moved into process */
-	    ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size));
-
-	    hp = &hfrag->mem[0];
-	    hp_end = &hfrag->mem[hfrag->used_size];
-	    if (any_heap_refs(hp, hp_end, literals, lit_bsize))
-		goto try_literal_gc;
-	}
-
-	/*
-	 * Message buffer fragments (matched messages) 
-         *  - off heap lists should already have been moved into
-         *    process off heap structure.
-         *  - Check for literals
-	 */
-	for (msgp = rp->msg_frag; msgp; msgp = msgp->next) {
-            hfrag = erts_message_to_heap_frag(msgp);
-	    for (; hfrag; hfrag = hfrag->next) {
-		Eterm *hp, *hp_end;
-		ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size));
-
-		hp = &hfrag->mem[0];
-		hp_end = &hfrag->mem[hfrag->used_size];
-
-                if (any_heap_refs(hp, hp_end, literals, lit_bsize))
-                    goto try_literal_gc;
-	    }
-	}
-
-	return am_false;
-
-    try_literal_gc:
-	need_gc |= ERTS_LITERAL_GC__;
-
-    try_gc:
-	need_gc |= ERTS_ORDINARY_GC__;
-
-	if ((done_gc & need_gc) == need_gc)
-	    return am_true;
-
-	if (!(flags & ERTS_CPC_ALLOW_GC))
-	    return am_aborted;
-
-	need_gc &= ~done_gc;
-
-	/*
-	 * Try to get rid of literals by by garbage collecting.
-	 * Clear both fvalue and ftrace.
-	 */
-
-	rp->freason = EXC_NULL;
-	rp->fvalue = NIL;
-	rp->ftrace = NIL;
-
-	if (need_gc & ERTS_ORDINARY_GC__) {
-	    FLAGS(rp) |= F_NEED_FULLSWEEP;
-	    *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity, fcalls);
-	    done_gc |= ERTS_ORDINARY_GC__;
-	}
-	if (need_gc & ERTS_LITERAL_GC__) {
-	    struct erl_off_heap_header* oh;
-	    oh = modp->old.code_hdr->literal_area->off_heap;
-	    *redsp += lit_bsize / 64; /* Need, better value... */
-	    erts_garbage_collect_literals(rp, (Eterm*)literals, lit_bsize, oh);
-	    done_gc |= ERTS_LITERAL_GC__;
-	}
-	need_gc = 0;
-    }
-
-#undef ERTS_ORDINARY_GC__
-#undef ERTS_LITERAL_GC__
-
-}
-
-#endif /*  !ERTS_NEW_PURGE_STRATEGY */
-
 static int
 any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
 {
@@ -1484,8 +1206,6 @@ hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp,
     }
 }
 
-#ifdef ERTS_NEW_PURGE_STRATEGY
-
 ErtsThrPrgrLaterOp later_literal_area_switch;
 
 #ifdef ERTS_SMP
@@ -1499,13 +1219,8 @@ complete_literal_area_switch(void *unused)
 }
 #endif
 
-#endif /* ERTS_NEW_PURGE_STRATEGY */
-
 BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0)
 {
-#ifndef ERTS_NEW_PURGE_STRATEGY
-    BIF_ERROR(BIF_P, EXC_NOTSUP);
-#else
     ErtsLiteralAreaRef *la_ref;
 
     if (BIF_P != erts_literal_area_collector)
@@ -1544,7 +1259,6 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0)
     ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
 #endif
 
-#endif /* ERTS_NEW_PURGE_STRATEGY */
 }
 
 void
@@ -1722,10 +1436,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 		code = (BeamInstr*) modp->old.code_hdr;
 		end = (BeamInstr *)((char *)code + modp->old.code_length);
 		erts_fun_purge_prepare(code, end);
-#if !defined(ERTS_NEW_PURGE_STRATEGY)
-		ASSERT(!erts_copy_literal_area);
-		erts_copy_literal_area = modp->old.code_hdr->literal_area;
-#endif
 	    }
 	    erts_runlock_old_code(code_ix);
 	}
@@ -1765,10 +1475,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 
 	erts_fun_purge_abort_prepare(purge_state.funs, purge_state.fe_ix);
 
-#if !defined(ERTS_NEW_PURGE_STRATEGY)
-	ASSERT(erts_copy_literal_area);
-	erts_copy_literal_area = NULL;
-#endif
 #ifndef ERTS_SMP
 	erts_fun_purge_abort_finalize(purge_state.funs, purge_state.fe_ix);
 	finalize_purge_operation(BIF_P, 0);
@@ -1877,14 +1583,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 
 	finalize_purge_operation(BIF_P, ret == am_true);
 
-#if !defined(ERTS_NEW_PURGE_STRATEGY)
-
-	ASSERT(erts_copy_literal_area == literals);
-	erts_copy_literal_area = NULL;
-	erts_release_literal_area(literals);
-
-#else /* ERTS_NEW_PURGE_STRATEGY */
-
 	if (literals) {
 	    ErtsLiteralAreaRef *ref;
 	    ref = erts_alloc(ERTS_ALC_T_LITERAL_REF,
@@ -1908,8 +1606,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 			       BIF_P->common.id);
 	}
 
-#endif /* ERTS_NEW_PURGE_STRATEGY */
-
 	return ret;
     }
 
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index ea66f165a0..7a35c02934 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -164,7 +164,7 @@ bif erts_internal:port_connect/2
 
 bif erts_internal:request_system_task/3
 bif erts_internal:request_system_task/4
-bif erts_internal:check_process_code/2
+bif erts_internal:check_process_code/1
 
 bif erts_internal:map_to_tuple_keys/1
 bif erts_internal:term_type/1
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 107c0c01d9..cb7278696f 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2883,27 +2883,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(AM_tag);
 #endif
     }
-    else if (ERTS_IS_ATOM_STR("check_process_code",BIF_ARG_1)) {
-	Eterm terms[3];
-	Sint length = 1;
-	Uint sz = 0;
-	Eterm *hp, res;
-	DECL_AM(direct_references);
-
-	terms[0] = AM_direct_references;
-#if !defined(ERTS_NEW_PURGE_STRATEGY)
-	{
-	    DECL_AM(indirect_references);
-	    terms[1] = AM_indirect_references;
-	    terms[2] = am_copy_literals;
-	    length = 3;
-	}
-#endif
-	erts_bld_list(NULL, &sz, length, terms);
-	hp = HAlloc(BIF_P, sz);
-	res = erts_bld_list(&hp, NULL, length, terms);
-	BIF_RET(res);
-    }
 
     BIF_ERROR(BIF_P, BADARG);
 }
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 781bf024dd..fb85bbff34 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -2264,7 +2264,6 @@ erl_start(int argc, char **argv)
 	ASSERT(erts_code_purger && erts_code_purger->common.id == pid);
 	erts_proc_inc_refc(erts_code_purger); 
 
-#ifdef ERTS_NEW_PURGE_STRATEGY
 	pid = erl_system_process_otp(otp_ring0_pid, "erts_literal_area_collector");
 	erts_literal_area_collector
 	    = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc,
@@ -2272,7 +2271,6 @@ erl_start(int argc, char **argv)
 	ASSERT(erts_literal_area_collector
 	       && erts_literal_area_collector->common.id == pid);
 	erts_proc_inc_refc(erts_literal_area_collector);
-#endif
 
 #ifdef ERTS_DIRTY_SCHEDULERS
 	pid = erl_system_process_otp(otp_ring0_pid, "erts_dirty_process_code_checker");
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 06266363b5..356f5ca71e 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -113,9 +113,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"export_tab",				NULL			},
     {	"fun_tab",				NULL			},
     {	"environ",				NULL			},
-#ifdef ERTS_NEW_PURGE_STRATEGY
     {	"release_literal_areas",		NULL			},
-#endif
 #endif
     {	"efile_drv",				"address"		},
     {	"drv_ev_state_grow",			NULL,   		},
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index fa8290028a..91eb82af7d 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -448,9 +448,7 @@ int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP;
 typedef enum {
     ERTS_PSTT_GC,	/* Garbage Collect */
     ERTS_PSTT_CPC,	/* Check Process Code */
-#ifdef ERTS_NEW_PURGE_STRATEGY
     ERTS_PSTT_CLA,	/* Copy Literal Area */
-#endif
     ERTS_PSTT_COHMQ,    /* Change off heap message queue */
     ERTS_PSTT_FTMQ      /* Flush trace msg queue */
 } ErtsProcSysTaskType;
@@ -10430,7 +10428,6 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 		fcalls = reds - CONTEXT_REDS;
 	    st_res = erts_check_process_code(c_p,
 					     st->arg[0],
-                                             unsigned_val(st->arg[1]),
 					     &cpc_reds,
 					     fcalls);
             reds -= cpc_reds;
@@ -10441,7 +10438,6 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	    }
 	    break;
         }
-#ifdef ERTS_NEW_PURGE_STRATEGY
 	case ERTS_PSTT_CLA: {
 	    int fcalls;
             int cla_reds = 0;
@@ -10461,7 +10457,6 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	    }
 	    break;
         }
-#endif
 	case ERTS_PSTT_COHMQ:
 	    reds -= erts_complete_off_heap_message_queue_change(c_p);
 	    st_res = am_true;
@@ -10516,11 +10511,9 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
 	case ERTS_PSTT_COHMQ:
 	    st_res = am_false;
 	    break;
-#ifdef ERTS_NEW_PURGE_STRATEGY
 	case ERTS_PSTT_CLA:
 	    st_res = am_ok;
 	    break;
-#endif
 #ifdef ERTS_SMP
         case ERTS_PSTT_FTMQ:
 	    reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN);
@@ -10695,8 +10688,6 @@ request_system_task(Process *c_p, Eterm requester, Eterm target,
     case am_check_process_code:
 	if (is_not_atom(st->arg[0]))
 	    goto badarg;
-	if (is_not_small(st->arg[1]) || (unsigned_val(st->arg[1]) & ~ERTS_CPC_ALL))
-	    goto badarg;
 	noproc_res = am_false;
 	st->type = ERTS_PSTT_CPC;
 	if (!rp)
@@ -10712,7 +10703,6 @@ request_system_task(Process *c_p, Eterm requester, Eterm target,
 #endif
 	break;
 
-#ifdef ERTS_NEW_PURGE_STRATEGY
     case am_copy_literals:
 	if (st->arg[0] != am_true && st->arg[0] != am_false)
 	    goto badarg;
@@ -10721,7 +10711,6 @@ request_system_task(Process *c_p, Eterm requester, Eterm target,
 	if (!rp)
 	    goto noproc;
 	break;
-#endif
 
     default:
 	goto badarg;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index d7dd6371b7..0fd9ab5e58 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1000,12 +1000,8 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg);
 Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2);
 
 /* beam_bif_load.c */
-#define ERTS_CPC_ALLOW_GC      (1 << 0)
-#define ERTS_CPC_ALL           ERTS_CPC_ALLOW_GC
-Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls);
-#ifdef ERTS_NEW_PURGE_STRATEGY
+Eterm erts_check_process_code(Process *c_p, Eterm module, int *redsp, int fcalls);
 Eterm erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed);
-#endif
 
 typedef struct ErtsLiteralArea_ {
     struct erl_off_heap_header *off_heap;
@@ -1017,9 +1013,7 @@ typedef struct ErtsLiteralArea_ {
     (sizeof(ErtsLiteralArea) + sizeof(Eterm)*((N) - 1))
 
 extern ErtsLiteralArea *erts_copy_literal_area;
-#ifdef ERTS_NEW_PURGE_STRATEGY
 extern Process *erts_literal_area_collector;
-#endif
 #ifdef ERTS_DIRTY_SCHEDULERS
 extern Process *erts_dirty_process_code_checker;
 #endif
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 879daaca0a..b467c5a9b6 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -792,14 +792,14 @@ allocate_init t I y
 #################################################################
 
 #
-# The BIFs erts_internal:check_process_code/2 must be called like a function,
+# The BIFs erts_internal:check_process_code/1 must be called like a function,
 # to ensure that c_p->i (program counter) is set correctly (an ordinary
 # BIF call doesn't set it).
 #
 
-call_ext u==2 Bif=u$bif:erts_internal:check_process_code/2 => i_call_ext Bif
-call_ext_last u==2 Bif=u$bif:erts_internal:check_process_code/2 D => i_call_ext_last Bif D
-call_ext_only u==2 Bif=u$bif:erts_internal:check_process_code/2 => i_call_ext_only Bif
+call_ext u==1 Bif=u$bif:erts_internal:check_process_code/1 => i_call_ext Bif
+call_ext_last u==1 Bif=u$bif:erts_internal:check_process_code/1 D => i_call_ext_last Bif D
+call_ext_only u==1 Bif=u$bif:erts_internal:check_process_code/1 => i_call_ext_only Bif
 
 #
 # The BIFs erlang:garbage_collect/0 must be called like a function,
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index dcf3447af9..bb328b5915 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -153,7 +153,7 @@ standard_bif_interface_0(nbif_ports_0, ports_0)
  * BIFs and primops that may do a GC (change heap limit and walk the native stack).
  * XXX: erase/1 and put/2 cannot fail
  */
-gc_bif_interface_2(nbif_erts_internal_check_process_code_2, hipe_erts_internal_check_process_code_2)
+gc_bif_interface_1(nbif_erts_internal_check_process_code_1, hipe_erts_internal_check_process_code_1)
 gc_bif_interface_1(nbif_erase_1, erase_1)
 gc_bif_interface_0(nbif_garbage_collect_0, garbage_collect_0)
 gc_nofail_primop_interface_1(nbif_gc_1, hipe_gc)
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index 9c03b3811c..9439b823ab 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -42,7 +42,7 @@
  */
 
 /* for -Wmissing-prototypes :-( */
-extern Eterm hipe_erts_internal_check_process_code_2(BIF_ALIST_2);
+extern Eterm hipe_erts_internal_check_process_code_1(BIF_ALIST_1);
 extern Eterm hipe_show_nstack_1(BIF_ALIST_1);
 
 /* Used when a BIF can trigger a stack walk. */
@@ -51,12 +51,12 @@ static __inline__ void hipe_set_narity(Process *p, unsigned int arity)
     p->hipe.narity = arity;
 }
 
-Eterm hipe_erts_internal_check_process_code_2(BIF_ALIST_2)
+Eterm hipe_erts_internal_check_process_code_1(BIF_ALIST_1)
 {
     Eterm ret;
 
-    hipe_set_narity(BIF_P, 2);
-    ret = erts_internal_check_process_code_2(BIF_P, BIF__ARGS);
+    hipe_set_narity(BIF_P, 1);
+    ret = erts_internal_check_process_code_1(BIF_P, BIF__ARGS);
     hipe_set_narity(BIF_P, 0);
     return ret;
 }
diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index 34515efa3d..82607424b5 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -22,8 +22,7 @@
 -export([all/0, suite/0, init_per_suite/1, end_per_suite/1, 
          versions/1,new_binary_types/1, call_purged_fun_code_gone/1,
 	 call_purged_fun_code_reload/1, call_purged_fun_code_there/1,
-         t_check_process_code/1,t_check_old_code/1,
-         t_check_process_code_ets/1,
+         t_check_old_code/1,
          external_fun/1,get_chunk/1,module_md5/1,make_stub/1,
          make_stub_many_funs/1,constant_pools/1,constant_refc_binaries/1,
          false_dependency/1,coverage/1,fun_confusion/1,
@@ -36,8 +35,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}].
 
 all() -> 
     [versions, new_binary_types, call_purged_fun_code_gone,
-     call_purged_fun_code_reload, call_purged_fun_code_there, t_check_process_code,
-     t_check_process_code_ets, t_check_old_code, external_fun, get_chunk,
+     call_purged_fun_code_reload, call_purged_fun_code_there,
+     t_check_old_code, external_fun, get_chunk,
      module_md5, make_stub, make_stub_many_funs,
      constant_pools, constant_refc_binaries, false_dependency,
      coverage, fun_confusion, t_copy_literals, t_copy_literals_frags].
@@ -276,98 +275,6 @@ call_purged_fun_test(Priv, Data, Type) ->
     end,
     ok.
 
-t_check_process_code(Config) when is_list(Config) ->
-    case check_process_code_handle(indirect_references) of
-	false -> {skipped, "check_process_code() ignores funs"};
-	true -> t_check_process_code_test(Config)
-    end.
-
-t_check_process_code_test(Config) ->
-    Priv = proplists:get_value(priv_dir, Config),
-    Data = proplists:get_value(data_dir, Config),
-    File = filename:join(Data, "my_code_test"),
-    Code = filename:join(Priv, "my_code_test"),
-
-    catch erlang:purge_module(my_code_test),
-    catch erlang:delete_module(my_code_test),
-    catch erlang:purge_module(my_code_test),
-
-    {ok,my_code_test} = c:c(File, [{outdir,Priv}]),
-
-    MyFun = fun(X, Y) -> X + Y end,	%Confuse things.
-    F = my_code_test:make_fun(42),
-    2 = fun_refc(F),
-    MyFun2 = fun(X, Y) -> X * Y end,	%Confuse things.
-    44 = F(2),
-
-    %% Delete the module and call the fun again.
-    true = erlang:delete_module(my_code_test),
-    2 = fun_refc(F),
-    45 = F(3),
-    {'EXIT',{undef,_}} = (catch my_code_test:make_fun(33)),
-
-    %% The fun should still be there, preventing purge.
-    true = erlang:check_process_code(self(), my_code_test),
-    gc(),
-    gc(),					%Place funs on the old heap.
-    true = erlang:check_process_code(self(), my_code_test),
-
-    %% Using the funs here guarantees that they will not be prematurely garbed.
-    48 = F(6),
-    3 = MyFun(1, 2),
-    12 = MyFun2(3, 4),
-
-    %% Kill all funs.
-    t_check_process_code1(Code, []).
-
-%% The real fun was killed, but we have some fakes which look similar.
-
-t_check_process_code1(Code, Fakes) ->
-    MyFun = fun(X, Y) -> X + Y + 1 end,	%Confuse things.
-    false = erlang:check_process_code(self(), my_code_test),
-    4 = MyFun(1, 2),
-    t_check_process_code2(Code, Fakes).
-
-t_check_process_code2(Code, _) ->
-    false = erlang:check_process_code(self(), my_code_test),
-    true = erlang:purge_module(my_code_test),
-
-    %% In the next test we will load the same module twice.
-    {module,my_code_test} = code:load_abs(Code),
-    F = my_code_test:make_fun(37),
-    2 = fun_refc(F),
-    false = erlang:check_process_code(self(), my_code_test),
-    {module,my_code_test} = code:load_abs(Code),
-    2 = fun_refc(F),
-
-    %% Still false because the fun with the same identify is found
-    %% in the current code.
-    false = erlang:check_process_code(self(), my_code_test),
-
-    %% Some fake funs in the same module should not do any difference.
-    false = erlang:check_process_code(self(), my_code_test),
-
-    38 = F(1),
-    t_check_process_code3(Code, F, []).
-
-t_check_process_code3(Code, F, Fakes) ->
-    Pid = spawn_link(fun() -> body(F, Fakes) end),
-    true = erlang:purge_module(my_code_test),
-    false = erlang:check_process_code(self(), my_code_test),
-    false = erlang:check_process_code(Pid, my_code_test),
-
-    true = erlang:delete_module(my_code_test),
-    true = erlang:check_process_code(self(), my_code_test),
-    true = erlang:check_process_code(Pid, my_code_test),
-    39 = F(2),
-    t_check_process_code4(Code, Pid).
-
-t_check_process_code4(_Code, Pid) ->
-    Pid ! drop_funs,
-    receive after 1 -> ok end,
-    false = erlang:check_process_code(Pid, my_code_test),
-    ok.
-
 body(F, Fakes) ->
     receive
         jog ->
@@ -388,72 +295,6 @@ gc() ->
     gc1().
 gc1() -> ok.
 
-%% Test check_process_code/2 in combination with a fun obtained from an ets table.
-t_check_process_code_ets(Config) when is_list(Config) ->
-    case check_process_code_handle(indirect_references) of
-	false ->
-	    {skipped, "check_process_code() ignores funs"};
-	true ->
-	    case test_server:is_native(?MODULE) of
-		true ->
-		    {skip,"Native code"};
-		false ->
-		    do_check_process_code_ets(Config)
-	    end
-    end.
-
-do_check_process_code_ets(Config) ->
-    Priv = proplists:get_value(priv_dir, Config),
-    Data = proplists:get_value(data_dir, Config),
-    File = filename:join(Data, "my_code_test"),
-
-    catch erlang:purge_module(my_code_test),
-    catch erlang:delete_module(my_code_test),
-    catch erlang:purge_module(my_code_test),
-    {ok,my_code_test} = c:c(File, [{outdir,Priv}]),
-
-    T = ets:new(my_code_test, []),
-    ets:insert(T, {7,my_code_test:make_fun(107)}),
-    ets:insert(T, {8,my_code_test:make_fun(108)}),
-    erlang:delete_module(my_code_test),
-    false = erlang:check_process_code(self(), my_code_test),
-    Body = fun() ->
-                   [{7,F1}] = ets:lookup(T, 7),
-                   [{8,F2}] = ets:lookup(T, 8),
-                   IdleLoop = fun() -> receive _X -> ok end end,
-                   RecLoop = fun(Again) ->
-                                     receive
-                                         call -> 110 = F1(3),
-                                                 100 = F2(-8),
-                                                 Again(Again);
-                                         {drop_funs,To} ->
-                                             To ! funs_dropped,
-                                             IdleLoop()
-                                     end
-                             end,
-                   true = erlang:check_process_code(self(), my_code_test),
-                   RecLoop(RecLoop)
-           end,
-    Pid = spawn_link(Body),
-    receive after 1 -> ok end,
-    true = erlang:check_process_code(Pid, my_code_test),
-    Pid ! call,
-    Pid ! {drop_funs,self()},
-
-    receive
-        funs_dropped -> ok;
-        Other -> ct:fail({unexpected,Other})
-    after 10000 ->
-              ct:fail(no_funs_dropped_answer)
-    end,
-
-    false = erlang:check_process_code(Pid, my_code_test),
-    ok.
-
-fun_refc(F) ->
-    {refc,Count} = erlang:fun_info(F, refc),
-    Count.
-
 
 %% Test the erlang:check_old_code/1 BIF.
 t_check_old_code(Config) when is_list(Config) ->
@@ -1137,9 +978,6 @@ flush() ->
 
 id(I) -> I.
 
-check_process_code_handle(What) ->
-    lists:member(What, erlang:system_info(check_process_code)).
-
 wait_until(Fun) ->
     case Fun() of
 	true ->
-- 
cgit v1.2.3


From 7be8e2309d87dbb6e922cb0ca56f031f9e4ec12b Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 30 Aug 2016 12:16:57 +0200
Subject: Avoid selective receive in code-purger process

The code purger process handles vast amounts of messages when
there are lots of processes alive. A single message in the
message queue that does not match will in such cases cause
lots of extra work. The code purger process now always picks
the first message in the message queue, and by this avoid
this extra work.
---
 erts/emulator/test/code_SUITE.erl | 62 +++++++++++++++++++++++++++++++++++++--
 1 file changed, 60 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index 82607424b5..8427bb134d 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -22,7 +22,7 @@
 -export([all/0, suite/0, init_per_suite/1, end_per_suite/1, 
          versions/1,new_binary_types/1, call_purged_fun_code_gone/1,
 	 call_purged_fun_code_reload/1, call_purged_fun_code_there/1,
-         t_check_old_code/1,
+         multi_proc_purge/1, t_check_old_code/1,
          external_fun/1,get_chunk/1,module_md5/1,make_stub/1,
          make_stub_many_funs/1,constant_pools/1,constant_refc_binaries/1,
          false_dependency/1,coverage/1,fun_confusion/1,
@@ -36,7 +36,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}].
 all() -> 
     [versions, new_binary_types, call_purged_fun_code_gone,
      call_purged_fun_code_reload, call_purged_fun_code_there,
-     t_check_old_code, external_fun, get_chunk,
+     multi_proc_purge, t_check_old_code, external_fun, get_chunk,
      module_md5, make_stub, make_stub_many_funs,
      constant_pools, constant_refc_binaries, false_dependency,
      coverage, fun_confusion, t_copy_literals, t_copy_literals_frags].
@@ -275,6 +275,64 @@ call_purged_fun_test(Priv, Data, Type) ->
     end,
     ok.
 
+multi_proc_purge(Config) when is_list(Config) ->
+    %%
+    %% Make sure purge requests aren't lost when
+    %% purger process is working.
+    %%
+    Priv = proplists:get_value(priv_dir, Config),
+    Data = proplists:get_value(data_dir, Config),
+    File1 = filename:join(Data, "my_code_test"),
+    File2 = filename:join(Data, "my_code_test2"),
+    
+    {ok,my_code_test} = c:c(File1, [{outdir,Priv}]),
+    {ok,my_code_test2} = c:c(File2, [{outdir,Priv}]),
+    erlang:delete_module(my_code_test),
+    erlang:delete_module(my_code_test2),
+
+    Self = self(),
+
+    Fun1 = fun () ->
+		   erts_code_purger:purge(my_code_test),
+		   Self ! {self(), done}
+	   end,
+    Fun2 = fun () ->
+		   erts_code_purger:soft_purge(my_code_test2),
+		   Self ! {self(), done}
+	   end,
+    Fun3 = fun () ->
+		   erts_code_purger:purge('__nonexisting_module__'),
+		   Self ! {self(), done}
+	   end,
+    Fun4 = fun () ->
+		   erts_code_purger:soft_purge('__another_nonexisting_module__'),
+		   Self ! {self(), done}
+	   end,
+
+    Pid1 = spawn_link(Fun1),
+    Pid2 = spawn_link(Fun2),
+    Pid3 = spawn_link(Fun3),
+    Pid4 = spawn_link(Fun4),
+    Pid5 = spawn_link(Fun1),
+    Pid6 = spawn_link(Fun2),
+    Pid7 = spawn_link(Fun3),
+    receive after 50 -> ok end,
+    Pid8 = spawn_link(Fun4),
+    Pid9 = spawn_link(Fun1),
+    Pid10 = spawn_link(Fun2),
+    Pid11 = spawn_link(Fun3),
+    Pid12 = spawn_link(Fun4),
+    Pid13 = spawn_link(Fun1),
+    receive after 50 -> ok end,
+    Pid14 = spawn_link(Fun2),
+    Pid15 = spawn_link(Fun3),
+    Pid16 = spawn_link(Fun4),
+
+    lists:foreach(fun (P) -> receive {P, done} -> ok end end,
+		  [Pid1, Pid2, Pid3, Pid4, Pid5, Pid6, Pid7, Pid8,
+		   Pid9, Pid10, Pid11, Pid12, Pid13, Pid14, Pid15, Pid16]),
+    ok.
+
 body(F, Fakes) ->
     receive
         jog ->
-- 
cgit v1.2.3


From c46ba734b6787e254b45b35ea135fc2da23190bb Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 23 Feb 2016 11:24:12 +0100
Subject: erts: Fix install of suspend handler

This commit makes sure to setup the suspend handler
to matter what +B option is given at the command line.
---
 erts/emulator/beam/break.c            |  4 ++--
 erts/emulator/beam/erl_init.c         |  1 +
 erts/emulator/sys/unix/erl_unix_sys.h |  1 +
 erts/emulator/sys/unix/sys.c          | 24 ++++++++++++++++--------
 4 files changed, 20 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 0ddf7f4e6d..298b30fff3 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -684,7 +684,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
        crash dump. */
     erts_thr_progress_fatal_error_block(&tpd_buf);
 
-#ifdef ERTS_THR_HAVE_SIG_FUNCS
+#ifdef ERTS_SYS_SUSPEND_SIGNAL
     /*
      * We suspend all scheduler threads so that we can dump some
      * data about the currently running processes and scheduler data.
@@ -818,7 +818,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
 
 #ifdef ERTS_SMP
 
-#if defined(ERTS_THR_HAVE_SIG_FUNCS)
+#ifdef ERTS_SYS_SUSPEND_SIGNAL
 
     /* We resume all schedulers so that we are in a known safe state
        when we write the rest of the crash dump */
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index e729574ec7..1718b23688 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -2134,6 +2134,7 @@ erl_start(int argc, char **argv)
 	init_break_handler();
     if (replace_intr)
 	erts_replace_intr();
+    sys_init_suspend_handler();
 #endif
 
     boot_argc = argc - i;  /* Number of arguments to init */
diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index 8d4e98bf3a..b55180c509 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -311,6 +311,7 @@ extern SIGFUNC sys_signal(int, SIGFUNC);
 extern void sys_sigrelease(int);
 extern void sys_sigblock(int);
 extern void sys_stop_cat(void);
+extern void sys_init_suspend_handler(void);
 
 /*
  * Handling of floating point exceptions.
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index d94b37430e..cbd47db37f 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -226,8 +226,10 @@ static erts_smp_atomic_t sys_misc_mem_sz;
 static void smp_sig_notify(char c);
 static int sig_notify_fds[2] = {-1, -1};
 
+#if !defined(ETHR_UNUSABLE_SIGUSRX) && defined(ERTS_THR_HAVE_SIG_FUNCS)
 static int sig_suspend_fds[2] = {-1, -1};
 #define ERTS_SYS_SUSPEND_SIGNAL SIGUSR2
+#endif
 
 #endif
 
@@ -872,7 +874,7 @@ sigusr1_exit(void)
 
 #else
 
-#ifdef ERTS_SMP
+#ifdef ERTS_SYS_SUSPEND_SIGNAL
 void
 sys_thr_suspend(erts_tid_t tid) {
     erts_thr_kill(tid, ERTS_SYS_SUSPEND_SIGNAL);
@@ -900,7 +902,7 @@ static RETSIGTYPE user_signal1(int signum)
 #endif
 }
 
-#ifdef ERTS_SMP
+#ifdef ERTS_SYS_SUSPEND_SIGNAL
 #if (defined(SIG_SIGSET) || defined(SIG_SIGNAL))
 static RETSIGTYPE suspend_signal(void)
 #else
@@ -913,7 +915,7 @@ static RETSIGTYPE suspend_signal(int signum)
      res = read(sig_suspend_fds[0], buf, sizeof(int));
    } while (res < 0 && errno == EINTR);
 }
-#endif /* #ifdef ERTS_SMP */
+#endif /* #ifdef ERTS_SYS_SUSPEND_SIGNAL */
 
 #endif /* #ifndef ETHR_UNUSABLE_SIGUSRX */
 
@@ -966,13 +968,17 @@ void init_break_handler(void)
    sys_signal(SIGINT, request_break);
 #ifndef ETHR_UNUSABLE_SIGUSRX
    sys_signal(SIGUSR1, user_signal1);
-#ifdef ERTS_SMP
-   sys_signal(ERTS_SYS_SUSPEND_SIGNAL, suspend_signal);
-#endif /* #ifdef ERTS_SMP */
 #endif /* #ifndef ETHR_UNUSABLE_SIGUSRX */
    sys_signal(SIGQUIT, do_quit);
 }
 
+void sys_init_suspend_handler(void)
+{
+#ifdef ERTS_SYS_SUSPEND_SIGNAL
+   sys_signal(ERTS_SYS_SUSPEND_SIGNAL, suspend_signal);
+#endif
+}
+
 int sys_max_files(void)
 {
    return(max_files);
@@ -990,7 +996,7 @@ static void block_signals(void)
 #endif /* #ifndef ETHR_UNUSABLE_SIGUSRX */
 #endif /* #ifndef ERTS_SMP */
 
-#if defined(ERTS_SMP) && !defined(ETHR_UNUSABLE_SIGUSRX)
+#ifdef ERTS_SYS_SUSPEND_SIGNAL
    sys_sigblock(ERTS_SYS_SUSPEND_SIGNAL);
 #endif
 
@@ -1009,7 +1015,7 @@ static void unblock_signals(void)
 #endif /* #ifndef ETHR_UNUSABLE_SIGUSRX */
 #endif /* #ifndef ERTS_SMP */
 
-#if defined(ERTS_SMP) && !defined(ETHR_UNUSABLE_SIGUSRX)
+#ifdef ERTS_SYS_SUSPEND_SIGNAL
    sys_sigrelease(ERTS_SYS_SUSPEND_SIGNAL);
 #endif
 
@@ -3248,12 +3254,14 @@ init_smp_sig_notify(void)
 
 static void
 init_smp_sig_suspend(void) {
+#ifdef ERTS_SYS_SUSPEND_SIGNAL
   if (pipe(sig_suspend_fds) < 0) {
     erts_exit(ERTS_ABORT_EXIT,
 	     "Failed to create sig_suspend pipe: %s (%d)\n",
 	     erl_errno_id(errno),
 	     errno);
   }
+#endif
 }
 
 #ifdef __DARWIN__
-- 
cgit v1.2.3


From ae3e177c514c35483184b0cb52c00b4452ca72ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 16 Aug 2016 08:27:39 +0200
Subject: compiler: Eliminate use of sys_pre_expand

sys_pre_expand previously did a lot more work, for example,
translating records and funs, but now is merely a grab bag
of small transformations. Move those transformations to
v3_core.
---
 erts/emulator/test/match_spec_SUITE.erl | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl
index 6733237b20..34e956bc21 100644
--- a/erts/emulator/test/match_spec_SUITE.erl
+++ b/erts/emulator/test/match_spec_SUITE.erl
@@ -427,13 +427,13 @@ silent_no_ms(Config) when is_list(Config) ->
               %%
               [{trace,Tracee,call,{?MODULE,f1,[start]}},
                {trace,Tracee,return_to,
-                {?MODULE,'-silent_no_ms/1-fun-2-',0}},
+                {?MODULE,'-silent_no_ms/1-fun-3-',0}},
                {trace,Tracee,call,{?MODULE,f2,[f,g]}},
                {trace,Tracee,return_to,
-                {?MODULE,'-silent_no_ms/1-fun-2-',0}},
+                {?MODULE,'-silent_no_ms/1-fun-3-',0}},
                {trace,Tracee,call,{erlang,integer_to_list,[2]}},
                {trace,Tracee,return_to,
-                {?MODULE,'-silent_no_ms/1-fun-2-',0}},
+                {?MODULE,'-silent_no_ms/1-fun-3-',0}},
                {trace,Tracee,call,{?MODULE,f2,[h,i]}},
                {trace,Tracee,return_to,{?MODULE,f3,2}}]
       end).
@@ -484,7 +484,7 @@ ms_trace2(Config) when is_list(Config) ->
               %%
               %% Expected: (no return_to for global call trace)
               %%
-              Origin = {match_spec_SUITE,'-ms_trace2/1-fun-0-',1},
+              Origin = {match_spec_SUITE,'-ms_trace2/1-fun-1-',1},
               [{trace_ts,Tracee,call,
                 {?MODULE,fn,
                  [[all],[call,return_to,{tracer,Tracer}]]},
@@ -574,7 +574,7 @@ ms_trace3(Config) when is_list(Config) ->
               %%
               %% Expected: (no return_to for global call trace)
               %%
-              Origin = {match_spec_SUITE,'-ms_trace3/1-fun-1-',2},
+              Origin = {match_spec_SUITE,'-ms_trace3/1-fun-2-',2},
               [{trace_ts,Controller,call,
                 {?MODULE,fn,[TraceeName,[all],
                              [call,return_to,send,'receive',
-- 
cgit v1.2.3


From d5ba65f0aa25768c1af7a1639eb73298b029eaf3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 23 Aug 2016 07:46:43 +0200
Subject: Slightly reduce code size of process_main()

clang will generate smaller code if we avoid reassigning the
reg and freg arrays. That can be easily arranged by passing them
as arguments to process_main().
---
 erts/emulator/beam/beam_emu.c    | 10 ++++------
 erts/emulator/beam/erl_init.c    |  2 +-
 erts/emulator/beam/erl_process.c |  3 ++-
 erts/emulator/beam/global.h      |  2 +-
 4 files changed, 8 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index c704323bb2..0ba06058a5 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1078,7 +1078,7 @@ static Eterm make_arglist(Process* c_p, Eterm* reg, int a);
 void
 init_emulator(void)
 {
-    process_main();
+    process_main(0, 0);
 }
 
 /*
@@ -1225,7 +1225,7 @@ init_emulator(void)
  * the instructions' C labels to the loader.
  * The second call starts execution of BEAM code. This call never returns.
  */
-void process_main(void)
+void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 {
     static int init_done = 0;
     Process* c_p = NULL;
@@ -1237,7 +1237,7 @@ void process_main(void)
     /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC,
      * in all other cases x0 is used.
      */
-    register Eterm* reg REG_xregs = NULL;
+    register Eterm* reg REG_xregs = x_reg_array;
 
     /*
      * Top of heap (next free location); grows upwards.
@@ -1264,7 +1264,7 @@ void process_main(void)
      * X registers and floating point registers are located in
      * scheduler specific data.
      */
-    register FloatDef *freg;
+    register FloatDef *freg = f_reg_array;
 
     /*
      * For keeping the negative old value of 'reds' when call saving is active.
@@ -1350,8 +1350,6 @@ void process_main(void)
 	start_time_i = c_p->i;
     }
 
-    reg = erts_proc_sched_data(c_p)->x_reg_array;
-    freg = erts_proc_sched_data(c_p)->f_reg_array;
     ERL_BITS_RELOAD_STATEP(c_p);
     {
 	int reds;
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 781bf024dd..2f68ece37b 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -2302,7 +2302,7 @@ erl_start(int argc, char **argv)
 #endif
 	set_main_stack_size();
 	erts_sched_init_time_sup(esdp);
-	process_main();
+	process_main(esdp->x_reg_array, esdp->f_reg_array);
     }
 #endif
 }
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index bcbb9b3115..b577ae95b0 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -8169,7 +8169,8 @@ sched_thread_func(void *vesdp)
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL);
 #endif
 
-    process_main();
+    process_main(esdp->x_reg_array, esdp->f_reg_array);
+
     /* No schedulers should *ever* terminate */
     erts_exit(ERTS_ABORT_EXIT,
 	     "Scheduler thread number %beu terminated\n",
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index d7dd6371b7..3030a095a8 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1200,7 +1200,7 @@ void print_pass_through(int, byte*, int);
 /* beam_emu.c */
 int catchlevel(Process*);
 void init_emulator(void);
-void process_main(void);
+void process_main(Eterm* x_reg_array, FloatDef* f_reg_array);
 void erts_dirty_process_main(ErtsSchedulerData *);
 Eterm build_stacktrace(Process* c_p, Eterm exc);
 Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value);
-- 
cgit v1.2.3


From 986d32a62b20c32338dac4dfd27c141c8f9be0fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 20 Jun 2016 13:25:04 +0200
Subject: Implement the new ceil/1 and floor/1 guard BIFs

Implement as ceil/1 and floor/1 as new guard BIFs (essentially part of
Erlang language). They are guard BIFs because trunc/1 is a guard
BIF. It would be strange to have trunc/1 as a part of the language, but
not ceil/1 and floor/1.
---
 erts/emulator/beam/bif.tab           |  7 +++
 erts/emulator/beam/erl_bif_guard.c   | 65 ++++++++++++++++++++++++++++
 erts/emulator/test/num_bif_SUITE.erl | 84 ++++++++++++++++++++++++++++--------
 3 files changed, 139 insertions(+), 17 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 7a35c02934..67eae35dec 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -658,6 +658,13 @@ bif erlang:has_prepared_code_on_load/1
 
 bif maps:take/2
 
+#
+# New in 20.0
+#
+
+gcbif erlang:floor/1
+gcbif erlang:ceil/1
+
 #
 # Obsolete
 #
diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c
index b42d2dc28b..458315f293 100644
--- a/erts/emulator/beam/erl_bif_guard.c
+++ b/erts/emulator/beam/erl_bif_guard.c
@@ -141,6 +141,39 @@ BIF_RETTYPE trunc_1(BIF_ALIST_1)
     BIF_RET(res);
 }
 
+BIF_RETTYPE floor_1(BIF_ALIST_1)
+{
+    Eterm res;
+    FloatDef f;
+
+    if (is_not_float(BIF_ARG_1)) {
+	if (is_integer(BIF_ARG_1))
+	    BIF_RET(BIF_ARG_1);
+	BIF_ERROR(BIF_P, BADARG);
+    }
+    GET_DOUBLE(BIF_ARG_1, f);
+    res = double_to_integer(BIF_P, floor(f.fd));
+    BIF_RET(res);
+}
+
+BIF_RETTYPE ceil_1(BIF_ALIST_1)
+{
+    Eterm res;
+    FloatDef f;
+
+    /* check arg */
+    if (is_not_float(BIF_ARG_1)) {
+	if (is_integer(BIF_ARG_1))
+	    BIF_RET(BIF_ARG_1);
+	BIF_ERROR(BIF_P, BADARG);
+    }
+    /* get the float */
+    GET_DOUBLE(BIF_ARG_1, f);
+
+    res = double_to_integer(BIF_P, ceil(f.fd));
+    BIF_RET(res);
+}
+
 BIF_RETTYPE round_1(BIF_ALIST_1)
 {
     Eterm res;
@@ -621,6 +654,38 @@ Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live)
 				reg, live);
 }
 
+Eterm erts_gc_floor_1(Process* p, Eterm* reg, Uint live)
+{
+    Eterm arg;
+    FloatDef f;
+
+    arg = reg[live];
+    if (is_not_float(arg)) {
+	if (is_integer(arg))  {
+	    return arg;
+	}
+	BIF_ERROR(p, BADARG);
+    }
+    GET_DOUBLE(arg, f);
+    return gc_double_to_integer(p, floor(f.fd), reg, live);
+}
+
+Eterm erts_gc_ceil_1(Process* p, Eterm* reg, Uint live)
+{
+    Eterm arg;
+    FloatDef f;
+
+    arg = reg[live];
+    if (is_not_float(arg)) {
+	if (is_integer(arg))  {
+	    return arg;
+	}
+	BIF_ERROR(p, BADARG);
+    }
+    GET_DOUBLE(arg, f);
+    return gc_double_to_integer(p, ceil(f.fd), reg, live);
+}
+
 static Eterm
 gc_double_to_integer(Process* p, double x, Eterm* reg, Uint live)
 {
diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl
index d1c9648017..1a1ab0e5e0 100644
--- a/erts/emulator/test/num_bif_SUITE.erl
+++ b/erts/emulator/test/num_bif_SUITE.erl
@@ -32,6 +32,8 @@
 %%	list_to_integer/1
 %%	round/1
 %%	trunc/1
+%%      floor/1
+%%      ceil/1
 %%	integer_to_binary/1
 %%	integer_to_binary/2
 %%	binary_to_integer/1
@@ -41,7 +43,7 @@
 	 t_float_to_string/1, t_integer_to_string/1,
 	 t_string_to_integer/1, t_list_to_integer_edge_cases/1,
 	 t_string_to_float_safe/1, t_string_to_float_risky/1,
-	 t_round/1, t_trunc/1
+	 t_round/1, t_trunc_and_friends/1
      ]).
 
 suite() -> [{ct_hooks,[ts_install_cth]}].
@@ -49,7 +51,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}].
 all() ->
     [t_abs, t_float, t_float_to_string, t_integer_to_string,
      {group, t_string_to_float}, t_string_to_integer, t_round,
-     t_trunc, t_list_to_integer_edge_cases].
+     t_trunc_and_friends, t_list_to_integer_edge_cases].
 
 groups() ->
     [{t_string_to_float, [],
@@ -295,30 +297,78 @@ t_round(Config) when is_list(Config) ->
     -4294967297 = -round(id(4294967296.9)),
     ok.
 
-t_trunc(Config) when is_list(Config) ->
-    0 = trunc(id(0.0)),
-    5 = trunc(id(5.3333)),
-    -10 = trunc(id(-10.978987)),
+%% Test trunc/1, floor/1, ceil/1, and round/1.
+t_trunc_and_friends(_Config) ->
+    MinusZero = 0.0 / (-1.0),
+    0 = trunc_and_friends(MinusZero),
+    0 = trunc_and_friends(0.0),
+    5 = trunc_and_friends(5.3333),
+    -10 = trunc_and_friends(-10.978987),
 
-    % The largest smallnum, converted to float (OTP-3722):
+    %% The largest smallnum, converted to float (OTP-3722):
     X = id((1 bsl 27) - 1),
-    F = id(X + 0.0),
+    F = X + 0.0,
     io:format("X = ~p/~w/~w, F = ~p/~w/~w, trunc(F) = ~p/~w/~w~n",
 	      [X, X, binary_to_list(term_to_binary(X)),
 	       F, F, binary_to_list(term_to_binary(F)),
-	       trunc(F), trunc(F), binary_to_list(term_to_binary(trunc(F)))]),
-    X = trunc(F),
-    X = trunc(F+1)-1,
-    X = trunc(F-1)+1,
-    X = -trunc(-F),
-    X = -trunc(-F-1)-1,
-    X = -trunc(-F+1)+1,
+	       trunc_and_friends(F),
+	       trunc_and_friends(F),
+	       binary_to_list(term_to_binary(trunc_and_friends(F)))]),
+    X = trunc_and_friends(F),
+    X = trunc_and_friends(F+1)-1,
+    X = trunc_and_friends(F-1)+1,
+    X = -trunc_and_friends(-F),
+    X = -trunc_and_friends(-F-1)-1,
+    X = -trunc_and_friends(-F+1)+1,
 
     %% Bignums.
-    4294967305 = trunc(id(4294967305.7)),
-    -4294967305 = trunc(id(-4294967305.7)),
+    4294967305 = trunc_and_friends(4294967305.7),
+    -4294967305 = trunc_and_friends(-4294967305.7),
+    18446744073709551616 = trunc_and_friends(float(1 bsl 64)),
+    -18446744073709551616 = trunc_and_friends(-float(1 bsl 64)),
+
+    %% Random.
+    t_trunc_and_friends_rand(100),
     ok.
 
+t_trunc_and_friends_rand(0) ->
+    ok;
+t_trunc_and_friends_rand(N) ->
+    F0 = rand:uniform() * math:pow(10, 50*rand:normal()),
+    F = case rand:uniform() of
+	    U when U < 0.5 -> -F0;
+	    _ -> F0
+	end,
+    _ = trunc_and_friends(F),
+    t_trunc_and_friends_rand(N-1).
+
+trunc_and_friends(F) ->
+    Trunc = trunc(F),
+    Floor = floor(F),
+    Ceil = ceil(F),
+    Round = round(F),
+
+    Trunc = trunc(Trunc),
+    Floor = floor(Floor),
+    Ceil = ceil(Ceil),
+    Round = round(Round),
+
+    Trunc = trunc(float(Trunc)),
+    Floor = floor(float(Floor)),
+    Ceil = ceil(float(Ceil)),
+    Round = round(float(Round)),
+
+    true = Floor =< Trunc andalso Trunc =< Ceil,
+    true = Ceil - Floor =< 1,
+    true = Round =:= Floor orelse Round =:= Ceil,
+
+    if
+	F < 0 ->
+	    Trunc = Ceil;
+	true ->
+	    Trunc = Floor
+    end,
+    Trunc.
 
 %% Tests integer_to_binary/1.
 
-- 
cgit v1.2.3


From 6d40cfd77f1d2f1e1403e4b41c0b53ae6499ea11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 27 Jun 2016 12:54:04 +0200
Subject: Add math:floor/1 and math:ceil/1

Add math:floor/1 and math:ceil/1 to avoid unnecessary conversions
in floating point expressions. That is, instead of having to write
float(floor(X)) as part of a floating point expressions, we can
write simply math:floor(X).
---
 erts/emulator/beam/bif.tab    |  2 ++
 erts/emulator/beam/erl_math.c | 10 ++++++++--
 2 files changed, 10 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 67eae35dec..0bd80ea684 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -664,6 +664,8 @@ bif maps:take/2
 
 gcbif erlang:floor/1
 gcbif erlang:ceil/1
+bif math:floor/1
+bif math:ceil/1
 
 #
 # Obsolete
diff --git a/erts/emulator/beam/erl_math.c b/erts/emulator/beam/erl_math.c
index fc0aaed18a..990fa63bd4 100644
--- a/erts/emulator/beam/erl_math.c
+++ b/erts/emulator/beam/erl_math.c
@@ -247,6 +247,12 @@ BIF_RETTYPE math_pow_2(BIF_ALIST_2)
     return math_call_2(BIF_P, pow, BIF_ARG_1, BIF_ARG_2);
 }
 
+BIF_RETTYPE math_ceil_1(BIF_ALIST_1)
+{
+    return math_call_1(BIF_P, ceil, BIF_ARG_1);
+}
 
-
-
+BIF_RETTYPE math_floor_1(BIF_ALIST_1)
+{
+    return math_call_1(BIF_P, floor, BIF_ARG_1);
+}
-- 
cgit v1.2.3


From 2a65c850ff50ea5ae0f0e4a239bcb88781dedaf8 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 22 Aug 2016 15:22:06 +0200
Subject: Fix process_SUITE system_task_blast and no_priority_inversion2

---
 erts/emulator/test/process_SUITE.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index dae8990f56..cf43369974 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -2428,7 +2428,7 @@ no_priority_inversion2(Config) when is_list(Config) ->
 
 request_gc(Pid, Prio) ->
     Ref = make_ref(),
-    erts_internal:request_system_task(Pid, Prio, {garbage_collect, Ref}),
+    erts_internal:request_system_task(Pid, Prio, {garbage_collect, Ref, major}),
     Ref.
 
 system_task_blast(Config) when is_list(Config) ->
-- 
cgit v1.2.3


From 7262d8630998517e97ec62a7e0d9cbdd23fdd9c8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 12 Sep 2016 15:05:47 +0200
Subject: erts: Fix trace_nif_SUITE to load nif lib only once

---
 erts/emulator/test/trace_nif_SUITE.erl              | 12 +++++++++---
 erts/emulator/test/trace_nif_SUITE_data/trace_nif.c | 11 ++++++-----
 2 files changed, 15 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/trace_nif_SUITE.erl b/erts/emulator/test/trace_nif_SUITE.erl
index 8d5bff2a48..7ac6fce234 100644
--- a/erts/emulator/test/trace_nif_SUITE.erl
+++ b/erts/emulator/test/trace_nif_SUITE.erl
@@ -265,10 +265,16 @@ nif_process() ->
     nif_process().    
 
 load_nif(Config) ->    
-    Path = proplists:get_value(data_dir, Config),
-
-    ok = erlang:load_nif(filename:join(Path,"trace_nif"), 0).
+    case is_nif_loaded() of
+        true ->
+            ok;
+        false ->
+            Path = proplists:get_value(data_dir, Config),
+            ok = erlang:load_nif(filename:join(Path,"trace_nif"), 0)
+    end.
 
+is_nif_loaded() ->
+    false.
 
 nif() ->
     {"Stub0",[]}. %exit("nif/0 stub called").
diff --git a/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c b/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c
index 26f2420b8b..0c29ff8da8 100644
--- a/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c
+++ b/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c
@@ -6,18 +6,18 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
     return 0;
 }
 
-static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
 {
     return 0;
 }
 
-static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
+static void unload(ErlNifEnv* env, void* priv_data)
 {
-    return 0;
 }
 
-static void unload(ErlNifEnv* env, void* priv_data)
+static ERL_NIF_TERM is_nif_loaded(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
+    return enif_make_atom(env,"true");
 }
 
 static ERL_NIF_TERM nif_0(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
@@ -38,9 +38,10 @@ static ERL_NIF_TERM nif_1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 
 static ErlNifFunc nif_funcs[] =
 {
+    {"is_nif_loaded", 0, is_nif_loaded},
     {"nif", 0, nif_0},
     {"nif", 1, nif_1}
 };
 
-ERL_NIF_INIT(trace_nif_SUITE,nif_funcs,load,reload,upgrade,unload)
+ERL_NIF_INIT(trace_nif_SUITE,nif_funcs,load,NULL,upgrade,unload)
 
-- 
cgit v1.2.3


From 4f8071d81aa0690caa3adc734d73a1cb004ad808 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 19 Sep 2016 14:34:57 +0200
Subject: erts: Remove deprecated nif 'reload' feature

and instead let erlang:load_nif/2 return {error, {reload, _}}
before even trying to load the library
if a NIF library has already been successfully loaded
for the calling module instance.
---
 erts/emulator/beam/erl_nif.c                       | 121 ++++++---------------
 erts/emulator/test/nif_SUITE.erl                   |  39 ++++---
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c      |  10 +-
 erts/emulator/test/nif_SUITE_data/tester.c         |   4 +-
 .../test/trace_call_time_SUITE_data/trace_nif.c    |   7 +-
 5 files changed, 62 insertions(+), 119 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index b53d3d4c19..7442e99cb3 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3145,7 +3145,6 @@ static ErlNifFunc* next_func(ErlNifEntry* entry, int* incrp, ErlNifFunc* func)
 BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 {
     static const char bad_lib[] = "bad_lib";
-    static const char reload[] = "reload";
     static const char upgrade[] = "upgrade";
     char* lib_name = NULL;
     void* handle = NULL;
@@ -3162,7 +3161,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     Eterm ret = am_ok;
     int veto;
     struct erl_module_nif* lib = NULL;
-    int reload_warning = 0;
     struct erl_module_instance* this_mi;
     struct erl_module_instance* prev_mi;
 
@@ -3222,8 +3220,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	this_mi = module_p->on_load;
     }
 
-    if (init_func == NULL &&
-	     (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) {
+    if (this_mi->nif != NULL) {
+        ret = load_nif_error(BIF_P,"reload","NIF library already loaded"
+                             " (reload disallowed since OTP 20).");
+    }
+    else if (init_func == NULL &&
+             (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) {
 	const char slogan[] = "Failed to load NIF library";
 	if (strstr(errdesc.str, lib_name) != NULL) {
 	    ret = load_nif_error(BIF_P, "load_failed", "%s: '%s'", slogan, errdesc.str);
@@ -3312,7 +3314,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	goto error;
     }
 
-    /* Call load, reload or upgrade:
+    /* Call load or upgrade:
      */
 
 
@@ -3324,82 +3326,33 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     ASSERT(opened_rt_list == NULL);
     lib->mod = module_p;
     env.mod_nif = lib;
-    if (this_mi->nif != NULL) { /*************** Reload ******************/
-	/*
-	 * Repeated load_nif calls from same Erlang module instance ("reload")
-	 * is deprecated and was only ment as a development feature not to
-	 * be used in production systems. (See warning below)
-	 */
-	int k, old_incr = 0;
-	ErlNifFunc* old_func;
-        lib->priv_data = this_mi->nif->priv_data;
-
-	ASSERT(this_mi->nif->entry != NULL);
-	if (entry->reload == NULL) {
-	    ret = load_nif_error(BIF_P,reload,"Reload not supported by this NIF library.");
-	    goto error;
-	}
-	/* Check that no NIF is removed */
-	old_func = this_mi->nif->entry->funcs;
-	for (k=0; k < this_mi->nif->entry->num_of_funcs; k++) {
-	    int incr = 0;
-	    ErlNifFunc* f = entry->funcs;
-	    for (i=0; i < entry->num_of_funcs; i++) {
-		if (old_func->arity == f->arity
-		    && sys_strcmp(old_func->name, f->name) == 0) {
-		    break;
-		}
-		f = next_func(entry, &incr, f);
-	    }
-	    if (i == entry->num_of_funcs) {
-		ret = load_nif_error(BIF_P,reload,"Reloaded library missing "
-				     "function %T:%s/%u\r\n", mod_atom,
-				     old_func->name, old_func->arity);
-		goto error;
-	    }
-	    old_func = next_func(this_mi->nif->entry, &old_incr, old_func);
-	}
-	erts_pre_nif(&env, BIF_P, lib, NULL);
-	veto = entry->reload(&env, &lib->priv_data, BIF_ARG_2);
-	erts_post_nif(&env);
-	if (veto) {
-	    ret = load_nif_error(BIF_P, reload, "Library reload-call unsuccessful.");
-	}
-	else {
-	    commit_opened_resource_types(lib);
-	    this_mi->nif->entry = NULL; /* to prevent 'unload' callback */
-	    erts_unload_nif(this_mi->nif);
-	    reload_warning = 1;
-	}
+
+    lib->priv_data = NULL;
+    if (prev_mi->nif != NULL) { /**************** Upgrade ***************/
+        void* prev_old_data = prev_mi->nif->priv_data;
+        if (entry->upgrade == NULL) {
+            ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library.");
+            goto error;
+        }
+        erts_pre_nif(&env, BIF_P, lib, NULL);
+        veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, BIF_ARG_2);
+        erts_post_nif(&env);
+        if (veto) {
+            prev_mi->nif->priv_data = prev_old_data;
+            ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful.");
+        }
+        else
+            commit_opened_resource_types(lib);
     }
-    else {
-	lib->priv_data = NULL;
-	if (prev_mi->nif != NULL) { /**************** Upgrade ***************/
-	    void* prev_old_data = prev_mi->nif->priv_data;
-	    if (entry->upgrade == NULL) {
-		ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library.");
-		goto error;
-	    }
-		erts_pre_nif(&env, BIF_P, lib, NULL);
-	    veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, BIF_ARG_2);
-	    erts_post_nif(&env);
-	    if (veto) {
-		prev_mi->nif->priv_data = prev_old_data;
-		ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful.");
-	    }
-	    else
-		commit_opened_resource_types(lib);
-	}
-	else if (entry->load != NULL) { /********* Initial load ***********/
-	    erts_pre_nif(&env, BIF_P, lib, NULL);
-	    veto = entry->load(&env, &lib->priv_data, BIF_ARG_2);
-	    erts_post_nif(&env);
-	    if (veto) {
-		ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful.");
-	    }
-	    else
-		commit_opened_resource_types(lib);
-	}
+    else if (entry->load != NULL) { /********* Initial load ***********/
+        erts_pre_nif(&env, BIF_P, lib, NULL);
+        veto = entry->load(&env, &lib->priv_data, BIF_ARG_2);
+        erts_post_nif(&env);
+        if (veto) {
+            ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful.");
+        }
+        else
+            commit_opened_resource_types(lib);
     }
     if (ret == am_ok) {
 	/*
@@ -3458,14 +3411,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     erts_release_code_write_permission();
     erts_free(ERTS_ALC_T_TMP, lib_name);
 
-    if (reload_warning) {
-	erts_dsprintf_buf_t* dsbufp = erts_create_logger_dsbuf();
-	erts_dsprintf(dsbufp,
-		      "Repeated calls to erlang:load_nif from module '%T'.\n\n"
-		      "The NIF reload mechanism is deprecated and must not "
-		      "be used in production systems.\n", mod_atom);
-	erts_send_warning_to_logger(BIF_P->group_leader, dsbufp);
-    }
     BIF_RET(ret);
 }
 
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 9c1694fa8a..701b46665c 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -28,7 +28,7 @@
 
 -export([all/0, suite/0,
 	 init_per_testcase/2, end_per_testcase/2,
-         basic/1, reload/1, upgrade/1, heap_frag/1,
+         basic/1, reload_error/1, upgrade/1, heap_frag/1,
          t_on_load/1,
 	 types/1, many_args/1, binaries/1, get_string/1, get_atom/1,
 	 maps/1,
@@ -68,7 +68,7 @@
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
 all() -> 
-    [basic, reload, upgrade, heap_frag, types, many_args,
+    [basic, reload_error, upgrade, heap_frag, types, many_args,
      t_on_load,
      binaries, get_string, get_atom, maps, api_macros, from_array,
      iolist_as_binary, resource, resource_binary,
@@ -112,8 +112,8 @@ basic(Config) when is_list(Config) ->
     true = lists:member(?MODULE, erlang:system_info(taints)),
     ok.
 
-%% Test reload callback in nif lib
-reload(Config) when is_list(Config) ->    
+%% Test old reload feature now always fails
+reload_error(Config) when is_list(Config) ->
     TmpMem = tmpmem(),
     ensure_lib_loaded(Config),
 
@@ -127,20 +127,20 @@ reload(Config) when is_list(Config) ->
     hold_nif_mod_priv_data(nif_mod:get_priv_data_ptr()),
     [{load,1,1,101},{get_priv_data_ptr,1,2,102}] = nif_mod_call_history(),    
         
-    ok = nif_mod:load_nif_lib(Config, 2),
-    2 = nif_mod:lib_version(),
-    [{reload,2,1,201},{lib_version,2,2,202}] = nif_mod_call_history(),    
+    {error, {reload, _}} = nif_mod:load_nif_lib(Config, 2),
+    1 = nif_mod:lib_version(),
+    [{lib_version,1,3,103}] = nif_mod_call_history(),
 
-    ok = nif_mod:load_nif_lib(Config, 1),
+    {error, {reload, _}} = nif_mod:load_nif_lib(Config, 1),
     1 = nif_mod:lib_version(),
-    [{reload,1,1,101},{lib_version,1,2,102}] = nif_mod_call_history(),    
+    [{lib_version,1,4,104}] = nif_mod_call_history(),
 
     true = erlang:delete_module(nif_mod),
     [] = nif_mod_call_history(),    
 
     %%false= check_process_code(Pid, nif_mod),
     true = erlang:purge_module(nif_mod),
-    [{unload,1,3,103}] = nif_mod_call_history(),    
+    [{unload,1,5,105}] = nif_mod_call_history(),
 
     true = lists:member(?MODULE, erlang:system_info(taints)),
     true = lists:member(nif_mod, erlang:system_info(taints)),
@@ -828,7 +828,7 @@ resource_binary_do() ->
 -define(RT_CREATE,1).
 -define(RT_TAKEOVER,2).
 
-%% Test resource takeover by module reload and upgrade
+%% Test resource takeover by module upgrade
 resource_takeover(Config) when is_list(Config) ->    
     TmpMem = tmpmem(),
     ensure_lib_loaded(Config),
@@ -893,6 +893,7 @@ resource_takeover(Config) when is_list(Config) ->
     ok = forget_resource(NGX1),
     ?CHECK([], nif_mod_call_history()), % no dtor
 
+    {module,nif_mod} = erlang:load_module(nif_mod,ModBin),
     ok = nif_mod:load_nif_lib(Config, 2,
                               [{resource_type, 0, ?RT_TAKEOVER, "resource_type_A",resource_dtor_A,
                                 ?RT_TAKEOVER},
@@ -911,7 +912,9 @@ resource_takeover(Config) when is_list(Config) ->
                                {resource_type, 4, ?RT_CREATE, "resource_type_null_goneY",null,
                                 ?RT_CREATE}
                               ]),
-    ?CHECK([{reload,2,1,201}], nif_mod_call_history()),
+    ?CHECK([{upgrade,2,1,201}], nif_mod_call_history()),
+    true = erlang:purge_module(nif_mod),
+    ?CHECK([{unload,1,1,106}], nif_mod_call_history()),
 
     BinA2 = read_resource(0,A2),
     ok = forget_resource(A2),
@@ -1221,11 +1224,19 @@ threading_do(Config) ->
     ok = tester:load_nif_lib(Config, "basic"),   
     ok = tester:run(),
 
+    erlang:load_module(tester,ModBin),
+    erlang:purge_module(tester),
     ok = tester:load_nif_lib(Config, "rwlock"),
     ok = tester:run(),
 
+    erlang:load_module(tester,ModBin),
+    erlang:purge_module(tester),
     ok = tester:load_nif_lib(Config, "tsd"),
-    ok = tester:run().
+    ok = tester:run(),
+
+    erlang:delete_module(tester),
+    erlang:purge_module(tester).
+
 
 %% Test NIF message sending
 send(Config) when is_list(Config) ->    
@@ -1513,13 +1524,13 @@ send3_new_state(State, Blob) ->
 neg(Config) when is_list(Config) ->
     TmpMem = tmpmem(),
     {'EXIT',{badarg,_}} = (catch erlang:load_nif(badarg, 0)),
-    {error,{load_failed,_}} = erlang:load_nif("pink_unicorn", 0),
     
     Data = proplists:get_value(data_dir, Config),
     File = filename:join(Data, "nif_mod"),
     {ok,nif_mod,Bin} = compile:file(File, [binary,return_errors]),
     {module,nif_mod} = erlang:load_module(nif_mod,Bin),
 
+    {error,{load_failed,_}} = nif_mod:load_nif_lib(Config, 0),
     {error,{bad_lib,_}} = nif_mod:load_nif_lib(Config, no_init),    
     verify_tmpmem(TmpMem),
     ok.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index f2b1ef9d24..99534a9694 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -184,14 +184,6 @@ static void resource_takeover(ErlNifEnv* env, PrivData* priv)
     msgenv_resource_type = rt;
 }
 
-static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
-{
-    PrivData* priv = (PrivData*) *priv_data;
-    add_call(env, priv, "reload");
-    resource_takeover(env,priv);
-    return 0;
-}
-
 static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
 {
     PrivData* priv = (PrivData*) *old_priv_data;
@@ -2094,4 +2086,4 @@ static ErlNifFunc nif_funcs[] =
     {"format_term_nif", 2, format_term}
 };
 
-ERL_NIF_INIT(nif_SUITE,nif_funcs,load,reload,upgrade,unload)
+ERL_NIF_INIT(nif_SUITE,nif_funcs,load,NULL,upgrade,unload)
diff --git a/erts/emulator/test/nif_SUITE_data/tester.c b/erts/emulator/test/nif_SUITE_data/tester.c
index 257b116322..9e0074d554 100644
--- a/erts/emulator/test/nif_SUITE_data/tester.c
+++ b/erts/emulator/test/nif_SUITE_data/tester.c
@@ -53,7 +53,7 @@ void testcase_free(void *ptr)
 
 void testcase_run(TestCaseState_t *tcs);
 
-static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
 {
     return 0;
 }
@@ -70,5 +70,5 @@ static ErlNifFunc nif_funcs[] =
     {"run", 0, run}
 };
 
-ERL_NIF_INIT(tester,nif_funcs,NULL,reload,NULL,NULL)
+ERL_NIF_INIT(tester,nif_funcs,NULL,NULL,upgrade,NULL)
 
diff --git a/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c
index 33b346aab7..0540c25cb9 100644
--- a/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c
+++ b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c
@@ -6,11 +6,6 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
     return 0;
 }
 
-static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
-{
-    return 0;
-}
-
 static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
 {
     return 0;
@@ -34,4 +29,4 @@ static ErlNifFunc nif_funcs[] =
     {"nif_dec", 1, nif_dec_1}
 };
 
-ERL_NIF_INIT(trace_call_time_SUITE,nif_funcs,load,reload,upgrade,unload)
+ERL_NIF_INIT(trace_call_time_SUITE,nif_funcs,load,NULL,upgrade,unload)
-- 
cgit v1.2.3


From a54773f05cd60045a4ac0f9a4ae753aa76185b61 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 9 Sep 2016 16:15:52 +0200
Subject: erts: Cuddle nif_SUITE:consume_timeslice

---
 erts/emulator/test/nif_SUITE.erl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 701b46665c..306f2091a1 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -1651,6 +1651,7 @@ consume_timeslice(Config) when is_list(Config) ->
     end.
 
 consume_timeslice_test(Config) when is_list(Config) ->
+    ensure_lib_loaded(Config),
     CONTEXT_REDS = 2000,
     Me = self(),
     Go = make_ref(),
@@ -1727,7 +1728,7 @@ consume_timeslice_test(Config) when is_list(Config) ->
 	    io:format("Reductions = ~p~n", [Reductions]),
 	    ok;
 	{RedDiff, Reductions} ->
-	    ct:fail({unexpected_reduction_count, Reductions})
+	    ct:fail({unexpected_reduction_count, Reductions, ExpReds})
     end,
     
     none = next_msg(P),
-- 
cgit v1.2.3


From db12deff975c013a0ac02cfb49a339f8e6af5938 Mon Sep 17 00:00:00 2001
From: Luis Rascao <luis.rascao@gmail.com>
Date: Sat, 10 Sep 2016 21:02:47 +0200
Subject: Allow for easier enabling of distribution debug

Also update already deprecated calls.
---
 erts/emulator/beam/dist.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 09c83f1117..9bfeaa3379 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -25,6 +25,7 @@
 
 /* define this to get a lot of debug output */
 /* #define ERTS_DIST_MSG_DBG */
+/* #define ERTS_RAW_DIST_MSG_DBG */
 
 #ifdef HAVE_CONFIG_H
 #  include "config.h"
-- 
cgit v1.2.3


From f1ffa5e90e5eecaac890e876760932d1bb1d9c86 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 19 Sep 2016 15:47:18 +0200
Subject: erts: Add ErtsSizeofMember macro

(in case it matters)
---
 erts/emulator/beam/sys.h | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 2d0628f70e..16e6c33367 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -103,6 +103,8 @@
 #define ErtsContainerStruct_(ptr, type, memberv) \
     ((type *)((char *)(1 ? (ptr) : ((type *)0)->memberv) - offsetof(type, memberv)))
 
+#define ErtsSizeofMember(type, member) sizeof(((type *)0)->member)
+
 #if defined (__WIN32__)
 #  include "erl_win_sys.h"
 #else
-- 
cgit v1.2.3


From ff7eb12d002afbffacae5429bd9bb0819aa2d146 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 1 Jun 2016 16:19:04 +0200
Subject: erts: Remove unnecessary access of 'is_resizing'

in tables without write_concurrency
and remove it totally #ifndef ERTS_SMP
---
 erts/emulator/beam/erl_db_hash.c | 21 ++++++++++-----------
 erts/emulator/beam/erl_db_hash.h |  2 +-
 2 files changed, 11 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 12ae086b31..16d97f05ee 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -670,8 +670,8 @@ int db_create_hash(Process *p, DbTable *tbl)
     tb->nsegs = NSEG_1;
     tb->nslots = SEGSZ;
 
-    erts_smp_atomic_init_nob(&tb->is_resizing, 0);
 #ifdef ERTS_SMP
+    erts_smp_atomic_init_nob(&tb->is_resizing, 0);
     if (tb->common.type & DB_FINE_LOCKED) {
 	erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
 	int i;
@@ -2604,23 +2604,22 @@ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2,
 static ERTS_INLINE int
 begin_resizing(DbTableHash* tb)
 {
+#ifdef ERTS_SMP
     if (DB_USING_FINE_LOCKING(tb))
-	return !erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1);
-    else {
-	if (erts_smp_atomic_read_nob(&tb->is_resizing))
-	    return 0;
-	erts_smp_atomic_set_nob(&tb->is_resizing, 1);
-	return 1;
-    }
+	return !erts_atomic_xchg_acqb(&tb->is_resizing, 1);
+    else
+        ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock));
+#endif
+    return 1;
 }
 
 static ERTS_INLINE void
 done_resizing(DbTableHash* tb)
 {
+#ifdef ERTS_SMP
     if (DB_USING_FINE_LOCKING(tb))
-	erts_smp_atomic_set_relb(&tb->is_resizing, 0);
-    else
-	erts_smp_atomic_set_nob(&tb->is_resizing, 0);
+	erts_atomic_set_relb(&tb->is_resizing, 0);
+#endif
 }
 
 /* Grow table with one new bucket.
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index e654363cd5..081ff8fafc 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -60,8 +60,8 @@ typedef struct db_table_hash {
     /* List of slots where elements have been deleted while table was fixed */
     erts_smp_atomic_t fixdel;  /* (FixedDeletion*) */	
     erts_smp_atomic_t nactive; /* Number of "active" slots */
-    erts_smp_atomic_t is_resizing; /* grow/shrink in progress */
 #ifdef ERTS_SMP
+    erts_smp_atomic_t is_resizing; /* grow/shrink in progress */
     DbTableHashFineLocks* locks;
 #endif
 #ifdef VALGRIND
-- 
cgit v1.2.3


From abc2f4fdae2d62b5d2843082dbb4437595973b38 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 23 Jun 2016 19:30:37 +0200
Subject: erts: Redesign ets with separate segment tables

* Keep it simple(r)
* To prepare for both dynamic sized segments and segtabs
---
 erts/emulator/beam/erl_db.h      |   2 -
 erts/emulator/beam/erl_db_hash.c | 246 +++++++++++++++------------------------
 erts/emulator/beam/erl_db_hash.h |   4 +-
 3 files changed, 97 insertions(+), 155 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index 1d26c49652..2f3c4a8e1b 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -267,7 +267,5 @@ erts_db_free_nt(ErtsAlcType_t type, void *ptr, Uint size)
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-#undef ERTS_DB_ALC_MEM_UPDATE_
-
 #endif /* #if defined(ERTS_WANT_DB_INTERNAL__) && !defined(ERTS_HAVE_DB_INTERNAL__) */
 
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 16d97f05ee..82f03e458a 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -84,14 +84,6 @@
 
 #include "erl_db_hash.h"
 
-#ifdef MYDEBUG /* Will fail test case ets_SUITE:memory */
-#  define IF_DEBUG(x) x
-#  define MY_ASSERT(x) ASSERT(x)
-#else
-#  define IF_DEBUG(x)
-#  define MY_ASSERT(x)
-#endif
-
 /* 
  * The following symbols can be manipulated to "tune" the linear hash array 
  */
@@ -102,7 +94,7 @@
 #define SEGSZ   (1 << SEGSZ_EXP)
 #define SEGSZ_MASK (SEGSZ-1)
 
-#define NSEG_1     2 /* Size of first segment table (must be at least 2) */ 
+#define NSEG_1   (ErtsSizeofMember(DbTableHash,first_segtab) / sizeof(struct segment*))
 #define NSEG_2   256 /* Size of second segment table */
 #define NSEG_INC 128 /* Number of segments to grow after that */
 
@@ -318,27 +310,21 @@ struct mp_info {
 
 /* A table segment */
 struct segment {
-    HashDbTerm* buckets[SEGSZ];
-#ifdef MYDEBUG
-    int is_ext_segment;
-#endif
+    HashDbTerm* buckets[1];
 };
+#define SIZEOF_SEGMENT(N) \
+    (offsetof(struct segment,buckets) + sizeof(HashDbTerm*)*(N))
 
-/* A segment that also contains a segment table */
-struct ext_segment {    
-    struct segment s; /* The segment itself. Must be first */
-	
+/* An extended segment table */
+struct ext_segtab {
+    ErtsThrPrgrLaterOp lop;
     struct segment** prev_segtab;  /* Used when table is shrinking */
-    int nsegs;                     /* Size of segtab */
+    int prev_nsegs;                /* Size of prev_segtab */
+    int nsegs;                     /* Size of this segtab */
     struct segment* segtab[1];     /* The segment table */
 };
-#define SIZEOF_EXTSEG(NSEGS) \
-    (offsetof(struct ext_segment,segtab) + sizeof(struct segment*)*(NSEGS))
-
-#if defined(DEBUG) || defined(VALGRIND)
-#  define EXTSEG(SEGTAB_PTR) \
-    ((struct ext_segment*) (((char*)(SEGTAB_PTR)) - offsetof(struct ext_segment,segtab)))
-#endif
+#define SIZEOF_EXT_SEGTAB(NSEGS) \
+    (offsetof(struct ext_segtab,segtab) + sizeof(struct segment*)*(NSEGS))
 
 
 static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb,
@@ -348,41 +334,14 @@ static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb,
 	erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t) segtab);
     else
 	erts_smp_atomic_set_nob(&tb->segtab, (erts_aint_t) segtab);
-#ifdef VALGRIND
-    tb->top_ptr_to_segment_with_active_segtab = EXTSEG(segtab);
-#endif
 }
 
 
-/* How the table segments relate to each other:
-
-    ext_segment:                      ext_segment:              "plain" segment
-   #=================#                #================#        #=============#
-   | bucket[0]       |<--+   +------->| bucket[256]    |     +->| bucket[512] |
-   | bucket[1]       |   |   |        |       [257]    |     |  |       [513] |
-   :                 :   |   |        :                :     |  :             :
-   | bucket[255]     |   |   |        |       [511]    |     |  |       [767] |
-   |-----------------|   |   |        |----------------|     |  #=============#
-   | prev_segtab=NULL|   |   |   +--<---prev_segtab    |     |
-   | nsegs = 2       |   |   |   |    | nsegs = 256    |     |
-+->| segtab[0] -->-------+---|---|--<---segtab[0]      |<-+  |
-|  | segtab[1] -->-----------+---|--<---segtab[1]      |  |  |
-|  #=================#           |    | segtab[2] -->-----|--+    ext_segment:         
-|                                |    :                :  |      #================#
-+----------------<---------------+    | segtab[255] ->----|----->| bucket[255*256]| 
-                                      #================#  |      |                | 
-                                                          |      :                :
-                                                          |      |----------------| 
-                                                          +----<---prev_segtab    | 
-                                                                 :                :
-*/
-
 
 /*
 ** Forward decl's (static functions)
 */
-static struct ext_segment* alloc_ext_seg(DbTableHash* tb, unsigned seg_ix,
-					 struct segment** old_segtab);
+static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix);
 static int alloc_seg(DbTableHash *tb);
 static int free_seg(DbTableHash *tb, int free_records);
 static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_smp_rwmtx_t** lck_ptr,
@@ -666,9 +625,13 @@ int db_create_hash(Process *p, DbTable *tbl)
     erts_smp_atomic_init_nob(&tb->nactive, SEGSZ);
     erts_smp_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL);
     erts_smp_atomic_init_nob(&tb->segtab, (erts_aint_t)NULL);
-    SET_SEGTAB(tb, alloc_ext_seg(tb,0,NULL)->segtab);
+    SET_SEGTAB(tb, tb->first_segtab);
     tb->nsegs = NSEG_1;
     tb->nslots = SEGSZ;
+    tb->first_segtab[0] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG,
+                                                          (DbTable *) tb,
+                                                          SIZEOF_SEGMENT(SEGSZ));
+    sys_memset(tb->first_segtab[0], 0, SIZEOF_SEGMENT(SEGSZ));
 
 #ifdef ERTS_SMP
     erts_smp_atomic_init_nob(&tb->is_resizing, 0);
@@ -2414,34 +2377,29 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern,
     return DB_ERROR_NONE;
 }
 
-static struct ext_segment* alloc_ext_seg(DbTableHash* tb, unsigned seg_ix,
-					 struct segment** old_segtab)
+static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix)
 {
-    int nsegs;
-    struct ext_segment* eseg;
+    struct segment** old_segtab = SEGTAB(tb);
+    int nsegs = 0;
+    struct ext_segtab* est;
     
+    ASSERT(seg_ix >= NSEG_1);
     switch (seg_ix) {
-    case 0: nsegs = NSEG_1; break;
-    case 1: nsegs = NSEG_2; break; 
-    default: nsegs = seg_ix + NSEG_INC; break;
-    }    
-    eseg = (struct ext_segment*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG,
-						   (DbTable *) tb,
-						   SIZEOF_EXTSEG(nsegs));
-    ASSERT(eseg != NULL);	
-    sys_memset(&eseg->s, 0, sizeof(struct segment));
-    IF_DEBUG(eseg->s.is_ext_segment = 1);
-    eseg->prev_segtab = old_segtab;
-    eseg->nsegs = nsegs;
-    if (old_segtab) {
-	ASSERT(nsegs > tb->nsegs);
-	sys_memcpy(eseg->segtab, old_segtab, tb->nsegs*sizeof(struct segment*));
-    }
+    case NSEG_1: nsegs = NSEG_2; break;
+    default:     nsegs = seg_ix + NSEG_INC; break;
+    }
+    ASSERT(nsegs > tb->nsegs);
+    est = (struct ext_segtab*) erts_db_alloc(ERTS_ALC_T_DB_SEG,
+                                             (DbTable *) tb,
+                                             SIZEOF_EXT_SEGTAB(nsegs));
+    est->nsegs = nsegs;
+    est->prev_segtab = old_segtab;
+    est->prev_nsegs = tb->nsegs;
+    sys_memcpy(est->segtab, old_segtab, tb->nsegs*sizeof(struct segment*));
 #ifdef DEBUG
-    sys_memset(&eseg->segtab[seg_ix], 0, (nsegs-seg_ix)*sizeof(struct segment*));
+    sys_memset(&est->segtab[seg_ix], 0, (nsegs-seg_ix)*sizeof(struct segment*));
 #endif
-    eseg->segtab[seg_ix] = &eseg->s;
-    return eseg;
+    return est;
 }
 
 /* Extend table with one new segment
@@ -2449,36 +2407,32 @@ static struct ext_segment* alloc_ext_seg(DbTableHash* tb, unsigned seg_ix,
 static int alloc_seg(DbTableHash *tb)
 {    
     int seg_ix = tb->nslots >> SEGSZ_EXP;
-
-    if (seg_ix+1 == tb->nsegs) { /* New segtab needed (extended segment) */
-	struct segment** segtab = SEGTAB(tb);
-	struct ext_segment* seg = alloc_ext_seg(tb, seg_ix, segtab);
-    	if (seg == NULL) return 0;
-	segtab[seg_ix] = &seg->s;
-	/* We don't use the new segtab until next call (see "shrink race") */
-    }
-    else { /* Just a new plain segment */
-	struct segment** segtab;
-	if (seg_ix == tb->nsegs) { /* Time to start use segtab from last call */
-	    struct ext_segment* eseg;
-	    eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1];
-	    MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment);
-	    SET_SEGTAB(tb, eseg->segtab);	    
-	    tb->nsegs = eseg->nsegs;
-	}
-	ASSERT(seg_ix < tb->nsegs);
-	segtab = SEGTAB(tb);
-	ASSERT(segtab[seg_ix] == NULL);
-	segtab[seg_ix] = (struct segment*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG,
-							     (DbTable *) tb,
-							     sizeof(struct segment));
-	if (segtab[seg_ix] == NULL) return 0;
-	sys_memset(segtab[seg_ix], 0, sizeof(struct segment));
-    }
+    struct segment** segtab;
+
+    if (seg_ix == tb->nsegs) { /* New segtab needed */
+	struct ext_segtab* est = alloc_ext_segtab(tb, seg_ix);
+        SET_SEGTAB(tb, est->segtab);
+        tb->nsegs = est->nsegs;
+    }
+    ASSERT(seg_ix < tb->nsegs);
+    segtab = SEGTAB(tb);
+    segtab[seg_ix] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG,
+                                                     (DbTable *) tb,
+                                                     SIZEOF_SEGMENT(SEGSZ));
+    sys_memset(segtab[seg_ix], 0, SIZEOF_SEGMENT(SEGSZ));
     tb->nslots += SEGSZ;
     return 1;
 }
 
+#ifdef ERTS_SMP
+static void dealloc_ext_segtab(void* lop_data)
+{
+    struct ext_segtab* est = (struct ext_segtab*) lop_data;
+
+    erts_free(ERTS_ALC_T_DB_SEG, est);
+}
+#endif
+
 /* Shrink table by freeing the top segment
 ** free_records: 1=free any records in segment, 0=assume segment is empty 
 */
@@ -2486,18 +2440,17 @@ static int free_seg(DbTableHash *tb, int free_records)
 {
     const int seg_ix = (tb->nslots >> SEGSZ_EXP) - 1;
     struct segment** const segtab = SEGTAB(tb);
-    struct ext_segment* const top = (struct ext_segment*) segtab[seg_ix];
-    int bytes;
+    struct segment* const segp = segtab[seg_ix];
     int nrecords = 0;
 
-    ASSERT(top != NULL); 
+    ASSERT(segp != NULL);
 #ifndef DEBUG
     if (free_records)
 #endif
     {	
 	int i;
 	for (i=0; i<SEGSZ; ++i) {
-	    HashDbTerm* p = top->s.buckets[i];	    
+	    HashDbTerm* p = segp->buckets[i];
 	    while(p != 0) {		
 		HashDbTerm* nxt = p->next;
 		ASSERT(free_records); /* segment not empty as assumed? */
@@ -2507,53 +2460,46 @@ static int free_seg(DbTableHash *tb, int free_records)
 	    }
 	}
     }
-
-    /* The "shrink race":
-     * We must avoid deallocating an extended segment while its segtab may
-     * still be used by other threads.
-     * The trick is to stop use a segtab one call earlier. That is, stop use
-     * a segtab when the segment above it is deallocated. When the segtab is
-     * later deallocated, it has not been used for a very long time.
-     * It is even theoretically safe as we have by then rehashed the entire
-     * segment, seizing *all* locks, so there cannot exist any retarded threads
-     * still hanging in BUCKET macro with an old segtab pointer.
-     * For this to work, we must of course allocate a new segtab one call
-     * earlier in alloc_seg() as well. And this is also the reason why
-     * the minimum size of the first segtab is 2 and not 1 (NSEG_1).
-     */
     
-    if (seg_ix == tb->nsegs-1 || seg_ix==0) { /* Dealloc extended segment */
-	MY_ASSERT(top->s.is_ext_segment);   
-    	ASSERT(segtab != top->segtab || seg_ix==0);    
-	bytes = SIZEOF_EXTSEG(top->nsegs);
-    }
-    else { /* Dealloc plain segment */
-	struct ext_segment* newtop = (struct ext_segment*) segtab[seg_ix-1];
-	MY_ASSERT(!top->s.is_ext_segment);
-	
-	if (segtab == newtop->segtab) { /* New top segment is extended */
-	    MY_ASSERT(newtop->s.is_ext_segment);
-	    if (newtop->prev_segtab != NULL) {
-		/* Time to use a smaller segtab */
-		SET_SEGTAB(tb, newtop->prev_segtab);
-		tb->nsegs = seg_ix;
-		ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs);
-	    }
-	    else {
-		ASSERT(NSEG_1 > 2 && seg_ix==1);
-	    }
-	}
-	bytes = sizeof(struct segment);
+    if (seg_ix >= NSEG_1) {
+        struct ext_segtab* est = ErtsContainerStruct_(segtab,struct ext_segtab,segtab);
+
+        if (seg_ix == est->prev_nsegs) { /* Dealloc extended segtab */
+            ASSERT(est->prev_segtab != NULL);
+            SET_SEGTAB(tb, est->prev_segtab);
+            tb->nsegs = est->prev_nsegs;
+
+#ifdef ERTS_SMP
+            if (!tb->common.is_thread_safe) {
+                /*
+                 * Table is doing a graceful shrink operation and we must avoid
+                 * deallocating this segtab while it may still be read by other
+                 * threads. Schedule deallocation with thread progress to make
+                 * sure no lingering threads are still hanging in BUCKET macro
+                 * with an old segtab pointer.
+                 */
+                Uint sz = SIZEOF_EXT_SEGTAB(est->nsegs);
+                ASSERT(sz == ERTS_ALC_DBG_BLK_SZ(est));
+                ERTS_DB_ALC_MEM_UPDATE_(tb, sz, 0);
+                erts_schedule_thr_prgr_later_cleanup_op(dealloc_ext_segtab,
+                                                        est,
+                                                        &est->lop,
+                                                        sz);
+            }
+            else
+#endif
+                erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable*)tb, est,
+                             SIZEOF_EXT_SEGTAB(est->nsegs));
+        }
+        else {
+
+        }
     }
+    erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, segp, SIZEOF_SEGMENT(SEGSZ));
     
-    erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb,
-		 (void*)top, bytes);
 #ifdef DEBUG
-    if (seg_ix > 0) {
-        segtab[seg_ix] = NULL;
-    } else {
-	SET_SEGTAB(tb, NULL);
-    }
+    if (seg_ix < tb->nsegs)
+        SEGTAB(tb)[seg_ix] = NULL;
 #endif
     tb->nslots -= SEGSZ;
     ASSERT(tb->nslots >= 0);
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index 081ff8fafc..2d9b5e308a 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -51,6 +51,7 @@ typedef struct db_table_hash {
     DbTableCommon common;
 
     erts_smp_atomic_t segtab;  /* The segment table (struct segment**) */
+    struct segment* first_segtab[1];
     erts_smp_atomic_t szm;     /* current size mask. */
     
     /* SMP: nslots and nsegs are protected by is_resizing or table write lock */
@@ -64,9 +65,6 @@ typedef struct db_table_hash {
     erts_smp_atomic_t is_resizing; /* grow/shrink in progress */
     DbTableHashFineLocks* locks;
 #endif
-#ifdef VALGRIND
-    struct ext_segment* top_ptr_to_segment_with_active_segtab;
-#endif
 } DbTableHash;
 
 
-- 
cgit v1.2.3


From 0d7a001039dbcab096397f27213b518113a9e5d0 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 24 Aug 2016 16:28:53 +0200
Subject: erts: Enable a smaller first hash segment for ets

---
 erts/emulator/beam/erl_db_hash.c | 67 ++++++++++++++++++++++++----------------
 1 file changed, 41 insertions(+), 26 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 82f03e458a..2d24f438ca 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -89,10 +89,20 @@
  */
 #define CHAIN_LEN 6                 /* Medium bucket chain len      */
 
-/* Number of slots per segment */
-#define SEGSZ_EXP  8
-#define SEGSZ   (1 << SEGSZ_EXP)
-#define SEGSZ_MASK (SEGSZ-1)
+/*
+** We want the first mandatory segment to be small (to reduce minimal footprint)
+** and larger extra segments (to reduce number of alloc/free calls).
+*/
+
+/* Number of slots in first segment */
+#define FIRST_SEGSZ_EXP  8
+#define FIRST_SEGSZ      (1 << FIRST_SEGSZ_EXP)
+#define FIRST_SEGSZ_MASK (FIRST_SEGSZ - 1)
+
+/* Number of slots per extra segment */
+#define EXT_SEGSZ_EXP  11
+#define EXT_SEGSZ   (1 << EXT_SEGSZ_EXP)
+#define EXT_SEGSZ_MASK (EXT_SEGSZ-1)
 
 #define NSEG_1   (ErtsSizeofMember(DbTableHash,first_segtab) / sizeof(struct segment*))
 #define NSEG_2   256 /* Size of second segment table */
@@ -115,7 +125,9 @@
 #define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive))
 #define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems))
 
-#define BUCKET(tb, i) SEGTAB(tb)[(i) >> SEGSZ_EXP]->buckets[(i) & SEGSZ_MASK]
+#define SLOT_IX_TO_SEG_IX(i) (((i)+(EXT_SEGSZ-FIRST_SEGSZ)) >> EXT_SEGSZ_EXP)
+
+#define BUCKET(tb, i) SEGTAB(tb)[SLOT_IX_TO_SEG_IX(i)]->buckets[(i) & EXT_SEGSZ_MASK]
 
 /*
  * When deleting a table, the number of records to delete.
@@ -422,7 +434,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle);
 static ERTS_INLINE void try_shrink(DbTableHash* tb)
 {
     int nactive = NACTIVE(tb);
-    if (nactive > SEGSZ && NITEMS(tb) < (nactive * CHAIN_LEN)
+    if (nactive > FIRST_SEGSZ && NITEMS(tb) < (nactive * CHAIN_LEN)
 	&& !IS_FIXED(tb)) {
 	shrink(tb, nactive);
     }
@@ -621,17 +633,17 @@ int db_create_hash(Process *p, DbTable *tbl)
 {
     DbTableHash *tb = &tbl->hash;
 
-    erts_smp_atomic_init_nob(&tb->szm, SEGSZ_MASK);
-    erts_smp_atomic_init_nob(&tb->nactive, SEGSZ);
+    erts_smp_atomic_init_nob(&tb->szm, FIRST_SEGSZ_MASK);
+    erts_smp_atomic_init_nob(&tb->nactive, FIRST_SEGSZ);
     erts_smp_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL);
     erts_smp_atomic_init_nob(&tb->segtab, (erts_aint_t)NULL);
     SET_SEGTAB(tb, tb->first_segtab);
     tb->nsegs = NSEG_1;
-    tb->nslots = SEGSZ;
+    tb->nslots = FIRST_SEGSZ;
     tb->first_segtab[0] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG,
                                                           (DbTable *) tb,
-                                                          SIZEOF_SEGMENT(SEGSZ));
-    sys_memset(tb->first_segtab[0], 0, SIZEOF_SEGMENT(SEGSZ));
+                                                          SIZEOF_SEGMENT(FIRST_SEGSZ));
+    sys_memset(tb->first_segtab[0], 0, SIZEOF_SEGMENT(FIRST_SEGSZ));
 
 #ifdef ERTS_SMP
     erts_smp_atomic_init_nob(&tb->is_resizing, 0);
@@ -649,7 +661,7 @@ int db_create_hash(Process *p, DbTable *tbl)
 	    erts_smp_rwmtx_init_opt_x(&tb->locks->lck_vec[i].lck, &rwmtx_opt,
 				      "db_hash_slot", make_small(i));
 	}
-	/* This important property is needed to guarantee that the buckets
+	/* This important property is needed to guarantee the two buckets
     	 * involved in a grow/shrink operation it protected by the same lock:
 	 */
 	ASSERT(erts_smp_atomic_read_nob(&tb->nactive) % DB_HASH_LOCK_CNT == 0);
@@ -2213,12 +2225,12 @@ static int db_free_table_continue_hash(DbTable *tbl)
 
     done /= 2;
     while(tb->nslots != 0) {
-	free_seg(tb, 1);
+	done += 1 + EXT_SEGSZ/64 + free_seg(tb, 1);
 
 	/*
 	 * If we have done enough work, get out here.
 	 */
-	if (++done >= (DELETE_RECORD_LIMIT / CHAIN_LEN / SEGSZ)) {
+	if (done >= DELETE_RECORD_LIMIT) {
 	    return 0;	/* Not done */
 	}
     }
@@ -2406,9 +2418,10 @@ static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix)
 */
 static int alloc_seg(DbTableHash *tb)
 {    
-    int seg_ix = tb->nslots >> SEGSZ_EXP;
+    int seg_ix = SLOT_IX_TO_SEG_IX(tb->nslots);
     struct segment** segtab;
 
+    ASSERT(seg_ix > 0);
     if (seg_ix == tb->nsegs) { /* New segtab needed */
 	struct ext_segtab* est = alloc_ext_segtab(tb, seg_ix);
         SET_SEGTAB(tb, est->segtab);
@@ -2418,9 +2431,9 @@ static int alloc_seg(DbTableHash *tb)
     segtab = SEGTAB(tb);
     segtab[seg_ix] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG,
                                                      (DbTable *) tb,
-                                                     SIZEOF_SEGMENT(SEGSZ));
-    sys_memset(segtab[seg_ix], 0, SIZEOF_SEGMENT(SEGSZ));
-    tb->nslots += SEGSZ;
+                                                     SIZEOF_SEGMENT(EXT_SEGSZ));
+    sys_memset(segtab[seg_ix], 0, SIZEOF_SEGMENT(EXT_SEGSZ));
+    tb->nslots += EXT_SEGSZ;
     return 1;
 }
 
@@ -2438,9 +2451,10 @@ static void dealloc_ext_segtab(void* lop_data)
 */
 static int free_seg(DbTableHash *tb, int free_records)
 {
-    const int seg_ix = (tb->nslots >> SEGSZ_EXP) - 1;
+    const int seg_ix = SLOT_IX_TO_SEG_IX(tb->nslots) - 1;
     struct segment** const segtab = SEGTAB(tb);
     struct segment* const segp = segtab[seg_ix];
+    Uint seg_sz;
     int nrecords = 0;
 
     ASSERT(segp != NULL);
@@ -2448,8 +2462,8 @@ static int free_seg(DbTableHash *tb, int free_records)
     if (free_records)
 #endif
     {	
-	int i;
-	for (i=0; i<SEGSZ; ++i) {
+	int i = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ;
+	while (i--) {
 	    HashDbTerm* p = segp->buckets[i];
 	    while(p != 0) {		
 		HashDbTerm* nxt = p->next;
@@ -2495,13 +2509,14 @@ static int free_seg(DbTableHash *tb, int free_records)
 
         }
     }
-    erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, segp, SIZEOF_SEGMENT(SEGSZ));
+    seg_sz = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ;
+    erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, segp, SIZEOF_SEGMENT(seg_sz));
     
 #ifdef DEBUG
     if (seg_ix < tb->nsegs)
         SEGTAB(tb)[seg_ix] = NULL;
 #endif
-    tb->nslots -= SEGSZ;
+    tb->nslots -= seg_sz;
     ASSERT(tb->nslots >= 0);
     return nrecords;
 }
@@ -2589,7 +2604,7 @@ static void grow(DbTableHash* tb, int nactive)
     /* Ensure that the slot nactive exists */
     if (nactive == tb->nslots) {
 	/* Time to get a new segment */    
-	ASSERT((nactive & SEGSZ_MASK) == 0);
+        ASSERT(((nactive-FIRST_SEGSZ) & EXT_SEGSZ_MASK) == 0);
 	if (!alloc_seg(tb)) goto abort;	    
     }
     ASSERT(nactive < tb->nslots);
@@ -2668,7 +2683,7 @@ static void shrink(DbTableHash* tb, int nactive)
 	int dst_ix = src_ix & low_szm;
 
 	ASSERT(dst_ix < src_ix);
-	ASSERT(nactive > SEGSZ);
+	ASSERT(nactive > FIRST_SEGSZ);
 	lck = WLOCK_HASH(tb, dst_ix);
 	/* Double check for racing table fixers */
 	if (!IS_FIXED(tb)) {
@@ -2697,7 +2712,7 @@ static void shrink(DbTableHash* tb, int nactive)
 	    }
 	    WUNLOCK_HASH(lck);
 	    
-	    if (tb->nslots - src_ix >= SEGSZ) {
+	    if (tb->nslots - src_ix >= EXT_SEGSZ) {
 		free_seg(tb, 0);
 	    }
 	}
-- 
cgit v1.2.3


From 92c98a138638541a710f17f21073b568362502f8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 24 Aug 2016 18:26:55 +0200
Subject: erts: Reduce ets hash load factor

for faster lookup/insert/delete
at the expense of about one word per object.
---
 erts/emulator/beam/erl_db_hash.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 2d24f438ca..1752ec5191 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -87,7 +87,8 @@
 /* 
  * The following symbols can be manipulated to "tune" the linear hash array 
  */
-#define CHAIN_LEN 6                 /* Medium bucket chain len      */
+#define GROW_LIMIT(NACTIVE) ((NACTIVE)*1)
+#define SHRINK_LIMIT(NACTIVE) ((NACTIVE) / 2)
 
 /*
 ** We want the first mandatory segment to be small (to reduce minimal footprint)
@@ -434,7 +435,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle);
 static ERTS_INLINE void try_shrink(DbTableHash* tb)
 {
     int nactive = NACTIVE(tb);
-    if (nactive > FIRST_SEGSZ && NITEMS(tb) < (nactive * CHAIN_LEN)
+    if (nactive > FIRST_SEGSZ && NITEMS(tb) < SHRINK_LIMIT(nactive)
 	&& !IS_FIXED(tb)) {
 	shrink(tb, nactive);
     }
@@ -837,7 +838,7 @@ Lnew:
     WUNLOCK_HASH(lck);
     {
 	int nactive = NACTIVE(tb);       
-	if (nitems > nactive * (CHAIN_LEN+1) && !IS_FIXED(tb)) {
+	if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) {
 	    grow(tb, nactive);
 	}
     }
@@ -2891,7 +2892,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle)
             WUNLOCK_HASH(lck);
             nactive = NACTIVE(tb);
 
-            if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) {
+            if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) {
                 grow(tb, nactive);
             }
         } else {
-- 
cgit v1.2.3


From 25eb3fe353cb0f5c381107e43a865d3a312c8c25 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 30 Aug 2016 11:49:20 +0200
Subject: erts: Suppress failed ETS memory checks

due to the grow/shrink hysteresis of the meta tables
---
 erts/emulator/beam/erl_bif_info.c |  8 ++++++++
 erts/emulator/beam/erl_db.c       | 30 +++++++++++++++++++++++++++
 erts/emulator/beam/erl_db.h       |  2 ++
 erts/emulator/beam/erl_db_hash.c  | 43 ++++++++++++++++++++++++++++++++++++++-
 erts/emulator/beam/erl_db_hash.h  |  2 ++
 5 files changed, 84 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 3fb866733c..abf20a90e4 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3547,6 +3547,10 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    size_t words = (sizeof(DbTable) + sizeof(Uint) - 1)/sizeof(Uint);
 	    BIF_RET(make_small((Uint) words));
 	}
+        else if (ERTS_IS_ATOM_STR("DbTable_meta", BIF_ARG_1)) {
+            /* Used by ets_SUITE (stdlib) */
+            BIF_RET(erts_ets_get_meta_state(BIF_P));
+        }
 	else if (ERTS_IS_ATOM_STR("check_io_debug", BIF_ARG_1)) {
 	    /* Used by driver_SUITE (emulator) */
 	    Uint sz, *szp;
@@ -4280,6 +4284,10 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
             }
             BIF_RET(am_ok);
         }
+        else if (ERTS_IS_ATOM_STR("DbTable_meta", BIF_ARG_1)) {
+            /* Used by ets_SUITE (stdlib) */
+            BIF_RET(erts_ets_restore_meta_state(BIF_P, BIF_ARG_2));
+        }
     }
 
     BIF_ERROR(BIF_P, BADARG);
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index bad34211a5..df4e34511f 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -3978,6 +3978,36 @@ erts_ets_colliding_names(Process* p, Eterm name, Uint cnt)
     return list;
 }
 
+/*
+ * For testing only
+ * Retreive meta table size state
+ */
+Eterm erts_ets_get_meta_state(Process* p)
+{
+    Eterm* hp = HAlloc(p, 3);
+    return TUPLE2(hp,
+                  erts_ets_hash_get_memstate(p, &meta_pid_to_tab->hash),
+                  erts_ets_hash_get_memstate(p, &meta_pid_to_fixed_tab->hash));
+}
+/*
+ * For testing only
+ * Restore a previously retrieved meta table size state.
+ * We do this to suppress failed memory checks
+ * caused by the hysteresis of meta tables grow/shrink limits.
+ */
+Eterm erts_ets_restore_meta_state(Process* p, Eterm meta_state)
+{
+    Eterm* tv;
+    Eterm* hp;
+    if (!is_tuple_arity(meta_state, 2))
+        return am_badarg;
+
+    tv = tuple_val(meta_state);
+    hp = HAlloc(p, 3);
+    return TUPLE2(hp,
+                  erts_ets_hash_restore_memstate(&meta_pid_to_tab->hash, tv[1]),
+                  erts_ets_hash_restore_memstate(&meta_pid_to_fixed_tab->hash, tv[2]));
+}
 
 #ifdef HARDDEBUG   /* Here comes some debug functions */
 
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index 2f3c4a8e1b..f7eb3dc45c 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -90,6 +90,8 @@ extern Export ets_select_continue_exp;
 extern erts_smp_atomic_t erts_ets_misc_mem_size;
 
 Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt);
+Eterm erts_ets_get_meta_state(Process* p);
+Eterm erts_ets_restore_meta_state(Process* p, Eterm target_state);
 
 Uint erts_db_get_max_tabs(void);
 
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 1752ec5191..581b135233 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -2726,7 +2726,6 @@ static void shrink(DbTableHash* tb, int nactive)
     done_resizing(tb);
 }
 
-
 /* Search a list of tuples for a matching key */
 
 static HashDbTerm* search_list(DbTableHash* tb, Eterm key, 
@@ -2977,6 +2976,48 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats)
     stats->std_dev_expected = sqrt(stats->avg_chain_len * (1 - 1.0/NACTIVE(tb)));
     stats->kept_items = kept_items;
 }
+
+/* For testing only */
+Eterm erts_ets_hash_get_memstate(Process* p, DbTableHash* tb)
+{
+    Eterm seg_cnt;
+    while (!begin_resizing(tb))
+        /*spinn*/;
+
+    seg_cnt = make_small(SLOT_IX_TO_SEG_IX(tb->nslots));
+    done_resizing(tb);
+    return seg_cnt;
+}
+/* For testing only */
+Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate)
+{
+    int seg_cnt, target;
+    int nactive;
+
+    if (!is_small(memstate))
+        return make_small(__LINE__);
+
+    target = signed_val(memstate);
+    if (target < 1)
+        return make_small(__LINE__);
+    while (1) {
+        while (!begin_resizing(tb))
+            /*spin*/;
+        seg_cnt = SLOT_IX_TO_SEG_IX(tb->nslots);
+        nactive = NACTIVE(tb);
+        done_resizing(tb);
+
+        if (target == seg_cnt)
+            return am_ok;
+        if (IS_FIXED(tb))
+            return make_small(__LINE__);
+        if (target < seg_cnt)
+            shrink(tb, nactive);
+        else
+            grow(tb, nactive);
+    }
+}
+
 #ifdef HARDDEBUG
 
 void db_check_table_hash(DbTable *tbl)
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index 2d9b5e308a..e209037878 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -107,5 +107,7 @@ typedef struct {
 }DbHashStats;
 
 void db_calc_stats_hash(DbTableHash* tb, DbHashStats*);
+Eterm erts_ets_hash_get_memstate(Process*, DbTableHash* tb);
+Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate);
 
 #endif /* _DB_HASH_H */
-- 
cgit v1.2.3


From fdc2f2b4a6f6314ae7c183dc4e39e19d05ac89e4 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 2 Sep 2016 20:04:47 +0200
Subject: erts: Fix ets_SUITE:memory

by simply asking for the size of struct ext_segtab
---
 erts/emulator/beam/erl_bif_info.c | 4 +++-
 erts/emulator/beam/erl_db_hash.c  | 5 +++++
 erts/emulator/beam/erl_db_hash.h  | 1 +
 3 files changed, 9 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index abf20a90e4..13391b7c67 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3545,7 +3545,9 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	else if (ERTS_IS_ATOM_STR("DbTable_words", BIF_ARG_1)) {
 	    /* Used by ets_SUITE (stdlib) */
 	    size_t words = (sizeof(DbTable) + sizeof(Uint) - 1)/sizeof(Uint);
-	    BIF_RET(make_small((Uint) words));
+            Eterm* hp = HAlloc(BIF_P ,3);
+	    BIF_RET(TUPLE2(hp, make_small((Uint) words),
+                           erts_ets_hash_sizeof_ext_segtab()));
 	}
         else if (ERTS_IS_ATOM_STR("DbTable_meta", BIF_ARG_1)) {
             /* Used by ets_SUITE (stdlib) */
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 581b135233..3f7e14d15d 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -2977,6 +2977,11 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats)
     stats->kept_items = kept_items;
 }
 
+/* For testing only */
+Eterm erts_ets_hash_sizeof_ext_segtab(void)
+{
+    return make_small(((SIZEOF_EXT_SEGTAB(0)-1) / sizeof(UWord)) + 1);
+}
 /* For testing only */
 Eterm erts_ets_hash_get_memstate(Process* p, DbTableHash* tb)
 {
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index e209037878..11bd6aa32a 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -107,6 +107,7 @@ typedef struct {
 }DbHashStats;
 
 void db_calc_stats_hash(DbTableHash* tb, DbHashStats*);
+Eterm erts_ets_hash_sizeof_ext_segtab(void);
 Eterm erts_ets_hash_get_memstate(Process*, DbTableHash* tb);
 Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate);
 
-- 
cgit v1.2.3


From 1573603177f5ba641390c67b0515defb2786ecfb Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 6 Sep 2016 17:42:30 +0200
Subject: erts: Tweak ets grow/shrink to keep up at contention

---
 erts/emulator/beam/erl_db_hash.c | 287 +++++++++++++++++++++------------------
 erts/emulator/beam/erl_db_hash.h |   8 +-
 2 files changed, 161 insertions(+), 134 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 3f7e14d15d..698e4c301b 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -194,6 +194,7 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix,
 #ifdef ERTS_SMP
 #  define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1)
 #  define GET_LOCK(tb,hval) (&(tb)->locks->lck_vec[(hval) & DB_HASH_LOCK_MASK].lck)
+#  define GET_LOCK_MAYBE(tb,hval) ((tb)->common.is_thread_safe ? NULL : GET_LOCK(tb,hval))
 
 /* Fine grained read lock */
 static ERTS_INLINE erts_smp_rwmtx_t* RLOCK_HASH(DbTableHash* tb, HashValue hval)
@@ -330,7 +331,9 @@ struct segment {
 
 /* An extended segment table */
 struct ext_segtab {
+#ifdef ERTS_SMP
     ErtsThrPrgrLaterOp lop;
+#endif
     struct segment** prev_segtab;  /* Used when table is shrinking */
     int prev_nsegs;                /* Size of prev_segtab */
     int nsegs;                     /* Size of this segtab */
@@ -355,14 +358,14 @@ static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb,
 ** Forward decl's (static functions)
 */
 static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix);
-static int alloc_seg(DbTableHash *tb);
+static void alloc_seg(DbTableHash *tb);
 static int free_seg(DbTableHash *tb, int free_records);
 static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_smp_rwmtx_t** lck_ptr,
 			HashDbTerm *list);
 static HashDbTerm* search_list(DbTableHash* tb, Eterm key, 
 			       HashValue hval, HashDbTerm *list);
-static void shrink(DbTableHash* tb, int nactive);
-static void grow(DbTableHash* tb, int nactive);
+static void shrink(DbTableHash* tb, int nitems);
+static void grow(DbTableHash* tb, int nitems);
 static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2,
 			   Uint sz, DbTableHash*);
 static int analyze_pattern(DbTableHash *tb, Eterm pattern, 
@@ -435,9 +438,10 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle);
 static ERTS_INLINE void try_shrink(DbTableHash* tb)
 {
     int nactive = NACTIVE(tb);
-    if (nactive > FIRST_SEGSZ && NITEMS(tb) < SHRINK_LIMIT(nactive)
+    int nitems = NITEMS(tb);
+    if (nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive)
 	&& !IS_FIXED(tb)) {
-	shrink(tb, nactive);
+	shrink(tb, nitems);
     }
 }	
 
@@ -839,7 +843,7 @@ Lnew:
     {
 	int nactive = NACTIVE(tb);       
 	if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) {
-	    grow(tb, nactive);
+	    grow(tb, nitems);
 	}
     }
     CHECK_TABLES();
@@ -2417,7 +2421,7 @@ static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix)
 
 /* Extend table with one new segment
 */
-static int alloc_seg(DbTableHash *tb)
+static void alloc_seg(DbTableHash *tb)
 {    
     int seg_ix = SLOT_IX_TO_SEG_IX(tb->nslots);
     struct segment** segtab;
@@ -2435,7 +2439,6 @@ static int alloc_seg(DbTableHash *tb)
                                                      SIZEOF_SEGMENT(EXT_SEGSZ));
     sys_memset(segtab[seg_ix], 0, SIZEOF_SEGMENT(EXT_SEGSZ));
     tb->nslots += EXT_SEGSZ;
-    return 1;
 }
 
 #ifdef ERTS_SMP
@@ -2506,9 +2509,6 @@ static int free_seg(DbTableHash *tb, int free_records)
                 erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable*)tb, est,
                              SIZEOF_EXT_SEGTAB(est->nsegs));
         }
-        else {
-
-        }
     }
     seg_sz = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ;
     erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, segp, SIZEOF_SEGMENT(seg_sz));
@@ -2584,85 +2584,93 @@ done_resizing(DbTableHash* tb)
 #endif
 }
 
-/* Grow table with one new bucket.
+/* Grow table with one or more new buckets.
 ** Allocate new segment if needed.
 */
-static void grow(DbTableHash* tb, int nactive)
+static void grow(DbTableHash* tb, int nitems)
 {
     HashDbTerm** pnext;
     HashDbTerm** to_pnext;
     HashDbTerm* p;
     erts_smp_rwmtx_t* lck;
-    int from_ix;
+    int nactive;
+    int from_ix, to_ix;
     int szm;
+    int loop_limit = 5;
 
-    if (!begin_resizing(tb))
-	return; /* already in progress */
-    if (NACTIVE(tb) != nactive) {
-	goto abort; /* already done (race) */
-    }
+    do {
+        if (!begin_resizing(tb))
+            return; /* already in progress */
+        nactive = NACTIVE(tb);
+        if (nitems <= GROW_LIMIT(nactive)) {
+            goto abort; /* already done (race) */
+        }
 
-    /* Ensure that the slot nactive exists */
-    if (nactive == tb->nslots) {
-	/* Time to get a new segment */    
-        ASSERT(((nactive-FIRST_SEGSZ) & EXT_SEGSZ_MASK) == 0);
-	if (!alloc_seg(tb)) goto abort;	    
-    }
-    ASSERT(nactive < tb->nslots);
+        /* Ensure that the slot nactive exists */
+        if (nactive == tb->nslots) {
+            /* Time to get a new segment */
+            ASSERT(((nactive-FIRST_SEGSZ) & EXT_SEGSZ_MASK) == 0);
+            alloc_seg(tb);
+        }
+        ASSERT(nactive < tb->nslots);
 
-    szm = erts_smp_atomic_read_nob(&tb->szm);
-    if (nactive <= szm) {
-	from_ix = nactive & (szm >> 1);
-    } else {
-	ASSERT(nactive == szm+1);
-	from_ix = 0;
-	szm = (szm<<1) | 1;
-    }
+        szm = erts_smp_atomic_read_nob(&tb->szm);
+        if (nactive <= szm) {
+            from_ix = nactive & (szm >> 1);
+        } else {
+            ASSERT(nactive == szm+1);
+            from_ix = 0;
+            szm = (szm<<1) | 1;
+        }
+        to_ix = nactive;
+
+        lck = WLOCK_HASH(tb, from_ix);
+        ERTS_SMP_ASSERT(lck == GET_LOCK_MAYBE(tb,to_ix));
+        /* Now a final double check (with the from_ix lock held)
+         * that we did not get raced by a table fixer.
+         */
+        if (IS_FIXED(tb)) {
+            WUNLOCK_HASH(lck);
+            goto abort;
+        }
+        erts_smp_atomic_set_nob(&tb->nactive, ++nactive);
+        if (from_ix == 0) {
+            if (DB_USING_FINE_LOCKING(tb))
+                erts_smp_atomic_set_relb(&tb->szm, szm);
+            else
+                erts_smp_atomic_set_nob(&tb->szm, szm);
+        }
+        done_resizing(tb);
 
-    lck = WLOCK_HASH(tb, from_ix);
-    /* Now a final double check (with the from_ix lock held)
-     * that we did not get raced by a table fixer.
-     */
-    if (IS_FIXED(tb)) {
-	WUNLOCK_HASH(lck);
-	goto abort;
-    }
-    erts_smp_atomic_inc_nob(&tb->nactive);
-    if (from_ix == 0) {
-	if (DB_USING_FINE_LOCKING(tb))
-	    erts_smp_atomic_set_relb(&tb->szm, szm);
-	else
-	    erts_smp_atomic_set_nob(&tb->szm, szm);
-    }
-    done_resizing(tb);
+        /* Finally, let's split the bucket. We try to do it in a smart way
+           to keep link order and avoid unnecessary updates of next-pointers */
+        pnext = &BUCKET(tb, from_ix);
+        p = *pnext;
+        to_pnext = &BUCKET(tb, to_ix);
+        while (p != NULL) {
+            if (p->hvalue == INVALID_HASH) { /* rare but possible with fine locking */
+                *pnext = p->next;
+                free_term(tb, p);
+                p = *pnext;
+            }
+            else {
+                int ix = p->hvalue & szm;
+                if (ix != from_ix) {
+                    ASSERT(ix == (from_ix ^ ((szm+1)>>1)));
+                    *to_pnext = p;
+                    /* Swap "from" and "to": */
+                    from_ix = ix;
+                    to_pnext = pnext;
+                }
+                pnext = &p->next;
+                p = *pnext;
+            }
+        }
+        *to_pnext = NULL;
+        WUNLOCK_HASH(lck);
 
-    /* Finally, let's split the bucket. We try to do it in a smart way
-       to keep link order and avoid unnecessary updates of next-pointers */
-    pnext = &BUCKET(tb, from_ix);
-    p = *pnext;
-    to_pnext = &BUCKET(tb, nactive);
-    while (p != NULL) {
-	if (p->hvalue == INVALID_HASH) { /* rare but possible with fine locking */
-	    *pnext = p->next;
-	    free_term(tb, p);
-	    p = *pnext;
-	}
-	else {
-	    int ix = p->hvalue & szm;
-	    if (ix != from_ix) {
-		ASSERT(ix == (from_ix ^ ((szm+1)>>1)));
-		*to_pnext = p;
-		/* Swap "from" and "to": */
-		from_ix = ix;
-		to_pnext = pnext;
-	    }
-	    pnext = &p->next;
-	    p = *pnext;		
-	}
-    }
-    *to_pnext = NULL;
+    }while (--loop_limit && nitems > GROW_LIMIT(nactive));
 
-    WUNLOCK_HASH(lck);
     return;
    
 abort:
@@ -2673,56 +2681,75 @@ abort:
 /* Shrink table by joining top bucket.
 ** Remove top segment if it gets empty.
 */
-static void shrink(DbTableHash* tb, int nactive)
-{     
-    if (!begin_resizing(tb))
-	return; /* already in progress */
-    if (NACTIVE(tb) == nactive) {
-	erts_smp_rwmtx_t* lck;
-	int src_ix = nactive - 1;
-	int low_szm = erts_smp_atomic_read_nob(&tb->szm) >> 1;
-	int dst_ix = src_ix & low_szm;
-
-	ASSERT(dst_ix < src_ix);
-	ASSERT(nactive > FIRST_SEGSZ);
-	lck = WLOCK_HASH(tb, dst_ix);
-	/* Double check for racing table fixers */
-	if (!IS_FIXED(tb)) {
-	    HashDbTerm** src_bp = &BUCKET(tb, src_ix);
-	    HashDbTerm** dst_bp = &BUCKET(tb, dst_ix);
-	    HashDbTerm** bp = src_bp;
-
-	    /* Q: Why join lists by appending "dst" at the end of "src"?
-	       A: Must step through "src" anyway to purge pseudo deleted. */
-	    while(*bp != NULL) {
-		if ((*bp)->hvalue == INVALID_HASH) {
-		    HashDbTerm* deleted = *bp;
-		    *bp = deleted->next;
-		    free_term(tb, deleted);
-		} else {
-		    bp = &(*bp)->next;
-		}
-	    }
-	    *bp = *dst_bp;
-	    *dst_bp = *src_bp;
-	    *src_bp = NULL;
-	    
-	    erts_smp_atomic_set_nob(&tb->nactive, src_ix);
-	    if (dst_ix == 0) {
-		erts_smp_atomic_set_relb(&tb->szm, low_szm);
-	    }
-	    WUNLOCK_HASH(lck);
-	    
-	    if (tb->nslots - src_ix >= EXT_SEGSZ) {
-		free_seg(tb, 0);
-	    }
-	}
-	else {
-	    WUNLOCK_HASH(lck);
-	}
+static void shrink(DbTableHash* tb, int nitems)
+{
+    HashDbTerm** src_bp;
+    HashDbTerm** dst_bp;
+    HashDbTerm** bp;
+    erts_smp_rwmtx_t* lck;
+    int src_ix, dst_ix, low_szm;
+    int nactive;
+    int loop_limit = 5;
 
-    }
-    /*else already done */
+    do {
+        if (!begin_resizing(tb))
+            return; /* already in progress */
+        nactive = NACTIVE(tb);
+        if (!(nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive))) {
+            goto abort; /* already done (race) */
+        }
+        src_ix = nactive - 1;
+        low_szm = erts_smp_atomic_read_nob(&tb->szm) >> 1;
+        dst_ix = src_ix & low_szm;
+
+        ASSERT(dst_ix < src_ix);
+        ASSERT(nactive > FIRST_SEGSZ);
+        lck = WLOCK_HASH(tb, dst_ix);
+        ERTS_SMP_ASSERT(lck == GET_LOCK_MAYBE(tb,src_ix));
+        /* Double check for racing table fixers */
+        if (IS_FIXED(tb)) {
+            WUNLOCK_HASH(lck);
+            goto abort;
+        }
+
+        src_bp = &BUCKET(tb, src_ix);
+        dst_bp = &BUCKET(tb, dst_ix);
+        bp = src_bp;
+
+        /*
+         * We join lists by appending "dst" at the end of "src"
+         * as we must step through "src" anyway to purge pseudo deleted.
+         */
+        while(*bp != NULL) {
+            if ((*bp)->hvalue == INVALID_HASH) {
+                HashDbTerm* deleted = *bp;
+                *bp = deleted->next;
+                free_term(tb, deleted);
+            } else {
+                bp = &(*bp)->next;
+            }
+        }
+        *bp = *dst_bp;
+        *dst_bp = *src_bp;
+        *src_bp = NULL;
+
+        nactive = src_ix;
+        erts_smp_atomic_set_nob(&tb->nactive, nactive);
+        if (dst_ix == 0) {
+            erts_smp_atomic_set_relb(&tb->szm, low_szm);
+        }
+        WUNLOCK_HASH(lck);
+
+        if (tb->nslots - src_ix >= EXT_SEGSZ) {
+            free_seg(tb, 0);
+        }
+        done_resizing(tb);
+
+    } while (--loop_limit
+             && nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive));
+    return;
+
+abort:
     done_resizing(tb);
 }
 
@@ -2892,7 +2919,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle)
             nactive = NACTIVE(tb);
 
             if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) {
-                grow(tb, nactive);
+                grow(tb, nitems);
             }
         } else {
             WUNLOCK_HASH(lck);
@@ -2997,7 +3024,6 @@ Eterm erts_ets_hash_get_memstate(Process* p, DbTableHash* tb)
 Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate)
 {
     int seg_cnt, target;
-    int nactive;
 
     if (!is_small(memstate))
         return make_small(__LINE__);
@@ -3009,7 +3035,6 @@ Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate)
         while (!begin_resizing(tb))
             /*spin*/;
         seg_cnt = SLOT_IX_TO_SEG_IX(tb->nslots);
-        nactive = NACTIVE(tb);
         done_resizing(tb);
 
         if (target == seg_cnt)
@@ -3017,9 +3042,9 @@ Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate)
         if (IS_FIXED(tb))
             return make_small(__LINE__);
         if (target < seg_cnt)
-            shrink(tb, nactive);
+            shrink(tb, 0);
         else
-            grow(tb, nactive);
+            grow(tb, INT_MAX);
     }
 }
 
@@ -3031,7 +3056,7 @@ void db_check_table_hash(DbTable *tbl)
     HashDbTerm* list;
     int j;
     
-    for (j = 0; j < tb->nactive; j++) {
+    for (j = 0; j < NACTIVE(tb); j++) {
 	if ((list = BUCKET(tb,j)) != 0) {
 	    while (list != 0) {
 		if (!is_tuple(make_tuple(list->dbterm.tpl))) {
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index 11bd6aa32a..6d25c73549 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -50,17 +50,19 @@ typedef struct db_table_hash_fine_locks {
 typedef struct db_table_hash {
     DbTableCommon common;
 
+    /* SMP: szm and nactive are write-protected by is_resizing or table write lock */
+    erts_smp_atomic_t szm;     /* current size mask. */
+    erts_smp_atomic_t nactive; /* Number of "active" slots */
+
     erts_smp_atomic_t segtab;  /* The segment table (struct segment**) */
     struct segment* first_segtab[1];
-    erts_smp_atomic_t szm;     /* current size mask. */
-    
+
     /* SMP: nslots and nsegs are protected by is_resizing or table write lock */
     int nslots;       /* Total number of slots */
     int nsegs;        /* Size of segment table */
 
     /* List of slots where elements have been deleted while table was fixed */
     erts_smp_atomic_t fixdel;  /* (FixedDeletion*) */	
-    erts_smp_atomic_t nactive; /* Number of "active" slots */
 #ifdef ERTS_SMP
     erts_smp_atomic_t is_resizing; /* grow/shrink in progress */
     DbTableHashFineLocks* locks;
-- 
cgit v1.2.3


From d100187bde4f3f555b4f3cd9561ec8c67a528895 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 13 Sep 2016 18:28:09 +0200
Subject: erts: Unify reduction count for ets:select

to be per object as the other select-variants
and not per table slot.
---
 erts/emulator/beam/erl_db_hash.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 698e4c301b..40df1c5356 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1494,6 +1494,7 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl,
 		    match_list = CONS(hp, match_res, match_list);
 		    ++got;
 		}
+                --num_left;
 	    }
 	    current = current->next;
 	}	
@@ -1511,7 +1512,6 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl,
 	    }
 	}
 	else { /* Key is variable */
-	    --num_left;
 
 	    if ((slot_ix=next_slot(tb,slot_ix,&lck)) == 0) {
 		slot_ix = -1;
-- 
cgit v1.2.3


From 3489b9b689073f428a23f7fc7a67774b7dda07be Mon Sep 17 00:00:00 2001
From: Tuncer Ayaz <tuncer.ayaz@gmail.com>
Date: Sun, 18 Sep 2016 12:33:08 +0200
Subject: Use more correct delimiters for erl_nif.h include

Anywhere but the beam sources we shouldn't #include "erl_nif.h", because
what "erl_nif.h" does is: (1) fail to find it outside of -I dirs, (2)
then treat it as if it was written like <erl_nif.h>. Using <erl_nif.h>
skips (1).

More information can be found in 6.10.2 of the C standard.

Because the examples use "erl_nif.h", NIF projects in the Erlang
ecosystem copy this verbatim and make the same mistake.
---
 erts/emulator/test/alloc_SUITE_data/testcase_driver.h     | 2 +-
 erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c | 2 +-
 erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c             | 2 +-
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c             | 2 +-
 erts/emulator/test/nif_SUITE_data/nif_mod.c               | 2 +-
 erts/emulator/test/nif_SUITE_data/testcase_driver.h       | 2 +-
 erts/emulator/test/nif_SUITE_data/tester.c                | 2 +-
 erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c | 2 +-
 erts/emulator/test/trace_nif_SUITE_data/trace_nif.c       | 2 +-
 erts/emulator/test/tracer_SUITE_data/tracer_test.c        | 2 +-
 10 files changed, 10 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/alloc_SUITE_data/testcase_driver.h b/erts/emulator/test/alloc_SUITE_data/testcase_driver.h
index f0ca91bd06..2b742dd7e3 100644
--- a/erts/emulator/test/alloc_SUITE_data/testcase_driver.h
+++ b/erts/emulator/test/alloc_SUITE_data/testcase_driver.h
@@ -20,7 +20,7 @@
 #ifndef TESTCASE_DRIVER_H__
 #define TESTCASE_DRIVER_H__
 
-#include "erl_nif.h"
+#include <erl_nif.h>
 #include <stdlib.h>
 
 typedef struct {
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
index a0019e5d95..daaff955bc 100644
--- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
+++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
@@ -17,7 +17,7 @@
  *
  * %CopyrightEnd%
  */
-#include "erl_nif.h"
+#include <erl_nif.h>
 #include <assert.h>
 #ifdef __WIN32__
 #include <windows.h>
diff --git a/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c
index e011aadce9..46ee8b5540 100644
--- a/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c
+++ b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c
@@ -24,7 +24,7 @@
  * Author: Rickard Green
  */
 
-#include "erl_nif.h"
+#include <erl_nif.h>
 
 #ifdef __WIN32__
 #  ifndef WIN32_LEAN_AND_MEAN
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index f2b1ef9d24..ed142478c6 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -17,7 +17,7 @@
  *
  * %CopyrightEnd%
  */
-#include "erl_nif.h"
+#include <erl_nif.h>
 
 #include <stdio.h>
 #include <string.h>
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.c b/erts/emulator/test/nif_SUITE_data/nif_mod.c
index fd8a0d0595..4e94e7901c 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_mod.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.c
@@ -17,7 +17,7 @@
  *
  * %CopyrightEnd%
  */
-#include "erl_nif.h"
+#include <erl_nif.h>
 #include <string.h>
 #include <stdio.h>
 
diff --git a/erts/emulator/test/nif_SUITE_data/testcase_driver.h b/erts/emulator/test/nif_SUITE_data/testcase_driver.h
index e32e63069a..feb10ecaea 100644
--- a/erts/emulator/test/nif_SUITE_data/testcase_driver.h
+++ b/erts/emulator/test/nif_SUITE_data/testcase_driver.h
@@ -20,7 +20,7 @@
 #ifndef TESTCASE_DRIVER_H__
 #define TESTCASE_DRIVER_H__
 
-#include "erl_nif.h"
+#include <erl_nif.h>
 #include <stdlib.h>
 #include <stdio.h>
 
diff --git a/erts/emulator/test/nif_SUITE_data/tester.c b/erts/emulator/test/nif_SUITE_data/tester.c
index 257b116322..248c31fd3a 100644
--- a/erts/emulator/test/nif_SUITE_data/tester.c
+++ b/erts/emulator/test/nif_SUITE_data/tester.c
@@ -1,4 +1,4 @@
-#include "erl_nif.h"
+#include <erl_nif.h>
 
 #include <stdio.h>
 #include <stdarg.h>
diff --git a/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c
index 33b346aab7..b8c8590546 100644
--- a/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c
+++ b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c
@@ -1,4 +1,4 @@
-#include "erl_nif.h"
+#include <erl_nif.h>
 
 
 static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
diff --git a/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c b/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c
index 26f2420b8b..fe9308bf4e 100644
--- a/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c
+++ b/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c
@@ -1,4 +1,4 @@
-#include "erl_nif.h"
+#include <erl_nif.h>
 
 
 static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
diff --git a/erts/emulator/test/tracer_SUITE_data/tracer_test.c b/erts/emulator/test/tracer_SUITE_data/tracer_test.c
index a26bb33600..b68e480215 100644
--- a/erts/emulator/test/tracer_SUITE_data/tracer_test.c
+++ b/erts/emulator/test/tracer_SUITE_data/tracer_test.c
@@ -18,7 +18,7 @@
  * %CopyrightEnd%
  */
 
-#include "erl_nif.h"
+#include <erl_nif.h>
 
 #include <stdio.h>
 #include <string.h>
-- 
cgit v1.2.3


From ca7578aca806a6eb3e816c918120db78f485f5cb Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 26 Sep 2016 15:05:21 +0200
Subject: Fix initialization dependencies between time, lock-check and lcnt

---
 erts/emulator/beam/erl_init.c       | 3 ---
 erts/emulator/beam/erl_lock_count.c | 4 ++--
 erts/emulator/sys/unix/sys.c        | 4 ++++
 erts/emulator/sys/win32/sys.c       | 8 ++++----
 4 files changed, 10 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index d9c3b0dcf4..611a98a30f 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -762,9 +762,6 @@ early_init(int *argc, char **argv) /*
     erts_thr_progress_pre_init();
 #endif
 
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init();
-#endif
 #ifdef ERTS_SMP
     erts_smp_atomic32_init_nob(&erts_writing_erl_crash_dump, 0L);
     erts_tsd_key_create(&erts_is_crash_dumping_key,"erts_is_crash_dumping_key");
diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index bd00480ba2..7eba2e34e9 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -274,11 +274,11 @@ void erts_lcnt_init() {
 
     lcnt_unlock();
 
-    /* set start timer and zero statistics */
-    erts_lcnt_clear_counters();
 }
 
 void erts_lcnt_late_init() {
+    /* set start timer and zero statistics */
+    erts_lcnt_clear_counters();
     erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler);
 }
 
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index b036b20b7b..e0a3bbdff0 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -559,6 +559,10 @@ erts_sys_pre_init(void)
 
     report_exit_list = NULL;
 
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    erts_lc_init();
+#endif
+
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_init();
 #endif
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index fce76db28f..b7360a758e 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -3188,16 +3188,16 @@ erts_sys_pre_init(void)
     eid.thread_create_parent_func = thr_create_cleanup;
 
     erts_thr_init(&eid);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    erts_lc_init();
 #endif
-
-    erts_init_sys_time_sup();
-
-#ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_init();
 #endif
 #endif
 
+    erts_init_sys_time_sup();
+
     erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0);
 }
 
-- 
cgit v1.2.3


From d85475bf9461d854c12589f78201fd77bea11a17 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 27 Sep 2016 11:05:44 +0200
Subject: erts: Rename __errno in order to avoid conflict on openbsd

---
 erts/emulator/sys/unix/sys.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 089efec3e8..b0e623a5b9 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -715,13 +715,13 @@ static RETSIGTYPE suspend_signal(void)
 static RETSIGTYPE suspend_signal(int signum)
 #endif
 {
-    int res, buf[1], __errno = errno;
+    int res, buf[1], tmp_errno = errno;
     do {
         res = read(sig_suspend_fds[0], buf, sizeof(int));
     } while (res < 0 && errno == EINTR);
 
     /* restore previous errno in case read changed it */
-    errno = __errno;
+    errno = tmp_errno;
 }
 #endif /* #ifdef ERTS_SYS_SUSPEND_SIGNAL */
 
-- 
cgit v1.2.3


From 23ecba248ade6e4f1a6ed58c6636598643916019 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 29 Sep 2016 16:43:44 +0200
Subject: Fix minor soft purge bug

A process calling a fun from a module currently being soft-purged
could race with failure of the soft purge. When this happened the
call triggered loading of the module which erroneously would
load new code for the module in the case no new generation was
currently loaded.
---
 erts/emulator/beam/erl_fun.c | 1 -
 1 file changed, 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index c639ba623f..5258d83a18 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -236,7 +236,6 @@ erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no)
 	ErlFunEntry *fe = funs[ix];
 	if (fe->address == unloaded_fun)
 	    fe->address = fe->pend_purge_address;
-	fe->pend_purge_address = NULL;
     }
 }
 
-- 
cgit v1.2.3


From b001b56e061454f3175d9d9a99e2efb945a696b9 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 4 Oct 2016 10:24:56 +0200
Subject: erts: Do tracer liveness check on current tracer

This fixes a fault introduced in 19.0 where an invalid
tracer would block setting of a new tracer on a process.
---
 erts/emulator/beam/erl_bif_trace.c | 10 +++++-----
 erts/emulator/test/trace_SUITE.erl | 32 ++++++++++++++++++++++++++++++--
 2 files changed, 35 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 66e5146da0..96275eb228 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -512,7 +512,7 @@ start_trace(Process *c_p, ErtsTracer tracer,
             && !ERTS_TRACER_COMPARE(ERTS_TRACER(port), tracer)) {
             /* This tracee is already being traced, and not by the
              * tracer to be */
-            if (erts_is_tracer_enabled(tracer, common)) {
+            if (erts_is_tracer_enabled(ERTS_TRACER(port), common)) {
                 /* The tracer is still in use */
                 return 1;
             }
@@ -715,8 +715,8 @@ Eterm erts_internal_trace_3(BIF_ALIST_3)
 		    Process* tracee_p = erts_pix2proc(i);
 		    if (! tracee_p) 
 			continue;
-                    start_trace(p, tracer, &tracee_p->common, on, mask);
-		    matches++;
+                    if (!start_trace(p, tracer, &tracee_p->common, on, mask))
+                        matches++;
 		}
 	    }
 	    if (ports || mods) {
@@ -730,8 +730,8 @@ Eterm erts_internal_trace_3(BIF_ALIST_3)
 		    state = erts_atomic32_read_nob(&tracee_port->state);
 		    if (state & ERTS_PORT_SFLGS_DEAD)
 			continue;
-                    start_trace(p, tracer, &tracee_port->common, on, mask);
-                    matches++;
+                    if (!start_trace(p, tracer, &tracee_port->common, on, mask))
+                        matches++;
 		}
 	    }
 	}
diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl
index da6a6bdea4..f846b0f4b9 100644
--- a/erts/emulator/test/trace_SUITE.erl
+++ b/erts/emulator/test/trace_SUITE.erl
@@ -30,7 +30,7 @@
 	 procs_trace/1, dist_procs_trace/1, procs_new_trace/1,
 	 suspend/1, mutual_suspend/1, suspend_exit/1, suspender_exit/1,
 	 suspend_system_limit/1, suspend_opts/1, suspend_waiting/1,
-	 new_clear/1, existing_clear/1,
+	 new_clear/1, existing_clear/1, tracer_die/1,
 	 set_on_spawn/1, set_on_first_spawn/1, cpu_timestamp/1,
 	 set_on_link/1, set_on_first_link/1,
 	 system_monitor_args/1, more_system_monitor_args/1,
@@ -54,7 +54,7 @@ all() ->
      send_trace, procs_trace, dist_procs_trace, suspend,
      mutual_suspend, suspend_exit, suspender_exit,
      suspend_system_limit, suspend_opts, suspend_waiting,
-     new_clear, existing_clear, set_on_spawn,
+     new_clear, existing_clear, tracer_die, set_on_spawn,
      set_on_first_spawn, set_on_link, set_on_first_link,
      system_monitor_args,
      more_system_monitor_args, system_monitor_long_gc_1,
@@ -1636,6 +1636,34 @@ existing_clear(Config) when is_list(Config) ->
 
     ok.
 
+%% Test that erlang:trace/3 can be called on processes where the
+%% tracer has died. OTP-13928
+tracer_die(Config) when is_list(Config) ->
+    Proc = spawn(fun receiver/0),
+
+    Tracer = spawn(fun receiver/0),
+    timer:sleep(1),
+    N = erlang:trace(existing, true, [send, {tracer, Tracer}]),
+    {flags, [send]} = erlang:trace_info(Proc, flags),
+    {tracer, Tracer} = erlang:trace_info(Proc, tracer),
+    exit(Tracer, die),
+
+    Tracer2 = spawn(fun receiver/0),
+    timer:sleep(1),
+    N = erlang:trace(existing, true, [send, {tracer, Tracer2}]),
+    {flags, [send]} = erlang:trace_info(Proc, flags),
+    {tracer, Tracer2} = erlang:trace_info(Proc, tracer),
+    exit(Tracer2, die),
+
+    Tracer3 = spawn(fun receiver/0),
+    timer:sleep(1),
+    1 = erlang:trace(Proc, true, [send, {tracer, Tracer3}]),
+    {flags, [send]} = erlang:trace_info(Proc, flags),
+    {tracer, Tracer3} = erlang:trace_info(Proc, tracer),
+    exit(Tracer3, die),
+
+    ok.
+
 %% Test that an invalid flag cause badarg
 bad_flag(Config) when is_list(Config) ->
     %% A bad flag could deadlock the SMP emulator in erts-5.5
-- 
cgit v1.2.3


From ee6e1bbfab12d5b1106fab2745fc73f7ad7b473e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 4 Oct 2016 16:30:17 +0200
Subject: erts: Print error code from failed NIF load/upgrade/reload

in Text part of error tuple, like

{error, {load, "Library load-call unsuccessful (606)}}
---
 erts/emulator/beam/erl_nif.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 3a547982da..9a873857b9 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3360,7 +3360,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	veto = entry->reload(&env, &lib->priv_data, BIF_ARG_2);
 	erts_post_nif(&env);
 	if (veto) {
-	    ret = load_nif_error(BIF_P, reload, "Library reload-call unsuccessful.");
+	    ret = load_nif_error(BIF_P, reload, "Library reload-call unsuccessful (%d).", veto);
 	}
 	else {
 	    commit_opened_resource_types(lib);
@@ -3382,7 +3382,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	    erts_post_nif(&env);
 	    if (veto) {
 		prev_mi->nif->priv_data = prev_old_data;
-		ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful.");
+		ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful (%d).", veto);
 	    }
 	    else
 		commit_opened_resource_types(lib);
@@ -3392,7 +3392,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	    veto = entry->load(&env, &lib->priv_data, BIF_ARG_2);
 	    erts_post_nif(&env);
 	    if (veto) {
-		ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful.");
+		ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful (%d).", veto);
 	    }
 	    else
 		commit_opened_resource_types(lib);
-- 
cgit v1.2.3


From d8e69ab77f32ce1503125fa464d6db3ba1c904d0 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 15 Sep 2016 14:38:09 +0200
Subject: erts: Refactor rename structs hipe_mfa and hipe_ref

---
 erts/emulator/hipe/hipe_bif0.c | 44 +++++++++++++++++++++---------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 3336fded7a..b6d0abb2d1 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -537,13 +537,13 @@ BIF_RETTYPE hipe_bifs_merge_term_1(BIF_ALIST_1)
     BIF_RET(val);
 }
 
-struct mfa_t {
+struct hipe_mfa {
     Eterm mod;
     Eterm fun;
     Uint  ari;
 };
 
-static int term_to_mfa(Eterm term, struct mfa_t *mfa)
+static int term_to_mfa(Eterm term, struct hipe_mfa *mfa)
 {
     Eterm mod, fun, a;
     Uint ari;
@@ -601,7 +601,7 @@ static Uint *hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity)
 
 Uint *hipe_bifs_find_pc_from_mfa(Eterm term)
 {
-    struct mfa_t mfa;
+    struct hipe_mfa mfa;
 
     if (!term_to_mfa(term, &mfa))
 	return NULL;
@@ -621,7 +621,7 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3)
     Eterm *pc;
     void *address;
     int is_closure;
-    struct mfa_t mfa;
+    struct hipe_mfa mfa;
 
     switch (BIF_ARG_3) {
       case am_false:
@@ -1017,7 +1017,7 @@ struct hipe_mfa_info {
     Eterm *beam_code;
     Uint orig_beam_op;
     struct hipe_mfa_info_list *refers_to;
-    struct ref *referred_from;
+    struct hipe_ref *referred_from;
 #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
     void *trampoline;
 #endif
@@ -1235,7 +1235,7 @@ void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int arity, void *trampol
 
 BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3)
 {
-    struct mfa_t mfa;
+    struct hipe_mfa mfa;
     void *address;
     int is_exported;
 
@@ -1257,7 +1257,7 @@ BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3)
 BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1)
 {
     Eterm lst;
-    struct mfa_t mfa;
+    struct hipe_mfa mfa;
     struct hipe_mfa_info *p;
 
     hipe_mfa_info_table_rwlock();
@@ -1426,7 +1426,7 @@ BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3)
 
 BIF_RETTYPE hipe_bifs_find_na_or_make_stub_2(BIF_ALIST_2)
 {
-    struct mfa_t mfa;
+    struct hipe_mfa mfa;
     void *address;
     int is_remote;
 
@@ -1512,12 +1512,12 @@ struct hipe_mfa_info_list {
     struct hipe_mfa_info_list *next;
 };
 
-struct ref {
+struct hipe_ref {
     struct hipe_mfa_info *caller_mfa;
     void *address;
     void *trampoline;
     unsigned int flags;
-    struct ref *next;
+    struct hipe_ref *next;
 };
 #define REF_FLAG_IS_LOAD_MFA		1	/* bit 0: 0 == call, 1 == load_mfa */
 #define REF_FLAG_IS_REMOTE		2	/* bit 1: 0 == local, 1 == remote */
@@ -1528,16 +1528,16 @@ struct ref {
  */
 BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
 {
-    struct mfa_t callee;
+    struct hipe_mfa callee;
     Eterm *tuple;
-    struct mfa_t caller;
+    struct hipe_mfa caller;
     void *address;
     void *trampoline;
     unsigned int flags;
     struct hipe_mfa_info *callee_mfa;
     struct hipe_mfa_info *caller_mfa;
     struct hipe_mfa_info_list *refers_to;
-    struct ref *ref;
+    struct hipe_ref *ref;
 
     if (!term_to_mfa(BIF_ARG_1, &callee))
 	goto badarg;
@@ -1607,9 +1607,9 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
  */
 BIF_RETTYPE hipe_bifs_mark_referred_from_1(BIF_ALIST_1) /* get_refs_from */
 {
-    struct mfa_t mfa;
+    struct hipe_mfa mfa;
     const struct hipe_mfa_info *p;
-    struct ref *ref;
+    struct hipe_ref *ref;
 
     if (!term_to_mfa(BIF_ARG_1, &mfa))
 	BIF_ERROR(BIF_P, BADARG);
@@ -1647,7 +1647,7 @@ static void hipe_purge_all_refs(void)
 		erts_free(ERTS_ALC_T_HIPE, to);
 	    }
 	    while (mfa->referred_from) {
-		struct ref* from = mfa->referred_from;
+		struct hipe_ref* from = mfa->referred_from;
 		mfa->referred_from = from->next;
 		erts_free(ERTS_ALC_T_HIPE, from);
 	    }
@@ -1659,10 +1659,10 @@ static void hipe_purge_all_refs(void)
 
 BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1)
 {
-    struct mfa_t mfa;
+    struct hipe_mfa mfa;
     struct hipe_mfa_info *caller_mfa, *callee_mfa;
     struct hipe_mfa_info_list *refers_to, *tmp_refers_to;
-    struct ref **prev, *ref;
+    struct hipe_ref **prev, *ref;
 
     if (BIF_ARG_1 == am_all) {
 	hipe_purge_all_refs();
@@ -1686,7 +1686,7 @@ BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1)
 			prev = &ref->next;
 			ref = ref->next;
 		    } else {
-			struct ref *tmp = ref;
+			struct hipe_ref *tmp = ref;
 			ref = ref->next;
 			*prev = ref;
 			erts_free(ERTS_ALC_T_HIPE, tmp);
@@ -1713,9 +1713,9 @@ BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1)
  */
 BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1)
 {
-    struct mfa_t mfa;
+    struct hipe_mfa mfa;
     struct hipe_mfa_info *p;
-    struct ref **prev, *ref;
+    struct hipe_ref **prev, *ref;
     int is_remote, res;
     void *new_address;
 
@@ -1738,7 +1738,7 @@ BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1)
 		    fprintf(stderr, "%s: patch failed\r\n", __FUNCTION__);
 		ref->flags &= ~REF_FLAG_PENDING_REDIRECT;
 		if (ref->flags & REF_FLAG_PENDING_REMOVE) {
-		    struct ref *tmp = ref;
+		    struct hipe_ref *tmp = ref;
 		    ref = ref->next;
 		    *prev = ref;
 		    erts_free(ERTS_ALC_T_HIPE, tmp);
-- 
cgit v1.2.3


From 3bffb5797c9498e2294b65ba24abec5842655a11 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 15 Sep 2016 15:04:32 +0200
Subject: erts: Refactor rename struct hipe_sdesc

---
 erts/emulator/hipe/hipe_bif0.c       |  2 +-
 erts/emulator/hipe/hipe_gc.c         |  8 ++++----
 erts/emulator/hipe/hipe_risc_gc.h    | 16 ++++++++--------
 erts/emulator/hipe/hipe_risc_glue.h  |  2 +-
 erts/emulator/hipe/hipe_risc_stack.c | 12 ++++++------
 erts/emulator/hipe/hipe_stack.c      | 28 ++++++++++++++--------------
 erts/emulator/hipe/hipe_stack.h      | 32 ++++++++++++++++----------------
 erts/emulator/hipe/hipe_x86_gc.h     | 18 +++++++++---------
 erts/emulator/hipe/hipe_x86_glue.h   |  2 +-
 erts/emulator/hipe/hipe_x86_stack.c  | 14 +++++++-------
 10 files changed, 67 insertions(+), 67 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index b6d0abb2d1..95c2d73971 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -665,7 +665,7 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3)
 
 BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1)
 {
-    struct sdesc *sdesc;
+    struct hipe_sdesc *sdesc;
 
     sdesc = hipe_decode_sdesc(BIF_ARG_1);
     if (!sdesc) {
diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c
index 68c65dea27..be665ccdb9 100644
--- a/erts/emulator/hipe/hipe_gc.c
+++ b/erts/emulator/hipe/hipe_gc.c
@@ -38,7 +38,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
     /* known nstack walk state */
     Eterm *nsp;
     Eterm *nsp_end;
-    const struct sdesc *sdesc;
+    const struct hipe_sdesc *sdesc;
     unsigned int sdesc_size;
     unsigned long ra;
     unsigned int i;
@@ -123,7 +123,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
     /* known nstack walk state */
     Eterm *nsp;
     Eterm *nsp_end;
-    const struct sdesc *sdesc;
+    const struct hipe_sdesc *sdesc;
     unsigned int sdesc_size;
     unsigned long ra;
     unsigned int i;
@@ -244,7 +244,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area,
     /* known nstack walk state */
     Eterm *nsp;
     Eterm *nsp_end;
-    const struct sdesc *sdesc;
+    const struct hipe_sdesc *sdesc;
     /* arch-specific nstack walk state */
     struct nstack_walk_state walk_state;
 
@@ -311,7 +311,7 @@ nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size)
 {
     Eterm *nsp;
     Eterm *nsp_end;
-    const struct sdesc *sdesc;
+    const struct hipe_sdesc *sdesc;
     /* arch-specific nstack walk state */
     struct nstack_walk_state walk_state;
 
diff --git a/erts/emulator/hipe/hipe_risc_gc.h b/erts/emulator/hipe/hipe_risc_gc.h
index 09568c140e..f019434f67 100644
--- a/erts/emulator/hipe/hipe_risc_gc.h
+++ b/erts/emulator/hipe/hipe_risc_gc.h
@@ -27,7 +27,7 @@
 /* arch wrapper includes hipe_${arch}_asm.h to define NR_ARG_REGS */
 
 struct nstack_walk_state {
-    const struct sdesc *sdesc0;	/* .sdesc0 must be a pointer rvalue */
+    const struct hipe_sdesc *sdesc0;	/* .sdesc0 must be a pointer rvalue */
 };
 
 static inline int nstack_walk_init_check(const Process *p)
@@ -43,20 +43,20 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p)
     return p->hipe.nsp + nstkarity;
 }
 
-static inline const struct sdesc*
+static inline const struct hipe_sdesc*
 nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state)
 {
-    const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra);
+    const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra);
     state->sdesc0 = sdesc;
     return sdesc;
 }
 
-static inline const struct sdesc*
+static inline const struct hipe_sdesc*
 nstack_walk_init_sdesc_ignore_trap(const Process *p,
 				   struct nstack_walk_state *state)
 {
     unsigned long ra = (unsigned long)p->hipe.nra;
-    const struct sdesc *sdesc;
+    const struct hipe_sdesc *sdesc;
     if (ra == (unsigned long)&nbif_stack_trap_ra)
 	ra = (unsigned long)p->hipe.ngra;
     sdesc = hipe_find_sdesc(ra);
@@ -64,7 +64,7 @@ nstack_walk_init_sdesc_ignore_trap(const Process *p,
     return sdesc;
 }
 
-static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0)
+static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0)
 {
     Eterm *nsp = p->hipe.nsp;
     p->hipe.nsp = nstack_walk_nsp_begin(p);
@@ -103,7 +103,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp
     return nsp >= nsp_end;
 }
 
-static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc)
+static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc)
 {
     return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc);
 }
@@ -114,7 +114,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i)
 }
 
 static inline unsigned long
-nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc)
+nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc)
 {
     return nsp[sdesc_fsize(sdesc)];
 }
diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h
index 0284265307..3d84731cbe 100644
--- a/erts/emulator/hipe/hipe_risc_glue.h
+++ b/erts/emulator/hipe/hipe_risc_glue.h
@@ -66,7 +66,7 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y)
 
 static __inline__ void hipe_arch_glue_init(void)
 {
-    static struct sdesc_with_exnra nbif_return_sdesc = {
+    static struct hipe_sdesc_with_exnra nbif_return_sdesc = {
 	.exnra = (unsigned long)&nbif_fail,
 	.sdesc = {
 	    .bucket = { .hvalue = (unsigned long)&nbif_return },
diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c
index dc98c96b8f..7528a3ab00 100644
--- a/erts/emulator/hipe/hipe_risc_stack.c
+++ b/erts/emulator/hipe/hipe_risc_stack.c
@@ -56,8 +56,8 @@ void hipe_print_nstack(Process *p)
 {
     Eterm *nsp;
     Eterm *nsp_end;
-    const struct sdesc *sdesc1;
-    const struct sdesc *sdesc;
+    const struct hipe_sdesc *sdesc1;
+    const struct hipe_sdesc *sdesc;
     unsigned long ra;
     unsigned long exnra;
     unsigned int mask;
@@ -175,7 +175,7 @@ void hipe_print_nstack(Process *p)
 #define MINSTACK	128
 #define NSKIPFRAMES	4
 
-void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc)
+void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc)
 {
     Eterm *nsp;
     Eterm *nsp_end;
@@ -216,7 +216,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc)
 void (*hipe_handle_stack_trap(Process *p))(void)
 {
     void (*ngra)(void) = p->hipe.ngra;
-    const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra);
+    const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra);
     hipe_update_stack_trap(p, sdesc);
     return ngra;
 }
@@ -237,7 +237,7 @@ void hipe_find_handler(Process *p)
     unsigned long ra;
     unsigned long exnra;
     unsigned int arity;
-    const struct sdesc *sdesc;
+    const struct hipe_sdesc *sdesc;
 
     nsp = p->hipe.nsp;
     nsp_end = p->hipe.nstend;
@@ -277,7 +277,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace)
     Eterm *nsp_end;
     unsigned long ra, prev_ra;
     unsigned int arity;
-    const struct sdesc *sdesc;
+    const struct hipe_sdesc *sdesc;
     int i;
 
     if (depth < 1)
diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c
index e2e6eb74b1..a26132431b 100644
--- a/erts/emulator/hipe/hipe_stack.c
+++ b/erts/emulator/hipe/hipe_stack.c
@@ -43,10 +43,10 @@
  */
 struct hipe_sdesc_table hipe_sdesc_table;
 
-static struct sdesc **alloc_bucket(unsigned int size)
+static struct hipe_sdesc **alloc_bucket(unsigned int size)
 {
-    unsigned long nbytes = size * sizeof(struct sdesc*);
-    struct sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes);
+    unsigned long nbytes = size * sizeof(struct hipe_sdesc*);
+    struct hipe_sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes);
     sys_memzero(bucket, nbytes);
     return bucket;
 }
@@ -54,7 +54,7 @@ static struct sdesc **alloc_bucket(unsigned int size)
 static void hipe_grow_sdesc_table(void)
 {
     unsigned int old_size, new_size, new_mask;
-    struct sdesc **old_bucket, **new_bucket;
+    struct hipe_sdesc **old_bucket, **new_bucket;
     unsigned int i;
 
     old_size = 1 << hipe_sdesc_table.log2size;
@@ -66,9 +66,9 @@ static void hipe_grow_sdesc_table(void)
     new_bucket = alloc_bucket(new_size);
     hipe_sdesc_table.bucket = new_bucket;
     for (i = 0; i < old_size; ++i) {
-	struct sdesc *b = old_bucket[i];
+	struct hipe_sdesc *b = old_bucket[i];
 	while (b != NULL) {
-	    struct sdesc *next = b->bucket.next;
+	    struct hipe_sdesc *next = b->bucket.next;
 	    unsigned int j = (b->bucket.hvalue >> HIPE_RA_LSR_COUNT) & new_mask;
 	    b->bucket.next = new_bucket[j];
 	    new_bucket[j] = b;
@@ -78,11 +78,11 @@ static void hipe_grow_sdesc_table(void)
     erts_free(ERTS_ALC_T_HIPE, old_bucket);
 }
 
-struct sdesc *hipe_put_sdesc(struct sdesc *sdesc)
+struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc *sdesc)
 {
     unsigned long ra;
     unsigned int i;
-    struct sdesc *chain;
+    struct hipe_sdesc *chain;
     unsigned int size;
 
     ra = sdesc->bucket.hvalue;
@@ -102,7 +102,7 @@ struct sdesc *hipe_put_sdesc(struct sdesc *sdesc)
     return sdesc;
 }
 
-void hipe_init_sdesc_table(struct sdesc *sdesc)
+void hipe_init_sdesc_table(struct hipe_sdesc *sdesc)
 {
     unsigned int log2size, size;
 
@@ -121,14 +121,14 @@ void hipe_init_sdesc_table(struct sdesc *sdesc)
  * representation. If different representations are needed in
  * the future, this code has to be made target dependent.
  */
-struct sdesc *hipe_decode_sdesc(Eterm arg)
+struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
 {
     Uint ra, exnra;
     Eterm *live;
     Uint fsize, arity, nlive, i, nslots, off;
     Uint livebitswords, sdescbytes;
     void *p;
-    struct sdesc *sdesc;
+    struct hipe_sdesc *sdesc;
 
     if (is_not_tuple(arg) ||
 	(tuple_val(arg))[0] != make_arityval(6) ||
@@ -159,14 +159,14 @@ struct sdesc *hipe_decode_sdesc(Eterm arg)
     /* Calculate number of bytes needed for the stack descriptor. */
     sdescbytes =
 	(exnra
-	 ? offsetof(struct sdesc_with_exnra, sdesc.livebits)
-	 : offsetof(struct sdesc, livebits))
+	 ? offsetof(struct hipe_sdesc_with_exnra, sdesc.livebits)
+	 : offsetof(struct hipe_sdesc, livebits))
 	+ livebitswords * sizeof(int);
     p = erts_alloc(ERTS_ALC_T_HIPE, sdescbytes);
     /* If we have an exception handler use the
        special sdesc_with_exnra structure. */
     if (exnra) {
-	struct sdesc_with_exnra *sdesc_we = p;
+	struct hipe_sdesc_with_exnra *sdesc_we = p;
 	sdesc_we->exnra = exnra;
 	sdesc = &(sdesc_we->sdesc);
     } else
diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h
index afa0ed4256..e9d05bf99c 100644
--- a/erts/emulator/hipe/hipe_stack.h
+++ b/erts/emulator/hipe/hipe_stack.h
@@ -30,10 +30,10 @@
 
 #include <stddef.h>	/* offsetof() */
 
-struct sdesc {
+struct hipe_sdesc {
     struct {
 	unsigned long hvalue;	/* return address */
-	struct sdesc *next;	/* hash collision chain */
+	struct hipe_sdesc *next;	/* hash collision chain */
     } bucket;
     unsigned int summary; /* frame size, exn handler presence flag, arity */
 #ifdef DEBUG
@@ -43,27 +43,27 @@ struct sdesc {
     unsigned int livebits[1]; /* size depends on arch & data in summary field */
 };
 
-struct sdesc_with_exnra {
+struct hipe_sdesc_with_exnra {
     unsigned long exnra;
-    struct sdesc sdesc;
+    struct hipe_sdesc sdesc;
 };
 
-static __inline__ unsigned int sdesc_fsize(const struct sdesc *sdesc)
+static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc)
 {
     return sdesc->summary >> 9;
 }
 
-static __inline__ unsigned int sdesc_arity(const struct sdesc *sdesc)
+static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc)
 {
     return sdesc->summary & 0xFF;
 }
 
-static __inline__ unsigned long sdesc_exnra(const struct sdesc *sdesc)
+static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc)
 {
     if ((sdesc->summary & (1<<8))) {
 	const char *tmp;
-	tmp = (const char*)sdesc - offsetof(struct sdesc_with_exnra, sdesc);
-	return ((const struct sdesc_with_exnra*)tmp)->exnra;
+	tmp = (const char*)sdesc - offsetof(struct hipe_sdesc_with_exnra, sdesc);
+	return ((const struct hipe_sdesc_with_exnra*)tmp)->exnra;
     }
     return 0;
 }
@@ -72,13 +72,13 @@ struct hipe_sdesc_table {
     unsigned int log2size;
     unsigned int mask;		/* INV: mask == (1 << log2size)-1 */
     unsigned int used;
-    struct sdesc **bucket;
+    struct hipe_sdesc **bucket;
 };
 extern struct hipe_sdesc_table hipe_sdesc_table;
 
-extern struct sdesc *hipe_put_sdesc(struct sdesc*);
-extern void hipe_init_sdesc_table(struct sdesc*);
-extern struct sdesc *hipe_decode_sdesc(Eterm);
+extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*);
+extern void hipe_init_sdesc_table(struct hipe_sdesc*);
+extern struct hipe_sdesc *hipe_decode_sdesc(Eterm);
 
 #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
 #define __builtin_expect(x, expected_value) (x)
@@ -86,10 +86,10 @@ extern struct sdesc *hipe_decode_sdesc(Eterm);
 #define likely(x)	__builtin_expect((x),1)
 #define unlikely(x)	__builtin_expect((x),0)
 
-static __inline__ const struct sdesc *hipe_find_sdesc(unsigned long ra)
+static __inline__ const struct hipe_sdesc *hipe_find_sdesc(unsigned long ra)
 {
     unsigned int i = (ra >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask;
-    const struct sdesc *sdesc = hipe_sdesc_table.bucket[i];
+    const struct hipe_sdesc *sdesc = hipe_sdesc_table.bucket[i];
     if (likely(sdesc->bucket.hvalue == ra))
 	return sdesc;
     do {
@@ -103,7 +103,7 @@ AEXTERN(void,nbif_stack_trap_ra,(void));
 extern void hipe_print_nstack(Process*);
 extern void hipe_find_handler(Process*);
 extern void (*hipe_handle_stack_trap(Process*))(void);
-extern void hipe_update_stack_trap(Process*, const struct sdesc*);
+extern void hipe_update_stack_trap(Process*, const struct hipe_sdesc*);
 extern int hipe_fill_stacktrace(Process*, int, Eterm**);
 
 #if 0 && defined(HIPE_NSTACK_GROWS_UP)
diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h
index 00fe03d8f9..c025259871 100644
--- a/erts/emulator/hipe/hipe_x86_gc.h
+++ b/erts/emulator/hipe/hipe_x86_gc.h
@@ -30,9 +30,9 @@
 
 struct nstack_walk_state {
 #ifdef SKIP_YOUNGEST_FRAME
-    const struct sdesc *sdesc0;	/* .sdesc0 must be a pointer rvalue */
+    const struct hipe_sdesc *sdesc0;	/* .sdesc0 must be a pointer rvalue */
 #else
-    struct sdesc sdesc0[1];	/* .sdesc0 must be a pointer rvalue */
+    struct hipe_sdesc sdesc0[1];	/* .sdesc0 must be a pointer rvalue */
 #endif
 };
 
@@ -57,11 +57,11 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p)
 #endif
 }
 
-static inline const struct sdesc*
+static inline const struct hipe_sdesc*
 nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state)
 {
 #ifdef SKIP_YOUNGEST_FRAME
-    const struct sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]);
+    const struct hipe_sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]);
     state->sdesc0 = sdesc;
     return sdesc;
 #else
@@ -81,13 +81,13 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state)
 #endif
 }
 
-static inline const struct sdesc*
+static inline const struct hipe_sdesc*
 nstack_walk_init_sdesc_ignore_trap(const Process *p,
 				   struct nstack_walk_state *state)
 {
 #ifdef SKIP_YOUNGEST_FRAME
     unsigned long ra = p->hipe.nsp[0];
-    const struct sdesc *sdesc;
+    const struct hipe_sdesc *sdesc;
     if (ra == (unsigned long)nbif_stack_trap_ra)
 	ra = (unsigned long)p->hipe.ngra;
     sdesc = hipe_find_sdesc(ra);
@@ -98,7 +98,7 @@ nstack_walk_init_sdesc_ignore_trap(const Process *p,
 #endif
 }
 
-static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0)
+static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0)
 {
 #ifdef SKIP_YOUNGEST_FRAME
     Eterm *nsp = p->hipe.nsp;
@@ -137,7 +137,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp
     return nsp >= nsp_end;
 }
 
-static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc)
+static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc)
 {
     return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc);
 }
@@ -148,7 +148,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i)
 }
 
 static inline unsigned long
-nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc)
+nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc)
 {
     return nsp[sdesc_fsize(sdesc)];
 }
diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h
index 818d7444e2..d0c8eec2bc 100644
--- a/erts/emulator/hipe/hipe_x86_glue.h
+++ b/erts/emulator/hipe/hipe_x86_glue.h
@@ -58,7 +58,7 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y)
 
 static __inline__ void hipe_arch_glue_init(void)
 {
-    static struct sdesc_with_exnra nbif_return_sdesc = {
+    static struct hipe_sdesc_with_exnra nbif_return_sdesc = {
 	.exnra = (unsigned long)nbif_fail,
 	.sdesc = {
 	    .bucket = { .hvalue = (unsigned long)nbif_return },
diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c
index f1559b1451..6975cc4669 100644
--- a/erts/emulator/hipe/hipe_x86_stack.c
+++ b/erts/emulator/hipe/hipe_x86_stack.c
@@ -52,9 +52,9 @@ void hipe_print_nstack(Process *p)
 {
     Eterm *nsp;
     Eterm *nsp_end;
-    struct sdesc sdesc0;
-    const struct sdesc *sdesc1;
-    const struct sdesc *sdesc;
+    struct hipe_sdesc sdesc0;
+    const struct hipe_sdesc *sdesc1;
+    const struct hipe_sdesc *sdesc;
     unsigned long ra;
     unsigned long exnra;
     unsigned int mask;
@@ -158,7 +158,7 @@ void hipe_print_nstack(Process *p)
 #define MINSTACK	128
 #define NSKIPFRAMES	4
 
-void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc)
+void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc)
 {
     Eterm *nsp;
     Eterm *nsp_end;
@@ -199,7 +199,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc)
 void (*hipe_handle_stack_trap(Process *p))(void)
 {
     void (*ngra)(void) = p->hipe.ngra;
-    const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra);
+    const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra);
     hipe_update_stack_trap(p, sdesc);
     return ngra;
 }
@@ -220,7 +220,7 @@ void hipe_find_handler(Process *p)
     unsigned long ra;
     unsigned long exnra;
     unsigned int arity;
-    const struct sdesc *sdesc;
+    const struct hipe_sdesc *sdesc;
     unsigned int nstkarity;
 
     nsp = p->hipe.nsp;
@@ -262,7 +262,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace)
     Eterm *nsp_end;
     unsigned long ra, prev_ra;
     unsigned int arity;
-    const struct sdesc *sdesc;
+    const struct hipe_sdesc *sdesc;
     unsigned int nstkarity;
     int i;
 
-- 
cgit v1.2.3


From 9cb85a8cd1e02263f34c57aac68d16a2f1c03180 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 4 Oct 2016 12:26:54 +0200
Subject: erts: Refactor hipe_sdesc.summary into bit fields

---
 erts/emulator/hipe/hipe_risc_glue.h |  4 +++-
 erts/emulator/hipe/hipe_stack.c     |  4 +++-
 erts/emulator/hipe/hipe_stack.h     | 10 ++++++----
 erts/emulator/hipe/hipe_x86_gc.h    |  4 +++-
 erts/emulator/hipe/hipe_x86_glue.h  |  4 +++-
 erts/emulator/hipe/hipe_x86_stack.c |  4 +++-
 6 files changed, 21 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h
index 3d84731cbe..09804e3016 100644
--- a/erts/emulator/hipe/hipe_risc_glue.h
+++ b/erts/emulator/hipe/hipe_risc_glue.h
@@ -70,7 +70,9 @@ static __inline__ void hipe_arch_glue_init(void)
 	.exnra = (unsigned long)&nbif_fail,
 	.sdesc = {
 	    .bucket = { .hvalue = (unsigned long)&nbif_return },
-	    .summary = (1<<8),
+            .fsize = 0,
+            .has_exnra = 1,
+            .arity = 0
 	},
     };
     hipe_init_sdesc_table(&nbif_return_sdesc.sdesc);
diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c
index a26132431b..17bef0718c 100644
--- a/erts/emulator/hipe/hipe_stack.c
+++ b/erts/emulator/hipe/hipe_stack.c
@@ -175,7 +175,9 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
     /* Initialise head of sdesc. */
     sdesc->bucket.next = 0;
     sdesc->bucket.hvalue = ra;
-    sdesc->summary = (fsize << 9) | (exnra ? (1<<8) : 0) | arity;
+    sdesc->fsize = fsize;
+    sdesc->has_exnra = (exnra ? 1 : 0);
+    sdesc->arity = arity;
     /* Clear all live-bits */
     for (i = 0; i < livebitswords; ++i)
 	sdesc->livebits[i] = 0;
diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h
index e9d05bf99c..81f2e53dbc 100644
--- a/erts/emulator/hipe/hipe_stack.h
+++ b/erts/emulator/hipe/hipe_stack.h
@@ -35,7 +35,9 @@ struct hipe_sdesc {
 	unsigned long hvalue;	/* return address */
 	struct hipe_sdesc *next;	/* hash collision chain */
     } bucket;
-    unsigned int summary; /* frame size, exn handler presence flag, arity */
+    unsigned int fsize : 23;    /* frame size */
+    unsigned int has_exnra : 1; /* exn handler presence flag */
+    unsigned int arity : 8;
 #ifdef DEBUG
     Eterm dbg_M, dbg_F;
     unsigned dbg_A;
@@ -50,17 +52,17 @@ struct hipe_sdesc_with_exnra {
 
 static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc)
 {
-    return sdesc->summary >> 9;
+    return sdesc->fsize;
 }
 
 static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc)
 {
-    return sdesc->summary & 0xFF;
+    return sdesc->arity;
 }
 
 static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc)
 {
-    if ((sdesc->summary & (1<<8))) {
+    if (sdesc->has_exnra) {
 	const char *tmp;
 	tmp = (const char*)sdesc - offsetof(struct hipe_sdesc_with_exnra, sdesc);
 	return ((const struct hipe_sdesc_with_exnra*)tmp)->exnra;
diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h
index c025259871..7802076f70 100644
--- a/erts/emulator/hipe/hipe_x86_gc.h
+++ b/erts/emulator/hipe/hipe_x86_gc.h
@@ -68,7 +68,9 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state)
     unsigned int nstkarity = p->hipe.narity - NR_ARG_REGS;
     if ((int)nstkarity < 0)
 	nstkarity = 0;
-    state->sdesc0[0].summary = (0 << 9) | (0 << 8) | nstkarity;
+    state->sdesc0[0].fsize = 0;
+    state->sdesc0[0].has_exnra = 0;
+    state->sdesc0[0].arity = nstkarity;
     state->sdesc0[0].livebits[0] = 0;
 # ifdef DEBUG
     state->sdesc0[0].dbg_M = 0;
diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h
index d0c8eec2bc..0e9fcd61f1 100644
--- a/erts/emulator/hipe/hipe_x86_glue.h
+++ b/erts/emulator/hipe/hipe_x86_glue.h
@@ -62,7 +62,9 @@ static __inline__ void hipe_arch_glue_init(void)
 	.exnra = (unsigned long)nbif_fail,
 	.sdesc = {
 	    .bucket = { .hvalue = (unsigned long)nbif_return },
-	    .summary = (1<<8),
+            .fsize = 0,
+            .has_exnra = 1,
+            .arity = 0,
 	#ifdef DEBUG
 	    .dbg_F = am_return,
 	#endif
diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c
index 6975cc4669..9426b166e2 100644
--- a/erts/emulator/hipe/hipe_x86_stack.c
+++ b/erts/emulator/hipe/hipe_x86_stack.c
@@ -71,7 +71,9 @@ void hipe_print_nstack(Process *p)
     nstkarity = p->hipe.narity - NR_ARG_REGS;
     if ((int)nstkarity < 0)
 	nstkarity = 0;
-    sdesc0.summary = nstkarity;
+    sdesc0.fsize = 0;
+    sdesc0.has_exnra = 0;
+    sdesc0.arity = nstkarity;
     sdesc0.livebits[0] = ~1;
     sdesc = &sdesc0;
 
-- 
cgit v1.2.3


From 65737b70b9b71d53147e4155f8974b0fad01e1f6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Sun, 8 May 2016 15:54:33 +0200
Subject: erts: Refactor module_start_staging

with a copy_module() function.
---
 erts/emulator/beam/module.c | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index 4f36377450..d4f6e17c56 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -181,6 +181,13 @@ static ErtsCodeIndex dbg_load_code_ix = 0;
 
 static int entries_at_start_staging = 0;
 
+static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod)
+{
+    dst_mod->curr = src_mod->curr;
+    dst_mod->old = src_mod->old;
+    dst_mod->on_load = src_mod->on_load;
+}
+
 void module_start_staging(void)
 {
     IndexTable* src = &module_tables[erts_active_code_ix()];
@@ -199,10 +206,7 @@ void module_start_staging(void)
 	src_mod = (Module*) erts_index_lookup(src, i);
 	dst_mod = (Module*) erts_index_lookup(dst, i);
 	ASSERT(src_mod->module == dst_mod->module);
-
-	dst_mod->curr = src_mod->curr;
-	dst_mod->old = src_mod->old;
-	dst_mod->on_load = src_mod->on_load;
+        copy_module(dst_mod, src_mod);
     }
 
     /*
@@ -213,10 +217,7 @@ void module_start_staging(void)
 	src_mod = (Module*) erts_index_lookup(src, i);
 	dst_mod = (Module*) index_put_entry(dst, src_mod);
 	ASSERT(dst_mod != src_mod);
-
-	dst_mod->curr = src_mod->curr;
-	dst_mod->old = src_mod->old;
-	dst_mod->on_load = src_mod->on_load;
+        copy_module(dst_mod, src_mod);
     }
     newsz = index_table_sz(dst);
     erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz));
-- 
cgit v1.2.3


From 099c60de4033d7b397d4b3fb47f183b52fcba855 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 15 Sep 2016 21:21:10 +0200
Subject: erts: Improve hipe load/upgrade/purge machinery

A step toward better integration of hipe load and purge

Highlights:

* code_server no longer needs to call hipe_unified_loader:post_beam_load/1
  Instead new internal function hipe_redirect_to_module()
  is called by loading BIFs to patch native call sites if needed.

* hipe_purge_module() is called by erts_internal:purge_module/2
  to purge any native code.

* struct hipe_mfa_info redesigned and only used for exported
  functions that are called from or implemented by native code.
  A list of native call sites (struct hipe_ref) are kept for each hipe_mfa_info.

* struct hipe_sdesc used by hipe_find_mfa_from_ra()
  to build native stack traces.
---
 erts/emulator/beam/beam_bif_load.c  |  91 +++--
 erts/emulator/beam/beam_emu.c       |   1 +
 erts/emulator/beam/beam_load.c      |  94 ++++-
 erts/emulator/beam/beam_load.h      |  20 +
 erts/emulator/beam/export.c         |   8 +
 erts/emulator/beam/module.c         |  69 +++-
 erts/emulator/beam/module.h         |  15 +
 erts/emulator/hipe/hipe_amd64.c     |  15 +
 erts/emulator/hipe/hipe_arch.h      |  10 +-
 erts/emulator/hipe/hipe_arm.c       |  10 +
 erts/emulator/hipe/hipe_bif0.c      | 764 +++++++++++++++++++-----------------
 erts/emulator/hipe/hipe_bif0.h      |   5 +-
 erts/emulator/hipe/hipe_bif0.tab    |   6 +-
 erts/emulator/hipe/hipe_ppc.c       |  10 +
 erts/emulator/hipe/hipe_risc_glue.h |  20 +-
 erts/emulator/hipe/hipe_stack.c     |  90 +++--
 erts/emulator/hipe/hipe_stack.h     |  18 +-
 erts/emulator/hipe/hipe_x86.c       |  14 +
 erts/emulator/hipe/hipe_x86_gc.h    |  14 +-
 erts/emulator/hipe/hipe_x86_glue.h  |  23 +-
 erts/emulator/hipe/hipe_x86_stack.c |   7 +-
 erts/emulator/test/code_SUITE.erl   |  22 +-
 22 files changed, 813 insertions(+), 513 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 153fa205ed..237513095a 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -36,6 +36,12 @@
 #include "erl_nif.h"
 #include "erl_bits.h"
 #include "erl_thr_progress.h"
+#ifdef HIPE
+#  include "hipe_bif0.h"
+#  define IF_HIPE(X) (X)
+#else
+#  define IF_HIPE(X) (0)
+#endif
 
 #ifdef HIPE
 #  include "hipe_stack.h"
@@ -169,6 +175,11 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
     if (res == BIF_ARG_1) {
 	erts_end_staging_code_ix();
 	erts_commit_staging_code_ix();
+#ifdef HIPE
+        if (!modp)
+	    modp = erts_get_module(BIF_ARG_1, erts_active_code_ix());
+        hipe_redirect_to_module(modp);
+#endif
     }
     else {
 	erts_abort_staging_code_ix();
@@ -241,7 +252,7 @@ struct m {
     Uint exception;
 };
 
-static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int);
+static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int, int);
 #ifdef ERTS_SMP
 static void smp_code_ix_commiter(void*);
 
@@ -377,8 +388,9 @@ finish_loading_1(BIF_ALIST_1)
     for (i = 0; i < n; i++) {
 	if (p[i].modp->curr.num_breakpoints > 0 ||
 	    p[i].modp->curr.num_traced_exports > 0 ||
-	    erts_is_default_trace_enabled()) {
-	    /* tracing involved, fallback with thread blocking */
+	    erts_is_default_trace_enabled() ||
+	    IF_HIPE(hipe_need_blocking(p[i].modp))) {
+	    /* tracing or hipe need thread blocking */
 	    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	    erts_smp_thr_progress_block();
 	    is_blocking = 1;
@@ -436,32 +448,36 @@ finish_loading_1(BIF_ALIST_1)
     }
 
 done:
-    return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n);
+    return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n, 1);
 }
 
 static Eterm
 staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
-		 struct m* loaded, int nloaded)
+		 struct m* mods, int nmods, int free_mods)
 {    
 #ifdef ERTS_SMP
     if (is_blocking || !commit)
 #endif
     {
 	if (commit) {
+	    int i;
 	    erts_end_staging_code_ix();
 	    erts_commit_staging_code_ix();
-	    if (loaded) {
-		int i;
-		for (i=0; i < nloaded; i++) {		
-		    set_default_trace_pattern(loaded[i].module);
+
+	    for (i=0; i < nmods; i++) {
+		if (mods[i].modp->curr.code_hdr) {
+		    set_default_trace_pattern(mods[i].module);
 		}
+	      #ifdef HIPE
+		hipe_redirect_to_module(mods[i].modp);
+	      #endif
 	    }
 	}
 	else {
 	    erts_abort_staging_code_ix();
 	}
-	if (loaded) {
-	    erts_free(ERTS_ALC_T_LOADER_TMP, loaded);
+	if (free_mods) {
+	    erts_free(ERTS_ALC_T_LOADER_TMP, mods);
 	}
 	if (is_blocking) {
 	    erts_smp_thr_progress_unblock();
@@ -474,8 +490,8 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
     else {
 	ASSERT(is_value(res));
 
-	if (loaded) {
-	    erts_free(ERTS_ALC_T_LOADER_TMP, loaded);
+	if (free_mods) {
+	    erts_free(ERTS_ALC_T_LOADER_TMP, mods);
 	}
 	erts_end_staging_code_ix();
 	/*
@@ -653,8 +669,9 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
 	}
 	else {
 	    if (modp->curr.num_breakpoints > 0 ||
-		modp->curr.num_traced_exports > 0) {
-		/* we have tracing, retry single threaded */
+		modp->curr.num_traced_exports > 0 ||
+		IF_HIPE(hipe_need_blocking(modp))) {
+		/* tracing or hipe need to go single threaded */
 		erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 		erts_smp_thr_progress_block();
 		is_blocking = 1;
@@ -668,7 +685,14 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
 	    success = 1;
 	}
     }
-    return staging_epilogue(BIF_P, success, res, is_blocking, NULL, 0);  
+    {
+	struct m mod;
+	Eterm retval;
+	mod.module = BIF_ARG_1;
+	mod.modp = modp;
+	retval = staging_epilogue(BIF_P, success, res, is_blocking, &mod, 1, 0);
+	return retval;
+    }
 }
 
 BIF_RETTYPE module_loaded_1(BIF_ALIST_1)
@@ -809,6 +833,9 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	}
 	modp->curr.code_hdr->on_load_function_ptr = NULL;
 	set_default_trace_pattern(BIF_ARG_1);
+      #ifdef HIPE
+        hipe_redirect_to_module(modp);
+      #endif
     } else if (BIF_ARG_2 == am_false) {
 	int i;
 
@@ -1619,15 +1646,18 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 		/*
 		 * Unload any NIF library
 		 */
-		if (modp->old.nif != NULL) {
+		if (modp->old.nif != NULL
+		    || IF_HIPE(hipe_purge_need_blocking(modp))) {
 		    /* ToDo: Do unload nif without blocking */
 		    erts_rwunlock_old_code(code_ix);
 		    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 		    erts_smp_thr_progress_block();
 		    is_blocking = 1;
 		    erts_rwlock_old_code(code_ix);
-		    erts_unload_nif(modp->old.nif);
-		    modp->old.nif = NULL;
+		    if (modp->old.nif) {
+		      erts_unload_nif(modp->old.nif);
+		      modp->old.nif = NULL;
+		    }
 		}
 
 		/*
@@ -1646,7 +1676,9 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 		modp->old.code_length = 0;
 		modp->old.catches = BEAM_CATCHES_NIL;
 		erts_remove_from_ranges(code);
-
+#ifdef HIPE
+		hipe_purge_module(modp);
+#endif
 		ERTS_BIF_PREP_RET(ret, am_true);
 	    }
 
@@ -1719,6 +1751,8 @@ delete_code(Module* modp)
 			 (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
 		    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
 		    ASSERT(modp->curr.num_traced_exports > 0);
+		    DBG_TRACE_MFA(ep->code[0],ep->code[1],ep->code[2],
+				  "export trace cleared, code_ix=%d", code_ix);
 		    erts_clear_export_break(modp, ep->code+3);
 		}
 		else ASSERT(ep->code[3] == (BeamInstr) em_call_error_handler
@@ -1727,17 +1761,16 @@ delete_code(Module* modp)
 	    ep->addressv[code_ix] = ep->code+3;
 	    ep->code[3] = (BeamInstr) em_call_error_handler;
 	    ep->code[4] = 0;
+	    DBG_TRACE_MFA(ep->code[0],ep->code[1],ep->code[2],
+			  "export invalidation, code_ix=%d", code_ix);
 	}
     }
 
     ASSERT(modp->curr.num_breakpoints == 0);
     ASSERT(modp->curr.num_traced_exports == 0);
+    DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "delete_code old.first_hipe_ref=%p", modp->curr.first_hipe_ref);
     modp->old = modp->curr;
-    modp->curr.code_hdr = NULL;
-    modp->curr.code_length = 0;
-    modp->curr.catches = BEAM_CATCHES_NIL;
-    modp->curr.nif = NULL;
-
+    erts_module_instance_init(&modp->curr);
 }
 
 
@@ -1751,9 +1784,11 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module)
      * if not, delete old code; error if old code already exists.
      */
 
-    if (modp->curr.code_hdr && modp->old.code_hdr)  {
-	return am_not_purged;
-    } else if (!modp->old.code_hdr) { /* Make the current version old. */
+    if (modp->curr.code_hdr) {
+	if (modp->old.code_hdr)  {
+	    return am_not_purged;
+	}
+	/* Make the current version old. */
 	delete_code(modp);
     }
     return NIL;
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 0ba06058a5..ccb0f786ec 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6105,6 +6105,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
     Uint sz;
     int i;
 
+    DBG_TRACE_MFA(fi[0], fi[1], fi[2], "call_error_handler");
     /*
      * Search for the error_handler module.
      */
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index d69b18e22f..9b206f9a23 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -482,7 +482,8 @@ static void free_literal_fragment(ErlHeapFragment*);
 static void loader_state_dtor(Binary* magic);
 static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 				  Eterm group_leader, Eterm module,
-				  BeamCodeHeader* code, Uint size);
+				  BeamCodeHeader* code, Uint size,
+				  void* hipe_code_start, UWord hipe_code_size);
 static int init_iff_file(LoaderState* stp, byte* code, Uint size);
 static int scan_iff_file(LoaderState* stp, Uint* chunk_types,
 			 Uint num_types, Uint num_mandatory);
@@ -842,9 +843,7 @@ erts_finish_loading(Binary* magic, Process* c_p,
 	    erts_alloc(ERTS_ALC_T_PREPARED_CODE,
 		       sizeof(struct erl_module_instance));
 	inst_p = mod_tab_p->on_load;
-	inst_p->nif = 0;
-        inst_p->num_breakpoints = 0;
-        inst_p->num_traced_exports = 0;
+        erts_module_instance_init(inst_p);
     }
 
     inst_p->code_hdr = stp->hdr;
@@ -1094,7 +1093,8 @@ loader_state_dtor(Binary* magic)
 static Eterm
 stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 		     Eterm group_leader, Eterm module,
-		     BeamCodeHeader* code_hdr, Uint size)
+		     BeamCodeHeader* code_hdr, Uint size,
+		     void* hipe_code_start, UWord hipe_code_size)
 {
     Module* modp;
     Eterm retval;
@@ -1117,6 +1117,17 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
     modp->curr.code_hdr = code_hdr;
     modp->curr.code_length = size;
     modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */
+#if defined(HIPE)
+    DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code new_hipe_refs = %p", modp->new_hipe_refs);
+    modp->curr.first_hipe_ref = modp->new_hipe_refs;
+    modp->curr.first_hipe_sdesc = modp->new_hipe_sdesc;
+    modp->curr.hipe_code_start = hipe_code_start;
+    modp->new_hipe_refs = NULL;
+    modp->new_hipe_sdesc = NULL;
+# ifdef DEBUG
+    modp->curr.hipe_code_size  = hipe_code_size;
+# endif
+#endif
 
     /*
      * Update ranges (used for finding a function from a PC value).
@@ -4773,9 +4784,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
 	}
 	ep = erts_export_put(stp->module, stp->export[i].function,
 			     stp->export[i].arity);
-	if (!on_load) {
-	    ep->addressv[erts_staging_code_ix()] = address;
-	} else {
+	if (on_load) {
 	    /*
 	     * on_load: Don't make any of the exported functions
 	     * callable yet. Keep any function in the current
@@ -4783,6 +4792,8 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
 	     */
 	    ep->code[4] = (BeamInstr) address;
 	}
+        else
+            ep->addressv[erts_staging_code_ix()] = address;
     }
 
     /*
@@ -5989,17 +6000,12 @@ code_module_md5_1(BIF_ALIST_1)
 static BeamInstr*
 make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode)
 {
+    DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", &fp[5]);
     fp[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
     fp[1] = native;
     fp[2] = mod;
     fp[3] = func;
     fp[4] = arity;
-#ifdef HIPE
-    if (native) {
-	fp[5] = BeamOpCode(op_move_return_n);
-	hipe_mfa_save_orig_beam_op(mod, func, arity, fp+5);
-    }
-#endif
     fp[5] = OpCode;
     return fp + WORDS_PER_FUNCTION;
 }
@@ -6087,6 +6093,8 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp)
 	if (stp->export[i].function == function && stp->export[i].arity == arity) {
 	    Export* ep = erts_export_put(mod, function, arity);
 	    ep->addressv[erts_staging_code_ix()] = fp+5;
+	    DBG_TRACE_MFA(mod,function,arity,"set beam stub at %p in export at %p (code_ix=%d)",
+			  fp+5, ep, erts_staging_code_ix());
 	    return;
 	}
     }
@@ -6282,6 +6290,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     byte* temp_alloc = NULL;
     byte* bytes;
     Uint size;
+    UWord hipe_code_start = NULL, hipe_code_size = 0;
 
     /*
      * Must initialize stp->lambdas here because the error handling code
@@ -6297,7 +6306,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 	goto error;
     }
     tp = tuple_val(Info);
-    if (tp[0] != make_arityval(3)) {
+    if (tp[0] != make_arityval(5)) {
       goto error;
     }
     Funcs = tp[1];
@@ -6314,6 +6323,15 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     }
     size = binary_size(Beam);
 
+#ifdef HIPE
+    if (!term_to_Uint(tp[4], &hipe_code_start))
+        goto error;
+# ifdef DEBUG
+    if (!term_to_Uint(tp[5], &hipe_code_size))
+        goto error;
+# endif
+#endif
+
     /*
      * Scan the Beam binary and read the interesting sections.
      */
@@ -6476,7 +6494,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
      */
 
     rval = stub_insert_new_code(p, 0, p->group_leader, Mod,
-				code_hdr, code_size);
+				code_hdr, code_size,
+				(void*)hipe_code_start, hipe_code_size);
     if (rval != NIL) {
 	goto error;
     }
@@ -6517,3 +6536,46 @@ static int safe_mul(UWord a, UWord b, UWord* resp)
     }
 }
 
+#ifdef ENABLE_DBG_TRACE_MFA
+
+#define MFA_MAX 10
+Eterm dbg_trace_m[MFA_MAX];
+Eterm dbg_trace_f[MFA_MAX];
+Uint  dbg_trace_a[MFA_MAX];
+unsigned int dbg_trace_ix = 0;
+
+void dbg_set_traced_mfa(const char* m, const char* f, Uint a)
+{
+    unsigned i = dbg_trace_ix++;
+    ASSERT(i < MFA_MAX);
+    dbg_trace_m[i] = am_atom_put(m, strlen(m));
+    dbg_trace_f[i] = am_atom_put(f, strlen(f));
+    dbg_trace_a[i] = a;
+}
+
+int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a)
+{
+    unsigned int i;
+    for (i = 0; i < dbg_trace_ix; ++i) {
+        if (m == dbg_trace_m[i] &&
+            (!f || (f == dbg_trace_f[i] && a == dbg_trace_a[i]))) {
+
+            return i+1;
+        }
+    }
+    return 0;
+}
+
+void dbg_vtrace_mfa(unsigned ix, const char* format, ...)
+{
+    va_list arglist;
+    va_start(arglist, format);
+    ASSERT(--ix < MFA_MAX);
+    erts_fprintf(stderr, "MFA TRACE %T:%T/%u: ",
+                 dbg_trace_m[ix], dbg_trace_f[ix], (int)dbg_trace_a[ix]);
+
+    erts_vfprintf(stderr, format, arglist);
+    va_end(arglist);
+}
+
+#endif /* ENABLE_DBG_TRACE_MFA */
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 6be4031822..5f0f34fcdd 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -151,4 +151,24 @@ struct BeamCodeLineTab_ {
 #define LOC_FILE(Loc) ((Loc) >> 24)
 #define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1))
 
+#ifdef DEBUG
+# define ENABLE_DBG_TRACE_MFA
+#endif
+
+#ifdef  ENABLE_DBG_TRACE_MFA
+
+void dbg_set_traced_mfa(const char* m, const char* f, Uint a);
+int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a);
+void dbg_vtrace_mfa(unsigned ix, const char* format, ...);
+#define DBG_TRACE_MFA(M,F,A,FMT, ...) do {\
+    unsigned ix;\
+    if ((ix=dbg_is_traced_mfa(M,F,A))) \
+        dbg_vtrace_mfa(ix, FMT"\n", ##__VA_ARGS__);\
+  }while(0)
+
+#else
+#  define dbg_set_traced_mfa(M,F,A)
+#  define DBG_TRACE_MFA(M,F,A,FMT, ...)
+#endif /* ENABLE_DBG_TRACE_MFA */
+
 #endif /* _BEAM_LOAD_H */
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 2a19211987..9da7fae1dc 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -145,6 +145,9 @@ export_alloc(struct export_entry* tmpl_e)
 	    blob->entryv[ix].ep = &blob->exp;
 	}
 	ix = 0;
+
+	DBG_TRACE_MFA(obj->code[0], obj->code[1], obj->code[2],
+		      "export allocation at %p", obj);
     }
     else { /* Existing entry in another table, use free entry in blob */
 	blob = entry_to_blob(tmpl_e);
@@ -163,9 +166,14 @@ export_free(struct export_entry* obj)
     obj->slot.index = -1;
     for (i=0; i < ERTS_NUM_CODE_IX; i++) {
 	if (blob->entryv[i].slot.index >= 0) {
+	    DBG_TRACE_MFA(blob->exp.code[0], blob->exp.code[1], blob->exp.code[2],
+			  "export entry slot %u freed for %p",
+			  (obj - blob->entryv), &blob->exp);
 	    return;
 	}
     }
+    DBG_TRACE_MFA(blob->exp.code[0], blob->exp.code[1], blob->exp.code[2],
+		      "export blob deallocation at %p", &blob->exp);
     erts_free(ERTS_ALC_T_EXPORT, blob);
     erts_smp_atomic_add_nob(&total_entries_bytes, -sizeof(*blob));
 }
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index d4f6e17c56..93ce8fcf9f 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -26,6 +26,7 @@
 #include "erl_vm.h"
 #include "global.h"
 #include "module.h"
+#include "beam_catches.h"
 
 #ifdef DEBUG
 #  define IF_DEBUG(x) x
@@ -67,6 +68,23 @@ static int module_cmp(Module* tmpl, Module* obj)
     return tmpl->module != obj->module;
 }
 
+void erts_module_instance_init(struct erl_module_instance* modi)
+{
+    modi->code_hdr = 0;
+    modi->code_length = 0;
+    modi->catches = BEAM_CATCHES_NIL;
+    modi->nif = NULL;
+    modi->num_breakpoints = 0;
+    modi->num_traced_exports = 0;
+#ifdef HIPE
+    modi->first_hipe_ref = NULL;
+    modi->first_hipe_sdesc = NULL;
+    modi->hipe_code_start = NULL;
+# ifdef DEBUG
+    modi->hipe_code_size = 0;
+# endif
+#endif
+}
 
 static Module* module_alloc(Module* tmpl)
 {
@@ -74,18 +92,16 @@ static Module* module_alloc(Module* tmpl)
     erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module));
 
     obj->module = tmpl->module;
-    obj->curr.code_hdr = 0;
-    obj->old.code_hdr = 0;
-    obj->curr.code_length = 0;
-    obj->old.code_length = 0;
     obj->slot.index = -1;
-    obj->curr.nif = NULL;
-    obj->old.nif = NULL;
-    obj->curr.num_breakpoints = 0;
-    obj->old.num_breakpoints  = 0;
-    obj->curr.num_traced_exports = 0;
-    obj->old.num_traced_exports = 0;
+    erts_module_instance_init(&obj->curr);
+    erts_module_instance_init(&obj->old);
     obj->on_load = 0;
+#ifdef HIPE
+    obj->first_hipe_mfa = NULL;
+    obj->new_hipe_refs = NULL;
+    obj->new_hipe_sdesc = NULL;
+#endif
+    DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc");
     return obj;
 }
 
@@ -139,19 +155,14 @@ erts_get_module(Eterm mod, ErtsCodeIndex code_ix)
     }
 }
 
-Module*
-erts_put_module(Eterm mod)
+
+static Module* put_module(Eterm mod, IndexTable* mod_tab)
 {
     Module e;
-    IndexTable* mod_tab;
     int oldsz, newsz;
     Module* res;
 
     ASSERT(is_atom(mod));
-    ERTS_SMP_LC_ASSERT(erts_initialized == 0
-		       || erts_has_code_write_permission());
-
-    mod_tab = &module_tables[erts_staging_code_ix()];
     e.module = atom_val(mod);
     oldsz = index_table_sz(mod_tab);
     res = (Module*) index_put_entry(mod_tab, (void*) &e);
@@ -160,6 +171,24 @@ erts_put_module(Eterm mod)
     return res;
 }
 
+Module*
+erts_put_module(Eterm mod)
+{
+    ERTS_SMP_LC_ASSERT(erts_initialized == 0
+		       || erts_has_code_write_permission());
+
+    return put_module(mod, &module_tables[erts_staging_code_ix()]);
+}
+
+Module*
+erts_put_active_module(Eterm mod)
+{
+    ASSERT(is_atom(mod));
+    //SVERK Why not? ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+    return put_module(mod, &module_tables[erts_active_code_ix()]);
+}
+
 Module *module_code(int i, ErtsCodeIndex code_ix)
 {
     return (Module*) erts_index_lookup(&module_tables[code_ix], i);
@@ -186,6 +215,11 @@ static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod)
     dst_mod->curr = src_mod->curr;
     dst_mod->old = src_mod->old;
     dst_mod->on_load = src_mod->on_load;
+#ifdef HIPE
+    dst_mod->first_hipe_mfa = src_mod->first_hipe_mfa;
+    dst_mod->new_hipe_refs  = src_mod->new_hipe_refs;
+    dst_mod->new_hipe_sdesc = src_mod->new_hipe_sdesc;
+#endif
 }
 
 void module_start_staging(void)
@@ -217,6 +251,7 @@ void module_start_staging(void)
 	src_mod = (Module*) erts_index_lookup(src, i);
 	dst_mod = (Module*) index_put_entry(dst, src_mod);
 	ASSERT(dst_mod != src_mod);
+
         copy_module(dst_mod, src_mod);
     }
     newsz = index_table_sz(dst);
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index 1c1afc8461..694583597b 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -30,6 +30,14 @@ struct erl_module_instance {
     struct erl_module_nif* nif;
     int num_breakpoints;
     int num_traced_exports;
+#ifdef HIPE
+    struct hipe_ref* first_hipe_ref;  /* all external hipe calls from this module */
+    struct hipe_sdesc* first_hipe_sdesc;  /* all stack descriptors for this module */
+    void* hipe_code_start;
+# ifdef DEBUG
+    UWord hipe_code_size;
+# endif
+#endif
 };
 
 typedef struct erl_module {
@@ -40,10 +48,17 @@ typedef struct erl_module {
     struct erl_module_instance curr;
     struct erl_module_instance old; /* protected by "old_code" rwlock */
     struct erl_module_instance* on_load;
+#ifdef HIPE
+    struct hipe_mfa_info* first_hipe_mfa;
+    struct hipe_ref* new_hipe_refs;
+    struct hipe_sdesc* new_hipe_sdesc;
+#endif
 } Module; 
 
+void erts_module_instance_init(struct erl_module_instance* modi);
 Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix);
 Module* erts_put_module(Eterm mod);
+Module* erts_put_active_module(Eterm mod); /* only while blocked */
 
 void init_module_table(void);
 void module_start_staging(void);
diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c
index 62739d2a78..df53f4db30 100644
--- a/erts/emulator/hipe/hipe_amd64.c
+++ b/erts/emulator/hipe/hipe_amd64.c
@@ -130,6 +130,13 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
     return alloc_code(nrbytes);
 }
 
+void hipe_free_code(void* code)
+{
+    ALLOC_CODE_STATS(--nr_allocs);
+    /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/
+    erts_free(ERTS_ALC_T_HIPE_EXEC, code);
+}
+
 /* Make stub for native code calling exported beam function.
 */
 void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
@@ -234,6 +241,14 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
     return code;
 }
 
+void hipe_free_native_stub(void* stub)
+{
+    ALLOC_CODE_STATS(++nr_allocs);
+    /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/
+
+    erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
+}
+
 void hipe_arch_print_pcb(struct hipe_process_state *p)
 {
 #define U(n,x) \
diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h
index 6f959815bb..df38a80069 100644
--- a/erts/emulator/hipe/hipe_arch.h
+++ b/erts/emulator/hipe/hipe_arch.h
@@ -31,22 +31,30 @@ extern int hipe_patch_insn(void *address, Uint value, Eterm type);
 extern int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline);
 
 extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p);
-extern void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity);
+extern void  hipe_free_code(void*);
+extern void *hipe_make_native_stub(void *exp, unsigned int beamArity);
+extern void  hipe_free_native_stub(void*);
+
 
 #if defined(__sparc__)
 #include "hipe_sparc.h"
+#include "hipe_sparc_asm.h"
 #endif
 #if defined(__i386__)
 #include "hipe_x86.h"
+#include "hipe_x86_asm.h"
 #endif
 #if defined(__x86_64__)
 #include "hipe_amd64.h"
+#include "hipe_amd64_asm.h"
 #endif
 #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
 #include "hipe_ppc.h"
+#include "hipe_ppc_asm.h"
 #endif
 #if defined(__arm__)
 #include "hipe_arm.h"
+#include "hipe_arm_asm.h"
 #endif
 
 #if !defined(AEXTERN)
diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c
index f8ef468341..64e35b62d8 100644
--- a/erts/emulator/hipe/hipe_arm.c
+++ b/erts/emulator/hipe/hipe_arm.c
@@ -198,6 +198,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
     return address;
 }
 
+void  hipe_free_code(void* code, unsigned int bytes)
+{
+    /*SVERK: Leaking code memory */
+}
+
 static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu)
 {
     unsigned int *address;
@@ -229,6 +234,11 @@ static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu)
     return address;
 }
 
+void hipe_free_native_stub(void* stub)
+{
+    /*SVERK: Leaking code stub */
+}
+
 /*
  * ARMv5's support for 32-bit immediates is effectively non-existent.
  * Hence, every 32-bit immediate is stored in memory and loaded via
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 95c2d73971..8be6dc9aa9 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -54,6 +54,7 @@
 
 #define BeamOpCode(Op)	((Uint)BeamOp(Op))
 
+
 int term_to_Sint32(Eterm term, Sint *sp)
 {
     Sint val;
@@ -644,30 +645,21 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3)
     pc = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari);
 
     if (pc) {
-	hipe_mfa_save_orig_beam_op(mfa.mod, mfa.fun, mfa.ari, pc);
-#if HIPE
-#ifdef DEBUG_LINKER
-	printf("%s: ", __FUNCTION__);
-	print_mfa(mfa.mod, mfa.fun, mfa.ari);
-	printf(": planting call trap to %p at BEAM pc %p\r\n", address, pc);
-#endif
+	DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "set beam call trap at %p -> %p", pc, address);
 	hipe_set_call_trap(pc, address, is_closure);
 	BIF_RET(am_true);
-#endif
     }
-#ifdef DEBUG_LINKER
-    printf("%s: ", __FUNCTION__);
-    print_mfa(mfa.mod, mfa.fun, mfa.ari);
-    printf(": no BEAM pc found\r\n");
-#endif
+    DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "failed set call trap to %p, no beam code found", address);
     BIF_RET(am_false);
 }
 
 BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1)
 {
     struct hipe_sdesc *sdesc;
+    Module* modp;
+    int do_commit;
 
-    sdesc = hipe_decode_sdesc(BIF_ARG_1);
+    sdesc = hipe_decode_sdesc(BIF_ARG_1, &do_commit);
     if (!sdesc) {
 	fprintf(stderr, "%s: bad sdesc!\r\n", __FUNCTION__);
 	BIF_ERROR(BIF_P, BADARG);
@@ -676,6 +668,21 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1)
 	fprintf(stderr, "%s: duplicate entry!\r\n", __FUNCTION__);
 	BIF_ERROR(BIF_P, BADARG);
     }
+
+    /*
+     * Link into list of sdesc's in same module instance
+     */
+    modp = erts_put_active_module(make_atom(sdesc->m_aix));
+    ASSERT(modp);
+    if (do_commit) { /* Direct "hipe-patching" of early loaded module */
+        sdesc->next_in_modi = modp->curr.first_hipe_sdesc;
+        modp->curr.first_hipe_sdesc = sdesc;
+    }
+    else {           /* Normal module loading/upgrade */
+        sdesc->next_in_modi = modp->new_hipe_sdesc;
+        modp->new_hipe_sdesc = sdesc;
+    }
+
     BIF_RET(NIL);
 }
 
@@ -970,7 +977,7 @@ BIF_RETTYPE hipe_bifs_get_fe_2(BIF_ALIST_2)
 
 	atom_buf[0] = '\0';
 	strncat(atom_buf, (char*)atom_tab(i)->name, atom_tab(i)->len);
-	printf("no fun entry for %s %ld:%ld\n", atom_buf, uniq, index);
+	printf("no fun entry for %s %ld:%ld\n", atom_buf, (unsigned long)uniq, (unsigned long)index);
 	BIF_ERROR(BIF_P, BADARG);
     }
     BIF_RET(address_to_term((void *)fe, BIF_P));
@@ -997,11 +1004,15 @@ BIF_RETTYPE hipe_bifs_set_native_address_in_fe_2(BIF_ALIST_2)
     BIF_RET(am_true);
 }
 
+struct hipe_ref_head {
+    struct hipe_ref_head* next;
+    struct hipe_ref_head* prev;
+};
+
 /*
- * MFA info hash table:
+ * An exported function called from or implemented by native code
  * - maps MFA to native code entry point
- * - the MFAs it calls (refers_to)
- * - the references to it (referred_from)
+ * - all references to it (callers)
  * - maps MFA to most recent trampoline [if powerpc or arm]
  */
 struct hipe_mfa_info {
@@ -1011,16 +1022,19 @@ struct hipe_mfa_info {
     } bucket;
     Eterm m;	/* atom */
     Eterm f;	/* atom */
-    unsigned int a;
+    unsigned int a : sizeof(int)*8 - 1;
+    unsigned int is_stub : 1;       /* if beam or not (yet) loaded */
     void *remote_address;
-    void *local_address;
-    Eterm *beam_code;
-    Uint orig_beam_op;
-    struct hipe_mfa_info_list *refers_to;
-    struct hipe_ref *referred_from;
+    void *new_address;
+    struct hipe_ref_head callers;   /* sentinel in list of hipe_ref's */
+    struct hipe_mfa_info* next_in_mod;
 #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
     void *trampoline;
 #endif
+#ifdef DEBUG
+    Export* dbg_export;
+#endif
+
 };
 
 static struct {
@@ -1038,6 +1052,25 @@ static struct {
     erts_smp_rwmtx_t lock;
 } hipe_mfa_info_table;
 
+
+/*
+ * An external native call site M:F(...)
+ * to be patched when the callee changes.
+ */
+struct hipe_ref {
+    struct hipe_ref_head head;    /* list of refs to same calleee */
+    void *address;
+    void *trampoline;
+    unsigned int flags;
+    struct hipe_ref* next_from_modi; /* list of refs from same module instance */
+#if defined(DEBUG)
+    struct hipe_mfa_info* callee;
+    Eterm caller_m, caller_f, caller_a;
+#endif
+};
+#define REF_FLAG_IS_LOAD_MFA		1	/* bit 0: 0 == call, 1 == load_mfa */
+
+
 static inline void hipe_mfa_info_table_init_lock(void)
 {
     erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock");
@@ -1108,15 +1141,18 @@ static struct hipe_mfa_info *hipe_mfa_info_table_alloc(Eterm m, Eterm f, unsigne
     res->m = m;
     res->f = f;
     res->a = arity;
+    res->is_stub = 0;
     res->remote_address = NULL;
-    res->local_address = NULL;
-    res->beam_code = NULL;
-    res->orig_beam_op = 0;
-    res->refers_to = NULL;
-    res->referred_from = NULL;
+    res->new_address = NULL;
+    res->callers.next = &res->callers;
+    res->callers.prev = &res->callers;
+    res->next_in_mod = NULL;
 #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
     res->trampoline = NULL;
 #endif
+#ifdef DEBUG
+    res->dbg_export = NULL;
+#endif
 
     return res;
 }
@@ -1154,22 +1190,13 @@ static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eter
     return NULL;
 }
 
-#if 0 /* XXX: unused */
-void *hipe_mfa_find_na(Eterm m, Eterm f, unsigned int arity)
-{
-    const struct hipe_mfa_info *p;
-
-    p = hipe_mfa_info_table_get(m, f, arity);
-    return p ? p->address : NULL;
-}
-#endif
-
 static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, unsigned int arity)
 {
     unsigned long h;
     unsigned int i;
     struct hipe_mfa_info *p;
     unsigned int size;
+    Module* modp;
 
     h = HIPE_MFA_HASH(m, f, arity);
     i = h & hipe_mfa_info_table.mask;
@@ -1189,23 +1216,47 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f,
     size = 1 << hipe_mfa_info_table.log2size;
     if (hipe_mfa_info_table.used > (4*size/5))		/* rehash at 80% */
 	hipe_mfa_info_table_grow();
+
+    modp = erts_put_active_module(m);
+    ASSERT(modp);
+    p->next_in_mod = modp->first_hipe_mfa;
+    modp->first_hipe_mfa = p;
+
+    DBG_TRACE_MFA(m,f,arity, "hipe_mfa_info allocated at %p", p);
+
     return p;
 }
 
-static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address, int is_exported)
+static void remove_mfa_info(struct hipe_mfa_info* rm)
+{
+    unsigned int i;
+    struct hipe_mfa_info *p;
+    struct hipe_mfa_info **prevp;
+
+    i = rm->bucket.hvalue & hipe_mfa_info_table.mask;
+    prevp = &hipe_mfa_info_table.bucket[i];
+    for (;;) {
+        p = *prevp;
+        ASSERT(p);
+        if (p == rm) {
+            *prevp = p->bucket.next;
+            ASSERT(hipe_mfa_info_table.used > 0);
+            hipe_mfa_info_table.used--;
+            return;
+        }
+        prevp = &p->bucket.next;
+    }
+}
+
+static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address)
 {
     struct hipe_mfa_info *p;
 
     hipe_mfa_info_table_rwlock();
     p = hipe_mfa_info_table_put_rwlocked(m, f, arity);
-#ifdef DEBUG_LINKER
-    printf("%s: ", __FUNCTION__);
-    print_mfa(m, f, arity);
-    printf(": changing address from %p to %p\r\n", p->local_address, address);
-#endif
-    p->local_address = address;
-    if (is_exported)
-	p->remote_address = address;
+    DBG_TRACE_MFA(m,f,arity,"set native address in hipe_mfa_info at %p", p);
+    p->new_address = address;
+
     hipe_mfa_info_table_rwunlock();
 }
 
@@ -1237,168 +1288,85 @@ BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3)
 {
     struct hipe_mfa mfa;
     void *address;
-    int is_exported;
 
-    if (!term_to_mfa(BIF_ARG_1, &mfa))
-	BIF_ERROR(BIF_P, BADARG);
-    address = term_to_address(BIF_ARG_2);
-    if (!address)
-	BIF_ERROR(BIF_P, BADARG);
-    if (BIF_ARG_3 == am_true)
-	is_exported = 1;
-    else if (BIF_ARG_3 == am_false)
-	is_exported = 0;
-    else
+    switch (BIF_ARG_3) {
+    case am_true: /* is_exported */
+        if (!term_to_mfa(BIF_ARG_1, &mfa))
+            BIF_ERROR(BIF_P, BADARG);
+        address = term_to_address(BIF_ARG_2);
+        if (!address)
+            BIF_ERROR(BIF_P, BADARG);
+        hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address);
+        break;
+    case am_false:
+        break; /* ignore local functions */
+    default:
 	BIF_ERROR(BIF_P, BADARG);
-    hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address, is_exported);
-    BIF_RET(NIL);
-}
-
-BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1)
-{
-    Eterm lst;
-    struct hipe_mfa mfa;
-    struct hipe_mfa_info *p;
-
-    hipe_mfa_info_table_rwlock();
-    lst = BIF_ARG_1;
-    while (is_list(lst)) {
-	if (!term_to_mfa(CAR(list_val(lst)), &mfa))
-	    break;
-	lst = CDR(list_val(lst));
-	p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
-	if (p) {
-	    p->remote_address = NULL;
-	    p->local_address = NULL;
-	    if (p->beam_code) {
-#ifdef DEBUG_LINKER
-		printf("%s: ", __FUNCTION__);
-		print_mfa(mfa.mod, mfa.fun, mfa.ari);
-		printf(": removing call trap from BEAM pc %p (new op %#lx)\r\n",
-		       p->beam_code, p->orig_beam_op);
-#endif
-		p->beam_code[0] = p->orig_beam_op;
-		p->beam_code = NULL;
-		p->orig_beam_op = 0;
-	    } else {
-#ifdef DEBUG_LINKER
-		printf("%s: ", __FUNCTION__);
-		print_mfa(mfa.mod, mfa.fun, mfa.ari);
-		printf(": no call trap to remove\r\n");
-#endif
-	    }
-	}
     }
-    hipe_mfa_info_table_rwunlock();
-    if (is_not_nil(lst))
-	BIF_ERROR(BIF_P, BADARG);
     BIF_RET(NIL);
 }
 
-void hipe_mfa_save_orig_beam_op(Eterm mod, Eterm fun, unsigned int ari, Eterm *pc)
+
+/* Ask if we need to block all threads
+ * while loading/deleting (beam) code for this module?
+ */
+int hipe_need_blocking(Module* modp)
 {
-    Uint orig_beam_op;
     struct hipe_mfa_info *p;
 
-    orig_beam_op = pc[0];
-    if (orig_beam_op != BeamOpCode(op_hipe_trap_call_closure) &&
-	orig_beam_op != BeamOpCode(op_hipe_trap_call)) {
-	hipe_mfa_info_table_rwlock();
-	p = hipe_mfa_info_table_put_rwlocked(mod, fun, ari);
-#ifdef DEBUG_LINKER
-	printf("%s: ", __FUNCTION__);
-	print_mfa(mod, fun, ari);
-	printf(": saving orig op %#lx from BEAM pc %p\r\n", orig_beam_op, pc);
-#endif
-	p->beam_code = pc;
-	p->orig_beam_op = orig_beam_op;
-	hipe_mfa_info_table_rwunlock();
-    } else {
-#ifdef DEBUG_LINKER
-	printf("%s: ", __FUNCTION__);
-	print_mfa(mod, fun, ari);
-	printf(": orig op %#lx already saved\r\n", orig_beam_op);
-#endif
+    /* Need to block if we have at least one native caller to this module
+     * or native code to make unaccessible.
+     */
+    hipe_mfa_info_table_rlock();
+    for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) {
+        ASSERT(!p->new_address);
+        if (p->callers.next != &p->callers || !p->is_stub) {
+            break;
+	}
     }
+    hipe_mfa_info_table_runlock();
+    return (p != NULL);
 }
 
-static void *hipe_make_stub(Eterm m, Eterm f, unsigned int arity, int is_remote)
-{
-    Export *export_entry;
-    void *StubAddress;
-
-    ASSERT(is_remote);
-
-    export_entry = erts_export_get_or_make_stub(m, f, arity);
-    StubAddress = hipe_make_native_stub(export_entry, arity);
-    if (!StubAddress)
-	erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n");
-    return StubAddress;
-}
-
-static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info **pp)
+static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a)
 {
     struct hipe_mfa_info *p;
-    void *address;
 
     p = hipe_mfa_info_table_get_locked(m, f, a);
-    if (p) {
-	/* find address, predicting for a runtime apply call */
-	address = p->remote_address;
-	if (!is_remote)
-	    address = p->local_address;
-	if (address)
-	    return address;
-
-	/* bummer, install stub, checking if one already existed */
-	address = p->remote_address;
-	if (address)
-	    return address;
-    }
-    /* Caller must take the slow path with the write lock held, but allow
-       it to avoid some work if it already holds the write lock.  */
-    if (pp)
-	*pp = p;
-    return NULL;
-}
-
-static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info *p)
-{
-    void *address;
-
-    if (!p)
-	p = hipe_mfa_info_table_put_rwlocked(m, f, a);
-    address = hipe_make_stub(m, f, a, is_remote);
-    /* XXX: how to tell if a BEAM MFA is exported or not? */
-    p->remote_address = address;
-    return address;
+    return p ? p->remote_address : NULL;
 }
 
-static void *hipe_get_na_nofail_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote)
+static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a)
 {
-    struct hipe_mfa_info *p;
-    void *address;
+    struct hipe_mfa_info *p = hipe_mfa_info_table_put_rwlocked(m, f, a);
 
-    address = hipe_get_na_try_locked(m, f, a, is_remote, &p);
-    if (address)
-	return address;
+    if (!p->remote_address) {
+        Export* export_entry = erts_export_get_or_make_stub(m, f, a);
+        void* stubAddress = hipe_make_native_stub(export_entry, a);
+        if (!stubAddress)
+            erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n");
 
-    address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, p);
-    return address;
+        p->remote_address = stubAddress;
+        p->is_stub = 1;
+#ifdef DEBUG
+        p->dbg_export = export_entry;
+#endif
+    }
+    return p->remote_address;
 }
 
-static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a, int is_remote)
+static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a)
 {
     void *address;
 
     hipe_mfa_info_table_rlock();
-    address = hipe_get_na_try_locked(m, f, a, is_remote, NULL);
+    address = hipe_get_na_try_locked(m, f, a);
     hipe_mfa_info_table_runlock();
     if (address)
 	return address;
 
     hipe_mfa_info_table_rwlock();
-    address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, NULL);
+    address = hipe_get_na_slow_rwlocked(m, f, a);
     hipe_mfa_info_table_rwunlock();
     return address;
 }
@@ -1408,7 +1376,7 @@ void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a)
 {
     if (is_not_atom(m) || is_not_atom(f) || a > 255)
 	return NULL;
-    return hipe_get_na_nofail(m, f, a, 1);
+    return hipe_get_na_nofail(m, f, a);
 }
 
 /* primop, but called like a BIF for error handling purposes */
@@ -1420,25 +1388,19 @@ BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3)
     if (is_not_atom(BIF_ARG_1) || is_not_atom(BIF_ARG_2))
 	BIF_ERROR(BIF_P, BADARG);
     arity = unsigned_val(BIF_ARG_3); /* no error check */
-    address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity, 1);
+    address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity);
     BIF_RET((Eterm)address);	/* semi-Ok */
 }
 
-BIF_RETTYPE hipe_bifs_find_na_or_make_stub_2(BIF_ALIST_2)
+BIF_RETTYPE hipe_bifs_find_na_or_make_stub_1(BIF_ALIST_1)
 {
     struct hipe_mfa mfa;
     void *address;
-    int is_remote;
 
     if (!term_to_mfa(BIF_ARG_1, &mfa))
 	BIF_ERROR(BIF_P, BADARG);
-    if (BIF_ARG_2 == am_true)
-	is_remote = 1;
-    else if (BIF_ARG_2 == am_false)
-	is_remote = 0;
-    else
-	BIF_ERROR(BIF_P, BADARG);
-    address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari, is_remote);
+
+    address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari);
     BIF_RET(address_to_term(address, BIF_P));
 }
 
@@ -1460,7 +1422,7 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2)
 	f = ep->code[1];
     } else
 	goto badfun;
-    address = hipe_get_na_nofail(m, f, BIF_ARG_2, 1);
+    address = hipe_get_na_nofail(m, f, BIF_ARG_2);
     BIF_RET((Eterm)address);
 
  badfun:
@@ -1471,60 +1433,19 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2)
 
 int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a)
 {
-    struct hipe_mfa_info *mfa;
-    long mfa_offset, ra_offset;
-    struct hipe_mfa_info **bucket;
-    unsigned int i, nrbuckets;
+    const struct hipe_sdesc* sdesc = hipe_find_sdesc((unsigned long)ra);
 
-    /* Note about locking: the table is only updated from the
-       loader, which runs with the rest of the system suspended. */
-    /* XXX: alas not true; see comment at hipe_mfa_info_table.lock */
-    hipe_mfa_info_table_rlock();
-    bucket = hipe_mfa_info_table.bucket;
-    nrbuckets = 1 << hipe_mfa_info_table.log2size;
-    mfa = NULL;
-    mfa_offset = LONG_MAX;
-    for (i = 0; i < nrbuckets; ++i) {
-	struct hipe_mfa_info *b = bucket[i];
-	while (b != NULL) {
-	    ra_offset = (char*)ra - (char*)b->local_address;
-	    if (ra_offset > 0 && ra_offset < mfa_offset) {
-		mfa_offset = ra_offset;
-		mfa = b;
-	    }
-	    b = b->bucket.next;
-	}
-    }
-    if (mfa) {
-	*m = mfa->m;
-	*f = mfa->f;
-	*a = mfa->a;
-    }
-    hipe_mfa_info_table_runlock();
-    return mfa ? 1 : 0;
-}
+    if (!sdesc || sdesc->m_aix == atom_val(am_Empty))
+        return 0;
 
-/*
- * Patch Reference Handling.
- */
-struct hipe_mfa_info_list {
-    struct hipe_mfa_info *mfa;
-    struct hipe_mfa_info_list *next;
-};
+    *m = make_atom(sdesc->m_aix);
+    *f = make_atom(sdesc->f_aix);
+    *a = sdesc->a;
+    return 1;
+}
 
-struct hipe_ref {
-    struct hipe_mfa_info *caller_mfa;
-    void *address;
-    void *trampoline;
-    unsigned int flags;
-    struct hipe_ref *next;
-};
-#define REF_FLAG_IS_LOAD_MFA		1	/* bit 0: 0 == call, 1 == load_mfa */
-#define REF_FLAG_IS_REMOTE		2	/* bit 1: 0 == local, 1 == remote */
-#define REF_FLAG_PENDING_REDIRECT	4	/* bit 2: 1 == pending redirect */
-#define REF_FLAG_PENDING_REMOVE		8	/* bit 3: 1 == pending remove */
 
-/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,'remote'|'local'})
+/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,DoCommit})
  */
 BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
 {
@@ -1535,9 +1456,9 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
     void *trampoline;
     unsigned int flags;
     struct hipe_mfa_info *callee_mfa;
-    struct hipe_mfa_info *caller_mfa;
-    struct hipe_mfa_info_list *refers_to;
     struct hipe_ref *ref;
+    Module* modp;
+    int do_commit;
 
     if (!term_to_mfa(BIF_ARG_1, &callee))
 	goto badarg;
@@ -1569,62 +1490,90 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
 	    goto badarg;
     }
     switch (tuple[5]) {
-      case am_local:
-	break;
-      case am_remote:
-	flags |= REF_FLAG_IS_REMOTE;
-	break;
-      default:
-	goto badarg;
+    case am_true: do_commit = 1; break;
+    case am_false: do_commit = 0; break;
+    default: goto badarg;
     }
     hipe_mfa_info_table_rwlock();
     callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari);
-    caller_mfa = hipe_mfa_info_table_put_rwlocked(caller.mod, caller.fun, caller.ari);
-
-    refers_to = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*refers_to));
-    refers_to->mfa = callee_mfa;
-    refers_to->next = caller_mfa->refers_to;
-    caller_mfa->refers_to = refers_to;
 
-    ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*ref));
-    ref->caller_mfa = caller_mfa;
+    ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref));
     ref->address = address;
     ref->trampoline = trampoline;
     ref->flags = flags;
-    ref->next = callee_mfa->referred_from;
-    callee_mfa->referred_from = ref;
+
+    /*
+     * Link into list of refs to same callee
+     */
+    ASSERT(callee_mfa->callers.next->prev == &callee_mfa->callers);
+    ASSERT(callee_mfa->callers.prev->next == &callee_mfa->callers);
+    ref->head.next = callee_mfa->callers.next;
+    ref->head.prev = &callee_mfa->callers;
+    ref->head.next->prev = &ref->head;
+    ref->head.prev->next = &ref->head;
+
+    /*
+     * Link into list of refs from same module instance
+     */
+    modp = erts_put_active_module(caller.mod);
+    ASSERT(modp);
+    if (do_commit) { /* Direct "hipe-patching" of early loaded module */
+        ref->next_from_modi = modp->curr.first_hipe_ref;
+        modp->curr.first_hipe_ref = ref;
+    }
+    else {           /* Normal module loading/upgrade */
+        ref->next_from_modi = modp->new_hipe_refs;
+        modp->new_hipe_refs = ref;
+    }
+
+#if defined(DEBUG)
+    ref->callee = callee_mfa;
+    ref->caller_m = caller.mod;
+    ref->caller_f = caller.fun;
+    ref->caller_a = caller.ari;
+#endif
     hipe_mfa_info_table_rwunlock();
 
+    DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p) do_commit=%d",
+		  ref, callee.mod, callee.fun, callee.ari, ref->address, do_commit);
+    DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p) do_commit=%d",
+		  ref, caller.mod, caller.fun, caller.ari, ref->address, do_commit);
     BIF_RET(NIL);
 
  badarg:
     BIF_ERROR(BIF_P, BADARG);
 }
 
-/* Given a CalleeMFA, mark each ref to it as pending-redirect.
- * This ensures that remove_refs_from() won't remove them: any
- * removal is instead done at the end of redirect_referred_from().
- */
-BIF_RETTYPE hipe_bifs_mark_referred_from_1(BIF_ALIST_1) /* get_refs_from */
+
+static void unlink_mfa_from_mod(struct hipe_mfa_info* unlink_me)
 {
-    struct hipe_mfa mfa;
-    const struct hipe_mfa_info *p;
-    struct hipe_ref *ref;
+    Module* modp = erts_get_module(unlink_me->m, erts_active_code_ix());
+    struct hipe_mfa_info** prevp = &modp->first_hipe_mfa;
+    struct hipe_mfa_info* p;
 
-    if (!term_to_mfa(BIF_ARG_1, &mfa))
-	BIF_ERROR(BIF_P, BADARG);
-    hipe_mfa_info_table_rwlock();
-    p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
-    if (p)
-	for (ref = p->referred_from; ref != NULL; ref = ref->next)
-	    ref->flags |= REF_FLAG_PENDING_REDIRECT;
-    hipe_mfa_info_table_rwunlock();
-    BIF_RET(NIL);
+    ASSERT(modp);
+    for (;;) {
+        p = *prevp;
+        ASSERT(p && p->m == unlink_me->m);
+        if (p == unlink_me) {
+            *prevp = p->next_in_mod;
+            break;
+        }
+         prevp = &p->next_in_mod;
+    }
+}
+
+static void purge_mfa(struct hipe_mfa_info* p)
+{
+    ASSERT(p->is_stub);
+    remove_mfa_info(p);
+    hipe_free_native_stub(p->remote_address);
+    erts_free(ERTS_ALC_T_HIPE, p);
 }
 
 /* Called by init:restart after unloading all hipe compiled modules
- * to work around bug causing execution of deallocated beam code.
- * Can be removed when delete/purge of native modules works better.
+ * to work around old bug that caused execution of deallocated beam code.
+ * Can be removed now when delete/purge of native modules works better.
  * Test: Do init:restart in debug compiled vm with hipe compiled kernel. 
  */
 static void hipe_purge_all_refs(void)
@@ -1634,126 +1583,205 @@ static void hipe_purge_all_refs(void)
 
     hipe_mfa_info_table_rwlock();
 
+    ASSERT(hipe_mfa_info_table.used == 0);
     bucket = hipe_mfa_info_table.bucket;
     nrbuckets = 1 << hipe_mfa_info_table.log2size;
     for (i = 0; i < nrbuckets; ++i) {	
+        ASSERT(bucket[i] == NULL);
 	while (bucket[i] != NULL) {
+            Module* modp;
 	    struct hipe_mfa_info* mfa = bucket[i];
 	    bucket[i] = mfa->bucket.next;
 
-	    while (mfa->refers_to) {
-		struct hipe_mfa_info_list *to = mfa->refers_to;
-		mfa->refers_to = to->next;
-		erts_free(ERTS_ALC_T_HIPE, to);
-	    }
-	    while (mfa->referred_from) {
-		struct hipe_ref* from = mfa->referred_from;
-		mfa->referred_from = from->next;
-		erts_free(ERTS_ALC_T_HIPE, from);
-	    }
+	    ASSERT(mfa->callers.next == &mfa->callers);
+
+            modp = erts_get_module(mfa->m, erts_active_code_ix());
+            if (modp) {
+                /* unsafe write to active module */
+                modp->first_hipe_mfa = NULL;
+            }
 	    erts_free(ERTS_ALC_T_HIPE, mfa);
 	}
     }
+    hipe_mfa_info_table.used = 0;
     hipe_mfa_info_table_rwunlock();
 }
 
 BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1)
 {
-    struct hipe_mfa mfa;
-    struct hipe_mfa_info *caller_mfa, *callee_mfa;
-    struct hipe_mfa_info_list *refers_to, *tmp_refers_to;
-    struct hipe_ref **prev, *ref;
-
     if (BIF_ARG_1 == am_all) {
 	hipe_purge_all_refs();
 	BIF_RET(am_ok);
     }
 
-    if (!term_to_mfa(BIF_ARG_1, &mfa))
-	BIF_ERROR(BIF_P, BADARG);
-    hipe_mfa_info_table_rwlock();
-    caller_mfa = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
-    if (caller_mfa) {
-	refers_to = caller_mfa->refers_to;
-	while (refers_to) {
-	    callee_mfa = refers_to->mfa;
-	    prev = &callee_mfa->referred_from;
-	    ref = *prev;
-	    while (ref) {
-		if (ref->caller_mfa == caller_mfa) {
-		    if (ref->flags & REF_FLAG_PENDING_REDIRECT) {
-			ref->flags |= REF_FLAG_PENDING_REMOVE;
-			prev = &ref->next;
-			ref = ref->next;
-		    } else {
-			struct hipe_ref *tmp = ref;
-			ref = ref->next;
-			*prev = ref;
-			erts_free(ERTS_ALC_T_HIPE, tmp);
-		    }
-		} else {
-		    prev = &ref->next;
-		    ref = ref->next;
-		}
-	    }
-	    tmp_refers_to = refers_to;
-	    refers_to = refers_to->next;
-	    erts_free(ERTS_ALC_T_HIPE, tmp_refers_to);
-	}
-	caller_mfa->refers_to = NULL;
+    ASSERT(!"hipe_bifs_remove_refs_from_1() called");
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+int hipe_purge_need_blocking(Module* modp)
+{
+    /* SVERK: Verify if this is really necessary */
+    return (modp->old.first_hipe_ref ||
+            modp->old.first_hipe_sdesc ||
+            (!modp->curr.code_hdr && modp->first_hipe_mfa));
+}
+
+void hipe_purge_module(Module* modp)
+{
+    struct hipe_ref* ref;
+    struct hipe_sdesc* sdesc;
+
+    ASSERT(modp);
+
+    DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module");
+
+    /*
+     * Remove all hipe_ref's (external calls) from the old module instance
+     */
+    ref = modp->old.first_hipe_ref;
+
+    while (ref) {
+	struct hipe_ref* free_ref = ref;
+
+        ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+	DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref,
+		      ref->callee->m, ref->callee->f, ref->callee->a);
+	DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref,
+		      ref->caller_m, ref->caller_f, ref->caller_a);
+	ASSERT(ref->caller_m == make_atom(modp->module));
+
+        /*
+         * Unlink from other refs to same callee
+         */
+        ASSERT(ref->head.next->prev == &ref->head);
+        ASSERT(ref->head.prev->next == &ref->head);
+        ASSERT(ref->head.next != &ref->head);
+        ASSERT(ref->head.prev != &ref->head);
+        ref->head.next->prev = ref->head.prev;
+        ref->head.prev->next = ref->head.next;
+
+        /*
+         * Was this the last ref to that callee?
+         */
+        if (ref->head.next == ref->head.prev) {
+            struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers);
+            if (p->is_stub) {
+                unlink_mfa_from_mod(p);
+                purge_mfa(p);
+           }
+        }
+
+	ref = ref->next_from_modi;
+	erts_free(ERTS_ALC_T_HIPE, free_ref);
+    }
+    modp->old.first_hipe_ref = NULL;
+
+    /*
+     * Remove all hipe_sdesc's for the old module instance
+     */
+    sdesc = modp->old.first_hipe_sdesc;
+
+    while (sdesc) {
+	struct hipe_sdesc* free_sdesc = sdesc;
+
+        ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+	DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue);
+	ASSERT(sdesc->m_aix == modp->module);
+
+	sdesc = sdesc->next_in_modi;
+        hipe_destruct_sdesc(free_sdesc);
+    }
+    modp->old.first_hipe_sdesc = NULL;
+
+    /*
+     * Remove unreferred hipe_mfa_info's
+     */
+    if (modp->curr.code_hdr == NULL) {
+        struct hipe_mfa_info** prevp = &modp->first_hipe_mfa;
+        struct hipe_mfa_info* p = *prevp;
+        ERTS_SMP_LC_ASSERT(!p || erts_smp_thr_progress_is_blocking());
+        for (; p; p = *prevp) {
+            if (p->callers.next == &p->callers) {
+                *prevp = p->next_in_mod;
+                purge_mfa(p);
+            }
+            else
+                prevp = &p->next_in_mod;
+        }
+    }
+    if (modp->old.hipe_code_start) {
+#ifdef DEBUG
+        sys_memset(modp->old.hipe_code_start, 0xfe, modp->old.hipe_code_size);
+#endif
+        hipe_free_code(modp->old.hipe_code_start);
+        modp->old.hipe_code_start = NULL;
     }
-    hipe_mfa_info_table_rwunlock();
-    BIF_RET(am_ok);
 }
 
 
-/* redirect_referred_from(CalleeMFA)
- * Redirect all pending-redirect refs in CalleeMFA's referred_from.
- * Then remove any pending-redirect && pending-remove refs from CalleeMFA's referred_from.
+/*
+ * Redirect all existing native calls to this module
  */
-BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1)
+void hipe_redirect_to_module(Module* modp)
 {
-    struct hipe_mfa mfa;
     struct hipe_mfa_info *p;
-    struct hipe_ref **prev, *ref;
-    int is_remote, res;
-    void *new_address;
+    struct hipe_ref_head* refh;
+
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+    for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) {
+        if (p->new_address) {
+            if (p->is_stub) {
+                hipe_free_native_stub(p->remote_address);
+                p->is_stub = 0;
+            }
+            DBG_TRACE_MFA(p->m, p->f, p->a, "Commit new_address %p", p->new_address);
+            p->remote_address = p->new_address;
+            p->new_address = NULL;
+#ifdef DEBUG
+            p->dbg_export = NULL;
+#endif
+        }
+        else if (!p->is_stub) {
+            Export* exp = erts_export_get_or_make_stub(p->m, p->f, p->a);
+            p->remote_address = hipe_make_native_stub(exp, p->a);
+            DBG_TRACE_MFA(p->m, p->f, p->a, "Commit stub %p", p->remote_address);
+            if (!p->remote_address)
+                erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n");
+            p->is_stub = 1;
+#ifdef DEBUG
+            p->dbg_export = exp;
+#endif
+        }
+        else {
+            DBG_TRACE_MFA(p->m, p->f, p->a, "Commit no-op, already stub");
+            ASSERT(p->remote_address && p->dbg_export);
+        }
 
-    if (!term_to_mfa(BIF_ARG_1, &mfa))
-	BIF_ERROR(BIF_P, BADARG);
-    hipe_mfa_info_table_rwlock();
-    p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
-    if (p) {
-	prev = &p->referred_from;
-	ref = *prev;
-	while (ref) {
-	    if (ref->flags & REF_FLAG_PENDING_REDIRECT) {
-		is_remote = ref->flags & REF_FLAG_IS_REMOTE;
-		new_address = hipe_get_na_nofail_rwlocked(p->m, p->f, p->a, is_remote);
-		if (ref->flags & REF_FLAG_IS_LOAD_MFA)
-		    res = hipe_patch_insn(ref->address, (Uint)new_address, am_load_mfa);
-		else
-		    res = hipe_patch_call(ref->address, new_address, ref->trampoline);
-		if (res)
-		    fprintf(stderr, "%s: patch failed\r\n", __FUNCTION__);
-		ref->flags &= ~REF_FLAG_PENDING_REDIRECT;
-		if (ref->flags & REF_FLAG_PENDING_REMOVE) {
-		    struct hipe_ref *tmp = ref;
-		    ref = ref->next;
-		    *prev = ref;
-		    erts_free(ERTS_ALC_T_HIPE, tmp);
-		} else {
-		    prev = &ref->next;
-		    ref = ref->next;
-		}
-	    } else {
-		prev = &ref->next;
-		ref = ref->next;
-	    }
+	DBG_TRACE_MFA(p->m,p->f,p->a,"START REDIRECT towards hipe_mfa_info at %p", p);
+	for (refh = p->callers.next; refh != &p->callers; refh = refh->next) {
+            struct hipe_ref* ref = (struct hipe_ref*) refh;
+	    int res;
+
+	    DBG_TRACE_MFA(p->m,p->f,p->a, "  REDIRECT ref at %p FROM %T:%T/%u (%p -> %p)",
+			  ref, ref->caller_m, ref->caller_f, ref->caller_a,
+			  ref->address, p->remote_address);
+
+	    DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a,
+			  "  REDIRECT ref at %p TO %T:%T/%u (%p -> %p)",
+			  ref, p->m,p->f,p->a, ref->address, p->remote_address);
+
+	    if (ref->flags & REF_FLAG_IS_LOAD_MFA)
+		res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa);
+	    else
+		res = hipe_patch_call(ref->address, p->remote_address, ref->trampoline);
+	    if (res)
+		fprintf(stderr, "%s: patch failed", __FUNCTION__);
 	}
+	DBG_TRACE_MFA(p->m,p->f,p->a,"DONE REDIRECT towards hipe_mfa_info at %p", p);
     }
-    hipe_mfa_info_table_rwunlock();
-    BIF_RET(NIL);
 }
 
 BIF_RETTYPE hipe_bifs_check_crc_1(BIF_ALIST_1)
diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h
index c9a8216368..1914405e2d 100644
--- a/erts/emulator/hipe/hipe_bif0.h
+++ b/erts/emulator/hipe/hipe_bif0.h
@@ -42,7 +42,10 @@ extern void hipe_primop_set_trampoline(Eterm name, void *trampoline);
 #endif
 
 /* needed in beam_load.c */
-void hipe_mfa_save_orig_beam_op(Eterm m, Eterm f, unsigned int a, Eterm *pc);
+int hipe_need_blocking(Module*);
+int hipe_purge_need_blocking(Module*);
+void hipe_purge_module(Module*);
+void hipe_redirect_to_module(Module* modp);
 
 /* these are also needed in hipe_amd64.c */
 extern void *term_to_address(Eterm);
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 99237aae05..eae84383df 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -54,7 +54,7 @@ bif hipe_bifs:set_native_address/3
 #bif hipe_bifs:address_to_fun/1
 
 bif hipe_bifs:set_funinfo_native_address/3
-bif hipe_bifs:invalidate_funinfo_native_addresses/1
+#bif hipe_bifs:invalidate_funinfo_native_addresses/1
 
 bif hipe_bifs:update_code_size/3
 bif hipe_bifs:code_size/1
@@ -72,7 +72,7 @@ bif hipe_bifs:term_to_word/1
 bif hipe_bifs:get_fe/2
 bif hipe_bifs:set_native_address_in_fe/2
 
-bif hipe_bifs:find_na_or_make_stub/2
+bif hipe_bifs:find_na_or_make_stub/1
 
 bif hipe_bifs:check_crc/1
 bif hipe_bifs:system_crc/0
@@ -84,9 +84,7 @@ bif hipe_bifs:patch_insn/3
 bif hipe_bifs:patch_call/3
 
 bif hipe_bifs:add_ref/2
-bif hipe_bifs:mark_referred_from/1
 bif hipe_bifs:remove_refs_from/1
-bif hipe_bifs:redirect_referred_from/1
 
 # atoms used by add_ref/2
 atom call
diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c
index 9b2048c457..a1a6e2ad02 100644
--- a/erts/emulator/hipe/hipe_ppc.c
+++ b/erts/emulator/hipe/hipe_ppc.c
@@ -214,6 +214,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
     return address;
 }
 
+void hipe_free_code(void* code, unsigned int bytes)
+{
+    /*SVERK: Leaking code memory */
+}
+
 static unsigned int *alloc_stub(Uint nrwords)
 {
     unsigned int *address;
@@ -241,6 +246,11 @@ static unsigned int *alloc_stub(Uint nrwords)
     return address;
 }
 
+void hipe_free_native_stub(void* stub)
+{
+    /*SVERK: Leaking code stubs */
+}
+
 static void patch_imm16(Uint32 *address, unsigned int imm16)
 {
     unsigned int insn = *address;
diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h
index 09804e3016..1369b392fe 100644
--- a/erts/emulator/hipe/hipe_risc_glue.h
+++ b/erts/emulator/hipe/hipe_risc_glue.h
@@ -66,15 +66,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y)
 
 static __inline__ void hipe_arch_glue_init(void)
 {
-    static struct hipe_sdesc_with_exnra nbif_return_sdesc = {
-	.exnra = (unsigned long)&nbif_fail,
-	.sdesc = {
-	    .bucket = { .hvalue = (unsigned long)&nbif_return },
-            .fsize = 0,
-            .has_exnra = 1,
-            .arity = 0
-	},
-    };
+    static struct hipe_sdesc_with_exnra nbif_return_sdesc;
+
+    nbif_return_sdesc.exnra = (unsigned long)nbif_fail;
+    nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return;
+    nbif_return_sdesc.sdesc.fsize = 0;
+    nbif_return_sdesc.sdesc.has_exnra = 1;
+    nbif_return_sdesc.sdesc.stk_nargs = 0;
+    nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty);
+    nbif_return_sdesc.sdesc.f_aix = atom_val(am_return);
+    nbif_return_sdesc.sdesc.a     = 0;
+
     hipe_init_sdesc_table(&nbif_return_sdesc.sdesc);
 }
 
diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c
index 17bef0718c..3d082668a6 100644
--- a/erts/emulator/hipe/hipe_stack.c
+++ b/erts/emulator/hipe/hipe_stack.c
@@ -102,6 +102,28 @@ struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc *sdesc)
     return sdesc;
 }
 
+void hipe_destruct_sdesc(struct hipe_sdesc *sdesc)
+{
+    unsigned int i;
+    struct hipe_sdesc** prevp;
+    void* free_me;
+
+    i = (sdesc->bucket.hvalue >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask;
+    prevp = &hipe_sdesc_table.bucket[i];
+
+    for (; *prevp != sdesc; prevp = &(*prevp)->bucket.next)
+        ASSERT(*prevp);
+
+    *prevp = sdesc->bucket.next;
+    hipe_sdesc_table.used -= 1;
+
+    if (sdesc->has_exnra)
+        free_me = ErtsContainerStruct(sdesc, struct hipe_sdesc_with_exnra, sdesc);
+    else
+        free_me = sdesc;
+    erts_free(ERTS_ALC_T_HIPE, free_me);
+}
+
 void hipe_init_sdesc_table(struct hipe_sdesc *sdesc)
 {
     unsigned int log2size, size;
@@ -121,31 +143,46 @@ void hipe_init_sdesc_table(struct hipe_sdesc *sdesc)
  * representation. If different representations are needed in
  * the future, this code has to be made target dependent.
  */
-struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
+struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp)
 {
     Uint ra, exnra;
     Eterm *live;
-    Uint fsize, arity, nlive, i, nslots, off;
+    Uint fsize, nargs, stk_nargs, nlive, i, nslots, off;
     Uint livebitswords, sdescbytes;
     void *p;
     struct hipe_sdesc *sdesc;
-
-    if (is_not_tuple(arg) ||
-	(tuple_val(arg))[0] != make_arityval(6) ||
-	term_to_Uint((tuple_val(arg))[1], &ra) == 0 ||
-	term_to_Uint((tuple_val(arg))[2], &exnra) == 0 ||
-	is_not_small((tuple_val(arg))[3]) ||
-	(fsize = unsigned_val((tuple_val(arg))[3])) > 65535 ||
-	is_not_small((tuple_val(arg))[4]) ||
-	(arity = unsigned_val((tuple_val(arg))[4])) > 255 ||
-	is_not_tuple((tuple_val(arg))[5]))
+    Eterm* mfa_tpl;
+    Eterm* tp;
+
+    if (is_not_tuple(arg))
+        return 0;
+
+    tp = tuple_val(arg);
+    if (tp[0] != make_arityval(7) ||
+        term_to_Uint(tp[1], &ra) == 0 ||
+	term_to_Uint(tp[2], &exnra) == 0 ||
+	is_not_small(tp[3]) ||
+	(fsize = unsigned_val(tp[3])) > 65535 ||
+	is_not_small(tp[4]) ||
+	(stk_nargs = unsigned_val(tp[4])) > 255 ||
+	is_not_tuple(tp[5]) ||
+        is_not_tuple(tp[6]) ||
+        (mfa_tpl = tuple_val(tp[6]))[0] != make_arityval(3) ||
+        is_not_atom(mfa_tpl[1]) ||
+        is_not_atom(mfa_tpl[2]) ||
+        is_not_small(mfa_tpl[3]) ||
+        (nargs = unsigned_val(mfa_tpl[3])) > 255)
 	return 0;
+
+    if (stk_nargs > nargs)
+        return 0;
+
     /* Get tuple with live slots */
-    live = tuple_val((tuple_val(arg))[5]) + 1;
+    live = tuple_val(tp[5]) + 1;
     /* Get number of live slots */
     nlive = arityval(live[-1]);
-    /* Calculate size of frame = locals + ra + arguments */
-    nslots = fsize + 1 + arity;
+    /* Calculate size of frame = locals + ra + stack arguments */
+    nslots = fsize + 1 + stk_nargs;
     /* Check that only valid slots are given. */
     for (i = 0; i < nlive; ++i) {
 	if (is_not_small(live[i]) ||
@@ -154,8 +191,14 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
 	    return 0;
     }
 
+    switch(tp[7]) {
+    case am_true: *do_commitp = 1; break;
+    case am_false: *do_commitp = 0; break;
+    default: return 0;
+    }
+
     /* Calculate number of words for the live bitmap. */
-    livebitswords = (fsize + arity + 1 + 31) / 32;
+    livebitswords = (fsize + stk_nargs + 1 + 31) / 32;
     /* Calculate number of bytes needed for the stack descriptor. */
     sdescbytes =
 	(exnra
@@ -172,12 +215,17 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
     } else
 	sdesc = p;
 
+    sdesc->m_aix = atom_val(mfa_tpl[1]);
+    sdesc->f_aix = atom_val(mfa_tpl[2]);
+    sdesc->a = nargs;
+
+
     /* Initialise head of sdesc. */
     sdesc->bucket.next = 0;
     sdesc->bucket.hvalue = ra;
     sdesc->fsize = fsize;
     sdesc->has_exnra = (exnra ? 1 : 0);
-    sdesc->arity = arity;
+    sdesc->stk_nargs = stk_nargs;
     /* Clear all live-bits */
     for (i = 0; i < livebitswords; ++i)
 	sdesc->livebits[i] = 0;
@@ -186,13 +234,5 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
 	off = unsigned_val(live[i]);
 	sdesc->livebits[off / 32] |= (1 << (off & 31));
     }
-#ifdef DEBUG
-    {
-	Eterm mfa_tpl = tuple_val(arg)[6];	
-	sdesc->dbg_M = tuple_val(mfa_tpl)[1];
-	sdesc->dbg_F = tuple_val(mfa_tpl)[2];
-	sdesc->dbg_A = tuple_val(mfa_tpl)[3];
-    }
-#endif
     return sdesc;
 }
diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h
index 81f2e53dbc..1863b0db8c 100644
--- a/erts/emulator/hipe/hipe_stack.h
+++ b/erts/emulator/hipe/hipe_stack.h
@@ -37,12 +37,12 @@ struct hipe_sdesc {
     } bucket;
     unsigned int fsize : 23;    /* frame size */
     unsigned int has_exnra : 1; /* exn handler presence flag */
-    unsigned int arity : 8;
-#ifdef DEBUG
-    Eterm dbg_M, dbg_F;
-    unsigned dbg_A;
-#endif
-    unsigned int livebits[1]; /* size depends on arch & data in summary field */
+    unsigned int stk_nargs : 8; /* arguments on stack */
+    Uint32 m_aix;
+    Uint32 f_aix;
+    Uint32 a;
+    struct hipe_sdesc* next_in_modi;
+    Uint32 livebits[1]; /* size depends on arch & data in summary field */
 };
 
 struct hipe_sdesc_with_exnra {
@@ -55,9 +55,10 @@ static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc)
     return sdesc->fsize;
 }
 
+/* Nr of arguments pushed on stack */
 static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc)
 {
-    return sdesc->arity;
+    return sdesc->stk_nargs;
 }
 
 static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc)
@@ -79,8 +80,9 @@ struct hipe_sdesc_table {
 extern struct hipe_sdesc_table hipe_sdesc_table;
 
 extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*);
+extern void hipe_destruct_sdesc(struct hipe_sdesc*);
 extern void hipe_init_sdesc_table(struct hipe_sdesc*);
-extern struct hipe_sdesc *hipe_decode_sdesc(Eterm);
+extern struct hipe_sdesc *hipe_decode_sdesc(Eterm, int* do_commitp);
 
 #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
 #define __builtin_expect(x, expected_value) (x)
diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c
index 5f6c8c200e..8a3ba8bc7f 100644
--- a/erts/emulator/hipe/hipe_x86.c
+++ b/erts/emulator/hipe/hipe_x86.c
@@ -184,6 +184,15 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
     return alloc_code(nrbytes);
 }
 
+void hipe_free_code(void* code, unsigned int bytes)
+{
+    /* SVERK: Leak !!!
+    ALLOC_CODE_STATS(--nr_allocs);
+    ALLOC_CODE_STATS(total_alloc -= bytes);
+
+    erts_free(ERTS_ALC_T_HIPE_EXEC, code);*/
+}
+
 void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
 {
     /*
@@ -264,6 +273,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
     return code;
 }
 
+void hipe_free_native_stub(void* stub)
+{
+    /* SVERK: leak leak drip drop */
+}
+
 void hipe_arch_print_pcb(struct hipe_process_state *p)
 {
 #define U(n,x) \
diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h
index 7802076f70..e92e6ea718 100644
--- a/erts/emulator/hipe/hipe_x86_gc.h
+++ b/erts/emulator/hipe/hipe_x86_gc.h
@@ -65,18 +65,14 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state)
     state->sdesc0 = sdesc;
     return sdesc;
 #else
-    unsigned int nstkarity = p->hipe.narity - NR_ARG_REGS;
-    if ((int)nstkarity < 0)
-	nstkarity = 0;
     state->sdesc0[0].fsize = 0;
     state->sdesc0[0].has_exnra = 0;
-    state->sdesc0[0].arity = nstkarity;
+    state->sdesc0[0].stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 :
+                                  p->hipe.narity - NR_ARG_REGS);
     state->sdesc0[0].livebits[0] = 0;
-# ifdef DEBUG
-    state->sdesc0[0].dbg_M = 0;
-    state->sdesc0[0].dbg_F = am_undefined;
-    state->sdesc0[0].dbg_A = 0;
-# endif
+    state->sdesc0[0].m_aix = 0;
+    state->sdesc0[0].f_aix = atom_val(am_undefined);
+    state->sdesc0[0].a     = 0;
     /* XXX: this appears to prevent a gcc-4.1.1 bug on x86 */
     __asm__ __volatile__("" : : "m"(*state) : "memory");
     return &state->sdesc0[0];
diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h
index 0e9fcd61f1..de2b061706 100644
--- a/erts/emulator/hipe/hipe_x86_glue.h
+++ b/erts/emulator/hipe/hipe_x86_glue.h
@@ -58,18 +58,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y)
 
 static __inline__ void hipe_arch_glue_init(void)
 {
-    static struct hipe_sdesc_with_exnra nbif_return_sdesc = {
-	.exnra = (unsigned long)nbif_fail,
-	.sdesc = {
-	    .bucket = { .hvalue = (unsigned long)nbif_return },
-            .fsize = 0,
-            .has_exnra = 1,
-            .arity = 0,
-	#ifdef DEBUG
-	    .dbg_F = am_return,
-	#endif
-	},
-    };
+    static struct hipe_sdesc_with_exnra nbif_return_sdesc;
+
+    nbif_return_sdesc.exnra = (unsigned long)nbif_fail;
+    nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return;
+    nbif_return_sdesc.sdesc.fsize = 0;
+    nbif_return_sdesc.sdesc.has_exnra = 1;
+    nbif_return_sdesc.sdesc.stk_nargs = 0;
+    nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty);
+    nbif_return_sdesc.sdesc.f_aix = atom_val(am_return);
+    nbif_return_sdesc.sdesc.a     = 0;
+
     hipe_init_sdesc_table(&nbif_return_sdesc.sdesc);
 }
 
diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c
index 9426b166e2..31582b3a2e 100644
--- a/erts/emulator/hipe/hipe_x86_stack.c
+++ b/erts/emulator/hipe/hipe_x86_stack.c
@@ -60,7 +60,6 @@ void hipe_print_nstack(Process *p)
     unsigned int mask;
     unsigned int sdesc_size;
     unsigned int i;
-    unsigned int nstkarity;
     static const char dashes[2*sizeof(long)+5] = {
 	[0 ... 2*sizeof(long)+3] = '-'
     };
@@ -68,12 +67,10 @@ void hipe_print_nstack(Process *p)
     nsp = p->hipe.nsp;
     nsp_end = p->hipe.nstend;
 
-    nstkarity = p->hipe.narity - NR_ARG_REGS;
-    if ((int)nstkarity < 0)
-	nstkarity = 0;
     sdesc0.fsize = 0;
     sdesc0.has_exnra = 0;
-    sdesc0.arity = nstkarity;
+    sdesc0.stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 :
+                        p->hipe.narity - NR_ARG_REGS);
     sdesc0.livebits[0] = ~1;
     sdesc = &sdesc0;
 
diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index 8427bb134d..b76ba6427f 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -445,62 +445,64 @@ module_md5_ok(Code) ->
 make_stub(Config) when is_list(Config) ->
     catch erlang:purge_module(my_code_test),
     MD5 = erlang:md5(<<>>),
+    Arg3 = {[], [], MD5, 0, 0},
 
     Data = proplists:get_value(data_dir, Config),
     File = filename:join(Data, "my_code_test"),
     {ok,my_code_test,Code} = compile:file(File, [binary]),
 
-    my_code_test = code:make_stub_module(my_code_test, Code, {[],[],MD5}),
+    my_code_test = code:make_stub_module(my_code_test, Code, Arg3),
     true = erlang:delete_module(my_code_test),
     true = erlang:purge_module(my_code_test),
 
     my_code_test = code:make_stub_module(my_code_test, 
                                          make_unaligned_sub_binary(Code),
-                                         {[],[],MD5}),
+                                         Arg3),
     true = erlang:delete_module(my_code_test),
     true = erlang:purge_module(my_code_test),
 
     my_code_test = code:make_stub_module(my_code_test, zlib:gzip(Code),
-                                         {[],[],MD5}),
+                                         Arg3),
     true = erlang:delete_module(my_code_test),
     true = erlang:purge_module(my_code_test),
 
     %% Should fail.
     {'EXIT',{badarg,_}} =
-    (catch code:make_stub_module(my_code_test, <<"bad">>, {[],[],MD5})),
+    (catch code:make_stub_module(my_code_test, <<"bad">>, Arg3)),
     {'EXIT',{badarg,_}} =
     (catch code:make_stub_module(my_code_test,
                                  bit_sized_binary(Code),
-                                 {[],[],MD5})),
+                                 Arg3)),
     {'EXIT',{badarg,_}} =
     (catch code:make_stub_module(my_code_test_with_wrong_name,
-                                 Code, {[],[],MD5})),
+                                 Code, Arg3)),
     ok.
 
 make_stub_many_funs(Config) when is_list(Config) ->
     catch erlang:purge_module(many_funs),
     MD5 = erlang:md5(<<>>),
+    Arg3 = {[], [], MD5, 0, 0},
 
     Data = proplists:get_value(data_dir, Config),
     File = filename:join(Data, "many_funs"),
     {ok,many_funs,Code} = compile:file(File, [binary]),
 
-    many_funs = code:make_stub_module(many_funs, Code, {[],[],MD5}),
+    many_funs = code:make_stub_module(many_funs, Code, Arg3),
     true = erlang:delete_module(many_funs),
     true = erlang:purge_module(many_funs),
     many_funs = code:make_stub_module(many_funs, 
                                       make_unaligned_sub_binary(Code),
-                                      {[],[],MD5}),
+                                      Arg3),
     true = erlang:delete_module(many_funs),
     true = erlang:purge_module(many_funs),
 
     %% Should fail.
     {'EXIT',{badarg,_}} =
-    (catch code:make_stub_module(many_funs, <<"bad">>, {[],[],MD5})),
+    (catch code:make_stub_module(many_funs, <<"bad">>, Arg3)),
     {'EXIT',{badarg,_}} =
     (catch code:make_stub_module(many_funs,
                                  bit_sized_binary(Code),
-                                 {[],[],MD5})),
+                                 Arg3)),
     ok.
 
 constant_pools(Config) when is_list(Config) ->
-- 
cgit v1.2.3


From 4d96d297044274c46deda1adfc567297449a9ba9 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 15 Sep 2016 19:23:27 +0200
Subject: erts: Remove unused hipe_bifs:code_size

and hipe_bifs:update_code_size
---
 erts/emulator/hipe/hipe_bif0.c   | 83 ----------------------------------------
 erts/emulator/hipe/hipe_bif0.tab |  3 --
 2 files changed, 86 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 8be6dc9aa9..6f9bfe2f40 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1835,89 +1835,6 @@ void hipe_patch_address(Uint *address, Eterm patchtype, Uint value)
     }
 }
 
-struct modinfo {
-    HashBucket bucket;		/* bucket.hvalue == atom_val(the module name) */
-    unsigned int code_size;
-};
-
-static Hash modinfo_table;
-
-static HashValue modinfo_hash(void *tmpl)
-{
-    Eterm mod = (Eterm)tmpl;
-    return atom_val(mod);
-}
-
-static int modinfo_cmp(void *tmpl, void *bucket)
-{
-    /* bucket->hvalue == modinfo_hash(tmpl), so just return 0 (match) */
-    return 0;
-}
-
-static void *modinfo_alloc(void *tmpl)
-{
-    struct modinfo *p;
-
-    p = (struct modinfo*)erts_alloc(ERTS_ALC_T_HIPE, sizeof(*p));
-    p->code_size = 0;
-    return &p->bucket;
-}
-
-static void init_modinfo_table(void)
-{
-    HashFunctions f;
-    static int init_done = 0;
-
-    if (init_done)
-	return;
-    init_done = 1;
-    f.hash = (H_FUN) modinfo_hash;
-    f.cmp = (HCMP_FUN) modinfo_cmp;
-    f.alloc = (HALLOC_FUN) modinfo_alloc;
-    f.free = (HFREE_FUN) NULL;
-    f.meta_alloc = (HMALLOC_FUN) erts_alloc;
-    f.meta_free = (HMFREE_FUN) erts_free;
-    f.meta_print = (HMPRINT_FUN) erts_print;
-    hash_init(ERTS_ALC_T_HIPE, &modinfo_table, "modinfo_table", 11, f);
-}
-
-BIF_RETTYPE hipe_bifs_update_code_size_3(BIF_ALIST_3)
-{
-    struct modinfo *p;
-    Sint code_size;
-
-    init_modinfo_table();
-
-    if (is_not_atom(BIF_ARG_1) ||
-	is_not_small(BIF_ARG_3) ||
-	(code_size = signed_val(BIF_ARG_3)) < 0)
-	BIF_ERROR(BIF_P, BADARG);
-
-    p = (struct modinfo*)hash_put(&modinfo_table, (void*)BIF_ARG_1);
-
-    if (is_nil(BIF_ARG_2))	/* some MFAs, not whole module */
-	p->code_size += code_size;
-    else			/* whole module */
-	p->code_size = code_size;
-    BIF_RET(NIL);
-}
-
-BIF_RETTYPE hipe_bifs_code_size_1(BIF_ALIST_1)
-{
-    struct modinfo *p;
-    unsigned int code_size;
-
-    init_modinfo_table();
-
-    if (is_not_atom(BIF_ARG_1))
-	BIF_ERROR(BIF_P, BADARG);
-
-    p = (struct modinfo*)hash_get(&modinfo_table, (void*)BIF_ARG_1);
-
-    code_size = p ? p->code_size : 0;
-    BIF_RET(make_small(code_size));
-}
-
 BIF_RETTYPE hipe_bifs_patch_insn_3(BIF_ALIST_3)
 {
     Uint *address, value;
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index eae84383df..ff9c1d06fb 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -56,9 +56,6 @@ bif hipe_bifs:set_native_address/3
 bif hipe_bifs:set_funinfo_native_address/3
 #bif hipe_bifs:invalidate_funinfo_native_addresses/1
 
-bif hipe_bifs:update_code_size/3
-bif hipe_bifs:code_size/1
-
 bif hipe_bifs:enter_sdesc/1
 
 bif hipe_bifs:bif_address/3
-- 
cgit v1.2.3


From c5a826b3d3a80726473fd9c2a7bea58b4363a993 Mon Sep 17 00:00:00 2001
From: Victor Ren <victorhge@gmail.com>
Date: Thu, 6 Oct 2016 18:27:07 +0800
Subject: Use atom value as hash value in process dictionary

In the origin implementation, the hash value of atom term is retrieved
from the atom table.  Reading the atom table is expensive since it is in
memory and leads to more cache missing.

The size of a process dictionary is usually small.  The atom value (the
 index) is unique and can be hash value for it.  Using the atom value
 directly should be more efficient.
---
 erts/emulator/beam/beam_load.c        | 2 +-
 erts/emulator/beam/erl_process_dict.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index d69b18e22f..8aecf01bfd 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4345,7 +4345,7 @@ hash_internal_genop_arg(LoaderState* stp, GenOpArg Key, Uint32* hx)
 {
     switch (Key.type) {
     case TAG_a:
-	*hx = atom_tab(atom_val(Key.val))->slot.bucket.hvalue;
+	*hx = atom_val(Key.val);
 	return 1;
     case TAG_i:
 	*hx = Key.val;
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index d8c2eaba94..e05d81158d 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -56,7 +56,7 @@
 #define MAKE_HASH(Term)                                \
     ((is_small(Term)) ? unsigned_val(Term) :           \
      ((is_atom(Term)) ?                                \
-      (atom_tab(atom_val(Term))->slot.bucket.hvalue) : \
+      atom_val(Term) :								   \
       make_internal_hash(Term)))
 
 #define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm))
-- 
cgit v1.2.3


From b1070efa22f18e19bc7c55fd69e0796f48abf256 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 10 Oct 2016 15:05:26 +0200
Subject: erts: Refactor process dict hash pre-calculation

with new function erts_pd_make_hx()
---
 erts/emulator/beam/beam_load.c        | 20 ++++++++++++--------
 erts/emulator/beam/erl_process_dict.c |  5 +++++
 erts/emulator/beam/erl_process_dict.h |  1 +
 3 files changed, 18 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 8aecf01bfd..165328f992 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -39,6 +39,7 @@
 #include "erl_binary.h"
 #include "erl_zlib.h"
 #include "erl_map.h"
+#include "erl_process_dict.h"
 
 #ifdef HIPE
 #include "hipe_bif0.h"
@@ -4343,22 +4344,25 @@ gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src,
 static int
 hash_internal_genop_arg(LoaderState* stp, GenOpArg Key, Uint32* hx)
 {
+    Eterm key_term;
     switch (Key.type) {
     case TAG_a:
-	*hx = atom_val(Key.val);
-	return 1;
+	key_term = Key.val;
+        break;
     case TAG_i:
-	*hx = Key.val;
-	return 1;
+	key_term = make_small(Key.val);
+        break;
     case TAG_n:
-	*hx = make_internal_hash(NIL);
-	return 1;
+        key_term = NIL;
+        break;
     case TAG_q:
-	*hx = make_internal_hash(stp->literals[Key.val].term);
-	return 1;
+	key_term = stp->literals[Key.val].term;
+	break;
     default:
 	return 0;
     }
+    *hx = erts_pd_make_hx(key_term);
+    return 1;
 }
 
 
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index e05d81158d..d443fff22c 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -408,6 +408,11 @@ static void pd_hash_erase_all(Process *p)
     }
 }
 
+Uint32 erts_pd_make_hx(Eterm key)
+{
+    return MAKE_HASH(key);
+}
+
 Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id)
 {
     unsigned int hval;
diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h
index 387562058c..db5e1c7442 100644
--- a/erts/emulator/beam/erl_process_dict.h
+++ b/erts/emulator/beam/erl_process_dict.h
@@ -43,6 +43,7 @@ void erts_deep_dictionary_dump(int to, void *to_arg,
 Eterm erts_dictionary_copy(struct process *p, ProcDict *pd);
 
 Eterm erts_pd_hash_get(struct process *p, Eterm id);
+Uint32 erts_pd_make_hx(Eterm key);
 Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id);
 
 #endif
-- 
cgit v1.2.3


From 855b3a9be724ffd3c9f7e311cf9d810099fa36ef Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Mon, 27 Jun 2016 20:18:22 +0200
Subject: erts: Refactor out func_info into struct

This commit adds two new structs to be used to represent
erlang code in erts.

ErtsCodeInfo is used to describe the i_func_info header
that is part of all Export entries and the prelude of
each function. This replaces all the BeamInstr * that
were previously used to point to these locations.
After this change the code should never use BeamInstr *
with offsets to figure out different parts of the
func_info header.

ErtsCodeMFA is a struct that is used to descripe a
MFA in code. It is used within ErtsCodeInfo and also
in Process->current.

All function that previously took Eterm * or BeamInstr *
to identify a MFA now use the ErtsCodeMFA or ErtsCodeInfo
where appropriate.

The code has been tested to work when adding a new field to the
ErtsCodeInfo struct, but some updates are needed in ops.tab to
make it work.
---
 erts/emulator/beam/beam_bif_load.c    |  46 ++---
 erts/emulator/beam/beam_bp.c          | 330 +++++++++++++++++-----------------
 erts/emulator/beam/beam_bp.h          |  46 ++---
 erts/emulator/beam/beam_debug.c       |  87 ++++-----
 erts/emulator/beam/beam_emu.c         | 309 ++++++++++++++++---------------
 erts/emulator/beam/beam_load.c        | 153 ++++++++--------
 erts/emulator/beam/beam_load.h        |  10 +-
 erts/emulator/beam/beam_ranges.c      |  18 +-
 erts/emulator/beam/bif.c              |  12 +-
 erts/emulator/beam/bif.h              |  16 +-
 erts/emulator/beam/break.c            |  18 +-
 erts/emulator/beam/code_ix.h          |  76 +++++++-
 erts/emulator/beam/erl_bif_info.c     |  39 ++--
 erts/emulator/beam/erl_bif_op.c       |   2 +-
 erts/emulator/beam/erl_bif_trace.c    | 226 +++++++++++------------
 erts/emulator/beam/erl_db_util.c      |   8 +-
 erts/emulator/beam/erl_fun.c          |   4 +-
 erts/emulator/beam/erl_nif.c          |  80 +++++----
 erts/emulator/beam/erl_printf_term.c  |   6 +-
 erts/emulator/beam/erl_process.c      |  21 +--
 erts/emulator/beam/erl_process.h      |  15 +-
 erts/emulator/beam/erl_process_dump.c |   7 +-
 erts/emulator/beam/erl_trace.c        |  62 ++++---
 erts/emulator/beam/erl_trace.h        |  11 +-
 erts/emulator/beam/erl_vm.h           |   7 +
 erts/emulator/beam/error.h            |   4 +-
 erts/emulator/beam/export.c           |  39 ++--
 erts/emulator/beam/export.h           |  18 +-
 erts/emulator/beam/external.c         |  17 +-
 erts/emulator/beam/global.h           |  14 +-
 erts/emulator/beam/utils.c            |  26 +--
 erts/emulator/hipe/hipe_bif0.c        |   6 +-
 erts/emulator/hipe/hipe_bif1.c        |   8 +-
 erts/emulator/hipe/hipe_debug.c       |  16 +-
 34 files changed, 943 insertions(+), 814 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 153fa205ed..f584b486ed 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -792,19 +792,19 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	 */
 	for (i = 0; i < export_list_size(code_ix); i++) {
 	    Export *ep = export_list(i,code_ix);
-	    if (ep == NULL || ep->code[0] != BIF_ARG_1) {
+	    if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) {
 		continue;
 	    }
-	    if (ep->code[4] != 0) {
-		ep->addressv[code_ix] = (void *) ep->code[4];
-		ep->code[4] = 0;
+	    if (ep->code[1] != 0) {
+		ep->addressv[code_ix] = (void *) ep->code[1];
+		ep->code[1] = 0;
 	    } else {
-		if (ep->addressv[code_ix] == ep->code+3 &&
-		    ep->code[3] == (BeamInstr) em_apply_bif) {
+		if (ep->addressv[code_ix] == ep->code &&
+		    ep->code[0] == (BeamInstr) em_apply_bif) {
 		    continue;
 		}
-		ep->addressv[code_ix] = ep->code+3;
-		ep->code[3] = (BeamInstr) em_call_error_handler;
+		ep->addressv[code_ix] = ep->code;
+		ep->code[0] = (BeamInstr) em_call_error_handler;
 	    }
 	}
 	modp->curr.code_hdr->on_load_function_ptr = NULL;
@@ -819,13 +819,13 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 
 	for (i = 0; i < export_list_size(code_ix); i++) {
 	    Export *ep = export_list(i,code_ix);
-	    if (ep == NULL || ep->code[0] != BIF_ARG_1) {
+	    if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) {
 		continue;
 	    }
-	    if (ep->code[3] == (BeamInstr) em_apply_bif) {
+	    if (ep->code[0] == (BeamInstr) em_apply_bif) {
 		continue;
 	    }
-	    ep->code[4] = 0;
+	    ep->code[1] = 0;
 	}
     }
     erts_smp_thr_progress_unblock();
@@ -849,9 +849,9 @@ set_default_trace_pattern(Eterm module)
 				   &trace_pattern_flags,
 				   &meta_tracer);
     if (trace_pattern_is_on) {
-	Eterm mfa[1];
-	mfa[0] = module;
-	(void) erts_set_trace_pattern(0, mfa, 1,
+        ErtsCodeMFA mfa;
+        mfa.module = module;
+	(void) erts_set_trace_pattern(0, &mfa, 1,
 				      match_spec,
 				      meta_match_spec,
 				      1, trace_pattern_flags,
@@ -1710,23 +1710,23 @@ delete_code(Module* modp)
 
     for (i = 0; i < export_list_size(code_ix); i++) {
 	Export *ep = export_list(i, code_ix);
-        if (ep != NULL && (ep->code[0] == module)) {
-	    if (ep->addressv[code_ix] == ep->code+3) {
-		if (ep->code[3] == (BeamInstr) em_apply_bif) {
+        if (ep != NULL && (ep->info.mfa.module == module)) {
+	    if (ep->addressv[code_ix] == ep->code) {
+		if (ep->code[0] == (BeamInstr) em_apply_bif) {
 		    continue;
 		}
-		else if (ep->code[3] ==
+		else if (ep->code[0] ==
 			 (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
 		    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
 		    ASSERT(modp->curr.num_traced_exports > 0);
-		    erts_clear_export_break(modp, ep->code+3);
+		    erts_clear_export_break(modp, &ep->info);
 		}
-		else ASSERT(ep->code[3] == (BeamInstr) em_call_error_handler
+		else ASSERT(ep->code[0] == (BeamInstr) em_call_error_handler
 			    || !erts_initialized);
 	    }
-	    ep->addressv[code_ix] = ep->code+3;
-	    ep->code[3] = (BeamInstr) em_call_error_handler;
-	    ep->code[4] = 0;
+	    ep->addressv[code_ix] = ep->code;
+	    ep->code[0] = (BeamInstr) em_call_error_handler;
+	    ep->code[1] = 0;
 	}
     }
 
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 920c8b1ed0..2cdf66f441 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -92,27 +92,27 @@ get_mtime(Process *c_p)
 /*
 ** Helpers
 */
-static ErtsTracer do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg,
+static ErtsTracer do_call_trace(Process* c_p, ErtsCodeInfo *info, Eterm* reg,
                                 int local, Binary* ms, ErtsTracer tracer);
 static void set_break(BpFunctions* f, Binary *match_spec, Uint break_flags,
 		      enum erts_break_op count_op, ErtsTracer tracer);
-static void set_function_break(BeamInstr *pc,
+static void set_function_break(ErtsCodeInfo *ci,
 			       Binary *match_spec,
 			       Uint break_flags,
 			       enum erts_break_op count_op,
 			       ErtsTracer tracer);
 
 static void clear_break(BpFunctions* f, Uint break_flags);
-static int clear_function_break(BeamInstr *pc, Uint break_flags);
+static int clear_function_break(ErtsCodeInfo *ci, Uint break_flags);
 
-static BpDataTime* get_time_break(BeamInstr *pc);
-static GenericBpData* check_break(BeamInstr *pc, Uint break_flags);
+static BpDataTime* get_time_break(ErtsCodeInfo *ci);
+static GenericBpData* check_break(ErtsCodeInfo *ci, Uint break_flags);
 
-static void bp_meta_unref(BpMetaTracer* bmt);
-static void bp_count_unref(BpCount* bcp);
-static void bp_time_unref(BpDataTime* bdt);
-static void consolidate_bp_data(Module* modp, BeamInstr* pc, int local);
-static void uninstall_breakpoint(BeamInstr* pc);
+static void bp_meta_unref(BpMetaTracer *bmt);
+static void bp_count_unref(BpCount *bcp);
+static void bp_time_unref(BpDataTime *bdt);
+static void consolidate_bp_data(Module *modp, ErtsCodeInfo *ci, int local);
+static void uninstall_breakpoint(ErtsCodeInfo *ci);
 
 /* bp_hash */
 #define BP_TIME_ADD(pi0, pi1)                       \
@@ -139,7 +139,7 @@ erts_bp_init(void) {
 
 
 void
-erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified)
+erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified)
 {
     ErtsCodeIndex code_ix = erts_active_code_ix();
     Uint max_funcs = 0;
@@ -164,41 +164,44 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified)
     i = 0;
     for (current = 0; current < num_modules; current++) {
 	BeamCodeHeader* code_hdr = module[current]->curr.code_hdr;
-	BeamInstr* code;
+	ErtsCodeInfo* ci;
 	Uint num_functions = (Uint)(UWord) code_hdr->num_functions;
 	Uint fi;
 
 	if (specified > 0) {
-	    if (mfa[0] != make_atom(module[current]->module)) {
+	    if (mfa->module != make_atom(module[current]->module)) {
 		/* Wrong module name */
 		continue;
 	    }
 	}
 
 	for (fi = 0; fi < num_functions; fi++) {
-	    BeamInstr* pc;
-	    int wi;
 
-	    code = code_hdr->functions[fi];
-	    ASSERT(code[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
-	    pc = code+5;
-	    if (erts_is_native_break(pc)) {
+	    ci = code_hdr->functions[fi];
+	    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+	    if (erts_is_native_break(ci)) {
 		continue;
 	    }
-	    if (is_nil(code[3])) { /* Ignore BIF stub */
+	    if (is_nil(ci->mfa.module)) { /* Ignore BIF stub */
 		continue;
 	    }
-	    for (wi = 0;
-		 wi < specified && (Eterm) code[2+wi] == mfa[wi];
-		 wi++) {
-		/* Empty loop body */
-	    }
-	    if (wi == specified) {
-		/* Store match */
-		f->matching[i].pc = pc;
-		f->matching[i].mod = module[current];
-		i++;
-	    }
+            switch (specified) {
+            case 3:
+                if (ci->mfa.arity != mfa->arity)
+                    continue;
+            case 2:
+                if (ci->mfa.function != mfa->function)
+                    continue;
+            case 1:
+                if (ci->mfa.module != mfa->module)
+                    continue;
+            case 0:
+                break;
+            }
+            /* Store match */
+            f->matching[i].ci = ci;
+            f->matching[i].mod = module[current];
+            i++;
 	}
     }
     f->matched = i;
@@ -206,7 +209,7 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified)
 }
 
 void
-erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified)
+erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified)
 {
     ErtsCodeIndex code_ix = erts_active_code_ix();
     int i;
@@ -218,27 +221,36 @@ erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified)
     for (i = 0; i < num_exps; i++) {
 	Export* ep = export_list(i, code_ix);
 	BeamInstr* pc;
-	int j;
 
-	for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) {
-	    /* Empty loop body */
-	}
-	if (j < specified) {
-	    continue;
-	}
-	pc = ep->code+3;
+        switch (specified) {
+        case 3:
+            if (mfa->arity != ep->info.mfa.arity)
+                continue;
+        case 2:
+            if (mfa->function != ep->info.mfa.function)
+                continue;
+        case 1:
+            if (mfa->module != ep->info.mfa.module)
+                continue;
+        case 0:
+            break;
+        default:
+            ASSERT(0);
+        }
+
+	pc = ep->code;
 	if (ep->addressv[code_ix] == pc) {
 	    if ((*pc == (BeamInstr) em_apply_bif ||
 		 *pc == (BeamInstr) em_call_error_handler)) {
 		continue;
 	    }
 	    ASSERT(*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint));
-	} else if (erts_is_native_break(ep->addressv[code_ix])) {
+	} else if (erts_is_native_break(erts_code_to_codeinfo(ep->addressv[code_ix]))) {
 	    continue;
 	}
 
-	f->matching[ne].pc = pc;
-	f->matching[ne].mod = erts_get_module(ep->code[0], code_ix);
+	f->matching[ne].ci = &ep->info;
+	f->matching[ne].mod = erts_get_module(ep->info.mfa.module, code_ix);
 	ne++;
 
     }
@@ -264,7 +276,7 @@ erts_consolidate_bp_data(BpFunctions* f, int local)
     ERTS_SMP_LC_ASSERT(erts_has_code_write_permission());
 
     for (i = 0; i < n; i++) {
-	consolidate_bp_data(fs[i].mod, fs[i].pc, local);
+	consolidate_bp_data(fs[i].mod, fs[i].ci, local);
     }
 }
 
@@ -276,14 +288,14 @@ erts_consolidate_bif_bp_data(void)
     ERTS_SMP_LC_ASSERT(erts_has_code_write_permission());
     for (i = 0; i < BIF_SIZE; i++) {
 	Export *ep = bif_export[i];
-	consolidate_bp_data(0, ep->code+3, 0);
+	consolidate_bp_data(0, &ep->info, 0);
     }
 }
 
 static void
-consolidate_bp_data(Module* modp, BeamInstr* pc, int local)
+consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local)
 {
-    GenericBp* g = (GenericBp *) pc[-4];
+    GenericBp* g = (GenericBp *) ci->native;
     GenericBpData* src;
     GenericBpData* dst;
     Uint flags;
@@ -329,9 +341,10 @@ consolidate_bp_data(Module* modp, BeamInstr* pc, int local)
 	    }
 	    ASSERT(modp->curr.num_breakpoints >= 0);
 	    ASSERT(modp->curr.num_traced_exports >= 0);
-	    ASSERT(*pc != (BeamInstr) BeamOp(op_i_generic_breakpoint));
+	    ASSERT(*erts_codeinfo_to_code(ci) !=
+                   (BeamInstr) BeamOp(op_i_generic_breakpoint));
 	}
-	pc[-4] = 0;
+	ci->native = 0;
 	Free(g);
 	return;
     }
@@ -380,8 +393,9 @@ erts_install_breakpoints(BpFunctions* f)
     BeamInstr br = (BeamInstr) BeamOp(op_i_generic_breakpoint);
 
     for (i = 0; i < n; i++) {
-	BeamInstr* pc = f->matching[i].pc;
-	GenericBp* g = (GenericBp *) pc[-4];
+	ErtsCodeInfo* ci = f->matching[i].ci;
+        BeamInstr *pc = erts_codeinfo_to_code(ci);
+	GenericBp* g = (GenericBp *) ci->native;
 	if (*pc != br && g) {
 	    Module* modp = f->matching[i].mod;
 
@@ -413,16 +427,16 @@ erts_uninstall_breakpoints(BpFunctions* f)
     Uint n = f->matched;
 
     for (i = 0; i < n; i++) {
-	BeamInstr* pc = f->matching[i].pc;
-	uninstall_breakpoint(pc);
+	uninstall_breakpoint(f->matching[i].ci);
     }
 }
 
 static void
-uninstall_breakpoint(BeamInstr* pc)
+uninstall_breakpoint(ErtsCodeInfo *ci)
 {
+    BeamInstr *pc = erts_codeinfo_to_code(ci);
     if (*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
-	GenericBp* g = (GenericBp *) pc[-4];
+	GenericBp* g = (GenericBp *) ci->native;
 	if (g->data[erts_active_bp_ix()].flags == 0) {
 	    /*
 	     * The following write is not protected by any lock. We
@@ -449,30 +463,30 @@ erts_set_mtrace_break(BpFunctions* f, Binary *match_spec, ErtsTracer tracer)
 }
 
 void
-erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local)
+erts_set_call_trace_bif(ErtsCodeInfo *ci, Binary *match_spec, int local)
 {
     Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE;
 
-    set_function_break(pc, match_spec, flags, 0, erts_tracer_nil);
+    set_function_break(ci, match_spec, flags, 0, erts_tracer_nil);
 }
 
 void
-erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, ErtsTracer tracer)
+erts_set_mtrace_bif(ErtsCodeInfo *ci, Binary *match_spec, ErtsTracer tracer)
 {
-    set_function_break(pc, match_spec, ERTS_BPF_META_TRACE, 0, tracer);
+    set_function_break(ci, match_spec, ERTS_BPF_META_TRACE, 0, tracer);
 }
 
 void
-erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op count_op)
+erts_set_time_trace_bif(ErtsCodeInfo *ci, enum erts_break_op count_op)
 {
-    set_function_break(pc, NULL,
+    set_function_break(ci, NULL,
 		       ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE,
 		       count_op, erts_tracer_nil);
 }
 
 void
-erts_clear_time_trace_bif(BeamInstr *pc) {
-    clear_function_break(pc, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE);
+erts_clear_time_trace_bif(ErtsCodeInfo *ci) {
+    clear_function_break(ci, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE);
 }
 
 void
@@ -501,14 +515,14 @@ erts_clear_trace_break(BpFunctions* f)
 }
 
 void
-erts_clear_call_trace_bif(BeamInstr *pc, int local)
+erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local)
 {
-    GenericBp* g = (GenericBp *) pc[-4];
+    GenericBp* g = (GenericBp *) ci->native;
 
     if (g) {
 	Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE;
 	if (g->data[erts_staging_bp_ix()].flags & flags) {
-	    clear_function_break(pc, flags);
+	    clear_function_break(ci, flags);
 	}
     }
 }
@@ -520,9 +534,9 @@ erts_clear_mtrace_break(BpFunctions* f)
 }
 
 void
-erts_clear_mtrace_bif(BeamInstr *pc)
+erts_clear_mtrace_bif(ErtsCodeInfo *ci)
 {
-    clear_function_break(pc, ERTS_BPF_META_TRACE);
+    clear_function_break(ci, ERTS_BPF_META_TRACE);
 }
 
 void
@@ -564,52 +578,48 @@ erts_clear_module_break(Module *modp) {
     }
     n = (Uint)(UWord) code_hdr->num_functions;
     for (i = 0; i < n; ++i) {
-	BeamInstr* pc;
-
-	pc = code_hdr->functions[i] + 5;
-	if (erts_is_native_break(pc)) {
+	ErtsCodeInfo *ci = code_hdr->functions[i];
+	if (erts_is_native_break(ci))
 	    continue;
-	}
-	clear_function_break(pc, ERTS_BPF_ALL);
+	clear_function_break(ci, ERTS_BPF_ALL);
     }
 
     erts_commit_staged_bp();
 
     for (i = 0; i < n; ++i) {
-	BeamInstr* pc;
-
-	pc = code_hdr->functions[i] + 5;
-	if (erts_is_native_break(pc)) {
+	ErtsCodeInfo *ci = code_hdr->functions[i];
+	if (erts_is_native_break(ci))
 	    continue;
-	}
-	uninstall_breakpoint(pc);
-	consolidate_bp_data(modp, pc, 1);
-	ASSERT(pc[-4] == 0);
+	uninstall_breakpoint(ci);
+	consolidate_bp_data(modp, ci, 1);
+	ASSERT(ci->native == 0);
     }
     return n;
 }
 
 void
-erts_clear_export_break(Module* modp, BeamInstr* pc)
+erts_clear_export_break(Module* modp, ErtsCodeInfo *ci)
 {
     ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
 
-    clear_function_break(pc, ERTS_BPF_ALL);
+    clear_function_break(ci, ERTS_BPF_ALL);
     erts_commit_staged_bp();
-    *pc = (BeamInstr) 0;
-    consolidate_bp_data(modp, pc, 0);
-    ASSERT(pc[-4] == 0);
+    *erts_codeinfo_to_code(ci) = (BeamInstr) 0;
+    consolidate_bp_data(modp, ci, 0);
+    ASSERT(ci->native == 0);
 }
 
 BeamInstr
-erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg)
+erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg)
 {
     GenericBp* g;
     GenericBpData* bp;
     Uint bp_flags;
     ErtsBpIndex ix = erts_active_bp_ix();
 
-    g = (GenericBp *) I[-4];
+    ASSERT(info->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+
+    g = (GenericBp *) info->native;
     bp = &g->data[ix];
     bp_flags = bp->flags;
     ASSERT((bp_flags & ~ERTS_BPF_ALL) == 0);
@@ -628,9 +638,9 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg)
 
     if (bp_flags & ERTS_BPF_LOCAL_TRACE) {
 	ASSERT((bp_flags & ERTS_BPF_GLOBAL_TRACE) == 0);
-	(void) do_call_trace(c_p, I, reg, 1, bp->local_ms, erts_tracer_true);
+	(void) do_call_trace(c_p, info, reg, 1, bp->local_ms, erts_tracer_true);
     } else if (bp_flags & ERTS_BPF_GLOBAL_TRACE) {
-	(void) do_call_trace(c_p, I, reg, 0, bp->local_ms, erts_tracer_true);
+	(void) do_call_trace(c_p, info, reg, 0, bp->local_ms, erts_tracer_true);
     }
 
     if (bp_flags & ERTS_BPF_META_TRACE) {
@@ -638,7 +648,8 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg)
 
 	old_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer);
 
-	new_tracer = do_call_trace(c_p, I, reg, 1, bp->meta_ms, old_tracer);
+	new_tracer = do_call_trace(c_p, info, reg, 1, bp->meta_ms, old_tracer);
+
 	if (!ERTS_TRACER_COMPARE(new_tracer, old_tracer)) {
             if (old_tracer == erts_smp_atomic_cmpxchg_acqb(
                     &bp->meta_tracer->tracer,
@@ -657,7 +668,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg)
 
     if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE) {
 	Eterm w;
-	erts_trace_time_call(c_p, I, bp->time);
+	erts_trace_time_call(c_p, info, bp->time);
 	w = (BeamInstr) *c_p->cp;
 	if (! (w == (BeamInstr) BeamOp(op_i_return_time_trace) ||
 	       w == (BeamInstr) BeamOp(op_return_trace) ||
@@ -665,7 +676,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg)
 	    Eterm* E = c_p->stop;
 	    ASSERT(c_p->htop <= E && E <= c_p->hend);
 	    if (E - 2 < c_p->htop) {
-		(void) erts_garbage_collect(c_p, 2, reg, I[-1]);
+		(void) erts_garbage_collect(c_p, 2, reg, info->mfa.arity);
 		ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	    }
 	    E = c_p->stop;
@@ -673,7 +684,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg)
 	    ASSERT(c_p->htop <= E && E <= c_p->hend);
 
 	    E -= 2;
-	    E[0] = make_cp(I);
+	    E[0] = make_cp(erts_codeinfo_to_code(info));
 	    E[1] = make_cp(c_p->cp);     /* original return address */
 	    c_p->cp = beam_return_time_trace;
 	    c_p->stop = E;
@@ -701,9 +712,9 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
     Export* ep = bif_export[bif_index];
     Uint32 flags = 0, flags_meta = 0;
     ErtsTracer meta_tracer = erts_tracer_nil;
-    int applying = (I == &(ep->code[3])); /* Yup, the apply code for a bif
-					   * is actually in the
-					   * export entry */
+    int applying = (I == ep->code); /* Yup, the apply code for a bif
+                                      * is actually in the
+                                      * export entry */
     BeamInstr *cp = p->cp;
     GenericBp* g;
     GenericBpData* bp = NULL;
@@ -711,7 +722,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 
     ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
-    g = (GenericBp *) ep->fake_op_func_info_for_hipe[1];
+    g = (GenericBp *) ep->info.native;
     if (g) {
 	bp = &g->data[erts_active_bp_ix()];
 	bp_flags = bp->flags;
@@ -727,7 +738,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
     if (bp_flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE) &&
 	IS_TRACED_FL(p, F_TRACE_CALLS)) {
 	int local = !!(bp_flags & ERTS_BPF_LOCAL_TRACE);
-	flags = erts_call_trace(p, ep->code, bp->local_ms, args,
+	flags = erts_call_trace(p, &ep->info, bp->local_ms, args,
 				local, &ERTS_TRACER(p));
     }
     if (bp_flags & ERTS_BPF_META_TRACE) {
@@ -735,7 +746,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 
         meta_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer);
         old_tracer = meta_tracer;
-	flags_meta = erts_call_trace(p, ep->code, bp->meta_ms, args,
+	flags_meta = erts_call_trace(p, &ep->info, bp->meta_ms, args,
 				     0, &meta_tracer);
 
 	if (!ERTS_TRACER_COMPARE(old_tracer, meta_tracer)) {
@@ -753,8 +764,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
     }
     if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE &&
 	IS_TRACED_FL(p, F_TRACE_CALLS)) {
-	BeamInstr *pc = (BeamInstr *)ep->code+3;
-	erts_trace_time_call(p, pc, bp->time);
+	erts_trace_time_call(p, &ep->info, bp->time);
     }
 
     /* Restore original continuation pointer (if changed). */
@@ -817,11 +827,11 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 	    class = exception_tag[GET_EXC_CLASS(reason)];
 
 	    if (flags_meta & MATCH_SET_EXCEPTION_TRACE) {
-		erts_trace_exception(p, ep->code, class, value,
+		erts_trace_exception(p, &ep->info.mfa, class, value,
 				     &meta_tracer);
 	    }
 	    if (flags & MATCH_SET_EXCEPTION_TRACE) {
-		erts_trace_exception(p, ep->code, class, value,
+		erts_trace_exception(p, &ep->info.mfa, class, value,
 				     &ERTS_TRACER(p));
 	    }
 	    if ((flags & MATCH_SET_RETURN_TO_TRACE) && p->catches > 0) {
@@ -852,11 +862,11 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 	}
     } else {
 	if (flags_meta & MATCH_SET_RX_TRACE) {
-	    erts_trace_return(p, ep->code, result, &meta_tracer);
+	    erts_trace_return(p, &ep->info.mfa, result, &meta_tracer);
 	}
 	/* MATCH_SET_RETURN_TO_TRACE cannot occur if(meta) */
 	if (flags & MATCH_SET_RX_TRACE) {
-	    erts_trace_return(p, ep->code, result, &ERTS_TRACER(p));
+	    erts_trace_return(p, &ep->info.mfa, result, &ERTS_TRACER(p));
 	}
 	if (flags & MATCH_SET_RETURN_TO_TRACE &&
             IS_TRACED_FL(p, F_TRACE_RETURN_TO)) {
@@ -875,7 +885,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 }
 
 static ErtsTracer
-do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg,
+do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg,
 	      int local, Binary* ms, ErtsTracer tracer)
 {
     Eterm* cpp;
@@ -916,7 +926,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg,
 	ASSERT(is_CP(*cpp));
     }
     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
-    flags = erts_call_trace(c_p, I-3, ms, reg, local, &tracer);
+    flags = erts_call_trace(c_p, info, ms, reg, local, &tracer);
     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
     if (cpp) {
 	c_p->cp = cp_save;
@@ -932,7 +942,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg,
     if (need) {
 	ASSERT(c_p->htop <= E && E <= c_p->hend);
 	if (E - need < c_p->htop) {
-	    (void) erts_garbage_collect(c_p, need, reg, I[-1]);
+	    (void) erts_garbage_collect(c_p, need, reg, info->mfa.arity);
 	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	    E = c_p->stop;
 	}
@@ -948,12 +958,12 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg,
 	E -= 3;
         c_p->stop = E;
 	ASSERT(c_p->htop <= E && E <= c_p->hend);
-	ASSERT(is_CP((Eterm) (UWord) (I - 3)));
+	ASSERT(is_CP((Eterm) (UWord) (&info->mfa.module)));
 	ASSERT(IS_TRACER_VALID(tracer));
 	E[2] = make_cp(c_p->cp);
         E[1] = copy_object(tracer, c_p);
-	E[0] = make_cp(I - 3); /* We ARE at the beginning of an
-				  instruction,
+	E[0] = make_cp(&info->mfa.module);
+                               /* We ARE at the beginning of an instruction,
 				  the funcinfo is above i. */
 	c_p->cp = (flags & MATCH_SET_EXCEPTION_TRACE) ?
 	    beam_exception_trace : beam_return_trace;
@@ -966,7 +976,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg,
 }
 
 void
-erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
+erts_trace_time_call(Process* c_p, ErtsCodeInfo *info, BpDataTime* bdt)
 {
     ErtsMonotonicTime time;
     process_breakpoint_time_t *pbt = NULL;
@@ -995,14 +1005,14 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
 	pbt = Alloc(sizeof(process_breakpoint_time_t));
 	(void) ERTS_PROC_SET_CALL_TIME(c_p, pbt);
     } else {
-	ASSERT(pbt->pc);
+	ASSERT(pbt->ci);
 	/* add time to previous code */
 	sitem.time = time - pbt->time;
 	sitem.pid = c_p->common.id;
 	sitem.count = 0;
 
 	/* previous breakpoint */
-	pbdt = get_time_break(pbt->pc);
+	pbdt = get_time_break(pbt->ci);
 
 	/* if null then the breakpoint was removed */
 	if (pbdt) {
@@ -1039,12 +1049,12 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
 	BP_TIME_ADD(item, &sitem);
     }
 
-    pbt->pc = I;
+    pbt->ci = info;
     pbt->time = time;
 }
 
 void
-erts_trace_time_return(Process *p, BeamInstr *pc)
+erts_trace_time_return(Process *p, ErtsCodeInfo *ci)
 {
     ErtsMonotonicTime time;
     process_breakpoint_time_t *pbt = NULL;
@@ -1074,14 +1084,14 @@ erts_trace_time_return(Process *p, BeamInstr *pc)
 	/* might have been removed due to
 	 * trace_pattern(false)
 	 */
-	ASSERT(pbt->pc);
+	ASSERT(pbt->ci);
 
 	sitem.time = time - pbt->time;
 	sitem.pid   = p->common.id;
 	sitem.count = 0;
 
 	/* previous breakpoint */
-	pbdt = get_time_break(pbt->pc);
+	pbdt = get_time_break(pbt->ci);
 
 	/* beware, the trace_pattern might have been removed */
 	if (pbdt) {
@@ -1098,16 +1108,16 @@ erts_trace_time_return(Process *p, BeamInstr *pc)
 	    }
 	}
 
-	pbt->pc = pc;
+	pbt->ci = ci;
 	pbt->time = time;
     }
 }
 
 int 
-erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local)
+erts_is_trace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, int local)
 {
     Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE;
-    GenericBpData* bp = check_break(pc, flags);
+    GenericBpData* bp = check_break(ci, flags);
 
     if (bp) {
 	if (match_spec_ret) {
@@ -1119,10 +1129,10 @@ erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local)
 }
 
 int
-erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret,
+erts_is_mtrace_break(ErtsCodeInfo *ci, Binary **match_spec_ret,
 		     ErtsTracer *tracer_ret)
 {
-    GenericBpData* bp = check_break(pc, ERTS_BPF_META_TRACE);
+    GenericBpData* bp = check_break(ci, ERTS_BPF_META_TRACE);
     
     if (bp) {
 	if (match_spec_ret) {
@@ -1137,20 +1147,20 @@ erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret,
 }
 
 int
-erts_is_native_break(BeamInstr *pc) {
+erts_is_native_break(ErtsCodeInfo *ci) {
 #ifdef HIPE
-    ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
-    return pc[0]  == (BeamInstr) BeamOp(op_hipe_trap_call)
-	|| pc[0]  == (BeamInstr) BeamOp(op_hipe_trap_call_closure);
+    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+    return erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call)
+	|| erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure);
 #else
     return 0;
 #endif
 }
 
 int 
-erts_is_count_break(BeamInstr *pc, Uint *count_ret)
+erts_is_count_break(ErtsCodeInfo *ci, Uint *count_ret)
 {
-    GenericBpData* bp = check_break(pc, ERTS_BPF_COUNT);
+    GenericBpData* bp = check_break(ci, ERTS_BPF_COUNT);
     
     if (bp) {
 	if (count_ret) {
@@ -1161,13 +1171,13 @@ erts_is_count_break(BeamInstr *pc, Uint *count_ret)
     return 0;
 }
 
-int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) {
+int erts_is_time_break(Process *p, ErtsCodeInfo *ci, Eterm *retval) {
     Uint i, ix;
     bp_time_hash_t hash;
     Uint size;
     Eterm *hp, t;
     bp_data_time_item_t *item = NULL;
-    BpDataTime *bdt = get_time_break(pc);
+    BpDataTime *bdt = get_time_break(ci);
 
     if (bdt) {
 	if (retval) {
@@ -1221,26 +1231,25 @@ int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) {
 }
 
 
-BeamInstr *
-erts_find_local_func(Eterm mfa[3]) {
+ErtsCodeInfo *
+erts_find_local_func(ErtsCodeMFA *mfa) {
     Module *modp;
     BeamCodeHeader* code_hdr;
-    BeamInstr* code_ptr;
+    ErtsCodeInfo* ci;
     Uint i,n;
 
-    if ((modp = erts_get_module(mfa[0], erts_active_code_ix())) == NULL)
+    if ((modp = erts_get_module(mfa->module, erts_active_code_ix())) == NULL)
 	return NULL;
     if ((code_hdr = modp->curr.code_hdr) == NULL)
 	return NULL;
     n = (BeamInstr) code_hdr->num_functions;
     for (i = 0; i < n; ++i) {
-	code_ptr = code_hdr->functions[i];
-	ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]);
-	ASSERT(mfa[0] == ((Eterm) code_ptr[2]) ||
-	       is_nil((Eterm) code_ptr[2]));
-	if (mfa[1] == ((Eterm) code_ptr[3]) &&
-	    ((BeamInstr) mfa[2]) == code_ptr[4]) {
-	    return code_ptr + 5;
+	ci = code_hdr->functions[i];
+	ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == ci->op);
+	ASSERT(mfa->module == ci->mfa.module || is_nil(ci->mfa.module));
+	if (mfa->function == ci->mfa.function &&
+	    mfa->arity == ci->mfa.arity) {
+	    return ci;
 	}
     }
     return NULL;
@@ -1369,7 +1378,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) {
 	     * the previous breakpoint.
 	     */
 
-	    pbdt = get_time_break(pbt->pc);
+	    pbdt = get_time_break(pbt->ci);
 	    if (pbdt) {
 		sitem.time = get_mtime(p) - pbt->time;
 		sitem.pid   = p->common.id;
@@ -1417,14 +1426,14 @@ set_break(BpFunctions* f, Binary *match_spec, Uint break_flags,
 
     n = f->matched;
     for (i = 0; i < n; i++) {
-	BeamInstr* pc = f->matching[i].pc;
-	set_function_break(pc, match_spec, break_flags,
+	set_function_break(f->matching[i].ci,
+                           match_spec, break_flags,
 			   count_op, tracer);
     }
 }
 
 static void
-set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags,
+set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags,
 		   enum erts_break_op count_op, ErtsTracer tracer)
 {
     GenericBp* g;
@@ -1433,7 +1442,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags,
     ErtsBpIndex ix = erts_staging_bp_ix();
 
     ERTS_SMP_LC_ASSERT(erts_has_code_write_permission());
-    g = (GenericBp *) pc[-4];
+    g = (GenericBp *) ci->native;
     if (g == 0) {
 	int i;
 	if (count_op == ERTS_BREAK_RESTART || count_op == ERTS_BREAK_PAUSE) {
@@ -1441,11 +1450,11 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags,
 	    return;
 	}
 	g = Alloc(sizeof(GenericBp));
-	g->orig_instr = *pc;
+	g->orig_instr = *erts_codeinfo_to_code(ci);
 	for (i = 0; i < ERTS_NUM_BP_IX; i++) {
 	    g->data[i].flags = 0;
 	}
-	pc[-4] = (BeamInstr) g;
+	ci->native = (BeamInstr) g;
     }
     bp = &g->data[ix];
 
@@ -1537,13 +1546,12 @@ clear_break(BpFunctions* f, Uint break_flags)
 
     n = f->matched;
     for (i = 0; i < n; i++) {
-	BeamInstr* pc = f->matching[i].pc;
-	clear_function_break(pc, break_flags);
+	clear_function_break(f->matching[i].ci, break_flags);
     }
 }
 
 static int
-clear_function_break(BeamInstr *pc, Uint break_flags)
+clear_function_break(ErtsCodeInfo *ci, Uint break_flags)
 {
     GenericBp* g;
     GenericBpData* bp;
@@ -1552,7 +1560,7 @@ clear_function_break(BeamInstr *pc, Uint break_flags)
 
     ERTS_SMP_LC_ASSERT(erts_has_code_write_permission());
 
-    if ((g = (GenericBp *) pc[-4]) == 0) {
+    if ((g = (GenericBp *) ci->native) == 0) {
 	return 1;
     }
 
@@ -1638,19 +1646,19 @@ bp_time_unref(BpDataTime* bdt)
 }
 
 static BpDataTime*
-get_time_break(BeamInstr *pc)
+get_time_break(ErtsCodeInfo *ci)
 {
-    GenericBpData* bp = check_break(pc, ERTS_BPF_TIME_TRACE);
+    GenericBpData* bp = check_break(ci, ERTS_BPF_TIME_TRACE);
     return bp ? bp->time : 0;
 }
 
 static GenericBpData*
-check_break(BeamInstr *pc, Uint break_flags)
+check_break(ErtsCodeInfo *ci, Uint break_flags)
 {
-    GenericBp* g = (GenericBp *) pc[-4];
+    GenericBp* g = (GenericBp *) ci->native;
 
-    ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
-    if (erts_is_native_break(pc)) {
+    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+    if (erts_is_native_break(ci)) {
 	return 0;
     }
     if (g) {
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index 541af77211..d32a6bcba5 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -46,7 +46,7 @@ typedef struct bp_data_time {     /* Call time */
 
 typedef struct {
     ErtsMonotonicTime time;
-    BeamInstr *pc;
+    ErtsCodeInfo *ci;
 } process_breakpoint_time_t; /* used within psd */
 
 typedef struct {
@@ -95,7 +95,7 @@ enum erts_break_op{
 typedef Uint32 ErtsBpIndex;
 
 typedef struct {
-    BeamInstr* pc;
+    ErtsCodeInfo *ci;
     Module* mod;
 } BpFunction;
 
@@ -116,8 +116,8 @@ void erts_commit_staged_bp(void);
 ERTS_GLB_INLINE ErtsBpIndex erts_active_bp_ix(void);
 ERTS_GLB_INLINE ErtsBpIndex erts_staging_bp_ix(void);
 
-void erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified);
-void erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified);
+void erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified);
+void erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified);
 void erts_bp_free_matched_functions(BpFunctions* f);
 
 void erts_install_breakpoints(BpFunctions* f);
@@ -128,15 +128,15 @@ void erts_consolidate_bif_bp_data(void);
 void erts_set_trace_break(BpFunctions *f, Binary *match_spec);
 void erts_clear_trace_break(BpFunctions *f);
 
-void erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local);
-void erts_clear_call_trace_bif(BeamInstr *pc, int local);
+void erts_set_call_trace_bif(ErtsCodeInfo *ci, Binary *match_spec, int local);
+void erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local);
 
 void erts_set_mtrace_break(BpFunctions *f, Binary *match_spec,
 			  ErtsTracer tracer);
 void erts_clear_mtrace_break(BpFunctions *f);
-void erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec,
+void erts_set_mtrace_bif(ErtsCodeInfo *ci, Binary *match_spec,
 			 ErtsTracer tracer);
-void erts_clear_mtrace_bif(BeamInstr *pc);
+void erts_clear_mtrace_bif(ErtsCodeInfo *ci);
 
 void erts_set_debug_break(BpFunctions *f);
 void erts_clear_debug_break(BpFunctions *f);
@@ -146,32 +146,32 @@ void erts_clear_count_break(BpFunctions *f);
 
 void erts_clear_all_breaks(BpFunctions* f);
 int erts_clear_module_break(Module *modp);
-void erts_clear_export_break(Module *modp, BeamInstr* pc);
+void erts_clear_export_break(Module *modp, ErtsCodeInfo* ci);
 
-BeamInstr erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg);
-BeamInstr erts_trace_break(Process *p, BeamInstr *pc, Eterm *args,
+BeamInstr erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *ci, Eterm* reg);
+BeamInstr erts_trace_break(Process *p, ErtsCodeInfo *ci, Eterm *args,
                            Uint32 *ret_flags, ErtsTracer *tracer);
 
-int erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local);
-int erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret,
+int erts_is_trace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, int local);
+int erts_is_mtrace_break(ErtsCodeInfo *ci, Binary **match_spec_ret,
 			 ErtsTracer *tracer_ret);
-int erts_is_mtrace_bif(BeamInstr *pc, Binary **match_spec_ret,
+int erts_is_mtrace_bif(ErtsCodeInfo *ci, Binary **match_spec_ret,
 		       ErtsTracer *tracer_ret);
-int erts_is_native_break(BeamInstr *pc);
-int erts_is_count_break(BeamInstr *pc, Uint *count_ret);
-int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *call_time);
+int erts_is_native_break(ErtsCodeInfo *ci);
+int erts_is_count_break(ErtsCodeInfo *ci, Uint *count_ret);
+int erts_is_time_break(Process *p, ErtsCodeInfo *ci, Eterm *call_time);
 
-void erts_trace_time_call(Process* c_p, BeamInstr* pc, BpDataTime* bdt);
-void erts_trace_time_return(Process* c_p, BeamInstr* pc);
+void erts_trace_time_call(Process* c_p, ErtsCodeInfo *ci, BpDataTime* bdt);
+void erts_trace_time_return(Process* c_p, ErtsCodeInfo *ci);
 void erts_schedule_time_break(Process *p, Uint out);
 void erts_set_time_break(BpFunctions *f, enum erts_break_op);
 void erts_clear_time_break(BpFunctions *f);
 
-int erts_is_time_trace_bif(Process *p, BeamInstr *pc, Eterm *call_time);
-void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op);
-void erts_clear_time_trace_bif(BeamInstr *pc);
+int erts_is_time_trace_bif(Process *p, ErtsCodeInfo *ci, Eterm *call_time);
+void erts_set_time_trace_bif(ErtsCodeInfo *ci, enum erts_break_op);
+void erts_clear_time_trace_bif(ErtsCodeInfo *ci);
 
-BeamInstr *erts_find_local_func(Eterm mfa[3]);
+ErtsCodeInfo *erts_find_local_func(ErtsCodeMFA *mfa);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 8f47ed4d9d..8d4be4667b 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -116,7 +116,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2)
     Eterm MFA = BIF_ARG_1;
     Eterm boolean = BIF_ARG_2;
     Eterm* tp;
-    Eterm mfa[3];
+    ErtsCodeMFA mfa;
     int i;
     int specified = 0;
     Eterm res;
@@ -132,23 +132,24 @@ erts_debug_breakpoint_2(BIF_ALIST_2)
     if (*tp != make_arityval(3)) {
 	goto error;
     }
-    mfa[0] = tp[1];
-    mfa[1] = tp[2];
-    mfa[2] = tp[3];
-    if (!is_atom(mfa[0]) || !is_atom(mfa[1]) ||
-	(!is_small(mfa[2]) && mfa[2] != am_Underscore)) {
+    if (!is_atom(tp[1]) || !is_atom(tp[2]) ||
+	(!is_small(tp[3]) && tp[3] != am_Underscore)) {
 	goto error;
     }
-    for (i = 0; i < 3 && mfa[i] != am_Underscore; i++, specified++) {
+    for (i = 0; i < 3 && tp[i+1] != am_Underscore; i++, specified++) {
 	/* Empty loop body */
     }
     for (i = specified; i < 3; i++) {
-	if (mfa[i] != am_Underscore) {
+	if (tp[i+1] != am_Underscore) {
 	    goto error;
 	}
     }
-    if (is_small(mfa[2])) {
-	mfa[2] = signed_val(mfa[2]);
+
+    mfa.module = tp[1];
+    mfa.function = tp[2];
+
+    if (is_small(tp[3])) {
+        mfa.arity = signed_val(tp[3]);
     }
 
     if (!erts_try_seize_code_write_permission(BIF_P)) {
@@ -158,7 +159,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2)
     erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
     erts_smp_thr_progress_block();
 
-    erts_bp_match_functions(&f, mfa, specified);
+    erts_bp_match_functions(&f, &mfa, specified);
     if (boolean == am_true) {
 	erts_set_debug_break(&f);
 	erts_install_breakpoints(&f);
@@ -242,17 +243,17 @@ erts_debug_disassemble_1(BIF_ALIST_1)
     Eterm* tp;
     Eterm bin;
     Eterm mfa;
-    BeamInstr* funcinfo = NULL;	/* Initialized to eliminate warning. */
+    ErtsCodeInfo *ci = NULL;
     BeamCodeHeader* code_hdr;
-    BeamInstr* code_ptr = NULL;	/* Initialized to eliminate warning. */
+    BeamInstr *code_ptr;
     BeamInstr instr;
     BeamInstr uaddr;
     Uint hsz;
     int i;
 
     if (term_to_UWord(addr, &uaddr)) {
-	code_ptr = (BeamInstr *) uaddr;
-	if ((funcinfo = find_function_from_pc(code_ptr)) == NULL) {
+	BeamInstr *pc = (BeamInstr *) uaddr;
+	if ((ci = find_function_from_pc(pc)) == NULL) {
 	    BIF_RET(am_false);
 	}
     } else if (is_tuple(addr)) {
@@ -283,24 +284,22 @@ erts_debug_disassemble_1(BIF_ALIST_1)
 	 * such as erts_debug:apply/4.  Then search for it in the module.
 	 */
 	if ((ep = erts_find_function(mod, name, arity, code_ix)) != NULL) {
-	    /* XXX: add "&& ep->address != ep->code+3" condition?
+	    /* XXX: add "&& ep->address != ep->code" condition?
 	     * Consider a traced function.
-	     * Its ep will have ep->address == ep->code+3.
+	     * Its ep will have ep->address == ep->code.
 	     * erts_find_function() will return the non-NULL ep.
 	     * Below we'll try to derive a code_ptr from ep->address.
 	     * But this code_ptr will point to the start of the Export,
 	     * not the function's func_info instruction. BOOM !?
 	     */
-	    code_ptr = ((BeamInstr *) ep->addressv[code_ix]) - 5;
-	    funcinfo = code_ptr+2;
+	    ci = erts_code_to_codeinfo(ep->addressv[code_ix]);
 	} else if (modp == NULL || (code_hdr = modp->curr.code_hdr) == NULL) {
 	    BIF_RET(am_undef);
 	} else {
 	    n = code_hdr->num_functions;
 	    for (i = 0; i < n; i++) {
-		code_ptr = code_hdr->functions[i];
-		if (code_ptr[3] == name && code_ptr[4] == arity) {
-		    funcinfo = code_ptr+2;
+		ci = code_hdr->functions[i];
+		if (ci->mfa.function == name && ci->mfa.arity == arity) {
 		    break;
 		}
 	    }
@@ -312,6 +311,8 @@ erts_debug_disassemble_1(BIF_ALIST_1)
 	goto error;
     }
 
+
+    code_ptr = erts_codeinfo_to_code(ci);
     dsbufp = erts_create_tmp_dsbuf(0);
     erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr);
     instr = (BeamInstr) code_ptr[0];
@@ -333,9 +334,10 @@ erts_debug_disassemble_1(BIF_ALIST_1)
     (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr);
     hp = HAlloc(p, hsz);
     addr = erts_bld_uword(&hp, NULL, (BeamInstr) code_ptr);
-    ASSERT(is_atom(funcinfo[0]) || funcinfo[0] == NIL);
-    ASSERT(is_atom(funcinfo[1]) || funcinfo[1] == NIL);
-    mfa = TUPLE3(hp, (Eterm) funcinfo[0], (Eterm) funcinfo[1], make_small((Eterm) funcinfo[2]));
+    ASSERT(is_atom(ci->mfa.module) || is_nil(ci->mfa.module));
+    ASSERT(is_atom(ci->mfa.function) || is_nil(ci->mfa.function == NIL));
+    mfa = TUPLE3(hp, ci->mfa.module, ci->mfa.function,
+                 make_small(ci->mfa.arity));
     hp += 4;
     return TUPLE3(hp, addr, bin, mfa);
 }
@@ -347,11 +349,12 @@ dbg_bt(Process* p, Eterm* sp)
 
     while (sp < stack) {
 	if (is_CP(*sp)) {
-	    BeamInstr* addr = find_function_from_pc(cp_val(*sp));
-	    if (addr)
+	    ErtsCodeInfo* ci = find_function_from_pc(cp_val(*sp));
+	    if (ci)
 		erts_fprintf(stderr,
 			     HEXF ": %T:%T/%bpu\n",
-			     addr, (Eterm) addr[0], (Eterm) addr[1], addr[2]);
+			     &ci->mfa.module, ci->mfa.module,
+                             ci->mfa.function, ci->mfa.arity);
 	}
 	sp++;
     }
@@ -360,17 +363,17 @@ dbg_bt(Process* p, Eterm* sp)
 void
 dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg)
 {
-    BeamInstr* f = find_function_from_pc(addr);
+    ErtsCodeInfo* ci = find_function_from_pc(addr);
 
-    if (f == NULL) {
+    if (ci == NULL) {
 	erts_fprintf(stderr, "???\n");
     } else {
 	int arity;
 	int i;
 
-	addr = f;
-	arity = addr[2];
-	erts_fprintf(stderr, HEXF ": %T:%T(", addr, (Eterm) addr[0], (Eterm) addr[1]);
+	arity = ci->mfa.arity;
+	erts_fprintf(stderr, HEXF ": %T:%T(", addr,
+                     ci->mfa.module, ci->mfa.function);
 	for (i = 0; i < arity; i++)
 	    erts_fprintf(stderr, i ? ", %T" : "%T", i ? reg[i] : x0);
 	erts_fprintf(stderr, ")\n");
@@ -546,22 +549,24 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	    break;
 	case 'f':		/* Destination label */
 	    {
-		BeamInstr* f = find_function_from_pc((BeamInstr *)*ap);
-		if (f+3 != (BeamInstr *) *ap) {
+		ErtsCodeInfo* ci = find_function_from_pc((BeamInstr *)*ap);
+		if (erts_codeinfo_to_code(ci) != (BeamInstr *) *ap) {
 		    erts_print(to, to_arg, "f(" HEXF ")", *ap);
 		} else {
-		    erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], f[2]);
+		    erts_print(to, to_arg, "%T:%T/%bpu", ci->mfa.module,
+                               ci->mfa.function, ci->mfa.arity);
 		}
 		ap++;
 	    }
 	    break;
 	case 'p':		/* Pointer (to label) */
 	    {
-		BeamInstr* f = find_function_from_pc((BeamInstr *)*ap);
-		if (f+3 != (BeamInstr *) *ap) {
+		ErtsCodeInfo* ci = find_function_from_pc((BeamInstr *)*ap);
+		if (erts_codeinfo_to_code(ci) != (BeamInstr *) *ap) {
 		    erts_print(to, to_arg, "p(" HEXF ")", *ap);
 		} else {
-		    erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], f[2]);
+		    erts_print(to, to_arg, "%T:%T/%bpu", ci->mfa.module,
+                               ci->mfa.function, ci->mfa.arity);
 		}
 		ap++;
 	    }
@@ -574,7 +579,9 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	    {
 		Export* ex = (Export *) *ap;
 		erts_print(to, to_arg,
-			   "%T:%T/%bpu", (Eterm) ex->code[0], (Eterm) ex->code[1], ex->code[2]);
+			   "%T:%T/%bpu", (Eterm) ex->info.mfa.module,
+                           (Eterm) ex->info.mfa.function,
+                           ex->info.mfa.arity);
 		ap++;
 	    }
 	    break;
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 0ba06058a5..4ff177f215 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -116,10 +116,10 @@ do {                                     \
 #define MAX(x, y) (((x) > (y)) ? (x) : (y))
 #endif
 
-#define GET_BIF_MODULE(p)  ((Eterm) (((Export *) p)->code[0]))
-#define GET_BIF_FUNCTION(p)  ((Eterm) (((Export *) p)->code[1]))
-#define GET_BIF_ARITY(p)  ((Eterm) (((Export *) p)->code[2]))
-#define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4]))
+#define GET_BIF_MODULE(p)  (p->info.mfa.module)
+#define GET_BIF_FUNCTION(p)  (p->info.mfa.function)
+#define GET_BIF_ARITY(p)  (p->info.mfa.arity)
+#define GET_BIF_ADDRESS(p) ((BifFunction) (p->code[1]))
 #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm))))
 
 
@@ -1045,7 +1045,7 @@ void** beam_ops;
 static BifFunction translate_gc_bif(void* gcf) NOINLINE;
 static BeamInstr* handle_error(Process* c_p, BeamInstr* pc,
 			       Eterm* reg, BifFunction bf) NOINLINE;
-static BeamInstr* call_error_handler(Process* p, BeamInstr* ip,
+static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa,
 				     Eterm* reg, Eterm func) NOINLINE;
 static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE;
 static BeamInstr* apply(Process* p, Eterm module, Eterm function,
@@ -1106,98 +1106,91 @@ init_emulator(void)
 
 #ifdef USE_VM_CALL_PROBES
 
-#define DTRACE_LOCAL_CALL(p, m, f, a)					\
+#define DTRACE_LOCAL_CALL(p, mfa)					\
     if (DTRACE_ENABLED(local_function_entry)) {				\
         DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);		\
-        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);			\
+        DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE);			\
         int depth = STACK_START(p) - STACK_TOP(p);			\
-        dtrace_fun_decode(p, m, f, a,					\
-                          process_name, mfa);				\
-        DTRACE3(local_function_entry, process_name, mfa, depth);	\
+        dtrace_fun_decode(p, mfa, process_name, mfa_buf);               \
+        DTRACE3(local_function_entry, process_name, mfa_buf, depth);	\
     }
 
-#define DTRACE_GLOBAL_CALL(p, m, f, a)					\
+#define DTRACE_GLOBAL_CALL(p, mfa)					\
     if (DTRACE_ENABLED(global_function_entry)) {			\
         DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);		\
-        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);			\
+        DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE);			\
         int depth = STACK_START(p) - STACK_TOP(p);			\
-        dtrace_fun_decode(p, m, f, a,					\
-                          process_name, mfa);				\
-        DTRACE3(global_function_entry, process_name, mfa, depth);	\
+        dtrace_fun_decode(p, mfa, process_name, mfa_buf);               \
+        DTRACE3(global_function_entry, process_name, mfa_buf, depth);	\
     }
 
-#define DTRACE_RETURN(p, m, f, a)                               \
+#define DTRACE_RETURN(p, mfa)                                    \
     if (DTRACE_ENABLED(function_return)) {                      \
         DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);     \
-        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);              \
+        DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE);          \
         int depth = STACK_START(p) - STACK_TOP(p);              \
-        dtrace_fun_decode(p, m, f, a,                           \
-                          process_name, mfa);                   \
-        DTRACE3(function_return, process_name, mfa, depth);     \
+        dtrace_fun_decode(p, mfa, process_name, mfa_buf);       \
+        DTRACE3(function_return, process_name, mfa_buf, depth); \
     }
 
-#define DTRACE_BIF_ENTRY(p, m, f, a)                            \
-    if (DTRACE_ENABLED(bif_entry)) {                            \
-        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);     \
-        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);              \
-        dtrace_fun_decode(p, m, f, a,                           \
-                          process_name, mfa);                   \
-        DTRACE2(bif_entry, process_name, mfa);                  \
+#define DTRACE_BIF_ENTRY(p, mfa)                                    \
+    if (DTRACE_ENABLED(bif_entry)) {                                \
+        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);         \
+        DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE);              \
+        dtrace_fun_decode(p, mfa, process_name, mfa_buf);           \
+        DTRACE2(bif_entry, process_name, mfa_buf);                  \
     }
 
-#define DTRACE_BIF_RETURN(p, m, f, a)                           \
-    if (DTRACE_ENABLED(bif_return)) {                           \
-        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);     \
-        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);              \
-        dtrace_fun_decode(p, m, f, a,                           \
-                          process_name, mfa);                   \
-        DTRACE2(bif_return, process_name, mfa);                 \
+#define DTRACE_BIF_RETURN(p, mfa)                                   \
+    if (DTRACE_ENABLED(bif_return)) {                               \
+        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);         \
+        DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE);              \
+        dtrace_fun_decode(p, mfa, process_name, mfa_buf);           \
+        DTRACE2(bif_return, process_name, mfa_buf);                 \
     }
 
-#define DTRACE_NIF_ENTRY(p, m, f, a)                            \
-    if (DTRACE_ENABLED(nif_entry)) {                            \
-        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);     \
-        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);              \
-        dtrace_fun_decode(p, m, f, a,                           \
-                          process_name, mfa);                   \
-        DTRACE2(nif_entry, process_name, mfa);                  \
+#define DTRACE_NIF_ENTRY(p, mfa)                                        \
+    if (DTRACE_ENABLED(nif_entry)) {                                    \
+        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);             \
+        DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE);                  \
+        dtrace_fun_decode(p, mfa, process_name, mfa_buf);               \
+        DTRACE2(nif_entry, process_name, mfa_buf);                      \
     }
 
-#define DTRACE_NIF_RETURN(p, m, f, a)                           \
-    if (DTRACE_ENABLED(nif_return)) {                           \
-        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);     \
-        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);              \
-        dtrace_fun_decode(p, m, f, a,                           \
-                          process_name, mfa);                   \
-        DTRACE2(nif_return, process_name, mfa);                 \
+#define DTRACE_NIF_RETURN(p, mfa)                                       \
+    if (DTRACE_ENABLED(nif_return)) {                                   \
+        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);             \
+        DTRACE_CHARBUF(mfa_mfa, DTRACE_TERM_BUF_SIZE);                  \
+        dtrace_fun_decode(p, mfa, process_name, mfa_buf);               \
+        DTRACE2(nif_return, process_name, mfa_buf);                     \
     }
 
 #define DTRACE_GLOBAL_CALL_FROM_EXPORT(p,e)                                                    \
     do {                                                                                       \
         if (DTRACE_ENABLED(global_function_entry)) {                                           \
             BeamInstr* fp = (BeamInstr *) (((Export *) (e))->addressv[erts_active_code_ix()]); \
-            DTRACE_GLOBAL_CALL((p), (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]);                     \
+            DTRACE_GLOBAL_CALL((p), erts_code_to_codemfa(fp));          \
         }                                                                                      \
     } while(0)
 
 #define DTRACE_RETURN_FROM_PC(p)                                                        \
     do {                                                                                \
-        BeamInstr* fp;                                                                  \
-        if (DTRACE_ENABLED(function_return) && (fp = find_function_from_pc((p)->cp))) { \
-            DTRACE_RETURN((p), (Eterm)fp[0], (Eterm)fp[1], (Uint)fp[2]);                \
+        ErtsCodeInfo* ci;                                                                  \
+        if (DTRACE_ENABLED(function_return) && (ci = find_function_from_pc((p)->cp))) { \
+            DTRACE_RETURN((p), &ci->mfa);                               \
         }                                                                               \
     } while(0)
 
 #else /* USE_VM_PROBES */
-#define DTRACE_LOCAL_CALL(p, m, f, a)        do {} while (0)
-#define DTRACE_GLOBAL_CALL(p, m, f, a)       do {} while (0)
+#define DTRACE_LOCAL_CALL(p, mfa)        do {} while (0)
+#define DTRACE_GLOBAL_CALL(p, mfa)       do {} while (0)
 #define DTRACE_GLOBAL_CALL_FROM_EXPORT(p, e) do {} while (0)
-#define DTRACE_RETURN(p, m, f, a)            do {} while (0)
+#define DTRACE_RETURN(p, mfa)            do {} while (0)
 #define DTRACE_RETURN_FROM_PC(p)             do {} while (0)
-#define DTRACE_BIF_ENTRY(p, m, f, a)         do {} while (0)
-#define DTRACE_BIF_RETURN(p, m, f, a)        do {} while (0)
-#define DTRACE_NIF_ENTRY(p, m, f, a)         do {} while (0)
-#define DTRACE_NIF_RETURN(p, m, f, a)        do {} while (0)
+#define DTRACE_BIF_ENTRY(p, mfa)         do {} while (0)
+#define DTRACE_BIF_RETURN(p, mfa)        do {} while (0)
+#define DTRACE_NIF_ENTRY(p, mfa)         do {} while (0)
+#define DTRACE_NIF_RETURN(p, mfa)        do {} while (0)
 #endif /* USE_VM_PROBES */
 
 #ifdef DEBUG
@@ -1324,8 +1317,8 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
     if (start_time != 0) {
         Sint64 diff = erts_timestamp_millis() - start_time;
 	if (diff > 0 && (Uint) diff >  erts_system_monitor_long_schedule) {
-	    BeamInstr *inptr = find_function_from_pc(start_time_i);
-	    BeamInstr *outptr = find_function_from_pc(c_p->i);
+	    ErtsCodeInfo *inptr = find_function_from_pc(start_time_i);
+	    ErtsCodeInfo *outptr = find_function_from_pc(c_p->i);
 	    monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff);
 	}
     }
@@ -1583,7 +1576,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
  /* FALL THROUGH */
  OpCase(i_call_only_f): {
      SET_I((BeamInstr *) Arg(0));
-     DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]);
+     DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
      Dispatch();
  }
 
@@ -1595,7 +1588,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
      RESTORE_CP(E);
      E = ADD_BYTE_OFFSET(E, Arg(1));
      SET_I((BeamInstr *) Arg(0));
-     DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]);
+     DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
      Dispatch();
  }
 
@@ -1607,7 +1600,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
  OpCase(i_call_f): {
      SET_CP(c_p, I+2);
      SET_I((BeamInstr *) Arg(0));
-     DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]);
+     DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
      Dispatch();
  }
 
@@ -2834,25 +2827,27 @@ do {						\
 	Eterm result;
 	BeamInstr *next;
 	ErlHeapFragment *live_hf_end;
+        Export *export = (Export*)Arg(0);
 
 
         if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) {
             /* If we have run out of reductions, we do a context
                switch before calling the bif */
-            c_p->arity = ((Export *)Arg(0))->code[2];
-            c_p->current = ((Export *)Arg(0))->code;
+            c_p->arity = GET_BIF_ARITY(export);
+            c_p->current = &export->info.mfa;
             goto context_switch3;
         }
 
-        ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_BIF_MODULE(Arg(0)), GET_BIF_ADDRESS(Arg(0)));
+        ERTS_MSACC_SET_BIF_STATE_CACHED_X(
+            GET_BIF_MODULE(export), GET_BIF_ADDRESS(export));
 
-	bf = GET_BIF_ADDRESS(Arg(0));
+	bf = GET_BIF_ADDRESS(export);
 
 	PRE_BIF_SWAPOUT(c_p);
 	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS - 1;
 	if (FCALLS <= 0) {
-	   save_calls(c_p, (Export *) Arg(0));
+	   save_calls(c_p, export);
 	}
 	PreFetch(1, next);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2866,7 +2861,7 @@ do {						\
 	ERTS_HOLE_CHECK(c_p);
 	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
 	if (ERTS_IS_GC_DESIRED(c_p)) {
-	    Uint arity = ((Export *)Arg(0))->code[2];
+	    Uint arity = GET_BIF_ARITY(export);
 	    result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity);
 	    E = c_p->stop;
 	}
@@ -3359,14 +3354,15 @@ do {						\
      * called from I[-3], I[-2], and I[-1] respectively.
      */
  context_switch_fun:
-    c_p->arity = I[-1] + 1;
+    /* Add one for the environment of the fun */
+    c_p->arity = erts_code_to_codemfa(I)->arity + 1;
     goto context_switch2;
 
  context_switch:
-    c_p->arity = I[-1];
+    c_p->arity = erts_code_to_codemfa(I)->arity;
 
- context_switch2:		/* Entry for fun calls. */
-    c_p->current = I-3;		/* Pointer to Mod, Func, Arity */
+ context_switch2: 		/* Entry for fun calls. */
+    c_p->current = erts_code_to_codemfa(I);
 
  context_switch3:
 
@@ -3507,7 +3503,8 @@ do {						\
      * code[4]: Not used
      */
     HEAVY_SWAPOUT;
-    I = call_error_handler(c_p, I-3, reg, am_undefined_function);
+    I = call_error_handler(c_p, erts_code_to_codemfa(I),
+                           reg, am_undefined_function);
     HEAVY_SWAPIN;
     if (I) {
 	Goto(*I);
@@ -3544,9 +3541,12 @@ do {						\
 	     * I[1]: Function pointer to NIF function
 	     * I[2]: Pointer to erl_module_nif
 	     * I[3]: Function pointer to dirty NIF
+             *
+             * This layout is determined by the NifExport struct
 	     */
 	    BifFunction vbf;
 	    ErlHeapFragment *live_hf_end;
+            ErtsCodeMFA *codemfa;
 
             if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
                 /* If we have run out of reductions, we do a context
@@ -3556,12 +3556,16 @@ do {						\
 
 	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
 
-	    DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
-	    c_p->current = I-3; /* current and vbf set to please handle_error */ 
+	    codemfa = erts_code_to_codemfa(I);
+
+            c_p->current = codemfa; /* current and vbf set to please handle_error */
+
+	    DTRACE_NIF_ENTRY(c_p, codemfa);
+
 	    SWAPOUT;
 	    c_p->fcalls = FCALLS - 1;
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
-	    bif_nif_arity = I[-1];
+	    bif_nif_arity = codemfa->arity;
 	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 
 	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -3588,19 +3592,19 @@ do {						\
 		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	    }
 
-	    DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
+	    DTRACE_NIF_RETURN(c_p, codemfa);
 	    goto apply_bif_or_nif_epilogue;
 	 
 	OpCase(apply_bif):
 	    /*
-	     * At this point, I points to the code[3] in the export entry for
+	     * At this point, I points to the code[0] in the export entry for
 	     * the BIF:
 	     *
-	     * code[0]: Module
-	     * code[1]: Function
-	     * code[2]: Arity
-	     * code[3]: &&apply_bif
-	     * code[4]: Function pointer to BIF function
+	     * code[-3]: Module
+	     * code[-2]: Function
+	     * code[-1]: Arity
+	     * code[0]: &&apply_bif
+	     * code[1]: Function pointer to BIF function
 	     */
 
             if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
@@ -3609,21 +3613,25 @@ do {						\
                 goto context_switch;
             }
 
-            ERTS_MSACC_SET_BIF_STATE_CACHED_X((Eterm)I[-3], (BifFunction)Arg(0));
+	    codemfa = erts_code_to_codemfa(I);
+
+            ERTS_MSACC_SET_BIF_STATE_CACHED_X(codemfa->module, (BifFunction)Arg(0));
+
 
-	    c_p->current = I-3;	/* In case we apply process_info/1,2 or load_nif/1 */
+            /* In case we apply process_info/1,2 or load_nif/1 */
+	    c_p->current = codemfa;
 	    c_p->i = I;		/* In case we apply check_process_code/2. */
 	    c_p->arity = 0;		/* To allow garbage collection on ourselves
 					 * (check_process_code/2).
 					 */
-	    DTRACE_BIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
+	    DTRACE_BIF_ENTRY(c_p, codemfa);
 
 	    SWAPOUT;
 	    ERTS_DBG_CHK_REDS(c_p, FCALLS - 1);
 	    c_p->fcalls = FCALLS - 1;
 	    vbf = (BifFunction) Arg(0);
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
-	    bif_nif_arity = I[-1];
+	    bif_nif_arity = codemfa->arity;
             ASSERT(bif_nif_arity <= 4);
 	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
@@ -3645,7 +3653,7 @@ do {						\
             if (ERTS_MSACC_IS_ENABLED_CACHED_X())
                 ERTS_MSACC_UPDATE_CACHE_X();
 	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
-	    DTRACE_BIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
+	    DTRACE_BIF_RETURN(c_p, codemfa);
 
 	apply_bif_or_nif_epilogue:
 	    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
@@ -3712,8 +3720,9 @@ do {						\
     goto find_func_info;
 
  OpCase(i_func_info_IaaI): {
+     ErtsCodeInfo *ci = (ErtsCodeInfo*)I;
      c_p->freason = EXC_FUNCTION_CLAUSE;
-     c_p->current = I + 2;
+     c_p->current = &ci->mfa;
      goto handle_error;
  }
 
@@ -4704,11 +4713,11 @@ do {						\
      */
 
  OpCase(return_trace): {
-     BeamInstr* code = (BeamInstr *) (UWord) E[0];
+     ErtsCodeMFA* mfa = (ErtsCodeMFA *)(E[0]);
      
      SWAPOUT;		/* Needed for shared heap */
      ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
-     erts_trace_return(c_p, code, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */);
+     erts_trace_return(c_p, mfa, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */);
      ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
      SWAPIN;
      c_p->cp = NULL;
@@ -4719,9 +4728,8 @@ do {						\
 
  OpCase(i_generic_breakpoint): {
      BeamInstr real_I;
-     ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
      HEAVY_SWAPOUT;
-     real_I = erts_generic_breakpoint(c_p, I, reg);
+     real_I = erts_generic_breakpoint(c_p, erts_code_to_codeinfo(I), reg);
      HEAVY_SWAPIN;
      ASSERT(VALID_INSTR(real_I));
      Goto(real_I);
@@ -4730,7 +4738,7 @@ do {						\
  OpCase(i_return_time_trace): {
      BeamInstr *pc = (BeamInstr *) (UWord) E[0];
      SWAPOUT;
-     erts_trace_time_return(c_p, pc);
+     erts_trace_time_return(c_p, erts_code_to_codeinfo(pc));
      SWAPIN;
      c_p->cp = NULL;
      SET_I((BeamInstr *) cp_val(E[1]));
@@ -4915,16 +4923,18 @@ do {						\
 	  * I[ 0]: &&lb_hipe_trap_call
 	  * ... remainder of original BEAM code
 	  */
-	 ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI));
-	 c_p->hipe.u.ncallee = (void(*)(void)) I[-4];
+         ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
+	 ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
+	 c_p->hipe.u.ncallee = (void(*)(void)) ci->native;
 	 ++hipe_trap_count;
-	 HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8));
+	 HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8));
      }
      OpCase(hipe_trap_call_closure): {
-       ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI));
-       c_p->hipe.u.ncallee = (void(*)(void)) I[-4];
+       ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
+       ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
+       c_p->hipe.u.ncallee = (void(*)(void)) ci->native;
        ++hipe_trap_count;
-       HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8));
+       HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8));
      }
      OpCase(hipe_trap_return): {
 	 HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RETURN);
@@ -4993,8 +5003,9 @@ do {						\
       * I[ 0]: &&lb_hipe_call_count
       * ... remainder of original BEAM code
       */
-     struct hipe_call_count *hcc = (struct hipe_call_count*)I[-4];
-     ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI));
+     ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
+     struct hipe_call_count *hcc = (struct hipe_call_count*)ci->native;
+     ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
      ASSERT(hcc != NULL);
      ASSERT(VALID_INSTR(hcc->opcode));
      ++(hcc->count);
@@ -5063,7 +5074,7 @@ do {						\
 
  OpCase(i_debug_breakpoint): {
      HEAVY_SWAPOUT;
-     I = call_error_handler(c_p, I-3, reg, am_breakpoint);
+     I = call_error_handler(c_p, erts_code_to_codemfa(I), reg, am_breakpoint);
      HEAVY_SWAPIN;
      if (I) {
 	 Goto(*I);
@@ -5136,10 +5147,10 @@ do {						\
 			      bif_table[i].name,
 			      bif_table[i].arity);
 	 bif_export[i] = ep;
-	 ep->code[3] = (BeamInstr) OpCode(apply_bif);
-	 ep->code[4] = (BeamInstr) bif_table[i].f;
+	 ep->code[0] = (BeamInstr) OpCode(apply_bif);
+	 ep->code[1] = (BeamInstr) bif_table[i].f;
 	 /* XXX: set func info for bifs */
-	 ep->fake_op_func_info_for_hipe[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
+	 ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI);
      }
 
      return;
@@ -5219,8 +5230,8 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
     goto do_dirty_schedule;
 
  context_switch:
-    c_p->arity = I[-1];
-    c_p->current = I-3;		/* Pointer to Mod, Func, Arity */
+    c_p->current = erts_code_to_codemfa(I);	/* Pointer to Mod, Func, Arity */
+    c_p->arity = c_p->current->arity;
 
     {
 	int reds_used;
@@ -5378,16 +5389,22 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 	     * I[1]: Function pointer to NIF function
 	     * I[2]: Pointer to erl_module_nif
 	     * I[3]: Function pointer to dirty NIF
+             *
+             * This layout is determined by the NifExport struct
 	     */
 	    BifFunction vbf;
+            ErtsCodeMFA *codemfa;
 
 	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
 
-	    DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
-	    c_p->current = I-3; /* current and vbf set to please handle_error */
+            codemfa = erts_code_to_codemfa(I);
+
+	    DTRACE_NIF_ENTRY(c_p, codemfa);
+            /* current and vbf set to please handle_error */
+	    c_p->current = codemfa;
 	    SWAPOUT;
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
-	    arity = I[-1];
+	    arity = codemfa->arity;
 	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 
 	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -5422,7 +5439,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	    }
 
-	    DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
+	    DTRACE_NIF_RETURN(c_p, codemfa);
 	    ERTS_HOLE_CHECK(c_p);
 	    SWAPIN;
 	    I = c_p->i;
@@ -5602,7 +5619,8 @@ next_catch(Process* c_p, Eterm *reg) {
 	/* Can not follow cp here - code may be unloaded */
 	BeamInstr *cpp = c_p->cp;
 	if (cpp == beam_exception_trace) {
-	    erts_trace_exception(c_p, cp_val(ptr[0]),
+            ErtsCodeMFA *mfa = (ErtsCodeMFA*)cp_val(ptr[0]);
+	    erts_trace_exception(c_p, mfa,
 				 reg[1], reg[2],
                                  ERTS_TRACER_FROM_ETERM(ptr+1));
 	    /* Skip return_trace parameters */
@@ -5630,7 +5648,8 @@ next_catch(Process* c_p, Eterm *reg) {
 		    if (is_catch(*ptr) && active_catches) goto found_catch;
 		}
 		if (cp_val(*prev) == beam_exception_trace) {
-		    erts_trace_exception(c_p, cp_val(ptr[0]),
+                    ErtsCodeMFA *mfa = (ErtsCodeMFA*)cp_val(ptr[0]);
+		    erts_trace_exception(c_p, mfa,
 					 reg[1], reg[2],
                                          ERTS_TRACER_FROM_ETERM(ptr+1));
 		}
@@ -5836,7 +5855,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 	for (i = 0; i < BIF_SIZE; i++) {
 	    if (bf == bif_table[i].f || bf == bif_table[i].traced) {
 		Export *ep = bif_export[i];
-		s->current = ep->code;
+		s->current = &ep->info.mfa;
 	        a = bif_table[i].arity;
 		break;
 	    }
@@ -5850,7 +5869,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 	     */
 	    ASSERT(c_p->current);
 	    s->current = c_p->current;
-	    a = s->current[2];
+	    a = s->current->arity;
 	}
 	/* Save first stack entry */
 	ASSERT(pc);
@@ -5875,7 +5894,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 	     (GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) ) {
 	    int a;
 	    ASSERT(s->current);
-	    a = s->current[2];
+	    a = s->current->arity;
 	    args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */
 	    /* Save first stack entry */
 	    ASSERT(c_p->cp);
@@ -6046,7 +6065,7 @@ build_stacktrace(Process* c_p, Eterm exc) {
 	erts_lookup_function_info(&fi, s->pc, 1);
     } else if (GET_EXC_INDEX(s->freason) ==
 	       GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) {
-	erts_lookup_function_info(&fi, s->current, 1);
+	erts_lookup_function_info(&fi, erts_codemfa_to_code(s->current), 1);
     } else {
 	erts_set_current_function(&fi, s->current);
     }
@@ -6055,8 +6074,8 @@ build_stacktrace(Process* c_p, Eterm exc) {
      * If fi.current is still NULL, default to the initial function
      * (e.g. spawn_link(erlang, abs, [1])).
      */
-    if (fi.current == NULL) {
-	erts_set_current_function(&fi, c_p->u.initial);
+    if (fi.ci == NULL) {
+	erts_set_current_function(&fi, &c_p->u.initial);
 	args = am_true; /* Just in case */
     } else {
 	args = get_args_from_exc(exc);
@@ -6072,7 +6091,7 @@ build_stacktrace(Process* c_p, Eterm exc) {
     heap_size = fi.needed + 2;
     for (i = 0; i < depth; i++) {
 	erts_lookup_function_info(stkp, s->trace[i], 1);
-	if (stkp->current) {
+	if (stkp->ci) {
 	    heap_size += stkp->needed + 2;
 	    stkp++;
 	}
@@ -6096,7 +6115,7 @@ build_stacktrace(Process* c_p, Eterm exc) {
 }
 
 static BeamInstr*
-call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
+call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func)
 {
     Eterm* hp;
     Export* ep;
@@ -6111,7 +6130,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
     ep = erts_find_function(erts_proc_get_error_handler(p), func, 3,
 			    erts_active_code_ix());
     if (ep == NULL) {		/* No error handler */
-	p->current = fi;
+	p->current = mfa;
 	p->freason = EXC_UNDEF;
 	return 0;
     }
@@ -6120,7 +6139,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
      * Create a list with all arguments in the x registers.
      */
 
-    arity = fi[2];
+    arity = mfa->arity;
     sz = 2 * arity;
     if (HeapWordsLeft(p) < sz) {
 	erts_garbage_collect(p, sz, reg, arity);
@@ -6136,8 +6155,8 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
     /*
      * Set up registers for call to error_handler:<func>/3.
      */
-    reg[0] = fi[0];
-    reg[1] = fi[1];
+    reg[0] = mfa->module;
+    reg[1] = mfa->function;
     reg[2] = args;
     return ep->addressv[erts_active_code_ix()];
 }
@@ -6433,7 +6452,7 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
     }
     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
-    c_p->current = bif_export[BIF_hibernate_3]->code;
+    c_p->current = &bif_export[BIF_hibernate_3]->info.mfa;
     c_p->flags |= F_HIBERNATE_SCHED; /* Needed also when woken! */
     return 1;
 }
@@ -6456,21 +6475,15 @@ call_fun(Process* p,		/* Current process. */
 
     if (is_fun_header(hdr)) {
 	ErlFunThing* funp = (ErlFunThing *) fun_val(fun);
-	ErlFunEntry* fe;
-	BeamInstr* code_ptr;
+	ErlFunEntry* fe = funp->fe;
+	BeamInstr* code_ptr = fe->address;
 	Eterm* var_ptr;
-	int actual_arity;
-	unsigned num_free;
-
-	fe = funp->fe;
-	num_free = funp->num_free;
-	code_ptr = fe->address;
-	actual_arity = (int) code_ptr[-1];
+	unsigned num_free = funp->num_free;
+        ErtsCodeMFA *mfa = erts_code_to_codemfa(code_ptr);
+	int actual_arity = mfa->arity;
 
 	if (actual_arity == arity+num_free) {
-	    DTRACE_LOCAL_CALL(p, (Eterm)code_ptr[-3],
-			(Eterm)code_ptr[-2],
-			code_ptr[-1]);
+	    DTRACE_LOCAL_CALL(p, mfa);
 	    if (num_free == 0) {
 		return code_ptr;
 	    } else {
@@ -6575,10 +6588,10 @@ call_fun(Process* p,		/* Current process. */
 	int actual_arity;
 
 	ep = *((Export **) (export_val(fun) + 1));
-	actual_arity = (int) ep->code[2];
+	actual_arity = ep->info.mfa.arity;
 
 	if (arity == actual_arity) {
-	    DTRACE_GLOBAL_CALL(p, ep->code[0], ep->code[1], (Uint)ep->code[2]);
+	    DTRACE_GLOBAL_CALL(p, &ep->info.mfa);
 	    return ep->addressv[erts_active_code_ix()];
 	} else {
 	    /*
@@ -7199,15 +7212,15 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity)
 	return 1;
     }
 
-    e.code[0] = Mod;
-    e.code[1] = Name;
-    e.code[2] = arity;
+    e.info.mfa.module = Mod;
+    e.info.mfa.function = Name;
+    e.info.mfa.arity = arity;
 
     if ((ep = export_get(&e)) == NULL) {
 	return 0;
     }
-    return ep->addressv[erts_active_code_ix()] == ep->code+3
-	&& (ep->code[3] == (BeamInstr) em_apply_bif);
+    return ep->addressv[erts_active_code_ix()] == ep->code
+	&& (ep->code[0] == (BeamInstr) em_apply_bif);
 }
 
 
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index d69b18e22f..c822cd8606 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -806,21 +806,21 @@ erts_finish_loading(Binary* magic, Process* c_p,
 
 	for (i = 0; i < export_list_size(code_ix); i++) {
 	    Export *ep = export_list(i, code_ix);
-	    if (ep == NULL || ep->code[0] != module) {
+	    if (ep == NULL || ep->info.mfa.module != module) {
 		continue;
 	    }
-	    if (ep->addressv[code_ix] == ep->code+3) {
-		if (ep->code[3] == (BeamInstr) em_apply_bif) {
+	    if (ep->addressv[code_ix] == ep->code) {
+		if (ep->code[0] == (BeamInstr) em_apply_bif) {
 		    continue;
-		} else if (ep->code[3] ==
+		} else if (ep->code[0] ==
 			   (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
 		    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
 		    ASSERT(mod_tab_p->curr.num_traced_exports > 0);
-		    erts_clear_export_break(mod_tab_p, ep->code+3);
-		    ep->addressv[code_ix] = (BeamInstr *) ep->code[4];
-		    ep->code[4] = 0;
+		    erts_clear_export_break(mod_tab_p, &ep->info);
+		    ep->addressv[code_ix] = (BeamInstr *) ep->code[1];
+		    ep->code[1] = 0;
 		}
-		ASSERT(ep->code[4] == 0);
+		ASSERT(ep->code[1] == 0);
 	    }
 	}
 	ASSERT(mod_tab_p->curr.num_breakpoints == 0);
@@ -1420,8 +1420,8 @@ load_import_table(LoaderState* stp)
 	 * the BIF function.
 	 */
 	if ((e = erts_active_export_entry(mod, func, arity)) != NULL) {
-	    if (e->code[3] == (BeamInstr) em_apply_bif) {
-		stp->import[i].bf = (BifFunction) e->code[4];
+	    if (e->code[0] == (BeamInstr) em_apply_bif) {
+		stp->import[i].bf = (BifFunction) e->code[1];
 		if (func == am_load_nif && mod == am_erlang && arity == 2) {
 		    stp->may_load_nif = 1;
 		}
@@ -1514,7 +1514,7 @@ is_bif(Eterm mod, Eterm func, unsigned arity)
     if (e == NULL) {
 	return 0;
     }
-    if (e->code[3] != (BeamInstr) em_apply_bif) {
+    if (e->code[0] != (BeamInstr) em_apply_bif) {
 	return 0;
     }
     if (mod == am_erlang && func == am_apply && arity == 3) {
@@ -1893,7 +1893,7 @@ load_code(LoaderState* stp)
      * by both the nif functionality and line instructions.
      */
     enum {
-	FUNC_INFO_SZ = 5
+	FUNC_INFO_SZ = sizeof(ErtsCodeInfo) / sizeof(Eterm)
     };
 
     code = stp->codev;
@@ -2565,8 +2565,11 @@ load_code(LoaderState* stp)
 		stp->function = code[ci-2];
 		stp->arity = code[ci-1];
 
+                /* When this assert is triggered, it is normally a sign that
+                   the size of the ops.tab i_func_info instruction is not
+                   the same as FUNC_INFO_SZ */
 		ASSERT(stp->labels[last_label].value == ci - FUNC_INFO_SZ);
-		stp->hdr->functions[function_number] = (BeamInstr*) stp->labels[last_label].patches;
+		stp->hdr->functions[function_number] = (ErtsCodeInfo*) stp->labels[last_label].patches;
 		offset = function_number;
 		stp->labels[last_label].patches = offset;
 		function_number++;
@@ -4513,7 +4516,7 @@ freeze_code(LoaderState* stp)
      * function table in the beginning of the file.
      */
 
-    code_hdr->functions[stp->num_functions] = (codev + stp->ci - 1);
+    code_hdr->functions[stp->num_functions] = (ErtsCodeInfo*)(codev + stp->ci - 1);
     CHKBLK(ERTS_ALC_T_CODE,code_hdr);
 
     /*
@@ -4781,7 +4784,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
 	     * callable yet. Keep any function in the current
 	     * code callable.
 	     */
-	    ep->code[4] = (BeamInstr) address;
+	    ep->code[1] = (BeamInstr) address;
 	}
     }
 
@@ -4959,7 +4962,7 @@ transform_engine(LoaderState* st)
 		if (i >= st->num_imports || st->import[i].bf == NULL)
 		    goto restart;
 		if (bif_number != -1 &&
-		    bif_export[bif_number]->code[4] != (BeamInstr) st->import[i].bf) {
+		    bif_export[bif_number]->code[1] != (BeamInstr) st->import[i].bf) {
 		    goto restart;
 		}
 	    }
@@ -5584,18 +5587,16 @@ functions_in_module(Process* p, /* Process whose heap to use. */
     hp = HAlloc(p, need);
     hp_end = hp + need;
     for (i = num_functions-1; i >= 0 ; i--) {
-	BeamInstr* func_info = code_hdr->functions[i];
-	Eterm name = (Eterm) func_info[3];
-	int arity = (int) func_info[4];
+	ErtsCodeInfo* ci = code_hdr->functions[i];
 	Eterm tuple;
 
 	/*
 	 * If the function name is [], this entry is a stub for
 	 * a BIF that should be ignored.
 	 */
-	ASSERT(is_atom(name) || is_nil(name));
-	if (is_atom(name)) {
-	    tuple = TUPLE2(hp, name, make_small(arity));
+	ASSERT(is_atom(ci->mfa.function) || is_nil(ci->mfa.function));
+	if (is_atom(ci->mfa.function)) {
+	    tuple = TUPLE2(hp, ci->mfa.function, make_small(ci->mfa.arity));
 	    hp += 3;
 	    result = CONS(hp, tuple, result);
 	    hp += 2;
@@ -5683,17 +5684,17 @@ native_addresses(Process* p, BeamCodeHeader* code_hdr)
     hp = HAlloc(p, need);
     hp_end = hp + need;
     for (i = num_functions-1; i >= 0 ; i--) {
-	BeamInstr* func_info = code_hdr->functions[i];
-	Eterm name = (Eterm) func_info[3];
-	int arity = (int) func_info[4];
+	ErtsCodeInfo *ci = code_hdr->functions[i];
 	Eterm tuple;
 
-	ASSERT(is_atom(name) || is_nil(name)); /* [] if BIF stub */
-	if (func_info[1] != 0) {
-	    Eterm addr;
-	    ASSERT(is_atom(name));
-	    addr = erts_bld_uint(&hp, NULL, func_info[1]);
-	    tuple = erts_bld_tuple(&hp, NULL, 3, name, make_small(arity), addr);
+	ASSERT(is_atom(ci->mfa.function)
+               || is_nil(ci->mfa.function)); /* [] if BIF stub */
+	if (ci->native != 0) {
+            Eterm addr;
+	    ASSERT(is_atom(ci->mfa.function));
+	    addr = erts_bld_uint(&hp, NULL, ci->native);
+	    tuple = erts_bld_tuple(&hp, NULL, 3, ci->mfa.function,
+                                   make_small(ci->mfa.arity), addr);
 	    result = erts_bld_cons(&hp, NULL, tuple, result);
 	}
     }
@@ -5719,11 +5720,11 @@ exported_from_module(Process* p, /* Process whose heap to use. */
     for (i = 0; i < export_list_size(code_ix); i++) {
 	Export* ep = export_list(i,code_ix);
 	
-	if (ep->code[0] == mod) {
+	if (ep->info.mfa.module == mod) {
 	    Eterm tuple;
 	    
-	    if (ep->addressv[code_ix] == ep->code+3 &&
-		ep->code[3] == (BeamInstr) em_call_error_handler) {
+	    if (ep->addressv[code_ix] == ep->code &&
+		ep->code[0] == (BeamInstr) em_call_error_handler) {
 		/* There is a call to the function, but it does not exist. */ 
 		continue;
 	    }
@@ -5733,7 +5734,8 @@ exported_from_module(Process* p, /* Process whose heap to use. */
 		hp = HAlloc(p, need);
 		hend = hp + need;
 	    }
-	    tuple = TUPLE2(hp, ep->code[1], make_small(ep->code[2]));
+	    tuple = TUPLE2(hp, ep->info.mfa.function,
+                           make_small(ep->info.mfa.arity));
 	    hp += 3;
 	    result = CONS(hp, tuple, result);
 	    hp += 2;
@@ -5807,7 +5809,6 @@ md5_of_module(Process* p, /* Process whose heap to use. */
 Eterm*
 erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p)
 {
-    BeamInstr* current = fi->current;
     Eterm loc = NIL;
 
     if (fi->loc != LINE_INVALID_LOCATION) {
@@ -5817,7 +5818,7 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p)
 	Eterm file_term = NIL;
 
 	if (file == 0) {
-	    Atom* ap = atom_tab(atom_val(fi->current[0]));
+	    Atom* ap = atom_tab(atom_val(fi->ci->mfa.module));
 	    file_term = buf_to_intlist(&hp, ".erl", 4, NIL);
 	    file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, file_term);
 	} else {
@@ -5836,10 +5837,12 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p)
     }
 
     if (is_list(args) || is_nil(args)) {
-	*mfa_p = TUPLE4(hp, current[0], current[1], args, loc);
+	*mfa_p = TUPLE4(hp, fi->ci->mfa.module, fi->ci->mfa.function,
+                        args, loc);
     } else {
-	Eterm arity = make_small(current[2]);
-	*mfa_p = TUPLE4(hp, current[0], current[1], arity, loc);
+	Eterm arity = make_small(fi->ci->mfa.arity);
+	*mfa_p = TUPLE4(hp, fi->ci->mfa.module, fi->ci->mfa.function,
+                        arity, loc);
     }
     return hp + 5;
 }
@@ -5850,9 +5853,9 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p)
  * the function.
  */
 void
-erts_set_current_function(FunctionInfo* fi, BeamInstr* current)
+erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa)
 {
-    fi->current = current;
+    fi->ci = erts_code_to_codeinfo(erts_codemfa_to_code(mfa));
     fi->needed = 5;
     fi->loc = LINE_INVALID_LOCATION;
 }
@@ -5861,13 +5864,13 @@ erts_set_current_function(FunctionInfo* fi, BeamInstr* current)
 /*
  * Returns a pointer to {module, function, arity}, or NULL if not found.
  */
-BeamInstr*
+ErtsCodeInfo*
 find_function_from_pc(BeamInstr* pc)
 {
     FunctionInfo fi;
 
     erts_lookup_function_info(&fi, pc, 0);
-    return fi.current;
+    return fi.ci;
 }
 
 /*
@@ -5984,24 +5987,25 @@ code_module_md5_1(BIF_ALIST_1)
     return res;
 }
 
-#define WORDS_PER_FUNCTION 6
+#define WORDS_PER_FUNCTION (sizeof(ErtsCodeInfo) / sizeof(UWord) + 1)
 
 static BeamInstr*
-make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode)
+make_stub(ErtsCodeInfo* info, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode)
 {
-    fp[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
-    fp[1] = native;
-    fp[2] = mod;
-    fp[3] = func;
-    fp[4] = arity;
+    ASSERT(WORDS_PER_FUNCTION == 6);
+    info->op = (BeamInstr) BeamOp(op_i_func_info_IaaI);
+    info->native = native;
+    info->mfa.module = mod;
+    info->mfa.function = func;
+    info->mfa.arity = arity;
 #ifdef HIPE
     if (native) {
-	fp[5] = BeamOpCode(op_move_return_n);
-	hipe_mfa_save_orig_beam_op(mod, func, arity, fp+5);
+        erts_codeinfo_to_code(info)[0] = BeamOpCode(op_move_return_n);
+	hipe_mfa_save_orig_beam_op(mod, func, arity, erts_codeinfo_to_code(info));
     }
 #endif
-    fp[5] = OpCode;
-    return fp + WORDS_PER_FUNCTION;
+    erts_codeinfo_to_code(info)[0] = OpCode;
+    return erts_codeinfo_to_code(info)+1;
 }
 
 static byte*
@@ -6060,22 +6064,19 @@ stub_read_export_table(LoaderState* stp)
 }
 
 static void
-stub_final_touch(LoaderState* stp, BeamInstr* fp)
+stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci)
 {
     unsigned int i;
     unsigned int n = stp->num_exps;
-    Eterm mod = fp[2];
-    Eterm function = fp[3];
-    int arity = fp[4];
 #ifdef HIPE
     Lambda* lp;
 #endif
 
-    if (is_bif(mod, function, arity)) {
-	fp[1] = 0;
-	fp[2] = 0;
-	fp[3] = 0;
-	fp[4] = 0;
+    if (is_bif(ci->mfa.module, ci->mfa.function, ci->mfa.arity)) {
+	ci->native = 0;
+	ci->mfa.module = 0;
+	ci->mfa.function = 0;
+	ci->mfa.arity = 0;
 	return;
     }
 
@@ -6084,9 +6085,12 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp)
      */
 
     for (i = 0; i < n; i++) {
-	if (stp->export[i].function == function && stp->export[i].arity == arity) {
-	    Export* ep = erts_export_put(mod, function, arity);
-	    ep->addressv[erts_staging_code_ix()] = fp+5;
+	if (stp->export[i].function == ci->mfa.function &&
+            stp->export[i].arity == ci->mfa.arity) {
+	    Export* ep = erts_export_put(ci->mfa.module,
+                                         ci->mfa.function,
+                                         ci->mfa.arity);
+	    ep->addressv[erts_staging_code_ix()] = erts_codeinfo_to_code(ci);
 	    return;
 	}
     }
@@ -6100,9 +6104,9 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp)
     n = stp->num_lambdas;
     for (i = 0, lp = stp->lambdas; i < n; i++, lp++) {
         ErlFunEntry* fe = stp->lambdas[i].fe;
-	if (lp->function == function && lp->arity == arity) {
-	    fp[5] = (Eterm) BeamOpCode(op_hipe_trap_call_closure);
-            fe->address = &(fp[5]);
+	if (lp->function == ci->mfa.function && lp->arity == ci->mfa.arity) {
+	    *erts_codeinfo_to_code(ci) = (Eterm) BeamOpCode(op_hipe_trap_call_closure);
+            fe->address = erts_codeinfo_to_code(ci);
 	}
     }
 #endif
@@ -6426,20 +6430,21 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 	 * Set the pointer and make the stub. Put a return instruction
 	 * as the body until we know what kind of trap we should put there.
 	 */
-	code_hdr->functions[i] = fp;
+	code_hdr->functions[i] = (ErtsCodeInfo*)fp;
 #ifdef HIPE
 	op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */
 #else
 	op = (Eterm) BeamOpCode(op_move_return_n);
 #endif
-	fp = make_stub(fp, Mod, func, arity, (Uint)native_address, op);
+	fp = make_stub((ErtsCodeInfo*)fp, Mod, func, arity,
+                       (Uint)native_address, op);
     }
 
     /*
      * Insert the last pointer and the int_code_end instruction.
      */
 
-    code_hdr->functions[i] = fp;
+    code_hdr->functions[i] = (ErtsCodeInfo*)fp;
     *fp++ = (BeamInstr) BeamOp(op_int_code_end);
 
     /*
@@ -6487,7 +6492,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 
     fp = code_base;
     for (i = 0; i < n; i++) {
-	stub_final_touch(stp, fp);
+	stub_final_touch(stp, (ErtsCodeInfo*)fp);
 	fp += WORDS_PER_FUNCTION;
     }
 
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 6be4031822..f67dfd7a81 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -37,14 +37,6 @@ typedef struct gen_op_entry {
 
 extern const GenOpEntry gen_opc[];
 
-#ifdef NO_JUMP_TABLE 
-#define BeamOp(Op) (Op)
-#else
-extern void** beam_ops;
-#define BeamOp(Op) beam_ops[(Op)]
-#endif
-
-
 extern BeamInstr beam_debug_apply[];
 extern BeamInstr* em_call_error_handler;
 extern BeamInstr* em_apply_bif;
@@ -115,7 +107,7 @@ typedef struct beam_code_header {
      * The actual loaded code (for the first function) start just beyond
      * this table.
      */
-    BeamInstr* functions[1];
+    ErtsCodeInfo* functions[1];
 
 }BeamCodeHeader;
 
diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c
index 55342a38c6..887f80e7e3 100644
--- a/erts/emulator/beam/beam_ranges.c
+++ b/erts/emulator/beam/beam_ranges.c
@@ -221,13 +221,13 @@ erts_ranges_sz(void)
 void
 erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info)
 {
-    BeamInstr** low;
-    BeamInstr** high;
-    BeamInstr** mid;
+    ErtsCodeInfo** low;
+    ErtsCodeInfo** high;
+    ErtsCodeInfo** mid;
     Range* rp;
     BeamCodeHeader* hdr;
 
-    fi->current = NULL;
+    fi->ci = NULL;
     fi->needed = 5;
     fi->loc = LINE_INVALID_LOCATION;
     rp = find_range(pc);
@@ -240,12 +240,12 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info)
     high = low + hdr->num_functions;
     while (low < high) {
 	mid = low + (high-low) / 2;
-	if (pc < mid[0]) {
+	if (pc < (BeamInstr*)(mid[0])) {
 	    high = mid;
-	} else if (pc < mid[1]) {
-	    fi->current = mid[0]+2;
+	} else if (pc < (BeamInstr*)(mid[1])) {
+	    fi->ci = mid[0];
 	    if (full_info) {
-		BeamInstr** fp = hdr->functions;
+		ErtsCodeInfo** fp = hdr->functions;
 		int idx = mid - fp;
 		lookup_loc(fi, pc, hdr, idx);
 	    }
@@ -316,7 +316,7 @@ lookup_loc(FunctionInfo* fi, const BeamInstr* pc,
 	    file = LOC_FILE(fi->loc);
 	    if (file == 0) {
 		/* Special case: Module name with ".erl" appended */
-		Atom* mod_atom = atom_tab(atom_val(fi->current[0]));
+		Atom* mod_atom = atom_tab(atom_val(fi->ci->mfa.module));
 		fi->needed += 2*(mod_atom->len+4);
 	    } else {
 		Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1]));
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 84eab5f651..324f7d4268 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4963,13 +4963,13 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a,
     int i;
     sys_memset((void *) ep, 0, sizeof(Export));
     for (i=0; i<ERTS_NUM_CODE_IX; i++) {
-	ep->addressv[i] = &ep->code[3];
+	ep->addressv[i] = ep->code;
     }
-    ep->code[0] = m;
-    ep->code[1] = f;
-    ep->code[2] = a;
-    ep->code[3] = (BeamInstr) em_apply_bif;
-    ep->code[4] = (BeamInstr) bif;
+    ep->info.mfa.module = m;
+    ep->info.mfa.function = f;
+    ep->info.mfa.arity = a;
+    ep->code[0] = (BeamInstr) em_apply_bif;
+    ep->code[1] = (BeamInstr) bif;
 }
 
 void erts_init_bif(void)
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 2203182a0d..0c85e19ef0 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -159,7 +159,7 @@ do {									\
 #define ERTS_BIF_ERROR_TRAPPED0(Proc, Reason, Bif)		\
 do {								\
     (Proc)->freason = (Reason);					\
-    (Proc)->current = (Bif)->code;				\
+    (Proc)->current = &(Bif)->info.mfa;                         \
     return THE_NON_VALUE; 					\
 } while (0)
 
@@ -167,7 +167,7 @@ do {								\
 do {								\
     Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
     (Proc)->freason = (Reason);					\
-    (Proc)->current = (Bif)->code;				\
+    (Proc)->current = &(Bif)->info.mfa;                         \
     reg[0] = (Eterm) (A0);					\
     return THE_NON_VALUE; 					\
 } while (0)
@@ -176,7 +176,7 @@ do {								\
 do {								\
     Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
     (Proc)->freason = (Reason);					\
-    (Proc)->current = (Bif)->code;				\
+    (Proc)->current = &(Bif)->info.mfa;                         \
     reg[0] = (Eterm) (A0);					\
     reg[1] = (Eterm) (A1);					\
     return THE_NON_VALUE; 					\
@@ -186,7 +186,7 @@ do {								\
 do {								\
     Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
     (Proc)->freason = (Reason);					\
-    (Proc)->current = (Bif)->code;				\
+    (Proc)->current = &(Bif)->info.mfa;                         \
     reg[0] = (Eterm) (A0);					\
     reg[1] = (Eterm) (A1);					\
     reg[2] = (Eterm) (A2);					\
@@ -202,7 +202,7 @@ do {						\
 #define ERTS_BIF_PREP_ERROR_TRAPPED0(Ret, Proc, Reason, Bif)	\
 do {								\
     (Proc)->freason = (Reason);					\
-    (Proc)->current = (Bif)->code;				\
+    (Proc)->current = &(Bif)->info.mfa;                         \
     (Ret) = THE_NON_VALUE;					\
 } while (0)
 
@@ -210,7 +210,7 @@ do {								\
 do {								\
     Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
     (Proc)->freason = (Reason);					\
-    (Proc)->current = (Bif)->code;				\
+    (Proc)->current = &(Bif)->info.mfa;                         \
     reg[0] = (Eterm) (A0);					\
     (Ret) = THE_NON_VALUE;					\
 } while (0)
@@ -219,7 +219,7 @@ do {								\
 do {								\
     Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
     (Proc)->freason = (Reason);					\
-    (Proc)->current = (Bif)->code;				\
+    (Proc)->current = &(Bif)->info.mfa;                         \
     reg[0] = (Eterm) (A0);					\
     reg[1] = (Eterm) (A1);					\
     (Ret) = THE_NON_VALUE;					\
@@ -229,7 +229,7 @@ do {								\
 do {								\
     Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
     (Proc)->freason = (Reason);					\
-    (Proc)->current = (Bif)->code;				\
+    (Proc)->current = &(Bif)->info.mfa;                         \
     reg[0] = (Eterm) (A0);					\
     reg[1] = (Eterm) (A1);					\
     reg[2] = (Eterm) (A2);					\
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 3c19e82b66..4ee00b53be 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -231,9 +231,9 @@ print_process_info(int to, void *to_arg, Process *p)
      * Display the initial function name
      */
     erts_print(to, to_arg, "Spawned as: %T:%T/%bpu\n",
-	       p->u.initial[INITIAL_MOD],
-	       p->u.initial[INITIAL_FUN],
-	       p->u.initial[INITIAL_ARI]);
+	       p->u.initial.module,
+	       p->u.initial.function,
+	       p->u.initial.arity);
     
     if (p->current != NULL) {
 	if (running) {
@@ -242,9 +242,9 @@ print_process_info(int to, void *to_arg, Process *p)
 	    erts_print(to, to_arg, "Current call: ");
 	}
 	erts_print(to, to_arg, "%T:%T/%bpu\n",
-		   p->current[0],
-		   p->current[1],
-		   p->current[2]);
+		   p->current->module,
+		   p->current->function,
+		   p->current->arity);
     }
 
     erts_print(to, to_arg, "Spawned by: %T\n", p->parent);
@@ -291,9 +291,9 @@ print_process_info(int to, void *to_arg, Process *p)
 		   erts_print(to, to_arg, "timeout");
 	     else
 		 erts_print(to, to_arg, "%T:%T/%bpu\n",
-			    scb->ct[j]->code[0],
-			    scb->ct[j]->code[1],
-			    scb->ct[j]->code[2]);
+			    scb->ct[j]->info.mfa.module,
+			    scb->ct[j]->info.mfa.function,
+			    scb->ct[j]->info.mfa.arity);
        }
        erts_print(to, to_arg, "\n");
     }
diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h
index 584a605771..1b451bf921 100644
--- a/erts/emulator/beam/code_ix.h
+++ b/erts/emulator/beam/code_ix.h
@@ -56,12 +56,49 @@
 #  endif
 #  include "sys.h"
 #endif
+
+#include "beam_opcodes.h"
+
 struct process;
 
 
 #define ERTS_NUM_CODE_IX 3
 typedef unsigned ErtsCodeIndex;
 
+typedef struct ErtsCodeMFA_ {
+    Eterm module;
+    Eterm function;
+    Uint  arity;
+} ErtsCodeMFA;
+
+/*
+ * The ErtsCodeInfo structure is used both in the Export entry
+ * and in the code as the function header.
+ */
+
+/* If you change the size of this, you also have to update the code
+   in ops.tab to reflect the new func_info size */
+typedef struct ErtsCodeInfo_ {
+    BeamInstr op;           /* OpCode(i_func_info) */
+    BeamInstr native;       /* Used by hipe and trace to store extra data */
+    ErtsCodeMFA mfa;
+} ErtsCodeInfo;
+
+/* Get the code associated with a ErtsCodeInfo ptr. */
+ERTS_GLB_INLINE
+BeamInstr *erts_codeinfo_to_code(ErtsCodeInfo *ci);
+
+/* Get the ErtsCodeInfo for from a code ptr. */
+ERTS_GLB_INLINE
+ErtsCodeInfo *erts_code_to_codeinfo(BeamInstr *I);
+
+/* Get the code associated with a ErtsCodeMFA ptr. */
+ERTS_GLB_INLINE
+BeamInstr *erts_codemfa_to_code(ErtsCodeMFA *mfa);
+
+/* Get the ErtsCodeMFA from a code ptr. */
+ERTS_GLB_INLINE
+ErtsCodeMFA *erts_code_to_codemfa(BeamInstr *I);
 
 /* Called once at emulator initialization.
  */
@@ -121,10 +158,47 @@ void erts_abort_staging_code_ix(void);
 int erts_has_code_write_permission(void);
 #endif
 
-
+/* module/function/arity can be NIL/NIL/-1 when the MFA is pointing to some
+   invalid code, for instance unloaded_fun. */
+#define ASSERT_MFA(MFA)                                                 \
+    ASSERT((is_atom((MFA)->module) || is_nil((MFA)->module)) &&         \
+           (is_atom((MFA)->function) || is_nil((MFA)->function)) &&     \
+           (((MFA)->arity >= 0 && (MFA)->arity < 1024) || (MFA)->arity == -1))
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
+ERTS_GLB_INLINE
+BeamInstr *erts_codeinfo_to_code(ErtsCodeInfo *ci)
+{
+    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op);
+    ASSERT_MFA(&ci->mfa);
+    return (BeamInstr*)(ci + 1);
+}
+
+ERTS_GLB_INLINE
+ErtsCodeInfo *erts_code_to_codeinfo(BeamInstr *I)
+{
+    ErtsCodeInfo *ci = ((ErtsCodeInfo *)(((char *)(I)) - sizeof(ErtsCodeInfo)));
+    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op);
+    ASSERT_MFA(&ci->mfa);
+    return ci;
+}
+
+ERTS_GLB_INLINE
+BeamInstr *erts_codemfa_to_code(ErtsCodeMFA *mfa)
+{
+    ASSERT_MFA(mfa);
+    return (BeamInstr*)(mfa + 1);
+}
+
+ERTS_GLB_INLINE
+ErtsCodeMFA *erts_code_to_codemfa(BeamInstr *I)
+{
+    ErtsCodeMFA *mfa = ((ErtsCodeMFA *)(((char *)(I)) - sizeof(ErtsCodeMFA)));
+    ASSERT_MFA(mfa);
+    return mfa;
+}
+
 extern erts_smp_atomic32_t the_active_code_index;
 extern erts_smp_atomic32_t the_staging_code_index;
 
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 75389107bb..a985ce4918 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1114,9 +1114,9 @@ process_info_aux(Process *BIF_P,
     case am_initial_call:
 	hp = HAlloc(BIF_P, 3+4);
 	res = TUPLE3(hp,
-		     rp->u.initial[INITIAL_MOD],
-		     rp->u.initial[INITIAL_FUN],
-		     make_small(rp->u.initial[INITIAL_ARI]));
+		     rp->u.initial.module,
+		     rp->u.initial.function,
+		     make_small(rp->u.initial.arity));
 	hp += 4;
 	break;
 
@@ -1563,9 +1563,9 @@ process_info_aux(Process *BIF_P,
 		    term = am_timeout;
 		else {
 		    term = TUPLE3(hp,
-				  scb->ct[j]->code[0],
-				  scb->ct[j]->code[1],
-				  make_small(scb->ct[j]->code[2]));
+				  scb->ct[j]->info.mfa.module,
+				  scb->ct[j]->info.mfa.function,
+				  make_small(scb->ct[j]->info.mfa.arity));
 		    hp += 4;
 		}
 		list = CONS(hp, term, list);
@@ -1614,10 +1614,12 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info)
 
     if (rp->current == NULL) {
 	erts_lookup_function_info(&fi, rp->i, full_info);
-	rp->current = fi.current;
+	rp->current = &fi.ci->mfa;
+        ASSERT_MFA(rp->current);
     } else if (full_info) {
+        ASSERT_MFA(rp->current);
 	erts_lookup_function_info(&fi, rp->i, full_info);
-	if (fi.current == NULL) {
+	if (fi.ci == NULL) {
 	    /* Use the current function without location info */
 	    erts_set_current_function(&fi, rp->current);
 	}
@@ -1633,9 +1635,9 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info)
 	 * instead if it can be looked up.
 	 */
 	erts_lookup_function_info(&fi2, rp->cp, full_info);
-	if (fi2.current) {
+	if (fi2.ci) {
 	    fi = fi2;
-	    rp->current = fi2.current;
+	    rp->current = &fi2.ci->mfa;
 	}
     }
 
@@ -1650,8 +1652,9 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info)
 	hp = erts_build_mfa_item(&fi, hp, am_true, &res);
     } else {
 	hp = HAlloc(BIF_P, 3+4);
-	res = TUPLE3(hp, rp->current[0],
-		     rp->current[1], make_small(rp->current[2]));
+	res = TUPLE3(hp, rp->current->module,
+		     rp->current->function,
+                     make_small(rp->current->arity));
 	hp += 4;
     }
     *hpp = hp;
@@ -1692,7 +1695,7 @@ current_stacktrace(Process* p, Process* rp, Eterm** hpp)
     heap_size = 3;
     for (i = 0; i < depth; i++) {
 	erts_lookup_function_info(stkp, s->trace[i], 1);
-	if (stkp->current) {
+	if (stkp->ci) {
 	    heap_size += stkp->needed + 2;
 	    stkp++;
 	}
@@ -3227,7 +3230,7 @@ fun_info_2(BIF_ALIST_2)
 	    break;
 	case am_module:
 	    hp = HAlloc(p, 3);
-	    val = exp->code[0];
+	    val = exp->info.mfa.module;
 	    break;
 	case am_new_index:
 	    hp = HAlloc(p, 3);
@@ -3255,11 +3258,11 @@ fun_info_2(BIF_ALIST_2)
 	    break;
 	case am_arity:
 	    hp = HAlloc(p, 3);
-	    val = make_small(exp->code[2]);
+	    val = make_small(exp->info.mfa.arity);
 	    break;
 	case am_name:
 	    hp = HAlloc(p, 3);
-	    val = exp->code[1];
+	    val = exp->info.mfa.function;
 	    break;
 	default:
 	    goto error;
@@ -3285,7 +3288,9 @@ fun_info_mfa_1(BIF_ALIST_1)
     } else if (is_export(fun)) {
 	Export* exp = (Export *) ((UWord) (export_val(fun))[1]);
 	hp = HAlloc(p, 4);
-	BIF_RET(TUPLE3(hp,exp->code[0],exp->code[1],make_small(exp->code[2])));
+	BIF_RET(TUPLE3(hp,exp->info.mfa.module,
+                       exp->info.mfa.function,
+                       make_small(exp->info.mfa.arity)));
     }
     BIF_ERROR(p, BADARG);
 }
diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c
index aecb8bf0c1..a594ec1493 100644
--- a/erts/emulator/beam/erl_bif_op.c
+++ b/erts/emulator/beam/erl_bif_op.c
@@ -260,7 +260,7 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2)
     } else if (is_export(arg1)) {
 	Export* exp = (Export *) (export_val(arg1)[1]);
 
-	if (exp->code[2] == (Uint) arity) {
+	if (exp->info.mfa.arity == (Uint) arity) {
 	    BIF_RET(am_true);
 	}
     }
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 66e5146da0..e3355208e5 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -125,7 +125,6 @@ erts_internal_trace_pattern_3(BIF_ALIST_3)
 static Eterm
 trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 {
-    DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */
     int i;
     int matches = -1;
     int specified = 0;
@@ -308,30 +307,30 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 	}
 	matches = 0;
     } else if (is_tuple(MFA)) {
+        ErtsCodeMFA mfa;
 	Eterm *tp = tuple_val(MFA);
 	if (tp[0] != make_arityval(3)) {
 	    goto error;
 	}
-	mfa[0] = tp[1];
-	mfa[1] = tp[2];
-	mfa[2] = tp[3];
-	if (!is_atom(mfa[0]) || !is_atom(mfa[1]) ||
-	    (!is_small(mfa[2]) && mfa[2] != am_Underscore)) {
+	if (!is_atom(tp[1]) || !is_atom(tp[2]) ||
+	    (!is_small(tp[3]) && tp[3] != am_Underscore)) {
 	    goto error;
 	}
-	for (i = 0; i < 3 && mfa[i] != am_Underscore; i++, specified++) {
+	for (i = 0; i < 3 && tp[i+1] != am_Underscore; i++, specified++) {
 	    /* Empty loop body */
 	}
 	for (i = specified; i < 3; i++) {
-	    if (mfa[i] != am_Underscore) {
+	    if (tp[i+1] != am_Underscore) {
 		goto error;
 	    }
 	}
-	if (is_small(mfa[2])) {
-	    mfa[2] = signed_val(mfa[2]);
+	mfa.module   = tp[1];
+	mfa.function = tp[2];
+	if (specified == 3) {
+            mfa.arity = signed_val(tp[3]);
 	}
 
-	matches = erts_set_trace_pattern(p, mfa, specified,
+	matches = erts_set_trace_pattern(p, &mfa, specified,
 					 match_prog_set, match_prog_set,
 					 on, flags, meta_tracer, 0);
     } else if (is_atom(MFA)) {
@@ -343,7 +342,6 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 
  error:
     MatchSetUnref(match_prog_set);
-    UnUseTmpHeap(3,p);
 
     ERTS_TRACER_CLEAR(&meta_tracer);
 
@@ -975,13 +973,14 @@ static int function_is_traced(Process *p,
     Export e;
     Export* ep;
     BeamInstr* pc;
+    ErtsCodeInfo *ci;
 
     /* First look for an export entry */
-    e.code[0] = mfa[0];
-    e.code[1] = mfa[1];
-    e.code[2] = mfa[2];
+    e.info.mfa.module = mfa[0];
+    e.info.mfa.function = mfa[1];
+    e.info.mfa.arity = mfa[2];
     if ((ep = export_get(&e)) != NULL) {
-	pc = ep->code+3;
+	pc = ep->code;
 	if (ep->addressv[erts_active_code_ix()] == pc &&
 	    *pc != (BeamInstr) em_call_error_handler) {
 
@@ -990,17 +989,17 @@ static int function_is_traced(Process *p,
 	    ASSERT(*pc == (BeamInstr) em_apply_bif ||
 		   *pc == (BeamInstr) BeamOp(op_i_generic_breakpoint));
 
-	    if (erts_is_trace_break(pc, ms, 0)) {
+	    if (erts_is_trace_break(&ep->info, ms, 0)) {
 		return FUNC_TRACE_GLOBAL_TRACE;
 	    }
 
-	    if (erts_is_trace_break(pc, ms, 1)) {
+	    if (erts_is_trace_break(&ep->info, ms, 1)) {
 		r |= FUNC_TRACE_LOCAL_TRACE;
 	    }
-	    if (erts_is_mtrace_break(pc, ms_meta, tracer_pid_meta)) {
+	    if (erts_is_mtrace_break(&ep->info, ms_meta, tracer_pid_meta)) {
 		r |= FUNC_TRACE_META_TRACE;
 	    }
-	    if (erts_is_time_break(p, pc, call_time)) {
+	    if (erts_is_time_break(p, &ep->info, call_time)) {
 		r |= FUNC_TRACE_TIME_TRACE;
 	    }
 	    return r ? r : FUNC_TRACE_UNTRACED;
@@ -1008,15 +1007,15 @@ static int function_is_traced(Process *p,
     }
     
     /* OK, now look for breakpoint tracing */
-    if ((pc = erts_find_local_func(mfa)) != NULL) {
+    if ((ci = erts_find_local_func(&e.info.mfa)) != NULL) {
 	int r = 
-	    (erts_is_trace_break(pc, ms, 1)
+	    (erts_is_trace_break(ci, ms, 1)
 	     ? FUNC_TRACE_LOCAL_TRACE : 0) 
-	    | (erts_is_mtrace_break(pc, ms_meta, tracer_pid_meta)
+	    | (erts_is_mtrace_break(ci, ms_meta, tracer_pid_meta)
 	       ? FUNC_TRACE_META_TRACE : 0)
-	    | (erts_is_count_break(pc, count)
+	    | (erts_is_count_break(ci, count)
 	       ? FUNC_TRACE_COUNT_TRACE : 0)
-	    | (erts_is_time_break(p, pc, call_time)
+	    | (erts_is_time_break(p, ci, call_time)
 	       ? FUNC_TRACE_TIME_TRACE : 0);
 	
 	return r ? r : FUNC_TRACE_UNTRACED;
@@ -1350,7 +1349,7 @@ trace_info_event(Process* p, Eterm event, Eterm key)
 #undef FUNC_TRACE_LOCAL_TRACE
 
 int
-erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
+erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
 		       Binary* match_prog_set, Binary *meta_match_prog_set,
 		       int on, struct trace_pattern_flags flags,
 		       ErtsTracer meta_tracer, int is_blocking)
@@ -1370,22 +1369,23 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
     n = finish_bp.e.matched;
 
     for (i = 0; i < n; i++) {
-	BeamInstr* pc = fp[i].pc;
-	Export* ep = ErtsContainerStruct(pc, Export, code[3]);
+        ErtsCodeInfo *ci = fp[i].ci;
+	BeamInstr* pc = erts_codeinfo_to_code(ci);
+	Export* ep = ErtsContainerStruct(ci, Export, info);
 
 	if (on && !flags.breakpoint) {
 	    /* Turn on global call tracing */
 	    if (ep->addressv[code_ix] != pc) {
 		fp[i].mod->curr.num_traced_exports++;
 #ifdef DEBUG
-		pc[-5] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
+		ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI);
 #endif
-		pc[0] = (BeamInstr) BeamOp(op_jump_f);
-		pc[1] = (BeamInstr) ep->addressv[code_ix];
+                ep->code[0] = (BeamInstr) BeamOp(op_jump_f);
+		ep->code[1] = (BeamInstr) ep->addressv[code_ix];
 	    }
-	    erts_set_call_trace_bif(pc, match_prog_set, 0);
+	    erts_set_call_trace_bif(ci, match_prog_set, 0);
 	    if (ep->addressv[code_ix] != pc) {
-		pc[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint);
+		ep->code[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint);
 	    }
 	} else if (!on && flags.breakpoint) {
 	    /* Turn off breakpoint tracing -- nothing to do here. */
@@ -1394,9 +1394,9 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
 	     * Turn off global tracing, either explicitly or implicitly
 	     * before turning on breakpoint tracing.
 	     */
-	    erts_clear_call_trace_bif(pc, 0);
-	    if (pc[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
-		pc[0] = (BeamInstr) BeamOp(op_jump_f);
+	    erts_clear_call_trace_bif(ci, 0);
+	    if (ep->code[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
+		ep->code[0] = (BeamInstr) BeamOp(op_jump_f);
 	    }
 	}
     }
@@ -1406,68 +1406,76 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
     */
     for (i = 0; i < BIF_SIZE; ++i) {
 	Export *ep = bif_export[i];
-	int j;
-	
+
 	if (!ExportIsBuiltIn(ep)) {
 	    continue;
 	}
-	
+
 	if (bif_table[i].f == bif_table[i].traced) {
 	    /* Trace wrapper same as regular function - untraceable */
 	    continue;
 	}
-	
-	for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) {
-	    /* Empty loop body */
-	}
-	if (j == specified) {
-	    BeamInstr* pc = (BeamInstr *)bif_export[i]->code + 3;
 
-	    if (! flags.breakpoint) { /* Export entry call trace */
-		if (on) {
-		    erts_clear_call_trace_bif(pc, 1);
-		    erts_clear_mtrace_bif(pc);
-		    erts_set_call_trace_bif(pc, match_prog_set, 0);
-		} else { /* off */
-		    erts_clear_call_trace_bif(pc, 0);
-		}
-		matches++;
-	    } else { /* Breakpoint call trace */
-		int m = 0;
-		
-		if (on) {
-		    if (flags.local) {
-			erts_clear_call_trace_bif(pc, 0);
-			erts_set_call_trace_bif(pc, match_prog_set, 1);
-			m = 1;
-		    }
-		    if (flags.meta) {
-			erts_set_mtrace_bif(pc, meta_match_prog_set,
-					    meta_tracer);
-			m = 1;
-		    }
-		    if (flags.call_time) {
-			erts_set_time_trace_bif(pc, on);
-			/* I don't want to remove any other tracers */
-			m = 1;
-		    }
-		} else { /* off */
-		    if (flags.local) {
-			erts_clear_call_trace_bif(pc, 1);
-			m = 1;
-		    }
-		    if (flags.meta) {
-			erts_clear_mtrace_bif(pc);
-			m = 1;
-		    }
-		    if (flags.call_time) {
-			erts_clear_time_trace_bif(pc);
-			m = 1;
-		    }
-		}
-		matches += m;
-	    }
-	}
+        switch (specified) {
+        case 3:
+            if (mfa->arity != ep->info.mfa.arity)
+                continue;
+        case 2:
+            if (mfa->function != ep->info.mfa.function)
+                continue;
+        case 1:
+            if (mfa->module != ep->info.mfa.module)
+                continue;
+        case 0:
+            break;
+        default:
+            ASSERT(0);
+        }
+
+        if (! flags.breakpoint) { /* Export entry call trace */
+            if (on) {
+                erts_clear_call_trace_bif(&ep->info, 1);
+                erts_clear_mtrace_bif(&ep->info);
+                erts_set_call_trace_bif(&ep->info, match_prog_set, 0);
+            } else { /* off */
+                erts_clear_call_trace_bif(&ep->info, 0);
+            }
+            matches++;
+        } else { /* Breakpoint call trace */
+            int m = 0;
+
+            if (on) {
+                if (flags.local) {
+                    erts_clear_call_trace_bif(&ep->info, 0);
+                    erts_set_call_trace_bif(&ep->info, match_prog_set, 1);
+                    m = 1;
+                }
+                if (flags.meta) {
+                    erts_set_mtrace_bif(&ep->info, meta_match_prog_set,
+                                        meta_tracer);
+                    m = 1;
+                }
+                if (flags.call_time) {
+                    erts_set_time_trace_bif(&ep->info, on);
+                    /* I don't want to remove any other tracers */
+                    m = 1;
+                }
+            } else { /* off */
+                if (flags.local) {
+                    erts_clear_call_trace_bif(&ep->info, 1);
+                    m = 1;
+                }
+                if (flags.meta) {
+                    erts_clear_mtrace_bif(&ep->info);
+                    m = 1;
+                }
+                if (flags.call_time) {
+                    erts_clear_time_trace_bif(&ep->info);
+                    m = 1;
+                }
+            }
+            matches += m;
+        }
     }
 
     /*
@@ -1669,10 +1677,9 @@ install_exp_breakpoints(BpFunctions* f)
     Uint i;
 
     for (i = 0; i < ne; i++) {
-	BeamInstr* pc = fp[i].pc;
-	Export* ep = ErtsContainerStruct(pc, Export, code[3]);
+	Export* ep = ErtsContainerStruct(fp[i].ci, Export, info);
 
-	ep->addressv[code_ix] = pc;
+	ep->addressv[code_ix] = ep->code;
     }
 }
 
@@ -1685,14 +1692,13 @@ uninstall_exp_breakpoints(BpFunctions* f)
     Uint i;
 
     for (i = 0; i < ne; i++) {
-	BeamInstr* pc = fp[i].pc;
-	Export* ep = ErtsContainerStruct(pc, Export, code[3]);
+	Export* ep = ErtsContainerStruct(fp[i].ci, Export, info);
 
-	if (ep->addressv[code_ix] != pc) {
+	if (ep->addressv[code_ix] != ep->code) {
 	    continue;
 	}
-	ASSERT(*pc == (BeamInstr) BeamOp(op_jump_f));
-	ep->addressv[code_ix] = (BeamInstr *) ep->code[4];
+	ASSERT(ep->code[0] == (BeamInstr) BeamOp(op_jump_f));
+	ep->addressv[code_ix] = (BeamInstr *) ep->code[1];
     }
 }
 
@@ -1705,15 +1711,14 @@ clean_export_entries(BpFunctions* f)
     Uint i;
 
     for (i = 0; i < ne; i++) {
-	BeamInstr* pc = fp[i].pc;
-	Export* ep = ErtsContainerStruct(pc, Export, code[3]);
+	Export* ep = ErtsContainerStruct(fp[i].ci, Export, info);
 
-	if (ep->addressv[code_ix] == pc) {
+	if (ep->addressv[code_ix] == ep->code) {
 	    continue;
 	}
-	if (*pc == (BeamInstr) BeamOp(op_jump_f)) {
-	    ep->code[3] = (BeamInstr) 0;
-	    ep->code[4] = (BeamInstr) 0;
+	if (ep->code[0] == (BeamInstr) BeamOp(op_jump_f)) {
+	    ep->code[0] = (BeamInstr) 0;
+	    ep->code[1] = (BeamInstr) 0;
 	}
     }
 }
@@ -1725,11 +1730,11 @@ setup_bif_trace(void)
 
     for (i = 0; i < BIF_SIZE; ++i) {
 	Export *ep = bif_export[i];
-	GenericBp* g = (GenericBp *) ep->fake_op_func_info_for_hipe[1];
+	GenericBp* g = (GenericBp *) ep->info.native;
 	if (g) {
 	    if (ExportIsBuiltIn(ep)) {
-		ASSERT(ep->code[4]);
-		ep->code[4] = (BeamInstr) bif_table[i].traced;
+		ASSERT(ep->code[1]);
+		ep->code[1] = (BeamInstr) bif_table[i].traced;
 	    }
 	}
     }
@@ -1743,12 +1748,11 @@ reset_bif_trace(void)
 
     for (i = 0; i < BIF_SIZE; ++i) {
 	Export *ep = bif_export[i];
-	BeamInstr* pc = ep->code+3;
-	GenericBp* g = (GenericBp *) pc[-4];
+	GenericBp* g = (GenericBp *) ep->info.native;
 	if (g && g->data[active].flags == 0) {
 	    if (ExportIsBuiltIn(ep)) {
-		ASSERT(ep->code[4]);
-		ep->code[4] = (BeamInstr) bif_table[i].f;
+		ASSERT(ep->code[1]);
+		ep->code[1] = (BeamInstr) bif_table[i].f;
 	    }
 	}
     }
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 6732b708a8..988467ce44 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1769,7 +1769,7 @@ Eterm db_prog_match(Process *c_p,
     Eterm t;
     Eterm *esp;
     MatchVariable* variables;
-    BeamInstr *cp;
+    ErtsCodeInfo *cp;
     const UWord *pc = prog->text;
     Eterm *ehp;
     Eterm ret;
@@ -2408,9 +2408,9 @@ restart:
 		ehp = HAllocX(build_proc, 4, HEAP_XTRA);
  		*esp++ = make_tuple(ehp);
 		ehp[0] = make_arityval(3);
-		ehp[1] = cp[0];
-		ehp[2] = cp[1];
-		ehp[3] = make_small((Uint) cp[2]);
+		ehp[1] = cp->mfa.module;
+		ehp[2] = cp->mfa.function;
+		ehp[3] = make_small((Uint) cp->mfa.arity);
 	    }
 	    break;
         case matchSilent:
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index c639ba623f..9deec946d5 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -49,8 +49,8 @@ static void fun_free(ErlFunEntry* obj);
  * to unloaded_fun[]. The -1 in unloaded_fun[0] will be interpreted
  * as an illegal arity when attempting to call a fun.
  */
-static BeamInstr unloaded_fun_code[3] = {NIL, -1, 0};
-static BeamInstr* unloaded_fun = unloaded_fun_code + 2;
+static BeamInstr unloaded_fun_code[4] = {NIL, NIL, -1, 0};
+static BeamInstr* unloaded_fun = unloaded_fun_code + 3;
 
 void
 erts_init_fun_table(void)
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 7442e99cb3..f0535451fc 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2285,7 +2285,7 @@ typedef struct {
     Export exp;
     struct erl_module_nif* m;
     NativeFunPtr fp;
-    BeamInstr *saved_current;
+    ErtsCodeMFA *saved_current;
     int exception_thrown;
     int saved_argc;
     int rootset_extra;
@@ -2328,9 +2328,9 @@ allocate_nif_sched_data(Process* proc, int argc)
     ep->rootset_extra = argc;
     ep->rootset[0] = NIL;
     for (i=0; i<ERTS_NUM_CODE_IX; i++) {
-	ep->exp.addressv[i] = &ep->exp.code[3];
+	ep->exp.addressv[i] = &ep->exp.code[0];
     }
-    ep->exp.code[3] = (BeamInstr) em_call_nif;
+    ep->exp.code[0] = (BeamInstr) em_call_nif;
     (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ep);
     return ep;
 }
@@ -2401,10 +2401,10 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec
 	ep->saved_argc = argc;
     }
     proc->i = (BeamInstr*) ep->exp.addressv[0];
-    ep->exp.code[0] = (BeamInstr) proc->current[0];
-    ep->exp.code[1] = (BeamInstr) proc->current[1];
-    ep->exp.code[2] = argc;
-    ep->exp.code[4] = (BeamInstr) direct_fp;
+    ep->exp.info.mfa.module = proc->current->module;
+    ep->exp.info.mfa.function = proc->current->function;
+    ep->exp.info.mfa.arity = argc;
+    ep->exp.code[1] = (BeamInstr) direct_fp;
     ep->m = env->mod_nif;
     ep->fp = indirect_fp;
     proc->freason = TRAP;
@@ -2426,7 +2426,7 @@ restore_nif_mfa(Process* proc, NifExport* ep, int exception)
     ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc)
 		       & ERTS_PROC_LOCK_MAIN);
 
-    ASSERT(ep->saved_current != &ep->exp.code[0]);
+    ASSERT(ep->saved_current != &ep->exp.info.mfa);
     proc->current = ep->saved_current;
     ep->saved_current = NULL;
     if (exception) {
@@ -2500,7 +2500,8 @@ execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 
     execution_state(env, &proc, NULL);
 
-    fp = (NativeFunPtr) proc->current[6];
+    ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa);
+    fp = ep->fp;
 
     ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc)));
 
@@ -2569,7 +2570,8 @@ schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[
 	erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
     }
 
-    fp = (NativeFunPtr) proc->current[6];
+    ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa);
+    fp = ep->fp;
 
     ASSERT(fp);
 
@@ -2629,7 +2631,8 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     ERL_NIF_TERM result;
 
     execution_state(env, &proc, NULL);
-    fp = (NativeFunPtr) proc->current[6];
+    ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa);
+    fp = ep->fp;
 
     ASSERT(!env->exception_thrown);
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
@@ -2697,7 +2700,7 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags,
 
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     ASSERT(ep);
-    ep->exp.code[1] = (BeamInstr) fun_name_atom;
+    ep->exp.info.mfa.function = (BeamInstr) fun_name_atom;
 
 done:
     if (scheduler < 0)
@@ -3011,17 +3014,16 @@ int enif_map_iterator_get_pair(ErlNifEnv *env,
  ***************************************************************************/
 
 
-static BeamInstr** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsigned arity)
+static ErtsCodeInfo** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsigned arity)
 {
     int n = (int) mod_code->num_functions;
     int j;
     for (j = 0; j < n; ++j) {
-	BeamInstr* code_ptr = (BeamInstr*) mod_code->functions[j];
-	ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
-	if (f_atom == ((Eterm) code_ptr[3])
-	    && arity == ((unsigned) code_ptr[4])) {
-
-	    return (BeamInstr**) &mod_code->functions[j];
+	ErtsCodeInfo* ci = mod_code->functions[j];
+	ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+	if (f_atom == ci->mfa.function
+	    && arity == ci->mfa.arity) {
+	    return mod_code->functions+j;
 	}
     }
     return NULL;
@@ -3156,7 +3158,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     Eterm mod_atom;
     const Atom* mod_atomp;
     Eterm f_atom;
-    BeamInstr* caller;
+    ErtsCodeInfo* caller;
     ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT;
     Eterm ret = am_ok;
     int veto;
@@ -3189,12 +3191,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 
     /* Find calling module */
     ASSERT(BIF_P->current != NULL);
-    ASSERT(BIF_P->current[0] == am_erlang
-	   && BIF_P->current[1] == am_load_nif 
-	   && BIF_P->current[2] == 2);
+    ASSERT(BIF_P->current->module == am_erlang
+	   && BIF_P->current->function == am_load_nif 
+	   && BIF_P->current->arity == 2);
     caller = find_function_from_pc(BIF_P->cp);
     ASSERT(caller != NULL);
-    mod_atom = caller[0];
+    mod_atom = caller->mfa.module;
     ASSERT(is_atom(mod_atom));
     module_p = erts_get_module(mod_atom, erts_active_code_ix());
     ASSERT(module_p != NULL);
@@ -3271,9 +3273,9 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	int incr = 0;
 	ErlNifFunc* f = entry->funcs;
 	for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) {
-	    BeamInstr** code_pp;
+	    ErtsCodeInfo** ci_pp;
 	    if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1)
-		|| (code_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) {
+		|| (ci_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) {
 		ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u",
 				     mod_atom, f->name, f->arity);
 	    }
@@ -3295,9 +3297,9 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 #endif
 	    }
 #ifdef ERTS_DIRTY_SCHEDULERS
-	    else if (code_pp[1] - code_pp[0] < (5+4))
+	    else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < (4))
 #else
-	    else if (code_pp[1] - code_pp[0] < (5+3))
+	    else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < (3))
 #endif
 	    {
 		ret = load_nif_error(BIF_P,bad_lib,"No explicit call to load_nif"
@@ -3365,31 +3367,33 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	this_mi->nif = lib;
 	for (i=0; i < entry->num_of_funcs; i++)
 	{
-	    BeamInstr* code_ptr;
+	    ErtsCodeInfo* ci;
+            BeamInstr *code_ptr;
 	    erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1);
-	    code_ptr = *get_func_pp(this_mi->code_hdr, f_atom, f->arity);
+	    ci = *get_func_pp(this_mi->code_hdr, f_atom, f->arity);
+            code_ptr = erts_codeinfo_to_code(ci);
 
-	    if (code_ptr[1] == 0) {
-		code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif);
+	    if (ci->native == 0) {
+		code_ptr[0] = (BeamInstr) BeamOp(op_call_nif);
 	    }
 	    else { /* Function traced, patch the original instruction word */
-		GenericBp* g = (GenericBp *) code_ptr[1];
-		ASSERT(code_ptr[5+0] ==
+		GenericBp* g = (GenericBp *) ci->native;
+		ASSERT(code_ptr[0] ==
 		       (BeamInstr) BeamOp(op_i_generic_breakpoint));
 		g->orig_instr = (BeamInstr) BeamOp(op_call_nif);
 	    }
 #ifdef ERTS_DIRTY_SCHEDULERS
 	    if ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7))
 		&& (entry->options & ERL_NIF_DIRTY_NIF_OPTION) && f->flags) {
-		code_ptr[5+3] = (BeamInstr) f->fptr;
-		code_ptr[5+1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ?
+		code_ptr[3] = (BeamInstr) f->fptr;
+		code_ptr[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ?
 		    (BeamInstr) schedule_dirty_io_nif :
 		    (BeamInstr) schedule_dirty_cpu_nif;
 	    }
 	    else
 #endif
-		code_ptr[5+1] = (BeamInstr) f->fptr;
-	    code_ptr[5+2] = (BeamInstr) lib;
+		code_ptr[1] = (BeamInstr) f->fptr;
+	    code_ptr[2] = (BeamInstr) lib;
 	    f = next_func(entry, &incr, f);
 	}
     }
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index 1a579704a8..b43a1b0190 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -526,8 +526,8 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) {
 	case EXPORT_DEF:
 	    {
 		Export* ep = *((Export **) (export_val(wobj) + 1));
-		Atom* module = atom_tab(atom_val(ep->code[0]));
-		Atom* name = atom_tab(atom_val(ep->code[1]));
+		Atom* module = atom_tab(atom_val(ep->info.mfa.module));
+		Atom* name = atom_tab(atom_val(ep->info.mfa.function));
 
 		PRINT_STRING(res, fn, arg, "#Fun<");
 		PRINT_BUF(res, fn, arg, module->name, module->len);
@@ -535,7 +535,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) {
 		PRINT_BUF(res, fn, arg, name->name, name->len);
 		PRINT_CHAR(res, fn, arg, '.');
 		PRINT_SWORD(res, fn, arg, 'd', 0, 1,
-			    (ErlPfSWord) ep->code[2]);
+			    (ErlPfSWord) ep->info.mfa.arity);
 		PRINT_CHAR(res, fn, arg, '>');
 	    }
 	    break;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 269518d8d6..f7f391e322 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11464,9 +11464,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->schedule_count = 0;
     ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0));
 
-    p->u.initial[INITIAL_MOD] = mod;
-    p->u.initial[INITIAL_FUN] = func;
-    p->u.initial[INITIAL_ARI] = (Uint) arity;
+    p->u.initial.module = mod;
+    p->u.initial.function = func;
+    p->u.initial.arity = (Uint) arity;
 
     /*
      * Must initialize binary lists here before copying binaries to process.
@@ -11508,7 +11508,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 
     /* No need to initialize p->fcalls. */
 
-    p->current = p->u.initial+INITIAL_MOD;
+    p->current = &p->u.initial;
 
     p->i = (BeamInstr *) beam_apply;
     p->cp = (BeamInstr *) beam_apply+1;
@@ -11769,9 +11769,9 @@ void erts_init_empty_process(Process *p)
     p->seq_trace_clock = 0;
     p->seq_trace_lastcnt = 0;
     p->seq_trace_token = NIL;
-    p->u.initial[0] = 0;
-    p->u.initial[1] = 0;
-    p->u.initial[2] = 0;
+    p->u.initial.module = 0;
+    p->u.initial.function = 0;
+    p->u.initial.arity = 0;
     p->catches = 0;
     p->cp = NULL;
     p->i = NULL;
@@ -13220,8 +13220,8 @@ erts_program_counter_info(int to, void *to_arg, Process *p)
 static void
 print_function_from_pc(int to, void *to_arg, BeamInstr* x)
 {
-    BeamInstr* addr = find_function_from_pc(x);
-    if (addr == NULL) {
+    ErtsCodeInfo *ci = find_function_from_pc(x);
+    if (ci == NULL) {
         if (x == beam_exit) {
             erts_print(to, to_arg, "<terminate process>");
         } else if (x == beam_continue_exit) {
@@ -13235,7 +13235,8 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x)
         }
     } else {
 	erts_print(to, to_arg, "%T:%T/%d + %d",
-		   addr[0], addr[1], addr[2], ((x-addr)-2) * sizeof(Eterm));
+		   ci->mfa.module, ci->mfa.function, ci->mfa.arity,
+                   (x-(BeamInstr*)ci) * sizeof(Eterm));
     }
 }
 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 3347a7a60e..b266d32e76 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -93,10 +93,6 @@ struct ErtsNodesMonitor_;
 #define ERTS_HEAP_FREE(Type, Ptr, Size)					\
      erts_free((Type), (Ptr))
 
-#define INITIAL_MOD 0
-#define INITIAL_FUN 1
-#define INITIAL_ARI 2
-
 #include "export.h"
 
 struct saved_calls {
@@ -1022,15 +1018,16 @@ struct process {
 #endif
     union {
 	void *terminate;
-	BeamInstr initial[3];	/* Initial module(0), function(1), arity(2), often used instead
-				   of pointer to funcinfo instruction, hence the BeamInstr datatype */
+	ErtsCodeMFA initial;	/* Initial module(0), function(1), arity(2),
+                                   often used instead of pointer to funcinfo
+                                   instruction. */
     } u;
-    BeamInstr* current;		/* Current Erlang function, part of the funcinfo:
+    ErtsCodeMFA* current;	/* Current Erlang function, part of the funcinfo:
 				 * module(0), function(1), arity(2)
 				 * (module and functions are tagged atoms;
-				 * arity an untagged integer). BeamInstr * because it references code
+				 * arity an untagged integer).
 				 */
-    
+
     /*
      * Information mainly for post-mortem use (erl crash dump).
      */
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index a70dfb8e73..645bedd8f1 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -334,8 +334,8 @@ stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg)
 static void
 print_function_from_pc(int to, void *to_arg, BeamInstr* x)
 {
-    BeamInstr* addr = find_function_from_pc(x);
-    if (addr == NULL) {
+    ErtsCodeInfo* ci = find_function_from_pc(x);
+    if (ci == NULL) {
         if (x == beam_exit) {
             erts_print(to, to_arg, "<terminate process>");
         } else if (x == beam_continue_exit) {
@@ -347,7 +347,8 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x)
         }
     } else {
 	erts_print(to, to_arg, "%T:%T/%bpu + %bpu",
-		   addr[0], addr[1], addr[2], ((x-addr)-2) * sizeof(Eterm));
+		   ci->mfa.module, ci->mfa.function, ci->mfa.arity,
+                   (x-(BeamInstr*)ci) * sizeof(Eterm));
     }
 }
 
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 8c84303997..566529d2d4 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -773,7 +773,7 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what)
 	curr_func = 0;
     else {
 	if (!p->current)
-	    p->current = find_function_from_pc(p->i);
+	    p->current = &find_function_from_pc(p->i)->mfa;
 	curr_func = p->current != NULL;
     }
 
@@ -781,7 +781,8 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what)
 	tmp = make_small(0);
     } else {
         hp = HAlloc(p, 4);
-	tmp = TUPLE3(hp,p->current[0],p->current[1],make_small(p->current[2]));
+	tmp = TUPLE3(hp,p->current->module,p->current->function,
+                     make_small(p->current->arity));
 	hp += 4;
     }
 
@@ -1027,14 +1028,14 @@ erts_trace_return_to(Process *p, BeamInstr *pc)
 {
     Eterm mfa;
 
-    BeamInstr *code_ptr = find_function_from_pc(pc);
+    ErtsCodeInfo *ci = find_function_from_pc(pc);
 
-
-    if (!code_ptr) {
+    if (!ci) {
 	mfa = am_undefined;
     } else {
         Eterm *hp = HAlloc(p, 4);
-	mfa = TUPLE3(hp, code_ptr[0], code_ptr[1], make_small(code_ptr[2]));
+	mfa = TUPLE3(hp, ci->mfa.module, ci->mfa.function,
+                     make_small(ci->mfa.arity));
     }
 
     send_to_tracer_nif(p, &p->common, p->common.id, NULL, TRACE_FUN_T_CALL,
@@ -1046,11 +1047,11 @@ erts_trace_return_to(Process *p, BeamInstr *pc)
  * or   {trace, Pid, return_from, {Mod, Name, Arity}, Retval}
  */
 void
-erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer)
+erts_trace_return(Process* p, ErtsCodeMFA *mfa,
+                  Eterm retval, ErtsTracer *tracer)
 {
     Eterm* hp;
-    Eterm mfa, mod, name;
-    int arity;
+    Eterm mfa_tuple;
     Uint meta_flags, *tracee_flags;
 
     ASSERT(tracer);
@@ -1084,15 +1085,13 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer)
 	tracee_flags = &meta_flags;
     }
 
-    mod = fi[0];
-    name = fi[1];
-    arity = fi[2];
-
     hp = HAlloc(p, 4);
-    mfa = TUPLE3(hp, mod, name, make_small(arity));
+    mfa_tuple = TUPLE3(hp, mfa->module, mfa->function,
+                       make_small(mfa->arity));
     hp += 4;
     send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id,
-                           NULL, TRACE_FUN_T_CALL, am_return_from, mfa, retval, am_true);
+                           NULL, TRACE_FUN_T_CALL, am_return_from, mfa_tuple,
+                           retval, am_true);
 }
 
 /* Send {trace_ts, Pid, exception_from, {Mod, Name, Arity}, {Class,Value}, 
@@ -1103,7 +1102,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer)
  * Where Class is atomic but Value is any term.
  */
 void
-erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value,
+erts_trace_exception(Process* p, ErtsCodeMFA *mfa, Eterm class, Eterm value,
 		     ErtsTracer *tracer)
 {
     Eterm* hp;
@@ -1142,7 +1141,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value,
     }
 
     hp = HAlloc(p, 7);;
-    mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm)mfa[2]));
+    mfa_tuple = TUPLE3(hp, mfa->module, mfa->function, make_small(mfa->arity));
     hp += 4;
     cv = TUPLE2(hp, class, value);
     hp += 3;
@@ -1165,7 +1164,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value,
  * if it is a pid or port we do a meta trace.
  */
 Uint32
-erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
+erts_call_trace(Process* p, ErtsCodeInfo *info, Binary *match_spec,
 		Eterm* args, int local, ErtsTracer *tracer)
 {
     Eterm* hp;
@@ -1244,7 +1243,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
      * such as size_object() and copy_struct(), we must make sure that we
      * temporarily convert any match contexts to sub binaries.
      */
-    arity = (Eterm) mfa[2];
+    arity = info->mfa.arity;
     for (i = 0; i < arity; i++) {
 	Eterm arg = args[i];
 	if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) {
@@ -1339,7 +1338,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
             hp += 2;
         }
     }
-    mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple);
+    mfa_tuple = TUPLE3(hp, info->mfa.module, info->mfa.function, mfa_tuple);
     hp += 4;
 
     /*
@@ -1459,7 +1458,8 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg)
 }
 
 void 
-monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint time)
+monitor_long_schedule_proc(Process *p, ErtsCodeInfo *in_fp,
+                           ErtsCodeInfo *out_fp, Uint time)
 {
     ErlHeapFragment *bp;
     ErlOffHeap *off_heap;
@@ -1490,11 +1490,13 @@ monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint
     hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, monitor_p);
     tmo = erts_bld_uint(&hp, NULL, time);
     if (in_fp != NULL) {
-	in_mfa = TUPLE3(hp,(Eterm) in_fp[0], (Eterm) in_fp[1], make_small(in_fp[2]));
+	in_mfa = TUPLE3(hp, in_fp->mfa.module, in_fp->mfa.function,
+                        make_small(in_fp->mfa.arity));
 	hp +=4;
     } 
     if (out_fp != NULL) {
-	out_mfa = TUPLE3(hp,(Eterm) out_fp[0], (Eterm) out_fp[1], make_small(out_fp[2]));
+	out_mfa = TUPLE3(hp, out_fp->mfa.module, out_fp->mfa.function,
+                         make_small(out_fp->mfa.arity));
 	hp +=4;
     } 
     tmo_tpl = TUPLE2(hp,am_timeout, tmo);
@@ -2129,7 +2131,7 @@ profile_runnable_proc(Process *p, Eterm status){
     Eterm *hp, msg;
     Eterm where = am_undefined;
     ErlHeapFragment *bp = NULL;
-    BeamInstr *current = NULL;
+    ErtsCodeInfo *ci = NULL;
 
 #ifndef ERTS_SMP
 #define LOCAL_HEAP_SIZE (4 + 6 + ERTS_TRACE_PATCH_TS_MAX_SIZE)
@@ -2151,14 +2153,14 @@ profile_runnable_proc(Process *p, Eterm status){
 
     if (!ERTS_PROC_IS_EXITING(p)) {
         if (p->current) {
-            current = p->current;
+            ci = erts_code_to_codeinfo(erts_codemfa_to_code(p->current));
         } else {
-            current = find_function_from_pc(p->i);
+            ci = find_function_from_pc(p->i);
         }
     }
 
 #ifdef ERTS_SMP
-    if (!current) {
+    if (!ci) {
 	hsz -= 4;
     }
 
@@ -2166,8 +2168,10 @@ profile_runnable_proc(Process *p, Eterm status){
     hp = bp->mem;
 #endif
 
-    if (current) {
-	where = TUPLE3(hp, current[0], current[1], make_small(current[2])); hp += 4;
+    if (ci) {
+	where = TUPLE3(hp, ci->mfa.module, ci->mfa.function,
+                       make_small(ci->mfa.arity));
+        hp += 4;
     } else {
 	where = make_small(0);
     }
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 0095d4386b..74df3e50e1 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -101,11 +101,11 @@ void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *);
 
 void trace_send(Process*, Eterm, Eterm);
 void trace_receive(Process*, Eterm, Eterm, ErtsTracingEvent*);
-Uint32 erts_call_trace(Process *p, BeamInstr mfa[], struct binary *match_spec,
+Uint32 erts_call_trace(Process *p, ErtsCodeInfo *info, struct binary *match_spec,
                        Eterm* args, int local, ErtsTracer *tracer);
-void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval,
+void erts_trace_return(Process* p, ErtsCodeMFA *mfa, Eterm retval,
                        ErtsTracer *tracer);
-void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value,
+void erts_trace_exception(Process* p, ErtsCodeMFA *mfa, Eterm class, Eterm value,
                           ErtsTracer *tracer);
 void erts_trace_return_to(Process *p, BeamInstr *pc);
 void trace_sched(Process*, ErtsProcLocks, Eterm);
@@ -134,7 +134,8 @@ void erts_system_profile_setup_active_schedulers(void);
 
 /* system_monitor */
 void monitor_long_gc(Process *p, Uint time);
-void monitor_long_schedule_proc(Process *p, BeamInstr *in_i, BeamInstr *out_i, Uint time);
+void monitor_long_schedule_proc(Process *p, ErtsCodeInfo *in_i,
+                                ErtsCodeInfo *out_i, Uint time);
 void monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time);
 void monitor_large_heap(Process *p);
 void monitor_generic(Process *p, Eterm type, Eterm spec);
@@ -176,7 +177,7 @@ struct trace_pattern_flags {
 };
 extern const struct trace_pattern_flags erts_trace_pattern_flags_off;
 extern int erts_call_time_breakpoint_tracing;
-int erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
+int erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
 			   struct binary* match_prog_set,
 			   struct binary *meta_match_prog_set,
 			   int on, struct trace_pattern_flags,
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 60c2349f36..93cfe08105 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -186,4 +186,11 @@ extern int erts_pd_initial_size;/* Initial Process dictionary table size */
 
 #include "erl_term.h"
 
+#ifdef NO_JUMP_TABLE 
+#define BeamOp(Op) (Op)
+#else
+extern void** beam_ops;
+#define BeamOp(Op) beam_ops[(Op)]
+#endif
+
 #endif	/* __ERL_VM_H__ */
diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h
index 6c33b12dd0..e431c3051b 100644
--- a/erts/emulator/beam/error.h
+++ b/erts/emulator/beam/error.h
@@ -21,6 +21,8 @@
 #ifndef __ERROR_H__
 #define __ERROR_H__
 
+#include "code_ix.h"
+
 /*
  * There are three primary exception classes:
  *
@@ -197,7 +199,7 @@ struct StackTrace {
     Eterm header;	/* bignum header - must be first in struct */
     Eterm freason; /* original exception reason is saved in the struct */
     BeamInstr* pc;
-    BeamInstr* current;
+    ErtsCodeMFA* current;
     int depth;	/* number of saved pointers in trace[] */
     BeamInstr *trace[1];  /* varying size - must be last in struct */
 };
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 2a19211987..0a36175398 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -103,7 +103,8 @@ static HashValue
 export_hash(struct export_entry* ee)
 {
     Export* x = ee->ep;
-    return EXPORT_HASH(x->code[0], x->code[1], x->code[2]);
+    return EXPORT_HASH(x->info.mfa.module, x->info.mfa.function,
+                       x->info.mfa.arity);
 }
 
 static int
@@ -111,9 +112,9 @@ export_cmp(struct export_entry* tmpl_e, struct export_entry* obj_e)
 {
     Export* tmpl = tmpl_e->ep;
     Export* obj = obj_e->ep;
-    return !(tmpl->code[0] == obj->code[0] &&
-	     tmpl->code[1] == obj->code[1] &&
-	     tmpl->code[2] == obj->code[2]);
+    return !(tmpl->info.mfa.module == obj->info.mfa.module &&
+	     tmpl->info.mfa.function == obj->info.mfa.function &&
+	     tmpl->info.mfa.arity == obj->info.mfa.arity);
 }
 
 
@@ -130,16 +131,16 @@ export_alloc(struct export_entry* tmpl_e)
 	blob = (struct export_blob*) erts_alloc(ERTS_ALC_T_EXPORT, sizeof(*blob));
 	erts_smp_atomic_add_nob(&total_entries_bytes, sizeof(*blob));
 	obj = &blob->exp;
-	obj->fake_op_func_info_for_hipe[0] = 0;
-	obj->fake_op_func_info_for_hipe[1] = 0;
-	obj->code[0] = tmpl->code[0];
-	obj->code[1] = tmpl->code[1];
-	obj->code[2] = tmpl->code[2];
-	obj->code[3] = (BeamInstr) em_call_error_handler;
-	obj->code[4] = 0;
+	obj->info.op =  0;
+	obj->info.native = 0;
+	obj->info.mfa.module = tmpl->info.mfa.module;
+	obj->info.mfa.function = tmpl->info.mfa.function;
+	obj->info.mfa.arity = tmpl->info.mfa.arity;
+	obj->code[0] = (BeamInstr) em_call_error_handler;
+	obj->code[1] = 0;
 
 	for (ix=0; ix<ERTS_NUM_CODE_IX; ix++) {
-	    obj->addressv[ix] = obj->code+3;
+	    obj->addressv[ix] = obj->code;
 
 	    blob->entryv[ix].slot.index = -1;
 	    blob->entryv[ix].ep = &blob->exp;
@@ -224,7 +225,9 @@ erts_find_export_entry(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix)
 	
     while (b != (HashBucket*) 0) {
 	Export* ep = ((struct export_entry*) b)->ep;
-	if (ep->code[0] == m && ep->code[1] == f && ep->code[2] == a) {
+	if (ep->info.mfa.module == m &&
+            ep->info.mfa.function == f &&
+            ep->info.mfa.arity == a) {
 	    return ep;
 	}
 	b = b->next;
@@ -237,9 +240,9 @@ static struct export_entry* init_template(struct export_templ* templ,
 {
     templ->entry.ep = &templ->exp;
     templ->entry.slot.index = -1;
-    templ->exp.code[0] = m;
-    templ->exp.code[1] = f;
-    templ->exp.code[2] = a;
+    templ->exp.info.mfa.module = m;
+    templ->exp.info.mfa.function = f;
+    templ->exp.info.mfa.arity = a;
     return &templ->entry;
 }
 
@@ -263,8 +266,8 @@ erts_find_function(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix)
 
     ee = hash_get(&export_tables[code_ix].htable, init_template(&templ, m, f, a));
     if (ee == NULL ||
-	(ee->ep->addressv[code_ix] == ee->ep->code+3 &&
-	 ee->ep->code[3] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) {
+	(ee->ep->addressv[code_ix] == ee->ep->code &&
+	 ee->ep->code[0] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) {
 	return NULL;
     }
     return ee->ep;
diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h
index 1e7bb8514b..a807efef94 100644
--- a/erts/emulator/beam/export.h
+++ b/erts/emulator/beam/export.h
@@ -33,22 +33,20 @@ typedef struct export
 {
     void* addressv[ERTS_NUM_CODE_IX];  /* Pointer to code for function. */
 
-    BeamInstr fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */
+    ErtsCodeInfo info; /* MUST be just before code[] */
+
     /*
-     * code[0]: Tagged atom for module.
-     * code[1]: Tagged atom for function.
-     * code[2]: Arity (untagged integer).
-     * code[3]: This entry is 0 unless the 'address' field points to it.
+     * code[0]: This entry is 0 unless the 'address' field points to it.
      *          Threaded code instruction to load function
      *		(em_call_error_handler), execute BIF (em_apply_bif),
      *		or a breakpoint instruction (op_i_generic_breakpoint).
-     * code[4]: Function pointer to BIF function (for BIFs only),
+     * code[1]: Function pointer to BIF function (for BIFs only),
      *		or pointer to threaded code if the module has an
      *		on_load function that has not been run yet, or pointer
-     *          to code for function code[3] is a breakpont instruction.
+     *          to code if function code[0] is a breakpoint instruction.
      *		Otherwise: 0.
      */
-    BeamInstr code[5];
+    BeamInstr code[2];
 } Export;
 
 
@@ -74,8 +72,8 @@ extern erts_smp_mtx_t export_staging_lock;
 
 #include "beam_load.h" /* For em_* extern declarations */ 
 #define ExportIsBuiltIn(EntryPtr) 			\
-(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->code + 3) && \
- ((EntryPtr)->code[3] == (BeamInstr) em_apply_bif))
+(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->code) && \
+ ((EntryPtr)->code[0] == (BeamInstr) em_apply_bif))
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index beed847578..c297bb26b3 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2826,9 +2826,10 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		Export* exp = *((Export **) (export_val(obj) + 1));
 		if ((dflags & DFLAG_EXPORT_PTR_TAG) != 0) {
 		    *ep++ = EXPORT_EXT;
-		    ep = enc_atom(acmp, exp->code[0], ep, dflags);
-		    ep = enc_atom(acmp, exp->code[1], ep, dflags);
-		    ep = enc_term(acmp, make_small(exp->code[2]), ep, dflags, off_heap);
+		    ep = enc_atom(acmp, exp->info.mfa.module, ep, dflags);
+		    ep = enc_atom(acmp, exp->info.mfa.function, ep, dflags);
+		    ep = enc_term(acmp, make_small(exp->info.mfa.arity),
+                                  ep, dflags, off_heap);
 		} else {
 		    /* Tag, arity */
 		    *ep++ = SMALL_TUPLE_EXT;
@@ -2836,10 +2837,10 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		    ep += 1;
 
 		    /* Module name */
-		    ep = enc_atom(acmp, exp->code[0], ep, dflags);
+		    ep = enc_atom(acmp, exp->info.mfa.module, ep, dflags);
 
 		    /* Function name */
-		    ep = enc_atom(acmp, exp->code[1], ep, dflags);
+		    ep = enc_atom(acmp, exp->info.mfa.function, ep, dflags);
 		}
 		break;
 	    }
@@ -4262,9 +4263,9 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 	    {
 		Export* ep = *((Export **) (export_val(obj) + 1));
 		result += 1;
-		result += encode_size_struct2(acmp, ep->code[0], dflags);
-		result += encode_size_struct2(acmp, ep->code[1], dflags);
-		result += encode_size_struct2(acmp, make_small(ep->code[2]), dflags);
+		result += encode_size_struct2(acmp, ep->info.mfa.module, dflags);
+		result += encode_size_struct2(acmp, ep->info.mfa.function, dflags);
+		result += encode_size_struct2(acmp, make_small(ep->info.mfa.arity), dflags);
 	    }
 	    break;
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 93996e8b41..c9e41bcac7 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1024,7 +1024,7 @@ extern Process *erts_code_purger;
 
 /* beam_load.c */
 typedef struct {
-    BeamInstr* current;		/* Pointer to: Mod, Name, Arity */
+    ErtsCodeInfo* ci;		/* Pointer to: Mod, Name, Arity */
     Uint needed;		/* Heap space needed for entire tuple */
     Uint32 loc;			/* Location in source code */
     Eterm* fname_ptr;		/* Pointer to fname table */
@@ -1041,10 +1041,10 @@ Eterm erts_finish_loading(Binary* loader_state, Process* c_p,
 Eterm erts_preload_module(Process *c_p, ErtsProcLocks c_p_locks,
 			  Eterm group_leader, Eterm* mod, byte* code, Uint size);
 void init_load(void);
-BeamInstr* find_function_from_pc(BeamInstr* pc);
+ErtsCodeInfo* find_function_from_pc(BeamInstr* pc);
 Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp,
 			   Eterm args, Eterm* mfa_p);
-void erts_set_current_function(FunctionInfo* fi, BeamInstr* current);
+void erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa);
 Eterm erts_module_info_0(Process* p, Eterm module);
 Eterm erts_module_info_1(Process* p, Eterm module, Eterm what);
 Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info);
@@ -1577,8 +1577,7 @@ int erts_beam_jump_table(void);
 ERTS_GLB_INLINE void dtrace_pid_str(Eterm pid, char *process_buf);
 ERTS_GLB_INLINE void dtrace_proc_str(Process *process, char *process_buf);
 ERTS_GLB_INLINE void dtrace_port_str(Port *port, char *port_buf);
-ERTS_GLB_INLINE void dtrace_fun_decode(Process *process,
-				       Eterm module, Eterm function, int arity,
+ERTS_GLB_INLINE void dtrace_fun_decode(Process *process, ErtsCodeMFA *mfa,
 				       char *process_buf, char *mfa_buf);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -1612,8 +1611,7 @@ dtrace_port_str(Port *port, char *port_buf)
 }
 
 ERTS_GLB_INLINE void
-dtrace_fun_decode(Process *process,
-                  Eterm module, Eterm function, int arity,
+dtrace_fun_decode(Process *process, ErtsCodeMFA *mfa,
                   char *process_buf, char *mfa_buf)
 {
     if (process_buf) {
@@ -1621,7 +1619,7 @@ dtrace_fun_decode(Process *process,
     }
 
     erts_snprintf(mfa_buf, DTRACE_TERM_BUF_SIZE, "%T:%T/%d",
-                  module, function, arity);
+                  mfa->module, mfa->function, mfa->arity);
 }
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 6786657faf..cd7131f5df 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -893,11 +893,11 @@ tail_recur:
 	{
 	    Export* ep = *((Export **) (export_val(term) + 1));
 
-	    hash = hash * FUNNY_NUMBER11 + ep->code[2];
+	    hash = hash * FUNNY_NUMBER11 + ep->info.mfa.arity;
 	    hash = hash*FUNNY_NUMBER1 +
-		(atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue);
+		(atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue);
 	    hash = hash*FUNNY_NUMBER1 +
-		(atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue);
+		(atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue);
 	    break;
 	}
 
@@ -1330,11 +1330,11 @@ make_hash2(Eterm term)
 	    {
 		Export* ep = *((Export **) (export_val(term) + 1));
 		UINT32_HASH_2
-		    (ep->code[2],
-		     atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue,
+		    (ep->info.mfa.arity,
+		     atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue,
 		     HCONST);
 		UINT32_HASH
-		    (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue,
+		    (atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue,
 		     HCONST_14);
 		goto hash2_common;
 	    }
@@ -2025,11 +2025,11 @@ tail_recur:
 	{
 	    Export* ep = *((Export **) (export_val(term) + 1));
 
-	    hash = hash * FUNNY_NUMBER11 + ep->code[2];
+	    hash = hash * FUNNY_NUMBER11 + ep->info.mfa.arity;
 	    hash = hash*FUNNY_NUMBER1 +
-		(atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue);
+		(atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue);
 	    hash = hash*FUNNY_NUMBER1 +
-		(atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue);
+		(atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue);
 	    break;
 	}
 
@@ -3306,13 +3306,15 @@ tailrecur_ne:
 		    Export* a_exp = *((Export **) (export_val(a) + 1));
 		    Export* b_exp = *((Export **) (export_val(b) + 1));
 
-		    if ((j = erts_cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) {
+		    if ((j = erts_cmp_atoms(a_exp->info.mfa.module,
+                                            b_exp->info.mfa.module)) != 0) {
 			RETURN_NEQ(j);
 		    }
-		    if ((j = erts_cmp_atoms(a_exp->code[1], b_exp->code[1])) != 0) {
+		    if ((j = erts_cmp_atoms(a_exp->info.mfa.function,
+                                            b_exp->info.mfa.function)) != 0) {
 			RETURN_NEQ(j);
 		    }
-		    ON_CMP_GOTO((Sint) a_exp->code[2] - (Sint) b_exp->code[2]);
+		    ON_CMP_GOTO((Sint) a_exp->info.mfa.arity - (Sint) b_exp->info.mfa.arity);
 		}
 		break;
 	    case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE):
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 3336fded7a..9faa6e1649 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1453,11 +1453,11 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2)
     hdr = *boxed_val(BIF_ARG_1);
     if (is_export_header(hdr)) {
 	Export *ep = (Export*)(export_val(BIF_ARG_1)[1]);
-	unsigned int actual_arity = ep->code[2];
+	unsigned int actual_arity = ep->info.mfa.arity;
 	if (actual_arity != BIF_ARG_2)
 	    goto badfun;
-	m = ep->code[0];
-	f = ep->code[1];
+	m = ep->info.mfa.module;
+	f = ep->info.mfa.function;
     } else
 	goto badfun;
     address = hipe_get_na_nofail(m, f, BIF_ARG_2, 1);
diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c
index 5e127755c6..0c66eb6abe 100644
--- a/erts/emulator/hipe/hipe_bif1.c
+++ b/erts/emulator/hipe/hipe_bif1.c
@@ -45,7 +45,7 @@ BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1)
     pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
     if (!pc)
 	BIF_ERROR(BIF_P, BADARG);
-    ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI));
+    ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI));
     if (pc[0] == BeamOpCode(op_hipe_trap_call))
 	BIF_ERROR(BIF_P, BADARG);
     if (pc[0] == BeamOpCode(op_hipe_call_count))
@@ -67,7 +67,7 @@ BIF_RETTYPE hipe_bifs_call_count_off_1(BIF_ALIST_1)
     pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
     if (!pc)
 	BIF_ERROR(BIF_P, BADARG);
-    ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI));
+    ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI));
     if (pc[0] != BeamOpCode(op_hipe_call_count))
 	BIF_RET(am_false);
     hcc = (struct hipe_call_count*)pc[-4];
@@ -86,7 +86,7 @@ BIF_RETTYPE hipe_bifs_call_count_get_1(BIF_ALIST_1)
     pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
     if (!pc)
 	BIF_ERROR(BIF_P, BADARG);
-    ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI));
+    ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI));
     if (pc[0] != BeamOpCode(op_hipe_call_count))
 	BIF_RET(am_false);
     hcc = (struct hipe_call_count*)pc[-4];
@@ -102,7 +102,7 @@ BIF_RETTYPE hipe_bifs_call_count_clear_1(BIF_ALIST_1)
     pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
     if (!pc)
 	BIF_ERROR(BIF_P, BADARG);
-    ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI));
+    ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI));
     if (pc[0] != BeamOpCode(op_hipe_call_count))
 	BIF_RET(am_false);
     hcc = (struct hipe_call_count*)pc[-4];
diff --git a/erts/emulator/hipe/hipe_debug.c b/erts/emulator/hipe/hipe_debug.c
index ace489452f..d4dcf2a653 100644
--- a/erts/emulator/hipe/hipe_debug.c
+++ b/erts/emulator/hipe/hipe_debug.c
@@ -62,10 +62,12 @@ static void print_beam_pc(BeamInstr *pc)
     } else if (pc == &beam_apply[1]) {
 	printf("normal-process-exit");
     } else {
-	BeamInstr *mfa = find_function_from_pc(pc);
-	if (mfa)
+	ErtsCodeInfo *ci = find_function_from_pc(pc);
+	if (ci)
 	    erts_printf("%T:%T/%bpu + 0x%bpx",
-			mfa[0], mfa[1], mfa[2], pc - &mfa[3]);
+			ci->mfa.module, ci->mfa.function,
+                        ci->mfa.arity,
+                        pc - erts_codeinfo_to_code(ci));
 	else
 	    printf("?");
     }
@@ -214,10 +216,10 @@ void hipe_print_pcb(Process *p)
     U("seq..clock ", seq_trace_clock);
     U("seq..astcnt", seq_trace_lastcnt);
     U("seq..token ", seq_trace_token);
-    U("intial[0]  ", u.initial[0]);
-    U("intial[1]  ", u.initial[1]);
-    U("intial[2]  ", u.initial[2]);
-    P("current    ", current);
+    U("intial.mod ", u.initial.module);
+    U("intial.fun ", u.initial.function);
+    U("intial.ari ", u.initial.arity);
+    U("current    ", current);
     P("cp         ", cp);
     P("i          ", i);
     U("catches    ", catches);
-- 
cgit v1.2.3


From 8e2490086b45b9ce4d51883e594c38e2e17b5b47 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 5 Oct 2016 12:16:35 +0200
Subject: erts: Refactor find_function_from_pc to return MFA

---
 erts/emulator/beam/beam_debug.c       | 52 +++++++++++++++++------------------
 erts/emulator/beam/beam_emu.c         |  8 +++---
 erts/emulator/beam/beam_load.c        | 14 +++++-----
 erts/emulator/beam/beam_ranges.c      |  6 ++--
 erts/emulator/beam/erl_bif_info.c     | 12 ++++----
 erts/emulator/beam/erl_db_util.c      |  8 +++---
 erts/emulator/beam/erl_nif.c          |  4 +--
 erts/emulator/beam/erl_process.c      |  8 +++---
 erts/emulator/beam/erl_process_dump.c |  8 +++---
 erts/emulator/beam/erl_trace.c        | 36 ++++++++++++------------
 erts/emulator/beam/erl_trace.h        |  4 +--
 erts/emulator/beam/global.h           |  4 +--
 erts/emulator/hipe/hipe_debug.c       | 10 +++----
 13 files changed, 86 insertions(+), 88 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 8d4be4667b..c855c734d6 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -243,7 +243,7 @@ erts_debug_disassemble_1(BIF_ALIST_1)
     Eterm* tp;
     Eterm bin;
     Eterm mfa;
-    ErtsCodeInfo *ci = NULL;
+    ErtsCodeMFA *cmfa = NULL;
     BeamCodeHeader* code_hdr;
     BeamInstr *code_ptr;
     BeamInstr instr;
@@ -253,7 +253,7 @@ erts_debug_disassemble_1(BIF_ALIST_1)
 
     if (term_to_UWord(addr, &uaddr)) {
 	BeamInstr *pc = (BeamInstr *) uaddr;
-	if ((ci = find_function_from_pc(pc)) == NULL) {
+	if ((cmfa = find_function_from_pc(pc)) == NULL) {
 	    BIF_RET(am_false);
 	}
     } else if (is_tuple(addr)) {
@@ -292,14 +292,14 @@ erts_debug_disassemble_1(BIF_ALIST_1)
 	     * But this code_ptr will point to the start of the Export,
 	     * not the function's func_info instruction. BOOM !?
 	     */
-	    ci = erts_code_to_codeinfo(ep->addressv[code_ix]);
+	    cmfa = erts_code_to_codemfa(ep->addressv[code_ix]);
 	} else if (modp == NULL || (code_hdr = modp->curr.code_hdr) == NULL) {
 	    BIF_RET(am_undef);
 	} else {
 	    n = code_hdr->num_functions;
 	    for (i = 0; i < n; i++) {
-		ci = code_hdr->functions[i];
-		if (ci->mfa.function == name && ci->mfa.arity == arity) {
+		cmfa = &code_hdr->functions[i]->mfa;
+		if (cmfa->function == name && cmfa->arity == arity) {
 		    break;
 		}
 	    }
@@ -312,7 +312,7 @@ erts_debug_disassemble_1(BIF_ALIST_1)
     }
 
 
-    code_ptr = erts_codeinfo_to_code(ci);
+    code_ptr = erts_codemfa_to_code(cmfa);
     dsbufp = erts_create_tmp_dsbuf(0);
     erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr);
     instr = (BeamInstr) code_ptr[0];
@@ -334,10 +334,10 @@ erts_debug_disassemble_1(BIF_ALIST_1)
     (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr);
     hp = HAlloc(p, hsz);
     addr = erts_bld_uword(&hp, NULL, (BeamInstr) code_ptr);
-    ASSERT(is_atom(ci->mfa.module) || is_nil(ci->mfa.module));
-    ASSERT(is_atom(ci->mfa.function) || is_nil(ci->mfa.function == NIL));
-    mfa = TUPLE3(hp, ci->mfa.module, ci->mfa.function,
-                 make_small(ci->mfa.arity));
+    ASSERT(is_atom(cmfa->module) || is_nil(cmfa->module));
+    ASSERT(is_atom(cmfa->function) || is_nil(cmfa->function == NIL));
+    mfa = TUPLE3(hp, cmfa->module, cmfa->function,
+                 make_small(cmfa->arity));
     hp += 4;
     return TUPLE3(hp, addr, bin, mfa);
 }
@@ -349,12 +349,12 @@ dbg_bt(Process* p, Eterm* sp)
 
     while (sp < stack) {
 	if (is_CP(*sp)) {
-	    ErtsCodeInfo* ci = find_function_from_pc(cp_val(*sp));
-	    if (ci)
+	    ErtsCodeMFA* cmfa = find_function_from_pc(cp_val(*sp));
+	    if (cmfa)
 		erts_fprintf(stderr,
 			     HEXF ": %T:%T/%bpu\n",
-			     &ci->mfa.module, ci->mfa.module,
-                             ci->mfa.function, ci->mfa.arity);
+			     &cmfa->module, cmfa->module,
+                             cmfa->function, cmfa->arity);
 	}
 	sp++;
     }
@@ -363,17 +363,17 @@ dbg_bt(Process* p, Eterm* sp)
 void
 dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg)
 {
-    ErtsCodeInfo* ci = find_function_from_pc(addr);
+    ErtsCodeMFA* cmfa = find_function_from_pc(addr);
 
-    if (ci == NULL) {
+    if (cmfa == NULL) {
 	erts_fprintf(stderr, "???\n");
     } else {
 	int arity;
 	int i;
 
-	arity = ci->mfa.arity;
+	arity = cmfa->arity;
 	erts_fprintf(stderr, HEXF ": %T:%T(", addr,
-                     ci->mfa.module, ci->mfa.function);
+                     cmfa->module, cmfa->function);
 	for (i = 0; i < arity; i++)
 	    erts_fprintf(stderr, i ? ", %T" : "%T", i ? reg[i] : x0);
 	erts_fprintf(stderr, ")\n");
@@ -549,24 +549,24 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	    break;
 	case 'f':		/* Destination label */
 	    {
-		ErtsCodeInfo* ci = find_function_from_pc((BeamInstr *)*ap);
-		if (erts_codeinfo_to_code(ci) != (BeamInstr *) *ap) {
+		ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap);
+		if (erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) {
 		    erts_print(to, to_arg, "f(" HEXF ")", *ap);
 		} else {
-		    erts_print(to, to_arg, "%T:%T/%bpu", ci->mfa.module,
-                               ci->mfa.function, ci->mfa.arity);
+		    erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module,
+                               cmfa->function, cmfa->arity);
 		}
 		ap++;
 	    }
 	    break;
 	case 'p':		/* Pointer (to label) */
 	    {
-		ErtsCodeInfo* ci = find_function_from_pc((BeamInstr *)*ap);
-		if (erts_codeinfo_to_code(ci) != (BeamInstr *) *ap) {
+		ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap);
+		if (erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) {
 		    erts_print(to, to_arg, "p(" HEXF ")", *ap);
 		} else {
-		    erts_print(to, to_arg, "%T:%T/%bpu", ci->mfa.module,
-                               ci->mfa.function, ci->mfa.arity);
+		    erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module,
+                               cmfa->function, cmfa->arity);
 		}
 		ap++;
 	    }
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 4ff177f215..2c47386925 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1317,8 +1317,8 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
     if (start_time != 0) {
         Sint64 diff = erts_timestamp_millis() - start_time;
 	if (diff > 0 && (Uint) diff >  erts_system_monitor_long_schedule) {
-	    ErtsCodeInfo *inptr = find_function_from_pc(start_time_i);
-	    ErtsCodeInfo *outptr = find_function_from_pc(c_p->i);
+	    ErtsCodeMFA *inptr = find_function_from_pc(start_time_i);
+	    ErtsCodeMFA *outptr = find_function_from_pc(c_p->i);
 	    monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff);
 	}
     }
@@ -6074,7 +6074,7 @@ build_stacktrace(Process* c_p, Eterm exc) {
      * If fi.current is still NULL, default to the initial function
      * (e.g. spawn_link(erlang, abs, [1])).
      */
-    if (fi.ci == NULL) {
+    if (fi.mfa == NULL) {
 	erts_set_current_function(&fi, &c_p->u.initial);
 	args = am_true; /* Just in case */
     } else {
@@ -6091,7 +6091,7 @@ build_stacktrace(Process* c_p, Eterm exc) {
     heap_size = fi.needed + 2;
     for (i = 0; i < depth; i++) {
 	erts_lookup_function_info(stkp, s->trace[i], 1);
-	if (stkp->ci) {
+	if (stkp->mfa) {
 	    heap_size += stkp->needed + 2;
 	    stkp++;
 	}
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index c822cd8606..76a92d41f7 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -5818,7 +5818,7 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p)
 	Eterm file_term = NIL;
 
 	if (file == 0) {
-	    Atom* ap = atom_tab(atom_val(fi->ci->mfa.module));
+	    Atom* ap = atom_tab(atom_val(fi->mfa->module));
 	    file_term = buf_to_intlist(&hp, ".erl", 4, NIL);
 	    file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, file_term);
 	} else {
@@ -5837,11 +5837,11 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p)
     }
 
     if (is_list(args) || is_nil(args)) {
-	*mfa_p = TUPLE4(hp, fi->ci->mfa.module, fi->ci->mfa.function,
+	*mfa_p = TUPLE4(hp, fi->mfa->module, fi->mfa->function,
                         args, loc);
     } else {
-	Eterm arity = make_small(fi->ci->mfa.arity);
-	*mfa_p = TUPLE4(hp, fi->ci->mfa.module, fi->ci->mfa.function,
+	Eterm arity = make_small(fi->mfa->arity);
+	*mfa_p = TUPLE4(hp, fi->mfa->module, fi->mfa->function,
                         arity, loc);
     }
     return hp + 5;
@@ -5855,7 +5855,7 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p)
 void
 erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa)
 {
-    fi->ci = erts_code_to_codeinfo(erts_codemfa_to_code(mfa));
+    fi->mfa = mfa;
     fi->needed = 5;
     fi->loc = LINE_INVALID_LOCATION;
 }
@@ -5864,13 +5864,13 @@ erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa)
 /*
  * Returns a pointer to {module, function, arity}, or NULL if not found.
  */
-ErtsCodeInfo*
+ErtsCodeMFA*
 find_function_from_pc(BeamInstr* pc)
 {
     FunctionInfo fi;
 
     erts_lookup_function_info(&fi, pc, 0);
-    return fi.ci;
+    return fi.mfa;
 }
 
 /*
diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c
index 887f80e7e3..9b0335e83d 100644
--- a/erts/emulator/beam/beam_ranges.c
+++ b/erts/emulator/beam/beam_ranges.c
@@ -227,7 +227,7 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info)
     Range* rp;
     BeamCodeHeader* hdr;
 
-    fi->ci = NULL;
+    fi->mfa = NULL;
     fi->needed = 5;
     fi->loc = LINE_INVALID_LOCATION;
     rp = find_range(pc);
@@ -243,7 +243,7 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info)
 	if (pc < (BeamInstr*)(mid[0])) {
 	    high = mid;
 	} else if (pc < (BeamInstr*)(mid[1])) {
-	    fi->ci = mid[0];
+	    fi->mfa = &mid[0]->mfa;
 	    if (full_info) {
 		ErtsCodeInfo** fp = hdr->functions;
 		int idx = mid - fp;
@@ -316,7 +316,7 @@ lookup_loc(FunctionInfo* fi, const BeamInstr* pc,
 	    file = LOC_FILE(fi->loc);
 	    if (file == 0) {
 		/* Special case: Module name with ".erl" appended */
-		Atom* mod_atom = atom_tab(atom_val(fi->ci->mfa.module));
+		Atom* mod_atom = atom_tab(atom_val(fi->mfa->module));
 		fi->needed += 2*(mod_atom->len+4);
 	    } else {
 		Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1]));
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index a985ce4918..dfc4beb719 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1614,12 +1614,10 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info)
 
     if (rp->current == NULL) {
 	erts_lookup_function_info(&fi, rp->i, full_info);
-	rp->current = &fi.ci->mfa;
-        ASSERT_MFA(rp->current);
+	rp->current = fi.mfa;
     } else if (full_info) {
-        ASSERT_MFA(rp->current);
 	erts_lookup_function_info(&fi, rp->i, full_info);
-	if (fi.ci == NULL) {
+	if (fi.mfa == NULL) {
 	    /* Use the current function without location info */
 	    erts_set_current_function(&fi, rp->current);
 	}
@@ -1635,9 +1633,9 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info)
 	 * instead if it can be looked up.
 	 */
 	erts_lookup_function_info(&fi2, rp->cp, full_info);
-	if (fi2.ci) {
+	if (fi2.mfa) {
 	    fi = fi2;
-	    rp->current = &fi2.ci->mfa;
+	    rp->current = fi2.mfa;
 	}
     }
 
@@ -1695,7 +1693,7 @@ current_stacktrace(Process* p, Process* rp, Eterm** hpp)
     heap_size = 3;
     for (i = 0; i < depth; i++) {
 	erts_lookup_function_info(stkp, s->trace[i], 1);
-	if (stkp->ci) {
+	if (stkp->mfa) {
 	    heap_size += stkp->needed + 2;
 	    stkp++;
 	}
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 988467ce44..4e987d5bee 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1769,7 +1769,7 @@ Eterm db_prog_match(Process *c_p,
     Eterm t;
     Eterm *esp;
     MatchVariable* variables;
-    ErtsCodeInfo *cp;
+    ErtsCodeMFA *cp;
     const UWord *pc = prog->text;
     Eterm *ehp;
     Eterm ret;
@@ -2408,9 +2408,9 @@ restart:
 		ehp = HAllocX(build_proc, 4, HEAP_XTRA);
  		*esp++ = make_tuple(ehp);
 		ehp[0] = make_arityval(3);
-		ehp[1] = cp->mfa.module;
-		ehp[2] = cp->mfa.function;
-		ehp[3] = make_small((Uint) cp->mfa.arity);
+		ehp[1] = cp->module;
+		ehp[2] = cp->function;
+		ehp[3] = make_small((Uint) cp->arity);
 	    }
 	    break;
         case matchSilent:
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index f0535451fc..167cded28f 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3158,7 +3158,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     Eterm mod_atom;
     const Atom* mod_atomp;
     Eterm f_atom;
-    ErtsCodeInfo* caller;
+    ErtsCodeMFA* caller;
     ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT;
     Eterm ret = am_ok;
     int veto;
@@ -3196,7 +3196,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	   && BIF_P->current->arity == 2);
     caller = find_function_from_pc(BIF_P->cp);
     ASSERT(caller != NULL);
-    mod_atom = caller->mfa.module;
+    mod_atom = caller->module;
     ASSERT(is_atom(mod_atom));
     module_p = erts_get_module(mod_atom, erts_active_code_ix());
     ASSERT(module_p != NULL);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index f7f391e322..d0a0310544 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -13220,8 +13220,8 @@ erts_program_counter_info(int to, void *to_arg, Process *p)
 static void
 print_function_from_pc(int to, void *to_arg, BeamInstr* x)
 {
-    ErtsCodeInfo *ci = find_function_from_pc(x);
-    if (ci == NULL) {
+    ErtsCodeMFA *cmfa = find_function_from_pc(x);
+    if (cmfa == NULL) {
         if (x == beam_exit) {
             erts_print(to, to_arg, "<terminate process>");
         } else if (x == beam_continue_exit) {
@@ -13235,8 +13235,8 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x)
         }
     } else {
 	erts_print(to, to_arg, "%T:%T/%d + %d",
-		   ci->mfa.module, ci->mfa.function, ci->mfa.arity,
-                   (x-(BeamInstr*)ci) * sizeof(Eterm));
+		   cmfa->module, cmfa->function, cmfa->arity,
+                   (x-(BeamInstr*)cmfa) * sizeof(Eterm));
     }
 }
 
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 645bedd8f1..e7a311b430 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -334,8 +334,8 @@ stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg)
 static void
 print_function_from_pc(int to, void *to_arg, BeamInstr* x)
 {
-    ErtsCodeInfo* ci = find_function_from_pc(x);
-    if (ci == NULL) {
+    ErtsCodeMFA* cmfa = find_function_from_pc(x);
+    if (cmfa == NULL) {
         if (x == beam_exit) {
             erts_print(to, to_arg, "<terminate process>");
         } else if (x == beam_continue_exit) {
@@ -347,8 +347,8 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x)
         }
     } else {
 	erts_print(to, to_arg, "%T:%T/%bpu + %bpu",
-		   ci->mfa.module, ci->mfa.function, ci->mfa.arity,
-                   (x-(BeamInstr*)ci) * sizeof(Eterm));
+		   cmfa->module, cmfa->function, cmfa->arity,
+                   (x-(BeamInstr*)cmfa) * sizeof(Eterm));
     }
 }
 
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 566529d2d4..9be4741ec8 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -773,7 +773,7 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what)
 	curr_func = 0;
     else {
 	if (!p->current)
-	    p->current = &find_function_from_pc(p->i)->mfa;
+	    p->current = find_function_from_pc(p->i);
 	curr_func = p->current != NULL;
     }
 
@@ -1028,14 +1028,14 @@ erts_trace_return_to(Process *p, BeamInstr *pc)
 {
     Eterm mfa;
 
-    ErtsCodeInfo *ci = find_function_from_pc(pc);
+    ErtsCodeMFA *cmfa = find_function_from_pc(pc);
 
-    if (!ci) {
+    if (!cmfa) {
 	mfa = am_undefined;
     } else {
         Eterm *hp = HAlloc(p, 4);
-	mfa = TUPLE3(hp, ci->mfa.module, ci->mfa.function,
-                     make_small(ci->mfa.arity));
+	mfa = TUPLE3(hp, cmfa->module, cmfa->function,
+                     make_small(cmfa->arity));
     }
 
     send_to_tracer_nif(p, &p->common, p->common.id, NULL, TRACE_FUN_T_CALL,
@@ -1458,8 +1458,8 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg)
 }
 
 void 
-monitor_long_schedule_proc(Process *p, ErtsCodeInfo *in_fp,
-                           ErtsCodeInfo *out_fp, Uint time)
+monitor_long_schedule_proc(Process *p, ErtsCodeMFA *in_fp,
+                           ErtsCodeMFA *out_fp, Uint time)
 {
     ErlHeapFragment *bp;
     ErlOffHeap *off_heap;
@@ -1490,13 +1490,13 @@ monitor_long_schedule_proc(Process *p, ErtsCodeInfo *in_fp,
     hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, monitor_p);
     tmo = erts_bld_uint(&hp, NULL, time);
     if (in_fp != NULL) {
-	in_mfa = TUPLE3(hp, in_fp->mfa.module, in_fp->mfa.function,
-                        make_small(in_fp->mfa.arity));
+	in_mfa = TUPLE3(hp, in_fp->module, in_fp->function,
+                        make_small(in_fp->arity));
 	hp +=4;
     } 
     if (out_fp != NULL) {
-	out_mfa = TUPLE3(hp, out_fp->mfa.module, out_fp->mfa.function,
-                         make_small(out_fp->mfa.arity));
+	out_mfa = TUPLE3(hp, out_fp->module, out_fp->function,
+                         make_small(out_fp->arity));
 	hp +=4;
     } 
     tmo_tpl = TUPLE2(hp,am_timeout, tmo);
@@ -2131,7 +2131,7 @@ profile_runnable_proc(Process *p, Eterm status){
     Eterm *hp, msg;
     Eterm where = am_undefined;
     ErlHeapFragment *bp = NULL;
-    ErtsCodeInfo *ci = NULL;
+    ErtsCodeMFA *cmfa = NULL;
 
 #ifndef ERTS_SMP
 #define LOCAL_HEAP_SIZE (4 + 6 + ERTS_TRACE_PATCH_TS_MAX_SIZE)
@@ -2153,14 +2153,14 @@ profile_runnable_proc(Process *p, Eterm status){
 
     if (!ERTS_PROC_IS_EXITING(p)) {
         if (p->current) {
-            ci = erts_code_to_codeinfo(erts_codemfa_to_code(p->current));
+            cmfa = p->current;
         } else {
-            ci = find_function_from_pc(p->i);
+            cmfa = find_function_from_pc(p->i);
         }
     }
 
 #ifdef ERTS_SMP
-    if (!ci) {
+    if (!cmfa) {
 	hsz -= 4;
     }
 
@@ -2168,9 +2168,9 @@ profile_runnable_proc(Process *p, Eterm status){
     hp = bp->mem;
 #endif
 
-    if (ci) {
-	where = TUPLE3(hp, ci->mfa.module, ci->mfa.function,
-                       make_small(ci->mfa.arity));
+    if (cmfa) {
+	where = TUPLE3(hp, cmfa->module, cmfa->function,
+                       make_small(cmfa->arity));
         hp += 4;
     } else {
 	where = make_small(0);
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 74df3e50e1..378b6de49c 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -134,8 +134,8 @@ void erts_system_profile_setup_active_schedulers(void);
 
 /* system_monitor */
 void monitor_long_gc(Process *p, Uint time);
-void monitor_long_schedule_proc(Process *p, ErtsCodeInfo *in_i,
-                                ErtsCodeInfo *out_i, Uint time);
+void monitor_long_schedule_proc(Process *p, ErtsCodeMFA *in_i,
+                                ErtsCodeMFA *out_i, Uint time);
 void monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time);
 void monitor_large_heap(Process *p);
 void monitor_generic(Process *p, Eterm type, Eterm spec);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index c9e41bcac7..7ee046ac12 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1024,7 +1024,7 @@ extern Process *erts_code_purger;
 
 /* beam_load.c */
 typedef struct {
-    ErtsCodeInfo* ci;		/* Pointer to: Mod, Name, Arity */
+    ErtsCodeMFA* mfa;		/* Pointer to: Mod, Name, Arity */
     Uint needed;		/* Heap space needed for entire tuple */
     Uint32 loc;			/* Location in source code */
     Eterm* fname_ptr;		/* Pointer to fname table */
@@ -1041,7 +1041,7 @@ Eterm erts_finish_loading(Binary* loader_state, Process* c_p,
 Eterm erts_preload_module(Process *c_p, ErtsProcLocks c_p_locks,
 			  Eterm group_leader, Eterm* mod, byte* code, Uint size);
 void init_load(void);
-ErtsCodeInfo* find_function_from_pc(BeamInstr* pc);
+ErtsCodeMFA* find_function_from_pc(BeamInstr* pc);
 Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp,
 			   Eterm args, Eterm* mfa_p);
 void erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa);
diff --git a/erts/emulator/hipe/hipe_debug.c b/erts/emulator/hipe/hipe_debug.c
index d4dcf2a653..222a11db3d 100644
--- a/erts/emulator/hipe/hipe_debug.c
+++ b/erts/emulator/hipe/hipe_debug.c
@@ -62,12 +62,12 @@ static void print_beam_pc(BeamInstr *pc)
     } else if (pc == &beam_apply[1]) {
 	printf("normal-process-exit");
     } else {
-	ErtsCodeInfo *ci = find_function_from_pc(pc);
-	if (ci)
+	ErtsCodeMFA *cmfa = find_function_from_pc(pc);
+	if (cmfa)
 	    erts_printf("%T:%T/%bpu + 0x%bpx",
-			ci->mfa.module, ci->mfa.function,
-                        ci->mfa.arity,
-                        pc - erts_codeinfo_to_code(ci));
+			cmfa->module, cmfa->function,
+                        cmfa->arity,
+                        pc - erts_codemfa_to_code(cmfa));
 	else
 	    printf("?");
     }
-- 
cgit v1.2.3


From 81e6a32de18bd3d8ba4b8801631e5c46892a5a18 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 6 Oct 2016 10:50:50 +0200
Subject: erts: Fix erts_debug:df with new func_info

---
 erts/emulator/beam/beam_debug.c | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index c855c734d6..68559e3992 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -252,8 +252,8 @@ erts_debug_disassemble_1(BIF_ALIST_1)
     int i;
 
     if (term_to_UWord(addr, &uaddr)) {
-	BeamInstr *pc = (BeamInstr *) uaddr;
-	if ((cmfa = find_function_from_pc(pc)) == NULL) {
+	code_ptr = (BeamInstr *) uaddr;
+	if ((cmfa = find_function_from_pc(code_ptr)) == NULL) {
 	    BIF_RET(am_false);
 	}
     } else if (is_tuple(addr)) {
@@ -307,12 +307,11 @@ erts_debug_disassemble_1(BIF_ALIST_1)
 		BIF_RET(am_undef);
 	    }
 	}
+        code_ptr = erts_codemfa_to_code(cmfa);
     } else {
 	goto error;
     }
 
-
-    code_ptr = erts_codemfa_to_code(cmfa);
     dsbufp = erts_create_tmp_dsbuf(0);
     erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr);
     instr = (BeamInstr) code_ptr[0];
@@ -335,7 +334,7 @@ erts_debug_disassemble_1(BIF_ALIST_1)
     hp = HAlloc(p, hsz);
     addr = erts_bld_uword(&hp, NULL, (BeamInstr) code_ptr);
     ASSERT(is_atom(cmfa->module) || is_nil(cmfa->module));
-    ASSERT(is_atom(cmfa->function) || is_nil(cmfa->function == NIL));
+    ASSERT(is_atom(cmfa->function) || is_nil(cmfa->function));
     mfa = TUPLE3(hp, cmfa->module, cmfa->function,
                  make_small(cmfa->arity));
     hp += 4;
@@ -550,7 +549,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	case 'f':		/* Destination label */
 	    {
 		ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap);
-		if (erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) {
+		if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) {
 		    erts_print(to, to_arg, "f(" HEXF ")", *ap);
 		} else {
 		    erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module,
@@ -562,7 +561,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 	case 'p':		/* Pointer (to label) */
 	    {
 		ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap);
-		if (erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) {
+		if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) {
 		    erts_print(to, to_arg, "p(" HEXF ")", *ap);
 		} else {
 		    erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module,
-- 
cgit v1.2.3


From ab3ea86a2ad0015eb8a5a46e01a1ad4ad51da4e1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 10 Oct 2016 17:36:42 +0200
Subject: erts: Refactor rename Export.code[] to Export.beam[]

to avoid scary merge errors.
---
 erts/emulator/beam/beam_bif_load.c | 32 ++++++++++++++++----------------
 erts/emulator/beam/beam_bp.c       |  4 ++--
 erts/emulator/beam/beam_emu.c      | 10 +++++-----
 erts/emulator/beam/beam_load.c     | 26 +++++++++++++-------------
 erts/emulator/beam/bif.c           |  6 +++---
 erts/emulator/beam/erl_bif_trace.c | 36 ++++++++++++++++++------------------
 erts/emulator/beam/erl_nif.c       |  6 +++---
 erts/emulator/beam/export.c        | 10 +++++-----
 erts/emulator/beam/export.h        | 14 +++++++-------
 9 files changed, 72 insertions(+), 72 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index f584b486ed..bce57fb7c1 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -795,16 +795,16 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	    if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) {
 		continue;
 	    }
-	    if (ep->code[1] != 0) {
-		ep->addressv[code_ix] = (void *) ep->code[1];
-		ep->code[1] = 0;
+	    if (ep->beam[1] != 0) {
+		ep->addressv[code_ix] = (void *) ep->beam[1];
+		ep->beam[1] = 0;
 	    } else {
-		if (ep->addressv[code_ix] == ep->code &&
-		    ep->code[0] == (BeamInstr) em_apply_bif) {
+		if (ep->addressv[code_ix] == ep->beam &&
+		    ep->beam[0] == (BeamInstr) em_apply_bif) {
 		    continue;
 		}
-		ep->addressv[code_ix] = ep->code;
-		ep->code[0] = (BeamInstr) em_call_error_handler;
+		ep->addressv[code_ix] = ep->beam;
+		ep->beam[0] = (BeamInstr) em_call_error_handler;
 	    }
 	}
 	modp->curr.code_hdr->on_load_function_ptr = NULL;
@@ -822,10 +822,10 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	    if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) {
 		continue;
 	    }
-	    if (ep->code[0] == (BeamInstr) em_apply_bif) {
+	    if (ep->beam[0] == (BeamInstr) em_apply_bif) {
 		continue;
 	    }
-	    ep->code[1] = 0;
+	    ep->beam[1] = 0;
 	}
     }
     erts_smp_thr_progress_unblock();
@@ -1711,22 +1711,22 @@ delete_code(Module* modp)
     for (i = 0; i < export_list_size(code_ix); i++) {
 	Export *ep = export_list(i, code_ix);
         if (ep != NULL && (ep->info.mfa.module == module)) {
-	    if (ep->addressv[code_ix] == ep->code) {
-		if (ep->code[0] == (BeamInstr) em_apply_bif) {
+	    if (ep->addressv[code_ix] == ep->beam) {
+		if (ep->beam[0] == (BeamInstr) em_apply_bif) {
 		    continue;
 		}
-		else if (ep->code[0] ==
+		else if (ep->beam[0] ==
 			 (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
 		    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
 		    ASSERT(modp->curr.num_traced_exports > 0);
 		    erts_clear_export_break(modp, &ep->info);
 		}
-		else ASSERT(ep->code[0] == (BeamInstr) em_call_error_handler
+		else ASSERT(ep->beam[0] == (BeamInstr) em_call_error_handler
 			    || !erts_initialized);
 	    }
-	    ep->addressv[code_ix] = ep->code;
-	    ep->code[0] = (BeamInstr) em_call_error_handler;
-	    ep->code[1] = 0;
+	    ep->addressv[code_ix] = ep->beam;
+	    ep->beam[0] = (BeamInstr) em_call_error_handler;
+	    ep->beam[1] = 0;
 	}
     }
 
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 2cdf66f441..62464f3864 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -238,7 +238,7 @@ erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified)
             ASSERT(0);
         }
 
-	pc = ep->code;
+	pc = ep->beam;
 	if (ep->addressv[code_ix] == pc) {
 	    if ((*pc == (BeamInstr) em_apply_bif ||
 		 *pc == (BeamInstr) em_call_error_handler)) {
@@ -712,7 +712,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
     Export* ep = bif_export[bif_index];
     Uint32 flags = 0, flags_meta = 0;
     ErtsTracer meta_tracer = erts_tracer_nil;
-    int applying = (I == ep->code); /* Yup, the apply code for a bif
+    int applying = (I == ep->beam); /* Yup, the apply code for a bif
                                       * is actually in the
                                       * export entry */
     BeamInstr *cp = p->cp;
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 2c47386925..8e88d6ccd5 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -119,7 +119,7 @@ do {                                     \
 #define GET_BIF_MODULE(p)  (p->info.mfa.module)
 #define GET_BIF_FUNCTION(p)  (p->info.mfa.function)
 #define GET_BIF_ARITY(p)  (p->info.mfa.arity)
-#define GET_BIF_ADDRESS(p) ((BifFunction) (p->code[1]))
+#define GET_BIF_ADDRESS(p) ((BifFunction) (p->beam[1]))
 #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm))))
 
 
@@ -5147,8 +5147,8 @@ do {						\
 			      bif_table[i].name,
 			      bif_table[i].arity);
 	 bif_export[i] = ep;
-	 ep->code[0] = (BeamInstr) OpCode(apply_bif);
-	 ep->code[1] = (BeamInstr) bif_table[i].f;
+	 ep->beam[0] = (BeamInstr) OpCode(apply_bif);
+	 ep->beam[1] = (BeamInstr) bif_table[i].f;
 	 /* XXX: set func info for bifs */
 	 ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI);
      }
@@ -7219,8 +7219,8 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity)
     if ((ep = export_get(&e)) == NULL) {
 	return 0;
     }
-    return ep->addressv[erts_active_code_ix()] == ep->code
-	&& (ep->code[0] == (BeamInstr) em_apply_bif);
+    return ep->addressv[erts_active_code_ix()] == ep->beam
+	&& (ep->beam[0] == (BeamInstr) em_apply_bif);
 }
 
 
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 76a92d41f7..13552ae88c 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -809,18 +809,18 @@ erts_finish_loading(Binary* magic, Process* c_p,
 	    if (ep == NULL || ep->info.mfa.module != module) {
 		continue;
 	    }
-	    if (ep->addressv[code_ix] == ep->code) {
-		if (ep->code[0] == (BeamInstr) em_apply_bif) {
+	    if (ep->addressv[code_ix] == ep->beam) {
+		if (ep->beam[0] == (BeamInstr) em_apply_bif) {
 		    continue;
-		} else if (ep->code[0] ==
+		} else if (ep->beam[0] ==
 			   (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
 		    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
 		    ASSERT(mod_tab_p->curr.num_traced_exports > 0);
 		    erts_clear_export_break(mod_tab_p, &ep->info);
-		    ep->addressv[code_ix] = (BeamInstr *) ep->code[1];
-		    ep->code[1] = 0;
+		    ep->addressv[code_ix] = (BeamInstr *) ep->beam[1];
+		    ep->beam[1] = 0;
 		}
-		ASSERT(ep->code[1] == 0);
+		ASSERT(ep->beam[1] == 0);
 	    }
 	}
 	ASSERT(mod_tab_p->curr.num_breakpoints == 0);
@@ -1420,8 +1420,8 @@ load_import_table(LoaderState* stp)
 	 * the BIF function.
 	 */
 	if ((e = erts_active_export_entry(mod, func, arity)) != NULL) {
-	    if (e->code[0] == (BeamInstr) em_apply_bif) {
-		stp->import[i].bf = (BifFunction) e->code[1];
+	    if (e->beam[0] == (BeamInstr) em_apply_bif) {
+		stp->import[i].bf = (BifFunction) e->beam[1];
 		if (func == am_load_nif && mod == am_erlang && arity == 2) {
 		    stp->may_load_nif = 1;
 		}
@@ -1514,7 +1514,7 @@ is_bif(Eterm mod, Eterm func, unsigned arity)
     if (e == NULL) {
 	return 0;
     }
-    if (e->code[0] != (BeamInstr) em_apply_bif) {
+    if (e->beam[0] != (BeamInstr) em_apply_bif) {
 	return 0;
     }
     if (mod == am_erlang && func == am_apply && arity == 3) {
@@ -4784,7 +4784,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
 	     * callable yet. Keep any function in the current
 	     * code callable.
 	     */
-	    ep->code[1] = (BeamInstr) address;
+	    ep->beam[1] = (BeamInstr) address;
 	}
     }
 
@@ -4962,7 +4962,7 @@ transform_engine(LoaderState* st)
 		if (i >= st->num_imports || st->import[i].bf == NULL)
 		    goto restart;
 		if (bif_number != -1 &&
-		    bif_export[bif_number]->code[1] != (BeamInstr) st->import[i].bf) {
+		    bif_export[bif_number]->beam[1] != (BeamInstr) st->import[i].bf) {
 		    goto restart;
 		}
 	    }
@@ -5723,8 +5723,8 @@ exported_from_module(Process* p, /* Process whose heap to use. */
 	if (ep->info.mfa.module == mod) {
 	    Eterm tuple;
 	    
-	    if (ep->addressv[code_ix] == ep->code &&
-		ep->code[0] == (BeamInstr) em_call_error_handler) {
+	    if (ep->addressv[code_ix] == ep->beam &&
+		ep->beam[0] == (BeamInstr) em_call_error_handler) {
 		/* There is a call to the function, but it does not exist. */ 
 		continue;
 	    }
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 324f7d4268..d886c2985e 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4963,13 +4963,13 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a,
     int i;
     sys_memset((void *) ep, 0, sizeof(Export));
     for (i=0; i<ERTS_NUM_CODE_IX; i++) {
-	ep->addressv[i] = ep->code;
+	ep->addressv[i] = ep->beam;
     }
     ep->info.mfa.module = m;
     ep->info.mfa.function = f;
     ep->info.mfa.arity = a;
-    ep->code[0] = (BeamInstr) em_apply_bif;
-    ep->code[1] = (BeamInstr) bif;
+    ep->beam[0] = (BeamInstr) em_apply_bif;
+    ep->beam[1] = (BeamInstr) bif;
 }
 
 void erts_init_bif(void)
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index e3355208e5..1a20cc911e 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -980,7 +980,7 @@ static int function_is_traced(Process *p,
     e.info.mfa.function = mfa[1];
     e.info.mfa.arity = mfa[2];
     if ((ep = export_get(&e)) != NULL) {
-	pc = ep->code;
+	pc = ep->beam;
 	if (ep->addressv[erts_active_code_ix()] == pc &&
 	    *pc != (BeamInstr) em_call_error_handler) {
 
@@ -1380,12 +1380,12 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
 #ifdef DEBUG
 		ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI);
 #endif
-                ep->code[0] = (BeamInstr) BeamOp(op_jump_f);
-		ep->code[1] = (BeamInstr) ep->addressv[code_ix];
+                ep->beam[0] = (BeamInstr) BeamOp(op_jump_f);
+		ep->beam[1] = (BeamInstr) ep->addressv[code_ix];
 	    }
 	    erts_set_call_trace_bif(ci, match_prog_set, 0);
 	    if (ep->addressv[code_ix] != pc) {
-		ep->code[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint);
+		ep->beam[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint);
 	    }
 	} else if (!on && flags.breakpoint) {
 	    /* Turn off breakpoint tracing -- nothing to do here. */
@@ -1395,8 +1395,8 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
 	     * before turning on breakpoint tracing.
 	     */
 	    erts_clear_call_trace_bif(ci, 0);
-	    if (ep->code[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
-		ep->code[0] = (BeamInstr) BeamOp(op_jump_f);
+	    if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
+		ep->beam[0] = (BeamInstr) BeamOp(op_jump_f);
 	    }
 	}
     }
@@ -1679,7 +1679,7 @@ install_exp_breakpoints(BpFunctions* f)
     for (i = 0; i < ne; i++) {
 	Export* ep = ErtsContainerStruct(fp[i].ci, Export, info);
 
-	ep->addressv[code_ix] = ep->code;
+	ep->addressv[code_ix] = ep->beam;
     }
 }
 
@@ -1694,11 +1694,11 @@ uninstall_exp_breakpoints(BpFunctions* f)
     for (i = 0; i < ne; i++) {
 	Export* ep = ErtsContainerStruct(fp[i].ci, Export, info);
 
-	if (ep->addressv[code_ix] != ep->code) {
+	if (ep->addressv[code_ix] != ep->beam) {
 	    continue;
 	}
-	ASSERT(ep->code[0] == (BeamInstr) BeamOp(op_jump_f));
-	ep->addressv[code_ix] = (BeamInstr *) ep->code[1];
+	ASSERT(ep->beam[0] == (BeamInstr) BeamOp(op_jump_f));
+	ep->addressv[code_ix] = (BeamInstr *) ep->beam[1];
     }
 }
 
@@ -1713,12 +1713,12 @@ clean_export_entries(BpFunctions* f)
     for (i = 0; i < ne; i++) {
 	Export* ep = ErtsContainerStruct(fp[i].ci, Export, info);
 
-	if (ep->addressv[code_ix] == ep->code) {
+	if (ep->addressv[code_ix] == ep->beam) {
 	    continue;
 	}
-	if (ep->code[0] == (BeamInstr) BeamOp(op_jump_f)) {
-	    ep->code[0] = (BeamInstr) 0;
-	    ep->code[1] = (BeamInstr) 0;
+	if (ep->beam[0] == (BeamInstr) BeamOp(op_jump_f)) {
+	    ep->beam[0] = (BeamInstr) 0;
+	    ep->beam[1] = (BeamInstr) 0;
 	}
     }
 }
@@ -1733,8 +1733,8 @@ setup_bif_trace(void)
 	GenericBp* g = (GenericBp *) ep->info.native;
 	if (g) {
 	    if (ExportIsBuiltIn(ep)) {
-		ASSERT(ep->code[1]);
-		ep->code[1] = (BeamInstr) bif_table[i].traced;
+		ASSERT(ep->beam[1]);
+		ep->beam[1] = (BeamInstr) bif_table[i].traced;
 	    }
 	}
     }
@@ -1751,8 +1751,8 @@ reset_bif_trace(void)
 	GenericBp* g = (GenericBp *) ep->info.native;
 	if (g && g->data[active].flags == 0) {
 	    if (ExportIsBuiltIn(ep)) {
-		ASSERT(ep->code[1]);
-		ep->code[1] = (BeamInstr) bif_table[i].f;
+		ASSERT(ep->beam[1]);
+		ep->beam[1] = (BeamInstr) bif_table[i].f;
 	    }
 	}
     }
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 167cded28f..d2f18667bf 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2328,9 +2328,9 @@ allocate_nif_sched_data(Process* proc, int argc)
     ep->rootset_extra = argc;
     ep->rootset[0] = NIL;
     for (i=0; i<ERTS_NUM_CODE_IX; i++) {
-	ep->exp.addressv[i] = &ep->exp.code[0];
+	ep->exp.addressv[i] = &ep->exp.beam[0];
     }
-    ep->exp.code[0] = (BeamInstr) em_call_nif;
+    ep->exp.beam[0] = (BeamInstr) em_call_nif;
     (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ep);
     return ep;
 }
@@ -2404,7 +2404,7 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec
     ep->exp.info.mfa.module = proc->current->module;
     ep->exp.info.mfa.function = proc->current->function;
     ep->exp.info.mfa.arity = argc;
-    ep->exp.code[1] = (BeamInstr) direct_fp;
+    ep->exp.beam[1] = (BeamInstr) direct_fp;
     ep->m = env->mod_nif;
     ep->fp = indirect_fp;
     proc->freason = TRAP;
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 0a36175398..060fde074c 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -136,11 +136,11 @@ export_alloc(struct export_entry* tmpl_e)
 	obj->info.mfa.module = tmpl->info.mfa.module;
 	obj->info.mfa.function = tmpl->info.mfa.function;
 	obj->info.mfa.arity = tmpl->info.mfa.arity;
-	obj->code[0] = (BeamInstr) em_call_error_handler;
-	obj->code[1] = 0;
+	obj->beam[0] = (BeamInstr) em_call_error_handler;
+	obj->beam[1] = 0;
 
 	for (ix=0; ix<ERTS_NUM_CODE_IX; ix++) {
-	    obj->addressv[ix] = obj->code;
+	    obj->addressv[ix] = obj->beam;
 
 	    blob->entryv[ix].slot.index = -1;
 	    blob->entryv[ix].ep = &blob->exp;
@@ -266,8 +266,8 @@ erts_find_function(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix)
 
     ee = hash_get(&export_tables[code_ix].htable, init_template(&templ, m, f, a));
     if (ee == NULL ||
-	(ee->ep->addressv[code_ix] == ee->ep->code &&
-	 ee->ep->code[0] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) {
+	(ee->ep->addressv[code_ix] == ee->ep->beam &&
+	 ee->ep->beam[0] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) {
 	return NULL;
     }
     return ee->ep;
diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h
index a807efef94..198b90c839 100644
--- a/erts/emulator/beam/export.h
+++ b/erts/emulator/beam/export.h
@@ -33,20 +33,20 @@ typedef struct export
 {
     void* addressv[ERTS_NUM_CODE_IX];  /* Pointer to code for function. */
 
-    ErtsCodeInfo info; /* MUST be just before code[] */
+    ErtsCodeInfo info; /* MUST be just before beam[] */
 
     /*
-     * code[0]: This entry is 0 unless the 'address' field points to it.
+     * beam[0]: This entry is 0 unless the 'addressv' field points to it.
      *          Threaded code instruction to load function
      *		(em_call_error_handler), execute BIF (em_apply_bif),
      *		or a breakpoint instruction (op_i_generic_breakpoint).
-     * code[1]: Function pointer to BIF function (for BIFs only),
+     * beam[1]: Function pointer to BIF function (for BIFs only),
      *		or pointer to threaded code if the module has an
      *		on_load function that has not been run yet, or pointer
-     *          to code if function code[0] is a breakpoint instruction.
+     *          to code if function beam[0] is a breakpoint instruction.
      *		Otherwise: 0.
      */
-    BeamInstr code[2];
+    BeamInstr beam[2];
 } Export;
 
 
@@ -72,8 +72,8 @@ extern erts_smp_mtx_t export_staging_lock;
 
 #include "beam_load.h" /* For em_* extern declarations */ 
 #define ExportIsBuiltIn(EntryPtr) 			\
-(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->code) && \
- ((EntryPtr)->code[0] == (BeamInstr) em_apply_bif))
+(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->beam) && \
+ ((EntryPtr)->beam[0] == (BeamInstr) em_apply_bif))
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-- 
cgit v1.2.3


From c283f299551d53b62027ae1a8b189285c57573e8 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 11 Oct 2016 08:14:04 +0200
Subject: erts: Fix some dtrace related compile issues

---
 erts/emulator/beam/beam_emu.c    | 31 ++++++++++++++-----------------
 erts/emulator/beam/erl_process.c |  7 ++++---
 2 files changed, 18 insertions(+), 20 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8e88d6ccd5..3b02df8bac 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1160,7 +1160,7 @@ init_emulator(void)
 #define DTRACE_NIF_RETURN(p, mfa)                                       \
     if (DTRACE_ENABLED(nif_return)) {                                   \
         DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);             \
-        DTRACE_CHARBUF(mfa_mfa, DTRACE_TERM_BUF_SIZE);                  \
+        DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE);                  \
         dtrace_fun_decode(p, mfa, process_name, mfa_buf);               \
         DTRACE2(nif_return, process_name, mfa_buf);                     \
     }
@@ -1175,9 +1175,9 @@ init_emulator(void)
 
 #define DTRACE_RETURN_FROM_PC(p)                                                        \
     do {                                                                                \
-        ErtsCodeInfo* ci;                                                                  \
-        if (DTRACE_ENABLED(function_return) && (ci = find_function_from_pc((p)->cp))) { \
-            DTRACE_RETURN((p), &ci->mfa);                               \
+        ErtsCodeMFA* cmfa;                                                                  \
+        if (DTRACE_ENABLED(function_return) && (cmfa = find_function_from_pc((p)->cp))) { \
+            DTRACE_RETURN((p), cmfa);                               \
         }                                                                               \
     } while(0)
 
@@ -1392,10 +1392,9 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
             if (ERTS_PROC_IS_EXITING(c_p)) {
                 strcpy(fun_buf, "<exiting>");
             } else {
-                BeamInstr *fptr = find_function_from_pc(c_p->i);
-                if (fptr) {
-                    dtrace_fun_decode(c_p, (Eterm)fptr[0],
-                                      (Eterm)fptr[1], (Uint)fptr[2],
+                ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i);
+                if (cmfa) {
+                    dtrace_fun_decode(c_p, cmfa,
                                       NULL, fun_buf);
                 } else {
                     erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)),
@@ -5356,11 +5355,9 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
             if (ERTS_PROC_IS_EXITING(c_p)) {
                 strcpy(fun_buf, "<exiting>");
             } else {
-                BeamInstr *fptr = find_function_from_pc(c_p->i);
-                if (fptr) {
-                    dtrace_fun_decode(c_p, (Eterm)fptr[0],
-                                      (Eterm)fptr[1], (Uint)fptr[2],
-                                      NULL, fun_buf);
+                ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i);
+                if (cmfa) {
+		    dtrace_fun_decode(c_p, cmfa, NULL, fun_buf);
                 } else {
                     erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)),
                                   "<unknown/%p>", *I);
@@ -6401,11 +6398,11 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
 
 #ifdef USE_VM_PROBES
     if (DTRACE_ENABLED(process_hibernate)) {
+        ErtsCodeMFA cmfa = { module, function, arity};
         DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);
-        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);
-        dtrace_fun_decode(c_p, module, function, arity,
-                          process_name, mfa);
-        DTRACE2(process_hibernate, process_name, mfa);
+        DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE);
+        dtrace_fun_decode(c_p, &cmfa, process_name, mfa_buf);
+        DTRACE2(process_hibernate, process_name, mfa_buf);
     }
 #endif
     /*
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index d0a0310544..dd5943b5cf 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11690,11 +11690,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 
 #ifdef USE_VM_PROBES
     if (DTRACE_ENABLED(process_spawn)) {
+        ErtsCodeMFA cmfa = {mod, func, arity};
         DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);
-        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);
+        DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE);
 
-        dtrace_fun_decode(p, mod, func, arity, process_name, mfa);
-        DTRACE2(process_spawn, process_name, mfa);
+        dtrace_fun_decode(p, &cmfa, process_name, mfa_buf);
+        DTRACE2(process_spawn, process_name, mfa_buf);
     }
 #endif
     return res;
-- 
cgit v1.2.3


From e22f99f0303bb5e70b128118bad320ee9368e4ba Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 12 Oct 2016 16:14:51 +0200
Subject: erts: Add strict size assert to fix alloc

and remove outdated min size adjustment.

The is similar to lost commit 31bc414dc9639ccf94f9011ed32
except we now assume and assert strict size equality.
---
 erts/emulator/beam/erl_alloc_util.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 2995f2f822..dd8ca5fac1 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1361,6 +1361,7 @@ fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size)
 	   && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
 
     fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE];
+    ASSERT(size == fix->type_size);
 
     res = fix->list;
     if (res) {
@@ -1372,8 +1373,6 @@ fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size)
 	fix_cpool_check_shrink(allctr, type, fix, NULL);
 	return res;
     }
-    if (size < 2*sizeof(UWord))
-	size += sizeof(UWord);
     if (size >= allctr->sbc_threshold) {
 	Block_t *blk;
 	blk = create_carrier(allctr, size, CFLG_SBC);
@@ -1493,6 +1492,7 @@ fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size)
 	   && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
 
     fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE];
+    ASSERT(size == fix->type_size);
 
     ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
     fix->u.nocpool.used++;
@@ -1515,8 +1515,6 @@ fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size)
 	ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
 	return res;
     }
-    if (size < 2*sizeof(UWord))
-	size += sizeof(UWord);
     if (fix->u.nocpool.limit < fix->u.nocpool.used)
 	fix->u.nocpool.limit = fix->u.nocpool.used;
     if (fix->u.nocpool.max_used < fix->u.nocpool.used)
-- 
cgit v1.2.3


From 2dfca60a24c2c6728a4b8431f37aeee05212a4ac Mon Sep 17 00:00:00 2001
From: Gabriele Santomaggio <gabriele.santomaggio@erlang-solutions.com>
Date: Thu, 13 Oct 2016 21:17:10 +0200
Subject: Add system_info(atom_limit)

Add system_info(atom_limit) to provide a way to retrieve the maximum
number of atoms allowed. Add tests and documentation for it too.

Also split system_info_SUITE:start_node/2 to start_node_ets/2 and
start_node_atm/2 to avoid code duplication.
---
 erts/emulator/beam/atom.c                |  6 ++++
 erts/emulator/beam/atom.h                |  1 +
 erts/emulator/beam/erl_bif_info.c        |  3 ++
 erts/emulator/test/system_info_SUITE.erl | 59 +++++++++++++++++++++++++-------
 4 files changed, 57 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index a5e778e4aa..d2fad24805 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -483,3 +483,9 @@ dump_atoms(int to, void *to_arg)
 	}
     }
 }
+
+Uint
+erts_get_atom_limit(void)
+{
+    return erts_atom_table.limit;
+}
\ No newline at end of file
diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h
index ae60904785..0d34c47a55 100644
--- a/erts/emulator/beam/atom.h
+++ b/erts/emulator/beam/atom.h
@@ -138,6 +138,7 @@ int atom_static_put(byte*, int);
 void init_atom_table(void);
 void atom_info(int, void *);
 void dump_atoms(int, void *);
+Uint erts_get_atom_limit(void);
 int erts_atom_get(const char* name, int len, Eterm* ap, ErtsAtomEncoding enc);
 void erts_atom_get_text_space_sizes(Uint *reserved, Uint *used);
 #endif
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 29ba12dfdb..cfe01f244e 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2859,6 +2859,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     else if (ERTS_IS_ATOM_STR("ets_limit",BIF_ARG_1)) {
         BIF_RET(make_small(erts_db_get_max_tabs()));
     }
+    else if (ERTS_IS_ATOM_STR("atom_limit",BIF_ARG_1)) {
+        BIF_RET(make_small(erts_get_atom_limit()));
+    }
     else if (ERTS_IS_ATOM_STR("tolerant_timeofday",BIF_ARG_1)) {
 	if (erts_has_time_correction()
 	    && erts_time_offset_state() == ERTS_TIME_OFFSET_FINAL) {
diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl
index a4aedb31f6..3d9e74472b 100644
--- a/erts/emulator/test/system_info_SUITE.erl
+++ b/erts/emulator/test/system_info_SUITE.erl
@@ -36,7 +36,7 @@
 -export([all/0, suite/0]).
 
 -export([process_count/1, system_version/1, misc_smoke_tests/1,
-         heap_size/1, wordsize/1, memory/1, ets_limit/1]).
+         heap_size/1, wordsize/1, memory/1, ets_limit/1, atom_limit/1]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
@@ -44,7 +44,7 @@ suite() ->
 
 all() -> 
     [process_count, system_version, misc_smoke_tests,
-     heap_size, wordsize, memory, ets_limit].
+     heap_size, wordsize, memory, ets_limit, atom_limit].
 
 %%%
 %%% The test cases -------------------------------------------------------------
@@ -472,6 +472,17 @@ mapn(_Fun, 0) ->
 mapn(Fun, N) ->
     [Fun(N) | mapn(Fun, N-1)].
 
+
+get_node_name(Config) ->
+    list_to_atom(atom_to_list(?MODULE)
+		 ++ "-"
+		 ++ atom_to_list(proplists:get_value(testcase, Config))
+		 ++ "-"
+		 ++ integer_to_list(erlang:system_time(second))
+		 ++ "-"
+		 ++ integer_to_list(erlang:unique_integer([positive]))).
+
+
 %% Verify system_info(ets_limit) reflects max ETS table settings.
 ets_limit(Config0) when is_list(Config0) ->
     Config = [{testcase,ets_limit}|Config0],
@@ -486,7 +497,7 @@ get_ets_limit(Config, EtsMax) ->
                0 -> [];
                _ -> [{"ERL_MAX_ETS_TABLES", integer_to_list(EtsMax)}]
            end,
-    {ok, Node} = start_node(Config, Envs),
+    {ok, Node} = start_node_ets(Config, Envs),
     Me = self(),
     Ref = make_ref(),
     spawn_link(Node,
@@ -502,16 +513,40 @@ get_ets_limit(Config, EtsMax) ->
     stop_node(Node),
     Res.
 
-start_node(Config, Envs) when is_list(Config) ->
+start_node_ets(Config, Envs) when is_list(Config) ->
+    Pa = filename:dirname(code:which(?MODULE)),
+    test_server:start_node(get_node_name(Config), peer,
+			   [{args, "-pa "++Pa}, {env, Envs}]).
+
+start_node_atm(Config, AtomsMax) when is_list(Config) ->
     Pa = filename:dirname(code:which(?MODULE)),
-    Name = list_to_atom(atom_to_list(?MODULE)
-                        ++ "-"
-                        ++ atom_to_list(proplists:get_value(testcase, Config))
-                        ++ "-"
-                        ++ integer_to_list(erlang:system_time(second))
-                        ++ "-"
-                        ++ integer_to_list(erlang:unique_integer([positive]))),
-    test_server:start_node(Name, peer, [{args, "-pa "++Pa}, {env, Envs}]).
+    test_server:start_node(get_node_name(Config), peer,
+			   [{args, "-pa "++ Pa ++ AtomsMax}]).
 
 stop_node(Node) ->
     test_server:stop_node(Node).
+
+
+%% Verify system_info(atom_limit) reflects max atoms settings
+%% (using " +t").
+atom_limit(Config0) when is_list(Config0) ->
+    Config = [{testcase,atom_limit}|Config0],
+    2186042 = get_atom_limit(Config, " +t 2186042 "),
+    ok.
+
+get_atom_limit(Config, AtomsMax) ->
+    {ok, Node} = start_node_atm(Config, AtomsMax),
+    Me = self(),
+    Ref = make_ref(),
+    spawn_link(Node,
+        fun() ->
+            Res = erlang:system_info(atom_limit),
+            unlink(Me),
+            Me ! {Ref, Res}
+        end),
+    receive
+        {Ref, Res} ->
+            Res
+    end,
+    stop_node(Node),
+    Res.
-- 
cgit v1.2.3


From 39072836944d00c288beebfd98b14593f9609006 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Magnus=20L=C3=A5ng?= <margnus1@telia.com>
Date: Thu, 19 May 2016 17:38:54 +0200
Subject: Add a loader state for HiPE code loading

Just like the BEAM loader state (as returned by
erlang:prepare_loading/2), the HiPE loader state is contained in a magic
binary.

Eventually, we will separate HiPE loading into a prepare and a finalise
phase, like the BEAM loader, where the prepare phase will be implemented
by hipe_unified_loader and the finalise phase be implemented in C by
hipe_load.c and beam_load.c, making prepare side-effect free and
finalise atomic. The finalise phase will be exposed through the
erlang:finish_loading/1 API, just like the BEAM loader, as this will
allow HiPE and BEAM modules to be mixed in the same atomic "commit".

The usage of a loader state makes it easier to keep track of all
resources allocated during loading, and will not only make it easy to
prevent leaks when hipe_unified_loader crashes, but also paves the way
for proper, leak-free, unloading of HiPE modules.
---
 erts/emulator/Makefile.in          |   2 +
 erts/emulator/beam/beam_bif_load.c |  14 +++--
 erts/emulator/beam/beam_load.c     |  69 +++++++++++++++----------
 erts/emulator/beam/module.c        |   5 +-
 erts/emulator/beam/module.h        |   9 ++--
 erts/emulator/hipe/hipe_bif0.c     |  88 +++++++++++++++++++++++++-------
 erts/emulator/hipe/hipe_bif0.tab   |   6 ++-
 erts/emulator/hipe/hipe_load.c     | 101 +++++++++++++++++++++++++++++++++++++
 erts/emulator/hipe/hipe_load.h     |  45 +++++++++++++++++
 erts/emulator/hipe/hipe_module.c   |  37 ++++++++++++++
 erts/emulator/hipe/hipe_module.h   |  42 +++++++++++++++
 11 files changed, 361 insertions(+), 57 deletions(-)
 create mode 100644 erts/emulator/hipe/hipe_load.c
 create mode 100644 erts/emulator/hipe/hipe_load.h
 create mode 100644 erts/emulator/hipe/hipe_module.c
 create mode 100644 erts/emulator/hipe/hipe_module.h

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 21bcbbab27..fa4de5805e 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -884,7 +884,9 @@ HIPE_OBJS= \
 	$(OBJDIR)/hipe_bif2.o \
 	$(OBJDIR)/hipe_debug.o \
 	$(OBJDIR)/hipe_gc.o \
+	$(OBJDIR)/hipe_load.o \
 	$(OBJDIR)/hipe_mode_switch.o \
+	$(OBJDIR)/hipe_module.o \
 	$(OBJDIR)/hipe_native_bif.o \
 	$(OBJDIR)/hipe_stack.o $(HIPE_ARCH_OBJS)
 ifdef HIPE_ENABLED
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 237513095a..3302de0787 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -150,7 +150,13 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1)
 BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
 {
     Module* modp;
-    Eterm res;
+    Eterm res, mod;
+
+    if (!ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_1) ||
+	is_not_atom(mod = erts_module_for_prepared_code
+		    (((ProcBin*)binary_val(BIF_ARG_1))->val))) {
+	BIF_ERROR(BIF_P, BADARG);
+    }
 
     if (!erts_try_seize_code_write_permission(BIF_P)) {
 	ERTS_BIF_YIELD3(bif_export[BIF_code_make_stub_module_3],
@@ -160,7 +166,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
     erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
     erts_smp_thr_progress_block();
 
-    modp = erts_get_module(BIF_ARG_1, erts_active_code_ix());
+    modp = erts_get_module(mod, erts_active_code_ix());
 
     if (modp && modp->curr.num_breakpoints > 0) {
 	ASSERT(modp->curr.code_hdr != NULL);
@@ -172,12 +178,12 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
 
     res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
 
-    if (res == BIF_ARG_1) {
+    if (res == mod) {
 	erts_end_staging_code_ix();
 	erts_commit_staging_code_ix();
 #ifdef HIPE
         if (!modp)
-	    modp = erts_get_module(BIF_ARG_1, erts_active_code_ix());
+	    modp = erts_get_module(mod, erts_active_code_ix());
         hipe_redirect_to_module(modp);
 #endif
     }
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 9b206f9a23..7d29f393e5 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -44,6 +44,7 @@
 #include "hipe_bif0.h"
 #include "hipe_mode_switch.h"
 #include "hipe_arch.h"
+#include "hipe_load.h"
 #endif
 
 ErlDrvBinary* erts_gzinflate_buffer(char*, int);
@@ -482,8 +483,8 @@ static void free_literal_fragment(ErlHeapFragment*);
 static void loader_state_dtor(Binary* magic);
 static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 				  Eterm group_leader, Eterm module,
-				  BeamCodeHeader* code, Uint size,
-				  void* hipe_code_start, UWord hipe_code_size);
+				  BeamCodeHeader* code_hdr, Uint size,
+				  HipeModule *hipe_code);
 static int init_iff_file(LoaderState* stp, byte* code, Uint size);
 static int scan_iff_file(LoaderState* stp, Uint* chunk_types,
 			 Uint num_types, Uint num_mandatory);
@@ -942,6 +943,13 @@ erts_module_for_prepared_code(Binary* magic)
     LoaderState* stp;
 
     if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != loader_state_dtor) {
+#ifdef HIPE
+	HipeLoaderState *hipe_stp;
+	if ((hipe_stp = hipe_get_loader_state(magic))
+	    && hipe_stp->text_segment != 0) {
+	    return hipe_stp->module;
+	}
+#endif
 	return NIL;
     }
     stp = ERTS_MAGIC_BIN_DATA(magic);
@@ -1094,7 +1102,7 @@ static Eterm
 stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 		     Eterm group_leader, Eterm module,
 		     BeamCodeHeader* code_hdr, Uint size,
-		     void* hipe_code_start, UWord hipe_code_size)
+		     HipeModule *hipe_code)
 {
     Module* modp;
     Eterm retval;
@@ -1121,12 +1129,9 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
     DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code new_hipe_refs = %p", modp->new_hipe_refs);
     modp->curr.first_hipe_ref = modp->new_hipe_refs;
     modp->curr.first_hipe_sdesc = modp->new_hipe_sdesc;
-    modp->curr.hipe_code_start = hipe_code_start;
     modp->new_hipe_refs = NULL;
     modp->new_hipe_sdesc = NULL;
-# ifdef DEBUG
-    modp->curr.hipe_code_size  = hipe_code_size;
-# endif
+    modp->curr.hipe_code = hipe_code;
 #endif
 
     /*
@@ -6271,10 +6276,13 @@ patch_funentries(Eterm Patchlist)
  */
 
 Eterm
-erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
+erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
 {
     Binary* magic;
+    Binary* hipe_magic;
     LoaderState* stp;
+    HipeLoaderState* hipe_stp;
+    HipeModule *hipe_code;
     BeamInstr Funcs;
     BeamInstr Patchlist;
     Eterm MD5Bin;
@@ -6290,7 +6298,6 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     byte* temp_alloc = NULL;
     byte* bytes;
     Uint size;
-    UWord hipe_code_start = NULL, hipe_code_size = 0;
 
     /*
      * Must initialize stp->lambdas here because the error handling code
@@ -6298,15 +6305,19 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
      */
     magic = erts_alloc_loader_state();
     stp = ERTS_MAGIC_BIN_DATA(magic);
+    hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code));
 
-    if (is_not_atom(Mod)) {
+    if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) ||
+	!(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val,
+	  hipe_stp = hipe_get_loader_state(hipe_magic)) ||
+	hipe_stp->module == NIL || hipe_stp->text_segment == 0) {
 	goto error;
     }
     if (is_not_tuple(Info)) {
 	goto error;
     }
     tp = tuple_val(Info);
-    if (tp[0] != make_arityval(5)) {
+    if (tp[0] != make_arityval(3)) {
       goto error;
     }
     Funcs = tp[1];
@@ -6323,20 +6334,11 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     }
     size = binary_size(Beam);
 
-#ifdef HIPE
-    if (!term_to_Uint(tp[4], &hipe_code_start))
-        goto error;
-# ifdef DEBUG
-    if (!term_to_Uint(tp[5], &hipe_code_size))
-        goto error;
-# endif
-#endif
-
     /*
      * Scan the Beam binary and read the interesting sections.
      */
 
-    stp->module = Mod;
+    stp->module = hipe_stp->module;
     stp->group_leader = p->group_leader;
     stp->num_functions = n;
     if (!init_iff_file(stp, bytes, size)) {
@@ -6450,7 +6452,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 #else
 	op = (Eterm) BeamOpCode(op_move_return_n);
 #endif
-	fp = make_stub(fp, Mod, func, arity, (Uint)native_address, op);
+	fp = make_stub(fp, hipe_stp->module, func, arity, (Uint)native_address,
+		       op);
     }
 
     /*
@@ -6489,13 +6492,19 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
       erts_free_aligned_binary_bytes(tmp);
     }
 
+    /*
+     * Initialise HiPE module
+     */
+    hipe_code->text_segment = hipe_stp->text_segment;
+    hipe_code->text_segment_size = hipe_stp->text_segment_size;
+    hipe_code->data_segment = hipe_stp->data_segment;
+
     /*
      * Insert the module in the module table.
      */
 
-    rval = stub_insert_new_code(p, 0, p->group_leader, Mod,
-				code_hdr, code_size,
-				(void*)hipe_code_start, hipe_code_size);
+    rval = stub_insert_new_code(p, 0, p->group_leader, hipe_stp->module,
+				code_hdr, code_size, hipe_code);
     if (rval != NIL) {
 	goto error;
     }
@@ -6511,12 +6520,20 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     }
 
     if (patch_funentries(Patchlist)) {
+	Eterm mod = hipe_stp->module;
+	/* Prevent code from being freed */
+	hipe_stp->text_segment = 0;
+	hipe_stp->data_segment = 0;
+
 	erts_free_aligned_binary_bytes(temp_alloc);
 	free_loader_state(magic);
-	return Mod;
+	hipe_free_loader_state(hipe_magic);
+
+	return mod;
     }
 
  error:
+    erts_free(ERTS_ALC_T_HIPE, hipe_code);
     erts_free_aligned_binary_bytes(temp_alloc);
     free_loader_state(magic);
     BIF_ERROR(p, BADARG);
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index 93ce8fcf9f..5c56ad20df 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -79,10 +79,7 @@ void erts_module_instance_init(struct erl_module_instance* modi)
 #ifdef HIPE
     modi->first_hipe_ref = NULL;
     modi->first_hipe_sdesc = NULL;
-    modi->hipe_code_start = NULL;
-# ifdef DEBUG
-    modi->hipe_code_size = 0;
-# endif
+    modi->hipe_code = NULL;
 #endif
 }
 
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index 694583597b..c3adaf12fc 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -23,6 +23,10 @@
 
 #include "index.h"
 
+#ifdef HIPE
+#include "hipe_module.h"
+#endif
+
 struct erl_module_instance {
     BeamCodeHeader* code_hdr;
     int code_length;		/* Length of loaded code in bytes. */
@@ -33,10 +37,7 @@ struct erl_module_instance {
 #ifdef HIPE
     struct hipe_ref* first_hipe_ref;  /* all external hipe calls from this module */
     struct hipe_sdesc* first_hipe_sdesc;  /* all stack descriptors for this module */
-    void* hipe_code_start;
-# ifdef DEBUG
-    UWord hipe_code_size;
-# endif
+    HipeModule *hipe_code;
 #endif
 };
 
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 6f9bfe2f40..daa198d695 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -44,6 +44,7 @@
 #include "hipe_mode_switch.h"
 #include "hipe_native_bif.h"
 #include "hipe_bif0.h"
+#include "hipe_load.h"
 /* We need hipe_literals.h for HIPE_SYSTEM_CRC, but it redefines
    a few constants. #undef them here to avoid warnings. */
 #undef F_TIMO
@@ -374,16 +375,32 @@ BIF_RETTYPE hipe_bifs_ref_set_2(BIF_ALIST_2)
     BIF_RET(BIF_ARG_1);
 }
 
+/*
+ * BIFs for loading code.
+ */
+
+static HipeLoaderState *get_loader_state(Eterm term)
+{
+    ProcBin *pb;
+
+    if (!ERTS_TERM_IS_MAGIC_BINARY(term)) return NULL;
+
+    pb = (ProcBin*) binary_val(term);
+    return hipe_get_loader_state(pb->val);
+}
+
+
 /*
  * Allocate memory and copy machine code to it.
  */
-BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2)
+BIF_RETTYPE hipe_bifs_enter_code_3(BIF_ALIST_3)
 {
     Uint nrbytes;
     void *bytes;
     void *address;
     Eterm trampolines;
     Eterm *hp;
+    HipeLoaderState *stp;
 #ifndef DEBUG
     ERTS_DECLARE_DUMMY(Uint bitoffs);
     ERTS_DECLARE_DUMMY(Uint bitsize);
@@ -392,13 +409,15 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2)
     Uint bitsize;
 #endif
 
-    if (is_not_binary(BIF_ARG_1))
+    if (is_not_binary(BIF_ARG_1) ||
+	(!(stp = get_loader_state(BIF_ARG_3))))
 	BIF_ERROR(BIF_P, BADARG);
     nrbytes = binary_size(BIF_ARG_1);
     ERTS_GET_BINARY_BYTES(BIF_ARG_1, bytes, bitoffs, bitsize);
     ASSERT(bitoffs == 0);
     ASSERT(bitsize == 0);
     trampolines = NIL;
+    // XXX: Trampolines are not tracked and freed
     address = hipe_alloc_code(nrbytes, BIF_ARG_2, &trampolines, BIF_P);
     if (!address) {
 	Uint nrcallees;
@@ -407,11 +426,15 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2)
 	    nrcallees = arityval(tuple_val(BIF_ARG_2)[0]);
 	else
 	    nrcallees = 0;
+	// XXX: Is there any reason to not just BIF_ERROR, so that the runtime
+	// survives?
 	erts_exit(ERTS_ERROR_EXIT, "%s: failed to allocate %lu bytes and %lu trampolines\r\n",
 		 __func__, (unsigned long)nrbytes, (unsigned long)nrcallees);
     }
     memcpy(address, bytes, nrbytes);
     hipe_flush_icache_range(address, nrbytes);
+    stp->text_segment = address;
+    stp->text_segment_size = nrbytes;
     hp = HAlloc(BIF_P, 3);
     hp[0] = make_arityval(2);
     hp[1] = address_to_term(address, BIF_P);
@@ -424,25 +447,34 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2)
 /*
  * Allocate memory for arbitrary non-Erlang data.
  */
-BIF_RETTYPE hipe_bifs_alloc_data_2(BIF_ALIST_2)
+BIF_RETTYPE hipe_bifs_alloc_data_3(BIF_ALIST_3)
 {
-    Uint align, nrbytes;
-    void *block;
+    Uint align;
+    HipeLoaderState *stp;
 
     if (is_not_small(BIF_ARG_1) || is_not_small(BIF_ARG_2) ||
+	(!(stp = get_loader_state(BIF_ARG_3))) ||
 	(align = unsigned_val(BIF_ARG_1), !IS_POWER_OF_TWO(align)))
 	BIF_ERROR(BIF_P, BADARG);
-    nrbytes = unsigned_val(BIF_ARG_2);
-    if (nrbytes == 0)
+
+    if (stp->data_segment_size || stp->data_segment)
+	BIF_ERROR(BIF_P, BADARG);
+
+    stp->data_segment_size = unsigned_val(BIF_ARG_2);
+    if (stp->data_segment_size == 0)
 	BIF_RET(make_small(0));
-    block = erts_alloc(ERTS_ALC_T_HIPE, nrbytes);
-    if ((unsigned long)block & (align-1)) {
-	fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte aligned\r\n",
-		__FUNCTION__, (unsigned long)nrbytes, block, (unsigned long)align);
-	erts_free(ERTS_ALC_T_HIPE, block);
+    stp->data_segment = erts_alloc(ERTS_ALC_T_HIPE, stp->data_segment_size);
+    if ((unsigned long)stp->data_segment & (align-1)) {
+	fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte "
+		"aligned\r\n",
+		__FUNCTION__, (unsigned long)stp->data_segment_size,
+		stp->data_segment, (unsigned long)align);
+	erts_free(ERTS_ALC_T_HIPE, stp->data_segment);
+	stp->data_segment = NULL;
+	stp->data_segment_size = 0;
 	BIF_ERROR(BIF_P, EXC_NOTSUP);
     }
-    BIF_RET(address_to_term(block, BIF_P));
+    BIF_RET(address_to_term(stp->data_segment, BIF_P));
 }
 
 /*
@@ -1711,12 +1743,14 @@ void hipe_purge_module(Module* modp)
                 prevp = &p->next_in_mod;
         }
     }
-    if (modp->old.hipe_code_start) {
+    if (modp->old.hipe_code && modp->old.hipe_code->text_segment) {
 #ifdef DEBUG
-        sys_memset(modp->old.hipe_code_start, 0xfe, modp->old.hipe_code_size);
+        sys_memset(modp->old.hipe_code->text_segment, 0xfe,
+                   modp->old.hipe_code->text_segment_size);
 #endif
-        hipe_free_code(modp->old.hipe_code_start);
-        modp->old.hipe_code_start = NULL;
+        hipe_free_code(modp->old.hipe_code->text_segment);
+        modp->old.hipe_code->text_segment = NULL;
+        modp->old.hipe_code->text_segment_size = 0;
     }
 }
 
@@ -1870,3 +1904,23 @@ BIF_RETTYPE hipe_bifs_patch_call_3(BIF_ALIST_3)
 	BIF_ERROR(BIF_P, BADARG);
     BIF_RET(NIL);
 }
+
+BIF_RETTYPE hipe_bifs_alloc_loader_state_1(BIF_ALIST_1)
+{
+    Binary *magic;
+    Eterm *hp;
+    Eterm res;
+
+    if (is_not_atom(BIF_ARG_1))
+	BIF_ERROR(BIF_P, BADARG);
+
+    magic = hipe_alloc_loader_state(BIF_ARG_1);
+
+    if (!magic)
+	BIF_ERROR(BIF_P, BADARG);
+
+    hp = HAlloc(BIF_P, PROC_BIN_SIZE);
+    res = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), magic);
+    erts_refc_dec(&magic->refc, 1);
+    BIF_RET(res);
+}
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index ff9c1d06fb..1e4bde5ef2 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -44,8 +44,8 @@ bif hipe_bifs:ref/1
 bif hipe_bifs:ref_get/1
 bif hipe_bifs:ref_set/2
 
-bif hipe_bifs:enter_code/2
-bif hipe_bifs:alloc_data/2
+bif hipe_bifs:enter_code/3
+bif hipe_bifs:alloc_data/3
 bif hipe_bifs:constants_size/0
 bif hipe_bifs:merge_term/1
 
@@ -83,6 +83,8 @@ bif hipe_bifs:patch_call/3
 bif hipe_bifs:add_ref/2
 bif hipe_bifs:remove_refs_from/1
 
+bif hipe_bifs:alloc_loader_state/1
+
 # atoms used by add_ref/2
 atom call
 atom load_mfa
diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
new file mode 100644
index 0000000000..5bce3f1aee
--- /dev/null
+++ b/erts/emulator/hipe/hipe_load.c
@@ -0,0 +1,101 @@
+/*
+ * %CopyrightBegin%
+
+ *
+ * Copyright Ericsson AB 2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * hipe_load.c
+ *
+ * HiPE atomic code loader
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include "sys.h"
+#include "global.h"
+#include "erl_binary.h"
+#include "hipe_load.h"
+
+/*
+ * This destructor function can safely be called multiple times.
+ */
+static void
+hipe_loader_state_dtor(Binary* magic)
+{
+    HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic);
+    if (stp->module == NIL) return;
+
+    erts_fprintf(stderr, "Destroying HiPE loader state for module %T\n",
+		 stp->module);
+
+    // TODO: Needs to be freed separately. We'd like have a unified executable
+    // code allocator, so postpone this for now.
+    /* if (stp->text_segment) */
+    /* 	erts_free(ERTS_ALC_T_HIPE, stp->text_segment); */
+    stp->text_segment = NULL;
+    stp->text_segment_size = 0;
+
+    if (stp->data_segment)
+	erts_free(ERTS_ALC_T_HIPE, stp->data_segment);
+    stp->data_segment = NULL;
+    stp->data_segment_size = 0;
+
+    stp->module = NIL;
+}
+
+Binary *hipe_alloc_loader_state(Eterm module)
+{
+    HipeLoaderState *stp;
+    Binary *magic;
+
+    if (is_not_atom(module)) return NULL;
+
+    erts_fprintf(stderr, "Creating HiPE loader state for module %T\n", module);
+
+    magic = erts_create_magic_binary(sizeof(HipeLoaderState),
+				     hipe_loader_state_dtor);
+    erts_refc_inc(&magic->refc, 1);
+    stp = ERTS_MAGIC_BIN_DATA(magic);
+
+    stp->module = module;
+    stp->text_segment = NULL;
+    stp->text_segment_size = 0;
+    stp->data_segment = NULL;
+    stp->data_segment_size = 0;
+
+    return magic;
+}
+
+void hipe_free_loader_state(Binary *magic)
+{
+    if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != hipe_loader_state_dtor)
+	return;
+
+    /* Why does BEAM do a refc_dec here? What is holding that reference? */
+    hipe_loader_state_dtor(magic);
+}
+
+HipeLoaderState *
+hipe_get_loader_state(Binary *magic)
+{
+    if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != hipe_loader_state_dtor)
+	return NULL;
+
+    return (HipeLoaderState*) ERTS_MAGIC_BIN_DATA(magic);
+}
diff --git a/erts/emulator/hipe/hipe_load.h b/erts/emulator/hipe/hipe_load.h
new file mode 100644
index 0000000000..9e24e915ed
--- /dev/null
+++ b/erts/emulator/hipe/hipe_load.h
@@ -0,0 +1,45 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * hipe_load.h
+ *
+ * HiPE atomic code loader
+ */
+#ifndef HIPE_LOAD_H
+#define HIPE_LOAD_H
+
+#include "global.h"
+
+typedef struct hipe_loader_state {
+    Eterm module;                   /* Module name, atom */
+
+    void *text_segment;
+    Uint text_segment_size;
+
+    void *data_segment;
+    Uint data_segment_size;
+
+} HipeLoaderState;
+
+extern Binary *hipe_alloc_loader_state(Eterm module);
+extern void hipe_free_loader_state(Binary *binary);
+extern HipeLoaderState *hipe_get_loader_state(Binary *binary);
+
+#endif /* HIPE_LOAD_H */
diff --git a/erts/emulator/hipe/hipe_module.c b/erts/emulator/hipe/hipe_module.c
new file mode 100644
index 0000000000..1553bf9087
--- /dev/null
+++ b/erts/emulator/hipe/hipe_module.c
@@ -0,0 +1,37 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include "sys.h"
+#include "hipe_module.h"
+
+void hipe_free_module(HipeModule *mod)
+{
+#ifdef DEBUG
+    sys_memzero(mod->text_segment, mod->text_segment_size);
+#endif
+    /* XXX: erts_free(ERTS_ALC_T_HIPE, mod->text_segment); */
+    if (mod->data_segment) /* Some modules lack data segments */
+	erts_free(ERTS_ALC_T_HIPE, mod->data_segment);
+
+    erts_free(ERTS_ALC_T_HIPE, mod);
+}
diff --git a/erts/emulator/hipe/hipe_module.h b/erts/emulator/hipe/hipe_module.h
new file mode 100644
index 0000000000..1e1302fc0a
--- /dev/null
+++ b/erts/emulator/hipe/hipe_module.h
@@ -0,0 +1,42 @@
+/*
+ * %CopyrightBegin%
+
+ *
+ * Copyright Ericsson AB 2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * hipe_module.h
+ *
+ *
+ */
+#ifndef HIPE_MODULE_H
+#define HIPE_MODULE_H
+
+/* Forward-declare type to resolve circular dependency with module.h */
+typedef struct hipe_module HipeModule;
+
+#include "global.h"
+
+struct hipe_module {
+    void *text_segment;
+    Uint text_segment_size;
+    void *data_segment;
+};
+
+extern void hipe_free_module(HipeModule *mod);
+
+#endif /* HIPE_MODULE_H */
-- 
cgit v1.2.3


From 5e879fb1d4a9273f0a6183da7509dcdbc24f9653 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Magnus=20L=C3=A5ng?= <margnus1@telia.com>
Date: Fri, 16 Sep 2016 19:33:38 +0200
Subject: erts: Check hipe stack in check_process_code

This is part of commit 1bd508921dd93086b05e7d0038b816b36c421d86.
I did not include the fun-checking as we have a new purge strategy
for funs in OTP 20. That remains to be solved some other way for hipe.
---
 erts/emulator/beam/beam_bif_load.c | 26 ++++++++++++++++-
 erts/emulator/hipe/hipe_gc.c       | 33 ++++++++++++++++++++++
 erts/emulator/hipe/hipe_stack.h    |  2 ++
 erts/emulator/hipe/hipe_x86_gc.h   |  1 +
 erts/emulator/test/hipe_SUITE.erl  | 57 ++++++++++++++++++++++++++++++++++++--
 5 files changed, 116 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 3302de0787..ec1db8d9c2 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1069,6 +1069,8 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls)
     BeamInstr* start;
     char* mod_start;
     Uint mod_size;
+    void *nat_start = NULL;
+    Uint nat_size = 0;
     Eterm* sp;
 
     *redsp += 1;
@@ -1099,6 +1101,20 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls)
 	}
     }
 
+#ifdef HIPE
+    /*
+     * Check all continuation pointers stored on the native stack if the module
+     * has native code.
+     */
+    if (modp->old.hipe_code) {
+	nat_start = modp->old.hipe_code->text_segment;
+	nat_size = modp->old.hipe_code->text_segment_size;
+	if (nat_size && nstack_any_cps_in_segment(rp, nat_start, nat_size)) {
+	    return am_true;
+	}
+    }
+#endif
+
     /* 
      * Check all continuation pointers stored in stackdump
      * and clear exception stackdump if there is a pointer
@@ -1115,8 +1131,16 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls)
 	    rp->ftrace = NIL;
 	} else {
 	    int i;
+	    char *area_start = mod_start;
+	    Uint area_size = mod_size;
+#ifdef HIPE
+	    if (rp->freason & EXF_NATIVE) {
+		area_start = nat_start;
+		area_size = nat_size;
+	    }
+#endif
 	    for (i = 0;  i < s->depth;  i++) {
-		if (ErtsInArea(s->trace[i], mod_start, mod_size)) {
+		if (ErtsInArea(s->trace[i], area_start, area_size)) {
 		    rp->freason = EXC_NULL;
 		    rp->fvalue = NIL;
 		    rp->ftrace = NIL;
diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c
index be665ccdb9..e6ce7ce628 100644
--- a/erts/emulator/hipe/hipe_gc.c
+++ b/erts/emulator/hipe/hipe_gc.c
@@ -356,3 +356,36 @@ nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size)
     }
     return 0;
 }
+
+int
+nstack_any_cps_in_segment(Process *p, char* seg_start, Uint seg_size)
+{
+    Eterm *nsp;
+    Eterm *nsp_end;
+    const struct hipe_sdesc *sdesc;
+    /* arch-specific nstack walk state */
+    struct nstack_walk_state walk_state;
+
+    if (!p->hipe.nstack || !nstack_walk_init_check(p))
+	return 0;
+    ASSERT(p->hipe.nsp && p->hipe.nstend);
+    nsp = nstack_walk_nsp_begin(p);
+    nsp_end = nstack_walk_nsp_end(p);
+    sdesc = nstack_walk_init_sdesc_ignore_trap(p, &walk_state);
+
+    /* Check the topmost frame */
+    if (ErtsInArea(sdesc->bucket.hvalue, seg_start, seg_size))
+	return 1;
+
+    while (!nstack_walk_nsp_reached_end(nsp, nsp_end)) {
+	unsigned sdesc_size = nstack_walk_frame_size(sdesc);
+	unsigned long ra = nstack_walk_frame_ra(nsp, sdesc);
+	if (ra == (unsigned long)nbif_stack_trap_ra)
+	    ra = (unsigned long)p->hipe.ngra;
+	if (ErtsInArea(ra, seg_start, seg_size))
+            return 1;
+        sdesc = hipe_find_sdesc(ra);
+        nsp = nstack_walk_next_frame(nsp, sdesc_size);
+    }
+    return 0;
+}
diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h
index 1863b0db8c..537755fefa 100644
--- a/erts/emulator/hipe/hipe_stack.h
+++ b/erts/emulator/hipe/hipe_stack.h
@@ -138,5 +138,7 @@ extern void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop
 extern Eterm *sweep_literals_nstack(Process *p, Eterm *n_htop, char *area,
 				    Uint area_size);
 extern int nstack_any_heap_ref_ptrs(Process *, char* mod_start, Uint mod_size);
+extern int nstack_any_cps_in_segment(Process *, char* seg_start, Uint seg_size);
+
 
 #endif /* HIPE_STACK_H */
diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h
index e92e6ea718..a703e24b8c 100644
--- a/erts/emulator/hipe/hipe_x86_gc.h
+++ b/erts/emulator/hipe/hipe_x86_gc.h
@@ -65,6 +65,7 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state)
     state->sdesc0 = sdesc;
     return sdesc;
 #else
+    state->sdesc0[0].bucket.hvalue = 0; /* for nstack_any_cps_in_segment */
     state->sdesc0[0].fsize = 0;
     state->sdesc0[0].has_exnra = 0;
     state->sdesc0[0].stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 :
diff --git a/erts/emulator/test/hipe_SUITE.erl b/erts/emulator/test/hipe_SUITE.erl
index a556b4ddc0..0b44dd7fb7 100644
--- a/erts/emulator/test/hipe_SUITE.erl
+++ b/erts/emulator/test/hipe_SUITE.erl
@@ -19,12 +19,17 @@
 %%
 
 -module(hipe_SUITE).
--export([all/0, t_copy_literals/1]).
+-export([all/0
+	,t_copy_literals/1
+	,t_purge/1
+	]).
 
 all() ->
     case erlang:system_info(hipe_architecture) of
 	undefined -> {skip, "HiPE is disabled"};
-	_ -> [t_copy_literals]
+	_ -> [t_copy_literals
+	     ,t_purge
+	     ]
     end.
 
 t_copy_literals(doc) ->
@@ -65,3 +70,51 @@ t_copy_literals(Config) when is_list(Config) ->
     true = erlang:delete_module(ref_cell),
     true = erlang:purge_module(ref_cell),
     ok.
+
+t_purge(doc) -> "Checks that native code is properly found and purged";
+t_purge(Config) when is_list(Config) ->
+    Data = proplists:get_value(data_dir, Config),
+    Priv = proplists:get_value(priv_dir, Config),
+    SrcFile = filename:join(Data, "ref_cell"),
+    BeamFile = filename:join(Priv, "ref_cell"),
+    {ok,ref_cell} = c:c(SrcFile, [{outdir,Priv},native]),
+    true = code:is_module_native(ref_cell),
+
+    PA = ref_cell:start_link(),
+
+    %% Unload, PA should still be running
+    true = erlang:delete_module(ref_cell),
+    %% Can't use ref_cel:call/2, it's in old code!
+    call(PA, {put_res_of, fun()-> hej end}),
+    hej = call(PA, get),
+
+    %% Load same module again
+    code:load_abs(BeamFile),
+    true = code:is_module_native(ref_cell),
+    PB = ref_cell:start_link(),
+
+    %% Purge old code, PA should be killed, PB should survive
+    unlink(PA),
+    ARef = monitor(process, PA),
+    true = erlang:purge_module(ref_cell),
+    receive {'DOWN', ARef, process, PA, killed} -> ok
+    after 1 -> ct:fail("PA was not killed")
+    end,
+
+    %% Unload, PB should still be running
+    true = erlang:delete_module(ref_cell),
+    call(PB, {put_res_of, fun()-> svejs end}),
+    svejs = call(PB, get),
+
+    unlink(PB),
+    BRef = monitor(process, PB),
+    true = erlang:purge_module(ref_cell),
+    receive {'DOWN', BRef, process, PB, killed} -> ok
+    after 1 -> ct:fail("PB was not killed")
+    end,
+
+    ok.
+
+call(Pid, Call) ->
+    Pid ! {Call, self()},
+    receive {Pid, Res} -> Res end.
-- 
cgit v1.2.3


From a82be01198c1f3714f29f0a0e3ce3ed6d6bce0a6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 16 Sep 2016 19:39:24 +0200
Subject: erts: Fill freed EXEC memory with illegal instructions on DEBUG

to get a nice SIGILL crash.
For x86 and x86_64 only.
---
 erts/emulator/beam/erl_alloc.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 3c2c9def3b..d96b90fd55 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -4129,12 +4129,20 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr)
     ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
     void *dptr;
     Uint size;
+    int free_pattern = n;
 
     ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
 
     dptr = check_memory_fence(ptr, &size, n, ERTS_ALC_O_FREE);
 
-    sys_memset((void *) dptr, n, size + FENCE_SZ);
+#ifdef ERTS_ALC_A_EXEC
+# if defined(__i386__) || defined(__x86_64__)
+    if (ERTS_ALC_T2A(ERTS_ALC_N2T(n)) == ERTS_ALC_A_EXEC) {
+        free_pattern = 0x0f; /* Illegal instruction */
+    }
+# endif
+#endif
+    sys_memset((void *) dptr, free_pattern, size + FENCE_SZ);
 
     (*real_af->free)(n, real_af->extra, dptr);
 
-- 
cgit v1.2.3


From e36c8d49359db1622ad381f705c54d460c4fb5e9 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 16 Sep 2016 20:01:11 +0200
Subject: erts: Use hipe_free_module

---
 erts/emulator/hipe/hipe_amd64.c  |  5 +++--
 erts/emulator/hipe/hipe_arch.h   |  4 ++--
 erts/emulator/hipe/hipe_bif0.c   | 11 +++--------
 erts/emulator/hipe/hipe_module.c |  6 ++----
 4 files changed, 10 insertions(+), 16 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c
index df53f4db30..96b8c22265 100644
--- a/erts/emulator/hipe/hipe_amd64.c
+++ b/erts/emulator/hipe/hipe_amd64.c
@@ -130,10 +130,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
     return alloc_code(nrbytes);
 }
 
-void hipe_free_code(void* code)
+void hipe_free_code(void* code, unsigned int bytes)
 {
     ALLOC_CODE_STATS(--nr_allocs);
-    /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/
+    ALLOC_CODE_STATS(total_alloc -= bytes);
+
     erts_free(ERTS_ALC_T_HIPE_EXEC, code);
 }
 
diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h
index df38a80069..059b8e7f29 100644
--- a/erts/emulator/hipe/hipe_arch.h
+++ b/erts/emulator/hipe/hipe_arch.h
@@ -30,8 +30,8 @@ extern void hipe_patch_load_fe(Uint *address, Uint value);
 extern int hipe_patch_insn(void *address, Uint value, Eterm type);
 extern int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline);
 
-extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p);
-extern void  hipe_free_code(void*);
+extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, struct process *p);
+extern void  hipe_free_code(void*, unsigned int);
 extern void *hipe_make_native_stub(void *exp, unsigned int beamArity);
 extern void  hipe_free_native_stub(void*);
 
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index daa198d695..46d4378f21 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1743,14 +1743,9 @@ void hipe_purge_module(Module* modp)
                 prevp = &p->next_in_mod;
         }
     }
-    if (modp->old.hipe_code && modp->old.hipe_code->text_segment) {
-#ifdef DEBUG
-        sys_memset(modp->old.hipe_code->text_segment, 0xfe,
-                   modp->old.hipe_code->text_segment_size);
-#endif
-        hipe_free_code(modp->old.hipe_code->text_segment);
-        modp->old.hipe_code->text_segment = NULL;
-        modp->old.hipe_code->text_segment_size = 0;
+    if (modp->old.hipe_code) {
+        hipe_free_module(modp->old.hipe_code);
+        modp->old.hipe_code = NULL;
     }
 }
 
diff --git a/erts/emulator/hipe/hipe_module.c b/erts/emulator/hipe/hipe_module.c
index 1553bf9087..469f077dd2 100644
--- a/erts/emulator/hipe/hipe_module.c
+++ b/erts/emulator/hipe/hipe_module.c
@@ -22,14 +22,12 @@
 #include "config.h"
 #endif
 #include "sys.h"
+#include "hipe_arch.h"
 #include "hipe_module.h"
 
 void hipe_free_module(HipeModule *mod)
 {
-#ifdef DEBUG
-    sys_memzero(mod->text_segment, mod->text_segment_size);
-#endif
-    /* XXX: erts_free(ERTS_ALC_T_HIPE, mod->text_segment); */
+    hipe_free_code(mod->text_segment, mod->text_segment_size);
     if (mod->data_segment) /* Some modules lack data segments */
 	erts_free(ERTS_ALC_T_HIPE, mod->data_segment);
 
-- 
cgit v1.2.3


From 966098ceb9dd9d18e9bcd37cd06b96045903e320 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 20 Sep 2016 16:16:50 +0200
Subject: erts: Move new hipe ref and sdesc lists to loader state

---
 erts/emulator/beam/beam_bif_load.c |   1 -
 erts/emulator/beam/beam_load.c     |  11 +--
 erts/emulator/beam/module.c        |   6 --
 erts/emulator/beam/module.h        |   4 -
 erts/emulator/hipe/hipe_bif0.c     | 150 +++++++++++++++++++++----------------
 erts/emulator/hipe/hipe_bif0.tab   |   2 +-
 erts/emulator/hipe/hipe_load.c     |   3 +
 erts/emulator/hipe/hipe_load.h     |   3 +
 erts/emulator/hipe/hipe_module.h   |   3 +
 9 files changed, 100 insertions(+), 83 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index ec1db8d9c2..f5d1ca7e54 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1798,7 +1798,6 @@ delete_code(Module* modp)
 
     ASSERT(modp->curr.num_breakpoints == 0);
     ASSERT(modp->curr.num_traced_exports == 0);
-    DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "delete_code old.first_hipe_ref=%p", modp->curr.first_hipe_ref);
     modp->old = modp->curr;
     erts_module_instance_init(&modp->curr);
 }
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 7d29f393e5..0eb390bf4c 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -1126,11 +1126,8 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
     modp->curr.code_length = size;
     modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */
 #if defined(HIPE)
-    DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code new_hipe_refs = %p", modp->new_hipe_refs);
-    modp->curr.first_hipe_ref = modp->new_hipe_refs;
-    modp->curr.first_hipe_sdesc = modp->new_hipe_sdesc;
-    modp->new_hipe_refs = NULL;
-    modp->new_hipe_sdesc = NULL;
+    DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code "
+                  "first_hipe_ref = %p", hipe_code->first_hipe_ref);
     modp->curr.hipe_code = hipe_code;
 #endif
 
@@ -6498,6 +6495,8 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
     hipe_code->text_segment = hipe_stp->text_segment;
     hipe_code->text_segment_size = hipe_stp->text_segment_size;
     hipe_code->data_segment = hipe_stp->data_segment;
+    hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs;
+    hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc;
 
     /*
      * Insert the module in the module table.
@@ -6524,6 +6523,8 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
 	/* Prevent code from being freed */
 	hipe_stp->text_segment = 0;
 	hipe_stp->data_segment = 0;
+        hipe_stp->new_hipe_refs = NULL;
+        hipe_stp->new_hipe_sdesc = NULL;
 
 	erts_free_aligned_binary_bytes(temp_alloc);
 	free_loader_state(magic);
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index 5c56ad20df..e9b18ba525 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -77,8 +77,6 @@ void erts_module_instance_init(struct erl_module_instance* modi)
     modi->num_breakpoints = 0;
     modi->num_traced_exports = 0;
 #ifdef HIPE
-    modi->first_hipe_ref = NULL;
-    modi->first_hipe_sdesc = NULL;
     modi->hipe_code = NULL;
 #endif
 }
@@ -95,8 +93,6 @@ static Module* module_alloc(Module* tmpl)
     obj->on_load = 0;
 #ifdef HIPE
     obj->first_hipe_mfa = NULL;
-    obj->new_hipe_refs = NULL;
-    obj->new_hipe_sdesc = NULL;
 #endif
     DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc");
     return obj;
@@ -214,8 +210,6 @@ static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod)
     dst_mod->on_load = src_mod->on_load;
 #ifdef HIPE
     dst_mod->first_hipe_mfa = src_mod->first_hipe_mfa;
-    dst_mod->new_hipe_refs  = src_mod->new_hipe_refs;
-    dst_mod->new_hipe_sdesc = src_mod->new_hipe_sdesc;
 #endif
 }
 
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index c3adaf12fc..9de610ccf7 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -35,8 +35,6 @@ struct erl_module_instance {
     int num_breakpoints;
     int num_traced_exports;
 #ifdef HIPE
-    struct hipe_ref* first_hipe_ref;  /* all external hipe calls from this module */
-    struct hipe_sdesc* first_hipe_sdesc;  /* all stack descriptors for this module */
     HipeModule *hipe_code;
 #endif
 };
@@ -51,8 +49,6 @@ typedef struct erl_module {
     struct erl_module_instance* on_load;
 #ifdef HIPE
     struct hipe_mfa_info* first_hipe_mfa;
-    struct hipe_ref* new_hipe_refs;
-    struct hipe_sdesc* new_hipe_sdesc;
 #endif
 } Module; 
 
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 46d4378f21..da895e4e8b 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -685,12 +685,17 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3)
     BIF_RET(am_false);
 }
 
-BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1)
+BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2)
 {
     struct hipe_sdesc *sdesc;
+    HipeLoaderState* stp;
     Module* modp;
     int do_commit;
 
+    stp = get_loader_state(BIF_ARG_2);
+    if (!stp)
+	BIF_ERROR(BIF_P, BADARG);
+
     sdesc = hipe_decode_sdesc(BIF_ARG_1, &do_commit);
     if (!sdesc) {
 	fprintf(stderr, "%s: bad sdesc!\r\n", __FUNCTION__);
@@ -707,12 +712,13 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1)
     modp = erts_put_active_module(make_atom(sdesc->m_aix));
     ASSERT(modp);
     if (do_commit) { /* Direct "hipe-patching" of early loaded module */
-        sdesc->next_in_modi = modp->curr.first_hipe_sdesc;
-        modp->curr.first_hipe_sdesc = sdesc;
+        ASSERT(modp->curr.hipe_code);
+        sdesc->next_in_modi = modp->curr.hipe_code->first_hipe_sdesc;
+        modp->curr.hipe_code->first_hipe_sdesc = sdesc;
     }
     else {           /* Normal module loading/upgrade */
-        sdesc->next_in_modi = modp->new_hipe_sdesc;
-        modp->new_hipe_sdesc = sdesc;
+        sdesc->next_in_modi = stp->new_hipe_sdesc;
+        stp->new_hipe_sdesc = sdesc;
     }
 
     BIF_RET(NIL);
@@ -1491,13 +1497,14 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
     struct hipe_ref *ref;
     Module* modp;
     int do_commit;
+    HipeLoaderState* stp;
 
     if (!term_to_mfa(BIF_ARG_1, &callee))
 	goto badarg;
     if (is_not_tuple(BIF_ARG_2))
 	goto badarg;
     tuple = tuple_val(BIF_ARG_2);
-    if (tuple[0] != make_arityval(5))
+    if (tuple[0] != make_arityval(6))
 	goto badarg;
     if (!term_to_mfa(tuple[1], &caller))
 	goto badarg;
@@ -1526,6 +1533,10 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
     case am_false: do_commit = 0; break;
     default: goto badarg;
     }
+    stp = get_loader_state(tuple[6]);
+    if (!stp)
+        goto badarg;
+
     hipe_mfa_info_table_rwlock();
     callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari);
 
@@ -1550,12 +1561,13 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
     modp = erts_put_active_module(caller.mod);
     ASSERT(modp);
     if (do_commit) { /* Direct "hipe-patching" of early loaded module */
-        ref->next_from_modi = modp->curr.first_hipe_ref;
-        modp->curr.first_hipe_ref = ref;
+        ASSERT(modp->curr.hipe_code);
+        ref->next_from_modi = modp->curr.hipe_code->first_hipe_ref;
+        modp->curr.hipe_code->first_hipe_ref = ref;
     }
     else {           /* Normal module loading/upgrade */
-        ref->next_from_modi = modp->new_hipe_refs;
-        modp->new_hipe_refs = ref;
+        ref->next_from_modi = stp->new_hipe_refs;
+        stp->new_hipe_refs = ref;
     }
 
 #if defined(DEBUG)
@@ -1653,9 +1665,12 @@ BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1)
 int hipe_purge_need_blocking(Module* modp)
 {
     /* SVERK: Verify if this is really necessary */
-    return (modp->old.first_hipe_ref ||
-            modp->old.first_hipe_sdesc ||
-            (!modp->curr.code_hdr && modp->first_hipe_mfa));
+    if (modp->old.hipe_code) {
+        if (modp->old.hipe_code->first_hipe_ref ||
+            modp->old.hipe_code->first_hipe_sdesc)
+            return 1;
+    }
+    return !modp->curr.code_hdr && modp->first_hipe_mfa;
 }
 
 void hipe_purge_module(Module* modp)
@@ -1667,68 +1682,75 @@ void hipe_purge_module(Module* modp)
 
     DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module");
 
-    /*
-     * Remove all hipe_ref's (external calls) from the old module instance
-     */
-    ref = modp->old.first_hipe_ref;
-
-    while (ref) {
-	struct hipe_ref* free_ref = ref;
-
-        ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
-
-	DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref,
-		      ref->callee->m, ref->callee->f, ref->callee->a);
-	DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref,
-		      ref->caller_m, ref->caller_f, ref->caller_a);
-	ASSERT(ref->caller_m == make_atom(modp->module));
-
+    if (modp->old.hipe_code) {
         /*
-         * Unlink from other refs to same callee
+         * Remove all hipe_ref's (external calls) from the old module instance
          */
-        ASSERT(ref->head.next->prev == &ref->head);
-        ASSERT(ref->head.prev->next == &ref->head);
-        ASSERT(ref->head.next != &ref->head);
-        ASSERT(ref->head.prev != &ref->head);
-        ref->head.next->prev = ref->head.prev;
-        ref->head.prev->next = ref->head.next;
+        ref = modp->old.hipe_code->first_hipe_ref;
+
+        while (ref) {
+            struct hipe_ref* free_ref = ref;
+
+            ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+            DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref,
+                          ref->callee->m, ref->callee->f, ref->callee->a);
+            DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref,
+                          ref->caller_m, ref->caller_f, ref->caller_a);
+            ASSERT(ref->caller_m == make_atom(modp->module));
+
+            /*
+             * Unlink from other refs to same callee
+             */
+            ASSERT(ref->head.next->prev == &ref->head);
+            ASSERT(ref->head.prev->next == &ref->head);
+            ASSERT(ref->head.next != &ref->head);
+            ASSERT(ref->head.prev != &ref->head);
+            ref->head.next->prev = ref->head.prev;
+            ref->head.prev->next = ref->head.next;
+
+            /*
+             * Was this the last ref to that callee?
+             */
+            if (ref->head.next == ref->head.prev) {
+                struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers);
+                if (p->is_stub) {
+                    unlink_mfa_from_mod(p);
+                    purge_mfa(p);
+               }
+            }
 
-        /*
-         * Was this the last ref to that callee?
-         */
-        if (ref->head.next == ref->head.prev) {
-            struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers);
-            if (p->is_stub) {
-                unlink_mfa_from_mod(p);
-                purge_mfa(p);
-           }
+            ref = ref->next_from_modi;
+            erts_free(ERTS_ALC_T_HIPE, free_ref);
         }
+        modp->old.hipe_code->first_hipe_ref = NULL;
 
-	ref = ref->next_from_modi;
-	erts_free(ERTS_ALC_T_HIPE, free_ref);
-    }
-    modp->old.first_hipe_ref = NULL;
+        /*
+         * Remove all hipe_sdesc's for the old module instance
+         */
+        sdesc = modp->old.hipe_code->first_hipe_sdesc;
 
-    /*
-     * Remove all hipe_sdesc's for the old module instance
-     */
-    sdesc = modp->old.first_hipe_sdesc;
+        while (sdesc) {
+            struct hipe_sdesc* free_sdesc = sdesc;
 
-    while (sdesc) {
-	struct hipe_sdesc* free_sdesc = sdesc;
+            ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
 
-        ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+            DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue);
+            ASSERT(sdesc->m_aix == modp->module);
 
-	DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue);
-	ASSERT(sdesc->m_aix == modp->module);
+            sdesc = sdesc->next_in_modi;
+            hipe_destruct_sdesc(free_sdesc);
+        }
+        modp->old.hipe_code->first_hipe_sdesc = NULL;
 
-	sdesc = sdesc->next_in_modi;
-        hipe_destruct_sdesc(free_sdesc);
+        hipe_free_module(modp->old.hipe_code);
+        modp->old.hipe_code = NULL;
     }
-    modp->old.first_hipe_sdesc = NULL;
+
 
     /*
-     * Remove unreferred hipe_mfa_info's
+     * Remove unreferred hipe_mfa_info's 
+     * when all module instances are removed (like in init:restart) 
      */
     if (modp->curr.code_hdr == NULL) {
         struct hipe_mfa_info** prevp = &modp->first_hipe_mfa;
@@ -1743,10 +1765,6 @@ void hipe_purge_module(Module* modp)
                 prevp = &p->next_in_mod;
         }
     }
-    if (modp->old.hipe_code) {
-        hipe_free_module(modp->old.hipe_code);
-        modp->old.hipe_code = NULL;
-    }
 }
 
 
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 1e4bde5ef2..8475b8ce76 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -56,7 +56,7 @@ bif hipe_bifs:set_native_address/3
 bif hipe_bifs:set_funinfo_native_address/3
 #bif hipe_bifs:invalidate_funinfo_native_addresses/1
 
-bif hipe_bifs:enter_sdesc/1
+bif hipe_bifs:enter_sdesc/2
 
 bif hipe_bifs:bif_address/3
 bif hipe_bifs:primop_address/1
diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
index 5bce3f1aee..cebbbafdd3 100644
--- a/erts/emulator/hipe/hipe_load.c
+++ b/erts/emulator/hipe/hipe_load.c
@@ -79,6 +79,9 @@ Binary *hipe_alloc_loader_state(Eterm module)
     stp->data_segment = NULL;
     stp->data_segment_size = 0;
 
+    stp->new_hipe_refs = NULL;
+    stp->new_hipe_sdesc = NULL;
+
     return magic;
 }
 
diff --git a/erts/emulator/hipe/hipe_load.h b/erts/emulator/hipe/hipe_load.h
index 9e24e915ed..17dd3c3c05 100644
--- a/erts/emulator/hipe/hipe_load.h
+++ b/erts/emulator/hipe/hipe_load.h
@@ -36,6 +36,9 @@ typedef struct hipe_loader_state {
     void *data_segment;
     Uint data_segment_size;
 
+    struct hipe_ref* new_hipe_refs;
+    struct hipe_sdesc* new_hipe_sdesc;
+
 } HipeLoaderState;
 
 extern Binary *hipe_alloc_loader_state(Eterm module);
diff --git a/erts/emulator/hipe/hipe_module.h b/erts/emulator/hipe/hipe_module.h
index 1e1302fc0a..b489f567cb 100644
--- a/erts/emulator/hipe/hipe_module.h
+++ b/erts/emulator/hipe/hipe_module.h
@@ -35,6 +35,9 @@ struct hipe_module {
     void *text_segment;
     Uint text_segment_size;
     void *data_segment;
+
+    struct hipe_ref* first_hipe_ref;  /* all external hipe calls from this module */
+    struct hipe_sdesc* first_hipe_sdesc;  /* all stack descriptors for this module */
 };
 
 extern void hipe_free_module(HipeModule *mod);
-- 
cgit v1.2.3


From e77d7a8417368617c4c228af9556a7f6a8f3e84c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 28 Sep 2016 20:55:40 +0200
Subject: erts: Fix early hipe patch loading

by introducing hipe_bifs:commit_patch_load/1
that creates the HipeModule.
---
 erts/emulator/beam/beam_load.c   | 39 ++++++++++++++++++++++++++
 erts/emulator/beam/global.h      |  1 +
 erts/emulator/hipe/hipe_bif0.c   | 59 ++++++++++++++--------------------------
 erts/emulator/hipe/hipe_bif0.tab |  1 +
 erts/emulator/hipe/hipe_load.c   |  5 ++++
 erts/emulator/hipe/hipe_stack.c  | 10 ++-----
 erts/emulator/hipe/hipe_stack.h  |  2 +-
 7 files changed, 69 insertions(+), 48 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 0eb390bf4c..4a833689e3 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -6540,6 +6540,45 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
     BIF_ERROR(p, BADARG);
 }
 
+int erts_commit_hipe_patch_load(Eterm hipe_magic_bin)
+{
+    Binary* hipe_magic;
+    HipeLoaderState* hipe_stp;
+    HipeModule *hipe_code;
+    Module* modp;
+
+    if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) ||
+	!(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val,
+	  hipe_stp = hipe_get_loader_state(hipe_magic)) ||
+	hipe_stp->module == NIL || hipe_stp->text_segment == 0) {
+	return 0;
+    }
+
+    modp = erts_get_module(hipe_stp->module, erts_active_code_ix());
+    if (!modp)
+	return 0;
+
+    /*
+     * Initialise HiPE module
+     */
+    hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code));
+    hipe_code->text_segment = hipe_stp->text_segment;
+    hipe_code->text_segment_size = hipe_stp->text_segment_size;
+    hipe_code->data_segment = hipe_stp->data_segment;
+    hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs;
+    hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc;
+
+    modp->curr.hipe_code = hipe_code;
+
+    /* Prevent code from being freed */
+    hipe_stp->text_segment = 0;
+    hipe_stp->data_segment = 0;
+    hipe_stp->new_hipe_refs = NULL;
+    hipe_stp->new_hipe_sdesc = NULL;
+
+    return 1;
+}
+
 #undef WORDS_PER_FUNCTION
 
 static int safe_mul(UWord a, UWord b, UWord* resp)
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 93996e8b41..29a0bc8d0b 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1048,6 +1048,7 @@ void erts_set_current_function(FunctionInfo* fi, BeamInstr* current);
 Eterm erts_module_info_0(Process* p, Eterm module);
 Eterm erts_module_info_1(Process* p, Eterm module, Eterm what);
 Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info);
+int erts_commit_hipe_patch_load(Eterm hipe_magic_bin);
 
 /* beam_ranges.c */
 void erts_init_ranges(void);
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index da895e4e8b..67508e6f5d 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -649,6 +649,14 @@ BIF_RETTYPE hipe_bifs_fun_to_address_1(BIF_ALIST_1)
     BIF_RET(address_to_term(pc, BIF_P));
 }
 
+BIF_RETTYPE hipe_bifs_commit_patch_load_1(BIF_ALIST_1)
+{
+    if (!erts_commit_hipe_patch_load(BIF_ARG_1))
+        BIF_ERROR(BIF_P, BADARG);
+
+    BIF_RET(am_ok);
+}
+
 BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3)
 {
     Eterm *pc;
@@ -689,14 +697,12 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2)
 {
     struct hipe_sdesc *sdesc;
     HipeLoaderState* stp;
-    Module* modp;
-    int do_commit;
 
     stp = get_loader_state(BIF_ARG_2);
     if (!stp)
 	BIF_ERROR(BIF_P, BADARG);
 
-    sdesc = hipe_decode_sdesc(BIF_ARG_1, &do_commit);
+    sdesc = hipe_decode_sdesc(BIF_ARG_1);
     if (!sdesc) {
 	fprintf(stderr, "%s: bad sdesc!\r\n", __FUNCTION__);
 	BIF_ERROR(BIF_P, BADARG);
@@ -709,17 +715,8 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2)
     /*
      * Link into list of sdesc's in same module instance
      */
-    modp = erts_put_active_module(make_atom(sdesc->m_aix));
-    ASSERT(modp);
-    if (do_commit) { /* Direct "hipe-patching" of early loaded module */
-        ASSERT(modp->curr.hipe_code);
-        sdesc->next_in_modi = modp->curr.hipe_code->first_hipe_sdesc;
-        modp->curr.hipe_code->first_hipe_sdesc = sdesc;
-    }
-    else {           /* Normal module loading/upgrade */
-        sdesc->next_in_modi = stp->new_hipe_sdesc;
-        stp->new_hipe_sdesc = sdesc;
-    }
+    sdesc->next_in_modi = stp->new_hipe_sdesc;
+    stp->new_hipe_sdesc = sdesc;
 
     BIF_RET(NIL);
 }
@@ -1483,7 +1480,7 @@ int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a)
 }
 
 
-/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,DoCommit})
+/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,LoaderState})
  */
 BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
 {
@@ -1495,8 +1492,6 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
     unsigned int flags;
     struct hipe_mfa_info *callee_mfa;
     struct hipe_ref *ref;
-    Module* modp;
-    int do_commit;
     HipeLoaderState* stp;
 
     if (!term_to_mfa(BIF_ARG_1, &callee))
@@ -1504,7 +1499,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
     if (is_not_tuple(BIF_ARG_2))
 	goto badarg;
     tuple = tuple_val(BIF_ARG_2);
-    if (tuple[0] != make_arityval(6))
+    if (tuple[0] != make_arityval(5))
 	goto badarg;
     if (!term_to_mfa(tuple[1], &caller))
 	goto badarg;
@@ -1528,12 +1523,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
 	if (!trampoline)
 	    goto badarg;
     }
-    switch (tuple[5]) {
-    case am_true: do_commit = 1; break;
-    case am_false: do_commit = 0; break;
-    default: goto badarg;
-    }
-    stp = get_loader_state(tuple[6]);
+    stp = get_loader_state(tuple[5]);
     if (!stp)
         goto badarg;
 
@@ -1558,17 +1548,8 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
     /*
      * Link into list of refs from same module instance
      */
-    modp = erts_put_active_module(caller.mod);
-    ASSERT(modp);
-    if (do_commit) { /* Direct "hipe-patching" of early loaded module */
-        ASSERT(modp->curr.hipe_code);
-        ref->next_from_modi = modp->curr.hipe_code->first_hipe_ref;
-        modp->curr.hipe_code->first_hipe_ref = ref;
-    }
-    else {           /* Normal module loading/upgrade */
-        ref->next_from_modi = stp->new_hipe_refs;
-        stp->new_hipe_refs = ref;
-    }
+    ref->next_from_modi = stp->new_hipe_refs;
+    stp->new_hipe_refs = ref;
 
 #if defined(DEBUG)
     ref->callee = callee_mfa;
@@ -1578,10 +1559,10 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
 #endif
     hipe_mfa_info_table_rwunlock();
 
-    DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p) do_commit=%d",
-		  ref, callee.mod, callee.fun, callee.ari, ref->address, do_commit);
-    DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p) do_commit=%d",
-		  ref, caller.mod, caller.fun, caller.ari, ref->address, do_commit);
+    DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p)",
+		  ref, callee.mod, callee.fun, callee.ari, ref->address);
+    DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p)",
+		  ref, caller.mod, caller.fun, caller.ari, ref->address);
     BIF_RET(NIL);
 
  badarg:
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 8475b8ce76..264ea2c34a 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -50,6 +50,7 @@ bif hipe_bifs:constants_size/0
 bif hipe_bifs:merge_term/1
 
 bif hipe_bifs:fun_to_address/1
+bif hipe_bifs:commit_patch_load/1
 bif hipe_bifs:set_native_address/3
 #bif hipe_bifs:address_to_fun/1
 
diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
index cebbbafdd3..48f0cab293 100644
--- a/erts/emulator/hipe/hipe_load.c
+++ b/erts/emulator/hipe/hipe_load.c
@@ -57,6 +57,11 @@ hipe_loader_state_dtor(Binary* magic)
     stp->data_segment_size = 0;
 
     stp->module = NIL;
+
+    ASSERT(!stp->new_hipe_refs && !stp->new_hipe_sdesc);
+    /* ToDO: Purge lists 'new_hipe_refs' and 'new_hipe_sdesc'
+     *       if this is a failed load.
+     */
 }
 
 Binary *hipe_alloc_loader_state(Eterm module)
diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c
index 3d082668a6..b80e44bc37 100644
--- a/erts/emulator/hipe/hipe_stack.c
+++ b/erts/emulator/hipe/hipe_stack.c
@@ -143,7 +143,7 @@ void hipe_init_sdesc_table(struct hipe_sdesc *sdesc)
  * representation. If different representations are needed in
  * the future, this code has to be made target dependent.
  */
-struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp)
+struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
 {
     Uint ra, exnra;
     Eterm *live;
@@ -158,7 +158,7 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp)
         return 0;
 
     tp = tuple_val(arg);
-    if (tp[0] != make_arityval(7) ||
+    if (tp[0] != make_arityval(6) ||
         term_to_Uint(tp[1], &ra) == 0 ||
 	term_to_Uint(tp[2], &exnra) == 0 ||
 	is_not_small(tp[3]) ||
@@ -191,12 +191,6 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp)
 	    return 0;
     }
 
-    switch(tp[7]) {
-    case am_true: *do_commitp = 1; break;
-    case am_false: *do_commitp = 0; break;
-    default: return 0;
-    }
-
     /* Calculate number of words for the live bitmap. */
     livebitswords = (fsize + stk_nargs + 1 + 31) / 32;
     /* Calculate number of bytes needed for the stack descriptor. */
diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h
index 537755fefa..7e30358767 100644
--- a/erts/emulator/hipe/hipe_stack.h
+++ b/erts/emulator/hipe/hipe_stack.h
@@ -82,7 +82,7 @@ extern struct hipe_sdesc_table hipe_sdesc_table;
 extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*);
 extern void hipe_destruct_sdesc(struct hipe_sdesc*);
 extern void hipe_init_sdesc_table(struct hipe_sdesc*);
-extern struct hipe_sdesc *hipe_decode_sdesc(Eterm, int* do_commitp);
+extern struct hipe_sdesc *hipe_decode_sdesc(Eterm);
 
 #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
 #define __builtin_expect(x, expected_value) (x)
-- 
cgit v1.2.3


From 870d49f0f8d383e62ea231700089933c4cf8fe2a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 29 Sep 2016 17:38:54 +0200
Subject: hipe,erts: Remove cached fun 'native_address'

Did not work with purge and made worse by new purge strategy.

Did yield terrible performance when fun thing is created *before*
fun code is loaded. Like when receiving not yet loaded fun
from other node. The cached 'native_address' in ErlFunThing
will not be updated leading to mode switch and error_handler
being called for every call to the fun from native code.
---
 erts/emulator/beam/beam_emu.c        | 3 ---
 erts/emulator/beam/erl_fun.h         | 3 ---
 erts/emulator/beam/external.c        | 4 ----
 erts/emulator/hipe/hipe_mkliterals.c | 3 ---
 4 files changed, 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index ccb0f786ec..3fa4fb5b3f 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6667,9 +6667,6 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
     funp->fe = fe;
     funp->num_free = num_free;
     funp->creator = p->common.id;
-#ifdef HIPE
-    funp->native_address = fe->native_address;
-#endif
     funp->arity = (int)fe->address[-1] - num_free;
     for (i = 0; i < num_free; i++) {
 	*hp++ = reg[i];
diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h
index 73c3e19c1c..6ba055d92c 100644
--- a/erts/emulator/beam/erl_fun.h
+++ b/erts/emulator/beam/erl_fun.h
@@ -57,9 +57,6 @@ typedef struct erl_fun_thing {
     Eterm thing_word;		/* Subtag FUN_SUBTAG. */
     ErlFunEntry* fe;		/* Pointer to fun entry. */
     struct erl_off_heap_header* next;
-#ifdef HIPE
-    UWord* native_address;	/* Native code for the fun. */
-#endif
     Uint arity;			/* The arity of the fun. */
     Uint num_free;		/* Number of free variables (in env). */
   /* -- The following may be compound Erlang terms ---------------------- */
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index beed847578..ca79f83184 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3749,7 +3749,6 @@ dec_term_atom_common:
 		if (funp->fe->native_address == NULL) {
 		    hipe_set_closure_stub(funp->fe, num_free);
 		}
-		funp->native_address = funp->fe->native_address;
 #endif
 		hp = factory->hp;
 
@@ -3821,9 +3820,6 @@ dec_term_atom_common:
 
 		funp->fe = erts_put_fun_entry(module, old_uniq, old_index);
 		funp->arity = funp->fe->address[-1] - num_free;
-#ifdef HIPE
-		funp->native_address = funp->fe->native_address;
-#endif
 		hp = factory->hp;
 
 		/* Environment */
diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c
index b9d4226705..4573980e1e 100644
--- a/erts/emulator/hipe/hipe_mkliterals.c
+++ b/erts/emulator/hipe/hipe_mkliterals.c
@@ -435,9 +435,6 @@ static const struct rts_param rts_params[] = {
        presence or absence of struct erl_fun_thing's "next" field. */
     { 5, "EFT_CREATOR", 1, offsetof(struct erl_fun_thing, creator) },
     { 6, "EFT_FE", 1, offsetof(struct erl_fun_thing, fe) },
-#ifdef HIPE
-    { 7, "EFT_NATIVE_ADDRESS", 1, offsetof(struct erl_fun_thing, native_address) },
-#endif
     { 8, "EFT_ARITY", 1, offsetof(struct erl_fun_thing, arity) },
     { 9, "EFT_NUM_FREE", 1, offsetof(struct erl_fun_thing, num_free) },
     { 10, "EFT_ENV", 1, offsetof(struct erl_fun_thing, env[0]) },
-- 
cgit v1.2.3


From 1e8588e8c310a628fb0215ef15dc3cc29f98c336 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Sun, 2 Oct 2016 16:17:21 +0200
Subject: erts: Refactor hipe_loader_state_dtor into a true destructor

that is only called once.

Basically switch hipe_free_loader_state and hipe_loader_state_dtor.
---
 erts/emulator/beam/beam_load.c |  2 +-
 erts/emulator/hipe/hipe_load.c | 26 +++++++++++---------------
 erts/emulator/hipe/hipe_load.h |  2 +-
 3 files changed, 13 insertions(+), 17 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 4a833689e3..813788f66c 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -6528,7 +6528,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
 
 	erts_free_aligned_binary_bytes(temp_alloc);
 	free_loader_state(magic);
-	hipe_free_loader_state(hipe_magic);
+	hipe_free_loader_state(hipe_stp);
 
 	return mod;
     }
diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
index 48f0cab293..6261857d8f 100644
--- a/erts/emulator/hipe/hipe_load.c
+++ b/erts/emulator/hipe/hipe_load.c
@@ -32,13 +32,8 @@
 #include "erl_binary.h"
 #include "hipe_load.h"
 
-/*
- * This destructor function can safely be called multiple times.
- */
-static void
-hipe_loader_state_dtor(Binary* magic)
+void hipe_free_loader_state(HipeLoaderState *stp)
 {
-    HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic);
     if (stp->module == NIL) return;
 
     erts_fprintf(stderr, "Destroying HiPE loader state for module %T\n",
@@ -64,6 +59,16 @@ hipe_loader_state_dtor(Binary* magic)
      */
 }
 
+static void
+hipe_loader_state_dtor(Binary* magic)
+{
+    HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic);
+
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(magic) == hipe_loader_state_dtor);
+
+    hipe_free_loader_state(stp);
+}
+
 Binary *hipe_alloc_loader_state(Eterm module)
 {
     HipeLoaderState *stp;
@@ -90,15 +95,6 @@ Binary *hipe_alloc_loader_state(Eterm module)
     return magic;
 }
 
-void hipe_free_loader_state(Binary *magic)
-{
-    if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != hipe_loader_state_dtor)
-	return;
-
-    /* Why does BEAM do a refc_dec here? What is holding that reference? */
-    hipe_loader_state_dtor(magic);
-}
-
 HipeLoaderState *
 hipe_get_loader_state(Binary *magic)
 {
diff --git a/erts/emulator/hipe/hipe_load.h b/erts/emulator/hipe/hipe_load.h
index 17dd3c3c05..40c8a8aa2a 100644
--- a/erts/emulator/hipe/hipe_load.h
+++ b/erts/emulator/hipe/hipe_load.h
@@ -42,7 +42,7 @@ typedef struct hipe_loader_state {
 } HipeLoaderState;
 
 extern Binary *hipe_alloc_loader_state(Eterm module);
-extern void hipe_free_loader_state(Binary *binary);
+extern void hipe_free_loader_state(HipeLoaderState*);
 extern HipeLoaderState *hipe_get_loader_state(Binary *binary);
 
 #endif /* HIPE_LOAD_H */
-- 
cgit v1.2.3


From 663360ea2bb7766ad2015fff7f3f8a8f53c3eef2 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Sun, 2 Oct 2016 22:49:59 +0200
Subject: erts: Refactor out hipe_purge_refs/sdesc

from hipe_purge_module.
---
 erts/emulator/hipe/hipe_bif0.c | 119 +++++++++++++++++++++++------------------
 erts/emulator/hipe/hipe_bif0.h |   2 +
 2 files changed, 68 insertions(+), 53 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 67508e6f5d..58eee1693f 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1654,11 +1654,66 @@ int hipe_purge_need_blocking(Module* modp)
     return !modp->curr.code_hdr && modp->first_hipe_mfa;
 }
 
-void hipe_purge_module(Module* modp)
+void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module)
 {
-    struct hipe_ref* ref;
-    struct hipe_sdesc* sdesc;
+    struct hipe_ref* ref = first_ref;
+
+    while (ref) {
+        struct hipe_ref* free_ref = ref;
+
+        ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+        DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref,
+                      ref->callee->m, ref->callee->f, ref->callee->a);
+        DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref,
+                      ref->caller_m, ref->caller_f, ref->caller_a);
+        ASSERT(ref->caller_m == caller_module);
+
+        /*
+         * Unlink from other refs to same callee
+         */
+        ASSERT(ref->head.next->prev == &ref->head);
+        ASSERT(ref->head.prev->next == &ref->head);
+        ASSERT(ref->head.next != &ref->head);
+        ASSERT(ref->head.prev != &ref->head);
+        ref->head.next->prev = ref->head.prev;
+        ref->head.prev->next = ref->head.next;
+
+        /*
+         * Was this the last ref to that callee?
+         */
+        if (ref->head.next == ref->head.prev) {
+            struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers);
+            if (p->is_stub) {
+                unlink_mfa_from_mod(p);
+                purge_mfa(p);
+            }
+        }
+
+        ref = ref->next_from_modi;
+        erts_free(ERTS_ALC_T_HIPE, free_ref);
+    }
+}
 
+void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module)
+{
+    struct hipe_sdesc* sdesc = first_sdesc;
+    while (sdesc) {
+        struct hipe_sdesc* free_sdesc = sdesc;
+
+        ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+        DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue);
+        ASSERT(make_atom(sdesc->m_aix) == module);
+
+        sdesc = sdesc->next_in_modi;
+        hipe_destruct_sdesc(free_sdesc);
+    }
+}
+
+
+void hipe_purge_module(Module* modp)
+{
     ASSERT(modp);
 
     DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module");
@@ -1667,62 +1722,20 @@ void hipe_purge_module(Module* modp)
         /*
          * Remove all hipe_ref's (external calls) from the old module instance
          */
-        ref = modp->old.hipe_code->first_hipe_ref;
-
-        while (ref) {
-            struct hipe_ref* free_ref = ref;
-
-            ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
-
-            DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref,
-                          ref->callee->m, ref->callee->f, ref->callee->a);
-            DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref,
-                          ref->caller_m, ref->caller_f, ref->caller_a);
-            ASSERT(ref->caller_m == make_atom(modp->module));
-
-            /*
-             * Unlink from other refs to same callee
-             */
-            ASSERT(ref->head.next->prev == &ref->head);
-            ASSERT(ref->head.prev->next == &ref->head);
-            ASSERT(ref->head.next != &ref->head);
-            ASSERT(ref->head.prev != &ref->head);
-            ref->head.next->prev = ref->head.prev;
-            ref->head.prev->next = ref->head.next;
-
-            /*
-             * Was this the last ref to that callee?
-             */
-            if (ref->head.next == ref->head.prev) {
-                struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers);
-                if (p->is_stub) {
-                    unlink_mfa_from_mod(p);
-                    purge_mfa(p);
-               }
-            }
-
-            ref = ref->next_from_modi;
-            erts_free(ERTS_ALC_T_HIPE, free_ref);
+        if (modp->old.hipe_code->first_hipe_ref) {
+            hipe_purge_refs(modp->old.hipe_code->first_hipe_ref,
+                            make_atom(modp->module));
+            modp->old.hipe_code->first_hipe_ref = NULL;
         }
-        modp->old.hipe_code->first_hipe_ref = NULL;
 
         /*
          * Remove all hipe_sdesc's for the old module instance
          */
-        sdesc = modp->old.hipe_code->first_hipe_sdesc;
-
-        while (sdesc) {
-            struct hipe_sdesc* free_sdesc = sdesc;
-
-            ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
-
-            DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue);
-            ASSERT(sdesc->m_aix == modp->module);
-
-            sdesc = sdesc->next_in_modi;
-            hipe_destruct_sdesc(free_sdesc);
+        if (modp->old.hipe_code->first_hipe_sdesc) {
+            hipe_purge_sdescs(modp->old.hipe_code->first_hipe_sdesc,
+                             make_atom(modp->module));
+            modp->old.hipe_code->first_hipe_sdesc = NULL;
         }
-        modp->old.hipe_code->first_hipe_sdesc = NULL;
 
         hipe_free_module(modp->old.hipe_code);
         modp->old.hipe_code = NULL;
diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h
index 1914405e2d..2b7c01b975 100644
--- a/erts/emulator/hipe/hipe_bif0.h
+++ b/erts/emulator/hipe/hipe_bif0.h
@@ -44,6 +44,8 @@ extern void hipe_primop_set_trampoline(Eterm name, void *trampoline);
 /* needed in beam_load.c */
 int hipe_need_blocking(Module*);
 int hipe_purge_need_blocking(Module*);
+void hipe_purge_refs(struct hipe_ref*, Eterm);
+void hipe_purge_sdescs(struct hipe_sdesc*, Eterm);
 void hipe_purge_module(Module*);
 void hipe_redirect_to_module(Module* modp);
 
-- 
cgit v1.2.3


From d9e8fa3d13be54246297ac9776e835ce0bdcd362 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Sun, 2 Oct 2016 22:51:49 +0200
Subject: erts: Free hipe_refs and hipe_sdesc of a failed load

---
 erts/emulator/hipe/hipe_load.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
index 6261857d8f..c05dd886a4 100644
--- a/erts/emulator/hipe/hipe_load.c
+++ b/erts/emulator/hipe/hipe_load.c
@@ -31,6 +31,7 @@
 #include "global.h"
 #include "erl_binary.h"
 #include "hipe_load.h"
+#include "hipe_bif0.h"
 
 void hipe_free_loader_state(HipeLoaderState *stp)
 {
@@ -51,12 +52,16 @@ void hipe_free_loader_state(HipeLoaderState *stp)
     stp->data_segment = NULL;
     stp->data_segment_size = 0;
 
-    stp->module = NIL;
+    if (stp->new_hipe_refs) {
+        hipe_purge_refs(stp->new_hipe_refs, stp->module);
+        stp->new_hipe_refs = NULL;
+    }
+    if (stp->new_hipe_sdesc) {
+        hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module);
+        stp->new_hipe_sdesc = NULL;
+    }
 
-    ASSERT(!stp->new_hipe_refs && !stp->new_hipe_sdesc);
-    /* ToDO: Purge lists 'new_hipe_refs' and 'new_hipe_sdesc'
-     *       if this is a failed load.
-     */
+    stp->module = NIL;
 }
 
 static void
-- 
cgit v1.2.3


From d347e91735cce9ace9c376ba4913fcf688da22f8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 3 Oct 2016 19:22:08 +0200
Subject: erts: Remove debug printout for hipe loader state

creation and destruction.
---
 erts/emulator/hipe/hipe_load.c | 5 -----
 1 file changed, 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
index c05dd886a4..fcbcf1a6a4 100644
--- a/erts/emulator/hipe/hipe_load.c
+++ b/erts/emulator/hipe/hipe_load.c
@@ -37,9 +37,6 @@ void hipe_free_loader_state(HipeLoaderState *stp)
 {
     if (stp->module == NIL) return;
 
-    erts_fprintf(stderr, "Destroying HiPE loader state for module %T\n",
-		 stp->module);
-
     // TODO: Needs to be freed separately. We'd like have a unified executable
     // code allocator, so postpone this for now.
     /* if (stp->text_segment) */
@@ -81,8 +78,6 @@ Binary *hipe_alloc_loader_state(Eterm module)
 
     if (is_not_atom(module)) return NULL;
 
-    erts_fprintf(stderr, "Creating HiPE loader state for module %T\n", module);
-
     magic = erts_create_magic_binary(sizeof(HipeLoaderState),
 				     hipe_loader_state_dtor);
     erts_refc_inc(&magic->refc, 1);
-- 
cgit v1.2.3


From 6eee26d55f32113e3ea4575e5500f2fc3c312fff Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 4 Oct 2016 21:46:53 +0200
Subject: erts: Enable exec_alloc for all hipe architectures

For non-amd64 it's a "normal" allocator with a
wrapper around mseg_alloc to call mprotect(PROT_EXEC).
---
 erts/emulator/beam/erl_alloc.c      | 16 +++++++++----
 erts/emulator/beam/erl_alloc_util.c | 48 ++++++++++++++++++++++++++++++++++++-
 erts/emulator/beam/erl_alloc_util.h |  6 +++++
 erts/emulator/sys/common/erl_mmap.c |  4 ++--
 erts/emulator/sys/common/erl_mmap.h |  5 ++--
 erts/emulator/sys/common/erl_mseg.c |  2 +-
 6 files changed, 70 insertions(+), 11 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index d96b90fd55..56aa78a08a 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -373,10 +373,16 @@ set_default_exec_alloc_opts(struct au_init *ip)
     ip->init.util.rmbcmt	= 0;
     ip->init.util.acul		= 0;
 
+# ifdef ERTS_HAVE_EXEC_MMAPPER
     ip->init.util.mseg_alloc    = &erts_alcu_mmapper_mseg_alloc;
     ip->init.util.mseg_realloc  = &erts_alcu_mmapper_mseg_realloc;
     ip->init.util.mseg_dealloc  = &erts_alcu_mmapper_mseg_dealloc;
     ip->init.util.mseg_mmapper  = &erts_exec_mmapper;
+# else
+    ip->init.util.mseg_alloc    = &erts_alcu_exec_mseg_alloc;
+    ip->init.util.mseg_realloc  = &erts_alcu_exec_mseg_realloc;
+    ip->init.util.mseg_dealloc  = &erts_alcu_exec_mseg_dealloc;
+# endif
 }
 #endif /* ERTS_ALC_A_EXEC */
 
@@ -1571,7 +1577,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 		    break;
                 case 'X':
                     if (has_prefix("scs", argv[i]+3)) {
-#ifdef ERTS_ALC_A_EXEC
+#ifdef ERTS_HAVE_EXEC_MMAPPER
                         init->mseg.exec_mmap.scs =
 #endif
                             get_mb_value(argv[i]+6, argv, &i);
@@ -2852,7 +2858,7 @@ erts_allocator_info(int to, void *arg)
         erts_print(to, arg, "=allocator:erts_mmap.literal_mmap\n");
         erts_mmap_info(&erts_literal_mmapper, &to, arg, NULL, NULL, &emis);
 #endif
-#ifdef ERTS_ALC_A_EXEC
+#ifdef ERTS_HAVE_EXEC_MMAPPER
         erts_print(to, arg, "=allocator:erts_mmap.exec_mmap\n");
         erts_mmap_info(&erts_exec_mmapper, &to, arg, NULL, NULL, &emis);
 #endif
@@ -3010,7 +3016,7 @@ erts_allocator_options(void *proc)
 #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
     terms[length++] = ERTS_MAKE_AM("literal_mmap");
 #endif
-#ifdef ERTS_ALC_A_EXEC
+#ifdef ERTS_HAVE_EXEC_MMAPPER
     terms[length++] = ERTS_MAKE_AM("exec_mmap");
 #endif
     features = length ? erts_bld_list(hpp, szp, length, terms) : NIL;
@@ -3102,7 +3108,7 @@ reply_alloc_info(void *vair)
 # if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
     struct erts_mmap_info_struct mmap_info_literal;
 # endif
-# ifdef ERTS_ALC_A_EXEC
+# ifdef ERTS_HAVE_EXEC_MMAPPER
     struct erts_mmap_info_struct mmap_info_exec;
 # endif
 #endif
@@ -3232,7 +3238,7 @@ reply_alloc_info(void *vair)
                                             erts_bld_atom(hpp,szp,"literal_mmap"),
                                             ainfo);
 #  endif
-#  ifdef ERTS_ALC_A_EXEC
+#  ifdef ERTS_HAVE_EXEC_MMAPPER
                     ai_list = erts_bld_cons(hpp, szp,
                                             ainfo, ai_list);
                     ainfo = (air->only_sz ? NIL :
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 2995f2f822..77be48035d 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -907,7 +907,9 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
 
 #elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
 
-/* Used by literal allocator that has its own mmapper (super carrier) */
+/* For allocators that have their own mmapper (super carrier),
+ * like literal_alloc and exec_alloc on amd64
+ */
 void*
 erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
 {
@@ -948,6 +950,50 @@ erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
 }
 #endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */
 
+#if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER)
+
+/*
+ * For exec_alloc on non-amd64 that just need memory with PROT_EXEC
+ */
+void*
+erts_alcu_exec_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
+{
+    void* res = erts_alcu_mseg_alloc(allctr, size_p, flags);
+
+    if (res) {
+        int r = mprotect(res, *size_p, PROT_EXEC | PROT_READ | PROT_WRITE);
+        ASSERT(r == 0); (void)r;
+    }
+    return res;
+}
+
+void*
+erts_alcu_exec_mseg_realloc(Allctr_t *allctr, void *seg,
+                            Uint old_size, Uint *new_size_p)
+{
+    void *res;
+
+    if (seg && old_size) {
+        int r = mprotect(seg, old_size, PROT_READ | PROT_WRITE);
+        ASSERT(r == 0); (void)r;
+    }
+    res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p);
+    if (res) {
+        int r = mprotect(res, *new_size_p, PROT_EXEC | PROT_READ | PROT_WRITE);
+        ASSERT(r == 0); (void)r;
+    }
+    return res;
+}
+
+void
+erts_alcu_exec_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags)
+{
+    int r = mprotect(seg, size, PROT_READ | PROT_WRITE);
+    ASSERT(r == 0); (void)r;
+    erts_alcu_mseg_dealloc(allctr, seg, size, flags);
+}
+#endif /* ERTS_ALC_A_EXEC && !ERTS_HAVE_EXEC_MMAPPER */
+
 #endif /* HAVE_ERTS_MSEG */
 
 void*
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index f50f09907a..2958bdf8d1 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -210,6 +210,12 @@ void* erts_alcu_mmapper_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
 void* erts_alcu_mmapper_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
 void  erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
 # endif
+
+# if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER)
+void* erts_alcu_exec_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
+void* erts_alcu_exec_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
+void  erts_alcu_exec_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
+# endif
 #endif /* HAVE_ERTS_MSEG */
 
 void* erts_alcu_sys_alloc(Allctr_t*, Uint *size_p, int superalign);
diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index 7bbb406f29..3ab03971a5 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -21,6 +21,7 @@
 #  include "config.h"
 #endif
 
+#define ERTS_WANT_MEM_MAPPERS
 #include "sys.h"
 #include "erl_process.h"
 #include "erl_smp.h"
@@ -358,12 +359,11 @@ char* erts_literals_start;
 UWord erts_literals_size;
 #endif
 
-#ifdef ERTS_ALC_A_EXEC
+#ifdef ERTS_HAVE_EXEC_MMAPPER
 ErtsMemMapper erts_exec_mmapper;
 #endif
 
 
-
 #define ERTS_MMAP_SIZE_SC_SA_INC(SZ) 						\
     do {									\
 	mm->size.supercarrier.used.total += (SZ);			\
diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h
index fa51b663fa..92e9eb9e41 100644
--- a/erts/emulator/sys/common/erl_mmap.h
+++ b/erts/emulator/sys/common/erl_mmap.h
@@ -159,9 +159,10 @@ Eterm erts_mmap_info_options(ErtsMemMapper*,
 extern ErtsMemMapper erts_dflt_mmapper;
 #  if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
 extern ErtsMemMapper erts_literal_mmapper;
-#  endif
-#  ifdef ERTS_ALC_A_EXEC
+#    ifdef ERTS_ALC_A_EXEC
+#      define ERTS_HAVE_EXEC_MMAPPER
 extern ErtsMemMapper erts_exec_mmapper;
+#    endif
 #  endif
 #endif /* ERTS_WANT_MEM_MAPPERS */
 
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index f3306a888c..e76082177e 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -1414,7 +1414,7 @@ erts_mseg_init(ErtsMsegInit_t *init)
 
     erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms");
 
-#ifdef ERTS_ALC_A_EXEC
+#ifdef ERTS_HAVE_EXEC_MMAPPER
     /* Initialize erts_exec_mapper *FIRST*, to increase probability
      * of getting low memory for HiPE AMD64's small code model.
      */
-- 
cgit v1.2.3


From d9149ef5e2591d34ba781a08b1ae3e1823f7a13a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 5 Oct 2016 17:54:15 +0200
Subject: erts: Fix old leak of hipe code on x86 32-bit

---
 erts/emulator/hipe/hipe_x86.c | 112 ++----------------------------------------
 1 file changed, 3 insertions(+), 109 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c
index 8a3ba8bc7f..b091bee6fb 100644
--- a/erts/emulator/hipe/hipe_x86.c
+++ b/erts/emulator/hipe/hipe_x86.c
@@ -24,7 +24,6 @@
 #include "config.h"
 #endif
 #include "global.h"
-#include <sys/mman.h>
 
 #include "hipe_arch.h"
 #include "hipe_native_bif.h"	/* nbif_callemu() */
@@ -70,110 +69,9 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
     return 0;
 }
 
-/*
- * Memory allocator for executable code.
- *
- * This is required on x86 because some combinations
- * of Linux kernels and CPU generations default to
- * non-executable memory mappings, causing ordinary
- * malloc() memory to be non-executable.
- */
-static unsigned int code_bytes;
-static char *code_next;
-
-#if 0	/* change to non-zero to get allocation statistics at exit() */
-static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost;
-static unsigned int atexit_done;
-
-static void alloc_code_stats(void)
-{
-    printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n",
-	   total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost);
-}
-
-static void atexit_alloc_code_stats(void)
-{
-    if (!atexit_done) {
-	atexit_done = 1;
-	(void)atexit(alloc_code_stats);
-    }
-}
-
-#define ALLOC_CODE_STATS(X)	do{X;}while(0)
-#else
-#define ALLOC_CODE_STATS(X)	do{}while(0)
-#endif
-
-/* FreeBSD 6.1 and Darwin breakage */
-#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-
-static int morecore(unsigned int alloc_bytes)
-{
-    unsigned int map_bytes;
-    char *map_hint, *map_start;
-
-    /* Page-align the amount to allocate. */
-    map_bytes = (alloc_bytes + 4095) & ~4095;
-
-    /* Round up small allocations. */
-    if (map_bytes < 1024*1024)
-	map_bytes = 1024*1024;
-    else
-	ALLOC_CODE_STATS(++nr_large);
-
-    /* Create a new memory mapping, ensuring it is executable
-       and in the low 2GB of the address space. Also attempt
-       to make it adjacent to the previous mapping. */
-    map_hint = code_next + code_bytes;
-    if ((unsigned long)map_hint & 4095)
-	abort();
-    map_start = mmap(map_hint, map_bytes,
-		     PROT_EXEC|PROT_READ|PROT_WRITE,
-		     MAP_PRIVATE|MAP_ANONYMOUS
-#ifdef __x86_64__
-		     |MAP_32BIT
-#endif
-		     ,
-		     -1, 0);
-    if (map_start == MAP_FAILED)
-	return -1;
-
-    ALLOC_CODE_STATS(total_mapped += map_bytes);
-
-    /* Merge adjacent mappings, so the trailing portion of the previous
-       mapping isn't lost. In practice this is quite successful. */
-    if (map_start == map_hint) {
-	ALLOC_CODE_STATS(++nr_joins);
-	code_bytes += map_bytes;
-    } else {
-	ALLOC_CODE_STATS(++nr_splits);
-	ALLOC_CODE_STATS(total_lost += code_bytes);
-	code_next = map_start;
-	code_bytes = map_bytes;
-    }
-
-    ALLOC_CODE_STATS(atexit_alloc_code_stats());
-
-    return 0;
-}
-
 static void *alloc_code(unsigned int alloc_bytes)
 {
-    void *res;
-
-    /* Align function entries. */
-    alloc_bytes = (alloc_bytes + 3) & ~3;
-
-    if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0)
-	return NULL;
-    ALLOC_CODE_STATS(++nr_allocs);
-    ALLOC_CODE_STATS(total_alloc += alloc_bytes);
-    res = code_next;
-    code_next += alloc_bytes;
-    code_bytes -= alloc_bytes;
-    return res;
+    return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes);
 }
 
 void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
@@ -186,11 +84,7 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
 
 void hipe_free_code(void* code, unsigned int bytes)
 {
-    /* SVERK: Leak !!!
-    ALLOC_CODE_STATS(--nr_allocs);
-    ALLOC_CODE_STATS(total_alloc -= bytes);
-
-    erts_free(ERTS_ALC_T_HIPE_EXEC, code);*/
+    erts_free(ERTS_ALC_T_HIPE_EXEC, code);
 }
 
 void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
@@ -275,7 +169,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
 
 void hipe_free_native_stub(void* stub)
 {
-    /* SVERK: leak leak drip drop */
+    erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
 }
 
 void hipe_arch_print_pcb(struct hipe_process_state *p)
-- 
cgit v1.2.3


From e048ebf4bddf8434d0f4402863c0f6313a741df0 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 5 Oct 2016 19:48:01 +0200
Subject: erts: Fix old leak of sparc hipe code

---
 erts/emulator/hipe/hipe_sparc.c | 109 ++++------------------------------------
 1 file changed, 11 insertions(+), 98 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_sparc.c b/erts/emulator/hipe/hipe_sparc.c
index 23020f34ee..75bfb4fef1 100644
--- a/erts/emulator/hipe/hipe_sparc.c
+++ b/erts/emulator/hipe/hipe_sparc.c
@@ -24,7 +24,6 @@
 #include "config.h"
 #endif
 #include "global.h"
-#include <sys/mman.h>
 
 #include "hipe_arch.h"
 #include "hipe_native_bif.h"	/* nbif_callemu() */
@@ -97,105 +96,9 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
     return 0;
 }
 
-/*
- * Memory allocator for executable code.
- *
- * This is required on x86 because some combinations
- * of Linux kernels and CPU generations default to
- * non-executable memory mappings, causing ordinary
- * malloc() memory to be non-executable.
- */
-static unsigned int code_bytes;
-static char *code_next;
-
-#if 0	/* change to non-zero to get allocation statistics at exit() */
-static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost;
-static unsigned int atexit_done;
-
-static void alloc_code_stats(void)
-{
-    printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n",
-	   total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost);
-}
-
-static void atexit_alloc_code_stats(void)
-{
-    if (!atexit_done) {
-	atexit_done = 1;
-	(void)atexit(alloc_code_stats);
-    }
-}
-
-#define ALLOC_CODE_STATS(X)	do{X;}while(0)
-#else
-#define ALLOC_CODE_STATS(X)	do{}while(0)
-#endif
-
-static int morecore(unsigned int alloc_bytes)
-{
-    unsigned int map_bytes;
-    char *map_hint, *map_start;
-
-    /* Page-align the amount to allocate. */
-    map_bytes = (alloc_bytes + 4095) & ~4095;
-
-    /* Round up small allocations. */
-    if (map_bytes < 1024*1024)
-	map_bytes = 1024*1024;
-    else
-	ALLOC_CODE_STATS(++nr_large);
-
-    /* Create a new memory mapping, ensuring it is executable
-       and in the low 2GB of the address space. Also attempt
-       to make it adjacent to the previous mapping. */
-    map_hint = code_next + code_bytes;
-    if ((unsigned long)map_hint & 4095)
-	abort();
-    map_start = mmap(map_hint, map_bytes,
-		     PROT_EXEC|PROT_READ|PROT_WRITE,
-		     MAP_PRIVATE|MAP_ANONYMOUS
-#ifdef __x86_64__
-		     |MAP_32BIT
-#endif
-		     ,
-		     -1, 0);
-    if (map_start == MAP_FAILED)
-	return -1;
-
-    ALLOC_CODE_STATS(total_mapped += map_bytes);
-
-    /* Merge adjacent mappings, so the trailing portion of the previous
-       mapping isn't lost. In practice this is quite successful. */
-    if (map_start == map_hint) {
-	ALLOC_CODE_STATS(++nr_joins);
-	code_bytes += map_bytes;
-    } else {
-	ALLOC_CODE_STATS(++nr_splits);
-	ALLOC_CODE_STATS(total_lost += code_bytes);
-	code_next = map_start;
-	code_bytes = map_bytes;
-    }
-
-    ALLOC_CODE_STATS(atexit_alloc_code_stats());
-
-    return 0;
-}
-
 static void *alloc_code(unsigned int alloc_bytes)
 {
-    void *res;
-
-    /* Align function entries. */
-    alloc_bytes = (alloc_bytes + 3) & ~3;
-
-    if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0)
-	return NULL;
-    ALLOC_CODE_STATS(++nr_allocs);
-    ALLOC_CODE_STATS(total_alloc += alloc_bytes);
-    res = code_next;
-    code_next += alloc_bytes;
-    code_bytes -= alloc_bytes;
-    return res;
+    return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes);
 }
 
 void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
@@ -206,6 +109,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
     return alloc_code(nrbytes);
 }
 
+void hipe_free_code(void* code, unsigned int nrbytes)
+{
+    erts_free(ERTS_ALC_T_HIPE_EXEC, code);
+}
+
 void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
 {
     unsigned int *code;
@@ -235,6 +143,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
     return code;
 }
 
+void hipe_free_native_stub(void* stub)
+{
+    erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
+}
+
 void hipe_arch_print_pcb(struct hipe_process_state *p)
 {
 #define U(n,x) \
-- 
cgit v1.2.3


From 25b28611ea06d3e2d3708d20e53118ec24ae8321 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 5 Oct 2016 19:09:28 +0200
Subject: erts: Fix old leak for arm hipe code

* Use erts_alloc(ERTS_ALC_T_HIPE_EXEC,_)
* Each module has its own trampolines
* Stubs do not use trampolines,
  they do their own long jumps.
---
 erts/emulator/hipe/hipe_arm.c  | 201 ++++++++++++-----------------------------
 erts/emulator/hipe/hipe_bif0.c |   1 -
 2 files changed, 57 insertions(+), 145 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c
index 64e35b62d8..b61939724c 100644
--- a/erts/emulator/hipe/hipe_arm.c
+++ b/erts/emulator/hipe/hipe_arm.c
@@ -25,7 +25,6 @@
 #endif
 #include "global.h"
 #include "erl_binary.h"
-#include <sys/mman.h>
 
 #include "hipe_arch.h"
 #include "hipe_native_bif.h"	/* nbif_callemu() */
@@ -57,30 +56,6 @@ void hipe_flush_icache_word(void *address)
     hipe_flush_icache_range(address, 4);
 }
 
-/*
- * Management of 32MB code segments for regular code and trampolines.
- */
-
-#define SEGMENT_NRBYTES	(32*1024*1024)	/* named constant, _not_ a tunable */
-
-static struct segment {
-    unsigned int *base;		/* [base,base+32MB[ */
-    unsigned int *code_pos;	/* INV: base <= code_pos <= tramp_pos  */
-    unsigned int *tramp_pos;	/* INV: tramp_pos <= base+32MB */
-    /* On ARM we always allocate a trampoline at base+32MB-8 for
-       nbif_callemu, so tramp_pos <= base+32MB-8. */
-} curseg;
-
-#define in_area(ptr,start,nbytes)	\
-	((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
-
-static void *new_code_mapping(void)
-{
-    return mmap(0, SEGMENT_NRBYTES,
-		PROT_EXEC|PROT_READ|PROT_WRITE,
-		MAP_PRIVATE|MAP_ANONYMOUS,
-		-1, 0);
-}
 
 static int check_callees(Eterm callees)
 {
@@ -107,136 +82,53 @@ static int check_callees(Eterm callees)
     return arity;
 }
 
-static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec)
-{
-    unsigned int *base, *address, *tramp_pos, nrfreewords;
-    int trampnr;
-    Eterm mfa, m, f;
-    unsigned int a, *trampoline;
+#define TRAMPOLINE_WORDS 2
 
-    m = NIL; f = NIL; a = 0; /* silence stupid compiler warning */
-    tramp_pos = curseg.tramp_pos;
-    address = curseg.code_pos;
-    nrfreewords = tramp_pos - address;
-    if (nrwords > nrfreewords)
-	return NULL;
-    curseg.code_pos = address + nrwords;
-    nrfreewords -= nrwords;
-
-    base = curseg.base;
-    for (trampnr = 1; trampnr <= nrcallees; ++trampnr) {
-	mfa = tuple_val(callees)[trampnr];
-	if (is_atom(mfa))
-	    trampoline = hipe_primop_get_trampoline(mfa);
-	else {
-	    m = tuple_val(mfa)[1];
-	    f = tuple_val(mfa)[2];
-	    a = unsigned_val(tuple_val(mfa)[3]);
-	    trampoline = hipe_mfa_get_trampoline(m, f, a);
-	}
-	if (!in_area(trampoline, base, SEGMENT_NRBYTES)) {
-	    if (nrfreewords < 2)
-		return NULL;
-	    nrfreewords -= 2;
-	    tramp_pos = trampoline = tramp_pos - 2;
-	    trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */
-	    trampoline[1] = 0;		/* callee's address */
-	    hipe_flush_icache_range(trampoline, 2*sizeof(int));
-	    if (is_atom(mfa))
-		hipe_primop_set_trampoline(mfa, trampoline);
-	    else
-		hipe_mfa_set_trampoline(m, f, a, trampoline);
-	}
-	trampvec[trampnr-1] = trampoline;
+static void generate_trampolines(Uint32* address,
+                                 int nrcallees, Eterm callees,
+                                 Uint32** trampvec)
+{
+    Uint32* trampoline = address;
+    int i;
+
+    for (i = 0; i < nrcallees; ++i) {
+        trampoline[0] = 0xE51FF004;     /* ldr pc, [pc,#-4] */
+        trampoline[1] = 0;		/* callee's address */
+	trampvec[i] = trampoline;
+        trampoline += TRAMPOLINE_WORDS;
     }
-    curseg.tramp_pos = tramp_pos;
-    return address;
+    hipe_flush_icache_range(address, nrcallees*2*sizeof(Uint32));
 }
 
 void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
 {
-    Uint nrwords;
+    Uint code_words;
     int nrcallees;
     Eterm trampvecbin;
-    unsigned int **trampvec;
-    unsigned int *address;
-    unsigned int *base;
-    struct segment oldseg;
+    Uint32 **trampvec;
+    Uint32 *address;
 
     if (nrbytes & 0x3)
 	return NULL;
-    nrwords = nrbytes >> 2;
+    code_words = nrbytes / sizeof(Uint32);
 
     nrcallees = check_callees(callees);
     if (nrcallees < 0)
 	return NULL;
-    trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*));
-    trampvec = (unsigned int**)binary_bytes(trampvecbin);
+    trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*));
+    trampvec = (Uint32**)binary_bytes(trampvecbin);
 
-    address = try_alloc(nrwords, nrcallees, callees, trampvec);
-    if (!address) {
-	base = new_code_mapping();
-	if (base == MAP_FAILED)
-	    return NULL;
-	oldseg = curseg;
-	curseg.base = base;
-	curseg.code_pos = base;
-	curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES);
-	curseg.tramp_pos -= 2;
-	curseg.tramp_pos[0] = 0xE51FF004;	/* ldr pc, [pc,#-4] */
-	curseg.tramp_pos[1] = (unsigned int)&nbif_callemu;
+    address = erts_alloc(ERTS_ALC_T_HIPE_EXEC,
+                         (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32));
 
-	address = try_alloc(nrwords, nrcallees, callees, trampvec);
-	if (!address) {
-	    munmap(base, SEGMENT_NRBYTES);
-	    curseg = oldseg;
-	    return NULL;
-	}
-	/* commit to new segment, ignore leftover space in old segment */
-    }
+    generate_trampolines(address + code_words, nrcallees, callees, trampvec);
     *trampolines = trampvecbin;
     return address;
 }
 
 void  hipe_free_code(void* code, unsigned int bytes)
 {
-    /*SVERK: Leaking code memory */
-}
-
-static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu)
-{
-    unsigned int *address;
-    unsigned int *base;
-    struct segment oldseg;
-
-    address = try_alloc(nrwords, 0, NIL, NULL);
-    if (!address) {
-	base = new_code_mapping();
-	if (base == MAP_FAILED)
-	    return NULL;
-	oldseg = curseg;
-	curseg.base = base;
-	curseg.code_pos = base;
-	curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES);
-	curseg.tramp_pos -= 2;
-	curseg.tramp_pos[0] = 0xE51FF004;	/* ldr pc, [pc,#-4] */
-	curseg.tramp_pos[1] = (unsigned int)&nbif_callemu;
-
-	address = try_alloc(nrwords, 0, NIL, NULL);
-	if (!address) {
-	    munmap(base, SEGMENT_NRBYTES);
-	    curseg = oldseg;
-	    return NULL;
-	}
-	/* commit to new segment, ignore leftover space in old segment */
-    }
-    *tramp_callemu = (unsigned int*)((char*)curseg.base + SEGMENT_NRBYTES) - 2;
-    return address;
-}
-
-void hipe_free_native_stub(void* stub)
-{
-    /*SVERK: Leaking code stub */
+    erts_free(ERTS_ALC_T_HIPE_EXEC, code);
 }
 
 /*
@@ -276,8 +168,8 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type)
 void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
 {
     unsigned int *code;
-    unsigned int *tramp_callemu;
     int callemu_offset;
+    int is_short_jmp;
 
     /*
      * Native code calls BEAM via a stub looking as follows:
@@ -287,36 +179,57 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
      * b nbif_callemu
      * .long callee_exp
      *
+     * or if nbif_callemu is too far away:
+     *
+     * mov r0, #beamArity
+     * ldr r8, [pc,#0] // callee_exp
+     * ldr pc, [pc,#0] // nbif_callemu
+     * .long callee_exp
+     * .long nbif_callemu
+     *
      * I'm using r0 and r8 since they aren't used for
-     * parameter passing in native code. The branch to
-     * nbif_callemu may need to go via a trampoline.
-     * (Trampolines are allowed to modify r12, but they don't.)
+     * parameter passing in native code.
      */
 
-    code = alloc_stub(4, &tramp_callemu);
+    code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 5*sizeof(Uint32));
     if (!code)
 	return NULL;
     callemu_offset = ((int)&nbif_callemu - ((int)&code[2] + 8)) >> 2;
-    if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) {
-	callemu_offset = ((int)tramp_callemu - ((int)&code[2] + 8)) >> 2;
-	if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF))
-	    abort();
+    is_short_jmp = (callemu_offset >= -0x00800000 &&
+                    callemu_offset <= 0x007FFFFF);
+#ifdef DEBUG
+    if (is_short_jmp && (callemu_offset % 3)==0) {
+        is_short_jmp = 0;
     }
+#endif
 
     /* mov r0, #beamArity */
     code[0] = 0xE3A00000 | (beamArity & 0xFF);
     /* ldr r8, [pc,#0] // callee_exp */
     code[1] = 0xE59F8000;
-    /* b nbif_callemu */
-    code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF);
+    if (is_short_jmp) {
+        /* b nbif_callemu */
+        code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF);
+    }
+    else {
+        /* ldr pc, [pc,#0] // nbif_callemu */
+        code[2] = 0xE59FF000;
+        /* .long nbif_callemu */
+        code[4] = (unsigned int)&nbif_callemu;
+    }
     /* .long callee_exp */
     code[3] = (unsigned int)callee_exp;
 
-    hipe_flush_icache_range(code, 4*sizeof(int));
+    hipe_flush_icache_range(code, 5*sizeof(Uint32));
 
     return code;
 }
 
+void hipe_free_native_stub(void* stub)
+{
+    erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
+}
+
 static void patch_b(Uint32 *address, Sint32 offset, Uint32 AA)
 {
     Uint32 oldI = *address;
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 58eee1693f..f9a657124a 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -417,7 +417,6 @@ BIF_RETTYPE hipe_bifs_enter_code_3(BIF_ALIST_3)
     ASSERT(bitoffs == 0);
     ASSERT(bitsize == 0);
     trampolines = NIL;
-    // XXX: Trampolines are not tracked and freed
     address = hipe_alloc_code(nrbytes, BIF_ARG_2, &trampolines, BIF_P);
     if (!address) {
 	Uint nrcallees;
-- 
cgit v1.2.3


From 65fbc0464cce8266024129fcf10ef0906f907eb4 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 12 Oct 2016 19:00:53 +0200
Subject: erts: Fix old leak for ppc hipe code

* Use erts_alloc(ERTS_ALC_T_HIPE_EXEC,_)
* Each module has its own trampolines
---
 erts/emulator/hipe/hipe_ppc.c | 178 ++++++++++--------------------------------
 1 file changed, 42 insertions(+), 136 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c
index a1a6e2ad02..4413748936 100644
--- a/erts/emulator/hipe/hipe_ppc.c
+++ b/erts/emulator/hipe/hipe_ppc.c
@@ -25,7 +25,6 @@
 #endif
 #include "global.h"
 #include "erl_binary.h"
-#include <sys/mman.h>
 
 #include "hipe_arch.h"
 #include "hipe_native_bif.h"	/* nbif_callemu() */
@@ -68,34 +67,6 @@ void hipe_flush_icache_range(void *address, unsigned int nbytes)
     asm volatile("sync\n\tisync");
 }
 
-/*
- * Management of 32MB code segments for regular code and trampolines.
- */
-
-#define SEGMENT_NRBYTES	(32*1024*1024)	/* named constant, _not_ a tunable */
-
-static struct segment {
-    unsigned int *base;		/* [base,base+32MB[ */
-    unsigned int *code_pos;	/* INV: base <= code_pos <= tramp_pos  */
-    unsigned int *tramp_pos;	/* INV: tramp_pos <= base+32MB */
-} curseg;
-
-#define in_area(ptr,start,nbytes)	\
-	((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
-
-/* Darwin breakage */
-#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-
-static void *new_code_mapping(void)
-{
-    return mmap(0, SEGMENT_NRBYTES,
-		PROT_EXEC|PROT_READ|PROT_WRITE,
-		MAP_PRIVATE|MAP_ANONYMOUS,
-		-1, 0);
-}
-
 static int check_callees(Eterm callees)
 {
     Eterm *tuple;
@@ -119,136 +90,71 @@ static int check_callees(Eterm callees)
     return arity;
 }
 
-static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec)
+
+static void generate_trampolines(Uint32* address,
+                                 int nrcallees, Eterm callees,
+                                 Uint32** trampvec)
 {
-    unsigned int *base, *address, *tramp_pos, nrfreewords;
-    int trampnr;
+    Uint32* trampoline = address;
+    int i;
 
-    tramp_pos = curseg.tramp_pos;
-    address = curseg.code_pos;
-    nrfreewords = tramp_pos - address;
-    if (nrwords > nrfreewords)
-	return NULL;
-    curseg.code_pos = address + nrwords;
-    nrfreewords -= nrwords;
-
-    base = curseg.base;
-    for (trampnr = 1; trampnr <= nrcallees; ++trampnr) {
-	Eterm mfa = tuple_val(callees)[trampnr];
-	Eterm m = tuple_val(mfa)[1];
-	Eterm f = tuple_val(mfa)[2];
-	unsigned int a = unsigned_val(tuple_val(mfa)[3]);
-	unsigned int *trampoline = hipe_mfa_get_trampoline(m, f, a);
-	if (!in_area(trampoline, base, SEGMENT_NRBYTES)) {
+    for (i = 0; i < nrcallees; ++i) {
 #if defined(__powerpc64__)
-	    if (nrfreewords < 7)
-		return NULL;
-	    nrfreewords -= 7;
-	    tramp_pos = trampoline = tramp_pos - 7;
-	    trampoline[0] = 0x3D600000; /* addis r11,r0,0 */
-	    trampoline[1] = 0x616B0000; /* ori r11,r11,0 */
-	    trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */
-	    trampoline[3] = 0x656B0000; /* oris r11,r11,0 */
-	    trampoline[4] = 0x616B0000; /* ori r11,r11,0 */
-	    trampoline[5] = 0x7D6903A6; /* mtctr r11 */
-	    trampoline[6] = 0x4E800420; /* bctr */
-	    hipe_flush_icache_range(trampoline, 7*sizeof(int));
+# define TRAMPOLINE_WORDS 7
+        trampoline[0] = 0x3D600000; /* addis r11,r0,0 */
+        trampoline[1] = 0x616B0000; /* ori r11,r11,0 */
+        trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */
+        trampoline[3] = 0x656B0000; /* oris r11,r11,0 */
+        trampoline[4] = 0x616B0000; /* ori r11,r11,0 */
+        trampoline[5] = 0x7D6903A6; /* mtctr r11 */
+        trampoline[6] = 0x4E800420; /* bctr */
 #else
-	    if (nrfreewords < 4)
-		return NULL;
-	    nrfreewords -= 4;
-	    tramp_pos = trampoline = tramp_pos - 4;
-	    trampoline[0] = 0x39600000; /* addi r11,r0,0 */
-	    trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */
-	    trampoline[2] = 0x7D6903A6; /* mtctr r11 */
-	    trampoline[3] = 0x4E800420; /* bctr */
-	    hipe_flush_icache_range(trampoline, 4*sizeof(int));
+# define TRAMPOLINE_WORDS 4
+        trampoline[0] = 0x39600000; /* addi r11,r0,0 */
+        trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */
+        trampoline[2] = 0x7D6903A6; /* mtctr r11 */
+        trampoline[3] = 0x4E800420; /* bctr */
 #endif
-	    hipe_mfa_set_trampoline(m, f, a, trampoline);
-	}
-	trampvec[trampnr-1] = trampoline;
+        trampvec[i] = trampoline;
+        trampoline += TRAMPOLINE_WORDS;
     }
-    curseg.tramp_pos = tramp_pos;
-    return address;
+    hipe_flush_icache_range(address, nrcallees*TRAMPOLINE_WORDS*sizeof(Uint32));
 }
 
 void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
 {
-    Uint nrwords;
+    Uint code_words;
     int nrcallees;
     Eterm trampvecbin;
-    unsigned int **trampvec;
-    unsigned int *address;
-    unsigned int *base;
-    struct segment oldseg;
+    Uint32 **trampvec;
+    Uint32 *address;
 
     if (nrbytes & 0x3)
 	return NULL;
-    nrwords = nrbytes >> 2;
+    code_words = nrbytes / sizeof(Uint32);
 
     nrcallees = check_callees(callees);
     if (nrcallees < 0)
 	return NULL;
-    trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*));
-    trampvec = (unsigned int**)binary_bytes(trampvecbin);
-
-    address = try_alloc(nrwords, nrcallees, callees, trampvec);
-    if (!address) {
-	base = new_code_mapping();
-	if (base == MAP_FAILED)
-	    return NULL;
-	oldseg = curseg;
-	curseg.base = base;
-	curseg.code_pos = base;
-	curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES);
-
-	address = try_alloc(nrwords, nrcallees, callees, trampvec);
-	if (!address) {
-	    munmap(base, SEGMENT_NRBYTES);
-	    curseg = oldseg;
-	    return NULL;
-	}
-	/* commit to new segment, ignore leftover space in old segment */
-    }
+    trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*));
+    trampvec = (Uint32**)binary_bytes(trampvecbin);
+
+    address = erts_alloc(ERTS_ALC_T_HIPE_EXEC,
+                         (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32));
+
+    generate_trampolines(address + code_words, nrcallees, callees, trampvec);
     *trampolines = trampvecbin;
     return address;
 }
 
 void hipe_free_code(void* code, unsigned int bytes)
 {
-    /*SVERK: Leaking code memory */
-}
-
-static unsigned int *alloc_stub(Uint nrwords)
-{
-    unsigned int *address;
-    unsigned int *base;
-    struct segment oldseg;
-
-    address = try_alloc(nrwords, 0, NIL, NULL);
-    if (!address) {
-	base = new_code_mapping();
-	if (base == MAP_FAILED)
-	    return NULL;
-	oldseg = curseg;
-	curseg.base = base;
-	curseg.code_pos = base;
-	curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES);
-
-	address = try_alloc(nrwords, 0, NIL, NULL);
-	if (!address) {
-	    munmap(base, SEGMENT_NRBYTES);
-	    curseg = oldseg;
-	    return NULL;
-	}
-	/* commit to new segment, ignore leftover space in old segment */
-    }
-    return address;
+    erts_free(ERTS_ALC_T_HIPE_EXEC, code);
 }
 
 void hipe_free_native_stub(void* stub)
 {
-    /*SVERK: Leaking code stubs */
+    erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
 }
 
 static void patch_imm16(Uint32 *address, unsigned int imm16)
@@ -298,12 +204,12 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type)
 
 void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
 {
-    unsigned int *code;
+    Uint32 *code;
 
     if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL)
 	abort();
 
-    code = alloc_stub(7);
+    code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 7*sizeof(Uint32));
     if (!code)
 	return NULL;
 
@@ -322,7 +228,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
     /* ba nbif_callemu */
     code[6] = 0x48000002 | (unsigned long)&nbif_callemu;
 
-    hipe_flush_icache_range(code, 7*sizeof(int));
+    hipe_flush_icache_range(code, 7*sizeof(Uint32));
 
     return code;
 }
@@ -370,7 +276,7 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type)
 
 void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
 {
-    unsigned int *code;
+    Uint32 *code;
 
     /*
      * Native code calls BEAM via a stub looking as follows:
@@ -393,7 +299,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
     if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL)
 	abort();
 
-    code = alloc_stub(4);
+    code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 4*sizeof(Uint32));
     if (!code)
 	return NULL;
 
@@ -406,7 +312,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
     /* ba nbif_callemu */
     code[3] = 0x48000002 | (unsigned long)&nbif_callemu;
 
-    hipe_flush_icache_range(code, 4*sizeof(int));
+    hipe_flush_icache_range(code, 4*sizeof(Uint32));
 
     return code;
 }
-- 
cgit v1.2.3


From 6cd1469d0c8f9af602332b20772134c1241f6429 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 6 Oct 2016 20:09:21 +0200
Subject: erts: Fix bug in stack walk on risc

Symptom: Got ra==NULL at nsp==nsp_end
when running basic_SUITE:basic_arith on arm

why has this not been detected before?
---
 erts/emulator/hipe/hipe_risc_stack.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c
index 7528a3ab00..4001bedeb6 100644
--- a/erts/emulator/hipe/hipe_risc_stack.c
+++ b/erts/emulator/hipe/hipe_risc_stack.c
@@ -292,7 +292,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace)
     ra = (unsigned long)p->hipe.nra;
     prev_ra = 0;
     i = 0;
-    for (;;) {
+    while (nsp < nsp_end) {
 	if (ra == (unsigned long)nbif_stack_trap_ra)
 	    ra = (unsigned long)p->hipe.ngra;
 	if (ra != prev_ra) {
@@ -302,8 +302,6 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace)
 		break;
 	    prev_ra = ra;
 	}
-	if (nsp >= nsp_end)
-	    break;
 	sdesc = hipe_find_sdesc(ra);
 	nsp += arity + sdesc_fsize(sdesc);
 	arity = sdesc_arity(sdesc);
-- 
cgit v1.2.3


From deed6619dee9db9c0347b25987f3a91d51424079 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 7 Oct 2016 16:13:55 +0200
Subject: erts: Let code:make_stub_module raise 'notsup'

if hipe is disabled.

Makes the code simpler to just ifdef away
a lot of hipe stuff.
---
 erts/emulator/beam/beam_bif_load.c | 10 +++++++---
 erts/emulator/beam/beam_load.c     | 29 +++++++++--------------------
 2 files changed, 16 insertions(+), 23 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index f5d1ca7e54..0a4d89d51b 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -149,6 +149,9 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1)
 
 BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
 {
+#if !defined(HIPE)
+    BIF_ERROR(BIF_P, EXC_NOTSUP);
+#else
     Module* modp;
     Eterm res, mod;
 
@@ -181,11 +184,9 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
     if (res == mod) {
 	erts_end_staging_code_ix();
 	erts_commit_staging_code_ix();
-#ifdef HIPE
         if (!modp)
 	    modp = erts_get_module(mod, erts_active_code_ix());
         hipe_redirect_to_module(modp);
-#endif
     }
     else {
 	erts_abort_staging_code_ix();
@@ -194,6 +195,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
     erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     erts_release_code_write_permission();
     return res;
+#endif
 }
 
 BIF_RETTYPE
@@ -1069,9 +1071,11 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls)
     BeamInstr* start;
     char* mod_start;
     Uint mod_size;
+    Eterm* sp;
+#ifdef HIPE
     void *nat_start = NULL;
     Uint nat_size = 0;
-    Eterm* sp;
+#endif
 
     *redsp += 1;
 
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 813788f66c..4833bcc6e9 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -481,10 +481,12 @@ static void free_loader_state(Binary* magic);
 static ErlHeapFragment* new_literal_fragment(Uint size);
 static void free_literal_fragment(ErlHeapFragment*);
 static void loader_state_dtor(Binary* magic);
+#ifdef HIPE
 static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 				  Eterm group_leader, Eterm module,
 				  BeamCodeHeader* code_hdr, Uint size,
 				  HipeModule *hipe_code);
+#endif
 static int init_iff_file(LoaderState* stp, byte* code, Uint size);
 static int scan_iff_file(LoaderState* stp, Uint* chunk_types,
 			 Uint num_types, Uint num_mandatory);
@@ -539,8 +541,6 @@ static Eterm compilation_info_for_module(Process* p, BeamCodeHeader*);
 static Eterm md5_of_module(Process* p, BeamCodeHeader*);
 static Eterm has_native(BeamCodeHeader*);
 static Eterm native_addresses(Process* p, BeamCodeHeader*);
-int patch_funentries(Eterm Patchlist);
-int patch(Eterm Addresses, Uint fe);
 static int safe_mul(UWord a, UWord b, UWord* resp);
 
 static int must_swap_floats;
@@ -1098,6 +1098,7 @@ loader_state_dtor(Binary* magic)
     ASSERT(stp->genop_blocks == 0);
 }
 
+#ifdef HIPE
 static Eterm
 stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 		     Eterm group_leader, Eterm module,
@@ -1125,11 +1126,9 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
     modp->curr.code_hdr = code_hdr;
     modp->curr.code_length = size;
     modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */
-#if defined(HIPE)
     DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code "
                   "first_hipe_ref = %p", hipe_code->first_hipe_ref);
     modp->curr.hipe_code = hipe_code;
-#endif
 
     /*
      * Update ranges (used for finding a function from a PC value).
@@ -1138,6 +1137,7 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
     erts_update_ranges((BeamInstr*)modp->curr.code_hdr, size);
     return NIL;
 }
+#endif
 
 static int
 init_iff_file(LoaderState* stp, byte* code, Uint size)
@@ -5997,6 +5997,7 @@ code_module_md5_1(BIF_ALIST_1)
     return res;
 }
 
+#ifdef HIPE
 #define WORDS_PER_FUNCTION 6
 
 static BeamInstr*
@@ -6075,9 +6076,7 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp)
     Eterm mod = fp[2];
     Eterm function = fp[3];
     int arity = fp[4];
-#ifdef HIPE
     Lambda* lp;
-#endif
 
     if (is_bif(mod, function, arity)) {
 	fp[1] = 0;
@@ -6106,7 +6105,6 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp)
      * Search the lambda table to find out which.
      */
     
-#ifdef HIPE
     n = stp->num_lambdas;
     for (i = 0, lp = stp->lambdas; i < n; i++, lp++) {
         ErlFunEntry* fe = stp->lambdas[i].fe;
@@ -6115,7 +6113,6 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp)
             fe->address = &(fp[5]);
 	}
     }
-#endif
     return;
 }
 
@@ -6124,10 +6121,9 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp)
    [{Adr, Patchtyppe} | Addresses]
    and the address of a fun_entry.
 */
-int 
+static int
 patch(Eterm Addresses, Uint fe) 
  {
-#ifdef HIPE
   Eterm* listp;
   Eterm tuple;
   Eterm* tp;
@@ -6163,15 +6159,13 @@ patch(Eterm Addresses, Uint fe)
 
   }
 
-#endif
   return 1;
 }
 
 
-int
+static int
 patch_funentries(Eterm Patchlist) 
  {
-#ifdef HIPE   
   while (!is_nil(Patchlist)) {
     Eterm Info;
     Eterm MFA;
@@ -6260,18 +6254,15 @@ patch_funentries(Eterm Patchlist)
       return 0;
 
   }
-#endif
   return 1; /* Signal that all went well */
 }
 
-
 /*
  * Do a dummy load of a module. No threaded code will be loaded.
  * Used for loading native code.
  * Will also patch all references to fun_entries to point to 
  * the new fun_entries created.
  */
-
 Eterm
 erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
 {
@@ -6444,11 +6435,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
 	 * as the body until we know what kind of trap we should put there.
 	 */
 	code_hdr->functions[i] = fp;
-#ifdef HIPE
 	op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */
-#else
-	op = (Eterm) BeamOpCode(op_move_return_n);
-#endif
 	fp = make_stub(fp, hipe_stp->module, func, arity, (Uint)native_address,
 		       op);
     }
@@ -6580,6 +6567,8 @@ int erts_commit_hipe_patch_load(Eterm hipe_magic_bin)
 }
 
 #undef WORDS_PER_FUNCTION
+#endif /* HIPE */
+
 
 static int safe_mul(UWord a, UWord b, UWord* resp)
 {
-- 
cgit v1.2.3


From 9aa58e5e93af074d1e5e54848c29d40d826de361 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 7 Oct 2016 16:18:07 +0200
Subject: erts: Remove code_SUITE:make_stub and make_stub_many_funs

Hard to unit test now when it takes a magic HipeLoaderState
as argument. All hipe testing should be enough exercise for
code:make_stub_module.
---
 erts/emulator/test/code_SUITE.erl | 69 ++-------------------------------------
 1 file changed, 3 insertions(+), 66 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index b76ba6427f..465ddfa7a8 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -23,8 +23,8 @@
          versions/1,new_binary_types/1, call_purged_fun_code_gone/1,
 	 call_purged_fun_code_reload/1, call_purged_fun_code_there/1,
          multi_proc_purge/1, t_check_old_code/1,
-         external_fun/1,get_chunk/1,module_md5/1,make_stub/1,
-         make_stub_many_funs/1,constant_pools/1,constant_refc_binaries/1,
+         external_fun/1,get_chunk/1,module_md5/1,
+         constant_pools/1,constant_refc_binaries/1,
          false_dependency/1,coverage/1,fun_confusion/1,
          t_copy_literals/1, t_copy_literals_frags/1]).
 
@@ -37,7 +37,7 @@ all() ->
     [versions, new_binary_types, call_purged_fun_code_gone,
      call_purged_fun_code_reload, call_purged_fun_code_there,
      multi_proc_purge, t_check_old_code, external_fun, get_chunk,
-     module_md5, make_stub, make_stub_many_funs,
+     module_md5,
      constant_pools, constant_refc_binaries, false_dependency,
      coverage, fun_confusion, t_copy_literals, t_copy_literals_frags].
 
@@ -442,69 +442,6 @@ module_md5_ok(Code) ->
     end.
 
 
-make_stub(Config) when is_list(Config) ->
-    catch erlang:purge_module(my_code_test),
-    MD5 = erlang:md5(<<>>),
-    Arg3 = {[], [], MD5, 0, 0},
-
-    Data = proplists:get_value(data_dir, Config),
-    File = filename:join(Data, "my_code_test"),
-    {ok,my_code_test,Code} = compile:file(File, [binary]),
-
-    my_code_test = code:make_stub_module(my_code_test, Code, Arg3),
-    true = erlang:delete_module(my_code_test),
-    true = erlang:purge_module(my_code_test),
-
-    my_code_test = code:make_stub_module(my_code_test, 
-                                         make_unaligned_sub_binary(Code),
-                                         Arg3),
-    true = erlang:delete_module(my_code_test),
-    true = erlang:purge_module(my_code_test),
-
-    my_code_test = code:make_stub_module(my_code_test, zlib:gzip(Code),
-                                         Arg3),
-    true = erlang:delete_module(my_code_test),
-    true = erlang:purge_module(my_code_test),
-
-    %% Should fail.
-    {'EXIT',{badarg,_}} =
-    (catch code:make_stub_module(my_code_test, <<"bad">>, Arg3)),
-    {'EXIT',{badarg,_}} =
-    (catch code:make_stub_module(my_code_test,
-                                 bit_sized_binary(Code),
-                                 Arg3)),
-    {'EXIT',{badarg,_}} =
-    (catch code:make_stub_module(my_code_test_with_wrong_name,
-                                 Code, Arg3)),
-    ok.
-
-make_stub_many_funs(Config) when is_list(Config) ->
-    catch erlang:purge_module(many_funs),
-    MD5 = erlang:md5(<<>>),
-    Arg3 = {[], [], MD5, 0, 0},
-
-    Data = proplists:get_value(data_dir, Config),
-    File = filename:join(Data, "many_funs"),
-    {ok,many_funs,Code} = compile:file(File, [binary]),
-
-    many_funs = code:make_stub_module(many_funs, Code, Arg3),
-    true = erlang:delete_module(many_funs),
-    true = erlang:purge_module(many_funs),
-    many_funs = code:make_stub_module(many_funs, 
-                                      make_unaligned_sub_binary(Code),
-                                      Arg3),
-    true = erlang:delete_module(many_funs),
-    true = erlang:purge_module(many_funs),
-
-    %% Should fail.
-    {'EXIT',{badarg,_}} =
-    (catch code:make_stub_module(many_funs, <<"bad">>, Arg3)),
-    {'EXIT',{badarg,_}} =
-    (catch code:make_stub_module(many_funs,
-                                 bit_sized_binary(Code),
-                                 Arg3)),
-    ok.
-
 constant_pools(Config) when is_list(Config) ->
     Data = proplists:get_value(data_dir, Config),
     File = filename:join(Data, "literals"),
-- 
cgit v1.2.3


From df9e09fc1f938f9902052dcb896b42558ee9779d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 7 Oct 2016 15:54:33 +0200
Subject: erts: Remove dead alloc stats in hipe_amd64.c

erlang:system_info({allocator, exec_alloc})
have stats if we need it.
---
 erts/emulator/hipe/hipe_amd64.c | 32 --------------------------------
 1 file changed, 32 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c
index 96b8c22265..3173afad0a 100644
--- a/erts/emulator/hipe/hipe_amd64.c
+++ b/erts/emulator/hipe/hipe_amd64.c
@@ -83,29 +83,6 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
     return 0;
 }
 
-#if 0	/* change to non-zero to get allocation statistics at exit() */
-static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost;
-static unsigned int atexit_done;
-
-static void alloc_code_stats(void)
-{
-    printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n",
-	   total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost);
-}
-
-static void atexit_alloc_code_stats(void)
-{
-    if (!atexit_done) {
-	atexit_done = 1;
-	(void)atexit(alloc_code_stats);
-    }
-}
-
-#define ALLOC_CODE_STATS(X)	do{X;}while(0)
-#else
-#define ALLOC_CODE_STATS(X)	do{}while(0)
-#endif
-
 /*
  * Memory allocator for executable code.
  *
@@ -116,9 +93,6 @@ static void atexit_alloc_code_stats(void)
  */
 static void *alloc_code(unsigned int alloc_bytes)
 {
-    ALLOC_CODE_STATS(++nr_allocs);
-    ALLOC_CODE_STATS(total_alloc += alloc_bytes);
-
     return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes);
 }
 
@@ -132,9 +106,6 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
 
 void hipe_free_code(void* code, unsigned int bytes)
 {
-    ALLOC_CODE_STATS(--nr_allocs);
-    ALLOC_CODE_STATS(total_alloc -= bytes);
-
     erts_free(ERTS_ALC_T_HIPE_EXEC, code);
 }
 
@@ -244,9 +215,6 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
 
 void hipe_free_native_stub(void* stub)
 {
-    ALLOC_CODE_STATS(++nr_allocs);
-    /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/
-
     erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
 }
 
-- 
cgit v1.2.3


From 8bb80fe76f5b4debe981e42ba70260c5a12b250f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 14 Oct 2016 14:43:22 +0200
Subject: erts: Cleanup hipe trampoline code

---
 erts/emulator/hipe/hipe_amd64.c |  4 +--
 erts/emulator/hipe/hipe_bif0.c  | 70 +++++++----------------------------------
 erts/emulator/hipe/hipe_bif0.h  |  8 -----
 erts/emulator/hipe/hipe_sparc.c |  4 +--
 erts/emulator/hipe/hipe_x86.c   |  4 +--
 5 files changed, 18 insertions(+), 72 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c
index 3173afad0a..e3cff4a4ba 100644
--- a/erts/emulator/hipe/hipe_amd64.c
+++ b/erts/emulator/hipe/hipe_amd64.c
@@ -73,8 +73,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
 {
     Sint rel32;
 
-    if (trampoline)
-	return -1;
+    ASSERT(trampoline == NULL);
+
     rel32 = (Sint)destAddress - (Sint)callAddress - 4;
     if ((Sint)(Sint32)rel32 != rel32)
 	return -1;
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index f9a657124a..0402a4de98 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -819,9 +819,6 @@ BIF_RETTYPE hipe_bifs_bif_address_3(BIF_ALIST_3)
 struct primop {
     HashBucket bucket;	/* bucket.hvalue == atom_val(name) */
     const void *address;
-#if defined(__arm__)
-    void *trampoline;
-#endif
 };
 
 static struct primop primops[] = {
@@ -880,29 +877,6 @@ static struct primop *primop_table_get(Eterm name)
     return hash_get(&primop_table, &tmpl);
 }
 
-#if defined(__arm__)
-static struct primop *primop_table_put(Eterm name)
-{
-    struct primop tmpl;
-
-    init_primop_table();
-    tmpl.bucket.hvalue = atom_val(name);
-    return hash_put(&primop_table, &tmpl);
-}
-
-void *hipe_primop_get_trampoline(Eterm name)
-{
-    struct primop *primop = primop_table_get(name);
-    return primop ? primop->trampoline : NULL;
-}
-
-void hipe_primop_set_trampoline(Eterm name, void *trampoline)
-{
-    struct primop *primop = primop_table_put(name);
-    primop->trampoline = trampoline;
-}
-#endif
-
 /*
  * hipe_bifs_primop_address(Atom) -> address or false
  */
@@ -1062,9 +1036,6 @@ struct hipe_mfa_info {
     void *new_address;
     struct hipe_ref_head callers;   /* sentinel in list of hipe_ref's */
     struct hipe_mfa_info* next_in_mod;
-#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
-    void *trampoline;
-#endif
 #ifdef DEBUG
     Export* dbg_export;
 #endif
@@ -1094,7 +1065,9 @@ static struct {
 struct hipe_ref {
     struct hipe_ref_head head;    /* list of refs to same calleee */
     void *address;
+#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
     void *trampoline;
+#endif
     unsigned int flags;
     struct hipe_ref* next_from_modi; /* list of refs from same module instance */
 #if defined(DEBUG)
@@ -1181,9 +1154,6 @@ static struct hipe_mfa_info *hipe_mfa_info_table_alloc(Eterm m, Eterm f, unsigne
     res->callers.next = &res->callers;
     res->callers.prev = &res->callers;
     res->next_in_mod = NULL;
-#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
-    res->trampoline = NULL;
-#endif
 #ifdef DEBUG
     res->dbg_export = NULL;
 #endif
@@ -1294,30 +1264,6 @@ static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address)
     hipe_mfa_info_table_rwunlock();
 }
 
-#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
-void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int arity)
-{
-    struct hipe_mfa_info *p;
-    void *trampoline;
-
-    hipe_mfa_info_table_rlock();
-    p = hipe_mfa_info_table_get_locked(m, f, arity);
-    trampoline = p ? p->trampoline : NULL;
-    hipe_mfa_info_table_runlock();
-    return trampoline;
-}
-
-void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int arity, void *trampoline)
-{
-    struct hipe_mfa_info *p;
-
-    hipe_mfa_info_table_rwlock();
-    p = hipe_mfa_info_table_put_rwlocked(m, f, arity);
-    p->trampoline = trampoline;
-    hipe_mfa_info_table_rwunlock();
-}
-#endif
-
 BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3)
 {
     struct hipe_mfa mfa;
@@ -1531,7 +1477,9 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
 
     ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref));
     ref->address = address;
+#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
     ref->trampoline = trampoline;
+#endif
     ref->flags = flags;
 
     /*
@@ -1815,8 +1763,14 @@ void hipe_redirect_to_module(Module* modp)
 
 	    if (ref->flags & REF_FLAG_IS_LOAD_MFA)
 		res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa);
-	    else
-		res = hipe_patch_call(ref->address, p->remote_address, ref->trampoline);
+	    else {
+#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
+                void* trampoline = ref->trampoline;
+#else
+                void* trampoline = NULL;
+#endif
+		res = hipe_patch_call(ref->address, p->remote_address, trampoline);
+            }
 	    if (res)
 		fprintf(stderr, "%s: patch failed", __FUNCTION__);
 	}
diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h
index 2b7c01b975..02f0d6c823 100644
--- a/erts/emulator/hipe/hipe_bif0.h
+++ b/erts/emulator/hipe/hipe_bif0.h
@@ -32,14 +32,6 @@ extern void hipe_mfa_info_table_init(void);
 extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a);
 extern BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3);
 extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a);
-#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
-extern void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int a);
-extern void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int a, void *trampoline);
-#endif
-#if defined(__arm__)
-extern void *hipe_primop_get_trampoline(Eterm name);
-extern void hipe_primop_set_trampoline(Eterm name, void *trampoline);
-#endif
 
 /* needed in beam_load.c */
 int hipe_need_blocking(Module*);
diff --git a/erts/emulator/hipe/hipe_sparc.c b/erts/emulator/hipe/hipe_sparc.c
index 75bfb4fef1..876b20bb15 100644
--- a/erts/emulator/hipe/hipe_sparc.c
+++ b/erts/emulator/hipe/hipe_sparc.c
@@ -87,8 +87,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
 {
     Uint32 relDest, newI;
 
-    if (trampoline)
-	return -1;
+    ASSERT(trampoline == NULL);
+
     relDest = (Uint32)((Sint32)destAddress - (Sint32)callAddress);
     newI = (1 << 30) | (relDest >> 2);
     *(Uint32*)callAddress = newI;
diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c
index b091bee6fb..c7e24673ac 100644
--- a/erts/emulator/hipe/hipe_x86.c
+++ b/erts/emulator/hipe/hipe_x86.c
@@ -61,8 +61,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
 {
     Uint rel32;
 
-    if (trampoline)
-	return -1;
+    ASSERT(trampoline == NULL);
+
     rel32 = (Uint)destAddress - (Uint)callAddress - 4;
     *(Uint32*)callAddress = rel32;
     hipe_flush_icache_word(callAddress);
-- 
cgit v1.2.3


From 7d5021bc9e1150b5d839438756851b3175f3ba85 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 12 Oct 2016 16:56:41 +0200
Subject: erts: Disable DBG_TRACE_MFA for debug build

---
 erts/emulator/beam/beam_load.h | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 5f0f34fcdd..43cf6597df 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -151,10 +151,16 @@ struct BeamCodeLineTab_ {
 #define LOC_FILE(Loc) ((Loc) >> 24)
 #define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1))
 
-#ifdef DEBUG
-# define ENABLE_DBG_TRACE_MFA
-#endif
 
+/*
+ * MFA event debug "tracing" usage:
+ *
+ * #define ENABLE_DBG_TRACE_MFA
+ * call dbg_set_traced_mfa("mymod","myfunc",arity)
+ * for the function(s) to trace, in some init function.
+ *
+ * Run and get stderr printouts when interesting things happen to your MFA.
+ */
 #ifdef  ENABLE_DBG_TRACE_MFA
 
 void dbg_set_traced_mfa(const char* m, const char* f, Uint a);
-- 
cgit v1.2.3


From 3582e6f420d84ddac64b55cb13100f1ae7f31b08 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 13 Oct 2016 20:59:58 +0200
Subject: erts: Replace unsafe Module.first_hipe_ref

with hash table mod2mfa_tab
---
 erts/emulator/beam/beam_bif_load.c |   2 +-
 erts/emulator/beam/module.c        |   7 +-
 erts/emulator/beam/module.h        |   3 -
 erts/emulator/hipe/hipe_bif0.c     | 196 ++++++++++++++++++++++++++++---------
 erts/emulator/hipe/hipe_bif0.h     |   6 +-
 erts/emulator/hipe/hipe_load.c     |   4 +-
 6 files changed, 155 insertions(+), 63 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 0a4d89d51b..c31abbb84f 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1711,7 +1711,7 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 		modp->old.catches = BEAM_CATCHES_NIL;
 		erts_remove_from_ranges(code);
 #ifdef HIPE
-		hipe_purge_module(modp);
+		hipe_purge_module(modp, is_blocking);
 #endif
 		ERTS_BIF_PREP_RET(ret, am_true);
 	    }
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index e9b18ba525..3e622c0f3a 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -91,9 +91,6 @@ static Module* module_alloc(Module* tmpl)
     erts_module_instance_init(&obj->curr);
     erts_module_instance_init(&obj->old);
     obj->on_load = 0;
-#ifdef HIPE
-    obj->first_hipe_mfa = NULL;
-#endif
     DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc");
     return obj;
 }
@@ -128,6 +125,7 @@ void init_module_table(void)
     erts_smp_atomic_init_nob(&tot_module_bytes, 0);
 }
 
+
 Module*
 erts_get_module(Eterm mod, ErtsCodeIndex code_ix)
 {
@@ -208,9 +206,6 @@ static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod)
     dst_mod->curr = src_mod->curr;
     dst_mod->old = src_mod->old;
     dst_mod->on_load = src_mod->on_load;
-#ifdef HIPE
-    dst_mod->first_hipe_mfa = src_mod->first_hipe_mfa;
-#endif
 }
 
 void module_start_staging(void)
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index 9de610ccf7..1d6d9dd463 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -47,9 +47,6 @@ typedef struct erl_module {
     struct erl_module_instance curr;
     struct erl_module_instance old; /* protected by "old_code" rwlock */
     struct erl_module_instance* on_load;
-#ifdef HIPE
-    struct hipe_mfa_info* first_hipe_mfa;
-#endif
 } Module; 
 
 void erts_module_instance_init(struct erl_module_instance* modi);
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 0402a4de98..4c9757e8d3 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1024,6 +1024,7 @@ struct hipe_ref_head {
  * - maps MFA to most recent trampoline [if powerpc or arm]
  */
 struct hipe_mfa_info {
+    HashBucket mod2mfa;
     struct {
 	unsigned long hvalue;
 	struct hipe_mfa_info *next;
@@ -1057,6 +1058,61 @@ static struct {
     erts_smp_rwmtx_t lock;
 } hipe_mfa_info_table;
 
+Hash mod2mfa_tab; /* map from module atom to list of hipe_mfa_info */
+
+static HashValue mod2mfa_hash(struct hipe_mfa_info* mfa)
+{
+    return mfa->mod2mfa.hvalue;
+}
+
+static int mod2mfa_cmp(HashBucket* tmpl, struct hipe_mfa_info* mfa)
+{
+    return tmpl->hvalue != mfa->mod2mfa.hvalue;
+}
+
+static struct hipe_mfa_info* mod2mfa_alloc(struct hipe_mfa_info* tmpl)
+{
+    return tmpl; /* hash_put always use mfa itself at template */
+}
+
+static void mod2mfa_free(struct hipe_mfa_info* mfa)
+{
+}
+
+static void mod2mfa_tab_init(void)
+{
+    HashFunctions f;
+    static int init_done = 0;
+
+    if (init_done)
+	return;
+    init_done = 1;
+
+    f.hash = (H_FUN) mod2mfa_hash;
+    f.cmp = (HCMP_FUN) mod2mfa_cmp;
+    f.alloc = (HALLOC_FUN) mod2mfa_alloc;
+    f.free = (HFREE_FUN) mod2mfa_free;
+    f.meta_alloc = (HMALLOC_FUN) erts_alloc;
+    f.meta_free = (HMFREE_FUN) erts_free;
+    f.meta_print = (HMPRINT_FUN) erts_print;
+
+    hash_init(ERTS_ALC_T_HIPE, &mod2mfa_tab, "mod2mfa_tab", 50, f);
+}
+
+static struct hipe_mfa_info* mod2mfa_get(Module* modp)
+{
+    HashBucket tmpl;
+    tmpl.hvalue = modp->module;
+    return hash_get(&mod2mfa_tab, &tmpl);
+}
+
+static struct hipe_mfa_info* mod2mfa_put(struct hipe_mfa_info* mfa)
+{
+    mfa->mod2mfa.hvalue = atom_val(mfa->m);
+    return hash_put(&mod2mfa_tab, mfa);
+}
+
+
 
 /*
  * An external native call site M:F(...)
@@ -1103,6 +1159,16 @@ static inline void hipe_mfa_info_table_rwunlock(void)
     erts_smp_rwmtx_rwunlock(&hipe_mfa_info_table.lock);
 }
 
+static ERTS_INLINE
+struct hipe_mfa_info* mod2mfa_get_safe(Module* modp)
+{
+    struct hipe_mfa_info* mfa;
+    hipe_mfa_info_table_rlock();
+    mfa = mod2mfa_get(modp);
+    hipe_mfa_info_table_runlock();
+    return mfa;
+}
+
 #define HIPE_MFA_HASH(M,F,A)	(atom_val(M) ^ atom_val(F) ^ (A))
 
 static struct hipe_mfa_info **hipe_mfa_info_table_alloc_bucket(unsigned int size)
@@ -1173,6 +1239,8 @@ void hipe_mfa_info_table_init(void)
     hipe_mfa_info_table.bucket = hipe_mfa_info_table_alloc_bucket(size);
 
     hipe_mfa_info_table_init_lock();
+
+    mod2mfa_tab_init();
 }
 
 static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eterm f, unsigned int arity)
@@ -1199,8 +1267,8 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f,
     unsigned long h;
     unsigned int i;
     struct hipe_mfa_info *p;
+    struct hipe_mfa_info *first_in_mod;
     unsigned int size;
-    Module* modp;
 
     h = HIPE_MFA_HASH(m, f, arity);
     i = h & hipe_mfa_info_table.mask;
@@ -1221,10 +1289,14 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f,
     if (hipe_mfa_info_table.used > (4*size/5))		/* rehash at 80% */
 	hipe_mfa_info_table_grow();
 
-    modp = erts_put_active_module(m);
-    ASSERT(modp);
-    p->next_in_mod = modp->first_hipe_mfa;
-    modp->first_hipe_mfa = p;
+    first_in_mod = mod2mfa_put(p);
+    if (p != first_in_mod) {
+        p->next_in_mod = first_in_mod->next_in_mod;
+        first_in_mod->next_in_mod = p;
+    }
+    else {
+        p->next_in_mod = NULL;
+    }
 
     DBG_TRACE_MFA(m,f,arity, "hipe_mfa_info allocated at %p", p);
 
@@ -1288,7 +1360,7 @@ BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3)
 
 
 /* Ask if we need to block all threads
- * while loading/deleting (beam) code for this module?
+ * while loading/deleting code for this module?
  */
 int hipe_need_blocking(Module* modp)
 {
@@ -1298,7 +1370,7 @@ int hipe_need_blocking(Module* modp)
      * or native code to make unaccessible.
      */
     hipe_mfa_info_table_rlock();
-    for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) {
+    for (p = mod2mfa_get(modp); p; p = p->next_in_mod) {
         ASSERT(!p->new_address);
         if (p->callers.next != &p->callers || !p->is_stub) {
             break;
@@ -1519,19 +1591,25 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
 
 static void unlink_mfa_from_mod(struct hipe_mfa_info* unlink_me)
 {
-    Module* modp = erts_get_module(unlink_me->m, erts_active_code_ix());
-    struct hipe_mfa_info** prevp = &modp->first_hipe_mfa;
     struct hipe_mfa_info* p;
 
-    ASSERT(modp);
-    for (;;) {
-        p = *prevp;
-        ASSERT(p && p->m == unlink_me->m);
-        if (p == unlink_me) {
-            *prevp = p->next_in_mod;
-            break;
-        }
-         prevp = &p->next_in_mod;
+    p = hash_get(&mod2mfa_tab, unlink_me);
+    ASSERT(p);
+    if (p == unlink_me) {
+        hash_erase(&mod2mfa_tab, p);
+        if (p->next_in_mod)
+            mod2mfa_put(p->next_in_mod);
+    }
+    else {
+        struct hipe_mfa_info** prevp;
+
+        do {
+            prevp = &p->next_in_mod;
+            p = *prevp;
+            ASSERT(p && p->m == unlink_me->m);
+        } while (p != unlink_me);
+
+        *prevp = p->next_in_mod;
     }
 }
 
@@ -1561,18 +1639,11 @@ static void hipe_purge_all_refs(void)
     for (i = 0; i < nrbuckets; ++i) {	
         ASSERT(bucket[i] == NULL);
 	while (bucket[i] != NULL) {
-            Module* modp;
 	    struct hipe_mfa_info* mfa = bucket[i];
 	    bucket[i] = mfa->bucket.next;
 
-	    ASSERT(mfa->callers.next == &mfa->callers);
-
-            modp = erts_get_module(mfa->m, erts_active_code_ix());
-            if (modp) {
-                /* unsafe write to active module */
-                modp->first_hipe_mfa = NULL;
-            }
-	    erts_free(ERTS_ALC_T_HIPE, mfa);
+            hash_erase(&mod2mfa_tab, mfa);
+            erts_free(ERTS_ALC_T_HIPE, mfa);
 	}
     }
     hipe_mfa_info_table.used = 0;
@@ -1598,18 +1669,22 @@ int hipe_purge_need_blocking(Module* modp)
             modp->old.hipe_code->first_hipe_sdesc)
             return 1;
     }
-    return !modp->curr.code_hdr && modp->first_hipe_mfa;
+    if (!modp->curr.code_hdr) {
+        return mod2mfa_get_safe(modp) != NULL;
+    }
+    return 0;
 }
 
-void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module)
+void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module,
+                     int is_blocking)
 {
     struct hipe_ref* ref = first_ref;
 
+    ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking());
+
     while (ref) {
         struct hipe_ref* free_ref = ref;
 
-        ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
-
         DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref,
                       ref->callee->m, ref->callee->f, ref->callee->a);
         DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref,
@@ -1632,8 +1707,12 @@ void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module)
         if (ref->head.next == ref->head.prev) {
             struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers);
             if (p->is_stub) {
+                if (!is_blocking)
+                    hipe_mfa_info_table_rwlock();
                 unlink_mfa_from_mod(p);
                 purge_mfa(p);
+                if (!is_blocking)
+                    hipe_mfa_info_table_rwunlock();
             }
         }
 
@@ -1642,14 +1721,18 @@ void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module)
     }
 }
 
-void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module)
+void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module,
+                       int is_blocking)
 {
     struct hipe_sdesc* sdesc = first_sdesc;
+
+    ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking());
+
+    ERTS_SMP_LC_ASSERT(is_blocking); /*XXX Fix safe sdesc destruction */
+
     while (sdesc) {
         struct hipe_sdesc* free_sdesc = sdesc;
 
-        ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
-
         DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue);
         ASSERT(make_atom(sdesc->m_aix) == module);
 
@@ -1659,10 +1742,12 @@ void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module)
 }
 
 
-void hipe_purge_module(Module* modp)
+void hipe_purge_module(Module* modp, int is_blocking)
 {
     ASSERT(modp);
 
+    ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking());
+
     DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module");
 
     if (modp->old.hipe_code) {
@@ -1670,8 +1755,10 @@ void hipe_purge_module(Module* modp)
          * Remove all hipe_ref's (external calls) from the old module instance
          */
         if (modp->old.hipe_code->first_hipe_ref) {
+            ERTS_SMP_LC_ASSERT(is_blocking);
+
             hipe_purge_refs(modp->old.hipe_code->first_hipe_ref,
-                            make_atom(modp->module));
+                            make_atom(modp->module), is_blocking);
             modp->old.hipe_code->first_hipe_ref = NULL;
         }
 
@@ -1679,8 +1766,10 @@ void hipe_purge_module(Module* modp)
          * Remove all hipe_sdesc's for the old module instance
          */
         if (modp->old.hipe_code->first_hipe_sdesc) {
+            ERTS_SMP_LC_ASSERT(is_blocking);
+
             hipe_purge_sdescs(modp->old.hipe_code->first_hipe_sdesc,
-                             make_atom(modp->module));
+                             make_atom(modp->module), is_blocking);
             modp->old.hipe_code->first_hipe_sdesc = NULL;
         }
 
@@ -1693,17 +1782,28 @@ void hipe_purge_module(Module* modp)
      * Remove unreferred hipe_mfa_info's 
      * when all module instances are removed (like in init:restart) 
      */
-    if (modp->curr.code_hdr == NULL) {
-        struct hipe_mfa_info** prevp = &modp->first_hipe_mfa;
-        struct hipe_mfa_info* p = *prevp;
-        ERTS_SMP_LC_ASSERT(!p || erts_smp_thr_progress_is_blocking());
-        for (; p; p = *prevp) {
-            if (p->callers.next == &p->callers) {
-                *prevp = p->next_in_mod;
-                purge_mfa(p);
+    if (is_blocking && modp->curr.code_hdr == NULL) {
+        struct hipe_mfa_info* was_first = mod2mfa_get(modp);
+        struct hipe_mfa_info* is_first = was_first;
+        struct hipe_mfa_info** prevp = &is_first;
+        struct hipe_mfa_info *p;
+
+        if (was_first) {
+            for (p = was_first ; p; p = *prevp) {
+                if (p->callers.next == &p->callers) {
+                    *prevp = p->next_in_mod;
+                    if (p != was_first)
+                        purge_mfa(p);
+                }
+                else
+                    prevp = &p->next_in_mod;
+            }
+            if (was_first != is_first) {
+                hash_erase(&mod2mfa_tab, was_first);
+                purge_mfa(was_first);
+                if (is_first)
+                    mod2mfa_put(is_first); 
             }
-            else
-                prevp = &p->next_in_mod;
         }
     }
 }
@@ -1719,7 +1819,7 @@ void hipe_redirect_to_module(Module* modp)
 
     ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
 
-    for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) {
+    for (p = mod2mfa_get(modp); p; p = p->next_in_mod) {
         if (p->new_address) {
             if (p->is_stub) {
                 hipe_free_native_stub(p->remote_address);
diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h
index 02f0d6c823..4a59bacc6e 100644
--- a/erts/emulator/hipe/hipe_bif0.h
+++ b/erts/emulator/hipe/hipe_bif0.h
@@ -36,9 +36,9 @@ extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned in
 /* needed in beam_load.c */
 int hipe_need_blocking(Module*);
 int hipe_purge_need_blocking(Module*);
-void hipe_purge_refs(struct hipe_ref*, Eterm);
-void hipe_purge_sdescs(struct hipe_sdesc*, Eterm);
-void hipe_purge_module(Module*);
+void hipe_purge_refs(struct hipe_ref*, Eterm, int is_blocking);
+void hipe_purge_sdescs(struct hipe_sdesc*, Eterm, int is_blocking);
+void hipe_purge_module(Module*, int is_blocking);
 void hipe_redirect_to_module(Module* modp);
 
 /* these are also needed in hipe_amd64.c */
diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
index fcbcf1a6a4..2998ed87a2 100644
--- a/erts/emulator/hipe/hipe_load.c
+++ b/erts/emulator/hipe/hipe_load.c
@@ -50,11 +50,11 @@ void hipe_free_loader_state(HipeLoaderState *stp)
     stp->data_segment_size = 0;
 
     if (stp->new_hipe_refs) {
-        hipe_purge_refs(stp->new_hipe_refs, stp->module);
+        hipe_purge_refs(stp->new_hipe_refs, stp->module, 0);
         stp->new_hipe_refs = NULL;
     }
     if (stp->new_hipe_sdesc) {
-        hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module);
+        hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module, 0);
         stp->new_hipe_sdesc = NULL;
     }
 
-- 
cgit v1.2.3


From a28149b2500a4db9b3c56b168dcd18effca87a3a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 17 Oct 2016 11:50:01 +0200
Subject: erts: Cleanup dead code

---
 erts/emulator/beam/module.c | 9 ---------
 erts/emulator/beam/module.h | 1 -
 2 files changed, 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index 3e622c0f3a..705dd1d43f 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -171,15 +171,6 @@ erts_put_module(Eterm mod)
     return put_module(mod, &module_tables[erts_staging_code_ix()]);
 }
 
-Module*
-erts_put_active_module(Eterm mod)
-{
-    ASSERT(is_atom(mod));
-    //SVERK Why not? ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
-
-    return put_module(mod, &module_tables[erts_active_code_ix()]);
-}
-
 Module *module_code(int i, ErtsCodeIndex code_ix)
 {
     return (Module*) erts_index_lookup(&module_tables[code_ix], i);
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index 1d6d9dd463..8f44e11f3d 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -52,7 +52,6 @@ typedef struct erl_module {
 void erts_module_instance_init(struct erl_module_instance* modi);
 Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix);
 Module* erts_put_module(Eterm mod);
-Module* erts_put_active_module(Eterm mod); /* only while blocked */
 
 void init_module_table(void);
 void module_start_staging(void);
-- 
cgit v1.2.3


From 0c480fab8830febf8911d2eb6618a144dc90b8cc Mon Sep 17 00:00:00 2001
From: Kostis Sagonas <kostis@it.uu.se>
Date: Tue, 18 Oct 2016 13:08:17 +0200
Subject: Change the return type of hipe_bifs:add_ref/2

---
 erts/emulator/hipe/hipe_bif0.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 453a452590..57fbbd9403 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1582,7 +1582,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
 		  ref, callee.mod, callee.fun, callee.ari, ref->address);
     DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p)",
 		  ref, caller.mod, caller.fun, caller.ari, ref->address);
-    BIF_RET(NIL);
+    BIF_RET(am_ok);
 
  badarg:
     BIF_ERROR(BIF_P, BADARG);
-- 
cgit v1.2.3


From 79abd75445dec4f1932868fcc52238a84d94cce0 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 21 Oct 2016 16:42:30 +0200
Subject: erts: Use mprotect for hipe on all arch except x86_64

Only x86_64 needs to reserve low virtual memory for its
amd64 small code model.
---
 erts/emulator/sys/common/erl_mmap.h | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h
index 92e9eb9e41..1049f7affc 100644
--- a/erts/emulator/sys/common/erl_mmap.h
+++ b/erts/emulator/sys/common/erl_mmap.h
@@ -157,13 +157,23 @@ Eterm erts_mmap_info_options(ErtsMemMapper*,
 #  include "erl_alloc_types.h"
 
 extern ErtsMemMapper erts_dflt_mmapper;
-#  if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+
+# if defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+
+#  if defined(ARCH_64)
 extern ErtsMemMapper erts_literal_mmapper;
-#    ifdef ERTS_ALC_A_EXEC
-#      define ERTS_HAVE_EXEC_MMAPPER
+#  endif
+
+#  if defined(ERTS_ALC_A_EXEC) && defined(__x86_64__)
+   /*
+    * On x86_64, exec_alloc employs its own super carrier 'erts_exec_mmaper'
+    * to ensure low memory for HiPE AMD64 small code model.
+    */
+#   define ERTS_HAVE_EXEC_MMAPPER
 extern ErtsMemMapper erts_exec_mmapper;
-#    endif
 #  endif
+
+# endif /* ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */
 #endif /* ERTS_WANT_MEM_MAPPERS */
 
 /*#define HARD_DEBUG_MSEG*/
-- 
cgit v1.2.3


From 6408400fc1ca9609169f6a0450f1c4671aced91c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 26 Oct 2016 16:42:05 +0200
Subject: erts: Avoid delayed fun undef race

Avoid suspending fun caller not just if purge is already done
but also if purge of another module has started. Another purge
of the same module again cannot happen as making current to old
transition includes thread progress.
---
 erts/emulator/beam/beam_bif_load.c | 4 ++--
 erts/emulator/beam/beam_emu.c      | 2 +-
 erts/emulator/beam/beam_load.h     | 3 ++-
 3 files changed, 5 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 5969197168..8af7703f51 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1623,10 +1623,10 @@ erts_purge_state_add_fun(ErlFunEntry *fe)
 }
 
 Export *
-erts_suspend_process_on_pending_purge_lambda(Process *c_p)
+erts_suspend_process_on_pending_purge_lambda(Process *c_p, ErlFunEntry* fe)
 {
     erts_smp_mtx_lock(&purge_state.mtx);
-    if (is_value(purge_state.module)) {
+    if (purge_state.module == fe->module) {
 	/*
 	 * The process c_p is about to call a fun in the code
 	 * that we are trying to purge. Suspend it and call
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index ef4cdf9d5a..3be5c0d24c 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6555,7 +6555,7 @@ call_fun(Process* p,		/* Current process. */
 		     * and let it try again when the purge operation is
 		     * done (may succeed or not).
 		     */
-		    ep = erts_suspend_process_on_pending_purge_lambda(p);
+		    ep = erts_suspend_process_on_pending_purge_lambda(p, fe);
 		    ASSERT(ep);
 		}
 		else {
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 1200bb9c6f..9be5e14e40 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -124,7 +124,8 @@ int erts_is_module_native(BeamCodeHeader* code);
 void erts_beam_bif_load_init(void);
 struct erl_fun_entry;
 void erts_purge_state_add_fun(struct erl_fun_entry *fe);
-Export *erts_suspend_process_on_pending_purge_lambda(Process *c_p);
+Export *erts_suspend_process_on_pending_purge_lambda(Process *c_p,
+                                                     struct erl_fun_entry*);
 
 /*
  * Layout of the line table.
-- 
cgit v1.2.3


From 3223059923314313ffff9c7b09ab0a73bf18238b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 26 Oct 2016 17:22:32 +0200
Subject: erts: Remove unused argument to hipe_set_closure_stub

---
 erts/emulator/beam/beam_load.c        | 2 +-
 erts/emulator/beam/external.c         | 2 +-
 erts/emulator/hipe/hipe_mode_switch.c | 7 ++-----
 erts/emulator/hipe/hipe_mode_switch.h | 2 +-
 4 files changed, 5 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 36cf64e08d..3cd395c2c1 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4848,7 +4848,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
 	    }
 	    fe->address = code_ptr;
 #ifdef HIPE
-	    hipe_set_closure_stub(fe, stp->lambdas[i].num_free);
+	    hipe_set_closure_stub(fe);
 #endif
 	}
     }
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 98df8a0726..a49b242d7c 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3748,7 +3748,7 @@ dec_term_atom_common:
 		funp->arity = arity;
 #ifdef HIPE
 		if (funp->fe->native_address == NULL) {
-		    hipe_set_closure_stub(funp->fe, num_free);
+		    hipe_set_closure_stub(funp->fe);
 		}
 #endif
 		hp = factory->hp;
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index ed95045292..0706f8d2c9 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -718,12 +718,9 @@ void hipe_empty_nstack(Process *p)
     p->hipe.nstend = NULL;
 }
 
-void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free)
+void hipe_set_closure_stub(ErlFunEntry *fe)
 {
-    unsigned arity;
-
-    arity = fe->arity;
-    fe->native_address = (Eterm*) hipe_closure_stub_address(arity);
+    fe->native_address = (Eterm*) hipe_closure_stub_address(fe->arity);
 }
 
 Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s)
diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h
index c40077d558..334e978307 100644
--- a/erts/emulator/hipe/hipe_mode_switch.h
+++ b/erts/emulator/hipe/hipe_mode_switch.h
@@ -59,7 +59,7 @@ void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure);
 Process *hipe_mode_switch(Process*, unsigned, Eterm*);
 void hipe_inc_nstack(Process *p);
 void hipe_empty_nstack(Process *p);
-void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free);
+void hipe_set_closure_stub(ErlFunEntry *fe);
 Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s);
 
 ERTS_GLB_INLINE void hipe_reserve_beam_trap_frame(Process*, Eterm reg[], unsigned arity);
-- 
cgit v1.2.3


From 21e49c8b2f3683b50750fb6a756810c8ba52b749 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 26 Oct 2016 17:44:14 +0200
Subject: erts: Fix new fun purge strategy for hipe

---
 erts/emulator/beam/erl_fun.c | 23 +++++++++++++++++++++--
 erts/emulator/beam/erl_fun.h |  3 +++
 2 files changed, 24 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index acb0d609d0..d9118d2549 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -31,6 +31,9 @@
 static Hash erts_fun_table;
 
 #include "erl_smp.h"
+#ifdef HIPE
+# include "hipe_mode_switch.h"
+#endif
 
 static erts_smp_rwmtx_t erts_fun_table_lock;
 
@@ -219,6 +222,10 @@ erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end)
 		fe->pend_purge_address = addr;
 		ERTS_SMP_WRITE_MEMORY_BARRIER;
 		fe->address = unloaded_fun;
+#ifdef HIPE
+                fe->pend_purge_native_address = fe->native_address;
+                hipe_set_closure_stub(fe);
+#endif
 		erts_purge_state_add_fun(fe);
 	    }
 	    b = b->next;
@@ -234,8 +241,12 @@ erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no)
 
     for (ix = 0; ix < no; ix++) {
 	ErlFunEntry *fe = funs[ix];
-	if (fe->address == unloaded_fun)
+	if (fe->address == unloaded_fun) {
 	    fe->address = fe->pend_purge_address;
+#ifdef HIPE
+            fe->native_address = fe->pend_purge_native_address;
+#endif
+        }
     }
 }
 
@@ -244,8 +255,12 @@ erts_fun_purge_abort_finalize(ErlFunEntry **funs, Uint no)
 {
     Uint ix;
 
-    for (ix = 0; ix < no; ix++)
+    for (ix = 0; ix < no; ix++) {
 	funs[ix]->pend_purge_address = NULL;
+#ifdef HIPE
+        funs[ix]->pend_purge_native_address = NULL;
+#endif
+    }
 }
 
 void
@@ -256,6 +271,9 @@ erts_fun_purge_complete(ErlFunEntry **funs, Uint no)
     for (ix = 0; ix < no; ix++) {
 	ErlFunEntry *fe = funs[ix];
 	fe->pend_purge_address = NULL;
+#ifdef HIPE
+        fe->pend_purge_native_address = NULL;
+#endif
 	if (erts_refc_dectest(&fe->refc, 0) == 0)
 	    erts_erase_fun_entry(fe);
     }
@@ -324,6 +342,7 @@ fun_alloc(ErlFunEntry* template)
     obj->pend_purge_address = NULL;
 #ifdef HIPE
     obj->native_address = NULL;
+    obj->pend_purge_native_address = NULL;
 #endif
     return obj;
 }
diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h
index 6ba055d92c..717bfee0fb 100644
--- a/erts/emulator/beam/erl_fun.h
+++ b/erts/emulator/beam/erl_fun.h
@@ -45,6 +45,9 @@ typedef struct erl_fun_entry {
     erts_refc_t refc;		/* Reference count: One for code + one for each
 				   fun object in each process. */
     BeamInstr *pend_purge_address; /* address stored during a pending purge */
+#ifdef HIPE
+    UWord* pend_purge_native_address;
+#endif
 } ErlFunEntry;
 
 /*
-- 
cgit v1.2.3


From c5aa2c8a4e943176e5e7dea7c8653aee93faa18e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 26 Oct 2016 17:49:44 +0200
Subject: erts: Expand purge fun tests for hipe

---
 erts/emulator/test/code_SUITE.erl                  | 168 ++-------------------
 .../code_SUITE_data/call_purged_fun_tester.erl     | 167 ++++++++++++++++++++
 2 files changed, 183 insertions(+), 152 deletions(-)
 create mode 100644 erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index 465ddfa7a8..774461c525 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -153,127 +153,23 @@ call_purged_fun_code_there(Config) when is_list(Config) ->
     ok.
 
 call_purged_fun_test(Priv, Data, Type) ->
-    File = filename:join(Data, "my_code_test2"),
-    Code = filename:join(Priv, "my_code_test2"),
-
-    catch erlang:purge_module(my_code_test2),
-    catch erlang:delete_module(my_code_test2),
-    catch erlang:purge_module(my_code_test2),
-
-    {ok,my_code_test2} = c:c(File, [{outdir,Priv}]),
-
-    T = ets:new(my_code_test2_fun_table, []),
-    ets:insert(T, {my_fun,my_code_test2:make_fun(4711)}),
-    ets:insert(T, {my_fun2,my_code_test2:make_fun2()}),
-
-    spawn(fun () ->
-		  [{my_fun2,F2}] = ets:lookup(T, my_fun2),
-		  F2(fun () ->
-			     receive after infinity -> ok end
-		     end,
-		     fun () -> ok end),
-		  exit(completed)
-	  end),
-
-    PurgeType = case Type of
-		    code_gone ->
-			ok = file:delete(Code++".beam"),
-			true;
-		    code_reload ->
-			true;
-		    code_there ->
-			false
-		end,
-
-    true = erlang:delete_module(my_code_test2),
-
-    Purge = start_purge(my_code_test2, PurgeType),
-
-    {P0, M0} = spawn_monitor(fun () ->
-				     [{my_fun,F}] = ets:lookup(T, my_fun),
-				     4712 = F(1),
-				     exit(completed)
-			     end),
-
-    wait_until(fun () ->
-		       {status, suspended}
-			   == process_info(P0, status)
-	       end),
-
-    ok = continue_purge(Purge),
-
-    {P1, M1} = spawn_monitor(fun () ->
-				     [{my_fun,F}] = ets:lookup(T, my_fun),
-				     4713 = F(2),
-				     exit(completed)
-			     end),
-    {P2, M2} = spawn_monitor(fun () ->
-				     [{my_fun,F}] = ets:lookup(T, my_fun),
-				     4714 = F(3),
-				     exit(completed)
-			     end),
-
-    wait_until(fun () ->
-		       {status, suspended}
-			   == process_info(P1, status)
-	       end),
-    wait_until(fun () ->
-		       {status, suspended}
-			   == process_info(P2, status)
-	       end),
-
-    {current_function,
-     {erts_code_purger,
-      pending_purge_lambda,
-      3}} = process_info(P0, current_function),
-    {current_function,
-     {erts_code_purger,
-      pending_purge_lambda,
-      3}} = process_info(P1, current_function),
-    {current_function,
-     {erts_code_purger,
-      pending_purge_lambda,
-      3}} = process_info(P2, current_function),
-
-    case Type of
-	code_there ->
-	    false = complete_purge(Purge);
-	_ ->
-	    {true, true} = complete_purge(Purge)
-    end,
+    OptsList = case erlang:system_info(hipe_architecture) of
+                   undefined -> [[]];
+                   _ -> [[], [native]]
+               end,
+    [call_purged_fun_test_do(Priv, Data, Type, CO, FO)
+     || CO <- OptsList, FO <- OptsList].
+
+
+call_purged_fun_test_do(Priv, Data, Type, CallerOpts, FunOpts) ->
+    io:format("Compile caller as ~p and funs as ~p\n", [CallerOpts, FunOpts]),
+    SrcFile = filename:join(Data, "call_purged_fun_tester.erl"),
+    ObjFile = filename:join(Priv, "call_purged_fun_tester.beam"),
+    {ok,Mod,Code} = compile:file(SrcFile, [binary, report | CallerOpts]),
+    {module,Mod} = code:load_binary(Mod, ObjFile, Code),
+
+    call_purged_fun_tester:do(Priv, Data, Type, FunOpts).
 
-    case Type of
-	code_gone ->
-	    receive
-		{'DOWN', M0, process, P0, Reason0} ->
-		    {undef, _} = Reason0
-	    end,
-	    receive
-		{'DOWN', M1, process, P1, Reason1} ->
-		    {undef, _} = Reason1
-	    end,
-	    receive
-		{'DOWN', M2, process, P2, Reason2} ->
-		    {undef, _} = Reason2
-	    end;
-	_ ->
-	    receive
-		{'DOWN', M0, process, P0, Reason0} ->
-		    completed = Reason0
-	    end,
-	    receive
-		{'DOWN', M1, process, P1, Reason1} ->
-		    completed = Reason1
-	    end,
-	    receive
-		{'DOWN', M2, process, P2, Reason2} ->
-		    completed = Reason2
-	    end,
-	    catch erlang:purge_module(my_code_test2),
-	    catch erlang:delete_module(my_code_test2),
-	    catch erlang:purge_module(my_code_test2)
-    end,
-    ok.
 
 multi_proc_purge(Config) when is_list(Config) ->
     %%
@@ -975,35 +871,3 @@ flush() ->
 
 id(I) -> I.
 
-wait_until(Fun) ->
-    case Fun() of
-	true ->
-	    ok;
-	false ->
-	    receive after 100 -> ok end,
-	    wait_until(Fun)
-    end.
-
-start_purge(Mod, Type) when is_atom(Mod)
-			    andalso ((Type == true)
-				     orelse (Type == false)) ->
-    Ref = make_ref(),
-    erts_code_purger ! {test_purge, Mod, self(), Type, Ref},
-    receive
-	{started, Ref} ->
-	    Ref
-    end.
-
-continue_purge(Ref) when is_reference(Ref) ->
-    erts_code_purger ! {continue, Ref},
-    receive
-	{continued, Ref} ->
-	    ok
-    end.
-
-complete_purge(Ref) when is_reference(Ref) ->
-    erts_code_purger ! {complete, Ref},
-    receive
-	{test_purge, Res, Ref} ->
-	    Res
-    end.
diff --git a/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl b/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl
new file mode 100644
index 0000000000..5e031abca8
--- /dev/null
+++ b/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl
@@ -0,0 +1,167 @@
+-module(call_purged_fun_tester).
+
+-export([do/4]).
+
+do(Priv, Data, Type, Opts) ->
+    File = filename:join(Data, "my_code_test2"),
+    Code = filename:join(Priv, "my_code_test2"),
+
+    catch erlang:purge_module(my_code_test2),
+    catch erlang:delete_module(my_code_test2),
+    catch erlang:purge_module(my_code_test2),
+
+    {ok,my_code_test2} = c:c(File, [{outdir,Priv} | Opts]),
+
+    IsNative = lists:member(native,Opts),
+    IsNative = code:is_module_native(my_code_test2),
+
+    T = ets:new(my_code_test2_fun_table, []),
+    ets:insert(T, {my_fun,my_code_test2:make_fun(4711)}),
+    ets:insert(T, {my_fun2,my_code_test2:make_fun2()}),
+
+    spawn(fun () ->
+		  [{my_fun2,F2}] = ets:lookup(T, my_fun2),
+		  F2(fun () ->
+			     receive after infinity -> ok end
+		     end,
+		     fun () -> ok end),
+		  exit(completed)
+	  end),
+
+    PurgeType = case Type of
+		    code_gone ->
+			ok = file:delete(Code++".beam"),
+			true;
+		    code_reload ->
+			true;
+		    code_there ->
+			false
+		end,
+
+    true = erlang:delete_module(my_code_test2),
+
+    Purge = start_purge(my_code_test2, PurgeType),
+
+    {P0, M0} = spawn_monitor(fun () ->
+				     [{my_fun,F}] = ets:lookup(T, my_fun),
+				     4712 = F(1),
+				     exit(completed)
+			     end),
+
+    wait_until(fun () ->
+		       {status, suspended}
+			   == process_info(P0, status)
+	       end),
+
+    ok = continue_purge(Purge),
+
+    {P1, M1} = spawn_monitor(fun () ->
+				     [{my_fun,F}] = ets:lookup(T, my_fun),
+				     4713 = F(2),
+				     exit(completed)
+			     end),
+    {P2, M2} = spawn_monitor(fun () ->
+				     [{my_fun,F}] = ets:lookup(T, my_fun),
+				     4714 = F(3),
+				     exit(completed)
+			     end),
+
+    wait_until(fun () ->
+		       {status, suspended}
+			   == process_info(P1, status)
+	       end),
+    wait_until(fun () ->
+		       {status, suspended}
+			   == process_info(P2, status)
+	       end),
+
+    {current_function,
+     {erts_code_purger,
+      pending_purge_lambda,
+      3}} = process_info(P0, current_function),
+    {current_function,
+     {erts_code_purger,
+      pending_purge_lambda,
+      3}} = process_info(P1, current_function),
+    {current_function,
+     {erts_code_purger,
+      pending_purge_lambda,
+      3}} = process_info(P2, current_function),
+
+    case Type of
+	code_there ->
+	    false = complete_purge(Purge);
+	_ ->
+	    {true, true} = complete_purge(Purge)
+    end,
+
+    case Type of
+	code_gone ->
+	    receive
+		{'DOWN', M0, process, P0, Reason0} ->
+		    {undef, _} = Reason0
+	    end,
+	    receive
+		{'DOWN', M1, process, P1, Reason1} ->
+		    {undef, _} = Reason1
+	    end,
+	    receive
+		{'DOWN', M2, process, P2, Reason2} ->
+		    {undef, _} = Reason2
+	    end;
+	_ ->
+	    receive
+		{'DOWN', M0, process, P0, Reason0} ->
+		    completed = Reason0
+	    end,
+	    receive
+		{'DOWN', M1, process, P1, Reason1} ->
+		    completed = Reason1
+	    end,
+	    receive
+		{'DOWN', M2, process, P2, Reason2} ->
+		    completed = Reason2
+	    end,
+	    catch erlang:purge_module(my_code_test2),
+	    catch erlang:delete_module(my_code_test2),
+	    catch erlang:purge_module(my_code_test2)
+    end,
+    ok.
+
+wait_until(Fun) ->
+    ok = wait_until(Fun, 20).
+
+wait_until(Fun, N) ->
+    case {Fun(),N} of
+	{true, _} ->
+	    ok;
+        {false, 0} ->
+            timeout;
+	{false, _} ->
+	    receive after 100 -> ok end,
+	    wait_until(Fun, N-1)
+    end.
+
+start_purge(Mod, Type) when is_atom(Mod)
+			    andalso ((Type == true)
+				     orelse (Type == false)) ->
+    Ref = make_ref(),
+    erts_code_purger ! {test_purge, Mod, self(), Type, Ref},
+    receive
+	{started, Ref} ->
+	    Ref
+    end.
+
+continue_purge(Ref) when is_reference(Ref) ->
+    erts_code_purger ! {continue, Ref},
+    receive
+	{continued, Ref} ->
+	    ok
+    end.
+
+complete_purge(Ref) when is_reference(Ref) ->
+    erts_code_purger ! {complete, Ref},
+    receive
+	{test_purge, Res, Ref} ->
+	    Res
+    end.
-- 
cgit v1.2.3


From e084d3d77eb15f58b2152e45ef84c27e7cf88403 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 28 Oct 2016 15:52:43 +0200
Subject: erts: Exclude random beam functions from hipe stacktrace

---
 erts/emulator/hipe/hipe_bif0.c        | 4 ++++
 erts/emulator/hipe/hipe_mode_switch.h | 6 ++++++
 2 files changed, 10 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 3336fded7a..dcb6c35bfa 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1476,6 +1476,10 @@ int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a)
     struct hipe_mfa_info **bucket;
     unsigned int i, nrbuckets;
 
+    if (hipe_is_ra_mode_switch(ra)) {
+        return 0;
+    }
+
     /* Note about locking: the table is only updated from the
        loader, which runs with the rest of the system suspended. */
     /* XXX: alas not true; see comment at hipe_mfa_info_table.lock */
diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h
index c40077d558..e54b81cf78 100644
--- a/erts/emulator/hipe/hipe_mode_switch.h
+++ b/erts/emulator/hipe/hipe_mode_switch.h
@@ -64,6 +64,7 @@ Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s);
 
 ERTS_GLB_INLINE void hipe_reserve_beam_trap_frame(Process*, Eterm reg[], unsigned arity);
 ERTS_GLB_INLINE void hipe_unreserve_beam_trap_frame(Process*);
+ERTS_GLB_INLINE int hipe_is_ra_mode_switch(const void* ra);
 
 extern Uint hipe_beam_pc_return[];
 extern Uint hipe_beam_pc_throw[];
@@ -112,6 +113,11 @@ ERTS_GLB_INLINE void hipe_unreserve_beam_trap_frame(Process *p)
     p->stop += 2;
 }
 
+ERTS_GLB_INLINE int hipe_is_ra_mode_switch(const void* ra)
+{
+    return ra == nbif_return;
+}
+
 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 #endif	/* ASM */
-- 
cgit v1.2.3


From 24485c421f15b00d0b5c97633d701373b53897a5 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 1 Nov 2016 11:40:48 +0100
Subject: erts: Add notsup error for load_nif/2 from hipe code

---
 erts/emulator/beam/erl_nif.c                        |  6 ++++++
 erts/emulator/test/nif_SUITE.erl                    | 18 ++++++++++++++++++
 erts/emulator/test/nif_SUITE_data/hipe_compiled.erl |  6 ++++++
 3 files changed, 30 insertions(+)
 create mode 100644 erts/emulator/test/nif_SUITE_data/hipe_compiled.erl

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index c5606f72a1..9872a3ab81 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3166,6 +3166,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     struct erl_module_instance* this_mi;
     struct erl_module_instance* prev_mi;
 
+    if (BIF_P->flags & F_HIPE_MODE) {
+	ret = load_nif_error(BIF_P, "notsup", "Calling load_nif from HiPE compiled "
+			     "modules not supported");
+	BIF_RET(ret);
+    }
+
     encoding = erts_get_native_filename_encoding();
     if (encoding == ERL_FILENAME_WIN_WCHAR) {
         /* Do not convert the lib name to utf-16le yet, do that in win32 specific code */
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 306f2091a1..3a76ee6aee 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -30,6 +30,7 @@
 	 init_per_testcase/2, end_per_testcase/2,
          basic/1, reload_error/1, upgrade/1, heap_frag/1,
          t_on_load/1,
+         hipe/1,
 	 types/1, many_args/1, binaries/1, get_string/1, get_atom/1,
 	 maps/1,
 	 api_macros/1,
@@ -70,6 +71,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}].
 all() -> 
     [basic, reload_error, upgrade, heap_frag, types, many_args,
      t_on_load,
+     hipe,
      binaries, get_string, get_atom, maps, api_macros, from_array,
      iolist_as_binary, resource, resource_binary,
      resource_takeover, threading, send, send2, send3,
@@ -88,6 +90,11 @@ all() ->
 init_per_testcase(t_on_load, Config) ->
     ets:new(nif_SUITE, [named_table]),
     Config;
+init_per_testcase(hipe, Config) ->
+    case erlang:system_info(hipe_architecture) of
+	undefined -> {skip, "HiPE is disabled"};
+	_ -> Config
+    end;
 init_per_testcase(_Case, Config) ->
     Config.
 
@@ -411,6 +418,17 @@ t_on_load(Config) when is_list(Config) ->
     verify_tmpmem(TmpMem),
     ok.
 
+hipe(Config) when is_list(Config) ->
+    Data = proplists:get_value(data_dir, Config),
+    Priv = proplists:get_value(priv_dir, Config),
+    Src = filename:join(Data, "hipe_compiled"),
+    {ok,hipe_compiled} = c:c(Src, [{outdir,Priv},native]),
+    true = code:is_module_native(hipe_compiled),
+    {error, {notsup,_}} = hipe_compiled:try_load_nif(),
+    true = code:delete(hipe_compiled),
+    false = code:purge(hipe_compiled),
+    ok.
+
 
 %% Test NIF building heap fragments
 heap_frag(Config) when is_list(Config) ->    
diff --git a/erts/emulator/test/nif_SUITE_data/hipe_compiled.erl b/erts/emulator/test/nif_SUITE_data/hipe_compiled.erl
new file mode 100644
index 0000000000..84ddbc8d63
--- /dev/null
+++ b/erts/emulator/test/nif_SUITE_data/hipe_compiled.erl
@@ -0,0 +1,6 @@
+-module(hipe_compiled).
+
+-export([try_load_nif/0]).
+
+try_load_nif() ->
+    erlang:load_nif("doesn't matter", 0).
-- 
cgit v1.2.3


From 500748fd912d71b6e7350b5e45a2208eb7c3bd65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 31 Oct 2016 15:20:07 +0100
Subject: erts: Fix SIGUSR1 crashdump generation

Do not generate a core when a crashdump is asked for.
Regression introduced in 56090db3ea417157a749bdd810fc61d117493f1f
---
 erts/emulator/sys/unix/sys.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 089efec3e8..ba22a100c2 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -672,7 +672,7 @@ sigusr1_exit(void)
     }
 
     prepare_crash_dump(secs);
-    erts_exit(ERTS_ERROR_EXIT, "Received SIGUSR1\n");
+    erts_exit(ERTS_DUMP_EXIT, "Received SIGUSR1\n");
 }
 
 #ifdef ETHR_UNUSABLE_SIGUSRX
-- 
cgit v1.2.3


From e21a6c9ec713219b0aa2b2a4224f97e713d7337f Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Tue, 25 Oct 2016 22:05:10 +0100
Subject: Add math:fmod/2 BIF

Returns the (floating point) remainder of first argument divided
by second argument.
---
 erts/emulator/beam/bif.tab    | 1 +
 erts/emulator/beam/erl_math.c | 5 +++++
 2 files changed, 6 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 852b1135fe..32600f4338 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -667,6 +667,7 @@ gcbif erlang:floor/1
 gcbif erlang:ceil/1
 bif math:floor/1
 bif math:ceil/1
+bif math:fmod/2
 
 #
 # Obsolete
diff --git a/erts/emulator/beam/erl_math.c b/erts/emulator/beam/erl_math.c
index 990fa63bd4..1f270eb55f 100644
--- a/erts/emulator/beam/erl_math.c
+++ b/erts/emulator/beam/erl_math.c
@@ -256,3 +256,8 @@ BIF_RETTYPE math_floor_1(BIF_ALIST_1)
 {
     return math_call_1(BIF_P, floor, BIF_ARG_1);
 }
+
+BIF_RETTYPE math_fmod_2(BIF_ALIST_2)
+{
+    return math_call_2(BIF_P, fmod, BIF_ARG_1, BIF_ARG_2);
+}
-- 
cgit v1.2.3


From 9d751cedd2d24ca476b12052fcc15215c0a384be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 2 Nov 2016 16:48:30 +0100
Subject: erts: Use cmd on windows for port_SUITE

---
 erts/emulator/test/port_SUITE.erl | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index 4323849465..d4e77d634a 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -976,21 +976,21 @@ try_bad_env(Env) ->
 %% Test that we can handle a very very large environment gracefully.
 huge_env(Config) when is_list(Config) ->
     ct:timetrap({minutes, 2}),
-    Vars = case os:type() of
-               {win32,_} -> 500;
-               _ ->
-                   %% We create a huge environment,
-                   %% 20000 variables is about 25MB
-                   %% which seems to be the limit on Linux.
-                   20000
-           end,
+    {Vars, Cmd} = case os:type() of
+                      {win32,_} -> {500, "cmd /q /c ls"};
+                      _ ->
+                          %% We create a huge environment,
+                          %% 20000 variables is about 25MB
+                          %% which seems to be the limit on Linux.
+                          {20000, "ls"}
+                  end,
     Env = [{[$a + I div (25*25*25*25) rem 25,
              $a + I div (25*25*25) rem 25,
              $a + I div (25*25) rem 25,
              $a+I div 25 rem 25, $a+I rem 25],
             lists:duplicate(100,$a+I rem 25)}
            || I <- lists:seq(1,Vars)],
-    try erlang:open_port({spawn,"ls"},[exit_status, {env, Env}]) of
+    try erlang:open_port({spawn,Cmd},[exit_status, {env, Env}]) of
         P ->
             receive
                 {P, {exit_status,N}} = M ->
@@ -1009,7 +1009,10 @@ huge_env(Config) when is_list(Config) ->
 %% Test to spawn program with command payload buffer
 %% just around pipe capacity (9f779819f6bda734c5953468f7798)
 pipe_limit_env(Config) when is_list(Config) ->
-    Cmd = "true",
+    Cmd = case os:type() of
+              {win32,_} -> "cmd /q /c true";
+              _ -> "true"
+          end,
     CmdSize = command_payload_size(Cmd),
     Limits = [4096, 16384, 65536], % Try a couple of common pipe buffer sizes
 
@@ -1026,7 +1029,7 @@ pipe_limit_env_do(Bytes, Cmd, CmdSize) ->
 	    try erlang:open_port({spawn,Cmd},[exit_status, {env, Env}]) of
 		P ->
 		    receive
-			{P, {exit_status,N}} = M ->
+			{P, {exit_status,N}} ->
 			    %% Bug caused exit_status 150 (EINVAL+128)
 			    0 = N
 		    end
-- 
cgit v1.2.3


From 5aa13e16ae81050509fceaf603650fc8594af7ec Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 9 Nov 2016 16:52:23 +0100
Subject: erts: Fix correct link flags for hipe_mkliterals

and no need for $(INCLUDED).
---
 erts/emulator/Makefile.in | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index e0260205e3..8772befe27 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -49,6 +49,7 @@ CREATE_DIRS=
 LDFLAGS=@LDFLAGS@
 ARFLAGS=rc
 OMIT_OMIT_FP=no
+TYPE_LIBS=
 
 DIRTY_SCHEDULER_SUPPORT=@DIRTY_SCHEDULER_SUPPORT@
 NEW_PURGE_STRATEGY=@NEW_PURGE_STRATEGY@
@@ -90,7 +91,7 @@ PURIFY =
 TYPEMARKER = .gcov
 TYPE_FLAGS = $(DEBUG_CFLAGS) -DERTS_GCOV -DNO_JUMP_TABLE -fprofile-arcs -ftest-coverage -O0 -DERTS_CAN_INLINE=0 -DERTS_INLINE=
 ifneq ($(findstring solaris,$(TARGET)),solaris)
-LIBS += -lgcov
+TYPE_LIBS = -lgcov
 endif
 ENABLE_ALLOC_TYPE_VARS += debug
 else
@@ -146,6 +147,8 @@ endif
 endif
 endif
 
+LIBS += $(TYPE_LIBS)
+
 comma:=,
 space:=
 space+=
@@ -931,7 +934,7 @@ $(OBJDIR)/%.o: hipe/%.c
 	$(V_CC) $(subst O2,O3, $(CFLAGS)) $(INCLUDES) -c $< -o $@
 
 $(BINDIR)/hipe_mkliterals$(TF_MARKER):	$(OBJDIR)/hipe_mkliterals.o
-	$(ld_verbose)$(CC) $(CFLAGS) $(INCLUDES) -o $@ $<
+	$(ld_verbose)$(CC) $(LDFLAGS) -o $@ $< $(TYPE_LIBS)
 
 $(OBJDIR)/hipe_mkliterals.o:	$(HIPE_ASM) $(TTF_DIR)/erl_alloc_types.h $(DTRACE_HEADERS) \
 	$(TTF_DIR)/OPCODES-GENERATED $(TARGET)/TABLES-GENERATED
-- 
cgit v1.2.3


From 4c3141e8765bcaaaf998dd84b4f53059c3fe066f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 15 Nov 2016 15:17:29 +0100
Subject: erts: Remove unused erl_crash_dump()

---
 erts/emulator/beam/break.c    | 10 ----------
 erts/emulator/sys/unix/sys.c  |  2 --
 erts/emulator/sys/win32/sys.c |  1 -
 3 files changed, 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 3c19e82b66..aa9bed5714 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -44,7 +44,6 @@
 static void process_killer(void);
 void do_break(void);
 void erl_crash_dump_v(char *file, int line, char* fmt, va_list args);
-void erl_crash_dump(char* file, int line, char* fmt, ...);
 
 #ifdef DEBUG
 static void bin_check(void);
@@ -884,12 +883,3 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     erts_fprintf(stderr,"done\n");
 }
 
-void
-erl_crash_dump(char* file, int line, char* fmt, ...)
-{
-  va_list args;
-  
-  va_start(args, fmt);
-  erl_crash_dump_v(file, line, fmt, args);
-  va_end(args);
-}
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 089efec3e8..4e0f2e6de3 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -90,8 +90,6 @@ extern void erl_sys_args(int*, char**);
 
 extern void erts_sys_init_float(void);
 
-extern void erl_crash_dump(char* file, int line, char* fmt, ...);
-
 
 #ifdef DEBUG
 static int debug_log = 0;
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index cf821b05cb..404e72d176 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -40,7 +40,6 @@ void erts_sys_init_float(void);
 void erl_start(int, char**);
 void erts_exit(int n, char*, ...);
 void erl_error(char*, va_list);
-void erl_crash_dump(char*, int, char*, ...);
 
 /*
  * Microsoft-specific function to map a WIN32 error code to a Posix errno.
-- 
cgit v1.2.3


From 9a1110de820988afb4e9d195a59872b214de2d26 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 17 Nov 2016 16:50:36 +0100
Subject: erts: Refactor crash dumping with cbprintf

Instead of passing around a file descriptor
use a function pointer to facilitate more advanced
backend write logic such as size limitation or compression.
---
 erts/emulator/beam/atom.c                  |  4 +-
 erts/emulator/beam/atom.h                  |  4 +-
 erts/emulator/beam/beam_debug.c            |  4 +-
 erts/emulator/beam/break.c                 | 97 +++++++++++++++---------------
 erts/emulator/beam/dist.c                  | 14 ++---
 erts/emulator/beam/erl_afit_alloc.c        |  4 +-
 erts/emulator/beam/erl_alloc.c             | 12 ++--
 erts/emulator/beam/erl_alloc.h             |  6 +-
 erts/emulator/beam/erl_alloc_util.c        | 32 +++++-----
 erts/emulator/beam/erl_alloc_util.h        | 10 +--
 erts/emulator/beam/erl_ao_firstfit_alloc.c |  4 +-
 erts/emulator/beam/erl_bestfit_alloc.c     |  4 +-
 erts/emulator/beam/erl_bif_info.c          |  2 +-
 erts/emulator/beam/erl_db.c                |  6 +-
 erts/emulator/beam/erl_db.h                |  2 +-
 erts/emulator/beam/erl_db_hash.c           |  4 +-
 erts/emulator/beam/erl_db_tree.c           |  4 +-
 erts/emulator/beam/erl_db_util.h           |  2 +-
 erts/emulator/beam/erl_debug.c             | 10 +--
 erts/emulator/beam/erl_fun.c               |  4 +-
 erts/emulator/beam/erl_fun.h               |  4 +-
 erts/emulator/beam/erl_goodfit_alloc.c     |  4 +-
 erts/emulator/beam/erl_hl_timer.c          |  4 +-
 erts/emulator/beam/erl_hl_timer.h          |  2 +-
 erts/emulator/beam/erl_instrument.c        | 74 ++++++++---------------
 erts/emulator/beam/erl_instrument.h        |  4 +-
 erts/emulator/beam/erl_nif.c               |  6 +-
 erts/emulator/beam/erl_node_tables.c       |  8 +--
 erts/emulator/beam/erl_node_tables.h       |  6 +-
 erts/emulator/beam/erl_port.h              |  2 +-
 erts/emulator/beam/erl_process.c           | 14 ++---
 erts/emulator/beam/erl_process.h           | 14 ++---
 erts/emulator/beam/erl_process_dict.c      |  6 +-
 erts/emulator/beam/erl_process_dict.h      |  6 +-
 erts/emulator/beam/erl_process_dump.c      | 48 +++++++--------
 erts/emulator/beam/export.c                |  2 +-
 erts/emulator/beam/export.h                |  2 +-
 erts/emulator/beam/global.h                | 16 ++---
 erts/emulator/beam/hash.c                  |  2 +-
 erts/emulator/beam/hash.h                  |  4 +-
 erts/emulator/beam/index.c                 |  2 +-
 erts/emulator/beam/index.h                 |  2 +-
 erts/emulator/beam/io.c                    |  4 +-
 erts/emulator/beam/module.c                |  2 +-
 erts/emulator/beam/module.h                |  2 +-
 erts/emulator/beam/register.c              |  2 +-
 erts/emulator/beam/register.h              |  2 +-
 erts/emulator/beam/sys.h                   | 19 +++---
 erts/emulator/beam/utils.c                 | 31 +++++-----
 erts/emulator/sys/common/erl_mmap.c        |  8 +--
 erts/emulator/sys/common/erl_mmap.h        |  5 +-
 erts/emulator/sys/common/erl_mseg.c        | 20 +++---
 erts/emulator/sys/common/erl_mseg.h        |  4 +-
 53 files changed, 271 insertions(+), 289 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index a5e778e4aa..2052afe52b 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -68,7 +68,7 @@ static Uint atom_space;		/* Amount of atom text space used */
 /*
  * Print info about atom tables
  */
-void atom_info(int to, void *to_arg)
+void atom_info(fmtfn_t to, void *to_arg)
 {
     int lock = !ERTS_IS_CRASH_DUMPING;
     if (lock)
@@ -470,7 +470,7 @@ init_atom_table(void)
 }
 
 void
-dump_atoms(int to, void *to_arg)
+dump_atoms(fmtfn_t to, void *to_arg)
 {
     int i = erts_atom_table.entries;
 
diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h
index ae60904785..a82efabb9f 100644
--- a/erts/emulator/beam/atom.h
+++ b/erts/emulator/beam/atom.h
@@ -136,8 +136,8 @@ Eterm erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc);
 int atom_erase(byte*, int);
 int atom_static_put(byte*, int);
 void init_atom_table(void);
-void atom_info(int, void *);
-void dump_atoms(int, void *);
+void atom_info(fmtfn_t, void *);
+void dump_atoms(fmtfn_t, void *);
 int erts_atom_get(const char* name, int len, Eterm* ap, ErtsAtomEncoding enc);
 void erts_atom_get_text_space_sizes(Uint *reserved, Uint *used);
 #endif
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index a4ad3e7886..21d336049f 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -50,7 +50,7 @@
 void dbg_bt(Process* p, Eterm* sp);
 void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg);
 
-static int print_op(int to, void *to_arg, int op, int size, BeamInstr* addr);
+static int print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr);
 
 BIF_RETTYPE
 erts_debug_same_2(BIF_ALIST_2)
@@ -377,7 +377,7 @@ dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg)
 }
 
 static int
-print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
+print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 {
     int i;
     BeamInstr tag;
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index aa9bed5714..2e8a384dec 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -49,17 +49,17 @@ void erl_crash_dump_v(char *file, int line, char* fmt, va_list args);
 static void bin_check(void);
 #endif
 
-static void print_garb_info(int to, void *to_arg, Process* p);
+static void print_garb_info(fmtfn_t to, void *to_arg, Process* p);
 #ifdef OPPROF
 static void dump_frequencies(void);
 #endif
 
-static void dump_attributes(int to, void *to_arg, byte* ptr, int size);
+static void dump_attributes(fmtfn_t to, void *to_arg, byte* ptr, int size);
 
 extern char* erts_system_version[];
 
 static void
-port_info(int to, void *to_arg)
+port_info(fmtfn_t to, void *to_arg)
 {
     int i, max = erts_ptab_max(&erts_port);
     for (i = 0; i < max; i++) {
@@ -70,7 +70,7 @@ port_info(int to, void *to_arg)
 }
 
 void
-process_info(int to, void *to_arg)
+process_info(fmtfn_t to, void *to_arg)
 {
     int i, max = erts_ptab_max(&erts_proc);
     for (i = 0; i < max; i++) {
@@ -147,14 +147,14 @@ process_killer(void)
 
 typedef struct {
     int is_first;
-    int to;
+    fmtfn_t to;
     void *to_arg;
 } PrintMonitorContext;
 
 static void doit_print_link(ErtsLink *lnk, void *vpcontext)
 {
     PrintMonitorContext *pcontext = vpcontext;
-    int to = pcontext->to;
+    fmtfn_t to = pcontext->to;
     void *to_arg = pcontext->to_arg;
 
     if (pcontext->is_first) {
@@ -169,7 +169,7 @@ static void doit_print_link(ErtsLink *lnk, void *vpcontext)
 static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext)
 {
     PrintMonitorContext *pcontext = vpcontext;
-    int to = pcontext->to;
+    fmtfn_t to = pcontext->to;
     void *to_arg = pcontext->to_arg;
     char *prefix = ", ";
  
@@ -196,7 +196,7 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext)
 			       
 /* Display info about an individual Erlang process */
 void
-print_process_info(int to, void *to_arg, Process *p)
+print_process_info(fmtfn_t to, void *to_arg, Process *p)
 {
     time_t approx_started;
     int garbing = 0;
@@ -299,7 +299,7 @@ print_process_info(int to, void *to_arg, Process *p)
 
     /* display the links only if there are any*/
     if (ERTS_P_LINKS(p) || ERTS_P_MONITORS(p)) {
-	PrintMonitorContext context = {1,to}; 
+	PrintMonitorContext context = {1, to, to_arg};
 	erts_print(to, to_arg,"Link list: [");
 	erts_doforall_links(ERTS_P_LINKS(p), &doit_print_link, &context);	
 	erts_doforall_monitors(ERTS_P_MONITORS(p), &doit_print_monitor, &context);
@@ -347,7 +347,7 @@ print_process_info(int to, void *to_arg, Process *p)
 }
 
 static void
-print_garb_info(int to, void *to_arg, Process* p)
+print_garb_info(fmtfn_t to, void *to_arg, Process* p)
 {
 #ifdef ERTS_SMP
     /* ERTS_SMP: A scheduler is probably concurrently doing gc... */
@@ -364,7 +364,7 @@ print_garb_info(int to, void *to_arg, Process* p)
 }
 
 void
-info(int to, void *to_arg)
+info(fmtfn_t to, void *to_arg)
 {
     erts_memory(&to, to_arg, NULL, THE_NON_VALUE);
     atom_info(to, to_arg);
@@ -380,7 +380,7 @@ info(int to, void *to_arg)
 }
 
 void
-loaded(int to, void *to_arg)
+loaded(fmtfn_t to, void *to_arg)
 {
     int i;
     int old = 0;
@@ -477,7 +477,7 @@ loaded(int to, void *to_arg)
 
 
 static void
-dump_attributes(int to, void *to_arg, byte* ptr, int size)
+dump_attributes(fmtfn_t to, void *to_arg, byte* ptr, int size)
 {
     while (size-- > 0) {
 	erts_print(to, to_arg, "%02X", *ptr++);
@@ -678,6 +678,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     int secs;
     int env_erl_crash_dump_seconds_set = 1;
     int i;
+    fmtfn_t to = &erts_write_fd;
+    void*   to_arg;
 
     if (ERTS_SOMEONE_IS_CRASH_DUMPING)
 	return;
@@ -768,39 +770,40 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     fd = open(dumpname,O_WRONLY | O_CREAT | O_TRUNC,0640);
     if (fd < 0)
 	return; /* Can't create the crash dump, skip it */
+    to_arg = (void*)&fd;
     time(&now);
-    erts_fdprintf(fd, "=erl_crash_dump:0.3\n%s", ctime(&now));
+    erts_cbprintf(to, to_arg, "=erl_crash_dump:0.3\n%s", ctime(&now));
 
     if (file != NULL)
-       erts_fdprintf(fd, "The error occurred in file %s, line %d\n", file, line);
+       erts_cbprintf(to, to_arg, "The error occurred in file %s, line %d\n", file, line);
 
     if (fmt != NULL && *fmt != '\0') {
-	erts_fdprintf(fd, "Slogan: ");
-	erts_vfdprintf(fd, fmt, args);
+	erts_cbprintf(to, to_arg, "Slogan: ");
+	erts_vcbprintf(to, to_arg, fmt, args);
     }
-    erts_fdprintf(fd, "System version: ");
-    erts_print_system_version(fd, NULL, NULL);
+    erts_cbprintf(to, to_arg, "System version: ");
+    erts_print_system_version(to, to_arg, NULL);
 #if ERTS_SAVED_COMPILE_TIME
-    erts_fdprintf(fd, "%s\n", "Compiled: " ERLANG_COMPILE_DATE);
+    erts_cbprintf(to, to_arg, "%s\n", "Compiled: " ERLANG_COMPILE_DATE);
 #endif
 
-    erts_fdprintf(fd, "Taints: ");
-    erts_print_nif_taints(fd, NULL);
-    erts_fdprintf(fd, "Atoms: %d\n", atom_table_size());
+    erts_cbprintf(to, to_arg, "Taints: ");
+    erts_print_nif_taints(to, to_arg);
+    erts_cbprintf(to, to_arg, "Atoms: %d\n", atom_table_size());
 
 #ifdef USE_THREADS
     /* We want to note which thread it was that called erts_exit */
     if (erts_get_scheduler_data()) {
-        erts_fdprintf(fd, "Calling Thread: scheduler:%d\n",
+        erts_cbprintf(to, to_arg, "Calling Thread: scheduler:%d\n",
                       erts_get_scheduler_data()->no);
     } else {
         if (!erts_thr_getname(erts_thr_self(), dumpnamebuf, MAXPATHLEN))
-            erts_fdprintf(fd, "Calling Thread: %s\n", dumpnamebuf);
+            erts_cbprintf(to, to_arg, "Calling Thread: %s\n", dumpnamebuf);
         else
-            erts_fdprintf(fd, "Calling Thread: %p\n", erts_thr_self());
+            erts_cbprintf(to, to_arg, "Calling Thread: %p\n", erts_thr_self());
     }
 #else
-    erts_fdprintf(fd, "Calling Thread: scheduler:1\n");
+    erts_cbprintf(to, to_arg, "Calling Thread: scheduler:1\n");
 #endif
 
 #if defined(ERTS_HAVE_TRY_CATCH)
@@ -815,8 +818,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
      */
     for (i = 0; i < erts_no_schedulers; i++) {
         ERTS_SYS_TRY_CATCH(
-            erts_print_scheduler_info(fd, NULL, ERTS_SCHEDULER_IX(i)),
-            erts_fdprintf(fd, "** crashed **\n"));
+            erts_print_scheduler_info(to, to_arg, ERTS_SCHEDULER_IX(i)),
+            erts_cbprintf(to, to_arg, "** crashed **\n"));
     }
 #endif
 
@@ -847,38 +850,38 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
 #ifndef ERTS_HAVE_TRY_CATCH
     /* This is safe to call here, as all schedulers are blocked */
     for (i = 0; i < erts_no_schedulers; i++) {
-        erts_print_scheduler_info(fd, NULL, ERTS_SCHEDULER_IX(i));
+        erts_print_scheduler_info(to, to_arg, ERTS_SCHEDULER_IX(i));
     }
 #endif
     
-    info(fd, NULL); /* General system info */
+    info(to, to_arg); /* General system info */
     if (erts_ptab_initialized(&erts_proc))
-	process_info(fd, NULL); /* Info about each process and port */
-    db_info(fd, NULL, 0);
-    erts_print_bif_timer_info(fd, NULL);
-    distribution_info(fd, NULL);
-    erts_fdprintf(fd, "=loaded_modules\n");
-    loaded(fd, NULL);
-    erts_dump_fun_entries(fd, NULL);
-    erts_deep_process_dump(fd, NULL);
-    erts_fdprintf(fd, "=atoms\n");
-    dump_atoms(fd, NULL);
+	process_info(to, to_arg); /* Info about each process and port */
+    db_info(to, to_arg, 0);
+    erts_print_bif_timer_info(to, to_arg);
+    distribution_info(to, to_arg);
+    erts_cbprintf(to, to_arg, "=loaded_modules\n");
+    loaded(to, to_arg);
+    erts_dump_fun_entries(to, to_arg);
+    erts_deep_process_dump(to, to_arg);
+    erts_cbprintf(to, to_arg, "=atoms\n");
+    dump_atoms(to, to_arg);
 
     /* Keep the instrumentation data at the end of the dump */
     if (erts_instr_memory_map || erts_instr_stat) {
-	erts_fdprintf(fd, "=instr_data\n");
+	erts_cbprintf(to, to_arg, "=instr_data\n");
 
 	if (erts_instr_stat) {
-	    erts_fdprintf(fd, "=memory_status\n");
-	    erts_instr_dump_stat_to_fd(fd, 0);
+	    erts_cbprintf(to, to_arg, "=memory_status\n");
+	    erts_instr_dump_stat_to(to, to_arg, 0);
 	}
 	if (erts_instr_memory_map) {
-	    erts_fdprintf(fd, "=memory_map\n");
-	    erts_instr_dump_memory_map_to_fd(fd);
+	    erts_cbprintf(to, to_arg, "=memory_map\n");
+	    erts_instr_dump_memory_map_to(to, to_arg);
 	}
     }
 
-    erts_fdprintf(fd, "=end\n");
+    erts_cbprintf(to, to_arg, "=end\n");
     close(fd);
     erts_fprintf(stderr,"done\n");
 }
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 09c83f1117..d79245e0e6 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -2402,13 +2402,13 @@ erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id)
 }
 
 struct print_to_data {
-    int to;
+    fmtfn_t to;
     void *arg;
 };
 
 static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp)
 {
-    int to = ((struct print_to_data *) vptdp)->to;
+    fmtfn_t to = ((struct print_to_data *) vptdp)->to;
     void *arg = ((struct print_to_data *) vptdp)->arg;
     Process *rp;
     ErtsMonitor *rmon;
@@ -2431,7 +2431,7 @@ static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp)
     }
 }    
 
-static void print_monitor_info(int to, void *arg, ErtsMonitor *mon)
+static void print_monitor_info(fmtfn_t to, void *arg, ErtsMonitor *mon)
 {
     struct print_to_data ptd = {to, arg};
     erts_doforall_monitors(mon,&doit_print_monitor_info,&ptd);
@@ -2457,7 +2457,7 @@ static void doit_print_link_info(ErtsLink *lnk, void *vptdp)
     } 
 }
 
-static void print_link_info(int to, void *arg, ErtsLink *lnk)
+static void print_link_info(fmtfn_t to, void *arg, ErtsLink *lnk)
 {
     struct print_to_data ptd = {to, arg};
     erts_doforall_links(lnk, &doit_print_link_info, (void *) &ptd);
@@ -2478,7 +2478,7 @@ static void doit_print_nodelink_info(ErtsLink *lnk, void *vpcontext)
 		   "Remote monitoring: %T %T\n", lnk->pid, pcontext->sysname);
 }
 
-static void print_nodelink_info(int to, void *arg, ErtsLink *lnk, Eterm sysname)
+static void print_nodelink_info(fmtfn_t to, void *arg, ErtsLink *lnk, Eterm sysname)
 {
     PrintNodeLinkContext context = {{to, arg}, sysname};
     erts_doforall_links(lnk, &doit_print_nodelink_info, &context);
@@ -2486,7 +2486,7 @@ static void print_nodelink_info(int to, void *arg, ErtsLink *lnk, Eterm sysname)
 
 
 static int
-info_dist_entry(int to, void *arg, DistEntry *dep, int visible, int connected)
+info_dist_entry(fmtfn_t to, void *arg, DistEntry *dep, int visible, int connected)
 {
 
   if (visible && connected) {
@@ -2535,7 +2535,7 @@ info_dist_entry(int to, void *arg, DistEntry *dep, int visible, int connected)
   return 0;
     
 }
-int distribution_info(int to, void *arg)	/* Called by break handler */
+int distribution_info(fmtfn_t to, void *arg)	/* Called by break handler */
 {
     DistEntry *dep;
 
diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c
index eda3ad870a..4ebe37ee1d 100644
--- a/erts/emulator/beam/erl_afit_alloc.c
+++ b/erts/emulator/beam/erl_afit_alloc.c
@@ -54,7 +54,7 @@ static void		link_free_block		(Allctr_t *, Block_t *);
 static void		unlink_free_block	(Allctr_t *, Block_t *);
 
 
-static Eterm		info_options		(Allctr_t *, char *, int *,
+static Eterm		info_options		(Allctr_t *, char *, fmtfn_t *,
 						 void *arg, Uint **, Uint *);
 static void		init_atoms		(void);
 
@@ -227,7 +227,7 @@ add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2)
 static Eterm
 info_options(Allctr_t *allctr,
 	     char *prefix,
-	     int *print_to_p,
+	     fmtfn_t *print_to_p,
 	     void *print_to_arg,
 	     Uint **hpp,
 	     Uint *szp)
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 3c2c9def3b..214fb1f2af 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -2113,7 +2113,7 @@ add_fix_values(UWord *ap, UWord *up, ErtsAlcUFixInfo_t *fi, ErtsAlcType_t type)
 }
 
 Eterm
-erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
+erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 {
 /*
  * NOTE! When updating this function, make sure to also update
@@ -2476,7 +2476,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 
     if (print_to_p) {
 	int i;
-	int to = *print_to_p;
+	fmtfn_t to = *print_to_p;
 	void *arg = print_to_arg;
 
 	/* Print result... */
@@ -2530,7 +2530,7 @@ struct aa_values {
 };
 
 Eterm
-erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
+erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc)
 {
 #define MAX_AA_VALUES (24)
     struct aa_values values[MAX_AA_VALUES];
@@ -2665,7 +2665,7 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
 
     if (print_to_p) {
 	/* Print result... */
-	int to = *print_to_p;
+	fmtfn_t to = *print_to_p;
 	void *arg = print_to_arg;
 
 	erts_print(to, arg, "=allocated_areas\n");
@@ -2779,7 +2779,7 @@ erts_alloc_util_allocators(void *proc)
 }
 
 void
-erts_allocator_info(int to, void *arg)
+erts_allocator_info(fmtfn_t to, void *arg)
 {
     ErtsAlcType_t a;
 
@@ -3110,7 +3110,7 @@ reply_alloc_info(void *vair)
     Eterm (*info_func)(Allctr_t *,
 		       int,
 		       int,
-		       int *,
+		       fmtfn_t *,
 		       void *,
 		       Uint **,
 		       Uint *) = (air->only_sz
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 925a081a02..56a3b73bf9 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -69,11 +69,11 @@ void *erts_sys_aligned_realloc(UWord alignment, void *ptr, UWord size, UWord old
 void erts_sys_aligned_free(UWord alignment, void *ptr);
 #endif
 
-Eterm erts_memory(int *, void *, void *, Eterm);
-Eterm erts_allocated_areas(int *, void *, void *);
+Eterm erts_memory(fmtfn_t *, void *, void *, Eterm);
+Eterm erts_allocated_areas(fmtfn_t *, void *, void *);
 
 Eterm erts_alloc_util_allocators(void *proc);
-void erts_allocator_info(int, void *);
+void erts_allocator_info(fmtfn_t, void *);
 Eterm erts_allocator_options(void *proc);
 
 struct process;
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 2995f2f822..f4ca689040 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -4473,7 +4473,7 @@ add_fix_types(Allctr_t *allctr, int internal, Uint **hpp, Uint *szp,
 static Eterm
 sz_info_fix(Allctr_t *allctr,
 	    int internal,
-	    int *print_to_p,
+	    fmtfn_t *print_to_p,
 	    void *print_to_arg,
 	    Uint **hpp,
 	    Uint *szp)
@@ -4494,7 +4494,7 @@ sz_info_fix(Allctr_t *allctr,
 		UWord used = fix->type_size * fix->u.cpool.used;
 
 		if (print_to_p) {
-		    int to = *print_to_p;
+		    fmtfn_t to = *print_to_p;
 		    void *arg = print_to_arg;
 		    erts_print(to,
 			       arg,
@@ -4522,7 +4522,7 @@ sz_info_fix(Allctr_t *allctr,
 	    UWord used = fix->type_size*fix->u.nocpool.used;
 
 	    if (print_to_p) {
-		int to = *print_to_p;
+		fmtfn_t to = *print_to_p;
 		void *arg = print_to_arg;
 		erts_print(to,
 			   arg,
@@ -4548,7 +4548,7 @@ static Eterm
 sz_info_carriers(Allctr_t *allctr,
 		 CarriersStats_t *cs,
 		 char *prefix,
-		 int *print_to_p,
+		 fmtfn_t *print_to_p,
 		 void *print_to_arg,
 		 Uint **hpp,
 		 Uint *szp)
@@ -4557,7 +4557,7 @@ sz_info_carriers(Allctr_t *allctr,
     UWord curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size;
 
     if (print_to_p) {
-	int to = *print_to_p;
+	fmtfn_t to = *print_to_p;
 	void *arg = print_to_arg;
 	erts_print(to,
 		   arg,
@@ -4598,7 +4598,7 @@ static Eterm
 info_cpool(Allctr_t *allctr,
 	   int sz_only,
 	   char *prefix,
-	   int *print_to_p,
+	   fmtfn_t *print_to_p,
 	   void *print_to_arg,
 	   Uint **hpp,
 	   Uint *szp)
@@ -4615,7 +4615,7 @@ info_cpool(Allctr_t *allctr,
     }
 
     if (print_to_p) {
-	int to = *print_to_p;
+	fmtfn_t to = *print_to_p;
 	void *arg = print_to_arg;
 	if (!sz_only)
 	    erts_print(to, arg, "%sblocks: %bpu\n", prefix, nob);
@@ -4652,7 +4652,7 @@ static Eterm
 info_carriers(Allctr_t *allctr,
 	      CarriersStats_t *cs,
 	      char *prefix,
-	      int *print_to_p,
+	      fmtfn_t *print_to_p,
 	      void *print_to_arg,
 	      Uint **hpp,
 	      Uint *szp)
@@ -4664,7 +4664,7 @@ info_carriers(Allctr_t *allctr,
     curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size;
 
     if (print_to_p) {
-	int to = *print_to_p;
+	fmtfn_t to = *print_to_p;
 	void *arg = print_to_arg;
 	erts_print(to,
 		   arg,
@@ -4790,7 +4790,7 @@ make_name_atoms(Allctr_t *allctr)
 
 static Eterm
 info_calls(Allctr_t *allctr,
-	   int *print_to_p,
+	   fmtfn_t *print_to_p,
 	   void *print_to_arg,
 	   Uint **hpp,
 	   Uint *szp)
@@ -4807,7 +4807,7 @@ info_calls(Allctr_t *allctr,
 	erts_print(TO, TOA, "%s%s calls: %b64u\n",PRFX,NAME,CC)
 
 	char *prefix = allctr->name_prefix;
-	int to = *print_to_p;
+	fmtfn_t to = *print_to_p;
 	void *arg = print_to_arg;
 
 	PRINT_CC_5(to, arg, prefix, "alloc",        allctr->calls.this_alloc);
@@ -4883,7 +4883,7 @@ info_calls(Allctr_t *allctr,
 
 static Eterm
 info_options(Allctr_t *allctr,
-             int *print_to_p,
+             fmtfn_t *print_to_p,
 	     void *print_to_arg,
 	     Uint **hpp,
 	     Uint *szp)
@@ -5035,7 +5035,7 @@ reset_max_values(CarriersStats_t *cs)
 \*                                                                         */
 
 Eterm
-erts_alcu_au_info_options(int *print_to_p, void *print_to_arg,
+erts_alcu_au_info_options(fmtfn_t *print_to_p, void *print_to_arg,
 			  Uint **hpp, Uint *szp)
 {
     Eterm res = THE_NON_VALUE;    
@@ -5078,7 +5078,7 @@ erts_alcu_au_info_options(int *print_to_p, void *print_to_arg,
 
 Eterm
 erts_alcu_info_options(Allctr_t *allctr,
-		       int *print_to_p,
+		       fmtfn_t *print_to_p,
 		       void *print_to_arg,
 		       Uint **hpp,
 		       Uint *szp)
@@ -5110,7 +5110,7 @@ Eterm
 erts_alcu_sz_info(Allctr_t *allctr,
 		  int internal,
 		  int begin_max_period,
-		  int *print_to_p,
+		  fmtfn_t *print_to_p,
 		  void *print_to_arg,
 		  Uint **hpp,
 		  Uint *szp)
@@ -5196,7 +5196,7 @@ Eterm
 erts_alcu_info(Allctr_t *allctr,
 	       int internal,
 	       int begin_max_period,
-	       int *print_to_p,
+	       fmtfn_t *print_to_p,
 	       void *print_to_arg,
 	       Uint **hpp,
 	       Uint *szp)
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index f50f09907a..81180382af 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -178,10 +178,10 @@ void *	erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t, void *, void *, Uint);
 void	erts_alcu_free_thr_pref(ErtsAlcType_t, void *, void *);
 #endif
 #endif
-Eterm	erts_alcu_au_info_options(int *, void *, Uint **, Uint *);
-Eterm	erts_alcu_info_options(Allctr_t *, int *, void *, Uint **, Uint *);
-Eterm	erts_alcu_sz_info(Allctr_t *, int, int, int *, void *, Uint **, Uint *);
-Eterm	erts_alcu_info(Allctr_t *, int, int, int *, void *, Uint **, Uint *);
+Eterm	erts_alcu_au_info_options(fmtfn_t *, void *, Uint **, Uint *);
+Eterm	erts_alcu_info_options(Allctr_t *, fmtfn_t *, void *, Uint **, Uint *);
+Eterm	erts_alcu_sz_info(Allctr_t *, int, int, fmtfn_t *, void *, Uint **, Uint *);
+Eterm	erts_alcu_info(Allctr_t *, int, int, fmtfn_t *, void *, Uint **, Uint *);
 void	erts_alcu_init(AlcUInit_t *);
 void    erts_alcu_current_size(Allctr_t *, AllctrSize_t *,
 			       ErtsAlcUFixInfo_t *, int);
@@ -586,7 +586,7 @@ struct Allctr_t_ {
 						 Block_t *, Uint);
     void		(*link_free_block)	(Allctr_t *, Block_t *);
     void		(*unlink_free_block)	(Allctr_t *, Block_t *);
-    Eterm		(*info_options)		(Allctr_t *, char *, int *,
+    Eterm		(*info_options)		(Allctr_t *, char *, fmtfn_t *,
 						 void *, Uint **, Uint *);
 
     Uint		(*get_next_mbc_size)	(Allctr_t *);
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index 7e239d1f5d..05ba1f9891 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -224,7 +224,7 @@ static AOFF_RBTree_t* rbt_search(AOFF_RBTree_t* root, Uint size);
 static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node);
 #endif
 
-static Eterm info_options(Allctr_t *, char *, int *, void *, Uint **, Uint *);
+static Eterm info_options(Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *);
 static void init_atoms(void);
 
 
@@ -1014,7 +1014,7 @@ add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2)
 static Eterm
 info_options(Allctr_t *allctr,
 	     char *prefix,
-	     int *print_to_p,
+	     fmtfn_t *print_to_p,
 	     void *print_to_arg,
 	     Uint **hpp,
 	     Uint *szp)
diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c
index 379cee39a1..6173c408e1 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.c
+++ b/erts/emulator/beam/erl_bestfit_alloc.c
@@ -104,7 +104,7 @@ static void		bf_link_free_block	(Allctr_t *, Block_t *);
 static ERTS_INLINE void	bf_unlink_free_block	(Allctr_t *, Block_t *);
 
 
-static Eterm		info_options		(Allctr_t *, char *, int *,
+static Eterm		info_options		(Allctr_t *, char *, fmtfn_t *,
 						 void *, Uint **, Uint *);
 static void		init_atoms		(void);
 
@@ -921,7 +921,7 @@ add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2)
 static Eterm
 info_options(Allctr_t *allctr,
 	     char *prefix,
-	     int *print_to_p,
+	     fmtfn_t *print_to_p,
 	     void *print_to_arg,
 	     Uint **hpp,
 	     Uint *szp)
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 29ba12dfdb..735aabbee3 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -317,7 +317,7 @@ make_link_list(Process *p, ErtsLink *root, Eterm tail)
 }
 
 int
-erts_print_system_version(int to, void *arg, Process *c_p)
+erts_print_system_version(fmtfn_t to, void *arg, Process *c_p)
 {
     int i, rc = -1;
     char *rc_str = "";
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 128a7b3865..dceadc46f4 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -201,7 +201,7 @@ static int free_table_cont(Process *p,
 			   DbTable *tb,
 			   int first,
 			   int clean_meta_tab);
-static void print_table(int to, void *to_arg, int show,  DbTable* tb);
+static void print_table(fmtfn_t to, void *to_arg, int show,  DbTable* tb);
 static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1);
 static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1);
 static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1);
@@ -3871,7 +3871,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
     return ret;
 }
 
-static void print_table(int to, void *to_arg, int show,  DbTable* tb)
+static void print_table(fmtfn_t to, void *to_arg, int show,  DbTable* tb)
 {
     erts_print(to, to_arg, "Table: %T\n", tb->common.id);
     erts_print(to, to_arg, "Name: %T\n", tb->common.the_name);
@@ -3891,7 +3891,7 @@ static void print_table(int to, void *to_arg, int show,  DbTable* tb)
     erts_print(to, to_arg, "Read Concurrency: %T\n", table_info(NULL, tb, am_read_concurrency));
 }
 
-void db_info(int to, void *to_arg, int show)    /* Called by break handler */
+void db_info(fmtfn_t to, void *to_arg, int show)    /* Called by break handler */
 {
     int i;
     for (i=0; i < db_max_tabs; i++) 
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index 1d26c49652..b0508f2e74 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -74,7 +74,7 @@ typedef enum {
 
 void init_db(ErtsDbSpinCount);
 int erts_db_process_exiting(Process *, ErtsProcLocks);
-void db_info(int, void *, int);
+void db_info(fmtfn_t, void *, int);
 void erts_db_foreach_table(void (*)(DbTable *, void *), void *);
 void erts_db_foreach_offheap(DbTable *,
 			     void (*func)(ErlOffHeap *, void *),
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 5e6fe4f460..390369fdb9 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -437,7 +437,7 @@ static int db_select_count_continue_hash(Process *p, DbTable *tbl,
 static int db_select_delete_continue_hash(Process *p, DbTable *tbl,
 					  Eterm continuation, Eterm *ret);
 static int db_take_hash(Process *, DbTable *, Eterm, Eterm *);
-static void db_print_hash(int to,
+static void db_print_hash(fmtfn_t to,
 			  void *to_arg,
 			  int show,
 			  DbTable *tbl);
@@ -2155,7 +2155,7 @@ int db_mark_all_deleted_hash(DbTable *tbl)
 
 
 /* Display hash table contents (for dump) */
-static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl)
+static void db_print_hash(fmtfn_t to, void *to_arg, int show, DbTable *tbl)
 {
     DbTableHash *tb = &tbl->hash;
     DbHashStats stats;
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 02d211a4bb..dd9403e132 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -385,7 +385,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
 static int db_select_delete_continue_tree(Process *p, DbTable *tbl, 
 					  Eterm continuation, Eterm *ret);
 static int db_take_tree(Process *, DbTable *, Eterm, Eterm *);
-static void db_print_tree(int to, void *to_arg,
+static void db_print_tree(fmtfn_t to, void *to_arg,
 			  int show, DbTable *tbl);
 static int db_free_table_tree(DbTable *tbl);
 
@@ -1740,7 +1740,7 @@ static int db_take_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
 
 
 /* Display tree contents (for dump) */
-static void db_print_tree(int to, void *to_arg, 
+static void db_print_tree(fmtfn_t to, void *to_arg,
 			  int show,
 			  DbTable *tbl)
 {
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 4acedbfed0..49e5f6b4cf 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -175,7 +175,7 @@ typedef struct db_table_method
     int (*db_free_table)(DbTable* db /* [in out] */ );
     int (*db_free_table_continue)(DbTable* db); /* [in out] */  
     
-    void (*db_print)(int to, 
+    void (*db_print)(fmtfn_t to,
 		     void* to_arg, 
 		     int show, 
 		     DbTable* tb /* [in out] */ );
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index 3e3bfa03a2..3526bb684d 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -60,10 +60,10 @@ static const char dashes[PTR_SIZE+3] = {
 
 void pps(Process*, Eterm*);
 void ptd(Process*, Eterm);
-void paranoid_display(int, void*, Process*, Eterm);
+void paranoid_display(fmtfn_t, void*, Process*, Eterm);
 static int dcount;
 
-static int pdisplay1(int to, void *to_arg, Process* p, Eterm obj);
+static int pdisplay1(fmtfn_t to, void *to_arg, Process* p, Eterm obj);
 
 void ptd(Process* p, Eterm x) 
 {
@@ -77,14 +77,14 @@ void ptd(Process* p, Eterm x)
  */
 
 void
-paranoid_display(int to, void *to_arg, Process* p, Eterm obj)
+paranoid_display(fmtfn_t to, void *to_arg, Process* p, Eterm obj)
 {
     dcount = 100000;
     pdisplay1(to, to_arg, p, obj);
 }
 
 static int
-pdisplay1(int to, void *to_arg, Process* p, Eterm obj)
+pdisplay1(fmtfn_t to, void *to_arg, Process* p, Eterm obj)
 {
     int i, k;
     Eterm* nobj;
@@ -201,7 +201,7 @@ pdisplay1(int to, void *to_arg, Process* p, Eterm obj)
 void
 pps(Process* p, Eterm* stop)
 {
-    int to = ERTS_PRINT_STDOUT;
+    fmtfn_t to = ERTS_PRINT_STDOUT;
     void *to_arg = NULL;
     Eterm* sp = STACK_START(p) - 1;
 
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index c639ba623f..dad77071b3 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -74,7 +74,7 @@ erts_init_fun_table(void)
 }
 
 void
-erts_fun_info(int to, void *to_arg)
+erts_fun_info(fmtfn_t to, void *to_arg)
 {
     int lock = !ERTS_IS_CRASH_DUMPING;
     if (lock)
@@ -264,7 +264,7 @@ erts_fun_purge_complete(ErlFunEntry **funs, Uint no)
 }
 
 void
-erts_dump_fun_entries(int to, void *to_arg)
+erts_dump_fun_entries(fmtfn_t to, void *to_arg)
 {
     int limit;
     HashBucket** bucket;
diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h
index 73c3e19c1c..caa55c730c 100644
--- a/erts/emulator/beam/erl_fun.h
+++ b/erts/emulator/beam/erl_fun.h
@@ -71,7 +71,7 @@ typedef struct erl_fun_thing {
 #define ERL_FUN_SIZE ((sizeof(ErlFunThing)/sizeof(Eterm))-1)
 
 void erts_init_fun_table(void);
-void erts_fun_info(int, void *);
+void erts_fun_info(fmtfn_t, void *);
 int erts_fun_table_sz(void);
 
 ErlFunEntry* erts_put_fun_entry(Eterm mod, int uniq, int index);
@@ -86,6 +86,6 @@ void erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end);
 void erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no);
 void erts_fun_purge_abort_finalize(ErlFunEntry **funs, Uint no);
 void erts_fun_purge_complete(ErlFunEntry **funs, Uint no);
-void erts_dump_fun_entries(int, void *);
+void erts_dump_fun_entries(fmtfn_t, void *);
 
 #endif
diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c
index 223ba193da..50aa41b4d2 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.c
+++ b/erts/emulator/beam/erl_goodfit_alloc.c
@@ -168,7 +168,7 @@ static Block_t *	get_free_block		(Allctr_t *, Uint,
 static void		link_free_block		(Allctr_t *, Block_t *);
 static void		unlink_free_block	(Allctr_t *, Block_t *);
 static void		update_last_aux_mbc	(Allctr_t *, Carrier_t *);
-static Eterm		info_options		(Allctr_t *, char *, int *,
+static Eterm		info_options		(Allctr_t *, char *, fmtfn_t *,
 						 void *, Uint **, Uint *);
 static void		init_atoms		(void);
 
@@ -551,7 +551,7 @@ add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2)
 static Eterm
 info_options(Allctr_t *allctr,
 	     char *prefix,
-	     int *print_to_p,
+	     fmtfn_t *print_to_p,
 	     void *print_to_arg,
 	     Uint **hpp,
 	     Uint *szp)
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index f1bef28186..d29d079fc5 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -2851,7 +2851,7 @@ erts_read_port_timer(Port *c_prt)
  */
 
 typedef struct {
-    int to;
+    fmtfn_t to;
     void *to_arg;
     ErtsMonotonicTime now;
 } ErtsBTMPrint;
@@ -2881,7 +2881,7 @@ btm_print(ErtsHLTimer *tmr, void *vbtmp)
 }
 
 void
-erts_print_bif_timer_info(int to, void *to_arg)
+erts_print_bif_timer_info(fmtfn_t to, void *to_arg)
 {
     ErtsBTMPrint btmp;
     int six;
diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h
index 0931bb8965..705be94532 100644
--- a/erts/emulator/beam/erl_hl_timer.h
+++ b/erts/emulator/beam/erl_hl_timer.h
@@ -72,7 +72,7 @@ erts_handle_canceled_timers(void *vesdp,
 #endif
 
 Uint erts_bif_timer_memory_size(void);
-void erts_print_bif_timer_info(int to, void *to_arg);
+void erts_print_bif_timer_info(fmtfn_t to, void *to_arg);
 
 void erts_debug_bif_timer_foreach(void (*func)(Eterm,
 					       Eterm,
diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c
index f84c63e7a4..4d4defd8b5 100644
--- a/erts/emulator/beam/erl_instrument.c
+++ b/erts/emulator/beam/erl_instrument.c
@@ -539,7 +539,7 @@ map_stat_free(ErtsAlcType_t n, void *extra, void *ptr)
 
 }
 
-static void dump_memory_map_to_stream(FILE *fp)
+static void dump_memory_map_to_stream(fmtfn_t to, void* to_arg)
 {
     ErtsAlcType_t n;
     MapStatBlock_t *bp;
@@ -551,7 +551,7 @@ static void dump_memory_map_to_stream(FILE *fp)
 
     /* Write header */
 
-    fprintf(fp,
+    erts_cbprintf(to, to_arg,
 	    "{instr_hdr,\n"
 	    " %lu,\n"
 	    " %lu,\n"
@@ -574,7 +574,7 @@ static void dump_memory_map_to_stream(FILE *fp)
 	else
 	    astr = ERTS_ALC_A2AD(ERTS_ALC_A_SYSTEM);
 
-	fprintf(fp,
+	erts_cbprintf(to, to_arg,
 		"%s{%s,%s,%s}%s",
 		(n == ERTS_ALC_N_MIN) ? "" : "  ",
 		ERTS_ALC_N2TD(n),
@@ -583,12 +583,12 @@ static void dump_memory_map_to_stream(FILE *fp)
 		(n == ERTS_ALC_N_MAX) ? "" : ",\n");
     }
 
-    fprintf(fp, "}}.\n");
+    erts_cbprintf(to, to_arg, "}}.\n");
 
     /* Write memory data */
     for (bp = mem_anchor; bp; bp = bp->next) {
 	if (is_internal_pid(bp->pid))
-	    fprintf(fp,
+	    erts_cbprintf(to, to_arg,
 		    "{%lu, %lu, %lu, {%lu,%lu,%lu}}.\n",
 		    (UWord) bp->type_no,
 		    (UWord) bp->mem,
@@ -597,7 +597,7 @@ static void dump_memory_map_to_stream(FILE *fp)
 		    (UWord) pid_number(bp->pid),
 		    (UWord) pid_serial(bp->pid));
 	else
-	    fprintf(fp,
+	    erts_cbprintf(to, to_arg,
 		    "{%lu, %lu, %lu, undefined}.\n",
 		    (UWord) bp->type_no,
 		    (UWord) bp->mem,
@@ -608,40 +608,29 @@ static void dump_memory_map_to_stream(FILE *fp)
 	erts_mtx_unlock(&instr_mutex);
 }
 
-int erts_instr_dump_memory_map_to_fd(int fd)
+int erts_instr_dump_memory_map_to(fmtfn_t to, void* to_arg)
 {
-    char buf[BUFSIZ];
-    FILE *f;
-
     if (!erts_instr_memory_map)
 	return 0;
 
-    f = fdopen(fd, "w");
-    if (f == NULL)
-	return 0;
-
-    /* Avoid allocating memory; we may have run out of it at this point. */
-    setbuf(f, buf);
-
-    dump_memory_map_to_stream(f);
-    fflush(f);
+    dump_memory_map_to_stream(to, to_arg);
     return 1;
 }
 
 int erts_instr_dump_memory_map(const char *name)
 {
-    FILE *f;
+    int fd;
 
     if (!erts_instr_memory_map)
 	return 0;
 
-    f = fopen(name, "w");
-    if (f == NULL)
+    fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, 0640);
+    if (fd < 0)
 	return 0;
 
-    dump_memory_map_to_stream(f);
+    dump_memory_map_to_stream(erts_write_fd, (void*)&fd);
 
-    fclose(f);
+    close(fd);
     return 1;
 }
 
@@ -998,19 +987,19 @@ erts_instr_get_stat(Process *proc, Eterm what, int begin_max_period)
 }
 
 static void
-dump_stat_to_stream(FILE *fp, int begin_max_period)
+dump_stat_to_stream(fmtfn_t to, void* to_arg, int begin_max_period)
 {
     ErtsAlcType_t i, a_max, a_min;
 
     erts_mtx_lock(&instr_mutex);
 
-    fprintf(fp,
+    erts_cbprintf(to, to_arg,
 	    "{instr_vsn,%lu}.\n",
 	    (unsigned long) ERTS_INSTR_VSN);
     
     update_max_ever_values(&stats->tot, 0, 0);
 
-    fprintf(fp,
+    erts_cbprintf(to, to_arg,
 	    "{total,[{total,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}]}.\n",
 	    (UWord) stats->tot.size,
 	    (UWord) stats->tot.max_size,
@@ -1038,7 +1027,7 @@ dump_stat_to_stream(FILE *fp, int begin_max_period)
 
     for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
 	if (erts_allctrs_info[i].enabled) {
-	    fprintf(fp,
+	    erts_cbprintf(to, to_arg,
 		    "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s",
 		    i == a_min ? "{allocators,\n [" : "  ",
 		    ERTS_ALC_A2AD(i),
@@ -1055,7 +1044,7 @@ dump_stat_to_stream(FILE *fp, int begin_max_period)
     update_max_ever_values(stats->c, ERTS_ALC_C_MIN, ERTS_ALC_C_MAX);
 
     for (i = ERTS_ALC_C_MIN; i <= ERTS_ALC_C_MAX; i++) {
-	fprintf(fp,
+	erts_cbprintf(to, to_arg,
 		"%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s",
 		i == ERTS_ALC_C_MIN ? "{classes,\n [" : "  ",
 		ERTS_ALC_C2CD(i),
@@ -1071,7 +1060,7 @@ dump_stat_to_stream(FILE *fp, int begin_max_period)
     update_max_ever_values(stats->n, ERTS_ALC_N_MIN, ERTS_ALC_N_MAX);
 
     for (i = ERTS_ALC_N_MIN; i <= ERTS_ALC_N_MAX; i++) {
-	fprintf(fp,
+	erts_cbprintf(to, to_arg,
 		"%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s",
 		i == ERTS_ALC_N_MIN ? "{types,\n [" : "  ",
 		ERTS_ALC_N2TD(i),
@@ -1095,40 +1084,29 @@ dump_stat_to_stream(FILE *fp, int begin_max_period)
 
 }
 
-int erts_instr_dump_stat_to_fd(int fd, int begin_max_period)
+int erts_instr_dump_stat_to(fmtfn_t to, void* to_arg, int begin_max_period)
 {
-    char buf[BUFSIZ];
-    FILE *fp;
-
     if (!erts_instr_stat)
 	return 0;
 
-    fp = fdopen(fd, "w");
-    if (fp == NULL)
-	return 0;
-
-    /* Avoid allocating memory; we may have run out of it at this point. */
-    setbuf(fp, buf);
-
-    dump_stat_to_stream(fp, begin_max_period);
-    fflush(fp);
+    dump_stat_to_stream(to, to_arg, begin_max_period);
     return 1;
 }
 
 int erts_instr_dump_stat(const char *name, int begin_max_period)
 {
-    FILE *file;
+    int fd;
 
     if (!erts_instr_stat)
 	return 0;
 
-    file = fopen(name, "w");
-    if (file == NULL)
+    fd = open(name, O_WRONLY | O_CREAT | O_TRUNC,0640);
+    if (fd < 0)
 	return 0;
 
-    dump_stat_to_stream(file, begin_max_period);
+    dump_stat_to_stream(erts_write_fd, (void*)&fd, begin_max_period);
 
-    fclose(file);
+    close(fd);
     return 1;
 }
 
diff --git a/erts/emulator/beam/erl_instrument.h b/erts/emulator/beam/erl_instrument.h
index 1f04c91d5e..351172b2fa 100644
--- a/erts/emulator/beam/erl_instrument.h
+++ b/erts/emulator/beam/erl_instrument.h
@@ -29,10 +29,10 @@ extern int erts_instr_memory_map;
 extern int erts_instr_stat;
 
 Uint  erts_instr_init(int stat, int map_stat);
-int   erts_instr_dump_memory_map_to_fd(int fd);
+int   erts_instr_dump_memory_map_to(fmtfn_t to, void* to_arg);
 int   erts_instr_dump_memory_map(const char *name);
 Eterm erts_instr_get_memory_map(Process *process);
-int   erts_instr_dump_stat_to_fd(int fd, int begin_max_period);
+int   erts_instr_dump_stat_to(fmtfn_t to, void* to_arg, int begin_max_period);
 int   erts_instr_dump_stat(const char *name, int begin_max_period);
 Eterm erts_instr_get_stat(Process *proc, Eterm what, int begin_max_period);
 Eterm erts_instr_get_type_info(Process *proc);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 3a547982da..20b6c76cb7 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3068,16 +3068,16 @@ Eterm erts_nif_taints(Process* p)
     return list;
 }
 
-void erts_print_nif_taints(int to, void* to_arg)
+void erts_print_nif_taints(fmtfn_t to, void* to_arg)
 {
     struct tainted_module_t* t;
     const char* delim = "";
     for (t=first_tainted_module ; t!=NULL; t=t->next) {
 	const Atom* atom = atom_tab(atom_val(t->module_atom));
-	erts_print(to,to_arg,"%s%.*s", delim, atom->len, atom->name);
+	erts_cbprintf(to,to_arg,"%s%.*s", delim, atom->len, atom->name);
 	delim = ",";
     }
-    erts_print(to,to_arg,"\n");
+    erts_cbprintf(to,to_arg,"\n");
 }
 
 
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 646f786651..70500ed6e1 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -188,7 +188,7 @@ dist_table_free(void *vdep)
 
 
 void
-erts_dist_table_info(int to, void *to_arg)
+erts_dist_table_info(fmtfn_t to, void *to_arg)
 {
     int lock = !ERTS_IS_CRASH_DUMPING;
     if (lock)
@@ -564,7 +564,7 @@ erts_node_table_size(void)
 }
 
 void
-erts_node_table_info(int to, void *to_arg)
+erts_node_table_info(fmtfn_t to, void *to_arg)
 {
     int lock = !ERTS_IS_CRASH_DUMPING;
     if (lock)
@@ -649,7 +649,7 @@ void erts_schedule_delete_node(ErlNode *enp)
 }
 
 struct pn_data {
-    int to;
+    fmtfn_t to;
     void *to_arg;
     Eterm sysname;
     int no_sysname;
@@ -679,7 +679,7 @@ static void print_node(void *venp, void *vpndp)
     pndp->no_total++;
 }
 
-void erts_print_node_info(int to,
+void erts_print_node_info(fmtfn_t to,
 			  void *to_arg,
 			  Eterm sysname,
 			  int *no_sysname,
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 7a4434acbf..47a6724c21 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -179,7 +179,7 @@ DistEntry *erts_find_or_insert_dist_entry(Eterm);
 DistEntry *erts_find_dist_entry(Eterm);
 void erts_schedule_delete_dist_entry(DistEntry *);
 Uint erts_dist_table_size(void);
-void erts_dist_table_info(int, void *);
+void erts_dist_table_info(fmtfn_t, void *);
 void erts_set_dist_entry_not_connected(DistEntry *);
 void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint);
 ErlNode *erts_find_or_insert_node(Eterm, Uint32);
@@ -187,8 +187,8 @@ void erts_schedule_delete_node(ErlNode *);
 void erts_set_this_node(Eterm, Uint);
 Uint erts_node_table_size(void);
 void erts_init_node_tables(int);
-void erts_node_table_info(int, void *);
-void erts_print_node_info(int, void *, Eterm, int*, int*);
+void erts_node_table_info(fmtfn_t, void *);
+void erts_print_node_info(fmtfn_t, void *, Eterm, int*, int*);
 Eterm erts_get_node_and_dist_references(struct process *);
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
 int erts_lc_is_de_rwlocked(DistEntry *);
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index f90844ccc8..c59b42cdae 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -369,7 +369,7 @@ Eterm erts_request_io_bytes(Process *c_p);
 #define ERTS_PORT_REDS_INFO		(CONTEXT_REDS/100)
 #define ERTS_PORT_REDS_TERMINATE	(CONTEXT_REDS/50)
 
-void print_port_info(Port *, int, void *);
+void print_port_info(Port *, fmtfn_t, void *);
 void erts_port_free(Port *);
 #ifndef ERTS_SMP
 void erts_port_cleanup(Port *);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index bc59147c6c..b989d3be7a 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -557,8 +557,8 @@ do {									\
  */
 
 static void exec_misc_ops(ErtsRunQueue *);
-static void print_function_from_pc(int to, void *to_arg, BeamInstr* x);
-static int stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg);
+static void print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x);
+static int stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg);
 
 static void aux_work_timeout(void *unused);
 static void aux_work_timeout_early_init(int no_schedulers);
@@ -13171,7 +13171,7 @@ erts_continue_exit_process(Process *p)
  */
 
 void
-erts_stack_dump(int to, void *to_arg, Process *p)
+erts_stack_dump(fmtfn_t to, void *to_arg, Process *p)
 {
     Eterm* sp;
     int yreg = -1;
@@ -13186,7 +13186,7 @@ erts_stack_dump(int to, void *to_arg, Process *p)
 }
 
 void
-erts_program_counter_info(int to, void *to_arg, Process *p)
+erts_program_counter_info(fmtfn_t to, void *to_arg, Process *p)
 {
     erts_aint32_t state;
     int i;
@@ -13216,7 +13216,7 @@ erts_program_counter_info(int to, void *to_arg, Process *p)
 }
 
 static void
-print_function_from_pc(int to, void *to_arg, BeamInstr* x)
+print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x)
 {
     BeamInstr* addr = find_function_from_pc(x);
     if (addr == NULL) {
@@ -13238,7 +13238,7 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x)
 }
 
 static int
-stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg)
+stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg)
 {
     Eterm x = *sp;
 
@@ -13270,7 +13270,7 @@ stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg)
  * Print scheduler information
  */
 void
-erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) {
+erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
     int i;
     erts_aint32_t flg;
     Process *p;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 3347a7a60e..9e5121abd5 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1837,12 +1837,12 @@ void erts_cleanup_empty_process(Process* p);
 #ifdef DEBUG
 void erts_debug_verify_clean_empty_process(Process* p);
 #endif
-void erts_stack_dump(int to, void *to_arg, Process *);
-void erts_limited_stack_trace(int to, void *to_arg, Process *);
-void erts_program_counter_info(int to, void *to_arg, Process *);
-void erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp);
-void erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg);
-void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg);
+void erts_stack_dump(fmtfn_t to, void *to_arg, Process *);
+void erts_limited_stack_trace(fmtfn_t to, void *to_arg, Process *);
+void erts_program_counter_info(fmtfn_t to, void *to_arg, Process *);
+void erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp);
+void erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg);
+void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg);
 
 Eterm erts_get_process_priority(Process *p);
 Eterm erts_set_process_priority(Process *p, Eterm prio);
@@ -1878,7 +1878,7 @@ void erts_handle_pending_exit(Process *, ErtsProcLocks);
 #define ERTS_PROC_PENDING_EXIT(P) 0
 #endif
 
-void erts_deep_process_dump(int, void *);
+void erts_deep_process_dump(fmtfn_t, void *);
 
 Eterm erts_get_reader_groups_map(Process *c_p);
 Eterm erts_debug_reader_groups_map(Process *c_p, int groups);
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index d8c2eaba94..42654604cb 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -156,7 +156,7 @@ erts_pd_set_initial_size(int size)
  * Called from break handler
  */
 void
-erts_dictionary_dump(int to, void *to_arg, ProcDict *pd)
+erts_dictionary_dump(fmtfn_t to, void *to_arg, ProcDict *pd)
 {
     unsigned int i;
 #ifdef DEBUG
@@ -196,8 +196,8 @@ erts_dictionary_dump(int to, void *to_arg, ProcDict *pd)
 }
 
 void
-erts_deep_dictionary_dump(int to, void *to_arg,
-			  ProcDict* pd, void (*cb)(int, void *, Eterm))
+erts_deep_dictionary_dump(fmtfn_t to, void *to_arg,
+			  ProcDict* pd, void (*cb)(fmtfn_t, void *, Eterm))
 {
     unsigned int i;
     Eterm t;
diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h
index 387562058c..b50a2af72c 100644
--- a/erts/emulator/beam/erl_process_dict.h
+++ b/erts/emulator/beam/erl_process_dict.h
@@ -37,9 +37,9 @@ typedef struct proc_dict {
 int erts_pd_set_initial_size(int size);
 Uint erts_dicts_mem_size(struct process *p);
 void erts_erase_dicts(struct process *p);
-void erts_dictionary_dump(int to, void *to_arg, ProcDict *pd);
-void erts_deep_dictionary_dump(int to, void *to_arg,
-			       ProcDict* pd, void (*cb)(int, void *, Eterm obj));
+void erts_dictionary_dump(fmtfn_t to, void *to_arg, ProcDict *pd);
+void erts_deep_dictionary_dump(fmtfn_t to, void *to_arg,
+			       ProcDict* pd, void (*cb)(fmtfn_t, void *, Eterm obj));
 Eterm erts_dictionary_copy(struct process *p, ProcDict *pd);
 
 Eterm erts_pd_hash_get(struct process *p, Eterm id);
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index a70dfb8e73..d8bb00e8c6 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -40,17 +40,17 @@
 
 #define OUR_NIL	_make_header(0,_TAG_HEADER_FLOAT)
 
-static void dump_process_info(int to, void *to_arg, Process *p);
-static void dump_element(int to, void *to_arg, Eterm x);
-static void dump_dist_ext(int to, void *to_arg, ErtsDistExternal *edep);
-static void dump_element_nl(int to, void *to_arg, Eterm x);
-static int stack_element_dump(int to, void *to_arg, Eterm* sp,
+static void dump_process_info(fmtfn_t to, void *to_arg, Process *p);
+static void dump_element(fmtfn_t to, void *to_arg, Eterm x);
+static void dump_dist_ext(fmtfn_t to, void *to_arg, ErtsDistExternal *edep);
+static void dump_element_nl(fmtfn_t to, void *to_arg, Eterm x);
+static int stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp,
 			      int yreg);
-static void stack_trace_dump(int to, void *to_arg, Eterm* sp);
-static void print_function_from_pc(int to, void *to_arg, BeamInstr* x);
-static void heap_dump(int to, void *to_arg, Eterm x);
-static void dump_binaries(int to, void *to_arg, Binary* root);
-static void dump_externally(int to, void *to_arg, Eterm term);
+static void stack_trace_dump(fmtfn_t to, void *to_arg, Eterm* sp);
+static void print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x);
+static void heap_dump(fmtfn_t to, void *to_arg, Eterm x);
+static void dump_binaries(fmtfn_t to, void *to_arg, Binary* root);
+static void dump_externally(fmtfn_t to, void *to_arg, Eterm term);
 
 static Binary* all_binaries;
 
@@ -60,7 +60,7 @@ extern BeamInstr beam_continue_exit[];
 
 
 void
-erts_deep_process_dump(int to, void *to_arg)
+erts_deep_process_dump(fmtfn_t to, void *to_arg)
 {
     int i, max = erts_ptab_max(&erts_proc);
 
@@ -117,7 +117,7 @@ Uint erts_process_memory(Process *p, int incl_msg_inq) {
 }
 
 static void
-dump_process_info(int to, void *to_arg, Process *p)
+dump_process_info(fmtfn_t to, void *to_arg, Process *p)
 {
     Eterm* sp;
     ErtsMessage* mp;
@@ -176,7 +176,7 @@ dump_process_info(int to, void *to_arg, Process *p)
 }
 
 static void
-dump_dist_ext(int to, void *to_arg, ErtsDistExternal *edep)
+dump_dist_ext(fmtfn_t to, void *to_arg, ErtsDistExternal *edep)
 {
     if (!edep)
 	erts_print(to, to_arg, "D0:E0:");
@@ -210,7 +210,7 @@ dump_dist_ext(int to, void *to_arg, ErtsDistExternal *edep)
 }
 
 static void
-dump_element(int to, void *to_arg, Eterm x)
+dump_element(fmtfn_t to, void *to_arg, Eterm x)
 {
     if (is_list(x)) {
 	erts_print(to, to_arg, "H" PTR_FMT, list_val(x));
@@ -240,14 +240,14 @@ dump_element(int to, void *to_arg, Eterm x)
 }
 
 static void
-dump_element_nl(int to, void *to_arg, Eterm x)
+dump_element_nl(fmtfn_t to, void *to_arg, Eterm x)
 {
     dump_element(to, to_arg, x);
     erts_putc(to, to_arg, '\n');
 }
 
 static void
-stack_trace_dump(int to, void *to_arg, Eterm *sp) {
+stack_trace_dump(fmtfn_t to, void *to_arg, Eterm *sp) {
     Eterm x = *sp;
     if (is_CP(x)) {
         erts_print(to, to_arg, "%p:", sp);
@@ -258,7 +258,7 @@ stack_trace_dump(int to, void *to_arg, Eterm *sp) {
 }
 
 void
-erts_limited_stack_trace(int to, void *to_arg, Process *p)
+erts_limited_stack_trace(fmtfn_t to, void *to_arg, Process *p)
 {
     Eterm* sp;
 
@@ -304,7 +304,7 @@ erts_limited_stack_trace(int to, void *to_arg, Process *p)
 }
 
 static int
-stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg)
+stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg)
 {
     Eterm x = *sp;
 
@@ -332,7 +332,7 @@ stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg)
 }
 
 static void
-print_function_from_pc(int to, void *to_arg, BeamInstr* x)
+print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x)
 {
     BeamInstr* addr = find_function_from_pc(x);
     if (addr == NULL) {
@@ -352,7 +352,7 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x)
 }
 
 static void
-heap_dump(int to, void *to_arg, Eterm x)
+heap_dump(fmtfn_t to, void *to_arg, Eterm x)
 {
     DeclareTmpHeapNoproc(last,1);
     Eterm* next = last;
@@ -512,7 +512,7 @@ heap_dump(int to, void *to_arg, Eterm x)
 }
 
 static void
-dump_binaries(int to, void *to_arg, Binary* current)
+dump_binaries(fmtfn_t to, void *to_arg, Binary* current)
 {
     while (current) {
 	long i;
@@ -530,7 +530,7 @@ dump_binaries(int to, void *to_arg, Binary* current)
 }
 
 static void
-dump_externally(int to, void *to_arg, Eterm term)
+dump_externally(fmtfn_t to, void *to_arg, Eterm term)
 {
     byte sbuf[1024]; /* encode and hope for the best ... */
     byte* s; 
@@ -573,7 +573,7 @@ dump_externally(int to, void *to_arg, Eterm term)
     }
 }
 
-void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg)
+void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg)
 {
     char *s;
     switch (erts_process_state2status(psflg)) {
@@ -591,7 +591,7 @@ void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg)
 }
 
 void
-erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg) {
+erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg) {
 
     int i;
 
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 2a19211987..7f043e191b 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -83,7 +83,7 @@ static struct export_blob* entry_to_blob(struct export_entry* ee)
 }
 
 void
-export_info(int to, void *to_arg)
+export_info(fmtfn_t to, void *to_arg)
 {
 #ifdef ERTS_SMP
     int lock = !ERTS_IS_CRASH_DUMPING;
diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h
index 1e7bb8514b..17fc4828ca 100644
--- a/erts/emulator/beam/export.h
+++ b/erts/emulator/beam/export.h
@@ -53,7 +53,7 @@ typedef struct export
 
 
 void init_export_table(void);
-void export_info(int, void *);
+void export_info(fmtfn_t, void *);
 
 ERTS_GLB_INLINE Export* erts_active_export_entry(Eterm m, Eterm f, unsigned a);
 Export* erts_export_put(Eterm mod, Eterm func, unsigned int arity);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index b2c76aa605..244e5edeb6 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -68,7 +68,7 @@ extern void erts_pre_dirty_nif(ErtsSchedulerData *,
 extern void erts_post_dirty_nif(struct enif_environment_t* env);
 #endif
 extern Eterm erts_nif_taints(Process* p);
-extern void erts_print_nif_taints(int to, void* to_arg);
+extern void erts_print_nif_taints(fmtfn_t to, void* to_arg);
 void erts_unload_nif(struct erl_module_nif* nif);
 extern void erl_nif_init(void);
 extern int erts_nif_get_funcs(struct erl_module_nif*,
@@ -1069,10 +1069,10 @@ void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info);
 void init_break_handler(void);
 void erts_set_ignore_break(void);
 void erts_replace_intr(void);
-void process_info(int, void *);
-void print_process_info(int, void *, Process*);
-void info(int, void *);
-void loaded(int, void *);
+void process_info(fmtfn_t, void *);
+void print_process_info(fmtfn_t, void *, Process*);
+void info(fmtfn_t, void *);
+void loaded(fmtfn_t, void *);
 
 /* erl_arith.c */
 double erts_get_positive_zero_float(void);
@@ -1161,7 +1161,7 @@ extern void erts_delete_nodes_monitors(Process *, ErtsProcLocks);
 extern Eterm erts_monitor_nodes(Process *, Eterm, Eterm);
 extern Eterm erts_processes_monitoring_nodes(Process *);
 extern int erts_do_net_exits(DistEntry*, Eterm);
-extern int distribution_info(int, void *);
+extern int distribution_info(fmtfn_t, void *);
 extern int is_node_name_atom(Eterm a);
 
 extern int erts_net_message(Port *, DistEntry *,
@@ -1350,7 +1350,7 @@ int erts_utf8_to_latin1(byte* dest, const byte* source, int slen);
 #define ERTS_UTF8_ANALYZE_MORE 3
 #define ERTS_UTF8_OK_MAX_CHARS 4
 
-void bin_write(int, void*, byte*, size_t);
+void bin_write(fmtfn_t, void*, byte*, size_t);
 Sint intlist_to_buf(Eterm, char*, Sint); /* most callers pass plain char*'s */
 
 struct Sint_buf {
@@ -1466,7 +1466,7 @@ Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live);
 
 Uint erts_current_reductions(Process* current, Process *p);
 
-int erts_print_system_version(int to, void *arg, Process *c_p);
+int erts_print_system_version(fmtfn_t to, void *arg, Process *c_p);
 
 int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg);
 
diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c
index cd038d100b..8548e30e8b 100644
--- a/erts/emulator/beam/hash.c
+++ b/erts/emulator/beam/hash.c
@@ -95,7 +95,7 @@ void hash_get_info(HashInfo *hi, Hash *h)
 **
 */
 
-void hash_info(int to, void *arg, Hash* h)
+void hash_info(fmtfn_t to, void *arg, Hash* h)
 {
     HashInfo hi;
 
diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h
index 4e769c0119..d319aaca83 100644
--- a/erts/emulator/beam/hash.h
+++ b/erts/emulator/beam/hash.h
@@ -37,7 +37,7 @@ typedef void (*HFREE_FUN)(void*);
 /* Meta functions */
 typedef void* (*HMALLOC_FUN)(int,size_t);
 typedef void (*HMFREE_FUN)(int,void*);
-typedef int (*HMPRINT_FUN)(int,void*,char*, ...);
+typedef int (*HMPRINT_FUN)(fmtfn_t,void*,char*, ...);
 
 /*
 ** This bucket must be placed in top of 
@@ -89,7 +89,7 @@ Hash* hash_init(int, Hash*, char*, int, HashFunctions);
 
 void  hash_delete(Hash*);
 void  hash_get_info(HashInfo*, Hash*);
-void  hash_info(int, void *, Hash*);
+void  hash_info(fmtfn_t, void *, Hash*);
 int   hash_table_sz(Hash *);
 
 void* hash_get(Hash*, void*);
diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c
index 26d6c04ea0..c86a2122f6 100644
--- a/erts/emulator/beam/index.c
+++ b/erts/emulator/beam/index.c
@@ -27,7 +27,7 @@
 #include "global.h"
 #include "index.h"
 
-void index_info(int to, void *arg, IndexTable *t)
+void index_info(fmtfn_t to, void *arg, IndexTable *t)
 {
     hash_info(to, arg, &t->htable);
     erts_print(to, arg, "=index_table:%s\n", t->htable.name);
diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h
index 0a109d8699..b2e3c0eab5 100644
--- a/erts/emulator/beam/index.h
+++ b/erts/emulator/beam/index.h
@@ -51,7 +51,7 @@ typedef struct index_table
 #define INDEX_PAGE_MASK ((1 << INDEX_PAGE_SHIFT)-1)
 
 IndexTable *erts_index_init(ErtsAlcType_t,IndexTable*,char*,int,int,HashFunctions);
-void index_info(int, void *, IndexTable*);
+void index_info(fmtfn_t, void *, IndexTable*);
 int index_table_sz(IndexTable *);
 
 int index_get(IndexTable*, void*);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 77dbe92241..4f131c74de 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -5480,7 +5480,7 @@ erts_request_io_bytes(Process *c_p)
 
 
 typedef struct {
-    int to;
+    fmtfn_t to;
     void *arg;
 } prt_one_lnk_data;
 
@@ -5497,7 +5497,7 @@ static void prt_one_lnk(ErtsLink *lnk, void *vprtd)
 }
 
 void
-print_port_info(Port *p, int to, void *arg)
+print_port_info(Port *p, fmtfn_t to, void *arg)
 {
     erts_aint32_t state = erts_atomic32_read_nob(&p->state);
 
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index 4f36377450..5978478323 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -50,7 +50,7 @@ static erts_smp_atomic_t tot_module_bytes;
 
 #include "erl_smp.h"
 
-void module_info(int to, void *to_arg)
+void module_info(fmtfn_t to, void *to_arg)
 {
     index_info(to, to_arg, &module_tables[erts_active_code_ix()]);
 }
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index 1c1afc8461..f105b3f401 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -48,7 +48,7 @@ Module* erts_put_module(Eterm mod);
 void init_module_table(void);
 void module_start_staging(void);
 void module_end_staging(int commit);
-void module_info(int, void *);
+void module_info(fmtfn_t, void *);
 
 Module *module_code(int, ErtsCodeIndex);
 int module_code_size(ErtsCodeIndex);
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index ac7096745e..acda51c9fc 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -102,7 +102,7 @@ is_proc_alive(Process *p)
     return !ERTS_PROC_IS_EXITING(p);
 }
 
-void register_info(int to, void *to_arg)
+void register_info(fmtfn_t to, void *to_arg)
 {
     int lock = !ERTS_IS_CRASH_DUMPING;
     if (lock)
diff --git a/erts/emulator/beam/register.h b/erts/emulator/beam/register.h
index d839f55d6b..27a314ca78 100644
--- a/erts/emulator/beam/register.h
+++ b/erts/emulator/beam/register.h
@@ -44,7 +44,7 @@ typedef struct reg_proc
 int process_reg_size(void);
 int process_reg_sz(void);
 void init_register_table(void);
-void register_info(int, void *);
+void register_info(fmtfn_t, void *);
 int erts_register_name(Process *, Eterm, Eterm);
 Eterm erts_whereis_name_to_id(Process *, Eterm);
 void erts_whereis_name(Process *, ErtsProcLocks,
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index dfe82cab44..4557f1534d 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -610,22 +610,21 @@ Uint erts_sys_misc_mem_sz(void);
 #include "erl_printf.h"
 
 /* Io constants to erts_print and erts_putc */
-#define ERTS_PRINT_STDERR	(2)
-#define ERTS_PRINT_STDOUT	(1)
-#define ERTS_PRINT_FILE		(-1)
-#define ERTS_PRINT_SBUF		(-2)
-#define ERTS_PRINT_SNBUF	(-3)
-#define ERTS_PRINT_DSBUF	(-4)
-
-#define ERTS_PRINT_MIN		ERTS_PRINT_DSBUF
+#define ERTS_PRINT_STDERR	((fmtfn_t)0)
+#define ERTS_PRINT_STDOUT	((fmtfn_t)1)
+#define ERTS_PRINT_FILE		((fmtfn_t)2)
+#define ERTS_PRINT_SBUF		((fmtfn_t)3)
+#define ERTS_PRINT_SNBUF	((fmtfn_t)4)
+#define ERTS_PRINT_DSBUF	((fmtfn_t)5)
+#define ERTS_PRINT_FD           ((fmtfn_t)6)
 
 typedef struct {
     char *buf;
     size_t size;
 } erts_print_sn_buf;
 
-int erts_print(int to, void *arg, char *format, ...);	/* in utils.c */
-int erts_putc(int to, void *arg, char);			/* in utils.c */
+int erts_print(fmtfn_t to, void *arg, char *format, ...);	/* in utils.c */
+int erts_putc(fmtfn_t to, void *arg, char);			/* in utils.c */
 
 /* logger stuff is declared here instead of in global.h, so sys files
    won't have to include global.h */
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 6786657faf..87ea4f05a1 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -348,40 +348,41 @@ int erts_fit_in_bits_uint(Uint value)
 }
 
 int
-erts_print(int to, void *arg, char *format, ...)
+erts_print(fmtfn_t to, void *arg, char *format, ...)
 {
     int res;
     va_list arg_list;
     va_start(arg_list, format);
 
-    if (to < ERTS_PRINT_MIN)
-	res = -EINVAL;
-    else {
-	switch (to) {
-	case ERTS_PRINT_STDOUT:
+    {
+	switch ((UWord)to) {
+	case (UWord)ERTS_PRINT_STDOUT:
 	    res = erts_vprintf(format, arg_list);
 	    break;
-	case ERTS_PRINT_STDERR:
+	case (UWord)ERTS_PRINT_STDERR:
 	    res = erts_vfprintf(stderr, format, arg_list);
 	    break;
-	case ERTS_PRINT_FILE:
+	case (UWord)ERTS_PRINT_FILE:
 	    res = erts_vfprintf((FILE *) arg, format, arg_list);
 	    break;
-	case ERTS_PRINT_SBUF:
+	case (UWord)ERTS_PRINT_SBUF:
 	    res = erts_vsprintf((char *) arg, format, arg_list);
 	    break;
-	case ERTS_PRINT_SNBUF:
+	case (UWord)ERTS_PRINT_SNBUF:
 	    res = erts_vsnprintf(((erts_print_sn_buf *) arg)->buf,
 				 ((erts_print_sn_buf *) arg)->size,
 				 format,
 				 arg_list);
 	    break;
-	case ERTS_PRINT_DSBUF:
+	case (UWord)ERTS_PRINT_DSBUF:
 	    res = erts_vdsprintf((erts_dsprintf_buf_t *) arg, format, arg_list);
 	    break;
-	default:
-	    res = erts_vfdprintf((int) to, format, arg_list);
+        case (UWord)ERTS_PRINT_FD:
+	    res = erts_vfdprintf((int)(SWord) arg, format, arg_list);
 	    break;
+        default:
+            res = erts_vcbprintf(to, arg, format, arg_list);
+            break;
 	}
     }
 
@@ -390,7 +391,7 @@ erts_print(int to, void *arg, char *format, ...)
 }
 
 int
-erts_putc(int to, void *arg, char c)
+erts_putc(fmtfn_t to, void *arg, char c)
 {
     return erts_print(to, arg, "%c", c);
 }
@@ -3876,7 +3877,7 @@ store_external_or_ref_in_proc_(Process *proc, Eterm ns)
     return store_external_or_ref_(&hp, &MSO(proc), ns);
 }
 
-void bin_write(int to, void *to_arg, byte* buf, size_t sz)
+void bin_write(fmtfn_t to, void *to_arg, byte* buf, size_t sz)
 {
     size_t i;
 
diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index 7bbb406f29..a5714f8325 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -2390,7 +2390,7 @@ add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2)
 }
 
 Eterm erts_mmap_info(ErtsMemMapper* mm,
-                     int *print_to_p,
+                     fmtfn_t *print_to_p,
                      void *print_to_arg,
                      Eterm** hpp, Uint* szp,
                      struct erts_mmap_info_struct* emis)
@@ -2431,7 +2431,7 @@ Eterm erts_mmap_info(ErtsMemMapper* mm,
 
 
     if (print_to_p) {
-        int to = *print_to_p;
+        fmtfn_t to = *print_to_p;
 	void *arg = print_to_arg;
         if (mm->supercarrier) {
             const char* prefix = "supercarrier ";
@@ -2485,7 +2485,7 @@ Eterm erts_mmap_info(ErtsMemMapper* mm,
 
 Eterm erts_mmap_info_options(ErtsMemMapper* mm,
                              char *prefix,
-                             int *print_to_p,
+                             fmtfn_t *print_to_p,
                              void *print_to_arg,
                              Uint **hpp,
                              Uint *szp)
@@ -2496,7 +2496,7 @@ Eterm erts_mmap_info_options(ErtsMemMapper* mm,
     Eterm res = THE_NON_VALUE;
 
     if (print_to_p) {
-        int to = *print_to_p;
+        fmtfn_t to = *print_to_p;
 	void *arg = print_to_arg;
         erts_print(to, arg, "%sscs: %bpu\n", prefix, scs);
         if (mm->supercarrier) {
diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h
index fa51b663fa..623b5188ac 100644
--- a/erts/emulator/sys/common/erl_mmap.h
+++ b/erts/emulator/sys/common/erl_mmap.h
@@ -22,6 +22,7 @@
 #define ERL_MMAP_H__
 
 #include "sys.h"
+#include "erl_printf.h"
 
 #define ERTS_MMAP_SUPERALIGNED_BITS (18)
 /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */
@@ -146,10 +147,10 @@ struct erts_mmap_info_struct
     UWord segs[6];
     UWord os_used;
 };
-Eterm erts_mmap_info(ErtsMemMapper*, int *print_to_p, void *print_to_arg,
+Eterm erts_mmap_info(ErtsMemMapper*, fmtfn_t *print_to_p, void *print_to_arg,
                      Eterm** hpp, Uint* szp, struct erts_mmap_info_struct*);
 Eterm erts_mmap_info_options(ErtsMemMapper*,
-                             char *prefix, int *print_to_p, void *print_to_arg,
+                             char *prefix, fmtfn_t *print_to_p, void *print_to_arg,
                              Uint **hpp, Uint *szp);
 
 
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index f3306a888c..882c93a83c 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -991,7 +991,7 @@ add_4tup(Uint **hpp, Uint *szp, Eterm *lp,
 static Eterm
 info_options(ErtsMsegAllctr_t *ma,
 	     char *prefix,
-	     int *print_to_p,
+	     fmtfn_t *print_to_p,
 	     void *print_to_arg,
 	     Uint **hpp,
 	     Uint *szp)
@@ -999,7 +999,7 @@ info_options(ErtsMsegAllctr_t *ma,
     Eterm res = NIL;
 
     if (print_to_p) {
-	int to = *print_to_p;
+	fmtfn_t to = *print_to_p;
 	void *arg = print_to_arg;
 	erts_print(to, arg, "%samcbf: %beu\n", prefix, ma->abs_max_cache_bad_fit);
 	erts_print(to, arg, "%srmcbf: %beu\n", prefix, ma->rel_max_cache_bad_fit);
@@ -1027,7 +1027,7 @@ info_options(ErtsMsegAllctr_t *ma,
 }
 
 static Eterm
-info_calls(ErtsMsegAllctr_t *ma, int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
+info_calls(ErtsMsegAllctr_t *ma, fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
 {
     Eterm res = THE_NON_VALUE;
 
@@ -1040,7 +1040,7 @@ info_calls(ErtsMsegAllctr_t *ma, int *print_to_p, void *print_to_arg, Uint **hpp
 	erts_print(TO, TOA, "mseg_%s calls: %b32u%09b32u\n", #CC,		\
 		   ma->calls.CC.giga_no, ma->calls.CC.no)
 
-	int to = *print_to_p;
+	fmtfn_t to = *print_to_p;
 	void *arg = print_to_arg;
 
 	PRINT_CC(to, arg, alloc);
@@ -1106,7 +1106,7 @@ info_calls(ErtsMsegAllctr_t *ma, int *print_to_p, void *print_to_arg, Uint **hpp
 }
 
 static Eterm
-info_status(ErtsMsegAllctr_t *ma, int *print_to_p, void *print_to_arg,
+info_status(ErtsMsegAllctr_t *ma, fmtfn_t *print_to_p, void *print_to_arg,
 	    int begin_new_max_period, int only_sz, Uint **hpp, Uint *szp)
 {
     Eterm res = THE_NON_VALUE;
@@ -1117,7 +1117,7 @@ info_status(ErtsMsegAllctr_t *ma, int *print_to_p, void *print_to_arg,
 	ma->segments.max_ever.sz = ma->segments.max.sz;
 
     if (print_to_p) {
-	int to = *print_to_p;
+	fmtfn_t to = *print_to_p;
 	void *arg = print_to_arg;
 
         if (!only_sz) {
@@ -1165,7 +1165,7 @@ info_status(ErtsMsegAllctr_t *ma, int *print_to_p, void *print_to_arg,
     return res;
 }
 
-static Eterm info_memkind(ErtsMsegAllctr_t *ma, int *print_to_p, void *print_to_arg,
+static Eterm info_memkind(ErtsMsegAllctr_t *ma, fmtfn_t *print_to_p, void *print_to_arg,
 			  int begin_max_per, int only_sz, Uint **hpp, Uint *szp)
 {
     Eterm res = THE_NON_VALUE;
@@ -1196,7 +1196,7 @@ static Eterm info_memkind(ErtsMsegAllctr_t *ma, int *print_to_p, void *print_to_
 }
 
 static Eterm
-info_version(ErtsMsegAllctr_t *ma, int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
+info_version(ErtsMsegAllctr_t *ma, fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
 {
     Eterm res = THE_NON_VALUE;
 
@@ -1218,7 +1218,7 @@ info_version(ErtsMsegAllctr_t *ma, int *print_to_p, void *print_to_arg, Uint **h
 
 Eterm
 erts_mseg_info_options(int ix,
-		       int *print_to_p, void *print_to_arg,
+		       fmtfn_t *print_to_p, void *print_to_arg,
 		       Uint **hpp, Uint *szp)
 {
     ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_IX(ix);
@@ -1231,7 +1231,7 @@ erts_mseg_info_options(int ix,
 
 Eterm
 erts_mseg_info(int ix,
-	       int *print_to_p,
+	       fmtfn_t *print_to_p,
 	       void *print_to_arg,
 	       int begin_max_per,
                int only_sz,
diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h
index a43b409e94..bba0dec499 100644
--- a/erts/emulator/sys/common/erl_mseg.h
+++ b/erts/emulator/sys/common/erl_mseg.h
@@ -98,8 +98,8 @@ Uint  erts_mseg_unit_size(void);
 void  erts_mseg_init(ErtsMsegInit_t *init);
 void  erts_mseg_late_init(void); /* Have to be called after all allocators,
 				   threads and timers have been initialized. */
-Eterm erts_mseg_info_options(int, int *, void*, Uint **, Uint *);
-Eterm erts_mseg_info(int, int *, void*, int, int, Uint **, Uint *);
+Eterm erts_mseg_info_options(int, fmtfn_t*, void*, Uint **, Uint *);
+Eterm erts_mseg_info(int, fmtfn_t *, void*, int, int, Uint **, Uint *);
 
 #endif /* #if HAVE_ERTS_MSEG */
 
-- 
cgit v1.2.3


From dcc7f6b2901a632ff8398c1382c15eaabe562714 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 16 Nov 2016 16:42:30 +0100
Subject: erts: Add ErtsStrToSint64

for simplify string to 64-bit integer parsing.
---
 erts/emulator/beam/sys.h              | 2 ++
 erts/emulator/sys/win32/erl_win_sys.h | 2 ++
 2 files changed, 4 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 4557f1534d..7740dd4373 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -378,6 +378,7 @@ typedef long          Sint64;
 #    ifdef LONG_MIN
 #      define ERTS_SINT64_MIN LONG_MIN
 #    endif
+#    define ErtsStrToSint64 strtol
 #  elif SIZEOF_LONG_LONG == 8
 #    define HAVE_INT64 1
 typedef unsigned long long Uint64;
@@ -391,6 +392,7 @@ typedef long long          Sint64;
 #    ifdef LLONG_MIN
 #      define ERTS_SINT64_MIN LLONG_MIN
 #    endif
+#    define ErtsStrToSint64 strtoll
 #  else
 #    error "No 64-bit integer type found"
 #  endif
diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h
index 04fbf23109..78005aada9 100644
--- a/erts/emulator/sys/win32/erl_win_sys.h
+++ b/erts/emulator/sys/win32/erl_win_sys.h
@@ -182,6 +182,8 @@ typedef LONGLONG ErtsMonotonicTime;
 typedef LONGLONG ErtsSysHrTime;
 #endif
 
+#define ErtsStrToSint64 _strtoi64
+
 typedef ErtsMonotonicTime ErtsSystemTime;
 typedef ErtsMonotonicTime ErtsSysPerfCounter;
 
-- 
cgit v1.2.3


From 27e865d01ad49664897662ccc123d825505c0ff6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 9 Nov 2016 18:41:43 +0100
Subject: erts: Fix all -Wundef errors

---
 erts/emulator/beam/beam_emu.c         | 2 +-
 erts/emulator/beam/erl_alloc_util.c   | 2 +-
 erts/emulator/beam/erl_bif_ddll.c     | 2 +-
 erts/emulator/beam/erl_drv_nif.h      | 4 ----
 erts/emulator/beam/erl_gc.c           | 6 +++---
 erts/emulator/beam/erl_msacc.h        | 8 ++++----
 erts/emulator/beam/erl_port.h         | 2 +-
 erts/emulator/beam/erl_process.h      | 4 ++--
 erts/emulator/hipe/hipe_x86_signal.c  | 2 +-
 erts/emulator/sys/common/erl_mmap.c   | 6 +++---
 erts/emulator/sys/unix/erl_unix_sys.h | 4 ++--
 11 files changed, 19 insertions(+), 23 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 7eaa7a0d8b..b24995ca3d 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5053,7 +5053,7 @@ do {						\
     } else {
         TestHeap(ERTS_SINT64_HEAP_SIZE(ts),0);
         r(0) = make_big(HTOP);
-#if defined(ARCH_32) || HALFWORD_HEAP
+#if defined(ARCH_32)
         if (ts >= (((Uint64) 1) << 32)) {
             *HTOP = make_pos_bignum_header(2);
             BIG_DIGIT(HTOP, 0) = (Uint) (ts & ((Uint) 0xffffffff));
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 163244237d..9c6e27fc32 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -6083,7 +6083,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 	goto error;
     allctr->min_block_size		= UNIT_CEILING(allctr->min_block_size
 						       + sizeof(FreeBlkFtr_t));
-#if ERTS_SMP
+#ifdef ERTS_SMP
     if (init->tpref) {
 	Uint sz = ABLK_HDR_SZ;
 	sz += (init->fix ? 
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index ef77201544..587859e413 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1099,7 +1099,7 @@ void erts_ddll_increment_port_count(DE_Handle *dh)
 void erts_ddll_decrement_port_count(DE_Handle *dh)
 {
     assert_drv_list_locked();
-#if DEBUG
+#ifdef DEBUG
     ASSERT(erts_smp_atomic32_dec_read_nob(&dh->port_count) >= 0);
 #else
     erts_smp_atomic32_dec_nob(&dh->port_count);
diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h
index 6ec5fbb895..2489099b5c 100644
--- a/erts/emulator/beam/erl_drv_nif.h
+++ b/erts/emulator/beam/erl_drv_nif.h
@@ -69,10 +69,6 @@ typedef enum {
 #  define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG
 #  undef SIZEOF_LONG_LONG
 #endif
-#ifdef HALFWORD_HEAP_EMULATOR
-#  define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR
-#  undef HALFWORD_HEAP_EMULATOR
-#endif
 #include "erl_int_sizes_config.h"
 #if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR
 #  error SIZEOF_CHAR mismatch
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index db262d35c5..6c752b9741 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -35,7 +35,7 @@
 #include "error.h"
 #include "big.h"
 #include "erl_gc.h"
-#if HIPE
+#ifdef HIPE
 #include "hipe_stack.h"
 #include "hipe_mode_switch.h"
 #endif
@@ -395,11 +395,11 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end,
 
     if (is_non_value(result)) {
 	if (p->freason == TRAP) {
-	  #if HIPE
+#ifdef HIPE
 	    if (regs == NULL) {
 		regs = erts_proc_sched_data(p)->x_reg_array;
 	    }
-	  #endif
+#endif
 	    cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls);
 	} else {
 	    cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls);
diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h
index ad7c8c5eee..7e02d8e101 100644
--- a/erts/emulator/beam/erl_msacc.h
+++ b/erts/emulator/beam/erl_msacc.h
@@ -22,7 +22,7 @@
 #define ERL_MSACC_H__
 
 /* Can be enabled/disabled via configure */
-#if ERTS_ENABLE_MSACC == 2
+#if defined(ERTS_ENABLE_MSACC) && ERTS_ENABLE_MSACC == 2
 #define ERTS_MSACC_EXTENDED_STATES 1
 #endif
 
@@ -66,7 +66,7 @@
 
 #define ERTS_MSACC_STATE_COUNT 7
 
-#if ERTS_MSACC_STATE_STRINGS && ERTS_ENABLE_MSACC
+#if defined(ERTS_MSACC_STATE_STRINGS) && defined(ERTS_ENABLE_MSACC)
 static char *erts_msacc_states[] = {
     "aux",
     "check_io",
@@ -104,7 +104,7 @@ static char *erts_msacc_states[] = {
 #define ERTS_MSACC_STATE_COUNT ERTS_MSACC_STATIC_STATE_COUNT
 #endif
 
-#if ERTS_MSACC_STATE_STRINGS
+#ifdef ERTS_MSACC_STATE_STRINGS
 static char *erts_msacc_states[] = {
     "alloc",
     "aux",
@@ -157,7 +157,7 @@ struct erl_msacc_t_ {
 
 };
 
-#if ERTS_ENABLE_MSACC
+#ifdef ERTS_ENABLE_MSACC
 
 #ifdef USE_THREADS
 extern erts_tsd_key_t erts_msacc_key;
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index f90844ccc8..0844dc0994 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -733,7 +733,7 @@ erts_thr_drvport2port(ErlDrvPort drvport, int lock_pdl)
     if (lock_pdl && prt->port_data_lock)
 	driver_pdl_lock(prt->port_data_lock);
 
-#if ERTS_ENABLE_LOCK_CHECK
+#ifdef ERTS_ENABLE_LOCK_CHECK
     if (!ERTS_IS_CRASH_DUMPING) {
 	if (erts_lc_is_emu_thr()) {
 	    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index b266d32e76..72344be504 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -21,6 +21,8 @@
 #ifndef __PROCESS_H__
 #define __PROCESS_H__
 
+#include "sys.h"
+
 #undef ERTS_INCLUDE_SCHEDULER_INTERNALS
 #if (defined(ERL_PROCESS_C__) \
      || defined(ERL_PORT_TASK_C__) \
@@ -37,8 +39,6 @@
 
 typedef struct process Process;
 
-#include "sys.h"
-
 #define ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
 #include "erl_process_lock.h" /* Only pull out important types... */
 #undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
diff --git a/erts/emulator/hipe/hipe_x86_signal.c b/erts/emulator/hipe/hipe_x86_signal.c
index 50d08b96d3..1a34ce786c 100644
--- a/erts/emulator/hipe/hipe_x86_signal.c
+++ b/erts/emulator/hipe/hipe_x86_signal.c
@@ -51,7 +51,7 @@
 #endif
 #include "hipe_signal.h"
 
-#if __GLIBC__ == 2 && (__GLIBC_MINOR__ >= 3)
+#if defined(__GLIBC__) && __GLIBC__ == 2 && (__GLIBC_MINOR__ >= 3)
 /*
  * __libc_sigaction() is the core routine.
  * Without libpthread, sigaction() and __sigaction() are both aliases
diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index 3ab03971a5..169740f8cf 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -1878,7 +1878,7 @@ erts_mremap(ErtsMemMapper* mm,
 	    return NULL;
 	}
 
-#if ERTS_HAVE_OS_MREMAP || ERTS_HAVE_GENUINE_OS_MMAP
+#if defined(ERTS_HAVE_OS_MREMAP) || defined(ERTS_HAVE_GENUINE_OS_MMAP)
 	superaligned = (ERTS_MMAPFLG_SUPERALIGNED & flags);
 
 	if (superaligned) {
@@ -1898,7 +1898,7 @@ erts_mremap(ErtsMemMapper* mm,
 	    }
 	}
 
-#if ERTS_HAVE_GENUINE_OS_MMAP
+#ifdef ERTS_HAVE_GENUINE_OS_MMAP
 	if (asize < old_size
 	    && (!superaligned
 		|| ERTS_IS_SUPERALIGNED(ptr))) {
@@ -1913,7 +1913,7 @@ erts_mremap(ErtsMemMapper* mm,
 	    return ptr;
 	}
 #endif
-#if ERTS_HAVE_OS_MREMAP
+#ifdef ERTS_HAVE_OS_MREMAP
 	if (superaligned)
 	    return remap_move(mm, flags, new_ptr, old_size, sizep);
 	else {
diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index b64b0d87f6..bd554238b7 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -46,10 +46,10 @@
 #include <signal.h>
 #include <setjmp.h>
 
-#if HAVE_SYS_SOCKETIO_H
+#ifdef HAVE_SYS_SOCKETIO_H
 #   include <sys/socketio.h>
 #endif
-#if HAVE_SYS_SOCKIO_H
+#ifdef HAVE_SYS_SOCKIO_H
 #   include <sys/sockio.h>
 #endif
 
-- 
cgit v1.2.3


From 011242f927259aa477ebe488a3ee3c4e22081488 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 10 Nov 2016 20:09:48 +0100
Subject: erts: Tweak PSTACK to avoid warning

warning: array subscript is below array bounds

and ok, it's technically undef behavior to set pointer
before first array element.
---
 erts/emulator/beam/global.h | 36 +++++++++++++++++++-----------------
 erts/emulator/beam/utils.c  |  6 ++----
 2 files changed, 21 insertions(+), 21 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 537aaf3177..507ef76e59 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -773,8 +773,8 @@ do {						\
 
 typedef struct ErtsPStack_ {
     byte* pstart;
-    byte* psp;
-    byte* pend;
+    int offs;   /* "stack pointer" as byte offset from pstart */
+    int size;   /* allocated size in bytes */
     ErtsAlcType_t alloc_type;
 }ErtsPStack;
 
@@ -785,8 +785,8 @@ void erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes);
 #define PSTACK_DECLARE(s, DEF_PSTACK_SIZE) \
 PSTACK_TYPE PSTK_DEF_STACK(s)[DEF_PSTACK_SIZE];                            \
 ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */                    \
-                 (byte*)(PSTK_DEF_STACK(s) - 1), /* psp */                 \
-                 (byte*)(PSTK_DEF_STACK(s) + (DEF_PSTACK_SIZE)), /* pend */\
+                 -(int)sizeof(PSTACK_TYPE), /* offs */                     \
+                 DEF_PSTACK_SIZE*sizeof(PSTACK_TYPE), /* size */           \
                  ERTS_ALC_T_ESTACK   /* alloc_type */                      \
 }
 
@@ -806,19 +806,21 @@ do {							\
     }							\
 } while(0)
 
-#define PSTACK_IS_EMPTY(s) (s.psp < s.pstart)
+#define PSTACK_IS_EMPTY(s) (s.offs < 0)
 
-#define PSTACK_COUNT(s) (((PSTACK_TYPE*)s.psp + 1) - (PSTACK_TYPE*)s.pstart)
+#define PSTACK_COUNT(s) ((s.offs + sizeof(PSTACK_TYPE)) / sizeof(PSTACK_TYPE))
 
-#define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), (PSTACK_TYPE*)(s.psp))
+#define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), \
+                       (PSTACK_TYPE*)(s.pstart + s.offs))
 
-#define PSTACK_PUSH(s) 		                                           \
-    (s.psp += sizeof(PSTACK_TYPE),                                         \
-     ((s.psp == s.pend) ? erl_grow_pstack(&s, PSTK_DEF_STACK(s),           \
-                                          sizeof(PSTACK_TYPE)) : (void)0), \
-     ((PSTACK_TYPE*) s.psp))
+#define PSTACK_PUSH(s) 		                                            \
+    (s.offs += sizeof(PSTACK_TYPE),                                         \
+     ((s.offs == s.size) ? erl_grow_pstack(&s, PSTK_DEF_STACK(s),           \
+                                          sizeof(PSTACK_TYPE)) : (void)0),  \
+     ((PSTACK_TYPE*) (s.pstart + s.offs)))
 
-#define PSTACK_POP(s) ((PSTACK_TYPE*) (s.psp -= sizeof(PSTACK_TYPE)))
+#define PSTACK_POP(s) ((s.offs -= sizeof(PSTACK_TYPE)), \
+                       (PSTACK_TYPE*)(s.pstart + s.offs))
 
 /*
  * Do not free the stack after this, it may have pointers into what
@@ -831,8 +833,8 @@ do {\
 	(dst)->pstart = erts_alloc(s.alloc_type,\
 				   sizeof(PSTK_DEF_STACK(s)));\
 	sys_memcpy((dst)->pstart, s.pstart, _pbytes);\
-	(dst)->psp = (dst)->pstart + _pbytes - sizeof(PSTACK_TYPE);\
-	(dst)->pend = (dst)->pstart + sizeof(PSTK_DEF_STACK(s));\
+	(dst)->offs = s.offs;\
+	(dst)->size = s.size;\
 	(dst)->alloc_type = s.alloc_type;\
     } else\
         *(dst) = s;\
@@ -847,8 +849,8 @@ do {						        \
     ASSERT(s.pstart == (byte*)PSTK_DEF_STACK(s));	\
     s = *(src);  /* struct copy */		        \
     (src)->pstart = NULL;			        \
-    ASSERT(s.psp >= (s.pstart - sizeof(PSTACK_TYPE)));  \
-    ASSERT(s.psp < s.pend);			        \
+    ASSERT(s.offs >= -(int)sizeof(PSTACK_TYPE));        \
+    ASSERT(s.offs < s.size);			        \
 } while (0)
 
 #define PSTACK_DESTROY_SAVED(pstack)\
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index cd7131f5df..80c116fdac 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -204,9 +204,8 @@ erl_grow_wstack(ErtsWStack* s, Uint need)
 void
 erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes)
 {
-    Uint old_size = s->pend - s->pstart;
+    Uint old_size = s->size;
     Uint new_size;
-    Uint sp_offs = s->psp - s->pstart;
 
     if (need_bytes < old_size)
 	new_size = 2 * old_size;
@@ -220,8 +219,7 @@ erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes)
 	sys_memcpy(new_ptr, s->pstart, old_size);
 	s->pstart = new_ptr;
     }
-    s->pend = s->pstart + new_size;
-    s->psp = s->pstart + sp_offs;
+    s->size = new_size;
 }
 
 /*
-- 
cgit v1.2.3


From 160a28a0912cf87e971ddcbfdb6f76d64581fe44 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 21 Nov 2016 19:26:02 +0100
Subject: erts: Mend broken checks in nif_SUITE

Macro CHECK did not fail the test case
just some nice logging that no one saw.

One ignored failure fixed; a delayed unload
after purge due to live resource with dtor.
---
 erts/emulator/test/nif_SUITE.erl | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 3a76ee6aee..37d6e68868 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -21,7 +21,7 @@
 -module(nif_SUITE).
 
 %%-define(line_trace,true).
--define(CHECK(Exp,Got), check(Exp,Got,?LINE)).
+-define(CHECK(Exp,Got), Exp = check(Exp,Got,?LINE)).
 %%-define(CHECK(Exp,Got), Exp = Got).
 
 -include_lib("common_test/include/ct.hrl").
@@ -932,7 +932,7 @@ resource_takeover(Config) when is_list(Config) ->
                               ]),
     ?CHECK([{upgrade,2,1,201}], nif_mod_call_history()),
     true = erlang:purge_module(nif_mod),
-    ?CHECK([{unload,1,1,106}], nif_mod_call_history()),
+    ?CHECK([], nif_mod_call_history()),  % BGX2 keeping lib loaded
 
     BinA2 = read_resource(0,A2),
     ok = forget_resource(A2),
@@ -945,8 +945,8 @@ resource_takeover(Config) when is_list(Config) ->
     ?CHECK([], nif_mod_call_history()),    % no dtor
 
     ok = forget_resource(BGX2),  % calling dtor in orphan library v1 still loaded
-    ?CHECK([{{resource_dtor_B_v1,BinBGX2},1,6,106}], nif_mod_call_history()),
-    % How to test that lib v1 is closed here?
+    ?CHECK([{{resource_dtor_B_v1,BinBGX2},1,6,106}, {unload,1,7,107}],
+           nif_mod_call_history()),
 
     ok = forget_resource(NGX2),
     ?CHECK([], nif_mod_call_history()),  % no dtor
@@ -1900,7 +1900,8 @@ check(Exp,Got,Line) ->
     case Got of
  	Exp -> Exp;	    
   	_ ->
-  	    io:format("CHECK at ~p: Expected ~p but got ~p\n",[Line,Exp,Got]),
+	    io:format("CHECK at line ~p\nExpected: ~p\nGot     : ~p\n",
+                      [Line,Exp,Got]),
  	    Got
     end.
 
-- 
cgit v1.2.3


From 491cd4c0c0a534ab89c3ebb4c413301c91c7167d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 22 Nov 2016 14:37:31 +0100
Subject: erts: Add env variable ERL_CRASH_DUMP_BYTES

to limit crash dump size
---
 erts/emulator/beam/break.c       | 35 +++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_init.c    | 13 ++++++++++++-
 erts/emulator/beam/global.h      |  1 +
 erts/emulator/test/bif_SUITE.erl | 29 +++++++++++++++++++++++++++--
 4 files changed, 75 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 2e8a384dec..33433df68a 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -661,6 +661,26 @@ bin_check(void)
 
 #endif
 
+static Sint64 crash_dump_limit = ERTS_SINT64_MAX;
+static Sint64 crash_dump_written = 0;
+
+static int crash_dump_limited_writer(void* vfdp, char* buf, size_t len)
+{
+    const char stop_msg[] = "\n=abort: CRASH DUMP SIZE LIMIT REACHED\n";
+
+    crash_dump_written += len;
+    if (crash_dump_written <= crash_dump_limit) {
+        return erts_write_fd(vfdp, buf, len);
+    }
+
+    len -= (crash_dump_written - crash_dump_limit);
+    erts_write_fd(vfdp, buf, len);
+    erts_write_fd(vfdp, (char*)stop_msg, sizeof(stop_msg)-1);
+
+    /* We assume that crash dump was called from erts_exit_vv() */
+    erts_exit_epilogue();
+}
+
 /* XXX THIS SHOULD BE IN SYSTEM !!!! */
 void
 erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
@@ -760,6 +780,21 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
 	return;
     }
 
+    crash_dump_limit = ERTS_SINT64_MAX;
+    envsz = sizeof(env);
+    if (erts_sys_getenv__("ERL_CRASH_DUMP_BYTES", env, &envsz) == 0) {
+        Sint64 limit;
+        char* endptr;
+        errno = 0;
+        limit = ErtsStrToSint64(env, &endptr, 10);
+        if (errno == 0 && limit >= 0 && endptr != env && *endptr == 0) {
+            if (limit == 0)
+                return;
+            crash_dump_limit = limit;
+            to = &crash_dump_limited_writer;
+        }
+    }
+
     if (erts_sys_getenv__("ERL_CRASH_DUMP",&dumpnamebuf[0],&dumpnamebufsize) != 0)
 	dumpname = "erl_crash.dump";
     else
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 781bf024dd..362239f765 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -2374,6 +2374,8 @@ system_cleanup(int flush_async)
     erts_exit_flush_async();
 }
 
+static int erts_exit_code;
+
 static __decl_noreturn void __noreturn
 erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2)
 {
@@ -2385,12 +2387,21 @@ erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2)
     if (fmt != NULL && *fmt != '\0')
 	erl_error(fmt, args2);	/* Print error message. */
 
-    /* Produce an Erlang core dump if error */
+    erts_exit_code = n;
+
+    /* Produce an Erlang crash dump if error */
     if (((n == ERTS_ERROR_EXIT && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT)
 	&& erts_initialized) {
 	erl_crash_dump_v((char*) NULL, 0, fmt, args1);
     }
 
+    erts_exit_epilogue();
+}
+
+__decl_noreturn void __noreturn erts_exit_epilogue(void)
+{
+    int n = erts_exit_code;
+
     sys_tty_reset(n);
 
     if (n == ERTS_INTR_EXIT)
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 244e5edeb6..d6df85034c 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1079,6 +1079,7 @@ double erts_get_positive_zero_float(void);
 
 /* config.c */
 
+__decl_noreturn void __noreturn erts_exit_epilogue(void);
 __decl_noreturn void __noreturn erts_exit(int n, char*, ...);
 __decl_noreturn void __noreturn erts_flush_async_exit(int n, char*, ...);
 void erl_error(char*, va_list);
diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl
index d31399e4af..b8d89126fe 100644
--- a/erts/emulator/test/bif_SUITE.erl
+++ b/erts/emulator/test/bif_SUITE.erl
@@ -31,6 +31,7 @@
 	 t_list_to_existing_atom/1,os_env/1,otp_7526/1,
 	 binary_to_atom/1,binary_to_existing_atom/1,
 	 atom_to_binary/1,min_max/1, erlang_halt/1,
+         erl_crash_dump_bytes/1,
 	 is_builtin/1]).
 
 suite() ->
@@ -43,6 +44,7 @@ all() ->
      t_list_to_existing_atom, os_env, otp_7526,
      display,
      atom_to_binary, binary_to_atom, binary_to_existing_atom,
+     erl_crash_dump_bytes,
      min_max, erlang_halt, is_builtin].
 
 %% Uses erlang:display to test that erts_printf does not do deep recursion
@@ -664,7 +666,7 @@ erlang_halt(Config) when is_list(Config) ->
                      [available_internal_state, true]),
     {badrpc,nodedown} = rpc:call(N4, erts_debug, set_internal_state,
                                  [broken_halt, "Validate correct crash dump"]),
-    ok = wait_until_stable_size(CrashDump,-1),
+    {ok,_} = wait_until_stable_size(CrashDump,-1),
     {ok, Bin} = file:read_file(CrashDump),
     case {string:str(binary_to_list(Bin),"\n=end\n"),
           string:str(binary_to_list(Bin),"\r\n=end\r\n")} of
@@ -681,11 +683,34 @@ wait_until_stable_size(File,PrevSz) ->
             wait_until_stable_size(File,PrevSz-1);
         {ok,#file_info{size = PrevSz }} when PrevSz /= -1 ->
             io:format("Crashdump file size was: ~p (~s)~n",[PrevSz,File]),
-            ok;
+            {ok,PrevSz};
         {ok,#file_info{size = NewSz }} ->
             wait_until_stable_size(File,NewSz)
     end.
 
+% Test erlang:halt with ERL_CRASH_DUMP_BYTES
+erl_crash_dump_bytes(Config) when is_list(Config) ->
+    Bytes = 1000,
+    CrashDump = do_limited_crash_dump(Config, Bytes),
+    {ok,ActualBytes} = wait_until_stable_size(CrashDump,-1),
+    true = ActualBytes < (Bytes + 100),
+
+    NoDump = do_limited_crash_dump(Config,0),
+    {error,enoent} = wait_until_stable_size(NoDump,-8),
+    ok.
+
+do_limited_crash_dump(Config, Bytes) ->
+    H = hostname(),
+    {ok,N} = slave:start(H, halt_node),
+    BytesStr = integer_to_list(Bytes),
+    CrashDump = filename:join(proplists:get_value(priv_dir,Config),
+                              "erl_crash." ++ BytesStr ++ ".dump"),
+    true = rpc:call(N, os, putenv, ["ERL_CRASH_DUMP",CrashDump]),
+    true = rpc:call(N, os, putenv, ["ERL_CRASH_DUMP_BYTES",BytesStr]),
+    {badrpc,nodedown} = rpc:call(N, erlang, halt, ["Testing ERL_CRASH_DUMP_BYTES"]),
+    CrashDump.
+
+
 is_builtin(_Config) ->
     Exp0 = [{M,F,A} || {M,_} <- code:all_loaded(),
 		       {F,A} <- M:module_info(exports)],
-- 
cgit v1.2.3


From d82ed483ae6bae52745d2725017d77cb2c7810c7 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 22 Nov 2016 15:53:51 +0100
Subject: Fix saving of original arguments when rescheduling via NifExport

---
 erts/emulator/beam/erl_nif.c | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 3a547982da..55bd9f7455 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2360,6 +2360,7 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec
     Eterm* reg;
     NifExport* ep;
     int i, scheduler;
+    int orig_argc;
 
     execution_state(env, &proc, &scheduler);
 
@@ -2370,11 +2371,14 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec
 
     reg = erts_proc_sched_data(proc)->x_reg_array;
 
+    ASSERT(!need_save || proc->current);
+    orig_argc = need_save ? (int) proc->current[2] : 0;
+
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     if (!ep)
-	ep = allocate_nif_sched_data(proc, argc);
-    else if (need_save && ep->rootset_extra < argc) {
-	NifExport* new_ep = allocate_nif_sched_data(proc, argc);
+	ep = allocate_nif_sched_data(proc, orig_argc);
+    else if (need_save && ep->rootset_extra < orig_argc) {
+	NifExport* new_ep = allocate_nif_sched_data(proc, orig_argc);
 	destroy_nif_export(ep);
 	ep = new_ep;
     }
@@ -2387,16 +2391,14 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec
     }
     if (scheduler > 0)
 	ERTS_VBUMP_ALL_REDS(proc);
-    for (i = 0; i < argc; i++) {
-	if (need_save)
-	    ep->rootset[i+1] = reg[i];
-	reg[i] = (Eterm) argv[i];
-    }
     if (need_save) {
-	ASSERT(proc->current);
 	ep->saved_current = proc->current;
-	ep->saved_argc = argc;
+	ep->saved_argc = orig_argc;
+	for (i = 0; i < orig_argc; i++)
+	    ep->rootset[i+1] = reg[i];
     }
+    for (i = 0; i < argc; i++)
+	reg[i] = (Eterm) argv[i];
     proc->i = (BeamInstr*) ep->exp.addressv[0];
     ep->exp.code[0] = (BeamInstr) proc->current[0];
     ep->exp.code[1] = (BeamInstr) proc->current[1];
-- 
cgit v1.2.3


From 7a6973a2fa238468bb47d175908e99d230fbc0b1 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 22 Nov 2016 16:00:25 +0100
Subject: Fix GC when NifExport is in use

---
 erts/emulator/beam/erl_gc.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 6f641a1ea7..af799d09da 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2325,6 +2325,11 @@ move_msgq_to_heap(Process *p)
 static Uint
 setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
 {
+    /*
+     * NOTE!
+     *   Remember to update offset_rootset() when changing
+     *   this function.
+     */
     Roots* roots;
     Uint n;
 
@@ -2969,6 +2974,12 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size,
 	offset_heap_ptr(objv, nobj, offs, area, area_size);
     }
     offset_off_heap(p, offs, area, area_size);
+    if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) {
+	Eterm* argv;
+	int argc;
+	if (erts_setup_nif_gc(p, &argv, &argc))
+	    offset_heap_ptr(argv, argc, offs, area, area_size);
+    }
 }
 
 static void
-- 
cgit v1.2.3


From ad94a4bdb24c93d3cafc7351c4c98f346a5f53cc Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 22 Nov 2016 16:14:57 +0100
Subject: Fix check_process_code() when NifExport is in use

---
 erts/emulator/beam/beam_bif_load.c | 11 +++++++++++
 erts/emulator/beam/erl_nif.c       | 11 +++++++++++
 erts/emulator/beam/erl_process.h   |  1 +
 3 files changed, 23 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 5969197168..74a333ecbe 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1086,6 +1086,11 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls
 	|| ErtsInArea(rp->cp, mod_start, mod_size)) {
 	return am_true;
     }
+ 
+    *redsp += 1;
+
+    if (erts_check_nif_export_in_area(rp, mod_start, mod_size))
+	return am_true;
 
     *redsp += (STACK_START(rp) - rp->stop) / 32;
 
@@ -1161,6 +1166,12 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls
 	|| ErtsInArea(rp->cp, mod_start, mod_size)) {
 	return am_true;
     }
+ 
+    *redsp += 1;
+
+    if (erts_check_nif_export_in_area(rp, mod_start, mod_size))
+	return am_true;
+
 
     /*
      * Check all continuation pointers stored on the stack.
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 55bd9f7455..e2f208badf 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2309,6 +2309,17 @@ erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj)
     return gc;
 }
 
+int
+erts_check_nif_export_in_area(Process *p, char *start, Uint size)
+{
+    NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p);
+    if (!nep || !nep->saved_current)
+	return 0;
+    if (ErtsInArea(nep->saved_current, start, size))
+	return 1;
+    return 0;
+}
+
 /*
  * Allocate a NifExport and set it in proc specific data
  */
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 3347a7a60e..435b53fb5f 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1586,6 +1586,7 @@ Uint64 erts_step_proc_interval(void);
 
 int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */
 void erts_destroy_nif_export(void *); /* see erl_nif.c */
+int erts_check_nif_export_in_area(Process *p, char *start, Uint size);
 
 ErtsProcList *erts_proclist_create(Process *);
 ErtsProcList *erts_proclist_copy(ErtsProcList *);
-- 
cgit v1.2.3


From 80490fe3e8f636f4de3af1d71cdb41721e88ac7c Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 22 Nov 2016 16:19:55 +0100
Subject: Silence debug warning when no beam jump table is used with dirty
 schedulers

---
 erts/emulator/beam/beam_emu.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index ef4cdf9d5a..c2000929d5 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5325,7 +5325,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 
 	I = c_p->i;
 
-	ASSERT(BeamOp(op_call_nif) == (BeamInstr *) *I);
+	ASSERT(em_call_nif == (BeamInstr *) *I);
 
 	/*
 	 * Set fcalls even though we ignore it, so we don't
-- 
cgit v1.2.3


From 696167d5e64365c28d626e117c136cb81e4c4028 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 21 Oct 2016 14:16:24 +0200
Subject: Fix alloc-util hard-debug

---
 erts/emulator/beam/erl_alloc_util.c | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 2995f2f822..7abe3846be 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -4153,9 +4153,9 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
 	    ASSERT(IS_LAST_BLK(blk));
 
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
-	    (*allctr->link_free_block)(allctr, blk, 0);
+	    (*allctr->link_free_block)(allctr, blk);
 	    HARD_CHECK_BLK_CARRIER(allctr, blk);
-	    (*allctr->unlink_free_block)(allctr, blk, 0);
+	    (*allctr->unlink_free_block)(allctr, blk);
 #endif
 	}
 #endif
@@ -6440,11 +6440,6 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
 
 	ASSERT(SBC2BLK(allctr, sbc) == iblk);
 	ASSERT(CARRIER_SZ(sbc) - SBC_HEADER_SIZE >= SBC_BLK_SZ(iblk));
-#if HAVE_ERTS_MSEG
-	if (IS_MSEG_CARRIER(sbc)) {
-	    ASSERT(CARRIER_SZ(sbc) % ERTS_SACRR_UNIT_SZ == 0);
-	}
-#endif
 	crr = sbc;
 	cl = &allctr->sbc_list;
     }
-- 
cgit v1.2.3


From a532df810da29fcb28d142d244b6b3c812fa33ca Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 17 Oct 2016 12:00:32 +0200
Subject: Fix dirty scheduler process priority

---
 erts/emulator/beam/erl_process.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index bc59147c6c..59ee84d115 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -6271,7 +6271,11 @@ check_dirty_enqueue_in_prio_queue(Process *c_p,
 	return -1*queue;
     }
 
-    *newp |= ERTS_PSFLG_IN_RUNQ;
+    /*
+     * Enqueue using process struct.
+     */
+    *newp &= ~ERTS_PSFLGS_PRQ_PRIO_MASK;
+    *newp |= ERTS_PSFLG_IN_RUNQ | (aprio << ERTS_PSFLGS_PRQ_PRIO_OFFSET);
     return queue;
 }
 
-- 
cgit v1.2.3


From dd2e99bdd726d4322c8e07c81731ad66ae05176e Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 21 Oct 2016 14:05:11 +0200
Subject: Fix send of exit signal to process executing dirty

---
 erts/emulator/beam/erl_process.c | 34 ++++++++++++++++++++++++++++------
 1 file changed, 28 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 59ee84d115..86a997de65 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -12405,6 +12405,8 @@ send_exit_signal(Process *c_p,		/* current process if and only
 	    else if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) {
 		/* Process not running ... */
 		ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL;
+		ErlHeapFragment *bp = NULL;
+		Eterm rsn_cpy;
 		if (need_locks
 		    && erts_smp_proc_trylock(rp, need_locks) == EBUSY) {
 		    /* ... but we havn't got all locks on it ... */
@@ -12417,12 +12419,32 @@ send_exit_signal(Process *c_p,		/* current process if and only
 		}
                 /* ...and we have all locks on it... */
                 *rp_locks = ERTS_PROC_LOCKS_ALL;
-                set_proc_exiting(rp,
-                                 state,
-                                 (is_immed(rsn)
-                                  ? rsn
-                                  : copy_object(rsn, rp)),
-                                 NULL);
+
+		state = erts_smp_atomic32_read_nob(&rp->state);
+
+		if (is_immed(rsn))
+		    rsn_cpy = rsn;
+		else {
+		    Eterm *hp;
+		    ErlOffHeap *ohp;
+		    Uint rsn_sz = size_object(rsn);
+#ifdef ERTS_DIRTY_SCHEDULERS
+		    if (state & (ERTS_PSFLG_DIRTY_RUNNING
+				 | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
+			bp = new_message_buffer(rsn_sz);
+			ohp = &bp->off_heap;
+			hp = &bp->mem[0];
+		    }
+		    else
+#endif
+		    {
+			hp = HAlloc(rp, rsn_sz);
+			ohp = &rp->off_heap;
+		    }
+		    rsn_cpy = copy_struct(rsn, rsn_sz, &hp, ohp);
+		}
+
+		set_proc_exiting(rp, state, rsn_cpy, bp);
 	    }
 	    else { /* Process running... */
 
-- 
cgit v1.2.3


From e9ac0c64a78c858d474757d435f3c7dce2a224b1 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 1 Nov 2016 16:08:21 +0100
Subject: Fix call time tracing with dirty schedulers

---
 erts/emulator/beam/beam_bp.c     | 2 +-
 erts/emulator/beam/beam_bp.h     | 2 +-
 erts/emulator/beam/erl_process.c | 4 ++++
 erts/emulator/beam/erl_process.h | 1 +
 4 files changed, 7 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 920c8b1ed0..bbb2e4f34f 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -1517,7 +1517,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags,
 	ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0);
 	bdt = Alloc(sizeof(BpDataTime));
 	erts_refc_init(&bdt->refc, 1);
-	bdt->n = erts_no_schedulers;
+	bdt->n = erts_no_total_schedulers;
 	bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n));
 	for (i = 0; i < bdt->n; i++) {
 	    bp_hash_init(&(bdt->hash[i]), 32);
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index 541af77211..7206ef471a 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -80,7 +80,7 @@ typedef struct generic_bp {
 #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2)
 
 #ifdef ERTS_SMP
-#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->no - 1)
+#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->thr_id - 1)
 #else
 #define bp_sched2ix_proc(p) (0)
 #endif
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 86a997de65..adf7779fe5 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -154,6 +154,7 @@ int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1;
 int ERTS_WRITE_UNLIKELY(erts_sched_compact_load);
 int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0;
 Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers);
+Uint ERTS_WRITE_UNLIKELY(erts_no_total_schedulers);
 Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers) = 0;
 Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers) = 0;
 
@@ -5908,9 +5909,12 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 
     n = (int) no_schedulers;
     erts_no_schedulers = n;
+    erts_no_total_schedulers = n;
 #ifdef ERTS_DIRTY_SCHEDULERS
     erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers;
+    erts_no_total_schedulers += no_dirty_cpu_schedulers;
     erts_no_dirty_io_schedulers = no_dirty_io_schedulers;
+    erts_no_total_schedulers += no_dirty_io_schedulers;
 #endif
 
     /* Create and initialize scheduler sleep info */
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 435b53fb5f..9214d05456 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -111,6 +111,7 @@ extern int erts_eager_check_io;
 extern int erts_sched_compact_load;
 extern int erts_sched_balance_util;
 extern Uint erts_no_schedulers;
+extern Uint erts_no_total_schedulers;
 #ifdef ERTS_DIRTY_SCHEDULERS
 extern Uint erts_no_dirty_cpu_schedulers;
 extern Uint erts_no_dirty_io_schedulers;
-- 
cgit v1.2.3


From ea15a660908c48a2b8cefe0ed90fc5c4a89fe572 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 22 Nov 2016 15:31:44 +0100
Subject: Fix scheduling of system tasks on processes executing dirty

---
 erts/emulator/beam/erl_process.c | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index adf7779fe5..92b3d9d1a8 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -6691,15 +6691,8 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
     erts_aint32_t fail_state, state, a, n, enq_prio;
     int enqueue; /* < 0 -> use proxy */
     unsigned int prof_runnable_procs;
-    int strict_fail_state;
 
     fail_state = *fail_state_p;
-    /*
-     * If fail state something other than just exiting process,
-     * ensure that the task wont be scheduled when the 
-     * receiver is in the failure state.
-     */
-    strict_fail_state = fail_state != ERTS_PSFLG_EXITING;
 
     res = 1; /* prepare for success */
     st->next = st->prev = st; /* Prep for empty prio queue */
@@ -6781,7 +6774,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
     /* Status lock prevents out of order "runnable proc" trace msgs */
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
 
-    if (!prof_runnable_procs && !strict_fail_state) {
+    if (!prof_runnable_procs) {
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 	locked = 0;
     }
@@ -6792,11 +6785,6 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
 	erts_aint32_t e;
 	n = e = a;
 
-	if (strict_fail_state && (a & fail_state)) {
-	    *fail_state_p = (a & fail_state);
-	    goto cleanup;
-	}
-
 	if (a & ERTS_PSFLG_FREE)
 	    goto cleanup; /* We don't want to schedule free processes... */
 
-- 
cgit v1.2.3


From ab647066f572d9456572a70c53687f9ab52502b8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 23 Nov 2016 15:33:29 +0100
Subject: erts: Remove one little space in crash dump

---
 erts/emulator/beam/break.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 33433df68a..dfbe1ced47 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -666,7 +666,7 @@ static Sint64 crash_dump_written = 0;
 
 static int crash_dump_limited_writer(void* vfdp, char* buf, size_t len)
 {
-    const char stop_msg[] = "\n=abort: CRASH DUMP SIZE LIMIT REACHED\n";
+    const char stop_msg[] = "\n=abort:CRASH DUMP SIZE LIMIT REACHED\n";
 
     crash_dump_written += len;
     if (crash_dump_written <= crash_dump_limit) {
-- 
cgit v1.2.3


From ec776506eac5342da3d45829fb67b669081f6439 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 23 Nov 2016 15:55:43 +0100
Subject: erts: Secure abi backward compat for tracer nifs

and simplify the code by handling the compatibility stuff
at loading by creating modern struct copies in create_lib().
---
 erts/emulator/beam/erl_nif.c | 128 +++++++++++++++++++++++--------------------
 erts/emulator/beam/erl_nif.h |  15 +++--
 2 files changed, 77 insertions(+), 66 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 9872a3ab81..9fec9b4939 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -71,10 +71,12 @@
 struct erl_module_nif {
     void* priv_data;
     void* handle;             /* "dlopen" */
-    struct enif_entry_t* entry;
+    struct enif_entry_t entry;
     erts_refc_t rt_cnt;       /* number of resource types */
     erts_refc_t rt_dtor_cnt;  /* number of resource types with destructors */
     Module* mod;           /* Can be NULL if orphan with dtor-resources left */ 
+
+    ErlNifFunc _funcs_copy_[1];  /* only used for old libs */
 };
 
 #ifdef DEBUG
@@ -1958,10 +1960,10 @@ static void close_lib(struct erl_module_nif* lib)
     ASSERT(lib->handle != NULL);
     ASSERT(erts_refc_read(&lib->rt_dtor_cnt,0) == 0);
 
-    if (lib->entry != NULL && lib->entry->unload != NULL) {
+    if (lib->entry.unload != NULL) {
 	struct enif_msg_environment_t msg_env;
         pre_nif_noproc(&msg_env, lib, NULL);
-	lib->entry->unload(&msg_env.env, lib->priv_data);
+	lib->entry.unload(&msg_env.env, lib->priv_data);
         post_nif_noproc(&msg_env);
     }
     if (!erts_is_static_nif(lib->handle))
@@ -3115,33 +3117,52 @@ static Eterm load_nif_error(Process* p, const char* atom, const char* format, ..
     return ret;
 }
 
+#define AT_LEAST_VERSION(E,MAJ,MIN) \
+    (((E)->major * 0x100 + (E)->minor) >= ((MAJ) * 0x100 + (MIN)))
+
 /*
- * The function below is for looping through ErlNifFunc arrays, helping
- * provide backwards compatibility across the version 2.7 change that added
- * the "flags" field to ErlNifFunc.
+ * Allocate erl_module_nif and make a _modern_ copy of the lib entry.
  */
-static ErlNifFunc* next_func(ErlNifEntry* entry, int* incrp, ErlNifFunc* func)
+static struct erl_module_nif* create_lib(const ErlNifEntry* src)
 {
-    ASSERT(incrp);
-    if (!*incrp) {
-	if (entry->major > 2 || (entry->major == 2 && entry->minor >= 7))
-	    *incrp = sizeof(ErlNifFunc);
-	else {
-	    /*
-	     * ErlNifFuncV1 below is what ErlNifFunc was before the
-	     * addition of the flags field for 2.7, and is needed to handle
-	     * backward compatibility.
-	     */
-	    typedef struct {
-		const char* name;
-		unsigned arity;
-		ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
-	    }ErlNifFuncV1;
-	    *incrp = sizeof(ErlNifFuncV1);
-	}
+    struct erl_module_nif* lib;
+    ErlNifEntry* dst;
+    Uint bytes = offsetof(struct erl_module_nif, _funcs_copy_);
+
+    if (!AT_LEAST_VERSION(src, 2, 7))
+        bytes += src->num_of_funcs * sizeof(ErlNifFunc);
+
+    lib = erts_alloc(ERTS_ALC_T_NIF, bytes);
+    dst = &lib->entry;
+
+    sys_memcpy(dst, src, offsetof(ErlNifEntry, vm_variant));
+
+    if (AT_LEAST_VERSION(src, 2, 1)) {
+        dst->vm_variant = src->vm_variant;
+    } else {
+        dst->vm_variant = "beam.vanilla";
     }
-    return (ErlNifFunc*) ((char*)func + *incrp);
-}
+    if (AT_LEAST_VERSION(src, 2, 7)) {
+        dst->options = src->options;
+    } else {
+        /*
+         * Make a modern copy of the ErlNifFunc array
+         */
+        struct ErlNifFunc_V1 {
+            const char* name;
+            unsigned arity;
+            ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+        }*src_funcs = (struct ErlNifFunc_V1*) src->funcs;
+        int i;
+        for (i = 0; i < src->num_of_funcs; ++i) {
+            sys_memcpy(&lib->_funcs_copy_[i], &src_funcs[i], sizeof(*src_funcs));
+            lib->_funcs_copy_[i].flags = 0;
+        }
+        dst->funcs = lib->_funcs_copy_;
+        dst->options = 0;
+    }
+    return lib;
+};
 
 
 BIF_RETTYPE load_nif_2(BIF_ALIST_2)
@@ -3261,7 +3282,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	ret = load_nif_error(BIF_P, bad_lib, "Library version (%d.%d) not compatible (with %d.%d).",
 			     entry->major, entry->minor, ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION);
     }   
-    else if (entry->minor >= 1
+    else if (AT_LEAST_VERSION(entry, 2, 1)
 	     && sys_strcmp(entry->vm_variant, ERL_NIF_VM_VARIANT) != 0) {
 	ret = load_nif_error(BIF_P, bad_lib, "Library (%s) not compiled for "
 			     "this vm variant (%s).",
@@ -3272,20 +3293,25 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 			     " match calling module '%T'", entry->name, mod_atom);
     }
     else {
-	/*erts_fprintf(stderr, "Found module %T\r\n", mod_atom);*/
+        lib = create_lib(entry);
+        entry = &lib->entry; /* Use a guaranteed modern lib entry from now on */
+
+        lib->handle = handle;
+        erts_refc_init(&lib->rt_cnt, 0);
+        erts_refc_init(&lib->rt_dtor_cnt, 0);
+        ASSERT(opened_rt_list == NULL);
+        lib->mod = module_p;
 
-	int maybe_dirty_nifs = ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7))
-				&& (entry->options & ERL_NIF_DIRTY_NIF_OPTION));
-	int incr = 0;
-	ErlNifFunc* f = entry->funcs;
-	for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) {
+        for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) {
 	    ErtsCodeInfo** ci_pp;
+            ErlNifFunc* f = &entry->funcs[i];
+
 	    if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1)
 		|| (ci_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) {
 		ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u",
 				     mod_atom, f->name, f->arity);
 	    }
-	    else if (maybe_dirty_nifs && f->flags) {
+	    else if (f->flags) {
 		/*
 		 * If the flags field is non-zero and this emulator was
 		 * built with dirty scheduler support, check that the flags
@@ -3314,7 +3340,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	    }
 	    /*erts_fprintf(stderr, "Found NIF %T:%s/%u\r\n",
 	      mod_atom, f->name, f->arity);*/
-	    f = next_func(entry, &incr, f);
 	}
     }
 
@@ -3325,14 +3350,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     /* Call load or upgrade:
      */
 
-
-    lib = erts_alloc(ERTS_ALC_T_NIF, sizeof(struct erl_module_nif));
-    lib->handle = handle;
-    lib->entry = entry;
-    erts_refc_init(&lib->rt_cnt, 0);
-    erts_refc_init(&lib->rt_dtor_cnt, 0);
-    ASSERT(opened_rt_list == NULL);
-    lib->mod = module_p;
     env.mod_nif = lib;
 
     lib->priv_data = NULL;
@@ -3349,8 +3366,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
             prev_mi->nif->priv_data = prev_old_data;
             ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful (%d).", veto);
         }
-        else
-            commit_opened_resource_types(lib);
     }
     else if (entry->load != NULL) { /********* Initial load ***********/
         erts_pre_nif(&env, BIF_P, lib, NULL);
@@ -3359,22 +3374,21 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
         if (veto) {
             ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful (%d).", veto);
         }
-        else
-            commit_opened_resource_types(lib);
     }
     if (ret == am_ok) {
+        commit_opened_resource_types(lib);
+
 	/*
 	** Everything ok, patch the beam code with op_call_nif
 	*/
 
-	int incr = 0;
-	ErlNifFunc* f = entry->funcs;
-
 	this_mi->nif = lib;
 	for (i=0; i < entry->num_of_funcs; i++)
 	{
+            ErlNifFunc* f = &entry->funcs[i];
 	    ErtsCodeInfo* ci;
             BeamInstr *code_ptr;
+
 	    erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1);
 	    ci = *get_func_pp(this_mi->code_hdr, f_atom, f->arity);
             code_ptr = erts_codeinfo_to_code(ci);
@@ -3389,8 +3403,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 		g->orig_instr = (BeamInstr) BeamOp(op_call_nif);
 	    }
 #ifdef ERTS_DIRTY_SCHEDULERS
-	    if ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7))
-		&& (entry->options & ERL_NIF_DIRTY_NIF_OPTION) && f->flags) {
+	    if (f->flags) {
 		code_ptr[3] = (BeamInstr) f->fptr;
 		code_ptr[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ?
 		    (BeamInstr) schedule_dirty_io_nif :
@@ -3400,7 +3413,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 #endif
 		code_ptr[1] = (BeamInstr) f->fptr;
 	    code_ptr[2] = (BeamInstr) lib;
-	    f = next_func(entry, &incr, f);
 	}
     }
     else {
@@ -3484,8 +3496,8 @@ void erl_nif_init()
 int erts_nif_get_funcs(struct erl_module_nif* mod,
                        ErlNifFunc **funcs)
 {
-    *funcs = mod->entry->funcs;
-    return mod->entry->num_of_funcs;
+    *funcs = mod->entry.funcs;
+    return mod->entry.num_of_funcs;
 }
 
 Eterm erts_nif_call_function(Process *p, Process *tracee,
@@ -3496,10 +3508,10 @@ Eterm erts_nif_call_function(Process *p, Process *tracee,
 #ifdef DEBUG
     /* Verify that function is part of this module */
     int i;
-    for (i = 0; i < mod->entry->num_of_funcs; i++)
-        if (fun == &(mod->entry->funcs[i]))
+    for (i = 0; i < mod->entry.num_of_funcs; i++)
+        if (fun == &(mod->entry.funcs[i]))
             break;
-    ASSERT(i < mod->entry->num_of_funcs);
+    ASSERT(i < mod->entry.num_of_funcs);
     if (p)
         ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN
                            || erts_smp_thr_progress_is_blocking());
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 494971e118..413b4f7343 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -49,7 +49,7 @@
 ** 2.8: 18.0 add enif_has_pending_exception
 ** 2.9: 18.2 enif_getenv
 ** 2.10: Time API
-** 2.11: 19.0 enif_snprintf
+** 2.11: 19.0 enif_snprintf 
 */
 #define ERL_NIF_MAJOR_VERSION 2
 #define ERL_NIF_MINOR_VERSION 11
@@ -116,12 +116,13 @@ typedef struct enif_entry_t
     int  (*reload) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info);
     int  (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info);
     void (*unload) (ErlNifEnv*, void* priv_data);
+
+    /* Added in 2.1 */
     const char* vm_variant;
-    unsigned options;
-}ErlNifEntry;
 
-/* Field bits for ErlNifEntry options */
-#define ERL_NIF_DIRTY_NIF_OPTION 1
+    /* Added in 2.7 */
+    unsigned options;   /* Unused. Can be set to 0 or 1 (dirty sched config) */
+}ErlNifEntry;
 
 
 typedef struct
@@ -266,8 +267,6 @@ extern TWinDynNifCallbacks WinDynNifCallbacks;
 #  define ERL_NIF_INIT_DECL(MODNAME) ERL_NIF_INIT_EXPORT ErlNifEntry* nif_init(ERL_NIF_INIT_ARGS)
 #endif
 
-#define ERL_NIF_ENTRY_OPTIONS ERL_NIF_DIRTY_NIF_OPTION
-
 #ifdef __cplusplus
 }
 #  define ERL_NIF_INIT_PROLOGUE extern "C" {
@@ -293,7 +292,7 @@ ERL_NIF_INIT_DECL(NAME)			\
 	FUNCS,				\
 	LOAD, RELOAD, UPGRADE, UNLOAD,	\
 	ERL_NIF_VM_VARIANT,		\
-	ERL_NIF_ENTRY_OPTIONS		\
+	1		                \
     };                                  \
     ERL_NIF_INIT_BODY;                  \
     return &entry;			\
-- 
cgit v1.2.3


From 3d5f679fdb59988369b198776d5e3930b594941a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 18 Nov 2016 21:43:42 +0100
Subject: erts: Add testing of old nif API

---
 erts/emulator/test/nif_SUITE.erl                   |  25 +-
 erts/emulator/test/nif_SUITE_data/Makefile.src     |   5 +-
 .../test/nif_SUITE_data/nif_api_2_4/README         |   6 +
 .../test/nif_SUITE_data/nif_api_2_4/erl_drv_nif.h  |  48 ++
 .../test/nif_SUITE_data/nif_api_2_4/erl_nif.h      | 237 ++++++++++
 .../nif_SUITE_data/nif_api_2_4/erl_nif_api_funcs.h | 503 +++++++++++++++++++++
 erts/emulator/test/nif_SUITE_data/nif_mod.1.2_4.c  |   4 +
 erts/emulator/test/nif_SUITE_data/nif_mod.2.2_4.c  |   4 +
 erts/emulator/test/nif_SUITE_data/nif_mod.3.2_4.c  |   4 +
 erts/emulator/test/nif_SUITE_data/nif_mod.c        |   9 +
 erts/emulator/test/nif_SUITE_data/nif_mod.erl      |  29 +-
 11 files changed, 864 insertions(+), 10 deletions(-)
 create mode 100644 erts/emulator/test/nif_SUITE_data/nif_api_2_4/README
 create mode 100644 erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_drv_nif.h
 create mode 100644 erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif.h
 create mode 100644 erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif_api_funcs.h
 create mode 100644 erts/emulator/test/nif_SUITE_data/nif_mod.1.2_4.c
 create mode 100644 erts/emulator/test/nif_SUITE_data/nif_mod.2.2_4.c
 create mode 100644 erts/emulator/test/nif_SUITE_data/nif_mod.3.2_4.c

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 37d6e68868..99213a6888 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -155,7 +155,13 @@ reload_error(Config) when is_list(Config) ->
     ok.
 
 %% Test upgrade callback in nif lib
-upgrade(Config) when is_list(Config) ->    
+upgrade(Config) when is_list(Config) ->
+    [begin io:format("Do test for API version \"~p\"", [API]),
+           upgrade_do([{nif_api_version,API}|Config])
+     end
+     || API <- ["", ".2_4"]].
+
+upgrade_do(Config) ->
     TmpMem = tmpmem(),
     ensure_lib_loaded(Config),
 
@@ -295,6 +301,12 @@ upgrade(Config) when is_list(Config) ->
 
 %% Test loading/upgrade in on_load
 t_on_load(Config) when is_list(Config) ->
+    [begin io:format("Do test for API version \"~p\"", [API]),
+           t_on_load_do([{nif_api_version,API}|Config])
+     end
+     || API <- ["", ".2_4"]].
+
+t_on_load_do(Config) ->
     TmpMem = tmpmem(),
     ensure_lib_loaded(Config),
 
@@ -306,6 +318,8 @@ t_on_load(Config) when is_list(Config) ->
     %% Use ETS to tell nif_mod:on_load what to do
     ets:insert(nif_SUITE, {data_dir, Data}),
     ets:insert(nif_SUITE, {lib_version, 1}),
+    API = proplists:get_value(nif_api_version, Config, ""),
+    ets:insert(nif_SUITE, {nif_api_version, API}),
     {module,nif_mod} = code:load_binary(nif_mod,File,Bin),
     hold_nif_mod_priv_data(nif_mod:get_priv_data_ptr()),
     [{load,1,1,101},{get_priv_data_ptr,1,2,102}] = nif_mod_call_history(),
@@ -848,6 +862,12 @@ resource_binary_do() ->
 
 %% Test resource takeover by module upgrade
 resource_takeover(Config) when is_list(Config) ->    
+    [begin io:format("Do test for API version \"~p\"", [API]),
+           resource_takeover_do([{nif_api_version,API}|Config])
+     end
+     || API <- ["", ".2_4"]].
+
+resource_takeover_do(Config) ->
     TmpMem = tmpmem(),
     ensure_lib_loaded(Config),
 
@@ -1169,6 +1189,9 @@ resource_takeover(Config) when is_list(Config) ->
     ok = forget_resource(AN7),
     [] = nif_mod_call_history(),
 
+    true = erlang:delete_module(nif_mod),
+    true = erlang:purge_module(nif_mod),
+
     true = lists:member(?MODULE, erlang:system_info(taints)),
     true = lists:member(nif_mod, erlang:system_info(taints)),
     verify_tmpmem(TmpMem),    
diff --git a/erts/emulator/test/nif_SUITE_data/Makefile.src b/erts/emulator/test/nif_SUITE_data/Makefile.src
index fbb8978771..cbe843b120 100644
--- a/erts/emulator/test/nif_SUITE_data/Makefile.src
+++ b/erts/emulator/test/nif_SUITE_data/Makefile.src
@@ -2,7 +2,10 @@
 NIF_LIBS = nif_SUITE.1@dll@ \
            nif_mod.1@dll@ \
            nif_mod.2@dll@ \
-           nif_mod.3@dll@
+           nif_mod.3@dll@ \
+	   nif_mod.1.2_4@dll@ \
+	   nif_mod.2.2_4@dll@ \
+	   nif_mod.3.2_4@dll@
 
 all: $(NIF_LIBS) basic@dll@ rwlock@dll@ tsd@dll@ echo_drv@dll@
 
diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_4/README b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/README
new file mode 100644
index 0000000000..7abd0319a6
--- /dev/null
+++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/README
@@ -0,0 +1,6 @@
+These are old genuine header files
+checked out from tag OTP_R16B 05f11890bdfec4bfc3a78e191
+
+I choose this API version (2.4) to test, as it's before
+the addition of 'options' in ErlNifEntry and 'flags' in ErlNifFunc
+and without include of generated erl_native_features_config.h.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_drv_nif.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_drv_nif.h
new file mode 100644
index 0000000000..ea013a49a3
--- /dev/null
+++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_drv_nif.h
@@ -0,0 +1,48 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Common structures for both erl_driver.h and erl_nif.h
+ */
+
+#ifndef __ERL_DRV_NIF_H__
+#define __ERL_DRV_NIF_H__
+
+typedef struct {
+    int driver_major_version;
+    int driver_minor_version;
+    char *erts_version;
+    char *otp_release;
+    int thread_support;
+    int smp_support;
+    int async_threads;
+    int scheduler_threads;
+    int nif_major_version;
+    int nif_minor_version;
+}  ErlDrvSysInfo;
+
+typedef struct {
+    int suggested_stack_size;
+} ErlDrvThreadOpts;
+
+#endif  /* __ERL_DRV_NIF_H__ */
+
+
+
+
diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif.h
new file mode 100644
index 0000000000..8006741a63
--- /dev/null
+++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif.h
@@ -0,0 +1,237 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2013. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/* Include file for writers of Native Implemented Functions. 
+*/
+
+#ifndef __ERL_NIF_H__
+#define __ERL_NIF_H__
+
+
+#include "erl_drv_nif.h"
+
+/* Version history:
+** 0.1: R13B03
+** 1.0: R13B04
+** 2.0: R14A
+** 2.1: R14B02 "vm_variant"
+** 2.2: R14B03 enif_is_exception
+** 2.3: R15 enif_make_reverse_list, enif_is_number
+** 2.4: R16 enif_consume_timeslice
+*/
+#define ERL_NIF_MAJOR_VERSION 2
+#define ERL_NIF_MINOR_VERSION 4
+
+#include <stdlib.h>
+
+#ifdef SIZEOF_CHAR
+#  define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR
+#  undef SIZEOF_CHAR
+#endif
+#ifdef SIZEOF_SHORT
+#  define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT
+#  undef SIZEOF_SHORT
+#endif
+#ifdef SIZEOF_INT
+#  define SIZEOF_INT_SAVED__ SIZEOF_INT
+#  undef SIZEOF_INT
+#endif
+#ifdef SIZEOF_LONG
+#  define SIZEOF_LONG_SAVED__ SIZEOF_LONG
+#  undef SIZEOF_LONG
+#endif
+#ifdef SIZEOF_LONG_LONG
+#  define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG
+#  undef SIZEOF_LONG_LONG
+#endif
+#ifdef HALFWORD_HEAP_EMULATOR
+#  define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR
+#  undef HALFWORD_HEAP_EMULATOR
+#endif
+#include "erl_int_sizes_config.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
+typedef unsigned __int64 ErlNifUInt64;
+typedef __int64 ErlNifSInt64;
+#elif SIZEOF_LONG == 8
+typedef unsigned long ErlNifUInt64;
+typedef long ErlNifSInt64;
+#elif SIZEOF_LONG_LONG == 8
+typedef unsigned long long ErlNifUInt64;
+typedef long long ErlNifSInt64;
+#else
+#error No 64-bit integer type
+#endif
+
+#ifdef HALFWORD_HEAP_EMULATOR
+#  define ERL_NIF_VM_VARIANT "beam.halfword" 
+typedef unsigned int ERL_NIF_TERM;
+#else
+#  define ERL_NIF_VM_VARIANT "beam.vanilla" 
+#  if SIZEOF_LONG == SIZEOF_VOID_P
+typedef unsigned long ERL_NIF_TERM;
+#  elif SIZEOF_LONG_LONG == SIZEOF_VOID_P
+typedef unsigned long long ERL_NIF_TERM;
+#  endif
+#endif
+
+struct enif_environment_t;
+typedef struct enif_environment_t ErlNifEnv;
+
+typedef struct
+{
+    const char* name;
+    unsigned arity;
+    ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+}ErlNifFunc;
+
+typedef struct enif_entry_t
+{
+    int major;
+    int minor;
+    const char* name;
+    int num_of_funcs;
+    ErlNifFunc* funcs;
+    int  (*load)   (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info);
+    int  (*reload) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info);
+    int  (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info);
+    void (*unload) (ErlNifEnv*, void* priv_data);
+    const char* vm_variant;
+}ErlNifEntry;
+
+
+
+typedef struct
+{
+    size_t size;
+    unsigned char* data;
+
+    /* Internals (avert your eyes) */
+    ERL_NIF_TERM bin_term;
+    void* ref_bin;
+}ErlNifBinary;
+
+typedef struct enif_resource_type_t ErlNifResourceType;
+typedef void ErlNifResourceDtor(ErlNifEnv*, void*);
+typedef enum
+{
+    ERL_NIF_RT_CREATE = 1,
+    ERL_NIF_RT_TAKEOVER = 2
+}ErlNifResourceFlags;
+
+typedef enum
+{
+    ERL_NIF_LATIN1 = 1
+}ErlNifCharEncoding;
+
+typedef struct
+{
+    ERL_NIF_TERM pid;  /* internal, may change */
+}ErlNifPid;
+
+typedef ErlDrvSysInfo ErlNifSysInfo;
+
+typedef struct ErlDrvTid_ *ErlNifTid;
+typedef struct ErlDrvMutex_ ErlNifMutex;
+typedef struct ErlDrvCond_ ErlNifCond;
+typedef struct ErlDrvRWLock_ ErlNifRWLock;
+typedef int ErlNifTSDKey;
+
+typedef ErlDrvThreadOpts ErlNifThreadOpts;
+
+#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
+#  define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS
+typedef struct {
+#  include "erl_nif_api_funcs.h"
+} TWinDynNifCallbacks;
+extern TWinDynNifCallbacks WinDynNifCallbacks;
+#  undef ERL_NIF_API_FUNC_DECL
+#endif
+
+#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) && !defined(STATIC_ERLANG_DRIVER)
+#  define ERL_NIF_API_FUNC_MACRO(NAME) (WinDynNifCallbacks.NAME)
+#  include "erl_nif_api_funcs.h"
+/* note that we have to keep ERL_NIF_API_FUNC_MACRO defined */
+
+#else /* non windows or included from emulator itself */
+
+#  define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) extern RET_TYPE NAME ARGS
+#  include "erl_nif_api_funcs.h"
+#  undef ERL_NIF_API_FUNC_DECL
+#endif
+
+
+#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
+#  define ERL_NIF_INIT_GLOB TWinDynNifCallbacks WinDynNifCallbacks;
+#  define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks)
+#  define ERL_NIF_INIT_BODY memcpy(&WinDynNifCallbacks,callbacks,sizeof(TWinDynNifCallbacks))
+#else 
+#  define ERL_NIF_INIT_GLOB
+#  define ERL_NIF_INIT_BODY
+#  define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* nif_init(void)
+#endif
+
+
+#ifdef __cplusplus
+}
+#  define ERL_NIF_INIT_PROLOGUE extern "C" {
+#  define ERL_NIF_INIT_EPILOGUE }
+#else
+#  define ERL_NIF_INIT_PROLOGUE
+#  define ERL_NIF_INIT_EPILOGUE
+#endif
+
+
+#define ERL_NIF_INIT(NAME, FUNCS, LOAD, RELOAD, UPGRADE, UNLOAD) \
+ERL_NIF_INIT_PROLOGUE                   \
+ERL_NIF_INIT_GLOB                       \
+ERL_NIF_INIT_DECL(NAME);		\
+ERL_NIF_INIT_DECL(NAME)			\
+{					\
+    static ErlNifEntry entry = 		\
+    {					\
+	ERL_NIF_MAJOR_VERSION,		\
+	ERL_NIF_MINOR_VERSION,		\
+	#NAME,				\
+	sizeof(FUNCS) / sizeof(*FUNCS),	\
+	FUNCS,				\
+	LOAD, RELOAD, UPGRADE, UNLOAD,	\
+	ERL_NIF_VM_VARIANT		\
+    };                                  \
+    ERL_NIF_INIT_BODY;                  \
+    return &entry;			\
+}                                       \
+ERL_NIF_INIT_EPILOGUE
+
+#if defined(USE_DYNAMIC_TRACE) && (defined(USE_DTRACE) || defined(USE_SYSTEMTAP))
+#define HAVE_USE_DTRACE 1
+#endif
+
+#ifdef HAVE_USE_DTRACE
+ERL_NIF_TERM erl_nif_user_trace_s1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+ERL_NIF_TERM erl_nif_user_trace_i4s4(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+ERL_NIF_TERM erl_nif_user_trace_n(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+#endif
+
+#endif /* __ERL_NIF_H__ */
+
diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif_api_funcs.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif_api_funcs.h
new file mode 100644
index 0000000000..2f841645e1
--- /dev/null
+++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif_api_funcs.h
@@ -0,0 +1,503 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2013. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#if !defined(ERL_NIF_API_FUNC_DECL) && !defined(ERL_NIF_API_FUNC_MACRO)
+#  error This file should not be included directly
+#endif
+
+/*
+** WARNING: add new ERL_NIF_API_FUNC_DECL entries at the bottom of the list
+** to keep compatibility on Windows!!!
+**
+** And don't forget to increase ERL_NIF_MINOR_VERSION in erl_nif.h
+** when adding functions to the API.
+*/
+#ifdef ERL_NIF_API_FUNC_DECL
+ERL_NIF_API_FUNC_DECL(void*,enif_priv_data,(ErlNifEnv*));
+ERL_NIF_API_FUNC_DECL(void*,enif_alloc,(size_t size));
+ERL_NIF_API_FUNC_DECL(void,enif_free,(void* ptr));
+ERL_NIF_API_FUNC_DECL(int,enif_is_atom,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_is_binary,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_is_ref,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_inspect_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, ErlNifBinary* bin));
+ERL_NIF_API_FUNC_DECL(int,enif_alloc_binary,(size_t size, ErlNifBinary* bin));
+ERL_NIF_API_FUNC_DECL(int,enif_realloc_binary,(ErlNifBinary* bin, size_t size));
+ERL_NIF_API_FUNC_DECL(void,enif_release_binary,(ErlNifBinary* bin));
+ERL_NIF_API_FUNC_DECL(int,enif_get_int,(ErlNifEnv*, ERL_NIF_TERM term, int* ip));
+ERL_NIF_API_FUNC_DECL(int,enif_get_ulong,(ErlNifEnv*, ERL_NIF_TERM term, unsigned long* ip));
+ERL_NIF_API_FUNC_DECL(int,enif_get_double,(ErlNifEnv*, ERL_NIF_TERM term, double* dp));
+ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM* head, ERL_NIF_TERM* tail));
+ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array));
+ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
+ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_ulong,(ErlNifEnv* env, unsigned long i));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_double,(ErlNifEnv* env, double d));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_atom,(ErlNifEnv* env, const char* name));
+ERL_NIF_API_FUNC_DECL(int,enif_make_existing_atom,(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple,(ErlNifEnv* env, unsigned cnt, ...));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list,(ErlNifEnv* env, unsigned cnt, ...));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_cell,(ErlNifEnv* env, ERL_NIF_TERM car, ERL_NIF_TERM cdr));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_string,(ErlNifEnv* env, const char* string, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_ref,(ErlNifEnv* env));
+
+ERL_NIF_API_FUNC_DECL(ErlNifMutex*,enif_mutex_create,(char *name));
+ERL_NIF_API_FUNC_DECL(void,enif_mutex_destroy,(ErlNifMutex *mtx));
+ERL_NIF_API_FUNC_DECL(int,enif_mutex_trylock,(ErlNifMutex *mtx));
+ERL_NIF_API_FUNC_DECL(void,enif_mutex_lock,(ErlNifMutex *mtx));
+ERL_NIF_API_FUNC_DECL(void,enif_mutex_unlock,(ErlNifMutex *mtx));
+ERL_NIF_API_FUNC_DECL(ErlNifCond*,enif_cond_create,(char *name));
+ERL_NIF_API_FUNC_DECL(void,enif_cond_destroy,(ErlNifCond *cnd));
+ERL_NIF_API_FUNC_DECL(void,enif_cond_signal,(ErlNifCond *cnd));
+ERL_NIF_API_FUNC_DECL(void,enif_cond_broadcast,(ErlNifCond *cnd));
+ERL_NIF_API_FUNC_DECL(void,enif_cond_wait,(ErlNifCond *cnd, ErlNifMutex *mtx));
+ERL_NIF_API_FUNC_DECL(ErlNifRWLock*,enif_rwlock_create,(char *name));
+ERL_NIF_API_FUNC_DECL(void,enif_rwlock_destroy,(ErlNifRWLock *rwlck));
+ERL_NIF_API_FUNC_DECL(int,enif_rwlock_tryrlock,(ErlNifRWLock *rwlck));
+ERL_NIF_API_FUNC_DECL(void,enif_rwlock_rlock,(ErlNifRWLock *rwlck));
+ERL_NIF_API_FUNC_DECL(void,enif_rwlock_runlock,(ErlNifRWLock *rwlck));
+ERL_NIF_API_FUNC_DECL(int,enif_rwlock_tryrwlock,(ErlNifRWLock *rwlck));
+ERL_NIF_API_FUNC_DECL(void,enif_rwlock_rwlock,(ErlNifRWLock *rwlck));
+ERL_NIF_API_FUNC_DECL(void,enif_rwlock_rwunlock,(ErlNifRWLock *rwlck));
+ERL_NIF_API_FUNC_DECL(int,enif_tsd_key_create,(char *name, ErlNifTSDKey *key));
+ERL_NIF_API_FUNC_DECL(void,enif_tsd_key_destroy,(ErlNifTSDKey key));
+ERL_NIF_API_FUNC_DECL(void,enif_tsd_set,(ErlNifTSDKey key, void *data));
+ERL_NIF_API_FUNC_DECL(void*,enif_tsd_get,(ErlNifTSDKey key));
+ERL_NIF_API_FUNC_DECL(ErlNifThreadOpts*,enif_thread_opts_create,(char *name));
+ERL_NIF_API_FUNC_DECL(void,enif_thread_opts_destroy,(ErlNifThreadOpts *opts));
+ERL_NIF_API_FUNC_DECL(int,enif_thread_create,(char *name,ErlNifTid *tid,void * (*func)(void *),void *args,ErlNifThreadOpts *opts));
+ERL_NIF_API_FUNC_DECL(ErlNifTid,enif_thread_self,(void));
+ERL_NIF_API_FUNC_DECL(int,enif_equal_tids,(ErlNifTid tid1, ErlNifTid tid2));
+ERL_NIF_API_FUNC_DECL(void,enif_thread_exit,(void *resp));
+ERL_NIF_API_FUNC_DECL(int,enif_thread_join,(ErlNifTid, void **respp));
+
+ERL_NIF_API_FUNC_DECL(void*,enif_realloc,(void* ptr, size_t size));
+ERL_NIF_API_FUNC_DECL(void,enif_system_info,(ErlNifSysInfo *sip, size_t si_size));
+ERL_NIF_API_FUNC_DECL(int,enif_fprintf,(void/* FILE* */ *filep, const char *format, ...));
+ERL_NIF_API_FUNC_DECL(int,enif_inspect_iolist_as_binary,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifBinary* bin));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_sub_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, size_t pos, size_t size));
+ERL_NIF_API_FUNC_DECL(int,enif_get_string,(ErlNifEnv*, ERL_NIF_TERM list, char* buf, unsigned len, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(int,enif_get_atom,(ErlNifEnv*, ERL_NIF_TERM atom, char* buf, unsigned len, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(int,enif_is_fun,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_is_pid,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_is_port,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_get_uint,(ErlNifEnv*, ERL_NIF_TERM term, unsigned* ip));
+ERL_NIF_API_FUNC_DECL(int,enif_get_long,(ErlNifEnv*, ERL_NIF_TERM term, long* ip));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_uint,(ErlNifEnv*, unsigned i));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_long,(ErlNifEnv*, long i));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt));
+ERL_NIF_API_FUNC_DECL(int,enif_is_empty_list,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type,(ErlNifEnv*, const char* module_str, const char* name_str, void (*dtor)(ErlNifEnv*,void *), ErlNifResourceFlags flags, ErlNifResourceFlags* tried));
+ERL_NIF_API_FUNC_DECL(void*,enif_alloc_resource,(ErlNifResourceType* type, size_t size));
+ERL_NIF_API_FUNC_DECL(void,enif_release_resource,(void* obj));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource,(ErlNifEnv*, void* obj));
+ERL_NIF_API_FUNC_DECL(int,enif_get_resource,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp));
+ERL_NIF_API_FUNC_DECL(size_t,enif_sizeof_resource,(void* obj));
+ERL_NIF_API_FUNC_DECL(unsigned char*,enif_make_new_binary,(ErlNifEnv*,size_t size,ERL_NIF_TERM* termp));
+ERL_NIF_API_FUNC_DECL(int,enif_is_list,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_is_tuple,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_get_atom_length,(ErlNifEnv*, ERL_NIF_TERM atom, unsigned* len, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(int,enif_get_list_length,(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_atom_len,(ErlNifEnv* env, const char* name, size_t len));
+ERL_NIF_API_FUNC_DECL(int, enif_make_existing_atom_len,(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM* atom, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_string_len,(ErlNifEnv* env, const char* string, size_t len, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(ErlNifEnv*,enif_alloc_env,(void));
+ERL_NIF_API_FUNC_DECL(void,enif_free_env,(ErlNifEnv* env));
+ERL_NIF_API_FUNC_DECL(void,enif_clear_env,(ErlNifEnv* env));
+ERL_NIF_API_FUNC_DECL(int,enif_send,(ErlNifEnv* env, const ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_copy,(ErlNifEnv* dst_env, ERL_NIF_TERM src_term));
+ERL_NIF_API_FUNC_DECL(ErlNifPid*,enif_self,(ErlNifEnv* caller_env, ErlNifPid* pid));
+ERL_NIF_API_FUNC_DECL(int,enif_get_local_pid,(ErlNifEnv* env, ERL_NIF_TERM, ErlNifPid* pid));
+ERL_NIF_API_FUNC_DECL(void,enif_keep_resource,(void* obj));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource_binary,(ErlNifEnv*,void* obj,const void* data, size_t size));
+#if SIZEOF_LONG != 8
+ERL_NIF_API_FUNC_DECL(int,enif_get_int64,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifSInt64* ip));
+ERL_NIF_API_FUNC_DECL(int,enif_get_uint64,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifUInt64* ip));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int64,(ErlNifEnv*, ErlNifSInt64));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_uint64,(ErlNifEnv*, ErlNifUInt64));
+#endif
+ERL_NIF_API_FUNC_DECL(int,enif_is_exception,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_make_reverse_list,(ErlNifEnv*, ERL_NIF_TERM term, ERL_NIF_TERM *list));
+ERL_NIF_API_FUNC_DECL(int,enif_is_number,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(void*,enif_dlopen,(const char* lib, void (*err_handler)(void*,const char*), void* err_arg));
+ERL_NIF_API_FUNC_DECL(void*,enif_dlsym,(void* handle, const char* symbol, void (*err_handler)(void*,const char*), void* err_arg));
+ERL_NIF_API_FUNC_DECL(int,enif_consume_timeslice,(ErlNifEnv*, int percent));
+
+/*
+** Add new entries here to keep compatibility on Windows!!!
+*/
+#endif
+
+/*
+** Please keep the ERL_NIF_API_FUNC_MACRO list below in the same order
+** as the ERL_NIF_API_FUNC_DECL list above
+*/
+#ifdef ERL_NIF_API_FUNC_MACRO
+#  define enif_priv_data ERL_NIF_API_FUNC_MACRO(enif_priv_data)
+#  define enif_alloc ERL_NIF_API_FUNC_MACRO(enif_alloc)
+#  define enif_free ERL_NIF_API_FUNC_MACRO(enif_free)
+#  define enif_is_atom ERL_NIF_API_FUNC_MACRO(enif_is_atom)
+#  define enif_is_binary ERL_NIF_API_FUNC_MACRO(enif_is_binary)
+#  define enif_is_ref ERL_NIF_API_FUNC_MACRO(enif_is_ref)
+#  define enif_inspect_binary ERL_NIF_API_FUNC_MACRO(enif_inspect_binary)
+#  define enif_alloc_binary ERL_NIF_API_FUNC_MACRO(enif_alloc_binary)
+#  define enif_realloc_binary ERL_NIF_API_FUNC_MACRO(enif_realloc_binary)
+#  define enif_release_binary ERL_NIF_API_FUNC_MACRO(enif_release_binary)
+#  define enif_get_int ERL_NIF_API_FUNC_MACRO(enif_get_int)
+#  define enif_get_ulong ERL_NIF_API_FUNC_MACRO(enif_get_ulong)
+#  define enif_get_double ERL_NIF_API_FUNC_MACRO(enif_get_double)
+#  define enif_get_tuple ERL_NIF_API_FUNC_MACRO(enif_get_tuple)
+#  define enif_get_list_cell ERL_NIF_API_FUNC_MACRO(enif_get_list_cell)
+#  define enif_is_identical ERL_NIF_API_FUNC_MACRO(enif_is_identical)
+#  define enif_compare ERL_NIF_API_FUNC_MACRO(enif_compare)
+
+#  define enif_make_binary ERL_NIF_API_FUNC_MACRO(enif_make_binary)
+#  define enif_make_badarg ERL_NIF_API_FUNC_MACRO(enif_make_badarg)
+#  define enif_make_int ERL_NIF_API_FUNC_MACRO(enif_make_int)
+#  define enif_make_ulong ERL_NIF_API_FUNC_MACRO(enif_make_ulong)
+#  define enif_make_double ERL_NIF_API_FUNC_MACRO(enif_make_double)
+#  define enif_make_atom ERL_NIF_API_FUNC_MACRO(enif_make_atom)
+#  define enif_make_existing_atom ERL_NIF_API_FUNC_MACRO(enif_make_existing_atom)
+#  define enif_make_tuple ERL_NIF_API_FUNC_MACRO(enif_make_tuple)
+#  define enif_make_list ERL_NIF_API_FUNC_MACRO(enif_make_list)
+#  define enif_make_list_cell ERL_NIF_API_FUNC_MACRO(enif_make_list_cell)
+#  define enif_make_string ERL_NIF_API_FUNC_MACRO(enif_make_string)
+#  define enif_make_ref ERL_NIF_API_FUNC_MACRO(enif_make_ref)
+
+#  define enif_mutex_create ERL_NIF_API_FUNC_MACRO(enif_mutex_create) 
+#  define enif_mutex_destroy ERL_NIF_API_FUNC_MACRO(enif_mutex_destroy) 
+#  define enif_mutex_trylock ERL_NIF_API_FUNC_MACRO(enif_mutex_trylock) 
+#  define enif_mutex_lock ERL_NIF_API_FUNC_MACRO(enif_mutex_lock) 
+#  define enif_mutex_unlock ERL_NIF_API_FUNC_MACRO(enif_mutex_unlock) 
+#  define enif_cond_create ERL_NIF_API_FUNC_MACRO(enif_cond_create) 
+#  define enif_cond_destroy ERL_NIF_API_FUNC_MACRO(enif_cond_destroy) 
+#  define enif_cond_signal ERL_NIF_API_FUNC_MACRO(enif_cond_signal) 
+#  define enif_cond_broadcast ERL_NIF_API_FUNC_MACRO(enif_cond_broadcast) 
+#  define enif_cond_wait ERL_NIF_API_FUNC_MACRO(enif_cond_wait) 
+#  define enif_rwlock_create ERL_NIF_API_FUNC_MACRO(enif_rwlock_create) 
+#  define enif_rwlock_destroy ERL_NIF_API_FUNC_MACRO(enif_rwlock_destroy) 
+#  define enif_rwlock_tryrlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_tryrlock) 
+#  define enif_rwlock_rlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_rlock) 
+#  define enif_rwlock_runlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_runlock) 
+#  define enif_rwlock_tryrwlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_tryrwlock) 
+#  define enif_rwlock_rwlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_rwlock) 
+#  define enif_rwlock_rwunlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_rwunlock) 
+#  define enif_tsd_key_create ERL_NIF_API_FUNC_MACRO(enif_tsd_key_create) 
+#  define enif_tsd_key_destroy ERL_NIF_API_FUNC_MACRO(enif_tsd_key_destroy) 
+#  define enif_tsd_set ERL_NIF_API_FUNC_MACRO(enif_tsd_set) 
+#  define enif_tsd_get ERL_NIF_API_FUNC_MACRO(enif_tsd_get) 
+#  define enif_thread_opts_create ERL_NIF_API_FUNC_MACRO(enif_thread_opts_create) 
+#  define enif_thread_opts_destroy ERL_NIF_API_FUNC_MACRO(enif_thread_opts_destroy) 
+#  define enif_thread_create ERL_NIF_API_FUNC_MACRO(enif_thread_create) 
+#  define enif_thread_self ERL_NIF_API_FUNC_MACRO(enif_thread_self) 
+#  define enif_equal_tids ERL_NIF_API_FUNC_MACRO(enif_equal_tids) 
+#  define enif_thread_exit ERL_NIF_API_FUNC_MACRO(enif_thread_exit) 
+#  define enif_thread_join ERL_NIF_API_FUNC_MACRO(enif_thread_join) 
+
+#  define enif_realloc ERL_NIF_API_FUNC_MACRO(enif_realloc) 
+#  define enif_system_info ERL_NIF_API_FUNC_MACRO(enif_system_info) 
+#  define enif_fprintf ERL_NIF_API_FUNC_MACRO(enif_fprintf) 
+#  define enif_inspect_iolist_as_binary ERL_NIF_API_FUNC_MACRO(enif_inspect_iolist_as_binary)
+#  define enif_make_sub_binary ERL_NIF_API_FUNC_MACRO(enif_make_sub_binary)
+#  define enif_get_string ERL_NIF_API_FUNC_MACRO(enif_get_string)
+#  define enif_get_atom ERL_NIF_API_FUNC_MACRO(enif_get_atom)
+#  define enif_is_fun ERL_NIF_API_FUNC_MACRO(enif_is_fun)
+#  define enif_is_pid ERL_NIF_API_FUNC_MACRO(enif_is_pid)
+#  define enif_is_port ERL_NIF_API_FUNC_MACRO(enif_is_port)
+#  define enif_get_uint ERL_NIF_API_FUNC_MACRO(enif_get_uint)
+#  define enif_get_long ERL_NIF_API_FUNC_MACRO(enif_get_long)
+#  define enif_make_uint ERL_NIF_API_FUNC_MACRO(enif_make_uint)
+#  define enif_make_long ERL_NIF_API_FUNC_MACRO(enif_make_long)
+#  define enif_make_tuple_from_array ERL_NIF_API_FUNC_MACRO(enif_make_tuple_from_array)
+#  define enif_make_list_from_array ERL_NIF_API_FUNC_MACRO(enif_make_list_from_array)
+#  define enif_is_empty_list ERL_NIF_API_FUNC_MACRO(enif_is_empty_list)
+#  define enif_open_resource_type ERL_NIF_API_FUNC_MACRO(enif_open_resource_type)
+#  define enif_alloc_resource ERL_NIF_API_FUNC_MACRO(enif_alloc_resource)
+#  define enif_release_resource ERL_NIF_API_FUNC_MACRO(enif_release_resource)
+#  define enif_make_resource ERL_NIF_API_FUNC_MACRO(enif_make_resource)
+#  define enif_get_resource ERL_NIF_API_FUNC_MACRO(enif_get_resource)
+#  define enif_sizeof_resource ERL_NIF_API_FUNC_MACRO(enif_sizeof_resource)
+#  define enif_make_new_binary ERL_NIF_API_FUNC_MACRO(enif_make_new_binary)
+#  define enif_is_list ERL_NIF_API_FUNC_MACRO(enif_is_list)
+#  define enif_is_tuple ERL_NIF_API_FUNC_MACRO(enif_is_tuple)
+#  define enif_get_atom_length ERL_NIF_API_FUNC_MACRO(enif_get_atom_length)
+#  define enif_get_list_length ERL_NIF_API_FUNC_MACRO(enif_get_list_length)
+#  define enif_make_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_atom_len)
+#  define enif_make_existing_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_existing_atom_len)
+#  define enif_make_string_len ERL_NIF_API_FUNC_MACRO(enif_make_string_len)
+#  define enif_alloc_env ERL_NIF_API_FUNC_MACRO(enif_alloc_env)
+#  define enif_free_env ERL_NIF_API_FUNC_MACRO(enif_free_env)
+#  define enif_clear_env ERL_NIF_API_FUNC_MACRO(enif_clear_env)
+#  define enif_send ERL_NIF_API_FUNC_MACRO(enif_send)
+#  define enif_make_copy ERL_NIF_API_FUNC_MACRO(enif_make_copy)
+#  define enif_self ERL_NIF_API_FUNC_MACRO(enif_self)
+#  define enif_get_local_pid ERL_NIF_API_FUNC_MACRO(enif_get_local_pid)
+#  define enif_keep_resource ERL_NIF_API_FUNC_MACRO(enif_keep_resource)
+#  define enif_make_resource_binary ERL_NIF_API_FUNC_MACRO(enif_make_resource_binary)
+#if  SIZEOF_LONG != 8
+#  define enif_get_int64 ERL_NIF_API_FUNC_MACRO(enif_get_int64)
+#  define enif_get_uint64 ERL_NIF_API_FUNC_MACRO(enif_get_uint64)
+#  define enif_make_int64 ERL_NIF_API_FUNC_MACRO(enif_make_int64)
+#  define enif_make_uint64 ERL_NIF_API_FUNC_MACRO(enif_make_uint64)
+#endif
+
+#  define enif_is_exception ERL_NIF_API_FUNC_MACRO(enif_is_exception)
+#  define enif_make_reverse_list ERL_NIF_API_FUNC_MACRO(enif_make_reverse_list)
+#  define enif_is_number ERL_NIF_API_FUNC_MACRO(enif_is_number)
+#  define enif_dlopen ERL_NIF_API_FUNC_MACRO(enif_dlopen)
+#  define enif_dlsym ERL_NIF_API_FUNC_MACRO(enif_dlsym)
+#  define enif_consume_timeslice ERL_NIF_API_FUNC_MACRO(enif_consume_timeslice)
+
+/*
+** Add new entries here
+*/
+#endif
+
+
+#if defined(__GNUC__) && !(defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
+
+/* Inline functions for compile time type checking of arguments to
+   variadic functions.
+*/
+
+#  define ERL_NIF_INLINE __inline__
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple1(ErlNifEnv* env,
+						    ERL_NIF_TERM e1)
+{
+    return enif_make_tuple(env, 1, e1);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple2(ErlNifEnv* env,
+						    ERL_NIF_TERM e1,
+						    ERL_NIF_TERM e2)
+{
+    return enif_make_tuple(env, 2, e1, e2);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple3(ErlNifEnv* env,
+						    ERL_NIF_TERM e1,
+						    ERL_NIF_TERM e2,
+						    ERL_NIF_TERM e3)
+{
+    return enif_make_tuple(env, 3, e1, e2, e3);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple4(ErlNifEnv* env,
+						    ERL_NIF_TERM e1,
+						    ERL_NIF_TERM e2,
+						    ERL_NIF_TERM e3,
+						    ERL_NIF_TERM e4)
+{
+    return enif_make_tuple(env, 4, e1, e2, e3, e4);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple5(ErlNifEnv* env,
+						    ERL_NIF_TERM e1,
+						    ERL_NIF_TERM e2,
+						    ERL_NIF_TERM e3,
+						    ERL_NIF_TERM e4,
+						    ERL_NIF_TERM e5)
+{
+    return enif_make_tuple(env, 5, e1, e2, e3, e4, e5);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple6(ErlNifEnv* env,
+						    ERL_NIF_TERM e1,
+						    ERL_NIF_TERM e2,
+						    ERL_NIF_TERM e3,
+						    ERL_NIF_TERM e4,
+						    ERL_NIF_TERM e5,
+						    ERL_NIF_TERM e6)
+{
+    return enif_make_tuple(env, 6, e1, e2, e3, e4, e5, e6);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple7(ErlNifEnv* env,
+						    ERL_NIF_TERM e1,
+						    ERL_NIF_TERM e2,
+						    ERL_NIF_TERM e3,
+						    ERL_NIF_TERM e4,
+						    ERL_NIF_TERM e5,
+						    ERL_NIF_TERM e6,
+						    ERL_NIF_TERM e7)
+{
+    return enif_make_tuple(env, 7, e1, e2, e3, e4, e5, e6, e7);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple8(ErlNifEnv* env,
+						    ERL_NIF_TERM e1,
+						    ERL_NIF_TERM e2,
+						    ERL_NIF_TERM e3,
+						    ERL_NIF_TERM e4,
+						    ERL_NIF_TERM e5,
+						    ERL_NIF_TERM e6,
+						    ERL_NIF_TERM e7,
+						    ERL_NIF_TERM e8)
+{
+    return enif_make_tuple(env, 8, e1, e2, e3, e4, e5, e6, e7, e8);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple9(ErlNifEnv* env,
+						    ERL_NIF_TERM e1,
+						    ERL_NIF_TERM e2,
+						    ERL_NIF_TERM e3,
+						    ERL_NIF_TERM e4,
+						    ERL_NIF_TERM e5,
+						    ERL_NIF_TERM e6,
+						    ERL_NIF_TERM e7,
+						    ERL_NIF_TERM e8,
+						    ERL_NIF_TERM e9)
+{
+    return enif_make_tuple(env, 9, e1, e2, e3, e4, e5, e6, e7, e8, e9);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list1(ErlNifEnv* env,
+						   ERL_NIF_TERM e1)
+{
+    return enif_make_list(env, 1, e1);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list2(ErlNifEnv* env,
+						   ERL_NIF_TERM e1,
+						   ERL_NIF_TERM e2)
+{
+    return enif_make_list(env, 2, e1, e2);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list3(ErlNifEnv* env,
+						   ERL_NIF_TERM e1,
+						   ERL_NIF_TERM e2,
+						   ERL_NIF_TERM e3)
+{
+    return enif_make_list(env, 3, e1, e2, e3);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list4(ErlNifEnv* env,
+						   ERL_NIF_TERM e1,
+						   ERL_NIF_TERM e2,
+						   ERL_NIF_TERM e3,
+						   ERL_NIF_TERM e4)
+{
+    return enif_make_list(env, 4, e1, e2, e3, e4);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list5(ErlNifEnv* env,
+						   ERL_NIF_TERM e1,
+						   ERL_NIF_TERM e2,
+						   ERL_NIF_TERM e3,
+						   ERL_NIF_TERM e4,
+						   ERL_NIF_TERM e5)
+{
+    return enif_make_list(env, 5, e1, e2, e3, e4, e5);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list6(ErlNifEnv* env,
+						   ERL_NIF_TERM e1,
+						   ERL_NIF_TERM e2,
+						   ERL_NIF_TERM e3,
+						   ERL_NIF_TERM e4,
+						   ERL_NIF_TERM e5,
+						   ERL_NIF_TERM e6)
+{
+    return enif_make_list(env, 6, e1, e2, e3, e4, e5, e6);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list7(ErlNifEnv* env,
+						   ERL_NIF_TERM e1,
+						   ERL_NIF_TERM e2,
+						   ERL_NIF_TERM e3,
+						   ERL_NIF_TERM e4,
+						   ERL_NIF_TERM e5,
+						   ERL_NIF_TERM e6,
+						   ERL_NIF_TERM e7)
+{
+    return enif_make_list(env, 7, e1, e2, e3, e4, e5, e6, e7);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list8(ErlNifEnv* env,
+						   ERL_NIF_TERM e1,
+						   ERL_NIF_TERM e2,
+						   ERL_NIF_TERM e3,
+						   ERL_NIF_TERM e4,
+						   ERL_NIF_TERM e5,
+						   ERL_NIF_TERM e6,
+						   ERL_NIF_TERM e7,
+						   ERL_NIF_TERM e8)
+{
+    return enif_make_list(env, 8, e1, e2, e3, e4, e5, e6, e7, e8);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list9(ErlNifEnv* env,
+						   ERL_NIF_TERM e1,
+						   ERL_NIF_TERM e2,
+						   ERL_NIF_TERM e3,
+						   ERL_NIF_TERM e4,
+						   ERL_NIF_TERM e5,
+						   ERL_NIF_TERM e6,
+						   ERL_NIF_TERM e7,
+						   ERL_NIF_TERM e8,
+						   ERL_NIF_TERM e9)
+{
+    return enif_make_list(env, 9, e1, e2, e3, e4, e5, e6, e7, e8, e9);
+}
+
+#  undef ERL_NIF_INLINE
+
+#else /* fallback with macros */
+
+#ifndef enif_make_list1
+#  define enif_make_list1(ENV,E1) enif_make_list(ENV,1,E1)
+#  define enif_make_list2(ENV,E1,E2) enif_make_list(ENV,2,E1,E2)
+#  define enif_make_list3(ENV,E1,E2,E3) enif_make_list(ENV,3,E1,E2,E3)
+#  define enif_make_list4(ENV,E1,E2,E3,E4) enif_make_list(ENV,4,E1,E2,E3,E4)
+#  define enif_make_list5(ENV,E1,E2,E3,E4,E5) enif_make_list(ENV,5,E1,E2,E3,E4,E5)
+#  define enif_make_list6(ENV,E1,E2,E3,E4,E5,E6) enif_make_list(ENV,6,E1,E2,E3,E4,E5,E6)
+#  define enif_make_list7(ENV,E1,E2,E3,E4,E5,E6,E7) enif_make_list(ENV,7,E1,E2,E3,E4,E5,E6,E7)
+#  define enif_make_list8(ENV,E1,E2,E3,E4,E5,E6,E7,E8) enif_make_list(ENV,8,E1,E2,E3,E4,E5,E6,E7,E8)
+#  define enif_make_list9(ENV,E1,E2,E3,E4,E5,E6,E7,E8,E9) enif_make_list(ENV,9,E1,E2,E3,E4,E5,E6,E7,E8,E9)
+#  define enif_make_tuple1(ENV,E1) enif_make_tuple(ENV,1,E1)
+#  define enif_make_tuple2(ENV,E1,E2) enif_make_tuple(ENV,2,E1,E2)
+#  define enif_make_tuple3(ENV,E1,E2,E3) enif_make_tuple(ENV,3,E1,E2,E3)
+#  define enif_make_tuple4(ENV,E1,E2,E3,E4) enif_make_tuple(ENV,4,E1,E2,E3,E4)
+#  define enif_make_tuple5(ENV,E1,E2,E3,E4,E5) enif_make_tuple(ENV,5,E1,E2,E3,E4,E5)
+#  define enif_make_tuple6(ENV,E1,E2,E3,E4,E5,E6) enif_make_tuple(ENV,6,E1,E2,E3,E4,E5,E6)
+#  define enif_make_tuple7(ENV,E1,E2,E3,E4,E5,E6,E7) enif_make_tuple(ENV,7,E1,E2,E3,E4,E5,E6,E7)
+#  define enif_make_tuple8(ENV,E1,E2,E3,E4,E5,E6,E7,E8) enif_make_tuple(ENV,8,E1,E2,E3,E4,E5,E6,E7,E8)
+#  define enif_make_tuple9(ENV,E1,E2,E3,E4,E5,E6,E7,E8,E9) enif_make_tuple(ENV,9,E1,E2,E3,E4,E5,E6,E7,E8,E9)
+#endif
+
+#endif /* __GNUC__ && !WIN32 */
+
+#ifndef enif_make_pid
+
+#  define enif_make_pid(ENV, PID) ((const ERL_NIF_TERM)((PID)->pid))
+
+#if SIZEOF_LONG == 8
+#  define enif_get_int64 enif_get_long
+#  define enif_get_uint64 enif_get_ulong
+#  define enif_make_int64 enif_make_long
+#  define enif_make_uint64 enif_make_ulong
+#endif
+
+#endif
+
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.1.2_4.c b/erts/emulator/test/nif_SUITE_data/nif_mod.1.2_4.c
new file mode 100644
index 0000000000..6d28dbb8ba
--- /dev/null
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.1.2_4.c
@@ -0,0 +1,4 @@
+#include "nif_api_2_4/erl_nif.h"
+
+#define NIF_LIB_VER 1
+#include "nif_mod.c"
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.2.2_4.c b/erts/emulator/test/nif_SUITE_data/nif_mod.2.2_4.c
new file mode 100644
index 0000000000..628fd42b52
--- /dev/null
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.2.2_4.c
@@ -0,0 +1,4 @@
+#include "nif_api_2_4/erl_nif.h"
+
+#define NIF_LIB_VER 2
+#include "nif_mod.c"
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.3.2_4.c b/erts/emulator/test/nif_SUITE_data/nif_mod.3.2_4.c
new file mode 100644
index 0000000000..bdbe8cf381
--- /dev/null
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.3.2_4.c
@@ -0,0 +1,4 @@
+#include "nif_api_2_4/erl_nif.h"
+
+#define NIF_LIB_VER 3
+#include "nif_mod.c"
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.c b/erts/emulator/test/nif_SUITE_data/nif_mod.c
index 4e94e7901c..98eac821b6 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_mod.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.c
@@ -237,6 +237,14 @@ static ERL_NIF_TERM lib_version(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg
     return enif_make_int(env, NIF_LIB_VER);
 }
 
+static ERL_NIF_TERM nif_api_version(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    /*ADD_CALL("nif_api_version");*/
+    return enif_make_tuple2(env,
+			    enif_make_int(env, ERL_NIF_MAJOR_VERSION),
+			    enif_make_int(env, ERL_NIF_MINOR_VERSION));
+}
+
 static ERL_NIF_TERM get_priv_data_ptr(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     ADD_CALL("get_priv_data_ptr");
@@ -279,6 +287,7 @@ static ERL_NIF_TERM get_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
 static ErlNifFunc nif_funcs[] =
 {
     {"lib_version", 0, lib_version},
+    {"nif_api_version", 0, nif_api_version},
     {"get_priv_data_ptr", 0, get_priv_data_ptr},
     {"make_new_resource", 2, make_new_resource},
     {"get_resource", 2, get_resource}
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.erl b/erts/emulator/test/nif_SUITE_data/nif_mod.erl
index 1fcc33faa4..8019cfcf82 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_mod.erl
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.erl
@@ -22,7 +22,7 @@
 
 -include_lib("common_test/include/ct.hrl").
 
--export([load_nif_lib/2, load_nif_lib/3, start/0, lib_version/0, call_history/0,
+-export([load_nif_lib/2, load_nif_lib/3, start/0, lib_version/0,
 	 get_priv_data_ptr/0, make_new_resource/2, get_resource/2]).
 
 -export([loop/0, upgrade/1]).
@@ -35,21 +35,32 @@
 on_load() ->
     [{data_dir, Path}] = ets:lookup(nif_SUITE, data_dir),
     [{lib_version, Ver}] = ets:lookup(nif_SUITE, lib_version),
-    erlang:load_nif(filename:join(Path,libname(Ver)), []).
+    [{nif_api_version, API}] = ets:lookup(nif_SUITE, nif_api_version),
+    R = erlang:load_nif(filename:join(Path,libname(Ver,API)), []),
+    check_api_version(R, API).
 
 -endif.
 
+check_api_version(Err, _) when Err =/= ok -> Err;
+check_api_version(ok, []) -> ok;
+check_api_version(ok, [$., MajC, $_ | MinS]) ->
+    {Maj, Min} = {list_to_integer([MajC]), list_to_integer(MinS)},
+    {Maj, Min} = nif_api_version(),
+    ok.
+
 load_nif_lib(Config, Ver) ->
     load_nif_lib(Config, Ver, []).
 
 load_nif_lib(Config, Ver, LoadInfo) ->
     Path = proplists:get_value(data_dir, Config),
-    erlang:load_nif(filename:join(Path,libname(Ver)), LoadInfo).
-
-libname(no_init) -> libname(3);
-libname(Ver) when is_integer(Ver) ->
-    "nif_mod." ++ integer_to_list(Ver).
+    API = proplists:get_value(nif_api_version, Config, ""),
+    R = erlang:load_nif(filename:join(Path,libname(Ver,API)), LoadInfo),
+    check_api_version(R, API).
 
+libname(no_init,API) -> libname(3,API);
+libname(Ver,API) when is_integer(Ver) ->
+    "nif_mod." ++ integer_to_list(Ver) ++ API.
+    
 start() ->
     spawn_opt(?MODULE,loop,[],
 	      [link, monitor]).
@@ -72,7 +83,9 @@ upgrade(Pid) ->
 lib_version() ->  % NIF
     undefined.
 
-call_history() -> ?nif_stub.    
+nif_api_version() -> %NIF
+    {undefined,undefined}.
+
 get_priv_data_ptr() -> ?nif_stub.
 make_new_resource(_,_) -> ?nif_stub.
 get_resource(_,_) -> ?nif_stub.
-- 
cgit v1.2.3


From bc1128ca7bb6ab06a2f15e94c5c9728fc6ba27a6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 21 Nov 2016 17:28:15 +0100
Subject: erts: Use test groups to repeat for different APIs

---
 erts/emulator/test/nif_SUITE.erl | 56 ++++++++++++++++------------------------
 1 file changed, 22 insertions(+), 34 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 99213a6888..7cac3bd8b3 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -26,7 +26,8 @@
 
 -include_lib("common_test/include/ct.hrl").
 
--export([all/0, suite/0,
+-export([all/0, suite/0, groups/0,
+         init_per_group/2, end_per_group/2,
 	 init_per_testcase/2, end_per_testcase/2,
          basic/1, reload_error/1, upgrade/1, heap_frag/1,
          t_on_load/1,
@@ -53,28 +54,20 @@
 
 -export([many_args_100/100]).
 
-
-%% -export([lib_version/0,call_history/0,hold_nif_mod_priv_data/1,nif_mod_call_history/0,
-%% 	 list_seq/1,type_test/0,tuple_2_list/1,is_identical/2,compare/2,
-%% 	 clone_bin/1,make_sub_bin/3,string_to_bin/2,atom_to_bin/2,macros/1,
-%% 	 tuple_2_list_and_tuple/1,iolist_2_bin/1,get_resource_type/1,alloc_resource/2,
-%% 	 make_resource/1,get_resource/2,release_resource/1,last_resource_dtor_call/0, suite/0,
-%% 	 make_new_resource/2,make_new_resource_binary/1,send_list_seq/2,send_new_blob/2,
-%% 	 alloc_msgenv/0,clear_msgenv/1,grow_blob/2,send_blob/2,send_blob_thread/3,
-%% 	 join_send_thread/1]).
-
-
 -define(nif_stub,nif_stub_error(?LINE)).
 
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
-all() -> 
-    [basic, reload_error, upgrade, heap_frag, types, many_args,
-     t_on_load,
+all() ->
+    [basic]
+        ++
+    [{group, G} || G <- api_groups()]
+        ++
+    [reload_error, heap_frag, types, many_args,
      hipe,
      binaries, get_string, get_atom, maps, api_macros, from_array,
      iolist_as_binary, resource, resource_binary,
-     resource_takeover, threading, send, send2, send3,
+     threading, send, send2, send3,
      send_threaded, neg, is_checks, get_length, make_atom,
      make_string,reverse_list_test,
      otp_9828,
@@ -87,6 +80,19 @@ all() ->
      nif_port_command,
      nif_snprintf].
 
+groups() ->
+    [{G, [], api_repeaters()} || G <- api_groups()].
+
+api_groups() -> [api_latest, api_2_4].
+
+api_repeaters() -> [upgrade, resource_takeover, t_on_load].
+
+init_per_group(api_latest, Config) -> Config;
+init_per_group(api_2_4, Config) ->
+    [{nif_api_version, ".2_4"} | Config].
+
+end_per_group(_,_) -> ok.
+
 init_per_testcase(t_on_load, Config) ->
     ets:new(nif_SUITE, [named_table]),
     Config;
@@ -156,12 +162,6 @@ reload_error(Config) when is_list(Config) ->
 
 %% Test upgrade callback in nif lib
 upgrade(Config) when is_list(Config) ->
-    [begin io:format("Do test for API version \"~p\"", [API]),
-           upgrade_do([{nif_api_version,API}|Config])
-     end
-     || API <- ["", ".2_4"]].
-
-upgrade_do(Config) ->
     TmpMem = tmpmem(),
     ensure_lib_loaded(Config),
 
@@ -301,12 +301,6 @@ upgrade_do(Config) ->
 
 %% Test loading/upgrade in on_load
 t_on_load(Config) when is_list(Config) ->
-    [begin io:format("Do test for API version \"~p\"", [API]),
-           t_on_load_do([{nif_api_version,API}|Config])
-     end
-     || API <- ["", ".2_4"]].
-
-t_on_load_do(Config) ->
     TmpMem = tmpmem(),
     ensure_lib_loaded(Config),
 
@@ -862,12 +856,6 @@ resource_binary_do() ->
 
 %% Test resource takeover by module upgrade
 resource_takeover(Config) when is_list(Config) ->    
-    [begin io:format("Do test for API version \"~p\"", [API]),
-           resource_takeover_do([{nif_api_version,API}|Config])
-     end
-     || API <- ["", ".2_4"]].
-
-resource_takeover_do(Config) ->
     TmpMem = tmpmem(),
     ensure_lib_loaded(Config),
 
-- 
cgit v1.2.3


From daa93a64340ba5e3118c2550416e2f324e049531 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 21 Nov 2016 18:35:23 +0100
Subject: erts: Change nif_SUITE to use binaries for raw pointers

---
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 15 +++++++++------
 erts/emulator/test/nif_SUITE_data/nif_mod.c   |  6 +++++-
 2 files changed, 14 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index dc6d8cea76..2c93891852 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -104,21 +104,24 @@ struct binary_resource {
 
 static int get_pointer(ErlNifEnv* env, ERL_NIF_TERM term, void** pp)
 {
-    ErlNifUInt64 i64;
-    int r = enif_get_uint64(env, term, &i64);
+    ErlNifBinary bin;
+    int r = enif_inspect_binary(env, term, &bin);
     if (r) {
-	*pp = (void*)i64;
+	*pp = *(void**)bin.data;
     }
     return r;
 }
 
 static ERL_NIF_TERM make_pointer(ErlNifEnv* env, void* p)
 {
-    ErlNifUInt64 i64 = (ErlNifUInt64) p;
-    return enif_make_uint64(env, i64);
+    void** bin_data;
+    ERL_NIF_TERM res;
+    ADD_CALL("get_priv_data_ptr");
+    bin_data = (void**)enif_make_new_binary(env, sizeof(void*), &res);
+    *bin_data = p;
+    return res;
 }
 
-
 static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
 {
     PrivData* data = enif_alloc(sizeof(PrivData));
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.c b/erts/emulator/test/nif_SUITE_data/nif_mod.c
index 98eac821b6..e6106b6036 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_mod.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.c
@@ -247,8 +247,12 @@ static ERL_NIF_TERM nif_api_version(ErlNifEnv* env, int argc, const ERL_NIF_TERM
 
 static ERL_NIF_TERM get_priv_data_ptr(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
+    NifModPrivData** bin_data;
+    ERL_NIF_TERM res;
     ADD_CALL("get_priv_data_ptr");
-    return enif_make_uint64(env, (ErlNifUInt64)priv_data(env));
+    bin_data = (NifModPrivData**)enif_make_new_binary(env, sizeof(void*), &res);
+    *bin_data = priv_data(env);
+    return res;
 }
 
 static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
-- 
cgit v1.2.3


From 99900a1e23adb08980c32e09846c281b698d8340 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 21 Nov 2016 18:32:57 +0100
Subject: erts: Add tests of nif API 2.0

---
 erts/emulator/test/nif_SUITE.erl                   |   6 +-
 erts/emulator/test/nif_SUITE_data/Makefile.src     |   3 +
 .../test/nif_SUITE_data/nif_api_2_0/README         |   5 +
 .../test/nif_SUITE_data/nif_api_2_0/erl_drv_nif.h  |  48 ++++
 .../test/nif_SUITE_data/nif_api_2_0/erl_nif.h      | 206 +++++++++++++++++
 .../nif_SUITE_data/nif_api_2_0/erl_nif_api_funcs.h | 257 +++++++++++++++++++++
 erts/emulator/test/nif_SUITE_data/nif_mod.1.2_0.c  |   4 +
 erts/emulator/test/nif_SUITE_data/nif_mod.2.2_0.c  |   4 +
 erts/emulator/test/nif_SUITE_data/nif_mod.3.2_0.c  |   4 +
 9 files changed, 535 insertions(+), 2 deletions(-)
 create mode 100644 erts/emulator/test/nif_SUITE_data/nif_api_2_0/README
 create mode 100644 erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_drv_nif.h
 create mode 100644 erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif.h
 create mode 100644 erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif_api_funcs.h
 create mode 100644 erts/emulator/test/nif_SUITE_data/nif_mod.1.2_0.c
 create mode 100644 erts/emulator/test/nif_SUITE_data/nif_mod.2.2_0.c
 create mode 100644 erts/emulator/test/nif_SUITE_data/nif_mod.3.2_0.c

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 7cac3bd8b3..01a5e84dc9 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -83,13 +83,15 @@ all() ->
 groups() ->
     [{G, [], api_repeaters()} || G <- api_groups()].
 
-api_groups() -> [api_latest, api_2_4].
+api_groups() -> [api_latest, api_2_4, api_2_0].
 
 api_repeaters() -> [upgrade, resource_takeover, t_on_load].
 
 init_per_group(api_latest, Config) -> Config;
 init_per_group(api_2_4, Config) ->
-    [{nif_api_version, ".2_4"} | Config].
+    [{nif_api_version, ".2_4"} | Config];
+init_per_group(api_2_0, Config) ->
+    [{nif_api_version, ".2_0"} | Config].
 
 end_per_group(_,_) -> ok.
 
diff --git a/erts/emulator/test/nif_SUITE_data/Makefile.src b/erts/emulator/test/nif_SUITE_data/Makefile.src
index cbe843b120..de06026780 100644
--- a/erts/emulator/test/nif_SUITE_data/Makefile.src
+++ b/erts/emulator/test/nif_SUITE_data/Makefile.src
@@ -3,6 +3,9 @@ NIF_LIBS = nif_SUITE.1@dll@ \
            nif_mod.1@dll@ \
            nif_mod.2@dll@ \
            nif_mod.3@dll@ \
+	   nif_mod.1.2_0@dll@ \
+	   nif_mod.2.2_0@dll@ \
+	   nif_mod.3.2_0@dll@ \
 	   nif_mod.1.2_4@dll@ \
 	   nif_mod.2.2_4@dll@ \
 	   nif_mod.3.2_4@dll@
diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_0/README b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/README
new file mode 100644
index 0000000000..a6ed36f634
--- /dev/null
+++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/README
@@ -0,0 +1,5 @@
+These are old genuine header files
+checked out from tag OTP_R14A c1e94fa9a6fe4ae717d35.
+
+I choose this API version (2.0) to test as it's
+before the addition of vm_variant in ErlNifEntry.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_drv_nif.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_drv_nif.h
new file mode 100644
index 0000000000..ea013a49a3
--- /dev/null
+++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_drv_nif.h
@@ -0,0 +1,48 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Common structures for both erl_driver.h and erl_nif.h
+ */
+
+#ifndef __ERL_DRV_NIF_H__
+#define __ERL_DRV_NIF_H__
+
+typedef struct {
+    int driver_major_version;
+    int driver_minor_version;
+    char *erts_version;
+    char *otp_release;
+    int thread_support;
+    int smp_support;
+    int async_threads;
+    int scheduler_threads;
+    int nif_major_version;
+    int nif_minor_version;
+}  ErlDrvSysInfo;
+
+typedef struct {
+    int suggested_stack_size;
+} ErlDrvThreadOpts;
+
+#endif  /* __ERL_DRV_NIF_H__ */
+
+
+
+
diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif.h
new file mode 100644
index 0000000000..936f03bce1
--- /dev/null
+++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif.h
@@ -0,0 +1,206 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/* Include file for writers of Native Implemented Functions. 
+*/
+
+#ifndef __ERL_NIF_H__
+#define __ERL_NIF_H__
+
+
+#include "erl_drv_nif.h"
+
+/* Version history:
+** 0.1: R13B03
+** 1.0: R13B04
+** 2.0: R14A
+*/
+#define ERL_NIF_MAJOR_VERSION 2
+#define ERL_NIF_MINOR_VERSION 0
+
+#include <stdlib.h>
+
+#ifdef SIZEOF_CHAR
+#  define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR
+#  undef SIZEOF_CHAR
+#endif
+#ifdef SIZEOF_SHORT
+#  define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT
+#  undef SIZEOF_SHORT
+#endif
+#ifdef SIZEOF_INT
+#  define SIZEOF_INT_SAVED__ SIZEOF_INT
+#  undef SIZEOF_INT
+#endif
+#ifdef SIZEOF_LONG
+#  define SIZEOF_LONG_SAVED__ SIZEOF_LONG
+#  undef SIZEOF_LONG
+#endif
+#ifdef SIZEOF_LONG_LONG
+#  define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG
+#  undef SIZEOF_LONG_LONG
+#endif
+#ifdef HALFWORD_HEAP_EMULATOR
+#  define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR
+#  undef HALFWORD_HEAP_EMULATOR
+#endif
+#include "erl_int_sizes_config.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef HALFWORD_HEAP_EMULATOR
+typedef unsigned int ERL_NIF_TERM;
+#else
+typedef unsigned long ERL_NIF_TERM;
+#endif
+
+struct enif_environment_t;
+typedef struct enif_environment_t ErlNifEnv;
+
+typedef struct
+{
+    const char* name;
+    unsigned arity;
+    ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+}ErlNifFunc;
+
+typedef struct enif_entry_t
+{
+    int major;
+    int minor;
+    const char* name;
+    int num_of_funcs;
+    ErlNifFunc* funcs;
+    int  (*load)   (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info);
+    int  (*reload) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info);
+    int  (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info);
+    void (*unload) (ErlNifEnv*, void* priv_data);    
+}ErlNifEntry;
+
+
+
+typedef struct
+{
+    size_t size;
+    unsigned char* data;
+
+    /* Internals (avert your eyes) */
+    ERL_NIF_TERM bin_term;
+    void* ref_bin;
+}ErlNifBinary;
+
+typedef struct enif_resource_type_t ErlNifResourceType;
+typedef void ErlNifResourceDtor(ErlNifEnv*, void*);
+typedef enum
+{
+    ERL_NIF_RT_CREATE = 1,
+    ERL_NIF_RT_TAKEOVER = 2
+}ErlNifResourceFlags;
+
+typedef enum
+{
+    ERL_NIF_LATIN1 = 1
+}ErlNifCharEncoding;
+
+typedef struct
+{
+    ERL_NIF_TERM pid;  /* internal, may change */
+}ErlNifPid;
+
+typedef ErlDrvSysInfo ErlNifSysInfo;
+
+typedef struct ErlDrvTid_ *ErlNifTid;
+typedef struct ErlDrvMutex_ ErlNifMutex;
+typedef struct ErlDrvCond_ ErlNifCond;
+typedef struct ErlDrvRWLock_ ErlNifRWLock;
+typedef int ErlNifTSDKey;
+
+typedef ErlDrvThreadOpts ErlNifThreadOpts;
+
+#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
+#  define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS
+typedef struct {
+#  include "erl_nif_api_funcs.h"
+} TWinDynNifCallbacks;
+extern TWinDynNifCallbacks WinDynNifCallbacks;
+#  undef ERL_NIF_API_FUNC_DECL
+#endif
+
+#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) && !defined(STATIC_ERLANG_DRIVER)
+#  define ERL_NIF_API_FUNC_MACRO(NAME) (WinDynNifCallbacks.NAME)
+#  include "erl_nif_api_funcs.h"
+/* note that we have to keep ERL_NIF_API_FUNC_MACRO defined */
+
+#else /* non windows or included from emulator itself */
+
+#  define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) extern RET_TYPE NAME ARGS
+#  include "erl_nif_api_funcs.h"
+#  undef ERL_NIF_API_FUNC_DECL
+#endif
+
+
+#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
+#  define ERL_NIF_INIT_GLOB TWinDynNifCallbacks WinDynNifCallbacks;
+#  define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks)
+#  define ERL_NIF_INIT_BODY memcpy(&WinDynNifCallbacks,callbacks,sizeof(TWinDynNifCallbacks))
+#else 
+#  define ERL_NIF_INIT_GLOB
+#  define ERL_NIF_INIT_BODY
+#  if defined(VXWORKS)
+#    define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* MODNAME  ## _init(void)
+#  else
+#    define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* nif_init(void)
+#  endif
+#endif
+
+
+#ifdef __cplusplus
+}
+#  define ERL_NIF_INIT_PROLOGUE extern "C" {
+#  define ERL_NIF_INIT_EPILOGUE }
+#else
+#  define ERL_NIF_INIT_PROLOGUE
+#  define ERL_NIF_INIT_EPILOGUE
+#endif
+
+
+#define ERL_NIF_INIT(NAME, FUNCS, LOAD, RELOAD, UPGRADE, UNLOAD) \
+ERL_NIF_INIT_PROLOGUE                   \
+ERL_NIF_INIT_GLOB                       \
+ERL_NIF_INIT_DECL(NAME)			\
+{					\
+    static ErlNifEntry entry = 		\
+    {					\
+	ERL_NIF_MAJOR_VERSION,		\
+	ERL_NIF_MINOR_VERSION,		\
+	#NAME,				\
+	sizeof(FUNCS) / sizeof(*FUNCS),	\
+	FUNCS,				\
+	LOAD, RELOAD, UPGRADE, UNLOAD	\
+    };                                  \
+    ERL_NIF_INIT_BODY;                  \
+    return &entry;			\
+}                                       \
+ERL_NIF_INIT_EPILOGUE
+
+
+#endif /* __ERL_NIF_H__ */
+
diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif_api_funcs.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif_api_funcs.h
new file mode 100644
index 0000000000..ef4e9580b0
--- /dev/null
+++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif_api_funcs.h
@@ -0,0 +1,257 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#if !defined(ERL_NIF_API_FUNC_DECL) && !defined(ERL_NIF_API_FUNC_MACRO)
+#  error This file should not be included directly
+#endif
+
+#ifdef ERL_NIF_API_FUNC_DECL
+ERL_NIF_API_FUNC_DECL(void*,enif_priv_data,(ErlNifEnv*));
+ERL_NIF_API_FUNC_DECL(void*,enif_alloc,(size_t size));
+ERL_NIF_API_FUNC_DECL(void,enif_free,(void* ptr));
+ERL_NIF_API_FUNC_DECL(int,enif_is_atom,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_is_binary,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_is_ref,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_inspect_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, ErlNifBinary* bin));
+ERL_NIF_API_FUNC_DECL(int,enif_alloc_binary,(size_t size, ErlNifBinary* bin));
+ERL_NIF_API_FUNC_DECL(int,enif_realloc_binary,(ErlNifBinary* bin, size_t size));
+ERL_NIF_API_FUNC_DECL(void,enif_release_binary,(ErlNifBinary* bin));
+ERL_NIF_API_FUNC_DECL(int,enif_get_int,(ErlNifEnv*, ERL_NIF_TERM term, int* ip));
+ERL_NIF_API_FUNC_DECL(int,enif_get_ulong,(ErlNifEnv*, ERL_NIF_TERM term, unsigned long* ip));
+ERL_NIF_API_FUNC_DECL(int,enif_get_double,(ErlNifEnv*, ERL_NIF_TERM term, double* dp));
+ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM* head, ERL_NIF_TERM* tail));
+ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array));
+ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
+ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_ulong,(ErlNifEnv* env, unsigned long i));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_double,(ErlNifEnv* env, double d));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_atom,(ErlNifEnv* env, const char* name));
+ERL_NIF_API_FUNC_DECL(int,enif_make_existing_atom,(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple,(ErlNifEnv* env, unsigned cnt, ...));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list,(ErlNifEnv* env, unsigned cnt, ...));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_cell,(ErlNifEnv* env, ERL_NIF_TERM car, ERL_NIF_TERM cdr));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_string,(ErlNifEnv* env, const char* string, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_ref,(ErlNifEnv* env));
+
+ERL_NIF_API_FUNC_DECL(ErlNifMutex*,enif_mutex_create,(char *name));
+ERL_NIF_API_FUNC_DECL(void,enif_mutex_destroy,(ErlNifMutex *mtx));
+ERL_NIF_API_FUNC_DECL(int,enif_mutex_trylock,(ErlNifMutex *mtx));
+ERL_NIF_API_FUNC_DECL(void,enif_mutex_lock,(ErlNifMutex *mtx));
+ERL_NIF_API_FUNC_DECL(void,enif_mutex_unlock,(ErlNifMutex *mtx));
+ERL_NIF_API_FUNC_DECL(ErlNifCond*,enif_cond_create,(char *name));
+ERL_NIF_API_FUNC_DECL(void,enif_cond_destroy,(ErlNifCond *cnd));
+ERL_NIF_API_FUNC_DECL(void,enif_cond_signal,(ErlNifCond *cnd));
+ERL_NIF_API_FUNC_DECL(void,enif_cond_broadcast,(ErlNifCond *cnd));
+ERL_NIF_API_FUNC_DECL(void,enif_cond_wait,(ErlNifCond *cnd, ErlNifMutex *mtx));
+ERL_NIF_API_FUNC_DECL(ErlNifRWLock*,enif_rwlock_create,(char *name));
+ERL_NIF_API_FUNC_DECL(void,enif_rwlock_destroy,(ErlNifRWLock *rwlck));
+ERL_NIF_API_FUNC_DECL(int,enif_rwlock_tryrlock,(ErlNifRWLock *rwlck));
+ERL_NIF_API_FUNC_DECL(void,enif_rwlock_rlock,(ErlNifRWLock *rwlck));
+ERL_NIF_API_FUNC_DECL(void,enif_rwlock_runlock,(ErlNifRWLock *rwlck));
+ERL_NIF_API_FUNC_DECL(int,enif_rwlock_tryrwlock,(ErlNifRWLock *rwlck));
+ERL_NIF_API_FUNC_DECL(void,enif_rwlock_rwlock,(ErlNifRWLock *rwlck));
+ERL_NIF_API_FUNC_DECL(void,enif_rwlock_rwunlock,(ErlNifRWLock *rwlck));
+ERL_NIF_API_FUNC_DECL(int,enif_tsd_key_create,(char *name, ErlNifTSDKey *key));
+ERL_NIF_API_FUNC_DECL(void,enif_tsd_key_destroy,(ErlNifTSDKey key));
+ERL_NIF_API_FUNC_DECL(void,enif_tsd_set,(ErlNifTSDKey key, void *data));
+ERL_NIF_API_FUNC_DECL(void*,enif_tsd_get,(ErlNifTSDKey key));
+ERL_NIF_API_FUNC_DECL(ErlNifThreadOpts*,enif_thread_opts_create,(char *name));
+ERL_NIF_API_FUNC_DECL(void,enif_thread_opts_destroy,(ErlNifThreadOpts *opts));
+ERL_NIF_API_FUNC_DECL(int,enif_thread_create,(char *name,ErlNifTid *tid,void * (*func)(void *),void *args,ErlNifThreadOpts *opts));
+ERL_NIF_API_FUNC_DECL(ErlNifTid,enif_thread_self,(void));
+ERL_NIF_API_FUNC_DECL(int,enif_equal_tids,(ErlNifTid tid1, ErlNifTid tid2));
+ERL_NIF_API_FUNC_DECL(void,enif_thread_exit,(void *resp));
+ERL_NIF_API_FUNC_DECL(int,enif_thread_join,(ErlNifTid, void **respp));
+
+ERL_NIF_API_FUNC_DECL(void*,enif_realloc,(void* ptr, size_t size));
+ERL_NIF_API_FUNC_DECL(void,enif_system_info,(ErlNifSysInfo *sip, size_t si_size));
+ERL_NIF_API_FUNC_DECL(int,enif_fprintf,(void/* FILE* */ *filep, const char *format, ...));
+ERL_NIF_API_FUNC_DECL(int,enif_inspect_iolist_as_binary,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifBinary* bin));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_sub_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, size_t pos, size_t size));
+ERL_NIF_API_FUNC_DECL(int,enif_get_string,(ErlNifEnv*, ERL_NIF_TERM list, char* buf, unsigned len, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(int,enif_get_atom,(ErlNifEnv*, ERL_NIF_TERM atom, char* buf, unsigned len, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(int,enif_is_fun,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_is_pid,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_is_port,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_get_uint,(ErlNifEnv*, ERL_NIF_TERM term, unsigned* ip));
+ERL_NIF_API_FUNC_DECL(int,enif_get_long,(ErlNifEnv*, ERL_NIF_TERM term, long* ip));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_uint,(ErlNifEnv*, unsigned i));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_long,(ErlNifEnv*, long i));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt));
+ERL_NIF_API_FUNC_DECL(int,enif_is_empty_list,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type,(ErlNifEnv*, const char* module_str, const char* name_str, void (*dtor)(ErlNifEnv*,void *), ErlNifResourceFlags flags, ErlNifResourceFlags* tried));
+ERL_NIF_API_FUNC_DECL(void*,enif_alloc_resource,(ErlNifResourceType* type, size_t size));
+ERL_NIF_API_FUNC_DECL(void,enif_release_resource,(void* obj));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource,(ErlNifEnv*, void* obj));
+ERL_NIF_API_FUNC_DECL(int,enif_get_resource,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp));
+ERL_NIF_API_FUNC_DECL(size_t,enif_sizeof_resource,(void* obj));
+ERL_NIF_API_FUNC_DECL(unsigned char*,enif_make_new_binary,(ErlNifEnv*,size_t size,ERL_NIF_TERM* termp));
+ERL_NIF_API_FUNC_DECL(int,enif_is_list,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_is_tuple,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_get_atom_length,(ErlNifEnv*, ERL_NIF_TERM atom, unsigned* len, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(int,enif_get_list_length,(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_atom_len,(ErlNifEnv* env, const char* name, size_t len));
+ERL_NIF_API_FUNC_DECL(int, enif_make_existing_atom_len,(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM* atom, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_string_len,(ErlNifEnv* env, const char* string, size_t len, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(ErlNifEnv*,enif_alloc_env,(void));
+ERL_NIF_API_FUNC_DECL(void,enif_free_env,(ErlNifEnv* env));
+ERL_NIF_API_FUNC_DECL(void,enif_clear_env,(ErlNifEnv* env));
+ERL_NIF_API_FUNC_DECL(int,enif_send,(ErlNifEnv* env, const ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_copy,(ErlNifEnv* dst_env, ERL_NIF_TERM src_term));
+ERL_NIF_API_FUNC_DECL(ErlNifPid*,enif_self,(ErlNifEnv* caller_env, ErlNifPid* pid));
+ERL_NIF_API_FUNC_DECL(int,enif_get_local_pid,(ErlNifEnv* env, ERL_NIF_TERM, ErlNifPid* pid));
+ERL_NIF_API_FUNC_DECL(void,enif_keep_resource,(void* obj));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource_binary,(ErlNifEnv*,void* obj,const void* data, size_t size));
+
+/*
+** Add last to keep compatibility on Windows!!!
+*/
+#endif
+
+#ifdef ERL_NIF_API_FUNC_MACRO
+#  define enif_priv_data ERL_NIF_API_FUNC_MACRO(enif_priv_data)
+#  define enif_alloc ERL_NIF_API_FUNC_MACRO(enif_alloc)
+#  define enif_free ERL_NIF_API_FUNC_MACRO(enif_free)
+#  define enif_is_atom ERL_NIF_API_FUNC_MACRO(enif_is_atom)
+#  define enif_is_binary ERL_NIF_API_FUNC_MACRO(enif_is_binary)
+#  define enif_is_ref ERL_NIF_API_FUNC_MACRO(enif_is_ref)
+#  define enif_inspect_binary ERL_NIF_API_FUNC_MACRO(enif_inspect_binary)
+#  define enif_alloc_binary ERL_NIF_API_FUNC_MACRO(enif_alloc_binary)
+#  define enif_realloc_binary ERL_NIF_API_FUNC_MACRO(enif_realloc_binary)
+#  define enif_release_binary ERL_NIF_API_FUNC_MACRO(enif_release_binary)
+#  define enif_get_int ERL_NIF_API_FUNC_MACRO(enif_get_int)
+#  define enif_get_ulong ERL_NIF_API_FUNC_MACRO(enif_get_ulong)
+#  define enif_get_double ERL_NIF_API_FUNC_MACRO(enif_get_double)
+#  define enif_get_tuple ERL_NIF_API_FUNC_MACRO(enif_get_tuple)
+#  define enif_get_list_cell ERL_NIF_API_FUNC_MACRO(enif_get_list_cell)
+#  define enif_is_identical ERL_NIF_API_FUNC_MACRO(enif_is_identical)
+#  define enif_compare ERL_NIF_API_FUNC_MACRO(enif_compare)
+
+#  define enif_make_binary ERL_NIF_API_FUNC_MACRO(enif_make_binary)
+#  define enif_make_badarg ERL_NIF_API_FUNC_MACRO(enif_make_badarg)
+#  define enif_make_int ERL_NIF_API_FUNC_MACRO(enif_make_int)
+#  define enif_make_ulong ERL_NIF_API_FUNC_MACRO(enif_make_ulong)
+#  define enif_make_double ERL_NIF_API_FUNC_MACRO(enif_make_double)
+#  define enif_make_atom ERL_NIF_API_FUNC_MACRO(enif_make_atom)
+#  define enif_make_existing_atom ERL_NIF_API_FUNC_MACRO(enif_make_existing_atom)
+#  define enif_make_tuple ERL_NIF_API_FUNC_MACRO(enif_make_tuple)
+#  define enif_make_list ERL_NIF_API_FUNC_MACRO(enif_make_list)
+#  define enif_make_list_cell ERL_NIF_API_FUNC_MACRO(enif_make_list_cell)
+#  define enif_make_string ERL_NIF_API_FUNC_MACRO(enif_make_string)
+#  define enif_make_ref ERL_NIF_API_FUNC_MACRO(enif_make_ref)
+
+#  define enif_mutex_create ERL_NIF_API_FUNC_MACRO(enif_mutex_create) 
+#  define enif_mutex_destroy ERL_NIF_API_FUNC_MACRO(enif_mutex_destroy) 
+#  define enif_mutex_trylock ERL_NIF_API_FUNC_MACRO(enif_mutex_trylock) 
+#  define enif_mutex_lock ERL_NIF_API_FUNC_MACRO(enif_mutex_lock) 
+#  define enif_mutex_unlock ERL_NIF_API_FUNC_MACRO(enif_mutex_unlock) 
+#  define enif_cond_create ERL_NIF_API_FUNC_MACRO(enif_cond_create) 
+#  define enif_cond_destroy ERL_NIF_API_FUNC_MACRO(enif_cond_destroy) 
+#  define enif_cond_signal ERL_NIF_API_FUNC_MACRO(enif_cond_signal) 
+#  define enif_cond_broadcast ERL_NIF_API_FUNC_MACRO(enif_cond_broadcast) 
+#  define enif_cond_wait ERL_NIF_API_FUNC_MACRO(enif_cond_wait) 
+#  define enif_rwlock_create ERL_NIF_API_FUNC_MACRO(enif_rwlock_create) 
+#  define enif_rwlock_destroy ERL_NIF_API_FUNC_MACRO(enif_rwlock_destroy) 
+#  define enif_rwlock_tryrlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_tryrlock) 
+#  define enif_rwlock_rlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_rlock) 
+#  define enif_rwlock_runlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_runlock) 
+#  define enif_rwlock_tryrwlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_tryrwlock) 
+#  define enif_rwlock_rwlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_rwlock) 
+#  define enif_rwlock_rwunlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_rwunlock) 
+#  define enif_tsd_key_create ERL_NIF_API_FUNC_MACRO(enif_tsd_key_create) 
+#  define enif_tsd_key_destroy ERL_NIF_API_FUNC_MACRO(enif_tsd_key_destroy) 
+#  define enif_tsd_set ERL_NIF_API_FUNC_MACRO(enif_tsd_set) 
+#  define enif_tsd_get ERL_NIF_API_FUNC_MACRO(enif_tsd_get) 
+#  define enif_thread_opts_create ERL_NIF_API_FUNC_MACRO(enif_thread_opts_create) 
+#  define enif_thread_opts_destroy ERL_NIF_API_FUNC_MACRO(enif_thread_opts_destroy) 
+#  define enif_thread_create ERL_NIF_API_FUNC_MACRO(enif_thread_create) 
+#  define enif_thread_self ERL_NIF_API_FUNC_MACRO(enif_thread_self) 
+#  define enif_equal_tids ERL_NIF_API_FUNC_MACRO(enif_equal_tids) 
+#  define enif_thread_exit ERL_NIF_API_FUNC_MACRO(enif_thread_exit) 
+#  define enif_thread_join ERL_NIF_API_FUNC_MACRO(enif_thread_join) 
+
+#  define enif_realloc ERL_NIF_API_FUNC_MACRO(enif_realloc) 
+#  define enif_system_info ERL_NIF_API_FUNC_MACRO(enif_system_info) 
+#  define enif_fprintf ERL_NIF_API_FUNC_MACRO(enif_fprintf) 
+#  define enif_inspect_iolist_as_binary ERL_NIF_API_FUNC_MACRO(enif_inspect_iolist_as_binary)
+#  define enif_make_sub_binary ERL_NIF_API_FUNC_MACRO(enif_make_sub_binary)
+#  define enif_get_string ERL_NIF_API_FUNC_MACRO(enif_get_string)
+#  define enif_get_atom ERL_NIF_API_FUNC_MACRO(enif_get_atom)
+#  define enif_is_fun ERL_NIF_API_FUNC_MACRO(enif_is_fun)
+#  define enif_is_pid ERL_NIF_API_FUNC_MACRO(enif_is_pid)
+#  define enif_is_port ERL_NIF_API_FUNC_MACRO(enif_is_port)
+#  define enif_get_uint ERL_NIF_API_FUNC_MACRO(enif_get_uint)
+#  define enif_get_long ERL_NIF_API_FUNC_MACRO(enif_get_long)
+#  define enif_make_uint ERL_NIF_API_FUNC_MACRO(enif_make_uint)
+#  define enif_make_long ERL_NIF_API_FUNC_MACRO(enif_make_long)
+#  define enif_make_tuple_from_array ERL_NIF_API_FUNC_MACRO(enif_make_tuple_from_array)
+#  define enif_make_list_from_array ERL_NIF_API_FUNC_MACRO(enif_make_list_from_array)
+#  define enif_is_empty_list ERL_NIF_API_FUNC_MACRO(enif_is_empty_list)
+#  define enif_open_resource_type ERL_NIF_API_FUNC_MACRO(enif_open_resource_type)
+#  define enif_alloc_resource ERL_NIF_API_FUNC_MACRO(enif_alloc_resource)
+#  define enif_release_resource ERL_NIF_API_FUNC_MACRO(enif_release_resource)
+#  define enif_make_resource ERL_NIF_API_FUNC_MACRO(enif_make_resource)
+#  define enif_get_resource ERL_NIF_API_FUNC_MACRO(enif_get_resource)
+#  define enif_sizeof_resource ERL_NIF_API_FUNC_MACRO(enif_sizeof_resource)
+#  define enif_make_new_binary ERL_NIF_API_FUNC_MACRO(enif_make_new_binary)
+#  define enif_is_list ERL_NIF_API_FUNC_MACRO(enif_is_list)
+#  define enif_is_tuple ERL_NIF_API_FUNC_MACRO(enif_is_tuple)
+#  define enif_get_atom_length ERL_NIF_API_FUNC_MACRO(enif_get_atom_length)
+#  define enif_get_list_length ERL_NIF_API_FUNC_MACRO(enif_get_list_length)
+#  define enif_make_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_atom_len)
+#  define enif_make_existing_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_existing_atom_len)
+#  define enif_make_string_len ERL_NIF_API_FUNC_MACRO(enif_make_string_len)
+#  define enif_alloc_env ERL_NIF_API_FUNC_MACRO(enif_alloc_env)
+#  define enif_free_env ERL_NIF_API_FUNC_MACRO(enif_free_env)
+#  define enif_clear_env ERL_NIF_API_FUNC_MACRO(enif_clear_env)
+#  define enif_send ERL_NIF_API_FUNC_MACRO(enif_send)
+#  define enif_make_copy ERL_NIF_API_FUNC_MACRO(enif_make_copy)
+#  define enif_self ERL_NIF_API_FUNC_MACRO(enif_self)
+#  define enif_get_local_pid ERL_NIF_API_FUNC_MACRO(enif_get_local_pid)
+#  define enif_keep_resource ERL_NIF_API_FUNC_MACRO(enif_keep_resource)
+#  define enif_make_resource_binary ERL_NIF_API_FUNC_MACRO(enif_make_resource_binary)
+#endif
+
+#ifndef enif_make_list1
+#  define enif_make_list1(ENV,E1) enif_make_list(ENV,1,E1)
+#  define enif_make_list2(ENV,E1,E2) enif_make_list(ENV,2,E1,E2)
+#  define enif_make_list3(ENV,E1,E2,E3) enif_make_list(ENV,3,E1,E2,E3)
+#  define enif_make_list4(ENV,E1,E2,E3,E4) enif_make_list(ENV,4,E1,E2,E3,E4)
+#  define enif_make_list5(ENV,E1,E2,E3,E4,E5) enif_make_list(ENV,5,E1,E2,E3,E4,E5)
+#  define enif_make_list6(ENV,E1,E2,E3,E4,E5,E6) enif_make_list(ENV,6,E1,E2,E3,E4,E5,E6)
+#  define enif_make_list7(ENV,E1,E2,E3,E4,E5,E6,E7) enif_make_list(ENV,7,E1,E2,E3,E4,E5,E6,E7)
+#  define enif_make_list8(ENV,E1,E2,E3,E4,E5,E6,E7,E8) enif_make_list(ENV,8,E1,E2,E3,E4,E5,E6,E7,E8)
+#  define enif_make_list9(ENV,E1,E2,E3,E4,E5,E6,E7,E8,E9) enif_make_list(ENV,9,E1,E2,E3,E4,E5,E6,E7,E8,E9)
+#  define enif_make_tuple1(ENV,E1) enif_make_tuple(ENV,1,E1)
+#  define enif_make_tuple2(ENV,E1,E2) enif_make_tuple(ENV,2,E1,E2)
+#  define enif_make_tuple3(ENV,E1,E2,E3) enif_make_tuple(ENV,3,E1,E2,E3)
+#  define enif_make_tuple4(ENV,E1,E2,E3,E4) enif_make_tuple(ENV,4,E1,E2,E3,E4)
+#  define enif_make_tuple5(ENV,E1,E2,E3,E4,E5) enif_make_tuple(ENV,5,E1,E2,E3,E4,E5)
+#  define enif_make_tuple6(ENV,E1,E2,E3,E4,E5,E6) enif_make_tuple(ENV,6,E1,E2,E3,E4,E5,E6)
+#  define enif_make_tuple7(ENV,E1,E2,E3,E4,E5,E6,E7) enif_make_tuple(ENV,7,E1,E2,E3,E4,E5,E6,E7)
+#  define enif_make_tuple8(ENV,E1,E2,E3,E4,E5,E6,E7,E8) enif_make_tuple(ENV,8,E1,E2,E3,E4,E5,E6,E7,E8)
+#  define enif_make_tuple9(ENV,E1,E2,E3,E4,E5,E6,E7,E8,E9) enif_make_tuple(ENV,9,E1,E2,E3,E4,E5,E6,E7,E8,E9)
+
+#  define enif_make_pid(ENV, PID) ((const ERL_NIF_TERM)((PID)->pid))
+#endif
+
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.1.2_0.c b/erts/emulator/test/nif_SUITE_data/nif_mod.1.2_0.c
new file mode 100644
index 0000000000..a554cc7f25
--- /dev/null
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.1.2_0.c
@@ -0,0 +1,4 @@
+#include "nif_api_2_0/erl_nif.h"
+
+#define NIF_LIB_VER 1
+#include "nif_mod.c"
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.2.2_0.c b/erts/emulator/test/nif_SUITE_data/nif_mod.2.2_0.c
new file mode 100644
index 0000000000..0731e6b5d0
--- /dev/null
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.2.2_0.c
@@ -0,0 +1,4 @@
+#include "nif_api_2_0/erl_nif.h"
+
+#define NIF_LIB_VER 2
+#include "nif_mod.c"
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.3.2_0.c b/erts/emulator/test/nif_SUITE_data/nif_mod.3.2_0.c
new file mode 100644
index 0000000000..d7e676b668
--- /dev/null
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.3.2_0.c
@@ -0,0 +1,4 @@
+#include "nif_api_2_0/erl_nif.h"
+
+#define NIF_LIB_VER 3
+#include "nif_mod.c"
-- 
cgit v1.2.3


From ece2fe5158d8a3212f5063572150ca991a66464c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 25 Nov 2016 19:34:29 +0100
Subject: erts: Fix code_SUITE:api_2_0,upgrade crash on win64

---
 erts/emulator/test/nif_SUITE.erl | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 01a5e84dc9..0d3910b2e2 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -91,7 +91,13 @@ init_per_group(api_latest, Config) -> Config;
 init_per_group(api_2_4, Config) ->
     [{nif_api_version, ".2_4"} | Config];
 init_per_group(api_2_0, Config) ->
-    [{nif_api_version, ".2_0"} | Config].
+    case {os:type(),erlang:system_info({wordsize, internal})} of
+        {{win32,_}, 8} ->
+            %% ERL_NIF_TERM was declared as 32-bit 'long' until 2.3
+            {skip, "API 2.0 buggy on Windows 64-bit"};
+        _ ->
+            [{nif_api_version, ".2_0"} | Config]
+    end.
 
 end_per_group(_,_) -> ok.
 
-- 
cgit v1.2.3


From a85a45b5a384deae60f8020d0805a93df141a7c3 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 24 Oct 2016 18:35:35 +0200
Subject: erts: Change handling of error event for driver_select

Make the code simpler.

May change the behavior as we may call both iready and oready
when getting an error. But doesn't that seem more correct.
---
 erts/emulator/sys/common/erl_check_io.c | 43 +++++++++++++--------------------
 1 file changed, 17 insertions(+), 26 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 44a77f3ea5..146f6ae5a1 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1699,31 +1699,22 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 
 	switch (state->type) {
 	case ERTS_EV_TYPE_DRV_SEL: { /* Requested via driver_select()... */
-	    ErtsPollEvents revents;
-	    ErtsPollEvents revent_mask;
-
-	    revent_mask = ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT);
-	    revent_mask |= state->events;
-	    revents = pollres[i].events & revent_mask;
-
-	    if (revents & ERTS_POLL_EV_ERR) {
-		/*
-		 * Let the driver handle the error condition. Only input,
-		 * only output, or nothing might have been selected.
-		 * We *do not* want to call a callback that corresponds
-		 * to an event not selected. revents might give us a clue
-		 * on which one to call.
-		 */ 
-		if ((revents & ERTS_POLL_EV_IN)
-		    || (!(revents & ERTS_POLL_EV_OUT)
-			&& state->events & ERTS_POLL_EV_IN)) {
-		    iready(state->driver.select->inport, state, current_cio_time);
-		}
-		else if (state->events & ERTS_POLL_EV_OUT) {
-		    oready(state->driver.select->outport, state, current_cio_time);
-		}
-	    }
-	    else if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) {
+	    ErtsPollEvents revents = pollres[i].events;
+
+            if (revents & ERTS_POLL_EV_ERR) {
+                /*
+                 * Handle error events by triggering all in/out events
+                 * that the driver has selected.
+                 * We *do not* want to call a callback that corresponds
+		 * to an event not selected.
+                 */
+                revents = state->events;
+            }
+            else {
+                revents &= (state->events | ERTS_POLL_EV_NVAL);
+            }
+
+            if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) {
 		if (revents & ERTS_POLL_EV_OUT) {
 		    oready(state->driver.select->outport, state, current_cio_time);
 		}
@@ -1731,7 +1722,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 		   was read (true also on the non-smp emulator since
 		   oready() may have been called); therefore, update
 		   revents... */
-		revents &= ~(~state->events & ERTS_POLL_EV_IN);
+		revents &= state->events;
 		if (revents & ERTS_POLL_EV_IN) {
 		    iready(state->driver.select->inport, state, current_cio_time);
 		}
-- 
cgit v1.2.3


From c3cefc3ac9c0487832fecb4eb1eadf1e49e4fa77 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 21 Oct 2016 17:01:55 +0200
Subject: erts: Refactor add_active_fd

to not inline cold reallocation code.
---
 erts/emulator/sys/common/erl_check_io.c | 38 ++++++++++++++++++---------------
 1 file changed, 21 insertions(+), 17 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 146f6ae5a1..e6c05faa4c 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -807,6 +807,24 @@ check_cleanup_active_fds(erts_aint_t current_cio_time)
     erts_smp_atomic32_set_relb(&pollset.active_fd.no, no);
 }
 
+static void grow_active_fds(void)
+{
+    ASSERT(pollset.active_fd.six == pollset.active_fd.eix);
+    pollset.active_fd.six = 0;
+    pollset.active_fd.eix = pollset.active_fd.size;
+    pollset.active_fd.size += ERTS_ACTIVE_FD_INC;
+    pollset.active_fd.array = erts_realloc(ERTS_ALC_T_ACTIVE_FD_ARR,
+                                           pollset.active_fd.array,
+                                           pollset.active_fd.size*sizeof(ErtsSysFdType));
+#ifdef DEBUG
+    {
+        int i;
+        for (i = pollset.active_fd.eix + 1; i < pollset.active_fd.size; i++)
+            pollset.active_fd.array[i] = ERTS_SYS_FD_INVALID;
+    }
+#endif
+}
+
 static ERTS_INLINE void
 add_active_fd(ErtsSysFdType fd)
 {
@@ -823,25 +841,11 @@ add_active_fd(ErtsSysFdType fd)
     eix++;
     if (eix >= size)
 	eix = 0;
-    if (pollset.active_fd.six == eix) {
-	pollset.active_fd.six = 0;
-	eix = size;
-	size += ERTS_ACTIVE_FD_INC;
-	pollset.active_fd.array = erts_realloc(ERTS_ALC_T_ACTIVE_FD_ARR,
-					       pollset.active_fd.array,
-					       sizeof(ErtsSysFdType)*size);
-	pollset.active_fd.size = size;
-#ifdef DEBUG
-	{
-	    int i;
-	    for (i = eix + 1; i < size; i++)
-		pollset.active_fd.array[i] = ERTS_SYS_FD_INVALID;
-	}
-#endif
+    pollset.active_fd.eix = eix;
 
+    if (pollset.active_fd.six == eix) {
+        grow_active_fds();
     }
-
-    pollset.active_fd.eix = eix;
 }
 
 int
-- 
cgit v1.2.3


From 16b9292ff0914f77ee7ab7e169def914a190f79b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 27 Oct 2016 16:16:52 +0200
Subject: erts: Rename some static function

---
 erts/emulator/sys/common/erl_check_io.c | 56 ++++++++++++++++-----------------
 1 file changed, 28 insertions(+), 28 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index e6c05faa4c..2c138e1746 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -211,24 +211,24 @@ static ERTS_INLINE void hash_erase_drv_ev_state(ErtsDrvEventState *state)
 #endif /* !ERTS_SYS_CONTINOUS_FD_NUMBERS */
 
 static void stale_drv_select(Eterm id, ErtsDrvEventState *state, int mode);
-static void select_steal(ErlDrvPort ix, ErtsDrvEventState *state, 
+static void drv_select_steal(ErlDrvPort ix, ErtsDrvEventState *state,
 			 int mode, int on);
-static void print_select_op(erts_dsprintf_buf_t *dsbufp,
-			    ErlDrvPort ix, ErtsSysFdType fd, int mode, int on);
+static void print_drv_select_op(erts_dsprintf_buf_t *dsbufp,
+                                ErlDrvPort ix, ErtsSysFdType fd, int mode, int on);
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
 static void select_large_fd_error(ErlDrvPort, ErtsSysFdType, int, int);
 #endif
 #if ERTS_CIO_HAVE_DRV_EVENT
-static void event_steal(ErlDrvPort ix, ErtsDrvEventState *state, 
+static void drv_event_steal(ErlDrvPort ix, ErtsDrvEventState *state,
 			ErlDrvEventData event_data);
-static void print_event_op(erts_dsprintf_buf_t *dsbufp,
-			   ErlDrvPort, ErtsSysFdType, ErlDrvEventData);    
+static void print_drv_event_op(erts_dsprintf_buf_t *dsbufp,
+                               ErlDrvPort, ErtsSysFdType, ErlDrvEventData);
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
 static void event_large_fd_error(ErlDrvPort, ErtsSysFdType, ErlDrvEventData);
 #endif
 #endif
-static void steal_pending_stop_select(erts_dsprintf_buf_t*, ErlDrvPort,
-				      ErtsDrvEventState*, int mode, int on);
+static void steal_pending_stop_use(erts_dsprintf_buf_t*, ErlDrvPort,
+                                   ErtsDrvEventState*, int mode, int on);
 
 #ifdef ERTS_SMP
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(removed_fd, struct removed_fd, 64, ERTS_ALC_T_FD_LIST)
@@ -922,12 +922,12 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
 
 #if ERTS_CIO_HAVE_DRV_EVENT
     if (state->type == ERTS_EV_TYPE_DRV_EV)
-	select_steal(ix, state, mode, on);
+	drv_select_steal(ix, state, mode, on);
 #endif
     if (state->type == ERTS_EV_TYPE_STOP_USE) {
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	print_select_op(dsbufp, ix, state->fd, mode, on);
-	steal_pending_stop_select(dsbufp, ix, state, mode, on);
+	print_drv_select_op(dsbufp, ix, state->fd, mode, on);
+	steal_pending_stop_use(dsbufp, ix, state, mode, on);
 	if (state->type == ERTS_EV_TYPE_STOP_USE) {
 	    ret = 0;
 	    goto done; /* stop_select still pending */
@@ -939,7 +939,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
 	if (state->type == ERTS_EV_TYPE_DRV_SEL) {
 	    Eterm owner = state->driver.select->inport;
 	    if (owner != id && is_not_nil(owner))
-		select_steal(ix, state, mode, on);
+		drv_select_steal(ix, state, mode, on);
 	}
 	ctl_events |= ERTS_POLL_EV_IN;
     }
@@ -947,7 +947,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
 	if (state->type == ERTS_EV_TYPE_DRV_SEL) {
 	    Eterm owner = state->driver.select->outport;
 	    if (owner != id && is_not_nil(owner))
-		select_steal(ix, state, mode, on);
+		drv_select_steal(ix, state, mode, on);
 	}
 	ctl_events |= ERTS_POLL_EV_OUT;
     }	
@@ -1130,12 +1130,12 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix,
 	if (state->driver.event->port == id) break;
 	/*fall through*/
     case ERTS_EV_TYPE_DRV_SEL:
-	event_steal(ix, state, event_data);
+	drv_event_steal(ix, state, event_data);
 	break;
     case ERTS_EV_TYPE_STOP_USE: {
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	print_event_op(dsbufp, ix, fd, event_data);
-	steal_pending_stop_select(dsbufp, ix, state, 0, 1);
+	print_drv_event_op(dsbufp, ix, fd, event_data);
+	steal_pending_stop_use(dsbufp, ix, state, 0, 1);
 	break;
       }
     }
@@ -1339,8 +1339,8 @@ steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode)
 }
 
 static void
-print_select_op(erts_dsprintf_buf_t *dsbufp,
-		ErlDrvPort ix, ErtsSysFdType fd, int mode, int on)
+print_drv_select_op(erts_dsprintf_buf_t *dsbufp,
+                    ErlDrvPort ix, ErtsSysFdType fd, int mode, int on)
 {
     Port *pp = erts_drvport2port(ix);
     erts_dsprintf(dsbufp,
@@ -1358,11 +1358,11 @@ print_select_op(erts_dsprintf_buf_t *dsbufp,
 }
 
 static void
-select_steal(ErlDrvPort ix, ErtsDrvEventState *state, int mode, int on)
+drv_select_steal(ErlDrvPort ix, ErtsDrvEventState *state, int mode, int on)
 {
     if (need2steal(state, mode)) {
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	print_select_op(dsbufp, ix, state->fd, mode, on);
+	print_drv_select_op(dsbufp, ix, state->fd, mode, on);
 	steal(dsbufp, state, mode);
 	erts_send_error_to_logger_nogl(dsbufp);
     }
@@ -1381,7 +1381,7 @@ static void
 select_large_fd_error(ErlDrvPort ix, ErtsSysFdType fd, int mode, int on)
 {
     erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-    print_select_op(dsbufp, ix, fd, mode, on);
+    print_drv_select_op(dsbufp, ix, fd, mode, on);
     erts_dsprintf(dsbufp, "failed: ");
     large_fd_error_common(dsbufp, fd);
     erts_send_error_to_logger_nogl(dsbufp);
@@ -1391,8 +1391,8 @@ select_large_fd_error(ErlDrvPort ix, ErtsSysFdType fd, int mode, int on)
 
 
 static void
-steal_pending_stop_select(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix,
-			  ErtsDrvEventState *state, int mode, int on)
+steal_pending_stop_use(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix,
+                       ErtsDrvEventState *state, int mode, int on)
 {
     ASSERT(state->type == ERTS_EV_TYPE_STOP_USE);
     erts_dsprintf(dsbufp, "failed: fd=%d (re)selected before stop_select "
@@ -1433,8 +1433,8 @@ steal_pending_stop_select(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix,
 #if ERTS_CIO_HAVE_DRV_EVENT
 
 static void
-print_event_op(erts_dsprintf_buf_t *dsbufp,
-	       ErlDrvPort ix, ErtsSysFdType fd, ErlDrvEventData event_data)
+print_drv_event_op(erts_dsprintf_buf_t *dsbufp,
+                   ErlDrvPort ix, ErtsSysFdType fd, ErlDrvEventData event_data)
 {
     Port *pp = erts_drvport2port(ix);
     erts_dsprintf(dsbufp, "driver_event(%p, %d, ", ix, (int) fd);
@@ -1451,11 +1451,11 @@ print_event_op(erts_dsprintf_buf_t *dsbufp,
 }
 
 static void
-event_steal(ErlDrvPort ix, ErtsDrvEventState *state, ErlDrvEventData event_data)
+drv_event_steal(ErlDrvPort ix, ErtsDrvEventState *state, ErlDrvEventData event_data)
 {
     if (need2steal(state, ERL_DRV_READ|ERL_DRV_WRITE)) {
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	print_event_op(dsbufp, ix, state->fd, event_data);
+	print_drv_event_op(dsbufp, ix, state->fd, event_data);
 	steal(dsbufp, state, ERL_DRV_READ|ERL_DRV_WRITE);
 	erts_send_error_to_logger_nogl(dsbufp);
     }
@@ -1470,7 +1470,7 @@ static void
 event_large_fd_error(ErlDrvPort ix, ErtsSysFdType fd, ErlDrvEventData event_data)
 {
     erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-    print_event_op(dsbufp, ix, fd, event_data);
+    print_drv_event_op(dsbufp, ix, fd, event_data);
     erts_dsprintf(dsbufp, "failed: ");
     large_fd_error_common(dsbufp, fd);
     erts_send_error_to_logger_nogl(dsbufp);
-- 
cgit v1.2.3


From 9a88fc02527063be5096bdf4d49a48b144471553 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 14 Oct 2016 18:48:00 +0200
Subject: erts: Add erts internal secret atom

---
 erts/emulator/beam/atom.c     | 5 ++++-
 erts/emulator/beam/atom.h     | 2 --
 erts/emulator/beam/atom.names | 3 +++
 3 files changed, 7 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index b47739059b..2b5ad097a0 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -199,7 +199,7 @@ atom_alloc(Atom* tmpl)
 static void
 atom_free(Atom* obj)
 {
-    erts_free(ERTS_ALC_T_ATOM, (void*) obj);
+    ASSERT(obj->slot.index == atom_val(am_ErtsSecretAtom));
 }
 
 static void latin1_to_utf8(byte* conv_buf, const byte** srcp, int* lenp)
@@ -467,6 +467,9 @@ init_atom_table(void)
 	atom_space -= a.len;
 	atom_tab(ix)->name = (byte*)erl_atom_names[i];
     }
+
+    /* Hide am_ErtsSecretAtom */
+    hash_erase(&erts_atom_table.htable, atom_tab(atom_val(am_ErtsSecretAtom)));
 }
 
 void
diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h
index f3b21e1687..abd3b44993 100644
--- a/erts/emulator/beam/atom.h
+++ b/erts/emulator/beam/atom.h
@@ -133,8 +133,6 @@ int atom_table_sz(void);	/* table size in bytes, excluding stored objects */
 
 Eterm am_atom_put(const char*, int); /* ONLY 7-bit ascii! */
 Eterm erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc);
-int atom_erase(byte*, int);
-int atom_static_put(byte*, int);
 void init_atom_table(void);
 void atom_info(fmtfn_t, void *);
 void dump_atoms(fmtfn_t, void *);
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 66af05c1f2..b1aeed7889 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -59,6 +59,9 @@ atom nocatch
 atom undefined_function
 atom undefined_lambda
 
+# Secret internal atom that can never be found by string lookup
+# and should never leak out to be seen by the user.
+atom ErtsSecretAtom='3RT$'
 
 # All other atoms.  Try to keep the order alphabetic.
 #
-- 
cgit v1.2.3


From f7a3650b6827242772f957d624dbb91b5e472744 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 29 Nov 2016 18:21:17 +0100
Subject: erts: Use secret atom as sysname for ETS compressed

---
 erts/emulator/beam/external.c | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index a49b242d7c..10e452fa25 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2163,12 +2163,8 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags)
  * We use this atom as sysname in local pid/port/refs
  * for the ETS compressed format (DFLAG_INTERNAL_TAGS).
  *
- * We used atom '' earlier but that turned out to cause problems
- * for buggy erl_interface/ic usage of c-nodes with empty node names.
- * A long atom reduces risk of nodes actually called this and the length
- * does not matter anyway as it's encoded with atom index (ATOM_INTERNAL_REF2).
  */
-#define INTERNAL_LOCAL_SYSNAME am_await_microstate_accounting_modifications
+#define INTERNAL_LOCAL_SYSNAME am_ErtsSecretAtom
 
 static byte*
 enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags)
-- 
cgit v1.2.3


From 6c76bfa96c12b9f29e66baf3a2df7e84bb9b05ef Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 29 Nov 2016 20:54:14 +0100
Subject: erts: Refactor BEAM_NIF_MIN_FUNC_SZ

to be declared once in beam_load.h
and get rid of #ifdef kludge.
---
 erts/emulator/beam/beam_load.c | 11 +++--------
 erts/emulator/beam/beam_load.h |  6 ++++++
 erts/emulator/beam/erl_nif.c   |  7 ++-----
 3 files changed, 11 insertions(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 3cd395c2c1..8f1faa6719 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2546,15 +2546,10 @@ load_code(LoaderState* stp)
 
 		if (stp->may_load_nif) {
 		    const int finfo_ix = ci - FUNC_INFO_SZ;
-#ifdef ERTS_DIRTY_SCHEDULERS
-		    enum { MIN_FUNC_SZ = 4 };
-#else
-		    enum { MIN_FUNC_SZ = 3 };
-#endif
-		    if (finfo_ix - last_func_start < MIN_FUNC_SZ && last_func_start) {		   
+		    if (finfo_ix - last_func_start < BEAM_NIF_MIN_FUNC_SZ && last_func_start) {
 			/* Must make room for call_nif op */
-			int pad = MIN_FUNC_SZ - (finfo_ix - last_func_start);
-			ASSERT(pad > 0 && pad < MIN_FUNC_SZ);
+			int pad = BEAM_NIF_MIN_FUNC_SZ - (finfo_ix - last_func_start);
+			ASSERT(pad > 0 && pad < BEAM_NIF_MIN_FUNC_SZ);
 			CodeNeed(pad);
 			sys_memmove(&code[finfo_ix+pad], &code[finfo_ix],
 				    FUNC_INFO_SZ*sizeof(BeamInstr));
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index d420a4346c..659b9c303f 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -111,6 +111,12 @@ typedef struct beam_code_header {
 
 }BeamCodeHeader;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+#  define BEAM_NIF_MIN_FUNC_SZ 4
+#else
+#  define BEAM_NIF_MIN_FUNC_SZ 3
+#endif
+
 void erts_release_literal_area(struct ErtsLiteralArea_* literal_area);
 int erts_is_module_native(BeamCodeHeader* code);
 void erts_beam_bif_load_init(void);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 202f713f67..5499512dd1 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3341,11 +3341,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 				     mod_atom, f->name, f->arity);
 #endif
 	    }
-#ifdef ERTS_DIRTY_SCHEDULERS
-	    else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < (4))
-#else
-	    else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < (3))
-#endif
+	    else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0])
+                     < BEAM_NIF_MIN_FUNC_SZ)
 	    {
 		ret = load_nif_error(BIF_P,bad_lib,"No explicit call to load_nif"
 				     " in module (%T:%s/%u too small)",
-- 
cgit v1.2.3


From b3441cbd116fede332e1e4cfee4d48331be1fa62 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 22 Nov 2016 13:39:24 +0100
Subject: erts: Add ?line macro for some hipe testing

---
 erts/emulator/test/code_SUITE.erl                  |   2 +-
 .../code_SUITE_data/call_purged_fun_tester.erl     | 108 ++++++++++++---------
 2 files changed, 65 insertions(+), 45 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index 774461c525..0742b77a52 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -155,7 +155,7 @@ call_purged_fun_code_there(Config) when is_list(Config) ->
 call_purged_fun_test(Priv, Data, Type) ->
     OptsList = case erlang:system_info(hipe_architecture) of
                    undefined -> [[]];
-                   _ -> [[], [native]]
+                   _ -> [[], [native,{d,hipe}]]
                end,
     [call_purged_fun_test_do(Priv, Data, Type, CO, FO)
      || CO <- OptsList, FO <- OptsList].
diff --git a/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl b/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl
index 5e031abca8..16dbbcec37 100644
--- a/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl
+++ b/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl
@@ -2,7 +2,27 @@
 
 -export([do/4]).
 
+%% Resurrect line macro when hipe compiled
+-ifdef(hipe).
+-define(line, put(the_line,?LINE),).
 do(Priv, Data, Type, Opts) ->
+    try do_it(Priv, Data, Type, Opts)
+    catch
+        C:E ->
+            ST = erlang:get_stacktrace(),
+            io:format("Caught exception from line ~p:\n~p\n",
+                      [get(the_line), ST]),
+            io:format("Message queue: ~p\n", [process_info(self(), messages)]),
+            erlang:raise(C, E, ST)
+    end.
+-else.
+-define(line,).
+do(P,D,T,O) ->
+    do_it(P,D,T,O).
+-endif.
+
+
+do_it(Priv, Data, Type, Opts) ->
     File = filename:join(Data, "my_code_test2"),
     Code = filename:join(Priv, "my_code_test2"),
 
@@ -10,12 +30,12 @@ do(Priv, Data, Type, Opts) ->
     catch erlang:delete_module(my_code_test2),
     catch erlang:purge_module(my_code_test2),
 
-    {ok,my_code_test2} = c:c(File, [{outdir,Priv} | Opts]),
+    ?line {ok,my_code_test2} = c:c(File, [{outdir,Priv} | Opts]),
 
-    IsNative = lists:member(native,Opts),
-    IsNative = code:is_module_native(my_code_test2),
+    ?line IsNative = lists:member(native,Opts),
+    ?line IsNative = code:is_module_native(my_code_test2),
 
-    T = ets:new(my_code_test2_fun_table, []),
+    ?line T = ets:new(my_code_test2_fun_table, []),
     ets:insert(T, {my_fun,my_code_test2:make_fun(4711)}),
     ets:insert(T, {my_fun2,my_code_test2:make_fun2()}),
 
@@ -28,7 +48,7 @@ do(Priv, Data, Type, Opts) ->
 		  exit(completed)
 	  end),
 
-    PurgeType = case Type of
+    ?line PurgeType = case Type of
 		    code_gone ->
 			ok = file:delete(Code++".beam"),
 			true;
@@ -38,89 +58,89 @@ do(Priv, Data, Type, Opts) ->
 			false
 		end,
 
-    true = erlang:delete_module(my_code_test2),
+    ?line true = erlang:delete_module(my_code_test2),
 
-    Purge = start_purge(my_code_test2, PurgeType),
+    ?line Purge = start_purge(my_code_test2, PurgeType),
 
-    {P0, M0} = spawn_monitor(fun () ->
-				     [{my_fun,F}] = ets:lookup(T, my_fun),
-				     4712 = F(1),
-				     exit(completed)
+    ?line {P0, M0} = spawn_monitor(fun () ->
+                                           ?line [{my_fun,F}] = ets:lookup(T, my_fun),
+                                           ?line 4712 = F(1),
+                                           exit(completed)
 			     end),
 
-    wait_until(fun () ->
-		       {status, suspended}
-			   == process_info(P0, status)
-	       end),
+    ?line ok = wait_until(fun () ->
+                                  {status, suspended}
+                                      == process_info(P0, status)
+                          end),
 
-    ok = continue_purge(Purge),
+    ?line ok = continue_purge(Purge),
 
-    {P1, M1} = spawn_monitor(fun () ->
-				     [{my_fun,F}] = ets:lookup(T, my_fun),
-				     4713 = F(2),
-				     exit(completed)
+    ?line {P1, M1} = spawn_monitor(fun () ->
+                                           ?line [{my_fun,F}] = ets:lookup(T, my_fun),
+                                           ?line 4713 = F(2),
+                                           exit(completed)
 			     end),
-    {P2, M2} = spawn_monitor(fun () ->
-				     [{my_fun,F}] = ets:lookup(T, my_fun),
-				     4714 = F(3),
-				     exit(completed)
+    ?line {P2, M2} = spawn_monitor(fun () ->
+                                           ?line [{my_fun,F}] = ets:lookup(T, my_fun),
+                                           ?line 4714 = F(3),
+                                           exit(completed)
 			     end),
 
-    wait_until(fun () ->
-		       {status, suspended}
-			   == process_info(P1, status)
-	       end),
-    wait_until(fun () ->
-		       {status, suspended}
-			   == process_info(P2, status)
-	       end),
+    ?line ok = wait_until(fun () ->
+                                  {status, suspended}
+                                      == process_info(P1, status)
+                          end),
+    ?line ok = wait_until(fun () ->
+                                  {status, suspended}
+                                      == process_info(P2, status)
+                          end),
 
-    {current_function,
+    ?line {current_function,
      {erts_code_purger,
       pending_purge_lambda,
       3}} = process_info(P0, current_function),
-    {current_function,
+    ?line {current_function,
      {erts_code_purger,
       pending_purge_lambda,
       3}} = process_info(P1, current_function),
-    {current_function,
+    ?line {current_function,
      {erts_code_purger,
       pending_purge_lambda,
       3}} = process_info(P2, current_function),
 
     case Type of
 	code_there ->
-	    false = complete_purge(Purge);
+	    ?line false = complete_purge(Purge);
 	_ ->
-	    {true, true} = complete_purge(Purge)
+	    ?line {true, true} = complete_purge(Purge)
     end,
 
     case Type of
 	code_gone ->
 	    receive
 		{'DOWN', M0, process, P0, Reason0} ->
-		    {undef, _} = Reason0
+		    ?line {undef, _} = Reason0
 	    end,
 	    receive
 		{'DOWN', M1, process, P1, Reason1} ->
-		    {undef, _} = Reason1
+		    ?line {undef, _} = Reason1
 	    end,
 	    receive
 		{'DOWN', M2, process, P2, Reason2} ->
-		    {undef, _} = Reason2
+		    ?line {undef, _} = Reason2
 	    end;
 	_ ->
 	    receive
 		{'DOWN', M0, process, P0, Reason0} ->
-		    completed = Reason0
+		    ?line completed = Reason0
 	    end,
 	    receive
 		{'DOWN', M1, process, P1, Reason1} ->
-		    completed = Reason1
+		    ?line completed = Reason1
 	    end,
 	    receive
 		{'DOWN', M2, process, P2, Reason2} ->
-		    completed = Reason2
+		    ?line completed = Reason2
 	    end,
 	    catch erlang:purge_module(my_code_test2),
 	    catch erlang:delete_module(my_code_test2),
@@ -129,7 +149,7 @@ do(Priv, Data, Type, Opts) ->
     ok.
 
 wait_until(Fun) ->
-    ok = wait_until(Fun, 20).
+    wait_until(Fun, 20).
 
 wait_until(Fun, N) ->
     case {Fun(),N} of
-- 
cgit v1.2.3


From 15789e168811d6e6bed082ea7c64db93e1d968e8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 17 Nov 2016 14:39:04 +0100
Subject: erts: Refactor test code

---
 .../code_SUITE_data/call_purged_fun_tester.erl     | 58 +++++++++-------------
 1 file changed, 24 insertions(+), 34 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl b/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl
index 16dbbcec37..10d00b7d2a 100644
--- a/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl
+++ b/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl
@@ -62,25 +62,25 @@ do_it(Priv, Data, Type, Opts) ->
 
     ?line Purge = start_purge(my_code_test2, PurgeType),
 
-    ?line {P0, M0} = spawn_monitor(fun () ->
+    ?line {P1, M1} = spawn_monitor(fun () ->
                                            ?line [{my_fun,F}] = ets:lookup(T, my_fun),
                                            ?line 4712 = F(1),
                                            exit(completed)
 			     end),
 
-    ?line ok = wait_until(fun () ->
+   ?line ok =  wait_until(fun () ->
                                   {status, suspended}
-                                      == process_info(P0, status)
+                                      == process_info(P1, status)
                           end),
 
     ?line ok = continue_purge(Purge),
 
-    ?line {P1, M1} = spawn_monitor(fun () ->
+    ?line {P2, M2} = spawn_monitor(fun () ->
                                            ?line [{my_fun,F}] = ets:lookup(T, my_fun),
                                            ?line 4713 = F(2),
                                            exit(completed)
 			     end),
-    ?line {P2, M2} = spawn_monitor(fun () ->
+    ?line {P3, M3} = spawn_monitor(fun () ->
                                            ?line [{my_fun,F}] = ets:lookup(T, my_fun),
                                            ?line 4714 = F(3),
                                            exit(completed)
@@ -88,25 +88,25 @@ do_it(Priv, Data, Type, Opts) ->
 
     ?line ok = wait_until(fun () ->
                                   {status, suspended}
-                                      == process_info(P1, status)
+                                      == process_info(P2, status)
                           end),
     ?line ok = wait_until(fun () ->
                                   {status, suspended}
-                                      == process_info(P2, status)
+                                      == process_info(P3, status)
                           end),
 
     ?line {current_function,
      {erts_code_purger,
       pending_purge_lambda,
-      3}} = process_info(P0, current_function),
+      3}} = process_info(P1, current_function),
     ?line {current_function,
      {erts_code_purger,
       pending_purge_lambda,
-      3}} = process_info(P1, current_function),
+      3}} = process_info(P2, current_function),
     ?line {current_function,
      {erts_code_purger,
       pending_purge_lambda,
-      3}} = process_info(P2, current_function),
+      3}} = process_info(P3, current_function),
 
     case Type of
 	code_there ->
@@ -117,37 +117,27 @@ do_it(Priv, Data, Type, Opts) ->
 
     case Type of
 	code_gone ->
-	    receive
-		{'DOWN', M0, process, P0, Reason0} ->
-		    ?line {undef, _} = Reason0
-	    end,
-	    receive
-		{'DOWN', M1, process, P1, Reason1} ->
-		    ?line {undef, _} = Reason1
-	    end,
-	    receive
-		{'DOWN', M2, process, P2, Reason2} ->
-		    ?line {undef, _} = Reason2
-	    end;
+            ?line {undef, _} = wait_for_down(P1,M1),
+            ?line {undef, _} = wait_for_down(P2,M2),
+            ?line {undef, _} = wait_for_down(P3,M3);
 	_ ->
-	    receive
-		{'DOWN', M0, process, P0, Reason0} ->
-		    ?line completed = Reason0
-	    end,
-	    receive
-		{'DOWN', M1, process, P1, Reason1} ->
-		    ?line completed = Reason1
-	    end,
-	    receive
-		{'DOWN', M2, process, P2, Reason2} ->
-		    ?line completed = Reason2
-	    end,
+            ?line completed = wait_for_down(P1,M1),
+            ?line completed = wait_for_down(P2,M2),
+            ?line completed = wait_for_down(P3,M3),
 	    catch erlang:purge_module(my_code_test2),
 	    catch erlang:delete_module(my_code_test2),
 	    catch erlang:purge_module(my_code_test2)
     end,
     ok.
 
+wait_for_down(P,M) ->
+    receive
+        {'DOWN', M, process, P, Reason} ->
+            Reason
+    after 1000 ->
+            timeout
+    end.
+
 wait_until(Fun) ->
     wait_until(Fun, 20).
 
-- 
cgit v1.2.3


From 20eea6a99b25b99db728fcae5b8303739e5c1cc8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 17 Nov 2016 17:35:17 +0100
Subject: erts: Fix race in code_SUITE:call_purged_fun_*

Must wait for process P0 to enter fun F2
before starting purge, to make sure it's not suspended.
---
 .../code_SUITE_data/call_purged_fun_tester.erl     | 29 ++++++++++++++--------
 1 file changed, 19 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl b/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl
index 10d00b7d2a..699f0c1161 100644
--- a/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl
+++ b/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl
@@ -39,14 +39,16 @@ do_it(Priv, Data, Type, Opts) ->
     ets:insert(T, {my_fun,my_code_test2:make_fun(4711)}),
     ets:insert(T, {my_fun2,my_code_test2:make_fun2()}),
 
-    spawn(fun () ->
-		  [{my_fun2,F2}] = ets:lookup(T, my_fun2),
-		  F2(fun () ->
-			     receive after infinity -> ok end
-		     end,
-		     fun () -> ok end),
-		  exit(completed)
-	  end),
+    Papa = self(),
+    {P0,M0} = spawn_monitor(fun () ->
+                                    [{my_fun2,F2}] = ets:lookup(T, my_fun2),
+                                    F2(fun () ->
+                                               Papa ! {self(),"going to sleep"},
+                                               receive {Papa,"wake up"} -> ok end
+                                       end,
+                                       fun () -> ok end),
+                                    exit(completed)
+                            end),
 
     ?line PurgeType = case Type of
 		    code_gone ->
@@ -60,6 +62,10 @@ do_it(Priv, Data, Type, Opts) ->
 
     ?line true = erlang:delete_module(my_code_test2),
 
+    ?line ok = receive {P0, "going to sleep"} -> ok
+               after 1000 -> timeout
+               end,
+
     ?line Purge = start_purge(my_code_test2, PurgeType),
 
     ?line {P1, M1} = spawn_monitor(fun () ->
@@ -110,9 +116,12 @@ do_it(Priv, Data, Type, Opts) ->
 
     case Type of
 	code_there ->
-	    ?line false = complete_purge(Purge);
+	    ?line false = complete_purge(Purge),
+            P0 ! {self(), "wake up"},
+            ?line completed = wait_for_down(P0,M0);
 	_ ->
-	    ?line {true, true} = complete_purge(Purge)
+	    ?line {true, true} = complete_purge(Purge),
+            ?line killed = wait_for_down(P0,M0)
     end,
 
     case Type of
-- 
cgit v1.2.3


From 0763a36867a702e3075b682973a079e0390144ce Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 23 Nov 2016 15:58:15 +0100
Subject: erts: Add enif_select & enif_open_resource_type_x

---
 erts/emulator/beam/erl_alloc.c                |   2 +
 erts/emulator/beam/erl_alloc.types            |   1 +
 erts/emulator/beam/erl_driver.h               |   9 +-
 erts/emulator/beam/erl_drv_nif.h              |   7 +
 erts/emulator/beam/erl_nif.c                  |  90 +--
 erts/emulator/beam/erl_nif.h                  |  20 +-
 erts/emulator/beam/erl_nif_api_funcs.h        |   4 +
 erts/emulator/beam/global.h                   |  29 +
 erts/emulator/sys/common/erl_check_io.c       | 759 ++++++++++++++++++++++++--
 erts/emulator/sys/common/erl_check_io.h       |  18 +
 erts/emulator/sys/common/erl_poll.c           |   1 +
 erts/emulator/sys/unix/sys.c                  |  10 +
 erts/emulator/test/nif_SUITE.erl              |  43 ++
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 190 ++++++-
 14 files changed, 1076 insertions(+), 107 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 40a45c961f..3ddf7a53e2 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -663,6 +663,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	= sizeof(ErtsDrvEventDataState);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)]
 	= sizeof(ErtsDrvSelectDataState);
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_SEL_D_STATE)]
+        = sizeof(ErtsNifSelectDataState);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)]
 	= sizeof(ErtsMessageRef);
 #ifdef ERTS_SMP
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 6e8710eb8a..7ea8c98008 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -396,6 +396,7 @@ type	DRV_TAB		LONG_LIVED	SYSTEM		drv_tab
 type	DRV_EV_STATE	LONG_LIVED	SYSTEM		driver_event_state
 type	DRV_EV_D_STATE	FIXED_SIZE	SYSTEM		driver_event_data_state
 type	DRV_SEL_D_STATE	FIXED_SIZE	SYSTEM		driver_select_data_state
+type	NIF_SEL_D_STATE	FIXED_SIZE	SYSTEM		enif_select_data_state
 type	FD_LIST		SHORT_LIVED	SYSTEM		fd_list
 type	ACTIVE_FD_ARR	SHORT_LIVED	SYSTEM		active_fd_array
 type	POLLSET		LONG_LIVED	SYSTEM		pollset
diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 97a69140c3..5bea92e198 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -76,11 +76,10 @@ typedef struct {
 #  endif
 #endif
 
-/* Values for mode arg to driver_select() */
-#define ERL_DRV_READ  (1 << 0)
-#define ERL_DRV_WRITE (1 << 1)
-#define ERL_DRV_USE   (1 << 2)
-#define ERL_DRV_USE_NO_CALLBACK (ERL_DRV_USE | (1 << 3))
+#define ERL_DRV_READ  ((int)ERL_NIF_SELECT_READ)
+#define ERL_DRV_WRITE ((int)ERL_NIF_SELECT_WRITE)
+#define ERL_DRV_USE   ((int)ERL_NIF_SELECT_STOP)
+#define ERL_DRV_USE_NO_CALLBACK (ERL_DRV_USE | (ERL_DRV_USE  << 1))
 
 /* Old deprecated */
 #define DO_READ  ERL_DRV_READ
diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h
index 2489099b5c..e4ebcdb1d4 100644
--- a/erts/emulator/beam/erl_drv_nif.h
+++ b/erts/emulator/beam/erl_drv_nif.h
@@ -49,6 +49,13 @@ typedef enum {
     ERL_DIRTY_JOB_IO_BOUND  = 2
 } ErlDirtyJobFlags;
 
+/* Values for enif_select AND mode arg for driver_select() */
+enum ErlNifSelectFlags {
+    ERL_NIF_SELECT_READ      = (1 << 0),
+    ERL_NIF_SELECT_WRITE     = (1 << 1),
+    ERL_NIF_SELECT_STOP      = (1 << 2)
+};
+
 #ifdef SIZEOF_CHAR
 #  define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR
 #  undef SIZEOF_CHAR
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 202f713f67..27abba7cfd 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1904,37 +1904,10 @@ int enif_snprintf(char *buffer, size_t size, const char* format, ...)
  **       Memory managed (GC'ed) "resource" objects       **
  ***********************************************************/
 
-
-struct enif_resource_type_t
-{
-    struct enif_resource_type_t* next;   /* list of all resource types */
-    struct enif_resource_type_t* prev;    
-    struct erl_module_nif* owner;  /* that created this type and thus implements the destructor*/
-    ErlNifResourceDtor* dtor;      /* user destructor function */
-    erts_refc_t refc;  /* num of resources of this type (HOTSPOT warning)
-                          +1 for active erl_module_nif */
-    Eterm module;
-    Eterm name;
-};
-
 /* dummy node in circular list */
 struct enif_resource_type_t resource_type_list; 
 
-typedef struct enif_resource_t
-{
-    struct enif_resource_type_t* type;
-#ifdef DEBUG
-    erts_refc_t nif_refc;
-# ifdef ARCH_32
-    byte align__[4];
-# endif
-#endif
-
-    char data[1];
-}ErlNifResource;
-
 #define SIZEOF_ErlNifResource(SIZE) (offsetof(ErlNifResource,data) + (SIZE))
-#define DATA_TO_RESOURCE(PTR) ((ErlNifResource*)((char*)(PTR) - offsetof(ErlNifResource,data)))
 
 static ErlNifResourceType* find_resource_type(Eterm module, Eterm name)
 {
@@ -1997,24 +1970,23 @@ struct opened_resource_type
 
     ErlNifResourceFlags op;
     ErlNifResourceType* type;
-    ErlNifResourceDtor* new_dtor;
+    ErlNifResourceTypeInit new_callbacks;
 };
 static struct opened_resource_type* opened_rt_list = NULL;
 
-ErlNifResourceType*
-enif_open_resource_type(ErlNifEnv* env,
-			const char* module_str, 
-			const char* name_str, 
-			ErlNifResourceDtor* dtor,
-			ErlNifResourceFlags flags,
-			ErlNifResourceFlags* tried)
+static
+ErlNifResourceType* open_resource_type(ErlNifEnv* env,
+                                       const char* name_str,
+                                       const ErlNifResourceTypeInit* init,
+                                       ErlNifResourceFlags flags,
+                                       ErlNifResourceFlags* tried,
+                                       size_t sizeof_init)
 {
     ErlNifResourceType* type = NULL;
     ErlNifResourceFlags op = flags;
     Eterm module_am, name_am;
 
     ASSERT(erts_smp_thr_progress_is_blocking());
-    ASSERT(module_str == NULL); /* for now... */
     module_am = make_atom(env->mod_nif->mod->module);
     name_am = enif_make_atom(env, name_str);
 
@@ -2048,7 +2020,9 @@ enif_open_resource_type(ErlNifEnv* env,
 						sizeof(struct opened_resource_type));
 	ort->op = op;
 	ort->type = type;
-	ort->new_dtor = dtor;
+        sys_memzero(&ort->new_callbacks, sizeof(ErlNifResourceTypeInit));
+        ASSERT(sizeof_init > 0 && sizeof_init <= sizeof(ErlNifResourceTypeInit));
+        sys_memcpy(&ort->new_callbacks, init, sizeof_init);
 	ort->next = opened_rt_list;
 	opened_rt_list = ort;
     }
@@ -2058,6 +2032,31 @@ enif_open_resource_type(ErlNifEnv* env,
     return type;
 }
 
+ErlNifResourceType*
+enif_open_resource_type(ErlNifEnv* env,
+                        const char* module_str,
+                        const char* name_str,
+			ErlNifResourceDtor* dtor,
+			ErlNifResourceFlags flags,
+			ErlNifResourceFlags* tried)
+{
+    ErlNifResourceTypeInit init =  {dtor, NULL};
+    ASSERT(module_str == NULL); /* for now... */
+    return open_resource_type(env, name_str, &init, flags, tried,
+                              sizeof(init));
+}
+
+ErlNifResourceType*
+enif_open_resource_type_x(ErlNifEnv* env,
+                          const char* name_str,
+                          const ErlNifResourceTypeInit* init,
+                          ErlNifResourceFlags flags,
+                          ErlNifResourceFlags* tried)
+{
+    return open_resource_type(env, name_str, init, flags, tried,
+                              env->mod_nif->entry.sizeof_ErlNifResourceTypeInit);
+}
+
 static void commit_opened_resource_types(struct erl_module_nif* lib)
 {
     while (opened_rt_list) {
@@ -2076,7 +2075,8 @@ static void commit_opened_resource_types(struct erl_module_nif* lib)
 	}
 
 	type->owner = lib;
-	type->dtor = ort->new_dtor;
+	type->dtor = ort->new_callbacks.dtor;
+        type->stop = ort->new_callbacks.stop;
 
 	if (type->dtor != NULL) {
 	    erts_refc_inc(&lib->rt_dtor_cnt, 1);
@@ -2124,6 +2124,15 @@ static void nif_resource_dtor(Binary* bin)
     }
 }
 
+void erts_resource_stop(ErlNifResource* resource)
+{
+    struct enif_msg_environment_t msg_env;
+    ASSERT(resource->type->stop);
+    pre_nif_noproc(&msg_env, resource->type->owner, NULL);
+    resource->type->stop(&msg_env.env, resource->data);
+    post_nif_noproc(&msg_env);
+}
+
 void* enif_alloc_resource(ErlNifResourceType* type, size_t size)
 {
     Binary* bin = erts_create_magic_binary_x(SIZEOF_ErlNifResource(size),
@@ -3174,6 +3183,11 @@ static struct erl_module_nif* create_lib(const ErlNifEntry* src)
         dst->funcs = lib->_funcs_copy_;
         dst->options = 0;
     }
+    if (AT_LEAST_VERSION(src, 2, 12)) {
+        dst->sizeof_ErlNifResourceTypeInit = src->sizeof_ErlNifResourceTypeInit;
+    } else {
+        dst->sizeof_ErlNifResourceTypeInit = 0;
+    }
     return lib;
 };
 
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 413b4f7343..ce8caaf729 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -52,7 +52,7 @@
 ** 2.11: 19.0 enif_snprintf 
 */
 #define ERL_NIF_MAJOR_VERSION 2
-#define ERL_NIF_MINOR_VERSION 11
+#define ERL_NIF_MINOR_VERSION 12
 
 /*
  * The emulator will refuse to load a nif-lib with a major version
@@ -122,6 +122,9 @@ typedef struct enif_entry_t
 
     /* Added in 2.7 */
     unsigned options;   /* Unused. Can be set to 0 or 1 (dirty sched config) */
+
+    /* Added in 2.12 */
+    size_t sizeof_ErlNifResourceTypeInit;
 }ErlNifEntry;
 
 
@@ -135,8 +138,20 @@ typedef struct
     void* ref_bin;
 }ErlNifBinary;
 
+typedef struct {
+    void (*dtor)(ErlNifEnv* env, void* obj);
+    void (*stop)(ErlNifEnv* env, void* obj); /* at ERL_NIF_SELECT_STOP event */
+} ErlNifResourceTypeInit;
+
 typedef struct enif_resource_type_t ErlNifResourceType;
 typedef void ErlNifResourceDtor(ErlNifEnv*, void*);
+typedef void ErlNifResourceStop(ErlNifEnv*, void*);
+typedef void ErlNifResourceExit(ErlNifEnv*, void*);
+
+//#ifndef ERL_SYS_DRV
+typedef int ErlNifEvent; /* An event to be selected on. */
+//#endif
+
 typedef enum
 {
     ERL_NIF_RT_CREATE = 1,
@@ -292,7 +307,8 @@ ERL_NIF_INIT_DECL(NAME)			\
 	FUNCS,				\
 	LOAD, RELOAD, UPGRADE, UNLOAD,	\
 	ERL_NIF_VM_VARIANT,		\
-	1		                \
+        1,                              \
+        sizeof(ErlNifResourceTypeInit)  \
     };                                  \
     ERL_NIF_INIT_BODY;                  \
     return &entry;			\
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 9a8f216773..9163ce25eb 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -175,6 +175,8 @@ ERL_NIF_API_FUNC_DECL(size_t, enif_binary_to_term, (ErlNifEnv *env, const unsign
 ERL_NIF_API_FUNC_DECL(int, enif_port_command, (ErlNifEnv *env, const ErlNifPort* to_port, ErlNifEnv *msg_env, ERL_NIF_TERM msg));
 ERL_NIF_API_FUNC_DECL(int,enif_thread_type,(void));
 ERL_NIF_API_FUNC_DECL(int,enif_snprintf,(char * buffer, size_t size, const char *format, ...));
+ERL_NIF_API_FUNC_DECL(int,enif_select,(ErlNifEnv* env, ErlNifEvent e, enum ErlNifSelectFlags flags, void* obj, ERL_NIF_TERM ref));
+ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type_x,(ErlNifEnv*, const char* name_str, const ErlNifResourceTypeInit*, ErlNifResourceFlags flags, ErlNifResourceFlags* tried));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
@@ -332,6 +334,8 @@ ERL_NIF_API_FUNC_DECL(int,enif_snprintf,(char * buffer, size_t size, const char
 #  define enif_port_command ERL_NIF_API_FUNC_MACRO(enif_port_command)
 #  define enif_thread_type ERL_NIF_API_FUNC_MACRO(enif_thread_type)
 #  define enif_snprintf ERL_NIF_API_FUNC_MACRO(enif_snprintf)
+#  define enif_select ERL_NIF_API_FUNC_MACRO(enif_select)
+#  define enif_open_resource_type_x ERL_NIF_API_FUNC_MACRO(enif_open_resource_type_x)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 2b2f3c5cdc..5c5693a315 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -42,6 +42,7 @@
 #include "erl_utils.h"
 #include "erl_port.h"
 #include "erl_gc.h"
+#include "erl_nif.h"
 
 struct enif_func_t;
 
@@ -58,6 +59,33 @@ struct enif_environment_t /* ErlNifEnv */
     Process *tracee;
     int exiting; /* boolean (dirty nifs might return in exiting state) */
 };
+struct enif_resource_type_t
+{
+    struct enif_resource_type_t* next;   /* list of all resource types */
+    struct enif_resource_type_t* prev;
+    struct erl_module_nif* owner;  /* that created this type and thus implements the destructor*/
+    ErlNifResourceDtor* dtor;      /* user destructor function */
+    ErlNifResourceStop* stop;
+    erts_refc_t refc;  /* num of resources of this type (HOTSPOT warning)
+                          +1 for active erl_module_nif */
+    Eterm module;
+    Eterm name;
+};
+typedef struct enif_resource_t
+{
+    struct enif_resource_type_t* type;
+#ifdef DEBUG
+    erts_refc_t nif_refc;
+# ifdef ARCH_32
+    byte align__[4];
+# endif
+#endif
+
+    char data[1];
+}ErlNifResource;
+
+#define DATA_TO_RESOURCE(PTR) ((ErlNifResource*)((char*)(PTR) - offsetof(ErlNifResource,data)))
+
 extern void erts_pre_nif(struct enif_environment_t*, Process*,
 			 struct erl_module_nif*, Process* tracee);
 extern void erts_post_nif(struct enif_environment_t* env);
@@ -67,6 +95,7 @@ extern void erts_pre_dirty_nif(ErtsSchedulerData *,
 			       struct erl_module_nif*);
 extern void erts_post_dirty_nif(struct enif_environment_t* env);
 #endif
+extern void erts_resource_stop(ErlNifResource* resource);
 extern Eterm erts_nif_taints(Process* p);
 extern void erts_print_nif_taints(fmtfn_t to, void* to_arg);
 void erts_unload_nif(struct erl_module_nif* nif);
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 2c138e1746..6f61fc8a28 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -53,6 +53,8 @@ typedef char EventStateType;
 #define ERTS_EV_TYPE_DRV_SEL  ((EventStateType) 1) /* driver_select */
 #define ERTS_EV_TYPE_DRV_EV   ((EventStateType) 2) /* driver_event */
 #define ERTS_EV_TYPE_STOP_USE ((EventStateType) 3) /* pending stop_select */
+#define ERTS_EV_TYPE_NIF      ((EventStateType) 4) /* enif_select */
+#define ERTS_EV_TYPE_STOP_NIF ((EventStateType) 5) /* pending nif stop */
 
 typedef char EventStateFlags;
 #define ERTS_EV_FLAG_USED         ((EventStateFlags) 1)   /* ERL_DRV_USE has been turned on */
@@ -122,7 +124,11 @@ typedef struct {
 #if ERTS_CIO_HAVE_DRV_EVENT
 	ErtsDrvEventDataState *event;     /* ERTS_EV_TYPE_DRV_EV */
 #endif
-	erts_driver_t* drv_ptr;           /* ERTS_EV_TYPE_STOP_USE */
+        ErtsNifSelectDataState *nif;      /* ERTS_EV_TYPE_NIF */
+        union {
+            erts_driver_t*  drv_ptr;    /* ERTS_EV_TYPE_STOP_USE */
+            ErlNifResource* resource;   /* ERTS_EV_TYPE_STOP_NIF */
+        }stop;
     } driver;
     ErtsPollEvents events;
     unsigned short remove_cnt; /* number of removed_fd's referring to this fd */
@@ -212,11 +218,18 @@ static ERTS_INLINE void hash_erase_drv_ev_state(ErtsDrvEventState *state)
 
 static void stale_drv_select(Eterm id, ErtsDrvEventState *state, int mode);
 static void drv_select_steal(ErlDrvPort ix, ErtsDrvEventState *state,
-			 int mode, int on);
+                             int mode, int on);
+static void nif_select_steal(ErtsDrvEventState *state, int mode,
+                             ErlNifResource* resource, Eterm ref);
+
 static void print_drv_select_op(erts_dsprintf_buf_t *dsbufp,
                                 ErlDrvPort ix, ErtsSysFdType fd, int mode, int on);
+static void print_nif_select_op(erts_dsprintf_buf_t*, ErtsSysFdType,
+                                int mode, ErlNifResource*, Eterm ref);
+
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-static void select_large_fd_error(ErlDrvPort, ErtsSysFdType, int, int);
+static void drv_select_large_fd_error(ErlDrvPort, ErtsSysFdType, int, int);
+static void nif_select_large_fd_error(ErtsSysFdType, int, ErlNifResource*,Eterm ref);
 #endif
 #if ERTS_CIO_HAVE_DRV_EVENT
 static void drv_event_steal(ErlDrvPort ix, ErtsDrvEventState *state,
@@ -227,8 +240,12 @@ static void print_drv_event_op(erts_dsprintf_buf_t *dsbufp,
 static void event_large_fd_error(ErlDrvPort, ErtsSysFdType, ErlDrvEventData);
 #endif
 #endif
-static void steal_pending_stop_use(erts_dsprintf_buf_t*, ErlDrvPort,
-                                   ErtsDrvEventState*, int mode, int on);
+static void
+steal_pending_stop_use(erts_dsprintf_buf_t*, ErlDrvPort, ErtsDrvEventState*,
+                       int mode, int on);
+static void
+steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErlNifResource*,
+                       ErtsDrvEventState *state, int mode, int on);
 
 #ifdef ERTS_SMP
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(removed_fd, struct removed_fd, 64, ERTS_ALC_T_FD_LIST)
@@ -263,6 +280,18 @@ alloc_drv_select_data(void)
     return dsp;
 }
 
+static ERTS_INLINE ErtsNifSelectDataState *
+alloc_nif_select_data(void)
+{
+    ErtsNifSelectDataState *dsp = erts_alloc(ERTS_ALC_T_NIF_SEL_D_STATE,
+					     sizeof(ErtsNifSelectDataState));
+    dsp->in.pid = NIL;
+    dsp->out.pid = NIL;
+    dsp->in.ddeselect_cnt = 0;
+    dsp->out.ddeselect_cnt = 0;
+    return dsp;
+}
+
 static ERTS_INLINE void
 free_drv_select_data(ErtsDrvSelectDataState *dsp)
 {
@@ -271,6 +300,12 @@ free_drv_select_data(ErtsDrvSelectDataState *dsp)
     erts_free(ERTS_ALC_T_DRV_SEL_D_STATE, dsp);   
 }
 
+static ERTS_INLINE void
+free_nif_select_data(ErtsNifSelectDataState *dsp)
+{
+    erts_free(ERTS_ALC_T_NIF_SEL_D_STATE, dsp);
+}
+
 #if ERTS_CIO_HAVE_DRV_EVENT
 
 static ERTS_INLINE ErtsDrvEventDataState *
@@ -352,6 +387,7 @@ forget_removed(struct pollset_info* psi)
     erts_smp_spin_unlock(&psi->removed_list_lock);
 
     while (fdlp) {
+        ErlNifResource* resource = NULL;
 	erts_driver_t* drv_ptr = NULL;
 	erts_smp_mtx_t* mtx;
 	ErtsSysFdType fd;
@@ -372,15 +408,25 @@ forget_removed(struct pollset_info* psi)
 	ASSERT(state->remove_cnt > 0);
 	if (--state->remove_cnt == 0) {
 	    switch (state->type) {
+	    case ERTS_EV_TYPE_STOP_NIF:
+		/* Now we can call stop */
+		resource = state->driver.stop.resource;
+                state->driver.stop.resource = NULL;
+                ASSERT(resource);
+		state->type = ERTS_EV_TYPE_NONE;
+		state->flags &= ~ERTS_EV_FLAG_USED;
+                goto case_ERTS_EV_TYPE_NONE;
+
 	    case ERTS_EV_TYPE_STOP_USE:
 		/* Now we can call stop_select */
-		drv_ptr = state->driver.drv_ptr;
+		drv_ptr = state->driver.stop.drv_ptr;
 		ASSERT(drv_ptr);
 		state->type = ERTS_EV_TYPE_NONE;
 		state->flags &= ~ERTS_EV_FLAG_USED;
-		state->driver.drv_ptr = NULL;  
+		state->driver.stop.drv_ptr = NULL;
 		/* Fall through */
-	    case ERTS_EV_TYPE_NONE:
+            case ERTS_EV_TYPE_NONE:
+            case_ERTS_EV_TYPE_NONE:
 #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
 		hash_erase_drv_ev_state(state);
 #endif
@@ -403,6 +449,11 @@ forget_removed(struct pollset_info* psi)
 		erts_ddll_dereference_driver(drv_ptr->handle);
 	    }
 	}
+        if (resource) {
+            erts_resource_stop(resource);
+            enif_release_resource(resource->data);
+        }
+
 	tofree = fdlp;
 	fdlp = fdlp->next;
 	removed_fd_free(tofree);
@@ -440,7 +491,8 @@ grow_drv_ev_state(int min_ix)
 #if ERTS_CIO_HAVE_DRV_EVENT
 	    drv_ev_state[i].driver.event = NULL;
 #endif
-	    drv_ev_state[i].driver.drv_ptr = NULL;
+	    drv_ev_state[i].driver.stop.drv_ptr = NULL;
+            drv_ev_state[i].driver.nif = NULL;
 	    drv_ev_state[i].events = 0;
 	    drv_ev_state[i].remove_cnt = 0;
 	    drv_ev_state[i].type = ERTS_EV_TYPE_NONE;
@@ -480,6 +532,7 @@ abort_tasks(ErtsDrvEventState *state, int mode)
 		       ERTS_EV_TYPE_DRV_EV);
 	    return;
 #endif
+        case ERTS_EV_TYPE_NIF:
 	case ERTS_EV_TYPE_NONE:
 	    return;
 	default:
@@ -534,6 +587,14 @@ deselect(ErtsDrvEventState *state, int mode)
 
     if (!(state->events)) {
 	switch (state->type) {
+        case ERTS_EV_TYPE_NIF:
+            state->driver.nif->in.pid = NIL;
+            state->driver.nif->out.pid = NIL;
+            state->driver.nif->in.ddeselect_cnt = 0;
+            state->driver.nif->out.ddeselect_cnt = 0;
+            enif_release_resource(state->driver.stop.resource);
+            state->driver.stop.resource = NULL;
+            break;
 	case ERTS_EV_TYPE_DRV_SEL:
 	    state->driver.select->inport = NIL;
 	    state->driver.select->outport = NIL;
@@ -569,7 +630,8 @@ check_fd_cleanup(ErtsDrvEventState *state,
 #if ERTS_CIO_HAVE_DRV_EVENT
 		 ErtsDrvEventDataState **free_event,
 #endif
-		 ErtsDrvSelectDataState **free_select)
+		 ErtsDrvSelectDataState **free_select,
+                 ErtsNifSelectDataState **free_nif)
 {
     erts_aint_t current_cio_time;
 
@@ -586,6 +648,12 @@ check_fd_cleanup(ErtsDrvEventState *state,
 	state->driver.select = NULL;
     }
 
+    *free_nif = NULL;
+    if (state->driver.nif && (state->type != ERTS_EV_TYPE_NIF)) {
+        *free_nif = state->driver.nif;
+        state->driver.nif = NULL;
+    }
+
 #if ERTS_CIO_HAVE_DRV_EVENT
     *free_event = NULL;
     if (state->driver.event
@@ -617,12 +685,14 @@ check_cleanup_active_fd(ErtsSysFdType fd,
 			ErtsPollControlEntry *pce,
 			int *pce_ix,
 #endif
-			erts_aint_t current_cio_time)
+			erts_aint_t current_cio_time,
+                        int may_sleep)
 {
     ErtsDrvEventState *state;
     int active = 0;
     erts_smp_mtx_t *mtx = fd_mtx(fd);
     void *free_select = NULL;
+    void *free_nif = NULL;
 #if ERTS_CIO_HAVE_DRV_EVENT
     void *free_event = NULL;
 #endif
@@ -682,6 +752,41 @@ check_cleanup_active_fd(ErtsSysFdType fd,
 	    }
 	}
 
+        if (state->driver.nif) {
+#if ERTS_CIO_DEFER_ACTIVE_EVENTS
+#  error Windows
+#endif
+            int do_wake = 0;
+            ErtsPollEvents rm_events = 0;
+            if (state->driver.nif->in.ddeselect_cnt) {
+                ASSERT(state->type == ERTS_EV_TYPE_NIF);
+                ASSERT(state->events & ERTS_POLL_EV_IN);
+                ASSERT(is_nil(state->driver.nif->in.pid));
+                if (may_sleep || state->driver.nif->in.ddeselect_cnt == 1) {
+                    rm_events = ERTS_POLL_EV_IN;
+                    state->driver.nif->in.ddeselect_cnt = 0;
+                }
+            }
+            if (state->driver.nif->out.ddeselect_cnt) {
+                ASSERT(state->type == ERTS_EV_TYPE_NIF);
+                ASSERT(state->events & ERTS_POLL_EV_OUT);
+                ASSERT(is_nil(state->driver.nif->out.pid));
+                if (may_sleep || state->driver.nif->out.ddeselect_cnt == 1) {
+                    rm_events |= ERTS_POLL_EV_OUT;
+                    state->driver.nif->out.ddeselect_cnt = 0;
+                }
+            }
+            if (rm_events) {
+                state->events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, rm_events, 0, &do_wake);
+            }
+            if (state->events)
+                active = 1;
+            else if (state->type != ERTS_EV_TYPE_NIF) {
+                free_nif = state->driver.nif;
+                state->driver.nif = NULL;
+            }
+        }
+
 #if ERTS_CIO_HAVE_DRV_EVENT
 	if (state->driver.event) {
 	    if (is_iotask_active(&state->driver.event->iotask, current_cio_time)) {
@@ -722,6 +827,8 @@ check_cleanup_active_fd(ErtsSysFdType fd,
 
     if (free_select)
 	free_drv_select_data(free_select);
+    if (free_nif)
+        free_nif_select_data(free_nif);
 #if ERTS_CIO_HAVE_DRV_EVENT
     if (free_event)
 	free_drv_event_data(free_event);
@@ -746,7 +853,7 @@ check_cleanup_active_fd(ErtsSysFdType fd,
 }
 
 static void
-check_cleanup_active_fds(erts_aint_t current_cio_time)
+check_cleanup_active_fds(erts_aint_t current_cio_time, int may_sleep)
 {
     int six = pollset.active_fd.six;
     int eix = pollset.active_fd.eix;
@@ -773,7 +880,8 @@ check_cleanup_active_fds(erts_aint_t current_cio_time)
 				     pctrl_entries,
 				     &pctrl_ix,
 #endif
-				     current_cio_time)) {
+				     current_cio_time,
+                                     may_sleep)) {
 	    no--;
 	    if (ix == six) {
 #ifdef DEBUG
@@ -831,7 +939,6 @@ add_active_fd(ErtsSysFdType fd)
     int eix = pollset.active_fd.eix;
     int size = pollset.active_fd.size;
     
-
     pollset.active_fd.array[eix] = fd;
 
     erts_smp_atomic32_set_relb(&pollset.active_fd.no,
@@ -867,6 +974,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
     ErtsDrvEventDataState *free_event = NULL;
 #endif
     ErtsDrvSelectDataState *free_select = NULL;
+    ErtsNifSelectDataState *free_nif = NULL;
 #ifdef USE_VM_PROBES
     DTRACE_CHARBUF(name, 64);
 #endif
@@ -882,7 +990,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
 	    return -1;    
 	}
 	if (fd >= max_fds) {
-	    select_large_fd_error(ix, fd, mode, on);
+	    drv_select_large_fd_error(ix, fd, mode, on);
 	    return -1;
 	}
 	grow_drv_ev_state(fd);
@@ -920,20 +1028,32 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
     }
 #endif
 
+    switch (state->type) {
 #if ERTS_CIO_HAVE_DRV_EVENT
-    if (state->type == ERTS_EV_TYPE_DRV_EV)
-	drv_select_steal(ix, state, mode, on);
+    case ERTS_EV_TYPE_DRV_EV:
 #endif
-    if (state->type == ERTS_EV_TYPE_STOP_USE) {
-	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	print_drv_select_op(dsbufp, ix, state->fd, mode, on);
-	steal_pending_stop_use(dsbufp, ix, state, mode, on);
-	if (state->type == ERTS_EV_TYPE_STOP_USE) {
-	    ret = 0;
-	    goto done; /* stop_select still pending */
-	}
-	ASSERT(state->type == ERTS_EV_TYPE_NONE);
-    }
+    case ERTS_EV_TYPE_NIF:
+        drv_select_steal(ix, state, mode, on);
+        break;
+    case ERTS_EV_TYPE_STOP_USE: {
+        erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
+        print_drv_select_op(dsbufp, ix, state->fd, mode, on);
+        steal_pending_stop_use(dsbufp, ix, state, mode, on);
+        if (state->type == ERTS_EV_TYPE_STOP_USE) {
+            ret = 0;
+            goto done; /* stop_select still pending */
+        }
+        ASSERT(state->type == ERTS_EV_TYPE_NONE);
+        break;
+    }
+    case ERTS_EV_TYPE_STOP_NIF: {
+        erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
+        print_drv_select_op(dsbufp, ix, state->fd, mode, on);
+        steal_pending_stop_nif(dsbufp, NULL, state, mode, on);
+        ASSERT(state->type == ERTS_EV_TYPE_NONE);
+        break;
+
+    }}
 
     if (mode & ERL_DRV_READ) {
 	if (state->type == ERTS_EV_TYPE_DRV_SEL) {
@@ -1037,7 +1157,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
 		else {
 		    /* Not safe to close fd, postpone stop_select callback. */
 		    state->type = ERTS_EV_TYPE_STOP_USE;
-		    state->driver.drv_ptr = drv_ptr;
+		    state->driver.stop.drv_ptr = drv_ptr;
 		    if (drv_ptr->handle) {
 			erts_ddll_reference_referenced_driver(drv_ptr->handle);
 		    }
@@ -1054,7 +1174,8 @@ done:
 #if ERTS_CIO_HAVE_DRV_EVENT
 		     &free_event,
 #endif
-		     &free_select);
+		     &free_select,
+                     &free_nif);
 
 done_unknown:
     erts_smp_mtx_unlock(fd_mtx(fd));
@@ -1067,6 +1188,9 @@ done_unknown:
     }
     if (free_select)
 	free_drv_select_data(free_select);
+    if (free_nif)
+        free_nif_select_data(free_nif);
+
 #if ERTS_CIO_HAVE_DRV_EVENT
     if (free_event)
 	free_drv_event_data(free_event);
@@ -1074,6 +1198,269 @@ done_unknown:
     return ret;
 }
 
+int
+ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
+                             ErlNifEvent e,
+                             enum ErlNifSelectFlags mode,
+                             void* obj,
+                             Eterm ref)
+{
+    int on;
+    const Eterm id = env->proc->common.id;
+    ErlNifResource* resource = DATA_TO_RESOURCE(obj);
+    ErtsSysFdType fd = (ErtsSysFdType) e;
+    ErtsPollEvents ctl_events = (ErtsPollEvents) 0;
+    ErtsPollEvents new_events, old_events;
+    ErtsDrvEventState *state;
+    int wake_poller;
+    int ret;
+    enum { NO_STOP=0, CALL_STOP, CALL_STOP_AND_RELEASE } call_stop = NO_STOP;
+#if ERTS_CIO_HAVE_DRV_EVENT
+    ErtsDrvEventDataState *free_event = NULL;
+#endif
+    ErtsDrvSelectDataState *free_select = NULL;
+    ErtsNifSelectDataState *free_nif = NULL;
+#ifdef USE_VM_PROBES
+    DTRACE_CHARBUF(name, 64);
+#endif
+
+#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
+    if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) {
+	if (fd < 0) {
+	    return -1;
+	}
+	if (fd >= max_fds) {
+	    nif_select_large_fd_error(fd, mode, resource, ref);
+            return -1;
+	}
+	grow_drv_ev_state(fd);
+    }
+#endif
+
+    erts_smp_mtx_lock(fd_mtx(fd));
+
+#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
+    state = &drv_ev_state[(int) fd];
+#else
+    state = hash_get_drv_ev_state(fd); /* may be NULL! */
+#endif
+
+    if (mode & ERL_NIF_SELECT_STOP) {
+        ASSERT(resource->type->stop);
+        if (IS_FD_UNKNOWN(state)) {
+            /* fast track to stop callback */
+            call_stop = CALL_STOP;
+            ret = 0;
+            goto done_unknown;
+        }
+        on = 0;
+        mode = ERL_DRV_READ | ERL_DRV_WRITE | ERL_DRV_USE;
+        wake_poller = 1; /* to eject fd from pollset (if needed) */
+    }
+    else {
+        on = 1;
+        ASSERT(mode);
+        wake_poller = 0;
+    }
+
+#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
+    if (state == NULL) {
+	state = hash_new_drv_ev_state(fd);
+    }
+#endif
+
+    switch (state->type) {
+    case ERTS_EV_TYPE_NIF:
+        /*
+         * Changing resource is considered stealing.
+         * Changing process and/or ref is ok (I think?).
+         */
+        if (state->driver.stop.resource != resource)
+            nif_select_steal(state, ERL_DRV_READ | ERL_DRV_WRITE, resource, ref);
+        break;
+#if ERTS_CIO_HAVE_DRV_EVENT
+    case ERTS_EV_TYPE_DRV_EV:
+#endif
+    case ERTS_EV_TYPE_DRV_SEL:
+        nif_select_steal(state, mode, resource, ref);
+        break;
+    case ERTS_EV_TYPE_STOP_USE: {
+        erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
+        print_nif_select_op(dsbufp, fd, mode, resource, ref);
+        steal_pending_stop_use(dsbufp, ERTS_INVALID_ERL_DRV_PORT, state, mode, on);
+        ASSERT(state->type == ERTS_EV_TYPE_NONE);
+        break;
+    }
+    case ERTS_EV_TYPE_STOP_NIF: {
+        erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
+        print_nif_select_op(dsbufp, fd, mode, resource, ref);
+        steal_pending_stop_nif(dsbufp, resource, state, mode, on);
+        if (state->type == ERTS_EV_TYPE_STOP_NIF) {
+            ret = 0;
+            goto done;
+        }
+        ASSERT(state->type == ERTS_EV_TYPE_NONE);
+        break;
+    }}
+
+    ASSERT(state->type == ERTS_EV_TYPE_NONE ||
+           state->type == ERTS_EV_TYPE_NIF);
+
+    if (mode & ERL_DRV_READ) {
+	ctl_events |= ERTS_POLL_EV_IN;
+    }
+    if (mode & ERL_DRV_WRITE) {
+	ctl_events |= ERTS_POLL_EV_OUT;
+    }
+
+    ASSERT((state->type == ERTS_EV_TYPE_NIF) ||
+	   (state->type == ERTS_EV_TYPE_NONE && !state->events));
+
+    new_events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, ctl_events, on, &wake_poller);
+
+    if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
+	if (state->type == ERTS_EV_TYPE_NIF && !state->events) {
+	    state->type = ERTS_EV_TYPE_NONE;
+	    state->flags &= ~ERTS_EV_FLAG_USED;
+	    state->driver.nif->in.pid = NIL;
+	    state->driver.nif->out.pid = NIL;
+            state->driver.nif->in.ddeselect_cnt = 0;
+            state->driver.nif->out.ddeselect_cnt = 0;
+            state->driver.stop.resource = NULL;
+	}
+	ret = -1;
+	goto done;
+    }
+
+    old_events = state->events;
+
+    ASSERT(on
+	   ? (new_events == (state->events | ctl_events))
+	   : (new_events == (state->events & ~ctl_events)));
+
+    ASSERT(state->type == ERTS_EV_TYPE_NIF
+	   || state->type == ERTS_EV_TYPE_NONE);
+
+    state->events = new_events;
+    if (ctl_events) {
+	if (on) {
+            Uint32* refn;
+	    if (!state->driver.nif)
+		state->driver.nif = alloc_nif_select_data();
+	    if (state->type == ERTS_EV_TYPE_NONE) {
+		state->type = ERTS_EV_TYPE_NIF;
+                state->driver.stop.resource = resource;
+                enif_keep_resource(resource->data);
+            }
+            ASSERT(state->type == ERTS_EV_TYPE_NIF);
+            ASSERT(state->driver.stop.resource == resource);
+	    if (ctl_events & ERTS_POLL_EV_IN) {
+                ASSERT(is_nil(state->driver.nif->in.pid));
+		state->driver.nif->in.pid = id;
+                if (is_immed(ref)) {
+                    state->driver.nif->in.immed = ref;
+                } else {
+                    ASSERT(is_internal_ref(ref));
+                    refn = internal_ref_numbers(ref);
+                    state->driver.nif->in.immed = THE_NON_VALUE;
+                    state->driver.nif->in.refn[0] = refn[0];
+                    state->driver.nif->in.refn[1] = refn[1];
+                    state->driver.nif->in.refn[2] = refn[2];
+                }
+                state->driver.nif->in.ddeselect_cnt = 0;
+            }
+	    if (ctl_events & ERTS_POLL_EV_OUT) {
+                ASSERT(is_nil(state->driver.nif->out.pid));
+		state->driver.nif->out.pid = id;
+                if (is_immed(ref)) {
+                    state->driver.nif->out.immed = ref;
+                } else {
+                    ASSERT(is_internal_ref(ref));
+                    refn = internal_ref_numbers(ref);
+                    state->driver.nif->out.immed = THE_NON_VALUE;
+                    state->driver.nif->out.refn[0] = refn[0];
+                    state->driver.nif->out.refn[1] = refn[1];
+                    state->driver.nif->out.refn[2] = refn[2];
+                }
+                state->driver.nif->out.ddeselect_cnt = 0;
+            }
+            state->flags |= ERTS_EV_FLAG_USED;
+	}
+	else { /* off */
+	    if (state->type == ERTS_EV_TYPE_NIF) {
+                //erts_fprintf(stderr, "SVERK: enif select clear fd=%d inpid=%T inrsrc=%p\n",
+                //             state->fd, state->driver.nif->inpid,
+                //             state->driver.nif->in.resource);
+                state->driver.nif->in.pid = NIL;
+                state->driver.nif->out.pid = NIL;
+                state->driver.nif->in.ddeselect_cnt = 0;
+                state->driver.nif->out.ddeselect_cnt = 0;
+                state->flags &= ~ERTS_EV_FLAG_USED;
+                if (old_events != 0) {
+                    remember_removed(state, &pollset);
+                }
+	    }
+            ASSERT(new_events==0);
+            if (state->remove_cnt == 0 || !wake_poller) {
+                /* Safe to close fd now as it is not in pollset
+		   or there was no need to eject fd (kernel poll) */
+                //erts_fprintf(stderr, "SVERK : enif_select calling stop before return\n");
+                if (state->type == ERTS_EV_TYPE_NIF) {
+                    ASSERT(state->driver.stop.resource == resource);
+                    call_stop = CALL_STOP_AND_RELEASE;
+                    state->driver.stop.resource = NULL;
+                }
+                else {
+                    ASSERT(!state->driver.stop.resource);
+                    call_stop = CALL_STOP;
+                }
+                state->type = ERTS_EV_TYPE_NONE;
+            }
+            else {
+                /* Not safe to close fd, postpone stop_select callback. */
+                //erts_fprintf(stderr, "SVERK: enif_select schedule stop\n");
+                if (state->type == ERTS_EV_TYPE_NONE) {
+                    ASSERT(!state->driver.stop.resource);
+                    state->driver.stop.resource = resource;
+                    enif_keep_resource(resource);
+                }
+                state->type = ERTS_EV_TYPE_STOP_NIF;
+	    }
+	}
+    }
+
+    ret = 0;
+
+done:
+
+    check_fd_cleanup(state,
+#if ERTS_CIO_HAVE_DRV_EVENT
+		     &free_event,
+#endif
+		     &free_select,
+                     &free_nif);
+
+done_unknown:
+    erts_smp_mtx_unlock(fd_mtx(fd));
+    if (call_stop) {
+        erts_resource_stop(resource);
+        if (call_stop == CALL_STOP_AND_RELEASE) {
+            enif_release_resource(resource->data);
+        }
+    }
+    if (free_select)
+	free_drv_select_data(free_select);
+    if (free_nif)
+        free_nif_select_data(free_nif);
+
+#if ERTS_CIO_HAVE_DRV_EVENT
+    if (free_event)
+	free_drv_event_data(free_event);
+#endif
+    return ret;
+}
+
+
 int
 ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix,
 			      ErlDrvEvent e,
@@ -1094,6 +1481,7 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix,
     ErtsDrvEventDataState *free_event;
 #endif
     ErtsDrvSelectDataState *free_select;
+    ErtsNifSelectDataState *free_nif;
     Port *prt = erts_drvport2port(ix);
 
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
@@ -1203,12 +1591,15 @@ done:
 #if ERTS_CIO_HAVE_DRV_EVENT
 		     &free_event,
 #endif
-		     &free_select);
+		     &free_select,
+                     &free_nif);
 
     erts_smp_mtx_unlock(fd_mtx(fd));
 
     if (free_select)
 	free_drv_select_data(free_select);
+    if (free_nif)
+        free_nif_select_data(free_nif);
 #if ERTS_CIO_HAVE_DRV_EVENT
     if (free_event)
 	free_drv_event_data(free_event);
@@ -1244,13 +1635,19 @@ need2steal(ErtsDrvEventState *state, int mode)
 				  state,
 				  ERL_DRV_WRITE);
 	break;
+    case ERTS_EV_TYPE_NIF:
+        ASSERT(state->driver.stop.resource);
+        do_steal = 1;
+        break;
+
 #if ERTS_CIO_HAVE_DRV_EVENT
     case ERTS_EV_TYPE_DRV_EV:
 	do_steal |= chk_stale(state->driver.event->port, state, 0);
 	break;
 #endif
     case ERTS_EV_TYPE_STOP_USE:
-	ASSERT(0);
+    case ERTS_EV_TYPE_STOP_NIF:
+        ASSERT(0);
 	break;
     default:
 	break;
@@ -1311,6 +1708,25 @@ steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode)
 	erts_dsprintf(dsbufp, "\n");
 	break;
     }
+    case ERTS_EV_TYPE_NIF: {
+        Eterm iid = state->driver.nif->in.pid;
+        Eterm oid = state->driver.nif->out.pid;
+        const char* with = "with";
+        ErlNifResourceType* rt = state->driver.stop.resource->type;
+
+        erts_dsprintf(dsbufp, "resource %T:%T", rt->module, rt->name);
+
+        if (is_not_nil(iid)) {
+            erts_dsprintf(dsbufp, " %s in-pid %T", with, iid);
+            with = "and";
+        }
+        if (is_not_nil(oid)) {
+            erts_dsprintf(dsbufp, " %s out-pid %T", with, oid);
+        }
+        deselect(state, 0);
+        erts_dsprintf(dsbufp, "\n");
+        break;
+    }
 #if ERTS_CIO_HAVE_DRV_EVENT
     case ERTS_EV_TYPE_DRV_EV: {
 	Eterm eid = state->driver.event->port;
@@ -1328,7 +1744,8 @@ steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode)
 	break;
     }
 #endif
-    case ERTS_EV_TYPE_STOP_USE: {
+    case ERTS_EV_TYPE_STOP_USE:
+    case ERTS_EV_TYPE_STOP_NIF: {
 	ASSERT(0);
 	break;
     }
@@ -1357,6 +1774,23 @@ print_drv_select_op(erts_dsprintf_buf_t *dsbufp,
     erts_dsprintf(dsbufp, "driver %T ", pp != ERTS_INVALID_ERL_DRV_PORT ? pp->common.id : NIL);
 }
 
+static void
+print_nif_select_op(erts_dsprintf_buf_t *dsbufp,
+                    ErtsSysFdType fd, int mode,
+                    ErlNifResource* resource, Eterm ref)
+{
+    erts_dsprintf(dsbufp,
+		  "enif_select(_, %d,%s%s%s, %T:%T, %T) ",
+		  (int) GET_FD(fd),
+		  mode & ERL_NIF_SELECT_READ ? " READ" : "",
+		  mode & ERL_NIF_SELECT_WRITE ? " WRITE" : "",
+		  mode & ERL_NIF_SELECT_STOP ? " STOP" : "",
+		  resource->type->module,
+                  resource->type->name,
+                  ref);
+}
+
+
 static void
 drv_select_steal(ErlDrvPort ix, ErtsDrvEventState *state, int mode, int on)
 {
@@ -1368,6 +1802,18 @@ drv_select_steal(ErlDrvPort ix, ErtsDrvEventState *state, int mode, int on)
     }
 }
 
+static void
+nif_select_steal(ErtsDrvEventState *state, int mode,
+                 ErlNifResource* resource, Eterm ref)
+{
+    if (need2steal(state, mode)) {
+	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
+	print_nif_select_op(dsbufp, state->fd, mode, resource, ref);
+	steal(dsbufp, state, mode);
+	erts_send_error_to_logger_nogl(dsbufp);
+    }
+}
+
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
 static void
 large_fd_error_common(erts_dsprintf_buf_t *dsbufp, ErtsSysFdType fd)
@@ -1378,7 +1824,7 @@ large_fd_error_common(erts_dsprintf_buf_t *dsbufp, ErtsSysFdType fd)
 }
 
 static void
-select_large_fd_error(ErlDrvPort ix, ErtsSysFdType fd, int mode, int on)
+drv_select_large_fd_error(ErlDrvPort ix, ErtsSysFdType fd, int mode, int on)
 {
     erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
     print_drv_select_op(dsbufp, ix, fd, mode, on);
@@ -1386,6 +1832,16 @@ select_large_fd_error(ErlDrvPort ix, ErtsSysFdType fd, int mode, int on)
     large_fd_error_common(dsbufp, fd);
     erts_send_error_to_logger_nogl(dsbufp);
 }
+static void
+nif_select_large_fd_error(ErtsSysFdType fd, int mode,
+                          ErlNifResource* resource, Eterm ref)
+{
+    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
+    print_nif_select_op(dsbufp, fd, mode, resource, ref);
+    erts_dsprintf(dsbufp, "failed: ");
+    large_fd_error_common(dsbufp, fd);
+    erts_send_error_to_logger_nogl(dsbufp);
+}
 #endif /* ERTS_SYS_CONTINOUS_FD_NUMBERS */
 
 
@@ -1394,38 +1850,78 @@ static void
 steal_pending_stop_use(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix,
                        ErtsDrvEventState *state, int mode, int on)
 {
+    int cancel = 0;
     ASSERT(state->type == ERTS_EV_TYPE_STOP_USE);
-    erts_dsprintf(dsbufp, "failed: fd=%d (re)selected before stop_select "
-		          "was called for driver %s\n",
-		  (int) GET_FD(state->fd), state->driver.drv_ptr->name);
-    erts_send_error_to_logger_nogl(dsbufp);
 
     if (on) {
 	/* Either fd-owner changed its mind about closing
 	 * or closed fd before stop_select callback and fd is now reused.
 	 * In either case stop_select should not be called.
-	 */	    
-	state->type = ERTS_EV_TYPE_NONE;
-	state->flags &= ~ERTS_EV_FLAG_USED;
-	if (state->driver.drv_ptr->handle) {
-	    erts_ddll_dereference_driver(state->driver.drv_ptr->handle);
-	}
-	state->driver.drv_ptr = NULL;
+	 */
+        cancel = 1;
     }
     else if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) {
 	Port *prt = erts_drvport2port(ix);
-	erts_driver_t* drv_ptr = prt != ERTS_INVALID_ERL_DRV_PORT ? prt->drv_ptr : NULL;
-	if (drv_ptr && drv_ptr != state->driver.drv_ptr) {
-	    /* Some other driver wants the stop_select callback */
-	    if (state->driver.drv_ptr->handle) {
-		erts_ddll_dereference_driver(state->driver.drv_ptr->handle);
-	    }
-	    if (drv_ptr->handle) {
-		erts_ddll_reference_referenced_driver(drv_ptr->handle);
-	    }
-	    state->driver.drv_ptr = drv_ptr;
-	}    
+	if (prt == ERTS_INVALID_ERL_DRV_PORT
+            || prt->drv_ptr != state->driver.stop.drv_ptr) {
+	    /* Some other driver or nif wants the stop_select callback */
+            cancel = 1;
+        }
+    }
+
+    if (cancel) {
+        erts_dsprintf(dsbufp, "called before stop_select was called for driver '%s'\n",
+                      state->driver.stop.drv_ptr->name);
+        if (state->driver.stop.drv_ptr->handle) {
+            erts_ddll_dereference_driver(state->driver.stop.drv_ptr->handle);
+        }
+        state->type = ERTS_EV_TYPE_NONE;
+        state->flags &= ~ERTS_EV_FLAG_USED;
+        state->driver.stop.drv_ptr = NULL;
+    }
+    else {
+        erts_dsprintf(dsbufp, "ignored repeated call\n");
+    }
+    erts_send_error_to_logger_nogl(dsbufp);
+}
+
+static void
+steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErlNifResource* resource,
+                       ErtsDrvEventState *state, int mode, int on)
+{
+    int cancel = 0;
+
+    ASSERT(state->type == ERTS_EV_TYPE_STOP_NIF);
+    ASSERT(state->driver.stop.resource);
+
+    if (on) {
+        ASSERT(mode & (ERL_NIF_SELECT_READ | ERL_NIF_SELECT_WRITE));
+        /* Either fd-owner changed its mind about closing
+         * or closed fd before stop callback and fd is now reused.
+         * In either case, stop should not be called.
+         */
+        cancel = 1;
     }
+    else if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE
+             && resource != state->driver.stop.resource) {
+        /* Some driver or other resource wants the stop callback */
+        cancel = 1;
+    }
+
+    if (cancel) {
+        ErlNifResourceType* rt = state->driver.stop.resource->type;
+        erts_dsprintf(dsbufp, "called before stop was called for NIF resource %T:%T\n",
+                      rt->module, rt->name);
+
+        enif_release_resource(state->driver.stop.resource);
+        state->type = ERTS_EV_TYPE_NONE;
+        state->flags &= ~ERTS_EV_FLAG_USED;
+        state->driver.stop.resource = NULL;
+    }
+    else {
+        erts_dsprintf(dsbufp, "ignored repeated call\n");
+    }
+    erts_send_error_to_logger_nogl(dsbufp);
 
 }
 
@@ -1557,6 +2053,48 @@ oready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time)
     }
 }
 
+static ERTS_INLINE void
+send_event_tuple(struct erts_nif_select_event* e, ErlNifResource* resource,
+                 Eterm event_atom)
+{
+    Process* rp = erts_proc_lookup(e->pid);
+    ErtsProcLocks rp_locks = 0;
+    ErtsMessage* mp;
+    ErlOffHeap* ohp;
+    ErtsBinary* bin;
+    Eterm* hp;
+    Uint hsz = 5 + PROC_BIN_SIZE + REF_THING_SIZE; /* {select, Resource, Ref, EventAtom} */
+    Eterm resource_term, ref_term, tuple;
+
+    if (!rp) {
+        erts_fprintf(stderr, "SVERK: Process %T not alive for msg %T\n", e->pid, event_atom);
+        return;
+    }
+
+    bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
+
+    mp = erts_alloc_message_heap(rp, &rp_locks, hsz, &hp, &ohp);
+
+    resource_term = erts_mk_magic_binary_term(&hp, ohp, &bin->binary);
+    if (is_value(e->immed)) {
+        ASSERT(is_immed(e->immed));
+        ref_term = e->immed;
+    }
+    else {
+        write_ref_thing(hp, e->refn[0], e->refn[1], e->refn[2]);
+        ref_term = make_internal_ref(hp);
+    }
+    hp += REF_THING_SIZE;
+    tuple = TUPLE4(hp, am_select, resource_term, ref_term, event_atom);
+
+    ERL_MESSAGE_TOKEN(mp) = am_undefined;
+    erts_queue_message(rp, rp_locks, mp, tuple, am_system);
+
+    if (rp_locks)
+        erts_smp_proc_unlock(rp, rp_locks);
+}
+
+
 #if ERTS_CIO_HAVE_DRV_EVENT
 static ERTS_INLINE void
 eready(Eterm id, ErtsDrvEventState *state, ErlDrvEventData event_data,
@@ -1602,6 +2140,8 @@ ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(int set,
     ERTS_CIO_POLL_INTR_TMD(pollset.ps, set, timeout_time);
 }
 
+#define ERTS_NIF_DELAYED_DESELECT 20
+
 void
 ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 {
@@ -1635,7 +2175,8 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
     current_cio_time++;
     erts_smp_atomic_set_relb(&erts_check_io_time, current_cio_time);
 
-    check_cleanup_active_fds(current_cio_time);
+    check_cleanup_active_fds(current_cio_time,
+                             timeout_time != ERTS_POLL_NO_TIMEOUT);
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_check_exact(NULL, 0); /* No locks should be locked */
@@ -1741,6 +2282,69 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 	    break;
 	}
 
+        case ERTS_EV_TYPE_NIF: { /* Requested via enif_select()... */
+            struct erts_nif_select_event in = {NIL};
+            struct erts_nif_select_event out = {NIL};
+            ErlNifResource* resource;
+            ErtsPollEvents revents = pollres[i].events;
+
+            if (revents & ERTS_POLL_EV_ERR) {
+                /*
+                 * Handle error events by triggering all in/out events
+                 * that the NIF has selected.
+                 * We *do not* want to send a message that corresponds
+		 * to an event not selected.
+                 */
+                revents = state->events;
+            }
+            else {
+                revents &= (state->events | ERTS_POLL_EV_NVAL);
+            }
+
+            if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) {
+                if (revents & ERTS_POLL_EV_OUT) {
+                    if (is_not_nil(state->driver.nif->out.pid)) {
+                        out = state->driver.nif->out;
+                        resource = state->driver.stop.resource;
+                        state->driver.nif->out.ddeselect_cnt = ERTS_NIF_DELAYED_DESELECT;
+                        state->driver.nif->out.pid = NIL;
+                        add_active_fd(state->fd);
+                    }
+                    else {
+                        ASSERT(state->driver.nif->out.ddeselect_cnt >= 2);
+                        state->driver.nif->out.ddeselect_cnt--;
+                    }
+                }
+                if (revents & ERTS_POLL_EV_IN) {
+                    if (is_not_nil(state->driver.nif->in.pid)) {
+                        in = state->driver.nif->in;
+                        resource = state->driver.stop.resource;
+                        state->driver.nif->in.ddeselect_cnt = ERTS_NIF_DELAYED_DESELECT;
+                        state->driver.nif->in.pid = NIL;
+                        add_active_fd(state->fd);
+                    }
+                    else {
+                        ASSERT(state->driver.nif->in.ddeselect_cnt >= 2);
+                        state->driver.nif->in.ddeselect_cnt--;
+                    }
+                }
+            }
+            else if (revents & ERTS_POLL_EV_NVAL) {
+                abort();
+            }
+
+#ifdef ERTS_SMP
+            erts_smp_mtx_unlock(fd_mtx(fd));
+#endif
+            if (is_not_nil(in.pid)) {
+                send_event_tuple(&in, resource, am_ready_input);
+            }
+            if (is_not_nil(out.pid)) {
+                send_event_tuple(&out, resource, am_ready_output);
+            }
+            goto next_pollres_unlocked;
+        }
+
 #if ERTS_CIO_HAVE_DRV_EVENT
 	case ERTS_EV_TYPE_DRV_EV: { /* Requested via driver_event()... */
 	    ErlDrvEventData event_data;
@@ -1781,6 +2385,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 #ifdef ERTS_SMP
 	erts_smp_mtx_unlock(fd_mtx(fd));
 #endif
+        next_pollres_unlocked:;
     }
 
     erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0);
@@ -2306,6 +2911,39 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters)
 		}
 	    }
 	}
+        else if (state->type == ERTS_EV_TYPE_NIF) {
+            ErlNifResource* r;
+            erts_printf("enif_select ");
+
+#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
+            if (internal) {
+                erts_printf("internal ");
+                err = 1;
+            }
+
+            if (cio_events == ep_events) {
+                erts_printf("ev=");
+                if (print_events(cio_events) != 0)
+                    err = 1;
+            }
+            else {
+                err = 1;
+                erts_printf("cio_ev=");
+                print_events(cio_events);
+                erts_printf(" ep_ev=");
+                print_events(ep_events);
+            }
+#else
+            if (print_events(cio_events) != 0)
+                err = 1;
+#endif
+            erts_printf(" inpid=%T dd_cnt=%b32d", state->driver.nif->in.pid,
+                        state->driver.nif->in.ddeselect_cnt);
+            erts_printf(" outpid=%T dd_cnt=%b32d", state->driver.nif->out.pid,
+                        state->driver.nif->out.ddeselect_cnt);
+            r = state->driver.stop.resource;
+            erts_printf(" resource=%p(%T:%T)", r, r->type->module, r->type->name);
+        }
 #if ERTS_CIO_HAVE_DRV_EVENT
 	else if (state->type == ERTS_EV_TYPE_DRV_EV) {
 	    Eterm id;
@@ -2362,11 +3000,12 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters)
 	    erts_printf("control_type=%d ", (int)state->type);
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
 	    if (cio_events == ep_events) {
-		erts_printf("ev=0x%b32x", (Uint32) cio_events);
+                erts_printf("ev=");
+                print_events(cio_events);
 	    }
 	    else {
-		erts_printf("cio_ev=0x%b32x", (Uint32) cio_events);
-		erts_printf(" ep_ev=0x%b32x", (Uint32) ep_events);
+		erts_printf("cio_ev="); print_events(cio_events);
+		erts_printf(" ep_ev="); print_events(ep_events);
 	    }
 #else
 	    erts_printf("ev=0x%b32x", (Uint32) cio_events);
@@ -2395,7 +3034,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
 #if ERTS_CIO_HAVE_DRV_EVENT
     null_des.driver.event = NULL;
 #endif
-    null_des.driver.drv_ptr = NULL;
+    null_des.driver.stop.drv_ptr = NULL;
     null_des.events = 0;
     null_des.remove_cnt = 0;
     null_des.type = ERTS_EV_TYPE_NONE;
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index 14f1ea3f43..242e7cfcf5 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -34,6 +34,8 @@
 
 int driver_select_kp(ErlDrvPort, ErlDrvEvent, int, int);
 int driver_select_nkp(ErlDrvPort, ErlDrvEvent, int, int);
+int enif_select_kp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, Eterm);
+int enif_select_nkp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, Eterm);
 int driver_event_kp(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
 int driver_event_nkp(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
 Uint erts_check_io_size_kp(void);
@@ -136,4 +138,20 @@ typedef struct {
     ErtsIoTask iniotask;
     ErtsIoTask outiotask;
 } ErtsDrvSelectDataState;
+
+struct erts_nif_select_event {
+    Eterm pid;
+    Eterm immed;
+    Uint32 refn[ERTS_REF_NUMBERS];
+    Sint32 ddeselect_cnt; /* 0:  No delayed deselect in progress
+                           * 1:  Do deselect before next poll
+                           * >1: Countdown of ignored events
+                           */
+};
+
+typedef struct {
+    struct erts_nif_select_event in;
+    struct erts_nif_select_event out;
+} ErtsNifSelectDataState;
+
 #endif /* #ifndef ERL_CHECK_IO_INTERNAL__ */
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index b8a28bcc18..0c4df3e13a 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -3030,6 +3030,7 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet ps,
 	    ev[fd] = 0;
 	else {
 	    ev[fd] = ps->fds_status[fd].events;
+            ASSERT(ps->fds_status[fd].used_events == ev[fd]);
             if (
 #if ERTS_POLL_USE_WAKEUP_PIPE
                 fd == ps->wake_fds[0] || fd == ps->wake_fds[1] ||
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 4b2edace0a..789b455f2d 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -161,6 +161,7 @@ int erts_use_kernel_poll = 0;
 
 struct {
     int (*select)(ErlDrvPort, ErlDrvEvent, int, int);
+    int (*enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, Eterm);
     int (*event)(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
     void (*check_io_as_interrupt)(void);
     void (*check_io_interrupt)(int);
@@ -184,6 +185,13 @@ driver_event(ErlDrvPort port, ErlDrvEvent event, ErlDrvEventData event_data)
     return (*io_func.event)(port, event, event_data);
 }
 
+int enif_select(ErlNifEnv* env, ErlNifEvent event,
+                enum ErlNifSelectFlags flags, void* obj, Eterm ref)
+{
+    return (*io_func.enif_select)(env, event, flags, obj, ref);
+}
+
+
 Eterm erts_check_io_info(void *p)
 {
     return (*io_func.info)(p);
@@ -201,6 +209,7 @@ init_check_io(void)
 {
     if (erts_use_kernel_poll) {
 	io_func.select			= driver_select_kp;
+        io_func.enif_select		= enif_select_kp;
 	io_func.event			= driver_event_kp;
 #ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
 	io_func.check_io_as_interrupt	= erts_check_io_async_sig_interrupt_kp;
@@ -216,6 +225,7 @@ init_check_io(void)
     }
     else {
 	io_func.select			= driver_select_nkp;
+        io_func.enif_select		= enif_select_nkp;
 	io_func.event			= driver_event_nkp;
 #ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
 	io_func.check_io_as_interrupt	= erts_check_io_async_sig_interrupt_nkp;
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 0d3910b2e2..8c761202d6 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -31,6 +31,7 @@
 	 init_per_testcase/2, end_per_testcase/2,
          basic/1, reload_error/1, upgrade/1, heap_frag/1,
          t_on_load/1,
+         select/1,
          hipe/1,
 	 types/1, many_args/1, binaries/1, get_string/1, get_atom/1,
 	 maps/1,
@@ -64,6 +65,7 @@ all() ->
     [{group, G} || G <- api_groups()]
         ++
     [reload_error, heap_frag, types, many_args,
+     select,
      hipe,
      binaries, get_string, get_atom, maps, api_macros, from_array,
      iolist_as_binary, resource, resource_binary,
@@ -434,6 +436,33 @@ t_on_load(Config) when is_list(Config) ->
     verify_tmpmem(TmpMem),
     ok.
 
+-define(ERL_NIF_SELECT_READ, (1 bsl 0)).
+-define(ERL_NIF_SELECT_WRITE, (1 bsl 1)).
+-define(ERL_NIF_SELECT_STOP, (1 bsl 2)).
+
+select(Config) when is_list(Config) ->
+    ensure_lib_loaded(Config),
+
+    Ref = make_ref(),
+    {R,W} = pipe_nif(),
+    ok = write_nif(W, <<"hej">>),
+    <<"hej">> = read_nif(R, 3),
+    eagain = read_nif(R, 3),
+    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,Ref),
+    [] = flush(),
+    ok = write_nif(W, <<"hej">>),
+    [{select, R, Ref, ready_input}] = flush(),
+    <<"hej">> = read_nif(R, 3),
+
+    %% To be extended...
+
+    0 = select_nif(R,?ERL_NIF_SELECT_STOP,R,Ref),
+    0 = select_nif(W,?ERL_NIF_SELECT_STOP,W,Ref),
+    timer:sleep(10),
+    true = is_closed_nif(R),
+    true = is_closed_nif(W),
+    ok.
+
 hipe(Config) when is_list(Config) ->
     Data = proplists:get_value(data_dir, Config),
     Priv = proplists:get_value(priv_dir, Config),
@@ -1910,6 +1939,15 @@ call(Pid,Cmd) ->
 receive_any() ->
     receive M -> M end.	     
 
+flush() ->
+    flush(10).
+flush(Timeout) ->
+    receive M ->
+            [M | flush(Timeout)]
+    after Timeout ->
+            []
+    end.
+
 repeat(0, _, Arg) ->
     Arg;
 repeat(N, Fun, Arg0) ->
@@ -2273,6 +2311,11 @@ term_to_binary_nif(_, _) -> ?nif_stub.
 binary_to_term_nif(_, _, _) -> ?nif_stub.
 port_command_nif(_, _) -> ?nif_stub.
 format_term_nif(_,_) -> ?nif_stub.
+select_nif(_,_,_,_) -> ?nif_stub.
+pipe_nif() -> ?nif_stub.
+write_nif(_,_) -> ?nif_stub.
+read_nif(_,_) -> ?nif_stub.
+is_closed_nif(_) -> ?nif_stub.
 
 %% maps
 is_map_nif(_) -> ?nif_stub.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 2c93891852..a4915b13c4 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -25,6 +25,8 @@
 #include <limits.h>
 #ifndef __WIN32__
 #include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
 #endif
 
 #include "nif_mod.h"
@@ -42,7 +44,9 @@ static ERL_NIF_TERM atom_second;
 static ERL_NIF_TERM atom_millisecond;
 static ERL_NIF_TERM atom_microsecond;
 static ERL_NIF_TERM atom_nanosecond;
-
+static ERL_NIF_TERM atom_eagain;
+static ERL_NIF_TERM atom_eof;
+static ERL_NIF_TERM atom_error;
 
 typedef struct
 {
@@ -102,6 +106,18 @@ struct binary_resource {
     unsigned size;
 };
 
+static ErlNifResourceType* fd_resource_type;
+static void fd_resource_dtor(ErlNifEnv* env, void* obj);
+static void fd_resource_stop(ErlNifEnv* env, void* obj);
+static ErlNifResourceTypeInit fd_rt_init = {
+    fd_resource_dtor,
+    fd_resource_stop
+};
+struct fd_resource {
+    int fd;
+};
+
+
 static int get_pointer(ErlNifEnv* env, ERL_NIF_TERM term, void** pp)
 {
     ErlNifBinary bin;
@@ -144,6 +160,9 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
     msgenv_resource_type =  enif_open_resource_type(env,NULL,"nif_SUITE.msgenv",
 						    msgenv_dtor,
 						    ERL_NIF_RT_CREATE, NULL);
+    fd_resource_type =  enif_open_resource_type_x(env, "nif_SUITE.fd",
+                                                  &fd_rt_init,
+                                                  ERL_NIF_RT_CREATE, NULL);
     atom_false = enif_make_atom(env,"false");
     atom_true = enif_make_atom(env,"true");
     atom_self = enif_make_atom(env,"self");
@@ -154,6 +173,9 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
     atom_millisecond = enif_make_atom(env,"millisecond");
     atom_microsecond = enif_make_atom(env,"microsecond");
     atom_nanosecond = enif_make_atom(env,"nanosecond");
+    atom_eagain = enif_make_atom(env, "eagain");
+    atom_eof = enif_make_atom(env, "eof");
+    atom_error = enif_make_atom(env, "error");
 
     *priv_data = data;
     return 0;
@@ -2010,6 +2032,165 @@ static ERL_NIF_TERM format_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg
 }
 
 
+static int get_fd(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifEvent* fd)
+{
+    struct fd_resource* rsrc;
+
+    if (!enif_get_resource(env, term, fd_resource_type, (void**)&rsrc)) {
+        return 0;
+    }
+    *fd = rsrc->fd;
+    return 1;
+}
+
+static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ErlNifEvent e;
+    enum ErlNifSelectFlags mode;
+    void* obj;
+    ERL_NIF_TERM ref;
+    int retval;
+
+    if (!get_fd(env, argv[0], &e)
+        || !enif_get_uint(env, argv[1], &mode)
+        || !enif_get_resource(env, argv[2], fd_resource_type, &obj))
+    {
+        return enif_make_badarg(env);
+    }
+
+    ref = argv[3];
+
+    retval = enif_select(env, e, mode, obj, ref);
+
+    return enif_make_int(env, retval);
+}
+
+static ERL_NIF_TERM pipe_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    struct fd_resource* read_rsrc;
+    struct fd_resource* write_rsrc;
+    ERL_NIF_TERM read_fd, write_fd;
+    unsigned char *inp, *outp;
+    int fds[2], flags;
+
+    if (pipe(fds) < 0)
+        return enif_make_string(env, "pipe failed", ERL_NIF_LATIN1);
+
+    if ((flags = fcntl(fds[0], F_GETFL, 0)) < 0
+        || fcntl(fds[0], F_SETFL, flags|O_NONBLOCK) < 0
+        || (flags = fcntl(fds[1], F_GETFL, 0)) < 0
+        || fcntl(fds[1], F_SETFL, flags|O_NONBLOCK) < 0) {
+        close(fds[0]);
+        close(fds[1]);
+        return enif_make_string(env, "fcntl failed on pipe", ERL_NIF_LATIN1);
+    }
+
+    read_rsrc  = enif_alloc_resource(fd_resource_type, sizeof(struct fd_resource));
+    write_rsrc = enif_alloc_resource(fd_resource_type, sizeof(struct fd_resource));
+    read_rsrc->fd  = fds[0];
+    write_rsrc->fd = fds[1];
+    read_fd  = enif_make_resource(env, read_rsrc);
+    write_fd = enif_make_resource(env, write_rsrc);
+    enif_release_resource(read_rsrc);
+    enif_release_resource(write_rsrc);
+
+    return enif_make_tuple2(env, read_fd, write_fd);
+}
+
+static ERL_NIF_TERM write_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ErlNifEvent fd;
+    ErlNifBinary bin;
+    int n, written = 0;
+
+    if (!get_fd(env, argv[0], &fd)
+        || !enif_inspect_binary(env, argv[1], &bin))
+        return enif_make_badarg(env);
+
+    for (;;) {
+        n = write(fd, bin.data + written, bin.size - written);
+        if (n >= 0) {
+            written += n;
+            if (written == bin.size) {
+                return atom_ok;
+            }
+        }
+        else if (errno == EAGAIN) {
+            return enif_make_tuple2(env, atom_eagain, enif_make_int(env, written));
+        }
+        else if (errno == EINTR) {
+            continue;
+        }
+        else {
+            return enif_make_tuple2(env, atom_error, enif_make_int(env, errno));
+        }
+    }
+}
+
+static ERL_NIF_TERM read_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ErlNifEvent fd;
+    unsigned char* buf;
+    int n, count, nread = 0;
+    ERL_NIF_TERM res;
+
+    if (!get_fd(env, argv[0], &fd)
+        || !enif_get_int(env, argv[1], &count) || count < 1)
+        return enif_make_badarg(env);
+
+    buf = enif_make_new_binary(env, count, &res);
+
+    for (;;) {
+        n = read(fd, buf, count);
+        if (n > 0) {
+            if (n < count) {
+                res = enif_make_sub_binary(env, res, 0, n);
+            }
+            return res;
+        }
+        else if (n == 0) {
+            return atom_eof;
+        }
+        else if (errno == EAGAIN) {
+            return atom_eagain;
+        }
+        else if (errno == EINTR) {
+            continue;
+        }
+        else {
+            return enif_make_tuple2(env, atom_error, enif_make_int(env, errno));
+        }
+    }
+}
+
+static ERL_NIF_TERM is_closed_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ErlNifEvent fd;
+
+    if (!get_fd(env, argv[0], &fd))
+        return enif_make_badarg(env);
+
+    return fd < 0 ? atom_true : atom_false;
+}
+
+static void fd_resource_dtor(ErlNifEnv* env, void* obj)
+{
+    struct fd_resource* rsrc = (struct fd_resource*)obj;
+    if (rsrc->fd >= 0)
+        close(rsrc->fd);
+}
+
+static void fd_resource_stop(ErlNifEnv* env, void* obj)
+{
+    struct fd_resource* rsrc = (struct fd_resource*)obj;
+    if (rsrc->fd >= 0) {
+        close(rsrc->fd);
+        rsrc->fd = -1;   /* thread safety ? */
+    }
+}
+
+
+
 static ErlNifFunc nif_funcs[] =
 {
     {"lib_version", 0, lib_version},
@@ -2086,7 +2267,12 @@ static ErlNifFunc nif_funcs[] =
     {"term_to_binary_nif", 2, term_to_binary},
     {"binary_to_term_nif", 3, binary_to_term},
     {"port_command_nif", 2, port_command},
-    {"format_term_nif", 2, format_term}
+    {"format_term_nif", 2, format_term},
+    {"select_nif", 4, select_nif},
+    {"pipe_nif", 0, pipe_nif},
+    {"write_nif", 2, write_nif},
+    {"read_nif", 2, read_nif},
+    {"is_closed_nif", 1, is_closed_nif}
 };
 
 ERL_NIF_INIT(nif_SUITE,nif_funcs,load,NULL,upgrade,unload)
-- 
cgit v1.2.3


From 73e6fc31e8ef1808fb079a4bac2f1ccff44efa3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 30 Nov 2016 15:16:17 +0100
Subject: erts: Add driver and nif locks to lcnt

---
 erts/emulator/beam/erl_drv_thread.c | 98 +++++++++++++++++++++++++++++++------
 1 file changed, 84 insertions(+), 14 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 92edce5176..0e6aadf568 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -54,6 +54,9 @@ fatal_error(int err, char *func)
 
 struct ErlDrvMutex_ {
     ethr_mutex mtx;
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_lock_t lcnt;
+#endif
     char *name;
 };
 
@@ -64,6 +67,9 @@ struct ErlDrvCond_ {
 
 struct ErlDrvRWLock_ {
     ethr_rwmutex rwmtx;
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_lock_t lcnt;
+#endif
     char *name;
 };
 
@@ -161,14 +167,17 @@ erl_drv_mutex_create(char *name)
 	opt.posix_compliant = 1;
 	if (ethr_mutex_init_opt(&dmtx->mtx, &opt) != 0) {
 	    erts_free(ERTS_ALC_T_DRV_MTX, (void *) dmtx);
-	    dmtx = NULL;
+	    return NULL;
 	}
-	else if (!name)
-	    dmtx->name = no_name;
-	else {
+	if (name) {
 	    dmtx->name = ((char *) dmtx) + sizeof(ErlDrvMutex);
 	    sys_strcpy(dmtx->name, name);
+        } else {
+	    dmtx->name = no_name;
 	}
+#ifdef ERTS_ENABLE_LOCK_COUNT
+        erts_lcnt_init_lock(&dmtx->lcnt, dmtx->name, ERTS_LCNT_LT_MUTEX);
+#endif
     }
     return dmtx;
 #else
@@ -180,7 +189,11 @@ void
 erl_drv_mutex_destroy(ErlDrvMutex *dmtx)
 {
 #ifdef USE_THREADS
-    int res = dmtx ? ethr_mutex_destroy(&dmtx->mtx) : EINVAL;
+    int res;
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_destroy_lock(&dmtx->lcnt);
+#endif
+    res = dmtx ? ethr_mutex_destroy(&dmtx->mtx) : EINVAL;
     if (res != 0)
 	fatal_error(res, "erl_drv_mutex_destroy()");
     erts_free(ERTS_ALC_T_DRV_MTX, (void *) dmtx);
@@ -202,9 +215,14 @@ int
 erl_drv_mutex_trylock(ErlDrvMutex *dmtx)
 {
 #ifdef USE_THREADS
+    int res;
     if (!dmtx)
 	fatal_error(EINVAL, "erl_drv_mutex_trylock()");
-    return ethr_mutex_trylock(&dmtx->mtx);
+    res = ethr_mutex_trylock(&dmtx->mtx);
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_trylock(&dmtx->lcnt, res);
+#endif
+    return res;
 #else
     return 0;
 #endif
@@ -216,8 +234,14 @@ erl_drv_mutex_lock(ErlDrvMutex *dmtx)
 #ifdef USE_THREADS
     if (!dmtx)
 	fatal_error(EINVAL, "erl_drv_mutex_lock()");
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_lock(&dmtx->lcnt);
+#endif
     ethr_mutex_lock(&dmtx->mtx);
 #endif
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_lock_post(&dmtx->lcnt);
+#endif
 }
 
 void
@@ -226,6 +250,9 @@ erl_drv_mutex_unlock(ErlDrvMutex *dmtx)
 #ifdef USE_THREADS
     if (!dmtx)
 	fatal_error(EINVAL, "erl_drv_mutex_unlock()");
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_unlock(&dmtx->lcnt);
+#endif
     ethr_mutex_unlock(&dmtx->mtx);
 #endif
 }
@@ -306,10 +333,18 @@ erl_drv_cond_wait(ErlDrvCond *dcnd, ErlDrvMutex *dmtx)
     if (!dcnd || !dmtx) {
 	fatal_error(EINVAL, "erl_drv_cond_wait()");
     }
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_unlock(&dmtx->lcnt);
+#endif
     while (1) {
 	int res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx);
-	if (res == 0)
+	if (res == 0) {
+#ifdef ERTS_ENABLE_LOCK_COUNT
+            erts_lcnt_lock(&dmtx->lcnt);
+            erts_lcnt_lock_post(&dmtx->lcnt);
+#endif
 	    break;
+        }
     }
 #endif
 }
@@ -324,14 +359,17 @@ erl_drv_rwlock_create(char *name)
     if (drwlck) {
 	if (ethr_rwmutex_init(&drwlck->rwmtx) != 0) {
 	    erts_free(ERTS_ALC_T_DRV_RWLCK, (void *) drwlck);
-	    drwlck = NULL;
+	    return NULL;
 	}
-	else if (!name)
-	    drwlck->name = no_name;
-	else {
+	if (name) {
 	    drwlck->name = ((char *) drwlck) + sizeof(ErlDrvRWLock);
 	    sys_strcpy(drwlck->name, name);
+        } else {
+	    drwlck->name = no_name;
 	}
+#ifdef ERTS_ENABLE_LOCK_COUNT
+        erts_lcnt_init_lock(&drwlck->lcnt, drwlck->name, ERTS_LCNT_LT_RWMUTEX);
+#endif
     }
     return drwlck;
 #else
@@ -343,7 +381,11 @@ void
 erl_drv_rwlock_destroy(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
-    int res = drwlck ? ethr_rwmutex_destroy(&drwlck->rwmtx) : EINVAL;
+    int res;
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_destroy_lock(&drwlck->lcnt);
+#endif
+    res = drwlck ? ethr_rwmutex_destroy(&drwlck->rwmtx) : EINVAL;
     if (res != 0)
 	fatal_error(res, "erl_drv_rwlock_destroy()");
     erts_free(ERTS_ALC_T_DRV_RWLCK, (void *) drwlck);
@@ -364,9 +406,14 @@ int
 erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
+    int res;
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_tryrlock()");
-    return ethr_rwmutex_tryrlock(&drwlck->rwmtx);
+    res = ethr_rwmutex_tryrlock(&drwlck->rwmtx);
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LCNT_LO_READ);
+#endif
+    return res;
 #else
     return 0;
 #endif
@@ -378,7 +425,13 @@ erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck)
 #ifdef USE_THREADS
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_rlock()");
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ);
+#endif
     ethr_rwmutex_rlock(&drwlck->rwmtx);
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_lock_post(&drwlck->lcnt);
+#endif
 #endif
 }
 
@@ -388,6 +441,9 @@ erl_drv_rwlock_runlock(ErlDrvRWLock *drwlck)
 #ifdef USE_THREADS
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_runlock()");
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ);
+#endif
     ethr_rwmutex_runlock(&drwlck->rwmtx);
 #endif
 }
@@ -396,9 +452,14 @@ int
 erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
+    int res;
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_tryrwlock()");
-    return ethr_rwmutex_tryrwlock(&drwlck->rwmtx);
+    res = ethr_rwmutex_tryrwlock(&drwlck->rwmtx);
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LCNT_LO_READ_WRITE);
+#endif
+    return res;
 #else
     return 0;
 #endif
@@ -410,7 +471,13 @@ erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck)
 #ifdef USE_THREADS
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_rwlock()");
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ_WRITE);
+#endif
     ethr_rwmutex_rwlock(&drwlck->rwmtx);
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_lock_post(&drwlck->lcnt);
+#endif
 #endif
 }
 
@@ -420,6 +487,9 @@ erl_drv_rwlock_rwunlock(ErlDrvRWLock *drwlck)
 #ifdef USE_THREADS
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_rwunlock()");
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ_WRITE);
+#endif
     ethr_rwmutex_rwunlock(&drwlck->rwmtx);
 #endif
 }
-- 
cgit v1.2.3


From 8e5c762d597b9a3ba4d78bcf57ebe379ad666599 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 2 Dec 2016 16:44:43 +0100
Subject: erts: Fix whitespace errors in lock counter

---
 erts/emulator/beam/erl_lock_count.c | 25 ++++++++++++++-----------
 erts/emulator/beam/erl_lock_count.h |  4 ++--
 2 files changed, 16 insertions(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index 481e92b2cd..6354fc8663 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -55,11 +55,11 @@ static erts_lcnt_thread_data_t *lcnt_thread_data[2048];
 /* local functions */
 
 static ERTS_INLINE void lcnt_lock(void) {
-    ethr_mutex_lock(&lcnt_data_lock); 
+    ethr_mutex_lock(&lcnt_data_lock);
 }
 
 static ERTS_INLINE void lcnt_unlock(void) {
-    ethr_mutex_unlock(&lcnt_data_lock); 
+    ethr_mutex_unlock(&lcnt_data_lock);
 }
 
 const int log2_tab64[64] = {
@@ -159,7 +159,7 @@ static erts_lcnt_thread_data_t *lcnt_thread_data_alloc(void) {
     lcnt_thread_data[eltd->id] = eltd;
 
     return eltd;
-} 
+}
 
 static erts_lcnt_thread_data_t *lcnt_get_thread_data(void) {
     return (erts_lcnt_thread_data_t *)ethr_tsd_get(lcnt_thr_data_key);
@@ -254,9 +254,9 @@ void erts_lcnt_init() {
     /* init lock */
     if (ethr_mutex_init(&lcnt_data_lock) != 0) abort();
 
-    /* init tsd */    
+    /* init tsd */
     lcnt_n_thr = 0;
-    ethr_tsd_key_create(&lcnt_thr_data_key,"lcnt_data");
+    ethr_tsd_key_create(&lcnt_thr_data_key, "lcnt_data");
 
     lcnt_lock();
 
@@ -352,11 +352,11 @@ void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock)
 
 /* interface to erl_threads.h */
 /* only lock on init and destroy, all others should use atomics */
-void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) { 
+void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) {
     erts_lcnt_init_lock_x(lock, name, flag, NIL);
 }
 
-void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { 
+void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) {
     int i;
     if (name == NULL) { ERTS_LCNT_CLEAR_FLAG(lock); return; }
     lcnt_lock();
@@ -382,7 +382,10 @@ void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eter
     erts_lcnt_list_insert(erts_lcnt_data->current_locks, lock);
     lcnt_unlock();
 }
-/* init empty, instead of zero struct */
+
+/* init empty, instead of zero struct
+ * used by process locks probes
+ */
 void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock) {
     lock->next = NULL;
     lock->prev = NULL;
@@ -444,7 +447,7 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
     }
 
     /* we cannot acquire w_lock if either w or r are taken */
-    /* we cannot acquire r_lock if w_lock is taken */	
+    /* we cannot acquire r_lock if w_lock is taken */
 
     if ((w_state > 0) || (r_state > 0)) {
         eltd->lock_in_conflict = 1;
@@ -561,7 +564,7 @@ void erts_lcnt_unlock(erts_lcnt_lock_t *lock) {
     if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
 #ifdef DEBUG
     {
-        erts_aint_t w_state;  
+        erts_aint_t w_state;
         erts_aint_t flowstate;
 
         /* flowstate */
@@ -647,7 +650,7 @@ Uint16 erts_lcnt_set_rt_opt(Uint16 opt) {
     return prev;
 }
 
-Uint16 erts_lcnt_clear_rt_opt(Uint16 opt) {			
+Uint16 erts_lcnt_clear_rt_opt(Uint16 opt) {
     Uint16 prev;
     prev = (erts_lcnt_rt_options & opt);
     erts_lcnt_rt_options &= ~opt;
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index 3e8dcefe69..4f838f7faa 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -148,13 +148,13 @@ typedef struct erts_lcnt_lock_stats_s {
 typedef struct erts_lcnt_lock_s {
     char *name;            /* lock name */
     Uint16 flag;           /* lock type */
-    Eterm id;              /* id if possible */ 
+    Eterm id;              /* id if possible */
 
 #ifdef DEBUG
     ethr_atomic_t flowstate;
 #endif
 
-    /* lock states */    
+    /* lock states */
     ethr_atomic_t w_state; /* 0 not taken, otherwise n threads waiting */
     ethr_atomic_t r_state; /* 0 not taken, > 0 -> writes will wait */
 
-- 
cgit v1.2.3


From 7d161f5b475575bd79bd90977b3a79334a8ec658 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 2 Dec 2016 17:20:00 +0100
Subject: erts: Remove unnecessary 'enable_lcnt' option in locks

---
 erts/emulator/beam/erl_alloc_util.c   |  8 ++++----
 erts/emulator/beam/erl_port_task.h    |  8 +++-----
 erts/emulator/beam/erl_process_lock.c | 21 +++++++--------------
 erts/emulator/beam/erl_smp.h          |  4 ++--
 erts/emulator/beam/erl_threads.h      | 33 +++++++++------------------------
 erts/emulator/beam/io.c               | 21 +++++++++------------
 6 files changed, 34 insertions(+), 61 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 61cce224dd..d346f8c8b6 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -6134,18 +6134,18 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 #ifdef USE_THREADS
     if (init->ts) {
 	allctr->thread_safe = 1;
-	
+
 #ifdef ERTS_ENABLE_LOCK_COUNT
 	erts_mtx_init_x_opt(&allctr->mutex,
 			    "alcu_allocator",
 			    make_small(allctr->alloc_no),
-			    ERTS_LCNT_LT_ALLOC,1);
+			    ERTS_LCNT_LT_ALLOC);
 #else
 	erts_mtx_init_x(&allctr->mutex,
 			"alcu_allocator",
-			make_small(allctr->alloc_no),1);
+			make_small(allctr->alloc_no));
 #endif /*ERTS_ENABLE_LOCK_COUNT*/
-	
+
 #ifdef DEBUG
 	allctr->debug.saved_tid = 0;
 #endif
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index 2a6bd165a3..e3550e878e 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -188,13 +188,11 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id)
     ptsp->taskq.in.last = NULL;
     erts_smp_atomic32_init_nob(&ptsp->flags, 0);
 #ifdef ERTS_SMP
-    erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id,
 #ifdef ERTS_ENABLE_LOCK_COUNT
-		    (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)
-#else
-		    1
+    if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK))
+        lock_str = NULL;
 #endif
-		    );
+    erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id);
 #endif
 }
 
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index a69185bc5c..180df229eb 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -123,7 +123,7 @@ erts_init_proc_lock(int cpus)
     for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) {
 #ifdef ERTS_ENABLE_LOCK_COUNT
 	erts_mtx_init_x(&erts_pix_locks[i].u.mtx,
-			"pix_lock", make_small(i), 1);
+			"pix_lock", make_small(i));
 #else
 	erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock");
 #endif
@@ -1027,39 +1027,32 @@ erts_proc_lock_init(Process *p)
 #endif
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
 
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    int do_lock_count = 1;
-#else
-    int do_lock_count = 0;
-#endif
-
-    erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id, do_lock_count);
+    erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id);
     ethr_mutex_lock(&p->lock.main.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.main.lc);
 #endif
-    erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id, do_lock_count);
+    erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id);
     ethr_mutex_lock(&p->lock.link.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.link.lc);
 #endif
-    erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id, do_lock_count);
+    erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id);
     ethr_mutex_lock(&p->lock.msgq.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.msgq.lc);
 #endif
-    erts_mtx_init_x(&p->lock.btm, "proc_btm", p->common.id, do_lock_count);
+    erts_mtx_init_x(&p->lock.btm, "proc_btm", p->common.id);
     ethr_mutex_lock(&p->lock.btm.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.btm.lc);
 #endif
-    erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id,
-		    do_lock_count);
+    erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id);
     ethr_mutex_lock(&p->lock.status.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.status.lc);
 #endif
-    erts_mtx_init_x(&p->lock.trace, "proc_trace", p->common.id, do_lock_count);
+    erts_mtx_init_x(&p->lock.trace, "proc_trace", p->common.id);
     ethr_mutex_lock(&p->lock.trace.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.trace.lc);
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index 713ed50b86..14be511f86 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -1065,7 +1065,7 @@ ERTS_GLB_INLINE void
 erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
 {
 #ifdef ERTS_SMP
-    erts_mtx_init_x(mtx, name, extra, 1);
+    erts_mtx_init_x(mtx, name, extra);
 #endif
 }
 
@@ -1073,7 +1073,7 @@ ERTS_GLB_INLINE void
 erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
 {
 #ifdef ERTS_SMP
-    erts_mtx_init_locked_x(mtx, name, extra, 1);
+    erts_mtx_init_locked_x(mtx, name, extra);
 #endif
 }
 
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index eccd49f2a9..9e75f6fee5 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -479,14 +479,9 @@ ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void));
 ERTS_GLB_INLINE erts_tid_t erts_thr_self(void);
 ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len);
 ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y);
-ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra,
-				     int enable_lcnt);
-ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra,
-					 Uint16 opt, int enable_lcnt);
-ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx,
-					    char *name,
-					    Eterm extra,
-					    int enable_lcnt);
+ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra);
+ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt);
+ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra);
 ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx);
@@ -2164,7 +2159,7 @@ erts_equal_tids(erts_tid_t x, erts_tid_t y)
 }
 
 ERTS_GLB_INLINE void
-erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt)
+erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra)
 {
 #ifdef USE_THREADS
     int res = ethr_mutex_init(&mtx->mtx);
@@ -2174,17 +2169,13 @@ erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt)
     erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    if (enable_lcnt)
-      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
-    else
-      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra);
+    erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
 #endif
 #endif
 }
 
 ERTS_GLB_INLINE void
-erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt,
-		    int enable_lcnt)
+erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt)
 {
 #ifdef USE_THREADS
     int res = ethr_mutex_init(&mtx->mtx);
@@ -2194,17 +2185,14 @@ erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt,
     erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    if (enable_lcnt)
-      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra);
-    else
-      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX | opt, extra);
+    erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra);
 #endif
 #endif
 }
 
 
 ERTS_GLB_INLINE void
-erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt)
+erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra)
 {
 #ifdef USE_THREADS
     int res = ethr_mutex_init(&mtx->mtx);
@@ -2214,10 +2202,7 @@ erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt
     erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    if (enable_lcnt)
-      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
-    else
-      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra);
+    erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
 #endif
     ethr_mutex_lock(&mtx->mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 4f131c74de..33b74f30b7 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -259,13 +259,11 @@ static ERTS_INLINE void port_init_instr(Port *prt
     ASSERT(prt->drv_ptr && prt->lock);
     if (!prt->drv_ptr->lock) {
 	char *lock_str = "port_lock";
-	erts_mtx_init_locked_x(prt->lock, lock_str, id,
 #ifdef ERTS_ENABLE_LOCK_COUNT
-			       (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)
-#else
-			       0
+        if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK))
+            lock_str = NULL;
 #endif
-			       );
+	erts_mtx_init_locked_x(prt->lock, lock_str, id);
     }
 #endif
     erts_port_task_init_sched(&prt->sched, id);
@@ -7107,7 +7105,7 @@ driver_pdl_create(ErlDrvPort dp)
 	return NULL;
     pdl = erts_alloc(ERTS_ALC_T_PORT_DATA_LOCK,
 		     sizeof(struct erl_drv_port_data_lock));
-    erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id, 1);
+    erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id);
     pdl_init_refc(pdl);
     erts_port_inc_refc(pp);
     pdl->prt = pp;
@@ -8290,14 +8288,13 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
 	erts_mtx_init_x(drv->lock,
 			"driver_lock",
 #if defined(ERTS_ENABLE_LOCK_CHECK) || defined(ERTS_ENABLE_LOCK_COUNT)
-			    erts_atom_put((byte *) drv->name,
-					  sys_strlen(drv->name),
-					  ERTS_ATOM_ENC_LATIN1,
-					  1),
+                        erts_atom_put((byte *) drv->name,
+				       sys_strlen(drv->name),
+                                       ERTS_ATOM_ENC_LATIN1,
+                                       1)
 #else
-			NIL,
+			NIL
 #endif
-			1
 	    );
     }
 #endif
-- 
cgit v1.2.3


From f2956b24845a14475d90959bf8f8f90807c49a8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Magnus=20L=C3=A5ng?= <margnus1@telia.com>
Date: Tue, 6 Dec 2016 16:29:56 +0100
Subject: erts: Fix missing HiPE trampolines on arm

8bb80fe76f5b replaced the "__arm__" macro used to test for the arm
architecture in hipe_bif0 with the "arm" macro, which is not universally
available. As this replacement is not motivated in the commit message,
nor replicated in any other file that uses the "__arm__" macro, this
seems to be an accident, and this commit reverts the replacement.

When compiled in an environment without the "arm" macro, upgrading hipe
code would occasionally not patch relocations to the new module due to
being out of range for a shortjump, and a trampoline not being provided
to do a longjump. Since this type of relocation patches are not expected
to be able to fail, there is no error handling, and aside from a
"hipe_redirect_to_module: patch failed" message, code upgrade would
proceed and lead to various incorrect behaviour.
---
 erts/emulator/hipe/hipe_bif0.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 57fbbd9403..5c29473443 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1121,7 +1121,7 @@ static struct hipe_mfa_info* mod2mfa_put(struct hipe_mfa_info* mfa)
 struct hipe_ref {
     struct hipe_ref_head head;    /* list of refs to same calleee */
     void *address;
-#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
+#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
     void *trampoline;
 #endif
     unsigned int flags;
@@ -1549,7 +1549,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
 
     ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref));
     ref->address = address;
-#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
+#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
     ref->trampoline = trampoline;
 #endif
     ref->flags = flags;
@@ -1864,7 +1864,7 @@ void hipe_redirect_to_module(Module* modp)
 	    if (ref->flags & REF_FLAG_IS_LOAD_MFA)
 		res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa);
 	    else {
-#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
+#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
                 void* trampoline = ref->trampoline;
 #else
                 void* trampoline = NULL;
-- 
cgit v1.2.3


From 3eddb0f762de248d3230b38bc9d478bfbc8e7331 Mon Sep 17 00:00:00 2001
From: Erlang/OTP <otp@erlang.org>
Date: Wed, 7 Dec 2016 13:15:31 +0100
Subject: Update copyright-year

---
 erts/emulator/beam/erl_hl_timer.h                         | 2 +-
 erts/emulator/beam/erl_msacc.c                            | 2 +-
 erts/emulator/beam/erl_msacc.h                            | 2 +-
 erts/emulator/nifs/common/erl_tracer_nif.c                | 2 +-
 erts/emulator/test/dirty_nif_SUITE.erl                    | 2 +-
 erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c | 2 +-
 erts/emulator/test/lttng_SUITE.erl                        | 2 +-
 erts/emulator/test/port_trace_SUITE.erl                   | 2 +-
 erts/emulator/test/tracer_SUITE.erl                       | 2 +-
 erts/emulator/test/tracer_SUITE_data/tracer_test.c        | 2 +-
 erts/emulator/test/tracer_test.erl                        | 2 +-
 11 files changed, 11 insertions(+), 11 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h
index 705be94532..9cdcd581a0 100644
--- a/erts/emulator/beam/erl_hl_timer.h
+++ b/erts/emulator/beam/erl_hl_timer.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2015. All Rights Reserved.
+ * Copyright Ericsson AB 2015-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index 421445fbad..7ddf49937f 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2014-2015. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h
index ad7c8c5eee..4c8e1c8e22 100644
--- a/erts/emulator/beam/erl_msacc.h
+++ b/erts/emulator/beam/erl_msacc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2014-2015. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/nifs/common/erl_tracer_nif.c b/erts/emulator/nifs/common/erl_tracer_nif.c
index c0cc48ff42..0bde60d057 100644
--- a/erts/emulator/nifs/common/erl_tracer_nif.c
+++ b/erts/emulator/nifs/common/erl_tracer_nif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson 2015. All Rights Reserved.
+ * Copyright Ericsson 2015-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl
index 658bdc41b6..a61fd92a18 100644
--- a/erts/emulator/test/dirty_nif_SUITE.erl
+++ b/erts/emulator/test/dirty_nif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2010-2014. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
index a0019e5d95..08efa23c81 100644
--- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
+++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2014. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/lttng_SUITE.erl b/erts/emulator/test/lttng_SUITE.erl
index 6b7ad836f5..c12f63706a 100644
--- a/erts/emulator/test/lttng_SUITE.erl
+++ b/erts/emulator/test/lttng_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/port_trace_SUITE.erl b/erts/emulator/test/port_trace_SUITE.erl
index 5d9a75bcd3..03efdc15db 100644
--- a/erts/emulator/test/port_trace_SUITE.erl
+++ b/erts/emulator/test/port_trace_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1999-2012. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/tracer_SUITE.erl b/erts/emulator/test/tracer_SUITE.erl
index 9eb55c9af3..730c43d8c2 100644
--- a/erts/emulator/test/tracer_SUITE.erl
+++ b/erts/emulator/test/tracer_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2013. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/tracer_SUITE_data/tracer_test.c b/erts/emulator/test/tracer_SUITE_data/tracer_test.c
index a26bb33600..d9543b7ab9 100644
--- a/erts/emulator/test/tracer_SUITE_data/tracer_test.c
+++ b/erts/emulator/test/tracer_SUITE_data/tracer_test.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2014. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/tracer_test.erl b/erts/emulator/test/tracer_test.erl
index 1da80bfe31..a82fd04d2e 100644
--- a/erts/emulator/test/tracer_test.erl
+++ b/erts/emulator/test/tracer_test.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2013. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
-- 
cgit v1.2.3


From 4a8e2aeee74f1c952c52d621ad13c781384a0172 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 6 Dec 2016 15:01:31 +0100
Subject: erts: Add SIGTERM signal handler

A received SIGTERM signal to beam will generate a {'stop','stop'} message
to the init process and terminate the beam nicely.
---
 erts/emulator/sys/unix/sys.c | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 99e0aea4b5..4362029608 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -409,6 +409,7 @@ erts_sys_pre_init(void)
 #ifdef ERTS_THR_HAVE_SIG_FUNCS
     sigemptyset(&thr_create_sigmask);
     sigaddset(&thr_create_sigmask, SIGINT);   /* block interrupt */
+    sigaddset(&thr_create_sigmask, SIGTERM);  /* block terminate signal */
     sigaddset(&thr_create_sigmask, SIGUSR1);  /* block user defined signal */
 #endif
 
@@ -653,6 +654,40 @@ static RETSIGTYPE request_break(int signum)
 #endif
 }
 
+static void stop_requested(void) {
+    Process* p = NULL;
+    Eterm msg, *hp;
+    ErtsProcLocks locks = 0;
+    ErlOffHeap *ohp;
+    Eterm id = erts_whereis_name_to_id(NULL, am_init);
+
+    if ((p = (erts_pid2proc_opt(NULL, 0, id, 0, ERTS_P2P_FLG_INC_REFC))) != NULL) {
+        ErtsMessage *msgp = erts_alloc_message_heap(p, &locks, 3, &hp, &ohp);
+
+        /* init ! {stop,stop} */
+        msg = TUPLE2(hp, am_stop, am_stop);
+        erts_queue_message(p, locks, msgp, msg, am_system);
+
+        if (locks)
+            erts_smp_proc_unlock(p, locks);
+        erts_proc_dec_refc(p);
+    }
+}
+
+#if (defined(SIG_SIGSET) || defined(SIG_SIGNAL))
+static RETSIGTYPE request_stop(void)
+#else
+static RETSIGTYPE request_stop(int signum)
+#endif
+{
+#ifdef ERTS_SMP
+    smp_sig_notify('S');
+#else
+    stop_requested();
+#endif
+}
+
+
 static ERTS_INLINE void
 sigusr1_exit(void)
 {
@@ -749,6 +784,7 @@ static RETSIGTYPE do_quit(int signum)
 /* Disable break */
 void erts_set_ignore_break(void) {
     sys_signal(SIGINT,  SIG_IGN);
+    sys_signal(SIGTERM, SIG_IGN);
     sys_signal(SIGQUIT, SIG_IGN);
     sys_signal(SIGTSTP, SIG_IGN);
 }
@@ -774,6 +810,7 @@ void erts_replace_intr(void) {
 void init_break_handler(void)
 {
    sys_signal(SIGINT, request_break);
+   sys_signal(SIGTERM, request_stop);
 #ifndef ETHR_UNUSABLE_SIGUSRX
    sys_signal(SIGUSR1, user_signal1);
 #endif /* #ifndef ETHR_UNUSABLE_SIGUSRX */
@@ -1287,6 +1324,9 @@ signal_dispatcher_thread_func(void *unused)
 	    switch (buf[i]) {
 	    case 0: /* Emulator initialized */
                 break;
+	    case 'S': /* SIGTERM */
+		stop_requested();
+		break;
 	    case 'I': /* SIGINT */
 		break_requested();
 		break;
-- 
cgit v1.2.3


From a07fc6fbcdf270e29332cd2556511b5f04e3d476 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 8 Dec 2016 18:36:25 +0100
Subject: erts: Fix hanging race in code_SUITE:versions

Monitor first, then ask to terminate.
---
 erts/emulator/test/code_SUITE.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index 34515efa3d..3ee14f2d1c 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -66,9 +66,9 @@ versions(Config) when is_list(Config) ->
     2 = versions:version(),
 
     %% Kill processes, unload code.
-    P1 ! P2 ! done,
     _ = monitor(process, P1),
     _ = monitor(process, P2),
+    P1 ! P2 ! done,
     receive
         {'DOWN',_,process,P1,normal} -> ok
     end,
-- 
cgit v1.2.3


From 2e1fb1a42098ac73cd44f60699a09c3c3e49f77f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 12 Dec 2016 16:24:49 +0100
Subject: Don't include problematic test cases in smoke test

In time_SUITE, the univ_to_local/1 and local_to_univ/1 test cases will
fail if they are run outside the CET timezone. The consistency/1
test case uses is not reliable in March and October because it
uses a simplified model for when the switch to/from DST is done.
---
 erts/emulator/test/emulator_smoke.spec | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/emulator_smoke.spec b/erts/emulator/test/emulator_smoke.spec
index 3219aeb823..b2d0de8835 100644
--- a/erts/emulator/test/emulator_smoke.spec
+++ b/erts/emulator/test/emulator_smoke.spec
@@ -1,3 +1,9 @@
-{suites,"../emulator_test",[smoke_test_SUITE,time_SUITE]}.
-{cases,"../emulator_test",crypto_SUITE,[t_md5]}.
-{cases,"../emulator_test",float_SUITE,[fpe,cmp_integer]}.
\ No newline at end of file
+{define,'Dir',"../emulator_test"}.
+{suites,'Dir',[smoke_test_SUITE]}.
+{suites,'Dir',[time_SUITE]}.
+{skip_cases,'Dir',time_SUITE,
+    [univ_to_local,local_to_univ],"Depends on CET timezone"}.
+{skip_cases,'Dir',time_SUITE,
+    [consistency],"Not reliable in October and March"}.
+{cases,'Dir',crypto_SUITE,[t_md5]}.
+{cases,'Dir',float_SUITE,[fpe,cmp_integer]}.
-- 
cgit v1.2.3


From 2f8d59aa9e8a96d094172db339fd94aae45a90b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?H=C3=A5kan=20Mattsson?= <hm@tail-f.com>
Date: Wed, 2 Nov 2016 12:56:39 +0100
Subject: erts: Make depth of current_stacktrace configurable

The BIF process_info(Pid, current_stacktrace) truncates the
stacktrace. The old behavior was to truncate long stacktraces to max
8 items. And this was hard coded. Now it is truncated to the value of
system_flag(backtrace_depth) instead. The backtrace_depth defaults to
8, but is configurable.
---
 erts/emulator/beam/erl_bif_info.c    |  4 ++--
 erts/emulator/test/process_SUITE.erl | 11 +++++++++++
 2 files changed, 13 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 735aabbee3..88a052cad7 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1672,11 +1672,11 @@ current_stacktrace(Process* p, Process* rp, Eterm** hpp)
     Eterm mfa;
     Eterm res = NIL;
 
-    depth = 8;
+    depth = erts_backtrace_depth;
     sz = offsetof(struct StackTrace, trace) + sizeof(BeamInstr *)*depth;
     s = (struct StackTrace *) erts_alloc(ERTS_ALC_T_TMP, sz);
     s->depth = 0;
-    if (rp->i) {
+    if (depth > 0 && rp->i) {
 	s->trace[s->depth++] = rp->i;
 	depth--;
     }
diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index 0f999e0efe..2289cbabc7 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -437,11 +437,22 @@ t_process_info(Config) when is_list(Config) ->
     verify_loc(Line2, Res2),
     pi_stacktrace([{?MODULE,t_process_info,1,?LINE}]),
 
+    verify_stacktrace_depth(),
+
     Gleader = group_leader(),
     {group_leader, Gleader} = process_info(self(), group_leader),
     {'EXIT',{badarg,_Info}} = (catch process_info('not_a_pid')),
     ok.
 
+verify_stacktrace_depth() ->
+    CS = current_stacktrace,
+    OldDepth = erlang:system_flag(backtrace_depth, 0),
+    {CS,[]} = erlang:process_info(self(), CS),
+    _ = erlang:system_flag(backtrace_depth, 8),
+    {CS,[{?MODULE,verify_stacktrace_depth,0,_},_|_]} =
+        erlang:process_info(self(), CS),
+    _ = erlang:system_flag(backtrace_depth, OldDepth).
+
 pi_stacktrace(Expected0) ->
     {Line,Res} = {?LINE,erlang:process_info(self(), current_stacktrace)},
     {current_stacktrace,Stack} = Res,
-- 
cgit v1.2.3


From 064a2630129caadbca58aaebe83f78735c1e374d Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 19 Dec 2016 14:06:12 +0100
Subject: Fix printout of timer data in crash dump

---
 erts/emulator/beam/erl_hl_timer.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index d29d079fc5..cc61f9abf1 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -2865,7 +2865,8 @@ btm_print(ErtsHLTimer *tmr, void *vbtmp)
 
     if (tmr->timeout <= btmp->now)
 	left = 0;
-    left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now);
+    else
+        left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now);
 
     receiver = ((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
 		? tmr->receiver.name
-- 
cgit v1.2.3


From f5627a66cec1b2e75264ee17b38f1e6d794bb671 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 10 Nov 2016 19:55:40 +0100
Subject: erts: Fix faulty printout in port_SUITE

---
 erts/emulator/test/port_SUITE.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index d4e77d634a..23594aa8c4 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -2292,7 +2292,7 @@ maybe_to_list(List) ->
     List.
 
 format({Eol,List}) ->
-    io_lib:format("tuple<~w,~s>",[Eol, maybe_to_list(List)]);
+    io_lib:format("tuple<~w,~w>",[Eol, maybe_to_list(List)]);
 format(List) when is_list(List) ->
     case list_at_least(50, List) of
         true ->
-- 
cgit v1.2.3


From d44f2e85c058e10082427a354a7d65e9af607501 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 25 Nov 2016 15:03:33 +0100
Subject: Fix stack-trace generated by a traced process

---
 erts/emulator/beam/beam_emu.c           | 45 +++++++++++++++++++++++++--------
 erts/emulator/test/call_trace_SUITE.erl |  6 ++---
 2 files changed, 36 insertions(+), 15 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 59a9ea1417..8c9bc1fa65 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5844,12 +5844,13 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
     s->depth = 0;
 
     /*
-     * If the failure was in a BIF other than 'error', 'exit' or
-     * 'throw', find the bif-table index and save the argument
-     * registers by consing up an arglist.
+     * If the failure was in a BIF other than 'error/1', 'error/2',
+     * 'exit/1' or 'throw/1', find the bif-table index and save the
+     * argument registers by consing up an arglist.
      */
-    if (bf != NULL && bf != error_1 && bf != error_2 &&
-	bf != exit_1 && bf != throw_1) {
+    if (bf != NULL && bf != error_1 && bf != error_2 && bf != exit_1
+	&& bf != throw_1 && bf != wrap_error_1 && bf != wrap_error_2
+	&& bf != wrap_exit_1 && bf != wrap_throw_1) {
         int i;
 	int a = 0;
 	for (i = 0; i < BIF_SIZE; i++) {
@@ -5945,12 +5946,30 @@ erts_save_stacktrace(Process* p, struct StackTrace* s, int depth)
 	    p->cp) {
 	    /* Cannot follow cp here - code may be unloaded */
 	    BeamInstr *cpp = p->cp;
+	    int trace_cp;
 	    if (cpp == beam_exception_trace || cpp == beam_return_trace) {
 		/* Skip return_trace parameters */
 		ptr += 2;
+		trace_cp = 1;
 	    } else if (cpp == beam_return_to_trace) {
 		/* Skip return_to_trace parameters */
 		ptr += 1;
+		trace_cp = 1;
+	    }
+	    else {
+		trace_cp = 0;
+	    }
+	    if (trace_cp && s->pc == cpp) {
+		/*
+		 * If process 'cp' points to a return/exception trace
+		 * instruction and 'cp' has been saved as 'pc' in
+		 * stacktrace, we need to update 'pc' in stacktrace
+		 * with the actual 'cp' located on the top of the
+		 * stack; otherwise, we will lose the top stackframe
+		 * when building the stack trace.
+		 */
+		ASSERT(is_CP(p->stop[0]));
+		s->pc = cp_val(p->stop[0]);
 	    }
 	}
 	while (ptr < STACK_START(p) && depth > 0) {
@@ -6070,12 +6089,15 @@ build_stacktrace(Process* c_p, Eterm exc) {
 	erts_set_current_function(&fi, s->current);
     }
 
+    depth = s->depth;
     /*
-     * If fi.current is still NULL, default to the initial function
+     * If fi.current is still NULL, and we have no
+     * stack at all, default to the initial function
      * (e.g. spawn_link(erlang, abs, [1])).
      */
     if (fi.current == NULL) {
-	erts_set_current_function(&fi, c_p->u.initial);
+	if (depth <= 0)
+	    erts_set_current_function(&fi, c_p->u.initial);
 	args = am_true; /* Just in case */
     } else {
 	args = get_args_from_exc(exc);
@@ -6085,10 +6107,9 @@ build_stacktrace(Process* c_p, Eterm exc) {
      * Look up all saved continuation pointers and calculate
      * needed heap space.
      */
-    depth = s->depth;
     stk = stkp = (FunctionInfo *) erts_alloc(ERTS_ALC_T_TMP,
 				      depth*sizeof(FunctionInfo));
-    heap_size = fi.needed + 2;
+    heap_size = fi.current ? fi.needed + 2 : 0;
     for (i = 0; i < depth; i++) {
 	erts_lookup_function_info(stkp, s->trace[i], 1);
 	if (stkp->current) {
@@ -6107,8 +6128,10 @@ build_stacktrace(Process* c_p, Eterm exc) {
 	res = CONS(hp, mfa, res);
 	hp += 2;
     }
-    hp = erts_build_mfa_item(&fi, hp, args, &mfa);
-    res = CONS(hp, mfa, res);
+    if (fi.current) {
+	hp = erts_build_mfa_item(&fi, hp, args, &mfa);
+	res = CONS(hp, mfa, res);
+    }
 
     erts_free(ERTS_ALC_T_TMP, (void *) stk);
     return res;
diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl
index 6ba6301c7c..f7ff04430a 100644
--- a/erts/emulator/test/call_trace_SUITE.erl
+++ b/erts/emulator/test/call_trace_SUITE.erl
@@ -1090,8 +1090,7 @@ exception_nocatch() ->
                        {trace,t2,exception_from,{erlang,throw,1},
                         {error,{nocatch,Q2}}}],
                       exception_from, {error,{nocatch,Q2}}),
-    expect({trace,T2,exit,{{nocatch,Q2},[{erlang,throw,[Q2],[]},
-                                         {?MODULE,deep_4,1,
+    expect({trace,T2,exit,{{nocatch,Q2},[{?MODULE,deep_4,1,
                                           Deep4LocThrow}]}}),
     Q3 = {dump,[dump,{dump}]},
     T3 = 
@@ -1100,8 +1099,7 @@ exception_nocatch() ->
                        {trace,t3,exception_from,{erlang,error,1},
                         {error,Q3}}],
                       exception_from, {error,Q3}),
-    expect({trace,T3,exit,{Q3,[{erlang,error,[Q3],[]},
-                               {?MODULE,deep_4,1,Deep4LocError}]}}),
+    expect({trace,T3,exit,{Q3,[{?MODULE,deep_4,1,Deep4LocError}]}}),
     T4 = 
     exception_nocatch(?LINE, '=', [17,4711], 5, [], 
                       exception_from, {error,{badmatch,4711}}),
-- 
cgit v1.2.3


From 0612bcedf520388d58a9441e9e6490bf86bf5a4b Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 28 Nov 2016 18:13:38 +0100
Subject: Fix stactrace for apply on error/[1,2], exit/1, or throw/1

---
 erts/emulator/beam/beam_emu.c | 185 +++++++++++++++++++++++++++++++++++++-----
 1 file changed, 163 insertions(+), 22 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8c9bc1fa65..66bccedd94 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1047,9 +1047,11 @@ static BeamInstr* handle_error(Process* c_p, BeamInstr* pc,
 			       Eterm* reg, BifFunction bf) NOINLINE;
 static BeamInstr* call_error_handler(Process* p, BeamInstr* ip,
 				     Eterm* reg, Eterm func) NOINLINE;
-static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE;
+static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity,
+			      BeamInstr *I, Uint offs) NOINLINE;
 static BeamInstr* apply(Process* p, Eterm module, Eterm function,
-			Eterm args, Eterm* reg) NOINLINE;
+			Eterm args, Eterm* reg,
+			BeamInstr *I, Uint offs) NOINLINE;
 static BeamInstr* call_fun(Process* p, int arity,
 			   Eterm* reg, Eterm args) NOINLINE;
 static BeamInstr* apply_fun(Process* p, Eterm fun,
@@ -3194,7 +3196,7 @@ do {						\
  OpCase(i_apply): {
      BeamInstr *next;
      HEAVY_SWAPOUT;
-     next = apply(c_p, r(0), x(1), x(2), reg);
+     next = apply(c_p, r(0), x(1), x(2), reg, NULL, 0);
      HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, I+1);
@@ -3208,7 +3210,7 @@ do {						\
  OpCase(i_apply_last_P): {
      BeamInstr *next;
      HEAVY_SWAPOUT;
-     next = apply(c_p, r(0), x(1), x(2), reg);
+     next = apply(c_p, r(0), x(1), x(2), reg, I, Arg(0));
      HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, (BeamInstr *) E[0]);
@@ -3223,7 +3225,7 @@ do {						\
  OpCase(i_apply_only): {
      BeamInstr *next;
      HEAVY_SWAPOUT;
-     next = apply(c_p, r(0), x(1), x(2), reg);
+     next = apply(c_p, r(0), x(1), x(2), reg, I, 0);
      HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_I(next);
@@ -3237,7 +3239,7 @@ do {						\
      BeamInstr *next;
 
      HEAVY_SWAPOUT;
-     next = fixed_apply(c_p, reg, Arg(0));
+     next = fixed_apply(c_p, reg, Arg(0), NULL, 0);
      HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, I+2);
@@ -3252,7 +3254,7 @@ do {						\
      BeamInstr *next;
 
      HEAVY_SWAPOUT;
-     next = fixed_apply(c_p, reg, Arg(0));
+     next = fixed_apply(c_p, reg, Arg(0), I, Arg(1));
      HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, (BeamInstr *) E[0]);
@@ -6228,8 +6230,107 @@ apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity,
     return ep;
 }
 
+static ERTS_INLINE void
+apply_bif_error_adjustment(Process *p, Export *ep,
+			   Eterm *reg, Uint arity,
+			   BeamInstr *I, Uint stack_offset)
+{
+    /*
+     * I is only set when the apply is a tail call, i.e.,
+     * from the instructions i_apply_only, i_apply_last_P,
+     * and apply_last_IP.
+     */
+    if (I
+	&& ep->code[3] == (BeamInstr) em_apply_bif
+	&& (ep == bif_export[BIF_error_1]
+	    || ep == bif_export[BIF_error_2]
+	    || ep == bif_export[BIF_exit_1]
+	    || ep == bif_export[BIF_throw_1])) {
+	/*
+	 * We are about to tail apply one of the BIFs
+	 * erlang:error/1, erlang:error/2, erlang:exit/1,
+	 * or erlang:throw/1. Error handling of these BIFs is
+	 * special!
+	 *
+	 * We need 'p->cp' to point into the calling
+	 * function when handling the error after the BIF has
+	 * been applied. This in order to get the topmost
+	 * stackframe correct. Without the following adjustment,
+	 * 'p->cp' will point into the function that called
+	 * current function when handling the error. We add a
+	 * dummy stackframe in order to achive this.
+	 *
+	 * Note that these BIFs unconditionally will cause
+	 * an exception to be raised. That is, our modifications
+	 * of 'p->cp' as well as the stack will be corrected by
+	 * the error handling code.
+	 *
+	 * If we find an exception/return-to trace continuation
+	 * pointer as the topmost continuation pointer, we do not
+	 * need to do anything since the information already will
+	 * be available for generation of the stacktrace.
+	 */
+	int apply_only = stack_offset == 0;
+	BeamInstr *cpp;
+
+	if (apply_only) {
+	    ASSERT(p->cp != NULL);
+	    cpp = p->cp;
+	}
+	else {
+	    ASSERT(is_CP(p->stop[0]));
+	    cpp = cp_val(p->stop[0]);
+	}
+
+	if (cpp != beam_exception_trace
+	    && cpp != beam_return_trace
+	    && cpp != beam_return_to_trace) {
+	    Uint need = stack_offset /* bytes */ / sizeof(Eterm);
+	    if (need == 0)
+		need = 1; /* i_apply_only */
+	    if (p->stop - p->htop < need)
+		erts_garbage_collect(p, (int) need, reg, arity+1);
+	    p->stop -= need;
+
+	    if (apply_only) {
+		/*
+		 * Called from the i_apply_only instruction.
+		 *
+		 * 'p->cp' contains continuation pointer pointing
+		 * into the function that called current function.
+		 * We push that continuation pointer onto the stack,
+		 * and set 'p->cp' to point into current function.
+		 */
+
+		p->stop[0] = make_cp(p->cp);
+		p->cp = I;
+	    }
+	    else {
+		/*
+		 * Called from an i_apply_last_p, or apply_last_IP,
+		 * instruction.
+		 *
+		 * Calling instruction will after we return read
+		 * a continuation pointer from the stack and write
+		 * it to 'p->cp', and then remove the topmost
+		 * stackframe of size 'stack_offset'.
+		 *
+		 * We have sized the dummy-stackframe so that it
+		 * will be removed by the instruction we currently
+		 * are executing, and leave the stackframe that
+		 * normally would have been removed intact.
+		 *
+		 */
+		p->stop[0] = make_cp(I);
+	    }
+	}
+    }
+}
+
 static BeamInstr*
-apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg)
+apply(
+Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg,
+BeamInstr *I, Uint stack_offset)
 {
     int arity;
     Export* ep;
@@ -6254,21 +6355,54 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg)
 	return 0;
     }
 
-    /* The module argument may be either an atom or an abstract module
-     * (currently implemented using tuples, but this might change).
-     */
-    this = THE_NON_VALUE;
-    if (is_not_atom(module)) {
-	Eterm* tp;
+    while (1) {
+	Eterm m, f, a;
+	/* The module argument may be either an atom or an abstract module
+	 * (currently implemented using tuples, but this might change).
+	 */
+	this = THE_NON_VALUE;
+	if (is_not_atom(module)) {
+	    Eterm* tp;
+
+	    if (is_not_tuple(module)) goto error;
+	    tp = tuple_val(module);
+	    if (arityval(tp[0]) < 1) goto error;
+	    this = module;
+	    module = tp[1];
+	    if (is_not_atom(module)) goto error;
+	}
 
-        if (is_not_tuple(module)) goto error;
-        tp = tuple_val(module);
-        if (arityval(tp[0]) < 1) goto error;
-        this = module;
-        module = tp[1];
-        if (is_not_atom(module)) goto error;
+	if (module != am_erlang || function != am_apply)
+	    break;
+
+	/* Adjust for multiple apply of apply/3... */
+
+	a = args;
+	if (is_list(a)) {
+	    Eterm *consp = list_val(a);
+	    m = CAR(consp);
+	    a = CDR(consp);
+	    if (is_list(a)) {
+		consp = list_val(a);
+		f = CAR(consp);
+		a = CDR(consp);
+		if (is_list(a)) {
+		    consp = list_val(a);
+		    a = CAR(consp);
+		    if (is_nil(CDR(consp))) {
+			/* erlang:apply/3 */
+			module = m;
+			function = f;
+			args = a;
+			if (is_not_atom(f))
+			    goto error;
+			continue;
+		    }
+		}
+	    }
+	}
+	break; /* != erlang:apply/3 */
     }
-    
     /*
      * Walk down the 3rd parameter of apply (the argument list) and copy
      * the parameters to the x registers (reg[]). If the module argument
@@ -6306,12 +6440,14 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg)
     } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) {
 	save_calls(p, ep);
     }
+    apply_bif_error_adjustment(p, ep, reg, arity, I, stack_offset);
     DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep);
     return ep->addressv[erts_active_code_ix()];
 }
 
 static BeamInstr*
-fixed_apply(Process* p, Eterm* reg, Uint arity)
+fixed_apply(Process* p, Eterm* reg, Uint arity,
+	    BeamInstr *I, Uint stack_offset)
 {
     Export* ep;
     Eterm module;
@@ -6341,6 +6477,10 @@ fixed_apply(Process* p, Eterm* reg, Uint arity)
         if (is_not_atom(module)) goto error;
         ++arity;
     }
+
+    /* Handle apply of apply/3... */
+    if (module == am_erlang && function == am_apply && arity == 3)
+	return apply(p, reg[0], reg[1], reg[2], reg, I, stack_offset);
     
     /*
      * Get the index into the export table, or failing that the export
@@ -6355,6 +6495,7 @@ fixed_apply(Process* p, Eterm* reg, Uint arity)
     } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) {
 	save_calls(p, ep);
     }
+    apply_bif_error_adjustment(p, ep, reg, arity, I, stack_offset);
     DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep);
     return ep->addressv[erts_active_code_ix()];
 }
-- 
cgit v1.2.3


From cece1693410a89e87343367e86b5dd82ac0e73b7 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 30 Nov 2016 16:46:26 +0100
Subject: New test cases testing stacktrace from apply on erlang:error()

---
 erts/emulator/test/bif_SUITE.erl | 173 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 170 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl
index b8d89126fe..f70fb0e501 100644
--- a/erts/emulator/test/bif_SUITE.erl
+++ b/erts/emulator/test/bif_SUITE.erl
@@ -32,7 +32,8 @@
 	 binary_to_atom/1,binary_to_existing_atom/1,
 	 atom_to_binary/1,min_max/1, erlang_halt/1,
          erl_crash_dump_bytes/1,
-	 is_builtin/1]).
+	 is_builtin/1, error_stacktrace/1,
+	 error_stacktrace_during_call_trace/1]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
@@ -44,8 +45,8 @@ all() ->
      t_list_to_existing_atom, os_env, otp_7526,
      display,
      atom_to_binary, binary_to_atom, binary_to_existing_atom,
-     erl_crash_dump_bytes,
-     min_max, erlang_halt, is_builtin].
+     erl_crash_dump_bytes, min_max, erlang_halt, is_builtin,
+     error_stacktrace, error_stacktrace_during_call_trace].
 
 %% Uses erlang:display to test that erts_printf does not do deep recursion
 display(Config) when is_list(Config) ->
@@ -727,6 +728,172 @@ is_builtin(_Config) ->
 
     ok.
 
+error_stacktrace(Config) when is_list(Config) ->
+    error_stacktrace_test().
+
+error_stacktrace_during_call_trace(Config) when is_list(Config) ->
+    Tracer = spawn_link(fun () ->
+				receive after infinity -> ok end
+			end),
+    Mprog = [{'_',[],[{exception_trace}]}],
+    erlang:trace_pattern({?MODULE,'_','_'}, Mprog, [local]),
+    1 = erlang:trace_pattern({erlang,error,2}, Mprog, [local]),
+    1 = erlang:trace_pattern({erlang,error,1}, Mprog, [local]),
+    erlang:trace(all, true, [call,return_to,timestamp,{tracer, Tracer}]),
+    try
+	error_stacktrace_test()
+    after
+	erlang:trace(all, false, [call,return_to,timestamp,{tracer, Tracer}]),
+	erlang:trace_pattern({erlang,error,2}, false, [local]),
+	erlang:trace_pattern({erlang,error,1}, false, [local]),
+	erlang:trace_pattern({?MODULE,'_','_'}, false, [local]),
+	unlink(Tracer),
+	exit(Tracer, kill),
+	Mon = erlang:monitor(process, Tracer),
+	receive
+	    {'DOWN', Mon, process, Tracer, _} -> ok
+	end
+    end,
+    ok.
+    
+
+error_stacktrace_test() ->
+    Types = [apply_const_last, apply_const, apply_last,
+	     apply, double_apply_const_last, double_apply_const,
+	     double_apply_last, double_apply, multi_apply_const_last,
+	     multi_apply_const, multi_apply_last, multi_apply,
+	     call_const_last, call_last, call_const, call],
+    lists:foreach(fun (Type) ->
+			  {Pid, Mon} = spawn_monitor(
+					 fun () ->
+						 stk([a,b,c,d], Type, error_2)
+					 end),
+			  receive
+			      {'DOWN', Mon, process, Pid, Reason} ->
+				  {oops, Stack} = Reason,
+%%				  io:format("Type: ~p Stack: ~p~n",
+%%					    [Type, Stack]),
+				  [{?MODULE, do_error_2, [Type], _},
+				   {?MODULE, stk, 3, _},
+				   {?MODULE, stk, 3, _}] = Stack
+			  end
+		  end,
+		  Types),
+    lists:foreach(fun (Type) ->
+			  {Pid, Mon} = spawn_monitor(
+					 fun () ->
+						 stk([a,b,c,d], Type, error_1)
+					 end),
+			  receive
+			      {'DOWN', Mon, process, Pid, Reason} ->
+				  {oops, Stack} = Reason,
+%%				  io:format("Type: ~p Stack: ~p~n",
+%%					    [Type, Stack]),
+				  [{?MODULE, do_error_1, 1, _},
+				   {?MODULE, stk, 3, _},
+				   {?MODULE, stk, 3, _}] = Stack
+			  end
+		  end,
+		  Types),
+    ok.
+
+stk([], Type, Func) ->
+    tail(Type, Func, jump),
+    ok;
+stk([_|L], Type, Func) ->
+    stk(L, Type, Func),
+    ok.
+
+tail(Type, Func, jump) ->
+    tail(Type, Func, do);
+tail(Type, error_1, do) ->
+    do_error_1(Type);
+tail(Type, error_2, do) ->
+    do_error_2(Type).
+
+do_error_2(apply_const_last) ->
+    erlang:apply(erlang, error, [oops, [apply_const_last]]);
+do_error_2(apply_const) ->
+    erlang:apply(erlang, error, [oops, [apply_const]]),
+    ok;
+do_error_2(apply_last) ->
+    erlang:apply(id(erlang), id(error), id([oops, [apply_last]]));
+do_error_2(apply) ->
+    erlang:apply(id(erlang), id(error), id([oops, [apply]])),
+    ok;
+do_error_2(double_apply_const_last) ->
+    erlang:apply(erlang, apply, [erlang, error, [oops, [double_apply_const_last]]]);
+do_error_2(double_apply_const) ->
+    erlang:apply(erlang, apply, [erlang, error, [oops, [double_apply_const]]]),
+    ok;
+do_error_2(double_apply_last) ->
+    erlang:apply(id(erlang), id(apply), [id(erlang), id(error), id([oops, [double_apply_last]])]);
+do_error_2(double_apply) ->
+    erlang:apply(id(erlang), id(apply), [id(erlang), id(error), id([oops, [double_apply]])]),
+    ok;
+do_error_2(multi_apply_const_last) ->
+    erlang:apply(erlang, apply, [erlang, apply, [erlang, apply, [erlang, error, [oops, [multi_apply_const_last]]]]]);
+do_error_2(multi_apply_const) ->
+    erlang:apply(erlang, apply, [erlang, apply, [erlang, apply, [erlang, error, [oops, [multi_apply_const]]]]]),
+    ok;
+do_error_2(multi_apply_last) ->
+    erlang:apply(id(erlang), id(apply), [id(erlang), id(apply), [id(erlang), id(apply), [id(erlang), id(error), id([oops, [multi_apply_last]])]]]);
+do_error_2(multi_apply) ->
+    erlang:apply(id(erlang), id(apply), [id(erlang), id(apply), [id(erlang), id(apply), [id(erlang), id(error), id([oops, [multi_apply]])]]]),
+    ok;
+do_error_2(call_const_last) ->
+    erlang:error(oops, [call_const_last]);
+do_error_2(call_last) ->
+    erlang:error(id(oops), id([call_last]));
+do_error_2(call_const) ->
+    erlang:error(oops, [call_const]),
+    ok;
+do_error_2(call) ->
+    erlang:error(id(oops), id([call])).
+
+
+do_error_1(apply_const_last) ->
+    erlang:apply(erlang, error, [oops]);
+do_error_1(apply_const) ->
+    erlang:apply(erlang, error, [oops]),
+    ok;
+do_error_1(apply_last) ->
+    erlang:apply(id(erlang), id(error), id([oops]));
+do_error_1(apply) ->
+    erlang:apply(id(erlang), id(error), id([oops])),
+    ok;
+do_error_1(double_apply_const_last) ->
+    erlang:apply(erlang, apply, [erlang, error, [oops]]);
+do_error_1(double_apply_const) ->
+    erlang:apply(erlang, apply, [erlang, error, [oops]]),
+    ok;
+do_error_1(double_apply_last) ->
+    erlang:apply(id(erlang), id(apply), [id(erlang), id(error), id([oops])]);
+do_error_1(double_apply) ->
+    erlang:apply(id(erlang), id(apply), [id(erlang), id(error), id([oops])]),
+    ok;
+do_error_1(multi_apply_const_last) ->
+    erlang:apply(erlang, apply, [erlang, apply, [erlang, apply, [erlang, error, [oops]]]]);
+do_error_1(multi_apply_const) ->
+    erlang:apply(erlang, apply, [erlang, apply, [erlang, apply, [erlang, error, [oops]]]]),
+    ok;
+do_error_1(multi_apply_last) ->
+    erlang:apply(id(erlang), id(apply), [id(erlang), id(apply), [id(erlang), id(apply), [id(erlang), id(error), id([oops])]]]);
+do_error_1(multi_apply) ->
+    erlang:apply(id(erlang), id(apply), [id(erlang), id(apply), [id(erlang), id(apply), [id(erlang), id(error), id([oops])]]]),
+    ok;
+do_error_1(call_const_last) ->
+    erlang:error(oops);
+do_error_1(call_last) ->
+    erlang:error(id(oops));
+do_error_1(call_const) ->
+    erlang:error(oops),
+    ok;
+do_error_1(call) ->
+    erlang:error(id(oops)).
+
+
+
 
 %% Helpers
     
-- 
cgit v1.2.3


From f34066894c3aa464636090213696cac1aeb67092 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 7 Dec 2016 17:36:17 +0100
Subject: erts: Tidy up in efile_drv.c

ERL_DRV_USE_NO_CALLBACK only meaningful when deselecting.
---
 erts/emulator/drivers/common/efile_drv.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c
index 3adb8db661..d64f015a6a 100644
--- a/erts/emulator/drivers/common/efile_drv.c
+++ b/erts/emulator/drivers/common/efile_drv.c
@@ -1937,7 +1937,8 @@ static void free_sendfile(void *data) {
 	MUTEX_LOCK(d->c.sendfile.q_mtx);
 	driver_deq(d->c.sendfile.port,1);
 	MUTEX_UNLOCK(d->c.sendfile.q_mtx);
-	driver_select(d->c.sendfile.port, (ErlDrvEvent)(long)d->c.sendfile.out_fd, ERL_DRV_USE_NO_CALLBACK|ERL_DRV_WRITE, 0);
+	driver_select(d->c.sendfile.port, (ErlDrvEvent)(long)d->c.sendfile.out_fd,
+                      ERL_DRV_USE_NO_CALLBACK|ERL_DRV_WRITE, 0);
     }
     EF_FREE(data);
 }
@@ -2555,7 +2556,7 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data)
 	      desc->sendfile_state = sending;
 	      desc->d = d;
 	      driver_select(desc->port, (ErlDrvEvent)(long)d->c.sendfile.out_fd,
-			    ERL_DRV_USE_NO_CALLBACK|ERL_DRV_WRITE, 1);
+			    ERL_DRV_USE|ERL_DRV_WRITE, 1);
 	  }
 	  break;
 #endif
-- 
cgit v1.2.3


From 9e334f98679ace2c0fb7b9893c962a9d1a80e41c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 7 Dec 2016 17:24:10 +0100
Subject: erts: Add ERL_ABORT_ON_FAILURE for driver_SUITE

---
 erts/emulator/test/driver_SUITE_data/chkio_drv.c | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/driver_SUITE_data/chkio_drv.c b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
index 614b68e865..8e5e81665c 100644
--- a/erts/emulator/test/driver_SUITE_data/chkio_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
@@ -1397,10 +1397,18 @@ static void assert_print(char* str, int line)
 static void assert_failed(ErlDrvPort port, char* str, int line)
 {
     char buf[30];
+    size_t bufsz = sizeof(buf);
+
     assert_print(str,line);
-    snprintf(buf,sizeof(buf),"failed_at_line_%d",line);
-    driver_failure_atom(port,buf);
-    /*abort();*/
+
+    if (erl_drv_getenv("ERL_ABORT_ON_FAILURE", buf, &bufsz) == 0
+        && (strcmp("true", buf) == 0 || strcmp("yes", buf) == 0)) {
+        abort();
+    }
+    else {
+        snprintf(buf,sizeof(buf),"failed_at_line_%d",line);
+        driver_failure_atom(port,buf);
+    }
 }
 
 #define my_driver_select(PORT,FD,MODE,ON) \
-- 
cgit v1.2.3


From 9ffe2d285943b661317cee2b00d779a2e75a3374 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 1 Dec 2016 19:54:22 +0100
Subject: erts: Add testing + some minor fixes

---
 erts/emulator/beam/erl_nif.h                  |  1 -
 erts/emulator/sys/common/erl_check_io.c       | 33 ++++++-----
 erts/emulator/test/nif_SUITE.erl              | 85 ++++++++++++++++++++++++++-
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c |  1 +
 4 files changed, 101 insertions(+), 19 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index ce8caaf729..bf0f7b1f15 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -146,7 +146,6 @@ typedef struct {
 typedef struct enif_resource_type_t ErlNifResourceType;
 typedef void ErlNifResourceDtor(ErlNifEnv*, void*);
 typedef void ErlNifResourceStop(ErlNifEnv*, void*);
-typedef void ErlNifResourceExit(ErlNifEnv*, void*);
 
 //#ifndef ERL_SYS_DRV
 typedef int ErlNifEvent; /* An event to be selected on. */
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 6f61fc8a28..aeda49b2c6 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1256,11 +1256,18 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
         on = 0;
         mode = ERL_DRV_READ | ERL_DRV_WRITE | ERL_DRV_USE;
         wake_poller = 1; /* to eject fd from pollset (if needed) */
+        ctl_events = ERTS_POLL_EV_IN | ERTS_POLL_EV_OUT;
     }
     else {
         on = 1;
         ASSERT(mode);
         wake_poller = 0;
+        if (mode & ERL_DRV_READ) {
+            ctl_events |= ERTS_POLL_EV_IN;
+        }
+        if (mode & ERL_DRV_WRITE) {
+            ctl_events |= ERTS_POLL_EV_OUT;
+        }
     }
 
 #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
@@ -1303,16 +1310,6 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
         break;
     }}
 
-    ASSERT(state->type == ERTS_EV_TYPE_NONE ||
-           state->type == ERTS_EV_TYPE_NIF);
-
-    if (mode & ERL_DRV_READ) {
-	ctl_events |= ERTS_POLL_EV_IN;
-    }
-    if (mode & ERL_DRV_WRITE) {
-	ctl_events |= ERTS_POLL_EV_OUT;
-    }
-
     ASSERT((state->type == ERTS_EV_TYPE_NIF) ||
 	   (state->type == ERTS_EV_TYPE_NONE && !state->events));
 
@@ -1355,7 +1352,6 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
             ASSERT(state->type == ERTS_EV_TYPE_NIF);
             ASSERT(state->driver.stop.resource == resource);
 	    if (ctl_events & ERTS_POLL_EV_IN) {
-                ASSERT(is_nil(state->driver.nif->in.pid));
 		state->driver.nif->in.pid = id;
                 if (is_immed(ref)) {
                     state->driver.nif->in.immed = ref;
@@ -1370,7 +1366,6 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
                 state->driver.nif->in.ddeselect_cnt = 0;
             }
 	    if (ctl_events & ERTS_POLL_EV_OUT) {
-                ASSERT(is_nil(state->driver.nif->out.pid));
 		state->driver.nif->out.pid = id;
                 if (is_immed(ref)) {
                     state->driver.nif->out.immed = ref;
@@ -1384,7 +1379,6 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
                 }
                 state->driver.nif->out.ddeselect_cnt = 0;
             }
-            state->flags |= ERTS_EV_FLAG_USED;
 	}
 	else { /* off */
 	    if (state->type == ERTS_EV_TYPE_NIF) {
@@ -1395,7 +1389,6 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
                 state->driver.nif->out.pid = NIL;
                 state->driver.nif->in.ddeselect_cnt = 0;
                 state->driver.nif->out.ddeselect_cnt = 0;
-                state->flags &= ~ERTS_EV_FLAG_USED;
                 if (old_events != 0) {
                     remember_removed(state, &pollset);
                 }
@@ -2063,7 +2056,7 @@ send_event_tuple(struct erts_nif_select_event* e, ErlNifResource* resource,
     ErlOffHeap* ohp;
     ErtsBinary* bin;
     Eterm* hp;
-    Uint hsz = 5 + PROC_BIN_SIZE + REF_THING_SIZE; /* {select, Resource, Ref, EventAtom} */
+    Uint hsz;
     Eterm resource_term, ref_term, tuple;
 
     if (!rp) {
@@ -2073,6 +2066,14 @@ send_event_tuple(struct erts_nif_select_event* e, ErlNifResource* resource,
 
     bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
 
+     /* {select, Resource, Ref, EventAtom} */
+    if (is_value(e->immed)) {
+        hsz = 5 + PROC_BIN_SIZE;
+    }
+    else {
+        hsz = 5 + PROC_BIN_SIZE + REF_THING_SIZE;
+    }
+
     mp = erts_alloc_message_heap(rp, &rp_locks, hsz, &hp, &ohp);
 
     resource_term = erts_mk_magic_binary_term(&hp, ohp, &bin->binary);
@@ -2083,8 +2084,8 @@ send_event_tuple(struct erts_nif_select_event* e, ErlNifResource* resource,
     else {
         write_ref_thing(hp, e->refn[0], e->refn[1], e->refn[2]);
         ref_term = make_internal_ref(hp);
+        hp += REF_THING_SIZE;
     }
-    hp += REF_THING_SIZE;
     tuple = TUPLE4(hp, am_select, resource_term, ref_term, event_atom);
 
     ERL_MESSAGE_TOKEN(mp) = am_undefined;
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 8c761202d6..2401ff708c 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -447,6 +447,8 @@ select(Config) when is_list(Config) ->
     {R,W} = pipe_nif(),
     ok = write_nif(W, <<"hej">>),
     <<"hej">> = read_nif(R, 3),
+
+    %% Wait for read
     eagain = read_nif(R, 3),
     0 = select_nif(R,?ERL_NIF_SELECT_READ,R,Ref),
     [] = flush(),
@@ -454,15 +456,94 @@ select(Config) when is_list(Config) ->
     [{select, R, Ref, ready_input}] = flush(),
     <<"hej">> = read_nif(R, 3),
 
-    %% To be extended...
+    %% Wait for write
+    Written = write_full(W, $a),
+    0 = select_nif(W,?ERL_NIF_SELECT_WRITE,W,Ref),
+    [] = flush(),
+    Half = byte_size(Written) div 2,
+    <<First:Half/binary,Second/binary>> = Written,
+    First = read_nif(R,Half),
+    [{select, W, Ref, ready_output}] = flush(),
+    Third = write_full(W, $A),
+    Half2 = byte_size(Second),
+    <<Second:Half2/binary, Third/binary>> = read_nif(R, byte_size(Written)),
+
+    %% Close write and wait for EOF
+    eagain = read_nif(R, 1),
+    0 = select_nif(W,?ERL_NIF_SELECT_STOP,W,Ref),
+    timer:sleep(10),
+    true = is_closed_nif(W),
+    [] = flush(),
+    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,Ref),
+    [{select, R, Ref, ready_input}] = flush(),
+    eof = read_nif(R,1),
 
     0 = select_nif(R,?ERL_NIF_SELECT_STOP,R,Ref),
-    0 = select_nif(W,?ERL_NIF_SELECT_STOP,W,Ref),
+    timer:sleep(10),
+    true = is_closed_nif(R),
+
+    select_2(Config).
+
+select_2(Config) ->
+    erlang:garbage_collect(),
+    {_,_,2} = last_resource_dtor_call(),
+
+    Ref1 = make_ref(),
+    Ref2 = make_ref(),
+    {R,W} = pipe_nif(),
+
+    %% Change ref
+    eagain = read_nif(R, 1),
+    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,Ref1),
+    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,Ref2),
+
+    [] = flush(),
+    ok = write_nif(W, <<"hej">>),
+    [{select, R, Ref2, ready_input}] = flush(),
+    <<"hej">> = read_nif(R, 3),
+
+    %% Change pid
+    eagain = read_nif(R, 1),
+    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,Ref1),
+    Papa = self(),
+    Pid2 = spawn_link(fun() ->
+                              0 = select_nif(R,?ERL_NIF_SELECT_READ,R,Ref1),
+                              [] = flush(),
+                              Papa ! sync,
+                              [{select, R, Ref1, ready_input}] = flush(),
+                              <<"hej">> = read_nif(R, 3),
+                              Papa ! done
+                      end),
+    sync = receive_any(),
+    ok = write_nif(W, <<"hej">>),
+    done = receive_any(),
+    [] = flush(),
+
+    0 = select_nif(R,?ERL_NIF_SELECT_STOP,R,Ref1),
+    0 = select_nif(W,?ERL_NIF_SELECT_STOP,W,Ref1),
     timer:sleep(10),
     true = is_closed_nif(R),
     true = is_closed_nif(W),
+
+    select_3(Config).
+
+select_3(Config) ->
+    erlang:garbage_collect(),
+    {_,_,2} = last_resource_dtor_call(),
     ok.
 
+
+
+write_full(W, C) ->
+    write_full(W, C, <<>>).
+write_full(W, C, Acc) ->
+    case write_nif(W, <<C>>) of
+        ok ->
+            write_full(W, (C+1) band 255, <<Acc/binary, C>>);
+        {eagain,0} ->
+            Acc
+    end.
+
 hipe(Config) when is_list(Config) ->
     Data = proplists:get_value(data_dir, Config),
     Priv = proplists:get_value(priv_dir, Config),
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index a4915b13c4..f6ccd3e6ba 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -2176,6 +2176,7 @@ static ERL_NIF_TERM is_closed_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM a
 static void fd_resource_dtor(ErlNifEnv* env, void* obj)
 {
     struct fd_resource* rsrc = (struct fd_resource*)obj;
+    resource_dtor(env, obj);
     if (rsrc->fd >= 0)
         close(rsrc->fd);
 }
-- 
cgit v1.2.3


From 387ff8e3347d21e9ca5ad3d8c3a694bc79d38bca Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 8 Dec 2016 19:26:14 +0100
Subject: Add stop arguments: fd and is_direct_call

---
 erts/emulator/beam/erl_nif.c                  |  5 ++-
 erts/emulator/beam/erl_nif.h                  | 16 +++----
 erts/emulator/beam/global.h                   |  2 +-
 erts/emulator/sys/common/erl_check_io.c       |  8 +++-
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 65 +++++++++++++--------------
 5 files changed, 49 insertions(+), 47 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 27abba7cfd..4e41944ccb 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2124,12 +2124,13 @@ static void nif_resource_dtor(Binary* bin)
     }
 }
 
-void erts_resource_stop(ErlNifResource* resource)
+void erts_resource_stop(ErlNifResource* resource, ErlNifEvent e,
+                        int is_direct_call)
 {
     struct enif_msg_environment_t msg_env;
     ASSERT(resource->type->stop);
     pre_nif_noproc(&msg_env, resource->type->owner, NULL);
-    resource->type->stop(&msg_env.env, resource->data);
+    resource->type->stop(&msg_env.env, resource->data, e, is_direct_call);
     post_nif_noproc(&msg_env);
 }
 
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index bf0f7b1f15..78e0fa1864 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -138,18 +138,18 @@ typedef struct
     void* ref_bin;
 }ErlNifBinary;
 
-typedef struct {
-    void (*dtor)(ErlNifEnv* env, void* obj);
-    void (*stop)(ErlNifEnv* env, void* obj); /* at ERL_NIF_SELECT_STOP event */
-} ErlNifResourceTypeInit;
+//#ifndef ERL_SYS_DRV
+typedef int ErlNifEvent; /* An event to be selected on. */
+//#endif
 
 typedef struct enif_resource_type_t ErlNifResourceType;
 typedef void ErlNifResourceDtor(ErlNifEnv*, void*);
-typedef void ErlNifResourceStop(ErlNifEnv*, void*);
+typedef void ErlNifResourceStop(ErlNifEnv*, void*, ErlNifEvent, int is_direct_call);
 
-//#ifndef ERL_SYS_DRV
-typedef int ErlNifEvent; /* An event to be selected on. */
-//#endif
+typedef struct {
+    ErlNifResourceDtor* dtor;
+    ErlNifResourceStop* stop;  /* at ERL_NIF_SELECT_STOP event */
+} ErlNifResourceTypeInit;
 
 typedef enum
 {
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 5c5693a315..2decb56544 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -95,7 +95,7 @@ extern void erts_pre_dirty_nif(ErtsSchedulerData *,
 			       struct erl_module_nif*);
 extern void erts_post_dirty_nif(struct enif_environment_t* env);
 #endif
-extern void erts_resource_stop(ErlNifResource* resource);
+extern void erts_resource_stop(ErlNifResource*, ErlNifEvent, int is_direct_call);
 extern Eterm erts_nif_taints(Process* p);
 extern void erts_print_nif_taints(fmtfn_t to, void* to_arg);
 void erts_unload_nif(struct erl_module_nif* nif);
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index aeda49b2c6..a533a10c22 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -450,7 +450,7 @@ forget_removed(struct pollset_info* psi)
 	    }
 	}
         if (resource) {
-            erts_resource_stop(resource);
+            erts_resource_stop(resource, (ErlNifEvent)fd, 0);
             enif_release_resource(resource->data);
         }
 
@@ -1436,7 +1436,7 @@ done:
 done_unknown:
     erts_smp_mtx_unlock(fd_mtx(fd));
     if (call_stop) {
-        erts_resource_stop(resource);
+        erts_resource_stop(resource, (ErlNifEvent)fd, 1);
         if (call_stop == CALL_STOP_AND_RELEASE) {
             enif_release_resource(resource->data);
         }
@@ -2141,6 +2141,10 @@ ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(int set,
     ERTS_CIO_POLL_INTR_TMD(pollset.ps, set, timeout_time);
 }
 
+/*
+ * Number of ignored events, for a lingering fd added by enif_select(),
+ * until we deselect fd-event from pollset.
+ */
 #define ERTS_NIF_DELAYED_DESELECT 20
 
 void
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index f6ccd3e6ba..878a9ffda9 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -108,13 +108,14 @@ struct binary_resource {
 
 static ErlNifResourceType* fd_resource_type;
 static void fd_resource_dtor(ErlNifEnv* env, void* obj);
-static void fd_resource_stop(ErlNifEnv* env, void* obj);
+static void fd_resource_stop(ErlNifEnv* env, void* obj, ErlNifEvent, int);
 static ErlNifResourceTypeInit fd_rt_init = {
     fd_resource_dtor,
     fd_resource_stop
 };
 struct fd_resource {
     int fd;
+    int was_selected;
 };
 
 
@@ -1128,10 +1129,6 @@ static ERL_NIF_TERM make_term_string(struct make_term_info* mti, int n)
 {
     return enif_make_string(mti->dst_env, "Hello!", ERL_NIF_LATIN1);
 }
-static ERL_NIF_TERM make_term_ref(struct make_term_info* mti, int n)
-{
-    return enif_make_ref(mti->dst_env);
-}
 static ERL_NIF_TERM make_term_sub_binary(struct make_term_info* mti, int n)
 {
     ERL_NIF_TERM orig;
@@ -1239,7 +1236,6 @@ static Make_term_Func* make_funcs[] = {
     make_term_atom,
     make_term_existing_atom,
     make_term_string,
-    //make_term_ref,
     make_term_sub_binary,
     make_term_uint,
     make_term_long,
@@ -2032,26 +2028,23 @@ static ERL_NIF_TERM format_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg
 }
 
 
-static int get_fd(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifEvent* fd)
+static int get_fd(ErlNifEnv* env, ERL_NIF_TERM term, struct fd_resource** rsrc)
 {
-    struct fd_resource* rsrc;
-
-    if (!enif_get_resource(env, term, fd_resource_type, (void**)&rsrc)) {
+    if (!enif_get_resource(env, term, fd_resource_type, (void**)rsrc)) {
         return 0;
     }
-    *fd = rsrc->fd;
     return 1;
 }
 
 static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    ErlNifEvent e;
+    struct fd_resource* fdr;
     enum ErlNifSelectFlags mode;
     void* obj;
     ERL_NIF_TERM ref;
     int retval;
 
-    if (!get_fd(env, argv[0], &e)
+    if (!get_fd(env, argv[0], &fdr)
         || !enif_get_uint(env, argv[1], &mode)
         || !enif_get_resource(env, argv[2], fd_resource_type, &obj))
     {
@@ -2060,7 +2053,8 @@ static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
 
     ref = argv[3];
 
-    retval = enif_select(env, e, mode, obj, ref);
+    fdr->was_selected = 1;
+    retval = enif_select(env, fdr->fd, mode, obj, ref);
 
     return enif_make_int(env, retval);
 }
@@ -2070,7 +2064,6 @@ static ERL_NIF_TERM pipe_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]
     struct fd_resource* read_rsrc;
     struct fd_resource* write_rsrc;
     ERL_NIF_TERM read_fd, write_fd;
-    unsigned char *inp, *outp;
     int fds[2], flags;
 
     if (pipe(fds) < 0)
@@ -2099,16 +2092,16 @@ static ERL_NIF_TERM pipe_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]
 
 static ERL_NIF_TERM write_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    ErlNifEvent fd;
+    struct fd_resource* fdr;
     ErlNifBinary bin;
     int n, written = 0;
 
-    if (!get_fd(env, argv[0], &fd)
+    if (!get_fd(env, argv[0], &fdr)
         || !enif_inspect_binary(env, argv[1], &bin))
         return enif_make_badarg(env);
 
     for (;;) {
-        n = write(fd, bin.data + written, bin.size - written);
+        n = write(fdr->fd, bin.data + written, bin.size - written);
         if (n >= 0) {
             written += n;
             if (written == bin.size) {
@@ -2129,19 +2122,19 @@ static ERL_NIF_TERM write_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
 
 static ERL_NIF_TERM read_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    ErlNifEvent fd;
+    struct fd_resource* fdr;
     unsigned char* buf;
-    int n, count, nread = 0;
+    int n, count;
     ERL_NIF_TERM res;
 
-    if (!get_fd(env, argv[0], &fd)
+    if (!get_fd(env, argv[0], &fdr)
         || !enif_get_int(env, argv[1], &count) || count < 1)
         return enif_make_badarg(env);
 
     buf = enif_make_new_binary(env, count, &res);
 
     for (;;) {
-        n = read(fd, buf, count);
+        n = read(fdr->fd, buf, count);
         if (n > 0) {
             if (n < count) {
                 res = enif_make_sub_binary(env, res, 0, n);
@@ -2165,29 +2158,33 @@ static ERL_NIF_TERM read_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]
 
 static ERL_NIF_TERM is_closed_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    ErlNifEvent fd;
+    struct fd_resource* fdr;
 
-    if (!get_fd(env, argv[0], &fd))
+    if (!get_fd(env, argv[0], &fdr))
         return enif_make_badarg(env);
 
-    return fd < 0 ? atom_true : atom_false;
+    return fdr->fd < 0 ? atom_true : atom_false;
 }
 
 static void fd_resource_dtor(ErlNifEnv* env, void* obj)
 {
-    struct fd_resource* rsrc = (struct fd_resource*)obj;
+    struct fd_resource* fdr = (struct fd_resource*)obj;
     resource_dtor(env, obj);
-    if (rsrc->fd >= 0)
-        close(rsrc->fd);
+    if (fdr->fd >= 0) {
+        assert(!fdr->was_selected);
+        close(fdr->fd);
+    }
 }
 
-static void fd_resource_stop(ErlNifEnv* env, void* obj)
+static void fd_resource_stop(ErlNifEnv* env, void* obj, ErlNifEvent fd,
+                             int is_direct_call)
 {
-    struct fd_resource* rsrc = (struct fd_resource*)obj;
-    if (rsrc->fd >= 0) {
-        close(rsrc->fd);
-        rsrc->fd = -1;   /* thread safety ? */
-    }
+    struct fd_resource* fdr = (struct fd_resource*)obj;
+    assert(fd == fdr->fd);
+    assert(fd >= 0);
+    close(fd);
+    fdr->fd = -1;   /* thread safety ? */
+    fdr->was_selected = 0;
 }
 
 
-- 
cgit v1.2.3


From 861b276f27952ecbb5a89748b86b7513946617f3 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 16 Dec 2016 16:26:08 +0100
Subject: nif_SUITE: Add last_fd_stop_call

---
 erts/emulator/test/nif_SUITE.erl              | 13 ++++++++++---
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 27 +++++++++++++++++++++++++--
 2 files changed, 35 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 2401ff708c..c2429c3405 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -444,7 +444,7 @@ select(Config) when is_list(Config) ->
     ensure_lib_loaded(Config),
 
     Ref = make_ref(),
-    {R,W} = pipe_nif(),
+    {{R, R_ptr}, {W, W_ptr}} = pipe_nif(),
     ok = write_nif(W, <<"hej">>),
     <<"hej">> = read_nif(R, 3),
 
@@ -472,6 +472,7 @@ select(Config) when is_list(Config) ->
     eagain = read_nif(R, 1),
     0 = select_nif(W,?ERL_NIF_SELECT_STOP,W,Ref),
     timer:sleep(10),
+    {1, {W_ptr,_}} = last_fd_stop_call(),
     true = is_closed_nif(W),
     [] = flush(),
     0 = select_nif(R,?ERL_NIF_SELECT_READ,R,Ref),
@@ -480,6 +481,7 @@ select(Config) when is_list(Config) ->
 
     0 = select_nif(R,?ERL_NIF_SELECT_STOP,R,Ref),
     timer:sleep(10),
+    {1, {R_ptr,_}} = last_fd_stop_call(),
     true = is_closed_nif(R),
 
     select_2(Config).
@@ -490,7 +492,7 @@ select_2(Config) ->
 
     Ref1 = make_ref(),
     Ref2 = make_ref(),
-    {R,W} = pipe_nif(),
+    {{R, R_ptr}, {W, W_ptr}} = pipe_nif(),
 
     %% Change ref
     eagain = read_nif(R, 1),
@@ -520,9 +522,13 @@ select_2(Config) ->
     [] = flush(),
 
     0 = select_nif(R,?ERL_NIF_SELECT_STOP,R,Ref1),
-    0 = select_nif(W,?ERL_NIF_SELECT_STOP,W,Ref1),
     timer:sleep(10),
+    {1, {R_ptr,_}} = last_fd_stop_call(),
     true = is_closed_nif(R),
+
+    0 = select_nif(W,?ERL_NIF_SELECT_STOP,W,Ref1),
+    timer:sleep(10),
+    {1, {W_ptr,1}} = last_fd_stop_call(),
     true = is_closed_nif(W),
 
     select_3(Config).
@@ -2397,6 +2403,7 @@ pipe_nif() -> ?nif_stub.
 write_nif(_,_) -> ?nif_stub.
 read_nif(_,_) -> ?nif_stub.
 is_closed_nif(_) -> ?nif_stub.
+last_fd_stop_call() -> ?nif_stub.
 
 %% maps
 is_map_nif(_) -> ?nif_stub.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 878a9ffda9..813d19ae90 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -2087,7 +2087,9 @@ static ERL_NIF_TERM pipe_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]
     enif_release_resource(read_rsrc);
     enif_release_resource(write_rsrc);
 
-    return enif_make_tuple2(env, read_fd, write_fd);
+    return enif_make_tuple2(env,
+               enif_make_tuple2(env, read_fd, make_pointer(env, read_rsrc)),
+               enif_make_tuple2(env, write_fd, make_pointer(env, write_rsrc)));
 }
 
 static ERL_NIF_TERM write_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
@@ -2176,17 +2178,37 @@ static void fd_resource_dtor(ErlNifEnv* env, void* obj)
     }
 }
 
+static struct {
+    void* obj;
+    int was_direct_call;
+}last_fd_stop;
+int fd_stop_cnt = 0;
+
 static void fd_resource_stop(ErlNifEnv* env, void* obj, ErlNifEvent fd,
                              int is_direct_call)
 {
     struct fd_resource* fdr = (struct fd_resource*)obj;
     assert(fd == fdr->fd);
     assert(fd >= 0);
+
+    last_fd_stop.obj = obj;
+    last_fd_stop.was_direct_call = is_direct_call;
+    fd_stop_cnt++;
+
     close(fd);
     fdr->fd = -1;   /* thread safety ? */
     fdr->was_selected = 0;
 }
 
+static ERL_NIF_TERM last_fd_stop_call(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ERL_NIF_TERM last, ret;
+    last = enif_make_tuple2(env, make_pointer(env, last_fd_stop.obj),
+                            enif_make_int(env, last_fd_stop.was_direct_call));
+    ret = enif_make_tuple2(env, enif_make_int(env, fd_stop_cnt), last);
+    fd_stop_cnt = 0;
+    return ret;
+}
 
 
 static ErlNifFunc nif_funcs[] =
@@ -2270,7 +2292,8 @@ static ErlNifFunc nif_funcs[] =
     {"pipe_nif", 0, pipe_nif},
     {"write_nif", 2, write_nif},
     {"read_nif", 2, read_nif},
-    {"is_closed_nif", 1, is_closed_nif}
+    {"is_closed_nif", 1, is_closed_nif},
+    {"last_fd_stop_call", 0, last_fd_stop_call}
 };
 
 ERL_NIF_INIT(nif_SUITE,nif_funcs,load,NULL,upgrade,unload)
-- 
cgit v1.2.3


From 4c5d33114edea787833d6aa1b0d51ea9d98b3321 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 16 Dec 2016 19:30:22 +0100
Subject: Add ErlNifSelectReturn

---
 erts/emulator/beam/erl_drv_nif.h              |  8 ++++++++
 erts/emulator/sys/common/erl_check_io.c       | 27 +++++++++++++--------------
 erts/emulator/sys/common/erl_check_io.h       |  4 ++--
 erts/emulator/sys/unix/sys.c                  |  2 +-
 erts/emulator/test/nif_SUITE.erl              | 18 +++++++++++++-----
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c |  4 ++--
 6 files changed, 39 insertions(+), 24 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h
index e4ebcdb1d4..46bb06d642 100644
--- a/erts/emulator/beam/erl_drv_nif.h
+++ b/erts/emulator/beam/erl_drv_nif.h
@@ -56,6 +56,14 @@ enum ErlNifSelectFlags {
     ERL_NIF_SELECT_STOP      = (1 << 2)
 };
 
+enum ErlNifSelectReturn {
+    ERL_NIF_SELECT_ERROR          = (1 << 0),
+    ERL_NIF_SELECT_STOP_CALLED    = (1 << 1),
+    ERL_NIF_SELECT_STOP_SCHEDULED = (1 << 2),
+    ERL_NIF_SELECT_INVALID_EVENT  = (1 << 3),
+    ERL_NIF_SELECT_FAILED         = (1 << 4)
+};
+
 #ifdef SIZEOF_CHAR
 #  define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR
 #  undef SIZEOF_CHAR
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index a533a10c22..4fc95624c7 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1198,7 +1198,7 @@ done_unknown:
     return ret;
 }
 
-int
+enum ErlNifSelectReturn
 ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
                              ErlNifEvent e,
                              enum ErlNifSelectFlags mode,
@@ -1213,7 +1213,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
     ErtsPollEvents new_events, old_events;
     ErtsDrvEventState *state;
     int wake_poller;
-    int ret;
+    enum ErlNifSelectReturn ret;
     enum { NO_STOP=0, CALL_STOP, CALL_STOP_AND_RELEASE } call_stop = NO_STOP;
 #if ERTS_CIO_HAVE_DRV_EVENT
     ErtsDrvEventDataState *free_event = NULL;
@@ -1227,11 +1227,11 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) {
 	if (fd < 0) {
-	    return -1;
+	    return ERL_NIF_SELECT_ERROR | ERL_NIF_SELECT_INVALID_EVENT;
 	}
 	if (fd >= max_fds) {
 	    nif_select_large_fd_error(fd, mode, resource, ref);
-            return -1;
+            return ERL_NIF_SELECT_ERROR | ERL_NIF_SELECT_INVALID_EVENT;
 	}
 	grow_drv_ev_state(fd);
     }
@@ -1250,7 +1250,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
         if (IS_FD_UNKNOWN(state)) {
             /* fast track to stop callback */
             call_stop = CALL_STOP;
-            ret = 0;
+            ret = ERL_NIF_SELECT_STOP_CALLED;
             goto done_unknown;
         }
         on = 0;
@@ -1303,7 +1303,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
         print_nif_select_op(dsbufp, fd, mode, resource, ref);
         steal_pending_stop_nif(dsbufp, resource, state, mode, on);
         if (state->type == ERTS_EV_TYPE_STOP_NIF) {
-            ret = 0;
+            ret = ERL_NIF_SELECT_STOP_SCHEDULED;  /* ?? */
             goto done;
         }
         ASSERT(state->type == ERTS_EV_TYPE_NONE);
@@ -1325,7 +1325,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
             state->driver.nif->out.ddeselect_cnt = 0;
             state->driver.stop.resource = NULL;
 	}
-	ret = -1;
+	ret = ERL_NIF_SELECT_ERROR | ERL_NIF_SELECT_FAILED;
 	goto done;
     }
 
@@ -1339,8 +1339,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
 	   || state->type == ERTS_EV_TYPE_NONE);
 
     state->events = new_events;
-    if (ctl_events) {
-	if (on) {
+    if (on) {
             Uint32* refn;
 	    if (!state->driver.nif)
 		state->driver.nif = alloc_nif_select_data();
@@ -1379,8 +1378,9 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
                 }
                 state->driver.nif->out.ddeselect_cnt = 0;
             }
-	}
-	else { /* off */
+            ret = 0;
+    }
+    else { /* off */
 	    if (state->type == ERTS_EV_TYPE_NIF) {
                 //erts_fprintf(stderr, "SVERK: enif select clear fd=%d inpid=%T inrsrc=%p\n",
                 //             state->fd, state->driver.nif->inpid,
@@ -1408,6 +1408,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
                     call_stop = CALL_STOP;
                 }
                 state->type = ERTS_EV_TYPE_NONE;
+                ret = ERL_NIF_SELECT_STOP_CALLED;
             }
             else {
                 /* Not safe to close fd, postpone stop_select callback. */
@@ -1418,12 +1419,10 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
                     enif_keep_resource(resource);
                 }
                 state->type = ERTS_EV_TYPE_STOP_NIF;
+                ret = ERL_NIF_SELECT_STOP_SCHEDULED;
 	    }
-	}
     }
 
-    ret = 0;
-
 done:
 
     check_fd_cleanup(state,
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index 242e7cfcf5..41e2e9210a 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -34,8 +34,8 @@
 
 int driver_select_kp(ErlDrvPort, ErlDrvEvent, int, int);
 int driver_select_nkp(ErlDrvPort, ErlDrvEvent, int, int);
-int enif_select_kp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, Eterm);
-int enif_select_nkp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, Eterm);
+enum ErlNifSelectReturn enif_select_kp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, Eterm);
+enum ErlNifSelectReturn enif_select_nkp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, Eterm);
 int driver_event_kp(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
 int driver_event_nkp(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
 Uint erts_check_io_size_kp(void);
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 789b455f2d..5ce57b7b1b 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -161,7 +161,7 @@ int erts_use_kernel_poll = 0;
 
 struct {
     int (*select)(ErlDrvPort, ErlDrvEvent, int, int);
-    int (*enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, Eterm);
+    enum ErlNifSelectReturn (*enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, Eterm);
     int (*event)(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
     void (*check_io_as_interrupt)(void);
     void (*check_io_interrupt)(int);
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index c2429c3405..8795a3b24a 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -440,6 +440,13 @@ t_on_load(Config) when is_list(Config) ->
 -define(ERL_NIF_SELECT_WRITE, (1 bsl 1)).
 -define(ERL_NIF_SELECT_STOP, (1 bsl 2)).
 
+-define(ERL_NIF_SELECT_ERROR, (1 bsl 0)).
+-define(ERL_NIF_SELECT_STOP_CALLED, (1 bsl 1)).
+-define(ERL_NIF_SELECT_STOP_SCHEDULED, (1 bsl 2)).
+-define(ERL_NIF_SELECT_INVALID_EVENT, (1 bsl 3)).
+-define(ERL_NIF_SELECT_FAILED, (1 bsl 4)).
+
+
 select(Config) when is_list(Config) ->
     ensure_lib_loaded(Config),
 
@@ -470,7 +477,7 @@ select(Config) when is_list(Config) ->
 
     %% Close write and wait for EOF
     eagain = read_nif(R, 1),
-    0 = select_nif(W,?ERL_NIF_SELECT_STOP,W,Ref),
+    check_stop_ret(select_nif(W,?ERL_NIF_SELECT_STOP,W,Ref)),
     timer:sleep(10),
     {1, {W_ptr,_}} = last_fd_stop_call(),
     true = is_closed_nif(W),
@@ -479,7 +486,7 @@ select(Config) when is_list(Config) ->
     [{select, R, Ref, ready_input}] = flush(),
     eof = read_nif(R,1),
 
-    0 = select_nif(R,?ERL_NIF_SELECT_STOP,R,Ref),
+    check_stop_ret(select_nif(R,?ERL_NIF_SELECT_STOP,R,Ref)),
     timer:sleep(10),
     {1, {R_ptr,_}} = last_fd_stop_call(),
     true = is_closed_nif(R),
@@ -521,12 +528,12 @@ select_2(Config) ->
     done = receive_any(),
     [] = flush(),
 
-    0 = select_nif(R,?ERL_NIF_SELECT_STOP,R,Ref1),
+    check_stop_ret(select_nif(R,?ERL_NIF_SELECT_STOP,R,Ref1)),
     timer:sleep(10),
     {1, {R_ptr,_}} = last_fd_stop_call(),
     true = is_closed_nif(R),
 
-    0 = select_nif(W,?ERL_NIF_SELECT_STOP,W,Ref1),
+    ?ERL_NIF_SELECT_STOP_CALLED = select_nif(W,?ERL_NIF_SELECT_STOP,W,Ref1),
     timer:sleep(10),
     {1, {W_ptr,1}} = last_fd_stop_call(),
     true = is_closed_nif(W),
@@ -538,7 +545,8 @@ select_3(Config) ->
     {_,_,2} = last_resource_dtor_call(),
     ok.
 
-
+check_stop_ret(?ERL_NIF_SELECT_STOP_CALLED) -> ok;    
+check_stop_ret(?ERL_NIF_SELECT_STOP_SCHEDULED) -> ok.
 
 write_full(W, C) ->
     write_full(W, C, <<>>).
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 813d19ae90..d2af081a22 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -2042,7 +2042,7 @@ static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
     enum ErlNifSelectFlags mode;
     void* obj;
     ERL_NIF_TERM ref;
-    int retval;
+    enum ErlNifSelectReturn retval;
 
     if (!get_fd(env, argv[0], &fdr)
         || !enif_get_uint(env, argv[1], &mode)
@@ -2056,7 +2056,7 @@ static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
     fdr->was_selected = 1;
     retval = enif_select(env, fdr->fd, mode, obj, ref);
 
-    return enif_make_int(env, retval);
+    return enif_make_int(env, (int)retval);
 }
 
 static ERL_NIF_TERM pipe_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
-- 
cgit v1.2.3


From ca7e946af9c2fdc86c1c74259ee7b6881c5aec1e Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikpelinux@gmail.com>
Date: Mon, 19 Dec 2016 11:17:05 +0100
Subject: erts: add erlang:system_info(atom_count)

---
 erts/emulator/beam/erl_bif_info.c        |  3 +++
 erts/emulator/test/system_info_SUITE.erl | 15 +++++++++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 0e0842e139..9a3b78ae8d 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2863,6 +2863,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     else if (ERTS_IS_ATOM_STR("atom_limit",BIF_ARG_1)) {
         BIF_RET(make_small(erts_get_atom_limit()));
     }
+    else if (ERTS_IS_ATOM_STR("atom_count",BIF_ARG_1)) {
+        BIF_RET(make_small(atom_table_size()));
+    }
     else if (ERTS_IS_ATOM_STR("tolerant_timeofday",BIF_ARG_1)) {
 	if (erts_has_time_correction()
 	    && erts_time_offset_state() == ERTS_TIME_OFFSET_FINAL) {
diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl
index 3d9e74472b..6a772bf7c9 100644
--- a/erts/emulator/test/system_info_SUITE.erl
+++ b/erts/emulator/test/system_info_SUITE.erl
@@ -36,7 +36,8 @@
 -export([all/0, suite/0]).
 
 -export([process_count/1, system_version/1, misc_smoke_tests/1,
-         heap_size/1, wordsize/1, memory/1, ets_limit/1, atom_limit/1]).
+         heap_size/1, wordsize/1, memory/1, ets_limit/1, atom_limit/1,
+         atom_count/1]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
@@ -44,7 +45,7 @@ suite() ->
 
 all() -> 
     [process_count, system_version, misc_smoke_tests,
-     heap_size, wordsize, memory, ets_limit, atom_limit].
+     heap_size, wordsize, memory, ets_limit, atom_limit, atom_count].
 
 %%%
 %%% The test cases -------------------------------------------------------------
@@ -550,3 +551,13 @@ get_atom_limit(Config, AtomsMax) ->
     end,
     stop_node(Node),
     Res.
+
+%% Verify that system_info(atom_count) works.
+atom_count(Config) when is_list(Config) ->
+    Limit = erlang:system_info(atom_limit),
+    Count1 = erlang:system_info(atom_count),
+    list_to_atom(integer_to_list(erlang:unique_integer())),
+    Count2 = erlang:system_info(atom_count),
+    true = Limit >= Count2,
+    true = Count2 > Count1,
+    ok.
-- 
cgit v1.2.3


From 5621aebf1375fa0e6eb938490c3391300e4efbfe Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 20 Dec 2016 15:45:03 +0100
Subject: erts: Add missing make rules for port_SUITE

Seems some default rule made it work before
until an upgrade of FreeBSD resulted in:

make: don't know how to make dead_port. Stop
---
 erts/emulator/test/port_SUITE_data/Makefile.src | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/port_SUITE_data/Makefile.src b/erts/emulator/test/port_SUITE_data/Makefile.src
index fb7685c4b6..3a343e6d17 100644
--- a/erts/emulator/test/port_SUITE_data/Makefile.src
+++ b/erts/emulator/test/port_SUITE_data/Makefile.src
@@ -20,6 +20,12 @@ echo_args@exe@: echo_args@obj@
 echo_args@obj@: echo_args.c
 	$(CC) -c -o echo_args@obj@ $(CFLAGS) echo_args.c
 
+dead_port@exe@: dead_port@obj@
+	$(LD) $(CROSSLDFLAGS) -o dead_port dead_port@obj@ @LIBS@
+
+dead_port@obj@: dead_port.c
+	$(CC) -c -o dead_port@obj@ $(CFLAGS) dead_port.c
+
 port_test.@EMULATOR@:		port_test.erl
 	@erl_name@ -compile port_test
 
-- 
cgit v1.2.3


From eb3e927c110ba8108f2797a23774db14ee70fd76 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 20 Dec 2016 15:46:24 +0100
Subject: erts: Fix some compile warnings for port_SUITE

---
 erts/emulator/test/port_SUITE_data/port_test.c | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/port_SUITE_data/port_test.c b/erts/emulator/test/port_SUITE_data/port_test.c
index cc3ebdf0f8..e199a0fc13 100644
--- a/erts/emulator/test/port_SUITE_data/port_test.c
+++ b/erts/emulator/test/port_SUITE_data/port_test.c
@@ -10,6 +10,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
+#include <ctype.h>
 
 #ifndef __WIN32__
 #include <unistd.h>
@@ -33,7 +34,7 @@
     exit(1); \
 }
 
-#define MAIN(argc, argv) main(argc, argv)
+#define ASSERT(e) ((void) ((e) ? 1 : abort()))
 
 extern int errno;
 
@@ -105,9 +106,7 @@ int err;
 #endif
 
 
-MAIN(argc, argv)
-int argc;
-char *argv[];
+int main(int argc, char *argv[])
 {
   int ret, fd_count;
   if((port_data = (PORT_TEST_DATA *) malloc(sizeof(PORT_TEST_DATA))) == NULL) {
@@ -377,9 +376,11 @@ write_reply(buf, size)
      int size;			/* Size of buffer to send. */
 {
     int n;			/* Temporary to hold size. */
+    int rv;
 
     if (port_data->slow_writes <= 0) {	/* Normal, "fast", write. */
-	write(port_data->fd_to_erl, buf, size);
+	rv = write(port_data->fd_to_erl, buf, size);
+        ASSERT(rv == size);
     } else {
 	/*
 	 * Write chunks with delays in between.
@@ -387,7 +388,8 @@ write_reply(buf, size)
 
 	while (size > 0) {
 	    n = size > port_data->slow_writes ? port_data->slow_writes : size;
-	    write(port_data->fd_to_erl, buf, n);
+	    rv = write(port_data->fd_to_erl, buf, n);
+            ASSERT(rv == n);
 	    size -= n;
 	    buf += n;
 	    if (size)
@@ -558,7 +560,7 @@ char* spec;			/* Specification for reply. */
     buf = (char *) malloc(total_size);
     if (buf == NULL) {
 	fprintf(stderr, "%s: insufficent memory for reply buffer of size %d\n",
-		port_data->progname, total_size);
+		port_data->progname, (int)total_size);
 	exit(1);
     }
 
-- 
cgit v1.2.3


From 3eef3cd38c6beb48bfe4c2e26ede2ab2d107c146 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 28 Dec 2016 18:21:44 +0100
Subject: Fix crash due to GC of node entry on dirty scheduler

The dirty scheduler failed to set a timer. Dispatch this job
to an ordinary scheduler.
---
 erts/emulator/beam/erl_hl_timer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index d29d079fc5..38bebc7576 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -2666,7 +2666,7 @@ erts_start_timer_callback(ErtsMonotonicTime tmo,
 				  tmo);
     twt = tmo < ERTS_TIMER_WHEEL_MSEC;
 
-    if (esdp)
+    if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp))
 	start_callback_timer(esdp,
 			     twt,
 			     timeout_pos,
-- 
cgit v1.2.3


From 32c71f85bd2d17a6c58f6d1fa450dfb8cf3de6cb Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 28 Dec 2016 18:27:54 +0100
Subject: Fix premature removal of process struct

Refc on process struct could reach zero while it was still
referred by dirty scheduler
---
 erts/emulator/beam/erl_process.c | 119 +++++++++++++++++++++++++++------------
 1 file changed, 82 insertions(+), 37 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b345c35a7e..6810210e4d 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2060,6 +2060,7 @@ handle_thr_prgr_later_op(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int wait
 #endif
     for (lops = 0; lops < ERTS_MAX_THR_PRGR_LATER_OPS; lops++) {
 	ErtsThrPrgrLaterOp *lop = awdp->later_op.first;
+
 	if (!erts_thr_progress_has_reached_this(current, lop->later))
 	    return aux_work & ~ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP;
 	awdp->later_op.first = lop->next;
@@ -6234,6 +6235,12 @@ check_dirty_enqueue_in_prio_queue(Process *c_p,
     int queue;
     erts_aint32_t dact, max_qbit;
 
+    /* Do not enqueue free process... */
+    if (actual & ERTS_PSFLG_FREE) {
+	*newp &= ~ERTS_PSFLGS_DIRTY_WORK;
+        return ERTS_ENQUEUE_NOT;
+    }
+
     /* Termination should be done on an ordinary scheduler */
     if ((*newp) & ERTS_PSFLG_EXITING) {
 	*newp &= ~ERTS_PSFLGS_DIRTY_WORK;
@@ -6426,6 +6433,9 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st
 
 /*
  * schedule_out_process() return with c_rq locked.
+ *
+ * Return non-zero value if caller should decrease
+ * reference count on the process when done with it...
  */
 static ERTS_INLINE int
 schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
@@ -6480,12 +6490,18 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
 
 	erts_smp_runq_lock(c_rq);
 
-	return 0;
-
+#if !defined(ERTS_SMP)
+        /* Decrement refc if process struct is free... */
+        return !!(n & ERTS_PSFLG_FREE);
+#else
+        /* Decrement refc if scheduled out from dirty scheduler... */
+	return !is_normal_sched;
+#endif
     }
     else {
 	Process* sched_p;
 
+        ASSERT(!(n & ERTS_PSFLG_FREE));
 	ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & (ERTS_PSFLG_ACTIVE_SYS
 						    | ERTS_PSFLG_DIRTY_ACTIVE_SYS)));
 
@@ -6501,11 +6517,14 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
 
 	erts_smp_runq_lock(runq);
 
+        if (is_normal_sched && sched_p == p && ERTS_RUNQ_IX_IS_DIRTY(runq->ix))
+            erts_proc_inc_refc(p); /* Needs to be done before enqueue_process() */
+
 	/* Enqueue the process */
 	enqueue_process(runq, (int) enq_prio, sched_p);
 
 	if (runq == c_rq)
-	    return 1;
+	    return 0;
 
 	erts_smp_runq_unlock(runq);
 
@@ -6513,9 +6532,15 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
 
 	erts_smp_runq_lock(c_rq);
 
-	return 1;
+        /*
+         * Decrement refc if process is scheduled out by a
+         * dirty scheduler, and we have not just scheduled
+         * the process using the ordinary process struct
+         * on a dirty run-queue again...
+         */
+        return !is_normal_sched && (sched_p != p
+                                    || !ERTS_RUNQ_IX_IS_DIRTY(runq->ix));
     }
-
 }
 
 static ERTS_INLINE void
@@ -6530,8 +6555,17 @@ add2runq(int enqueue, erts_aint32_t prio,
     if (runq) {
 	Process *sched_p;
 
-	if (enqueue > 0)
+	if (enqueue > 0) {
 	    sched_p = proc;
+	    /*
+	     * Refc on process struct (i.e. true struct,
+	     * not proxy-struct) increased while in a
+	     * dirty run-queue or executing on a dirty
+	     * scheduler.
+	     */
+            if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix))
+                erts_proc_inc_refc(proc);
+        }
 	else {
 	    Process *pxy;
 
@@ -9587,40 +9621,45 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	esdp->reductions += reds;
 
-	/* schedule_out_process() returns with rq locked! */
-	schedule_out_process(rq, state, p, proxy_p, is_normal_sched);
-	proxy_p = NULL;
+        {
+            int dec_refc;
 
-	ERTS_PROC_REDUCTIONS_EXECUTED(esdp, rq,
-				      (int) ERTS_PSFLGS_GET_USR_PRIO(state),
-				      reds,
-				      actual_reds);
+            /* schedule_out_process() returns with rq locked! */
+            dec_refc = schedule_out_process(rq, state, p,
+                                            proxy_p, is_normal_sched);
+            proxy_p = NULL;
 
-	esdp->current_process = NULL;
+            ERTS_PROC_REDUCTIONS_EXECUTED(esdp, rq,
+                                          (int) ERTS_PSFLGS_GET_USR_PRIO(state),
+                                          reds,
+                                          actual_reds);
+
+            esdp->current_process = NULL;
 #ifdef ERTS_SMP
-	p->scheduler_data = NULL;
+            p->scheduler_data = NULL;
 #endif
 
-	erts_smp_proc_unlock(p, (ERTS_PROC_LOCK_MAIN
-				 | ERTS_PROC_LOCK_STATUS
-				 | ERTS_PROC_LOCK_TRACE));
+            erts_smp_proc_unlock(p, (ERTS_PROC_LOCK_MAIN
+                                     | ERTS_PROC_LOCK_STATUS
+                                     | ERTS_PROC_LOCK_TRACE));
 
-        ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER);
+            ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER);
 
-	if (state & ERTS_PSFLG_FREE) {
-	    if (!is_normal_sched) {
-		ASSERT(p->flags & F_DELAYED_DEL_PROC);
-		erts_proc_dec_refc(p);
-	    }
-	    else {
 #ifdef ERTS_SMP
-		ASSERT(esdp->free_process == p);
-		esdp->free_process = NULL;
-#else
-		erts_proc_dec_refc(p);
+            if (state & ERTS_PSFLG_FREE) {
+                if (!is_normal_sched) {
+                    ASSERT(p->flags & F_DELAYED_DEL_PROC);
+                }
+                else {
+                    ASSERT(esdp->free_process == p);
+                    esdp->free_process = NULL;
+                }
+            }
 #endif
-	    }
-	}
+
+            if (dec_refc)
+                erts_proc_dec_refc(p);
+        }
 
 #ifdef ERTS_SMP
 	ASSERT(!esdp->free_process);
@@ -9896,8 +9935,12 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    if (!(state & ERTS_PSFLG_PROXY))
 		psflg_band_mask &= ~ERTS_PSFLG_IN_RUNQ;
 	    else {
+                Eterm pid = p->common.id;
 		proxy_p = p;
-		p = erts_proc_lookup_raw(proxy_p->common.id);
+		p = (is_normal_sched
+                     ? erts_proc_lookup_raw(pid)
+                     : erts_pid2proc_opt(NULL, 0, pid, 0,
+                                         ERTS_P2P_FLG_INC_REFC));
 		if (!p) {
 		    free_proxy_proc(proxy_p);
 		    proxy_p = NULL;
@@ -9929,6 +9972,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 					    | ERTS_PSFLG_FREE)))
 #ifdef ERTS_DIRTY_SCHEDULERS
 				| (((state & (ERTS_PSFLG_RUNNING
+
 					      | ERTS_PSFLG_FREE
 					      | ERTS_PSFLG_RUNNING_SYS
 					      | ERTS_PSFLG_EXITING))
@@ -9961,6 +10005,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 			    /* free and not queued by proxy */
 			    erts_proc_dec_refc(p);
 			}
+                        if (!is_normal_sched)
+                            erts_proc_dec_refc(p);
 			goto pick_next_process;
 		    }
 		    state = new;
@@ -13091,15 +13137,14 @@ erts_continue_exit_process(Process *p)
 	}
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-	if (a & (ERTS_PSFLG_DIRTY_RUNNING
-		 | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
+        if (a & (ERTS_PSFLG_DIRTY_RUNNING
+                 | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
 	    p->flags |= F_DELAYED_DEL_PROC;
 	    delay_del_proc = 1;
 	    /*
-	     * The dirty scheduler will also decrease
-	     * refc when done...
+	     * The dirty scheduler decrease refc
+             * when done with the process...
 	     */
-	    erts_proc_inc_refc(p);
 	}
 #endif
 
-- 
cgit v1.2.3


From 88ae8fd6c952cd1f1edba1321de0dc97d8eafaf2 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 28 Dec 2016 18:56:18 +0100
Subject: Leave dirty work in dirty run-queues on multi scheduling block

---
 erts/emulator/beam/erl_process.c | 30 ++++++------------------------
 1 file changed, 6 insertions(+), 24 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 6810210e4d..fba81db1cd 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -7165,16 +7165,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
     ASSERT(sched_type != ERTS_SCHED_NORMAL || no != 1);
 
-    if (sched_type != ERTS_SCHED_NORMAL) {
-	if (erts_smp_mtx_trylock(&schdlr_sspnd.mtx) == EBUSY) {
-	    erts_smp_runq_unlock(esdp->run_queue);
-	    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-	    erts_smp_runq_lock(esdp->run_queue);
-	}
-	if (schdlr_sspnd.msb.ongoing)
-	    evacuate_run_queue(esdp->run_queue, &sbp);
+    if (sched_type != ERTS_SCHED_NORMAL)
 	erts_smp_runq_unlock(esdp->run_queue);
-    }
     else {
 	evacuate_run_queue(esdp->run_queue, &sbp);
 
@@ -7187,8 +7179,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 	sched_wall_time_change(esdp, 0);
 
-	erts_smp_mtx_lock(&schdlr_sspnd.mtx);
     }
+    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 
     flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED);
     if (flgs & ERTS_SSI_FLG_SUSPENDED) {
@@ -7304,24 +7296,14 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 	    while (1) {
 		ErtsMonotonicTime current_time;
-		erts_aint32_t qmask;
 		erts_aint32_t flgs;
 
-		qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue)
-			 & ERTS_RUNQ_FLGS_QMASK);
-
-		if (sched_type != ERTS_SCHED_NORMAL) {
-		    if (qmask) {
-			erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-			erts_smp_runq_lock(esdp->run_queue);
-			if (schdlr_sspnd.msb.ongoing)
-			    evacuate_run_queue(esdp->run_queue, &sbp);
-			erts_smp_runq_unlock(esdp->run_queue);
-			erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-		    }
+		if (sched_type != ERTS_SCHED_NORMAL)
 		    aux_work = 0;
-		}
 		else {
+		    erts_aint32_t qmask;
+		    qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue)
+			     & ERTS_RUNQ_FLGS_QMASK);
 
 		    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 
-- 
cgit v1.2.3


From efc331a04191a124ee7428c25f2ec05ded748faf Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 28 Dec 2016 19:20:29 +0100
Subject: Fix VM global GC info for dirty schedulers

---
 erts/emulator/beam/erl_gc.c         | 39 +++++++++++++++++++++++++++++++++++--
 erts/emulator/beam/erl_lock_check.c |  1 +
 2 files changed, 38 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index af799d09da..cb48b31b7e 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -176,6 +176,13 @@ typedef struct {
     erts_smp_atomic32_t refc;
 } ErtsGCInfoReq;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+static struct {
+    erts_mtx_t mtx;
+    ErtsGCInfo info;
+} dirty_gc;
+#endif
+
 static ERTS_INLINE int
 gc_cost(Uint gc_moved_live_words, Uint resize_moved_words)
 {
@@ -259,6 +266,11 @@ erts_init_gc(void)
       init_gc_info(&esdp->gc_info);
     }
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_smp_mtx_init(&dirty_gc.mtx, "dirty_gc_info");
+    init_gc_info(&dirty_gc.info);
+#endif
+
     init_gcireq_alloc();
 }
 
@@ -735,8 +747,19 @@ do_major_collection:
 	    monitor_large_heap(p);
     }
 
-    esdp->gc_info.garbage_cols++;
-    esdp->gc_info.reclaimed += reclaimed_now;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	erts_mtx_lock(&dirty_gc.mtx);
+	dirty_gc.info.garbage_cols++;
+	dirty_gc.info.reclaimed += reclaimed_now;
+	erts_mtx_unlock(&dirty_gc.mtx);
+    }
+    else
+#endif
+    {
+	esdp->gc_info.garbage_cols++;
+	esdp->gc_info.reclaimed += reclaimed_now;
+    }
     
     FLAGS(p) &= ~F_FORCE_GC;
     p->live_hf_end = ERTS_INVALID_HFRAG_PTR;
@@ -3017,6 +3040,18 @@ reply_gc_info(void *vgcirp)
 
     reclaimed = esdp->gc_info.reclaimed;
     garbage_cols = esdp->gc_info.garbage_cols;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    /*
+     * Add dirty schedulers info on requesting
+     * schedulers info
+     */
+    if (gcirp->req_sched == esdp->no) {
+	erts_mtx_lock(&dirty_gc.mtx);
+	reclaimed += dirty_gc.info.reclaimed;
+	garbage_cols += dirty_gc.info.garbage_cols;
+	erts_mtx_unlock(&dirty_gc.mtx);
+    }
+#endif
 
     sz = 0;
     hpp = NULL;
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 06266363b5..13a4b2cd93 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -129,6 +129,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"run_queue",				"address"		},
 #ifdef ERTS_DIRTY_SCHEDULERS
     {   "dirty_run_queue_sleep_list",		"address"		},
+    {	"dirty_gc_info",			NULL			},
 #endif
     {	"process_table",			NULL			},
     {	"cpu_info",				NULL			},
-- 
cgit v1.2.3


From 765e63dbffc4ec7f8cc3a7a1463151d6d863746f Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 30 Dec 2016 17:15:53 +0100
Subject: Multi scheduling block bug-fixes

---
 erts/emulator/beam/erl_process.c | 80 +++++++++++++++++++++++++---------------
 erts/emulator/beam/erl_process.h |  2 +-
 2 files changed, 52 insertions(+), 30 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b345c35a7e..58097ad42f 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -7055,12 +7055,18 @@ typedef struct {
 } ErtsSchdlrSspndResume;
 
 static void
-schdlr_sspnd_resume_proc(Eterm pid)
+schdlr_sspnd_resume_proc(ErtsSchedType sched_type, Eterm pid)
 {
-    Process *p = erts_pid2proc(NULL, 0, pid, ERTS_PROC_LOCK_STATUS);
+    Process *p;
+    p = erts_pid2proc_opt(NULL, 0, pid, ERTS_PROC_LOCK_STATUS,
+                          (sched_type != ERTS_SCHED_NORMAL
+                           ? ERTS_P2P_FLG_INC_REFC
+                           : 0));
     if (p) {
 	resume_process(p, ERTS_PROC_LOCK_STATUS);
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+        if (sched_type != ERTS_SCHED_NORMAL)
+            erts_proc_dec_refc(p);
     }
 }
 
@@ -7069,17 +7075,20 @@ schdlr_sspnd_resume_procs(ErtsSchedType sched_type,
 			  ErtsSchdlrSspndResume *resume)
 {
     if (is_internal_pid(resume->onln.chngr)) {
-	schdlr_sspnd_resume_proc(resume->onln.chngr);
+	schdlr_sspnd_resume_proc(sched_type,
+                                 resume->onln.chngr);
 	resume->onln.chngr = NIL;
     }
     if (is_internal_pid(resume->onln.nxt)) {
-	schdlr_sspnd_resume_proc(resume->onln.nxt);
+	schdlr_sspnd_resume_proc(sched_type,
+                                 resume->onln.nxt);
 	resume->onln.nxt = NIL;
     }
     while (resume->msb.chngrs) {
 	ErtsProcList *plp = resume->msb.chngrs;
 	resume->msb.chngrs = plp->next;
-	schdlr_sspnd_resume_proc(plp->pid);
+	schdlr_sspnd_resume_proc(sched_type,
+                                 plp->pid);
 	proclist_destroy(plp);
     }
 }
@@ -7198,15 +7207,18 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 			(void) erts_proclist_fetch(&msb[i]->chngq, &end_plp);
 			/* resume processes that initiated the multi scheduling block... */
 			plp = msb[i]->chngq;
-			while (plp) {
-			    erts_proclist_store_last(&msb[i]->blckrs,
-						     proclist_copy(plp));
-			    plp = plp->next;
-			}
-			if (end_plp)
+			if (plp) {
+                            ASSERT(end_plp);
+                            ASSERT(msb[i]->ongoing);
+                            do {
+                                erts_proclist_store_last(&msb[i]->blckrs,
+                                                         proclist_copy(plp));
+                                plp = plp->next;
+                            } while (plp);
 			    end_plp->next = resume.msb.chngrs;
-			resume.msb.chngrs = msb[i]->chngq;
-			msb[i]->chngq = NULL;
+                            resume.msb.chngrs = msb[i]->chngq;
+                            msb[i]->chngq = NULL;
+                        }
 		    }
 		}
 	    }
@@ -7415,12 +7427,23 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 	schdlr_sspnd_inc_nscheds(&schdlr_sspnd.active, sched_type);
 	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
-	if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
-	    && schdlr_sspnd.online == schdlr_sspnd.active) {
-	    erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
-					    ~ERTS_SCHDLR_SSPND_CHNG_MSB);
-	}
-
+        if (changing) {
+            if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
+                && !schdlr_sspnd.msb.ongoing
+                && schdlr_sspnd.online == schdlr_sspnd.active) {
+                erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+                                                ~ERTS_SCHDLR_SSPND_CHNG_MSB);
+            }
+            if ((changing & ERTS_SCHDLR_SSPND_CHNG_NMSB)
+                && !schdlr_sspnd.nmsb.ongoing
+                && (schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+                                             ERTS_SCHED_NORMAL)
+                    == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+                                                ERTS_SCHED_NORMAL))) {
+                erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+                                                ~ERTS_SCHDLR_SSPND_CHNG_NMSB);
+            }
+        }
 	ASSERT(no <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, sched_type));
 	ASSERT((sched_type == ERTS_SCHED_NORMAL
 		? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)
@@ -7553,7 +7576,7 @@ abort_sched_onln_chng_waitq(Process *p)
     erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
 
     if (is_internal_pid(resume))
-	schdlr_sspnd_resume_proc(resume);
+	schdlr_sspnd_resume_proc(ERTS_SCHED_NORMAL, resume);
 }
 
 ErtsSchedSuspendResult
@@ -7951,21 +7974,20 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
     }
     else {  /* ------ UNBLOCK ------ */
 	if (p->flags & have_blckd_flg) {
-	    ErtsProcList *plps[2];
+	    ErtsProcList **plpps[3] = {0};
 	    ErtsProcList *plp;
-	    int limit = 0;
 
-	    plps[limit++] = erts_proclist_peek_first(msbp->blckrs);
-	    if (all)
-		plps[limit++] = erts_proclist_peek_first(msbp->chngq);
+            plpps[0] = &msbp->blckrs;
+            if (all)
+                plpps[1] = &msbp->chngq;
 
-	    for (ix = 0; ix < limit; ix++) {
-		plp = plps[ix];
+	    for (ix = 0; plpps[ix]; ix++) {
+                plp = erts_proclist_peek_first(*plpps[ix]);
 		while (plp) {
 		    ErtsProcList *tmp_plp = plp;
-		    plp = erts_proclist_peek_next(msbp->blckrs, plp);
+		    plp = erts_proclist_peek_next(*plpps[ix], plp);
 		    if (erts_proclist_same(tmp_plp, p)) {
-			erts_proclist_remove(&msbp->blckrs, tmp_plp);
+			erts_proclist_remove(plpps[ix], tmp_plp);
 			proclist_destroy(tmp_plp);
 			if (!all)
 			    break;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 9f7084c127..68fbb10602 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1684,7 +1684,7 @@ ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **list)
 	return NULL;
     else {
 	ErtsProcList *res = *list;
-	if (res == *list)
+	if (res->next == *list)
 	    *list = NULL;
 	else
 	    *list = res->next;
-- 
cgit v1.2.3


From 67355d1f2f4efb61ef9345751c15c0263cdc97a0 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 2 Jan 2017 13:09:47 +0100
Subject: Fix faulty assertion

---
 erts/emulator/beam/erl_process.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b345c35a7e..d15430b902 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -8604,8 +8604,15 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 	     * from being selected for normal execution regardless
 	     * of locks held or not held on it...
 	     */
-	    ASSERT(!((ERTS_PSFLG_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS)
-		     & erts_smp_atomic32_read_nob(&rp->state)));
+#ifdef DEBUG
+	    {
+		erts_aint32_t state;
+		state = erts_smp_atomic32_read_nob(&rp->state);
+		ASSERT((state & ERTS_PSFLG_PENDING_EXIT)
+		       || !(state & (ERTS_PSFLG_RUNNING
+				     | ERTS_PSFLG_DIRTY_RUNNING_SYS)));
+	    }
+#endif
 
 	    if (!suspend)
 		resume_process(rp, pid_locks|ERTS_PROC_LOCK_STATUS);
-- 
cgit v1.2.3


From fb4e0f8726d2e3033164fe6e44c49351c6b641de Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 21 Dec 2016 17:41:43 +0100
Subject: erts: Make erts_dbg_within_proc available

for debug assertions.
---
 erts/emulator/beam/erl_gc.c  | 12 +++---------
 erts/emulator/beam/erl_gc.h  |  7 +++----
 erts/emulator/hipe/hipe_gc.c |  6 +++---
 3 files changed, 9 insertions(+), 16 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 818f89f0e3..d907e7b351 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -3317,8 +3317,8 @@ erts_max_heap_size(Eterm arg, Uint *max_heap_size, Uint *max_heap_flags)
 
 #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG)
 
-static int
-within2(Eterm *ptr, Process *p, Eterm *real_htop)
+int
+erts_dbg_within_proc(Eterm *ptr, Process *p, Eterm *real_htop)
 {
     ErlHeapFragment* bp;
     ErtsMessage* mp;
@@ -3362,12 +3362,6 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop)
     return 0;
 }
 
-int
-within(Eterm *ptr, Process *p)
-{
-    return within2(ptr, p, NULL);
-}
-
 #endif
 
 #ifdef ERTS_OFFHEAP_DEBUG
@@ -3422,7 +3416,7 @@ erts_check_off_heap2(Process *p, Eterm *htop)
 	else if (oheap <= u.ep && u.ep < ohtop)
 	    old = 1;
 	else {
-	    ERTS_CHK_OFFHEAP_ASSERT(within2(u.ep, p, htop));
+	    ERTS_CHK_OFFHEAP_ASSERT(erts_dbg_within_proc(u.ep, p, htop));
 	}
     }
 
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 54ea9ca3c0..9684478c1b 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -69,10 +69,6 @@ do {                                                                    \
     while (nelts--) *HTOP++ = *PTR++;                                   \
 } while(0)
 
-#if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG)
-int within(Eterm *ptr, Process *p);
-#endif
-
 #define ErtsInYoungGen(TPtr, Ptr, OldHeap, OldHeapSz)			\
     (!erts_is_literal((TPtr), (Ptr))					\
      & !ErtsInArea((Ptr), (OldHeap), (OldHeapSz)))
@@ -157,5 +153,8 @@ void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*);
 void erts_free_heap_frags(struct process* p);
 Eterm erts_max_heap_size_map(Sint, Uint, Eterm **, Uint *);
 int erts_max_heap_size(Eterm, Uint *, Uint *);
+#if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG)
+int erts_dbg_within_proc(Eterm *ptr, Process *p, Eterm* real_htop);
+#endif
 
 #endif /* __ERL_GC_H__ */
diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c
index e6ce7ce628..cf0435adc9 100644
--- a/erts/emulator/hipe/hipe_gc.c
+++ b/erts/emulator/hipe/hipe_gc.c
@@ -99,7 +99,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
 		    if (IS_MOVED_CONS(val)) {
 			*nsp_i = ptr[1];
 		    } else if (!erts_is_literal(gval, ptr)) {
-			ASSERT(within(ptr, p));
+			ASSERT(erts_dbg_within_proc(ptr, p, NULL));
 			MOVE_CONS(ptr, val, n_htop, nsp_i);
 		    }
 		}
@@ -208,7 +208,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
 		    } else if (ErtsInArea(ptr, mature, mature_size)) {
 			MOVE_BOXED(ptr, val, old_htop, nsp_i);
 		    } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
-			ASSERT(within(ptr, p));
+			ASSERT(erts_dbg_within_proc(ptr, p, NULL));
 			MOVE_BOXED(ptr, val, n_htop, nsp_i);
 		    }
 		} else if (is_list(gval)) {
@@ -219,7 +219,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
 		    } else if (ErtsInArea(ptr, mature, mature_size)) {
 			MOVE_CONS(ptr, val, old_htop, nsp_i);
 		    } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
-			ASSERT(within(ptr, p));
+			ASSERT(erts_dbg_within_proc(ptr, p, NULL));
 			MOVE_CONS(ptr, val, n_htop, nsp_i);
 		    }
 		}
-- 
cgit v1.2.3


From dc72c879b68fdf96edfec360e8c311ff7389ecc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 16 Dec 2016 10:08:16 +0100
Subject: erts: Remove whitespace errors

* and some minor refactoring
---
 erts/emulator/sys/unix/sys.c | 78 ++++++++++++--------------------------------
 1 file changed, 20 insertions(+), 58 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 2fc802a2c6..2ce0d56dde 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -153,7 +153,7 @@ volatile int erts_break_requested = 0;
 static struct termios initial_tty_mode;
 static int replace_intr = 0;
 /* assume yes initially, ttsl_init will clear it */
-int using_oldshell = 1; 
+int using_oldshell = 1;
 
 #ifdef ERTS_ENABLE_KERNEL_POLL
 
@@ -798,11 +798,11 @@ void erts_replace_intr(void) {
 
   if (isatty(0)) {
     tcgetattr(0, &mode);
-    
+
     /* here's an example of how to replace ctrl-c with ctrl-u */
     /* mode.c_cc[VKILL] = 0;
        mode.c_cc[VINTR] = CKILL; */
-    
+
     mode.c_cc[VINTR] = 0;	/* disable ctrl-c */
     tcsetattr(0, TCSANOW, &mode);
     replace_intr = 1;
@@ -856,10 +856,7 @@ get_number(char **str_ptr)
     }
 }
 
-void
-os_flavor(char* namebuf, 	/* Where to return the name. */
-	  unsigned size) 	/* Size of name buffer. */
-{
+void os_flavor(char* namebuf, unsigned size) {
     struct utsname uts;		/* Information about the system. */
     char* s;
 
@@ -872,22 +869,16 @@ os_flavor(char* namebuf, 	/* Where to return the name. */
     strcpy(namebuf, uts.sysname);
 }
 
-void
-os_version(pMajor, pMinor, pBuild)
-int* pMajor;			/* Pointer to major version. */
-int* pMinor;			/* Pointer to minor version. */
-int* pBuild;			/* Pointer to build number. */
-{
+void os_version(int *pMajor, int *pMinor, int *pBuild) {
     struct utsname uts;		/* Information about the system. */
     char* release;		/* Pointer to the release string:
-				 * X.Y or X.Y.Z.
-				 */
+				 * X.Y or X.Y.Z.  */
 
     (void) uname(&uts);
     release = uts.release;
-    *pMajor = get_number(&release);
-    *pMinor = get_number(&release);
-    *pBuild = get_number(&release);
+    *pMajor = get_number(&release); /* Pointer to major version. */
+    *pMinor = get_number(&release); /* Pointer to minor version. */
+    *pBuild = get_number(&release); /* Pointer to build number. */
 }
 
 void init_getenv_state(GETENV_STATE *state)
@@ -922,7 +913,7 @@ void erts_do_break_handling(void)
 {
     struct termios temp_mode;
     int saved = 0;
-    
+
     /*
      * Most functions that do_break() calls are intentionally not thread safe;
      * therefore, make sure that all threads but this one are blocked before
@@ -940,14 +931,14 @@ void erts_do_break_handling(void)
       tcsetattr(0,TCSANOW,&initial_tty_mode);
       saved = 1;
     }
-    
+
     /* call the break handling function, reset the flag */
     do_break();
 
     ERTS_UNSET_BREAK_REQUESTED;
 
     fflush(stdout);
-    
+
     /* after break we go back to saved settings */
     if (using_oldshell && !replace_intr) {
       SET_NONBLOCKING(1);
@@ -1055,37 +1046,12 @@ erts_sys_unsetenv(char *key)
     return res;
 }
 
-void
-sys_init_io(void)
-{
-}
-
-#if (0) /* unused? */
-static int write_fill(fd, buf, len)
-int fd, len;
-char *buf;
-{
-    int i, done = 0;
-    
-    do {
-	if ((i = write(fd, buf+done, len-done)) < 0) {
-	    if (errno != EINTR)
-		return (i);
-	    i = 0;
-	}
-	done += i;
-    } while (done < len);
-    return (len);
-}
-#endif
+void sys_init_io(void) { }
+void erts_sys_alloc_init(void) { }
 
 extern const char pre_loaded_code[];
 extern Preload pre_loaded[];
 
-void erts_sys_alloc_init(void)
-{
-}
-
 #if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
 void *erts_sys_aligned_alloc(UWord alignment, UWord size)
 {
@@ -1186,9 +1152,7 @@ void sys_preload_end(Preload* p)
    Here we assume that all schedulers are stopped so that erl_poll
    does not interfere with the select below.
 */
-int sys_get_key(fd)
-int fd;
-{
+int sys_get_key(int fd) {
     int c, ret;
     unsigned char rbuf[64];
     fd_set fds;
@@ -1207,15 +1171,14 @@ int fd;
         if (c <= 0)
             return c;
     }
-
-    return rbuf[0]; 
+    return rbuf[0];
 }
 
 
 extern int erts_initialized;
 void
 erl_assert_error(const char* expr, const char* func, const char* file, int line)
-{   
+{
     fflush(stdout);
     fprintf(stderr, "%s:%d:%s() Assertion failed: %s\n",
             file, line, func, expr);
@@ -1240,7 +1203,7 @@ erl_debug(char* fmt, ...)
 {
     char sbuf[1024];		/* Temporary buffer. */
     va_list va;
-    
+
     if (debug_log) {
 	va_start(va, fmt);
 	vsprintf(sbuf, fmt, va);
@@ -1388,9 +1351,9 @@ init_smp_sig_suspend(void) {
 int erts_darwin_main_thread_pipe[2];
 int erts_darwin_main_thread_result_pipe[2];
 
-static void initialize_darwin_main_thread_pipes(void) 
+static void initialize_darwin_main_thread_pipes(void)
 {
-    if (pipe(erts_darwin_main_thread_pipe) < 0 || 
+    if (pipe(erts_darwin_main_thread_pipe) < 0 ||
 	pipe(erts_darwin_main_thread_result_pipe) < 0) {
 	erts_exit(ERTS_ERROR_EXIT,"Fatal error initializing Darwin main thread stealing");
     }
@@ -1556,5 +1519,4 @@ erl_sys_args(int* argc, char** argv)
 	    argv[j++] = argv[i];
     }
     *argc = j;
-
 }
-- 
cgit v1.2.3


From c9060f9b4289bf9a669ac651a1a7aeb97a28652c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 13 Apr 2015 18:03:33 +0200
Subject: erts: Add SIGHUP signal handler

A received SIGHUP signal to beam will generate a '{notify, sighup}' message
to the registered process 'erl_signal_server'. 'erl_signal_server' is a
gen_event process.
---
 erts/emulator/beam/atom.names |  2 ++
 erts/emulator/beam/register.c | 12 ++++++------
 erts/emulator/sys/unix/sys.c  | 41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 49 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index b1aeed7889..363b17e27c 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -239,6 +239,7 @@ atom Eq='=:='
 atom Eqeq='=='
 atom erl_tracer
 atom erlang
+atom erl_signal_server
 atom ERROR='ERROR'
 atom error_handler
 atom error_logger
@@ -590,6 +591,7 @@ atom set_tcw
 atom set_tcw_fake
 atom separate
 atom shared
+atom sighup
 atom silent
 atom size
 atom sl_alloc
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index acda51c9fc..7f60710124 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -272,13 +272,13 @@ erts_whereis_name_to_id(Process *c_p, Eterm name)
     int ix;
     HashBucket* b;
 #ifdef ERTS_SMP
-    ErtsProcLocks c_p_locks = c_p ? ERTS_PROC_LOCK_MAIN : 0;
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    if (c_p) ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p);
-#endif
-
+    ErtsProcLocks c_p_locks = 0;
+    if (c_p) {
+        c_p_locks = ERTS_PROC_LOCK_MAIN;
+        ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p);
+    }
     reg_safe_read_lock(c_p, &c_p_locks);
+
     if (c_p && !c_p_locks)
         erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 #endif
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 2ce0d56dde..a099662b16 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -408,6 +408,7 @@ erts_sys_pre_init(void)
     sigemptyset(&thr_create_sigmask);
     sigaddset(&thr_create_sigmask, SIGINT);   /* block interrupt */
     sigaddset(&thr_create_sigmask, SIGTERM);  /* block terminate signal */
+    sigaddset(&thr_create_sigmask, SIGHUP);   /* block sighups */
     sigaddset(&thr_create_sigmask, SIGUSR1);  /* block user defined signal */
 #endif
 
@@ -625,6 +626,28 @@ int erts_sys_prepare_crash_dump(int secs)
     return prepare_crash_dump(secs);
 }
 
+static void signal_notify_requested(Eterm type) {
+    Process* p = NULL;
+    Eterm msg, *hp;
+    ErtsProcLocks locks = 0;
+    ErlOffHeap *ohp;
+
+    Eterm id = erts_whereis_name_to_id(NULL, am_erl_signal_server);
+
+    if ((p = (erts_pid2proc_opt(NULL, 0, id, 0, ERTS_P2P_FLG_INC_REFC))) != NULL) {
+        ErtsMessage *msgp = erts_alloc_message_heap(p, &locks, 3, &hp, &ohp);
+
+        /* erl_signal_server ! {notify, sighup} */
+        msg = TUPLE2(hp, am_notify, type);
+        erts_queue_message(p, locks, msgp, msg, am_system);
+
+        if (locks)
+            erts_smp_proc_unlock(p, locks);
+        erts_proc_dec_refc(p);
+    }
+}
+
+
 static ERTS_INLINE void
 break_requested(void)
 {
@@ -783,10 +806,24 @@ static RETSIGTYPE do_quit(int signum)
 #endif
 }
 
+#if (defined(SIG_SIGSET) || defined(SIG_SIGNAL))
+static RETSIGTYPE request_sighup(void)
+#else
+static RETSIGTYPE request_sighup(int signum)
+#endif
+{
+#ifdef ERTS_SMP
+    smp_sig_notify('H');
+#else
+    signal_notify_requested(am_sighup);
+#endif
+}
+
 /* Disable break */
 void erts_set_ignore_break(void) {
     sys_signal(SIGINT,  SIG_IGN);
     sys_signal(SIGTERM, SIG_IGN);
+    sys_signal(SIGHUP,  SIG_IGN);
     sys_signal(SIGQUIT, SIG_IGN);
     sys_signal(SIGTSTP, SIG_IGN);
 }
@@ -813,6 +850,7 @@ void init_break_handler(void)
 {
    sys_signal(SIGINT, request_break);
    sys_signal(SIGTERM, request_stop);
+   sys_signal(SIGHUP, request_sighup);
 #ifndef ETHR_UNUSABLE_SIGUSRX
    sys_signal(SIGUSR1, user_signal1);
 #endif /* #ifndef ETHR_UNUSABLE_SIGUSRX */
@@ -1289,6 +1327,9 @@ signal_dispatcher_thread_func(void *unused)
 	    switch (buf[i]) {
 	    case 0: /* Emulator initialized */
                 break;
+            case 'H': /* SIGHUP */
+                signal_notify_requested(am_sighup);
+                break;
 	    case 'S': /* SIGTERM */
 		stop_requested();
 		break;
-- 
cgit v1.2.3


From 5f4d538b2abfd1ab3f495879996edaa004309623 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 5 Jan 2017 20:06:04 +0100
Subject: erts: Assert sufficient space need after GC

---
 erts/emulator/beam/erl_gc.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index af799d09da..50f09277fa 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -765,6 +765,7 @@ do_major_collection:
 
     ASSERT(!p->mbuf);
     ASSERT(!ERTS_IS_GC_DESIRED(p));
+    ASSERT(need <= HEAP_LIMIT(p) - HEAP_TOP(p));
 
     return reds;
 }
-- 
cgit v1.2.3


From 287edce0090a14fdd88a9e10d13327e74e52831a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 5 Jan 2017 20:43:49 +0100
Subject: erts: Fix GC tracing to use temp heap

Can't use HAlloc as it might consume part of callers 'need'.
---
 erts/emulator/beam/erl_gc.c    | 3 ++-
 erts/emulator/beam/erl_nif.c   | 4 ++--
 erts/emulator/beam/erl_trace.c | 5 ++++-
 3 files changed, 8 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 50f09277fa..b05209402d 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -602,7 +602,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     Uint reclaimed_now = 0;
     Eterm gc_trace_end_tag;
     int reds;
-    ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */
+    ErtsMonotonicTime start_time;
     ErtsSchedulerData *esdp;
     erts_aint32_t state;
     ERTS_MSACC_PUSH_STATE_M();
@@ -610,6 +610,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
 #endif
 
+    ERTS_UNDEF(start_time, 0);
     ERTS_CHK_MBUF_SZ(p);
 
     ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls)
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 6b265a8b80..19ce0f6965 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3561,8 +3561,8 @@ Eterm erts_nif_call_function(Process *p, Process *tracee,
 #endif
     if (p) {
         /* This is almost a normal nif call like in beam_emu,
-           except that any heap fragment created in the nif will be
-           discarded without checking if anything in it is live.
+           except that any heap consumed by the nif will be
+           released without checking if anything in it is live.
            This is because we cannot do a GC here as we don't know
            the number of live registers that have to be preserved.
            This means that any heap part of the returned term may
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 8c84303997..ac9e91e31f 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1436,6 +1436,7 @@ void
 trace_gc(Process *p, Eterm what, Uint size, Eterm msg)
 {
     ErtsTracerNif *tnif = NULL;
+    Eterm* o_hp = NULL;
     Eterm* hp;
     Uint sz = 0;
     Eterm tup;
@@ -1446,7 +1447,7 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg)
         if (is_non_value(msg)) {
 
             (void) erts_process_gc_info(p, &sz, NULL, 0, 0);
-            hp = HAlloc(p, sz + 3 + 2);
+            o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, (sz + 3 + 2) * sizeof(Eterm));
 
             msg = erts_process_gc_info(p, NULL, &hp, 0, 0);
             tup = TUPLE2(hp, am_wordsize, make_small(size)); hp += 3;
@@ -1455,6 +1456,8 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg)
 
         send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_GC,
                            what, msg, THE_NON_VALUE, am_true);
+        if (o_hp)
+            erts_free(ERTS_ALC_T_TMP, o_hp);
     }
 }
 
-- 
cgit v1.2.3


From 5bf38ca930b75189cc83ed01e4a34eeb6a0892d2 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 9 Jan 2017 13:57:34 +0100
Subject: erts: Fix error report message for erlang:delete/1

---
 erts/emulator/beam/beam_bif_load.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index ea1323d651..644fd9fbf9 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -658,7 +658,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
 	}
 	else if (modp->old.code_hdr) {
 	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	    erts_dsprintf(dsbufp, "Module %T must be purged before loading\n",
+	    erts_dsprintf(dsbufp, "Module %T must be purged before deleting\n",
 			  BIF_ARG_1);
 	    erts_send_error_to_logger(BIF_P->group_leader, dsbufp);
 	    ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
-- 
cgit v1.2.3


From 282b0f62f39168c63279e1d4875c3bda43a0366c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 21 Dec 2016 18:12:14 +0100
Subject: erts: Cleanup and extra assertions in nif_SUITE.c

---
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index d2af081a22..d30876433c 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -133,7 +133,6 @@ static ERL_NIF_TERM make_pointer(ErlNifEnv* env, void* p)
 {
     void** bin_data;
     ERL_NIF_TERM res;
-    ADD_CALL("get_priv_data_ptr");
     bin_data = (void**)enif_make_new_binary(env, sizeof(void*), &res);
     *bin_data = p;
     return res;
@@ -412,8 +411,7 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
     ErlNifSInt64 sint64;
     ErlNifUInt64 uint64;
     double d;
-    ERL_NIF_TERM atom, ref1, ref2, term;
-    size_t len;
+    ERL_NIF_TERM atom, ref1, ref2;
 
     sint = INT_MIN;
     do {
@@ -1047,6 +1045,7 @@ struct make_term_info
 {
     ErlNifEnv* caller_env;
     ErlNifEnv* dst_env;
+    int dst_env_valid;
     ERL_NIF_TERM reuse[MAKE_TERM_REUSE_LEN];
     unsigned reuse_push;
     unsigned reuse_pull;
@@ -1076,6 +1075,7 @@ static ERL_NIF_TERM pull_term(struct make_term_info* mti)
 	mti->reuse_push < MAKE_TERM_REUSE_LEN) {
 	mti->reuse_pull = 0;
 	if (mti->reuse_push == 0) {
+            assert(mti->dst_env_valid);
 	    mti->reuse[0] = enif_make_list(mti->dst_env, 0);
 	}
     }
@@ -1259,6 +1259,7 @@ static unsigned num_of_make_funcs()
 static int make_term_n(struct make_term_info* mti, int n, ERL_NIF_TERM* res)
 {
     if (n < num_of_make_funcs()) {
+        assert(mti->dst_env_valid);
 	*res = make_funcs[n](mti, n);
 	push_term(mti, *res);
 	return 1;
@@ -1275,6 +1276,7 @@ static ERL_NIF_TERM make_blob(ErlNifEnv* caller_env, ErlNifEnv* dst_env,
     struct make_term_info mti;
     mti.caller_env = caller_env;
     mti.dst_env = dst_env;
+    mti.dst_env_valid = 1;
     mti.reuse_push = 0;
     mti.reuse_pull = 0;
     mti.resource_type = priv->rt_arr[0].t;
@@ -1315,6 +1317,7 @@ static ERL_NIF_TERM alloc_msgenv(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
 						       sizeof(*mti));
     mti->caller_env = NULL;
     mti->dst_env = enif_alloc_env();
+    mti->dst_env_valid = 1;
     mti->reuse_push = 0;
     mti->reuse_pull = 0;
     mti->resource_type = priv->rt_arr[0].t;
@@ -1346,6 +1349,7 @@ static ERL_NIF_TERM clear_msgenv(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
 	return enif_make_badarg(env);
     }
     enif_clear_env(mti.p->dst_env);
+    mti.p->dst_env_valid = 1;
     mti.p->reuse_pull = 0;
     mti.p->reuse_push = 0;
     mti.p->blob = enif_make_list(mti.p->dst_env, 0);
@@ -1380,6 +1384,8 @@ static ERL_NIF_TERM send_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
     }
     copy = enif_make_copy(env, mti.p->blob);
     res = enif_send(env, &to, mti.p->dst_env, mti.p->blob);
+    if (res)
+        mti.p->dst_env_valid = 0;
     return enif_make_tuple3(env, atom_ok, enif_make_int(env,res), copy);
 }
 
@@ -1387,7 +1393,6 @@ static ERL_NIF_TERM send3_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
 {
     mti_t mti;
     ErlNifPid to;
-    ERL_NIF_TERM copy;
     int res;
     if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)
 	|| !enif_get_local_pid(env, argv[1], &to)) {
@@ -1397,6 +1402,8 @@ static ERL_NIF_TERM send3_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
 				   enif_make_copy(mti.p->dst_env, argv[2]),
 				   mti.p->blob);
     res = enif_send(env, &to, mti.p->dst_env, mti.p->blob);
+    if (res)
+        mti.p->dst_env_valid = 0;
     return enif_make_int(env,res);
 }
 
@@ -1413,6 +1420,8 @@ void* threaded_sender(void *arg)
     mti.p->send_it = 0;
     enif_mutex_unlock(mti.p->mtx);
     mti.p->send_res = enif_send(NULL, &mti.p->to_pid, mti.p->dst_env, mti.p->blob);
+    if (mti.p->send_res)
+        mti.p->dst_env_valid = 0;
     return NULL;
 }
 
-- 
cgit v1.2.3


From 4b97d656a9e1774fd4ff984014f085ad661a4fdb Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 21 Dec 2016 18:13:02 +0100
Subject: nif_SUITE: Send message from stop callback

---
 erts/emulator/test/nif_SUITE.erl              |  9 +++++----
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 16 ++++++++++++++++
 2 files changed, 21 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 8795a3b24a..7739a0bc22 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -478,7 +478,7 @@ select(Config) when is_list(Config) ->
     %% Close write and wait for EOF
     eagain = read_nif(R, 1),
     check_stop_ret(select_nif(W,?ERL_NIF_SELECT_STOP,W,Ref)),
-    timer:sleep(10),
+    [{fd_resource_stop, W_ptr, _}] = flush(),
     {1, {W_ptr,_}} = last_fd_stop_call(),
     true = is_closed_nif(W),
     [] = flush(),
@@ -487,7 +487,7 @@ select(Config) when is_list(Config) ->
     eof = read_nif(R,1),
 
     check_stop_ret(select_nif(R,?ERL_NIF_SELECT_STOP,R,Ref)),
-    timer:sleep(10),
+    [{fd_resource_stop, R_ptr, _}] = flush(),
     {1, {R_ptr,_}} = last_fd_stop_call(),
     true = is_closed_nif(R),
 
@@ -529,12 +529,13 @@ select_2(Config) ->
     [] = flush(),
 
     check_stop_ret(select_nif(R,?ERL_NIF_SELECT_STOP,R,Ref1)),
-    timer:sleep(10),
+    [{fd_resource_stop, R_ptr, _}] = flush(),
     {1, {R_ptr,_}} = last_fd_stop_call(),
     true = is_closed_nif(R),
 
+    %% Stop without previous read/write select
     ?ERL_NIF_SELECT_STOP_CALLED = select_nif(W,?ERL_NIF_SELECT_STOP,W,Ref1),
-    timer:sleep(10),
+    [{fd_resource_stop, W_ptr, 1}] = flush(),
     {1, {W_ptr,1}} = last_fd_stop_call(),
     true = is_closed_nif(W),
 
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index d30876433c..c4f9611ec8 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -47,6 +47,7 @@ static ERL_NIF_TERM atom_nanosecond;
 static ERL_NIF_TERM atom_eagain;
 static ERL_NIF_TERM atom_eof;
 static ERL_NIF_TERM atom_error;
+static ERL_NIF_TERM atom_fd_resource_stop;
 
 typedef struct
 {
@@ -116,6 +117,7 @@ static ErlNifResourceTypeInit fd_rt_init = {
 struct fd_resource {
     int fd;
     int was_selected;
+    ErlNifPid pid;
 };
 
 
@@ -176,6 +178,7 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
     atom_eagain = enif_make_atom(env, "eagain");
     atom_eof = enif_make_atom(env, "eof");
     atom_error = enif_make_atom(env, "error");
+    atom_fd_resource_stop = enif_make_atom(env, "fd_resource_stop");
 
     *priv_data = data;
     return 0;
@@ -2063,6 +2066,7 @@ static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
     ref = argv[3];
 
     fdr->was_selected = 1;
+    enif_self(env, &fdr->pid);
     retval = enif_select(env, fdr->fd, mode, obj, ref);
 
     return enif_make_int(env, (int)retval);
@@ -2207,6 +2211,18 @@ static void fd_resource_stop(ErlNifEnv* env, void* obj, ErlNifEvent fd,
     close(fd);
     fdr->fd = -1;   /* thread safety ? */
     fdr->was_selected = 0;
+
+    {
+        ErlNifEnv* msg_env = enif_alloc_env();
+        ERL_NIF_TERM msg;
+        msg = enif_make_tuple3(msg_env,
+                               atom_fd_resource_stop,
+                               make_pointer(msg_env, obj),
+                               enif_make_int(msg_env, is_direct_call));
+
+        enif_send(env, &fdr->pid, msg_env, msg);
+        enif_free_env(msg_env);
+    }
 }
 
 static ERL_NIF_TERM last_fd_stop_call(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
-- 
cgit v1.2.3


From a67afb4e2e1e0dda8a8f818c844a2036e703c6f6 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 10 Jan 2017 16:13:07 +0100
Subject: Fix processes() BIF test cases

---
 erts/emulator/test/process_SUITE.erl | 10 ++++++++++
 1 file changed, 10 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index 0f999e0efe..0f492eb982 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -1611,6 +1611,7 @@ spawn_initial_hangarounds(_Cleaner, NP, Max, Len, HAs) when NP > Max ->
     {Len, HAs};
 spawn_initial_hangarounds(Cleaner, NP, Max, Len, HAs) ->
     Skip = 30,
+    wait_for_proc_slots(Skip+3),
     HA1 = spawn_opt(?MODULE, hangaround, [Cleaner, initial_hangaround],
 		    [{priority, low}]),
     HA2 = spawn_opt(?MODULE, hangaround, [Cleaner, initial_hangaround],
@@ -1620,6 +1621,15 @@ spawn_initial_hangarounds(Cleaner, NP, Max, Len, HAs) ->
     spawn_drop(Skip),
     spawn_initial_hangarounds(Cleaner, NP+Skip, Max, Len+3, [HA1,HA2,HA3|HAs]).
 
+wait_for_proc_slots(MinFreeSlots) ->
+    case erlang:system_info(process_limit) - erlang:system_info(process_count) of
+        FreeSlots when FreeSlots < MinFreeSlots ->
+            receive after 10 -> ok end,
+            wait_for_proc_slots(MinFreeSlots);
+        _FreeSlots ->
+            ok
+    end.
+
 spawn_drop(N) when N =< 0 ->
     ok;
 spawn_drop(N) ->
-- 
cgit v1.2.3


From 8df00f51a4a2c0c6788b3dcf3c6be56c50e44461 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 4 Jan 2017 17:51:54 +0100
Subject: Fix issues with abandoned heap

---
 erts/emulator/beam/beam_bif_load.c    |  11 +++
 erts/emulator/beam/erl_gc.c           | 158 +++++++++++++++++++---------------
 erts/emulator/beam/erl_message.c      |   2 +-
 erts/emulator/beam/erl_process_dump.c |   3 +
 4 files changed, 103 insertions(+), 71 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index ea1323d651..1307fe22b5 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -995,6 +995,12 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed
     if (any_heap_refs(c_p->heap, c_p->htop, literals, lit_bsize))
 	goto literal_gc;
     *redsp += 1;
+    if (c_p->abandoned_heap) {
+	if (any_heap_refs(c_p->abandoned_heap, c_p->abandoned_heap + c_p->heap_sz,
+			  literals, lit_bsize))
+	    goto literal_gc;
+	*redsp += 1;
+    }
     if (any_heap_refs(c_p->old_heap, c_p->old_htop, literals, lit_bsize))
 	goto literal_gc;
 
@@ -1295,6 +1301,11 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls
 #endif
 	if (any_heap_refs(rp->heap, rp->htop, literals, lit_bsize))
 	    goto try_literal_gc;
+	if (rp->abandoned_heap) {
+	    if (any_heap_refs(rp->abandoned_heap, rp->abandoned_heap + rp->heap_sz,
+			      literals, lit_bsize))
+		goto try_literal_gc;
+	}
 	if (any_heap_refs(rp->old_heap, rp->old_htop, literals, lit_bsize))
 	    goto try_literal_gc;
 
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index af799d09da..561e7fbdee 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -110,7 +110,7 @@ typedef struct {
 
 static Uint setup_rootset(Process*, Eterm*, int, Rootset*);
 static void cleanup_rootset(Rootset *rootset);
-static void remove_message_buffers(Process* p);
+static void deallocate_previous_young_generation(Process *c_p);
 static Eterm *full_sweep_heaps(Process *p,
 			       int hibernate,
 			       Eterm *n_heap, Eterm* n_htop,
@@ -153,7 +153,7 @@ static void init_gc_info(ErtsGCInfo *gcip);
 static Uint64 next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz);
 
 #ifdef HARDDEBUG
-static void disallow_heap_frag_ref_in_heap(Process* p);
+static void disallow_heap_frag_ref_in_heap(Process *p, Eterm *heap, Eterm *htop);
 static void disallow_heap_frag_ref_in_old_heap(Process* p);
 #endif
 
@@ -477,24 +477,26 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int
     hsz = ssz + need + ERTS_DELAY_GC_EXTRA_FREE;
 
     hfrag = new_message_buffer(hsz);
-    hfrag->next = p->mbuf;
-    p->mbuf = hfrag;
-    p->mbuf_sz += hsz;
     p->heap = p->htop = &hfrag->mem[0];
     p->hend = hend = &hfrag->mem[hsz];
     p->stop = stop = hend - ssz;
     sys_memcpy((void *) stop, (void *) orig_stop, ssz * sizeof(Eterm));
 
     if (p->abandoned_heap) {
-	/* Active heap already in a fragment; adjust it... */
-	ErlHeapFragment *hfrag = ((ErlHeapFragment *)
-				  (((char *) orig_heap)
-				   - offsetof(ErlHeapFragment, mem)));
-	Uint unused = orig_hend - orig_htop;
-	ASSERT(hfrag->used_size == hfrag->alloc_size);
-	ASSERT(hfrag->used_size >= unused);
-	hfrag->used_size -= unused;
-	p->mbuf_sz -= unused;
+	/*
+         * Active heap already in a fragment; adjust it and
+         * save it into mbuf list...
+         */
+        ErlHeapFragment *hfrag = ((ErlHeapFragment *)
+                                  (((char *) orig_heap)
+                                   - offsetof(ErlHeapFragment, mem)));
+        Uint used = orig_htop - orig_heap;
+        hfrag->used_size = used;
+        p->mbuf_sz += used;
+        ASSERT(hfrag->used_size <= hfrag->alloc_size);
+        ASSERT(!hfrag->off_heap.first && !hfrag->off_heap.overhead);
+        hfrag->next = p->mbuf;
+        p->mbuf = hfrag;
     }
     else {
 	/* Do not leave a hole in the abandoned heap... */
@@ -556,17 +558,14 @@ young_gen_usage(Process *p)
 	}
     }
 
+    hsz += p->htop - p->heap;
     aheap = p->abandoned_heap;
-    if (!aheap)
-	hsz += p->htop - p->heap;
-    else {
+    if (aheap) {
 	/* used in orig heap */
 	if (p->flags & F_ABANDONED_HEAP_USE)
 	    hsz += aheap[p->heap_sz-1];
 	else
 	    hsz += p->heap_sz;
-	/* Remove unused part in latest fragment */
-	hsz -= p->hend - p->htop;
     }
     return hsz;
 }
@@ -789,6 +788,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
 	   >= erts_proc_sched_data(p)->virtual_reds);
 }
 
+
 /*
  * Place all living data on a the new heap; deallocate any old heap.
  * Meant to be used by hibernate/3.
@@ -835,11 +835,11 @@ erts_garbage_collect_hibernate(Process* p)
 			    p->arg_reg,
 			    p->arity);
 
-    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
-		   (p->abandoned_heap
-		    ? p->abandoned_heap
-		    : p->heap),
-		   p->heap_sz * sizeof(Eterm));
+#ifdef HARDDEBUG
+    disallow_heap_frag_ref_in_heap(p, heap, htop);
+#endif
+
+    deallocate_previous_young_generation(p);
 
     p->heap = heap;
     p->high_water = htop;
@@ -874,8 +874,6 @@ erts_garbage_collect_hibernate(Process* p)
     sys_memcpy((void *) heap, (void *) p->heap, actual_size*sizeof(Eterm));
     ERTS_HEAP_FREE(ERTS_ALC_T_TMP_HEAP, p->heap, p->heap_sz*sizeof(Eterm));
 
-    remove_message_buffers(p);
-
     p->stop = p->hend = heap + heap_size;
 
     offs = heap - p->heap;
@@ -1494,22 +1492,16 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
     }
 #endif
 
-    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
-		   (p->abandoned_heap
-		    ? p->abandoned_heap
-		    : HEAP_START(p)),
-		   HEAP_SIZE(p) * sizeof(Eterm));
-    p->abandoned_heap = NULL;
-    p->flags &= ~F_ABANDONED_HEAP_USE;
+#ifdef HARDDEBUG
+    disallow_heap_frag_ref_in_heap(p, n_heap, n_htop);
+#endif
+
+    deallocate_previous_young_generation(p);
+
     HEAP_START(p) = n_heap;
     HEAP_TOP(p) = n_htop;
     HEAP_SIZE(p) = new_sz;
     HEAP_END(p) = n_heap + new_sz;
-
-#ifdef HARDDEBUG
-    disallow_heap_frag_ref_in_heap(p);
-#endif
-    remove_message_buffers(p);
 }
 
 /*
@@ -1598,13 +1590,12 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
     }
 #endif
 
-    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
-		   (p->abandoned_heap
-		    ? p->abandoned_heap
-		    : HEAP_START(p)),
-		   p->heap_sz * sizeof(Eterm));
-    p->abandoned_heap = NULL;
-    p->flags &= ~F_ABANDONED_HEAP_USE;
+#ifdef HARDDEBUG
+    disallow_heap_frag_ref_in_heap(p, n_heap, n_htop);
+#endif
+
+    deallocate_previous_young_generation(p);
+
     HEAP_START(p) = n_heap;
     HEAP_TOP(p) = n_htop;
     HEAP_SIZE(p) = new_sz;
@@ -1613,11 +1604,6 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
 
     HIGH_WATER(p) = HEAP_TOP(p);
 
-#ifdef HARDDEBUG
-    disallow_heap_frag_ref_in_heap(p);
-#endif
-    remove_message_buffers(p);
-
     if (p->flags & F_ON_HEAP_MSGQ)
 	move_msgq_to_heap(p);
 
@@ -1770,22 +1756,56 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
     return adjusted;
 }
 
-/*
- * Remove all message buffers.
- */
 static void
-remove_message_buffers(Process* p)
+deallocate_previous_young_generation(Process *c_p)
 {
-    if (MBUF(p) != NULL) {
-	free_message_buffer(MBUF(p));
-	MBUF(p) = NULL;
+    Eterm *orig_heap;
+
+    if (!c_p->abandoned_heap) {
+	orig_heap = c_p->heap;
+	ASSERT(!(c_p->flags & F_ABANDONED_HEAP_USE));
+    }
+    else {
+	ErlHeapFragment *hfrag;
+
+	orig_heap = c_p->abandoned_heap;
+	c_p->abandoned_heap = NULL;
+	c_p->flags &= ~F_ABANDONED_HEAP_USE;
+
+	/*
+	 * Temporary heap located in heap fragment
+	 * only referred to by 'c_p->heap'. Add it to
+	 * 'c_p->mbuf' list and deallocate it as any
+	 * other heap fragment...
+	 */
+	hfrag = ((ErlHeapFragment *)
+		 (((char *) c_p->heap)
+		  - offsetof(ErlHeapFragment, mem)));
+
+	ASSERT(!hfrag->off_heap.first);
+	ASSERT(!hfrag->off_heap.overhead);
+	ASSERT(!hfrag->next);
+	ASSERT(c_p->htop - c_p->heap <= hfrag->alloc_size);
+
+	hfrag->next = c_p->mbuf;
+	c_p->mbuf = hfrag;
+    }
+
+    if (c_p->mbuf) {
+	free_message_buffer(c_p->mbuf);
+	c_p->mbuf = NULL;
     }    
-    if (p->msg_frag) {
-	erts_cleanup_messages(p->msg_frag);
-	p->msg_frag = NULL;
+    if (c_p->msg_frag) {
+	erts_cleanup_messages(c_p->msg_frag);
+	c_p->msg_frag = NULL;
     }
-    MBUF_SIZE(p) = 0;    
+    c_p->mbuf_sz = 0;  
+
+    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
+		   orig_heap,
+		   c_p->heap_sz * sizeof(Eterm));
 }
+
 #ifdef HARDDEBUG
 
 /*
@@ -1797,19 +1817,15 @@ remove_message_buffers(Process* p)
  */
 
 static void
-disallow_heap_frag_ref_in_heap(Process* p)
+disallow_heap_frag_ref_in_heap(Process *p, Eterm *heap, Eterm *htop)
 {
     Eterm* hp;
-    Eterm* htop;
-    Eterm* heap;
     Uint heap_size;
 
     if (p->mbuf == 0) {
 	return;
     }
 
-    htop = p->htop;
-    heap = p->heap;
     heap_size = (htop - heap)*sizeof(Eterm);
 
     hp = heap;
@@ -3297,13 +3313,15 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop)
     ErtsMessage* mp;
     Eterm *htop, *heap;
 
-    if (p->abandoned_heap)
+    if (p->abandoned_heap) {
 	ERTS_GET_ORIG_HEAP(p, heap, htop);
-    else {
-	heap = p->heap;
-	htop = real_htop ? real_htop : HEAP_TOP(p);
+	if (heap <= ptr && ptr < htop)
+	    return 1;
     }
 
+    heap = p->heap;
+    htop = real_htop ? real_htop : HEAP_TOP(p);
+
     if (OLD_HEAP(p) && (OLD_HEAP(p) <= ptr && ptr < OLD_HEND(p))) {
         return 1;
     }
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index e4c696ae3b..f45e6974cd 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -193,7 +193,7 @@ free_message_buffer(ErlHeapFragment* bp)
 
 	erts_cleanup_offheap(&bp->off_heap);
 	ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp,
-		       ERTS_HEAP_FRAG_SIZE(bp->size));	
+		       ERTS_HEAP_FRAG_SIZE(bp->alloc_size));	
 	bp = next_bp;
     }while (bp != NULL);
 }
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index d8bb00e8c6..a19db74763 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -90,9 +90,12 @@ Uint erts_process_memory(Process *p, int incl_msg_inq) {
   erts_doforall_links(ERTS_P_LINKS(p), &erts_one_link_size, &size);
   erts_doforall_monitors(ERTS_P_MONITORS(p), &erts_one_mon_size, &size);
   size += (p->heap_sz + p->mbuf_sz) * sizeof(Eterm);
+  if (p->abandoned_heap)
+      size += (p->hend - p->heap) * sizeof(Eterm);
   if (p->old_hend && p->old_heap)
     size += (p->old_hend - p->old_heap) * sizeof(Eterm);
 
+
   size += p->msg.len * sizeof(ErtsMessage);
 
   for (mp = p->msg.first; mp; mp = mp->next)
-- 
cgit v1.2.3


From a0d79ee6eeca9d99fbb644717ee2df57b824a527 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 10 Jan 2017 20:40:56 +0100
Subject: Fix call_time trace for NIFs

---
 erts/emulator/beam/beam_emu.c | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 59a9ea1417..6ac156a15d 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3550,18 +3550,11 @@ do {						\
 	    BifFunction vbf;
 	    ErlHeapFragment *live_hf_end;
 
-            if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
-                /* If we have run out of reductions, we do a context
-                   switch before calling the nif */
-                goto context_switch;
-            }
-
 	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
 
 	    DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
 	    c_p->current = I-3; /* current and vbf set to please handle_error */ 
-	    SWAPOUT;
-	    c_p->fcalls = FCALLS - 1;
+	    HEAVY_SWAPOUT;
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
 	    bif_nif_arity = I[-1];
 	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
-- 
cgit v1.2.3


From c057b468ee7535f199aa01301ff93e059fad4b07 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 11 Jan 2017 18:18:11 +0100
Subject: Fix call time tracing with dirty schedulers

---
 erts/emulator/beam/beam_bp.c        | 60 +++++++++++++++++++++++++++++++++----
 erts/emulator/beam/beam_bp.h        |  6 ++--
 erts/emulator/beam/erl_bif_trace.c  |  7 +++++
 erts/emulator/beam/erl_lock_check.c |  1 +
 4 files changed, 64 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index bbb2e4f34f..0df2df0eaa 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -74,6 +74,9 @@ extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */
 
 erts_smp_atomic32_t erts_active_bp_index;
 erts_smp_atomic32_t erts_staging_bp_index;
+#ifdef ERTS_DIRTY_SCHEDULERS
+erts_smp_mtx_t erts_dirty_bp_ix_mtx;
+#endif
 
 /*
  * Inlined helpers
@@ -85,6 +88,31 @@ get_mtime(Process *c_p)
     return erts_get_monotonic_time(erts_proc_sched_data(c_p));
 }
 
+static ERTS_INLINE Uint32
+acquire_bp_sched_ix(Process *c_p)
+{
+    ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
+    ASSERT(esdp);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	erts_smp_mtx_lock(&erts_dirty_bp_ix_mtx);
+        return (Uint32) erts_no_schedulers;
+    }
+#endif
+    return (Uint32) esdp->no - 1;
+}
+
+static ERTS_INLINE void
+release_bp_sched_ix(Uint32 ix)
+{
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ix == (Uint32) erts_no_schedulers)
+        erts_smp_mtx_unlock(&erts_dirty_bp_ix_mtx);
+#endif
+}
+
+
+
 /* *************************************************************************
 ** Local prototypes
 */
@@ -135,6 +163,9 @@ void
 erts_bp_init(void) {
     erts_smp_atomic32_init_nob(&erts_active_bp_index, 0);
     erts_smp_atomic32_init_nob(&erts_staging_bp_index, 1);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_smp_mtx_init(&erts_dirty_bp_ix_mtx, "dirty_break_point_index");
+#endif
 }
 
 
@@ -973,6 +1004,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
     bp_data_time_item_t sitem, *item = NULL;
     bp_time_hash_t *h = NULL;
     BpDataTime *pbdt = NULL;
+    Uint32 six = acquire_bp_sched_ix(c_p);
 
     ASSERT(c_p);
     ASSERT(erts_smp_atomic32_read_acqb(&c_p->state) & (ERTS_PSFLG_RUNNING
@@ -980,7 +1012,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
 
     /* get previous timestamp and breakpoint
      * from the process psd  */
-
+    
     pbt = ERTS_PROC_GET_CALL_TIME(c_p);
     time = get_mtime(c_p);
 
@@ -1006,7 +1038,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
 
 	/* if null then the breakpoint was removed */
 	if (pbdt) {
-	    h = &(pbdt->hash[bp_sched2ix_proc(c_p)]);
+	    h = &(pbdt->hash[six]);
 
 	    ASSERT(h);
 	    ASSERT(h->item);
@@ -1027,7 +1059,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
 
     /* this breakpoint */
     ASSERT(bdt);
-    h = &(bdt->hash[bp_sched2ix_proc(c_p)]);
+    h = &(bdt->hash[six]);
 
     ASSERT(h);
     ASSERT(h->item);
@@ -1041,6 +1073,8 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
 
     pbt->pc = I;
     pbt->time = time;
+
+    release_bp_sched_ix(six);
 }
 
 void
@@ -1051,6 +1085,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc)
     bp_data_time_item_t sitem, *item = NULL;
     bp_time_hash_t *h = NULL;
     BpDataTime *pbdt = NULL;
+    Uint32 six = acquire_bp_sched_ix(p);
 
     ASSERT(p);
     ASSERT(erts_smp_atomic32_read_acqb(&p->state) & (ERTS_PSFLG_RUNNING
@@ -1071,6 +1106,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc)
      */
 
     if (pbt) {
+
 	/* might have been removed due to
 	 * trace_pattern(false)
 	 */
@@ -1085,7 +1121,8 @@ erts_trace_time_return(Process *p, BeamInstr *pc)
 
 	/* beware, the trace_pattern might have been removed */
 	if (pbdt) {
-	    h = &(pbdt->hash[bp_sched2ix_proc(p)]);
+
+	    h = &(pbdt->hash[six]);
 
 	    ASSERT(h);
 	    ASSERT(h->item);
@@ -1096,11 +1133,15 @@ erts_trace_time_return(Process *p, BeamInstr *pc)
 	    } else {
 		BP_TIME_ADD(item, &sitem);
 	    }
+
 	}
 
 	pbt->pc = pc;
 	pbt->time = time;
+
     }
+
+    release_bp_sched_ix(six);
 }
 
 int 
@@ -1353,6 +1394,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) {
     bp_data_time_item_t sitem, *item = NULL;
     bp_time_hash_t *h = NULL;
     BpDataTime *pbdt = NULL;
+    Uint32 six = acquire_bp_sched_ix(p);
 
     ASSERT(p);
 
@@ -1375,7 +1417,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) {
 		sitem.pid   = p->common.id;
 		sitem.count = 0;
 
-		h = &(pbdt->hash[bp_sched2ix_proc(p)]);
+		h = &(pbdt->hash[six]);
 
 		ASSERT(h);
 		ASSERT(h->item);
@@ -1401,6 +1443,8 @@ void erts_schedule_time_break(Process *p, Uint schedule) {
 	    break;
 	}
     } /* pbt */
+
+    release_bp_sched_ix(six);
 }
 
 /* *************************************************************************
@@ -1517,7 +1561,11 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags,
 	ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0);
 	bdt = Alloc(sizeof(BpDataTime));
 	erts_refc_init(&bdt->refc, 1);
-	bdt->n = erts_no_total_schedulers;
+#ifdef ERTS_DIRTY_SCHEDULERS
+	bdt->n = erts_no_schedulers + 1;
+#else
+	bdt->n = erts_no_schedulers;
+#endif
 	bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n));
 	for (i = 0; i < bdt->n; i++) {
 	    bp_hash_init(&(bdt->hash[i]), 32);
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index 7206ef471a..4743e4fc2f 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -79,10 +79,8 @@ typedef struct generic_bp {
 #define ERTS_BP_CALL_TIME_SCHEDULE_OUT     (1)
 #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2)
 
-#ifdef ERTS_SMP
-#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->thr_id - 1)
-#else
-#define bp_sched2ix_proc(p) (0)
+#ifdef ERTS_DIRTY_SCHEDULERS
+extern erts_smp_mtx_t erts_dirty_bp_ix_mtx;
 #endif
 
 enum erts_break_op{
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 96275eb228..0627526d7e 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1062,9 +1062,16 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 	erts_smp_thr_progress_block();
     }
 #endif
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_smp_mtx_lock(&erts_dirty_bp_ix_mtx);
+#endif
+
 
     r = function_is_traced(p, mfa, &ms, &ms_meta, &meta, &count, &call_time);
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_smp_mtx_unlock(&erts_dirty_bp_ix_mtx);
+#endif
 #ifdef ERTS_SMP
     if ( (key == am_call_time) || (key == am_all)) {
 	erts_smp_thr_progress_unblock();
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 13a4b2cd93..5d4823c077 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -130,6 +130,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
 #ifdef ERTS_DIRTY_SCHEDULERS
     {   "dirty_run_queue_sleep_list",		"address"		},
     {	"dirty_gc_info",			NULL			},
+    {	"dirty_break_point_index",		NULL			},
 #endif
     {	"process_table",			NULL			},
     {	"cpu_info",				NULL			},
-- 
cgit v1.2.3


From af5169d85fcd545e3c857a219db081a62f33404d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 12 Jan 2017 13:58:26 +0100
Subject: erts: Fix race bug between export fun creation and code loading

Symptom: SEGV crash on ARM in delete_code() -> export_list().
Could probably happen on other machines as well.

Problem: Staging export table was iterated in an unsafe way
while an entry was added for a new export fun.

Solution: Correct write order and some memory barriers.
---
 erts/emulator/beam/beam_bif_load.c | 14 ++++++++------
 erts/emulator/beam/beam_load.c     | 11 ++++++-----
 erts/emulator/beam/export.c        |  2 +-
 erts/emulator/beam/index.c         |  9 ++++++++-
 erts/emulator/beam/index.h         | 14 ++++++++++++++
 5 files changed, 37 insertions(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 5969197168..93f5ed500b 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -786,7 +786,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
     }
 
     if (BIF_ARG_2 == am_true) {
-	int i;
+	int i, num_exps;
 
 	/*
 	 * Make the code with the on_load function current.
@@ -802,7 +802,8 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	/*
 	 * The on_load function succeded. Fix up export entries.
 	 */
-	for (i = 0; i < export_list_size(code_ix); i++) {
+	num_exps = export_list_size(code_ix);
+	for (i = 0; i < num_exps; i++) {
 	    Export *ep = export_list(i,code_ix);
 	    if (ep == NULL || ep->code[0] != BIF_ARG_1) {
 		continue;
@@ -822,14 +823,15 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	modp->curr.code_hdr->on_load_function_ptr = NULL;
 	set_default_trace_pattern(BIF_ARG_1);
     } else if (BIF_ARG_2 == am_false) {
-	int i;
+	int i, num_exps;
 
 	/*
 	 * The on_load function failed. Remove references to the
 	 * code that is about to be purged from the export entries.
 	 */
 
-	for (i = 0; i < export_list_size(code_ix); i++) {
+	num_exps = export_list_size(code_ix);
+	for (i = 0; i < num_exps; i++) {
 	    Export *ep = export_list(i,code_ix);
 	    if (ep == NULL || ep->code[0] != BIF_ARG_1) {
 		continue;
@@ -2011,9 +2013,9 @@ delete_code(Module* modp)
 {
     ErtsCodeIndex code_ix = erts_staging_code_ix();
     Eterm module = make_atom(modp->module);
-    int i;
+    int i, num_exps = export_list_size(code_ix);
 
-    for (i = 0; i < export_list_size(code_ix); i++) {
+    for (i = 0; i < num_exps; i++) {
 	Export *ep = export_list(i, code_ix);
         if (ep != NULL && (ep->code[0] == module)) {
 	    if (ep->addressv[code_ix] == ep->code+3) {
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 0afdedf6c2..3f2bdf3f9d 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -797,14 +797,14 @@ erts_finish_loading(Binary* magic, Process* c_p,
     } else {
 	ErtsCodeIndex code_ix = erts_staging_code_ix();
 	Eterm module = stp->module;
-	int i;
+	int i, num_exps;
 
 	/*
 	 * There is an -on_load() function. We will keep the current
 	 * code, but we must turn off any tracing.
 	 */
-
-	for (i = 0; i < export_list_size(code_ix); i++) {
+        num_exps = export_list_size(code_ix);
+	for (i = 0; i < num_exps; i++) {
 	    Export *ep = export_list(i, code_ix);
 	    if (ep == NULL || ep->code[0] != module) {
 		continue;
@@ -5754,12 +5754,13 @@ exported_from_module(Process* p, /* Process whose heap to use. */
                      ErtsCodeIndex code_ix,
 		     Eterm mod) /* Tagged atom for module. */
 {
-    int i;
+    int i, num_exps;
     Eterm* hp = NULL;
     Eterm* hend = NULL;
     Eterm result = NIL;
 
-    for (i = 0; i < export_list_size(code_ix); i++) {
+    num_exps = export_list_size(code_ix);
+    for (i = 0; i < num_exps; i++) {
 	Export* ep = export_list(i,code_ix);
 	
 	if (ep->code[0] == mod) {
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 2a19211987..2a007ce860 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -348,7 +348,7 @@ Export *export_list(int i, ErtsCodeIndex code_ix)
 
 int export_list_size(ErtsCodeIndex code_ix)
 {
-    return export_tables[code_ix].entries;
+    return erts_index_num_entries(&export_tables[code_ix]);
 }
 
 int export_table_sz(void)
diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c
index 26d6c04ea0..4cebb4d8fa 100644
--- a/erts/emulator/beam/index.c
+++ b/erts/emulator/beam/index.c
@@ -91,9 +91,16 @@ index_put_entry(IndexTable* t, void* tmpl)
 	t->seg_table[ix>>INDEX_PAGE_SHIFT] = erts_alloc(t->type, sz);
 	t->size += INDEX_PAGE_SIZE;
     }
-    t->entries++;
     p->index = ix;
     t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK] = p;
+
+    /*
+     * Do a write barrier here to allow readers to do lock free iteration.
+     * erts_index_num_entries() does matching read barrier.
+     */
+    ERTS_SMP_WRITE_MEMORY_BARRIER;
+    t->entries++;
+
     return p;
 }
 
diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h
index 0a109d8699..532dec0168 100644
--- a/erts/emulator/beam/index.h
+++ b/erts/emulator/beam/index.h
@@ -65,6 +65,7 @@ void index_erase_latest_from(IndexTable*, Uint ix);
 
 ERTS_GLB_INLINE int index_put(IndexTable*, void*);
 ERTS_GLB_INLINE IndexSlot* erts_index_lookup(IndexTable*, Uint);
+ERTS_GLB_INLINE int erts_index_num_entries(IndexTable* t);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
@@ -78,6 +79,19 @@ erts_index_lookup(IndexTable* t, Uint ix)
 {
     return t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK];
 }
+
+ERTS_GLB_INLINE int erts_index_num_entries(IndexTable* t)
+{
+    int ret = t->entries;
+    /*
+     * Do a read barrier here to allow lock free iteration
+     * on tables where entries are never erased.
+     * index_put_entry() does matching write barrier.
+     */
+    ERTS_SMP_READ_MEMORY_BARRIER;
+    return ret;
+}
+
 #endif
 
 #endif
-- 
cgit v1.2.3


From 6bcdd45abd97134fddfb5b0307b1d256337b0c67 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 28 Dec 2016 18:23:06 +0100
Subject: Reduction counting on non-tail return

---
 erts/emulator/beam/beam_emu.c    | 21 +++++++++++++++++----
 erts/emulator/beam/erl_vm.h      |  2 +-
 erts/emulator/test/nif_SUITE.erl |  2 +-
 3 files changed, 19 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index c554bd73b6..bb77dbd955 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -633,21 +633,34 @@ void** beam_ops;
       y[4] = xt4;				\
  } while (0)
 
+#define DispatchReturn                          \
+do {                                            \
+    if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
+        FCALLS--;				\
+        Goto(*I);                               \
+    }                                           \
+    else {					\
+        c_p->current = NULL;                    \
+        c_p->arity = 1;                         \
+        goto context_switch3;			\
+    }						\
+} while (0)
+
 #define MoveReturn(Src)				\
     x(0) = (Src);				\
     I = c_p->cp;				\
     ASSERT(VALID_INSTR(*c_p->cp));		\
     c_p->cp = 0;				\
     CHECK_TERM(r(0));				\
-    Goto(*I)
+    DispatchReturn
 
 #define DeallocateReturn(Deallocate)       \
   do {                                     \
     int words_to_pop = (Deallocate);       \
-    SET_I((BeamInstr *) cp_val(*E));                     \
+    SET_I((BeamInstr *) cp_val(*E));       \
     E = ADD_BYTE_OFFSET(E, words_to_pop);  \
     CHECK_TERM(r(0));                      \
-    Goto(*I);                              \
+    DispatchReturn;                        \
   } while (0)
 
 #define MoveDeallocateReturn(Src, Deallocate)	\
@@ -1681,7 +1694,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
     c_p->cp = 0;
     CHECK_TERM(r(0));
     HEAP_SPACE_VERIFIED(0);
-    Goto(*I);
+    DispatchReturn;
  }
 
     /*
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 93cfe08105..d88cafc5bf 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -36,7 +36,7 @@
 #define EMULATOR "BEAM"
 #define SEQ_TRACE 1
 
-#define CONTEXT_REDS 2000	/* Swap process out after this number */
+#define CONTEXT_REDS 4000	/* Swap process out after this number */
 #define MAX_ARG 255	        /* Max number of arguments allowed */
 #define MAX_REG 1024            /* Max number of x(N) registers used */
 
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 0d3910b2e2..36d512e388 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -1689,7 +1689,7 @@ consume_timeslice(Config) when is_list(Config) ->
 
 consume_timeslice_test(Config) when is_list(Config) ->
     ensure_lib_loaded(Config),
-    CONTEXT_REDS = 2000,
+    CONTEXT_REDS = 4000,
     Me = self(),
     Go = make_ref(),
     RedDiff = make_ref(),
-- 
cgit v1.2.3


From 5aff60d96efac96a41b514ed167f13eb787a415f Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 26 Sep 2016 17:05:50 +0200
Subject: Support for dirty BIFs

---
 erts/emulator/Makefile.in                          |  75 ++-
 erts/emulator/beam/atom.names                      |   9 +
 erts/emulator/beam/beam_bif_load.c                 |   6 +
 erts/emulator/beam/beam_debug.c                    | 347 ++++++++++
 erts/emulator/beam/beam_emu.c                      | 117 ++--
 erts/emulator/beam/bif.c                           | 317 ++++++++-
 erts/emulator/beam/bif.h                           |  87 ++-
 erts/emulator/beam/bif.tab                         |   3 +
 erts/emulator/beam/erl_bif_info.c                  |   3 +
 erts/emulator/beam/erl_dirty_bif.tab               |  82 +++
 erts/emulator/beam/erl_gc.c                        |  24 +-
 erts/emulator/beam/erl_msacc.c                     |   4 +-
 erts/emulator/beam/erl_msacc.h                     |   2 +-
 erts/emulator/beam/erl_nfunc_sched.c               | 144 +++++
 erts/emulator/beam/erl_nfunc_sched.h               | 289 +++++++++
 erts/emulator/beam/erl_nif.c                       | 707 ++++++++-------------
 erts/emulator/beam/erl_process.c                   |  18 +-
 erts/emulator/beam/erl_process.h                   |   8 +-
 erts/emulator/beam/error.h                         |  88 +--
 erts/emulator/beam/global.h                        |  14 +-
 erts/emulator/beam/utils.c                         |   2 +
 erts/emulator/hipe/hipe_amd64_bifs.m4              |  12 +-
 erts/emulator/hipe/hipe_arm_bifs.m4                |   4 +-
 erts/emulator/hipe/hipe_bif0.c                     |  11 +-
 erts/emulator/hipe/hipe_bif0.h                     |   2 +-
 erts/emulator/hipe/hipe_bif2.c                     |  10 +-
 erts/emulator/hipe/hipe_bif_list.m4                |  34 +-
 erts/emulator/hipe/hipe_native_bif.c               |  33 +-
 erts/emulator/hipe/hipe_native_bif.h               |  20 +-
 erts/emulator/hipe/hipe_ppc_bifs.m4                |   4 +-
 erts/emulator/hipe/hipe_sparc_bifs.m4              |   4 +-
 erts/emulator/hipe/hipe_x86_bifs.m4                |  12 +-
 erts/emulator/test/Makefile                        |   1 +
 erts/emulator/test/call_trace_SUITE.erl            |   2 +-
 erts/emulator/test/dirty_bif_SUITE.erl             | 583 +++++++++++++++++
 erts/emulator/test/dirty_bif_SUITE_data/.gitignore |   0
 erts/emulator/test/old_scheduler_SUITE.erl         |   4 +-
 erts/emulator/test/port_SUITE.erl                  |   4 +-
 erts/emulator/test/process_SUITE.erl               |  28 +-
 erts/emulator/test/z_SUITE.erl                     |  18 +-
 erts/emulator/utils/make_tables                    | 161 ++++-
 41 files changed, 2560 insertions(+), 733 deletions(-)
 create mode 100644 erts/emulator/beam/erl_dirty_bif.tab
 create mode 100644 erts/emulator/beam/erl_nfunc_sched.c
 create mode 100644 erts/emulator/beam/erl_nfunc_sched.h
 create mode 100644 erts/emulator/test/dirty_bif_SUITE.erl
 create mode 100644 erts/emulator/test/dirty_bif_SUITE_data/.gitignore

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index ce50022683..eb27703d6a 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -52,6 +52,7 @@ OMIT_OMIT_FP=no
 TYPE_LIBS=
 
 DIRTY_SCHEDULER_SUPPORT=@DIRTY_SCHEDULER_SUPPORT@
+DIRTY_SCHEDULER_TEST=@DIRTY_SCHEDULER_TEST@
 
 ifeq ($(TYPE),debug)
 PURIFY =
@@ -181,9 +182,24 @@ ENABLE_ALLOC_TYPE_VARS += smp nofrag
 M4FLAGS += -DERTS_SMP=1
 ifeq ($(DIRTY_SCHEDULER_SUPPORT),yes)
 THR_DEFS += -DERTS_DIRTY_SCHEDULERS
-endif
+DS_SUPPORT=yes
 
-else
+ifeq ($(DIRTY_SCHEDULER_TEST),yes)
+DS_TEST=yes
+THR_DEFS += -DERTS_DIRTY_SCHEDULERS_TEST
+else # DIRTY_SCHEDULER_TEST
+DS_TEST=no
+endif # DIRTY_SCHEDULER_TEST
+
+else # DIRTY_SCHEDULER_SUPPORT
+DS_SUPPORT=no
+DS_TEST=no
+endif # DIRTY_SCHEDULER_SUPPORT
+
+else # FLAVOR
+
+DS_SUPPORT=no
+DS_TEST=no
 
 # If flavor isn't one of the above, it *is* plain flavor...
 override FLAVOR=plain
@@ -548,8 +564,10 @@ GENERATE += $(TTF_DIR)/OPCODES-GENERATED
 
 # bif and atom table
 ATOMS=	beam/atom.names
+DIRTY_BIFS = beam/erl_dirty_bif.tab
 BIFS =	beam/bif.tab
 ifdef HIPE_ENABLED
+HIPE=yes
 HIPE_ARCH64_TAB=hipe/hipe_bif64.tab
 HIPE_x86_TAB=hipe/hipe_x86.tab
 HIPE_amd64_TAB=hipe/hipe_amd64.tab $(HIPE_ARCH64_TAB)
@@ -559,20 +577,26 @@ HIPE_ppc64_TAB=hipe/hipe_ppc64.tab $(HIPE_ARCH64_TAB)
 HIPE_arm_TAB=hipe/hipe_arm.tab
 HIPE_ARCH_TAB=$(HIPE_$(ARCH)_TAB)
 BIFS += hipe/hipe_bif0.tab hipe/hipe_bif1.tab hipe/hipe_bif2.tab $(HIPE_ARCH_TAB)
-endif
-
-$(TARGET)/erl_bif_table.c \
-$(TARGET)/erl_bif_table.h \
-$(TARGET)/erl_bif_wrap.c \
-$(TARGET)/erl_bif_list.h \
-$(TARGET)/erl_gc_bifs.c \
-$(TARGET)/erl_atom_table.c \
-$(TARGET)/erl_atom_table.h \
-	: $(TARGET)/TABLES-GENERATED
-$(TARGET)/TABLES-GENERATED: $(ATOMS) $(BIFS) utils/make_tables
-	$(gen_verbose)LANG=C $(PERL) utils/make_tables -src $(TARGET) -include $(TARGET)\
-		$(ATOMS) $(BIFS) && echo $? >$(TARGET)/TABLES-GENERATED
-GENERATE += $(TARGET)/TABLES-GENERATED
+HIPE_NBIF_FILES=$(TTF_DIR)/hipe_nbif_impl.h $(TTF_DIR)/hipe_nbif_impl.c
+else
+HIPE=no
+HIPE_NBIF_FILES=
+endif
+
+$(TTF_DIR)/erl_bif_table.c \
+$(TTF_DIR)/erl_bif_table.h \
+$(TTF_DIR)/erl_bif_wrap.c \
+$(TTF_DIR)/erl_bif_list.h \
+$(TTF_DIR)/erl_atom_table.c \
+$(TTF_DIR)/erl_atom_table.h \
+$(TTF_DIR)/erl_gc_bifs.c \
+$(TTF_DIR)/erl_dirty_bif_wrap.c \
+$(HIPE_NBIF_FILES) \
+	: $(TTF_DIR)/TABLES-GENERATED
+$(TTF_DIR)/TABLES-GENERATED: $(ATOMS) $(DIRTY_BIFS) $(BIFS) utils/make_tables
+	$(gen_verbose)LANG=C $(PERL) utils/make_tables -src $(TTF_DIR) -include $(TTF_DIR)\
+		-ds $(DS_SUPPORT) -dst $(DS_TEST) -hipe $(HIPE) $(ATOMS) $(DIRTY_BIFS) $(BIFS) && echo $? >$(TTF_DIR)/TABLES-GENERATED
+GENERATE += $(TTF_DIR)/TABLES-GENERATED
 
 $(TTF_DIR)/erl_alloc_types.h: beam/erl_alloc.types utils/make_alloc_types
 	$(gen_verbose)LANG=C $(PERL) utils/make_alloc_types -src $< -dst $@ $(ENABLE_ALLOC_TYPE_VARS)
@@ -750,8 +774,8 @@ RUN_OBJS = \
 	$(OBJDIR)/erl_bif_info.o	$(OBJDIR)/erl_bif_op.o \
 	$(OBJDIR)/erl_bif_os.o		$(OBJDIR)/erl_bif_lists.o \
 	$(OBJDIR)/erl_bif_trace.o	$(OBJDIR)/erl_bif_unique.o \
-	$(OBJDIR)/erl_bif_wrap.o \
-	$(OBJDIR)/erl_gc_bifs.o \
+	$(OBJDIR)/erl_bif_wrap.o	$(OBJDIR)/erl_nfunc_sched.o \
+	$(OBJDIR)/erl_gc_bifs.o 	$(OBJDIR)/erl_dirty_bif_wrap.o \
 	$(OBJDIR)/erl_trace.o		$(OBJDIR)/copy.o \
 	$(OBJDIR)/utils.o		$(OBJDIR)/bif.o \
 	$(OBJDIR)/io.o 			$(OBJDIR)/erl_printf_term.o\
@@ -882,6 +906,7 @@ HIPE_noarch_OBJS=
 HIPE_ARCH_OBJS=$(HIPE_$(ARCH)_OBJS)
 
 HIPE_OBJS= \
+	$(OBJDIR)/hipe_nbif_impl.o \
 	$(OBJDIR)/hipe_bif0.o \
 	$(OBJDIR)/hipe_bif1.o \
 	$(OBJDIR)/hipe_bif2.o \
@@ -917,7 +942,7 @@ $(OBJS): $(TTF_DIR)/GENERATED
 ########################################
 # HiPE section
 
-M4FLAGS += -DTARGET=$(TARGET) -DOPSYS=$(OPSYS) -DARCH=$(ARCH)
+M4FLAGS += -DTARGET=$(TARGET) -DTTF_DIR=$(TTF_DIR) -DOPSYS=$(OPSYS) -DARCH=$(ARCH)
 
 $(TTF_DIR)/%.S: hipe/%.m4
 	$(m4_verbose)m4 $(M4FLAGS) $< > $@
@@ -938,7 +963,7 @@ $(BINDIR)/hipe_mkliterals$(TF_MARKER):	$(OBJDIR)/hipe_mkliterals.o
 	$(ld_verbose)$(CC) $(LDFLAGS) -o $@ $< $(TYPE_LIBS)
 
 $(OBJDIR)/hipe_mkliterals.o:	$(HIPE_ASM) $(TTF_DIR)/erl_alloc_types.h $(DTRACE_HEADERS) \
-	$(TTF_DIR)/OPCODES-GENERATED $(TARGET)/TABLES-GENERATED
+	$(TTF_DIR)/OPCODES-GENERATED $(TTF_DIR)/TABLES-GENERATED
 
 $(TTF_DIR)/hipe_literals.h:	$(BINDIR)/hipe_mkliterals$(TF_MARKER)
 	$(gen_verbose)$(BINDIR)/hipe_mkliterals$(TF_MARKER) -c > $@
@@ -947,7 +972,7 @@ $(OBJDIR)/hipe_x86_glue.o:	hipe/hipe_x86_glue.S \
 	$(TTF_DIR)/hipe_x86_asm.h $(TTF_DIR)/hipe_literals.h \
 	hipe/hipe_mode_switch.h
 $(TTF_DIR)/hipe_x86_bifs.S:	hipe/hipe_x86_bifs.m4 hipe/hipe_x86_asm.m4 \
-	hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h
+	hipe/hipe_bif_list.m4 $(TTF_DIR)/erl_bif_list.h hipe/hipe_gbif_list.h
 $(OBJDIR)/hipe_x86_bifs.o:	$(TTF_DIR)/hipe_x86_bifs.S \
 	$(TTF_DIR)/hipe_literals.h
 
@@ -955,7 +980,7 @@ $(OBJDIR)/hipe_amd64_glue.o:	hipe/hipe_amd64_glue.S \
 	$(TTF_DIR)/hipe_amd64_asm.h $(TTF_DIR)/hipe_literals.h \
 	hipe/hipe_mode_switch.h
 $(TTF_DIR)/hipe_amd64_bifs.S:	hipe/hipe_amd64_bifs.m4 hipe/hipe_amd64_asm.m4 \
-	hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h
+	hipe/hipe_bif_list.m4 $(TTF_DIR)/erl_bif_list.h hipe/hipe_gbif_list.h
 $(OBJDIR)/hipe_amd64_bifs.o:	$(TTF_DIR)/hipe_amd64_bifs.S \
 	$(TTF_DIR)/hipe_literals.h
 
@@ -963,21 +988,21 @@ $(OBJDIR)/hipe_sparc_glue.o:	hipe/hipe_sparc_glue.S \
 	$(TTF_DIR)/hipe_sparc_asm.h hipe/hipe_mode_switch.h \
 	$(TTF_DIR)/hipe_literals.h
 $(TTF_DIR)/hipe_sparc_bifs.S:	hipe/hipe_sparc_bifs.m4 hipe/hipe_sparc_asm.m4 \
-	hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h
+	hipe/hipe_bif_list.m4 $(TTF_DIR)/erl_bif_list.h hipe/hipe_gbif_list.h
 $(OBJDIR)/hipe_sparc_bifs.o:	$(TTF_DIR)/hipe_sparc_bifs.S \
 	$(TTF_DIR)/hipe_literals.h
 
 $(OBJDIR)/hipe_ppc_glue.o:	hipe/hipe_ppc_glue.S $(TTF_DIR)/hipe_ppc_asm.h \
 	hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h
 $(TTF_DIR)/hipe_ppc_bifs.S:	hipe/hipe_ppc_bifs.m4 hipe/hipe_ppc_asm.m4 \
-	hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h
+	hipe/hipe_bif_list.m4 $(TTF_DIR)/erl_bif_list.h hipe/hipe_gbif_list.h
 $(OBJDIR)/hipe_ppc_bifs.o:	$(TTF_DIR)/hipe_ppc_bifs.S \
 	$(TTF_DIR)/hipe_literals.h
 
 $(OBJDIR)/hipe_arm_glue.o:	hipe/hipe_arm_glue.S $(TTF_DIR)/hipe_arm_asm.h \
 	hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h
 $(TTF_DIR)/hipe_arm_bifs.S:	hipe/hipe_arm_bifs.m4 hipe/hipe_arm_asm.m4 \
-	hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h
+	hipe/hipe_bif_list.m4 $(TTF_DIR)/erl_bif_list.h hipe/hipe_gbif_list.h
 $(OBJDIR)/hipe_arm_bifs.o:	$(TTF_DIR)/hipe_arm_bifs.S \
 	$(TTF_DIR)/hipe_literals.h
 
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index b1aeed7889..7df350116a 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -76,6 +76,7 @@ atom ac
 atom accessor
 atom active
 atom active_tasks
+atom alive
 atom all
 atom all_but_first
 atom all_names
@@ -199,10 +200,15 @@ atom dexit
 atom depth
 atom dgroup_leader
 atom dictionary
+atom dirty_bif_exception
+atom dirty_bif_result
+atom dirty_bif_trap
 atom dirty_cpu
 atom dirty_cpu_schedulers_online
 atom dirty_execution
 atom dirty_io
+atom dirty_nif_exception
+atom dirty_nif_finalizer
 atom disable_trace
 atom disabled
 atom discard
@@ -243,6 +249,7 @@ atom ERROR='ERROR'
 atom error_handler
 atom error_logger
 atom erts_code_purger
+atom erts_debug
 atom erts_internal
 atom ets
 atom ETS_TRANSFER='ETS-TRANSFER'
@@ -573,6 +580,7 @@ atom scientific
 atom scope
 atom second
 atom seconds
+atom send
 atom send_to_non_existing_process
 atom sensitive
 atom sequential_tracer
@@ -665,6 +673,7 @@ atom value
 atom values
 atom version
 atom visible
+atom wait
 atom waiting
 atom wall_clock
 atom warning
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 09331eb8ad..36bbe81ed5 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -36,6 +36,7 @@
 #include "erl_nif.h"
 #include "erl_bits.h"
 #include "erl_thr_progress.h"
+#include "erl_nfunc_sched.h"
 #ifdef HIPE
 #  include "hipe_bif0.h"
 #  define IF_HIPE(X) (X)
@@ -1102,6 +1103,11 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls)
  
     *redsp += 1;
 
+    if (erts_check_nif_export_in_area(rp, mod_start, mod_size))
+	return am_true;
+
+    *redsp += 1;
+
     if (erts_check_nif_export_in_area(rp, mod_start, mod_size))
 	return am_true;
 
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index e72d7f8de4..3ffbb0364c 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -39,6 +39,7 @@
 #include "beam_bp.h"
 #include "erl_binary.h"
 #include "erl_thr_progress.h"
+#include "erl_nfunc_sched.h"
 
 #ifdef ARCH_64
 # define HEXF "%016bpX"
@@ -764,3 +765,349 @@ static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif)
 	erts_print(to, to_arg, "%T/%u", name, arity);
     }
 }
+
+/*
+ * Dirty BIF testing.
+ *
+ * The erts_debug:dirty_cpu/2, erts_debug:dirty_io/1, and
+ * erts_debug:dirty/3 BIFs are used by the dirty_bif_SUITE
+ * test suite.
+ */
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+static int ms_wait(Process *c_p, Eterm etimeout, int busy);
+static int dirty_send_message(Process *c_p, Eterm to, Eterm tag);
+#endif
+static BIF_RETTYPE dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I);
+
+/*
+ * erts_debug:dirty_cpu/2 is statically determined to execute on
+ * a dirty CPU scheduler (see erts_dirty_bif.tab).
+ */
+BIF_RETTYPE
+erts_debug_dirty_cpu_2(BIF_ALIST_2)
+{
+    return dirty_test(BIF_P, am_dirty_cpu, BIF_ARG_1, BIF_ARG_2, BIF_I);
+}
+
+/*
+ * erts_debug:dirty_io/2 is statically determined to execute on
+ * a dirty I/O scheduler (see erts_dirty_bif.tab).
+ */
+BIF_RETTYPE
+erts_debug_dirty_io_2(BIF_ALIST_2)
+{
+    return dirty_test(BIF_P, am_dirty_io, BIF_ARG_1, BIF_ARG_2, BIF_I);
+}
+
+/*
+ * erts_debug:dirty/3 executes on a normal scheduler.
+ */
+BIF_RETTYPE
+erts_debug_dirty_3(BIF_ALIST_3)
+{
+#ifdef ERTS_DIRTY_SCHEDULERS
+    Eterm argv[2];
+    switch (BIF_ARG_1) {
+    case am_normal:
+	return dirty_test(BIF_P, am_normal, BIF_ARG_2, BIF_ARG_3, BIF_I);
+    case am_dirty_cpu:
+	argv[0] = BIF_ARG_2;
+	argv[1] = BIF_ARG_3;
+	return erts_schedule_bif(BIF_P,
+				 argv,
+				 BIF_I,
+				 erts_debug_dirty_cpu_2,
+				 ERTS_SCHED_DIRTY_CPU,
+				 am_erts_debug,
+				 am_dirty_cpu,
+				 2);
+    case am_dirty_io:
+	argv[0] = BIF_ARG_2;
+	argv[1] = BIF_ARG_3;
+	return erts_schedule_bif(BIF_P,
+				 argv,
+				 BIF_I,
+				 erts_debug_dirty_io_2,
+				 ERTS_SCHED_DIRTY_IO,
+				 am_erts_debug,
+				 am_dirty_io,
+				 2);
+    default:
+	BIF_ERROR(BIF_P, EXC_BADARG);
+    }
+#else
+	BIF_ERROR(BIF_P, EXC_UNDEF);
+#endif
+}
+
+
+static BIF_RETTYPE
+dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I)
+{
+    BIF_RETTYPE ret;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (am_scheduler == arg1) {
+	ErtsSchedulerData *esdp;
+	if (arg2 != am_type) 
+	    goto badarg;
+	esdp = erts_proc_sched_data(c_p);
+	if (!esdp)
+	    ERTS_BIF_PREP_RET(ret, am_error);
+	else if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+	    ERTS_BIF_PREP_RET(ret, am_normal);
+	else if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp))
+	    ERTS_BIF_PREP_RET(ret, am_dirty_cpu);
+	else if (ERTS_SCHEDULER_IS_DIRTY_IO(esdp))
+	    ERTS_BIF_PREP_RET(ret, am_dirty_io);
+	else
+	    ERTS_BIF_PREP_RET(ret, am_error);
+    }
+    else if (am_error == arg1) {
+	switch (arg2) {
+	case am_notsup:
+	    ERTS_BIF_PREP_ERROR(ret, c_p, EXC_NOTSUP);
+	    break;
+	case am_undef:
+	    ERTS_BIF_PREP_ERROR(ret, c_p, EXC_UNDEF);
+	    break;
+	case am_badarith:
+	    ERTS_BIF_PREP_ERROR(ret, c_p, EXC_BADARITH);
+	    break;
+	case am_noproc:
+	    ERTS_BIF_PREP_ERROR(ret, c_p, EXC_NOPROC);
+	    break;
+	case am_system_limit:
+	    ERTS_BIF_PREP_ERROR(ret, c_p, SYSTEM_LIMIT);
+	    break;
+	case am_badarg:
+	default:
+	    goto badarg;
+	}
+    }
+    else if (am_copy == arg1) {
+	int i;
+	Eterm res;
+
+	for (res = NIL, i = 0; i < 1000; i++) {
+	    Eterm *hp, sz;
+	    Eterm cpy;
+	    /* We do not want this to be optimized,
+	       but rather the oposite... */
+	    sz = size_object(arg2);
+	    hp = HAlloc(c_p, sz);
+	    cpy = copy_struct(arg2, sz, &hp, &c_p->off_heap);
+	    hp = HAlloc(c_p, 2);
+	    res = CONS(hp, cpy, res);
+	}
+
+	ERTS_BIF_PREP_RET(ret, res);
+    }
+    else if (am_send == arg1) {
+	dirty_send_message(c_p, arg2, am_ok);
+	ERTS_BIF_PREP_RET(ret, am_ok);
+    }
+    else if (ERTS_IS_ATOM_STR("wait", arg1)) {
+	if (!ms_wait(c_p, arg2, type == am_dirty_cpu))
+	    goto badarg;
+	ERTS_BIF_PREP_RET(ret, am_ok);
+    }
+    else if (ERTS_IS_ATOM_STR("reschedule", arg1)) {
+	/*
+	 * Reschedule operation after decrement of two until we reach
+	 * zero. Switch between dirty scheduler types when 'n' is
+	 * evenly divided by 4. If the initial value wasn't evenly
+	 * dividable by 2, throw badarg exception.
+	 */
+	Eterm next_type;
+	Sint n;
+	if (!term_to_Sint(arg2, &n) || n < 0)
+	    goto badarg;
+	if (n == 0)
+	    ERTS_BIF_PREP_RET(ret, am_ok);
+	else {
+	    Eterm argv[3];
+	    Eterm eint = erts_make_integer((Uint) (n - 2), c_p);
+	    if (n % 4 != 0)
+		next_type = type;
+	    else {
+		switch (type) {
+		case am_dirty_cpu: next_type = am_dirty_io; break;
+		case am_dirty_io: next_type = am_normal; break;
+		case am_normal: next_type = am_dirty_cpu; break;
+		default: goto badarg;
+		}
+	    }
+	    switch (next_type) {
+	    case am_dirty_io:
+		argv[0] = arg1;
+		argv[1] = eint;
+		ret = erts_schedule_bif(c_p,
+					argv,
+					I,
+					erts_debug_dirty_io_2,
+					ERTS_SCHED_DIRTY_IO,
+					am_erts_debug,
+					am_dirty_io,
+					2);
+		break;
+	    case am_dirty_cpu:
+		argv[0] = arg1;
+		argv[1] = eint;
+		ret = erts_schedule_bif(c_p,
+					argv,
+					I,
+					erts_debug_dirty_cpu_2,
+					ERTS_SCHED_DIRTY_CPU,
+					am_erts_debug,
+					am_dirty_cpu,
+					2);
+		break;
+	    case am_normal:
+		argv[0] = am_normal;
+		argv[1] = arg1;
+		argv[2] = eint;
+		ret = erts_schedule_bif(c_p,
+					argv,
+					I,
+					erts_debug_dirty_3,
+					ERTS_SCHED_NORMAL,
+					am_erts_debug,
+					am_dirty,
+					3);
+		break;
+	    default:
+		goto badarg;
+	    }
+	}
+    }
+    else if (ERTS_IS_ATOM_STR("ready_wait6_done", arg1)) {
+	ERTS_DECL_AM(ready);
+	ERTS_DECL_AM(done);
+	dirty_send_message(c_p, arg2, AM_ready);
+	ms_wait(c_p, make_small(6000), 0);
+	dirty_send_message(c_p, arg2, AM_done);
+	ERTS_BIF_PREP_RET(ret, am_ok);
+    }
+    else if (ERTS_IS_ATOM_STR("alive_waitexiting", arg1)) {
+	Process *real_c_p = erts_proc_shadow2real(c_p);
+	Eterm *hp, *hp2;
+	Uint sz;
+	int i;
+	if (ERTS_PROC_IS_EXITING(real_c_p))
+	    goto badarg;
+	dirty_send_message(c_p, arg2, am_alive);
+	/* Wait until dead */
+
+	while (!ERTS_PROC_IS_EXITING(real_c_p))
+	    erts_thr_yield();
+
+	ms_wait(c_p, make_small(1000), 0);
+
+	/* Should still be able to allocate memory */
+	hp = HAlloc(c_p, 3); /* Likely on heap */
+	sz = 10000;
+	hp2 = HAlloc(c_p, sz); /* Likely in heap fragment */
+	*hp2 = make_pos_bignum_header(sz);
+	for (i = 1; i < sz; i++)
+	    hp2[i] = (Eterm) 4711;
+	ERTS_BIF_PREP_RET(ret, TUPLE2(hp, am_ok, make_big(hp2)));
+    }
+    else {
+    badarg:
+	ERTS_BIF_PREP_ERROR(ret, c_p, BADARG);
+    }
+#else
+    ERTS_BIF_PREP_ERROR(ret, c_p, EXC_UNDEF);
+#endif
+    return ret;
+}
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+static int
+dirty_send_message(Process *c_p, Eterm to, Eterm tag)
+{
+    ErtsProcLocks c_p_locks, rp_locks;
+    Process *rp, *real_c_p;
+    Eterm msg, *hp;
+    ErlOffHeap *ohp;
+    ErtsMessage *mp;
+
+    ASSERT(is_immed(tag));
+
+    real_c_p = erts_proc_shadow2real(c_p);
+    if (real_c_p != c_p)
+	c_p_locks = 0;
+    else
+	c_p_locks = ERTS_PROC_LOCK_MAIN;
+
+    ASSERT(real_c_p->common.id == c_p->common.id);
+
+    rp = erts_pid2proc_opt(real_c_p, c_p_locks,
+			   to, 0,
+			   ERTS_P2P_FLG_INC_REFC);
+
+    if (!rp)
+	return 0;
+
+    rp_locks = 0;
+    mp = erts_alloc_message_heap(rp, &rp_locks, 3, &hp, &ohp);
+
+    msg = TUPLE2(hp, tag, c_p->common.id);
+    erts_queue_message(rp, rp_locks, mp, msg, c_p->common.id);
+
+    if (rp == real_c_p)
+	rp_locks &= ~c_p_locks;
+    if (rp_locks)
+	erts_smp_proc_unlock(rp, rp_locks);
+
+    erts_proc_dec_refc(rp);
+
+    return 1;
+}
+
+static int
+ms_wait(Process *c_p, Eterm etimeout, int busy)
+{
+    ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
+    ErtsMonotonicTime time, timeout_time;
+    Sint64 ms;
+
+    if (!term_to_Sint64(etimeout, &ms))
+	return 0;
+
+    time = erts_get_monotonic_time(esdp);
+
+    if (ms < 0)
+	timeout_time = time;
+    else
+	timeout_time = time + ERTS_MSEC_TO_MONOTONIC(ms);
+
+    while (time < timeout_time) {
+	if (busy)
+	    erts_thr_yield();
+	else {
+	    ErtsMonotonicTime timeout = timeout_time - time;
+
+#ifdef __WIN32__
+	    Sleep((DWORD) ERTS_MONOTONIC_TO_MSEC(timeout));
+#else
+	    {
+		ErtsMonotonicTime to = ERTS_MONOTONIC_TO_USEC(timeout);
+		struct timeval tv;
+
+		tv.tv_sec = (long) to / (1000*1000);
+		tv.tv_usec = (long) to % (1000*1000);
+
+		select(0, NULL, NULL, NULL, &tv);
+	    }
+#endif
+	}
+
+	time = erts_get_monotonic_time(esdp);
+    }
+    return 1;
+}
+
+#endif /* ERTS_DIRTY_SCHEDULERS */
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index bb77dbd955..befd2989f9 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -38,6 +38,7 @@
 #include "beam_bp.h"
 #include "beam_catches.h"
 #include "erl_thr_progress.h"
+#include "erl_nfunc_sched.h"
 #ifdef HIPE
 #include "hipe_mode_switch.h"
 #include "hipe_bif1.h"
@@ -214,6 +215,7 @@ BeamInstr beam_continue_exit[1];
 BeamInstr* em_call_error_handler;
 BeamInstr* em_apply_bif;
 BeamInstr* em_call_nif;
+BeamInstr* em_call_bif_e;
 
 
 /* NOTE These should be the only variables containing trace instructions.
@@ -2581,7 +2583,7 @@ do {						\
 
  OpCase(bif1_fbsd):
     {
-	Eterm (*bf)(Process*, Eterm*);
+	ErtsBifFunc bf;
 	Eterm tmp_reg[1];
 	Eterm result;
 
@@ -2592,7 +2594,7 @@ do {						\
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	ERTS_CHK_MBUF_SZ(c_p);
-	result = (*bf)(c_p, tmp_reg);
+	result = (*bf)(c_p, tmp_reg, I);
 	ERTS_CHK_MBUF_SZ(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
@@ -2613,19 +2615,19 @@ do {						\
 
  OpCase(bif1_body_bsd):
     {
-	Eterm (*bf)(Process*, Eterm*);
+	ErtsBifFunc bf;
 
 	Eterm tmp_reg[1];
 	Eterm result;
 
 	GetArg1(1, tmp_reg[0]);
-	bf = (BifFunction) Arg(0);
+	bf = (ErtsBifFunc) Arg(0);
 	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	ERTS_CHK_MBUF_SZ(c_p);
-	result = (*bf)(c_p, tmp_reg);
+	result = (*bf)(c_p, tmp_reg, I);
 	ERTS_CHK_MBUF_SZ(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
@@ -2775,17 +2777,17 @@ do {						\
  OpCase(i_bif2_fbssd):
     {
 	Eterm tmp_reg[2];
-	Eterm (*bf)(Process*, Eterm*);
+	ErtsBifFunc bf;
 	Eterm result;
 
 	GetArg2(2, tmp_reg[0], tmp_reg[1]);
-	bf = (BifFunction) Arg(1);
+	bf = (ErtsBifFunc) Arg(1);
 	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	ERTS_CHK_MBUF_SZ(c_p);
-	result = (*bf)(c_p, tmp_reg);
+	result = (*bf)(c_p, tmp_reg, I);
 	ERTS_CHK_MBUF_SZ(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
@@ -2806,15 +2808,15 @@ do {						\
  OpCase(i_bif2_body_bssd):
     {
 	Eterm tmp_reg[2];
-	Eterm (*bf)(Process*, Eterm*);
+	ErtsBifFunc bf;
 	Eterm result;
 
 	GetArg2(1, tmp_reg[0], tmp_reg[1]);
-	bf = (BifFunction) Arg(0);
+	bf = (ErtsBifFunc) Arg(0);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	ERTS_CHK_MBUF_SZ(c_p);
-	result = (*bf)(c_p, tmp_reg);
+	result = (*bf)(c_p, tmp_reg, I);
 	ERTS_CHK_MBUF_SZ(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
@@ -2837,7 +2839,7 @@ do {						\
      */
  OpCase(call_bif_e):
     {
-        Eterm (*bf)(Process*, Eterm*, BeamInstr*);
+	ErtsBifFunc bf;
 	Eterm result;
 	BeamInstr *next;
 	ErlHeapFragment *live_hf_end;
@@ -3644,7 +3646,7 @@ do {						\
 	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	    {
-		Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf;
+		ErtsBifFunc bf = vbf;
 		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 		live_hf_end = c_p->mbuf;
 		ERTS_CHK_MBUF_SZ(c_p);
@@ -5137,6 +5139,7 @@ do {						\
      em_call_error_handler = OpCode(call_error_handler);
      em_apply_bif = OpCode(apply_bif);
      em_call_nif = OpCode(call_nif);
+     em_call_bif_e = OpCode(call_bif_e);
 
      beam_apply[0]             = (BeamInstr) OpCode(i_apply);
      beam_apply[1]             = (BeamInstr) OpCode(normal_exit);
@@ -5342,8 +5345,6 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 
 	I = c_p->i;
 
-	ASSERT(em_call_nif == (BeamInstr *) *I);
-
 	/*
 	 * Set fcalls even though we ignore it, so we don't
 	 * confuse code accessing it...
@@ -5379,10 +5380,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
     }
 
     {
-#ifdef DEBUG
-	Eterm result;
-#endif
-	Eterm arity;
+	int exiting;
 
 	{
 	    /*
@@ -5398,7 +5396,6 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
              *
              * This layout is determined by the NifExport struct
 	     */
-	    BifFunction vbf;
             ErtsCodeMFA *codemfa;
 
 	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
@@ -5406,44 +5403,29 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
             codemfa = erts_code_to_codemfa(I);
 
 	    DTRACE_NIF_ENTRY(c_p, codemfa);
-            /* current and vbf set to please handle_error */
 	    c_p->current = codemfa;
 	    SWAPOUT;
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
-	    arity = codemfa->arity;
 	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 
 	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	    {
-		typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
-		NifF* fp = vbf = (NifF*) I[1];
-		struct enif_environment_t env;
-		ASSERT(!c_p->scheduler_data);
-
-		erts_pre_dirty_nif(esdp, &env, c_p,
-				   (struct erl_module_nif*)I[2]);
-
-#ifdef DEBUG
-		result =
-#else
-		(void)
-#endif
-		    (*fp)(&env, arity, reg);
-
-		erts_post_dirty_nif(&env);
+	    if (em_apply_bif == (BeamInstr *) *I) {
+		exiting = erts_call_dirty_bif(esdp, c_p, I, reg);
+	    }
+	    else {
+		ASSERT(em_call_nif == (BeamInstr *) *I);
+		exiting = erts_call_dirty_nif(esdp, c_p, I, reg);
+	    }
 
-		ASSERT(!is_value(result));
-		ASSERT(c_p->freason == TRAP);
-		ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
+	    ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
 
-		PROCESS_MAIN_CHK_LOCKS(c_p);
-		ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
-		ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-		ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
-		if (env.exiting)
-		    goto do_dirty_schedule;
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	    }
+	    PROCESS_MAIN_CHK_LOCKS(c_p);
+	    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
+	    if (exiting)
+		goto do_dirty_schedule;
+	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 
 	    DTRACE_NIF_RETURN(c_p, codemfa);
 	    ERTS_HOLE_CHECK(c_p);
@@ -5529,10 +5511,15 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
     Eterm* hp;
     Eterm Value = c_p->fvalue;
     Eterm Args = am_true;
-    c_p->i = pc;    /* In case we call erts_exit(). */
 
     ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */
 
+    if (c_p->freason & EXF_RESTORE_NIF) {
+	erts_nif_export_restore_error(c_p, &pc, reg, &bf);
+    }
+
+    c_p->i = pc;    /* In case we call erts_exit(). */
+
     /*
      * Check if we have an arglist for the top level call. If so, this
      * is encoded in Value, so we have to dig out the real Value as well
@@ -5829,6 +5816,18 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) {
  * error terms.)
  */
 
+static void
+save_stacktrace_current(struct StackTrace *s, Process *c_p, ErtsCodeMFA *mfa)
+{
+    NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
+    if (!nep || &nep->exp.info.mfa != mfa)
+	s->current = mfa;
+    else {
+	s->mfa = *mfa;
+	s->current = &s->mfa;
+    }
+}
+
 static void
 save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 		Eterm args) {
@@ -5858,12 +5857,10 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 	&& bf != throw_1 && bf != wrap_error_1 && bf != wrap_error_2
 	&& bf != wrap_exit_1 && bf != wrap_throw_1) {
         int i;
-	int a = 0;
+	int a;
 	for (i = 0; i < BIF_SIZE; i++) {
 	    if (bf == bif_table[i].f || bf == bif_table[i].traced) {
-		Export *ep = bif_export[i];
-		s->current = &ep->info.mfa;
-	        a = bif_table[i].arity;
+		save_stacktrace_current(s, c_p, &bif_export[i]->info.mfa);
 		break;
 	    }
 	}
@@ -5875,9 +5872,11 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 	     * OR it is a NIF called by call_nif where current is also set.
 	     */
 	    ASSERT(c_p->current);
-	    s->current = c_p->current;
-	    a = s->current->arity;
+	    save_stacktrace_current(s, c_p, c_p->current);
 	}
+
+	a = s->current->arity;
+
 	/* Save first stack entry */
 	ASSERT(pc);
 	if (depth > 0) {
@@ -5892,7 +5891,9 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 	s->pc = NULL;
 	args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */
     } else {
-	s->current = c_p->current;
+
+	save_stacktrace_current(s, c_p, c_p->current);
+
         /* 
 	 * For a function_clause error, the arguments are in the beam
 	 * registers, c_p->cp is valid, and c_p->current is set.
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index d886c2985e..7bcb7c196d 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4958,7 +4958,7 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
 Export bif_return_trap_export;
 
 void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a,
-			   Eterm (*bif)(BIF_ALIST_0))
+			   Eterm (*bif)(BIF_ALIST))
 {
     int i;
     sys_memset((void *) ep, 0, sizeof(Export));
@@ -5019,6 +5019,321 @@ void erts_init_bif(void)
     erts_smp_atomic32_init_nob(&msacc, ERTS_MSACC_IS_ENABLED());
 }
 
+/*
+ * Scheduling of BIFs via NifExport...
+ */
+#define ERTS_WANT_NFUNC_SCHED_INTERNALS__
+#include "erl_nfunc_sched.h"
+
+#define ERTS_SCHED_BIF_TRAP_MARKER ((void *) (UWord) 1)
+
+static void
+schedule(Process *c_p, Process *dirty_shadow_proc,
+	 ErtsCodeMFA *mfa, BifFunction *nif, BeamInstr *pc,
+	 ErtsBifFunc dfunc, void *ifunc,
+	 Eterm module, Eterm function,
+	 int argc, Eterm *argv)
+{
+    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
+    (void) erts_nif_export_schedule(c_p, dirty_shadow_proc,
+				    mfa, nif, pc,
+				    (BeamInstr) em_apply_bif,
+				    dfunc, ifunc,
+				    module, function,
+				    argc, argv);
+}
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+static BIF_RETTYPE dirty_bif_result(BIF_ALIST_1)
+{
+    NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P);
+    erts_nif_export_restore(BIF_P, nep);
+    BIF_RET(BIF_ARG_1);
+}
+
+static BIF_RETTYPE dirty_bif_trap(BIF_ALIST)
+{
+    NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P);
+
+    /*
+     * Arity and argument registers already set
+     * correct by call to dirty_bif_trap()...
+     */
+
+    ASSERT(BIF_P->arity == nep->exp.info.mfa.arity);
+
+    erts_nif_export_restore(BIF_P, nep);
+
+    BIF_P->i = (BeamInstr *) nep->func;
+    BIF_P->freason = TRAP;
+    return THE_NON_VALUE;
+}
+
+static BIF_RETTYPE dirty_bif_exception(BIF_ALIST_2)
+{
+    Eterm freason;
+
+    ASSERT(is_small(BIF_ARG_1));
+
+    freason = signed_val(BIF_ARG_1);
+
+    /* Restore orig info for error and clear nif export in handle_error() */
+    freason |= EXF_RESTORE_NIF;
+
+    BIF_P->fvalue = BIF_ARG_2;
+
+    BIF_ERROR(BIF_P, freason);
+}
+
+#endif /* ERTS_DIRTY_SCHEDULERS */
+
+extern BeamInstr* em_call_bif_e;
+static BIF_RETTYPE call_bif(Process *c_p, Eterm *reg, BeamInstr *I);
+
+BIF_RETTYPE
+erts_schedule_bif(Process *proc,
+		  Eterm *argv,
+		  BeamInstr *i,
+		  ErtsBifFunc bif,
+		  ErtsSchedType sched_type,
+		  Eterm mod,
+		  Eterm func,
+		  int argc)
+{
+    Process *c_p, *dirty_shadow_proc;
+    ErtsCodeMFA *mfa;
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
+	dirty_shadow_proc = proc;
+	c_p = proc->next;
+	ASSERT(c_p->common.id == dirty_shadow_proc->common.id);
+	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+    }
+    else
+#endif
+    {
+	dirty_shadow_proc = NULL;
+	c_p = proc;
+    }
+
+    if (!ERTS_PROC_IS_EXITING(c_p)) {
+	Export *exp;
+	BifFunction obif, dbif, ibif;
+	BeamInstr *pc;
+
+	/*
+	 * obif - original bif
+	 * dbif - direct bif
+	 * ibif - indirect bif
+	 */
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+	erts_aint32_t set, mask;
+	mask = (ERTS_PSFLG_DIRTY_CPU_PROC
+		| ERTS_PSFLG_DIRTY_IO_PROC);
+	switch (sched_type) {
+	case ERTS_SCHED_DIRTY_CPU:
+	    set = ERTS_PSFLG_DIRTY_CPU_PROC;
+	    dbif = bif;
+	    ibif = NULL;
+	    break;
+	case ERTS_SCHED_DIRTY_IO:
+	    set = ERTS_PSFLG_DIRTY_IO_PROC;
+	    dbif = bif;
+	    ibif = NULL;
+	    break;
+	case ERTS_SCHED_NORMAL:
+	default:
+	    set = 0;
+	    dbif = call_bif;
+	    ibif = bif;
+	    break;
+	}
+
+	(void) erts_smp_atomic32_read_bset_nob(&c_p->state, mask, set);
+#else
+	dbif = call_bif;
+	ibif = bif;
+#endif
+
+	if (i == NULL) {
+	    ERTS_INTERNAL_ERROR("Missing instruction pointer");
+	    obif = NULL;
+	}
+#ifdef HIPE
+	else if (proc->flags & F_HIPE_MODE) {
+	    /* Pointer to bif export in i */
+	    exp = (Export *) i;
+	    pc = c_p->cp;
+	    obif = (BifFunction) exp->beam[1];
+	    mfa = &exp->info.mfa;
+	}
+#endif
+	else if (em_call_bif_e == (BeamInstr *) *i) {
+	    /* Pointer to bif export in i+1 */
+	    exp = (Export *) i[1];
+	    pc = i;
+	    obif = (BifFunction) exp->beam[1];
+	    mfa = &exp->info.mfa;
+	}
+	else if (em_apply_bif == (BeamInstr *) *i) {
+	    /* Pointer to bif in i+1, and mfa in i-3 */	    
+	    obif = (BifFunction) i[1];
+	    pc = c_p->cp;
+	    mfa = erts_code_to_codemfa(i);
+	}
+	else {
+	    ERTS_INTERNAL_ERROR("erts_schedule_bif() called "
+				"from unexpected instruction");
+	    obif = NULL;
+	}
+	ASSERT(bif);
+
+	if (argc < 0) { /* reschedule original call */
+	    mod = mfa->module;
+	    func = mfa->function;
+	    argc = (int) mfa->arity;
+	}
+
+	schedule(c_p, dirty_shadow_proc, mfa, obif,
+		 pc, dbif, ibif, mod, func, argc, argv);
+    }
+
+    if (dirty_shadow_proc)
+	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+
+    return THE_NON_VALUE;
+}
+
+static BIF_RETTYPE
+call_bif(Process *c_p, Eterm *reg, BeamInstr *I)
+{
+    NifExport *nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I);
+    ErtsBifFunc bif = (ErtsBifFunc) nep->func;
+    BIF_RETTYPE ret;
+
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
+
+    nep->func = ERTS_SCHED_BIF_TRAP_MARKER;
+
+    ASSERT(bif);
+
+    ret = (*bif)(c_p, reg, I);
+
+    if (is_value(ret))
+	erts_nif_export_restore(c_p, nep);
+    else if (c_p->freason != TRAP)
+	c_p->freason |= EXF_RESTORE_NIF; /* restore in handle_error() */
+    else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) {
+	/* BIF did an ordinary trap... */
+	erts_nif_export_restore(c_p, nep);
+    }
+    /* else:
+     *   BIF rescheduled itself using erts_schedule_bif().
+     */
+
+    return ret;
+}
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+int
+erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg)
+{
+    BIF_RETTYPE result;
+    int exiting;
+    Process *dirty_shadow_proc;
+    ErtsBifFunc bf;
+    NifExport *nep;
+#ifdef DEBUG
+    Eterm *c_p_htop;
+    erts_aint32_t state;
+
+    ASSERT(!c_p->scheduler_data);
+    state = erts_smp_atomic32_read_nob(&c_p->state);
+    ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING)
+	   && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)));
+    ASSERT(esdp);
+
+#endif
+
+    nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I);
+    ASSERT(nep == ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p));
+
+    nep->func = ERTS_SCHED_BIF_TRAP_MARKER;
+
+    bf = (ErtsBifFunc) I[1];
+
+    erts_smp_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC
+						  | ERTS_PSFLG_DIRTY_IO_PROC));
+
+    dirty_shadow_proc = erts_make_dirty_shadow_proc(esdp, c_p);
+
+    dirty_shadow_proc->freason = c_p->freason;
+    dirty_shadow_proc->fvalue = c_p->fvalue;
+    dirty_shadow_proc->ftrace = c_p->ftrace;
+    dirty_shadow_proc->cp = c_p->cp;
+    dirty_shadow_proc->i = c_p->i;
+
+#ifdef DEBUG
+    c_p_htop = c_p->htop;
+#endif
+
+    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+
+    result = (*bf)(dirty_shadow_proc, reg, I);
+
+    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+
+    ASSERT(c_p_htop == c_p->htop);
+    ASSERT(dirty_shadow_proc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
+    ASSERT(dirty_shadow_proc->next == c_p);
+
+    exiting = ERTS_PROC_IS_EXITING(c_p);
+
+    if (!exiting) {
+	if (is_value(result))
+	    schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL, dirty_bif_result,
+		     NULL, am_erts_internal, am_dirty_bif_result, 1, &result);
+	else if (dirty_shadow_proc->freason != TRAP) {
+	    Eterm argv[2];
+	    ASSERT(dirty_shadow_proc->freason <= MAX_SMALL);
+	    argv[0] = make_small(dirty_shadow_proc->freason);
+	    argv[1] = dirty_shadow_proc->fvalue;
+	    schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL,
+		     dirty_bif_exception, NULL, am_erts_internal,
+		     am_dirty_bif_exception, 2, argv);
+	}
+	else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) {
+	    /* Dirty BIF did an ordinary trap... */
+	    ASSERT(!(erts_smp_atomic32_read_nob(&c_p->state)
+		     & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)));
+	    schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL,
+		     dirty_bif_trap, (void *) dirty_shadow_proc->i,
+		     am_erts_internal, am_dirty_bif_trap,
+		     dirty_shadow_proc->arity, reg);
+	}
+	/* else:
+	 *   BIF rescheduled itself using erts_schedule_bif().
+	 */
+	c_p->freason = dirty_shadow_proc->freason;
+	c_p->fvalue = dirty_shadow_proc->fvalue;
+	c_p->ftrace = dirty_shadow_proc->ftrace;
+	c_p->cp = dirty_shadow_proc->cp;
+	c_p->i = dirty_shadow_proc->i;
+	c_p->arity = dirty_shadow_proc->arity;
+    }
+
+    erts_flush_dirty_shadow_proc(dirty_shadow_proc);
+
+    return exiting;
+}
+
+#endif /* ERTS_DIRTY_SCHEDULERS */
+
+
 #ifdef HARDDEBUG
 /*
 You'll need this line in bif.tab to be able to use this debug bif
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 0c85e19ef0..ad55b19d0a 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -29,17 +29,35 @@ extern Export *erts_convert_time_unit_trap;
 
 #define BIF_P A__p
 
-#define BIF_ALIST_0 Process* A__p, Eterm* BIF__ARGS
-#define BIF_ALIST_1 Process* A__p, Eterm* BIF__ARGS
-#define BIF_ALIST_2 Process* A__p, Eterm* BIF__ARGS
-#define BIF_ALIST_3 Process* A__p, Eterm* BIF__ARGS
-#define BIF_ALIST_4 Process* A__p, Eterm* BIF__ARGS
+#define BIF_ALIST Process* A__p, Eterm* BIF__ARGS, BeamInstr *A__I
+#define BIF_CALL_ARGS A__p, BIF__ARGS, A__I
+
+#define BIF_ALIST_0 BIF_ALIST
+#define BIF_ALIST_1 BIF_ALIST
+#define BIF_ALIST_2 BIF_ALIST
+#define BIF_ALIST_3 BIF_ALIST
+#define BIF_ALIST_4 BIF_ALIST
 
 #define BIF_ARG_1  (BIF__ARGS[0])
 #define BIF_ARG_2  (BIF__ARGS[1])
 #define BIF_ARG_3  (BIF__ARGS[2])
 #define BIF_ARG_4  (BIF__ARGS[3])
 
+#define BIF_I A__I
+
+/* NBIF_* is for bif calls from native code... */
+
+#define NBIF_ALIST Process* A__p, Eterm* BIF__ARGS
+#define NBIF_CALL_ARGS A__p, BIF__ARGS
+
+#define NBIF_ALIST_0 NBIF_ALIST
+#define NBIF_ALIST_1 NBIF_ALIST
+#define NBIF_ALIST_2 NBIF_ALIST
+#define NBIF_ALIST_3 NBIF_ALIST
+#define NBIF_ALIST_4 NBIF_ALIST
+
+typedef BIF_RETTYPE (*ErtsBifFunc)(BIF_ALIST);
+
 #define ERTS_IS_PROC_OUT_OF_REDS(p)		\
     ((p)->fcalls > 0				\
      ? 0					\
@@ -480,6 +498,49 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
 					 Eterm args[],
 					 int nargs);
 
+typedef enum {
+    ERTS_SCHED_NORMAL,
+    ERTS_SCHED_DIRTY_CPU,
+    ERTS_SCHED_DIRTY_IO
+} ErtsSchedType;
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+int erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p,
+			BeamInstr *I, Eterm *reg);
+#endif
+
+BIF_RETTYPE
+erts_schedule_bif(Process *proc,
+		  Eterm *argv,
+		  BeamInstr *i,
+		  ErtsBifFunc dbf,
+		  ErtsSchedType sched_type,
+		  Eterm mod,
+		  Eterm func,
+		  int argc);
+
+ERTS_GLB_INLINE BIF_RETTYPE
+erts_reschedule_bif(Process *proc,
+		    Eterm *argv,
+		    BeamInstr *i,
+		    ErtsBifFunc dbf,
+		    ErtsSchedType sched_type);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE BIF_RETTYPE
+erts_reschedule_bif(Process *proc,
+		    Eterm *argv,
+		    BeamInstr *i,
+		    ErtsBifFunc dbf,
+		    ErtsSchedType sched_type)
+{
+    return erts_schedule_bif(proc, argv, i, dbf, sched_type,
+			     THE_NON_VALUE, THE_NON_VALUE, -1);
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
 #ifdef ERL_WANT_HIPE_BIF_WRAPPER__
 
 #ifndef HIPE
@@ -510,16 +571,16 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
 
 
 #define HIPE_WRAPPER_BIF_DISABLE_GC(BIF_NAME, ARITY)			\
-BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p,	\
-						     Eterm* args);	\
-BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p,	\
-						     Eterm* args)	\
+BIF_RETTYPE								\
+nbif_impl_hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (NBIF_ALIST);		\
+BIF_RETTYPE								\
+nbif_impl_hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (NBIF_ALIST)		\
 {									\
     BIF_RETTYPE  res;							\
-    hipe_reserve_beam_trap_frame(c_p, args, ARITY);			\
-    res =  BIF_NAME ## _ ## ARITY (c_p, args);				\
-    if (is_value(res) || c_p->freason != TRAP) {			\
-	hipe_unreserve_beam_trap_frame(c_p);				\
+    hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, ARITY);		\
+    res = nbif_impl_ ## BIF_NAME ## _ ## ARITY (NBIF_CALL_ARGS);	\
+    if (is_value(res) || BIF_P->freason != TRAP) {			\
+	hipe_unreserve_beam_trap_frame(BIF_P);				\
     }									\
     return res;								\
 }
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 32600f4338..f461da913e 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -420,6 +420,9 @@ bif erts_debug:set_internal_state/2
 bif erts_debug:display/1
 bif erts_debug:dist_ext_to_term/2
 bif erts_debug:instructions/0
+bif erts_debug:dirty_cpu/2
+bif erts_debug:dirty_io/2
+bif erts_debug:dirty/3
 
 #
 # Monitor testing bif's...
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 009d2b72d3..4659cb5a7b 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -93,6 +93,9 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
 #ifdef USE_THREADS
 #if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
 				     " [ds:%beu:%beu:%beu]"
+#endif
+#if defined(ERTS_DIRTY_SCHEDULERS_TEST)
+				     " [dirty-schedulers-TEST]"
 #endif
 				     " [async-threads:%d]"
 #endif
diff --git a/erts/emulator/beam/erl_dirty_bif.tab b/erts/emulator/beam/erl_dirty_bif.tab
new file mode 100644
index 0000000000..69421dcfcc
--- /dev/null
+++ b/erts/emulator/beam/erl_dirty_bif.tab
@@ -0,0 +1,82 @@
+#
+# %CopyrightBegin%
+#
+# Copyright Ericsson AB 2016. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# %CopyrightEnd%
+#
+
+#
+# Static declaration of BIFs that should execute on dirty schedulers.
+#
+# <dirty-bif-decl> ::= <type> <bif>
+# <bif> ::= <module> ":" <name> "/" <arity>
+# <type> ::= dirty-cpu | dirty-io | dirty-cpu-test | dirty-io-test
+#
+# When dirty scheduler support is available, a BIF declared with the
+# 'dirty-cpu' type will unconditionally execute on a dirty CPU scheduler,
+# and a BIF declared with the type 'dirty-io' will unconditionally execute
+# on a dirty IO scheduler. When dirty scheduler support is not available
+# all BIFs will of course execute on normal schedulers.
+#
+# When the emulator has been configured with the debug option
+# '--enable-dirty-schedulers-test', BIFs with the types 'dirty-cpu-test',
+# and 'dirty-io-test' will unconditionally execute on dirty schedulers.
+# When this debug option has not been enabled, these BIFs will be executed
+# on normal schedulers.
+#
+# BIFs marked as 'ubif' in ./bif.tab will be ignored, i.e., will always
+# execute on normal schedulers.
+#
+
+# --- Dirty BIFs ---
+
+dirty-cpu erts_debug:dirty_cpu/2
+dirty-io erts_debug:dirty_io/2
+
+# --- TEST of Dirty BIF functionality ---
+#  Functions below will execute on dirty schedulers when emulator has
+#  been configured for testing dirty schedulers. This is used for test
+#  and debug purposes only. We really do *not* want to execute these
+#  on dirty schedulers on a real system.
+
+dirty-cpu-test erlang:'++'/2
+dirty-cpu-test erlang:append/2
+dirty-cpu-test erlang:'--'/2
+dirty-cpu-test erlang:subtract/2
+dirty-cpu-test erlang:iolist_size/1
+dirty-cpu-test erlang:make_tuple/2
+dirty-cpu-test erlang:make_tuple/3
+dirty-cpu-test erlang:append_element/2
+dirty-cpu-test erlang:insert_element/3
+dirty-cpu-test erlang:delete_element/2
+dirty-cpu-test erlang:atom_to_list/1
+dirty-cpu-test erlang:list_to_atom/1
+dirty-cpu-test erlang:list_to_existing_atom/1
+dirty-cpu-test erlang:integer_to_list/1
+dirty-cpu-test erlang:string_to_integer/1
+dirty-cpu-test erlang:list_to_integer/1
+dirty-cpu-test erlang:list_to_integer/2
+dirty-cpu-test erlang:float_to_list/1
+dirty-cpu-test erlang:float_to_list/2
+dirty-cpu-test erlang:float_to_binary/1
+dirty-cpu-test erlang:float_to_binary/2
+dirty-cpu-test erlang:string_to_float/1
+dirty-cpu-test erlang:list_to_float/1
+dirty-cpu-test erlang:binary_to_float/1
+dirty-cpu-test erlang:tuple_to_list/1
+dirty-cpu-test erlang:list_to_tuple/1
+dirty-cpu-test erlang:display/1
+dirty-cpu-test erlang:display_string/1
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 643b46c861..6093b0cf39 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -42,6 +42,7 @@
 #include "dtrace-wrapper.h"
 #include "erl_bif_unique.h"
 #include "dist.h"
+#include "erl_nfunc_sched.h"
 
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20
@@ -2423,17 +2424,10 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
     }
 
     /*
-     * If a NIF has saved arguments, they need to be added
+     * If a NIF or BIF has saved arguments, they need to be added
      */
-    if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) {
-	Eterm* argv;
-	int argc;
-	if (erts_setup_nif_gc(p, &argv, &argc)) {
-	    roots[n].v = argv;
-	    roots[n].sz = argc;
-	    n++;
-	}
-    }
+    if (erts_setup_nif_export_rootset(p, &roots[n].v, &roots[n].sz))
+	n++;
 
     ASSERT(n <= rootset->size);
 
@@ -2985,6 +2979,8 @@ static void ERTS_INLINE
 offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size,
 	       Eterm* objv, int nobj)
 {
+    Eterm *v;
+    Uint sz;
     if (p->dictionary)  {
 	offset_heap(ERTS_PD_START(p->dictionary),
 		    ERTS_PD_SIZE(p->dictionary),
@@ -3005,12 +3001,8 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size,
 	offset_heap_ptr(objv, nobj, offs, area, area_size);
     }
     offset_off_heap(p, offs, area, area_size);
-    if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) {
-	Eterm* argv;
-	int argc;
-	if (erts_setup_nif_gc(p, &argv, &argc))
-	    offset_heap_ptr(argv, argc, offs, area, area_size);
-    }
+    if (erts_setup_nif_export_rootset(p, &v, &sz))
+	offset_heap_ptr(v, sz, offs, area, area_size);
 }
 
 static void
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index 7ddf49937f..66bb55e6c8 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -137,8 +137,8 @@ void erts_msacc_init_thread(char *type, int id, int managed) {
 void erts_msacc_set_bif_state(ErtsMsAcc *__erts_msacc_cache, Eterm mod, void *fn) {
 
 #ifdef ERTS_MSACC_EXTENDED_BIFS
-#define BIF_LIST(Mod,Func,Arity,FuncAddr,Num)                           \
-    if (fn == &FuncAddr) {                                             \
+#define BIF_LIST(Mod,Func,Arity,BifFuncAddr,FuncAddr,Num)	       \
+    if (fn == &BifFuncAddr) {                                             \
         ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATIC_STATE_COUNT + Num); \
     } else
 #include "erl_bif_list.h"
diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h
index 9ef86a1293..d64ef8c8b9 100644
--- a/erts/emulator/beam/erl_msacc.h
+++ b/erts/emulator/beam/erl_msacc.h
@@ -122,7 +122,7 @@ static char *erts_msacc_states[] = {
     "sleep",
     "timers"
 #ifdef ERTS_MSACC_EXTENDED_BIFS
-#define BIF_LIST(Mod,Func,Arity,FuncAddr,Num)   \
+#define BIF_LIST(Mod,Func,Arity,BifFuncAddr,FuncAddr,Num)	\
         ,"bif_" #Mod "_" #Func "_" #Arity
 #include "erl_bif_list.h"
 #undef BIF_LIST
diff --git a/erts/emulator/beam/erl_nfunc_sched.c b/erts/emulator/beam/erl_nfunc_sched.c
new file mode 100644
index 0000000000..0333545255
--- /dev/null
+++ b/erts/emulator/beam/erl_nfunc_sched.c
@@ -0,0 +1,144 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#define ERTS_WANT_NFUNC_SCHED_INTERNALS__
+
+#include "global.h"
+#include "erl_process.h"
+#include "bif.h"
+#include "erl_nfunc_sched.h"
+
+NifExport *
+erts_new_proc_nif_export(Process *c_p, int argc)
+{
+    size_t size;
+    int i;
+    NifExport *nep, *old_nep;
+
+    size = sizeof(NifExport) + (argc-1)*sizeof(Eterm);
+    nep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, size);
+
+    for (i = 0; i < ERTS_NUM_CODE_IX; i++)
+	nep->exp.addressv[i] = &nep->exp.beam[0];
+
+    nep->argc = -1; /* unused marker */
+    nep->argv_size = argc;
+    old_nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(c_p, nep);
+    if (old_nep)
+	erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, old_nep);
+    return nep;
+}
+
+void
+erts_destroy_nif_export(Process *p)
+{
+    NifExport *nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL);
+    if (nep) {
+	if (nep->m)
+	    erts_nif_export_cleanup_nif_mod(nep);
+	erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, nep);
+    }
+}
+
+NifExport *
+erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc,
+			 ErtsCodeMFA *mfa, void *nif, BeamInstr *pc,
+			 BeamInstr instr,
+			 void *dfunc, void *ifunc,
+			 Eterm mod, Eterm func,
+			 int argc, const Eterm *argv)
+{
+    Process *used_proc;
+    ErtsSchedulerData *esdp;
+    Eterm* reg;
+    NifExport* nep;
+    int i;
+
+    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
+		       & ERTS_PROC_LOCK_MAIN);
+
+    if (dirty_shadow_proc) {
+	esdp = erts_get_scheduler_data();
+	ASSERT(esdp && ERTS_SCHEDULER_IS_DIRTY(esdp));
+
+	used_proc = dirty_shadow_proc;
+    }
+    else {
+	esdp = erts_proc_sched_data(c_p);
+	ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp));
+
+	used_proc = c_p;
+	ERTS_VBUMP_ALL_REDS(c_p);
+    }
+
+    reg = esdp->x_reg_array;
+
+    if (mfa)
+	nep = erts_get_proc_nif_export(c_p, (int) mfa->arity);
+    else {
+	/* If no mfa, this is not the first schedule... */
+	nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
+	ASSERT(nep && nep->argc >= 0);
+    }
+
+    if (nep->argc < 0) {
+	/*
+	 * First schedule; save things that might
+	 * need to be restored...
+	 */
+	for (i = 0; i < (int) mfa->arity; i++)
+	    nep->argv[i] = reg[i];
+	nep->pc = pc;
+	nep->cp = c_p->cp;
+	ASSERT(nif);
+	nep->nif = nif;
+	nep->mfa = mfa;
+	nep->current = c_p->current;
+	ASSERT(argc >= 0);
+	nep->argc = (int) mfa->arity;
+	nep->m = NULL;
+
+	ASSERT(!erts_check_nif_export_in_area(c_p,
+					      (char *) nep,
+					      (sizeof(NifExport)
+					       + (sizeof(Eterm)
+						  *(nep->argc-1)))));
+    }
+    /* Copy new arguments into register array if necessary... */
+    if (reg != argv) {
+	for (i = 0; i < argc; i++)
+	    reg[i] = argv[i];
+    }
+    ASSERT(is_atom(mod) && is_atom(func));
+    nep->exp.info.mfa.module = mod;
+    nep->exp.info.mfa.function = func;
+    nep->exp.info.mfa.arity = (Uint) argc;
+    nep->exp.beam[0] = (BeamInstr) instr; /* call_nif || apply_bif */
+    nep->exp.beam[1] = (BeamInstr) dfunc;
+    nep->func = ifunc;
+    used_proc->arity = argc;
+    used_proc->freason = TRAP;
+    used_proc->i = (BeamInstr*) nep->exp.addressv[0];
+    return nep;
+}
diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h
new file mode 100644
index 0000000000..72de7f0eb5
--- /dev/null
+++ b/erts/emulator/beam/erl_nfunc_sched.h
@@ -0,0 +1,289 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifndef ERL_NFUNC_SCHED_H__
+#define ERL_NFUNC_SCHED_H__
+
+#include "erl_process.h"
+#include "bif.h"
+
+/*
+ * NIF exports need a few more items than the Export struct provides,
+ * including the erl_module_nif* and a NIF function pointer, so the
+ * NifExport below adds those. The Export member must be first in the
+ * struct. A number of values are stored for error handling purposes
+ * only.
+ *
+ * 'argc' is >= 0 when NifExport is in use, and < 0 when not.
+ */
+
+typedef struct {
+    Export exp;
+    struct erl_module_nif* m; /* NIF module, or NULL if BIF */
+    void *func;		/* Indirect NIF or BIF to execute (may be unused) */
+    ErtsCodeMFA *current;/* Current as set when originally called */
+    /* --- The following is only used on error --- */
+    BeamInstr *pc;	/* Program counter */
+    BeamInstr *cp;	/* Continuation pointer */
+    void *nif;		/* Original NIF/BIF call */
+    ErtsCodeMFA *mfa;	/* MFA of original call */
+    int argc;		/* Number of arguments in original call */
+    int argv_size;	/* Allocated size of argv */
+    Eterm argv[1];	/* Saved arguments from the original call */
+} NifExport;
+
+NifExport *erts_new_proc_nif_export(Process *c_p, int argc);
+void erts_destroy_nif_export(Process *p);
+NifExport *erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc,
+				    ErtsCodeMFA *mfa, void *nif, BeamInstr *pc,
+				    BeamInstr instr,
+				    void *dfunc, void *ifunc,
+				    Eterm mod, Eterm func,
+				    int argc, const Eterm *argv);
+void erts_nif_export_cleanup_nif_mod(NifExport *ep); /* erl_nif.c */
+ERTS_GLB_INLINE NifExport *erts_get_proc_nif_export(Process *c_p, int extra);
+ERTS_GLB_INLINE int erts_setup_nif_export_rootset(Process* proc, Eterm** objv,
+						  Uint* nobj);
+ERTS_GLB_INLINE int erts_check_nif_export_in_area(Process *p,
+						  char *start, Uint size);
+ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep);
+ERTS_GLB_INLINE void erts_nif_export_restore_error(Process* c_p, BeamInstr **pc,
+						   Eterm *reg, void **nif);
+ERTS_GLB_INLINE Process *erts_proc_shadow2real(Process *c_p);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE NifExport *
+erts_get_proc_nif_export(Process *c_p, int argc)
+{
+    NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
+    if (!nep || (nep->argc < 0 && nep->argv_size < argc))
+	return erts_new_proc_nif_export(c_p, argc);
+    return nep;
+}
+
+/*
+ * If a process has saved arguments, they need to be part of the GC
+ * rootset. The function below is called from setup_rootset() in
+ * erl_gc.c. Any exception term saved in the NifExport is also made
+ * part of the GC rootset here; it always resides in rootset[0].
+ */
+ERTS_GLB_INLINE int
+erts_setup_nif_export_rootset(Process* proc, Eterm** objv, Uint* nobj)
+{
+    NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
+
+    if (!ep || ep->argc <= 0)
+	return 0;
+
+    *objv = ep->argv;
+    *nobj = ep->argc;
+    return 1;
+}
+
+/*
+ * Check if nif export points into code area...
+ */
+ERTS_GLB_INLINE int
+erts_check_nif_export_in_area(Process *p, char *start, Uint size)
+{
+    NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p);
+    if (!nep || nep->argc < 0)
+	return 0;
+    if (ErtsInArea(nep->pc, start, size))
+	return 1;
+    if (ErtsInArea(nep->cp, start, size))
+	return 1;
+    if (ErtsInArea(nep->mfa, start, size))
+	return 1;
+    if (ErtsInArea(nep->current, start, size))
+	return 1;
+    return 0;
+}
+
+ERTS_GLB_INLINE void
+erts_nif_export_restore(Process *c_p, NifExport *ep)
+{
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
+    ERTS_SMP_LC_ASSERT(!(c_p->static_flags
+			 & ERTS_STC_FLG_SHADOW_PROC));
+    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
+		       & ERTS_PROC_LOCK_MAIN);
+
+    c_p->current = ep->current;
+    ep->argc = -1; /* Unused nif-export marker... */
+}
+
+ERTS_GLB_INLINE void
+erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, Eterm *reg,
+			      void **nif)
+{
+    NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
+    int ix;
+
+    ASSERT(nep);
+    *pc = nep->pc;
+    c_p->cp = nep->cp;
+    *nif = nep->nif;
+    for (ix = 0; ix < nep->argc; ix++)
+	reg[ix] = nep->argv[ix];
+    erts_nif_export_restore(c_p, nep);
+}
+
+ERTS_GLB_INLINE Process *
+erts_proc_shadow2real(Process *c_p)
+{
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
+	Process *real_c_p = c_p->next;
+	ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
+	ASSERT(real_c_p->common.id == c_p->common.id);
+	return real_c_p;
+    }
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
+#endif
+    return c_p;
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#endif /* ERL_NFUNC_SCHED_H__ */
+
+#if defined(ERTS_WANT_NFUNC_SCHED_INTERNALS__) && !defined(ERTS_NFUNC_SCHED_INTERNALS__)
+#define ERTS_NFUNC_SCHED_INTERNALS__
+
+#define ERTS_I_BEAM_OP_TO_NIF_EXPORT(I)					\
+    (ASSERT(BeamOp(op_apply_bif) == (BeamInstr *) (*(I))		\
+	    || BeamOp(op_call_nif) == (BeamInstr *) (*(I))),		\
+     ((NifExport *) (((char *) (I)) - offsetof(NifExport, exp.beam[0]))))
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+#include "erl_message.h"
+#include <stddef.h>
+
+ERTS_GLB_INLINE void erts_flush_dirty_shadow_proc(Process *sproc);
+ERTS_GLB_INLINE void erts_cache_dirty_shadow_proc(Process *sproc);
+ERTS_GLB_INLINE Process *erts_make_dirty_shadow_proc(ErtsSchedulerData *esdp,
+						     Process *c_p);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE void
+erts_flush_dirty_shadow_proc(Process *sproc)
+{
+    Process *c_p = sproc->next;
+
+    ASSERT(sproc->common.id == c_p->common.id);
+    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
+		       & ERTS_PROC_LOCK_MAIN);
+
+    ASSERT(c_p->stop == sproc->stop);
+    ASSERT(c_p->hend == sproc->hend);
+    ASSERT(c_p->heap == sproc->heap);
+    ASSERT(c_p->abandoned_heap == sproc->abandoned_heap);
+    ASSERT(c_p->heap_sz == sproc->heap_sz);
+    ASSERT(c_p->high_water == sproc->high_water);
+    ASSERT(c_p->old_heap == sproc->old_heap);
+    ASSERT(c_p->old_htop == sproc->old_htop);
+    ASSERT(c_p->old_hend == sproc->old_hend);
+
+    ASSERT(c_p->htop <= sproc->htop && sproc->htop <= c_p->stop);
+
+    c_p->htop = sproc->htop;
+
+    if (!c_p->mbuf)
+	c_p->mbuf = sproc->mbuf;
+    else if (sproc->mbuf) {
+	ErlHeapFragment *bp;
+	for (bp = sproc->mbuf; bp->next; bp = bp->next)
+	    ASSERT(!bp->off_heap.first);
+	bp->next = c_p->mbuf;
+	c_p->mbuf = sproc->mbuf;
+    }
+
+    c_p->mbuf_sz += sproc->mbuf_sz;
+
+    if (!c_p->off_heap.first)
+	c_p->off_heap.first = sproc->off_heap.first;
+    else if (sproc->off_heap.first) {
+	struct erl_off_heap_header *ohhp;
+	for (ohhp = sproc->off_heap.first; ohhp->next; ohhp = ohhp->next)
+	    ;
+	ohhp->next = c_p->off_heap.first;
+	c_p->off_heap.first = sproc->off_heap.first;
+    }
+
+    c_p->off_heap.overhead += sproc->off_heap.overhead;
+}
+
+ERTS_GLB_INLINE void
+erts_cache_dirty_shadow_proc(Process *sproc)
+{
+    Process *c_p = sproc->next;
+    ASSERT(c_p);
+    ASSERT(sproc->common.id == c_p->common.id);
+    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
+		       & ERTS_PROC_LOCK_MAIN);
+
+    sproc->htop = c_p->htop;
+    sproc->stop = c_p->stop;
+    sproc->hend = c_p->hend;
+    sproc->heap = c_p->heap;
+    sproc->abandoned_heap = c_p->abandoned_heap;
+    sproc->heap_sz = c_p->heap_sz;
+    sproc->high_water = c_p->high_water;
+    sproc->old_hend = c_p->old_hend;
+    sproc->old_htop = c_p->old_htop;
+    sproc->old_heap = c_p->old_heap;
+    sproc->mbuf = NULL;
+    sproc->mbuf_sz = 0;
+    ERTS_INIT_OFF_HEAP(&sproc->off_heap);
+}
+
+ERTS_GLB_INLINE Process *
+erts_make_dirty_shadow_proc(ErtsSchedulerData *esdp, Process *c_p)
+{
+    Process *sproc;
+
+    ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp));
+
+    sproc = esdp->dirty_shadow_process;
+    ASSERT(sproc);
+    ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
+    ASSERT(erts_smp_atomic32_read_nob(&sproc->state)
+	   == (ERTS_PSFLG_ACTIVE
+	       | ERTS_PSFLG_DIRTY_RUNNING
+	       | ERTS_PSFLG_PROXY));
+
+    sproc->next = c_p;
+    sproc->common.id = c_p->common.id;
+
+    erts_cache_dirty_shadow_proc(sproc);
+
+    return sproc;
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#endif /* ERTS_DIRTY_SCHEDULERS */
+
+#endif /* defined(ERTS_WANT_NFUNC_SCHED_INTERNALS__) && !defined(ERTS_NFUNC_SCHED_INTERNALS__) */
+
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 5499512dd1..ee4b22f7ca 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -55,6 +55,9 @@
 #include "dtrace-wrapper.h"
 #include "erl_process.h"
 #include "erl_bif_unique.h"
+#undef ERTS_WANT_NFUNC_SCHED_INTERNALS__
+#define ERTS_WANT_NFUNC_SCHED_INTERNALS__
+#include "erl_nfunc_sched.h"
 #if defined(USE_DYNAMIC_TRACE) && (defined(USE_DTRACE) || defined(USE_SYSTEMTAP))
 #define HAVE_USE_DTRACE 1
 #endif
@@ -79,8 +82,11 @@ struct erl_module_nif {
     ErlNifFunc _funcs_copy_[1];  /* only used for old libs */
 };
 
+typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]);
+
 #ifdef DEBUG
 #  define READONLY_CHECK
+#  define ERTS_DBG_NIF_NOT_SCHED_MARKER ((void *) (UWord) 1)
 #endif
 #ifdef READONLY_CHECK
 #  define ADD_READONLY_CHECK(ENV,PTR,SIZE) add_readonly_check(ENV,PTR,SIZE)
@@ -219,38 +225,6 @@ static void cache_env(ErlNifEnv* env);
 static void full_flush_env(ErlNifEnv *env);
 static void flush_env(ErlNifEnv* env);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-void erts_pre_dirty_nif(ErtsSchedulerData *esdp,
-			ErlNifEnv* env, Process* p,
-			struct erl_module_nif* mod_nif)
-{
-    Process *sproc;
-#ifdef DEBUG
-    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
-
-    ASSERT(!p->scheduler_data);
-    ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING)
-	   && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)));
-    ASSERT(esdp);
-#endif
-
-    erts_pre_nif(env, p, mod_nif, NULL);
-
-    sproc = esdp->dirty_shadow_process;
-    ASSERT(sproc);
-    ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
-    ASSERT(erts_smp_atomic32_read_nob(&sproc->state)
-	   == (ERTS_PSFLG_ACTIVE
-	       | ERTS_PSFLG_DIRTY_RUNNING
-	       | ERTS_PSFLG_PROXY));
-
-    sproc->next = p;
-    sproc->common.id = p->common.id;
-    env->proc = sproc;
-    full_cache_env(env);
-}
-#endif
-
 /* Temporary object header, auto-deallocated when NIF returns
  * or when independent environment is cleared.
  */
@@ -278,115 +252,158 @@ void erts_post_nif(ErlNifEnv* env)
     env->exiting = ERTS_PROC_IS_EXITING(env->proc);
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-void erts_post_dirty_nif(ErlNifEnv* env)
+
+/*
+ * Initialize a NifExport struct. Create it if needed and store it in the
+ * proc. The direct_fp function is what will be invoked by op_call_nif, and
+ * the indirect_fp function, if not NULL, is what the direct_fp function
+ * will call. If the allocated NifExport isn't enough to hold all of argv,
+ * allocate a larger one. Save 'current' and registers if first time this
+ * call is scheduled.
+ */
+
+static ERTS_INLINE ERL_NIF_TERM
+schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp,
+	 Eterm mod, Eterm func_name, int argc, const ERL_NIF_TERM argv[])
 {
-    Process *c_p;
-    ASSERT(env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
-    ASSERT(env->proc->next);
-    erts_unblock_fpe(env->fpe_was_unmasked);
-    full_flush_env(env);
-    free_tmp_objs(env);
-    c_p = env->proc->next;
-    env->exiting = ERTS_PROC_IS_EXITING(c_p);
-    ERTS_VBUMP_ALL_REDS(c_p);
+    Export *exp;
+    NifExport *ep;
+    Process *c_p, *dirty_shadow_proc;
+
+    execution_state(env, &c_p, NULL);
+    if (c_p == env->proc)
+	dirty_shadow_proc = NULL;
+    else
+	dirty_shadow_proc = env->proc;
+
+    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
+
+    exp = ErtsContainerStruct(c_p->current, Export, info.mfa);
+
+    ep = erts_nif_export_schedule(c_p, dirty_shadow_proc,
+				  c_p->current,
+				  (BifFunction) exp->beam[1],
+				  c_p->cp,
+				  (BeamInstr) em_call_nif,
+				  direct_fp, indirect_fp,
+				  mod, func_name,
+				  argc, (const Eterm *) argv);
+    if (!ep->m) {
+	/* First time this call is scheduled... */
+	erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1);
+	ep->m = env->mod_nif;
+    }
+    return (ERL_NIF_TERM) THE_NON_VALUE;
 }
-#endif
 
-static void full_flush_env(ErlNifEnv* env)
-{
 #ifdef ERTS_DIRTY_SCHEDULERS
-    if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
-	/* Dirty nif call using shadow process struct */
-	Process *c_p = env->proc->next;
-
-	ASSERT(is_scheduler() < 0);
-	ASSERT(env->proc->common.id == c_p->common.id);
-	ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
-			   & ERTS_PROC_LOCK_MAIN);
-
-	if (!env->heap_frag) {
-	    ASSERT(env->hp_end == HEAP_LIMIT(c_p));
-	    ASSERT(env->hp >= HEAP_TOP(c_p));
-	    ASSERT(env->hp <= HEAP_LIMIT(c_p));
-	    HEAP_TOP(c_p) = env->hp;
+
+static ERL_NIF_TERM dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+
+int
+erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg)
+{
+    int exiting;
+    ERL_NIF_TERM *argv = (ERL_NIF_TERM *) reg;
+    NifExport *nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I);
+    ErtsCodeMFA *codemfa = erts_code_to_codemfa(I);
+    NativeFunPtr dirty_nif = (NativeFunPtr) I[1];
+    ErlNifEnv env;
+    ERL_NIF_TERM result;
+#ifdef DEBUG
+    erts_aint32_t state = erts_smp_atomic32_read_nob(&c_p->state);
+
+    ASSERT(nep == ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p));
+
+    ASSERT(!c_p->scheduler_data);
+    ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING)
+	&& !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)));
+    ASSERT(esdp);
+
+    nep->func = ERTS_DBG_NIF_NOT_SCHED_MARKER;
+#endif
+
+    erts_pre_nif(&env, c_p, nep->m, NULL);
+
+    env.proc = erts_make_dirty_shadow_proc(esdp, c_p);
+
+    env.proc->freason = EXC_NULL;
+    env.proc->fvalue = NIL;
+    env.proc->ftrace = NIL;
+    env.proc->i = c_p->i;
+
+    ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p)));
+
+    erts_smp_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC
+						   | ERTS_PSFLG_DIRTY_IO_PROC));
+
+    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+
+    result = (*dirty_nif)(&env, codemfa->arity, argv); /* Call dirty NIF */
+
+    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+
+    ASSERT(env.proc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
+    ASSERT(env.proc->next == c_p);
+
+    exiting = ERTS_PROC_IS_EXITING(c_p);
+
+    if (!exiting) {
+	if (env.exception_thrown) {
+	schedule_exception:
+	    schedule(&env, dirty_nif_exception, NULL,
+		     am_erts_internal, am_dirty_nif_exception,
+		     1, &env.proc->fvalue);
+	}
+	else if (is_value(result)) {
+	    schedule(&env, dirty_nif_finalizer, NULL,
+		     am_erts_internal, am_dirty_nif_finalizer,
+		     1, &result);
+	}
+	else if (env.proc->freason != TRAP) { /* user returned garbage... */
+	    ERTS_DECL_AM(badreturn);
+	    (void) enif_raise_exception(&env, AM_badreturn);
+	    goto schedule_exception;
 	}
 	else {
-	    Uint usz;
-	    ASSERT(env->hp_end != HEAP_LIMIT(c_p));
-	    ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);
-
-	    HEAP_TOP(c_p) = HEAP_TOP(env->proc);
-	    usz = env->hp - env->heap_frag->mem;
-	    env->proc->mbuf_sz += usz - env->heap_frag->used_size;
-	    env->heap_frag->used_size = usz;
-
-	    ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
-
-	    if (c_p->mbuf) {
-		ErlHeapFragment *bp;
-		for (bp = env->proc->mbuf; bp->next; bp = bp->next)
-		    ;
-		bp->next = c_p->mbuf;
-	    }
+	    /* Rescheduled by dirty NIF call... */
+	    ASSERT(nep->func != ERTS_DBG_NIF_NOT_SCHED_MARKER);
+	}
+	c_p->i = env.proc->i;
+	c_p->arity = env.proc->arity;
+    }
 
-	    c_p->mbuf = env->proc->mbuf;
-	    c_p->mbuf_sz += env->proc->mbuf_sz;
+#ifdef DEBUG
+    if (nep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER)
+	nep->func = NULL;
+#endif
 
-	}
+    erts_unblock_fpe(env.fpe_was_unmasked);
+    full_flush_env(&env);
+    free_tmp_objs(&env);
 
-	if (!c_p->off_heap.first)
-	    c_p->off_heap.first = env->proc->off_heap.first;
-	else if (env->proc->off_heap.first) {
-	    struct erl_off_heap_header *ohhp;
-	    for (ohhp = env->proc->off_heap.first; ohhp->next; ohhp = ohhp->next)
-		;
-	    ohhp->next = c_p->off_heap.first;
-	    c_p->off_heap.first = env->proc->off_heap.first;
-	}
-	c_p->off_heap.overhead += env->proc->off_heap.overhead;
+    return exiting;
+}
 
-	return;
-    }
 #endif
 
+static void full_flush_env(ErlNifEnv* env)
+{
     flush_env(env);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)
+	/* Dirty nif call using shadow process struct */
+	erts_flush_dirty_shadow_proc(env->proc);
+#endif
 }
 
 static void full_cache_env(ErlNifEnv* env)
 {    
 #ifdef ERTS_DIRTY_SCHEDULERS
-    if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
-	/* Dirty nif call using shadow process struct */
-	Process *sproc = env->proc;
-	Process *c_p = sproc->next;
-	ASSERT(c_p);
-	ASSERT(is_scheduler() < 0);
-	ASSERT(env->proc->common.id == c_p->common.id);
-	ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
-			   & ERTS_PROC_LOCK_MAIN);
-
-	sproc->htop = c_p->htop;
-	sproc->stop = c_p->stop;
-	sproc->hend = c_p->hend;
-	sproc->heap = c_p->heap;
-	sproc->abandoned_heap = c_p->abandoned_heap;
-	sproc->heap_sz = c_p->heap_sz;
-	sproc->high_water = c_p->high_water;
-	sproc->old_hend = c_p->old_hend;
-	sproc->old_htop = c_p->old_htop;
-	sproc->old_heap = c_p->old_heap;
-	sproc->mbuf = NULL;
-	sproc->mbuf_sz = 0;
-	ERTS_INIT_OFF_HEAP(&sproc->off_heap);
-
-	env->hp_end = HEAP_LIMIT(c_p);
-	env->hp = HEAP_TOP(c_p);
-	env->heap_frag = NULL;
-	return;
-    }
+    if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)
+	erts_cache_dirty_shadow_proc(env->proc);
 #endif
-
     cache_env(env);
 }
 
@@ -1303,22 +1320,15 @@ Eterm enif_make_badarg(ErlNifEnv* env)
 
 Eterm enif_raise_exception(ErlNifEnv* env, ERL_NIF_TERM reason)
 {
-    Process *c_p;
-
-    execution_state(env, &c_p, NULL);
-
     env->exception_thrown = 1;
-    c_p->fvalue = reason;
-    BIF_ERROR(c_p, EXC_ERROR);
+    env->proc->fvalue = reason;
+    BIF_ERROR(env->proc, EXC_ERROR);
 }
 
 int enif_has_pending_exception(ErlNifEnv* env, ERL_NIF_TERM* reason)
 {
-    if (env->exception_thrown && reason != NULL) {
-	Process *c_p;
-	execution_state(env, &c_p, NULL);
-	*reason = c_p->fvalue;
-    }
+    if (env->exception_thrown && reason != NULL)
+	*reason = env->proc->fvalue;
     return env->exception_thrown;
 }
 
@@ -2269,188 +2279,28 @@ int enif_consume_timeslice(ErlNifEnv* env, int percent)
     return ERTS_BIF_REDS_LEFT(proc) == 0;
 }
 
-/*
- * NIF exports need a few more items than the Export struct provides,
- * including the erl_module_nif* and a NIF function pointer, so the
- * NifExport below adds those. The Export member must be first in the
- * struct. The saved_current, exception_thrown, saved_argc, rootset_extra, and
- * rootset members are used to track the MFA, any pending exception, and
- * arguments of the top NIF in case a chain of one or more
- * enif_schedule_nif() calls results in an exception, since in that case
- * the original MFA and registers have to be restored before returning to
- * Erlang to ensure stacktrace information associated with the exception is
- * correct.
- */
-typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]);
-
-typedef struct {
-    Export exp;
-    struct erl_module_nif* m;
-    NativeFunPtr fp;
-    ErtsCodeMFA *saved_current;
-    int exception_thrown;
-    int saved_argc;
-    int rootset_extra;
-    Eterm rootset[1];
-} NifExport;
-
-/*
- * If a process has saved arguments, they need to be part of the GC
- * rootset. The function below is called from setup_rootset() in
- * erl_gc.c. This function is declared in erl_process.h. Any exception term
- * saved in the NifExport is also made part of the GC rootset here; it
- * always resides in rootset[0].
- */
-int
-erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj)
-{
-    NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    int gc = ep && (ep->saved_argc > 0 || ep->rootset[0] != NIL);
-
-    if (gc) {
-	*objv = ep->rootset;
-	*nobj = 1 + ep->saved_argc;
-    }
-    return gc;
-}
-
-int
-erts_check_nif_export_in_area(Process *p, char *start, Uint size)
-{
-    NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p);
-    if (!nep || !nep->saved_current)
-	return 0;
-    if (ErtsInArea(nep->saved_current, start, size))
-	return 1;
-    return 0;
-}
-
-/*
- * Allocate a NifExport and set it in proc specific data
- */
-static NifExport*
-allocate_nif_sched_data(Process* proc, int argc)
-{
-    NifExport* ep;
-    size_t total;
-    int i;
-
-    total = sizeof(NifExport) + argc*sizeof(Eterm);
-    ep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, total);
-    sys_memset((void*) ep, 0, total);
-    ep->rootset_extra = argc;
-    ep->rootset[0] = NIL;
-    for (i=0; i<ERTS_NUM_CODE_IX; i++) {
-	ep->exp.addressv[i] = &ep->exp.beam[0];
-    }
-    ep->exp.beam[0] = (BeamInstr) em_call_nif;
-    (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ep);
-    return ep;
-}
-
 static ERTS_INLINE void
-destroy_nif_export(NifExport *nif_export)
+nif_export_cleanup_nif_mod(NifExport *ep)
 {
-    erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, (void *) nif_export);
+    if (erts_refc_dectest(&ep->m->rt_dtor_cnt, 0) == 0 && ep->m->mod == NULL)
+	close_lib(ep->m);
+    ep->m = NULL;
 }
 
 void
-erts_destroy_nif_export(void *nif_export)
+erts_nif_export_cleanup_nif_mod(NifExport *ep)
 {
-    destroy_nif_export((NifExport *) nif_export);
+    nif_export_cleanup_nif_mod(ep);
 }
 
-/*
- * Initialize a NifExport struct. Create it if needed and store it in the
- * proc. The direct_fp function is what will be invoked by op_call_nif, and
- * the indirect_fp function, if not NULL, is what the direct_fp function
- * will call. If the allocated NifExport isn't enough to hold all of argv,
- * allocate a larger one. Save MFA and registers only if the need_save
- * parameter is true.
- */
-static ERL_NIF_TERM
-init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp,
-		    int need_save, int argc, const ERL_NIF_TERM argv[])
+static ERTS_INLINE void
+nif_export_restore(Process *c_p, NifExport *ep)
 {
-    Process* proc;
-    Eterm* reg;
-    NifExport* ep;
-    int i, scheduler;
-    int orig_argc;
-
-    execution_state(env, &proc, &scheduler);
-
-    ASSERT(scheduler);
-
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc)
-		       & ERTS_PROC_LOCK_MAIN);
-
-    reg = erts_proc_sched_data(proc)->x_reg_array;
-
-    ASSERT(!need_save || proc->current);
-    orig_argc = need_save ? (int) proc->current->arity : 0;
-
-    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    if (!ep)
-	ep = allocate_nif_sched_data(proc, orig_argc);
-    else if (need_save && ep->rootset_extra < orig_argc) {
-	NifExport* new_ep = allocate_nif_sched_data(proc, orig_argc);
-	destroy_nif_export(ep);
-	ep = new_ep;
-    }
-    if (env->exception_thrown) {
-	ep->exception_thrown = 1;
-	ep->rootset[0] = proc->fvalue;
-    } else {
-	ep->exception_thrown = 0;
-	ep->rootset[0] = NIL;
-    }
-    if (scheduler > 0)
-	ERTS_VBUMP_ALL_REDS(proc);
-    if (need_save) {
-	ep->saved_current = proc->current;
-	ep->saved_argc = orig_argc;
-	for (i = 0; i < orig_argc; i++)
-	    ep->rootset[i+1] = reg[i];
-    }
-    for (i = 0; i < argc; i++)
-	reg[i] = (Eterm) argv[i];
-    proc->i = (BeamInstr*) ep->exp.addressv[0];
-    ep->exp.info.mfa.module = proc->current->module;
-    ep->exp.info.mfa.function = proc->current->function;
-    ep->exp.info.mfa.arity = argc;
-    ep->exp.beam[1] = (BeamInstr) direct_fp;
-    ep->m = env->mod_nif;
-    ep->fp = indirect_fp;
-    proc->freason = TRAP;
-    proc->arity = argc;
-    return THE_NON_VALUE;
+    erts_nif_export_restore(c_p, ep);
+    ASSERT(ep->m);
+    nif_export_cleanup_nif_mod(ep);
 }
 
-/*
- * Restore saved MFA and registers. Registers are restored only when the
- * exception flag is true.
- */
-static void
-restore_nif_mfa(Process* proc, NifExport* ep, int exception)
-{
-    int i;
-
-    ERTS_SMP_LC_ASSERT(!(proc->static_flags
-			 & ERTS_STC_FLG_SHADOW_PROC));
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc)
-		       & ERTS_PROC_LOCK_MAIN);
-
-    ASSERT(ep->saved_current != &ep->exp.info.mfa);
-    proc->current = ep->saved_current;
-    ep->saved_current = NULL;
-    if (exception) {
-	Eterm* reg = erts_proc_sched_data(proc)->x_reg_array;
-	for (i = 0; i < ep->saved_argc; i++)
-	    reg[i] = ep->rootset[i+1];
-    }
-    ep->saved_argc = 0;
-}
 
 #ifdef ERTS_DIRTY_SCHEDULERS
 
@@ -2459,7 +2309,7 @@ restore_nif_mfa(Process* proc, NifExport* ep, int exception)
  * switch the process off a dirty scheduler thread and back onto a regular
  * scheduler thread, and then return the result from the dirty NIF. It also
  * restores the original NIF MFA when necessary based on the value of
- * ep->fp set by execute_dirty_nif via init_nif_sched_data -- non-NULL
+ * ep->func set by execute_dirty_nif via init_nif_sched_data -- non-NULL
  * means restore, NULL means do not restore.
  */
 static ERL_NIF_TERM
@@ -2474,9 +2324,7 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc)));
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     ASSERT(ep);
-    ASSERT(!ep->exception_thrown);
-    if (ep->fp)
-	restore_nif_mfa(proc, ep, 0);
+    nif_export_restore(proc, ep);
     return argv[0];
 }
 
@@ -2486,148 +2334,100 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 static ERL_NIF_TERM
 dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
+    ERL_NIF_TERM ret;
     Process* proc;
     NifExport* ep;
+    Eterm exception;
 
     execution_state(env, &proc, NULL);
 
+    ASSERT(argc == 1);
     ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc)));
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     ASSERT(ep);
-    ASSERT(ep->exception_thrown);
-    if (ep->fp)
-	restore_nif_mfa(proc, ep, 1);
-    return enif_raise_exception(env, ep->rootset[0]);
+    exception = argv[0]; /* argv overwritten by restore below... */
+    nif_export_cleanup_nif_mod(ep);
+    ret = enif_raise_exception(env, exception);
+
+    /* Restore orig info for error and clear nif export in handle_error() */
+    proc->freason |= EXF_RESTORE_NIF;
+    return ret;
 }
 
 /*
- * Dirty NIF execution wrapper function. Invoke an application's dirty NIF,
- * then check the result and schedule the appropriate finalizer function
- * where needed. Also restore the original NIF MFA when appropriate.
+ * Dirty NIF scheduling wrapper function. Schedule a dirty NIF to execute.
+ * The dirty scheduler thread type (CPU or I/O) is indicated in flags
+ * parameter.
  */
-static ERL_NIF_TERM
-execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+static ERTS_INLINE ERL_NIF_TERM
+schedule_dirty_nif(ErlNifEnv* env, int flags, NativeFunPtr fp,
+		   Eterm func_name, int argc, const ERL_NIF_TERM argv[])
 {
     Process* proc;
-    NativeFunPtr fp;
-    NifExport* ep;
-    ERL_NIF_TERM result;
-
-    execution_state(env, &proc, NULL);
 
-    ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa);
-    fp = ep->fp;
-
-    ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc)));
-
-    /*
-     * Set ep->fp to NULL before the native call so we know later whether it scheduled another NIF for execution
-     */
-    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    ASSERT(ep && fp);
-
-    ep->fp = NULL;
-    erts_smp_atomic32_read_band_mb(&proc->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC
-						   | ERTS_PSFLG_DIRTY_IO_PROC));
+    ASSERT(is_atom(func_name));
+    ASSERT(fp);
 
-    erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
+    ASSERT(flags==ERL_NIF_DIRTY_JOB_IO_BOUND || flags==ERL_NIF_DIRTY_JOB_CPU_BOUND);
 
-    result = (*fp)(env, argc, argv);
+    execution_state(env, &proc, NULL);
 
-    erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
+    (void) erts_smp_atomic32_read_bset_nob(&proc->state,
+					   (ERTS_PSFLG_DIRTY_CPU_PROC
+					    | ERTS_PSFLG_DIRTY_IO_PROC),
+					   (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND
+					    ? ERTS_PSFLG_DIRTY_CPU_PROC
+					    : ERTS_PSFLG_DIRTY_IO_PROC));
 
-    if (erts_refc_dectest(&env->mod_nif->rt_dtor_cnt, 0) == 0 && env->mod_nif->mod == NULL)
-	close_lib(env->mod_nif);
-    /*
-     * If no more NIFs were scheduled by the native call via
-     * enif_schedule_nif(), then ep->fp will still be NULL as set above, in
-     * which case we need to restore the original NIF calling
-     * context. Reuse fp essentially as a boolean for this, passing it to
-     * init_nif_sched_data below. Both dirty_nif_exception and
-     * dirty_nif_finalizer then check ep->fp to decide whether or not to
-     * restore the original calling context.
-     */
-    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    ASSERT(ep);
-    if (ep->fp)
-	fp = NULL;
-    if (is_non_value(result) || env->exception_thrown) {
-	if (proc->freason != TRAP) {
-	    return init_nif_sched_data(env, dirty_nif_exception, fp, 0, argc, argv);
-	} else {
-	    if (ep->fp == NULL)
-		restore_nif_mfa(proc, ep, 1);
-	    return THE_NON_VALUE;
-	}
-    }
-    else
-	return init_nif_sched_data(env, dirty_nif_finalizer, fp, 0, 1, &result);
+    return schedule(env, fp, NULL, proc->current->module, func_name, argc, argv);
 }
 
-/*
- * Dirty NIF scheduling wrapper function. Schedule a dirty NIF to execute
- * via the execute_dirty_nif() wrapper function. The dirty scheduler thread
- * type (CPU or I/O) is indicated in flags parameter.
- */
 static ERTS_INLINE ERL_NIF_TERM
-schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[])
+static_schedule_dirty_nif(ErlNifEnv* env, erts_aint32_t dirty_psflg,
+			     int argc, const ERL_NIF_TERM argv[])
 {
-    ERL_NIF_TERM result;
-    erts_aint32_t act, dirty_flag;
-    Process* proc;
+    Process *proc;
+    NifExport *ep;
+    Eterm mod, func;
     NativeFunPtr fp;
-    NifExport* ep;
-    int need_save, scheduler;
 
-    execution_state(env, &proc, &scheduler);
-    if (scheduler <= 0) {
-	ASSERT(scheduler < 0);
-	erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
-    }
+    execution_state(env, &proc, NULL);
+
+    /*
+     * Called in order to schedule statically determined
+     * dirty NIF calls...
+     *
+     * Note that 'current' does not point into a NifExport
+     * structure; only a structure with similar
+     * parts (located in code).
+     */
 
     ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa);
-    fp = ep->fp;
+    mod = proc->current->module;
+    func = proc->current->function;
+    fp = (NativeFunPtr) ep->func;
 
+    ASSERT(is_atom(mod) && is_atom(func));
     ASSERT(fp);
 
-    ASSERT(flags==ERL_NIF_DIRTY_JOB_IO_BOUND || flags==ERL_NIF_DIRTY_JOB_CPU_BOUND);
-
-    if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND)
-	dirty_flag = ERTS_PSFLG_DIRTY_CPU_PROC;
-    else
-	dirty_flag = ERTS_PSFLG_DIRTY_IO_PROC;
-
-    act = erts_smp_atomic32_read_bor_nob(&proc->state, dirty_flag);
-    if (!(act & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)))
-	erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1);
-    else if ((act & (ERTS_PSFLG_DIRTY_CPU_PROC
-		     | ERTS_PSFLG_DIRTY_IO_PROC)) & ~dirty_flag) {
-	/* clear other flag... */
-	if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND)
-	    dirty_flag = ERTS_PSFLG_DIRTY_IO_PROC;
-	else
-	    dirty_flag = ERTS_PSFLG_DIRTY_CPU_PROC;
-	erts_smp_atomic32_read_band_nob(&proc->state, ~dirty_flag);
-    }
+    (void) erts_smp_atomic32_read_bset_nob(&proc->state,
+					   (ERTS_PSFLG_DIRTY_CPU_PROC
+					    | ERTS_PSFLG_DIRTY_IO_PROC),
+					   dirty_psflg);
 
-    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    need_save = (ep == NULL || !ep->saved_current);
-    result = init_nif_sched_data(env, execute_dirty_nif, fp, need_save, argc, argv);
-    if (scheduler <= 0)
-	erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
-    return result;
+    return schedule(env, fp, NULL, mod, func, argc, argv);
 }
 
 static ERL_NIF_TERM
-schedule_dirty_io_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+static_schedule_dirty_io_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    return schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_IO_BOUND, argc, argv);
+    return static_schedule_dirty_nif(env, ERTS_PSFLG_DIRTY_IO_PROC, argc, argv);
 }
 
 static ERL_NIF_TERM
-schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+static_schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    return schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_CPU_BOUND, argc, argv);
+    return static_schedule_dirty_nif(env, ERTS_PSFLG_DIRTY_CPU_PROC, argc, argv);
 }
 
 #endif /* ERTS_DIRTY_SCHEDULERS */
@@ -2646,23 +2446,42 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     ERL_NIF_TERM result;
 
     execution_state(env, &proc, NULL);
-    ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa);
-    fp = ep->fp;
 
-    ASSERT(!env->exception_thrown);
-    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
+    ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa);
+    fp = ep->func;
     ASSERT(ep);
-    ep->fp = NULL;
+    ASSERT(!env->exception_thrown);
+
+    fp = (NativeFunPtr) ep->func;
+
+#ifdef DEBUG
+    ep->func = ERTS_DBG_NIF_NOT_SCHED_MARKER;
+#endif
+
     result = (*fp)(env, argc, argv);
-    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    ASSERT(ep);
-    /*
-     * If no NIFs were scheduled by the native call via
-     * enif_schedule_nif(), then ep->fp will still be NULL as set above, in
-     * which case we need to restore the original NIF MFA.
-     */
-    if (ep->fp == NULL)
-	restore_nif_mfa(proc, ep, env->exception_thrown);
+
+    ASSERT(ep == ERTS_PROC_GET_NIF_TRAP_EXPORT(proc));
+
+    if (is_value(result) || proc->freason != TRAP) {
+	/* Done (not rescheduled)... */
+	ASSERT(ep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER);
+	if (!env->exception_thrown)
+	    nif_export_restore(proc, ep);
+	else {
+	    nif_export_cleanup_nif_mod(ep);
+	    /*
+	     * Restore orig info for error and clear nif
+	     * export in handle_error()
+	     */
+	    proc->freason |= EXF_RESTORE_NIF;
+	}
+    }
+
+#ifdef DEBUG
+    if (ep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER)
+	ep->func = NULL;
+#endif
+
     return result;
 }
 
@@ -2672,9 +2491,8 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags,
 		  int argc, const ERL_NIF_TERM argv[])
 {
     Process* proc;
-    NifExport* ep;
     ERL_NIF_TERM fun_name_atom, result;
-    int need_save, scheduler;
+    int scheduler;
 
     if (argc > MAX_ARG)
 	return enif_make_badarg(env);
@@ -2689,35 +2507,16 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags,
 	erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
     }
 
-    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    need_save = (ep == NULL || !ep->saved_current);
-
-    if (flags) {
+    if (flags == 0)
+	result = schedule(env, execute_nif, fp, proc->current->module,
+			  fun_name_atom, argc, argv);
 #ifdef ERTS_DIRTY_SCHEDULERS
-	NativeFunPtr sched_fun;
-	int chkflgs = (flags & (ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND));
-	if (chkflgs == ERL_NIF_DIRTY_JOB_IO_BOUND)
-	    sched_fun = schedule_dirty_io_nif;
-	else if (chkflgs == ERL_NIF_DIRTY_JOB_CPU_BOUND)
-	    sched_fun = schedule_dirty_cpu_nif;
-	else {
-	    result = enif_make_badarg(env);
-	    goto done;
-	}
-	result = init_nif_sched_data(env, sched_fun, fp, need_save, argc, argv);
-#else
-	result = enif_make_badarg(env);
+    else if (!(flags & ~(ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND)))
+	result = schedule_dirty_nif(env, flags, fp, fun_name_atom, argc, argv);
 #endif
-	goto done;
-    }
     else
-	result = init_nif_sched_data(env, execute_nif, fp, need_save, argc, argv);
-
-    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    ASSERT(ep);
-    ep->exp.info.mfa.function = (BeamInstr) fun_name_atom;
+	result = enif_make_badarg(env);
 
-done:
     if (scheduler < 0)
 	erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
 
@@ -3416,8 +3215,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	    if (f->flags) {
 		code_ptr[3] = (BeamInstr) f->fptr;
 		code_ptr[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ?
-		    (BeamInstr) schedule_dirty_io_nif :
-		    (BeamInstr) schedule_dirty_cpu_nif;
+		    (BeamInstr) static_schedule_dirty_io_nif :
+		    (BeamInstr) static_schedule_dirty_cpu_nif;
 	    }
 	    else
 #endif
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 6928b282ee..955b98b35b 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -48,6 +48,7 @@
 #include "erl_bif_unique.h"
 #define ERTS_WANT_TIMER_WHEEL_API
 #include "erl_time.h"
+#include "erl_nfunc_sched.h"
 
 #define ERTS_CHECK_TIME_REDS CONTEXT_REDS
 #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0)
@@ -193,12 +194,6 @@ static ErtsAuxWorkData *aux_thread_aux_work_data;
 #define ERTS_SCHDLR_SSPND_CHNG_ONLN		(((erts_aint32_t) 1) << 2)
 #define ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN	(((erts_aint32_t) 1) << 3)
 
-typedef enum {
-    ERTS_SCHED_NORMAL,
-    ERTS_SCHED_DIRTY_CPU,
-    ERTS_SCHED_DIRTY_IO
-} ErtsSchedType;
-
 typedef struct {
     int ongoing;
     ErtsProcList *blckrs;
@@ -10102,9 +10097,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		goto sunlock_sched_out_proc;
 	    }
 
-	    ASSERT((state & ERTS_PSFLG_DIRTY_ACTIVE_SYS)
-		   || *p->i == (BeamInstr) em_call_nif);
-
 	    ASSERT(rq == ERTS_DIRTY_CPU_RUNQ
 		   ? (state & (ERTS_PSFLG_DIRTY_CPU_PROC
 			       | ERTS_PSFLG_DIRTY_ACTIVE_SYS))
@@ -11976,7 +11968,6 @@ delete_process(Process* p)
     ErtsPSD *psd;
     struct saved_calls *scb;
     process_breakpoint_time_t *pbt;
-    void *nif_export;
 
     VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id));
     VERBOSE(DEBUG_SHCOPY, ("[pid=%T] delete process: %p %p %p %p\n", p->common.id,
@@ -11993,9 +11984,7 @@ delete_process(Process* p)
     if (pbt)
         erts_free(ERTS_ALC_T_BPD, (void *) pbt);
 
-    nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL);
-    if (nif_export)
-	erts_destroy_nif_export(nif_export);
+    erts_destroy_nif_export(p);
 
     /* Cleanup psd */
 
@@ -13564,6 +13553,9 @@ erts_dbg_check_halloc_lock(Process *p)
     ErtsSchedulerData *esdp;
     if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p))
 	return 1;
+    if ((p->static_flags & ERTS_STC_FLG_SHADOW_PROC)
+	&& ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()))
+	return 1;
     if (p->common.id == ERTS_INVALID_PID)
 	return 1;
     esdp = erts_proc_sched_data(p);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 7193b7d1db..58a50f8cf4 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -838,8 +838,8 @@ typedef struct {
 #define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
 #define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN
 
-#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ((ErtsProcLocks) 0)
-#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ((ErtsProcLocks) 0)
+#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
+#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN
 
 typedef struct {
     ErtsProcLocks get_locks;
@@ -1582,10 +1582,6 @@ Uint64 erts_get_proc_interval(void);
 Uint64 erts_ensure_later_proc_interval(Uint64);
 Uint64 erts_step_proc_interval(void);
 
-int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */
-void erts_destroy_nif_export(void *); /* see erl_nif.c */
-int erts_check_nif_export_in_area(Process *p, char *start, Uint size);
-
 ErtsProcList *erts_proclist_create(Process *);
 ErtsProcList *erts_proclist_copy(ErtsProcList *);
 void erts_proclist_destroy(ErtsProcList *);
diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h
index e431c3051b..99a8ff6bad 100644
--- a/erts/emulator/beam/error.h
+++ b/erts/emulator/beam/error.h
@@ -38,15 +38,12 @@
  * Some of these have convenient aliases, like BADARG and BADARITH.
  */
 
-/*
- * Bits 0-1 index the 'exception class tag' table.
- */
-#define EXC_CLASSBITS 3
-#define GET_EXC_CLASS(x) ((x) & EXC_CLASSBITS)
-
 /*
  * Exception class tags (indices into the 'exception_tag' array)
  */
+#define EXTAG_OFFSET	0
+#define EXTAG_BITS	2
+
 #define EXTAG_ERROR	0
 #define EXTAG_EXIT	1
 #define EXTAG_THROWN	2
@@ -54,20 +51,31 @@
 #define NUMBER_EXC_TAGS 3	/* The number of exception class tags */
 
 /*
- * Exit code flags (bits 2-7)
+ * Index to the 'exception class tag' table.
+ */
+#define EXC_CLASSBITS	((1<<EXTAG_BITS)-1)
+#define GET_EXC_CLASS(x) ((x) & EXC_CLASSBITS)
+
+/*
+ * Exit code flags
  *
  * These flags make is easier and quicker to decide what to do with the
  * exception in the early stages, before a handler is found, and also
  * maintains some separation between the class tag and the actions.
  */
-#define EXF_PANIC	(1<<2)	/* ignore catches */
-#define EXF_THROWN	(1<<3)	/* nonlocal return */
-#define EXF_LOG		(1<<4)	/* write to logger on termination */
-#define EXF_NATIVE	(1<<5)	/* occurred in native code */
-#define EXF_SAVETRACE	(1<<6)	/* save stack trace in internal form */
-#define EXF_ARGLIST	(1<<7)	/* has arglist for top of trace */
+#define EXF_OFFSET	EXTAG_BITS
+#define EXF_BITS	7
 
-#define EXC_FLAGBITS 0x00fc
+#define EXF_PANIC	(1<<(0+EXF_OFFSET))	/* ignore catches */
+#define EXF_THROWN	(1<<(1+EXF_OFFSET))	/* nonlocal return */
+#define EXF_LOG		(1<<(2+EXF_OFFSET))	/* write to logger on termination */
+#define EXF_NATIVE	(1<<(3+EXF_OFFSET))	/* occurred in native code */
+#define EXF_SAVETRACE	(1<<(4+EXF_OFFSET))	/* save stack trace in internal form */
+#define EXF_ARGLIST	(1<<(5+EXF_OFFSET))	/* has arglist for top of trace */
+#define EXF_RESTORE_NIF	(1<<(6+EXF_OFFSET))	/* restore original bif/nif */
+
+#define EXC_FLAGBITS	(((1<<(EXF_BITS+EXF_OFFSET))-1) \
+			 & ~((1<<(EXF_OFFSET))-1))
 
 /*
  * The primary fields of an exception code
@@ -77,11 +85,16 @@
 #define NATIVE_EXCEPTION(x) ((x) | EXF_NATIVE)
 
 /*
- * Bits 8-12 of the error code are used for indexing into
+ * Error code used for indexing into
  * the short-hand error descriptor table.
  */
-#define EXC_INDEXBITS 0x1f00
-#define GET_EXC_INDEX(x) (((x) & EXC_INDEXBITS) >> 8)
+#define EXC_OFFSET	(EXF_OFFSET+EXF_BITS)
+#define EXC_BITS	5
+
+#define EXC_INDEXBITS	(((1<<(EXC_BITS+EXC_OFFSET))-1) \
+			 & ~((1<<(EXC_OFFSET))-1))
+
+#define GET_EXC_INDEX(x) (((x) & EXC_INDEXBITS) >> EXC_OFFSET)
 
 /*
  * Exit codes used for raising a fresh exception. The primary exceptions
@@ -107,46 +120,46 @@
 					/* Error with given arglist term
 					 * (exit reason in p->fvalue) */
 
-#define EXC_NORMAL		((1 << 8) | EXC_EXIT)
+#define EXC_NORMAL		((1 << EXC_OFFSET) | EXC_EXIT)
 					/* Normal exit (reason 'normal') */
-#define EXC_INTERNAL_ERROR	((2 << 8) | EXC_ERROR | EXF_PANIC)
+#define EXC_INTERNAL_ERROR	((2 << EXC_OFFSET) | EXC_ERROR | EXF_PANIC)
 					/* Things that shouldn't happen */
-#define EXC_BADARG		((3 << 8) | EXC_ERROR)
+#define EXC_BADARG		((3 << EXC_OFFSET) | EXC_ERROR)
 					/* Bad argument to a BIF */
-#define EXC_BADARITH		((4 << 8) | EXC_ERROR)
+#define EXC_BADARITH		((4 << EXC_OFFSET) | EXC_ERROR)
 					/* Bad arithmetic */
-#define EXC_BADMATCH		((5 << 8) | EXC_ERROR)
+#define EXC_BADMATCH		((5 << EXC_OFFSET) | EXC_ERROR)
 					/* Bad match in function body */
-#define EXC_FUNCTION_CLAUSE	((6 << 8) | EXC_ERROR)
+#define EXC_FUNCTION_CLAUSE	((6 << EXC_OFFSET) | EXC_ERROR)
 					 /* No matching function head */
-#define EXC_CASE_CLAUSE		((7 << 8) | EXC_ERROR)
+#define EXC_CASE_CLAUSE		((7 << EXC_OFFSET) | EXC_ERROR)
 					/* No matching case clause */
-#define EXC_IF_CLAUSE		((8 << 8) | EXC_ERROR)
+#define EXC_IF_CLAUSE		((8 << EXC_OFFSET) | EXC_ERROR)
 					/* No matching if clause */
-#define EXC_UNDEF		((9 << 8) | EXC_ERROR)
+#define EXC_UNDEF		((9 << EXC_OFFSET) | EXC_ERROR)
 				 	/* No farity that matches */
-#define EXC_BADFUN		((10 << 8) | EXC_ERROR)
+#define EXC_BADFUN		((10 << EXC_OFFSET) | EXC_ERROR)
 					/* Not an existing fun */
-#define EXC_BADARITY		((11 << 8) | EXC_ERROR)
+#define EXC_BADARITY		((11 << EXC_OFFSET) | EXC_ERROR)
 					/* Attempt to call fun with
 					 * wrong number of arguments. */
-#define EXC_TIMEOUT_VALUE	((12 << 8) | EXC_ERROR)
+#define EXC_TIMEOUT_VALUE	((12 << EXC_OFFSET) | EXC_ERROR)
 					/* Bad time out value */
-#define EXC_NOPROC		((13 << 8) | EXC_ERROR)
+#define EXC_NOPROC		((13 << EXC_OFFSET) | EXC_ERROR)
 					/* No process or port */
-#define EXC_NOTALIVE		((14 << 8) | EXC_ERROR)
+#define EXC_NOTALIVE		((14 << EXC_OFFSET) | EXC_ERROR)
 					/* Not distributed */
-#define EXC_SYSTEM_LIMIT	((15 << 8) | EXC_ERROR)
+#define EXC_SYSTEM_LIMIT	((15 << EXC_OFFSET) | EXC_ERROR)
 					/* Ran out of something */
-#define EXC_TRY_CLAUSE		((16 << 8) | EXC_ERROR)
+#define EXC_TRY_CLAUSE		((16 << EXC_OFFSET) | EXC_ERROR)
 					/* No matching try clause */
-#define EXC_NOTSUP		((17 << 8) | EXC_ERROR)
+#define EXC_NOTSUP		((17 << EXC_OFFSET) | EXC_ERROR)
 					/* Not supported */
 
-#define EXC_BADMAP		((18 << 8) | EXC_ERROR)
+#define EXC_BADMAP		((18 << EXC_OFFSET) | EXC_ERROR)
 					/* Bad map */
 
-#define EXC_BADKEY		((19 << 8) | EXC_ERROR)
+#define EXC_BADKEY		((19 << EXC_OFFSET) | EXC_ERROR)
 					/* Bad key in map */
 
 #define NUMBER_EXIT_CODES 20	/* The number of exit code indices */
@@ -154,7 +167,7 @@
 /*
  * Internal pseudo-error codes.
  */
-#define TRAP		(1 << 8)	/* BIF Trap to erlang code */
+#define TRAP		(1 << EXC_OFFSET)	/* BIF Trap to erlang code */
 
 /*
  * Aliases for some common exit codes.
@@ -201,6 +214,7 @@ struct StackTrace {
     BeamInstr* pc;
     ErtsCodeMFA* current;
     int depth;	/* number of saved pointers in trace[] */
+    ErtsCodeMFA mfa; /* in case we need to make a copy of mfa */
     BeamInstr *trace[1];  /* varying size - must be last in struct */
 };
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 2b2f3c5cdc..d5ca3b04eb 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -61,12 +61,6 @@ struct enif_environment_t /* ErlNifEnv */
 extern void erts_pre_nif(struct enif_environment_t*, Process*,
 			 struct erl_module_nif*, Process* tracee);
 extern void erts_post_nif(struct enif_environment_t* env);
-#ifdef ERTS_DIRTY_SCHEDULERS
-extern void erts_pre_dirty_nif(ErtsSchedulerData *,
-			       struct enif_environment_t*, Process*,
-			       struct erl_module_nif*);
-extern void erts_post_dirty_nif(struct enif_environment_t* env);
-#endif
 extern Eterm erts_nif_taints(Process* p);
 extern void erts_print_nif_taints(fmtfn_t to, void* to_arg);
 void erts_unload_nif(struct erl_module_nif* nif);
@@ -78,6 +72,12 @@ extern Eterm erts_nif_call_function(Process *p, Process *tracee,
                                     struct enif_func_t *,
                                     int argc, Eterm *argv);
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+int erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p,
+			BeamInstr *I, Eterm *reg);
+#endif /* ERTS_DIRTY_SCHEDULERS */
+
+
 /* Driver handle (wrapper for old plain handle) */
 #define ERL_DE_OK      0
 #define ERL_DE_UNLOAD  1
@@ -993,7 +993,7 @@ void erts_queue_monitor_message(Process *,
 				Eterm,
 				Eterm);
 void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a,
-			   Eterm (*bif)(Process*,Eterm*));
+			   Eterm (*bif)(Process*, Eterm*, BeamInstr*));
 void erts_init_bif(void);
 Eterm erl_send(Process *p, Eterm to, Eterm msg);
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index ec502d5a78..3fa48da1ec 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -56,6 +56,8 @@
 #ifdef HIPE
 #  include "hipe_mode_switch.h"
 #endif
+#define ERTS_WANT_NFUNC_SCHED_INTERNALS__
+#include "erl_nfunc_sched.h"
 
 #undef M_TRIM_THRESHOLD
 #undef M_TOP_PAD
diff --git a/erts/emulator/hipe/hipe_amd64_bifs.m4 b/erts/emulator/hipe/hipe_amd64_bifs.m4
index 21739726bb..dca3887564 100644
--- a/erts/emulator/hipe/hipe_amd64_bifs.m4
+++ b/erts/emulator/hipe/hipe_amd64_bifs.m4
@@ -41,11 +41,11 @@ define(HANDLE_GOT_MBUF,`
 
 `#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
 #  define CALL_BIF(F) \
-		movq CSYM(F)@GOTPCREL(%rip), %r11; \
+		movq CSYM(nbif_impl_##F)@GOTPCREL(%rip), %r11; \
 		movq %r11, P_BIF_CALLEE(P); \
 		call CSYM(hipe_debug_bif_wrapper)
 #else
-#  define CALL_BIF(F)	call	CSYM(F)
+#  define CALL_BIF(F)	call	CSYM(nbif_impl_##F)
 #endif'
 
 /*
@@ -595,13 +595,9 @@ noproc_primop_interface_0(nbif_handle_fp_exception, erts_restore_fpu)
 #endif /* NO_FPE_SIGNALS */
 
 /*
- * Implement gc_bif_interface_0 as nofail_primop_interface_0.
- */
-define(gc_bif_interface_0,`nofail_primop_interface_0($1, $2)')
-
-/*
- * Implement gc_bif_interface_N as standard_bif_interface_N (N=1,2,3).
+ * Implement gc_bif_interface_N as standard_bif_interface_N.
  */
+define(gc_bif_interface_0,`standard_bif_interface_0($1, $2)')
 define(gc_bif_interface_1,`standard_bif_interface_1($1, $2)')
 define(gc_bif_interface_2,`standard_bif_interface_2($1, $2)')
 define(gc_bif_interface_3,`standard_bif_interface_3($1, $2)')
diff --git a/erts/emulator/hipe/hipe_arm_bifs.m4 b/erts/emulator/hipe/hipe_arm_bifs.m4
index d7a2fec04a..a9097dabde 100644
--- a/erts/emulator/hipe/hipe_arm_bifs.m4
+++ b/erts/emulator/hipe/hipe_arm_bifs.m4
@@ -30,9 +30,9 @@ include(`hipe/hipe_arm_asm.m4')
 	.arm
 
 `#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
-#  define CALL_BIF(F)	ldr r14, =F; str r14, [r0, #P_BIF_CALLEE]; bl hipe_debug_bif_wrapper
+#  define CALL_BIF(F)	ldr r14, =nbif_impl_##F; str r14, [r0, #P_BIF_CALLEE]; bl hipe_debug_bif_wrapper
 #else
-#  define CALL_BIF(F)	bl	F
+#  define CALL_BIF(F)	bl	nbif_impl_##F
 #endif'
 
 define(TEST_GOT_MBUF,`ldr r1, [P, #P_MBUF]	/* `TEST_GOT_MBUF' */
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 5c29473443..9c6ac4bd9c 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -732,7 +732,7 @@ struct nbif {
 };
 
 static struct nbif nbifs[BIF_SIZE] = {
-#define BIF_LIST(MOD,FUN,ARY,CFUN,IX)	\
+#define BIF_LIST(MOD,FUN,ARY,BIF,CFUN,IX)	\
 	{ {0,0}, MOD, FUN, ARY, &nbif_##CFUN },
 #include "erl_bif_list.h"
 #undef BIF_LIST
@@ -905,7 +905,8 @@ BIF_RETTYPE hipe_bifs_term_to_word_1(BIF_ALIST_1)
 }
 
 /* XXX: this is really a primop, not a BIF */
-BIF_RETTYPE hipe_conv_big_to_float(BIF_ALIST_1)
+/* Called via standard_bif_interface_1 */
+BIF_RETTYPE nbif_impl_hipe_conv_big_to_float(NBIF_ALIST_1)
 {
     Eterm res;
     Eterm *hp;
@@ -1432,7 +1433,8 @@ void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a)
 }
 
 /* primop, but called like a BIF for error handling purposes */
-BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3)
+/* Called via standard_bif_interface_3 */
+BIF_RETTYPE nbif_impl_hipe_find_na_or_make_stub(NBIF_ALIST_3)
 {
     Uint arity;
     void *address;
@@ -1457,7 +1459,8 @@ BIF_RETTYPE hipe_bifs_find_na_or_make_stub_1(BIF_ALIST_1)
 }
 
 /* primop, but called like a BIF for error handling purposes */
-BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2)
+/* Called via standard_bif_interface_2 */
+BIF_RETTYPE nbif_impl_hipe_nonclosure_address(NBIF_ALIST_2)
 {
     Eterm hdr, m, f;
     void *address;
diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h
index 4a59bacc6e..811c3801c1 100644
--- a/erts/emulator/hipe/hipe_bif0.h
+++ b/erts/emulator/hipe/hipe_bif0.h
@@ -30,7 +30,7 @@ extern Uint *hipe_bifs_find_pc_from_mfa(Eterm mfa);
 
 extern void hipe_mfa_info_table_init(void);
 extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a);
-extern BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3);
+extern BIF_RETTYPE nbif_impl_hipe_find_na_or_make_stub(NBIF_ALIST_3);
 extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a);
 
 /* needed in beam_load.c */
diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c
index dfd34e31d4..e04d3d32d1 100644
--- a/erts/emulator/hipe/hipe_bif2.c
+++ b/erts/emulator/hipe/hipe_bif2.c
@@ -155,7 +155,7 @@ BIF_RETTYPE hipe_bifs_modeswitch_debug_off_0(BIF_ALIST_0)
 
 #if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
 
-BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1);
+BIF_RETTYPE hipe_debug_bif_wrapper(NBIF_ALIST_1);
 
 #    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
        if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\
@@ -163,13 +163,13 @@ BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1);
 #    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
         if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
 
-BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1)
+BIF_RETTYPE hipe_debug_bif_wrapper(NBIF_ALIST_1)
 {
-    typedef BIF_RETTYPE Bif(BIF_ALIST_1);
-    Bif* fp = (Bif*) (BIF_P->hipe.bif_callee);
+    typedef BIF_RETTYPE nBif(NBIF_ALIST_1);
+    nBif* fp = (nBif*) (BIF_P->hipe.bif_callee);
     BIF_RETTYPE res;
     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(BIF_P);
-    res = (*fp)(BIF_P, BIF__ARGS);
+    res = (*fp)(NBIF_CALL_ARGS);
     ERTS_SMP_REQ_PROC_MAIN_LOCK(BIF_P);
     return res;
 }
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index bb328b5915..ec0d0f5a0d 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -71,6 +71,32 @@
  ****************************************************************/
 
 /*
+ * NOTE:
+ *  Beam BIFs have the prototype:
+ *    Eterm (*BIF)(Process *c_p, Eterm *regs, UWord *I)
+ *  Native BIFs have the prototype:
+ *    Eterm (*BIF)(Process *c_p, Eterm *regs)
+ *
+ * Beam BIFs expect 'I' to contain current instruction
+ * pointer when called from beam, and expect 'I' to
+ * contain a pointer to the export entry of the BIF
+ * when called from native code. In order to facilitate
+ * this, beam BIFs are called via wrapper functions
+ * when called from native code. These wrapper functions
+ * are auto-generated (by utils/make_tables) and have
+ * the function names nbif_impl_<BIF>.
+ *
+ * The standard_bif_interface_*() and
+ * gc_bif_interface_*() will add the prefix and
+ * thus call nbif_impl_<cbif_name>. That is, all
+ * functions (true BIFs as well as other c-functions)
+ * called via these interfaces have to be named
+ * nbif_impl_<FUNC>.
+ */
+
+/*
+ * See NOTE above!
+ *
  * standard_bif_interface_0(nbif_name, cbif_name)
  * standard_bif_interface_1(nbif_name, cbif_name)
  * standard_bif_interface_2(nbif_name, cbif_name)
@@ -93,6 +119,8 @@
  */
 
 /*
+ * See NOTE above!
+ *
  * gc_bif_interface_0(nbif_name, cbif_name)
  * gc_bif_interface_1(nbif_name, cbif_name)
  * gc_bif_interface_2(nbif_name, cbif_name)
@@ -247,7 +275,7 @@ nocons_nofail_primop_interface_5(nbif_bs_put_big_integer, hipe_bs_put_big_intege
 noproc_primop_interface_5(nbif_bs_put_big_integer, hipe_bs_put_big_integer)
 ')dnl
 
-gc_bif_interface_0(nbif_check_get_msg, hipe_check_get_msg)
+nofail_primop_interface_0(nbif_check_get_msg, hipe_check_get_msg)
 
 #`ifdef' NO_FPE_SIGNALS
 nocons_nofail_primop_interface_0(nbif_emulate_fpe, hipe_emulate_fpe)
@@ -291,8 +319,8 @@ gc_bif_interface_2(nbif_maps_merge_2, hipe_wrapper_maps_merge_2)
  * BIF_LIST(ModuleAtom,FunctionAtom,Arity,CFun,Index)
  */
 
-define(BIF_LIST,`standard_bif_interface_$3(nbif_$4, $4)')
-include(TARGET/`erl_bif_list.h')
+define(BIF_LIST,`standard_bif_interface_$3(nbif_$5, $5)')
+include(TTF_DIR/`erl_bif_list.h')
 
 /*
  * Guard BIFs.
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index 9439b823ab..0c0dc79591 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -42,8 +42,8 @@
  */
 
 /* for -Wmissing-prototypes :-( */
-extern Eterm hipe_erts_internal_check_process_code_1(BIF_ALIST_1);
-extern Eterm hipe_show_nstack_1(BIF_ALIST_1);
+extern Eterm nbif_impl_hipe_erts_internal_check_process_code_1(NBIF_ALIST_1);
+extern Eterm nbif_impl_hipe_show_nstack_1(NBIF_ALIST_1);
 
 /* Used when a BIF can trigger a stack walk. */
 static __inline__ void hipe_set_narity(Process *p, unsigned int arity)
@@ -51,22 +51,24 @@ static __inline__ void hipe_set_narity(Process *p, unsigned int arity)
     p->hipe.narity = arity;
 }
 
-Eterm hipe_erts_internal_check_process_code_1(BIF_ALIST_1)
+/* Called via standard_bif_interface_2 */
+Eterm nbif_impl_hipe_erts_internal_check_process_code_1(NBIF_ALIST_1)
 {
     Eterm ret;
 
     hipe_set_narity(BIF_P, 1);
-    ret = erts_internal_check_process_code_1(BIF_P, BIF__ARGS);
+    ret = nbif_impl_erts_internal_check_process_code_1(NBIF_CALL_ARGS);
     hipe_set_narity(BIF_P, 0);
     return ret;
 }
 
-Eterm hipe_show_nstack_1(BIF_ALIST_1)
+/* Called via standard_bif_interface_1 */
+Eterm nbif_impl_hipe_show_nstack_1(NBIF_ALIST_1)
 {
     Eterm ret;
 
     hipe_set_narity(BIF_P, 1);
-    ret = hipe_bifs_show_nstack_1(BIF_P, BIF__ARGS);
+    ret = nbif_impl_hipe_bifs_show_nstack_1(NBIF_CALL_ARGS);
     hipe_set_narity(BIF_P, 0);
     return ret;
 }
@@ -89,7 +91,7 @@ void hipe_gc(Process *p, Eterm need)
  *  has begun.
  * XXX: BUG: native code should check return status
  */
-BIF_RETTYPE hipe_set_timeout(BIF_ALIST_1)
+BIF_RETTYPE nbif_impl_hipe_set_timeout(NBIF_ALIST_1)
 {
     Process* p = BIF_P;
     Eterm timeout_value = BIF_ARG_1;
@@ -280,10 +282,10 @@ static struct StackTrace *get_trace_from_exc(Eterm exc)
  * This does what the (misnamed) Beam instruction 'raise_ss' does,
  * namely, a proper re-throw of an exception that was caught by 'try'.
  */
-
-BIF_RETTYPE hipe_rethrow(BIF_ALIST_2)
+/* Called via standard_bif_interface_2 */
+BIF_RETTYPE nbif_impl_hipe_rethrow(NBIF_ALIST_2)
 {
-    Process* c_p = BIF_P;
+    Process *c_p = BIF_P;
     Eterm exc = BIF_ARG_1;
     Eterm value = BIF_ARG_2;
 
@@ -407,7 +409,7 @@ Eterm hipe_bs_utf8_size(Eterm arg)
 	return make_small(4);
 }
 
-BIF_RETTYPE hipe_bs_put_utf8(BIF_ALIST_3)
+BIF_RETTYPE nbif_impl_hipe_bs_put_utf8(NBIF_ALIST_3)
 {
     Process* p = BIF_P;
     Eterm arg = BIF_ARG_1;
@@ -468,7 +470,7 @@ Eterm hipe_bs_put_utf16(Process *p, Eterm arg, byte *base, unsigned int offset,
     return new_offset;
 }
 
-BIF_RETTYPE hipe_bs_put_utf16be(BIF_ALIST_3)
+BIF_RETTYPE nbif_impl_hipe_bs_put_utf16be(NBIF_ALIST_3)
 {
     Process *p = BIF_P;
     Eterm arg = BIF_ARG_1;
@@ -477,7 +479,7 @@ BIF_RETTYPE hipe_bs_put_utf16be(BIF_ALIST_3)
     return hipe_bs_put_utf16(p, arg, base, offset, 0);
 }
 
-BIF_RETTYPE hipe_bs_put_utf16le(BIF_ALIST_3)
+BIF_RETTYPE nbif_impl_hipe_bs_put_utf16le(NBIF_ALIST_3)
 {
     Process *p = BIF_P;
     Eterm arg = BIF_ARG_1;
@@ -495,7 +497,7 @@ static int validate_unicode(Eterm arg)
     return 1;
 }
 
-BIF_RETTYPE hipe_bs_validate_unicode(BIF_ALIST_1)
+BIF_RETTYPE nbif_impl_hipe_bs_validate_unicode(NBIF_ALIST_1)
 {
     Process *p = BIF_P;
     Eterm arg = BIF_ARG_1;
@@ -513,7 +515,8 @@ int hipe_bs_validate_unicode_retract(ErlBinMatchBuffer* mb, Eterm arg)
     return 1;
 }
 
-BIF_RETTYPE hipe_is_divisible(BIF_ALIST_2)
+/* Called via standard_bif_interface_2 */
+BIF_RETTYPE nbif_impl_hipe_is_divisible(NBIF_ALIST_2)
 {
     /* Arguments are Eterm-sized unsigned integers */
     Uint dividend = BIF_ARG_1;
diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h
index a02d26087b..38f874888b 100644
--- a/erts/emulator/hipe/hipe_native_bif.h
+++ b/erts/emulator/hipe/hipe_native_bif.h
@@ -74,27 +74,27 @@ AEXTERN(void,nbif_select_msg,(Process*));
 AEXTERN(Eterm,nbif_cmp_2,(void));
 AEXTERN(Eterm,nbif_eq_2,(void));
 
-BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2);
-BIF_RETTYPE hipe_conv_big_to_float(BIF_ALIST_1);
+BIF_RETTYPE nbif_impl_hipe_nonclosure_address(NBIF_ALIST_2);
+BIF_RETTYPE nbif_impl_hipe_conv_big_to_float(NBIF_ALIST_1);
 void hipe_fclearerror_error(Process*);
 void hipe_select_msg(Process*);
 void hipe_gc(Process*, Eterm);
-BIF_RETTYPE hipe_set_timeout(BIF_ALIST_1);
+BIF_RETTYPE nbif_impl_hipe_set_timeout(NBIF_ALIST_1);
 void hipe_handle_exception(Process*);
-BIF_RETTYPE hipe_rethrow(BIF_ALIST_2);
+BIF_RETTYPE nbif_impl_hipe_rethrow(NBIF_ALIST_2);
 char *hipe_bs_allocate(int);
 Binary *hipe_bs_reallocate(Binary*, int);
 int hipe_bs_put_small_float(Process*, Eterm, Uint, byte*, unsigned, unsigned);
 void hipe_bs_put_bits(Eterm, Uint, byte*, unsigned, unsigned);
 Eterm hipe_bs_utf8_size(Eterm);
-BIF_RETTYPE hipe_bs_put_utf8(BIF_ALIST_3);
+BIF_RETTYPE nbif_impl_hipe_bs_put_utf8(NBIF_ALIST_3);
 Eterm hipe_bs_utf16_size(Eterm);
-BIF_RETTYPE hipe_bs_put_utf16be(BIF_ALIST_3);
-BIF_RETTYPE hipe_bs_put_utf16le(BIF_ALIST_3);
-BIF_RETTYPE hipe_bs_validate_unicode(BIF_ALIST_1);
+BIF_RETTYPE nbif_impl_hipe_bs_put_utf16be(NBIF_ALIST_3);
+BIF_RETTYPE nbif_impl_hipe_bs_put_utf16le(NBIF_ALIST_3);
+BIF_RETTYPE nbif_impl_hipe_bs_validate_unicode(NBIF_ALIST_1);
 struct erl_bin_match_buffer;
 int hipe_bs_validate_unicode_retract(struct erl_bin_match_buffer*, Eterm);
-BIF_RETTYPE hipe_is_divisible(BIF_ALIST_2);
+BIF_RETTYPE nbif_impl_hipe_is_divisible(NBIF_ALIST_2);
 
 #ifdef NO_FPE_SIGNALS
 AEXTERN(void,nbif_emulate_fpe,(Process*));
@@ -129,7 +129,7 @@ void hipe_atomic_inc(int*);
 void hipe_clear_timeout(Process*);
 #endif
 
-#define BIF_LIST(M,F,A,C,I)	AEXTERN(Eterm,nbif_##C,(void));
+#define BIF_LIST(M,F,A,B,C,I)	AEXTERN(Eterm,nbif_##C,(void));
 #include "erl_bif_list.h"
 #undef BIF_LIST
 
diff --git a/erts/emulator/hipe/hipe_ppc_bifs.m4 b/erts/emulator/hipe/hipe_ppc_bifs.m4
index b540562185..79a8bef77d 100644
--- a/erts/emulator/hipe/hipe_ppc_bifs.m4
+++ b/erts/emulator/hipe/hipe_ppc_bifs.m4
@@ -26,9 +26,9 @@ include(`hipe/hipe_ppc_asm.m4')
 #`include' "hipe_literals.h"
 
 `#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
-#  define CALL_BIF(F)	STORE_IA(CSYM(F), P_BIF_CALLEE(P), r29); bl CSYM(hipe_debug_bif_wrapper) 
+#  define CALL_BIF(F)	STORE_IA(CSYM(nbif_impl_##F), P_BIF_CALLEE(P), r29); bl CSYM(hipe_debug_bif_wrapper) 
 #else
-#  define CALL_BIF(F)	bl	CSYM(F)
+#  define CALL_BIF(F)	bl	CSYM(nbif_impl_##F)
 #endif'
 
 	.text
diff --git a/erts/emulator/hipe/hipe_sparc_bifs.m4 b/erts/emulator/hipe/hipe_sparc_bifs.m4
index 1389beaa61..14330c2f1c 100644
--- a/erts/emulator/hipe/hipe_sparc_bifs.m4
+++ b/erts/emulator/hipe/hipe_sparc_bifs.m4
@@ -29,9 +29,9 @@ include(`hipe/hipe_sparc_asm.m4')
 	.align	4
 
 `#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
-#  define CALL_BIF(F)	set F, %o7; st %o7, [%o0+P_BIF_CALLEE]; call hipe_debug_bif_wrapper 
+#  define CALL_BIF(F)	set nbif_impl_##F, %o7; st %o7, [%o0+P_BIF_CALLEE]; call hipe_debug_bif_wrapper 
 #else
-#  define CALL_BIF(F)	call	F
+#  define CALL_BIF(F)	call	nbif_impl_##F
 #endif'
 
 /*
diff --git a/erts/emulator/hipe/hipe_x86_bifs.m4 b/erts/emulator/hipe/hipe_x86_bifs.m4
index c0c149733c..aecf67dc1b 100644
--- a/erts/emulator/hipe/hipe_x86_bifs.m4
+++ b/erts/emulator/hipe/hipe_x86_bifs.m4
@@ -32,9 +32,9 @@ include(`hipe/hipe_x86_asm.m4')
 #endif'
 
 `#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
-#  define CALL_BIF(F)	movl $CSYM(F), P_BIF_CALLEE(P); call CSYM(hipe_debug_bif_wrapper) 
+#  define CALL_BIF(F)	movl $CSYM(nbif_impl_##F), P_BIF_CALLEE(P); call CSYM(hipe_debug_bif_wrapper) 
 #else
-#  define CALL_BIF(F)	call	CSYM(F)
+#  define CALL_BIF(F)	call	CSYM(nbif_impl_##F)
 #endif'
 
 define(TEST_GOT_MBUF,`movl P_MBUF(P), %edx	/* `TEST_GOT_MBUF' */
@@ -666,13 +666,9 @@ noproc_primop_interface_0(nbif_handle_fp_exception, erts_restore_fpu)
 #endif /* NO_FPE_SIGNALS */
 
 /*
- * Implement gc_bif_interface_0 as nofail_primop_interface_0.
- */
-define(gc_bif_interface_0,`nofail_primop_interface_0($1, $2)')
-
-/*
- * Implement gc_bif_interface_N as standard_bif_interface_N (N=1,2,3).
+ * Implement gc_bif_interface_N as standard_bif_interface_N.
  */
+define(gc_bif_interface_0,`standard_bif_interface_0($1, $2)')
 define(gc_bif_interface_1,`standard_bif_interface_1($1, $2)')
 define(gc_bif_interface_2,`standard_bif_interface_2($1, $2)')
 define(gc_bif_interface_3,`standard_bif_interface_3($1, $2)')
diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index 2e48c475d5..7c9927c4f3 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -53,6 +53,7 @@ MODULES= \
 	crypto_SUITE \
 	ddll_SUITE \
 	decode_packet_SUITE \
+	dirty_bif_SUITE \
 	dirty_nif_SUITE \
 	distribution_SUITE \
 	driver_SUITE \
diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl
index f7ff04430a..2e303ba9a8 100644
--- a/erts/emulator/test/call_trace_SUITE.erl
+++ b/erts/emulator/test/call_trace_SUITE.erl
@@ -45,7 +45,7 @@ suite() ->
     [{ct_hooks,[ts_install_cth]},
      {timetrap, {seconds, 30}}].
 
-all() -> 
+all() ->
     Common = [errors, on_load],
     NotHipe = [process_specs, basic, flags, pam, change_pam,
                upgrade,
diff --git a/erts/emulator/test/dirty_bif_SUITE.erl b/erts/emulator/test/dirty_bif_SUITE.erl
new file mode 100644
index 0000000000..01ff8d6efd
--- /dev/null
+++ b/erts/emulator/test/dirty_bif_SUITE.erl
@@ -0,0 +1,583 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2014. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(dirty_bif_SUITE).
+
+%%-define(line_trace,true).
+-define(CHECK(Exp,Got), check(Exp,Got,?LINE)).
+%%-define(CHECK(Exp,Got), Exp = Got).
+
+-include_lib("common_test/include/ct.hrl").
+
+-export([all/0, suite/0,
+	 init_per_suite/1, end_per_suite/1,
+	 init_per_testcase/2, end_per_testcase/2,
+	 dirty_bif/1, dirty_bif_exception/1,
+	 dirty_bif_multischedule/1,
+	 dirty_bif_multischedule_exception/1,
+	 dirty_scheduler_exit/1,
+	 dirty_call_while_terminated/1,
+	 dirty_heap_access/1,
+	 dirty_process_info/1,
+	 dirty_process_register/1,
+	 dirty_process_trace/1,
+	 code_purge/1]).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+%%
+%% All these tests utilize the debug BIFs:
+%% - erts_debug:dirty_cpu/2 - Statically determined
+%%   to (begin to) execute on a dirty CPU scheduler.
+%% - erts_debug:dirty_io/2 - Statically determined
+%%   to (begin to) execute on a dirty IO scheduler.
+%% - erts_debug:dirty/3
+%% Their implementations are located in
+%% $ERL_TOP/erts/emulator/beam/beam_debug.c
+%%
+
+all() ->
+    [dirty_bif,
+     dirty_bif_multischedule,
+     dirty_bif_exception,
+     dirty_bif_multischedule_exception,
+     dirty_scheduler_exit,
+     dirty_call_while_terminated,
+     dirty_heap_access,
+     dirty_process_info,
+     dirty_process_register,
+     dirty_process_trace,
+     code_purge].
+
+init_per_suite(Config) ->
+    try erlang:system_info(dirty_cpu_schedulers) of
+	N when is_integer(N), N > 0 ->
+	    Config
+    catch _:_ ->
+	    {skipped, "No dirty scheduler support"}
+    end.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_testcase(Case, Config) ->
+    [{testcase, Case} | Config].
+
+end_per_testcase(_Case, _Config) ->
+    ok.
+
+dirty_bif(Config) when is_list(Config) ->
+    dirty_cpu = erts_debug:dirty_cpu(scheduler,type),
+    dirty_io = erts_debug:dirty_io(scheduler,type),
+    normal = erts_debug:dirty(normal,scheduler,type),
+    dirty_cpu = erts_debug:dirty(dirty_cpu,scheduler,type),
+    dirty_io = erts_debug:dirty(dirty_io,scheduler,type),
+    ok.
+
+dirty_bif_multischedule(Config) when is_list(Config) ->
+    ok = erts_debug:dirty_cpu(reschedule,1000),
+    ok = erts_debug:dirty_io(reschedule,1000),
+    ok = erts_debug:dirty(normal,reschedule,1000),
+    ok.
+    
+
+dirty_bif_exception(Config) when is_list(Config) ->
+    lists:foreach(fun (Error) ->
+			  ErrorType = case Error of
+					  _ when is_atom(Error) -> Error;
+					  _ -> badarg
+				      end,
+			  try
+			      erts_debug:dirty_cpu(error, Error),
+			      ct:fail(expected_exception)
+			  catch
+			      error:ErrorType ->
+				  [{erts_debug,dirty_cpu,[error, Error],_}|_]
+				      = erlang:get_stacktrace(),
+				  ok
+			  end,
+			  try
+			      apply(erts_debug,dirty_cpu,[error, Error]),
+			      ct:fail(expected_exception)
+			  catch
+			      error:ErrorType ->
+				  [{erts_debug,dirty_cpu,[error, Error],_}|_]
+				      = erlang:get_stacktrace(),
+				  ok
+			  end,
+			  try
+			      erts_debug:dirty_io(error, Error),
+			      ct:fail(expected_exception)
+			  catch
+			      error:ErrorType ->
+				  [{erts_debug,dirty_io,[error, Error],_}|_]
+				      = erlang:get_stacktrace(),
+				  ok
+			  end,
+			  try
+			      apply(erts_debug,dirty_io,[error, Error]),
+			      ct:fail(expected_exception)
+			  catch
+			      error:ErrorType ->
+				  [{erts_debug,dirty_io,[error, Error],_}|_]
+				      = erlang:get_stacktrace(),
+				  ok
+			  end,
+			  try
+			      erts_debug:dirty(normal, error, Error),
+			      ct:fail(expected_exception)
+			  catch
+			      error:ErrorType ->
+				  [{erts_debug,dirty,[normal, error, Error],_}|_]
+				      = erlang:get_stacktrace(),
+				  ok
+			  end,
+			  try
+			      apply(erts_debug,dirty,[normal, error, Error]),
+			      ct:fail(expected_exception)
+			  catch
+			      error:ErrorType ->
+				  [{erts_debug,dirty,[normal, error, Error],_}|_]
+				      = erlang:get_stacktrace(),
+				  ok
+			  end,
+			  try
+			      erts_debug:dirty(dirty_cpu, error, Error),
+			      ct:fail(expected_exception)
+			  catch
+			      error:ErrorType ->
+				  [{erts_debug,dirty,[dirty_cpu, error, Error],_}|_]
+				      = erlang:get_stacktrace(),
+				  ok
+			  end,
+			  try
+			      apply(erts_debug,dirty,[dirty_cpu, error, Error]),
+			      ct:fail(expected_exception)
+			  catch
+			      error:ErrorType ->
+				  [{erts_debug,dirty,[dirty_cpu, error, Error],_}|_]
+				      = erlang:get_stacktrace(),
+				  ok
+			  end,
+			  try
+			      erts_debug:dirty(dirty_io, error, Error),
+			      ct:fail(expected_exception)
+			  catch
+			      error:ErrorType ->
+				  [{erts_debug,dirty,[dirty_io, error, Error],_}|_]
+				      = erlang:get_stacktrace(),
+				  ok
+			  end,
+			  try
+			      apply(erts_debug,dirty,[dirty_io, error, Error]),
+			      ct:fail(expected_exception)
+			  catch
+			      error:ErrorType ->
+				  [{erts_debug,dirty,[dirty_io, error, Error],_}|_]
+				      = erlang:get_stacktrace(),
+				  ok
+			  end
+		  end,
+		  [badarg, undef, badarith, system_limit, noproc,
+		   make_ref(), {another, "heap", term_to_binary("term")}]),
+    ok.
+
+
+dirty_bif_multischedule_exception(Config) when is_list(Config) ->
+    try
+	erts_debug:dirty_cpu(reschedule,1001)
+    catch
+	error:badarg ->
+	    [{erts_debug,dirty_cpu,[reschedule, 1001],_}|_]
+		= erlang:get_stacktrace(),
+	    ok
+    end,
+    try
+	erts_debug:dirty_io(reschedule,1001)
+    catch
+	error:badarg ->
+	    [{erts_debug,dirty_io,[reschedule, 1001],_}|_]
+		= erlang:get_stacktrace(),
+	    ok
+    end,
+    try
+	erts_debug:dirty(normal,reschedule,1001)
+    catch
+	error:badarg ->
+	    [{erts_debug,dirty,[normal,reschedule,1001],_}|_]
+		= erlang:get_stacktrace(),
+	    ok
+    end.
+
+dirty_scheduler_exit(Config) when is_list(Config) ->
+    {ok, Node} = start_node(Config, "+SDio 1"),
+    [ok] = mcall(Node,
+                 [fun() ->
+			  Start = erlang:monotonic_time(millisecond),
+                          ok = test_dirty_scheduler_exit(),
+			  End = erlang:monotonic_time(millisecond),
+			  io:format("Time=~p ms~n", [End-Start]),
+			  ok
+                  end]),
+    stop_node(Node),
+    ok.
+
+test_dirty_scheduler_exit() ->
+    process_flag(trap_exit,true),
+    test_dse(10,[]).
+test_dse(0,Pids) ->
+    timer:sleep(100),
+    kill_dse(Pids,[]);
+test_dse(N,Pids) ->
+    Pid = spawn_link(fun () -> erts_debug:dirty_io(wait, 5000) end),
+    test_dse(N-1,[Pid|Pids]).
+
+kill_dse([],Killed) ->
+    wait_dse(Killed);
+kill_dse([Pid|Pids],AlreadyKilled) ->
+    exit(Pid,kill),
+    kill_dse(Pids,[Pid|AlreadyKilled]).
+
+wait_dse([]) ->
+    ok;
+wait_dse([Pid|Pids]) ->
+    receive
+        {'EXIT',Pid,Reason} ->
+	    killed = Reason
+    end,
+    wait_dse(Pids).
+
+dirty_call_while_terminated(Config) when is_list(Config) ->
+    Me = self(),
+    Bin = list_to_binary(lists:duplicate(4711, $r)),
+    {value, {BinAddr, 4711, 1}} = lists:keysearch(4711, 2,
+						  element(2,
+							  process_info(self(),
+								       binary))),
+    {Dirty, DM} = spawn_opt(fun () ->
+				    erts_debug:dirty_cpu(alive_waitexiting, Me),
+				    blipp:blupp(Bin)
+			    end,
+			    [monitor,link]),
+    receive {alive, Dirty} -> ok end,
+    {value, {BinAddr, 4711, 2}} = lists:keysearch(4711, 2,
+						  element(2,
+							  process_info(self(),
+								       binary))),
+    Reason = die_dirty_process,
+    OT = process_flag(trap_exit, true),
+    exit(Dirty, Reason),
+    receive
+	{'DOWN', DM, process, Dirty, R0} ->
+	    R0 = Reason
+    end,
+    receive
+	{'EXIT', Dirty, R1} ->
+	    R1 = Reason
+    end,
+    undefined = process_info(Dirty),
+    undefined = process_info(Dirty, status),
+    false = erlang:is_process_alive(Dirty),
+    false = lists:member(Dirty, processes()),
+    %% Binary still refered by Dirty process not yet cleaned up
+    %% since the dirty bif has not yet returned...
+    {value, {BinAddr, 4711, 2}} = lists:keysearch(4711, 2,
+						  element(2,
+							  process_info(self(),
+								       binary))),
+    receive after 2000 -> ok end,
+    receive
+	Msg ->
+	    ct:fail({unexpected_message, Msg})
+    after
+	0 ->
+	    ok
+    end,
+    {value, {BinAddr, 4711, 1}} = lists:keysearch(4711, 2,
+						  element(2,
+							  process_info(self(),
+								       binary))),
+    process_flag(trap_exit, OT),
+    try
+	blipp:blupp(Bin)
+    catch
+	_ : _ -> ok
+    end.
+
+dirty_heap_access(Config) when is_list(Config) ->
+    {ok, Node} = start_node(Config),
+    Me = self(),
+    RGL = rpc:call(Node,erlang,whereis,[init]),
+    Ref = rpc:call(Node,erlang,make_ref,[]),
+    Dirty = spawn_link(fun () ->
+			       Res = erts_debug:dirty_cpu(copy, Ref),
+			       garbage_collect(),
+			       Me ! {self(), Res},
+			       receive after infinity -> ok end
+		       end),
+    {N, R} = access_dirty_heap(Dirty, RGL, 0, 0),
+    receive
+	{_Pid, Res} ->
+	    1000 = length(Res),
+	    lists:foreach(fun (X) -> Ref = X end, Res)
+    end,
+    unlink(Dirty),
+    exit(Dirty, kill),
+    stop_node(Node),
+    {comment, integer_to_list(N) ++ " GL change loops; "
+     ++ integer_to_list(R) ++ " while running dirty"}.
+
+access_dirty_heap(Dirty, RGL, N, R) ->
+    case process_info(Dirty, status) of
+	{status, waiting} ->
+	    {N, R};
+	{status, Status} ->
+	    {group_leader, GL} = process_info(Dirty, group_leader),
+	    true = group_leader(RGL, Dirty),
+	    {group_leader, RGL} = process_info(Dirty, group_leader),
+	    true = group_leader(GL, Dirty),
+	    {group_leader, GL} = process_info(Dirty, group_leader),
+	    access_dirty_heap(Dirty, RGL, N+1, case Status of
+						   running ->
+						       R+1;
+						   _ ->
+						       R
+					       end)
+    end.
+
+%% These tests verify that processes that access a process executing a
+%% dirty BIF where the main lock is needed for that access do not get
+%% blocked. Each test passes its pid to dirty_sleeper, which sends an
+%% 'alive' message when it's running on a dirty scheduler and just before
+%% it starts a 6 second sleep. When it receives the message, it verifies
+%% that access to the dirty process is as it expects.  After the dirty
+%% process finishes its 6 second sleep but before it returns from the dirty
+%% scheduler, it sends a 'done' message. If the tester already received
+%% that message, the test fails because it means attempting to access the
+%% dirty process waited for that process to return to a regular scheduler,
+%% so verify that we haven't received that message, and also verify that
+%% the dirty process is still alive immediately after accessing it.
+dirty_process_info(Config) when is_list(Config) ->
+    access_dirty_process(
+      Config,
+      fun() -> ok end,
+      fun(BifPid) ->
+	      PI = process_info(BifPid),
+	      {current_function,{erts_debug,dirty_io,2}} =
+		  lists:keyfind(current_function, 1, PI),
+	      ok
+      end,
+      fun(_) -> ok end).
+
+dirty_process_register(Config) when is_list(Config) ->
+    access_dirty_process(
+      Config,
+      fun() -> ok end,
+      fun(BifPid) ->
+	      register(test_dirty_process_register, BifPid),
+	      BifPid = whereis(test_dirty_process_register),
+	      unregister(test_dirty_process_register),
+	      false = lists:member(test_dirty_process_register,
+				   registered()),
+	      ok
+      end,
+      fun(_) -> ok end).
+
+dirty_process_trace(Config) when is_list(Config) ->
+    access_dirty_process(
+      Config,
+      fun() ->
+	      erlang:trace_pattern({erts_debug,dirty_io,2},
+				   [{'_',[],[{return_trace}]}],
+				   [local,meta]),
+	      ok
+      end,
+      fun(BifPid) ->
+	      erlang:trace(BifPid, true, [call,timestamp]),
+	      ok
+      end,
+      fun(BifPid) ->
+	      receive
+		  {done, BifPid} ->
+		      receive
+			  {trace_ts,BifPid,call,{erts_debug,dirty_io,_},_} ->
+			      ok
+		      after
+			  0 ->
+			      error(missing_trace_call_message)
+		      end %%,
+		      %% receive
+		      %% 	  {trace_ts,BifPid,return_from,{erts_debug,dirty_io,2},
+		      %% 	   ok,_} ->
+		      %% 	      ok
+		      %% after
+		      %% 	  100 ->
+		      %% 	      error(missing_trace_return_message)
+		      %% end
+	      after
+		  6500 ->
+		      error(missing_done_message)
+	      end,
+	      ok
+      end).
+
+dirty_code_test_code() ->
+    "
+-module(dirty_code_test).
+
+-export([func/1]).
+
+func(Fun) ->
+  Fun(),
+  blipp:blapp().
+
+".
+
+code_purge(Config) when is_list(Config) ->
+    Path = ?config(data_dir, Config),
+    File = filename:join(Path, "dirty_code_test.erl"),
+    ok = file:write_file(File, dirty_code_test_code()),
+    {ok, dirty_code_test, Bin} = compile:file(File, [binary]),
+    {module, dirty_code_test} = erlang:load_module(dirty_code_test, Bin),
+    Start = erlang:monotonic_time(),
+    {Pid1, Mon1} = spawn_monitor(fun () ->
+				       dirty_code_test:func(fun () ->
+								    %% Sleep for 6 seconds
+								    %% in dirty bif...
+								    erts_debug:dirty_io(wait,6000)
+							    end)
+			       end),
+    {module, dirty_code_test} = erlang:load_module(dirty_code_test, Bin),
+    {Pid2, Mon2} = spawn_monitor(fun () ->
+				       dirty_code_test:func(fun () ->
+								    %% Sleep for 6 seconds
+								    %% in dirty bif...
+								    erts_debug:dirty_io(wait,6000)
+							    end)
+				 end),
+    receive
+	{'DOWN', Mon1, process, Pid1, _} ->
+	    ct:fail(premature_death)
+    after 100 ->
+	    ok
+    end,
+    true = erlang:purge_module(dirty_code_test),
+    receive
+	{'DOWN', Mon1, process, Pid1, Reason1} ->
+	    killed = Reason1
+    end,
+    receive
+	{'DOWN', Mon2, process, Pid2, _} ->
+	    ct:fail(premature_death)
+    after 100 ->
+	    ok
+    end,
+    true = erlang:delete_module(dirty_code_test),
+    receive
+	{'DOWN', Mon2, process, Pid2, _} ->
+	    ct:fail(premature_death)
+    after 100 ->
+	    ok
+    end,
+    true = erlang:purge_module(dirty_code_test),
+    receive
+	{'DOWN', Mon2, process, Pid2, Reason2} ->
+	    killed = Reason2
+    end,
+    End = erlang:monotonic_time(),
+    Time = erlang:convert_time_unit(End-Start, native, milli_seconds),
+    io:format("Time=~p~n", [Time]),
+    true = Time =< 1000,
+    ok.
+
+%%
+%% Internal...
+%%
+
+access_dirty_process(Config, Start, Test, Finish) ->
+    {ok, Node} = start_node(Config, ""),
+    [ok] = mcall(Node,
+		 [fun() ->
+			  ok = test_dirty_process_access(Start, Test, Finish)
+		  end]),
+    stop_node(Node),
+    ok.
+
+test_dirty_process_access(Start, Test, Finish) ->
+    ok = Start(),
+    Self = self(),
+    BifPid = spawn_link(fun() ->
+				ok = erts_debug:dirty_io(ready_wait6_done, Self)
+			end),
+    ok = receive
+	     {ready, BifPid} ->
+		 ok = Test(BifPid),
+		 receive
+		     {done, BifPid} ->
+			 error(dirty_process_info_blocked)
+		 after
+		     0 ->
+			 true = erlang:is_process_alive(BifPid),
+			 ok
+		 end
+	 after
+	     3000 ->
+		 error(timeout)
+	 end,
+    ok = Finish(BifPid).
+
+receive_any() ->
+    receive M -> M end.
+
+start_node(Config) ->
+    start_node(Config, "").
+
+start_node(Config, Args) when is_list(Config) ->
+    Pa = filename:dirname(code:which(?MODULE)),
+    Name = list_to_atom(atom_to_list(?MODULE)
+			++ "-"
+			++ atom_to_list(proplists:get_value(testcase, Config))
+			++ "-"
+			++ integer_to_list(erlang:system_time(second))
+			++ "-"
+			++ integer_to_list(erlang:unique_integer([positive]))),
+    test_server:start_node(Name, slave, [{args, "-pa "++Pa++" "++Args}]).
+
+stop_node(Node) ->
+    test_server:stop_node(Node).
+
+mcall(Node, Funs) ->
+    Parent = self(),
+    Refs = lists:map(fun (Fun) ->
+                             Ref = make_ref(),
+                             spawn_link(Node,
+                                        fun () ->
+                                                Res = Fun(),
+                                                unlink(Parent),
+                                                Parent ! {Ref, Res}
+                                        end),
+                             Ref
+                     end, Funs),
+    lists:map(fun (Ref) ->
+                      receive
+                          {Ref, Res} ->
+                              Res
+                      end
+              end, Refs).
diff --git a/erts/emulator/test/dirty_bif_SUITE_data/.gitignore b/erts/emulator/test/dirty_bif_SUITE_data/.gitignore
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/erts/emulator/test/old_scheduler_SUITE.erl b/erts/emulator/test/old_scheduler_SUITE.erl
index ffe7d40139..8515a87df8 100644
--- a/erts/emulator/test/old_scheduler_SUITE.erl
+++ b/erts/emulator/test/old_scheduler_SUITE.erl
@@ -64,11 +64,11 @@ all() ->
 init_per_testcase(_Case, Config) ->
     %% main test process needs max prio
     Prio = process_flag(priority, max),
-    MS = erlang:system_flag(multi_scheduling, block),
+    MS = erlang:system_flag(multi_scheduling, block_normal),
     [{prio,Prio},{multi_scheduling, MS}|Config].
 
 end_per_testcase(_Case, Config) ->
-    erlang:system_flag(multi_scheduling, unblock),
+    erlang:system_flag(multi_scheduling, unblock_normal),
     Prio=proplists:get_value(prio, Config),
     process_flag(priority, Prio),
     ok.
diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index 23594aa8c4..2a13b2d2f4 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -2066,13 +2066,13 @@ exit_status_msb_test(Config, SleepSecs) when is_list(Config) ->
     StartedTime = (erlang:monotonic_time(microsecond) - Start)/1000000,
     io:format("StartedTime = ~p~n", [StartedTime]),
     true = StartedTime < SleepSecs,
-    erlang:system_flag(multi_scheduling, block),
+    erlang:system_flag(multi_scheduling, block_normal),
     lists:foreach(fun (P) -> receive {P, done} -> ok end end, Procs),
     DoneTime = (erlang:monotonic_time(microsecond) - Start)/1000000,
     io:format("DoneTime = ~p~n", [DoneTime]),
     true = DoneTime > SleepSecs,
     ok = verify_multi_scheduling_blocked(),
-    erlang:system_flag(multi_scheduling, unblock),
+    erlang:system_flag(multi_scheduling, unblock_normal),
     case {length(lists:usort(lists:flatten(SIds))), NoSchedsOnln} of
         {N, N} ->
             ok;
diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index 5712c9fa74..e14185e881 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -1028,9 +1028,9 @@ low_prio(Config) when is_list(Config) ->
 	1 ->
 	    ok = low_prio_test(Config);
 	_ -> 
-	    erlang:system_flag(multi_scheduling, block),
+	    erlang:system_flag(multi_scheduling, block_normal),
 	    ok = low_prio_test(Config),
-	    erlang:system_flag(multi_scheduling, unblock),
+	    erlang:system_flag(multi_scheduling, unblock_normal),
 	    {comment,
 		   "Test not written for SMP runtime system. "
 		   "Multi scheduling blocked during test."}
@@ -1097,9 +1097,9 @@ yield(Config) when is_list(Config) ->
 	     ++ ") is enabled. Testcase gets messed up by modfied "
 	     "timing."};
 	_ ->
-	    MS = erlang:system_flag(multi_scheduling, block),
+	    MS = erlang:system_flag(multi_scheduling, block_normal),
 	    yield_test(),
-	    erlang:system_flag(multi_scheduling, unblock),
+	    erlang:system_flag(multi_scheduling, unblock_normal),
 	    case MS of
 		blocked ->
 		    {comment,
@@ -1679,7 +1679,7 @@ processes_bif_test() ->
 	true ->
 	    %% Do it again with a process suspended while
 	    %% in the processes/0 bif.
-	    erlang:system_flag(multi_scheduling, block),
+	    erlang:system_flag(multi_scheduling, block_normal),
 	    Suspendee = spawn_link(fun () ->
 						 Tester ! {suspend_me, self()},
 						 Tester ! {self(),
@@ -1692,7 +1692,7 @@ processes_bif_test() ->
 					 end),
 	    receive {suspend_me, Suspendee} -> ok end,
 	    erlang:suspend_process(Suspendee),
-	    erlang:system_flag(multi_scheduling, unblock),
+	    erlang:system_flag(multi_scheduling, unblock_normal),
 	    
 	    [{status,suspended},{current_function,{erlang,ptab_list_continue,2}}] =
 		process_info(Suspendee, [status, current_function]),
@@ -1732,10 +1732,10 @@ do_processes_bif_test(WantReds, DieTest, Processes) ->
 		    Splt = NoTestProcs div 10,
 		    {TP1, TP23} = lists:split(Splt, TestProcs),
 		    {TP2, TP3} = lists:split(Splt, TP23),
-		    erlang:system_flag(multi_scheduling, block),
+		    erlang:system_flag(multi_scheduling, block_normal),
 		    Tester ! DoIt,
 		    receive GetGoing -> ok end,
-		    erlang:system_flag(multi_scheduling, unblock),
+		    erlang:system_flag(multi_scheduling, unblock_normal),
 		    SpawnProcesses(high),
 		    lists:foreach( fun (P) ->
 				SpawnHangAround(),
@@ -1944,7 +1944,7 @@ processes_gc_trap(Config) when is_list(Config) ->
 	    processes()
     end,
 
-    erlang:system_flag(multi_scheduling, block),
+    erlang:system_flag(multi_scheduling, block_normal),
     Suspendee = spawn_link(fun () ->
 					 Tester ! {suspend_me, self()},
 					 Tester ! {self(),
@@ -1954,7 +1954,7 @@ processes_gc_trap(Config) when is_list(Config) ->
 				 end),
     receive {suspend_me, Suspendee} -> ok end,
     erlang:suspend_process(Suspendee),
-    erlang:system_flag(multi_scheduling, unblock),
+    erlang:system_flag(multi_scheduling, unblock_normal),
 	    
     [{status,suspended}, {current_function,{erlang,ptab_list_continue,2}}]
 	= process_info(Suspendee, [status, current_function]),
@@ -2161,7 +2161,7 @@ processes_term_proc_list_test(MustChk) ->
 		end)
     end,
     SpawnSuspendProcessesProc = fun () ->
-		  erlang:system_flag(multi_scheduling, block),
+		  erlang:system_flag(multi_scheduling, block_normal),
 		  P = spawn_link(fun () ->
 					 Tester ! {suspend_me, self()},
 					 Tester ! {self(),
@@ -2171,7 +2171,7 @@ processes_term_proc_list_test(MustChk) ->
 				 end),
 		  receive {suspend_me, P} -> ok end,
 		  erlang:suspend_process(P),
-		  erlang:system_flag(multi_scheduling, unblock),
+		  erlang:system_flag(multi_scheduling, unblock_normal),
 		  [{status,suspended},
 		   {current_function,{erlang,ptab_list_continue,2}}]
 		      = process_info(P, [status, current_function]),
@@ -2232,7 +2232,7 @@ processes_term_proc_list_test(MustChk) ->
     S8 = SpawnSuspendProcessesProc(),
     ?CHK_TERM_PROC_LIST(MustChk, 7),
 
-    erlang:system_flag(multi_scheduling, block),
+    erlang:system_flag(multi_scheduling, block_normal),
     Exit(S8),
     ?CHK_TERM_PROC_LIST(MustChk, 7),
     Exit(S5),
@@ -2241,7 +2241,7 @@ processes_term_proc_list_test(MustChk) ->
     ?CHK_TERM_PROC_LIST(MustChk, 6),
     Exit(S6),
     ?CHK_TERM_PROC_LIST(MustChk, 0),
-    erlang:system_flag(multi_scheduling, unblock),
+    erlang:system_flag(multi_scheduling, unblock_normal),
     as_expected.
 
 
diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl
index ab56018373..d663cc548c 100644
--- a/erts/emulator/test/z_SUITE.erl
+++ b/erts/emulator/test/z_SUITE.erl
@@ -68,8 +68,8 @@ schedulers_alive(Config) when is_list(Config) ->
               enabled ->
                   io:format("Testing blocking process exit~n"),
                   BF = fun () ->
-                               blocked = erlang:system_flag(multi_scheduling,
-                                                            block),
+                               blocked_normal = erlang:system_flag(multi_scheduling,
+								   block_normal),
                                Master ! {self(), blocking},
                                receive after infinity -> ok end
                        end,
@@ -77,21 +77,21 @@ schedulers_alive(Config) when is_list(Config) ->
                   Mon = erlang:monitor(process, Blocker),
                   receive {Blocker, blocking} -> ok end,
                   [Blocker]
-                  = erlang:system_info(multi_scheduling_blockers),
+                  = erlang:system_info(normal_multi_scheduling_blockers),
                   unlink(Blocker),
                   exit(Blocker, kill),
                   receive {'DOWN', Mon, _, _, _} -> ok end,
                   enabled = erlang:system_info(multi_scheduling),
-                  [] = erlang:system_info(multi_scheduling_blockers),
+                  [] = erlang:system_info(normal_multi_scheduling_blockers),
                   ok
           end,
           io:format("Testing blocked~n"),
-          erlang:system_flag(multi_scheduling, block),
+          erlang:system_flag(multi_scheduling, block_normal),
           case erlang:system_info(multi_scheduling) of
               enabled ->
                   ct:fail(multi_scheduling_enabled);
-              blocked ->
-                  [Master] = erlang:system_info(multi_scheduling_blockers);
+              blocked_normal ->
+                  [Master] = erlang:system_info(normal_multi_scheduling_blockers);
               disabled -> ok
           end,
           Ps = lists:map(
@@ -109,8 +109,8 @@ schedulers_alive(Config) when is_list(Config) ->
                                 unlink(P),
                                 exit(P, bang)
                         end, Ps),
-          case erlang:system_flag(multi_scheduling, unblock) of
-              blocked -> ct:fail(multi_scheduling_blocked);
+          case erlang:system_flag(multi_scheduling, unblock_normal) of
+              blocked_normal -> ct:fail(multi_scheduling_blocked);
               disabled -> ok;
               enabled -> ok
           end,
diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables
index 27f9dcc878..1d3023ee9a 100755
--- a/erts/emulator/utils/make_tables
+++ b/erts/emulator/utils/make_tables
@@ -36,6 +36,9 @@ use File::Basename;
 #    <-src>/erl_am.c
 #    <-src>/erl_bif_table.c
 #    <-src>/erl_bif_wrap.c
+#    <-src>/erl_dirty_bif_wrap.c
+#    <-src>/hipe_nbif_impl.c
+#    <-include>/hipe_nbif_impl.h
 #    <-include>/erl_atom_table.h
 #    <-include>/erl_bif_table.h
 #
@@ -51,9 +54,13 @@ my %atom;
 my %atom_alias;
 my %aliases;
 my $auto_alias_num = 0;
+my %dirty_bif_tab;
 
 my @bif;
-my @bif_type;
+my @bif_info;
+my $dirty_schedulers = 'no';
+my $dirty_schedulers_test = 'no';
+my $hipe = 'no';
 
 while (@ARGV && $ARGV[0] =~ /^-(\w+)/) {
     my $opt = shift;
@@ -65,6 +72,18 @@ while (@ARGV && $ARGV[0] =~ /^-(\w+)/) {
 	$include = shift;
 	die "No directory for -include argument specified"
 	    unless defined $include;
+    } elsif($opt eq '-ds') {
+	$dirty_schedulers = shift;
+	die "No -ds argument specified"
+	    unless defined $dirty_schedulers;
+    } elsif($opt eq '-dst') {
+	$dirty_schedulers_test = shift;
+	die "No -dst argument specified"
+	    unless defined $dirty_schedulers_test;
+    } elsif($opt eq '-hipe') {
+	$hipe = shift;
+	die "No -hipe argument specified"
+	    unless defined $hipe;
     } else {
 	usage("bad option: $opt");
     }
@@ -84,12 +103,31 @@ while (<>) {
 	my($bif,$alias) = (@args);
 	$bif =~ m@^([a-z_.'0-9]+):(.*)/(\d)$@ or error("invalid BIF");
 	my($mod,$name,$arity) = ($1,$2,$3);
+	my $mfa = "$mod:$name/$arity";
 	save_atoms($mod, $name);
 	unless (defined $alias) {
 	    $alias = "";
 	    $alias = "${mod}_" unless $mod eq 'erlang';
 	    $alias .= "${name}_$arity";
 	}
+	my $sched_type;
+	my $alias3 = $alias;
+
+	$sched_type = $dirty_bif_tab{$mfa};
+
+	if (!$sched_type or ($type eq 'ubif')) {
+	    $sched_type = 'normal';
+	}
+	elsif ($sched_type eq 'dirty_cpu') {
+	    $alias3 = "schedule_dirty_cpu_$alias"
+	}
+	elsif ($sched_type eq 'dirty_io') {
+	    $alias3 = "schedule_dirty_io_$alias"
+	}
+	else {
+	    error("invalid sched_type: $sched_type");
+	}
+
 	my $wrapper;
 	if ($type eq 'bif') {
 	    $wrapper = "wrap_$alias";
@@ -97,8 +135,25 @@ while (<>) {
 	    $wrapper = $alias;
 	}
 	push(@bif, ["am_$atom_alias{$mod}","am_$atom_alias{$name}",$arity,
-		    $alias,$wrapper]);
-	push(@bif_type, $type);
+		    $alias3,$wrapper,$alias]);
+	push(@bif_info, [$type, $sched_type, $alias3, $alias]);
+    } elsif ($type eq 'dirty-cpu' or $type eq 'dirty-io'
+	     or $type eq 'dirty-cpu-test' or $type eq 'dirty-io-test') {
+	if ($dirty_schedulers eq 'yes') {
+	    my($bif,$other) = (@args);
+	    $bif =~ m@^([a-z_.'0-9]+):(.*)/(\d)$@ or error("invalid BIF");
+	    my($mod,$name,$arity) = ($1,$2,$3);
+	    my $mfa = "$mod:$name/$arity";
+	    if (($type eq 'dirty-cpu')
+		or (($dirty_schedulers_test eq 'yes')
+		    and ($type eq 'dirty-cpu-test'))) {
+		$dirty_bif_tab{$mfa} = 'dirty_cpu';
+	    } elsif (($type eq 'dirty-io')
+		or (($dirty_schedulers_test eq 'yes')
+		    and ($type eq 'dirty-io-test'))) {
+		$dirty_bif_tab{$mfa} = 'dirty_io';
+	    }
+	}
     } else {
 	error("invalid line");
     }
@@ -147,7 +202,7 @@ open_file("$include/erl_bif_list.h");
 my $i;
 for ($i = 0; $i < @bif; $i++) {
     # module atom, function atom, arity, C function, table index
-    print "BIF_LIST($bif[$i]->[0],$bif[$i]->[1],$bif[$i]->[2],$bif[$i]->[3],$i)\n";
+    print "BIF_LIST($bif[$i]->[0],$bif[$i]->[1],$bif[$i]->[2],$bif[$i]->[3],$bif[$i]->[5],$i)\n";
 }
 
 #
@@ -167,6 +222,7 @@ typedef struct bif_entry {
     int arity;
     BifFunction f;
     BifFunction traced;
+    BifFunction impl;
 } BifEntry;
 
 typedef struct erts_gc_bif {
@@ -184,21 +240,28 @@ EOF
 
 my $i;
 for ($i = 0; $i < @bif; $i++) {
-    print "#define BIF_$bif[$i]->[3] $i\n";
+    print "#define BIF_$bif_info[$i]->[3] $i\n";
 }
 
 print "\n";
 
 for ($i = 0; $i < @bif; $i++) {
-    my $args = join(', ', 'Process*', 'Eterm*');
-    my $name = $bif[$i]->[3];
+    my $args = join(', ', 'Process*', 'Eterm*', 'UWord*');
+    my $name = $bif_info[$i]->[3];
     print "Eterm $name($args);\n";
-    print "Eterm wrap_$name($args, UWord *I);\n";
+    print "Eterm wrap_$name($args);\n";
     print "Eterm erts_gc_$name(Process* p, Eterm* reg, Uint live);\n"
-	if $bif_type[$i] eq 'gcbif';
+	if $bif_info[$i]->[0] eq 'gcbif';
+    print "Eterm $bif_info[$i]->[2]($args);\n"
+	unless $bif_info[$i]->[1] eq 'normal';
     print "\n";
 }
-print "#endif\n";
+
+if ($hipe eq 'yes') {
+    print "\n#include \"hipe_nbif_impl.h\"\n";
+}
+
+print "\n#endif\n";
 
 #
 # Generate the bif table file.
@@ -229,7 +292,7 @@ includes("export.h", "sys.h", "erl_vm.h", "global.h", "erl_process.h", "bif.h",
 for ($i = 0; $i < @bif; $i++) {
     next if $bif[$i]->[3] eq $bif[$i]->[4]; # Skip unwrapped bifs
     my $arity = $bif[$i]->[2];
-    my $func = $bif[$i]->[3];
+    my $func = $bif_info[$i]->[3];
     print "Eterm\n";
     print "wrap_$func(Process* p, Eterm* args, UWord* I)\n";
     print "{\n";
@@ -247,7 +310,7 @@ includes("export.h", "sys.h", "erl_vm.h", "global.h", "erl_process.h", "bif.h",
 	 "erl_bif_table.h");
 print "const ErtsGcBif erts_gc_bifs[] = {\n";
 for ($i = 0; $i < @bif; $i++) {
-    next unless $bif_type[$i] eq 'gcbif';
+    next unless $bif_info[$i]->[0] eq 'gcbif';
     my $arity = $bif[$i]->[2];
     my $func = $bif[$i]->[3];
     print "  {$func, erts_gc_$func},\n";
@@ -255,6 +318,80 @@ for ($i = 0; $i < @bif; $i++) {
 print "  {0, 0}\n";
 print "};\n";
 
+#
+# Generate the dirty bif wrappers file.
+#
+
+open_file("$src/erl_dirty_bif_wrap.c");
+my $i;
+includes("erl_process.h", "erl_nfunc_sched.h", "erl_bif_table.h", "erl_atom_table.h");
+for ($i = 0; $i < @bif_info; $i++) {
+    next if $bif_info[$i]->[1] eq 'normal';
+    my $dtype;
+    if ($bif_info[$i]->[1] eq 'dirty_cpu') {
+	$dtype = "ERTS_SCHED_DIRTY_CPU";
+    }
+    else {
+	$dtype = "ERTS_SCHED_DIRTY_IO";
+    }
+print <<EOF;
+Eterm $bif_info[$i]->[2](Process *c_p, Eterm *regs, BeamInstr *I)
+{
+    return erts_reschedule_bif(c_p, regs, I, $bif_info[$i]->[3], $dtype);
+}
+
+EOF
+
+}
+
+if ($hipe eq 'yes') {
+
+    #
+    # Generate the nbif_impl bif wrappers file.
+    #
+
+    open_file("$src/hipe_nbif_impl.h");
+    print <<EOF;
+
+#ifndef HIPE_NBIF_IMPL_H__
+#define HIPE_NBIF_IMPL_H__
+
+EOF
+
+    my $i;
+    for ($i = 0; $i < @bif; $i++) {
+	print <<EOF;
+Eterm nbif_impl_$bif[$i]->[5](Process *c_p, Eterm *regs);
+EOF
+   }
+
+   print <<EOF;
+
+#endif /* ERL_HIPE_NBIF_IMPL_H__ */
+
+EOF
+
+    #
+    # Generate the nbif_impl bif wrappers file.
+    #
+
+    open_file("$src/hipe_nbif_impl.c");
+    my $i;
+    includes("erl_process.h", "erl_nfunc_sched.h", "erl_bif_table.h", "erl_atom_table.h");
+    for ($i = 0; $i < @bif; $i++) {
+
+	print <<EOF;
+Eterm nbif_impl_$bif[$i]->[5](Process *c_p, Eterm *regs)
+{
+    return $bif[$i]->[3](c_p, regs, (UWord *) bif_export\[BIF_$bif[$i]->[5]\]);
+}
+
+EOF
+
+   }
+
+} # hipe
+
 #
 # Utilities follow.
 #
-- 
cgit v1.2.3


From 95ec5d385cfba23c770d946871c0197bf374ff3c Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 28 Oct 2016 18:34:14 +0200
Subject: Optimize handling of BIF errors

---
 erts/emulator/Makefile.in            |   4 +-
 erts/emulator/beam/beam_emu.c        | 139 ++++++++++++++++++-----------------
 erts/emulator/beam/bif.c             |  25 +++----
 erts/emulator/beam/erl_nfunc_sched.c |   4 +-
 erts/emulator/beam/erl_nfunc_sched.h |  10 +--
 erts/emulator/beam/erl_nif.c         |   4 -
 erts/emulator/beam/error.h           |   1 -
 erts/emulator/utils/make_tables      |  24 ++++--
 8 files changed, 108 insertions(+), 103 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index eb27703d6a..18fd7f320b 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -589,7 +589,7 @@ $(TTF_DIR)/erl_bif_wrap.c \
 $(TTF_DIR)/erl_bif_list.h \
 $(TTF_DIR)/erl_atom_table.c \
 $(TTF_DIR)/erl_atom_table.h \
-$(TTF_DIR)/erl_gc_bifs.c \
+$(TTF_DIR)/erl_guard_bifs.c \
 $(TTF_DIR)/erl_dirty_bif_wrap.c \
 $(HIPE_NBIF_FILES) \
 	: $(TTF_DIR)/TABLES-GENERATED
@@ -775,7 +775,7 @@ RUN_OBJS = \
 	$(OBJDIR)/erl_bif_os.o		$(OBJDIR)/erl_bif_lists.o \
 	$(OBJDIR)/erl_bif_trace.o	$(OBJDIR)/erl_bif_unique.o \
 	$(OBJDIR)/erl_bif_wrap.o	$(OBJDIR)/erl_nfunc_sched.o \
-	$(OBJDIR)/erl_gc_bifs.o 	$(OBJDIR)/erl_dirty_bif_wrap.o \
+	$(OBJDIR)/erl_guard_bifs.o 	$(OBJDIR)/erl_dirty_bif_wrap.o \
 	$(OBJDIR)/erl_trace.o		$(OBJDIR)/copy.o \
 	$(OBJDIR)/utils.o		$(OBJDIR)/bif.o \
 	$(OBJDIR)/io.o 			$(OBJDIR)/erl_printf_term.o\
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index befd2989f9..cfdf61eed0 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1057,9 +1057,10 @@ do {                                            \
  * The following functions are called directly by process_main().
  * Don't inline them.
  */
-static BifFunction translate_gc_bif(void* gcf) NOINLINE;
+static ErtsCodeMFA *ubif2mfa(void* uf) NOINLINE;
+static ErtsCodeMFA *gcbif2mfa(void* gcf) NOINLINE;
 static BeamInstr* handle_error(Process* c_p, BeamInstr* pc,
-			       Eterm* reg, BifFunction bf) NOINLINE;
+			       Eterm* reg, ErtsCodeMFA* bif_mfa) NOINLINE;
 static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa,
 				     Eterm* reg, Eterm func) NOINLINE;
 static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity,
@@ -1088,7 +1089,7 @@ static BeamInstr* next_catch(Process* c_p, Eterm *reg);
 static void terminate_proc(Process* c_p, Eterm Value);
 static Eterm add_stacktrace(Process* c_p, Eterm Value, Eterm exc);
 static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg,
-			     BifFunction bf, Eterm args);
+			    ErtsCodeMFA *bif_mfa, Eterm args);
 static struct StackTrace * get_trace_from_exc(Eterm exc);
 static Eterm make_arglist(Process* c_p, Eterm* reg, int a);
 
@@ -2640,7 +2641,7 @@ do {						\
 	}
 	reg[0] = tmp_reg[0];
 	SWAPOUT;
-	I = handle_error(c_p, I, reg, bf);
+	I = handle_error(c_p, I, reg, ubif2mfa((void *) bf));
 	goto post_error_handling;
     }
 
@@ -2676,7 +2677,7 @@ do {						\
 	    Goto(*I);
 	}
 	x(0) = x(live);
-	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
+	I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf));
 	goto post_error_handling;
     }
 
@@ -2721,7 +2722,7 @@ do {						\
 	live--;
 	x(0) = x(live);
 	x(1) = x(live+1);
-	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
+	I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf));
 	goto post_error_handling;
     }
 
@@ -2767,7 +2768,7 @@ do {						\
 	x(0) = x(live);
 	x(1) = x(live+1);
 	x(2) = x(live+2);
-	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
+	I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf));
 	goto post_error_handling;
     }
 
@@ -2829,7 +2830,7 @@ do {						\
 	reg[0] = tmp_reg[0];
 	reg[1] = tmp_reg[1];
 	SWAPOUT;
-	I = handle_error(c_p, I, reg, bf);
+	I = handle_error(c_p, I, reg, ubif2mfa((void *) bf));
 	goto post_error_handling;
     }
 
@@ -2906,7 +2907,7 @@ do {						\
 	 * Error handling.  SWAPOUT is not needed because it was done above.
 	 */
 	ASSERT(c_p->stop == E);
-	I = handle_error(c_p, I, reg, bf);
+	I = handle_error(c_p, I, reg, &export->info.mfa);
 	goto post_error_handling;
     }
 
@@ -3210,7 +3211,7 @@ do {						\
 	 SET_I(next);
 	 Dispatch();
      }
-     I = handle_error(c_p, I, reg, apply_3);
+     I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa);
      goto post_error_handling;
  }
 
@@ -3225,7 +3226,7 @@ do {						\
 	 SET_I(next);
 	 Dispatch();
      }
-     I = handle_error(c_p, I, reg, apply_3);
+     I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa);
      goto post_error_handling;
  }
 
@@ -3238,7 +3239,7 @@ do {						\
 	 SET_I(next);
 	 Dispatch();
      }
-     I = handle_error(c_p, I, reg, apply_3);
+     I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa);
      goto post_error_handling;
  }
 
@@ -3253,7 +3254,7 @@ do {						\
 	 SET_I(next);
 	 Dispatch();
      }
-     I = handle_error(c_p, I, reg, apply_3);
+     I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa);
      goto post_error_handling;
  }
 
@@ -3269,7 +3270,7 @@ do {						\
 	 SET_I(next);
 	 Dispatch();
      }
-     I = handle_error(c_p, I, reg, apply_3);
+     I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa);
      goto post_error_handling;
  }
 
@@ -3690,7 +3691,7 @@ do {						\
 		}
 		Dispatch();
 	    }
-	    I = handle_error(c_p, c_p->cp, reg, vbf);
+	    I = handle_error(c_p, c_p->cp, reg, c_p->current);
 	    goto post_error_handling;
 	}
     }
@@ -5045,7 +5046,7 @@ do {						\
 	 goto do_schedule;
      } else {
 	 HEAVY_SWAPIN;
-	 I = handle_error(c_p, I, reg, hibernate_3);
+	 I = handle_error(c_p, I, reg, &bif_export[BIF_hibernate_3]->info.mfa);
 	 goto post_error_handling;
      }
  }
@@ -5437,17 +5438,28 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 #endif /* ERTS_DIRTY_SCHEDULERS */
 }
 
-static BifFunction
-translate_gc_bif(void* gcf)
+static ErtsCodeMFA *
+gcbif2mfa(void* gcf)
 {
-    const ErtsGcBif* p;
-
-    for (p = erts_gc_bifs; p->bif != 0; p++) {
-	if (p->gc_bif == gcf) {
-	    return p->bif;
-	}
+    int i;
+    for (i = 0; erts_gc_bifs[i].bif; i++) {
+	if (erts_gc_bifs[i].gc_bif == gcf)
+	    return &bif_export[erts_gc_bifs[i].exp_ix]->info.mfa;
     }
     erts_exit(ERTS_ERROR_EXIT, "bad gc bif");
+    return NULL;
+}
+
+static ErtsCodeMFA *
+ubif2mfa(void* uf)
+{
+    int i;
+    for (i = 0; erts_u_bifs[i].bif; i++) {
+	if (erts_u_bifs[i].bif == uf)
+	    return &bif_export[erts_u_bifs[i].exp_ix]->info.mfa;
+    }
+    erts_exit(ERTS_ERROR_EXIT, "bad u bif");
+    return NULL;
 }
 
 /*
@@ -5506,7 +5518,7 @@ Eterm error_atom[NUMBER_EXIT_CODES] = {
  */
 
 static BeamInstr*
-handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
+handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, ErtsCodeMFA *bif_mfa)
 {
     Eterm* hp;
     Eterm Value = c_p->fvalue;
@@ -5514,9 +5526,16 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
 
     ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */
 
-    if (c_p->freason & EXF_RESTORE_NIF) {
-	erts_nif_export_restore_error(c_p, &pc, reg, &bf);
+    if (c_p->freason & EXF_RESTORE_NIF)
+	erts_nif_export_restore_error(c_p, &pc, reg, &bif_mfa);
+
+#ifdef DEBUG
+    if (bif_mfa) {
+	/* Verify that bif_mfa does not point into our nif export */
+	NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
+	ASSERT(!nep || !ErtsInArea(bif_mfa, (char *)nep, sizeof(NifExport)));
     }
+#endif
 
     c_p->i = pc;    /* In case we call erts_exit(). */
 
@@ -5542,7 +5561,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
      * more modular.
      */
     if (c_p->freason & EXF_SAVETRACE) {
-        save_stacktrace(c_p, pc, reg, bf, Args);
+        save_stacktrace(c_p, pc, reg, bif_mfa, Args);
     }
 
     /*
@@ -5817,23 +5836,12 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) {
  */
 
 static void
-save_stacktrace_current(struct StackTrace *s, Process *c_p, ErtsCodeMFA *mfa)
-{
-    NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
-    if (!nep || &nep->exp.info.mfa != mfa)
-	s->current = mfa;
-    else {
-	s->mfa = *mfa;
-	s->current = &s->mfa;
-    }
-}
-
-static void
-save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
-		Eterm args) {
+save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg,
+		ErtsCodeMFA *bif_mfa, Eterm args) {
     struct StackTrace* s;
     int sz;
     int depth = erts_backtrace_depth;    /* max depth (never negative) */
+
     if (depth > 0) {
 	/* There will always be a current function */
 	depth --;
@@ -5850,33 +5858,29 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 
     /*
      * If the failure was in a BIF other than 'error/1', 'error/2',
-     * 'exit/1' or 'throw/1', find the bif-table index and save the
-     * argument registers by consing up an arglist.
+     * 'exit/1' or 'throw/1', save BIF-MFA and save the argument
+     * registers by consing up an arglist.
      */
-    if (bf != NULL && bf != error_1 && bf != error_2 && bf != exit_1
-	&& bf != throw_1 && bf != wrap_error_1 && bf != wrap_error_2
-	&& bf != wrap_exit_1 && bf != wrap_throw_1) {
-        int i;
-	int a;
-	for (i = 0; i < BIF_SIZE; i++) {
-	    if (bf == bif_table[i].f || bf == bif_table[i].traced) {
-		save_stacktrace_current(s, c_p, &bif_export[i]->info.mfa);
+    if (bif_mfa) {
+	if (bif_mfa->module == am_erlang) {
+	    switch (bif_mfa->function) {
+	    case am_error:
+		if (bif_mfa->arity == 1 || bif_mfa->arity == 2)
+		    goto non_bif_stacktrace;
+		break;
+	    case am_exit:
+		if (bif_mfa->arity == 1)
+		    goto non_bif_stacktrace;
+		break;
+	    case am_throw:
+		if (bif_mfa->arity == 1)
+		    goto non_bif_stacktrace;
+		break;
+	    default:
 		break;
 	    }
 	}
-	if (i >= BIF_SIZE) {
-	    /* 
-	     * The Bif does not really exist (no BIF entry).  It is a
-	     * TRAP and traps are called through apply_bif, which also
-	     * sets c_p->current (luckily).
-	     * OR it is a NIF called by call_nif where current is also set.
-	     */
-	    ASSERT(c_p->current);
-	    save_stacktrace_current(s, c_p, c_p->current);
-	}
-
-	a = s->current->arity;
-
+	s->current = bif_mfa;
 	/* Save first stack entry */
 	ASSERT(pc);
 	if (depth > 0) {
@@ -5889,11 +5893,12 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 	    depth--;
 	}
 	s->pc = NULL;
-	args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */
+	args = make_arglist(c_p, reg, bif_mfa->arity); /* Overwrite CAR(c_p->ftrace) */
     } else {
 
-	save_stacktrace_current(s, c_p, c_p->current);
+    non_bif_stacktrace:
 
+	s->current = c_p->current;
         /* 
 	 * For a function_clause error, the arguments are in the beam
 	 * registers, c_p->cp is valid, and c_p->current is set.
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 7bcb7c196d..5f0564474f 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -5027,17 +5027,16 @@ void erts_init_bif(void)
 
 #define ERTS_SCHED_BIF_TRAP_MARKER ((void *) (UWord) 1)
 
-static void
+static ERTS_INLINE void
 schedule(Process *c_p, Process *dirty_shadow_proc,
-	 ErtsCodeMFA *mfa, BifFunction *nif, BeamInstr *pc,
+	 ErtsCodeMFA *mfa, BeamInstr *pc,
 	 ErtsBifFunc dfunc, void *ifunc,
 	 Eterm module, Eterm function,
 	 int argc, Eterm *argv)
 {
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
     (void) erts_nif_export_schedule(c_p, dirty_shadow_proc,
-				    mfa, nif, pc,
-				    (BeamInstr) em_apply_bif,
+				    mfa, pc, (BeamInstr) em_apply_bif,
 				    dfunc, ifunc,
 				    module, function,
 				    argc, argv);
@@ -5120,11 +5119,10 @@ erts_schedule_bif(Process *proc,
 
     if (!ERTS_PROC_IS_EXITING(c_p)) {
 	Export *exp;
-	BifFunction obif, dbif, ibif;
+	BifFunction dbif, ibif;
 	BeamInstr *pc;
 
 	/*
-	 * obif - original bif
 	 * dbif - direct bif
 	 * ibif - indirect bif
 	 */
@@ -5160,14 +5158,12 @@ erts_schedule_bif(Process *proc,
 
 	if (i == NULL) {
 	    ERTS_INTERNAL_ERROR("Missing instruction pointer");
-	    obif = NULL;
 	}
 #ifdef HIPE
 	else if (proc->flags & F_HIPE_MODE) {
 	    /* Pointer to bif export in i */
 	    exp = (Export *) i;
 	    pc = c_p->cp;
-	    obif = (BifFunction) exp->beam[1];
 	    mfa = &exp->info.mfa;
 	}
 #endif
@@ -5175,19 +5171,16 @@ erts_schedule_bif(Process *proc,
 	    /* Pointer to bif export in i+1 */
 	    exp = (Export *) i[1];
 	    pc = i;
-	    obif = (BifFunction) exp->beam[1];
 	    mfa = &exp->info.mfa;
 	}
 	else if (em_apply_bif == (BeamInstr *) *i) {
 	    /* Pointer to bif in i+1, and mfa in i-3 */	    
-	    obif = (BifFunction) i[1];
 	    pc = c_p->cp;
 	    mfa = erts_code_to_codemfa(i);
 	}
 	else {
 	    ERTS_INTERNAL_ERROR("erts_schedule_bif() called "
 				"from unexpected instruction");
-	    obif = NULL;
 	}
 	ASSERT(bif);
 
@@ -5197,8 +5190,8 @@ erts_schedule_bif(Process *proc,
 	    argc = (int) mfa->arity;
 	}
 
-	schedule(c_p, dirty_shadow_proc, mfa, obif,
-		 pc, dbif, ibif, mod, func, argc, argv);
+	schedule(c_p, dirty_shadow_proc, mfa, pc, dbif, ibif,
+		 mod, func, argc, argv);
     }
 
     if (dirty_shadow_proc)
@@ -5295,14 +5288,14 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *
 
     if (!exiting) {
 	if (is_value(result))
-	    schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL, dirty_bif_result,
+	    schedule(c_p, dirty_shadow_proc, NULL, NULL, dirty_bif_result,
 		     NULL, am_erts_internal, am_dirty_bif_result, 1, &result);
 	else if (dirty_shadow_proc->freason != TRAP) {
 	    Eterm argv[2];
 	    ASSERT(dirty_shadow_proc->freason <= MAX_SMALL);
 	    argv[0] = make_small(dirty_shadow_proc->freason);
 	    argv[1] = dirty_shadow_proc->fvalue;
-	    schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL,
+	    schedule(c_p, dirty_shadow_proc, NULL, NULL,
 		     dirty_bif_exception, NULL, am_erts_internal,
 		     am_dirty_bif_exception, 2, argv);
 	}
@@ -5310,7 +5303,7 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *
 	    /* Dirty BIF did an ordinary trap... */
 	    ASSERT(!(erts_smp_atomic32_read_nob(&c_p->state)
 		     & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)));
-	    schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL,
+	    schedule(c_p, dirty_shadow_proc, NULL, NULL,
 		     dirty_bif_trap, (void *) dirty_shadow_proc->i,
 		     am_erts_internal, am_dirty_bif_trap,
 		     dirty_shadow_proc->arity, reg);
diff --git a/erts/emulator/beam/erl_nfunc_sched.c b/erts/emulator/beam/erl_nfunc_sched.c
index 0333545255..bc3fd83d7a 100644
--- a/erts/emulator/beam/erl_nfunc_sched.c
+++ b/erts/emulator/beam/erl_nfunc_sched.c
@@ -63,7 +63,7 @@ erts_destroy_nif_export(Process *p)
 
 NifExport *
 erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc,
-			 ErtsCodeMFA *mfa, void *nif, BeamInstr *pc,
+			 ErtsCodeMFA *mfa, BeamInstr *pc,
 			 BeamInstr instr,
 			 void *dfunc, void *ifunc,
 			 Eterm mod, Eterm func,
@@ -111,8 +111,6 @@ erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc,
 	    nep->argv[i] = reg[i];
 	nep->pc = pc;
 	nep->cp = c_p->cp;
-	ASSERT(nif);
-	nep->nif = nif;
 	nep->mfa = mfa;
 	nep->current = c_p->current;
 	ASSERT(argc >= 0);
diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h
index 72de7f0eb5..d7eccb28ba 100644
--- a/erts/emulator/beam/erl_nfunc_sched.h
+++ b/erts/emulator/beam/erl_nfunc_sched.h
@@ -52,7 +52,7 @@ typedef struct {
 NifExport *erts_new_proc_nif_export(Process *c_p, int argc);
 void erts_destroy_nif_export(Process *p);
 NifExport *erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc,
-				    ErtsCodeMFA *mfa, void *nif, BeamInstr *pc,
+				    ErtsCodeMFA *mfa, BeamInstr *pc,
 				    BeamInstr instr,
 				    void *dfunc, void *ifunc,
 				    Eterm mod, Eterm func,
@@ -65,7 +65,7 @@ ERTS_GLB_INLINE int erts_check_nif_export_in_area(Process *p,
 						  char *start, Uint size);
 ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep);
 ERTS_GLB_INLINE void erts_nif_export_restore_error(Process* c_p, BeamInstr **pc,
-						   Eterm *reg, void **nif);
+						   Eterm *reg, ErtsCodeMFA **nif_mfa);
 ERTS_GLB_INLINE Process *erts_proc_shadow2real(Process *c_p);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -132,8 +132,8 @@ erts_nif_export_restore(Process *c_p, NifExport *ep)
 }
 
 ERTS_GLB_INLINE void
-erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, Eterm *reg,
-			      void **nif)
+erts_nif_export_restore_error(Process* c_p, BeamInstr **pc,
+			      Eterm *reg, ErtsCodeMFA **nif_mfa)
 {
     NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
     int ix;
@@ -141,7 +141,7 @@ erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, Eterm *reg,
     ASSERT(nep);
     *pc = nep->pc;
     c_p->cp = nep->cp;
-    *nif = nep->nif;
+    *nif_mfa = nep->mfa;
     for (ix = 0; ix < nep->argc; ix++)
 	reg[ix] = nep->argv[ix];
     erts_nif_export_restore(c_p, nep);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index ee4b22f7ca..fd756692f9 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -266,7 +266,6 @@ static ERTS_INLINE ERL_NIF_TERM
 schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp,
 	 Eterm mod, Eterm func_name, int argc, const ERL_NIF_TERM argv[])
 {
-    Export *exp;
     NifExport *ep;
     Process *c_p, *dirty_shadow_proc;
 
@@ -278,11 +277,8 @@ schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp,
 
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
 
-    exp = ErtsContainerStruct(c_p->current, Export, info.mfa);
-
     ep = erts_nif_export_schedule(c_p, dirty_shadow_proc,
 				  c_p->current,
-				  (BifFunction) exp->beam[1],
 				  c_p->cp,
 				  (BeamInstr) em_call_nif,
 				  direct_fp, indirect_fp,
diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h
index 99a8ff6bad..64c08b1570 100644
--- a/erts/emulator/beam/error.h
+++ b/erts/emulator/beam/error.h
@@ -214,7 +214,6 @@ struct StackTrace {
     BeamInstr* pc;
     ErtsCodeMFA* current;
     int depth;	/* number of saved pointers in trace[] */
-    ErtsCodeMFA mfa; /* in case we need to make a copy of mfa */
     BeamInstr *trace[1];  /* varying size - must be last in struct */
 };
 
diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables
index 1d3023ee9a..47e1528958 100755
--- a/erts/emulator/utils/make_tables
+++ b/erts/emulator/utils/make_tables
@@ -37,6 +37,7 @@ use File::Basename;
 #    <-src>/erl_bif_table.c
 #    <-src>/erl_bif_wrap.c
 #    <-src>/erl_dirty_bif_wrap.c
+#    <-src>/erl_guard_bifs.c
 #    <-src>/hipe_nbif_impl.c
 #    <-include>/hipe_nbif_impl.h
 #    <-include>/erl_atom_table.h
@@ -228,11 +229,18 @@ typedef struct bif_entry {
 typedef struct erts_gc_bif {
     BifFunction bif;
     BifFunction gc_bif;
+    int exp_ix;
 } ErtsGcBif;
 
+typedef struct erts_u_bif {
+    BifFunction bif;
+    int exp_ix;
+} ErtsUBif;
+
 extern BifEntry bif_table[];
 extern Export* bif_export[];
 extern const ErtsGcBif erts_gc_bifs[];
+extern const ErtsUBif erts_u_bifs[];
 
 #define BIF_SIZE $bif_size
 
@@ -304,18 +312,24 @@ for ($i = 0; $i < @bif; $i++) {
 # Generate erl_gc_bifs.c.
 #
 
-open_file("$src/erl_gc_bifs.c");
+open_file("$src/erl_guard_bifs.c");
 my $i;
 includes("export.h", "sys.h", "erl_vm.h", "global.h", "erl_process.h", "bif.h",
 	 "erl_bif_table.h");
 print "const ErtsGcBif erts_gc_bifs[] = {\n";
 for ($i = 0; $i < @bif; $i++) {
     next unless $bif_info[$i]->[0] eq 'gcbif';
-    my $arity = $bif[$i]->[2];
-    my $func = $bif[$i]->[3];
-    print "  {$func, erts_gc_$func},\n";
+    print "  {$bif[$i]->[3], erts_gc_$bif[$i]->[3], BIF_$bif[$i]->[5]},\n";
+}
+print "  {NULL, NULL, -1}\n";
+print "};\n";
+
+print "const ErtsUBif erts_u_bifs[] = {\n";
+for ($i = 0; $i < @bif; $i++) {
+    next unless $bif_info[$i]->[0] eq 'ubif';
+    print "  {$bif[$i]->[3], BIF_$bif[$i]->[5]},\n";
 }
-print "  {0, 0}\n";
+print "  {NULL, -1}\n";
 print "};\n";
 
 #
-- 
cgit v1.2.3


From 04e119e22a68d686b9e8df17c0a4836c4a5b91ea Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 8 Nov 2016 09:51:03 +0100
Subject: Return and exception trace for nif-export scheduled BIFs

The support is somewhat primitive, since it is determined at
call time if trace on return or exception should be sent.
---
 erts/emulator/beam/beam_bp.c         | 25 ++++++++++++++++++
 erts/emulator/beam/bif.c             |  8 +++---
 erts/emulator/beam/erl_alloc.c       |  7 +++++
 erts/emulator/beam/erl_alloc.types   |  3 ++-
 erts/emulator/beam/erl_nfunc_sched.c | 40 ++++++++++++++++++++++++++++-
 erts/emulator/beam/erl_nfunc_sched.h | 50 +++++++++++++++++++++++++++++++++---
 erts/emulator/beam/erl_nif.c         |  8 +++---
 erts/emulator/beam/erl_trace.h       |  5 ++++
 8 files changed, 133 insertions(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 455362f5ae..27329a339e 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -32,6 +32,7 @@
 #include "erl_binary.h"
 #include "beam_bp.h"
 #include "erl_term.h"
+#include "erl_nfunc_sched.h"
 
 /* *************************************************************************
 ** Macros
@@ -805,6 +806,30 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 
     result = func(p, args, I);
 
+    if (erts_nif_export_check_save_trace(p, result,
+					 applying, ep,
+					 cp, flags,
+					 flags_meta, I,
+					 meta_tracer)) {
+	/*
+	 * erts_bif_trace_epilogue() will be called
+	 * later when appropriate via the NIF export
+	 * scheduling functionality...
+	 */
+	return result;
+    }
+
+    return erts_bif_trace_epilogue(p, result, applying, ep, cp,
+				   flags, flags_meta, I,
+				   meta_tracer);
+}
+
+Eterm
+erts_bif_trace_epilogue(Process *p, Eterm result, int applying,
+			Export* ep, BeamInstr *cp, Uint32 flags,
+			Uint32 flags_meta, BeamInstr* I,
+			ErtsTracer meta_tracer)
+{
     if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) {
 	BeamInstr i_return_trace      = beam_return_trace[0];
 	BeamInstr i_return_to_trace   = beam_return_to_trace[0];
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 5f0564474f..cae346267c 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -5047,7 +5047,7 @@ schedule(Process *c_p, Process *dirty_shadow_proc,
 static BIF_RETTYPE dirty_bif_result(BIF_ALIST_1)
 {
     NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P);
-    erts_nif_export_restore(BIF_P, nep);
+    erts_nif_export_restore(BIF_P, nep, BIF_ARG_1);
     BIF_RET(BIF_ARG_1);
 }
 
@@ -5062,7 +5062,7 @@ static BIF_RETTYPE dirty_bif_trap(BIF_ALIST)
 
     ASSERT(BIF_P->arity == nep->exp.info.mfa.arity);
 
-    erts_nif_export_restore(BIF_P, nep);
+    erts_nif_export_restore(BIF_P, nep, THE_NON_VALUE);
 
     BIF_P->i = (BeamInstr *) nep->func;
     BIF_P->freason = TRAP;
@@ -5216,12 +5216,12 @@ call_bif(Process *c_p, Eterm *reg, BeamInstr *I)
     ret = (*bif)(c_p, reg, I);
 
     if (is_value(ret))
-	erts_nif_export_restore(c_p, nep);
+	erts_nif_export_restore(c_p, nep, ret);
     else if (c_p->freason != TRAP)
 	c_p->freason |= EXF_RESTORE_NIF; /* restore in handle_error() */
     else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) {
 	/* BIF did an ordinary trap... */
-	erts_nif_export_restore(c_p, nep);
+	erts_nif_export_restore(c_p, nep, ret);
     }
     /* else:
      *   BIF rescheduled itself using erts_schedule_bif().
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 40a45c961f..4d990a9c56 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -44,6 +44,7 @@
 #include "erl_hl_timer.h"
 #include "erl_cpu_topology.h"
 #include "erl_thr_queue.h"
+#include "erl_nfunc_sched.h"
 #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE)
 #include "erl_check_io.h"
 #endif
@@ -679,6 +680,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_ABIF_TIMER)]
 	= erts_timer_type_size(ERTS_ALC_T_ABIF_TIMER);
 #endif
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_EXP_TRACE)]
+	= sizeof(NifExportTrace);
 
 #ifdef HARD_DEBUG
     hdbg_init();
@@ -2437,6 +2440,10 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 		       fi,
 		       ERTS_ALC_T_ABIF_TIMER);
 #endif
+	add_fix_values(&size.processes,
+		       &size.processes_used,
+		       fi,
+		       ERTS_ALC_T_NIF_EXP_TRACE);
     }
 
     if (want.atom || want.atom_used) {
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 6e8710eb8a..70eca5b49c 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -376,7 +376,8 @@ type	DB_MS_PSDO_PROC	LONG_LIVED	ETS		db_match_pseudo_proc
 type	SCHDLR_DATA	LONG_LIVED	SYSTEM		scheduler_data
 type  	LL_TEMP_TERM 	LONG_LIVED	SYSTEM		ll_temp_term
 
-type	NIF_TRAP_EXPORT	STANDARD	CODE		nif_trap_export_entry
+type	NIF_TRAP_EXPORT	STANDARD	PROCESSES	nif_trap_export_entry
+type	NIF_EXP_TRACE	FIXED_SIZE	PROCESSES	nif_export_trace
 type	EXPORT		LONG_LIVED	CODE		export_entry
 type	MONITOR_SH	FIXED_SIZE	PROCESSES	monitor_sh
 type	NLINK_SH	FIXED_SIZE	PROCESSES	nlink_sh
diff --git a/erts/emulator/beam/erl_nfunc_sched.c b/erts/emulator/beam/erl_nfunc_sched.c
index bc3fd83d7a..1bebc1eda4 100644
--- a/erts/emulator/beam/erl_nfunc_sched.c
+++ b/erts/emulator/beam/erl_nfunc_sched.c
@@ -28,6 +28,7 @@
 #include "erl_process.h"
 #include "bif.h"
 #include "erl_nfunc_sched.h"
+#include "erl_trace.h"
 
 NifExport *
 erts_new_proc_nif_export(Process *c_p, int argc)
@@ -44,9 +45,12 @@ erts_new_proc_nif_export(Process *c_p, int argc)
 
     nep->argc = -1; /* unused marker */
     nep->argv_size = argc;
+    nep->trace = NULL;
     old_nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(c_p, nep);
-    if (old_nep)
+    if (old_nep) {
+	ASSERT(!nep->trace);
 	erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, old_nep);
+    }
     return nep;
 }
 
@@ -61,6 +65,40 @@ erts_destroy_nif_export(Process *p)
     }
 }
 
+void
+erts_nif_export_save_trace(Process *c_p, NifExport *nep, int applying,
+			   Export* ep, BeamInstr *cp, Uint32 flags,
+			   Uint32 flags_meta, BeamInstr* I,
+			   ErtsTracer meta_tracer)
+{
+    NifExportTrace *netp;
+    ASSERT(nep && nep->argc >= 0);
+    ASSERT(!nep->trace);
+    netp = erts_alloc(ERTS_ALC_T_NIF_EXP_TRACE,
+		      sizeof(NifExportTrace));
+    netp->applying = applying;
+    netp->ep = ep;
+    netp->cp = cp;
+    netp->flags = flags;
+    netp->flags_meta = flags_meta;
+    netp->I = I;
+    netp->meta_tracer = NIL;
+    erts_tracer_update(&netp->meta_tracer, meta_tracer);
+    nep->trace = netp;
+}
+
+void
+erts_nif_export_restore_trace(Process *c_p, Eterm result, NifExport *nep)
+{
+    NifExportTrace *netp = nep->trace;
+    nep->trace = NULL;
+    erts_bif_trace_epilogue(c_p, result, netp->applying, netp->ep,
+			    netp->cp, netp->flags, netp->flags_meta,
+			    netp->I, netp->meta_tracer);
+    erts_tracer_update(&netp->meta_tracer, NIL);
+    erts_free(ERTS_ALC_T_NIF_EXP_TRACE, netp);
+}
+
 NifExport *
 erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc,
 			 ErtsCodeMFA *mfa, BeamInstr *pc,
diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h
index d7eccb28ba..5c98957cb7 100644
--- a/erts/emulator/beam/erl_nfunc_sched.h
+++ b/erts/emulator/beam/erl_nfunc_sched.h
@@ -23,6 +23,17 @@
 
 #include "erl_process.h"
 #include "bif.h"
+#include "error.h"
+
+typedef struct {
+    int applying;
+    Export* ep;
+    BeamInstr *cp;
+    Uint32 flags;
+    Uint32 flags_meta;
+    BeamInstr* I;
+    ErtsTracer meta_tracer;
+} NifExportTrace;
 
 /*
  * NIF exports need a few more items than the Export struct provides,
@@ -39,6 +50,7 @@ typedef struct {
     struct erl_module_nif* m; /* NIF module, or NULL if BIF */
     void *func;		/* Indirect NIF or BIF to execute (may be unused) */
     ErtsCodeMFA *current;/* Current as set when originally called */
+    NifExportTrace *trace;
     /* --- The following is only used on error --- */
     BeamInstr *pc;	/* Program counter */
     BeamInstr *cp;	/* Continuation pointer */
@@ -50,6 +62,11 @@ typedef struct {
 } NifExport;
 
 NifExport *erts_new_proc_nif_export(Process *c_p, int argc);
+void erts_nif_export_save_trace(Process *c_p, NifExport *nep, int applying,
+				Export* ep, BeamInstr *cp, Uint32 flags,
+				Uint32 flags_meta, BeamInstr* I,
+				ErtsTracer meta_tracer);
+void erts_nif_export_restore_trace(Process *c_p, Eterm result, NifExport *nep);
 void erts_destroy_nif_export(Process *p);
 NifExport *erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc,
 				    ErtsCodeMFA *mfa, BeamInstr *pc,
@@ -63,9 +80,15 @@ ERTS_GLB_INLINE int erts_setup_nif_export_rootset(Process* proc, Eterm** objv,
 						  Uint* nobj);
 ERTS_GLB_INLINE int erts_check_nif_export_in_area(Process *p,
 						  char *start, Uint size);
-ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep);
+ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep,
+					     Eterm result);
 ERTS_GLB_INLINE void erts_nif_export_restore_error(Process* c_p, BeamInstr **pc,
 						   Eterm *reg, ErtsCodeMFA **nif_mfa);
+ERTS_GLB_INLINE int erts_nif_export_check_save_trace(Process *c_p, Eterm result,
+						     int applying, Export* ep,
+						     BeamInstr *cp, Uint32 flags,
+						     Uint32 flags_meta, BeamInstr* I,
+						     ErtsTracer meta_tracer);
 ERTS_GLB_INLINE Process *erts_proc_shadow2real(Process *c_p);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -119,7 +142,7 @@ erts_check_nif_export_in_area(Process *p, char *start, Uint size)
 }
 
 ERTS_GLB_INLINE void
-erts_nif_export_restore(Process *c_p, NifExport *ep)
+erts_nif_export_restore(Process *c_p, NifExport *ep, Eterm result)
 {
     ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
     ERTS_SMP_LC_ASSERT(!(c_p->static_flags
@@ -129,6 +152,8 @@ erts_nif_export_restore(Process *c_p, NifExport *ep)
 
     c_p->current = ep->current;
     ep->argc = -1; /* Unused nif-export marker... */
+    if (ep->trace)
+	erts_nif_export_restore_trace(c_p, result, ep);
 }
 
 ERTS_GLB_INLINE void
@@ -144,7 +169,26 @@ erts_nif_export_restore_error(Process* c_p, BeamInstr **pc,
     *nif_mfa = nep->mfa;
     for (ix = 0; ix < nep->argc; ix++)
 	reg[ix] = nep->argv[ix];
-    erts_nif_export_restore(c_p, nep);
+    erts_nif_export_restore(c_p, nep, THE_NON_VALUE);
+}
+
+ERTS_GLB_INLINE int
+erts_nif_export_check_save_trace(Process *c_p, Eterm result,
+				 int applying, Export* ep,
+				 BeamInstr *cp, Uint32 flags,
+				 Uint32 flags_meta, BeamInstr* I,
+				 ErtsTracer meta_tracer)
+{
+    if (is_non_value(result) && c_p->freason == TRAP) {
+	NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
+	if (nep && nep->argc >= 0) {
+	    erts_nif_export_save_trace(c_p, nep, applying, ep,
+				       cp, flags, flags_meta,
+				       I, meta_tracer);
+	    return 1;
+	}
+    }
+    return 0;
 }
 
 ERTS_GLB_INLINE Process *
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index fd756692f9..d946844f15 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2290,9 +2290,9 @@ erts_nif_export_cleanup_nif_mod(NifExport *ep)
 }
 
 static ERTS_INLINE void
-nif_export_restore(Process *c_p, NifExport *ep)
+nif_export_restore(Process *c_p, NifExport *ep, Eterm res)
 {
-    erts_nif_export_restore(c_p, ep);
+    erts_nif_export_restore(c_p, ep, res);
     ASSERT(ep->m);
     nif_export_cleanup_nif_mod(ep);
 }
@@ -2320,7 +2320,7 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc)));
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     ASSERT(ep);
-    nif_export_restore(proc, ep);
+    nif_export_restore(proc, ep, argv[0]);
     return argv[0];
 }
 
@@ -2462,7 +2462,7 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 	/* Done (not rescheduled)... */
 	ASSERT(ep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER);
 	if (!env->exception_thrown)
-	    nif_export_restore(proc, ep);
+	    nif_export_restore(proc, ep, result);
 	else {
 	    nif_export_cleanup_nif_mod(ep);
 	    /*
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 378b6de49c..01fe1e5e23 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -143,6 +143,11 @@ Uint erts_trace_flag2bit(Eterm flag);
 int erts_trace_flags(Eterm List, 
 		 Uint *pMask, ErtsTracer *pTracer, int *pCpuTimestamp);
 Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I);
+Eterm
+erts_bif_trace_epilogue(Process *p, Eterm result, int applying,
+			Export* ep, BeamInstr *cp, Uint32 flags,
+			Uint32 flags_meta, BeamInstr* I,
+			ErtsTracer meta_tracer);
 
 #ifdef ERTS_SMP
 void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp);
-- 
cgit v1.2.3


From d0e88c0c69f94625daf9cafa192bac97115e9072 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 16 Dec 2016 16:35:57 +0100
Subject: Perform potentially long GC on dirty schedulers if available

---
 erts/emulator/beam/beam_bif_load.c   |  26 ++-
 erts/emulator/beam/beam_emu.c        | 119 ++++++-----
 erts/emulator/beam/bif.c             |   9 +-
 erts/emulator/beam/bif.tab           |   1 -
 erts/emulator/beam/erl_gc.c          | 169 +++++++++++++---
 erts/emulator/beam/erl_gc.h          |   9 +-
 erts/emulator/beam/erl_nfunc_sched.h |   1 -
 erts/emulator/beam/erl_process.c     | 375 +++++++++++++++++++++++++++--------
 erts/emulator/beam/erl_process.h     |  45 ++++-
 erts/emulator/beam/ops.tab           |   7 +-
 erts/emulator/hipe/hipe_bif_list.m4  |   2 +-
 11 files changed, 563 insertions(+), 200 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 36bbe81ed5..22576503ef 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -913,7 +913,7 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed
 
     la = ERTS_COPY_LITERAL_AREA();
     if (!la)
-	return am_ok;
+        goto return_ok;
 
     oh = la->off_heap;
     literals = (char *) &la->start[0];
@@ -977,6 +977,11 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed
 	 * this is not completely certain). We go for
 	 * the GC directly instead of scanning everything
 	 * one more time...
+	 *
+	 * Also note that calling functions expect a
+	 * major GC to be performed if gc_allowed is set
+	 * to true. If you change this, you need to fix
+	 * callers...
 	 */
 	goto literal_gc;
     }
@@ -1051,6 +1056,13 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed
 	}
     }
 
+return_ok:
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p)))
+	c_p->flags &= ~F_DIRTY_CLA;
+#endif
+
     return am_ok;
 
 literal_gc:
@@ -1061,13 +1073,13 @@ literal_gc:
     if (c_p->flags & F_DISABLE_GC)
 	return THE_NON_VALUE;
 
-    FLAGS(c_p) |= F_NEED_FULLSWEEP;
-
-    *redsp += erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg, c_p->arity, fcalls);
+    *redsp += erts_garbage_collect_literals(c_p, (Eterm *) literals, lit_bsize,
+					    oh, fcalls);
 
-    erts_garbage_collect_literals(c_p, (Eterm *) literals, lit_bsize, oh);
-
-    *redsp += lit_bsize / 64; /* Need, better value... */
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (c_p->flags & F_DIRTY_CLA)
+	return THE_NON_VALUE;
+#endif
 
     return am_ok;
 }
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index cfdf61eed0..85d92321b8 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5327,10 +5327,25 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
     ASSERT(!(c_p->flags & F_HIPE_MODE));
     ERTS_MSACC_UPDATE_CACHE_X();
 
-    reg = esdp->x_reg_array;
-    {
+    /*
+     * Set fcalls even though we ignore it, so we don't
+     * confuse code accessing it...
+     */
+    if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+	c_p->fcalls = 0;
+    else
+	c_p->fcalls = CONTEXT_REDS;
+
+    if (erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_DIRTY_RUNNING_SYS) {
+	erts_execute_dirty_system_task(c_p);
+	goto do_dirty_schedule;
+    }
+    else {
+	ErtsCodeMFA *codemfa;
 	Eterm* argp;
-	int i;
+	int i, exiting;
+
+	reg = esdp->x_reg_array;
 
 	argp = c_p->arg_reg;
 	for (i = c_p->arity - 1; i >= 0; i--) {
@@ -5346,15 +5361,6 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 
 	I = c_p->i;
 
-	/*
-	 * Set fcalls even though we ignore it, so we don't
-	 * confuse code accessing it...
-	 */
-	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
-	    c_p->fcalls = 0;
-	else
-	    c_p->fcalls = CONTEXT_REDS;
-
 	SWAPIN;
 
 #ifdef USE_VM_PROBES
@@ -5378,62 +5384,55 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
             DTRACE2(process_scheduled, process_buf, fun_buf);
         }
 #endif
-    }
 
-    {
-	int exiting;
-
-	{
-	    /*
-	     * call_nif is always first instruction in function:
-	     *
-	     * I[-3]: Module
-	     * I[-2]: Function
-	     * I[-1]: Arity
-	     * I[0]: &&call_nif
-	     * I[1]: Function pointer to NIF function
-	     * I[2]: Pointer to erl_module_nif
-	     * I[3]: Function pointer to dirty NIF
-             *
-             * This layout is determined by the NifExport struct
-	     */
-            ErtsCodeMFA *codemfa;
+	/*
+	 * call_nif is always first instruction in function:
+	 *
+	 * I[-3]: Module
+	 * I[-2]: Function
+	 * I[-1]: Arity
+	 * I[0]: &&call_nif
+	 * I[1]: Function pointer to NIF function
+	 * I[2]: Pointer to erl_module_nif
+	 * I[3]: Function pointer to dirty NIF
+	 *
+	 * This layout is determined by the NifExport struct
+	 */
 
-	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
+	ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
 
-            codemfa = erts_code_to_codemfa(I);
+	codemfa = erts_code_to_codemfa(I);
 
-	    DTRACE_NIF_ENTRY(c_p, codemfa);
-	    c_p->current = codemfa;
-	    SWAPOUT;
-	    PROCESS_MAIN_CHK_LOCKS(c_p);
-	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	DTRACE_NIF_ENTRY(c_p, codemfa);
+	c_p->current = codemfa;
+	SWAPOUT;
+	PROCESS_MAIN_CHK_LOCKS(c_p);
+	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 
-	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	    if (em_apply_bif == (BeamInstr *) *I) {
-		exiting = erts_call_dirty_bif(esdp, c_p, I, reg);
-	    }
-	    else {
-		ASSERT(em_call_nif == (BeamInstr *) *I);
-		exiting = erts_call_dirty_nif(esdp, c_p, I, reg);
-	    }
+	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+	if (em_apply_bif == (BeamInstr *) *I) {
+	    exiting = erts_call_dirty_bif(esdp, c_p, I, reg);
+	}
+	else {
+	    ASSERT(em_call_nif == (BeamInstr *) *I);
+	    exiting = erts_call_dirty_nif(esdp, c_p, I, reg);
+	}
 
-	    ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
+	ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
 
-	    PROCESS_MAIN_CHK_LOCKS(c_p);
-	    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
-	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
-	    if (exiting)
-		goto do_dirty_schedule;
-	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+	PROCESS_MAIN_CHK_LOCKS(c_p);
+	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+	ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
+	if (exiting)
+	    goto do_dirty_schedule;
+	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 
-	    DTRACE_NIF_RETURN(c_p, codemfa);
-	    ERTS_HOLE_CHECK(c_p);
-	    SWAPIN;
-	    I = c_p->i;
-	    goto context_switch;
-	}
+	DTRACE_NIF_RETURN(c_p, codemfa);
+	ERTS_HOLE_CHECK(c_p);
+	SWAPIN;
+	I = c_p->i;
+	goto context_switch;
     }
 #endif /* ERTS_DIRTY_SCHEDULERS */
 }
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index cae346267c..a5b66db35c 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3865,13 +3865,6 @@ BIF_RETTYPE now_0(BIF_ALIST_0)
 
 /**********************************************************************/
 
-BIF_RETTYPE garbage_collect_0(BIF_ALIST_0)
-{
-    FLAGS(BIF_P) |= F_NEED_FULLSWEEP;
-    erts_garbage_collect(BIF_P, 0, NULL, 0);
-    return am_true;
-}
-
 /*
  * Pass atom 'minor' for relaxed generational GC run. This is only
  * recommendation, major run may still be chosen by VM.
@@ -4324,7 +4317,7 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2)
 		erts_smp_proc_unlock(new_member, ERTS_PROC_LOCK_STATUS);
 		if (new_member == BIF_P
 		    || !(erts_smp_atomic32_read_nob(&new_member->state)
-			 & (ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS))) {
+			 & ERTS_PSFLG_DIRTY_RUNNING)) {
 		    new_member->group_leader = STORE_NC_IN_PROC(new_member,
 								BIF_ARG_1);
 		}
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index f461da913e..47fdcfa7a4 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -68,7 +68,6 @@ gcbif erlang:float/1
 bif erlang:float_to_list/1
 bif erlang:float_to_list/2
 bif erlang:fun_info/2
-bif erlang:garbage_collect/0
 bif erts_internal:garbage_collect/1
 bif erlang:get/0
 bif erlang:get/1
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 6093b0cf39..5b299d9acf 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -118,11 +118,14 @@ static Eterm *full_sweep_heaps(Process *p,
 			       char *oh, Uint oh_size,
 			       Eterm *objv, int nobj);
 static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
-			   int need, Eterm* objv, int nobj, int fcalls);
+			   int need, Eterm* objv, int nobj, int fcalls,
+			   Uint max_young_gen_usage);
 static int major_collection(Process* p, ErlHeapFragment *live_hf_end,
-			    int need, Eterm* objv, int nobj, Uint *recl);
+			    int need, Eterm* objv, int nobj,
+			    Uint ygen_usage, Uint *recl);
 static int minor_collection(Process* p, ErlHeapFragment *live_hf_end,
-			    int need, Eterm* objv, int nobj, Uint *recl);
+			    int need, Eterm* objv, int nobj,
+			    Uint ygen_usage, Uint *recl);
 static void do_minor(Process *p, ErlHeapFragment *live_hf_end,
 		     char *mature, Uint mature_size,
 		     Uint new_sz, Eterm* objv, int nobj);
@@ -413,15 +416,15 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end,
 		regs = erts_proc_sched_data(p)->x_reg_array;
 	    }
 #endif
-	    cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls);
+	    cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls, 0);
 	} else {
-	    cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls);
+	    cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls, 0);
 	}
     } else {
 	Eterm val[1];
 
 	val[0] = result;
-	cost = garbage_collect(p, live_hf_end, 0, val, 1, p->fcalls);
+	cost = garbage_collect(p, live_hf_end, 0, val, 1, p->fcalls, 0);
 	result = val[0];
     }
     BUMP_REDS(p, cost);
@@ -599,6 +602,32 @@ young_gen_usage(Process *p)
 	}							\
     } while (0)
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+static ERTS_INLINE void
+check_for_possibly_long_gc(Process *p, Uint ygen_usage)
+{
+    int major;
+    Uint sz;
+
+    major = (p->flags & F_NEED_FULLSWEEP) || GEN_GCS(p) >= MAX_GEN_GCS(p);
+
+    sz = ygen_usage;
+    sz += p->hend - p->stop;
+    if (p->flags & F_ON_HEAP_MSGQ)
+        sz += p->msg.len;
+    if (major)
+	sz += p->old_htop - p->old_heap;
+
+    if (sz >= ERTS_POTENTIALLY_LONG_GC_HSIZE) {
+        ASSERT(!(p->flags & (F_DISABLE_GC|F_DELAY_GC)));
+	p->flags |= major ? F_DIRTY_MAJOR_GC : F_DIRTY_MINOR_GC;
+        erts_schedule_dirty_sys_execution(p);
+    }
+}
+
+#endif
+
 /*
  * Garbage collect a process.
  *
@@ -609,13 +638,15 @@ young_gen_usage(Process *p)
  */
 static int
 garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
-		int need, Eterm* objv, int nobj, int fcalls)
+		int need, Eterm* objv, int nobj, int fcalls,
+		Uint max_young_gen_usage)
 {
     Uint reclaimed_now = 0;
+    Uint ygen_usage;
     Eterm gc_trace_end_tag;
     int reds;
     ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */
-    ErtsSchedulerData *esdp;
+    ErtsSchedulerData *esdp = erts_proc_sched_data(p);
     erts_aint32_t state;
     ERTS_MSACC_PUSH_STATE_M();
 #ifdef USE_VM_PROBES
@@ -624,13 +655,26 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
 
     ERTS_CHK_MBUF_SZ(p);
 
-    ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls)
-	   >= erts_proc_sched_data(p)->virtual_reds);
+    ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) >= esdp->virtual_reds);
 
     state = erts_smp_atomic32_read_nob(&p->state);
 
-    if (p->flags & (F_DISABLE_GC|F_DELAY_GC) || state & ERTS_PSFLG_EXITING)
+    if ((p->flags & (F_DISABLE_GC|F_DELAY_GC)) || state & ERTS_PSFLG_EXITING) {
+#ifdef ERTS_DIRTY_SCHEDULERS
+    delay_gc_before_start:
+#endif
 	return delay_garbage_collection(p, live_hf_end, need, fcalls);
+    }
+
+    ygen_usage = max_young_gen_usage ? max_young_gen_usage : young_gen_usage(p);
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	check_for_possibly_long_gc(p, ygen_usage);
+	if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC))
+	    goto delay_gc_before_start;
+    }
+#endif
 
     if (p->abandoned_heap)
 	live_hf_end = ERTS_INVALID_HFRAG_PTR;
@@ -639,8 +683,6 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
 
     ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_GC);
 
-    esdp = erts_get_scheduler_data();
-
     erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
     if (erts_system_monitor_long_gc != 0)
 	start_time = erts_get_monotonic_time(esdp);
@@ -667,14 +709,25 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
             trace_gc(p, am_gc_minor_start, need, THE_NON_VALUE);
         }
         DTRACE2(gc_minor_start, pidbuf, need);
-        reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
+        reds = minor_collection(p, live_hf_end, need, objv, nobj,
+				ygen_usage, &reclaimed_now);
         DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
         if (reds == -1) {
             if (IS_TRACED_FL(p, F_TRACE_GC)) {
                 trace_gc(p, am_gc_minor_end, reclaimed_now, THE_NON_VALUE);
             }
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+		p->flags |= F_NEED_FULLSWEEP;
+		check_for_possibly_long_gc(p, ygen_usage);
+		if (p->flags & F_DIRTY_MAJOR_GC)
+		    goto delay_gc_after_start;
+	    }
+#endif
             goto do_major_collection;
         }
+        if (ERTS_SCHEDULER_IS_DIRTY(esdp))
+            p->flags &= ~F_DIRTY_MINOR_GC;
         gc_trace_end_tag = am_gc_minor_end;
     } else {
 do_major_collection:
@@ -683,7 +736,10 @@ do_major_collection:
             trace_gc(p, am_gc_major_start, need, THE_NON_VALUE);
         }
         DTRACE2(gc_major_start, pidbuf, need);
-        reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
+        reds = major_collection(p, live_hf_end, need, objv, nobj,
+				ygen_usage, &reclaimed_now);
+        if (ERTS_SCHEDULER_IS_DIRTY(esdp))
+            p->flags &= ~(F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC);
         DTRACE2(gc_major_end, pidbuf, reclaimed_now);
         gc_trace_end_tag = am_gc_major_end;
         ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC);
@@ -713,6 +769,9 @@ do_major_collection:
                               am_kill, NIL, NULL, 0);
         erts_smp_proc_unlock(p, locks & ERTS_PROC_LOCKS_ALL_MINOR);
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    delay_gc_after_start:
+#endif
         /* erts_send_exit_signal looks for ERTS_PSFLG_GC, so
            we have to remove it after the signal is sent */
         erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
@@ -795,7 +854,7 @@ do_major_collection:
 int
 erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fcalls)
 {
-    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls);
+    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls, 0);
     int reds_left = ERTS_REDS_LEFT(p, fcalls);
     if (reds > reds_left)
 	reds = reds_left;
@@ -806,7 +865,7 @@ erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fca
 void
 erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
 {
-    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls);
+    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls, 0);
     BUMP_REDS(p, reds);
     ASSERT(CONTEXT_REDS - ERTS_BIF_REDS_LEFT(p)
 	   >= erts_proc_sched_data(p)->virtual_reds);
@@ -832,6 +891,20 @@ erts_garbage_collect_hibernate(Process* p)
     if (p->flags & F_DISABLE_GC)
 	ERTS_INTERNAL_ERROR("GC disabled");
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p)))
+        p->flags &= ~(F_DIRTY_GC_HIBERNATE|F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC);
+    else {
+	Uint flags = p->flags;
+	p->flags |= F_NEED_FULLSWEEP;
+	check_for_possibly_long_gc(p, (p->htop - p->heap) + p->mbuf_sz);
+	if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) {
+	    p->flags = flags|F_DIRTY_GC_HIBERNATE;
+	    return;
+	}
+	p->flags = flags;
+    }
+#endif
     /*
      * Preliminaries.
      */
@@ -843,7 +916,6 @@ erts_garbage_collect_hibernate(Process* p)
      * Do it.
      */
 
-
     heap_size = p->heap_sz + (p->old_htop - p->old_heap) + p->mbuf_sz;
 
     heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_TMP_HEAP,
@@ -984,10 +1056,11 @@ static ERTS_INLINE void offset_nstack(Process* p, Sint offs,
 
 #endif /* HIPE */
 
-void
+int
 erts_garbage_collect_literals(Process* p, Eterm* literals,
 			      Uint byte_lit_size,
-			      struct erl_off_heap_header* oh)
+			      struct erl_off_heap_header* oh,
+			      int fcalls)
 {
     Uint lit_size = byte_lit_size / sizeof(Eterm);
     Uint old_heap_size;
@@ -999,20 +1072,49 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
     Uint area_size;
     Eterm* old_htop;
     Uint n;
+    Uint ygen_usage = 0;
     struct erl_off_heap_header** prev = NULL;
+    Sint64 reds;
+
+    if (p->flags & (F_DISABLE_GC|F_DELAY_GC))
+	ERTS_INTERNAL_ERROR("GC disabled");
+
+    /*
+     * First an ordinary major collection...
+     */
+
+    p->flags |= F_NEED_FULLSWEEP;
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p)))
+	p->flags &= ~F_DIRTY_CLA;
+    else {
+	ygen_usage = young_gen_usage(p);
+	check_for_possibly_long_gc(p,
+				   (byte_lit_size/sizeof(Uint)
+				    + 2*ygen_usage));
+	if (p->flags & F_DIRTY_MAJOR_GC) {
+	    p->flags |= F_DIRTY_CLA;
+	    return 10;
+	}
+    }
+#endif
+
+    reds = (Sint64) garbage_collect(p, ERTS_INVALID_HFRAG_PTR, 0,
+				    p->arg_reg, p->arity, fcalls,
+				    ygen_usage);
+
+    ASSERT(!(p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)));
 
-    if (p->flags & F_DISABLE_GC)
-	return;
     /*
      * Set GC state.
      */
     erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
 
     /*
-     * We assume that the caller has already done a major collection
-     * (which has discarded the old heap), so that we don't have to cope
-     * with pointer to literals on the old heap. We will now allocate
-     * an old heap to contain the literals.
+     * Just did a major collection (which has discarded the old heap),
+     * so that we don't have to cope with pointer to literals on the
+     * old heap. We will now allocate an old heap to contain the literals.
      */
     
     ASSERT(p->old_heap == 0);	/* Must NOT have an old heap yet. */
@@ -1155,15 +1257,21 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
      * Restore status.
      */
     erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
+
+    reds += (Sint64) gc_cost((p->htop - p->heap) + byte_lit_size/sizeof(Uint), 0);
+    if (reds > INT_MAX)
+	return INT_MAX;
+    return (int) reds;
 }
 
 static int
 minor_collection(Process* p, ErlHeapFragment *live_hf_end,
-		 int need, Eterm* objv, int nobj, Uint *recl)
+		 int need, Eterm* objv, int nobj,
+		 Uint ygen_usage, Uint *recl)
 {
     Eterm *mature = p->abandoned_heap ? p->abandoned_heap : p->heap;
     Uint mature_size = p->high_water - mature;
-    Uint size_before = young_gen_usage(p);
+    Uint size_before = ygen_usage;
 
     /*
      * Check if we have gone past the max heap size limit
@@ -1534,7 +1642,8 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
 
 static int
 major_collection(Process* p, ErlHeapFragment *live_hf_end,
-		 int need, Eterm* objv, int nobj, Uint *recl)
+		 int need, Eterm* objv, int nobj,
+		 Uint ygen_usage, Uint *recl)
 {
     Uint size_before, size_after, stack_size;
     Eterm* n_heap;
@@ -1552,7 +1661,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
      * to receive all live data.
      */
 
-    size_before = young_gen_usage(p);
+    size_before = ygen_usage;
     size_before += p->old_htop - p->old_heap;
     stack_size = p->hend - p->stop;
 
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 54ea9ca3c0..0dfdf52028 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -25,6 +25,8 @@
 
 /* GC declarations shared by beam/erl_gc.c and hipe/hipe_gc.c */
 
+#define ERTS_POTENTIALLY_LONG_GC_HSIZE (128*1024) /* Words */
+
 #include "erl_map.h"
 
 #if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -145,9 +147,10 @@ void erts_garbage_collect_hibernate(struct process* p);
 Eterm erts_gc_after_bif_call_lhf(struct process* p, ErlHeapFragment *live_hf_end,
 				 Eterm result, Eterm* regs, Uint arity);
 Eterm erts_gc_after_bif_call(struct process* p, Eterm result, Eterm* regs, Uint arity);
-void erts_garbage_collect_literals(struct process* p, Eterm* literals,
-				   Uint lit_size,
-				   struct erl_off_heap_header* oh);
+int erts_garbage_collect_literals(struct process* p, Eterm* literals,
+				  Uint lit_size,
+				  struct erl_off_heap_header* oh,
+				  int fcalls);
 Uint erts_next_heap_size(Uint, Uint);
 Eterm erts_heap_sizes(struct process* p);
 
diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h
index 5c98957cb7..55a3a6dbf6 100644
--- a/erts/emulator/beam/erl_nfunc_sched.h
+++ b/erts/emulator/beam/erl_nfunc_sched.h
@@ -54,7 +54,6 @@ typedef struct {
     /* --- The following is only used on error --- */
     BeamInstr *pc;	/* Program counter */
     BeamInstr *cp;	/* Continuation pointer */
-    void *nif;		/* Original NIF/BIF call */
     ErtsCodeMFA *mfa;	/* MFA of original call */
     int argc;		/* Number of arguments in original call */
     int argv_size;	/* Allocated size of argv */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 955b98b35b..da4b468d66 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -6439,10 +6439,30 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
     int enqueue; /* < 0 -> use proxy */
     ErtsRunQueue* runq;
 
-    if (is_normal_sched)
-	running_flgs = ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS;
-    else
+    if (!is_normal_sched)
 	running_flgs = ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS;
+    else {
+	running_flgs = ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS;
+#ifdef ERTS_DIRTY_SCHEDULERS
+        if (state & ERTS_PSFLG_DIRTY_ACTIVE_SYS
+            && (p->flags & (F_DELAY_GC|F_DISABLE_GC))) {
+            /*
+             * Delay dirty GC; will be enabled automatically
+             * again by next GC...
+             */
+
+            /*
+             * No normal execution until dirty CLA or hibernat has
+             * been handled...
+             */
+            ASSERT(!(p->flags & (F_DIRTY_CLA | F_DIRTY_GC_HIBERNATE)));
+
+            state = erts_smp_atomic32_read_band_nob(&p->state,
+                                                    ~ERTS_PSFLG_DIRTY_ACTIVE_SYS);
+            state &= ~ERTS_PSFLG_DIRTY_ACTIVE_SYS;
+        }
+#endif
+    }
 
     a = state;
 
@@ -8642,8 +8662,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 		erts_aint32_t state;
 		state = erts_smp_atomic32_read_nob(&rp->state);
 		ASSERT((state & ERTS_PSFLG_PENDING_EXIT)
-		       || !(state & (ERTS_PSFLG_RUNNING
-				     | ERTS_PSFLG_DIRTY_RUNNING_SYS)));
+		       || !(state & ERTS_PSFLG_RUNNING));
 	    }
 #endif
 
@@ -9981,6 +10000,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 					      | ERTS_PSFLG_FREE
 					      | ERTS_PSFLG_RUNNING_SYS
+					      | ERTS_PSFLG_DIRTY_RUNNING_SYS
 					      | ERTS_PSFLG_EXITING))
 				    == ERTS_PSFLG_EXITING)
 				   & (!!is_normal_sched))
@@ -9992,7 +10012,14 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 					    | ERTS_PSFLG_PENDING_EXIT
 					    | ERTS_PSFLG_ACTIVE_SYS
 					    | ERTS_PSFLG_DIRTY_ACTIVE_SYS))
-				  != ERTS_PSFLG_SUSPENDED));
+				  != ERTS_PSFLG_SUSPENDED)
+#ifdef ERTS_DIRTY_SCHEDULERS
+                               & (!(state & (ERTS_PSFLG_EXITING
+                                             | ERTS_PSFLG_PENDING_EXIT))
+                                  | (!!is_normal_sched))
+#endif
+                    );
+
 		if (run_process) {
 		    if (state & (ERTS_PSFLG_ACTIVE_SYS
 				 | ERTS_PSFLG_DIRTY_ACTIVE_SYS))
@@ -10133,65 +10160,70 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    }
 	}
 
-	if (state & (ERTS_PSFLG_RUNNING_SYS
-		     | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
-	    /*
-	     * GC is normally never delayed when a process
-	     * is scheduled out, but might be when executing
-	     * hand written beam assembly in
-	     * prim_eval:'receive'. If GC is delayed we are
-	     * not allowed to execute system tasks.
-	     */
-	    if (!(p->flags & F_DELAY_GC)) {
-		int cost = execute_sys_tasks(p, &state, reds);
-		calls += cost;
-		reds -= cost;
-		if (reds <= 0
+        if (is_normal_sched) {
+
+            if (state & ERTS_PSFLG_RUNNING_SYS) {
+                /*
+                 * GC is normally never delayed when a process
+                 * is scheduled out, but might be when executing
+                 * hand written beam assembly in
+                 * prim_eval:'receive'. If GC is delayed we are
+                 * not allowed to execute system tasks.
+                 */
+                if (!(p->flags & F_DELAY_GC)) {
+                    int cost = execute_sys_tasks(p, &state, reds);
+                    calls += cost;
+                    reds -= cost;
+                    if (reds <= 0)
+                        goto sched_out_proc;
 #ifdef ERTS_DIRTY_SCHEDULERS
-		    || !is_normal_sched
-		    || (state & ERTS_PSFLGS_DIRTY_WORK)
+                    if (state & ERTS_PSFLGS_DIRTY_WORK)
+                        goto sched_out_proc;
 #endif
-		    ) {
-		    goto sched_out_proc;
-		}
-	    }
+                }
 
-	    ASSERT(state & psflg_running_sys);
-	    ASSERT(!(state & psflg_running));
+                ASSERT(state & psflg_running_sys);
+                ASSERT(!(state & psflg_running));
 
-	    while (1) {
-		erts_aint32_t n, e;
+                while (1) {
+                    erts_aint32_t n, e;
 
-		if (((state & (ERTS_PSFLG_SUSPENDED
-			       | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE)
-		    && !(state & ERTS_PSFLG_EXITING)) {
-		    goto sched_out_proc;
-		}
+                    if (((state & (ERTS_PSFLG_SUSPENDED
+                                   | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE)
+                        && !(state & ERTS_PSFLG_EXITING)) {
+                        goto sched_out_proc;
+                    }
 
-		n = e = state;
-		n &= ~psflg_running_sys;
-		n |= psflg_running;
+                    n = e = state;
+                    n &= ~psflg_running_sys;
+                    n |= psflg_running;
 
-		state = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
-		if (state == e) {
-		    state = n;
-		    break;
-		}
+                    state = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
+                    if (state == e) {
+                        state = n;
+                        break;
+                    }
 
-		ASSERT(state & psflg_running_sys);
-		ASSERT(!(state & psflg_running));
-	    }
-	}
+                    ASSERT(state & psflg_running_sys);
+                    ASSERT(!(state & psflg_running));
+                }
+            }
 
-	if (ERTS_IS_GC_DESIRED(p) && !ERTS_SCHEDULER_IS_DIRTY_IO(esdp)) {
-	    if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) {
-		int cost = scheduler_gc_proc(p, reds);
-		calls += cost;
-		reds -= cost;
-		if (reds <= 0)
-		    goto sched_out_proc;
-	    }
-	}
+            if (ERTS_IS_GC_DESIRED(p)) {
+                if (!(state & ERTS_PSFLG_EXITING)
+                    && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) {
+                    int cost = scheduler_gc_proc(p, reds);
+                    calls += cost;
+                    reds -= cost;
+                    if (reds <= 0)
+                        goto sched_out_proc;
+#ifdef ERTS_DIRTY_SCHEDULERS
+                    if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC))
+                        goto sched_out_proc;
+#endif
+                }
+            }
+        }
 
 	if (proxy_p) {
 	    free_proxy_proc(proxy_p);
@@ -10203,7 +10235,13 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
 	/* Never run a suspended process */
-	ASSERT(!(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&p->state)));
+#ifdef DEBUG
+        {
+            erts_aint32_t dstate = erts_smp_atomic32_read_nob(&p->state);
+            ASSERT(!(ERTS_PSFLG_SUSPENDED & dstate)
+                   || (ERTS_PSFLG_DIRTY_RUNNING_SYS & dstate));
+        }
+#endif
 
 	ASSERT(erts_proc_read_refc(p) > 0);
 
@@ -10224,9 +10262,18 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 }
 
 static int
-notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result)
+notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st,
+			 Eterm st_result, int normal_sched)
 {
-    Process *rp = erts_proc_lookup(st->requester);
+    Process *rp;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (!normal_sched)
+	rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
+			       st->requester, 0,
+			       ERTS_P2P_FLG_INC_REFC);
+    else
+#endif
+	rp = erts_proc_lookup(st->requester);
     if (rp) {
 	ErtsProcLocks rp_locks;
 	ErlOffHeap *ohp;
@@ -10274,6 +10321,11 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result)
 
 	if (rp_locks)
 	    erts_smp_proc_unlock(rp, rp_locks);
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+	if (!normal_sched)
+	    erts_proc_dec_refc(rp);
+#endif
     }
 
     erts_cleanup_offheap(&st->off_heap);
@@ -10429,18 +10481,23 @@ done:
 }
 
 static void save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio);
+#ifdef ERTS_DIRTY_SCHEDULERS
+static void save_dirty_task(Process *c_p, ErtsProcSysTask *st);
+#endif
 
 static int
 execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 {
-    int garbage_collected = 0;
+    int minor_gc = 0, major_gc = 0;
     erts_aint32_t state = *statep;
     int reds = in_reds;
     int qmask = 0;
 
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p)));
     ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
 
     do {
+	ErtsProcSysTaskType type;
 	ErtsProcSysTask *st;
 	int st_prio;
 	Eterm st_res;
@@ -10458,7 +10515,9 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	if (!st)
 	    break;
 
-	switch (st->type) {
+	type = st->type;
+
+	switch (type) {
         case ERTS_PSTT_GC_MAJOR:
         case ERTS_PSTT_GC_MINOR:
 	    if (c_p->flags & F_DISABLE_GC) {
@@ -10467,12 +10526,23 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 		reds--;
 	    }
 	    else {
-		if (!garbage_collected) {
-                    if (st->type == ERTS_PSTT_GC_MAJOR) {
+		if (!minor_gc
+		    || (!major_gc && type == ERTS_PSTT_GC_MAJOR)) {
+                    if (type == ERTS_PSTT_GC_MAJOR) {
                         FLAGS(c_p) |= F_NEED_FULLSWEEP;
                     }
 		    reds -= scheduler_gc_proc(c_p, reds);
-		    garbage_collected = 1;
+#ifdef ERTS_DIRTY_SCHEDULERS
+		    if (c_p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) {
+			save_dirty_task(c_p, st);
+			st = NULL;
+			break;
+		    }
+#endif
+		    if (type == ERTS_PSTT_GC_MAJOR)
+			minor_gc = major_gc = 1;
+		    else
+			minor_gc = 1;
 		}
 		st_res = am_true;
 	    }
@@ -10499,20 +10569,31 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	case ERTS_PSTT_CLA: {
 	    int fcalls;
             int cla_reds = 0;
+	    int do_gc;
+
 	    if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
 		fcalls = reds;
 	    else
 		fcalls = reds - CONTEXT_REDS;
-	    st_res = erts_proc_copy_literal_area(c_p,
-						 &cla_reds,
-						 fcalls,
-						 st->arg[0] == am_true);
+	    do_gc = st->arg[0] == am_true;
+	    st_res = erts_proc_copy_literal_area(c_p, &cla_reds,
+						 fcalls, do_gc);
             reds -= cla_reds;
 	    if (is_non_value(st_res)) {
+#ifdef ERTS_DIRTY_SCHEDULERS
+		if (c_p->flags & F_DIRTY_CLA) {
+		    save_dirty_task(c_p, st);
+		    st = NULL;
+		    break;
+		}
+#endif
 		/* Needed gc, but gc was disabled */
 		save_gc_task(c_p, st, st_prio);
 		st = NULL;
+		break;
 	    }
+	    if (do_gc) /* We did a major gc */
+		minor_gc = major_gc = 1;
 	    break;
         }
 	case ERTS_PSTT_COHMQ:
@@ -10531,7 +10612,7 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	}
 
 	if (st)
-	    reds += notify_sys_task_executed(c_p, st, st_res);
+	    reds += notify_sys_task_executed(c_p, st, st_res, 1);
 
 	state = erts_smp_atomic32_read_acqb(&c_p->state);
     } while (qmask && reds > 0);
@@ -10559,9 +10640,18 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
 	Eterm st_res;
 	int st_prio;
 
-	st = fetch_sys_task(c_p, state, &qmask, &st_prio);
-	if (!st)
-	    break;
+#ifdef ERTS_DIRTY_SCHEDULERS
+	if (c_p->dirty_sys_tasks) {
+	    st = c_p->dirty_sys_tasks;
+	    c_p->dirty_sys_tasks = st->next;
+	}
+	else
+#endif
+	{
+	    st = fetch_sys_task(c_p, state, &qmask, &st_prio);
+	    if (!st)
+		break;
+	}
 
 	switch (st->type) {
         case ERTS_PSTT_GC_MAJOR:
@@ -10585,7 +10675,7 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
 	    break;
 	}
 
-	reds += notify_sys_task_executed(c_p, st, st_res);
+	reds += notify_sys_task_executed(c_p, st, st_res, 1);
 
 	state = erts_smp_atomic32_read_acqb(&c_p->state);
     } while (qmask && reds < max_reds);
@@ -10595,6 +10685,92 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
 
 #ifdef ERTS_DIRTY_SCHEDULERS
 
+void
+erts_execute_dirty_system_task(Process *c_p)
+{
+    Eterm cla_res = THE_NON_VALUE;
+    ErtsProcSysTask *stasks;
+
+    /*
+     * If multiple operations, perform them in the following
+     * order (in order to avoid unnecessary GC):
+     *  1. Copy Literal Area (implies major GC).
+     *  2. GC Hibernate (implies major GC if not woken).
+     *  3. Major GC (implies minor GC).
+     *  4. Minor GC.
+     *
+     * System task requests are handled after the actual
+     * operations have been performed...
+     */
+
+    ASSERT(!(c_p->flags & (F_DELAY_GC|F_DISABLE_GC)));
+
+    if (c_p->flags & F_DIRTY_CLA) {
+	int cla_reds = 0;
+	cla_res = erts_proc_copy_literal_area(c_p, &cla_reds, c_p->fcalls, 1);
+	ASSERT(is_value(cla_res));
+    }
+
+    if (c_p->flags & F_DIRTY_GC_HIBERNATE) {
+	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
+	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
+	if (c_p->msg.len)
+            c_p->flags &= ~F_DIRTY_GC_HIBERNATE; /* operation aborted... */
+        else {
+	    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
+	    c_p->fvalue = NIL;
+	    erts_garbage_collect_hibernate(c_p);
+	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+	    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
+	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+	}
+	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
+    }
+
+    if (c_p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) {
+        if (c_p->flags & F_DIRTY_MAJOR_GC)
+            c_p->flags |= F_NEED_FULLSWEEP;
+	(void) erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg,
+					   c_p->arity, c_p->fcalls);
+    }
+
+    ASSERT(!(c_p->flags & (F_DIRTY_CLA
+			   | F_DIRTY_GC_HIBERNATE
+			   | F_DIRTY_MAJOR_GC
+			   | F_DIRTY_MINOR_GC)));
+
+    stasks = c_p->dirty_sys_tasks;
+    c_p->dirty_sys_tasks = NULL;
+    
+    while (stasks) {
+	Eterm st_res;
+	ErtsProcSysTask *st = stasks;
+	stasks = st->next;
+
+	switch (st->type) {
+	case ERTS_PSTT_CLA:
+	    ASSERT(is_value(st_res));
+	    st_res = cla_res;
+	    break;
+	case ERTS_PSTT_GC_MAJOR:
+	    st_res = am_true;
+	    break;
+	case ERTS_PSTT_GC_MINOR:
+	    st_res = am_true;
+	    break;
+
+	default:
+	    ERTS_INTERNAL_ERROR("Not supported dirty system task");
+	    break;
+	}
+
+	(void) notify_sys_task_executed(c_p, st, st_res, 0);
+
+    }
+
+    erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_DIRTY_ACTIVE_SYS);
+}
+
 static BIF_RETTYPE
 dispatch_system_task(Process *c_p, erts_aint_t fail_state,
 		     ErtsProcSysTask *st, Eterm target,
@@ -10797,7 +10973,7 @@ request_system_task(Process *c_p, Eterm requester, Eterm target,
 	    ERTS_INTERNAL_ERROR("Unknown failure schedule_process_sys_task()");
 	    failure = am_internal_error;
 	}
-	notify_sys_task_executed(c_p, st, failure);
+	notify_sys_task_executed(c_p, st, failure, 1);
     }
 
     ERTS_BIF_PREP_RET(ret, am_ok);
@@ -11012,6 +11188,15 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio)
     }
 }
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+static void
+save_dirty_task(Process *c_p, ErtsProcSysTask *st)
+{
+    st->next = c_p->dirty_sys_tasks;
+    c_p->dirty_sys_tasks = st;
+}
+#endif
+
 int
 erts_set_gc_state(Process *c_p, int enable)
 {
@@ -11348,6 +11533,7 @@ static void early_init_process_struct(void *varg, Eterm data)
     proc->common.id = make_internal_pid(data);
 #ifdef ERTS_DIRTY_SCHEDULERS
     erts_smp_atomic32_init_nob(&proc->dirty_state, 0);
+    proc->dirty_sys_tasks = NULL;
 #endif
     erts_smp_atomic32_init_relb(&proc->state, arg->state);
 
@@ -11856,6 +12042,7 @@ void erts_init_empty_process(Process *p)
 
 #ifdef ERTS_DIRTY_SCHEDULERS
     erts_smp_atomic32_init_nob(&p->dirty_state, 0);
+    p->dirty_sys_tasks = NULL;
 #endif
     erts_smp_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL);
 
@@ -12443,7 +12630,9 @@ send_exit_signal(Process *c_p,		/* current process if and only
 		}
 		set_proc_exiting(c_p, state, rsn, NULL);
 	    }
-	    else if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) {
+	    else if (!(state & (ERTS_PSFLG_RUNNING
+                                | ERTS_PSFLG_RUNNING_SYS
+                                | ERTS_PSFLG_DIRTY_RUNNING_SYS))) {
 		/* Process not running ... */
 		ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL;
 		ErlHeapFragment *bp = NULL;
@@ -12470,8 +12659,7 @@ send_exit_signal(Process *c_p,		/* current process if and only
 		    ErlOffHeap *ohp;
 		    Uint rsn_sz = size_object(rsn);
 #ifdef ERTS_DIRTY_SCHEDULERS
-		    if (state & (ERTS_PSFLG_DIRTY_RUNNING
-				 | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
+		    if (state & ERTS_PSFLG_DIRTY_RUNNING) {
 			bp = new_message_buffer(rsn_sz);
 			ohp = &bp->off_heap;
 			hp = &bp->mem[0];
@@ -12488,7 +12676,7 @@ send_exit_signal(Process *c_p,		/* current process if and only
 		set_proc_exiting(rp, state, rsn_cpy, bp);
 	    }
 	    else { /* Process running... */
-
+ 
 		/*
 		 * The pending exit will be discovered when the process
 		 * is scheduled out if not discovered earlier.
@@ -12510,8 +12698,35 @@ send_exit_signal(Process *c_p,		/* current process if and only
 							  &bp->off_heap);
 		    rp->pending_exit.bp = bp;
 		}
-		erts_smp_atomic32_read_bor_relb(&rp->state,
-						ERTS_PSFLG_PENDING_EXIT);
+
+		/*
+		 * If no dirty work has been scheduled, pending exit will
+                 * be discovered when the process is scheduled. If dirty work
+                 * has been scheduled, we may need to add it to a normal run
+                 * queue...
+		 */
+#ifndef ERTS_DIRTY_SCHEDULERS
+                (void) erts_smp_atomic32_read_bor_relb(&rp->state,
+                                                       ERTS_PSFLG_PENDING_EXIT);
+#else
+                {
+                    erts_aint32_t a = erts_smp_atomic32_read_nob(&rp->state);
+                    while (1) {
+                        erts_aint32_t n, e;
+                        int dwork;
+                        n = e = a;
+                        n |= ERTS_PSFLG_PENDING_EXIT;
+                        dwork = !!(n & ERTS_PSFLGS_DIRTY_WORK);
+                        n &= ~ERTS_PSFLGS_DIRTY_WORK;
+                        a = erts_smp_atomic32_cmpxchg_mb(&rp->state, n, e);
+                        if (a == e) {
+                            if (dwork)
+                                erts_schedule_process(rp, n, *rp_locks);
+                            break;
+                        }
+                    }
+                }
+#endif
 	    }
 	}
 	/* else:
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 58a50f8cf4..3938c4c913 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1060,6 +1060,9 @@ struct process {
     Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */
 
     ErtsProcSysTaskQs *sys_task_qs;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ErtsProcSysTask *dirty_sys_tasks;
+#endif
 
     erts_smp_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -1384,14 +1387,18 @@ extern int erts_system_profile_ts_type;
 #define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
 #define F_DISABLE_GC         (1 << 11) /* Disable GC (see below) */
 #define F_OFF_HEAP_MSGQ      (1 << 12) /* Off heap msg queue */
-#define F_ON_HEAP_MSGQ       (1 << 13) /* Off heap msg queue */
+#define F_ON_HEAP_MSGQ       (1 << 13) /* On heap msg queue */
 #define F_OFF_HEAP_MSGQ_CHNG (1 << 14) /* Off heap msg queue changing */
 #define F_ABANDONED_HEAP_USE (1 << 15) /* Have usage of abandoned heap */
 #define F_DELAY_GC           (1 << 16) /* Similar to disable GC (see below) */
 #define F_SCHDLR_ONLN_WAITQ  (1 << 17) /* Process enqueued waiting to change schedulers online */
 #define F_HAVE_BLCKD_NMSCHED (1 << 18) /* Process has blocked normal multi-scheduling */
-#define F_HIPE_MODE          (1 << 19)
+#define F_HIPE_MODE          (1 << 19) /* Process is executing in HiPE mode */
 #define F_DELAYED_DEL_PROC   (1 << 20) /* Delay delete process (dirty proc exit case) */
+#define F_DIRTY_CLA          (1 << 21) /* Dirty copy literal area scheduled */
+#define F_DIRTY_GC_HIBERNATE (1 << 22) /* Dirty GC hibernate scheduled */
+#define F_DIRTY_MAJOR_GC     (1 << 23) /* Dirty major GC scheduled */
+#define F_DIRTY_MINOR_GC     (1 << 24) /* Dirty minor GC scheduled */
 
 /*
  * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent
@@ -1573,7 +1580,9 @@ void erts_init_scheduling(int, int
 			  , int, int, int
 #endif
 			  );
-
+#ifdef ERTS_DIRTY_SCHEDULERS
+void erts_execute_dirty_system_task(Process *c_p);
+#endif
 int erts_set_gc_state(Process *c_p, int enable);
 Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
 Eterm erts_system_check_request(Process *c_p);
@@ -1918,6 +1927,8 @@ ErtsSchedulerData *erts_get_scheduler_data(void)
 void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks);
 
 ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks);
+ERTS_GLB_INLINE void erts_schedule_dirty_sys_execution(Process *c_p);
+
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 ERTS_GLB_INLINE void
 erts_proc_notify_new_message(Process *p, ErtsProcLocks locks)
@@ -1927,6 +1938,34 @@ erts_proc_notify_new_message(Process *p, ErtsProcLocks locks)
     if (!(state & ERTS_PSFLG_ACTIVE))
 	erts_schedule_process(p, state, locks);
 }
+
+ERTS_GLB_INLINE void
+erts_schedule_dirty_sys_execution(Process *c_p)
+{
+    erts_aint32_t a, n, e;
+
+    a = erts_smp_atomic32_read_nob(&c_p->state);
+
+    /*
+     * Only a currently executing process schedules
+     * itself for dirty-sys execution...
+     */
+
+    ASSERT(a & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS));
+
+    /* Don't set dirty-active-sys if we are about to exit... */
+
+    while (!(a & (ERTS_PSFLG_DIRTY_ACTIVE_SYS
+                  | ERTS_PSFLG_EXITING
+                  | ERTS_PSFLG_PENDING_EXIT))) {
+        e = a;
+        n = a | ERTS_PSFLG_DIRTY_ACTIVE_SYS;
+        a = erts_smp_atomic32_cmpxchg_mb(&c_p->state, n, e);
+        if (a == e)
+            break; /* dirty-active-sys set */
+    }
+}
+
 #endif
 
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 4ef04d020a..ec36b23059 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -802,15 +802,10 @@ call_ext_last u==1 Bif=u$bif:erts_internal:check_process_code/1 D => i_call_ext_
 call_ext_only u==1 Bif=u$bif:erts_internal:check_process_code/1 => i_call_ext_only Bif
 
 #
-# The BIFs erlang:garbage_collect/0 must be called like a function,
+# The BIFs erts_internal:garbage_collect/1 must be called like a function,
 # to allow them to invoke the garbage collector. (The stack pointer must
 # be saved and p->arity must be zeroed, which is not done on ordinary BIF calls.)
 #
-
-call_ext u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext Bif
-call_ext_last u==0 Bif=u$bif:erlang:garbage_collect/0 D => i_call_ext_last Bif D
-call_ext_only u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext_only Bif
-
 call_ext u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext Bif
 call_ext_last u==1 Bif=u$bif:erts_internal:garbage_collect/1 D => i_call_ext_last Bif D
 call_ext_only u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext_only Bif
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index ec0d0f5a0d..f034c4700c 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -183,7 +183,7 @@ standard_bif_interface_0(nbif_ports_0, ports_0)
  */
 gc_bif_interface_1(nbif_erts_internal_check_process_code_1, hipe_erts_internal_check_process_code_1)
 gc_bif_interface_1(nbif_erase_1, erase_1)
-gc_bif_interface_0(nbif_garbage_collect_0, garbage_collect_0)
+gc_bif_interface_1(nbif_erts_internal_garbage_collect_1, erts_internal_garbage_collect_1)
 gc_nofail_primop_interface_1(nbif_gc_1, hipe_gc)
 gc_bif_interface_2(nbif_put_2, put_2)
 
-- 
cgit v1.2.3


From 9e9e7bf64fa97663dd4fb645c014880205fb46db Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 9 Jan 2017 15:13:58 +0100
Subject: erts: Add assertions for correct ErlNifEnv

when constructing container terms.
---
 erts/emulator/beam/erl_nif.c | 107 +++++++++++++++++++++++++++++++++++++++++--
 erts/emulator/beam/global.h  |   7 +++
 2 files changed, 110 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 5499512dd1..10514f6b3f 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -89,6 +89,14 @@ static void add_readonly_check(ErlNifEnv*, unsigned char* ptr, unsigned sz);
 #  define ADD_READONLY_CHECK(ENV,PTR,SIZE) ((void)0)
 #endif
 
+#ifdef ERTS_NIF_ASSERT_IN_ENV
+#  define ASSERT_IN_ENV(ENV, TERM, NR, TYPE) dbg_assert_in_env(ENV, TERM, NR, TYPE, __func__)
+static void dbg_assert_in_env(ErlNifEnv*, Eterm term, int nr, const char* type, const char* func);
+#  include "erl_gc.h"
+#else
+#  define ASSERT_IN_ENV(ENV, TERM, NR, TYPE)
+#endif
+
 #ifdef DEBUG
 static int is_offheap(const ErlOffHeap* off_heap);
 #endif
@@ -196,6 +204,9 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
 
     ASSERT(p->common.id != ERTS_INVALID_PID);
 
+#ifdef ERTS_NIF_ASSERT_IN_ENV
+    env->dbg_disable_assert_in_env = 0;
+#endif
 #if defined(DEBUG) && defined(ERTS_DIRTY_SCHEDULERS)
     {
 	ErtsSchedulerData *esdp = erts_get_scheduler_data();
@@ -474,11 +485,15 @@ setup_nif_env(struct enif_msg_environment_t* msg_env,
     HEAP_END(&msg_env->phony_proc) = phony_heap;
     MBUF(&msg_env->phony_proc) = NULL;
     msg_env->phony_proc.common.id = ERTS_INVALID_PID;
+    msg_env->env.tracee = tracee;
+
 #ifdef FORCE_HEAP_FRAGS
     msg_env->phony_proc.space_verified = 0;
     msg_env->phony_proc.space_verified_from = NULL;
 #endif
-    msg_env->env.tracee = tracee;
+#ifdef ERTS_NIF_ASSERT_IN_ENV
+    msg_env->env.dbg_disable_assert_in_env = 0;
+#endif
 }
 
 ErlNifEnv* enif_alloc_env(void)
@@ -1586,6 +1601,9 @@ int enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len,
 
 ERL_NIF_TERM enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...)
 {
+#ifdef ERTS_NIF_ASSERT_IN_ENV
+    int nr = 0;
+#endif
     Eterm* hp = alloc_heap(env,cnt+1);
     Eterm ret = make_tuple(hp);
     va_list ap;
@@ -1593,7 +1611,9 @@ ERL_NIF_TERM enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...)
     *hp++ = make_arityval(cnt);
     va_start(ap,cnt);
     while (cnt--) {
-	*hp++ = va_arg(ap,Eterm);	   
+        Eterm elem = va_arg(ap,Eterm);
+        ASSERT_IN_ENV(env, elem, ++nr, "tuple");
+	*hp++ = elem;
     }
     va_end(ap);
     return ret;
@@ -1601,12 +1621,16 @@ ERL_NIF_TERM enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...)
 
 ERL_NIF_TERM enif_make_tuple_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], unsigned cnt)
 {
+#ifdef ERTS_NIF_ASSERT_IN_ENV
+    int nr = 0;
+#endif
     Eterm* hp = alloc_heap(env,cnt+1);
     Eterm ret = make_tuple(hp);
     const Eterm* src = arr;
 
     *hp++ = make_arityval(cnt);
     while (cnt--) {
+        ASSERT_IN_ENV(env, *src, ++nr, "tuple");
 	*hp++ = *src++;	   
     }
     return ret;
@@ -1617,6 +1641,8 @@ ERL_NIF_TERM enif_make_list_cell(ErlNifEnv* env, Eterm car, Eterm cdr)
     Eterm* hp = alloc_heap(env,2);
     Eterm ret = make_list(hp);
 
+    ASSERT_IN_ENV(env, car, 0, "head of list cell");
+    ASSERT_IN_ENV(env, cdr, 0, "tail of list cell");
     CAR(hp) = car;
     CDR(hp) = cdr;
     return ret;
@@ -1628,6 +1654,9 @@ ERL_NIF_TERM enif_make_list(ErlNifEnv* env, unsigned cnt, ...)
 	return NIL;
     }
     else {
+#ifdef ERTS_NIF_ASSERT_IN_ENV
+        int nr = 0;
+#endif
 	Eterm* hp = alloc_heap(env,cnt*2);
 	Eterm ret = make_list(hp);
 	Eterm* last = &ret;
@@ -1635,8 +1664,10 @@ ERL_NIF_TERM enif_make_list(ErlNifEnv* env, unsigned cnt, ...)
 
 	va_start(ap,cnt);
 	while (cnt--) {
+            Eterm term = va_arg(ap,Eterm);
 	    *last = make_list(hp);
-	    *hp = va_arg(ap,Eterm);
+            ASSERT_IN_ENV(env, term, ++nr, "list");
+	    *hp = term;
 	    last = ++hp;
 	    ++hp;
 	}
@@ -1648,14 +1679,19 @@ ERL_NIF_TERM enif_make_list(ErlNifEnv* env, unsigned cnt, ...)
 
 ERL_NIF_TERM enif_make_list_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], unsigned cnt)
 {
+#ifdef ERTS_NIF_ASSERT_IN_ENV
+    int nr = 0;
+#endif
     Eterm* hp = alloc_heap(env,cnt*2);
     Eterm ret = make_list(hp);
     Eterm* last = &ret;
     const Eterm* src = arr;
 
     while (cnt--) {
+        Eterm term = *src++;
 	*last = make_list(hp);
-	*hp = *src++;
+        ASSERT_IN_ENV(env, term, ++nr, "list");
+	*hp = term;
 	last = ++hp;
 	++hp;
     }
@@ -2789,6 +2825,10 @@ int enif_make_map_put(ErlNifEnv* env,
     if (!is_map(map_in)) {
 	return 0;
     }
+    ASSERT_IN_ENV(env, map_in, 0, "old map");
+    ASSERT_IN_ENV(env, key, 0, "key");
+    ASSERT_IN_ENV(env, value, 0, "value");
+
     flush_env(env);
     *map_out = erts_maps_put(env->proc, key, value, map_in);
     cache_env(env);
@@ -2823,6 +2863,10 @@ int enif_make_map_update(ErlNifEnv* env,
 	return 0;
     }
 
+    ASSERT_IN_ENV(env, map_in, 0, "old map");
+    ASSERT_IN_ENV(env, key, 0, "key");
+    ASSERT_IN_ENV(env, value, 0, "value");
+
     flush_env(env);
     res = erts_maps_update(env->proc, key, value, map_in, map_out);
     cache_env(env);
@@ -3543,6 +3587,9 @@ Eterm erts_nif_call_function(Process *p, Process *tracee,
         clear_offheap(&MSO(p));
 
         erts_pre_nif(&env, p, mod, tracee);
+#ifdef ERTS_NIF_ASSERT_IN_ENV
+        env.dbg_disable_assert_in_env = 1;
+#endif
         nif_result = (*fun->fptr)(&env, argc, argv);
         if (env.exception_thrown)
             nif_result = THE_NON_VALUE;
@@ -3565,6 +3612,9 @@ Eterm erts_nif_call_function(Process *p, Process *tracee,
            so we create a phony one. */
         struct enif_msg_environment_t msg_env;
         pre_nif_noproc(&msg_env, mod, tracee);
+#ifdef ERTS_NIF_ASSERT_IN_ENV
+        msg_env.env.dbg_disable_assert_in_env = 1;
+#endif
         nif_result = (*fun->fptr)(&msg_env.env, argc, argv);
         if (msg_env.env.exception_thrown)
             nif_result = THE_NON_VALUE;
@@ -3629,6 +3679,55 @@ static unsigned calc_checksum(unsigned char* ptr, unsigned size)
 
 #endif /* READONLY_CHECK */
 
+#ifdef ERTS_NIF_ASSERT_IN_ENV
+static void dbg_assert_in_env(ErlNifEnv* env, Eterm term,
+                              int nr, const char* type, const char* func)
+{
+    Uint saved_used_size;
+    Eterm* real_htop;
+
+    if (is_immed(term)
+        || (is_non_value(term) && env->exception_thrown)
+        || erts_is_literal(term, ptr_val(term)))
+        return;
+
+    if (env->dbg_disable_assert_in_env) {
+        /*
+         * Trace nifs may cheat as built terms are discarded after return.
+         * ToDo: Check if 'term' is part of argv[].
+         */
+        return;
+    }
+
+    if (env->heap_frag) {
+        ASSERT(env->heap_frag == MBUF(env->proc));
+        ASSERT(env->hp >= env->heap_frag->mem);
+        ASSERT(env->hp <= env->heap_frag->mem + env->heap_frag->alloc_size);
+        saved_used_size = env->heap_frag->used_size;
+        env->heap_frag->used_size = env->hp - env->heap_frag->mem;
+        real_htop = NULL;
+    }
+    else {
+        real_htop = env->hp;
+    }
+    if (!erts_dbg_within_proc(ptr_val(term), env->proc, real_htop)) {
+        fprintf(stderr, "\r\nFAILED ASSERTION in %s:\r\n", func);
+        if (nr) {
+            fprintf(stderr, "Term #%d of the %s is not from same ErlNifEnv.",
+                    nr, type);
+        }
+        else {
+            fprintf(stderr, "The %s is not from the same ErlNifEnv.", type);
+        }
+        fprintf(stderr, "\r\nABORTING\r\n");
+        abort();
+    }
+    if (env->heap_frag) {
+        env->heap_frag->used_size = saved_used_size;
+    }
+}
+#endif
+
 #ifdef HAVE_USE_DTRACE
 
 #define MESSAGE_BUFSIZ 1024
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 2b2f3c5cdc..5d07490152 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -45,6 +45,9 @@
 
 struct enif_func_t;
 
+#ifdef  DEBUG
+#  define ERTS_NIF_ASSERT_IN_ENV
+#endif
 struct enif_environment_t /* ErlNifEnv */
 {
     struct erl_module_nif* mod_nif;
@@ -57,6 +60,10 @@ struct enif_environment_t /* ErlNifEnv */
     int exception_thrown; /* boolean */
     Process *tracee;
     int exiting; /* boolean (dirty nifs might return in exiting state) */
+
+#ifdef ERTS_NIF_ASSERT_IN_ENV
+    int dbg_disable_assert_in_env;
+#endif
 };
 extern void erts_pre_nif(struct enif_environment_t*, Process*,
 			 struct erl_module_nif*, Process* tracee);
-- 
cgit v1.2.3


From 10ace079fb6d0d626a16a5c6726c6bb8e6bb3878 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 21 Dec 2016 17:43:28 +0100
Subject: erts: Cleanup enif_make_reverse_list

---
 erts/emulator/beam/erl_nif.c | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 10514f6b3f..cd44eb880e 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1724,13 +1724,9 @@ void enif_system_info(ErlNifSysInfo *sip, size_t si_size)
     driver_system_info(sip, si_size);
 }
 
-int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list) {
-    Eterm *listptr, ret = NIL, *hp;
-
-    if (is_nil(term)) {
-	*list = term;
-        return 1;
-    }
+int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list)
+{
+    Eterm *listptr, ret, *hp;
 
     ret = NIL;
 
-- 
cgit v1.2.3


From 1239d92556cc0a7921648cbd4abd9f9a397b29b8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 21 Dec 2016 18:12:14 +0100
Subject: erts: Cleanup and extra assertions in nif_SUITE.c

---
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 2c93891852..4decb7f418 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -116,7 +116,6 @@ static ERL_NIF_TERM make_pointer(ErlNifEnv* env, void* p)
 {
     void** bin_data;
     ERL_NIF_TERM res;
-    ADD_CALL("get_priv_data_ptr");
     bin_data = (void**)enif_make_new_binary(env, sizeof(void*), &res);
     *bin_data = p;
     return res;
@@ -389,8 +388,7 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
     ErlNifSInt64 sint64;
     ErlNifUInt64 uint64;
     double d;
-    ERL_NIF_TERM atom, ref1, ref2, term;
-    size_t len;
+    ERL_NIF_TERM atom, ref1, ref2;
 
     sint = INT_MIN;
     do {
@@ -1024,6 +1022,7 @@ struct make_term_info
 {
     ErlNifEnv* caller_env;
     ErlNifEnv* dst_env;
+    int dst_env_valid;
     ERL_NIF_TERM reuse[MAKE_TERM_REUSE_LEN];
     unsigned reuse_push;
     unsigned reuse_pull;
@@ -1053,6 +1052,7 @@ static ERL_NIF_TERM pull_term(struct make_term_info* mti)
 	mti->reuse_push < MAKE_TERM_REUSE_LEN) {
 	mti->reuse_pull = 0;
 	if (mti->reuse_push == 0) {
+            assert(mti->dst_env_valid);
 	    mti->reuse[0] = enif_make_list(mti->dst_env, 0);
 	}
     }
@@ -1241,6 +1241,7 @@ static unsigned num_of_make_funcs()
 static int make_term_n(struct make_term_info* mti, int n, ERL_NIF_TERM* res)
 {
     if (n < num_of_make_funcs()) {
+        assert(mti->dst_env_valid);
 	*res = make_funcs[n](mti, n);
 	push_term(mti, *res);
 	return 1;
@@ -1257,6 +1258,7 @@ static ERL_NIF_TERM make_blob(ErlNifEnv* caller_env, ErlNifEnv* dst_env,
     struct make_term_info mti;
     mti.caller_env = caller_env;
     mti.dst_env = dst_env;
+    mti.dst_env_valid = 1;
     mti.reuse_push = 0;
     mti.reuse_pull = 0;
     mti.resource_type = priv->rt_arr[0].t;
@@ -1297,6 +1299,7 @@ static ERL_NIF_TERM alloc_msgenv(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
 						       sizeof(*mti));
     mti->caller_env = NULL;
     mti->dst_env = enif_alloc_env();
+    mti->dst_env_valid = 1;
     mti->reuse_push = 0;
     mti->reuse_pull = 0;
     mti->resource_type = priv->rt_arr[0].t;
@@ -1328,6 +1331,7 @@ static ERL_NIF_TERM clear_msgenv(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
 	return enif_make_badarg(env);
     }
     enif_clear_env(mti.p->dst_env);
+    mti.p->dst_env_valid = 1;
     mti.p->reuse_pull = 0;
     mti.p->reuse_push = 0;
     mti.p->blob = enif_make_list(mti.p->dst_env, 0);
@@ -1362,6 +1366,8 @@ static ERL_NIF_TERM send_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
     }
     copy = enif_make_copy(env, mti.p->blob);
     res = enif_send(env, &to, mti.p->dst_env, mti.p->blob);
+    if (res)
+        mti.p->dst_env_valid = 0;
     return enif_make_tuple3(env, atom_ok, enif_make_int(env,res), copy);
 }
 
@@ -1369,7 +1375,6 @@ static ERL_NIF_TERM send3_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
 {
     mti_t mti;
     ErlNifPid to;
-    ERL_NIF_TERM copy;
     int res;
     if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)
 	|| !enif_get_local_pid(env, argv[1], &to)) {
@@ -1379,6 +1384,8 @@ static ERL_NIF_TERM send3_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
 				   enif_make_copy(mti.p->dst_env, argv[2]),
 				   mti.p->blob);
     res = enif_send(env, &to, mti.p->dst_env, mti.p->blob);
+    if (res)
+        mti.p->dst_env_valid = 0;
     return enif_make_int(env,res);
 }
 
@@ -1395,6 +1402,8 @@ void* threaded_sender(void *arg)
     mti.p->send_it = 0;
     enif_mutex_unlock(mti.p->mtx);
     mti.p->send_res = enif_send(NULL, &mti.p->to_pid, mti.p->dst_env, mti.p->blob);
+    if (mti.p->send_res)
+        mti.p->dst_env_valid = 0;
     return NULL;
 }
 
-- 
cgit v1.2.3


From 9e862df2bf26b9b1d79ba3b447945b7c0612e3d0 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 12 Jan 2017 18:12:38 +0100
Subject: erts: Add macro ERTS_PROC_LOCKS_HIGHER_THAN

to safeguard against bugs due to future proc lock changes.

The two places now using ERTS_PROC_LOCKS_HIGHER_THAN were
kind of bugs as BTM and TRACE locks were missing. But there was
probably no way to get there with BTM or TRACE locked.
---
 erts/emulator/beam/erl_message.c      | 10 ++++++----
 erts/emulator/beam/erl_process_lock.h |  4 ++++
 2 files changed, 10 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 118adc0c1b..04f9640a17 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -285,9 +285,11 @@ erts_queue_dist_message(Process *rcvr,
     if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
 	if (erts_smp_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
 	    ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
-	    if (rcvr_locks & ERTS_PROC_LOCK_STATUS) {
-		erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_STATUS);
-		need_locks |= ERTS_PROC_LOCK_STATUS;
+            ErtsProcLocks unlocks =
+                rcvr_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
+	    if (unlocks) {
+		erts_smp_proc_unlock(rcvr, unlocks);
+		need_locks |= unlocks;
 	    }
 	    erts_smp_proc_lock(rcvr, need_locks);
 	}
@@ -406,7 +408,7 @@ queue_messages(Process* receiver,
 	    if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
 		goto exiting;
 
-            need_locks = receiver_locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+            need_locks = receiver_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
 	    if (need_locks) {
 		erts_smp_proc_unlock(receiver, need_locks);
 	    }
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 2cccf0697a..e7d68f0a09 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -219,6 +219,10 @@ typedef struct erts_proc_lock_t_ {
 #define ERTS_PROC_LOCKS_ALL_MINOR	(ERTS_PROC_LOCKS_ALL \
                                          & ~ERTS_PROC_LOCK_MAIN)
 
+/* All locks we first must unlock to lock L */
+#define ERTS_PROC_LOCKS_HIGHER_THAN(L) \
+  (ERTS_PROC_LOCKS_ALL & (~(L) & ~((L)-1)))
+
 
 #define ERTS_PIX_LOCKS_BITS		10
 #define ERTS_NO_OF_PIX_LOCKS		(1 << ERTS_PIX_LOCKS_BITS)
-- 
cgit v1.2.3


From 5de2d73abd5d369676e43acd17a9d9db9795ccd5 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 13 Jan 2017 14:22:31 +0100
Subject: Fix memory leak of temporary heap

This bug was introduced in previous commit, and has never been
released in an official OTP version.
---
 erts/emulator/beam/erl_gc.c      | 11 +++++------
 erts/emulator/beam/erl_gc.h      |  1 +
 erts/emulator/beam/erl_process.c | 18 +-----------------
 3 files changed, 7 insertions(+), 23 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 561e7fbdee..ace1524326 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -110,7 +110,6 @@ typedef struct {
 
 static Uint setup_rootset(Process*, Eterm*, int, Rootset*);
 static void cleanup_rootset(Rootset *rootset);
-static void deallocate_previous_young_generation(Process *c_p);
 static Eterm *full_sweep_heaps(Process *p,
 			       int hibernate,
 			       Eterm *n_heap, Eterm* n_htop,
@@ -839,7 +838,7 @@ erts_garbage_collect_hibernate(Process* p)
     disallow_heap_frag_ref_in_heap(p, heap, htop);
 #endif
 
-    deallocate_previous_young_generation(p);
+    erts_deallocate_young_generation(p);
 
     p->heap = heap;
     p->high_water = htop;
@@ -1496,7 +1495,7 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
     disallow_heap_frag_ref_in_heap(p, n_heap, n_htop);
 #endif
 
-    deallocate_previous_young_generation(p);
+    erts_deallocate_young_generation(p);
 
     HEAP_START(p) = n_heap;
     HEAP_TOP(p) = n_htop;
@@ -1594,7 +1593,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
     disallow_heap_frag_ref_in_heap(p, n_heap, n_htop);
 #endif
 
-    deallocate_previous_young_generation(p);
+    erts_deallocate_young_generation(p);
 
     HEAP_START(p) = n_heap;
     HEAP_TOP(p) = n_htop;
@@ -1756,8 +1755,8 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
     return adjusted;
 }
 
-static void
-deallocate_previous_young_generation(Process *c_p)
+void
+erts_deallocate_young_generation(Process *c_p)
 {
     Eterm *orig_heap;
 
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 54ea9ca3c0..9a177b7c59 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -157,5 +157,6 @@ void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*);
 void erts_free_heap_frags(struct process* p);
 Eterm erts_max_heap_size_map(Sint, Uint, Eterm **, Uint *);
 int erts_max_heap_size(Eterm, Uint *, Uint *);
+void erts_deallocate_young_generation(Process *c_p);
 
 #endif /* __ERL_GC_H__ */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b345c35a7e..8d13723188 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11912,7 +11912,6 @@ erts_cleanup_empty_process(Process* p)
 static void
 delete_process(Process* p)
 {
-    Eterm *heap;
     ErtsPSD *psd;
     struct saved_calls *scb;
     process_breakpoint_time_t *pbt;
@@ -11967,13 +11966,8 @@ delete_process(Process* p)
     hipe_delete_process(&p->hipe);
 #endif
 
-    heap = p->abandoned_heap ? p->abandoned_heap : p->heap;
+    erts_deallocate_young_generation(p);
 
-#ifdef DEBUG
-    sys_memset(heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm));
-#endif
-
-    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) heap, p->heap_sz*sizeof(Eterm));
     if (p->old_heap != NULL) {
 
 #ifdef DEBUG
@@ -11985,16 +11979,6 @@ delete_process(Process* p)
 		       (p->old_hend-p->old_heap)*sizeof(Eterm));
     }
 
-    /*
-     * Free all pending message buffers.
-     */
-    if (p->mbuf != NULL) {	
-	free_message_buffer(p->mbuf);
-    }
-
-    if (p->msg_frag)
-	erts_cleanup_messages(p->msg_frag);
-
     erts_erase_dicts(p);
 
     /* free all pending messages */
-- 
cgit v1.2.3


From 7350e237d66f5098b53de7fa58e725753c04d530 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 13 Jan 2017 17:46:58 +0100
Subject: Do not automatically define HARDDEBUG when DEBUG is defined

---
 erts/emulator/beam/erl_gc.c | 4 ++++
 erts/emulator/beam/erl_gc.h | 4 ----
 2 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index af799d09da..46be91c5d5 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -59,6 +59,10 @@
 #  define ERTS_GC_ASSERT(B) ((void) 1)
 #endif
 
+#if defined(DEBUG) && 0
+#  define HARDDEBUG 1
+#endif
+
 /*
  * Returns number of elements in an array.
  */
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 54ea9ca3c0..49cc43a48f 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -27,10 +27,6 @@
 
 #include "erl_map.h"
 
-#if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF
-#  define HARDDEBUG 1
-#endif
-
 #define IS_MOVED_BOXED(x)	(!is_header((x)))
 #define IS_MOVED_CONS(x)	(is_non_value((x)))
 
-- 
cgit v1.2.3


From d74d7a4cc0808e1f6e710e5d17ec55bfe093cdd9 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 13 Jan 2017 18:19:30 +0100
Subject: erts: Fix enif_send from noproc and no msg_env

Only try direct allocation on receiver heap when called
in a real process context to avoid trylock on our own main lock.
---
 erts/emulator/beam/erl_nif.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 6b265a8b80..dd4b88e4a3 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -690,7 +690,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
         Uint sz = size_object(msg);
 	ErlOffHeap *ohp;
         Eterm *hp;
-	if (env && !env->tracee) {
+	if (c_p && !env->tracee) {
 	    full_flush_env(env);
 	    mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
 	    full_cache_env(env);
-- 
cgit v1.2.3


From 43eebdd25a0d4a17f076f017cb3ea7da5cfd1808 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 22 Dec 2016 17:48:51 +0100
Subject: Switch between scheduler types when multi-scheduling is blocked

---
 erts/emulator/beam/bif.h           |   6 -
 erts/emulator/beam/erl_port_task.c |   2 +-
 erts/emulator/beam/erl_process.c   | 996 ++++++++++++++++++++++++-------------
 erts/emulator/beam/erl_process.h   |  22 +-
 4 files changed, 674 insertions(+), 352 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index ad55b19d0a..01cca90a7a 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -498,12 +498,6 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
 					 Eterm args[],
 					 int nargs);
 
-typedef enum {
-    ERTS_SCHED_NORMAL,
-    ERTS_SCHED_DIRTY_CPU,
-    ERTS_SCHED_DIRTY_IO
-} ErtsSchedType;
-
 #ifdef ERTS_DIRTY_SCHEDULERS
 int erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p,
 			BeamInstr *I, Eterm *reg);
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 3102e44c11..4836b9e2d3 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -936,7 +936,7 @@ enqueue_port(ErtsRunQueue *runq, Port *pp)
     erts_smp_inc_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL);
 
 #ifdef ERTS_SMP
-    if (runq->halt_in_progress)
+    if (ERTS_RUNQ_FLGS_GET_NOB(runq) & ERTS_RUNQ_FLG_HALTING)
 	erts_non_empty_runq(runq);
 #endif
 }
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index da4b468d66..084ecb14ec 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -107,9 +107,31 @@
 #define LOW_BIT       (1 << PRIORITY_LOW)
 #define PORT_BIT      (1 << ERTS_PORT_PRIO_LEVEL)
 
-#define ERTS_EMPTY_RUNQ(RQ)					\
-    ((ERTS_RUNQ_FLGS_GET_NOB((RQ)) & ERTS_RUNQ_FLGS_QMASK) == 0	\
-     && (RQ)->misc.start == NULL)
+#define ERTS_IS_RUNQ_EMPTY_FLGS(FLGS) \
+    (!((FLGS) & (ERTS_RUNQ_FLGS_QMASK|ERTS_RUNQ_FLG_MISC_OP)))
+
+#define ERTS_IS_RUNQ_EMPTY_PORTS_FLGS(FLGS) \
+    (!((FLGS) & (PORT_BIT|ERTS_RUNQ_FLG_MISC_OP)))
+
+#define ERTS_EMPTY_RUNQ(RQ) \
+    ERTS_IS_RUNQ_EMPTY_FLGS(ERTS_RUNQ_FLGS_GET_NOB((RQ)))
+
+#define ERTS_EMPTY_RUNQ_PORTS(RQ) \
+    ERTS_IS_RUNQ_EMPTY_FLGS(ERTS_RUNQ_FLGS_GET_NOB((RQ)))
+
+static ERTS_INLINE int
+runq_got_work_to_execute_flags(Uint32 flags)
+{
+    if (flags & ERTS_RUNQ_FLG_HALTING)
+        return !ERTS_IS_RUNQ_EMPTY_PORTS_FLGS(flags);
+    return !ERTS_IS_RUNQ_EMPTY_FLGS(flags);
+}
+
+static ERTS_INLINE int
+runq_got_work_to_execute(ErtsRunQueue *rq)
+{
+    return runq_got_work_to_execute_flags(ERTS_RUNQ_FLGS_GET_NOB(rq));
+}
 
 #undef RUNQ_READ_RQ
 #undef RUNQ_SET_RQ
@@ -141,9 +163,6 @@ do { \
 #  define ERTS_DBG_VERIFY_VALID_RUNQP(RQP)
 #endif
 
-#define ERTS_EMPTY_RUNQ_PORTS(RQ) \
-    (RUNQ_READ_LEN(&(RQ)->ports.info.len) == 0 && (RQ)->misc.start == NULL)
-
 const Process erts_invalid_process = {{ERTS_INVALID_PID}};
 
 extern BeamInstr beam_apply[];
@@ -200,158 +219,139 @@ typedef struct {
     ErtsProcList *chngq;
 } ErtsMultiSchedulingBlock;
 
+typedef struct {
+    Uint32 normal;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    Uint32 dirty_cpu;
+    Uint32 dirty_io;
+#endif
+} ErtsSchedTypeCounters;
+
 static struct {
     erts_smp_mtx_t mtx;
-    Uint32 online;
-    Uint32 curr_online;
-    Uint32 active;
+    ErtsSchedTypeCounters online;
+    ErtsSchedTypeCounters curr_online;
+    ErtsSchedTypeCounters active;
     erts_smp_atomic32_t changing;
     ErtsProcList *chngq;
     Eterm changer;
     ErtsMultiSchedulingBlock nmsb; /* Normal multi Scheduling Block */
     ErtsMultiSchedulingBlock msb; /* Multi Scheduling Block */
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ErtsSchedType last_msb_dirty_type;
+#endif
 } schdlr_sspnd;
 
-#define ERTS_SCHDLR_SSPND_S_BITS	10
-#define ERTS_SCHDLR_SSPND_DCS_BITS	11
-#define ERTS_SCHDLR_SSPND_DIS_BITS	11
-
-#define ERTS_SCHDLR_SSPND_S_MASK	((1 << ERTS_SCHDLR_SSPND_S_BITS)-1)
-#define ERTS_SCHDLR_SSPND_DCS_MASK	((1 << ERTS_SCHDLR_SSPND_DCS_BITS)-1)
-#define ERTS_SCHDLR_SSPND_DIS_MASK	((1 << ERTS_SCHDLR_SSPND_DIS_BITS)-1)
-
-#define ERTS_SCHDLR_SSPND_S_SHIFT	0
-#define ERTS_SCHDLR_SSPND_DCS_SHIFT	(ERTS_SCHDLR_SSPND_S_SHIFT \
-					 + ERTS_SCHDLR_SSPND_S_BITS)
-#define ERTS_SCHDLR_SSPND_DIS_SHIFT	(ERTS_SCHDLR_SSPND_DCS_SHIFT \
-					 + ERTS_SCHDLR_SSPND_DCS_BITS)
-
-#if (ERTS_SCHDLR_SSPND_S_BITS		\
-     + ERTS_SCHDLR_SSPND_DCS_BITS	\
-     + ERTS_SCHDLR_SSPND_DIS_BITS) > 32
-#  error Wont fit in Uint32
-#endif
+static void init_scheduler_suspend(void);
 
-#if (ERTS_MAX_NO_OF_SCHEDULERS-1) > ERTS_SCHDLR_SSPND_S_MASK
-#  error Max no schedulers wont fit in its bit-field
-#endif
-#if ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS > ERTS_SCHDLR_SSPND_DCS_MASK
-#  error Max no dirty cpu schedulers wont fit in its bit-field
-#endif
-#if ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS > ERTS_SCHDLR_SSPND_DIS_MASK
-#  error Max no dirty io schedulers wont fit in its bit-field
+static ERTS_INLINE Uint32
+schdlr_sspnd_eq_nscheds(ErtsSchedTypeCounters *val1p, ErtsSchedTypeCounters *val2p)
+{
+    int res = val1p->normal == val2p->normal;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    res &= val1p->dirty_cpu == val2p->dirty_cpu;
+    res &= val1p->dirty_io == val2p->dirty_io;
 #endif
-
-#define ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(S, DCS, DIS)			\
-    ((((Uint32) (((S) & ERTS_SCHDLR_SSPND_S_MASK))-1)			\
-      << ERTS_SCHDLR_SSPND_S_SHIFT)					\
-     | ((((Uint32) ((DCS) & ERTS_SCHDLR_SSPND_DCS_MASK))		\
-	 << ERTS_SCHDLR_SSPND_DCS_SHIFT))				\
-     | ((((Uint32) ((DIS) & ERTS_SCHDLR_SSPND_DIS_MASK))		\
-	 << ERTS_SCHDLR_SSPND_DIS_SHIFT)))
-
-static void init_scheduler_suspend(void);
+    return res;
+}
 
 static ERTS_INLINE Uint32
-schdlr_sspnd_get_nscheds(Uint32 *valp, ErtsSchedType type)
+schdlr_sspnd_get_nscheds(ErtsSchedTypeCounters *valp,
+                         ErtsSchedType type)
 {
-    Uint32 res = (Uint32) (*valp);
     switch (type) {
     case ERTS_SCHED_NORMAL:
-	res >>= ERTS_SCHDLR_SSPND_S_SHIFT;
-	res &= (Uint32) ERTS_SCHDLR_SSPND_S_MASK;
-	res++;
-	break;
+        return valp->normal;
+#ifdef ERTS_DIRTY_SCHEDULERS
     case ERTS_SCHED_DIRTY_CPU:
-	res >>= ERTS_SCHDLR_SSPND_DCS_SHIFT;
-	res &= (Uint32) ERTS_SCHDLR_SSPND_DCS_MASK;
-	break;
+        return valp->dirty_cpu;
     case ERTS_SCHED_DIRTY_IO:
-	res >>= ERTS_SCHDLR_SSPND_DIS_SHIFT;
-	res &= (Uint32) ERTS_SCHDLR_SSPND_DIS_MASK;
-	break;
+        return valp->dirty_io;
+#else
+    case ERTS_SCHED_DIRTY_CPU:
+    case ERTS_SCHED_DIRTY_IO:
+        return 0;
+#endif
     default:
 	ERTS_INTERNAL_ERROR("Invalid scheduler type");
 	return 0;
     }
+}
 
+static ERTS_INLINE Uint32
+schdlr_sspnd_get_nscheds_tot(ErtsSchedTypeCounters *valp)
+{
+    Uint32 res = valp->normal;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    res += valp->dirty_cpu;
+    res += valp->dirty_io;
+#endif
     return res;
 }
 
 static ERTS_INLINE void
-schdlr_sspnd_dec_nscheds(Uint32 *valp, ErtsSchedType type)
+schdlr_sspnd_dec_nscheds(ErtsSchedTypeCounters *valp,
+                         ErtsSchedType type)
 {
     ASSERT(schdlr_sspnd_get_nscheds(valp, type) > 0);
 
     switch (type) {
     case ERTS_SCHED_NORMAL:
-	*valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT;
+        valp->normal--;
 	break;
+#ifdef ERTS_DIRTY_SCHEDULERS
     case ERTS_SCHED_DIRTY_CPU:
-	*valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT;
+        valp->dirty_cpu--;
 	break;
     case ERTS_SCHED_DIRTY_IO:
-	*valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT;
+        valp->dirty_io--;
 	break;
+#endif
     default:
 	ERTS_INTERNAL_ERROR("Invalid scheduler type");
     }
 }
 
 static ERTS_INLINE void
-schdlr_sspnd_inc_nscheds(Uint32 *valp, ErtsSchedType type)
+schdlr_sspnd_inc_nscheds(ErtsSchedTypeCounters *valp,
+                         ErtsSchedType type)
 {
     switch (type) {
     case ERTS_SCHED_NORMAL:
-	ASSERT(schdlr_sspnd_get_nscheds(valp, type)
-	       < ERTS_MAX_NO_OF_SCHEDULERS-1);
-	*valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT;
+        valp->normal++;
 	break;
+#ifdef ERTS_DIRTY_SCHEDULERS
     case ERTS_SCHED_DIRTY_CPU:
-	ASSERT(schdlr_sspnd_get_nscheds(valp, type)
-	       < ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS);
-	*valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT;
+        valp->dirty_cpu++;
 	break;
     case ERTS_SCHED_DIRTY_IO:
-	ASSERT(schdlr_sspnd_get_nscheds(valp, type)
-	       < ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS);
-	*valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT;
+        valp->dirty_io++;
 	break;
+#endif
     default:
 	ERTS_INTERNAL_ERROR("Invalid scheduler type");
     }
 }
 
 static ERTS_INLINE void
-schdlr_sspnd_set_nscheds(Uint32 *valp, ErtsSchedType type, Uint32 no)
+schdlr_sspnd_set_nscheds(ErtsSchedTypeCounters *valp,
+                         ErtsSchedType type, Uint32 no)
 {
-    Uint32 val = *valp;
-
     switch (type) {
     case ERTS_SCHED_NORMAL:
-	ASSERT(no > 0);
-	val &= ~(((Uint32) ERTS_SCHDLR_SSPND_S_MASK)
-		 << ERTS_SCHDLR_SSPND_S_SHIFT);
-	val |= (((no-1) & ((Uint32) ERTS_SCHDLR_SSPND_S_MASK))
-		<< ERTS_SCHDLR_SSPND_S_SHIFT);
+        valp->normal = no;
 	break;
+#ifdef ERTS_DIRTY_SCHEDULERS
     case ERTS_SCHED_DIRTY_CPU:
-	val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK)
-		 << ERTS_SCHDLR_SSPND_DCS_SHIFT);
-	val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK))
-		 << ERTS_SCHDLR_SSPND_DCS_SHIFT);
+        valp->dirty_cpu = no;
 	break;
     case ERTS_SCHED_DIRTY_IO:
-	val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK)
-		 << ERTS_SCHDLR_SSPND_DIS_SHIFT);
-	val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK))
-		<< ERTS_SCHDLR_SSPND_DIS_SHIFT);
+        valp->dirty_io = no;
 	break;
+#endif
     default:
 	ERTS_INTERNAL_ERROR("Invalid scheduler type");
     }
-
-    *valp = val;
 }
 
 static struct {
@@ -2293,7 +2293,8 @@ static ERTS_INLINE erts_aint32_t
 handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
 {
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_REAP_PORTS);
-    awdp->esdp->run_queue->halt_in_progress = 1;
+    ERTS_RUNQ_FLGS_SET(awdp->esdp->run_queue, ERTS_RUNQ_FLG_HALTING);
+
     if (erts_smp_atomic32_dec_read_acqb(&erts_halt_progress) == 0) {
 	int i, max = erts_ptab_max(&erts_port);
 	erts_smp_atomic32_set_nob(&erts_halt_progress, 1);
@@ -2833,11 +2834,12 @@ static erts_aint32_t
 sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi)
 {
     erts_aint32_t oflgs;
-    erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING
-			   | ERTS_SSI_FLG_WAITING);
+    erts_aint32_t nflgs;
     erts_aint32_t xflgs = 0;
 
     do {
+        nflgs = (xflgs & ERTS_SSI_FLG_MSB_EXEC);
+        nflgs |= ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING;
 	oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
 	if (oflgs == xflgs)
 	    return nflgs;
@@ -2859,7 +2861,7 @@ sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi)
 	if (oflgs == xflgs)
 	    return nflgs;
 	xflgs = oflgs;
-	nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED;
+	nflgs |= oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC);
     } while (oflgs & ERTS_SSI_FLG_WAITING);
     return oflgs;
 }
@@ -2909,7 +2911,7 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type)
 	    return oflgs;
 	}
 	xflgs = oflgs;
-	nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED;
+	nflgs |= oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC);
     }
 }
 
@@ -3044,6 +3046,8 @@ aux_thread(void *unused)
     return NULL;
 }
 
+static void suspend_scheduler(ErtsSchedulerData *esdp);
+
 #endif /* ERTS_SMP */
 
 static void
@@ -3209,8 +3213,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	}
 
-	if (flgs & ~ERTS_SSI_FLG_SUSPENDED)
-	    erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+	if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC))
+	    erts_smp_atomic32_read_band_nob(&ssi->flags,
+                                            (ERTS_SSI_FLG_SUSPENDED
+                                             | ERTS_SSI_FLG_MSB_EXEC));
 
 	if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
 	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
@@ -3409,8 +3415,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    erts_smp_runq_lock(rq);
 	}
 	clear_sys_scheduling();
-	if (flgs & ~ERTS_SSI_FLG_SUSPENDED)
-	    erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+	if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC))
+	    erts_smp_atomic32_read_band_nob(&ssi->flags,
+                                            (ERTS_SSI_FLG_SUSPENDED
+                                             | ERTS_SSI_FLG_MSB_EXEC));
 #endif
 	if (!working)
 	    sched_wall_time_change(esdp, working = 1);
@@ -3433,17 +3441,54 @@ ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi)
 	oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, nflgs, xflgs);
 	if (oflgs == xflgs)
 	    return oflgs;
-	nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED;
+	nflgs = oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC);
 	xflgs = oflgs;
     }
 }
 
+static ERTS_INLINE void
+ssi_wake(ErtsSchedulerSleepInfo *ssi)
+{
+    erts_sched_finish_poke(ssi, ssi_flags_set_wake(ssi));
+}
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+
 static void
-wake_scheduler(ErtsRunQueue *rq)
+dcpu_sched_ix_suspend_wake(Uint ix)
 {
-    ErtsSchedulerSleepInfo *ssi;
-    erts_aint32_t flgs;
+    ErtsSchedulerSleepInfo* ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+    erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+    ssi_wake(ssi);
+}
+
+static void
+dio_sched_ix_suspend_wake(Uint ix)
+{
+    ErtsSchedulerSleepInfo* ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
+    erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+    ssi_wake(ssi);
+}
+
+static void
+dcpu_sched_ix_wake(Uint ix)
+{
+    ssi_wake(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix));
+}
+
+#if 0
+static void
+dio_sched_ix_wake(Uint ix)
+{
+    ssi_wake(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix));
+}
+#endif
 
+#endif
+
+static void
+wake_scheduler(ErtsRunQueue *rq)
+{
     /*
      * The unlocked run queue is not strictly necessary
      * from a thread safety or deadlock prevention
@@ -3455,10 +3500,7 @@ wake_scheduler(ErtsRunQueue *rq)
     ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq)
 		       || ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
 
-    ssi = rq->scheduler->ssi;
-
-    flgs = ssi_flags_set_wake(ssi);
-    erts_sched_finish_poke(ssi, flgs);
+    ssi_wake(rq->scheduler->ssi);
 }
 
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -3505,6 +3547,13 @@ wake_dirty_schedulers(ErtsRunQueue *rq, int one)
 	} while (ssi);
     }
 }
+
+static void
+wake_dirty_scheduler(ErtsRunQueue *rq)
+{
+    wake_dirty_schedulers(rq, 1);
+}
+
 #endif
 
 #define ERTS_NO_USED_RUNQS_SHIFT 16
@@ -3645,7 +3694,7 @@ smp_notify_inc_runq(ErtsRunQueue *runq)
     if (runq) {
 #ifdef ERTS_DIRTY_SCHEDULERS
 	if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix))
-	    wake_dirty_schedulers(runq, 1);
+	    wake_dirty_scheduler(runq);
 	else
 #endif
 	    wake_scheduler(runq);
@@ -3949,8 +3998,7 @@ suspend_run_queue(ErtsRunQueue *rq)
     wake_scheduler(rq);
 }
 
-static void scheduler_ix_resume_wake(Uint ix);
-static void scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi);
+static void nrml_sched_ix_resume_wake(Uint ix);
 
 static ERTS_INLINE void
 resume_run_queue(ErtsRunQueue *rq)
@@ -3958,16 +4006,19 @@ resume_run_queue(ErtsRunQueue *rq)
     int pix;
     Uint32 oflgs;
 
+    ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
+
     erts_smp_runq_lock(rq);
 
     oflgs = ERTS_RUNQ_FLGS_READ_BSET(rq,
 				     (ERTS_RUNQ_FLG_OUT_OF_WORK
 				      | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK
-				      | ERTS_RUNQ_FLG_SUSPENDED),
+				      | ERTS_RUNQ_FLG_SUSPENDED
+                                      | ERTS_RUNQ_FLG_MSB_EXEC),
 				     (ERTS_RUNQ_FLG_OUT_OF_WORK
 				      | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK));
 
-    if (oflgs & ERTS_RUNQ_FLG_SUSPENDED) {
+    if (oflgs & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_MSB_EXEC)) {
 	erts_aint32_t len;
 
 	rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
@@ -3986,10 +4037,7 @@ resume_run_queue(ErtsRunQueue *rq)
 
     erts_smp_runq_unlock(rq);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-    if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
-#endif
-	scheduler_ix_resume_wake(rq->ix);
+    nrml_sched_ix_resume_wake(rq->ix);
 }
 
 typedef struct {
@@ -4047,28 +4095,22 @@ evacuate_run_queue(ErtsRunQueue *rq,
     int prio_q;
     ErtsRunQueue *to_rq;
     ErtsMigrationPaths *mps;
-    ErtsMigrationPath *mp = NULL;
+    ErtsMigrationPath *mp;
 
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
     (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-    if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
-#endif
-    {
-	mps = erts_get_migration_paths_managed();
-	mp = &mps->mpath[rq->ix];
-    }
+    ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
+
+    mps = erts_get_migration_paths_managed();
+    mp = &mps->mpath[rq->ix];
 
     /* Evacuate scheduled misc ops */
 
     if (rq->misc.start) {
 	ErtsMiscOpList *start, *end;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-	ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
-#endif
 	to_rq = mp->misc_evac_runq;
 	if (!to_rq)
 	    return;
@@ -4077,6 +4119,7 @@ evacuate_run_queue(ErtsRunQueue *rq,
 	end = rq->misc.end;
 	rq->misc.start = NULL;
 	rq->misc.end = NULL;
+        ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP);
 	erts_smp_runq_unlock(rq);
 
 	erts_smp_runq_lock(to_rq);
@@ -4097,9 +4140,6 @@ evacuate_run_queue(ErtsRunQueue *rq,
     if (rq->ports.start) {
 	Port *prt;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-	ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
-#endif
 	to_rq = mp->prio[ERTS_PORT_PRIO_LEVEL].runq;
 	if (!to_rq)
 	    return;
@@ -4137,15 +4177,10 @@ evacuate_run_queue(ErtsRunQueue *rq,
 	int notify = 0;
 	to_rq = NULL;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-	if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
-#endif
-	{
-	    if (!mp->prio[prio_q].runq)
-		return;
-	    if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq)
-		return;
-	}
+        if (!mp->prio[prio_q].runq)
+            return;
+        if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq)
+            return;
 
 	proc = dequeue_process(rq, prio_q, &state);
 	while (proc) {
@@ -4201,11 +4236,6 @@ evacuate_run_queue(ErtsRunQueue *rq,
 		goto handle_next_proc;
 	    }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-	    if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
-		clear_proc_dirty_queue_bit(real_proc, rq, qbit);
-#endif
-
 	    if (ERTS_PSFLG_BOUND & real_state) {
 		/* Bound processes get stuck here... */
 		proc->next = NULL;
@@ -4219,16 +4249,7 @@ evacuate_run_queue(ErtsRunQueue *rq,
 		int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state);
 		erts_smp_runq_unlock(rq);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-		if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
-		    /*
-		     * dirty run queues evacuate only to run
-		     * queue 0 during multi-scheduling blocking
-		     */
-		    to_rq = ERTS_RUNQ_IX(0);
-		else
-#endif
-		    to_rq = mp->prio[prio].runq;
+                to_rq = mp->prio[prio].runq;
 		RUNQ_SET_RQ(&proc->run_queue, to_rq);
 
 		erts_smp_runq_lock(to_rq);
@@ -4263,7 +4284,7 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq,
 
     erts_smp_runq_lock(vrq);
 
-    if (rq->halt_in_progress)
+    if (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_HALTING)
 	goto no_procs;
 
     /*
@@ -4362,8 +4383,7 @@ check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix)
 {
     ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix);
     Uint32 flags = ERTS_RUNQ_FLGS_GET(vrq);
-    if ((flags & (ERTS_RUNQ_FLG_NONEMPTY
-		  | ERTS_RUNQ_FLG_PROTECTED)) == ERTS_RUNQ_FLG_NONEMPTY)
+    if (runq_got_work_to_execute_flags(flags) & (!(flags & ERTS_RUNQ_FLG_PROTECTED)))
 	return try_steal_task_from_victim(rq, rq_lockedp, vrq, flags);
     else
 	return 0;
@@ -4431,11 +4451,9 @@ try_steal_task(ErtsRunQueue *rq)
     if (!rq_locked)
 	erts_smp_runq_lock(rq);
 
-    if (!res)
-	res = rq->halt_in_progress ?
-	    !ERTS_EMPTY_RUNQ_PORTS(rq) : !ERTS_EMPTY_RUNQ(rq);
-
-    return res;
+    if (res)
+        return res;
+    return runq_got_work_to_execute(rq);
 }
 
 /* Run queue balancing */
@@ -5347,7 +5365,7 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags)
 #ifdef ERTS_DIRTY_SCHEDULERS
 		if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) {
 		    if (rq->waiting) {
-			wake_dirty_schedulers(rq, 1);
+			wake_dirty_scheduler(rq);
 		    }
 		} else
 #endif
@@ -5710,13 +5728,29 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER,
 					   MAX_REG * sizeof(FloatDef));
 #ifdef ERTS_DIRTY_SCHEDULERS
+    esdp->run_queue = runq;
     if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) {
 	esdp->no = 0;
+        if (runq == ERTS_DIRTY_CPU_RUNQ)
+            esdp->type = ERTS_SCHED_DIRTY_CPU;
+        else {
+            ASSERT(runq == ERTS_DIRTY_IO_RUNQ);
+            esdp->type = ERTS_SCHED_DIRTY_IO;
+        }
 	ERTS_DIRTY_SCHEDULER_NO(esdp) = (Uint) num;
+        if (num == 1) {
+            /*
+             * Multi-scheduling block functionality depends
+             * on finding dirty scheduler number 1 here...
+             */
+            runq->scheduler = esdp;
+        }
     }
     else {
+        esdp->type = ERTS_SCHED_NORMAL;
 	esdp->no = (Uint) num;
 	ERTS_DIRTY_SCHEDULER_NO(esdp) = 0;
+        runq->scheduler = esdp;
     }
     esdp->dirty_shadow_process = shadow_proc;
     if (shadow_proc) {
@@ -5728,6 +5762,8 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 	shadow_proc->static_flags = ERTS_STC_FLG_SHADOW_PROC;
     }
 #else
+    runq->scheduler = esdp;
+    esdp->run_queue = runq;
     esdp->no = (Uint) num;
 #endif
 
@@ -5740,9 +5776,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 
     erts_init_atom_cache_map(&esdp->atom_cache_map);
 
-    esdp->run_queue = runq;
-    esdp->run_queue->scheduler = esdp;
-
     esdp->last_monotonic_time = 0;
     esdp->check_time_reds = 0;
 
@@ -5858,7 +5891,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	erts_smp_atomic32_set_nob(&rq->len, 0);
 	rq->wakeup_other = 0;
 	rq->wakeup_other_reds = 0;
-	rq->halt_in_progress = 0;
 
 	rq->procs.pending_exiters = NULL;
 	rq->procs.context_switches = 0;
@@ -6974,15 +7006,8 @@ resume_process(Process *p, ErtsProcLocks locks)
 
 #ifdef ERTS_SMP
 
-static void
-scheduler_ix_resume_wake(Uint ix)
-{
-    ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix);
-    scheduler_ssi_resume_wake(ssi);
-}
-
-static void
-scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi)
+static ERTS_INLINE void
+sched_resume_wake__(ErtsSchedulerSleepInfo *ssi)
 {
     erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING
 			   | ERTS_SSI_FLG_TSE_SLEEPING
@@ -6996,9 +7021,31 @@ scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi)
 	    break;
 	}
 	xflgs = oflgs;
-    } while (oflgs & ERTS_SSI_FLG_SUSPENDED);
+    } while (oflgs & (ERTS_SSI_FLG_MSB_EXEC|ERTS_SSI_FLG_SUSPENDED));
+}
+
+static void
+nrml_sched_ix_resume_wake(Uint ix)
+{
+    sched_resume_wake__(ERTS_SCHED_SLEEP_INFO_IX(ix));
 }
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+static void
+dcpu_sched_ix_resume_wake(Uint ix)
+{
+    sched_resume_wake__(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix));
+}
+
+static void
+dio_sched_ix_resume_wake(Uint ix)
+{
+    sched_resume_wake__(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix));
+}
+
+#endif
+
 static erts_aint32_t
 sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct)
 {
@@ -7078,9 +7125,18 @@ static void
 init_scheduler_suspend(void)
 {
     erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd");
-    schdlr_sspnd.online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0);
-    schdlr_sspnd.curr_online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0);
-    schdlr_sspnd.active = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0);
+    schdlr_sspnd.online.normal = 1;
+    schdlr_sspnd.curr_online.normal = 1;
+    schdlr_sspnd.active.normal = 1;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    schdlr_sspnd.online.dirty_cpu = 0;
+    schdlr_sspnd.curr_online.dirty_cpu = 0;
+    schdlr_sspnd.active.dirty_cpu = 0;
+    schdlr_sspnd.online.dirty_io = 0;
+    schdlr_sspnd.curr_online.dirty_io = 0;
+    schdlr_sspnd.active.dirty_io = 0;
+    schdlr_sspnd.last_msb_dirty_type = ERTS_SCHED_DIRTY_IO;
+#endif
     erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0);
     schdlr_sspnd.chngq = NULL;
     schdlr_sspnd.changer = am_false;
@@ -7141,6 +7197,252 @@ schdlr_sspnd_resume_procs(ErtsSchedType sched_type,
     }
 }
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+static ERTS_INLINE int
+have_dirty_work(void)
+{
+    return !(ERTS_EMPTY_RUNQ(ERTS_DIRTY_CPU_RUNQ)
+             | ERTS_EMPTY_RUNQ(ERTS_DIRTY_IO_RUNQ));
+}
+
+#define ERTS_MSB_NONE_PRIO_BIT PORT_BIT
+
+static ERTS_INLINE Uint32
+msb_runq_prio_bit(Uint32 flgs)
+{
+    int pbit;
+
+    pbit = (int) (flgs & ERTS_RUNQ_FLGS_PROCS_QMASK);
+    if (flgs & PORT_BIT) {
+        /* rate ports as proc prio high */
+        pbit |= HIGH_BIT;
+    }
+    if (flgs & ERTS_RUNQ_FLG_MISC_OP) {
+        /* rate misc ops as proc prio normal */
+        pbit |= NORMAL_BIT;
+    }
+    if (flgs & LOW_BIT) {
+        /* rate low prio as normal (avoid starvation) */
+        pbit |= NORMAL_BIT;
+    }
+    if (!pbit)
+        pbit = (int) ERTS_MSB_NONE_PRIO_BIT;
+    else
+        pbit &= -pbit; /* least significant bit set... */
+    ASSERT(pbit);
+
+    /* High prio low value; low prio high value... */
+    return (Uint32) pbit;
+}
+
+static ERTS_INLINE void
+msb_runq_prio_bits(Uint32 *nrmlp, Uint32 *dcpup, Uint32 *diop)
+{
+    Uint32 flgs = ERTS_RUNQ_FLGS_GET(ERTS_RUNQ_IX(0));
+    if (flgs & ERTS_RUNQ_FLG_HALTING) {
+        /*
+         * Emulator is halting; only execute port jobs
+         * on normal scheduler. Ensure that we switch
+         * to the normal scheduler.
+         */
+        *nrmlp = HIGH_BIT;
+        *dcpup = ERTS_MSB_NONE_PRIO_BIT;
+        *diop = ERTS_MSB_NONE_PRIO_BIT;
+    }
+    else {
+        *nrmlp = msb_runq_prio_bit(flgs);
+
+        flgs = ERTS_RUNQ_FLGS_GET(ERTS_DIRTY_CPU_RUNQ);
+        *dcpup = msb_runq_prio_bit(flgs);
+
+        flgs = ERTS_RUNQ_FLGS_GET(ERTS_DIRTY_IO_RUNQ);
+        *diop = msb_runq_prio_bit(flgs);
+    }
+}
+
+static int
+msb_scheduler_type_switch(ErtsSchedType sched_type,
+                          ErtsSchedulerData *esdp,
+                          long no)
+{
+    Uint32 nrml_prio, dcpu_prio, dio_prio;
+    ErtsSchedType exec_type;
+    ErtsRunQueue *exec_rq;
+#ifdef DEBUG
+    erts_aint32_t dbg_val;
+#endif
+
+    ASSERT(schdlr_sspnd.msb.ongoing);
+
+    /*
+     * This function determines how to switch
+     * between scheduler types when multi-scheduling
+     * is blocked.
+     *
+     * If no dirty work exist, we always select
+     * execution of normal scheduler. If nothing
+     * executes, normal scheduler 1 should be waiting
+     * in sys_schedule(), otherwise we cannot react
+     * on I/O events.
+     *
+     * We unconditionally switch back to normal
+     * scheduler after executing dirty in order to
+     * make sure we check for I/O...
+     */
+
+    msb_runq_prio_bits(&nrml_prio, &dcpu_prio, &dio_prio);
+
+    exec_type = ERTS_SCHED_NORMAL;
+    if (sched_type == ERTS_SCHED_NORMAL) {
+
+        /*
+         * Check priorities of work in the
+         * different run-queues and determine
+         * run-queue with highest prio job...
+         */
+
+        if ((dcpu_prio == ERTS_MSB_NONE_PRIO_BIT)
+            & (dio_prio == ERTS_MSB_NONE_PRIO_BIT)) {
+            /*
+             * No dirty work exist; continue on normal
+             * scheduler...
+             */
+            return 0;
+        }
+
+        if (dcpu_prio < nrml_prio) {
+            exec_type = ERTS_SCHED_DIRTY_CPU;
+            if (dio_prio < dcpu_prio)
+                exec_type = ERTS_SCHED_DIRTY_IO;
+        }
+        else {
+            if (dio_prio < nrml_prio)
+                exec_type = ERTS_SCHED_DIRTY_IO;
+        }
+
+        /*
+         * Make sure to alternate between dirty types
+         * inbetween normal execution if highest 
+         * priorities are equal.
+         */
+
+        if (exec_type == ERTS_SCHED_NORMAL) {
+            if (dcpu_prio == nrml_prio)
+                exec_type = ERTS_SCHED_DIRTY_CPU;
+            else if (dio_prio == nrml_prio)
+                exec_type = ERTS_SCHED_DIRTY_IO;
+            else {
+                /*
+                 * Normal work has higher prio than
+                 * dirty work; continue on normal
+                 * scheduler...
+                 */
+                return 0;
+            }
+        }
+
+        ASSERT(exec_type != ERTS_SCHED_NORMAL);
+        if (dio_prio == dcpu_prio) {
+            /* Alter between dirty types... */
+            if (schdlr_sspnd.last_msb_dirty_type == ERTS_SCHED_DIRTY_IO)
+                exec_type = ERTS_SCHED_DIRTY_CPU;
+            else
+                exec_type = ERTS_SCHED_DIRTY_IO;
+        }
+    }
+
+    ASSERT(sched_type != exec_type);
+
+    if (exec_type != ERTS_SCHED_NORMAL)
+        schdlr_sspnd.last_msb_dirty_type = exec_type;
+    else {
+        erts_aint32_t calls;
+        /*
+         * Going back to normal scheduler after
+         * dirty execution; make sure it will check
+         * for I/O...
+         */
+        if (ERTS_USE_MODIFIED_TIMING())
+            calls = ERTS_MODIFIED_TIMING_INPUT_REDS + 1;
+        else
+            calls = INPUT_REDUCTIONS + 1;
+        erts_smp_atomic32_set_nob(&function_calls, calls);
+
+        if ((nrml_prio == ERTS_MSB_NONE_PRIO_BIT)
+            & ((dcpu_prio != ERTS_MSB_NONE_PRIO_BIT)
+               | (dio_prio != ERTS_MSB_NONE_PRIO_BIT))) {
+            /*
+             * We have dirty work, but an empty
+             * normal run-queue.
+             *
+             * Since the normal run-queue is
+             * empty, the normal scheduler will
+             * go to sleep when selected for
+             * execution. We have dirty work to
+             * do, so we only want it to check
+             * I/O, and then come back here and
+             * switch to dirty execution.
+             *
+             * To prevent the scheduler from going
+             * to sleep we trick it into believing
+             * it has work to do...
+             */
+            ERTS_RUNQ_FLGS_SET_NOB(ERTS_RUNQ_IX(0),
+                                   ERTS_RUNQ_FLG_MISC_OP);
+        }
+    }
+
+    /*
+     * Suspend this scheduler and wake up scheduler
+     * number one of another type...
+     */
+#ifdef DEBUG
+    dbg_val =
+#else
+    (void)
+#endif
+        erts_smp_atomic32_read_bset_mb(&esdp->ssi->flags,
+                                       (ERTS_SSI_FLG_SUSPENDED
+                                        | ERTS_SSI_FLG_MSB_EXEC),
+                                       ERTS_SSI_FLG_SUSPENDED);
+    ASSERT(dbg_val & ERTS_SSI_FLG_MSB_EXEC);
+
+    switch (exec_type) {
+    case ERTS_SCHED_NORMAL:
+        exec_rq = ERTS_RUNQ_IX(0);
+        break;
+    case ERTS_SCHED_DIRTY_CPU:
+        exec_rq = ERTS_DIRTY_CPU_RUNQ;
+        break;
+    case ERTS_SCHED_DIRTY_IO:
+        exec_rq = ERTS_DIRTY_IO_RUNQ;
+        break;
+    default:
+        ERTS_INTERNAL_ERROR("Invalid scheduler type");
+        exec_rq = NULL;
+        break;
+    }
+
+#ifdef DEBUG
+    dbg_val =
+#else
+    (void)
+#endif
+        erts_smp_atomic32_read_bset_mb(&exec_rq->scheduler->ssi->flags,
+                                       (ERTS_SSI_FLG_SUSPENDED
+                                        | ERTS_SSI_FLG_MSB_EXEC),
+                                       ERTS_SSI_FLG_MSB_EXEC);
+    ASSERT(dbg_val & ERTS_SSI_FLG_SUSPENDED);
+
+    wake_scheduler(exec_rq);
+
+    return 1; /* suspend this scheduler... */
+
+}
+
+#endif
+
 static void
 suspend_scheduler(ErtsSchedulerData *esdp)
 {
@@ -7166,32 +7468,49 @@ suspend_scheduler(ErtsSchedulerData *esdp)
      * Regardless of why a scheduler is suspended, it ends up here.
      */
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-    if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+
+#if !defined(ERTS_DIRTY_SCHEDULERS)
+
+    sched_type = ERTS_SCHED_NORMAL;
+    online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN;
+    no = esdp->no;
+    ASSERT(no != 1);
+
+#else
+
+    sched_type = esdp->type;
+    switch (sched_type) {
+    case ERTS_SCHED_NORMAL:
+        online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN;
+        no = esdp->no;
+        break;
+    case ERTS_SCHED_DIRTY_CPU:
+        online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN;
+        no = ERTS_DIRTY_SCHEDULER_NO(esdp);
+        break;
+    case ERTS_SCHED_DIRTY_IO:
+        online_flag = 0;
 	no = ERTS_DIRTY_SCHEDULER_NO(esdp);
-	if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) {
-	    online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN;
-	    sched_type = ERTS_SCHED_DIRTY_CPU;
-	}
-	else {
-	    online_flag = 0;
-	    sched_type = ERTS_SCHED_DIRTY_IO;
-	}
+        break;
+    default:
+        ERTS_INTERNAL_ERROR("Invalid scheduler type");
+        return;
     }
-    else
-#endif
-    {
-	online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN;
-	no = esdp->no;
-	sched_type = ERTS_SCHED_NORMAL;
+
+    if (erts_smp_atomic32_read_nob(&ssi->flags) & ERTS_SSI_FLG_MSB_EXEC) {
+        ASSERT(no == 1);
+        if (!msb_scheduler_type_switch(sched_type, esdp, no))
+            return;
+        /* Suspend and let scheduler 1 of another type execute... */
     }
 
-    ASSERT(sched_type != ERTS_SCHED_NORMAL || no != 1);
+#endif
 
     if (sched_type != ERTS_SCHED_NORMAL)
 	erts_smp_runq_unlock(esdp->run_queue);
     else {
-	evacuate_run_queue(esdp->run_queue, &sbp);
+        if (no != 1)
+            evacuate_run_queue(esdp->run_queue, &sbp);
 
 	erts_smp_runq_unlock(esdp->run_queue);
 
@@ -7200,8 +7519,6 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 	if (erts_system_profile_flags.scheduler)
 	    profile_scheduler(make_small(esdp->no), am_inactive);
 
-	sched_wall_time_change(esdp, 0);
-
     }
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 
@@ -7210,9 +7527,6 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 	schdlr_sspnd_dec_nscheds(&schdlr_sspnd.active, sched_type);
 
-	ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
-					ERTS_SCHED_NORMAL) >= 1);
-
 	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
 
 	while (1) {
@@ -7234,9 +7548,13 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 						    ERTS_SCHED_NORMAL) == 1) {
 			clr_flg = ERTS_SCHDLR_SSPND_CHNG_NMSB;
 		    }
-		    else if (schdlr_sspnd.active
-			     == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)) {
-			clr_flg = ERTS_SCHDLR_SSPND_CHNG_MSB;
+		    else if (schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+                                                      ERTS_SCHED_NORMAL) == 1
+                             && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+                                                         ERTS_SCHED_DIRTY_CPU) == 0
+                             && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+                                                         ERTS_SCHED_DIRTY_IO) == 0) {
+                        clr_flg = ERTS_SCHDLR_SSPND_CHNG_MSB;
 		    }
 
 		    if (clr_flg) {
@@ -7308,10 +7626,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 		}
 	    }
 
-	    if (curr_online
-		&& (sched_type == ERTS_SCHED_NORMAL
-		    ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)
-		    : !schdlr_sspnd.msb.ongoing)) {
+	    if (curr_online) {
 		flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
 		if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
 		    break;
@@ -7327,13 +7642,11 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 		if (sched_type != ERTS_SCHED_NORMAL)
 		    aux_work = 0;
 		else {
-		    erts_aint32_t qmask;
-		    qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue)
-			     & ERTS_RUNQ_FLGS_QMASK);
+                    int evacuate = no == 1 ? 0 : !ERTS_EMPTY_RUNQ(esdp->run_queue);
 
 		    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 
-		    if (aux_work|qmask) {
+		    if (aux_work|evacuate) {
 			if (!thr_prgr_active) {
 			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
 			    sched_wall_time_change(esdp, 1);
@@ -7345,7 +7658,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 			if (aux_work && erts_thr_progress_update(esdp))
 			    erts_thr_progress_leader_update(esdp);
-			if (qmask) {
+			if (evacuate) {
 			    erts_smp_runq_lock(esdp->run_queue);
 			    evacuate_run_queue(esdp->run_queue, &sbp);
 			    erts_smp_runq_unlock(esdp->run_queue);
@@ -7460,7 +7773,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
         if (changing) {
             if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
                 && !schdlr_sspnd.msb.ongoing
-                && schdlr_sspnd.online == schdlr_sspnd.active) {
+                && schdlr_sspnd_eq_nscheds(&schdlr_sspnd.online,
+                                           &schdlr_sspnd.active)) {
                 erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
                                                 ~ERTS_SCHDLR_SSPND_CHNG_MSB);
             }
@@ -7475,19 +7789,16 @@ suspend_scheduler(ErtsSchedulerData *esdp)
             }
         }
 	ASSERT(no <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, sched_type));
-	ASSERT((sched_type == ERTS_SCHED_NORMAL
-		? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)
-		: !schdlr_sspnd.msb.ongoing));
     }
 
     erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
 
-    ASSERT(!resume.msb.chngrs);
     schdlr_sspnd_resume_procs(sched_type, &resume);
 
     ASSERT(curr_online);
 
-    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+    if (sched_type == ERTS_SCHED_NORMAL) {
+	(void) erts_get_monotonic_time(esdp);
 	if (erts_system_profile_flags.scheduler)
 	    profile_scheduler(make_small(esdp->no), am_active);
 
@@ -7497,8 +7808,6 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 	}
     }
 
-    if (sched_type == ERTS_SCHED_NORMAL)
-	(void) erts_get_monotonic_time(esdp);
     erts_smp_runq_lock(esdp->run_queue);
     non_empty_runq(esdp->run_queue);
 
@@ -7768,10 +8077,8 @@ erts_set_schedulers_online(Process *p,
 		    erts_sched_poke(ssi);
 		}
 	    } else {
-		for (ix = dirty_online; ix < dirty_no; ix++) {
-		    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-		    scheduler_ssi_resume_wake(ssi);
-		}
+		for (ix = dirty_online; ix < dirty_no; ix++)
+		    dcpu_sched_ix_resume_wake(ix);
 	    }
 	}
 	if (!dirty_only)
@@ -7799,19 +8106,19 @@ erts_set_schedulers_online(Process *p,
     else /* if decrease */ {
 #ifdef ERTS_DIRTY_SCHEDULERS
 	if (change_dirty) {
-	    ErtsSchedulerSleepInfo* ssi;
 	    if (schdlr_sspnd.msb.ongoing) {
-		for (ix = dirty_no; ix < dirty_online; ix++) {
-		    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-		    erts_sched_poke(ssi);
-		}
-	    } else {
-		for (ix = dirty_no; ix < dirty_online; ix++) {
-		    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-		    erts_smp_atomic32_read_bor_nob(&ssi->flags,
-						   ERTS_SSI_FLG_SUSPENDED);
-		}
-		wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
+		for (ix = dirty_no; ix < dirty_online; ix++)
+		    erts_sched_poke(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix));
+	    }
+            else {
+		for (ix = dirty_no; ix < dirty_online; ix++)
+                    dcpu_sched_ix_suspend_wake(ix);
+                /*
+                 * Newly suspended scheduler may have just been
+                 * about to handle a task. Make sure someone takes
+                 * care of such a task...
+                 */
+                dcpu_sched_ix_wake(0);
 	    }
 	}
 	if (!dirty_only)
@@ -7874,9 +8181,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 {
     int resume_proc, ix, res, have_unlocked_plocks = 0;
     ErtsProcList *plp;
-#ifdef ERTS_DIRTY_SCHEDULERS
-    ErtsSchedulerSleepInfo* ssi;
-#endif
     ErtsMultiSchedulingBlock *msbp;
     erts_aint32_t chng_flg;
     int have_blckd_flg;
@@ -7923,8 +8227,9 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 	    erts_proclist_store_last(&msbp->blckrs, plp);
 	    p->flags |= have_blckd_flg;
 	    ASSERT(normal
-		   ? 1 == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, ERTS_SCHED_NORMAL)
-		   : schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0));
+                   ? 1 == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+                                                   ERTS_SCHED_NORMAL)
+                   : schdlr_sspnd_get_nscheds_tot(&schdlr_sspnd.active) == 1);
 	    ASSERT(erts_proc_sched_data(p)->no == 1);
 	    if (schdlr_sspnd.msb.ongoing)
 		res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
@@ -7942,59 +8247,41 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 	    }
 	    ASSERT(!msbp->ongoing);
 	    msbp->ongoing = 1;
-	    if (schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)
-		|| (normal && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
-						       ERTS_SCHED_NORMAL) == 1)) {
-		ASSERT(erts_proc_sched_data(p)->no == 1);
-		plp = proclist_create(p);
-		erts_proclist_store_last(&msbp->blckrs, plp);
-		if (schdlr_sspnd.msb.ongoing)
-		    res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
-		else
-		    res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED;
-	    }
-	    else {
-		erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing,
-					       chng_flg);
-		change_no_used_runqs(1);
-		for (ix = 1; ix < erts_no_run_queues; ix++)
-		    suspend_run_queue(ERTS_RUNQ_IX(ix));
 
-		for (ix = 1; ix < online; ix++) {
-		    ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
-		    wake_scheduler(rq);
-		}
+            erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing,
+                                           chng_flg);
+            change_no_used_runqs(1);
+            for (ix = 1; ix < erts_no_run_queues; ix++)
+                suspend_run_queue(ERTS_RUNQ_IX(ix));
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-		if (!normal) {
-		    for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
-			ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-			erts_smp_atomic32_read_bor_nob(&ssi->flags,
-						       ERTS_SSI_FLG_SUSPENDED);
-		    }
-		    wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
+            for (ix = 1; ix < online; ix++) {
+                ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
+                wake_scheduler(rq);
+            }
 
-		    for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
-			ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
-			erts_smp_atomic32_read_bor_nob(&ssi->flags,
-						       ERTS_SSI_FLG_SUSPENDED);
-		    }
-		    wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0);
-		}
+#ifdef ERTS_DIRTY_SCHEDULERS
+            if (!normal) {
+                ERTS_RUNQ_FLGS_SET_NOB(ERTS_RUNQ_IX(0), ERTS_RUNQ_FLG_MSB_EXEC);
+                erts_smp_atomic32_read_bor_nob(&ERTS_RUNQ_IX(0)->scheduler->ssi->flags,
+                                               ERTS_SSI_FLG_MSB_EXEC);
+                for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++)
+                    dcpu_sched_ix_suspend_wake(ix);
+                for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++)
+                    dio_sched_ix_suspend_wake(ix);
+            }
 #endif
 
-	    wait_until_msb:
+        wait_until_msb:
 
-		ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing));
+            ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing));
 
-		plp = proclist_create(p);
-		erts_proclist_store_last(&msbp->chngq, plp);
-		resume_proc = 0;
-		if (schdlr_sspnd.msb.ongoing)
-		    res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED;
-		else
-		    res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED;
-	    }
+            plp = proclist_create(p);
+            erts_proclist_store_last(&msbp->chngq, plp);
+            resume_proc = 0;
+            if (schdlr_sspnd.msb.ongoing)
+                res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED;
+            else
+                res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED;
 	    ASSERT(erts_proc_sched_data(p));
 	}
     }
@@ -8026,27 +8313,19 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 	    }
 	}
 	if (!msbp->blckrs && !msbp->chngq) {
-	    int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
-							ERTS_SCHED_NORMAL);
+	    int online;
 	    erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing,
 					   chng_flg);
 	    p->flags &= ~have_blckd_flg;
 	    msbp->ongoing = 0;
-	    if (online == 1) {
-		/* No normal schedulers to resume */
-		ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
-						ERTS_SCHED_NORMAL) == 1);
-#ifndef ERTS_DIRTY_SCHEDULERS
-		erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
-						~chng_flg);
-#endif
-	    }
-	    else if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) {
-		if (plocks) {
+            if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) {
+                if (plocks) {
 		    have_unlocked_plocks = 1;
 		    erts_smp_proc_unlock(p, plocks);
 		}
 
+                online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+							ERTS_SCHED_NORMAL);
 		change_no_used_runqs(online);
 
 		/* Resume all online run queues */
@@ -8057,19 +8336,15 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 		    suspend_run_queue(ERTS_RUNQ_IX(ix));
 	    }
 #ifdef ERTS_DIRTY_SCHEDULERS
-	    if (!normal) {
-		ASSERT(!schdlr_sspnd.msb.ongoing);
+            if (!schdlr_sspnd.msb.ongoing) {
+                /* Get rid of msb-exec flag in run-queue of scheduler 1 */
+                resume_run_queue(ERTS_RUNQ_IX(0));
 		online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
 							ERTS_SCHED_DIRTY_CPU);
-		for (ix = 0; ix < online; ix++) {
-		    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-		    scheduler_ssi_resume_wake(ssi);
-		}
-
-		for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
-		    ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
-		    scheduler_ssi_resume_wake(ssi);
-		}
+		for (ix = 0; ix < online; ix++)
+		    dcpu_sched_ix_resume_wake(ix);
+		for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++)
+		    dio_sched_ix_resume_wake(ix);
 	    }
 #endif
 	}
@@ -9734,7 +10009,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	if (!is_normal_sched) {
 	    if (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
-		& ERTS_SSI_FLG_SUSPENDED) {
+		& (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) {
 		suspend_scheduler(esdp);
 	    }
 	}
@@ -9744,8 +10019,10 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	    ASSERT(is_normal_sched);
 
-	    if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) {
-		if (flags & ERTS_RUNQ_FLG_SUSPENDED) {
+	    if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND
+                         | ERTS_RUNQ_FLG_SUSPENDED
+                         | ERTS_RUNQ_FLG_MSB_EXEC)) {
+		if (flags & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_MSB_EXEC)) {
 		    (void) ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_EXEC);
 		    suspend_scheduler(esdp);
 		    flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC);
@@ -9784,13 +10061,12 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
 
-	if (!is_normal_sched && rq->halt_in_progress) {
+	if (!is_normal_sched & !!(flags & ERTS_RUNQ_FLG_HALTING)) {
 	    /* Wait for emulator to terminate... */
 	    while (1)
 		erts_milli_sleep(1000*1000);
 	}
-	else if ((!(flags & ERTS_RUNQ_FLGS_QMASK) && !rq->misc.start)
-		 || (rq->halt_in_progress && ERTS_EMPTY_RUNQ_PORTS(rq))) {
+	else if (!runq_got_work_to_execute_flags(flags)) {
 	    /* Prepare for scheduler wait */
 #ifdef ERTS_SMP
 	    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
@@ -9804,21 +10080,44 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    if (flags & ERTS_RUNQ_FLG_INACTIVE)
 		empty_runq(rq);
 	    else {
-		if (is_normal_sched && try_steal_task(rq))
-		    goto continue_check_activities_to_run;
-
-		empty_runq(rq);
-
-		/*
-		 * Check for ERTS_RUNQ_FLG_SUSPENDED has to be done
-		 * after trying to steal a task.
-		 */
-		flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
-		if (flags & ERTS_RUNQ_FLG_SUSPENDED) {
-		    non_empty_runq(rq);
-		    flags |= ERTS_RUNQ_FLG_NONEMPTY;
-		    goto continue_check_activities_to_run_known_flags;
+                ASSERT(!runq_got_work_to_execute(rq));
+		if (!is_normal_sched) {
+                    /* Dirty scheduler */
+                    if (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
+                        & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) {
+                        /* Go suspend... */
+                        goto continue_check_activities_to_run_known_flags;
+                    }
+                }
+                else {
+                    /* Normal scheduler */
+                    if (try_steal_task(rq))
+                        goto continue_check_activities_to_run;
+                    /*
+                     * Check for suspend has to be done after trying
+                     * to steal a task...
+                     */
+                    flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
+                    if ((flags & ERTS_RUNQ_FLG_SUSPENDED)
+#ifdef ERTS_DIRTY_SCHEDULERS
+                        /* If multi scheduling block and we have
+                         * dirty work, suspend and let dirty
+                         * scheduler handle work... */
+                        || ((((flags & (ERTS_RUNQ_FLG_HALTING
+                                       | ERTS_RUNQ_FLG_MSB_EXEC))
+                              == ERTS_RUNQ_FLG_MSB_EXEC))
+                            && have_dirty_work())
+#endif
+                        ) {
+                        non_empty_runq(rq);
+                        flags |= ERTS_RUNQ_FLG_NONEMPTY;
+                        /*
+                         * Go suspend...
+                         */
+                        goto continue_check_activities_to_run_known_flags;
+                    }
 		}
+		empty_runq(rq);
 	    }
 #endif
 
@@ -9870,7 +10169,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 #endif
 	}
 
-	if (rq->misc.start)
+        if (flags & ERTS_RUNQ_FLG_MISC_OP)
 	    exec_misc_ops(rq);
 
 #ifdef ERTS_SMP
@@ -9881,13 +10180,15 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	 * Find a new port to run.
 	 */
 
-	if (RUNQ_READ_LEN(&rq->ports.info.len)) {
+        flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
+
+	if (flags & PORT_BIT) {
 	    int have_outstanding_io;
 	    have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port);
 	    if ((!erts_eager_check_io
 		 && have_outstanding_io
 		 && fcalls > 2*input_reductions)
-		|| rq->halt_in_progress) {
+		|| (flags & ERTS_RUNQ_FLG_HALTING)) {
 		/*
 		 * If we have performed more than 2*INPUT_REDUCTIONS since
 		 * last call to erl_sys_schedule() and we still haven't
@@ -11419,6 +11720,8 @@ erts_schedule_misc_op(void (*func)(void *), void *arg)
     non_empty_runq(rq);
 #endif
 
+    ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP);
+
     erts_smp_runq_unlock(rq);
 
     smp_notify_inc_runq(rq);
@@ -11449,6 +11752,9 @@ exec_misc_ops(ErtsRunQueue *rq)
 	    rq->misc.end = NULL;
     }
 
+    if (!rq->misc.start)
+        ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP);
+
     erts_smp_runq_unlock(rq);
 
     while (molp) {
@@ -13576,6 +13882,8 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
                 erts_print(to, to_arg, "WAITING"); break;
             case ERTS_SSI_FLG_SUSPENDED:
                 erts_print(to, to_arg, "SUSPENDED"); break;
+            case ERTS_SSI_FLG_MSB_EXEC:
+                erts_print(to, to_arg, "MSB_EXEC"); break;
             default:
                 erts_print(to, to_arg, "UNKNOWN(%d)", flg); break;
             }
@@ -13685,6 +13993,12 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
                 erts_print(to, to_arg, "NONEMPTY"); break;
             case ERTS_RUNQ_FLG_PROTECTED:
                 erts_print(to, to_arg, "PROTECTED"); break;
+            case ERTS_RUNQ_FLG_EXEC:
+                erts_print(to, to_arg, "EXEC"); break;
+            case ERTS_RUNQ_FLG_MSB_EXEC:
+                erts_print(to, to_arg, "MSB_EXEC"); break;
+            case ERTS_RUNQ_FLG_MISC_OP:
+                erts_print(to, to_arg, "MISC_OP"); break;
             default:
                 erts_print(to, to_arg, "UNKNOWN(%d)", flg); break;
             }
@@ -13734,11 +14048,11 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
  * A nice system halt closing all open port goes as follows:
  * 1) This function schedules the aux work ERTS_SSI_AUX_WORK_REAP_PORTS
  *    on all schedulers, then schedules itself out.
- * 2) All shedulers detect this and set the flag halt_in_progress
+ * 2) All shedulers detect this and set the flag ERTS_RUNQ_FLG_HALTING
  *    on their run queue. The last scheduler sets all non-closed ports
  *    ERTS_PORT_SFLG_HALT. Global atomic erts_halt_progress is used
  *    as refcount to determine which is last.
- * 3) While the run ques has flag halt_in_progress no processes
+ * 3) While the run queues has flag ERTS_RUNQ_FLG_HALTING no processes
  *    will be scheduled, only ports.
  * 4) When the last port closes that scheduler calls erlang:halt/1.
  *    The same global atomic is used as refcount.
@@ -13753,8 +14067,8 @@ void erts_halt(int code)
 					     erts_no_schedulers,
 					     -1)) {
 #ifdef ERTS_DIRTY_SCHEDULERS
-	ERTS_DIRTY_CPU_RUNQ->halt_in_progress = 1;
-	ERTS_DIRTY_IO_RUNQ->halt_in_progress = 1;
+        ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_CPU_RUNQ, ERTS_RUNQ_FLG_HALTING);
+        ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_IO_RUNQ, ERTS_RUNQ_FLG_HALTING);
 #endif
 	erts_halt_code = code;
 	notify_reap_ports_relb();
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 3938c4c913..fc2055332b 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -170,8 +170,14 @@ extern int erts_sched_thread_suggested_stack_size;
   (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6))
 #define ERTS_RUNQ_FLG_EXEC \
   (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7))
+#define ERTS_RUNQ_FLG_MSB_EXEC \
+  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 8))
+#define ERTS_RUNQ_FLG_MISC_OP \
+  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 9))
+#define ERTS_RUNQ_FLG_HALTING \
+  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 10))
 
-#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8)
+#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 11)
 
 #define ERTS_RUNQ_FLGS_MIGRATION_QMASKS	\
   (ERTS_RUNQ_FLGS_EMIGRATE_QMASK	\
@@ -262,8 +268,9 @@ typedef enum {
 #define ERTS_SSI_FLG_TSE_SLEEPING 	(((erts_aint32_t) 1) << 2)
 #define ERTS_SSI_FLG_WAITING		(((erts_aint32_t) 1) << 3)
 #define ERTS_SSI_FLG_SUSPENDED	 	(((erts_aint32_t) 1) << 4)
+#define ERTS_SSI_FLG_MSB_EXEC	 	(((erts_aint32_t) 1) << 5)
 
-#define ERTS_SSI_FLGS_MAX                                       5
+#define ERTS_SSI_FLGS_MAX                                       6
 
 #define ERTS_SSI_FLGS_SLEEP_TYPE			\
  (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING)
@@ -274,7 +281,8 @@ typedef enum {
 #define ERTS_SSI_FLGS_ALL				\
  (ERTS_SSI_FLGS_SLEEP					\
   | ERTS_SSI_FLG_WAITING				\
-  | ERTS_SSI_FLG_SUSPENDED)
+  | ERTS_SSI_FLG_SUSPENDED                              \
+  | ERTS_SSI_FLG_MSB_EXEC)
 
 /*
  * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency
@@ -393,6 +401,12 @@ typedef struct {
     Process* last;
 } ErtsRunPrioQueue;
 
+typedef enum {
+    ERTS_SCHED_NORMAL,
+    ERTS_SCHED_DIRTY_CPU,
+    ERTS_SCHED_DIRTY_IO
+} ErtsSchedType;
+
 typedef struct ErtsSchedulerData_ ErtsSchedulerData;
 
 typedef struct ErtsRunQueue_ ErtsRunQueue;
@@ -478,7 +492,6 @@ struct ErtsRunQueue_ {
     erts_smp_atomic32_t len;
     int wakeup_other;
     int wakeup_other_reds;
-    int halt_in_progress;
 
     struct {
 	ErtsProcList *pending_exiters;
@@ -642,6 +655,7 @@ struct ErtsSchedulerData_ {
 #endif
     ErtsSchedulerSleepInfo *ssi;
     Process *current_process;
+    ErtsSchedType type;
     Uint no;			/* Scheduler number for normal schedulers */
 #ifdef ERTS_DIRTY_SCHEDULERS
     ErtsDirtySchedId dirty_no;  /* Scheduler number for dirty schedulers */
-- 
cgit v1.2.3


From 82ba60f0ead61f344e2a7ebd383f62e99cb0375d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 16 Jan 2017 18:01:20 +0100
Subject: erts: Fix port_trace_SUITE to join threads

when port is closed before driver may be unloaded.
---
 .../emulator/test/port_trace_SUITE_data/echo_drv.c | 39 ++++++++++++++++------
 1 file changed, 29 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/port_trace_SUITE_data/echo_drv.c b/erts/emulator/test/port_trace_SUITE_data/echo_drv.c
index b545523192..20ec33a594 100644
--- a/erts/emulator/test/port_trace_SUITE_data/echo_drv.c
+++ b/erts/emulator/test/port_trace_SUITE_data/echo_drv.c
@@ -2,23 +2,30 @@
 #include "erl_driver.h"
 #include <errno.h>
 #include <string.h>
+#include <assert.h>
 
 
 /* -------------------------------------------------------------------------
 ** Data types
 **/
 
+struct my_thread {
+    struct my_thread* next;
+    ErlDrvTid tid;
+};
 
 typedef struct _erl_drv_data {
     ErlDrvPort   erlang_port;
     ErlDrvTermData caller;
+    struct my_thread* threads;
 } EchoDrvData;
 
 struct remote_send_term {
-    char *buf;
-    int len;
+    struct my_thread thread;
     ErlDrvTermData port;
     ErlDrvTermData caller;
+    int len;
+    char buf[1]; /* buf[len] */
 };
 
 #define ECHO_DRV_NOOP                 0
@@ -86,7 +93,7 @@ static ErlDrvEntry echo_drv_entry = {
     NULL
 };
 
-static void send_term_thread(void *);
+static void* send_term_thread(void *);
 
 /* -------------------------------------------------------------------------
 ** Entry functions
@@ -111,10 +118,22 @@ static EchoDrvData *echo_drv_start(ErlDrvPort port, char *command)
     EchoDrvData *echo_drv_data_p = driver_alloc(sizeof(EchoDrvData));
     echo_drv_data_p->erlang_port = port;
     echo_drv_data_p->caller = driver_caller(port);
+    echo_drv_data_p->threads = NULL;
     return echo_drv_data_p;
 }
 
-static void echo_drv_stop(EchoDrvData *data_p) {
+static void echo_drv_stop(EchoDrvData *data_p)
+{
+    struct my_thread* thr = data_p->threads;
+
+    while (thr) {
+        struct my_thread* next = thr->next;
+        void* exit_value;
+        int ret = erl_drv_thread_join(thr->tid, &exit_value);
+        assert(ret == 0 && exit_value == NULL);
+        driver_free(thr);
+        thr = next;
+    }
     driver_free(data_p);
 }
 
@@ -212,14 +231,14 @@ static void echo_drv_output(ErlDrvData drv_data, char *buf, ErlDrvSizeT len) {
     }
     case ECHO_DRV_REMOTE_SEND_TERM:
     {
-        ErlDrvTid tid;
-        struct remote_send_term *t = malloc(sizeof(struct remote_send_term));
+        struct remote_send_term *t = driver_alloc(sizeof(struct remote_send_term) + len);
         t->len = len-1;
-        t->buf = malloc(len-1);
         t->port = driver_mk_port(port);
         t->caller = data_p->caller;
         memcpy(t->buf, buf+1, t->len);
-        erl_drv_thread_create("tmp_thread", &tid, send_term_thread, t, NULL);
+        erl_drv_thread_create("tmp_thread", &t->thread.tid, send_term_thread, t, NULL);
+        t->thread.next = data_p->threads;
+        data_p->threads = &t->thread;
         break;
     }
     case ECHO_DRV_SAVE_CALLER:
@@ -262,7 +281,7 @@ static ErlDrvSSizeT echo_drv_call(ErlDrvData drv_data,
     return len-command;
 }
 
-static void send_term_thread(void *a)
+static void* send_term_thread(void *a)
 {
     struct remote_send_term *t = (struct remote_send_term*)a;
     ErlDrvTermData term[] = {
@@ -273,5 +292,5 @@ static void send_term_thread(void *a)
             ERL_DRV_TUPLE, 3};
     erl_drv_send_term(t->port, t->caller,
                       term, sizeof(term) / sizeof(ErlDrvTermData));
-    return;
+    return NULL;
 }
-- 
cgit v1.2.3


From fd1fb9070333fe5b91f4528841a4b0fe71bbf096 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 12 Jan 2017 14:25:30 +0100
Subject: Always return info from system_info(dirty_[cpu|io]_schedulers*)

---
 erts/emulator/beam/erl_bif_info.c      | 14 ++++++++++++--
 erts/emulator/test/dirty_bif_SUITE.erl |  8 ++++----
 erts/emulator/test/dirty_nif_SUITE.erl |  8 ++++----
 erts/emulator/test/scheduler_SUITE.erl | 22 ++++++++--------------
 4 files changed, 28 insertions(+), 24 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 4659cb5a7b..5f2122b969 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2680,20 +2680,30 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	erts_schedulers_state(NULL, NULL, &active, NULL, NULL, NULL, NULL, NULL);
 	BIF_RET(make_small(active));
 #endif
-#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS)
     } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers", BIF_ARG_1)) {
 	Uint dirty_cpu;
+#ifdef ERTS_DIRTY_SCHEDULERS
 	erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, NULL, NULL);
+#else
+        dirty_cpu = 0;
+#endif
 	BIF_RET(make_small(dirty_cpu));
     } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers_online", BIF_ARG_1)) {
 	Uint dirty_cpu_onln;
+#ifdef ERTS_DIRTY_SCHEDULERS
 	erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, NULL, NULL);
+#else
+        dirty_cpu_onln = 0;
+#endif
 	BIF_RET(make_small(dirty_cpu_onln));
     } else if (ERTS_IS_ATOM_STR("dirty_io_schedulers", BIF_ARG_1)) {
 	Uint dirty_io;
+#ifdef ERTS_DIRTY_SCHEDULERS
 	erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, NULL, &dirty_io, NULL);
-	BIF_RET(make_small(dirty_io));
+#else
+        dirty_io = 0;
 #endif
+	BIF_RET(make_small(dirty_io));
     } else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) {
 	res = make_small(erts_no_run_queues);
 	BIF_RET(res);
diff --git a/erts/emulator/test/dirty_bif_SUITE.erl b/erts/emulator/test/dirty_bif_SUITE.erl
index 01ff8d6efd..308323594d 100644
--- a/erts/emulator/test/dirty_bif_SUITE.erl
+++ b/erts/emulator/test/dirty_bif_SUITE.erl
@@ -67,10 +67,10 @@ all() ->
      code_purge].
 
 init_per_suite(Config) ->
-    try erlang:system_info(dirty_cpu_schedulers) of
-	N when is_integer(N), N > 0 ->
-	    Config
-    catch _:_ ->
+    case erlang:system_info(dirty_cpu_schedulers) of
+	N when N > 0 ->
+	    Config;
+        _ ->
 	    {skipped, "No dirty scheduler support"}
     end.
 
diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl
index a61fd92a18..991ba0acc8 100644
--- a/erts/emulator/test/dirty_nif_SUITE.erl
+++ b/erts/emulator/test/dirty_nif_SUITE.erl
@@ -54,8 +54,8 @@ all() ->
      dirty_nif_send_traced].
 
 init_per_suite(Config) ->
-    try erlang:system_info(dirty_cpu_schedulers) of
-	N when is_integer(N), N > 0 ->
+    case erlang:system_info(dirty_cpu_schedulers) of
+	N when N > 0 ->
 	    case lib_loaded() of
 		false ->
 		    ok = erlang:load_nif(
@@ -64,8 +64,8 @@ init_per_suite(Config) ->
 		true ->
 		    ok
 	    end,
-	    Config
-    catch _:_ ->
+	    Config;
+        _ ->
 	    {skipped, "No dirty scheduler support"}
     end.
 
diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl
index 3aee15a8fc..b178dede5b 100644
--- a/erts/emulator/test/scheduler_SUITE.erl
+++ b/erts/emulator/test/scheduler_SUITE.erl
@@ -1095,16 +1095,13 @@ scheduler_threads(Config) when is_list(Config) ->
     end.
 
 dirty_scheduler_threads(Config) when is_list(Config) ->
-    SmpSupport = erlang:system_info(smp_support),
-    try
-	erlang:system_info(dirty_cpu_schedulers),
-	dirty_scheduler_threads_test(Config, SmpSupport)
-    catch
-	error:badarg ->
-	    {skipped, "No dirty scheduler support"}
+    case erlang:system_info(dirty_cpu_schedulers) of
+        0 -> {skipped, "No dirty scheduler support"};
+        _ -> dirty_scheduler_threads_test(Config)
     end.
 
-dirty_scheduler_threads_test(Config, SmpSupport) ->
+dirty_scheduler_threads_test(Config) ->
+    SmpSupport = erlang:system_info(smp_support),
     {Sched, SchedOnln, _} = get_dsstate(Config, ""),
     {HalfSched, HalfSchedOnln} = case SmpSupport of
                                      false -> {1,1};
@@ -1374,12 +1371,9 @@ sst2_loop(N) ->
     sst2_loop(N-1).
 
 sst3_loop(S, N) ->
-    try erlang:system_info(dirty_cpu_schedulers) of
-	DS ->
-	    sst3_loop_with_dirty_schedulers(S, DS, N)
-    catch
-	error:badarg ->
-	    sst3_loop_normal_schedulers_only(S, N)
+    case erlang:system_info(dirty_cpu_schedulers) of
+        0 -> sst3_loop_normal_schedulers_only(S, N);
+	DS -> sst3_loop_with_dirty_schedulers(S, DS, N)
     end.
 
 sst3_loop_normal_schedulers_only(_S, 0) ->
-- 
cgit v1.2.3


From f9459092940943876dff040ee997515b96fd5d50 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 4 Jan 2017 18:10:26 +0100
Subject: Scheduler wall time support for dirty schedulers

---
 erts/emulator/beam/atom.names           |   3 +-
 erts/emulator/beam/beam_debug.c         |  13 +-
 erts/emulator/beam/bif.c                |   2 +-
 erts/emulator/beam/erl_bif_info.c       |   7 +-
 erts/emulator/beam/erl_process.c        | 366 ++++++++++++++++++++++++++------
 erts/emulator/beam/erl_process.h        |   9 +-
 erts/emulator/test/statistics_SUITE.erl | 131 ++++++++++--
 7 files changed, 445 insertions(+), 86 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 7df350116a..dd0a42b5ba 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -574,6 +574,8 @@ atom safe
 atom save_calls
 atom scheduler 
 atom scheduler_id
+atom scheduler_wall_time
+atom scheduler_wall_time_all
 atom schedulers_online
 atom scheme
 atom scientific
@@ -678,7 +680,6 @@ atom waiting
 atom wall_clock
 atom warning
 atom warning_msg
-atom scheduler_wall_time
 atom wordsize
 atom write_concurrency
 atom xor
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 3ffbb0364c..8326d348af 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -994,13 +994,20 @@ dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I)
 	Eterm *hp, *hp2;
 	Uint sz;
 	int i;
+	ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
+        int dirty_io = esdp->type == ERTS_SCHED_DIRTY_IO;
+
 	if (ERTS_PROC_IS_EXITING(real_c_p))
 	    goto badarg;
 	dirty_send_message(c_p, arg2, am_alive);
-	/* Wait until dead */
 
-	while (!ERTS_PROC_IS_EXITING(real_c_p))
-	    erts_thr_yield();
+	/* Wait until dead */
+	while (!ERTS_PROC_IS_EXITING(real_c_p)) {
+            if (dirty_io)
+                ms_wait(c_p, make_small(100), 0);
+            else
+                erts_thr_yield();
+        }
 
 	ms_wait(c_p, make_small(1000), 0);
 
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index a5b66db35c..65c370c55b 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4601,7 +4601,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	    erts_aint32_t new = BIF_ARG_2 == am_true ? 1 : 0;
 	    erts_aint32_t old = erts_smp_atomic32_xchg_nob(&sched_wall_time,
 							   new);
-	    Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new);
+	    Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new, 0, 0);
 	    ASSERT(is_value(ref));
 	    BIF_TRAP2(await_sched_wall_time_mod_trap,
 		      BIF_P,
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 5f2122b969..5ee19aead8 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3385,7 +3385,12 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
     Eterm* hp;
 
     if (BIF_ARG_1 == am_scheduler_wall_time) {
-	res = erts_sched_wall_time_request(BIF_P, 0, 0);
+	res = erts_sched_wall_time_request(BIF_P, 0, 0, 1, 0);
+	if (is_non_value(res))
+	    BIF_RET(am_undefined);
+	BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res);
+    } else if (BIF_ARG_1 == am_scheduler_wall_time_all) {
+	res = erts_sched_wall_time_request(BIF_P, 0, 0, 1, 1);
 	if (is_non_value(res))
 	    BIF_RET(am_undefined);
 	BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 084ecb14ec..8c9aa45a81 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -804,15 +804,28 @@ erts_late_init_process(void)
 
 }
 
+#define ERTS_SCHED_WTIME_IDLE ~((Uint64) 0)
+
 static void
-init_sched_wall_time(ErtsSchedWallTime *swtp)
+init_sched_wall_time(ErtsSchedulerData *esdp, Uint64 time_stamp)
 {
-    swtp->need = erts_sched_balance_util;
-    swtp->enabled = 0;
-    swtp->start = 0;
-    swtp->working.total = 0;
-    swtp->working.start = 0;
-    swtp->working.currently = 0;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (esdp->type != ERTS_SCHED_NORMAL) {
+        erts_atomic32_init_nob(&esdp->sched_wall_time.u.mod, 0);
+        esdp->sched_wall_time.enabled = 1;
+        esdp->sched_wall_time.start = time_stamp;
+        esdp->sched_wall_time.working.total = 0;
+        esdp->sched_wall_time.working.start = ERTS_SCHED_WTIME_IDLE;
+    }
+    else
+#endif
+    {
+        esdp->sched_wall_time.u.need = erts_sched_balance_util;
+        esdp->sched_wall_time.enabled = 0;
+        esdp->sched_wall_time.start = 0;
+        esdp->sched_wall_time.working.total = 0;
+        esdp->sched_wall_time.working.start = 0;
+    }
 }
 
 static ERTS_INLINE Uint64
@@ -1003,31 +1016,145 @@ init_runq_sched_util(ErtsRunQueueSchedUtil *rqsu, int enabled)
 
 #endif /* ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT */
 
-static ERTS_INLINE void
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+typedef struct {
+    Uint64 working;
+    Uint64 total;
+} ErtsDirtySchedWallTime;
+
+static void
+read_dirty_sched_wall_time(ErtsSchedulerData *esdp, ErtsDirtySchedWallTime *info)
+{
+    erts_aint32_t mod1;
+    Uint64 working, start, ts;
+
+    mod1 = erts_atomic32_read_nob(&esdp->sched_wall_time.u.mod);
+
+    while (1) {
+        erts_aint32_t mod2;
+
+        /* Spin until values are not written... */
+        while (1) {
+            if ((mod1 & 1) == 0)
+                break;
+            ERTS_SPIN_BODY;
+            mod1 = erts_atomic32_read_nob(&esdp->sched_wall_time.u.mod);
+        }
+
+        ERTS_THR_READ_MEMORY_BARRIER;
+
+        working = esdp->sched_wall_time.working.total;
+        start = esdp->sched_wall_time.working.start;
+
+        ERTS_THR_READ_MEMORY_BARRIER;
+
+        mod2 = erts_atomic32_read_nob(&esdp->sched_wall_time.u.mod);
+        if (mod1 == mod2)
+            break;
+        mod1 = mod2;
+    }
+
+    ts = sched_wall_time_ts();
+    ts -= esdp->sched_wall_time.start;
+
+    info->total = ts;
+
+    if (start == ERTS_SCHED_WTIME_IDLE || ts < start)
+        info->working = working;
+    else
+        info->working = working + (ts - start);
+
+    if (info->working > info->total)
+        info->working = info->total;
+}
+
+#endif
+
+#ifdef ERTS_SMP
+
+static void
+dirty_sched_wall_time_change(ErtsSchedulerData *esdp, int working)
+{
+    erts_aint32_t mod;
+    Uint64 ts = sched_wall_time_ts();
+
+    ts -= esdp->sched_wall_time.start;
+
+    /*
+     * This thread is the only thread writing in
+     * this sched_wall_time struct. We set 'mod' to
+     * an odd value while writing...
+     */
+    mod = erts_atomic32_read_dirty(&esdp->sched_wall_time.u.mod);
+    ASSERT((mod & 1) == 0);
+    mod++;
+
+    erts_atomic32_set_nob(&esdp->sched_wall_time.u.mod, mod);
+    ERTS_THR_WRITE_MEMORY_BARRIER;
+
+    if (working) {
+        ASSERT(esdp->sched_wall_time.working.start
+               == ERTS_SCHED_WTIME_IDLE);
+
+        esdp->sched_wall_time.working.start = ts;
+
+    }
+    else {
+        Uint64 total;
+
+        ASSERT(esdp->sched_wall_time.working.start
+               != ERTS_SCHED_WTIME_IDLE);
+
+        total = esdp->sched_wall_time.working.total;
+        total += ts - esdp->sched_wall_time.working.start;
+
+        esdp->sched_wall_time.working.total = total;
+        esdp->sched_wall_time.working.start = ERTS_SCHED_WTIME_IDLE;
+
+
+    }
+
+    ERTS_THR_WRITE_MEMORY_BARRIER;
+    mod++;
+    erts_atomic32_set_nob(&esdp->sched_wall_time.u.mod, mod);
+
+#if 0
+    if (!working) {
+        ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_BUSY_WAIT);
+    } else {
+        ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_OTHER);
+    }
+#endif
+}
+
+#endif /* ERTS_SMP */
+
+static void
 sched_wall_time_change(ErtsSchedulerData *esdp, int working)
 {
-    if (esdp->sched_wall_time.need) {
+    if (esdp->sched_wall_time.u.need) {
 	Uint64 ts = sched_wall_time_ts();
 #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
-	update_avg_sched_util(esdp, ts, working);
+        update_avg_sched_util(esdp, ts, working);
 #endif
 	if (esdp->sched_wall_time.enabled) {
 	    if (working) {
-#ifdef DEBUG
-		ASSERT(!esdp->sched_wall_time.working.currently);
-		esdp->sched_wall_time.working.currently = 1;
-#endif
+                ASSERT(esdp->sched_wall_time.working.start
+                       == ERTS_SCHED_WTIME_IDLE);
 		ts -= esdp->sched_wall_time.start;
 		esdp->sched_wall_time.working.start = ts;
 	    }
 	    else {
-#ifdef DEBUG
-		ASSERT(esdp->sched_wall_time.working.currently);
-		esdp->sched_wall_time.working.currently = 0;
-#endif
+                ASSERT(esdp->sched_wall_time.working.start
+                       != ERTS_SCHED_WTIME_IDLE);
 		ts -= esdp->sched_wall_time.start;
 		ts -= esdp->sched_wall_time.working.start;
 		esdp->sched_wall_time.working.total += ts;
+#ifdef DEBUG
+                esdp->sched_wall_time.working.start
+                    = ERTS_SCHED_WTIME_IDLE;
+#endif
 	    }
 	}
     }
@@ -1047,6 +1174,10 @@ typedef struct {
     Eterm ref_heap[REF_THING_SIZE];
     Uint req_sched;
     erts_smp_atomic32_t refc;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    int want_dirty_cpu;
+    int want_dirty_io;
+#endif
 } ErtsSchedWallTimeReq;
 
 typedef struct {
@@ -1068,6 +1199,7 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(screq,
 				 5,
 				 ERTS_ALC_T_SYS_CHECK_REQ)
 
+
 static void
 reply_sched_wall_time(void *vswtrp)
 {
@@ -1091,23 +1223,23 @@ reply_sched_wall_time(void *vswtrp)
 #endif
     if (swtrp->set) {
 	if (!swtrp->enable && esdp->sched_wall_time.enabled) {
-	    esdp->sched_wall_time.need = erts_sched_balance_util;
+	    esdp->sched_wall_time.u.need = erts_sched_balance_util;
 	    esdp->sched_wall_time.enabled = 0;
 	}
 	else if (swtrp->enable && !esdp->sched_wall_time.enabled) {
 	    Uint64 ts = sched_wall_time_ts();
-	    esdp->sched_wall_time.need = 1;
+	    esdp->sched_wall_time.u.need = 1;
 	    esdp->sched_wall_time.enabled = 1;
 	    esdp->sched_wall_time.start = ts;
 	    esdp->sched_wall_time.working.total = 0;
 	    esdp->sched_wall_time.working.start = 0;
-	    esdp->sched_wall_time.working.currently = 1;
 	}
     }
 
     if (esdp->sched_wall_time.enabled) {
 	Uint64 ts = sched_wall_time_ts();
-	ASSERT(esdp->sched_wall_time.working.currently);
+        ASSERT(esdp->sched_wall_time.working.start
+               != ERTS_SCHED_WTIME_IDLE);
 	ts -= esdp->sched_wall_time.start;
 	total = ts;
 	ts -= esdp->sched_wall_time.working.start;
@@ -1118,30 +1250,117 @@ reply_sched_wall_time(void *vswtrp)
     hpp = NULL;
     szp = &sz;
 
-    while (1) {
-	if (hpp)
-	    ref_copy = STORE_NC(hpp, ohp, swtrp->ref);
-	else
-	    *szp += REF_THING_SIZE;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (esdp->sched_wall_time.enabled
+        && swtrp->req_sched == esdp->no
+        && (swtrp->want_dirty_cpu || swtrp->want_dirty_io)) {
+        /* Reply with info about this scheduler and all dirty schedulers... */
+        ErtsDirtySchedWallTime *dswt;
+        int ix, no_dirty_scheds, want_dcpu, want_dio, soffset;
+
+        want_dcpu = swtrp->want_dirty_cpu;
+        want_dio = swtrp->want_dirty_io;
+
+        no_dirty_scheds = 0;
+        if (want_dcpu)
+            no_dirty_scheds += erts_no_dirty_cpu_schedulers;
+        if (want_dio)
+            no_dirty_scheds += erts_no_dirty_io_schedulers;
+
+        ASSERT(no_dirty_scheds);
+
+        dswt = erts_alloc(ERTS_ALC_T_TMP,
+                          sizeof(ErtsDirtySchedWallTime)
+                          * no_dirty_scheds);
+
+        for (ix = 0; ix < no_dirty_scheds; ix++) {
+            ErtsSchedulerData *esdp;
+            if (want_dcpu && ix < erts_no_dirty_cpu_schedulers)
+                esdp = &erts_aligned_dirty_cpu_scheduler_data[ix].esd;
+            else {
+                int dio_ix = ix - erts_no_dirty_cpu_schedulers;
+                esdp = &erts_aligned_dirty_io_scheduler_data[dio_ix].esd;
+            }
+            read_dirty_sched_wall_time(esdp, &dswt[ix]);
+        }
 
-	if (swtrp->set)
-	    msg = ref_copy;
-	else {
-	    msg = (!esdp->sched_wall_time.enabled
-		   ? am_notsup
-		   : erts_bld_tuple(hpp, szp, 3,
-				    make_small(esdp->no),
-				    erts_bld_uint64(hpp, szp, working),
-				    erts_bld_uint64(hpp, szp, total)));
+        soffset = erts_no_schedulers + 1;
 
-	    msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg);
-	}
-	if (hpp)
-	    break;
+        if (!want_dcpu) {
+            ASSERT(want_dio);
+            soffset += erts_no_dirty_cpu_schedulers;
+        }
+
+        while (1) {
+            if (hpp)
+                ref_copy = STORE_NC(hpp, ohp, swtrp->ref);
+            else
+                *szp += REF_THING_SIZE;
+
+            ASSERT(!swtrp->set);
+
+            /* info about dirty schedulers... */
+            msg = NIL;
+            for (ix = no_dirty_scheds-1; ix >= 0; ix--) {
+                msg = erts_bld_cons(hpp, szp,
+                                    erts_bld_tuple(hpp, szp, 3,
+                                                   make_small(ix+soffset),
+                                                   erts_bld_uint64(hpp, szp,
+                                                                   dswt[ix].working),
+                                                   erts_bld_uint64(hpp, szp,
+                                                                   dswt[ix].total)),
+                                    msg);
+            }
+            /* info about this scheduler... */
+            msg = erts_bld_cons(hpp, szp,
+                                erts_bld_tuple(hpp, szp, 3,
+                                               make_small(esdp->no),
+                                               erts_bld_uint64(hpp, szp, working),
+                                               erts_bld_uint64(hpp, szp, total)),
+                                msg);
+
+            msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg);
+
+            if (hpp)
+                break;
+
+            mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
+            szp = NULL;
+            hpp = &hp;
+        }
 
-	mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
-	szp = NULL;
-	hpp = &hp;
+        erts_free(ERTS_ALC_T_TMP, dswt);
+    }
+    else
+#endif
+    {
+        /* Reply with info about this scheduler only... */
+
+        while (1) {
+            if (hpp)
+                ref_copy = STORE_NC(hpp, ohp, swtrp->ref);
+            else
+                *szp += REF_THING_SIZE;
+
+            if (swtrp->set)
+                msg = ref_copy;
+            else {
+                msg = (!esdp->sched_wall_time.enabled
+                       ? am_undefined
+                       : erts_bld_tuple(hpp, szp, 3,
+                                        make_small(esdp->no),
+                                        erts_bld_uint64(hpp, szp, working),
+                                        erts_bld_uint64(hpp, szp, total)));
+
+                msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg);
+            }
+            if (hpp)
+                break;
+
+            mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
+            szp = NULL;
+            hpp = &hp;
+        }
     }
 
     erts_queue_message(rp, rp_locks, mp, msg, am_system);
@@ -1159,7 +1378,8 @@ reply_sched_wall_time(void *vswtrp)
 }
 
 Eterm
-erts_sched_wall_time_request(Process *c_p, int set, int enable)
+erts_sched_wall_time_request(Process *c_p, int set, int enable,
+                             int want_dirty_cpu, int want_dirty_io)
 {
     ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
     Eterm ref;
@@ -1181,6 +1401,10 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable)
     swtrp->proc = c_p;
     swtrp->ref = STORE_NC(&hp, NULL, ref);
     swtrp->req_sched = esdp->no;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    swtrp->want_dirty_cpu = want_dirty_cpu;
+    swtrp->want_dirty_io = want_dirty_io;
+#endif
     erts_smp_atomic32_init_nob(&swtrp->refc,
 			       (erts_aint32_t) erts_no_schedulers);
 
@@ -3106,8 +3330,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
     tse_wait:
 
-	if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && thr_prgr_active != working)
-	    sched_wall_time_change(esdp, thr_prgr_active);
+	if (ERTS_SCHEDULER_IS_DIRTY(esdp))
+	    dirty_sched_wall_time_change(esdp, working = 0);
+        else if (thr_prgr_active != working)
+	    sched_wall_time_change(esdp, working = thr_prgr_active);
 
 	while (1) {
 	    ErtsMonotonicTime current_time = 0;
@@ -3218,10 +3444,12 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
                                             (ERTS_SSI_FLG_SUSPENDED
                                              | ERTS_SSI_FLG_MSB_EXEC));
 
-	if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
+	if (ERTS_SCHEDULER_IS_DIRTY(esdp))
+	    dirty_sched_wall_time_change(esdp, working = 1);
+        else if (!thr_prgr_active) {
 	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
 	    sched_wall_time_change(esdp, 1);
-	}
+        }
 
 	erts_smp_runq_lock(rq);
 	sched_active(esdp->no, rq);
@@ -5712,7 +5940,8 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 		    ErtsSchedulerSleepInfo* ssi,
 		    ErtsRunQueue* runq,
 		    char** daww_ptr, size_t daww_sz,
-		    Process *shadow_proc)
+		    Process *shadow_proc,
+                    Uint64 time_stamp)
 {
     esdp->timer_wheel = NULL;
 #ifdef ERTS_SMP
@@ -5794,7 +6023,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 
     esdp->reductions = 0;
 
-    init_sched_wall_time(&esdp->sched_wall_time);
+    init_sched_wall_time(esdp, time_stamp);
     erts_port_task_handle_init(&esdp->nosuspend_port_task_handle);
 }
 
@@ -6008,17 +6237,18 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 
     erts_aligned_scheduler_data = 
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
-					   n*sizeof(ErtsAlignedSchedulerData));					   
+					   n*sizeof(ErtsAlignedSchedulerData));
 
     for (ix = 0; ix < n; ix++) {
 	ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix);
 	init_scheduler_data(esdp, ix+1, ERTS_SCHED_SLEEP_INFO_IX(ix),
 			    ERTS_RUNQ_IX(ix), &daww_ptr, daww_sz,
-			    NULL);
+			    NULL, 0);
     }
 
 #ifdef ERTS_DIRTY_SCHEDULERS
     {
+        Uint64 ts = sched_wall_time_ts();
 	int dirty_scheds = no_dirty_cpu_schedulers + no_dirty_io_schedulers;
 	int adspix = 0;
 	ErtsAlignedDirtyShadowProcess *adsp =
@@ -6038,13 +6268,13 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	    ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
 	    init_scheduler_data(esdp, ix+1, ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix),
 				ERTS_DIRTY_CPU_RUNQ, NULL, 0,
-				&adsp[adspix++].dsp);
+				&adsp[adspix++].dsp, ts);
 	}
 	for (ix = 0; ix < no_dirty_io_schedulers; ix++) {
 	    ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
 	    init_scheduler_data(esdp, ix+1, ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix),
 				ERTS_DIRTY_IO_RUNQ, NULL, 0,
-				&adsp[adspix++].dsp);
+				&adsp[adspix++].dsp, ts);
 	}
     }
 #endif
@@ -7506,8 +7736,10 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 #endif
 
-    if (sched_type != ERTS_SCHED_NORMAL)
+    if (sched_type != ERTS_SCHED_NORMAL) {
 	erts_smp_runq_unlock(esdp->run_queue);
+        dirty_sched_wall_time_change(esdp, 0);
+    }
     else {
         if (no != 1)
             evacuate_run_queue(esdp->run_queue, &sbp);
@@ -7518,8 +7750,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 	if (erts_system_profile_flags.scheduler)
 	    profile_scheduler(make_small(esdp->no), am_inactive);
-
     }
+
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 
     flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED);
@@ -7797,15 +8029,17 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
     ASSERT(curr_online);
 
-    if (sched_type == ERTS_SCHED_NORMAL) {
-	(void) erts_get_monotonic_time(esdp);
-	if (erts_system_profile_flags.scheduler)
-	    profile_scheduler(make_small(esdp->no), am_active);
+    if (sched_type != ERTS_SCHED_NORMAL)
+        dirty_sched_wall_time_change(esdp, 1);
+    else {
+        (void) erts_get_monotonic_time(esdp);
+        if (erts_system_profile_flags.scheduler)
+            profile_scheduler(make_small(esdp->no), am_active);
 
-	if (!thr_prgr_active) {
-	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
-	    sched_wall_time_change(esdp, 1);
-	}
+        if (!thr_prgr_active) {
+            erts_thr_progress_active(esdp, thr_prgr_active = 1);
+            sched_wall_time_change(esdp, 1);
+        }
     }
 
     erts_smp_runq_lock(esdp->run_queue);
@@ -8518,6 +8752,8 @@ sched_dirty_cpu_thread_func(void *vesdp)
     callbacks.wait = NULL;
     callbacks.finalize_wait = NULL;
 
+    dirty_sched_wall_time_change(esdp, 1);
+
     esdp->thr_id += erts_no_schedulers;
 
     erts_msacc_init_thread("dirty_cpu_scheduler", no, 0);
@@ -8565,6 +8801,8 @@ sched_dirty_io_thread_func(void *vesdp)
     callbacks.wait = NULL;
     callbacks.finalize_wait = NULL;
 
+    dirty_sched_wall_time_change(esdp, 1);
+
     esdp->thr_id += erts_no_schedulers + erts_no_dirty_cpu_schedulers;
 
     erts_msacc_init_thread("dirty_io_scheduler", no, 0);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index fc2055332b..ce0989883c 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -550,13 +550,15 @@ do {								\
 } while (0)
 
 typedef struct {
-    int need; /* "+sbu true" or scheduler_wall_time enabled */
+    union {
+        erts_atomic32_t mod; /* on dirty schedulers */
+        int need; /* "+sbu true" or scheduler_wall_time enabled */
+    } u;
     int enabled;
     Uint64 start;
     struct {
 	Uint64 total;
 	Uint64 start;
-	int currently;
     } working;
 } ErtsSchedWallTime;
 
@@ -1598,7 +1600,8 @@ void erts_init_scheduling(int, int
 void erts_execute_dirty_system_task(Process *c_p);
 #endif
 int erts_set_gc_state(Process *c_p, int enable);
-Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
+Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable,
+                                   int dirty_cpu, int want_dirty_io);
 Eterm erts_system_check_request(Process *c_p);
 Eterm erts_gc_info_request(Process *c_p);
 Uint64 erts_get_proc_interval(void);
diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl
index a1f12ba93c..f51244485b 100644
--- a/erts/emulator/test/statistics_SUITE.erl
+++ b/erts/emulator/test/statistics_SUITE.erl
@@ -28,6 +28,8 @@
 	 runtime_update/1, runtime_diff/1,
 	 run_queue_one/1,
 	 scheduler_wall_time/1,
+         scheduler_wall_time_all/1,
+         msb_scheduler_wall_time/1,
 	 reductions/1, reductions_big/1, garbage_collection/1, io/1,
 	 badarg/1, run_queues_lengths_active_tasks/1, msacc/1]).
 
@@ -43,7 +45,9 @@ suite() ->
 
 all() -> 
     [{group, wall_clock}, {group, runtime}, reductions,
-     reductions_big, {group, run_queue}, scheduler_wall_time,
+     reductions_big, {group, run_queue},
+     scheduler_wall_time, scheduler_wall_time_all,
+     msb_scheduler_wall_time,
      garbage_collection, io, badarg,
      run_queues_lengths_active_tasks,
      msacc].
@@ -271,35 +275,64 @@ hog_iter(0, Mon) ->
 
 %% Tests that statistics(scheduler_wall_time) works as intended
 scheduler_wall_time(Config) when is_list(Config) ->
+    scheduler_wall_time_test(scheduler_wall_time).
+
+%% Tests that statistics(scheduler_wall_time_all) works as intended
+scheduler_wall_time_all(Config) when is_list(Config) ->
+    scheduler_wall_time_test(scheduler_wall_time_all).
+
+scheduler_wall_time_test(Type) ->
     %% Should return undefined if system_flag is not turned on yet
-    undefined = statistics(scheduler_wall_time),
+    undefined = statistics(Type),
     %% Turn on statistics
     false = erlang:system_flag(scheduler_wall_time, true),
     try
         Schedulers = erlang:system_info(schedulers_online),
+        DirtyCPUSchedulers = erlang:system_info(dirty_cpu_schedulers_online),
+        DirtyIOSchedulers = erlang:system_info(dirty_io_schedulers),
+        TotLoadSchedulers = case Type of
+                                scheduler_wall_time_all ->
+                                    Schedulers + DirtyCPUSchedulers + DirtyIOSchedulers;
+                                scheduler_wall_time ->
+                                    Schedulers + DirtyCPUSchedulers
+                        end,
+                            
         %% Let testserver and everyone else finish their work
         timer:sleep(1500),
         %% Empty load
-        EmptyLoad = get_load(),
+        EmptyLoad = get_load(Type),
         {false, _} = {lists:any(fun(Load) -> Load > 50 end, EmptyLoad),EmptyLoad},
         MeMySelfAndI = self(),
         StartHog = fun() ->
-                           Pid = spawn(?MODULE, hog, [self()]),
+                           Pid = spawn_link(?MODULE, hog, [self()]),
                            receive hog_started -> MeMySelfAndI ! go end,
                            Pid
                    end,
+        StartDirtyHog = fun(Func) ->
+                                F = fun () ->
+                                            erts_debug:Func(alive_waitexiting,
+                                                            MeMySelfAndI)
+                                    end,
+                                Pid = spawn_link(F),
+                                receive {alive, Pid} -> ok end,
+                                Pid
+                        end,
         P1 = StartHog(),
         %% Max on one, the other schedulers empty (hopefully)
         %% Be generous the process can jump between schedulers
         %% which is ok and we don't want the test to fail for wrong reasons
-        _L1 = [S1Load|EmptyScheds1] = get_load(),
+        _L1 = [S1Load|EmptyScheds1] = get_load(Type),
         {true,_}  = {S1Load > 50,S1Load},
         {false,_} = {lists:any(fun(Load) -> Load > 50 end, EmptyScheds1),EmptyScheds1},
         {true,_}  = {lists:sum(EmptyScheds1) < 60,EmptyScheds1},
 
         %% 50% load
         HalfHogs = [StartHog() || _ <- lists:seq(1, (Schedulers-1) div 2)],
-        HalfLoad = lists:sum(get_load()) div Schedulers,
+        HalfDirtyCPUHogs = [StartDirtyHog(dirty_cpu)
+                            || _ <- lists:seq(1, DirtyCPUSchedulers div 2)],
+        HalfDirtyIOHogs = [StartDirtyHog(dirty_io)
+                           || _ <- lists:seq(1, DirtyIOSchedulers div 2)],
+        HalfLoad = lists:sum(get_load(Type)) div TotLoadSchedulers,
         if Schedulers < 2, HalfLoad > 80 -> ok; %% Ok only one scheduler online and one hog
            %% We want roughly 50% load
            HalfLoad > 40, HalfLoad < 60 -> ok;
@@ -308,23 +341,30 @@ scheduler_wall_time(Config) when is_list(Config) ->
 
         %% 100% load
         LastHogs = [StartHog() || _ <- lists:seq(1, Schedulers div 2)],
-        FullScheds = get_load(),
+        LastDirtyCPUHogs = [StartDirtyHog(dirty_cpu)
+                            || _ <- lists:seq(1, DirtyCPUSchedulers div 2)],
+        LastDirtyIOHogs = [StartDirtyHog(dirty_io)
+                           || _ <- lists:seq(1, DirtyIOSchedulers div 2)],
+        FullScheds = get_load(Type),
         {false,_} = {lists:any(fun(Load) -> Load < 80 end, FullScheds),FullScheds},
-        FullLoad = lists:sum(FullScheds) div Schedulers,
+        FullLoad = lists:sum(FullScheds) div TotLoadSchedulers,
         if FullLoad > 90 -> ok;
            true -> exit({fullload, FullLoad})
         end,
 
 	KillHog = fun (HP) ->
 			  HPM = erlang:monitor(process, HP),
+                          unlink(HP),
 			  exit(HP, kill),
 			  receive
 			      {'DOWN', HPM, process, HP, killed} ->
 				  ok
 			  end
 		  end,
-        [KillHog(Pid) || Pid <- [P1|HalfHogs++LastHogs]],
-        AfterLoad = get_load(),
+        [KillHog(Pid) || Pid <- [P1|HalfHogs++HalfDirtyCPUHogs++HalfDirtyIOHogs
+                                 ++LastHogs++LastDirtyCPUHogs++LastDirtyIOHogs]],
+        receive after 2000 -> ok end, %% Give dirty schedulers time to complete...
+        AfterLoad = get_load(Type),
 	io:format("AfterLoad=~p~n", [AfterLoad]),
         {false,_} = {lists:any(fun(Load) -> Load > 25 end, AfterLoad),AfterLoad},
         true = erlang:system_flag(scheduler_wall_time, false)
@@ -332,16 +372,81 @@ scheduler_wall_time(Config) when is_list(Config) ->
         erlang:system_flag(scheduler_wall_time, false)
     end.
 
-get_load() ->
-    Start = erlang:statistics(scheduler_wall_time),
+get_load(Type) ->
+    Start = erlang:statistics(Type),
     timer:sleep(1500),
-    End = erlang:statistics(scheduler_wall_time),
+    End = erlang:statistics(Type),
     lists:reverse(lists:sort(load_percentage(lists:sort(Start),lists:sort(End)))).
 
 load_percentage([{Id, WN, TN}|Ss], [{Id, WP, TP}|Ps]) ->
     [100*(WN-WP) div (TN-TP)|load_percentage(Ss, Ps)];
 load_percentage([], []) -> [].
 
+count(0) ->
+    ok;
+count(N) ->
+    count(N-1).
+
+msb_swt_hog(true) ->
+    count(1000000),
+    erts_debug:dirty_cpu(wait, 10),
+    erts_debug:dirty_io(wait, 10),
+    msb_swt_hog(true);
+msb_swt_hog(false) ->
+    count(1000000),
+    msb_swt_hog(false).
+
+msb_scheduler_wall_time(Config) ->
+    erlang:system_flag(scheduler_wall_time, true),
+    Dirty = erlang:system_info(dirty_cpu_schedulers) /= 0,
+    Hogs = lists:map(fun (_) ->
+                             spawn_opt(fun () ->
+                                               msb_swt_hog(Dirty)
+                                       end, [{priority,low}, link, monitor])
+                     end, lists:seq(1,10)),
+    erlang:system_flag(multi_scheduling, block),
+    try
+        SWT1 = lists:sort(statistics(scheduler_wall_time_all)),
+        %% io:format("SWT1 = ~p~n", [SWT1]),
+        receive after 4000 -> ok end,
+        SWT2 = lists:sort(statistics(scheduler_wall_time_all)),
+        %% io:format("SWT2 = ~p~n", [SWT2]),
+        SWT = lists:zip(SWT1, SWT2),
+        io:format("SU = ~p~n", [lists:map(fun({{I, A0, T0}, {I, A1, T1}}) ->
+                                                  {I, (A1 - A0)/(T1 - T0)} end,
+                                         SWT)]),
+        {A, T} = lists:foldl(fun({{_, A0, T0}, {_, A1, T1}}, {Ai,Ti}) ->
+                                     {Ai + (A1 - A0), Ti + (T1 - T0)}
+                             end,
+                             {0, 0},
+                             SWT),
+        TSU = A/T,
+        WSU = ((TSU * (erlang:system_info(schedulers)
+                       + erlang:system_info(dirty_cpu_schedulers)
+                       + erlang:system_info(dirty_io_schedulers)))
+               / 1),
+        %% Weighted scheduler utilization should be
+        %% very close to 1.0, i.e., we execute the
+        %% same time as one thread executing all
+        %% the time...
+        io:format("WSU = ~p~n", [WSU]),
+        true = 0.9 < WSU andalso WSU < 1.1,
+        ok
+    after
+        erlang:system_flag(multi_scheduling, unblock),
+        erlang:system_flag(scheduler_wall_time, false),
+        lists:foreach(fun ({HP, _HM}) ->
+                              unlink(HP),
+                              exit(HP, kill)
+                      end, Hogs),
+        lists:foreach(fun ({HP, HM}) ->
+                              receive
+                                  {'DOWN', HM, process, HP, _} ->
+                                      ok
+                              end
+                      end, Hogs),
+        ok
+    end.
 
 %% Tests that statistics(garbage_collection) is callable.
 %% It is not clear how to test anything more.
-- 
cgit v1.2.3


From a0a5bc55ac2805e6411ae5faa01b3ad532738345 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 17 Jan 2017 14:38:49 +0100
Subject: Remove debug printout and unnecessary GC

---
 erts/emulator/hipe/hipe_native_bif.c | 10 ----------
 1 file changed, 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index 9c03b3811c..801cef2e56 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -226,11 +226,6 @@ void hipe_handle_exception(Process *c_p)
 
     ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */
 
-    if (c_p->mbuf) {
-	erts_printf("%s line %u: p==%p, p->mbuf==%p\n", __FUNCTION__, __LINE__, c_p, c_p->mbuf);
-	/* erts_garbage_collect(c_p, 0, NULL, 0); */
-    }
-
     /*
      * Check if we have an arglist for the top level call. If so, this
      * is encoded in Value, so we have to dig out the real Value as well
@@ -259,11 +254,6 @@ void hipe_handle_exception(Process *c_p)
     /* Synthesized to avoid having to generate code for it. */
     c_p->def_arg_reg[0] = exception_tag[GET_EXC_CLASS(c_p->freason)];
 
-    if (c_p->mbuf) {
-	/* erts_printf("%s line %u: p==%p, p->mbuf==%p, p->lastbif==%p\n", __FUNCTION__, __LINE__, c_p, c_p->mbuf, c_p->hipe.lastbif); */
-	erts_garbage_collect(c_p, 0, NULL, 0);
-    }
-
     hipe_find_handler(c_p);
 }
 
-- 
cgit v1.2.3


From c3fe109d6f631fde600676db7e27209fe12911c1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 17 Jan 2017 15:12:17 +0100
Subject: erts: Fix zlib crash on mac for inflateGetDictionary

Work around broken build system by checking
actual zlib version in runtime.
---
 erts/emulator/drivers/common/zlib_drv.c | 46 ++++++++++++++++++++++++++-------
 1 file changed, 37 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/zlib_drv.c b/erts/emulator/drivers/common/zlib_drv.c
index 066cf87c9d..e8afddb01b 100644
--- a/erts/emulator/drivers/common/zlib_drv.c
+++ b/erts/emulator/drivers/common/zlib_drv.c
@@ -252,9 +252,9 @@ static int zlib_output(ZLibData* d)
     return zlib_output_init(d);
 }
 
-#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY
 static int zlib_inflate_get_dictionary(ZLibData* d)
 {
+#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY
     ErlDrvBinary* dbin = driver_alloc_binary(INFL_DICT_SZ);
     uInt dlen = 0;
     int res = inflateGetDictionary(&d->s, (unsigned char*)dbin->orig_bytes, &dlen);
@@ -263,8 +263,11 @@ static int zlib_inflate_get_dictionary(ZLibData* d)
     }
     driver_free_binary(dbin);
     return res;
-}
+#else
+    abort(); /* never called, just to silence 'unresolved symbol'
+                for non-optimizing compiler */
 #endif
+}
 
 static int zlib_inflate(ZLibData* d, int flush)
 {
@@ -448,10 +451,35 @@ static void zlib_free(void* data, void* addr)
     driver_free(addr);
 }
 
+#if defined(__APPLE__) && defined(__MACH__) && defined(HAVE_ZLIB_INFLATEGETDICTIONARY)
+
+/* Work around broken build system with runtime version test */
+static int have_inflateGetDictionary;
+
+static int zlib_init()
+{
+    unsigned int v[4] = {0, 0, 0, 0};
+    unsigned hexver;
+
+    sscanf(zlibVersion(), "%u.%u.%u.%u", &v[0], &v[1], &v[2], &v[3]);
+
+    hexver = (v[0] << (8*3)) | (v[1] << (8*2)) | (v[2] << (8)) | v[3];
+
+    have_inflateGetDictionary = (hexver >= 0x1020701); /* 1.2.7.1 */
+
+    return 0;
+}
+#else /* trust configure got it right */
+#  ifdef HAVE_ZLIB_INFLATEGETDICTIONARY
+#    define have_inflateGetDictionary 1
+#  else
+#    define have_inflateGetDictionary 0
+#  endif
 static int zlib_init()
 {
     return 0;
 }
+#endif
 
 static ErlDrvData zlib_start(ErlDrvPort port, char* buf)
 {
@@ -605,14 +633,14 @@ static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *bu
 	return zlib_return(res, rbuf, rlen);
 
     case INFLATE_GETDICT:
-#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY
-        if (d->state != ST_INFLATE) goto badarg;
-        res = zlib_inflate_get_dictionary(d);
+        if (have_inflateGetDictionary) {
+            if (d->state != ST_INFLATE) goto badarg;
+            res = zlib_inflate_get_dictionary(d);
+        } else {
+            errno = ENOTSUP;
+            res = Z_ERRNO;
+        }
         return zlib_return(res, rbuf, rlen);
-#else
-        errno = ENOTSUP;
-        return zlib_return(Z_ERRNO, rbuf, rlen);
-#endif
 
     case INFLATE_SYNC:
 	if (d->state != ST_INFLATE) goto badarg;
-- 
cgit v1.2.3


From 505305a4b6a6372921808c4e5ec56db51b6a308b Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 18 Jan 2017 13:30:20 +0100
Subject: Remove double check of NifExport when checking process code

---
 erts/emulator/beam/beam_bif_load.c | 5 -----
 1 file changed, 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index c8dde8caf8..a5891a0ec2 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1117,11 +1117,6 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls)
  
     *redsp += 1;
 
-    if (erts_check_nif_export_in_area(rp, mod_start, mod_size))
-	return am_true;
-
-    *redsp += 1;
-
     if (erts_check_nif_export_in_area(rp, mod_start, mod_size))
 	return am_true;
 
-- 
cgit v1.2.3


From bfcf88311828b93a833ce96ad1a518b8eca08552 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 18 Jan 2017 14:40:25 +0100
Subject: Change exception for enif_schedule_nif() with dirty flags

---
 erts/emulator/beam/erl_nif.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index b860759fa2..af3ca3afa3 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2506,10 +2506,13 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags,
     if (flags == 0)
 	result = schedule(env, execute_nif, fp, proc->current->module,
 			  fun_name_atom, argc, argv);
+    else if (!(flags & ~(ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND))) {
 #ifdef ERTS_DIRTY_SCHEDULERS
-    else if (!(flags & ~(ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND)))
 	result = schedule_dirty_nif(env, flags, fp, fun_name_atom, argc, argv);
+#else
+        result = enif_raise_exception(env, am_notsup);
 #endif
+    }
     else
 	result = enif_make_badarg(env);
 
-- 
cgit v1.2.3


From 40c82769def0cfa59e75669ff1c6fc4abcecd764 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 15 Dec 2016 18:07:13 +0100
Subject: erts: Handle SIGTERM via signal service instead

---
 erts/emulator/beam/atom.names |  1 +
 erts/emulator/sys/unix/sys.c  | 28 ++++------------------------
 2 files changed, 5 insertions(+), 24 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 363b17e27c..52a3413796 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -592,6 +592,7 @@ atom set_tcw_fake
 atom separate
 atom shared
 atom sighup
+atom sigterm
 atom silent
 atom size
 atom sl_alloc
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index a099662b16..cb9ed2de76 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -679,26 +679,6 @@ static RETSIGTYPE request_break(int signum)
 #endif
 }
 
-static void stop_requested(void) {
-    Process* p = NULL;
-    Eterm msg, *hp;
-    ErtsProcLocks locks = 0;
-    ErlOffHeap *ohp;
-    Eterm id = erts_whereis_name_to_id(NULL, am_init);
-
-    if ((p = (erts_pid2proc_opt(NULL, 0, id, 0, ERTS_P2P_FLG_INC_REFC))) != NULL) {
-        ErtsMessage *msgp = erts_alloc_message_heap(p, &locks, 3, &hp, &ohp);
-
-        /* init ! {stop,stop} */
-        msg = TUPLE2(hp, am_stop, am_stop);
-        erts_queue_message(p, locks, msgp, msg, am_system);
-
-        if (locks)
-            erts_smp_proc_unlock(p, locks);
-        erts_proc_dec_refc(p);
-    }
-}
-
 #if (defined(SIG_SIGSET) || defined(SIG_SIGNAL))
 static RETSIGTYPE request_stop(void)
 #else
@@ -706,9 +686,9 @@ static RETSIGTYPE request_stop(int signum)
 #endif
 {
 #ifdef ERTS_SMP
-    smp_sig_notify('S');
+    smp_sig_notify('T');
 #else
-    stop_requested();
+    signal_notify_requested(am_sigterm);
 #endif
 }
 
@@ -1330,8 +1310,8 @@ signal_dispatcher_thread_func(void *unused)
             case 'H': /* SIGHUP */
                 signal_notify_requested(am_sighup);
                 break;
-	    case 'S': /* SIGTERM */
-		stop_requested();
+	    case 'T': /* SIGTERM */
+                signal_notify_requested(am_sigterm);
 		break;
 	    case 'I': /* SIGINT */
 		break_requested();
-- 
cgit v1.2.3


From 2d3bf84d8167b50728b0a5411a4e2dfa71d52c10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 16 Dec 2016 12:24:13 +0100
Subject: erts: Handle SIGUSR1 via signal service instead

---
 erts/emulator/beam/atom.names       |  1 +
 erts/emulator/beam/sys.h            | 14 -------------
 erts/emulator/sys/common/erl_poll.c |  1 -
 erts/emulator/sys/unix/sys.c        | 42 ++++---------------------------------
 4 files changed, 5 insertions(+), 53 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 52a3413796..668abfaaee 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -593,6 +593,7 @@ atom separate
 atom shared
 atom sighup
 atom sigterm
+atom sigusr1
 atom silent
 atom size
 atom sl_alloc
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 4b3ac594a0..ceea794cc4 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -487,20 +487,6 @@ extern volatile int erts_break_requested;
 void erts_do_break_handling(void);
 #endif
 
-#ifdef ERTS_WANT_GOT_SIGUSR1
-#  ifndef UNIX
-#    define ERTS_GOT_SIGUSR1 0
-#  else
-#    ifdef ERTS_SMP
-extern erts_smp_atomic32_t erts_got_sigusr1;
-#      define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read_mb(&erts_got_sigusr1))
-#    else
-extern volatile int erts_got_sigusr1;
-#      define ERTS_GOT_SIGUSR1 erts_got_sigusr1
-#    endif
-#  endif
-#endif
-
 #ifdef ERTS_SMP
 extern erts_smp_atomic32_t erts_writing_erl_crash_dump;
 extern erts_tsd_key_t erts_is_crash_dumping_key;
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index b8a28bcc18..5e7ae8953a 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -51,7 +51,6 @@
 #ifndef WANT_NONBLOCKING
 #  define WANT_NONBLOCKING
 #endif
-#define ERTS_WANT_GOT_SIGUSR1
 
 #include "erl_poll.h"
 #if ERTS_POLL_USE_KQUEUE
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index cb9ed2de76..de9c1b2ca0 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -50,7 +50,6 @@
 #endif
 
 #define ERTS_WANT_BREAK_HANDLING
-#define ERTS_WANT_GOT_SIGUSR1
 #define WANT_NONBLOCKING    /* must define this to pull in defs from sys.h */
 #include "sys.h"
 #include "erl_thr_progress.h"
@@ -96,18 +95,10 @@ static int debug_log = 0;
 #endif
 
 #ifdef ERTS_SMP
-erts_smp_atomic32_t erts_got_sigusr1;
-#define ERTS_SET_GOT_SIGUSR1 \
-  erts_smp_atomic32_set_mb(&erts_got_sigusr1, 1)
-#define ERTS_UNSET_GOT_SIGUSR1 \
-  erts_smp_atomic32_set_mb(&erts_got_sigusr1, 0)
 static erts_smp_atomic32_t have_prepared_crash_dump;
 #define ERTS_PREPARED_CRASH_DUMP \
   ((int) erts_smp_atomic32_xchg_nob(&have_prepared_crash_dump, 1))
 #else
-volatile int erts_got_sigusr1;
-#define ERTS_SET_GOT_SIGUSR1 (erts_got_sigusr1 = 1)
-#define ERTS_UNSET_GOT_SIGUSR1 (erts_got_sigusr1 = 0)
 static volatile int have_prepared_crash_dump;
 #define ERTS_PREPARED_CRASH_DUMP \
   (have_prepared_crash_dump++)
@@ -430,11 +421,9 @@ erts_sys_pre_init(void)
 
 #ifdef ERTS_SMP
     erts_smp_atomic32_init_nob(&erts_break_requested, 0);
-    erts_smp_atomic32_init_nob(&erts_got_sigusr1, 0);
     erts_smp_atomic32_init_nob(&have_prepared_crash_dump, 0);
 #else
     erts_break_requested = 0;
-    erts_got_sigusr1 = 0;
     have_prepared_crash_dump = 0;
 #endif
 
@@ -692,29 +681,6 @@ static RETSIGTYPE request_stop(int signum)
 #endif
 }
 
-
-static ERTS_INLINE void
-sigusr1_exit(void)
-{
-    char env[21]; /* enough to hold any 64-bit integer */
-    size_t envsz;
-    int i, secs = -1;
-
-    /* We do this at interrupt level, since the main reason for
-     * wanting to generate a crash dump in this way is that the emulator
-     * is hung somewhere, so it won't be able to poll any flag we set here.
-     */
-    ERTS_SET_GOT_SIGUSR1;
-
-    envsz = sizeof(env);
-    if ((i = erts_sys_getenv_raw("ERL_CRASH_DUMP_SECONDS", env, &envsz)) >= 0) {
-	secs = i != 0 ? 0 : atoi(env);
-    }
-
-    prepare_crash_dump(secs);
-    erts_exit(ERTS_DUMP_EXIT, "Received SIGUSR1\n");
-}
-
 #ifdef ETHR_UNUSABLE_SIGUSRX
 #warning "Unusable SIGUSR1 & SIGUSR2. Disabling use of these signals"
 
@@ -744,7 +710,7 @@ static RETSIGTYPE user_signal1(int signum)
 #ifdef ERTS_SMP
    smp_sig_notify('1');
 #else
-   sigusr1_exit();
+   signal_notify_requested(am_sigusr1);
 #endif
 }
 
@@ -1313,15 +1279,15 @@ signal_dispatcher_thread_func(void *unused)
 	    case 'T': /* SIGTERM */
                 signal_notify_requested(am_sigterm);
 		break;
+	    case '1': /* SIGUSR1 */
+                signal_notify_requested(am_sigusr1);
+		break;
 	    case 'I': /* SIGINT */
 		break_requested();
 		break;
 	    case 'Q': /* SIGQUIT */
 		quit_requested();
 		break;
-	    case '1': /* SIGUSR1 */
-		sigusr1_exit();
-		break;
 	    default:
 		erts_exit(ERTS_ABORT_EXIT,
 			 "signal-dispatcher thread received unknown "
-- 
cgit v1.2.3


From 6bebb90f7c94e1aa95aff06fc5c40dd07696edc2 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 19 Jan 2017 16:28:35 +0100
Subject: Introduce erts_proc_lookup_inc_refc()

---
 erts/emulator/beam/erl_process_lock.c | 35 +++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_process_lock.h |  2 ++
 2 files changed, 37 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index 180df229eb..a93f1755c8 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -1006,6 +1006,41 @@ erts_pid2proc_opt(Process *c_p,
     return proc;
 }
 
+static ERTS_INLINE
+Process *proc_lookup_inc_refc(Eterm pid, int allow_exit)
+{
+    Process *proc;
+#ifdef ERTS_SMP
+    ErtsThrPrgrDelayHandle dhndl;
+
+    dhndl = erts_thr_progress_unmanaged_delay();
+#endif
+
+    proc = erts_proc_lookup_raw(pid);
+    if (proc) {
+        if (!allow_exit && ERTS_PROC_IS_EXITING(proc))
+            proc = NULL;
+        else
+            erts_proc_inc_refc(proc);
+    }
+
+#ifdef ERTS_SMP
+    erts_thr_progress_unmanaged_continue(dhndl);
+#endif
+
+    return proc;
+}
+
+Process *erts_proc_lookup_inc_refc(Eterm pid)
+{
+    return proc_lookup_inc_refc(pid, 0);
+}
+
+Process *erts_proc_lookup_raw_inc_refc(Eterm pid)
+{
+    return proc_lookup_inc_refc(pid, 1);
+}
+
 void
 erts_proc_lock_init(Process *p)
 {
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 2cccf0697a..773529384f 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -940,6 +940,8 @@ void erts_proc_safelock(Process *a_proc,
 #define erts_pid2proc(PROC, HL, PID, NL) \
   erts_pid2proc_opt((PROC), (HL), (PID), (NL), 0)
 
+Process *erts_proc_lookup_inc_refc(Eterm pid);
+Process *erts_proc_lookup_raw_inc_refc(Eterm pid);
 
 ERTS_GLB_INLINE Process *erts_pix2proc(int ix);
 ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid);
-- 
cgit v1.2.3


From eaecd838ebc3410efcbe0f3b1717543d6a4650b0 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 19 Jan 2017 17:24:14 +0100
Subject: Fix unused warning

---
 erts/emulator/beam/erl_process.c | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index f80ebdf31b..641a8fb3e8 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -127,11 +127,13 @@ runq_got_work_to_execute_flags(Uint32 flags)
     return !ERTS_IS_RUNQ_EMPTY_FLGS(flags);
 }
 
+#ifdef ERTS_SMP
 static ERTS_INLINE int
 runq_got_work_to_execute(ErtsRunQueue *rq)
 {
     return runq_got_work_to_execute_flags(ERTS_RUNQ_FLGS_GET_NOB(rq));
 }
+#endif
 
 #undef RUNQ_READ_RQ
 #undef RUNQ_SET_RQ
-- 
cgit v1.2.3


From 1623d7a85908221a118356bec64693901405659d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 20 Jan 2017 16:28:42 +0100
Subject: erts: Fix thread suspend in crashdump

* move signal handler setup
---
 erts/emulator/beam/break.c    | 2 ++
 erts/emulator/beam/erl_init.c | 1 -
 erts/emulator/beam/sys.h      | 3 +++
 erts/emulator/sys/unix/sys.c  | 3 +--
 4 files changed, 6 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index dfbe1ced47..61907cc7ff 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -717,6 +717,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
      * We have to be very very careful when doing this as the schedulers
      * could be anywhere.
      */
+    sys_init_suspend_handler();
+
     for (i = 0; i < erts_no_schedulers; i++) {
         erts_tid_t tid = ERTS_SCHEDULER_IX(i)->tid;
         if (!erts_equal_tids(tid,erts_thr_self()))
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 2fd97208cc..070cc3f2d0 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -2220,7 +2220,6 @@ erl_start(int argc, char **argv)
 	init_break_handler();
     if (replace_intr)
 	erts_replace_intr();
-    sys_init_suspend_handler();
 #endif
 
     boot_argc = argc - i;  /* Number of arguments to init */
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 7740dd4373..41a7ff4c16 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -859,9 +859,12 @@ int erts_sys_unsetenv(char *key);
 char *erts_read_env(char *key);
 void erts_free_read_env(void *value);
 
+#if defined(ERTS_SMP)
 #if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX)
 extern void sys_thr_resume(erts_tid_t tid);
 extern void sys_thr_suspend(erts_tid_t tid);
+#define ERTS_SYS_SUSPEND_SIGNAL SIGUSR2
+#endif
 #endif
 
 /* utils.c */
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 2fc802a2c6..e135dbff99 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -119,9 +119,8 @@ erts_smp_atomic_t sys_misc_mem_sz;
 static void smp_sig_notify(char c);
 static int sig_notify_fds[2] = {-1, -1};
 
-#if !defined(ETHR_UNUSABLE_SIGUSRX) && defined(ERTS_THR_HAVE_SIG_FUNCS)
+#ifdef ERTS_SYS_SUSPEND_SIGNAL
 static int sig_suspend_fds[2] = {-1, -1};
-#define ERTS_SYS_SUSPEND_SIGNAL SIGUSR2
 #endif
 
 #endif
-- 
cgit v1.2.3


From e27119948fc6ab28bea81019720bddaac5b655a7 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 23 Jan 2017 15:13:19 +0100
Subject: erts: Fix binary_to_term for compressed and zlib >=  v1.2.9

Problem: z_stream was incorrectly copied with memcpy
which just happened to work with zlib < v1.2.9.

Solution: Avoid copying z_stream.
---
 erts/emulator/beam/external.c | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 656de7c49a..4491d48683 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1193,6 +1193,7 @@ typedef struct B2TContext_t {
     } u;
 } B2TContext;
 
+static B2TContext* b2t_export_context(Process*, B2TContext* src);
 
 static uLongf binary2term_uncomp_size(byte* data, Sint size)
 {
@@ -1225,7 +1226,7 @@ static uLongf binary2term_uncomp_size(byte* data, Sint size)
 
 static ERTS_INLINE int
 binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size,
-		    B2TContext* ctx)
+		    B2TContext** ctxp, Process* p)
 {
     byte *bytes = data;
     Sint size = data_size;
@@ -1239,8 +1240,8 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size,
     size--;
     if (size < 5 || *bytes != COMPRESSED) {
 	state->extp = bytes;
-        if (ctx)
-	    ctx->state = B2TSizeInit;
+        if (ctxp)
+	    (*ctxp)->state = B2TSizeInit;
     }
     else  {
 	uLongf dest_len = (Uint32) get_int32(bytes+1);
@@ -1257,16 +1258,26 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size,
                 return -1;
 	    }
 	    state->extp = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, dest_len);
-            ctx->reds -= dest_len;
+            if (ctxp)
+                (*ctxp)->reds -= dest_len;
 	}
 	state->exttmp = 1;
-        if (ctx) {
+        if (ctxp) {
+            /*
+             * Start decompression by exporting trap context
+             * so we don't have to deal with deep-copying z_stream.
+             */
+            B2TContext* ctx = b2t_export_context(p, *ctxp);
+            ASSERT(state = &(*ctxp)->b2ts);
+            state = &ctx->b2ts;
+
 	    if (erl_zlib_inflate_start(&ctx->u.uc.stream, bytes, size) != Z_OK)
 		return -1;
 
 	    ctx->u.uc.dbytes = state->extp;
 	    ctx->u.uc.dleft = dest_len;
 	    ctx->state = B2TUncompressChunk;
+            *ctxp = ctx;
         }
 	else {
 	    uLongf dlen = dest_len;
@@ -1308,7 +1319,7 @@ erts_binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size
 {
     Sint res;
 
-    if (binary2term_prepare(state, data, data_size, NULL) < 0 ||
+    if (binary2term_prepare(state, data, data_size, NULL, NULL) < 0 ||
         (res=decoded_size(state->extp, state->extp + state->extsize, 0, NULL)) < 0) {
 
         if (state->exttmp)
@@ -1435,7 +1446,7 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con
             if (ctx->aligned_alloc) {
                 ctx->reds -= bin_size / 8;
             }
-            if (binary2term_prepare(&ctx->b2ts, bytes, bin_size, ctx) < 0) {
+            if (binary2term_prepare(&ctx->b2ts, bytes, bin_size, &ctx, p) < 0) {
 		ctx->state = B2TBadArg;
 	    }
             break;
-- 
cgit v1.2.3


From c5d9b970fb5b3a7143ec8bb2e8194514b5b04e25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 1 Aug 2016 14:35:04 +0200
Subject: erts: Remove broken hash from Erlang

erlang:hash/2 has been deprecated for a while, time to remove it.
---
 erts/emulator/beam/bif.c       |  19 ---
 erts/emulator/beam/bif.tab     |   6 -
 erts/emulator/beam/erl_utils.h |   1 -
 erts/emulator/beam/utils.c     | 264 +----------------------------------------
 4 files changed, 5 insertions(+), 285 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 65c370c55b..a3acf87000 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4723,25 +4723,6 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 
 /**********************************************************************/
 
-BIF_RETTYPE hash_2(BIF_ALIST_2)
-{
-    Uint32 hash;
-    Sint range;
-
-    if (is_not_small(BIF_ARG_2)) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
-    if ((range = signed_val(BIF_ARG_2)) <= 0) {  /* [1..MAX_SMALL] */
-	BIF_ERROR(BIF_P, BADARG);
-    }
-#if defined(ARCH_64)
-    if (range > ((1L << 27) - 1))
-	BIF_ERROR(BIF_P, BADARG);
-#endif
-    hash = make_broken_hash(BIF_ARG_1);
-    BIF_RET(make_small(1 + (hash % range)));   /* [1..range] */
-}
-
 BIF_RETTYPE phash_2(BIF_ALIST_2)
 {
     Uint32 hash;
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 47fdcfa7a4..476c4b9632 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -670,9 +670,3 @@ gcbif erlang:ceil/1
 bif math:floor/1
 bif math:ceil/1
 bif math:fmod/2
-
-#
-# Obsolete
-#
-
-bif erlang:hash/2
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 81800752f0..47289a0af1 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -117,7 +117,6 @@ int erts_fit_in_bits_int32(Sint32);
 int erts_fit_in_bits_uint(Uint);
 Sint erts_list_length(Eterm);
 int erts_is_builtin(Eterm, Eterm, int);
-Uint32 make_broken_hash(Eterm);
 Uint32 block_hash(byte *, unsigned, Uint32);
 Uint32 make_hash2(Eterm);
 Uint32 make_hash(Eterm);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 3fa48da1ec..74a0681dd5 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -700,12 +700,7 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length,
 /* make a hash index from an erlang term */
 
 /*
-** There are three hash functions.
-** make_broken_hash: the one used for backward compatibility
-** is called from the bif erlang:hash/2. Should never be used
-** as it a) hashes only a part of binaries, b) hashes bignums really poorly,
-** c) hashes bignums differently on different endian processors and d) hashes 
-** small integers with different weights on different bytes.
+** There are two hash functions.
 **
 ** make_hash: A hash function that will give the same values for the same
 ** terms regardless of the internal representation. Small integers are 
@@ -1045,6 +1040,10 @@ tail_recur:
     DESTROY_WSTACK(stack);
     return hash;
 
+#undef MAKE_HASH_TUPLE_OP
+#undef MAKE_HASH_TERM_ARRAY_OP
+#undef MAKE_HASH_CDR_PRE_OP
+#undef MAKE_HASH_CDR_POST_OP
 #undef UINT32_HASH_STEP
 #undef UINT32_HASH_RET
 }
@@ -1954,259 +1953,6 @@ make_internal_hash(Eterm term)
 #undef HCONST
 #undef MIX
 
-
-Uint32 make_broken_hash(Eterm term)
-{
-    Uint32 hash = 0;
-    DECLARE_WSTACK(stack);
-    unsigned op;
-tail_recur:
-    op = tag_val_def(term);
-    for (;;) {
-    switch (op) {
-    case NIL_DEF:
-	hash = hash*FUNNY_NUMBER3 + 1;
-	break;
-    case ATOM_DEF:
-	hash = hash*FUNNY_NUMBER1 +
-	    (atom_tab(atom_val(term))->slot.bucket.hvalue);
-	break;
-    case SMALL_DEF:
-#if defined(ARCH_64)
-    {
-	Sint y1 = signed_val(term);
-	Uint y2 = y1 < 0 ? -(Uint)y1 : y1;
-	Uint32 y3 = (Uint32) (y2 >> 32);
-	int arity = 1;
-
-#if defined(WORDS_BIGENDIAN)
-	if (!IS_SSMALL28(y1))
-	{   /* like a bignum */
-	    Uint32 y4 = (Uint32) y2;
-	    hash = hash*FUNNY_NUMBER2 + ((y4 << 16) | (y4 >> 16));
-	    if (y3) {
-		hash = hash*FUNNY_NUMBER2 + ((y3 << 16) | (y3 >> 16));
-		arity++;
-	    }
-	    hash = hash * (y1 < 0 ? FUNNY_NUMBER3 : FUNNY_NUMBER2) + arity;
-	} else {
-	    hash = hash*FUNNY_NUMBER2 + (((Uint) y1) & 0xfffffff);
-	}
-#else
-	if  (!IS_SSMALL28(y1))
-	{   /* like a bignum */
-	    hash = hash*FUNNY_NUMBER2 + ((Uint32) y2);
-	    if (y3)
-	    {
-		hash = hash*FUNNY_NUMBER2 + y3;
-		arity++;
-	    }
-	    hash = hash * (y1 < 0 ? FUNNY_NUMBER3 : FUNNY_NUMBER2) + arity;
-	} else {
-	    hash = hash*FUNNY_NUMBER2 + (((Uint) y1) & 0xfffffff);
-	}
-#endif
-    }
-#else
-	hash = hash*FUNNY_NUMBER2 + unsigned_val(term);
-#endif
-	break;
-
-    case BINARY_DEF:
-	{
-	    size_t sz = binary_size(term);
-	    size_t i = (sz < 15) ? sz : 15;
-
-	    hash = hash_binary_bytes(term, i, hash);
-	    hash = hash*FUNNY_NUMBER4 + sz;
-	    break;
-	}
-
-    case EXPORT_DEF:
-	{
-	    Export* ep = *((Export **) (export_val(term) + 1));
-
-	    hash = hash * FUNNY_NUMBER11 + ep->info.mfa.arity;
-	    hash = hash*FUNNY_NUMBER1 +
-		(atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue);
-	    hash = hash*FUNNY_NUMBER1 +
-		(atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue);
-	    break;
-	}
-
-    case FUN_DEF:
-	{
-	    ErlFunThing* funp = (ErlFunThing *) fun_val(term);
-	    Uint num_free = funp->num_free;
-
-	    hash = hash * FUNNY_NUMBER10 + num_free;
-	    hash = hash*FUNNY_NUMBER1 +
-		(atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue);
-	    hash = hash*FUNNY_NUMBER2 + funp->fe->old_index;
-	    hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq;
-	    if (num_free > 0) {
-		if (num_free > 1) {
-		    WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_TERM_ARRAY_OP);
-		}
-		term = funp->env[0];
-		goto tail_recur;
-	    }
-	    break;
-	}
-
-    case PID_DEF:
-	hash = hash*FUNNY_NUMBER5 + internal_pid_number(term);
-	break;
-    case EXTERNAL_PID_DEF:
-	hash = hash*FUNNY_NUMBER5 + external_pid_number(term);
-        break;
-    case PORT_DEF:
-	hash = hash*FUNNY_NUMBER9 + internal_port_number(term);
-	break;
-    case EXTERNAL_PORT_DEF:
-	hash = hash*FUNNY_NUMBER9 + external_port_number(term);
-	break;
-    case REF_DEF:
-	hash = hash*FUNNY_NUMBER9 + internal_ref_numbers(term)[0];
-	break;
-    case EXTERNAL_REF_DEF:
-	hash = hash*FUNNY_NUMBER9 + external_ref_numbers(term)[0];
-	break;
-    case FLOAT_DEF:
-	{
-            FloatDef ff;
-            GET_DOUBLE(term, ff);
-            if (ff.fd == 0.0f) {
-                /* ensure positive 0.0 */
-                ff.fd = erts_get_positive_zero_float();
-            }
-            hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]);
-	}
-	break;
-    case MAKE_HASH_CDR_PRE_OP:
-	term = (Eterm) WSTACK_POP(stack);
-	if (is_not_list(term)) {
-	    WSTACK_PUSH(stack, (UWord) MAKE_HASH_CDR_POST_OP);
-	    goto tail_recur;
-	}
-	/*fall through*/
-    case LIST_DEF:
-	{
-	    Eterm* list = list_val(term);
-	    WSTACK_PUSH2(stack, (UWord) CDR(list),
-			 (UWord) MAKE_HASH_CDR_PRE_OP);
-	    term = CAR(list);
-	    goto tail_recur;
-	}
-
-    case MAKE_HASH_CDR_POST_OP:
-	hash *= FUNNY_NUMBER8;
-	break;
-
-     case BIG_DEF:
-	{
-	    Eterm* ptr  = big_val(term);
-	    int is_neg = BIG_SIGN(ptr);
-	    Uint arity = BIG_ARITY(ptr);
-	    Uint i = arity;
-	    ptr++;
-#if D_EXP == 16
-	    /* hash over 32 bit LE */
-
-	    while(i--) {
-		hash = hash*FUNNY_NUMBER2 + *ptr++;
-	    }
-#elif D_EXP == 32
-
-#if defined(WORDS_BIGENDIAN)
-	    while(i--) {
-		Uint d = *ptr++;
-		hash = hash*FUNNY_NUMBER2 + ((d << 16) | (d >> 16));
-	    }
-#else
-	    while(i--) {
-		hash = hash*FUNNY_NUMBER2 + *ptr++;
-	    }
-#endif
-
-#elif D_EXP == 64
-	    {
-	      Uint32 h = 0, l;
-#if defined(WORDS_BIGENDIAN)
-	      while(i--) {
-		  Uint d = *ptr++;
-		  l = d & 0xffffffff;
-		  h = d >> 32;
-		  hash = hash*FUNNY_NUMBER2 + ((l << 16) | (l >> 16));
-		  if (h || i)
-		      hash = hash*FUNNY_NUMBER2 + ((h << 16) | (h >> 16));
-	      }
-#else
-	      while(i--) {
-		  Uint d = *ptr++;
-		  l = d & 0xffffffff;
-		  h = d >> 32;
-		  hash = hash*FUNNY_NUMBER2 + l;
-		  if (h || i)
-		      hash = hash*FUNNY_NUMBER2 + h;
-	      }
-#endif
-	      /* adjust arity to match 32 bit mode */
-	      arity = (arity << 1) - (h == 0);
-	    }
-
-#else
-#error "unsupported D_EXP size"
-#endif
-	    hash = hash * (is_neg ? FUNNY_NUMBER3 : FUNNY_NUMBER2) + arity;
-	}
-	break;
-
-    case MAP_DEF:
-        hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term);
-        break;
-    case TUPLE_DEF:
-	{
-	    Eterm* ptr = tuple_val(term);
-	    Uint arity = arityval(*ptr);
-
-	    WSTACK_PUSH3(stack, (UWord) arity, (UWord) (ptr+1), (UWord) arity);
-	    op = MAKE_HASH_TUPLE_OP;
-	}/*fall through*/
-    case MAKE_HASH_TUPLE_OP:
-    case MAKE_HASH_TERM_ARRAY_OP:
-	{
-	    Uint i = (Uint) WSTACK_POP(stack);
-	    Eterm* ptr = (Eterm*) WSTACK_POP(stack);
-	    if (i != 0) {
-		term = *ptr;
-		WSTACK_PUSH3(stack, (UWord)(ptr+1), (UWord) i-1, (UWord) op);
-		goto tail_recur;
-	    }
-	    if (op == MAKE_HASH_TUPLE_OP) {
-		Uint32 arity = (UWord) WSTACK_POP(stack);
-		hash = hash*FUNNY_NUMBER9 + arity;
-	    }
-	    break;
-	}
-
-    default:
-	erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_broken_hash\n");
-	return 0;
-      }
-      if (WSTACK_ISEMPTY(stack)) break;
-      op = (Uint) WSTACK_POP(stack);
-    }
-
-    DESTROY_WSTACK(stack);
-    return hash;
-
-#undef MAKE_HASH_TUPLE_OP
-#undef MAKE_HASH_TERM_ARRAY_OP
-#undef MAKE_HASH_CDR_PRE_OP
-#undef MAKE_HASH_CDR_POST_OP
-}
-
 static Eterm
 do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp,
 			   ErlHeapFragment **bp, Process **p, Uint sz)
-- 
cgit v1.2.3


From 52097fab56edbbd8c6f8a57ec3b3f33aa60c5bb9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 1 Aug 2016 14:49:47 +0200
Subject: Update test cases for erlang:hash/2 removal

---
 erts/emulator/test/binary_SUITE.erl      |   2 -
 erts/emulator/test/fun_SUITE.erl         |  35 +--------
 erts/emulator/test/hash_SUITE.erl        |  44 ++----------
 erts/emulator/test/map_SUITE.erl         |  24 -------
 erts/emulator/test/trace_local_SUITE.erl | 118 +++++++++++++++----------------
 5 files changed, 65 insertions(+), 158 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index 1c7d278bb0..238a8aa42d 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -19,7 +19,6 @@
 %%
 
 -module(binary_SUITE).
--compile({nowarn_deprecated_function, {erlang,hash,2}}).
 
 %% Tests binaries and the BIFs:
 %%	list_to_binary/1
@@ -392,7 +391,6 @@ test_hash(List) ->
     Bin = list_to_binary(List),
     Sbin = make_sub_binary(List),
     Unaligned = make_unaligned_sub_binary(Sbin),
-    test_hash_1(Bin, Sbin, Unaligned, fun erlang:hash/2),
     test_hash_1(Bin, Sbin, Unaligned, fun erlang:phash/2),
     test_hash_1(Bin, Sbin, Unaligned, fun erlang:phash2/2).
 
diff --git a/erts/emulator/test/fun_SUITE.erl b/erts/emulator/test/fun_SUITE.erl
index 26fa955e3c..e4640909aa 100644
--- a/erts/emulator/test/fun_SUITE.erl
+++ b/erts/emulator/test/fun_SUITE.erl
@@ -19,12 +19,11 @@
 %%
 
 -module(fun_SUITE).
--compile({nowarn_deprecated_function, {erlang,hash,2}}).
 
 -export([all/0, suite/0,
 	 bad_apply/1,bad_fun_call/1,badarity/1,ext_badarity/1,
 	 equality/1,ordering/1,
-	 fun_to_port/1,t_hash/1,t_phash/1,t_phash2/1,md5/1,
+	 fun_to_port/1,t_phash/1,t_phash2/1,md5/1,
 	 refc/1,refc_ets/1,refc_dist/1,
 	 const_propagation/1,t_arity/1,t_is_function2/1,
 	 t_fun_info/1,t_fun_info_mfa/1]).
@@ -38,9 +37,9 @@ suite() ->
      {timetrap, {minutes, 1}}].
 
 
-all() -> 
+all() ->
     [bad_apply, bad_fun_call, badarity, ext_badarity,
-     equality, ordering, fun_to_port, t_hash, t_phash,
+     equality, ordering, fun_to_port, t_phash,
      t_phash2, md5, refc, refc_ets, refc_dist,
      const_propagation, t_arity, t_is_function2, t_fun_info,
      t_fun_info_mfa].
@@ -412,33 +411,6 @@ build_io_list(N) ->
 	1 -> [7,L|L]
     end.
 
-%% Test the hash/2 BIF on funs.
-t_hash(Config) when is_list(Config) ->
-    F1 = fun(_X) -> 1 end,
-    F2 = fun(_X) -> 2 end,
-    true = hash(F1) /= hash(F2),
-
-    G1 = make_fun(1, 2, 3),
-    G2 = make_fun(1, 2, 3),
-    G3 = make_fun(1, 2, 4),
-    true = hash(G1) == hash(G2),
-    true = hash(G2) /= hash(G3),
-
-    FF0 = fun erlang:abs/1,
-    FF1 = fun erlang:exit/1,
-    FF2 = fun erlang:exit/2,
-    FF3 = fun blurf:exit/2,
-    true = hash(FF0) =/= hash(FF1),
-    true = hash(FF0) =/= hash(FF2),
-    true = hash(FF0) =/= hash(FF3),
-    true = hash(FF1) =/= hash(FF2),
-    true = hash(FF1) =/= hash(FF3),
-    true = hash(FF2) =/= hash(FF3),
-    ok.
-
-hash(Term) ->
-    erlang:hash(Term, 16#7ffffff).
-
 %% Test the phash/2 BIF on funs.
 t_phash(Config) when is_list(Config) ->
     F1 = fun(_X) -> 1 end,
@@ -461,7 +433,6 @@ t_phash(Config) when is_list(Config) ->
     true = phash(FF1) =/= phash(FF2),
     true = phash(FF1) =/= phash(FF3),
     true = phash(FF2) =/= phash(FF3),
-    
     ok.
 
 phash(Term) ->
diff --git a/erts/emulator/test/hash_SUITE.erl b/erts/emulator/test/hash_SUITE.erl
index a39d101b0d..3cbb3c7d5f 100644
--- a/erts/emulator/test/hash_SUITE.erl
+++ b/erts/emulator/test/hash_SUITE.erl
@@ -34,7 +34,6 @@
 -export([basic_test/0,cmp_test/1,range_test/0,spread_test/1,
 	 phash2_test/0, otp_5292_test/0,
          otp_7127_test/0]).
--compile({nowarn_deprecated_function, {erlang,hash,2}}).
 
 %%
 %% Define to run outside of test server
@@ -130,24 +129,12 @@ test_hash_zero(Config) when is_list(Config) ->
 %%
 basic_test() ->
     685556714 = erlang:phash({a,b,c},16#FFFFFFFF),
-    14468079 = erlang:hash({a,b,c},16#7FFFFFF),
     37442646 =  erlang:phash([a,b,c,{1,2,3},c:pid(0,2,3),
 				    16#77777777777777],16#FFFFFFFF),
-    Comment = case erlang:hash([a,b,c,{1,2,3},c:pid(0,2,3),
-				      16#77777777777777],16#7FFFFFF) of
-			102727602 ->
-			    big = erlang:system_info(endian),
-			    "Big endian machine";
-			105818829 ->
-			    little = erlang:system_info(endian),
-			    "Little endian machine"
-		    end,
     ExternalReference = <<131,114,0,3,100,0,13,110,111,110,111,100,101,64,
 			 110,111,104,111,115,116,0,0,0,0,122,0,0,0,0,0,0,0,0>>,
     1113403635 = erlang:phash(binary_to_term(ExternalReference),
 				    16#FFFFFFFF),
-    123 = erlang:hash(binary_to_term(ExternalReference),
-			    16#7FFFFFF),
     ExternalFun = <<131,117,0,0,0,3,103,100,0,13,110,111,110,111,100,101,64,
 		   110,111,104,111,115,116,0,0,0,38,0,0,0,0,0,100,0,8,101,
 		   114,108,95,101,118,97,108,97,20,98,5,182,139,98,108,0,0,
@@ -166,11 +153,9 @@ basic_test() ->
 		   64,110,111,104,111,115,116,0,0,0,22,0,0,0,0,0,106>>,
     170987488 = erlang:phash(binary_to_term(ExternalFun),
 				   16#FFFFFFFF),
-    124460689 = erlang:hash(binary_to_term(ExternalFun),
-				  16#7FFFFFF),
     case (catch erlang:phash(1,0)) of
 	{'EXIT',{badarg, _}} ->
-	    {comment, Comment};
+	    ok;
 	_ ->
 	    exit(phash_accepted_zero_as_range)
     end.
@@ -193,7 +178,6 @@ range_test() ->
 	end,
     F(1,16#100000000,F).
 
-    
 
 spread_test(N) ->
     test_fun(N,{erlang,phash},16#50000000000,fun(X) ->
@@ -419,7 +403,7 @@ phash2_test() ->
 	 {"abc"++[1009], 290369864},
 	 {"abc"++[1009]++"de", 4134369195},
 	 {"1234567890123456", 963649519},
-	 
+
 	 %% tuple
 	 {{}, 221703996},
 	 {{{}}, 2165044361},
@@ -452,30 +436,15 @@ f3(X, Y) ->
 -endif.
 
 otp_5292_test() ->
-    H = fun(E) -> [erlang:hash(E, 16#7FFFFFF),
-                   erlang:hash(-E, 16#7FFFFFF)]
-        end,
-    S1 = md5([md5(hash_int(S, E, H)) || {Start, N, Sz} <- d(), 
-                                        {S, E} <- int(Start, N, Sz)]),
     PH = fun(E) -> [erlang:phash(E, 1 bsl 32),
                     erlang:phash(-E, 1 bsl 32),
                     erlang:phash2(E, 1 bsl 32),
                     erlang:phash2(-E, 1 bsl 32)]
             end,
-    S2 = md5([md5(hash_int(S, E, PH)) || {Start, N, Sz} <- d(), 
+    S2 = md5([md5(hash_int(S, E, PH)) || {Start, N, Sz} <- d(),
                                          {S, E} <- int(Start, N, Sz)]),
-    Comment = case S1 of 
-			<<4,248,208,156,200,131,7,1,173,13,239,173,112,81,16,174>> ->
-			    big = erlang:system_info(endian),
-                            "Big endian machine";
-                        <<180,28,33,231,239,184,71,125,76,47,227,241,78,184,176,233>> ->
-			    little = erlang:system_info(endian),
-                            "Little endian machine"
-                    end,
     <<124,81,198,121,174,233,19,137,10,83,33,80,226,111,238,99>> = S2,
-    2 = erlang:hash(1, (1 bsl 27) -1),
-    {'EXIT', _} = (catch erlang:hash(1, (1 bsl 27))),
-    {comment, Comment}.
+    ok.
 
 d() ->
     [%% Start,          NumOfIntervals, SizeOfInterval
@@ -495,8 +464,6 @@ md5(T) ->
     erlang:md5(term_to_binary(T)).   
 
 bit_level_binaries_do() ->
-    [3511317,7022633,14044578,28087749,56173436,112344123,90467083|_] =
-	bit_level_all_different(fun erlang:hash/2),
     [3511317,7022633,14044578,28087749,56173436,112344123,90467083|_] =
 	bit_level_all_different(fun erlang:phash/2),
     [102233154,19716,102133857,4532024,123369135,24565730,109558721|_] =
@@ -535,9 +502,7 @@ bit_level_all_different(Hash) ->
     Hashes1.
 
 test_hash_phash(Bitstr, Rem) ->
-    Hash = erlang:hash(Bitstr, Rem),
     Hash = erlang:phash(Bitstr, Rem),
-    Hash = erlang:hash(unaligned_sub_bitstr(Bitstr), Rem),
     Hash = erlang:phash(unaligned_sub_bitstr(Bitstr), Rem).
 
 test_phash2(Bitstr, Rem) ->
@@ -555,7 +520,6 @@ hash_zero_test() ->
           binary_to_term(<<131,70,128,0,0,0,0,0,0,0>>)], %% -0.0
     ok = hash_zero_test(Zs,fun(T) -> erlang:phash2(T, 1 bsl 32) end),
     ok = hash_zero_test(Zs,fun(T) -> erlang:phash(T, 1 bsl 32) end),
-    ok = hash_zero_test(Zs,fun(T) -> erlang:hash(T, (1 bsl 27) - 1) end),
     ok.
 
 hash_zero_test([Z|Zs],F) ->
diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl
index 5af676c409..dfa1629112 100644
--- a/erts/emulator/test/map_SUITE.erl
+++ b/erts/emulator/test/map_SUITE.erl
@@ -18,7 +18,6 @@
 %%
 -module(map_SUITE).
 -export([all/0, suite/0]).
--compile({nowarn_deprecated_function, {erlang,hash,2}}).
 
 -export([t_build_and_match_literals/1, t_build_and_match_literals_large/1,
          t_update_literals/1, t_update_literals_large/1,
@@ -2130,8 +2129,6 @@ t_erlang_hash(Config) when is_list(Config) ->
 
     ok = t_bif_erlang_phash2(),
     ok = t_bif_erlang_phash(),
-    ok = t_bif_erlang_hash(),
-
     ok.
 
 t_bif_erlang_phash2() ->
@@ -2174,27 +2171,6 @@ t_bif_erlang_phash() ->
     2620391445 = erlang:phash(M2,Sz), % 3590546636
     ok.
 
-t_bif_erlang_hash() ->
-    Sz = 1 bsl 27 - 1,
-    39684169 = erlang:hash(#{},Sz),  % 5158
-    33673142 = erlang:hash(#{ a => 1, "a" => 2, <<"a">> => 3, {a,b} => 4 },Sz), % 71555838
-    95337869 = erlang:hash(#{ 1 => a, 2 => "a", 3 => <<"a">>, 4 => {a,b} },Sz), % 5497225
-    108959561 = erlang:hash(#{ 1 => a },Sz), % 126071654
-    59623150 = erlang:hash(#{ a => 1 },Sz), % 126426236
-
-    42775386 = erlang:hash(#{{} => <<>>},Sz), % 101655720
-    71692856 = erlang:hash(#{<<>> => {}},Sz), % 101655720
-
-    M0 = #{ a => 1, "key" => <<"value">> },
-    M1 = maps:remove("key",M0),
-    M2 = M1#{ "key" => <<"value">> },
-
-    70254632 = erlang:hash(M0,Sz), % 38260486
-    59623150 = erlang:hash(M1,Sz), % 126426236
-    70254632 = erlang:hash(M2,Sz), % 38260486
-    ok.
-
-
 t_map_encode_decode(Config) when is_list(Config) ->
     <<131,116,0,0,0,0>> = erlang:term_to_binary(#{}),
     Pairs = [
diff --git a/erts/emulator/test/trace_local_SUITE.erl b/erts/emulator/test/trace_local_SUITE.erl
index c297acd78b..5b65889f4a 100644
--- a/erts/emulator/test/trace_local_SUITE.erl
+++ b/erts/emulator/test/trace_local_SUITE.erl
@@ -19,7 +19,6 @@
 %%
 
 -module(trace_local_SUITE).
--compile({nowarn_deprecated_function, {erlang,hash,2}}).
 
 -export([basic_test/0, bit_syntax_test/0, return_test/0,
 	 on_and_off_test/0, stack_grow_test/0,
@@ -65,7 +64,7 @@
 init_per_testcase(_Case, Config) ->
     Config.
 
-end_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, _Config) ->
     shutdown(),
 
     %% Reloading the module will clear all trace patterns, and
@@ -78,7 +77,7 @@ suite() ->
     [{ct_hooks,[ts_install_cth]},
      {timetrap, {minutes, 2}}].
 
-all() -> 
+all() ->
     case test_server:is_native(trace_local_SUITE) of
         true -> [not_run];
         false ->
@@ -98,7 +97,7 @@ all() ->
     end.
 
 
-not_run(Config) when is_list(Config) -> 
+not_run(Config) when is_list(Config) ->
     {skipped,"Native code"}.
 
 %% Tests basic local call-trace
@@ -300,7 +299,7 @@ basic_test() ->
     NumMatches = erlang:trace_pattern({?MODULE,'_','_'},[],[local]),
     NumMatches = erlang:trace_pattern({?MODULE,'_','_'},[],[local]),
     erlang:trace_pattern({?MODULE,slave,'_'},false,[local]),
-    [1,1,1,1] = apply_slave(?MODULE,exported_wrap,[1]),
+    [1,1,1,997] = apply_slave(?MODULE,exported_wrap,[1]),
     ?CT(?MODULE,exported_wrap,[1]),
     ?CT(?MODULE,exported,[1]),
     ?CT(?MODULE,local,[1]),
@@ -308,17 +307,17 @@ basic_test() ->
     ?CT(?MODULE,local_tail,[1]),
     erlang:trace_pattern({?MODULE,'_','_'},[],[]),
     erlang:trace_pattern({?MODULE,slave,'_'},false,[local]),
-    [1,1,1,1] = apply_slave(?MODULE,exported_wrap,[1]),
+    [1,1,1,997] = apply_slave(?MODULE,exported_wrap,[1]),
     ?CT(?MODULE,exported_wrap,[1]),
-    [1,1,1,1] = lambda_slave(fun() ->
-                                     exported_wrap(1)
-                             end),
-    ?NM, 
+    [1,1,1,997] = lambda_slave(fun() ->
+                                       exported_wrap(1)
+                               end),
+    ?NM,
     erlang:trace_pattern({?MODULE,'_','_'},[],[local]),
     erlang:trace_pattern({?MODULE,slave,'_'},false,[local]),
-    [1,1,1,1] = lambda_slave(fun() ->
-                                     exported_wrap(1)
-                             end),
+    [1,1,1,997] = lambda_slave(fun() ->
+                                       exported_wrap(1)
+                               end),
     ?CT(?MODULE,_,_), %% The fun
     ?CT(?MODULE,exported_wrap,[1]),
     ?CT(?MODULE,exported,[1]),
@@ -379,36 +378,36 @@ return_test() ->
     setup([call]),
     erlang:trace_pattern({?MODULE,'_','_'},[{'_',[],[{return_trace}]}],
                          [local]),
-    erlang:trace_pattern({erlang,hash,'_'},[{'_',[],[{return_trace}]}],
+    erlang:trace_pattern({erlang,phash2,'_'},[{'_',[],[{return_trace}]}],
                          [local]),
     erlang:trace_pattern({?MODULE,slave,'_'},false,[local]),
-    [1,1,1,1] = apply_slave(?MODULE,exported_wrap,[1]),
-    ?CT(?MODULE,exported_wrap,[1]), 
+    [1,1,1,997] = apply_slave(?MODULE,exported_wrap,[1]),
+    ?CT(?MODULE,exported_wrap,[1]),
     ?CT(?MODULE,exported,[1]),
     ?CT(?MODULE,local,[1]),
     ?CT(?MODULE,local2,[1]),
     ?CT(?MODULE,local_tail,[1]),
-    ?CT(erlang,hash,[1,1]),
-    ?RF(erlang,hash,2,1),
-    ?RF(?MODULE,local_tail,1,[1,1]),
-    ?RF(?MODULE,local2,1,[1,1]),
-    ?RF(?MODULE,local,1,[1,1,1]),
-    ?RF(?MODULE,exported,1,[1,1,1,1]),
-    ?RF(?MODULE,exported_wrap,1,[1,1,1,1]),
+    ?CT(erlang,phash2,[1,1023]),
+    ?RF(erlang,phash2,2,997),
+    ?RF(?MODULE,local_tail,1,[1,997]),
+    ?RF(?MODULE,local2,1,[1,997]),
+    ?RF(?MODULE,local,1,[1,1,997]),
+    ?RF(?MODULE,exported,1,[1,1,1,997]),
+    ?RF(?MODULE,exported_wrap,1,[1,1,1,997]),
     shutdown(),
     setup([call,return_to]),
     erlang:trace_pattern({?MODULE,'_','_'},[],
                          [local]),
-    erlang:trace_pattern({erlang,hash,'_'},[],
+    erlang:trace_pattern({erlang,phash2,'_'},[],
                          [local]),
     erlang:trace_pattern({?MODULE,slave,'_'},false,[local]),
-    [1,1,1,1] = apply_slave(?MODULE,exported_wrap,[1]),
-    ?CT(?MODULE,exported_wrap,[1]), 
+    [1,1,1,997] = apply_slave(?MODULE,exported_wrap,[1]),
+    ?CT(?MODULE,exported_wrap,[1]),
     ?CT(?MODULE,exported,[1]),
     ?CT(?MODULE,local,[1]),
     ?CT(?MODULE,local2,[1]),
     ?CT(?MODULE,local_tail,[1]),
-    ?CT(erlang,hash,[1,1]),
+    ?CT(erlang,phash2,[1,1023]),
     ?RT(?MODULE,local_tail,1),
     ?RT(?MODULE,local,1),
     ?RT(?MODULE,exported,1),
@@ -417,25 +416,25 @@ return_test() ->
     setup([call,return_to]),
     erlang:trace_pattern({?MODULE,'_','_'},[{'_',[],[{return_trace}]}],
                          [local]),
-    erlang:trace_pattern({erlang,hash,'_'},[{'_',[],[{return_trace}]}],
+    erlang:trace_pattern({erlang,phash2,'_'},[{'_',[],[{return_trace}]}],
                          [local]),
     erlang:trace_pattern({?MODULE,slave,'_'},false,[local]),
-    [1,1,1,1] = apply_slave(?MODULE,exported_wrap,[1]),
-    ?CT(?MODULE,exported_wrap,[1]), 
+    [1,1,1,997] = apply_slave(?MODULE,exported_wrap,[1]),
+    ?CT(?MODULE,exported_wrap,[1]),
     ?CT(?MODULE,exported,[1]),
     ?CT(?MODULE,local,[1]),
     ?CT(?MODULE,local2,[1]),
     ?CT(?MODULE,local_tail,[1]),
-    ?CT(erlang,hash,[1,1]),
-    ?RF(erlang,hash,2,1),
+    ?CT(erlang,phash2,[1,1023]),
+    ?RF(erlang,phash2,2,997),
     ?RT(?MODULE,local_tail,1),
-    ?RF(?MODULE,local_tail,1,[1,1]),
-    ?RF(?MODULE,local2,1,[1,1]),
+    ?RF(?MODULE,local_tail,1,[1,997]),
+    ?RF(?MODULE,local2,1,[1,997]),
     ?RT(?MODULE,local,1),
-    ?RF(?MODULE,local,1,[1,1,1]),
+    ?RF(?MODULE,local,1,[1,1,997]),
     ?RT(?MODULE,exported,1),
-    ?RF(?MODULE,exported,1,[1,1,1,1]),
-    ?RF(?MODULE,exported_wrap,1,[1,1,1,1]),
+    ?RF(?MODULE,exported,1,[1,1,1,997]),
+    ?RF(?MODULE,exported_wrap,1,[1,1,1,997]),
     ?RT(?MODULE,slave,2),
     shutdown(),
     ?NM,
@@ -446,7 +445,6 @@ return_test() ->
     erlang:trace_pattern({'_','_','_'},[],[local]),
     apply_slave(erlang,trace,[Pid, false, [all]]),
     shutdown(),
-
     ok.
 
 on_and_off_test() ->
@@ -456,72 +454,72 @@ on_and_off_test() ->
     LocalTail = fun() ->
                         local_tail(1)
                 end,
-    [1,1] = lambda_slave(LocalTail),
+    [1,997] = lambda_slave(LocalTail),
     ?CT(?MODULE,local_tail,[1]),
     erlang:trace(Pid,true,[return_to]),
-    [1,1] = lambda_slave(LocalTail),
+    [1,997] = lambda_slave(LocalTail),
     ?CT(?MODULE,local_tail,[1]),
     ?RT(?MODULE,_,_),
     0 = erlang:trace_pattern({?MODULE,local_tail,1},[],[global]),
-    [1,1] = lambda_slave(LocalTail),
+    [1,997] = lambda_slave(LocalTail),
     ?NM,
     1 = erlang:trace_pattern({?MODULE,exported_wrap,1},[],[global]),
-    [1,1,1,1] = apply_slave(?MODULE,exported_wrap,[1]),
+    [1,1,1,997] = apply_slave(?MODULE,exported_wrap,[1]),
     ?CT(?MODULE,exported_wrap,[1]),
     1 = erlang:trace_pattern({?MODULE,exported_wrap,1},[],[local]),
-    [1,1,1,1] = apply_slave(?MODULE,exported_wrap,[1]),
+    [1,1,1,997] = apply_slave(?MODULE,exported_wrap,[1]),
     ?CT(?MODULE,exported_wrap,[1]),
     ?RT(?MODULE,slave,2),
-    1 = erlang:trace_pattern({erlang,hash,2},[],[local]),
-    [1,1,1,1] = apply_slave(?MODULE,exported_wrap,[1]),
+    1 = erlang:trace_pattern({erlang,phash2,2},[],[local]),
+    [1,1,1,997] = apply_slave(?MODULE,exported_wrap,[1]),
     ?CT(?MODULE,exported_wrap,[1]),
-    ?CT(erlang,hash,[1,1]),
+    ?CT(erlang,phash2,[1,1023]),
     ?RT(?MODULE,local_tail,1),
     ?RT(?MODULE,slave,2),
     erlang:trace(Pid,true,[timestamp]),
-    [1,1,1,1] = apply_slave(?MODULE,exported_wrap,[1]),
+    [1,1,1,997] = apply_slave(?MODULE,exported_wrap,[1]),
     ?CTT(?MODULE,exported_wrap,[1]),
-    ?CTT(erlang,hash,[1,1]),
+    ?CTT(erlang,phash2,[1,1023]),
     ?RTT(?MODULE,local_tail,1),
     ?RTT(?MODULE,slave,2),
     erlang:trace(Pid,false,[return_to,timestamp]),
-    [1,1,1,1] = apply_slave(?MODULE,exported_wrap,[1]),
+    [1,1,1,997] = apply_slave(?MODULE,exported_wrap,[1]),
     ?CT(?MODULE,exported_wrap,[1]),
-    ?CT(erlang,hash,[1,1]),
+    ?CT(erlang,phash2,[1,1023]),
     erlang:trace(Pid,true,[return_to]),
-    1 = erlang:trace_pattern({erlang,hash,2},[],[]),
-    [1,1,1,1] = apply_slave(?MODULE,exported_wrap,[1]),
+    1 = erlang:trace_pattern({erlang,phash2,2},[],[]),
+    [1,1,1,997] = apply_slave(?MODULE,exported_wrap,[1]),
     ?CT(?MODULE,exported_wrap,[1]),
-    ?CT(erlang,hash,[1,1]),
+    ?CT(erlang,phash2,[1,1023]),
     ?RT(?MODULE,slave,2),
     1 = erlang:trace_pattern({?MODULE,exported_wrap,1},[],[]),
-    [1,1,1,1] = apply_slave(?MODULE,exported_wrap,[1]),
+    [1,1,1,997] = apply_slave(?MODULE,exported_wrap,[1]),
     ?CT(?MODULE,exported_wrap,[1]),
-    ?CT(erlang,hash,[1,1]),
+    ?CT(erlang,phash2,[1,1023]),
     shutdown(),
     erlang:trace_pattern({'_','_','_'},false,[local]),
     N = erlang:trace_pattern({erlang,'_','_'},true,[local]),
     case erlang:trace_pattern({erlang,'_','_'},false,[local]) of
-        N -> 
+        N ->
             ok;
         Else ->
             exit({number_mismatch, {expected, N}, {got, Else}})
     end,
     case erlang:trace_pattern({erlang,'_','_'},false,[local]) of
-        N -> 
+        N ->
             ok;
         Else2 ->
             exit({number_mismatch, {expected, N}, {got, Else2}})
     end,
     M = erlang:trace_pattern({erlang,'_','_'},true,[]),
     case erlang:trace_pattern({erlang,'_','_'},false,[]) of
-        M -> 
+        M ->
             ok;
         Else3 ->
             exit({number_mismatch, {expected, N}, {got, Else3}})
     end,
     case erlang:trace_pattern({erlang,'_','_'},false,[]) of
-        M -> 
+        M ->
             ok;
         Else4 ->
             exit({number_mismatch, {expected, N}, {got, Else4}})
@@ -930,7 +928,7 @@ local2(Val) ->
     local_tail(Val). %% Tail recursive call
 
 local_tail(Val) ->
-    [Val , erlang:hash(1,1)].
+    [Val , erlang:phash2(1,1023)].
 
 
 
-- 
cgit v1.2.3


From 26b59dfe67ef551cd94765557cdd8c79794bcc38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Valim?= <jose.valim@plataformatec.com.br>
Date: Tue, 31 May 2016 14:28:54 +0200
Subject: Add new AtU8 beam chunk
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The new chunk stores atoms encoded in UTF-8.

beam_lib has also been modified to handle the new
'utf8_atoms' attribute while the 'atoms' attribute
may be a missing chunk from now on.

The binary_to_atom/2 BIF can now encode any utf8
binary with up to 255 characters.

The list_to_atom/1 BIF can now accept codepoints
higher than 255 with up to 255 characters (thanks
to Björn Gustavsson).
---
 erts/emulator/beam/atom.c         | 35 +++++++++++------
 erts/emulator/beam/atom.h         |  3 ++
 erts/emulator/beam/beam_load.c    | 66 ++++++++++++++++++++++---------
 erts/emulator/beam/bif.c          | 14 +++----
 erts/emulator/beam/erl_unicode.c  | 83 ++++++++++++++++-----------------------
 erts/emulator/beam/global.h       |  1 +
 erts/emulator/beam/utils.c        | 62 +++++++++++++++++++++++++++++
 erts/emulator/test/bif_SUITE.erl  | 47 +++++++++++++++++++---
 erts/emulator/test/code_SUITE.erl | 10 ++---
 9 files changed, 224 insertions(+), 97 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index 2b5ad097a0..2055c29190 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -233,10 +233,10 @@ need_convertion:
 }
 
 /*
- * erts_atom_put() may fail. If it fails THE_NON_VALUE is returned!
+ * erts_atom_put_index() may fail. Returns negative indexes for errors.
  */
-Eterm
-erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc)
+int
+erts_atom_put_index(const byte *name, int len, ErtsAtomEncoding enc, int trunc)
 {
     byte utf8_copy[MAX_ATOM_SZ_FROM_LATIN1];
     const byte *text = name;
@@ -253,7 +253,7 @@ erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc)
 	if (trunc)
 	    tlen = 0;
 	else
-	    return THE_NON_VALUE;
+	    return ATOM_MAX_CHARS_ERROR;
     }
 
     switch (enc) {
@@ -262,7 +262,7 @@ erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc)
 	    if (trunc)
 		tlen = MAX_ATOM_CHARACTERS;
 	    else
-		return THE_NON_VALUE;
+		return ATOM_MAX_CHARS_ERROR;
 	}
 #ifdef DEBUG
 	for (aix = 0; aix < len; aix++) {
@@ -276,7 +276,7 @@ erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc)
 	    if (trunc)
 		tlen = MAX_ATOM_CHARACTERS;
 	    else
-		return THE_NON_VALUE;
+		return ATOM_MAX_CHARS_ERROR;
 	}
 	no_latin1_chars = tlen;
 	latin1_to_utf8(utf8_copy, &text, &tlen);
@@ -284,7 +284,7 @@ erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc)
     case ERTS_ATOM_ENC_UTF8:
 	/* First sanity check; need to verify later */
 	if (tlen > MAX_ATOM_SZ_LIMIT && !trunc)
-	    return THE_NON_VALUE;
+	    return ATOM_MAX_CHARS_ERROR;
 	break;
     }
 
@@ -295,7 +295,7 @@ erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc)
     atom_read_unlock();
     if (aix >= 0) {
 	/* Already in table no need to verify it */
-	return make_atom(aix);
+	return aix;
     }
 
     if (enc == ERTS_ATOM_ENC_UTF8) {
@@ -314,13 +314,13 @@ erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc)
 	case ERTS_UTF8_OK_MAX_CHARS:
 	    /* Truncated... */
 	    if (!trunc)
-		return THE_NON_VALUE;
+		return ATOM_MAX_CHARS_ERROR;
 	    ASSERT(no_chars == MAX_ATOM_CHARACTERS);
 	    tlen = err_pos - text;
 	    break;
 	default:
 	    /* Bad utf8... */
-	    return THE_NON_VALUE;
+	    return ATOM_BAD_ENCODING_ERROR;
 	}
     }
 
@@ -333,7 +333,20 @@ erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc)
     atom_write_lock();
     aix = index_put(&erts_atom_table, (void*) &a);
     atom_write_unlock();
-    return make_atom(aix);
+    return aix;
+}
+
+/*
+ * erts_atom_put() may fail. If it fails THE_NON_VALUE is returned!
+ */
+Eterm
+erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc)
+{
+    int aix = erts_atom_put_index(name, len, enc, trunc);
+    if (aix >= 0)
+	return make_atom(aix);
+    else
+	return THE_NON_VALUE;
 }
 
 Eterm
diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h
index abd3b44993..be998a46bd 100644
--- a/erts/emulator/beam/atom.h
+++ b/erts/emulator/beam/atom.h
@@ -29,6 +29,8 @@
 #define MAX_ATOM_SZ_LIMIT (4*MAX_ATOM_CHARACTERS) /* theoretical byte limit */
 #define ATOM_LIMIT (1024*1024)
 #define MIN_ATOM_TABLE_SIZE 8192
+#define ATOM_BAD_ENCODING_ERROR -1
+#define ATOM_MAX_CHARS_ERROR -2
 
 #ifndef ARCH_32
 /* Internal atom cache needs MAX_ATOM_TABLE_SIZE to be less than an
@@ -133,6 +135,7 @@ int atom_table_sz(void);	/* table size in bytes, excluding stored objects */
 
 Eterm am_atom_put(const char*, int); /* ONLY 7-bit ascii! */
 Eterm erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc);
+int erts_atom_put_index(const byte *name, int len, ErtsAtomEncoding enc, int trunc);
 void init_atom_table(void);
 void atom_info(fmtfn_t, void *);
 void dump_atoms(fmtfn_t, void *);
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 8f1faa6719..1899ffb079 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -157,13 +157,15 @@ typedef struct {
 #define STR_CHUNK 2
 #define IMP_CHUNK 3
 #define EXP_CHUNK 4
-#define NUM_MANDATORY 5
+#define MIN_MANDATORY 1
+#define MAX_MANDATORY 5
 
 #define LAMBDA_CHUNK 5
 #define LITERAL_CHUNK 6
 #define ATTR_CHUNK 7
 #define COMPILE_CHUNK 8
 #define LINE_CHUNK 9
+#define UTF8_ATOM_CHUNK 10
 
 #define NUM_CHUNK_TYPES (sizeof(chunk_types)/sizeof(chunk_types[0]))
 
@@ -173,9 +175,13 @@ typedef struct {
 
 static Uint chunk_types[] = {
     /*
-     * Mandatory chunk types -- these MUST be present.
+     * Atom chunk types -- Atom or AtU8 MUST be present.
      */
     MakeIffId('A', 't', 'o', 'm'), /* 0 */
+
+    /*
+     * Mandatory chunk types -- these MUST be present.
+     */
     MakeIffId('C', 'o', 'd', 'e'), /* 1 */
     MakeIffId('S', 't', 'r', 'T'), /* 2 */
     MakeIffId('I', 'm', 'p', 'T'), /* 3 */
@@ -189,6 +195,7 @@ static Uint chunk_types[] = {
     MakeIffId('A', 't', 't', 'r'), /* 7 */
     MakeIffId('C', 'I', 'n', 'f'), /* 8 */
     MakeIffId('L', 'i', 'n', 'e'), /* 9 */
+    MakeIffId('A', 't', 'U', '8'), /* 10 */
 };
 
 /*
@@ -490,9 +497,9 @@ static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 #endif
 static int init_iff_file(LoaderState* stp, byte* code, Uint size);
 static int scan_iff_file(LoaderState* stp, Uint* chunk_types,
-			 Uint num_types, Uint num_mandatory);
+			 Uint num_types);
 static int verify_chunks(LoaderState* stp);
-static int load_atom_table(LoaderState* stp);
+static int load_atom_table(LoaderState* stp, ErtsAtomEncoding enc);
 static int load_import_table(LoaderState* stp);
 static int read_export_table(LoaderState* stp);
 static int is_bif(Eterm mod, Eterm func, unsigned arity);
@@ -629,7 +636,7 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader,
     CHKALLOC();
     CHKBLK(ERTS_ALC_T_CODE,stp->code);
     if (!init_iff_file(stp, code, unloaded_size) ||
-	!scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY) ||
+	!scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES) ||
 	!verify_chunks(stp)) {
 	goto load_error;
     }
@@ -674,9 +681,16 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader,
      */
 
     CHKBLK(ERTS_ALC_T_CODE,stp->code);
-    define_file(stp, "atom table", ATOM_CHUNK);
-    if (!load_atom_table(stp)) {
-	goto load_error;
+    if (stp->chunks[UTF8_ATOM_CHUNK].size > 0) {
+        define_file(stp, "utf8 atom table", UTF8_ATOM_CHUNK);
+        if (!load_atom_table(stp, ERTS_ATOM_ENC_UTF8)) {
+            goto load_error;
+        }
+    } else {
+        define_file(stp, "atom table", ATOM_CHUNK);
+        if (!load_atom_table(stp, ERTS_ATOM_ENC_LATIN1)) {
+            goto load_error;
+        }
     }
 
     /*
@@ -1212,7 +1226,7 @@ init_iff_file(LoaderState* stp, byte* code, Uint size)
  * Scan the IFF file. The header should have been verified by init_iff_file().
  */
 static int
-scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory)
+scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types)
 {
     Uint count;
     Uint id;
@@ -1291,7 +1305,16 @@ verify_chunks(LoaderState* stp)
     MD5_CTX context;
 
     MD5Init(&context);
-    for (i = 0; i < NUM_MANDATORY; i++) {
+
+    if (stp->chunks[UTF8_ATOM_CHUNK].start != NULL) {
+	MD5Update(&context, stp->chunks[UTF8_ATOM_CHUNK].start, stp->chunks[UTF8_ATOM_CHUNK].size);
+    } else if (stp->chunks[ATOM_CHUNK].start != NULL) {
+	MD5Update(&context, stp->chunks[ATOM_CHUNK].start, stp->chunks[ATOM_CHUNK].size);
+    } else {
+        LoadError0(stp, "mandatory chunk of type 'Atom' or 'AtU8' not found\n");
+    }
+
+    for (i = MIN_MANDATORY; i < MAX_MANDATORY; i++) {
 	if (stp->chunks[i].start != NULL) {
 	    MD5Update(&context, stp->chunks[i].start, stp->chunks[i].size);
 	} else {
@@ -1352,7 +1375,7 @@ verify_chunks(LoaderState* stp)
 }
 
 static int
-load_atom_table(LoaderState* stp)
+load_atom_table(LoaderState* stp, ErtsAtomEncoding enc)
 {
     unsigned int i;
 
@@ -1371,7 +1394,7 @@ load_atom_table(LoaderState* stp)
 
 	GetByte(stp, n);
 	GetString(stp, atom, n);
-	stp->atom[i] = erts_atom_put(atom, n, ERTS_ATOM_ENC_LATIN1, 1);
+	stp->atom[i] = erts_atom_put(atom, n, enc, 1);
     }
 
     /*
@@ -5937,7 +5960,7 @@ code_get_chunk_2(BIF_ALIST_2)
 	goto error;
     }
     if (!init_iff_file(stp, start, binary_size(Bin)) ||
-	!scan_iff_file(stp, &chunk, 1, 1) ||
+	!scan_iff_file(stp, &chunk, 1) ||
 	stp->chunks[0].start == NULL) {
 	res = am_undefined;
 	goto done;
@@ -5986,7 +6009,7 @@ code_module_md5_1(BIF_ALIST_1)
     }
     stp->module = THE_NON_VALUE; /* Suppress diagnostiscs */
     if (!init_iff_file(stp, bytes, binary_size(Bin)) ||
-	!scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY) ||
+	!scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES) ||
 	!verify_chunks(stp)) {
 	res = am_undefined;
 	goto done;
@@ -6335,7 +6358,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
     if (!init_iff_file(stp, bytes, size)) {
 	goto error;
     }
-    if (!scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY) ||
+    if (!scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES) ||
 	!verify_chunks(stp)) {
 	goto error;
     }
@@ -6343,9 +6366,16 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
     if (!read_code_header(stp)) {
 	goto error;
     }
-    define_file(stp, "atom table", ATOM_CHUNK);
-    if (!load_atom_table(stp)) {
-	goto error;
+    if (stp->chunks[UTF8_ATOM_CHUNK].size > 0) {
+        define_file(stp, "utf8 atom table", UTF8_ATOM_CHUNK);
+        if (!load_atom_table(stp, ERTS_ATOM_ENC_UTF8)) {
+            goto error;
+        }
+    } else {
+        define_file(stp, "atom table", ATOM_CHUNK);
+        if (!load_atom_table(stp, ERTS_ATOM_ENC_LATIN1)) {
+            goto error;
+        }
     }
     define_file(stp, "export table", EXP_CHUNK);
     if (!stub_read_export_table(stp)) {
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index d886c2985e..95bf13c07c 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3022,8 +3022,8 @@ BIF_RETTYPE atom_to_list_1(BIF_ALIST_1)
 BIF_RETTYPE list_to_atom_1(BIF_ALIST_1)
 {
     Eterm res;
-    char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_CHARACTERS);
-    Sint i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS);
+    byte *buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_SZ_LIMIT);
+    Sint i = erts_unicode_list_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS);
 
     if (i < 0) {
 	erts_free(ERTS_ALC_T_TMP, (void *) buf);
@@ -3033,7 +3033,7 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1)
 	}
 	BIF_ERROR(BIF_P, BADARG);
     }
-    res = erts_atom_put((byte *) buf, i, ERTS_ATOM_ENC_LATIN1, 1);
+    res = erts_atom_put(buf, i, ERTS_ATOM_ENC_UTF8, 1);
     ASSERT(is_atom(res));
     erts_free(ERTS_ALC_T_TMP, (void *) buf);
     BIF_RET(res);
@@ -3043,17 +3043,17 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1)
  
 BIF_RETTYPE list_to_existing_atom_1(BIF_ALIST_1)
 {
-    Sint i;
-    char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_CHARACTERS);
+    byte *buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_SZ_LIMIT);
+    Sint i = erts_unicode_list_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS);
 
-    if ((i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS)) < 0) {
+    if (i < 0) {
     error:
 	erts_free(ERTS_ALC_T_TMP, (void *) buf);
 	BIF_ERROR(BIF_P, BADARG);
     } else {
 	Eterm a;
 	
-	if (erts_atom_get(buf, i, &a, ERTS_ATOM_ENC_LATIN1)) {
+	if (erts_atom_get((char *) buf, i, &a, ERTS_ATOM_ENC_UTF8)) {
 	    erts_free(ERTS_ALC_T_TMP, (void *) buf);
 	    BIF_RET(a);
 	} else {
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index bd5e1482fb..8919898181 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -1890,74 +1890,57 @@ binary_to_atom(Process* proc, Eterm bin, Eterm enc, int must_exist)
     byte* bytes;
     byte *temp_alloc = NULL;
     Uint bin_size;
+    Eterm a;
 
     if ((bytes = erts_get_aligned_binary_bytes(bin, &temp_alloc)) == 0) {
 	BIF_ERROR(proc, BADARG);
     }
     bin_size = binary_size(bin);
+
     if (enc == am_latin1) {
-	Eterm a;
-	if (bin_size > MAX_ATOM_CHARACTERS) {
-	system_limit:
-	    erts_free_aligned_binary_bytes(temp_alloc);
-	    BIF_ERROR(proc, SYSTEM_LIMIT);
-	}
 	if (!must_exist) {
-	    a = erts_atom_put((byte *) bytes,
-			      bin_size,
-			      ERTS_ATOM_ENC_LATIN1,
-			      0);
-    	    erts_free_aligned_binary_bytes(temp_alloc);
-	    if (is_non_value(a))
-		goto badarg;
-	    BIF_RET(a);
-	} else if (erts_atom_get((char *)bytes, bin_size, &a, ERTS_ATOM_ENC_LATIN1)) {
-	    erts_free_aligned_binary_bytes(temp_alloc);
-	    BIF_RET(a);
-	} else {
+	    int lix = erts_atom_put_index((byte *) bytes,
+					  bin_size,
+					  ERTS_ATOM_ENC_LATIN1,
+					  0);
+	    if (lix == ATOM_BAD_ENCODING_ERROR) {
+	    badarg:
+		erts_free_aligned_binary_bytes(temp_alloc);
+		BIF_ERROR(proc, BADARG);
+	    } else if (lix == ATOM_MAX_CHARS_ERROR) {
+	    system_limit:
+		erts_free_aligned_binary_bytes(temp_alloc);
+		BIF_ERROR(proc, SYSTEM_LIMIT);
+	    }
+
+	    a = make_atom(lix);
+	} else if (!erts_atom_get((char *)bytes, bin_size, &a, ERTS_ATOM_ENC_LATIN1)) {
 	    goto badarg;
 	}
-    } else if (enc == am_utf8 || enc == am_unicode) {
-	Eterm res;
-	Uint num_chars = 0;
-	const byte* p = bytes;
-	Uint left = bin_size;
 
-	while (left) {
-	    if (++num_chars > MAX_ATOM_CHARACTERS) {
+    } else if (enc == am_utf8 || enc == am_unicode) {
+	if (!must_exist) {
+	    int uix = erts_atom_put_index((byte *) bytes,
+					  bin_size,
+					  ERTS_ATOM_ENC_UTF8,
+					  0);
+	    if (uix == ATOM_BAD_ENCODING_ERROR) {
+		goto badarg;
+	    } else if (uix == ATOM_MAX_CHARS_ERROR) {
 		goto system_limit;
 	    }
-	    if ((p[0] & 0x80) == 0) {
-		++p;
-		--left;
-	    }
-	    else if (left >= 2
-		     && (p[0] & 0xFE) == 0xC2 /* only allow latin1 subset */
-		     && (p[1] & 0xC0) == 0x80) {
-		p += 2;
-		left -= 2;
-	    }
-	    else goto badarg;
-	}
 
-	if (!must_exist) {
-	    res = erts_atom_put((byte *) bytes,
-				bin_size,
-				ERTS_ATOM_ENC_UTF8,
-				0);
+	    a = make_atom(uix);
 	}
-	else if (!erts_atom_get((char*)bytes, bin_size, &res, ERTS_ATOM_ENC_UTF8)) {
+	else if (!erts_atom_get((char*)bytes, bin_size, &a, ERTS_ATOM_ENC_UTF8)) {
 	    goto badarg;
 	}
-	erts_free_aligned_binary_bytes(temp_alloc);
-	if (is_non_value(res))
-	    goto badarg;
-	BIF_RET(res);
     } else {
-    badarg:
-	erts_free_aligned_binary_bytes(temp_alloc);
-	BIF_ERROR(proc, BADARG);
+	goto badarg;
     }
+
+    erts_free_aligned_binary_bytes(temp_alloc);
+    BIF_RET(a);
 }
 
 BIF_RETTYPE binary_to_atom_2(BIF_ALIST_2)
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 2b2f3c5cdc..9f2b43d216 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1373,6 +1373,7 @@ int erts_utf8_to_latin1(byte* dest, const byte* source, int slen);
 
 void bin_write(fmtfn_t, void*, byte*, size_t);
 Sint intlist_to_buf(Eterm, char*, Sint); /* most callers pass plain char*'s */
+Sint erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len);
 
 struct Sint_buf {
 #if defined(ARCH_64)
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index ec502d5a78..36b818505c 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3923,6 +3923,68 @@ intlist_to_buf(Eterm list, char *buf, Sint len)
     return -2;			/* not enough space */
 }
 
+/* Fill buf with the contents of the unicode list.
+ * Return the number of bytes in the buffer,
+ * or -1 for type error,
+ * or -2 for not enough buffer space (buffer contains truncated result).
+ */
+Sint
+erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len)
+{
+    Eterm* listptr;
+    Sint sz = 0;
+
+    if (is_nil(list)) {
+	return 0;
+    }
+    if (is_not_list(list)) {
+	return -1;
+    }
+    listptr = list_val(list);
+
+    while (len-- > 0) {
+	Sint val;
+
+	if (is_not_small(CAR(listptr))) {
+	    return -1;
+	}
+	val = signed_val(CAR(listptr));
+	if (0 <= val && val < 0x80) {
+	    buf[sz] = val;
+	    sz++;
+	} else if (val < 0x800) {
+	    buf[sz+0] = 0xC0 | (val >> 6);
+	    buf[sz+1] = 0x80 | (val & 0x3F);
+	    sz += 2;
+	} else if (val < 0x10000UL) {
+	    if (0xD800 <= val && val <= 0xDFFF) {
+		return -1;
+	    }
+	    buf[sz+0] = 0xE0 | (val >> 12);
+	    buf[sz+1] = 0x80 | ((val >> 6) & 0x3F);
+	    buf[sz+2] = 0x80 | (val & 0x3F);
+	    sz += 3;
+	} else if (val < 0x110000) {
+	    buf[sz+0] = 0xF0 | (val >> 18);
+	    buf[sz+1] = 0x80 | ((val >> 12) & 0x3F);
+	    buf[sz+2] = 0x80 | ((val >> 6) & 0x3F);
+	    buf[sz+3] = 0x80 | (val & 0x3F);
+	    sz += 4;
+	} else {
+	    return -1;
+	}
+	list = CDR(listptr);
+	if (is_nil(list)) {
+	    return sz;
+	}
+	if (is_not_list(list)) {
+	    return -1;
+	}
+	listptr = list_val(list);
+    }
+    return -2;			/* not enough space */
+}
+
 /*
 ** Convert an integer to a byte list
 ** return pointer to converted stuff (need not to be at start of buf!)
diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl
index f70fb0e501..339c827602 100644
--- a/erts/emulator/test/bif_SUITE.erl
+++ b/erts/emulator/test/bif_SUITE.erl
@@ -26,7 +26,7 @@
 -export([all/0, suite/0,
 	 display/1, display_huge/0,
 	 erl_bif_types/1,guard_bifs_in_erl_bif_types/1,
-	 shadow_comments/1,
+	 shadow_comments/1,list_to_utf8_atom/1,
 	 specs/1,improper_bif_stubs/1,auto_imports/1,
 	 t_list_to_existing_atom/1,os_env/1,otp_7526/1,
 	 binary_to_atom/1,binary_to_existing_atom/1,
@@ -43,7 +43,7 @@ all() ->
     [erl_bif_types, guard_bifs_in_erl_bif_types, shadow_comments,
      specs, improper_bif_stubs, auto_imports,
      t_list_to_existing_atom, os_env, otp_7526,
-     display,
+     display, list_to_utf8_atom,
      atom_to_binary, binary_to_atom, binary_to_existing_atom,
      erl_crash_dump_bytes, min_max, erlang_halt, is_builtin,
      error_stacktrace, error_stacktrace_during_call_trace].
@@ -339,6 +339,38 @@ check_stub({_,F,A}, B) ->
 	    ct:fail(invalid_body)
     end.
 
+list_to_utf8_atom(Config) when is_list(Config) ->
+    'hello' = atom_roundtrip("hello"),
+    'こんにちは' = atom_roundtrip("こんにちは"),
+
+    %% Test all edge cases.
+    _ = atom_roundtrip([16#80]),
+    _ = atom_roundtrip([16#7F]),
+    _ = atom_roundtrip([16#FF]),
+    _ = atom_roundtrip([16#100]),
+    _ = atom_roundtrip([16#7FF]),
+    _ = atom_roundtrip([16#800]),
+    _ = atom_roundtrip([16#D7FF]),
+    atom_badarg([16#D800]),
+    atom_badarg([16#DFFF]),
+    _ = atom_roundtrip([16#E000]),
+    _ = atom_roundtrip([16#FFFF]),
+    _ = atom_roundtrip([16#1000]),
+    _ = atom_roundtrip([16#10FFFF]),
+    atom_badarg([16#110000]),
+    ok.
+
+atom_roundtrip(String) ->
+    Atom = list_to_atom(String),
+    Atom = list_to_existing_atom(String),
+    String = atom_to_list(Atom),
+    Atom.
+
+atom_badarg(String) ->
+    {'EXIT',{badarg,_}} = (catch list_to_atom(String)),
+    {'EXIT',{badarg,_}} = (catch list_to_existing_atom(String)),
+    ok.
+
 t_list_to_existing_atom(Config) when is_list(Config) ->
     all = list_to_existing_atom("all"),
     ?MODULE = list_to_existing_atom(?MODULE_STRING),
@@ -429,6 +461,8 @@ binary_to_atom(Config) when is_list(Config) ->
     Long = lists:seq(0, 254),
     LongAtom = list_to_atom(Long),
     LongBin = list_to_binary(Long),
+    UnicodeLongAtom = list_to_atom([$é || _ <- lists:seq(0, 254)]),
+    UnicodeLongBin = << <<"é"/utf8>> || _ <- lists:seq(0, 254)>>,
 
     %% latin1
     '' = test_binary_to_atom(<<>>, latin1),
@@ -440,12 +474,17 @@ binary_to_atom(Config) when is_list(Config) ->
     '' = test_binary_to_atom(<<>>, utf8),
     HalfLongAtom = test_binary_to_atom(HalfLongBin, utf8),
     HalfLongAtom = test_binary_to_atom(HalfLongBin, unicode),
+    UnicodeLongAtom = test_binary_to_atom(UnicodeLongBin, utf8),
+    UnicodeLongAtom = test_binary_to_atom(UnicodeLongBin, unicode),
     [] = [C || C <- lists:seq(128, 255),
 		     begin
 			 list_to_atom([C]) =/=
 			     test_binary_to_atom(<<C/utf8>>, utf8)
 		     end],
 
+    <<"こんにちは"/utf8>> =
+	atom_to_binary(test_binary_to_atom(<<"こんにちは"/utf8>>, utf8), utf8),
+
     %% badarg failures.
     fail_binary_to_atom(atom),
     fail_binary_to_atom(42),
@@ -464,10 +503,6 @@ binary_to_atom(Config) when is_list(Config) ->
     ?BADARG(binary_to_atom(id(<<255>>), utf8)),
     ?BADARG(binary_to_atom(id(<<255,0>>), utf8)),
     ?BADARG(binary_to_atom(id(<<16#C0,16#80>>), utf8)), %Overlong 0.
-    [?BADARG(binary_to_atom(<<C/utf8>>, utf8)) || C <- lists:seq(256, 16#D7FF)],
-    [?BADARG(binary_to_atom(<<C/utf8>>, utf8)) || C <- lists:seq(16#E000, 16#FFFD)],
-    [?BADARG(binary_to_atom(<<C/utf8>>, utf8)) || C <- lists:seq(16#10000, 16#8FFFF)],
-    [?BADARG(binary_to_atom(<<C/utf8>>, utf8)) || C <- lists:seq(16#90000, 16#10FFFF)],
 
     %% system_limit failures.
     ?SYS_LIMIT(binary_to_atom(id(<<0:512/unit:8,255>>), utf8)),
diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index b29520ab9f..d07166ed98 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -296,16 +296,16 @@ get_chunk(Config) when is_list(Config) ->
     {ok,my_code_test,Code} = compile:file(File, [binary]),
 
     %% Should work.
-    Chunk = get_chunk_ok("Atom", Code),
-    Chunk = get_chunk_ok("Atom", make_sub_binary(Code)),
-    Chunk = get_chunk_ok("Atom", make_unaligned_sub_binary(Code)),
+    Chunk = get_chunk_ok("AtU8", Code),
+    Chunk = get_chunk_ok("AtU8", make_sub_binary(Code)),
+    Chunk = get_chunk_ok("AtU8", make_unaligned_sub_binary(Code)),
 
     %% Should fail.
-    {'EXIT',{badarg,_}} = (catch code:get_chunk(bit_sized_binary(Code), "Atom")),
+    {'EXIT',{badarg,_}} = (catch code:get_chunk(bit_sized_binary(Code), "AtU8")),
     {'EXIT',{badarg,_}} = (catch code:get_chunk(Code, "bad chunk id")),
 
     %% Invalid beam code or missing chunk should return 'undefined'.
-    undefined = code:get_chunk(<<"not a beam module">>, "Atom"),
+    undefined = code:get_chunk(<<"not a beam module">>, "AtU8"),
     undefined = code:get_chunk(Code, "XXXX"),
 
     ok.
-- 
cgit v1.2.3


From afbec46949f0d31448386234efa89cec78b905e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 16 Jan 2017 11:52:39 +0100
Subject: erts: Add OS signal tests

---
 erts/emulator/test/Makefile                        |   1 +
 erts/emulator/test/os_signal_SUITE.erl             | 354 +++++++++++++++++++++
 .../test/os_signal_SUITE_data/Makefile.src         |   6 +
 .../test/os_signal_SUITE_data/os_signal_nif.c      |  66 ++++
 4 files changed, 427 insertions(+)
 create mode 100644 erts/emulator/test/os_signal_SUITE.erl
 create mode 100644 erts/emulator/test/os_signal_SUITE_data/Makefile.src
 create mode 100644 erts/emulator/test/os_signal_SUITE_data/os_signal_nif.c

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index 2e48c475d5..5ec6ff1901 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -85,6 +85,7 @@ MODULES= \
 	num_bif_SUITE \
 	message_queue_data_SUITE \
 	op_SUITE \
+	os_signal_SUITE \
 	port_SUITE \
 	port_bif_SUITE \
 	process_SUITE \
diff --git a/erts/emulator/test/os_signal_SUITE.erl b/erts/emulator/test/os_signal_SUITE.erl
new file mode 100644
index 0000000000..2b9343663d
--- /dev/null
+++ b/erts/emulator/test/os_signal_SUITE.erl
@@ -0,0 +1,354 @@
+%%
+%% %CopyrightBegin%
+%% 
+%% Copyright Ericsson AB 2017. All Rights Reserved.
+%% 
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%% 
+%% %CopyrightEnd%
+%%
+
+%%
+%% File:    os_signal_SUITE.erl
+%% Author:  Björn-Egil Dahlberg
+%% Created: 2017-01-13
+%%
+
+-module(os_signal_SUITE).
+
+-include_lib("common_test/include/ct.hrl").
+-export([all/0, suite/0]).
+-export([init_per_testcase/2, end_per_testcase/2]).
+-export([init_per_suite/1, end_per_suite/1]).
+
+-export([set_alarm/1, fork/0, get_exit_code/1]).
+
+% Test cases
+-export([set_unset/1,
+         t_sighup/1,
+         t_sigusr1/1,
+         t_sigusr2/1,
+         t_sigterm/1,
+         t_sigalrm/1,
+         t_sigchld/1,
+         t_sigchld_fork/1]).
+
+-define(signal_server, erl_signal_server).
+
+suite() ->
+    [{ct_hooks,[ts_install_cth]},
+     {timetrap, {minutes, 2}}].
+
+all() ->
+    case os:type() of
+        {win32, _} -> [];
+        _ -> [set_unset,
+              t_sighup,
+              t_sigusr1,
+              t_sigusr2,
+              t_sigterm,
+              t_sigalrm,
+              t_sigchld,
+              t_sigchld_fork]
+    end.
+
+init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
+    Pid = erlang:whereis(?signal_server),
+    true = erlang:unregister(?signal_server),
+    [{signal_server, Pid}|Config].
+
+end_per_testcase(_Func, Config) ->
+    case proplists:get_value(signal_server, Config) of
+        undefined -> ok;
+        Pid ->
+            true = erlang:register(?signal_server, Pid),
+            ok
+    end.
+
+init_per_suite(Config) ->
+    load_nif(Config),
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+%% tests
+
+set_unset(_Config) ->
+    Signals = [sighup, %sigint,
+               sigquit, sigill,
+               sigabrt,
+               sigalrm, sigterm,
+               sigusr1, sigusr2,
+               sigchld,
+               sigstop, sigtstp],
+    F1 = fun(Sig) -> true = erts_internal:set_signal(Sig,handle) end,
+    F2 = fun(Sig) -> true = erts_internal:set_signal(Sig,default) end,
+    F3 = fun(Sig) -> true = erts_internal:set_signal(Sig,ignore) end,
+    %% set handle
+    ok = lists:foreach(F1, Signals),
+    %% set ignore
+    ok = lists:foreach(F2, Signals),
+    %% set default
+    ok = lists:foreach(F3, Signals),
+    ok.
+
+t_sighup(_Config) ->
+    Pid1 = setup_service(),
+    OsPid = os:getpid(),
+    erts_internal:set_signal(sighup, handle),
+    ok = kill("HUP", OsPid),
+    ok = kill("HUP", OsPid),
+    ok = kill("HUP", OsPid),
+    Msgs1 = fetch_msgs(Pid1),
+    io:format("Msgs1: ~p~n", [Msgs1]),
+    [{notify,sighup},
+     {notify,sighup},
+     {notify,sighup}] = Msgs1,
+    %% no proc
+    ok = kill("HUP", OsPid),
+    ok = kill("HUP", OsPid),
+    ok = kill("HUP", OsPid),
+    %% ignore
+    Pid2 = setup_service(),
+    erts_internal:set_signal(sighup, ignore),
+    ok = kill("HUP", OsPid),
+    ok = kill("HUP", OsPid),
+    ok = kill("HUP", OsPid),
+    Msgs2 = fetch_msgs(Pid2),
+    io:format("Msgs2: ~p~n", [Msgs2]),
+    [] = Msgs2,
+    %% reset to handle (it's the default)
+    erts_internal:set_signal(sighup, handle),
+    ok.
+
+t_sigusr1(_Config) ->
+    Pid1 = setup_service(),
+    OsPid = os:getpid(),
+    erts_internal:set_signal(sigusr1, handle),
+    ok = kill("USR1", OsPid),
+    ok = kill("USR1", OsPid),
+    ok = kill("USR1", OsPid),
+    Msgs1 = fetch_msgs(Pid1),
+    io:format("Msgs1: ~p~n", [Msgs1]),
+    [{notify,sigusr1},
+     {notify,sigusr1},
+     {notify,sigusr1}] = Msgs1,
+    %% no proc
+    ok = kill("USR1", OsPid),
+    ok = kill("USR1", OsPid),
+    ok = kill("USR1", OsPid),
+    %% ignore
+    Pid2 = setup_service(),
+    erts_internal:set_signal(sigusr1, ignore),
+    ok = kill("USR1", OsPid),
+    ok = kill("USR1", OsPid),
+    ok = kill("USR1", OsPid),
+    Msgs2 = fetch_msgs(Pid2),
+    io:format("Msgs2: ~p~n", [Msgs2]),
+    [] = Msgs2,
+    %% reset to ignore (it's the default)
+    erts_internal:set_signal(sigusr1, handle),
+    ok.
+
+t_sigusr2(_Config) ->
+    Pid1 = setup_service(),
+    OsPid = os:getpid(),
+    erts_internal:set_signal(sigusr2, handle),
+    ok = kill("USR2", OsPid),
+    ok = kill("USR2", OsPid),
+    ok = kill("USR2", OsPid),
+    Msgs1 = fetch_msgs(Pid1),
+    io:format("Msgs1: ~p~n", [Msgs1]),
+    [{notify,sigusr2},
+     {notify,sigusr2},
+     {notify,sigusr2}] = Msgs1,
+    %% no proc
+    ok = kill("USR2", OsPid),
+    ok = kill("USR2", OsPid),
+    ok = kill("USR2", OsPid),
+    %% ignore
+    Pid2 = setup_service(),
+    erts_internal:set_signal(sigusr2, ignore),
+    ok = kill("USR2", OsPid),
+    ok = kill("USR2", OsPid),
+    ok = kill("USR2", OsPid),
+    Msgs2 = fetch_msgs(Pid2),
+    io:format("Msgs2: ~p~n", [Msgs2]),
+    [] = Msgs2,
+    %% reset to ignore (it's the default)
+    erts_internal:set_signal(sigusr2, ignore),
+    ok.
+
+t_sigterm(_Config) ->
+    Pid1 = setup_service(),
+    OsPid = os:getpid(),
+    erts_internal:set_signal(sigterm, handle),
+    ok = kill("TERM", OsPid),
+    ok = kill("TERM", OsPid),
+    ok = kill("TERM", OsPid),
+    Msgs1 = fetch_msgs(Pid1),
+    io:format("Msgs1: ~p~n", [Msgs1]),
+    [{notify,sigterm},
+     {notify,sigterm},
+     {notify,sigterm}] = Msgs1,
+    %% no proc
+    ok = kill("TERM", OsPid),
+    ok = kill("TERM", OsPid),
+    ok = kill("TERM", OsPid),
+    %% ignore
+    Pid2 = setup_service(),
+    erts_internal:set_signal(sigterm, ignore),
+    ok = kill("TERM", OsPid),
+    ok = kill("TERM", OsPid),
+    ok = kill("TERM", OsPid),
+    Msgs2 = fetch_msgs(Pid2),
+    io:format("Msgs2: ~p~n", [Msgs2]),
+    [] = Msgs2,
+    %% reset to handle (it's the default)
+    erts_internal:set_signal(sigterm, handle),
+    ok.
+
+t_sigchld(_Config) ->
+    Pid1 = setup_service(),
+    OsPid = os:getpid(),
+    erts_internal:set_signal(sigchld, handle),
+    ok = kill("CHLD", OsPid),
+    ok = kill("CHLD", OsPid),
+    ok = kill("CHLD", OsPid),
+    Msgs1 = fetch_msgs(Pid1),
+    io:format("Msgs1: ~p~n", [Msgs1]),
+    [{notify,sigchld},
+     {notify,sigchld},
+     {notify,sigchld}] = Msgs1,
+    %% no proc
+    ok = kill("CHLD", OsPid),
+    ok = kill("CHLD", OsPid),
+    ok = kill("CHLD", OsPid),
+    %% ignore
+    Pid2 = setup_service(),
+    erts_internal:set_signal(sigchld, ignore),
+    ok = kill("CHLD", OsPid),
+    ok = kill("CHLD", OsPid),
+    ok = kill("CHLD", OsPid),
+    Msgs2 = fetch_msgs(Pid2),
+    io:format("Msgs2: ~p~n", [Msgs2]),
+    [] = Msgs2,
+    %% reset to handle (it's the default)
+    erts_internal:set_signal(sigchld, ignore),
+    ok.
+
+
+t_sigalrm(_Config) ->
+    Pid1 = setup_service(),
+    true = erts_internal:set_signal(sigalrm, handle),
+    ok = os_signal_SUITE:set_alarm(1),
+    receive after 3000 -> ok end,
+    Msgs1 = fetch_msgs(Pid1),
+    [{notify,sigalrm}] = Msgs1,
+    io:format("Msgs1: ~p~n", [Msgs1]),
+    erts_internal:set_signal(sigalrm, ignore),
+    Pid2 = setup_service(),
+    ok = os_signal_SUITE:set_alarm(1),
+    receive after 3000 -> ok end,
+    Msgs2 = fetch_msgs(Pid2),
+    [] = Msgs2,
+    io:format("Msgs2: ~p~n", [Msgs2]),
+    Pid3 = setup_service(),
+    erts_internal:set_signal(sigalrm, handle),
+    ok = os_signal_SUITE:set_alarm(1),
+    receive after 3000 -> ok end,
+    Msgs3 = fetch_msgs(Pid3),
+    [{notify,sigalrm}] = Msgs3,
+    io:format("Msgs3: ~p~n", [Msgs3]),
+    erts_internal:set_signal(sigalrm, ignore),
+    ok.
+
+t_sigchld_fork(_Config) ->
+    Pid1 = setup_service(),
+    true = erts_internal:set_signal(sigchld, handle),
+    {ok,OsPid} = os_signal_SUITE:fork(),
+    receive after 3000 -> ok end,
+    Msgs1 = fetch_msgs(Pid1),
+    io:format("Msgs1: ~p~n", [Msgs1]),
+    [{notify,sigchld}] = Msgs1,
+    {ok,Status} = os_signal_SUITE:get_exit_code(OsPid),
+    io:format("exit status from ~w : ~w~n", [OsPid,Status]),
+    42 = Status,
+    %% reset to ignore (it's the default)
+    erts_internal:set_signal(sigchld, ignore),
+    ok.
+
+
+%% nif stubs
+
+set_alarm(_Secs) -> no.
+fork() -> no.
+get_exit_code(_OsPid) -> no.
+
+%% aux
+
+setup_service() ->
+    Pid = spawn_link(fun msgs/0),
+    true = erlang:register(?signal_server, Pid),
+    Pid.
+
+msgs() ->
+    msgs([]).
+msgs(Ms) ->
+    receive
+        {Pid, fetch_msgs} -> Pid ! {self(), lists:reverse(Ms)};
+        Msg ->
+            msgs([Msg|Ms])
+    end.
+
+fetch_msgs(Pid) ->
+    Pid ! {self(), fetch_msgs},
+    receive {Pid, Msgs} -> Msgs end.
+
+kill(Signal, Pid) ->
+    {0,_} = run("kill", ["-s", Signal, Pid]),
+    receive after 200 -> ok end,
+    ok.
+
+load_nif(Config) ->
+    Path = proplists:get_value(data_dir, Config),
+    ok = erlang:load_nif(filename:join(Path,"os_signal_nif"), 0).
+
+run(Program0, Args) -> run(".", Program0, Args).
+run(Cwd, Program0, Args) when is_list(Cwd) ->
+    Program = case os:find_executable(Program0) of
+                  Path when is_list(Path) ->
+                      Path;
+                  false ->
+                      exit(no)
+              end,
+    Options = [{args,Args},binary,exit_status,stderr_to_stdout,
+               {line,4096}, {cd, Cwd}],
+    try open_port({spawn_executable,Program}, Options) of
+        Port ->
+            run_loop(Port, [])
+    catch
+        error:_ ->
+                      exit(no)
+    end.
+
+run_loop(Port, Output) ->
+    receive
+        {Port,{exit_status,Status}} ->
+            {Status,lists:reverse(Output)};
+        {Port,{data,{eol,Bin}}} ->
+            run_loop(Port, [Bin|Output]);
+        _Msg ->
+            run_loop(Port, Output)
+    end.
diff --git a/erts/emulator/test/os_signal_SUITE_data/Makefile.src b/erts/emulator/test/os_signal_SUITE_data/Makefile.src
new file mode 100644
index 0000000000..a7f5cdbba5
--- /dev/null
+++ b/erts/emulator/test/os_signal_SUITE_data/Makefile.src
@@ -0,0 +1,6 @@
+
+NIF_LIBS = os_signal_nif@dll@
+
+all: $(NIF_LIBS)
+
+@SHLIB_RULES@
diff --git a/erts/emulator/test/os_signal_SUITE_data/os_signal_nif.c b/erts/emulator/test/os_signal_SUITE_data/os_signal_nif.c
new file mode 100644
index 0000000000..78e1348383
--- /dev/null
+++ b/erts/emulator/test/os_signal_SUITE_data/os_signal_nif.c
@@ -0,0 +1,66 @@
+#include <sys/wait.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <stdio.h>
+
+#include <erl_nif.h>
+
+static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+{
+    return 0;
+}
+
+static ERL_NIF_TERM set_alarm(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    int t;
+    if (!enif_get_int(env, argv[0], &t)) {
+        return enif_make_badarg(env);
+    }
+
+    alarm(t);
+
+    return enif_make_atom(env, "ok");
+}
+
+static ERL_NIF_TERM fork_0(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    pid_t pid;
+
+    pid = fork();
+
+    if (pid == 0) {
+        /* child */
+        exit(42);
+    }
+
+    return enif_make_tuple(env, 2,
+            enif_make_atom(env, "ok"),
+            enif_make_int(env, (int)pid));
+}
+
+static ERL_NIF_TERM get_exit_code(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    int x;
+    pid_t pid;
+    if (!enif_get_int(env, argv[0], &x)) {
+        return enif_make_badarg(env);
+    }
+
+    pid = (pid_t) x;
+
+    waitpid(pid, &x, 0);
+
+    return enif_make_tuple(env, 2,
+            enif_make_atom(env, "ok"),
+            enif_make_int(env, WEXITSTATUS(x)));
+}
+
+
+static ErlNifFunc nif_funcs[] =
+{
+    {"set_alarm", 1, set_alarm},
+    {"fork", 0, fork_0},
+    {"get_exit_code", 1, get_exit_code}
+};
+
+ERL_NIF_INIT(os_signal_SUITE,nif_funcs,load,NULL,NULL,NULL)
-- 
cgit v1.2.3


From 120f04387ade07ef5b8b6d20a04de7d21e0c40ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 5 Jan 2017 17:17:52 +0100
Subject: erts: Use generic signal handler

---
 erts/emulator/beam/atom.names |  14 ++-
 erts/emulator/beam/bif.c      |  14 ++-
 erts/emulator/beam/bif.tab    |   1 +
 erts/emulator/beam/global.h   |   3 +
 erts/emulator/sys/unix/sys.c  | 251 ++++++++++++++++++++++--------------------
 erts/emulator/sys/win32/sys.c |   4 +
 6 files changed, 165 insertions(+), 122 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 668abfaaee..72a7b9e312 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -194,6 +194,7 @@ atom current_stacktrace
 atom data
 atom debug_flags
 atom decimals
+atom default
 atom delay_trap
 atom dexit
 atom depth
@@ -306,6 +307,7 @@ atom global
 atom Gt='>'
 atom grun
 atom group_leader
+atom handle
 atom have_dt_utag
 atom heap_block_size
 atom heap_size
@@ -566,7 +568,7 @@ atom running_procs
 atom runtime
 atom safe
 atom save_calls
-atom scheduler 
+atom scheduler
 atom scheduler_id
 atom schedulers_online
 atom scheme
@@ -594,6 +596,16 @@ atom shared
 atom sighup
 atom sigterm
 atom sigusr1
+atom sigusr2
+atom sigill
+atom sigchld
+atom sigabrt
+atom sigalrm
+atom sigstop
+atom sigint
+atom sigsegv
+atom sigtstp
+atom sigquit
 atom silent
 atom size
 atom sl_alloc
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index d886c2985e..452bfef71a 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -5273,7 +5273,7 @@ BIF_RETTYPE dt_restore_tag_1(BIF_ALIST_1)
 	    SEQ_TRACE_TOKEN(BIF_P) = am_have_dt_utag;
 	}
     }
-#else    
+#else
     if (BIF_ARG_1 != am_true) {
 	BIF_ERROR(BIF_P,BADARG);
     }
@@ -5281,4 +5281,16 @@ BIF_RETTYPE dt_restore_tag_1(BIF_ALIST_1)
     BIF_RET(am_true);
 }
 
+BIF_RETTYPE erts_internal_set_signal_2(BIF_ALIST_2) {
+    if (is_atom(BIF_ARG_1) && ((BIF_ARG_2 == am_ignore) ||
+                               (BIF_ARG_2 == am_default) ||
+                               (BIF_ARG_2 == am_handle))) {
+        if (!erts_set_signal(BIF_ARG_1, BIF_ARG_2))
+            goto error;
+
+        BIF_RET(am_true);
+    }
 
+error:
+    BIF_ERROR(BIF_P, BADARG);
+}
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 32600f4338..318a4fd264 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -668,6 +668,7 @@ gcbif erlang:ceil/1
 bif math:floor/1
 bif math:ceil/1
 bif math:fmod/2
+bif erts_internal:set_signal/2
 
 #
 # Obsolete
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 2b2f3c5cdc..f0f959e97a 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1070,6 +1070,9 @@ void print_process_info(fmtfn_t, void *, Process*);
 void info(fmtfn_t, void *);
 void loaded(fmtfn_t, void *);
 
+/* sighandler sys.c */
+int erts_set_signal(Eterm signal, Eterm type);
+
 /* erl_arith.c */
 double erts_get_positive_zero_float(void);
 
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index de9c1b2ca0..4175dc6a41 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -107,7 +107,7 @@ static volatile int have_prepared_crash_dump;
 erts_smp_atomic_t sys_misc_mem_sz;
 
 #if defined(ERTS_SMP)
-static void smp_sig_notify(char c);
+static void smp_sig_notify(int signum);
 static int sig_notify_fds[2] = {-1, -1};
 
 #if !defined(ETHR_UNUSABLE_SIGUSRX) && defined(ERTS_THR_HAVE_SIG_FUNCS)
@@ -395,14 +395,6 @@ erts_sys_pre_init(void)
     /* After creation in parent */
     eid.thread_create_parent_func = thr_create_cleanup,
 
-#ifdef ERTS_THR_HAVE_SIG_FUNCS
-    sigemptyset(&thr_create_sigmask);
-    sigaddset(&thr_create_sigmask, SIGINT);   /* block interrupt */
-    sigaddset(&thr_create_sigmask, SIGTERM);  /* block terminate signal */
-    sigaddset(&thr_create_sigmask, SIGHUP);   /* block sighups */
-    sigaddset(&thr_create_sigmask, SIGUSR1);  /* block user defined signal */
-#endif
-
     erts_thr_init(&eid);
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -654,33 +646,15 @@ break_requested(void)
   ERTS_CHK_IO_AS_INTR(); /* Make sure we don't sleep in poll */
 }
 
-/* set up signal handlers for break and quit */
-#if (defined(SIG_SIGSET) || defined(SIG_SIGNAL))
-static RETSIGTYPE request_break(void)
-#else
 static RETSIGTYPE request_break(int signum)
-#endif
 {
 #ifdef ERTS_SMP
-    smp_sig_notify('I');
+    smp_sig_notify(signum);
 #else
     break_requested();
 #endif
 }
 
-#if (defined(SIG_SIGSET) || defined(SIG_SIGNAL))
-static RETSIGTYPE request_stop(void)
-#else
-static RETSIGTYPE request_stop(int signum)
-#endif
-{
-#ifdef ERTS_SMP
-    smp_sig_notify('T');
-#else
-    signal_notify_requested(am_sigterm);
-#endif
-}
-
 #ifdef ETHR_UNUSABLE_SIGUSRX
 #warning "Unusable SIGUSR1 & SIGUSR2. Disabling use of these signals"
 
@@ -701,19 +675,6 @@ sys_thr_resume(erts_tid_t tid) {
 }
 #endif
 
-#if (defined(SIG_SIGSET) || defined(SIG_SIGNAL))
-static RETSIGTYPE user_signal1(void)
-#else
-static RETSIGTYPE user_signal1(int signum)
-#endif
-{
-#ifdef ERTS_SMP
-   smp_sig_notify('1');
-#else
-   signal_notify_requested(am_sigusr1);
-#endif
-}
-
 #ifdef ERTS_SYS_SUSPEND_SIGNAL
 #if (defined(SIG_SIGSET) || defined(SIG_SIGNAL))
 static RETSIGTYPE suspend_signal(void)
@@ -733,43 +694,101 @@ static RETSIGTYPE suspend_signal(int signum)
 
 #endif /* #ifndef ETHR_UNUSABLE_SIGUSRX */
 
-static void
-quit_requested(void)
+/*
+ Signal      Action   Comment
+ ─────────────────────────────────────────────────────────────
+  SIGHUP     Term     Hangup detected on controlling terminal or death of controlling process
+  SIGINT     Term     Interrupt from keyboard
+  SIGQUIT    Core     Quit from keyboard
+  SIGILL     Core     Illegal Instruction
+  SIGABRT    Core     Abort signal from abort(3)
+ !SIGFPE     Core     Floating point exception
+ !SIGKILL    Term     Kill signal
+ !SIGSEGV    Core     Invalid memory reference
+ !SIGPIPE    Term     Broken pipe: write to pipe with no readers
+  SIGALRM    Term     Timer signal from alarm(2)
+  SIGTERM    Term     Termination signal
+  SIGUSR1    Term     User-defined signal 1
+  SIGUSR2    Term     User-defined signal 2
+ !SIGCHLD    Ign      Child stopped or terminated
+ !SIGCONT    Cont     Continue if stopped
+  SIGSTOP    Stop     Stop process
+  SIGTSTP    Stop     Stop typed at terminal
+ !SIGTTIN    Stop     Terminal input for background process
+ !SIGTTOU    Stop     Terminal output for background process
+*/
+
+
+static ERTS_INLINE int
+signalterm_to_signum(Eterm signal)
 {
-    erts_exit(ERTS_INTR_EXIT, "");
+    switch (signal) {
+    case am_sighup:  return SIGHUP;
+    case am_sigint:  return SIGINT;
+    case am_sigquit: return SIGQUIT;
+    case am_sigill:  return SIGILL;
+    case am_sigabrt: return SIGABRT;
+    /* case am_sigsegv: return SIGSEGV; */
+    case am_sigalrm: return SIGALRM;
+    case am_sigterm: return SIGTERM;
+    case am_sigusr1: return SIGUSR1;
+    case am_sigusr2: return SIGUSR2;
+    case am_sigchld: return SIGCHLD;
+    case am_sigstop: return SIGSTOP;
+    case am_sigtstp: return SIGTSTP;
+    default:         return 0;
+    }
 }
 
-#if (defined(SIG_SIGSET) || defined(SIG_SIGNAL))
-static RETSIGTYPE do_quit(void)
-#else
-static RETSIGTYPE do_quit(int signum)
-#endif
+static ERTS_INLINE Eterm
+signum_to_signalterm(int signum)
 {
-#ifdef ERTS_SMP
-    smp_sig_notify('Q');
-#else
-    quit_requested();
-#endif
+    switch (signum) {
+    case SIGHUP:  return am_sighup;
+    case SIGINT:  return am_sigint;    /* ^c */
+    case SIGILL:  return am_sigill;
+    case SIGABRT: return am_sigabrt;
+    /* case SIGSEGV: return am_sigsegv; */
+    case SIGALRM: return am_sigalrm;
+    case SIGTERM: return am_sigterm;
+    case SIGQUIT: return am_sigquit;   /* ^\ */
+    case SIGUSR1: return am_sigusr1;
+    case SIGUSR2: return am_sigusr2;
+    case SIGCHLD: return am_sigchld;
+    case SIGSTOP: return am_sigstop;
+    case SIGTSTP: return am_sigtstp;   /* ^z */
+    default:      return am_error;
+    }
 }
 
-#if (defined(SIG_SIGSET) || defined(SIG_SIGNAL))
-static RETSIGTYPE request_sighup(void)
-#else
-static RETSIGTYPE request_sighup(int signum)
-#endif
+static RETSIGTYPE generic_signal_handler(int signum)
 {
 #ifdef ERTS_SMP
-    smp_sig_notify('H');
+    smp_sig_notify(signum);
 #else
-    signal_notify_requested(am_sighup);
+    Eterm signal = signum_to_signalterm(signum);
+    signal_notify_requested(signal);
 #endif
 }
 
+int erts_set_signal(Eterm signal, Eterm type) {
+    int signum;
+    if ((signum = signalterm_to_signum(signal)) > 0) {
+        if (type == am_ignore) {
+            sys_signal(signum, SIG_IGN);
+        } else if (type == am_default) {
+            sys_signal(signum, SIG_DFL);
+        } else {
+            sys_signal(signum, generic_signal_handler);
+        }
+        return 1;
+    }
+    return 0;
+}
+
 /* Disable break */
 void erts_set_ignore_break(void) {
     sys_signal(SIGINT,  SIG_IGN);
-    sys_signal(SIGTERM, SIG_IGN);
-    sys_signal(SIGHUP,  SIG_IGN);
     sys_signal(SIGQUIT, SIG_IGN);
     sys_signal(SIGTSTP, SIG_IGN);
 }
@@ -794,13 +813,13 @@ void erts_replace_intr(void) {
 
 void init_break_handler(void)
 {
-   sys_signal(SIGINT, request_break);
-   sys_signal(SIGTERM, request_stop);
-   sys_signal(SIGHUP, request_sighup);
+   sys_signal(SIGINT,  request_break);
+   sys_signal(SIGTERM, generic_signal_handler);
+   sys_signal(SIGHUP,  generic_signal_handler);
 #ifndef ETHR_UNUSABLE_SIGUSRX
-   sys_signal(SIGUSR1, user_signal1);
+   sys_signal(SIGUSR1, generic_signal_handler);
 #endif /* #ifndef ETHR_UNUSABLE_SIGUSRX */
-   sys_signal(SIGQUIT, do_quit);
+   sys_signal(SIGQUIT, generic_signal_handler);
 }
 
 void sys_init_suspend_handler(void)
@@ -1216,14 +1235,14 @@ erl_sys_schedule(int runnable)
 static erts_smp_tid_t sig_dispatcher_tid;
 
 static void
-smp_sig_notify(char c)
+smp_sig_notify(int signum)
 {
     int res;
     do {
 	/* write() is async-signal safe (according to posix) */
-	res = write(sig_notify_fds[1], &c, 1);
+	res = write(sig_notify_fds[1], &signum, sizeof(int));
     } while (res < 0 && errno == EINTR);
-    if (res != 1) {
+    if (res != sizeof(int)) {
 	char msg[] =
 	    "smp_sig_notify(): Failed to notify signal-dispatcher thread "
 	    "about received signal";
@@ -1239,63 +1258,55 @@ signal_dispatcher_thread_func(void *unused)
     erts_lc_set_thread_name("signal_dispatcher");
 #endif
     while (1) {
-	char buf[32];
-	int res, i;
+        union {int signum; char buf[4];} sb;
+        Eterm signal;
+	int res, i = 0;
 	/* Block on read() waiting for a signal notification to arrive... */
-	res = read(sig_notify_fds[0], (void *) &buf[0], 32);
+
+        do {
+            res = read(sig_notify_fds[0], (void *) &sb.buf[i], sizeof(int) - i);
+            i += res;
+        } while ((i != sizeof(int) && res >= 0) || (res < 0 && errno == EINTR));
+
 	if (res < 0) {
-	    if (errno == EINTR)
-		continue;
 	    erts_exit(ERTS_ABORT_EXIT,
 		     "signal-dispatcher thread got unexpected error: %s (%d)\n",
 		     erl_errno_id(errno),
 		     errno);
 	}
-	for (i = 0; i < res; i++) {
-	    /*
-	     * NOTE 1: The signal dispatcher thread should not do work
-	     *         that takes a substantial amount of time (except
-	     *         perhaps in test and debug builds). It needs to
-	     *         be responsive, i.e, it should only dispatch work
-	     *         to other threads.
-	     *
-	     * NOTE 2: The signal dispatcher thread is not a blockable
-	     *         thread (i.e., not a thread managed by the
-	     *         erl_thr_progress module). This is intentional.
-	     *         We want to be able to interrupt writing of a crash
-	     *         dump by hitting C-c twice. Since it isn't a
-	     *         blockable thread it is important that it doesn't
-	     *         change the state of any data that a blocking thread
-	     *         expects to have exclusive access to (unless the
-	     *         signal dispatcher itself explicitly is blocking all
-	     *         blockable threads).
-	     */
-	    switch (buf[i]) {
-	    case 0: /* Emulator initialized */
-                break;
-            case 'H': /* SIGHUP */
-                signal_notify_requested(am_sighup);
+        /*
+         * NOTE 1: The signal dispatcher thread should not do work
+         *         that takes a substantial amount of time (except
+         *         perhaps in test and debug builds). It needs to
+         *         be responsive, i.e, it should only dispatch work
+         *         to other threads.
+         *
+         * NOTE 2: The signal dispatcher thread is not a blockable
+         *         thread (i.e., not a thread managed by the
+         *         erl_thr_progress module). This is intentional.
+         *         We want to be able to interrupt writing of a crash
+         *         dump by hitting C-c twice. Since it isn't a
+         *         blockable thread it is important that it doesn't
+         *         change the state of any data that a blocking thread
+         *         expects to have exclusive access to (unless the
+         *         signal dispatcher itself explicitly is blocking all
+         *         blockable threads).
+         */
+        switch (sb.signum) {
+            case 0: continue;
+            case SIGINT:
+                break_requested();
                 break;
-	    case 'T': /* SIGTERM */
-                signal_notify_requested(am_sigterm);
-		break;
-	    case '1': /* SIGUSR1 */
-                signal_notify_requested(am_sigusr1);
-		break;
-	    case 'I': /* SIGINT */
-		break_requested();
-		break;
-	    case 'Q': /* SIGQUIT */
-		quit_requested();
-		break;
-	    default:
-		erts_exit(ERTS_ABORT_EXIT,
-			 "signal-dispatcher thread received unknown "
-			 "signal notification: '%c'\n",
-			 buf[i]);
-	    }
-	}
-	ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
+            default:
+                if ((signal = signum_to_signalterm(sb.signum)) == am_error) {
+                    erts_exit(ERTS_ABORT_EXIT,
+                            "signal-dispatcher thread received unknown "
+                            "signal notification: '%d'\n",
+                            sb.signum);
+                }
+                signal_notify_requested(signal);
+        }
+        ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
     }
     return NULL;
 }
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index f3881e0736..408b4f47ab 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -294,6 +294,10 @@ int erts_sys_prepare_crash_dump(int secs)
     return 0;
 }
 
+int erts_set_signal(Eterm signal, Eterm type) {
+    return 0;
+}
+
 static void
 init_console(void)
 {
-- 
cgit v1.2.3


From d3506c8dc3d355b7a62085da508b41866e431a37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 20 Jan 2017 14:27:40 +0100
Subject: erts: Do not enable SIGINT

---
 erts/emulator/sys/unix/sys.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 4175dc6a41..ce87e3c7a6 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -698,7 +698,7 @@ static RETSIGTYPE suspend_signal(int signum)
  Signal      Action   Comment
  ─────────────────────────────────────────────────────────────
   SIGHUP     Term     Hangup detected on controlling terminal or death of controlling process
-  SIGINT     Term     Interrupt from keyboard
+ !SIGINT     Term     Interrupt from keyboard
   SIGQUIT    Core     Quit from keyboard
   SIGILL     Core     Illegal Instruction
   SIGABRT    Core     Abort signal from abort(3)
@@ -724,7 +724,7 @@ signalterm_to_signum(Eterm signal)
 {
     switch (signal) {
     case am_sighup:  return SIGHUP;
-    case am_sigint:  return SIGINT;
+    /* case am_sigint:  return SIGINT; */
     case am_sigquit: return SIGQUIT;
     case am_sigill:  return SIGILL;
     case am_sigabrt: return SIGABRT;
@@ -745,7 +745,7 @@ signum_to_signalterm(int signum)
 {
     switch (signum) {
     case SIGHUP:  return am_sighup;
-    case SIGINT:  return am_sigint;    /* ^c */
+    /* case SIGINT:  return am_sigint; */    /* ^c */
     case SIGILL:  return am_sigill;
     case SIGABRT: return am_sigabrt;
     /* case SIGSEGV: return am_sigsegv; */
-- 
cgit v1.2.3


From 314fddc2e2e4cd8587ce00dfce328245a315832f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 20 Jan 2017 16:28:42 +0100
Subject: erts: Fix thread suspend in crashdump

* move signal handler setup
---
 erts/emulator/beam/break.c    | 2 ++
 erts/emulator/beam/erl_init.c | 1 -
 erts/emulator/beam/sys.h      | 3 +++
 erts/emulator/sys/unix/sys.c  | 3 +--
 4 files changed, 6 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 6e1e94b95b..2fc61ab436 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -717,6 +717,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
      * We have to be very very careful when doing this as the schedulers
      * could be anywhere.
      */
+    sys_init_suspend_handler();
+
     for (i = 0; i < erts_no_schedulers; i++) {
         erts_tid_t tid = ERTS_SCHEDULER_IX(i)->tid;
         if (!erts_equal_tids(tid,erts_thr_self()))
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index c5904b375e..88bd002a8c 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -2220,7 +2220,6 @@ erl_start(int argc, char **argv)
 	init_break_handler();
     if (replace_intr)
 	erts_replace_intr();
-    sys_init_suspend_handler();
 #endif
 
     boot_argc = argc - i;  /* Number of arguments to init */
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index ceea794cc4..a11fb16b39 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -851,9 +851,12 @@ int erts_sys_unsetenv(char *key);
 char *erts_read_env(char *key);
 void erts_free_read_env(void *value);
 
+#if defined(ERTS_SMP)
 #if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX)
 extern void sys_thr_resume(erts_tid_t tid);
 extern void sys_thr_suspend(erts_tid_t tid);
+#define ERTS_SYS_SUSPEND_SIGNAL SIGUSR2
+#endif
 #endif
 
 /* utils.c */
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index ce87e3c7a6..a25d865c91 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -110,9 +110,8 @@ erts_smp_atomic_t sys_misc_mem_sz;
 static void smp_sig_notify(int signum);
 static int sig_notify_fds[2] = {-1, -1};
 
-#if !defined(ETHR_UNUSABLE_SIGUSRX) && defined(ERTS_THR_HAVE_SIG_FUNCS)
+#ifdef ERTS_SYS_SUSPEND_SIGNAL
 static int sig_suspend_fds[2] = {-1, -1};
-#define ERTS_SYS_SUSPEND_SIGNAL SIGUSR2
 #endif
 
 #endif
-- 
cgit v1.2.3


From 9ccb988eeb79815a33e0dd138278e0c29ed3ca6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 23 Jan 2017 16:05:07 +0100
Subject: erts: Do not handle SIGILL

* Remove SIGILL from signal whitelist
---
 erts/emulator/sys/unix/sys.c           | 8 ++++----
 erts/emulator/test/os_signal_SUITE.erl | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index a25d865c91..cb20c690b4 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -699,7 +699,7 @@ static RETSIGTYPE suspend_signal(int signum)
   SIGHUP     Term     Hangup detected on controlling terminal or death of controlling process
  !SIGINT     Term     Interrupt from keyboard
   SIGQUIT    Core     Quit from keyboard
-  SIGILL     Core     Illegal Instruction
+ !SIGILL     Core     Illegal Instruction
   SIGABRT    Core     Abort signal from abort(3)
  !SIGFPE     Core     Floating point exception
  !SIGKILL    Term     Kill signal
@@ -725,7 +725,7 @@ signalterm_to_signum(Eterm signal)
     case am_sighup:  return SIGHUP;
     /* case am_sigint:  return SIGINT; */
     case am_sigquit: return SIGQUIT;
-    case am_sigill:  return SIGILL;
+    /* case am_sigill:  return SIGILL; */
     case am_sigabrt: return SIGABRT;
     /* case am_sigsegv: return SIGSEGV; */
     case am_sigalrm: return SIGALRM;
@@ -745,12 +745,12 @@ signum_to_signalterm(int signum)
     switch (signum) {
     case SIGHUP:  return am_sighup;
     /* case SIGINT:  return am_sigint; */    /* ^c */
-    case SIGILL:  return am_sigill;
+    case SIGQUIT: return am_sigquit;   /* ^\ */
+    /* case SIGILL:  return am_sigill; */
     case SIGABRT: return am_sigabrt;
     /* case SIGSEGV: return am_sigsegv; */
     case SIGALRM: return am_sigalrm;
     case SIGTERM: return am_sigterm;
-    case SIGQUIT: return am_sigquit;   /* ^\ */
     case SIGUSR1: return am_sigusr1;
     case SIGUSR2: return am_sigusr2;
     case SIGCHLD: return am_sigchld;
diff --git a/erts/emulator/test/os_signal_SUITE.erl b/erts/emulator/test/os_signal_SUITE.erl
index 2b9343663d..7c3950114f 100644
--- a/erts/emulator/test/os_signal_SUITE.erl
+++ b/erts/emulator/test/os_signal_SUITE.erl
@@ -86,7 +86,7 @@ end_per_suite(_Config) ->
 
 set_unset(_Config) ->
     Signals = [sighup, %sigint,
-               sigquit, sigill,
+               sigquit, %sigill,
                sigabrt,
                sigalrm, sigterm,
                sigusr1, sigusr2,
-- 
cgit v1.2.3


From d4bd17da9759af54227891a90d9fb83d5bfe6d7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 24 Jan 2017 10:59:30 +0100
Subject: erts: Use os module instead of erts_internal for set_signal/2

* Add specs
* Change return signature to 'ok' instead of 'true'
---
 erts/emulator/beam/bif.c               | 14 ----------
 erts/emulator/beam/bif.tab             |  2 +-
 erts/emulator/beam/erl_bif_os.c        | 14 ++++++++++
 erts/emulator/test/os_signal_SUITE.erl | 48 +++++++++++++++++-----------------
 4 files changed, 39 insertions(+), 39 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 452bfef71a..1048300cf7 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -5280,17 +5280,3 @@ BIF_RETTYPE dt_restore_tag_1(BIF_ALIST_1)
 #endif
     BIF_RET(am_true);
 }
-
-BIF_RETTYPE erts_internal_set_signal_2(BIF_ALIST_2) {
-    if (is_atom(BIF_ARG_1) && ((BIF_ARG_2 == am_ignore) ||
-                               (BIF_ARG_2 == am_default) ||
-                               (BIF_ARG_2 == am_handle))) {
-        if (!erts_set_signal(BIF_ARG_1, BIF_ARG_2))
-            goto error;
-
-        BIF_RET(am_true);
-    }
-
-error:
-    BIF_ERROR(BIF_P, BADARG);
-}
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 318a4fd264..f90fae6041 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -668,7 +668,7 @@ gcbif erlang:ceil/1
 bif math:floor/1
 bif math:ceil/1
 bif math:fmod/2
-bif erts_internal:set_signal/2
+bif os:set_signal/2
 
 #
 # Obsolete
diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c
index 46777d3aa5..edc3c82b23 100644
--- a/erts/emulator/beam/erl_bif_os.c
+++ b/erts/emulator/beam/erl_bif_os.c
@@ -203,3 +203,17 @@ BIF_RETTYPE os_unsetenv_1(BIF_ALIST_1)
     }
     BIF_RET(am_true);
 }
+
+BIF_RETTYPE os_set_signal_2(BIF_ALIST_2) {
+    if (is_atom(BIF_ARG_1) && ((BIF_ARG_2 == am_ignore) ||
+                               (BIF_ARG_2 == am_default) ||
+                               (BIF_ARG_2 == am_handle))) {
+        if (!erts_set_signal(BIF_ARG_1, BIF_ARG_2))
+            goto error;
+
+        BIF_RET(am_ok);
+    }
+
+error:
+    BIF_ERROR(BIF_P, BADARG);
+}
diff --git a/erts/emulator/test/os_signal_SUITE.erl b/erts/emulator/test/os_signal_SUITE.erl
index 7c3950114f..9aa49a453e 100644
--- a/erts/emulator/test/os_signal_SUITE.erl
+++ b/erts/emulator/test/os_signal_SUITE.erl
@@ -92,9 +92,9 @@ set_unset(_Config) ->
                sigusr1, sigusr2,
                sigchld,
                sigstop, sigtstp],
-    F1 = fun(Sig) -> true = erts_internal:set_signal(Sig,handle) end,
-    F2 = fun(Sig) -> true = erts_internal:set_signal(Sig,default) end,
-    F3 = fun(Sig) -> true = erts_internal:set_signal(Sig,ignore) end,
+    F1 = fun(Sig) -> ok = os:set_signal(Sig,handle) end,
+    F2 = fun(Sig) -> ok = os:set_signal(Sig,default) end,
+    F3 = fun(Sig) -> ok = os:set_signal(Sig,ignore) end,
     %% set handle
     ok = lists:foreach(F1, Signals),
     %% set ignore
@@ -106,7 +106,7 @@ set_unset(_Config) ->
 t_sighup(_Config) ->
     Pid1 = setup_service(),
     OsPid = os:getpid(),
-    erts_internal:set_signal(sighup, handle),
+    os:set_signal(sighup, handle),
     ok = kill("HUP", OsPid),
     ok = kill("HUP", OsPid),
     ok = kill("HUP", OsPid),
@@ -121,7 +121,7 @@ t_sighup(_Config) ->
     ok = kill("HUP", OsPid),
     %% ignore
     Pid2 = setup_service(),
-    erts_internal:set_signal(sighup, ignore),
+    os:set_signal(sighup, ignore),
     ok = kill("HUP", OsPid),
     ok = kill("HUP", OsPid),
     ok = kill("HUP", OsPid),
@@ -129,13 +129,13 @@ t_sighup(_Config) ->
     io:format("Msgs2: ~p~n", [Msgs2]),
     [] = Msgs2,
     %% reset to handle (it's the default)
-    erts_internal:set_signal(sighup, handle),
+    os:set_signal(sighup, handle),
     ok.
 
 t_sigusr1(_Config) ->
     Pid1 = setup_service(),
     OsPid = os:getpid(),
-    erts_internal:set_signal(sigusr1, handle),
+    os:set_signal(sigusr1, handle),
     ok = kill("USR1", OsPid),
     ok = kill("USR1", OsPid),
     ok = kill("USR1", OsPid),
@@ -150,7 +150,7 @@ t_sigusr1(_Config) ->
     ok = kill("USR1", OsPid),
     %% ignore
     Pid2 = setup_service(),
-    erts_internal:set_signal(sigusr1, ignore),
+    os:set_signal(sigusr1, ignore),
     ok = kill("USR1", OsPid),
     ok = kill("USR1", OsPid),
     ok = kill("USR1", OsPid),
@@ -158,13 +158,13 @@ t_sigusr1(_Config) ->
     io:format("Msgs2: ~p~n", [Msgs2]),
     [] = Msgs2,
     %% reset to ignore (it's the default)
-    erts_internal:set_signal(sigusr1, handle),
+    os:set_signal(sigusr1, handle),
     ok.
 
 t_sigusr2(_Config) ->
     Pid1 = setup_service(),
     OsPid = os:getpid(),
-    erts_internal:set_signal(sigusr2, handle),
+    os:set_signal(sigusr2, handle),
     ok = kill("USR2", OsPid),
     ok = kill("USR2", OsPid),
     ok = kill("USR2", OsPid),
@@ -179,7 +179,7 @@ t_sigusr2(_Config) ->
     ok = kill("USR2", OsPid),
     %% ignore
     Pid2 = setup_service(),
-    erts_internal:set_signal(sigusr2, ignore),
+    os:set_signal(sigusr2, ignore),
     ok = kill("USR2", OsPid),
     ok = kill("USR2", OsPid),
     ok = kill("USR2", OsPid),
@@ -187,13 +187,13 @@ t_sigusr2(_Config) ->
     io:format("Msgs2: ~p~n", [Msgs2]),
     [] = Msgs2,
     %% reset to ignore (it's the default)
-    erts_internal:set_signal(sigusr2, ignore),
+    os:set_signal(sigusr2, ignore),
     ok.
 
 t_sigterm(_Config) ->
     Pid1 = setup_service(),
     OsPid = os:getpid(),
-    erts_internal:set_signal(sigterm, handle),
+    os:set_signal(sigterm, handle),
     ok = kill("TERM", OsPid),
     ok = kill("TERM", OsPid),
     ok = kill("TERM", OsPid),
@@ -208,7 +208,7 @@ t_sigterm(_Config) ->
     ok = kill("TERM", OsPid),
     %% ignore
     Pid2 = setup_service(),
-    erts_internal:set_signal(sigterm, ignore),
+    os:set_signal(sigterm, ignore),
     ok = kill("TERM", OsPid),
     ok = kill("TERM", OsPid),
     ok = kill("TERM", OsPid),
@@ -216,13 +216,13 @@ t_sigterm(_Config) ->
     io:format("Msgs2: ~p~n", [Msgs2]),
     [] = Msgs2,
     %% reset to handle (it's the default)
-    erts_internal:set_signal(sigterm, handle),
+    os:set_signal(sigterm, handle),
     ok.
 
 t_sigchld(_Config) ->
     Pid1 = setup_service(),
     OsPid = os:getpid(),
-    erts_internal:set_signal(sigchld, handle),
+    os:set_signal(sigchld, handle),
     ok = kill("CHLD", OsPid),
     ok = kill("CHLD", OsPid),
     ok = kill("CHLD", OsPid),
@@ -237,7 +237,7 @@ t_sigchld(_Config) ->
     ok = kill("CHLD", OsPid),
     %% ignore
     Pid2 = setup_service(),
-    erts_internal:set_signal(sigchld, ignore),
+    os:set_signal(sigchld, ignore),
     ok = kill("CHLD", OsPid),
     ok = kill("CHLD", OsPid),
     ok = kill("CHLD", OsPid),
@@ -245,19 +245,19 @@ t_sigchld(_Config) ->
     io:format("Msgs2: ~p~n", [Msgs2]),
     [] = Msgs2,
     %% reset to handle (it's the default)
-    erts_internal:set_signal(sigchld, ignore),
+    os:set_signal(sigchld, ignore),
     ok.
 
 
 t_sigalrm(_Config) ->
     Pid1 = setup_service(),
-    true = erts_internal:set_signal(sigalrm, handle),
+    ok = os:set_signal(sigalrm, handle),
     ok = os_signal_SUITE:set_alarm(1),
     receive after 3000 -> ok end,
     Msgs1 = fetch_msgs(Pid1),
     [{notify,sigalrm}] = Msgs1,
     io:format("Msgs1: ~p~n", [Msgs1]),
-    erts_internal:set_signal(sigalrm, ignore),
+    os:set_signal(sigalrm, ignore),
     Pid2 = setup_service(),
     ok = os_signal_SUITE:set_alarm(1),
     receive after 3000 -> ok end,
@@ -265,18 +265,18 @@ t_sigalrm(_Config) ->
     [] = Msgs2,
     io:format("Msgs2: ~p~n", [Msgs2]),
     Pid3 = setup_service(),
-    erts_internal:set_signal(sigalrm, handle),
+    os:set_signal(sigalrm, handle),
     ok = os_signal_SUITE:set_alarm(1),
     receive after 3000 -> ok end,
     Msgs3 = fetch_msgs(Pid3),
     [{notify,sigalrm}] = Msgs3,
     io:format("Msgs3: ~p~n", [Msgs3]),
-    erts_internal:set_signal(sigalrm, ignore),
+    os:set_signal(sigalrm, ignore),
     ok.
 
 t_sigchld_fork(_Config) ->
     Pid1 = setup_service(),
-    true = erts_internal:set_signal(sigchld, handle),
+    ok = os:set_signal(sigchld, handle),
     {ok,OsPid} = os_signal_SUITE:fork(),
     receive after 3000 -> ok end,
     Msgs1 = fetch_msgs(Pid1),
@@ -286,7 +286,7 @@ t_sigchld_fork(_Config) ->
     io:format("exit status from ~w : ~w~n", [OsPid,Status]),
     42 = Status,
     %% reset to ignore (it's the default)
-    erts_internal:set_signal(sigchld, ignore),
+    os:set_signal(sigchld, ignore),
     ok.
 
 
-- 
cgit v1.2.3


From 808b2f4d53e446aed07f85716c5c4b85abb3d18a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 2 Feb 2017 16:08:38 +0100
Subject: erts: Add size of literals to module code size

in crash dump and (l)oaded command in break menu.
---
 erts/emulator/beam/break.c | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 61907cc7ff..5d34c83c1a 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -379,6 +379,16 @@ info(fmtfn_t to, void *to_arg)
 
 }
 
+static int code_size(struct erl_module_instance* modi)
+{
+    ErtsLiteralArea* lit = modi->code_hdr->literal_area;
+    int size = modi->code_length;
+    if (lit) {
+        size += (lit->end - lit->start) * sizeof(Eterm);
+    }
+    return size;
+}
+
 void
 loaded(fmtfn_t to, void *to_arg)
 {
@@ -399,9 +409,9 @@ loaded(fmtfn_t to, void *to_arg)
 	if ((modp = module_code(i, code_ix)) != NULL &&
 	    ((modp->curr.code_length != 0) ||
 	     (modp->old.code_length != 0))) {
-	    cur += modp->curr.code_length;
+	    cur += code_size(&modp->curr);
 	    if (modp->old.code_length != 0) {
-		old += modp->old.code_length;
+		old += code_size(&modp->old);
 	    }
 	}
     }
@@ -422,12 +432,12 @@ loaded(fmtfn_t to, void *to_arg)
 	    ((modp->curr.code_length != 0) ||
 	     (modp->old.code_length != 0))) {
 		erts_print(to, to_arg, "%T", make_atom(modp->module));
-		cur += modp->curr.code_length;
-		erts_print(to, to_arg, " %d", modp->curr.code_length );
+		cur += code_size(&modp->curr);
+		erts_print(to, to_arg, " %d", code_size(&modp->curr));
 		if (modp->old.code_length != 0) {
 		    erts_print(to, to_arg, " (%d old)",
-			       modp->old.code_length );
-		    old += modp->old.code_length;
+			       code_size(&modp->old));
+		    old += code_size(&modp->old);
 		}
 		erts_print(to, to_arg, "\n");
 	    }
@@ -442,7 +452,7 @@ loaded(fmtfn_t to, void *to_arg)
 		erts_print(to, to_arg, "%T", make_atom(modp->module));
 		erts_print(to, to_arg, "\n");
 		erts_print(to, to_arg, "Current size: %d\n",
-			   modp->curr.code_length);
+			   code_size(&modp->curr));
 		code = modp->curr.code_hdr;
 		if (code != NULL && code->attr_ptr) {
 		    erts_print(to, to_arg, "Current attributes: ");
@@ -456,7 +466,7 @@ loaded(fmtfn_t to, void *to_arg)
 		}
 
 		if (modp->old.code_length != 0) {
-		    erts_print(to, to_arg, "Old size: %d\n", modp->old.code_length);
+		    erts_print(to, to_arg, "Old size: %d\n", code_size(&modp->old));
 		    code = modp->old.code_hdr;
 		    if (code->attr_ptr) {
 			erts_print(to, to_arg, "Old attributes: ");
-- 
cgit v1.2.3


From 9d927001ba16517042d3099612a2c9b130c6f62a Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 2 Feb 2017 19:21:36 +0100
Subject: Dirty schedulers should not touch scheduler data pointed to by
 process struct

---
 erts/emulator/beam/erl_process.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b345c35a7e..cfa2b62061 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9598,7 +9598,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	esdp->current_process = NULL;
 #ifdef ERTS_SMP
-	p->scheduler_data = NULL;
+	if (is_normal_sched)
+	    p->scheduler_data = NULL;
 #endif
 
 	erts_smp_proc_unlock(p, (ERTS_PROC_LOCK_MAIN
@@ -10013,8 +10014,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	state = erts_smp_atomic32_read_nob(&p->state);
 
-	ASSERT(!p->scheduler_data);
 #ifndef ERTS_DIRTY_SCHEDULERS
+	ASSERT(!p->scheduler_data);
 	p->scheduler_data = esdp;
 #else /* ERTS_DIRTY_SCHEDULERS */
 	if (is_normal_sched) {
@@ -10025,6 +10026,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 		goto sched_out_proc;
 	    }
+	    ASSERT(!p->scheduler_data);
 	    p->scheduler_data = esdp;
 	}
 	else {
-- 
cgit v1.2.3


From ea530357b27635278ae8a3260e735ea39df5c283 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 3 Feb 2017 12:05:49 +0100
Subject: Fix merge commit

This fixes commit f0867aa2ccbbf5677e0577bba08f8b7bc53ec0ed
---
 erts/emulator/beam/erl_init.c |  1 -
 erts/emulator/sys/unix/sys.c  | 46 -------------------------------------------
 2 files changed, 47 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 2fd97208cc..070cc3f2d0 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -2220,7 +2220,6 @@ erl_start(int argc, char **argv)
 	init_break_handler();
     if (replace_intr)
 	erts_replace_intr();
-    sys_init_suspend_handler();
 #endif
 
     boot_argc = argc - i;  /* Number of arguments to init */
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index fa95473db8..e135dbff99 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -819,61 +819,15 @@ void init_break_handler(void)
 }
 
 void sys_init_suspend_handler(void)
-<<<<<<< HEAD
 {
 #ifdef ERTS_SYS_SUSPEND_SIGNAL
    sys_signal(ERTS_SYS_SUSPEND_SIGNAL, suspend_signal);
-=======
-{
-#ifdef ERTS_SYS_SUSPEND_SIGNAL
-   sys_signal(ERTS_SYS_SUSPEND_SIGNAL, suspend_signal);
-#endif
-}
-
-int sys_max_files(void)
-{
-   return(max_files);
-}
-
-static void block_signals(void)
-{
-#if !CHLDWTHR
-   sys_sigblock(SIGCHLD);
-#endif
-#ifndef ERTS_SMP
-   sys_sigblock(SIGINT);
-#ifndef ETHR_UNUSABLE_SIGUSRX
-   sys_sigblock(SIGUSR1);
-#endif /* #ifndef ETHR_UNUSABLE_SIGUSRX */
-#endif /* #ifndef ERTS_SMP */
-
-#ifdef ERTS_SYS_SUSPEND_SIGNAL
-   sys_sigblock(ERTS_SYS_SUSPEND_SIGNAL);
->>>>>>> maint-18
 #endif
 }
 
 int sys_max_files(void)
 {
-<<<<<<< HEAD
    return(max_files);
-=======
-    /* Update erl_child_setup.c if changed */
-#if !CHLDWTHR
-   sys_sigrelease(SIGCHLD);
-#endif
-#ifndef ERTS_SMP
-   sys_sigrelease(SIGINT);
-#ifndef ETHR_UNUSABLE_SIGUSRX
-   sys_sigrelease(SIGUSR1);
-#endif /* #ifndef ETHR_UNUSABLE_SIGUSRX */
-#endif /* #ifndef ERTS_SMP */
-
-#ifdef ERTS_SYS_SUSPEND_SIGNAL
-   sys_sigrelease(ERTS_SYS_SUSPEND_SIGNAL);
-#endif
-
->>>>>>> maint-18
 }
 
 /************************** OS info *******************************/
-- 
cgit v1.2.3


From 7eff21e2b0b86f798cce29835cd85a414d3d1be5 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 2 Feb 2017 18:19:49 +0100
Subject: Use a hole-marker that cannot be mistaken for a valid term on the
 heap

---
 erts/emulator/beam/erl_init.c |  5 +++++
 erts/emulator/beam/erl_vm.h   | 17 +++++++++++++++--
 2 files changed, 20 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 2fd97208cc..6c744553c3 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -117,6 +117,11 @@ const int etp_big_endian = 1;
 const int etp_big_endian = 0;
 #endif
 const Eterm etp_the_non_value = THE_NON_VALUE;
+#ifdef ERTS_HOLE_MARKER
+const Eterm etp_hole_marker = ERTS_HOLE_MARKER;
+#else
+const Eterm etp_hole_marker = 0;
+#endif
 
 /*
  * Note about VxWorks: All variables must be initialized by executable code,
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index f97716d030..899e2a275a 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -110,8 +110,21 @@
 #define HeapWordsLeft(p) (HEAP_LIMIT(p) - HEAP_TOP(p))
 
 #if defined(DEBUG) || defined(CHECK_FOR_HOLES)
-# define ERTS_HOLE_MARKER (((0xdeadbeef << 24) << 8) | 0xdeadbeef)
-#endif /* egil: 32-bit ? */
+
+/*
+ * ERTS_HOLE_MARKER must *not* be mistaken for a valid term
+ * on the heap...
+ */
+#  ifdef ARCH_64
+#    define ERTS_HOLE_MARKER \
+    make_catch(UWORD_CONSTANT(0xdeadbeaf00000000) >> _TAG_IMMED2_SIZE)
+/* Will (at the time of writing) appear as 0xdeadbeaf0000001b */
+#  else
+#    define ERTS_HOLE_MARKER \
+    make_catch(UWORD_CONSTANT(0xdead0000) >> _TAG_IMMED2_SIZE)
+/* Will (at the time of writing) appear as 0xdead001b */
+#  endif
+#endif
 
 /*
  * Allocate heap memory on the ordinary heap, NEVER in a heap
-- 
cgit v1.2.3


From 3fe6f3c0caecec522c7e14353eda6bbb86c9e8d6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 3 Feb 2017 15:22:48 +0100
Subject: erts: Add deallocation veto for magic destructors

A magic destructor can return 0 and thereby take control
and prolong the lifetime of a magic binary.
---
 erts/emulator/beam/beam_load.c      |  5 +++--
 erts/emulator/beam/binary.c         |  6 ++++--
 erts/emulator/beam/dist.c           |  4 +++-
 erts/emulator/beam/dist.h           |  2 +-
 erts/emulator/beam/erl_bif_binary.c | 15 ++++++++-------
 erts/emulator/beam/erl_bif_re.c     |  3 ++-
 erts/emulator/beam/erl_binary.h     | 16 ++++++++++------
 erts/emulator/beam/erl_db_util.c    |  5 +++--
 erts/emulator/beam/erl_db_util.h    |  2 +-
 erts/emulator/beam/erl_map.c        |  3 ++-
 erts/emulator/beam/erl_nif.c        |  3 ++-
 erts/emulator/beam/erl_ptab.c       |  6 ++++--
 erts/emulator/beam/erl_unicode.c    |  3 ++-
 erts/emulator/beam/external.c       |  6 ++++--
 erts/emulator/beam/global.h         |  2 +-
 erts/emulator/hipe/hipe_load.c      |  3 ++-
 16 files changed, 52 insertions(+), 32 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 309465bcd3..119c004f1f 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -481,7 +481,7 @@ typedef struct LoaderState {
 static void free_loader_state(Binary* magic);
 static ErlHeapFragment* new_literal_fragment(Uint size);
 static void free_literal_fragment(ErlHeapFragment*);
-static void loader_state_dtor(Binary* magic);
+static int loader_state_dtor(Binary* magic);
 #ifdef HIPE
 static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 				  Eterm group_leader, Eterm module,
@@ -1012,7 +1012,7 @@ static void free_literal_fragment(ErlHeapFragment* bp)
 /*
  * This destructor function can safely be called multiple times.
  */
-static void
+static int
 loader_state_dtor(Binary* magic)
 {
     LoaderState* stp = ERTS_MAGIC_BIN_DATA(magic);
@@ -1097,6 +1097,7 @@ loader_state_dtor(Binary* magic)
      */
 
     ASSERT(stp->genop_blocks == 0);
+    return 1;
 }
 
 #ifdef HIPE
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 071a356260..9894992b70 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -355,9 +355,10 @@ typedef struct {
     Uint bitoffs;
 } ErtsB2LState;
 
-static void b2l_state_destructor(Binary *mbp)
+static int b2l_state_destructor(Binary *mbp)
 {
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == b2l_state_destructor);
+    return 1;
 }
 
 static BIF_RETTYPE
@@ -729,12 +730,13 @@ list_to_binary_engine(ErtsL2BState *sp)
     }
 }
 
-static void
+static int
 l2b_state_destructor(Binary *mbp)
 {
     ErtsL2BState *sp = ERTS_MAGIC_BIN_DATA(mbp); 
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == l2b_state_destructor);
     DESTROY_SAVED_ESTACK(&sp->buf.iolist.estack);
+    return 1;
 }
 
 static ERTS_INLINE Eterm
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 990edb274e..9e8f853279 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -713,7 +713,7 @@ static void clear_dist_entry(DistEntry *dep)
     }
 }
 
-void erts_dsend_context_dtor(Binary* ctx_bin)
+int erts_dsend_context_dtor(Binary* ctx_bin)
 {
     ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(ctx_bin);
     switch (ctx->dss.phase) {
@@ -730,6 +730,8 @@ void erts_dsend_context_dtor(Binary* ctx_bin)
     }
     if (ctx->dep_to_deref)
 	erts_deref_dist_entry(ctx->dep_to_deref);
+
+    return 1;
 }
 
 Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx)
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index e82b416286..8799e54057 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -375,7 +375,7 @@ extern int erts_dsig_send_monitor(ErtsDSigData *, Eterm, Eterm, Eterm);
 extern int erts_dsig_send_m_exit(ErtsDSigData *, Eterm, Eterm, Eterm, Eterm);
 
 extern int erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx);
-extern void erts_dsend_context_dtor(Binary*);
+extern int erts_dsend_context_dtor(Binary*);
 extern Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx);
 
 extern int erts_dist_command(Port *prt, int reds);
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 6e10980b6b..24debedce2 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -239,13 +239,13 @@ static void dump_ac_node(ACNode *node, int indent, int ch);
 /*
  * Callback for the magic binary
  */
-static void cleanup_my_data_ac(Binary *bp)
+static int cleanup_my_data_ac(Binary *bp)
 {
-    return;
+    return 1;
 }
-static void cleanup_my_data_bm(Binary *bp)
+static int cleanup_my_data_bm(Binary *bp)
 {
-    return;
+    return 1;
 }
 
 /*
@@ -2066,7 +2066,7 @@ static int do_search_backward(CommonData *cd, Uint *posp, Uint *redsp)
     }
 }
 
-static void cleanup_common_data(Binary *bp)
+static int cleanup_common_data(Binary *bp)
 {
     int i;
     CommonData *cd;
@@ -2083,7 +2083,7 @@ static void cleanup_common_data(Binary *bp)
 	    break;
 	}
     }
-    return;
+    return 1;
 }
 
 static BIF_RETTYPE do_longest_common(Process *p, Eterm list, int direction)
@@ -2563,7 +2563,7 @@ typedef struct {
 
 #define BINARY_COPY_LOOP_FACTOR 100
 
-static void cleanup_copy_bin_state(Binary *bp)
+static int cleanup_copy_bin_state(Binary *bp)
 {
     CopyBinState *cbs = (CopyBinState *) ERTS_MAGIC_BIN_DATA(bp);
     if (cbs->result != NULL) {
@@ -2583,6 +2583,7 @@ static void cleanup_copy_bin_state(Binary *bp)
 	break;
     }
     cbs->source_type =  BC_TYPE_EMPTY;
+    return 1;
 }
 
 /*
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index ff7746ce1d..391fee5753 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -600,10 +600,11 @@ static void cleanup_restart_context(RestartContext *rc)
     }
 }
 
-static void cleanup_restart_context_bin(Binary *bp)
+static int cleanup_restart_context_bin(Binary *bp)
 {
     RestartContext *rc = ERTS_MAGIC_BIN_DATA(bp);
     cleanup_restart_context(rc);
+    return 1;
 }
 
 /*
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index fdc5efea98..52efa06816 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -189,10 +189,10 @@ ERTS_GLB_INLINE Binary *erts_bin_realloc_fnf(Binary *bp, Uint size);
 ERTS_GLB_INLINE Binary *erts_bin_realloc(Binary *bp, Uint size);
 ERTS_GLB_INLINE void erts_bin_free(Binary *bp);
 ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size,
-                                                  void (*destructor)(Binary *),
+                                                  int (*destructor)(Binary *),
                                                   int unaligned);
 ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size,
-						 void (*destructor)(Binary *));
+						 int (*destructor)(Binary *));
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
@@ -320,8 +320,12 @@ erts_bin_realloc(Binary *bp, Uint size)
 ERTS_GLB_INLINE void
 erts_bin_free(Binary *bp)
 {
-    if (bp->flags & BIN_FLAG_MAGIC)
-	ERTS_MAGIC_BIN_DESTRUCTOR(bp)(bp);
+    if (bp->flags & BIN_FLAG_MAGIC) {
+	if (!ERTS_MAGIC_BIN_DESTRUCTOR(bp)(bp)) {
+            /* Destructor took control of the deallocation */
+            return;
+        }
+    }
     if (bp->flags & BIN_FLAG_DRV)
 	erts_free(ERTS_ALC_T_DRV_BINARY, (void *) bp);
     else
@@ -329,7 +333,7 @@ erts_bin_free(Binary *bp)
 }
 
 ERTS_GLB_INLINE Binary *
-erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *),
+erts_create_magic_binary_x(Uint size, int (*destructor)(Binary *),
                            int unaligned)
 {
     Uint bsize = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_SIZE(size)
@@ -348,7 +352,7 @@ erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *),
 }
 
 ERTS_GLB_INLINE Binary *
-erts_create_magic_binary(Uint size, void (*destructor)(Binary *))
+erts_create_magic_binary(Uint size, int (*destructor)(Binary *))
 {
     return erts_create_magic_binary_x(size, destructor, 0);
 }
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 4e987d5bee..991d63a18b 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1702,17 +1702,18 @@ error: /* Here is were we land when compilation failed. */
 /*
 ** Free a match program (in a binary)
 */
-void erts_db_match_prog_destructor(Binary *bprog)
+int erts_db_match_prog_destructor(Binary *bprog)
 {
     MatchProg *prog;
     if (bprog == NULL)
-	return;
+	return 1;
     prog = Binary2MatchProg(bprog);
     if (prog->term_save != NULL) {
 	free_message_buffer(prog->term_save); 
     }
     if (prog->saved_program_buf != NULL)
 	free_message_buffer(prog->saved_program_buf);
+    return 1;
 }
 
 void
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 49e5f6b4cf..3e29fdcbcf 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -355,7 +355,7 @@ Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr);
 Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags);
 Binary *db_match_set_compile(Process *p, Eterm matchexpr, 
 			     Uint flags);
-void erts_db_match_prog_destructor(Binary *);
+int erts_db_match_prog_destructor(Binary *);
 
 typedef struct match_prog {
     ErlHeapFragment *term_save; /* Only if needed, a list of message 
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 979a0040b0..3015158b24 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -1188,12 +1188,13 @@ typedef struct HashmapMergeContext_ {
 #endif
 } HashmapMergeContext;
 
-static void hashmap_merge_ctx_destructor(Binary* ctx_bin)
+static int hashmap_merge_ctx_destructor(Binary* ctx_bin)
 {
     HashmapMergeContext* ctx = (HashmapMergeContext*) ERTS_MAGIC_BIN_DATA(ctx_bin);
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == hashmap_merge_ctx_destructor);
 
     PSTACK_DESTROY_SAVED(&ctx->pstack);
+    return 1;
 }
 
 BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1) {
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 85fa53a886..fbe94708a5 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2109,7 +2109,7 @@ static void rollback_opened_resource_types(void)
 }
 
 
-static void nif_resource_dtor(Binary* bin)
+static int nif_resource_dtor(Binary* bin)
 {
     ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin);
     ErlNifResourceType* type = resource->type;
@@ -2128,6 +2128,7 @@ static void nif_resource_dtor(Binary* bin)
 	steal_resource_type(type);
 	erts_free(ERTS_ALC_T_NIF, type);
     }
+    return 1;
 }
 
 void erts_resource_stop(ErlNifResource* resource, ErlNifEvent e,
diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index 22830a19c4..578ae14b9f 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -733,7 +733,7 @@ erts_ptab_delete_element(ErtsPTab *ptab,
  * erts_ptab_list() implements BIFs listing the content of the table,
  * e.g. erlang:processes/0.
  */
-static void cleanup_ptab_list_bif_data(Binary *bp);
+static int cleanup_ptab_list_bif_data(Binary *bp);
 static int ptab_list_bif_engine(Process *c_p, Eterm *res_accp, Binary *mbp);
 
 
@@ -787,7 +787,7 @@ erts_ptab_list(Process *c_p, ErtsPTab *ptab)
     return ret_val;
 }
 
-static void
+static int
 cleanup_ptab_list_bif_data(Binary *bp)
 {
     ErtsPTabListBifData *ptlbdp = ERTS_MAGIC_BIN_DATA(bp);
@@ -875,6 +875,8 @@ cleanup_ptab_list_bif_data(Binary *bp)
 
     ERTS_PTAB_LIST_DBG_TRACE(ptlbdp->debug.caller, return);
     ERTS_PTAB_LIST_DBG_CLEANUP(ptlbdp);
+
+    return 1;
 }
 
 static int
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index bd5e1482fb..3167ca5eef 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -123,10 +123,11 @@ static void cleanup_restart_context(RestartContext *rc)
     }
 }
 
-static void cleanup_restart_context_bin(Binary *bp)
+static int cleanup_restart_context_bin(Binary *bp)
 {
     RestartContext *rc = ERTS_MAGIC_BIN_DATA(bp);
     cleanup_restart_context(rc);
+    return 1;
 }
 
 static RestartContext *get_rc_from_bin(Eterm bin)
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 10e452fa25..cb3486fde2 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1387,12 +1387,13 @@ static void b2t_destroy_context(B2TContext* context)
     }
 }
 
-static void b2t_context_destructor(Binary *context_bin)
+static int b2t_context_destructor(Binary *context_bin)
 {
     B2TContext* ctx = (B2TContext*) ERTS_MAGIC_BIN_DATA(context_bin);
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor);
 
     b2t_destroy_context(ctx);
+    return 1;
 }
 
 static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1)
@@ -1797,7 +1798,7 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) {
 #endif
 #define TERM_TO_BINARY_MEMCPY_FACTOR 8
 
-static void ttb_context_destructor(Binary *context_bin)
+static int ttb_context_destructor(Binary *context_bin)
 {
     TTBContext *context = ERTS_MAGIC_BIN_DATA(context_bin);
     if (context->alive) {
@@ -1831,6 +1832,7 @@ static void ttb_context_destructor(Binary *context_bin)
 	    break;
 	}
     }
+    return 1;
 }
 
 static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags, 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 2158f54a49..91e5bbbd98 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -274,7 +274,7 @@ typedef struct binary {
 typedef struct {
     ERTS_INTERNAL_BINARY_FIELDS
     SWord orig_size;
-    void (*destructor)(Binary *);
+    int (*destructor)(Binary *);
     union {
         struct {
             ERTS_BINARY_STRUCT_ALIGNMENT
diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
index 2998ed87a2..87c5004d2b 100644
--- a/erts/emulator/hipe/hipe_load.c
+++ b/erts/emulator/hipe/hipe_load.c
@@ -61,7 +61,7 @@ void hipe_free_loader_state(HipeLoaderState *stp)
     stp->module = NIL;
 }
 
-static void
+static int
 hipe_loader_state_dtor(Binary* magic)
 {
     HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic);
@@ -69,6 +69,7 @@ hipe_loader_state_dtor(Binary* magic)
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(magic) == hipe_loader_state_dtor);
 
     hipe_free_loader_state(stp);
+    return 1;
 }
 
 Binary *hipe_alloc_loader_state(Eterm module)
-- 
cgit v1.2.3


From 7111434c0eb1fdd6576a99ca94cdc2b20be9b9af Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 3 Feb 2017 17:40:45 +0100
Subject: erts: Rename ErlNifResource as ErtsResource

as it's not part of the API
---
 erts/emulator/beam/erl_nif.c            | 24 ++++++++++++------------
 erts/emulator/beam/global.h             |  8 ++++----
 erts/emulator/sys/common/erl_check_io.c | 28 ++++++++++++++--------------
 3 files changed, 30 insertions(+), 30 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index fbe94708a5..b7425b9e45 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1913,7 +1913,7 @@ int enif_snprintf(char *buffer, size_t size, const char* format, ...)
 /* dummy node in circular list */
 struct enif_resource_type_t resource_type_list; 
 
-#define SIZEOF_ErlNifResource(SIZE) (offsetof(ErlNifResource,data) + (SIZE))
+#define SIZEOF_ErlNifResource(SIZE) (offsetof(ErtsResource,data) + (SIZE))
 
 static ErlNifResourceType* find_resource_type(Eterm module, Eterm name)
 {
@@ -2111,7 +2111,7 @@ static void rollback_opened_resource_types(void)
 
 static int nif_resource_dtor(Binary* bin)
 {
-    ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin);
+    ErtsResource* resource = (ErtsResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin);
     ErlNifResourceType* type = resource->type;
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor);
 
@@ -2131,7 +2131,7 @@ static int nif_resource_dtor(Binary* bin)
     return 1;
 }
 
-void erts_resource_stop(ErlNifResource* resource, ErlNifEvent e,
+void erts_resource_stop(ErtsResource* resource, ErlNifEvent e,
                         int is_direct_call)
 {
     struct enif_msg_environment_t msg_env;
@@ -2146,7 +2146,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t size)
     Binary* bin = erts_create_magic_binary_x(SIZEOF_ErlNifResource(size),
                                              &nif_resource_dtor,
                                              1); /* unaligned */
-    ErlNifResource* resource = ERTS_MAGIC_BIN_UNALIGNED_DATA(bin);
+    ErtsResource* resource = ERTS_MAGIC_BIN_UNALIGNED_DATA(bin);
 
     ASSERT(type->owner && type->next && type->prev); /* not allowed in load/upgrade */
     resource->type = type;
@@ -2160,7 +2160,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t size)
 
 void enif_release_resource(void* obj)
 {
-    ErlNifResource* resource = DATA_TO_RESOURCE(obj);
+    ErtsResource* resource = DATA_TO_RESOURCE(obj);
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
 
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor);
@@ -2174,7 +2174,7 @@ void enif_release_resource(void* obj)
 
 void enif_keep_resource(void* obj)
 {
-    ErlNifResource* resource = DATA_TO_RESOURCE(obj);
+    ErtsResource* resource = DATA_TO_RESOURCE(obj);
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
 
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor);
@@ -2186,7 +2186,7 @@ void enif_keep_resource(void* obj)
 
 ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj)
 {
-    ErlNifResource* resource = DATA_TO_RESOURCE(obj);
+    ErtsResource* resource = DATA_TO_RESOURCE(obj);
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
     Eterm* hp = alloc_heap(env,PROC_BIN_SIZE);
     return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary);
@@ -2207,7 +2207,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ
 {
     ProcBin* pb;
     Binary* mbin;
-    ErlNifResource* resource;
+    ErtsResource* resource;
     if (!ERTS_TERM_IS_MAGIC_BINARY(term)) {
 	return 0;
     }
@@ -2216,7 +2216,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ
 	return 0; / * Or should we allow "resource binaries" as handles? * /
     }*/
     mbin = pb->val;
-    resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin);
+    resource = (ErtsResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin);
     if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor
 	|| resource->type != type) {	
 	return 0;
@@ -2227,9 +2227,9 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ
 
 size_t enif_sizeof_resource(void* obj)
 {
-    ErlNifResource* resource = DATA_TO_RESOURCE(obj);
+    ErtsResource* resource = DATA_TO_RESOURCE(obj);
     Binary* bin = &ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource)->binary;
-    return ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(bin) - offsetof(ErlNifResource,data);
+    return ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(bin) - offsetof(ErtsResource,data);
 }
 
 
@@ -3303,7 +3303,7 @@ erts_unload_nif(struct erl_module_nif* lib)
 
 void erl_nif_init()
 {
-    ERTS_CT_ASSERT((offsetof(ErlNifResource,data) % 8) == ERTS_MAGIC_BIN_BYTES_TO_ALIGN);
+    ERTS_CT_ASSERT((offsetof(ErtsResource,data) % 8) == ERTS_MAGIC_BIN_BYTES_TO_ALIGN);
 
     resource_type_list.next = &resource_type_list;
     resource_type_list.prev = &resource_type_list;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 91e5bbbd98..86c38f8e8c 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -71,7 +71,7 @@ struct enif_resource_type_t
     Eterm module;
     Eterm name;
 };
-typedef struct enif_resource_t
+typedef struct
 {
     struct enif_resource_type_t* type;
 #ifdef DEBUG
@@ -82,14 +82,14 @@ typedef struct enif_resource_t
 #endif
 
     char data[1];
-}ErlNifResource;
+}ErtsResource;
 
-#define DATA_TO_RESOURCE(PTR) ((ErlNifResource*)((char*)(PTR) - offsetof(ErlNifResource,data)))
+#define DATA_TO_RESOURCE(PTR) ((ErtsResource*)((char*)(PTR) - offsetof(ErtsResource,data)))
 
 extern void erts_pre_nif(struct enif_environment_t*, Process*,
 			 struct erl_module_nif*, Process* tracee);
 extern void erts_post_nif(struct enif_environment_t* env);
-extern void erts_resource_stop(ErlNifResource*, ErlNifEvent, int is_direct_call);
+extern void erts_resource_stop(ErtsResource*, ErlNifEvent, int is_direct_call);
 extern Eterm erts_nif_taints(Process* p);
 extern void erts_print_nif_taints(fmtfn_t to, void* to_arg);
 void erts_unload_nif(struct erl_module_nif* nif);
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 4fc95624c7..79f567d16c 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -127,7 +127,7 @@ typedef struct {
         ErtsNifSelectDataState *nif;      /* ERTS_EV_TYPE_NIF */
         union {
             erts_driver_t*  drv_ptr;    /* ERTS_EV_TYPE_STOP_USE */
-            ErlNifResource* resource;   /* ERTS_EV_TYPE_STOP_NIF */
+            ErtsResource* resource;   /* ERTS_EV_TYPE_STOP_NIF */
         }stop;
     } driver;
     ErtsPollEvents events;
@@ -220,16 +220,16 @@ static void stale_drv_select(Eterm id, ErtsDrvEventState *state, int mode);
 static void drv_select_steal(ErlDrvPort ix, ErtsDrvEventState *state,
                              int mode, int on);
 static void nif_select_steal(ErtsDrvEventState *state, int mode,
-                             ErlNifResource* resource, Eterm ref);
+                             ErtsResource* resource, Eterm ref);
 
 static void print_drv_select_op(erts_dsprintf_buf_t *dsbufp,
                                 ErlDrvPort ix, ErtsSysFdType fd, int mode, int on);
 static void print_nif_select_op(erts_dsprintf_buf_t*, ErtsSysFdType,
-                                int mode, ErlNifResource*, Eterm ref);
+                                int mode, ErtsResource*, Eterm ref);
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
 static void drv_select_large_fd_error(ErlDrvPort, ErtsSysFdType, int, int);
-static void nif_select_large_fd_error(ErtsSysFdType, int, ErlNifResource*,Eterm ref);
+static void nif_select_large_fd_error(ErtsSysFdType, int, ErtsResource*,Eterm ref);
 #endif
 #if ERTS_CIO_HAVE_DRV_EVENT
 static void drv_event_steal(ErlDrvPort ix, ErtsDrvEventState *state,
@@ -244,7 +244,7 @@ static void
 steal_pending_stop_use(erts_dsprintf_buf_t*, ErlDrvPort, ErtsDrvEventState*,
                        int mode, int on);
 static void
-steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErlNifResource*,
+steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource*,
                        ErtsDrvEventState *state, int mode, int on);
 
 #ifdef ERTS_SMP
@@ -387,7 +387,7 @@ forget_removed(struct pollset_info* psi)
     erts_smp_spin_unlock(&psi->removed_list_lock);
 
     while (fdlp) {
-        ErlNifResource* resource = NULL;
+        ErtsResource* resource = NULL;
 	erts_driver_t* drv_ptr = NULL;
 	erts_smp_mtx_t* mtx;
 	ErtsSysFdType fd;
@@ -1207,7 +1207,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
 {
     int on;
     const Eterm id = env->proc->common.id;
-    ErlNifResource* resource = DATA_TO_RESOURCE(obj);
+    ErtsResource* resource = DATA_TO_RESOURCE(obj);
     ErtsSysFdType fd = (ErtsSysFdType) e;
     ErtsPollEvents ctl_events = (ErtsPollEvents) 0;
     ErtsPollEvents new_events, old_events;
@@ -1769,7 +1769,7 @@ print_drv_select_op(erts_dsprintf_buf_t *dsbufp,
 static void
 print_nif_select_op(erts_dsprintf_buf_t *dsbufp,
                     ErtsSysFdType fd, int mode,
-                    ErlNifResource* resource, Eterm ref)
+                    ErtsResource* resource, Eterm ref)
 {
     erts_dsprintf(dsbufp,
 		  "enif_select(_, %d,%s%s%s, %T:%T, %T) ",
@@ -1796,7 +1796,7 @@ drv_select_steal(ErlDrvPort ix, ErtsDrvEventState *state, int mode, int on)
 
 static void
 nif_select_steal(ErtsDrvEventState *state, int mode,
-                 ErlNifResource* resource, Eterm ref)
+                 ErtsResource* resource, Eterm ref)
 {
     if (need2steal(state, mode)) {
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
@@ -1826,7 +1826,7 @@ drv_select_large_fd_error(ErlDrvPort ix, ErtsSysFdType fd, int mode, int on)
 }
 static void
 nif_select_large_fd_error(ErtsSysFdType fd, int mode,
-                          ErlNifResource* resource, Eterm ref)
+                          ErtsResource* resource, Eterm ref)
 {
     erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
     print_nif_select_op(dsbufp, fd, mode, resource, ref);
@@ -1878,7 +1878,7 @@ steal_pending_stop_use(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix,
 }
 
 static void
-steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErlNifResource* resource,
+steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource* resource,
                        ErtsDrvEventState *state, int mode, int on)
 {
     int cancel = 0;
@@ -2046,7 +2046,7 @@ oready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time)
 }
 
 static ERTS_INLINE void
-send_event_tuple(struct erts_nif_select_event* e, ErlNifResource* resource,
+send_event_tuple(struct erts_nif_select_event* e, ErtsResource* resource,
                  Eterm event_atom)
 {
     Process* rp = erts_proc_lookup(e->pid);
@@ -2289,7 +2289,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
         case ERTS_EV_TYPE_NIF: { /* Requested via enif_select()... */
             struct erts_nif_select_event in = {NIL};
             struct erts_nif_select_event out = {NIL};
-            ErlNifResource* resource;
+            ErtsResource* resource;
             ErtsPollEvents revents = pollres[i].events;
 
             if (revents & ERTS_POLL_EV_ERR) {
@@ -2916,7 +2916,7 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters)
 	    }
 	}
         else if (state->type == ERTS_EV_TYPE_NIF) {
-            ErlNifResource* r;
+            ErtsResource* r;
             erts_printf("enif_select ");
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-- 
cgit v1.2.3


From fc0477a67641b9ba344de595b7fec2431208f8e6 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 24 Jan 2017 19:50:25 +0100
Subject: Atomic reference count of binaries also in non-SMP

NIF resources was not handled in a thread-safe manner in the runtime
system without SMP support.

As a consequence of this fix, the following driver functions are now
thread-safe also in the runtime system without SMP support:
- driver_free_binary()
- driver_realloc_binary()
- driver_binary_get_refc()
- driver_binary_inc_refc()
- driver_binary_dec_refc()
---
 erts/emulator/beam/beam_bp.c          |  18 ++---
 erts/emulator/beam/beam_bp.h          |   6 +-
 erts/emulator/beam/beam_emu.c         |   2 +-
 erts/emulator/beam/beam_load.c        |   4 +-
 erts/emulator/beam/break.c            |   2 +-
 erts/emulator/beam/copy.c             |  12 ++--
 erts/emulator/beam/dist.c             |   4 +-
 erts/emulator/beam/erl_bif_ddll.c     |  22 +++---
 erts/emulator/beam/erl_bif_info.c     |   2 +-
 erts/emulator/beam/erl_db.c           |  20 +++---
 erts/emulator/beam/erl_db_util.c      |   2 +-
 erts/emulator/beam/erl_db_util.h      |   4 +-
 erts/emulator/beam/erl_fun.c          |  20 +++---
 erts/emulator/beam/erl_fun.h          |   2 +-
 erts/emulator/beam/erl_gc.c           |   6 +-
 erts/emulator/beam/erl_message.c      |   2 +-
 erts/emulator/beam/erl_monitors.c     |   2 +-
 erts/emulator/beam/erl_node_tables.c  |  50 ++++++-------
 erts/emulator/beam/erl_node_tables.h  |   8 +--
 erts/emulator/beam/erl_process_dump.c |   2 +-
 erts/emulator/beam/external.c         |   2 +-
 erts/emulator/beam/global.h           |   2 +-
 erts/emulator/beam/io.c               |  31 ++------
 erts/emulator/beam/sys.h              | 128 +++++++++++++++++++++++++++++++---
 erts/emulator/beam/utils.c            |   2 +-
 erts/emulator/hipe/hipe_bif0.c        |   2 +-
 erts/emulator/hipe/hipe_native_bif.c  |   2 +-
 27 files changed, 221 insertions(+), 138 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index bbb2e4f34f..4e4429d1dc 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -347,17 +347,17 @@ consolidate_bp_data(Module* modp, BeamInstr* pc, int local)
     }
     if (flags & ERTS_BPF_META_TRACE) {
 	dst->meta_tracer = src->meta_tracer;
-	erts_refc_inc(&dst->meta_tracer->refc, 1);
+	erts_smp_refc_inc(&dst->meta_tracer->refc, 1);
 	dst->meta_ms = src->meta_ms;
 	MatchSetRef(dst->meta_ms);
     }
     if (flags & ERTS_BPF_COUNT) {
 	dst->count = src->count;
-	erts_refc_inc(&dst->count->refc, 1);
+	erts_smp_refc_inc(&dst->count->refc, 1);
     }
     if (flags & ERTS_BPF_TIME_TRACE) {
 	dst->time = src->time;
-	erts_refc_inc(&dst->time->refc, 1);
+	erts_smp_refc_inc(&dst->time->refc, 1);
 	ASSERT(dst->time->hash);
     }
 }
@@ -1498,7 +1498,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags,
 	MatchSetRef(match_spec);
 	bp->meta_ms = match_spec;
 	bmt = Alloc(sizeof(BpMetaTracer));
-	erts_refc_init(&bmt->refc, 1);
+	erts_smp_refc_init(&bmt->refc, 1);
         erts_tracer_update(&meta_tracer, tracer); /* copy tracer */
 	erts_smp_atomic_init_nob(&bmt->tracer, (erts_aint_t)meta_tracer);
 	bp->meta_tracer = bmt;
@@ -1507,7 +1507,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags,
 
 	ASSERT((bp->flags & ERTS_BPF_COUNT) == 0);
 	bcp = Alloc(sizeof(BpCount));
-	erts_refc_init(&bcp->refc, 1);
+	erts_smp_refc_init(&bcp->refc, 1);
 	erts_smp_atomic_init_nob(&bcp->acount, 0);
 	bp->count = bcp;
     } else if (break_flags & ERTS_BPF_TIME_TRACE) {
@@ -1516,7 +1516,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags,
 
 	ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0);
 	bdt = Alloc(sizeof(BpDataTime));
-	erts_refc_init(&bdt->refc, 1);
+	erts_smp_refc_init(&bdt->refc, 1);
 	bdt->n = erts_no_total_schedulers;
 	bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n));
 	for (i = 0; i < bdt->n; i++) {
@@ -1583,7 +1583,7 @@ clear_function_break(BeamInstr *pc, Uint break_flags)
 static void
 bp_meta_unref(BpMetaTracer* bmt)
 {
-    if (erts_refc_dectest(&bmt->refc, 0) <= 0) {
+    if (erts_smp_refc_dectest(&bmt->refc, 0) <= 0) {
         ErtsTracer trc = erts_smp_atomic_read_nob(&bmt->tracer);
         ERTS_TRACER_CLEAR(&trc);
 	Free(bmt);
@@ -1593,7 +1593,7 @@ bp_meta_unref(BpMetaTracer* bmt)
 static void
 bp_count_unref(BpCount* bcp)
 {
-    if (erts_refc_dectest(&bcp->refc, 0) <= 0) {
+    if (erts_smp_refc_dectest(&bcp->refc, 0) <= 0) {
 	Free(bcp);
     }
 }
@@ -1601,7 +1601,7 @@ bp_count_unref(BpCount* bcp)
 static void
 bp_time_unref(BpDataTime* bdt)
 {
-    if (erts_refc_dectest(&bdt->refc, 0) <= 0) {
+    if (erts_smp_refc_dectest(&bdt->refc, 0) <= 0) {
 	Uint i = 0;
 	Uint j = 0;
 	Process *h_p = NULL;
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index 7206ef471a..27194e31be 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -41,7 +41,7 @@ typedef struct {
 typedef struct bp_data_time {     /* Call time */
     Uint n;
     bp_time_hash_t *hash;
-    erts_refc_t refc;
+    erts_smp_refc_t refc;
 } BpDataTime;
 
 typedef struct {
@@ -51,12 +51,12 @@ typedef struct {
 
 typedef struct {
     erts_smp_atomic_t acount;
-    erts_refc_t refc;
+    erts_smp_refc_t refc;
 } BpCount;
 
 typedef struct {
     erts_smp_atomic_t tracer;
-    erts_refc_t refc;
+    erts_smp_refc_t refc;
 } BpMetaTracer;
 
 typedef struct generic_bp_data {
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 59a9ea1417..9413bc582d 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6678,7 +6678,7 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
     p->htop = hp + needed;
     funp = (ErlFunThing *) hp;
     hp = funp->env;
-    erts_refc_inc(&fe->refc, 2);
+    erts_smp_refc_inc(&fe->refc, 2);
     funp->thing_word = HEADER_FUN;
     funp->next = MSO(p).first;
     MSO(p).first = (struct erl_off_heap_header*) funp;
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 3f2bdf3f9d..88b6e89a02 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4867,7 +4867,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
 		/*
 		 * We are hiding a pointer into older code.
 		 */
-		erts_refc_dec(&fe->refc, 1);
+		erts_smp_refc_dec(&fe->refc, 1);
 	    }
 	    fe->address = code_ptr;
 #ifdef HIPE
@@ -6287,7 +6287,7 @@ patch_funentries(Eterm Patchlist)
      *
      * Reproduced on a debug emulator with stdlib_test/qlc_SUITE:join_merge
      *
-     * erts_refc_dec(&fe->refc, 1);
+     * erts_smp_refc_dec(&fe->refc, 1);
      */
 
     if (!patch(Addresses, (Uint) fe))
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index dfbe1ced47..8020dd484c 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -647,7 +647,7 @@ bin_check(void)
 		erts_printf("%p orig_size: %bpd, norefs = %bpd\n",
 			    bp->val, 
 			    bp->val->orig_size, 
-			    erts_smp_atomic_read_nob(&bp->val->refc));
+			    erts_refc_read(&bp->val->refc, 1));
 	    }
 	}
 	if (printed) {
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index ccc4cbad43..78db141cfc 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -826,7 +826,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 		    funp = (ErlFunThing *) tp;
 		    funp->next = off_heap->first;
 		    off_heap->first = (struct erl_off_heap_header*) funp;
-		    erts_refc_inc(&funp->fe->refc, 2);
+		    erts_smp_refc_inc(&funp->fe->refc, 2);
 		    *argp = make_fun(tp);
 		}
 		break;
@@ -845,7 +845,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 
 		  etp->next = off_heap->first;
 		  off_heap->first = (struct erl_off_heap_header*)etp;
-		  erts_refc_inc(&etp->node->refc, 2);
+		  erts_smp_refc_inc(&etp->node->refc, 2);
 
 		  *argp = make_external(tp);
 		}
@@ -1491,7 +1491,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 		}
 		funp->next = off_heap->first;
 		off_heap->first = (struct erl_off_heap_header*) funp;
-		erts_refc_inc(&funp->fe->refc, 2);
+		erts_smp_refc_inc(&funp->fe->refc, 2);
 		goto cleanup_next;
 	    }
 	    case MAP_SUBTAG:
@@ -1626,7 +1626,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 		}
 		etp->next = off_heap->first;
 		off_heap->first = (struct erl_off_heap_header*) etp;
-		erts_refc_inc(&etp->node->refc, 2);
+		erts_smp_refc_inc(&etp->node->refc, 2);
 		goto cleanup_next;
 	    }
 	    default:
@@ -1802,7 +1802,7 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    case FUN_SUBTAG:
 		{
 		    ErlFunThing* funp = (ErlFunThing *) (tp-1);
-		    erts_refc_inc(&funp->fe->refc, 2);
+		    erts_smp_refc_inc(&funp->fe->refc, 2);
 		}
 		goto off_heap_common;
 	    case EXTERNAL_PID_SUBTAG:
@@ -1810,7 +1810,7 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    case EXTERNAL_REF_SUBTAG:
 		{
 		    ExternalThing* etp = (ExternalThing *) (tp-1);
-		    erts_refc_inc(&etp->node->refc, 2);
+		    erts_smp_refc_inc(&etp->node->refc, 2);
 		}
 	    off_heap_common:
 		{
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index d79245e0e6..52b2174609 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -2081,7 +2081,7 @@ erts_dist_command(Port *prt, int reds_limit)
 
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 
-    erts_refc_inc(&dep->refc, 1); /* Otherwise dist_entry might be
+    erts_smp_refc_inc(&dep->refc, 1); /* Otherwise dist_entry might be
 				     removed if port command fails */
 
     erts_smp_atomic_set_mb(&dep->dist_cmd_scheduled, 0);
@@ -2514,7 +2514,7 @@ info_dist_entry(fmtfn_t to, void *arg, DistEntry *dep, int visible, int connecte
 
   erts_print(to, arg, "Name: %T", dep->sysname);
 #ifdef DEBUG
-  erts_print(to, arg, " (refc=%d)", erts_refc_read(&dep->refc, 0));
+  erts_print(to, arg, " (refc=%d)", erts_smp_refc_read(&dep->refc, 0));
 #endif
   erts_print(to, arg, "\n");
   if (!connected && is_nil(dep->cid)) {
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index ef77201544..bace51bbbb 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1109,25 +1109,25 @@ void erts_ddll_decrement_port_count(DE_Handle *dh)
 static void first_ddll_reference(DE_Handle *dh) 
 {
     assert_drv_list_rwlocked();
-    erts_refc_init(&(dh->refc),1);
+    erts_smp_refc_init(&(dh->refc),1);
 }
 
 void erts_ddll_reference_driver(DE_Handle *dh)
 {
     assert_drv_list_locked();
-    if (erts_refc_inctest(&(dh->refc),1) == 1) {
-	erts_refc_inc(&(dh->refc),2); /* add a reference for the scheduled operation */
+    if (erts_smp_refc_inctest(&(dh->refc),1) == 1) {
+	erts_smp_refc_inc(&(dh->refc),2); /* add a reference for the scheduled operation */
     }
 }
 
 void erts_ddll_reference_referenced_driver(DE_Handle *dh)
 {
-    erts_refc_inc(&(dh->refc),2);
+    erts_smp_refc_inc(&(dh->refc),2);
 }
 
 void erts_ddll_dereference_driver(DE_Handle *dh)
 {
-    if (erts_refc_dectest(&(dh->refc),0) == 0) {
+    if (erts_smp_refc_dectest(&(dh->refc),0) == 0) {
 	/* No lock here, but if the driver is referenced again,
 	   the scheduled deletion is added as a reference too, see above */
 	erts_schedule_misc_op(ddll_no_more_references, (void *) dh);
@@ -1150,11 +1150,11 @@ static void restore_process_references(DE_Handle *dh)
 {
     DE_ProcEntry *p;
     assert_drv_list_rwlocked();
-    ASSERT(erts_refc_read(&(dh->refc),0) == 0);
+    ASSERT(erts_smp_refc_read(&(dh->refc),0) == 0);
     for(p  = dh->procs;p != NULL; p = p->next) {
 	if (p->awaiting_status == ERL_DE_PROC_LOADED) {
 	    ASSERT(p->flags & ERL_DE_FL_DEREFERENCED);
-	    erts_refc_inc(&(dh->refc),1);
+	    erts_smp_refc_inc(&(dh->refc),1);
 	    p->flags &= ~ERL_DE_FL_DEREFERENCED;
 	}
     }
@@ -1176,9 +1176,9 @@ static void ddll_no_more_references(void *vdh)
 
     lock_drv_list();
 
-    x = erts_refc_read(&(dh->refc),0);
+    x = erts_smp_refc_read(&(dh->refc),0);
     if (x > 0) {
-	x = erts_refc_dectest(&(dh->refc),0); /* delete the reference added for me */
+	x = erts_smp_refc_dectest(&(dh->refc),0); /* delete the reference added for me */
     }
 
 
@@ -1643,7 +1643,7 @@ static int load_driver_entry(DE_Handle **dhp, char *path, char *name)
     dh->handle = NULL;
     dh->procs = NULL;
     erts_smp_atomic32_init_nob(&dh->port_count, 0);
-    erts_refc_init(&(dh->refc), (erts_aint_t) 0);
+    erts_smp_refc_init(&(dh->refc), (erts_aint_t) 0);
     dh->status = -1;
     dh->reload_full_path = NULL;
     dh->reload_driver_name = NULL;
@@ -1681,7 +1681,7 @@ static int reload_driver_entry(DE_Handle *dh)
     dh->reload_full_path = NULL;
     dh->reload_driver_name = NULL;
 
-    ASSERT(erts_refc_read(&(dh->refc),0) == 0);
+    ASSERT(erts_smp_refc_read(&(dh->refc),0) == 0);
     ASSERT(dh->full_path != NULL);
     erts_free(ERTS_ALC_T_DDLL_HANDLE, (void *) dh->full_path);
     dh->full_path = NULL;
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 735aabbee3..adf9306029 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -174,7 +174,7 @@ bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh)
 	    if (szp)
 		*szp += 4+2;
 	    if (hpp) {
-		Uint refc = (Uint) erts_smp_atomic_read_nob(&pb->val->refc);
+		Uint refc = (Uint) erts_refc_read(&pb->val->refc, 1);
 		tuple = TUPLE3(*hpp, val, orig_size, make_small(refc));
 		res = CONS(*hpp + 4, tuple, res);
 		*hpp += 4+2;
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index dceadc46f4..3cc2b21a20 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -266,7 +266,7 @@ static void schedule_free_dbtable(DbTable* tb)
      *  	     Caller is *not* allowed to access the specialized part
      *  	     (hash or tree) of *tb after this function has returned.
      */
-    ASSERT(erts_refc_read(&tb->common.ref, 0) == 0);
+    ASSERT(erts_smp_refc_read(&tb->common.ref, 0) == 0);
     erts_schedule_thr_prgr_later_cleanup_op(free_dbtable,
 					    (void *) tb,
 					    &tb->release.data,
@@ -600,11 +600,11 @@ done:
 */
 static ERTS_INLINE void local_fix_table(DbTable* tb)
 {
-    erts_refc_inc(&tb->common.ref, 1);
+    erts_smp_refc_inc(&tb->common.ref, 1);
 }	    
 static ERTS_INLINE void local_unfix_table(DbTable* tb)
 {	
-    if (erts_refc_dectest(&tb->common.ref, 0) == 0) {
+    if (erts_smp_refc_dectest(&tb->common.ref, 0) == 0) {
 	ASSERT(IS_HASH_TABLE(tb->common.status));
 	db_unfix_table_hash(&(tb->hash));
     }
@@ -1487,7 +1487,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     tb->common.type = status & ERTS_ETS_TABLE_TYPES;
     /* Note, 'type' is *read only* from now on... */
 #endif
-    erts_refc_init(&tb->common.ref, 0);
+    erts_smp_refc_init(&tb->common.ref, 0);
     db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ),
 		 "db_tab", "db_tab_fix");
     tb->common.keypos = keypos;
@@ -2990,7 +2990,7 @@ void init_db(ErtsDbSpinCount db_spin_count)
     meta_pid_to_tab->common.meth   = &db_hash;
     meta_pid_to_tab->common.compress = 0;
 
-    erts_refc_init(&meta_pid_to_tab->common.ref, 0);
+    erts_smp_refc_init(&meta_pid_to_tab->common.ref, 0);
     /* Neither rwlock or fixlock used
     db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/
 
@@ -3021,7 +3021,7 @@ void init_db(ErtsDbSpinCount db_spin_count)
     meta_pid_to_fixed_tab->common.meth   = &db_hash;
     meta_pid_to_fixed_tab->common.compress = 0;
 
-    erts_refc_init(&meta_pid_to_fixed_tab->common.ref, 0);
+    erts_smp_refc_init(&meta_pid_to_fixed_tab->common.ref, 0);
     /* Neither rwlock or fixlock used
     db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/
 
@@ -3382,7 +3382,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 			    if ((*pp)->pid == pid) {
 				DbFixation* fix = *pp;
 				erts_aint_t diff = -((erts_aint_t) fix->counter);
-				erts_refc_add(&tb->common.ref,diff,0);
+				erts_smp_refc_add(&tb->common.ref,diff,0);
 				*pp = fix->next;
 				erts_db_free(ERTS_ALC_T_DB_FIXATION,
 					     tb, fix, sizeof(DbFixation));
@@ -3458,7 +3458,7 @@ static void fix_table_locked(Process* p, DbTable* tb)
 #ifdef ERTS_SMP
     erts_smp_mtx_lock(&tb->common.fixlock);
 #endif
-    erts_refc_inc(&tb->common.ref,1);
+    erts_smp_refc_inc(&tb->common.ref,1);
     fix = tb->common.fixations;
     if (fix == NULL) {
 	tb->common.time.monotonic
@@ -3514,7 +3514,7 @@ static void unfix_table_locked(Process* p,  DbTable* tb,
     for (pp = &tb->common.fixations; *pp != NULL; pp = &(*pp)->next) {
 	if ((*pp)->pid == p->common.id) {
 	    DbFixation* fix = *pp;
-	    erts_refc_dec(&tb->common.ref,0);
+	    erts_smp_refc_dec(&tb->common.ref,0);
 	    --(fix->counter);
 	    ASSERT(fix->counter >= 0);
 	    if (fix->counter > 0) {
@@ -3563,7 +3563,7 @@ static void free_fixations_locked(DbTable *tb)
     fix = tb->common.fixations;
     while (fix != NULL) {
 	erts_aint_t diff = -((erts_aint_t) fix->counter);
-	erts_refc_add(&tb->common.ref,diff,0);
+	erts_smp_refc_add(&tb->common.ref,diff,0);
 	next_fix = fix->next;
 	db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
 	db_erase_bag_exact2(meta_pid_to_fixed_tab,
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 6732b708a8..f4db7ca3dd 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -3107,7 +3107,7 @@ void db_cleanup_offheap_comp(DbTerm* obj)
 	    break;
 	case FUN_SUBTAG:
 	    ASSERT(u.pb != &tmp);
-	    if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) {
+	    if (erts_smp_refc_dectest(&u.fun->fe->refc, 0) == 0) {
 		erts_erase_fun_entry(u.fun->fe);
 	    }
 	    break;
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 49e5f6b4cf..3fb063797e 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -211,7 +211,7 @@ typedef struct db_fixation {
  */
 
 typedef struct db_table_common {
-    erts_refc_t ref;          /* fixation counter */
+    erts_smp_refc_t ref;          /* fixation counter */
 #ifdef ERTS_SMP
     erts_smp_rwmtx_t rwlock;  /* rw lock on table */
     erts_smp_mtx_t fixlock;   /* Protects fixations,megasec,sec,microsec */
@@ -260,7 +260,7 @@ typedef struct db_table_common {
 				  (DB_BAG | DB_SET | DB_DUPLICATE_BAG)))
 #define IS_TREE_TABLE(Status) (!!((Status) & \
 				  DB_ORDERED_SET))
-#define NFIXED(T) (erts_refc_read(&(T)->common.ref,0))
+#define NFIXED(T) (erts_smp_refc_read(&(T)->common.ref,0))
 #define IS_FIXED(T) (NFIXED(T) != 0) 
 
 /*
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index d0a57f0ad0..fd4d5b1c5c 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -110,9 +110,9 @@ erts_put_fun_entry(Eterm mod, int uniq, int index)
     fe = (ErlFunEntry *) hash_put(&erts_fun_table, (void*) &template);
     sys_memset(fe->uniq, 0, sizeof(fe->uniq));
     fe->index = 0;
-    refc = erts_refc_inctest(&fe->refc, 0);
+    refc = erts_smp_refc_inctest(&fe->refc, 0);
     if (refc < 2) /* New or pending delete */
-	erts_refc_inc(&fe->refc, 1);
+	erts_smp_refc_inc(&fe->refc, 1);
     erts_fun_write_unlock();
     return fe;
 }
@@ -134,9 +134,9 @@ erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index,
     sys_memcpy(fe->uniq, uniq, sizeof(fe->uniq));
     fe->index = index;
     fe->arity = arity;
-    refc = erts_refc_inctest(&fe->refc, 0);
+    refc = erts_smp_refc_inctest(&fe->refc, 0);
     if (refc < 2) /* New or pending delete */
-	erts_refc_inc(&fe->refc, 1);
+	erts_smp_refc_inc(&fe->refc, 1);
     erts_fun_write_unlock();
     return fe;
 }
@@ -161,9 +161,9 @@ erts_get_fun_entry(Eterm mod, int uniq, int index)
     erts_fun_read_lock();
     ret = (ErlFunEntry *) hash_get(&erts_fun_table, (void*) &template);
     if (ret) {
-	erts_aint_t refc = erts_refc_inctest(&ret->refc, 1);
+	erts_aint_t refc = erts_smp_refc_inctest(&ret->refc, 1);
 	if (refc < 2) /* Pending delete */
-	    erts_refc_inc(&ret->refc, 1);
+	    erts_smp_refc_inc(&ret->refc, 1);
     }
     erts_fun_read_unlock();
     return ret;
@@ -184,7 +184,7 @@ erts_erase_fun_entry(ErlFunEntry* fe)
      * We have to check refc again since someone might have looked up
      * the fun entry and incremented refc after last check.
      */
-    if (erts_refc_dectest(&fe->refc, -1) <= 0)
+    if (erts_smp_refc_dectest(&fe->refc, -1) <= 0)
 #endif
     {
 	if (fe->address != unloaded_fun)
@@ -256,7 +256,7 @@ erts_fun_purge_complete(ErlFunEntry **funs, Uint no)
     for (ix = 0; ix < no; ix++) {
 	ErlFunEntry *fe = funs[ix];
 	fe->pend_purge_address = NULL;
-	if (erts_refc_dectest(&fe->refc, 0) == 0)
+	if (erts_smp_refc_dectest(&fe->refc, 0) == 0)
 	    erts_erase_fun_entry(fe);
     }
     ERTS_SMP_WRITE_MEMORY_BARRIER;
@@ -288,7 +288,7 @@ erts_dump_fun_entries(fmtfn_t to, void *to_arg)
 #ifdef HIPE
 	    erts_print(to, to_arg, "Native_address: %p\n", fe->native_address);
 #endif
-	    erts_print(to, to_arg, "Refc: %ld\n", erts_refc_read(&fe->refc, 1));
+	    erts_print(to, to_arg, "Refc: %ld\n", erts_smp_refc_read(&fe->refc, 1));
 	    b = b->next;
 	}
     }
@@ -319,7 +319,7 @@ fun_alloc(ErlFunEntry* template)
     obj->old_uniq = template->old_uniq;
     obj->old_index = template->old_index;
     obj->module = template->module;
-    erts_refc_init(&obj->refc, -1);
+    erts_smp_refc_init(&obj->refc, -1);
     obj->address = unloaded_fun;
     obj->pend_purge_address = NULL;
 #ifdef HIPE
diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h
index caa55c730c..0fd0d62343 100644
--- a/erts/emulator/beam/erl_fun.h
+++ b/erts/emulator/beam/erl_fun.h
@@ -42,7 +42,7 @@ typedef struct erl_fun_entry {
 
     Uint arity;			/* The arity of the fun. */
     Eterm module;		/* Tagged atom for module. */
-    erts_refc_t refc;		/* Reference count: One for code + one for each
+    erts_smp_refc_t refc;		/* Reference count: One for code + one for each
 				   fun object in each process. */
     BeamInstr *pend_purge_address; /* address stored during a pending purge */
 } ErlFunEntry;
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index af799d09da..704c282869 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2718,7 +2718,7 @@ sweep_off_heap(Process *p, int fullsweep)
 	    case FUN_SUBTAG:
 		{
 		    ErlFunEntry* fe = ((ErlFunThing*)ptr)->fe;
-		    if (erts_refc_dectest(&fe->refc, 0) == 0) {
+		    if (erts_smp_refc_dectest(&fe->refc, 0) == 0) {
 			erts_erase_fun_entry(fe);
 		    }
 		    break;
@@ -3374,12 +3374,12 @@ erts_check_off_heap2(Process *p, Eterm *htop)
 	    refc = erts_refc_read(&u.pb->val->refc, 1);		
 	    break;
 	case FUN_SUBTAG:
-	    refc = erts_refc_read(&u.fun->fe->refc, 1);
+	    refc = erts_smp_refc_read(&u.fun->fe->refc, 1);
 	    break;
 	case EXTERNAL_PID_SUBTAG:
 	case EXTERNAL_PORT_SUBTAG:
 	case EXTERNAL_REF_SUBTAG:
-	    refc = erts_refc_read(&u.ext->node->refc, 1);
+	    refc = erts_smp_refc_read(&u.ext->node->refc, 1);
 	    break;
 	default:
 	    ASSERT(!"erts_check_off_heap2: Invalid thing_word");
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index e4c696ae3b..13c597adce 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -172,7 +172,7 @@ erts_cleanup_offheap(ErlOffHeap *offheap)
 	    }
 	    break;
 	case FUN_SUBTAG:
-	    if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) {
+	    if (erts_smp_refc_dectest(&u.fun->fe->refc, 0) == 0) {
 		erts_erase_fun_entry(u.fun->fe);		    
 	    }
 	    break;
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
index 910598690d..c207dea10f 100644
--- a/erts/emulator/beam/erl_monitors.c
+++ b/erts/emulator/beam/erl_monitors.c
@@ -104,7 +104,7 @@ do {								\
 	    (*((Hp)++)) = boxed_val((From))[i__];		\
 	if (is_external((To))) {				\
 	    external_thing_ptr((To))->next = NULL;		\
-	    erts_refc_inc(&(external_thing_ptr((To))->node->refc), 2);\
+	    erts_smp_refc_inc(&(external_thing_ptr((To))->node->refc), 2);\
 	}							\
     }								\
 } while (0)
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 70500ed6e1..467a05f950 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -98,7 +98,7 @@ dist_table_alloc(void *dep_tmpl)
     dist_entries++;
 
     dep->prev				= NULL;
-    erts_refc_init(&dep->refc, -1);
+    erts_smp_refc_init(&dep->refc, -1);
     erts_smp_rwmtx_init_opt_x(&dep->rwmtx, &rwmtx_opt, "dist_entry", chnl_nr);
     dep->sysname			= sysname;
     dep->cid				= NIL;
@@ -208,7 +208,7 @@ erts_channel_no_to_dist_entry(Uint cno)
  * to the node name is used as channel no.
  */
     if(cno == ERST_INTERNAL_CHANNEL_NO) {
-	erts_refc_inc(&erts_this_dist_entry->refc, 2);
+	erts_smp_refc_inc(&erts_this_dist_entry->refc, 2);
 	return erts_this_dist_entry;
     }
 
@@ -231,16 +231,16 @@ erts_sysname_to_connected_dist_entry(Eterm sysname)
     de.sysname = sysname;
   
     if(erts_this_dist_entry->sysname == sysname) {
-	erts_refc_inc(&erts_this_dist_entry->refc, 2);
+	erts_smp_refc_inc(&erts_this_dist_entry->refc, 2);
 	return erts_this_dist_entry;
     }
 
     erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
     res_dep = (DistEntry *) hash_get(&erts_dist_table, (void *) &de);
     if (res_dep) {
-	erts_aint_t refc = erts_refc_inctest(&res_dep->refc, 1);
+	erts_aint_t refc = erts_smp_refc_inctest(&res_dep->refc, 1);
 	if (refc < 2) /* Pending delete */
-	    erts_refc_inc(&res_dep->refc, 1);
+	    erts_smp_refc_inc(&res_dep->refc, 1);
     }
     erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
     if (res_dep) {
@@ -267,9 +267,9 @@ DistEntry *erts_find_or_insert_dist_entry(Eterm sysname)
     de.sysname = sysname;
     erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx);
     res = hash_put(&erts_dist_table, (void *) &de);
-    refc = erts_refc_inctest(&res->refc, 0);
+    refc = erts_smp_refc_inctest(&res->refc, 0);
     if (refc < 2) /* New or pending delete */
-	erts_refc_inc(&res->refc, 1);
+	erts_smp_refc_inc(&res->refc, 1);
     erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
     return res;
 }
@@ -282,9 +282,9 @@ DistEntry *erts_find_dist_entry(Eterm sysname)
     erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
     res = hash_get(&erts_dist_table, (void *) &de);
     if (res) {
-	erts_aint_t refc = erts_refc_inctest(&res->refc, 1);
+	erts_aint_t refc = erts_smp_refc_inctest(&res->refc, 1);
 	if (refc < 2) /* Pending delete */
-	    erts_refc_inc(&res->refc, 1);
+	    erts_smp_refc_inc(&res->refc, 1);
     }
     erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
     return res;
@@ -311,7 +311,7 @@ static void try_delete_dist_entry(void *vdep)
      *
      * If refc > 0, the entry is in use. Keep the entry.
      */
-    refc = erts_refc_dectest(&dep->refc, -1);
+    refc = erts_smp_refc_dectest(&dep->refc, -1);
     if (refc == -1)
 	(void) hash_erase(&erts_dist_table, (void *) dep);
     erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
@@ -518,7 +518,7 @@ node_table_alloc(void *venp_tmpl)
 
     node_entries++;
 
-    erts_refc_init(&enp->refc, -1);
+    erts_smp_refc_init(&enp->refc, -1);
     enp->creation = ((ErlNode *) venp_tmpl)->creation;
     enp->sysname = ((ErlNode *) venp_tmpl)->sysname;
     enp->dist_entry = erts_find_or_insert_dist_entry(((ErlNode *) venp_tmpl)->sysname);
@@ -585,9 +585,9 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint32 creation)
     erts_smp_rwmtx_rlock(&erts_node_table_rwmtx);
     res = hash_get(&erts_node_table, (void *) &ne);
     if (res && res != erts_this_node) {
-	erts_aint_t refc = erts_refc_inctest(&res->refc, 0);
+	erts_aint_t refc = erts_smp_refc_inctest(&res->refc, 0);
 	if (refc < 2) /* New or pending delete */
-	    erts_refc_inc(&res->refc, 1);
+	    erts_smp_refc_inc(&res->refc, 1);
     }
     erts_smp_rwmtx_runlock(&erts_node_table_rwmtx);
     if (res)
@@ -597,9 +597,9 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint32 creation)
     res = hash_put(&erts_node_table, (void *) &ne);
     ASSERT(res);
     if (res != erts_this_node) {
-	erts_aint_t refc = erts_refc_inctest(&res->refc, 0);
+	erts_aint_t refc = erts_smp_refc_inctest(&res->refc, 0);
 	if (refc < 2) /* New or pending delete */
-	    erts_refc_inc(&res->refc, 1);
+	    erts_smp_refc_inc(&res->refc, 1);
     }
     erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx);
     return res;
@@ -626,7 +626,7 @@ static void try_delete_node(void *venp)
      *
      * If refc > 0, the entry is in use. Keep the entry.
      */
-    refc = erts_refc_dectest(&enp->refc, -1);
+    refc = erts_smp_refc_dectest(&enp->refc, -1);
     if (refc == -1)
 	(void) hash_erase(&erts_node_table, (void *) enp);
     erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx);
@@ -672,7 +672,7 @@ static void print_node(void *venp, void *vpndp)
 	erts_print(pndp->to, pndp->to_arg, " %d", enp->creation);
 #ifdef DEBUG
 	erts_print(pndp->to, pndp->to_arg, " (refc=%ld)",
-		   erts_refc_read(&enp->refc, 0));
+		   erts_smp_refc_read(&enp->refc, 0));
 #endif
 	pndp->no_sysname++;
     }
@@ -715,19 +715,19 @@ void
 erts_set_this_node(Eterm sysname, Uint creation)
 {
     ERTS_SMP_LC_ASSERT(erts_thr_progress_is_blocking());
-    ASSERT(erts_refc_read(&erts_this_dist_entry->refc, 2));
+    ASSERT(erts_smp_refc_read(&erts_this_dist_entry->refc, 2));
 
-    if (erts_refc_dectest(&erts_this_node->refc, 0) == 0)
+    if (erts_smp_refc_dectest(&erts_this_node->refc, 0) == 0)
         try_delete_node(erts_this_node);
 
-    if (erts_refc_dectest(&erts_this_dist_entry->refc, 0) == 0)
+    if (erts_smp_refc_dectest(&erts_this_dist_entry->refc, 0) == 0)
         try_delete_dist_entry(erts_this_dist_entry);
 
     erts_this_node = NULL; /* to make sure refc is bumped for this node */
     erts_this_node = erts_find_or_insert_node(sysname, creation);
     erts_this_dist_entry = erts_this_node->dist_entry;
 
-    erts_refc_inc(&erts_this_dist_entry->refc, 2);
+    erts_smp_refc_inc(&erts_this_dist_entry->refc, 2);
 
     erts_this_node_sysname = erts_this_node_sysname_BUFFER;
     erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER),
@@ -789,13 +789,13 @@ void erts_init_node_tables(int dd_sec)
     node_tmpl.creation = 0;
     erts_this_node = hash_put(&erts_node_table, &node_tmpl);
      /* +1 for erts_this_node */
-    erts_refc_init(&erts_this_node->refc, 1);
+    erts_smp_refc_init(&erts_this_node->refc, 1);
 
     ASSERT(erts_this_node->dist_entry != NULL);
     erts_this_dist_entry = erts_this_node->dist_entry;
     /* +1 for erts_this_dist_entry */
     /* +1 for erts_this_node->dist_entry */
-    erts_refc_init(&erts_this_dist_entry->refc, 2);
+    erts_smp_refc_init(&erts_this_dist_entry->refc, 2);
 
 
     erts_this_node_sysname = erts_this_node_sysname_BUFFER;
@@ -1623,7 +1623,7 @@ reference_table_term(Uint **hpp, Uint *szp)
 
 	tup = MK_2TUP(referred_nodes[i].node->sysname,
 		      MK_UINT(referred_nodes[i].node->creation));
-	tup = MK_3TUP(tup, MK_UINT(erts_refc_read(&referred_nodes[i].node->refc, 0)), nril);
+	tup = MK_3TUP(tup, MK_UINT(erts_smp_refc_read(&referred_nodes[i].node->refc, 0)), nril);
 	nl = MK_CONS(tup, nl);
     }
 
@@ -1684,7 +1684,7 @@ reference_table_term(Uint **hpp, Uint *szp)
 
 	/* DistList = [{Dist, Refc, ReferenceIdList}] */
 	tup = MK_3TUP(referred_dists[i].dist->sysname,
-		      MK_UINT(erts_refc_read(&referred_dists[i].dist->refc, 0)),
+		      MK_UINT(erts_smp_refc_read(&referred_dists[i].dist->refc, 0)),
 		      dril);
 	dl = MK_CONS(tup, dl);
     }
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 47a6724c21..35051173d0 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -107,7 +107,7 @@ typedef struct dist_entry_ {
     HashBucket hash_bucket;     /* Hash bucket */
     struct dist_entry_ *next;	/* Next entry in dist_table (not sorted) */
     struct dist_entry_ *prev;	/* Previous entry in dist_table (not sorted) */
-    erts_refc_t refc;		/* Reference count */
+    erts_smp_refc_t refc;		/* Reference count */
 
     erts_smp_rwmtx_t rwmtx;     /* Protects all fields below until lck_mtx. */
     Eterm sysname;		/* name@host atom for efficiency */
@@ -149,7 +149,7 @@ typedef struct dist_entry_ {
 
 typedef struct erl_node_ {
   HashBucket hash_bucket;	/* Hash bucket */
-  erts_refc_t refc;		/* Reference count */
+  erts_smp_refc_t refc;		/* Reference count */
   Eterm	sysname;		/* name@host atom for efficiency */
   Uint32 creation;		/* Creation */
   DistEntry *dist_entry;	/* Corresponding dist entry */
@@ -210,7 +210,7 @@ ERTS_GLB_INLINE void
 erts_deref_dist_entry(DistEntry *dep)
 {
     ASSERT(dep);
-    if (erts_refc_dectest(&dep->refc, 0) == 0)
+    if (erts_smp_refc_dectest(&dep->refc, 0) == 0)
 	erts_schedule_delete_dist_entry(dep);
 }
 
@@ -218,7 +218,7 @@ ERTS_GLB_INLINE void
 erts_deref_node_entry(ErlNode *np)
 {
     ASSERT(np);
-    if (erts_refc_dectest(&np->refc, 0) == 0)
+    if (erts_smp_refc_dectest(&np->refc, 0) == 0)
 	erts_schedule_delete_node(np);
 }
 
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index d8bb00e8c6..c5c62c5438 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -443,7 +443,7 @@ heap_dump(fmtfn_t to, void *to_arg, Eterm x)
 			ProcBin* pb = (ProcBin *) binary_val(x);
 			Binary* val = pb->val;
 
-			if (erts_smp_atomic_xchg_nob(&val->refc, 0) != 0) {
+			if (erts_atomic_xchg_nob(&val->refc, 0) != 0) {
 			    val->flags = (UWord) all_binaries;
 			    all_binaries = val;
 			}
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index beed847578..7963e745c7 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -616,7 +616,7 @@ erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize)
     sys_memcpy((void *) ep, (void *) edep, dist_ext_sz);
     ep += dist_ext_sz;
     if (new_edep->dep)
-	erts_refc_inc(&new_edep->dep->refc, 1);
+	erts_smp_refc_inc(&new_edep->dep->refc, 1);
     new_edep->extp = ep;
     new_edep->ext_endp = ep + ext_sz;
     new_edep->heap_size = -1;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index d6df85034c..26aa39b65a 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -127,7 +127,7 @@ typedef struct {
     void         *handle;             /* Handle for DLL or SO (for dyn. drivers). */
     DE_ProcEntry *procs;              /* List of pids that have loaded this driver,
 				         or that wait for it to change state */
-    erts_refc_t  refc;                /* Number of ports/processes having
+    erts_smp_refc_t  refc;                /* Number of ports/processes having
 					 references to the driver */
     erts_smp_atomic32_t port_count;   /* Number of ports using the driver */
     Uint         flags;               /* ERL_DE_FL_KILL_PORTS */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 4f131c74de..31396d06c6 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -6890,12 +6890,6 @@ ErlDrvSizeT driver_vec_to_buf(ErlIOVec *vec, char *buf, ErlDrvSizeT len)
     return (orig_len - len);
 }
 
-
-/*
- * - driver_alloc_binary() is thread safe (efile driver depend on it).
- * - driver_realloc_binary(), and driver_free_binary() are *not* thread safe.
- */
-
 /*
  * reference count on driver binaries...
  */
@@ -6938,26 +6932,15 @@ driver_alloc_binary(ErlDrvSizeT size)
     return Binary2ErlDrvBinary(bin);
 }
 
-/* Reallocate space hold by binary */
+/* Reallocate space held by binary */
 
 ErlDrvBinary* driver_realloc_binary(ErlDrvBinary* bin, ErlDrvSizeT size)
 {
     Binary* oldbin;
     Binary* newbin;
 
-    if (!bin) {
-	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	erts_dsprintf(dsbufp,
-		      "Bad use of driver_realloc_binary(%p, %lu): "
-		      "called with ",
-		      bin, (unsigned long)size);
-	if (!bin) {
-	    erts_dsprintf(dsbufp, "NULL pointer as first argument");
-	}
-	erts_send_warning_to_logger_nogl(dsbufp);
-	if (!bin)
-	    return driver_alloc_binary(size);
-    }
+    if (!bin)
+	return driver_alloc_binary(size);
 
     oldbin = ErlDrvBinary2Binary(bin);
     newbin = (Binary *) erts_bin_realloc_fnf(oldbin, size);
@@ -6971,14 +6954,8 @@ ErlDrvBinary* driver_realloc_binary(ErlDrvBinary* bin, ErlDrvSizeT size)
 void driver_free_binary(ErlDrvBinary* dbin)
 {
     Binary *bin;
-    if (!dbin) {
-	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	erts_dsprintf(dsbufp,
-		      "Bad use of driver_free_binary(%p): called with "
-		      "NULL pointer as argument", dbin);
-	erts_send_warning_to_logger_nogl(dsbufp);
+    if (!dbin)
 	return;
-    }
 
     bin = ErlDrvBinary2Binary(dbin);
     if (erts_refc_dectest(&bin->refc, 0) == 0)
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 7740dd4373..acffef0d2c 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -894,7 +894,7 @@ void sys_alloc_stat(SysAllocStat *);
 #define ERTS_REFC_DEBUG
 #endif
 
-typedef erts_smp_atomic_t erts_refc_t;
+typedef erts_atomic_t erts_refc_t;
 
 ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val);
 ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val);
@@ -913,27 +913,27 @@ ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp,
 ERTS_GLB_INLINE void
 erts_refc_init(erts_refc_t *refcp, erts_aint_t val)
 {
-    erts_smp_atomic_init_nob((erts_smp_atomic_t *) refcp, val);
+    erts_atomic_init_nob((erts_atomic_t *) refcp, val);
 }
 
 ERTS_GLB_INLINE void
 erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val)
 {
 #ifdef ERTS_REFC_DEBUG
-    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
+    erts_aint_t val = erts_atomic_inc_read_nob((erts_atomic_t *) refcp);
     if (val < min_val)
 	erts_exit(ERTS_ABORT_EXIT,
 		 "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n",
 		 val, min_val);
 #else
-    erts_smp_atomic_inc_nob((erts_smp_atomic_t *) refcp);
+    erts_atomic_inc_nob((erts_atomic_t *) refcp);
 #endif
 }
 
 ERTS_GLB_INLINE erts_aint_t
 erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val)
 {
-    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
+    erts_aint_t val = erts_atomic_inc_read_nob((erts_atomic_t *) refcp);
 #ifdef ERTS_REFC_DEBUG
     if (val < min_val)
 	erts_exit(ERTS_ABORT_EXIT,
@@ -947,20 +947,20 @@ ERTS_GLB_INLINE void
 erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val)
 {
 #ifdef ERTS_REFC_DEBUG
-    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
+    erts_aint_t val = erts_atomic_dec_read_nob((erts_atomic_t *) refcp);
     if (val < min_val)
 	erts_exit(ERTS_ABORT_EXIT,
 		 "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n",
 		 val, min_val);
 #else
-    erts_smp_atomic_dec_nob((erts_smp_atomic_t *) refcp);
+    erts_atomic_dec_nob((erts_atomic_t *) refcp);
 #endif
 }
 
 ERTS_GLB_INLINE erts_aint_t
 erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val)
 {
-    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
+    erts_aint_t val = erts_atomic_dec_read_nob((erts_atomic_t *) refcp);
 #ifdef ERTS_REFC_DEBUG
     if (val < min_val)
 	erts_exit(ERTS_ABORT_EXIT,
@@ -974,20 +974,20 @@ ERTS_GLB_INLINE void
 erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val)
 {
 #ifdef ERTS_REFC_DEBUG
-    erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff);
+    erts_aint_t val = erts_atomic_add_read_nob((erts_atomic_t *) refcp, diff);
     if (val < min_val)
 	erts_exit(ERTS_ABORT_EXIT,
 		 "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n",
 		 diff, val, min_val);
 #else
-    erts_smp_atomic_add_nob((erts_smp_atomic_t *) refcp, diff);
+    erts_atomic_add_nob((erts_atomic_t *) refcp, diff);
 #endif
 }
 
 ERTS_GLB_INLINE erts_aint_t
 erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
 {
-    erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp);
+    erts_aint_t val = erts_atomic_read_nob((erts_atomic_t *) refcp);
 #ifdef ERTS_REFC_DEBUG
     if (val < min_val)
 	erts_exit(ERTS_ABORT_EXIT,
@@ -999,6 +999,112 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
+typedef erts_smp_atomic_t erts_smp_refc_t;
+
+ERTS_GLB_INLINE void erts_smp_refc_init(erts_smp_refc_t *refcp, erts_aint_t val);
+ERTS_GLB_INLINE void erts_smp_refc_inc(erts_smp_refc_t *refcp, erts_aint_t min_val);
+ERTS_GLB_INLINE erts_aint_t erts_smp_refc_inctest(erts_smp_refc_t *refcp,
+                                                  erts_aint_t min_val);
+ERTS_GLB_INLINE void erts_smp_refc_dec(erts_smp_refc_t *refcp, erts_aint_t min_val);
+ERTS_GLB_INLINE erts_aint_t erts_smp_refc_dectest(erts_smp_refc_t *refcp,
+                                                  erts_aint_t min_val);
+ERTS_GLB_INLINE void erts_smp_refc_add(erts_smp_refc_t *refcp, erts_aint_t diff,
+                                       erts_aint_t min_val);
+ERTS_GLB_INLINE erts_aint_t erts_smp_refc_read(erts_smp_refc_t *refcp,
+                                               erts_aint_t min_val);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE void
+erts_smp_refc_init(erts_smp_refc_t *refcp, erts_aint_t val)
+{
+    erts_smp_atomic_init_nob((erts_smp_atomic_t *) refcp, val);
+}
+
+ERTS_GLB_INLINE void
+erts_smp_refc_inc(erts_smp_refc_t *refcp, erts_aint_t min_val)
+{
+#ifdef ERTS_REFC_DEBUG
+    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
+    if (val < min_val)
+	erts_exit(ERTS_ABORT_EXIT,
+		 "erts_smp_refc_inc(): Bad refc found (refc=%ld < %ld)!\n",
+		 val, min_val);
+#else
+    erts_smp_atomic_inc_nob((erts_smp_atomic_t *) refcp);
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint_t
+erts_smp_refc_inctest(erts_smp_refc_t *refcp, erts_aint_t min_val)
+{
+    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
+#ifdef ERTS_REFC_DEBUG
+    if (val < min_val)
+	erts_exit(ERTS_ABORT_EXIT,
+		 "erts_smp_refc_inctest(): Bad refc found (refc=%ld < %ld)!\n",
+		 val, min_val);
+#endif
+    return val;
+}
+
+ERTS_GLB_INLINE void
+erts_smp_refc_dec(erts_smp_refc_t *refcp, erts_aint_t min_val)
+{
+#ifdef ERTS_REFC_DEBUG
+    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
+    if (val < min_val)
+	erts_exit(ERTS_ABORT_EXIT,
+		 "erts_smp_refc_dec(): Bad refc found (refc=%ld < %ld)!\n",
+		 val, min_val);
+#else
+    erts_smp_atomic_dec_nob((erts_smp_atomic_t *) refcp);
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint_t
+erts_smp_refc_dectest(erts_smp_refc_t *refcp, erts_aint_t min_val)
+{
+    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
+#ifdef ERTS_REFC_DEBUG
+    if (val < min_val)
+	erts_exit(ERTS_ABORT_EXIT,
+		 "erts_smp_refc_dectest(): Bad refc found (refc=%ld < %ld)!\n",
+		 val, min_val);
+#endif
+    return val;
+}
+
+ERTS_GLB_INLINE void
+erts_smp_refc_add(erts_smp_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val)
+{
+#ifdef ERTS_REFC_DEBUG
+    erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff);
+    if (val < min_val)
+	erts_exit(ERTS_ABORT_EXIT,
+		 "erts_smp_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n",
+		 diff, val, min_val);
+#else
+    erts_smp_atomic_add_nob((erts_smp_atomic_t *) refcp, diff);
+#endif
+}
+
+ERTS_GLB_INLINE erts_aint_t
+erts_smp_refc_read(erts_smp_refc_t *refcp, erts_aint_t min_val)
+{
+    erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp);
+#ifdef ERTS_REFC_DEBUG
+    if (val < min_val)
+	erts_exit(ERTS_ABORT_EXIT,
+		 "erts_smp_refc_read(): Bad refc found (refc=%ld < %ld)!\n",
+		 val, min_val);
+#endif
+    return val;
+}
+
+#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+
 #ifdef ERTS_ENABLE_KERNEL_POLL
 extern int erts_use_kernel_poll;
 #endif
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 87ea4f05a1..7f7e38c22a 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3842,7 +3842,7 @@ store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns)
 	for(i = 0; i < size; i++)
 	    to_hp[i] = from_hp[i];
 
-	erts_refc_inc(&((ExternalThing *) to_hp)->node->refc, 2);
+	erts_smp_refc_inc(&((ExternalThing *) to_hp)->node->refc, 2);
 
 	((struct erl_off_heap_header*) to_hp)->next = oh->first;
 	oh->first = (struct erl_off_heap_header*) to_hp;
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index dcb6c35bfa..165f33eecd 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -992,7 +992,7 @@ BIF_RETTYPE hipe_bifs_set_native_address_in_fe_2(BIF_ALIST_2)
 	BIF_ERROR(BIF_P, BADARG);
 
     fe->native_address = native_address;
-    if (erts_refc_dectest(&fe->refc, 0) == 0)
+    if (erts_smp_refc_dectest(&fe->refc, 0) == 0)
 	erts_erase_fun_entry(fe);
     BIF_RET(am_true);
 }
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index 9c03b3811c..709acdb816 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -331,7 +331,7 @@ char *hipe_bs_allocate(int len)
     Binary *bptr;
 
     bptr = erts_bin_nrml_alloc(len);
-    erts_smp_atomic_init_nob(&bptr->refc, 1);
+    erts_refc_init(&bptr->refc, 1);
     return bptr->orig_bytes;
 }
 
-- 
cgit v1.2.3


From aefe39da715130f3d1df10084495d3b7ee48337e Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 24 Jan 2017 20:03:57 +0100
Subject: Implement erts_refc_inc_unless()

---
 erts/emulator/beam/sys.h | 55 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 16b38b8c6f..c6ea8049c3 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -893,6 +893,9 @@ typedef erts_atomic_t erts_refc_t;
 
 ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val);
 ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val);
+ERTS_GLB_INLINE erts_aint_t erts_refc_inc_unless(erts_refc_t *refcp,
+                                                 erts_aint_t unless_val,
+                                                 erts_aint_t min_val);
 ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp,
 					      erts_aint_t min_val);
 ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val);
@@ -925,6 +928,30 @@ erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val)
 #endif
 }
 
+ERTS_GLB_INLINE erts_aint_t
+erts_refc_inc_unless(erts_refc_t *refcp,
+                     erts_aint_t unless_val,
+                     erts_aint_t min_val)
+{
+    erts_aint_t val = erts_atomic_read_nob((erts_atomic_t *) refcp);
+    while (1) {
+        erts_aint_t exp, new;
+#ifdef ERTS_REFC_DEBUG
+        if (val < 0)
+            erts_exit(ERTS_ABORT_EXIT,
+                      "erts_refc_inc_unless(): Bad refc found (refc=%ld < %ld)!\n",
+                      val, min_val);
+#endif
+        if (val == unless_val)
+            return val;
+        new = val + 1;
+        exp = val;
+        val = erts_atomic_cmpxchg_nob((erts_atomic_t *) refcp, new, exp);
+        if (val == exp)
+            return new;
+    }
+}
+
 ERTS_GLB_INLINE erts_aint_t
 erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val)
 {
@@ -998,6 +1025,9 @@ typedef erts_smp_atomic_t erts_smp_refc_t;
 
 ERTS_GLB_INLINE void erts_smp_refc_init(erts_smp_refc_t *refcp, erts_aint_t val);
 ERTS_GLB_INLINE void erts_smp_refc_inc(erts_smp_refc_t *refcp, erts_aint_t min_val);
+ERTS_GLB_INLINE erts_aint_t erts_smp_refc_inc_unless(erts_smp_refc_t *refcp,
+                                                     erts_aint_t unless_val,
+                                                     erts_aint_t min_val);
 ERTS_GLB_INLINE erts_aint_t erts_smp_refc_inctest(erts_smp_refc_t *refcp,
                                                   erts_aint_t min_val);
 ERTS_GLB_INLINE void erts_smp_refc_dec(erts_smp_refc_t *refcp, erts_aint_t min_val);
@@ -1030,6 +1060,31 @@ erts_smp_refc_inc(erts_smp_refc_t *refcp, erts_aint_t min_val)
 #endif
 }
 
+ERTS_GLB_INLINE erts_aint_t
+erts_smp_refc_inc_unless(erts_smp_refc_t *refcp,
+                         erts_aint_t unless_val,
+                         erts_aint_t min_val)
+{
+    erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp);
+    while (1) {
+        erts_aint_t exp, new;
+#ifdef ERTS_REFC_DEBUG
+        if (val < 0)
+            erts_exit(ERTS_ABORT_EXIT,
+                      "erts_smp_refc_inc_unless(): Bad refc found (refc=%ld < %ld)!\n",
+                      val, min_val);
+#endif
+        if (val == unless_val)
+            return val;
+        new = val + 1;
+        exp = val;
+        val = erts_smp_atomic_cmpxchg_nob((erts_smp_atomic_t *) refcp, new, exp);
+        if (val == exp)
+            return new;
+    }
+}
+
+
 ERTS_GLB_INLINE erts_aint_t
 erts_smp_refc_inctest(erts_smp_refc_t *refcp, erts_aint_t min_val)
 {
-- 
cgit v1.2.3


From b079018e38272604ffacfece9b97924a9e39df5c Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 23 Jan 2017 17:10:18 +0100
Subject: Implement magic references

Magic references are *intentionally* indistinguishable from ordinary
references for the Erlang software. Magic references do not change
the language, and are intended as a pure runtime internal optimization.

An ordinary reference is typically used as a key in some table. A
magic reference has a direct pointer to a reference counted magic
binary. This makes it possible to implement various things without
having to do lookups in a table, but instead access the data directly.
Besides very fast lookups this can also improve scalability by
removing a potentially contended table. A couple of examples of
planned future usage of magic references are ETS table identifiers,
and BIF timer identifiers.

Besides future optimizations using magic references it should also
be possible to replace the exposed magic binary cludge with magic
references. That is, magic binaries that are exposed as empty
binaries to the Erlang software.
---
 erts/emulator/beam/atom.names                 |   1 +
 erts/emulator/beam/bif.c                      |  15 +-
 erts/emulator/beam/binary.c                   |   9 +-
 erts/emulator/beam/copy.c                     |  60 ++++-
 erts/emulator/beam/erl_alloc.c                |  19 +-
 erts/emulator/beam/erl_alloc.types            |   4 +
 erts/emulator/beam/erl_bif_ddll.c             |  14 +-
 erts/emulator/beam/erl_bif_info.c             |  87 ++++++-
 erts/emulator/beam/erl_bif_port.c             |  14 +-
 erts/emulator/beam/erl_bif_trace.c            |   2 +-
 erts/emulator/beam/erl_bif_unique.c           | 310 ++++++++++++++++++++++-
 erts/emulator/beam/erl_bif_unique.h           | 342 ++++++++++++++++++++++++--
 erts/emulator/beam/erl_binary.h               | 159 +++++++++++-
 erts/emulator/beam/erl_db_util.c              |  25 +-
 erts/emulator/beam/erl_debug.c                |   2 +-
 erts/emulator/beam/erl_gc.c                   |  27 +-
 erts/emulator/beam/erl_hl_timer.c             |  12 +-
 erts/emulator/beam/erl_init.c                 |  12 +-
 erts/emulator/beam/erl_lock_check.c           |   1 +
 erts/emulator/beam/erl_message.c              |   5 +
 erts/emulator/beam/erl_monitors.c             |  25 +-
 erts/emulator/beam/erl_monitors.h             |   2 +-
 erts/emulator/beam/erl_msacc.c                |  10 +-
 erts/emulator/beam/erl_nif.c                  |   4 +-
 erts/emulator/beam/erl_node_container_utils.h |  37 +--
 erts/emulator/beam/erl_node_tables.c          |   5 +-
 erts/emulator/beam/erl_printf_term.c          |   5 +-
 erts/emulator/beam/erl_process.c              |  11 +-
 erts/emulator/beam/erl_term.c                 |  40 ++-
 erts/emulator/beam/erl_term.h                 | 248 +++++++++++++++----
 erts/emulator/beam/erl_time_sup.c             |  15 +-
 erts/emulator/beam/external.c                 |  56 +++--
 erts/emulator/beam/global.h                   | 133 +---------
 erts/emulator/beam/io.c                       |  72 +++---
 erts/emulator/beam/utils.c                    |  96 ++++----
 erts/emulator/test/node_container_SUITE.erl   |  41 ++-
 36 files changed, 1475 insertions(+), 445 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 1efe5fe300..df2866b40e 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -379,6 +379,7 @@ atom long_schedule
 atom low
 atom Lt='<'
 atom machine
+atom magic_ref
 atom major
 atom match
 atom match_limit
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index a2d2433ed8..bda7b9224c 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -199,7 +199,7 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
 		goto res_no_proc;
 	    case ERTS_PORT_OP_SCHEDULED:
 		if (refp) {
-		    ASSERT(is_internal_ref(ref));
+		    ASSERT(is_internal_ordinary_ref(ref));
 		    BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true);
 		}
 	    default:
@@ -782,7 +782,7 @@ local_name_monitor(Process *self, Eterm type, Eterm target_name)
         case ERTS_PORT_OP_DONE:
             return ret;
         case ERTS_PORT_OP_SCHEDULED: { /* Scheduled a signal */
-            ASSERT(is_internal_ref(ret));
+            ASSERT(is_internal_ordinary_ref(ret));
             BIF_TRAP3(await_port_send_result_trap, self,
                       ret, am_true, ret);
             /* bif_trap returns */
@@ -1180,7 +1180,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 		res = erts_port_unlink(BIF_P, prt, BIF_P->common.id, refp);
 
 		if (refp && res == ERTS_PORT_OP_SCHEDULED) {
-		    ASSERT(is_internal_ref(ref));
+		    ASSERT(is_internal_ordinary_ref(ref));
 		    BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true);
 		}
 	    }
@@ -1586,7 +1586,7 @@ BIF_RETTYPE exit_2(BIF_ALIST_2)
 	     ERTS_BIF_CHK_EXITED(BIF_P);
 
 	     if (refp && res == ERTS_PORT_OP_SCHEDULED) {
-		 ASSERT(is_internal_ref(ref));
+		 ASSERT(is_internal_ordinary_ref(ref));
 		 BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true);
 	     }
 
@@ -2221,7 +2221,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx)
 		/* Fall through */
 	    case ERTS_PORT_OP_SCHEDULED:
 		if (is_not_nil(*refp)) {
-		    ASSERT(is_internal_ref(*refp));
+		    ASSERT(is_internal_ordinary_ref(*refp));
 		    ret_val = SEND_AWAIT_RESULT;
 		}
 		break;
@@ -2409,7 +2409,7 @@ BIF_RETTYPE send_3(BIF_ALIST_3)
 	ERTS_BIF_PREP_YIELD_RETURN(retval, p, am_ok);
         break;
     case SEND_AWAIT_RESULT:
-	ASSERT(is_internal_ref(ref));
+	ASSERT(is_internal_ordinary_ref(ref));
 	ERTS_BIF_PREP_TRAP3(retval, await_port_send_result_trap, p, ref, am_nosuspend, am_ok);
 	break;
     case SEND_BADARG:
@@ -2526,7 +2526,7 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg)
 	ERTS_BIF_PREP_YIELD_RETURN(retval, p, msg);
         break;
     case SEND_AWAIT_RESULT:
-	ASSERT(is_internal_ref(ref));
+	ASSERT(is_internal_ordinary_ref(ref));
 	ERTS_BIF_PREP_TRAP3(retval,
 			    await_port_send_result_trap, p, ref, msg, msg);
 	break;
@@ -4109,6 +4109,7 @@ BIF_RETTYPE ref_to_list_1(BIF_ALIST_1)
 {
     if (is_not_ref(BIF_ARG_1))
 	BIF_ERROR(BIF_P, BADARG);
+    erts_magic_ref_save_bin(BIF_ARG_1);
     BIF_RET(term2list_dsprintf(BIF_P, BIF_ARG_1));
 }
 
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 071a356260..a09950ec40 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -47,13 +47,8 @@ void
 erts_init_binary(void)
 {
     /* Verify Binary alignment... */
-    if ((((UWord) &((Binary *) 0)->orig_bytes[0]) % ((UWord) 8)) != 0) {
-	/* I assume that any compiler should be able to optimize this
-	   away. If not, this test is not very expensive... */
-	erts_exit(ERTS_ABORT_EXIT,
-		 "Internal error: Address of orig_bytes[0] of a Binary"
-		 " is *not* 8-byte aligned\n");
-    }
+    ERTS_CT_ASSERT((offsetof(Binary,orig_bytes) % 8) == 0);
+    ERTS_CT_ASSERT((offsetof(ErtsMagicBinary,u.aligned.data) % 8) == 0);
 
     erts_init_trap_export(&binary_to_list_continue_export,
 			  am_erts_internal, am_binary_to_list_continue, 1,
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index d20102cb2c..85db23393c 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -858,20 +858,24 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 	    case EXTERNAL_PORT_SUBTAG:
 	    case EXTERNAL_REF_SUBTAG:
 		{
-		  ExternalThing *etp = (ExternalThing *) htop;
-
+		  ExternalThing *etp = (ExternalThing *) objp;
+		  erts_smp_refc_inc(&etp->node->refc, 2);
+		}
+	    L_off_heap_node_container_common:
+		{
+		  struct erl_off_heap_header *ohhp;
+		  ohhp = (struct erl_off_heap_header *) htop;
 		  i  = thing_arityval(hdr) + 1;
+		  *argp = make_boxed(htop);
 		  tp = htop;
 
 		  while (i--)  {
 		    *htop++ = *objp++;
 		  }
 
-		  etp->next = off_heap->first;
-		  off_heap->first = (struct erl_off_heap_header*)etp;
-		  erts_smp_refc_inc(&etp->node->refc, 2);
+		  ohhp->next = off_heap->first;
+		  off_heap->first = ohhp;
 
-		  *argp = make_external(tp);
 		}
 		break;
 	    case MAP_SUBTAG:
@@ -899,6 +903,13 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 	    case BIN_MATCHSTATE_SUBTAG:
 		erts_exit(ERTS_ABORT_EXIT,
 			 "copy_struct: matchstate term not allowed");
+	    case REF_SUBTAG:
+		if (is_magic_ref_thing(objp)) {
+		    ErtsMRefThing *mreft = (ErtsMRefThing *) objp;
+		    erts_refc_inc(&mreft->mb->refc, 2);
+		    goto L_off_heap_node_container_common;
+		}
+		/* Fall through... */
 	    default:
 		i = thing_arityval(hdr)+1;
 		hbot -= i;
@@ -1649,20 +1660,33 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 	    }
 	    case EXTERNAL_PID_SUBTAG:
 	    case EXTERNAL_PORT_SUBTAG:
-	    case EXTERNAL_REF_SUBTAG: {
-		ExternalThing *etp = (ExternalThing *) hp;
+	    case EXTERNAL_REF_SUBTAG:
+	    {
+		ExternalThing *etp = (ExternalThing *) ptr;
+		erts_smp_refc_inc(&etp->node->refc, 2);
+	    }
+	  off_heap_node_container_common:
+	    {
+		struct erl_off_heap_header *ohhp;
+		ohhp = (struct erl_off_heap_header *) hp;
 		sz = thing_arityval(hdr);
-		*resp = make_external(hp);
+		*resp = make_boxed(hp);
 		*hp++ = hdr;
 		ptr++;
 		while (sz-- > 0) {
 		    *hp++ = *ptr++;
 		}
-		etp->next = off_heap->first;
-		off_heap->first = (struct erl_off_heap_header*) etp;
-		erts_smp_refc_inc(&etp->node->refc, 2);
+		ohhp->next = off_heap->first;
+		off_heap->first = ohhp;
 		goto cleanup_next;
 	    }
+	    case REF_SUBTAG:
+		if (is_magic_ref_thing(ptr)) {
+		    ErtsMRefThing *mreft = (ErtsMRefThing *) ptr;
+		    erts_refc_inc(&mreft->mb->refc, 2);
+		    goto off_heap_node_container_common;
+		}
+		/* Fall through... */
 	    default:
 		sz = thing_arityval(hdr);
 		*resp = make_boxed(hp);
@@ -1859,6 +1883,15 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    off_heap->first = ohh;
 		}
 		break;
+	    case REF_SUBTAG: {
+		ErtsRefThing *rtp = (ErtsRefThing *) (tp - 1);
+		if (is_magic_ref_thing(rtp)) {
+		    ErtsMRefThing *mreft = (ErtsMRefThing *) rtp;
+		    erts_refc_inc(&mreft->mb->refc, 2);
+		    goto off_heap_common;
+		}
+		/* Fall through... */
+	    }
 	    default:
 		{
 		    int tari = header_arity(val);
@@ -1959,6 +1992,9 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite
 	    ASSERT(ptr + header_arity(val) < end);
 	    MOVE_BOXED(ptr, val, hp, &dummy_ref);
 	    switch (val & _HEADER_SUBTAG_MASK) {
+	    case REF_SUBTAG:
+		if (is_ordinary_ref_thing(hdr))
+		    break;
 	    case REFC_BINARY_SUBTAG:
 	    case FUN_SUBTAG:
 	    case EXTERNAL_PID_SUBTAG:
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 4d990a9c56..e433de0f35 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -48,6 +48,7 @@
 #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE)
 #include "erl_check_io.h"
 #endif
+#include "erl_bif_unique.h"
 
 #define GET_ERL_GF_ALLOC_IMPL
 #include "erl_goodfit_alloc.h"
@@ -152,8 +153,7 @@ typedef struct {
     int internal;
     Uint req_sched;
     Process *proc;
-    Eterm ref;
-    Eterm ref_heap[REF_THING_SIZE];
+    ErtsIRefStorage iref;
     int allocs[ERTS_ALC_INFO_A_END - ERTS_ALC_A_MIN + 1];
 } ErtsAllocInfoReq;
 
@@ -682,6 +682,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 #endif
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_EXP_TRACE)]
 	= sizeof(NifExportTrace);
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MREF_NSCHED_ENT)]
+	= sizeof(ErtsNSchedMagicRefTableEntry);
 
 #ifdef HARD_DEBUG
     hdbg_init();
@@ -3139,9 +3141,10 @@ reply_alloc_info(void *vair)
     while (1) {
 
 	if (hpp)
-	    ref_copy = STORE_NC(hpp, ohp, air->ref);
+	    ref_copy = erts_iref_storage_make_ref(&air->iref,
+                                                  hpp, ohp, 0);
 	else
-	    *szp += REF_THING_SIZE;
+	    *szp += erts_iref_storage_heap_size(&air->iref);
 
 	ai_list = NIL;
 	for (i = 0; air->allocs[i] != ERTS_ALC_A_INVALID; i++);
@@ -3370,8 +3373,10 @@ reply_alloc_info(void *vair)
     erts_smp_proc_unlock(rp, rp_locks);
     erts_proc_dec_refc(rp);
 
-    if (erts_smp_atomic32_dec_read_nob(&air->refc) == 0)
+    if (erts_smp_atomic32_dec_read_nob(&air->refc) == 0) {
+        erts_iref_storage_clean(&air->iref);
 	aireq_free(air);
+    }
 }
 
 int
@@ -3384,7 +3389,6 @@ erts_request_alloc_info(struct process *c_p,
     ErtsAllocInfoReq *air = aireq_alloc();
     Eterm req_ai[ERTS_ALC_INFO_A_END] = {0};
     Eterm alist;
-    Eterm *hp;
     int airix = 0, ai;
 
     air->req_sched = erts_get_scheduler_id();
@@ -3398,8 +3402,7 @@ erts_request_alloc_info(struct process *c_p,
     if (is_not_internal_ref(ref))
 	return 0;
 
-    hp = &air->ref_heap[0];
-    air->ref = STORE_NC(&hp, NULL, ref);
+    erts_iref_storage_save(&air->iref, ref);
 
     if (is_not_list(allocs))
 	return 0;
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 70eca5b49c..c253ea82c8 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -280,6 +280,10 @@ type    TRACE_MSG_QUEUE SHORT_LIVED     SYSTEM          trace_message_queue
 type    SCHED_ASYNC_JOB SHORT_LIVED     SYSTEM          async_calls
 type    DIRTY_START     STANDARD	PROCESSES       dirty_start
 type    DIRTY_SL	SHORT_LIVED	SYSTEM		dirty_short_lived
+type    MREF_NSCHED_ENT	FIXED_SIZE	SYSTEM		nsched_magic_ref_entry
+type    MREF_ENT	STANDARD	SYSTEM		magic_ref_entry
+type	MREF_TAB_BKTS	STANDARD	SYSTEM		magic_ref_table_buckets
+type	MREF_TAB	LONG_LIVED	SYSTEM		magic_ref_table
 
 +if threads_no_smp
 # Need thread safe allocs, but std_alloc and fix_alloc are not;
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index af90cfe40e..2ce872d2bc 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1476,8 +1476,10 @@ static void add_proc_loaded_deref(DE_Handle *dh, Process *proc)
 
 static Eterm copy_ref(Eterm ref, Eterm *hp)
 {
-    RefThing *ptr = ref_thing_ptr(ref);
-    memcpy(hp, ptr, sizeof(RefThing));
+    ErtsORefThing *ptr;
+    ASSERT(is_internal_ordinary_ref(ref));
+    ptr = ordinary_ref_thing_ptr(ref);
+    memcpy(hp, ptr, sizeof(ErtsORefThing));
     return (make_internal_ref(hp));
 }
 
@@ -1720,10 +1722,10 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type,
 	Eterm e;
 	mp = erts_alloc_message_heap(proc, &rp_locks,
 				     (6 /* tuple */ + 3 /* Error tuple */ + 
-				      REF_THING_SIZE + need),
+				      ERTS_REF_THING_SIZE + need),
 				     &hp, &ohp);
 	r = copy_ref(ref,hp);
-	hp += REF_THING_SIZE;
+	hp += ERTS_REF_THING_SIZE;
 	e = build_load_error_hp(hp, errcode);
 	hp += need;
 	mess = TUPLE2(hp,tag,e);
@@ -1731,10 +1733,10 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type,
 	mess = TUPLE5(hp,type,r,am_driver,driver_name,mess);
     } else {	
 	mp = erts_alloc_message_heap(proc, &rp_locks,
-				     6 /* tuple */ + REF_THING_SIZE,
+				     6 /* tuple */ + ERTS_REF_THING_SIZE,
 				     &hp, &ohp);
 	r = copy_ref(ref,hp);
-	hp += REF_THING_SIZE;
+	hp += ERTS_REF_THING_SIZE;
 	mess = TUPLE5(hp,type,r,am_driver,driver_name,tag);
     }
     erts_queue_message(proc, rp_locks, mp, mess, am_system);
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 37dcb5e7a6..be113b7c88 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -187,6 +187,32 @@ bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh)
     return res;
 }
 
+static Eterm
+bld_magic_ref_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh)
+{
+    struct erl_off_heap_header* ohh;
+    Eterm res = NIL;
+    Eterm tuple;
+
+    for (ohh = oh->first; ohh; ohh = ohh->next) {
+	if (is_ref_thing_header((*((Eterm *) ohh)))) {
+            ErtsMRefThing *mrtp = (ErtsMRefThing *) ohh;
+	    Eterm val = erts_bld_uword(hpp, szp, (UWord) mrtp->mb);
+	    Eterm orig_size = erts_bld_uint(hpp, szp, mrtp->mb->orig_size);
+    
+	    if (szp)
+		*szp += 4+2;
+	    if (hpp) {
+		Uint refc = (Uint) erts_refc_read(&mrtp->mb->refc, 1);
+		tuple = TUPLE3(*hpp, val, orig_size, make_small(refc));
+		res = CONS(*hpp + 4, tuple, res);
+		*hpp += 4+2;
+	    }
+	}
+    }
+    return res;
+}
+
 
 /*
   make_monitor_list:
@@ -613,7 +639,8 @@ static Eterm pi_args[] = {
     am_current_location,
     am_current_stacktrace,
     am_message_queue_data,
-    am_garbage_collection_info
+    am_garbage_collection_info,
+    am_magic_ref
 };
 
 #define ERTS_PI_ARGS ((int) (sizeof(pi_args)/sizeof(Eterm)))
@@ -664,6 +691,7 @@ pi_arg2ix(Eterm arg)
     case am_current_stacktrace:			return 31;
     case am_message_queue_data:			return 32;
     case am_garbage_collection_info:		return 33;
+    case am_magic_ref:                          return 34;
     default:					return -1;
     }
 }
@@ -1599,6 +1627,14 @@ process_info_aux(Process *BIF_P,
 	hp = HAlloc(BIF_P, 3);
 	break;
 
+    case am_magic_ref: {
+	Uint sz = 3;
+	(void) bld_magic_ref_bin_list(NULL, &sz, &MSO(rp));
+	hp = HAlloc(BIF_P, sz);
+	res = bld_magic_ref_bin_list(&hp, NULL, &MSO(rp));
+	break;
+    }
+
     default:
 	return THE_NON_VALUE; /* will produce badarg */
 
@@ -3531,6 +3567,11 @@ BIF_RETTYPE error_logger_warning_map_0(BIF_ALIST_0)
 
 static erts_smp_atomic_t available_internal_state;
 
+static void empty_magic_ref_destructor(Binary *bin)
+{
+
+}
+
 BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 {
     /*
@@ -3896,6 +3937,34 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 		}
 		return make_atom(ix);
 	    }
+	    else if (ERTS_IS_ATOM_STR("magic_ref", tp[1])) {
+                Binary *bin;
+                UWord bin_addr, refc;
+                Eterm bin_addr_term, refc_term, test_type;
+                Uint sz;
+                Eterm *hp;
+                if (!is_internal_magic_ref(tp[2])) {
+                    if (is_internal_ordinary_ref(tp[2])) {
+                        ErtsORefThing *rtp;
+                        rtp = (ErtsORefThing *) internal_ref_val(tp[2]);
+                        if (erts_is_ref_numbers_magic(rtp->num))
+                            BIF_RET(am_true);
+                    }
+                    BIF_RET(am_false);
+                }
+                bin = erts_magic_ref2bin(tp[2]);
+                refc = erts_refc_read(&bin->refc, 1);
+                bin_addr = (UWord) bin;
+                sz = 4;
+                erts_bld_uword(NULL, &sz, bin_addr);
+                erts_bld_uword(NULL, &sz, refc);
+                hp = HAlloc(BIF_P, sz);
+                bin_addr_term = erts_bld_uword(&hp, NULL, bin_addr);
+                refc_term = erts_bld_uword(&hp, NULL, refc);
+                test_type = (ERTS_MAGIC_BIN_DESTRUCTOR(bin) == empty_magic_ref_destructor
+                             ? am_true : am_false);
+                BIF_RET(TUPLE3(hp, bin_addr_term, refc_term, test_type));
+	    }
 
 	    break;
 	}
@@ -3984,7 +4053,6 @@ static void broken_halt_test(Eterm bif_arg_2)
     erts_exit(ERTS_DUMP_EXIT, "%T", bif_arg_2);
 }
 
-
 BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 {
     /*
@@ -4314,6 +4382,21 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
             /* Used by ets_SUITE (stdlib) */
             BIF_RET(erts_ets_restore_meta_state(BIF_P, BIF_ARG_2));
         }
+        else if (ERTS_IS_ATOM_STR("make", BIF_ARG_1)) {
+            if (ERTS_IS_ATOM_STR("magic_ref", BIF_ARG_2)) {
+                Binary *bin = erts_create_magic_binary(0, empty_magic_ref_destructor);
+                UWord bin_addr = (UWord) bin;
+                Eterm bin_addr_term, magic_ref, res;
+                Eterm *hp;
+                Uint sz = ERTS_MAGIC_REF_THING_SIZE + 3;
+                erts_bld_uword(NULL, &sz, bin_addr);
+                hp = HAlloc(BIF_P, sz);
+                bin_addr_term = erts_bld_uword(&hp, NULL, bin_addr);
+                magic_ref = erts_mk_magic_ref(&hp, &BIF_P->off_heap, bin);
+                res = TUPLE2(hp, magic_ref, bin_addr_term);
+                BIF_RET(res);
+            }
+        }
     }
 
     BIF_ERROR(BIF_P, BADARG);
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 90e78a9b0b..09d2c5376b 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -214,7 +214,7 @@ BIF_RETTYPE erts_internal_port_command_3(BIF_ALIST_3)
 	ASSERT(!(flags & ERTS_PORT_SIG_FLG_FORCE));
 	/* Fall through... */
     case ERTS_PORT_OP_SCHEDULED:
-	ASSERT(is_internal_ref(ref));
+	ASSERT(is_internal_ordinary_ref(ref));
 	ERTS_BIF_PREP_RET(res, ref);
 	break;
     case ERTS_PORT_OP_DONE:
@@ -260,7 +260,7 @@ BIF_RETTYPE erts_internal_port_call_3(BIF_ALIST_3)
 	retval = am_badarg;
 	break;
     case ERTS_PORT_OP_SCHEDULED:
-	ASSERT(is_internal_ref(retval));
+	ASSERT(is_internal_ordinary_ref(retval));
 	break;
     case ERTS_PORT_OP_DONE:
 	ASSERT(is_not_internal_ref(retval));
@@ -310,7 +310,7 @@ BIF_RETTYPE erts_internal_port_control_3(BIF_ALIST_3)
 	retval = am_badarg;
 	break;
     case ERTS_PORT_OP_SCHEDULED:
-	ASSERT(is_internal_ref(retval));
+	ASSERT(is_internal_ordinary_ref(retval));
 	break;
     case ERTS_PORT_OP_DONE:
 	ASSERT(is_not_internal_ref(retval));
@@ -356,7 +356,7 @@ BIF_RETTYPE erts_internal_port_close_1(BIF_ALIST_1)
     case ERTS_PORT_OP_DROPPED:
 	BIF_RET(am_badarg);
     case ERTS_PORT_OP_SCHEDULED:
-	ASSERT(is_internal_ref(ref));
+	ASSERT(is_internal_ordinary_ref(ref));
 	BIF_RET(ref);
     case ERTS_PORT_OP_DONE:
 	BIF_RET(am_true);
@@ -389,7 +389,7 @@ BIF_RETTYPE erts_internal_port_connect_2(BIF_ALIST_2)
     case ERTS_PORT_OP_DROPPED:
 	BIF_RET(am_badarg);
     case ERTS_PORT_OP_SCHEDULED:
-	ASSERT(is_internal_ref(ref));
+	ASSERT(is_internal_ordinary_ref(ref));
 	BIF_RET(ref);
 	break;
     case ERTS_PORT_OP_DONE:
@@ -428,7 +428,7 @@ BIF_RETTYPE erts_internal_port_info_1(BIF_ALIST_1)
     case ERTS_PORT_OP_DROPPED:
 	BIF_RET(am_undefined);
     case ERTS_PORT_OP_SCHEDULED:
-	ASSERT(is_internal_ref(retval));
+	ASSERT(is_internal_ordinary_ref(retval));
 	BIF_RET(retval);
     case ERTS_PORT_OP_DONE:
 	ASSERT(is_not_internal_ref(retval));
@@ -467,7 +467,7 @@ BIF_RETTYPE erts_internal_port_info_2(BIF_ALIST_2)
     case ERTS_PORT_OP_DROPPED:
 	BIF_RET(am_undefined);
     case ERTS_PORT_OP_SCHEDULED:
-	ASSERT(is_internal_ref(retval));
+	ASSERT(is_internal_ordinary_ref(retval));
 	BIF_RET(retval);
     case ERTS_PORT_OP_DONE:
 	ASSERT(is_not_internal_ref(retval));
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index a480754bdb..f471390501 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -2363,7 +2363,7 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2)
 typedef struct {
     Process *proc;
     Eterm ref;
-    Eterm ref_heap[REF_THING_SIZE];
+    Eterm ref_heap[ERTS_REF_THING_SIZE];
     Eterm target;
     erts_smp_atomic32_t refc;
 } ErtsTraceDeliveredAll;
diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c
index 7c70217d8d..3fd4c87094 100644
--- a/erts/emulator/beam/erl_bif_unique.c
+++ b/erts/emulator/beam/erl_bif_unique.c
@@ -22,12 +22,15 @@
 #  include "config.h"
 #endif
 
+#define ERL_BIF_UNIQUE_C__
 #include "sys.h"
 #include "erl_vm.h"
 #include "erl_alloc.h"
 #include "export.h"
 #include "bif.h"
 #include "erl_bif_unique.h"
+#include "hash.h"
+#include "erl_binary.h"
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
  * Reference                                                         *
@@ -58,9 +61,20 @@ static union {
 static Uint32 max_thr_id;
 #endif
 
+static void init_magic_ref_tables(void);
+
+static Uint64 ref_init_value;
+
 static void
 init_reference(void)
 {
+    SysTimeval tv;
+    sys_gettimeofday(&tv);
+    ref_init_value = 0;
+    ref_init_value |= (Uint64) tv.tv_sec;
+    ref_init_value |= ((Uint64) tv.tv_usec) << 32;
+    ref_init_value *= (Uint64) 268438039;
+    ref_init_value += (Uint64) tv.tv_usec;
 #ifdef DEBUG
     max_thr_id = (Uint32) erts_no_schedulers;
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -68,11 +82,13 @@ init_reference(void)
     max_thr_id += (Uint32) erts_no_dirty_io_schedulers;
 #endif
 #endif
-    erts_atomic64_init_nob(&global_reference.count, 0);
+    erts_atomic64_init_nob(&global_reference.count,
+			   (erts_aint64_t) ref_init_value);
+    init_magic_ref_tables();
 }
 
 static ERTS_INLINE void
-global_make_ref_in_array(Uint32 thr_id, Uint32 ref[ERTS_MAX_REF_NUMBERS])
+global_make_ref_in_array(Uint32 thr_id, Uint32 ref[ERTS_REF_NUMBERS])
 {
     Uint64 value;
 
@@ -82,7 +98,7 @@ global_make_ref_in_array(Uint32 thr_id, Uint32 ref[ERTS_MAX_REF_NUMBERS])
 }
 
 static ERTS_INLINE void
-make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS])
+make_ref_in_array(Uint32 ref[ERTS_REF_NUMBERS])
 {
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
     if (esdp)
@@ -92,15 +108,23 @@ make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS])
 }
 
 void
-erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS])
+erts_make_ref_in_array(Uint32 ref[ERTS_REF_NUMBERS])
+{
+    make_ref_in_array(ref);
+}
+
+void
+erts_make_magic_ref_in_array(Uint32 ref[ERTS_REF_NUMBERS])
 {
     make_ref_in_array(ref);
+    ASSERT(!(ref[1] & ERTS_REF1_MAGIC_MARKER_BIT__));
+    ref[1] |= ERTS_REF1_MAGIC_MARKER_BIT__;
 }
 
-Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE])
+Eterm erts_make_ref_in_buffer(Eterm buffer[ERTS_REF_THING_SIZE])
 {
     Eterm* hp = buffer;
-    Uint32 ref[ERTS_MAX_REF_NUMBERS];
+    Uint32 ref[ERTS_REF_NUMBERS];
 
     make_ref_in_array(ref);
     write_ref_thing(hp, ref[0], ref[1], ref[2]);
@@ -110,11 +134,11 @@ Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE])
 Eterm erts_make_ref(Process *c_p)
 {
     Eterm* hp;
-    Uint32 ref[ERTS_MAX_REF_NUMBERS];
+    Uint32 ref[ERTS_REF_NUMBERS];
 
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
 
-    hp = HAlloc(c_p, REF_THING_SIZE);
+    hp = HAlloc(c_p, ERTS_REF_THING_SIZE);
 
     make_ref_in_array(ref);
     write_ref_thing(hp, ref[0], ref[1], ref[2]);
@@ -122,6 +146,272 @@ Eterm erts_make_ref(Process *c_p)
     return make_internal_ref(hp);
 }
 
+/*
+ * Magic reference tables
+ */
+
+typedef struct {
+    HashBucket hash;
+    ErtsMagicBinary *mb;
+    Uint64 value;
+    Uint32 thr_id;
+} ErtsMagicRefTableEntry;
+
+typedef struct {
+    erts_rwmtx_t rwmtx;
+    Hash hash;
+    char name[32];
+} ErtsMagicRefTable;
+
+typedef struct {
+    union {
+	ErtsMagicRefTable table;
+	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsMagicRefTable))];
+    } u;
+} ErtsAlignedMagicRefTable;
+
+ErtsAlignedMagicRefTable *magic_ref_table;
+
+ErtsMagicBinary *
+erts_magic_ref_lookup_bin__(Uint32 refn[ERTS_REF_NUMBERS])
+{
+    ErtsMagicRefTableEntry tmpl;
+    ErtsMagicRefTableEntry *tep;
+    ErtsMagicBinary *mb;
+    ErtsMagicRefTable *tblp;
+
+    ASSERT(erts_is_ref_numbers_magic(refn));
+
+    tmpl.value = erts_get_ref_numbers_value(refn);
+    tmpl.thr_id = erts_get_ref_numbers_thr_id(refn);
+    if (tmpl.thr_id > erts_no_schedulers)
+	tblp = &magic_ref_table[0].u.table;
+    else
+	tblp = &magic_ref_table[tmpl.thr_id].u.table;
+
+    erts_rwmtx_rlock(&tblp->rwmtx);
+
+    tep = (ErtsMagicRefTableEntry *) hash_get(&tblp->hash, &tmpl);
+    if (!tep)
+	mb = NULL;
+    else {
+	erts_aint_t refc;
+	mb = tep->mb;
+        refc = erts_refc_inc_unless(&mb->refc, 0, 0);
+        if (refc == 0)
+            mb = NULL;
+    }
+
+    erts_rwmtx_runlock(&tblp->rwmtx);
+
+    return mb;
+}
+
+void
+erts_magic_ref_save_bin__(Eterm ref)
+{
+    ErtsMagicRefTableEntry tmpl;
+    ErtsMagicRefTableEntry *tep;
+    ErtsMRefThing *mrtp;
+    ErtsMagicRefTable *tblp;
+    Uint32 *refn;
+
+    ASSERT(is_internal_magic_ref(ref));
+
+    mrtp = (ErtsMRefThing *) internal_ref_val(ref);
+    refn = mrtp->mb->refn;
+
+    tmpl.value = erts_get_ref_numbers_value(refn);
+    tmpl.thr_id = erts_get_ref_numbers_thr_id(refn);
+
+    if (tmpl.thr_id > erts_no_schedulers)
+	tblp = &magic_ref_table[0].u.table;
+    else
+	tblp = &magic_ref_table[tmpl.thr_id].u.table;
+
+    erts_rwmtx_rlock(&tblp->rwmtx);
+
+    tep = (ErtsMagicRefTableEntry *) hash_get(&tblp->hash, &tmpl);
+
+    erts_rwmtx_runlock(&tblp->rwmtx);
+
+    if (!tep) {
+	ErtsMagicRefTableEntry *used_tep;
+	
+	ASSERT(tmpl.value == erts_get_ref_numbers_value(refn));
+	ASSERT(tmpl.thr_id == erts_get_ref_numbers_thr_id(refn));
+
+	if (tblp != &magic_ref_table[0].u.table) {
+	    tep = erts_alloc(ERTS_ALC_T_MREF_NSCHED_ENT,
+			     sizeof(ErtsNSchedMagicRefTableEntry));
+	}
+	else {
+	    tep = erts_alloc(ERTS_ALC_T_MREF_ENT,
+			     sizeof(ErtsMagicRefTableEntry));
+	    tep->thr_id = tmpl.thr_id;
+	}
+
+	tep->value = tmpl.value;
+	tep->mb = mrtp->mb;
+
+	erts_rwmtx_rwlock(&tblp->rwmtx);
+
+	used_tep = hash_put(&tblp->hash, tep);
+
+	erts_rwmtx_rwunlock(&tblp->rwmtx);
+
+	if (used_tep != tep) {
+	    if (tblp != &magic_ref_table[0].u.table)
+		erts_free(ERTS_ALC_T_MREF_NSCHED_ENT, (void *) tep);
+	    else
+		erts_free(ERTS_ALC_T_MREF_ENT, (void *) tep);
+	}
+    }
+}
+
+void
+erts_magic_ref_remove_bin(Uint32 refn[ERTS_REF_NUMBERS])
+{
+    ErtsMagicRefTableEntry tmpl;
+    ErtsMagicRefTableEntry *tep;
+    ErtsMagicRefTable *tblp;
+
+    tmpl.value = erts_get_ref_numbers_value(refn);
+    tmpl.thr_id = erts_get_ref_numbers_thr_id(refn);
+
+    if (tmpl.thr_id > erts_no_schedulers)
+	tblp = &magic_ref_table[0].u.table;
+    else
+	tblp = &magic_ref_table[tmpl.thr_id].u.table;
+
+    erts_rwmtx_rlock(&tblp->rwmtx);
+
+    tep = (ErtsMagicRefTableEntry *) hash_get(&tblp->hash, &tmpl);
+
+    erts_rwmtx_runlock(&tblp->rwmtx);
+
+    if (tep) {
+
+	ASSERT(tmpl.value == erts_get_ref_numbers_value(refn));
+	ASSERT(tmpl.thr_id == erts_get_ref_numbers_thr_id(refn));
+
+	erts_rwmtx_rwlock(&tblp->rwmtx);
+
+	tep = hash_remove(&tblp->hash, &tmpl);
+	ASSERT(tep);
+
+	erts_rwmtx_rwunlock(&tblp->rwmtx);
+
+	if (tblp != &magic_ref_table[0].u.table)
+	    erts_free(ERTS_ALC_T_MREF_NSCHED_ENT, (void *) tep);
+	else
+	    erts_free(ERTS_ALC_T_MREF_ENT, (void *) tep);
+    }
+}
+
+static int nsched_mreft_cmp(void *ve1, void *ve2)
+{
+    ErtsNSchedMagicRefTableEntry *e1 = ve1;
+    ErtsNSchedMagicRefTableEntry *e2 = ve2;
+    return e1->value != e2->value;
+}
+
+static int non_nsched_mreft_cmp(void *ve1, void *ve2)
+{
+    ErtsMagicRefTableEntry *e1 = ve1;
+    ErtsMagicRefTableEntry *e2 = ve2;
+    return e1->value != e2->value || e1->thr_id != e2->thr_id;
+}
+
+static HashValue nsched_mreft_hash(void *ve)
+{
+    ErtsNSchedMagicRefTableEntry *e = ve;
+    return (HashValue) e->value;
+}
+
+static HashValue non_nsched_mreft_hash(void *ve)
+{
+    ErtsMagicRefTableEntry *e = ve;
+    HashValue h;
+    h = (HashValue) e->thr_id;
+    h *= 268440163;
+    h += (HashValue) e->value;
+    return h;
+}
+
+static void *mreft_alloc(void *ve)
+{
+    /*
+     * We allocate the element before
+     * hash_put() and pass it as
+     * template which we get as
+     * input...
+     */
+    return ve;
+}
+
+static void mreft_free(void *ve)
+{
+    /*
+     * We free the element ourselves
+     * after hash_remove()...
+     */
+}
+
+static void *mreft_meta_alloc(int i, size_t size)
+{
+    return erts_alloc(ERTS_ALC_T_MREF_TAB_BKTS, size);
+}
+
+static void mreft_meta_free(int i, void *ptr)
+{
+    erts_free(ERTS_ALC_T_MREF_TAB_BKTS, ptr);
+}
+
+static void
+init_magic_ref_tables(void)
+{
+    HashFunctions hash_funcs;
+    int i;
+    ErtsMagicRefTable *tblp;
+
+    magic_ref_table = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_MREF_TAB,
+							 (sizeof(ErtsAlignedMagicRefTable)
+							  * (erts_no_schedulers + 1)));
+
+    hash_funcs.hash = non_nsched_mreft_hash;
+    hash_funcs.cmp = non_nsched_mreft_cmp;
+
+    hash_funcs.alloc = mreft_alloc;
+    hash_funcs.free = mreft_free;
+    hash_funcs.meta_alloc = mreft_meta_alloc;
+    hash_funcs.meta_free = mreft_meta_free;
+    hash_funcs.meta_print = erts_print;
+
+    tblp = &magic_ref_table[0].u.table;
+    erts_snprintf(&tblp->name[0], sizeof(tblp->name),
+		  "magic_ref_table_0");
+    hash_init(0, &tblp->hash, &tblp->name[0], 1, hash_funcs);
+    erts_rwmtx_init(&tblp->rwmtx, "magic_ref_table");
+
+    hash_funcs.hash = nsched_mreft_hash;
+    hash_funcs.cmp = nsched_mreft_cmp;
+
+    for (i = 1; i <= erts_no_schedulers; i++) {
+	ErtsMagicRefTable *tblp = &magic_ref_table[i].u.table;
+	erts_snprintf(&tblp->name[0], sizeof(tblp->name),
+		      "magic_ref_table_%d", i);
+	hash_init(0, &tblp->hash, &tblp->name[0], 1, hash_funcs);
+	erts_rwmtx_init(&tblp->rwmtx, "magic_ref_table");
+    }
+}
+
+void erts_ref_bin_free(ErtsMagicBinary *mb)
+{
+    erts_bin_free((Binary *) mb);
+}
+
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
  * Unique Integer                                                    *
 \*                                                                   */
@@ -498,7 +788,7 @@ void
 erts_sched_bif_unique_init(ErtsSchedulerData *esdp)
 {
     esdp->unique = (Uint64) 0;
-    esdp->ref = (Uint64) 0;
+    esdp->ref = (Uint64) ref_init_value;
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
@@ -513,7 +803,7 @@ BIF_RETTYPE make_ref_0(BIF_ALIST_0)
 
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
 
-    hp = HAlloc(BIF_P, REF_THING_SIZE);
+    hp = HAlloc(BIF_P, ERTS_REF_THING_SIZE);
 
     res = erts_sched_make_ref_in_buffer(erts_proc_sched_data(BIF_P), hp);
 
diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h
index c6481864d0..6cd6cc3e09 100644
--- a/erts/emulator/beam/erl_bif_unique.h
+++ b/erts/emulator/beam/erl_bif_unique.h
@@ -21,16 +21,25 @@
 #ifndef ERTS_BIF_UNIQUE_H__
 #define ERTS_BIF_UNIQUE_H__
 
+#include "erl_term.h"
 #include "erl_process.h"
 #include "big.h"
+#define ERTS_BINARY_TYPES_ONLY__
+#include "erl_binary.h"
+#undef ERTS_BINARY_TYPES_ONLY__
 
 void erts_bif_unique_init(void);
 void erts_sched_bif_unique_init(ErtsSchedulerData *esdp);
 
 /* reference */
 Eterm erts_make_ref(Process *);
-Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]);
-void erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]);
+Eterm erts_make_ref_in_buffer(Eterm buffer[ERTS_REF_THING_SIZE]);
+void erts_make_ref_in_array(Uint32 ref[ERTS_REF_NUMBERS]);
+void erts_make_magic_ref_in_array(Uint32 ref[ERTS_REF_NUMBERS]);
+void erts_magic_ref_remove_bin(Uint32 refn[ERTS_REF_NUMBERS]);
+void erts_magic_ref_save_bin__(Eterm ref);
+ErtsMagicBinary *erts_magic_ref_lookup_bin__(Uint32 refn[ERTS_REF_NUMBERS]);
+
 
 /* strict monotonic counter */
 
@@ -67,19 +76,35 @@ Eterm erts_debug_make_unique_integer(Process *c_p,
 				     Eterm etval1);
 
 
-ERTS_GLB_INLINE void erts_set_ref_numbers(Uint32 ref[ERTS_MAX_REF_NUMBERS],
+ERTS_GLB_INLINE void erts_set_ref_numbers(Uint32 ref[ERTS_REF_NUMBERS],
 					  Uint32 thr_id, Uint64 value);
-ERTS_GLB_INLINE Uint32 erts_get_ref_numbers_thr_id(Uint32 ref[ERTS_MAX_REF_NUMBERS]);
-ERTS_GLB_INLINE Uint64 erts_get_ref_numbers_value(Uint32 ref[ERTS_MAX_REF_NUMBERS]);
+ERTS_GLB_INLINE Uint32 erts_get_ref_numbers_thr_id(Uint32 ref[ERTS_REF_NUMBERS]);
+ERTS_GLB_INLINE int erts_is_ref_numbers_magic(Uint32 ref[ERTS_REF_NUMBERS]);
+ERTS_GLB_INLINE Uint64 erts_get_ref_numbers_value(Uint32 ref[ERTS_REF_NUMBERS]);
 ERTS_GLB_INLINE void erts_sched_make_ref_in_array(ErtsSchedulerData *esdp,
-						  Uint32 ref[ERTS_MAX_REF_NUMBERS]);
+						  Uint32 ref[ERTS_REF_NUMBERS]);
+ERTS_GLB_INLINE void erts_sched_make_magic_ref_in_array(ErtsSchedulerData *esdp,
+							Uint32 ref[ERTS_REF_NUMBERS]);
 ERTS_GLB_INLINE Eterm erts_sched_make_ref_in_buffer(ErtsSchedulerData *esdp,
-						    Eterm buffer[REF_THING_SIZE]);
+						    Eterm buffer[ERTS_REF_THING_SIZE]);
+ERTS_GLB_INLINE Eterm erts_mk_magic_ref(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp);
+ERTS_GLB_INLINE Binary *erts_magic_ref2bin(Eterm mref);
+ERTS_GLB_INLINE void erts_magic_ref_save_bin(Eterm ref);
+ERTS_GLB_INLINE ErtsMagicBinary *erts_magic_ref_lookup_bin(Uint32 ref[ERTS_REF_NUMBERS]);
+
+#define ERTS_REF1_MAGIC_MARKER_BIT_NO__ \
+    (_REF_NUM_SIZE-1)
+#define ERTS_REF1_MAGIC_MARKER_BIT__ \
+    (((Uint32) 1) << ERTS_REF1_MAGIC_MARKER_BIT_NO__)
+#define ERTS_REF1_THR_ID_MASK__ \
+    (ERTS_REF1_MAGIC_MARKER_BIT__-1)
+#define ERTS_REF1_NUM_MASK__ \
+    (~(ERTS_REF1_THR_ID_MASK__|ERTS_REF1_MAGIC_MARKER_BIT__))
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_INLINE void
-erts_set_ref_numbers(Uint32 ref[ERTS_MAX_REF_NUMBERS], Uint32 thr_id, Uint64 value)
+erts_set_ref_numbers(Uint32 ref[ERTS_REF_NUMBERS], Uint32 thr_id, Uint64 value)
 {
     /*
      * We cannot use thread id in the first 18-bit word since
@@ -87,30 +112,39 @@ erts_set_ref_numbers(Uint32 ref[ERTS_MAX_REF_NUMBERS], Uint32 thr_id, Uint64 val
      * we did, we would get really poor hash values. Instead
      * we have to shuffle the bits a bit.
      */
-    ASSERT(thr_id == (thr_id & ((Uint32) 0x3ffff)));
-    ref[0] = (Uint32) (value & ((Uint64) 0x3ffff));
-    ref[1] = (((Uint32) (value & ((Uint64) 0xfffc0000)))
-	      | (thr_id & ((Uint32) 0x3ffff)));
+    ASSERT(thr_id == (thr_id & ((Uint32) ERTS_REF1_THR_ID_MASK__)));
+    ref[0] = (Uint32) (value & ((Uint64) REF_MASK));
+    ref[1] = (((Uint32) (value & ((Uint64) ERTS_REF1_NUM_MASK__)))
+	      | (thr_id & ((Uint32) ERTS_REF1_THR_ID_MASK__)));
     ref[2] = (Uint32) ((value >> 32) & ((Uint64) 0xffffffff));
 }
 
 ERTS_GLB_INLINE Uint32
-erts_get_ref_numbers_thr_id(Uint32 ref[ERTS_MAX_REF_NUMBERS])
+erts_get_ref_numbers_thr_id(Uint32 ref[ERTS_REF_NUMBERS])
 {
-    return ref[1] & ((Uint32) 0x3ffff);
+    return ref[1] & ((Uint32) ERTS_REF1_THR_ID_MASK__);
+}
+
+ERTS_GLB_INLINE int
+erts_is_ref_numbers_magic(Uint32 ref[ERTS_REF_NUMBERS])
+{
+    return !!(ref[1] & ERTS_REF1_MAGIC_MARKER_BIT__);
 }
 
 ERTS_GLB_INLINE Uint64
-erts_get_ref_numbers_value(Uint32 ref[ERTS_MAX_REF_NUMBERS])
+erts_get_ref_numbers_value(Uint32 ref[ERTS_REF_NUMBERS])
 {
+    ERTS_CT_ASSERT((ERTS_REF1_NUM_MASK__ | REF_MASK) == 0xffffffff);
+    ERTS_CT_ASSERT((ERTS_REF1_NUM_MASK__ & REF_MASK) == 0);
+
     return (((((Uint64) ref[2]) & ((Uint64) 0xffffffff)) << 32)
-	    | (((Uint64) ref[1]) & ((Uint64) 0xfffc0000))
-	    | (((Uint64) ref[0]) & ((Uint64) 0x3ffff)));
+	    | (((Uint64) ref[1]) & ((Uint64) ERTS_REF1_NUM_MASK__))
+	    | (((Uint64) ref[0]) & ((Uint64) REF_MASK)));
 }
 
 ERTS_GLB_INLINE void
 erts_sched_make_ref_in_array(ErtsSchedulerData *esdp,
-			     Uint32 ref[ERTS_MAX_REF_NUMBERS])
+			     Uint32 ref[ERTS_REF_NUMBERS])
 {
     Uint64 value;
 
@@ -119,18 +153,286 @@ erts_sched_make_ref_in_array(ErtsSchedulerData *esdp,
     erts_set_ref_numbers(ref, (Uint32) esdp->thr_id, value);
 }
 
+ERTS_GLB_INLINE void
+erts_sched_make_magic_ref_in_array(ErtsSchedulerData *esdp,
+				   Uint32 ref[ERTS_REF_NUMBERS])
+{
+    erts_sched_make_ref_in_array(esdp, ref);
+    ASSERT(!(ref[1] & ERTS_REF1_MAGIC_MARKER_BIT__));
+    ref[1] |= ERTS_REF1_MAGIC_MARKER_BIT__;
+}
+
 ERTS_GLB_INLINE Eterm
 erts_sched_make_ref_in_buffer(ErtsSchedulerData *esdp,
-			      Eterm buffer[REF_THING_SIZE])
+			      Eterm buffer[ERTS_REF_THING_SIZE])
 {
     Eterm* hp = buffer;
-    Uint32 ref[ERTS_MAX_REF_NUMBERS];
+    Uint32 ref[ERTS_REF_NUMBERS];
 
     erts_sched_make_ref_in_array(esdp, ref);
     write_ref_thing(hp, ref[0], ref[1], ref[2]);
     return make_internal_ref(hp);
 }
 
+ERTS_GLB_INLINE Eterm
+erts_mk_magic_ref(Eterm **hpp, ErlOffHeap *ohp, Binary *bp)
+{
+    Eterm *hp = *hpp;
+    ASSERT(bp->flags & BIN_FLAG_MAGIC);
+    write_magic_ref_thing(hp, ohp, (ErtsMagicBinary *) bp);
+    *hpp += ERTS_MAGIC_REF_THING_SIZE;
+    erts_refc_inc(&bp->refc, 1);
+    return make_internal_ref(hp);
+}
+
+ERTS_GLB_INLINE Binary *
+erts_magic_ref2bin(Eterm mref)
+{
+    ErtsMRefThing *mrtp;
+    ASSERT(is_internal_magic_ref(mref));
+    mrtp = (ErtsMRefThing *) internal_ref_val(mref);
+    return (Binary *) mrtp->mb;
+}
+
+/*
+ * Save the magic binary of a ref when the
+ * ref is exposed to the outside world...
+ */
+ERTS_GLB_INLINE void
+erts_magic_ref_save_bin(Eterm ref)
+{
+    if (is_internal_magic_ref(ref))
+	erts_magic_ref_save_bin__(ref);
+}
+
+/*
+ * Look up the magic binary of a magic ref
+ * when the ref comes from the outside world...
+ */
+ERTS_GLB_INLINE ErtsMagicBinary *
+erts_magic_ref_lookup_bin(Uint32 ref[ERTS_REF_NUMBERS])
+{
+    if (!erts_is_ref_numbers_magic(ref))
+	return NULL;
+    return erts_magic_ref_lookup_bin__(ref);
+}
+
+
 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
+
+/*
+ * Storage of internal refs in misc structures...
+ */
+
+#include "erl_message.h"
+
+#if ERTS_REF_NUMBERS != 3
+#  error fix this...
+#endif
+
+ERTS_GLB_INLINE int erts_internal_ref_number_cmp(Uint32 num1[ERTS_REF_NUMBERS],
+						 Uint32 num2[ERTS_REF_NUMBERS]);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE int
+erts_internal_ref_number_cmp(Uint32 num1[ERTS_REF_NUMBERS],
+			     Uint32 num2[ERTS_REF_NUMBERS])
+{
+    if (num1[2] != num2[2])
+	return (int) ((Sint64) num1[2] - (Sint64) num2[2]);
+    if (num1[1] != num2[1])
+	return (int) ((Sint64) num1[1] - (Sint64) num2[1]);
+    if (num1[0] != num2[0])
+	return (int) ((Sint64) num1[0] - (Sint64) num2[0]);
+    return 0;
+}
+
+#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+/* Iref storage for all internal references... */ 
+typedef struct {
+    Uint32 is_magic;
+    union {
+	ErtsMagicBinary *mb;
+	Uint32 num[ERTS_REF_NUMBERS];
+    } u;
+} ErtsIRefStorage;
+
+void erts_ref_bin_free(ErtsMagicBinary *mb);
+
+ERTS_GLB_INLINE void erts_iref_storage_save(ErtsIRefStorage *iref, Eterm ref);
+ERTS_GLB_INLINE void erts_iref_storage_clean(ErtsIRefStorage *iref);
+ERTS_GLB_INLINE Uint erts_iref_storage_heap_size(ErtsIRefStorage *iref);
+ERTS_GLB_INLINE Eterm erts_iref_storage_make_ref(ErtsIRefStorage *iref,
+						 Eterm **hpp, ErlOffHeap *ohp,
+						 int clean_storage);
+ERTS_GLB_INLINE int erts_iref_storage_cmp(ErtsIRefStorage *iref1,
+					  ErtsIRefStorage *iref2);
+
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE void
+erts_iref_storage_save(ErtsIRefStorage *iref, Eterm ref)
+{
+    Eterm *hp;
+
+    ERTS_CT_ASSERT(ERTS_REF_NUMBERS == 3);
+    ASSERT(is_internal_ref(ref));
+
+    hp = boxed_val(ref);
+
+    if (is_ordinary_ref_thing(hp)) {
+	ErtsORefThing *rtp = (ErtsORefThing *) hp;
+	iref->is_magic = 0;
+	iref->u.num[0] = rtp->num[0];
+	iref->u.num[1] = rtp->num[1];
+	iref->u.num[2] = rtp->num[2];
+    }
+    else {
+	ErtsMRefThing *mrtp = (ErtsMRefThing *) hp;
+	ASSERT(is_magic_ref_thing(hp));
+	iref->is_magic = 1;
+	iref->u.mb = mrtp->mb;
+	erts_refc_inc(&mrtp->mb->refc, 1);
+    }
+}
+
+ERTS_GLB_INLINE void
+erts_iref_storage_clean(ErtsIRefStorage *iref)
+{
+    if (iref->is_magic && erts_refc_dectest(&iref->u.mb->refc, 0) == 0)
+	erts_ref_bin_free(iref->u.mb);
+#ifdef DEBUG
+    memset((void *) iref, 0xf, sizeof(ErtsIRefStorage));
+#endif
+}
+
+ERTS_GLB_INLINE Uint
+erts_iref_storage_heap_size(ErtsIRefStorage *iref)
+{
+    return iref->is_magic ? ERTS_MAGIC_REF_THING_SIZE : ERTS_REF_THING_SIZE;
+}
+
+ERTS_GLB_INLINE Eterm
+erts_iref_storage_make_ref(ErtsIRefStorage *iref,
+			   Eterm **hpp, ErlOffHeap *ohp,
+			   int clean_storage)
+{
+    Eterm *hp = *hpp;
+    if (!iref->is_magic) {
+	write_ref_thing(hp, iref->u.num[0], iref->u.num[1],
+			iref->u.num[2]);
+	*hpp += ERTS_REF_THING_SIZE;
+    }
+    else {
+	write_magic_ref_thing(hp, ohp, iref->u.mb);
+	*hpp += ERTS_MAGIC_REF_THING_SIZE;
+	/*
+	 * If we clean storage, the term inherits the
+	 * refc increment of the cleaned storage...
+	 */
+	if (!clean_storage)
+	    erts_refc_inc(&iref->u.mb->refc, 1);
+    }
+
+#ifdef DEBUG
+    if (clean_storage)
+	memset((void *) iref, 0xf, sizeof(ErtsIRefStorage));
+#endif
+
+    return make_internal_ref(hp);
+}
+
+ERTS_GLB_INLINE int
+erts_iref_storage_cmp(ErtsIRefStorage *iref1,
+		      ErtsIRefStorage *iref2)
+{
+    Uint32 *num1 = iref1->is_magic ? iref1->u.mb->refn : iref1->u.num;
+    Uint32 *num2 = iref2->is_magic ? iref2->u.mb->refn : iref2->u.num;
+    return erts_internal_ref_number_cmp(num1, num2);
+}
+
+#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+
+/* OIref storage for ordinary internal references only... */ 
+typedef struct {
+    Uint32 num[ERTS_REF_NUMBERS];
+} ErtsOIRefStorage;
+
+ERTS_GLB_INLINE void erts_oiref_storage_save(ErtsOIRefStorage *oiref,
+					     Eterm ref);
+ERTS_GLB_INLINE Eterm erts_oiref_storage_make_ref(ErtsOIRefStorage *oiref,
+						  Eterm **hpp);
+ERTS_GLB_INLINE int erts_oiref_storage_cmp(ErtsOIRefStorage *oiref1,
+					   ErtsOIRefStorage *oiref2);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE void
+erts_oiref_storage_save(ErtsOIRefStorage *oiref, Eterm ref)
+{
+    ErtsORefThing *rtp;
+    ERTS_CT_ASSERT(ERTS_REF_NUMBERS == 3);
+    ASSERT(is_internal_ordinary_ref(ref));
+
+    rtp = (ErtsORefThing *) internal_ref_val(ref);
+
+    oiref->num[0] = rtp->num[0];
+    oiref->num[1] = rtp->num[1];
+    oiref->num[2] = rtp->num[2];
+}
+
+ERTS_GLB_INLINE Eterm
+erts_oiref_storage_make_ref(ErtsOIRefStorage *oiref, Eterm **hpp)
+{
+    Eterm *hp = *hpp;
+    ERTS_CT_ASSERT(ERTS_REF_NUMBERS == 3);
+    write_ref_thing(hp, oiref->num[0], oiref->num[1], oiref->num[2]);
+    *hpp += ERTS_REF_THING_SIZE;
+    return make_internal_ref(hp);
+}
+
+ERTS_GLB_INLINE int
+erts_oiref_storage_cmp(ErtsOIRefStorage *oiref1,
+		       ErtsOIRefStorage *oiref2)
+{
+    return erts_internal_ref_number_cmp(oiref1->num, oiref2->num);
+}
+
+#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+
+ERTS_GLB_INLINE Eterm
+erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS]);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE Eterm
+erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS])
+{
+    Eterm *hp = HAlloc(c_p, ERTS_REF_THING_SIZE);
+    write_ref_thing(hp, ref[0], ref[1], ref[2]);
+    return make_internal_ref(hp);
+}
+
+#endif
+
 #endif /*  ERTS_BIF_UNIQUE_H__ */
+
+#if (defined(ERTS_ALLOC_C__) || defined(ERL_BIF_UNIQUE_C__)) \
+    && !defined(ERTS_BIF_UNIQUE_H__FIX_ALLOC_TYPES__)
+#define ERTS_BIF_UNIQUE_H__FIX_ALLOC_TYPES__
+
+#include "hash.h"
+
+typedef struct {
+    HashBucket hash;
+    ErtsMagicBinary *mb;
+    Uint64 value;
+} ErtsNSchedMagicRefTableEntry;
+
+#endif
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index fdc5efea98..c811a002ce 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -18,11 +18,146 @@
  * %CopyrightEnd%
  */
 
-#ifndef __ERL_BINARY_H
-#define __ERL_BINARY_H
+#ifndef ERL_BINARY_H__TYPES__
+#define ERL_BINARY_H__TYPES__
+
+/*
+** Just like the driver binary but with initial flags
+** Note that the two structures Binary and ErlDrvBinary HAVE to
+** be equal except for extra fields in the beginning of the struct.
+** ErlDrvBinary is defined in erl_driver.h.
+** When driver_alloc_binary is called, a Binary is allocated, but 
+** the pointer returned is to the address of the first element that
+** also occurs in the ErlDrvBinary struct (driver.*binary takes care if this).
+** The driver need never know about additions to the internal Binary of the
+** emulator. One should however NEVER be sloppy when mixing ErlDrvBinary
+** and Binary, the macros below can convert one type to the other, as they both
+** in reality are equal.
+*/
+
+#ifdef ARCH_32
+ /* *DO NOT USE* only for alignment. */
+#define ERTS_BINARY_STRUCT_ALIGNMENT Uint32 align__;
+#else
+#define ERTS_BINARY_STRUCT_ALIGNMENT
+#endif
+
+/* Add fields in ERTS_INTERNAL_BINARY_FIELDS, otherwise the drivers crash */
+#define ERTS_INTERNAL_BINARY_FIELDS				\
+    UWord flags;						\
+    erts_refc_t refc;						\
+    ERTS_BINARY_STRUCT_ALIGNMENT
+
+typedef struct binary {
+    ERTS_INTERNAL_BINARY_FIELDS
+    SWord orig_size;
+    char orig_bytes[1]; /* to be continued */
+} Binary;
+
+#define ERTS_SIZEOF_Binary(Sz) \
+    (offsetof(Binary,orig_bytes) + (Sz))
+
+#if ERTS_REF_NUMBERS != 3
+#error "Update ErtsMagicBinary"
+#endif
+
+typedef struct magic_binary ErtsMagicBinary;
+struct magic_binary {
+    ERTS_INTERNAL_BINARY_FIELDS
+    SWord orig_size;
+    void (*destructor)(Binary *);
+    Uint32 refn[ERTS_REF_NUMBERS];
+    ErtsAlcType_t alloc_type;
+    union {
+        struct {
+            ERTS_BINARY_STRUCT_ALIGNMENT
+            char data[1];
+        } aligned;
+        struct {
+            char data[1];
+        } unaligned;
+    } u;
+};
+
+#ifdef ARCH_32
+#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 4
+#else
+#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 0
+#endif
+
+typedef union {
+    Binary binary;
+    ErtsMagicBinary magic_binary;
+    struct {
+	ERTS_INTERNAL_BINARY_FIELDS
+	ErlDrvBinary binary;
+    } driver;
+} ErtsBinary;
+
+/*
+ * 'Binary' alignment:
+ *   Address of orig_bytes[0] of a Binary should always be 8-byte aligned.
+ * It is assumed that the flags, refc, and orig_size fields are 4 bytes on
+ * 32-bits architectures and 8 bytes on 64-bits architectures.
+ */
+
+#define ERTS_MAGIC_BIN_REFN(BP) \
+  ((ErtsBinary *) (BP))->magic_binary.refn
+#define ERTS_MAGIC_BIN_ATYPE(BP) \
+  ((ErtsBinary *) (BP))->magic_binary.alloc_type
+#define ERTS_MAGIC_DATA_OFFSET \
+  (offsetof(ErtsMagicBinary,u.aligned.data) - offsetof(Binary,orig_bytes))
+#define ERTS_MAGIC_BIN_DESTRUCTOR(BP) \
+  ((ErtsBinary *) (BP))->magic_binary.destructor
+#define ERTS_MAGIC_BIN_DATA(BP) \
+  ((void *) ((ErtsBinary *) (BP))->magic_binary.u.aligned.data)
+#define ERTS_MAGIC_BIN_DATA_SIZE(BP) \
+  ((BP)->orig_size - ERTS_MAGIC_DATA_OFFSET)
+#define ERTS_MAGIC_DATA_OFFSET \
+  (offsetof(ErtsMagicBinary,u.aligned.data) - offsetof(Binary,orig_bytes))
+#define ERTS_MAGIC_BIN_ORIG_SIZE(Sz) \
+  (ERTS_MAGIC_DATA_OFFSET + (Sz))
+#define ERTS_MAGIC_BIN_SIZE(Sz) \
+  (offsetof(ErtsMagicBinary,u.aligned.data) + (Sz))
+#define ERTS_MAGIC_BIN_FROM_DATA(DATA) \
+  ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,u.aligned.data)))
+
+/* On 32-bit arch these macro variants will save memory
+   by not forcing 8-byte alignment for the magic payload.
+*/
+#define ERTS_MAGIC_BIN_UNALIGNED_DATA(BP) \
+  ((void *) ((ErtsBinary *) (BP))->magic_binary.u.unaligned.data)
+#define ERTS_MAGIC_UNALIGNED_DATA_OFFSET \
+  (offsetof(ErtsMagicBinary,u.unaligned.data) - offsetof(Binary,orig_bytes))
+#define ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(BP) \
+  ((BP)->orig_size - ERTS_MAGIC_UNALIGNED_DATA_OFFSET)
+#define ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(Sz) \
+  (ERTS_MAGIC_UNALIGNED_DATA_OFFSET + (Sz))
+#define ERTS_MAGIC_BIN_UNALIGNED_SIZE(Sz) \
+  (offsetof(ErtsMagicBinary,u.unaligned.data) + (Sz))
+#define ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(DATA) \
+  ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,u.unaligned.data)))
+
+
+#define Binary2ErlDrvBinary(B) (&((ErtsBinary *) (B))->driver.binary)
+#define ErlDrvBinary2Binary(D) ((Binary *) \
+				(((char *) (D)) \
+				 - offsetof(ErtsBinary, driver.binary)))
+
+/* A "magic" binary flag */
+#define BIN_FLAG_MAGIC      1
+#define BIN_FLAG_USR1       2 /* Reserved for use by different modules too mark */
+#define BIN_FLAG_USR2       4 /*  certain binaries as special (used by ets) */
+#define BIN_FLAG_DRV        8
+
+#endif /* ERL_BINARY_H__TYPES__ */
+
+#if !defined(ERL_BINARY_H__) && !defined(ERTS_BINARY_TYPES_ONLY__)
+#define ERL_BINARY_H__
 
 #include "erl_threads.h"
 #include "bif.h"
+#include "erl_bif_unique.h"
 
 /*
  * Maximum number of bytes to place in a heap binary.
@@ -190,6 +325,7 @@ ERTS_GLB_INLINE Binary *erts_bin_realloc(Binary *bp, Uint size);
 ERTS_GLB_INLINE void erts_bin_free(Binary *bp);
 ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size,
                                                   void (*destructor)(Binary *),
+                                                   ErtsAlcType_t alloc_type,
                                                   int unaligned);
 ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size,
 						 void (*destructor)(Binary *));
@@ -320,9 +456,12 @@ erts_bin_realloc(Binary *bp, Uint size)
 ERTS_GLB_INLINE void
 erts_bin_free(Binary *bp)
 {
-    if (bp->flags & BIN_FLAG_MAGIC)
+    if (bp->flags & BIN_FLAG_MAGIC) {
 	ERTS_MAGIC_BIN_DESTRUCTOR(bp)(bp);
-    if (bp->flags & BIN_FLAG_DRV)
+	erts_magic_ref_remove_bin(ERTS_MAGIC_BIN_REFN(bp));
+        erts_free(ERTS_MAGIC_BIN_ATYPE(bp), (void *) bp);
+    }
+    else if (bp->flags & BIN_FLAG_DRV)
 	erts_free(ERTS_ALC_T_DRV_BINARY, (void *) bp);
     else
 	erts_free(ERTS_ALC_T_BINARY, (void *) bp);
@@ -330,29 +469,33 @@ erts_bin_free(Binary *bp)
 
 ERTS_GLB_INLINE Binary *
 erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *),
+                           ErtsAlcType_t alloc_type,
                            int unaligned)
 {
     Uint bsize = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_SIZE(size)
                            : ERTS_MAGIC_BIN_SIZE(size);
-    Binary* bptr = erts_alloc_fnf(ERTS_ALC_T_BINARY, bsize);
+    Binary* bptr = erts_alloc_fnf(alloc_type, bsize);
     ASSERT(bsize > size);
     if (!bptr)
-	erts_alloc_n_enomem(ERTS_ALC_T2N(ERTS_ALC_T_BINARY), bsize);
+	erts_alloc_n_enomem(ERTS_ALC_T2N(alloc_type), bsize);
     ERTS_CHK_BIN_ALIGNMENT(bptr);
     bptr->flags = BIN_FLAG_MAGIC;
     bptr->orig_size = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(size)
                                 : ERTS_MAGIC_BIN_ORIG_SIZE(size);
     erts_refc_init(&bptr->refc, 0);
     ERTS_MAGIC_BIN_DESTRUCTOR(bptr) = destructor;
+    ERTS_MAGIC_BIN_ATYPE(bptr) = alloc_type;
+    erts_make_magic_ref_in_array(ERTS_MAGIC_BIN_REFN(bptr));
     return bptr;
 }
 
 ERTS_GLB_INLINE Binary *
 erts_create_magic_binary(Uint size, void (*destructor)(Binary *))
 {
-    return erts_create_magic_binary_x(size, destructor, 0);
+    return erts_create_magic_binary_x(size, destructor,
+                                      ERTS_ALC_T_BINARY, 0);
 }
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-#endif /* !__ERL_BINARY_H */
+#endif /* !ERL_BINARY_H__ */
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 5537935802..0ab42394ce 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -5289,24 +5289,35 @@ void db_match_dis(Binary *bp)
 	case matchEqRef:
 	    ++t;
 	    {
-		RefThing *rt = (RefThing *) t;
+		Uint32 *num;
 		int ri;
-		n = thing_arityval(rt->header);
-		erts_printf("EqRef\t(%d) {", (int) n);
+
+		if (is_ordinary_ref_thing(t)) {
+		    ErtsORefThing *rt = (ErtsORefThing *) t;
+		    num = rt->num;
+		    t += TermWords(ERTS_REF_THING_SIZE);
+		}
+		else {
+		    ErtsMRefThing *mrt = (ErtsMRefThing *) t;
+		    ASSERT(is_magic_ref_thing(t));
+		    num = mrt->mb->refn;
+		    t += TermWords(ERTS_MAGIC_REF_THING_SIZE);
+		}
+
+		erts_printf("EqRef\t(%d) {", (int) ERTS_REF_NUMBERS);
 		first = 1;
-		for (ri = 0; ri < n; ++ri) {
+		for (ri = 0; ri < ERTS_REF_NUMBERS; ++ri) {
 		    if (first)
 			first = 0;
 		    else
 			erts_printf(", ");
 #if defined(ARCH_64)
-		    erts_printf("0x%016bex", rt->data.ui[ri]);
+		    erts_printf("0x%016bex", num[ri]);
 #else
-		    erts_printf("0x%08bex", rt->data.ui[ri]);
+		    erts_printf("0x%08bex", num[ri]);
 #endif
 		}
 	    }
-	    t += TermWords(REF_THING_SIZE);
 	    erts_printf("}\n");
 	    break;
 	case matchEqBig:
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index 3526bb684d..517dad9cbb 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -130,7 +130,7 @@ pdisplay1(fmtfn_t to, void *to_arg, Process* p, Eterm obj)
 	Uint32 *ref_num;
 	erts_print(to, to_arg, "#Ref<%lu", ref_channel_no(obj));
 	ref_num = ref_numbers(obj);
-	for (i = ref_no_of_numbers(obj)-1; i >= 0; i--)
+	for (i = ref_no_numbers(obj)-1; i >= 0; i--)
 	    erts_print(to, to_arg, ",%lu", ref_num[i]);
 	erts_print(to, to_arg, ">");
 	break;
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 897dbbe82b..807d9d15e4 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -178,7 +178,7 @@ Uint erts_test_long_gc_sleep; /* Only used for testing... */
 typedef struct {
     Process *proc;
     Eterm ref;
-    Eterm ref_heap[REF_THING_SIZE];
+    Eterm ref_heap[ERTS_REF_THING_SIZE];
     Uint req_sched;
     erts_smp_atomic32_t refc;
 } ErtsGCInfoReq;
@@ -2354,6 +2354,9 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap,
 	    switch (val & _HEADER_SUBTAG_MASK) {
 	    case ARITYVAL_SUBTAG:
 		break;
+	    case REF_SUBTAG:
+		if (is_ordinary_ref_thing(fhp - 1))
+		    goto the_default;
 	    case REFC_BINARY_SUBTAG:
 	    case FUN_SUBTAG:
 	    case EXTERNAL_PID_SUBTAG:
@@ -2363,6 +2366,7 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap,
 		cpy_sz = thing_arityval(val);
 		goto cpy_words;
 	    default:
+	    the_default:
 		cpy_sz = header_arity(val);
 
 	    cpy_words:
@@ -2862,6 +2866,15 @@ sweep_off_heap(Process *p, int fullsweep)
 		    }
 		    break;
 		}
+	    case REF_SUBTAG:
+		{
+		    ErtsMagicBinary *bptr;
+		    ASSERT(is_magic_ref_thing(ptr));
+		    bptr = ((ErtsMRefThing *) ptr)->mb;
+		    if (erts_refc_dectest(&bptr->refc, 0) == 0)
+			erts_bin_free((Binary *) bptr);		    
+		    break;
+		}
 	    default:
 		ASSERT(is_external_header(ptr->thing_word));
 		erts_deref_node_entry(((ExternalThing*)ptr)->node);
@@ -2883,7 +2896,8 @@ sweep_off_heap(Process *p, int fullsweep)
 	}
 	else {
 	    ASSERT(is_fun_header(ptr->thing_word) ||
-		   is_external_header(ptr->thing_word));
+		   is_external_header(ptr->thing_word)
+		   || is_magic_ref_thing(ptr));
 	    prev = &ptr->next;
 	    ptr = ptr->next;
 	}
@@ -2978,6 +2992,9 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 	      }
 	      tari = thing_arityval(val);
 	      switch (thing_subtag(val)) {
+	      case REF_SUBTAG:
+		  if (is_ordinary_ref_thing(hp))
+		      break;
 	      case REFC_BINARY_SUBTAG:
 	      case FUN_SUBTAG:
 	      case EXTERNAL_PID_SUBTAG:
@@ -3175,7 +3192,7 @@ reply_gc_info(void *vgcirp)
 	if (hpp)
 	    ref_copy = STORE_NC(hpp, ohp, gcirp->ref);
 	else
-	    *szp += REF_THING_SIZE;
+	    *szp += ERTS_REF_THING_SIZE;
 
 	msg = erts_bld_tuple(hpp, szp, 3,
 			     make_small(esdp->no),
@@ -3526,6 +3543,10 @@ erts_check_off_heap2(Process *p, Eterm *htop)
 	case EXTERNAL_REF_SUBTAG:
 	    refc = erts_smp_refc_read(&u.ext->node->refc, 1);
 	    break;
+	case REF_SUBTAG:
+	    ASSERT(is_magic_ref_thing(u.hdr));
+	    refc = erts_refc_read(&u.mref->mb->refc, 1);
+	    break;
 	default:
 	    ASSERT(!"erts_check_off_heap2: Invalid thing_word");
 	}
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 647fa26811..26be8c7edf 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -1771,7 +1771,7 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos,
 
     esdp = erts_proc_sched_data(c_p);
 
-    hp = HAlloc(c_p, REF_THING_SIZE);
+    hp = HAlloc(c_p, ERTS_REF_THING_SIZE);
     ref = erts_sched_make_ref_in_buffer(esdp, hp);
 
     ASSERT(erts_get_ref_numbers_thr_id(
@@ -1939,7 +1939,7 @@ access_sched_local_btm(Process *c_p, Eterm pid,
 	Eterm *hp_end;
 #endif
 
-	hsz = REF_THING_SIZE;
+	hsz = ERTS_REF_THING_SIZE;
 	if (async) {
 	    refn = trefn; /* timer ref */
 	    hsz += 4; /* 3-tuple */
@@ -1973,7 +1973,7 @@ access_sched_local_btm(Process *c_p, Eterm pid,
 			refn[1],
 			refn[2]);
 	ref = make_internal_ref(hp);
-	hp += REF_THING_SIZE;
+	hp += ERTS_REF_THING_SIZE;
 
 	msg = (async
 	       ? TUPLE3(hp, (cancel
@@ -2087,7 +2087,7 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
 	ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN;
 	ErlOffHeap *ohp;
 
-	hsz = 4 + REF_THING_SIZE;
+	hsz = 4 + ERTS_REF_THING_SIZE;
 	if (time_left > (Sint64) MAX_SMALL)
 	    hsz += ERTS_SINT64_HEAP_SIZE(time_left);
 
@@ -2103,7 +2103,7 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
 			trefn[1],
 			trefn[2]);
 	tref = make_internal_ref(hp);
-	hp += REF_THING_SIZE;
+	hp += ERTS_REF_THING_SIZE;
 
 	if (time_left < 0)
 	    res = am_false;
@@ -2189,7 +2189,7 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info)
 	    Eterm *hp, rref;
 	    Uint32 *rrefn;
 
-	    hp = HAlloc(c_p, REF_THING_SIZE);
+	    hp = HAlloc(c_p, ERTS_REF_THING_SIZE);
 	    rref = erts_sched_make_ref_in_buffer(esdp, hp);
 	    rrefn = internal_ref_numbers(rref);
 
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 0646665b07..b2c307e826 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -111,10 +111,12 @@ const int etp_lock_check = 1;
 #else
 const int etp_lock_check = 0;
 #endif
-#ifdef WORDS_BIGENDIAN
-const int etp_big_endian = 1;
+const int etp_endianness = ERTS_ENDIANNESS;
+const Eterm etp_ref_header = ERTS_REF_THING_HEADER;
+#ifdef ERTS_MAGIC_REF_THING_HEADER
+const Eterm etp_magic_ref_header = ERTS_MAGIC_REF_THING_HEADER;
 #else
-const int etp_big_endian = 0;
+const Eterm etp_magic_ref_header = ERTS_REF_THING_HEADER;
 #endif
 const Eterm etp_the_non_value = THE_NON_VALUE;
 #ifdef ERTS_HOLE_MARKER
@@ -772,6 +774,8 @@ early_init(int *argc, char **argv) /*
     H_MAX_SIZE = H_DEFAULT_MAX_SIZE;
     H_MAX_FLAGS = MAX_HEAP_SIZE_KILL|MAX_HEAP_SIZE_LOG;
 
+    erts_term_init();
+
     erts_initialized = 0;
 
     erts_use_sender_punish = 1;
@@ -1143,6 +1147,8 @@ early_init(int *argc, char **argv) /*
     no_schedulers_online = schdlrs_onln;
 
     erts_no_schedulers = (Uint) no_schedulers;
+#else
+    erts_no_schedulers = 1;
 #endif
 #ifdef ERTS_DIRTY_SCHEDULERS
     erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers = dirty_cpu_scheds;
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 08dcbed91c..c98581a45e 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -156,6 +156,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"tracer_mtx", 				NULL			},
     {   "port_table",                           NULL                    },
 #endif
+    {	"magic_ref_table",			"address"		},
     {	"mtrace_op",				NULL			},
     {	"instr_x",				NULL			},
     {	"instr",				NULL			},
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 5e160de080..f181c1e3cb 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -176,6 +176,11 @@ erts_cleanup_offheap(ErlOffHeap *offheap)
 		erts_erase_fun_entry(u.fun->fe);		    
 	    }
 	    break;
+	case REF_SUBTAG:
+	    ASSERT(is_magic_ref_thing(u.hdr));
+	    if (erts_refc_dectest(&u.mref->mb->refc, 0) == 0)
+		erts_bin_free((Binary *)u.mref->mb);		    
+	    break;
 	default:
 	    ASSERT(is_external_header(u.hdr->thing_word));
 	    erts_deref_node_entry(u.ext->node);
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
index c207dea10f..bdfc7f30d9 100644
--- a/erts/emulator/beam/erl_monitors.c
+++ b/erts/emulator/beam/erl_monitors.c
@@ -45,6 +45,7 @@
 #include "bif.h"
 #include "big.h"
 #include "erl_monitors.h"
+#include "erl_bif_unique.h"
 
 #define STACK_NEED 50
 #define MAX_MONITORS 0xFFFFFFFFUL
@@ -79,7 +80,24 @@ static ERTS_INLINE int cmp_mon_ref(Eterm ref1, Eterm ref2)
     b2 = boxed_val(ref2);
     if (is_ref_thing_header(*b1)) {
 	if (is_ref_thing_header(*b2)) {
-	    return memcmp(b1+1,b2+1,ERTS_REF_WORDS*sizeof(Uint));
+	    Uint32 *num1, *num2;
+	    if (is_ordinary_ref_thing(b1)) {
+		ErtsORefThing *rtp = (ErtsORefThing *) b1;
+		num1 = rtp->num;
+	    }
+	    else {
+		ErtsMRefThing *mrtp = (ErtsMRefThing *) b1;
+		num1 = mrtp->mb->refn;
+	    }
+	    if (is_ordinary_ref_thing(b2)) {
+		ErtsORefThing *rtp = (ErtsORefThing *) b2;
+		num2 = rtp->num;
+	    }
+	    else {
+		ErtsMRefThing *mrtp = (ErtsMRefThing *) b2;
+		num2 = mrtp->mb->refn;
+	    }
+	    return erts_internal_ref_number_cmp(num1, num2);
 	}
 	return -1;
     }
@@ -97,7 +115,8 @@ do {								\
 	Uint i__;						\
 	Uint len__;						\
 	ASSERT((Hp));						\
-	ASSERT(is_internal_ref((From)) || is_external((From)));	\
+	ASSERT(is_internal_ordinary_ref((From))                 \
+               || is_external((From)));                         \
 	(To) = make_boxed((Hp));				\
 	len__ = thing_arityval(*boxed_val((From))) + 1;		\
 	for(i__ = 0; i__ < len__; i__++)			\
@@ -339,6 +358,8 @@ void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, Eterm pid,
     int state = 0;
     ErtsMonitor **this = root;
     Sint c;
+
+    ASSERT(is_internal_ordinary_ref(ref) || is_external_ref(ref));
   
     dstack[0] = DIR_END;
     for (;;) {
diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h
index 9e2beedea3..bb493bf336 100644
--- a/erts/emulator/beam/erl_monitors.h
+++ b/erts/emulator/beam/erl_monitors.h
@@ -91,7 +91,7 @@
 
 /* Size of a monitor without heap, in words (fixalloc) */
 #define ERTS_MONITOR_SIZE ((sizeof(ErtsMonitor) - sizeof(Uint))/sizeof(Uint))
-#define ERTS_MONITOR_SH_SIZE (ERTS_MONITOR_SIZE + REF_THING_SIZE)
+#define ERTS_MONITOR_SH_SIZE (ERTS_MONITOR_SIZE + ERTS_REF_THING_SIZE)
 #define ERTS_LINK_SIZE ((sizeof(ErtsLink) - sizeof(Uint))/sizeof(Uint))
 #define ERTS_LINK_SH_SIZE ERTS_LINK_SIZE /* Size of fix-alloced links */
 
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index 66bb55e6c8..1c3160efaf 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -66,9 +66,9 @@ static Uint msacc_unmanaged_count = 0;
 #endif
 
 #if ERTS_MSACC_STATE_COUNT < MAP_SMALL_MAP_LIMIT
-#define DEFAULT_MSACC_MSG_SIZE (3 + 1 + ERTS_MSACC_STATE_COUNT * 2 + 3 + REF_THING_SIZE)
+#define DEFAULT_MSACC_MSG_SIZE (3 + 1 + ERTS_MSACC_STATE_COUNT * 2 + 3 + ERTS_REF_THING_SIZE)
 #else
-#define DEFAULT_MSACC_MSG_SIZE (3 + ERTS_MSACC_STATE_COUNT * 3 + 3 + REF_THING_SIZE)
+#define DEFAULT_MSACC_MSG_SIZE (3 + ERTS_MSACC_STATE_COUNT * 3 + 3 + ERTS_REF_THING_SIZE)
 #endif
 
 /* we have to split initiation as atoms are not inited in early init */
@@ -212,7 +212,7 @@ typedef struct {
     int action;
     Process *proc;
     Eterm ref;
-    Eterm ref_heap[REF_THING_SIZE];
+    Eterm ref_heap[ERTS_REF_THING_SIZE];
     Uint req_sched;
     erts_smp_atomic32_t refc;
 } ErtsMSAccReq;
@@ -245,7 +245,7 @@ static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) {
 
         if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx);
 
-        hp = erts_produce_heap(&factory, REF_THING_SIZE + 3 /* tuple */, 0);
+        hp = erts_produce_heap(&factory, ERTS_REF_THING_SIZE + 3 /* tuple */, 0);
         ref_copy = STORE_NC(&hp, &msgp->hfrag.off_heap, msaccrp->ref);
         msg = erts_msacc_gather_stats(msacc, &factory);
         msg = TUPLE2(hp, ref_copy, msg);
@@ -255,7 +255,7 @@ static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) {
         erts_factory_close(&factory);
     } else {
         ErlOffHeap *ohp = NULL;
-        msgp = erts_alloc_message_heap(rp, &rp_locks, REF_THING_SIZE, &hp, &ohp);
+        msgp = erts_alloc_message_heap(rp, &rp_locks, ERTS_REF_THING_SIZE, &hp, &ohp);
         msg = STORE_NC(&hp, &msgp->hfrag.off_heap, msaccrp->ref);
     }
 
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 7c7a32f234..20b00bd4ba 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1721,7 +1721,7 @@ ERL_NIF_TERM enif_make_string_len(ErlNifEnv* env, const char* string,
 
 ERL_NIF_TERM enif_make_ref(ErlNifEnv* env)
 {
-    Eterm* hp = alloc_heap(env, REF_THING_SIZE);
+    Eterm* hp = alloc_heap(env, ERTS_REF_THING_SIZE);
     return erts_make_ref_in_buffer(hp);
 }
 
@@ -1967,7 +1967,6 @@ typedef struct enif_resource_t
     byte align__[4];
 # endif
 #endif
-
     char data[1];
 }ErlNifResource;
 
@@ -2166,6 +2165,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t size)
 {
     Binary* bin = erts_create_magic_binary_x(SIZEOF_ErlNifResource(size),
                                              &nif_resource_dtor,
+                                             ERTS_ALC_T_BINARY,
                                              1); /* unaligned */
     ErlNifResource* resource = ERTS_MAGIC_BIN_UNALIGNED_DATA(bin);
 
diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h
index 0c76c6fe7d..e27f1d121a 100644
--- a/erts/emulator/beam/erl_node_container_utils.h
+++ b/erts/emulator/beam/erl_node_container_utils.h
@@ -255,19 +255,16 @@ extern ErtsPTab erts_port;
  * Refs                                                                    *
 \*                                                                         */
 
+#define internal_ref_no_numbers(x)	ERTS_REF_NUMBERS
+#define internal_ref_numbers(x)		(is_internal_ordinary_ref((x)) \
+					 ? internal_ordinary_ref_numbers((x)) \
+					 : (ASSERT(is_internal_magic_ref((x))), \
+					    internal_magic_ref_numbers((x))))
 #if defined(ARCH_64)
 
-#define internal_ref_no_of_numbers(x)					\
-  (internal_ref_data((x))[0])
-#define internal_thing_ref_no_of_numbers(thing)			        \
-  (internal_thing_ref_data(thing)[0])
-#define internal_ref_numbers(x)						\
-  (&internal_ref_data((x))[1])
-#define internal_thing_ref_numbers(thing)				\
-  (&internal_thing_ref_data(thing)[1])
-#define external_ref_no_of_numbers(x)					\
+#define external_ref_no_numbers(x)					\
   (external_ref_data((x))[0])
-#define external_thing_ref_no_of_numbers(thing)			        \
+#define external_thing_ref_no_numbers(thing)			        \
   (external_thing_ref_data(thing)[0])
 #define external_ref_numbers(x)						\
   (&external_ref_data((x))[1])
@@ -277,12 +274,8 @@ extern ErtsPTab erts_port;
 
 #else
 
-#define internal_ref_no_of_numbers(x)	(internal_ref_data_words((x)))
-#define internal_thing_ref_no_of_numbers(t) (internal_thing_ref_data_words(t))
-#define internal_ref_numbers(x)		(internal_ref_data((x)))
-#define internal_thing_ref_numbers(t)   (internal_thing_ref_data(t))
-#define external_ref_no_of_numbers(x)	(external_ref_data_words((x)))
-#define external_thing_ref_no_of_numbers(t) (external_thing_ref_data_words((t)))
+#define external_ref_no_numbers(x)	(external_ref_data_words((x)))
+#define external_thing_ref_no_numbers(t) (external_thing_ref_data_words((t)))
 #define external_ref_numbers(x)		(external_ref_data((x)))
 #define external_thing_ref_numbers(t)   (external_thing_ref_data((t)))
 
@@ -299,15 +292,9 @@ extern ErtsPTab erts_port;
 #define internal_ref_channel_no(x)	(internal_channel_no((x)))
 #define external_ref_channel_no(x)	(external_channel_no((x)))
 
-#define ref_data_words(x)		(is_internal_ref((x))		\
-					 ? internal_ref_data_words((x))	\
-					 : external_ref_data_words((x)))
-#define ref_data(x)			(is_internal_ref((x))		\
-					 ? internal_ref_data((x))	\
-					 : external_ref_data((x)))
-#define ref_no_of_numbers(x)		(is_internal_ref((x))		\
-					 ? internal_ref_no_of_numbers((x))\
-					 : external_ref_no_of_numbers((x)))
+#define ref_no_numbers(x)		(is_internal_ref((x))		\
+					 ? internal_ref_no_numbers((x))\
+					 : external_ref_no_numbers((x)))
 #define ref_numbers(x)			(is_internal_ref((x))		\
 					 ? internal_ref_numbers((x))	\
 					 : external_ref_numbers((x)))
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 467a05f950..060fbd5883 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1126,8 +1126,8 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id)
 
     for (u.hdr = oh->first; u.hdr; u.hdr = u.hdr->next) {
 	switch (thing_subtag(u.hdr->thing_word)) {
-	case REFC_BINARY_SUBTAG:
-	    if(IsMatchProgBinary(u.pb->val)) {
+	case REF_SUBTAG:
+	    if(IsMatchProgBinary(u.mref->mb)) {
 		InsertedBin *ib;
 		int insert_bin = 1;
 		for (ib = inserted_bins; ib; ib = ib->next)
@@ -1152,6 +1152,7 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id)
 		}
 	    }		
 	    break;
+	case REFC_BINARY_SUBTAG:
 	case FUN_SUBTAG:
 	    break; /* No need to */
 	default:
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index b43a1b0190..6b64bbd2f1 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -382,12 +382,15 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) {
 	    break;
 	}
 	case REF_DEF:
+            if (!ERTS_IS_CRASH_DUMPING)
+                erts_magic_ref_save_bin(obj);
+            /* fall through... */
 	case EXTERNAL_REF_DEF:
 	    PRINT_STRING(res, fn, arg, "#Ref<");
 	    PRINT_UWORD(res, fn, arg, 'u', 0, 1,
 			(ErlPfUWord) ref_channel_no(wobj));
 	    ref_num = ref_numbers(wobj);
-	    for (i = ref_no_of_numbers(wobj)-1; i >= 0; i--) {
+	    for (i = ref_no_numbers(wobj)-1; i >= 0; i--) {
 		PRINT_CHAR(res, fn, arg, '.');
 		PRINT_UWORD(res, fn, arg, 'u', 0, 1, (ErlPfUWord) ref_num[i]);
 	    }
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 3691e31922..87d71fdd22 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1173,7 +1173,7 @@ typedef struct {
     int enable;
     Process *proc;
     Eterm ref;
-    Eterm ref_heap[REF_THING_SIZE];
+    Eterm ref_heap[ERTS_REF_THING_SIZE];
     Uint req_sched;
     erts_smp_atomic32_t refc;
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -1185,7 +1185,7 @@ typedef struct {
 typedef struct {
     Process *proc;
     Eterm ref;
-    Eterm ref_heap[REF_THING_SIZE];
+    Eterm ref_heap[ERTS_REF_THING_SIZE];
     Uint req_sched;
     erts_smp_atomic32_t refc;
 } ErtsSystemCheckReq;
@@ -1297,7 +1297,7 @@ reply_sched_wall_time(void *vswtrp)
             if (hpp)
                 ref_copy = STORE_NC(hpp, ohp, swtrp->ref);
             else
-                *szp += REF_THING_SIZE;
+                *szp += ERTS_REF_THING_SIZE;
 
             ASSERT(!swtrp->set);
 
@@ -1342,7 +1342,7 @@ reply_sched_wall_time(void *vswtrp)
             if (hpp)
                 ref_copy = STORE_NC(hpp, ohp, swtrp->ref);
             else
-                *szp += REF_THING_SIZE;
+                *szp += ERTS_REF_THING_SIZE;
 
             if (swtrp->set)
                 msg = ref_copy;
@@ -1444,7 +1444,7 @@ reply_system_check(void *vscrp)
     ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
 #endif
 
-    sz = REF_THING_SIZE;
+    sz = ERTS_REF_THING_SIZE;
     mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
     hpp = &hp;
     msg = STORE_NC(hpp, ohp, scrp->ref);
@@ -6381,7 +6381,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	erts_tsd_set(sched_data_key, (void *) esdp);
 #endif
     }
-    erts_no_schedulers = 1;
     erts_no_dirty_cpu_schedulers = 0;
     erts_no_dirty_io_schedulers = 0;
 #endif
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index 7d857ad326..6b25728af7 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -59,6 +59,42 @@ erts_set_literal_tag(Eterm *term, Eterm *hp_start, Eterm hsz)
 #endif
 }
 
+void
+erts_term_init(void)
+{
+#ifdef ERTS_ORDINARY_REF_MARKER
+    /* Ordinary and magic references of same size... */
+
+    ErtsRefThing ref_thing;
+
+    ERTS_CT_ASSERT(ERTS_ORDINARY_REF_MARKER == ~((Uint32)0));
+    ref_thing.m.header = ERTS_REF_THING_HEADER;
+    ref_thing.m.mb = (ErtsMagicBinary *) ~((UWord) 3);
+    ref_thing.m.next = (struct erl_off_heap_header *) ~((UWord) 3);
+    if (ref_thing.o.marker == ERTS_ORDINARY_REF_MARKER)
+        ERTS_INTERNAL_ERROR("Cannot differentiate between magic and ordinary references");
+
+    ERTS_CT_ASSERT(offsetof(ErtsORefThing,marker) != 0);
+    ERTS_CT_ASSERT(sizeof(ErtsORefThing) == sizeof(ErtsMRefThing));
+#  ifdef ERTS_MAGIC_REF_THING_HEADER
+#    error Magic ref thing header should not have been defined...
+#  endif
+
+#else
+    /* Ordinary and magic references of different sizes... */
+
+#  ifndef ERTS_MAGIC_REF_THING_HEADER
+#    error Magic ref thing header should have been defined...
+#  endif
+    ERTS_CT_ASSERT(sizeof(ErtsORefThing) != sizeof(ErtsMRefThing));
+
+#endif
+
+    ERTS_CT_ASSERT(ERTS_REF_THING_SIZE*sizeof(Eterm) == sizeof(ErtsORefThing));
+    ERTS_CT_ASSERT(ERTS_MAGIC_REF_THING_SIZE*sizeof(Eterm) == sizeof(ErtsMRefThing));
+
+}
+
 /*
  * XXX: define NUMBER_CODE() here when new representation is used
  */
@@ -95,8 +131,8 @@ ET_DEFINE_CHECKED(Eterm*,tuple_val,Wterm,is_tuple);
 ET_DEFINE_CHECKED(struct erl_node_*,internal_pid_node,Eterm,is_internal_pid);
 ET_DEFINE_CHECKED(struct erl_node_*,internal_port_node,Eterm,is_internal_port);
 ET_DEFINE_CHECKED(Eterm*,internal_ref_val,Wterm,is_internal_ref);
-ET_DEFINE_CHECKED(Uint,internal_ref_data_words,Wterm,is_internal_ref);
-ET_DEFINE_CHECKED(Uint32*,internal_ref_data,Wterm,is_internal_ref);
+ET_DEFINE_CHECKED(Uint32*,internal_magic_ref_numbers,Wterm,is_internal_magic_ref);
+ET_DEFINE_CHECKED(Uint32*,internal_ordinary_ref_numbers,Wterm,is_internal_ordinary_ref);
 ET_DEFINE_CHECKED(struct erl_node_*,internal_ref_node,Eterm,is_internal_ref);
 ET_DEFINE_CHECKED(Eterm*,external_val,Wterm,is_external);
 ET_DEFINE_CHECKED(Uint,external_data_words,Wterm,is_external);
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index c3234ee349..a602a8f7c6 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -23,6 +23,8 @@
 
 #include "erl_mmap.h"
 
+void erts_term_init(void);
+
 typedef UWord Wterm;  /* Full word terms */
 
 struct erl_node_; /* Declared in erl_node_tables.h */
@@ -718,73 +720,224 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_port_node,Eterm)
 #define ERTS_MAX_REF_NUMBERS	3
 #define ERTS_REF_NUMBERS	ERTS_MAX_REF_NUMBERS
 
-#if defined(ARCH_64)
-#  define ERTS_REF_WORDS	(ERTS_REF_NUMBERS/2 + 1)
-#  define ERTS_REF_32BIT_WORDS  (ERTS_REF_NUMBERS+1)
-#else
-#  define ERTS_REF_WORDS	ERTS_REF_NUMBERS
-#  define ERTS_REF_32BIT_WORDS  ERTS_REF_NUMBERS
+#ifndef ERTS_ENDIANNESS
+# error ERTS_ENDIANNESS not defined...
 #endif
 
-typedef struct {
-    Eterm      header;
-    union {
-	Uint32 ui32[ERTS_REF_32BIT_WORDS];
-	Uint   ui[ERTS_REF_WORDS];
-    } data;
-} RefThing;
-
-#define REF_THING_SIZE (sizeof(RefThing)/sizeof(Uint))
-#define REF_THING_HEAD_SIZE (sizeof(Eterm)/sizeof(Uint))
+#if ERTS_REF_NUMBERS != 3
+#  error "A new reference layout for 64-bit needs to be implemented..."
+#endif
 
-#define make_ref_thing_header(DW) \
-  _make_header((DW)+REF_THING_HEAD_SIZE-1,_TAG_HEADER_REF)
+struct magic_binary;
 
 #if defined(ARCH_64)
 
+# define ERTS_ORDINARY_REF_MARKER (~((Uint32) 0))
+
+typedef struct {
+    Eterm header;
+#if ERTS_ENDIANNESS <= 0
+    Uint32 marker;
+#endif
+    Uint32 num[ERTS_REF_NUMBERS];
+#if ERTS_ENDIANNESS > 0
+    Uint32 marker;
+#endif
+} ErtsORefThing;
+
+typedef struct {
+    Eterm header;
+    struct magic_binary *mb;
+    struct erl_off_heap_header* next;
+#if !ERTS_ENDIANNESS
+    Uint32 num[ERTS_REF_NUMBERS];
+    Uint32 marker;
+#endif
+} ErtsMRefThing;
+
 /*
- * Ref layout on a 64-bit little endian machine:
+ * Ordinary ref layout on a 64-bit little endian machine:
  *
  * 63             31             0
  * +--------------+--------------+
  * |         Thing word          |
  * +--------------+--------------+
- * |  Data 0      | 32-bit arity |
+ * |  Data 0      | 0xffffffff   |
  * +--------------+--------------+
  * |  Data 2      | Data 1       |
  * +--------------+--------------+
  *
- * Data is stored as an Uint32 array with 32-bit arity as first number.
+ * Ordinary ref layout on a 64-bit big endian machine:
+ *
+ * 63             31             0
+ * +--------------+--------------+
+ * |         Thing word          |
+ * +--------------+--------------+
+ * |  Data 0      | Data 1       |
+ * +--------------+--------------+
+ * |  Data 2      | 0xffffffff   |
+ * +--------------+--------------+
+ *
+ * Magic Ref layout on a 64-bit machine:
+ *
+ * 63             31             0
+ * +--------------+--------------+
+ * |         Thing word          |
+ * +--------------+--------------+
+ * |  Magic Binary Pointer       |
+ * +--------------+--------------+
+ * |  Next Off Heap Pointer      |
+ * +--------------+--------------+
+ *
+ * Both pointers in the magic ref are 64-bit aligned. That is,
+ * least significant bits are zero. The marker 32-bit word is
+ * placed over the least significant bits of one of the pointers.
+ * That is, we can distinguish between magic and ordinary ref
+ * by looking at the marker field.
+ * 
  */
 
 #define write_ref_thing(Hp, R0, R1, R2)					\
 do {									\
-  ((RefThing *) (Hp))->header  = make_ref_thing_header(ERTS_REF_WORDS);	\
-  ((RefThing *) (Hp))->data.ui32[0] = ERTS_REF_NUMBERS;			\
-  ((RefThing *) (Hp))->data.ui32[1] = (R0);				\
-  ((RefThing *) (Hp))->data.ui32[2] = (R1);				\
-  ((RefThing *) (Hp))->data.ui32[3] = (R2);				\
+  ((ErtsORefThing *) (Hp))->header = ERTS_REF_THING_HEADER;		\
+  ((ErtsORefThing *) (Hp))->marker = ERTS_ORDINARY_REF_MARKER;		\
+  ((ErtsORefThing *) (Hp))->num[0] = (R0);				\
+  ((ErtsORefThing *) (Hp))->num[1] = (R1);				\
+  ((ErtsORefThing *) (Hp))->num[2] = (R2);				\
 } while (0)
 
-#else
+#if ERTS_ENDIANNESS
+/* Known big or little endian */
+
+#define write_magic_ref_thing(Hp, Ohp, Binp)				\
+do {									\
+  ((ErtsMRefThing *) (Hp))->header = ERTS_REF_THING_HEADER;		\
+  ((ErtsMRefThing *) (Hp))->mb = (Binp);				\
+  ((ErtsMRefThing *) (Hp))->next = (Ohp)->first;			\
+  (Ohp)->first = (struct erl_off_heap_header*) (Hp);			\
+  ASSERT(erts_is_ref_numbers_magic((Binp)->refn));			\
+} while (0)
+
+#else /* !ERTS_ENDIANNESS */
+
+#define write_magic_ref_thing(Hp, Ohp, Binp)				\
+do {									\
+  ((ErtsMRefThing *) (Hp))->header = ERTS_MAGIC_REF_THING_HEADER;	\
+  ((ErtsMRefThing *) (Hp))->mb = (Binp);				\
+  ((ErtsMRefThing *) (Hp))->next = (Ohp)->first;			\
+  (Ohp)->first = (struct erl_off_heap_header*) (Hp);			\
+  ((ErtsMRefThing *) (Hp))->marker = 0;					\
+  ((ErtsMRefThing *) (Hp))->num[0] = (Binp)->refn[0];			\
+  ((ErtsMRefThing *) (Hp))->num[1] = (Binp)->refn[1];			\
+  ((ErtsMRefThing *) (Hp))->num[2] = (Binp)->refn[2];			\
+  ASSERT(erts_is_ref_numbers_magic((Binp)->refn));			\
+} while (0)
+
+#endif /* !ERTS_ENDIANNESS */
+
+#else /* ARCH_32 */
+
+typedef struct {
+    Eterm header;
+    Uint32 num[ERTS_REF_NUMBERS];
+} ErtsORefThing;
+
+typedef struct {
+    Eterm header;
+    struct magic_binary *mb;
+    struct erl_off_heap_header* next;
+} ErtsMRefThing;
+
 
 #define write_ref_thing(Hp, R0, R1, R2)					\
 do {									\
-  ((RefThing *) (Hp))->header  = make_ref_thing_header(ERTS_REF_WORDS);	\
-  ((RefThing *) (Hp))->data.ui32[0] = (R0);				\
-  ((RefThing *) (Hp))->data.ui32[1] = (R1);				\
-  ((RefThing *) (Hp))->data.ui32[2] = (R2);				\
+  ((ErtsORefThing *) (Hp))->header = ERTS_REF_THING_HEADER;		\
+  ((ErtsORefThing *) (Hp))->num[0] = (R0);				\
+  ((ErtsORefThing *) (Hp))->num[1] = (R1);				\
+  ((ErtsORefThing *) (Hp))->num[2] = (R2);				\
 } while (0)
 
+#define write_magic_ref_thing(Hp, Ohp, Binp)				\
+do {									\
+  ((ErtsMRefThing *) (Hp))->header = ERTS_MAGIC_REF_THING_HEADER;	\
+  ((ErtsMRefThing *) (Hp))->mb = (Binp);				\
+  ((ErtsMRefThing *) (Hp))->next = (Ohp)->first;			\
+  (Ohp)->first = (struct erl_off_heap_header*) (Hp);			\
+  ASSERT(erts_is_ref_numbers_magic(&(Binp)->refn));			\
+} while (0)
+
+#endif /* ARCH_32 */
+
+typedef union {
+    ErtsMRefThing m;
+    ErtsORefThing o;
+} ErtsRefThing;
+
+#define ERTS_REF_THING_SIZE (sizeof(ErtsORefThing)/sizeof(Uint))
+#define ERTS_MAGIC_REF_THING_SIZE (sizeof(ErtsMRefThing)/sizeof(Uint))
+#define ERTS_MAX_INTERNAL_REF_SIZE (sizeof(ErtsRefThing)/sizeof(Uint))
+
+#define make_ref_thing_header(Words) \
+    _make_header((Words)-1,_TAG_HEADER_REF)
+
+#define ERTS_REF_THING_HEADER _make_header(ERTS_REF_THING_SIZE-1,_TAG_HEADER_REF)
+
+#if defined(ARCH_64) && ERTS_ENDIANNESS /* All internal refs of same size... */
+
+#  undef ERTS_MAGIC_REF_THING_HEADER
+
+#  define is_ref_thing_header(x) ((x) == ERTS_REF_THING_HEADER)
+
+#define is_ordinary_ref_thing(x)					\
+    (ASSERT(is_ref_thing_header(*((Eterm *)(x)))),			\
+     ((ErtsRefThing *) (x))->o.marker == ERTS_ORDINARY_REF_MARKER)
+
+#define is_magic_ref_thing(x)						\
+    (!is_ordinary_ref_thing((x)))
+
+#define is_internal_magic_ref(x)					\
+    ((_unchecked_is_boxed((x)) && *boxed_val((x)) == ERTS_REF_THING_HEADER) \
+     && is_magic_ref_thing(boxed_val((x))))
+
+#define is_internal_ordinary_ref(x)					\
+    ((_unchecked_is_boxed((x)) && *boxed_val((x)) == ERTS_REF_THING_HEADER) \
+     && is_ordinary_ref_thing(boxed_val((x))))
+
+#else /* Ordinary and magic references of different sizes... */
+
+#  define ERTS_MAGIC_REF_THING_HEADER					\
+    _make_header(ERTS_MAGIC_REF_THING_SIZE-1,_TAG_HEADER_REF)
+
+#  define is_ref_thing_header(x)					\
+    (((x) & _TAG_HEADER_MASK) == _TAG_HEADER_REF)
+
+#define is_ordinary_ref_thing(x)					\
+    (ASSERT(is_ref_thing_header(*((Eterm *)(x)))),			\
+     *((Eterm *)(x)) == ERTS_REF_THING_HEADER)
+
+#define is_magic_ref_thing(x)						\
+    (ASSERT(is_ref_thing_header(*((Eterm *)(x)))),			\
+     *((Eterm *)(x)) == ERTS_MAGIC_REF_THING_HEADER)
+
+#define is_internal_magic_ref(x)					\
+    (_unchecked_is_boxed((x)) && *boxed_val((x)) == ERTS_MAGIC_REF_THING_HEADER)
+
+#define is_internal_ordinary_ref(x)					\
+    (_unchecked_is_boxed((x)) && *boxed_val((x)) == ERTS_REF_THING_HEADER)
+
 #endif
 
-#define is_ref_thing_header(x)	(((x) & _TAG_HEADER_MASK) == _TAG_HEADER_REF)
 #define make_internal_ref(x)	make_boxed((Eterm*)(x))
 
-#define _unchecked_ref_thing_ptr(x) \
-  ((RefThing*) _unchecked_internal_ref_val(x))
-#define ref_thing_ptr(x) \
-  ((RefThing*) internal_ref_val(x))
+#define _unchecked_ordinary_ref_thing_ptr(x) \
+  ((ErtsORefThing*) _unchecked_internal_ref_val(x))
+#define ordinary_ref_thing_ptr(x) \
+  ((ErtsORefThing*) internal_ref_val(x))
+
+#define _unchecked_magic_ref_thing_ptr(x) \
+  ((ErtsMRefThing*) _unchecked_internal_ref_val(x))
+#define magic_ref_thing_ptr(x) \
+  ((ErtsMRefThing*) internal_ref_val(x))
 
 #define is_internal_ref(x) \
     (_unchecked_is_boxed((x)) && is_ref_thing_header(*boxed_val((x))))
@@ -796,16 +949,21 @@ do {									\
 _ET_DECLARE_CHECKED(Eterm*,internal_ref_val,Wterm)
 #define internal_ref_val(x) _ET_APPLY(internal_ref_val,(x))
 
-#define internal_thing_ref_data_words(t) (thing_arityval(*(Eterm*)(t)))
-#define _unchecked_internal_ref_data_words(x) \
- (_unchecked_thing_arityval(*_unchecked_internal_ref_val(x)))
-_ET_DECLARE_CHECKED(Uint,internal_ref_data_words,Wterm)
-#define internal_ref_data_words(x) _ET_APPLY(internal_ref_data_words,(x))
+#define internal_ordinary_thing_ref_numbers(ort) (((ErtsORefThing *)(ort))->num)
+#define _unchecked_internal_ordinary_ref_numbers(x) (internal_ordinary_thing_ref_numbers(_unchecked_ordinary_ref_thing_ptr(x)))
+_ET_DECLARE_CHECKED(Uint32*,internal_ordinary_ref_numbers,Wterm)
+#define internal_ordinary_ref_numbers(x) _ET_APPLY(internal_ordinary_ref_numbers,(x))
+
+#if defined(ARCH_64) && !ERTS_ENDIANNESS
+#define internal_magic_thing_ref_numbers(mrt) (((ErtsMRefThing *)(mrt))->num)
+#else
+#define internal_magic_thing_ref_numbers(mrt) (((ErtsMRefThing *)(mrt))->mb->refn)
+#endif
+
+#define _unchecked_internal_magic_ref_numbers(x) (internal_magic_thing_ref_numbers(_unchecked_magic_ref_thing_ptr(x)))
+_ET_DECLARE_CHECKED(Uint32*,internal_magic_ref_numbers,Wterm)
+#define internal_magic_ref_numbers(x) _ET_APPLY(internal_magic_ref_numbers,(x))
 
-#define internal_thing_ref_data(thing) ((thing)->data.ui32)
-#define _unchecked_internal_ref_data(x) (internal_thing_ref_data(_unchecked_ref_thing_ptr(x)))
-_ET_DECLARE_CHECKED(Uint32*,internal_ref_data,Wterm)
-#define internal_ref_data(x) _ET_APPLY(internal_ref_data,(x))
 
 #define _unchecked_internal_ref_node(x) erts_this_node
 _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm)
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 6aa2a7500f..1d747e5fab 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1831,7 +1831,10 @@ erts_demonitor_time_offset(Eterm ref)
     ErtsMonitor *mon;
     ASSERT(is_internal_ref(ref));
     erts_smp_mtx_lock(&erts_get_time_mtx);
-    mon = erts_remove_monitor(&time_offset_monitors, ref);
+    if (is_internal_ordinary_ref(ref))
+        mon = erts_remove_monitor(&time_offset_monitors, ref);
+    else
+        mon = NULL;
     if (!mon)
 	res = 0;
     else {
@@ -1848,7 +1851,7 @@ erts_demonitor_time_offset(Eterm ref)
 typedef struct {
     Eterm pid;
     Eterm ref;
-    Eterm heap[REF_THING_SIZE];
+    Eterm heap[ERTS_REF_THING_SIZE];
 } ErtsTimeOffsetMonitorInfo;
 
 typedef struct {
@@ -1869,11 +1872,11 @@ save_time_offset_monitor(ErtsMonitor *mon, void *vcntxt)
     cntxt->to_mon_info[mix].pid = mon->pid;
     to_hp = &cntxt->to_mon_info[mix].heap[0];
 
-    ASSERT(is_internal_ref(mon->ref));
+    ASSERT(is_internal_ordinary_ref(mon->ref));
     from_hp = internal_ref_val(mon->ref);
-    ASSERT(thing_arityval(*from_hp) + 1 == REF_THING_SIZE);
+    ASSERT(thing_arityval(*from_hp) + 1 == ERTS_REF_THING_SIZE);
 
-    for (hix = 0; hix < REF_THING_SIZE; hix++)
+    for (hix = 0; hix < ERTS_REF_THING_SIZE; hix++)
 	to_hp[hix] = from_hp[hix];
 
     cntxt->to_mon_info[mix].ref
@@ -1933,7 +1936,7 @@ send_time_offset_changed_notifications(void *new_offsetp)
 	hp = (Eterm *) (tmp + no_monitors*sizeof(ErtsTimeOffsetMonitorInfo));
 
 	hsz = 6; /* 5-tuple */
-	hsz += REF_THING_SIZE;
+	hsz += ERTS_REF_THING_SIZE;
 	hsz += ERTS_SINT64_HEAP_SIZE(new_offset);
 
 	if (IS_SSMALL(new_offset))
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 6bcddb9e69..64b17a3f57 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2579,7 +2579,9 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 
 	    ASSERT(dflags & DFLAG_EXTENDED_REFERENCES);
 
-	    i = ref_no_of_numbers(obj);
+	    erts_magic_ref_save_bin(obj);
+
+	    i = ref_no_numbers(obj);
 	    put_int16(i, ep);
 	    ep += 2;
 	    ep = enc_atom(acmp, sysname, ep, dflags);
@@ -3435,26 +3437,35 @@ dec_term_atom_common:
 		r0 = get_int32(ep); /* allow full word */
 		ep += 4;
 
-	    ref_ext_common:
+	    ref_ext_common: {
+		ErtsORefThing *rtp;
+
 		if (ref_words > ERTS_MAX_REF_NUMBERS)
 		    goto error;
 
 		node = dec_get_node(sysname, cre);
 		if(node == erts_this_node) {
-		    RefThing *rtp = (RefThing *) hp;
-		    ref_num = (Uint32 *) (hp + REF_THING_HEAD_SIZE);
+	
+		    rtp = (ErtsORefThing *) hp;
+		    ref_num = &rtp->num[0];
+		    if (ref_words != ERTS_REF_NUMBERS) {
+                        int i;
+                        if (ref_words > ERTS_REF_NUMBERS)
+                            goto error; /* Not a ref that we created... */
+                        for (i = ref_words; i < ERTS_REF_NUMBERS; i++)
+                            ref_num[i] = 0;
+		    }
 
-#if defined(ARCH_64)
-		    hp += REF_THING_HEAD_SIZE + ref_words/2 + 1;
-		    rtp->header = make_ref_thing_header(ref_words/2 + 1);
-#else
-		    hp += REF_THING_HEAD_SIZE + ref_words;
-		    rtp->header = make_ref_thing_header(ref_words);
+#ifdef ERTS_ORDINARY_REF_MARKER
+		    rtp->marker = ERTS_ORDINARY_REF_MARKER;
 #endif
+		    hp += ERTS_REF_THING_SIZE;
+		    rtp->header = ERTS_REF_THING_HEADER;
 		    *objp = make_internal_ref(rtp);
 		}
 		else {
 		    ExternalThing *etp = (ExternalThing *) hp;
+		    rtp = NULL;
 #if defined(ARCH_64)
 		    hp += EXTERNAL_THING_HEAD_SIZE + ref_words/2 + 1;
 #else
@@ -3472,12 +3483,13 @@ dec_term_atom_common:
 		    factory->off_heap->first = (struct erl_off_heap_header*)etp;
 		    *objp = make_external_ref(etp);
 		    ref_num = &(etp->data.ui32[0]);
-		}
-
 #if defined(ARCH_64)
-		*(ref_num++) = ref_words /* 32-bit arity */;
+		    *(ref_num++) = ref_words /* 32-bit arity */;
 #endif
+		}
+
 		ref_num[0] = r0;
+
 		for(i = 1; i < ref_words; i++) {
 		    ref_num[i] = get_int32(ep);
 		    ep += 4;
@@ -3486,8 +3498,24 @@ dec_term_atom_common:
 		if ((1 + ref_words) % 2)
 		    ref_num[ref_words] = 0;
 #endif
+		if (node == erts_this_node) {
+		    /* Check if it was a magic reference... */
+		    ErtsMagicBinary *mb = erts_magic_ref_lookup_bin(ref_num);
+		    if (mb) {
+			/*
+			 * Was a magic ref; adjust it...
+			 *
+			 * Refc on binary was increased by lookup above...
+			 */
+			ASSERT(rtp);
+			hp = (Eterm *) rtp;
+			write_magic_ref_thing(hp, factory->off_heap, mb);
+			hp += ERTS_MAGIC_REF_THING_SIZE;
+		    }
+		}
 		break;
 	    }
+	    }
 	case BINARY_EXT:
 	    {
 		n = get_int32(ep);
@@ -4109,7 +4137,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
             /*fall through*/
 	case REF_DEF:
 	    ASSERT(dflags & DFLAG_EXTENDED_REFERENCES);
-	    i = ref_no_of_numbers(obj);
+	    i = ref_no_numbers(obj);
 	    result += (1 + 2 + encode_size_struct2(acmp, ref_node_name(obj), dflags) +
 		       1 + 4*i);
 	    break;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 713fb81c77..187a948a7b 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -42,6 +42,9 @@
 #include "erl_utils.h"
 #include "erl_port.h"
 #include "erl_gc.h"
+#define ERTS_BINARY_TYPES_ONLY__
+#include "erl_binary.h"
+#undef ERTS_BINARY_TYPES_ONLY__
 
 struct enif_func_t;
 
@@ -126,7 +129,7 @@ typedef struct de_proc_entry {
 			                PROC_AWAIT_LOAD == Wants to be notified when we
 			                reloaded the driver (old was locked) */
     Uint    flags;                   /* ERL_FL_DE_DEREFERENCED when reload in progress */
-    Eterm   heap[REF_THING_SIZE];    /* "ref heap" */
+    Eterm   heap[ERTS_REF_THING_SIZE];    /* "ref heap" */
     struct  de_proc_entry *next;
 } DE_ProcEntry;
 
@@ -213,118 +216,6 @@ extern Eterm erts_ddll_monitor_driver(Process *p,
 				      Eterm description,
 				      ErtsProcLocks plocks);
 
-/*
-** Just like the driver binary but with initial flags
-** Note that the two structures Binary and ErlDrvBinary HAVE to
-** be equal except for extra fields in the beginning of the struct.
-** ErlDrvBinary is defined in erl_driver.h.
-** When driver_alloc_binary is called, a Binary is allocated, but 
-** the pointer returned is to the address of the first element that
-** also occurs in the ErlDrvBinary struct (driver.*binary takes care if this).
-** The driver need never know about additions to the internal Binary of the
-** emulator. One should however NEVER be sloppy when mixing ErlDrvBinary
-** and Binary, the macros below can convert one type to the other, as they both
-** in reality are equal.
-*/
-
-#ifdef ARCH_32
- /* *DO NOT USE* only for alignment. */
-#define ERTS_BINARY_STRUCT_ALIGNMENT Uint32 align__;
-#else
-#define ERTS_BINARY_STRUCT_ALIGNMENT
-#endif
-
-/* Add fields in ERTS_INTERNAL_BINARY_FIELDS, otherwise the drivers crash */
-#define ERTS_INTERNAL_BINARY_FIELDS				\
-    UWord flags;							\
-    erts_refc_t refc;						\
-    ERTS_BINARY_STRUCT_ALIGNMENT
-
-typedef struct binary {
-    ERTS_INTERNAL_BINARY_FIELDS
-    SWord orig_size;
-    char orig_bytes[1]; /* to be continued */
-} Binary;
-
-#define ERTS_SIZEOF_Binary(Sz) \
-    (offsetof(Binary,orig_bytes) + (Sz))
-
-typedef struct {
-    ERTS_INTERNAL_BINARY_FIELDS
-    SWord orig_size;
-    void (*destructor)(Binary *);
-    union {
-        struct {
-            ERTS_BINARY_STRUCT_ALIGNMENT
-            char data[1];
-        } aligned;
-        struct {
-            char data[1];
-        } unaligned;
-    } u;
-} ErtsMagicBinary;
-
-#ifdef ARCH_32
-#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 4
-#else
-#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 0
-#endif
-
-typedef union {
-    Binary binary;
-    ErtsMagicBinary magic_binary;
-    struct {
-	ERTS_INTERNAL_BINARY_FIELDS
-	ErlDrvBinary binary;
-    } driver;
-} ErtsBinary;
-
-/*
- * 'Binary' alignment:
- *   Address of orig_bytes[0] of a Binary should always be 8-byte aligned.
- * It is assumed that the flags, refc, and orig_size fields are 4 bytes on
- * 32-bits architectures and 8 bytes on 64-bits architectures.
- */
-
-#define ERTS_MAGIC_BIN_DESTRUCTOR(BP) \
-  ((ErtsBinary *) (BP))->magic_binary.destructor
-#define ERTS_MAGIC_BIN_DATA(BP) \
-  ((void *) ((ErtsBinary *) (BP))->magic_binary.u.aligned.data)
-#define ERTS_MAGIC_DATA_OFFSET \
-  (offsetof(ErtsMagicBinary,u.aligned.data) - offsetof(Binary,orig_bytes))
-#define ERTS_MAGIC_BIN_ORIG_SIZE(Sz) \
-  (ERTS_MAGIC_DATA_OFFSET + (Sz))
-#define ERTS_MAGIC_BIN_SIZE(Sz) \
-  (offsetof(ErtsMagicBinary,u.aligned.data) + (Sz))
-
-/* On 32-bit arch these macro variants will save memory
-   by not forcing 8-byte alignment for the magic payload.
-*/
-#define ERTS_MAGIC_BIN_UNALIGNED_DATA(BP) \
-  ((void *) ((ErtsBinary *) (BP))->magic_binary.u.unaligned.data)
-#define ERTS_MAGIC_UNALIGNED_DATA_OFFSET \
-  (offsetof(ErtsMagicBinary,u.unaligned.data) - offsetof(Binary,orig_bytes))
-#define ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(BP) \
-  ((BP)->orig_size - ERTS_MAGIC_UNALIGNED_DATA_OFFSET)
-#define ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(Sz) \
-  (ERTS_MAGIC_UNALIGNED_DATA_OFFSET + (Sz))
-#define ERTS_MAGIC_BIN_UNALIGNED_SIZE(Sz) \
-  (offsetof(ErtsMagicBinary,u.unaligned.data) + (Sz))
-#define ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(DATA) \
-  ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,u.unaligned.data)))
-
-
-#define Binary2ErlDrvBinary(B) (&((ErtsBinary *) (B))->driver.binary)
-#define ErlDrvBinary2Binary(D) ((Binary *) \
-				(((char *) (D)) \
-				 - offsetof(ErtsBinary, driver.binary)))
-
-/* A "magic" binary flag */
-#define BIN_FLAG_MAGIC      1
-#define BIN_FLAG_USR1       2 /* Reserved for use by different modules too mark */
-#define BIN_FLAG_USR2       4 /*  certain binaries as special (used by ets) */
-#define BIN_FLAG_DRV        8
-
 /*
  * This structure represents one type of a binary in a process.
  */
@@ -386,6 +277,7 @@ union erl_off_heap_ptr {
     ProcBin *pb;
     struct erl_fun_thing* fun;
     struct external_thing_* ext;
+    ErtsMRefThing *mref;
     Eterm* ep;
     void* voidp;
 };
@@ -978,21 +870,6 @@ void erts_bif_info_init(void);
 
 /* bif.c */
 
-ERTS_GLB_INLINE Eterm
-erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS]);
-
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-
-ERTS_GLB_INLINE Eterm
-erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS])
-{
-    Eterm *hp = HAlloc(c_p, REF_THING_SIZE);
-    write_ref_thing(hp, ref[0], ref[1], ref[2]);
-    return make_internal_ref(hp);
-}
-
-#endif
-
 void erts_queue_monitor_message(Process *,
 				ErtsProcLocks*,
 				Eterm,
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 5446ebaab0..dad24d7ef6 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1445,7 +1445,7 @@ finalize_force_imm_drv_call(ErtsTryImmDrvCallState *sp)
     erts_unblock_fpe(sp->fpe_was_unmasked);
 }
 
-#define ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE (REF_THING_SIZE + 3)
+#define ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE (ERTS_REF_THING_SIZE + 3)
 
 static ERTS_INLINE void
 queue_port_sched_op_reply(Process *rp,
@@ -1460,7 +1460,7 @@ queue_port_sched_op_reply(Process *rp,
 
     ref= make_internal_ref(hp);
     write_ref_thing(hp, ref_num[0], ref_num[1], ref_num[2]);
-    hp += REF_THING_SIZE;
+    hp += ERTS_REF_THING_SIZE;
 
     msg = TUPLE2(hp, ref, msg);
 
@@ -3099,7 +3099,7 @@ port_monitor(Port *prt, erts_aint32_t state, Eterm origin,
 
     ASSERT(is_pid(origin));
     ASSERT(is_atom(name) || is_port(name) || name == NIL);
-    ASSERT(is_internal_ref(ref));
+    ASSERT(is_internal_ordinary_ref(ref));
 
     if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) {
         ErtsProcLocks p_locks = ERTS_PROC_LOCK_LINK;
@@ -3124,7 +3124,7 @@ static int
 port_sig_monitor(Port *prt, erts_aint32_t state, int op,
                  ErtsProc2PortSigData *sigdp)
 {
-    Eterm hp[REF_THING_SIZE];
+    Eterm hp[ERTS_REF_THING_SIZE];
     Eterm ref = make_internal_ref(&hp);
     write_ref_thing(hp, sigdp->ref[0], sigdp->ref[1], sigdp->ref[2]);
 
@@ -3245,7 +3245,7 @@ static int
 port_sig_demonitor(Port *prt, erts_aint32_t state, int op,
                    ErtsProc2PortSigData *sigdp)
 {
-    Eterm hp[REF_THING_SIZE];
+    Eterm hp[ERTS_REF_THING_SIZE];
     Eterm ref = make_internal_ref(&hp);
     write_ref_thing(hp, sigdp->u.demonitor.ref[0],
                     sigdp->u.demonitor.ref[1],
@@ -3302,10 +3302,10 @@ ErtsPortOpResult erts_port_demonitor(Process *origin, ErtsDemonitorMode mode,
     sigdp->u.demonitor.origin = origin->common.id;
     sigdp->u.demonitor.name = target->common.id;
     {
-        RefThing *reft = ref_thing_ptr(ref);
+	Uint32 *nums = internal_ref_numbers(ref);
         /* Start from 1 skip ref arity */
         sys_memcpy(sigdp->u.demonitor.ref,
-                   internal_thing_ref_numbers(reft),
+		   nums,
                    sizeof(sigdp->u.demonitor.ref));
     }
 
@@ -5413,7 +5413,7 @@ reply_io_bytes(void *vreq)
 	    rp_locks = ERTS_PROC_LOCK_MAIN;
 	}
 
-	hsz = 5 /* 4-tuple */ + REF_THING_SIZE;
+	hsz = 5 /* 4-tuple */ + ERTS_REF_THING_SIZE;
 
 	erts_bld_uint64(NULL, &hsz, in);
 	erts_bld_uint64(NULL, &hsz, out);
@@ -5422,7 +5422,7 @@ reply_io_bytes(void *vreq)
 
 	ref = make_internal_ref(hp);
 	write_ref_thing(hp, req->refn[0], req->refn[1], req->refn[2]);
-	hp += REF_THING_SIZE;
+	hp += ERTS_REF_THING_SIZE;
 
 	ein = erts_bld_uint64(&hp, NULL, in);
 	eout = erts_bld_uint64(&hp, NULL, out);
@@ -5451,7 +5451,7 @@ erts_request_io_bytes(Process *c_p)
     ErtsIOBytesReq *req = erts_alloc(ERTS_ALC_T_IOB_REQ,
 				     sizeof(ErtsIOBytesReq));
 
-    hp = HAlloc(c_p, REF_THING_SIZE);
+    hp = HAlloc(c_p, ERTS_REF_THING_SIZE);
     ref = erts_sched_make_ref_in_buffer(esdp, hp);
     refn = internal_ref_numbers(ref);
 
@@ -7609,20 +7609,16 @@ erl_drv_convert_time_unit(ErlDrvTime val,
 
 static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon)
 {
-    RefThing *refp;
-    ASSERT(is_internal_ref(ref));
-    ERTS_CT_ASSERT(sizeof(RefThing) <= sizeof(ErlDrvMonitor));
-    refp = ref_thing_ptr(ref);
-    memset(mon,0,sizeof(ErlDrvMonitor));
-    memcpy(mon,refp,sizeof(RefThing));
+    ERTS_CT_ASSERT(sizeof(ErtsOIRefStorage) <= sizeof(ErlDrvMonitor));
+    erts_oiref_storage_save((ErtsOIRefStorage *) mon, ref);
 }
 
 
 static int do_driver_monitor_process(Port *prt,
-				     Eterm *buf,
 				     ErlDrvTermData process,
 				     ErlDrvMonitor *monitor)
 {
+    Eterm buf[ERTS_REF_THING_SIZE];
     Process *rp;
     Eterm ref;
 
@@ -7665,26 +7661,22 @@ int driver_monitor_process(ErlDrvPort drvport,
     /* Now (in SMP) we should have either the port lock (if we have a scheduler) or the port data lock
        (if we're a driver thread) */
     ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock));
-    {
-	DeclareTmpHeapNoproc(buf,REF_THING_SIZE);
-	UseTmpHeapNoproc(REF_THING_SIZE);
-	ret = do_driver_monitor_process(prt,buf,process,monitor);
-	UnUseTmpHeapNoproc(REF_THING_SIZE);
-    }
+    ret = do_driver_monitor_process(prt,process,monitor);
     DRV_MONITOR_UNLOCK_PDL(prt);
     return ret;
 }
 
-static int do_driver_demonitor_process(Port *prt, Eterm *buf,
-				       const ErlDrvMonitor *monitor)
+static int do_driver_demonitor_process(Port *prt, const ErlDrvMonitor *monitor)
 {
+    Eterm heap[ERTS_REF_THING_SIZE];
+    Eterm *hp = &heap[0];
     Process *rp;
     Eterm ref;
     ErtsMonitor *mon;
     Eterm to;
 
-    memcpy(buf,monitor,sizeof(Eterm)*REF_THING_SIZE);
-    ref = make_internal_ref(buf);
+    ref = erts_oiref_storage_make_ref((ErtsOIRefStorage *) monitor, &hp),
+
     mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref);
     if (mon == NULL) {
 	return 1;
@@ -7728,25 +7720,21 @@ int driver_demonitor_process(ErlDrvPort drvport,
     /* Now we should have either the port lock (if we have a scheduler) or the port data lock
        (if we're a driver thread) */
     ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock));
-    {
-	DeclareTmpHeapNoproc(buf,REF_THING_SIZE);
-	UseTmpHeapNoproc(REF_THING_SIZE);
-	ret = do_driver_demonitor_process(prt,buf,monitor);
-	UnUseTmpHeapNoproc(REF_THING_SIZE);
-    }
+    ret = do_driver_demonitor_process(prt,monitor);
     DRV_MONITOR_UNLOCK_PDL(prt);
     return ret;
 }
 
-static ErlDrvTermData do_driver_get_monitored_process(Port *prt, Eterm *buf,
-					    const ErlDrvMonitor *monitor)
+static ErlDrvTermData do_driver_get_monitored_process(Port *prt,const ErlDrvMonitor *monitor)
 {
     Eterm ref;
     ErtsMonitor *mon;
     Eterm to;
+    Eterm heap[ERTS_REF_THING_SIZE];
+    Eterm *hp = &heap[0];
+
+    ref = erts_oiref_storage_make_ref((ErtsOIRefStorage *) monitor, &hp),
 
-    memcpy(buf,monitor,sizeof(Eterm)*REF_THING_SIZE);
-    ref = make_internal_ref(buf);
     mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref);
     if (mon == NULL) {
 	return driver_term_nil;
@@ -7774,12 +7762,7 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport,
     /* Now we should have either the port lock (if we have a scheduler) or the port data lock
        (if we're a driver thread) */
     ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock));
-    {
-	DeclareTmpHeapNoproc(buf,REF_THING_SIZE);
-	UseTmpHeapNoproc(REF_THING_SIZE);
-	ret = do_driver_get_monitored_process(prt,buf,monitor);
-	UnUseTmpHeapNoproc(REF_THING_SIZE);
-    }
+    ret = do_driver_get_monitored_process(prt,monitor);
     DRV_MONITOR_UNLOCK_PDL(prt);
     return ret;
 }
@@ -7788,7 +7771,8 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport,
 int driver_compare_monitors(const ErlDrvMonitor *monitor1,
 			    const ErlDrvMonitor *monitor2)
 {
-    return memcmp(monitor1,monitor2,sizeof(ErlDrvMonitor));
+    return erts_oiref_storage_cmp((ErtsOIRefStorage *) monitor1,
+				  (ErtsOIRefStorage *) monitor2);
 }
 
 void erts_fire_port_monitor(Port *prt, Eterm ref)
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 25940edab7..fc459e17ad 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1838,7 +1838,7 @@ make_internal_hash(Eterm term)
 	    break;
 	    case REF_SUBTAG:
 		UINT32_HASH(internal_ref_numbers(term)[0], HCONST_7);
-                ASSERT(internal_ref_no_of_numbers(term) == 3);
+                ASSERT(internal_ref_no_numbers(term) == 3);
                 UINT32_HASH_2(internal_ref_numbers(term)[1],
                               internal_ref_numbers(term)[2], HCONST_8);
                 goto pop_next;
@@ -1847,7 +1847,7 @@ make_internal_hash(Eterm term)
             {
                 ExternalThing* thing = external_thing_ptr(term);
 
-                ASSERT(external_thing_ref_no_of_numbers(thing) == 3);
+                ASSERT(external_thing_ref_no_numbers(thing) == 3);
                 /* See limitation #2 */
             #ifdef ARCH_64
                 POINTER_HASH(thing->node, HCONST_7);
@@ -2486,22 +2486,20 @@ tailrecur_ne:
 
 		anum = external_thing_ref_numbers(athing);
 		bnum = external_thing_ref_numbers(bthing);
-		alen = external_thing_ref_no_of_numbers(athing);
-		blen = external_thing_ref_no_of_numbers(bthing);
+		alen = external_thing_ref_no_numbers(athing);
+		blen = external_thing_ref_no_numbers(bthing);
 
 		goto ref_common;
+
 	    case REF_SUBTAG:
-		    if (!is_internal_ref(b))
-			goto not_equal;
 
-		    {
-			RefThing* athing = ref_thing_ptr(a);
-			RefThing* bthing = ref_thing_ptr(b);
-			alen = internal_thing_ref_no_of_numbers(athing);
-			blen = internal_thing_ref_no_of_numbers(bthing);
-			anum = internal_thing_ref_numbers(athing);
-			bnum = internal_thing_ref_numbers(bthing);
-		    }
+		if (!is_internal_ref(b))
+		    goto not_equal;
+
+		alen = internal_ref_no_numbers(a);
+		anum = internal_ref_numbers(a);
+		blen = internal_ref_no_numbers(b);
+		bnum = internal_ref_numbers(b);
 
 	    ref_common:
 		    ASSERT(alen > 0 && blen > 0);
@@ -3133,25 +3131,21 @@ tailrecur_ne:
 		 */
 
 		if (is_internal_ref(b)) {
-		    RefThing* bthing = ref_thing_ptr(b);
 		    bnode = erts_this_node;
-		    bnum = internal_thing_ref_numbers(bthing);
-		    blen = internal_thing_ref_no_of_numbers(bthing);
+		    blen = internal_ref_no_numbers(b);
+		    bnum = internal_ref_numbers(b);
 		} else if(is_external_ref(b)) {
 		    ExternalThing* bthing = external_thing_ptr(b);
 		    bnode = bthing->node;
 		    bnum = external_thing_ref_numbers(bthing);
-		    blen = external_thing_ref_no_of_numbers(bthing);
+		    blen = external_thing_ref_no_numbers(bthing);
 		} else {
 		    a_tag = REF_DEF;
 		    goto mixed_types;
 		}
-		{
-		    RefThing* athing = ref_thing_ptr(a);
-		    anode = erts_this_node;
-		    anum = internal_thing_ref_numbers(athing);
-		    alen = internal_thing_ref_no_of_numbers(athing);
-		}
+		anode = erts_this_node;
+		alen = internal_ref_no_numbers(a);
+		anum = internal_ref_numbers(a);
 
 	    ref_common:
 		CMP_NODES(anode, bnode);
@@ -3181,15 +3175,14 @@ tailrecur_ne:
 		goto pop_next;
 	    case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE):
 		if (is_internal_ref(b)) {
-		    RefThing* bthing = ref_thing_ptr(b);
 		    bnode = erts_this_node;
-		    bnum = internal_thing_ref_numbers(bthing);
-		    blen = internal_thing_ref_no_of_numbers(bthing);
+		    blen = internal_ref_no_numbers(b);
+		    bnum = internal_ref_numbers(b);
 		} else if (is_external_ref(b)) {
 		    ExternalThing* bthing = external_thing_ptr(b);
 		    bnode = bthing->node;
 		    bnum = external_thing_ref_numbers(bthing);
-		    blen = external_thing_ref_no_of_numbers(bthing);
+		    blen = external_thing_ref_no_numbers(bthing);
 		} else {
 		    a_tag = EXTERNAL_REF_DEF;
 		    goto mixed_types;
@@ -3198,7 +3191,7 @@ tailrecur_ne:
 		    ExternalThing* athing = external_thing_ptr(a);
 		    anode = athing->node;
 		    anum = external_thing_ref_numbers(athing);
-		    alen = external_thing_ref_no_of_numbers(athing);
+		    alen = external_thing_ref_no_numbers(athing);
 		}
 		goto ref_common;
 	    default:
@@ -3575,40 +3568,39 @@ not_equal:
 Eterm
 store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns)
 {
+    struct erl_off_heap_header *ohhp;
     Uint i;
     Uint size;
-    Uint *from_hp;
-    Uint *to_hp = *hpp;
+    Eterm *from_hp;
+    Eterm *to_hp = *hpp;
 
     ASSERT(is_external(ns) || is_internal_ref(ns));
 
-    if(is_external(ns)) {
-	from_hp = external_val(ns);
-	size = thing_arityval(*from_hp) + 1;
-	*hpp += size;
-
-	for(i = 0; i < size; i++)
-	    to_hp[i] = from_hp[i];
-
-	erts_smp_refc_inc(&((ExternalThing *) to_hp)->node->refc, 2);
-
-	((struct erl_off_heap_header*) to_hp)->next = oh->first;
-	oh->first = (struct erl_off_heap_header*) to_hp;
-
-	return make_external(to_hp);
-    }
-
-    /* Internal ref */
-    from_hp = internal_ref_val(ns);
-
+    from_hp = boxed_val(ns);
     size = thing_arityval(*from_hp) + 1;
-
     *hpp += size;
 
     for(i = 0; i < size; i++)
 	to_hp[i] = from_hp[i];
 
-    return make_internal_ref(to_hp);
+    if (is_external_header(*from_hp)) {
+	ExternalThing *etp = (ExternalThing *) from_hp;
+	ASSERT(is_external(ns));
+	erts_smp_refc_inc(&etp->node->refc, 2);
+    }
+    else if (is_ordinary_ref_thing(from_hp))
+	return make_internal_ref(to_hp);
+    else {
+	ErtsMRefThing *mreft = (ErtsMRefThing *) from_hp;
+	ASSERT(is_magic_ref_thing(from_hp));
+	erts_refc_inc(&mreft->mb->refc, 2);
+    }
+
+    ohhp = (struct erl_off_heap_header*) to_hp;
+    ohhp->next = oh->first;
+    oh->first = ohhp;
+
+    return make_boxed(to_hp);
 }
 
 Eterm
diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl
index af18545bff..7356f31b75 100644
--- a/erts/emulator/test/node_container_SUITE.erl
+++ b/erts/emulator/test/node_container_SUITE.erl
@@ -50,7 +50,8 @@
          port_wrap/1,
          bad_nc/1,
          unique_pid/1,
-         iter_max_procs/1]).
+         iter_max_procs/1,
+         magic_ref/1]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
@@ -62,7 +63,7 @@ all() ->
      node_table_gc, dist_link_refc, dist_monitor_refc,
      node_controller_refc, ets_refc, match_spec_refc,
      timer_refc, otp_4715, pid_wrap, port_wrap, bad_nc,
-     unique_pid, iter_max_procs].
+     unique_pid, iter_max_procs, magic_ref].
 
 init_per_suite(Config) ->
     Config.
@@ -889,10 +890,46 @@ chk_max_proc_line_until(NoMoreTests, Res) ->
               chk_max_proc_line_until(NoMoreTests, Res)
     end.
 
+magic_ref(Config) when is_list(Config) ->
+    {MRef0, Addr0} = erts_debug:set_internal_state(make, magic_ref),
+    true = is_reference(MRef0),
+    {Addr0, 1, true} = erts_debug:get_internal_state({magic_ref,MRef0}),
+    MRef1 = binary_to_term(term_to_binary(MRef0)),
+    {Addr0, 2, true} = erts_debug:get_internal_state({magic_ref,MRef1}),
+    MRef0 = MRef1,
+    Me = self(),
+    {Pid, Mon} = spawn_opt(fun () ->
+				   receive
+				       {Me, MRef} ->
+					   Me ! {self(), erts_debug:get_internal_state({magic_ref,MRef})}
+				   end
+			   end,
+			   [link, monitor]),
+    Pid ! {self(), MRef0},
+    receive
+	{Pid, Info} ->
+	    {Addr0, 3, true} = Info
+    end,
+    receive
+	{'DOWN', Mon, process, Pid, _} ->
+	    ok
+    end,
+    {Addr0, 2, true} = erts_debug:get_internal_state({magic_ref,MRef0}),
+    id(MRef0),
+    id(MRef1),
+    MRefExt = term_to_binary(erts_debug:set_internal_state(make, magic_ref)),
+    garbage_collect(),
+    {MRef2, _Addr2} = binary_to_term(MRefExt),
+    true = is_reference(MRef2),
+    true = erts_debug:get_internal_state({magic_ref,MRef2}),
+    ok.
 %%
 %% -- Internal utils ---------------------------------------------------------
 %%
 
+id(X) ->
+    X.
+
 -define(ND_REFS, erts_debug:get_internal_state(node_and_dist_references)).
 
 node_container_refc_check(Node) when is_atom(Node) ->
-- 
cgit v1.2.3


From 7fee9c57d89be05c980d1684329b26ac991bb26a Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 23 Jan 2017 17:16:28 +0100
Subject: Use magic refs for NIF resources

---
 erts/emulator/beam/erl_nif.c                  | 32 ++++++++-------
 erts/emulator/test/nif_SUITE.erl              |  9 +++--
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 56 ++++++++++++++++++---------
 3 files changed, 61 insertions(+), 36 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 20b00bd4ba..1de56ce0a1 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2209,34 +2209,40 @@ ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj)
 {
     ErlNifResource* resource = DATA_TO_RESOURCE(obj);
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
-    Eterm* hp = alloc_heap(env,PROC_BIN_SIZE);
-    return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary);
+    Eterm* hp = alloc_heap(env, ERTS_MAGIC_REF_THING_SIZE);
+    return erts_mk_magic_ref(&hp, &MSO(env->proc), &bin->binary);
 }
 
 ERL_NIF_TERM enif_make_resource_binary(ErlNifEnv* env, void* obj,
 				       const void* data, size_t size)
 {
-    Eterm bin = enif_make_resource(env, obj);
-    ProcBin* pb = (ProcBin*) binary_val(bin);
+    ErlNifResource* resource = DATA_TO_RESOURCE(obj);
+    ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
+    Eterm* hp = alloc_heap(env,PROC_BIN_SIZE);
+    Eterm ebin = erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary);
+    ProcBin* pb = (ProcBin*) binary_val(ebin);
     pb->bytes = (byte*) data;
     pb->size = size;
-    return bin;
+    return ebin;
 }
 
 int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type,
 		      void** objp)
 {
-    ProcBin* pb;
     Binary* mbin;
     ErlNifResource* resource;
-    if (!ERTS_TERM_IS_MAGIC_BINARY(term)) {
-	return 0;
+    if (is_internal_magic_ref(term))
+	mbin = erts_magic_ref2bin(term);
+    else {
+	ProcBin* pb;
+	if (!ERTS_TERM_IS_MAGIC_BINARY(term))
+	    return 0;
+	pb = (ProcBin*) binary_val(term);
+	/*if (pb->size != 0) {	
+	  return 0; / * Or should we allow "resource binaries" as handles? * /
+	  }*/
+	mbin = pb->val;
     }
-    pb = (ProcBin*) binary_val(term);
-    /*if (pb->size != 0) {	
-	return 0; / * Or should we allow "resource binaries" as handles? * /
-    }*/
-    mbin = pb->val;
     resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin);
     if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor
 	|| resource->type != type) {	
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 36d512e388..8959be8690 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -748,7 +748,7 @@ resource_hugo_do(Type) ->
     HugoBin = <<"Hugo Hacker">>,
     HugoPtr = alloc_resource(Type, HugoBin),
     Hugo = make_resource(HugoPtr),
-    <<>> = Hugo,
+    true = is_reference(Hugo),
     release_resource(HugoPtr),
     erlang:garbage_collect(),
     {HugoPtr,HugoBin} = get_resource(Type,Hugo),
@@ -782,7 +782,7 @@ resource_otto_do(Type) ->
     OttoBin = <<"Otto Ordonnans">>,
     OttoPtr = alloc_resource(Type, OttoBin),
     Otto = make_resource(OttoPtr),
-    <<>> = Otto,
+    true = is_reference(Otto),
     %% forget resource term but keep referenced by NIF
     {OttoPtr, OttoBin}.    
 
@@ -805,8 +805,9 @@ resource_new_do2(Type) ->
     BinB = <<"NewB">>,
     ResA = make_new_resource(Type, BinA),
     ResB = make_new_resource(Type, BinB),
-    <<>> = ResA,
-    <<>> = ResB,
+    true = is_reference(ResA),
+    true = is_reference(ResB),
+    ResA /= ResB,
     {PtrA,BinA} = get_resource(Type, ResA),
     {PtrB,BinB} = get_resource(Type, ResB),
     true = (PtrA =/= PtrB),
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 4decb7f418..7331c5b2cd 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -1027,6 +1027,7 @@ struct make_term_info
     unsigned reuse_push;
     unsigned reuse_pull;
     ErlNifResourceType* resource_type;
+    void *resource;
     ERL_NIF_TERM other_term;
     ERL_NIF_TERM blob;
     ErlNifPid to_pid;
@@ -1139,12 +1140,7 @@ static ERL_NIF_TERM make_term_list0(struct make_term_info* mti, int n)
 }
 static ERL_NIF_TERM make_term_resource(struct make_term_info* mti, int n)
 {
-    void* resource = enif_alloc_resource(mti->resource_type, 10);
-    ERL_NIF_TERM term;
-    fill(resource, 10, n); 
-    term = enif_make_resource(mti->dst_env, resource);
-    enif_release_resource(resource);
-    return term;
+    return enif_make_resource(mti->dst_env, mti->resource);
 }
 static ERL_NIF_TERM make_term_new_binary(struct make_term_info* mti, int n)
 {
@@ -1249,23 +1245,39 @@ static int make_term_n(struct make_term_info* mti, int n, ERL_NIF_TERM* res)
     return 0;
 }
 
-static ERL_NIF_TERM make_blob(ErlNifEnv* caller_env, ErlNifEnv* dst_env,
-			      ERL_NIF_TERM other_term)
+
+static void
+init_make_blob(struct make_term_info *mti,
+	       ErlNifEnv* caller_env,
+	       ERL_NIF_TERM other_term)
 {
     PrivData* priv = (PrivData*) enif_priv_data(caller_env);
+    mti->caller_env = caller_env;
+    mti->resource_type = priv->rt_arr[0].t;
+    mti->resource = enif_alloc_resource(mti->resource_type, 10);
+    fill(mti->resource, 10, 17); 
+    mti->other_term = other_term;
+}
+
+static void
+fini_make_blob(struct make_term_info *mti)
+{
+    enif_release_resource(mti->resource);
+}
+
+static ERL_NIF_TERM make_blob(struct make_term_info *mti,
+			      ErlNifEnv* dst_env)
+{
     ERL_NIF_TERM term, list;
     int n = 0;
-    struct make_term_info mti;
-    mti.caller_env = caller_env;
-    mti.dst_env = dst_env;
-    mti.dst_env_valid = 1;
-    mti.reuse_push = 0;
-    mti.reuse_pull = 0;
-    mti.resource_type = priv->rt_arr[0].t;
-    mti.other_term = other_term;
+
+    mti->reuse_push = 0;
+    mti->reuse_pull = 0;
+    mti->dst_env = dst_env;
+    mti->dst_env_valid = 1;
 
     list = enif_make_list(dst_env, 0); 
-    while (make_term_n(&mti, n++, &term)) {
+    while (make_term_n(mti, n++, &term)) {
 	list = enif_make_list_cell(dst_env, term, list);
     }
     return list;
@@ -1277,13 +1289,16 @@ static ERL_NIF_TERM send_new_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM a
     ERL_NIF_TERM msg, copy;
     ErlNifEnv* msg_env;
     int res;
+    struct make_term_info mti;
     
     if (!enif_get_local_pid(env, argv[0], &to)) {
 	return enif_make_badarg(env);
     }
     msg_env = enif_alloc_env();
-    msg = make_blob(env,msg_env, argv[1]);
-    copy = make_blob(env,env, argv[1]);
+    init_make_blob(&mti, env, argv[1]);
+    msg = make_blob(&mti,msg_env);
+    copy = make_blob(&mti,env);
+    fini_make_blob(&mti);
     res = enif_send(env, &to, msg_env, msg);
     enif_free_env(msg_env);
     return enif_make_tuple3(env, atom_ok, enif_make_int(env,res), copy);
@@ -1303,6 +1318,8 @@ static ERL_NIF_TERM alloc_msgenv(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
     mti->reuse_push = 0;
     mti->reuse_pull = 0;
     mti->resource_type = priv->rt_arr[0].t;
+    mti->resource = enif_alloc_resource(mti->resource_type, 10);
+    fill(mti->resource, 10, 17); 
     mti->other_term = enif_make_list(mti->dst_env, 0);
     mti->blob = enif_make_list(mti->dst_env, 0);
     mti->mtx = enif_mutex_create("nif_SUITE:mtx");
@@ -1320,6 +1337,7 @@ static void msgenv_dtor(ErlNifEnv* env, void* obj)
     if (mti->dst_env != NULL) {
 	enif_free_env(mti->dst_env);
     }
+    enif_release_resource(mti->resource);
     enif_mutex_destroy(mti->mtx);
     enif_cond_destroy(mti->cond);
 }
-- 
cgit v1.2.3


From 7c70239985c4591ef2770a59a1bf62b51f5108cc Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 23 Jan 2017 17:06:42 +0100
Subject: Use magic refs in trapping processes()/ports() BIFs

---
 erts/emulator/beam/erl_ptab.c | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index 22830a19c4..a132b1d334 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -771,18 +771,18 @@ erts_ptab_list(Process *c_p, ErtsPTab *ptab)
     }
     else {
 	Eterm *hp;
-	Eterm magic_bin;
+	Eterm magic_ref;
 	ERTS_PTAB_LIST_DBG_CHK_RESLIST(res_acc);
-	hp = HAlloc(c_p, PROC_BIN_SIZE);
-	ERTS_PTAB_LIST_DBG_SAVE_HEAP_ALLOC(ptlbdp, hp, PROC_BIN_SIZE);
-	magic_bin = erts_mk_magic_binary_term(&hp, &MSO(c_p), mbp);
+	hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE);
+	ERTS_PTAB_LIST_DBG_SAVE_HEAP_ALLOC(ptlbdp, hp, ERTS_MAGIC_REF_THING_SIZE);
+	magic_ref = erts_mk_magic_ref(&hp, &MSO(c_p), mbp);
 	ERTS_PTAB_LIST_DBG_VERIFY_HEAP_ALLOC_USED(ptlbdp, hp);
 	ERTS_PTAB_LIST_DBG_TRACE(c_p->common.id, trap);
 	ERTS_BIF_PREP_YIELD2(ret_val,
 			     &ptab_list_continue_export,
 			     c_p,
 			     res_acc,
-			     magic_bin);
+			     magic_ref);
     }
     return ret_val;
 }
@@ -1287,9 +1287,7 @@ static BIF_RETTYPE ptab_list_continue(BIF_ALIST_2)
 
     res_acc = BIF_ARG_1;
 
-    ERTS_PTAB_LIST_ASSERT(ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_2));
-
-    mbp = ((ProcBin *) binary_val(BIF_ARG_2))->val;
+    mbp = erts_magic_ref2bin(BIF_ARG_2);
 
     ERTS_PTAB_LIST_ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp)
 			  == cleanup_ptab_list_bif_data);
-- 
cgit v1.2.3


From 5a97997217e5c3f901e8fefbd7bbf6c64652c9a8 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 23 Jan 2017 17:07:23 +0100
Subject: Use magic refs for code loading state

---
 erts/emulator/beam/beam_bif_load.c | 26 ++++++++++++--------------
 erts/emulator/beam/beam_load.c     |  8 ++++----
 erts/emulator/hipe/hipe_bif0.c     | 11 ++++-------
 3 files changed, 20 insertions(+), 25 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index a5891a0ec2..4ba8c2a669 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -156,11 +156,13 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
     Module* modp;
     Eterm res, mod;
 
-    if (!ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_1) ||
-	is_not_atom(mod = erts_module_for_prepared_code
-		    (((ProcBin*)binary_val(BIF_ARG_1))->val))) {
+    if (!is_internal_magic_ref(BIF_ARG_1))
+	BIF_ERROR(BIF_P, BADARG);
+
+    mod = erts_module_for_prepared_code(erts_magic_ref2bin(BIF_ARG_1));
+
+    if (is_not_atom(mod))
 	BIF_ERROR(BIF_P, BADARG);
-    }
 
     if (!erts_try_seize_code_write_permission(BIF_P)) {
 	ERTS_BIF_YIELD3(bif_export[BIF_code_make_stub_module_3],
@@ -229,8 +231,8 @@ prepare_loading_2(BIF_ALIST_2)
 	res = TUPLE2(hp, am_error, reason);
 	BIF_RET(res);
     }
-    hp = HAlloc(BIF_P, PROC_BIN_SIZE);
-    res = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), magic);
+    hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
+    res = erts_mk_magic_ref(&hp, &MSO(BIF_P), magic);
     erts_refc_dec(&magic->refc, 1);
     BIF_RET(res);
 }
@@ -239,15 +241,13 @@ BIF_RETTYPE
 has_prepared_code_on_load_1(BIF_ALIST_1)
 {
     Eterm res;
-    ProcBin* pb;
 
-    if (!ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_1)) {
+    if (!is_internal_magic_ref(BIF_ARG_1)) {
     error:
 	BIF_ERROR(BIF_P, BADARG);
     }
 
-    pb = (ProcBin*) binary_val(BIF_ARG_1);
-    res = erts_has_code_on_load(pb->val);
+    res = erts_has_code_on_load(erts_magic_ref2bin(BIF_ARG_1));
     if (res == NIL) {
 	goto error;
     }
@@ -333,13 +333,11 @@ finish_loading_1(BIF_ALIST_1)
     for (i = 0; i < n; i++) {
 	Eterm* cons = list_val(BIF_ARG_1);
 	Eterm term = CAR(cons);
-	ProcBin* pb;
 
-	if (!ERTS_TERM_IS_MAGIC_BINARY(term)) {
+	if (!is_internal_magic_ref(term)) {
 	    goto badarg;
 	}
-	pb = (ProcBin*) binary_val(term);
-	p[i].code = pb->val;
+	p[i].code = erts_magic_ref2bin(term);
 	p[i].module = erts_module_for_prepared_code(p[i].code);
 	if (p[i].module == NIL) {
 	    goto badarg;
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 9f38dd4c3a..e75e7afd54 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -6322,8 +6322,8 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
     stp = ERTS_MAGIC_BIN_DATA(magic);
     hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code));
 
-    if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) ||
-	!(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val,
+    if (!is_internal_magic_ref(hipe_magic_bin) ||
+	!(hipe_magic = erts_magic_ref2bin(hipe_magic_bin),
 	  hipe_stp = hipe_get_loader_state(hipe_magic)) ||
 	hipe_stp->module == NIL || hipe_stp->text_segment == 0) {
 	goto error;
@@ -6568,8 +6568,8 @@ int erts_commit_hipe_patch_load(Eterm hipe_magic_bin)
     HipeModule *hipe_code;
     Module* modp;
 
-    if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) ||
-	!(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val,
+    if (!is_internal_magic_ref(hipe_magic_bin) ||
+	!(hipe_magic = erts_magic_ref2bin(hipe_magic_bin),
 	  hipe_stp = hipe_get_loader_state(hipe_magic)) ||
 	hipe_stp->module == NIL || hipe_stp->text_segment == 0) {
 	return 0;
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 3011860e51..688c82ab7a 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -381,12 +381,9 @@ BIF_RETTYPE hipe_bifs_ref_set_2(BIF_ALIST_2)
 
 static HipeLoaderState *get_loader_state(Eterm term)
 {
-    ProcBin *pb;
+    if (!is_internal_magic_ref(term)) return NULL;
 
-    if (!ERTS_TERM_IS_MAGIC_BINARY(term)) return NULL;
-
-    pb = (ProcBin*) binary_val(term);
-    return hipe_get_loader_state(pb->val);
+    return hipe_get_loader_state(erts_magic_ref2bin(term));
 }
 
 
@@ -1982,8 +1979,8 @@ BIF_RETTYPE hipe_bifs_alloc_loader_state_1(BIF_ALIST_1)
     if (!magic)
 	BIF_ERROR(BIF_P, BADARG);
 
-    hp = HAlloc(BIF_P, PROC_BIN_SIZE);
-    res = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), magic);
+    hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
+    res = erts_mk_magic_ref(&hp, &MSO(BIF_P), magic);
     erts_refc_dec(&magic->refc, 1);
     BIF_RET(res);
 }
-- 
cgit v1.2.3


From 8d4dd97bcbd0988b08f8f8141ec7cfb17a16aa4a Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 23 Jan 2017 18:44:39 +0100
Subject: Use magic refs for re:run() static NIFs trap

---
 erts/emulator/beam/erl_bif_re.c | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index ff7746ce1d..ba183f24e8 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -1319,17 +1319,17 @@ handle_iolist:
 	Binary *mbp = erts_create_magic_binary(sizeof(RestartContext),
 					       cleanup_restart_context_bin);
 	RestartContext *restartp = ERTS_MAGIC_BIN_DATA(mbp);
-	Eterm magic_bin;
+	Eterm magic_ref;
 	Eterm *hp;
 	memcpy(restartp,&restart,sizeof(RestartContext));
 	BUMP_ALL_REDS(p);
-	hp = HAlloc(p, PROC_BIN_SIZE);
-	magic_bin = erts_mk_magic_binary_term(&hp, &MSO(p), mbp);
+	hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
+	magic_ref = erts_mk_magic_ref(&hp, &MSO(p), mbp);
 	BIF_TRAP3(&re_exec_trap_export, 
 		  p,
 		  arg1,
 		  arg2 /* To avoid GC of precompiled code, XXX: not utilized yet */,
-		  magic_bin);
+		  magic_ref);
     }
 
     res = build_exec_return(p, rc, &restart, arg1);
@@ -1366,9 +1366,7 @@ static BIF_RETTYPE re_exec_trap(BIF_ALIST_3)
     Uint loop_limit_tmp;
     Eterm res;
 
-    ASSERT(ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_3));
-
-    mbp = ((ProcBin *) binary_val(BIF_ARG_3))->val;
+    mbp = erts_magic_ref2bin(BIF_ARG_3);
 
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp)
 	   == cleanup_restart_context_bin);
-- 
cgit v1.2.3


From 2a78349342b9f72651c016b650321bb317098a3c Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 23 Jan 2017 21:26:22 +0100
Subject: Use magic refs for compiled match specs

---
 erts/emulator/beam/erl_db.c          | 18 +++++----------
 erts/emulator/beam/erl_db_hash.c     | 32 +++++++++++++--------------
 erts/emulator/beam/erl_db_tree.c     | 40 +++++++++++++++------------------
 erts/emulator/beam/erl_db_util.c     | 15 -------------
 erts/emulator/beam/erl_db_util.h     | 43 ++++++++++++++++++++++++++++++++----
 erts/emulator/beam/erl_node_tables.c |  8 +++----
 6 files changed, 82 insertions(+), 74 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 6a915a88a1..9f077dd407 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -2779,7 +2779,7 @@ BIF_RETTYPE ets_info_2(BIF_ALIST_2)
 
 BIF_RETTYPE ets_is_compiled_ms_1(BIF_ALIST_1)
 {
-    if (erts_db_is_compiled_ms(BIF_ARG_1)) {
+    if (erts_db_get_match_prog_binary(BIF_ARG_1)) {
 	BIF_RET(am_true);
     } else {
 	BIF_RET(am_false);
@@ -2794,9 +2794,9 @@ BIF_RETTYPE ets_match_spec_compile_1(BIF_ALIST_1)
 	BIF_ERROR(BIF_P, BADARG);
     }
 
-    hp = HAlloc(BIF_P, PROC_BIN_SIZE);
+    hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
 
-    BIF_RET(erts_mk_magic_binary_term(&hp, &MSO(BIF_P), mp));
+    BIF_RET(erts_db_make_match_prog_ref(BIF_P, mp, &hp));
 }
 
 BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3)
@@ -2805,24 +2805,18 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3)
     int i = 0;
     Eterm *hp;
     Eterm lst;
-    ProcBin *bp;
     Binary *mp;
     Eterm res;
     Uint32 dummy;
 
-    if (!(is_list(BIF_ARG_1) || BIF_ARG_1 == NIL) || !is_binary(BIF_ARG_2)) {
+    if (!(is_list(BIF_ARG_1) || BIF_ARG_1 == NIL)) {
     error:
 	BIF_ERROR(BIF_P, BADARG);
     }
     
-    bp = (ProcBin*) binary_val(BIF_ARG_2);
-    if (thing_subtag(bp->thing_word) != REFC_BINARY_SUBTAG) {
+    mp = erts_db_get_match_prog_binary(BIF_ARG_2);
+    if (!mp)
 	goto error;
-    }
-    mp = bp->val;
-    if (!IsMatchProgBinary(mp)) {
-	goto error;
-    }
 
     if (BIF_ARG_1 == NIL) {
 	BIF_RET(BIF_ARG_3);
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index b2a231467b..bb29d56aa7 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1287,15 +1287,13 @@ static int db_select_continue_hash(Process *p,
     if (arityval(*tptr) != 6)
 	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
     
-    if (!is_small(tptr[2]) || !is_small(tptr[3]) || !is_binary(tptr[4]) || 
+    if (!is_small(tptr[2]) || !is_small(tptr[3]) || 
 	!(is_list(tptr[5]) || tptr[5] == NIL) || !is_small(tptr[6]))
 	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
     if ((chunk_size = signed_val(tptr[3])) < 0)
 	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
-    if (!(thing_subtag(*binary_val(tptr[4])) == REFC_BINARY_SUBTAG))
-	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
-    mp = ((ProcBin *) binary_val(tptr[4]))->val;
-    if (!IsMatchProgBinary(mp))
+    mp = erts_db_get_match_prog_binary(tptr[4]);
+    if (!mp)
 	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
     all_objects = mp->flags & BIN_FLAG_ALL_OBJECTS;
     match_list = tptr[5];
@@ -1554,8 +1552,8 @@ done:
 					 been in 'user space' */ 
 	}
 	if (rest != NIL || slot_ix >= 0) { /* Need more calls */
-	    hp = HAlloc(p,3+7+PROC_BIN_SIZE);
-	    mpb =db_make_mp_binary(p,(mpi.mp),&hp);
+	    hp = HAlloc(p,3+7+ERTS_MAGIC_REF_THING_SIZE);
+	    mpb = erts_db_make_match_prog_ref(p,(mpi.mp),&hp);
 	    if (mpi.all_objects)
 		(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
 	    continuation = TUPLE6(hp, tb->common.id,make_small(slot_ix), 
@@ -1580,8 +1578,8 @@ trap:
     BUMP_ALL_REDS(p);
     if (mpi.all_objects)
 	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
-    hp = HAlloc(p,7+PROC_BIN_SIZE);
-    mpb =db_make_mp_binary(p,(mpi.mp),&hp);
+    hp = HAlloc(p,7+ERTS_MAGIC_REF_THING_SIZE);
+    mpb = erts_db_make_match_prog_ref(p,(mpi.mp),&hp);
     continuation = TUPLE6(hp, tb->common.id, make_small(slot_ix), 
 			  make_small(chunk_size), 
 			  mpb, match_list, 
@@ -1693,15 +1691,15 @@ done:
 trap:
     BUMP_ALL_REDS(p);
     if (IS_USMALL(0, got)) {
-	hp = HAlloc(p,  PROC_BIN_SIZE + 5);
+	hp = HAlloc(p,  ERTS_MAGIC_REF_THING_SIZE + 5);
 	egot = make_small(got);
     }
     else {
-	hp = HAlloc(p, BIG_UINT_HEAP_SIZE + PROC_BIN_SIZE + 5);
+	hp = HAlloc(p, BIG_UINT_HEAP_SIZE + ERTS_MAGIC_REF_THING_SIZE + 5);
 	egot = uint_to_big(got, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    mpb = db_make_mp_binary(p,mpi.mp,&hp);
+    mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
     continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix), 
 			  mpb, 
 			  egot);
@@ -1838,15 +1836,15 @@ done:
 trap:
     BUMP_ALL_REDS(p);
     if (IS_USMALL(0, got)) {
-	hp = HAlloc(p,  PROC_BIN_SIZE + 5);
+	hp = HAlloc(p,  ERTS_MAGIC_REF_THING_SIZE + 5);
 	egot = make_small(got);
     }
     else {
-	hp = HAlloc(p, BIG_UINT_HEAP_SIZE + PROC_BIN_SIZE + 5);
+	hp = HAlloc(p, BIG_UINT_HEAP_SIZE + ERTS_MAGIC_REF_THING_SIZE + 5);
 	egot = uint_to_big(got, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    mpb = db_make_mp_binary(p,mpi.mp,&hp);
+    mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
     continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix), 
 			  mpb, 
 			  egot);
@@ -1887,7 +1885,7 @@ static int db_select_delete_continue_hash(Process *p,
     
     tptr = tuple_val(continuation);
     slot_ix = unsigned_val(tptr[2]);
-    mp = ((ProcBin *) binary_val(tptr[3]))->val;
+    mp = erts_db_get_match_prog_binary_unchecked(tptr[3]);
     if (is_big(tptr[4])) {
 	got = big_to_uint32(tptr[4]);
     } else {
@@ -1999,7 +1997,7 @@ static int db_select_count_continue_hash(Process *p,
     
     tptr = tuple_val(continuation);
     slot_ix = unsigned_val(tptr[2]);
-    mp = ((ProcBin *) binary_val(tptr[3]))->val;
+    mp = erts_db_get_match_prog_binary_unchecked(tptr[3]);
     if (is_big(tptr[4])) {
 	got = big_to_uint32(tptr[4]);
     } else {
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index dd9403e132..c4ecd2ba37 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -935,17 +935,15 @@ static int db_select_continue_tree(Process *p,
     if (arityval(*tptr) != 8)
 	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
     
-    if (!is_small(tptr[4]) || !is_binary(tptr[5]) || 
+    if (!is_small(tptr[4]) ||
 	!(is_list(tptr[6]) || tptr[6] == NIL) || !is_small(tptr[7]) ||
 	!is_small(tptr[8]))
 	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
     
     lastkey = tptr[2];
     end_condition = tptr[3];
-    if (!(thing_subtag(*binary_val(tptr[5])) == REFC_BINARY_SUBTAG))
-	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
-    mp = ((ProcBin *) binary_val(tptr[5]))->val;
-    if (!IsMatchProgBinary(mp))
+    mp = erts_db_get_match_prog_binary(tptr[5]);
+    if (!mp)
 	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
     chunk_size = signed_val(tptr[4]);
 
@@ -1145,11 +1143,11 @@ static int db_select_tree(Process *p, DbTable *tbl,
 
     key = GETKEY(tb, sc.lastobj);
     sz = size_object(key);
-    hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE);
+    hp = HAlloc(p, 9 + sz + ERTS_MAGIC_REF_THING_SIZE);
     key = copy_struct(key, sz, &hp, &MSO(p));
     if (mpi.all_objects)
 	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
-    mpb=db_make_mp_binary(p,mpi.mp,&hp);
+    mpb= erts_db_make_match_prog_ref(p,mpi.mp,&hp);
 	    
     continuation = TUPLE8
 	(hp,
@@ -1208,10 +1206,8 @@ static int db_select_count_continue_tree(Process *p,
     
     lastkey = tptr[2];
     end_condition = tptr[3];
-    if (!(thing_subtag(*binary_val(tptr[4])) == REFC_BINARY_SUBTAG))
-	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
-    mp = ((ProcBin *) binary_val(tptr[4]))->val;
-    if (!IsMatchProgBinary(mp))
+    mp = erts_db_get_match_prog_binary(tptr[4]);
+    if (!mp)
 	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
 
     sc.p = p;
@@ -1338,18 +1334,18 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
     key = GETKEY(tb, sc.lastobj);
     sz = size_object(key);
     if (IS_USMALL(0, sc.got)) {
-	hp = HAlloc(p, sz + PROC_BIN_SIZE + 6);
+	hp = HAlloc(p, sz + ERTS_MAGIC_REF_THING_SIZE + 6);
 	egot = make_small(sc.got);
     }
     else {
-	hp = HAlloc(p, BIG_UINT_HEAP_SIZE + sz + PROC_BIN_SIZE + 6);
+	hp = HAlloc(p, BIG_UINT_HEAP_SIZE + sz + ERTS_MAGIC_REF_THING_SIZE + 6);
 	egot = uint_to_big(sc.got, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
     key = copy_struct(key, sz, &hp, &MSO(p));
     if (mpi.all_objects)
 	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
-    mpb = db_make_mp_binary(p,mpi.mp,&hp);
+    mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
 	    
     continuation = TUPLE5
 	(hp,
@@ -1470,11 +1466,11 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
 
 	key = GETKEY(tb, sc.lastobj);
 	sz = size_object(key);
-	hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE);
+	hp = HAlloc(p, 9 + sz + ERTS_MAGIC_REF_THING_SIZE);
 	key = copy_struct(key, sz, &hp, &MSO(p));
 	if (mpi.all_objects)
 	    (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
-	mpb = db_make_mp_binary(p,mpi.mp,&hp);
+	mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
 	
 	continuation = TUPLE8
 	    (hp,
@@ -1495,12 +1491,12 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
 
     key = GETKEY(tb, sc.lastobj);
     sz = size_object(key);
-    hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE);
+    hp = HAlloc(p, 9 + sz + ERTS_MAGIC_REF_THING_SIZE);
     key = copy_struct(key, sz, &hp, &MSO(p));
 
     if (mpi.all_objects)
 	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
-    mpb = db_make_mp_binary(p,mpi.mp,&hp);    
+    mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
     continuation = TUPLE8
 	(hp,
 	 tb->common.id,
@@ -1558,7 +1554,7 @@ static int db_select_delete_continue_tree(Process *p,
     sc.erase_lastterm = 0; /* Before first RET_TO_BIF */
     sc.lastterm = NULL;
 
-    mp = ((ProcBin *) binary_val(tptr[4]))->val;
+    mp = erts_db_get_match_prog_binary_unchecked(tptr[4]);
     sc.p = p;
     sc.tb = tb;
     if (is_big(tptr[5])) {
@@ -1682,16 +1678,16 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
     key = GETKEY(tb, (sc.lastterm)->dbterm.tpl);
     sz = size_object(key);
     if (IS_USMALL(0, sc.accum)) {
-	hp = HAlloc(p, sz + PROC_BIN_SIZE + 6);
+	hp = HAlloc(p, sz + ERTS_MAGIC_REF_THING_SIZE + 6);
 	eaccsum = make_small(sc.accum);
     }
     else {
-	hp = HAlloc(p, BIG_UINT_HEAP_SIZE + sz + PROC_BIN_SIZE + 6);
+	hp = HAlloc(p, BIG_UINT_HEAP_SIZE + sz + ERTS_MAGIC_REF_THING_SIZE + 6);
 	eaccsum = uint_to_big(sc.accum, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
     key = copy_struct(key, sz, &hp, &MSO(p));
-    mpb = db_make_mp_binary(p,mpi.mp,&hp);
+    mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
     
     continuation = TUPLE5
 	(hp,
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 0ab42394ce..070e29578f 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2545,14 +2545,6 @@ success:
 }
 
 
-/*
- * Convert a match program to a "magic" binary to return up to erlang
- */
-Eterm db_make_mp_binary(Process *p, Binary *mp, Eterm **hpp)
-{
-    return erts_mk_magic_binary_term(hpp, &MSO(p), mp);
-}
-
 DMCErrInfo *db_new_dmc_err_info(void) 
 {
     DMCErrInfo *ret = erts_alloc(ERTS_ALC_T_DB_DMC_ERR_INFO,
@@ -3266,13 +3258,6 @@ int db_has_variable(Eterm node) {
     return 0;
 }
 
-int erts_db_is_compiled_ms(Eterm term)
-{
-    return (is_binary(term)
-	    && (thing_subtag(*binary_val(term)) == REFC_BINARY_SUBTAG)
-	    && IsMatchProgBinary((((ProcBin *) binary_val(term))->val)));
-}
-
 /* 
 ** Local (static) utilities.
 */
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 3fb063797e..b2a3bb6c20 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -23,6 +23,7 @@
 
 #include "global.h"
 #include "erl_message.h"
+#include "erl_bif_unique.h"
 
 /*#define HARDDEBUG 1*/
 
@@ -456,10 +457,44 @@ Eterm db_format_dmc_err_info(Process *p, DMCErrInfo *ei);
 void db_free_dmc_err_info(DMCErrInfo *ei);
 /* Completely free's an error info structure, including all recorded 
    errors */
-Eterm db_make_mp_binary(Process *p, Binary *mp, Eterm **hpp);
-/* Convert a match program to a erlang "magic" binary to be returned to userspace,
-   increments the reference counter. */
-int erts_db_is_compiled_ms(Eterm term);
+
+ERTS_GLB_INLINE Eterm erts_db_make_match_prog_ref(Process *p, Binary *mp, Eterm **hpp);
+ERTS_GLB_INLINE Binary *erts_db_get_match_prog_binary(Eterm term);
+ERTS_GLB_INLINE Binary *erts_db_get_match_prog_binary_unchecked(Eterm term);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+/*
+ * Convert a match program to a "magic" ref to return up to erlang
+ */
+ERTS_GLB_INLINE Eterm erts_db_make_match_prog_ref(Process *p, Binary *mp, Eterm **hpp)
+{
+    return erts_mk_magic_ref(hpp, &MSO(p), mp);
+}
+
+ERTS_GLB_INLINE Binary *
+erts_db_get_match_prog_binary_unchecked(Eterm term)
+{
+    Binary *bp = erts_magic_ref2bin(term);
+    ASSERT(bp->flags & BIN_FLAG_MAGIC);
+    ASSERT((ERTS_MAGIC_BIN_DESTRUCTOR(bp) == erts_db_match_prog_destructor));
+    return bp;
+}
+
+ERTS_GLB_INLINE Binary *
+erts_db_get_match_prog_binary(Eterm term)
+{
+    Binary *bp;
+    if (!is_internal_magic_ref(term))
+	return NULL;
+    bp = erts_magic_ref2bin(term);
+    ASSERT(bp->flags & BIN_FLAG_MAGIC);
+    if (ERTS_MAGIC_BIN_DESTRUCTOR(bp) != erts_db_match_prog_destructor)
+	return NULL;
+    return bp;
+}
+
+#endif
 
 /*
 ** Convenience when compiling into Binary structures
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 060fbd5883..5e54e5aef2 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1131,7 +1131,7 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id)
 		InsertedBin *ib;
 		int insert_bin = 1;
 		for (ib = inserted_bins; ib; ib = ib->next)
-		    if(ib->bin_val == u.pb->val) {
+		    if(ib->bin_val == (Binary *) u.mref->mb) {
 			insert_bin = 0;
 			break;
 		    }
@@ -1140,12 +1140,12 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id)
 		    Uint *hp = &id_heap[0];
 		    InsertedBin *nib;
 		    UseTmpHeapNoproc(BIG_UINT_HEAP_SIZE);
-		    a.id = erts_bld_uint(&hp, NULL, (Uint) u.pb->val);
-		    erts_match_prog_foreach_offheap(u.pb->val,
+		    a.id = erts_bld_uint(&hp, NULL, (Uint) u.mref->mb);
+		    erts_match_prog_foreach_offheap((Binary *) u.mref->mb,
 						    insert_offheap2,
 						    (void *) &a);
 		    nib = erts_alloc(ERTS_ALC_T_NC_TMP, sizeof(InsertedBin));
-		    nib->bin_val = u.pb->val;
+		    nib->bin_val = (Binary *) u.mref->mb;
 		    nib->next = inserted_bins;
 		    inserted_bins = nib;
 		    UnUseTmpHeapNoproc(BIG_UINT_HEAP_SIZE);
-- 
cgit v1.2.3


From 94df64424e41952d340c0082c70f79e6b6dbabd3 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 24 Jan 2017 15:07:09 +0100
Subject: Use magic refs for list_to_binary/binary_to_list traps

---
 erts/emulator/beam/binary.c | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index a09950ec40..d38097fb12 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -440,18 +440,17 @@ binary_to_list(Process *c_p, Eterm *hp, Eterm tail, byte *bytes,
 	sp->size = size;
 	sp->bitoffs = bitoffs;
 
-	hp = HAlloc(c_p, PROC_BIN_SIZE);
-	mb = erts_mk_magic_binary_term(&hp, &MSO(c_p), mbp);
+	hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE);
+	mb = erts_mk_magic_ref(&hp, &MSO(c_p), mbp);
 	return binary_to_list_chunk(c_p, mb, sp, reds_left, 0);
     }
 }
 
 static BIF_RETTYPE binary_to_list_continue(BIF_ALIST_1)
 {
-    Binary *mbp = ((ProcBin *) binary_val(BIF_ARG_1))->val;
+    Binary *mbp = erts_magic_ref2bin(BIF_ARG_1);
 
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == b2l_state_destructor);
-
     ASSERT(BIF_P->flags & F_DISABLE_GC);
 
     return binary_to_list_chunk(BIF_P,
@@ -787,8 +786,8 @@ list_to_binary_chunk(Eterm mb_eterm,
 	    ERTS_L2B_STATE_MOVE(new_sp, sp);
 	    sp = new_sp;
 
-	    hp = HAlloc(c_p, PROC_BIN_SIZE);
-	    mb_eterm = erts_mk_magic_binary_term(&hp, &MSO(c_p), mbp);
+	    hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE);
+	    mb_eterm = erts_mk_magic_ref(&hp, &MSO(c_p), mbp);
 
 	    ASSERT(is_value(mb_eterm));
 
@@ -834,9 +833,9 @@ list_to_binary_chunk(Eterm mb_eterm,
 
 static BIF_RETTYPE list_to_binary_continue(BIF_ALIST_1)
 {
-    Binary *mbp = ((ProcBin *) binary_val(BIF_ARG_1))->val;
-    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == l2b_state_destructor);
+    Binary *mbp = erts_magic_ref2bin(BIF_ARG_1);
 
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == l2b_state_destructor);
     ASSERT(BIF_P->flags & F_DISABLE_GC);
 
     return list_to_binary_chunk(BIF_ARG_1,
-- 
cgit v1.2.3


From 3bdb334a95e9dd49c1ada55b6f442099fdf7f72b Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 24 Jan 2017 15:10:55 +0100
Subject: Use magic refs for binary compile patterns

---
 erts/emulator/beam/erl_bif_binary.c | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 6e10980b6b..e57cd06cec 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -1010,8 +1010,8 @@ BIF_RETTYPE binary_compile_pattern_1(BIF_ALIST_1)
     if (do_binary_match_compile(BIF_ARG_1,&tag,&bin)) {
 	BIF_ERROR(BIF_P,BADARG);
     }
-    hp = HAlloc(BIF_P, PROC_BIN_SIZE+3);
-    ret = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin);
+    hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE+3);
+    ret = erts_mk_magic_ref(&hp, &MSO(BIF_P), bin);
     ret = TUPLE2(hp, tag, ret);
     BIF_RET(ret);
 }
@@ -1426,11 +1426,11 @@ binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Uint flags)
 	    goto badarg;
 	}
 	if (((tp[1] != am_bm) && (tp[1] != am_ac)) ||
-	    !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) {
+	    !is_internal_magic_ref(tp[2])) {
 	    goto badarg;
 	}
 	bfs.type = tp[1];
-	bin = ((ProcBin *) binary_val(tp[2]))->val;
+	bin = erts_magic_ref2bin(tp[2]);
 	if (bfs.type == am_bm &&
 	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) {
 	    goto badarg;
@@ -1448,8 +1448,8 @@ binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Uint flags)
     bfs.global_result = &do_match_global_result;
     runres = do_binary_find(p, arg1, &bfs, bin, NIL, &result);
     if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) {
-	Eterm *hp = HAlloc(p, PROC_BIN_SIZE);
-	bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin);
+	Eterm *hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
+	bin_term = erts_mk_magic_ref(&hp, &MSO(p), bin);
     } else if (bin_term == NIL) {
 	erts_bin_free(bin);
     }
@@ -1512,11 +1512,11 @@ binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 	    goto badarg;
 	}
 	if (((tp[1] != am_bm) && (tp[1] != am_ac)) ||
-	    !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) {
+	    !is_internal_magic_ref(tp[2])) {
 	    goto badarg;
 	}
 	bfs.type = tp[1];
-	bin = ((ProcBin *) binary_val(tp[2]))->val;
+	bin = erts_magic_ref2bin(tp[2]);
 	if (bfs.type == am_bm &&
 	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) {
 	    goto badarg;
@@ -1534,8 +1534,8 @@ binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
     bfs.global_result = &do_split_global_result;
     runres = do_binary_find(p, arg1, &bfs, bin, NIL, &result);
     if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) {
-	Eterm *hp = HAlloc(p, PROC_BIN_SIZE);
-	bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin);
+	Eterm *hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
+	bin_term = erts_mk_magic_ref(&hp, &MSO(p), bin);
     } else if (bin_term == NIL) {
 	erts_bin_free(bin);
     }
@@ -1767,7 +1767,8 @@ static BIF_RETTYPE binary_find_trap(BIF_ALIST_3)
 {
     int runres;
     Eterm result;
-    Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val;
+    Binary *bin = erts_magic_ref2bin(BIF_ARG_3);
+
     runres = do_binary_find(BIF_P, BIF_ARG_1, NULL, bin, BIF_ARG_2, &result);
     if (runres == DO_BIN_MATCH_OK) {
 	BIF_RET(result);
@@ -2186,8 +2187,8 @@ static BIF_RETTYPE do_longest_common(Process *p, Eterm list, int direction)
 		cd[i].type = CL_TYPE_HEAP;
 	    }
 	}
-	hp = HAlloc(p, PROC_BIN_SIZE);
-	bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), mb);
+	hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
+	bin_term = erts_mk_magic_ref(&hp, &MSO(p), mb);
 	BUMP_ALL_REDS(p);
 	BIF_TRAP3(trapper, p, bin_term, epos,list);
     }
@@ -2214,8 +2215,7 @@ static BIF_RETTYPE do_longest_common_trap(Process *p, Eterm bin_term, Eterm curr
 #else
     term_to_Uint(current_pos, &pos);
 #endif
-    ASSERT(ERTS_TERM_IS_MAGIC_BINARY(bin_term));
-    bin = ((ProcBin *) binary_val(bin_term))->val;
+    bin = erts_magic_ref2bin(bin_term);
     cd = (CommonData *) ERTS_MAGIC_BIN_DATA(bin);
     if (direction == DIRECTION_PREFIX) {
 	trapper = &binary_longest_prefix_trap_export;
@@ -2680,8 +2680,8 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en)
 	cbs->source_size = size;
 	cbs->result_pos = pos;
 	cbs->times_left = n-i;
-	hp = HAlloc(p,PROC_BIN_SIZE);
-	trap_term = erts_mk_magic_binary_term(&hp, &MSO(p), mb);
+	hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
+	trap_term = erts_mk_magic_ref(&hp, &MSO(p), mb);
 	BUMP_ALL_REDS(p);
 	BIF_TRAP2(&binary_copy_trap_export, p, bin, trap_term);
     } else {
@@ -2716,7 +2716,7 @@ BIF_RETTYPE binary_copy_trap(BIF_ALIST_2)
     Uint reds = get_reds(BIF_P, BINARY_COPY_LOOP_FACTOR);
     byte *t;
     Uint pos;
-    Binary *mb = ((ProcBin *) binary_val(BIF_ARG_2))->val;
+    Binary *mb = erts_magic_ref2bin(BIF_ARG_2);
     CopyBinState *cbs = (CopyBinState *) ERTS_MAGIC_BIN_DATA(mb);
     Uint opos;
 
-- 
cgit v1.2.3


From 40d55ced08579e9fc0ade28ba0810bf800690394 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 24 Jan 2017 15:11:28 +0100
Subject: Use magic refs for unicode static NIFs traps

---
 erts/emulator/beam/erl_unicode.c | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 8919898181..dd7c589c3d 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -129,13 +129,9 @@ static void cleanup_restart_context_bin(Binary *bp)
     cleanup_restart_context(rc);
 }
 
-static RestartContext *get_rc_from_bin(Eterm bin)
+static RestartContext *get_rc_from_bin(Eterm mref)
 {
-    Binary *mbp;
-    ASSERT(ERTS_TERM_IS_MAGIC_BINARY(bin));
-
-    mbp = ((ProcBin *) binary_val(bin))->val;
-
+    Binary *mbp = erts_magic_ref2bin(mref);
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp)
 	   == cleanup_restart_context_bin);
     return (RestartContext *) ERTS_MAGIC_BIN_DATA(mbp);    
@@ -148,8 +144,8 @@ static Eterm make_magic_bin_for_restart(Process *p, RestartContext *rc)
     RestartContext *restartp = ERTS_MAGIC_BIN_DATA(mbp);
     Eterm *hp;
     memcpy(restartp,rc,sizeof(RestartContext));
-    hp = HAlloc(p, PROC_BIN_SIZE);
-    return erts_mk_magic_binary_term(&hp, &MSO(p), mbp);
+    hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
+    return erts_mk_magic_ref(&hp, &MSO(p), mbp);
 }
 
 	
-- 
cgit v1.2.3


From 13f02f2bb6e1e3ce709abc9ff846b24331ada58d Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 24 Jan 2017 17:29:18 +0100
Subject: Use magic refs for distributed send trap context

---
 erts/emulator/beam/bif.c  | 2 +-
 erts/emulator/beam/dist.c | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index bda7b9224c..2190c47262 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2446,7 +2446,7 @@ BIF_RETTYPE send_2(BIF_ALIST_2)
 
 static BIF_RETTYPE dsend_continue_trap_1(BIF_ALIST_1)
 {
-    Binary* bin = ((ProcBin*) binary_val(BIF_ARG_1))->val;
+    Binary* bin = erts_magic_ref2bin(BIF_ARG_1);
     ErtsSendContext* ctx = (ErtsSendContext*) ERTS_MAGIC_BIN_DATA(bin);
     Sint initial_reds = (Sint) (ERTS_BIF_REDS_LEFT(BIF_P) * TERM_TO_BINARY_LOOP_FACTOR);
     int result;
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 89d91065c2..390f2a0db3 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -741,7 +741,7 @@ Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx)
     Binary* ctx_bin = erts_create_magic_binary(sizeof(struct exported_ctx),
 					       erts_dsend_context_dtor);
     struct exported_ctx* dst = ERTS_MAGIC_BIN_DATA(ctx_bin);
-    Eterm* hp = HAlloc(p, PROC_BIN_SIZE);
+    Eterm* hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
 
     sys_memcpy(&dst->ctx, ctx, sizeof(ErtsSendContext));
     ASSERT(ctx->dss.ctl == make_tuple(ctx->ctl_heap));
@@ -750,7 +750,7 @@ Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx)
 	sys_memcpy(&dst->acm, ctx->dss.acmp, sizeof(ErtsAtomCacheMap));
 	dst->ctx.dss.acmp = &dst->acm;
     }
-    return erts_mk_magic_binary_term(&hp, &MSO(p), ctx_bin);
+    return erts_mk_magic_ref(&hp, &MSO(p), ctx_bin);
 }
 
 
-- 
cgit v1.2.3


From cf0e60eb5b1a6a2fec388bc650235b9eded335ec Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 24 Jan 2017 17:30:03 +0100
Subject: Use magic refs binary_to_term/term_to_binary trap context

---
 erts/emulator/beam/external.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 64b17a3f57..60df7b8a08 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1079,7 +1079,7 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1)
     Eterm *tp = tuple_val(BIF_ARG_1);
     Eterm Term = tp[1];
     Eterm bt = tp[2];
-    Binary *bin = ((ProcBin *) binary_val(bt))->val;
+    Binary *bin = erts_magic_ref2bin(bt);
     Eterm res = erts_term_to_binary_int(BIF_P, Term, 0, 0,bin);
     if (is_tuple(res)) {
 	ASSERT(BIF_P->flags & F_DISABLE_GC);
@@ -1408,7 +1408,7 @@ static void b2t_context_destructor(Binary *context_bin)
 
 static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1)
 {
-    Binary *context_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val;
+    Binary *context_bin = erts_magic_ref2bin(BIF_ARG_1);
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor);
 
     return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin, NULL,
@@ -1442,8 +1442,8 @@ static B2TContext* b2t_export_context(Process* p, B2TContext* src)
     if (ctx->state >= B2TDecode && ctx->u.dc.next == &src->u.dc.res) {
         ctx->u.dc.next = &ctx->u.dc.res;
     }
-    hp = HAlloc(p, PROC_BIN_SIZE);
-    ctx->trap_bin = erts_mk_magic_binary_term(&hp, &MSO(p), context_b);
+    hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
+    ctx->trap_bin = erts_mk_magic_ref(&hp, &MSO(p), context_b);
     return ctx;
 }
 
@@ -1871,8 +1871,8 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 
 #define RETURN_STATE()							\
     do {								\
-	hp = HAlloc(p, PROC_BIN_SIZE+3);				\
-	c_term = erts_mk_magic_binary_term(&hp, &MSO(p), context_b);	\
+	hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE+3);                    \
+	c_term = erts_mk_magic_ref(&hp, &MSO(p), context_b);            \
 	res = TUPLE2(hp, Term, c_term);					\
 	BUMP_ALL_REDS(p);                                               \
 	return res;							\
-- 
cgit v1.2.3


From 3a34a18c2f318a4a9ea475ee80fe05bd64f05070 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 24 Jan 2017 17:31:01 +0100
Subject: Use magic refs for maps merge trap context

---
 erts/emulator/beam/erl_map.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 979a0040b0..990be8efb2 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -1197,7 +1197,7 @@ static void hashmap_merge_ctx_destructor(Binary* ctx_bin)
 }
 
 BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1) {
-    Binary* ctx_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val;
+    Binary* ctx_bin = erts_magic_ref2bin(BIF_ARG_1);
 
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == hashmap_merge_ctx_destructor);
 
@@ -1453,9 +1453,9 @@ trap:  /* Yield */
                                                  hashmap_merge_ctx_destructor);
         ctx = ERTS_MAGIC_BIN_DATA(ctx_b);
         sys_memcpy(ctx, &local_ctx, sizeof(HashmapMergeContext));
-        hp = HAlloc(p, PROC_BIN_SIZE);
+        hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
         ASSERT(ctx->trap_bin == THE_NON_VALUE);
-        ctx->trap_bin = erts_mk_magic_binary_term(&hp, &MSO(p), ctx_b);
+        ctx->trap_bin = erts_mk_magic_ref(&hp, &MSO(p), ctx_b);
 
         erts_set_gc_state(p, 0);
     }
-- 
cgit v1.2.3


From 28735b9c2c6390df593b05f300151addbd01e367 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 24 Jan 2017 17:31:53 +0100
Subject: Adjust the only usage of exposed magic binaries

---
 erts/emulator/beam/erl_nif.c | 40 ++++++++++++++++++++++++++++------------
 erts/emulator/beam/global.h  | 35 -----------------------------------
 2 files changed, 28 insertions(+), 47 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 1de56ce0a1..3674a29bf8 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2218,12 +2218,22 @@ ERL_NIF_TERM enif_make_resource_binary(ErlNifEnv* env, void* obj,
 {
     ErlNifResource* resource = DATA_TO_RESOURCE(obj);
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
+    ErlOffHeap *ohp = &MSO(env->proc);
     Eterm* hp = alloc_heap(env,PROC_BIN_SIZE);
-    Eterm ebin = erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary);
-    ProcBin* pb = (ProcBin*) binary_val(ebin);
-    pb->bytes = (byte*) data;
+    ProcBin* pb = (ProcBin *) hp;
+
+    pb->thing_word = HEADER_PROC_BIN;
     pb->size = size;
-    return ebin;
+    pb->next = ohp->first;
+    ohp->first = (struct erl_off_heap_header*) pb;
+    pb->val = &bin->binary;
+    pb->bytes = (byte*) data;
+    pb->flags = 0;
+
+    OH_OVERHEAD(ohp, size / sizeof(Eterm));
+    erts_refc_inc(&bin->binary.refc, 1);
+
+    return make_binary(hp);
 }
 
 int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type,
@@ -2234,14 +2244,20 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ
     if (is_internal_magic_ref(term))
 	mbin = erts_magic_ref2bin(term);
     else {
-	ProcBin* pb;
-	if (!ERTS_TERM_IS_MAGIC_BINARY(term))
-	    return 0;
-	pb = (ProcBin*) binary_val(term);
-	/*if (pb->size != 0) {	
-	  return 0; / * Or should we allow "resource binaries" as handles? * /
-	  }*/
-	mbin = pb->val;
+        Eterm *hp;
+        if (!is_binary(term))
+            return 0;
+        hp = binary_val(term);
+        if (thing_subtag(*hp) != REFC_BINARY_SUBTAG)
+            return 0;
+        /*
+        if (((ProcBin *) hp)->size != 0) {	
+            return 0; / * Or should we allow "resource binaries" as handles? * /
+        }
+        */
+        mbin = ((ProcBin *) hp)->val;
+        if (!(mbin->flags & BIN_FLAG_MAGIC))
+            return 0;
     }
     resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin);
     if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 187a948a7b..fff22fe9c1 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -237,41 +237,6 @@ typedef struct proc_bin {
  */
 #define PROC_BIN_SIZE (sizeof(ProcBin)/sizeof(Eterm))
 
-ERTS_GLB_INLINE Eterm erts_mk_magic_binary_term(Eterm **hpp,
-						ErlOffHeap *ohp,
-						Binary *mbp);
-
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-
-ERTS_GLB_INLINE Eterm
-erts_mk_magic_binary_term(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp)
-{
-    ProcBin *pb = (ProcBin *) *hpp;
-    *hpp += PROC_BIN_SIZE;
-
-    ASSERT(mbp->flags & BIN_FLAG_MAGIC);
-
-    pb->thing_word = HEADER_PROC_BIN;
-    pb->size = 0;
-    pb->next = ohp->first;
-    ohp->first = (struct erl_off_heap_header*) pb;
-    pb->val = mbp;
-    pb->bytes = (byte *) mbp->orig_bytes;
-    pb->flags = 0;
-
-    erts_refc_inc(&mbp->refc, 1);
-
-    return make_binary(pb);    
-}
-
-#endif
-
-#define ERTS_TERM_IS_MAGIC_BINARY(T) \
-  (is_binary((T)) \
-   && (thing_subtag(*binary_val((T))) == REFC_BINARY_SUBTAG) \
-   && (((ProcBin *) binary_val((T)))->val->flags & BIN_FLAG_MAGIC))
-
-
 union erl_off_heap_ptr {
     struct erl_off_heap_header* hdr;
     ProcBin *pb;
-- 
cgit v1.2.3


From 8a72a253e2eda59e510d953f0b6eb21b50e06d0e Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 24 Jan 2017 20:43:08 +0100
Subject: Add binary overhead for magic ref/binaries

---
 erts/emulator/beam/erl_bif_unique.h |  2 ++
 erts/emulator/beam/erl_gc.c         | 59 +++++++++++++++++++++++++++++--------
 erts/emulator/beam/external.c       |  2 ++
 erts/emulator/beam/utils.c          |  4 ++-
 4 files changed, 53 insertions(+), 14 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h
index 6cd6cc3e09..27c2a15a5e 100644
--- a/erts/emulator/beam/erl_bif_unique.h
+++ b/erts/emulator/beam/erl_bif_unique.h
@@ -182,6 +182,7 @@ erts_mk_magic_ref(Eterm **hpp, ErlOffHeap *ohp, Binary *bp)
     write_magic_ref_thing(hp, ohp, (ErtsMagicBinary *) bp);
     *hpp += ERTS_MAGIC_REF_THING_SIZE;
     erts_refc_inc(&bp->refc, 1);
+    OH_OVERHEAD(ohp, bp->orig_size / sizeof(Eterm));
     return make_internal_ref(hp);
 }
 
@@ -329,6 +330,7 @@ erts_iref_storage_make_ref(ErtsIRefStorage *iref,
     }
     else {
 	write_magic_ref_thing(hp, ohp, iref->u.mb);
+        OH_OVERHEAD(ohp, iref->u.mb->orig_size / sizeof(Eterm));
 	*hpp += ERTS_MAGIC_REF_THING_SIZE;
 	/*
 	 * If we clean storage, the term inherits the
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 807d9d15e4..cb3a189e8f 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2800,6 +2800,16 @@ link_live_proc_bin(struct shrink_cand_data *shrink,
     *prevppp = &pbp->next;
 }
 
+#ifdef ERTS_MAGIC_REF_THING_HEADER
+/*
+ * ERTS_MAGIC_REF_THING_HEADER only defined when there
+ * is a size difference between magic and ordinary references...
+ */
+# define ERTS_USED_MAGIC_REF_THING_HEADER__ ERTS_MAGIC_REF_THING_HEADER
+#else
+# define ERTS_USED_MAGIC_REF_THING_HEADER__ ERTS_REF_THING_HEADER
+#endif
+
 
 static void
 sweep_off_heap(Process *p, int fullsweep)
@@ -2832,7 +2842,8 @@ sweep_off_heap(Process *p, int fullsweep)
 	    ASSERT(!ErtsInArea(ptr, oheap, oheap_sz));
 	    *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word);
 	    ASSERT(!IS_MOVED_BOXED(ptr->thing_word));
-	    if (ptr->thing_word == HEADER_PROC_BIN) {
+	    switch (ptr->thing_word) {
+	    case HEADER_PROC_BIN: {
 		int to_new_heap = !ErtsInArea(ptr, oheap, oheap_sz);
 		ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1)));
 		if (to_new_heap) {
@@ -2841,13 +2852,28 @@ sweep_off_heap(Process *p, int fullsweep)
 		    BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/
 		}		
 		link_live_proc_bin(&shrink, &prev, &ptr, to_new_heap);
-	    }
-	    else {
+                break;
+            }
+            case ERTS_USED_MAGIC_REF_THING_HEADER__: {
+                Uint size;
+		int to_new_heap = !ErtsInArea(ptr, oheap, oheap_sz);
+                ASSERT(is_magic_ref_thing(ptr));
+		ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1)));
+                size = (Uint) ((ErtsMRefThing *) ptr)->mb->orig_size;
+		if (to_new_heap)
+		    bin_vheap += size / sizeof(Eterm);
+                else
+		    BIN_OLD_VHEAP(p) += size / sizeof(Eterm); /* for binary gc (words)*/
+                /* fall through... */
+            }
+            default:
 		prev = &ptr->next;
 		ptr = ptr->next;
 	    }
 	}
-	else if (!ErtsInArea(ptr, oheap, oheap_sz)) {
+	else if (ErtsInArea(ptr, oheap, oheap_sz))
+            break; /* and let old-heap loop continue */
+        else {
 	    /* garbage */
 	    switch (thing_subtag(ptr->thing_word)) {
 	    case REFC_BINARY_SUBTAG:
@@ -2881,7 +2907,6 @@ sweep_off_heap(Process *p, int fullsweep)
 	    }
 	    *prev = ptr = ptr->next;
 	}
-	else break; /* and let old-heap loop continue */
     }
 
     /* The rest of the list resides on old-heap, and we just did a
@@ -2890,16 +2915,24 @@ sweep_off_heap(Process *p, int fullsweep)
     while (ptr) {
 	ASSERT(ErtsInArea(ptr, oheap, oheap_sz));
 	ASSERT(!IS_MOVED_BOXED(ptr->thing_word));       
-	if (ptr->thing_word == HEADER_PROC_BIN) {
+        switch (ptr->thing_word) {
+        case HEADER_PROC_BIN:
 	    BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/
 	    link_live_proc_bin(&shrink, &prev, &ptr, 0);
-	}
-	else {
-	    ASSERT(is_fun_header(ptr->thing_word) ||
-		   is_external_header(ptr->thing_word)
-		   || is_magic_ref_thing(ptr));
-	    prev = &ptr->next;
-	    ptr = ptr->next;
+            break;
+        case ERTS_USED_MAGIC_REF_THING_HEADER__:
+            ASSERT(is_magic_ref_thing(ptr));
+            BIN_OLD_VHEAP(p) +=
+                (((Uint) ((ErtsMRefThing *) ptr)->mb->orig_size)
+                 / sizeof(Eterm)); /* for binary gc (words)*/
+            /* fall through... */
+        default:
+            ASSERT(is_fun_header(ptr->thing_word) ||
+                   is_external_header(ptr->thing_word)
+                   || is_magic_ref_thing(ptr));
+            prev = &ptr->next;
+            ptr = ptr->next;
+            break;
 	}
     }
 
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 60df7b8a08..f9cba0aec0 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3510,6 +3510,8 @@ dec_term_atom_common:
 			ASSERT(rtp);
 			hp = (Eterm *) rtp;
 			write_magic_ref_thing(hp, factory->off_heap, mb);
+                        OH_OVERHEAD(factory->off_heap,
+                                    mb->orig_size / sizeof(Eterm));
 			hp += ERTS_MAGIC_REF_THING_SIZE;
 		    }
 		}
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index fc459e17ad..092a5320ba 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3592,8 +3592,10 @@ store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns)
 	return make_internal_ref(to_hp);
     else {
 	ErtsMRefThing *mreft = (ErtsMRefThing *) from_hp;
+        ErtsMagicBinary *mb = mreft->mb;
 	ASSERT(is_magic_ref_thing(from_hp));
-	erts_refc_inc(&mreft->mb->refc, 2);
+	erts_refc_inc(&mb->refc, 2);
+        OH_OVERHEAD(oh, mb->orig_size / sizeof(Eterm));
     }
 
     ohhp = (struct erl_off_heap_header*) to_hp;
-- 
cgit v1.2.3


From bb69faa1ab31b93bf4ef200d437d6eb52c88b61a Mon Sep 17 00:00:00 2001
From: Raimo Niskanen <raimo@erlang.org>
Date: Fri, 3 Feb 2017 12:14:17 +0100
Subject: Use fstat if it exists in efile_openfile

---
 erts/emulator/drivers/unix/unix_efile.c | 89 +++++++++++++++++----------------
 1 file changed, 46 insertions(+), 43 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c
index bfe0807df8..3ff68a8859 100644
--- a/erts/emulator/drivers/unix/unix_efile.c
+++ b/erts/emulator/drivers/unix/unix_efile.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -79,11 +79,10 @@
  * Macros for testing file types.
  */
 
-#define ISDIR(st) (((st).st_mode & S_IFMT) == S_IFDIR)
-#define ISREG(st) (((st).st_mode & S_IFMT) == S_IFREG)
-#define ISDEV(st) \
-  (((st).st_mode&S_IFMT) == S_IFCHR || ((st).st_mode&S_IFMT) == S_IFBLK)
-#define ISLNK(st) (((st).st_mode & S_IFLNK) == S_IFLNK)
+#define ISDIR(st) (S_ISDIR((st).st_mode))
+#define ISREG(st) (S_ISREG((st).st_mode))
+#define ISDEV(st) (S_ISCHR((st).st_mode) || S_ISBLK((st).st_mode))
+#define ISLNK(st) (S_ISLNK((st).st_mode))
 #ifdef NO_UMASK
 #define FILE_MODE 0644
 #define DIR_MODE  0755
@@ -366,33 +365,6 @@ efile_openfile(Efile_error* errInfo,	/* Where to return error codes. */
     int fd;
     int mode;			/* Open mode. */
 
-    if (stat(name, &statbuf) < 0) {
-	/* statbuf is undefined: if the caller depends on it,
-	   i.e. invoke_read_file(), fail the call immediately */
-	if (pSize && flags == EFILE_MODE_READ)
-	    return check_error(-1, errInfo);
-    } else if (!ISREG(statbuf)) {
-	/*
-	 * For UNIX only, here is some ugly code to allow
-	 * /dev/null to be opened as a file.
-	 *
-	 * Assumption: The i-node number for /dev/null cannot be zero.
-	 */
-	static ino_t dev_null_ino = 0;
-
-	if (dev_null_ino == 0) {
-	    struct stat nullstatbuf;
-	    
-	    if (stat("/dev/null", &nullstatbuf) >= 0) {
-		dev_null_ino = nullstatbuf.st_ino;
-	    }
-	}
-	if (!(dev_null_ino && statbuf.st_ino == dev_null_ino)) {
-	    errno = EISDIR;
-	    return check_error(-1, errInfo);
-	}
-    }
-
     switch (flags & (EFILE_MODE_READ|EFILE_MODE_WRITE)) {
     case EFILE_MODE_READ:
 	mode = O_RDONLY;
@@ -411,16 +383,13 @@ efile_openfile(Efile_error* errInfo,	/* Where to return error codes. */
 	return check_error(-1, errInfo);
     }
 
-
     if (flags & EFILE_MODE_APPEND) {
 	mode &= ~O_TRUNC;
 	mode |= O_APPEND;
     }
-
     if (flags & EFILE_MODE_EXCL) {
 	mode |= O_EXCL;
     }
-
     if (flags & EFILE_MODE_SYNC) {
 #ifdef O_SYNC
 	mode |= O_SYNC;
@@ -430,15 +399,49 @@ efile_openfile(Efile_error* errInfo,	/* Where to return error codes. */
 #endif
     }
 
-    fd = open(name, mode, FILE_MODE);
+#ifdef HAVE_FSTAT
+    while (((fd = open(name, mode, FILE_MODE)) < 0) && (errno == EINTR));
+    if (!check_error(fd, errInfo)) return 0;
+#endif
+
+    if (
+#ifdef HAVE_FSTAT
+        fstat(fd, &statbuf) < 0
+#else
+        stat(name, &statbuf) < 0
+#endif
+        ) {
+        /* statbuf is undefined: if the caller depends on it,
+           i.e. invoke_read_file(), fail the call immediately */
+        if (pSize && flags == EFILE_MODE_READ) {
+            check_error(-1, errInfo);
+#ifdef HAVE_FSTAT
+            efile_closefile(fd);
+#endif
+            return 0;
+        }
+    }
+    else if (! ISREG(statbuf)) {
+        struct stat nullstatbuf;
+	/*
+	 * For UNIX only, here is some ugly code to allow
+	 * /dev/null to be opened as a file.
+	 */
+        if ( (stat("/dev/null", &nullstatbuf) < 0)
+             || (statbuf.st_ino != nullstatbuf.st_ino)
+             || (statbuf.st_dev != nullstatbuf.st_dev) ) {
+	    errno = EISDIR;
+	    return check_error(-1, errInfo);
+        }
+    }
 
-    if (!check_error(fd, errInfo))
-	return 0;
+#ifndef HAVE_FSTAT
+    while (((fd = open(name, mode, FILE_MODE)) < 0) && (errno == EINTR));
+    if (!check_error(fd, errInfo)) return 0;
+#endif
 
     *pfd = fd;
-    if (pSize) {
-	*pSize = statbuf.st_size;
-    }
+    if (pSize) *pSize = statbuf.st_size;
     return 1;
 }
 
@@ -460,7 +463,7 @@ efile_may_openfile(Efile_error* errInfo, char *name) {
 void
 efile_closefile(int fd)
 {
-    close(fd);
+    while((close(fd) < 0) && (errno == EINTR));
 }
 
 int
-- 
cgit v1.2.3


From 75fdce43ef567668bb89508b9b8ce0df7efaa569 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 6 Feb 2017 17:15:52 +0100
Subject: erts: Add enif_monitor_process and enif_demonitor_process

---
 erts/emulator/beam/bif.c                      |   4 +-
 erts/emulator/beam/break.c                    |  24 +-
 erts/emulator/beam/dist.c                     |  25 +-
 erts/emulator/beam/erl_bif_info.c             |  90 +++--
 erts/emulator/beam/erl_bits.c                 |   3 -
 erts/emulator/beam/erl_driver.h               |   7 -
 erts/emulator/beam/erl_drv_nif.h              |   8 +
 erts/emulator/beam/erl_lock_check.c           |   3 +
 erts/emulator/beam/erl_monitors.c             |  47 ++-
 erts/emulator/beam/erl_monitors.h             |  19 +-
 erts/emulator/beam/erl_nif.c                  | 402 ++++++++++++++++++-
 erts/emulator/beam/erl_nif.h                  |  22 +-
 erts/emulator/beam/erl_nif_api_funcs.h        |   4 +
 erts/emulator/beam/erl_node_tables.c          |   4 +-
 erts/emulator/beam/erl_process.c              |  50 +--
 erts/emulator/beam/erl_time_sup.c             |   2 +-
 erts/emulator/beam/global.h                   |  34 +-
 erts/emulator/beam/io.c                       |  20 +-
 erts/emulator/sys/common/erl_check_io.c       |   2 +
 erts/emulator/test/nif_SUITE.erl              | 248 +++++++++++-
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 540 +++++++++++++++++++++++++-
 erts/emulator/test/nif_SUITE_data/nif_mod.c   |   2 +
 22 files changed, 1389 insertions(+), 171 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 65c370c55b..51057c3ebb 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -361,7 +361,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
 					    c_p->common.id, 
 					    (mon->name != NIL
 					     ? mon->name
-					     : mon->pid), 
+					     : mon->u.pid), 
 					    ref,
 					    0);
             res = (code == ERTS_DSIG_SEND_YIELD ? am_yield : am_true);
@@ -498,7 +498,7 @@ BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip)
        res = am_true;
        break;
    case MON_ORIGIN:
-       to = mon->pid;
+       to = mon->u.pid;
        *multip = am_false;
        if (is_atom(to)) {
            /* Monitoring a name at node to */
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 6e1e94b95b..5509c50a8e 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -178,19 +178,29 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext)
 	prefix = "";
     }
 
-    if (mon->type == MON_ORIGIN) {
-	if (is_atom(mon->pid)) { /* dist by name */
-	    ASSERT(is_node_name_atom(mon->pid));
+    switch (mon->type) {
+    case MON_ORIGIN:
+	if (is_atom(mon->u.pid)) { /* dist by name */
+	    ASSERT(is_node_name_atom(mon->u.pid));
 	    erts_print(to, to_arg, "%s{to,{%T,%T},%T}", prefix, mon->name,
-		       mon->pid, mon->ref);
+		       mon->u.pid, mon->ref);
 	} else if (is_atom(mon->name)){ /* local by name */
 	    erts_print(to, to_arg, "%s{to,{%T,%T},%T}", prefix, mon->name,
 		       erts_this_dist_entry->sysname, mon->ref);
 	} else { /* local and distributed by pid */
-	    erts_print(to, to_arg, "%s{to,%T,%T}", prefix, mon->pid, mon->ref);
+	    erts_print(to, to_arg, "%s{to,%T,%T}", prefix, mon->u.pid, mon->ref);
 	}
-    } else { /* MON_TARGET */
-	erts_print(to, to_arg, "%s{from,%T,%T}", prefix, mon->pid, mon->ref);
+	break;
+    case MON_TARGET:
+	erts_print(to, to_arg, "%s{from,%T,%T}", prefix, mon->u.pid, mon->ref);
+	break;
+    case MON_NIF_TARGET: {
+        ErtsResource* rsrc = mon->u.resource;
+	/*SVERK: Print resource-ref? */
+        erts_print(to, to_arg, "%s{from,{%T,%T},%T}", prefix, rsrc->type->module,
+                   rsrc->type->name, mon->ref);
+	break;
+    }
     }
 }
 			       
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 9e8f853279..016abaf717 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -259,7 +259,7 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp)
     DistEntry *dep = ((NetExitsContext *) vnecp)->dep;
     ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK;
 
-    rp = erts_pid2proc(NULL, 0, mon->pid, rp_locks);
+    rp = erts_pid2proc(NULL, 0, mon->u.pid, rp_locks);
     if (!rp)
 	goto done;
 
@@ -278,10 +278,11 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp)
 	rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref);
 	/* ASSERT(rmon != NULL); can happen during process exit */
 	if (rmon != NULL) {
+            ASSERT(rmon->type == MON_ORIGIN);
 	    ASSERT(is_atom(rmon->name) || is_nil(rmon->name));
 	    watched = (is_atom(rmon->name)
 		       ? TUPLE2(lhp, rmon->name, dep->sysname)
-		       : rmon->pid);
+		       : rmon->u.pid);
 #ifdef ERTS_SMP
 	    rp_locks |= ERTS_PROC_LOCKS_MSG_SEND;
 	    erts_smp_proc_lock(rp, ERTS_PROC_LOCKS_MSG_SEND);
@@ -1391,7 +1392,7 @@ int erts_net_message(Port *prt,
 	if (mon == NULL) {
 	    break;
 	}
-	watched = mon->pid;
+	watched = mon->u.pid;
 	erts_destroy_monitor(mon);
 	rp = erts_pid2proc_opt(NULL, 0,
 			       watched, ERTS_PROC_LOCK_LINK,
@@ -1549,7 +1550,7 @@ int erts_net_message(Port *prt,
 	if (mon == NULL) {
 	    break;
 	}
-	rp = erts_pid2proc(NULL, 0, mon->pid, rp_locks);
+	rp = erts_pid2proc(NULL, 0, mon->u.pid, rp_locks);
 
 	erts_destroy_monitor(mon);
 	if (rp == NULL) {
@@ -1566,7 +1567,7 @@ int erts_net_message(Port *prt,
 	
 	watched = (is_not_nil(mon->name)
 		   ? TUPLE2(&lhp[0], mon->name, sysname)
-		   : mon->pid);
+		   : mon->u.pid);
 	
 	erts_queue_monitor_message(rp, &rp_locks,
 				   ref, am_process, watched, reason);
@@ -2415,21 +2416,21 @@ static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp)
     void *arg = ((struct print_to_data *) vptdp)->arg;
     Process *rp;
     ErtsMonitor *rmon;
-    rp = erts_proc_lookup(mon->pid);
+    rp = erts_proc_lookup(mon->u.pid);
     if (!rp || (rmon = erts_lookup_monitor(ERTS_P_MONITORS(rp), mon->ref)) == NULL) {
-	erts_print(to, arg, "Warning, stray monitor for: %T\n", mon->pid);
+	erts_print(to, arg, "Warning, stray monitor for: %T\n", mon->u.pid);
     } else if (mon->type == MON_ORIGIN) {
 	/* Local pid is being monitored */
 	erts_print(to, arg, "Remotely monitored by: %T %T\n",
-		   mon->pid, rmon->pid);
+		   mon->u.pid, rmon->u.pid);
     } else {
-	erts_print(to, arg, "Remote monitoring: %T ", mon->pid);
-	if (is_not_atom(rmon->pid))
-	    erts_print(to, arg, "%T\n", rmon->pid);
+	erts_print(to, arg, "Remote monitoring: %T ", mon->u.pid);
+	if (is_not_atom(rmon->u.pid))
+	    erts_print(to, arg, "%T\n", rmon->u.pid);
 	else
 	    erts_print(to, arg, "{%T, %T}\n",
 		       rmon->name,
-		       rmon->pid); /* which in this case is the 
+		       rmon->u.pid); /* which in this case is the 
 				      remote system name... */
     }
 }    
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 5ee19aead8..9f32e0e7bd 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -202,8 +202,10 @@ bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh)
 static void do_calc_mon_size(ErtsMonitor *mon, void *vpsz)
 {
     Uint *psz = vpsz;
-    *psz += IS_CONST(mon->ref) ? 0 : NC_HEAP_SIZE(mon->ref);
-    *psz += IS_CONST(mon->pid) ? 0 : NC_HEAP_SIZE(mon->pid);
+    *psz += NC_HEAP_SIZE(mon->ref);
+    *psz += (mon->type == MON_NIF_TARGET ?
+	     erts_resource_ref_size(mon->u.resource) :
+	     (is_immed(mon->u.pid) ? 0 : NC_HEAP_SIZE(mon->u.pid)));
     *psz += 8; /* CONS + 5-tuple */ 
 }
 
@@ -218,12 +220,11 @@ static void do_make_one_mon_element(ErtsMonitor *mon, void * vpmlc)
 {
     MonListContext *pmlc = vpmlc;
     Eterm tup;
-    Eterm r = (IS_CONST(mon->ref)
-	       ? mon->ref
-	       : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->ref));
-    Eterm p = (IS_CONST(mon->pid)
-	       ? mon->pid
-	       : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->pid));
+    Eterm r = STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->ref);
+    Eterm p = (mon->type == MON_NIF_TARGET ?
+	       erts_bld_resource_ref(&(pmlc->hp), &MSO(pmlc->p), mon->u.resource)
+	       : (is_immed(mon->u.pid) ? mon->u.pid
+		  : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->u.pid)));
     tup = TUPLE5(pmlc->hp, pmlc->tag, make_small(mon->type), r, p, mon->name);
     pmlc->hp += 6;
     pmlc->res = CONS(pmlc->hp, tup, pmlc->res);
@@ -262,7 +263,7 @@ make_monitor_list(Process *p, ErtsMonitor *root)
 static void do_calc_lnk_size(ErtsLink *lnk, void *vpsz)
 {
     Uint *psz = vpsz;
-    *psz += IS_CONST(lnk->pid) ? 0 : NC_HEAP_SIZE(lnk->pid);
+    *psz += is_immed(lnk->pid) ? 0 : NC_HEAP_SIZE(lnk->pid);
     if (lnk->type != LINK_NODE && ERTS_LINK_ROOT(lnk) != NULL) { 
 	/* Node links use this pointer as ref counter... */
 	erts_doforall_links(ERTS_LINK_ROOT(lnk),&do_calc_lnk_size,vpsz);
@@ -282,7 +283,7 @@ static void do_make_one_lnk_element(ErtsLink *lnk, void * vpllc)
     LnkListContext *pllc = vpllc;
     Eterm tup;
     Eterm old_res, targets = NIL;
-    Eterm p = (IS_CONST(lnk->pid)
+    Eterm p = (is_immed(lnk->pid)
 	       ? lnk->pid
 	       : STORE_NC(&(pllc->hp), &MSO(pllc->p), lnk->pid));
     if (lnk->type == LINK_NODE) {
@@ -366,8 +367,12 @@ erts_print_system_version(fmtfn_t to, void *arg, Process *c_p)
 typedef struct {
     /* {Entity,Node} = {monitor.Name,monitor.Pid} for external by name
      * {Entity,Node} = {monitor.Pid,NIL} for external/external by pid
-     * {Entity,Node} = {monitor.Name,erlang:node()} for internal by name */
-    Eterm entity;
+     * {Entity,Node} = {monitor.Name,erlang:node()} for internal by name 
+     * {Entity,Node} = {monitor.resource,MON_NIF_TARGET}*/
+    union {
+	Eterm term;
+	ErtsResource* resource;
+    }entity;
     Eterm node;
     /* pid is actual target being monitored, no matter pid/port or name */
     Eterm pid;
@@ -413,7 +418,7 @@ static void collect_one_link(ErtsLink *lnk, void *vmicp)
     if (!(lnk->type == LINK_PID)) {
 	return;
     }
-    micp->mi[micp->mi_i].entity = lnk->pid;
+    micp->mi[micp->mi_i].entity.term = lnk->pid;
     micp->sz += 2 + NC_HEAP_SIZE(lnk->pid);
     micp->mi_i++;
 } 
@@ -426,20 +431,20 @@ static void collect_one_origin_monitor(ErtsMonitor *mon, void *vmicp)
 	return;
     }
     EXTEND_MONITOR_INFOS(micp);
-    if (is_atom(mon->pid)) { /* external by name */
-	micp->mi[micp->mi_i].entity = mon->name;
-        micp->mi[micp->mi_i].node = mon->pid;
+    if (is_atom(mon->u.pid)) { /* external by name */
+	micp->mi[micp->mi_i].entity.term = mon->name;
+        micp->mi[micp->mi_i].node = mon->u.pid;
         micp->sz += 3; /* need one 2-tuple */
-    } else if (is_external_pid(mon->pid)) { /* external by pid */
-	micp->mi[micp->mi_i].entity = mon->pid;
+    } else if (is_external_pid(mon->u.pid)) { /* external by pid */
+	micp->mi[micp->mi_i].entity.term = mon->u.pid;
         micp->mi[micp->mi_i].node = NIL;
-        micp->sz += NC_HEAP_SIZE(mon->pid);
+        micp->sz += NC_HEAP_SIZE(mon->u.pid);
     } else if (!is_nil(mon->name)) { /* internal by name */
-	micp->mi[micp->mi_i].entity = mon->name;
+	micp->mi[micp->mi_i].entity.term = mon->name;
         micp->mi[micp->mi_i].node = erts_this_dist_entry->sysname;
         micp->sz += 3; /* need one 2-tuple */
     } else { /* internal by pid */
-	micp->mi[micp->mi_i].entity = mon->pid;
+	micp->mi[micp->mi_i].entity.term = mon->u.pid;
         micp->mi[micp->mi_i].node = NIL;
 	/* no additional heap space needed */
     }
@@ -447,7 +452,7 @@ static void collect_one_origin_monitor(ErtsMonitor *mon, void *vmicp)
     /* have always pid at hand, to assist with figuring out if its a port or
      * a process, when we monitored by name and process_info is requested.
      * See: erl_bif_info.c:process_info_aux section for am_monitors */
-    micp->mi[micp->mi_i].pid = mon->pid;
+    micp->mi[micp->mi_i].pid = mon->u.pid;
 
     micp->mi_i++;
     micp->sz += 2 + 3; /* For a cons cell and a 2-tuple */
@@ -457,15 +462,24 @@ static void collect_one_target_monitor(ErtsMonitor *mon, void *vmicp)
 {
     MonitorInfoCollection *micp = vmicp;
  
-    if (mon->type != MON_TARGET) {
-	return;
+    if (mon->type != MON_TARGET && mon->type != MON_NIF_TARGET) {
+        return;
     }
 
     EXTEND_MONITOR_INFOS(micp);
   
-    micp->mi[micp->mi_i].node = NIL;
-    micp->mi[micp->mi_i].entity = mon->pid;
-    micp->sz += (NC_HEAP_SIZE(mon->pid) + 2 /* cons */);
+
+    if (mon->type == MON_NIF_TARGET) {
+	micp->mi[micp->mi_i].entity.resource = mon->u.resource;
+        micp->mi[micp->mi_i].node = make_small(MON_NIF_TARGET);
+        micp->sz += erts_resource_ref_size(mon->u.resource);
+    }
+    else {
+	micp->mi[micp->mi_i].entity.term = mon->u.pid;
+        micp->mi[micp->mi_i].node = NIL;
+        micp->sz += NC_HEAP_SIZE(mon->u.pid);
+    }
+    micp->sz += 2; /* cons */;
     micp->mi_i++;
 }
 
@@ -1194,7 +1208,7 @@ process_info_aux(Process *BIF_P,
 	hp = HAlloc(BIF_P, 3 + mic.sz);
 	res = NIL;
 	for (i = 0; i < mic.mi_i; i++) {
-	    item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); 
+	    item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity.term); 
 	    res = CONS(hp, item, res);
 	    hp += 2;
 	}
@@ -1212,7 +1226,7 @@ process_info_aux(Process *BIF_P,
         hp = HAlloc(BIF_P, 3 + mic.sz);
 	res = NIL;
 	for (i = 0; i < mic.mi_i; i++) {
-	    if (is_atom(mic.mi[i].entity)) {
+	    if (is_atom(mic.mi[i].entity.term)) {
 		/* Monitor by name. 
                  * Build {process|port, {Name, Node}} and cons it.
 		 */
@@ -1224,7 +1238,7 @@ process_info_aux(Process *BIF_P,
                     || is_port(mic.mi[i].pid)
                     || is_atom(mic.mi[i].pid));
 
-		t1 = TUPLE2(hp, mic.mi[i].entity, mic.mi[i].node);
+		t1 = TUPLE2(hp, mic.mi[i].entity.term, mic.mi[i].node);
 		hp += 3;
                 t2 = TUPLE2(hp, m_type, t1);
 		hp += 3;
@@ -1234,7 +1248,7 @@ process_info_aux(Process *BIF_P,
 	    else {
                 /* Monitor by pid. Build {process|port, Pid} and cons it. */
 		Eterm t;
-		Eterm pid = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity);
+		Eterm pid = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity.term);
 
                 Eterm m_type = is_port(mic.mi[i].pid) ? am_port : am_process;
                 ASSERT(is_pid(mic.mi[i].pid)
@@ -1261,7 +1275,12 @@ process_info_aux(Process *BIF_P,
 
 	res = NIL;
 	for (i = 0; i < mic.mi_i; ++i) {
-	    item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); 
+            if (mic.mi[i].node == make_small(MON_NIF_TARGET)) {
+                item = erts_bld_resource_ref(&hp, &MSO(BIF_P), mic.mi[i].entity.resource);
+            }
+            else {
+                item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity.term);
+            }
 	    res = CONS(hp, item, res);
 	    hp += 2;
 	}
@@ -2950,7 +2969,7 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt,
 	if (hpp) {
 	    res = NIL;
 	    for (i = 0; i < mic.mi_i; i++) {
-		item = STORE_NC(hpp, ohp, mic.mi[i].entity);
+		item = STORE_NC(hpp, ohp, mic.mi[i].entity.term);
 		res = CONS(*hpp, item, res);
 		*hpp += 2;
 	    }
@@ -2981,7 +3000,7 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt,
 		Eterm t;
                 Eterm m_type;
 
-                item = STORE_NC(hpp, ohp, mic.mi[i].entity);
+                item = STORE_NC(hpp, ohp, mic.mi[i].entity.term);
                 m_type = is_port(item) ? am_port : am_process;
                 t = TUPLE2(*hpp, m_type, item);
 		*hpp += 3;
@@ -3010,7 +3029,8 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt,
         if (hpp) {
             res = NIL;
             for (i = 0; i < mic.mi_i; ++i) {
-                item = STORE_NC(hpp, ohp, mic.mi[i].entity);
+                ASSERT(mic.mi[i].node == NIL);
+                item = STORE_NC(hpp, ohp, mic.mi[i].entity.term);
                 res = CONS(*hpp, item, res);
                 *hpp += 2;
             }
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 6bf52fb303..885e955332 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -110,9 +110,6 @@ erts_init_bits(void)
 {
     ERTS_CT_ASSERT(offsetof(Binary,orig_bytes) % 8 == 0);
     ERTS_CT_ASSERT(offsetof(ErtsMagicBinary,u.aligned.data) % 8 == 0);
-    ERTS_CT_ASSERT(ERTS_MAGIC_BIN_BYTES_TO_ALIGN ==
-                   (offsetof(ErtsMagicBinary,u.aligned.data)
-                    - offsetof(ErtsMagicBinary,u.unaligned.data)));
     ERTS_CT_ASSERT(offsetof(ErtsBinary,driver.binary.orig_bytes)
                 == offsetof(Binary,orig_bytes));
 
diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 5bea92e198..b386b68cff 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -175,13 +175,6 @@ struct erl_drv_event_data {
 #endif
 typedef struct erl_drv_event_data *ErlDrvEventData; /* Event data */
 
-/*
- * A driver monitor
- */
-typedef struct {
-    unsigned char data[sizeof(void *)*4];
-} ErlDrvMonitor;
-
 typedef struct {
     unsigned long megasecs;
     unsigned long secs;
diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h
index 46bb06d642..8de4a7855d 100644
--- a/erts/emulator/beam/erl_drv_nif.h
+++ b/erts/emulator/beam/erl_drv_nif.h
@@ -64,6 +64,14 @@ enum ErlNifSelectReturn {
     ERL_NIF_SELECT_FAILED         = (1 << 4)
 };
 
+/*
+ * A driver monitor
+ */
+typedef struct {
+    unsigned char data[sizeof(void *)*4];
+} ErlDrvMonitor;
+
+
 #ifdef SIZEOF_CHAR
 #  define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR
 #  undef SIZEOF_CHAR
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 08dcbed91c..0798faf53a 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -89,6 +89,9 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"hipe_mfait_lock",			NULL			},
 #endif
     {	"nodes_monitors",			NULL			},
+#ifdef ERTS_SMP
+    {	"resource_monitors",			"address"	        },
+#endif
     {   "driver_list",                          NULL                    },
     {	"proc_link",				"pid"			},
     {	"proc_msgq",				"pid"			},
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
index 910598690d..e5be66be79 100644
--- a/erts/emulator/beam/erl_monitors.c
+++ b/erts/emulator/beam/erl_monitors.c
@@ -91,7 +91,7 @@ static ERTS_INLINE int cmp_mon_ref(Eterm ref1, Eterm ref2)
 	    
 #define CP_LINK_VAL(To, Hp, From)				\
 do {								\
-    if (IS_CONST(From))						\
+    if (is_immed(From))						\
 	(To) = (From);						\
     else {							\
 	Uint i__;						\
@@ -109,15 +109,15 @@ do {								\
     }								\
 } while (0)
 
-static ErtsMonitor *create_monitor(Uint type, Eterm ref, Eterm pid, Eterm name)
+static ErtsMonitor *create_monitor(Uint type, Eterm ref, UWord entity, Eterm name)
 {
      Uint mon_size = ERTS_MONITOR_SIZE;
      ErtsMonitor *n;
      Eterm *hp;
 
      mon_size += NC_HEAP_SIZE(ref);
-     if (!IS_CONST(pid)) {
-	 mon_size += NC_HEAP_SIZE(pid);
+     if (type != MON_NIF_TARGET && is_not_immed(entity)) {
+	 mon_size += NC_HEAP_SIZE(entity);
      }
 
      if (mon_size <= ERTS_MONITOR_SH_SIZE) {
@@ -136,7 +136,10 @@ static ErtsMonitor *create_monitor(Uint type, Eterm ref, Eterm pid, Eterm name)
      n->balance = 0;            /* Always the same initial value */
      n->name = name; /* atom() or [] */
      CP_LINK_VAL(n->ref, hp, ref); /*XXX Unneccesary check, never immediate*/
-     CP_LINK_VAL(n->pid, hp, pid);
+     if (type == MON_NIF_TARGET)
+         n->u.resource = (ErtsResource*)entity;
+     else
+         CP_LINK_VAL(n->u.pid, hp, (Eterm)entity);
 
      return n;
 }
@@ -147,7 +150,7 @@ static ErtsLink *create_link(Uint type, Eterm pid)
      ErtsLink *n;
      Eterm *hp;
 
-     if (!IS_CONST(pid)) {
+     if (is_not_immed(pid)) {
 	 lnk_size += NC_HEAP_SIZE(pid);
      }
 
@@ -206,16 +209,16 @@ void erts_destroy_monitor(ErtsMonitor *mon)
     Uint mon_size = ERTS_MONITOR_SIZE;
     ErlNode *node;
 
-    ASSERT(!IS_CONST(mon->ref));
+    ASSERT(is_not_immed(mon->ref));
     mon_size +=  NC_HEAP_SIZE(mon->ref);
     if (is_external(mon->ref)) {
 	node = external_thing_ptr(mon->ref)->node;
 	erts_deref_node_entry(node);
     }
-    if (!IS_CONST(mon->pid)) {
-	mon_size += NC_HEAP_SIZE(mon->pid);
-	if (is_external(mon->pid)) {
-	    node = external_thing_ptr(mon->pid)->node;
+    if (mon->type != MON_NIF_TARGET && is_not_immed(mon->u.pid)) {
+	mon_size += NC_HEAP_SIZE(mon->u.pid);
+	if (is_external(mon->u.pid)) {
+	    node = external_thing_ptr(mon->u.pid)->node;
 	    erts_deref_node_entry(node);
 	}
     }
@@ -234,7 +237,7 @@ void erts_destroy_link(ErtsLink *lnk)
 
     ASSERT(lnk->type == LINK_NODE || ERTS_LINK_ROOT(lnk) == NULL);
 
-    if (!IS_CONST(lnk->pid)) {
+    if (is_not_immed(lnk->pid)) {
 	lnk_size += NC_HEAP_SIZE(lnk->pid);
 	if (is_external(lnk->pid)) {
 	    node = external_thing_ptr(lnk->pid)->node;
@@ -329,7 +332,7 @@ static void insertion_rotation(int dstack[], int dpos,
     }
 }
 
-void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, Eterm pid, 
+void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, UWord entity, 
 		      Eterm name)
 {
     void *tstack[STACK_NEED];
@@ -344,7 +347,7 @@ void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, Eterm pid,
     for (;;) {
 	if (!*this) { /* Found our place */
 	    state = 1;
-	    *this = create_monitor(type,ref,pid,name);
+	    *this = create_monitor(type,ref,entity,name);
 	    break;
 	} else if ((c = CMP_MON_REF(ref,(*this)->ref)) < 0) { 
 	    /* go left */
@@ -914,8 +917,12 @@ static void erts_dump_monitors(ErtsMonitor *root, int indent)
     if (root == NULL)
 	return;
     erts_dump_monitors(root->right,indent+2);
-    erts_printf("%*s[%b16d:%b16u:%T:%T:%T]\n", indent, "", root->balance,
-		root->type, root->ref, root->pid, root->name);
+    erts_printf("%*s[%b16d:%b16u:%T:%T", indent, "", root->balance,
+		root->type, root->ref, root->name);
+    if (root->type == MON_NIF_TARGET)
+	erts_printf(":%p]\n", root->u.resource);
+    else
+	erts_printf(":%T]\n", root->u.pid);
     erts_dump_monitors(root->left,indent+2);
 }
 
@@ -1030,7 +1037,7 @@ void erts_one_link_size(ErtsLink *lnk, void *vpu)
 {
     Uint *pu = vpu;
     *pu += ERTS_LINK_SIZE*sizeof(Uint);
-    if(!IS_CONST(lnk->pid))
+    if(is_not_immed(lnk->pid))
 	*pu += NC_HEAP_SIZE(lnk->pid)*sizeof(Uint);
     if (lnk->type != LINK_NODE && ERTS_LINK_ROOT(lnk) != NULL) {
 	erts_doforall_links(ERTS_LINK_ROOT(lnk),&erts_one_link_size,vpu);
@@ -1040,8 +1047,8 @@ void erts_one_mon_size(ErtsMonitor *mon, void *vpu)
 {
     Uint *pu = vpu;
     *pu += ERTS_MONITOR_SIZE*sizeof(Uint);
-    if(!IS_CONST(mon->pid))
-	*pu += NC_HEAP_SIZE(mon->pid)*sizeof(Uint);
-    if(!IS_CONST(mon->ref))
+    if(mon->type != MON_NIF_TARGET && is_not_immed(mon->u.pid))
+	*pu += NC_HEAP_SIZE(mon->u.pid)*sizeof(Uint);
+    if(is_not_immed(mon->ref))
 	*pu += NC_HEAP_SIZE(mon->ref)*sizeof(Uint);
 }
diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h
index 9e2beedea3..f90ac930e5 100644
--- a/erts/emulator/beam/erl_monitors.h
+++ b/erts/emulator/beam/erl_monitors.h
@@ -82,8 +82,9 @@
 
 /* Type tags for monitors */
 #define MON_ORIGIN 1
-#define MON_TARGET 3
-#define MON_TIME_OFFSET 7
+#define MON_TARGET 2
+#define MON_NIF_TARGET 3
+#define MON_TIME_OFFSET 4
 
 /* Type tags for links */
 #define LINK_PID 1   /* ...Or port */
@@ -105,11 +106,15 @@ typedef struct erts_monitor_or_link {
 typedef struct erts_monitor {
     struct erts_monitor *left, *right; 
     Sint16 balance;
-    Uint16 type;  /* MON_ORIGIN | MON_TARGET | MON_TIME_OFFSET */
+    Uint16 type;  /* MON_ORIGIN | MON_TARGET | MON_NIF_TARGET | MON_TIME_OFFSET */
     Eterm ref;
-    Eterm pid;    /* In case of distributed named monitor, this is the
-		     nodename atom in MON_ORIGIN process, otherwise a pid or
-		     , in case of a MON_TARGET, a port */
+    union {
+        Eterm pid;  /* In case of distributed named monitor, this is the
+		     * nodename atom in MON_ORIGIN process, otherwise a pid or,
+		     * in case of a MON_TARGET, a port
+                     */
+        struct ErtsResource_* resource; /* MON_NIF_TARGET */
+    }u;
     Eterm name;   /* When monitoring a named process: atom() else [] */
     Uint heap[1]; /* Larger in reality */
 } ErtsMonitor;
@@ -144,7 +149,7 @@ Uint erts_tot_link_lh_size(void);
 
 /* Prototypes */
 void erts_destroy_monitor(ErtsMonitor *mon);
-void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, Eterm pid,
+void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, UWord entity,
 		      Eterm name);
 ErtsMonitor *erts_remove_monitor(ErtsMonitor **root, Eterm ref);
 ErtsMonitor *erts_lookup_monitor(ErtsMonitor *root, Eterm ref);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index b7425b9e45..62191c4abf 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1913,8 +1913,6 @@ int enif_snprintf(char *buffer, size_t size, const char* format, ...)
 /* dummy node in circular list */
 struct enif_resource_type_t resource_type_list; 
 
-#define SIZEOF_ErlNifResource(SIZE) (offsetof(ErtsResource,data) + (SIZE))
-
 static ErlNifResourceType* find_resource_type(Eterm module, Eterm name)
 {
     ErlNifResourceType* type;
@@ -2083,6 +2081,7 @@ static void commit_opened_resource_types(struct erl_module_nif* lib)
 	type->owner = lib;
 	type->dtor = ort->new_callbacks.dtor;
         type->stop = ort->new_callbacks.stop;
+        type->down = ort->new_callbacks.down;
 
 	if (type->dtor != NULL) {
 	    erts_refc_inc(&lib->rt_dtor_cnt, 1);
@@ -2108,12 +2107,148 @@ static void rollback_opened_resource_types(void)
     }
 }
 
+struct destroy_monitor_ctx
+{
+    Binary* resource_bin;
+    int exiting_procs;
+    int scheduler;
+};
+
+static void destroy_one_monitor(ErtsMonitor* mon, void* context)
+{
+    struct destroy_monitor_ctx* ctx = (struct destroy_monitor_ctx*) context;
+    Process* rp;
+    ErtsMonitor *rmon = NULL;
+    int is_exiting;
+
+    ASSERT(mon->type == MON_ORIGIN);
+    ASSERT(is_internal_pid(mon->u.pid));
+    ASSERT(is_internal_ref(mon->ref));
+
+    if (ctx->scheduler > 0) { /* Normal scheduler */
+        rp = erts_proc_lookup(mon->u.pid);
+    }
+    else {
+#ifdef ERTS_SMP
+        rp = erts_proc_lookup_inc_refc(mon->u.pid);
+#else
+        ASSERT(!"nif monitor destruction in non-scheduler thread");
+        rp = NULL;
+#endif
+    }
+
+    if (!rp) {
+        is_exiting = 1;
+    }
+    if (rp) {
+        erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK);
+        if (ERTS_PROC_IS_EXITING(rp)) {
+            is_exiting = 1;
+        } else {
+            rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref);
+            ASSERT(rmon);
+            is_exiting = 0;
+        }
+        erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+#ifdef ERTS_SMP
+        if (ctx->scheduler <= 0)
+            erts_proc_dec_refc(rp);
+#endif
+    }
+    if (is_exiting) {
+        ctx->exiting_procs++;
+        /* +1 for exiting process */
+        erts_refc_inc(&ctx->resource_bin->refc, ctx->exiting_procs);
+    }
+
+    /* ToDo: Delay destruction after monitor_locks */
+    if (rmon) {
+        ASSERT(rmon->type == MON_NIF_TARGET);
+        ASSERT(&ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rmon->u.pid)->binary == ctx->resource_bin);
+        erts_destroy_monitor(rmon);
+    }
+    erts_destroy_monitor(mon);
+}
+
+static int destroy_all_monitors(ErtsMonitor* monitors, Binary* bin)
+{
+    struct destroy_monitor_ctx ctx;
+
+    execution_state(NULL, NULL, &ctx.scheduler);
+
+    ctx.resource_bin = bin;
+    ctx.exiting_procs = 0;
+    erts_sweep_monitors(monitors, &destroy_one_monitor, &ctx);
+    return ctx.exiting_procs == 0;
+}
+
+
+#ifdef ERTS_SMP
+#  define NIF_RESOURCE_DTOR &nif_resource_dtor
+#else
+#  define NIF_RESOURCE_DTOR &nosmp_nif_resource_dtor_prologue
+
+/*
+ * NO-SMP: Always run resource destructor on scheduler thread 
+ *         as we may have to remove process monitors.
+ */
+static int nif_resource_dtor(Binary*);
+
+static void nosmp_nif_resource_dtor_scheduled(void* vbin)
+{
+    erts_bin_free((Binary*)vbin);
+}
+
+static int nosmp_nif_resource_dtor_prologue(Binary* bin)
+{
+    if (is_scheduler()) {
+        return nif_resource_dtor(bin);
+    }
+    else {
+        erts_schedule_misc_aux_work(1, nosmp_nif_resource_dtor_scheduled, bin);
+        return 0; /* do not free */
+    }
+}
+
+#endif /* !ERTS_SMP */
 
 static int nif_resource_dtor(Binary* bin)
 {
     ErtsResource* resource = (ErtsResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin);
     ErlNifResourceType* type = resource->type;
-    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor);
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == NIF_RESOURCE_DTOR);
+
+    if (resource->monitors) {
+        ErtsResourceMonitors* rm = resource->monitors;
+        ErtsMonitor* root;
+
+        ASSERT(type->down);
+        erts_smp_mtx_lock(&rm->lock);
+        ASSERT(erts_refc_read(&bin->refc, 0) == 0);
+        root = rm->root;
+        if (root) {
+            rm->root = NULL;
+            if (!destroy_all_monitors(root, bin)) {
+                /*
+                 * Resource death struggle prolonged to serve exiting process(es).
+                 * Destructor will be called again when last exiting process
+                 * tries to fire its MON_NIF_TARGET monitor (and fails).
+                 *
+                 * This resource is doomed. It has no "real" references and
+                 * should get not get called upon to do anything except the
+                 * final destructor call.
+                 */
+                ASSERT(erts_refc_read(&bin->refc, 1));
+#ifdef DEBUG
+                resource->dbg_is_dying = 1;
+#endif
+                erts_smp_mtx_unlock(&rm->lock);
+                return 0;
+            }
+        }
+        erts_smp_mtx_unlock(&rm->lock);
+        erts_smp_mtx_destroy(&rm->lock);
+    }
 
     if (type->dtor != NULL) {
         struct enif_msg_environment_t msg_env;
@@ -2141,20 +2276,99 @@ void erts_resource_stop(ErtsResource* resource, ErlNifEvent e,
     post_nif_noproc(&msg_env);
 }
 
-void* enif_alloc_resource(ErlNifResourceType* type, size_t size)
+/* SVERK SVERK: Move to ?.h */
+void erts_ref_to_driver_monitor(Eterm ref, ErlDrvMonitor*);
+
+void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref)
 {
-    Binary* bin = erts_create_magic_binary_x(SIZEOF_ErlNifResource(size),
-                                             &nif_resource_dtor,
-                                             1); /* unaligned */
-    ErtsResource* resource = ERTS_MAGIC_BIN_UNALIGNED_DATA(bin);
+    ErtsMonitor* rmon;
+    ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
+    struct enif_msg_environment_t msg_env;
+    ErlNifPid nif_pid;
+    ErlNifMonitor nif_monitor;
+    ErtsResourceMonitors* rmp = resource->monitors;
+
+    ASSERT(rmp);
+    ASSERT(resource->type->down);
+
+    erts_smp_mtx_lock(&rmp->lock);
+    rmon = erts_remove_monitor(&rmp->root, ref);
+    if (!rmon) {
+        erts_smp_mtx_unlock(&rmp->lock);
+        /* -1 for exiting process */
+        if (erts_refc_dectest(&bin->binary.refc, 0) == 0) {
+            erts_bin_free(&bin->binary);
+        }
+        return;
+    }
+    ASSERT(!resource->dbg_is_dying);
+    if (erts_refc_inctest(&bin->binary.refc, 1) < 2) {
+        /*
+         * Racing resource destruction. 
+         * To avoid a more complex refc-dance with destructing thread
+         * we avoid calling 'down' and just silently remove the monitor.
+         * There are no real references left to this resource and we have
+         * monitors_lock, so it's safe to reset refc back to zero.
+         * This can happen even for non smp as destructor calls may be scheduled.
+         */
+        erts_refc_init(&bin->binary.refc, 0);
+        erts_smp_mtx_unlock(&rmp->lock);
+    }
+    else {
+        erts_smp_mtx_unlock(&rmp->lock);
+
+        ASSERT(rmon->u.pid == pid);
+        erts_ref_to_driver_monitor(ref, &nif_monitor);
+        nif_pid.pid = pid;
+        pre_nif_noproc(&msg_env, resource->type->owner, NULL);
+        resource->type->down(&msg_env.env, resource->data, &nif_pid, &nif_monitor);
+        post_nif_noproc(&msg_env);
+
+        if (erts_refc_dectest(&bin->binary.refc, 0) == 0) {
+            erts_bin_free(&bin->binary);
+        }
+    }
+    erts_destroy_monitor(rmon);
+}
+
+void* enif_alloc_resource(ErlNifResourceType* type, size_t data_sz)
+{
+    size_t magic_sz = offsetof(ErtsResource,data);
+    Binary* bin;
+    ErtsResource* resource;
+    size_t monitors_offs;
+
+    if (type->down) {
+        /* Put ErtsResourceMonitors after user data and properly aligned */
+        monitors_offs = ((data_sz + ERTS_ALLOC_ALIGN_BYTES - 1)
+                         & ~((size_t)ERTS_ALLOC_ALIGN_BYTES - 1));
+        magic_sz += monitors_offs + sizeof(ErtsResourceMonitors);
+    }
+    else {
+        ERTS_UNDEF(monitors_offs, 0);
+        magic_sz += data_sz;
+    }
+    bin = erts_create_magic_binary_x(magic_sz, NIF_RESOURCE_DTOR,
+                                     1); /* unaligned */
+    resource = ERTS_MAGIC_BIN_UNALIGNED_DATA(bin);
 
     ASSERT(type->owner && type->next && type->prev); /* not allowed in load/upgrade */
     resource->type = type;
     erts_refc_inc(&bin->refc, 1);
 #ifdef DEBUG
     erts_refc_init(&resource->nif_refc, 1);
+    resource->dbg_is_dying = 0;
 #endif
     erts_refc_inc(&resource->type->refc, 2);
+    if (type->down) {
+        resource->monitors = (ErtsResourceMonitors*) (resource->data + monitors_offs);
+        erts_smp_mtx_init(&resource->monitors->lock, "resource_monitors");
+        resource->monitors->root = NULL;
+        resource->monitors->user_data_sz = data_sz;
+    }
+    else {
+        resource->monitors = NULL;
+    }
     return resource->data;
 }
 
@@ -2163,7 +2377,8 @@ void enif_release_resource(void* obj)
     ErtsResource* resource = DATA_TO_RESOURCE(obj);
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
 
-    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor);
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == NIF_RESOURCE_DTOR);
+    ASSERT(!resource->dbg_is_dying);
 #ifdef DEBUG
     erts_refc_dec(&resource->nif_refc, 0);
 #endif
@@ -2177,18 +2392,27 @@ void enif_keep_resource(void* obj)
     ErtsResource* resource = DATA_TO_RESOURCE(obj);
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
 
-    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor);
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == NIF_RESOURCE_DTOR);
+    ASSERT(!resource->dbg_is_dying);
 #ifdef DEBUG
     erts_refc_inc(&resource->nif_refc, 1);
 #endif
     erts_refc_inc(&bin->binary.refc, 2);
 }
 
+Eterm erts_bld_resource_ref(Eterm** hpp, ErlOffHeap* oh, ErtsResource* resource)
+{
+    ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
+    ASSERT(!resource->dbg_is_dying);
+    return erts_mk_magic_binary_term(hpp, oh, &bin->binary);
+}
+
 ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj)
 {
     ErtsResource* resource = DATA_TO_RESOURCE(obj);
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
     Eterm* hp = alloc_heap(env,PROC_BIN_SIZE);
+    ASSERT(!resource->dbg_is_dying);
     return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary);
 }
 
@@ -2217,7 +2441,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ
     }*/
     mbin = pb->val;
     resource = (ErtsResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin);
-    if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor
+    if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != NIF_RESOURCE_DTOR
 	|| resource->type != type) {	
 	return 0;
     }
@@ -2228,8 +2452,13 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ
 size_t enif_sizeof_resource(void* obj)
 {
     ErtsResource* resource = DATA_TO_RESOURCE(obj);
-    Binary* bin = &ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource)->binary;
-    return ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(bin) - offsetof(ErtsResource,data);
+    if (resource->monitors) {
+        return resource->monitors->user_data_sz;
+    }
+    else {
+        Binary* bin = &ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource)->binary;
+        return ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(bin) - offsetof(ErtsResource,data);
+    }
 }
 
 
@@ -2830,6 +3059,150 @@ int enif_map_iterator_get_pair(ErlNifEnv *env,
     return 0;
 }
 
+int enif_monitor_process(ErlNifEnv* env, void* obj, const ErlNifPid* target_pid,
+                         ErlNifMonitor* monitor)
+{
+    int scheduler;
+    ErtsResource* rsrc = DATA_TO_RESOURCE(obj);
+    Process *rp;
+    Eterm tmp[REF_THING_SIZE];
+    Eterm ref;
+    int retval;
+
+    ASSERT(ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc)->magic_binary.destructor
+           == NIF_RESOURCE_DTOR);
+    ASSERT(!rsrc->dbg_is_dying);
+    ASSERT(!rsrc->monitors == !rsrc->type->down);
+
+
+    if (!rsrc->monitors) {
+        ASSERT(!rsrc->type->down);
+        return -1;
+    }
+    ASSERT(rsrc->type->down);
+
+    execution_state(env, NULL, &scheduler);
+
+#ifdef ERTS_SMP
+    if (scheduler > 0) /* Normal scheduler */
+        rp = erts_proc_lookup_raw(target_pid->pid);
+    else
+        rp = erts_proc_lookup_raw_inc_refc(target_pid->pid);
+#else
+    if (scheduler <= 0) {
+        erts_exit(ERTS_ABORT_EXIT, "enif_monitor_process: called from "
+                  "non-scheduler thread on non-SMP VM");
+    }
+    rp = erts_proc_lookup(target_pid->pid);
+#endif
+
+    if (!rp)
+        return 1;
+
+    ref = erts_make_ref_in_buffer(tmp);
+
+    erts_smp_mtx_lock(&rsrc->monitors->lock);
+    erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK);
+    if (ERTS_PSFLG_FREE & erts_smp_atomic32_read_nob(&rp->state)) {
+        retval = 1;
+    }
+    else {
+        erts_add_monitor(&rsrc->monitors->root, MON_ORIGIN, ref, rp->common.id, NIL);
+        erts_add_monitor(&ERTS_P_MONITORS(rp), MON_NIF_TARGET, ref, (UWord)rsrc, NIL);
+        retval = 0;
+    }
+    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+    erts_smp_mtx_unlock(&rsrc->monitors->lock);
+
+#ifdef ERTS_SMP
+    if (scheduler <= 0)
+        erts_proc_dec_refc(rp);
+#endif
+    if (monitor)
+        erts_ref_to_driver_monitor(ref,monitor);
+
+    return retval;
+}
+
+int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monitor)
+{
+    int scheduler;
+    ErtsResource* rsrc = DATA_TO_RESOURCE(obj);
+    ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc);
+    Process *rp;
+    ErtsMonitor *mon;
+    ErtsMonitor *rmon = NULL;
+    Eterm ref_heap[REF_THING_SIZE];
+    Eterm ref;
+    int is_exiting;
+
+    ASSERT(bin->magic_binary.destructor == NIF_RESOURCE_DTOR);
+    ASSERT(!rsrc->dbg_is_dying);
+
+    execution_state(env, NULL, &scheduler);
+
+    memcpy(ref_heap, monitor, sizeof(Eterm)*REF_THING_SIZE);
+    ref = make_internal_ref(ref_heap);
+    erts_smp_mtx_lock(&rsrc->monitors->lock);
+    mon = erts_remove_monitor(&rsrc->monitors->root, ref);
+
+    if (mon == NULL) {
+        erts_smp_mtx_unlock(&rsrc->monitors->lock);
+        return 1;
+    }
+
+    ASSERT(mon->type == MON_ORIGIN);
+    ASSERT(is_internal_pid(mon->u.pid));
+
+#ifdef ERTS_SMP
+    if (scheduler > 0) /* Normal scheduler */
+        rp = erts_proc_lookup(mon->u.pid);
+    else
+        rp = erts_proc_lookup_inc_refc(mon->u.pid);
+#else
+    if (scheduler <= 0) {
+        erts_exit(ERTS_ABORT_EXIT, "enif_demonitor_process: called from "
+                  "non-scheduler thread on non-SMP VM");
+    }
+    rp = erts_proc_lookup(mon->u.pid);
+#endif
+
+    if (!rp) {
+        is_exiting = 1;
+    }
+    else {
+        erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK);
+        if (ERTS_PROC_IS_EXITING(rp)) {
+            is_exiting = 1;
+        } else {
+            rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref);
+            ASSERT(rmon);
+            is_exiting = 0;
+        }
+        erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+
+#ifdef ERTS_SMP
+        if (scheduler <= 0)
+            erts_proc_dec_refc(rp);
+#endif
+    }
+    if (is_exiting) {
+        /* +1 for exiting process */
+        erts_refc_inc(&bin->binary.refc, 2);
+    }
+    erts_smp_mtx_unlock(&rsrc->monitors->lock);
+
+    if (rmon) {
+        ASSERT(rmon->type == MON_NIF_TARGET);
+        ASSERT(rmon->u.resource == rsrc);
+        erts_destroy_monitor(rmon);
+    }
+    erts_destroy_monitor(mon);
+
+    return 0;
+}
+
+
 /***************************************************************************
  **                              load_nif/2                               **
  ***************************************************************************/
@@ -3303,7 +3676,8 @@ erts_unload_nif(struct erl_module_nif* lib)
 
 void erl_nif_init()
 {
-    ERTS_CT_ASSERT((offsetof(ErtsResource,data) % 8) == ERTS_MAGIC_BIN_BYTES_TO_ALIGN);
+    ERTS_CT_ASSERT((offsetof(ErtsResource,data) % 8)
+                   == ERTS_MAGIC_BIN_BYTES_TO_ALIGN);
 
     resource_type_list.next = &resource_type_list;
     resource_type_list.prev = &resource_type_list;
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 78e0fa1864..5248f287ee 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -142,15 +142,6 @@ typedef struct
 typedef int ErlNifEvent; /* An event to be selected on. */
 //#endif
 
-typedef struct enif_resource_type_t ErlNifResourceType;
-typedef void ErlNifResourceDtor(ErlNifEnv*, void*);
-typedef void ErlNifResourceStop(ErlNifEnv*, void*, ErlNifEvent, int is_direct_call);
-
-typedef struct {
-    ErlNifResourceDtor* dtor;
-    ErlNifResourceStop* stop;  /* at ERL_NIF_SELECT_STOP event */
-} ErlNifResourceTypeInit;
-
 typedef enum
 {
     ERL_NIF_RT_CREATE = 1,
@@ -172,6 +163,19 @@ typedef struct
     ERL_NIF_TERM port_id;  /* internal, may change */
 }ErlNifPort;
 
+typedef ErlDrvMonitor ErlNifMonitor;
+
+typedef struct enif_resource_type_t ErlNifResourceType;
+typedef void ErlNifResourceDtor(ErlNifEnv*, void*);
+typedef void ErlNifResourceStop(ErlNifEnv*, void*, ErlNifEvent, int is_direct_call);
+typedef void ErlNifResourceDown(ErlNifEnv*, void*, ErlNifPid*, ErlNifMonitor*);
+
+typedef struct {
+    ErlNifResourceDtor* dtor;
+    ErlNifResourceStop* stop;  /* at ERL_NIF_SELECT_STOP event */
+    ErlNifResourceDown* down;  /* enif_monitor_process */
+} ErlNifResourceTypeInit;
+
 typedef ErlDrvSysInfo ErlNifSysInfo;
 
 typedef struct ErlDrvTid_ *ErlNifTid;
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 9163ce25eb..a9ed246962 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -177,6 +177,8 @@ ERL_NIF_API_FUNC_DECL(int,enif_thread_type,(void));
 ERL_NIF_API_FUNC_DECL(int,enif_snprintf,(char * buffer, size_t size, const char *format, ...));
 ERL_NIF_API_FUNC_DECL(int,enif_select,(ErlNifEnv* env, ErlNifEvent e, enum ErlNifSelectFlags flags, void* obj, ERL_NIF_TERM ref));
 ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type_x,(ErlNifEnv*, const char* name_str, const ErlNifResourceTypeInit*, ErlNifResourceFlags flags, ErlNifResourceFlags* tried));
+ERL_NIF_API_FUNC_DECL(int, enif_monitor_process,(ErlNifEnv*,void* obj,const ErlNifPid*,ErlDrvMonitor *monitor));
+ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const ErlDrvMonitor *monitor));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
@@ -336,6 +338,8 @@ ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type_x,(ErlNifEnv*,
 #  define enif_snprintf ERL_NIF_API_FUNC_MACRO(enif_snprintf)
 #  define enif_select ERL_NIF_API_FUNC_MACRO(enif_select)
 #  define enif_open_resource_type_x ERL_NIF_API_FUNC_MACRO(enif_open_resource_type_x)
+#  define enif_monitor_process ERL_NIF_API_FUNC_MACRO(enif_monitor_process)
+#  define enif_demonitor_process ERL_NIF_API_FUNC_MACRO(enif_demonitor_process)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 70500ed6e1..f463f7fdf4 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1165,8 +1165,8 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id)
 static void doit_insert_monitor(ErtsMonitor *monitor, void *p)
 {
     Eterm *idp = p;
-    if(is_external(monitor->pid))
-	insert_node(external_thing_ptr(monitor->pid)->node, MONITOR_REF, *idp);
+    if(monitor->type != MON_NIF_TARGET && is_external(monitor->u.pid))
+	insert_node(external_thing_ptr(monitor->u.pid)->node, MONITOR_REF, *idp);
     if(is_external(monitor->ref))
 	insert_node(external_thing_ptr(monitor->ref)->node, MONITOR_REF, *idp);
 }
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index f80ebdf31b..9db77585f2 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -13320,9 +13320,9 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
     switch (mon->type) {
     case MON_ORIGIN:
 	/* We are monitoring someone else, we need to demonitor that one.. */
-	if (is_atom(mon->pid)) { /* remote by name */
-	    ASSERT(is_node_name_atom(mon->pid));
-	    dep = erts_sysname_to_connected_dist_entry(mon->pid);
+	if (is_atom(mon->u.pid)) { /* remote by name */
+	    ASSERT(is_node_name_atom(mon->u.pid));
+	    dep = erts_sysname_to_connected_dist_entry(mon->u.pid);
 	    if (dep) {
 		erts_smp_de_links_lock(dep);
 		rmon = erts_remove_monitor(&(dep->monitors), mon->ref);
@@ -13333,7 +13333,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 						 ERTS_DSP_NO_LOCK, 0);
 		    if (code == ERTS_DSIG_PREP_CONNECTED) {
 			code = erts_dsig_send_demonitor(&dsd,
-							rmon->pid,
+							rmon->u.pid,
 							mon->name,
 							mon->ref,
 							1);
@@ -13344,10 +13344,10 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		erts_deref_dist_entry(dep);
 	    }
 	} else {
-            ASSERT(is_pid(mon->pid) || is_port(mon->pid));
+            ASSERT(is_pid(mon->u.pid) || is_port(mon->u.pid));
             /* if is local by pid or name */
-            if (is_internal_pid(mon->pid)) {
-                Process *rp = erts_pid2proc(NULL, 0, mon->pid, ERTS_PROC_LOCK_LINK);
+            if (is_internal_pid(mon->u.pid)) {
+                Process *rp = erts_pid2proc(NULL, 0, mon->u.pid, ERTS_PROC_LOCK_LINK);
 		if (!rp) {
 		    goto done;
 		}
@@ -13357,9 +13357,9 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		    goto done;
 		}
 		erts_destroy_monitor(rmon);
-            } else if (is_internal_port(mon->pid)) {
+            } else if (is_internal_port(mon->u.pid)) {
                 /* Is a local port */
-                Port *prt = erts_port_lookup_raw(mon->pid);
+                Port *prt = erts_port_lookup_raw(mon->u.pid);
                 if (!prt) {
                     goto done;
                 }
@@ -13367,8 +13367,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
                                     ERTS_PORT_DEMONITOR_ORIGIN_ON_DEATHBED,
                                     prt, mon->ref, NULL);
             } else { /* remote by pid */
-		ASSERT(is_external_pid(mon->pid));
-		dep = external_pid_dist_entry(mon->pid);
+		ASSERT(is_external_pid(mon->u.pid));
+		dep = external_pid_dist_entry(mon->u.pid);
 		ASSERT(dep != NULL);
 		if (dep) {
 		    erts_smp_de_links_lock(dep);
@@ -13380,8 +13380,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 						     ERTS_DSP_NO_LOCK, 0);
 			if (code == ERTS_DSIG_PREP_CONNECTED) {
 			    code = erts_dsig_send_demonitor(&dsd,
-							    rmon->pid,
-							    mon->pid,
+							    rmon->u.pid,
+							    mon->u.pid,
 							    mon->ref,
 							    1);
 			    ASSERT(code == ERTS_DSIG_SEND_OK);
@@ -13393,22 +13393,21 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 	}
 	break;
     case MON_TARGET:
-	ASSERT(mon->type == MON_TARGET);
-	ASSERT(is_pid(mon->pid) || is_internal_port(mon->pid));
-	if (is_internal_port(mon->pid)) {
-	    Port *prt = erts_id2port(mon->pid);
+	ASSERT(is_pid(mon->u.pid) || is_internal_port(mon->u.pid));
+	if (is_internal_port(mon->u.pid)) {
+	    Port *prt = erts_id2port(mon->u.pid);
 	    if (prt == NULL) {
 		goto done;
 	    }
 	    erts_fire_port_monitor(prt, mon->ref);
 	    erts_port_release(prt); 
-	} else if (is_internal_pid(mon->pid)) {/* local by name or pid */
+	} else if (is_internal_pid(mon->u.pid)) {/* local by name or pid */
 	    Eterm watched;
             Process *rp;
 	    DeclareTmpHeapNoproc(lhp,3);
 	    ErtsProcLocks rp_locks = (ERTS_PROC_LOCK_LINK
 				      | ERTS_PROC_LOCKS_MSG_SEND);
-	    rp = erts_pid2proc(NULL, 0, mon->pid, rp_locks);
+	    rp = erts_pid2proc(NULL, 0, mon->u.pid, rp_locks);
 	    if (rp == NULL) {
 		goto done;
 	    }
@@ -13427,8 +13426,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 	    /* else: demonitor while we exited, i.e. do nothing... */
 	    erts_smp_proc_unlock(rp, rp_locks);
 	} else { /* external by pid or name */
-	    ASSERT(is_external_pid(mon->pid));    
-	    dep = external_pid_dist_entry(mon->pid);
+	    ASSERT(is_external_pid(mon->u.pid));    
+	    dep = external_pid_dist_entry(mon->u.pid);
 	    ASSERT(dep != NULL);
 	    if (dep) {
 		erts_smp_de_links_lock(dep);
@@ -13440,10 +13439,10 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 						 ERTS_DSP_NO_LOCK, 0);
 		    if (code == ERTS_DSIG_PREP_CONNECTED) {
 			code = erts_dsig_send_m_exit(&dsd,
-						     mon->pid,
+						     mon->u.pid,
 						     (rmon->name != NIL
 						      ? rmon->name
-						      : rmon->pid),
+						      : rmon->u.pid),
 						     mon->ref,
 						     pcontext->reason);
 			ASSERT(code == ERTS_DSIG_SEND_OK);
@@ -13453,6 +13452,11 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 	    }
 	}
 	break;
+    case MON_NIF_TARGET:
+        erts_fire_nif_monitor(mon->u.resource,
+                              pcontext->p->common.id,
+                              mon->ref);
+        break;
     case MON_TIME_OFFSET:
 	erts_demonitor_time_offset(mon->ref);
 	break;
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 6aa2a7500f..c01cc26246 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1866,7 +1866,7 @@ save_time_offset_monitor(ErtsMonitor *mon, void *vcntxt)
 
     cntxt = (ErtsTimeOffsetMonitorContext *) vcntxt;
     mix = (cntxt->ix)++;
-    cntxt->to_mon_info[mix].pid = mon->pid;
+    cntxt->to_mon_info[mix].pid = mon->u.pid;
     to_hp = &cntxt->to_mon_info[mix].heap[0];
 
     ASSERT(is_internal_ref(mon->ref));
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 86c38f8e8c..511e357d14 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -66,30 +66,47 @@ struct enif_resource_type_t
     struct erl_module_nif* owner;  /* that created this type and thus implements the destructor*/
     ErlNifResourceDtor* dtor;      /* user destructor function */
     ErlNifResourceStop* stop;
+    ErlNifResourceDown* down;
     erts_refc_t refc;  /* num of resources of this type (HOTSPOT warning)
                           +1 for active erl_module_nif */
     Eterm module;
     Eterm name;
 };
+
 typedef struct
+{
+    erts_smp_mtx_t lock;
+    ErtsMonitor* root;
+    size_t user_data_sz;
+} ErtsResourceMonitors;
+
+typedef struct ErtsResource_
 {
     struct enif_resource_type_t* type;
+    ErtsResourceMonitors* monitors;
+#ifdef ARCH_32
+    byte align__[4];
+#endif
 #ifdef DEBUG
     erts_refc_t nif_refc;
-# ifdef ARCH_32
-    byte align__[4];
+    int dbg_is_dying;
+# ifdef ARCH_64
+    byte dbg_align__[4];
 # endif
 #endif
-
     char data[1];
 }ErtsResource;
 
-#define DATA_TO_RESOURCE(PTR) ((ErtsResource*)((char*)(PTR) - offsetof(ErtsResource,data)))
+#define DATA_TO_RESOURCE(PTR) ErtsContainerStruct(PTR, ErtsResource, data)
+#define erts_resource_ref_size(P) PROC_BIN_SIZE
+
+extern Eterm erts_bld_resource_ref(Eterm** hp, ErlOffHeap*, ErtsResource*);
 
 extern void erts_pre_nif(struct enif_environment_t*, Process*,
 			 struct erl_module_nif*, Process* tracee);
 extern void erts_post_nif(struct enif_environment_t* env);
 extern void erts_resource_stop(ErtsResource*, ErlNifEvent, int is_direct_call);
+void erts_fire_nif_monitor(ErtsResource*, Eterm pid, Eterm ref);
 extern Eterm erts_nif_taints(Process* p);
 extern void erts_print_nif_taints(fmtfn_t to, void* to_arg);
 void erts_unload_nif(struct erl_module_nif* nif);
@@ -286,11 +303,9 @@ typedef struct {
     } u;
 } ErtsMagicBinary;
 
-#ifdef ARCH_32
-#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 4
-#else
-#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 0
-#endif
+#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN \
+  (offsetof(ErtsMagicBinary,u.aligned.data) - \
+   offsetof(ErtsMagicBinary,u.unaligned.data))
 
 typedef union {
     Binary binary;
@@ -1306,6 +1321,7 @@ void erts_stale_drv_select(Eterm, ErlDrvPort, ErlDrvEvent, int, int);
 
 Port *erts_get_heart_port(void);
 void erts_emergency_close_ports(void);
+void erts_ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon);
 
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT)
 void erts_lcnt_enable_io_lock_count(int enable);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 33b74f30b7..7fa0280396 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -4192,8 +4192,8 @@ static void sweep_one_monitor(ErtsMonitor *mon, void *vpsc)
         ErtsMonitor *rmon;
         Process *rp;
 
-        ASSERT(is_internal_pid(mon->pid));
-        rp = erts_pid2proc(NULL, 0, mon->pid, ERTS_PROC_LOCK_LINK);
+        ASSERT(is_internal_pid(mon->u.pid));
+        rp = erts_pid2proc(NULL, 0, mon->u.pid, ERTS_PROC_LOCK_LINK);
         if (!rp) {
             goto done;
         }
@@ -4294,7 +4294,7 @@ port_fire_one_monitor(ErtsMonitor *mon, void *ctx0)
     Process      *origin;
     ErtsProcLocks origin_locks;
 
-    if (mon->type != MON_TARGET || ! is_pid(mon->pid)) {
+    if (mon->type != MON_TARGET || ! is_pid(mon->u.pid)) {
         return;
     }
     /*
@@ -4303,7 +4303,7 @@ port_fire_one_monitor(ErtsMonitor *mon, void *ctx0)
      */
     origin_locks = ERTS_PROC_LOCKS_MSG_SEND | ERTS_PROC_LOCK_LINK;
 
-    origin = erts_pid2proc(NULL, 0, mon->pid, origin_locks);
+    origin = erts_pid2proc(NULL, 0, mon->u.pid, origin_locks);
     if (origin) {
         DeclareTmpHeapNoproc(lhp,3);
         SweepContext    *ctx = (SweepContext *)ctx0;
@@ -5485,7 +5485,7 @@ typedef struct {
 static void prt_one_monitor(ErtsMonitor *mon, void *vprtd)
 {
     prt_one_lnk_data *prtd = (prt_one_lnk_data *) vprtd;
-    erts_print(prtd->to, prtd->arg, "(%T,%T)", mon->pid,mon->ref);
+    erts_print(prtd->to, prtd->arg, "(%T,%T)", mon->u.pid, mon->ref);
 }
 
 static void prt_one_lnk(ErtsLink *lnk, void *vprtd)
@@ -7630,7 +7630,7 @@ erl_drv_convert_time_unit(ErlDrvTime val,
 						    (int) to);
 }
 
-static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon)
+void erts_ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon)
 {
     RefThing *refp;
     ASSERT(is_internal_ref(ref));
@@ -7664,7 +7664,7 @@ static int do_driver_monitor_process(Port *prt,
     erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, ref, prt->common.id, NIL);
 
     erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-    ref_to_driver_monitor(ref,monitor);
+    erts_ref_to_driver_monitor(ref,monitor);
     return 0;
 }
 
@@ -7713,7 +7713,7 @@ static int do_driver_demonitor_process(Port *prt, Eterm *buf,
 	return 1;
     }
     ASSERT(mon->type == MON_ORIGIN);
-    to = mon->pid;
+    to = mon->u.pid;
     ASSERT(is_internal_pid(to));
     rp = erts_pid2proc_opt(NULL,
 			   0,
@@ -7775,7 +7775,7 @@ static ErlDrvTermData do_driver_get_monitored_process(Port *prt, Eterm *buf,
 	return driver_term_nil;
     }
     ASSERT(mon->type == MON_ORIGIN);
-    to = mon->pid;
+    to = mon->u.pid;
     ASSERT(is_internal_pid(to));
     return (ErlDrvTermData) to;
 }
@@ -7831,7 +7831,7 @@ void erts_fire_port_monitor(Port *prt, Eterm ref)
     }
     callback = prt->drv_ptr->process_exit;
     ASSERT(callback != NULL);
-    ref_to_driver_monitor(ref,&drv_monitor);
+    erts_ref_to_driver_monitor(ref,&drv_monitor);
     ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT);
     DRV_MONITOR_UNLOCK_PDL(prt);
 #ifdef USE_VM_PROBES
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 79f567d16c..5f4eca68d0 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1224,6 +1224,8 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
     DTRACE_CHARBUF(name, 64);
 #endif
 
+    ASSERT(!resource->dbg_is_dying);
+
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) {
 	if (fd < 0) {
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 27276e6646..e3af9f7454 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -32,6 +32,12 @@
          basic/1, reload_error/1, upgrade/1, heap_frag/1,
          t_on_load/1,
          select/1,
+         monitor_process_a/1,
+         monitor_process_b/1,
+         monitor_process_c/1,
+         monitor_process_d/1,
+         demonitor_process/1,
+         monitor_frenzy/1,
          hipe/1,
 	 types/1, many_args/1, binaries/1, get_string/1, get_atom/1,
 	 maps/1,
@@ -57,6 +63,8 @@
 
 -define(nif_stub,nif_stub_error(?LINE)).
 
+-define(is_resource, is_binary).  % to be is_reference
+
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
 all() ->
@@ -66,6 +74,8 @@ all() ->
         ++
     [reload_error, heap_frag, types, many_args,
      select,
+     {group, monitor},
+     monitor_frenzy,
      hipe,
      binaries, get_string, get_atom, maps, api_macros, from_array,
      iolist_as_binary, resource, resource_binary,
@@ -83,13 +93,19 @@ all() ->
      nif_snprintf].
 
 groups() ->
-    [{G, [], api_repeaters()} || G <- api_groups()].
+    [{G, [], api_repeaters()} || G <- api_groups()]
+        ++
+    [{monitor, [], [monitor_process_a,
+                    monitor_process_b,
+                    monitor_process_c,
+                    monitor_process_d,
+                    demonitor_process]}].
+
 
 api_groups() -> [api_latest, api_2_4, api_2_0].
 
 api_repeaters() -> [upgrade, resource_takeover, t_on_load].
 
-init_per_group(api_latest, Config) -> Config;
 init_per_group(api_2_4, Config) ->
     [{nif_api_version, ".2_4"} | Config];
 init_per_group(api_2_0, Config) ->
@@ -99,7 +115,8 @@ init_per_group(api_2_0, Config) ->
             {skip, "API 2.0 buggy on Windows 64-bit"};
         _ ->
             [{nif_api_version, ".2_0"} | Config]
-    end.
+    end;
+init_per_group(_, Config) -> Config.
 
 end_per_group(_,_) -> ok.
 
@@ -559,6 +576,216 @@ write_full(W, C, Acc) ->
             Acc
     end.
 
+%% Basic monitoring of one process that terminates 
+monitor_process_a(Config) ->
+    ensure_lib_loaded(Config),
+
+    F = fun(Terminator, UseMsgEnv) ->
+                Pid = spawn(fun() ->
+                                    receive
+                                        {exit, Arg} -> exit(Arg);
+                                        return -> ok;
+                                        BadMatch -> goodmatch = BadMatch
+                                    end
+                            end),
+                R_ptr = alloc_monitor_resource_nif(),
+                {0, Mon} = monitor_process_nif(R_ptr, Pid, UseMsgEnv, self()),
+                [R_ptr] = monitored_by(Pid),
+                Terminator(Pid),
+                [{monitor_resource_down, R_ptr, Pid, Mon}] = flush(),
+                [] = last_resource_dtor_call(),
+                ok = release_resource(R_ptr),
+                {R_ptr, _, 1} = last_resource_dtor_call()
+        end,
+
+    T1 = fun(Pid) -> Pid ! {exit, 17} end,
+    T2 = fun(Pid) -> Pid ! return end,
+    T3 = fun(Pid) -> Pid ! badmatch end,
+    T4 = fun(Pid) -> exit(Pid, 18) end,
+
+    [F(T, UME) || T <- [T1,T2,T3,T4], UME <- [true, false]],
+
+    ok.
+
+%% Test auto-demonitoring at resource destruction
+monitor_process_b(Config) ->
+    ensure_lib_loaded(Config),
+
+    Pid = spawn_link(fun() ->
+                             receive
+                                 return -> ok
+                             end
+                     end),
+    R_ptr = alloc_monitor_resource_nif(),
+    {0,_} = monitor_process_nif(R_ptr, Pid, true, self()),
+    [R_ptr] = monitored_by(Pid),
+    ok = release_resource(R_ptr),
+    [] = flush(),
+    {R_ptr, _, 1} = last_resource_dtor_call(),
+    [] = monitored_by(Pid),
+    Pid ! return,
+    ok.
+
+%% Test termination of monitored process holding last resource ref
+monitor_process_c(Config) ->
+    ensure_lib_loaded(Config),
+
+    Papa = self(),
+    Pid = spawn_link(fun() ->
+                             R_ptr = alloc_monitor_resource_nif(),
+                             {0,Mon} = monitor_process_nif(R_ptr, self(), true, Papa),
+                             [R_ptr] = monitored_by(self()),
+                             put(store, make_resource(R_ptr)),
+                             ok = release_resource(R_ptr),
+                             [] = last_resource_dtor_call(),
+                             Papa ! {self(), done, R_ptr, Mon},
+                             exit
+                     end),
+    [{Pid, done, R_ptr, Mon},
+     {monitor_resource_down, R_ptr, Pid, Mon}] = flush(),
+    {R_ptr, _, 1} = last_resource_dtor_call(),
+    ok.
+
+%% Test race of resource dtor called when monitored process is exiting
+monitor_process_d(Config) ->
+    ensure_lib_loaded(Config),
+
+    Papa = self(),
+    {Target,TRef} = spawn_monitor(fun() ->
+                                          nothing = receive_any()
+                                  end),
+    
+    R_ptr = alloc_monitor_resource_nif(),
+    {0,_} = monitor_process_nif(R_ptr, Target, true, self()),
+    [Papa, R_ptr] = monitored_by(Target),
+    
+    exit(Target, die),
+    ok = release_resource(R_ptr),
+
+    [{'DOWN', TRef, process, Target, die}] = flush(),  %% no monitor_resource_down
+    {R_ptr, _, 1} = last_resource_dtor_call(),
+
+    ok.
+
+%% Test basic demonitoring
+demonitor_process(Config) ->
+    ensure_lib_loaded(Config),
+
+    Pid = spawn_link(fun() ->
+                             receive
+                                 return -> ok
+                             end
+                     end),
+    R_ptr = alloc_monitor_resource_nif(),
+    {0,MonBin1} = monitor_process_nif(R_ptr, Pid, true, self()),
+    [R_ptr] = monitored_by(Pid),
+    {0,MonBin2} = monitor_process_nif(R_ptr, Pid, true, self()),
+    [R_ptr, R_ptr] = monitored_by(Pid),
+    0 = demonitor_process_nif(R_ptr, MonBin1),
+    [R_ptr] = monitored_by(Pid),
+    1 = demonitor_process_nif(R_ptr, MonBin1),
+    0 = demonitor_process_nif(R_ptr, MonBin2),
+    [] = monitored_by(Pid),
+    1 = demonitor_process_nif(R_ptr, MonBin2),
+
+    ok = release_resource(R_ptr),
+    [] = flush(),
+    {R_ptr, _, 1} = last_resource_dtor_call(),
+    [] = monitored_by(Pid),
+    Pid ! return,
+    ok.
+
+
+monitored_by(Pid) ->
+    {monitored_by, List0} = process_info(Pid, monitored_by),
+    List1 = lists:map(fun(E) when ?is_resource(E) ->
+                              {Ptr, _} = get_resource(monitor_resource_type, E),
+                              Ptr;
+                         (E) -> E
+                      end,
+                      List0),
+    erlang:garbage_collect(),
+    lists:sort(List1).
+
+-define(FRENZY_RAND_BITS, 25).
+
+monitor_frenzy(Config) ->
+    ensure_lib_loaded(Config),
+
+    Procs1 = processes(),
+    io:format("~p processes before: ~p\n", [length(Procs1), Procs1]),
+
+    %% Spawn first worker process
+    Master = self(),
+    spawn_link(fun() ->
+                       SelfPix = monitor_frenzy_nif(init, ?FRENZY_RAND_BITS, 0, 0),
+                       unlink(Master),
+                       frenzy(SelfPix, undefined)
+          end),
+    receive after 5*1000 -> ok end,
+
+    io:format("stats = ~p\n", [monitor_frenzy_nif(stats, 0, 0, 0)]),
+
+    Pids = monitor_frenzy_nif(stop, 0, 0, 0),
+    io:format("stats = ~p\n", [monitor_frenzy_nif(stats, 0, 0, 0)]),
+
+    lists:foreach(fun(P) -> exit(P, stop) end, Pids),
+
+    io:format("stats = ~p\n", [monitor_frenzy_nif(stats, 0, 0, 0)]),
+
+    Procs2 = processes(),
+    io:format("~p processes after: ~p\n", [length(Procs2), Procs2]),
+    ok.
+
+
+frenzy(_SelfPix, done) ->
+    ok;
+frenzy(SelfPix, State0) ->
+    Rnd = rand:uniform(1 bsl (?FRENZY_RAND_BITS+2)) - 1,
+    Op = Rnd band 3,
+    State1 = frenzy_do_op(SelfPix, Op, (Rnd bsr 2), State0),
+    frenzy(SelfPix, State1).
+
+frenzy_do_op(SelfPix, Op, Rnd, Pid0) ->
+    case Op of
+        0 -> % add/remove process
+            Papa = self(),
+            NewPid = case Pid0 of
+                         undefined -> % Prepare new process to be added
+                             spawn(fun() ->
+                                           MRef = monitor(process, Papa),
+                                           case receive_any() of
+                                               {go, MyPix, MyState} ->
+                                                   demonitor(MRef, [flush]),
+                                                   frenzy(MyPix, MyState);
+                                               {'DOWN', MRef, process, Papa, _} ->
+                                                   ok
+                                           end
+                                   end);
+                         _ ->
+                             Pid0
+                     end,
+            case monitor_frenzy_nif(Op, Rnd, SelfPix, NewPid) of
+                NewPix when is_integer(NewPix) ->
+                    NewPid ! {go, NewPix, undefined},
+                    undefined;
+                ExitPid when is_pid(ExitPid) ->
+                    false = (ExitPid =:= self()),
+                    exit(ExitPid,die),
+                    NewPid;
+                done ->
+                    done
+            end;
+        _ ->
+            case monitor_frenzy_nif(Op, Rnd, SelfPix, undefined) of
+                ok -> Pid0;
+                0 -> Pid0;
+                1 -> Pid0;
+                done -> done
+            end
+    end.
+
+
 hipe(Config) when is_list(Config) ->
     Data = proplists:get_value(data_dir, Config),
     Priv = proplists:get_value(priv_dir, Config),
@@ -814,6 +1041,7 @@ maps(Config) when is_list(Config) ->
     {1, M2} = make_map_remove_nif(M2, "key3"),
     {0, undefined} = make_map_remove_nif(self(), key),
 
+    verify_tmpmem(TmpMem),
     ok.
  
 %% Test macros enif_make_list<N> and enif_make_tuple<N>
@@ -1302,7 +1530,7 @@ resource_takeover(Config) when is_list(Config) ->
     [{load,1,1,101},{get_priv_data_ptr,1,2,102}] = nif_mod_call_history(),
 
     {NA7,BinNA7} = make_resource(0, Holder, "NA7"),
-    {AN7,BinAN7} = make_resource(1, Holder, "AN7"),
+    {AN7,_BinAN7} = make_resource(1, Holder, "AN7"),
 
     ok = forget_resource(NA7),
     [{{resource_dtor_A_v1,BinNA7},1,_,_}] = nif_mod_call_history(),
@@ -1793,7 +2021,7 @@ otp_9828(Config) ->
     ensure_lib_loaded(Config, 1),
     otp_9828_loop(<<"I'm alive!">>, 1000).
 
-otp_9828_loop(Bin, 0) ->
+otp_9828_loop(_Bin, 0) ->
     ok;
 otp_9828_loop(Bin, Val) ->
     WrtBin = <<Bin/binary, Val:32>>,
@@ -2075,7 +2303,7 @@ nif_raise_exceptions(NifFunc) ->
 -define(ERL_NIF_TIME_ERROR, -9223372036854775808).
 -define(TIME_UNITS, [second, millisecond, microsecond, nanosecond]).
 
-nif_monotonic_time(Config) ->
+nif_monotonic_time(_Config) ->
     ?ERL_NIF_TIME_ERROR = monotonic_time(invalid_time_unit),
     mtime_loop(1000000).
 
@@ -2100,7 +2328,7 @@ chk_mtime([TU|TUs]) ->
     end,
     chk_mtime(TUs).
 
-nif_time_offset(Config) ->
+nif_time_offset(_Config) ->
     ?ERL_NIF_TIME_ERROR = time_offset(invalid_time_unit),
     toffs_loop(1000000).
 
@@ -2138,7 +2366,7 @@ chk_toffs([TU|TUs]) ->
     end,
     chk_toffs(TUs).
 
-nif_convert_time_unit(Config) ->
+nif_convert_time_unit(_Config) ->
     ?ERL_NIF_TIME_ERROR = convert_time_unit(0, second, invalid_time_unit),
     ?ERL_NIF_TIME_ERROR = convert_time_unit(0, invalid_time_unit, second),
     ?ERL_NIF_TIME_ERROR = convert_time_unit(0, invalid_time_unit, invalid_time_unit),
@@ -2413,6 +2641,10 @@ write_nif(_,_) -> ?nif_stub.
 read_nif(_,_) -> ?nif_stub.
 is_closed_nif(_) -> ?nif_stub.
 last_fd_stop_call() -> ?nif_stub.
+alloc_monitor_resource_nif() -> ?nif_stub.
+monitor_process_nif(_,_,_,_) -> ?nif_stub.
+demonitor_process_nif(_,_) -> ?nif_stub.
+monitor_frenzy_nif(_,_,_,_) -> ?nif_stub.
 
 %% maps
 is_map_nif(_) -> ?nif_stub.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index c4f9611ec8..ee925512d2 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -31,6 +31,27 @@
 
 #include "nif_mod.h"
 
+#if 0
+static ErlNifMutex* dbg_trace_lock;
+#define DBG_TRACE_INIT dbg_trace_lock = enif_mutex_create("nif_SUITE.DBG_TRACE")
+#define DBG_TRACE_FINI enif_mutex_destroy(dbg_trace_lock)
+#define DBG_TRACE_LOCK enif_mutex_lock(dbg_trace_lock)
+#define DBG_TRACE_UNLOCK enif_mutex_unlock(dbg_trace_lock)
+#define DBG_TRACE0(FMT) do {DBG_TRACE_LOCK; enif_fprintf(stderr, FMT); DBG_TRACE_UNLOCK; }while(0)
+#define DBG_TRACE1(FMT, A) do {DBG_TRACE_LOCK; enif_fprintf(stderr, FMT, A); DBG_TRACE_UNLOCK; }while(0)
+#define DBG_TRACE2(FMT, A, B) do {DBG_TRACE_LOCK; enif_fprintf(stderr, FMT, A, B); DBG_TRACE_UNLOCK; }while(0)
+#define DBG_TRACE3(FMT, A, B, C) do {DBG_TRACE_LOCK; enif_fprintf(stderr, FMT, A, B, C); DBG_TRACE_UNLOCK; }while(0)
+#define DBG_TRACE4(FMT, A, B, C, D) do {DBG_TRACE_LOCK; enif_fprintf(stderr, FMT, A, B, C, D); DBG_TRACE_UNLOCK; }while(0)
+#else
+#define DBG_TRACE_INIT
+#define DBG_TRACE_FINI
+#define DBG_TRACE0(FMT)
+#define DBG_TRACE1(FMT, A)
+#define DBG_TRACE2(FMT, A, B)
+#define DBG_TRACE3(FMT, A, B, C)
+#define DBG_TRACE4(FMT, A, B, C, D)
+#endif
+
 static int static_cntA; /* zero by default */
 static int static_cntB = NIF_SUITE_LIB_VER * 100;
 
@@ -48,6 +69,12 @@ static ERL_NIF_TERM atom_eagain;
 static ERL_NIF_TERM atom_eof;
 static ERL_NIF_TERM atom_error;
 static ERL_NIF_TERM atom_fd_resource_stop;
+static ERL_NIF_TERM atom_monitor_resource_type;
+static ERL_NIF_TERM atom_monitor_resource_down;
+static ERL_NIF_TERM atom_init;
+static ERL_NIF_TERM atom_stats;
+static ERL_NIF_TERM atom_done;
+static ERL_NIF_TERM atom_stop;
 
 typedef struct
 {
@@ -120,6 +147,27 @@ struct fd_resource {
     ErlNifPid pid;
 };
 
+static ErlNifResourceType* monitor_resource_type;
+static void monitor_resource_dtor(ErlNifEnv* env, void* obj);
+static void monitor_resource_down(ErlNifEnv*, void* obj, ErlNifPid*, ErlNifMonitor*);
+static ErlNifResourceTypeInit monitor_rt_init = {
+    monitor_resource_dtor,
+    NULL,
+    monitor_resource_down
+};
+struct monitor_resource {
+    ErlNifPid receiver;
+    int use_msgenv;
+};
+
+static ErlNifResourceType* frenzy_resource_type;
+static void frenzy_resource_dtor(ErlNifEnv* env, void* obj);
+static void frenzy_resource_down(ErlNifEnv*, void* obj, ErlNifPid*, ErlNifMonitor*);
+static ErlNifResourceTypeInit frenzy_rt_init = {
+    frenzy_resource_dtor,
+    NULL,
+    frenzy_resource_down
+};
 
 static int get_pointer(ErlNifEnv* env, ERL_NIF_TERM term, void** pp)
 {
@@ -148,6 +196,8 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
     data->call_history = NULL;
     data->nif_mod = NULL;
 
+    DBG_TRACE_INIT;
+
     add_call(env, data, "load");
 
     data->rt_arr[0].t = enif_open_resource_type(env,NULL,"Gold",resource_dtor,
@@ -165,6 +215,13 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
     fd_resource_type =  enif_open_resource_type_x(env, "nif_SUITE.fd",
                                                   &fd_rt_init,
                                                   ERL_NIF_RT_CREATE, NULL);
+    monitor_resource_type = enif_open_resource_type_x(env, "nif_SUITE.monitor",
+                                                      &monitor_rt_init,
+                                                      ERL_NIF_RT_CREATE, NULL);
+    frenzy_resource_type = enif_open_resource_type_x(env, "nif_SUITE.monitor_frenzy",
+						      &frenzy_rt_init,
+						      ERL_NIF_RT_CREATE, NULL);
+
     atom_false = enif_make_atom(env,"false");
     atom_true = enif_make_atom(env,"true");
     atom_self = enif_make_atom(env,"self");
@@ -179,6 +236,12 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
     atom_eof = enif_make_atom(env, "eof");
     atom_error = enif_make_atom(env, "error");
     atom_fd_resource_stop = enif_make_atom(env, "fd_resource_stop");
+    atom_monitor_resource_type = enif_make_atom(env, "monitor_resource_type");
+    atom_monitor_resource_down = enif_make_atom(env, "monitor_resource_down");
+    atom_init = enif_make_atom(env,"init");
+    atom_stats = enif_make_atom(env,"stats");
+    atom_done = enif_make_atom(env,"done");
+    atom_stop = enif_make_atom(env,"stop");
 
     *priv_data = data;
     return 0;
@@ -232,6 +295,7 @@ static void unload(ErlNifEnv* env, void* priv_data)
 	}
 	enif_free(priv_data);
     }
+    DBG_TRACE_FINI;
 }
 
 static ERL_NIF_TERM lib_version(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
@@ -855,6 +919,9 @@ static ERL_NIF_TERM get_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
     if (enif_is_identical(argv[0], atom_binary_resource_type)) {
 	type.t = binary_resource_type;
     }
+    else if (enif_is_identical(argv[0], atom_monitor_resource_type)) {
+	type.t = monitor_resource_type;
+    }
     else {
 	get_pointer(env, argv[0], &type.vp);
     }
@@ -1496,7 +1563,6 @@ static ERL_NIF_TERM send_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
 
 static ERL_NIF_TERM send_copy_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    ErlNifEnv* menv;
     ErlNifPid pid;
     int ret;
     if (!enif_get_local_pid(env, argv[0], &pid)) {
@@ -2236,6 +2302,472 @@ static ERL_NIF_TERM last_fd_stop_call(ErlNifEnv* env, int argc, const ERL_NIF_TE
 }
 
 
+static void monitor_resource_dtor(ErlNifEnv* env, void* obj)
+{
+    resource_dtor(env, obj);
+}
+
+static ERL_NIF_TERM make_monitor(ErlNifEnv* env, const ErlNifMonitor* mon)
+{
+    ERL_NIF_TERM mon_bin;
+    memcpy(enif_make_new_binary(env, sizeof(ErlNifMonitor), &mon_bin),
+           mon, sizeof(ErlNifMonitor));
+    return mon_bin;
+}
+
+static int get_monitor(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifMonitor* mon)
+{
+    ErlNifBinary bin;
+    if (!enif_inspect_binary(env, term, &bin)
+        || bin.size != sizeof(ErlNifMonitor))
+        return 0;
+    memcpy(mon, bin.data, bin.size);
+    return 1;
+}
+
+static void monitor_resource_down(ErlNifEnv* env, void* obj, ErlNifPid* pid,
+                                  ErlNifMonitor* mon)
+{
+    struct monitor_resource* rsrc = (struct monitor_resource*)obj;
+    ErlNifEnv* build_env;
+    ErlNifEnv* msg_env;
+    ERL_NIF_TERM msg;
+
+    if (rsrc->use_msgenv) {
+        msg_env = enif_alloc_env();
+        build_env = msg_env;
+    }
+    else {
+        msg_env = NULL;
+        build_env = env;
+    }
+
+    msg = enif_make_tuple4(build_env,
+                           atom_monitor_resource_down,
+                           make_pointer(build_env, obj),
+                           enif_make_pid(build_env, pid),
+                           make_monitor(build_env, mon));
+
+    enif_send(env, &rsrc->receiver, msg_env, msg);
+    if (msg_env)
+        enif_free_env(msg_env);
+}
+
+static ERL_NIF_TERM alloc_monitor_resource_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    struct monitor_resource* rsrc;
+
+    rsrc  = enif_alloc_resource(monitor_resource_type, sizeof(struct monitor_resource));
+
+    return make_pointer(env,rsrc);
+}
+
+static ERL_NIF_TERM monitor_process_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    struct monitor_resource* rsrc;
+    ErlNifPid target;
+    ErlNifMonitor mon;
+    int res;
+
+    if (!get_pointer(env, argv[0], (void**)&rsrc)
+        || !enif_get_local_pid(env, argv[1], &target)
+        || !enif_get_local_pid(env, argv[3], &rsrc->receiver)) {
+        return enif_make_badarg(env);
+    }
+
+    rsrc->use_msgenv = (argv[2] == atom_true);
+    res = enif_monitor_process(env, rsrc, &target, &mon);
+
+    return enif_make_tuple2(env, enif_make_int(env, res), make_monitor(env, &mon));
+}
+
+static ERL_NIF_TERM demonitor_process_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    struct monitor_resource* rsrc;
+    ErlNifMonitor mon;
+    int res;
+
+    if (!get_pointer(env, argv[0], (void**)&rsrc)
+        || !get_monitor(env, argv[1], &mon)) {
+        return enif_make_badarg(env);
+    }
+
+    res = enif_demonitor_process(env, rsrc, &mon);
+
+    return enif_make_int(env, res);
+}
+
+/*********** monitor_frenzy ************/
+
+struct frenzy_rand_bits
+{
+    unsigned int source;
+    unsigned int bits_consumed;
+};
+
+static unsigned int frenzy_rand_bits_max;
+
+unsigned rand_bits(struct frenzy_rand_bits* rnd, unsigned int nbits)
+{
+    unsigned int res;
+
+    rnd->bits_consumed += nbits;
+    assert(rnd->bits_consumed <= frenzy_rand_bits_max);
+    res = rnd->source & ((1 << nbits)-1);
+    rnd->source >>= nbits;
+    return res;
+}
+
+#define FRENZY_PROCS_MAX_BITS 4
+#define FRENZY_PROCS_MAX (1 << FRENZY_PROCS_MAX_BITS)
+
+#define FRENZY_RESOURCES_MAX_BITS 4
+#define FRENZY_RESOURCES_MAX (1 << FRENZY_RESOURCES_MAX_BITS)
+
+#define FRENZY_MONITORS_MAX_BITS 4
+#define FRENZY_MONITORS_MAX (1 << FRENZY_MONITORS_MAX_BITS)
+
+struct frenzy_monitor {
+    ErlNifMutex* lock;
+    enum { 
+        MON_FREE, MON_FREE_DOWN, MON_FREE_DEMONITOR,
+        MON_TRYING, MON_ACTIVE, MON_PENDING
+    } state;
+    ErlNifMonitor mon;
+    ErlNifPid pid;
+    unsigned int use_cnt;
+};
+
+struct frenzy_resource {
+    unsigned int rix;
+    struct frenzy_monitor monv[FRENZY_MONITORS_MAX];
+};
+struct frenzy_reslot {
+    ErlNifMutex* lock;
+    struct frenzy_resource* obj;
+    unsigned long alloc_cnt;
+    unsigned long release_cnt;
+};
+static struct frenzy_reslot resv[FRENZY_RESOURCES_MAX];
+
+static ERL_NIF_TERM monitor_frenzy_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    struct frenzy_proc {
+        ErlNifPid pid;
+        int is_free;
+    };
+    static struct frenzy_proc procs[FRENZY_PROCS_MAX];
+    static struct frenzy_proc* proc_refs[FRENZY_PROCS_MAX];
+    static unsigned int nprocs, old_nprocs;
+    static ErlNifMutex* procs_lock;
+    static unsigned long spawn_cnt = 0;
+    static unsigned long kill_cnt = 0;
+    static unsigned long proc_histogram[FRENZY_PROCS_MAX];
+
+    static const unsigned int primes[] = {7, 13, 17, 19};
+
+    struct frenzy_resource* r;
+    struct frenzy_rand_bits rnd;
+    unsigned int op, inc, my_nprocs;
+    unsigned int mix;  /* r->monv[] index */
+    unsigned int rix;  /* resv[] index */
+    unsigned int pix;  /* procs[] index */
+    unsigned int ref_ix; /* proc_refs[] index */
+    int self_pix, rv;
+    ERL_NIF_TERM retval = atom_error;
+    const ERL_NIF_TERM Op = argv[0];
+    const ERL_NIF_TERM Rnd = argv[1];
+    const ERL_NIF_TERM SelfPix = argv[2];
+    const ERL_NIF_TERM NewPid = argv[3];
+
+    if (enif_is_atom(env, Op)) {
+        if (Op == atom_init) {
+            if (procs_lock || !enif_get_uint(env, Rnd, &frenzy_rand_bits_max))
+                return enif_make_badarg(env);
+
+            procs_lock = enif_mutex_create("nif_SUITE:monitor_frenzy.procs");
+            nprocs = 0;
+            old_nprocs = 0;
+            for (pix = 0; pix < FRENZY_PROCS_MAX; pix++) {
+                proc_refs[pix] = &procs[pix];
+                procs[pix].is_free = 1;
+                proc_histogram[pix] = 0;
+            }
+            for (rix = 0; rix < FRENZY_RESOURCES_MAX; rix++) {
+                resv[rix].lock = enif_mutex_create("nif_SUITE:monitor_frenzy.resv.lock");
+                resv[rix].obj = NULL;
+                resv[rix].alloc_cnt = 0;
+                resv[rix].release_cnt = 0;
+            }
+
+            /* Add self as first process */
+            enif_self(env, &procs[0].pid);
+            procs[0].is_free = 0;
+            old_nprocs = ++nprocs;
+
+            spawn_cnt = 1;
+            kill_cnt = 0;
+            return enif_make_uint(env, 0);  /* SelfPix */
+        }
+        else if (Op == atom_stats) {
+            ERL_NIF_TERM hist[FRENZY_PROCS_MAX];
+            unsigned long res_alloc_cnt = 0;
+            unsigned long res_release_cnt = 0;
+            for (ref_ix = 0; ref_ix < FRENZY_PROCS_MAX; ref_ix++) {
+                hist[ref_ix] = enif_make_ulong(env, proc_histogram[ref_ix]);
+            }
+            for (rix = 0; rix < FRENZY_RESOURCES_MAX; rix++) {
+                res_alloc_cnt += resv[rix].alloc_cnt;
+                res_release_cnt += resv[rix].release_cnt;
+            }
+
+            return
+            enif_make_list4(env,
+                            enif_make_tuple2(env, enif_make_string(env, "proc_histogram", ERL_NIF_LATIN1),
+                                             enif_make_list_from_array(env, hist, FRENZY_PROCS_MAX)),
+                            enif_make_tuple2(env, enif_make_string(env, "spawn_cnt", ERL_NIF_LATIN1),
+                                             enif_make_ulong(env, spawn_cnt)),
+                            enif_make_tuple2(env, enif_make_string(env, "kill_cnt", ERL_NIF_LATIN1),
+                                             enif_make_ulong(env, kill_cnt)),
+                            enif_make_tuple3(env, enif_make_string(env, "resource_alloc", ERL_NIF_LATIN1),
+                                             enif_make_ulong(env, res_alloc_cnt),
+                                             enif_make_ulong(env, res_release_cnt)));
+
+        }
+        else if (Op == atom_stop && procs_lock) {  /* stop all */
+            retval = enif_make_list(env, 0);
+            enif_mutex_lock(procs_lock);
+            for (ref_ix = 0; ref_ix < nprocs; ref_ix++) {
+                assert(!proc_refs[ref_ix]->is_free);
+                retval = enif_make_list_cell(env, enif_make_pid(env, &proc_refs[ref_ix]->pid),
+                                             retval);
+                proc_refs[ref_ix]->is_free = 1;
+            }
+            kill_cnt += nprocs;
+            nprocs = 0;
+            old_nprocs = 0;
+            enif_mutex_unlock(procs_lock);
+            return retval;
+        }
+        return enif_make_badarg(env);
+    }
+
+    if (!enif_get_int(env, SelfPix, &self_pix) ||
+        !enif_get_uint(env, Op, &op) ||
+        !enif_get_uint(env, Rnd, &rnd.source))
+        return enif_make_badarg(env);
+
+    rnd.bits_consumed = 0;
+    switch (op) {
+    case 0:  { /* add/remove process */
+        ErlNifPid self;
+        enif_self(env, &self);
+
+        ref_ix = rand_bits(&rnd, FRENZY_PROCS_MAX_BITS) % FRENZY_PROCS_MAX;
+        enif_mutex_lock(procs_lock);
+        if (procs[self_pix].is_free || procs[self_pix].pid.pid != self.pid) {
+            /* Some one already removed me */
+            enif_mutex_unlock(procs_lock);
+            return atom_done;
+        }
+        if (ref_ix >= nprocs || nprocs < 2) { /* add process */
+            ref_ix = nprocs++;
+            pix = proc_refs[ref_ix] - procs;
+            assert(procs[pix].is_free);
+            if (!enif_get_local_pid(env, NewPid, &procs[pix].pid))
+                abort();
+            procs[pix].is_free = 0;
+            spawn_cnt++;
+            proc_histogram[ref_ix]++;
+            old_nprocs = nprocs;
+            enif_mutex_unlock(procs_lock);
+            DBG_TRACE2("Add pid %T, nprocs = %u\n", NewPid, nprocs);
+            retval = enif_make_uint(env, pix);
+        }
+        else { /* remove process */
+            pix = proc_refs[ref_ix] - procs;
+            if (pix == self_pix) {
+                ref_ix = (ref_ix + 1) % nprocs;
+                pix = proc_refs[ref_ix] - procs;
+            }
+            assert(procs[pix].pid.pid != self.pid);
+            assert(!procs[pix].is_free);
+            retval = enif_make_pid(env, &procs[pix].pid);
+            --nprocs;
+            assert(!proc_refs[nprocs]->is_free);
+            if (ref_ix != nprocs) {
+                struct frenzy_proc* tmp = proc_refs[ref_ix];
+                proc_refs[ref_ix] = proc_refs[nprocs];
+                proc_refs[nprocs] = tmp;
+            }
+            procs[pix].is_free = 1;
+            proc_histogram[nprocs]++;
+            kill_cnt++;
+            enif_mutex_unlock(procs_lock);
+            DBG_TRACE2("Removed pid %T, nprocs = %u\n", retval, nprocs);
+        }
+	break;
+    }
+    case 1:
+    case 2: /* create/delete/lookup resource */
+	rix = rand_bits(&rnd, FRENZY_RESOURCES_MAX_BITS) % FRENZY_RESOURCES_MAX;
+	inc = primes[rand_bits(&rnd, 2)];
+	while (enif_mutex_trylock(resv[rix].lock) == EBUSY) {
+	    rix = (rix + inc) % FRENZY_RESOURCES_MAX;
+	}
+	if (resv[rix].obj == NULL) {
+	    r = enif_alloc_resource(frenzy_resource_type,
+				    sizeof(struct frenzy_resource));
+	    resv[rix].obj = r;
+            resv[rix].alloc_cnt++;
+	    r->rix = rix;
+            for (mix = 0; mix < FRENZY_MONITORS_MAX; mix++) {
+                r->monv[mix].lock = enif_mutex_create("nif_SUITE:monitor_frenzy.monv.lock");
+                r->monv[mix].state = MON_FREE;
+                r->monv[mix].use_cnt = 0;
+                r->monv[mix].pid.pid = 0; /* null-pid */
+            }
+            DBG_TRACE2("New resource at r=%p rix=%u\n", r, rix);
+	}
+	else if (rand_bits(&rnd, 3) == 0) {
+            r = resv[rix].obj;
+            resv[rix].obj = NULL;
+            resv[rix].release_cnt++;
+	    enif_mutex_unlock(resv[rix].lock);
+            DBG_TRACE2("Delete resource at r=%p rix=%u\n", r, rix);
+	    enif_release_resource(r);
+	    retval = atom_ok;
+	    break;
+	}
+	else {
+	    r = resv[rix].obj;
+	}
+        enif_keep_resource(r);
+        enif_mutex_unlock(resv[rix].lock);
+
+        /* monitor/demonitor */
+
+        mix = rand_bits(&rnd, FRENZY_MONITORS_MAX_BITS) % FRENZY_MONITORS_MAX;
+        inc = primes[rand_bits(&rnd, 2)];
+        while (enif_mutex_trylock(r->monv[mix].lock) == EBUSY) {
+            mix = (mix + inc) % FRENZY_MONITORS_MAX;
+        }
+        switch (r->monv[mix].state) {
+        case MON_FREE:
+        case MON_FREE_DOWN:
+        case MON_FREE_DEMONITOR: {   /* do monitor */
+            /*
+             * Use an old possibly larger value of 'nprocs', to increase
+             * probability of monitoring an already terminated process
+             */
+            my_nprocs = old_nprocs;
+            if (my_nprocs > 0) {
+                int save_state = r->monv[mix].state;
+                ref_ix = rand_bits(&rnd, FRENZY_PROCS_MAX_BITS) % my_nprocs;
+                pix = proc_refs[ref_ix] - procs;
+                r->monv[mix].pid.pid = procs[pix].pid.pid; /* "atomic" */
+                r->monv[mix].state = MON_TRYING;
+                rv = enif_monitor_process(env, r, &r->monv[mix].pid, &r->monv[mix].mon);
+                if (rv == 0) {
+                    r->monv[mix].state = MON_ACTIVE;
+                    r->monv[mix].use_cnt++;
+                    DBG_TRACE3("Monitor from r=%p rix=%u to %T\n",
+                                 r, r->rix, r->monv[mix].pid.pid);
+                }
+                else {
+                    r->monv[mix].state = save_state;
+                    DBG_TRACE4("Monitor from r=%p rix=%u to %T FAILED with %d\n",
+                               r, r->rix, r->monv[mix].pid.pid, rv);
+                }
+                retval = enif_make_int(env,rv);
+            }
+            else {
+                DBG_TRACE0("No pids to monitor\n");
+                retval = atom_ok;
+            }
+            break;
+        }
+        case MON_ACTIVE: /* do demonitor */
+            rv = enif_demonitor_process(env, r, &r->monv[mix].mon);
+            if (rv == 0) {
+                DBG_TRACE3("Demonitor from r=%p rix=%u to %T\n",
+                           r, r->rix, r->monv[mix].pid.pid);
+                r->monv[mix].state = MON_FREE_DEMONITOR;
+            }
+            else {
+                DBG_TRACE4("Demonitor from r=%p rix=%u to %T FAILED with %d\n",
+                           r, r->rix, r->monv[mix].pid.pid, rv);
+                r->monv[mix].state = MON_PENDING;
+            }
+            retval = enif_make_int(env,rv);
+            break;
+
+        case MON_PENDING: /* waiting for 'down' callback, do nothing */
+            retval = atom_ok;
+            break;
+        default:
+            abort();
+            break;
+        }
+        enif_mutex_unlock(r->monv[mix].lock);
+        enif_release_resource(r);
+	break;
+
+    case 3: /* no-op */
+        retval = atom_ok;
+        break;
+    }
+
+    {
+        int percent = (rand_bits(&rnd, 6) + 1) * 2;  /* 2 to 128 */
+        if (percent <= 100)
+            enif_consume_timeslice(env, percent);
+    }
+
+    return retval;
+}
+
+static void frenzy_resource_dtor(ErlNifEnv* env, void* obj)
+{
+    struct frenzy_resource* r = (struct frenzy_resource*) obj;
+    unsigned int mix;
+
+    DBG_TRACE2("DTOR r=%p rix=%u\n", r, r->rix);
+
+    for (mix = 0; mix < FRENZY_MONITORS_MAX; mix++) {
+        assert(r->monv[mix].state != MON_PENDING);
+        enif_mutex_destroy(r->monv[mix].lock);
+        r->monv[mix].lock = NULL;
+    }
+
+}
+
+static void frenzy_resource_down(ErlNifEnv* env, void* obj, ErlNifPid* pid,
+                                 ErlNifMonitor* mon)
+{
+    struct frenzy_resource* r = (struct frenzy_resource*) obj;
+    unsigned int mix;
+
+    DBG_TRACE3("DOWN pid=%T, r=%p rix=%u\n", pid->pid, r, r->rix);
+
+    for (mix = 0; mix < FRENZY_MONITORS_MAX; mix++) {
+        if (r->monv[mix].pid.pid == pid->pid && r->monv[mix].state >= MON_TRYING) {
+            enif_mutex_lock(r->monv[mix].lock);
+            if (memcmp(mon, &r->monv[mix].mon, sizeof(*mon)) == 0) {
+                assert(r->monv[mix].state >= MON_ACTIVE);
+                r->monv[mix].state = MON_FREE_DOWN;
+                enif_mutex_unlock(r->monv[mix].lock);
+                return;
+            }
+            enif_mutex_unlock(r->monv[mix].lock);
+        }
+    }
+    enif_fprintf(stderr, "DOWN called for unknown monitor\n");
+    abort();
+}
+
+
+
 static ErlNifFunc nif_funcs[] =
 {
     {"lib_version", 0, lib_version},
@@ -2318,7 +2850,11 @@ static ErlNifFunc nif_funcs[] =
     {"write_nif", 2, write_nif},
     {"read_nif", 2, read_nif},
     {"is_closed_nif", 1, is_closed_nif},
-    {"last_fd_stop_call", 0, last_fd_stop_call}
+    {"last_fd_stop_call", 0, last_fd_stop_call},
+    {"alloc_monitor_resource_nif", 0, alloc_monitor_resource_nif},
+    {"monitor_process_nif", 4, monitor_process_nif},
+    {"demonitor_process_nif", 2, demonitor_process_nif},
+    {"monitor_frenzy_nif", 4, monitor_frenzy_nif}
 };
 
 ERL_NIF_INIT(nif_SUITE,nif_funcs,load,NULL,upgrade,unload)
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.c b/erts/emulator/test/nif_SUITE_data/nif_mod.c
index e6106b6036..04699d3327 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_mod.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.c
@@ -176,6 +176,7 @@ static void do_load_info(ErlNifEnv* env, ERL_NIF_TERM load_info, int* retvalp)
     CHECK(enif_is_empty_list(env, head));
 }
 
+#if NIF_LIB_VER != 3
 static int load(ErlNifEnv* env, void** priv, ERL_NIF_TERM load_info)
 {
     NifModPrivData* data;
@@ -230,6 +231,7 @@ static void unload(ErlNifEnv* env, void* priv)
     add_call(env, data, "unload");
     NifModPrivData_release(data);
 }
+#endif /*  NIF_LIB_VER != 3 */
 
 static ERL_NIF_TERM lib_version(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-- 
cgit v1.2.3


From 1dadbe55812d462169ff87b6ba041d6e8c5829fb Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 7 Feb 2017 18:02:21 +0100
Subject: erts: Fix bad_fd_in_pollset error case for enif_select

---
 erts/emulator/sys/common/erl_check_io.c | 65 +++++++++++++++++++--------------
 1 file changed, 37 insertions(+), 28 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 5f4eca68d0..5c3042fb99 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -2119,7 +2119,7 @@ eready(Eterm id, ErtsDrvEventState *state, ErlDrvEventData event_data,
 }
 #endif
 
-static void bad_fd_in_pollset( ErtsDrvEventState *, Eterm, Eterm, ErtsPollEvents);
+static void bad_fd_in_pollset(ErtsDrvEventState *, Eterm inport, Eterm outport);
 
 #ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
 void
@@ -2280,9 +2280,8 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 	    }
 	    else if (revents & ERTS_POLL_EV_NVAL) {
 		bad_fd_in_pollset(state,
-				  state->driver.select->inport,
-				  state->driver.select->outport,
-				  state->events);
+                                  state->driver.select->inport,
+                                  state->driver.select->outport);
 		add_active_fd(state->fd);
 	    }
 	    break;
@@ -2336,7 +2335,8 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
                 }
             }
             else if (revents & ERTS_POLL_EV_NVAL) {
-                abort();
+                bad_fd_in_pollset(state, NIL, NIL);
+                add_active_fd(state->fd);
             }
 
 #ifdef ERTS_SMP
@@ -2400,9 +2400,9 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 }
 
 static void
-bad_fd_in_pollset(ErtsDrvEventState *state, Eterm inport, 
-		  Eterm outport, ErtsPollEvents events)
+bad_fd_in_pollset(ErtsDrvEventState *state, Eterm inport, Eterm outport)
 {
+    ErtsPollEvents events = state->events;
     erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
 
     if (events & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) {
@@ -2426,27 +2426,36 @@ bad_fd_in_pollset(ErtsDrvEventState *state, Eterm inport,
 	erts_dsprintf(dsbufp,
 		      "Bad %s fd in erts_poll()! fd=%d, ",
 		      io_str, (int) state->fd);
-	if (is_nil(port)) {
-	    ErtsPortNames *ipnp = erts_get_port_names(inport, ERTS_INVALID_ERL_DRV_PORT);
-	    ErtsPortNames *opnp = erts_get_port_names(outport, ERTS_INVALID_ERL_DRV_PORT);
-	    erts_dsprintf(dsbufp, "ports=%T/%T, drivers=%s/%s, names=%s/%s\n",
-			  is_nil(inport) ? am_undefined : inport,
-			  is_nil(outport) ? am_undefined : outport,
-			  ipnp->driver_name ? ipnp->driver_name : "<unknown>",
-			  opnp->driver_name ? opnp->driver_name : "<unknown>",
-			  ipnp->name ? ipnp->name : "<unknown>",
-			  opnp->name ? opnp->name : "<unknown>");
-	    erts_free_port_names(ipnp);
-	    erts_free_port_names(opnp);
-	}
-	else {
-	    ErtsPortNames *pnp = erts_get_port_names(port, ERTS_INVALID_ERL_DRV_PORT);
-	    erts_dsprintf(dsbufp, "port=%T, driver=%s, name=%s\n",
-			  is_nil(port) ? am_undefined : port,
-			  pnp->driver_name ? pnp->driver_name : "<unknown>",
-			  pnp->name ? pnp->name : "<unknown>");
-	    erts_free_port_names(pnp);
-	}
+        if (state->type == ERTS_EV_TYPE_DRV_SEL) {
+            if (is_nil(port)) {
+                ErtsPortNames *ipnp = erts_get_port_names(inport, ERTS_INVALID_ERL_DRV_PORT);
+                ErtsPortNames *opnp = erts_get_port_names(outport, ERTS_INVALID_ERL_DRV_PORT);
+                erts_dsprintf(dsbufp, "ports=%T/%T, drivers=%s/%s, names=%s/%s\n",
+                              is_nil(inport) ? am_undefined : inport,
+                              is_nil(outport) ? am_undefined : outport,
+                              ipnp->driver_name ? ipnp->driver_name : "<unknown>",
+                              opnp->driver_name ? opnp->driver_name : "<unknown>",
+                              ipnp->name ? ipnp->name : "<unknown>",
+                              opnp->name ? opnp->name : "<unknown>");
+                erts_free_port_names(ipnp);
+                erts_free_port_names(opnp);
+            }
+            else {
+                ErtsPortNames *pnp = erts_get_port_names(port, ERTS_INVALID_ERL_DRV_PORT);
+                erts_dsprintf(dsbufp, "port=%T, driver=%s, name=%s\n",
+                              is_nil(port) ? am_undefined : port,
+                              pnp->driver_name ? pnp->driver_name : "<unknown>",
+                              pnp->name ? pnp->name : "<unknown>");
+                erts_free_port_names(pnp);
+            }
+        }
+        else {
+            ErlNifResourceType* rt;
+            ASSERT(state->type == ERTS_EV_TYPE_NIF);
+            ASSERT(state->driver.stop.resource);
+            rt = state->driver.stop.resource->type;
+            erts_dsprintf(dsbufp, "resource={%T,%T}\n", rt->module, rt->name);
+        }
     }
     else {
 	erts_dsprintf(dsbufp, "Bad fd in erts_poll()! fd=%d\n", (int) state->fd);
-- 
cgit v1.2.3


From 2d3de607e346e6b965f410e8c4e126cd38c6603e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 7 Feb 2017 18:13:58 +0100
Subject: erts: Beautify enif_select

indentation and comments only
---
 erts/emulator/sys/common/erl_check_io.c | 147 ++++++++++++++++----------------
 1 file changed, 72 insertions(+), 75 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 5c3042fb99..c8d958e72f 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1342,87 +1342,84 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
 
     state->events = new_events;
     if (on) {
-            Uint32* refn;
-	    if (!state->driver.nif)
-		state->driver.nif = alloc_nif_select_data();
-	    if (state->type == ERTS_EV_TYPE_NONE) {
-		state->type = ERTS_EV_TYPE_NIF;
-                state->driver.stop.resource = resource;
-                enif_keep_resource(resource->data);
-            }
-            ASSERT(state->type == ERTS_EV_TYPE_NIF);
-            ASSERT(state->driver.stop.resource == resource);
-	    if (ctl_events & ERTS_POLL_EV_IN) {
-		state->driver.nif->in.pid = id;
-                if (is_immed(ref)) {
-                    state->driver.nif->in.immed = ref;
-                } else {
-                    ASSERT(is_internal_ref(ref));
-                    refn = internal_ref_numbers(ref);
-                    state->driver.nif->in.immed = THE_NON_VALUE;
-                    state->driver.nif->in.refn[0] = refn[0];
-                    state->driver.nif->in.refn[1] = refn[1];
-                    state->driver.nif->in.refn[2] = refn[2];
-                }
-                state->driver.nif->in.ddeselect_cnt = 0;
+        Uint32* refn;
+        if (!state->driver.nif)
+            state->driver.nif = alloc_nif_select_data();
+        if (state->type == ERTS_EV_TYPE_NONE) {
+            state->type = ERTS_EV_TYPE_NIF;
+            state->driver.stop.resource = resource;
+            enif_keep_resource(resource->data);
+        }
+        ASSERT(state->type == ERTS_EV_TYPE_NIF);
+        ASSERT(state->driver.stop.resource == resource);
+        if (ctl_events & ERTS_POLL_EV_IN) {
+            state->driver.nif->in.pid = id;
+            if (is_immed(ref)) {
+                state->driver.nif->in.immed = ref;
+            } else {
+                ASSERT(is_internal_ref(ref));
+                refn = internal_ref_numbers(ref);
+                state->driver.nif->in.immed = THE_NON_VALUE;
+                state->driver.nif->in.refn[0] = refn[0];
+                state->driver.nif->in.refn[1] = refn[1];
+                state->driver.nif->in.refn[2] = refn[2];
             }
-	    if (ctl_events & ERTS_POLL_EV_OUT) {
-		state->driver.nif->out.pid = id;
-                if (is_immed(ref)) {
-                    state->driver.nif->out.immed = ref;
-                } else {
-                    ASSERT(is_internal_ref(ref));
-                    refn = internal_ref_numbers(ref);
-                    state->driver.nif->out.immed = THE_NON_VALUE;
-                    state->driver.nif->out.refn[0] = refn[0];
-                    state->driver.nif->out.refn[1] = refn[1];
-                    state->driver.nif->out.refn[2] = refn[2];
-                }
-                state->driver.nif->out.ddeselect_cnt = 0;
+            state->driver.nif->in.ddeselect_cnt = 0;
+        }
+        if (ctl_events & ERTS_POLL_EV_OUT) {
+            state->driver.nif->out.pid = id;
+            if (is_immed(ref)) {
+                state->driver.nif->out.immed = ref;
+            } else {
+                ASSERT(is_internal_ref(ref));
+                refn = internal_ref_numbers(ref);
+                state->driver.nif->out.immed = THE_NON_VALUE;
+                state->driver.nif->out.refn[0] = refn[0];
+                state->driver.nif->out.refn[1] = refn[1];
+                state->driver.nif->out.refn[2] = refn[2];
             }
-            ret = 0;
+            state->driver.nif->out.ddeselect_cnt = 0;
+        }
+        ret = 0;
     }
     else { /* off */
-	    if (state->type == ERTS_EV_TYPE_NIF) {
-                //erts_fprintf(stderr, "SVERK: enif select clear fd=%d inpid=%T inrsrc=%p\n",
-                //             state->fd, state->driver.nif->inpid,
-                //             state->driver.nif->in.resource);
-                state->driver.nif->in.pid = NIL;
-                state->driver.nif->out.pid = NIL;
-                state->driver.nif->in.ddeselect_cnt = 0;
-                state->driver.nif->out.ddeselect_cnt = 0;
-                if (old_events != 0) {
-                    remember_removed(state, &pollset);
-                }
-	    }
-            ASSERT(new_events==0);
-            if (state->remove_cnt == 0 || !wake_poller) {
-                /* Safe to close fd now as it is not in pollset
-		   or there was no need to eject fd (kernel poll) */
-                //erts_fprintf(stderr, "SVERK : enif_select calling stop before return\n");
-                if (state->type == ERTS_EV_TYPE_NIF) {
-                    ASSERT(state->driver.stop.resource == resource);
-                    call_stop = CALL_STOP_AND_RELEASE;
-                    state->driver.stop.resource = NULL;
-                }
-                else {
-                    ASSERT(!state->driver.stop.resource);
-                    call_stop = CALL_STOP;
-                }
-                state->type = ERTS_EV_TYPE_NONE;
-                ret = ERL_NIF_SELECT_STOP_CALLED;
+        if (state->type == ERTS_EV_TYPE_NIF) {
+            state->driver.nif->in.pid = NIL;
+            state->driver.nif->out.pid = NIL;
+            state->driver.nif->in.ddeselect_cnt = 0;
+            state->driver.nif->out.ddeselect_cnt = 0;
+            if (old_events != 0) {
+                remember_removed(state, &pollset);
+            }
+        }
+        ASSERT(new_events==0);
+        if (state->remove_cnt == 0 || !wake_poller) {
+            /*
+             * Safe to close fd now as it is not in pollset
+             * or there was no need to eject fd (kernel poll)
+             */
+            if (state->type == ERTS_EV_TYPE_NIF) {
+                ASSERT(state->driver.stop.resource == resource);
+                call_stop = CALL_STOP_AND_RELEASE;
+                state->driver.stop.resource = NULL;
             }
             else {
-                /* Not safe to close fd, postpone stop_select callback. */
-                //erts_fprintf(stderr, "SVERK: enif_select schedule stop\n");
-                if (state->type == ERTS_EV_TYPE_NONE) {
-                    ASSERT(!state->driver.stop.resource);
-                    state->driver.stop.resource = resource;
-                    enif_keep_resource(resource);
-                }
-                state->type = ERTS_EV_TYPE_STOP_NIF;
-                ret = ERL_NIF_SELECT_STOP_SCHEDULED;
-	    }
+                ASSERT(!state->driver.stop.resource);
+                call_stop = CALL_STOP;
+            }
+            state->type = ERTS_EV_TYPE_NONE;
+            ret = ERL_NIF_SELECT_STOP_CALLED;
+        }
+        else {
+            /* Not safe to close fd, postpone stop_select callback. */
+            if (state->type == ERTS_EV_TYPE_NONE) {
+                ASSERT(!state->driver.stop.resource);
+                state->driver.stop.resource = resource;
+                enif_keep_resource(resource);
+            }
+            state->type = ERTS_EV_TYPE_STOP_NIF;
+            ret = ERL_NIF_SELECT_STOP_SCHEDULED;
+        }
     }
 
 done:
-- 
cgit v1.2.3


From d85e74e0c0e4bc66c875e2fd5f54d89255df0047 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 9 Feb 2017 15:23:11 +0100
Subject: erts: Add pid argument to enif_select

---
 erts/emulator/beam/erl_nif_api_funcs.h        |  2 +-
 erts/emulator/sys/common/erl_check_io.c       |  7 ++--
 erts/emulator/sys/common/erl_check_io.h       |  4 +-
 erts/emulator/sys/unix/sys.c                  |  6 +--
 erts/emulator/test/nif_SUITE.erl              | 54 +++++++++++++++++----------
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 16 ++++++--
 6 files changed, 57 insertions(+), 32 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index a9ed246962..1fc6842acc 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -175,7 +175,7 @@ ERL_NIF_API_FUNC_DECL(size_t, enif_binary_to_term, (ErlNifEnv *env, const unsign
 ERL_NIF_API_FUNC_DECL(int, enif_port_command, (ErlNifEnv *env, const ErlNifPort* to_port, ErlNifEnv *msg_env, ERL_NIF_TERM msg));
 ERL_NIF_API_FUNC_DECL(int,enif_thread_type,(void));
 ERL_NIF_API_FUNC_DECL(int,enif_snprintf,(char * buffer, size_t size, const char *format, ...));
-ERL_NIF_API_FUNC_DECL(int,enif_select,(ErlNifEnv* env, ErlNifEvent e, enum ErlNifSelectFlags flags, void* obj, ERL_NIF_TERM ref));
+ERL_NIF_API_FUNC_DECL(int,enif_select,(ErlNifEnv* env, ErlNifEvent e, enum ErlNifSelectFlags flags, void* obj, const ErlNifPid* pid, ERL_NIF_TERM ref));
 ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type_x,(ErlNifEnv*, const char* name_str, const ErlNifResourceTypeInit*, ErlNifResourceFlags flags, ErlNifResourceFlags* tried));
 ERL_NIF_API_FUNC_DECL(int, enif_monitor_process,(ErlNifEnv*,void* obj,const ErlNifPid*,ErlDrvMonitor *monitor));
 ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const ErlDrvMonitor *monitor));
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index c8d958e72f..089f10fd8e 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1203,10 +1203,10 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
                              ErlNifEvent e,
                              enum ErlNifSelectFlags mode,
                              void* obj,
+                             const ErlNifPid* pid,
                              Eterm ref)
 {
     int on;
-    const Eterm id = env->proc->common.id;
     ErtsResource* resource = DATA_TO_RESOURCE(obj);
     ErtsSysFdType fd = (ErtsSysFdType) e;
     ErtsPollEvents ctl_events = (ErtsPollEvents) 0;
@@ -1342,6 +1342,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
 
     state->events = new_events;
     if (on) {
+        const Eterm recipient = pid ? pid->pid : env->proc->common.id;
         Uint32* refn;
         if (!state->driver.nif)
             state->driver.nif = alloc_nif_select_data();
@@ -1353,7 +1354,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
         ASSERT(state->type == ERTS_EV_TYPE_NIF);
         ASSERT(state->driver.stop.resource == resource);
         if (ctl_events & ERTS_POLL_EV_IN) {
-            state->driver.nif->in.pid = id;
+            state->driver.nif->in.pid = recipient;
             if (is_immed(ref)) {
                 state->driver.nif->in.immed = ref;
             } else {
@@ -1367,7 +1368,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
             state->driver.nif->in.ddeselect_cnt = 0;
         }
         if (ctl_events & ERTS_POLL_EV_OUT) {
-            state->driver.nif->out.pid = id;
+            state->driver.nif->out.pid = recipient;
             if (is_immed(ref)) {
                 state->driver.nif->out.immed = ref;
             } else {
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index 41e2e9210a..4f9efeefcd 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -34,8 +34,8 @@
 
 int driver_select_kp(ErlDrvPort, ErlDrvEvent, int, int);
 int driver_select_nkp(ErlDrvPort, ErlDrvEvent, int, int);
-enum ErlNifSelectReturn enif_select_kp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, Eterm);
-enum ErlNifSelectReturn enif_select_nkp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, Eterm);
+enum ErlNifSelectReturn enif_select_kp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
+enum ErlNifSelectReturn enif_select_nkp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
 int driver_event_kp(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
 int driver_event_nkp(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
 Uint erts_check_io_size_kp(void);
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index a852550915..4843ee4ba2 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -161,7 +161,7 @@ int erts_use_kernel_poll = 0;
 
 struct {
     int (*select)(ErlDrvPort, ErlDrvEvent, int, int);
-    enum ErlNifSelectReturn (*enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, Eterm);
+    enum ErlNifSelectReturn (*enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
     int (*event)(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
     void (*check_io_as_interrupt)(void);
     void (*check_io_interrupt)(int);
@@ -186,9 +186,9 @@ driver_event(ErlDrvPort port, ErlDrvEvent event, ErlDrvEventData event_data)
 }
 
 int enif_select(ErlNifEnv* env, ErlNifEvent event,
-                enum ErlNifSelectFlags flags, void* obj, Eterm ref)
+                enum ErlNifSelectFlags flags, void* obj, const ErlNifPid* pid, Eterm ref)
 {
-    return (*io_func.enif_select)(env, event, flags, obj, ref);
+    return (*io_func.enif_select)(env, event, flags, obj, pid, ref);
 }
 
 
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index e3af9f7454..de26e73c3d 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -468,21 +468,31 @@ select(Config) when is_list(Config) ->
     ensure_lib_loaded(Config),
 
     Ref = make_ref(),
+    Ref2 = make_ref(),
     {{R, R_ptr}, {W, W_ptr}} = pipe_nif(),
     ok = write_nif(W, <<"hej">>),
     <<"hej">> = read_nif(R, 3),
 
     %% Wait for read
     eagain = read_nif(R, 3),
-    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,Ref),
+    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,null,Ref),
     [] = flush(),
     ok = write_nif(W, <<"hej">>),
     [{select, R, Ref, ready_input}] = flush(),
+    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,self(),Ref2),
+    [{select, R, Ref2, ready_input}] = flush(),
+    Papa = self(),
+    Pid = spawn_link(fun() ->
+                             [{select, R, Ref, ready_input}] = flush(),
+                             Papa ! {self(), done}
+                     end),
+    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,Pid,Ref),
+    {Pid, done} = receive_any(1000),
     <<"hej">> = read_nif(R, 3),
 
     %% Wait for write
     Written = write_full(W, $a),
-    0 = select_nif(W,?ERL_NIF_SELECT_WRITE,W,Ref),
+    0 = select_nif(W,?ERL_NIF_SELECT_WRITE,W,self(),Ref),
     [] = flush(),
     Half = byte_size(Written) div 2,
     <<First:Half/binary,Second/binary>> = Written,
@@ -494,16 +504,16 @@ select(Config) when is_list(Config) ->
 
     %% Close write and wait for EOF
     eagain = read_nif(R, 1),
-    check_stop_ret(select_nif(W,?ERL_NIF_SELECT_STOP,W,Ref)),
+    check_stop_ret(select_nif(W,?ERL_NIF_SELECT_STOP,W,null,Ref)),
     [{fd_resource_stop, W_ptr, _}] = flush(),
     {1, {W_ptr,_}} = last_fd_stop_call(),
     true = is_closed_nif(W),
     [] = flush(),
-    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,Ref),
+    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,self(),Ref),
     [{select, R, Ref, ready_input}] = flush(),
     eof = read_nif(R,1),
 
-    check_stop_ret(select_nif(R,?ERL_NIF_SELECT_STOP,R,Ref)),
+    check_stop_ret(select_nif(R,?ERL_NIF_SELECT_STOP,R,null,Ref)),
     [{fd_resource_stop, R_ptr, _}] = flush(),
     {1, {R_ptr,_}} = last_fd_stop_call(),
     true = is_closed_nif(R),
@@ -520,8 +530,8 @@ select_2(Config) ->
 
     %% Change ref
     eagain = read_nif(R, 1),
-    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,Ref1),
-    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,Ref2),
+    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,null,Ref1),
+    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,self(),Ref2),
 
     [] = flush(),
     ok = write_nif(W, <<"hej">>),
@@ -530,35 +540,35 @@ select_2(Config) ->
 
     %% Change pid
     eagain = read_nif(R, 1),
-    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,Ref1),
+    0 = select_nif(R,?ERL_NIF_SELECT_READ,R,null,Ref1),
     Papa = self(),
-    Pid2 = spawn_link(fun() ->
-                              0 = select_nif(R,?ERL_NIF_SELECT_READ,R,Ref1),
-                              [] = flush(),
-                              Papa ! sync,
-                              [{select, R, Ref1, ready_input}] = flush(),
-                              <<"hej">> = read_nif(R, 3),
-                              Papa ! done
-                      end),
+    spawn_link(fun() ->
+                       0 = select_nif(R,?ERL_NIF_SELECT_READ,R,null,Ref1),
+                       [] = flush(),
+                       Papa ! sync,
+                       [{select, R, Ref1, ready_input}] = flush(),
+                       <<"hej">> = read_nif(R, 3),
+                       Papa ! done
+               end),
     sync = receive_any(),
     ok = write_nif(W, <<"hej">>),
     done = receive_any(),
     [] = flush(),
 
-    check_stop_ret(select_nif(R,?ERL_NIF_SELECT_STOP,R,Ref1)),
+    check_stop_ret(select_nif(R,?ERL_NIF_SELECT_STOP,R,null,Ref1)),
     [{fd_resource_stop, R_ptr, _}] = flush(),
     {1, {R_ptr,_}} = last_fd_stop_call(),
     true = is_closed_nif(R),
 
     %% Stop without previous read/write select
-    ?ERL_NIF_SELECT_STOP_CALLED = select_nif(W,?ERL_NIF_SELECT_STOP,W,Ref1),
+    ?ERL_NIF_SELECT_STOP_CALLED = select_nif(W,?ERL_NIF_SELECT_STOP,W,null,Ref1),
     [{fd_resource_stop, W_ptr, 1}] = flush(),
     {1, {W_ptr,1}} = last_fd_stop_call(),
     true = is_closed_nif(W),
 
     select_3(Config).
 
-select_3(Config) ->
+select_3(_Config) ->
     erlang:garbage_collect(),
     {_,_,2} = last_resource_dtor_call(),
     ok.
@@ -2263,6 +2273,10 @@ call(Pid,Cmd) ->
 receive_any() ->
     receive M -> M end.	     
 
+receive_any(Timeout) ->
+    receive M -> M
+    after Timeout -> timeout end.
+
 flush() ->
     flush(10).
 flush(Timeout) ->
@@ -2635,7 +2649,7 @@ term_to_binary_nif(_, _) -> ?nif_stub.
 binary_to_term_nif(_, _, _) -> ?nif_stub.
 port_command_nif(_, _) -> ?nif_stub.
 format_term_nif(_,_) -> ?nif_stub.
-select_nif(_,_,_,_) -> ?nif_stub.
+select_nif(_,_,_,_,_) -> ?nif_stub.
 pipe_nif() -> ?nif_stub.
 write_nif(_,_) -> ?nif_stub.
 read_nif(_,_) -> ?nif_stub.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index ee925512d2..6cf02c6efe 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -75,6 +75,7 @@ static ERL_NIF_TERM atom_init;
 static ERL_NIF_TERM atom_stats;
 static ERL_NIF_TERM atom_done;
 static ERL_NIF_TERM atom_stop;
+static ERL_NIF_TERM atom_null;
 
 typedef struct
 {
@@ -242,6 +243,7 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
     atom_stats = enif_make_atom(env,"stats");
     atom_done = enif_make_atom(env,"done");
     atom_stop = enif_make_atom(env,"stop");
+    atom_null = enif_make_atom(env,"null");
 
     *priv_data = data;
     return 0;
@@ -2119,6 +2121,7 @@ static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
     struct fd_resource* fdr;
     enum ErlNifSelectFlags mode;
     void* obj;
+    ErlNifPid nifpid, *pid = NULL;
     ERL_NIF_TERM ref;
     enum ErlNifSelectReturn retval;
 
@@ -2129,11 +2132,16 @@ static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
         return enif_make_badarg(env);
     }
 
-    ref = argv[3];
+    if (argv[3] != atom_null) {
+	if (!enif_get_local_pid(env, argv[3], &nifpid))
+	    return enif_make_badarg(env);
+	pid = &nifpid;
+    }
+    ref = argv[4];
 
     fdr->was_selected = 1;
     enif_self(env, &fdr->pid);
-    retval = enif_select(env, fdr->fd, mode, obj, ref);
+    retval = enif_select(env, fdr->fd, mode, obj, pid, ref);
 
     return enif_make_int(env, (int)retval);
 }
@@ -2160,7 +2168,9 @@ static ERL_NIF_TERM pipe_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]
     read_rsrc  = enif_alloc_resource(fd_resource_type, sizeof(struct fd_resource));
     write_rsrc = enif_alloc_resource(fd_resource_type, sizeof(struct fd_resource));
     read_rsrc->fd  = fds[0];
+    read_rsrc->was_selected = 0;
     write_rsrc->fd = fds[1];
+    write_rsrc->was_selected = 0;
     read_fd  = enif_make_resource(env, read_rsrc);
     write_fd = enif_make_resource(env, write_rsrc);
     enif_release_resource(read_rsrc);
@@ -2845,7 +2855,7 @@ static ErlNifFunc nif_funcs[] =
     {"binary_to_term_nif", 3, binary_to_term},
     {"port_command_nif", 2, port_command},
     {"format_term_nif", 2, format_term},
-    {"select_nif", 4, select_nif},
+    {"select_nif", 5, select_nif},
     {"pipe_nif", 0, pipe_nif},
     {"write_nif", 2, write_nif},
     {"read_nif", 2, read_nif},
-- 
cgit v1.2.3


From a8d5234c77634df9522727ff200cef4fcab49c22 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 9 Feb 2017 17:30:30 +0100
Subject: erts: Change return value for enif_select

to negative int as error and positive as success.
---
 erts/emulator/beam/erl_drv_nif.h              |  8 --------
 erts/emulator/beam/erl_nif.h                  |  6 ++++++
 erts/emulator/sys/common/erl_check_io.c       | 14 +++++++++-----
 erts/emulator/sys/common/erl_check_io.h       |  4 ++--
 erts/emulator/sys/unix/sys.c                  |  2 +-
 erts/emulator/test/nif_SUITE.erl              |  9 ++++-----
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c |  4 ++--
 7 files changed, 24 insertions(+), 23 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h
index 8de4a7855d..2d21dac87a 100644
--- a/erts/emulator/beam/erl_drv_nif.h
+++ b/erts/emulator/beam/erl_drv_nif.h
@@ -56,14 +56,6 @@ enum ErlNifSelectFlags {
     ERL_NIF_SELECT_STOP      = (1 << 2)
 };
 
-enum ErlNifSelectReturn {
-    ERL_NIF_SELECT_ERROR          = (1 << 0),
-    ERL_NIF_SELECT_STOP_CALLED    = (1 << 1),
-    ERL_NIF_SELECT_STOP_SCHEDULED = (1 << 2),
-    ERL_NIF_SELECT_INVALID_EVENT  = (1 << 3),
-    ERL_NIF_SELECT_FAILED         = (1 << 4)
-};
-
 /*
  * A driver monitor
  */
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 5248f287ee..9ff28a30cc 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -142,6 +142,12 @@ typedef struct
 typedef int ErlNifEvent; /* An event to be selected on. */
 //#endif
 
+/* Return bits from enif_select: */
+#define ERL_NIF_SELECT_STOP_CALLED    (1 << 0)
+#define ERL_NIF_SELECT_STOP_SCHEDULED (1 << 1)
+#define ERL_NIF_SELECT_INVALID_EVENT  (1 << 2)
+#define ERL_NIF_SELECT_FAILED         (1 << 3)
+
 typedef enum
 {
     ERL_NIF_RT_CREATE = 1,
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 089f10fd8e..2964b09b8c 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1198,7 +1198,7 @@ done_unknown:
     return ret;
 }
 
-enum ErlNifSelectReturn
+int
 ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
                              ErlNifEvent e,
                              enum ErlNifSelectFlags mode,
@@ -1213,7 +1213,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
     ErtsPollEvents new_events, old_events;
     ErtsDrvEventState *state;
     int wake_poller;
-    enum ErlNifSelectReturn ret;
+    int ret;
     enum { NO_STOP=0, CALL_STOP, CALL_STOP_AND_RELEASE } call_stop = NO_STOP;
 #if ERTS_CIO_HAVE_DRV_EVENT
     ErtsDrvEventDataState *free_event = NULL;
@@ -1229,11 +1229,11 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) {
 	if (fd < 0) {
-	    return ERL_NIF_SELECT_ERROR | ERL_NIF_SELECT_INVALID_EVENT;
+	    return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT;
 	}
 	if (fd >= max_fds) {
 	    nif_select_large_fd_error(fd, mode, resource, ref);
-            return ERL_NIF_SELECT_ERROR | ERL_NIF_SELECT_INVALID_EVENT;
+            return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT;
 	}
 	grow_drv_ev_state(fd);
     }
@@ -1327,7 +1327,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
             state->driver.nif->out.ddeselect_cnt = 0;
             state->driver.stop.resource = NULL;
 	}
-	ret = ERL_NIF_SELECT_ERROR | ERL_NIF_SELECT_FAILED;
+	ret = INT_MIN | ERL_NIF_SELECT_FAILED;
 	goto done;
     }
 
@@ -2518,6 +2518,10 @@ static void drv_ev_state_free(void *des)
 void
 ERTS_CIO_EXPORT(erts_init_check_io)(void)
 {
+    ERTS_CT_ASSERT((INT_MIN & (ERL_NIF_SELECT_STOP_CALLED |
+                               ERL_NIF_SELECT_STOP_SCHEDULED |
+                               ERL_NIF_SELECT_INVALID_EVENT |
+                               ERL_NIF_SELECT_FAILED)) == 0);
     erts_smp_atomic_init_nob(&erts_check_io_time, 0);
     erts_smp_atomic_init_nob(&pollset.in_poll_wait, 0);
 
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index 4f9efeefcd..f02d6c1f62 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -34,8 +34,8 @@
 
 int driver_select_kp(ErlDrvPort, ErlDrvEvent, int, int);
 int driver_select_nkp(ErlDrvPort, ErlDrvEvent, int, int);
-enum ErlNifSelectReturn enif_select_kp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
-enum ErlNifSelectReturn enif_select_nkp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
+int enif_select_kp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
+int enif_select_nkp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
 int driver_event_kp(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
 int driver_event_nkp(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
 Uint erts_check_io_size_kp(void);
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 4843ee4ba2..f4bdc129e1 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -161,7 +161,7 @@ int erts_use_kernel_poll = 0;
 
 struct {
     int (*select)(ErlDrvPort, ErlDrvEvent, int, int);
-    enum ErlNifSelectReturn (*enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
+    int (*enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
     int (*event)(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
     void (*check_io_as_interrupt)(void);
     void (*check_io_interrupt)(int);
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index de26e73c3d..d88ac01e46 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -457,11 +457,10 @@ t_on_load(Config) when is_list(Config) ->
 -define(ERL_NIF_SELECT_WRITE, (1 bsl 1)).
 -define(ERL_NIF_SELECT_STOP, (1 bsl 2)).
 
--define(ERL_NIF_SELECT_ERROR, (1 bsl 0)).
--define(ERL_NIF_SELECT_STOP_CALLED, (1 bsl 1)).
--define(ERL_NIF_SELECT_STOP_SCHEDULED, (1 bsl 2)).
--define(ERL_NIF_SELECT_INVALID_EVENT, (1 bsl 3)).
--define(ERL_NIF_SELECT_FAILED, (1 bsl 4)).
+-define(ERL_NIF_SELECT_STOP_CALLED, (1 bsl 0)).
+-define(ERL_NIF_SELECT_STOP_SCHEDULED, (1 bsl 1)).
+-define(ERL_NIF_SELECT_INVALID_EVENT, (1 bsl 2)).
+-define(ERL_NIF_SELECT_FAILED, (1 bsl 3)).
 
 
 select(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 6cf02c6efe..05b4b05e16 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -2123,7 +2123,7 @@ static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
     void* obj;
     ErlNifPid nifpid, *pid = NULL;
     ERL_NIF_TERM ref;
-    enum ErlNifSelectReturn retval;
+    int retval;
 
     if (!get_fd(env, argv[0], &fdr)
         || !enif_get_uint(env, argv[1], &mode)
@@ -2143,7 +2143,7 @@ static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
     enif_self(env, &fdr->pid);
     retval = enif_select(env, fdr->fd, mode, obj, pid, ref);
 
-    return enif_make_int(env, (int)retval);
+    return enif_make_int(env, retval);
 }
 
 static ERL_NIF_TERM pipe_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
-- 
cgit v1.2.3


From e93c9afd2415e3d500fe631d047c75fe47487baf Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 9 Feb 2017 19:51:15 +0100
Subject: erts: Try fix enif_select for windows

by simply disable "delayed deselect".
---
 erts/emulator/sys/common/erl_check_io.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 2964b09b8c..352e51a877 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -753,10 +753,6 @@ check_cleanup_active_fd(ErtsSysFdType fd,
 	}
 
         if (state->driver.nif) {
-#if ERTS_CIO_DEFER_ACTIVE_EVENTS
-#  error Windows
-#endif
-            int do_wake = 0;
             ErtsPollEvents rm_events = 0;
             if (state->driver.nif->in.ddeselect_cnt) {
                 ASSERT(state->type == ERTS_EV_TYPE_NIF);
@@ -777,7 +773,9 @@ check_cleanup_active_fd(ErtsSysFdType fd,
                 }
             }
             if (rm_events) {
-                state->events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, rm_events, 0, &do_wake);
+                int do_wake = 0;
+                state->events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd,
+                                                  rm_events, 0, &do_wake);
             }
             if (state->events)
                 active = 1;
@@ -2140,11 +2138,16 @@ ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(int set,
     ERTS_CIO_POLL_INTR_TMD(pollset.ps, set, timeout_time);
 }
 
+#if !ERTS_CIO_DEFER_ACTIVE_EVENTS
 /*
  * Number of ignored events, for a lingering fd added by enif_select(),
  * until we deselect fd-event from pollset.
  */
-#define ERTS_NIF_DELAYED_DESELECT 20
+# define ERTS_NIF_DELAYED_DESELECT 20
+#else
+/* Disable delayed deselect as pollset cannot handle active events */
+# define ERTS_NIF_DELAYED_DESELECT 1
+#endif
 
 void
 ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
-- 
cgit v1.2.3


From 7e9f0932f9d06edca0c30d4a2b9d57f34677344c Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 31 Jan 2017 13:30:20 +0100
Subject: Magic indirection

---
 erts/emulator/beam/erl_alloc.c     |  2 ++
 erts/emulator/beam/erl_alloc.types |  1 +
 erts/emulator/beam/erl_binary.h    | 44 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 47 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index e433de0f35..6708d96d56 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -684,6 +684,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	= sizeof(NifExportTrace);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MREF_NSCHED_ENT)]
 	= sizeof(ErtsNSchedMagicRefTableEntry);
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MINDIRECTION)]
+	= ERTS_MAGIC_BIN_UNALIGNED_SIZE(sizeof(ErtsMagicIndirectionWord));
 
 #ifdef HARD_DEBUG
     hdbg_init();
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index c253ea82c8..f88985b7f5 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -284,6 +284,7 @@ type    MREF_NSCHED_ENT	FIXED_SIZE	SYSTEM		nsched_magic_ref_entry
 type    MREF_ENT	STANDARD	SYSTEM		magic_ref_entry
 type	MREF_TAB_BKTS	STANDARD	SYSTEM		magic_ref_table_buckets
 type	MREF_TAB	LONG_LIVED	SYSTEM		magic_ref_table
+type	MINDIRECTION	FIXED_SIZE	SYSTEM		magic_indirection
 
 +if threads_no_smp
 # Need thread safe allocs, but std_alloc and fix_alloc are not;
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index c811a002ce..4d9c4d6eac 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -300,6 +300,17 @@ BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is
 BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen);
 
 
+typedef union {
+    /*
+     * These two are almost always of
+     * the same size, but when fallback
+     * atomics are used they might
+     * differ in size.
+     */
+    erts_smp_atomic_t smp_atomic_word;
+    erts_atomic_t atomic_word;
+} ErtsMagicIndirectionWord;
+
 #if defined(__i386__) || !defined(__GNUC__)
 /*
  * Doubles aren't required to be 8-byte aligned on intel x86.
@@ -329,6 +340,9 @@ ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size,
                                                   int unaligned);
 ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size,
 						 void (*destructor)(Binary *));
+ERTS_GLB_INLINE Binary *erts_create_magic_indirection(void (*destructor)(Binary *));
+ERTS_GLB_INLINE erts_smp_atomic_t *erts_smp_binary_to_magic_indirection(Binary *bp);
+ERTS_GLB_INLINE erts_atomic_t *erts_binary_to_magic_indirection(Binary *bp);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
@@ -496,6 +510,36 @@ erts_create_magic_binary(Uint size, void (*destructor)(Binary *))
                                       ERTS_ALC_T_BINARY, 0);
 }
 
+ERTS_GLB_INLINE Binary *
+erts_create_magic_indirection(void (*destructor)(Binary *))
+{
+    return erts_create_magic_binary_x(sizeof(ErtsMagicIndirectionWord),
+                                      destructor,
+                                      ERTS_ALC_T_MINDIRECTION,
+                                      1); /* Not 64-bit aligned,
+                                             but word aligned */
+}
+
+ERTS_GLB_INLINE erts_smp_atomic_t *
+erts_smp_binary_to_magic_indirection(Binary *bp)
+{
+    ErtsMagicIndirectionWord *mip;
+    ASSERT(bp->flags & BIN_FLAG_MAGIC);
+    ASSERT(ERTS_MAGIC_BIN_ATYPE(bp) == ERTS_ALC_T_MINDIRECTION);
+    mip = ERTS_MAGIC_BIN_UNALIGNED_DATA(bp);
+    return &mip->smp_atomic_word;
+}
+
+ERTS_GLB_INLINE erts_atomic_t *
+erts_binary_to_magic_indirection(Binary *bp)
+{
+    ErtsMagicIndirectionWord *mip;
+    ASSERT(bp->flags & BIN_FLAG_MAGIC);
+    ASSERT(ERTS_MAGIC_BIN_ATYPE(bp) == ERTS_ALC_T_MINDIRECTION);
+    mip = ERTS_MAGIC_BIN_UNALIGNED_DATA(bp);
+    return &mip->atomic_word;
+}
+
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 #endif /* !ERL_BINARY_H__ */
-- 
cgit v1.2.3


From 7c06ca6231b812965305522284dd9f2653ced98d Mon Sep 17 00:00:00 2001
From: Andrew Dryga <andrew@dryga.com>
Date: Tue, 14 Feb 2017 11:30:41 +0200
Subject: Fixed typos in erts

---
 erts/emulator/beam/beam_emu.c                      | 10 +++++-----
 erts/emulator/beam/bif.c                           |  4 ++--
 erts/emulator/beam/binary.c                        |  4 ++--
 erts/emulator/beam/dist.c                          |  4 ++--
 erts/emulator/beam/dist.h                          |  4 ++--
 erts/emulator/beam/dtrace-wrapper.h                |  2 +-
 erts/emulator/beam/erl_alloc.types                 |  2 +-
 erts/emulator/beam/erl_alloc_util.c                |  4 ++--
 erts/emulator/beam/erl_bif_binary.c                |  2 +-
 erts/emulator/beam/erl_gc.c                        |  2 +-
 erts/emulator/beam/erl_lock_count.h                |  2 +-
 erts/emulator/beam/erl_monitors.c                  |  4 ++--
 erts/emulator/beam/erl_process.c                   |  2 +-
 erts/emulator/beam/erl_ptab.c                      |  2 +-
 erts/emulator/beam/erl_time_sup.c                  |  2 +-
 erts/emulator/beam/io.c                            |  2 +-
 erts/emulator/drivers/common/efile_drv.c           |  6 +++---
 erts/emulator/drivers/common/gzio.c                |  2 +-
 erts/emulator/drivers/common/inet_drv.c            |  4 ++--
 erts/emulator/drivers/unix/sig_drv.c               |  2 +-
 erts/emulator/internal_doc/DelayedDealloc.md       | 18 +++++++++---------
 erts/emulator/internal_doc/PortSignals.md          |  2 +-
 erts/emulator/internal_doc/SuperCarrier.md         |  2 +-
 erts/emulator/internal_doc/ThreadProgress.md       |  8 ++++----
 erts/emulator/internal_doc/Tracing.md              |  2 +-
 erts/emulator/pcre/pcre_exec.c                     |  2 +-
 erts/emulator/sys/common/erl_mseg.c                |  2 +-
 erts/emulator/sys/unix/erl_child_setup.h           |  2 +-
 erts/emulator/sys/win32/erl_poll.c                 |  6 +++---
 erts/emulator/sys/win32/sys.c                      |  8 ++++----
 erts/emulator/test/binary_SUITE.erl                |  2 +-
 erts/emulator/test/bs_match_int_SUITE.erl          |  2 +-
 erts/emulator/test/call_trace_SUITE.erl            |  2 +-
 erts/emulator/test/ddll_SUITE.erl                  |  2 +-
 erts/emulator/test/dirty_nif_SUITE.erl             |  2 +-
 erts/emulator/test/estone_SUITE.erl                |  2 +-
 erts/emulator/test/float_SUITE.erl                 |  2 +-
 erts/emulator/test/hibernate_SUITE.erl             |  2 +-
 erts/emulator/test/match_spec_SUITE.erl            |  2 +-
 erts/emulator/test/node_container_SUITE.erl        |  2 +-
 erts/emulator/test/num_bif_SUITE.erl               |  2 +-
 erts/emulator/test/port_SUITE.erl                  |  2 +-
 erts/emulator/test/port_SUITE_data/port_test.c     |  2 +-
 erts/emulator/test/port_bif_SUITE_data/port_test.c |  2 +-
 erts/emulator/test/system_info_SUITE.erl           |  2 +-
 erts/emulator/test/time_SUITE.erl                  |  2 +-
 erts/emulator/test/z_SUITE.erl                     |  2 +-
 47 files changed, 76 insertions(+), 76 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 5e275c8fc5..49f932a1c2 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -126,7 +126,7 @@ do {                                     \
 
 /*
  * We reuse some of fields in the save area in the process structure.
- * This is safe to do, since this space is only activly used when
+ * This is safe to do, since this space is only actively used when
  * the process is switched out.
  */
 #define REDS_IN(p)  ((p)->def_arg_reg[5])
@@ -220,7 +220,7 @@ BeamInstr* em_call_bif_e;
 
 /* NOTE These should be the only variables containing trace instructions.
 **      Sometimes tests are form the instruction value, and sometimes
-**      for the refering variable (one of these), and rouge references
+**      for the referring variable (one of these), and rouge references
 **      will most likely cause chaos.
 */
 BeamInstr beam_return_to_trace[1];   /* OpCode(i_return_to_trace) */
@@ -2993,7 +2993,7 @@ do {						\
  }
 
  /*
-  * An error occured in an arithmetic operation or test that could
+  * An error occurred in an arithmetic operation or test that could
   * appear either in a head or in a body.
   * In a head, execution should continue at failure address in Arg(0).
   * In a body, Arg(0) == 0 and an exception should be raised.
@@ -6268,7 +6268,7 @@ apply_bif_error_adjustment(Process *p, Export *ep,
 	 * stackframe correct. Without the following adjustment,
 	 * 'p->cp' will point into the function that called
 	 * current function when handling the error. We add a
-	 * dummy stackframe in order to achive this.
+	 * dummy stackframe in order to achieve this.
 	 *
 	 * Note that these BIFs unconditionally will cause
 	 * an exception to be raised. That is, our modifications
@@ -6612,7 +6612,7 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
 #ifndef ERTS_SMP
 	if (ERTS_PROC_IS_EXITING(c_p)) {
 	    /*
-	     * See comment in the begining of the function...
+	     * See comment in the beginning of the function...
 	     *
 	     * This second test is needed since gc might be traced.
 	     */
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 2190c47262..4c9b9887c1 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -467,7 +467,7 @@ demonitor_local_port(Process *origin, Eterm ref, Eterm target)
 }
 
 /* Can return atom true, false, yield, internal_error, badarg or
- * THE_NON_VALUE if error occured or trap has been set up
+ * THE_NON_VALUE if error occurred or trap has been set up
  */
 static
 BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip)
@@ -597,7 +597,7 @@ BIF_RETTYPE demonitor_2(BIF_ALIST_2)
 
     switch (demonitor(BIF_P, BIF_ARG_1, &multi)) {
     case THE_NON_VALUE:
-        /* If other error occured or trap has been set up - pass through */
+        /* If other error occurred or trap has been set up - pass through */
         BIF_RET(THE_NON_VALUE);
     case am_false:
 	if (info)
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index d38097fb12..8f907ee8a2 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -102,7 +102,7 @@ new_binary(Process *p, byte *buf, Uint len)
     pb->flags = 0;
 
     /*
-     * Miscellanous updates. Return the tagged binary.
+     * Miscellaneous updates. Return the tagged binary.
      */
     OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
     return make_binary(pb);
@@ -139,7 +139,7 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, Uint len)
     pb->flags = 0;
 
     /*
-     * Miscellanous updates. Return the tagged binary.
+     * Miscellaneous updates. Return the tagged binary.
      */
     OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
     return make_binary(pb);
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 390f2a0db3..0d25eb123e 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -264,7 +264,7 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp)
 	goto done;
 
     if (mon->type == MON_ORIGIN) {
-	/* local pid is beeing monitored */
+	/* local pid is being monitored */
 	rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref);
 	/* ASSERT(rmon != NULL); nope, can happen during process exit */
 	if (rmon != NULL) {
@@ -795,7 +795,7 @@ erts_dsig_send_unlink(ErtsDSigData *dsdp, Eterm local, Eterm remote)
 }
 
 
-/* A local process that's beeing monitored by a remote one exits. We send:
+/* A local process that's being monitored by a remote one exits. We send:
    {DOP_MONITOR_P_EXIT, Local pid or name, Remote pid, ref, reason},
    which is rather sad as only the ref is needed, no pid's... */
 int
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index e82b416286..39670de506 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -115,8 +115,8 @@ extern int erts_is_alive;
  * erts_dsig_prepare() prepares a send of a distributed signal.
  * One of the values defined below are returned. If the returned
  * value is another than ERTS_DSIG_PREP_CONNECTED, the
- * distributed signal cannot be sent before apropriate actions
- * have been taken. Apropriate actions would typically be setting
+ * distributed signal cannot be sent before appropriate actions
+ * have been taken. Appropriate actions would typically be setting
  * up the connection.
  */
 
diff --git a/erts/emulator/beam/dtrace-wrapper.h b/erts/emulator/beam/dtrace-wrapper.h
index 6f70d5961e..f6fc28b801 100644
--- a/erts/emulator/beam/dtrace-wrapper.h
+++ b/erts/emulator/beam/dtrace-wrapper.h
@@ -74,7 +74,7 @@
 
 #if defined(_SDT_PROBE) && !defined(STAP_PROBE11)
 /* SLF: This is Ubuntu 11-style SystemTap hackery */
-/* work arround for missing STAP macro */
+/* workaround for missing STAP macro */
 #define STAP_PROBE11(provider,name,arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10,arg11) \
   _SDT_PROBE(provider, name, 11, \
              (arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10,arg11))
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index f88985b7f5..295db2fe9b 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -65,7 +65,7 @@
 
 # --- Allocator declarations -------------------------------------------------
 #
-# If, and only if, the same thread performes *all* allocations,
+# If, and only if, the same thread performs *all* allocations,
 # reallocations and deallocations of all memory types that are handled
 # by a specific allocator (<ALLOCATOR> in type declaration), set 
 # <MULTI_THREAD> for this specific allocator to false; otherwise, set
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index d346f8c8b6..a6bac06478 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -5186,7 +5186,7 @@ erts_alcu_sz_info(Allctr_t *allctr,
 
     ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
 
-    /* Update sbc values not continously updated */
+    /* Update sbc values not continuously updated */
     allctr->sbcs.blocks.curr.no
 	= allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no;
     allctr->sbcs.blocks.max.no = allctr->sbcs.max.no;
@@ -5272,7 +5272,7 @@ erts_alcu_info(Allctr_t *allctr,
 
     ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
 
-    /* Update sbc values not continously updated */
+    /* Update sbc values not continuously updated */
     allctr->sbcs.blocks.curr.no
 	= allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no;
     allctr->sbcs.blocks.max.no = allctr->sbcs.max.no;
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index e57cd06cec..64a5c1cb29 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -380,7 +380,7 @@ static void ac_compute_failure_functions(ACTrie *act, ACNode **qbuff)
 		qbuff[qt++] = child;
 		/* Search for correct failure function, follow the parent's
 		   failure function until you find a similar transition
-		   funtion to this child's */
+		   function to this child's */
 		r =  parent->h;
 		while (r != NULL && r->g[i] == NULL) {
 		    r = r->h;
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index cb3a189e8f..8cc7fc0142 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2834,7 +2834,7 @@ sweep_off_heap(Process *p, int fullsweep)
     prev = &MSO(p).first;
     ptr = MSO(p).first;
 
-    /* Firts part of the list will reside on the (old) new-heap.
+    /* First part of the list will reside on the (old) new-heap.
      * Keep if moved, otherwise deref.
      */
     while (ptr) {
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index 4f838f7faa..71cd73ee27 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -129,7 +129,7 @@ typedef struct {
 
 typedef struct erts_lcnt_lock_stats_s {
     /* "tries" and "colls" needs to be atomic since
-     * trylock busy does not aquire a lock and there
+     * trylock busy does not acquire a lock and there
      * is no post action to rectify the situation
      */
 
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
index bdfc7f30d9..f813f19dbc 100644
--- a/erts/emulator/beam/erl_monitors.c
+++ b/erts/emulator/beam/erl_monitors.c
@@ -24,7 +24,7 @@
  * key in the monitor case and the pid of the linked process as key in the 
  * link case. Lookups the order of the references is somewhat special. Local 
  * references are strictly smaller than remote references and are sorted 
- * by inlined comparision functionality. Remote references are handled by the
+ * by inlined comparison functionality. Remote references are handled by the
  * usual cmp function.
  * Each Monitor is tagged with different tags depending on which end of the 
  * monitor it is.
@@ -154,7 +154,7 @@ static ErtsMonitor *create_monitor(Uint type, Eterm ref, Eterm pid, Eterm name)
      n->type = (Uint16) type;
      n->balance = 0;            /* Always the same initial value */
      n->name = name; /* atom() or [] */
-     CP_LINK_VAL(n->ref, hp, ref); /*XXX Unneccesary check, never immediate*/
+     CP_LINK_VAL(n->ref, hp, ref); /*XXX Unnecessary check, never immediate*/
      CP_LINK_VAL(n->pid, hp, pid);
 
      return n;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 87d71fdd22..c5a7b67764 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -5050,7 +5050,7 @@ check_balance(ErtsRunQueue *c_rq)
 	    sched_util_balancing = 1;
 	    /*
 	     * In order to avoid renaming a large amount of fields
-	     * we write utilization values instead of lenght values
+	     * we write utilization values instead of length values
 	     * in the 'max_len' and 'migration_limit' fields...
 	     */
 	    for (qix = 0; qix < blnc_no_rqs; qix++) {
diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index a132b1d334..828a029e84 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -474,7 +474,7 @@ erts_ptab_init_table(ErtsPTab *ptab,
 	 * we don't want to shrink the size to ERTS_PTAB_MAX_SIZE/2.
 	 *
 	 * In order to fix this, we insert a pointer from the table
-	 * to the invalid_element, wich will be interpreted as a
+	 * to the invalid_element, which will be interpreted as a
 	 * slot currently being modified. This way we will be able to
 	 * have ERTS_PTAB_MAX_SIZE-1 valid elements in the table while
 	 * still having a table size of the power of 2.
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 1d747e5fab..ed40539b78 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1502,7 +1502,7 @@ static time_t gregday(int year, int month, int day)
     pyear = gyear - 1;
     ndays = (pyear/4) - (pyear/100) + (pyear/400) + pyear*365 + 366;
   }
-  /* number of days in all months preceeding month */
+  /* number of days in all months preceding month */
   for (m = 1; m < month; m++)
     ndays += mdays[m];
   /* Extra day if leap year and March or later */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index dad24d7ef6..7e49e9c48f 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -3431,7 +3431,7 @@ void erts_init_io(int port_tab_size,
 			 NULL,
 			 (ErtsPTabElementCommon *) &erts_invalid_port.common,
 			 port_tab_size,
-			 common_element_size, /* Doesn't need to be excact */
+			 common_element_size, /* Doesn't need to be exact */
 			 "port_table",
 			 legacy_port_tab,
 			 1);
diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c
index d64f015a6a..173a39533d 100644
--- a/erts/emulator/drivers/common/efile_drv.c
+++ b/erts/emulator/drivers/common/efile_drv.c
@@ -2331,9 +2331,9 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data)
 	free_read(data);
 	break;
       case FILE_READ_LINE:
-	  /* The read_line stucture differs from the read structure.
-	     The data->read_offset and d->c.read_line.read_offset are copies, as are 
-	     data->read_size and d->c.read_line.read_size 
+	  /* The read_line structure differs from the read structure.
+	     The data->read_offset and d->c.read_line.read_offset are copies, as are
+	     data->read_size and d->c.read_line.read_size
              The read_line function does not kniow in advance how large the binary has to be,
 	     why new allocation (but not reallocation of the old binary, for obvious reasons) 
 	     may happen in the worker thread. */
diff --git a/erts/emulator/drivers/common/gzio.c b/erts/emulator/drivers/common/gzio.c
index 1ef1602ec9..f60c781894 100644
--- a/erts/emulator/drivers/common/gzio.c
+++ b/erts/emulator/drivers/common/gzio.c
@@ -308,7 +308,7 @@ ErtsGzFile erts_gzopen (path, mode)
 /* ===========================================================================
      Read a byte from a gz_stream; update next_in and avail_in. Return EOF
    for end of file.
-   IN assertion: the stream s has been sucessfully opened for reading.
+   IN assertion: the stream s has been successfully opened for reading.
 */
 local int get_byte(s)
     gz_stream *s;
diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 1885338ce5..278abe4e00 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -728,7 +728,7 @@ static int is_nonzero(const char *s, size_t n)
 #define TCP_ADDF_PENDING_SHUTDOWN \
 		(TCP_ADDF_PENDING_SHUT_WR | TCP_ADDF_PENDING_SHUT_RDWR)
 #define TCP_ADDF_SHOW_ECONNRESET   64 /* Tell user about incoming RST */
-#define TCP_ADDF_DELAYED_ECONNRESET 128 /* An ECONNRESET error occured on send or shutdown */
+#define TCP_ADDF_DELAYED_ECONNRESET 128 /* An ECONNRESET error occurred on send or shutdown */
 #define TCP_ADDF_SHUTDOWN_WR_DONE 256 /* A shutdown(sock, SHUT_WR) or SHUT_RDWR was made */
 #define TCP_ADDF_LINGER_ZERO 	  512 /* Discard driver queue on port close */
 
@@ -4132,8 +4132,8 @@ static char *inet_set_faddress(int family, inet_address* dst,
 
 /* Get a inaddr structure
 ** src = inaddr structure
-** *len is the lenght of structure
 ** dst is filled with [F,P1,P0,X1,....] 
+** *len is the length of structure
 ** where F is the family code (coded)
 ** and *len is the length of dst on return 
 ** (suitable to deliver to erlang)
diff --git a/erts/emulator/drivers/unix/sig_drv.c b/erts/emulator/drivers/unix/sig_drv.c
index 68ad6b9156..18f2038431 100644
--- a/erts/emulator/drivers/unix/sig_drv.c
+++ b/erts/emulator/drivers/unix/sig_drv.c
@@ -18,7 +18,7 @@
  * %CopyrightEnd%
  */
 
-/* Purpose: demonstrate how to include interupt handlers in erlang */
+/* Purpose: demonstrate how to include interrupt handlers in erlang */
 
 #ifdef HAVE_CONFIG_H
 #  include "config.h"
diff --git a/erts/emulator/internal_doc/DelayedDealloc.md b/erts/emulator/internal_doc/DelayedDealloc.md
index b7d87b839f..4b7c774141 100644
--- a/erts/emulator/internal_doc/DelayedDealloc.md
+++ b/erts/emulator/internal_doc/DelayedDealloc.md
@@ -19,7 +19,7 @@ the Erlang VM where memory allocation/deallocation is frequent and
 references to memory also are passed around between threads this
 solution will also scale poorly due to lock contention.
 
-Functionality Used to Adress This problem
+Functionality Used to Address This problem
 -----------------------------------------
 
 In order to reduce contention due to locking of allocator instances we
@@ -44,12 +44,12 @@ deallocation.
 The "message box" is implemented using a lock free single linked list
 through the memory blocks to deallocate. The order of the elements in
 this list is not important. Insertion of new free blocks will be made
-somewhere near the end of this list. Requirering that the new blocks
+somewhere near the end of this list. Requiring that the new blocks
 need to be inserted at the end would cause unnecessary contention when
 large amount of memory blocks are inserted simultaneous by multiple
 threads.
 
-The data structure refering to this single linked list cover two cache
+The data structure referring to this single linked list cover two cache
 lines. One cache line containing information about the head of the
 list, and one cache line containing information about the tail of the
 list. This in order to reduce cache line ping ponging of this data
@@ -65,21 +65,21 @@ list. In the uncontended case it will point to the end of the list,
 but when simultaneous insert operations are performed it will point to
 something near the end of the list.
 
-When insterting an element one will try to write a pointer to the new
+When inserting an element one will try to write a pointer to the new
 element in the next pointer of the element pointed to by the last
 pointer. This is done using an atomic compare and swap that expects
-the next pointer to be `NULL`. If this succeds the thread performing
+the next pointer to be `NULL`. If this succeeds the thread performing
 this operation moves the last pointer to point to the newly inserted
 element.
 
 If the atomic compare and swap described above failed, the last
 pointer didn't point to the last element. In this case we need to
-insert the new element somewhere inbetween the element that the last
+insert the new element somewhere between the element that the last
 pointer pointed to and the actual last element. If we do it this way
 the last pointer will eventually end up at the last element when
 threads stop adding new elements. When trying to insert somewhere near
 the end and failing to do so, the inserting thread sometimes moves to
-the next element and somtimes tries with the same element again. This
+the next element and sometimes tries with the same element again. This
 in order to spread the inserted elements during heavy contention. That
 is, we try to spread the modifications of memory to different
 locations instead of letting all threads continue to try to modify the
@@ -87,7 +87,7 @@ same location in memory.
 
 ### Head ###
 
-The head contains pointers to begining of the list (`head.first`), and
+The head contains pointers to beginning of the list (`head.first`), and
 to the first block which other threads may refer to
 (`head.unref_end`). Blocks between these pointers are only refered to
 by the head part of the data structure which is only used by the
@@ -142,7 +142,7 @@ contains this "marker" element.
 
 ### Contention ###
 
-When elements are continously inserted by threads not owning the
+When elements are continuously inserted by threads not owning the
 allocator instance, the thread owning the allocator instance will be
 able to work more or less undisturbed by other threads at the head end
 of the list. At the tail end large amounts of simultaneous inserts may
diff --git a/erts/emulator/internal_doc/PortSignals.md b/erts/emulator/internal_doc/PortSignals.md
index b1afb7c5cb..8782ae4e17 100644
--- a/erts/emulator/internal_doc/PortSignals.md
+++ b/erts/emulator/internal_doc/PortSignals.md
@@ -204,7 +204,7 @@ high limit is 8 KB and the low limit is 4 KB.
 
 Previously all operations sending signals to ports began by acquiring
 the port lock, then performed preparations for sending the signal, and
-then finaly sent the signal. The preparations typically included
+then finally sent the signal. The preparations typically included
 inspecting the state of the port, and preparing the data to pass along
 with the signal. The preparation of data is frequently quite time
 consuming, and did not really depend on the port. That is we would
diff --git a/erts/emulator/internal_doc/SuperCarrier.md b/erts/emulator/internal_doc/SuperCarrier.md
index 0ad6af41de..acf722ea37 100644
--- a/erts/emulator/internal_doc/SuperCarrier.md
+++ b/erts/emulator/internal_doc/SuperCarrier.md
@@ -151,7 +151,7 @@ To find the smallest free segment that will satisfy a carrier allocation
 size (`stree`). We search in this tree at allocation. If no free segment of
 sufficient size was found, the area (`sa` or `sua`) is instead expanded.
 If two or more free segments with equal size exist, the one at lowest
-address is choosen for `sa` and highest address for `sua`.
+address is chosen for `sa` and highest address for `sua`.
 
 At carrier deallocation, we want to coalesce with any adjacent free
 segments, to form one large free segment. To do that, all free
diff --git a/erts/emulator/internal_doc/ThreadProgress.md b/erts/emulator/internal_doc/ThreadProgress.md
index 6118bcf0f6..03a802f904 100644
--- a/erts/emulator/internal_doc/ThreadProgress.md
+++ b/erts/emulator/internal_doc/ThreadProgress.md
@@ -60,7 +60,7 @@ threads are managed threads.
 ### Thread Progress Events ###
 
 Any thread in the system may use the thread progress functionality in
-order to determine when the following events have occured at least
+order to determine when the following events have occurred at least
 once in all managed threads:
 
 1.  The thread has returned from other code to a known state in the
@@ -160,7 +160,7 @@ calling the following functions:
 *   `int erts_thr_progress_leader_update(ErtsSchedulerData *esdp)` -
     Leader update thread progress.
 
-Unmanaged threads can delay thread progress beeing made:
+Unmanaged threads can delay thread progress being made:
 
 *   `ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay(void)` -
     Delay thread progress.
@@ -251,7 +251,7 @@ doing so. If not zero, the leader isn't allowed to increment the
 global counter, and needs to wait before it can do this. When it is
 zero, it swaps the `waiting` and `current` counters before increasing
 the global counter. From now on the new `waiting` counter will
-decrease, so that it eventualy will reach zero, making it possible to
+decrease, so that it eventually will reach zero, making it possible to
 increment the global counter the next time. If we only used one
 reference counter it would potentially be held above zero for ever by
 different unmanaged threads.
@@ -261,7 +261,7 @@ prevent the next increment of the global counter, but instead the
 increment after that. This is sufficient since the global counter
 needs to be incremented two times before thread progress has been
 made. It is also desirable not to prevent the first increment, since
-the likelyhood increases that the delay is withdrawn before any
+the likelihood increases that the delay is withdrawn before any
 increment of the global counter is delayed. That is, the operation
 will cause as little disruption as possible.
 
diff --git a/erts/emulator/internal_doc/Tracing.md b/erts/emulator/internal_doc/Tracing.md
index 728f315263..7f97f64765 100644
--- a/erts/emulator/internal_doc/Tracing.md
+++ b/erts/emulator/internal_doc/Tracing.md
@@ -51,7 +51,7 @@ the new instrumented code. Normally loaded code can only be reached
 through external functions calls. Trace settings must be activated
 instantaneously without the need of external function calls.
 
-The choosen solution is instead for tracing to use the technique of
+The chosen solution is instead for tracing to use the technique of
 replication applied on the data structures for breakpoints. Two
 generations of breakpoints are kept and indentified by index of 0 and
 1. The global atomic variables `erts_active_bp_index` will determine
diff --git a/erts/emulator/pcre/pcre_exec.c b/erts/emulator/pcre/pcre_exec.c
index 1cab78cdd8..07e662ff07 100644
--- a/erts/emulator/pcre/pcre_exec.c
+++ b/erts/emulator/pcre/pcre_exec.c
@@ -1134,7 +1134,7 @@ for (;;)
     the result of a recursive call to match() whatever happened so it was
     possible to reduce stack usage by turning this into a tail recursion,
     except in the case of a possibly empty group. However, now that there is
-    the possiblity of (*THEN) occurring in the final alternative, this
+    the possibility of (*THEN) occurring in the final alternative, this
     optimization is no longer always possible.
 
     We can optimize if we know there are no (*THEN)s in the pattern; at present
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index 4dd986f4c8..1e05fd3490 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -378,7 +378,7 @@ static ERTS_INLINE int cache_bless_segment(ErtsMsegAllctr_t *ma, void *seg, UWor
 
     ASSERT(!MSEG_FLG_IS_2POW(flags) || (MSEG_FLG_IS_2POW(flags) && MAP_IS_ALIGNED(seg) && IS_2POW(size)));
 
-    /* The idea is that sbc caching is prefered over mbc caching.
+    /* The idea is that sbc caching is preferred over mbc caching.
      * Blocks are normally allocated in mb carriers and thus cached there.
      * Large blocks has no such cache and it is up to mseg to cache them to speed things up.
      */
diff --git a/erts/emulator/sys/unix/erl_child_setup.h b/erts/emulator/sys/unix/erl_child_setup.h
index a28b136bfc..b61e557ddf 100644
--- a/erts/emulator/sys/unix/erl_child_setup.h
+++ b/erts/emulator/sys/unix/erl_child_setup.h
@@ -17,7 +17,7 @@
  * 
  * %CopyrightEnd%
  *
- * This file defines the interface inbetween erts and child_setup.
+ * This file defines the interface between erts and child_setup.
  */
 
 #ifndef _ERL_UNIX_FORKER_H
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index f23c7ab03d..93013a412b 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -842,9 +842,9 @@ event_happened:
 	    ASSERT(WAIT_OBJECT_0 < i && i < WAIT_OBJECT_0+w->active_events);
 	    notify_io_ready(ps);
 
-	    /* 
-	     * The main thread wont start working on our arrays untill we're
-	     * stopped, so we can work in peace although the main thread runs 
+	    /*
+	     * The main thread wont start working on our arrays until we're
+	     * stopped, so we can work in peace although the main thread runs
 	     */
 	    ASSERT(i >= WAIT_OBJECT_0+1);
 	    i -= WAIT_OBJECT_0;
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index 408b4f47ab..40e5595533 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -496,7 +496,7 @@ struct driver_data {
     int outBufSize;		/* Size of output buffer. */
     byte *outbuf;		/* Buffer to use for overlapped write. */
     ErlDrvPort port_num;	/* The port handle. */
-    int packet_bytes;		/* 0: continous stream, 1, 2, or 4: the number
+    int packet_bytes;		/* 0: continuous stream, 1, 2, or 4: the number
 				 * of bytes in the packet header.
 				 */
     HANDLE port_pid;		/* PID of the port process. */
@@ -1427,7 +1427,7 @@ int parse_command(wchar_t* cmd){
  *
  * If new == NULL we just calculate the length.
  *
- * The reason for having to quote all of the is becasue CreateProcessW removes
+ * The reason for having to quote all of the is because CreateProcessW removes
  * one level of escaping since it takes a single long command line rather
  * than the argument chunks that unix uses.
  */
@@ -2486,7 +2486,7 @@ output(ErlDrvData drv_data, char* buf, ErlDrvSizeT len)
  *	event object has been signaled, indicating that there is
  *	something to read on the corresponding file handle.
  *
- *	If the port is working in the continous stream mode (packet_bytes == 0),
+ *	If the port is working in the continuous stream mode (packet_bytes == 0),
  *	whatever data read will be sent straight to Erlang.
  *
  * Results:
@@ -2527,7 +2527,7 @@ ready_input(ErlDrvData drv_data, ErlDrvEvent ready_event)
 #endif
 
     if (error == NO_ERROR) {
-	if (pb == 0) { /* Continous stream. */
+	if (pb == 0) { /* Continuous stream. */
 #ifdef DEBUG
 	    DEBUGF(("ready_input: %d: ", bytesRead));
 	    erl_bin_write(dp->inbuf, 16, bytesRead);
diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index 238a8aa42d..324730d562 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -1008,7 +1008,7 @@ ordering(Config) when is_list(Config) ->
 
     ok.
 
-%% Test that comparisions between binaries with different alignment work.
+%% Test that comparison between binaries with different alignment work.
 unaligned_order(Config) when is_list(Config) ->
     L = lists:seq(0, 7),
     [test_unaligned_order(I, J) || I <- L, J <- L], 
diff --git a/erts/emulator/test/bs_match_int_SUITE.erl b/erts/emulator/test/bs_match_int_SUITE.erl
index a7bd4b8ac3..32bfa2f1fa 100644
--- a/erts/emulator/test/bs_match_int_SUITE.erl
+++ b/erts/emulator/test/bs_match_int_SUITE.erl
@@ -247,7 +247,7 @@ match_huge_int(Config) when is_list(Config) ->
 	8 ->
 	    %% An attempt will be made to allocate heap space for
 	    %% the bignum (which will probably fail); only if the
-	    %% allocation succeds will the matching fail because
+	    %% allocation succeeds will the matching fail because
 	    %% the binary is too small.
 	    ok
     end,
diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl
index 2e303ba9a8..e74631e916 100644
--- a/erts/emulator/test/call_trace_SUITE.erl
+++ b/erts/emulator/test/call_trace_SUITE.erl
@@ -233,7 +233,7 @@ basic() ->
     trace_func({'_','_','_'}, false),
     [b,a] = lists:reverse([a,b]),
 
-    %% Read out the remaing trace messages.
+    %% Read out the remaining trace messages.
 
     ?MODULE:expect({trace,Self,call,{lists,seq,[1,10]}}),
     ?MODULE:expect({trace,Self,call,{erlang,list_to_integer,["777"]}}),
diff --git a/erts/emulator/test/ddll_SUITE.erl b/erts/emulator/test/ddll_SUITE.erl
index 93b6f2d956..e7e518f82b 100644
--- a/erts/emulator/test/ddll_SUITE.erl
+++ b/erts/emulator/test/ddll_SUITE.erl
@@ -805,7 +805,7 @@ reference_count(Config) when is_list(Config) ->
 
     Pid1 ! {self(), die},
     test_server:sleep(200),   % Give time to unload.
-    % Verify that the driver was automaticly unloaded when the
+    % Verify that the driver was automatically unloaded when the
     % process died.
     {error, not_loaded}=erl_ddll:unload_driver(echo_drv),
     ok.
diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl
index 991ba0acc8..5ba0d85ff3 100644
--- a/erts/emulator/test/dirty_nif_SUITE.erl
+++ b/erts/emulator/test/dirty_nif_SUITE.erl
@@ -214,7 +214,7 @@ dirty_call_while_terminated(Config) when is_list(Config) ->
     undefined = process_info(Dirty, status),
     false = erlang:is_process_alive(Dirty),
     false = lists:member(Dirty, processes()),
-    %% Binary still refered by Dirty process not yet cleaned up
+    %% Binary still referred by Dirty process not yet cleaned up
     %% since the dirty nif has not yet returned...
     {value, {BinAddr, 4711, 2}} = lists:keysearch(4711, 2,
 						  element(2,
diff --git a/erts/emulator/test/estone_SUITE.erl b/erts/emulator/test/estone_SUITE.erl
index 3ce849b88e..35f695ffe5 100644
--- a/erts/emulator/test/estone_SUITE.erl
+++ b/erts/emulator/test/estone_SUITE.erl
@@ -708,7 +708,7 @@ alloc(I) ->
 
 %% Time to call bif's
 %% Lot's of element stuff which reflects the record code which
-%% is becomming more and more common
+%% is becoming more and more common
 bif_dispatch(0) ->
     0;
 bif_dispatch(I) ->
diff --git a/erts/emulator/test/float_SUITE.erl b/erts/emulator/test/float_SUITE.erl
index 36b1f9179f..ad33ad705b 100644
--- a/erts/emulator/test/float_SUITE.erl
+++ b/erts/emulator/test/float_SUITE.erl
@@ -208,7 +208,7 @@ span_cmp(Axis, Incr, Length) ->
 %% for both negative and positive numbers.
 %%
 %% Axis: The number around which to do the tests eg. (1 bsl 58) - 1.0
-%% Incr: How much to increment the test numbers inbetween each test.
+%% Incr: How much to increment the test numbers in-between each test.
 %% Length: Length/2 is the number of Incr away from Axis to test on the
 %%         negative and positive plane.
 %% Diff: How much the float and int should differ when comparing
diff --git a/erts/emulator/test/hibernate_SUITE.erl b/erts/emulator/test/hibernate_SUITE.erl
index 6f8ce02266..90693595a7 100644
--- a/erts/emulator/test/hibernate_SUITE.erl
+++ b/erts/emulator/test/hibernate_SUITE.erl
@@ -349,7 +349,7 @@ clean_dict() ->
     lists:foreach(fun ({Key, _}) -> erase(Key) end, Dict).
 
 %%
-%% Wake up and then immediatly bif trap with a lengthy computation.
+%% Wake up and then immediately bif trap with a lengthy computation.
 %%
 
 wake_up_and_bif_trap(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl
index 34e956bc21..971a047309 100644
--- a/erts/emulator/test/match_spec_SUITE.erl
+++ b/erts/emulator/test/match_spec_SUITE.erl
@@ -646,7 +646,7 @@ destructive_in_test_bif(Config) when is_list(Config) ->
 			       ([],[{'_',[],[{message,{get_tcw}}]}],trace),
     ok.
 
-%% Test that the comparision between boxed and small does not crash emulator
+%% Test that the comparison between boxed and small does not crash emulator
 boxed_and_small(Config) when is_list(Config) ->
     {ok, Node} = start_node(match_spec_suite_other),
     ok = rpc:call(Node,?MODULE,do_boxed_and_small,[]),
diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl
index 7356f31b75..291d3ee30d 100644
--- a/erts/emulator/test/node_container_SUITE.erl
+++ b/erts/emulator/test/node_container_SUITE.erl
@@ -153,7 +153,7 @@ ttbtteq_do_remote(RNode) ->
 %%
 %% Test case: round_trip_eq
 %%
-%% Tests that node containers that are sent beteen nodes stay equal to themselves.
+%% Tests that node containers that are sent between nodes stay equal to themselves.
 round_trip_eq(Config) when is_list(Config) ->
     ThisNode = {node(), erlang:system_info(creation)},
     NodeFirstName = get_nodefirstname(),
diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl
index 1a1ab0e5e0..8016a21424 100644
--- a/erts/emulator/test/num_bif_SUITE.erl
+++ b/erts/emulator/test/num_bif_SUITE.erl
@@ -108,7 +108,7 @@ t_float(Config) when is_list(Config) ->
     4294967305.0 = float(id(4294967305)),
     -4294967305.0 = float(id(-4294967305)),
 
-    %% Extremly big bignums.
+    %% Extremely big bignums.
     Big = id(list_to_integer(id(lists:duplicate(2000, $1)))),
     {'EXIT', {badarg, _}} = (catch float(Big)),
 
diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index 2a13b2d2f4..c117554f90 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -2131,7 +2131,7 @@ ping(Config, Sizes, HSize, CmdLine, Options) ->
 %% Sizes = Size of packets to generated.
 %% HSize = Header size: 1, 2, or 4
 %% CmdLine = Additional command line options.
-%% Options = Addtional port options.
+%% Options = Additional port options.
 
 expect_input(Config, Sizes, HSize, CmdLine, Options) ->
     expect_input1(Config, Sizes, {HSize, CmdLine, Options}, [], []).
diff --git a/erts/emulator/test/port_SUITE_data/port_test.c b/erts/emulator/test/port_SUITE_data/port_test.c
index e199a0fc13..fa97b4c9d0 100644
--- a/erts/emulator/test/port_SUITE_data/port_test.c
+++ b/erts/emulator/test/port_SUITE_data/port_test.c
@@ -41,7 +41,7 @@ extern int errno;
 typedef struct {
     char* progname;	        /* Name of this program (from argv[0]). */
     int header_size;	        /* Number of bytes in each packet header:
-				 * 1, 2, or 4, or 0 for a continous byte stream. */
+				 * 1, 2, or 4, or 0 for a continuous byte stream. */
     int fd_from_erl;	        /* File descriptor from Erlang. */
     int fd_to_erl;	        /* File descriptor to Erlang. */
     unsigned char* io_buf;      /* Buffer for file i/o. */
diff --git a/erts/emulator/test/port_bif_SUITE_data/port_test.c b/erts/emulator/test/port_bif_SUITE_data/port_test.c
index 28324a56a6..923ab99ccc 100644
--- a/erts/emulator/test/port_bif_SUITE_data/port_test.c
+++ b/erts/emulator/test/port_bif_SUITE_data/port_test.c
@@ -39,7 +39,7 @@ extern int errno;
 typedef struct {
     char* progname;	        /* Name of this program (from argv[0]). */
     int header_size;	        /* Number of bytes in each packet header:
-				 * 1, 2, or 4, or 0 for a continous byte stream. */
+				 * 1, 2, or 4, or 0 for a continuous byte stream. */
     int fd_from_erl;	        /* File descriptor from Erlang. */
     int fd_to_erl;	        /* File descriptor to Erlang. */
     unsigned char* io_buf;      /* Buffer for file i/o. */
diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl
index 6a772bf7c9..6bd1eb1e1e 100644
--- a/erts/emulator/test/system_info_SUITE.erl
+++ b/erts/emulator/test/system_info_SUITE.erl
@@ -174,7 +174,7 @@ memory(Config) when is_list(Config) ->
     %%
 
     erts_debug:set_internal_state(available_internal_state, true),
-    %% Use a large heap size on the controling process in
+    %% Use a large heap size on the controlling process in
     %% order to avoid changes in its heap size during
     %% comparisons.
     MinHeapSize = process_flag(min_heap_size, 1024*1024), 
diff --git a/erts/emulator/test/time_SUITE.erl b/erts/emulator/test/time_SUITE.erl
index 9501569814..c13d03bcc4 100644
--- a/erts/emulator/test/time_SUITE.erl
+++ b/erts/emulator/test/time_SUITE.erl
@@ -132,7 +132,7 @@ local_to_univ_utc(Config) when is_list(Config) ->
     end.
 
 
-%% Tests conversion from univeral to local time.
+%% Tests conversion from universal to local time.
 
 univ_to_local(Config) when is_list(Config) ->
     test_univ_to_local(test_data()).
diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl
index d663cc548c..a90701c5d2 100644
--- a/erts/emulator/test/z_SUITE.erl
+++ b/erts/emulator/test/z_SUITE.erl
@@ -226,7 +226,7 @@ pollset_size(Config) when is_list(Config) ->
 				 "Pollset size information not available"}
 		  end;
 	      false ->
-		  %% Somtimes we have fewer descriptors in the
+		  %% Sometimes we have fewer descriptors in the
 		  %% pollset at the end than when we started, but
 		  %% that is ok as long as there are at least 2
 		  %% descriptors (dist listen socket and
-- 
cgit v1.2.3


From d73423cdba178c166f25e00a4608ccc8d0465937 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 14 Feb 2017 15:53:04 +0100
Subject: erts: Fix round/1 for large floats

1> round(6209607916799025.0).
6209607916799026

Problem: Adding/subtracting 0.5 and large double floats between
(1 bsl 52) and (1 bsl 53) does not give reliable results.

Solution: Use round() function in math.h.
---
 erts/emulator/beam/erl_bif_guard.c   | 5 ++---
 erts/emulator/test/guard_SUITE.erl   | 1 +
 erts/emulator/test/num_bif_SUITE.erl | 3 +++
 3 files changed, 6 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c
index b42d2dc28b..74cf7cf11a 100644
--- a/erts/emulator/beam/erl_bif_guard.c
+++ b/erts/emulator/beam/erl_bif_guard.c
@@ -157,7 +157,7 @@ BIF_RETTYPE round_1(BIF_ALIST_1)
     GET_DOUBLE(BIF_ARG_1, f);
 
     /* round it and return the resultant integer */
-    res = double_to_integer(BIF_P, (f.fd > 0.0) ? f.fd + 0.5 : f.fd - 0.5);
+    res = double_to_integer(BIF_P, round(f.fd));
     BIF_RET(res);
 }
 
@@ -597,8 +597,7 @@ Eterm erts_gc_round_1(Process* p, Eterm* reg, Uint live)
     }
     GET_DOUBLE(arg, f);
 
-    return gc_double_to_integer(p, (f.fd > 0.0) ? f.fd + 0.5 : f.fd - 0.5,
-				reg, live);
+    return gc_double_to_integer(p, round(f.fd), reg, live);
 }
 
 Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live)
diff --git a/erts/emulator/test/guard_SUITE.erl b/erts/emulator/test/guard_SUITE.erl
index e155e5f49f..54ee710363 100644
--- a/erts/emulator/test/guard_SUITE.erl
+++ b/erts/emulator/test/guard_SUITE.erl
@@ -317,6 +317,7 @@ guard_bifs(Config) when is_list(Config) ->
     try_gbif('float/1', Big, float(id(Big))),
     try_gbif('trunc/1', Float, 387924.0),
     try_gbif('round/1', Float, 387925.0),
+    try_gbif('round/1', 6209607916799025.0, 6209607916799025),
     try_gbif('length/1', [], 0),
 
     try_gbif('length/1', [a], 1),
diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl
index d1c9648017..bb85738454 100644
--- a/erts/emulator/test/num_bif_SUITE.erl
+++ b/erts/emulator/test/num_bif_SUITE.erl
@@ -293,6 +293,9 @@ t_round(Config) when is_list(Config) ->
     4294967297 = round(id(4294967296.9)),
     -4294967296 = -round(id(4294967296.1)),
     -4294967297 = -round(id(4294967296.9)),
+
+    6209607916799025 = round(id(6209607916799025.0)),
+    -6209607916799025 = round(id(-6209607916799025.0)),
     ok.
 
 t_trunc(Config) when is_list(Config) ->
-- 
cgit v1.2.3


From 118de47d703e303aea7f4575849a37c11416ba14 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 14 Feb 2017 18:26:31 +0100
Subject: erts: Add deallocation veto for magic destructors

A magic destructor can return 0 and thereby take control
and prolong the lifetime of a magic binary.
---
 erts/emulator/beam/beam_load.c      |  5 +++--
 erts/emulator/beam/binary.c         |  6 ++++--
 erts/emulator/beam/dist.c           |  4 +++-
 erts/emulator/beam/dist.h           |  2 +-
 erts/emulator/beam/erl_bif_binary.c | 15 ++++++++-------
 erts/emulator/beam/erl_bif_info.c   |  4 ++--
 erts/emulator/beam/erl_bif_re.c     |  3 ++-
 erts/emulator/beam/erl_binary.h     | 19 +++++++++++--------
 erts/emulator/beam/erl_db_util.c    |  5 +++--
 erts/emulator/beam/erl_db_util.h    |  2 +-
 erts/emulator/beam/erl_map.c        |  3 ++-
 erts/emulator/beam/erl_nif.c        |  3 ++-
 erts/emulator/beam/erl_ptab.c       |  6 ++++--
 erts/emulator/beam/erl_unicode.c    |  3 ++-
 erts/emulator/beam/external.c       |  6 ++++--
 erts/emulator/hipe/hipe_load.c      |  3 ++-
 16 files changed, 54 insertions(+), 35 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index e75e7afd54..48206a75a8 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -488,7 +488,7 @@ typedef struct LoaderState {
 static void free_loader_state(Binary* magic);
 static ErlHeapFragment* new_literal_fragment(Uint size);
 static void free_literal_fragment(ErlHeapFragment*);
-static void loader_state_dtor(Binary* magic);
+static int loader_state_dtor(Binary* magic);
 #ifdef HIPE
 static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 				  Eterm group_leader, Eterm module,
@@ -1026,7 +1026,7 @@ static void free_literal_fragment(ErlHeapFragment* bp)
 /*
  * This destructor function can safely be called multiple times.
  */
-static void
+static int
 loader_state_dtor(Binary* magic)
 {
     LoaderState* stp = ERTS_MAGIC_BIN_DATA(magic);
@@ -1111,6 +1111,7 @@ loader_state_dtor(Binary* magic)
      */
 
     ASSERT(stp->genop_blocks == 0);
+    return 1;
 }
 
 #ifdef HIPE
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index d38097fb12..be43f6e1ac 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -350,9 +350,10 @@ typedef struct {
     Uint bitoffs;
 } ErtsB2LState;
 
-static void b2l_state_destructor(Binary *mbp)
+static int b2l_state_destructor(Binary *mbp)
 {
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == b2l_state_destructor);
+    return 1;
 }
 
 static BIF_RETTYPE
@@ -723,12 +724,13 @@ list_to_binary_engine(ErtsL2BState *sp)
     }
 }
 
-static void
+static int
 l2b_state_destructor(Binary *mbp)
 {
     ErtsL2BState *sp = ERTS_MAGIC_BIN_DATA(mbp); 
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == l2b_state_destructor);
     DESTROY_SAVED_ESTACK(&sp->buf.iolist.estack);
+    return 1;
 }
 
 static ERTS_INLINE Eterm
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 390f2a0db3..da1083100b 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -713,7 +713,7 @@ static void clear_dist_entry(DistEntry *dep)
     }
 }
 
-void erts_dsend_context_dtor(Binary* ctx_bin)
+int erts_dsend_context_dtor(Binary* ctx_bin)
 {
     ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(ctx_bin);
     switch (ctx->dss.phase) {
@@ -730,6 +730,8 @@ void erts_dsend_context_dtor(Binary* ctx_bin)
     }
     if (ctx->dep_to_deref)
 	erts_deref_dist_entry(ctx->dep_to_deref);
+
+    return 1;
 }
 
 Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx)
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index e82b416286..8799e54057 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -375,7 +375,7 @@ extern int erts_dsig_send_monitor(ErtsDSigData *, Eterm, Eterm, Eterm);
 extern int erts_dsig_send_m_exit(ErtsDSigData *, Eterm, Eterm, Eterm, Eterm);
 
 extern int erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx);
-extern void erts_dsend_context_dtor(Binary*);
+extern int erts_dsend_context_dtor(Binary*);
 extern Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx);
 
 extern int erts_dist_command(Port *prt, int reds);
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index e57cd06cec..deab371e3e 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -239,13 +239,13 @@ static void dump_ac_node(ACNode *node, int indent, int ch);
 /*
  * Callback for the magic binary
  */
-static void cleanup_my_data_ac(Binary *bp)
+static int cleanup_my_data_ac(Binary *bp)
 {
-    return;
+    return 1;
 }
-static void cleanup_my_data_bm(Binary *bp)
+static int cleanup_my_data_bm(Binary *bp)
 {
-    return;
+    return 1;
 }
 
 /*
@@ -2067,7 +2067,7 @@ static int do_search_backward(CommonData *cd, Uint *posp, Uint *redsp)
     }
 }
 
-static void cleanup_common_data(Binary *bp)
+static int cleanup_common_data(Binary *bp)
 {
     int i;
     CommonData *cd;
@@ -2084,7 +2084,7 @@ static void cleanup_common_data(Binary *bp)
 	    break;
 	}
     }
-    return;
+    return 1;
 }
 
 static BIF_RETTYPE do_longest_common(Process *p, Eterm list, int direction)
@@ -2563,7 +2563,7 @@ typedef struct {
 
 #define BINARY_COPY_LOOP_FACTOR 100
 
-static void cleanup_copy_bin_state(Binary *bp)
+static int cleanup_copy_bin_state(Binary *bp)
 {
     CopyBinState *cbs = (CopyBinState *) ERTS_MAGIC_BIN_DATA(bp);
     if (cbs->result != NULL) {
@@ -2583,6 +2583,7 @@ static void cleanup_copy_bin_state(Binary *bp)
 	break;
     }
     cbs->source_type =  BC_TYPE_EMPTY;
+    return 1;
 }
 
 /*
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index be113b7c88..5a8776076e 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3567,9 +3567,9 @@ BIF_RETTYPE error_logger_warning_map_0(BIF_ALIST_0)
 
 static erts_smp_atomic_t available_internal_state;
 
-static void empty_magic_ref_destructor(Binary *bin)
+static int empty_magic_ref_destructor(Binary *bin)
 {
-
+    return 1;
 }
 
 BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index ba183f24e8..a66b05c6ff 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -600,10 +600,11 @@ static void cleanup_restart_context(RestartContext *rc)
     }
 }
 
-static void cleanup_restart_context_bin(Binary *bp)
+static int cleanup_restart_context_bin(Binary *bp)
 {
     RestartContext *rc = ERTS_MAGIC_BIN_DATA(bp);
     cleanup_restart_context(rc);
+    return 1;
 }
 
 /*
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 4d9c4d6eac..db259be2a7 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -65,7 +65,7 @@ typedef struct magic_binary ErtsMagicBinary;
 struct magic_binary {
     ERTS_INTERNAL_BINARY_FIELDS
     SWord orig_size;
-    void (*destructor)(Binary *);
+    int (*destructor)(Binary *);
     Uint32 refn[ERTS_REF_NUMBERS];
     ErtsAlcType_t alloc_type;
     union {
@@ -335,12 +335,12 @@ ERTS_GLB_INLINE Binary *erts_bin_realloc_fnf(Binary *bp, Uint size);
 ERTS_GLB_INLINE Binary *erts_bin_realloc(Binary *bp, Uint size);
 ERTS_GLB_INLINE void erts_bin_free(Binary *bp);
 ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size,
-                                                  void (*destructor)(Binary *),
+                                                  int (*destructor)(Binary *),
                                                    ErtsAlcType_t alloc_type,
                                                   int unaligned);
 ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size,
-						 void (*destructor)(Binary *));
-ERTS_GLB_INLINE Binary *erts_create_magic_indirection(void (*destructor)(Binary *));
+						 int (*destructor)(Binary *));
+ERTS_GLB_INLINE Binary *erts_create_magic_indirection(int (*destructor)(Binary *));
 ERTS_GLB_INLINE erts_smp_atomic_t *erts_smp_binary_to_magic_indirection(Binary *bp);
 ERTS_GLB_INLINE erts_atomic_t *erts_binary_to_magic_indirection(Binary *bp);
 
@@ -471,7 +471,10 @@ ERTS_GLB_INLINE void
 erts_bin_free(Binary *bp)
 {
     if (bp->flags & BIN_FLAG_MAGIC) {
-	ERTS_MAGIC_BIN_DESTRUCTOR(bp)(bp);
+        if (!ERTS_MAGIC_BIN_DESTRUCTOR(bp)(bp)) {
+            /* Destructor took control of the deallocation */
+            return;
+        }
 	erts_magic_ref_remove_bin(ERTS_MAGIC_BIN_REFN(bp));
         erts_free(ERTS_MAGIC_BIN_ATYPE(bp), (void *) bp);
     }
@@ -482,7 +485,7 @@ erts_bin_free(Binary *bp)
 }
 
 ERTS_GLB_INLINE Binary *
-erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *),
+erts_create_magic_binary_x(Uint size, int (*destructor)(Binary *),
                            ErtsAlcType_t alloc_type,
                            int unaligned)
 {
@@ -504,14 +507,14 @@ erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *),
 }
 
 ERTS_GLB_INLINE Binary *
-erts_create_magic_binary(Uint size, void (*destructor)(Binary *))
+erts_create_magic_binary(Uint size, int (*destructor)(Binary *))
 {
     return erts_create_magic_binary_x(size, destructor,
                                       ERTS_ALC_T_BINARY, 0);
 }
 
 ERTS_GLB_INLINE Binary *
-erts_create_magic_indirection(void (*destructor)(Binary *))
+erts_create_magic_indirection(int (*destructor)(Binary *))
 {
     return erts_create_magic_binary_x(sizeof(ErtsMagicIndirectionWord),
                                       destructor,
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 070e29578f..3ac2bdd3d6 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1702,17 +1702,18 @@ error: /* Here is were we land when compilation failed. */
 /*
 ** Free a match program (in a binary)
 */
-void erts_db_match_prog_destructor(Binary *bprog)
+int erts_db_match_prog_destructor(Binary *bprog)
 {
     MatchProg *prog;
     if (bprog == NULL)
-	return;
+	return 1;
     prog = Binary2MatchProg(bprog);
     if (prog->term_save != NULL) {
 	free_message_buffer(prog->term_save); 
     }
     if (prog->saved_program_buf != NULL)
 	free_message_buffer(prog->saved_program_buf);
+    return 1;
 }
 
 void
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index b2a3bb6c20..471fefe3cb 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -356,7 +356,7 @@ Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr);
 Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags);
 Binary *db_match_set_compile(Process *p, Eterm matchexpr, 
 			     Uint flags);
-void erts_db_match_prog_destructor(Binary *);
+int erts_db_match_prog_destructor(Binary *);
 
 typedef struct match_prog {
     ErlHeapFragment *term_save; /* Only if needed, a list of message 
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 990be8efb2..9062e44b10 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -1188,12 +1188,13 @@ typedef struct HashmapMergeContext_ {
 #endif
 } HashmapMergeContext;
 
-static void hashmap_merge_ctx_destructor(Binary* ctx_bin)
+static int hashmap_merge_ctx_destructor(Binary* ctx_bin)
 {
     HashmapMergeContext* ctx = (HashmapMergeContext*) ERTS_MAGIC_BIN_DATA(ctx_bin);
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == hashmap_merge_ctx_destructor);
 
     PSTACK_DESTROY_SAVED(&ctx->pstack);
+    return 1;
 }
 
 BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1) {
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 3674a29bf8..61ae57a7f0 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2140,7 +2140,7 @@ static void rollback_opened_resource_types(void)
 }
 
 
-static void nif_resource_dtor(Binary* bin)
+static int nif_resource_dtor(Binary* bin)
 {
     ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin);
     ErlNifResourceType* type = resource->type;
@@ -2159,6 +2159,7 @@ static void nif_resource_dtor(Binary* bin)
 	steal_resource_type(type);
 	erts_free(ERTS_ALC_T_NIF, type);
     }
+    return 1;
 }
 
 void* enif_alloc_resource(ErlNifResourceType* type, size_t size)
diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index a132b1d334..ce28708ece 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -733,7 +733,7 @@ erts_ptab_delete_element(ErtsPTab *ptab,
  * erts_ptab_list() implements BIFs listing the content of the table,
  * e.g. erlang:processes/0.
  */
-static void cleanup_ptab_list_bif_data(Binary *bp);
+static int cleanup_ptab_list_bif_data(Binary *bp);
 static int ptab_list_bif_engine(Process *c_p, Eterm *res_accp, Binary *mbp);
 
 
@@ -787,7 +787,7 @@ erts_ptab_list(Process *c_p, ErtsPTab *ptab)
     return ret_val;
 }
 
-static void
+static int
 cleanup_ptab_list_bif_data(Binary *bp)
 {
     ErtsPTabListBifData *ptlbdp = ERTS_MAGIC_BIN_DATA(bp);
@@ -875,6 +875,8 @@ cleanup_ptab_list_bif_data(Binary *bp)
 
     ERTS_PTAB_LIST_DBG_TRACE(ptlbdp->debug.caller, return);
     ERTS_PTAB_LIST_DBG_CLEANUP(ptlbdp);
+
+    return 1;
 }
 
 static int
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index dd7c589c3d..01629db9ad 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -123,10 +123,11 @@ static void cleanup_restart_context(RestartContext *rc)
     }
 }
 
-static void cleanup_restart_context_bin(Binary *bp)
+static int cleanup_restart_context_bin(Binary *bp)
 {
     RestartContext *rc = ERTS_MAGIC_BIN_DATA(bp);
     cleanup_restart_context(rc);
+    return 1;
 }
 
 static RestartContext *get_rc_from_bin(Eterm mref)
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index f9cba0aec0..205a7711ec 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1398,12 +1398,13 @@ static void b2t_destroy_context(B2TContext* context)
     }
 }
 
-static void b2t_context_destructor(Binary *context_bin)
+static int b2t_context_destructor(Binary *context_bin)
 {
     B2TContext* ctx = (B2TContext*) ERTS_MAGIC_BIN_DATA(context_bin);
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor);
 
     b2t_destroy_context(ctx);
+    return 1;
 }
 
 static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1)
@@ -1808,7 +1809,7 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) {
 #endif
 #define TERM_TO_BINARY_MEMCPY_FACTOR 8
 
-static void ttb_context_destructor(Binary *context_bin)
+static int ttb_context_destructor(Binary *context_bin)
 {
     TTBContext *context = ERTS_MAGIC_BIN_DATA(context_bin);
     if (context->alive) {
@@ -1842,6 +1843,7 @@ static void ttb_context_destructor(Binary *context_bin)
 	    break;
 	}
     }
+    return 1;
 }
 
 static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags, 
diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
index 2998ed87a2..87c5004d2b 100644
--- a/erts/emulator/hipe/hipe_load.c
+++ b/erts/emulator/hipe/hipe_load.c
@@ -61,7 +61,7 @@ void hipe_free_loader_state(HipeLoaderState *stp)
     stp->module = NIL;
 }
 
-static void
+static int
 hipe_loader_state_dtor(Binary* magic)
 {
     HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic);
@@ -69,6 +69,7 @@ hipe_loader_state_dtor(Binary* magic)
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(magic) == hipe_loader_state_dtor);
 
     hipe_free_loader_state(stp);
+    return 1;
 }
 
 Binary *hipe_alloc_loader_state(Eterm module)
-- 
cgit v1.2.3


From fc5a4a227968001cb031802c5a122cdbb0323b25 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 16 Feb 2017 19:11:54 +0100
Subject: Handle magic refs in db_cleanup_offheap_comp()

---
 erts/emulator/beam/erl_db_util.c | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 3ac2bdd3d6..6f30b1d3dd 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -3104,6 +3104,11 @@ void db_cleanup_offheap_comp(DbTerm* obj)
 		erts_erase_fun_entry(u.fun->fe);
 	    }
 	    break;
+	case REF_SUBTAG:
+	    ASSERT(is_magic_ref_thing(u.hdr));
+	    if (erts_refc_dectest(&u.mref->mb->refc, 0) == 0)
+		erts_bin_free((Binary *)u.mref->mb);		    
+	    break;
 	default:
 	    ASSERT(is_external_header(u.hdr->thing_word));
 	    ASSERT(u.pb != &tmp);
-- 
cgit v1.2.3


From 5263d10ba361cf0666cb6b2730132a018ba67bd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 17 Feb 2017 12:03:08 +0100
Subject: Teach make_preload to handle the new 'AtU8' chunk

26b59dfe67 introduced the new 'AtU8' chunk to support
Unicode atoms.

make_preload strips the pre-loaded BEAM files so that they
only contain essential chunks. It expects to find the old
'Atom' chunk.

Teach make_preload to read the new 'AtU8' chunk instead of the old
chunk. Also produce a nice error message if someone by mistake
compiles the pre-loaded modules with an OTP 19 compiler.
---
 erts/emulator/utils/make_preload | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/make_preload b/erts/emulator/utils/make_preload
index 8b629d9517..bcb2e42614 100755
--- a/erts/emulator/utils/make_preload
+++ b/erts/emulator/utils/make_preload
@@ -90,7 +90,7 @@ foreach $file (@ARGV) {
 	open(FILE, $file) or error("failed to read $file: $!");
 	binmode(FILE);
 	$_ = <FILE>;
-	$_ = beam_strip($_);
+	$_ = beam_strip($_, $file);
 	close(FILE);
 	
 	push(@modules, "  {\"$module\", " . length($_) . ", preloaded_$module},\n");
@@ -147,20 +147,20 @@ sub error {
 }
 
 sub beam_strip {
-    my($beam) = @_;
+    my($beam,$file) = @_;
 
 
     my $size_left = length($beam);
     my %chunk;
     my %needed_chunk = ('Code' => 1,
-			'Atom' => 1,
+			'AtU8' => 1,
 			'ImpT' => 1,
 			'ExpT' => 1,
 			'StrT' => 1,
 			'FunT' => 1,
 			'LitT' => 1);
 
-    die "can't read Beam files for OTP R4 or earlier (sorry)"
+    die "$file: can't read Beam files for OTP R4 or earlier (sorry)"
 	if $beam =~ /^\x7fBEAM!/;
 
     #
@@ -177,7 +177,7 @@ sub beam_strip {
     die "form size $size greater than size ", $size_left, " of module"
 	if $size > $size_left;
     $size_left -= 4;
-    die "not a BEAM file: IFF form type is not 'BEAM'"
+    die "$file: not a BEAM file: IFF form type is not 'BEAM'"
 	unless $beam_id eq 'BEAM';
 
     #
@@ -196,6 +196,14 @@ sub beam_strip {
 	$size_left = length($beam);
     }
 
+    #
+    # Abort if there is no new-style 'AtU8' atom chunk.
+    #
+
+    exists $chunk{'AtU8'} or
+        die "$file: no 'AtU8' chunk (re-compile with " .
+        "OTP 20 or later)\n";
+
     #
     # Create a new beam file with only the useful chunk types.
     #
-- 
cgit v1.2.3


From 21a737586fbaff6794f7b737ea215b8daa35ea3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Magnus=20L=C3=A5ng?= <margnus1@telia.com>
Date: Sun, 19 Feb 2017 10:51:10 +0100
Subject: erts: Drop workarounds for ErLLVM w/ LLVM < 3.9

---
 erts/emulator/beam/erl_process.h      | 13 ++-----------
 erts/emulator/hipe/hipe_mode_switch.c | 27 ---------------------------
 2 files changed, 2 insertions(+), 38 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index ce0989883c..8bf372dad5 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -958,9 +958,11 @@ struct process {
     Eterm* stop;		/* Stack top */
     Eterm* heap;		/* Heap start */
     Eterm* hend;		/* Heap end */
+    Eterm* abandoned_heap;
     Uint heap_sz;		/* Size of heap in words */
     Uint min_heap_size;         /* Minimum size of heap (in words). */
     Uint min_vheap_size;        /* Minimum size of virtual heap (in words). */
+    Uint max_heap_size;         /* Maximum size of heap (in words). */
 
 #if !defined(NO_FPE_SIGNALS) || defined(HIPE)
     volatile unsigned long fp_exception;
@@ -972,16 +974,6 @@ struct process {
     struct hipe_process_state hipe;
 #endif
 
-    /*
-     * Moved to after "struct hipe_process_state hipe", as a temporary fix for
-     * LLVM hard-coding offsetof(struct process, hipe.nstack) (sic!)
-     * (see void X86FrameLowering::adjustForHiPEPrologue(...) in
-     * lib/Target/X86/X86FrameLowering.cpp).
-     *
-     * Used to be below "Eterm* hend".
-     */
-    Eterm* abandoned_heap;
-
     /*
      * Saved x registers.
      */
@@ -1061,7 +1053,6 @@ struct process {
     Eterm *old_hend;            /* Heap pointers for generational GC. */
     Eterm *old_htop;
     Eterm *old_heap;
-    Uint max_heap_size;         /* Maximum size of heap (in words). */
     Uint16 gen_gcs;		/* Number of (minor) generational GCs. */
     Uint16 max_gen_gcs;		/* Max minor gen GCs before fullsweep. */
     ErlOffHeap off_heap;	/* Off-heap data updated by copy_struct(). */
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index 0706f8d2c9..f11223d8b0 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -174,33 +174,6 @@ void hipe_mode_switch_init(void)
 	make_catch(beam_catches_cons(hipe_beam_pc_throw, BEAM_CATCHES_NIL));
 
     hipe_mfa_info_table_init();
-
-#if (defined(__i386__) || defined(__x86_64__)) && defined(__linux__)
-    /* Verify that the offset of c-p->hipe does not change.
-       The ErLLVM hipe backend depends on it being in a specific
-       position. Kostis et al has promised to fix this in upstream
-       llvm by OTP 20, so it should be possible to remove these asserts
-       after that. */
-    ERTS_CT_ASSERT(sizeof(ErtsPTabElementCommon) ==
-                   (sizeof(Eterm) +                   /* id */
-                    sizeof(((ErtsPTabElementCommon*)0)->refc) +
-                    sizeof(ErtsTracer) +              /* tracer */
-                    sizeof(Uint) +                    /* trace_flags */
-                    sizeof(erts_smp_atomic_t) +       /* timer */
-                    sizeof(((ErtsPTabElementCommon*)0)->u)));
-
-    ERTS_CT_ASSERT(offsetof(Process, hipe) ==
-                   (sizeof(ErtsPTabElementCommon) +   /* common */
-                    sizeof(Eterm*) +                  /* htop */
-                    sizeof(Eterm*) +                  /* stop */
-                    sizeof(Eterm*) +                  /* heap */
-                    sizeof(Eterm*) +                  /* hend */
-                    sizeof(Uint) +                    /* heap_sz */
-                    sizeof(Uint) +                    /* min_heap_size */
-                    sizeof(Uint) +                    /* min_vheap_size */
-                    sizeof(volatile unsigned long))); /* fp_exception */
-#endif
-
 }
 
 void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure)
-- 
cgit v1.2.3


From 0a144d1fc4bebefcc74b0cb63fe76809bc041789 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 17 Feb 2017 15:26:01 +0100
Subject: Ensure prim_eval:'receive' wont clobber def_arg_reg[0]

def_arg_reg[0] is used for storage of timeout instruction
when a 'receive after' is executed. When a process was
scheduled out inside prim_eval:'receive'/0 due to a function
call, def_arg_reg[0] was overwritten due to storage of live
registers.

prim_eval:'receive'/2 now calls arg_reg_alloc/0 which
bumps all reductions and then calls arg_reg_alloc/7
which will cause an allocation of a new arg_reg array
since def_arg_reg only can hold 6 values. This ensures
that the timeout instruction in def_arg_reg[0] used
for the timeout wont be overwritten.
---
 erts/emulator/test/Makefile            |  1 +
 erts/emulator/test/prim_eval_SUITE.erl | 78 ++++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 erts/emulator/test/prim_eval_SUITE.erl

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index 2e48c475d5..453f819d1b 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -87,6 +87,7 @@ MODULES= \
 	op_SUITE \
 	port_SUITE \
 	port_bif_SUITE \
+	prim_eval_SUITE \
 	process_SUITE \
 	pseudoknot_SUITE \
 	receive_SUITE \
diff --git a/erts/emulator/test/prim_eval_SUITE.erl b/erts/emulator/test/prim_eval_SUITE.erl
new file mode 100644
index 0000000000..3f4965f96d
--- /dev/null
+++ b/erts/emulator/test/prim_eval_SUITE.erl
@@ -0,0 +1,78 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(prim_eval_SUITE).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_testcase/2, end_per_testcase/2,
+	 init_per_group/2, end_per_group/2]).
+
+-export(['ERL-365'/1]).
+
+init_per_testcase(_Case, Config) ->
+    Config.
+
+end_per_testcase(_Case, _Config) ->
+    ok.
+
+suite() ->
+    [{ct_hooks,[ts_install_cth]},
+     {timetrap,{minutes,1}}].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+all() ->
+    ['ERL-365'].
+
+'ERL-365'(Config) when is_list(Config) ->
+    %% def_arg_reg[0] is used for storage of timeout instruction
+    %% when a 'receive after' is executed. When a process was
+    %% scheduled out inside prim_eval:'receive'/0 due to a function
+    %% call, def_arg_reg[0] was overwritten due to storage of live
+    %% registers.
+    P = spawn_link(fun () ->
+                           prim_eval:'receive'(fun (_M) -> 
+                                                       erlang:bump_reductions((1 bsl 27)-1),
+                                                       id(true),
+                                                       nomatch
+                                               end,
+                                               200)
+                   end),
+    receive after 100 -> ok end,
+    P ! {wont, match},
+    receive after 200 -> ok end,
+    ok.
+
+
+
+id(X) ->
+    X.
-- 
cgit v1.2.3


From 7dc39b7f02bf1e12541e91298a44d09d052d8fde Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 16 Feb 2017 19:23:51 +0100
Subject: Fix driver monitor implementation

---
 erts/emulator/beam/io.c | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 7e49e9c48f..e0cc756887 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -7609,10 +7609,21 @@ erl_drv_convert_time_unit(ErlDrvTime val,
 
 static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon)
 {
-    ERTS_CT_ASSERT(sizeof(ErtsOIRefStorage) <= sizeof(ErlDrvMonitor));
-    erts_oiref_storage_save((ErtsOIRefStorage *) mon, ref);
+    ERTS_CT_ASSERT(ERTS_REF_THING_SIZE*sizeof(Uint) <= sizeof(ErlDrvMonitor));
+    ASSERT(is_internal_ordinary_ref(ref));
+    sys_memcpy((void *) mon, (void *) internal_ref_val(ref),
+               ERTS_REF_THING_SIZE*sizeof(Uint));
 }
 
+static Eterm driver_monitor_to_ref(Eterm *hp, const ErlDrvMonitor *mon)
+{
+    Eterm ref;
+    ERTS_CT_ASSERT(ERTS_REF_THING_SIZE*sizeof(Uint) <= sizeof(ErlDrvMonitor));
+    sys_memcpy((void *) hp, (void *) mon, ERTS_REF_THING_SIZE*sizeof(Uint));
+    ref = make_internal_ref(hp);
+    ASSERT(is_internal_ordinary_ref(ref));
+    return ref;
+}
 
 static int do_driver_monitor_process(Port *prt,
 				     ErlDrvTermData process,
@@ -7669,13 +7680,12 @@ int driver_monitor_process(ErlDrvPort drvport,
 static int do_driver_demonitor_process(Port *prt, const ErlDrvMonitor *monitor)
 {
     Eterm heap[ERTS_REF_THING_SIZE];
-    Eterm *hp = &heap[0];
     Process *rp;
     Eterm ref;
     ErtsMonitor *mon;
     Eterm to;
 
-    ref = erts_oiref_storage_make_ref((ErtsOIRefStorage *) monitor, &hp),
+    ref = driver_monitor_to_ref(heap, monitor);
 
     mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref);
     if (mon == NULL) {
@@ -7731,9 +7741,8 @@ static ErlDrvTermData do_driver_get_monitored_process(Port *prt,const ErlDrvMoni
     ErtsMonitor *mon;
     Eterm to;
     Eterm heap[ERTS_REF_THING_SIZE];
-    Eterm *hp = &heap[0];
 
-    ref = erts_oiref_storage_make_ref((ErtsOIRefStorage *) monitor, &hp),
+    ref = driver_monitor_to_ref(heap, monitor);
 
     mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref);
     if (mon == NULL) {
@@ -7767,12 +7776,11 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport,
     return ret;
 }
 
-
 int driver_compare_monitors(const ErlDrvMonitor *monitor1,
 			    const ErlDrvMonitor *monitor2)
 {
-    return erts_oiref_storage_cmp((ErtsOIRefStorage *) monitor1,
-				  (ErtsOIRefStorage *) monitor2);
+    return sys_memcmp((void *) monitor1, (void *) monitor2,
+                      ERTS_REF_THING_SIZE*sizeof(Eterm));
 }
 
 void erts_fire_port_monitor(Port *prt, Eterm ref)
-- 
cgit v1.2.3


From 5adbf961a3c79a6782f8be8336ec26594754e9e8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 20 Feb 2017 19:16:48 +0100
Subject: erts: Add enif_compare_monitors

# Conflicts:
#	erts/emulator/test/nif_SUITE_data/nif_SUITE.c
---
 erts/emulator/beam/erl_nif.c                  |  5 +++++
 erts/emulator/beam/erl_nif_api_funcs.h        |  2 ++
 erts/emulator/test/nif_SUITE.erl              | 11 +++++++----
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 15 ++++++++++++++-
 4 files changed, 28 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 62191c4abf..cd6fbf276b 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3202,6 +3202,11 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit
     return 0;
 }
 
+int enif_compare_monitors(const ErlNifMonitor *monitor1,
+                          const ErlNifMonitor *monitor2)
+{
+    return memcmp(monitor1,monitor2,sizeof(ErlNifMonitor));
+}
 
 /***************************************************************************
  **                              load_nif/2                               **
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 1fc6842acc..01d9e386ed 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -179,6 +179,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_select,(ErlNifEnv* env, ErlNifEvent e, enum ErlNi
 ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type_x,(ErlNifEnv*, const char* name_str, const ErlNifResourceTypeInit*, ErlNifResourceFlags flags, ErlNifResourceFlags* tried));
 ERL_NIF_API_FUNC_DECL(int, enif_monitor_process,(ErlNifEnv*,void* obj,const ErlNifPid*,ErlDrvMonitor *monitor));
 ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const ErlDrvMonitor *monitor));
+ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const ErlNifMonitor*));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
@@ -340,6 +341,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const Er
 #  define enif_open_resource_type_x ERL_NIF_API_FUNC_MACRO(enif_open_resource_type_x)
 #  define enif_monitor_process ERL_NIF_API_FUNC_MACRO(enif_monitor_process)
 #  define enif_demonitor_process ERL_NIF_API_FUNC_MACRO(enif_demonitor_process)
+#  define enif_compare_monitors ERL_NIF_API_FUNC_MACRO(enif_compare_monitors)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index d88ac01e46..9932b526b9 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -598,10 +598,11 @@ monitor_process_a(Config) ->
                                     end
                             end),
                 R_ptr = alloc_monitor_resource_nif(),
-                {0, Mon} = monitor_process_nif(R_ptr, Pid, UseMsgEnv, self()),
+                {0, Mon1} = monitor_process_nif(R_ptr, Pid, UseMsgEnv, self()),
                 [R_ptr] = monitored_by(Pid),
                 Terminator(Pid),
-                [{monitor_resource_down, R_ptr, Pid, Mon}] = flush(),
+                [{monitor_resource_down, R_ptr, Pid, Mon2}] = flush(),
+                0 = compare_monitors_nif(Mon1, Mon2),
                 [] = last_resource_dtor_call(),
                 ok = release_resource(R_ptr),
                 {R_ptr, _, 1} = last_resource_dtor_call()
@@ -650,8 +651,9 @@ monitor_process_c(Config) ->
                              Papa ! {self(), done, R_ptr, Mon},
                              exit
                      end),
-    [{Pid, done, R_ptr, Mon},
-     {monitor_resource_down, R_ptr, Pid, Mon}] = flush(),
+    [{Pid, done, R_ptr, Mon1},
+     {monitor_resource_down, R_ptr, Pid, Mon2}] = flush(),
+    compare_monitors_nif(Mon1, Mon2),
     {R_ptr, _, 1} = last_resource_dtor_call(),
     ok.
 
@@ -2657,6 +2659,7 @@ last_fd_stop_call() -> ?nif_stub.
 alloc_monitor_resource_nif() -> ?nif_stub.
 monitor_process_nif(_,_,_,_) -> ?nif_stub.
 demonitor_process_nif(_,_) -> ?nif_stub.
+compare_monitors_nif(_,_) -> ?nif_stub.
 monitor_frenzy_nif(_,_,_,_) -> ?nif_stub.
 
 %% maps
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 05b4b05e16..a9622e3b8d 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -2407,6 +2407,18 @@ static ERL_NIF_TERM demonitor_process_nif(ErlNifEnv* env, int argc, const ERL_NI
     return enif_make_int(env, res);
 }
 
+static ERL_NIF_TERM compare_monitors_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ErlNifMonitor m1, m2;
+    if (!get_monitor(env, argv[0], &m1)
+        || !get_monitor(env, argv[1], &m2)) {
+        return enif_make_badarg(env);
+    }
+
+    return enif_make_int(env, enif_compare_monitors(&m1, &m2));
+}
+
+
 /*********** monitor_frenzy ************/
 
 struct frenzy_rand_bits
@@ -2763,7 +2775,7 @@ static void frenzy_resource_down(ErlNifEnv* env, void* obj, ErlNifPid* pid,
     for (mix = 0; mix < FRENZY_MONITORS_MAX; mix++) {
         if (r->monv[mix].pid.pid == pid->pid && r->monv[mix].state >= MON_TRYING) {
             enif_mutex_lock(r->monv[mix].lock);
-            if (memcmp(mon, &r->monv[mix].mon, sizeof(*mon)) == 0) {
+            if (enif_compare_monitors(mon, &r->monv[mix].mon) == 0) {
                 assert(r->monv[mix].state >= MON_ACTIVE);
                 r->monv[mix].state = MON_FREE_DOWN;
                 enif_mutex_unlock(r->monv[mix].lock);
@@ -2864,6 +2876,7 @@ static ErlNifFunc nif_funcs[] =
     {"alloc_monitor_resource_nif", 0, alloc_monitor_resource_nif},
     {"monitor_process_nif", 4, monitor_process_nif},
     {"demonitor_process_nif", 2, demonitor_process_nif},
+    {"compare_monitors_nif", 2, compare_monitors_nif},
     {"monitor_frenzy_nif", 4, monitor_frenzy_nif}
 };
 
-- 
cgit v1.2.3


From cb2a5bd9e86ba49d9bbc83b3d8383fbe0cc90715 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 14 Feb 2017 19:25:16 +0100
Subject: erts: Avoid revival of dying resource by dec_term

---
 erts/emulator/beam/erl_binary.h         |  8 ++-
 erts/emulator/beam/erl_nif.c            | 88 ++++++++++++++++-----------------
 erts/emulator/beam/global.h             | 12 ++---
 erts/emulator/sys/common/erl_check_io.c |  2 +-
 4 files changed, 54 insertions(+), 56 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index db259be2a7..946d3cfe03 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -79,11 +79,9 @@ struct magic_binary {
     } u;
 };
 
-#ifdef ARCH_32
-#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 4
-#else
-#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 0
-#endif
+#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN \
+    (offsetof(ErtsMagicBinary,u.aligned.data) - \
+     offsetof(ErtsMagicBinary,u.unaligned.data))
 
 typedef union {
     Binary binary;
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 7e7b78ec14..daec4ac9e9 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2141,7 +2141,7 @@ static void rollback_opened_resource_types(void)
 
 struct destroy_monitor_ctx
 {
-    Binary* resource_bin;
+    ErtsResource* resource;
     int exiting_procs;
     int scheduler;
 };
@@ -2188,30 +2188,26 @@ static void destroy_one_monitor(ErtsMonitor* mon, void* context)
 #endif
     }
     if (is_exiting) {
-        ctx->exiting_procs++;
-        /* +1 for exiting process */
-        erts_refc_inc(&ctx->resource_bin->refc, ctx->exiting_procs);
+        ctx->resource->monitors->pending_failed_fire++;
     }
 
     /* ToDo: Delay destruction after monitor_locks */
     if (rmon) {
         ASSERT(rmon->type == MON_NIF_TARGET);
-        ASSERT(&ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rmon->u.pid)->binary == ctx->resource_bin);
+        ASSERT(rmon->u.resource == ctx->resource);
         erts_destroy_monitor(rmon);
     }
     erts_destroy_monitor(mon);
 }
 
-static int destroy_all_monitors(ErtsMonitor* monitors, Binary* bin)
+static void destroy_all_monitors(ErtsMonitor* monitors, ErtsResource* resource)
 {
     struct destroy_monitor_ctx ctx;
 
     execution_state(NULL, NULL, &ctx.scheduler);
 
-    ctx.resource_bin = bin;
-    ctx.exiting_procs = 0;
+    ctx.resource = resource;
     erts_sweep_monitors(monitors, &destroy_one_monitor, &ctx);
-    return ctx.exiting_procs == 0;
 }
 
 
@@ -2252,31 +2248,32 @@ static int nif_resource_dtor(Binary* bin)
 
     if (resource->monitors) {
         ErtsResourceMonitors* rm = resource->monitors;
-        ErtsMonitor* root;
 
         ASSERT(type->down);
         erts_smp_mtx_lock(&rm->lock);
         ASSERT(erts_refc_read(&bin->refc, 0) == 0);
-        root = rm->root;
-        if (root) {
+        if (rm->root) {
+            ASSERT(!rm->is_dying);
+            destroy_all_monitors(rm->root, resource);
             rm->root = NULL;
-            if (!destroy_all_monitors(root, bin)) {
-                /*
-                 * Resource death struggle prolonged to serve exiting process(es).
-                 * Destructor will be called again when last exiting process
-                 * tries to fire its MON_NIF_TARGET monitor (and fails).
-                 *
-                 * This resource is doomed. It has no "real" references and
-                 * should get not get called upon to do anything except the
-                 * final destructor call.
-                 */
-                ASSERT(erts_refc_read(&bin->refc, 1));
-#ifdef DEBUG
-                resource->dbg_is_dying = 1;
-#endif
-                erts_smp_mtx_unlock(&rm->lock);
-                return 0;
-            }
+        }
+        if (rm->pending_failed_fire) {
+            /*
+             * Resource death struggle prolonged to serve exiting process(es).
+             * Destructor will be called again when last exiting process
+             * tries to fire its MON_NIF_TARGET monitor (and fails).
+             *
+             * This resource is doomed. It has no "real" references and
+             * should get not get called upon to do anything except the
+             * final destructor call.
+             *
+             * We keep refc at 0 and use a separate counter for exiting
+             * processes to avoid resource getting revived by "dec_term".
+             */
+            ASSERT(!rm->is_dying);
+            rm->is_dying = 1;
+            erts_smp_mtx_unlock(&rm->lock);
+            return 0;
         }
         erts_smp_mtx_unlock(&rm->lock);
         erts_smp_mtx_destroy(&rm->lock);
@@ -2326,24 +2323,24 @@ void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref)
     erts_smp_mtx_lock(&rmp->lock);
     rmon = erts_remove_monitor(&rmp->root, ref);
     if (!rmon) {
+        int free_me = (--rmp->pending_failed_fire == 0) && rmp->is_dying;
+        ASSERT(rmp->pending_failed_fire >= 0);
         erts_smp_mtx_unlock(&rmp->lock);
-        /* -1 for exiting process */
-        if (erts_refc_dectest(&bin->binary.refc, 0) == 0) {
+
+        if (free_me) {
+            ASSERT(erts_refc_read(&bin->binary.refc, 0) == 0);
             erts_bin_free(&bin->binary);
         }
         return;
     }
-    ASSERT(!resource->dbg_is_dying);
-    if (erts_refc_inctest(&bin->binary.refc, 1) < 2) {
+    ASSERT(!rmp->is_dying);
+    if (erts_refc_inc_unless(&bin->binary.refc, 0, 0) == 0) {
         /*
          * Racing resource destruction. 
          * To avoid a more complex refc-dance with destructing thread
          * we avoid calling 'down' and just silently remove the monitor.
-         * There are no real references left to this resource and we have
-         * monitors_lock, so it's safe to reset refc back to zero.
          * This can happen even for non smp as destructor calls may be scheduled.
          */
-        erts_refc_init(&bin->binary.refc, 0);
         erts_smp_mtx_unlock(&rmp->lock);
     }
     else {
@@ -2390,13 +2387,14 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t data_sz)
     erts_refc_inc(&bin->refc, 1);
 #ifdef DEBUG
     erts_refc_init(&resource->nif_refc, 1);
-    resource->dbg_is_dying = 0;
 #endif
     erts_refc_inc(&resource->type->refc, 2);
     if (type->down) {
         resource->monitors = (ErtsResourceMonitors*) (resource->data + monitors_offs);
         erts_smp_mtx_init(&resource->monitors->lock, "resource_monitors");
         resource->monitors->root = NULL;
+        resource->monitors->pending_failed_fire = 0;
+        resource->monitors->is_dying = 0;
         resource->monitors->user_data_sz = data_sz;
     }
     else {
@@ -2411,7 +2409,7 @@ void enif_release_resource(void* obj)
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
 
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == NIF_RESOURCE_DTOR);
-    ASSERT(!resource->dbg_is_dying);
+    ASSERT(!(resource->monitors && resource->monitors->is_dying));
 #ifdef DEBUG
     erts_refc_dec(&resource->nif_refc, 0);
 #endif
@@ -2426,7 +2424,7 @@ void enif_keep_resource(void* obj)
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
 
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == NIF_RESOURCE_DTOR);
-    ASSERT(!resource->dbg_is_dying);
+    ASSERT(!(resource->monitors && resource->monitors->is_dying));
 #ifdef DEBUG
     erts_refc_inc(&resource->nif_refc, 1);
 #endif
@@ -2436,7 +2434,7 @@ void enif_keep_resource(void* obj)
 Eterm erts_bld_resource_ref(Eterm** hpp, ErlOffHeap* oh, ErtsResource* resource)
 {
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
-    ASSERT(!resource->dbg_is_dying);
+    ASSERT(!(resource->monitors && resource->monitors->is_dying));
     return erts_mk_magic_ref(hpp, oh, &bin->binary);
 }
 
@@ -2445,6 +2443,7 @@ ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj)
     ErtsResource* resource = DATA_TO_RESOURCE(obj);
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
     Eterm* hp = alloc_heap(env, ERTS_MAGIC_REF_THING_SIZE);
+    ASSERT(!(resource->monitors && resource->monitors->is_dying));
     return erts_mk_magic_ref(&hp, &MSO(env->proc), &bin->binary);
 }
 
@@ -3136,7 +3135,7 @@ int enif_monitor_process(ErlNifEnv* env, void* obj, const ErlNifPid* target_pid,
 
     ASSERT(ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc)->magic_binary.destructor
            == NIF_RESOURCE_DTOR);
-    ASSERT(!rsrc->dbg_is_dying);
+    ASSERT(!(rsrc->monitors && rsrc->monitors->is_dying));
     ASSERT(!rsrc->monitors == !rsrc->type->down);
 
 
@@ -3193,7 +3192,9 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit
 {
     int scheduler;
     ErtsResource* rsrc = DATA_TO_RESOURCE(obj);
+#ifdef DEBUG
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc);
+#endif
     Process *rp;
     ErtsMonitor *mon;
     ErtsMonitor *rmon = NULL;
@@ -3202,7 +3203,7 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit
     int is_exiting;
 
     ASSERT(bin->magic_binary.destructor == NIF_RESOURCE_DTOR);
-    ASSERT(!rsrc->dbg_is_dying);
+    ASSERT(!(rsrc->monitors && rsrc->monitors->is_dying));
 
     execution_state(env, NULL, &scheduler);
 
@@ -3252,8 +3253,7 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit
 #endif
     }
     if (is_exiting) {
-        /* +1 for exiting process */
-        erts_refc_inc(&bin->binary.refc, 2);
+        rsrc->monitors->pending_failed_fire++;
     }
     erts_smp_mtx_unlock(&rsrc->monitors->lock);
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index d4a1225bdd..776f2c599b 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -87,6 +87,9 @@ typedef struct
 {
     erts_smp_mtx_t lock;
     ErtsMonitor* root;
+    int pending_failed_fire;
+    int is_dying;
+
     size_t user_data_sz;
 } ErtsResourceMonitors;
 
@@ -94,14 +97,11 @@ typedef struct ErtsResource_
 {
     struct enif_resource_type_t* type;
     ErtsResourceMonitors* monitors;
-#ifdef ARCH_32
-    byte align__[4];
-#endif
 #ifdef DEBUG
     erts_refc_t nif_refc;
-    int dbg_is_dying;
-# ifdef ARCH_64
-    byte dbg_align__[4];
+#else
+# ifdef ARCH_32
+    byte align__[4];
 # endif
 #endif
     char data[1];
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 2214a1937a..1c97df4201 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1223,7 +1223,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
     DTRACE_CHARBUF(name, 64);
 #endif
 
-    ASSERT(!resource->dbg_is_dying);
+    ASSERT(!(resource->monitors && resource->monitors->is_dying));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) {
-- 
cgit v1.2.3


From 68ad03bc3661f4bf82afbbde65a8d04861a6f799 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 13 Feb 2017 18:40:32 +0100
Subject: Expand nif_SUITE:monitor_frenzy with binary_to_term

to provoke resource revival race.
---
 erts/emulator/test/nif_SUITE.erl              | 32 +++++++++++++++++++--------
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 29 ++++++++++++++----------
 2 files changed, 40 insertions(+), 21 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 5857ed51e4..d29537e3ef 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -720,6 +720,8 @@ monitored_by(Pid) ->
 
 -define(FRENZY_RAND_BITS, 25).
 
+%% Exercise monitoring from NIF resources by randomly
+%% create/destruct processes, resources and monitors.
 monitor_frenzy(Config) ->
     ensure_lib_loaded(Config),
 
@@ -731,7 +733,7 @@ monitor_frenzy(Config) ->
     spawn_link(fun() ->
                        SelfPix = monitor_frenzy_nif(init, ?FRENZY_RAND_BITS, 0, 0),
                        unlink(Master),
-                       frenzy(SelfPix, undefined)
+                       frenzy(SelfPix, {undefined, []})
           end),
     receive after 5*1000 -> ok end,
 
@@ -757,7 +759,7 @@ frenzy(SelfPix, State0) ->
     State1 = frenzy_do_op(SelfPix, Op, (Rnd bsr 2), State0),
     frenzy(SelfPix, State1).
 
-frenzy_do_op(SelfPix, Op, Rnd, Pid0) ->
+frenzy_do_op(SelfPix, Op, Rnd, {Pid0,RBins}=State0) ->
     case Op of
         0 -> % add/remove process
             Papa = self(),
@@ -778,24 +780,36 @@ frenzy_do_op(SelfPix, Op, Rnd, Pid0) ->
                      end,
             case monitor_frenzy_nif(Op, Rnd, SelfPix, NewPid) of
                 NewPix when is_integer(NewPix) ->
-                    NewPid ! {go, NewPix, undefined},
-                    undefined;
+                    NewPid ! {go, NewPix, {undefined, []}},
+                    {undefined, RBins};
                 ExitPid when is_pid(ExitPid) ->
                     false = (ExitPid =:= self()),
                     exit(ExitPid,die),
-                    NewPid;
+                    {NewPid, RBins};
                 done ->
                     done
             end;
+
+        3 ->
+            %% Try provoke revival-race of resource from magic ref external format
+            _ = [binary_to_term(B) || B <- RBins],
+            {Pid0, []};
         _ ->
             case monitor_frenzy_nif(Op, Rnd, SelfPix, undefined) of
-                ok -> Pid0;
-                0 -> Pid0;
-                1 -> Pid0;
+                Rsrc when ?is_resource(Rsrc) ->
+                    %% Store resource in ext format only, for later revival
+                    State1 = {Pid0, [term_to_binary(Rsrc) | RBins]},
+                    gc_and_return(State1);
+                ok -> State0;
+                0 -> State0;
+                1 -> State0;
                 done -> done
             end
     end.
 
+gc_and_return(RetVal) ->
+    erlang:garbage_collect(),
+    RetVal.
 
 hipe(Config) when is_list(Config) ->
     Data = proplists:get_value(data_dir, Config),
@@ -1171,7 +1185,7 @@ resource_new_do2(Type) ->
     ResB = make_new_resource(Type, BinB),
     true = is_reference(ResA),
     true = is_reference(ResB),
-    ResA /= ResB,
+    true = (ResA /= ResB),
     {PtrA,BinA} = get_resource(Type, ResA),
     {PtrB,BinB} = get_resource(Type, ResB),
     true = (PtrA =/= PtrB),
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index d5dd379d76..97cb7e55d6 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -2669,19 +2669,24 @@ static ERL_NIF_TERM monitor_frenzy_nif(ErlNifEnv* env, int argc, const ERL_NIF_T
             }
             DBG_TRACE2("New resource at r=%p rix=%u\n", r, rix);
 	}
-	else if (rand_bits(&rnd, 3) == 0) {
+        else {
+            unsigned int resource_op = rand_bits(&rnd, 3);
             r = resv[rix].obj;
-            resv[rix].obj = NULL;
-            resv[rix].release_cnt++;
-	    enif_mutex_unlock(resv[rix].lock);
-            DBG_TRACE2("Delete resource at r=%p rix=%u\n", r, rix);
-	    enif_release_resource(r);
-	    retval = atom_ok;
-	    break;
-	}
-	else {
-	    r = resv[rix].obj;
-	}
+            if (resource_op == 0) {
+                resv[rix].obj = NULL;
+                resv[rix].release_cnt++;
+                enif_mutex_unlock(resv[rix].lock);
+                DBG_TRACE2("Delete resource at r=%p rix=%u\n", r, rix);
+                enif_release_resource(r);
+                retval = atom_ok;
+                break;
+            }
+            else if (resource_op == 1) {
+                retval = enif_make_resource(env, r);
+                enif_mutex_unlock(resv[rix].lock);
+                break;
+            }
+        }
         enif_keep_resource(r);
         enif_mutex_unlock(resv[rix].lock);
 
-- 
cgit v1.2.3


From 471a22dd081da31933c883afe686007793812075 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 13 Feb 2017 19:58:45 +0100
Subject: Expand nif_SUITE:monitor_frenzy to verify dtor calls

---
 erts/emulator/test/nif_SUITE.erl              |  7 ++++-
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 39 ++++++++++++++++++++++++---
 2 files changed, 42 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index d29537e3ef..8439b28010 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -742,7 +742,12 @@ monitor_frenzy(Config) ->
     Pids = monitor_frenzy_nif(stop, 0, 0, 0),
     io:format("stats = ~p\n", [monitor_frenzy_nif(stats, 0, 0, 0)]),
 
-    lists:foreach(fun(P) -> exit(P, stop) end, Pids),
+    lists:foreach(fun(P) ->
+                          MRef = monitor(process, P),
+                          exit(P, stop),
+                          {'DOWN', MRef, process, P, _} = receive_any()
+                  end,
+                  Pids),
 
     io:format("stats = ~p\n", [monitor_frenzy_nif(stats, 0, 0, 0)]),
 
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 97cb7e55d6..4f003d11f5 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -2484,9 +2484,11 @@ struct frenzy_resource {
 };
 struct frenzy_reslot {
     ErlNifMutex* lock;
+    int stopped;
     struct frenzy_resource* obj;
     unsigned long alloc_cnt;
     unsigned long release_cnt;
+    unsigned long dtor_cnt;
 };
 static struct frenzy_reslot resv[FRENZY_RESOURCES_MAX];
 
@@ -2536,8 +2538,10 @@ static ERL_NIF_TERM monitor_frenzy_nif(ErlNifEnv* env, int argc, const ERL_NIF_T
             for (rix = 0; rix < FRENZY_RESOURCES_MAX; rix++) {
                 resv[rix].lock = enif_mutex_create("nif_SUITE:monitor_frenzy.resv.lock");
                 resv[rix].obj = NULL;
+                resv[rix].stopped = 0;
                 resv[rix].alloc_cnt = 0;
                 resv[rix].release_cnt = 0;
+                resv[rix].dtor_cnt = 0;
             }
 
             /* Add self as first process */
@@ -2553,12 +2557,14 @@ static ERL_NIF_TERM monitor_frenzy_nif(ErlNifEnv* env, int argc, const ERL_NIF_T
             ERL_NIF_TERM hist[FRENZY_PROCS_MAX];
             unsigned long res_alloc_cnt = 0;
             unsigned long res_release_cnt = 0;
+            unsigned long res_dtor_cnt = 0;
             for (ref_ix = 0; ref_ix < FRENZY_PROCS_MAX; ref_ix++) {
                 hist[ref_ix] = enif_make_ulong(env, proc_histogram[ref_ix]);
             }
             for (rix = 0; rix < FRENZY_RESOURCES_MAX; rix++) {
                 res_alloc_cnt += resv[rix].alloc_cnt;
                 res_release_cnt += resv[rix].release_cnt;
+                res_dtor_cnt += resv[rix].dtor_cnt;
             }
 
             return
@@ -2569,12 +2575,29 @@ static ERL_NIF_TERM monitor_frenzy_nif(ErlNifEnv* env, int argc, const ERL_NIF_T
                                              enif_make_ulong(env, spawn_cnt)),
                             enif_make_tuple2(env, enif_make_string(env, "kill_cnt", ERL_NIF_LATIN1),
                                              enif_make_ulong(env, kill_cnt)),
-                            enif_make_tuple3(env, enif_make_string(env, "resource_alloc", ERL_NIF_LATIN1),
+                            enif_make_tuple4(env, enif_make_string(env, "resource_alloc", ERL_NIF_LATIN1),
                                              enif_make_ulong(env, res_alloc_cnt),
-                                             enif_make_ulong(env, res_release_cnt)));
+                                             enif_make_ulong(env, res_release_cnt),
+                                             enif_make_ulong(env, res_dtor_cnt)));
 
         }
         else if (Op == atom_stop && procs_lock) {  /* stop all */
+
+            /* Release all resources */
+            for (rix = 0; rix < FRENZY_RESOURCES_MAX; rix++) {
+                enif_mutex_lock(resv[rix].lock);
+                r = resv[rix].obj;
+                if (r) {
+                    resv[rix].obj =  NULL;
+                    resv[rix].release_cnt++;
+                }
+                resv[rix].stopped = 1;
+                enif_mutex_unlock(resv[rix].lock);
+                if (r)
+                    enif_release_resource(r);
+            }
+
+            /* Remove and return all pids */
             retval = enif_make_list(env, 0);
             enif_mutex_lock(procs_lock);
             for (ref_ix = 0; ref_ix < nprocs; ref_ix++) {
@@ -2587,6 +2610,7 @@ static ERL_NIF_TERM monitor_frenzy_nif(ErlNifEnv* env, int argc, const ERL_NIF_T
             nprocs = 0;
             old_nprocs = 0;
             enif_mutex_unlock(procs_lock);
+
             return retval;
         }
         return enif_make_badarg(env);
@@ -2655,7 +2679,12 @@ static ERL_NIF_TERM monitor_frenzy_nif(ErlNifEnv* env, int argc, const ERL_NIF_T
 	while (enif_mutex_trylock(resv[rix].lock) == EBUSY) {
 	    rix = (rix + inc) % FRENZY_RESOURCES_MAX;
 	}
-	if (resv[rix].obj == NULL) {
+        if (resv[rix].stopped) {
+            retval = atom_done;
+            enif_mutex_unlock(resv[rix].lock);
+            break;
+        }
+        else if (resv[rix].obj == NULL) {
 	    r = enif_alloc_resource(frenzy_resource_type,
 				    sizeof(struct frenzy_resource));
 	    resv[rix].obj = r;
@@ -2779,6 +2808,10 @@ static void frenzy_resource_dtor(ErlNifEnv* env, void* obj)
 
     DBG_TRACE2("DTOR r=%p rix=%u\n", r, r->rix);
 
+    enif_mutex_lock(resv[r->rix].lock);
+    resv[r->rix].dtor_cnt++;
+    enif_mutex_unlock(resv[r->rix].lock);
+
     for (mix = 0; mix < FRENZY_MONITORS_MAX; mix++) {
         assert(r->monv[mix].state != MON_PENDING);
         enif_mutex_destroy(r->monv[mix].lock);
-- 
cgit v1.2.3


From 839ad04bbacb329c89568371bbf5a28d4b2bba25 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 14 Feb 2017 14:23:15 +0100
Subject: Fix whitebox monitor tests

---
 erts/emulator/test/erl_link_SUITE.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/erl_link_SUITE.erl b/erts/emulator/test/erl_link_SUITE.erl
index 89e1aefb50..9258897764 100644
--- a/erts/emulator/test/erl_link_SUITE.erl
+++ b/erts/emulator/test/erl_link_SUITE.erl
@@ -60,7 +60,7 @@
 
 % These are to be kept in sync with erl_monitors.h 
 -define(MON_ORIGIN, 1).
--define(MON_TARGET, 3).
+-define(MON_TARGET, 2).
 
 
 -record(erl_link, {type = ?LINK_UNDEF,
@@ -69,7 +69,7 @@
 
 % This is to be kept in sync with erl_bif_info.c (make_monitor_list)
 
--record(erl_monitor, {type,        % MON_ORIGIN or MON_TARGET (1 or 3)
+-record(erl_monitor, {type,        % MON_ORIGIN or MON_TARGET
                       ref,
                       pid,         % Process or nodename
                       name = []}). % registered name or []
-- 
cgit v1.2.3


From 28e6aeb8c0cc0e8b8821be957e3ee9c1eb015de6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 14 Feb 2017 16:41:19 +0100
Subject: Fix enif_select for windows

---
 erts/emulator/beam/erl_nif.h            | 8 +++++---
 erts/emulator/sys/common/erl_check_io.c | 3 ++-
 2 files changed, 7 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 9ff28a30cc..ac45f3ac81 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -138,9 +138,11 @@ typedef struct
     void* ref_bin;
 }ErlNifBinary;
 
-//#ifndef ERL_SYS_DRV
-typedef int ErlNifEvent; /* An event to be selected on. */
-//#endif
+#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
+typedef void* ErlNifEvent; /* FIXME: Use 'HANDLE' somehow without breaking existing source */
+#else
+typedef int ErlNifEvent;
+#endif
 
 /* Return bits from enif_select: */
 #define ERL_NIF_SELECT_STOP_CALLED    (1 << 0)
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 1c97df4201..8191b6e53f 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -201,7 +201,8 @@ static ERTS_INLINE ErtsDrvEventState* hash_new_drv_ev_state(ErtsSysFdType fd)
 #if ERTS_CIO_HAVE_DRV_EVENT
     tmpl.driver.event = NULL;
 #endif
-    tmpl.driver.drv_ptr = NULL;
+    tmpl.driver.nif = NULL;
+    tmpl.driver.stop.drv_ptr = NULL;
     tmpl.events = 0;
     tmpl.remove_cnt = 0;
     tmpl.type = ERTS_EV_TYPE_NONE;
-- 
cgit v1.2.3


From d7abb5e2efc5dfd0f100b9a8978bc9e87e15cd2d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 16 Feb 2017 14:42:24 +0100
Subject: erts: Skip nif_SUITE:select on windows

for now...
---
 erts/emulator/test/nif_SUITE.erl              |  7 +++++++
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 15 ++++++++++++---
 2 files changed, 19 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 8439b28010..c1d9632fb5 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -128,6 +128,13 @@ init_per_testcase(hipe, Config) ->
 	undefined -> {skip, "HiPE is disabled"};
 	_ -> Config
     end;
+init_per_testcase(select, Config) ->
+    case os:type() of
+        {win32,_} ->
+            {skip, "Test not yet implemented for windows"};
+        _ ->
+            Config
+    end;
 init_per_testcase(_Case, Config) ->
     Config.
 
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 4f003d11f5..69c5f09bd5 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -23,10 +23,10 @@
 #include <string.h>
 #include <assert.h>
 #include <limits.h>
+#include <errno.h>
 #ifndef __WIN32__
 #include <unistd.h>
 #include <fcntl.h>
-#include <errno.h>
 #endif
 
 #include "nif_mod.h"
@@ -143,7 +143,7 @@ static ErlNifResourceTypeInit fd_rt_init = {
     fd_resource_stop
 };
 struct fd_resource {
-    int fd;
+    ErlNifEvent fd;
     int was_selected;
     ErlNifPid pid;
 };
@@ -2144,7 +2144,7 @@ static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
     int retval;
 
     if (!get_fd(env, argv[0], &fdr)
-        || !enif_get_uint(env, argv[1], &mode)
+        || !enif_get_uint(env, argv[1], (unsigned int*)&mode)
         || !enif_get_resource(env, argv[2], fd_resource_type, &obj))
     {
         return enif_make_badarg(env);
@@ -2164,6 +2164,7 @@ static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
     return enif_make_int(env, retval);
 }
 
+#ifndef __WIN32__
 static ERL_NIF_TERM pipe_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     struct fd_resource* read_rsrc;
@@ -2274,15 +2275,21 @@ static ERL_NIF_TERM is_closed_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM a
 
     return fdr->fd < 0 ? atom_true : atom_false;
 }
+#endif /* !__WIN32__ */
+
 
 static void fd_resource_dtor(ErlNifEnv* env, void* obj)
 {
     struct fd_resource* fdr = (struct fd_resource*)obj;
     resource_dtor(env, obj);
+#ifdef __WIN32__
+    abort();
+#else
     if (fdr->fd >= 0) {
         assert(!fdr->was_selected);
         close(fdr->fd);
     }
+#endif
 }
 
 static struct {
@@ -2924,10 +2931,12 @@ static ErlNifFunc nif_funcs[] =
     {"port_command_nif", 2, port_command},
     {"format_term_nif", 2, format_term},
     {"select_nif", 5, select_nif},
+#ifndef __WIN32__
     {"pipe_nif", 0, pipe_nif},
     {"write_nif", 2, write_nif},
     {"read_nif", 2, read_nif},
     {"is_closed_nif", 1, is_closed_nif},
+#endif
     {"last_fd_stop_call", 0, last_fd_stop_call},
     {"alloc_monitor_resource_nif", 0, alloc_monitor_resource_nif},
     {"monitor_process_nif", 4, monitor_process_nif},
-- 
cgit v1.2.3


From d8d8301a252579a000b24bab87d26549da0e813a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 17 Feb 2017 17:54:03 +0100
Subject: Remove faulty debug ASSERT

Why did I add that?
---
 erts/emulator/sys/common/erl_poll.c | 1 -
 1 file changed, 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index 3cb0eb31f5..5e7ae8953a 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -3029,7 +3029,6 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet ps,
 	    ev[fd] = 0;
 	else {
 	    ev[fd] = ps->fds_status[fd].events;
-            ASSERT(ps->fds_status[fd].used_events == ev[fd]);
             if (
 #if ERTS_POLL_USE_WAKEUP_PIPE
                 fd == ps->wake_fds[0] || fd == ps->wake_fds[1] ||
-- 
cgit v1.2.3


From af7cf70ca22a34add7836963d086ca0764f4fbae Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 20 Feb 2017 20:20:29 +0100
Subject: Fix ErlNifMonitor handling

---
 erts/emulator/beam/erl_nif.c                  | 7 ++-----
 erts/emulator/beam/global.h                   | 1 +
 erts/emulator/beam/io.c                       | 6 +++---
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 4 ++--
 4 files changed, 8 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index daec4ac9e9..e6da4c1a76 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2305,9 +2305,6 @@ void erts_resource_stop(ErtsResource* resource, ErlNifEvent e,
     post_nif_noproc(&msg_env);
 }
 
-/* SVERK SVERK: Move to ?.h */
-void erts_ref_to_driver_monitor(Eterm ref, ErlDrvMonitor*);
-
 void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref)
 {
     ErtsMonitor* rmon;
@@ -3207,8 +3204,8 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit
 
     execution_state(env, NULL, &scheduler);
 
-    memcpy(ref_heap, monitor, sizeof(Eterm)*ERTS_REF_THING_SIZE);
-    ref = make_internal_ref(ref_heap);
+    ref = erts_driver_monitor_to_ref(ref_heap, monitor);
+
     erts_smp_mtx_lock(&rsrc->monitors->lock);
     mon = erts_remove_monitor(&rsrc->monitors->root, ref);
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 776f2c599b..c4c848f49f 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1176,6 +1176,7 @@ void erts_stale_drv_select(Eterm, ErlDrvPort, ErlDrvEvent, int, int);
 Port *erts_get_heart_port(void);
 void erts_emergency_close_ports(void);
 void erts_ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon);
+Eterm erts_driver_monitor_to_ref(Eterm* hp, const ErlDrvMonitor *mon);
 
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT)
 void erts_lcnt_enable_io_lock_count(int enable);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 9b525cc100..84dc00f641 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -7615,7 +7615,7 @@ void erts_ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon)
                ERTS_REF_THING_SIZE*sizeof(Uint));
 }
 
-static Eterm driver_monitor_to_ref(Eterm *hp, const ErlDrvMonitor *mon)
+Eterm erts_driver_monitor_to_ref(Eterm *hp, const ErlDrvMonitor *mon)
 {
     Eterm ref;
     ERTS_CT_ASSERT(ERTS_REF_THING_SIZE*sizeof(Uint) <= sizeof(ErlDrvMonitor));
@@ -7685,7 +7685,7 @@ static int do_driver_demonitor_process(Port *prt, const ErlDrvMonitor *monitor)
     ErtsMonitor *mon;
     Eterm to;
 
-    ref = driver_monitor_to_ref(heap, monitor);
+    ref = erts_driver_monitor_to_ref(heap, monitor);
 
     mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref);
     if (mon == NULL) {
@@ -7742,7 +7742,7 @@ static ErlDrvTermData do_driver_get_monitored_process(Port *prt,const ErlDrvMoni
     Eterm to;
     Eterm heap[ERTS_REF_THING_SIZE];
 
-    ref = driver_monitor_to_ref(heap, monitor);
+    ref = erts_driver_monitor_to_ref(heap, monitor);
 
     mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref);
     if (mon == NULL) {
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 69c5f09bd5..8fe5ee809a 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -2708,7 +2708,7 @@ static ERL_NIF_TERM monitor_frenzy_nif(ErlNifEnv* env, int argc, const ERL_NIF_T
         else {
             unsigned int resource_op = rand_bits(&rnd, 3);
             r = resv[rix].obj;
-            if (resource_op == 0) {
+            if (resource_op == 0) {      /* delete resource */
                 resv[rix].obj = NULL;
                 resv[rix].release_cnt++;
                 enif_mutex_unlock(resv[rix].lock);
@@ -2717,7 +2717,7 @@ static ERL_NIF_TERM monitor_frenzy_nif(ErlNifEnv* env, int argc, const ERL_NIF_T
                 retval = atom_ok;
                 break;
             }
-            else if (resource_op == 1) {
+            else if (resource_op == 1) {  /* return resource */
                 retval = enif_make_resource(env, r);
                 enif_mutex_unlock(resv[rix].lock);
                 break;
-- 
cgit v1.2.3


From 3671dd2d426208cf0639530b294da083b5e75acd Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 21 Feb 2017 09:52:51 +0100
Subject: erts: Emasculate binaries that are scheduled in port_control

---
 erts/emulator/beam/io.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 31396d06c6..9bf29ee230 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -4884,10 +4884,16 @@ erts_port_control(Process* c_p,
 	ASSERT(!tmp_alloced);
 	if (*ebinp == HEADER_SUB_BIN)
 	    ebinp = binary_val(((ErlSubBin *) ebinp)->orig);
+
 	if (*ebinp != HEADER_PROC_BIN)
 	    copy = 1;
 	else {
-	    binp = ((ProcBin *) ebinp)->val;
+            ProcBin *pb = (ProcBin *) ebinp;
+
+            if (pb->flags)
+                erts_emasculate_writable_binary(pb);
+
+	    binp = pb->val;
 	    ASSERT(bufp <= bufp + size);
 	    ASSERT(binp->orig_bytes <= bufp
 		   && bufp + size <= binp->orig_bytes + binp->orig_size);
-- 
cgit v1.2.3


From faaf8ffe2902de80c91fbed1e74b062b94edd792 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 21 Feb 2017 15:03:30 +0100
Subject: erts: Fix literal size bug when only old instance exists

fix for already merged but not releases 808b2f4d53e446aed07f85716c5c4b85abb3d18a
---
 erts/emulator/beam/break.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 5d34c83c1a..71934e1376 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -381,10 +381,12 @@ info(fmtfn_t to, void *to_arg)
 
 static int code_size(struct erl_module_instance* modi)
 {
-    ErtsLiteralArea* lit = modi->code_hdr->literal_area;
     int size = modi->code_length;
-    if (lit) {
-        size += (lit->end - lit->start) * sizeof(Eterm);
+
+    if (modi->code_hdr) {
+        ErtsLiteralArea* lit = modi->code_hdr->literal_area;
+        if (lit)
+            size += (lit->end - lit->start) * sizeof(Eterm);
     }
     return size;
 }
-- 
cgit v1.2.3


From d4e2691d35fc7e1964e58be524d826752001f2a5 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 21 Feb 2017 15:16:59 +0100
Subject: erts: Beautify loaded()

by removing some unnecessary conditions and
remove unused and faulty summation for 'cur' and 'old'.
---
 erts/emulator/beam/break.c | 30 ++++++++++--------------------
 1 file changed, 10 insertions(+), 20 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 71934e1376..2efdd1ab89 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -408,13 +408,9 @@ loaded(fmtfn_t to, void *to_arg)
      * Calculate and print totals.
      */
     for (i = 0; i < module_code_size(code_ix); i++) {
-	if ((modp = module_code(i, code_ix)) != NULL &&
-	    ((modp->curr.code_length != 0) ||
-	     (modp->old.code_length != 0))) {
+	if ((modp = module_code(i, code_ix)) != NULL) {
 	    cur += code_size(&modp->curr);
-	    if (modp->old.code_length != 0) {
-		old += code_size(&modp->old);
-	    }
+            old += code_size(&modp->old);
 	}
     }
     erts_print(to, to_arg, "Current code: %d\n", cur);
@@ -430,26 +426,20 @@ loaded(fmtfn_t to, void *to_arg)
 	    /*
 	     * Interactive dump; keep it brief.
 	     */
-	    if (modp != NULL &&
-	    ((modp->curr.code_length != 0) ||
-	     (modp->old.code_length != 0))) {
-		erts_print(to, to_arg, "%T", make_atom(modp->module));
-		cur += code_size(&modp->curr);
-		erts_print(to, to_arg, " %d", code_size(&modp->curr));
-		if (modp->old.code_length != 0) {
-		    erts_print(to, to_arg, " (%d old)",
-			       code_size(&modp->old));
-		    old += code_size(&modp->old);
-		}
+	    if (modp != NULL && ((modp->curr.code_length != 0) ||
+                                 (modp->old.code_length != 0))) {
+		erts_print(to, to_arg, "%T %d", make_atom(modp->module),
+                           code_size(&modp->curr));
+		if (modp->old.code_length != 0)
+		    erts_print(to, to_arg, " (%d old)", code_size(&modp->old));
 		erts_print(to, to_arg, "\n");
 	    }
 	} else {
 	    /*
 	     * To crash dump; make it parseable.
 	     */
-	    if (modp != NULL &&
-		((modp->curr.code_length != 0) ||
-		 (modp->old.code_length != 0))) {
+	    if (modp != NULL && ((modp->curr.code_length != 0) ||
+                                 (modp->old.code_length != 0))) {
 		erts_print(to, to_arg, "=mod:");
 		erts_print(to, to_arg, "%T", make_atom(modp->module));
 		erts_print(to, to_arg, "\n");
-- 
cgit v1.2.3


From 058800faf6b8fa4842aa0f8c3683cab26d399f60 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 22 Feb 2017 15:57:33 +0100
Subject: Fix nif_SUITE:select for old linux

with pipe capacity equal to PIPE_BUF
meaning pipe must be empty to be writable.
---
 erts/emulator/test/nif_SUITE.erl | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index c1d9632fb5..693db42e58 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -500,13 +500,8 @@ select(Config) when is_list(Config) ->
     Written = write_full(W, $a),
     0 = select_nif(W,?ERL_NIF_SELECT_WRITE,W,self(),Ref),
     [] = flush(),
-    Half = byte_size(Written) div 2,
-    <<First:Half/binary,Second/binary>> = Written,
-    First = read_nif(R,Half),
+    Written = read_nif(R,byte_size(Written)),
     [{select, W, Ref, ready_output}] = flush(),
-    Third = write_full(W, $A),
-    Half2 = byte_size(Second),
-    <<Second:Half2/binary, Third/binary>> = read_nif(R, byte_size(Written)),
 
     %% Close write and wait for EOF
     eagain = read_nif(R, 1),
-- 
cgit v1.2.3


From d8c0f76c800051629cd9cf7279bf075435708a1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 22 Feb 2017 16:52:29 +0100
Subject: erts: Introduce erts_internal:maps_to_list/2

---
 erts/emulator/beam/bif.tab   |  1 +
 erts/emulator/beam/erl_map.c | 66 +++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 60 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index a6510ace81..ce2cffa498 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -671,3 +671,4 @@ bif math:floor/1
 bif math:ceil/1
 bif math:fmod/2
 bif os:set_signal/2
+bif erts_internal:maps_to_list/2
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 9062e44b10..a80f5d6a16 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -91,7 +91,7 @@ static BIF_RETTYPE hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_
 static Export hashmap_merge_trap_export;
 static BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1);
 static Uint hashmap_subtree_size(Eterm node);
-static Eterm hashmap_to_list(Process *p, Eterm map);
+static Eterm hashmap_to_list(Process *p, Eterm map, Sint n);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
 static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node, Eterm *value);
@@ -161,13 +161,58 @@ BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) {
 
 	BIF_RET(res);
     } else if (is_hashmap(BIF_ARG_1)) {
-	return hashmap_to_list(BIF_P, BIF_ARG_1);
+	return hashmap_to_list(BIF_P, BIF_ARG_1, -1);
     }
 
     BIF_P->fvalue = BIF_ARG_1;
     BIF_ERROR(BIF_P, BADMAP);
 }
 
+/* erts_internal:maps_to_list/2
+ *
+ * This function should be removed once iterators are in place.
+ * Never document it.
+ * Never encourage its usage.
+ *
+ * A negative value in ARG 2 means the entire map.
+ */
+
+BIF_RETTYPE erts_internal_maps_to_list_2(BIF_ALIST_2) {
+    Sint m;
+    if (term_to_Sint(BIF_ARG_2, &m)) {
+        if (is_flatmap(BIF_ARG_1)) {
+            Uint n;
+            Eterm* hp;
+            Eterm *ks,*vs, res, tup;
+            flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1);
+
+            ks  = flatmap_get_keys(mp);
+            vs  = flatmap_get_values(mp);
+            n   = flatmap_get_size(mp);
+
+            if (m >= 0) {
+                n = m < n ? m : n;
+            }
+
+            hp  = HAlloc(BIF_P, (2 + 3) * n);
+            res = NIL;
+
+            while(n--) {
+                tup = TUPLE2(hp, ks[n], vs[n]); hp += 3;
+                res = CONS(hp, tup, res); hp += 2;
+            }
+
+            BIF_RET(res);
+        } else if (is_hashmap(BIF_ARG_1)) {
+            return hashmap_to_list(BIF_P, BIF_ARG_1, m);
+        }
+        BIF_P->fvalue = BIF_ARG_1;
+        BIF_ERROR(BIF_P, BADMAP);
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+
 /* maps:find/2
  * return value if key *matches* a key in the map
  */
@@ -1917,15 +1962,22 @@ BIF_RETTYPE maps_values_1(BIF_ALIST_1) {
     BIF_ERROR(BIF_P, BADMAP);
 }
 
-static Eterm hashmap_to_list(Process *p, Eterm node) {
+static Eterm hashmap_to_list(Process *p, Eterm node, Sint m) {
     DECLARE_WSTACK(stack);
     Eterm *hp, *kv;
-    Eterm res = NIL;
+    Eterm tup, res = NIL;
+    Uint n = hashmap_size(node);
 
-    hp  = HAlloc(p, hashmap_size(node) * (2 + 3));
+    if (m >= 0) {
+        n = m < n ? m : n;
+    }
+
+    hp  = HAlloc(p, n * (2 + 3));
     hashmap_iterator_init(&stack, node, 0);
-    while ((kv=hashmap_iterator_next(&stack)) != NULL) {
-	Eterm tup = TUPLE2(hp, CAR(kv), CDR(kv));
+    while (n--) {
+        kv  = hashmap_iterator_next(&stack);
+        ASSERT(kv != NULL);
+	tup = TUPLE2(hp, CAR(kv), CDR(kv));
 	hp += 3;
 	res = CONS(hp, tup, res);
 	hp += 2;
-- 
cgit v1.2.3


From ec3d7602ca96819f23ab9bc59e370c6defc9cd8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Thu, 23 Feb 2017 11:50:05 +0100
Subject: erts: Test erts_internal:maps_to_list/2

---
 erts/emulator/test/map_SUITE.erl | 66 ++++++++++++++++++++++++++++++----------
 1 file changed, 50 insertions(+), 16 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl
index dfa1629112..02f3c89318 100644
--- a/erts/emulator/test/map_SUITE.erl
+++ b/erts/emulator/test/map_SUITE.erl
@@ -52,6 +52,7 @@
          t_bif_map_values/1,
          t_bif_map_to_list/1,
          t_bif_map_from_list/1,
+         t_bif_erts_internal_maps_to_list/1,
 
          %% erlang
          t_erlang_hash/1,
@@ -118,6 +119,7 @@ all() -> [t_build_and_match_literals, t_build_and_match_literals_large,
           t_bif_map_update,
           t_bif_map_values,
           t_bif_map_to_list, t_bif_map_from_list,
+          t_bif_erts_internal_maps_to_list,
 
           %% erlang
           t_erlang_hash, t_map_encode_decode,
@@ -2362,23 +2364,55 @@ t_bif_map_from_list(Config) when is_list(Config) ->
     {'EXIT', {badarg,_}} = (catch maps:from_list(id(42))),
     ok.
 
-t_bif_build_and_check(Config) when is_list(Config) ->
-    ok = check_build_and_remove(750,[
-				      fun(K) -> [K,K] end,
-				      fun(K) -> [float(K),K] end,
-				      fun(K) -> K end,
-				      fun(K) -> {1,K} end,
-				      fun(K) -> {K} end,
-				      fun(K) -> [K|K] end,
-				      fun(K) -> [K,1,2,3,4] end,
-				      fun(K) -> {K,atom} end,
-				      fun(K) -> float(K) end,
-				      fun(K) -> integer_to_list(K) end,
-				      fun(K) -> list_to_atom(integer_to_list(K)) end,
-				      fun(K) -> [K,{K,[K,{K,[K]}]}] end,
-				      fun(K) -> <<K:32>> end
-			      ]),
+t_bif_erts_internal_maps_to_list(Config) when is_list(Config) ->
+    %% small maps
+    [] = erts_internal:maps_to_list(#{},-1),
+    [] = erts_internal:maps_to_list(#{},-2),
+    [] = erts_internal:maps_to_list(#{},10),
+    [{a,1},{b,2}] = lists:sort(erts_internal:maps_to_list(#{a=>1,b=>2}, 2)),
+    [{a,1},{b,2}] = lists:sort(erts_internal:maps_to_list(#{a=>1,b=>2}, -1)),
+    [{_,_}] = erts_internal:maps_to_list(#{a=>1,b=>2}, 1),
+    [{a,1},{b,2},{c,3}] = lists:sort(erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},-2)),
+    [{a,1},{b,2},{c,3}] = lists:sort(erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},3)),
+    [{a,1},{b,2},{c,3}] = lists:sort(erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},5)),
+    [{_,_},{_,_}] = erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},2),
+    [{_,_}] = erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},1),
+    [] = erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},0),
+
+    %% big maps
+    M = maps:from_list([{I,ok}||I <- lists:seq(1,500)]),
+    [] = erts_internal:maps_to_list(M,0),
+    [{_,_}] = erts_internal:maps_to_list(M,1),
+    [{_,_},{_,_}] = erts_internal:maps_to_list(M,2),
+    Ls1 = erts_internal:maps_to_list(M,10),
+    10 = length(Ls1),
+    Ls2 = erts_internal:maps_to_list(M,20),
+    20 = length(Ls2),
+    Ls3 = erts_internal:maps_to_list(M,120),
+    120 = length(Ls3),
+    Ls4 = erts_internal:maps_to_list(M,-1),
+    500 = length(Ls4),
 
+    %% error cases
+    {'EXIT', {{badmap,[{a,b},b]},_}} = (catch erts_internal:maps_to_list(id([{a,b},b]),id(1))),
+    {'EXIT', {badarg,_}} = (catch erts_internal:maps_to_list(id(#{}),id(a))),
+    {'EXIT', {badarg,_}} = (catch erts_internal:maps_to_list(id(#{1=>2}),id(<<>>))),
+    ok.
+
+t_bif_build_and_check(Config) when is_list(Config) ->
+    ok = check_build_and_remove(750,[fun(K) -> [K,K] end,
+				     fun(K) -> [float(K),K] end,
+				     fun(K) -> K end,
+				     fun(K) -> {1,K} end,
+				     fun(K) -> {K} end,
+				     fun(K) -> [K|K] end,
+				     fun(K) -> [K,1,2,3,4] end,
+				     fun(K) -> {K,atom} end,
+				     fun(K) -> float(K) end,
+				     fun(K) -> integer_to_list(K) end,
+				     fun(K) -> list_to_atom(integer_to_list(K)) end,
+				     fun(K) -> [K,{K,[K,{K,[K]}]}] end,
+				     fun(K) -> <<K:32>> end]),
     ok.
 
 check_build_and_remove(_,[]) -> ok;
-- 
cgit v1.2.3


From 0e566efe903fd2482c51762f1939bc292dca43e7 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 23 Feb 2017 19:26:07 +0100
Subject: Remove debug printout and comment

---
 erts/emulator/beam/break.c              | 1 -
 erts/emulator/sys/common/erl_check_io.c | 1 -
 2 files changed, 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 787f1e4b7e..6780c28580 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -196,7 +196,6 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext)
 	break;
     case MON_NIF_TARGET: {
         ErtsResource* rsrc = mon->u.resource;
-	/*SVERK: Print resource-ref? */
         erts_print(to, to_arg, "%s{from,{%T,%T},%T}", prefix, rsrc->type->module,
                    rsrc->type->name, mon->ref);
 	break;
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 8191b6e53f..1ef9fa2a05 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -2059,7 +2059,6 @@ send_event_tuple(struct erts_nif_select_event* e, ErtsResource* resource,
     Eterm resource_term, ref_term, tuple;
 
     if (!rp) {
-        erts_fprintf(stderr, "SVERK: Process %T not alive for msg %T\n", e->pid, event_atom);
         return;
     }
 
-- 
cgit v1.2.3


From cfe4b5c4aaa0d86f9d98095515ae6582a053cc47 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 27 Feb 2017 10:11:47 +0100
Subject: erts: Fix emasculation of binaries in port_control

This fixes a bug introduced in 3671dd2d
---
 erts/emulator/beam/io.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 9bf29ee230..38e403d91a 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -4889,10 +4889,18 @@ erts_port_control(Process* c_p,
 	    copy = 1;
 	else {
             ProcBin *pb = (ProcBin *) ebinp;
+            int offset = bufp - pb->val->orig_bytes;
 
-            if (pb->flags)
+	    ASSERT(pb->val->orig_bytes <= bufp
+		   && bufp + size <= pb->val->orig_bytes + pb->val->orig_size);
+
+            if (pb->flags) {
                 erts_emasculate_writable_binary(pb);
 
+                /* The procbin may have been reallocated, so update bufp */
+                bufp = pb->val->orig_bytes + offset;
+            }
+
 	    binp = pb->val;
 	    ASSERT(bufp <= bufp + size);
 	    ASSERT(binp->orig_bytes <= bufp
-- 
cgit v1.2.3


From c0f82eef818a71a8cb475d9b27a3c30074fe3b77 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 27 Feb 2017 12:01:50 +0100
Subject: erts: Fix proc arity value when exiting

Not setting it correctly will cause the swapin later
to fail in debug build as REDS_IN will overwrite
the value in reg[5].
---
 erts/emulator/beam/beam_emu.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 49f932a1c2..8be0f58227 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1324,6 +1324,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
     goto do_schedule1;
 
  do_schedule:
+    ASSERT(c_p->arity < 6);
     ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
     if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
 	reds_used = REDS_IN(c_p) - FCALLS;
@@ -1924,6 +1925,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	     SWAPOUT;
 	     c_p->flags &= ~F_DELAY_GC;
+             c_p->arity = 0;
 	     goto do_schedule; /* Will be rescheduled for exit */
 	 }
 	 ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
@@ -2166,6 +2168,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 		  * in limbo forever.
 		  */
 		 SWAPOUT;
+                 c_p->arity = 0;
 		 goto do_schedule;
 	     }
 #endif
-- 
cgit v1.2.3


From 4bbbf4ff9b3fe4be24bf8d57780c84d5b3ca0f77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 27 Feb 2017 15:27:05 +0100
Subject: erts: Convert small sub-binaries to heap-binaries

In many cases sub-binaries costs more memory than converting them to heap-binaries.

Sub-binaries also has a hidden cost of pinning larger binaries in memory.
By converting binaries this cost is reduced.

Byte aligned sub-binaries upto 24 bytes (64-bit) or 12 bytes (32-bit) are converted.
---
 erts/emulator/beam/erl_binary.h | 18 +----------
 erts/emulator/beam/erl_bits.h   | 17 ++++++++++
 erts/emulator/beam/erl_gc.c     | 33 +++++++++++++++++++
 erts/emulator/beam/erl_gc.h     | 70 ++++++++++++++++++++++++++++-------------
 4 files changed, 100 insertions(+), 38 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 946d3cfe03..6ff71ec6d1 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -156,6 +156,7 @@ typedef union {
 #include "erl_threads.h"
 #include "bif.h"
 #include "erl_bif_unique.h"
+#include "erl_bits.h"
 
 /*
  * Maximum number of bytes to place in a heap binary.
@@ -163,23 +164,6 @@ typedef union {
 
 #define ERL_ONHEAP_BIN_LIMIT 64
 
-/*
- * This structure represents a SUB_BINARY.
- *
- * Note: The last field (orig) is not counted in arityval in the header.
- * This simplifies garbage collection.
- */
-
-typedef struct erl_sub_bin {
-    Eterm thing_word;		/* Subtag SUB_BINARY_SUBTAG. */
-    Uint size;			/* Binary size in bytes. */
-    Uint offs;			/* Offset into original binary. */
-    byte bitsize; 
-    byte bitoffs; 
-    byte is_writable;		/* The underlying binary is writable */
-    Eterm orig;			/* Original binary (REFC or HEAP binary). */
-} ErlSubBin;
-
 #define ERL_SUB_BIN_SIZE (sizeof(ErlSubBin)/sizeof(Eterm))
 #define HEADER_SUB_BIN	_make_header(ERL_SUB_BIN_SIZE-2,_TAG_HEADER_SUB_BIN)
 
diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h
index 4bd5b24157..1b4546722f 100644
--- a/erts/emulator/beam/erl_bits.h
+++ b/erts/emulator/beam/erl_bits.h
@@ -21,6 +21,23 @@
 #ifndef __ERL_BITS_H__
 #define __ERL_BITS_H__
 
+/*
+ * This structure represents a SUB_BINARY.
+ *
+ * Note: The last field (orig) is not counted in arityval in the header.
+ * This simplifies garbage collection.
+ */
+
+typedef struct erl_sub_bin {
+    Eterm thing_word;		/* Subtag SUB_BINARY_SUBTAG. */
+    Uint size;			/* Binary size in bytes. */
+    Uint offs;			/* Offset into original binary. */
+    byte bitsize;
+    byte bitoffs;
+    byte is_writable;		/* The underlying binary is writable */
+    Eterm orig;			/* Original binary (REFC or HEAP binary). */
+} ErlSubBin;
+
 /*
  * This structure represents a binary to be matched.
  */
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 8cc7fc0142..26f6b6339d 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -3256,6 +3256,39 @@ reply_gc_info(void *vgcirp)
 	gcireq_free(vgcirp);
 }
 
+void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig) {
+    Eterm *ptr = *pp;
+    Eterm *htop = *hpp;
+    Eterm gval;
+    ErlSubBin *sb = (ErlSubBin *)ptr;
+    ErlHeapBin *hb = (ErlHeapBin *)htop;
+    Eterm *real_bin;
+    byte *bs;
+
+    real_bin = binary_val(follow_moved(sb->orig, (Eterm)0));
+
+    if (*real_bin == HEADER_PROC_BIN) {
+        bs = ((ProcBin *) real_bin)->bytes + sb->offs;
+    } else {
+        bs = (byte *)(&(((ErlHeapBin *) real_bin)->data)) + sb->offs;
+    }
+
+    hb->thing_word = header_heap_bin(sb->size);
+    hb->size = sb->size;
+    sys_memcpy((byte *)hb->data, bs, sb->size);
+
+    gval  = make_boxed(htop);
+    *orig = gval;
+    *ptr  = gval;
+
+    ptr  += ERL_SUB_BIN_SIZE;
+    htop += heap_bin_size(sb->size);
+
+    *hpp = htop;
+    *pp  = ptr;
+}
+
+
 Eterm
 erts_gc_info_request(Process *c_p)
 {
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 1cce426d21..2e2be2ee47 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -28,6 +28,8 @@
 #define ERTS_POTENTIALLY_LONG_GC_HSIZE (128*1024) /* Words */
 
 #include "erl_map.h"
+#include "erl_fun.h"
+#include "erl_bits.h"
 
 #define IS_MOVED_BOXED(x)	(!is_header((x)))
 #define IS_MOVED_CONS(x)	(is_non_value((x)))
@@ -45,27 +47,53 @@ do {									\
     HTOP += 2;			/* update tospace htop */		\
 } while(0)
 
-#define MOVE_BOXED(PTR,HDR,HTOP,ORIG)                                   \
-do {                                                                    \
-    Eterm gval;                                                         \
-    Sint nelts;                                                         \
-                                                                        \
-    ASSERT(is_header(HDR));                                             \
-    nelts = header_arity(HDR);                                          \
-    switch ((HDR) & _HEADER_SUBTAG_MASK) {                              \
-    case SUB_BINARY_SUBTAG: nelts++; break;                             \
-    case MAP_SUBTAG:                                                    \
-        if (is_flatmap_header(HDR)) nelts+=flatmap_get_size(PTR) + 1;   \
-        else nelts += hashmap_bitcount(MAP_HEADER_VAL(HDR));            \
-    break;                                                              \
-    case FUN_SUBTAG: nelts+=((ErlFunThing*)(PTR))->num_free+1; break;   \
-    }                                                                   \
-    gval    = make_boxed(HTOP);                                         \
-    *ORIG   = gval;                                                     \
-    *HTOP++ = HDR;                                                      \
-    *PTR++  = gval;                                                     \
-    while (nelts--) *HTOP++ = *PTR++;                                   \
-} while(0)
+#define MOVE_BOXED(PTR,HDR,HTOP,ORIG) \
+    do { \
+        move_boxed(&(PTR), HDR, &(HTOP), ORIG); \
+    } while(0)
+
+void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig);
+ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig)
+{
+    Eterm gval;
+    Sint nelts;
+    Eterm *ptr = *pp;
+    Eterm *htop = *hpp;
+
+    ASSERT(is_header(hdr));
+    nelts = header_arity(hdr);
+    switch ((hdr) & _HEADER_SUBTAG_MASK) {
+    case SUB_BINARY_SUBTAG:
+        {
+            ErlSubBin *sb = (ErlSubBin *)ptr;
+            /* convert sub-binary to heap-binary if applicable */
+            if (sb->bitsize == 0 && sb->bitoffs == 0 &&
+                sb->is_writable == 0 && sb->size <= sizeof(Eterm) * 3) {
+                erts_sub_binary_to_heap_binary(pp, hpp, orig);
+                return;
+            }
+        }
+        nelts++;
+        break;
+    case MAP_SUBTAG:
+        if (is_flatmap_header(hdr)) nelts+=flatmap_get_size(ptr) + 1;
+        else nelts += hashmap_bitcount(MAP_HEADER_VAL(hdr));
+    break;
+    case FUN_SUBTAG: nelts+=((ErlFunThing*)(ptr))->num_free+1; break;
+    }
+    gval    = make_boxed(htop);
+    *orig   = gval;
+    *htop++ = hdr;
+    *ptr++  = gval;
+    while (nelts--) *htop++ = *ptr++;
+
+    *hpp = htop;
+    *pp  = ptr;
+}
+#endif
 
 #define ErtsInYoungGen(TPtr, Ptr, OldHeap, OldHeapSz)			\
     (!erts_is_literal((TPtr), (Ptr))					\
-- 
cgit v1.2.3


From 6b51f402e4e656776565233021edcea8f4d1590b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 28 Feb 2017 15:22:35 +0100
Subject: erts: Refactor MOVE_BOXED to inline function

---
 erts/emulator/beam/copy.c    |  2 +-
 erts/emulator/beam/erl_gc.c  | 30 +++++++++++++++---------------
 erts/emulator/beam/erl_gc.h  |  5 -----
 erts/emulator/hipe/hipe_gc.c |  8 ++++----
 4 files changed, 20 insertions(+), 25 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 85db23393c..a8e37c0173 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -1990,7 +1990,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite
 	if (is_header(val)) {
 	    struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp;
 	    ASSERT(ptr + header_arity(val) < end);
-	    MOVE_BOXED(ptr, val, hp, &dummy_ref);
+	    move_boxed(&ptr, val, &hp, &dummy_ref);
 	    switch (val & _HEADER_SUBTAG_MASK) {
 	    case REF_SUBTAG:
 		if (is_ordinary_ref_thing(hdr))
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 26f6b6339d..a8c8bf3db5 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1173,7 +1173,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
 		    ASSERT(is_boxed(val));
                     *g_ptr++ = val;
 		} else if (ErtsInArea(ptr, area, area_size)) {
-                    MOVE_BOXED(ptr,val,old_htop,g_ptr++);
+                    move_boxed(&ptr,val,&old_htop,g_ptr++);
 		} else {
 		    g_ptr++;
 		}
@@ -1479,9 +1479,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
 		    ASSERT(is_boxed(val));
                     *g_ptr++ = val;
                 } else if (ErtsInArea(ptr, mature, mature_size)) {
-                    MOVE_BOXED(ptr,val,old_htop,g_ptr++);
+                    move_boxed(&ptr,val,&old_htop,g_ptr++);
                 } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
-                    MOVE_BOXED(ptr,val,n_htop,g_ptr++);
+                    move_boxed(&ptr,val,&n_htop,g_ptr++);
                 } else {
 		    g_ptr++;
 		}
@@ -1538,9 +1538,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
 		    ASSERT(is_boxed(val));
 		    *n_hp++ = val;
 		} else if (ErtsInArea(ptr, mature, mature_size)) {
-		    MOVE_BOXED(ptr,val,old_htop,n_hp++);
+		    move_boxed(&ptr,val,&old_htop,n_hp++);
 		} else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
-		    MOVE_BOXED(ptr,val,n_htop,n_hp++);
+		    move_boxed(&ptr,val,&n_htop,n_hp++);
 		} else {
 		    n_hp++;
 		}
@@ -1574,10 +1574,10 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
 			    *origptr = val;
 			    mb->base = binary_bytes(val);
 			} else if (ErtsInArea(ptr, mature, mature_size)) {
-			    MOVE_BOXED(ptr,val,old_htop,origptr);
+			    move_boxed(&ptr,val,&old_htop,origptr);
 			    mb->base = binary_bytes(mb->orig);
 			} else if (ErtsInYoungGen(*origptr, ptr, oh, oh_size)) {
-			    MOVE_BOXED(ptr,val,n_htop,origptr);
+			    move_boxed(&ptr,val,&n_htop,origptr);
 			    mb->base = binary_bytes(mb->orig);
 			}
 		    }
@@ -1792,7 +1792,7 @@ full_sweep_heaps(Process *p,
 	    Eterm* ptr;
 	    Eterm val;
 	    Eterm gval = *g_ptr;
-	    
+
 	    switch (primary_tag(gval)) {
 
 	    case TAG_PRIMARY_BOXED: {
@@ -1802,7 +1802,7 @@ full_sweep_heaps(Process *p,
 		    ASSERT(is_boxed(val));
 		    *g_ptr++ = val;
 		} else if (!erts_is_literal(gval, ptr)) {
-		    MOVE_BOXED(ptr,val,n_htop,g_ptr++);
+		    move_boxed(&ptr,val,&n_htop,g_ptr++);
 		} else {
 		    g_ptr++;
 		}
@@ -2104,7 +2104,7 @@ sweep(Eterm *n_hp, Eterm *n_htop,
 		ASSERT(is_boxed(val));
 		*n_hp++ = val;
 	    } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) {
-		MOVE_BOXED(ptr,val,n_htop,n_hp++);
+		move_boxed(&ptr,val,&n_htop,n_hp++);
 	    } else {
 		n_hp++;
 	    }
@@ -2129,7 +2129,7 @@ sweep(Eterm *n_hp, Eterm *n_htop,
 		if (header_is_bin_matchstate(gval)) {
 		    ErlBinMatchState *ms = (ErlBinMatchState*) n_hp;
 		    ErlBinMatchBuffer *mb = &(ms->mb);
-		    Eterm* origptr;	
+		    Eterm* origptr;
 		    origptr = &(mb->orig);
 		    ptr = boxed_val(*origptr);
 		    val = *ptr;
@@ -2137,7 +2137,7 @@ sweep(Eterm *n_hp, Eterm *n_htop,
 			*origptr = val;
 			mb->base = binary_bytes(*origptr);
 		    } else if (ERTS_IS_IN_SWEEP_AREA(*origptr, ptr)) {
-			MOVE_BOXED(ptr,val,n_htop,origptr); 
+			move_boxed(&ptr,val,&n_htop,origptr);
 			mb->base = binary_bytes(*origptr);
 		    }
 		}
@@ -2200,7 +2200,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
 		ASSERT(is_boxed(val));
 		*heap_ptr++ = val;
 	    } else if (ErtsInArea(ptr, src, src_size)) {
-		MOVE_BOXED(ptr,val,htop,heap_ptr++);
+		move_boxed(&ptr,val,&htop,heap_ptr++);
 	    } else {
 		heap_ptr++;
 	    }
@@ -2233,7 +2233,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
 			*origptr = val;
 			mb->base = binary_bytes(*origptr);
 		    } else if (ErtsInArea(ptr, src, src_size)) {
-			MOVE_BOXED(ptr,val,htop,origptr);
+			move_boxed(&ptr,val,&htop,origptr);
 			mb->base = binary_bytes(*origptr);
 		    }
 		}
@@ -2266,7 +2266,7 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size)
 	ASSERT(val != ERTS_HOLE_MARKER);
 	if (is_header(val)) {
 	    ASSERT(ptr + header_arity(val) < end);
-	    MOVE_BOXED(ptr, val, n_htop, &dummy_ref);	    
+	    move_boxed(&ptr, val, &n_htop, &dummy_ref);
 	}
 	else { /* must be a cons cell */
 	    ASSERT(ptr+1 < end);
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 2e2be2ee47..8e09904766 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -47,11 +47,6 @@ do {									\
     HTOP += 2;			/* update tospace htop */		\
 } while(0)
 
-#define MOVE_BOXED(PTR,HDR,HTOP,ORIG) \
-    do { \
-        move_boxed(&(PTR), HDR, &(HTOP), ORIG); \
-    } while(0)
-
 void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig);
 ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig);
 
diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c
index cf0435adc9..77ec4ee6c0 100644
--- a/erts/emulator/hipe/hipe_gc.c
+++ b/erts/emulator/hipe/hipe_gc.c
@@ -91,7 +91,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
 			ASSERT(is_boxed(val));
 			*nsp_i = val;
 		    } else if (!erts_is_literal(gval, ptr)) {
-			MOVE_BOXED(ptr, val, n_htop, nsp_i);
+			move_boxed(&ptr, val, &n_htop, nsp_i);
 		    }
 		} else if (is_list(gval)) {
 		    Eterm *ptr = list_val(gval);
@@ -206,10 +206,10 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
 			ASSERT(is_boxed(val));
 			*nsp_i = val;
 		    } else if (ErtsInArea(ptr, mature, mature_size)) {
-			MOVE_BOXED(ptr, val, old_htop, nsp_i);
+			move_boxed(&ptr, val, &old_htop, nsp_i);
 		    } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
 			ASSERT(erts_dbg_within_proc(ptr, p, NULL));
-			MOVE_BOXED(ptr, val, n_htop, nsp_i);
+			move_boxed(&ptr, val, &n_htop, nsp_i);
 		    }
 		} else if (is_list(gval)) {
 		    Eterm *ptr = list_val(gval);
@@ -278,7 +278,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area,
 			ASSERT(is_boxed(val));
 			*nsp_i = val;
 		    } else if (ErtsInArea(ptr, area, area_size)) {
-			MOVE_BOXED(ptr, val, old_htop, nsp_i);
+			move_boxed(&ptr, val, &old_htop, nsp_i);
 		    }
 		} else if (is_list(gval)) {
 		    Eterm *ptr = list_val(gval);
-- 
cgit v1.2.3


From 3e638439e80e388aa66a2047735a47c954cd70c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 28 Feb 2017 15:32:58 +0100
Subject: erts: Refactor MOVE_CONS to inline function

---
 erts/emulator/beam/copy.c    |  2 +-
 erts/emulator/beam/erl_gc.c  | 18 +++++++++---------
 erts/emulator/beam/erl_gc.h  | 32 ++++++++++++++++++--------------
 erts/emulator/hipe/hipe_gc.c |  8 ++++----
 4 files changed, 32 insertions(+), 28 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index a8e37c0173..e567eabc82 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -2007,7 +2007,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite
 	}
 	else { /* must be a cons cell */
 	    ASSERT(ptr+1 < end);
-	    MOVE_CONS(ptr, val, hp, &dummy_ref);
+	    move_cons(&ptr, val, &hp, &dummy_ref);
 	    ptr += 2;
 	}
     }
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index a8c8bf3db5..50805d9cd9 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1184,7 +1184,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
                 if (IS_MOVED_CONS(val)) { /* Moved */
                     *g_ptr++ = ptr[1];
 		} else if (ErtsInArea(ptr, area, area_size)) {
-                    MOVE_CONS(ptr,val,old_htop,g_ptr++);
+                    move_cons(&ptr,val,&old_htop,g_ptr++);
                 } else {
 		    g_ptr++;
 		}
@@ -1494,9 +1494,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
                 if (IS_MOVED_CONS(val)) { /* Moved */
                     *g_ptr++ = ptr[1];
                 } else if (ErtsInArea(ptr, mature, mature_size)) {
-                    MOVE_CONS(ptr,val,old_htop,g_ptr++);
+                    move_cons(&ptr,val,&old_htop,g_ptr++);
                 } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
-                    MOVE_CONS(ptr,val,n_htop,g_ptr++);
+                    move_cons(&ptr,val,&n_htop,g_ptr++);
                 } else {
 		    g_ptr++;
 		}
@@ -1552,9 +1552,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
 		if (IS_MOVED_CONS(val)) {
 		    *n_hp++ = ptr[1];
 		} else if (ErtsInArea(ptr, mature, mature_size)) {
-		    MOVE_CONS(ptr,val,old_htop,n_hp++);
+		    move_cons(&ptr,val,&old_htop,n_hp++);
 		} else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
-		    MOVE_CONS(ptr,val,n_htop,n_hp++);
+		    move_cons(&ptr,val,&n_htop,n_hp++);
 		} else {
 		    n_hp++;
 		}
@@ -1815,7 +1815,7 @@ full_sweep_heaps(Process *p,
 		if (IS_MOVED_CONS(val)) {
 		    *g_ptr++ = ptr[1];
 		} else if (!erts_is_literal(gval, ptr)) {
-		    MOVE_CONS(ptr,val,n_htop,g_ptr++);
+		    move_cons(&ptr,val,&n_htop,g_ptr++);
 		} else {
 		    g_ptr++;
 		}
@@ -2116,7 +2116,7 @@ sweep(Eterm *n_hp, Eterm *n_htop,
 	    if (IS_MOVED_CONS(val)) {
 		*n_hp++ = ptr[1];
 	    } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) {
-		MOVE_CONS(ptr,val,n_htop,n_hp++);
+		move_cons(&ptr,val,&n_htop,n_hp++);
 	    } else {
 		n_hp++;
 	    }
@@ -2212,7 +2212,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
 	    if (IS_MOVED_CONS(val)) {
 		*heap_ptr++ = ptr[1];
 	    } else if (ErtsInArea(ptr, src, src_size)) {
-		MOVE_CONS(ptr,val,htop,heap_ptr++);
+		move_cons(&ptr,val,&htop,heap_ptr++);
 	    } else {
 		heap_ptr++;
 	    }
@@ -2270,7 +2270,7 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size)
 	}
 	else { /* must be a cons cell */
 	    ASSERT(ptr+1 < end);
-	    MOVE_CONS(ptr, val, n_htop, &dummy_ref);
+	    move_cons(&ptr, val, &n_htop, &dummy_ref);
 	    ptr += 2;
 	}
     }
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 8e09904766..414aff1e06 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -33,23 +33,27 @@
 
 #define IS_MOVED_BOXED(x)	(!is_header((x)))
 #define IS_MOVED_CONS(x)	(is_non_value((x)))
+void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig);
+
+ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig);
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig)
+{
+    Eterm *ptr  = *pp;
+    Eterm *htop = *hpp;
+    Eterm gval;
 
-#define MOVE_CONS(PTR,CAR,HTOP,ORIG)					\
-do {									\
-    Eterm gval;								\
-									\
-    HTOP[0] = CAR;		/* copy car */				\
-    HTOP[1] = PTR[1];		/* copy cdr */				\
-    gval = make_list(HTOP);	/* new location */			\
-    *ORIG = gval;		/* redirect original reference */	\
-    PTR[0] = THE_NON_VALUE;	/* store forwarding indicator */	\
-    PTR[1] = gval;		/* store forwarding address */		\
-    HTOP += 2;			/* update tospace htop */		\
-} while(0)
+    htop[0] = car;               /* copy car */
+    htop[1] = ptr[1];            /* copy cdr */
+    gval    = make_list(htop);   /* new location */
+    *orig   = gval;              /* redirect original reference */
+    ptr[0]  = THE_NON_VALUE;     /* store forwarding indicator */
+    ptr[1]  = gval;              /* store forwarding address */
+    *hpp   += 2;                 /* update tospace htop */
+}
+#endif
 
-void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig);
 ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig);
-
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig)
 {
diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c
index 77ec4ee6c0..2311beb34a 100644
--- a/erts/emulator/hipe/hipe_gc.c
+++ b/erts/emulator/hipe/hipe_gc.c
@@ -100,7 +100,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
 			*nsp_i = ptr[1];
 		    } else if (!erts_is_literal(gval, ptr)) {
 			ASSERT(erts_dbg_within_proc(ptr, p, NULL));
-			MOVE_CONS(ptr, val, n_htop, nsp_i);
+			move_cons(&ptr, val, &n_htop, nsp_i);
 		    }
 		}
 	    }
@@ -217,10 +217,10 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
 		    if (IS_MOVED_CONS(val)) {
 			*nsp_i = ptr[1];
 		    } else if (ErtsInArea(ptr, mature, mature_size)) {
-			MOVE_CONS(ptr, val, old_htop, nsp_i);
+			move_cons(&ptr, val, &old_htop, nsp_i);
 		    } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
 			ASSERT(erts_dbg_within_proc(ptr, p, NULL));
-			MOVE_CONS(ptr, val, n_htop, nsp_i);
+			move_cons(&ptr, val, &n_htop, nsp_i);
 		    }
 		}
 	    }
@@ -286,7 +286,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area,
 		    if (IS_MOVED_CONS(val)) {
 			*nsp_i = ptr[1];
 		    } else if (ErtsInArea(ptr, area, area_size)) {
-			MOVE_CONS(ptr, val, old_htop, nsp_i);
+			move_cons(&ptr, val, &old_htop, nsp_i);
 		    }
 		}
 	    }
-- 
cgit v1.2.3


From 0d6dc895744c34c9c52fd42f4801a8a941864ae3 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 13 Feb 2017 18:22:39 +0100
Subject: Introduce references as table identifiers

---
 erts/emulator/beam/erl_db.c      | 98 ++++++++++++++++++++++++++++++++++++++--
 erts/emulator/beam/erl_db_util.h |  2 +
 2 files changed, 96 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 9f077dd407..d886c2546a 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -43,6 +43,7 @@
 #include "erl_db.h"
 #include "bif.h"
 #include "big.h"
+#include "erl_binary.h"
 
 
 erts_smp_atomic_t erts_ets_misc_mem_size;
@@ -113,6 +114,75 @@ static struct {
 #define MARK_SLOT_DEAD(i) (meta_main_tab[(i)].u.next_free |= 2)
 #define GET_ANY_SLOT_TAB(i) ((DbTable*)(meta_main_tab[(i)].u.next_free & ~(1|2))) /* dead or alive */
 
+static void schedule_free_dbtable(DbTable* tb);
+
+static void table_dec_refc(DbTable *tb, erts_aint_t min_val)
+{
+    if (erts_smp_refc_dectest(&tb->common.refc, min_val) == 0)
+	schedule_free_dbtable(tb);
+}
+
+static int
+db_table_tid_destructor(Binary *unused)
+{
+    return 1;
+}
+
+static ERTS_INLINE void
+make_btid(DbTable *tb)
+{
+    Binary *btid = erts_create_magic_indirection(db_table_tid_destructor);
+    erts_smp_atomic_t *tbref = erts_smp_binary_to_magic_indirection(btid);
+    erts_smp_atomic_init_nob(tbref, (erts_aint_t) tb);
+    tb->common.btid = btid;
+    /*
+     * Table and magic indirection refer eachother,
+     * and table is refered once by being alive...
+     */
+    erts_smp_refc_init(&tb->common.refc, 2);
+    erts_refc_inc(&btid->refc, 1);
+}
+
+static DbTable *
+tid2tab(Eterm tid)
+{
+    DbTable *tb;
+    Binary *btid;
+    erts_smp_atomic_t *tbref;
+    if (!is_internal_magic_ref(tid))
+        return NULL;
+
+    btid = erts_magic_ref2bin(tid);
+    if (ERTS_MAGIC_BIN_DESTRUCTOR(btid) != db_table_tid_destructor)
+        return NULL;
+
+    tbref = erts_smp_binary_to_magic_indirection(btid);
+    tb = (DbTable *) erts_smp_atomic_read_nob(tbref);
+
+    ASSERT(!tb || tb->common.btid == btid);
+
+    return tb;
+}
+
+static ERTS_INLINE void
+tid_clear(DbTable *tb)
+{
+    DbTable *rtb;
+    Binary *btid = tb->common.btid;
+    erts_smp_atomic_t *tbref = erts_smp_binary_to_magic_indirection(btid);
+    rtb = (DbTable *) erts_smp_atomic_xchg_nob(tbref, (erts_aint_t) NULL);
+    ASSERT(!rtb || tb == rtb);
+    if (rtb)
+        table_dec_refc(tb, 1);
+}
+
+static ERTS_INLINE Eterm
+make_tid(Process *c_p, DbTable *tb)
+{
+    Eterm *hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE);
+    return erts_mk_magic_ref(&hp, &c_p->off_heap, tb->common.btid);
+}
+
 static ERTS_INLINE erts_smp_rwmtx_t *
 get_meta_main_tab_lock(unsigned slot)
 {
@@ -252,6 +322,10 @@ free_dbtable(void *vtb)
 	erts_smp_mtx_destroy(&tb->common.fixlock);
 #endif
 	ASSERT(is_immed(tb->common.heir_data));
+
+        if (tb->common.btid && erts_refc_dectest(&tb->common.btid->refc, 0) == 0)
+            erts_bin_free(tb->common.btid);
+
 	erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));
 }
 
@@ -266,6 +340,7 @@ static void schedule_free_dbtable(DbTable* tb)
      *  	     Caller is *not* allowed to access the specialized part
      *  	     (hash or tree) of *tb after this function has returned.
      */
+    ASSERT(erts_smp_refc_read(&tb->common.refc, 0) == 0);
     ASSERT(erts_smp_refc_read(&tb->common.ref, 0) == 0);
     erts_schedule_thr_prgr_later_cleanup_op(free_dbtable,
 					    (void *) tb,
@@ -375,7 +450,7 @@ DbTable* db_get_table_aux(Process *p,
 			  db_lock_kind_t kind,
 			  int meta_already_locked)
 {
-    DbTable *tb = NULL;
+    DbTable *tb;
     erts_smp_rwmtx_t *mtl = NULL;
 
     /*
@@ -385,7 +460,10 @@ DbTable* db_get_table_aux(Process *p,
      */
     ASSERT(erts_get_scheduler_data());
 
-    if (is_small(id)) {
+    tb = tid2tab(id);
+    if (tb)
+        id = tb->common.id;
+    else if (is_small(id)) {
 	Uint slot = unsigned_val(id) & meta_main_tab_slot_mask;
 	if (!meta_already_locked) {
 	    mtl = get_meta_main_tab_lock(slot);
@@ -1505,6 +1583,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 	meth->db_create(BIF_P, tb);
     ASSERT(cret == DB_ERROR_NONE);
 
+    make_btid(tb);
+
     erts_smp_spin_lock(&meta_main_tab_main_lock);
 
     if (meta_main_tab_cnt >= db_max_tabs) {
@@ -1539,6 +1619,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     tb->common.id = ret;
     tb->common.slot = slot;           /* store slot for erase */
 
+    if (!is_named)
+        ret = make_tid(BIF_P, tb);
+
     mmtl = get_meta_main_tab_lock(slot);
     erts_smp_rwmtx_rwlock(mmtl);
     meta_main_tab[slot].u.tb = tb;
@@ -1551,11 +1634,13 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 	free_slot(slot);
 	erts_smp_rwmtx_rwunlock(mmtl);
 
+        tid_clear(tb);
+
 	db_lock(tb,LCK_WRITE);
 	free_heir_data(tb);
 	tb->common.meth->db_free_table(tb);
-	schedule_free_dbtable(tb);
 	db_unlock(tb,LCK_WRITE);
+        table_dec_refc(tb, 0);
 	BIF_ERROR(BIF_P, BADARG);
     }
     
@@ -1740,6 +1825,8 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 	UnUseTmpHeap(3,BIF_P);
     }    
 
+    tid_clear(tb);
+
     mmtl = get_meta_main_tab_lock(tb->common.slot);
 #ifdef ERTS_SMP
     if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) {
@@ -2969,6 +3056,7 @@ void init_db(ErtsDbSpinCount db_spin_count)
 			     erts_smp_atomic_read_nob(&init_tb.common.memory_size));
 
     meta_pid_to_tab->common.id = NIL;
+    meta_pid_to_tab->common.btid = NULL;
     meta_pid_to_tab->common.the_name = am_true;
     meta_pid_to_tab->common.status = (DB_NORMAL | DB_BAG | DB_PUBLIC | DB_FINE_LOCKED);
 #ifdef ERTS_SMP
@@ -3000,6 +3088,7 @@ void init_db(ErtsDbSpinCount db_spin_count)
 			     erts_smp_atomic_read_nob(&init_tb.common.memory_size));
 
     meta_pid_to_fixed_tab->common.id = NIL;
+    meta_pid_to_fixed_tab->common.btid = NULL;
     meta_pid_to_fixed_tab->common.the_name = am_true;
     meta_pid_to_fixed_tab->common.status = (DB_NORMAL | DB_BAG | DB_PUBLIC | DB_FINE_LOCKED);
 #ifdef ERTS_SMP
@@ -3295,6 +3384,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 #endif
 			first_call = (tb->common.status & DB_DELETE) == 0;
 			if (first_call) {
+                            tid_clear(tb);
 			    /* Clear all access bits. */
 			    tb->common.status &= ~(DB_PROTECTED
 						   | DB_PUBLIC
@@ -3707,7 +3797,7 @@ static int free_table_cont(Process *p,
 				make_small(tb->common.slot));
 	    db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
 	}
-	schedule_free_dbtable(tb);
+        table_dec_refc(tb, 0);
 	BUMP_REDS(p, 100);
 	return 0;
     }
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 471fefe3cb..3f507b559f 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -212,6 +212,7 @@ typedef struct db_fixation {
  */
 
 typedef struct db_table_common {
+    erts_smp_refc_t refc;     /* reference count of table struct */
     erts_smp_refc_t ref;          /* fixation counter */
 #ifdef ERTS_SMP
     erts_smp_rwmtx_t rwlock;  /* rw lock on table */
@@ -225,6 +226,7 @@ typedef struct db_table_common {
     Uint64 heir_started_interval;  /* To further identify the heir */
     Eterm the_name;           /* an atom */
     Eterm id;                 /* atom | integer */
+    Binary *btid;
     DbTableMethod* meth;      /* table methods */
     erts_smp_atomic_t nitems; /* Total number of items in table */
     erts_smp_atomic_t memory_size;/* Total memory size. NOTE: in bytes! */
-- 
cgit v1.2.3


From ee7494d5d5c3388a8bb1f012f6411a90f89d8ea6 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 13 Feb 2017 18:36:00 +0100
Subject: Rename fixation count in ets table to avoid confusion

---
 erts/emulator/beam/erl_db.c      | 20 ++++++++++----------
 erts/emulator/beam/erl_db_util.h |  4 ++--
 2 files changed, 12 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index d886c2546a..ad11d72d5f 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -341,7 +341,7 @@ static void schedule_free_dbtable(DbTable* tb)
      *  	     (hash or tree) of *tb after this function has returned.
      */
     ASSERT(erts_smp_refc_read(&tb->common.refc, 0) == 0);
-    ASSERT(erts_smp_refc_read(&tb->common.ref, 0) == 0);
+    ASSERT(erts_smp_refc_read(&tb->common.fix_count, 0) == 0);
     erts_schedule_thr_prgr_later_cleanup_op(free_dbtable,
 					    (void *) tb,
 					    &tb->release.data,
@@ -678,11 +678,11 @@ done:
 */
 static ERTS_INLINE void local_fix_table(DbTable* tb)
 {
-    erts_smp_refc_inc(&tb->common.ref, 1);
+    erts_smp_refc_inc(&tb->common.fix_count, 1);
 }	    
 static ERTS_INLINE void local_unfix_table(DbTable* tb)
 {	
-    if (erts_smp_refc_dectest(&tb->common.ref, 0) == 0) {
+    if (erts_smp_refc_dectest(&tb->common.fix_count, 0) == 0) {
 	ASSERT(IS_HASH_TABLE(tb->common.status));
 	db_unfix_table_hash(&(tb->hash));
     }
@@ -1565,7 +1565,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     tb->common.type = status & ERTS_ETS_TABLE_TYPES;
     /* Note, 'type' is *read only* from now on... */
 #endif
-    erts_smp_refc_init(&tb->common.ref, 0);
+    erts_smp_refc_init(&tb->common.fix_count, 0);
     db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ),
 		 "db_tab", "db_tab_fix");
     tb->common.keypos = keypos;
@@ -3072,7 +3072,7 @@ void init_db(ErtsDbSpinCount db_spin_count)
     meta_pid_to_tab->common.meth   = &db_hash;
     meta_pid_to_tab->common.compress = 0;
 
-    erts_smp_refc_init(&meta_pid_to_tab->common.ref, 0);
+    erts_smp_refc_init(&meta_pid_to_tab->common.fix_count, 0);
     /* Neither rwlock or fixlock used
     db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/
 
@@ -3104,7 +3104,7 @@ void init_db(ErtsDbSpinCount db_spin_count)
     meta_pid_to_fixed_tab->common.meth   = &db_hash;
     meta_pid_to_fixed_tab->common.compress = 0;
 
-    erts_smp_refc_init(&meta_pid_to_fixed_tab->common.ref, 0);
+    erts_smp_refc_init(&meta_pid_to_fixed_tab->common.fix_count, 0);
     /* Neither rwlock or fixlock used
     db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/
 
@@ -3466,7 +3466,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 			    if ((*pp)->pid == pid) {
 				DbFixation* fix = *pp;
 				erts_aint_t diff = -((erts_aint_t) fix->counter);
-				erts_smp_refc_add(&tb->common.ref,diff,0);
+				erts_smp_refc_add(&tb->common.fix_count,diff,0);
 				*pp = fix->next;
 				erts_db_free(ERTS_ALC_T_DB_FIXATION,
 					     tb, fix, sizeof(DbFixation));
@@ -3542,7 +3542,7 @@ static void fix_table_locked(Process* p, DbTable* tb)
 #ifdef ERTS_SMP
     erts_smp_mtx_lock(&tb->common.fixlock);
 #endif
-    erts_smp_refc_inc(&tb->common.ref,1);
+    erts_smp_refc_inc(&tb->common.fix_count,1);
     fix = tb->common.fixations;
     if (fix == NULL) {
 	tb->common.time.monotonic
@@ -3598,7 +3598,7 @@ static void unfix_table_locked(Process* p,  DbTable* tb,
     for (pp = &tb->common.fixations; *pp != NULL; pp = &(*pp)->next) {
 	if ((*pp)->pid == p->common.id) {
 	    DbFixation* fix = *pp;
-	    erts_smp_refc_dec(&tb->common.ref,0);
+	    erts_smp_refc_dec(&tb->common.fix_count,0);
 	    --(fix->counter);
 	    ASSERT(fix->counter >= 0);
 	    if (fix->counter > 0) {
@@ -3647,7 +3647,7 @@ static void free_fixations_locked(DbTable *tb)
     fix = tb->common.fixations;
     while (fix != NULL) {
 	erts_aint_t diff = -((erts_aint_t) fix->counter);
-	erts_smp_refc_add(&tb->common.ref,diff,0);
+	erts_smp_refc_add(&tb->common.fix_count,diff,0);
 	next_fix = fix->next;
 	db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
 	db_erase_bag_exact2(meta_pid_to_fixed_tab,
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 3f507b559f..60dedd5713 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -213,7 +213,7 @@ typedef struct db_fixation {
 
 typedef struct db_table_common {
     erts_smp_refc_t refc;     /* reference count of table struct */
-    erts_smp_refc_t ref;          /* fixation counter */
+    erts_smp_refc_t fix_count;/* fixation counter */
 #ifdef ERTS_SMP
     erts_smp_rwmtx_t rwlock;  /* rw lock on table */
     erts_smp_mtx_t fixlock;   /* Protects fixations,megasec,sec,microsec */
@@ -263,7 +263,7 @@ typedef struct db_table_common {
 				  (DB_BAG | DB_SET | DB_DUPLICATE_BAG)))
 #define IS_TREE_TABLE(Status) (!!((Status) & \
 				  DB_ORDERED_SET))
-#define NFIXED(T) (erts_smp_refc_read(&(T)->common.ref,0))
+#define NFIXED(T) (erts_smp_refc_read(&(T)->common.fix_count,0))
 #define IS_FIXED(T) (NFIXED(T) != 0) 
 
 /*
-- 
cgit v1.2.3


From d1ad45f0940697f04f334c078a2287cd51e45ad5 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 17 Feb 2017 18:23:11 +0100
Subject: Implement ets:all() using scheduler specific data

---
 erts/emulator/beam/bif.tab         |   2 +-
 erts/emulator/beam/break.c         |   2 +
 erts/emulator/beam/erl_alloc.types |   1 +
 erts/emulator/beam/erl_db.c        | 559 +++++++++++++++++++++++++++++++------
 erts/emulator/beam/erl_db.h        |  42 ++-
 erts/emulator/beam/erl_db_util.h   |   9 +-
 erts/emulator/beam/erl_init.c      |   1 +
 erts/emulator/beam/erl_process.c   |  50 ++++
 erts/emulator/beam/erl_process.h   |  16 +-
 9 files changed, 590 insertions(+), 92 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index a6510ace81..08c8c3adbd 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -324,7 +324,7 @@ bif erlang:match_spec_test/3
 # Bifs in ets module.
 #
 
-bif ets:all/0
+bif ets:internal_request_all/0
 bif ets:new/2
 bif ets:delete/1
 bif ets:delete/2
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 4dad5736c5..0b40d70cb7 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -512,6 +512,8 @@ do_break(void)
     erts_free_read_env(mode);
 #endif /* __WIN32__ */
 
+    ASSERT(erts_smp_thr_progress_is_blocking());
+
     erts_printf("\n"
 		"BREAK: (a)bort (c)ontinue (p)roc info (i)nfo (l)oaded\n"
 		"       (v)ersion (k)ill (D)b-tables (d)istribution\n");
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 3d5de72ee7..32f84c8593 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -227,6 +227,7 @@ type	DB_DMC_ERROR	ETS		ETS		db_dmc_error
 type	DB_DMC_ERR_INFO	ETS		ETS		db_dmc_error_info
 type	DB_TERM		ETS		ETS		db_term
 type	DB_PROC_CLEANUP SHORT_LIVED	ETS		db_proc_cleanup_state
+type	ETS_ALL_REQ	SHORT_LIVED	ETS		ets_all_request
 type	INSTR_INFO	LONG_LIVED	SYSTEM		instr_info
 type	LOGGER_DSBUF	TEMPORARY	SYSTEM		logger_dsbuf
 type	TMP_DSBUF	TEMPORARY	SYSTEM		tmp_dsbuf
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index ad11d72d5f..8c4c76e187 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -114,7 +114,12 @@ static struct {
 #define MARK_SLOT_DEAD(i) (meta_main_tab[(i)].u.next_free |= 2)
 #define GET_ANY_SLOT_TAB(i) ((DbTable*)(meta_main_tab[(i)].u.next_free & ~(1|2))) /* dead or alive */
 
+static void
+send_ets_transfer_message(Process *c_p, Process *proc,
+                          ErtsProcLocks *locks,
+                          DbTable *tb, Eterm heir_data);
 static void schedule_free_dbtable(DbTable* tb);
+static void delete_sched_table(Process *c_p, DbTable *tb);
 
 static void table_dec_refc(DbTable *tb, erts_aint_t min_val)
 {
@@ -164,16 +169,32 @@ tid2tab(Eterm tid)
     return tb;
 }
 
+static ERTS_INLINE int
+is_table_alive(DbTable *tb)
+{
+    erts_smp_atomic_t *tbref;
+    DbTable *rtb;
+
+    tbref = erts_smp_binary_to_magic_indirection(tb->common.btid);
+    rtb = (DbTable *) erts_smp_atomic_read_nob(tbref);
+
+    ASSERT(!rtb || rtb == tb);
+
+    return !!rtb;
+}
+
 static ERTS_INLINE void
-tid_clear(DbTable *tb)
+tid_clear(Process *c_p, DbTable *tb)
 {
     DbTable *rtb;
     Binary *btid = tb->common.btid;
     erts_smp_atomic_t *tbref = erts_smp_binary_to_magic_indirection(btid);
     rtb = (DbTable *) erts_smp_atomic_xchg_nob(tbref, (erts_aint_t) NULL);
     ASSERT(!rtb || tb == rtb);
-    if (rtb)
+    if (rtb) {
         table_dec_refc(tb, 1);
+        delete_sched_table(c_p, tb);
+    }
 }
 
 static ERTS_INLINE Eterm
@@ -348,6 +369,99 @@ static void schedule_free_dbtable(DbTable* tb)
 					    sizeof(DbTable));
 }
 
+static ERTS_INLINE void
+save_sched_table(Process *c_p, DbTable *tb)
+{
+    ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
+    DbTable *first;
+
+    ASSERT(esdp);
+    esdp->ets_tables.count++;
+    erts_smp_refc_inc(&tb->common.refc, 1);
+
+    first = esdp->ets_tables.clist;
+    if (!first) {
+        tb->common.all.next = tb->common.all.prev = tb;
+        esdp->ets_tables.clist = tb;
+    }
+    else {
+        tb->common.all.prev = first->common.all.prev;
+        tb->common.all.next = first;
+        tb->common.all.prev->common.all.next = tb;
+        first->common.all.prev = tb;
+    }
+}
+
+static ERTS_INLINE void
+remove_sched_table(ErtsSchedulerData *esdp, DbTable *tb)
+{
+    ErtsEtsAllYieldData *eaydp;
+    ASSERT(esdp);
+    ASSERT(erts_get_ref_numbers_thr_id(ERTS_MAGIC_BIN_REFN(tb->common.btid))
+           == (Uint32) esdp->no);
+
+    ASSERT(esdp->ets_tables.count > 0);
+    esdp->ets_tables.count--;
+
+    eaydp = ERTS_SCHED_AUX_YIELD_DATA(esdp, ets_all);
+    if (eaydp->ongoing) {
+        /* ets:all() op process list from last to first... */
+        if (eaydp->tab == tb) {
+            if (eaydp->tab == esdp->ets_tables.clist)
+                eaydp->tab = NULL;
+            else
+                eaydp->tab = tb->common.all.prev;
+        }
+    }
+
+    if (tb->common.all.next == tb) {
+        ASSERT(tb->common.all.prev == tb);
+        ASSERT(esdp->ets_tables.clist == tb);
+        esdp->ets_tables.clist = NULL;
+    }
+    else {
+#ifdef DEBUG
+        DbTable *tmp = esdp->ets_tables.clist;
+        do {
+            if (tmp == tb) break;
+            tmp = tmp->common.all.next;
+        } while (tmp != esdp->ets_tables.clist);
+        ASSERT(tmp == tb);
+#endif
+        tb->common.all.prev->common.all.next = tb->common.all.next;
+        tb->common.all.next->common.all.prev = tb->common.all.prev;
+
+        if (esdp->ets_tables.clist == tb)
+            esdp->ets_tables.clist = tb->common.all.next;
+
+    }
+
+    table_dec_refc(tb, 0);
+}
+
+static void
+scheduled_remove_sched_table(void *vtb)
+{
+    remove_sched_table(erts_get_scheduler_data(), (DbTable *) vtb);
+}
+
+static void
+delete_sched_table(Process *c_p, DbTable *tb)
+{
+    ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
+    Uint32 sid;
+
+    ASSERT(esdp);
+
+    ASSERT(tb->common.btid);
+    sid = erts_get_ref_numbers_thr_id(ERTS_MAGIC_BIN_REFN(tb->common.btid));
+    ASSERT(1 <= sid && sid <= erts_no_schedulers);
+    if (sid == (Uint32) esdp->no)
+        remove_sched_table(esdp, tb);
+    else
+        erts_schedule_misc_aux_work((int) sid, scheduled_remove_sched_table, tb);
+}
+
 static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock,
 				     char *rwname, char* fixname)
 {
@@ -1322,6 +1436,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
 {
     DbTable* tb;
     Eterm ret;
+    Eterm old_name;
     erts_smp_rwmtx_t *lck1, *lck2;
 
 #ifdef HARDDEBUG
@@ -1338,12 +1453,10 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
 
     (void) meta_name_tab_bucket(BIF_ARG_2, &lck1);
 
-    if (is_small(BIF_ARG_1)) {
-	Uint slot = unsigned_val(BIF_ARG_1) & meta_main_tab_slot_mask;
-	lck2 = get_meta_main_tab_lock(slot);
-    }
-    else if (is_atom(BIF_ARG_1)) {
-	(void) meta_name_tab_bucket(BIF_ARG_1, &lck2);
+    if (is_atom(BIF_ARG_1)) {
+        old_name = BIF_ARG_1;
+    named_tab:
+	(void) meta_name_tab_bucket(old_name, &lck2);
 	if (lck1 == lck2)
 	    lck2 = NULL;
 	else if (lck1 > lck2) {
@@ -1353,7 +1466,18 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
 	}
     }
     else {
-	BIF_ERROR(BIF_P, BADARG);
+        Uint slot;
+        tb = tid2tab(BIF_ARG_1);
+        if (!tb)
+            BIF_ERROR(BIF_P, BADARG);
+        else {
+            if (is_atom(tb->common.id)) {
+                old_name = tb->common.id;
+                goto named_tab;
+            }
+            slot = unsigned_val(tb->common.id) & meta_main_tab_slot_mask;
+            lck2 = get_meta_main_tab_lock(slot);
+        }
     }
 
     erts_smp_rwmtx_rwlock(lck1);
@@ -1378,7 +1502,10 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
     tb->common.id = tb->common.the_name = BIF_ARG_2;
 
  done:
-    ret = tb->common.id;
+    if (is_atom(tb->common.id))
+        ret = tb->common.id;
+    else
+        ret = BIF_ARG_1;
     db_unlock(tb, LCK_WRITE);
     erts_smp_rwmtx_rwunlock(lck1);
     if (lck2)
@@ -1622,6 +1749,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     if (!is_named)
         ret = make_tid(BIF_P, tb);
 
+    save_sched_table(BIF_P, tb);
+
     mmtl = get_meta_main_tab_lock(slot);
     erts_smp_rwmtx_rwlock(mmtl);
     meta_main_tab[slot].u.tb = tb;
@@ -1634,7 +1763,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 	free_slot(slot);
 	erts_smp_rwmtx_rwunlock(mmtl);
 
-        tid_clear(tb);
+        tid_clear(BIF_P, tb);
 
 	db_lock(tb,LCK_WRITE);
 	free_heir_data(tb);
@@ -1825,7 +1954,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 	UnUseTmpHeap(3,BIF_P);
     }    
 
-    tid_clear(tb);
+    tid_clear(BIF_P, tb);
 
     mmtl = get_meta_main_tab_lock(tb->common.slot);
 #ifdef ERTS_SMP
@@ -1913,12 +2042,8 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
     db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
 
     db_unlock(tb,LCK_WRITE);
-    erts_send_message(BIF_P, to_proc, &to_locks,
-		      TUPLE4(buf, am_ETS_TRANSFER,
-			     tb->common.id,
-			     from_pid,
-			     BIF_ARG_3), 
-                      0);
+    send_ets_transfer_message(BIF_P, to_proc, &to_locks,
+                              tb, BIF_ARG_3);
     erts_smp_proc_unlock(to_proc, to_locks);
     UnUseTmpHeap(5,BIF_P);
     BIF_RET(am_true);
@@ -2188,46 +2313,254 @@ BIF_RETTYPE ets_select_delete_2(BIF_ALIST_2)
     return result;
 }
 
-/* 
-** Return a list of tables on this node 
-*/
-BIF_RETTYPE ets_all_0(BIF_ALIST_0)
+/*
+ * ets:all/0
+ *
+ * ets:all() calls ets:internal_request_all/0 which
+ * requests information about all tables from
+ * each scheduler thread. Each scheduler replies
+ * to the calling process with information about
+ * existing tables created on that specific scheduler.
+ */
+
+struct ErtsEtsAllReq_ {
+    erts_smp_atomic32_t refc;
+    Process *proc;
+    ErtsOIRefStorage ref;
+    ErtsEtsAllReqList list[1]; /* one per scheduler */
+};
+
+#define ERTS_ETS_ALL_REQ_SIZE           \
+    (sizeof(ErtsEtsAllReq)              \
+     + (sizeof(ErtsEtsAllReqList)       \
+        * (erts_no_schedulers - 1)))
+
+typedef struct {
+    ErtsEtsAllReq *ongoing;
+    ErlHeapFragment *hfrag;
+    DbTable *tab;
+    ErtsEtsAllReq *queue;
+} ErtsEtsAllData;
+
+/* Tables handled before yielding */
+#define ERTS_ETS_ALL_TB_YCNT 200
+/*
+ * Min yield count required before starting
+ * an operation that will require yield.
+ */
+#define ERTS_ETS_ALL_TB_YCNT_START 10
+
+#ifdef DEBUG
+/* Test yielding... */
+#undef ERTS_ETS_ALL_TB_YCNT
+#undef ERTS_ETS_ALL_TB_YCNT_START
+#define ERTS_ETS_ALL_TB_YCNT 10
+#define ERTS_ETS_ALL_TB_YCNT_START 1
+#endif
+
+static int
+ets_all_reply(ErtsSchedulerData *esdp, ErtsEtsAllReq **reqpp,
+              ErlHeapFragment **hfragpp, DbTable **tablepp,
+              int *yield_count_p)
 {
-    DbTable* tb;
-    Eterm previous;
-    int i;
-    Eterm* hp;
-    Eterm* hendp;
-    int t_tabs_cnt;
-    int t_top;
+    ErtsEtsAllReq *reqp = *reqpp;
+    ErlHeapFragment *hfragp = *hfragpp;
+    int ycount = *yield_count_p;
+    DbTable *tb, *first;
+    Uint sz;
+    Eterm list, msg, ref, *hp;
+    ErlOffHeap *ohp;
+    ErtsMessage *mp;
 
-    erts_smp_spin_lock(&meta_main_tab_main_lock);
-    t_tabs_cnt = meta_main_tab_cnt;
-    t_top = meta_main_tab_top;
-    erts_smp_spin_unlock(&meta_main_tab_main_lock);
+    /*
+     * - save_sched_table() inserts at end of circular list.
+     *
+     * - This function scans from the end so we know that
+     *   the amount of tables to scan wont grow even if we
+     *   yield.
+     *
+     * - remove_sched_table() updates the table we yielded
+     *   on if it removes it.
+     */
+
+    if (hfragp) {
+        /* Restart of a yielded operation... */
+        ASSERT(hfragp->used_size < hfragp->alloc_size);
+        ohp = &hfragp->off_heap;
+        hp = &hfragp->mem[hfragp->used_size];
+        list = *hp;
+        hfragp->used_size = hfragp->alloc_size;
+        first = esdp->ets_tables.clist;
+        tb = *tablepp;
+    }
+    else {
+        /* A new operation... */
+        ASSERT(!*tablepp);
 
-    hp = HAlloc(BIF_P, 2*t_tabs_cnt);
-    hendp = hp + 2*t_tabs_cnt;
+        /* Max heap size needed... */
+        sz = esdp->ets_tables.count;
+        sz *= ERTS_MAGIC_REF_THING_SIZE + 2;
+        sz += 3 + ERTS_REF_THING_SIZE;
+        hfragp = new_message_buffer(sz);
 
-    previous = NIL;
-    for(i = 0; i < t_top; i++) {
-	erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(i);
-	erts_smp_rwmtx_rlock(mmtl);
-	if (IS_SLOT_ALIVE(i)) {
-	    if (hp == hendp) {
-		/* Racing table creator, grab some more heap space */
-		t_tabs_cnt = 10;
-		hp = HAlloc(BIF_P, 2*t_tabs_cnt);
-		hendp = hp + 2*t_tabs_cnt;
-	    }
-	    tb = meta_main_tab[i].u.tb;
-	    previous = CONS(hp, tb->common.id, previous);
-	    hp += 2;
-	}
-	erts_smp_rwmtx_runlock(mmtl);
+        hp = &hfragp->mem[0];
+        ohp = &hfragp->off_heap;
+        list = NIL;
+        first = esdp->ets_tables.clist;
+        tb = first ? first->common.all.prev : NULL;
+    }
+
+    if (tb) {
+        while (1) {
+            if (is_table_alive(tb)) {
+                Eterm tid;
+                if (is_atom(tb->common.id))
+                    tid = tb->common.id;
+                else
+                    tid = erts_mk_magic_ref(&hp, ohp, tb->common.btid);
+                list = CONS(hp, tid, list);
+                hp += 2;
+            }
+
+            if (tb == first)
+                break;
+
+            tb = tb->common.all.prev;
+
+            if (--ycount <= 0) {
+                sz = hp - &hfragp->mem[0];
+                ASSERT(hfragp->alloc_size > sz + 1);
+                *hp = list;
+                hfragp->used_size = sz;
+                *hfragpp = hfragp;
+                *reqpp = reqp;
+                *tablepp = tb;
+                *yield_count_p = 0;
+                return 1; /* Yield! */
+            }
+        }
+    }
+
+    ref = erts_oiref_storage_make_ref(&reqp->ref, &hp);
+    msg = TUPLE2(hp, ref, list);
+    hp += 3;
+
+    sz = hp - &hfragp->mem[0];
+    ASSERT(sz <= hfragp->alloc_size);
+
+    hfragp = erts_resize_message_buffer(hfragp, sz, &msg, 1);
+
+    mp = erts_alloc_message(0, NULL);
+    mp->data.heap_frag = hfragp;
+
+    erts_queue_message(reqp->proc, 0, mp, msg, am_system);
+
+    erts_proc_dec_refc(reqp->proc);
+
+    if (erts_smp_atomic32_dec_read_nob(&reqp->refc) == 0)
+        erts_free(ERTS_ALC_T_ETS_ALL_REQ, reqp);
+
+    *reqpp = NULL;
+    *hfragpp = NULL;
+    *tablepp = NULL;
+    *yield_count_p = ycount;
+
+    return 0;
+}
+
+int
+erts_handle_yielded_ets_all_request(ErtsSchedulerData *esdp,
+                                    ErtsEtsAllYieldData *eaydp)
+{
+    int ix = (int) esdp->no - 1;
+    int yc = ERTS_ETS_ALL_TB_YCNT;
+
+    while (1) {
+        if (!eaydp->ongoing) {
+            ErtsEtsAllReq *ongoing;
+
+            if (!eaydp->queue)
+                return 0; /* All work completed! */
+
+            if (yc < ERTS_ETS_ALL_TB_YCNT_START && yc > esdp->ets_tables.count)
+                return 1; /* Yield! */
+
+            eaydp->ongoing = ongoing = eaydp->queue;
+            if (ongoing->list[ix].next == ongoing)
+                eaydp->queue = NULL;
+            else {
+                ongoing->list[ix].next->list[ix].prev = ongoing->list[ix].prev;
+                ongoing->list[ix].prev->list[ix].next = ongoing->list[ix].next;
+                eaydp->queue = ongoing->list[ix].next;
+            }
+            ASSERT(!eaydp->hfrag);
+            ASSERT(!eaydp->tab);
+        }
+
+        if (ets_all_reply(esdp, &eaydp->ongoing, &eaydp->hfrag, &eaydp->tab, &yc))
+            return 1; /* Yield! */
+    }
+}
+
+static void
+handle_ets_all_request(void *vreq)
+{
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    ErtsEtsAllYieldData *eayp = ERTS_SCHED_AUX_YIELD_DATA(esdp, ets_all);
+    ErtsEtsAllReq *req = (ErtsEtsAllReq *) vreq;
+
+    if (!eayp->ongoing && !eayp->queue) {
+        /* No ets:all() operations ongoing... */
+        ErlHeapFragment *hf = NULL;
+        DbTable *tb = NULL;
+        int yc = ERTS_ETS_ALL_TB_YCNT;
+        if (ets_all_reply(esdp, &req, &hf, &tb, &yc)) {
+            /* Yielded... */
+            ASSERT(hf);
+            eayp->ongoing = req;
+            eayp->hfrag = hf;
+            eayp->tab = tb;
+            erts_notify_new_aux_yield_work(esdp);
+        }
+    }
+    else {
+        /* Ongoing ets:all() operations; queue up this request... */
+        int ix = (int) esdp->no - 1;
+        if (!eayp->queue) {
+            req->list[ix].next = req;
+            req->list[ix].prev = req;
+            eayp->queue = req;
+        }
+        else {
+            req->list[ix].next = eayp->queue;
+            req->list[ix].prev = eayp->queue->list[ix].prev;
+            eayp->queue->list[ix].prev = req;
+            req->list[ix].prev->list[ix].next = req;
+        }
     }
-    HRelease(BIF_P, hendp, hp);
-    BIF_RET(previous);
+}
+
+BIF_RETTYPE ets_internal_request_all_0(BIF_ALIST_0)
+{
+    Eterm ref = erts_make_ref(BIF_P);
+    ErtsEtsAllReq *req = erts_alloc(ERTS_ALC_T_ETS_ALL_REQ,
+                                    ERTS_ETS_ALL_REQ_SIZE);
+    erts_smp_atomic32_init_nob(&req->refc,
+			       (erts_aint32_t) erts_no_schedulers);
+    erts_oiref_storage_save(&req->ref, ref);
+    req->proc = BIF_P;
+    erts_proc_add_refc(BIF_P, (Sint) erts_no_schedulers);
+
+#ifdef ERTS_SMP
+    if (erts_no_schedulers > 1)
+	erts_schedule_multi_misc_aux_work(1,
+					  erts_no_schedulers,
+                                          handle_ets_all_request,
+					  (void *) req);
+#endif
+
+    handle_ets_all_request((void *) req);
+    BIF_RET(ref);
 }
 
 
@@ -2788,7 +3121,7 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1)
     */
 
     if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL) {
-	if (is_atom(BIF_ARG_1) || is_small(BIF_ARG_1)) {
+	if (is_atom(BIF_ARG_1) || is_ref(BIF_ARG_1)) {
 	    BIF_RET(am_undefined);
 	}
 	BIF_ERROR(BIF_P, BADARG);
@@ -2850,7 +3183,7 @@ BIF_RETTYPE ets_info_2(BIF_ALIST_2)
     Eterm ret = THE_NON_VALUE;
 
     if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL) {
-	if (is_atom(BIF_ARG_1) || is_small(BIF_ARG_1)) {
+	if (is_atom(BIF_ARG_1) || is_ref(BIF_ARG_1)) {
 	    BIF_RET(am_undefined);
 	}
 	BIF_ERROR(BIF_P, BADARG);
@@ -3140,6 +3473,19 @@ void init_db(ErtsDbSpinCount db_spin_count)
     ms_delete_all = CONS(hp, ms_delete_all,NIL);
 }
 
+void
+erts_ets_sched_spec_data_init(ErtsSchedulerData *esdp)
+{
+    ErtsEtsAllYieldData *eaydp = ERTS_SCHED_AUX_YIELD_DATA(esdp, ets_all);
+    eaydp->ongoing = NULL;
+    eaydp->hfrag = NULL;
+    eaydp->tab = NULL;
+    eaydp->queue = NULL;
+    esdp->ets_tables.clist = NULL;
+    esdp->ets_tables.count = 0;
+}
+
+
 #define ARRAY_CHUNK 100
 
 typedef enum {
@@ -3288,17 +3634,46 @@ retry:
 	ASSERT(arityval(*tpv) == 1);
 	heir_data = tpv[1];
     }
-    erts_send_message(p, to_proc, &to_locks,
-		      TUPLE4(buf,
-			     am_ETS_TRANSFER,
-			     tb->common.id,
-			     p->common.id,
-			     heir_data), 
-                      0);
+    send_ets_transfer_message(p, to_proc, &to_locks, tb, heir_data);
     erts_smp_proc_unlock(to_proc, to_locks);
     return !0;
 }
 
+static void
+send_ets_transfer_message(Process *c_p, Process *proc,
+                          ErtsProcLocks *locks,
+                          DbTable *tb, Eterm heir_data)
+{
+    Uint hsz, hd_sz;
+    ErtsMessage *mp;
+    Eterm *hp;
+    ErlOffHeap *ohp;
+    Eterm tid, hd_copy, msg, sender;
+
+    hsz = 5;
+    if (is_not_atom(tb->common.id))
+        hsz += ERTS_MAGIC_REF_THING_SIZE;
+    if (is_immed(heir_data))
+        hd_sz = 0;
+    else {
+        hd_sz = size_object(heir_data);
+        hsz += hd_sz;
+    }
+
+    mp = erts_alloc_message_heap(proc, locks, hsz, &hp, &ohp);
+    if (is_atom(tb->common.id))
+        tid = tb->common.id;
+    else
+        tid = erts_mk_magic_ref(&hp, ohp, tb->common.btid);
+    if (!hd_sz)
+        hd_copy = heir_data;
+    else
+        hd_copy = copy_struct(heir_data, hd_sz, &hp, ohp);
+    sender = c_p->common.id;
+    msg = TUPLE4(hp, am_ETS_TRANSFER, tid, sender, hd_copy);
+    erts_queue_message(proc, *locks, mp, msg, sender);
+}
+
 /*
  * erts_db_process_exiting() is called when a process terminates.
  * It returns 0 when completely done, and !0 when it wants to
@@ -3384,7 +3759,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 #endif
 			first_call = (tb->common.status & DB_DELETE) == 0;
 			if (first_call) {
-                            tid_clear(tb);
+                            tid_clear(c_p, tb);
 			    /* Clear all access bits. */
 			    tb->common.status &= ~(DB_PROTECTED
 						   | DB_PUBLIC
@@ -3975,21 +4350,30 @@ static void print_table(fmtfn_t to, void *to_arg, int show,  DbTable* tb)
     erts_print(to, to_arg, "Read Concurrency: %T\n", table_info(NULL, tb, am_read_concurrency));
 }
 
+typedef struct {
+    fmtfn_t to;
+    void *to_arg;
+    int show;
+} ErtsPrintDbInfo;
+
+static void
+db_info_print(DbTable *tb, void *vpdbip)
+{
+    ErtsPrintDbInfo *pdbip = (ErtsPrintDbInfo *) vpdbip;
+    erts_print(pdbip->to, pdbip->to_arg, "=ets:%T\n", tb->common.owner);
+    erts_print(pdbip->to, pdbip->to_arg, "Slot: %bpu\n", (Uint) tb);
+    print_table(pdbip->to, pdbip->to_arg, pdbip->show, tb);
+}
+
 void db_info(fmtfn_t to, void *to_arg, int show)    /* Called by break handler */
 {
-    int i;
-    for (i=0; i < db_max_tabs; i++) 
-	if (IS_SLOT_ALIVE(i)) {
-	    erts_print(to, to_arg, "=ets:%T\n", meta_main_tab[i].u.tb->common.owner);
-	    erts_print(to, to_arg, "Slot: %d\n", i);
-	    print_table(to, to_arg, show, meta_main_tab[i].u.tb);
-	}
-#ifdef DEBUG
-    erts_print(to, to_arg, "=internal_ets: Process to table index\n");
-    print_table(to, to_arg, show, meta_pid_to_tab);
-    erts_print(to, to_arg, "=internal_ets: Process to fixation index\n");
-    print_table(to, to_arg, show, meta_pid_to_fixed_tab);
-#endif
+    ErtsPrintDbInfo pdbi;
+
+    pdbi.to = to;
+    pdbi.to_arg = to_arg;
+    pdbi.show = show;
+
+    erts_db_foreach_table(db_info_print, &pdbi);
 }
 
 Uint
@@ -4004,15 +4388,22 @@ erts_get_ets_misc_mem_size(void)
 void
 erts_db_foreach_table(void (*func)(DbTable *, void *), void *arg)
 {
-    int i, j;
-    j = 0;
-    for(i = 0; (i < db_max_tabs && j < meta_main_tab_cnt); i++) {
-	if (IS_SLOT_ALIVE(i)) {
-	    j++;
-	    (*func)(meta_main_tab[i].u.tb, arg);
-	}
+    int ix;
+
+    ASSERT(erts_smp_thr_progress_is_blocking());
+
+    for (ix = 0; ix < erts_no_schedulers; ix++) {
+        ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix);
+        DbTable *first = esdp->ets_tables.clist;
+        if (first) {
+            DbTable *tb = first;
+            do {
+                if (is_table_alive(tb))
+                    (*func)(tb, arg);
+                tb = tb->common.all.next;
+            } while (tb != first);
+        }
     }
-    ASSERT(j == meta_main_tab_cnt);
 }
 
 /* SMP Note: May only be used when system is locked */
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index 852440ff31..1c53401590 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -24,8 +24,37 @@
  *
  */
 
-#ifndef __DB_H__
-#define __DB_H__
+#ifndef ERTS_DB_SCHED_SPEC_TYPES__
+#define ERTS_DB_SCHED_SPEC_TYPES__
+
+union db_table;
+typedef union db_table DbTable;
+
+typedef struct ErtsEtsAllReq_ ErtsEtsAllReq;
+
+typedef struct {
+    ErtsEtsAllReq *next;
+    ErtsEtsAllReq *prev;
+} ErtsEtsAllReqList;
+
+typedef struct {
+    ErtsEtsAllReq *ongoing;
+    ErlHeapFragment *hfrag;
+    DbTable *tab;
+    ErtsEtsAllReq *queue;
+} ErtsEtsAllYieldData;
+
+typedef struct {
+    Uint count;
+    DbTable *clist;
+} ErtsEtsTables;
+
+#endif /* ERTS_DB_SCHED_SPEC_TYPES__ */
+
+#ifndef ERTS_ONLY_SCHED_SPEC_ETS_DATA
+
+#ifndef ERL_DB_H__
+#define ERL_DB_H__
 
 #include "sys.h"
 #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
@@ -46,6 +75,12 @@ typedef struct {
     ErtsThrPrgrLaterOp data;
 } DbTableRelease;
 
+struct ErtsSchedulerData_;
+int erts_handle_yielded_ets_all_request(struct ErtsSchedulerData_ *esdp,
+                                        ErtsEtsAllYieldData *eadp);
+
+void erts_ets_sched_spec_data_init(struct ErtsSchedulerData_ *esdp);
+
 /*
  * So, the structure for a database table, NB this is only
  * interesting in db.c.
@@ -95,7 +130,7 @@ Eterm erts_ets_restore_meta_state(Process* p, Eterm target_state);
 
 Uint erts_db_get_max_tabs(void);
 
-#endif
+#endif /* ERL_DB_H__ */
 
 #if defined(ERTS_WANT_DB_INTERNAL__) && !defined(ERTS_HAVE_DB_INTERNAL__)
 #define ERTS_HAVE_DB_INTERNAL__
@@ -271,3 +306,4 @@ erts_db_free_nt(ErtsAlcType_t type, void *ptr, Uint size)
 
 #endif /* #if defined(ERTS_WANT_DB_INTERNAL__) && !defined(ERTS_HAVE_DB_INTERNAL__) */
 
+#endif /* !ERTS_ONLY_SCHED_SPEC_ETS_DATA */
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 60dedd5713..3c13035ec5 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -75,9 +75,6 @@ typedef struct db_term {
      */
 } DbTerm;
 
-union db_table;
-typedef union db_table DbTable;
-
 #define DB_MUST_RESIZE 1
 #define DB_NEW_OBJECT 2
 #define DB_INC_TRY_GROW 4
@@ -203,6 +200,11 @@ typedef struct db_fixation {
     struct db_fixation *next;
 } DbFixation;
 
+typedef struct {
+    DbTable *next;
+    DbTable *prev;
+} DbTableList;
+
 /*
  * This structure contains data for all different types of database
  * tables. Note that these fields must match the same fields
@@ -214,6 +216,7 @@ typedef struct db_fixation {
 typedef struct db_table_common {
     erts_smp_refc_t refc;     /* reference count of table struct */
     erts_smp_refc_t fix_count;/* fixation counter */
+    DbTableList all;
 #ifdef ERTS_SMP
     erts_smp_rwmtx_t rwlock;  /* rw lock on table */
     erts_smp_mtx_t fixlock;   /* Protects fixations,megasec,sec,microsec */
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index b2c307e826..61477af316 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -2307,6 +2307,7 @@ erl_start(int argc, char **argv)
 #endif
 	set_main_stack_size();
 	erts_sched_init_time_sup(esdp);
+        erts_ets_sched_spec_data_init(esdp);
 	process_main(esdp->x_reg_array, esdp->f_reg_array);
     }
 #endif
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 52d9f9ddf7..b816ba59fb 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -602,6 +602,7 @@ dbg_chk_aux_work_val(erts_aint32_t value)
     valid |= ERTS_SSI_AUX_WORK_REAP_PORTS;
 #endif
     valid |= ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
+    valid |= ERTS_SSI_AUX_WORK_YIELD;
 
     if (~valid & value)
 	erts_exit(ERTS_ABORT_EXIT,
@@ -690,6 +691,8 @@ erts_pre_init_process(void)
 	= "SET_TMO";
     erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX]
 	= "MSEG_CACHE_CHECK";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_YIELD_IX]
+	= "YIELD";
     erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_REAP_PORTS_IX]
 	= "REAP_PORTS";
     erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX]
@@ -2557,6 +2560,48 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
     return aux_work & ~ERTS_SSI_AUX_WORK_REAP_PORTS;
 }
 
+void
+erts_notify_new_aux_yield_work(ErtsSchedulerData *esdp)
+{
+    ASSERT(esdp == erts_get_scheduler_data());
+    /* Always called by the scheduler itself... */
+    set_aux_work_flags_wakeup_nob(esdp->ssi, ERTS_SSI_AUX_WORK_YIELD);
+}
+
+static ERTS_INLINE erts_aint32_t
+handle_yield(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
+{
+    int yield = 0;
+    /*
+     * Yield operations are always requested by the scheduler itself.
+     *
+     * The following handlers should *not* set the ERTS_SSI_AUX_WORK_YIELD
+     * flag in order to indicate more work. They should instead return
+     * information so this "main handler" can manipulate the flag...
+     *
+     * The following handlers should be able to handle being called
+     * even though no work is to be done...
+     */
+
+    /* Various yielding operations... */
+
+    yield |= erts_handle_yielded_ets_all_request(awdp->esdp,
+                                                 &awdp->yield.ets_all);
+
+    /*
+     * Other yielding operations...
+     *
+     */
+
+    if (!yield) {
+        unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_YIELD);
+        return aux_work & ~ERTS_SSI_AUX_WORK_YIELD;
+    }
+
+    return aux_work;
+}
+
+
 #if HAVE_ERTS_MSEG
 
 static ERTS_INLINE erts_aint32_t
@@ -2697,6 +2742,9 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
 		    handle_mseg_cache_check);
 #endif
 
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_YIELD,
+		    handle_yield);
+
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_REAP_PORTS,
 		    handle_reap_ports);
 
@@ -8727,6 +8775,8 @@ sched_thread_func(void *vesdp)
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL);
 #endif
 
+    erts_ets_sched_spec_data_init(esdp);
+
     process_main(esdp->x_reg_array, esdp->f_reg_array);
 
     /* No schedulers should *ever* terminate */
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index ce0989883c..58f1e15c23 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -63,6 +63,9 @@ typedef struct process Process;
 #define ERTS_ONLY_INCLUDE_TRACE_FLAGS
 #include "erl_trace.h"
 #undef ERTS_ONLY_INCLUDE_TRACE_FLAGS
+#define ERTS_ONLY_SCHED_SPEC_ETS_DATA
+#include "erl_db.h"
+#undef ERTS_ONLY_SCHED_SPEC_ETS_DATA
 
 #ifdef HIPE
 #include "hipe_process.h"
@@ -312,6 +315,7 @@ typedef enum {
     ERTS_SSI_AUX_WORK_PENDING_EXITERS_IX,
     ERTS_SSI_AUX_WORK_SET_TMO_IX,
     ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
+    ERTS_SSI_AUX_WORK_YIELD_IX,
     ERTS_SSI_AUX_WORK_REAP_PORTS_IX,
     ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */
 
@@ -348,6 +352,8 @@ typedef enum {
     (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
 #define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
     (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX)
+#define ERTS_SSI_AUX_WORK_YIELD \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_YIELD_IX)
 #define ERTS_SSI_AUX_WORK_REAP_PORTS \
     (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX)
 #define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \
@@ -612,6 +618,10 @@ typedef struct {
 	ErtsDelayedAuxWorkWakeupJob *job;
     } delayed_wakeup;
 #endif
+    struct {
+        ErtsEtsAllYieldData ets_all;
+        /* Other yielding operations... */
+    } yield;
     struct {
 	struct {
 	    erts_aint32_t flags;
@@ -621,6 +631,10 @@ typedef struct {
     } debug;
 } ErtsAuxWorkData;
 
+#define ERTS_SCHED_AUX_YIELD_DATA(ESDP, NAME) \
+    (&(ESDP)->aux_work_data.yield.NAME)
+void erts_notify_new_aux_yield_work(ErtsSchedulerData *esdp);
+
 #ifdef ERTS_DIRTY_SCHEDULERS
 typedef enum {
     ERTS_DIRTY_CPU_SCHEDULER,
@@ -688,7 +702,7 @@ struct ErtsSchedulerData_ {
     ErtsSchedWallTime sched_wall_time;
     ErtsGCInfo gc_info;
     ErtsPortTaskHandle nosuspend_port_task_handle;
-
+    ErtsEtsTables ets_tables;
 #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
     erts_alloc_verify_func_t verify_unused_temp_alloc;
     Allctr_t *verify_unused_temp_alloc_data;
-- 
cgit v1.2.3


From 7488cfc986e1608e73f904071556eb0543d6a346 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 2 Mar 2017 16:59:38 +0100
Subject: erts: Add enif_inspect_binary emasculation of refc bins

---
 erts/emulator/beam/erl_nif.c | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index e6da4c1a76..63a4a997da 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -998,16 +998,14 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin)
 	byte* raw_ptr;
     }u;
 
-    if (is_boxed(bin_term) && *binary_val(bin_term) == HEADER_SUB_BIN) {
-	ErlSubBin* sb = (ErlSubBin*) binary_val(bin_term);
-	if (sb->is_writable) {
-	    ProcBin* pb = (ProcBin*) binary_val(sb->orig);
-	    ASSERT(pb->thing_word == HEADER_PROC_BIN);
-	    if (pb->flags) {
-		erts_emasculate_writable_binary(pb);
-		sb->is_writable = 0;
-	    }
-	}
+    if (is_binary(bin_term)) {
+        ProcBin *pb = (ProcBin*) binary_val(bin_term);
+        if (pb->thing_word == HEADER_SUB_BIN) {
+            ErlSubBin* sb = (ErlSubBin*) pb;
+            pb = (ProcBin*) binary_val(sb->orig);
+        }
+        if (pb->thing_word == HEADER_PROC_BIN && pb->flags)
+            erts_emasculate_writable_binary(pb);
     }
     u.tmp = NULL;
     bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, allocator,
@@ -1024,7 +1022,7 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin)
     bin->bin_term = bin_term;
     bin->size = binary_size(bin_term);
     bin->ref_bin = NULL;
-    ADD_READONLY_CHECK(env, bin->data, bin->size); 
+    ADD_READONLY_CHECK(env, bin->data, bin->size);
     return 1;
 }
 
-- 
cgit v1.2.3


From 5cce8836ccffbaf71879b2efbb5788278e7f7e90 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 8 Mar 2017 14:41:54 +0100
Subject: erts: Fix faulty ASSERT for failed dec_term

when a recursive call to dec_term() has already done erts_factory_undo
and set factory->hp to NULL.
---
 erts/emulator/beam/external.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 205a7711ec..186b250c7e 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -4010,7 +4010,7 @@ error:
 	    factory->hp = hp;   /* the largest must be the freshest */
 	}
     }
-    else ASSERT(factory->hp == hp);
+    else ASSERT(!factory->hp || factory->hp == hp);
 
 error_hamt:
     erts_factory_undo(factory);
-- 
cgit v1.2.3


From 65b04e233e09e3cc2e0fda3c28e155b95c5a4baf Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 7 Mar 2017 17:04:40 +0100
Subject: erts: Do not generate atoms on old latin1 external format

term_to_binary will always generate utf8 atoms
ATOM_UTF8_EXT and SMALL_ATOM_UTF8_EXT.

Old latin1 atoms, ATOM_EXT and SMALL_ATOM_EXT, are still decoded.
---
 erts/emulator/beam/dist.h     |  1 +
 erts/emulator/beam/external.c | 39 +++------------------------------------
 2 files changed, 4 insertions(+), 36 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 8f6be1061a..6ed36a478e 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -53,6 +53,7 @@
 			       | DFLAG_EXPORT_PTR_TAG		\
 			       | DFLAG_BIT_BINARIES             \
 			       | DFLAG_MAP_TAG                  \
+                               | DFLAG_UTF8_ATOMS               \
                                | DFLAG_BIG_CREATION)
 
 /* opcodes used in distribution messages */
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 186b250c7e..285ae4ac78 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2093,7 +2093,6 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags)
 {
     int iix;
     int len;
-    int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS);
 
     ASSERT(is_atom(atom));
 
@@ -2122,8 +2121,8 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags)
     if (iix < 0) {
 	Atom *a = atom_tab(atom_val(atom));
 	len = a->len;
-	if (utf8_atoms || a->latin1_chars < 0) {
-	    if (len > 255) {
+        {
+            if (len > 255) {
 		*ep++ = ATOM_UTF8_EXT;
 		put_int16(len, ep);
 		ep += 2;
@@ -2135,32 +2134,6 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags)
 	    }
 	    sys_memcpy((char *) ep, (char *) a->name, len);
 	}
-	else {
-	    if (a->latin1_chars <= 255 && (dflags & DFLAG_SMALL_ATOM_TAGS)) {
-		*ep++ = SMALL_ATOM_EXT;
-		if (len == a->latin1_chars) {
-		    sys_memcpy(ep+1, a->name, len);
-		}
-		else {
-		    len = erts_utf8_to_latin1(ep+1, a->name, len);
-		    ASSERT(len == a->latin1_chars);
-		}
-		put_int8(len, ep);
-		ep++;
-	    }
-	    else {
-		*ep++ = ATOM_EXT;
-		if (len == a->latin1_chars) {
-		    sys_memcpy(ep+2, a->name, len);
-		}
-		else {
-		    len = erts_utf8_to_latin1(ep+2, a->name, len);
-		    ASSERT(len == a->latin1_chars);
-		}
-		put_int16(len, ep);
-		ep += 2;
-	    }	    
-	}
 	ep += len;
 	return ep;
     }
@@ -4085,19 +4058,13 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 	    else {
 		Atom *a = atom_tab(atom_val(obj));
 		int alen;
-		if ((dflags & DFLAG_UTF8_ATOMS) || a->latin1_chars < 0) {
+                {
 		    alen = a->len;
 		    result += 1 + 1 + alen;
 		    if (alen > 255) {
 			result++; /* ATOM_UTF8_EXT (not small) */
 		    }
 		}
-		else {
-		    alen = a->latin1_chars;
-		    result += 1 + 1 + alen;
-		    if (alen > 255 || !(dflags & DFLAG_SMALL_ATOM_TAGS))
-			result++; /* ATOM_EXT (not small) */
-		}
 		insert_acache_map(acmp, obj, dflags);
 	    }
 	    break;
-- 
cgit v1.2.3


From 26c3cd82529836cb5b6eefbf7f92f318fd91f847 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 10 Mar 2017 15:00:46 +0100
Subject: Update copyright year

---
 erts/emulator/beam/beam_bif_load.c    | 2 +-
 erts/emulator/beam/beam_bp.c          | 2 +-
 erts/emulator/beam/beam_bp.h          | 2 +-
 erts/emulator/beam/beam_emu.c         | 2 +-
 erts/emulator/beam/beam_load.c        | 2 +-
 erts/emulator/beam/break.c            | 2 +-
 erts/emulator/beam/copy.c             | 2 +-
 erts/emulator/beam/dist.c             | 2 +-
 erts/emulator/beam/erl_bif_ddll.c     | 2 +-
 erts/emulator/beam/erl_bif_guard.c    | 2 +-
 erts/emulator/beam/erl_bif_info.c     | 2 +-
 erts/emulator/beam/erl_bif_trace.c    | 2 +-
 erts/emulator/beam/erl_db.c           | 2 +-
 erts/emulator/beam/erl_db_util.c      | 2 +-
 erts/emulator/beam/erl_db_util.h      | 2 +-
 erts/emulator/beam/erl_fun.c          | 2 +-
 erts/emulator/beam/erl_fun.h          | 2 +-
 erts/emulator/beam/erl_gc.c           | 2 +-
 erts/emulator/beam/erl_gc.h           | 2 +-
 erts/emulator/beam/erl_hl_timer.c     | 2 +-
 erts/emulator/beam/erl_init.c         | 2 +-
 erts/emulator/beam/erl_lock_check.c   | 2 +-
 erts/emulator/beam/erl_message.c      | 2 +-
 erts/emulator/beam/erl_monitors.c     | 2 +-
 erts/emulator/beam/erl_nif.c          | 2 +-
 erts/emulator/beam/erl_node_tables.c  | 2 +-
 erts/emulator/beam/erl_node_tables.h  | 2 +-
 erts/emulator/beam/erl_process.c      | 2 +-
 erts/emulator/beam/erl_process_dump.c | 2 +-
 erts/emulator/beam/erl_trace.c        | 2 +-
 erts/emulator/beam/erl_vm.h           | 2 +-
 erts/emulator/beam/export.c           | 2 +-
 erts/emulator/beam/external.c         | 2 +-
 erts/emulator/beam/global.h           | 2 +-
 erts/emulator/beam/index.c            | 2 +-
 erts/emulator/beam/index.h            | 2 +-
 erts/emulator/beam/io.c               | 2 +-
 erts/emulator/beam/sys.h              | 2 +-
 erts/emulator/beam/utils.c            | 2 +-
 erts/emulator/hipe/hipe_bif0.c        | 2 +-
 erts/emulator/hipe/hipe_native_bif.c  | 2 +-
 erts/emulator/sys/unix/sys.c          | 2 +-
 erts/emulator/test/Makefile           | 2 +-
 erts/emulator/test/guard_SUITE.erl    | 2 +-
 erts/emulator/test/num_bif_SUITE.erl  | 2 +-
 erts/emulator/test/process_SUITE.erl  | 2 +-
 46 files changed, 46 insertions(+), 46 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 7f5e9ff691..27d6823d3c 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index a66d9f68d9..6d723a4b92 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index 9e4ab3cd20..7372b9b258 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 1c83d8f02e..1ad13c32e3 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 88b6e89a02..2bad3ab4c5 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 4c7c910419..74a8a3b852 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 78db141cfc..de68adbeb6 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 52b2174609..3611eeee02 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index bace51bbbb..4948975851 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c
index 74cf7cf11a..ea508bd1c4 100644
--- a/erts/emulator/beam/erl_bif_guard.c
+++ b/erts/emulator/beam/erl_bif_guard.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2006-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 95a56a3de9..7bd45916f5 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 0627526d7e..4fed09f085 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 3cc2b21a20..23c8a5bae9 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index f4db7ca3dd..3c5bfb2552 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 3fb063797e..b30f0580dd 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index fd4d5b1c5c..53a1801b72 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h
index 0fd0d62343..b1d5cddb34 100644
--- a/erts/emulator/beam/erl_fun.h
+++ b/erts/emulator/beam/erl_fun.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 9b7744068e..bb36be0848 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 2521379664..76dd74c866 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2007-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 647fa26811..a73f4ecfc7 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2015-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2015-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index f65a06c85f..e8b8739852 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 5d4823c077..c53bdc685b 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 019e079e67..cdd771ef7d 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
index c207dea10f..7244ae144f 100644
--- a/erts/emulator/beam/erl_monitors.c
+++ b/erts/emulator/beam/erl_monitors.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2004-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 4fa15a8dd8..479a2f4809 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 467a05f950..f37393bce4 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 35051173d0..489da1ba17 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 485949c84a..5c0322a7f6 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 4a74cc2b82..8d66391d55 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2003-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index ac9e91e31f..00bb114c26 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 899e2a275a..3d8c647386 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 00fa452f79..6cf6fed31a 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 0ef8114bac..9436259011 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 26aa39b65a..19286e1310 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c
index cd834e2c12..a1f6f54543 100644
--- a/erts/emulator/beam/index.c
+++ b/erts/emulator/beam/index.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h
index 10f5d1eb39..6c07571df6 100644
--- a/erts/emulator/beam/index.h
+++ b/erts/emulator/beam/index.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 38e403d91a..c3eb610fdc 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 79ec34e717..d49edad6dc 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 7f7e38c22a..d90c282c7e 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 165f33eecd..c0ebc34771 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index a2165958a9..860791074d 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index e135dbff99..6459fa064b 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index 453f819d1b..b8e907f769 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1997-2016. All Rights Reserved.
+# Copyright Ericsson AB 1997-2017. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/guard_SUITE.erl b/erts/emulator/test/guard_SUITE.erl
index 54ee710363..1a93a9f5c2 100644
--- a/erts/emulator/test/guard_SUITE.erl
+++ b/erts/emulator/test/guard_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl
index bb85738454..f31e73c85b 100644
--- a/erts/emulator/test/num_bif_SUITE.erl
+++ b/erts/emulator/test/num_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index 0a6eb7ffac..ab1587a90c 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
-- 
cgit v1.2.3


From 5d95b5ce9eb08f2087443c1be8de8044552daecc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 13 Mar 2017 16:21:38 +0100
Subject: erts: Fix os_signal_SUITE test

---
 erts/emulator/test/os_signal_SUITE.erl | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/os_signal_SUITE.erl b/erts/emulator/test/os_signal_SUITE.erl
index 9aa49a453e..6bafb0e18c 100644
--- a/erts/emulator/test/os_signal_SUITE.erl
+++ b/erts/emulator/test/os_signal_SUITE.erl
@@ -323,7 +323,10 @@ kill(Signal, Pid) ->
 
 load_nif(Config) ->
     Path = proplists:get_value(data_dir, Config),
-    ok = erlang:load_nif(filename:join(Path,"os_signal_nif"), 0).
+    case erlang:load_nif(filename:join(Path,"os_signal_nif"), 0) of
+        ok -> ok;
+        {error,{reload,_}} -> ok
+    end.
 
 run(Program0, Args) -> run(".", Program0, Args).
 run(Cwd, Program0, Args) when is_list(Cwd) ->
-- 
cgit v1.2.3


From aec471e80bf086ae556c9c9b442c3d6cc864c6c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 14 Mar 2017 09:55:27 +0100
Subject: erts: Fix signal pipe

---
 erts/emulator/sys/unix/sys.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 0d1ed17449..e836f9bcc8 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -1274,8 +1274,8 @@ signal_dispatcher_thread_func(void *unused)
 
         do {
             res = read(sig_notify_fds[0], (void *) &sb.buf[i], sizeof(int) - i);
-            i += res;
-        } while ((i != sizeof(int) && res >= 0) || (res < 0 && errno == EINTR));
+            i += res > 0 ? res : 0;
+        } while ((i < sizeof(int) && res >= 0) || (res < 0 && errno == EINTR));
 
 	if (res < 0) {
 	    erts_exit(ERTS_ABORT_EXIT,
-- 
cgit v1.2.3


From b4b8529371f4c157ab741f30a8aeeccf16c102bc Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 14 Mar 2017 12:07:23 +0100
Subject: erts: Remove fun_r13_SUITE

---
 erts/emulator/test/Makefile          |  1 -
 erts/emulator/test/fun_r13_SUITE.erl | 74 ------------------------------------
 2 files changed, 75 deletions(-)
 delete mode 100644 erts/emulator/test/fun_r13_SUITE.erl

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index 5478932b13..927a9366b9 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -66,7 +66,6 @@ MODULES= \
 	exception_SUITE \
 	float_SUITE \
 	fun_SUITE \
-	fun_r13_SUITE \
 	gc_SUITE \
 	guard_SUITE \
 	hash_SUITE \
diff --git a/erts/emulator/test/fun_r13_SUITE.erl b/erts/emulator/test/fun_r13_SUITE.erl
deleted file mode 100644
index a45ed08b9d..0000000000
--- a/erts/emulator/test/fun_r13_SUITE.erl
+++ /dev/null
@@ -1,74 +0,0 @@
-%%
-%% %CopyrightBegin%
-%%
-%% Copyright Ericsson AB 2007-2016. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%
-%% %CopyrightEnd%
-%%
-
--module(fun_r13_SUITE).
--compile(r13).
-
--export([all/0, suite/0,
-         dist_old_release/1]).
-
--include_lib("common_test/include/ct.hrl").
-
-suite() ->
-    [{ct_hooks,[ts_install_cth]},
-     {timetrap, {minutes, 1}}].
-
-all() ->
-    [dist_old_release].
-
-dist_old_release(Config) when is_list(Config) ->
-    case test_server:is_release_available("r12b") of
-        true -> do_dist_old(Config);
-        false -> {skip,"No R12B found"}
-    end.
-
-do_dist_old(Config) when is_list(Config) ->
-    Pa = filename:dirname(code:which(?MODULE)),
-    Name = fun_dist_r12,
-    {ok,Node} = test_server:start_node(Name, peer,
-                                       [{args,"-pa "++Pa},
-                                        {erl,[{release,"r12b"}]}]),
-
-    Pid = spawn_link(Node,
-                     fun() ->
-                             receive
-                                 Fun when is_function(Fun) ->
-                                     R12BFun = fun(H) -> cons(H, [b,c]) end,
-                                     Fun(Fun, R12BFun)
-                             end
-                     end),
-    Self = self(),
-    Fun = fun(F, R12BFun) ->
-                  {pid,Self} = erlang:fun_info(F, pid),
-                  {module,?MODULE} = erlang:fun_info(F, module),
-                  Self ! {ok,F,R12BFun}
-          end,
-    Pid ! Fun,
-    receive
-        {ok,Fun,R12BFun} ->
-            [a,b,c] = R12BFun(a);
-        Other ->
-            ct:fail({bad_message,Other})
-    end,
-    true = test_server:stop_node(Node),
-    ok.
-
-cons(H, T) ->
-    [H|T].
-- 
cgit v1.2.3


From a603a15503c6e11290c354c8c5c18578ccd8acff Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 16 Mar 2017 18:24:43 +0100
Subject: erts: Change HIPE allocations from sys_alloc

to long lived, short lived and native stack.
---
 erts/emulator/beam/beam_load.c        |  6 +++---
 erts/emulator/beam/erl_alloc.types    |  5 +++--
 erts/emulator/hipe/hipe_bif0.c        | 24 ++++++++++++------------
 erts/emulator/hipe/hipe_bif1.c        |  4 ++--
 erts/emulator/hipe/hipe_load.c        |  2 +-
 erts/emulator/hipe/hipe_mode_switch.c |  6 +++---
 erts/emulator/hipe/hipe_module.c      |  4 ++--
 erts/emulator/hipe/hipe_process.h     |  2 +-
 erts/emulator/hipe/hipe_stack.c       |  8 ++++----
 erts/emulator/hipe/hipe_x86_signal.c  |  2 +-
 10 files changed, 32 insertions(+), 31 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 48206a75a8..6eea963016 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -6321,7 +6321,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
      */
     magic = erts_alloc_loader_state();
     stp = ERTS_MAGIC_BIN_DATA(magic);
-    hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code));
+    hipe_code = erts_alloc(ERTS_ALC_T_HIPE_LL, sizeof(*hipe_code));
 
     if (!is_internal_magic_ref(hipe_magic_bin) ||
 	!(hipe_magic = erts_magic_ref2bin(hipe_magic_bin),
@@ -6556,7 +6556,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
     }
 
  error:
-    erts_free(ERTS_ALC_T_HIPE, hipe_code);
+    erts_free(ERTS_ALC_T_HIPE_LL, hipe_code);
     erts_free_aligned_binary_bytes(temp_alloc);
     free_loader_state(magic);
     BIF_ERROR(p, BADARG);
@@ -6583,7 +6583,7 @@ int erts_commit_hipe_patch_load(Eterm hipe_magic_bin)
     /*
      * Initialise HiPE module
      */
-    hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code));
+    hipe_code = erts_alloc(ERTS_ALC_T_HIPE_LL, sizeof(*hipe_code));
     hipe_code->text_segment = hipe_stp->text_segment;
     hipe_code->text_segment_size = hipe_stp->text_segment_size;
     hipe_code->data_segment = hipe_stp->data_segment;
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 3d5de72ee7..46ab941041 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -350,8 +350,9 @@ type	SL_MPATHS	SHORT_LIVED	SYSTEM		sl_migration_paths
 
 +if hipe
 
-# Currently most hipe code use this type.
-type	HIPE		SYSTEM		SYSTEM		hipe_data
+type	HIPE_LL		LONG_LIVED	SYSTEM		hipe_long_lived
+type	HIPE_SL		SHORT_LIVED	SYSTEM		hipe_short_lived
+type	HIPE_STK	STANDARD	SYSTEM		hipe_nstack
 
 +if exec_alloc
 type	HIPE_EXEC	EXEC            CODE		hipe_code
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 688c82ab7a..4babb2db4e 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -459,13 +459,13 @@ BIF_RETTYPE hipe_bifs_alloc_data_3(BIF_ALIST_3)
     stp->data_segment_size = unsigned_val(BIF_ARG_2);
     if (stp->data_segment_size == 0)
 	BIF_RET(make_small(0));
-    stp->data_segment = erts_alloc(ERTS_ALC_T_HIPE, stp->data_segment_size);
+    stp->data_segment = erts_alloc(ERTS_ALC_T_HIPE_LL, stp->data_segment_size);
     if ((unsigned long)stp->data_segment & (align-1)) {
 	fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte "
 		"aligned\r\n",
 		__FUNCTION__, (unsigned long)stp->data_segment_size,
 		stp->data_segment, (unsigned long)align);
-	erts_free(ERTS_ALC_T_HIPE, stp->data_segment);
+	erts_free(ERTS_ALC_T_HIPE_LL, stp->data_segment);
 	stp->data_segment = NULL;
 	stp->data_segment_size = 0;
 	BIF_ERROR(BIF_P, EXC_NOTSUP);
@@ -545,7 +545,7 @@ static void init_const_term_table(void)
     f.meta_alloc = (HMALLOC_FUN) erts_alloc;
     f.meta_free = (HMFREE_FUN) erts_free;
     f.meta_print = (HMPRINT_FUN) erts_print;
-    hash_init(ERTS_ALC_T_HIPE, &const_term_table, "const_term_table", 97, f);
+    hash_init(ERTS_ALC_T_HIPE_LL, &const_term_table, "const_term_table", 97, f);
 }
 
 BIF_RETTYPE hipe_bifs_merge_term_1(BIF_ALIST_1)
@@ -859,7 +859,7 @@ static void init_primop_table(void)
     f.meta_free = (HMFREE_FUN) erts_free;
     f.meta_print = (HMPRINT_FUN) erts_print;
 
-    hash_init(ERTS_ALC_T_HIPE, &primop_table, "primop_table", 50, f);
+    hash_init(ERTS_ALC_T_HIPE_LL, &primop_table, "primop_table", 50, f);
 
     for (i = 0; i < sizeof(primops)/sizeof(primops[0]); ++i)
 	hash_put(&primop_table, &primops[i]);
@@ -1094,7 +1094,7 @@ static void mod2mfa_tab_init(void)
     f.meta_free = (HMFREE_FUN) erts_free;
     f.meta_print = (HMPRINT_FUN) erts_print;
 
-    hash_init(ERTS_ALC_T_HIPE, &mod2mfa_tab, "mod2mfa_tab", 50, f);
+    hash_init(ERTS_ALC_T_HIPE_LL, &mod2mfa_tab, "mod2mfa_tab", 50, f);
 }
 
 static struct hipe_mfa_info* mod2mfa_get(Module* modp)
@@ -1172,7 +1172,7 @@ struct hipe_mfa_info* mod2mfa_get_safe(Module* modp)
 static struct hipe_mfa_info **hipe_mfa_info_table_alloc_bucket(unsigned int size)
 {
     unsigned long nbytes = size * sizeof(struct hipe_mfa_info*);
-    struct hipe_mfa_info **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes);
+    struct hipe_mfa_info **bucket = erts_alloc(ERTS_ALC_T_HIPE_LL, nbytes);
     sys_memzero(bucket, nbytes);
     return bucket;
 }
@@ -1201,14 +1201,14 @@ static void hipe_mfa_info_table_grow(void)
 	    b = next;
 	}
     }
-    erts_free(ERTS_ALC_T_HIPE, old_bucket);
+    erts_free(ERTS_ALC_T_HIPE_LL, old_bucket);
 }
 
 static struct hipe_mfa_info *hipe_mfa_info_table_alloc(Eterm m, Eterm f, unsigned int arity)
 {
     struct hipe_mfa_info *res;
 
-    res = (struct hipe_mfa_info*)erts_alloc(ERTS_ALC_T_HIPE, sizeof(*res));
+    res = (struct hipe_mfa_info*)erts_alloc(ERTS_ALC_T_HIPE_LL, sizeof(*res));
     res->m = m;
     res->f = f;
     res->a = arity;
@@ -1547,7 +1547,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
     hipe_mfa_info_table_rwlock();
     callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari);
 
-    ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref));
+    ref = erts_alloc(ERTS_ALC_T_HIPE_LL, sizeof(struct hipe_ref));
     ref->address = address;
 #if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
     ref->trampoline = trampoline;
@@ -1618,7 +1618,7 @@ static void purge_mfa(struct hipe_mfa_info* p)
     ASSERT(p->is_stub);
     remove_mfa_info(p);
     hipe_free_native_stub(p->remote_address);
-    erts_free(ERTS_ALC_T_HIPE, p);
+    erts_free(ERTS_ALC_T_HIPE_LL, p);
 }
 
 /* Called by init:restart after unloading all hipe compiled modules
@@ -1643,7 +1643,7 @@ static void hipe_purge_all_refs(void)
 	    bucket[i] = mfa->bucket.next;
 
             hash_erase(&mod2mfa_tab, mfa);
-            erts_free(ERTS_ALC_T_HIPE, mfa);
+            erts_free(ERTS_ALC_T_HIPE_LL, mfa);
 	}
     }
     hipe_mfa_info_table.used = 0;
@@ -1717,7 +1717,7 @@ void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module,
         }
 
         ref = ref->next_from_modi;
-        erts_free(ERTS_ALC_T_HIPE, free_ref);
+        erts_free(ERTS_ALC_T_HIPE_LL, free_ref);
     }
 }
 
diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c
index 0c66eb6abe..0ba0fa5172 100644
--- a/erts/emulator/hipe/hipe_bif1.c
+++ b/erts/emulator/hipe/hipe_bif1.c
@@ -50,7 +50,7 @@ BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1)
 	BIF_ERROR(BIF_P, BADARG);
     if (pc[0] == BeamOpCode(op_hipe_call_count))
 	BIF_RET(NIL);
-    hcc = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hcc));
+    hcc = erts_alloc(ERTS_ALC_T_HIPE_SL, sizeof(*hcc));
     hcc->count = 0;
     hcc->opcode = pc[0];
     pc[-4] = (Eterm)hcc;
@@ -74,7 +74,7 @@ BIF_RETTYPE hipe_bifs_call_count_off_1(BIF_ALIST_1)
     count = hcc->count;
     pc[0] = hcc->opcode;
     pc[-4] = (Eterm)NULL;
-    erts_free(ERTS_ALC_T_HIPE, hcc);
+    erts_free(ERTS_ALC_T_HIPE_SL, hcc);
     BIF_RET(make_small(count));
 }
 
diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
index 87c5004d2b..0b53880628 100644
--- a/erts/emulator/hipe/hipe_load.c
+++ b/erts/emulator/hipe/hipe_load.c
@@ -45,7 +45,7 @@ void hipe_free_loader_state(HipeLoaderState *stp)
     stp->text_segment_size = 0;
 
     if (stp->data_segment)
-	erts_free(ERTS_ALC_T_HIPE, stp->data_segment);
+	erts_free(ERTS_ALC_T_HIPE_LL, stp->data_segment);
     stp->data_segment = NULL;
     stp->data_segment_size = 0;
 
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index f11223d8b0..712f65f629 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -664,7 +664,7 @@ void hipe_inc_nstack(Process *p)
 {
     unsigned old_size = p->hipe.nstend - p->hipe.nstack;
     unsigned new_size = hipe_next_nstack_size(old_size);
-    Eterm *new_nstack = erts_alloc(ERTS_ALC_T_HIPE, new_size*sizeof(Eterm));
+    Eterm *new_nstack = erts_alloc(ERTS_ALC_T_HIPE_STK, new_size*sizeof(Eterm));
     unsigned used_size = p->hipe.nstend - p->hipe.nsp;
 
     sys_memcpy(new_nstack+new_size-used_size, p->hipe.nsp, used_size*sizeof(Eterm));
@@ -673,7 +673,7 @@ void hipe_inc_nstack(Process *p)
     if (p->hipe.nstblacklim)
 	p->hipe.nstblacklim = new_nstack + new_size - (p->hipe.nstend - p->hipe.nstblacklim);
     if (p->hipe.nstack)
-	erts_free(ERTS_ALC_T_HIPE, p->hipe.nstack);
+	erts_free(ERTS_ALC_T_HIPE_STK, p->hipe.nstack);
     p->hipe.nstack = new_nstack;
     p->hipe.nstend = new_nstack + new_size;
     p->hipe.nsp = new_nstack + new_size - used_size;
@@ -683,7 +683,7 @@ void hipe_inc_nstack(Process *p)
 void hipe_empty_nstack(Process *p)
 {
     if (p->hipe.nstack) {
-	erts_free(ERTS_ALC_T_HIPE, p->hipe.nstack);
+	erts_free(ERTS_ALC_T_HIPE_STK, p->hipe.nstack);
     }
     p->hipe.nstgraylim = NULL;
     p->hipe.nsp = NULL;
diff --git a/erts/emulator/hipe/hipe_module.c b/erts/emulator/hipe/hipe_module.c
index 469f077dd2..2e99a30556 100644
--- a/erts/emulator/hipe/hipe_module.c
+++ b/erts/emulator/hipe/hipe_module.c
@@ -29,7 +29,7 @@ void hipe_free_module(HipeModule *mod)
 {
     hipe_free_code(mod->text_segment, mod->text_segment_size);
     if (mod->data_segment) /* Some modules lack data segments */
-	erts_free(ERTS_ALC_T_HIPE, mod->data_segment);
+	erts_free(ERTS_ALC_T_HIPE_LL, mod->data_segment);
 
-    erts_free(ERTS_ALC_T_HIPE, mod);
+    erts_free(ERTS_ALC_T_HIPE_LL, mod);
 }
diff --git a/erts/emulator/hipe/hipe_process.h b/erts/emulator/hipe/hipe_process.h
index a8d5972280..36b6ffc021 100644
--- a/erts/emulator/hipe/hipe_process.h
+++ b/erts/emulator/hipe/hipe_process.h
@@ -79,7 +79,7 @@ static __inline__ void hipe_init_process(struct hipe_process_state *p)
 static __inline__ void hipe_delete_process(struct hipe_process_state *p)
 {
     if (p->nstack)
-	erts_free(ERTS_ALC_T_HIPE, (void*)p->nstack);
+	erts_free(ERTS_ALC_T_HIPE_STK, (void*)p->nstack);
 }
 
 #ifdef ERTS_SMP
diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c
index b80e44bc37..d0f0407489 100644
--- a/erts/emulator/hipe/hipe_stack.c
+++ b/erts/emulator/hipe/hipe_stack.c
@@ -46,7 +46,7 @@ struct hipe_sdesc_table hipe_sdesc_table;
 static struct hipe_sdesc **alloc_bucket(unsigned int size)
 {
     unsigned long nbytes = size * sizeof(struct hipe_sdesc*);
-    struct hipe_sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes);
+    struct hipe_sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE_LL, nbytes);
     sys_memzero(bucket, nbytes);
     return bucket;
 }
@@ -75,7 +75,7 @@ static void hipe_grow_sdesc_table(void)
 	    b = next;
 	}
     }
-    erts_free(ERTS_ALC_T_HIPE, old_bucket);
+    erts_free(ERTS_ALC_T_HIPE_LL, old_bucket);
 }
 
 struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc *sdesc)
@@ -121,7 +121,7 @@ void hipe_destruct_sdesc(struct hipe_sdesc *sdesc)
         free_me = ErtsContainerStruct(sdesc, struct hipe_sdesc_with_exnra, sdesc);
     else
         free_me = sdesc;
-    erts_free(ERTS_ALC_T_HIPE, free_me);
+    erts_free(ERTS_ALC_T_HIPE_LL, free_me);
 }
 
 void hipe_init_sdesc_table(struct hipe_sdesc *sdesc)
@@ -199,7 +199,7 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
 	 ? offsetof(struct hipe_sdesc_with_exnra, sdesc.livebits)
 	 : offsetof(struct hipe_sdesc, livebits))
 	+ livebitswords * sizeof(int);
-    p = erts_alloc(ERTS_ALC_T_HIPE, sdescbytes);
+    p = erts_alloc(ERTS_ALC_T_HIPE_LL, sdescbytes);
     /* If we have an exception handler use the
        special sdesc_with_exnra structure. */
     if (exnra) {
diff --git a/erts/emulator/hipe/hipe_x86_signal.c b/erts/emulator/hipe/hipe_x86_signal.c
index 1a34ce786c..b24b9148a2 100644
--- a/erts/emulator/hipe/hipe_x86_signal.c
+++ b/erts/emulator/hipe/hipe_x86_signal.c
@@ -267,7 +267,7 @@ void hipe_thread_signal_init(void)
 {
     /* Stack don't really need to be cache aligned.
        We use it to suppress false leak report from valgrind */
-    hipe_sigaltstack(erts_alloc_permanent_cache_aligned(ERTS_ALC_T_HIPE, SIGSTKSZ));
+    hipe_sigaltstack(erts_alloc_permanent_cache_aligned(ERTS_ALC_T_HIPE_LL, SIGSTKSZ));
 }
 #endif
 
-- 
cgit v1.2.3


From a6ccff7ba9659cfac6d94f49b2aa39faf13e7e7d Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 17 Mar 2017 17:59:03 +0100
Subject: Fix scheduler_SUITE:update_cpu_info test-case

---
 erts/emulator/test/scheduler_SUITE.erl | 78 +++++++++++++++++++++++++++-------
 1 file changed, 62 insertions(+), 16 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl
index b178dede5b..885298ce34 100644
--- a/erts/emulator/test/scheduler_SUITE.erl
+++ b/erts/emulator/test/scheduler_SUITE.erl
@@ -807,21 +807,31 @@ update_cpu_info(Config) when is_list(Config) ->
 			      %% Nothing much to test; just a smoke test
 			      ok;
 			  Onln0 ->
-			      %% unset least significant bit
-			      Aff = (OldAff band (OldAff - 1)),
-			      set_affinity_mask(Aff),
-			      Onln1 = Avail - 1,
-			      case adjust_schedulers_online() of
-					{Onln0, Onln1} ->
-					    Onln1 = erlang:system_info(schedulers_online),
-					    receive after 500 -> ok end,
-					    io:format("TEST - Affinity mask: ~p - Schedulers online: ~p - Scheduler bindings: ~p~n",
-							    [Aff, Onln1, erlang:system_info(scheduler_bindings)]),
-					    unchanged = adjust_schedulers_online(),
-					    ok;
-					Fail ->
-					    ct:fail(Fail)
-				    end
+			      Cpus = bits_in_mask(OldAff),
+			      RmCpus = case Cpus > Onln0 of
+					   true -> Cpus - Onln0 + 1;
+					   false -> Onln0 - Cpus + 1
+				       end,
+			      Onln1 = Cpus - RmCpus,
+			      case Onln1 > 0 of
+				  false ->
+				      %% Nothing much to test; just a smoke test
+				      ok;
+				  true ->
+				      Aff = restrict_affinity_mask(OldAff, RmCpus),
+				      set_affinity_mask(Aff),
+				      case adjust_schedulers_online() of
+					  {Onln0, Onln1} ->
+					      Onln1 = erlang:system_info(schedulers_online),
+					      receive after 500 -> ok end,
+					      io:format("TEST - Affinity mask: ~p - Schedulers online: ~p - Scheduler bindings: ~p~n",
+							[Aff, Onln1, erlang:system_info(scheduler_bindings)]),
+					      unchanged = adjust_schedulers_online(),
+					      ok;
+					  Fail ->
+					      ct:fail(Fail)
+				      end
+			      end
 		      end
 		  after
 		      set_affinity_mask(OldAff),
@@ -835,13 +845,49 @@ update_cpu_info(Config) when is_list(Config) ->
 		  end
 	  end.
 
+bits_in_mask(Mask) ->
+    bits_in_mask(Mask, 0, 0).
+
+bits_in_mask(0, Shift, N) ->
+    N;
+bits_in_mask(Mask, Shift, N) ->
+    case Mask band (1 bsl Shift) of
+	0 -> bits_in_mask(Mask, Shift+1, N);
+	_ -> bits_in_mask(Mask band (bnot (1 bsl Shift)),
+			  Shift+1, N+1)
+    end.
+
+restrict_affinity_mask(Mask, N) ->
+    try
+	restrict_affinity_mask(Mask, 0, N)
+    catch
+	throw : Reason ->
+	    exit({Reason, Mask, N})
+    end.
+
+restrict_affinity_mask(Mask, _Shift, 0) ->
+    Mask;
+restrict_affinity_mask(0, _Shift, _N) ->
+    throw(overresticted_affinity_mask);
+restrict_affinity_mask(Mask, Shift, N) ->
+    case Mask band (1 bsl Shift) of
+	0 -> restrict_affinity_mask(Mask, Shift+1, N);
+	_ -> restrict_affinity_mask(Mask band (bnot (1 bsl Shift)),
+				    Shift+1, N-1)
+    end.
+
 adjust_schedulers_online() ->
     case erlang:system_info(update_cpu_info) of
 	unchanged ->
 	    unchanged;
 	changed ->
 	    Avail = erlang:system_info(logical_processors_available),
-	    {erlang:system_flag(schedulers_online, Avail), Avail}
+	    Scheds = erlang:system_info(schedulers),
+	    SOnln = case Avail > Scheds of
+	    	    	 true -> Scheds;
+			 false -> Avail
+		    end,
+	    {erlang:system_flag(schedulers_online, SOnln), SOnln}
     end.
 
 read_affinity(Data) ->
-- 
cgit v1.2.3


From 303f4781e485f66e1eb6c5db77ff9523e5f69b7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Mon, 20 Mar 2017 11:49:15 +0100
Subject: erts: Fix erl_async include files for no-threads

---
 erts/emulator/beam/erl_async.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_async.h b/erts/emulator/beam/erl_async.h
index 473c7686e5..c884a5040d 100644
--- a/erts/emulator/beam/erl_async.h
+++ b/erts/emulator/beam/erl_async.h
@@ -27,7 +27,6 @@ extern int erts_async_max_threads;
 #define ERTS_ASYNC_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */
 extern int erts_async_thread_suggested_stack_size;
 
-#ifdef USE_THREADS
 
 #ifdef ERTS_SMP
 /*
@@ -47,6 +46,10 @@ extern int erts_async_thread_suggested_stack_size;
 #  define ERTS_USE_ASYNC_READY_Q 0
 #endif
 
+#ifndef USE_THREADS
+#  undef ERTS_USE_ASYNC_READY_Q
+#  define ERTS_USE_ASYNC_READY_Q 0
+#endif /* !USE_THREADS */
 #if ERTS_USE_ASYNC_READY_Q
 int erts_check_async_ready(void *);
 int erts_async_ready_clean(void *, void *);
@@ -58,10 +61,7 @@ void *erts_get_async_ready_queue(Uint sched_id);
 #endif
 #endif /* ERTS_USE_ASYNC_READY_Q */
 
-#endif /* USE_THREADS */
-
 void erts_init_async(void);
 void erts_exit_flush_async(void);
 
-
 #endif /* ERL_ASYNC_H__ */
-- 
cgit v1.2.3


From 92616102d59ae7c51c39176b264b84b32cc21f14 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 20 Mar 2017 17:31:45 +0100
Subject: Fix type of MAKE_HASH

---
 erts/emulator/beam/erl_process_dict.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 8311fde025..7cfdf20341 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -54,9 +54,9 @@
 #define HASH_RANGE(PDict) ((PDict)->usedSlots)
 
 #define MAKE_HASH(Term)                                \
-    ((is_small(Term)) ? unsigned_val(Term) :           \
+    ((is_small(Term)) ? (Uint32) unsigned_val(Term) :  \
      ((is_atom(Term)) ?                                \
-      atom_val(Term) :								   \
+      (Uint32) atom_val(Term) :                        \
       make_internal_hash(Term)))
 
 #define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm))
-- 
cgit v1.2.3


From 950d9f4276f70856bbd64d4815b4c0c0cf87366d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 20 Mar 2017 19:21:25 +0100
Subject: erts: Make generated files depend on Makefile

in order to regenerate them if config has changed.

ERL-379

Maybe a better solution would have been to run 'clean' target
or stop and ask user to run make clean.
---
 erts/emulator/Makefile.in | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 18fd7f320b..7ea0111e59 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -656,6 +656,10 @@ generate: $(TTF_DIR)/GENERATED $(PRELOAD_SRC)
 
 $(TTF_DIR)/GENERATED: $(GENERATE)
 	$(gen_verbose)echo $? >$(TTF_DIR)/GENERATED
+
+# Regenerate if Makefile has changed
+$(GENERATE): $(TARGET)/Makefile
+
 endif
 
 $(TARGET)/erlang_dtrace.h: beam/erlang_dtrace.d
-- 
cgit v1.2.3


From f92d862c1af230a873c2b2d30fea575ff3dafd56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 21 Mar 2017 14:37:03 +0100
Subject: erts: Make sigterm signal safe for non-smp beam

The signal handler will now schedule a sigterm message instead of sending
the message in the signal handler. The signal handler must refrain from
memory allocations and thus the event is scheduled.
---
 erts/emulator/beam/sys.h                |  6 ++++++
 erts/emulator/sys/common/erl_check_io.c | 10 ++++++++++
 erts/emulator/sys/unix/sys.c            | 20 +++++++++++++++++---
 3 files changed, 33 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index d49edad6dc..dd4f05686b 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -481,6 +481,12 @@ extern volatile int erts_break_requested;
 void erts_do_break_handling(void);
 #endif
 
+#if !defined(ERTS_SMP) && !defined(__WIN32__)
+extern volatile Uint erts_signal_sigterm;
+#define ERTS_SIGNAL_SIGTERM erts_signal_sigterm
+void erts_handle_signal_sigterm(void);
+#endif
+
 #ifdef ERTS_WANT_GOT_SIGUSR1
 #  ifndef UNIX
 #    define ERTS_GOT_SIGUSR1 0
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 44a77f3ea5..8f2f1f9521 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1617,6 +1617,11 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 	erts_do_break_handling();
 #endif
 
+#ifdef ERTS_SIGNAL_SIGTERM
+    if (ERTS_SIGNAL_SIGTERM)
+	erts_handle_signal_sigterm();
+#endif
+
     /* Figure out timeout value */
     timeout_time = (do_wait
 		    ? erts_check_next_timeout_time(esdp)
@@ -1654,6 +1659,11 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 	erts_do_break_handling();
 #endif
 
+#ifdef ERTS_SIGNAL_SIGTERM
+    if (ERTS_SIGNAL_SIGTERM)
+	erts_handle_signal_sigterm();
+#endif
+
     if (poll_ret != 0) {
 	erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0);
 	forget_removed(&pollset);
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 6459fa064b..55411f83d0 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -148,11 +148,18 @@ volatile int erts_break_requested = 0;
 #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1)
 #define ERTS_UNSET_BREAK_REQUESTED (erts_break_requested = 0)
 #endif
+
+#ifndef ERTS_SMP
+volatile Uint erts_signal_sigterm = 0;
+#define ERTS_SET_SIGNAL_SIGTERM (erts_signal_sigterm = 1)
+#define ERTS_CLEAR_SIGNAL_SIGTERM (erts_signal_sigterm = 0)
+#endif
+
 /* set early so the break handler has access to initial mode */
 static struct termios initial_tty_mode;
 static int replace_intr = 0;
 /* assume yes initially, ttsl_init will clear it */
-int using_oldshell = 1; 
+int using_oldshell = 1;
 
 #ifdef ERTS_ENABLE_KERNEL_POLL
 
@@ -684,11 +691,11 @@ static RETSIGTYPE request_stop(int signum)
 #ifdef ERTS_SMP
     smp_sig_notify('S');
 #else
-    stop_requested();
+    ERTS_SET_SIGNAL_SIGTERM;
+    ERTS_CHK_IO_AS_INTR();
 #endif
 }
 
-
 static ERTS_INLINE void
 sigusr1_exit(void)
 {
@@ -958,6 +965,13 @@ void erts_do_break_handling(void)
     erts_smp_thr_progress_unblock();
 }
 
+#ifdef ERTS_SIGNAL_SIGTERM
+void erts_handle_signal_sigterm(void) {
+    ERTS_CLEAR_SIGNAL_SIGTERM;
+    stop_requested();
+}
+#endif
+
 /* Fills in the systems representation of the jam/beam process identifier.
 ** The Pid is put in STRING representation in the supplied buffer,
 ** no interpretatione of this should be done by the rest of the
-- 
cgit v1.2.3


From 0dec7ddb341b2b7d2643f86dd9e9497b3f5268a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 21 Mar 2017 10:48:15 +0100
Subject: erts: Don't allocate memory during signal handling

Allocations and message sending are now scheduled by a signal,
via a signal state bitmap, instead of doing it directly in the signal handler.
---
 erts/emulator/beam/sys.h                |  6 ++++
 erts/emulator/sys/common/erl_check_io.c | 14 ++++++++
 erts/emulator/sys/unix/sys.c            | 64 +++++++++++++++++++++++++++++++--
 3 files changed, 82 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index c6ea8049c3..144dd60d21 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -487,6 +487,12 @@ extern volatile int erts_break_requested;
 void erts_do_break_handling(void);
 #endif
 
+#if !defined(ERTS_SMP) && !defined(__WIN32__)
+extern volatile Uint erts_signal_state;
+#define ERTS_SIGNAL_STATE erts_signal_state
+void erts_handle_signal_state(void);
+#endif
+
 #ifdef ERTS_SMP
 extern erts_smp_atomic32_t erts_writing_erl_crash_dump;
 extern erts_tsd_key_t erts_is_crash_dumping_key;
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 1ef9fa2a05..00c70268df 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -2169,6 +2169,12 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 	erts_do_break_handling();
 #endif
 
+#ifdef ERTS_SIGNAL_STATE /* ifndef ERTS_SMP */
+    if (ERTS_SIGNAL_STATE) {
+        erts_handle_signal_state();
+    }
+#endif
+
     /* Figure out timeout value */
     timeout_time = (do_wait
 		    ? erts_check_next_timeout_time(esdp)
@@ -2207,6 +2213,14 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 	erts_do_break_handling();
 #endif
 
+
+#ifdef ERTS_SIGNAL_STATE /* ifndef ERTS_SMP */
+    if (ERTS_SIGNAL_STATE) {
+        erts_handle_signal_state();
+    }
+#endif
+
+
     if (poll_ret != 0) {
 	erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0);
 	forget_removed(&pollset);
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index e836f9bcc8..b48b3f8804 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -139,6 +139,28 @@ volatile int erts_break_requested = 0;
 #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1)
 #define ERTS_UNSET_BREAK_REQUESTED (erts_break_requested = 0)
 #endif
+
+#ifndef ERTS_SMP
+static Eterm signalstate_sigterm[] = {
+    am_sigint,   /* 0 */
+    am_sighup,   /* 1 */
+    am_sigquit,  /* 2 */
+    am_sigabrt,  /* 3 */
+    am_sigalrm,  /* 4 */
+    am_sigterm,  /* 5 */
+    am_sigusr1,  /* 6 */
+    am_sigusr2,  /* 7 */
+    am_sigchld,  /* 8 */
+    am_sigstop,  /* 9 */
+    am_sigtstp   /* 10 */
+};
+
+volatile Uint erts_signal_state = 0;
+#define ERTS_SET_SIGNAL_STATE(S) (erts_signal_state |= signum_to_signalstate(S))
+#define ERTS_CLEAR_SIGNAL_STATE (erts_signal_state = 0)
+static ERTS_INLINE Uint signum_to_signalstate(int signum);
+#endif
+
 /* set early so the break handler has access to initial mode */
 static struct termios initial_tty_mode;
 static int replace_intr = 0;
@@ -770,13 +792,34 @@ signum_to_signalterm(int signum)
     }
 }
 
+#ifndef ERTS_SMP
+static ERTS_INLINE Uint
+signum_to_signalstate(int signum)
+{
+    switch (signum) {
+    case SIGINT:  return (1 <<  0);
+    case SIGHUP:  return (1 <<  1);
+    case SIGQUIT: return (1 <<  2);
+    case SIGABRT: return (1 <<  3);
+    case SIGALRM: return (1 <<  4);
+    case SIGTERM: return (1 <<  5);
+    case SIGUSR1: return (1 <<  6);
+    case SIGUSR2: return (1 <<  7);
+    case SIGCHLD: return (1 <<  8);
+    case SIGSTOP: return (1 <<  9);
+    case SIGTSTP: return (1 << 10);
+    default:      return 0;
+    }
+}
+#endif
+
 static RETSIGTYPE generic_signal_handler(int signum)
 {
 #ifdef ERTS_SMP
     smp_sig_notify(signum);
 #else
-    Eterm signal = signum_to_signalterm(signum);
-    signal_notify_requested(signal);
+    ERTS_SET_SIGNAL_STATE(signum);
+    ERTS_CHK_IO_AS_INTR(); /* Make sure we don't sleep in poll */
 #endif
 }
 
@@ -962,6 +1005,23 @@ void erts_do_break_handling(void)
     erts_smp_thr_progress_unblock();
 }
 
+#ifdef ERTS_SIGNAL_STATE
+void erts_handle_signal_state(void) {
+    Uint signal_state = ERTS_SIGNAL_STATE;
+    Uint i = 0;
+
+    ERTS_CLEAR_SIGNAL_STATE;
+
+    while (signal_state) {
+        if (signal_state & 0x1) {
+            signal_notify_requested(signalstate_sigterm[i]);
+        }
+        i++;
+        signal_state = signal_state >> 1;
+    }
+}
+#endif
+
 /* Fills in the systems representation of the jam/beam process identifier.
 ** The Pid is put in STRING representation in the supplied buffer,
 ** no interpretatione of this should be done by the rest of the
-- 
cgit v1.2.3


From 25ee16c99376650e61f3af2fd6fdc1fc3e36a6de Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 23 Feb 2017 12:08:12 +0100
Subject: list_to_ref/1

---
 erts/emulator/beam/bif.c   | 126 +++++++++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/bif.tab |   1 +
 2 files changed, 127 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 614bedab12..2d37f977c0 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4254,6 +4254,132 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1)
     BIF_ERROR(BIF_P, BADARG);
 }
 
+BIF_RETTYPE list_to_ref_1(BIF_ALIST_1)
+{
+    /*
+     * A valid reference is on the format
+     * "#Ref<N.X.Y.Z>" where N, X, Y, and Z are
+     * 32-bit integers (i.e., max 10 characters).
+     */
+    Eterm *hp;
+    Eterm res;
+    Uint32 refn[ERTS_MAX_REF_NUMBERS];
+    int n = 0;
+    Uint ints[1 + ERTS_MAX_REF_NUMBERS] = {0};
+    char* cp;
+    Sint i;
+    DistEntry *dep = NULL;
+    char buf[5 /* #Ref< */
+             + (1 + ERTS_MAX_REF_NUMBERS)*(10 + 1) /* N.X.Y.Z> */
+             + 1 /* \0 */];
+
+    /* walk down the list and create a C string */
+    if ((i = intlist_to_buf(BIF_ARG_1, buf, sizeof(buf)-1)) < 0)
+	goto bad;
+
+    buf[i] = '\0';		/* null terminal */
+
+    cp = &buf[0];
+    if (*cp++ != '#') goto bad;
+    if (*cp++ != 'R') goto bad;
+    if (*cp++ != 'e') goto bad;
+    if (*cp++ != 'f') goto bad;
+    if (*cp++ != '<') goto bad;
+
+    for (i = 0; i < sizeof(ints)/sizeof(Uint); i++) {
+        if (*cp < '0' || *cp > '9') goto bad;
+
+        while (*cp >= '0' && *cp <= '9') {
+            ints[i] = 10*ints[i] + (*cp - '0');
+            cp++;
+        }
+
+        n++;
+        if (ints[i] > ~((Uint32) 0)) goto bad;
+        if (*cp == '>') break;
+        if (*cp++ != '.') goto bad;
+    }
+
+    if (*cp++ != '>') goto bad;
+    if (*cp != '\0') goto bad;
+
+    if (n < 2) goto bad;
+
+    for (n = 0; i > 0; i--)
+        refn[n++] = (Uint32) ints[i];
+
+    ASSERT(n <= ERTS_MAX_REF_NUMBERS);
+
+    dep = erts_channel_no_to_dist_entry(ints[0]);
+
+    if (!dep)
+	goto bad;
+
+    if(dep == erts_this_dist_entry) {
+        ErtsMagicBinary *mb;
+        Uint32 sid;
+        if (refn[0] > MAX_REFERENCE) goto bad;
+        if (n != ERTS_REF_NUMBERS) goto bad;
+        sid = erts_get_ref_numbers_thr_id(refn);
+        if (sid > erts_no_schedulers) goto bad;
+        mb = erts_magic_ref_lookup_bin(refn);
+        if (mb) {
+            hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
+            res = erts_mk_magic_ref(&hp, &BIF_P->off_heap,
+                                    (Binary *) mb);
+        }
+        else {
+            hp = HAlloc(BIF_P, ERTS_REF_THING_SIZE);
+            write_ref_thing(hp, refn[0], refn[1], refn[2]);
+            res = make_internal_ref(hp);
+        }
+    }
+    else {
+      ExternalThing *etp;
+      ErlNode *enp;
+      Uint hsz;
+      int j;
+
+      if (is_nil(dep->cid))
+	  goto bad;
+      
+      enp = erts_find_or_insert_node(dep->sysname, dep->creation);
+      ASSERT(enp != erts_this_node);
+
+      hsz = EXTERNAL_THING_HEAD_SIZE;
+#if defined(ARCH_64)
+      hsz += n/2 + 1;
+#else
+      hsz += n;
+#endif
+
+      etp = (ExternalThing *) HAlloc(BIF_P, hsz);
+      etp->header = make_external_ref_header(n/2);
+      etp->next = BIF_P->off_heap.first;
+      etp->node = enp;
+      i = 0;
+#if defined(ARCH_64)
+      etp->data.ui32[i] = n;
+#endif
+      for (j = 0; j < n; j++) {
+          etp->data.ui32[i] = refn[j];
+          i++;
+      }
+
+      BIF_P->off_heap.first = (struct erl_off_heap_header*) etp;
+      res = make_external_ref(etp);
+    }
+
+    erts_deref_dist_entry(dep);
+    BIF_RET(res);
+
+ bad:
+    if (dep)
+	erts_deref_dist_entry(dep);
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+
 /**********************************************************************/
 
 BIF_RETTYPE group_leader_0(BIF_ALIST_0)
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 08c8c3adbd..56da8703c2 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -88,6 +88,7 @@ bif erlang:list_to_binary/1
 bif erlang:list_to_float/1
 bif erlang:list_to_integer/1
 bif erlang:list_to_pid/1
+bif erlang:list_to_ref/1
 bif erlang:list_to_tuple/1
 bif erlang:loaded/0
 bif erlang:localtime/0
-- 
cgit v1.2.3


From 8b2a0f9940ffde4533d029a87ee30e290552db61 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 27 Feb 2017 16:18:41 +0100
Subject: Fix node_container_SUITE

---
 erts/emulator/beam/erl_node_tables.c | 41 ++++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 14 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index e2072fe30f..89b627aaf5 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -850,14 +850,15 @@ static Eterm AM_timer;
 static Eterm AM_delayed_delete_timer;
 
 static void setup_reference_table(void);
-static Eterm reference_table_term(Uint **hpp, Uint *szp);
+static Eterm reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp);
 static void delete_reference_table(void);
 
-#if BIG_UINT_HEAP_SIZE > 3 /* 2-tuple */
-#define ID_HEAP_SIZE BIG_UINT_HEAP_SIZE
-#else
-#define ID_HEAP_SIZE 3 /* 2-tuple */
-#endif
+#undef ERTS_MAX__
+#define ERTS_MAX__(A, B) ((A) > (B) ? (A) : (B))
+
+#define ID_HEAP_SIZE \
+    ERTS_MAX__(ERTS_MAGIC_REF_THING_SIZE, \
+               ERTS_MAX__(BIG_UINT_HEAP_SIZE, 3))
 
 typedef struct node_referrer_ {
     struct node_referrer_ *next;
@@ -870,6 +871,7 @@ typedef struct node_referrer_ {
     int system_ref;
     Eterm id;
     Uint id_heap[ID_HEAP_SIZE];
+    ErlOffHeap off_heap;
 } NodeReferrer;
 
 typedef struct {
@@ -942,7 +944,7 @@ erts_get_node_and_dist_references(struct process *proc)
 
     /* Get term size */
     size = 0;
-    (void) reference_table_term(NULL, &size);
+    (void) reference_table_term(NULL, NULL, &size);
 
     hp = HAlloc(proc, size);
 #ifdef DEBUG
@@ -951,7 +953,7 @@ erts_get_node_and_dist_references(struct process *proc)
 #endif
 
     /* Write term */
-    res = reference_table_term(&hp, NULL);
+    res = reference_table_term(&hp, &proc->off_heap, NULL);
 
     ASSERT(endp == hp);
 
@@ -1048,13 +1050,14 @@ insert_node_referrer(ReferredNode *referred_node, int type, Eterm id)
 	nrp = (NodeReferrer *) erts_alloc(ERTS_ALC_T_NC_TMP,
 					  sizeof(NodeReferrer));
 	nrp->next = referred_node->referrers;
+        ERTS_INIT_OFF_HEAP(&nrp->off_heap);
 	referred_node->referrers = nrp;
 	if(IS_CONST(id))
 	    nrp->id = id;
 	else {
 	    Uint *hp = &nrp->id_heap[0];
-	    ASSERT(is_big(id) || is_tuple(id));
-	    nrp->id = copy_struct(id, size_object(id), &hp, NULL);
+	    ASSERT(is_big(id) || is_tuple(id) || is_internal_magic_ref(id));
+	    nrp->id = copy_struct(id, size_object(id), &hp, &nrp->off_heap);
 	}
 	nrp->heap_ref = 0;
 	nrp->link_ref = 0;
@@ -1211,10 +1214,20 @@ insert_links2(ErtsLink *lnk, Eterm id)
 static void
 insert_ets_table(DbTable *tab, void *unused)
 {
+    ErlOffHeap off_heap;
+    Eterm heap[ERTS_MAGIC_REF_THING_SIZE];
     struct insert_offheap2_arg a;
     a.type = ETS_REF;
-    a.id = tab->common.id;
+    if (tab->common.status & DB_NAMED_TABLE)
+        a.id = tab->common.the_name;
+    else {
+        Eterm *hp = &heap[0];
+        ERTS_INIT_OFF_HEAP(&off_heap);
+        a.id = erts_mk_magic_ref(&hp, &off_heap, tab->common.btid);
+    }
     erts_db_foreach_offheap(tab, insert_offheap2, (void *) &a);
+    if (is_not_atom(a.id))
+        erts_cleanup_offheap(&off_heap);
 }
 
 static void
@@ -1518,7 +1531,7 @@ setup_reference_table(void)
  */
 
 static Eterm
-reference_table_term(Uint **hpp, Uint *szp)
+reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp)
 {
 #undef  MK_2TUP
 #undef  MK_3TUP
@@ -1573,12 +1586,11 @@ reference_table_term(Uint **hpp, Uint *szp)
 
 	    nrid = nrp->id;
 	    if (!IS_CONST(nrp->id)) {
-
 		Uint nrid_sz = size_object(nrp->id);
 		if (szp)
 		    *szp += nrid_sz;
 		if (hpp)
-		    nrid = copy_struct(nrp->id, nrid_sz, hpp, NULL);
+		    nrid = copy_struct(nrp->id, nrid_sz, hpp, ohp);
 	    }
 
 	    if (is_internal_pid(nrid) || nrid == am_error_logger) {
@@ -1713,6 +1725,7 @@ delete_reference_table(void)
 	NodeReferrer *tnrp;
 	nrp = referred_nodes[i].referrers;
 	while(nrp) {
+            erts_cleanup_offheap(&nrp->off_heap);
 	    tnrp = nrp;
 	    nrp = nrp->next;
 	    erts_free(ERTS_ALC_T_NC_TMP, (void *) tnrp);
-- 
cgit v1.2.3


From 8d98992b9fa3059554c6cd5ff45ee3b47805fa1c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 15 Feb 2017 19:19:02 +0100
Subject: erts: Add ERTS_RBT_YIELD_STAT_INIT to erl_rbtree

for dynamic initialization of yield state.
---
 erts/emulator/beam/erl_rbtree.h | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_rbtree.h b/erts/emulator/beam/erl_rbtree.h
index 5fefaea978..a10b1b1081 100644
--- a/erts/emulator/beam/erl_rbtree.h
+++ b/erts/emulator/beam/erl_rbtree.h
@@ -105,7 +105,10 @@
  * <ERTS_RBT_PREFIX>_rbt_yield_state_t.
  *
  * The yield state should be statically initialized by
- * ERTS_RBT_YIELD_STAT_INITER.
+ * ERTS_RBT_YIELD_STAT_INITER
+ *
+ * or dynamically initialized with
+ * ERTS_RBT_YIELD_STAT_INIT(<ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate)
  *
  *
  * The following API functions are implemented if corresponding
@@ -422,6 +425,13 @@
 #ifndef ERTS_RBT_YIELD_STAT_INITER
 #  define ERTS_RBT_YIELD_STAT_INITER {NULL, 0}
 #endif
+#ifndef ERTS_RBT_YIELD_STAT_INIT
+#  define ERTS_RBT_YIELD_STAT_INIT(YS) \
+    do {                               \
+        (YS)->x = NULL;                \
+        (YS)->up = 0;                  \
+    } while (0)
+#endif
 
 #define ERTS_RBT_CONCAT_MACRO_VALUES___(X, Y) \
     X ## Y
-- 
cgit v1.2.3


From 832091f4dbf27c8bdc572383cbdeba5307d8e41c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 15 Feb 2017 19:23:28 +0100
Subject: erts: Correct erl_rbtree comments about yielding

true is yielding, false is done.

and correct return value for unused foreach_ordered__
---
 erts/emulator/beam/erl_rbtree.h | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_rbtree.h b/erts/emulator/beam/erl_rbtree.h
index a10b1b1081..ebde5caca4 100644
--- a/erts/emulator/beam/erl_rbtree.h
+++ b/erts/emulator/beam/erl_rbtree.h
@@ -181,8 +181,8 @@
  *         Operate by calling the operator 'op' on each element.
  *         Order is undefined.
  *
- *         Yield when 'ylimit' elements has been processed. Zero is
- *         returned when yielding, and a non-zero value is returned when
+ *         Yield when 'ylimit' elements has been processed. True is
+ *         returned when yielding, and false is returned when
  *         the whole tree has been processed. The tree should not be
  *         modified until all of it has been processed.
  *
@@ -198,8 +198,8 @@
  *         Order is undefined. Each element should be destroyed
  *         by 'op'.
  *
- *         Yield when 'ylimit' elements has been processed. Zero is
- *         returned when yielding, and a non-zero value is returned when
+ *         Yield when 'ylimit' elements has been processed. True is
+ *         returned when yielding, and false is returned when
  *         the whole tree has been processed.
  *
  *         'arg' is passed as argument to 'op'.
@@ -231,8 +231,8 @@
  *         Operate by calling the operator 'op' on each element from
  *         smallest towards larger elements.
  *
- *         Yield when 'ylimit' elements has been processed. Zero is
- *         returned when yielding, and a non-zero value is returned when
+ *         Yield when 'ylimit' elements has been processed. True is
+ *         returned when yielding, and false is returned when
  *         the whole tree has been processed. The tree should not be
  *         modified until all of it has been processed.
  *
@@ -247,8 +247,8 @@
  *         Operate by calling the operator 'op' on each element from
  *         largest towards smaller elements.
  *
- *         Yield when 'ylimit' elements has been processed. Zero is
- *         returned when yielding, and a non-zero value is returned when
+ *         Yield when 'ylimit' elements has been processed. True is
+ *         returned when yielding, and false is returned when
  *         the whole tree has been processed. The tree should not be
  *         modified until all of it has been processed.
  *
@@ -299,8 +299,8 @@
  *         Note that elements are often destroyed in another order
  *         than the order that the elements are operated on.
  *
- *         Yield when 'ylimit' elements has been processed. Zero is
- *         returned when yielding, and a non-zero value is returned when
+ *         Yield when 'ylimit' elements has been processed. True is
+ *         returned when yielding, and false is returned when
  *         the whole tree has been processed. The tree should not be
  *         modified until all of it has been processed.
  *
@@ -321,8 +321,8 @@
  *         Note that elements are often destroyed in another order
  *         than the order that the elements are operated on.
  *
- *         Yield when 'ylimit' elements has been processed. Zero is
- *         returned when yielding, and a non-zero value is returned when
+ *         Yield when 'ylimit' elements has been processed. True is
+ *         returned when yielding, and false is returned when
  *         the whole tree has been processed. The tree should not be
  *         modified until all of it has been processed.
  *
@@ -1374,7 +1374,7 @@ ERTS_RBT_FUNC__(foreach_ordered__)(ERTS_RBT_T **root,
 		    ystate->x = NULL;
 		    ystate->up = 0;
 		}
-		return 1; /* Done */
+		return 0; /* Done */
 	    }
 	    x = p;
 	}
-- 
cgit v1.2.3


From b174a38c5f09859d3714fe5a81773aa5f2d19417 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 21 Feb 2017 16:07:50 +0100
Subject: erts: Replace meta_pid_to{_fixed}_tab with linked lists

from process psd through all owned/fixed tables.

As meta_pid_to{_fixed}_tab maps to slot in meta_main_tab
which is planned for destruction.

In this commit we no longer seize table lock while
freeing the table (free_table_cont) as it's not needed
and makes the code a bit simpler. Any concurrent operation
on the table will only access lock, owner and status
and then bail out.
---
 erts/emulator/beam/erl_bif_info.c   |    8 -
 erts/emulator/beam/erl_db.c         | 1030 ++++++++++++++++-------------------
 erts/emulator/beam/erl_db.h         |    4 +-
 erts/emulator/beam/erl_db_hash.c    |  152 +-----
 erts/emulator/beam/erl_db_hash.h    |   10 -
 erts/emulator/beam/erl_db_util.h    |   20 +-
 erts/emulator/beam/erl_lock_check.c |    6 +-
 erts/emulator/beam/erl_process.c    |   38 +-
 erts/emulator/beam/erl_process.h    |   16 +-
 erts/emulator/beam/erl_rbtree.h     |   28 +-
 10 files changed, 545 insertions(+), 767 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 06a73ffea5..025f99330a 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3634,10 +3634,6 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    BIF_RET(TUPLE2(hp, make_small((Uint) words),
                            erts_ets_hash_sizeof_ext_segtab()));
 	}
-        else if (ERTS_IS_ATOM_STR("DbTable_meta", BIF_ARG_1)) {
-            /* Used by ets_SUITE (stdlib) */
-            BIF_RET(erts_ets_get_meta_state(BIF_P));
-        }
 	else if (ERTS_IS_ATOM_STR("check_io_debug", BIF_ARG_1)) {
 	    /* Used by driver_SUITE (emulator) */
 	    Uint sz, *szp;
@@ -4398,10 +4394,6 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
             }
             BIF_RET(am_ok);
         }
-        else if (ERTS_IS_ATOM_STR("DbTable_meta", BIF_ARG_1)) {
-            /* Used by ets_SUITE (stdlib) */
-            BIF_RET(erts_ets_restore_meta_state(BIF_P, BIF_ARG_2));
-        }
         else if (ERTS_IS_ATOM_STR("make", BIF_ARG_1)) {
             if (ERTS_IS_ATOM_STR("magic_ref", BIF_ARG_2)) {
                 Binary *bin = erts_create_magic_binary(0, empty_magic_ref_destructor);
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 8c4c76e187..24de21cc24 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -19,9 +19,7 @@
  */
 
 /*
- * This file contains the bif interface functions and
- * the handling of the "meta tables" ie the tables of 
- * db tables.
+ * This file contains the 'ets' bif interface functions.
  */
 
 /*
@@ -75,6 +73,113 @@ enum DbIterSafety {
 
 #define DID_TRAP(P,Ret) (!is_value(Ret) && ((P)->freason == TRAP))
 
+/*
+ * "fixed_tabs": list of all fixed tables for a process
+ */
+#ifdef DEBUG
+static int fixed_tabs_find(DbFixation* first, DbFixation* fix);
+#endif
+
+static void fixed_tabs_insert(Process* p, DbFixation* fix)
+{
+    DbFixation* first = erts_psd_get(p, ERTS_PSD_ETS_FIXED_TABLES);
+
+    if (!first) {
+        fix->tabs.next = fix->tabs.prev = fix;
+        erts_psd_set(p, ERTS_PSD_ETS_FIXED_TABLES, fix);
+    }
+    else {
+        ASSERT(!fixed_tabs_find(first, fix));
+        fix->tabs.prev = first->tabs.prev;
+        fix->tabs.next = first;
+        fix->tabs.prev->tabs.next = fix;
+        first->tabs.prev = fix;
+    }
+}
+
+static void fixed_tabs_delete(Process *p, DbFixation* fix)
+{
+    if (fix->tabs.next == fix) {
+        DbFixation* old;
+        ASSERT(fix->tabs.prev == fix);
+        old = erts_psd_set(p, ERTS_PSD_ETS_FIXED_TABLES, NULL);
+        ASSERT(old == fix); (void)old;
+    }
+    else {
+        DbFixation *first = (DbFixation*) erts_psd_get(p, ERTS_PSD_ETS_FIXED_TABLES);
+
+        ASSERT(fixed_tabs_find(first, fix));
+        fix->tabs.prev->tabs.next = fix->tabs.next;
+        fix->tabs.next->tabs.prev = fix->tabs.prev;
+
+        if (fix == first)
+            erts_psd_set(p, ERTS_PSD_ETS_FIXED_TABLES, fix->tabs.next);
+    }
+}
+
+#ifdef DEBUG
+static int fixed_tabs_find(DbFixation* first, DbFixation* fix)
+{
+    DbFixation* p;
+
+    if (!first) {
+        first = (DbFixation*) erts_psd_get(fix->procs.p, ERTS_PSD_ETS_FIXED_TABLES);
+    }
+    p = first;
+    do {
+        if (p == fix)
+            return 1;
+        ASSERT(p->procs.p == fix->procs.p);
+        ASSERT(p->tabs.next->tabs.prev == p);
+        p = p->tabs.next;
+    } while (p != first);
+    return 0;
+}
+#endif
+
+
+/*
+ * fixing_procs: tree of all processes fixating a table
+ */
+#define ERTS_RBT_PREFIX fixing_procs
+#define ERTS_RBT_T DbFixation
+#define ERTS_RBT_KEY_T Process*
+#define ERTS_RBT_FLAGS_T int
+#define ERTS_RBT_INIT_EMPTY_TNODE(T)                    \
+    do {						\
+	(T)->procs.parent = NULL;			\
+	(T)->procs.right = NULL;				\
+	(T)->procs.left = NULL;				\
+    } while (0)
+#define ERTS_RBT_IS_RED(T)        ((T)->procs.is_red)
+#define ERTS_RBT_SET_RED(T)       ((T)->procs.is_red = 1)
+#define ERTS_RBT_IS_BLACK(T)      (!(T)->procs.is_red)
+#define ERTS_RBT_SET_BLACK(T)     ((T)->procs.is_red = 0)
+#define ERTS_RBT_GET_FLAGS(T)     ((T)->procs.is_red)
+#define ERTS_RBT_SET_FLAGS(T, F)  ((T)->procs.is_red = (F))
+#define ERTS_RBT_GET_PARENT(T)    ((T)->procs.parent)
+#define ERTS_RBT_SET_PARENT(T, P) ((T)->procs.parent = (P))
+#define ERTS_RBT_GET_RIGHT(T)     ((T)->procs.right)
+#define ERTS_RBT_SET_RIGHT(T, R)  ((T)->procs.right = (R))
+#define ERTS_RBT_GET_LEFT(T)      ((T)->procs.left)
+#define ERTS_RBT_SET_LEFT(T, L)   ((T)->procs.left = (L))
+#define ERTS_RBT_GET_KEY(T)       ((T)->procs.p)
+#define ERTS_RBT_IS_LT(KX, KY)    ((KX) < (KY))
+#define ERTS_RBT_IS_EQ(KX, KY)    ((KX) == (KY))
+
+#define ERTS_RBT_WANT_INSERT
+#define ERTS_RBT_WANT_LOOKUP
+#define ERTS_RBT_WANT_DELETE
+#define ERTS_RBT_WANT_FOREACH
+#define ERTS_RBT_WANT_FOREACH_DESTROY
+#ifdef DEBUG
+# define ERTS_RBT_WANT_LOOKUP
+#endif
+#define ERTS_RBT_UNDEF
+
+#include "erl_rbtree.h"
+
+
 
 /* 
 ** The main meta table, containing all ets tables.
@@ -148,6 +253,12 @@ make_btid(DbTable *tb)
     erts_refc_inc(&btid->refc, 1);
 }
 
+static ERTS_INLINE DbTable* btid2tab(Binary* btid)
+{
+    erts_smp_atomic_t *tbref = erts_smp_binary_to_magic_indirection(btid);
+    return (DbTable *) erts_smp_atomic_read_nob(tbref);
+}
+
 static DbTable *
 tid2tab(Eterm tid)
 {
@@ -272,8 +383,6 @@ int user_requested_db_max_tabs;
 int erts_ets_realloc_always_moves;
 int erts_ets_always_compress;
 static int db_max_tabs;
-static DbTable *meta_pid_to_tab; /* Pid mapped to owned tables */
-static DbTable *meta_pid_to_fixed_tab; /* Pid mapped to fixed tables */
 static Eterm ms_delete_all;
 static Eterm ms_delete_all_buff[8]; /* To compare with for deletion 
 				       of all objects */
@@ -286,12 +395,9 @@ static void fix_table_locked(Process* p, DbTable* tb);
 static void unfix_table_locked(Process* p,  DbTable* tb, db_lock_kind_t* kind);
 static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data);
 static void free_heir_data(DbTable*);
-static void free_fixations_locked(DbTable *tb);
+static void free_fixations_locked(Process* p, DbTable *tb);
 
-static int free_table_cont(Process *p,
-			   DbTable *tb,
-			   int first,
-			   int clean_meta_tab);
+static int free_table_cont(Process *p, DbTable *tb);
 static void print_table(fmtfn_t to, void *to_arg, int show,  DbTable* tb);
 static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1);
 static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1);
@@ -328,15 +434,6 @@ free_dbtable(void *vtb)
 	}
 	erts_fprintf(stderr, "ets: free_dbtable(%T) deleted!!!\r\n",
 		     tb->common.id);
-
-	erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_tab common.memory_size = %ld\n",
-		     erts_smp_atomic_read_nob(&meta_pid_to_tab->common.memory_size));
-	print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_tab);
-
-
-	erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_fixed_tab common.memory_size = %ld\n",
-		     erts_smp_atomic_read_nob(&meta_pid_to_fixed_tab->common.memory_size));
-	print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_fixed_tab);
 #endif
 #ifdef ERTS_SMP
 	erts_smp_rwmtx_destroy(&tb->common.rwlock);
@@ -462,6 +559,62 @@ delete_sched_table(Process *c_p, DbTable *tb)
         erts_schedule_misc_aux_work((int) sid, scheduled_remove_sched_table, tb);
 }
 
+static ERTS_INLINE void
+save_owned_table(Process *c_p, DbTable *tb)
+{
+    DbTable *first;
+
+    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
+
+    first = (DbTable*) erts_psd_get(c_p, ERTS_PSD_ETS_OWNED_TABLES);
+
+    erts_smp_refc_inc(&tb->common.refc, 1);
+
+    if (!first) {
+        tb->common.owned.next = tb->common.owned.prev = tb;
+        erts_psd_set(c_p, ERTS_PSD_ETS_OWNED_TABLES, tb);
+    }
+    else {
+        tb->common.owned.prev = first->common.owned.prev;
+        tb->common.owned.next = first;
+        tb->common.owned.prev->common.owned.next = tb;
+        first->common.owned.prev = tb;
+    }
+    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
+}
+
+static ERTS_INLINE void
+delete_owned_table(Process *p, DbTable *tb)
+{
+    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+    if (tb->common.owned.next == tb) {
+        DbTable* old;
+        ASSERT(tb->common.owned.prev == tb);
+        old = erts_psd_set(p, ERTS_PSD_ETS_OWNED_TABLES, NULL);
+        ASSERT(old == tb); (void)old;
+    }
+    else {
+        DbTable *first = (DbTable*) erts_psd_get(p, ERTS_PSD_ETS_OWNED_TABLES);
+#ifdef DEBUG
+        DbTable *tmp = first;
+        do {
+            if (tmp == tb) break;
+            tmp = tmp->common.owned.next;
+        } while (tmp != first);
+        ASSERT(tmp == tb);
+#endif
+        tb->common.owned.prev->common.owned.next = tb->common.owned.next;
+        tb->common.owned.next->common.owned.prev = tb->common.owned.prev;
+
+        if (tb == first)
+            erts_psd_set(p, ERTS_PSD_ETS_OWNED_TABLES, tb->common.owned.next);
+    }
+    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+
+    table_dec_refc(tb, 1);
+}
+
+
 static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock,
 				     char *rwname, char* fixname)
 {
@@ -483,7 +636,6 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock,
 static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind)
 {
 #ifdef ERTS_SMP
-    ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab);
     if (tb->common.type & DB_FINE_LOCKED) {
 	if (kind == LCK_WRITE) {	   
 	    erts_smp_rwmtx_rwlock(&tb->common.rwlock);
@@ -516,8 +668,6 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
      * to follow the tb pointer!
      */
 #ifdef ERTS_SMP
-    ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab);
-
     if (tb->common.type & DB_FINE_LOCKED) {
 	if (kind == LCK_WRITE) {
 	    ASSERT(tb->common.is_thread_safe);
@@ -543,20 +693,6 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
 #endif
 }
 
-
-static ERTS_INLINE void db_meta_lock(DbTable* tb, db_lock_kind_t kind)
-{
-    ASSERT(tb == meta_pid_to_tab || tb == meta_pid_to_fixed_tab);
-    ASSERT(kind != LCK_WRITE);
-    /* As long as we only lock for READ we don't have to lock at all. */
-}
-
-static ERTS_INLINE void db_meta_unlock(DbTable* tb, db_lock_kind_t kind)
-{
-    ASSERT(tb == meta_pid_to_tab || tb == meta_pid_to_fixed_tab);
-    ASSERT(kind != LCK_WRITE);
-}
-
 static ERTS_INLINE
 DbTable* db_get_table_aux(Process *p,
 			  Eterm id,
@@ -1544,7 +1680,6 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 #ifdef DEBUG
     int cret;
 #endif
-    DeclareTmpHeap(meta_tuple,3,BIF_P);
     DbTableMethod* meth;
     erts_smp_rwmtx_t *mmtl;
 
@@ -1701,7 +1836,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 
     erts_smp_atomic_init_nob(&tb->common.nitems, 0);
 
-    tb->common.fixations = NULL;
+    tb->common.fixing_procs = NULL;
     tb->common.compress = is_compressed;
 
 #ifdef DEBUG
@@ -1774,32 +1909,15 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     }
     
     BIF_P->flags |= F_USING_DB; /* So we can remove tb if p dies */
+    save_owned_table(BIF_P, tb);
 
 #ifdef HARDDEBUG
     erts_fprintf(stderr,
 		"ets:new(%T,%T)=%T; Process: %T, initial: %T:%T/%bpu\n",
 		 BIF_ARG_1, BIF_ARG_2, ret, BIF_P->common.id,
 		 BIF_P->u.initial[0], BIF_P->u.initial[1], BIF_P->u.initial[2]);
-	erts_fprintf(stderr, "ets: new: meta_pid_to_tab common.memory_size = %ld\n",
-		     erts_smp_atomic_read_nob(&meta_pid_to_tab->common.memory_size));
-	erts_fprintf(stderr, "ets: new: meta_pid_to_fixed_tab common.memory_size = %ld\n",
-		     erts_smp_atomic_read_nob(&meta_pid_to_fixed_tab->common.memory_size));
 #endif
 
-    UseTmpHeap(3,BIF_P);
-
-    db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
-    if (db_put_hash(meta_pid_to_tab,
-		    TUPLE2(meta_tuple,
-			   BIF_P->common.id,
-			   make_small(slot)),
-		    0) != DB_ERROR_NONE) {
-	erts_exit(ERTS_ERROR_EXIT,"Could not update ets metadata.");
-    }
-    db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
-
-    UnUseTmpHeap(3,BIF_P);
-
     BIF_RET(ret);
 }
 
@@ -1929,7 +2047,6 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
     tb->common.status |= DB_DELETE;
 
     if (tb->common.owner != BIF_P->common.id) {
-	DeclareTmpHeap(meta_tuple,3,BIF_P);
 
 	/*
 	 * The table is being deleted by a process other than its owner.
@@ -1937,22 +2054,19 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 	 * current process will be killed (e.g. by an EXIT signal), we will
 	 * now transfer the ownership to the current process.
 	 */
-	UseTmpHeap(3,BIF_P);
-	db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
-	db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner,
-			    make_small(tb->common.slot));
-
-	BIF_P->flags |= F_USING_DB;
-	tb->common.owner = BIF_P->common.id;
-
-	db_put_hash(meta_pid_to_tab,
-		    TUPLE2(meta_tuple,
-			   BIF_P->common.id,
-			   make_small(tb->common.slot)),
-		    0);
-	db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
-	UnUseTmpHeap(3,BIF_P);
-    }    
+
+        Process *rp = erts_proc_lookup_raw(tb->common.owner);
+        /*
+         * Process 'rp' might be exiting, but our table lock prevents it
+         * from terminating as it cannot complete erts_db_process_exiting().
+         */
+        ASSERT(!(ERTS_PSFLG_FREE & erts_smp_atomic32_read_nob(&rp->state)));
+
+        delete_owned_table(rp, tb);
+        BIF_P->flags |= F_USING_DB;
+        tb->common.owner = BIF_P->common.id;
+        save_owned_table(BIF_P, tb);
+    }
 
     tid_clear(BIF_P, tb);
 
@@ -1978,10 +2092,10 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
     free_heir_data(tb);
     tb->common.heir = am_none;
 
-    free_fixations_locked(tb);
-
-    trap = free_table_cont(BIF_P, tb, 1, 1);
+    free_fixations_locked(BIF_P, tb);
     db_unlock(tb, LCK_WRITE);
+
+    trap = free_table_cont(BIF_P, tb);
     if (trap) {
 	/*
 	 * Package the DbTable* pointer into a bignum so that it can be safely
@@ -2006,7 +2120,6 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
 {
     Process* to_proc = NULL;
     ErtsProcLocks to_locks = ERTS_PROC_LOCK_MAIN;
-    DeclareTmpHeap(buf,5,BIF_P);
     Eterm to_pid = BIF_ARG_2;
     Eterm from_pid;
     DbTable* tb = NULL;
@@ -2028,18 +2141,10 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
 	goto badarg;  /* or should we be idempotent? return false maybe */
     }
 
-    UseTmpHeap(5,BIF_P);
-    db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
-    db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner,
-			make_small(tb->common.slot));
-
+    delete_owned_table(BIF_P, tb);
     to_proc->flags |= F_USING_DB;
     tb->common.owner = to_pid;
-
-    db_put_hash(meta_pid_to_tab,
-		TUPLE2(buf,to_pid,make_small(tb->common.slot)),
-		0);
-    db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
+    save_owned_table(to_proc, tb);
 
     db_unlock(tb,LCK_WRITE);
     send_ets_transfer_message(BIF_P, to_proc, &to_locks,
@@ -3273,7 +3378,6 @@ int erts_ets_rwmtx_spin_count = -1;
 
 void init_db(ErtsDbSpinCount db_spin_count)
 {
-    DbTable init_tb;
     int i;
     Eterm *hp;
     unsigned bits;
@@ -3379,72 +3483,6 @@ void init_db(ErtsDbSpinCount db_spin_count)
     db_initialize_hash();
     db_initialize_tree();
 
-    /*TT*/
-    /* Create meta table invertion. */
-    erts_smp_atomic_init_nob(&init_tb.common.memory_size, 0);
-    meta_pid_to_tab = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE,
-					       &init_tb,
-					       sizeof(DbTable));
-    erts_smp_atomic_init_nob(&meta_pid_to_tab->common.memory_size,
-			     erts_smp_atomic_read_nob(&init_tb.common.memory_size));
-
-    meta_pid_to_tab->common.id = NIL;
-    meta_pid_to_tab->common.btid = NULL;
-    meta_pid_to_tab->common.the_name = am_true;
-    meta_pid_to_tab->common.status = (DB_NORMAL | DB_BAG | DB_PUBLIC | DB_FINE_LOCKED);
-#ifdef ERTS_SMP
-    meta_pid_to_tab->common.type
-	= meta_pid_to_tab->common.status & ERTS_ETS_TABLE_TYPES;
-    /* Note, 'type' is *read only* from now on... */
-    meta_pid_to_tab->common.is_thread_safe = 0;
-#endif
-    meta_pid_to_tab->common.keypos = 1;
-    meta_pid_to_tab->common.owner  = NIL;
-    erts_smp_atomic_init_nob(&meta_pid_to_tab->common.nitems, 0);
-    meta_pid_to_tab->common.slot   = -1;
-    meta_pid_to_tab->common.meth   = &db_hash;
-    meta_pid_to_tab->common.compress = 0;
-
-    erts_smp_refc_init(&meta_pid_to_tab->common.fix_count, 0);
-    /* Neither rwlock or fixlock used
-    db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/
-
-    if (db_create_hash(NULL, meta_pid_to_tab) != DB_ERROR_NONE) {
-	erts_exit(ERTS_ERROR_EXIT,"Unable to create ets metadata tables.");
-    }
-
-    erts_smp_atomic_set_nob(&init_tb.common.memory_size, 0);
-    meta_pid_to_fixed_tab = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE,
-						     &init_tb,
-						     sizeof(DbTable));
-    erts_smp_atomic_init_nob(&meta_pid_to_fixed_tab->common.memory_size,
-			     erts_smp_atomic_read_nob(&init_tb.common.memory_size));
-
-    meta_pid_to_fixed_tab->common.id = NIL;
-    meta_pid_to_fixed_tab->common.btid = NULL;
-    meta_pid_to_fixed_tab->common.the_name = am_true;
-    meta_pid_to_fixed_tab->common.status = (DB_NORMAL | DB_BAG | DB_PUBLIC | DB_FINE_LOCKED);
-#ifdef ERTS_SMP
-    meta_pid_to_fixed_tab->common.type
-	= meta_pid_to_fixed_tab->common.status & ERTS_ETS_TABLE_TYPES;
-    /* Note, 'type' is *read only* from now on... */
-    meta_pid_to_fixed_tab->common.is_thread_safe = 0;
-#endif
-    meta_pid_to_fixed_tab->common.keypos = 1;
-    meta_pid_to_fixed_tab->common.owner  = NIL;
-    erts_smp_atomic_init_nob(&meta_pid_to_fixed_tab->common.nitems, 0);
-    meta_pid_to_fixed_tab->common.slot   = -1;
-    meta_pid_to_fixed_tab->common.meth   = &db_hash;
-    meta_pid_to_fixed_tab->common.compress = 0;
-
-    erts_smp_refc_init(&meta_pid_to_fixed_tab->common.fix_count, 0);
-    /* Neither rwlock or fixlock used
-    db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/
-
-    if (db_create_hash(NULL, meta_pid_to_fixed_tab) != DB_ERROR_NONE) {
-	erts_exit(ERTS_ERROR_EXIT,"Unable to create ets metadata tables.");
-    }
-
     /* Non visual BIF to trap to. */
     erts_init_trap_export(&ets_select_delete_continue_exp,
 			  am_ets, am_atom_put("delete_trap",11), 1,
@@ -3486,82 +3524,6 @@ erts_ets_sched_spec_data_init(ErtsSchedulerData *esdp)
 }
 
 
-#define ARRAY_CHUNK 100
-
-typedef enum {
-    ErtsDbProcCleanupProgressTables,
-    ErtsDbProcCleanupProgressFixations,
-    ErtsDbProcCleanupProgressDone,
-} ErtsDbProcCleanupProgress;
-
-typedef enum {
-    ErtsDbProcCleanupOpGetTables,
-    ErtsDbProcCleanupOpDeleteTables,
-    ErtsDbProcCleanupOpGetFixations,
-    ErtsDbProcCleanupOpDeleteFixations,
-    ErtsDbProcCleanupOpDone
-} ErtsDbProcCleanupOperation;
-
-typedef struct {
-    ErtsDbProcCleanupProgress progress;
-    ErtsDbProcCleanupOperation op;
-    struct {
-	Eterm arr[ARRAY_CHUNK];
-	int size;
-	int ix;
-	int clean_ix;
-    } slots;
-} ErtsDbProcCleanupState;
-
-
-static void
-proc_exit_cleanup_tables_meta_data(Eterm pid, ErtsDbProcCleanupState *state)
-{
-    ASSERT(state->slots.clean_ix <= state->slots.ix);
-    if (state->slots.clean_ix < state->slots.ix) {
-	db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
-	if (state->slots.size < ARRAY_CHUNK
-	    && state->slots.ix == state->slots.size) {
-	    Eterm dummy;
-	    db_erase_hash(meta_pid_to_tab,pid,&dummy);
-	}
-	else {
-	    int ix;
-	    /* Need to erase each explicitly */
-	    for (ix = state->slots.clean_ix; ix < state->slots.ix; ix++)
-		db_erase_bag_exact2(meta_pid_to_tab,
-				    pid,
-				    state->slots.arr[ix]);
-	}
-	db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
-	state->slots.clean_ix = state->slots.ix;
-    }
-}
-
-static void
-proc_exit_cleanup_fixations_meta_data(Eterm pid, ErtsDbProcCleanupState *state)
-{
-    ASSERT(state->slots.clean_ix <= state->slots.ix);
-    if (state->slots.clean_ix < state->slots.ix) {
-	db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
-	if (state->slots.size < ARRAY_CHUNK
-	    && state->slots.ix == state->slots.size) {
-	    Eterm dummy;
-	    db_erase_hash(meta_pid_to_fixed_tab,pid,&dummy);
-	}
-	else {
-	    int ix;
-	    /* Need to erase each explicitly */
-	    for (ix = state->slots.clean_ix; ix < state->slots.ix; ix++)
-		db_erase_bag_exact2(meta_pid_to_fixed_tab,
-				    pid,
-				    state->slots.arr[ix]);
-	}
-	db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
-	state->slots.clean_ix = state->slots.ix;
-    }
-}
-
 /* In: Table LCK_WRITE
 ** Return TRUE : ok, table not mine and NOT locked anymore.
 ** Return FALSE: failed, table still mine (LCK_WRITE)
@@ -3570,7 +3532,6 @@ static int give_away_to_heir(Process* p, DbTable* tb)
 {
     Process* to_proc;
     ErtsProcLocks to_locks = ERTS_PROC_LOCK_MAIN;
-    DeclareTmpHeap(buf,5,p);
     Eterm to_pid;
     UWord heir_data;
 
@@ -3614,19 +3575,12 @@ retry:
 	erts_smp_proc_unlock(to_proc, to_locks);
 	return 0; /* heir dead and pid reused, table still mine */
     }
-    UseTmpHeap(5,p);
-    db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
-    db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner,
-			make_small(tb->common.slot));
     
+    delete_owned_table(p, tb);
     to_proc->flags |= F_USING_DB;
     tb->common.owner = to_pid;
-    
-    db_put_hash(meta_pid_to_tab,
-		TUPLE2(buf,to_pid,make_small(tb->common.slot)),
-		0);
-    db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
-    UnUseTmpHeap(5,p);
+    save_owned_table(to_proc, tb);
+
     db_unlock(tb,LCK_WRITE);
     heir_data = tb->common.heir_data;
     if (!is_immed(heir_data)) {
@@ -3674,6 +3628,56 @@ send_ets_transfer_message(Process *c_p, Process *proc,
     erts_queue_message(proc, *locks, mp, msg, sender);
 }
 
+
+/* Auto-release fixation from exiting process */
+static void proc_cleanup_fixed_table(Process* p, DbFixation* fix)
+{
+    DbTable* tb = btid2tab(fix->tabs.btid);
+
+    ASSERT(fix->procs.p == p); (void)p;
+    if (tb) {
+	db_lock(tb, LCK_WRITE_REC);
+	if (!(tb->common.status & DB_DELETE)) {
+	    erts_aint_t diff;
+    #ifdef ERTS_SMP
+	    erts_smp_mtx_lock(&tb->common.fixlock);
+    #endif
+
+	    ASSERT(fixing_procs_rbt_lookup(tb->common.fixing_procs, p));
+
+	    diff = -((erts_aint_t) fix->counter);
+	    erts_smp_refc_add(&tb->common.fix_count,diff,0);
+	    fix->counter = 0;
+
+	    fixing_procs_rbt_delete(&tb->common.fixing_procs, fix);
+
+    #ifdef ERTS_SMP
+	    erts_smp_mtx_unlock(&tb->common.fixlock);
+    #endif
+	    if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)) {
+		db_unfix_table_hash(&(tb->hash));
+	    }
+
+	    ASSERT(sizeof(DbFixation) == ERTS_ALC_DBG_BLK_SZ(fix));
+	    ERTS_DB_ALC_MEM_UPDATE_(tb, sizeof(DbFixation), 0);
+	}
+	else {
+	    ASSERT(fix->counter == 0);
+	}
+	db_unlock(tb, LCK_WRITE_REC);
+    }
+    else {
+	ASSERT(fix->counter == 0);
+    }
+
+    if (erts_refc_dectest(&fix->tabs.btid->refc, 0) == 0) {
+	erts_bin_free(fix->tabs.btid);
+    }
+    erts_free(ERTS_ALC_T_DB_FIXATION, fix);
+    ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation));
+}
+
+
 /*
  * erts_db_process_exiting() is called when a process terminates.
  * It returns 0 when completely done, and !0 when it wants to
@@ -3687,277 +3691,157 @@ send_ets_transfer_message(Process *c_p, Process *proc,
 int
 erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 {
-    ErtsDbProcCleanupState *state = (ErtsDbProcCleanupState *) c_p->u.terminate;
+    typedef struct {
+        enum {
+            GET_OWNED_TABLE,
+            FREE_OWNED_TABLE,
+            UNFIX_TABLES,
+        }op;
+        DbTable *tb;
+    } CleanupState;
+    CleanupState *state = (CleanupState *) c_p->u.terminate;
     Eterm pid = c_p->common.id;
-    ErtsDbProcCleanupState default_state;
-    int ret;
+    CleanupState default_state;
 
     if (!state) {
 	state = &default_state;
-	state->progress = ErtsDbProcCleanupProgressTables;
-	state->op = ErtsDbProcCleanupOpGetTables;
+	state->op = GET_OWNED_TABLE;
+        state->tb = NULL;
     }
 
-    while (!0) {
+    do {
 	switch (state->op) {
-	case ErtsDbProcCleanupOpGetTables:
-	    state->slots.size = ARRAY_CHUNK;
-	    db_meta_lock(meta_pid_to_tab, LCK_READ);
-	    ret = db_get_element_array(meta_pid_to_tab,
-				       pid,
-				       2,
-				       state->slots.arr,
-				       &state->slots.size);
-	    db_meta_unlock(meta_pid_to_tab, LCK_READ);
-	    if (ret == DB_ERROR_BADKEY) {
-		/* Done with tables; now fixations */
-		state->progress = ErtsDbProcCleanupProgressFixations;
-		state->op = ErtsDbProcCleanupOpGetFixations;
-		break;
-	    } else if (ret != DB_ERROR_NONE) {
-		ERTS_DB_INTERNAL_ERROR("Inconsistent ets table metadata");
-	    }
-
-	    state->slots.ix = 0;
-	    state->slots.clean_ix = 0;
-	    state->op = ErtsDbProcCleanupOpDeleteTables;
-	    /* Fall through */
-
-	case ErtsDbProcCleanupOpDeleteTables:
-
-	    while (state->slots.ix < state->slots.size) {
-		DbTable *tb = NULL;
-		Sint ix = unsigned_val(state->slots.arr[state->slots.ix]);
-		erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix);
-		erts_smp_rwmtx_rlock(mmtl);
-		if (!IS_SLOT_FREE(ix)) {
-		    tb = GET_ANY_SLOT_TAB(ix);
-		    ASSERT(tb);
-		}
-		erts_smp_rwmtx_runlock(mmtl);
-		if (tb) {
-		    int do_yield;
-		    db_lock(tb, LCK_WRITE);
-		    /* Ownership may have changed since
-		       we looked up the table. */
-		    if (tb->common.owner != pid) {
-			do_yield = 0;
-			db_unlock(tb, LCK_WRITE);
-		    }
-		    else if (tb->common.heir != am_none
-			     && tb->common.heir != pid
-			     && give_away_to_heir(c_p, tb)) {
-			do_yield = 0;
-		    }
-		    else {
-			int first_call;
-#ifdef HARDDEBUG
-			erts_fprintf(stderr,
-				     "erts_db_process_exiting(); Table: %T, "
-				     "Process: %T\n",
-				     tb->common.id, pid);
-#endif
-			first_call = (tb->common.status & DB_DELETE) == 0;
-			if (first_call) {
-                            tid_clear(c_p, tb);
-			    /* Clear all access bits. */
-			    tb->common.status &= ~(DB_PROTECTED
-						   | DB_PUBLIC
-						   | DB_PRIVATE);
-			    tb->common.status |= DB_DELETE;
-
-			    if (is_atom(tb->common.id))
-				remove_named_tab(tb, 0);
-
-			    free_heir_data(tb);
-			    free_fixations_locked(tb);
-			}
-
-			do_yield = free_table_cont(c_p, tb, first_call, 0);
-			db_unlock(tb, LCK_WRITE);
-		    }		    
-		    if (do_yield)
-			goto yield;
-		}
-		state->slots.ix++;
-		if (ERTS_BIF_REDS_LEFT(c_p) <= 0)
-		    goto yield;
-	    }
+        case GET_OWNED_TABLE: {
+            DbTable* tb;
+            erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
+            tb = (DbTable*) erts_psd_get(c_p, ERTS_PSD_ETS_OWNED_TABLES);
+            erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
+
+            if (!tb) {
+                /* Done with owned tables; now fixations */
+                state->op = UNFIX_TABLES;
+                break;
+            }
 
-	    proc_exit_cleanup_tables_meta_data(pid, state);
-	    state->op = ErtsDbProcCleanupOpGetTables;
-	    break;
+            ASSERT(tb != state->tb);
+            state->tb = tb;
+            db_lock(tb, LCK_WRITE);
+            /*
+             *  Ownership may have changed since we looked up the table.
+             */
+            if (tb->common.owner != pid) {
+                db_unlock(tb, LCK_WRITE);
+                break;
+            }
+            if (tb->common.heir != am_none
+                && tb->common.heir != pid
+                && give_away_to_heir(c_p, tb)) {
+                break;
+            }
+            tid_clear(c_p, tb);
+            /* Clear all access bits. */
+            tb->common.status &= ~(DB_PROTECTED | DB_PUBLIC | DB_PRIVATE);
+            tb->common.status |= DB_DELETE;
+
+            if (is_atom(tb->common.id))
+                remove_named_tab(tb, 0);
+
+            free_heir_data(tb);
+            free_fixations_locked(c_p, tb);
+            db_unlock(tb, LCK_WRITE);
+            state->op = FREE_OWNED_TABLE;
+            /* fall through */
+        }
+        case FREE_OWNED_TABLE:
+            if (free_table_cont(c_p, state->tb))
+                goto yield;
 
-	case ErtsDbProcCleanupOpGetFixations:
-	    state->slots.size = ARRAY_CHUNK;
-	    db_meta_lock(meta_pid_to_fixed_tab, LCK_READ);
-	    ret = db_get_element_array(meta_pid_to_fixed_tab, 
-				       pid,
-				       2,
-				       state->slots.arr,
-				       &state->slots.size);
-	    db_meta_unlock(meta_pid_to_fixed_tab, LCK_READ);
-
-	    if (ret == DB_ERROR_BADKEY) {
-		/* Done */
-		state->progress = ErtsDbProcCleanupProgressDone;
-		state->op = ErtsDbProcCleanupOpDone;
-		break;
-	    } else if (ret != DB_ERROR_NONE) {
-		ERTS_DB_INTERNAL_ERROR("Inconsistent ets fix table metadata");
-	    }
+            state->op = GET_OWNED_TABLE;
 
-	    state->slots.ix = 0;
-	    state->slots.clean_ix = 0;
-	    state->op = ErtsDbProcCleanupOpDeleteFixations;
-	    /* Fall through */
+            break;
 
-	case ErtsDbProcCleanupOpDeleteFixations:
+	case UNFIX_TABLES: {
+	    DbFixation* fix;
 
-	    while (state->slots.ix < state->slots.size) {
-		DbTable *tb = NULL;
-		Sint ix = unsigned_val(state->slots.arr[state->slots.ix]);
-		erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix);
-		erts_smp_rwmtx_rlock(mmtl);
-		if (IS_SLOT_ALIVE(ix)) {
-		    tb = meta_main_tab[ix].u.tb;
-		    ASSERT(tb);
-		}
-		erts_smp_rwmtx_runlock(mmtl);
-		if (tb) {
-		    int reds = 0;
-
-		    db_lock(tb, LCK_WRITE_REC);
-		    if (!(tb->common.status & DB_DELETE)) {
-			DbFixation** pp;
-
-			#ifdef ERTS_SMP
-			erts_smp_mtx_lock(&tb->common.fixlock);
-			#endif
-			reds = 10;
-
-			for (pp = &tb->common.fixations; *pp != NULL;
-			      pp = &(*pp)->next) {
-			    if ((*pp)->pid == pid) {
-				DbFixation* fix = *pp;
-				erts_aint_t diff = -((erts_aint_t) fix->counter);
-				erts_smp_refc_add(&tb->common.fix_count,diff,0);
-				*pp = fix->next;
-				erts_db_free(ERTS_ALC_T_DB_FIXATION,
-					     tb, fix, sizeof(DbFixation));
-				ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation));
-				break;
-			    }
-			}
-			#ifdef ERTS_SMP
-			erts_smp_mtx_unlock(&tb->common.fixlock);
-			#endif
-			if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)) {
-			    db_unfix_table_hash(&(tb->hash));
-			    reds += 40;
-			}
-		    }
-		    db_unlock(tb, LCK_WRITE_REC);
-		    BUMP_REDS(c_p, reds);
-		}
-		state->slots.ix++;
-		if (ERTS_BIF_REDS_LEFT(c_p) <= 0)
-		    goto yield;
-	    }
+            fix = (DbFixation*) erts_psd_get(c_p, ERTS_PSD_ETS_FIXED_TABLES);
 
-	    proc_exit_cleanup_fixations_meta_data(pid, state);
-	    state->op = ErtsDbProcCleanupOpGetFixations;
-	    break;
+            if (!fix) {
+                /* Done */
 
-	case ErtsDbProcCleanupOpDone:
+                if (state != &default_state)
+                    erts_free(ERTS_ALC_T_DB_PROC_CLEANUP, state);
+                c_p->u.terminate = NULL;
+                return 0;
+            }
 
-	    if (state != &default_state)
-		erts_free(ERTS_ALC_T_DB_PROC_CLEANUP, state);
-	    c_p->u.terminate = NULL;
-	    return 0;
+            fixed_tabs_delete(c_p, fix);
+            proc_cleanup_fixed_table(c_p, fix);
 
+            BUMP_REDS(c_p, 10);
+            break;
+        }
 	default:
 	    ERTS_DB_INTERNAL_ERROR("Bad internal state");
-	}
-    }
-
- yield:
+        }
 
-    switch (state->progress) {
-    case ErtsDbProcCleanupProgressTables:
-	proc_exit_cleanup_tables_meta_data(pid, state);
-	break;
-    case ErtsDbProcCleanupProgressFixations:
-	proc_exit_cleanup_fixations_meta_data(pid, state);
-	break;
-    default:
-	break;
-    }
+    } while (ERTS_BIF_REDS_LEFT(c_p) > 0);
 
-    ASSERT(c_p->u.terminate == (void *) state
-	   || state == &default_state);
+ yield:
 
     if (state == &default_state) {
 	c_p->u.terminate = erts_alloc(ERTS_ALC_T_DB_PROC_CLEANUP,
-				      sizeof(ErtsDbProcCleanupState));
-	sys_memcpy(c_p->u.terminate,
-		   (void*) state,
-		   sizeof(ErtsDbProcCleanupState));
+				      sizeof(CleanupState));
+	sys_memcpy(c_p->u.terminate, (void*) state, sizeof(CleanupState));
     }
+    else
+        ASSERT(state == c_p->u.terminate);
 
     return !0;
 }
 
+
 /*  SMP note: table only need to be LCK_READ locked */
 static void fix_table_locked(Process* p, DbTable* tb)
 {
     DbFixation *fix;
-    DeclareTmpHeap(meta_tuple,3,p);
 
 #ifdef ERTS_SMP
     erts_smp_mtx_lock(&tb->common.fixlock);
 #endif
     erts_smp_refc_inc(&tb->common.fix_count,1);
-    fix = tb->common.fixations;
+    fix = tb->common.fixing_procs;
     if (fix == NULL) {
 	tb->common.time.monotonic
 	    = erts_get_monotonic_time(erts_proc_sched_data(p));
 	tb->common.time.offset = erts_get_time_offset();
     }
     else {
-	for (; fix != NULL; fix = fix->next) {
-	    if (fix->pid == p->common.id) {
-		++(fix->counter);
+	fix = fixing_procs_rbt_lookup(fix, p);
+	if (fix) {
+	    ASSERT(fixed_tabs_find(NULL, fix));
+	    ++(fix->counter);
+
 #ifdef ERTS_SMP
-		erts_smp_mtx_unlock(&tb->common.fixlock);
+	    erts_smp_mtx_unlock(&tb->common.fixlock);
 #endif
-		return;
-	    }
+	    return;
 	}
     }
     fix = (DbFixation *) erts_db_alloc(ERTS_ALC_T_DB_FIXATION,
 				       tb, sizeof(DbFixation));
     ERTS_ETS_MISC_MEM_ADD(sizeof(DbFixation));
-    fix->pid = p->common.id;
+    fix->tabs.btid = tb->common.btid;
+    erts_refc_inc(&fix->tabs.btid->refc, 2);
+    fix->procs.p = p;
     fix->counter = 1;
-    fix->next = tb->common.fixations;
-    tb->common.fixations = fix;
+    fixing_procs_rbt_insert(&tb->common.fixing_procs, fix);
+
 #ifdef ERTS_SMP
     erts_smp_mtx_unlock(&tb->common.fixlock);
 #endif
-    p->flags |= F_USING_DB;        
-    UseTmpHeap(3,p);
-    db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
-    if (db_put_hash(meta_pid_to_fixed_tab,
-		    TUPLE2(meta_tuple,
-			   p->common.id,
-			   make_small(tb->common.slot)),
-		    0) != DB_ERROR_NONE) {
-	UnUseTmpHeap(3,p);
-	erts_exit(ERTS_ERROR_EXIT,"Could not insert ets metadata in safe_fixtable.");
-    }	
-    UnUseTmpHeap(3,p);
-    db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
+    p->flags |= F_USING_DB;
+
+    fixed_tabs_insert(p, fix);
 }
 
 /* SMP note: May re-lock table 
@@ -3965,28 +3849,26 @@ static void fix_table_locked(Process* p, DbTable* tb)
 static void unfix_table_locked(Process* p,  DbTable* tb,
 			       db_lock_kind_t* kind_p)
 {
-    DbFixation** pp;
+    DbFixation* fix;
 
 #ifdef ERTS_SMP
     erts_smp_mtx_lock(&tb->common.fixlock);
 #endif
-    for (pp = &tb->common.fixations; *pp != NULL; pp = &(*pp)->next) {
-	if ((*pp)->pid == p->common.id) {
-	    DbFixation* fix = *pp;
-	    erts_smp_refc_dec(&tb->common.fix_count,0);
-	    --(fix->counter);
-	    ASSERT(fix->counter >= 0);
-	    if (fix->counter > 0) {
-		break;
-	    }
-	    *pp = fix->next;
+    fix = fixing_procs_rbt_lookup(tb->common.fixing_procs, p);
+
+    if (fix) {
+	erts_smp_refc_dec(&tb->common.fix_count,0);
+	--(fix->counter);
+	ASSERT(fix->counter >= 0);
+	if (fix->counter == 0) {
+	    fixing_procs_rbt_delete(&tb->common.fixing_procs, fix);
 #ifdef ERTS_SMP
 	    erts_smp_mtx_unlock(&tb->common.fixlock);
 #endif
-	    db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
-	    db_erase_bag_exact2(meta_pid_to_fixed_tab,
-				p->common.id, make_small(tb->common.slot));
-	    db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
+	    fixed_tabs_delete(p, fix);
+
+	    erts_refc_dec(&fix->tabs.btid->refc, 1);
+
 	    erts_db_free(ERTS_ALC_T_DB_FIXATION,
 			 tb, (void *) fix, sizeof(DbFixation));
 	    ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation));
@@ -4013,29 +3895,83 @@ unlocked:
     }
 }
 
-/* Assume that tb is WRITE locked */
-static void free_fixations_locked(DbTable *tb)
+struct free_fixations_ctx
 {
-    DbFixation *fix;
-    DbFixation *next_fix;
-
-    fix = tb->common.fixations;
-    while (fix != NULL) {
-	erts_aint_t diff = -((erts_aint_t) fix->counter);
-	erts_smp_refc_add(&tb->common.fix_count,diff,0);
-	next_fix = fix->next;
-	db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
-	db_erase_bag_exact2(meta_pid_to_fixed_tab,
-			    fix->pid,
-			    make_small(tb->common.slot));
-	db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
-	erts_db_free(ERTS_ALC_T_DB_FIXATION,
-		     tb, (void *) fix, sizeof(DbFixation));
-	ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation));
-
-	fix = next_fix;
-    }
-    tb->common.fixations = NULL;
+    Process* p;
+    DbTable* tb;
+};
+
+static void free_fixations_op(DbFixation* fix, void* vctx)
+{
+    struct free_fixations_ctx* ctx = (struct free_fixations_ctx*) vctx;
+    erts_aint_t diff;
+#ifdef DEBUG
+    DbTable* dbg_tb = btid2tab(fix->tabs.btid);
+#endif
+
+    ASSERT(!dbg_tb || dbg_tb == ctx->tb);
+    ASSERT(fix->counter > 0);
+    ASSERT(ctx->tb->common.status & DB_DELETE);
+
+    diff = -((erts_aint_t) fix->counter);
+    erts_smp_refc_add(&ctx->tb->common.fix_count, diff, 0);
+
+#ifdef ERTS_SMP
+    if (fix->procs.p != ctx->p) { /* Fixated by other process */
+        fix->counter = 0;
+
+        /* Fake memory stats for table */
+        ASSERT(sizeof(DbFixation) == ERTS_ALC_DBG_BLK_SZ(fix));
+        ERTS_DB_ALC_MEM_UPDATE_(ctx->tb, sizeof(DbFixation), 0);
+
+        erts_schedule_ets_free_fixation(fix->procs.p->common.id, fix);
+        /*
+         * Either sys task is scheduled and erts_db_execute_free_fixation()
+         * will remove 'fix' or process will exit, drop sys task and
+         * proc_cleanup_fixed_table() will remove 'fix'.
+         */
+    }
+    else
+#endif
+    {
+        fixed_tabs_delete(fix->procs.p, fix);
+
+        if (erts_refc_dectest(&fix->tabs.btid->refc, 0) == 0) {
+            erts_bin_free(fix->tabs.btid);
+        }
+
+        erts_db_free(ERTS_ALC_T_DB_FIXATION,
+		     ctx->tb, (void *) fix, sizeof(DbFixation));
+        ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation));
+    }
+}
+
+#ifdef ERTS_SMP
+int erts_db_execute_free_fixation(Process* p, DbFixation* fix)
+{
+    ASSERT(fix->counter == 0);
+    fixed_tabs_delete(p, fix);
+
+    if (erts_refc_dectest(&fix->tabs.btid->refc, 0) == 0) {
+        erts_bin_free(fix->tabs.btid);
+    }
+    erts_free(ERTS_ALC_T_DB_FIXATION, fix);
+    ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation));
+    return 1;
+}
+#endif
+
+static void free_fixations_locked(Process* p, DbTable *tb)
+{
+    struct free_fixations_ctx ctx;
+
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rwlocked(&tb->common.rwlock));
+
+    ctx.p = p;
+    ctx.tb = tb;
+    fixing_procs_rbt_foreach_destroy(&tb->common.fixing_procs,
+                                     free_fixations_op, &ctx);
+    tb->common.fixing_procs = NULL;
 }
 
 static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data)
@@ -4107,9 +4043,7 @@ static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1)
 
     ASSERT(*ptr == make_pos_bignum_header(1));
 
-    db_lock(tb, LCK_WRITE);
-    trap = free_table_cont(p, tb, 0, 1);
-    db_unlock(tb, LCK_WRITE);
+    trap = free_table_cont(p, tb);
 
     if (trap) {
 	BIF_TRAP1(&ets_delete_continue_exp, p, cont);
@@ -4123,21 +4057,11 @@ static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1)
 /*
  * free_table_cont() returns 0 when done and !0 when more work is needed.
  */
-static int free_table_cont(Process *p,
-			   DbTable *tb,
-			   int first,
-			   int clean_meta_tab)
+static int free_table_cont(Process *p, DbTable *tb)
 {
     Eterm result;
     erts_smp_rwmtx_t *mmtl;
 
-#ifdef HARDDEBUG
-    if (!first) {
-	erts_fprintf(stderr,"ets: free_table_cont %T (continue)\r\n",
-		     tb->common.id);
-    }
-#endif
-
     result = tb->common.meth->db_free_table_continue(tb);
 
     if (result == 0) {
@@ -4157,27 +4081,36 @@ static int free_table_cont(Process *p,
 	/* Completely done - we will not get called again. */
 	mmtl = get_meta_main_tab_lock(tb->common.slot);
 #ifdef ERTS_SMP
-	if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) {
-	    erts_smp_rwmtx_rwunlock(&tb->common.rwlock);
-	    erts_smp_rwmtx_rwlock(mmtl);
-	    erts_smp_rwmtx_rwlock(&tb->common.rwlock);
-	}
+        erts_smp_rwmtx_rwlock(mmtl);
 #endif
 	free_slot(tb->common.slot);
 	erts_smp_rwmtx_rwunlock(mmtl);
 
-	if (clean_meta_tab) {
-	    db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
-	    db_erase_bag_exact2(meta_pid_to_tab,tb->common.owner,
-				make_small(tb->common.slot));
-	    db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
-	}
+        delete_owned_table(p, tb);
         table_dec_refc(tb, 0);
 	BUMP_REDS(p, 100);
 	return 0;
     }
 }
 
+struct fixing_procs_info_ctx
+{
+    Process* p;
+    Eterm list;
+};
+
+static void fixing_procs_info_op(DbFixation* fix, void* vctx)
+{
+    struct fixing_procs_info_ctx* ctx = (struct fixing_procs_info_ctx*) vctx;
+    Eterm* hp;
+    Eterm tpl;
+
+    hp = HAllocX(ctx->p, 5, 100);
+    tpl = TUPLE2(hp, fix->procs.p->common.id, make_small(fix->counter));
+    hp += 3;
+    ctx->list = CONS(hp, tpl, ctx->list);
+}
+
 static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 {
     Eterm ret = THE_NON_VALUE;
@@ -4249,9 +4182,9 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	if (IS_FIXED(tb)) {
 	    Uint need;
 	    Eterm *hp;
-	    Eterm tpl, lst;
-	    DbFixation *fix;
+	    Eterm time;
 	    Sint64 mtime;
+	    struct fixing_procs_info_ctx ctx;
 	    
 	    need = 3;
 	    if (use_monotonic) {
@@ -4264,19 +4197,15 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 		mtime = 0;
 		need += 4;
 	    }
-	    for (fix = tb->common.fixations; fix != NULL; fix = fix->next) {
-		need += 5;
-	    }
+	    ctx.p = p;
+	    ctx.list = NIL;
+	    fixing_procs_rbt_foreach(tb->common.fixing_procs,
+				     fixing_procs_info_op,
+				     &ctx);
+
 	    hp = HAlloc(p, need);
-	    lst = NIL;
-	    for (fix = tb->common.fixations; fix != NULL; fix = fix->next) {
-		tpl = TUPLE2(hp,fix->pid,make_small(fix->counter));
-		hp += 3;
-		lst = CONS(hp,tpl,lst);
-		hp += 2;
-	    }
 	    if (use_monotonic)
-		tpl = (IS_SSMALL(mtime)
+		time = (IS_SSMALL(mtime)
 		       ? make_small(mtime)
 		       : erts_sint64_to_big(mtime, &hp));
 	    else {
@@ -4284,10 +4213,10 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 		erts_make_timestamp_value(&ms, &s, &us,
 					  tb->common.time.monotonic,
 					  tb->common.time.offset);
-		tpl = TUPLE3(hp, make_small(ms), make_small(s), make_small(us));
+		time = TUPLE3(hp, make_small(ms), make_small(s), make_small(us));
 		hp += 4;
 	    }
-	    ret = TUPLE2(hp, tpl, lst);
+	    ret = TUPLE2(hp, time, ctx.list);
 	} else {
 	    ret = am_false;
 	}
@@ -4453,37 +4382,6 @@ erts_ets_colliding_names(Process* p, Eterm name, Uint cnt)
     return list;
 }
 
-/*
- * For testing only
- * Retreive meta table size state
- */
-Eterm erts_ets_get_meta_state(Process* p)
-{
-    Eterm* hp = HAlloc(p, 3);
-    return TUPLE2(hp,
-                  erts_ets_hash_get_memstate(p, &meta_pid_to_tab->hash),
-                  erts_ets_hash_get_memstate(p, &meta_pid_to_fixed_tab->hash));
-}
-/*
- * For testing only
- * Restore a previously retrieved meta table size state.
- * We do this to suppress failed memory checks
- * caused by the hysteresis of meta tables grow/shrink limits.
- */
-Eterm erts_ets_restore_meta_state(Process* p, Eterm meta_state)
-{
-    Eterm* tv;
-    Eterm* hp;
-    if (!is_tuple_arity(meta_state, 2))
-        return am_badarg;
-
-    tv = tuple_val(meta_state);
-    hp = HAlloc(p, 3);
-    return TUPLE2(hp,
-                  erts_ets_hash_restore_memstate(&meta_pid_to_tab->hash, tv[1]),
-                  erts_ets_hash_restore_memstate(&meta_pid_to_fixed_tab->hash, tv[2]));
-}
-
 #ifdef HARDDEBUG   /* Here comes some debug functions */
 
 void db_check_tables(void)
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index 1c53401590..cbf4b9e007 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -109,6 +109,7 @@ typedef enum {
 
 void init_db(ErtsDbSpinCount);
 int erts_db_process_exiting(Process *, ErtsProcLocks);
+int erts_db_execute_free_fixation(Process*, DbFixation*);
 void db_info(fmtfn_t, void *, int);
 void erts_db_foreach_table(void (*)(DbTable *, void *), void *);
 void erts_db_foreach_offheap(DbTable *,
@@ -125,9 +126,6 @@ extern Export ets_select_continue_exp;
 extern erts_smp_atomic_t erts_ets_misc_mem_size;
 
 Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt);
-Eterm erts_ets_get_meta_state(Process* p);
-Eterm erts_ets_restore_meta_state(Process* p, Eterm target_state);
-
 Uint erts_db_get_max_tabs(void);
 
 #endif /* ERL_DB_H__ */
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index bb29d56aa7..3afc58bb79 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -903,70 +903,6 @@ done:
     RUNLOCK_HASH(lck);
     return DB_ERROR_NONE;
 }
-
-int db_get_element_array(DbTable *tbl, 
-			 Eterm key,
-			 int ndex, 
-			 Eterm *ret,
-			 int *num_ret)
-{
-    DbTableHash *tb = &tbl->hash;
-    HashValue hval;
-    int ix;
-    HashDbTerm* b1;
-    int num = 0;
-    int retval;
-    erts_smp_rwmtx_t* lck;
-
-    ASSERT(!IS_FIXED(tbl)); /* no support for fixed tables here */
-
-    hval = MAKE_HASH(key);
-    lck = RLOCK_HASH(tb, hval);
-    ix = hash_to_ix(tb, hval);
-    b1 = BUCKET(tb, ix);
-
-    while(b1 != 0) {
-	if (has_live_key(tb,b1,key,hval)) {
-	    if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) {
-		HashDbTerm* b;
-		HashDbTerm* b2 = b1->next;
-
-		while(b2 != NULL && has_live_key(tb,b2,key,hval)) {
-		    if (ndex > arityval(b2->dbterm.tpl[0])) {
-			retval = DB_ERROR_BADITEM;
-			goto done;
-		    }
-		    b2 = b2->next;
-		}
-
-		b = b1;
-		while(b != b2) {
-		    if (num < *num_ret) {
-			ret[num++] = b->dbterm.tpl[ndex];
-		    } else {
-			retval = DB_ERROR_NONE;
-			goto done;
-		    }
-		    b = b->next;
-		}
-		*num_ret = num;
-	    }
-	    else {
-		ASSERT(*num_ret > 0);
-		ret[0] = b1->dbterm.tpl[ndex];
-		*num_ret = 1;
-	    }
-	    retval = DB_ERROR_NONE;
-	    goto done;
-	}
-	b1 = b1->next;
-    }
-    retval = DB_ERROR_BADKEY;
-done:
-    RUNLOCK_HASH(lck);
-    return retval;
-}
-    
     
 static int db_member_hash(DbTable *tbl, Eterm key, Eterm *ret)
 {
@@ -1058,54 +994,6 @@ done:
     return retval;
 }
 
-/*
- * Very internal interface, removes elements of arity two from 
- * BAG. Used for the PID meta table
- */
-int db_erase_bag_exact2(DbTable *tbl, Eterm key, Eterm value)
-{
-    DbTableHash *tb = &tbl->hash;
-    HashValue hval;
-    int ix;
-    HashDbTerm** bp;
-    HashDbTerm* b;
-    erts_smp_rwmtx_t* lck;
-    int found = 0;
-
-    hval = MAKE_HASH(key);
-    lck = WLOCK_HASH(tb,hval);
-    ix = hash_to_ix(tb, hval);
-    bp = &BUCKET(tb, ix);
-    b = *bp;
-
-    ASSERT(!IS_FIXED(tb));
-    ASSERT((tb->common.status & DB_BAG));
-    ASSERT(!tb->common.compress);
-
-    while(b != 0) {
-	if (has_live_key(tb,b,key,hval)) {
-	    found = 1;
-	    if ((arityval(b->dbterm.tpl[0]) == 2) && 
-		EQ(value, b->dbterm.tpl[2])) {
-		*bp = b->next;
-		free_term(tb, b);
-		erts_smp_atomic_dec_nob(&tb->common.nitems);
-		b = *bp;
-		break;
-	    }
-	} else if (found) {
-		break;
-	}
-	bp = &b->next;
-	b = b->next;
-    }
-    WUNLOCK_HASH(lck);
-    if (found) {
-	try_shrink(tb);
-    }
-    return DB_ERROR_NONE;
-}
-	
 /*
 ** NB, this is for the db_erase/2 bif.
 */
@@ -2207,7 +2095,7 @@ static int db_free_table_continue_hash(DbTable *tbl)
     DbTableHash *tb = &tbl->hash;
     int done;
     FixedDeletion* fixdel = (FixedDeletion*) erts_smp_atomic_read_acqb(&tb->fixdel);
-    ERTS_SMP_LC_ASSERT(IS_TAB_WLOCKED(tb));
+    ERTS_SMP_LC_ASSERT(IS_TAB_WLOCKED(tb) || (tb->common.status & DB_DELETE));
 
     done = 0;
     while (fixdel != NULL) {
@@ -3007,44 +2895,6 @@ Eterm erts_ets_hash_sizeof_ext_segtab(void)
 {
     return make_small(((SIZEOF_EXT_SEGTAB(0)-1) / sizeof(UWord)) + 1);
 }
-/* For testing only */
-Eterm erts_ets_hash_get_memstate(Process* p, DbTableHash* tb)
-{
-    Eterm seg_cnt;
-    while (!begin_resizing(tb))
-        /*spinn*/;
-
-    seg_cnt = make_small(SLOT_IX_TO_SEG_IX(tb->nslots));
-    done_resizing(tb);
-    return seg_cnt;
-}
-/* For testing only */
-Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate)
-{
-    int seg_cnt, target;
-
-    if (!is_small(memstate))
-        return make_small(__LINE__);
-
-    target = signed_val(memstate);
-    if (target < 1)
-        return make_small(__LINE__);
-    while (1) {
-        while (!begin_resizing(tb))
-            /*spin*/;
-        seg_cnt = SLOT_IX_TO_SEG_IX(tb->nslots);
-        done_resizing(tb);
-
-        if (target == seg_cnt)
-            return am_ok;
-        if (IS_FIXED(tb))
-            return make_small(__LINE__);
-        if (target < seg_cnt)
-            shrink(tb, 0);
-        else
-            grow(tb, INT_MAX);
-    }
-}
 
 #ifdef HARDDEBUG
 
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index 6d25c73549..cd02418370 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -88,14 +88,6 @@ int db_get_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret);
 
 int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret);
 
-int db_get_element_array(DbTable *tbl, 
-			 Eterm key,
-			 int ndex, 
-			 Eterm *ret,
-			 int *num_ret); 
-
-int db_erase_bag_exact2(DbTable *tbl, Eterm key, Eterm value);
-
 /* not yet in method table */
 int db_mark_all_deleted_hash(DbTable *tbl);
 
@@ -110,7 +102,5 @@ typedef struct {
 
 void db_calc_stats_hash(DbTableHash* tb, DbHashStats*);
 Eterm erts_ets_hash_sizeof_ext_segtab(void);
-Eterm erts_ets_hash_get_memstate(Process*, DbTableHash* tb);
-Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate);
 
 #endif /* _DB_HASH_H */
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 3c13035ec5..673ec80604 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -195,9 +195,20 @@ typedef struct db_table_method
 } DbTableMethod;
 
 typedef struct db_fixation {
-    Eterm pid;
+    /* Node in fixed_tabs list */
+    struct {
+        struct db_fixation *next, *prev;
+        Binary* btid;
+    } tabs;
+
+    /* Node in fixing_procs tree */
+    struct {
+        struct db_fixation *left, *right, *parent;
+        int is_red;
+        Process* p;
+    } procs;
+
     Uint counter;
-    struct db_fixation *next;
 } DbFixation;
 
 typedef struct {
@@ -217,9 +228,10 @@ typedef struct db_table_common {
     erts_smp_refc_t refc;     /* reference count of table struct */
     erts_smp_refc_t fix_count;/* fixation counter */
     DbTableList all;
+    DbTableList owned;
 #ifdef ERTS_SMP
     erts_smp_rwmtx_t rwlock;  /* rw lock on table */
-    erts_smp_mtx_t fixlock;   /* Protects fixations,megasec,sec,microsec */
+    erts_smp_mtx_t fixlock;   /* Protects fixing_procs and time */
     int is_thread_safe;       /* No fine locking inside table needed */
     Uint32 type;              /* table type, *read only* after creation */
 #endif
@@ -237,7 +249,7 @@ typedef struct db_table_common {
 	ErtsMonotonicTime monotonic;
 	ErtsMonotonicTime offset;
     } time;
-    DbFixation* fixations;    /* List of processes who have done safe_fixtable,
+    DbFixation* fixing_procs; /* Tree of processes who have done safe_fixtable,
                                  "local" fixations not included. */ 
     /* All 32-bit fields */
     Uint32 status;            /* bit masks defined  below */
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 6ff9aea5ab..7f7c06da8e 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -100,12 +100,12 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"dist_entry_links",			"address"		},
     {   "code_write_permission",                NULL                    },
     {	"purge_state",		      		NULL			},
-    {	"proc_status",				"pid"			},
-    {	"proc_trace",				"pid"			},
-    {   "ports_snapshot",                       NULL                    },
     {	"meta_name_tab",	         	"address"		},
     {	"meta_main_tab_slot",			"address"		},
     {	"db_tab",				"address"		},
+    {	"proc_status",				"pid"			},
+    {	"proc_trace",				"pid"			},
+    {   "ports_snapshot",                       NULL                    },
     {	"db_tab_fix",				"address"		},
     {	"meta_main_tab_main",			NULL 			},
     {	"db_hash_slot",				"address"		},
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b816ba59fb..88fae30845 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -449,7 +449,8 @@ typedef enum {
     ERTS_PSTT_CPC,	/* Check Process Code */
     ERTS_PSTT_CLA,	/* Copy Literal Area */
     ERTS_PSTT_COHMQ,    /* Change off heap message queue */
-    ERTS_PSTT_FTMQ      /* Flush trace msg queue */
+    ERTS_PSTT_FTMQ,     /* Flush trace msg queue */
+    ERTS_PSTT_ETS_FREE_FIXATION
 } ErtsProcSysTaskType;
 
 #define ERTS_MAX_PROC_SYS_TASK_ARGS 2
@@ -729,6 +730,16 @@ erts_pre_init_process(void)
 	= ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS;
     erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].set_locks
 	= ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS;
+
+    erts_psd_required_locks[ERTS_PSD_ETS_OWNED_TABLES].get_locks
+        = ERTS_PSD_ETS_OWNED_TABLES_GET_LOCKS;
+    erts_psd_required_locks[ERTS_PSD_ETS_OWNED_TABLES].set_locks
+        = ERTS_PSD_ETS_OWNED_TABLES_SET_LOCKS;
+
+    erts_psd_required_locks[ERTS_PSD_ETS_FIXED_TABLES].get_locks
+        = ERTS_PSD_ETS_FIXED_TABLES_GET_LOCKS;
+    erts_psd_required_locks[ERTS_PSD_ETS_FIXED_TABLES].set_locks
+        = ERTS_PSD_ETS_FIXED_TABLES_SET_LOCKS;
 #endif
 }
 
@@ -11197,6 +11208,12 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	    reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN);
 	    st_res = am_true;
 	    break;
+#endif
+#ifdef ERTS_SMP
+        case ERTS_PSTT_ETS_FREE_FIXATION:
+	    reds -= erts_db_execute_free_fixation(c_p, (DbFixation*)st->arg[0]);
+	    st_res = am_true;
+	    break;
 #endif
 	default:
 	    ERTS_INTERNAL_ERROR("Invalid process sys task type");
@@ -11249,7 +11266,8 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
         case ERTS_PSTT_GC_MAJOR:
         case ERTS_PSTT_GC_MINOR:
 	case ERTS_PSTT_CPC:
-	case ERTS_PSTT_COHMQ:
+        case ERTS_PSTT_COHMQ:
+        case ERTS_PSTT_ETS_FREE_FIXATION:
 	    st_res = am_false;
 	    break;
 	case ERTS_PSTT_CLA:
@@ -11603,13 +11621,12 @@ erts_internal_request_system_task_4(BIF_ALIST_4)
 }
 
 static void
-erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type)
+erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type, void* arg)
 {
     Process *rp = erts_proc_lookup(pid);
     if (rp) {
 	ErtsProcSysTask *st;
 	erts_aint32_t state, fail_state;
-	int i;
 
 	st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK,
 			ERTS_PROC_SYS_TASK_SIZE(0));
@@ -11618,8 +11635,7 @@ erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type)
 	st->reply_tag = NIL;
 	st->req_id = NIL;
 	st->req_id_sz = 0;
-	for (i = 0; i < ERTS_MAX_PROC_SYS_TASK_ARGS; i++)
-	    st->arg[i] = NIL;
+        st->arg[0] = (Eterm)arg;
 	ERTS_INIT_OFF_HEAP(&st->off_heap);
 	state = erts_smp_atomic32_read_nob(&rp->state);
 
@@ -11635,7 +11651,13 @@ erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type)
 void
 erts_schedule_complete_off_heap_message_queue_change(Eterm pid)
 {
-    erts_schedule_generic_sys_task(pid, ERTS_PSTT_COHMQ);
+    erts_schedule_generic_sys_task(pid, ERTS_PSTT_COHMQ, NULL);
+}
+
+void
+erts_schedule_ets_free_fixation(Eterm pid, DbFixation* fix)
+{
+    erts_schedule_generic_sys_task(pid, ERTS_PSTT_ETS_FREE_FIXATION, fix);
 }
 
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -11683,7 +11705,7 @@ erts_schedule_flush_trace_messages(Process *proc, int force_on_proc)
     dhndl = erts_thr_progress_unmanaged_delay();
 #endif
 
-    erts_schedule_generic_sys_task(pid, ERTS_PSTT_FTMQ);
+    erts_schedule_generic_sys_task(pid, ERTS_PSTT_FTMQ, NULL);
 
 #ifdef ERTS_SMP
     erts_thr_progress_unmanaged_continue(dhndl);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 58f1e15c23..f20dc68799 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -836,14 +836,16 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
 #define ERTS_PSD_CALL_TIME_BP			3
 #define ERTS_PSD_DELAYED_GC_TASK_QS		4
 #define ERTS_PSD_NIF_TRAP_EXPORT		5
-#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF	6
+#define ERTS_PSD_ETS_OWNED_TABLES               6
+#define ERTS_PSD_ETS_FIXED_TABLES               7
+#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF	8
 
-#define ERTS_PSD_SIZE				7
+#define ERTS_PSD_SIZE				9
 
 #if !defined(HIPE)
 #  undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF
 #  undef ERTS_PSD_SIZE
-#  define ERTS_PSD_SIZE 6
+#  define ERTS_PSD_SIZE 8
 #endif
 
 typedef struct {
@@ -871,6 +873,12 @@ typedef struct {
 #define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
 #define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN
 
+#define ERTS_PSD_ETS_OWNED_TABLES_GET_LOCKS ERTS_PROC_LOCK_STATUS
+#define ERTS_PSD_ETS_OWNED_TABLES_SET_LOCKS ERTS_PROC_LOCK_STATUS
+
+#define ERTS_PSD_ETS_FIXED_TABLES_GET_LOCKS ERTS_PROC_LOCK_MAIN
+#define ERTS_PSD_ETS_FIXED_TABLES_SET_LOCKS ERTS_PROC_LOCK_MAIN
+
 typedef struct {
     ErtsProcLocks get_locks;
     ErtsProcLocks set_locks;
@@ -1805,6 +1813,8 @@ void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *),
 					     ErtsThrPrgrLaterOp *,
 					     UWord);
 void erts_schedule_complete_off_heap_message_queue_change(Eterm pid);
+struct db_fixation;
+void erts_schedule_ets_free_fixation(Eterm pid, struct db_fixation*);
 void erts_schedule_flush_trace_messages(Process *proc, int force_on_proc);
 int erts_flush_trace_messages(Process *c_p, ErtsProcLocks locks);
 
diff --git a/erts/emulator/beam/erl_rbtree.h b/erts/emulator/beam/erl_rbtree.h
index ebde5caca4..6a42853957 100644
--- a/erts/emulator/beam/erl_rbtree.h
+++ b/erts/emulator/beam/erl_rbtree.h
@@ -486,12 +486,12 @@ typedef struct {
 #if defined(ERTS_RBT_HARD_DEBUG) \
     && (defined(ERTS_RBT_WANT_DELETE) \
 	|| defined(ERTS_RBT_NEED_INSERT__))
-static void ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root);
+static void ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root, ERTS_RBT_T *node);
 #  define ERTS_RBT_NEED_HDBG_CHECK_TREE__
-#  define ERTS_RBT_HDBG_CHECK_TREE__(R) \
-    ERTS_RBT_FUNC__(hdbg_check_tree)((R))
+#  define ERTS_RBT_HDBG_CHECK_TREE__(R,N) \
+    ERTS_RBT_FUNC__(hdbg_check_tree)((R),(N))
 #else
-#  define ERTS_RBT_HDBG_CHECK_TREE__(R) ((void) 1)
+#  define ERTS_RBT_HDBG_CHECK_TREE__(R,N) ((void) 1)
 #endif
 
 #ifdef ERTS_RBT_NEED_ROTATE__
@@ -644,7 +644,7 @@ ERTS_RBT_FUNC__(delete)(ERTS_RBT_T **root, ERTS_RBT_T *n)
     ERTS_RBT_T null_x; /* null_x is used to get the fixup started when we
 			  splice out a node without children. */
 
-    ERTS_RBT_HDBG_CHECK_TREE__(*root);
+    ERTS_RBT_HDBG_CHECK_TREE__(*root, n);
 
     ERTS_RBT_INIT_EMPTY_TNODE(&null_x);
 
@@ -862,7 +862,7 @@ ERTS_RBT_FUNC__(delete)(ERTS_RBT_T **root, ERTS_RBT_T *n)
 	}
     }
 
-    ERTS_RBT_HDBG_CHECK_TREE__(*root);
+    ERTS_RBT_HDBG_CHECK_TREE__(*root, NULL);
 
 }
 
@@ -992,7 +992,7 @@ ERTS_RBT_FUNC__(insert_aux__)(ERTS_RBT_T **root, ERTS_RBT_T *n, int lookup)
 {
     ERTS_RBT_KEY_T kn = ERTS_RBT_GET_KEY(n);
 
-    ERTS_RBT_HDBG_CHECK_TREE__(*root);
+    ERTS_RBT_HDBG_CHECK_TREE__(*root, NULL);
 
     ERTS_RBT_INIT_EMPTY_TNODE(n);	
 
@@ -1014,7 +1014,7 @@ ERTS_RBT_FUNC__(insert_aux__)(ERTS_RBT_T **root, ERTS_RBT_T *n, int lookup)
 
 	    if (lookup && ERTS_RBT_IS_EQ(kn, kx)) {
 
-		ERTS_RBT_HDBG_CHECK_TREE__(*root);
+		ERTS_RBT_HDBG_CHECK_TREE__(*root, NULL);
 
 		return x;
 	    }
@@ -1048,7 +1048,7 @@ ERTS_RBT_FUNC__(insert_aux__)(ERTS_RBT_T **root, ERTS_RBT_T *n, int lookup)
 	    ERTS_RBT_FUNC__(insert_fixup__)(root, n);
     }
 
-    ERTS_RBT_HDBG_CHECK_TREE__(*root);
+    ERTS_RBT_HDBG_CHECK_TREE__(*root, n);
 
     return NULL;
 }
@@ -1589,15 +1589,17 @@ ERTS_RBT_FUNC__(debug_print)(FILE *filep, ERTS_RBT_T *x, int indent,
 #ifdef ERTS_RBT_NEED_HDBG_CHECK_TREE__
 
 static void
-ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root)
+ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root, ERTS_RBT_T *n)
 {
     int black_depth = -1, no_black = 0;
     ERTS_RBT_T *c, *p, *x = root;
     ERTS_RBT_KEY_T kx;
     ERTS_RBT_KEY_T kc;
 
-    if (!x)
+    if (!x) {
+        ERTS_RBT_ASSERT(!n);
 	return;
+    }
 
     ERTS_RBT_ASSERT(!ERTS_RBT_GET_PARENT(x));
 
@@ -1607,6 +1609,9 @@ ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root)
 
 	    while (1) {
 
+                if (x == n)
+                    n = NULL;
+
 		if (ERTS_RBT_IS_BLACK(x))
 		    no_black++;
 		else {
@@ -1678,6 +1683,7 @@ ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root)
 	    if (!p) {
 		ERTS_RBT_ASSERT(root == x);
 		ERTS_RBT_ASSERT(no_black == 0);
+                ERTS_RBT_ASSERT(!n);
 		return; /* Done */
 	    }
 
-- 
cgit v1.2.3


From b9388df82dd1149fd603371c6ee12f5d7dbcec30 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 23 Feb 2017 18:24:15 +0100
Subject: erts: Print table id as ref in crashdump and break menu

---
 erts/emulator/beam/erl_db.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 24de21cc24..7faec9a057 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -4261,7 +4261,19 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 
 static void print_table(fmtfn_t to, void *to_arg, int show,  DbTable* tb)
 {
-    erts_print(to, to_arg, "Table: %T\n", tb->common.id);
+    Eterm tid;
+    Eterm heap[ERTS_MAGIC_REF_THING_SIZE];
+
+    if (is_atom(tb->common.id)) {
+        tid = tb->common.id;
+    } else {
+        ErlOffHeap oh;
+        ERTS_INIT_OFF_HEAP(&oh);
+        write_magic_ref_thing(heap, &oh, (ErtsMagicBinary *) tb->common.btid);
+        tid = make_internal_ref(heap);
+    }
+
+    erts_print(to, to_arg, "Table: %T\n", tid);
     erts_print(to, to_arg, "Name: %T\n", tb->common.the_name);
 
     tb->common.meth->db_print(to, to_arg, show, tb);
-- 
cgit v1.2.3


From e214bb4a0f86a6f956dd473f66b1a6c2f7085591 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 24 Feb 2017 20:08:34 +0100
Subject: erts: Pass tid argument down to trapping functions

to get rid of meta table lookup by integer (tb->common.id)
---
 erts/emulator/beam/erl_db.c      | 13 ++++++-------
 erts/emulator/beam/erl_db_hash.c | 32 +++++++++++++++++---------------
 erts/emulator/beam/erl_db_tree.c | 26 +++++++++++++-------------
 erts/emulator/beam/erl_db_util.h | 16 ++++++++++------
 4 files changed, 46 insertions(+), 41 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 7faec9a057..3f92a69df1 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -2391,7 +2391,7 @@ BIF_RETTYPE ets_select_delete_2(BIF_ALIST_2)
     if (safety == ITER_UNSAFE) {
 	local_fix_table(tb);
     }
-    cret = tb->common.meth->db_select_delete(BIF_P, tb, BIF_ARG_2, &ret);
+    cret = tb->common.meth->db_select_delete(BIF_P, tb, BIF_ARG_1, BIF_ARG_2, &ret);
 
     if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) {
 	fix_table_locked(BIF_P,tb);
@@ -2770,7 +2770,7 @@ ets_select3(Process* p, Eterm arg1, Eterm arg2, Eterm arg3)
     if (safety == ITER_UNSAFE) {
 	local_fix_table(tb);
     }
-    cret = tb->common.meth->db_select_chunk(p, tb,
+    cret = tb->common.meth->db_select_chunk(p, tb, arg1,
 					    arg2, chunk_size,
 					    0 /* not reversed */,
 					    &ret);
@@ -2939,8 +2939,7 @@ ets_select2(Process* p, Eterm arg1, Eterm arg2)
 	local_fix_table(tb);
     }
 
-    cret = tb->common.meth->db_select(p, tb, arg2,
-				      0, &ret);
+    cret = tb->common.meth->db_select(p, tb, arg1, arg2, 0, &ret);
 
     if (DID_TRAP(p,ret) && safety != ITER_SAFE) {
 	fix_table_locked(p, tb);
@@ -3031,7 +3030,7 @@ BIF_RETTYPE ets_select_count_2(BIF_ALIST_2)
     if (safety == ITER_UNSAFE) {
 	local_fix_table(tb);
     }
-    cret = tb->common.meth->db_select_count(BIF_P,tb,BIF_ARG_2, &ret);
+    cret = tb->common.meth->db_select_count(BIF_P,tb, BIF_ARG_1, BIF_ARG_2, &ret);
 
     if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) {
 	fix_table_locked(BIF_P, tb);
@@ -3085,7 +3084,7 @@ BIF_RETTYPE ets_select_reverse_3(BIF_ALIST_3)
     if (safety == ITER_UNSAFE) {
 	local_fix_table(tb);
     }
-    cret = tb->common.meth->db_select_chunk(BIF_P,tb,
+    cret = tb->common.meth->db_select_chunk(BIF_P,tb, BIF_ARG_1,
 					    BIF_ARG_2, chunk_size, 
 					    1 /* reversed */, &ret);
     if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) {
@@ -3135,7 +3134,7 @@ BIF_RETTYPE ets_select_reverse_2(BIF_ALIST_2)
     if (safety == ITER_UNSAFE) {
 	local_fix_table(tb);
     }
-    cret = tb->common.meth->db_select(BIF_P,tb,BIF_ARG_2,
+    cret = tb->common.meth->db_select(BIF_P,tb, BIF_ARG_1, BIF_ARG_2,
 				      1 /*reversed*/, &ret);
 
     if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) {
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 3afc58bb79..adb2376c34 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -393,20 +393,20 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object,Eterm *ret);
 static int db_slot_hash(Process *p, DbTable *tbl, 
 			Eterm slot_term, Eterm *ret);
 
-static int db_select_chunk_hash(Process *p, DbTable *tbl, 
+static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid,
 				Eterm pattern, Sint chunk_size,
 				int reverse, Eterm *ret);
-static int db_select_hash(Process *p, DbTable *tbl, 
+static int db_select_hash(Process *p, DbTable *tbl, Eterm tid,
 			  Eterm pattern, int reverse, Eterm *ret);
-static int db_select_count_hash(Process *p, DbTable *tbl, 
+static int db_select_count_hash(Process *p, DbTable *tbl,  Eterm tid,
 				Eterm pattern, Eterm *ret);
-static int db_select_delete_hash(Process *p, DbTable *tbl, 
+static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid,
 				 Eterm pattern, Eterm *ret);
 
 static int db_select_continue_hash(Process *p, DbTable *tbl, 
 				   Eterm continuation, Eterm *ret);
 
-static int db_select_count_continue_hash(Process *p, DbTable *tbl, 
+static int db_select_count_continue_hash(Process *p, DbTable *tbl,
 					 Eterm continuation, Eterm *ret);
 
 static int db_select_delete_continue_hash(Process *p, DbTable *tbl,
@@ -1289,14 +1289,14 @@ trap:
 
 }
 
-static int db_select_hash(Process *p, DbTable *tbl, 
+static int db_select_hash(Process *p, DbTable *tbl, Eterm tid,
 			  Eterm pattern, int reverse,
 			  Eterm *ret)
 {
-    return db_select_chunk_hash(p, tbl, pattern, 0, reverse, ret);
+    return db_select_chunk_hash(p, tbl, tid, pattern, 0, reverse, ret);
 }
 
-static int db_select_chunk_hash(Process *p, DbTable *tbl, 
+static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid,
 				Eterm pattern, Sint chunk_size, 
 				int reverse, /* not used */
 				Eterm *ret)
@@ -1444,7 +1444,7 @@ done:
 	    mpb = erts_db_make_match_prog_ref(p,(mpi.mp),&hp);
 	    if (mpi.all_objects)
 		(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
-	    continuation = TUPLE6(hp, tb->common.id,make_small(slot_ix), 
+	    continuation = TUPLE6(hp, tid, make_small(slot_ix),
 				  make_small(chunk_size),  
 				  mpb, rest, 
 				  make_small(rest_size));
@@ -1468,7 +1468,7 @@ trap:
 	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
     hp = HAlloc(p,7+ERTS_MAGIC_REF_THING_SIZE);
     mpb = erts_db_make_match_prog_ref(p,(mpi.mp),&hp);
-    continuation = TUPLE6(hp, tb->common.id, make_small(slot_ix), 
+    continuation = TUPLE6(hp, tid, make_small(slot_ix),
 			  make_small(chunk_size), 
 			  mpb, match_list, 
 			  make_small(got));
@@ -1482,7 +1482,8 @@ trap:
 }
 
 static int db_select_count_hash(Process *p, 
-				DbTable *tbl, 
+				DbTable *tbl,
+                                Eterm tid,
 				Eterm pattern,
 				Eterm *ret)
 {
@@ -1588,7 +1589,7 @@ trap:
 	hp += BIG_UINT_HEAP_SIZE;
     }
     mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
-    continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix), 
+    continuation = TUPLE4(hp, tid, make_small(slot_ix),
 			  mpb, 
 			  egot);
     mpi.mp = NULL; /*otherwise the return macro will destroy it */
@@ -1601,6 +1602,7 @@ trap:
 
 static int db_select_delete_hash(Process *p,
 				 DbTable *tbl,
+                                 Eterm tid,
 				 Eterm pattern,
 				 Eterm *ret)
 {
@@ -1733,7 +1735,7 @@ trap:
 	hp += BIG_UINT_HEAP_SIZE;
     }
     mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
-    continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix), 
+    continuation = TUPLE4(hp, tid, make_small(slot_ix),
 			  mpb, 
 			  egot);
     mpi.mp = NULL; /*otherwise the return macro will destroy it */
@@ -1847,7 +1849,7 @@ trap:
 	egot = uint_to_big(got, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix), 
+    continuation = TUPLE4(hp, tptr[1], make_small(slot_ix),
 			  tptr[3], 
 			  egot);
     RET_TO_BIF(bif_trap1(&ets_select_delete_continue_exp, p, 
@@ -1938,7 +1940,7 @@ trap:
 	egot = uint_to_big(got, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix), 
+    continuation = TUPLE4(hp, tptr[1], make_small(slot_ix),
 			  tptr[3], 
 			  egot);
     RET_TO_BIF(bif_trap1(&ets_select_count_continue_exp, p, 
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index c4ecd2ba37..3c2dd65e61 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -369,18 +369,18 @@ static int db_erase_tree(DbTable *tbl, Eterm key, Eterm *ret);
 static int db_erase_object_tree(DbTable *tbl, Eterm object,Eterm *ret);
 static int db_slot_tree(Process *p, DbTable *tbl, 
 			Eterm slot_term,  Eterm *ret);
-static int db_select_tree(Process *p, DbTable *tbl, 
+static int db_select_tree(Process *p, DbTable *tbl, Eterm tid,
 			  Eterm pattern, int reversed, Eterm *ret);
-static int db_select_count_tree(Process *p, DbTable *tbl, 
+static int db_select_count_tree(Process *p, DbTable *tbl, Eterm tid,
 				Eterm pattern,  Eterm *ret);
-static int db_select_chunk_tree(Process *p, DbTable *tbl, 
+static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid,
 				Eterm pattern, Sint chunk_size,
 				int reversed, Eterm *ret);
 static int db_select_continue_tree(Process *p, DbTable *tbl,
 				   Eterm continuation, Eterm *ret);
 static int db_select_count_continue_tree(Process *p, DbTable *tbl,
 					 Eterm continuation, Eterm *ret);
-static int db_select_delete_tree(Process *p, DbTable *tbl, 
+static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm tid,
 				 Eterm pattern,  Eterm *ret);
 static int db_select_delete_continue_tree(Process *p, DbTable *tbl, 
 					  Eterm continuation, Eterm *ret);
@@ -1058,7 +1058,7 @@ static int db_select_continue_tree(Process *p,
 }
 
 
-static int db_select_tree(Process *p, DbTable *tbl, 
+static int db_select_tree(Process *p, DbTable *tbl, Eterm tid,
 			  Eterm pattern, int reverse, Eterm *ret)
 {
     /* Strategy: Traverse backwards to build resulting list from tail to head */
@@ -1151,7 +1151,7 @@ static int db_select_tree(Process *p, DbTable *tbl,
 	    
     continuation = TUPLE8
 	(hp,
-	 tb->common.id,
+	 tid,
 	 key,
 	 sc.end_condition, /* From the match program, needn't be copied */
 	 make_small(0), /* Chunk size of zero means not chunked to the
@@ -1263,7 +1263,7 @@ static int db_select_count_continue_tree(Process *p,
 }
 
 
-static int db_select_count_tree(Process *p, DbTable *tbl, 
+static int db_select_count_tree(Process *p, DbTable *tbl, Eterm tid,
 				Eterm pattern, Eterm *ret)
 {
     DbTableTree *tb = &tbl->tree;
@@ -1349,7 +1349,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
 	    
     continuation = TUPLE5
 	(hp,
-	 tb->common.id,
+	 tid,
 	 key,
 	 sc.end_condition, /* From the match program, needn't be copied */
 	 mpb,
@@ -1363,7 +1363,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
 
 }
 
-static int db_select_chunk_tree(Process *p, DbTable *tbl, 
+static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid,
 				Eterm pattern, Sint chunk_size,
 				int reverse,
 				Eterm *ret)
@@ -1474,7 +1474,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
 	
 	continuation = TUPLE8
 	    (hp,
-	     tb->common.id,
+	     tid,
 	     key,
 	     sc.end_condition, /* From the match program, 
 				  needn't be copied */
@@ -1499,7 +1499,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
     mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
     continuation = TUPLE8
 	(hp,
-	 tb->common.id,
+	 tid,
 	 key,
 	 sc.end_condition, /* From the match program, needn't be copied */
 	 make_small(chunk_size),
@@ -1605,7 +1605,7 @@ static int db_select_delete_continue_tree(Process *p,
 #undef RET_TO_BIF
 }
 
-static int db_select_delete_tree(Process *p, DbTable *tbl, 
+static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm tid,
 				 Eterm pattern, Eterm *ret)
 {
     DbTableTree *tb = &tbl->tree;
@@ -1691,7 +1691,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
     
     continuation = TUPLE5
 	(hp,
-	 tb->common.id,
+	 tid,
 	 key,
 	 sc.end_condition, /* From the match program, needn't be copied */
 	 mpb,
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 673ec80604..958b504a60 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -135,30 +135,34 @@ typedef struct db_table_method
 		   Eterm slot, 
 		   Eterm* ret);
     int (*db_select_chunk)(Process* p, 
-			   DbTable* tb, /* [in out] */ 
+			   DbTable* tb, /* [in out] */
+                           Eterm tid,
 			   Eterm pattern,
 			   Sint chunk_size,
 			   int reverse,
 			   Eterm* ret);
     int (*db_select)(Process* p, 
-		     DbTable* tb, /* [in out] */ 
+		     DbTable* tb, /* [in out] */
+                     Eterm tid,
 		     Eterm pattern,
 		     int reverse,
 		     Eterm* ret);
     int (*db_select_delete)(Process* p, 
-			    DbTable* tb, /* [in out] */ 
+			    DbTable* tb, /* [in out] */
+                            Eterm tid,
 			    Eterm pattern,
 			    Eterm* ret);
     int (*db_select_continue)(Process* p, 
-			      DbTable* tb, /* [in out] */ 
+			      DbTable* tb, /* [in out] */
 			      Eterm continuation,
 			      Eterm* ret);
     int (*db_select_delete_continue)(Process* p, 
-				     DbTable* tb, /* [in out] */ 
+				     DbTable* tb, /* [in out] */
 				     Eterm continuation,
 				     Eterm* ret);
     int (*db_select_count)(Process* p, 
-			   DbTable* tb, /* [in out] */ 
+			   DbTable* tb, /* [in out] */
+                           Eterm tid,
 			   Eterm pattern, 
 			   Eterm* ret);
     int (*db_select_count_continue)(Process* p, 
-- 
cgit v1.2.3


From 14982307b07e899e888e5bfb93b3a138d4948459 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 24 Feb 2017 20:43:54 +0100
Subject: erts: Remove meta_main_tab

\o/
 O
/ \

Also removed the body for CHECK_TABLES enabled by HARDDEBUG.
Removed quite useless check for hash,
but kept dead check code for tree.
---
 erts/emulator/beam/erl_db.c         | 201 ++----------------------------------
 erts/emulator/beam/erl_db_hash.c    |  28 -----
 erts/emulator/beam/erl_db_tree.c    |   9 +-
 erts/emulator/beam/erl_db_util.h    |  11 --
 erts/emulator/beam/erl_lock_check.c |   2 -
 5 files changed, 11 insertions(+), 240 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 3f92a69df1..11936ed9dc 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -179,45 +179,12 @@ static int fixed_tabs_find(DbFixation* first, DbFixation* fix)
 
 #include "erl_rbtree.h"
 
-
-
-/* 
-** The main meta table, containing all ets tables.
-*/
-#ifdef ERTS_SMP
-
-#define ERTS_META_MAIN_TAB_LOCK_TAB_BITS 8
-#define ERTS_META_MAIN_TAB_LOCK_TAB_SIZE (1 << ERTS_META_MAIN_TAB_LOCK_TAB_BITS)
-#define ERTS_META_MAIN_TAB_LOCK_TAB_MASK (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE - 1)
-
-typedef union {
-    erts_smp_rwmtx_t rwmtx;
-    byte cache_line_align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(
-				sizeof(erts_smp_rwmtx_t))];
-} erts_meta_main_tab_lock_t;
-
-static erts_meta_main_tab_lock_t *meta_main_tab_locks;
-
+#ifdef HARDDEBUG
+# error Do something useful with CHECK_TABLES maybe
+#else
+# define CHECK_TABLES()
 #endif
-static struct {
-    union {
-	DbTable *tb;     /* Only directly readable if slot is ALIVE */
-	UWord next_free;  /* (index<<2)|1 if slot is FREE */
-    }u;
-} *meta_main_tab;
 
-/* A slot in meta_main_tab can have three states:
- * FREE : Free to use for new table. Part of linked free-list.
- * ALIVE: Contains a table
- * DEAD : Contains a table that is being removed.
- */
-#define IS_SLOT_FREE(i)	(meta_main_tab[(i)].u.next_free & 1)
-#define IS_SLOT_DEAD(i) (meta_main_tab[(i)].u.next_free & 2)
-#define IS_SLOT_ALIVE(i) (!(meta_main_tab[(i)].u.next_free & (1|2)))
-#define GET_NEXT_FREE_SLOT(i) (meta_main_tab[(i)].u.next_free >> 2)
-#define SET_NEXT_FREE_SLOT(i,next) (meta_main_tab[(i)].u.next_free = ((next)<<2)|1)
-#define MARK_SLOT_DEAD(i) (meta_main_tab[(i)].u.next_free |= 2)
-#define GET_ANY_SLOT_TAB(i) ((DbTable*)(meta_main_tab[(i)].u.next_free & ~(1|2))) /* dead or alive */
 
 static void
 send_ets_transfer_message(Process *c_p, Process *proc,
@@ -315,23 +282,6 @@ make_tid(Process *c_p, DbTable *tb)
     return erts_mk_magic_ref(&hp, &c_p->off_heap, tb->common.btid);
 }
 
-static ERTS_INLINE erts_smp_rwmtx_t *
-get_meta_main_tab_lock(unsigned slot)
-{
-#ifdef ERTS_SMP
-    return &meta_main_tab_locks[slot & ERTS_META_MAIN_TAB_LOCK_TAB_MASK].rwmtx;
-#else
-    return NULL;
-#endif
-}
-
-static erts_smp_spinlock_t meta_main_tab_main_lock;
-static Uint meta_main_tab_first_free;   /* Index of first free slot */
-static int meta_main_tab_cnt;		/* Number of active tables */
-static int meta_main_tab_top;           /* Highest ever used slot + 1 */
-static Uint meta_main_tab_slot_mask;    /* The slot index part of an unnamed table id */
-static Uint meta_main_tab_seq_incr;
-static Uint meta_main_tab_seq_cnt = 0;  /* To give unique(-ish) table identifiers */
 
 /* 
 ** The meta hash table of all NAMED ets tables
@@ -713,22 +663,6 @@ DbTable* db_get_table_aux(Process *p,
     tb = tid2tab(id);
     if (tb)
         id = tb->common.id;
-    else if (is_small(id)) {
-	Uint slot = unsigned_val(id) & meta_main_tab_slot_mask;
-	if (!meta_already_locked) {
-	    mtl = get_meta_main_tab_lock(slot);
-	    erts_smp_rwmtx_rlock(mtl);
-	}
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
-	else {
-	    erts_smp_rwmtx_t *test_mtl = get_meta_main_tab_lock(slot);
-	    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(test_mtl)
-			       || erts_lc_rwmtx_is_rwlocked(test_mtl));
-	}
-#endif
-	if (slot < db_max_tabs && IS_SLOT_ALIVE(slot))
-	    tb = meta_main_tab[slot].u.tb;
-    }
     else if (is_atom(id)) {
 	struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl);
 	if (!meta_already_locked)
@@ -779,18 +713,6 @@ DbTable* db_get_table(Process *p,
     return db_get_table_aux(p, id, what, kind, 0);
 }
 
-/* Requires meta_main_tab_locks[slot] locked.
-*/
-static ERTS_INLINE void free_slot(int slot)
-{
-    ASSERT(!IS_SLOT_FREE(slot));
-    erts_smp_spin_lock(&meta_main_tab_main_lock);
-    SET_NEXT_FREE_SLOT(slot,meta_main_tab_first_free);
-    meta_main_tab_first_free = slot;
-    meta_main_tab_cnt--;
-    erts_smp_spin_unlock(&meta_main_tab_main_lock);
-}
-
 static int insert_named_tab(Eterm name_atom, DbTable* tb, int have_lock)
 {
     int ret = 0;
@@ -1602,7 +1524,6 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
 	}
     }
     else {
-        Uint slot;
         tb = tid2tab(BIF_ARG_1);
         if (!tb)
             BIF_ERROR(BIF_P, BADARG);
@@ -1611,8 +1532,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
                 old_name = tb->common.id;
                 goto named_tab;
             }
-            slot = unsigned_val(tb->common.id) & meta_main_tab_slot_mask;
-            lck2 = get_meta_main_tab_lock(slot);
+            lck2 = NULL;
         }
     }
 
@@ -1665,7 +1585,6 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
 BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 {
     DbTable* tb = NULL;
-    int slot;
     Eterm list;
     Eterm val;
     Eterm ret;
@@ -1681,7 +1600,6 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     int cret;
 #endif
     DbTableMethod* meth;
-    erts_smp_rwmtx_t *mmtl;
 
     if (is_not_atom(BIF_ARG_1)) {
 	BIF_ERROR(BIF_P, BADARG);
@@ -1847,57 +1765,21 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 
     make_btid(tb);
 
-    erts_smp_spin_lock(&meta_main_tab_main_lock);
-
-    if (meta_main_tab_cnt >= db_max_tabs) {
-	erts_smp_spin_unlock(&meta_main_tab_main_lock);
-	erts_send_error_to_logger_str(BIF_P->group_leader,
-				      "** Too many db tables **\n");
-	free_heir_data(tb);
-	tb->common.meth->db_free_table(tb);
-	free_dbtable((void *) tb);
-	BIF_ERROR(BIF_P, SYSTEM_LIMIT);
-    }
-
-    slot = meta_main_tab_first_free;
-    ASSERT(slot>=0 && slot<db_max_tabs);
-    meta_main_tab_first_free = GET_NEXT_FREE_SLOT(slot);
-    meta_main_tab_cnt++;
-    if (slot >= meta_main_tab_top) {
-	ASSERT(slot == meta_main_tab_top);
-	meta_main_tab_top = slot + 1;
-    }
-
     if (is_named) {
 	ret = BIF_ARG_1;
+        tb->common.id = ret;
     }
     else {
-	ret = make_small(slot | meta_main_tab_seq_cnt);
-	meta_main_tab_seq_cnt += meta_main_tab_seq_incr;
-	ASSERT((unsigned_val(ret) & meta_main_tab_slot_mask) == slot);
+        ret = make_tid(BIF_P, tb);
+        tb->common.id = THE_NON_VALUE;  /* or what? */
     }
-    erts_smp_spin_unlock(&meta_main_tab_main_lock);
-
-    tb->common.id = ret;
-    tb->common.slot = slot;           /* store slot for erase */
 
     if (!is_named)
         ret = make_tid(BIF_P, tb);
 
     save_sched_table(BIF_P, tb);
 
-    mmtl = get_meta_main_tab_lock(slot);
-    erts_smp_rwmtx_rwlock(mmtl);
-    meta_main_tab[slot].u.tb = tb;
-    ASSERT(IS_SLOT_ALIVE(slot));
-    erts_smp_rwmtx_rwunlock(mmtl);
-
     if (is_named && !insert_named_tab(BIF_ARG_1, tb, 0)) {
-	mmtl = get_meta_main_tab_lock(slot);
-	erts_smp_rwmtx_rwlock(mmtl);
-	free_slot(slot);
-	erts_smp_rwmtx_rwunlock(mmtl);
-
         tid_clear(BIF_P, tb);
 
 	db_lock(tb,LCK_WRITE);
@@ -2024,7 +1906,6 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 {
     int trap;
     DbTable* tb;
-    erts_smp_rwmtx_t *mmtl;
 
 #ifdef HARDDEBUG
     erts_fprintf(stderr,
@@ -2070,21 +1951,6 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 
     tid_clear(BIF_P, tb);
 
-    mmtl = get_meta_main_tab_lock(tb->common.slot);
-#ifdef ERTS_SMP
-    if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) {
-	/*
-	 * We keep our increased refc over this op in order to
-	 * prevent the table from disapearing.
-	 */
-	db_unlock(tb, LCK_WRITE);
-	erts_smp_rwmtx_rwlock(mmtl);
-	db_lock(tb, LCK_WRITE);
-    }
-#endif
-    /* We must keep the slot, to be found by db_proc_dead() if process dies */
-    MARK_SLOT_DEAD(tb->common.slot);
-    erts_smp_rwmtx_rwunlock(mmtl);
     if (is_atom(tb->common.id))
 	remove_named_tab(tb, 0);
     
@@ -3425,16 +3291,6 @@ void init_db(ErtsDbSpinCount db_spin_count)
     if (erts_ets_rwmtx_spin_count >= 0)
 	rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count;
 
-    meta_main_tab_locks =
-	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_DB_TABLES,
-					   sizeof(erts_meta_main_tab_lock_t)
-					   * ERTS_META_MAIN_TAB_LOCK_TAB_SIZE);
-
-    for (i = 0; i < ERTS_META_MAIN_TAB_LOCK_TAB_SIZE; i++) {
-	erts_smp_rwmtx_init_opt_x(&meta_main_tab_locks[i].rwmtx, &rwmtx_opt,
-				  "meta_main_tab_slot", make_small(i));
-    }
-    erts_smp_spinlock_init(&meta_main_tab_main_lock, "meta_main_tab_main");
     for (i=0; i<META_NAME_TAB_LOCK_CNT; i++) {
 	erts_smp_rwmtx_init_opt_x(&meta_name_tab_rwlocks[i].lck, &rwmtx_opt,
 				  "meta_name_tab", make_small(i));
@@ -3454,20 +3310,6 @@ void init_db(ErtsDbSpinCount db_spin_count)
 	erts_exit(ERTS_ERROR_EXIT,"Max limit for ets tabled too high %u (max %u).",
 		 db_max_tabs, ((Uint)1)<<SMALL_BITS);
     }
-    meta_main_tab_slot_mask = (((Uint)1)<<bits) - 1;
-    meta_main_tab_seq_incr = (((Uint)1)<<bits);
-
-    size = sizeof(*meta_main_tab)*db_max_tabs;
-    meta_main_tab = erts_db_alloc_nt(ERTS_ALC_T_DB_TABLES, size);
-    ERTS_ETS_MISC_MEM_ADD(size);
-
-    meta_main_tab_cnt = 0;
-    meta_main_tab_top = 0;
-    for (i=1; i<db_max_tabs; i++) {
-	SET_NEXT_FREE_SLOT(i-1,i);
-    }
-    SET_NEXT_FREE_SLOT(db_max_tabs-1, (Uint)-1);
-    meta_main_tab_first_free = 0;
 
     meta_name_tab_mask = (((Uint) 1)<<(bits-1)) - 1; /* At least half the size of main tab */
     size = sizeof(struct meta_name_tab_entry)*(meta_name_tab_mask+1);
@@ -4059,7 +3901,6 @@ static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1)
 static int free_table_cont(Process *p, DbTable *tb)
 {
     Eterm result;
-    erts_smp_rwmtx_t *mmtl;
 
     result = tb->common.meth->db_free_table_continue(tb);
 
@@ -4078,13 +3919,6 @@ static int free_table_cont(Process *p, DbTable *tb)
 		     tb->common.id);
 #endif
 	/* Completely done - we will not get called again. */
-	mmtl = get_meta_main_tab_lock(tb->common.slot);
-#ifdef ERTS_SMP
-        erts_smp_rwmtx_rwlock(mmtl);
-#endif
-	free_slot(tb->common.slot);
-	erts_smp_rwmtx_rwunlock(mmtl);
-
         delete_owned_table(p, tb);
         table_dec_refc(tb, 0);
 	BUMP_REDS(p, 100);
@@ -4393,22 +4227,3 @@ erts_ets_colliding_names(Process* p, Eterm name, Uint cnt)
     return list;
 }
 
-#ifdef HARDDEBUG   /* Here comes some debug functions */
-
-void db_check_tables(void)
-{
-#ifdef ERTS_SMP
-    return;
-#else
-    int i;
-
-    for (i = 0; i < db_max_tabs; i++) {
-	if (IS_SLOT_ALIVE(i)) {
-	    DbTable* tb = meta_main_tab[i].t; 
-	    tb->common.meth->db_check_table(tb);
-	}
-    }
-#endif
-}
-
-#endif /* HARDDEBUG */
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index adb2376c34..781886a822 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -529,11 +529,6 @@ DbTableMethod db_hash =
     db_free_table_continue_hash,
     db_print_hash,
     db_foreach_offheap_hash,
-#ifdef HARDDEBUG
-    db_check_table_hash,
-#else
-    NULL,
-#endif
     db_lookup_dbterm_hash,
     db_finalize_dbterm_hash
 };
@@ -846,7 +841,6 @@ Lnew:
 	    grow(tb, nitems);
 	}
     }
-    CHECK_TABLES();
     return DB_ERROR_NONE;
 
 Ldone:
@@ -871,7 +865,6 @@ get_term_list(Process *p, DbTableHash *tb, Eterm key, HashValue hval,
         }
     }
     copy = build_term_list(p, b1, b2, sz, tb);
-    CHECK_TABLES();
     if (bend) {
         *bend = b2;
     }
@@ -2898,24 +2891,3 @@ Eterm erts_ets_hash_sizeof_ext_segtab(void)
     return make_small(((SIZEOF_EXT_SEGTAB(0)-1) / sizeof(UWord)) + 1);
 }
 
-#ifdef HARDDEBUG
-
-void db_check_table_hash(DbTable *tbl)
-{
-    DbTableHash *tb = &tbl->hash;
-    HashDbTerm* list;
-    int j;
-    
-    for (j = 0; j < NACTIVE(tb); j++) {
-	if ((list = BUCKET(tb,j)) != 0) {
-	    while (list != 0) {
-		if (!is_tuple(make_tuple(list->dbterm.tpl))) {
-		    erts_exit(ERTS_ERROR_EXIT, "Bad term in slot %d of ets table", j);
-		}
-		list = list->next;
-	    }
-	}
-    }
-}
-
-#endif
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 3c2dd65e61..3dd06fd7bb 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -180,7 +180,6 @@ static ERTS_INLINE TreeDbTerm* replace_dbterm(DbTableTree *tb, TreeDbTerm* old,
 static TreeDbTerm *traverse_until(TreeDbTerm *t, int *current, int to);
 static void check_slot_pos(DbTableTree *tb);
 static void check_saved_stack(DbTableTree *tb);
-static int check_table_tree(DbTableTree* tb, TreeDbTerm *t);
 
 #define TREE_DEBUG
 #endif
@@ -442,11 +441,6 @@ DbTableMethod db_tree =
     db_free_table_continue_tree,
     db_print_tree,
     db_foreach_offheap_tree,
-#ifdef HARDDEBUG
-    db_check_table_tree,
-#else
-    NULL,
-#endif
     db_lookup_dbterm_tree,
     db_finalize_dbterm_tree
 
@@ -3130,6 +3124,9 @@ static void do_dump_tree2(DbTableTree* tb, int to, void *to_arg, int show,
 
 #ifdef HARDDEBUG
 
+/*
+ * No called, but kept as it might come to use
+ */
 void db_check_table_tree(DbTable *tbl)
 {
     DbTableTree *tb = &tbl->tree;
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 958b504a60..e22113b72f 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -185,7 +185,6 @@ typedef struct db_table_method
     void (*db_foreach_offheap)(DbTable* db,  /* [in out] */ 
 			       void (*func)(ErlOffHeap *, void *),
 			       void *arg);
-    void (*db_check_table)(DbTable* tb);
 
     /* Lookup a dbterm for updating. Return false if not found. */
     int (*db_lookup_dbterm)(Process *, DbTable *, Eterm key, Eterm obj,
@@ -257,7 +256,6 @@ typedef struct db_table_common {
                                  "local" fixations not included. */ 
     /* All 32-bit fields */
     Uint32 status;            /* bit masks defined  below */
-    int slot;                 /* slot index in meta_main_tab */
     int keypos;               /* defaults to 1 */
     int compress;
 } DbTableCommon;
@@ -527,14 +525,5 @@ erts_db_get_match_prog_binary(Eterm term)
 #define Binary2MatchProg(BP) \
   (ASSERT(IsMatchProgBinary((BP))), \
    ((MatchProg *) ERTS_MAGIC_BIN_DATA((BP))))
-/*
-** Debugging 
-*/
-#ifdef HARDDEBUG
-void db_check_tables(void); /* in db.c */
-#define CHECK_TABLES() db_check_tables()
-#else 
-#define CHECK_TABLES()
-#endif
 
 #endif /* _DB_UTIL_H */
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 7f7c06da8e..da73469516 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -101,13 +101,11 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {   "code_write_permission",                NULL                    },
     {	"purge_state",		      		NULL			},
     {	"meta_name_tab",	         	"address"		},
-    {	"meta_main_tab_slot",			"address"		},
     {	"db_tab",				"address"		},
     {	"proc_status",				"pid"			},
     {	"proc_trace",				"pid"			},
     {   "ports_snapshot",                       NULL                    },
     {	"db_tab_fix",				"address"		},
-    {	"meta_main_tab_main",			NULL 			},
     {	"db_hash_slot",				"address"		},
     {	"node_table",				NULL			},
     {	"dist_table",				NULL			},
-- 
cgit v1.2.3


From ba23c44add22306495b66d7258f835db00f12868 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 27 Feb 2017 16:18:05 +0100
Subject: erts: Remove now redundant 'id' from DbTableCommon

'the_name' keeps name of all tables.
'type' & DB_NAMED_TABLE mark tables as named.
table ref id is built from magic bin when needed.
---
 erts/emulator/beam/erl_db.c      | 94 ++++++++++++++++++++--------------------
 erts/emulator/beam/erl_db_util.h |  5 ++-
 2 files changed, 50 insertions(+), 49 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 11936ed9dc..5d14530d84 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -261,6 +261,17 @@ is_table_alive(DbTable *tb)
     return !!rtb;
 }
 
+static ERTS_INLINE int
+is_table_named(DbTable *tb)
+{
+#ifdef ERTS_SMP
+    return tb->common.type & DB_NAMED_TABLE;
+#else
+    return tb->common.status & DB_NAMED_TABLE;
+#endif
+}
+
+
 static ERTS_INLINE void
 tid_clear(Process *c_p, DbTable *tb)
 {
@@ -660,10 +671,7 @@ DbTable* db_get_table_aux(Process *p,
      */
     ASSERT(erts_get_scheduler_data());
 
-    tb = tid2tab(id);
-    if (tb)
-        id = tb->common.id;
-    else if (is_atom(id)) {
+    if (is_atom(id)) {
 	struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl);
 	if (!meta_already_locked)
 	    erts_smp_rwmtx_rlock(mtl);
@@ -672,7 +680,7 @@ DbTable* db_get_table_aux(Process *p,
 			       || erts_lc_rwmtx_is_rwlocked(mtl));
 	    mtl = NULL;
 	}
-
+        tb = NULL;
 	if (bucket->pu.tb != NULL) {
 	    if (is_atom(bucket->u.name_atom)) { /* single */
 		if (bucket->u.name_atom == id)
@@ -690,11 +698,13 @@ DbTable* db_get_table_aux(Process *p,
 	    }
 	}
     }
+    else
+        tb = tid2tab(id);
+
     if (tb) {
 	db_lock(tb, kind);
-	if (tb->common.id != id
-	    || ((tb->common.status & what) == 0
-		&& p->common.id != tb->common.owner)) {
+	if ((tb->common.status & what) == 0
+            && p->common.id != tb->common.owner) {
 	    db_unlock(tb, kind);
 	    tb = NULL;
 	}
@@ -777,9 +787,10 @@ static int remove_named_tab(DbTable *tb, int have_lock)
 {
     int ret = 0;
     erts_smp_rwmtx_t* rwlock;
-    Eterm name_atom = tb->common.id;
+    Eterm name_atom = tb->common.the_name;
     struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom,
 							      &rwlock);
+    ASSERT(is_table_named(tb));
 #ifdef ERTS_SMP
     if (!have_lock && erts_smp_rwmtx_tryrwlock(rwlock) == EBUSY) {
 	db_unlock(tb, LCK_WRITE);
@@ -1528,8 +1539,8 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
         if (!tb)
             BIF_ERROR(BIF_P, BADARG);
         else {
-            if (is_atom(tb->common.id)) {
-                old_name = tb->common.id;
+            if (is_table_named(tb)) {
+                old_name = tb->common.the_name;
                 goto named_tab;
             }
             lck2 = NULL;
@@ -1544,24 +1555,19 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
     if (!tb)
 	goto badarg;
 
-    if (is_not_atom(tb->common.id)) { /* Not a named table */
-	tb->common.the_name = BIF_ARG_2;
-	goto done;
-    }
+    if (is_table_named(tb)) {
+        if (!insert_named_tab(BIF_ARG_2, tb, 1))
+            goto badarg;
 
-    if (!insert_named_tab(BIF_ARG_2, tb, 1))
-	goto badarg;
-
-    if (!remove_named_tab(tb, 1))
-	erts_exit(ERTS_ERROR_EXIT,"Could not find named tab %s", tb->common.id);
-
-    tb->common.id = tb->common.the_name = BIF_ARG_2;
-
- done:
-    if (is_atom(tb->common.id))
-        ret = tb->common.id;
-    else
+        if (!remove_named_tab(tb, 1))
+            erts_exit(ERTS_ERROR_EXIT,"Could not find named tab %s", tb->common.the_name);
+        ret = BIF_ARG_2;
+    }
+    else { /* Not a named table */
         ret = BIF_ARG_1;
+    }
+    tb->common.the_name = BIF_ARG_2;
+
     db_unlock(tb, LCK_WRITE);
     erts_smp_rwmtx_rwunlock(lck1);
     if (lck2)
@@ -1691,6 +1697,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 	}
 	else if (val == am_named_table) {
 	    is_named = 1;
+            status |= DB_NAMED_TABLE;
 	}
 	else if (val == am_compressed) {
 	    is_compressed = 1;
@@ -1765,16 +1772,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 
     make_btid(tb);
 
-    if (is_named) {
-	ret = BIF_ARG_1;
-        tb->common.id = ret;
-    }
-    else {
-        ret = make_tid(BIF_P, tb);
-        tb->common.id = THE_NON_VALUE;  /* or what? */
-    }
-
-    if (!is_named)
+    if (is_named)
+        ret = BIF_ARG_1;
+    else
         ret = make_tid(BIF_P, tb);
 
     save_sched_table(BIF_P, tb);
@@ -1951,7 +1951,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 
     tid_clear(BIF_P, tb);
 
-    if (is_atom(tb->common.id))
+    if (is_table_named(tb))
 	remove_named_tab(tb, 0);
     
     /* disable inheritance */
@@ -2385,8 +2385,8 @@ ets_all_reply(ErtsSchedulerData *esdp, ErtsEtsAllReq **reqpp,
         while (1) {
             if (is_table_alive(tb)) {
                 Eterm tid;
-                if (is_atom(tb->common.id))
-                    tid = tb->common.id;
+                if (is_table_named(tb))
+                    tid = tb->common.the_name;
                 else
                     tid = erts_mk_magic_ref(&hp, ohp, tb->common.btid);
                 list = CONS(hp, tid, list);
@@ -3446,7 +3446,7 @@ send_ets_transfer_message(Process *c_p, Process *proc,
     Eterm tid, hd_copy, msg, sender;
 
     hsz = 5;
-    if (is_not_atom(tb->common.id))
+    if (!is_table_named(tb))
         hsz += ERTS_MAGIC_REF_THING_SIZE;
     if (is_immed(heir_data))
         hd_sz = 0;
@@ -3456,8 +3456,8 @@ send_ets_transfer_message(Process *c_p, Process *proc,
     }
 
     mp = erts_alloc_message_heap(proc, locks, hsz, &hp, &ohp);
-    if (is_atom(tb->common.id))
-        tid = tb->common.id;
+    if (is_table_named(tb))
+        tid = tb->common.the_name;
     else
         tid = erts_mk_magic_ref(&hp, ohp, tb->common.btid);
     if (!hd_sz)
@@ -3584,7 +3584,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
             tb->common.status &= ~(DB_PROTECTED | DB_PUBLIC | DB_PRIVATE);
             tb->common.status |= DB_DELETE;
 
-            if (is_atom(tb->common.id))
+            if (is_table_named(tb))
                 remove_named_tab(tb, 0);
 
             free_heir_data(tb);
@@ -3990,7 +3990,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
     } else if (What == am_node) {
 	ret = erts_this_dist_entry->sysname;
     } else if (What == am_named_table) {
-	ret = is_atom(tb->common.id) ? am_true : am_false;
+	ret = is_table_named(tb) ? am_true : am_false;
     } else if (What == am_compressed) {
 	ret = tb->common.compress ? am_true : am_false;
     }
@@ -4097,8 +4097,8 @@ static void print_table(fmtfn_t to, void *to_arg, int show,  DbTable* tb)
     Eterm tid;
     Eterm heap[ERTS_MAGIC_REF_THING_SIZE];
 
-    if (is_atom(tb->common.id)) {
-        tid = tb->common.id;
+    if (is_table_named(tb)) {
+        tid = tb->common.the_name;
     } else {
         ErlOffHeap oh;
         ERTS_INIT_OFF_HEAP(&oh);
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index e22113b72f..198f339751 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -243,7 +243,6 @@ typedef struct db_table_common {
     UWord heir_data;          /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */
     Uint64 heir_started_interval;  /* To further identify the heir */
     Eterm the_name;           /* an atom */
-    Eterm id;                 /* atom | integer */
     Binary *btid;
     DbTableMethod* meth;      /* table methods */
     erts_smp_atomic_t nitems; /* Total number of items in table */
@@ -273,8 +272,10 @@ typedef struct db_table_common {
 #define DB_ORDERED_SET   (1 << 9)
 #define DB_DELETE        (1 << 10) /* table is being deleted */
 #define DB_FREQ_READ     (1 << 11)
+#define DB_NAMED_TABLE   (1 << 12)
 
-#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET|DB_FINE_LOCKED|DB_FREQ_READ)
+#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET\
+                              |DB_FINE_LOCKED|DB_FREQ_READ|DB_NAMED_TABLE)
 
 #define IS_HASH_TABLE(Status) (!!((Status) & \
 				  (DB_BAG | DB_SET | DB_DUPLICATE_BAG)))
-- 
cgit v1.2.3


From dc7048b07adc0bf7f6dc841bbb210d950e7a6ba9 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 28 Feb 2017 18:29:22 +0100
Subject: erts: Cleanup table status bits

---
 erts/emulator/beam/erl_db.c      |  2 +-
 erts/emulator/beam/erl_db_util.h | 24 +++++++++++-------------
 2 files changed, 12 insertions(+), 14 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 5d14530d84..9c597afabf 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1614,7 +1614,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 	BIF_ERROR(BIF_P, BADARG);
     }
 
-    status = DB_NORMAL | DB_SET | DB_PROTECTED;
+    status = DB_SET | DB_PROTECTED;
     keypos = 1;
     is_named = 0;
 #ifdef ERTS_SMP
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 198f339751..4fb285186e 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -260,19 +260,17 @@ typedef struct db_table_common {
 } DbTableCommon;
 
 /* These are status bit patterns */
-#define DB_NORMAL        (1 << 0)
-#define DB_PRIVATE       (1 << 1)
-#define DB_PROTECTED     (1 << 2)
-#define DB_PUBLIC        (1 << 3)
-#define DB_BAG           (1 << 4)
-#define DB_SET           (1 << 5)
-/*#define DB_LHASH         (1 << 6)*/
-#define DB_FINE_LOCKED   (1 << 7)  /* fine grained locking enabled */
-#define DB_DUPLICATE_BAG (1 << 8)
-#define DB_ORDERED_SET   (1 << 9)
-#define DB_DELETE        (1 << 10) /* table is being deleted */
-#define DB_FREQ_READ     (1 << 11)
-#define DB_NAMED_TABLE   (1 << 12)
+#define DB_PRIVATE       (1 << 0)
+#define DB_PROTECTED     (1 << 1)
+#define DB_PUBLIC        (1 << 2)
+#define DB_DELETE        (1 << 3) /* table is being deleted */
+#define DB_SET           (1 << 4)
+#define DB_BAG           (1 << 5)
+#define DB_DUPLICATE_BAG (1 << 6)
+#define DB_ORDERED_SET   (1 << 7)
+#define DB_FINE_LOCKED   (1 << 8) /* write_concurrency */
+#define DB_FREQ_READ     (1 << 9) /* read_concurrency */
+#define DB_NAMED_TABLE   (1 << 10)
 
 #define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET\
                               |DB_FINE_LOCKED|DB_FREQ_READ|DB_NAMED_TABLE)
-- 
cgit v1.2.3


From b9240f72148c536b038c0d26b1ef83da7c951429 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 2 Mar 2017 19:58:04 +0100
Subject: erts: Improve reduction count during table cleanup

---
 erts/emulator/beam/erl_db.c      | 76 ++++++++++++++++++++++------------------
 erts/emulator/beam/erl_db_hash.c | 30 ++++++++--------
 erts/emulator/beam/erl_db_hash.h |  2 +-
 erts/emulator/beam/erl_db_tree.c | 28 +++++++--------
 erts/emulator/beam/erl_db_util.h |  2 +-
 5 files changed, 73 insertions(+), 65 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 9c597afabf..2f188b5391 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -356,9 +356,9 @@ static void fix_table_locked(Process* p, DbTable* tb);
 static void unfix_table_locked(Process* p,  DbTable* tb, db_lock_kind_t* kind);
 static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data);
 static void free_heir_data(DbTable*);
-static void free_fixations_locked(Process* p, DbTable *tb);
+static SWord free_fixations_locked(Process* p, DbTable *tb);
 
-static int free_table_cont(Process *p, DbTable *tb);
+static SWord free_table_continue(Process *p, DbTable *tb, SWord reds);
 static void print_table(fmtfn_t to, void *to_arg, int show,  DbTable* tb);
 static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1);
 static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1);
@@ -393,8 +393,6 @@ free_dbtable(void *vtb)
 			 erts_smp_atomic_read_nob(&tb->common.memory_size)-sizeof(DbTable),
 			 tb->common.fixations);
 	}
-	erts_fprintf(stderr, "ets: free_dbtable(%T) deleted!!!\r\n",
-		     tb->common.id);
 #endif
 #ifdef ERTS_SMP
 	erts_smp_rwmtx_destroy(&tb->common.rwlock);
@@ -1904,7 +1902,8 @@ BIF_RETTYPE ets_lookup_element_3(BIF_ALIST_3)
  */
 BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 {
-    int trap;
+    SWord initial_reds = ERTS_BIF_REDS_LEFT(BIF_P);
+    SWord reds = initial_reds;
     DbTable* tb;
 
 #ifdef HARDDEBUG
@@ -1958,11 +1957,10 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
     free_heir_data(tb);
     tb->common.heir = am_none;
 
-    free_fixations_locked(BIF_P, tb);
+    reds -= free_fixations_locked(BIF_P, tb);
     db_unlock(tb, LCK_WRITE);
 
-    trap = free_table_cont(BIF_P, tb);
-    if (trap) {
+    if (free_table_continue(BIF_P, tb, reds) < 0) {
 	/*
 	 * Package the DbTable* pointer into a bignum so that it can be safely
 	 * passed through a trap. We used to pass the DbTable* pointer directly
@@ -1972,9 +1970,11 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 	Eterm *hp = HAlloc(BIF_P, 2);
 	hp[0] = make_pos_bignum_header(1);
 	hp[1] = (Eterm) tb;
+        BUMP_ALL_REDS(BIF_P);
 	BIF_TRAP1(&ets_delete_continue_exp, BIF_P, make_big(hp));
     }
     else {
+        BUMP_REDS(BIF_P, (initial_reds - reds));
 	BIF_RET(am_true);
     }
 }
@@ -3471,9 +3471,10 @@ send_ets_transfer_message(Process *c_p, Process *proc,
 
 
 /* Auto-release fixation from exiting process */
-static void proc_cleanup_fixed_table(Process* p, DbFixation* fix)
+static SWord proc_cleanup_fixed_table(Process* p, DbFixation* fix)
 {
     DbTable* tb = btid2tab(fix->tabs.btid);
+    SWord work = 0;
 
     ASSERT(fix->procs.p == p); (void)p;
     if (tb) {
@@ -3496,7 +3497,7 @@ static void proc_cleanup_fixed_table(Process* p, DbFixation* fix)
 	    erts_smp_mtx_unlock(&tb->common.fixlock);
     #endif
 	    if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)) {
-		db_unfix_table_hash(&(tb->hash));
+		work += db_unfix_table_hash(&(tb->hash));
 	    }
 
 	    ASSERT(sizeof(DbFixation) == ERTS_ALC_DBG_BLK_SZ(fix));
@@ -3516,6 +3517,9 @@ static void proc_cleanup_fixed_table(Process* p, DbFixation* fix)
     }
     erts_free(ERTS_ALC_T_DB_FIXATION, fix);
     ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation));
+    ++work;
+
+    return work;
 }
 
 
@@ -3543,6 +3547,8 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
     CleanupState *state = (CleanupState *) c_p->u.terminate;
     Eterm pid = c_p->common.id;
     CleanupState default_state;
+    SWord initial_reds = ERTS_BIF_REDS_LEFT(c_p);
+    SWord reds = initial_reds;
 
     if (!state) {
 	state = &default_state;
@@ -3588,17 +3594,17 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
                 remove_named_tab(tb, 0);
 
             free_heir_data(tb);
-            free_fixations_locked(c_p, tb);
+            reds -= free_fixations_locked(c_p, tb);
             db_unlock(tb, LCK_WRITE);
             state->op = FREE_OWNED_TABLE;
-            /* fall through */
+            break;
         }
         case FREE_OWNED_TABLE:
-            if (free_table_cont(c_p, state->tb))
+            reds = free_table_continue(c_p, state->tb, reds);
+            if (reds < 0)
                 goto yield;
 
             state->op = GET_OWNED_TABLE;
-
             break;
 
 	case UNFIX_TABLES: {
@@ -3612,20 +3618,21 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
                 if (state != &default_state)
                     erts_free(ERTS_ALC_T_DB_PROC_CLEANUP, state);
                 c_p->u.terminate = NULL;
+
+                BUMP_REDS(c_p, (initial_reds - reds));
                 return 0;
             }
 
             fixed_tabs_delete(c_p, fix);
-            proc_cleanup_fixed_table(c_p, fix);
+            reds -= proc_cleanup_fixed_table(c_p, fix);
 
-            BUMP_REDS(c_p, 10);
             break;
         }
 	default:
 	    ERTS_DB_INTERNAL_ERROR("Bad internal state");
         }
 
-    } while (ERTS_BIF_REDS_LEFT(c_p) > 0);
+    } while (reds > 0);
 
  yield:
 
@@ -3740,6 +3747,7 @@ struct free_fixations_ctx
 {
     Process* p;
     DbTable* tb;
+    SWord cnt;
 };
 
 static void free_fixations_op(DbFixation* fix, void* vctx)
@@ -3785,6 +3793,7 @@ static void free_fixations_op(DbFixation* fix, void* vctx)
 		     ctx->tb, (void *) fix, sizeof(DbFixation));
         ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation));
     }
+    ctx->cnt++;
 }
 
 #ifdef ERTS_SMP
@@ -3802,7 +3811,7 @@ int erts_db_execute_free_fixation(Process* p, DbFixation* fix)
 }
 #endif
 
-static void free_fixations_locked(Process* p, DbTable *tb)
+static SWord free_fixations_locked(Process* p, DbTable *tb)
 {
     struct free_fixations_ctx ctx;
 
@@ -3810,9 +3819,11 @@ static void free_fixations_locked(Process* p, DbTable *tb)
 
     ctx.p = p;
     ctx.tb = tb;
+    ctx.cnt = 0;
     fixing_procs_rbt_foreach_destroy(&tb->common.fixing_procs,
                                      free_fixations_op, &ctx);
     tb->common.fixing_procs = NULL;
+    return ctx.cnt;
 }
 
 static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data)
@@ -3876,42 +3887,40 @@ static void free_heir_data(DbTable* tb)
 
 static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1)
 {
-    Process *p = BIF_P;
+    SWord initial_reds = ERTS_BIF_REDS_LEFT(BIF_P);
+    SWord reds = initial_reds;
     Eterm cont = BIF_ARG_1;
-    int trap;
     Eterm* ptr = big_val(cont);
     DbTable *tb = *((DbTable **) (UWord) (ptr + 1));
 
     ASSERT(*ptr == make_pos_bignum_header(1));
 
-    trap = free_table_cont(p, tb);
-
-    if (trap) {
-	BIF_TRAP1(&ets_delete_continue_exp, p, cont);
+    if (free_table_continue(BIF_P, tb, reds) < 0) {
+        BUMP_ALL_REDS(BIF_P);
+        BIF_TRAP1(&ets_delete_continue_exp, BIF_P, cont);
     }
     else {
+        BUMP_REDS(BIF_P, (initial_reds - reds));
 	BIF_RET(am_true);
     }
 }
 
 
 /*
- * free_table_cont() returns 0 when done and !0 when more work is needed.
+ * free_table_continue() returns reductions left
+ * done if >= 0
+ * yield if < 0
  */
-static int free_table_cont(Process *p, DbTable *tb)
+static SWord free_table_continue(Process *p, DbTable *tb, SWord reds)
 {
-    Eterm result;
-
-    result = tb->common.meth->db_free_table_continue(tb);
+    reds = tb->common.meth->db_free_table_continue(tb, reds);
 
-    if (result == 0) {
+    if (reds < 0) {
 #ifdef HARDDEBUG
 	erts_fprintf(stderr,"ets: free_table_cont %T (continue begin)\r\n",
 		     tb->common.id);
 #endif
 	/* More work to be done. Let other processes work and call us again. */
-	BUMP_ALL_REDS(p);
-	return !0;
     }
     else {
 #ifdef HARDDEBUG
@@ -3921,9 +3930,8 @@ static int free_table_cont(Process *p, DbTable *tb)
 	/* Completely done - we will not get called again. */
         delete_owned_table(p, tb);
         table_dec_refc(tb, 0);
-	BUMP_REDS(p, 100);
-	return 0;
     }
+    return reds;
 }
 
 struct fixing_procs_info_ctx
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 781886a822..18405342da 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -418,7 +418,7 @@ static void db_print_hash(fmtfn_t to,
 			  DbTable *tbl);
 static int db_free_table_hash(DbTable *tbl);
 
-static int db_free_table_continue_hash(DbTable *tbl);
+static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds);
 
 
 static void db_foreach_offheap_hash(DbTable *,
@@ -576,9 +576,10 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel)
 ** Table interface routines ie what's called by the bif's 
 */
 
-void db_unfix_table_hash(DbTableHash *tb)
+SWord db_unfix_table_hash(DbTableHash *tb)
 {
     FixedDeletion* fixdel;
+    SWord work = 0;
 
     ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rwlocked(&tb->common.rwlock)
 		       || (erts_smp_lc_rwmtx_is_rlocked(&tb->common.rwlock)
@@ -599,7 +600,7 @@ restart:
 	    if (!IS_FIXED(tb)) {
 		goto restart; /* unfixed again! */
 	    }
-	    return;
+	    return work;
 	}
 	if (ix < NACTIVE(tb)) {
 	    bp = &BUCKET(tb, ix);
@@ -609,6 +610,7 @@ restart:
 		if (b->hvalue == INVALID_HASH) {
 		    *bp = b->next;
 		    free_term(tb, b);
+		    work++;
 		    b = *bp;
 		} else {
 		    bp = &b->next;
@@ -624,9 +626,12 @@ restart:
 		     (void *) fx,
 		     sizeof(FixedDeletion));
 	ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion));
+	work++;
     }
 
     /* ToDo: Maybe try grow/shrink the table as well */
+
+    return work;
 }
 
 int db_create_hash(Process *p, DbTable *tbl)
@@ -2080,19 +2085,17 @@ static void db_print_hash(fmtfn_t to, void *to_arg, int show, DbTable *tbl)
 /* release all memory occupied by a single table */
 static int db_free_table_hash(DbTable *tbl)
 {
-    while (!db_free_table_continue_hash(tbl))
+    while (db_free_table_continue_hash(tbl, ERTS_SWORD_MAX) < 0)
 	;
     return 0;
 }
 
-static int db_free_table_continue_hash(DbTable *tbl)
+static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds)
 {
     DbTableHash *tb = &tbl->hash;
-    int done;
     FixedDeletion* fixdel = (FixedDeletion*) erts_smp_atomic_read_acqb(&tb->fixdel);
     ERTS_SMP_LC_ASSERT(IS_TAB_WLOCKED(tb) || (tb->common.status & DB_DELETE));
 
-    done = 0;
     while (fixdel != NULL) {
 	FixedDeletion *fx = fixdel;
 
@@ -2102,22 +2105,21 @@ static int db_free_table_continue_hash(DbTable *tbl)
 		     (void *) fx,
 		     sizeof(FixedDeletion));
 	ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion));
-	if (++done >= 2*DELETE_RECORD_LIMIT) {
+	if (--reds < 0) {
 	    erts_smp_atomic_set_relb(&tb->fixdel, (erts_aint_t)fixdel);
-	    return 0;		/* Not done */
+	    return reds;		/* Not done */
 	}
     }
     erts_smp_atomic_set_relb(&tb->fixdel, (erts_aint_t)NULL);
 
-    done /= 2;
     while(tb->nslots != 0) {
-	done += 1 + EXT_SEGSZ/64 + free_seg(tb, 1);
+	reds -= EXT_SEGSZ/64 + free_seg(tb, 1);
 
 	/*
 	 * If we have done enough work, get out here.
 	 */
-	if (done >= DELETE_RECORD_LIMIT) {
-	    return 0;	/* Not done */
+	if (reds < 0) {
+	    return reds;	/* Not done */
 	}
     }
 #ifdef ERTS_SMP
@@ -2132,7 +2134,7 @@ static int db_free_table_continue_hash(DbTable *tbl)
     }
 #endif    
     ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size) == sizeof(DbTable));
-    return 1;			/* Done */
+    return reds;			/* Done */
 }
 
 
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index cd02418370..c340c72311 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -75,7 +75,7 @@ typedef struct db_table_hash {
 ** table types. The process is always an [in out] parameter.
 */
 void db_initialize_hash(void);
-void db_unfix_table_hash(DbTableHash *tb /* [in out] */);
+SWord db_unfix_table_hash(DbTableHash *tb);
 Uint db_kept_items_hash(DbTableHash *tb);
 
 /* Interface for meta pid table */
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 3dd06fd7bb..14a7451267 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -282,7 +282,7 @@ struct select_delete_context {
 static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key);
 static TreeDbTerm *linkout_object_tree(DbTableTree *tb, 
 				       Eterm object);
-static int do_free_tree_cont(DbTableTree *tb, int num_left);
+static SWord do_free_tree_continue(DbTableTree *tb, SWord reds);
 static void free_term(DbTableTree *tb, TreeDbTerm* p);
 static int balance_left(TreeDbTerm **this); 
 static int balance_right(TreeDbTerm **this); 
@@ -388,7 +388,7 @@ static void db_print_tree(fmtfn_t to, void *to_arg,
 			  int show, DbTable *tbl);
 static int db_free_table_tree(DbTable *tbl);
 
-static int db_free_table_continue_tree(DbTable *tbl);
+static SWord db_free_table_continue_tree(DbTable *tbl, SWord);
 
 static void db_foreach_offheap_tree(DbTable *,
 				    void (*)(ErlOffHeap *, void *),
@@ -1751,23 +1751,22 @@ static void db_print_tree(fmtfn_t to, void *to_arg,
 /* release all memory occupied by a single table */
 static int db_free_table_tree(DbTable *tbl)
 {
-    while (!db_free_table_continue_tree(tbl))
+    while (db_free_table_continue_tree(tbl, ERTS_SWORD_MAX) < 0)
 	;
     return 1;
 }
 
-static int db_free_table_continue_tree(DbTable *tbl)
+static SWord db_free_table_continue_tree(DbTable *tbl, SWord reds)
 {
     DbTableTree *tb = &tbl->tree;
-    int result;
 
     if (!tb->deletion) {
 	tb->static_stack.pos = 0;
 	tb->deletion = 1;
 	PUSH_NODE(&tb->static_stack, tb->root);
     }
-    result = do_free_tree_cont(tb, DELETE_RECORD_LIMIT);
-    if (result) {		/* Completely done. */
+    reds = do_free_tree_continue(tb, reds);
+    if (reds >= 0) {		/* Completely done. */
 	erts_db_free(ERTS_ALC_T_DB_STK,
 		     (DbTable *) tb,
 		     (void *) tb->static_stack.array,
@@ -1775,7 +1774,7 @@ static int db_free_table_continue_tree(DbTable *tbl)
 	ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size)
 	       == sizeof(DbTable));
     }
-    return result;
+    return reds;
 }
 
 static int db_delete_all_objects_tree(Process* p, DbTable* tbl)
@@ -2058,7 +2057,7 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern,
     return DB_ERROR_NONE;
 }
 
-static int do_free_tree_cont(DbTableTree *tb, int num_left)
+static SWord do_free_tree_continue(DbTableTree *tb, SWord reds)
 {
     TreeDbTerm *root;
     TreeDbTerm *p;
@@ -2077,15 +2076,14 @@ static int do_free_tree_cont(DbTableTree *tb, int num_left)
 		root = p;
 	    } else {
 		free_term(tb, root);
-		if (--num_left > 0) {
-		    break;
-		} else {
-		    return 0;	/* Done enough for now */
-		}
+		if (--reds < 0) {
+                    return reds;   /* Done enough for now */
+                }
+                break;
 	    }
 	}
     }
-    return 1;
+    return reds;
 }
 
 /*
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 4fb285186e..72298842d7 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -175,7 +175,7 @@ typedef struct db_table_method
 				 DbTable* db /* [in out] */ );
 
     int (*db_free_table)(DbTable* db /* [in out] */ );
-    int (*db_free_table_continue)(DbTable* db); /* [in out] */  
+    SWord (*db_free_table_continue)(DbTable* db, SWord reds);
     
     void (*db_print)(fmtfn_t to,
 		     void* to_arg, 
-- 
cgit v1.2.3


From 3fcf58b142e22adb754b312576570c904655f877 Mon Sep 17 00:00:00 2001
From: Deepak Goel <dgoel@mesosphere.com>
Date: Wed, 15 Feb 2017 15:13:38 -0800
Subject: This patch fixes the issue in which erlang fails to start if the
 hostname is 64 characters on a linux system.

---
 erts/emulator/beam/erl_mtrace.c         |  7 ++++---
 erts/emulator/drivers/common/inet_drv.c | 13 +++++++------
 2 files changed, 11 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c
index e275867928..bb6f8660f1 100644
--- a/erts/emulator/beam/erl_mtrace.c
+++ b/erts/emulator/beam/erl_mtrace.c
@@ -572,7 +572,7 @@ void erts_mtrace_pre_init(void)
 
 void erts_mtrace_init(char *receiver, char *nodename)
 {
-    char hostname[MAXHOSTNAMELEN];
+    char hostname[MAXHOSTNAMELEN + 1];
     char pid[21]; /* enough for a 64 bit number */
 
     socket_desc = ERTS_SOCK_INVALID_SOCKET;
@@ -613,9 +613,10 @@ void erts_mtrace_init(char *receiver, char *nodename)
 	}
 	tracep = trace_buffer;
 	endp = trace_buffer + TRACE_BUF_SZ;
-	if (erts_sock_gethostname(hostname, MAXHOSTNAMELEN) != 0)
+        /* gethostname requires that the len is max(hostname) + 1 */
+	if (erts_sock_gethostname(hostname, MAXHOSTNAMELEN + 1) != 0)
 	    hostname[0] = '\0';
-	hostname[MAXHOSTNAMELEN-1] = '\0';
+	hostname[MAXHOSTNAMELEN] = '\0';
 	sys_get_pid(pid, sizeof(pid));
 	write_trace_header(nodename ? nodename : "", pid, hostname);
 	erts_mtrace_update_heap_size();
diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 1885338ce5..b619e6ebb9 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -8345,10 +8345,10 @@ static ErlDrvData inet_start(ErlDrvPort port, int size, int protocol)
     return (ErlDrvData)desc;
 }
 
-
-#ifndef MAXHOSTNAMELEN
-#define MAXHOSTNAMELEN 256
-#endif
+/* MAXHOSTNAMELEN could be 64 or 255 depending
+on the platform. Instead, use INET_MAXHOSTNAMELEN
+which is always 255 across all platforms */
+#define INET_MAXHOSTNAMELEN 255
 
 /*
 ** common TCP/UDP/SCTP control command
@@ -8525,13 +8525,14 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf,
     }
 	
     case INET_REQ_GETHOSTNAME: { /* get host name */
-	char tbuf[MAXHOSTNAMELEN];
+	char tbuf[INET_MAXHOSTNAMELEN + 1];
 
 	DEBUGF(("inet_ctl(%ld): GETHOSTNAME\r\n", (long)desc->port)); 
 	if (len != 0)
 	    return ctl_error(EINVAL, rbuf, rsize);
 
-	if (IS_SOCKET_ERROR(sock_hostname(tbuf, MAXHOSTNAMELEN)))
+        /* gethostname requires len to be max(hostname) + 1 */
+	if (IS_SOCKET_ERROR(sock_hostname(tbuf, INET_MAXHOSTNAMELEN + 1)))
 	    return ctl_error(sock_errno(), rbuf, rsize);
 	return ctl_reply(INET_REP_OK, tbuf, strlen(tbuf), rbuf, rsize);
     }
-- 
cgit v1.2.3


From 5b7e50500f6cb3e680b277f871392ac706c5c1d7 Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Fri, 3 Jun 2016 00:39:01 +0100
Subject: ETS: Allow for matchspec-based replacement

---
 erts/emulator/beam/bif.tab       |   1 +
 erts/emulator/beam/erl_db.c      |  96 +++++++++
 erts/emulator/beam/erl_db.h      |   1 +
 erts/emulator/beam/erl_db_hash.c | 318 ++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_db_tree.c | 407 +++++++++++++++++++++++++++++++++++++--
 erts/emulator/beam/erl_db_util.h |   8 +
 6 files changed, 817 insertions(+), 14 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 6f50297fc5..4140938210 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -361,6 +361,7 @@ bif ets:select_reverse/1
 bif ets:select_reverse/2
 bif ets:select_reverse/3
 bif ets:select_delete/2
+bif ets:select_replace/2
 bif ets:match_spec_compile/1
 bif ets:match_spec_run_r/3
 
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 2f188b5391..81fe79a92e 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -362,6 +362,7 @@ static SWord free_table_continue(Process *p, DbTable *tb, SWord reds);
 static void print_table(fmtfn_t to, void *to_arg, int show,  DbTable* tb);
 static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1);
 static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1);
+static BIF_RETTYPE ets_select_replace_1(BIF_ALIST_1);
 static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1);
 static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1);
 static Eterm table_info(Process* p, DbTable* tb, Eterm What);
@@ -376,6 +377,7 @@ static BIF_RETTYPE ets_select3(Process* p, Eterm arg1, Eterm arg2, Eterm arg3);
  */
 Export ets_select_delete_continue_exp;
 Export ets_select_count_continue_exp;
+Export ets_select_replace_continue_exp;
 Export ets_select_continue_exp;
 
 /*
@@ -2922,6 +2924,95 @@ BIF_RETTYPE ets_select_count_2(BIF_ALIST_2)
     return result;
 }
 
+/*
+ ** This is for trapping, cannot be called directly.
+ */
+static BIF_RETTYPE ets_select_replace_1(BIF_ALIST_1)
+{
+    Process *p = BIF_P;
+    Eterm a1 = BIF_ARG_1;
+    BIF_RETTYPE result;
+    DbTable* tb;
+    int cret;
+    Eterm ret;
+    Eterm *tptr;
+    db_lock_kind_t kind = LCK_WRITE_REC;
+
+    CHECK_TABLES();
+    ASSERT(is_tuple(a1));
+    tptr = tuple_val(a1);
+    ASSERT(arityval(*tptr) >= 1);
+
+    if ((tb = db_get_table(p, tptr[1], DB_WRITE, kind)) == NULL) {
+        BIF_ERROR(p,BADARG);
+    }
+
+    cret = tb->common.meth->db_select_replace_continue(p,tb,a1,&ret);
+
+    if(!DID_TRAP(p,ret) && ITERATION_SAFETY(p,tb) != ITER_SAFE) {
+        unfix_table_locked(p, tb, &kind);
+    }
+
+    db_unlock(tb, kind);
+
+    switch (cret) {
+    case DB_ERROR_NONE:
+        ERTS_BIF_PREP_RET(result, ret);
+        break;
+    default:
+        ERTS_BIF_PREP_ERROR(result, p, BADARG);
+        break;
+    }
+    erts_match_set_release_result(p);
+
+    return result;
+}
+
+
+BIF_RETTYPE ets_select_replace_2(BIF_ALIST_2)
+{
+    BIF_RETTYPE result;
+    DbTable* tb;
+    int cret;
+    Eterm ret;
+    enum DbIterSafety safety;
+
+    CHECK_TABLES();
+
+    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+    safety = ITERATION_SAFETY(BIF_P,tb);
+    if (safety == ITER_UNSAFE) {
+        local_fix_table(tb);
+    }
+    cret = tb->common.meth->db_select_replace(BIF_P, tb, BIF_ARG_2, &ret);
+
+    if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) {
+        fix_table_locked(BIF_P,tb);
+    }
+    if (safety == ITER_UNSAFE) {
+        local_unfix_table(tb);
+    }
+    db_unlock(tb, LCK_WRITE_REC);
+
+    switch (cret) {
+    case DB_ERROR_NONE:
+        ERTS_BIF_PREP_RET(result, ret);
+        break;
+    case DB_ERROR_SYSRES:
+        ERTS_BIF_PREP_ERROR(result, BIF_P, SYSTEM_LIMIT);
+        break;
+    default:
+        ERTS_BIF_PREP_ERROR(result, BIF_P, BADARG);
+        break;
+    }
+
+    erts_match_set_release_result(BIF_P);
+
+    return result;
+}
+
 
 BIF_RETTYPE ets_select_reverse_3(BIF_ALIST_3)
 {
@@ -3334,6 +3425,11 @@ void init_db(ErtsDbSpinCount db_spin_count)
 			  am_ets, am_atom_put("count_trap",11), 1,
 			  &ets_select_count_1);
 
+    /* Non visual BIF to trap to. */
+    erts_init_trap_export(&ets_select_replace_continue_exp,
+                          am_ets, am_atom_put("replace_trap",11), 1,
+                          &ets_select_replace_1);
+
     /* Non visual BIF to trap to. */
     erts_init_trap_export(&ets_select_continue_exp,
 			  am_ets, am_atom_put("select_trap",11), 1,
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index cbf4b9e007..4ff9f224e8 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -122,6 +122,7 @@ extern int erts_ets_realloc_always_moves;  /* set in erl_init */
 extern int erts_ets_always_compress;  /* set in erl_init */
 extern Export ets_select_delete_continue_exp;
 extern Export ets_select_count_continue_exp;
+extern Export ets_select_replace_continue_exp;
 extern Export ets_select_continue_exp;
 extern erts_smp_atomic_t erts_ets_misc_mem_size;
 
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 18405342da..a4ca6dce77 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -288,6 +288,9 @@ static ERTS_INLINE Sint next_slot_w(DbTableHash* tb, Uint ix,
 #endif
 }
 
+#ifndef MIN
+#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
+#endif
 
 /*
  * Some special binary flags
@@ -403,6 +406,9 @@ static int db_select_count_hash(Process *p, DbTable *tbl,  Eterm tid,
 static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid,
 				 Eterm pattern, Eterm *ret);
 
+static int db_select_replace_hash(Process *p, DbTable *tbl,
+                                  Eterm pattern, Eterm *ret);
+
 static int db_select_continue_hash(Process *p, DbTable *tbl, 
 				   Eterm continuation, Eterm *ret);
 
@@ -411,6 +417,10 @@ static int db_select_count_continue_hash(Process *p, DbTable *tbl,
 
 static int db_select_delete_continue_hash(Process *p, DbTable *tbl,
 					  Eterm continuation, Eterm *ret);
+
+static int db_select_replace_continue_hash(Process *p, DbTable *tbl,
+                                           Eterm continuation, Eterm *ret);
+
 static int db_take_hash(Process *, DbTable *, Eterm, Eterm *);
 static void db_print_hash(fmtfn_t to,
 			  void *to_arg,
@@ -523,6 +533,8 @@ DbTableMethod db_hash =
     db_select_delete_continue_hash,
     db_select_count_hash,
     db_select_count_continue_hash,
+    db_select_replace_hash,
+    db_select_replace_continue_hash,
     db_take_hash,
     db_delete_all_objects_hash,
     db_free_table_hash,
@@ -1989,6 +2001,312 @@ static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
     return DB_ERROR_NONE;
 }
 
+static int db_select_replace_hash(Process *p,
+                                  DbTable *tbl,
+                                  Eterm pattern,
+                                  Eterm *ret)
+{
+    DbTableHash *tb = &tbl->hash;
+    struct mp_info mpi;
+    Uint slot_ix = 0;
+    HashDbTerm **current = NULL, **initial = NULL, **iter = NULL;
+    HashDbTerm *new = NULL, *next = NULL;
+    HashValue hval = INVALID_HASH;
+    unsigned current_list_pos = 0;
+    Eterm key;
+    Eterm match_res;
+    Eterm *hp;
+    int num_left = 1000;
+    Uint got = 0;
+    Eterm continuation;
+    int errcode;
+    Eterm mpb;
+    Eterm egot;
+    erts_smp_rwmtx_t* lck;
+
+#define RET_TO_BIF(Term,RetVal) do {		\
+	if (mpi.mp != NULL) {			\
+	    erts_bin_free(mpi.mp);		\
+	}					\
+	if (mpi.lists != mpi.dlists) {		\
+	    erts_free(ERTS_ALC_T_DB_SEL_LIST,	\
+		      (void *) mpi.lists);	\
+	}					\
+	*ret = (Term);				\
+	return RetVal;				\
+    } while(0)
+
+
+    if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) {
+        RET_TO_BIF(NIL,errcode);
+    }
+
+    if (!mpi.something_can_match) {
+        RET_TO_BIF(make_small(0), DB_ERROR_NONE);
+        /* can't possibly match anything */
+    }
+
+    if (!mpi.key_given) {
+        /* Run this code if pattern is variable or GETKEY(pattern)  */
+        /* is a variable                                            */
+        lck = WLOCK_HASH(tb,slot_ix);
+        current = &BUCKET(tb,slot_ix);
+    } else {
+        /* We have at least one */
+        slot_ix = mpi.lists[current_list_pos].ix;
+        lck = WLOCK_HASH(tb, slot_ix);
+        current = mpi.lists[current_list_pos++].bucket;
+        ASSERT(*current == BUCKET(tb,slot_ix));
+    }
+
+
+    initial = current;
+    for(;;) {
+        if ((*current) == NULL) {
+            if (mpi.key_given) {  /* Key is bound */
+                WUNLOCK_HASH(lck);
+                if (current_list_pos == mpi.num_lists) {
+                    goto done;
+                } else {
+                    slot_ix = mpi.lists[current_list_pos].ix;
+                    lck = WLOCK_HASH(tb, slot_ix);
+                    current = mpi.lists[current_list_pos].bucket;
+                    ASSERT(mpi.lists[current_list_pos].bucket == &BUCKET(tb,slot_ix));
+                    ++current_list_pos;
+                }
+            } else {
+                if ((slot_ix=next_slot_w(tb,slot_ix,&lck)) == 0) {
+                    goto done;
+                }
+                if (num_left <= 0) {
+                    WUNLOCK_HASH(lck);
+                    goto trap;
+                }
+                current = &BUCKET(tb,slot_ix);
+            }
+            initial = current;
+        }
+        else if ((*current)->hvalue == INVALID_HASH) {
+            current = &((*current)->next);
+        }
+        else if (tb->common.status & DB_BAG) {
+            match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0,
+                                        &(*current)->dbterm, NULL, 0);
+            if (is_value(match_res) &&
+                is_value(key = db_getkey(tb->common.keypos, match_res)) &&
+                eq(key, GETKEY(tb, (*current)->dbterm.tpl)))
+            {
+                // we need to make sure we don't end up with duplicate objects;
+                // it's quite inefficient
+                int object_exists = 0;
+                for (iter = initial; *iter != NULL; iter = &((*iter)->next))
+                    if (((*iter)->hvalue != INVALID_HASH)
+                            && (*iter != *current)
+                            && db_eq(&tb->common, match_res, &(*iter)->dbterm))
+                    {
+                        object_exists = 1;
+                        break;
+                    }
+
+                if (!object_exists) {
+                    next = (*current)->next;
+                    hval = (*current)->hvalue;
+                    new = replace_dbterm(tb, *current, match_res);
+                    new->next = next;
+                    new->hvalue = hval;
+                    *current = new;
+                    ++got;
+                }
+            }
+            --num_left;
+            current = &((*current)->next);
+        }
+        else {
+            match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0,
+                                        &(*current)->dbterm, NULL, 0);
+            if (is_value(match_res) &&
+                is_value(key = db_getkey(tb->common.keypos, match_res)) &&
+                eq(key, GETKEY(tb, (*current)->dbterm.tpl)))
+            {
+                next = (*current)->next;
+                hval = (*current)->hvalue;
+                new = replace_dbterm(tb, *current, match_res);
+                new->next = next;
+                new->hvalue = hval;
+                *current = new;
+                ++got;
+            }
+            --num_left;
+            current = &((*current)->next);
+        }
+    }
+done:
+    // the more objects we've replaced, the more reductions we've consumed
+    BUMP_REDS(p, MIN(2000, (1000 - num_left) + (int)got));
+    RET_TO_BIF(erts_make_integer(got,p),DB_ERROR_NONE);
+trap:
+    BUMP_ALL_REDS(p);
+    if (IS_USMALL(0, got)) {
+        hp = HAlloc(p,  PROC_BIN_SIZE + 5);
+        egot = make_small(got);
+    }
+    else {
+        hp = HAlloc(p, BIG_UINT_HEAP_SIZE + PROC_BIN_SIZE + 5);
+        egot = uint_to_big(got, hp);
+        hp += BIG_UINT_HEAP_SIZE;
+    }
+    mpb = db_make_mp_binary(p,mpi.mp,&hp);
+    continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix),
+            mpb,
+            egot);
+    mpi.mp = NULL; /*otherwise the return macro will destroy it */
+    RET_TO_BIF(bif_trap1(&ets_select_replace_continue_exp, p,
+                continuation),
+            DB_ERROR_NONE);
+
+#undef RET_TO_BIF
+
+}
+
+/*
+** This is called when select_replace traps
+*/
+static int db_select_replace_continue_hash(Process *p,
+					   DbTable *tbl,
+					   Eterm continuation,
+					   Eterm *ret)
+{
+    DbTableHash *tb = &tbl->hash;
+    Uint slot_ix;
+    HashDbTerm **current = NULL, **initial = NULL, **iter = NULL;
+    HashDbTerm *new = NULL, *next = NULL;
+    HashValue hval = INVALID_HASH;
+    Eterm key;
+    Eterm match_res;
+    Eterm *hp;
+    int num_left = 1000;
+    Uint got, prev_got;
+    Eterm *tptr;
+    Binary *mp;
+    Eterm egot;
+    erts_smp_rwmtx_t* lck;
+
+#define RET_TO_BIF(Term,RetVal) do {		\
+	*ret = (Term);				\
+	return RetVal;				\
+    } while(0)
+
+
+    tptr = tuple_val(continuation);
+    slot_ix = unsigned_val(tptr[2]);
+    mp = ((ProcBin *) binary_val(tptr[3]))->val;
+    if (is_big(tptr[4])) {
+        got = big_to_uint32(tptr[4]);
+    } else {
+        got = unsigned_val(tptr[4]);
+    }
+    prev_got = got;
+
+    lck = WLOCK_HASH(tb,slot_ix);
+    if (slot_ix >= NACTIVE(tb)) {
+        WUNLOCK_HASH(lck);
+        goto done;
+    }
+    current = &BUCKET(tb,slot_ix);
+    initial = current;
+
+    for(;;) {
+        if ((*current) == NULL) {
+            if ((slot_ix=next_slot_w(tb,slot_ix,&lck)) == 0) {
+                goto done;
+            }
+            if (num_left <= 0) {
+                WUNLOCK_HASH(lck);
+                goto trap;
+            }
+            current = &BUCKET(tb,slot_ix);
+            initial = current;
+        }
+        else if ((*current)->hvalue == INVALID_HASH) {
+            current = &((*current)->next);
+        }
+        else if (tb->common.status & DB_BAG) {
+            match_res = db_match_dbterm(&tb->common, p, mp, 0,
+                                        &(*current)->dbterm, NULL, 0);
+            if (is_value(match_res) &&
+                is_value(key = db_getkey(tb->common.keypos, match_res)) &&
+                eq(key, GETKEY(tb, (*current)->dbterm.tpl)))
+            {
+                // we need to make sure we don't end up with duplicate objects;
+                // it's quite inefficient
+                int object_exists = 0;
+                for (iter = initial; *iter != NULL; iter = &((*iter)->next))
+                    if (((*iter)->hvalue != INVALID_HASH)
+                            && (*iter != *current)
+                            && db_eq(&tb->common, match_res, &(*iter)->dbterm))
+                    {
+                        object_exists = 1;
+                        break;
+                    }
+
+                if (!object_exists) {
+                    next = (*current)->next;
+                    hval = (*current)->hvalue;
+                    new = replace_dbterm(tb, *current, match_res);
+                    new->next = next;
+                    new->hvalue = hval;
+                    *current = new;
+                    ++got;
+                }
+            }
+            --num_left;
+            current = &((*current)->next);
+        }
+        else {
+            match_res = db_match_dbterm(&tb->common, p, mp, 0,
+                                        &(*current)->dbterm, NULL, 0);
+            if (is_value(match_res) &&
+                is_value(key = db_getkey(tb->common.keypos, match_res)) &&
+                eq(key, GETKEY(tb, (*current)->dbterm.tpl)))
+            {
+                next = (*current)->next;
+                hval = (*current)->hvalue;
+                new = replace_dbterm(tb, *current, match_res);
+                new->next = next;
+                new->hvalue = hval;
+                *current = new;
+                ++got;
+            }
+            --num_left;
+            current = &((*current)->next);
+        }
+    }
+done:
+    // the more objects we've replaced, the more reductions we've consumed
+    BUMP_REDS(p, MIN(2000, (1000 - num_left) + (int)(got - prev_got)));
+    RET_TO_BIF(erts_make_integer(got,p),DB_ERROR_NONE);
+trap:
+    BUMP_ALL_REDS(p);
+    if (IS_USMALL(0, got)) {
+        hp = HAlloc(p,  5);
+        egot = make_small(got);
+    }
+    else {
+        hp = HAlloc(p, BIG_UINT_HEAP_SIZE + 5);
+        egot = uint_to_big(got, hp);
+        hp += BIG_UINT_HEAP_SIZE;
+    }
+    continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix),
+            tptr[3],
+            egot);
+    RET_TO_BIF(bif_trap1(&ets_select_replace_continue_exp, p,
+                continuation),
+            DB_ERROR_NONE);
+
+#undef RET_TO_BIF
+
+}
+
 /*
 ** Other interface routines (not directly coupled to one bif)
 */
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 14a7451267..af1296367a 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -76,9 +76,18 @@
      ((Dtt->pos) ? 			\
       (Dtt)->array[(Dtt)->pos - 1] : NULL)
 
-#define EMPTY_NODE(Dtt) (TOP_NODE(Dtt) == NULL)
+#define TOPN_NODE(Dtt, Pos)                   \
+     (((Pos) < Dtt->pos) ? 			\
+      (Dtt)->array[(Dtt)->pos - ((Pos) + 1)] : NULL)
+
+#define REPLACE_TOP_NODE(Dtt, Node)          \
+     if ((Dtt)->pos) (Dtt)->array[(Dtt)->pos - 1] = (Node)
 
+#define EMPTY_NODE(Dtt) (TOP_NODE(Dtt) == NULL)
 
+#ifndef MIN
+#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
+#endif
 
 /* Obtain table static stack if available. NULL if not.
 ** Must be released with release_stack()
@@ -225,9 +234,9 @@ struct mp_info {
     Eterm most;                 /* The highest matching key (possibly 
 				 * partially bound expression) */
 
-    TreeDbTerm *save_term;      /* If the key is completely bound, this
-				 * will be the Tree node we're searching
-				 * for, otherwise it will be useless */
+    TreeDbTerm **save_term;      /* If the key is completely bound, this
+	 			  * will be the Tree node we're searching
+				  * for, otherwise it will be useless */
     Binary *mp;                 /* The compiled match program */
 };
 
@@ -276,6 +285,21 @@ struct select_delete_context {
     int keypos;
 };
 
+/*
+ * Used by doit_select_replace
+ */
+struct select_replace_context {
+    Process *p;
+    DbTableTree *tb;
+    Binary *mp;
+    Eterm end_condition;
+    Eterm *lastobj;
+    Sint32 max;
+    int keypos;
+    int all_objects;
+    Sint replaced;
+};
+
 /*
 ** Forward declarations 
 */
@@ -290,6 +314,7 @@ static int delsub(TreeDbTerm **this);
 static TreeDbTerm *slot_search(Process *p, DbTableTree *tb, Sint slot);
 static TreeDbTerm *find_node(DbTableTree *tb, Eterm key);
 static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key);
+static TreeDbTerm **find_ptr(DbTableTree *tb, DbTreeStack*, TreeDbTerm *this);
 static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key);
 static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key);
 static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack*,
@@ -311,8 +336,16 @@ static void traverse_forward(DbTableTree *tb,
 					 TreeDbTerm *,
 					 void *,
 					 int),
-			     void *context); 
-static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret,
+			     void *context);
+static void traverse_update_backwards(DbTableTree *tb,
+                                      DbTreeStack*,
+                                      Eterm lastkey,
+                                      int (*doit)(DbTableTree *tb,
+                                                  TreeDbTerm **, // out
+                                                  void *,
+                                                  int),
+                                      void *context);
+static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm ***ret,
 		     Eterm *partly_bound_key);
 static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key);
 static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done);
@@ -335,6 +368,10 @@ static int doit_select_delete(DbTableTree *tb,
 			      TreeDbTerm *this,
 			      void *ptr,
 			      int forward);
+static int doit_select_replace(DbTableTree *tb,
+                               TreeDbTerm **this_ptr,
+                               void *ptr,
+                               int forward);
 
 static int partly_bound_can_match_lesser(Eterm partly_bound_1, 
 					 Eterm partly_bound_2);
@@ -383,6 +420,10 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm tid,
 				 Eterm pattern,  Eterm *ret);
 static int db_select_delete_continue_tree(Process *p, DbTable *tbl, 
 					  Eterm continuation, Eterm *ret);
+static int db_select_replace_tree(Process *p, DbTable *tbl,
+                                  Eterm pattern, Eterm *ret);
+static int db_select_replace_continue_tree(Process *p, DbTable *tbl,
+                                           Eterm continuation, Eterm *ret);
 static int db_take_tree(Process *, DbTable *, Eterm, Eterm *);
 static void db_print_tree(fmtfn_t to, void *to_arg,
 			  int show, DbTable *tbl);
@@ -435,6 +476,8 @@ DbTableMethod db_tree =
     db_select_delete_continue_tree,
     db_select_count_tree,
     db_select_count_continue_tree,
+    db_select_replace_tree,
+    db_select_replace_continue_tree,
     db_take_tree,
     db_delete_all_objects_tree,
     db_free_table_tree,
@@ -1103,7 +1146,7 @@ static int db_select_tree(Process *p, DbTable *tbl, Eterm tid,
 
     if (!mpi.got_partial && mpi.some_limitation && 
 	CMP_EQ(mpi.least,mpi.most)) {
-	doit_select(tb,mpi.save_term,&sc,0 /* direction doesn't matter */);
+	doit_select(tb,*(mpi.save_term),&sc,0 /* direction doesn't matter */);
 	RET_TO_BIF(sc.accum,DB_ERROR_NONE);
     }
 
@@ -1306,7 +1349,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, Eterm tid,
 
     if (!mpi.got_partial && mpi.some_limitation && 
 	CMP_EQ(mpi.least,mpi.most)) {
-	doit_select_count(tb,mpi.save_term,&sc,0 /* dummy */);
+	doit_select_count(tb,*(mpi.save_term),&sc,0 /* dummy */);
 	RET_TO_BIF(erts_make_integer(sc.got,p),DB_ERROR_NONE);
     }
 
@@ -1409,7 +1452,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid,
 
     if (!mpi.got_partial && mpi.some_limitation && 
 	CMP_EQ(mpi.least,mpi.most)) {
-	doit_select(tb,mpi.save_term,&sc, 0 /* direction doesn't matter */);
+	doit_select(tb,*(mpi.save_term),&sc, 0 /* direction doesn't matter */);
 	if (sc.accum != NIL) {
 	    hp=HAlloc(p, 3);
 	    RET_TO_BIF(TUPLE2(hp,sc.accum,am_EOT),DB_ERROR_NONE);
@@ -1650,7 +1693,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm tid,
 
     if (!mpi.got_partial && mpi.some_limitation && 
 	CMP_EQ(mpi.least,mpi.most)) {
-	doit_select_delete(tb,mpi.save_term,&sc, 0 /* direction doesn't 
+	doit_select_delete(tb,*(mpi.save_term),&sc, 0 /* direction doesn't
 						      matter */);
 	RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE);
     }
@@ -1702,6 +1745,212 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm tid,
 
 }
 
+static int db_select_replace_continue_tree(Process *p,
+                                           DbTable *tbl,
+                                           Eterm continuation,
+                                           Eterm *ret)
+{
+    DbTableTree *tb = &tbl->tree;
+    DbTreeStack* stack;
+    struct select_replace_context sc;
+    unsigned sz;
+    Eterm *hp;
+    Eterm lastkey;
+    Eterm end_condition;
+    Binary *mp;
+    Eterm key;
+    Eterm *tptr;
+    Eterm ereplaced;
+    Sint prev_replaced;
+
+
+#define RET_TO_BIF(Term, State) do { *ret = (Term); return State; } while(0);
+
+    /* Decode continuation. We know it's a tuple and everything else as
+       this is only called by ourselves */
+
+    /* continuation:
+       {Table, Lastkey, EndCondition, MatchProgBin, HowManyReplaced}*/
+
+    tptr = tuple_val(continuation);
+
+    if (arityval(*tptr) != 5)
+        erts_exit(ERTS_ERROR_EXIT,"Internal error in ets:select_replace/1");
+
+    lastkey = tptr[2];
+    end_condition = tptr[3];
+    if (!(thing_subtag(*binary_val(tptr[4])) == REFC_BINARY_SUBTAG))
+        RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
+    mp = ((ProcBin *) binary_val(tptr[4]))->val;
+    if (!IsMatchProgBinary(mp))
+        RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
+
+    sc.p = p;
+    sc.mp = mp;
+    sc.end_condition = NIL;
+    sc.lastobj = NULL;
+    sc.max = 1000;
+    sc.keypos = tb->common.keypos;
+    if (is_big(tptr[5])) {
+        sc.replaced = big_to_uint32(tptr[5]);
+    } else {
+        sc.replaced = unsigned_val(tptr[5]);
+    }
+    prev_replaced = sc.replaced;
+
+    stack = get_any_stack(tb);
+    traverse_update_backwards(tb, stack, lastkey, &doit_select_replace, &sc);
+    release_stack(tb,stack);
+
+    // the more objects we've replaced, the more reductions we've consumed
+    BUMP_REDS(p, MIN(2000, (1000 - sc.max) + (sc.replaced - prev_replaced)));
+
+    if (sc.max > 0) {
+        RET_TO_BIF(erts_make_integer(sc.replaced,p), DB_ERROR_NONE);
+    }
+    key = GETKEY(tb, sc.lastobj);
+    if (end_condition != NIL &&
+            (cmp_partly_bound(end_condition,key) > 0)) {
+        /* done anyway */
+        RET_TO_BIF(make_small(sc.replaced),DB_ERROR_NONE);
+    }
+    /* Not done yet, let's trap. */
+    sz = size_object(key);
+    if (IS_USMALL(0, sc.replaced)) {
+        hp = HAlloc(p, sz + 6);
+        ereplaced = make_small(sc.replaced);
+    }
+    else {
+        hp = HAlloc(p, BIG_UINT_HEAP_SIZE + sz + 6);
+        ereplaced = uint_to_big(sc.replaced, hp);
+        hp += BIG_UINT_HEAP_SIZE;
+    }
+    key = copy_struct(key, sz, &hp, &MSO(p));
+    continuation = TUPLE5
+        (hp,
+         tptr[1],
+         key,
+         tptr[3],
+         tptr[4],
+         ereplaced);
+    RET_TO_BIF(bif_trap1(&ets_select_replace_continue_exp, p, continuation),
+            DB_ERROR_NONE);
+
+#undef RET_TO_BIF
+}
+
+static int db_select_replace_tree(Process *p, DbTable *tbl,
+                                  Eterm pattern, Eterm *ret)
+{
+    DbTableTree *tb = &tbl->tree;
+    DbTreeStack* stack;
+    struct select_replace_context sc;
+    struct mp_info mpi;
+    Eterm lastkey = THE_NON_VALUE;
+    Eterm key;
+    Eterm continuation;
+    unsigned sz;
+    Eterm *hp;
+    TreeDbTerm *this;
+    int errcode;
+    Eterm ereplaced;
+    Eterm mpb;
+
+
+#define RET_TO_BIF(Term,RetVal) do { 	       	\
+	if (mpi.mp != NULL) {			\
+	    erts_bin_free(mpi.mp);       	\
+	}					\
+	*ret = (Term); 				\
+	return RetVal; 			        \
+    } while(0)
+
+    mpi.mp = NULL;
+
+    sc.lastobj = NULL;
+    sc.p = p;
+    sc.tb = tb;
+    sc.max = 1000;
+    sc.end_condition = NIL;
+    sc.keypos = tb->common.keypos;
+    sc.replaced = 0;
+
+    if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) {
+        RET_TO_BIF(NIL,errcode);
+    }
+
+    if (!mpi.something_can_match) {
+        RET_TO_BIF(make_small(0),DB_ERROR_NONE);
+        /* can't possibly match anything */
+    }
+
+    sc.mp = mpi.mp;
+    sc.all_objects = mpi.all_objects;
+
+    stack = get_static_stack(tb);
+    if (!mpi.got_partial && mpi.some_limitation &&
+            CMP_EQ(mpi.least,mpi.most)) {
+        TreeDbTerm* term = *(mpi.save_term);
+        doit_select_replace(tb,mpi.save_term,&sc,0 /* dummy */);
+        if (stack != NULL) {
+            if (TOP_NODE(stack) == term)
+                // throw away potentially invalid reference
+                REPLACE_TOP_NODE(stack, *(mpi.save_term));
+            release_stack(tb, stack);
+        }
+        RET_TO_BIF(erts_make_integer(sc.replaced,p),DB_ERROR_NONE);
+    }
+
+    if (stack == NULL)
+        stack = get_any_stack(tb);
+
+    if (mpi.some_limitation) {
+        if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) {
+            lastkey = GETKEY(tb, this->dbterm.tpl);
+        }
+        sc.end_condition = mpi.least;
+    }
+
+    traverse_update_backwards(tb, stack, lastkey, &doit_select_replace, &sc);
+    release_stack(tb,stack);
+    // the more objects we've replaced, the more reductions we've consumed
+    BUMP_REDS(p, MIN(2000, (1000 - sc.max) + sc.replaced));
+    if (sc.max > 0) {
+        RET_TO_BIF(erts_make_integer(sc.replaced,p),DB_ERROR_NONE);
+    }
+
+    key = GETKEY(tb, sc.lastobj);
+    sz = size_object(key);
+    if (IS_USMALL(0, sc.replaced)) {
+        hp = HAlloc(p, sz + PROC_BIN_SIZE + 6);
+        ereplaced = make_small(sc.replaced);
+    }
+    else {
+        hp = HAlloc(p, BIG_UINT_HEAP_SIZE + sz + PROC_BIN_SIZE + 6);
+        ereplaced = uint_to_big(sc.replaced, hp);
+        hp += BIG_UINT_HEAP_SIZE;
+    }
+    key = copy_struct(key, sz, &hp, &MSO(p));
+    if (mpi.all_objects)
+        (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
+    mpb = db_make_mp_binary(p,mpi.mp,&hp);
+
+    continuation = TUPLE5
+        (hp,
+         tb->common.id,
+         key,
+         sc.end_condition, /* From the match program, needn't be copied */
+         mpb,
+         ereplaced);
+
+    /* Don't free mpi.mp, so don't use macro */
+    *ret = bif_trap1(&ets_select_replace_continue_exp, p, continuation);
+    return DB_ERROR_NONE;
+
+#undef RET_TO_BIF
+
+}
+
 static int db_take_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
 {
     DbTableTree *tb = &tbl->tree;
@@ -2011,7 +2260,7 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern,
 	++i;
 
 	partly_bound = NIL;
-	res = key_given(tb, tpl, &mpi->save_term, &partly_bound);
+	res = key_given(tb, tpl, &(mpi->save_term), &partly_bound);
 	if ( res >= 0 ) {   /* Can match something */
 	    key = 0;
 	    mpi->something_can_match = 1;
@@ -2514,6 +2763,58 @@ static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key)
     return this;
 }
 
+/*
+ * Find node and return the address of the node pointer (NULL if not found)
+ * Tries to reuse the existing stack for performance.
+ */
+
+static TreeDbTerm **find_ptr(DbTableTree *tb, DbTreeStack *stack, TreeDbTerm *this) {
+    Eterm key = GETKEY(tb, this->dbterm.tpl);
+    TreeDbTerm *tmp;
+    TreeDbTerm *parent;
+    Sint c;
+
+    if(( tmp = TOP_NODE(stack)) != NULL) {
+	if (!cmp_key_eq(tb,key,tmp)) {
+	    /* Start from the beginning */
+	    stack->pos = stack->slot = 0;
+	}
+    }
+    if (EMPTY_NODE(stack)) { /* Have to rebuild the stack */
+	if (( tmp = tb->root ) == NULL)
+	    return NULL;
+	for (;;) {
+	    PUSH_NODE(stack, tmp);
+	    if (( c = cmp_key(tb,key,tmp) ) < 0) {
+		if (tmp->left == NULL) /* We are at the next
+					   and the element does
+					   not exist */
+		    break;
+		else
+		    tmp = tmp->left;
+	    } else if (c > 0) {
+		if (tmp->right == NULL) /* Done */
+		    return NULL;
+		else
+		    tmp = tmp->right;
+	    } else
+		break;
+	}
+    }
+
+    if (TOP_NODE(stack) != this)
+        return NULL;
+
+    parent = TOPN_NODE(stack, 1);
+    if (parent == NULL)
+        return ((this != tb->root) ? NULL : &(tb->root));
+    if (parent->left == this)
+        return &(parent->left);
+    if (parent->right == this)
+        return &(parent->right);
+    return NULL;
+}
+
 static int
 db_lookup_dbterm_tree(Process *p, DbTable *tbl, Eterm key, Eterm obj,
                       DbUpdateHandle* handle)
@@ -2654,14 +2955,61 @@ static void traverse_forward(DbTableTree *tb,
     }
 }
 
+/*
+ * Traverse the tree with an update callback function, used by db_select_replace
+ */
+static void traverse_update_backwards(DbTableTree *tb,
+                                      DbTreeStack* stack,
+                                      Eterm lastkey,
+                                      int (*doit)(DbTableTree*,
+                                                  TreeDbTerm**,
+                                                  void*,
+                                                  int),
+                                      void* context)
+{
+    int res;
+    TreeDbTerm *this, *next, **this_ptr;
+
+    if (lastkey == THE_NON_VALUE) {
+        stack->pos = stack->slot = 0;
+        if (( this = tb->root ) == NULL) {
+            return;
+        }
+        while (this != NULL) {
+            PUSH_NODE(stack, this);
+            this = this->right;
+        }
+        this = TOP_NODE(stack);
+        this_ptr = find_ptr(tb, stack, this);
+        ASSERT(this_ptr != NULL);
+        res = (*doit)(tb, this_ptr, context, 0);
+        REPLACE_TOP_NODE(stack, *this_ptr);
+        next = find_prev(tb, stack, GETKEY(tb, (*this_ptr)->dbterm.tpl));
+        if (!res)
+            return;
+    } else {
+        next = find_prev(tb, stack, lastkey);
+    }
+
+    while ((this = next) != NULL) {
+        this_ptr = find_ptr(tb, stack, this);
+        ASSERT(this_ptr != NULL);
+        res = (*doit)(tb, this_ptr, context, 0);
+        REPLACE_TOP_NODE(stack, *this_ptr);
+        next = find_prev(tb, stack, GETKEY(tb, (*this_ptr)->dbterm.tpl));
+        if (!res)
+            return;
+    }
+}
+
 /*
  * Returns 0 if not given 1 if given and -1 on no possible match
  * if key is given; *ret is set to point to the object concerned.
  */
-static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, 
+static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm ***ret,
 		     Eterm *partly_bound)
 {
-    TreeDbTerm *this;
+    TreeDbTerm **this;
     Eterm key;
 
     ASSERT(ret != NULL);
@@ -2671,7 +3019,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret,
     if (is_non_value(key))
 	return -1;  /* can't possibly match anything */
     if (!db_has_variable(key)) {   /* Bound key */
-	if (( this = find_node(tb, key) ) == NULL) {
+	if (( this = find_node2(tb, key) ) == NULL) {
 	    return -1;
 	}
 	*ret = this;
@@ -3094,6 +3442,37 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr,
     return 1;
 }
 
+static int doit_select_replace(DbTableTree *tb, TreeDbTerm **this, void *ptr,
+                               int forward)
+{
+    struct select_replace_context *sc = (struct select_replace_context *) ptr;
+    Eterm ret, key;
+
+    sc->lastobj = (*this)->dbterm.tpl;
+
+    /* Always backwards traversing */
+    if (sc->end_condition != NIL &&
+	(cmp_partly_bound(sc->end_condition,
+			  GETKEY_WITH_POS(sc->keypos, (*this)->dbterm.tpl)) > 0)) {
+	return 0;
+    }
+    ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0,
+			  &(*this)->dbterm, NULL, 0);
+
+    if (is_value(ret) &&
+        is_value(key = db_getkey(tb->common.keypos, ret)) &&
+        (cmp_key(tb, key, *this) == 0))
+    {
+        *this = replace_dbterm(tb, *this, ret);
+        sc->lastobj = (*this)->dbterm.tpl;
+        ++(sc->replaced);
+    }
+    if (--(sc->max) <= 0) {
+	return 0;
+    }
+    return 1;
+}
+
 #ifdef TREE_DEBUG
 static void do_dump_tree2(DbTableTree* tb, int to, void *to_arg, int show,
 			  TreeDbTerm *t, int offset)
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 72298842d7..89ef79d2dc 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -169,6 +169,14 @@ typedef struct db_table_method
 				    DbTable* tb, /* [in out] */ 
 				    Eterm continuation, 
 				    Eterm* ret);
+    int (*db_select_replace)(Process* p,
+            DbTable* tb, /* [in out] */
+            Eterm pattern,
+            Eterm* ret);
+    int (*db_select_replace_continue)(Process* p,
+            DbTable* tb, /* [in out] */
+            Eterm continuation,
+            Eterm* ret);
     int (*db_take)(Process *, DbTable *, Eterm, Eterm *);
 
     int (*db_delete_all_objects)(Process* p,
-- 
cgit v1.2.3


From e15319fcdb5c99514cd63d7a02d04c97587e8853 Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Mon, 27 Jun 2016 22:09:37 +0100
Subject: Disable ets:select_replace/2 for bags

The existing implementation presented both
semantic inconsistencies and performance issues.
---
 erts/emulator/beam/erl_db.c      |  8 +++++
 erts/emulator/beam/erl_db_hash.c | 74 +++-------------------------------------
 2 files changed, 12 insertions(+), 70 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 81fe79a92e..d77c585628 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -2982,6 +2982,14 @@ BIF_RETTYPE ets_select_replace_2(BIF_ALIST_2)
     if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
         BIF_ERROR(BIF_P, BADARG);
     }
+
+    if (tb->common.status & DB_BAG) {
+        /* Bag implementation presented both semantic consistency
+           and performance issues */
+        db_unlock(tb, LCK_WRITE_REC);
+        BIF_ERROR(BIF_P, BADARG);
+    }
+
     safety = ITERATION_SAFETY(BIF_P,tb);
     if (safety == ITER_UNSAFE) {
         local_fix_table(tb);
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index a4ca6dce77..96d74bb050 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -2009,7 +2009,7 @@ static int db_select_replace_hash(Process *p,
     DbTableHash *tb = &tbl->hash;
     struct mp_info mpi;
     Uint slot_ix = 0;
-    HashDbTerm **current = NULL, **initial = NULL, **iter = NULL;
+    HashDbTerm **current = NULL;
     HashDbTerm *new = NULL, *next = NULL;
     HashValue hval = INVALID_HASH;
     unsigned current_list_pos = 0;
@@ -2036,6 +2036,8 @@ static int db_select_replace_hash(Process *p,
 	return RetVal;				\
     } while(0)
 
+    /* Bag implementation presented both semantic consistency and performance issues */
+    ASSERT(!(tb->common.status & DB_BAG));
 
     if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) {
         RET_TO_BIF(NIL,errcode);
@@ -2060,7 +2062,6 @@ static int db_select_replace_hash(Process *p,
     }
 
 
-    initial = current;
     for(;;) {
         if ((*current) == NULL) {
             if (mpi.key_given) {  /* Key is bound */
@@ -2084,43 +2085,10 @@ static int db_select_replace_hash(Process *p,
                 }
                 current = &BUCKET(tb,slot_ix);
             }
-            initial = current;
         }
         else if ((*current)->hvalue == INVALID_HASH) {
             current = &((*current)->next);
         }
-        else if (tb->common.status & DB_BAG) {
-            match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0,
-                                        &(*current)->dbterm, NULL, 0);
-            if (is_value(match_res) &&
-                is_value(key = db_getkey(tb->common.keypos, match_res)) &&
-                eq(key, GETKEY(tb, (*current)->dbterm.tpl)))
-            {
-                // we need to make sure we don't end up with duplicate objects;
-                // it's quite inefficient
-                int object_exists = 0;
-                for (iter = initial; *iter != NULL; iter = &((*iter)->next))
-                    if (((*iter)->hvalue != INVALID_HASH)
-                            && (*iter != *current)
-                            && db_eq(&tb->common, match_res, &(*iter)->dbterm))
-                    {
-                        object_exists = 1;
-                        break;
-                    }
-
-                if (!object_exists) {
-                    next = (*current)->next;
-                    hval = (*current)->hvalue;
-                    new = replace_dbterm(tb, *current, match_res);
-                    new->next = next;
-                    new->hvalue = hval;
-                    *current = new;
-                    ++got;
-                }
-            }
-            --num_left;
-            current = &((*current)->next);
-        }
         else {
             match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0,
                                         &(*current)->dbterm, NULL, 0);
@@ -2178,7 +2146,7 @@ static int db_select_replace_continue_hash(Process *p,
 {
     DbTableHash *tb = &tbl->hash;
     Uint slot_ix;
-    HashDbTerm **current = NULL, **initial = NULL, **iter = NULL;
+    HashDbTerm **current = NULL;
     HashDbTerm *new = NULL, *next = NULL;
     HashValue hval = INVALID_HASH;
     Eterm key;
@@ -2213,7 +2181,6 @@ static int db_select_replace_continue_hash(Process *p,
         goto done;
     }
     current = &BUCKET(tb,slot_ix);
-    initial = current;
 
     for(;;) {
         if ((*current) == NULL) {
@@ -2225,43 +2192,10 @@ static int db_select_replace_continue_hash(Process *p,
                 goto trap;
             }
             current = &BUCKET(tb,slot_ix);
-            initial = current;
         }
         else if ((*current)->hvalue == INVALID_HASH) {
             current = &((*current)->next);
         }
-        else if (tb->common.status & DB_BAG) {
-            match_res = db_match_dbterm(&tb->common, p, mp, 0,
-                                        &(*current)->dbterm, NULL, 0);
-            if (is_value(match_res) &&
-                is_value(key = db_getkey(tb->common.keypos, match_res)) &&
-                eq(key, GETKEY(tb, (*current)->dbterm.tpl)))
-            {
-                // we need to make sure we don't end up with duplicate objects;
-                // it's quite inefficient
-                int object_exists = 0;
-                for (iter = initial; *iter != NULL; iter = &((*iter)->next))
-                    if (((*iter)->hvalue != INVALID_HASH)
-                            && (*iter != *current)
-                            && db_eq(&tb->common, match_res, &(*iter)->dbterm))
-                    {
-                        object_exists = 1;
-                        break;
-                    }
-
-                if (!object_exists) {
-                    next = (*current)->next;
-                    hval = (*current)->hvalue;
-                    new = replace_dbterm(tb, *current, match_res);
-                    new->next = next;
-                    new->hvalue = hval;
-                    *current = new;
-                    ++got;
-                }
-            }
-            --num_left;
-            current = &((*current)->next);
-        }
         else {
             match_res = db_match_dbterm(&tb->common, p, mp, 0,
                                         &(*current)->dbterm, NULL, 0);
-- 
cgit v1.2.3


From 3323f01156a698e2d21c2bb1c51b58cf47444aa0 Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Sun, 29 Jan 2017 20:20:34 +0000
Subject: Deduplicate select* code on ETS hash tables

Refactor existing solution into a common iteration loop parameterized
using stateful callbacks.
---
 erts/emulator/beam/erl_db_hash.c | 1992 +++++++++++++++++++-------------------
 erts/emulator/beam/erl_db_tree.c |    3 +-
 2 files changed, 1015 insertions(+), 980 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 96d74bb050..1b5c8fe422 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -355,8 +355,6 @@ static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb,
 	erts_smp_atomic_set_nob(&tb->segtab, (erts_aint_t) segtab);
 }
 
-
-
 /*
 ** Forward decl's (static functions)
 */
@@ -401,23 +399,21 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid,
 				int reverse, Eterm *ret);
 static int db_select_hash(Process *p, DbTable *tbl, Eterm tid,
 			  Eterm pattern, int reverse, Eterm *ret);
-static int db_select_count_hash(Process *p, DbTable *tbl,  Eterm tid,
-				Eterm pattern, Eterm *ret);
-static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid,
-				 Eterm pattern, Eterm *ret);
-
-static int db_select_replace_hash(Process *p, DbTable *tbl,
-                                  Eterm pattern, Eterm *ret);
-
-static int db_select_continue_hash(Process *p, DbTable *tbl, 
+static int db_select_continue_hash(Process *p, DbTable *tbl,
 				   Eterm continuation, Eterm *ret);
 
+static int db_select_count_hash(Process *p, DbTable *tbl, Eterm tid,
+				Eterm pattern, Eterm *ret);
 static int db_select_count_continue_hash(Process *p, DbTable *tbl,
 					 Eterm continuation, Eterm *ret);
 
+static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid,
+				 Eterm pattern, Eterm *ret);
 static int db_select_delete_continue_hash(Process *p, DbTable *tbl,
 					  Eterm continuation, Eterm *ret);
 
+static int db_select_replace_hash(Process *p, DbTable *tbl,
+                                  Eterm pattern, Eterm *ret);
 static int db_select_replace_continue_hash(Process *p, DbTable *tbl,
                                            Eterm continuation, Eterm *ret);
 
@@ -1151,816 +1147,1093 @@ static BIF_RETTYPE bif_trap1(Export *bif,
 {
     BIF_TRAP1(bif, p, p1);
 }
-    
-/*
- * Continue collecting select matches, this may happen either due to a trap
- * or when the user calls ets:select/1
- */
-static int db_select_continue_hash(Process *p, 
-				   DbTable *tbl, 
-				   Eterm continuation, 
-				   Eterm *ret)
-{
-    DbTableHash *tb = &tbl->hash;
-    Sint slot_ix; 
-    Sint save_slot_ix;
-    Sint chunk_size;
-    int all_objects;
-    Binary *mp;
-    int num_left = 1000;
-    HashDbTerm *current = 0;
-    Eterm match_list;
-    Eterm *hp;
-    Eterm match_res;
-    Sint got;
-    Eterm *tptr;
-    erts_smp_rwmtx_t* lck;
-
-#define RET_TO_BIF(Term, State) do { *ret = (Term); return State; } while(0);
-
-    /* Decode continuation. We know it's a tuple but not the arity or anything else */
-
-    tptr = tuple_val(continuation);
-
-    if (arityval(*tptr) != 6)
-	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
-    
-    if (!is_small(tptr[2]) || !is_small(tptr[3]) || 
-	!(is_list(tptr[5]) || tptr[5] == NIL) || !is_small(tptr[6]))
-	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
-    if ((chunk_size = signed_val(tptr[3])) < 0)
-	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
-    mp = erts_db_get_match_prog_binary(tptr[4]);
-    if (!mp)
-	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
-    all_objects = mp->flags & BIN_FLAG_ALL_OBJECTS;
-    match_list = tptr[5];
-    if ((got = signed_val(tptr[6])) < 0)
-	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
-
-    slot_ix = signed_val(tptr[2]);
-    if (slot_ix < 0 /* EOT */ 
-	|| (chunk_size && got >= chunk_size)) {       
-	goto done; /* Already got all or enough in the match_list */
-    }
-
-    lck = RLOCK_HASH(tb,slot_ix);
-    if (slot_ix >= NACTIVE(tb)) {
-	RUNLOCK_HASH(lck);	
-	RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
-    }
-
-    while ((current = BUCKET(tb,slot_ix)) == NULL) {
-	slot_ix = next_slot(tb, slot_ix, &lck);
-	if (slot_ix == 0) {
-	    slot_ix = -1; /* EOT */
-	    goto done;	   
-	}
-    }	  
-    for(;;) {
-	if (current->hvalue != INVALID_HASH && 
-	    (match_res = db_match_dbterm(&tb->common, p, mp, all_objects,
-					 &current->dbterm, &hp, 2),
-	     is_value(match_res))) {
-
-	    match_list = CONS(hp, match_res, match_list);
-	    ++got;
-	}
-
-	--num_left;
-	save_slot_ix = slot_ix;
-	if ((current = next(tb, (Uint*)&slot_ix, &lck, current)) == NULL) {
-	    slot_ix = -1; /* EOT */
-	    break;
-	}
-	if (slot_ix != save_slot_ix) { 
-	    if (chunk_size && got >= chunk_size) {
-		RUNLOCK_HASH(lck);
-		break;
-	    }    
-	    if (num_left <= 0 || MBUF(p)) {
-		/*
-		 * We have either reached our limit, or just created some heap fragments.
-		 * Since many heap fragments will make the GC slower, trap and GC now.
-		 */
-		RUNLOCK_HASH(lck);
-		goto trap;
-	    }
-	}
-    }
-done:
-    BUMP_REDS(p, 1000 - num_left);
-    if (chunk_size) {
-	Eterm continuation;
-	Eterm rest = NIL;
-	Sint rest_size = 0;
-
-	if (got > chunk_size) { /* Cannot write destructively here, 
-				   the list may have 
-				   been in user space */
-	    rest = NIL;
-	    hp = HAlloc(p, (got - chunk_size) * 2); 
-	    while (got-- > chunk_size) {
-		rest = CONS(hp, CAR(list_val(match_list)), rest);
-		hp += 2;
-		match_list = CDR(list_val(match_list));
-		++rest_size;
-	    }
-	}
-	if (rest != NIL || slot_ix >= 0) {
-	    hp = HAlloc(p,3+7);
-	    continuation = TUPLE6(hp, tptr[1], make_small(slot_ix), 
-				  tptr[3], tptr[4], rest, 
-				  make_small(rest_size));
-	    hp += 7;
-	    RET_TO_BIF(TUPLE2(hp, match_list, continuation),DB_ERROR_NONE);
-	} else {
-	    if (match_list != NIL) {
-		hp = HAlloc(p, 3);
-		RET_TO_BIF(TUPLE2(hp, match_list, am_EOT),DB_ERROR_NONE);
-	    } else {
-		RET_TO_BIF(am_EOT, DB_ERROR_NONE);
-	    }
-	}
-    }
-    RET_TO_BIF(match_list,DB_ERROR_NONE);
-
-trap:
-    BUMP_ALL_REDS(p);
-
-    hp = HAlloc(p,7);
-    continuation = TUPLE6(hp, tptr[1], make_small(slot_ix), tptr[3],
-			  tptr[4], match_list, make_small(got));
-    RET_TO_BIF(bif_trap1(&ets_select_continue_exp, p, 
-			 continuation), 
-	       DB_ERROR_NONE);
 
-#undef RET_TO_BIF
 
-}
+/*
+ * Match traversal callbacks
+ */
 
-static int db_select_hash(Process *p, DbTable *tbl, Eterm tid,
-			  Eterm pattern, int reverse,
-			  Eterm *ret)
-{
-    return db_select_chunk_hash(p, tbl, tid, pattern, 0, reverse, ret);
-}
+/* Called when no match is possible.
+ *      context_ptr: Pointer to context
+ *      ret: Pointer to traversal function term return.
+ *
+ * Both the direct return value and 'ret' are used as the traversal function return values.
+ */
+typedef int (*mtraversal_on_nothing_can_match_t)(void* context_ptr, Eterm* ret);
+
+/* Called for each match result.
+ *      context_ptr: Pointer to context
+ *      slot_ix: Current slot index
+ *      current_ptr_ptr: Triple pointer to either the bucket or the 'next' pointer in the previous element;
+ *                       can be (carefully) used to adjust iteration when deleting or replacing elements.
+ *      match_res: The result of running the match program against the current term.
+ *
+ * Should return 1 for successful match, 0 otherwise.
+ */
+typedef int (*mtraversal_on_match_res_t)(void* context_ptr, Sint slot_ix, HashDbTerm*** current_ptr_ptr,
+                                         Eterm match_res);
+
+/* Called when either we've matched enough elements in this cycle or EOT was reached.
+ *      context_ptr: Pointer to context
+ *      slot_ix: Current slot index
+ *      got: How many elements have been matched so far
+ *      iterations_left: Number of intended iterations (down from an initial max.) left in this traversal cycle
+ *      mpp: Double pointer to the compiled match program
+ *      ret: Pointer to traversal function term return.
+ *
+ * Both the direct return value and 'ret' are used as the traversal function return values.
+ * If *mpp is set to NULL, it won't be deallocated (useful for trapping.)
+ */
+typedef int (*mtraversal_on_loop_ended_t)(void* context_ptr, Sint slot_ix, Sint got,
+                                          Sint iterations_left, Binary** mpp, Eterm* ret);
+
+/* Called when it's time to trap
+ *      context_ptr: Pointer to context
+ *      slot_ix: Current slot index
+ *      got: How many elements have been matched so far
+ *      mpp: Double pointer to the compiled match program
+ *      ret: Pointer to traversal function term return.
+ *
+ * Both the direct return value and 'ret' are used as the traversal function return values.
+ * If *mpp is set to NULL, it won't be deallocated (useful for trapping.)
+ */
+typedef int (*mtraversal_on_trap_t)(void* context_ptr, Sint slot_ix, Sint got, Binary** mpp, Eterm* ret);
 
-static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid,
-				Eterm pattern, Sint chunk_size, 
-				int reverse, /* not used */
-				Eterm *ret)
+/*
+ * Begin hash table match traversal
+ */
+static int match_traverse(Process* p, DbTableHash* tb, Eterm pattern,
+                          Sint chunk_size,      /* If 0, no chunking */
+                          Sint iterations_left, /* Nr. of iterations left */
+                          Eterm** hpp,          /* Heap */
+                          int lock_for_write,   /* Set to 1 if we're going to delete or
+                                                   modify existing terms */
+                          mtraversal_on_nothing_can_match_t on_nothing_can_match,
+                          mtraversal_on_match_res_t on_match_res,
+                          mtraversal_on_loop_ended_t on_loop_ended,
+                          mtraversal_on_trap_t on_trap,
+                          void* context_ptr, /* State for callbacks above */
+                          Eterm* ret)
 {
-    DbTableHash *tb = &tbl->hash;
+    Sint slot_ix = -1;                /* Slot index */
+    HashDbTerm** current_ptr = NULL;  /* Refers to either the bucket pointer or
+                                       * the 'next' pointer in the previous term
+                                       */
+    HashDbTerm* saved_current = NULL; /* Helper to avoid double skip on match */
     struct mp_info mpi;
-    Sint slot_ix;
-    HashDbTerm *current = 0;
-    unsigned current_list_pos = 0;
-    Eterm match_list;
+    unsigned current_list_pos = 0;    /* Prefound buckets list index */
     Eterm match_res;
-    Eterm *hp;
-    int num_left = 1000;
-    Uint got = 0;
-    Eterm continuation;
-    int errcode;
-    Eterm mpb;
-    erts_smp_rwmtx_t* lck;
-
-
-#define RET_TO_BIF(Term,RetVal) do {		\
-	if (mpi.mp != NULL) {			\
-	    erts_bin_free(mpi.mp);		\
-	}					\
-	if (mpi.lists != mpi.dlists) {		\
-	    erts_free(ERTS_ALC_T_DB_SEL_LIST,	\
-		      (void *) mpi.lists);	\
-	}					\
-	*ret = (Term);				\
-	return RetVal;				\
-    } while(0)
+    Sint got = 0;                     /* Matched terms counter */
+    erts_smp_rwmtx_t* lck = NULL;     /* Slot lock */
+    int ret_value = DB_ERROR_NONE;
+#ifdef ERTS_SMP
+    erts_smp_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue)
+        = (lock_for_write ? WLOCK_HASH : RLOCK_HASH);
+    void (*unlock_hash_function)(erts_smp_rwmtx_t*)
+        = (lock_for_write ? WUNLOCK_HASH : RUNLOCK_HASH);
+#else
+    #define lock_hash_function(tb, hval) NULL
+    #define unlock_hash_function(lck) ((void)lck)
+#endif
+    Sint (*next_slot_function)(DbTableHash*, Uint, erts_smp_rwmtx_t**)
+        = (lock_for_write ? next_slot_w : next_slot);
 
+    *ret = NIL;
 
-    if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) {
-	RET_TO_BIF(NIL,errcode);
+    if ((ret_value = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) {
+        goto done;
     }
 
     if (!mpi.something_can_match) {
-	if (chunk_size) {
-	    RET_TO_BIF(am_EOT, DB_ERROR_NONE); /* We're done */
-	}  
-	RET_TO_BIF(NIL, DB_ERROR_NONE);
-	/* can't possibly match anything */
+        /* Can't possibly match anything */
+        ret_value = on_nothing_can_match(context_ptr, ret);
+        goto done;
+    }
+
+    if (mpi.all_objects) {
+        mpi.mp->flags |= BIN_FLAG_ALL_OBJECTS;
     }
 
+    /*
+     * Look for initial slot / bucket
+     */
     if (!mpi.key_given) {
-    /* Run this code if pattern is variable or GETKEY(pattern)  */
-    /* is a variable                                            */
-	slot_ix = 0;
-	lck = RLOCK_HASH(tb,slot_ix);
-	for (;;) { 
-	    ASSERT(slot_ix < NACTIVE(tb));
-	    if ((current = BUCKET(tb,slot_ix)) != NULL) {
-		break;
-	    }
-	    slot_ix = next_slot(tb,slot_ix,&lck);
-	    if (slot_ix == 0) {
-		if (chunk_size) {
-		    RET_TO_BIF(am_EOT, DB_ERROR_NONE); /* We're done */
-		}  
-		RET_TO_BIF(NIL,DB_ERROR_NONE); 
-	    }
-	}
+        /* Run this code if pattern is variable or GETKEY(pattern)  */
+        /* is a variable                                            */
+        slot_ix = 0;
+        lck = lock_hash_function(tb,slot_ix);
+        for (;;) {
+            ASSERT(slot_ix < NACTIVE(tb));
+            if (*(current_ptr = &BUCKET(tb,slot_ix)) != NULL) {
+                break;
+            }
+            slot_ix = next_slot_function(tb,slot_ix,&lck);
+            if (slot_ix == 0) {
+                ret_value = on_loop_ended(context_ptr, slot_ix, got, iterations_left, &mpi.mp, ret);
+                goto done;
+            }
+        }
     } else {
-	/* We have at least one */
-	slot_ix = mpi.lists[current_list_pos].ix;
-	lck = RLOCK_HASH(tb, slot_ix);
-	current = *(mpi.lists[current_list_pos].bucket);
-	ASSERT(current == BUCKET(tb,slot_ix));
-	++current_list_pos;
+        /* We have at least one */
+        slot_ix = mpi.lists[current_list_pos].ix;
+        lck = lock_hash_function(tb, slot_ix);
+        current_ptr = mpi.lists[current_list_pos].bucket;
+        ASSERT(*current_ptr == BUCKET(tb,slot_ix));
+        ++current_list_pos;
     }
 
-    match_list = NIL;
-
+    /*
+     * Execute traversal cycle
+     */
     for(;;) {
-	if (current != NULL) {
-	    if (current->hvalue != INVALID_HASH) {
-		match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0,
-					    &current->dbterm, &hp, 2);
-		if (is_value(match_res)) {
-		    match_list = CONS(hp, match_res, match_list);
-		    ++got;
-		}
-                --num_left;
-	    }
-	    current = current->next;
-	}	
-	else if (mpi.key_given) {  /* Key is bound */
-	    RUNLOCK_HASH(lck);
-	    if (current_list_pos == mpi.num_lists) {
-		slot_ix = -1; /* EOT */
-		goto done;
-	    } else {
-		slot_ix = mpi.lists[current_list_pos].ix;
-		lck = RLOCK_HASH(tb, slot_ix);
-		current = *(mpi.lists[current_list_pos].bucket);
-		ASSERT(mpi.lists[current_list_pos].bucket == &BUCKET(tb,slot_ix));
-		++current_list_pos;		       
-	    }
-	}
-	else { /* Key is variable */
-
-	    if ((slot_ix=next_slot(tb,slot_ix,&lck)) == 0) {
-		slot_ix = -1;
-		break;
-	    }
-	    if (chunk_size && got >= chunk_size) {
-		RUNLOCK_HASH(lck);
-		break;
-	    }    
-	    if (num_left <= 0 || MBUF(p)) {
-		/*
-		 * We have either reached our limit, or just created some heap fragments.
-		 * Since many heap fragments will make the GC slower, trap and GC now.
-		 */
-		RUNLOCK_HASH(lck);
-		goto trap;
-	    }
-	    current = BUCKET(tb,slot_ix);
+        if (*current_ptr != NULL) {
+            if ((*current_ptr)->hvalue != INVALID_HASH) {
+                match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0,
+                                            &(*current_ptr)->dbterm, hpp, 2);
+                saved_current = *current_ptr;
+                if (on_match_res(context_ptr, slot_ix, &current_ptr, match_res)) {
+                    ++got;
+                }
+                --iterations_left;
+                if (*current_ptr != saved_current) {
+                    /* Don't advance to next, the callback did it already */
+                    continue;
+                }
+            }
+            current_ptr = &((*current_ptr)->next);
+        }
+        else if (mpi.key_given) {  /* Key is bound */
+            unlock_hash_function(lck);
+            if (current_list_pos == mpi.num_lists) {
+                ret_value = on_loop_ended(context_ptr, slot_ix, got, iterations_left, &mpi.mp, ret);
+                goto done;
+            } else {
+                slot_ix = mpi.lists[current_list_pos].ix;
+                lck = lock_hash_function(tb, slot_ix);
+                current_ptr = mpi.lists[current_list_pos].bucket;
+                ASSERT(mpi.lists[current_list_pos].bucket == &BUCKET(tb,slot_ix));
+                ++current_list_pos;
+            }
+        }
+        else { /* Key is variable */
+            if ((slot_ix = next_slot_function(tb,slot_ix,&lck)) == 0) {
+                slot_ix = -1;
+                break;
+            }
+            if (chunk_size && got >= chunk_size) {
+                unlock_hash_function(lck);
+                break;
+            }
+            if (iterations_left <= 0 || MBUF(p)) {
+                /*
+                 * We have either reached our limit, or just created some heap fragments.
+                 * Since many heap fragments will make the GC slower, trap and GC now.
+                 */
+                unlock_hash_function(lck);
+                ret_value = on_trap(context_ptr, slot_ix, got, &mpi.mp, ret);
+                goto done;
+            }
+            current_ptr = &BUCKET(tb,slot_ix);
         }
-    }
-done:
-    BUMP_REDS(p, 1000 - num_left);
-    if (chunk_size) {
-	Eterm continuation;
-	Eterm rest = NIL;
-	Sint rest_size = 0;
-
-	if (mpi.all_objects)
-	    (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
-	if (got > chunk_size) { /* Split list in return value and 'rest' */
-	    Eterm tmp = match_list;
-	    rest = match_list;
-	    while (got-- > chunk_size + 1) { 
-		tmp = CDR(list_val(tmp));
-		++rest_size;
-	    }
-	    ++rest_size;
-	    match_list = CDR(list_val(tmp));
-	    CDR(list_val(tmp)) = NIL; /* Destructive, the list has never 
-					 been in 'user space' */ 
-	}
-	if (rest != NIL || slot_ix >= 0) { /* Need more calls */
-	    hp = HAlloc(p,3+7+ERTS_MAGIC_REF_THING_SIZE);
-	    mpb = erts_db_make_match_prog_ref(p,(mpi.mp),&hp);
-	    if (mpi.all_objects)
-		(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
-	    continuation = TUPLE6(hp, tid, make_small(slot_ix),
-				  make_small(chunk_size),  
-				  mpb, rest, 
-				  make_small(rest_size));
-	    mpi.mp = NULL; /*otherwise the return macro will destroy it */
-	    hp += 7;
-	    RET_TO_BIF(TUPLE2(hp, match_list, continuation),DB_ERROR_NONE);
-	} else { /* All data is exhausted */
-	    if (match_list != NIL) { /* No more data to search but still a
-					result to return to the caller */
-		hp = HAlloc(p, 3);
-		RET_TO_BIF(TUPLE2(hp, match_list, am_EOT),DB_ERROR_NONE);
-	    } else { /* Reached the end of the ttable with no data to return */
-		RET_TO_BIF(am_EOT, DB_ERROR_NONE);
-	    }
-	}
-    }
-    RET_TO_BIF(match_list,DB_ERROR_NONE);
-trap:
-    BUMP_ALL_REDS(p);
-    if (mpi.all_objects)
-	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
-    hp = HAlloc(p,7+ERTS_MAGIC_REF_THING_SIZE);
-    mpb = erts_db_make_match_prog_ref(p,(mpi.mp),&hp);
-    continuation = TUPLE6(hp, tid, make_small(slot_ix),
-			  make_small(chunk_size), 
-			  mpb, match_list, 
-			  make_small(got));
-    mpi.mp = NULL; /*otherwise the return macro will destroy it */
-    RET_TO_BIF(bif_trap1(&ets_select_continue_exp, p, 
-			 continuation), 
-	       DB_ERROR_NONE);
-
-#undef RET_TO_BIF
-
-}
-
-static int db_select_count_hash(Process *p, 
-				DbTable *tbl,
-                                Eterm tid,
-				Eterm pattern,
-				Eterm *ret)
-{
-    DbTableHash *tb = &tbl->hash;
-    struct mp_info mpi;
-    Uint slot_ix = 0;
-    HashDbTerm* current = NULL;
-    unsigned current_list_pos = 0;
-    Eterm *hp;
-    int num_left = 1000;
-    Uint got = 0;
-    Eterm continuation;
-    int errcode;
-    Eterm egot;
-    Eterm mpb;
-    erts_smp_rwmtx_t* lck;
-
-#define RET_TO_BIF(Term,RetVal) do {		\
-	if (mpi.mp != NULL) {			\
-	    erts_bin_free(mpi.mp);		\
-	}					\
-	if (mpi.lists != mpi.dlists) {		\
-	    erts_free(ERTS_ALC_T_DB_SEL_LIST,	\
-		      (void *) mpi.lists);	\
-	}					\
-	*ret = (Term);				\
-	return RetVal;				\
-    } while(0)
-
-
-    if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) {
-	RET_TO_BIF(NIL,errcode);
-    }
-
-    if (!mpi.something_can_match) {
-	RET_TO_BIF(make_small(0), DB_ERROR_NONE);
-	/* can't possibly match anything */
     }
 
-    if (!mpi.key_given) {
-    /* Run this code if pattern is variable or GETKEY(pattern)  */
-    /* is a variable                                            */      
-	slot_ix = 0;
-	lck = RLOCK_HASH(tb,slot_ix);
-	current = BUCKET(tb,slot_ix);
-    } else {
-	/* We have at least one */
-	slot_ix = mpi.lists[current_list_pos].ix;
-	lck = RLOCK_HASH(tb, slot_ix);
-	current = *(mpi.lists[current_list_pos].bucket);
-	ASSERT(current == BUCKET(tb,slot_ix));
-	++current_list_pos;
-    }
+    ret_value = on_loop_ended(context_ptr, slot_ix, got, iterations_left, &mpi.mp, ret);
 
-    for(;;) {
-	if (current != NULL) {
-	    if (current->hvalue != INVALID_HASH) {
-		if (db_match_dbterm(&tb->common, p, mpi.mp, 0,
-				    &current->dbterm, NULL,0) == am_true) {
-		    ++got;
-		}
-		--num_left;
-	    }
-	    current = current->next;
-	}
-	else { /* next bucket */
-	    if (mpi.key_given) {  /* Key is bound */
-		RUNLOCK_HASH(lck);
-		if (current_list_pos == mpi.num_lists) {
-		    goto done;
-		} else {
-		    slot_ix = mpi.lists[current_list_pos].ix;
-		    lck = RLOCK_HASH(tb, slot_ix);
-		    current = *(mpi.lists[current_list_pos].bucket);
-		    ASSERT(mpi.lists[current_list_pos].bucket == &BUCKET(tb,slot_ix));
-		    ++current_list_pos;		       
-		}
-	    }
-	    else {
-		if ((slot_ix=next_slot(tb,slot_ix,&lck)) == 0) {
-		    goto done;
-		}
-		if (num_left <= 0) {
-		    RUNLOCK_HASH(lck);
-		    goto trap;
-		}		
-		current = BUCKET(tb,slot_ix);
-	    }
-	}
-    }
 done:
-    BUMP_REDS(p, 1000 - num_left);
-    RET_TO_BIF(erts_make_integer(got,p),DB_ERROR_NONE);
-trap:
-    BUMP_ALL_REDS(p);
-    if (IS_USMALL(0, got)) {
-	hp = HAlloc(p,  ERTS_MAGIC_REF_THING_SIZE + 5);
-	egot = make_small(got);
+    /* We should only jump directly to this label if
+     * we've already called on_nothing_can_match / on_loop_ended / on_trap
+     */
+    if (mpi.mp != NULL) {
+        erts_bin_free(mpi.mp);
     }
-    else {
-	hp = HAlloc(p, BIG_UINT_HEAP_SIZE + ERTS_MAGIC_REF_THING_SIZE + 5);
-	egot = uint_to_big(got, hp);
-	hp += BIG_UINT_HEAP_SIZE;
+    if (mpi.lists != mpi.dlists) {
+        erts_free(ERTS_ALC_T_DB_SEL_LIST,
+                (void *) mpi.lists);
     }
-    mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
-    continuation = TUPLE4(hp, tid, make_small(slot_ix),
-			  mpb, 
-			  egot);
-    mpi.mp = NULL; /*otherwise the return macro will destroy it */
-    RET_TO_BIF(bif_trap1(&ets_select_count_continue_exp, p, 
-			 continuation), 
-	       DB_ERROR_NONE);
+    return ret_value;
 
-#undef RET_TO_BIF
+#ifndef SMP
+#undef lock_hash_function
+#undef unlock_hash_function
+#endif
 }
 
-static int db_select_delete_hash(Process *p,
-				 DbTable *tbl,
-                                 Eterm tid,
-				 Eterm pattern,
-				 Eterm *ret)
+/*
+ * Continue hash table match traversal
+ */
+static int match_traverse_continue(Process* p, DbTableHash* tb,
+                                   Sint chunk_size,      /* If 0, no chunking */
+                                   Sint iterations_left, /* Nr. of iterations left */
+                                   Eterm** hpp,          /* Heap */
+                                   Sint slot_ix,         /* Slot index to resume traversal from */
+                                   Sint got,             /* Matched terms counter */
+                                   Binary** mpp,         /* Existing match program */
+                                   int lock_for_write,   /* Set to 1 if we're going to delete or
+                                                            modify existing terms */
+                                   mtraversal_on_match_res_t on_match_res,
+                                   mtraversal_on_loop_ended_t on_loop_ended,
+                                   mtraversal_on_trap_t on_trap,
+                                   void* context_ptr, /* For callbacks */
+                                   Eterm* ret)
 {
-    DbTableHash *tb = &tbl->hash;
-    struct mp_info mpi;
-    Uint slot_ix = 0;
-    HashDbTerm **current = NULL;
-    unsigned current_list_pos = 0;
-    Eterm *hp;
-    int num_left = 1000;
-    Uint got = 0;
-    Eterm continuation;
-    int errcode;
-    Uint last_pseudo_delete = (Uint)-1;
-    Eterm mpb;
-    Eterm egot;
+    int all_objects = (*mpp)->flags & BIN_FLAG_ALL_OBJECTS;
+    HashDbTerm** current_ptr = NULL;  /* Refers to either the bucket pointer or
+                                       * the 'next' pointer in the previous term
+                                       */
+    HashDbTerm* saved_current = NULL; /* Helper to avoid double skip on match */
+    Eterm match_res = NIL;
+    erts_smp_rwmtx_t* lck = NULL;
+    int ret_value = DB_ERROR_NONE;
 #ifdef ERTS_SMP
-    erts_aint_t fixated_by_me = tb->common.is_thread_safe ? 0 : 1; /* ToDo: something nicer */
+    erts_smp_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue)
+        = (lock_for_write ? WLOCK_HASH : RLOCK_HASH);
+    void (*unlock_hash_function)(erts_smp_rwmtx_t*)
+        = (lock_for_write ? WUNLOCK_HASH : RUNLOCK_HASH);
 #else
-    erts_aint_t fixated_by_me = 0;
+    #define lock_hash_function(tb, hval) NULL
+    #define unlock_hash_function(lck) ((void)lck)
 #endif
-    erts_smp_rwmtx_t* lck;
+    Sint (*next_slot_function)(DbTableHash* tb, Uint ix, erts_smp_rwmtx_t** lck_ptr)
+        = (lock_for_write ? next_slot_w : next_slot);
 
-#define RET_TO_BIF(Term,RetVal) do {		\
-	if (mpi.mp != NULL) {			\
-	    erts_bin_free(mpi.mp);		\
-	}					\
-	if (mpi.lists != mpi.dlists) {		\
-	    erts_free(ERTS_ALC_T_DB_SEL_LIST,	\
-		      (void *) mpi.lists);	\
-	}					\
-	*ret = (Term);				\
-	return RetVal;				\
-    } while(0)
+    *ret = NIL;
 
+    if (got < 0)
+        return DB_ERROR_BADPARAM;
 
-    if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) {
-	RET_TO_BIF(NIL,errcode);
+    if (slot_ix < 0 /* EOT */
+       || (chunk_size && got >= chunk_size))
+    {
+        /* Already got all or enough in the match_list */
+        ret_value = on_loop_ended(context_ptr, slot_ix, got, iterations_left, mpp, ret);
+        goto done;
     }
 
-    if (!mpi.something_can_match) {
-	RET_TO_BIF(make_small(0), DB_ERROR_NONE);
-	/* can't possibly match anything */
+    lck = lock_hash_function(tb, slot_ix);
+    if (slot_ix >= NACTIVE(tb)) { /* Is this possible? */
+        unlock_hash_function(lck);
+        ret_value = DB_ERROR_BADPARAM;
+        goto done;
     }
 
-    if (!mpi.key_given) {
-	/* Run this code if pattern is variable or GETKEY(pattern)  */
-	/* is a variable                                            */
-	lck = WLOCK_HASH(tb,slot_ix);
-	current = &BUCKET(tb,slot_ix);
-    } else {
-	/* We have at least one */
-	slot_ix = mpi.lists[current_list_pos].ix;
-	lck = WLOCK_HASH(tb, slot_ix);
-	current = mpi.lists[current_list_pos++].bucket; 
-	ASSERT(*current == BUCKET(tb,slot_ix));
+    /*
+     * Resume traversal cycle from where we left
+     */
+    current_ptr = &BUCKET(tb,slot_ix);
+    for(;;) {
+        if (*current_ptr != NULL) {
+            if ((*current_ptr)->hvalue != INVALID_HASH) {
+                match_res = db_match_dbterm(&tb->common, p, *mpp, all_objects,
+                                            &(*current_ptr)->dbterm, hpp, 2);
+                saved_current = *current_ptr;
+                if (on_match_res(context_ptr, slot_ix, &current_ptr, match_res)) {
+                    ++got;
+                }
+                --iterations_left;
+                if (*current_ptr != saved_current) {
+                    /* Don't advance to next, the callback did it already */
+                    continue;
+                }
+            }
+            current_ptr = &((*current_ptr)->next);
+        }
+        else {
+            if ((slot_ix=next_slot_function(tb,slot_ix,&lck)) == 0) {
+                slot_ix = -1;
+                break;
+            }
+            if (chunk_size && got >= chunk_size) {
+                unlock_hash_function(lck);
+                break;
+            }
+            if (iterations_left <= 0 || MBUF(p)) {
+                /*
+                 * We have either reached our limit, or just created some heap fragments.
+                 * Since many heap fragments will make the GC slower, trap and GC now.
+                 */
+                unlock_hash_function(lck);
+                ret_value = on_trap(context_ptr, slot_ix, got, mpp, ret);
+                goto done;
+            }
+            current_ptr = &BUCKET(tb,slot_ix);
+        }
     }
 
+    ret_value = on_loop_ended(context_ptr, slot_ix, got, iterations_left, mpp, ret);
 
-    for(;;) {
-	if ((*current) == NULL) {
-	    if (mpi.key_given) {  /* Key is bound */
-		WUNLOCK_HASH(lck);
-		if (current_list_pos == mpi.num_lists) {
-		    goto done;
-		} else {
-		    slot_ix = mpi.lists[current_list_pos].ix;
-		    lck = WLOCK_HASH(tb, slot_ix);
-		    current = mpi.lists[current_list_pos].bucket;
-		    ASSERT(mpi.lists[current_list_pos].bucket == &BUCKET(tb,slot_ix));
-		    ++current_list_pos;
-		}
-	    } else {
-		if ((slot_ix=next_slot_w(tb,slot_ix,&lck)) == 0) {
-		    goto done;
-		}
-		if (num_left <= 0) {
-		    WUNLOCK_HASH(lck);
-		    goto trap;
-		}		
-		current = &BUCKET(tb,slot_ix);
-	    } 
-	}
-	else if ((*current)->hvalue == INVALID_HASH) {
-	    current = &((*current)->next);
-	} 
-	else {
-	    int did_erase = 0;
-	    if (db_match_dbterm(&tb->common, p, mpi.mp, 0,
-				&(*current)->dbterm, NULL, 0) == am_true) {
-                HashDbTerm *del;
-		if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */
-		    if (slot_ix != last_pseudo_delete) {
-                        if (!add_fixed_deletion(tb, slot_ix, fixated_by_me))
-                            goto do_erase;
-                        last_pseudo_delete = slot_ix;
-		    }
-		    (*current)->hvalue = INVALID_HASH;
-		} else {
-                do_erase:
-		    del = *current;
-		    *current = (*current)->next;
-		    free_term(tb, del);
-		    did_erase = 1;
-		}
-		erts_smp_atomic_dec_nob(&tb->common.nitems);
-		++got;
-	    }	    
-	    --num_left;
-	    if (!did_erase) {
-		current = &((*current)->next);
-	    }
-	}
-    }
 done:
-    BUMP_REDS(p, 1000 - num_left);
-    if (got) {
-	try_shrink(tb);
-    }
-    RET_TO_BIF(erts_make_integer(got,p),DB_ERROR_NONE);
-trap:
+    /* We should only jump directly to this label if
+     * we've already called on_loop_ended / on_trap
+     */
+    return ret_value;
+
+#ifndef SMP
+#undef lock_hash_function
+#undef unlock_hash_function
+#endif
+}
+
+
+/*
+ * Common traversal trapping/continuation code;
+ * used by select_count, select_delete and select_replace,
+ * as well as their continuation-handling counterparts.
+ */
+
+static ERTS_INLINE int on_mtraversal_simple_trap(Export* trap_function,
+                                                 Process* p,
+                                                 DbTableHash* tb,
+                                                 Eterm tid,
+                                                 Eterm* prev_continuation_tptr,
+                                                 Sint slot_ix,
+                                                 Sint got,
+                                                 Binary** mpp,
+                                                 Eterm* ret)
+{
+    Eterm* hp = NULL;
+    Eterm egot = NIL;
+    Eterm mpb = NIL;
+    Eterm continuation = NIL;
+    int is_first_trap = (prev_continuation_tptr == NULL);
+    size_t base_halloc_sz = (is_first_trap ? PROC_BIN_SIZE : 0);
+
     BUMP_ALL_REDS(p);
     if (IS_USMALL(0, got)) {
-	hp = HAlloc(p,  ERTS_MAGIC_REF_THING_SIZE + 5);
+	hp = HAlloc(p,  base_halloc_sz + 5);
 	egot = make_small(got);
     }
     else {
-	hp = HAlloc(p, BIG_UINT_HEAP_SIZE + ERTS_MAGIC_REF_THING_SIZE + 5);
+	hp = HAlloc(p, base_halloc_sz + BIG_UINT_HEAP_SIZE + 5);
 	egot = uint_to_big(got, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
-    continuation = TUPLE4(hp, tid, make_small(slot_ix),
-			  mpb, 
-			  egot);
-    mpi.mp = NULL; /*otherwise the return macro will destroy it */
-    RET_TO_BIF(bif_trap1(&ets_select_delete_continue_exp, p, 
-			 continuation), 
-	       DB_ERROR_NONE);
 
-#undef RET_TO_BIF
+    if (is_first_trap) {
+        mpb = db_make_mp_binary(p, *mpp, &hp);
+        *mpp = NULL; /* otherwise the caller will destroy it */
+    }
+    else {
+        mpb = prev_continuation_tptr[3];
+    }
 
+    continuation = TUPLE4(
+            hp,
+            tid,
+            make_small(slot_ix),
+            mpb,
+            egot);
+    *ret = bif_trap1(trap_function, p, continuation);
+    return DB_ERROR_NONE;
 }
-/*
-** This is called when select_delete traps
-*/
-static int db_select_delete_continue_hash(Process *p, 
-					  DbTable *tbl,
-					  Eterm continuation,
-					  Eterm *ret)
+
+static ERTS_INLINE int unpack_simple_mtraversal_continuation(Eterm continuation,
+                                                             Eterm** tptr_ptr,
+                                                             Eterm* tid_ptr,
+                                                             Sint* slot_ix_p,
+                                                             Binary** mpp,
+                                                             Sint* got_p)
 {
-    DbTableHash *tb = &tbl->hash;
-    Uint slot_ix;
-    Uint last_pseudo_delete = (Uint)-1;
-    HashDbTerm **current = NULL;
-    Eterm *hp;
-    int num_left = 1000;
-    Uint got;
-    Eterm *tptr;
-    Binary *mp;
-    Eterm egot;
-    int fixated_by_me = ONLY_WRITER(p,tb) ? 0 : 1; /* ToDo: something nicer */
-    erts_smp_rwmtx_t* lck;
+    Eterm* tptr = NULL;
+    ASSERT(is_tuple(continuation));
+    tptr = tuple_val(continuation);
+    if (arityval(*tptr) != 4)
+        return 1;
 
-#define RET_TO_BIF(Term,RetVal) do {		\
-	*ret = (Term);				\
-	return RetVal;				\
-    } while(0)
+    if (! is_small(tptr[2])
+            || !(is_binary(tptr[3]) && thing_subtag(*binary_val(tptr[3])) == REFC_BINARY_SUBTAG)
+            || !(is_big(tptr[4]) || is_small(tptr[4])))
+    {
+        return 1;
+    }
 
-    
-    tptr = tuple_val(continuation);
-    slot_ix = unsigned_val(tptr[2]);
-    mp = erts_db_get_match_prog_binary_unchecked(tptr[3]);
+    *tptr_ptr = tptr;
+    *tid_ptr = tptr[1];
+    *slot_ix_p = unsigned_val(tptr[2]);
+    *mpp = ((ProcBin *) binary_val(tptr[3]))->val;
     if (is_big(tptr[4])) {
-	got = big_to_uint32(tptr[4]);
-    } else {
-	got = unsigned_val(tptr[4]);
+        *got_p = big_to_uint32(tptr[4]);
     }
-    
-    lck = WLOCK_HASH(tb,slot_ix);
-    if (slot_ix >= NACTIVE(tb)) {
-	WUNLOCK_HASH(lck);
-	goto done;
-    } 
-    current = &BUCKET(tb,slot_ix);
-
-    for(;;) {
-	if ((*current) == NULL) {
-	    if ((slot_ix=next_slot_w(tb,slot_ix,&lck)) == 0) {
-		goto done;
-	    }
-	    if (num_left <= 0) {
-		WUNLOCK_HASH(lck);
-		goto trap;
-	    }		
-	    current = &BUCKET(tb,slot_ix);
-	}
-	else if ((*current)->hvalue == INVALID_HASH) {
-	    current = &((*current)->next);
-	} 
-	else {
-	    int did_erase = 0;
-	    if (db_match_dbterm(&tb->common, p, mp, 0,
-				&(*current)->dbterm, NULL, 0) == am_true) {
-                HashDbTerm *del;
-		if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */
-		    if (slot_ix != last_pseudo_delete) {
-                        if (!add_fixed_deletion(tb, slot_ix, fixated_by_me))
-                            goto do_erase;
-			last_pseudo_delete = slot_ix;
-		    }
-		    (*current)->hvalue = INVALID_HASH;
-		} else {
-                do_erase:
-		    del = *current;
-		    *current = (*current)->next;
-		    free_term(tb, del);
-		    did_erase = 1;
-		}
-		erts_smp_atomic_dec_nob(&tb->common.nitems);
-		++got;
-	    }
-	    
-	    --num_left;
-	    if (!did_erase) {
-		current = &((*current)->next);
-	    }
-	}
+    else {
+        *got_p = unsigned_val(tptr[4]);
     }
-done:
-    BUMP_REDS(p, 1000 - num_left);
-    if (got) {
-	try_shrink(tb);
+    return 0;
+}
+
+
+/*
+ *
+ * select / select_chunk match traversal
+ *
+ */
+
+#define MAX_SELECT_CHUNK_ITERATIONS 1000
+
+typedef struct {
+    Process* p;
+    DbTableHash* tb;
+    Eterm tid;
+    Eterm* hp;
+    Sint chunk_size;
+    Eterm match_list;
+    Eterm* prev_continuation_tptr;
+} mtraversal_select_chunk_context_t;
+
+static int mtraversal_select_chunk_on_nothing_can_match(void* context_ptr, Eterm* ret) {
+    mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr;
+    *ret = (sc_context_ptr->chunk_size > 0 ? am_EOT : NIL);
+    return DB_ERROR_NONE;
+}
+
+static int mtraversal_select_chunk_on_match_res(void* context_ptr, Sint slot_ix,
+                                                HashDbTerm*** current_ptr_ptr,
+                                                Eterm match_res)
+{
+    mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr;
+    if (is_value(match_res)) {
+        sc_context_ptr->match_list = CONS(sc_context_ptr->hp, match_res, sc_context_ptr->match_list);
+        return 1;
     }
-    RET_TO_BIF(erts_make_integer(got,p),DB_ERROR_NONE);
-trap:
-    BUMP_ALL_REDS(p);
-    if (IS_USMALL(0, got)) {
-	hp = HAlloc(p,  5);
-	egot = make_small(got);
+    return 0;
+}
+
+static int mtraversal_select_chunk_on_loop_ended(void* context_ptr, Sint slot_ix, Sint got,
+                                                 Sint iterations_left, Binary** mpp, Eterm* ret)
+{
+    mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr;
+    Eterm mpb = NIL;
+
+    if (iterations_left == MAX_SELECT_CHUNK_ITERATIONS) {
+        /* We didn't get to iterate a single time, which means EOT */
+        ASSERT(sc_context_ptr->match_list == NIL);
+        *ret = (sc_context_ptr->chunk_size > 0 ? am_EOT : NIL);
+        return DB_ERROR_NONE;
     }
     else {
-	hp = HAlloc(p, BIG_UINT_HEAP_SIZE + 5);
-	egot = uint_to_big(got, hp);
-	hp += BIG_UINT_HEAP_SIZE;
+        ASSERT(iterations_left < MAX_SELECT_CHUNK_ITERATIONS);
+        BUMP_REDS(sc_context_ptr->p, MAX_SELECT_CHUNK_ITERATIONS - iterations_left);
+        if (sc_context_ptr->chunk_size) {
+            Eterm continuation;
+            Eterm rest = NIL;
+            Sint rest_size = 0;
+
+            if (got > sc_context_ptr->chunk_size) { /* Split list in return value and 'rest' */
+                Eterm tmp = sc_context_ptr->match_list;
+                rest = sc_context_ptr->match_list;
+                while (got-- > sc_context_ptr->chunk_size + 1) {
+                    tmp = CDR(list_val(tmp));
+                    ++rest_size;
+                }
+                ++rest_size;
+                sc_context_ptr->match_list = CDR(list_val(tmp));
+                CDR(list_val(tmp)) = NIL; /* Destructive, the list has never
+                                             been in 'user space' */
+            }
+            if (rest != NIL || slot_ix >= 0) { /* Need more calls */
+                sc_context_ptr->hp = HAlloc(sc_context_ptr->p, 3 + 7 + PROC_BIN_SIZE);
+                mpb = db_make_mp_binary(sc_context_ptr->p, *mpp, &sc_context_ptr->hp);
+                continuation = TUPLE6(
+                        sc_context_ptr->hp,
+                        sc_context_ptr->tid,
+                        make_small(slot_ix),
+                        make_small(sc_context_ptr->chunk_size),
+                        mpb, rest,
+                        make_small(rest_size));
+                *mpp = NULL; /* Otherwise the caller will destroy it */
+                sc_context_ptr->hp += 7;
+                *ret = TUPLE2(sc_context_ptr->hp, sc_context_ptr->match_list, continuation);
+                return DB_ERROR_NONE;
+            } else { /* All data is exhausted */
+                if (sc_context_ptr->match_list != NIL) { /* No more data to search but still a
+                                                            result to return to the caller */
+                    sc_context_ptr->hp = HAlloc(sc_context_ptr->p, 3);
+                    *ret = TUPLE2(sc_context_ptr->hp, sc_context_ptr->match_list, am_EOT);
+                    return DB_ERROR_NONE;
+                } else { /* Reached the end of the ttable with no data to return */
+                    *ret = am_EOT;
+                    return DB_ERROR_NONE;
+                }
+            }
+        }
+        *ret = sc_context_ptr->match_list;
+        return DB_ERROR_NONE;
+    }
+}
+
+static int mtraversal_select_chunk_on_trap(void* context_ptr, Sint slot_ix, Sint got,
+                                           Binary** mpp, Eterm* ret)
+{
+    mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr;
+    Eterm mpb = NIL;
+    Eterm continuation = NIL;
+    Eterm* hp = NULL;
+
+    BUMP_ALL_REDS(sc_context_ptr->p);
+
+    if (sc_context_ptr->prev_continuation_tptr == NULL) {
+        /* First time we're trapping */
+        hp = HAlloc(sc_context_ptr->p, 7 + PROC_BIN_SIZE);
+        mpb = db_make_mp_binary(sc_context_ptr->p, *mpp, &hp);
+        continuation = TUPLE6(
+                hp,
+                sc_context_ptr->tid,
+                make_small(slot_ix),
+                make_small(sc_context_ptr->chunk_size),
+                mpb,
+                sc_context_ptr->match_list,
+                make_small(got));
+        *mpp = NULL; /* otherwise the caller will destroy it */
     }
-    continuation = TUPLE4(hp, tptr[1], make_small(slot_ix),
-			  tptr[3], 
-			  egot);
-    RET_TO_BIF(bif_trap1(&ets_select_delete_continue_exp, p, 
-			 continuation), 
-	       DB_ERROR_NONE);
+    else {
+        /* Not the first time we're trapping; reuse continuation terms */
+        hp = HAlloc(sc_context_ptr->p, 7);
+        continuation = TUPLE6(
+                hp,
+                sc_context_ptr->prev_continuation_tptr[1],
+                make_small(slot_ix),
+                sc_context_ptr->prev_continuation_tptr[3],
+                sc_context_ptr->prev_continuation_tptr[4],
+                sc_context_ptr->match_list,
+                make_small(got));
+    }
+    *ret = bif_trap1(&ets_select_continue_exp, sc_context_ptr->p, continuation);
+    return DB_ERROR_NONE;
+}
 
-#undef RET_TO_BIF
+static int db_select_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, int reverse, Eterm *ret) {
+    return db_select_chunk_hash(p, tbl, tid, pattern, 0, reverse, ret);
+}
 
+static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Sint chunk_size,
+                                int reverse, Eterm *ret)
+{
+    mtraversal_select_chunk_context_t sc_context = {0};
+    sc_context.p = p;
+    sc_context.tb = &tbl->hash;
+    sc_context.tid = tid;
+    sc_context.hp = NULL;
+    sc_context.chunk_size = chunk_size;
+    sc_context.match_list = NIL;
+    sc_context.prev_continuation_tptr = NULL;
+
+    return match_traverse(
+            sc_context.p, sc_context.tb, pattern, sc_context.chunk_size,
+            MAX_SELECT_CHUNK_ITERATIONS,
+            &sc_context.hp, 0,
+            mtraversal_select_chunk_on_nothing_can_match,
+            mtraversal_select_chunk_on_match_res,
+            mtraversal_select_chunk_on_loop_ended,
+            mtraversal_select_chunk_on_trap,
+            &sc_context, ret);
 }
-    
+
 /*
-** This is called when select_count traps
-*/
-static int db_select_count_continue_hash(Process *p, 
-					 DbTable *tbl,
-					 Eterm continuation,
-					 Eterm *ret)
+ *
+ * select_continue match traversal
+ *
+ */
+
+static int mtraversal_select_chunk_continue_on_loop_ended(void* context_ptr, Sint slot_ix, Sint got,
+                                                          Sint iterations_left, Binary** mpp, Eterm* ret)
 {
-    DbTableHash *tb = &tbl->hash;
-    Uint slot_ix;
-    HashDbTerm* current;
-    Eterm *hp;
-    int num_left = 1000;
-    Uint got;
-    Eterm *tptr;
-    Binary *mp;
-    Eterm egot;
-    erts_smp_rwmtx_t* lck;
+    mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr;
+    Eterm continuation = NIL;
+    Eterm rest = NIL;
+    Eterm* hp = NULL;
+
+    ASSERT(iterations_left <= MAX_SELECT_CHUNK_ITERATIONS);
+    BUMP_REDS(sc_context_ptr->p, MAX_SELECT_CHUNK_ITERATIONS - iterations_left);
+    if (sc_context_ptr->chunk_size) {
+        Sint rest_size = 0;
+        if (got > sc_context_ptr->chunk_size) {
+            /* Cannot write destructively here,
+               the list may have
+               been in user space */
+            rest = NIL;
+            hp = HAlloc(sc_context_ptr->p, (got - sc_context_ptr->chunk_size) * 2);
+            while (got-- > sc_context_ptr->chunk_size) {
+                rest = CONS(hp, CAR(list_val(sc_context_ptr->match_list)), rest);
+                hp += 2;
+                sc_context_ptr->match_list = CDR(list_val(sc_context_ptr->match_list));
+                ++rest_size;
+            }
+        }
+        if (rest != NIL || slot_ix >= 0) {
+            hp = HAlloc(sc_context_ptr->p, 3 + 7);
+            continuation = TUPLE6(
+                    hp,
+                    sc_context_ptr->prev_continuation_tptr[1],
+                    make_small(slot_ix),
+                    sc_context_ptr->prev_continuation_tptr[3],
+                    sc_context_ptr->prev_continuation_tptr[4],
+                    rest,
+                    make_small(rest_size));
+            hp += 7;
+            *ret = TUPLE2(hp, sc_context_ptr->match_list, continuation);
+            return DB_ERROR_NONE;
+        } else {
+            if (sc_context_ptr->match_list != NIL) {
+                hp = HAlloc(sc_context_ptr->p, 3);
+                *ret = TUPLE2(hp, sc_context_ptr->match_list, am_EOT);
+                return DB_ERROR_NONE;
+            } else {
+                *ret = am_EOT;
+                return DB_ERROR_NONE;
+            }
+        }
+    }
+    *ret = sc_context_ptr->match_list;
+    return DB_ERROR_NONE;
+}
 
-#define RET_TO_BIF(Term,RetVal) do {		\
-	*ret = (Term);				\
-	return RetVal;				\
-    } while(0)
+/*
+ * This is called when select traps
+ */
+static int db_select_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) {
+    mtraversal_select_chunk_context_t sc_context = {0};
+    Eterm* tptr = NULL;
+    Eterm tid = NIL;
+    Binary* mp = NULL;
+    Sint got = 0;
+    Sint slot_ix = 0;
+    Sint chunk_size = 0;
+    Eterm match_list = NIL;
+    Sint iterations_left = MAX_SELECT_CHUNK_ITERATIONS;
+    *ret = NIL;
 
-    
+    /* Decode continuation. We know it's a tuple but not the arity or anything else */
+    ASSERT(is_tuple(continuation));
     tptr = tuple_val(continuation);
-    slot_ix = unsigned_val(tptr[2]);
-    mp = erts_db_get_match_prog_binary_unchecked(tptr[3]);
-    if (is_big(tptr[4])) {
-	got = big_to_uint32(tptr[4]);
-    } else {
-	got = unsigned_val(tptr[4]);
-    }
-    
 
-    lck = RLOCK_HASH(tb, slot_ix);
-    if (slot_ix >= NACTIVE(tb)) { /* Is this posible? */
-	RUNLOCK_HASH(lck);
-	goto done;
-    }     
-    current = BUCKET(tb,slot_ix);
-    
-    for(;;) {
-	if (current != NULL) {
-	    if (current->hvalue == INVALID_HASH) {
-		current = current->next;
-		continue;
-	    }
-	    if (db_match_dbterm(&tb->common, p, mp, 0, &current->dbterm,
-				NULL, 0) == am_true) {
-		++got;
-	    }
-	    --num_left;
-	    current = current->next;
-	}
-	else { /* next bucket */
-            if ((slot_ix = next_slot(tb,slot_ix,&lck)) == 0) {
-		goto done;
-	    }
-	    if (num_left <= 0) {
-		RUNLOCK_HASH(lck);
-		goto trap;
-	    }
-	    current = BUCKET(tb,slot_ix);
-	}
+    if (arityval(*tptr) != 6)
+        return DB_ERROR_BADPARAM;
+
+    if (!is_small(tptr[2]) || !is_small(tptr[3]) || !is_binary(tptr[4]) ||
+            !(is_list(tptr[5]) || tptr[5] == NIL) || !is_small(tptr[6]))
+        return DB_ERROR_BADPARAM;
+    if ((chunk_size = signed_val(tptr[3])) < 0)
+        return DB_ERROR_BADPARAM;
+    if (!(thing_subtag(*binary_val(tptr[4])) == REFC_BINARY_SUBTAG))
+        return DB_ERROR_BADPARAM;
+
+    mp = ((ProcBin *) binary_val(tptr[4]))->val;
+    if (!IsMatchProgBinary(mp))
+        return DB_ERROR_BADPARAM;
+
+    if ((got = signed_val(tptr[6])) < 0)
+        return DB_ERROR_BADPARAM;
+
+    tid = tptr[1];
+    slot_ix = signed_val(tptr[2]);
+    match_list = tptr[5];
+
+    /* Proceed */
+    sc_context.p = p;
+    sc_context.tb = &tbl->hash;
+    sc_context.tid = tid;
+    sc_context.hp = NULL;
+    sc_context.chunk_size = chunk_size;
+    sc_context.match_list = match_list;
+    sc_context.prev_continuation_tptr = tptr;
+
+    return match_traverse_continue(
+            sc_context.p, sc_context.tb, sc_context.chunk_size,
+            iterations_left, &sc_context.hp, slot_ix, got, &mp, 0,
+            mtraversal_select_chunk_on_match_res, /* Reuse callback */
+            mtraversal_select_chunk_continue_on_loop_ended,
+            mtraversal_select_chunk_on_trap,      /* Reuse callback */
+            &sc_context, ret);
+}
+
+#undef MAX_SELECT_CHUNK_ITERATIONS
+
+
+/*
+ *
+ * select_count match traversal
+ *
+ */
+
+#define MAX_SELECT_COUNT_ITERATIONS 1000
+
+typedef struct {
+    Process* p;
+    DbTableHash* tb;
+    Eterm tid;
+    Eterm* hp;
+    Eterm* prev_continuation_tptr;
+} mtraversal_select_count_context_t;
+
+static int mtraversal_select_count_on_nothing_can_match(void* context_ptr, Eterm* ret) {
+    *ret = make_small(0);
+    return DB_ERROR_NONE;
+}
+
+static int mtraversal_select_count_on_match_res(void* context_ptr, Sint slot_ix,
+                                                HashDbTerm*** current_ptr_ptr,
+                                                Eterm match_res)
+{
+    return (match_res == am_true);
+}
+
+static int mtraversal_select_count_on_loop_ended(void* context_ptr, Sint slot_ix, Sint got,
+                                                 Sint iterations_left, Binary** mpp, Eterm* ret)
+{
+    mtraversal_select_count_context_t* scnt_context_ptr = (mtraversal_select_count_context_t*) context_ptr;
+    ASSERT(iterations_left <= MAX_SELECT_COUNT_ITERATIONS);
+    BUMP_REDS(scnt_context_ptr->p, MAX_SELECT_COUNT_ITERATIONS - iterations_left);
+    *ret = erts_make_integer(got, scnt_context_ptr->p);
+    return DB_ERROR_NONE;
+}
+
+static int mtraversal_select_count_on_trap(void* context_ptr, Sint slot_ix, Sint got,
+                                           Binary** mpp, Eterm* ret)
+{
+    mtraversal_select_count_context_t* scnt_context_ptr = (mtraversal_select_count_context_t*) context_ptr;
+    return on_mtraversal_simple_trap(
+            &ets_select_count_continue_exp,
+            scnt_context_ptr->p,
+            scnt_context_ptr->tb,
+            scnt_context_ptr->tid,
+            scnt_context_ptr->prev_continuation_tptr,
+            slot_ix, got, mpp, ret);
+}
+
+static int db_select_count_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret) {
+    mtraversal_select_count_context_t scnt_context = {0};
+    Sint iterations_left = MAX_SELECT_COUNT_ITERATIONS;
+    Sint chunk_size = 0;
+
+    scnt_context.p = p;
+    scnt_context.tb = &tbl->hash;
+    scnt_context.tid = tid;
+    scnt_context.hp = NULL;
+    scnt_context.prev_continuation_tptr = NULL;
+
+    return match_traverse(
+            scnt_context.p, scnt_context.tb,
+            pattern, chunk_size,
+            iterations_left, NULL, 0,
+            mtraversal_select_count_on_nothing_can_match,
+            mtraversal_select_count_on_match_res,
+            mtraversal_select_count_on_loop_ended,
+            mtraversal_select_count_on_trap,
+            &scnt_context, ret);
+}
+
+/*
+ * This is called when select_count traps
+ */
+static int db_select_count_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) {
+    mtraversal_select_count_context_t scnt_context = {0};
+    Eterm* tptr = NULL;
+    Eterm tid = NIL;
+    Binary* mp = NULL;
+    Sint got = 0;
+    Sint slot_ix = 0;
+    Sint chunk_size = 0;
+    *ret = NIL;
+
+    if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) {
+        return DB_ERROR_BADPARAM;
     }
-done:
-    BUMP_REDS(p, 1000 - num_left);
-    RET_TO_BIF(erts_make_integer(got,p),DB_ERROR_NONE);
-trap:
-    BUMP_ALL_REDS(p);
-    if (IS_USMALL(0, got)) {
-	hp = HAlloc(p, 5);
-	egot = make_small(got);
+
+    scnt_context.p = p;
+    scnt_context.tb = &tbl->hash;
+    scnt_context.tid = tid;
+    scnt_context.hp = NULL;
+    scnt_context.prev_continuation_tptr = tptr;
+
+    return match_traverse_continue(
+            scnt_context.p, scnt_context.tb, chunk_size,
+            MAX_SELECT_COUNT_ITERATIONS,
+            NULL, slot_ix, got, &mp, 0,
+            mtraversal_select_count_on_match_res,  /* Reuse callback */
+            mtraversal_select_count_on_loop_ended, /* Reuse callback */
+            mtraversal_select_count_on_trap,       /* Reuse callback */
+            &scnt_context, ret);
+}
+
+#undef MAX_SELECT_COUNT_ITERATIONS
+
+
+/*
+ *
+ * select_delete match traversal
+ *
+ */
+
+#define MAX_SELECT_DELETE_ITERATIONS 1000
+
+typedef struct {
+    Process* p;
+    DbTableHash* tb;
+    Eterm tid;
+    Eterm* hp;
+    Eterm* prev_continuation_tptr;
+    erts_aint_t fixated_by_me;
+    Uint last_pseudo_delete;
+} mtraversal_select_delete_context_t;
+
+static int mtraversal_select_delete_on_nothing_can_match(void* context_ptr, Eterm* ret) {
+    *ret = make_small(0);
+    return DB_ERROR_NONE;
+}
+
+static int mtraversal_select_delete_on_match_res(void* context_ptr, Sint slot_ix,
+                                                 HashDbTerm*** current_ptr_ptr,
+                                                 Eterm match_res)
+{
+    HashDbTerm** current_ptr = *current_ptr_ptr;
+    mtraversal_select_delete_context_t* sd_context_ptr = (mtraversal_select_delete_context_t*) context_ptr;
+    HashDbTerm* del = NULL;
+    if (match_res != am_true)
+        return 0;
+
+    if (NFIXED(sd_context_ptr->tb) > sd_context_ptr->fixated_by_me) { /* fixated by others? */
+        if (slot_ix != sd_context_ptr->last_pseudo_delete) {
+            if (!add_fixed_deletion(sd_context_ptr->tb, slot_ix, sd_context_ptr->fixated_by_me))
+                goto do_erase;
+            sd_context_ptr->last_pseudo_delete = slot_ix;
+        }
+        (*current_ptr)->hvalue = INVALID_HASH;
     }
     else {
-	hp = HAlloc(p, BIG_UINT_HEAP_SIZE + 5);
-	egot = uint_to_big(got, hp);
-	hp += BIG_UINT_HEAP_SIZE;
+    do_erase:
+        del = *current_ptr;
+        *current_ptr = (*current_ptr)->next; // replace pointer to term using next
+        free_term(sd_context_ptr->tb, del);
     }
-    continuation = TUPLE4(hp, tptr[1], make_small(slot_ix),
-			  tptr[3], 
-			  egot);
-    RET_TO_BIF(bif_trap1(&ets_select_count_continue_exp, p, 
-			 continuation), 
-	       DB_ERROR_NONE);
+    erts_smp_atomic_dec_nob(&sd_context_ptr->tb->common.nitems);
 
-#undef RET_TO_BIF
+    return 1;
+}
 
+static int mtraversal_select_delete_on_loop_ended(void* context_ptr, Sint slot_ix, Sint got,
+                                                  Sint iterations_left, Binary** mpp, Eterm* ret)
+{
+    mtraversal_select_delete_context_t* sd_context_ptr = (mtraversal_select_delete_context_t*) context_ptr;
+    ASSERT(iterations_left <= MAX_SELECT_DELETE_ITERATIONS);
+    BUMP_REDS(sd_context_ptr->p, MAX_SELECT_DELETE_ITERATIONS - iterations_left);
+    if (got) {
+	try_shrink(sd_context_ptr->tb);
+    }
+    *ret = erts_make_integer(got, sd_context_ptr->p);
+    return DB_ERROR_NONE;
 }
-    
+
+static int mtraversal_select_delete_on_trap(void* context_ptr, Sint slot_ix, Sint got,
+                                            Binary** mpp, Eterm* ret)
+{
+    mtraversal_select_delete_context_t* sd_context_ptr = (mtraversal_select_delete_context_t*) context_ptr;
+    return on_mtraversal_simple_trap(
+            &ets_select_delete_continue_exp,
+            sd_context_ptr->p,
+            sd_context_ptr->tb,
+            sd_context_ptr->tid,
+            sd_context_ptr->prev_continuation_tptr,
+            slot_ix, got, mpp, ret);
+}
+
+static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret) {
+    mtraversal_select_delete_context_t sd_context = {0};
+    Sint chunk_size = 0;
+
+    sd_context.p = p;
+    sd_context.tb = &tbl->hash;
+    sd_context.tid = tid;
+    sd_context.hp = NULL;
+    sd_context.prev_continuation_tptr = NULL;
+#ifdef ERTS_SMP
+    sd_context.fixated_by_me = sd_context.tb->common.is_thread_safe ? 0 : 1; /* TODO: something nicer */
+#else
+    sd_context.fixated_by_me = 0;
+#endif
+    sd_context.last_pseudo_delete = (Uint) -1;
+
+    return match_traverse(
+            sd_context.p, sd_context.tb, pattern, chunk_size,
+            MAX_SELECT_DELETE_ITERATIONS, NULL, 1,
+            mtraversal_select_delete_on_nothing_can_match,
+            mtraversal_select_delete_on_match_res,
+            mtraversal_select_delete_on_loop_ended,
+            mtraversal_select_delete_on_trap,
+            &sd_context, ret);
+}
+
+/*
+ * This is called when select_delete traps
+ */
+static int db_select_delete_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) {
+    mtraversal_select_delete_context_t sd_context = {0};
+    Eterm* tptr = NULL;
+    Eterm tid = NIL;
+    Binary* mp = NULL;
+    Sint got = 0;
+    Sint slot_ix = 0;
+    Sint chunk_size = 0;
+    *ret = NIL;
+
+    if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) {
+        return DB_ERROR_BADPARAM;
+    }
+
+    sd_context.p = p;
+    sd_context.tb = &tbl->hash;
+    sd_context.tid = tid;
+    sd_context.hp = NULL;
+    sd_context.prev_continuation_tptr = tptr;
+    sd_context.fixated_by_me = ONLY_WRITER(p, sd_context.tb) ? 0 : 1; /* TODO: something nicer */
+    sd_context.last_pseudo_delete = (Uint) -1;
+
+    return match_traverse_continue(
+            sd_context.p, sd_context.tb, chunk_size,
+            MAX_SELECT_DELETE_ITERATIONS,
+            NULL, slot_ix, got, &mp, 1,
+            mtraversal_select_delete_on_match_res,  /* Reuse callback */
+            mtraversal_select_delete_on_loop_ended, /* Reuse callback */
+            mtraversal_select_delete_on_trap,       /* Reuse callback */
+            &sd_context, ret);
+}
+
+#undef MAX_SELECT_DELETE_ITERATIONS
+
+
+/*
+ *
+ * select_replace match traversal
+ *
+ */
+
+#define MAX_SELECT_REPLACE_ITERATIONS 1000
+
+typedef struct {
+    Process* p;
+    DbTableHash* tb;
+    Eterm tid;
+    Eterm* hp;
+    Eterm* prev_continuation_tptr;
+} mtraversal_select_replace_context_t;
+
+static int mtraversal_select_replace_on_nothing_can_match(void* context_ptr, Eterm* ret) {
+    *ret = make_small(0);
+    return DB_ERROR_NONE;
+}
+
+static int mtraversal_select_replace_on_match_res(void* context_ptr, Sint slot_ix,
+                                                  HashDbTerm*** current_ptr_ptr,
+                                                  Eterm match_res)
+{
+    mtraversal_select_replace_context_t* sr_context_ptr = (mtraversal_select_replace_context_t*) context_ptr;
+    DbTableHash* tb = sr_context_ptr->tb;
+    Eterm key = NIL;
+    HashDbTerm* new = NULL;
+    HashDbTerm* next = NULL;
+    HashValue hval = INVALID_HASH;
+
+    if (is_value(match_res) &&
+            is_value(key = db_getkey(tb->common.keypos, match_res)) &&
+            eq(key, GETKEY(tb, (**current_ptr_ptr)->dbterm.tpl)))
+    {
+        next = (**current_ptr_ptr)->next;
+        hval = (**current_ptr_ptr)->hvalue;
+        new = replace_dbterm(tb, **current_ptr_ptr, match_res);
+        new->next = next;
+        new->hvalue = hval;
+        **current_ptr_ptr = new; /* replace 'next' pointer in previous object */
+        *current_ptr_ptr = &((**current_ptr_ptr)->next); /* advance to next object */
+        return 1;
+    }
+    return 0;
+}
+
+static int mtraversal_select_replace_on_loop_ended(void* context_ptr, Sint slot_ix, Sint got,
+                                                   Sint iterations_left, Binary** mpp, Eterm* ret)
+{
+    mtraversal_select_replace_context_t* sr_context_ptr = (mtraversal_select_replace_context_t*) context_ptr;
+    ASSERT(iterations_left <= MAX_SELECT_REPLACE_ITERATIONS);
+    /* the more objects we've replaced, the more reductions we've consumed */
+    BUMP_REDS(sr_context_ptr->p,
+              MIN(MAX_SELECT_REPLACE_ITERATIONS * 2,
+                  (MAX_SELECT_REPLACE_ITERATIONS - iterations_left) + (int)got));
+    *ret = erts_make_integer(got, sr_context_ptr->p);
+    return DB_ERROR_NONE;
+}
+
+static int mtraversal_select_replace_on_trap(void* context_ptr, Sint slot_ix, Sint got,
+                                             Binary** mpp, Eterm* ret)
+{
+    mtraversal_select_replace_context_t* sr_context_ptr = (mtraversal_select_replace_context_t*) context_ptr;
+    return on_mtraversal_simple_trap(
+            &ets_select_replace_continue_exp,
+            sr_context_ptr->p,
+            sr_context_ptr->tb,
+            sr_context_ptr->tid,
+            sr_context_ptr->prev_continuation_tptr,
+            slot_ix, got, mpp, ret);
+}
+
+static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm pattern, Eterm *ret)
+{
+    mtraversal_select_replace_context_t sr_context = {0};
+    Sint chunk_size = 0;
+
+    /* Bag implementation presented both semantic consistency and performance issues,
+     * unsupported for now
+     */
+    ASSERT(!(tb->common.status & DB_BAG));
+
+    sr_context.p = p;
+    sr_context.tb = &tbl->hash;
+    sr_context.tid = NIL; // TODO
+    sr_context.hp = NULL;
+    sr_context.prev_continuation_tptr = NULL;
+
+    return match_traverse(
+            sr_context.p, sr_context.tb, pattern, chunk_size,
+            MAX_SELECT_REPLACE_ITERATIONS, NULL, 1,
+            mtraversal_select_replace_on_nothing_can_match,
+            mtraversal_select_replace_on_match_res,
+            mtraversal_select_replace_on_loop_ended,
+            mtraversal_select_replace_on_trap,
+            &sr_context, ret);
+}
+
+/*
+ * This is called when select_replace traps
+ */
+static int db_select_replace_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret)
+{
+    mtraversal_select_replace_context_t sr_context = {0};
+    Eterm* tptr = NULL;
+    Eterm tid = NIL;
+    Binary* mp = NULL;
+    Sint got = 0;
+    Sint slot_ix = 0;
+    Sint chunk_size = 0;
+    *ret = NIL;
+
+    if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) {
+        return DB_ERROR_BADPARAM;
+    }
+
+    /* Proceed */
+    sr_context.p = p;
+    sr_context.tb = &tbl->hash;
+    sr_context.tid = tid;
+    sr_context.hp = NULL;
+    sr_context.prev_continuation_tptr = tptr;
+
+    return match_traverse_continue(
+            sr_context.p, sr_context.tb, chunk_size,
+            MAX_SELECT_REPLACE_ITERATIONS,
+            NULL, slot_ix, got, &mp, 1,
+            mtraversal_select_replace_on_match_res,  /* Reuse callback */
+            mtraversal_select_replace_on_loop_ended, /* Reuse callback */
+            mtraversal_select_replace_on_trap,       /* Reuse callback */
+            &sr_context, ret);
+}
+
+
 static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
 {
     DbTableHash *tb = &tbl->hash;
@@ -2001,245 +2274,6 @@ static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
     return DB_ERROR_NONE;
 }
 
-static int db_select_replace_hash(Process *p,
-                                  DbTable *tbl,
-                                  Eterm pattern,
-                                  Eterm *ret)
-{
-    DbTableHash *tb = &tbl->hash;
-    struct mp_info mpi;
-    Uint slot_ix = 0;
-    HashDbTerm **current = NULL;
-    HashDbTerm *new = NULL, *next = NULL;
-    HashValue hval = INVALID_HASH;
-    unsigned current_list_pos = 0;
-    Eterm key;
-    Eterm match_res;
-    Eterm *hp;
-    int num_left = 1000;
-    Uint got = 0;
-    Eterm continuation;
-    int errcode;
-    Eterm mpb;
-    Eterm egot;
-    erts_smp_rwmtx_t* lck;
-
-#define RET_TO_BIF(Term,RetVal) do {		\
-	if (mpi.mp != NULL) {			\
-	    erts_bin_free(mpi.mp);		\
-	}					\
-	if (mpi.lists != mpi.dlists) {		\
-	    erts_free(ERTS_ALC_T_DB_SEL_LIST,	\
-		      (void *) mpi.lists);	\
-	}					\
-	*ret = (Term);				\
-	return RetVal;				\
-    } while(0)
-
-    /* Bag implementation presented both semantic consistency and performance issues */
-    ASSERT(!(tb->common.status & DB_BAG));
-
-    if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) {
-        RET_TO_BIF(NIL,errcode);
-    }
-
-    if (!mpi.something_can_match) {
-        RET_TO_BIF(make_small(0), DB_ERROR_NONE);
-        /* can't possibly match anything */
-    }
-
-    if (!mpi.key_given) {
-        /* Run this code if pattern is variable or GETKEY(pattern)  */
-        /* is a variable                                            */
-        lck = WLOCK_HASH(tb,slot_ix);
-        current = &BUCKET(tb,slot_ix);
-    } else {
-        /* We have at least one */
-        slot_ix = mpi.lists[current_list_pos].ix;
-        lck = WLOCK_HASH(tb, slot_ix);
-        current = mpi.lists[current_list_pos++].bucket;
-        ASSERT(*current == BUCKET(tb,slot_ix));
-    }
-
-
-    for(;;) {
-        if ((*current) == NULL) {
-            if (mpi.key_given) {  /* Key is bound */
-                WUNLOCK_HASH(lck);
-                if (current_list_pos == mpi.num_lists) {
-                    goto done;
-                } else {
-                    slot_ix = mpi.lists[current_list_pos].ix;
-                    lck = WLOCK_HASH(tb, slot_ix);
-                    current = mpi.lists[current_list_pos].bucket;
-                    ASSERT(mpi.lists[current_list_pos].bucket == &BUCKET(tb,slot_ix));
-                    ++current_list_pos;
-                }
-            } else {
-                if ((slot_ix=next_slot_w(tb,slot_ix,&lck)) == 0) {
-                    goto done;
-                }
-                if (num_left <= 0) {
-                    WUNLOCK_HASH(lck);
-                    goto trap;
-                }
-                current = &BUCKET(tb,slot_ix);
-            }
-        }
-        else if ((*current)->hvalue == INVALID_HASH) {
-            current = &((*current)->next);
-        }
-        else {
-            match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0,
-                                        &(*current)->dbterm, NULL, 0);
-            if (is_value(match_res) &&
-                is_value(key = db_getkey(tb->common.keypos, match_res)) &&
-                eq(key, GETKEY(tb, (*current)->dbterm.tpl)))
-            {
-                next = (*current)->next;
-                hval = (*current)->hvalue;
-                new = replace_dbterm(tb, *current, match_res);
-                new->next = next;
-                new->hvalue = hval;
-                *current = new;
-                ++got;
-            }
-            --num_left;
-            current = &((*current)->next);
-        }
-    }
-done:
-    // the more objects we've replaced, the more reductions we've consumed
-    BUMP_REDS(p, MIN(2000, (1000 - num_left) + (int)got));
-    RET_TO_BIF(erts_make_integer(got,p),DB_ERROR_NONE);
-trap:
-    BUMP_ALL_REDS(p);
-    if (IS_USMALL(0, got)) {
-        hp = HAlloc(p,  PROC_BIN_SIZE + 5);
-        egot = make_small(got);
-    }
-    else {
-        hp = HAlloc(p, BIG_UINT_HEAP_SIZE + PROC_BIN_SIZE + 5);
-        egot = uint_to_big(got, hp);
-        hp += BIG_UINT_HEAP_SIZE;
-    }
-    mpb = db_make_mp_binary(p,mpi.mp,&hp);
-    continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix),
-            mpb,
-            egot);
-    mpi.mp = NULL; /*otherwise the return macro will destroy it */
-    RET_TO_BIF(bif_trap1(&ets_select_replace_continue_exp, p,
-                continuation),
-            DB_ERROR_NONE);
-
-#undef RET_TO_BIF
-
-}
-
-/*
-** This is called when select_replace traps
-*/
-static int db_select_replace_continue_hash(Process *p,
-					   DbTable *tbl,
-					   Eterm continuation,
-					   Eterm *ret)
-{
-    DbTableHash *tb = &tbl->hash;
-    Uint slot_ix;
-    HashDbTerm **current = NULL;
-    HashDbTerm *new = NULL, *next = NULL;
-    HashValue hval = INVALID_HASH;
-    Eterm key;
-    Eterm match_res;
-    Eterm *hp;
-    int num_left = 1000;
-    Uint got, prev_got;
-    Eterm *tptr;
-    Binary *mp;
-    Eterm egot;
-    erts_smp_rwmtx_t* lck;
-
-#define RET_TO_BIF(Term,RetVal) do {		\
-	*ret = (Term);				\
-	return RetVal;				\
-    } while(0)
-
-
-    tptr = tuple_val(continuation);
-    slot_ix = unsigned_val(tptr[2]);
-    mp = ((ProcBin *) binary_val(tptr[3]))->val;
-    if (is_big(tptr[4])) {
-        got = big_to_uint32(tptr[4]);
-    } else {
-        got = unsigned_val(tptr[4]);
-    }
-    prev_got = got;
-
-    lck = WLOCK_HASH(tb,slot_ix);
-    if (slot_ix >= NACTIVE(tb)) {
-        WUNLOCK_HASH(lck);
-        goto done;
-    }
-    current = &BUCKET(tb,slot_ix);
-
-    for(;;) {
-        if ((*current) == NULL) {
-            if ((slot_ix=next_slot_w(tb,slot_ix,&lck)) == 0) {
-                goto done;
-            }
-            if (num_left <= 0) {
-                WUNLOCK_HASH(lck);
-                goto trap;
-            }
-            current = &BUCKET(tb,slot_ix);
-        }
-        else if ((*current)->hvalue == INVALID_HASH) {
-            current = &((*current)->next);
-        }
-        else {
-            match_res = db_match_dbterm(&tb->common, p, mp, 0,
-                                        &(*current)->dbterm, NULL, 0);
-            if (is_value(match_res) &&
-                is_value(key = db_getkey(tb->common.keypos, match_res)) &&
-                eq(key, GETKEY(tb, (*current)->dbterm.tpl)))
-            {
-                next = (*current)->next;
-                hval = (*current)->hvalue;
-                new = replace_dbterm(tb, *current, match_res);
-                new->next = next;
-                new->hvalue = hval;
-                *current = new;
-                ++got;
-            }
-            --num_left;
-            current = &((*current)->next);
-        }
-    }
-done:
-    // the more objects we've replaced, the more reductions we've consumed
-    BUMP_REDS(p, MIN(2000, (1000 - num_left) + (int)(got - prev_got)));
-    RET_TO_BIF(erts_make_integer(got,p),DB_ERROR_NONE);
-trap:
-    BUMP_ALL_REDS(p);
-    if (IS_USMALL(0, got)) {
-        hp = HAlloc(p,  5);
-        egot = make_small(got);
-    }
-    else {
-        hp = HAlloc(p, BIG_UINT_HEAP_SIZE + 5);
-        egot = uint_to_big(got, hp);
-        hp += BIG_UINT_HEAP_SIZE;
-    }
-    continuation = TUPLE4(hp, tb->common.id, make_small(slot_ix),
-            tptr[3],
-            egot);
-    RET_TO_BIF(bif_trap1(&ets_select_replace_continue_exp, p,
-                continuation),
-            DB_ERROR_NONE);
-
-#undef RET_TO_BIF
-
-}
 
 /*
 ** Other interface routines (not directly coupled to one bif)
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index af1296367a..83cbf059c5 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -1843,6 +1843,7 @@ static int db_select_replace_tree(Process *p, DbTable *tbl,
                                   Eterm pattern, Eterm *ret)
 {
     DbTableTree *tb = &tbl->tree;
+    Eterm tid = NIL; // TODO
     DbTreeStack* stack;
     struct select_replace_context sc;
     struct mp_info mpi;
@@ -1937,7 +1938,7 @@ static int db_select_replace_tree(Process *p, DbTable *tbl,
 
     continuation = TUPLE5
         (hp,
-         tb->common.id,
+         tid,
          key,
          sc.end_condition, /* From the match program, needn't be copied */
          mpb,
-- 
cgit v1.2.3


From 36d93952f6ca64192f05e0482fa55270103c8d97 Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Tue, 14 Feb 2017 22:31:31 +0000
Subject: Use magic refs on revamped ETS code

---
 erts/emulator/beam/erl_db_hash.c | 27 +++++++++++----------------
 erts/emulator/beam/erl_db_tree.c | 12 ++++--------
 2 files changed, 15 insertions(+), 24 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 1b5c8fe422..44d11dd5e7 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1495,7 +1495,7 @@ static ERTS_INLINE int on_mtraversal_simple_trap(Export* trap_function,
     Eterm mpb = NIL;
     Eterm continuation = NIL;
     int is_first_trap = (prev_continuation_tptr == NULL);
-    size_t base_halloc_sz = (is_first_trap ? PROC_BIN_SIZE : 0);
+    size_t base_halloc_sz = (is_first_trap ? ERTS_MAGIC_REF_THING_SIZE : 0);
 
     BUMP_ALL_REDS(p);
     if (IS_USMALL(0, got)) {
@@ -1509,7 +1509,7 @@ static ERTS_INLINE int on_mtraversal_simple_trap(Export* trap_function,
     }
 
     if (is_first_trap) {
-        mpb = db_make_mp_binary(p, *mpp, &hp);
+        mpb = erts_db_make_match_prog_ref(p, *mpp, &hp);
         *mpp = NULL; /* otherwise the caller will destroy it */
     }
     else {
@@ -1539,17 +1539,14 @@ static ERTS_INLINE int unpack_simple_mtraversal_continuation(Eterm continuation,
     if (arityval(*tptr) != 4)
         return 1;
 
-    if (! is_small(tptr[2])
-            || !(is_binary(tptr[3]) && thing_subtag(*binary_val(tptr[3])) == REFC_BINARY_SUBTAG)
-            || !(is_big(tptr[4]) || is_small(tptr[4])))
-    {
+    if (! is_small(tptr[2]) || !(is_big(tptr[4]) || is_small(tptr[4]))) {
         return 1;
     }
 
     *tptr_ptr = tptr;
     *tid_ptr = tptr[1];
     *slot_ix_p = unsigned_val(tptr[2]);
-    *mpp = ((ProcBin *) binary_val(tptr[3]))->val;
+    *mpp = erts_db_get_match_prog_binary_unchecked(tptr[3]);
     if (is_big(tptr[4])) {
         *got_p = big_to_uint32(tptr[4]);
     }
@@ -1629,8 +1626,8 @@ static int mtraversal_select_chunk_on_loop_ended(void* context_ptr, Sint slot_ix
                                              been in 'user space' */
             }
             if (rest != NIL || slot_ix >= 0) { /* Need more calls */
-                sc_context_ptr->hp = HAlloc(sc_context_ptr->p, 3 + 7 + PROC_BIN_SIZE);
-                mpb = db_make_mp_binary(sc_context_ptr->p, *mpp, &sc_context_ptr->hp);
+                sc_context_ptr->hp = HAlloc(sc_context_ptr->p, 3 + 7 + ERTS_MAGIC_REF_THING_SIZE);
+                mpb = erts_db_make_match_prog_ref(sc_context_ptr->p, *mpp, &sc_context_ptr->hp);
                 continuation = TUPLE6(
                         sc_context_ptr->hp,
                         sc_context_ptr->tid,
@@ -1671,8 +1668,8 @@ static int mtraversal_select_chunk_on_trap(void* context_ptr, Sint slot_ix, Sint
 
     if (sc_context_ptr->prev_continuation_tptr == NULL) {
         /* First time we're trapping */
-        hp = HAlloc(sc_context_ptr->p, 7 + PROC_BIN_SIZE);
-        mpb = db_make_mp_binary(sc_context_ptr->p, *mpp, &hp);
+        hp = HAlloc(sc_context_ptr->p, 7 + ERTS_MAGIC_REF_THING_SIZE);
+        mpb = erts_db_make_match_prog_ref(sc_context_ptr->p, *mpp, &hp);
         continuation = TUPLE6(
                 hp,
                 sc_context_ptr->tid,
@@ -1807,16 +1804,14 @@ static int db_select_continue_hash(Process* p, DbTable* tbl, Eterm continuation,
     if (arityval(*tptr) != 6)
         return DB_ERROR_BADPARAM;
 
-    if (!is_small(tptr[2]) || !is_small(tptr[3]) || !is_binary(tptr[4]) ||
+    if (!is_small(tptr[2]) || !is_small(tptr[3]) ||
             !(is_list(tptr[5]) || tptr[5] == NIL) || !is_small(tptr[6]))
         return DB_ERROR_BADPARAM;
     if ((chunk_size = signed_val(tptr[3])) < 0)
         return DB_ERROR_BADPARAM;
-    if (!(thing_subtag(*binary_val(tptr[4])) == REFC_BINARY_SUBTAG))
-        return DB_ERROR_BADPARAM;
 
-    mp = ((ProcBin *) binary_val(tptr[4]))->val;
-    if (!IsMatchProgBinary(mp))
+    mp = erts_db_get_match_prog_binary(tptr[4]);
+    if (mp == NULL)
         return DB_ERROR_BADPARAM;
 
     if ((got = signed_val(tptr[6])) < 0)
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 83cbf059c5..fea888b12d 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -1779,11 +1779,7 @@ static int db_select_replace_continue_tree(Process *p,
 
     lastkey = tptr[2];
     end_condition = tptr[3];
-    if (!(thing_subtag(*binary_val(tptr[4])) == REFC_BINARY_SUBTAG))
-        RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
-    mp = ((ProcBin *) binary_val(tptr[4]))->val;
-    if (!IsMatchProgBinary(mp))
-        RET_TO_BIF(NIL,DB_ERROR_BADPARAM);
+    mp = erts_db_get_match_prog_binary_unchecked(tptr[4]);
 
     sc.p = p;
     sc.mp = mp;
@@ -1923,18 +1919,18 @@ static int db_select_replace_tree(Process *p, DbTable *tbl,
     key = GETKEY(tb, sc.lastobj);
     sz = size_object(key);
     if (IS_USMALL(0, sc.replaced)) {
-        hp = HAlloc(p, sz + PROC_BIN_SIZE + 6);
+        hp = HAlloc(p, sz + ERTS_MAGIC_REF_THING_SIZE + 6);
         ereplaced = make_small(sc.replaced);
     }
     else {
-        hp = HAlloc(p, BIG_UINT_HEAP_SIZE + sz + PROC_BIN_SIZE + 6);
+        hp = HAlloc(p, BIG_UINT_HEAP_SIZE + sz + ERTS_MAGIC_REF_THING_SIZE + 6);
         ereplaced = uint_to_big(sc.replaced, hp);
         hp += BIG_UINT_HEAP_SIZE;
     }
     key = copy_struct(key, sz, &hp, &MSO(p));
     if (mpi.all_objects)
         (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
-    mpb = db_make_mp_binary(p,mpi.mp,&hp);
+    mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
 
     continuation = TUPLE5
         (hp,
-- 
cgit v1.2.3


From ed71ea35bad9a511125c82ce42160cad9fa8311f Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Sat, 25 Feb 2017 22:00:17 +0000
Subject: Reject unsafe matchspecs on ets:select_replace/2

Preemptively fail operation with badarg if the replacement object
might have a different key.
---
 erts/emulator/beam/erl_db_hash.c | 63 +++++++++++++++++++--------
 erts/emulator/beam/erl_db_tree.c | 52 +++++++++++++++-------
 erts/emulator/beam/erl_db_util.c | 94 ++++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_db_util.h |  1 +
 4 files changed, 177 insertions(+), 33 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 44d11dd5e7..ac31569181 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -355,6 +355,9 @@ static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb,
 	erts_smp_atomic_set_nob(&tb->segtab, (erts_aint_t) segtab);
 }
 
+/* Used by select_replace on analyze_pattern */
+typedef int (*extra_match_validator_t)(int keypos, Eterm match, Eterm guard, Eterm body);
+
 /*
 ** Forward decl's (static functions)
 */
@@ -369,8 +372,9 @@ static void shrink(DbTableHash* tb, int nitems);
 static void grow(DbTableHash* tb, int nitems);
 static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2,
 			   Uint sz, DbTableHash*);
-static int analyze_pattern(DbTableHash *tb, Eterm pattern, 
-			   struct mp_info *mpi);
+static int analyze_pattern(DbTableHash *tb, Eterm pattern,
+                           extra_match_validator_t extra_validator, /* Optional callback */
+                           struct mp_info *mpi);
 
 /*
  *  Method interface functions
@@ -1202,7 +1206,9 @@ typedef int (*mtraversal_on_trap_t)(void* context_ptr, Sint slot_ix, Sint got, B
 /*
  * Begin hash table match traversal
  */
-static int match_traverse(Process* p, DbTableHash* tb, Eterm pattern,
+static int match_traverse(Process* p, DbTableHash* tb,
+                          Eterm pattern,
+                          extra_match_validator_t extra_match_validator, /* Optional */
                           Sint chunk_size,      /* If 0, no chunking */
                           Sint iterations_left, /* Nr. of iterations left */
                           Eterm** hpp,          /* Heap */
@@ -1240,7 +1246,9 @@ static int match_traverse(Process* p, DbTableHash* tb, Eterm pattern,
 
     *ret = NIL;
 
-    if ((ret_value = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) {
+    if ((ret_value = analyze_pattern(tb, pattern, extra_match_validator, &mpi))
+            != DB_ERROR_NONE)
+    {
         goto done;
     }
 
@@ -1713,7 +1721,9 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid, Eterm patte
     sc_context.prev_continuation_tptr = NULL;
 
     return match_traverse(
-            sc_context.p, sc_context.tb, pattern, sc_context.chunk_size,
+            sc_context.p, sc_context.tb,
+            pattern, NULL,
+            sc_context.chunk_size,
             MAX_SELECT_CHUNK_ITERATIONS,
             &sc_context.hp, 0,
             mtraversal_select_chunk_on_nothing_can_match,
@@ -1906,8 +1916,8 @@ static int db_select_count_hash(Process *p, DbTable *tbl, Eterm tid, Eterm patte
 
     return match_traverse(
             scnt_context.p, scnt_context.tb,
-            pattern, chunk_size,
-            iterations_left, NULL, 0,
+            pattern, NULL,
+            chunk_size, iterations_left, NULL, 0,
             mtraversal_select_count_on_nothing_can_match,
             mtraversal_select_count_on_match_res,
             mtraversal_select_count_on_loop_ended,
@@ -2046,7 +2056,9 @@ static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, Eterm patt
     sd_context.last_pseudo_delete = (Uint) -1;
 
     return match_traverse(
-            sd_context.p, sd_context.tb, pattern, chunk_size,
+            sd_context.p, sd_context.tb,
+            pattern, NULL,
+            chunk_size,
             MAX_SELECT_DELETE_ITERATIONS, NULL, 1,
             mtraversal_select_delete_on_nothing_can_match,
             mtraversal_select_delete_on_match_res,
@@ -2120,15 +2132,18 @@ static int mtraversal_select_replace_on_match_res(void* context_ptr, Sint slot_i
 {
     mtraversal_select_replace_context_t* sr_context_ptr = (mtraversal_select_replace_context_t*) context_ptr;
     DbTableHash* tb = sr_context_ptr->tb;
+#ifdef DEBUG
     Eterm key = NIL;
+#endif
     HashDbTerm* new = NULL;
     HashDbTerm* next = NULL;
     HashValue hval = INVALID_HASH;
 
-    if (is_value(match_res) &&
-            is_value(key = db_getkey(tb->common.keypos, match_res)) &&
-            eq(key, GETKEY(tb, (**current_ptr_ptr)->dbterm.tpl)))
-    {
+    if (is_value(match_res)) {
+#ifdef DEBUG
+        ASSERT(is_value(key = db_getkey(tb->common.keypos, match_res)));
+        ASSERT(eq(key, GETKEY(tb, (**current_ptr_ptr)->dbterm.tpl)));
+#endif
         next = (**current_ptr_ptr)->next;
         hval = (**current_ptr_ptr)->hvalue;
         new = replace_dbterm(tb, **current_ptr_ptr, match_res);
@@ -2184,7 +2199,9 @@ static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm pattern, Eterm
     sr_context.prev_continuation_tptr = NULL;
 
     return match_traverse(
-            sr_context.p, sr_context.tb, pattern, chunk_size,
+            sr_context.p, sr_context.tb,
+            pattern, db_match_keeps_key,
+            chunk_size,
             MAX_SELECT_REPLACE_ITERATIONS, NULL, 1,
             mtraversal_select_replace_on_nothing_can_match,
             mtraversal_select_replace_on_match_res,
@@ -2428,7 +2445,8 @@ static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds)
 ** slots should be searched. Also compiles the match program
 */
 static int analyze_pattern(DbTableHash *tb, Eterm pattern, 
-			   struct mp_info *mpi)
+                           extra_match_validator_t extra_validator, /* Optional callback */
+                           struct mp_info *mpi)
 {
     Eterm *ptpl;
     Eterm lst, tpl, ttpl;
@@ -2466,7 +2484,10 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern,
 
     i = 0;
     for(lst = pattern; is_list(lst); lst = CDR(list_val(lst))) {
-	Eterm body;
+        Eterm match = NIL;
+        Eterm guard = NIL;
+        Eterm body = NIL;
+
 	ttpl = CAR(list_val(lst));
 	if (!is_tuple(ttpl)) {
 	    if (buff != sbuff) { 
@@ -2481,9 +2502,17 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern,
 	    }
 	    return DB_ERROR_BADPARAM;
 	}
-	matches[i] = tpl = ptpl[1];
-	guards[i] = ptpl[2];
+	matches[i] = match = tpl = ptpl[1];
+	guards[i] = guard = ptpl[2];
 	bodies[i] = body = ptpl[3];
+
+        if(extra_validator != NULL && !extra_validator(tb->common.keypos, match, guard, body)) {
+	    if (buff != sbuff) {
+		erts_free(ERTS_ALC_T_DB_TMP, buff);
+	    }
+            return DB_ERROR_BADPARAM;
+        }
+
 	if (!is_list(body) || CDR(list_val(body)) != NIL ||
 	    CAR(list_val(body)) != am_DollarUnderscore) {
 	    mpi->all_objects = 0;
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index fea888b12d..949f8c46b6 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -300,6 +300,9 @@ struct select_replace_context {
     Sint replaced;
 };
 
+/* Used by select_replace on analyze_pattern */
+typedef int (*extra_match_validator_t)(int keypos, Eterm match, Eterm guard, Eterm body);
+
 /*
 ** Forward declarations 
 */
@@ -351,7 +354,8 @@ static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key);
 static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done);
 
 static int analyze_pattern(DbTableTree *tb, Eterm pattern, 
-			   struct mp_info *mpi);
+                           extra_match_validator_t extra_validator, /* Optional callback */
+                           struct mp_info *mpi);
 static int doit_select(DbTableTree *tb,
 		       TreeDbTerm *this,
 		       void *ptr,
@@ -1132,7 +1136,7 @@ static int db_select_tree(Process *p, DbTable *tbl, Eterm tid,
     sc.got = 0;
     sc.chunk_size = 0;
 
-    if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) {
+    if ((errcode = analyze_pattern(tb, pattern, NULL, &mpi)) != DB_ERROR_NONE) {
 	RET_TO_BIF(NIL,errcode);
     }
 
@@ -1335,7 +1339,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, Eterm tid,
     sc.keypos = tb->common.keypos;
     sc.got = 0;
 
-    if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) {
+    if ((errcode = analyze_pattern(tb, pattern, NULL, &mpi)) != DB_ERROR_NONE) {
 	RET_TO_BIF(NIL,errcode);
     }
 
@@ -1438,7 +1442,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid,
     sc.got = 0;
     sc.chunk_size = chunk_size;
 
-    if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) {
+    if ((errcode = analyze_pattern(tb, pattern, NULL, &mpi)) != DB_ERROR_NONE) {
 	RET_TO_BIF(NIL,errcode);
     }
 
@@ -1680,7 +1684,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm tid,
     sc.keypos = tb->common.keypos;
     sc.tb = tb;
     
-    if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) {
+    if ((errcode = analyze_pattern(tb, pattern, NULL, &mpi)) != DB_ERROR_NONE) {
 	RET_TO_BIF(0,errcode);
     }
 
@@ -1872,7 +1876,7 @@ static int db_select_replace_tree(Process *p, DbTable *tbl,
     sc.keypos = tb->common.keypos;
     sc.replaced = 0;
 
-    if ((errcode = analyze_pattern(tb, pattern, &mpi)) != DB_ERROR_NONE) {
+    if ((errcode = analyze_pattern(tb, pattern, db_match_keeps_key, &mpi)) != DB_ERROR_NONE) {
         RET_TO_BIF(NIL,errcode);
     }
 
@@ -2193,8 +2197,9 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb,
 ** For the select functions, analyzes the pattern and determines which
 ** part of the tree should be searched. Also compiles the match program
 */
-static int analyze_pattern(DbTableTree *tb, Eterm pattern, 
-			   struct mp_info *mpi)
+static int analyze_pattern(DbTableTree *tb, Eterm pattern,
+                           extra_match_validator_t extra_validator, /* Optional callback */
+                           struct mp_info *mpi)
 {
     Eterm lst, tpl, ttpl;
     Eterm *matches,*guards, *bodies;
@@ -2232,7 +2237,10 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern,
 
     i = 0;
     for(lst = pattern; is_list(lst); lst = CDR(list_val(lst))) {
-	Eterm body;
+        Eterm match = NIL;
+        Eterm guard = NIL;
+        Eterm body = NIL;
+
 	ttpl = CAR(list_val(lst));
 	if (!is_tuple(ttpl)) {
 	    if (buff != sbuff) { 
@@ -2247,9 +2255,17 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern,
 	    }
 	    return DB_ERROR_BADPARAM;
 	}
-	matches[i] = tpl = ptpl[1];
-	guards[i] = ptpl[2];
+	matches[i] = match = tpl = ptpl[1];
+	guards[i] = guard = ptpl[2];
 	bodies[i] = body = ptpl[3];
+
+        if(extra_validator != NULL && !extra_validator(tb->common.keypos, match, guard, body)) {
+	    if (buff != sbuff) {
+		erts_free(ERTS_ALC_T_DB_TMP, buff);
+	    }
+            return DB_ERROR_BADPARAM;
+        }
+
 	if (!is_list(body) || CDR(list_val(body)) != NIL ||
 	    CAR(list_val(body)) != am_DollarUnderscore) {
 	    mpi->all_objects = 0;
@@ -3443,7 +3459,10 @@ static int doit_select_replace(DbTableTree *tb, TreeDbTerm **this, void *ptr,
                                int forward)
 {
     struct select_replace_context *sc = (struct select_replace_context *) ptr;
-    Eterm ret, key;
+    Eterm ret = NIL;
+#ifdef DEBUG
+    Eterm key = NIL;
+#endif
 
     sc->lastobj = (*this)->dbterm.tpl;
 
@@ -3456,10 +3475,11 @@ static int doit_select_replace(DbTableTree *tb, TreeDbTerm **this, void *ptr,
     ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0,
 			  &(*this)->dbterm, NULL, 0);
 
-    if (is_value(ret) &&
-        is_value(key = db_getkey(tb->common.keypos, ret)) &&
-        (cmp_key(tb, key, *this) == 0))
-    {
+    if (is_value(ret)) {
+#ifdef DEBUG
+        ASSERT(is_value(key = db_getkey(tb->common.keypos, ret)));
+        ASSERT(cmp_key(tb, key, *this) == 0);
+#endif
         *this = replace_dbterm(tb, *this, ret);
         sc->lastobj = (*this)->dbterm.tpl;
         ++(sc->replaced);
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 6f30b1d3dd..3b2e7f4407 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1119,6 +1119,100 @@ error:
     return NULL;
 }
 
+/* This is used by select_replace */
+int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body) {
+    Eterm match_key = NIL;
+    int match_key_variable = -1;
+    Eterm* body_list = NULL;
+    Eterm single_body_term = NIL;
+    Eterm* single_body_term_tpl = NULL;
+    Eterm single_body_subterm = NIL;
+    Eterm single_body_subterm_key = NIL;
+    int single_body_subterm_key_variable = -1;
+    Eterm* single_body_subterm_key_tpl = NULL;
+
+    if (!is_list(body)) {
+        return 0;
+    }
+
+    body_list = list_val(body);
+    if (CDR(body_list) != NIL) {
+        return 0;
+    }
+
+    single_body_term = CAR(body_list);
+    if (single_body_term == am_DollarUnderscore) {
+        /* same tuple is returned */
+        return 1;
+    }
+
+    if (!is_tuple(single_body_term)) {
+        return 0;
+    }
+
+    single_body_term_tpl = tuple_val(single_body_term);
+    if (arityval(*single_body_term_tpl) != 1) {
+        // not the 1-element tuple we're expecting
+        return 0;
+    }
+
+    match_key = db_getkey(keypos, match);
+    if (!is_value(match_key)) {
+        // can't get key out of match
+        return 0;
+    }
+
+    single_body_subterm = single_body_term_tpl[1];
+    single_body_subterm_key = db_getkey(keypos, single_body_subterm);
+    if (!is_value(single_body_subterm_key)) {
+        // can't get key out of single body subterm
+        return 0;
+    }
+
+    match_key_variable = db_is_variable(match_key);
+    single_body_subterm_key_variable = db_is_variable(single_body_subterm_key);
+    if (match_key_variable != -1 && match_key_variable == single_body_subterm_key_variable) {
+        /* tuple with same key is returned */
+        return 1;
+    }
+
+    if (!is_tuple(single_body_subterm_key)) {
+        /* can't possibly be an element instruction */
+        return 0;
+    }
+
+    single_body_subterm_key_tpl = tuple_val(single_body_subterm_key);
+    if (arityval(*single_body_subterm_key_tpl) != 3) {
+        /* can't possibly be an element instruction */
+        return 0;
+    }
+
+    if (single_body_subterm_key_tpl[1] != am_element) {
+        /* tag is not of an element instruction */
+        return 0;
+    }
+    if (single_body_subterm_key_tpl[3] != am_DollarUnderscore) {
+        /* even if it's an element instruction, it's not fetching from the original tuple */
+        return 0;
+    }
+
+    if (is_big(single_body_subterm_key_tpl[2])
+        && (big_to_uint32(single_body_subterm_key_tpl[2]) != keypos))
+    {
+        /* the key comes from a different position */
+        return 0;
+    }
+
+    if (is_small(single_body_subterm_key_tpl[2])
+        && (unsigned_val(single_body_subterm_key_tpl[2]) != keypos))
+    {
+        /* the key comes from a different position */
+        return 0;
+    }
+
+    return 1;
+}
+
 /* This is used when tracing */
 Eterm erts_match_set_lint(Process *p, Eterm matchexpr) {
     return db_match_set_lint(p, matchexpr, DCOMP_TRACE);
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 89ef79d2dc..1a84f20e5d 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -382,6 +382,7 @@ Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr);
 Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags);
 Binary *db_match_set_compile(Process *p, Eterm matchexpr, 
 			     Uint flags);
+int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body);
 int erts_db_match_prog_destructor(Binary *);
 
 typedef struct match_prog {
-- 
cgit v1.2.3


From d3edd7a070d50ae99cb42f0564013f122bda80b2 Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Sat, 25 Feb 2017 22:03:23 +0000
Subject: Fix typo that broke debug builds

---
 erts/emulator/beam/erl_db_hash.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index ac31569181..21af322632 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -2190,7 +2190,7 @@ static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm pattern, Eterm
     /* Bag implementation presented both semantic consistency and performance issues,
      * unsupported for now
      */
-    ASSERT(!(tb->common.status & DB_BAG));
+    ASSERT(!(tbl->hash.common.status & DB_BAG));
 
     sr_context.p = p;
     sr_context.tb = &tbl->hash;
-- 
cgit v1.2.3


From 4855e8bd2d9933020479a4fe684a0cb8bbaf6f21 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 2 Mar 2017 13:37:58 +0100
Subject: Add more complete key-safety check

---
 erts/emulator/beam/erl_db_util.c | 145 ++++++++++++++++++++++++++++++---------
 1 file changed, 111 insertions(+), 34 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 3b2e7f4407..ab45c3fe0f 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1119,17 +1119,111 @@ error:
     return NULL;
 }
 
+/*
+ * Compare a matching term 'a' with a constructing term 'b' for equality.
+ *
+ * Returns true if 'b' is guaranteed to always construct
+ * the same term as 'a' has matched.
+ */
+static int db_match_eq_body(Eterm a, Eterm b)
+{
+    DECLARE_ESTACK(s);
+    Uint arity;
+    Eterm *ap, *bp;
+    int const_mode = 0;
+    const Eterm CONST_MODE_OFF = THE_NON_VALUE;
+
+    while (1) {
+        switch(b & _TAG_PRIMARY_MASK) {
+        case TAG_PRIMARY_LIST:
+            if (!is_list(a))
+                return 0;
+            ESTACK_PUSH2(s, CDR(list_val(a)), CDR(list_val(b)));
+            a = CAR(list_val(a));
+            b = CAR(list_val(b));
+            continue; /* loop without pop */
+
+        case TAG_PRIMARY_BOXED:
+            if (is_tuple(b)) {
+                bp = tuple_val(b);
+                if (!const_mode) {
+                    if (bp[0] == make_arityval(1) && is_tuple(bp[1])) {
+                        b = bp[1]; /* double-tuple syntax */
+                    }
+                    else if (bp[0] == make_arityval(2) && bp[1] == am_const) {
+                        ESTACK_PUSH(s, CONST_MODE_OFF);
+                        const_mode = 1;   /* {const, term()} syntax */
+                        b = bp[2];
+                        continue; /* loop without pop */
+                    }
+                    else
+                        return 0; /* function call or invalid tuple syntax */
+                }
+                if (!is_tuple(a))
+                    return 0;
+
+                ap = tuple_val(a);
+                bp = tuple_val(b);
+                if (ap[0] != bp[0])
+                    return 0;
+                arity = arityval(ap[0]);
+                if (arity > 0) {
+                    a = *(++ap);
+                    b = *(++bp);
+                    while(--arity) {
+                        ESTACK_PUSH2(s, *(++ap), *(++bp));
+                    }
+                    continue; /* loop without pop */
+                }
+            }
+            else if (is_map(b)) {
+                /* We don't know what other pairs the matched map may contain */
+                return 0;
+            }
+            else if (!eq(a,b)) /* other boxed */
+                return 0;
+            break;
+
+        case TAG_PRIMARY_IMMED1:
+            if (a != b || a == am_Underscore || a == am_DollarDollar
+                || a == am_DollarUnderscore
+                || (const_mode && db_is_variable(a) >= 0)) {
+
+                return 0;
+            }
+            break;
+        default:
+            erts_exit(ERTS_ABORT_EXIT, "db_compare: "
+                      "Bad object on ESTACK: 0x%bex\n", b);
+        }
+
+pop_next:
+        if (ESTACK_ISEMPTY(s))
+            break; /* done */
+
+        b = ESTACK_POP(s);
+        if (b == CONST_MODE_OFF) {
+            ASSERT(const_mode);
+            const_mode = 0;
+            goto pop_next;
+        }
+        a = ESTACK_POP(s);
+    }
+
+    DESTROY_ESTACK(s);
+    return 1;
+}
+
 /* This is used by select_replace */
-int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body) {
-    Eterm match_key = NIL;
-    int match_key_variable = -1;
-    Eterm* body_list = NULL;
-    Eterm single_body_term = NIL;
-    Eterm* single_body_term_tpl = NULL;
-    Eterm single_body_subterm = NIL;
-    Eterm single_body_subterm_key = NIL;
-    int single_body_subterm_key_variable = -1;
-    Eterm* single_body_subterm_key_tpl = NULL;
+int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body)
+{
+    Eterm match_key;
+    Eterm* body_list;
+    Eterm single_body_term;
+    Eterm* single_body_term_tpl;
+    Eterm single_body_subterm;
+    Eterm single_body_subterm_key;
+    Eterm* single_body_subterm_key_tpl;
 
     if (!is_list(body)) {
         return 0;
@@ -1169,9 +1263,7 @@ int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body) {
         return 0;
     }
 
-    match_key_variable = db_is_variable(match_key);
-    single_body_subterm_key_variable = db_is_variable(single_body_subterm_key);
-    if (match_key_variable != -1 && match_key_variable == single_body_subterm_key_variable) {
+    if (db_match_eq_body(match_key, single_body_subterm_key)) {
         /* tuple with same key is returned */
         return 1;
     }
@@ -1187,30 +1279,15 @@ int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body) {
         return 0;
     }
 
-    if (single_body_subterm_key_tpl[1] != am_element) {
-        /* tag is not of an element instruction */
-        return 0;
-    }
-    if (single_body_subterm_key_tpl[3] != am_DollarUnderscore) {
-        /* even if it's an element instruction, it's not fetching from the original tuple */
-        return 0;
-    }
-
-    if (is_big(single_body_subterm_key_tpl[2])
-        && (big_to_uint32(single_body_subterm_key_tpl[2]) != keypos))
-    {
-        /* the key comes from a different position */
-        return 0;
-    }
-
-    if (is_small(single_body_subterm_key_tpl[2])
-        && (unsigned_val(single_body_subterm_key_tpl[2]) != keypos))
+    if (single_body_subterm_key_tpl[1] == am_element &&
+        single_body_subterm_key_tpl[3] == am_DollarUnderscore &&
+        single_body_subterm_key_tpl[2] == make_small(keypos))
     {
-        /* the key comes from a different position */
-        return 0;
+        /* {element, KeyPos, '$_'} */
+        return 1;
     }
 
-    return 1;
+    return 0;
 }
 
 /* This is used when tracing */
-- 
cgit v1.2.3


From 42ece2ed3a7022adf0568d5cd64d8df8c8e64b4b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 9 Mar 2017 16:09:10 +0100
Subject: Cleanup some unnecessary variable initialization

---
 erts/emulator/beam/erl_db_hash.c | 12 +++++-------
 erts/emulator/beam/erl_db_tree.c | 10 ++++------
 2 files changed, 9 insertions(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 21af322632..569419265b 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -2132,16 +2132,14 @@ static int mtraversal_select_replace_on_match_res(void* context_ptr, Sint slot_i
 {
     mtraversal_select_replace_context_t* sr_context_ptr = (mtraversal_select_replace_context_t*) context_ptr;
     DbTableHash* tb = sr_context_ptr->tb;
-#ifdef DEBUG
-    Eterm key = NIL;
-#endif
-    HashDbTerm* new = NULL;
-    HashDbTerm* next = NULL;
-    HashValue hval = INVALID_HASH;
+    HashDbTerm* new;
+    HashDbTerm* next;
+    HashValue hval;
 
     if (is_value(match_res)) {
 #ifdef DEBUG
-        ASSERT(is_value(key = db_getkey(tb->common.keypos, match_res)));
+        Eterm key = db_getkey(tb->common.keypos, match_res);
+        ASSERT(is_value(key));
         ASSERT(eq(key, GETKEY(tb, (**current_ptr_ptr)->dbterm.tpl)));
 #endif
         next = (**current_ptr_ptr)->next;
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 949f8c46b6..fadd63be34 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -3459,10 +3459,7 @@ static int doit_select_replace(DbTableTree *tb, TreeDbTerm **this, void *ptr,
                                int forward)
 {
     struct select_replace_context *sc = (struct select_replace_context *) ptr;
-    Eterm ret = NIL;
-#ifdef DEBUG
-    Eterm key = NIL;
-#endif
+    Eterm ret;
 
     sc->lastobj = (*this)->dbterm.tpl;
 
@@ -3477,8 +3474,9 @@ static int doit_select_replace(DbTableTree *tb, TreeDbTerm **this, void *ptr,
 
     if (is_value(ret)) {
 #ifdef DEBUG
-        ASSERT(is_value(key = db_getkey(tb->common.keypos, ret)));
-        ASSERT(cmp_key(tb, key, *this) == 0);
+        Eterm key = db_getkey(tb->common.keypos, ret);
+        ASSERT(is_value(key));
+        ASSERT(cmp_key(tb, key, old) == 0);
 #endif
         *this = replace_dbterm(tb, *this, ret);
         sc->lastobj = (*this)->dbterm.tpl;
-- 
cgit v1.2.3


From 865f06f93db7a68533bbdc4c961dd45699d6a07f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 9 Mar 2017 16:15:10 +0100
Subject: erts: Fix benign bug in match spec machine

Looks like this line has truly been dead code
as ets has (so far) always been using ERTS_PAM_COPY_RESULT
and matchPushExpr is not generated for tracing.
---
 erts/emulator/beam/erl_db_util.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index ab45c3fe0f..d4dbe3d434 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2343,7 +2343,7 @@ restart:
 		}
 	    }
 	    else {
-		*esp = term;
+		*esp++ = term;
 	    }
 	    break;
 	case matchPushArrayAsList:
-- 
cgit v1.2.3


From 5e18e917f29ed46caffa1211eb52ade01d24366a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 9 Mar 2017 16:23:04 +0100
Subject: erts: Optimize ets:select_replace to not use heap

for temporary matchspec results.

ToDo: Would be even nicer if PAM could allocate and build
the ETS objects without extra copy_struct needed.
---
 erts/emulator/beam/erl_db_hash.c |  3 ++-
 erts/emulator/beam/erl_db_tree.c | 11 +++++++++--
 erts/emulator/beam/erl_db_util.c |  7 ++++++-
 3 files changed, 17 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 569419265b..92b833468d 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -2144,9 +2144,10 @@ static int mtraversal_select_replace_on_match_res(void* context_ptr, Sint slot_i
 #endif
         next = (**current_ptr_ptr)->next;
         hval = (**current_ptr_ptr)->hvalue;
-        new = replace_dbterm(tb, **current_ptr_ptr, match_res);
+        new = new_dbterm(tb, match_res);
         new->next = next;
         new->hvalue = hval;
+        free_term(tb, **current_ptr_ptr);
         **current_ptr_ptr = new; /* replace 'next' pointer in previous object */
         *current_ptr_ptr = &((**current_ptr_ptr)->next); /* advance to next object */
         return 1;
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index fadd63be34..60bf7fc979 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -3473,13 +3473,20 @@ static int doit_select_replace(DbTableTree *tb, TreeDbTerm **this, void *ptr,
 			  &(*this)->dbterm, NULL, 0);
 
     if (is_value(ret)) {
+        TreeDbTerm* new;
+        TreeDbTerm* old = *this;
 #ifdef DEBUG
         Eterm key = db_getkey(tb->common.keypos, ret);
         ASSERT(is_value(key));
         ASSERT(cmp_key(tb, key, old) == 0);
 #endif
-        *this = replace_dbterm(tb, *this, ret);
-        sc->lastobj = (*this)->dbterm.tpl;
+        new = new_dbterm(tb, ret);
+        new->left = old->left;
+        new->right = old->right;
+        new->balance = old->balance;
+        sc->lastobj = new->dbterm.tpl;
+        *this = new;
+        free_term(tb, old);
         ++(sc->replaced);
     }
     if (--(sc->max) <= 0) {
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index d4dbe3d434..03cc11bdc4 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -5332,6 +5332,7 @@ void db_free_tmp_uncompressed(DbTerm* obj)
 Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog,
 			     int all, DbTerm* obj, Eterm** hpp, Uint extra)
 {
+    enum erts_pam_run_flags flags;
     Uint32 dummy;
     Eterm res;
 
@@ -5339,9 +5340,13 @@ Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog,
 	obj = db_alloc_tmp_uncompressed(tb, obj);
     }
 
+    flags = (hpp ?
+             ERTS_PAM_COPY_RESULT | ERTS_PAM_CONTIGUOUS_TUPLE :
+             ERTS_PAM_TMP_RESULT  | ERTS_PAM_CONTIGUOUS_TUPLE);
+
     res = db_prog_match(c_p, c_p,
                         bprog, make_tuple(obj->tpl), NULL, 0,
-			ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy);
+			flags, &dummy);
 
     if (is_value(res) && hpp!=NULL) {
 	*hpp = HAlloc(c_p, extra);
-- 
cgit v1.2.3


From 7cb2ca89f96d9724267b46d2f1eb52a2cbe7c06f Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Wed, 22 Mar 2017 23:56:56 +0000
Subject: Use ETS table id references on select_replace

---
 erts/emulator/beam/erl_db.c      | 2 +-
 erts/emulator/beam/erl_db_hash.c | 6 +++---
 erts/emulator/beam/erl_db_tree.c | 5 ++---
 erts/emulator/beam/erl_db_util.h | 1 +
 4 files changed, 7 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index d77c585628..378328856d 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -2994,7 +2994,7 @@ BIF_RETTYPE ets_select_replace_2(BIF_ALIST_2)
     if (safety == ITER_UNSAFE) {
         local_fix_table(tb);
     }
-    cret = tb->common.meth->db_select_replace(BIF_P, tb, BIF_ARG_2, &ret);
+    cret = tb->common.meth->db_select_replace(BIF_P, tb, BIF_ARG_1, BIF_ARG_2, &ret);
 
     if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) {
         fix_table_locked(BIF_P,tb);
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 92b833468d..297bef3023 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -416,7 +416,7 @@ static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid,
 static int db_select_delete_continue_hash(Process *p, DbTable *tbl,
 					  Eterm continuation, Eterm *ret);
 
-static int db_select_replace_hash(Process *p, DbTable *tbl,
+static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm tid,
                                   Eterm pattern, Eterm *ret);
 static int db_select_replace_continue_hash(Process *p, DbTable *tbl,
                                            Eterm continuation, Eterm *ret);
@@ -2181,7 +2181,7 @@ static int mtraversal_select_replace_on_trap(void* context_ptr, Sint slot_ix, Si
             slot_ix, got, mpp, ret);
 }
 
-static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm pattern, Eterm *ret)
+static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret)
 {
     mtraversal_select_replace_context_t sr_context = {0};
     Sint chunk_size = 0;
@@ -2193,7 +2193,7 @@ static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm pattern, Eterm
 
     sr_context.p = p;
     sr_context.tb = &tbl->hash;
-    sr_context.tid = NIL; // TODO
+    sr_context.tid = tid;
     sr_context.hp = NULL;
     sr_context.prev_continuation_tptr = NULL;
 
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 60bf7fc979..6334723d39 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -424,7 +424,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm tid,
 				 Eterm pattern,  Eterm *ret);
 static int db_select_delete_continue_tree(Process *p, DbTable *tbl, 
 					  Eterm continuation, Eterm *ret);
-static int db_select_replace_tree(Process *p, DbTable *tbl,
+static int db_select_replace_tree(Process *p, DbTable *tbl, Eterm tid,
                                   Eterm pattern, Eterm *ret);
 static int db_select_replace_continue_tree(Process *p, DbTable *tbl,
                                            Eterm continuation, Eterm *ret);
@@ -1839,11 +1839,10 @@ static int db_select_replace_continue_tree(Process *p,
 #undef RET_TO_BIF
 }
 
-static int db_select_replace_tree(Process *p, DbTable *tbl,
+static int db_select_replace_tree(Process *p, DbTable *tbl, Eterm tid,
                                   Eterm pattern, Eterm *ret)
 {
     DbTableTree *tb = &tbl->tree;
-    Eterm tid = NIL; // TODO
     DbTreeStack* stack;
     struct select_replace_context sc;
     struct mp_info mpi;
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 1a84f20e5d..9be77fcefa 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -171,6 +171,7 @@ typedef struct db_table_method
 				    Eterm* ret);
     int (*db_select_replace)(Process* p,
             DbTable* tb, /* [in out] */
+            Eterm tid,
             Eterm pattern,
             Eterm* ret);
     int (*db_select_replace_continue)(Process* p,
-- 
cgit v1.2.3


From 4dd2ccbdf374b4ce631d4d796bb690a55149e160 Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Thu, 23 Mar 2017 00:32:44 +0000
Subject: Remove redundant variable initializations

---
 erts/emulator/beam/erl_db_hash.c | 111 ++++++++++++++++++---------------------
 erts/emulator/beam/erl_db_tree.c |   6 +--
 2 files changed, 55 insertions(+), 62 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 297bef3023..0d908fc2c8 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1221,17 +1221,17 @@ static int match_traverse(Process* p, DbTableHash* tb,
                           void* context_ptr, /* State for callbacks above */
                           Eterm* ret)
 {
-    Sint slot_ix = -1;                /* Slot index */
-    HashDbTerm** current_ptr = NULL;  /* Refers to either the bucket pointer or
-                                       * the 'next' pointer in the previous term
-                                       */
-    HashDbTerm* saved_current = NULL; /* Helper to avoid double skip on match */
+    Sint slot_ix;                  /* Slot index */
+    HashDbTerm** current_ptr;      /* Refers to either the bucket pointer or
+                                    * the 'next' pointer in the previous term
+                                    */
+    HashDbTerm* saved_current;     /* Helper to avoid double skip on match */
     struct mp_info mpi;
-    unsigned current_list_pos = 0;    /* Prefound buckets list index */
+    unsigned current_list_pos = 0; /* Prefound buckets list index */
     Eterm match_res;
-    Sint got = 0;                     /* Matched terms counter */
-    erts_smp_rwmtx_t* lck = NULL;     /* Slot lock */
-    int ret_value = DB_ERROR_NONE;
+    Sint got = 0;                  /* Matched terms counter */
+    erts_smp_rwmtx_t* lck;         /* Slot lock */
+    int ret_value;
 #ifdef ERTS_SMP
     erts_smp_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue)
         = (lock_for_write ? WLOCK_HASH : RLOCK_HASH);
@@ -1244,8 +1244,6 @@ static int match_traverse(Process* p, DbTableHash* tb,
     Sint (*next_slot_function)(DbTableHash*, Uint, erts_smp_rwmtx_t**)
         = (lock_for_write ? next_slot_w : next_slot);
 
-    *ret = NIL;
-
     if ((ret_value = analyze_pattern(tb, pattern, extra_match_validator, &mpi))
             != DB_ERROR_NONE)
     {
@@ -1385,13 +1383,13 @@ static int match_traverse_continue(Process* p, DbTableHash* tb,
                                    Eterm* ret)
 {
     int all_objects = (*mpp)->flags & BIN_FLAG_ALL_OBJECTS;
-    HashDbTerm** current_ptr = NULL;  /* Refers to either the bucket pointer or
+    HashDbTerm** current_ptr;  /* Refers to either the bucket pointer or
                                        * the 'next' pointer in the previous term
                                        */
-    HashDbTerm* saved_current = NULL; /* Helper to avoid double skip on match */
-    Eterm match_res = NIL;
-    erts_smp_rwmtx_t* lck = NULL;
-    int ret_value = DB_ERROR_NONE;
+    HashDbTerm* saved_current; /* Helper to avoid double skip on match */
+    Eterm match_res;
+    erts_smp_rwmtx_t* lck;
+    int ret_value;
 #ifdef ERTS_SMP
     erts_smp_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue)
         = (lock_for_write ? WLOCK_HASH : RLOCK_HASH);
@@ -1404,8 +1402,6 @@ static int match_traverse_continue(Process* p, DbTableHash* tb,
     Sint (*next_slot_function)(DbTableHash* tb, Uint ix, erts_smp_rwmtx_t** lck_ptr)
         = (lock_for_write ? next_slot_w : next_slot);
 
-    *ret = NIL;
-
     if (got < 0)
         return DB_ERROR_BADPARAM;
 
@@ -1498,10 +1494,10 @@ static ERTS_INLINE int on_mtraversal_simple_trap(Export* trap_function,
                                                  Binary** mpp,
                                                  Eterm* ret)
 {
-    Eterm* hp = NULL;
-    Eterm egot = NIL;
-    Eterm mpb = NIL;
-    Eterm continuation = NIL;
+    Eterm* hp;
+    Eterm egot;
+    Eterm mpb;
+    Eterm continuation;
     int is_first_trap = (prev_continuation_tptr == NULL);
     size_t base_halloc_sz = (is_first_trap ? ERTS_MAGIC_REF_THING_SIZE : 0);
 
@@ -1541,7 +1537,7 @@ static ERTS_INLINE int unpack_simple_mtraversal_continuation(Eterm continuation,
                                                              Binary** mpp,
                                                              Sint* got_p)
 {
-    Eterm* tptr = NULL;
+    Eterm* tptr;
     ASSERT(is_tuple(continuation));
     tptr = tuple_val(continuation);
     if (arityval(*tptr) != 4)
@@ -1605,7 +1601,7 @@ static int mtraversal_select_chunk_on_loop_ended(void* context_ptr, Sint slot_ix
                                                  Sint iterations_left, Binary** mpp, Eterm* ret)
 {
     mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr;
-    Eterm mpb = NIL;
+    Eterm mpb;
 
     if (iterations_left == MAX_SELECT_CHUNK_ITERATIONS) {
         /* We didn't get to iterate a single time, which means EOT */
@@ -1668,9 +1664,9 @@ static int mtraversal_select_chunk_on_trap(void* context_ptr, Sint slot_ix, Sint
                                            Binary** mpp, Eterm* ret)
 {
     mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr;
-    Eterm mpb = NIL;
-    Eterm continuation = NIL;
-    Eterm* hp = NULL;
+    Eterm mpb;
+    Eterm continuation;
+    Eterm* hp;
 
     BUMP_ALL_REDS(sc_context_ptr->p);
 
@@ -1711,7 +1707,7 @@ static int db_select_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, in
 static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Sint chunk_size,
                                 int reverse, Eterm *ret)
 {
-    mtraversal_select_chunk_context_t sc_context = {0};
+    mtraversal_select_chunk_context_t sc_context;
     sc_context.p = p;
     sc_context.tb = &tbl->hash;
     sc_context.tid = tid;
@@ -1743,9 +1739,9 @@ static int mtraversal_select_chunk_continue_on_loop_ended(void* context_ptr, Sin
                                                           Sint iterations_left, Binary** mpp, Eterm* ret)
 {
     mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr;
-    Eterm continuation = NIL;
+    Eterm continuation;
     Eterm rest = NIL;
-    Eterm* hp = NULL;
+    Eterm* hp;
 
     ASSERT(iterations_left <= MAX_SELECT_CHUNK_ITERATIONS);
     BUMP_REDS(sc_context_ptr->p, MAX_SELECT_CHUNK_ITERATIONS - iterations_left);
@@ -1755,7 +1751,6 @@ static int mtraversal_select_chunk_continue_on_loop_ended(void* context_ptr, Sin
             /* Cannot write destructively here,
                the list may have
                been in user space */
-            rest = NIL;
             hp = HAlloc(sc_context_ptr->p, (got - sc_context_ptr->chunk_size) * 2);
             while (got-- > sc_context_ptr->chunk_size) {
                 rest = CONS(hp, CAR(list_val(sc_context_ptr->match_list)), rest);
@@ -1797,15 +1792,14 @@ static int mtraversal_select_chunk_continue_on_loop_ended(void* context_ptr, Sin
  */
 static int db_select_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) {
     mtraversal_select_chunk_context_t sc_context = {0};
-    Eterm* tptr = NULL;
-    Eterm tid = NIL;
-    Binary* mp = NULL;
-    Sint got = 0;
-    Sint slot_ix = 0;
-    Sint chunk_size = 0;
-    Eterm match_list = NIL;
+    Eterm* tptr;
+    Eterm tid;
+    Binary* mp;
+    Sint got;
+    Sint slot_ix;
+    Sint chunk_size;
+    Eterm match_list;
     Sint iterations_left = MAX_SELECT_CHUNK_ITERATIONS;
-    *ret = NIL;
 
     /* Decode continuation. We know it's a tuple but not the arity or anything else */
     ASSERT(is_tuple(continuation));
@@ -1930,11 +1924,11 @@ static int db_select_count_hash(Process *p, DbTable *tbl, Eterm tid, Eterm patte
  */
 static int db_select_count_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) {
     mtraversal_select_count_context_t scnt_context = {0};
-    Eterm* tptr = NULL;
-    Eterm tid = NIL;
-    Binary* mp = NULL;
-    Sint got = 0;
-    Sint slot_ix = 0;
+    Eterm* tptr;
+    Eterm tid;
+    Binary* mp;
+    Sint got;
+    Sint slot_ix;
     Sint chunk_size = 0;
     *ret = NIL;
 
@@ -1990,7 +1984,7 @@ static int mtraversal_select_delete_on_match_res(void* context_ptr, Sint slot_ix
 {
     HashDbTerm** current_ptr = *current_ptr_ptr;
     mtraversal_select_delete_context_t* sd_context_ptr = (mtraversal_select_delete_context_t*) context_ptr;
-    HashDbTerm* del = NULL;
+    HashDbTerm* del;
     if (match_res != am_true)
         return 0;
 
@@ -2072,13 +2066,12 @@ static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, Eterm patt
  */
 static int db_select_delete_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) {
     mtraversal_select_delete_context_t sd_context = {0};
-    Eterm* tptr = NULL;
-    Eterm tid = NIL;
-    Binary* mp = NULL;
-    Sint got = 0;
-    Sint slot_ix = 0;
+    Eterm* tptr;
+    Eterm tid;
+    Binary* mp;
+    Sint got;
+    Sint slot_ix;
     Sint chunk_size = 0;
-    *ret = NIL;
 
     if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) {
         return DB_ERROR_BADPARAM;
@@ -2215,11 +2208,11 @@ static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pat
 static int db_select_replace_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret)
 {
     mtraversal_select_replace_context_t sr_context = {0};
-    Eterm* tptr = NULL;
-    Eterm tid = NIL;
-    Binary* mp = NULL;
-    Sint got = 0;
-    Sint slot_ix = 0;
+    Eterm* tptr;
+    Eterm tid ;
+    Binary* mp;
+    Sint got;
+    Sint slot_ix;
     Sint chunk_size = 0;
     *ret = NIL;
 
@@ -2483,9 +2476,9 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern,
 
     i = 0;
     for(lst = pattern; is_list(lst); lst = CDR(list_val(lst))) {
-        Eterm match = NIL;
-        Eterm guard = NIL;
-        Eterm body = NIL;
+        Eterm match;
+        Eterm guard;
+        Eterm body;
 
 	ttpl = CAR(list_val(lst));
 	if (!is_tuple(ttpl)) {
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 6334723d39..ab8da6ccf6 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -2236,9 +2236,9 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern,
 
     i = 0;
     for(lst = pattern; is_list(lst); lst = CDR(list_val(lst))) {
-        Eterm match = NIL;
-        Eterm guard = NIL;
-        Eterm body = NIL;
+        Eterm match;
+        Eterm guard;
+        Eterm body;
 
 	ttpl = CAR(list_val(lst));
 	if (!is_tuple(ttpl)) {
-- 
cgit v1.2.3


From 14d709b0e07e899161a40bb43fc43fd6916f59ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 29 Aug 2014 18:27:45 +0200
Subject: compiler: Add is_tagged_tuple instruction

Rewrite the instruction stream on tagged tuple tests.
Tagged tuples means a tuple of any arity with an atom as its first element.
Typically records, ok-tuples and error-tuples.

from:
    ...
    {test,is_tuple,Fail,[Src]}.
    {test,test_arity,Fail,[Src,Sz]}.
    ...
    {get_tuple_element,Src,0,Dst}.
    ...
    {test,is_eq_exact,Fail,[Dst,Atom]}.
    ...
to:
    ...
    {test,is_tagged_tuple,Fail,[Src,Sz,Atom]}.
    ...
---
 erts/emulator/beam/beam_emu.c |  9 +++++++++
 erts/emulator/beam/ops.tab    | 14 ++++++++++++++
 2 files changed, 23 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8be0f58227..9a91fdce08 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -850,6 +850,15 @@ do {                                            \
   } while (0)
 #endif
 
+#define IsTaggedTuple(Src,Arityval,Tag,Fail) \
+  do {                                       \
+    if (!(is_tuple(Src) &&                   \
+         (tuple_val(Src))[0] == Arityval &&  \
+         (tuple_val(Src))[1] == Tag)) {      \
+        Fail;                                \
+    }                                        \
+  } while (0)
+
 #define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; }
 
 #define IsBinary(Src, Fail) \
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index ec36b23059..9b5bd7a749 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -620,6 +620,20 @@ test_heap Need u==1 | put_list Y=y x==0 x==0 => test_heap_1_put_list Need Y
 %macro: test_heap_1_put_list TestHeapPutList -pack
 test_heap_1_put_list I y
 
+#
+# is_tagged_tuple Fail=f Src=rxy Arity Atom=a
+#
+
+is_tagged_tuple Fail Literal=q Arity Atom => \
+    move Literal x | is_tagged_tuple Fail x Arity Atom
+is_tagged_tuple Fail=f c Arity Atom  => jump Fail
+
+%macro:is_tagged_tuple IsTaggedTuple -fail_action
+
+is_tagged_tuple f r A a
+is_tagged_tuple f x A a
+is_tagged_tuple f y A a
+
 # Test tuple & arity (head)
 
 is_tuple Fail Literal=q => move Literal x | is_tuple Fail x
-- 
cgit v1.2.3


From 5f4324e9d9a3bd99cd06f622cb3c9c50783764ac Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 24 Mar 2017 11:42:18 +0100
Subject: Add node_container_SUITE spec

---
 erts/emulator/test/Makefile                           | 3 ++-
 erts/emulator/test/emulator_node_container_SUITE.spec | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)
 create mode 100644 erts/emulator/test/emulator_node_container_SUITE.spec

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index 5478932b13..186f9fef8d 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -157,7 +157,8 @@ EMAKEFILE=Emakefile
 TEST_SPEC_FILES= emulator.spec \
 		 emulator.spec.win \
 		 emulator_bench.spec \
-		 emulator_smoke.spec
+		 emulator_smoke.spec \
+                 emulator_node_container_SUITE.spec
 
 # ----------------------------------------------------
 # Release directory specification
diff --git a/erts/emulator/test/emulator_node_container_SUITE.spec b/erts/emulator/test/emulator_node_container_SUITE.spec
new file mode 100644
index 0000000000..77c28ba7ae
--- /dev/null
+++ b/erts/emulator/test/emulator_node_container_SUITE.spec
@@ -0,0 +1,2 @@
+{enable_builtin_hooks, false}.
+{suites,"../emulator_test",node_container_SUITE}.
-- 
cgit v1.2.3


From 1bd867a8dbd88ffcee38b2179d42b563e5078820 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 24 Mar 2017 14:37:45 +0100
Subject: Fix dirty scheduler type tests

Old test for dirty schedulers didn't work with Visual C++
---
 erts/emulator/beam/beam_debug.c  | 18 ++++++++++++-----
 erts/emulator/beam/erl_nif.c     | 17 ++++++++++------
 erts/emulator/beam/erl_process.c | 42 ++++++++++++++++++++++++++--------------
 erts/emulator/beam/erl_process.h | 27 +++++---------------------
 4 files changed, 57 insertions(+), 47 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 8326d348af..b2663d3931 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -853,15 +853,23 @@ dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I)
 	    goto badarg;
 	esdp = erts_proc_sched_data(c_p);
 	if (!esdp)
-	    ERTS_BIF_PREP_RET(ret, am_error);
-	else if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+            goto scheduler_type_error;
+      
+        switch (esdp->type) {
+        case ERTS_SCHED_NORMAL:
 	    ERTS_BIF_PREP_RET(ret, am_normal);
-	else if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp))
+            break;
+        case ERTS_SCHED_DIRTY_CPU:
 	    ERTS_BIF_PREP_RET(ret, am_dirty_cpu);
-	else if (ERTS_SCHEDULER_IS_DIRTY_IO(esdp))
+            break;
+        case ERTS_SCHED_DIRTY_IO:
 	    ERTS_BIF_PREP_RET(ret, am_dirty_io);
-	else
+            break;
+        default:
+        scheduler_type_error:
 	    ERTS_BIF_PREP_RET(ret, am_error);
+            break;
+        }
     }
     else if (am_error == arg1) {
 	switch (arg2) {
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index e6da4c1a76..7c7d265257 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2820,14 +2820,19 @@ enif_thread_type(void)
     if (!esdp)
 	return ERL_NIF_THR_UNDEFINED;
 
-    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+    switch (esdp->type) {
+    case ERTS_SCHED_NORMAL:
 	return ERL_NIF_THR_NORMAL_SCHEDULER;
-
-    if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp))
+#ifdef ERTS_DIRTY_SCHEDULERS
+    case ERTS_SCHED_DIRTY_CPU:
 	return ERL_NIF_THR_DIRTY_CPU_SCHEDULER;
-
-    ASSERT(ERTS_SCHEDULER_IS_DIRTY_IO(esdp));
-    return ERL_NIF_THR_DIRTY_IO_SCHEDULER;
+    case ERTS_SCHED_DIRTY_IO:
+        return ERL_NIF_THR_DIRTY_IO_SCHEDULER;
+#endif
+    default:
+        ERTS_INTERNAL_ERROR("Invalid scheduler type");
+	return -1;
+    }
 }
 
 /* Maps */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 52d9f9ddf7..1a30f70588 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -5888,14 +5888,30 @@ erts_sched_set_wake_cleanup_threshold(char *str)
 static void
 init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
 {
-    if (!esdp)
-	awdp->sched_id = 0;
+    int id = 0;
+    if (esdp) {
+        switch (esdp->type) {
+        case ERTS_SCHED_NORMAL:
+            id = (int) esdp->no;
+            break;
 #ifdef ERTS_DIRTY_SCHEDULERS
-    else if (ERTS_SCHEDULER_IS_DIRTY(esdp))
-	awdp->sched_id = (int) ERTS_DIRTY_SCHEDULER_NO(esdp);
+        case ERTS_SCHED_DIRTY_CPU:
+            id = (int) erts_no_schedulers;
+            id += (int) esdp->dirty_no;
+            break;
+        case ERTS_SCHED_DIRTY_IO:
+            id = (int) erts_no_schedulers;
+            id += (int) erts_no_dirty_cpu_schedulers;
+            id += (int) esdp->dirty_no;
+            break;
 #endif
-    else
-	awdp->sched_id = (int) esdp->no;
+        default:
+            ERTS_INTERNAL_ERROR("Invalid scheduler type");
+            break;
+        }
+    }
+
+    awdp->sched_id = id;
     awdp->esdp = esdp;
     awdp->ssi = esdp ? esdp->ssi : NULL;
 #ifdef ERTS_SMP
@@ -5968,7 +5984,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
             ASSERT(runq == ERTS_DIRTY_IO_RUNQ);
             esdp->type = ERTS_SCHED_DIRTY_IO;
         }
-	ERTS_DIRTY_SCHEDULER_NO(esdp) = (Uint) num;
+        esdp->dirty_no = (Uint) num;
         if (num == 1) {
             /*
              * Multi-scheduling block functionality depends
@@ -5980,7 +5996,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     else {
         esdp->type = ERTS_SCHED_NORMAL;
 	esdp->no = (Uint) num;
-	ERTS_DIRTY_SCHEDULER_NO(esdp) = 0;
+	esdp->dirty_no = 0;
         runq->scheduler = esdp;
     }
     esdp->dirty_shadow_process = shadow_proc;
@@ -7717,11 +7733,11 @@ suspend_scheduler(ErtsSchedulerData *esdp)
         break;
     case ERTS_SCHED_DIRTY_CPU:
         online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN;
-        no = ERTS_DIRTY_SCHEDULER_NO(esdp);
+        no = esdp->dirty_no;
         break;
     case ERTS_SCHED_DIRTY_IO:
         online_flag = 0;
-	no = ERTS_DIRTY_SCHEDULER_NO(esdp);
+	no = esdp->dirty_no;
         break;
     default:
         ERTS_INTERNAL_ERROR("Invalid scheduler type");
@@ -8743,8 +8759,7 @@ sched_dirty_cpu_thread_func(void *vesdp)
 {
     ErtsThrPrgrCallbacks callbacks;
     ErtsSchedulerData *esdp = vesdp;
-    Uint no = ERTS_DIRTY_SCHEDULER_NO(esdp);
-    ERTS_DIRTY_SCHEDULER_TYPE(esdp) = ERTS_DIRTY_CPU_SCHEDULER;
+    Uint no = esdp->dirty_no;
     ASSERT(no != 0);
     ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(no-1)->event = erts_tse_fetch();
     callbacks.arg = (void *) esdp->ssi;
@@ -8792,8 +8807,7 @@ sched_dirty_io_thread_func(void *vesdp)
 {
     ErtsThrPrgrCallbacks callbacks;
     ErtsSchedulerData *esdp = vesdp;
-    Uint no = ERTS_DIRTY_SCHEDULER_NO(esdp);
-    ERTS_DIRTY_SCHEDULER_TYPE(esdp) = ERTS_DIRTY_IO_SCHEDULER;
+    Uint no = esdp->dirty_no;
     ASSERT(no != 0);
     ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(no-1)->event = erts_tse_fetch();
     callbacks.arg = (void *) esdp->ssi;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 8bf372dad5..5369148f87 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -627,13 +627,6 @@ typedef enum {
     ERTS_DIRTY_IO_SCHEDULER
 } ErtsDirtySchedulerType;
 
-typedef union {
-    struct {
-	ErtsDirtySchedulerType type: 1;
-	Uint num: sizeof(Uint)*8 - 1;
-    } s;
-    Uint no;
-} ErtsDirtySchedId;
 #endif
 
 struct ErtsSchedulerData_ {
@@ -660,7 +653,7 @@ struct ErtsSchedulerData_ {
     ErtsSchedType type;
     Uint no;			/* Scheduler number for normal schedulers */
 #ifdef ERTS_DIRTY_SCHEDULERS
-    ErtsDirtySchedId dirty_no;  /* Scheduler number for dirty schedulers */
+    Uint dirty_no;  /* Scheduler number for dirty schedulers */
     Process *dirty_shadow_process;
 #endif
     Port *current_port;
@@ -1556,23 +1549,13 @@ extern int erts_system_profile_ts_type;
 #define ERTS_DIRTY_IO_SCHEDULER_IX(IX)					\
   (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_io_schedulers),		\
    &erts_aligned_dirty_io_scheduler_data[(IX)].esd)
-#define ERTS_DIRTY_SCHEDULER_NO(ESDP)					\
-  ((ESDP)->dirty_no.s.num)
-#define ERTS_DIRTY_SCHEDULER_TYPE(ESDP)					\
-  ((ESDP)->dirty_no.s.type)
-#ifdef ERTS_SMP
 #define ERTS_SCHEDULER_IS_DIRTY(ESDP)					\
-  ((ESDP)->dirty_no.s.num != 0)
+  ((ESDP)->type != ERTS_SCHED_NORMAL)
 #define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP)				\
-    (ERTS_SCHEDULER_IS_DIRTY((ESDP)) & ((ESDP)->dirty_no.s.type == 0))
+    ((ESDP)->type == ERTS_SCHED_DIRTY_CPU)
 #define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP)				\
-    (ERTS_SCHEDULER_IS_DIRTY((ESDP)) & ((ESDP)->dirty_no.s.type == 1))
-#else
-#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
-#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
-#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
-#endif
-#else
+    ((ESDP)->type == ERTS_SCHED_DIRTY_IO)
+#else /* !ERTS_DIRTY_SCHEDULERS */
 #define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
 #define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
 #define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
-- 
cgit v1.2.3


From 44b5223489576a6ec719cbdb3bbdb31624326a32 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 24 Mar 2017 18:21:26 +0100
Subject: Fix double hit bug of select/3 with bound key

---
 erts/emulator/beam/erl_db_hash.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 0d908fc2c8..b58fe7a185 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1311,7 +1311,7 @@ static int match_traverse(Process* p, DbTableHash* tb,
         else if (mpi.key_given) {  /* Key is bound */
             unlock_hash_function(lck);
             if (current_list_pos == mpi.num_lists) {
-                ret_value = on_loop_ended(context_ptr, slot_ix, got, iterations_left, &mpi.mp, ret);
+                ret_value = on_loop_ended(context_ptr, -1, got, iterations_left, &mpi.mp, ret);
                 goto done;
             } else {
                 slot_ix = mpi.lists[current_list_pos].ix;
-- 
cgit v1.2.3


From 29bf1c9b7f33d64871c0f973be4a732850d98572 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Magnus=20L=C3=A5ng?= <margnus1@telia.com>
Date: Fri, 24 Mar 2017 16:23:52 +0100
Subject: Make hipe_bifs:alloc_data/3 pad addr to alignment

7814ec18b made hipe_bifs:alloc_data/3 expect alignments greater than 8
from erts_alloc(), which is not something that it guarantees. Fix it up
so that it pads the allocation up to the required alignment, in case it
does not initially satisfy the alignment requirement.
---
 erts/emulator/hipe/hipe_bif0.c | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 4babb2db4e..8b420b9e9b 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -447,6 +447,7 @@ BIF_RETTYPE hipe_bifs_alloc_data_3(BIF_ALIST_3)
 {
     Uint align;
     HipeLoaderState *stp;
+    void *aligned_block;
 
     if (is_not_small(BIF_ARG_1) || is_not_small(BIF_ARG_2) ||
 	(!(stp = get_loader_state(BIF_ARG_3))) ||
@@ -459,18 +460,14 @@ BIF_RETTYPE hipe_bifs_alloc_data_3(BIF_ALIST_3)
     stp->data_segment_size = unsigned_val(BIF_ARG_2);
     if (stp->data_segment_size == 0)
 	BIF_RET(make_small(0));
+
+    stp->data_segment_size += align-1; /* Make room to align the pointer */
     stp->data_segment = erts_alloc(ERTS_ALC_T_HIPE_LL, stp->data_segment_size);
-    if ((unsigned long)stp->data_segment & (align-1)) {
-	fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte "
-		"aligned\r\n",
-		__FUNCTION__, (unsigned long)stp->data_segment_size,
-		stp->data_segment, (unsigned long)align);
-	erts_free(ERTS_ALC_T_HIPE_LL, stp->data_segment);
-	stp->data_segment = NULL;
-	stp->data_segment_size = 0;
-	BIF_ERROR(BIF_P, EXC_NOTSUP);
-    }
-    BIF_RET(address_to_term(stp->data_segment, BIF_P));
+
+    /* Align the pointer */
+    aligned_block = (void*)((UWord)(stp->data_segment + align - 1)
+			    & ~(UWord)(align-1));
+    BIF_RET(address_to_term(aligned_block, BIF_P));
 }
 
 /*
-- 
cgit v1.2.3


From 1db00aa35f99b6876e1049b5162b31acc4f541c1 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Wed, 22 Mar 2017 16:17:43 -0700
Subject: erts: Get rid of some unused function warnings on os x

---
 erts/emulator/beam/erl_process.c | 2 ++
 erts/emulator/sys/unix/sys.c     | 2 ++
 2 files changed, 4 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 88fae30845..4fca47d399 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -280,6 +280,7 @@ schdlr_sspnd_get_nscheds(ErtsSchedTypeCounters *valp,
     }
 }
 
+#ifdef DEBUG
 static ERTS_INLINE Uint32
 schdlr_sspnd_get_nscheds_tot(ErtsSchedTypeCounters *valp)
 {
@@ -290,6 +291,7 @@ schdlr_sspnd_get_nscheds_tot(ErtsSchedTypeCounters *valp)
 #endif
     return res;
 }
+#endif
 
 static ERTS_INLINE void
 schdlr_sspnd_dec_nscheds(ErtsSchedTypeCounters *valp,
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index b48b3f8804..4fd35c4efb 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -538,11 +538,13 @@ void sys_sigrelease(int sig)
     sigprocmask(SIG_UNBLOCK, &mask, (sigset_t *)NULL);
 }
 
+#ifdef ERTS_HAVE_TRY_CATCH
 void erts_sys_sigsegv_handler(int signo) {
     if (signo == SIGSEGV) {
         longjmp(erts_sys_sigsegv_jmp, 1);
     }
 }
+#endif
 
 /*
  * Function returns 1 if we can read from all values in between
-- 
cgit v1.2.3


From 2079ac0b81d7796d3296d82ed4d11b91f699e139 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Mon, 27 Mar 2017 07:13:37 -0700
Subject: erts: Fix erts_debug:df function info output

The func_info instruction should also be dumped, so that we know
which function is which in the dump. This was accidentally removed
when introducing the new codeinfo/codemfa api.
---
 erts/emulator/beam/beam_debug.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 8326d348af..3f4b0c0c16 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -308,7 +308,7 @@ erts_debug_disassemble_1(BIF_ALIST_1)
 		BIF_RET(am_undef);
 	    }
 	}
-        code_ptr = erts_codemfa_to_code(cmfa);
+        code_ptr = (BeamInstr*)erts_code_to_codeinfo(erts_codemfa_to_code(cmfa));
     } else {
 	goto error;
     }
-- 
cgit v1.2.3


From 158de690ba539f4048ee0634e1d2ee1ffb76a68f Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Mon, 27 Mar 2017 07:20:19 -0700
Subject: erts: Fix two compiler warnings on OS X

---
 erts/emulator/beam/erl_process.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 4fca47d399..2cd65b786f 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11361,7 +11361,7 @@ erts_execute_dirty_system_task(Process *c_p)
 
 	switch (st->type) {
 	case ERTS_PSTT_CLA:
-	    ASSERT(is_value(st_res));
+	    ASSERT(is_value(cla_res));
 	    st_res = cla_res;
 	    break;
 	case ERTS_PSTT_GC_MAJOR:
-- 
cgit v1.2.3


From eb9fc2a4361037f06991d30f697f0e3ff6394117 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 28 Mar 2017 17:49:10 +0200
Subject: Fix use of uninitialized variable 'ret'

DID_TRAP will read 'ret' even when error is returned.

Found by valgrind.
---
 erts/emulator/beam/erl_db_hash.c | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index b58fe7a185..7ab27df00c 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1247,6 +1247,7 @@ static int match_traverse(Process* p, DbTableHash* tb,
     if ((ret_value = analyze_pattern(tb, pattern, extra_match_validator, &mpi))
             != DB_ERROR_NONE)
     {
+        *ret = NIL;
         goto done;
     }
 
@@ -1402,8 +1403,10 @@ static int match_traverse_continue(Process* p, DbTableHash* tb,
     Sint (*next_slot_function)(DbTableHash* tb, Uint ix, erts_smp_rwmtx_t** lck_ptr)
         = (lock_for_write ? next_slot_w : next_slot);
 
-    if (got < 0)
+    if (got < 0) {
+        *ret = NIL;
         return DB_ERROR_BADPARAM;
+    }
 
     if (slot_ix < 0 /* EOT */
        || (chunk_size && got >= chunk_size))
@@ -1416,6 +1419,7 @@ static int match_traverse_continue(Process* p, DbTableHash* tb,
     lck = lock_hash_function(tb, slot_ix);
     if (slot_ix >= NACTIVE(tb)) { /* Is this possible? */
         unlock_hash_function(lck);
+        *ret = NIL;
         ret_value = DB_ERROR_BADPARAM;
         goto done;
     }
@@ -1806,20 +1810,20 @@ static int db_select_continue_hash(Process* p, DbTable* tbl, Eterm continuation,
     tptr = tuple_val(continuation);
 
     if (arityval(*tptr) != 6)
-        return DB_ERROR_BADPARAM;
+        goto badparam;
 
     if (!is_small(tptr[2]) || !is_small(tptr[3]) ||
             !(is_list(tptr[5]) || tptr[5] == NIL) || !is_small(tptr[6]))
-        return DB_ERROR_BADPARAM;
+        goto badparam;
     if ((chunk_size = signed_val(tptr[3])) < 0)
-        return DB_ERROR_BADPARAM;
+        goto badparam;
 
     mp = erts_db_get_match_prog_binary(tptr[4]);
     if (mp == NULL)
-        return DB_ERROR_BADPARAM;
+        goto badparam;
 
     if ((got = signed_val(tptr[6])) < 0)
-        return DB_ERROR_BADPARAM;
+        goto badparam;
 
     tid = tptr[1];
     slot_ix = signed_val(tptr[2]);
@@ -1841,6 +1845,10 @@ static int db_select_continue_hash(Process* p, DbTable* tbl, Eterm continuation,
             mtraversal_select_chunk_continue_on_loop_ended,
             mtraversal_select_chunk_on_trap,      /* Reuse callback */
             &sc_context, ret);
+
+badparam:
+    *ret = NIL;
+    return DB_ERROR_BADPARAM;
 }
 
 #undef MAX_SELECT_CHUNK_ITERATIONS
@@ -1933,6 +1941,7 @@ static int db_select_count_continue_hash(Process* p, DbTable* tbl, Eterm continu
     *ret = NIL;
 
     if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) {
+        *ret = NIL;
         return DB_ERROR_BADPARAM;
     }
 
@@ -2074,6 +2083,7 @@ static int db_select_delete_continue_hash(Process* p, DbTable* tbl, Eterm contin
     Sint chunk_size = 0;
 
     if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) {
+        *ret = NIL;
         return DB_ERROR_BADPARAM;
     }
 
@@ -2217,6 +2227,7 @@ static int db_select_replace_continue_hash(Process* p, DbTable* tbl, Eterm conti
     *ret = NIL;
 
     if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) {
+        *ret = NIL;
         return DB_ERROR_BADPARAM;
     }
 
-- 
cgit v1.2.3


From d71498ac1bfa204138ec7d7ae9fb97d0ffe071b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Tue, 28 Mar 2017 16:29:01 +0200
Subject: erts: Fix faulty assert for refs in copy sharing

---
 erts/emulator/beam/copy.c     |  5 -----
 erts/emulator/beam/erl_term.h | 15 +++++++++++++--
 2 files changed, 13 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index e567eabc82..264ba89e8b 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -286,11 +286,6 @@ do {                                                                    \
     (dst) = result;                                                     \
 } while(0)
 
-#define BOXED_VISITED_MASK	 ((Eterm) 3)
-#define BOXED_VISITED		 ((Eterm) 1)
-#define BOXED_SHARED_UNPROCESSED ((Eterm) 2)
-#define BOXED_SHARED_PROCESSED	 ((Eterm) 3)
-
 #define COUNT_OFF_HEAP (0)
 
 /*
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index a602a8f7c6..097d580d99 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -873,6 +873,12 @@ typedef union {
     ErtsORefThing o;
 } ErtsRefThing;
 
+/* for copy sharing */
+#define BOXED_VISITED_MASK	 ((Eterm) 3)
+#define BOXED_VISITED		 ((Eterm) 1)
+#define BOXED_SHARED_UNPROCESSED ((Eterm) 2)
+#define BOXED_SHARED_PROCESSED	 ((Eterm) 3)
+
 #define ERTS_REF_THING_SIZE (sizeof(ErtsORefThing)/sizeof(Uint))
 #define ERTS_MAGIC_REF_THING_SIZE (sizeof(ErtsMRefThing)/sizeof(Uint))
 #define ERTS_MAX_INTERNAL_REF_SIZE (sizeof(ErtsRefThing)/sizeof(Uint))
@@ -888,9 +894,14 @@ typedef union {
 
 #  define is_ref_thing_header(x) ((x) == ERTS_REF_THING_HEADER)
 
-#define is_ordinary_ref_thing(x)					\
-    (ASSERT(is_ref_thing_header(*((Eterm *)(x)))),			\
+#ifdef SHCOPY
+#define is_ordinary_ref_thing(x)                                           \
+    (((ErtsRefThing *) (x))->o.marker == ERTS_ORDINARY_REF_MARKER)
+#else
+#define is_ordinary_ref_thing(x)                                           \
+    (ASSERT(is_ref_thing_header((*((Eterm *)(x))) & ~BOXED_VISITED_MASK)), \
      ((ErtsRefThing *) (x))->o.marker == ERTS_ORDINARY_REF_MARKER)
+#endif
 
 #define is_magic_ref_thing(x)						\
     (!is_ordinary_ref_thing((x)))
-- 
cgit v1.2.3


From 0de8b6897b15b9ee881b42ad96b0720f9c17b556 Mon Sep 17 00:00:00 2001
From: Raimo Niskanen <raimo@erlang.org>
Date: Wed, 29 Mar 2017 11:50:06 +0200
Subject: Close FD after trying to open a directory

---
 erts/emulator/drivers/unix/unix_efile.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c
index 3ff68a8859..0acc2432a7 100644
--- a/erts/emulator/drivers/unix/unix_efile.c
+++ b/erts/emulator/drivers/unix/unix_efile.c
@@ -430,6 +430,9 @@ efile_openfile(Efile_error* errInfo,	/* Where to return error codes. */
         if ( (stat("/dev/null", &nullstatbuf) < 0)
              || (statbuf.st_ino != nullstatbuf.st_ino)
              || (statbuf.st_dev != nullstatbuf.st_dev) ) {
+#ifdef HAVE_FSTAT
+            efile_closefile(fd);
+#endif
 	    errno = EISDIR;
 	    return check_error(-1, errInfo);
         }
-- 
cgit v1.2.3


From 0c2fea970cf8b24d103bcba66bf858eea5663f74 Mon Sep 17 00:00:00 2001
From: Kostis Sagonas <kostis@it.uu.se>
Date: Thu, 30 Mar 2017 00:04:27 +0200
Subject: Add a missing 0 to an address, which was suspiciously missing

---
 erts/emulator/hipe/elf64ppc.x | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/elf64ppc.x b/erts/emulator/hipe/elf64ppc.x
index 46d2632970..bb14a6cd29 100644
--- a/erts/emulator/hipe/elf64ppc.x
+++ b/erts/emulator/hipe/elf64ppc.x
@@ -28,7 +28,7 @@ SEARCH_DIR("/mnt/archive/cross-ppc64/ppc64-unknown-linux/lib");
 SECTIONS
 {
   /* Read-only sections, merged into text segment: */
-  PROVIDE (__executable_start = 0x0180000); . = 0x01800000 + SIZEOF_HEADERS;
+  PROVIDE (__executable_start = 0x01800000); . = 0x01800000 + SIZEOF_HEADERS;
   .interp         : { *(.interp) }
   .hash           : { *(.hash) }
   .dynsym         : { *(.dynsym) }
-- 
cgit v1.2.3


From 74dc036b67427987f54b45beb3adcccd2cf2e27a Mon Sep 17 00:00:00 2001
From: Manuel Rubio <manuel@altenwald.com>
Date: Fri, 31 Mar 2017 01:14:07 +0100
Subject: Add re:version/0

---
 erts/emulator/beam/bif.tab      |  1 +
 erts/emulator/beam/erl_bif_re.c | 11 +++++++++++
 2 files changed, 12 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 6f50297fc5..b0819c6427 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -392,6 +392,7 @@ bif erl_ddll:demonitor/1
 #
 # Bifs in the re module 
 #
+bif re:version/0
 bif re:compile/1
 bif re:compile/2
 bif re:run/2
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index a66b05c6ff..5dbf2a7de0 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -477,6 +477,17 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con
  * Compile BIFs
  */
 
+BIF_RETTYPE
+re_version_0(BIF_ALIST_0)
+{
+    Eterm ret;
+    size_t version_size = 0;
+    byte *version = (byte *) erts_pcre_version();
+    version_size = strlen((const char *) version);
+    ret = new_binary(BIF_P, version, version_size);
+    BIF_RET(ret);
+}
+
 static BIF_RETTYPE
 re_compile(Process* p, Eterm arg1, Eterm arg2)
 {
-- 
cgit v1.2.3


From 566ff495cb0f5f87306c81fc8d8ab2323b34d840 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 4 Apr 2017 18:13:22 +0200
Subject: erts: Refactor ErtsCodeInfo.native

into union with actual usage types.
---
 erts/emulator/beam/beam_bp.c       | 26 +++++++++++++-------------
 erts/emulator/beam/beam_emu.c      |  6 +++---
 erts/emulator/beam/beam_load.c     | 12 +++++++-----
 erts/emulator/beam/code_ix.h       |  8 +++++++-
 erts/emulator/beam/erl_bif_trace.c |  4 ++--
 erts/emulator/beam/erl_nif.c       |  4 ++--
 erts/emulator/beam/export.c        |  2 +-
 7 files changed, 35 insertions(+), 27 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index b32c74ce7a..1efe7536d6 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -327,7 +327,7 @@ erts_consolidate_bif_bp_data(void)
 static void
 consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local)
 {
-    GenericBp* g = (GenericBp *) ci->native;
+    GenericBp* g = ci->u.gen_bp;
     GenericBpData* src;
     GenericBpData* dst;
     Uint flags;
@@ -376,7 +376,7 @@ consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local)
 	    ASSERT(*erts_codeinfo_to_code(ci) !=
                    (BeamInstr) BeamOp(op_i_generic_breakpoint));
 	}
-	ci->native = 0;
+	ci->u.gen_bp = NULL;
 	Free(g);
 	return;
     }
@@ -427,7 +427,7 @@ erts_install_breakpoints(BpFunctions* f)
     for (i = 0; i < n; i++) {
 	ErtsCodeInfo* ci = f->matching[i].ci;
         BeamInstr *pc = erts_codeinfo_to_code(ci);
-	GenericBp* g = (GenericBp *) ci->native;
+	GenericBp* g = ci->u.gen_bp;
 	if (*pc != br && g) {
 	    Module* modp = f->matching[i].mod;
 
@@ -468,7 +468,7 @@ uninstall_breakpoint(ErtsCodeInfo *ci)
 {
     BeamInstr *pc = erts_codeinfo_to_code(ci);
     if (*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
-	GenericBp* g = (GenericBp *) ci->native;
+	GenericBp* g = ci->u.gen_bp;
 	if (g->data[erts_active_bp_ix()].flags == 0) {
 	    /*
 	     * The following write is not protected by any lock. We
@@ -549,7 +549,7 @@ erts_clear_trace_break(BpFunctions* f)
 void
 erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local)
 {
-    GenericBp* g = (GenericBp *) ci->native;
+    GenericBp* g = ci->u.gen_bp;
 
     if (g) {
 	Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE;
@@ -624,7 +624,7 @@ erts_clear_module_break(Module *modp) {
 	    continue;
 	uninstall_breakpoint(ci);
 	consolidate_bp_data(modp, ci, 1);
-	ASSERT(ci->native == 0);
+	ASSERT(ci->u.gen_bp == NULL);
     }
     return n;
 }
@@ -638,7 +638,7 @@ erts_clear_export_break(Module* modp, ErtsCodeInfo *ci)
     erts_commit_staged_bp();
     *erts_codeinfo_to_code(ci) = (BeamInstr) 0;
     consolidate_bp_data(modp, ci, 0);
-    ASSERT(ci->native == 0);
+    ASSERT(ci->u.gen_bp == NULL);
 }
 
 BeamInstr
@@ -651,7 +651,7 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg)
 
     ASSERT(info->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
 
-    g = (GenericBp *) info->native;
+    g = info->u.gen_bp;
     bp = &g->data[ix];
     bp_flags = bp->flags;
     ASSERT((bp_flags & ~ERTS_BPF_ALL) == 0);
@@ -754,7 +754,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 
     ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
-    g = (GenericBp *) ep->info.native;
+    g = ep->info.u.gen_bp;
     if (g) {
 	bp = &g->data[erts_active_bp_ix()];
 	bp_flags = bp->flags;
@@ -1511,7 +1511,7 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags,
     ErtsBpIndex ix = erts_staging_bp_ix();
 
     ERTS_SMP_LC_ASSERT(erts_has_code_write_permission());
-    g = (GenericBp *) ci->native;
+    g = ci->u.gen_bp;
     if (g == 0) {
 	int i;
 	if (count_op == ERTS_BREAK_RESTART || count_op == ERTS_BREAK_PAUSE) {
@@ -1523,7 +1523,7 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags,
 	for (i = 0; i < ERTS_NUM_BP_IX; i++) {
 	    g->data[i].flags = 0;
 	}
-	ci->native = (BeamInstr) g;
+	ci->u.gen_bp = g;
     }
     bp = &g->data[ix];
 
@@ -1633,7 +1633,7 @@ clear_function_break(ErtsCodeInfo *ci, Uint break_flags)
 
     ERTS_SMP_LC_ASSERT(erts_has_code_write_permission());
 
-    if ((g = (GenericBp *) ci->native) == 0) {
+    if ((g = ci->u.gen_bp) == NULL) {
 	return 1;
     }
 
@@ -1728,7 +1728,7 @@ get_time_break(ErtsCodeInfo *ci)
 static GenericBpData*
 check_break(ErtsCodeInfo *ci, Uint break_flags)
 {
-    GenericBp* g = (GenericBp *) ci->native;
+    GenericBp* g = ci->u.gen_bp;
 
     ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
     if (erts_is_native_break(ci)) {
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 9a91fdce08..3291f92b71 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -4948,14 +4948,14 @@ do {						\
 	  */
          ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
 	 ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
-	 c_p->hipe.u.ncallee = (void(*)(void)) ci->native;
+	 c_p->hipe.u.ncallee = ci->u.ncallee;
 	 ++hipe_trap_count;
 	 HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8));
      }
      OpCase(hipe_trap_call_closure): {
        ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
        ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
-       c_p->hipe.u.ncallee = (void(*)(void)) ci->native;
+       c_p->hipe.u.ncallee = ci->u.ncallee;
        ++hipe_trap_count;
        HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8));
      }
@@ -5027,7 +5027,7 @@ do {						\
       * ... remainder of original BEAM code
       */
      ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
-     struct hipe_call_count *hcc = (struct hipe_call_count*)ci->native;
+     struct hipe_call_count *hcc = ci->u.hcc;
      ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
      ASSERT(hcc != NULL);
      ASSERT(VALID_INSTR(hcc->opcode));
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 6eea963016..ae98b37956 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -5708,12 +5708,13 @@ erts_is_module_native(BeamCodeHeader* code_hdr)
 static Eterm
 native_addresses(Process* p, BeamCodeHeader* code_hdr)
 {
+    Eterm result = NIL;
+#ifdef HIPE
     int i;
     Eterm* hp;
     Uint num_functions;
     Uint need;
     Eterm* hp_end;
-    Eterm result = NIL;
 
     num_functions = code_hdr->num_functions;
     need = (6+BIG_UINT_HEAP_SIZE)*num_functions;
@@ -5725,16 +5726,17 @@ native_addresses(Process* p, BeamCodeHeader* code_hdr)
 
 	ASSERT(is_atom(ci->mfa.function)
                || is_nil(ci->mfa.function)); /* [] if BIF stub */
-	if (ci->native != 0) {
+	if (ci->u.ncallee != NULL) {
             Eterm addr;
 	    ASSERT(is_atom(ci->mfa.function));
-	    addr = erts_bld_uint(&hp, NULL, ci->native);
+	    addr = erts_bld_uint(&hp, NULL, (Uint)ci->u.ncallee);
 	    tuple = erts_bld_tuple(&hp, NULL, 3, ci->mfa.function,
                                    make_small(ci->mfa.arity), addr);
 	    result = erts_bld_cons(&hp, NULL, tuple, result);
 	}
     }
     HRelease(p, hp_end, hp);
+#endif
     return result;
 }
 
@@ -6033,7 +6035,7 @@ make_stub(ErtsCodeInfo* info, Eterm mod, Eterm func, Uint arity, Uint native, Be
     DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", erts_codeinfo_to_code(info));
     ASSERT(WORDS_PER_FUNCTION == 6);
     info->op = (BeamInstr) BeamOp(op_i_func_info_IaaI);
-    info->native = native;
+    info->u.ncallee = (void (*)(void)) native;
     info->mfa.module = mod;
     info->mfa.function = func;
     info->mfa.arity = arity;
@@ -6104,7 +6106,7 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci)
     Lambda* lp;
 
     if (is_bif(ci->mfa.module, ci->mfa.function, ci->mfa.arity)) {
-	ci->native = 0;
+	ci->u.ncallee = NULL;
 	ci->mfa.module = 0;
 	ci->mfa.function = 0;
 	ci->mfa.arity = 0;
diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h
index 1b451bf921..e802ad5dd7 100644
--- a/erts/emulator/beam/code_ix.h
+++ b/erts/emulator/beam/code_ix.h
@@ -80,7 +80,13 @@ typedef struct ErtsCodeMFA_ {
    in ops.tab to reflect the new func_info size */
 typedef struct ErtsCodeInfo_ {
     BeamInstr op;           /* OpCode(i_func_info) */
-    BeamInstr native;       /* Used by hipe and trace to store extra data */
+    union {
+        struct generic_bp* gen_bp;     /* Trace breakpoint */
+#ifdef HIPE
+        void (*ncallee)(void);
+        struct hipe_call_count* hcc;
+#endif
+    }u;
     ErtsCodeMFA mfa;
 } ErtsCodeInfo;
 
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index f471390501..023bfca797 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1737,7 +1737,7 @@ setup_bif_trace(void)
 
     for (i = 0; i < BIF_SIZE; ++i) {
 	Export *ep = bif_export[i];
-	GenericBp* g = (GenericBp *) ep->info.native;
+	GenericBp* g = ep->info.u.gen_bp;
 	if (g) {
 	    if (ExportIsBuiltIn(ep)) {
 		ASSERT(ep->beam[1]);
@@ -1755,7 +1755,7 @@ reset_bif_trace(void)
 
     for (i = 0; i < BIF_SIZE; ++i) {
 	Export *ep = bif_export[i];
-	GenericBp* g = (GenericBp *) ep->info.native;
+	GenericBp* g = ep->info.u.gen_bp;
 	if (g && g->data[active].flags == 0) {
 	    if (ExportIsBuiltIn(ep)) {
 		ASSERT(ep->beam[1]);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 63a4a997da..deed74ee49 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3653,11 +3653,11 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	    ci = *get_func_pp(this_mi->code_hdr, f_atom, f->arity);
             code_ptr = erts_codeinfo_to_code(ci);
 
-	    if (ci->native == 0) {
+	    if (ci->u.gen_bp == NULL) {
 		code_ptr[0] = (BeamInstr) BeamOp(op_call_nif);
 	    }
 	    else { /* Function traced, patch the original instruction word */
-		GenericBp* g = (GenericBp *) ci->native;
+		GenericBp* g = ci->u.gen_bp;
 		ASSERT(code_ptr[0] ==
 		       (BeamInstr) BeamOp(op_i_generic_breakpoint));
 		g->orig_instr = (BeamInstr) BeamOp(op_call_nif);
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 33ed6d7ec1..ecfef28c57 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -132,7 +132,7 @@ export_alloc(struct export_entry* tmpl_e)
 	erts_smp_atomic_add_nob(&total_entries_bytes, sizeof(*blob));
 	obj = &blob->exp;
 	obj->info.op =  0;
-	obj->info.native = 0;
+	obj->info.u.gen_bp = NULL;
 	obj->info.mfa.module = tmpl->info.mfa.module;
 	obj->info.mfa.function = tmpl->info.mfa.function;
 	obj->info.mfa.arity = tmpl->info.mfa.arity;
-- 
cgit v1.2.3


From cbaddb165393002bd76f4d70247662588d3dee26 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 4 Apr 2017 18:55:46 +0200
Subject: Refactor hipe specific code to use ErtsCodeInfo

instead of ugly negative indexing.
---
 erts/emulator/hipe/hipe_bif0.c        | 32 +++++++++++++-------------
 erts/emulator/hipe/hipe_bif0.h        |  2 +-
 erts/emulator/hipe/hipe_bif1.c        | 42 +++++++++++++++++++++--------------
 erts/emulator/hipe/hipe_mode_switch.c |  7 +++---
 erts/emulator/hipe/hipe_mode_switch.h |  2 +-
 5 files changed, 46 insertions(+), 39 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 8b420b9e9b..e51e9f2733 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -603,10 +603,7 @@ static void print_mfa(Eterm mod, Eterm fun, unsigned int ari)
 }
 #endif
 
-/*
- * Convert {M,F,A} to pointer to first insn after initial func_info.
- */
-static Uint *hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity)
+static ErtsCodeInfo* hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity)
 {
     Module *modp;
     BeamCodeHeader* code_hdr;
@@ -617,15 +614,15 @@ static Uint *hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity)
 	return NULL;
     n = code_hdr->num_functions;
     for (i = 0; i < n; ++i) {
-	Uint *code_ptr = (Uint*)code_hdr->functions[i];
-	ASSERT(code_ptr[0] == BeamOpCode(op_i_func_info_IaaI));
-	if (code_ptr[3] == name && code_ptr[4] == arity)
-	    return code_ptr+5;
+	ErtsCodeInfo *ci = code_hdr->functions[i];
+	ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
+	if (ci->mfa.function == name && ci->mfa.arity == arity)
+	    return ci;
     }
     return NULL;
 }
 
-Uint *hipe_bifs_find_pc_from_mfa(Eterm term)
+ErtsCodeInfo* hipe_bifs_find_pc_from_mfa(Eterm term)
 {
     struct hipe_mfa mfa;
 
@@ -636,10 +633,10 @@ Uint *hipe_bifs_find_pc_from_mfa(Eterm term)
 
 BIF_RETTYPE hipe_bifs_fun_to_address_1(BIF_ALIST_1)
 {
-    Eterm *pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
-    if (!pc)
+    ErtsCodeInfo* ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
+    if (!ci)
 	BIF_ERROR(BIF_P, BADARG);
-    BIF_RET(address_to_term(pc, BIF_P));
+    BIF_RET(address_to_term(erts_codeinfo_to_code(ci), BIF_P));
 }
 
 BIF_RETTYPE hipe_bifs_commit_patch_load_1(BIF_ALIST_1)
@@ -652,7 +649,7 @@ BIF_RETTYPE hipe_bifs_commit_patch_load_1(BIF_ALIST_1)
 
 BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3)
 {
-    Eterm *pc;
+    ErtsCodeInfo *ci;
     void *address;
     int is_closure;
     struct hipe_mfa mfa;
@@ -675,11 +672,12 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3)
        simply have called hipe_bifs_find_pc_from_mfa(). */
     if (!term_to_mfa(BIF_ARG_1, &mfa))
 	BIF_ERROR(BIF_P, BADARG);
-    pc = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari);
+    ci = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari);
 
-    if (pc) {
-	DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "set beam call trap at %p -> %p", pc, address);
-	hipe_set_call_trap(pc, address, is_closure);
+    if (ci) {
+	DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "set beam call trap at %p -> %p",
+                      erts_codeinfo_to_code(ci), address);
+	hipe_set_call_trap(ci, address, is_closure);
 	BIF_RET(am_true);
     }
     DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "failed set call trap to %p, no beam code found", address);
diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h
index 811c3801c1..d6be473ff5 100644
--- a/erts/emulator/hipe/hipe_bif0.h
+++ b/erts/emulator/hipe/hipe_bif0.h
@@ -26,7 +26,7 @@
 #ifndef HIPE_BIF0_H
 #define HIPE_BIF0_H
 
-extern Uint *hipe_bifs_find_pc_from_mfa(Eterm mfa);
+extern ErtsCodeInfo *hipe_bifs_find_pc_from_mfa(Eterm mfa);
 
 extern void hipe_mfa_info_table_init(void);
 extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a);
diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c
index 0ba0fa5172..0d4c1539b6 100644
--- a/erts/emulator/hipe/hipe_bif1.c
+++ b/erts/emulator/hipe/hipe_bif1.c
@@ -39,13 +39,15 @@
 
 BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1)
 {
+    ErtsCodeInfo *ci;
     Eterm *pc;
     struct hipe_call_count *hcc;
 
-    pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
-    if (!pc)
+    ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
+    if (!ci)
 	BIF_ERROR(BIF_P, BADARG);
-    ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI));
+    ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
+    pc = erts_codeinfo_to_code(ci);
     if (pc[0] == BeamOpCode(op_hipe_trap_call))
 	BIF_ERROR(BIF_P, BADARG);
     if (pc[0] == BeamOpCode(op_hipe_call_count))
@@ -53,59 +55,65 @@ BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1)
     hcc = erts_alloc(ERTS_ALC_T_HIPE_SL, sizeof(*hcc));
     hcc->count = 0;
     hcc->opcode = pc[0];
-    pc[-4] = (Eterm)hcc;
+    ci->u.hcc = hcc;
     pc[0] = BeamOpCode(op_hipe_call_count);
     BIF_RET(am_true);
 }
 
 BIF_RETTYPE hipe_bifs_call_count_off_1(BIF_ALIST_1)
 {
+    ErtsCodeInfo* ci;
     Eterm *pc;
     struct hipe_call_count *hcc;
     unsigned count;
 
-    pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
-    if (!pc)
+    ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
+    if (!ci)
 	BIF_ERROR(BIF_P, BADARG);
-    ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI));
+    ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
+    pc = erts_codeinfo_to_code(ci);
     if (pc[0] != BeamOpCode(op_hipe_call_count))
 	BIF_RET(am_false);
-    hcc = (struct hipe_call_count*)pc[-4];
+    hcc = ci->u.hcc;
     count = hcc->count;
     pc[0] = hcc->opcode;
-    pc[-4] = (Eterm)NULL;
+    ci->u.hcc = NULL;
     erts_free(ERTS_ALC_T_HIPE_SL, hcc);
     BIF_RET(make_small(count));
 }
 
 BIF_RETTYPE hipe_bifs_call_count_get_1(BIF_ALIST_1)
 {
+    ErtsCodeInfo* ci;
     Eterm *pc;
     struct hipe_call_count *hcc;
 
-    pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
-    if (!pc)
+    ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
+    if (!ci)
 	BIF_ERROR(BIF_P, BADARG);
-    ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI));
+    ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
+    pc = erts_codeinfo_to_code(ci);
     if (pc[0] != BeamOpCode(op_hipe_call_count))
 	BIF_RET(am_false);
-    hcc = (struct hipe_call_count*)pc[-4];
+    hcc = ci->u.hcc;
     BIF_RET(make_small(hcc->count));
 }
 
 BIF_RETTYPE hipe_bifs_call_count_clear_1(BIF_ALIST_1)
 {
+    ErtsCodeInfo* ci;
     Eterm *pc;
     struct hipe_call_count *hcc;
     unsigned count;
 
-    pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
-    if (!pc)
+    ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
+    if (!ci)
 	BIF_ERROR(BIF_P, BADARG);
-    ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI));
+    ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
+    pc = erts_codeinfo_to_code(ci);
     if (pc[0] != BeamOpCode(op_hipe_call_count))
 	BIF_RET(am_false);
-    hcc = (struct hipe_call_count*)pc[-4];
+    hcc = ci->u.hcc;
     count = hcc->count;
     hcc->count = 0;
     BIF_RET(make_small(count));
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index 712f65f629..1270a94986 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -176,14 +176,15 @@ void hipe_mode_switch_init(void)
     hipe_mfa_info_table_init();
 }
 
-void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure)
+void hipe_set_call_trap(ErtsCodeInfo* ci, void *nfun, int is_closure)
 {
-    HIPE_ASSERT(bfun[-5] == BeamOpCode(op_i_func_info_IaaI));
+    BeamInstr* bfun = erts_codeinfo_to_code(ci);
+    HIPE_ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
     bfun[0] =
 	is_closure
 	? BeamOpCode(op_hipe_trap_call_closure)
 	: BeamOpCode(op_hipe_trap_call);
-    bfun[-4] = (Uint)nfun;
+    ci->u.ncallee = (void (*)(void)) nfun;
 }
 
 static __inline__ void
diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h
index 334e978307..7b896872a6 100644
--- a/erts/emulator/hipe/hipe_mode_switch.h
+++ b/erts/emulator/hipe/hipe_mode_switch.h
@@ -55,7 +55,7 @@
 extern int hipe_modeswitch_debug;
 
 void hipe_mode_switch_init(void);
-void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure);
+void hipe_set_call_trap(ErtsCodeInfo*, void *nfun, int is_closure);
 Process *hipe_mode_switch(Process*, unsigned, Eterm*);
 void hipe_inc_nstack(Process *p);
 void hipe_empty_nstack(Process *p);
-- 
cgit v1.2.3


From 74f11bdee247626ed6a3afe558bde4989687e642 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 27 Mar 2017 14:39:49 +0200
Subject: Update PCRE to version 8.40

---
 erts/emulator/pcre/local_config.h       |   12 +-
 erts/emulator/pcre/pcre-8.33.tar.bz2    |  Bin 1440869 -> 0 bytes
 erts/emulator/pcre/pcre-8.33_1370.diff  |   60 -
 erts/emulator/pcre/pcre-8.40.tar.bz2    |  Bin 0 -> 1560119 bytes
 erts/emulator/pcre/pcre.h               |   24 +-
 erts/emulator/pcre/pcre_byte_order.c    |    6 +-
 erts/emulator/pcre/pcre_chartables.c    |    2 +-
 erts/emulator/pcre/pcre_compile.c       | 4549 +++++++++++++++--------
 erts/emulator/pcre/pcre_config.c        |    4 +
 erts/emulator/pcre/pcre_dfa_exec.c      |  230 +-
 erts/emulator/pcre/pcre_exec.c          |  903 +++--
 erts/emulator/pcre/pcre_fullinfo.c      |    4 +
 erts/emulator/pcre/pcre_get.c           |   23 +-
 erts/emulator/pcre/pcre_globals.c       |    4 +-
 erts/emulator/pcre/pcre_internal.h      |  441 +--
 erts/emulator/pcre/pcre_jit_compile.c   | 6074 ++++++++++++++++++++-----------
 erts/emulator/pcre/pcre_latin_1_table.c |    2 +-
 erts/emulator/pcre/pcre_maketables.c    |   27 +-
 erts/emulator/pcre/pcre_string_utils.c  |    8 +-
 erts/emulator/pcre/pcre_study.c         |  236 +-
 erts/emulator/pcre/pcre_tables.c        |  349 +-
 erts/emulator/pcre/pcre_ucd.c           | 5048 +++++++++++++------------
 erts/emulator/pcre/pcre_xclass.c        |   94 +-
 erts/emulator/pcre/ucp.h                |   31 +-
 24 files changed, 11040 insertions(+), 7091 deletions(-)
 delete mode 100644 erts/emulator/pcre/pcre-8.33.tar.bz2
 delete mode 100644 erts/emulator/pcre/pcre-8.33_1370.diff
 create mode 100644 erts/emulator/pcre/pcre-8.40.tar.bz2

(limited to 'erts/emulator')

diff --git a/erts/emulator/pcre/local_config.h b/erts/emulator/pcre/local_config.h
index 791d7f5a6b..6a0b7b4d4d 100644
--- a/erts/emulator/pcre/local_config.h
+++ b/erts/emulator/pcre/local_config.h
@@ -55,6 +55,11 @@
    --disable-stack-for-recursion). */
 #define NO_RECURSE 
 
+/* The value of PARENS_NEST_LIMIT specifies the maximum depth of nested
+   parentheses (of any kind) in a pattern. This limits the amount of system
+   stack that is used while compiling a pattern. */
+#define PARENS_NEST_LIMIT 250
+
 /* Define if linking statically (TODO: make nice with Libtool) */
 #define PCRE_STATIC 1
 
@@ -74,8 +79,11 @@
 /* Define to enable support for Unicode properties */
 #define SUPPORT_UCP
 
-/* Define to enable support for the UTF-8 Unicode encoding. */
+/* Define to any value to enable support for the UTF-8/16/32 Unicode encoding.
+   This will work even in an EBCDIC environment, but it is incompatible with
+   the EBCDIC macro. That is, PCRE can support *either* EBCDIC code *or*
+   ASCII/UTF-8/16/32, but not both at once. */
 #define SUPPORT_UTF 
 
 /* Version number of package */
-#define VERSION "8.33"
+#define VERSION "8.40"
diff --git a/erts/emulator/pcre/pcre-8.33.tar.bz2 b/erts/emulator/pcre/pcre-8.33.tar.bz2
deleted file mode 100644
index 0cea71c8b6..0000000000
Binary files a/erts/emulator/pcre/pcre-8.33.tar.bz2 and /dev/null differ
diff --git a/erts/emulator/pcre/pcre-8.33_1370.diff b/erts/emulator/pcre/pcre-8.33_1370.diff
deleted file mode 100644
index d62398985d..0000000000
--- a/erts/emulator/pcre/pcre-8.33_1370.diff
+++ /dev/null
@@ -1,60 +0,0 @@
---- code/trunk/pcre_exec.c	2013/07/02 18:37:36	1346
-+++ code/trunk/pcre_exec.c	2013/07/26 10:03:38	1350
-@@ -5637,7 +5637,7 @@
-           }
-         }
- 
--      /* Match extended Unicode sequences. We will get here only if the
-+      /* Match extended Unicode grapheme clusters. We will get here only if the
-       support is in the binary; otherwise a compile-time error occurs. */
- 
-       else if (ctype == OP_EXTUNI)
-@@ -5670,21 +5670,41 @@
-         /* eptr is now past the end of the maximum run */
- 
-         if (possessive) continue;    /* No backtracking */
-+         
-         for(;;)
-           {
--          if (eptr == pp) goto TAIL_RECURSE;
-+          int lgb, rgb; 
-+          PCRE_PUCHAR fptr;
-+            
-+          if (eptr == pp) goto TAIL_RECURSE;   /* At start of char run */
-           RMATCH(eptr, ecode, offset_top, md, eptrb, RM45);
-           if (rrc != MATCH_NOMATCH) RRETURN(rrc);
-+
-+          /* Backtracking over an extended grapheme cluster involves inspecting
-+          the previous two characters (if present) to see if a break is
-+          permitted between them. */
-+ 
-           eptr--;
--          for (;;)                        /* Move back over one extended */
-+          if (!utf) c = *eptr; else
-+            {
-+            BACKCHAR(eptr);
-+            GETCHAR(c, eptr);
-+            }
-+          rgb = UCD_GRAPHBREAK(c);
-+
-+          for (;;)
-             {
--            if (!utf) c = *eptr; else
-+            if (eptr == pp) goto TAIL_RECURSE;   /* At start of char run */
-+            fptr = eptr - 1;
-+            if (!utf) c = *fptr; else
-               {
--              BACKCHAR(eptr);
--              GETCHAR(c, eptr);
-+              BACKCHAR(fptr);
-+              GETCHAR(c, fptr);
-               }
--            if (UCD_CATEGORY(c) != ucp_M) break;
--            eptr--;
-+            lgb = UCD_GRAPHBREAK(c);        
-+            if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break;
-+            eptr = fptr;
-+            rgb = lgb;
-             }
-           }
-         }
diff --git a/erts/emulator/pcre/pcre-8.40.tar.bz2 b/erts/emulator/pcre/pcre-8.40.tar.bz2
new file mode 100644
index 0000000000..6147917f4e
Binary files /dev/null and b/erts/emulator/pcre/pcre-8.40.tar.bz2 differ
diff --git a/erts/emulator/pcre/pcre.h b/erts/emulator/pcre/pcre.h
index 57efdd01f5..9cbd9c0293 100644
--- a/erts/emulator/pcre/pcre.h
+++ b/erts/emulator/pcre/pcre.h
@@ -5,7 +5,7 @@
 /* This is the public header file for the PCRE library, to be #included by
 applications that call the PCRE functions.
 
-           Copyright (c) 1997-2013 University of Cambridge
+           Copyright (c) 1997-2014 University of Cambridge
 
 -----------------------------------------------------------------------------
 Redistribution and use in source and binary forms, with or without
@@ -43,9 +43,9 @@ POSSIBILITY OF SUCH DAMAGE.
 /* The current PCRE version information. */
 
 #define PCRE_MAJOR          8
-#define PCRE_MINOR          33
+#define PCRE_MINOR          40
 #define PCRE_PRERELEASE     
-#define PCRE_DATE           2013-05-28
+#define PCRE_DATE           2017-01-11
 
 /* When an application links to a PCRE DLL in Windows, the symbols that are
 imported have to be identified as such. When building PCRE, the appropriate
@@ -151,7 +151,10 @@ with J. */
 #define PCRE_NEVER_UTF          0x00010000  /* C1        ) Overlaid */
 #define PCRE_DFA_SHORTEST       0x00010000  /*      D    ) Overlaid */
 
-#define PCRE_DFA_RESTART        0x00020000  /*      D   */
+/* This pair use the same bit. */
+#define PCRE_NO_AUTO_POSSESS    0x00020000  /* C1        ) Overlaid */
+#define PCRE_DFA_RESTART        0x00020000  /*      D    ) Overlaid */
+
 #define PCRE_FIRSTLINE          0x00040000  /* C3       */
 #define PCRE_DUPNAMES           0x00080000  /* C1       */
 #define PCRE_NEWLINE_CR         0x00100000  /* C3 E D   */
@@ -281,6 +284,7 @@ with J. */
 #define PCRE_INFO_REQUIREDCHARFLAGS 22
 #define PCRE_INFO_MATCHLIMIT        23
 #define PCRE_INFO_RECURSIONLIMIT    24
+#define PCRE_INFO_MATCH_EMPTY       25
 
 /* Request types for pcre_config(). Do not re-arrange, in order to remain
 compatible. */
@@ -298,6 +302,7 @@ compatible. */
 #define PCRE_CONFIG_UTF16                  10
 #define PCRE_CONFIG_JITTARGET              11
 #define PCRE_CONFIG_UTF32                  12
+#define PCRE_CONFIG_PARENS_LIMIT           13
 
 /* Request types for pcre_study(). Do not re-arrange, in order to remain
 compatible. */
@@ -513,24 +518,28 @@ PCRE_EXP_DECL void  (*erts_pcre_free)(void *);
 PCRE_EXP_DECL void *(*erts_pcre_stack_malloc)(size_t);
 PCRE_EXP_DECL void  (*erts_pcre_stack_free)(void *);
 PCRE_EXP_DECL int   (*erts_pcre_callout)(erts_pcre_callout_block *);
+PCRE_EXP_DECL int   (*erts_pcre_stack_guard)(void);
 #else
 PCRE_EXP_DECL void *(*pcre_malloc)(size_t);
 PCRE_EXP_DECL void  (*pcre_free)(void *);
 PCRE_EXP_DECL void *(*pcre_stack_malloc)(size_t);
 PCRE_EXP_DECL void  (*pcre_stack_free)(void *);
 PCRE_EXP_DECL int   (*pcre_callout)(pcre_callout_block *);
+PCRE_EXP_DECL int   (*pcre_stack_guard)(void);
 #endif
 PCRE_EXP_DECL void *(*pcre16_malloc)(size_t);
 PCRE_EXP_DECL void  (*pcre16_free)(void *);
 PCRE_EXP_DECL void *(*pcre16_stack_malloc)(size_t);
 PCRE_EXP_DECL void  (*pcre16_stack_free)(void *);
 PCRE_EXP_DECL int   (*pcre16_callout)(pcre16_callout_block *);
+PCRE_EXP_DECL int   (*pcre16_stack_guard)(void);
 
 PCRE_EXP_DECL void *(*pcre32_malloc)(size_t);
 PCRE_EXP_DECL void  (*pcre32_free)(void *);
 PCRE_EXP_DECL void *(*pcre32_stack_malloc)(size_t);
 PCRE_EXP_DECL void  (*pcre32_stack_free)(void *);
 PCRE_EXP_DECL int   (*pcre32_callout)(pcre32_callout_block *);
+PCRE_EXP_DECL int   (*pcre32_stack_guard)(void);
 #else   /* VPCOMPAT */
 #if defined(ERLANG_INTEGRATION)
 PCRE_EXP_DECL void *erts_pcre_malloc(size_t);
@@ -538,12 +547,14 @@ PCRE_EXP_DECL void  erts_pcre_free(void *);
 PCRE_EXP_DECL void *erts_pcre_stack_malloc(size_t);
 PCRE_EXP_DECL void  erts_pcre_stack_free(void *);
 PCRE_EXP_DECL int   erts_pcre_callout(erts_pcre_callout_block *);
+PCRE_EXP_DECL int   erts_pcre_stack_guard(void);
 #else
 PCRE_EXP_DECL void *pcre_malloc(size_t);
 PCRE_EXP_DECL void  pcre_free(void *);
 PCRE_EXP_DECL void *pcre_stack_malloc(size_t);
 PCRE_EXP_DECL void  pcre_stack_free(void *);
 PCRE_EXP_DECL int   pcre_callout(pcre_callout_block *);
+PCRE_EXP_DECL int   pcre_stack_guard(void);
 #endif
 
 PCRE_EXP_DECL void *pcre16_malloc(size_t);
@@ -551,12 +562,14 @@ PCRE_EXP_DECL void  pcre16_free(void *);
 PCRE_EXP_DECL void *pcre16_stack_malloc(size_t);
 PCRE_EXP_DECL void  pcre16_stack_free(void *);
 PCRE_EXP_DECL int   pcre16_callout(pcre16_callout_block *);
+PCRE_EXP_DECL int   pcre16_stack_guard(void);
 
 PCRE_EXP_DECL void *pcre32_malloc(size_t);
 PCRE_EXP_DECL void  pcre32_free(void *);
 PCRE_EXP_DECL void *pcre32_stack_malloc(size_t);
 PCRE_EXP_DECL void  pcre32_stack_free(void *);
 PCRE_EXP_DECL int   pcre32_callout(pcre32_callout_block *);
+PCRE_EXP_DECL int   pcre32_stack_guard(void);
 #endif  /* VPCOMPAT */
 
 /* User defined callback which provides a stack just before the match starts. */
@@ -811,6 +824,9 @@ PCRE_EXP_DECL void pcre16_assign_jit_stack(pcre16_extra *,
                   pcre16_jit_callback, void *);
 PCRE_EXP_DECL void pcre32_assign_jit_stack(pcre32_extra *,
                   pcre32_jit_callback, void *);
+PCRE_EXP_DECL void pcre_jit_free_unused_memory(void);
+PCRE_EXP_DECL void pcre16_jit_free_unused_memory(void);
+PCRE_EXP_DECL void pcre32_jit_free_unused_memory(void);
 
 #ifdef ERLANG_INTEGRATION
 PCRE_EXP_DECL void  erts_pcre_free_restart_data(void *restart_data);
diff --git a/erts/emulator/pcre/pcre_byte_order.c b/erts/emulator/pcre/pcre_byte_order.c
index 710676988f..f99317c44e 100644
--- a/erts/emulator/pcre/pcre_byte_order.c
+++ b/erts/emulator/pcre/pcre_byte_order.c
@@ -6,7 +6,7 @@
 and semantics are as close as possible to those of the Perl 5 language.
 
                        Written by Philip Hazel
-           Copyright (c) 1997-2013 University of Cambridge
+           Copyright (c) 1997-2014 University of Cambridge
 
 -----------------------------------------------------------------------------
 Redistribution and use in source and binary forms, with or without
@@ -316,9 +316,9 @@ while(TRUE)
   ptr++;
   }
 /* Control should never reach here in 16/32 bit mode. */
-#endif /* !COMPILE_PCRE8 */
-
+#else  /* In 8-bit mode, the pattern does not need to be processed. */
 return 0;
+#endif /* !COMPILE_PCRE8 */
 }
 
 /* End of pcre_byte_order.c */
diff --git a/erts/emulator/pcre/pcre_chartables.c b/erts/emulator/pcre/pcre_chartables.c
index 0d7ecd5261..b3d9020f25 100644
--- a/erts/emulator/pcre/pcre_chartables.c
+++ b/erts/emulator/pcre/pcre_chartables.c
@@ -163,7 +163,7 @@ graph, print, punct, and cntrl. Other classes are built from combinations. */
 */
 
   0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /*   0-  7 */
-  0x00,0x01,0x01,0x00,0x01,0x01,0x00,0x00, /*   8- 15 */
+  0x00,0x01,0x01,0x01,0x01,0x01,0x00,0x00, /*   8- 15 */
   0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /*  16- 23 */
   0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /*  24- 31 */
   0x01,0x00,0x00,0x00,0x80,0x00,0x00,0x00, /*    - '  */
diff --git a/erts/emulator/pcre/pcre_compile.c b/erts/emulator/pcre/pcre_compile.c
index d48126a55d..6e841c9cf8 100644
--- a/erts/emulator/pcre/pcre_compile.c
+++ b/erts/emulator/pcre/pcre_compile.c
@@ -6,7 +6,7 @@
 and semantics are as close as possible to those of the Perl 5 language.
 
                        Written by Philip Hazel
-           Copyright (c) 1997-2013 University of Cambridge
+           Copyright (c) 1997-2016 University of Cambridge
 
 -----------------------------------------------------------------------------
 Redistribution and use in source and binary forms, with or without
@@ -48,8 +48,8 @@ supporting internal functions that are not used by other modules. */
 #endif
 
 #define NLBLOCK cd             /* Block containing newline information */
-#define PSSTART start_pattern  /* Field containing processed string start */
-#define PSEND   end_pattern    /* Field containing processed string end */
+#define PSSTART start_pattern  /* Field containing pattern start */
+#define PSEND   end_pattern    /* Field containing pattern end */
 
 #include "pcre_internal.h"
 
@@ -116,6 +116,13 @@ kicks in at the same number of forward references in all cases. */
 #define COMPILE_WORK_SIZE (2048*LINK_SIZE)
 #define COMPILE_WORK_SIZE_MAX (100*COMPILE_WORK_SIZE)
 
+/* This value determines the size of the initial vector that is used for
+remembering named groups during the pre-compile. It is allocated on the stack,
+but if it is too small, it is expanded using malloc(), in a similar way to the
+workspace. The value is the number of slots in the list. */
+
+#define NAMED_GROUP_LIST_SIZE  20
+
 /* The overrun tests check for a slightly smaller size so that they detect the
 overrun before it actually does run off the end of the data block. */
 
@@ -168,7 +175,7 @@ static const short int escapes[] = {
      -ESC_Z,                  CHAR_LEFT_SQUARE_BRACKET,
      CHAR_BACKSLASH,          CHAR_RIGHT_SQUARE_BRACKET,
      CHAR_CIRCUMFLEX_ACCENT,  CHAR_UNDERSCORE,
-     CHAR_GRAVE_ACCENT,       7,
+     CHAR_GRAVE_ACCENT,       ESC_a,
      -ESC_b,                  0,
      -ESC_d,                  ESC_e,
      ESC_f,                   0,
@@ -196,9 +203,9 @@ static const short int escapes[] = {
 /*  68 */     0,     0,    '|',     ',',    '%',   '_',    '>',    '?',
 /*  70 */     0,     0,      0,       0,      0,     0,      0,      0,
 /*  78 */     0,   '`',    ':',     '#',    '@',  '\'',    '=',    '"',
-/*  80 */     0,     7, -ESC_b,       0, -ESC_d, ESC_e,  ESC_f,      0,
+/*  80 */     0, ESC_a, -ESC_b,       0, -ESC_d, ESC_e,  ESC_f,      0,
 /*  88 */-ESC_h,     0,      0,     '{',      0,     0,      0,      0,
-/*  90 */     0,     0, -ESC_k,     'l',      0, ESC_n,      0, -ESC_p,
+/*  90 */     0,     0, -ESC_k,       0,      0, ESC_n,      0, -ESC_p,
 /*  98 */     0, ESC_r,      0,     '}',      0,     0,      0,      0,
 /*  A0 */     0,   '~', -ESC_s, ESC_tee,      0,-ESC_v, -ESC_w,      0,
 /*  A8 */     0,-ESC_z,      0,       0,      0,   '[',      0,      0,
@@ -213,6 +220,12 @@ static const short int escapes[] = {
 /*  F0 */     0,     0,      0,       0,      0,     0,      0,      0,
 /*  F8 */     0,     0,      0,       0,      0,     0,      0,      0
 };
+
+/* We also need a table of characters that may follow \c in an EBCDIC
+environment for characters 0-31. */
+
+static unsigned char ebcdic_escape_c[] = "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_";
+
 #endif
 
 
@@ -254,11 +267,25 @@ static const verbitem verbs[] = {
 static const int verbcount = sizeof(verbs)/sizeof(verbitem);
 
 
+/* Substitutes for [[:<:]] and [[:>:]], which mean start and end of word in
+another regex library. */
+
+static const pcre_uchar sub_start_of_word[] = {
+  CHAR_BACKSLASH, CHAR_b, CHAR_LEFT_PARENTHESIS, CHAR_QUESTION_MARK,
+  CHAR_EQUALS_SIGN, CHAR_BACKSLASH, CHAR_w, CHAR_RIGHT_PARENTHESIS, '\0' };
+
+static const pcre_uchar sub_end_of_word[] = {
+  CHAR_BACKSLASH, CHAR_b, CHAR_LEFT_PARENTHESIS, CHAR_QUESTION_MARK,
+  CHAR_LESS_THAN_SIGN, CHAR_EQUALS_SIGN, CHAR_BACKSLASH, CHAR_w,
+  CHAR_RIGHT_PARENTHESIS, '\0' };
+
+
 /* Tables of names of POSIX character classes and their lengths. The names are
 now all in a single string, to reduce the number of relocations when a shared
 library is dynamically loaded. The list of lengths is terminated by a zero
 length entry. The first three must be alpha, lower, upper, as this is assumed
-for handling case independence. */
+for handling case independence. The indices for graph, print, and punct are
+needed, so identify them. */
 
 static const char posix_names[] =
   STRING_alpha0 STRING_lower0 STRING_upper0 STRING_alnum0
@@ -269,6 +296,11 @@ static const char posix_names[] =
 static const pcre_uint8 posix_name_lengths[] = {
   5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 6, 0 };
 
+#define PC_GRAPH  8
+#define PC_PRINT  9
+#define PC_PUNCT 10
+
+
 /* Table of class bit maps for each POSIX class. Each class is formed from a
 base map, with an optional addition or removal of another map. Then, for some
 classes, there is some additional tweaking: for [:blank:] the vertical space
@@ -296,9 +328,8 @@ static const int posix_class_maps[] = {
   cbit_xdigit,-1,          0              /* xdigit */
 };
 
-/* Table of substitutes for \d etc when PCRE_UCP is set. The POSIX class
-substitutes must be in the order of the names, defined above, and there are
-both positive and negative cases. NULL means no substitute. */
+/* Table of substitutes for \d etc when PCRE_UCP is set. They are replaced by
+Unicode property escapes. */
 
 #ifdef SUPPORT_UCP
 static const pcre_uchar string_PNd[]  = {
@@ -323,12 +354,18 @@ static const pcre_uchar string_pXwd[] = {
 static const pcre_uchar *substitutes[] = {
   string_PNd,           /* \D */
   string_pNd,           /* \d */
-  string_PXsp,          /* \S */       /* NOTE: Xsp is Perl space */
-  string_pXsp,          /* \s */
+  string_PXsp,          /* \S */   /* Xsp is Perl space, but from 8.34, Perl */
+  string_pXsp,          /* \s */   /* space and POSIX space are the same. */
   string_PXwd,          /* \W */
   string_pXwd           /* \w */
 };
 
+/* The POSIX class substitutes must be in the order of the POSIX class names,
+defined above, and there are both positive and negative cases. NULL means no
+general substitute of a Unicode property escape (\p or \P). However, for some
+POSIX classes (e.g. graph, print, punct) a special property code is compiled
+directly. */
+
 static const pcre_uchar string_pL[] =   {
   CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET,
   CHAR_L, CHAR_RIGHT_CURLY_BRACKET, '\0' };
@@ -376,8 +413,8 @@ static const pcre_uchar *posix_substitutes[] = {
   NULL,                 /* graph */
   NULL,                 /* print */
   NULL,                 /* punct */
-  string_pXps,          /* space */    /* NOTE: Xps is POSIX space */
-  string_pXwd,          /* word */
+  string_pXps,          /* space */   /* Xps is POSIX space, but from 8.34 */
+  string_pXwd,          /* word  */   /* Perl and POSIX space are the same */
   NULL,                 /* xdigit */
   /* Negated cases */
   string_PL,            /* ^alpha */
@@ -391,8 +428,8 @@ static const pcre_uchar *posix_substitutes[] = {
   NULL,                 /* ^graph */
   NULL,                 /* ^print */
   NULL,                 /* ^punct */
-  string_PXps,          /* ^space */   /* NOTE: Xps is POSIX space */
-  string_PXwd,          /* ^word */
+  string_PXps,          /* ^space */  /* Xps is POSIX space, but from 8.34 */
+  string_PXwd,          /* ^word */   /* Perl and POSIX space are the same */
   NULL                  /* ^xdigit */
 };
 #define POSIX_SUBSIZE (sizeof(posix_substitutes) / sizeof(pcre_uchar *))
@@ -428,7 +465,7 @@ static const char error_texts[] =
   "range out of order in character class\0"
   "nothing to repeat\0"
   /* 10 */
-  "operand of unlimited repeat could match the empty string\0"  /** DEAD **/
+  "internal error: invalid forward reference offset\0"
   "internal error: unexpected repeat\0"
   "unrecognized character after (? or (?-\0"
   "POSIX named classes are supported only within a class\0"
@@ -449,14 +486,14 @@ static const char error_texts[] =
   "lookbehind assertion is not fixed length\0"
   "malformed number or name after (?(\0"
   "conditional group contains more than two branches\0"
-  "assertion expected after (?(\0"
+  "assertion expected after (?( or (?(?C)\0"
   "(?R or (?[+-]digits must be followed by )\0"
   /* 30 */
   "unknown POSIX class name\0"
   "POSIX collating elements are not supported\0"
   "this version of PCRE is compiled without UTF support\0"
   "spare error\0"  /** DEAD **/
-  "character value in \\x{...} sequence is too large\0"
+  "character value in \\x{} or \\o{} is too large\0"
   /* 35 */
   "invalid condition (?(0)\0"
   "\\C not allowed in lookbehind assertion\0"
@@ -497,7 +534,11 @@ static const char error_texts[] =
   "different names for subpatterns of the same number are not allowed\0"
   "(*MARK) must have an argument\0"
   "this version of PCRE is not compiled with Unicode property support\0"
+#ifndef EBCDIC
   "\\c must be followed by an ASCII character\0"
+#else
+  "\\c must be followed by a letter or one of [\\]^_?\0"
+#endif
   "\\k is not followed by a braced, angle-bracketed, or quoted name\0"
   /* 70 */
   "internal error: unknown opcode in find_fixedlength()\0"
@@ -510,6 +551,17 @@ static const char error_texts[] =
   "character value in \\u.... sequence is too large\0"
   "invalid UTF-32 string\0"
   "setting UTF is disabled by the application\0"
+  "non-hex character in \\x{} (closing brace missing?)\0"
+  /* 80 */
+  "non-octal character in \\o{} (closing brace missing?)\0"
+  "missing opening brace after \\o\0"
+  "parentheses are too deeply nested\0"
+  "invalid range in character class\0"
+  "group name must start with a non-digit\0"
+  /* 85 */
+  "parentheses are too deeply nested (stack check)\0"
+  "digits missing in \\x{} or \\o{}\0"
+  "regular expression is too complicated\0"
   ;
 
 /* Table to identify digits and hex digits. This is used when compiling
@@ -649,6 +701,183 @@ static const pcre_uint8 ebcdic_chartab[] = { /* chartable partial dup */
 #endif
 
 
+/* This table is used to check whether auto-possessification is possible
+between adjacent character-type opcodes. The left-hand (repeated) opcode is
+used to select the row, and the right-hand opcode is use to select the column.
+A value of 1 means that auto-possessification is OK. For example, the second
+value in the first row means that \D+\d can be turned into \D++\d.
+
+The Unicode property types (\P and \p) have to be present to fill out the table
+because of what their opcode values are, but the table values should always be
+zero because property types are handled separately in the code. The last four
+columns apply to items that cannot be repeated, so there is no need to have
+rows for them. Note that OP_DIGIT etc. are generated only when PCRE_UCP is
+*not* set. When it is set, \d etc. are converted into OP_(NOT_)PROP codes. */
+
+#define APTROWS (LAST_AUTOTAB_LEFT_OP - FIRST_AUTOTAB_OP + 1)
+#define APTCOLS (LAST_AUTOTAB_RIGHT_OP - FIRST_AUTOTAB_OP + 1)
+
+static const pcre_uint8 autoposstab[APTROWS][APTCOLS] = {
+/* \D \d \S \s \W \w  . .+ \C \P \p \R \H \h \V \v \X \Z \z  $ $M */
+  { 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 },  /* \D */
+  { 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1 },  /* \d */
+  { 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1 },  /* \S */
+  { 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 },  /* \s */
+  { 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 },  /* \W */
+  { 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1 },  /* \w */
+  { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0 },  /* .  */
+  { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 },  /* .+ */
+  { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 },  /* \C */
+  { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },  /* \P */
+  { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },  /* \p */
+  { 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0 },  /* \R */
+  { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0 },  /* \H */
+  { 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0 },  /* \h */
+  { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0 },  /* \V */
+  { 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0 },  /* \v */
+  { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 }   /* \X */
+};
+
+
+/* This table is used to check whether auto-possessification is possible
+between adjacent Unicode property opcodes (OP_PROP and OP_NOTPROP). The
+left-hand (repeated) opcode is used to select the row, and the right-hand
+opcode is used to select the column. The values are as follows:
+
+  0   Always return FALSE (never auto-possessify)
+  1   Character groups are distinct (possessify if both are OP_PROP)
+  2   Check character categories in the same group (general or particular)
+  3   TRUE if the two opcodes are not the same (PROP vs NOTPROP)
+
+  4   Check left general category vs right particular category
+  5   Check right general category vs left particular category
+
+  6   Left alphanum vs right general category
+  7   Left space vs right general category
+  8   Left word vs right general category
+
+  9   Right alphanum vs left general category
+ 10   Right space vs left general category
+ 11   Right word vs left general category
+
+ 12   Left alphanum vs right particular category
+ 13   Left space vs right particular category
+ 14   Left word vs right particular category
+
+ 15   Right alphanum vs left particular category
+ 16   Right space vs left particular category
+ 17   Right word vs left particular category
+*/
+
+static const pcre_uint8 propposstab[PT_TABSIZE][PT_TABSIZE] = {
+/* ANY LAMP GC  PC  SC ALNUM SPACE PXSPACE WORD CLIST UCNC */
+  { 0,  0,  0,  0,  0,    0,    0,      0,   0,    0,   0 },  /* PT_ANY */
+  { 0,  3,  0,  0,  0,    3,    1,      1,   0,    0,   0 },  /* PT_LAMP */
+  { 0,  0,  2,  4,  0,    9,   10,     10,  11,    0,   0 },  /* PT_GC */
+  { 0,  0,  5,  2,  0,   15,   16,     16,  17,    0,   0 },  /* PT_PC */
+  { 0,  0,  0,  0,  2,    0,    0,      0,   0,    0,   0 },  /* PT_SC */
+  { 0,  3,  6, 12,  0,    3,    1,      1,   0,    0,   0 },  /* PT_ALNUM */
+  { 0,  1,  7, 13,  0,    1,    3,      3,   1,    0,   0 },  /* PT_SPACE */
+  { 0,  1,  7, 13,  0,    1,    3,      3,   1,    0,   0 },  /* PT_PXSPACE */
+  { 0,  0,  8, 14,  0,    0,    1,      1,   3,    0,   0 },  /* PT_WORD */
+  { 0,  0,  0,  0,  0,    0,    0,      0,   0,    0,   0 },  /* PT_CLIST */
+  { 0,  0,  0,  0,  0,    0,    0,      0,   0,    0,   3 }   /* PT_UCNC */
+};
+
+/* This table is used to check whether auto-possessification is possible
+between adjacent Unicode property opcodes (OP_PROP and OP_NOTPROP) when one
+specifies a general category and the other specifies a particular category. The
+row is selected by the general category and the column by the particular
+category. The value is 1 if the particular category is not part of the general
+category. */
+
+static const pcre_uint8 catposstab[7][30] = {
+/* Cc Cf Cn Co Cs Ll Lm Lo Lt Lu Mc Me Mn Nd Nl No Pc Pd Pe Pf Pi Po Ps Sc Sk Sm So Zl Zp Zs */
+  { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 },  /* C */
+  { 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 },  /* L */
+  { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 },  /* M */
+  { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 },  /* N */
+  { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1 },  /* P */
+  { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1 },  /* S */
+  { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0 }   /* Z */
+};
+
+/* This table is used when checking ALNUM, (PX)SPACE, SPACE, and WORD against
+a general or particular category. The properties in each row are those
+that apply to the character set in question. Duplication means that a little
+unnecessary work is done when checking, but this keeps things much simpler
+because they can all use the same code. For more details see the comment where
+this table is used.
+
+Note: SPACE and PXSPACE used to be different because Perl excluded VT from
+"space", but from Perl 5.18 it's included, so both categories are treated the
+same here. */
+
+static const pcre_uint8 posspropstab[3][4] = {
+  { ucp_L, ucp_N, ucp_N, ucp_Nl },  /* ALNUM, 3rd and 4th values redundant */
+  { ucp_Z, ucp_Z, ucp_C, ucp_Cc },  /* SPACE and PXSPACE, 2nd value redundant */
+  { ucp_L, ucp_N, ucp_P, ucp_Po }   /* WORD */
+};
+
+/* This table is used when converting repeating opcodes into possessified
+versions as a result of an explicit possessive quantifier such as ++. A zero
+value means there is no possessified version - in those cases the item in
+question must be wrapped in ONCE brackets. The table is truncated at OP_CALLOUT
+because all relevant opcodes are less than that. */
+
+static const pcre_uint8 opcode_possessify[] = {
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   /* 0 - 15  */
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   /* 16 - 31 */
+
+  0,                       /* NOTI */
+  OP_POSSTAR, 0,           /* STAR, MINSTAR */
+  OP_POSPLUS, 0,           /* PLUS, MINPLUS */
+  OP_POSQUERY, 0,          /* QUERY, MINQUERY */
+  OP_POSUPTO, 0,           /* UPTO, MINUPTO */
+  0,                       /* EXACT */
+  0, 0, 0, 0,              /* POS{STAR,PLUS,QUERY,UPTO} */
+
+  OP_POSSTARI, 0,          /* STARI, MINSTARI */
+  OP_POSPLUSI, 0,          /* PLUSI, MINPLUSI */
+  OP_POSQUERYI, 0,         /* QUERYI, MINQUERYI */
+  OP_POSUPTOI, 0,          /* UPTOI, MINUPTOI */
+  0,                       /* EXACTI */
+  0, 0, 0, 0,              /* POS{STARI,PLUSI,QUERYI,UPTOI} */
+
+  OP_NOTPOSSTAR, 0,        /* NOTSTAR, NOTMINSTAR */
+  OP_NOTPOSPLUS, 0,        /* NOTPLUS, NOTMINPLUS */
+  OP_NOTPOSQUERY, 0,       /* NOTQUERY, NOTMINQUERY */
+  OP_NOTPOSUPTO, 0,        /* NOTUPTO, NOTMINUPTO */
+  0,                       /* NOTEXACT */
+  0, 0, 0, 0,              /* NOTPOS{STAR,PLUS,QUERY,UPTO} */
+
+  OP_NOTPOSSTARI, 0,       /* NOTSTARI, NOTMINSTARI */
+  OP_NOTPOSPLUSI, 0,       /* NOTPLUSI, NOTMINPLUSI */
+  OP_NOTPOSQUERYI, 0,      /* NOTQUERYI, NOTMINQUERYI */
+  OP_NOTPOSUPTOI, 0,       /* NOTUPTOI, NOTMINUPTOI */
+  0,                       /* NOTEXACTI */
+  0, 0, 0, 0,              /* NOTPOS{STARI,PLUSI,QUERYI,UPTOI} */
+
+  OP_TYPEPOSSTAR, 0,       /* TYPESTAR, TYPEMINSTAR */
+  OP_TYPEPOSPLUS, 0,       /* TYPEPLUS, TYPEMINPLUS */
+  OP_TYPEPOSQUERY, 0,      /* TYPEQUERY, TYPEMINQUERY */
+  OP_TYPEPOSUPTO, 0,       /* TYPEUPTO, TYPEMINUPTO */
+  0,                       /* TYPEEXACT */
+  0, 0, 0, 0,              /* TYPEPOS{STAR,PLUS,QUERY,UPTO} */
+
+  OP_CRPOSSTAR, 0,         /* CRSTAR, CRMINSTAR */
+  OP_CRPOSPLUS, 0,         /* CRPLUS, CRMINPLUS */
+  OP_CRPOSQUERY, 0,        /* CRQUERY, CRMINQUERY */
+  OP_CRPOSRANGE, 0,        /* CRRANGE, CRMINRANGE */
+  0, 0, 0, 0,              /* CRPOS{STAR,PLUS,QUERY,RANGE} */
+
+  0, 0, 0,                 /* CLASS, NCLASS, XCLASS */
+  0, 0,                    /* REF, REFI */
+  0, 0,                    /* DNREF, DNREFI */
+  0, 0                     /* RECURSE, CALLOUT */
+};
+
+
 
 /*************************************************
 *            Find an error text                  *
@@ -676,6 +905,7 @@ return s;
 }
 
 
+
 /*************************************************
 *           Expand the workspace                 *
 *************************************************/
@@ -753,16 +983,15 @@ return (*p == CHAR_RIGHT_CURLY_BRACKET);
 *************************************************/
 
 /* This function is called when a \ has been encountered. It either returns a
-positive value for a simple escape such as \n, or 0 for a data character
-which will be placed in chptr. A backreference to group n is returned as
-negative n. When UTF-8 is enabled, a positive value greater than 255 may
-be returned in chptr.
-On entry,ptr is pointing at the \. On exit, it is on the final character of the
-escape sequence.
+positive value for a simple escape such as \n, or 0 for a data character which
+will be placed in chptr. A backreference to group n is returned as negative n.
+When UTF-8 is enabled, a positive value greater than 255 may be returned in
+chptr. On entry, ptr is pointing at the \. On exit, it is on the final
+character of the escape sequence.
 
 Arguments:
   ptrptr         points to the pattern position pointer
-  chptr          points to the data character
+  chptr          points to a returned data character
   errorcodeptr   points to the errorcode variable
   bracount       number of previous extracting brackets
   options        the options bits
@@ -966,16 +1195,20 @@ else
     break;
 
     /* The handling of escape sequences consisting of a string of digits
-    starting with one that is not zero is not straightforward. By experiment,
-    the way Perl works seems to be as follows:
+    starting with one that is not zero is not straightforward. Perl has changed
+    over the years. Nowadays \g{} for backreferences and \o{} for octal are
+    recommended to avoid the ambiguities in the old syntax.
 
     Outside a character class, the digits are read as a decimal number. If the
-    number is less than 10, or if there are that many previous extracting
-    left brackets, then it is a back reference. Otherwise, up to three octal
-    digits are read to form an escaped byte. Thus \123 is likely to be octal
-    123 (cf \0123, which is octal 012 followed by the literal 3). If the octal
-    value is greater than 377, the least significant 8 bits are taken. Inside a
-    character class, \ followed by a digit is always an octal number. */
+    number is less than 8 (used to be 10), or if there are that many previous
+    extracting left brackets, then it is a back reference. Otherwise, up to
+    three octal digits are read to form an escaped byte. Thus \123 is likely to
+    be octal 123 (cf \0123, which is octal 012 followed by the literal 3). If
+    the octal value is greater than 377, the least significant 8 bits are
+    taken. \8 and \9 are treated as the literal characters 8 and 9.
+
+    Inside a character class, \ followed by a digit is always either a literal
+    8 or 9 or an octal number. */
 
     case CHAR_1: case CHAR_2: case CHAR_3: case CHAR_4: case CHAR_5:
     case CHAR_6: case CHAR_7: case CHAR_8: case CHAR_9:
@@ -1002,7 +1235,7 @@ else
         *errorcodeptr = ERR61;
         break;
         }
-      if (s < 10 || s <= bracount)
+      if (s < 8 || s <= bracount)  /* Check for back reference */
         {
         escape = -s;
         break;
@@ -1010,16 +1243,14 @@ else
       ptr = oldptr;      /* Put the pointer back and fall through */
       }
 
-    /* Handle an octal number following \. If the first digit is 8 or 9, Perl
-    generates a binary zero byte and treats the digit as a following literal.
-    Thus we have to pull back the pointer by one. */
+    /* Handle a digit following \ when the number is not a back reference. If
+    the first digit is 8 or 9, Perl used to generate a binary zero byte and
+    then treat the digit as a following literal. At least by Perl 5.18 this
+    changed so as not to insert the binary zero. */
 
-    if ((c = *ptr) >= CHAR_8)
-      {
-      ptr--;
-      c = 0;
-      break;
-      }
+    if ((c = *ptr) >= CHAR_8) break;
+
+    /* Fall through with a digit less than 8 */
 
     /* \0 always starts an octal number, but we may drop through to here with a
     larger first octal digit. The original code used just to take the least
@@ -1036,15 +1267,51 @@ else
 #endif
     break;
 
-    /* \x is complicated. \x{ddd} is a character number which can be greater
-    than 0xff in utf or non-8bit mode, but only if the ddd are hex digits.
-    If not, { is treated as a data character. */
+    /* \o is a relatively new Perl feature, supporting a more general way of
+    specifying character codes in octal. The only supported form is \o{ddd}. */
+
+    case CHAR_o:
+    if (ptr[1] != CHAR_LEFT_CURLY_BRACKET) *errorcodeptr = ERR81; else
+    if (ptr[2] == CHAR_RIGHT_CURLY_BRACKET) *errorcodeptr = ERR86; else
+      {
+      ptr += 2;
+      c = 0;
+      overflow = FALSE;
+      while (*ptr >= CHAR_0 && *ptr <= CHAR_7)
+        {
+        register pcre_uint32 cc = *ptr++;
+        if (c == 0 && cc == CHAR_0) continue;     /* Leading zeroes */
+#ifdef COMPILE_PCRE32
+        if (c >= 0x20000000l) { overflow = TRUE; break; }
+#endif
+        c = (c << 3) + cc - CHAR_0 ;
+#if defined COMPILE_PCRE8
+        if (c > (utf ? 0x10ffffU : 0xffU)) { overflow = TRUE; break; }
+#elif defined COMPILE_PCRE16
+        if (c > (utf ? 0x10ffffU : 0xffffU)) { overflow = TRUE; break; }
+#elif defined COMPILE_PCRE32
+        if (utf && c > 0x10ffffU) { overflow = TRUE; break; }
+#endif
+        }
+      if (overflow)
+        {
+        while (*ptr >= CHAR_0 && *ptr <= CHAR_7) ptr++;
+        *errorcodeptr = ERR34;
+        }
+      else if (*ptr == CHAR_RIGHT_CURLY_BRACKET)
+        {
+        if (utf && c >= 0xd800 && c <= 0xdfff) *errorcodeptr = ERR73;
+        }
+      else *errorcodeptr = ERR80;
+      }
+    break;
+
+    /* \x is complicated. In JavaScript, \x must be followed by two hexadecimal
+    numbers. Otherwise it is a lowercase x letter. */
 
     case CHAR_x:
     if ((options & PCRE_JAVASCRIPT_COMPAT) != 0)
       {
-      /* In JavaScript, \x must be followed by two hexadecimal numbers.
-      Otherwise it is a lowercase x letter. */
       if (MAX_255(ptr[1]) && (digitab[ptr[1]] & ctype_xdigit) != 0
         && MAX_255(ptr[2]) && (digitab[ptr[2]] & ctype_xdigit) != 0)
         {
@@ -1061,73 +1328,91 @@ else
 #endif
           }
         }
-      break;
-      }
+      }    /* End JavaScript handling */
 
-    if (ptr[1] == CHAR_LEFT_CURLY_BRACKET)
-      {
-      const pcre_uchar *pt = ptr + 2;
+    /* Handle \x in Perl's style. \x{ddd} is a character number which can be
+    greater than 0xff in utf or non-8bit mode, but only if the ddd are hex
+    digits. If not, { used to be treated as a data character. However, Perl
+    seems to read hex digits up to the first non-such, and ignore the rest, so
+    that, for example \x{zz} matches a binary zero. This seems crazy, so PCRE
+    now gives an error. */
 
-      c = 0;
-      overflow = FALSE;
-      while (MAX_255(*pt) && (digitab[*pt] & ctype_xdigit) != 0)
+    else
+      {
+      if (ptr[1] == CHAR_LEFT_CURLY_BRACKET)
         {
-        register pcre_uint32 cc = *pt++;
-        if (c == 0 && cc == CHAR_0) continue;     /* Leading zeroes */
+        ptr += 2;
+        if (*ptr == CHAR_RIGHT_CURLY_BRACKET)
+          {
+          *errorcodeptr = ERR86;
+          break;
+          }
+        c = 0;
+        overflow = FALSE;
+        while (MAX_255(*ptr) && (digitab[*ptr] & ctype_xdigit) != 0)
+          {
+          register pcre_uint32 cc = *ptr++;
+          if (c == 0 && cc == CHAR_0) continue;     /* Leading zeroes */
 
 #ifdef COMPILE_PCRE32
-        if (c >= 0x10000000l) { overflow = TRUE; break; }
+          if (c >= 0x10000000l) { overflow = TRUE; break; }
 #endif
 
 #ifndef EBCDIC  /* ASCII/UTF-8 coding */
-        if (cc >= CHAR_a) cc -= 32;               /* Convert to upper case */
-        c = (c << 4) + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10));
+          if (cc >= CHAR_a) cc -= 32;               /* Convert to upper case */
+          c = (c << 4) + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10));
 #else           /* EBCDIC coding */
-        if (cc >= CHAR_a && cc <= CHAR_z) cc += 64;  /* Convert to upper case */
-        c = (c << 4) + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10));
+          if (cc >= CHAR_a && cc <= CHAR_z) cc += 64;  /* Convert to upper case */
+          c = (c << 4) + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10));
 #endif
 
 #if defined COMPILE_PCRE8
-        if (c > (utf ? 0x10ffffU : 0xffU)) { overflow = TRUE; break; }
+          if (c > (utf ? 0x10ffffU : 0xffU)) { overflow = TRUE; break; }
 #elif defined COMPILE_PCRE16
-        if (c > (utf ? 0x10ffffU : 0xffffU)) { overflow = TRUE; break; }
+          if (c > (utf ? 0x10ffffU : 0xffffU)) { overflow = TRUE; break; }
 #elif defined COMPILE_PCRE32
-        if (utf && c > 0x10ffffU) { overflow = TRUE; break; }
+          if (utf && c > 0x10ffffU) { overflow = TRUE; break; }
 #endif
-        }
+          }
 
-      if (overflow)
-        {
-        while (MAX_255(*pt) && (digitab[*pt] & ctype_xdigit) != 0) pt++;
-        *errorcodeptr = ERR34;
-        }
+        if (overflow)
+          {
+          while (MAX_255(*ptr) && (digitab[*ptr] & ctype_xdigit) != 0) ptr++;
+          *errorcodeptr = ERR34;
+          }
 
-      if (*pt == CHAR_RIGHT_CURLY_BRACKET)
-        {
-        if (utf && c >= 0xd800 && c <= 0xdfff) *errorcodeptr = ERR73;
-        ptr = pt;
-        break;
-        }
+        else if (*ptr == CHAR_RIGHT_CURLY_BRACKET)
+          {
+          if (utf && c >= 0xd800 && c <= 0xdfff) *errorcodeptr = ERR73;
+          }
 
-      /* If the sequence of hex digits does not end with '}', then we don't
-      recognize this construct; fall through to the normal \x handling. */
-      }
+        /* If the sequence of hex digits does not end with '}', give an error.
+        We used just to recognize this construct and fall through to the normal
+        \x handling, but nowadays Perl gives an error, which seems much more
+        sensible, so we do too. */
 
-    /* Read just a single-byte hex-defined char */
+        else *errorcodeptr = ERR79;
+        }   /* End of \x{} processing */
 
-    c = 0;
-    while (i++ < 2 && MAX_255(ptr[1]) && (digitab[ptr[1]] & ctype_xdigit) != 0)
-      {
-      pcre_uint32 cc;                          /* Some compilers don't like */
-      cc = *(++ptr);                           /* ++ in initializers */
+      /* Read a single-byte hex-defined char (up to two hex digits after \x) */
+
+      else
+        {
+        c = 0;
+        while (i++ < 2 && MAX_255(ptr[1]) && (digitab[ptr[1]] & ctype_xdigit) != 0)
+          {
+          pcre_uint32 cc;                          /* Some compilers don't like */
+          cc = *(++ptr);                           /* ++ in initializers */
 #ifndef EBCDIC  /* ASCII/UTF-8 coding */
-      if (cc >= CHAR_a) cc -= 32;              /* Convert to upper case */
-      c = c * 16 + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10));
+          if (cc >= CHAR_a) cc -= 32;              /* Convert to upper case */
+          c = c * 16 + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10));
 #else           /* EBCDIC coding */
-      if (cc <= CHAR_z) cc += 64;              /* Convert to upper case */
-      c = c * 16 + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10));
+          if (cc <= CHAR_z) cc += 64;              /* Convert to upper case */
+          c = c * 16 + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10));
 #endif
-      }
+          }
+        }     /* End of \xdd handling */
+      }       /* End of Perl-style \x handling */
     break;
 
     /* For \c, a following letter is upper-cased; then the 0x40 bit is flipped.
@@ -1152,7 +1437,16 @@ else
     c ^= 0x40;
 #else             /* EBCDIC coding */
     if (c >= CHAR_a && c <= CHAR_z) c += 64;
-    c ^= 0xC0;
+    if (c == CHAR_QUESTION_MARK)
+      c = ('\\' == 188 && '`' == 74)? 0x5f : 0xff;
+    else
+      {
+      for (i = 0; i < 32; i++)
+        {
+        if (c == ebcdic_escape_c[i]) break;
+        }
+      if (i < 32) c = i; else *errorcodeptr = ERR68;
+      }
 #endif
     break;
 
@@ -1193,6 +1487,8 @@ if ((options & PCRE_UCP) != 0 && escape >= ESC_D && escape <= ESC_w)
 return escape;
 }
 
+
+
 #ifdef SUPPORT_UCP
 /*************************************************
 *               Handle \P and \p                 *
@@ -1290,7 +1586,6 @@ return FALSE;
 
 
 
-
 /*************************************************
 *         Read repeat counts                     *
 *************************************************/
@@ -1316,29 +1611,29 @@ read_repeat_counts(const pcre_uchar *p, int *minp, int *maxp, int *errorcodeptr)
 int min = 0;
 int max = -1;
 
-/* Read the minimum value and do a paranoid check: a negative value indicates
-an integer overflow. */
-
-while (IS_DIGIT(*p)) min = min * 10 + (int)(*p++ - CHAR_0);
-if (min < 0 || min > 65535)
+while (IS_DIGIT(*p))
   {
-  *errorcodeptr = ERR5;
-  return p;
+  min = min * 10 + (int)(*p++ - CHAR_0);
+  if (min > 65535)
+    {
+    *errorcodeptr = ERR5;
+    return p;
+    }
   }
 
-/* Read the maximum value if there is one, and again do a paranoid on its size.
-Also, max must not be less than min. */
-
 if (*p == CHAR_RIGHT_CURLY_BRACKET) max = min; else
   {
   if (*(++p) != CHAR_RIGHT_CURLY_BRACKET)
     {
     max = 0;
-    while(IS_DIGIT(*p)) max = max * 10 + (int)(*p++ - CHAR_0);
-    if (max < 0 || max > 65535)
+    while(IS_DIGIT(*p))
       {
-      *errorcodeptr = ERR5;
-      return p;
+      max = max * 10 + (int)(*p++ - CHAR_0);
+      if (max > 65535)
+        {
+        *errorcodeptr = ERR5;
+        return p;
+        }
       }
     if (max < min)
       {
@@ -1348,9 +1643,6 @@ if (*p == CHAR_RIGHT_CURLY_BRACKET) max = min; else
     }
   }
 
-/* Fill in the required variables, and pass back the pointer to the terminating
-'}'. */
-
 *minp = min;
 *maxp = max;
 return p;
@@ -1359,424 +1651,126 @@ return p;
 
 
 /*************************************************
-*  Subroutine for finding forward reference      *
+*      Find first significant op code            *
 *************************************************/
 
-/* This recursive function is called only from find_parens() below. The
-top-level call starts at the beginning of the pattern. All other calls must
-start at a parenthesis. It scans along a pattern's text looking for capturing
-subpatterns, and counting them. If it finds a named pattern that matches the
-name it is given, it returns its number. Alternatively, if the name is NULL, it
-returns when it reaches a given numbered subpattern. Recursion is used to keep
-track of subpatterns that reset the capturing group numbers - the (?| feature.
-
-This function was originally called only from the second pass, in which we know
-that if (?< or (?' or (?P< is encountered, the name will be correctly
-terminated because that is checked in the first pass. There is now one call to
-this function in the first pass, to check for a recursive back reference by
-name (so that we can make the whole group atomic). In this case, we need check
-only up to the current position in the pattern, and that is still OK because
-and previous occurrences will have been checked. To make this work, the test
-for "end of pattern" is a check against cd->end_pattern in the main loop,
-instead of looking for a binary zero. This means that the special first-pass
-call can adjust cd->end_pattern temporarily. (Checks for binary zero while
-processing items within the loop are OK, because afterwards the main loop will
-terminate.)
+/* This is called by several functions that scan a compiled expression looking
+for a fixed first character, or an anchoring op code etc. It skips over things
+that do not influence this. For some calls, it makes sense to skip negative
+forward and all backward assertions, and also the \b assertion; for others it
+does not.
 
 Arguments:
-  ptrptr       address of the current character pointer (updated)
-  cd           compile background data
-  name         name to seek, or NULL if seeking a numbered subpattern
-  lorn         name length, or subpattern number if name is NULL
-  xmode        TRUE if we are in /x mode
-  utf          TRUE if we are in UTF-8 / UTF-16 / UTF-32 mode
-  count        pointer to the current capturing subpattern number (updated)
-
-Returns:       the number of the named subpattern, or -1 if not found
+  code         pointer to the start of the group
+  skipassert   TRUE if certain assertions are to be skipped
+
+Returns:       pointer to the first significant opcode
 */
 
-static int
-find_parens_sub(pcre_uchar **ptrptr, compile_data *cd, const pcre_uchar *name, int lorn,
-  BOOL xmode, BOOL utf, int *count)
+static const pcre_uchar*
+first_significant_code(const pcre_uchar *code, BOOL skipassert)
 {
-pcre_uchar *ptr = *ptrptr;
-int start_count = *count;
-int hwm_count = start_count;
-BOOL dup_parens = FALSE;
-
-/* If the first character is a parenthesis, check on the type of group we are
-dealing with. The very first call may not start with a parenthesis. */
-
-if (ptr[0] == CHAR_LEFT_PARENTHESIS)
+for (;;)
   {
-  /* Handle specials such as (*SKIP) or (*UTF8) etc. */
-
-  if (ptr[1] == CHAR_ASTERISK)
+  switch ((int)*code)
     {
-    ptr += 2;
-    while (ptr < cd->end_pattern && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++;
-    }
-
-  /* Handle a normal, unnamed capturing parenthesis. */
+    case OP_ASSERT_NOT:
+    case OP_ASSERTBACK:
+    case OP_ASSERTBACK_NOT:
+    if (!skipassert) return code;
+    do code += GET(code, 1); while (*code == OP_ALT);
+    code += PRIV(OP_lengths)[*code];
+    break;
 
-  else if (ptr[1] != CHAR_QUESTION_MARK)
-    {
-    *count += 1;
-    if (name == NULL && *count == lorn) return *count;
-    ptr++;
-    }
+    case OP_WORD_BOUNDARY:
+    case OP_NOT_WORD_BOUNDARY:
+    if (!skipassert) return code;
+    /* Fall through */
 
-  /* All cases now have (? at the start. Remember when we are in a group
-  where the parenthesis numbers are duplicated. */
+    case OP_CALLOUT:
+    case OP_CREF:
+    case OP_DNCREF:
+    case OP_RREF:
+    case OP_DNRREF:
+    case OP_DEF:
+    code += PRIV(OP_lengths)[*code];
+    break;
 
-  else if (ptr[2] == CHAR_VERTICAL_LINE)
-    {
-    ptr += 3;
-    dup_parens = TRUE;
+    default:
+    return code;
     }
+  }
+/* Control never reaches here */
+}
 
-  /* Handle comments; all characters are allowed until a ket is reached. */
 
-  else if (ptr[2] == CHAR_NUMBER_SIGN)
-    {
-    for (ptr += 3; *ptr != CHAR_NULL; ptr++)
-      if (*ptr == CHAR_RIGHT_PARENTHESIS) break;
-    goto FAIL_EXIT;
-    }
 
-  /* Handle a condition. If it is an assertion, just carry on so that it
-  is processed as normal. If not, skip to the closing parenthesis of the
-  condition (there can't be any nested parens). */
+/*************************************************
+*        Find the fixed length of a branch       *
+*************************************************/
 
-  else if (ptr[2] == CHAR_LEFT_PARENTHESIS)
-    {
-    ptr += 2;
-    if (ptr[1] != CHAR_QUESTION_MARK)
-      {
-      while (*ptr != CHAR_NULL && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++;
-      if (*ptr != CHAR_NULL) ptr++;
-      }
-    }
+/* Scan a branch and compute the fixed length of subject that will match it,
+if the length is fixed. This is needed for dealing with backward assertions.
+In UTF8 mode, the result is in characters rather than bytes. The branch is
+temporarily terminated with OP_END when this function is called.
 
-  /* Start with (? but not a condition. */
+This function is called when a backward assertion is encountered, so that if it
+fails, the error message can point to the correct place in the pattern.
+However, we cannot do this when the assertion contains subroutine calls,
+because they can be forward references. We solve this by remembering this case
+and doing the check at the end; a flag specifies which mode we are running in.
 
-  else
-    {
-    ptr += 2;
-    if (*ptr == CHAR_P) ptr++;                      /* Allow optional P */
+Arguments:
+  code     points to the start of the pattern (the bracket)
+  utf      TRUE in UTF-8 / UTF-16 / UTF-32 mode
+  atend    TRUE if called when the pattern is complete
+  cd       the "compile data" structure
+  recurses    chain of recurse_check to catch mutual recursion
 
-    /* We have to disambiguate (?<! and (?<= from (?<name> for named groups */
+Returns:   the fixed length,
+             or -1 if there is no fixed length,
+             or -2 if \C was encountered (in UTF-8 mode only)
+             or -3 if an OP_RECURSE item was encountered and atend is FALSE
+             or -4 if an unknown opcode was encountered (internal error)
+*/
 
-    if ((*ptr == CHAR_LESS_THAN_SIGN && ptr[1] != CHAR_EXCLAMATION_MARK &&
-        ptr[1] != CHAR_EQUALS_SIGN) || *ptr == CHAR_APOSTROPHE)
-      {
-      pcre_uchar term;
-      const pcre_uchar *thisname;
-      *count += 1;
-      if (name == NULL && *count == lorn) return *count;
-      term = *ptr++;
-      if (term == CHAR_LESS_THAN_SIGN) term = CHAR_GREATER_THAN_SIGN;
-      thisname = ptr;
-      while (*ptr != term) ptr++;
-      if (name != NULL && lorn == (int)(ptr - thisname) &&
-          STRNCMP_UC_UC(name, thisname, (unsigned int)lorn) == 0)
-        return *count;
-      term++;
-      }
-    }
-  }
+static int
+find_fixedlength(pcre_uchar *code, BOOL utf, BOOL atend, compile_data *cd,
+  recurse_check *recurses)
+{
+int length = -1;
+recurse_check this_recurse;
+register int branchlength = 0;
+register pcre_uchar *cc = code + 1 + LINK_SIZE;
 
-/* Past any initial parenthesis handling, scan for parentheses or vertical
-bars. Stop if we get to cd->end_pattern. Note that this is important for the
-first-pass call when this value is temporarily adjusted to stop at the current
-position. So DO NOT change this to a test for binary zero. */
+/* Scan along the opcodes for this branch. If we get to the end of the
+branch, check the length against that of the other branches. */
 
-for (; ptr < cd->end_pattern; ptr++)
+for (;;)
   {
-  /* Skip over backslashed characters and also entire \Q...\E */
+  int d;
+  pcre_uchar *ce, *cs;
+  register pcre_uchar op = *cc;
 
-  if (*ptr == CHAR_BACKSLASH)
+  switch (op)
     {
-    if (*(++ptr) == CHAR_NULL) goto FAIL_EXIT;
-    if (*ptr == CHAR_Q) for (;;)
-      {
-      while (*(++ptr) != CHAR_NULL && *ptr != CHAR_BACKSLASH) {};
-      if (*ptr == CHAR_NULL) goto FAIL_EXIT;
-      if (*(++ptr) == CHAR_E) break;
-      }
-    continue;
-    }
-
-  /* Skip over character classes; this logic must be similar to the way they
-  are handled for real. If the first character is '^', skip it. Also, if the
-  first few characters (either before or after ^) are \Q\E or \E we skip them
-  too. This makes for compatibility with Perl. Note the use of STR macros to
-  encode "Q\\E" so that it works in UTF-8 on EBCDIC platforms. */
+    /* We only need to continue for OP_CBRA (normal capturing bracket) and
+    OP_BRA (normal non-capturing bracket) because the other variants of these
+    opcodes are all concerned with unlimited repeated groups, which of course
+    are not of fixed length. */
 
-  if (*ptr == CHAR_LEFT_SQUARE_BRACKET)
-    {
-    BOOL negate_class = FALSE;
-    for (;;)
-      {
-      if (ptr[1] == CHAR_BACKSLASH)
-        {
-        if (ptr[2] == CHAR_E)
-          ptr+= 2;
-        else if (STRNCMP_UC_C8(ptr + 2,
-                 STR_Q STR_BACKSLASH STR_E, 3) == 0)
-          ptr += 4;
-        else
-          break;
-        }
-      else if (!negate_class && ptr[1] == CHAR_CIRCUMFLEX_ACCENT)
-        {
-        negate_class = TRUE;
-        ptr++;
-        }
-      else break;
-      }
-
-    /* If the next character is ']', it is a data character that must be
-    skipped, except in JavaScript compatibility mode. */
-
-    if (ptr[1] == CHAR_RIGHT_SQUARE_BRACKET &&
-        (cd->external_options & PCRE_JAVASCRIPT_COMPAT) == 0)
-      ptr++;
-
-    while (*(++ptr) != CHAR_RIGHT_SQUARE_BRACKET)
-      {
-      if (*ptr == CHAR_NULL) return -1;
-      if (*ptr == CHAR_BACKSLASH)
-        {
-        if (*(++ptr) == CHAR_NULL) goto FAIL_EXIT;
-        if (*ptr == CHAR_Q) for (;;)
-          {
-          while (*(++ptr) != CHAR_NULL && *ptr != CHAR_BACKSLASH) {};
-          if (*ptr == CHAR_NULL) goto FAIL_EXIT;
-          if (*(++ptr) == CHAR_E) break;
-          }
-        continue;
-        }
-      }
-    continue;
-    }
-
-  /* Skip comments in /x mode */
-
-  if (xmode && *ptr == CHAR_NUMBER_SIGN)
-    {
-    ptr++;
-    while (*ptr != CHAR_NULL)
-      {
-      if (IS_NEWLINE(ptr)) { ptr += cd->nllen - 1; break; }
-      ptr++;
-#ifdef SUPPORT_UTF
-      if (utf) FORWARDCHAR(ptr);
-#endif
-      }
-    if (*ptr == CHAR_NULL) goto FAIL_EXIT;
-    continue;
-    }
-
-  /* Check for the special metacharacters */
-
-  if (*ptr == CHAR_LEFT_PARENTHESIS)
-    {
-    int rc = find_parens_sub(&ptr, cd, name, lorn, xmode, utf, count);
-    if (rc > 0) return rc;
-    if (*ptr == CHAR_NULL) goto FAIL_EXIT;
-    }
-
-  else if (*ptr == CHAR_RIGHT_PARENTHESIS)
-    {
-    if (dup_parens && *count < hwm_count) *count = hwm_count;
-    goto FAIL_EXIT;
-    }
-
-  else if (*ptr == CHAR_VERTICAL_LINE && dup_parens)
-    {
-    if (*count > hwm_count) hwm_count = *count;
-    *count = start_count;
-    }
-  }
-
-FAIL_EXIT:
-*ptrptr = ptr;
-return -1;
-}
-
-
-
-
-/*************************************************
-*       Find forward referenced subpattern       *
-*************************************************/
-
-/* This function scans along a pattern's text looking for capturing
-subpatterns, and counting them. If it finds a named pattern that matches the
-name it is given, it returns its number. Alternatively, if the name is NULL, it
-returns when it reaches a given numbered subpattern. This is used for forward
-references to subpatterns. We used to be able to start this scan from the
-current compiling point, using the current count value from cd->bracount, and
-do it all in a single loop, but the addition of the possibility of duplicate
-subpattern numbers means that we have to scan from the very start, in order to
-take account of such duplicates, and to use a recursive function to keep track
-of the different types of group.
-
-Arguments:
-  cd           compile background data
-  name         name to seek, or NULL if seeking a numbered subpattern
-  lorn         name length, or subpattern number if name is NULL
-  xmode        TRUE if we are in /x mode
-  utf          TRUE if we are in UTF-8 / UTF-16 / UTF-32 mode
-
-Returns:       the number of the found subpattern, or -1 if not found
-*/
-
-static int
-find_parens(compile_data *cd, const pcre_uchar *name, int lorn, BOOL xmode,
-  BOOL utf)
-{
-pcre_uchar *ptr = (pcre_uchar *)cd->start_pattern;
-int count = 0;
-int rc;
-
-/* If the pattern does not start with an opening parenthesis, the first call
-to find_parens_sub() will scan right to the end (if necessary). However, if it
-does start with a parenthesis, find_parens_sub() will return when it hits the
-matching closing parens. That is why we have to have a loop. */
-
-for (;;)
-  {
-  rc = find_parens_sub(&ptr, cd, name, lorn, xmode, utf, &count);
-  if (rc > 0 || *ptr++ == CHAR_NULL) break;
-  }
-
-return rc;
-}
-
-
-
-
-/*************************************************
-*      Find first significant op code            *
-*************************************************/
-
-/* This is called by several functions that scan a compiled expression looking
-for a fixed first character, or an anchoring op code etc. It skips over things
-that do not influence this. For some calls, it makes sense to skip negative
-forward and all backward assertions, and also the \b assertion; for others it
-does not.
-
-Arguments:
-  code         pointer to the start of the group
-  skipassert   TRUE if certain assertions are to be skipped
-
-Returns:       pointer to the first significant opcode
-*/
-
-static const pcre_uchar*
-first_significant_code(const pcre_uchar *code, BOOL skipassert)
-{
-for (;;)
-  {
-  switch ((int)*code)
-    {
-    case OP_ASSERT_NOT:
-    case OP_ASSERTBACK:
-    case OP_ASSERTBACK_NOT:
-    if (!skipassert) return code;
-    do code += GET(code, 1); while (*code == OP_ALT);
-    code += PRIV(OP_lengths)[*code];
-    break;
-
-    case OP_WORD_BOUNDARY:
-    case OP_NOT_WORD_BOUNDARY:
-    if (!skipassert) return code;
-    /* Fall through */
-
-    case OP_CALLOUT:
-    case OP_CREF:
-    case OP_NCREF:
-    case OP_RREF:
-    case OP_NRREF:
-    case OP_DEF:
-    code += PRIV(OP_lengths)[*code];
-    break;
-
-    default:
-    return code;
-    }
-  }
-/* Control never reaches here */
-}
-
-
-
-
-/*************************************************
-*        Find the fixed length of a branch       *
-*************************************************/
-
-/* Scan a branch and compute the fixed length of subject that will match it,
-if the length is fixed. This is needed for dealing with backward assertions.
-In UTF8 mode, the result is in characters rather than bytes. The branch is
-temporarily terminated with OP_END when this function is called.
-
-This function is called when a backward assertion is encountered, so that if it
-fails, the error message can point to the correct place in the pattern.
-However, we cannot do this when the assertion contains subroutine calls,
-because they can be forward references. We solve this by remembering this case
-and doing the check at the end; a flag specifies which mode we are running in.
-
-Arguments:
-  code     points to the start of the pattern (the bracket)
-  utf      TRUE in UTF-8 / UTF-16 / UTF-32 mode
-  atend    TRUE if called when the pattern is complete
-  cd       the "compile data" structure
-
-Returns:   the fixed length,
-             or -1 if there is no fixed length,
-             or -2 if \C was encountered (in UTF-8 mode only)
-             or -3 if an OP_RECURSE item was encountered and atend is FALSE
-             or -4 if an unknown opcode was encountered (internal error)
-*/
-
-static int
-find_fixedlength(pcre_uchar *code, BOOL utf, BOOL atend, compile_data *cd)
-{
-int length = -1;
-
-register int branchlength = 0;
-register pcre_uchar *cc = code + 1 + LINK_SIZE;
-
-/* Scan along the opcodes for this branch. If we get to the end of the
-branch, check the length against that of the other branches. */
-
-for (;;)
-  {
-  int d;
-  pcre_uchar *ce, *cs;
-  register pcre_uchar op = *cc;
-
-  switch (op)
-    {
-    /* We only need to continue for OP_CBRA (normal capturing bracket) and
-    OP_BRA (normal non-capturing bracket) because the other variants of these
-    opcodes are all concerned with unlimited repeated groups, which of course
-    are not of fixed length. */
-
-    case OP_CBRA:
-    case OP_BRA:
-    case OP_ONCE:
-    case OP_ONCE_NC:
-    case OP_COND:
-    d = find_fixedlength(cc + ((op == OP_CBRA)? IMM2_SIZE : 0), utf, atend, cd);
-    if (d < 0) return d;
-    branchlength += d;
-    do cc += GET(cc, 1); while (*cc == OP_ALT);
-    cc += 1 + LINK_SIZE;
-    break;
+    case OP_CBRA:
+    case OP_BRA:
+    case OP_ONCE:
+    case OP_ONCE_NC:
+    case OP_COND:
+    d = find_fixedlength(cc + ((op == OP_CBRA)? IMM2_SIZE : 0), utf, atend, cd,
+      recurses);
+    if (d < 0) return d;
+    branchlength += d;
+    do cc += GET(cc, 1); while (*cc == OP_ALT);
+    cc += 1 + LINK_SIZE;
+    break;
 
     /* Reached end of a branch; if it's a ket it is the end of a nested call.
     If it's ALT it is an alternation in a nested call. An ACCEPT is effectively
@@ -1805,7 +1799,15 @@ for (;;)
     cs = ce = (pcre_uchar *)cd->start_code + GET(cc, 1);  /* Start subpattern */
     do ce += GET(ce, 1); while (*ce == OP_ALT);           /* End subpattern */
     if (cc > cs && cc < ce) return -1;                    /* Recursion */
-    d = find_fixedlength(cs + IMM2_SIZE, utf, atend, cd);
+    else   /* Check for mutual recursion */
+      {
+      recurse_check *r = recurses;
+      for (r = recurses; r != NULL; r = r->prev) if (r->group == cs) break;
+      if (r != NULL) return -1;   /* Mutual recursion */
+      }
+    this_recurse.prev = recurses;
+    this_recurse.group = cs;
+    d = find_fixedlength(cs + IMM2_SIZE, utf, atend, cd, &this_recurse);
     if (d < 0) return d;
     branchlength += d;
     cc += 1 + LINK_SIZE;
@@ -1818,7 +1820,7 @@ for (;;)
     case OP_ASSERTBACK:
     case OP_ASSERTBACK_NOT:
     do cc += GET(cc, 1); while (*cc == OP_ALT);
-    cc += PRIV(OP_lengths)[*cc];
+    cc += 1 + LINK_SIZE;
     break;
 
     /* Skip over things that don't match chars */
@@ -1837,13 +1839,13 @@ for (;;)
     case OP_COMMIT:
     case OP_CREF:
     case OP_DEF:
+    case OP_DNCREF:
+    case OP_DNRREF:
     case OP_DOLL:
     case OP_DOLLM:
     case OP_EOD:
     case OP_EODN:
     case OP_FAIL:
-    case OP_NCREF:
-    case OP_NRREF:
     case OP_NOT_WORD_BOUNDARY:
     case OP_PRUNE:
     case OP_REVERSE:
@@ -1938,16 +1940,20 @@ for (;;)
 
     switch (*cc)
       {
-      case OP_CRPLUS:
-      case OP_CRMINPLUS:
       case OP_CRSTAR:
       case OP_CRMINSTAR:
+      case OP_CRPLUS:
+      case OP_CRMINPLUS:
       case OP_CRQUERY:
       case OP_CRMINQUERY:
+      case OP_CRPOSSTAR:
+      case OP_CRPOSPLUS:
+      case OP_CRPOSQUERY:
       return -1;
 
       case OP_CRRANGE:
       case OP_CRMINRANGE:
+      case OP_CRPOSRANGE:
       if (GET2(cc,1) != GET2(cc,1+IMM2_SIZE)) return -1;
       branchlength += (int)GET2(cc,1);
       cc += 1 + 2 * IMM2_SIZE;
@@ -2016,6 +2022,8 @@ for (;;)
     case OP_QUERYI:
     case OP_REF:
     case OP_REFI:
+    case OP_DNREF:
+    case OP_DNREFI:
     case OP_SBRA:
     case OP_SBRAPOS:
     case OP_SCBRA:
@@ -2052,7 +2060,6 @@ for (;;)
 
 
 
-
 /*************************************************
 *    Scan compiled regex for specific bracket    *
 *************************************************/
@@ -2154,32 +2161,60 @@ for (;;)
       {
       case OP_CHAR:
       case OP_CHARI:
+      case OP_NOT:
+      case OP_NOTI:
       case OP_EXACT:
       case OP_EXACTI:
+      case OP_NOTEXACT:
+      case OP_NOTEXACTI:
       case OP_UPTO:
       case OP_UPTOI:
+      case OP_NOTUPTO:
+      case OP_NOTUPTOI:
       case OP_MINUPTO:
       case OP_MINUPTOI:
+      case OP_NOTMINUPTO:
+      case OP_NOTMINUPTOI:
       case OP_POSUPTO:
       case OP_POSUPTOI:
+      case OP_NOTPOSUPTO:
+      case OP_NOTPOSUPTOI:
       case OP_STAR:
       case OP_STARI:
+      case OP_NOTSTAR:
+      case OP_NOTSTARI:
       case OP_MINSTAR:
       case OP_MINSTARI:
+      case OP_NOTMINSTAR:
+      case OP_NOTMINSTARI:
       case OP_POSSTAR:
       case OP_POSSTARI:
+      case OP_NOTPOSSTAR:
+      case OP_NOTPOSSTARI:
       case OP_PLUS:
       case OP_PLUSI:
+      case OP_NOTPLUS:
+      case OP_NOTPLUSI:
       case OP_MINPLUS:
       case OP_MINPLUSI:
+      case OP_NOTMINPLUS:
+      case OP_NOTMINPLUSI:
       case OP_POSPLUS:
       case OP_POSPLUSI:
+      case OP_NOTPOSPLUS:
+      case OP_NOTPOSPLUSI:
       case OP_QUERY:
       case OP_QUERYI:
+      case OP_NOTQUERY:
+      case OP_NOTQUERYI:
       case OP_MINQUERY:
       case OP_MINQUERYI:
+      case OP_NOTMINQUERY:
+      case OP_NOTMINQUERYI:
       case OP_POSQUERY:
       case OP_POSQUERYI:
+      case OP_NOTPOSQUERY:
+      case OP_NOTPOSQUERYI:
       if (HAS_EXTRALEN(code[-1])) code += GET_EXTRALEN(code[-1]);
       break;
       }
@@ -2354,15 +2389,18 @@ Arguments:
   endcode     points to where to stop
   utf         TRUE if in UTF-8 / UTF-16 / UTF-32 mode
   cd          contains pointers to tables etc.
+  recurses    chain of recurse_check to catch mutual recursion
 
 Returns:      TRUE if what is matched could be empty
 */
 
 static BOOL
 could_be_empty_branch(const pcre_uchar *code, const pcre_uchar *endcode,
-  BOOL utf, compile_data *cd)
+  BOOL utf, compile_data *cd, recurse_check *recurses)
 {
 register pcre_uchar c;
+recurse_check this_recurse;
+
 for (code = first_significant_code(code + PRIV(OP_lengths)[*code], TRUE);
      code < endcode;
      code = first_significant_code(code + PRIV(OP_lengths)[c], TRUE))
@@ -2390,25 +2428,47 @@ for (code = first_significant_code(code + PRIV(OP_lengths)[*code], TRUE);
 
   if (c == OP_RECURSE)
     {
-    const pcre_uchar *scode;
+    const pcre_uchar *scode = cd->start_code + GET(code, 1);
+    const pcre_uchar *endgroup = scode;
     BOOL empty_branch;
 
-    /* Test for forward reference */
+    /* Test for forward reference or uncompleted reference. This is disabled
+    when called to scan a completed pattern by setting cd->start_workspace to
+    NULL. */
 
-    for (scode = cd->start_workspace; scode < cd->hwm; scode += LINK_SIZE)
-      if ((int)GET(scode, 0) == (int)(code + 1 - cd->start_code)) return TRUE;
+    if (cd->start_workspace != NULL)
+      {
+      const pcre_uchar *tcode;
+      for (tcode = cd->start_workspace; tcode < cd->hwm; tcode += LINK_SIZE)
+        if ((int)GET(tcode, 0) == (int)(code + 1 - cd->start_code)) return TRUE;
+      if (GET(scode, 1) == 0) return TRUE;    /* Unclosed */
+      }
 
-    /* Not a forward reference, test for completed backward reference */
+    /* If the reference is to a completed group, we need to detect whether this
+    is a recursive call, as otherwise there will be an infinite loop. If it is
+    a recursion, just skip over it. Simple recursions are easily detected. For
+    mutual recursions we keep a chain on the stack. */
 
-    empty_branch = FALSE;
-    scode = cd->start_code + GET(code, 1);
-    if (GET(scode, 1) == 0) return TRUE;    /* Unclosed */
+    do endgroup += GET(endgroup, 1); while (*endgroup == OP_ALT);
+    if (code >= scode && code <= endgroup) continue;  /* Simple recursion */
+    else
+      {
+      recurse_check *r = recurses;
+      for (r = recurses; r != NULL; r = r->prev)
+        if (r->group == scode) break;
+      if (r != NULL) continue;   /* Mutual recursion */
+      }
+
+    /* Completed reference; scan the referenced group, remembering it on the
+    stack chain to detect mutual recursions. */
 
-    /* Completed backwards reference */
+    empty_branch = FALSE;
+    this_recurse.prev = recurses;
+    this_recurse.group = scode;
 
     do
       {
-      if (could_be_empty_branch(scode, endcode, utf, cd))
+      if (could_be_empty_branch(scode, endcode, utf, cd, &this_recurse))
         {
         empty_branch = TRUE;
         break;
@@ -2448,7 +2508,7 @@ for (code = first_significant_code(code + PRIV(OP_lengths)[*code], TRUE);
   if (c == OP_BRA  || c == OP_BRAPOS ||
       c == OP_CBRA || c == OP_CBRAPOS ||
       c == OP_ONCE || c == OP_ONCE_NC ||
-      c == OP_COND)
+      c == OP_COND || c == OP_SCOND)
     {
     BOOL empty_branch;
     if (GET(code, 1) == 0) return TRUE;    /* Hit unclosed bracket */
@@ -2464,8 +2524,8 @@ for (code = first_significant_code(code + PRIV(OP_lengths)[*code], TRUE);
       empty_branch = FALSE;
       do
         {
-        if (!empty_branch && could_be_empty_branch(code, endcode, utf, cd))
-          empty_branch = TRUE;
+        if (!empty_branch && could_be_empty_branch(code, endcode, utf, cd,
+          recurses)) empty_branch = TRUE;
         code += GET(code, 1);
         }
       while (*code == OP_ALT);
@@ -2506,15 +2566,19 @@ for (code = first_significant_code(code + PRIV(OP_lengths)[*code], TRUE);
       case OP_CRMINSTAR:
       case OP_CRQUERY:
       case OP_CRMINQUERY:
+      case OP_CRPOSSTAR:
+      case OP_CRPOSQUERY:
       break;
 
       default:                   /* Non-repeat => class must match */
       case OP_CRPLUS:            /* These repeats aren't empty */
       case OP_CRMINPLUS:
+      case OP_CRPOSPLUS:
       return FALSE;
 
       case OP_CRRANGE:
       case OP_CRMINRANGE:
+      case OP_CRPOSRANGE:
       if (GET2(ccode, 1) > 0) return FALSE;  /* Minimum > 0 */
       break;
       }
@@ -2522,34 +2586,57 @@ for (code = first_significant_code(code + PRIV(OP_lengths)[*code], TRUE);
 
     /* Opcodes that must match a character */
 
+    case OP_ANY:
+    case OP_ALLANY:
+    case OP_ANYBYTE:
+
     case OP_PROP:
     case OP_NOTPROP:
+    case OP_ANYNL:
+
+    case OP_NOT_HSPACE:
+    case OP_HSPACE:
+    case OP_NOT_VSPACE:
+    case OP_VSPACE:
     case OP_EXTUNI:
+
     case OP_NOT_DIGIT:
     case OP_DIGIT:
     case OP_NOT_WHITESPACE:
     case OP_WHITESPACE:
     case OP_NOT_WORDCHAR:
     case OP_WORDCHAR:
-    case OP_ANY:
-    case OP_ALLANY:
-    case OP_ANYBYTE:
+
     case OP_CHAR:
     case OP_CHARI:
     case OP_NOT:
     case OP_NOTI:
+
     case OP_PLUS:
+    case OP_PLUSI:
     case OP_MINPLUS:
-    case OP_POSPLUS:
-    case OP_EXACT:
+    case OP_MINPLUSI:
+
     case OP_NOTPLUS:
+    case OP_NOTPLUSI:
     case OP_NOTMINPLUS:
+    case OP_NOTMINPLUSI:
+
+    case OP_POSPLUS:
+    case OP_POSPLUSI:
     case OP_NOTPOSPLUS:
+    case OP_NOTPOSPLUSI:
+
+    case OP_EXACT:
+    case OP_EXACTI:
     case OP_NOTEXACT:
-    case OP_TYPEPLUS:
+    case OP_NOTEXACTI:
+
+    case OP_TYPEPLUS:
     case OP_TYPEMINPLUS:
     case OP_TYPEPOSPLUS:
     case OP_TYPEEXACT:
+
     return FALSE;
 
     /* These are going to continue, as they may be empty, but we have to
@@ -2583,30 +2670,58 @@ for (code = first_significant_code(code + PRIV(OP_lengths)[*code], TRUE);
     return TRUE;
 
     /* In UTF-8 mode, STAR, MINSTAR, POSSTAR, QUERY, MINQUERY, POSQUERY, UPTO,
-    MINUPTO, and POSUPTO may be followed by a multibyte character */
+    MINUPTO, and POSUPTO and their caseless and negative versions may be
+    followed by a multibyte character. */
 
 #if defined SUPPORT_UTF && !defined COMPILE_PCRE32
     case OP_STAR:
     case OP_STARI:
+    case OP_NOTSTAR:
+    case OP_NOTSTARI:
+
     case OP_MINSTAR:
     case OP_MINSTARI:
+    case OP_NOTMINSTAR:
+    case OP_NOTMINSTARI:
+
     case OP_POSSTAR:
     case OP_POSSTARI:
+    case OP_NOTPOSSTAR:
+    case OP_NOTPOSSTARI:
+
     case OP_QUERY:
     case OP_QUERYI:
+    case OP_NOTQUERY:
+    case OP_NOTQUERYI:
+
     case OP_MINQUERY:
     case OP_MINQUERYI:
+    case OP_NOTMINQUERY:
+    case OP_NOTMINQUERYI:
+
     case OP_POSQUERY:
     case OP_POSQUERYI:
+    case OP_NOTPOSQUERY:
+    case OP_NOTPOSQUERYI:
+
     if (utf && HAS_EXTRALEN(code[1])) code += GET_EXTRALEN(code[1]);
     break;
 
     case OP_UPTO:
     case OP_UPTOI:
+    case OP_NOTUPTO:
+    case OP_NOTUPTOI:
+
     case OP_MINUPTO:
     case OP_MINUPTOI:
+    case OP_NOTMINUPTO:
+    case OP_NOTMINUPTOI:
+
     case OP_POSUPTO:
     case OP_POSUPTOI:
+    case OP_NOTPOSUPTO:
+    case OP_NOTPOSUPTOI:
+
     if (utf && HAS_EXTRALEN(code[1 + IMM2_SIZE])) code += GET_EXTRALEN(code[1 + IMM2_SIZE]);
     break;
 #endif
@@ -2660,7 +2775,7 @@ could_be_empty(const pcre_uchar *code, const pcre_uchar *endcode,
 {
 while (bcptr != NULL && bcptr->current_branch >= code)
   {
-  if (!could_be_empty_branch(bcptr->current_branch, endcode, utf, cd))
+  if (!could_be_empty_branch(bcptr->current_branch, endcode, utf, cd, NULL))
     return FALSE;
   bcptr = bcptr->outer;
   }
@@ -2670,755 +2785,1396 @@ return TRUE;
 
 
 /*************************************************
-*           Check for POSIX class syntax         *
+*        Base opcode of repeated opcodes         *
 *************************************************/
 
-/* This function is called when the sequence "[:" or "[." or "[=" is
-encountered in a character class. It checks whether this is followed by a
-sequence of characters terminated by a matching ":]" or ".]" or "=]". If we
-reach an unescaped ']' without the special preceding character, return FALSE.
+/* Returns the base opcode for repeated single character type opcodes. If the
+opcode is not a repeated character type, it returns with the original value.
 
-Originally, this function only recognized a sequence of letters between the
-terminators, but it seems that Perl recognizes any sequence of characters,
-though of course unknown POSIX names are subsequently rejected. Perl gives an
-"Unknown POSIX class" error for [:f\oo:] for example, where previously PCRE
-didn't consider this to be a POSIX class. Likewise for [:1234:].
+Arguments:  c opcode
+Returns:    base opcode for the type
+*/
 
-The problem in trying to be exactly like Perl is in the handling of escapes. We
-have to be sure that [abc[:x\]pqr] is *not* treated as containing a POSIX
-class, but [abc[:x\]pqr:]] is (so that an error can be generated). The code
-below handles the special case of \], but does not try to do any other escape
-processing. This makes it different from Perl for cases such as [:l\ower:]
-where Perl recognizes it as the POSIX class "lower" but PCRE does not recognize
-"l\ower". This is a lesser evil that not diagnosing bad classes when Perl does,
-I think.
+static pcre_uchar
+get_repeat_base(pcre_uchar c)
+{
+return (c > OP_TYPEPOSUPTO)? c :
+       (c >= OP_TYPESTAR)?   OP_TYPESTAR :
+       (c >= OP_NOTSTARI)?   OP_NOTSTARI :
+       (c >= OP_NOTSTAR)?    OP_NOTSTAR :
+       (c >= OP_STARI)?      OP_STARI :
+                             OP_STAR;
+}
 
-A user pointed out that PCRE was rejecting [:a[:digit:]] whereas Perl was not.
-It seems that the appearance of a nested POSIX class supersedes an apparent
-external class. For example, [:a[:digit:]b:] matches "a", "b", ":", or
-a digit.
 
-In Perl, unescaped square brackets may also appear as part of class names. For
-example, [:a[:abc]b:] gives unknown POSIX class "[:abc]b:]". However, for
-[:a[:abc]b][b:] it gives unknown POSIX class "[:abc]b][b:]", which does not
-seem right at all. PCRE does not allow closing square brackets in POSIX class
-names.
+
+#ifdef SUPPORT_UCP
+/*************************************************
+*        Check a character and a property        *
+*************************************************/
+
+/* This function is called by check_auto_possessive() when a property item
+is adjacent to a fixed character.
 
 Arguments:
-  ptr      pointer to the initial [
-  endptr   where to return the end pointer
+  c            the character
+  ptype        the property type
+  pdata        the data for the type
+  negated      TRUE if it's a negated property (\P or \p{^)
 
-Returns:   TRUE or FALSE
+Returns:       TRUE if auto-possessifying is OK
 */
 
 static BOOL
-check_posix_syntax(const pcre_uchar *ptr, const pcre_uchar **endptr)
+check_char_prop(pcre_uint32 c, unsigned int ptype, unsigned int pdata,
+  BOOL negated)
 {
-pcre_uchar terminator;          /* Don't combine these lines; the Solaris cc */
-terminator = *(++ptr);   /* compiler warns about "non-constant" initializer. */
-for (++ptr; *ptr != CHAR_NULL; ptr++)
+const pcre_uint32 *p;
+const ucd_record *prop = GET_UCD(c);
+
+switch(ptype)
   {
-  if (*ptr == CHAR_BACKSLASH && ptr[1] == CHAR_RIGHT_SQUARE_BRACKET)
-    ptr++;
-  else if (*ptr == CHAR_RIGHT_SQUARE_BRACKET) return FALSE;
-  else
+  case PT_LAMP:
+  return (prop->chartype == ucp_Lu ||
+          prop->chartype == ucp_Ll ||
+          prop->chartype == ucp_Lt) == negated;
+
+  case PT_GC:
+  return (pdata == PRIV(ucp_gentype)[prop->chartype]) == negated;
+
+  case PT_PC:
+  return (pdata == prop->chartype) == negated;
+
+  case PT_SC:
+  return (pdata == prop->script) == negated;
+
+  /* These are specials */
+
+  case PT_ALNUM:
+  return (PRIV(ucp_gentype)[prop->chartype] == ucp_L ||
+          PRIV(ucp_gentype)[prop->chartype] == ucp_N) == negated;
+
+  /* Perl space used to exclude VT, but from Perl 5.18 it is included, which
+  means that Perl space and POSIX space are now identical. PCRE was changed
+  at release 8.34. */
+
+  case PT_SPACE:    /* Perl space */
+  case PT_PXSPACE:  /* POSIX space */
+  switch(c)
     {
-    if (*ptr == terminator && ptr[1] == CHAR_RIGHT_SQUARE_BRACKET)
-      {
-      *endptr = ptr;
-      return TRUE;
-      }
-    if (*ptr == CHAR_LEFT_SQUARE_BRACKET &&
-         (ptr[1] == CHAR_COLON || ptr[1] == CHAR_DOT ||
-          ptr[1] == CHAR_EQUALS_SIGN) &&
-        check_posix_syntax(ptr, endptr))
-      return FALSE;
+    HSPACE_CASES:
+    VSPACE_CASES:
+    return negated;
+
+    default:
+    return (PRIV(ucp_gentype)[prop->chartype] == ucp_Z) == negated;
+    }
+  break;  /* Control never reaches here */
+
+  case PT_WORD:
+  return (PRIV(ucp_gentype)[prop->chartype] == ucp_L ||
+          PRIV(ucp_gentype)[prop->chartype] == ucp_N ||
+          c == CHAR_UNDERSCORE) == negated;
+
+  case PT_CLIST:
+  p = PRIV(ucd_caseless_sets) + prop->caseset;
+  for (;;)
+    {
+    if (c < *p) return !negated;
+    if (c == *p++) return negated;
     }
+  break;  /* Control never reaches here */
   }
+
 return FALSE;
 }
-
+#endif  /* SUPPORT_UCP */
 
 
 
 /*************************************************
-*          Check POSIX class name                *
+*        Fill the character property list        *
 *************************************************/
 
-/* This function is called to check the name given in a POSIX-style class entry
-such as [:alnum:].
+/* Checks whether the code points to an opcode that can take part in auto-
+possessification, and if so, fills a list with its properties.
 
 Arguments:
-  ptr        points to the first letter
-  len        the length of the name
-
-Returns:     a value representing the name, or -1 if unknown
+  code        points to start of expression
+  utf         TRUE if in UTF-8 / UTF-16 / UTF-32 mode
+  fcc         points to case-flipping table
+  list        points to output list
+              list[0] will be filled with the opcode
+              list[1] will be non-zero if this opcode
+                can match an empty character string
+              list[2..7] depends on the opcode
+
+Returns:      points to the start of the next opcode if *code is accepted
+              NULL if *code is not accepted
 */
 
-static int
-check_posix_name(const pcre_uchar *ptr, int len)
+static const pcre_uchar *
+get_chr_property_list(const pcre_uchar *code, BOOL utf,
+  const pcre_uint8 *fcc, pcre_uint32 *list)
 {
-const char *pn = posix_names;
-register int yield = 0;
-while (posix_name_lengths[yield] != 0)
+pcre_uchar c = *code;
+pcre_uchar base;
+const pcre_uchar *end;
+pcre_uint32 chr;
+
+#ifdef SUPPORT_UCP
+pcre_uint32 *clist_dest;
+const pcre_uint32 *clist_src;
+#else
+utf = utf;  /* Suppress "unused parameter" compiler warning */
+#endif
+
+list[0] = c;
+list[1] = FALSE;
+code++;
+
+if (c >= OP_STAR && c <= OP_TYPEPOSUPTO)
   {
-  if (len == posix_name_lengths[yield] &&
-    STRNCMP_UC_C8(ptr, pn, (unsigned int)len) == 0) return yield;
-  pn += posix_name_lengths[yield] + 1;
-  yield++;
-  }
-return -1;
-}
+  base = get_repeat_base(c);
+  c -= (base - OP_STAR);
 
+  if (c == OP_UPTO || c == OP_MINUPTO || c == OP_EXACT || c == OP_POSUPTO)
+    code += IMM2_SIZE;
 
-/*************************************************
-*    Adjust OP_RECURSE items in repeated group   *
-*************************************************/
+  list[1] = (c != OP_PLUS && c != OP_MINPLUS && c != OP_EXACT && c != OP_POSPLUS);
 
-/* OP_RECURSE items contain an offset from the start of the regex to the group
-that is referenced. This means that groups can be replicated for fixed
-repetition simply by copying (because the recursion is allowed to refer to
-earlier groups that are outside the current group). However, when a group is
-optional (i.e. the minimum quantifier is zero), OP_BRAZERO or OP_SKIPZERO is
-inserted before it, after it has been compiled. This means that any OP_RECURSE
-items within it that refer to the group itself or any contained groups have to
-have their offsets adjusted. That one of the jobs of this function. Before it
-is called, the partially compiled regex must be temporarily terminated with
-OP_END.
+  switch(base)
+    {
+    case OP_STAR:
+    list[0] = OP_CHAR;
+    break;
 
-This function has been extended with the possibility of forward references for
-recursions and subroutine calls. It must also check the list of such references
-for the group we are dealing with. If it finds that one of the recursions in
-the current group is on this list, it adjusts the offset in the list, not the
-value in the reference (which is a group number).
+    case OP_STARI:
+    list[0] = OP_CHARI;
+    break;
 
-Arguments:
-  group      points to the start of the group
-  adjust     the amount by which the group is to be moved
-  utf        TRUE in UTF-8 / UTF-16 / UTF-32 mode
-  cd         contains pointers to tables etc.
-  save_hwm   the hwm forward reference pointer at the start of the group
+    case OP_NOTSTAR:
+    list[0] = OP_NOT;
+    break;
 
-Returns:     nothing
-*/
+    case OP_NOTSTARI:
+    list[0] = OP_NOTI;
+    break;
 
-static void
-adjust_recurse(pcre_uchar *group, int adjust, BOOL utf, compile_data *cd,
-  pcre_uchar *save_hwm)
-{
-pcre_uchar *ptr = group;
+    case OP_TYPESTAR:
+    list[0] = *code;
+    code++;
+    break;
+    }
+  c = list[0];
+  }
 
-while ((ptr = (pcre_uchar *)find_recurse(ptr, utf)) != NULL)
+switch(c)
   {
-  int offset;
-  pcre_uchar *hc;
+  case OP_NOT_DIGIT:
+  case OP_DIGIT:
+  case OP_NOT_WHITESPACE:
+  case OP_WHITESPACE:
+  case OP_NOT_WORDCHAR:
+  case OP_WORDCHAR:
+  case OP_ANY:
+  case OP_ALLANY:
+  case OP_ANYNL:
+  case OP_NOT_HSPACE:
+  case OP_HSPACE:
+  case OP_NOT_VSPACE:
+  case OP_VSPACE:
+  case OP_EXTUNI:
+  case OP_EODN:
+  case OP_EOD:
+  case OP_DOLL:
+  case OP_DOLLM:
+  return code;
+
+  case OP_CHAR:
+  case OP_NOT:
+  GETCHARINCTEST(chr, code);
+  list[2] = chr;
+  list[3] = NOTACHAR;
+  return code;
 
-  /* See if this recursion is on the forward reference list. If so, adjust the
-  reference. */
+  case OP_CHARI:
+  case OP_NOTI:
+  list[0] = (c == OP_CHARI) ? OP_CHAR : OP_NOT;
+  GETCHARINCTEST(chr, code);
+  list[2] = chr;
+
+#ifdef SUPPORT_UCP
+  if (chr < 128 || (chr < 256 && !utf))
+    list[3] = fcc[chr];
+  else
+    list[3] = UCD_OTHERCASE(chr);
+#elif defined SUPPORT_UTF || !defined COMPILE_PCRE8
+  list[3] = (chr < 256) ? fcc[chr] : chr;
+#else
+  list[3] = fcc[chr];
+#endif
+
+  /* The othercase might be the same value. */
+
+  if (chr == list[3])
+    list[3] = NOTACHAR;
+  else
+    list[4] = NOTACHAR;
+  return code;
 
-  for (hc = save_hwm; hc < cd->hwm; hc += LINK_SIZE)
+#ifdef SUPPORT_UCP
+  case OP_PROP:
+  case OP_NOTPROP:
+  if (code[0] != PT_CLIST)
     {
-    offset = (int)GET(hc, 0);
-    if (cd->start_code + offset == ptr + 1)
-      {
-      PUT(hc, 0, offset + adjust);
-      break;
-      }
+    list[2] = code[0];
+    list[3] = code[1];
+    return code + 2;
     }
 
-  /* Otherwise, adjust the recursion offset if it's after the start of this
-  group. */
+  /* Convert only if we have enough space. */
 
-  if (hc >= cd->hwm)
+  clist_src = PRIV(ucd_caseless_sets) + code[1];
+  clist_dest = list + 2;
+  code += 2;
+
+  do {
+     if (clist_dest >= list + 8)
+       {
+       /* Early return if there is not enough space. This should never
+       happen, since all clists are shorter than 5 character now. */
+       list[2] = code[0];
+       list[3] = code[1];
+       return code;
+       }
+     *clist_dest++ = *clist_src;
+     }
+  while(*clist_src++ != NOTACHAR);
+
+  /* All characters are stored. The terminating NOTACHAR
+  is copied form the clist itself. */
+
+  list[0] = (c == OP_PROP) ? OP_CHAR : OP_NOT;
+  return code;
+#endif
+
+  case OP_NCLASS:
+  case OP_CLASS:
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+  case OP_XCLASS:
+  if (c == OP_XCLASS)
+    end = code + GET(code, 0) - 1;
+  else
+#endif
+    end = code + 32 / sizeof(pcre_uchar);
+
+  switch(*end)
     {
-    offset = (int)GET(ptr, 1);
-    if (cd->start_code + offset >= group) PUT(ptr, 1, offset + adjust);
-    }
+    case OP_CRSTAR:
+    case OP_CRMINSTAR:
+    case OP_CRQUERY:
+    case OP_CRMINQUERY:
+    case OP_CRPOSSTAR:
+    case OP_CRPOSQUERY:
+    list[1] = TRUE;
+    end++;
+    break;
 
-  ptr += 1 + LINK_SIZE;
+    case OP_CRPLUS:
+    case OP_CRMINPLUS:
+    case OP_CRPOSPLUS:
+    end++;
+    break;
+
+    case OP_CRRANGE:
+    case OP_CRMINRANGE:
+    case OP_CRPOSRANGE:
+    list[1] = (GET2(end, 1) == 0);
+    end += 1 + 2 * IMM2_SIZE;
+    break;
+    }
+  list[2] = (pcre_uint32)(end - code);
+  return end;
   }
+return NULL;    /* Opcode not accepted */
 }
 
 
 
 /*************************************************
-*        Insert an automatic callout point       *
+*    Scan further character sets for match       *
 *************************************************/
 
-/* This function is called when the PCRE_AUTO_CALLOUT option is set, to insert
-callout points before each pattern item.
+/* Checks whether the base and the current opcode have a common character, in
+which case the base cannot be possessified.
 
 Arguments:
-  code           current code pointer
-  ptr            current pattern pointer
-  cd             pointers to tables etc
+  code        points to the byte code
+  utf         TRUE in UTF-8 / UTF-16 / UTF-32 mode
+  cd          static compile data
+  base_list   the data list of the base opcode
 
-Returns:         new code pointer
+Returns:      TRUE if the auto-possessification is possible
 */
 
-static pcre_uchar *
-auto_callout(pcre_uchar *code, const pcre_uchar *ptr, compile_data *cd)
+static BOOL
+compare_opcodes(const pcre_uchar *code, BOOL utf, const compile_data *cd,
+  const pcre_uint32 *base_list, const pcre_uchar *base_end, int *rec_limit)
 {
-*code++ = OP_CALLOUT;
-*code++ = 255;
-PUT(code, 0, (int)(ptr - cd->start_pattern));  /* Pattern offset */
-PUT(code, LINK_SIZE, 0);                       /* Default length */
-return code + 2 * LINK_SIZE;
-}
+pcre_uchar c;
+pcre_uint32 list[8];
+const pcre_uint32 *chr_ptr;
+const pcre_uint32 *ochr_ptr;
+const pcre_uint32 *list_ptr;
+const pcre_uchar *next_code;
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+const pcre_uchar *xclass_flags;
+#endif
+const pcre_uint8 *class_bitset;
+const pcre_uint8 *set1, *set2, *set_end;
+pcre_uint32 chr;
+BOOL accepted, invert_bits;
+BOOL entered_a_group = FALSE;
 
+if (*rec_limit == 0) return FALSE;
+--(*rec_limit);
 
+/* Note: the base_list[1] contains whether the current opcode has greedy
+(represented by a non-zero value) quantifier. This is a different from
+other character type lists, which stores here that the character iterator
+matches to an empty string (also represented by a non-zero value). */
 
-/*************************************************
-*         Complete a callout item                *
-*************************************************/
+for(;;)
+  {
+  /* All operations move the code pointer forward.
+  Therefore infinite recursions are not possible. */
 
-/* A callout item contains the length of the next item in the pattern, which
-we can't fill in till after we have reached the relevant point. This is used
-for both automatic and manual callouts.
+  c = *code;
 
-Arguments:
-  previous_callout   points to previous callout item
-  ptr                current pattern pointer
-  cd                 pointers to tables etc
+  /* Skip over callouts */
 
-Returns:             nothing
-*/
+  if (c == OP_CALLOUT)
+    {
+    code += PRIV(OP_lengths)[c];
+    continue;
+    }
 
-static void
-complete_callout(pcre_uchar *previous_callout, const pcre_uchar *ptr, compile_data *cd)
-{
-int length = (int)(ptr - cd->start_pattern - GET(previous_callout, 2));
-PUT(previous_callout, 2 + LINK_SIZE, length);
-}
+  if (c == OP_ALT)
+    {
+    do code += GET(code, 1); while (*code == OP_ALT);
+    c = *code;
+    }
 
+  switch(c)
+    {
+    case OP_END:
+    case OP_KETRPOS:
+    /* TRUE only in greedy case. The non-greedy case could be replaced by
+    an OP_EXACT, but it is probably not worth it. (And note that OP_EXACT
+    uses more memory, which we cannot get at this stage.) */
 
+    return base_list[1] != 0;
 
-#ifdef SUPPORT_UCP
-/*************************************************
-*           Get othercase range                  *
-*************************************************/
+    case OP_KET:
+    /* If the bracket is capturing, and referenced by an OP_RECURSE, or
+    it is an atomic sub-pattern (assert, once, etc.) the non-greedy case
+    cannot be converted to a possessive form. */
 
-/* This function is passed the start and end of a class range, in UTF-8 mode
-with UCP support. It searches up the characters, looking for ranges of
-characters in the "other" case. Each call returns the next one, updating the
-start address. A character with multiple other cases is returned on its own
-with a special return value.
+    if (base_list[1] == 0) return FALSE;
 
-Arguments:
-  cptr        points to starting character value; updated
-  d           end value
-  ocptr       where to put start of othercase range
-  odptr       where to put end of othercase range
+    switch(*(code - GET(code, 1)))
+      {
+      case OP_ASSERT:
+      case OP_ASSERT_NOT:
+      case OP_ASSERTBACK:
+      case OP_ASSERTBACK_NOT:
+      case OP_ONCE:
+      case OP_ONCE_NC:
+      /* Atomic sub-patterns and assertions can always auto-possessify their
+      last iterator. However, if the group was entered as a result of checking
+      a previous iterator, this is not possible. */
+
+      return !entered_a_group;
+      }
 
-Yield:        -1 when no more
-               0 when a range is returned
-              >0 the CASESET offset for char with multiple other cases
-                in this case, ocptr contains the original
-*/
+    code += PRIV(OP_lengths)[c];
+    continue;
 
-static int
-get_othercase_range(pcre_uint32 *cptr, pcre_uint32 d, pcre_uint32 *ocptr,
-  pcre_uint32 *odptr)
-{
-pcre_uint32 c, othercase, next;
-unsigned int co;
+    case OP_ONCE:
+    case OP_ONCE_NC:
+    case OP_BRA:
+    case OP_CBRA:
+    next_code = code + GET(code, 1);
+    code += PRIV(OP_lengths)[c];
 
-/* Find the first character that has an other case. If it has multiple other
-cases, return its case offset value. */
+    while (*next_code == OP_ALT)
+      {
+      if (!compare_opcodes(code, utf, cd, base_list, base_end, rec_limit))
+        return FALSE;
+      code = next_code + 1 + LINK_SIZE;
+      next_code += GET(next_code, 1);
+      }
 
-for (c = *cptr; c <= d; c++)
-  {
-  if ((co = UCD_CASESET(c)) != 0)
+    entered_a_group = TRUE;
+    continue;
+
+    case OP_BRAZERO:
+    case OP_BRAMINZERO:
+
+    next_code = code + 1;
+    if (*next_code != OP_BRA && *next_code != OP_CBRA
+        && *next_code != OP_ONCE && *next_code != OP_ONCE_NC) return FALSE;
+
+    do next_code += GET(next_code, 1); while (*next_code == OP_ALT);
+
+    /* The bracket content will be checked by the
+    OP_BRA/OP_CBRA case above. */
+    next_code += 1 + LINK_SIZE;
+    if (!compare_opcodes(next_code, utf, cd, base_list, base_end, rec_limit))
+      return FALSE;
+
+    code += PRIV(OP_lengths)[c];
+    continue;
+
+    default:
+    break;
+    }
+
+  /* Check for a supported opcode, and load its properties. */
+
+  code = get_chr_property_list(code, utf, cd->fcc, list);
+  if (code == NULL) return FALSE;    /* Unsupported */
+
+  /* If either opcode is a small character list, set pointers for comparing
+  characters from that list with another list, or with a property. */
+
+  if (base_list[0] == OP_CHAR)
     {
-    *ocptr = c++;   /* Character that has the set */
-    *cptr = c;      /* Rest of input range */
-    return (int)co;
+    chr_ptr = base_list + 2;
+    list_ptr = list;
+    }
+  else if (list[0] == OP_CHAR)
+    {
+    chr_ptr = list + 2;
+    list_ptr = base_list;
     }
-  if ((othercase = UCD_OTHERCASE(c)) != c) break;
-  }
 
-if (c > d) return -1;  /* Reached end of range */
+  /* Character bitsets can also be compared to certain opcodes. */
 
-*ocptr = othercase;
-next = othercase + 1;
+  else if (base_list[0] == OP_CLASS || list[0] == OP_CLASS
+#ifdef COMPILE_PCRE8
+      /* In 8 bit, non-UTF mode, OP_CLASS and OP_NCLASS are the same. */
+      || (!utf && (base_list[0] == OP_NCLASS || list[0] == OP_NCLASS))
+#endif
+      )
+    {
+#ifdef COMPILE_PCRE8
+    if (base_list[0] == OP_CLASS || (!utf && base_list[0] == OP_NCLASS))
+#else
+    if (base_list[0] == OP_CLASS)
+#endif
+      {
+      set1 = (pcre_uint8 *)(base_end - base_list[2]);
+      list_ptr = list;
+      }
+    else
+      {
+      set1 = (pcre_uint8 *)(code - list[2]);
+      list_ptr = base_list;
+      }
 
-for (++c; c <= d; c++)
-  {
-  if (UCD_OTHERCASE(c) != next) break;
-  next++;
-  }
+    invert_bits = FALSE;
+    switch(list_ptr[0])
+      {
+      case OP_CLASS:
+      case OP_NCLASS:
+      set2 = (pcre_uint8 *)
+        ((list_ptr == list ? code : base_end) - list_ptr[2]);
+      break;
 
-*odptr = next - 1;     /* End of othercase range */
-*cptr = c;             /* Rest of input range */
-return 0;
-}
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+      case OP_XCLASS:
+      xclass_flags = (list_ptr == list ? code : base_end) - list_ptr[2] + LINK_SIZE;
+      if ((*xclass_flags & XCL_HASPROP) != 0) return FALSE;
+      if ((*xclass_flags & XCL_MAP) == 0)
+        {
+        /* No bits are set for characters < 256. */
+        if (list[1] == 0) return TRUE;
+        /* Might be an empty repeat. */
+        continue;
+        }
+      set2 = (pcre_uint8 *)(xclass_flags + 1);
+      break;
+#endif
 
+      case OP_NOT_DIGIT:
+      invert_bits = TRUE;
+      /* Fall through */
+      case OP_DIGIT:
+      set2 = (pcre_uint8 *)(cd->cbits + cbit_digit);
+      break;
 
+      case OP_NOT_WHITESPACE:
+      invert_bits = TRUE;
+      /* Fall through */
+      case OP_WHITESPACE:
+      set2 = (pcre_uint8 *)(cd->cbits + cbit_space);
+      break;
 
-/*************************************************
-*        Check a character and a property        *
-*************************************************/
+      case OP_NOT_WORDCHAR:
+      invert_bits = TRUE;
+      /* Fall through */
+      case OP_WORDCHAR:
+      set2 = (pcre_uint8 *)(cd->cbits + cbit_word);
+      break;
 
-/* This function is called by check_auto_possessive() when a property item
-is adjacent to a fixed character.
+      default:
+      return FALSE;
+      }
 
-Arguments:
-  c            the character
-  ptype        the property type
-  pdata        the data for the type
-  negated      TRUE if it's a negated property (\P or \p{^)
+    /* Because the sets are unaligned, we need
+    to perform byte comparison here. */
+    set_end = set1 + 32;
+    if (invert_bits)
+      {
+      do
+        {
+        if ((*set1++ & ~(*set2++)) != 0) return FALSE;
+        }
+      while (set1 < set_end);
+      }
+    else
+      {
+      do
+        {
+        if ((*set1++ & *set2++) != 0) return FALSE;
+        }
+      while (set1 < set_end);
+      }
 
-Returns:       TRUE if auto-possessifying is OK
-*/
+    if (list[1] == 0) return TRUE;
+    /* Might be an empty repeat. */
+    continue;
+    }
+
+  /* Some property combinations also acceptable. Unicode property opcodes are
+  processed specially; the rest can be handled with a lookup table. */
+
+  else
+    {
+    pcre_uint32 leftop, rightop;
+
+    leftop = base_list[0];
+    rightop = list[0];
 
-static BOOL
-check_char_prop(pcre_uint32 c, unsigned int ptype, unsigned int pdata, BOOL negated)
-{
 #ifdef SUPPORT_UCP
-const pcre_uint32 *p;
-#endif
+    accepted = FALSE; /* Always set in non-unicode case. */
+    if (leftop == OP_PROP || leftop == OP_NOTPROP)
+      {
+      if (rightop == OP_EOD)
+        accepted = TRUE;
+      else if (rightop == OP_PROP || rightop == OP_NOTPROP)
+        {
+        int n;
+        const pcre_uint8 *p;
+        BOOL same = leftop == rightop;
+        BOOL lisprop = leftop == OP_PROP;
+        BOOL risprop = rightop == OP_PROP;
+        BOOL bothprop = lisprop && risprop;
+
+        /* There's a table that specifies how each combination is to be
+        processed:
+          0   Always return FALSE (never auto-possessify)
+          1   Character groups are distinct (possessify if both are OP_PROP)
+          2   Check character categories in the same group (general or particular)
+          3   Return TRUE if the two opcodes are not the same
+          ... see comments below
+        */
+
+        n = propposstab[base_list[2]][list[2]];
+        switch(n)
+          {
+          case 0: break;
+          case 1: accepted = bothprop; break;
+          case 2: accepted = (base_list[3] == list[3]) != same; break;
+          case 3: accepted = !same; break;
 
-const ucd_record *prop = GET_UCD(c);
+          case 4:  /* Left general category, right particular category */
+          accepted = risprop && catposstab[base_list[3]][list[3]] == same;
+          break;
 
-switch(ptype)
-  {
-  case PT_LAMP:
-  return (prop->chartype == ucp_Lu ||
-          prop->chartype == ucp_Ll ||
-          prop->chartype == ucp_Lt) == negated;
+          case 5:  /* Right general category, left particular category */
+          accepted = lisprop && catposstab[list[3]][base_list[3]] == same;
+          break;
 
-  case PT_GC:
-  return (pdata == PRIV(ucp_gentype)[prop->chartype]) == negated;
+          /* This code is logically tricky. Think hard before fiddling with it.
+          The posspropstab table has four entries per row. Each row relates to
+          one of PCRE's special properties such as ALNUM or SPACE or WORD.
+          Only WORD actually needs all four entries, but using repeats for the
+          others means they can all use the same code below.
+
+          The first two entries in each row are Unicode general categories, and
+          apply always, because all the characters they include are part of the
+          PCRE character set. The third and fourth entries are a general and a
+          particular category, respectively, that include one or more relevant
+          characters. One or the other is used, depending on whether the check
+          is for a general or a particular category. However, in both cases the
+          category contains more characters than the specials that are defined
+          for the property being tested against. Therefore, it cannot be used
+          in a NOTPROP case.
+
+          Example: the row for WORD contains ucp_L, ucp_N, ucp_P, ucp_Po.
+          Underscore is covered by ucp_P or ucp_Po. */
+
+          case 6:  /* Left alphanum vs right general category */
+          case 7:  /* Left space vs right general category */
+          case 8:  /* Left word vs right general category */
+          p = posspropstab[n-6];
+          accepted = risprop && lisprop ==
+            (list[3] != p[0] &&
+             list[3] != p[1] &&
+            (list[3] != p[2] || !lisprop));
+          break;
 
-  case PT_PC:
-  return (pdata == prop->chartype) == negated;
+          case 9:   /* Right alphanum vs left general category */
+          case 10:  /* Right space vs left general category */
+          case 11:  /* Right word vs left general category */
+          p = posspropstab[n-9];
+          accepted = lisprop && risprop ==
+            (base_list[3] != p[0] &&
+             base_list[3] != p[1] &&
+            (base_list[3] != p[2] || !risprop));
+          break;
 
-  case PT_SC:
-  return (pdata == prop->script) == negated;
+          case 12:  /* Left alphanum vs right particular category */
+          case 13:  /* Left space vs right particular category */
+          case 14:  /* Left word vs right particular category */
+          p = posspropstab[n-12];
+          accepted = risprop && lisprop ==
+            (catposstab[p[0]][list[3]] &&
+             catposstab[p[1]][list[3]] &&
+            (list[3] != p[3] || !lisprop));
+          break;
 
-  /* These are specials */
+          case 15:  /* Right alphanum vs left particular category */
+          case 16:  /* Right space vs left particular category */
+          case 17:  /* Right word vs left particular category */
+          p = posspropstab[n-15];
+          accepted = lisprop && risprop ==
+            (catposstab[p[0]][base_list[3]] &&
+             catposstab[p[1]][base_list[3]] &&
+            (base_list[3] != p[3] || !risprop));
+          break;
+          }
+        }
+      }
 
-  case PT_ALNUM:
-  return (PRIV(ucp_gentype)[prop->chartype] == ucp_L ||
-          PRIV(ucp_gentype)[prop->chartype] == ucp_N) == negated;
+    else
+#endif  /* SUPPORT_UCP */
 
-  case PT_SPACE:    /* Perl space */
-  return (PRIV(ucp_gentype)[prop->chartype] == ucp_Z ||
-          c == CHAR_HT || c == CHAR_NL || c == CHAR_FF || c == CHAR_CR)
-          == negated;
+    accepted = leftop >= FIRST_AUTOTAB_OP && leftop <= LAST_AUTOTAB_LEFT_OP &&
+           rightop >= FIRST_AUTOTAB_OP && rightop <= LAST_AUTOTAB_RIGHT_OP &&
+           autoposstab[leftop - FIRST_AUTOTAB_OP][rightop - FIRST_AUTOTAB_OP];
 
-  case PT_PXSPACE:  /* POSIX space */
-  return (PRIV(ucp_gentype)[prop->chartype] == ucp_Z ||
-          c == CHAR_HT || c == CHAR_NL || c == CHAR_VT ||
-          c == CHAR_FF || c == CHAR_CR)
-          == negated;
+    if (!accepted) return FALSE;
 
-  case PT_WORD:
-  return (PRIV(ucp_gentype)[prop->chartype] == ucp_L ||
-          PRIV(ucp_gentype)[prop->chartype] == ucp_N ||
-          c == CHAR_UNDERSCORE) == negated;
+    if (list[1] == 0) return TRUE;
+    /* Might be an empty repeat. */
+    continue;
+    }
 
-#ifdef SUPPORT_UCP
-  case PT_CLIST:
-  p = PRIV(ucd_caseless_sets) + prop->caseset;
-  for (;;)
+  /* Control reaches here only if one of the items is a small character list.
+  All characters are checked against the other side. */
+
+  do
     {
-    if (c < *p) return !negated;
-    if (c == *p++) return negated;
-    }
-  break;  /* Control never reaches here */
+    chr = *chr_ptr;
+
+    switch(list_ptr[0])
+      {
+      case OP_CHAR:
+      ochr_ptr = list_ptr + 2;
+      do
+        {
+        if (chr == *ochr_ptr) return FALSE;
+        ochr_ptr++;
+        }
+      while(*ochr_ptr != NOTACHAR);
+      break;
+
+      case OP_NOT:
+      ochr_ptr = list_ptr + 2;
+      do
+        {
+        if (chr == *ochr_ptr)
+          break;
+        ochr_ptr++;
+        }
+      while(*ochr_ptr != NOTACHAR);
+      if (*ochr_ptr == NOTACHAR) return FALSE;   /* Not found */
+      break;
+
+      /* Note that OP_DIGIT etc. are generated only when PCRE_UCP is *not*
+      set. When it is set, \d etc. are converted into OP_(NOT_)PROP codes. */
+
+      case OP_DIGIT:
+      if (chr < 256 && (cd->ctypes[chr] & ctype_digit) != 0) return FALSE;
+      break;
+
+      case OP_NOT_DIGIT:
+      if (chr > 255 || (cd->ctypes[chr] & ctype_digit) == 0) return FALSE;
+      break;
+
+      case OP_WHITESPACE:
+      if (chr < 256 && (cd->ctypes[chr] & ctype_space) != 0) return FALSE;
+      break;
+
+      case OP_NOT_WHITESPACE:
+      if (chr > 255 || (cd->ctypes[chr] & ctype_space) == 0) return FALSE;
+      break;
+
+      case OP_WORDCHAR:
+      if (chr < 255 && (cd->ctypes[chr] & ctype_word) != 0) return FALSE;
+      break;
+
+      case OP_NOT_WORDCHAR:
+      if (chr > 255 || (cd->ctypes[chr] & ctype_word) == 0) return FALSE;
+      break;
+
+      case OP_HSPACE:
+      switch(chr)
+        {
+        HSPACE_CASES: return FALSE;
+        default: break;
+        }
+      break;
+
+      case OP_NOT_HSPACE:
+      switch(chr)
+        {
+        HSPACE_CASES: break;
+        default: return FALSE;
+        }
+      break;
+
+      case OP_ANYNL:
+      case OP_VSPACE:
+      switch(chr)
+        {
+        VSPACE_CASES: return FALSE;
+        default: break;
+        }
+      break;
+
+      case OP_NOT_VSPACE:
+      switch(chr)
+        {
+        VSPACE_CASES: break;
+        default: return FALSE;
+        }
+      break;
+
+      case OP_DOLL:
+      case OP_EODN:
+      switch (chr)
+        {
+        case CHAR_CR:
+        case CHAR_LF:
+        case CHAR_VT:
+        case CHAR_FF:
+        case CHAR_NEL:
+#ifndef EBCDIC
+        case 0x2028:
+        case 0x2029:
+#endif  /* Not EBCDIC */
+        return FALSE;
+        }
+      break;
+
+      case OP_EOD:    /* Can always possessify before \z */
+      break;
+
+#ifdef SUPPORT_UCP
+      case OP_PROP:
+      case OP_NOTPROP:
+      if (!check_char_prop(chr, list_ptr[2], list_ptr[3],
+            list_ptr[0] == OP_NOTPROP))
+        return FALSE;
+      break;
+#endif
+
+      case OP_NCLASS:
+      if (chr > 255) return FALSE;
+      /* Fall through */
+
+      case OP_CLASS:
+      if (chr > 255) break;
+      class_bitset = (pcre_uint8 *)
+        ((list_ptr == list ? code : base_end) - list_ptr[2]);
+      if ((class_bitset[chr >> 3] & (1 << (chr & 7))) != 0) return FALSE;
+      break;
+
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+      case OP_XCLASS:
+      if (PRIV(xclass)(chr, (list_ptr == list ? code : base_end) -
+          list_ptr[2] + LINK_SIZE, utf)) return FALSE;
+      break;
 #endif
+
+      default:
+      return FALSE;
+      }
+
+    chr_ptr++;
+    }
+  while(*chr_ptr != NOTACHAR);
+
+  /* At least one character must be matched from this opcode. */
+
+  if (list[1] == 0) return TRUE;
   }
 
-return FALSE;
+/* Control never reaches here. There used to be a fail-save return FALSE; here,
+but some compilers complain about an unreachable statement. */
+
 }
-#endif  /* SUPPORT_UCP */
 
 
 
 /*************************************************
-*     Check if auto-possessifying is possible    *
+*    Scan compiled regex for auto-possession     *
 *************************************************/
 
-/* This function is called for unlimited repeats of certain items, to see
-whether the next thing could possibly match the repeated item. If not, it makes
-sense to automatically possessify the repeated item.
+/* Replaces single character iterations with their possessive alternatives
+if appropriate. This function modifies the compiled opcode!
 
 Arguments:
-  previous      pointer to the repeated opcode
-  utf           TRUE in UTF-8 / UTF-16 / UTF-32 mode
-  ptr           next character in pattern
-  options       options bits
-  cd            contains pointers to tables etc.
+  code        points to start of the byte code
+  utf         TRUE in UTF-8 / UTF-16 / UTF-32 mode
+  cd          static compile data
 
-Returns:        TRUE if possessifying is wanted
+Returns:      nothing
 */
 
-static BOOL
-check_auto_possessive(const pcre_uchar *previous, BOOL utf,
-  const pcre_uchar *ptr, int options, compile_data *cd)
+static void
+auto_possessify(pcre_uchar *code, BOOL utf, const compile_data *cd)
 {
-pcre_uint32 c = NOTACHAR;
-pcre_uint32 next;
-int escape;
-pcre_uchar op_code = *previous++;
-
-/* Skip whitespace and comments in extended mode */
+register pcre_uchar c;
+const pcre_uchar *end;
+pcre_uchar *repeat_opcode;
+pcre_uint32 list[8];
+int rec_limit;
 
-if ((options & PCRE_EXTENDED) != 0)
+for (;;)
   {
-  for (;;)
+  c = *code;
+
+  /* When a pattern with bad UTF-8 encoding is compiled with NO_UTF_CHECK,
+  it may compile without complaining, but may get into a loop here if the code
+  pointer points to a bad value. This is, of course a documentated possibility,
+  when NO_UTF_CHECK is set, so it isn't a bug, but we can detect this case and
+  just give up on this optimization. */
+
+  if (c >= OP_TABLE_LENGTH) return;
+
+  if (c >= OP_STAR && c <= OP_TYPEPOSUPTO)
     {
-    while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_space) != 0) ptr++;
-    if (*ptr == CHAR_NUMBER_SIGN)
+    c -= get_repeat_base(c) - OP_STAR;
+    end = (c <= OP_MINUPTO) ?
+      get_chr_property_list(code, utf, cd->fcc, list) : NULL;
+    list[1] = c == OP_STAR || c == OP_PLUS || c == OP_QUERY || c == OP_UPTO;
+
+    rec_limit = 1000;
+    if (end != NULL && compare_opcodes(end, utf, cd, list, end, &rec_limit))
       {
-      ptr++;
-      while (*ptr != CHAR_NULL)
+      switch(c)
         {
-        if (IS_NEWLINE(ptr)) { ptr += cd->nllen; break; }
-        ptr++;
-#ifdef SUPPORT_UTF
-        if (utf) FORWARDCHAR(ptr);
+        case OP_STAR:
+        *code += OP_POSSTAR - OP_STAR;
+        break;
+
+        case OP_MINSTAR:
+        *code += OP_POSSTAR - OP_MINSTAR;
+        break;
+
+        case OP_PLUS:
+        *code += OP_POSPLUS - OP_PLUS;
+        break;
+
+        case OP_MINPLUS:
+        *code += OP_POSPLUS - OP_MINPLUS;
+        break;
+
+        case OP_QUERY:
+        *code += OP_POSQUERY - OP_QUERY;
+        break;
+
+        case OP_MINQUERY:
+        *code += OP_POSQUERY - OP_MINQUERY;
+        break;
+
+        case OP_UPTO:
+        *code += OP_POSUPTO - OP_UPTO;
+        break;
+
+        case OP_MINUPTO:
+        *code += OP_POSUPTO - OP_MINUPTO;
+        break;
+        }
+      }
+    c = *code;
+    }
+  else if (c == OP_CLASS || c == OP_NCLASS || c == OP_XCLASS)
+    {
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+    if (c == OP_XCLASS)
+      repeat_opcode = code + GET(code, 1);
+    else
 #endif
+      repeat_opcode = code + 1 + (32 / sizeof(pcre_uchar));
+
+    c = *repeat_opcode;
+    if (c >= OP_CRSTAR && c <= OP_CRMINRANGE)
+      {
+      /* end must not be NULL. */
+      end = get_chr_property_list(code, utf, cd->fcc, list);
+
+      list[1] = (c & 1) == 0;
+
+      rec_limit = 1000;
+      if (compare_opcodes(end, utf, cd, list, end, &rec_limit))
+        {
+        switch (c)
+          {
+          case OP_CRSTAR:
+          case OP_CRMINSTAR:
+          *repeat_opcode = OP_CRPOSSTAR;
+          break;
+
+          case OP_CRPLUS:
+          case OP_CRMINPLUS:
+          *repeat_opcode = OP_CRPOSPLUS;
+          break;
+
+          case OP_CRQUERY:
+          case OP_CRMINQUERY:
+          *repeat_opcode = OP_CRPOSQUERY;
+          break;
+
+          case OP_CRRANGE:
+          case OP_CRMINRANGE:
+          *repeat_opcode = OP_CRPOSRANGE;
+          break;
+          }
         }
       }
-    else break;
+    c = *code;
     }
-  }
 
-/* If the next item is one that we can handle, get its value. A non-negative
-value is a character, a negative value is an escape value. */
+  switch(c)
+    {
+    case OP_END:
+    return;
 
-if (*ptr == CHAR_BACKSLASH)
-  {
-  int temperrorcode = 0;
-  escape = check_escape(&ptr, &next, &temperrorcode, cd->bracount, options,
-    FALSE);
-  if (temperrorcode != 0) return FALSE;
-  ptr++;    /* Point after the escape sequence */
-  }
-else if (!MAX_255(*ptr) || (cd->ctypes[*ptr] & ctype_meta) == 0)
-  {
-  escape = 0;
-#ifdef SUPPORT_UTF
-  if (utf) { GETCHARINC(next, ptr); } else
+    case OP_TYPESTAR:
+    case OP_TYPEMINSTAR:
+    case OP_TYPEPLUS:
+    case OP_TYPEMINPLUS:
+    case OP_TYPEQUERY:
+    case OP_TYPEMINQUERY:
+    case OP_TYPEPOSSTAR:
+    case OP_TYPEPOSPLUS:
+    case OP_TYPEPOSQUERY:
+    if (code[1] == OP_PROP || code[1] == OP_NOTPROP) code += 2;
+    break;
+
+    case OP_TYPEUPTO:
+    case OP_TYPEMINUPTO:
+    case OP_TYPEEXACT:
+    case OP_TYPEPOSUPTO:
+    if (code[1 + IMM2_SIZE] == OP_PROP || code[1 + IMM2_SIZE] == OP_NOTPROP)
+      code += 2;
+    break;
+
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+    case OP_XCLASS:
+    code += GET(code, 1);
+    break;
 #endif
-  next = *ptr++;
-  }
-else return FALSE;
 
-/* Skip whitespace and comments in extended mode */
+    case OP_MARK:
+    case OP_PRUNE_ARG:
+    case OP_SKIP_ARG:
+    case OP_THEN_ARG:
+    code += code[1];
+    break;
+    }
 
-if ((options & PCRE_EXTENDED) != 0)
-  {
-  for (;;)
+  /* Add in the fixed length from the table */
+
+  code += PRIV(OP_lengths)[c];
+
+  /* In UTF-8 mode, opcodes that are followed by a character may be followed by
+  a multi-byte character. The length in the table is a minimum, so we have to
+  arrange to skip the extra bytes. */
+
+#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+  if (utf) switch(c)
     {
-    while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_space) != 0) ptr++;
-    if (*ptr == CHAR_NUMBER_SIGN)
-      {
-      ptr++;
-      while (*ptr != CHAR_NULL)
-        {
-        if (IS_NEWLINE(ptr)) { ptr += cd->nllen; break; }
-        ptr++;
-#ifdef SUPPORT_UTF
-        if (utf) FORWARDCHAR(ptr);
-#endif
-        }
-      }
-    else break;
+    case OP_CHAR:
+    case OP_CHARI:
+    case OP_NOT:
+    case OP_NOTI:
+    case OP_STAR:
+    case OP_MINSTAR:
+    case OP_PLUS:
+    case OP_MINPLUS:
+    case OP_QUERY:
+    case OP_MINQUERY:
+    case OP_UPTO:
+    case OP_MINUPTO:
+    case OP_EXACT:
+    case OP_POSSTAR:
+    case OP_POSPLUS:
+    case OP_POSQUERY:
+    case OP_POSUPTO:
+    case OP_STARI:
+    case OP_MINSTARI:
+    case OP_PLUSI:
+    case OP_MINPLUSI:
+    case OP_QUERYI:
+    case OP_MINQUERYI:
+    case OP_UPTOI:
+    case OP_MINUPTOI:
+    case OP_EXACTI:
+    case OP_POSSTARI:
+    case OP_POSPLUSI:
+    case OP_POSQUERYI:
+    case OP_POSUPTOI:
+    case OP_NOTSTAR:
+    case OP_NOTMINSTAR:
+    case OP_NOTPLUS:
+    case OP_NOTMINPLUS:
+    case OP_NOTQUERY:
+    case OP_NOTMINQUERY:
+    case OP_NOTUPTO:
+    case OP_NOTMINUPTO:
+    case OP_NOTEXACT:
+    case OP_NOTPOSSTAR:
+    case OP_NOTPOSPLUS:
+    case OP_NOTPOSQUERY:
+    case OP_NOTPOSUPTO:
+    case OP_NOTSTARI:
+    case OP_NOTMINSTARI:
+    case OP_NOTPLUSI:
+    case OP_NOTMINPLUSI:
+    case OP_NOTQUERYI:
+    case OP_NOTMINQUERYI:
+    case OP_NOTUPTOI:
+    case OP_NOTMINUPTOI:
+    case OP_NOTEXACTI:
+    case OP_NOTPOSSTARI:
+    case OP_NOTPOSPLUSI:
+    case OP_NOTPOSQUERYI:
+    case OP_NOTPOSUPTOI:
+    if (HAS_EXTRALEN(code[-1])) code += GET_EXTRALEN(code[-1]);
+    break;
     }
+#else
+  (void)(utf);  /* Keep compiler happy by referencing function argument */
+#endif
   }
+}
 
-/* If the next thing is itself optional, we have to give up. */
 
-if (*ptr == CHAR_ASTERISK || *ptr == CHAR_QUESTION_MARK ||
-  STRNCMP_UC_C8(ptr, STR_LEFT_CURLY_BRACKET STR_0 STR_COMMA, 3) == 0)
-    return FALSE;
 
-/* If the previous item is a character, get its value. */
+/*************************************************
+*           Check for POSIX class syntax         *
+*************************************************/
 
-if (op_code == OP_CHAR || op_code == OP_CHARI ||
-    op_code == OP_NOT || op_code == OP_NOTI)
-  {
-#ifdef SUPPORT_UTF
-  GETCHARTEST(c, previous);
-#else
-  c = *previous;
-#endif
-  }
+/* This function is called when the sequence "[:" or "[." or "[=" is
+encountered in a character class. It checks whether this is followed by a
+sequence of characters terminated by a matching ":]" or ".]" or "=]". If we
+reach an unescaped ']' without the special preceding character, return FALSE.
 
-/* Now compare the next item with the previous opcode. First, handle cases when
-the next item is a character. */
+Originally, this function only recognized a sequence of letters between the
+terminators, but it seems that Perl recognizes any sequence of characters,
+though of course unknown POSIX names are subsequently rejected. Perl gives an
+"Unknown POSIX class" error for [:f\oo:] for example, where previously PCRE
+didn't consider this to be a POSIX class. Likewise for [:1234:].
 
-if (escape == 0)
-  {
-  /* For a caseless UTF match, the next character may have more than one other
-  case, which maps to the special PT_CLIST property. Check this first. */
+The problem in trying to be exactly like Perl is in the handling of escapes. We
+have to be sure that [abc[:x\]pqr] is *not* treated as containing a POSIX
+class, but [abc[:x\]pqr:]] is (so that an error can be generated). The code
+below handles the special cases \\ and \], but does not try to do any other
+escape processing. This makes it different from Perl for cases such as
+[:l\ower:] where Perl recognizes it as the POSIX class "lower" but PCRE does
+not recognize "l\ower". This is a lesser evil than not diagnosing bad classes
+when Perl does, I think.
 
-#ifdef SUPPORT_UCP
-  if (utf && c != NOTACHAR && (options & PCRE_CASELESS) != 0)
-    {
-    unsigned int ocs = UCD_CASESET(next);
-    if (ocs > 0) return check_char_prop(c, PT_CLIST, ocs, op_code >= OP_NOT);
-    }
-#endif
+A user pointed out that PCRE was rejecting [:a[:digit:]] whereas Perl was not.
+It seems that the appearance of a nested POSIX class supersedes an apparent
+external class. For example, [:a[:digit:]b:] matches "a", "b", ":", or
+a digit.
 
-  switch(op_code)
-    {
-    case OP_CHAR:
-    return c != next;
+In Perl, unescaped square brackets may also appear as part of class names. For
+example, [:a[:abc]b:] gives unknown POSIX class "[:abc]b:]". However, for
+[:a[:abc]b][b:] it gives unknown POSIX class "[:abc]b][b:]", which does not
+seem right at all. PCRE does not allow closing square brackets in POSIX class
+names.
 
-    /* For CHARI (caseless character) we must check the other case. If we have
-    Unicode property support, we can use it to test the other case of
-    high-valued characters. We know that next can have only one other case,
-    because multi-other-case characters are dealt with above. */
+Arguments:
+  ptr      pointer to the initial [
+  endptr   where to return the end pointer
 
-    case OP_CHARI:
-    if (c == next) return FALSE;
-#ifdef SUPPORT_UTF
-    if (utf)
-      {
-      pcre_uint32 othercase;
-      if (next < 128) othercase = cd->fcc[next]; else
-#ifdef SUPPORT_UCP
-      othercase = UCD_OTHERCASE(next);
-#else
-      othercase = NOTACHAR;
-#endif
-      return c != othercase;
-      }
-    else
-#endif  /* SUPPORT_UTF */
-    return (c != TABLE_GET(next, cd->fcc, next));  /* Not UTF */
+Returns:   TRUE or FALSE
+*/
 
-    case OP_NOT:
-    return c == next;
+static BOOL
+check_posix_syntax(const pcre_uchar *ptr, const pcre_uchar **endptr)
+{
+pcre_uchar terminator;          /* Don't combine these lines; the Solaris cc */
+terminator = *(++ptr);   /* compiler warns about "non-constant" initializer. */
+for (++ptr; *ptr != CHAR_NULL; ptr++)
+  {
+  if (*ptr == CHAR_BACKSLASH &&
+      (ptr[1] == CHAR_RIGHT_SQUARE_BRACKET ||
+       ptr[1] == CHAR_BACKSLASH))
+    ptr++;
+  else if ((*ptr == CHAR_LEFT_SQUARE_BRACKET && ptr[1] == terminator) ||
+            *ptr == CHAR_RIGHT_SQUARE_BRACKET) return FALSE;
+  else if (*ptr == terminator && ptr[1] == CHAR_RIGHT_SQUARE_BRACKET)
+    {
+    *endptr = ptr;
+    return TRUE;
+    }
+  }
+return FALSE;
+}
 
-    case OP_NOTI:
-    if (c == next) return TRUE;
-#ifdef SUPPORT_UTF
-    if (utf)
-      {
-      pcre_uint32 othercase;
-      if (next < 128) othercase = cd->fcc[next]; else
-#ifdef SUPPORT_UCP
-      othercase = UCD_OTHERCASE(next);
-#else
-      othercase = NOTACHAR;
-#endif
-      return c == othercase;
-      }
-    else
-#endif  /* SUPPORT_UTF */
-    return (c == TABLE_GET(next, cd->fcc, next));  /* Not UTF */
 
-    /* Note that OP_DIGIT etc. are generated only when PCRE_UCP is *not* set.
-    When it is set, \d etc. are converted into OP_(NOT_)PROP codes. */
 
-    case OP_DIGIT:
-    return next > 255 || (cd->ctypes[next] & ctype_digit) == 0;
 
-    case OP_NOT_DIGIT:
-    return next <= 255 && (cd->ctypes[next] & ctype_digit) != 0;
+/*************************************************
+*          Check POSIX class name                *
+*************************************************/
 
-    case OP_WHITESPACE:
-    return next > 255 || (cd->ctypes[next] & ctype_space) == 0;
+/* This function is called to check the name given in a POSIX-style class entry
+such as [:alnum:].
 
-    case OP_NOT_WHITESPACE:
-    return next <= 255 && (cd->ctypes[next] & ctype_space) != 0;
+Arguments:
+  ptr        points to the first letter
+  len        the length of the name
 
-    case OP_WORDCHAR:
-    return next > 255 || (cd->ctypes[next] & ctype_word) == 0;
+Returns:     a value representing the name, or -1 if unknown
+*/
 
-    case OP_NOT_WORDCHAR:
-    return next <= 255 && (cd->ctypes[next] & ctype_word) != 0;
+static int
+check_posix_name(const pcre_uchar *ptr, int len)
+{
+const char *pn = posix_names;
+register int yield = 0;
+while (posix_name_lengths[yield] != 0)
+  {
+  if (len == posix_name_lengths[yield] &&
+    STRNCMP_UC_C8(ptr, pn, (unsigned int)len) == 0) return yield;
+  pn += posix_name_lengths[yield] + 1;
+  yield++;
+  }
+return -1;
+}
 
-    case OP_HSPACE:
-    case OP_NOT_HSPACE:
-    switch(next)
-      {
-      HSPACE_CASES:
-      return op_code == OP_NOT_HSPACE;
 
-      default:
-      return op_code != OP_NOT_HSPACE;
-      }
+/*************************************************
+*    Adjust OP_RECURSE items in repeated group   *
+*************************************************/
 
-    case OP_ANYNL:
-    case OP_VSPACE:
-    case OP_NOT_VSPACE:
-    switch(next)
-      {
-      VSPACE_CASES:
-      return op_code == OP_NOT_VSPACE;
+/* OP_RECURSE items contain an offset from the start of the regex to the group
+that is referenced. This means that groups can be replicated for fixed
+repetition simply by copying (because the recursion is allowed to refer to
+earlier groups that are outside the current group). However, when a group is
+optional (i.e. the minimum quantifier is zero), OP_BRAZERO or OP_SKIPZERO is
+inserted before it, after it has been compiled. This means that any OP_RECURSE
+items within it that refer to the group itself or any contained groups have to
+have their offsets adjusted. That one of the jobs of this function. Before it
+is called, the partially compiled regex must be temporarily terminated with
+OP_END.
 
-      default:
-      return op_code != OP_NOT_VSPACE;
-      }
+This function has been extended to cope with forward references for recursions
+and subroutine calls. It must check the list of such references for the
+group we are dealing with. If it finds that one of the recursions in the
+current group is on this list, it does not adjust the value in the reference
+(which is a group number). After the group has been scanned, all the offsets in
+the forward reference list for the group are adjusted.
 
-#ifdef SUPPORT_UCP
-    case OP_PROP:
-    return check_char_prop(next, previous[0], previous[1], FALSE);
+Arguments:
+  group      points to the start of the group
+  adjust     the amount by which the group is to be moved
+  utf        TRUE in UTF-8 / UTF-16 / UTF-32 mode
+  cd         contains pointers to tables etc.
+  save_hwm_offset   the hwm forward reference offset at the start of the group
 
-    case OP_NOTPROP:
-    return check_char_prop(next, previous[0], previous[1], TRUE);
-#endif
+Returns:     nothing
+*/
 
-    default:
-    return FALSE;
+static void
+adjust_recurse(pcre_uchar *group, int adjust, BOOL utf, compile_data *cd,
+  size_t save_hwm_offset)
+{
+int offset;
+pcre_uchar *hc;
+pcre_uchar *ptr = group;
+
+while ((ptr = (pcre_uchar *)find_recurse(ptr, utf)) != NULL)
+  {
+  for (hc = (pcre_uchar *)cd->start_workspace + save_hwm_offset; hc < cd->hwm;
+       hc += LINK_SIZE)
+    {
+    offset = (int)GET(hc, 0);
+    if (cd->start_code + offset == ptr + 1) break;
     }
-  }
 
-/* Handle the case when the next item is \d, \s, etc. Note that when PCRE_UCP
-is set, \d turns into ESC_du rather than ESC_d, etc., so ESC_d etc. are
-generated only when PCRE_UCP is *not* set, that is, when only ASCII
-characteristics are recognized. Similarly, the opcodes OP_DIGIT etc. are
-replaced by OP_PROP codes when PCRE_UCP is set. */
+  /* If we have not found this recursion on the forward reference list, adjust
+  the recursion's offset if it's after the start of this group. */
 
-switch(op_code)
-  {
-  case OP_CHAR:
-  case OP_CHARI:
-  switch(escape)
+  if (hc >= cd->hwm)
     {
-    case ESC_d:
-    return c > 255 || (cd->ctypes[c] & ctype_digit) == 0;
+    offset = (int)GET(ptr, 1);
+    if (cd->start_code + offset >= group) PUT(ptr, 1, offset + adjust);
+    }
 
-    case ESC_D:
-    return c <= 255 && (cd->ctypes[c] & ctype_digit) != 0;
+  ptr += 1 + LINK_SIZE;
+  }
 
-    case ESC_s:
-    return c > 255 || (cd->ctypes[c] & ctype_space) == 0;
+/* Now adjust all forward reference offsets for the group. */
 
-    case ESC_S:
-    return c <= 255 && (cd->ctypes[c] & ctype_space) != 0;
+for (hc = (pcre_uchar *)cd->start_workspace + save_hwm_offset; hc < cd->hwm;
+     hc += LINK_SIZE)
+  {
+  offset = (int)GET(hc, 0);
+  PUT(hc, 0, offset + adjust);
+  }
+}
 
-    case ESC_w:
-    return c > 255 || (cd->ctypes[c] & ctype_word) == 0;
 
-    case ESC_W:
-    return c <= 255 && (cd->ctypes[c] & ctype_word) != 0;
 
-    case ESC_h:
-    case ESC_H:
-    switch(c)
-      {
-      HSPACE_CASES:
-      return escape != ESC_h;
+/*************************************************
+*        Insert an automatic callout point       *
+*************************************************/
 
-      default:
-      return escape == ESC_h;
-      }
+/* This function is called when the PCRE_AUTO_CALLOUT option is set, to insert
+callout points before each pattern item.
 
-    case ESC_v:
-    case ESC_V:
-    switch(c)
-      {
-      VSPACE_CASES:
-      return escape != ESC_v;
+Arguments:
+  code           current code pointer
+  ptr            current pattern pointer
+  cd             pointers to tables etc
 
-      default:
-      return escape == ESC_v;
-      }
+Returns:         new code pointer
+*/
 
-    /* When PCRE_UCP is set, these values get generated for \d etc. Find
-    their substitutions and process them. The result will always be either
-    ESC_p or ESC_P. Then fall through to process those values. */
+static pcre_uchar *
+auto_callout(pcre_uchar *code, const pcre_uchar *ptr, compile_data *cd)
+{
+*code++ = OP_CALLOUT;
+*code++ = 255;
+PUT(code, 0, (int)(ptr - cd->start_pattern));  /* Pattern offset */
+PUT(code, LINK_SIZE, 0);                       /* Default length */
+return code + 2 * LINK_SIZE;
+}
 
-#ifdef SUPPORT_UCP
-    case ESC_du:
-    case ESC_DU:
-    case ESC_wu:
-    case ESC_WU:
-    case ESC_su:
-    case ESC_SU:
-      {
-      int temperrorcode = 0;
-      ptr = substitutes[escape - ESC_DU];
-      escape = check_escape(&ptr, &next, &temperrorcode, 0, options, FALSE);
-      if (temperrorcode != 0) return FALSE;
-      ptr++;    /* For compatibility */
-      }
-    /* Fall through */
 
-    case ESC_p:
-    case ESC_P:
-      {
-      unsigned int ptype = 0, pdata = 0;
-      int errorcodeptr;
-      BOOL negated;
 
-      ptr--;      /* Make ptr point at the p or P */
-      if (!get_ucp(&ptr, &negated, &ptype, &pdata, &errorcodeptr))
-        return FALSE;
-      ptr++;      /* Point past the final curly ket */
+/*************************************************
+*         Complete a callout item                *
+*************************************************/
 
-      /* If the property item is optional, we have to give up. (When generated
-      from \d etc by PCRE_UCP, this test will have been applied much earlier,
-      to the original \d etc. At this point, ptr will point to a zero byte. */
+/* A callout item contains the length of the next item in the pattern, which
+we can't fill in till after we have reached the relevant point. This is used
+for both automatic and manual callouts.
 
-      if (*ptr == CHAR_ASTERISK || *ptr == CHAR_QUESTION_MARK ||
-        STRNCMP_UC_C8(ptr, STR_LEFT_CURLY_BRACKET STR_0 STR_COMMA, 3) == 0)
-          return FALSE;
+Arguments:
+  previous_callout   points to previous callout item
+  ptr                current pattern pointer
+  cd                 pointers to tables etc
 
-      /* Do the property check. */
+Returns:             nothing
+*/
 
-      return check_char_prop(c, ptype, pdata, (escape == ESC_P) != negated);
-      }
-#endif
+static void
+complete_callout(pcre_uchar *previous_callout, const pcre_uchar *ptr, compile_data *cd)
+{
+int length = (int)(ptr - cd->start_pattern - GET(previous_callout, 2));
+PUT(previous_callout, 2 + LINK_SIZE, length);
+}
 
-    default:
-    return FALSE;
-    }
 
-  /* In principle, support for Unicode properties should be integrated here as
-  well. It means re-organizing the above code so as to get hold of the property
-  values before switching on the op-code. However, I wonder how many patterns
-  combine ASCII \d etc with Unicode properties? (Note that if PCRE_UCP is set,
-  these op-codes are never generated.) */
 
-  case OP_DIGIT:
-  return escape == ESC_D || escape == ESC_s || escape == ESC_W ||
-         escape == ESC_h || escape == ESC_v || escape == ESC_R;
+#ifdef SUPPORT_UCP
+/*************************************************
+*           Get othercase range                  *
+*************************************************/
 
-  case OP_NOT_DIGIT:
-  return escape == ESC_d;
+/* This function is passed the start and end of a class range, in UTF-8 mode
+with UCP support. It searches up the characters, looking for ranges of
+characters in the "other" case. Each call returns the next one, updating the
+start address. A character with multiple other cases is returned on its own
+with a special return value.
 
-  case OP_WHITESPACE:
-  return escape == ESC_S || escape == ESC_d || escape == ESC_w;
+Arguments:
+  cptr        points to starting character value; updated
+  d           end value
+  ocptr       where to put start of othercase range
+  odptr       where to put end of othercase range
 
-  case OP_NOT_WHITESPACE:
-  return escape == ESC_s || escape == ESC_h || escape == ESC_v || escape == ESC_R;
+Yield:        -1 when no more
+               0 when a range is returned
+              >0 the CASESET offset for char with multiple other cases
+                in this case, ocptr contains the original
+*/
 
-  case OP_HSPACE:
-  return escape == ESC_S || escape == ESC_H || escape == ESC_d ||
-         escape == ESC_w || escape == ESC_v || escape == ESC_R;
+static int
+get_othercase_range(pcre_uint32 *cptr, pcre_uint32 d, pcre_uint32 *ocptr,
+  pcre_uint32 *odptr)
+{
+pcre_uint32 c, othercase, next;
+unsigned int co;
 
-  case OP_NOT_HSPACE:
-  return escape == ESC_h;
+/* Find the first character that has an other case. If it has multiple other
+cases, return its case offset value. */
 
-  /* Can't have \S in here because VT matches \S (Perl anomaly) */
-  case OP_ANYNL:
-  case OP_VSPACE:
-  return escape == ESC_V || escape == ESC_d || escape == ESC_w;
+for (c = *cptr; c <= d; c++)
+  {
+  if ((co = UCD_CASESET(c)) != 0)
+    {
+    *ocptr = c++;   /* Character that has the set */
+    *cptr = c;      /* Rest of input range */
+    return (int)co;
+    }
+  if ((othercase = UCD_OTHERCASE(c)) != c) break;
+  }
 
-  case OP_NOT_VSPACE:
-  return escape == ESC_v || escape == ESC_R;
+if (c > d) return -1;  /* Reached end of range */
 
-  case OP_WORDCHAR:
-  return escape == ESC_W || escape == ESC_s || escape == ESC_h ||
-         escape == ESC_v || escape == ESC_R;
+/* Found a character that has a single other case. Search for the end of the
+range, which is either the end of the input range, or a character that has zero
+or more than one other cases. */
 
-  case OP_NOT_WORDCHAR:
-  return escape == ESC_w || escape == ESC_d;
+*ocptr = othercase;
+next = othercase + 1;
 
-  default:
-  return FALSE;
+for (++c; c <= d; c++)
+  {
+  if ((co = UCD_CASESET(c)) != 0 || UCD_OTHERCASE(c) != next) break;
+  next++;
   }
 
-/* Control does not reach here */
+*odptr = next - 1;     /* End of othercase range */
+*cptr = c;             /* Rest of input range */
+return 0;
 }
+#endif  /* SUPPORT_UCP */
 
 
 
@@ -3448,6 +4204,7 @@ add_to_class(pcre_uint8 *classbits, pcre_uchar **uchardptr, int options,
   compile_data *cd, pcre_uint32 start, pcre_uint32 end)
 {
 pcre_uint32 c;
+pcre_uint32 classbits_end = (end <= 0xff ? end : 0xff);
 int n8 = 0;
 
 /* If caseless matching is required, scan the range and process alternate
@@ -3482,7 +4239,11 @@ if ((options & PCRE_CASELESS) != 0)
       range. Otherwise, use a recursive call to add the additional range. */
 
       else if (oc < start && od >= start - 1) start = oc; /* Extend downwards */
-      else if (od > end && oc <= end + 1) end = od;       /* Extend upwards */
+      else if (od > end && oc <= end + 1)
+        {
+        end = od;       /* Extend upwards */
+        if (end > classbits_end) classbits_end = (end <= 0xff ? end : 0xff);
+        }
       else n8 += add_to_class(classbits, uchardptr, options, cd, oc, od);
       }
     }
@@ -3491,7 +4252,7 @@ if ((options & PCRE_CASELESS) != 0)
 
   /* Not UTF-mode, or no UCP */
 
-  for (c = start; c <= end && c < 256; c++)
+  for (c = start; c <= classbits_end; c++)
     {
     SETBIT(classbits, cd->fcc[c]);
     n8++;
@@ -3516,22 +4277,21 @@ in all cases. */
 
 #endif /* COMPILE_PCRE[8|16] */
 
-/* If all characters are less than 256, use the bit map. Otherwise use extra
-data. */
+/* Use the bitmap for characters < 256. Otherwise use extra data.*/
 
-if (end < 0x100)
+for (c = start; c <= classbits_end; c++)
   {
-  for (c = start; c <= end; c++)
-    {
-    n8++;
-    SETBIT(classbits, c);
-    }
+  /* Regardless of start, c will always be <= 255. */
+  SETBIT(classbits, c);
+  n8++;
   }
 
-else
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+if (start <= 0xff) start = 0xff + 1;
+
+if (end >= start)
   {
   pcre_uchar *uchardata = *uchardptr;
-
 #ifdef SUPPORT_UTF
   if ((options & PCRE_UTF8) != 0)  /* All UTFs use the same flag bit */
     {
@@ -3571,6 +4331,7 @@ else
 
   *uchardptr = uchardata;   /* Updata extra data pointer */
   }
+#endif /* SUPPORT_UTF || !COMPILE_PCRE8 */
 
 return n8;    /* Number of 8-bit characters */
 }
@@ -3671,22 +4432,22 @@ to find out the amount of memory needed, as well as during the real compile
 phase. The value of lengthptr distinguishes the two phases.
 
 Arguments:
-  optionsptr     pointer to the option bits
-  codeptr        points to the pointer to the current code point
-  ptrptr         points to the current pattern pointer
-  errorcodeptr   points to error code variable
-  firstcharptr    place to put the first required character
+  optionsptr        pointer to the option bits
+  codeptr           points to the pointer to the current code point
+  ptrptr            points to the current pattern pointer
+  errorcodeptr      points to error code variable
+  firstcharptr      place to put the first required character
   firstcharflagsptr place to put the first character flags, or a negative number
-  reqcharptr     place to put the last required character
-  reqcharflagsptr place to put the last required character flags, or a negative number
-  bcptr          points to current branch chain
-  cond_depth     conditional nesting depth
-  cd             contains pointers to tables etc.
-  lengthptr      NULL during the real compile phase
-                 points to length accumulator during pre-compile phase
-
-Returns:         TRUE on success
-                 FALSE, with *errorcodeptr set non-zero on error
+  reqcharptr        place to put the last required character
+  reqcharflagsptr   place to put the last required character flags, or a negative number
+  bcptr             points to current branch chain
+  cond_depth        conditional nesting depth
+  cd                contains pointers to tables etc.
+  lengthptr         NULL during the real compile phase
+                    points to length accumulator during pre-compile phase
+
+Returns:            TRUE on success
+                    FALSE, with *errorcodeptr set non-zero on error
 */
 
 static BOOL
@@ -3722,7 +4483,7 @@ const pcre_uchar *tempptr;
 const pcre_uchar *nestptr = NULL;
 pcre_uchar *previous = NULL;
 pcre_uchar *previous_callout = NULL;
-pcre_uchar *save_hwm = NULL;
+size_t item_hwm_offset = 0;
 pcre_uint8 classbits[32];
 
 /* We can fish out the UTF-8 setting once and for all into a BOOL, but we
@@ -3792,6 +4553,9 @@ for (;; ptr++)
   BOOL reset_bracount;
   int class_has_8bitchar;
   int class_one_char;
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+  BOOL xclass_has_prop;
+#endif
   int newoptions;
   int recno;
   int refsign;
@@ -3804,6 +4568,10 @@ for (;; ptr++)
   pcre_uint32 ec;
   pcre_uchar mcbuffer[8];
 
+  /* Come here to restart the loop without advancing the pointer. */
+
+  REDO_LOOP:
+
   /* Get next character in the pattern */
 
   c = *ptr;
@@ -3829,7 +4597,8 @@ for (;; ptr++)
     if (code > cd->start_workspace + cd->workspace_size -
         WORK_SIZE_SAFETY_MARGIN)                       /* Check for overrun */
       {
-      *errorcodeptr = ERR52;
+      *errorcodeptr = (code >= cd->start_workspace + cd->workspace_size)?
+        ERR52 : ERR87;
       goto FAILED;
       }
 
@@ -3877,16 +4646,16 @@ for (;; ptr++)
   /* In the real compile phase, just check the workspace used by the forward
   reference list. */
 
-  else if (cd->hwm > cd->start_workspace + cd->workspace_size -
-           WORK_SIZE_SAFETY_MARGIN)
+  else if (cd->hwm > cd->start_workspace + cd->workspace_size)
     {
     *errorcodeptr = ERR52;
     goto FAILED;
     }
 
-  /* If in \Q...\E, check for the end; if not, we have a literal */
+  /* If in \Q...\E, check for the end; if not, we have a literal. Otherwise an
+  isolated \E is ignored. */
 
-  if (inescq && c != CHAR_NULL)
+  if (c != CHAR_NULL)
     {
     if (c == CHAR_BACKSLASH && ptr[1] == CHAR_E)
       {
@@ -3894,7 +4663,7 @@ for (;; ptr++)
       ptr++;
       continue;
       }
-    else
+    else if (inescq)
       {
       if (previous_callout != NULL)
         {
@@ -3909,58 +4678,97 @@ for (;; ptr++)
         }
       goto NORMAL_CHAR;
       }
-    }
-
-  /* Fill in length of a previous callout, except when the next thing is
-  a quantifier. */
 
-  is_quantifier =
-    c == CHAR_ASTERISK || c == CHAR_PLUS || c == CHAR_QUESTION_MARK ||
-    (c == CHAR_LEFT_CURLY_BRACKET && is_counted_repeat(ptr+1));
+    /* Check for the start of a \Q...\E sequence. We must do this here rather
+    than later in case it is immediately followed by \E, which turns it into a
+    "do nothing" sequence. */
 
-  if (!is_quantifier && previous_callout != NULL &&
-       after_manual_callout-- <= 0)
-    {
-    if (lengthptr == NULL)      /* Don't attempt in pre-compile phase */
-      complete_callout(previous_callout, ptr, cd);
-    previous_callout = NULL;
+    if (c == CHAR_BACKSLASH && ptr[1] == CHAR_Q)
+      {
+      inescq = TRUE;
+      ptr++;
+      continue;
+      }
     }
 
   /* In extended mode, skip white space and comments. */
 
   if ((options & PCRE_EXTENDED) != 0)
     {
-    if (MAX_255(*ptr) && (cd->ctypes[c] & ctype_space) != 0) continue;
+    const pcre_uchar *wscptr = ptr;
+    while (MAX_255(c) && (cd->ctypes[c] & ctype_space) != 0) c = *(++ptr);
     if (c == CHAR_NUMBER_SIGN)
       {
       ptr++;
       while (*ptr != CHAR_NULL)
         {
-        if (IS_NEWLINE(ptr)) { ptr += cd->nllen - 1; break; }
+        if (IS_NEWLINE(ptr))         /* For non-fixed-length newline cases, */
+          {                          /* IS_NEWLINE sets cd->nllen. */
+          ptr += cd->nllen;
+          break;
+          }
         ptr++;
 #ifdef SUPPORT_UTF
         if (utf) FORWARDCHAR(ptr);
 #endif
         }
-      if (*ptr != CHAR_NULL) continue;
+      }
 
-      /* Else fall through to handle end of string */
-      c = 0;
+    /* If we skipped any characters, restart the loop. Otherwise, we didn't see
+    a comment. */
+
+    if (ptr > wscptr) goto REDO_LOOP;
+    }
+
+  /* Skip over (?# comments. We need to do this here because we want to know if
+  the next thing is a quantifier, and these comments may come between an item
+  and its quantifier. */
+
+  if (c == CHAR_LEFT_PARENTHESIS && ptr[1] == CHAR_QUESTION_MARK &&
+      ptr[2] == CHAR_NUMBER_SIGN)
+    {
+    ptr += 3;
+    while (*ptr != CHAR_NULL && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++;
+    if (*ptr == CHAR_NULL)
+      {
+      *errorcodeptr = ERR18;
+      goto FAILED;
       }
+    continue;
+    }
+
+  /* See if the next thing is a quantifier. */
+
+  is_quantifier =
+    c == CHAR_ASTERISK || c == CHAR_PLUS || c == CHAR_QUESTION_MARK ||
+    (c == CHAR_LEFT_CURLY_BRACKET && is_counted_repeat(ptr+1));
+
+  /* Fill in length of a previous callout, except when the next thing is a
+  quantifier or when processing a property substitution string in UCP mode. */
+
+  if (!is_quantifier && previous_callout != NULL && nestptr == NULL &&
+       after_manual_callout-- <= 0)
+    {
+    if (lengthptr == NULL)      /* Don't attempt in pre-compile phase */
+      complete_callout(previous_callout, ptr, cd);
+    previous_callout = NULL;
     }
 
-  /* No auto callout for quantifiers. */
+  /* Create auto callout, except for quantifiers, or while processing property
+  strings that are substituted for \w etc in UCP mode. */
 
-  if ((options & PCRE_AUTO_CALLOUT) != 0 && !is_quantifier)
+  if ((options & PCRE_AUTO_CALLOUT) != 0 && !is_quantifier && nestptr == NULL)
     {
     previous_callout = code;
     code = auto_callout(code, ptr, cd);
     }
 
+  /* Process the next pattern item. */
+
   switch(c)
     {
     /* ===================================================================*/
-    case 0:                        /* The branch terminates at string end */
+    case CHAR_NULL:                /* The branch terminates at string end */
     case CHAR_VERTICAL_LINE:       /* or | or ) */
     case CHAR_RIGHT_PARENTHESIS:
     *firstcharptr = firstchar;
@@ -3990,7 +4798,8 @@ for (;; ptr++)
     previous = NULL;
     if ((options & PCRE_MULTILINE) != 0)
       {
-      if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE;
+      if (firstcharflags == REQ_UNSET)
+        zerofirstcharflags = firstcharflags = REQ_NONE;
       *code++ = OP_CIRCM;
       }
     else *code++ = OP_CIRC;
@@ -4011,6 +4820,7 @@ for (;; ptr++)
     zeroreqchar = reqchar;
     zeroreqcharflags = reqcharflags;
     previous = code;
+    item_hwm_offset = cd->hwm - cd->start_workspace;
     *code++ = ((options & PCRE_DOTALL) != 0)? OP_ALLANY: OP_ANY;
     break;
 
@@ -4038,8 +4848,31 @@ for (;; ptr++)
       }
     goto NORMAL_CHAR;
 
+    /* In another (POSIX) regex library, the ugly syntax [[:<:]] and [[:>:]] is
+    used for "start of word" and "end of word". As these are otherwise illegal
+    sequences, we don't break anything by recognizing them. They are replaced
+    by \b(?=\w) and \b(?<=\w) respectively. Sequences like [a[:<:]] are
+    erroneous and are handled by the normal code below. */
+
     case CHAR_LEFT_SQUARE_BRACKET:
+    if (STRNCMP_UC_C8(ptr+1, STRING_WEIRD_STARTWORD, 6) == 0)
+      {
+      nestptr = ptr + 7;
+      ptr = sub_start_of_word;
+      goto REDO_LOOP;
+      }
+
+    if (STRNCMP_UC_C8(ptr+1, STRING_WEIRD_ENDWORD, 6) == 0)
+      {
+      nestptr = ptr + 7;
+      ptr = sub_end_of_word;
+      goto REDO_LOOP;
+      }
+
+    /* Handle a real character class. */
+
     previous = code;
+    item_hwm_offset = cd->hwm - cd->start_workspace;
 
     /* PCRE supports POSIX class stuff inside a class. Perl gives an error if
     they are encountered at the top level, so we'll do that too. */
@@ -4095,13 +4928,26 @@ for (;; ptr++)
 
     should_flip_negation = FALSE;
 
+    /* Extended class (xclass) will be used when characters > 255
+    might match. */
+
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+    xclass = FALSE;
+    class_uchardata = code + LINK_SIZE + 2;   /* For XCLASS items */
+    class_uchardata_base = class_uchardata;   /* Save the start */
+#endif
+
     /* For optimization purposes, we track some properties of the class:
     class_has_8bitchar will be non-zero if the class contains at least one <
     256 character; class_one_char will be 1 if the class contains just one
-    character. */
+    character; xclass_has_prop will be TRUE if unicode property checks
+    are present in the class. */
 
     class_has_8bitchar = 0;
     class_one_char = 0;
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+    xclass_has_prop = FALSE;
+#endif
 
     /* Initialize the 32-char bit map to all zeros. We build the map in a
     temporary bit of memory, in case the class contains fewer than two
@@ -4110,12 +4956,6 @@ for (;; ptr++)
 
     memset(classbits, 0, 32 * sizeof(pcre_uint8));
 
-#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
-    xclass = FALSE;
-    class_uchardata = code + LINK_SIZE + 2;   /* For XCLASS items */
-    class_uchardata_base = class_uchardata;   /* Save the start */
-#endif
-
     /* Process characters until ] is reached. By writing this as a "do" it
     means that an initial ] is taken as a data character. At the start of the
     loop, c contains the first byte of the character. */
@@ -4138,10 +4978,11 @@ for (;; ptr++)
       (which is on the stack). We have to remember that there was XCLASS data,
       however. */
 
+      if (class_uchardata > class_uchardata_base) xclass = TRUE;
+
       if (lengthptr != NULL && class_uchardata > class_uchardata_base)
         {
-        xclass = TRUE;
-        *lengthptr += class_uchardata - class_uchardata_base;
+        *lengthptr += (int)(class_uchardata - class_uchardata_base);
         class_uchardata = class_uchardata_base;
         }
 #endif
@@ -4203,24 +5044,77 @@ for (;; ptr++)
           posix_class = 0;
 
         /* When PCRE_UCP is set, some of the POSIX classes are converted to
-        different escape sequences that use Unicode properties. */
+        different escape sequences that use Unicode properties \p or \P. Others
+        that are not available via \p or \P generate XCL_PROP/XCL_NOTPROP
+        directly. */
 
 #ifdef SUPPORT_UCP
         if ((options & PCRE_UCP) != 0)
           {
+          unsigned int ptype = 0;
           int pc = posix_class + ((local_negate)? POSIX_SUBSIZE/2 : 0);
+
+          /* The posix_substitutes table specifies which POSIX classes can be
+          converted to \p or \P items. */
+
           if (posix_substitutes[pc] != NULL)
             {
             nestptr = tempptr + 1;
             ptr = posix_substitutes[pc] - 1;
             continue;
             }
+
+          /* There are three other classes that generate special property calls
+          that are recognized only in an XCLASS. */
+
+          else switch(posix_class)
+            {
+            case PC_GRAPH:
+            ptype = PT_PXGRAPH;
+            /* Fall through */
+            case PC_PRINT:
+            if (ptype == 0) ptype = PT_PXPRINT;
+            /* Fall through */
+            case PC_PUNCT:
+            if (ptype == 0) ptype = PT_PXPUNCT;
+            *class_uchardata++ = local_negate? XCL_NOTPROP : XCL_PROP;
+            *class_uchardata++ = ptype;
+            *class_uchardata++ = 0;
+            xclass_has_prop = TRUE;
+            ptr = tempptr + 1;
+            continue;
+
+            /* For the other POSIX classes (ascii, cntrl, xdigit) we are going
+            to fall through to the non-UCP case and build a bit map for
+            characters with code points less than 256. If we are in a negated
+            POSIX class, characters with code points greater than 255 must
+            either all match or all not match. In the special case where we
+            have not yet generated any xclass data, and this is the final item
+            in the overall class, we need do nothing: later on, the opcode
+            OP_NCLASS will be used to indicate that characters greater than 255
+            are acceptable. If we have already seen an xclass item or one may
+            follow (we have to assume that it might if this is not the end of
+            the class), explicitly list all wide codepoints, which will then
+            either not match or match, depending on whether the class is or is
+            not negated. */
+
+            default:
+            if (local_negate &&
+                (xclass || tempptr[2] != CHAR_RIGHT_SQUARE_BRACKET))
+              {
+              *class_uchardata++ = XCL_RANGE;
+              class_uchardata += PRIV(ord2utf)(0x100, class_uchardata);
+              class_uchardata += PRIV(ord2utf)(0x10ffff, class_uchardata);
+              }
+            break;
+            }
           }
 #endif
-        /* In the non-UCP case, we build the bit map for the POSIX class in a
-        chunk of local store because we may be adding and subtracting from it,
-        and we don't want to subtract bits that may be in the main map already.
-        At the end we or the result into the bit map that is being built. */
+        /* In the non-UCP case, or when UCP makes no difference, we build the
+        bit map for the POSIX class in a chunk of local store because we may be
+        adding and subtracting from it, and we don't want to subtract bits that
+        may be in the main map already. At the end we or the result into the
+        bit map that is being built. */
 
         posix_class *= 3;
 
@@ -4337,21 +5231,20 @@ for (;; ptr++)
             for (c = 0; c < 32; c++) classbits[c] |= ~cbits[c+cbit_word];
             continue;
 
-            /* Perl 5.004 onwards omits VT from \s, but we must preserve it
-            if it was previously set by something earlier in the character
-            class. Luckily, the value of CHAR_VT is 0x0b in both ASCII and
-            EBCDIC, so we lazily just adjust the appropriate bit. */
+            /* Perl 5.004 onwards omitted VT from \s, but restored it at Perl
+            5.18. Before PCRE 8.34, we had to preserve the VT bit if it was
+            previously set by something earlier in the character class.
+            Luckily, the value of CHAR_VT is 0x0b in both ASCII and EBCDIC, so
+            we could just adjust the appropriate bit. From PCRE 8.34 we no
+            longer treat \s and \S specially. */
 
             case ESC_s:
-            classbits[0] |= cbits[cbit_space];
-            classbits[1] |= cbits[cbit_space+1] & ~0x08;
-            for (c = 2; c < 32; c++) classbits[c] |= cbits[c+cbit_space];
+            for (c = 0; c < 32; c++) classbits[c] |= cbits[c+cbit_space];
             continue;
 
             case ESC_S:
             should_flip_negation = TRUE;
             for (c = 0; c < 32; c++) classbits[c] |= ~cbits[c+cbit_space];
-            classbits[1] |= 0x08;    /* Perl 5.004 onwards omits VT from \s */
             continue;
 
             /* The rest apply in both UCP and non-UCP cases. */
@@ -4376,9 +5269,9 @@ for (;; ptr++)
               cd, PRIV(vspace_list));
             continue;
 
-#ifdef SUPPORT_UCP
             case ESC_p:
             case ESC_P:
+#ifdef SUPPORT_UCP
               {
               BOOL negated;
               unsigned int ptype = 0, pdata = 0;
@@ -4388,9 +5281,13 @@ for (;; ptr++)
                 XCL_PROP : XCL_NOTPROP;
               *class_uchardata++ = ptype;
               *class_uchardata++ = pdata;
+              xclass_has_prop = TRUE;
               class_has_8bitchar--;                /* Undo! */
               continue;
               }
+#else
+            *errorcodeptr = ERR45;
+            goto FAILED;
 #endif
             /* Unrecognized escapes are faulted if PCRE is running in its
             strict mode. By default, for compatibility with Perl, they are
@@ -4473,26 +5370,43 @@ for (;; ptr++)
 #endif
         d = *ptr;  /* Not UTF-8 mode */
 
-        /* The second part of a range can be a single-character escape, but
-        not any of the other escapes. Perl 5.6 treats a hyphen as a literal
-        in such circumstances. */
+        /* The second part of a range can be a single-character escape
+        sequence, but not any of the other escapes. Perl treats a hyphen as a
+        literal in such circumstances. However, in Perl's warning mode, a
+        warning is given, so PCRE now faults it as it is almost certainly a
+        mistake on the user's part. */
 
-        if (!inescq && d == CHAR_BACKSLASH)
+        if (!inescq)
           {
-          int descape;
-          descape = check_escape(&ptr, &d, errorcodeptr, cd->bracount, options, TRUE);
-          if (*errorcodeptr != 0) goto FAILED;
+          if (d == CHAR_BACKSLASH)
+            {
+            int descape;
+            descape = check_escape(&ptr, &d, errorcodeptr, cd->bracount, options, TRUE);
+            if (*errorcodeptr != 0) goto FAILED;
 
-          /* \b is backspace; any other special means the '-' was literal. */
+            /* 0 means a character was put into d; \b is backspace; any other
+            special causes an error. */
 
-          if (descape != 0)
-            {
-            if (descape == ESC_b) d = CHAR_BS; else
+            if (descape != 0)
               {
-              ptr = oldptr;
-              goto CLASS_SINGLE_CHARACTER;  /* A few lines below */
+              if (descape == ESC_b) d = CHAR_BS; else
+                {
+                *errorcodeptr = ERR83;
+                goto FAILED;
+                }
               }
             }
+
+          /* A hyphen followed by a POSIX class is treated in the same way. */
+
+          else if (d == CHAR_LEFT_SQUARE_BRACKET &&
+                   (ptr[1] == CHAR_COLON || ptr[1] == CHAR_DOT ||
+                    ptr[1] == CHAR_EQUALS_SIGN) &&
+                   check_posix_syntax(ptr, &tempptr))
+            {
+            *errorcodeptr = ERR83;
+            goto FAILED;
+            }
           }
 
         /* Check that the two values are in the correct order. Optimize
@@ -4530,16 +5444,20 @@ for (;; ptr++)
       CLASS_SINGLE_CHARACTER:
       if (class_one_char < 2) class_one_char++;
 
-      /* If class_one_char is 1, we have the first single character in the
-      class, and there have been no prior ranges, or XCLASS items generated by
-      escapes. If this is the final character in the class, we can optimize by
-      turning the item into a 1-character OP_CHAR[I] if it's positive, or
-      OP_NOT[I] if it's negative. In the positive case, it can cause firstchar
-      to be set. Otherwise, there can be no first char if this item is first,
-      whatever repeat count may follow. In the case of reqchar, save the
-      previous value for reinstating. */
+      /* If xclass_has_prop is false and class_one_char is 1, we have the first
+      single character in the class, and there have been no prior ranges, or
+      XCLASS items generated by escapes. If this is the final character in the
+      class, we can optimize by turning the item into a 1-character OP_CHAR[I]
+      if it's positive, or OP_NOT[I] if it's negative. In the positive case, it
+      can cause firstchar to be set. Otherwise, there can be no first char if
+      this item is first, whatever repeat count may follow. In the case of
+      reqchar, save the previous value for reinstating. */
 
-      if (class_one_char == 1 && ptr[1] == CHAR_RIGHT_SQUARE_BRACKET)
+      if (!inescq &&
+#ifdef SUPPORT_UCP
+          !xclass_has_prop &&
+#endif
+          class_one_char == 1 && ptr[1] == CHAR_RIGHT_SQUARE_BRACKET)
         {
         ptr++;
         zeroreqchar = reqchar;
@@ -4655,16 +5573,46 @@ for (;; ptr++)
     actual compiled code. */
 
 #ifdef SUPPORT_UTF
-    if (xclass && (!should_flip_negation || (options & PCRE_UCP) != 0))
+    if (xclass && (xclass_has_prop || !should_flip_negation ||
+        (options & PCRE_UCP) != 0))
 #elif !defined COMPILE_PCRE8
-    if (xclass && !should_flip_negation)
+    if (xclass && (xclass_has_prop || !should_flip_negation))
 #endif
 #if defined SUPPORT_UTF || !defined COMPILE_PCRE8
       {
+      /* For non-UCP wide characters, in a non-negative class containing \S or
+      similar (should_flip_negation is set), all characters greater than 255
+      must be in the class. */
+
+      if (
+#if defined COMPILE_PCRE8
+           utf &&
+#endif
+           should_flip_negation && !negate_class && (options & PCRE_UCP) == 0)
+        {
+        *class_uchardata++ = XCL_RANGE;
+        if (utf)   /* Will always be utf in the 8-bit library */
+          {
+          class_uchardata += PRIV(ord2utf)(0x100, class_uchardata);
+          class_uchardata += PRIV(ord2utf)(0x10ffff, class_uchardata);
+          }
+        else       /* Can only happen for the 16-bit & 32-bit libraries */
+          {
+#if defined COMPILE_PCRE16
+          *class_uchardata++ = 0x100;
+          *class_uchardata++ = 0xffffu;
+#elif defined COMPILE_PCRE32
+          *class_uchardata++ = 0x100;
+          *class_uchardata++ = 0xffffffffu;
+#endif
+          }
+        }
+
       *class_uchardata++ = XCL_END;    /* Marks the end of extra data */
       *code++ = OP_XCLASS;
       code += LINK_SIZE;
       *code = negate_class? XCL_NOT:0;
+      if (xclass_has_prop) *code |= XCL_HASPROP;
 
       /* If the map is required, move up the extra data to make room for it;
       otherwise just move the code pointer to the end of the extra data. */
@@ -4674,6 +5622,8 @@ for (;; ptr++)
         *code++ |= XCL_MAP;
         memmove(code + (32 / sizeof(pcre_uchar)), code,
           IN_UCHARS(class_uchardata - code));
+        if (negate_class && !xclass_has_prop)
+          for (c = 0; c < 32; c++) classbits[c] = ~classbits[c];
         memcpy(code, classbits, 32);
         code = class_uchardata + (32 / sizeof(pcre_uchar));
         }
@@ -4684,6 +5634,12 @@ for (;; ptr++)
       PUT(previous, 1, (int)(code - previous));
       break;   /* End of class handling */
       }
+
+    /* Even though any XCLASS list is now discarded, we must allow for
+    its memory. */
+
+    if (lengthptr != NULL)
+      *lengthptr += (int)(class_uchardata - class_uchardata_base);
 #endif
 
     /* If there are no characters > 255, or they are all to be included or
@@ -4756,6 +5712,34 @@ for (;; ptr++)
 
     tempcode = previous;
 
+    /* Before checking for a possessive quantifier, we must skip over
+    whitespace and comments in extended mode because Perl allows white space at
+    this point. */
+
+    if ((options & PCRE_EXTENDED) != 0)
+      {
+      const pcre_uchar *p = ptr + 1;
+      for (;;)
+        {
+        while (MAX_255(*p) && (cd->ctypes[*p] & ctype_space) != 0) p++;
+        if (*p != CHAR_NUMBER_SIGN) break;
+        p++;
+        while (*p != CHAR_NULL)
+          {
+          if (IS_NEWLINE(p))         /* For non-fixed-length newline cases, */
+            {                        /* IS_NEWLINE sets cd->nllen. */
+            p += cd->nllen;
+            break;
+            }
+          p++;
+#ifdef SUPPORT_UTF
+          if (utf) FORWARDCHAR(p);
+#endif
+          }           /* Loop for comment characters */
+        }             /* Loop for multiple comments */
+      ptr = p - 1;    /* Character before the next significant one. */
+      }
+
     /* If the next character is '+', we have a possessive quantifier. This
     implies greediness, whatever the setting of the PCRE_UNGREEDY option.
     If the next character is '?' this is a minimizing repeat, by default,
@@ -4850,19 +5834,6 @@ for (;; ptr++)
           }
         }
 
-      /* If the repetition is unlimited, it pays to see if the next thing on
-      the line is something that cannot possibly match this character. If so,
-      automatically possessifying this item gains some performance in the case
-      where the match fails. */
-
-      if (!possessive_quantifier &&
-          repeat_max < 0 &&
-          check_auto_possessive(previous, utf, ptr + 1, options, cd))
-        {
-        repeat_type = 0;    /* Force greedy */
-        possessive_quantifier = TRUE;
-        }
-
       goto OUTPUT_SINGLE_REPEAT;   /* Code shared with single character types */
       }
 
@@ -4880,14 +5851,6 @@ for (;; ptr++)
       op_type = OP_TYPESTAR - OP_STAR;  /* Use type opcodes */
       c = *previous;
 
-      if (!possessive_quantifier &&
-          repeat_max < 0 &&
-          check_auto_possessive(previous, utf, ptr + 1, options, cd))
-        {
-        repeat_type = 0;    /* Force greedy */
-        possessive_quantifier = TRUE;
-        }
-
       OUTPUT_SINGLE_REPEAT:
       if (*previous == OP_PROP || *previous == OP_NOTPROP)
         {
@@ -5036,13 +5999,12 @@ for (;; ptr++)
     /* If previous was a character class or a back reference, we put the repeat
     stuff after it, but just skip the item if the repeat was {0,0}. */
 
-    else if (*previous == OP_CLASS ||
-             *previous == OP_NCLASS ||
+    else if (*previous == OP_CLASS || *previous == OP_NCLASS ||
 #if defined SUPPORT_UTF || !defined COMPILE_PCRE8
              *previous == OP_XCLASS ||
 #endif
-             *previous == OP_REF ||
-             *previous == OP_REFI)
+             *previous == OP_REF   || *previous == OP_REFI ||
+             *previous == OP_DNREF || *previous == OP_DNREFI)
       {
       if (repeat_max == 0)
         {
@@ -5070,13 +6032,15 @@ for (;; ptr++)
     opcodes such as BRA and CBRA, as this is the place where they get converted
     into the more special varieties such as BRAPOS and SBRA. A test for >=
     OP_ASSERT and <= OP_COND includes ASSERT, ASSERT_NOT, ASSERTBACK,
-    ASSERTBACK_NOT, ONCE, BRA, CBRA, and COND. Originally, PCRE did not allow
-    repetition of assertions, but now it does, for Perl compatibility. */
+    ASSERTBACK_NOT, ONCE, ONCE_NC, BRA, BRAPOS, CBRA, CBRAPOS, and COND.
+    Originally, PCRE did not allow repetition of assertions, but now it does,
+    for Perl compatibility. */
 
     else if (*previous >= OP_ASSERT && *previous <= OP_COND)
       {
       register int i;
       int len = (int)(code - previous);
+      size_t base_hwm_offset = item_hwm_offset;
       pcre_uchar *bralink = NULL;
       pcre_uchar *brazeroptr = NULL;
 
@@ -5089,7 +6053,7 @@ for (;; ptr++)
       /* There is no sense in actually repeating assertions. The only potential
       use of repetition is in cases when the assertion is optional. Therefore,
       if the minimum is greater than zero, just ignore the repeat. If the
-      maximum is not not zero or one, set it to 1. */
+      maximum is not zero or one, set it to 1. */
 
       if (*previous < OP_ONCE)    /* Assertion */
         {
@@ -5131,7 +6095,7 @@ for (;; ptr++)
         if (repeat_max <= 1)    /* Covers 0, 1, and unlimited */
           {
           *code = OP_END;
-          adjust_recurse(previous, 1, utf, cd, save_hwm);
+          adjust_recurse(previous, 1, utf, cd, item_hwm_offset);
           memmove(previous + 1, previous, IN_UCHARS(len));
           code++;
           if (repeat_max == 0)
@@ -5155,7 +6119,7 @@ for (;; ptr++)
           {
           int offset;
           *code = OP_END;
-          adjust_recurse(previous, 2 + LINK_SIZE, utf, cd, save_hwm);
+          adjust_recurse(previous, 2 + LINK_SIZE, utf, cd, item_hwm_offset);
           memmove(previous + 2 + LINK_SIZE, previous, IN_UCHARS(len));
           code += 2 + LINK_SIZE;
           *previous++ = OP_BRAZERO + repeat_type;
@@ -5218,26 +6182,25 @@ for (;; ptr++)
             for (i = 1; i < repeat_min; i++)
               {
               pcre_uchar *hc;
-              pcre_uchar *this_hwm = cd->hwm;
+              size_t this_hwm_offset = cd->hwm - cd->start_workspace;
               memcpy(code, previous, IN_UCHARS(len));
 
               while (cd->hwm > cd->start_workspace + cd->workspace_size -
-                     WORK_SIZE_SAFETY_MARGIN - (this_hwm - save_hwm))
+                     WORK_SIZE_SAFETY_MARGIN -
+                     (this_hwm_offset - base_hwm_offset))
                 {
-                int save_offset = save_hwm - cd->start_workspace;
-                int this_offset = this_hwm - cd->start_workspace;
                 *errorcodeptr = expand_workspace(cd);
                 if (*errorcodeptr != 0) goto FAILED;
-                save_hwm = (pcre_uchar *)cd->start_workspace + save_offset;
-                this_hwm = (pcre_uchar *)cd->start_workspace + this_offset;
                 }
 
-              for (hc = save_hwm; hc < this_hwm; hc += LINK_SIZE)
+              for (hc = (pcre_uchar *)cd->start_workspace + base_hwm_offset;
+                   hc < (pcre_uchar *)cd->start_workspace + this_hwm_offset;
+                   hc += LINK_SIZE)
                 {
                 PUT(cd->hwm, 0, GET(hc, 0) + len);
                 cd->hwm += LINK_SIZE;
                 }
-              save_hwm = this_hwm;
+              base_hwm_offset = this_hwm_offset;
               code += len;
               }
             }
@@ -5282,7 +6245,7 @@ for (;; ptr++)
         else for (i = repeat_max - 1; i >= 0; i--)
           {
           pcre_uchar *hc;
-          pcre_uchar *this_hwm = cd->hwm;
+          size_t this_hwm_offset = cd->hwm - cd->start_workspace;
 
           *code++ = OP_BRAZERO + repeat_type;
 
@@ -5304,22 +6267,21 @@ for (;; ptr++)
           copying them. */
 
           while (cd->hwm > cd->start_workspace + cd->workspace_size -
-                 WORK_SIZE_SAFETY_MARGIN - (this_hwm - save_hwm))
+                 WORK_SIZE_SAFETY_MARGIN -
+                 (this_hwm_offset - base_hwm_offset))
             {
-            int save_offset = save_hwm - cd->start_workspace;
-            int this_offset = this_hwm - cd->start_workspace;
             *errorcodeptr = expand_workspace(cd);
             if (*errorcodeptr != 0) goto FAILED;
-            save_hwm = (pcre_uchar *)cd->start_workspace + save_offset;
-            this_hwm = (pcre_uchar *)cd->start_workspace + this_offset;
             }
 
-          for (hc = save_hwm; hc < this_hwm; hc += LINK_SIZE)
+          for (hc = (pcre_uchar *)cd->start_workspace + base_hwm_offset;
+               hc < (pcre_uchar *)cd->start_workspace + this_hwm_offset;
+               hc += LINK_SIZE)
             {
             PUT(cd->hwm, 0, GET(hc, 0) + len + ((i != 0)? 2+LINK_SIZE : 1));
             cd->hwm += LINK_SIZE;
             }
-          save_hwm = this_hwm;
+          base_hwm_offset = this_hwm_offset;
           code += len;
           }
 
@@ -5392,7 +6354,7 @@ for (;; ptr++)
             pcre_uchar *scode = bracode;
             do
               {
-              if (could_be_empty_branch(scode, ketcode, utf, cd))
+              if (could_be_empty_branch(scode, ketcode, utf, cd, NULL))
                 {
                 *bracode += OP_SBRA - OP_BRA;
                 break;
@@ -5402,6 +6364,12 @@ for (;; ptr++)
             while (*scode == OP_ALT);
             }
 
+          /* A conditional group with only one branch has an implicit empty
+          alternative branch. */
+
+          if (*bracode == OP_COND && bracode[GET(bracode,1)] != OP_ALT)
+            *bracode = OP_SCOND;
+
           /* Handle possessive quantifiers. */
 
           if (possessive_quantifier)
@@ -5415,11 +6383,11 @@ for (;; ptr++)
               {
               int nlen = (int)(code - bracode);
               *code = OP_END;
-              adjust_recurse(bracode, 1 + LINK_SIZE, utf, cd, save_hwm);
+              adjust_recurse(bracode, 1 + LINK_SIZE, utf, cd, item_hwm_offset);
               memmove(bracode + 1 + LINK_SIZE, bracode, IN_UCHARS(nlen));
               code += 1 + LINK_SIZE;
               nlen += 1 + LINK_SIZE;
-              *bracode = OP_BRAPOS;
+              *bracode = (*bracode == OP_COND)? OP_BRAPOS : OP_SBRAPOS;
               *code++ = OP_KETRPOS;
               PUTINC(code, 0, nlen);
               PUT(bracode, 1, nlen);
@@ -5462,43 +6430,105 @@ for (;; ptr++)
       goto FAILED;
       }
 
-    /* If the character following a repeat is '+', or if certain optimization
-    tests above succeeded, possessive_quantifier is TRUE. For some opcodes,
-    there are special alternative opcodes for this case. For anything else, we
-    wrap the entire repeated item inside OP_ONCE brackets. Logically, the '+'
-    notation is just syntactic sugar, taken from Sun's Java package, but the
-    special opcodes can optimize it.
+    /* If the character following a repeat is '+', possessive_quantifier is
+    TRUE. For some opcodes, there are special alternative opcodes for this
+    case. For anything else, we wrap the entire repeated item inside OP_ONCE
+    brackets. Logically, the '+' notation is just syntactic sugar, taken from
+    Sun's Java package, but the special opcodes can optimize it.
 
     Some (but not all) possessively repeated subpatterns have already been
     completely handled in the code just above. For them, possessive_quantifier
-    is always FALSE at this stage.
-
-    Note that the repeated item starts at tempcode, not at previous, which
-    might be the first part of a string whose (former) last char we repeated.
-
-    Possessifying an 'exact' quantifier has no effect, so we can ignore it. But
-    an 'upto' may follow. We skip over an 'exact' item, and then test the
-    length of what remains before proceeding. */
+    is always FALSE at this stage. Note that the repeated item starts at
+    tempcode, not at previous, which might be the first part of a string whose
+    (former) last char we repeated. */
 
     if (possessive_quantifier)
       {
       int len;
 
-      if (*tempcode == OP_TYPEEXACT)
+      /* Possessifying an EXACT quantifier has no effect, so we can ignore it.
+      However, QUERY, STAR, or UPTO may follow (for quantifiers such as {5,6},
+      {5,}, or {5,10}). We skip over an EXACT item; if the length of what
+      remains is greater than zero, there's a further opcode that can be
+      handled. If not, do nothing, leaving the EXACT alone. */
+
+      switch(*tempcode)
+        {
+        case OP_TYPEEXACT:
         tempcode += PRIV(OP_lengths)[*tempcode] +
           ((tempcode[1 + IMM2_SIZE] == OP_PROP
           || tempcode[1 + IMM2_SIZE] == OP_NOTPROP)? 2 : 0);
+        break;
 
-      else if (*tempcode == OP_EXACT || *tempcode == OP_NOTEXACT)
-        {
+        /* CHAR opcodes are used for exacts whose count is 1. */
+
+        case OP_CHAR:
+        case OP_CHARI:
+        case OP_NOT:
+        case OP_NOTI:
+        case OP_EXACT:
+        case OP_EXACTI:
+        case OP_NOTEXACT:
+        case OP_NOTEXACTI:
         tempcode += PRIV(OP_lengths)[*tempcode];
 #ifdef SUPPORT_UTF
         if (utf && HAS_EXTRALEN(tempcode[-1]))
           tempcode += GET_EXTRALEN(tempcode[-1]);
+#endif
+        break;
+
+        /* For the class opcodes, the repeat operator appears at the end;
+        adjust tempcode to point to it. */
+
+        case OP_CLASS:
+        case OP_NCLASS:
+        tempcode += 1 + 32/sizeof(pcre_uchar);
+        break;
+
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+        case OP_XCLASS:
+        tempcode += GET(tempcode, 1);
+        break;
 #endif
         }
 
+      /* If tempcode is equal to code (which points to the end of the repeated
+      item), it means we have skipped an EXACT item but there is no following
+      QUERY, STAR, or UPTO; the value of len will be 0, and we do nothing. In
+      all other cases, tempcode will be pointing to the repeat opcode, and will
+      be less than code, so the value of len will be greater than 0. */
+
       len = (int)(code - tempcode);
+      if (len > 0)
+        {
+        unsigned int repcode = *tempcode;
+
+        /* There is a table for possessifying opcodes, all of which are less
+        than OP_CALLOUT. A zero entry means there is no possessified version.
+        */
+
+        if (repcode < OP_CALLOUT && opcode_possessify[repcode] > 0)
+          *tempcode = opcode_possessify[repcode];
+
+        /* For opcode without a special possessified version, wrap the item in
+        ONCE brackets. Because we are moving code along, we must ensure that any
+        pending recursive references are updated. */
+
+        else
+          {
+          *code = OP_END;
+          adjust_recurse(tempcode, 1 + LINK_SIZE, utf, cd, item_hwm_offset);
+          memmove(tempcode + 1 + LINK_SIZE, tempcode, IN_UCHARS(len));
+          code += 1 + LINK_SIZE;
+          len += 1 + LINK_SIZE;
+          tempcode[0] = OP_ONCE;
+          *code++ = OP_KET;
+          PUTINC(code, 0, len);
+          PUT(tempcode, 1, len);
+          }
+        }
+
+#ifdef NEVER
       if (len > 0) switch (*tempcode)
         {
         case OP_STAR:  *tempcode = OP_POSSTAR; break;
@@ -5526,12 +6556,17 @@ for (;; ptr++)
         case OP_TYPEQUERY: *tempcode = OP_TYPEPOSQUERY; break;
         case OP_TYPEUPTO:  *tempcode = OP_TYPEPOSUPTO; break;
 
+        case OP_CRSTAR:   *tempcode = OP_CRPOSSTAR; break;
+        case OP_CRPLUS:   *tempcode = OP_CRPOSPLUS; break;
+        case OP_CRQUERY:  *tempcode = OP_CRPOSQUERY; break;
+        case OP_CRRANGE:  *tempcode = OP_CRPOSRANGE; break;
+
         /* Because we are moving code along, we must ensure that any
         pending recursive references are updated. */
 
         default:
         *code = OP_END;
-        adjust_recurse(tempcode, 1 + LINK_SIZE, utf, cd, save_hwm);
+        adjust_recurse(tempcode, 1 + LINK_SIZE, utf, cd, item_hwm_offset);
         memmove(tempcode + 1 + LINK_SIZE, tempcode, IN_UCHARS(len));
         code += 1 + LINK_SIZE;
         len += 1 + LINK_SIZE;
@@ -5541,6 +6576,7 @@ for (;; ptr++)
         PUT(tempcode, 1, len);
         break;
         }
+#endif
       }
 
     /* In all case we no longer have a previous item. We also set the
@@ -5559,15 +6595,10 @@ for (;; ptr++)
     parenthesis forms.  */
 
     case CHAR_LEFT_PARENTHESIS:
-    newoptions = options;
-    skipbytes = 0;
-    bravalue = OP_CBRA;
-    save_hwm = cd->hwm;
-    reset_bracount = FALSE;
+    ptr++;
 
-    /* First deal with various "verbs" that can be introduced by '*'. */
+    /* Now deal with various "verbs" that can be introduced by '*'. */
 
-    ptr++;
     if (ptr[0] == CHAR_ASTERISK && (ptr[1] == ':'
          || (MAX_255(ptr[1]) && ((cd->ctypes[ptr[1]] & ctype_letter) != 0))))
       {
@@ -5626,8 +6657,21 @@ for (;; ptr++)
             cd->had_accept = TRUE;
             for (oc = cd->open_caps; oc != NULL; oc = oc->next)
               {
-              *code++ = OP_CLOSE;
-              PUT2INC(code, 0, oc->number);
+              if (lengthptr != NULL)
+                {
+#ifdef COMPILE_PCRE8
+                *lengthptr += 1 + IMM2_SIZE;
+#elif defined COMPILE_PCRE16
+                *lengthptr += 2 + IMM2_SIZE;
+#elif defined COMPILE_PCRE32
+                *lengthptr += 4 + IMM2_SIZE;
+#endif
+                }
+              else
+                {
+                *code++ = OP_CLOSE;
+                PUT2INC(code, 0, oc->number);
+                }
               }
             setverb = *code++ =
               (cd->assert_depth > 0)? OP_ASSERT_ACCEPT : OP_ACCEPT;
@@ -5656,9 +6700,17 @@ for (;; ptr++)
               goto FAILED;
               }
             setverb = *code++ = verbs[i].op_arg;
-            *code++ = arglen;
-            memcpy(code, arg, IN_UCHARS(arglen));
-            code += arglen;
+            if (lengthptr != NULL)    /* In pass 1 just add in the length */
+              {                       /* to avoid potential workspace */
+              *lengthptr += arglen;   /* overflow. */
+              *code++ = 0;
+              }
+            else
+              {
+              *code++ = arglen;
+              memcpy(code, arg, IN_UCHARS(arglen));
+              code += arglen;
+              }
             *code++ = 0;
             }
 
@@ -5688,10 +6740,18 @@ for (;; ptr++)
       goto FAILED;
       }
 
+    /* Initialize for "real" parentheses */
+
+    newoptions = options;
+    skipbytes = 0;
+    bravalue = OP_CBRA;
+    item_hwm_offset = cd->hwm - cd->start_workspace;
+    reset_bracount = FALSE;
+
     /* Deal with the extended parentheses; all are introduced by '?', and the
     appearance of any of them means that this is not a capturing group. */
 
-    else if (*ptr == CHAR_QUESTION_MARK)
+    if (*ptr == CHAR_QUESTION_MARK)
       {
       int i, set, unset, namelen;
       int *optset;
@@ -5700,20 +6760,10 @@ for (;; ptr++)
 
       switch (*(++ptr))
         {
-        case CHAR_NUMBER_SIGN:                 /* Comment; skip to ket */
-        ptr++;
-        while (*ptr != CHAR_NULL && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++;
-        if (*ptr == CHAR_NULL)
-          {
-          *errorcodeptr = ERR18;
-          goto FAILED;
-          }
-        continue;
-
-
         /* ------------------------------------------------------------ */
         case CHAR_VERTICAL_LINE:  /* Reset capture count for each branch */
         reset_bracount = TRUE;
+        cd->dupgroups = TRUE;     /* Record (?| encountered */
         /* Fall through */
 
         /* ------------------------------------------------------------ */
@@ -5729,17 +6779,16 @@ for (;; ptr++)
         tempptr = ptr;
 
         /* A condition can be an assertion, a number (referring to a numbered
-        group), a name (referring to a named group), or 'R', referring to
-        recursion. R<digits> and R&name are also permitted for recursion tests.
+        group's having been set), a name (referring to a named group), or 'R',
+        referring to recursion. R<digits> and R&name are also permitted for
+        recursion tests.
 
-        There are several syntaxes for testing a named group: (?(name)) is used
-        by Python; Perl 5.10 onwards uses (?(<name>) or (?('name')).
+        There are ways of testing a named group: (?(name)) is used by Python;
+        Perl 5.10 onwards uses (?(<name>) or (?('name')).
 
-        There are two unfortunate ambiguities, caused by history. (a) 'R' can
-        be the recursive thing or the name 'R' (and similarly for 'R' followed
-        by digits), and (b) a number could be a name that consists of digits.
-        In both cases, we look for a name first; if not found, we try the other
-        cases.
+        There is one unfortunate ambiguity, caused by history. 'R' can be the
+        recursive thing or the name 'R' (and similarly for 'R' followed by
+        digits). We look for a name first; if not found, we try the other case.
 
         For compatibility with auto-callouts, we allow a callout to be
         specified before a condition that is an assertion. First, check for the
@@ -5751,6 +6800,15 @@ for (;; ptr++)
           for (i = 3;; i++) if (!IS_DIGIT(ptr[i])) break;
           if (ptr[i] == CHAR_RIGHT_PARENTHESIS)
             tempptr += i + 1;
+
+          /* tempptr should now be pointing to the opening parenthesis of the
+          assertion condition. */
+
+          if (*tempptr != CHAR_LEFT_PARENTHESIS)
+            {
+            *errorcodeptr = ERR28;
+            goto FAILED;
+            }
           }
 
         /* For conditions that are assertions, check the syntax, and then exit
@@ -5760,19 +6818,28 @@ for (;; ptr++)
         if (tempptr[1] == CHAR_QUESTION_MARK &&
               (tempptr[2] == CHAR_EQUALS_SIGN ||
                tempptr[2] == CHAR_EXCLAMATION_MARK ||
-               tempptr[2] == CHAR_LESS_THAN_SIGN))
+                 (tempptr[2] == CHAR_LESS_THAN_SIGN &&
+                   (tempptr[3] == CHAR_EQUALS_SIGN ||
+                    tempptr[3] == CHAR_EXCLAMATION_MARK))))
+          {
+          cd->iscondassert = TRUE;
           break;
+          }
 
-        /* Most other conditions use OP_CREF (a couple change to OP_RREF
-        below), and all need to skip 1+IMM2_SIZE bytes at the start of the group. */
+        /* Other conditions use OP_CREF/OP_DNCREF/OP_RREF/OP_DNRREF, and all
+        need to skip at least 1+IMM2_SIZE bytes at the start of the group. */
 
         code[1+LINK_SIZE] = OP_CREF;
         skipbytes = 1+IMM2_SIZE;
-        refsign = -1;
+        refsign = -1;     /* => not a number */
+        namelen = -1;     /* => not a name; must set to avoid warning */
+        name = NULL;      /* Always set to avoid warning */
+        recno = 0;        /* Always set to avoid warning */
 
         /* Check for a test for recursion in a named group. */
 
-        if (ptr[1] == CHAR_R && ptr[2] == CHAR_AMPERSAND)
+        ptr++;
+        if (*ptr == CHAR_R && ptr[1] == CHAR_AMPERSAND)
           {
           terminator = -1;
           ptr += 2;
@@ -5780,14 +6847,15 @@ for (;; ptr++)
           }
 
         /* Check for a test for a named group's having been set, using the Perl
-        syntax (?(<name>) or (?('name') */
+        syntax (?(<name>) or (?('name'), and also allow for the original PCRE
+        syntax of (?(name) or for (?(+n), (?(-n), and just (?(n). */
 
-        else if (ptr[1] == CHAR_LESS_THAN_SIGN)
+        else if (*ptr == CHAR_LESS_THAN_SIGN)
           {
           terminator = CHAR_GREATER_THAN_SIGN;
           ptr++;
           }
-        else if (ptr[1] == CHAR_APOSTROPHE)
+        else if (*ptr == CHAR_APOSTROPHE)
           {
           terminator = CHAR_APOSTROPHE;
           ptr++;
@@ -5795,35 +6863,60 @@ for (;; ptr++)
         else
           {
           terminator = CHAR_NULL;
-          if (ptr[1] == CHAR_MINUS || ptr[1] == CHAR_PLUS) refsign = *(++ptr);
+          if (*ptr == CHAR_MINUS || *ptr == CHAR_PLUS) refsign = *ptr++;
+            else if (IS_DIGIT(*ptr)) refsign = 0;
+          }
+
+        /* Handle a number */
+
+        if (refsign >= 0)
+          {
+          while (IS_DIGIT(*ptr))
+            {
+            if (recno > INT_MAX / 10 - 1)  /* Integer overflow */
+              {
+              while (IS_DIGIT(*ptr)) ptr++;
+              *errorcodeptr = ERR61;
+              goto FAILED;
+              }
+            recno = recno * 10 + (int)(*ptr - CHAR_0);
+            ptr++;
+            }
           }
 
-        /* We now expect to read a name; any thing else is an error */
+        /* Otherwise we expect to read a name; anything else is an error. When
+        a name is one of a number of duplicates, a different opcode is used and
+        it needs more memory. Unfortunately we cannot tell whether a name is a
+        duplicate in the first pass, so we have to allow for more memory. */
 
-        if (!MAX_255(ptr[1]) || (cd->ctypes[ptr[1]] & ctype_word) == 0)
+        else
           {
-          ptr += 1;  /* To get the right offset */
-          *errorcodeptr = ERR28;
-          goto FAILED;
+          if (IS_DIGIT(*ptr))
+            {
+            *errorcodeptr = ERR84;
+            goto FAILED;
+            }
+          if (!MAX_255(*ptr) || (cd->ctypes[*ptr] & ctype_word) == 0)
+            {
+            *errorcodeptr = ERR28;   /* Assertion expected */
+            goto FAILED;
+            }
+          name = ptr++;
+          while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_word) != 0)
+            {
+            ptr++;
+            }
+          namelen = (int)(ptr - name);
+          if (lengthptr != NULL) skipbytes += IMM2_SIZE;
           }
 
-        /* Read the name, but also get it as a number if it's all digits */
-
-        recno = 0;
-        name = ++ptr;
-        while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_word) != 0)
-          {
-          if (recno >= 0)
-            recno = (IS_DIGIT(*ptr))? recno * 10 + (int)(*ptr - CHAR_0) : -1;
-          ptr++;
-          }
-        namelen = (int)(ptr - name);
+        /* Check the terminator */
 
         if ((terminator > 0 && *ptr++ != (pcre_uchar)terminator) ||
             *ptr++ != CHAR_RIGHT_PARENTHESIS)
           {
-          ptr--;      /* Error offset */
-          *errorcodeptr = ERR26;
+          ptr--;                  /* Error offset */
+          *errorcodeptr = ERR26;  /* Malformed number or name */
           goto FAILED;
           }
 
@@ -5832,63 +6925,75 @@ for (;; ptr++)
         if (lengthptr != NULL) break;
 
         /* In the real compile we do the work of looking for the actual
-        reference. If the string started with "+" or "-" we require the rest to
-        be digits, in which case recno will be set. */
+        reference. If refsign is not negative, it means we have a number in
+        recno. */
 
-        if (refsign > 0)
+        if (refsign >= 0)
           {
           if (recno <= 0)
             {
-            *errorcodeptr = ERR58;
+            *errorcodeptr = ERR35;
             goto FAILED;
             }
-          recno = (refsign == CHAR_MINUS)?
-            cd->bracount - recno + 1 : recno +cd->bracount;
+          if (refsign != 0) recno = (refsign == CHAR_MINUS)?
+            cd->bracount - recno + 1 : recno + cd->bracount;
           if (recno <= 0 || recno > cd->final_bracount)
             {
             *errorcodeptr = ERR15;
             goto FAILED;
             }
           PUT2(code, 2+LINK_SIZE, recno);
+          if (recno > cd->top_backref) cd->top_backref = recno;
           break;
           }
 
-        /* Otherwise (did not start with "+" or "-"), start by looking for the
-        name. If we find a name, add one to the opcode to change OP_CREF or
-        OP_RREF into OP_NCREF or OP_NRREF. These behave exactly the same,
-        except they record that the reference was originally to a name. The
-        information is used to check duplicate names. */
+        /* Otherwise look for the name. */
 
         slot = cd->name_table;
         for (i = 0; i < cd->names_found; i++)
           {
-          if (STRNCMP_UC_UC(name, slot+IMM2_SIZE, namelen) == 0) break;
+          if (STRNCMP_UC_UC(name, slot+IMM2_SIZE, namelen) == 0 &&
+            slot[IMM2_SIZE+namelen] == 0) break;
           slot += cd->name_entry_size;
           }
 
-        /* Found a previous named subpattern */
+        /* Found the named subpattern. If the name is duplicated, add one to
+        the opcode to change CREF/RREF into DNCREF/DNRREF and insert
+        appropriate data values. Otherwise, just insert the unique subpattern
+        number. */
 
         if (i < cd->names_found)
           {
-          recno = GET2(slot, 0);
-          PUT2(code, 2+LINK_SIZE, recno);
-          code[1+LINK_SIZE]++;
-          }
-
-        /* Search the pattern for a forward reference */
+          int offset = i++;
+          int count = 1;
+          recno = GET2(slot, 0);   /* Number from first found */
+          if (recno > cd->top_backref) cd->top_backref = recno;
+          for (; i < cd->names_found; i++)
+            {
+            slot += cd->name_entry_size;
+            if (STRNCMP_UC_UC(name, slot+IMM2_SIZE, namelen) != 0 ||
+              (slot+IMM2_SIZE)[namelen] != 0) break;
+            count++;
+            }
 
-        else if ((i = find_parens(cd, name, namelen,
-                        (options & PCRE_EXTENDED) != 0, utf)) > 0)
-          {
-          PUT2(code, 2+LINK_SIZE, i);
-          code[1+LINK_SIZE]++;
+          if (count > 1)
+            {
+            PUT2(code, 2+LINK_SIZE, offset);
+            PUT2(code, 2+LINK_SIZE+IMM2_SIZE, count);
+            skipbytes += IMM2_SIZE;
+            code[1+LINK_SIZE]++;
+            }
+          else  /* Not a duplicated name */
+            {
+            PUT2(code, 2+LINK_SIZE, recno);
+            }
           }
 
         /* If terminator == CHAR_NULL it means that the name followed directly
         after the opening parenthesis [e.g. (?(abc)...] and in this case there
         are some further alternatives to try. For the cases where terminator !=
-        0 [things like (?(<name>... or (?('name')... or (?(R&name)... ] we have
-        now checked all the possibilities, so give an error. */
+        CHAR_NULL [things like (?(<name>... or (?('name')... or (?(R&name)... ]
+        we have now checked all the possibilities, so give an error. */
 
         else if (terminator != CHAR_NULL)
           {
@@ -5909,6 +7014,11 @@ for (;; ptr++)
               *errorcodeptr = ERR15;
               goto FAILED;
               }
+            if (recno > INT_MAX / 10 - 1)   /* Integer overflow */
+              {
+              *errorcodeptr = ERR61;
+              goto FAILED;
+              }
             recno = recno * 10 + name[i] - CHAR_0;
             }
           if (recno == 0) recno = RREF_ANY;
@@ -5925,19 +7035,11 @@ for (;; ptr++)
           skipbytes = 1;
           }
 
-        /* Check for the "name" actually being a subpattern number. We are
-        in the second pass here, so final_bracount is set. */
-
-        else if (recno > 0 && recno <= cd->final_bracount)
-          {
-          PUT2(code, 2+LINK_SIZE, recno);
-          }
-
-        /* Either an unidentified subpattern, or a reference to (?(0) */
+        /* Reference to an unidentified subpattern. */
 
         else
           {
-          *errorcodeptr = (recno == 0)? ERR35: ERR15;
+          *errorcodeptr = ERR15;
           goto FAILED;
           }
         break;
@@ -5950,11 +7052,18 @@ for (;; ptr++)
         ptr++;
         break;
 
+        /* Optimize (?!) to (*FAIL) unless it is quantified - which is a weird
+        thing to do, but Perl allows all assertions to be quantified, and when
+        they contain capturing parentheses there may be a potential use for
+        this feature. Not that that applies to a quantified (?!) but we allow
+        it for uniformity. */
 
         /* ------------------------------------------------------------ */
         case CHAR_EXCLAMATION_MARK:            /* Negative lookahead */
         ptr++;
-        if (*ptr == CHAR_RIGHT_PARENTHESIS)    /* Optimize (?!) */
+        if (*ptr == CHAR_RIGHT_PARENTHESIS && ptr[1] != CHAR_ASTERISK &&
+             ptr[1] != CHAR_PLUS && ptr[1] != CHAR_QUESTION_MARK &&
+            (ptr[1] != CHAR_LEFT_CURLY_BRACKET || !is_counted_repeat(ptr+2)))
           {
           *code++ = OP_FAIL;
           previous = NULL;
@@ -6047,124 +7156,110 @@ for (;; ptr++)
         /* ------------------------------------------------------------ */
         DEFINE_NAME:    /* Come here from (?< handling */
         case CHAR_APOSTROPHE:
+        terminator = (*ptr == CHAR_LESS_THAN_SIGN)?
+          CHAR_GREATER_THAN_SIGN : CHAR_APOSTROPHE;
+        name = ++ptr;
+        if (IS_DIGIT(*ptr))
           {
-          terminator = (*ptr == CHAR_LESS_THAN_SIGN)?
-            CHAR_GREATER_THAN_SIGN : CHAR_APOSTROPHE;
-          name = ++ptr;
+          *errorcodeptr = ERR84;   /* Group name must start with non-digit */
+          goto FAILED;
+          }
+        while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_word) != 0) ptr++;
+        namelen = (int)(ptr - name);
 
-          while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_word) != 0) ptr++;
-          namelen = (int)(ptr - name);
+        /* In the pre-compile phase, do a syntax check, remember the longest
+        name, and then remember the group in a vector, expanding it if
+        necessary. Duplicates for the same number are skipped; other duplicates
+        are checked for validity. In the actual compile, there is nothing to
+        do. */
+
+        if (lengthptr != NULL)
+          {
+          named_group *ng;
+          pcre_uint32 number = cd->bracount + 1;
 
-          /* In the pre-compile phase, just do a syntax check. */
+          if (*ptr != (pcre_uchar)terminator)
+            {
+            *errorcodeptr = ERR42;
+            goto FAILED;
+            }
 
-          if (lengthptr != NULL)
+          if (cd->names_found >= MAX_NAME_COUNT)
             {
-            if (*ptr != (pcre_uchar)terminator)
-              {
-              *errorcodeptr = ERR42;
-              goto FAILED;
-              }
-            if (cd->names_found >= MAX_NAME_COUNT)
+            *errorcodeptr = ERR49;
+            goto FAILED;
+            }
+
+          if (namelen + IMM2_SIZE + 1 > cd->name_entry_size)
+            {
+            cd->name_entry_size = namelen + IMM2_SIZE + 1;
+            if (namelen > MAX_NAME_SIZE)
               {
-              *errorcodeptr = ERR49;
+              *errorcodeptr = ERR48;
               goto FAILED;
               }
-            if (namelen + IMM2_SIZE + 1 > cd->name_entry_size)
+            }
+
+          /* Scan the list to check for duplicates. For duplicate names, if the
+          number is the same, break the loop, which causes the name to be
+          discarded; otherwise, if DUPNAMES is not set, give an error.
+          If it is set, allow the name with a different number, but continue
+          scanning in case this is a duplicate with the same number. For
+          non-duplicate names, give an error if the number is duplicated. */
+
+          ng = cd->named_groups;
+          for (i = 0; i < cd->names_found; i++, ng++)
+            {
+            if (namelen == ng->length &&
+                STRNCMP_UC_UC(name, ng->name, namelen) == 0)
               {
-              cd->name_entry_size = namelen + IMM2_SIZE + 1;
-              if (namelen > MAX_NAME_SIZE)
+              if (ng->number == number) break;
+              if ((options & PCRE_DUPNAMES) == 0)
                 {
-                *errorcodeptr = ERR48;
+                *errorcodeptr = ERR43;
                 goto FAILED;
                 }
+              cd->dupnames = TRUE;  /* Duplicate names exist */
+              }
+            else if (ng->number == number)
+              {
+              *errorcodeptr = ERR65;
+              goto FAILED;
               }
             }
 
-          /* In the real compile, create the entry in the table, maintaining
-          alphabetical order. Duplicate names for different numbers are
-          permitted only if PCRE_DUPNAMES is set. Duplicate names for the same
-          number are always OK. (An existing number can be re-used if (?|
-          appears in the pattern.) In either event, a duplicate name results in
-          a duplicate entry in the table, even if the number is the same. This
-          is because the number of names, and hence the table size, is computed
-          in the pre-compile, and it affects various numbers and pointers which
-          would all have to be modified, and the compiled code moved down, if
-          duplicates with the same number were omitted from the table. This
-          doesn't seem worth the hassle. However, *different* names for the
-          same number are not permitted. */
-
-          else
+          if (i >= cd->names_found)     /* Not a duplicate with same number */
             {
-            BOOL dupname = FALSE;
-            slot = cd->name_table;
+            /* Increase the list size if necessary */
 
-            for (i = 0; i < cd->names_found; i++)
+            if (cd->names_found >= cd->named_group_list_size)
               {
-              int crc = memcmp(name, slot+IMM2_SIZE, IN_UCHARS(namelen));
-              if (crc == 0)
-                {
-                if (slot[IMM2_SIZE+namelen] == 0)
-                  {
-                  if (GET2(slot, 0) != cd->bracount + 1 &&
-                      (options & PCRE_DUPNAMES) == 0)
-                    {
-                    *errorcodeptr = ERR43;
-                    goto FAILED;
-                    }
-                  else dupname = TRUE;
-                  }
-                else crc = -1;      /* Current name is a substring */
-                }
+              int newsize = cd->named_group_list_size * 2;
+              named_group *newspace = (PUBL(malloc))
+                (newsize * sizeof(named_group));
 
-              /* Make space in the table and break the loop for an earlier
-              name. For a duplicate or later name, carry on. We do this for
-              duplicates so that in the simple case (when ?(| is not used) they
-              are in order of their numbers. */
-
-              if (crc < 0)
+              if (newspace == NULL)
                 {
-                memmove(slot + cd->name_entry_size, slot,
-                  IN_UCHARS((cd->names_found - i) * cd->name_entry_size));
-                break;
+                *errorcodeptr = ERR21;
+                goto FAILED;
                 }
 
-              /* Continue the loop for a later or duplicate name */
-
-              slot += cd->name_entry_size;
-              }
-
-            /* For non-duplicate names, check for a duplicate number before
-            adding the new name. */
-
-            if (!dupname)
-              {
-              pcre_uchar *cslot = cd->name_table;
-              for (i = 0; i < cd->names_found; i++)
-                {
-                if (cslot != slot)
-                  {
-                  if (GET2(cslot, 0) == cd->bracount + 1)
-                    {
-                    *errorcodeptr = ERR65;
-                    goto FAILED;
-                    }
-                  }
-                else i--;
-                cslot += cd->name_entry_size;
-                }
+              memcpy(newspace, cd->named_groups,
+                cd->named_group_list_size * sizeof(named_group));
+              if (cd->named_group_list_size > NAMED_GROUP_LIST_SIZE)
+                (PUBL(free))((void *)cd->named_groups);
+              cd->named_groups = newspace;
+              cd->named_group_list_size = newsize;
               }
 
-            PUT2(slot, 0, cd->bracount + 1);
-            memcpy(slot + IMM2_SIZE, name, IN_UCHARS(namelen));
-            slot[IMM2_SIZE + namelen] = 0;
+            cd->named_groups[cd->names_found].name = name;
+            cd->named_groups[cd->names_found].length = namelen;
+            cd->named_groups[cd->names_found].number = number;
+            cd->names_found++;
             }
           }
 
-        /* In both pre-compile and compile, count the number of names we've
-        encountered. */
-
-        cd->names_found++;
-        ptr++;                    /* Move past > or ' */
+        ptr++;                    /* Move past > or ' in both passes. */
         goto NUMBERED_GROUP;
 
 
@@ -6182,6 +7277,11 @@ for (;; ptr++)
 
         NAMED_REF_OR_RECURSE:
         name = ++ptr;
+        if (IS_DIGIT(*ptr))
+          {
+          *errorcodeptr = ERR84;   /* Group name must start with non-digit */
+          goto FAILED;
+          }
         while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_word) != 0) ptr++;
         namelen = (int)(ptr - name);
 
@@ -6194,7 +7294,8 @@ for (;; ptr++)
 
         if (lengthptr != NULL)
           {
-          const pcre_uchar *temp;
+          named_group *ng;
+          recno = 0;
 
           if (namelen == 0)
             {
@@ -6212,27 +7313,76 @@ for (;; ptr++)
             goto FAILED;
             }
 
-          /* The name table does not exist in the first pass, so we cannot
-          do a simple search as in the code below. Instead, we have to scan the
-          pattern to find the number. It is important that we scan it only as
-          far as we have got because the syntax of named subpatterns has not
-          been checked for the rest of the pattern, and find_parens() assumes
-          correct syntax. In any case, it's a waste of resources to scan
-          further. We stop the scan at the current point by temporarily
-          adjusting the value of cd->endpattern. */
-
-          temp = cd->end_pattern;
-          cd->end_pattern = ptr;
-          recno = find_parens(cd, name, namelen,
-            (options & PCRE_EXTENDED) != 0, utf);
-          cd->end_pattern = temp;
-          if (recno < 0) recno = 0;    /* Forward ref; set dummy number */
+          /* Count named back references. */
+
+          if (!is_recurse) cd->namedrefcount++;
+
+          /* We have to allow for a named reference to a duplicated name (this
+          cannot be determined until the second pass). This needs an extra
+          16-bit data item. */
+
+          *lengthptr += IMM2_SIZE;
+
+          /* If this is a forward reference and we are within a (?|...) group,
+          the reference may end up as the number of a group which we are
+          currently inside, that is, it could be a recursive reference. In the
+          real compile this will be picked up and the reference wrapped with
+          OP_ONCE to make it atomic, so we must space in case this occurs. */
+
+          /* In fact, this can happen for a non-forward reference because
+          another group with the same number might be created later. This
+          issue is fixed "properly" in PCRE2. As PCRE1 is now in maintenance
+          only mode, we finesse the bug by allowing more memory always. */
+
+          *lengthptr += 4 + 4*LINK_SIZE;
+
+          /* It is even worse than that. The current reference may be to an
+          existing named group with a different number (so apparently not
+          recursive) but which later on is also attached to a group with the
+          current number. This can only happen if $(| has been previous
+          encountered. In that case, we allow yet more memory, just in case.
+          (Again, this is fixed "properly" in PCRE2. */
+
+          if (cd->dupgroups) *lengthptr += 4 + 4*LINK_SIZE;
+
+          /* Otherwise, check for recursion here. The name table does not exist
+          in the first pass; instead we must scan the list of names encountered
+          so far in order to get the number. If the name is not found, leave
+          the value of recno as 0 for a forward reference. */
+
+          /* This patch (removing "else") fixes a problem when a reference is
+          to multiple identically named nested groups from within the nest.
+          Once again, it is not the "proper" fix, and it results in an
+          over-allocation of memory. */
+
+          /* else */
+            {
+            ng = cd->named_groups;
+            for (i = 0; i < cd->names_found; i++, ng++)
+              {
+              if (namelen == ng->length &&
+                  STRNCMP_UC_UC(name, ng->name, namelen) == 0)
+                {
+                open_capitem *oc;
+                recno = ng->number;
+                if (is_recurse) break;
+                for (oc = cd->open_caps; oc != NULL; oc = oc->next)
+                  {
+                  if (oc->number == recno)
+                    {
+                    oc->flag = TRUE;
+                    break;
+                    }
+                  }
+                }
+              }
+            }
           }
 
-        /* In the real compile, seek the name in the table. We check the name
+        /* In the real compile, search the name table. We check the name
         first, and then check that we have reached the end of the name in the
-        table. That way, if the name that is longer than any in the table,
-        the comparison will fail without reading beyond the table entry. */
+        table. That way, if the name is longer than any in the table, the
+        comparison will fail without reading beyond the table entry. */
 
         else
           {
@@ -6245,30 +7395,88 @@ for (;; ptr++)
             slot += cd->name_entry_size;
             }
 
-          if (i < cd->names_found)         /* Back reference */
+          if (i < cd->names_found)
             {
             recno = GET2(slot, 0);
             }
-          else if ((recno =                /* Forward back reference */
-                    find_parens(cd, name, namelen,
-                      (options & PCRE_EXTENDED) != 0, utf)) <= 0)
+          else
             {
             *errorcodeptr = ERR15;
             goto FAILED;
             }
           }
 
-        /* In both phases, we can now go to the code than handles numerical
-        recursion or backreferences. */
+        /* In both phases, for recursions, we can now go to the code than
+        handles numerical recursion. */
 
         if (is_recurse) goto HANDLE_RECURSION;
-          else goto HANDLE_REFERENCE;
+
+        /* In the second pass we must see if the name is duplicated. If so, we
+        generate a different opcode. */
+
+        if (lengthptr == NULL && cd->dupnames)
+          {
+          int count = 1;
+          unsigned int index = i;
+          pcre_uchar *cslot = slot + cd->name_entry_size;
+
+          for (i++; i < cd->names_found; i++)
+            {
+            if (STRCMP_UC_UC(slot + IMM2_SIZE, cslot + IMM2_SIZE) != 0) break;
+            count++;
+            cslot += cd->name_entry_size;
+            }
+
+          if (count > 1)
+            {
+            if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE;
+            previous = code;
+            item_hwm_offset = cd->hwm - cd->start_workspace;
+            *code++ = ((options & PCRE_CASELESS) != 0)? OP_DNREFI : OP_DNREF;
+            PUT2INC(code, 0, index);
+            PUT2INC(code, 0, count);
+
+            /* Process each potentially referenced group. */
+
+            for (; slot < cslot; slot += cd->name_entry_size)
+              {
+              open_capitem *oc;
+              recno = GET2(slot, 0);
+              cd->backref_map |= (recno < 32)? (1 << recno) : 1;
+              if (recno > cd->top_backref) cd->top_backref = recno;
+
+              /* Check to see if this back reference is recursive, that it, it
+              is inside the group that it references. A flag is set so that the
+              group can be made atomic. */
+
+              for (oc = cd->open_caps; oc != NULL; oc = oc->next)
+                {
+                if (oc->number == recno)
+                  {
+                  oc->flag = TRUE;
+                  break;
+                  }
+                }
+              }
+
+            continue;  /* End of back ref handling */
+            }
+          }
+
+        /* First pass, or a non-duplicated name. */
+
+        goto HANDLE_REFERENCE;
 
 
         /* ------------------------------------------------------------ */
-        case CHAR_R:              /* Recursion */
-        ptr++;                    /* Same as (?0)      */
-        /* Fall through */
+        case CHAR_R:              /* Recursion, same as (?0) */
+        recno = 0;
+        if (*(++ptr) != CHAR_RIGHT_PARENTHESIS)
+          {
+          *errorcodeptr = ERR29;
+          goto FAILED;
+          }
+        goto HANDLE_RECURSION;
 
 
         /* ------------------------------------------------------------ */
@@ -6305,7 +7513,15 @@ for (;; ptr++)
 
           recno = 0;
           while(IS_DIGIT(*ptr))
+            {
+            if (recno > INT_MAX / 10 - 1) /* Integer overflow */
+              {
+              while (IS_DIGIT(*ptr)) ptr++;
+              *errorcodeptr = ERR61;
+              goto FAILED;
+              }
             recno = recno * 10 + *ptr++ - CHAR_0;
+            }
 
           if (*ptr != (pcre_uchar)terminator)
             {
@@ -6342,6 +7558,7 @@ for (;; ptr++)
           HANDLE_RECURSION:
 
           previous = code;
+          item_hwm_offset = cd->hwm - cd->start_workspace;
           called = cd->start_code;
 
           /* When we are actually compiling, find the bracket that is being
@@ -6361,8 +7578,7 @@ for (;; ptr++)
 
             if (called == NULL)
               {
-              if (find_parens(cd, NULL, recno,
-                    (options & PCRE_EXTENDED) != 0, utf) < 0)
+              if (recno > cd->final_bracount)
                 {
                 *errorcodeptr = ERR15;
                 goto FAILED;
@@ -6450,39 +7666,15 @@ for (;; ptr++)
         newoptions = (options | set) & (~unset);
 
         /* If the options ended with ')' this is not the start of a nested
-        group with option changes, so the options change at this level. If this
-        item is right at the start of the pattern, the options can be
-        abstracted and made external in the pre-compile phase, and ignored in
-        the compile phase. This can be helpful when matching -- for instance in
-        caseless checking of required bytes.
-
-        If the code pointer is not (cd->start_code + 1 + LINK_SIZE), we are
-        definitely *not* at the start of the pattern because something has been
-        compiled. In the pre-compile phase, however, the code pointer can have
-        that value after the start, because it gets reset as code is discarded
-        during the pre-compile. However, this can happen only at top level - if
-        we are within parentheses, the starting BRA will still be present. At
-        any parenthesis level, the length value can be used to test if anything
-        has been compiled at that level. Thus, a test for both these conditions
-        is necessary to ensure we correctly detect the start of the pattern in
-        both phases.
-
+        group with option changes, so the options change at this level.
         If we are not at the pattern start, reset the greedy defaults and the
         case value for firstchar and reqchar. */
 
         if (*ptr == CHAR_RIGHT_PARENTHESIS)
           {
-          if (code == cd->start_code + 1 + LINK_SIZE &&
-               (lengthptr == NULL || *lengthptr == 2 + 2*LINK_SIZE))
-            {
-            cd->external_options = newoptions;
-            }
-          else
-            {
-            greedy_default = ((newoptions & PCRE_UNGREEDY) != 0);
-            greedy_non_default = greedy_default ^ 1;
-            req_caseopt = ((newoptions & PCRE_CASELESS) != 0)? REQ_CASELESS:0;
-            }
+          greedy_default = ((newoptions & PCRE_UNGREEDY) != 0);
+          greedy_non_default = greedy_default ^ 1;
+          req_caseopt = ((newoptions & PCRE_CASELESS) != 0)? REQ_CASELESS:0;
 
           /* Change options at this level, and pass them back for use
           in subsequent branches. */
@@ -6521,12 +7713,35 @@ for (;; ptr++)
       skipbytes = IMM2_SIZE;
       }
 
-    /* Process nested bracketed regex. Assertions used not to be repeatable,
-    but this was changed for Perl compatibility, so all kinds can now be
-    repeated. We copy code into a non-register variable (tempcode) in order to
-    be able to pass its address because some compilers complain otherwise. */
+    /* Process nested bracketed regex. First check for parentheses nested too
+    deeply. */
+
+    if ((cd->parens_depth += 1) > PARENS_NEST_LIMIT)
+      {
+      *errorcodeptr = ERR82;
+      goto FAILED;
+      }
+
+    /* All assertions used not to be repeatable, but this was changed for Perl
+    compatibility. All kinds can now be repeated except for assertions that are
+    conditions (Perl also forbids these to be repeated). We copy code into a
+    non-register variable (tempcode) in order to be able to pass its address
+    because some compilers complain otherwise. At the start of a conditional
+    group whose condition is an assertion, cd->iscondassert is set. We unset it
+    here so as to allow assertions later in the group to be quantified. */
+
+    if (bravalue >= OP_ASSERT && bravalue <= OP_ASSERTBACK_NOT &&
+        cd->iscondassert)
+      {
+      previous = NULL;
+      cd->iscondassert = FALSE;
+      }
+    else
+      {
+      previous = code;
+      item_hwm_offset = cd->hwm - cd->start_workspace;
+      }
 
-    previous = code;                      /* For handling repetition */
     *code = bravalue;
     tempcode = code;
     tempreqvary = cd->req_varyopt;        /* Save value before bracket */
@@ -6555,6 +7770,8 @@ for (;; ptr++)
          ))
       goto FAILED;
 
+    cd->parens_depth -= 1;
+
     /* If this was an atomic group and there are no capturing groups within it,
     generate OP_ONCE_NC instead of OP_ONCE. */
 
@@ -6702,15 +7919,17 @@ for (;; ptr++)
         }
       }
 
-    /* For a forward assertion, we take the reqchar, if set. This can be
-    helpful if the pattern that follows the assertion doesn't set a different
-    char. For example, it's useful for /(?=abcde).+/. We can't set firstchar
-    for an assertion, however because it leads to incorrect effect for patterns
-    such as /(?=a)a.+/ when the "real" "a" would then become a reqchar instead
-    of a firstchar. This is overcome by a scan at the end if there's no
-    firstchar, looking for an asserted first char. */
-
-    else if (bravalue == OP_ASSERT && subreqcharflags >= 0)
+    /* For a forward assertion, we take the reqchar, if set, provided that the
+    group has also set a first char. This can be helpful if the pattern that
+    follows the assertion doesn't set a different char. For example, it's
+    useful for /(?=abcde).+/. We can't set firstchar for an assertion, however
+    because it leads to incorrect effect for patterns such as /(?=a)a.+/ when
+    the "real" "a" would then become a reqchar instead of a firstchar. This is
+    overcome by a scan at the end if there's no firstchar, looking for an
+    asserted first char. */
+
+    else if (bravalue == OP_ASSERT && subreqcharflags >= 0 &&
+             subfirstcharflags >= 0)
       {
       reqchar = subreqchar;
       reqcharflags = subreqcharflags;
@@ -6736,16 +7955,6 @@ for (;; ptr++)
       c = ec;
     else
       {
-      if (escape == ESC_Q)            /* Handle start of quoted string */
-        {
-        if (ptr[1] == CHAR_BACKSLASH && ptr[2] == CHAR_E)
-          ptr += 2;               /* avoid empty string */
-            else inescq = TRUE;
-        continue;
-        }
-
-      if (escape == ESC_E) continue;  /* Perl ignores an orphan \E */
-
       /* For metasequences that actually match a character, we disable the
       setting of a first character if it hasn't already been set. */
 
@@ -6769,51 +7978,38 @@ for (;; ptr++)
       if (escape == ESC_g)
         {
         const pcre_uchar *p;
-        save_hwm = cd->hwm;   /* Normally this is set when '(' is read */
+        pcre_uint32 cf;
+
+        item_hwm_offset = cd->hwm - cd->start_workspace;   /* Normally this is set when '(' is read */
         terminator = (*(++ptr) == CHAR_LESS_THAN_SIGN)?
           CHAR_GREATER_THAN_SIGN : CHAR_APOSTROPHE;
 
         /* These two statements stop the compiler for warning about possibly
         unset variables caused by the jump to HANDLE_NUMERICAL_RECURSION. In
-        fact, because we actually check for a number below, the paths that
+        fact, because we do the check for a number below, the paths that
         would actually be in error are never taken. */
 
         skipbytes = 0;
         reset_bracount = FALSE;
 
-        /* Test for a name */
+        /* If it's not a signed or unsigned number, treat it as a name. */
 
-        if (ptr[1] != CHAR_PLUS && ptr[1] != CHAR_MINUS)
+        cf = ptr[1];
+        if (cf != CHAR_PLUS && cf != CHAR_MINUS && !IS_DIGIT(cf))
           {
-          BOOL is_a_number = TRUE;
-          for (p = ptr + 1; *p != CHAR_NULL && *p != (pcre_uchar)terminator; p++)
-            {
-            if (!MAX_255(*p)) { is_a_number = FALSE; break; }
-            if ((cd->ctypes[*p] & ctype_digit) == 0) is_a_number = FALSE;
-            if ((cd->ctypes[*p] & ctype_word) == 0) break;
-            }
-          if (*p != (pcre_uchar)terminator)
-            {
-            *errorcodeptr = ERR57;
-            break;
-            }
-          if (is_a_number)
-            {
-            ptr++;
-            goto HANDLE_NUMERICAL_RECURSION;
-            }
           is_recurse = TRUE;
           goto NAMED_REF_OR_RECURSE;
           }
 
-        /* Test a signed number in angle brackets or quotes. */
+        /* Signed or unsigned number (cf = ptr[1]) is known to be plus or minus
+        or a digit. */
 
         p = ptr + 2;
         while (IS_DIGIT(*p)) p++;
         if (*p != (pcre_uchar)terminator)
           {
           *errorcodeptr = ERR57;
-          break;
+          goto FAILED;
           }
         ptr++;
         goto HANDLE_NUMERICAL_RECURSION;
@@ -6828,7 +8024,7 @@ for (;; ptr++)
           ptr[1] != CHAR_APOSTROPHE && ptr[1] != CHAR_LEFT_CURLY_BRACKET))
           {
           *errorcodeptr = ERR69;
-          break;
+          goto FAILED;
           }
         is_recurse = FALSE;
         terminator = (*(++ptr) == CHAR_LESS_THAN_SIGN)?
@@ -6846,9 +8042,13 @@ for (;; ptr++)
         open_capitem *oc;
         recno = -escape;
 
-        HANDLE_REFERENCE:    /* Come here from named backref handling */
+        /* Come here from named backref handling when the reference is to a
+        single group (i.e. not to a duplicated name. */
+
+        HANDLE_REFERENCE:
         if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE;
         previous = code;
+        item_hwm_offset = cd->hwm - cd->start_workspace;
         *code++ = ((options & PCRE_CASELESS) != 0)? OP_REFI : OP_REF;
         PUT2INC(code, 0, recno);
         cd->backref_map |= (recno < 32)? (1 << recno) : 1;
@@ -6878,6 +8078,7 @@ for (;; ptr++)
         if (!get_ucp(&ptr, &negated, &ptype, &pdata, errorcodeptr))
           goto FAILED;
         previous = code;
+        item_hwm_offset = cd->hwm - cd->start_workspace;
         *code++ = ((escape == ESC_p) != negated)? OP_PROP : OP_NOTPROP;
         *code++ = ptype;
         *code++ = pdata;
@@ -6918,6 +8119,7 @@ for (;; ptr++)
 
           {
           previous = (escape > ESC_b && escape < ESC_Z)? code : NULL;
+          item_hwm_offset = cd->hwm - cd->start_workspace;
           *code++ = (!utf && escape == ESC_C)? OP_ALLANY : escape;
           }
         }
@@ -6943,8 +8145,8 @@ for (;; ptr++)
 
     /* ===================================================================*/
     /* Handle a literal character. It is guaranteed not to be whitespace or #
-    when the extended flag is set. If we are in UTF-8 mode, it may be a
-    multi-byte literal character. */
+    when the extended flag is set. If we are in a UTF mode, it may be a
+    multi-unit literal character. */
 
     default:
     NORMAL_CHAR:
@@ -6961,6 +8163,7 @@ for (;; ptr++)
 
     ONE_CHAR:
     previous = code;
+    item_hwm_offset = cd->hwm - cd->start_workspace;
 
     /* For caseless UTF-8 mode when UCP support is available, check whether
     this character has more than one other case. If so, generate a special
@@ -6975,7 +8178,8 @@ for (;; ptr++)
         *code++ = OP_PROP;
         *code++ = PT_CLIST;
         *code++ = c;
-        if (firstcharflags == REQ_UNSET) firstcharflags = zerofirstcharflags = REQ_NONE;
+        if (firstcharflags == REQ_UNSET)
+          firstcharflags = zerofirstcharflags = REQ_NONE;
         break;
         }
       }
@@ -7064,24 +8268,24 @@ out the amount of memory needed, as well as during the real compile phase. The
 value of lengthptr distinguishes the two phases.
 
 Arguments:
-  options        option bits, including any changes for this subpattern
-  codeptr        -> the address of the current code pointer
-  ptrptr         -> the address of the current pattern pointer
-  errorcodeptr   -> pointer to error code variable
-  lookbehind     TRUE if this is a lookbehind assertion
-  reset_bracount TRUE to reset the count for each branch
-  skipbytes      skip this many bytes at start (for brackets and OP_COND)
-  cond_depth     depth of nesting for conditional subpatterns
-  firstcharptr    place to put the first required character
+  options           option bits, including any changes for this subpattern
+  codeptr           -> the address of the current code pointer
+  ptrptr            -> the address of the current pattern pointer
+  errorcodeptr      -> pointer to error code variable
+  lookbehind        TRUE if this is a lookbehind assertion
+  reset_bracount    TRUE to reset the count for each branch
+  skipbytes         skip this many bytes at start (for brackets and OP_COND)
+  cond_depth        depth of nesting for conditional subpatterns
+  firstcharptr      place to put the first required character
   firstcharflagsptr place to put the first character flags, or a negative number
-  reqcharptr     place to put the last required character
-  reqcharflagsptr place to put the last required character flags, or a negative number
-  bcptr          pointer to the chain of currently open branches
-  cd             points to the data block with tables pointers etc.
-  lengthptr      NULL during the real compile phase
-                 points to length accumulator during pre-compile phase
-
-Returns:         TRUE on success
+  reqcharptr        place to put the last required character
+  reqcharflagsptr   place to put the last required character flags, or a negative number
+  bcptr             pointer to the chain of currently open branches
+  cd                points to the data block with tables pointers etc.
+  lengthptr         NULL during the real compile phase
+                    points to length accumulator during pre-compile phase
+
+Returns:            TRUE on success
 */
 
 static BOOL
@@ -7107,6 +8311,17 @@ int length;
 unsigned int orig_bracount;
 unsigned int max_bracount;
 branch_chain bc;
+size_t save_hwm_offset;
+
+/* If set, call the external function that checks for stack availability. */
+
+if (PUBL(stack_guard) != NULL && PUBL(stack_guard)())
+  {
+  *errorcodeptr= ERR85;
+  return FALSE;
+  }
+
+/* Miscellaneous initialization */
 
 bc.outer = bcptr;
 bc.current_branch = code;
@@ -7114,6 +8329,8 @@ bc.current_branch = code;
 firstchar = reqchar = 0;
 firstcharflags = reqcharflags = REQ_UNSET;
 
+save_hwm_offset = cd->hwm - cd->start_workspace;
+
 /* Accumulate the length for use in the pre-compile phase. Start with the
 length of the BRA and KET and any extra bytes that are required at the
 beginning. We accumulate in a local variable to save frequent testing of
@@ -7255,7 +8472,7 @@ for (;;)
       int fixed_length;
       *code = OP_END;
       fixed_length = find_fixedlength(last_branch,  (options & PCRE_UTF8) != 0,
-        FALSE, cd);
+        FALSE, cd, NULL);
       DPRINTF(("fixed length = %d\n", fixed_length));
       if (fixed_length == -3)
         {
@@ -7307,12 +8524,16 @@ for (;;)
 
     /* If it was a capturing subpattern, check to see if it contained any
     recursive back references. If so, we must wrap it in atomic brackets.
-    In any event, remove the block from the chain. */
+    Because we are moving code along, we must ensure that any pending recursive
+    references are updated. In any event, remove the block from the chain. */
 
     if (capnumber > 0)
       {
       if (cd->open_caps->flag)
         {
+        *code = OP_END;
+        adjust_recurse(start_bracket, 1 + LINK_SIZE,
+          (options & PCRE_UTF8) != 0, cd, save_hwm_offset);
         memmove(start_bracket + 1 + LINK_SIZE, start_bracket,
           IN_UCHARS(code - start_bracket));
         *start_bracket = OP_ONCE;
@@ -7497,8 +8718,8 @@ matching and for non-DOTALL patterns that start with .* (which must start at
 the beginning or after \n). As in the case of is_anchored() (see above), we
 have to take account of back references to capturing brackets that contain .*
 because in that case we can't make the assumption. Also, the appearance of .*
-inside atomic brackets or in a pattern that contains *PRUNE or *SKIP does not
-count, because once again the assumption no longer holds.
+inside atomic brackets or in an assertion, or in a pattern that contains *PRUNE
+or *SKIP does not count, because once again the assumption no longer holds.
 
 Arguments:
   code           points to start of expression (the bracket)
@@ -7507,13 +8728,14 @@ Arguments:
                   the less precise approach
   cd             points to the compile data
   atomcount      atomic group level
+  inassert       TRUE if in an assertion
 
 Returns:         TRUE or FALSE
 */
 
 static BOOL
 is_startline(const pcre_uchar *code, unsigned int bracket_map,
-  compile_data *cd, int atomcount)
+  compile_data *cd, int atomcount, BOOL inassert)
 {
 do {
    const pcre_uchar *scode = first_significant_code(
@@ -7532,14 +8754,15 @@ do {
      switch (*scode)
        {
        case OP_CREF:
-       case OP_NCREF:
+       case OP_DNCREF:
        case OP_RREF:
-       case OP_NRREF:
+       case OP_DNRREF:
        case OP_DEF:
+       case OP_FAIL:
        return FALSE;
 
        default:     /* Assertion */
-       if (!is_startline(scode, bracket_map, cd, atomcount)) return FALSE;
+       if (!is_startline(scode, bracket_map, cd, atomcount, TRUE)) return FALSE;
        do scode += GET(scode, 1); while (*scode == OP_ALT);
        scode += 1 + LINK_SIZE;
        break;
@@ -7553,7 +8776,7 @@ do {
    if (op == OP_BRA  || op == OP_BRAPOS ||
        op == OP_SBRA || op == OP_SBRAPOS)
      {
-     if (!is_startline(scode, bracket_map, cd, atomcount)) return FALSE;
+     if (!is_startline(scode, bracket_map, cd, atomcount, inassert)) return FALSE;
      }
 
    /* Capturing brackets */
@@ -7563,33 +8786,33 @@ do {
      {
      int n = GET2(scode, 1+LINK_SIZE);
      int new_map = bracket_map | ((n < 32)? (1 << n) : 1);
-     if (!is_startline(scode, new_map, cd, atomcount)) return FALSE;
+     if (!is_startline(scode, new_map, cd, atomcount, inassert)) return FALSE;
      }
 
    /* Positive forward assertions */
 
    else if (op == OP_ASSERT)
      {
-     if (!is_startline(scode, bracket_map, cd, atomcount)) return FALSE;
+     if (!is_startline(scode, bracket_map, cd, atomcount, TRUE)) return FALSE;
      }
 
    /* Atomic brackets */
 
    else if (op == OP_ONCE || op == OP_ONCE_NC)
      {
-     if (!is_startline(scode, bracket_map, cd, atomcount + 1)) return FALSE;
+     if (!is_startline(scode, bracket_map, cd, atomcount + 1, inassert)) return FALSE;
      }
 
    /* .* means "start at start or after \n" if it isn't in atomic brackets or
-   brackets that may be referenced, as long as the pattern does not contain
-   *PRUNE or *SKIP, because these break the feature. Consider, for example,
-   /.*?a(*PRUNE)b/ with the subject "aab", which matches "ab", i.e. not at the
-   start of a line. */
+   brackets that may be referenced or an assertion, as long as the pattern does
+   not contain *PRUNE or *SKIP, because these break the feature. Consider, for
+   example, /.*?a(*PRUNE)b/ with the subject "aab", which matches "ab", i.e.
+   not at the start of a line. */
 
    else if (op == OP_TYPESTAR || op == OP_TYPEMINSTAR || op == OP_TYPEPOSSTAR)
      {
      if (scode[1] != OP_ANY || (bracket_map & cd->backref_map) != 0 ||
-         atomcount > 0 || cd->had_pruneorskip)
+         atomcount > 0 || cd->had_pruneorskip || inassert)
        return FALSE;
      }
 
@@ -7618,13 +8841,14 @@ return TRUE;
 discarded, because they can cause conflicts with actual literals that follow.
 However, if we end up without a first char setting for an unanchored pattern,
 it is worth scanning the regex to see if there is an initial asserted first
-char. If all branches start with the same asserted char, or with a bracket all
-of whose alternatives start with the same asserted char (recurse ad lib), then
-we return that char, otherwise -1.
+char. If all branches start with the same asserted char, or with a
+non-conditional bracket all of whose alternatives start with the same asserted
+char (recurse ad lib), then we return that char, with the flags set to zero or
+REQ_CASELESS; otherwise return zero with REQ_NONE in the flags.
 
 Arguments:
   code       points to start of expression (the bracket)
-  flags       points to the first char flags, or to REQ_NONE
+  flags      points to the first char flags, or to REQ_NONE
   inassert   TRUE if in an assertion
 
 Returns:     the fixed first char, or 0 with REQ_NONE in flags
@@ -7661,7 +8885,6 @@ do {
      case OP_ASSERT:
      case OP_ONCE:
      case OP_ONCE_NC:
-     case OP_COND:
      d = find_firstassertedchar(scode, &dflags, op == OP_ASSERT);
      if (dflags < 0)
        return 0;
@@ -7705,6 +8928,61 @@ return c;
 
 
 
+/*************************************************
+*     Add an entry to the name/number table      *
+*************************************************/
+
+/* This function is called between compiling passes to add an entry to the
+name/number table, maintaining alphabetical order. Checking for permitted
+and forbidden duplicates has already been done.
+
+Arguments:
+  cd           the compile data block
+  name         the name to add
+  length       the length of the name
+  groupno      the group number
+
+Returns:       nothing
+*/
+
+static void
+add_name(compile_data *cd, const pcre_uchar *name, int length,
+  unsigned int groupno)
+{
+int i;
+pcre_uchar *slot = cd->name_table;
+
+for (i = 0; i < cd->names_found; i++)
+  {
+  int crc = memcmp(name, slot+IMM2_SIZE, IN_UCHARS(length));
+  if (crc == 0 && slot[IMM2_SIZE+length] != 0)
+    crc = -1; /* Current name is a substring */
+
+  /* Make space in the table and break the loop for an earlier name. For a
+  duplicate or later name, carry on. We do this for duplicates so that in the
+  simple case (when ?(| is not used) they are in order of their numbers. In all
+  cases they are in the order in which they appear in the pattern. */
+
+  if (crc < 0)
+    {
+    memmove(slot + cd->name_entry_size, slot,
+      IN_UCHARS((cd->names_found - i) * cd->name_entry_size));
+    break;
+    }
+
+  /* Continue the loop for a later or duplicate name */
+
+  slot += cd->name_entry_size;
+  }
+
+PUT2(slot, 0, groupno);
+memcpy(slot + IMM2_SIZE, name, IN_UCHARS(length));
+slot[IMM2_SIZE + length] = 0;
+cd->names_found++;
+}
+
+
+
 /*************************************************
 *        Compile a Regular Expression            *
 *************************************************/
@@ -7809,6 +9087,11 @@ new memory is obtained from malloc(). */
 
 pcre_uchar cworkspace[COMPILE_WORK_SIZE];
 
+/* This vector is used for remembering name groups during the pre-compile. In a
+similar way to cworkspace, it can be expanded using malloc() if necessary. */
+
+named_group named_groups[NAMED_GROUP_LIST_SIZE];
+
 /* Set this early so that early errors get offset 0. */
 
 ptr = (const pcre_uchar *)pattern;
@@ -7888,6 +9171,8 @@ PCRE_UTF8 == PCRE_UTF16 == PCRE_UTF32. */
     { skipatstart += 6; options |= PCRE_UTF8; continue; }
   else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UCP_RIGHTPAR, 4) == 0)
     { skipatstart += 6; options |= PCRE_UCP; continue; }
+  else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_NO_AUTO_POSSESS_RIGHTPAR, 16) == 0)
+    { skipatstart += 18; options |= PCRE_NO_AUTO_POSSESS; continue; }
   else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_NO_START_OPT_RIGHTPAR, 13) == 0)
     { skipatstart += 15; options |= PCRE_NO_START_OPTIMIZE; continue; }
 
@@ -8071,13 +9356,20 @@ cd->bracount = cd->final_bracount = 0;
 cd->names_found = 0;
 cd->name_entry_size = 0;
 cd->name_table = NULL;
+cd->dupnames = FALSE;
+cd->dupgroups = FALSE;
+cd->namedrefcount = 0;
 cd->start_code = cworkspace;
 cd->hwm = cworkspace;
+cd->iscondassert = FALSE;
 cd->start_workspace = cworkspace;
 cd->workspace_size = COMPILE_WORK_SIZE;
+cd->named_groups = named_groups;
+cd->named_group_list_size = NAMED_GROUP_LIST_SIZE;
 cd->start_pattern = (const pcre_uchar *)pattern;
 cd->end_pattern = (const pcre_uchar *)(pattern + STRLEN_UC((const pcre_uchar *)pattern));
 cd->req_varyopt = 0;
+cd->parens_depth = 0;
 cd->assert_depth = 0;
 cd->max_lookbehind = 0;
 cd->external_options = options;
@@ -8092,6 +9384,7 @@ outside can help speed up starting point checks. */
 ptr += skipatstart;
 code = cworkspace;
 *code = OP_BRA;
+
 (void)compile_regex(cd->external_options, &code, &ptr, &errorcode, FALSE,
   FALSE, 0, 0, &firstchar, &firstcharflags, &reqchar, &reqcharflags, NULL,
   cd, &length);
@@ -8106,14 +9399,16 @@ if (length > MAX_PATTERN_SIZE)
   goto PCRE_EARLY_ERROR_RETURN;
   }
 
-/* Compute the size of data block needed and get it, either from malloc or
-externally provided function. Integer overflow should no longer be possible
-because nowadays we limit the maximum value of cd->names_found and
-cd->name_entry_size. */
+/* Compute the size of the data block for storing the compiled pattern. Integer
+overflow should no longer be possible because nowadays we limit the maximum
+value of cd->names_found and cd->name_entry_size. */
 
-size = sizeof(REAL_PCRE) + (length + cd->names_found * cd->name_entry_size) * sizeof(pcre_uchar);
-re = (REAL_PCRE *)(PUBL(malloc))(size);
+size = sizeof(REAL_PCRE) +
+  (length + cd->names_found * cd->name_entry_size) * sizeof(pcre_uchar);
 
+/* Get the memory. */
+
+re = (REAL_PCRE *)(PUBL(malloc))(size);
 if (re == NULL)
   {
   errorcode = ERR21;
@@ -8154,20 +9449,35 @@ field; this time it's used for remembering forward references to subpatterns.
 */
 
 cd->final_bracount = cd->bracount;  /* Save for checking forward references */
+cd->parens_depth = 0;
 cd->assert_depth = 0;
 cd->bracount = 0;
 cd->max_lookbehind = 0;
-cd->names_found = 0;
 cd->name_table = (pcre_uchar *)re + re->name_table_offset;
 codestart = cd->name_table + re->name_entry_size * re->name_count;
 cd->start_code = codestart;
 cd->hwm = (pcre_uchar *)(cd->start_workspace);
+cd->iscondassert = FALSE;
 cd->req_varyopt = 0;
 cd->had_accept = FALSE;
 cd->had_pruneorskip = FALSE;
 cd->check_lookbehind = FALSE;
 cd->open_caps = NULL;
 
+/* If any named groups were found, create the name/number table from the list
+created in the first pass. */
+
+if (cd->names_found > 0)
+  {
+  int i = cd->names_found;
+  named_group *ng = cd->named_groups;
+  cd->names_found = 0;
+  for (; i > 0; i--, ng++)
+    add_name(cd, ng->name, ng->length, ng->number);
+  if (cd->named_group_list_size > NAMED_GROUP_LIST_SIZE)
+    (PUBL(free))((void *)cd->named_groups);
+  }
+
 /* Set up a starting, non-extracting bracket, then compile the expression. On
 error, errorcode will be set non-zero, so we don't need to look at the result
 of the function here. */
@@ -8220,6 +9530,16 @@ if (cd->hwm > cd->start_workspace)
     int offset, recno;
     cd->hwm -= LINK_SIZE;
     offset = GET(cd->hwm, 0);
+
+    /* Check that the hwm handling hasn't gone wrong. This whole area is
+    rewritten in PCRE2 because there are some obscure cases. */
+
+    if (offset == 0 || codestart[offset-1] != OP_RECURSE)
+      {
+      errorcode = ERR10;
+      break;
+      }
+
     recno = GET(codestart, offset);
     if (recno != prev_recno)
       {
@@ -8231,16 +9551,31 @@ if (cd->hwm > cd->start_workspace)
     }
   }
 
-/* If the workspace had to be expanded, free the new memory. */
+/* If the workspace had to be expanded, free the new memory. Set the pointer to
+NULL to indicate that forward references have been filled in. */
 
 if (cd->workspace_size > COMPILE_WORK_SIZE)
   (PUBL(free))((void *)cd->start_workspace);
+cd->start_workspace = NULL;
 
 /* Give an error if there's back reference to a non-existent capturing
 subpattern. */
 
 if (errorcode == 0 && re->top_backref > re->top_bracket) errorcode = ERR15;
 
+/* Unless disabled, check whether any single character iterators can be
+auto-possessified. The function overwrites the appropriate opcode values, so
+the type of the pointer must be cast. NOTE: the intermediate variable "temp" is
+used in this code because at least one compiler gives a warning about loss of
+"const" attribute if the cast (pcre_uchar *)codestart is used directly in the
+function call. */
+
+if (errorcode == 0 && (options & PCRE_NO_AUTO_POSSESS) == 0)
+  {
+  pcre_uchar *temp = (pcre_uchar *)codestart;
+  auto_possessify(temp, utf, cd);
+  }
+
 /* If there were any lookbehind assertions that contained OP_RECURSE
 (recursions or subroutine calls), a flag is set for them to be checked here,
 because they may contain forward references. Actual recursions cannot be fixed
@@ -8249,7 +9584,7 @@ OP_RECURSE that are not fixed length get a diagnosic with a useful offset. The
 exceptional ones forgo this. We scan the pattern to check that they are fixed
 length, and set their lengths. */
 
-if (cd->check_lookbehind)
+if (errorcode == 0 && cd->check_lookbehind)
   {
   pcre_uchar *cc = (pcre_uchar *)codestart;
 
@@ -8269,7 +9604,7 @@ if (cd->check_lookbehind)
       int end_op = *be;
       *be = OP_END;
       fixed_length = find_fixedlength(cc, (re->options & PCRE_UTF8) != 0, TRUE,
-        cd);
+        cd, NULL);
       *be = end_op;
       DPRINTF(("fixed length = %d\n", fixed_length));
       if (fixed_length < 0)
@@ -8349,7 +9684,7 @@ if ((re->options & PCRE_ANCHORED) == 0)
       re->flags |= PCRE_FIRSTSET;
       }
 
-    else if (is_startline(codestart, 0, cd, 0)) re->flags |= PCRE_STARTLINE;
+    else if (is_startline(codestart, 0, cd, 0, FALSE)) re->flags |= PCRE_STARTLINE;
     }
   }
 
@@ -8438,6 +9773,20 @@ if (code - codestart > length)
   }
 #endif   /* PCRE_DEBUG */
 
+/* Check for a pattern than can match an empty string, so that this information
+can be provided to applications. */
+
+do
+  {
+  if (could_be_empty_branch(codestart, code, utf, cd, NULL))
+    {
+    re->flags |= PCRE_MATCH_EMPTY;
+    break;
+    }
+  codestart += GET(codestart, 1);
+  }
+while (*codestart == OP_ALT);
+
 #if defined COMPILE_PCRE8
 return (pcre *)re;
 #elif defined COMPILE_PCRE16
diff --git a/erts/emulator/pcre/pcre_config.c b/erts/emulator/pcre/pcre_config.c
index 06fa3d324f..297f0e14bc 100644
--- a/erts/emulator/pcre/pcre_config.c
+++ b/erts/emulator/pcre/pcre_config.c
@@ -171,6 +171,10 @@ switch (what)
   *((int *)where) = POSIX_MALLOC_THRESHOLD;
   break;
 
+  case PCRE_CONFIG_PARENS_LIMIT:
+  *((unsigned long int *)where) = PARENS_NEST_LIMIT;
+  break;
+
   case PCRE_CONFIG_MATCH_LIMIT:
   *((unsigned long int *)where) = MATCH_LIMIT;
   break;
diff --git a/erts/emulator/pcre/pcre_dfa_exec.c b/erts/emulator/pcre/pcre_dfa_exec.c
index f5718a3b33..529f40685b 100644
--- a/erts/emulator/pcre/pcre_dfa_exec.c
+++ b/erts/emulator/pcre/pcre_dfa_exec.c
@@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language (but see
 below for why this module is different).
 
                        Written by Philip Hazel
-           Copyright (c) 1997-2013 University of Cambridge
+           Copyright (c) 1997-2014 University of Cambridge
 
 -----------------------------------------------------------------------------
 Redistribution and use in source and binary forms, with or without
@@ -121,7 +121,7 @@ static const pcre_uint8 coptable[] = {
   0, 0,                          /* \P, \p                                 */
   0, 0, 0, 0, 0,                 /* \R, \H, \h, \V, \v                     */
   0,                             /* \X                                     */
-  0, 0, 0, 0, 0, 0,              /* \Z, \z, ^, ^M, $, $M                   */
+  0, 0, 0, 0, 0, 0,              /* \Z, \z, $, $M, ^, ^M                   */
   1,                             /* Char                                   */
   1,                             /* Chari                                  */
   1,                             /* not                                    */
@@ -152,11 +152,14 @@ static const pcre_uint8 coptable[] = {
   /* Character class & ref repeats                                         */
   0, 0, 0, 0, 0, 0,              /* *, *?, +, +?, ?, ??                    */
   0, 0,                          /* CRRANGE, CRMINRANGE                    */
+  0, 0, 0, 0,                    /* Possessive *+, ++, ?+, CRPOSRANGE      */
   0,                             /* CLASS                                  */
   0,                             /* NCLASS                                 */
   0,                             /* XCLASS - variable length               */
   0,                             /* REF                                    */
   0,                             /* REFI                                   */
+  0,                             /* DNREF                                  */
+  0,                             /* DNREFI                                 */
   0,                             /* RECURSE                                */
   0,                             /* CALLOUT                                */
   0,                             /* Alt                                    */
@@ -172,8 +175,8 @@ static const pcre_uint8 coptable[] = {
   0, 0,                          /* ONCE, ONCE_NC                          */
   0, 0, 0, 0, 0,                 /* BRA, BRAPOS, CBRA, CBRAPOS, COND       */
   0, 0, 0, 0, 0,                 /* SBRA, SBRAPOS, SCBRA, SCBRAPOS, SCOND  */
-  0, 0,                          /* CREF, NCREF                            */
-  0, 0,                          /* RREF, NRREF                            */
+  0, 0,                          /* CREF, DNCREF                           */
+  0, 0,                          /* RREF, DNRREF                           */
   0,                             /* DEF                                    */
   0, 0, 0,                       /* BRAZERO, BRAMINZERO, BRAPOSZERO        */
   0, 0, 0,                       /* MARK, PRUNE, PRUNE_ARG                 */
@@ -195,7 +198,7 @@ static const pcre_uint8 poptable[] = {
   1, 1,                          /* \P, \p                                 */
   1, 1, 1, 1, 1,                 /* \R, \H, \h, \V, \v                     */
   1,                             /* \X                                     */
-  0, 0, 0, 0, 0, 0,              /* \Z, \z, ^, ^M, $, $M                   */
+  0, 0, 0, 0, 0, 0,              /* \Z, \z, $, $M, ^, ^M                   */
   1,                             /* Char                                   */
   1,                             /* Chari                                  */
   1,                             /* not                                    */
@@ -221,11 +224,14 @@ static const pcre_uint8 poptable[] = {
   /* Character class & ref repeats                                         */
   1, 1, 1, 1, 1, 1,              /* *, *?, +, +?, ?, ??                    */
   1, 1,                          /* CRRANGE, CRMINRANGE                    */
+  1, 1, 1, 1,                    /* Possessive *+, ++, ?+, CRPOSRANGE      */
   1,                             /* CLASS                                  */
   1,                             /* NCLASS                                 */
   1,                             /* XCLASS - variable length               */
   0,                             /* REF                                    */
   0,                             /* REFI                                   */
+  0,                             /* DNREF                                  */
+  0,                             /* DNREFI                                 */
   0,                             /* RECURSE                                */
   0,                             /* CALLOUT                                */
   0,                             /* Alt                                    */
@@ -241,8 +247,8 @@ static const pcre_uint8 poptable[] = {
   0, 0,                          /* ONCE, ONCE_NC                          */
   0, 0, 0, 0, 0,                 /* BRA, BRAPOS, CBRA, CBRAPOS, COND       */
   0, 0, 0, 0, 0,                 /* SBRA, SBRAPOS, SCBRA, SCBRAPOS, SCOND  */
-  0, 0,                          /* CREF, NCREF                            */
-  0, 0,                          /* RREF, NRREF                            */
+  0, 0,                          /* CREF, DNCREF                           */
+  0, 0,                          /* RREF, DNRREF                           */
   0,                             /* DEF                                    */
   0, 0, 0,                       /* BRAZERO, BRAMINZERO, BRAPOSZERO        */
   0, 0, 0,                       /* MARK, PRUNE, PRUNE_ARG                 */
@@ -1095,15 +1101,23 @@ for (;;)
                PRIV(ucp_gentype)[prop->chartype] == ucp_N;
           break;
 
-          case PT_SPACE:    /* Perl space */
-          OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z ||
-               c == CHAR_HT || c == CHAR_NL || c == CHAR_FF || c == CHAR_CR;
-          break;
+          /* Perl space used to exclude VT, but from Perl 5.18 it is included,
+          which means that Perl space and POSIX space are now identical. PCRE
+          was changed at release 8.34. */
 
+          case PT_SPACE:    /* Perl space */
           case PT_PXSPACE:  /* POSIX space */
-          OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z ||
-               c == CHAR_HT || c == CHAR_NL || c == CHAR_VT ||
-               c == CHAR_FF || c == CHAR_CR;
+          switch(c)
+            {
+            HSPACE_CASES:
+            VSPACE_CASES:
+            OK = TRUE;
+            break;
+
+            default:
+            OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z;
+            break;
+            }
           break;
 
           case PT_WORD:
@@ -1345,15 +1359,23 @@ for (;;)
                PRIV(ucp_gentype)[prop->chartype] == ucp_N;
           break;
 
-          case PT_SPACE:    /* Perl space */
-          OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z ||
-               c == CHAR_HT || c == CHAR_NL || c == CHAR_FF || c == CHAR_CR;
-          break;
+          /* Perl space used to exclude VT, but from Perl 5.18 it is included,
+          which means that Perl space and POSIX space are now identical. PCRE
+          was changed at release 8.34. */
 
+          case PT_SPACE:    /* Perl space */
           case PT_PXSPACE:  /* POSIX space */
-          OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z ||
-               c == CHAR_HT || c == CHAR_NL || c == CHAR_VT ||
-               c == CHAR_FF || c == CHAR_CR;
+          switch(c)
+            {
+            HSPACE_CASES:
+            VSPACE_CASES:
+            OK = TRUE;
+            break;
+
+            default:
+            OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z;
+            break;
+            }
           break;
 
           case PT_WORD:
@@ -1452,7 +1474,7 @@ for (;;)
           goto ANYNL01;
 
           case CHAR_CR:
-          if (ptr + 1 < end_subject && RAWUCHARTEST(ptr + 1) == CHAR_LF) ncount = 1;
+          if (ptr + 1 < end_subject && UCHAR21TEST(ptr + 1) == CHAR_LF) ncount = 1;
           /* Fall through */
 
           ANYNL01:
@@ -1589,15 +1611,23 @@ for (;;)
                PRIV(ucp_gentype)[prop->chartype] == ucp_N;
           break;
 
-          case PT_SPACE:    /* Perl space */
-          OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z ||
-               c == CHAR_HT || c == CHAR_NL || c == CHAR_FF || c == CHAR_CR;
-          break;
+          /* Perl space used to exclude VT, but from Perl 5.18 it is included,
+          which means that Perl space and POSIX space are now identical. PCRE
+          was changed at release 8.34. */
 
+          case PT_SPACE:    /* Perl space */
           case PT_PXSPACE:  /* POSIX space */
-          OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z ||
-               c == CHAR_HT || c == CHAR_NL || c == CHAR_VT ||
-               c == CHAR_FF || c == CHAR_CR;
+          switch(c)
+            {
+            HSPACE_CASES:
+            VSPACE_CASES:
+            OK = TRUE;
+            break;
+
+            default:
+            OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z;
+            break;
+            }
           break;
 
           case PT_WORD:
@@ -1713,7 +1743,7 @@ for (;;)
           goto ANYNL02;
 
           case CHAR_CR:
-          if (ptr + 1 < end_subject && RAWUCHARTEST(ptr + 1) == CHAR_LF) ncount = 1;
+          if (ptr + 1 < end_subject && UCHAR21TEST(ptr + 1) == CHAR_LF) ncount = 1;
           /* Fall through */
 
           ANYNL02:
@@ -1858,15 +1888,23 @@ for (;;)
                PRIV(ucp_gentype)[prop->chartype] == ucp_N;
           break;
 
-          case PT_SPACE:    /* Perl space */
-          OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z ||
-               c == CHAR_HT || c == CHAR_NL || c == CHAR_FF || c == CHAR_CR;
-          break;
+          /* Perl space used to exclude VT, but from Perl 5.18 it is included,
+          which means that Perl space and POSIX space are now identical. PCRE
+          was changed at release 8.34. */
 
+          case PT_SPACE:    /* Perl space */
           case PT_PXSPACE:  /* POSIX space */
-          OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z ||
-               c == CHAR_HT || c == CHAR_NL || c == CHAR_VT ||
-               c == CHAR_FF || c == CHAR_CR;
+          switch(c)
+            {
+            HSPACE_CASES:
+            VSPACE_CASES:
+            OK = TRUE;
+            break;
+
+            default:
+            OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z;
+            break;
+            }
           break;
 
           case PT_WORD:
@@ -1975,7 +2013,7 @@ for (;;)
           goto ANYNL03;
 
           case CHAR_CR:
-          if (ptr + 1 < end_subject && RAWUCHARTEST(ptr + 1) == CHAR_LF) ncount = 1;
+          if (ptr + 1 < end_subject && UCHAR21TEST(ptr + 1) == CHAR_LF) ncount = 1;
           /* Fall through */
 
           ANYNL03:
@@ -2173,7 +2211,7 @@ for (;;)
           if ((md->moptions & PCRE_PARTIAL_HARD) != 0)
             reset_could_continue = TRUE;
           }
-        else if (RAWUCHARTEST(ptr + 1) == CHAR_LF)
+        else if (UCHAR21TEST(ptr + 1) == CHAR_LF)
           {
           ADD_NEW_DATA(-(state_offset + 1), 0, 1);
           }
@@ -2534,31 +2572,65 @@ for (;;)
           {
           case OP_CRSTAR:
           case OP_CRMINSTAR:
+          case OP_CRPOSSTAR:
           ADD_ACTIVE(next_state_offset + 1, 0);
-          if (isinclass) { ADD_NEW(state_offset, 0); }
+          if (isinclass)
+            {
+            if (*ecode == OP_CRPOSSTAR)
+              {
+              active_count--;           /* Remove non-match possibility */
+              next_active_state--;
+              }
+            ADD_NEW(state_offset, 0);
+            }
           break;
 
           case OP_CRPLUS:
           case OP_CRMINPLUS:
+          case OP_CRPOSPLUS:
           count = current_state->count;  /* Already matched */
           if (count > 0) { ADD_ACTIVE(next_state_offset + 1, 0); }
-          if (isinclass) { count++; ADD_NEW(state_offset, count); }
+          if (isinclass)
+            {
+            if (count > 0 && *ecode == OP_CRPOSPLUS)
+              {
+              active_count--;           /* Remove non-match possibility */
+              next_active_state--;
+              }
+            count++;
+            ADD_NEW(state_offset, count);
+            }
           break;
 
           case OP_CRQUERY:
           case OP_CRMINQUERY:
+          case OP_CRPOSQUERY:
           ADD_ACTIVE(next_state_offset + 1, 0);
-          if (isinclass) { ADD_NEW(next_state_offset + 1, 0); }
+          if (isinclass)
+            {
+            if (*ecode == OP_CRPOSQUERY)
+              {
+              active_count--;           /* Remove non-match possibility */
+              next_active_state--;
+              }
+            ADD_NEW(next_state_offset + 1, 0);
+            }
           break;
 
           case OP_CRRANGE:
           case OP_CRMINRANGE:
+          case OP_CRPOSRANGE:
           count = current_state->count;  /* Already matched */
           if (count >= (int)GET2(ecode, 1))
             { ADD_ACTIVE(next_state_offset + 1 + 2 * IMM2_SIZE, 0); }
           if (isinclass)
             {
             int max = (int)GET2(ecode, 1 + IMM2_SIZE);
+            if (*ecode == OP_CRPOSRANGE)
+              {
+              active_count--;           /* Remove non-match possibility */
+              next_active_state--;
+              }
             if (++count >= max && max != 0)   /* Max 0 => no limit */
               { ADD_NEW(next_state_offset + 1 + 2 * IMM2_SIZE, 0); }
             else
@@ -2658,21 +2730,24 @@ for (;;)
 
         condcode = code[LINK_SIZE+1];
 
-        /* Back reference conditions are not supported */
+        /* Back reference conditions and duplicate named recursion conditions
+        are not supported */
 
-        if (condcode == OP_CREF || condcode == OP_NCREF)
+        if (condcode == OP_CREF || condcode == OP_DNCREF ||
+            condcode == OP_DNRREF)
           return PCRE_ERROR_DFA_UCOND;
 
-        /* The DEFINE condition is always false */
+        /* The DEFINE condition is always false, and the assertion (?!) is
+        converted to OP_FAIL. */
 
-        if (condcode == OP_DEF)
+        if (condcode == OP_DEF || condcode == OP_FAIL)
           { ADD_ACTIVE(state_offset + codelink + LINK_SIZE + 1, 0); }
 
         /* The only supported version of OP_RREF is for the value RREF_ANY,
         which means "test if in any recursion". We can't test for specifically
         recursed groups. */
 
-        else if (condcode == OP_RREF || condcode == OP_NRREF)
+        else if (condcode == OP_RREF)
           {
           int value = GET2(code, LINK_SIZE + 2);
           if (value != RREF_ANY) return PCRE_ERROR_DFA_UCOND;
@@ -3176,7 +3251,7 @@ md->callout_data = NULL;
 
 if (extra_data != NULL)
   {
-  unsigned int flags = extra_data->flags;
+  unsigned long int flags = extra_data->flags;
   if ((flags & PCRE_EXTRA_STUDY_DATA) != 0)
     study = (const pcre_study_data *)extra_data->study_data;
   if ((flags & PCRE_EXTRA_MATCH_LIMIT) != 0) return PCRE_ERROR_DFA_UMLIMIT;
@@ -3400,7 +3475,7 @@ for (;;)
 
     if (((options | re->options) & PCRE_NO_START_OPTIMIZE) == 0)
       {
-      /* Advance to a known first char. */
+      /* Advance to a known first pcre_uchar (i.e. data item) */
 
       if (has_first_char)
         {
@@ -3408,12 +3483,12 @@ for (;;)
           {
           pcre_uchar csc;
           while (current_subject < end_subject &&
-                 (csc = RAWUCHARTEST(current_subject)) != first_char && csc != first_char2)
+                 (csc = UCHAR21TEST(current_subject)) != first_char && csc != first_char2)
             current_subject++;
           }
         else
           while (current_subject < end_subject &&
-                 RAWUCHARTEST(current_subject) != first_char)
+                 UCHAR21TEST(current_subject) != first_char)
             current_subject++;
         }
 
@@ -3443,36 +3518,26 @@ for (;;)
           ANYCRLF, and we are now at a LF, advance the match position by one
           more character. */
 
-          if (RAWUCHARTEST(current_subject - 1) == CHAR_CR &&
+          if (UCHAR21TEST(current_subject - 1) == CHAR_CR &&
                (md->nltype == NLTYPE_ANY || md->nltype == NLTYPE_ANYCRLF) &&
                current_subject < end_subject &&
-               RAWUCHARTEST(current_subject) == CHAR_NL)
+               UCHAR21TEST(current_subject) == CHAR_NL)
             current_subject++;
           }
         }
 
-      /* Or to a non-unique first char after study */
+      /* Advance to a non-unique first pcre_uchar after study */
 
       else if (start_bits != NULL)
         {
         while (current_subject < end_subject)
           {
-          register pcre_uint32 c = RAWUCHARTEST(current_subject);
+          register pcre_uint32 c = UCHAR21TEST(current_subject);
 #ifndef COMPILE_PCRE8
           if (c > 255) c = 255;
 #endif
-          if ((start_bits[c/8] & (1 << (c&7))) == 0)
-            {
-            current_subject++;
-#if defined SUPPORT_UTF && defined COMPILE_PCRE8
-            /* In non 8-bit mode, the iteration will stop for
-            characters > 255 at the beginning or not stop at all. */
-            if (utf)
-              ACROSSCHAR(current_subject < end_subject, *current_subject,
-                current_subject++);
-#endif
-            }
-          else break;
+          if ((start_bits[c/8] & (1 << (c&7))) != 0) break;
+          current_subject++;
           }
         }
       }
@@ -3491,19 +3556,20 @@ for (;;)
       /* If the pattern was studied, a minimum subject length may be set. This
       is a lower bound; no actual string of that length may actually match the
       pattern. Although the value is, strictly, in characters, we treat it as
-      bytes to avoid spending too much time in this optimization. */
+      in pcre_uchar units to avoid spending too much time in this optimization.
+      */
 
       if (study != NULL && (study->flags & PCRE_STUDY_MINLEN) != 0 &&
           (pcre_uint32)(end_subject - current_subject) < study->minlength)
         return PCRE_ERROR_NOMATCH;
 
-      /* If req_char is set, we know that that character must appear in the
-      subject for the match to succeed. If the first character is set, req_char
-      must be later in the subject; otherwise the test starts at the match
-      point. This optimization can save a huge amount of work in patterns with
-      nested unlimited repeats that aren't going to match. Writing separate
-      code for cased/caseless versions makes it go faster, as does using an
-      autoincrement and backing off on a match.
+      /* If req_char is set, we know that that pcre_uchar must appear in the
+      subject for the match to succeed. If the first pcre_uchar is set,
+      req_char must be later in the subject; otherwise the test starts at the
+      match point. This optimization can save a huge amount of work in patterns
+      with nested unlimited repeats that aren't going to match. Writing
+      separate code for cased/caseless versions makes it go faster, as does
+      using an autoincrement and backing off on a match.
 
       HOWEVER: when the subject string is very, very long, searching to its end
       can take a long time, and give bad performance on quite ordinary
@@ -3523,7 +3589,7 @@ for (;;)
             {
             while (p < end_subject)
               {
-              register pcre_uint32 pp = RAWUCHARINCTEST(p);
+              register pcre_uint32 pp = UCHAR21INCTEST(p);
               if (pp == req_char || pp == req_char2) { p--; break; }
               }
             }
@@ -3531,18 +3597,18 @@ for (;;)
             {
             while (p < end_subject)
               {
-              if (RAWUCHARINCTEST(p) == req_char) { p--; break; }
+              if (UCHAR21INCTEST(p) == req_char) { p--; break; }
               }
             }
 
-          /* If we can't find the required character, break the matching loop,
+          /* If we can't find the required pcre_uchar, break the matching loop,
           which will cause a return or PCRE_ERROR_NOMATCH. */
 
           if (p >= end_subject) break;
 
-          /* If we have found the required character, save the point where we
+          /* If we have found the required pcre_uchar, save the point where we
           found it, so that we don't search again next time round the loop if
-          the start hasn't passed this character yet. */
+          the start hasn't passed this point yet. */
 
           req_char_ptr = p;
           }
@@ -3599,9 +3665,9 @@ for (;;)
   not contain any explicit matches for \r or \n, and the newline option is CRLF
   or ANY or ANYCRLF, advance the match position by one more character. */
 
-  if (RAWUCHARTEST(current_subject - 1) == CHAR_CR &&
+  if (UCHAR21TEST(current_subject - 1) == CHAR_CR &&
       current_subject < end_subject &&
-      RAWUCHARTEST(current_subject) == CHAR_NL &&
+      UCHAR21TEST(current_subject) == CHAR_NL &&
       (re->flags & PCRE_HASCRORLF) == 0 &&
         (md->nltype == NLTYPE_ANY ||
          md->nltype == NLTYPE_ANYCRLF ||
diff --git a/erts/emulator/pcre/pcre_exec.c b/erts/emulator/pcre/pcre_exec.c
index 1cab78cdd8..9dcfadffde 100644
--- a/erts/emulator/pcre/pcre_exec.c
+++ b/erts/emulator/pcre/pcre_exec.c
@@ -6,7 +6,7 @@
 and semantics are as close as possible to those of the Perl 5 language.
 
                        Written by Philip Hazel
-           Copyright (c) 1997-2013 University of Cambridge
+           Copyright (c) 1997-2014 University of Cambridge
 
 -----------------------------------------------------------------------------
 Redistribution and use in source and binary forms, with or without
@@ -111,8 +111,8 @@ because the offset vector is always a multiple of 3 long. */
 
 /* Min and max values for the common repeats; for the maxima, 0 => infinity */
 
-static const char rep_min[] = { 0, 0, 1, 1, 0, 0 };
-static const char rep_max[] = { 0, 0, 0, 0, 1, 1 };
+static const char rep_min[] = { 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, };
+static const char rep_max[] = { 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, };
 
 #ifdef PCRE_DEBUG
 /*************************************************
@@ -138,7 +138,7 @@ pcre_uint32 c;
 BOOL utf = md->utf;
 if (is_subject && length > md->end_subject - p) length = md->end_subject - p;
 while (length-- > 0)
-  if (isprint(c = RAWUCHARINCTEST(p))) printf("%c", (char)c); else printf("\\x{%02x}", c);
+  if (isprint(c = UCHAR21INCTEST(p))) printf("%c", (char)c); else printf("\\x{%02x}", c);
 }
 #endif
 
@@ -187,7 +187,7 @@ match_ref(int offset, register PCRE_PUCHAR eptr, int length, match_data *md,
 {
 PCRE_PUCHAR eptr_start = eptr;
 register PCRE_PUCHAR p = md->start_subject + md->offset_vector[offset];
-#ifdef SUPPORT_UTF
+#if defined SUPPORT_UTF && defined SUPPORT_UCP
 BOOL utf = md->utf;
 #endif
 
@@ -215,8 +215,7 @@ ASCII characters. */
 
 if (caseless)
   {
-#ifdef SUPPORT_UTF
-#ifdef SUPPORT_UCP
+#if defined SUPPORT_UTF && defined SUPPORT_UCP
   if (utf)
     {
     /* Match characters up to the end of the reference. NOTE: the number of
@@ -249,7 +248,6 @@ if (caseless)
       }
     }
   else
-#endif
 #endif
 
   /* The same code works when not in UTF-8 mode and in UTF-8 mode when there
@@ -259,8 +257,8 @@ if (caseless)
       {
       pcre_uint32 cc, cp;
       if (eptr >= md->end_subject) return -2;   /* Partial match */
-      cc = RAWUCHARTEST(eptr);
-      cp = RAWUCHARTEST(p);
+      cc = UCHAR21TEST(eptr);
+      cp = UCHAR21TEST(p);
       if (TABLE_GET(cp, md->lcc, cp) != TABLE_GET(cc, md->lcc, cc)) return -1;
       p++;
       eptr++;
@@ -276,7 +274,7 @@ else
   while (length-- > 0)
     {
     if (eptr >= md->end_subject) return -2;   /* Partial match */
-    if (RAWUCHARINCTEST(p) != RAWUCHARINCTEST(eptr)) return -1;
+    if (UCHAR21INCTEST(p) != UCHAR21INCTEST(eptr)) return -1;
     }
   }
 
@@ -332,17 +330,13 @@ enum { RM1=1, RM2,  RM3,  RM4,  RM5,  RM6,  RM7,  RM8,  RM9,  RM10,
        RM31,  RM32, RM33, RM34, RM35, RM36, RM37, RM38, RM39, RM40,
        RM41,  RM42, RM43, RM44, RM45, RM46, RM47, RM48, RM49, RM50,
        RM51,  RM52, RM53, RM54, RM55, RM56, RM57, RM58, RM59, RM60,
-       RM61,  RM62, RM63, RM64, RM65, RM66, RM67, RM68 
-};
+       RM61,  RM62, RM63, RM64, RM65, RM66, RM67 };
 
 /* These versions of the macros use the stack, as normal. There are debugging
 versions and production versions. Note that the "rw" argument of RMATCH isn't
 actually used in this definition. */
 
 #ifndef NO_RECURSE
-#ifdef ERLANG_INTEGRATION
-#error ERLANG_INTEGRATION only together with NO_RECURSE
-#endif
 #define REGISTER register
 
 #ifdef PCRE_DEBUG
@@ -997,7 +991,7 @@ for (;;)
     /* Continue as from after the group, updating the offsets high water
     mark, since extracts may have been taken. */
 
-    do ecode += GET(ecode, 1); while (*ecode == OP_ALT);
+    do ecode += GET(ecode, 1); while (*ecode == OP_ALT); /* LOOP_COUNT: Ok */
 
     offset_top = md->end_offset_top;
     eptr = md->end_match_ptr;
@@ -1192,7 +1186,7 @@ for (;;)
           const pcre_uchar *scode = ecode;
           if (*scode != OP_ONCE)           /* If not at start, find it */
             {
-            while (*scode == OP_ALT) scode += GET(scode, 1);
+            while (*scode == OP_ALT) scode += GET(scode, 1); /* LOOP_COUNT: Ok */
             scode -= GET(scode, 1);
             }
           if (md->once_target == scode) rrc = MATCH_NOMATCH;
@@ -1230,87 +1224,81 @@ for (;;)
     printf("\n");
 #endif
 
-    if (offset < md->offset_max)
-      {
-      matched_once = FALSE;
-      code_offset = (int)(ecode - md->start_code);
-
-      save_offset1 = md->offset_vector[offset];
-      save_offset2 = md->offset_vector[offset+1];
-      save_offset3 = md->offset_vector[md->offset_end - number];
-      save_capture_last = md->capture_last;
+    if (offset >= md->offset_max) goto POSSESSIVE_NON_CAPTURE;
 
-      DPRINTF(("saving %d %d %d\n", save_offset1, save_offset2, save_offset3));
+    matched_once = FALSE;
+    code_offset = (int)(ecode - md->start_code);
 
-      /* Each time round the loop, save the current subject position for use
-      when the group matches. For MATCH_MATCH, the group has matched, so we
-      restart it with a new subject starting position, remembering that we had
-      at least one match. For MATCH_NOMATCH, carry on with the alternatives, as
-      usual. If we haven't matched any alternatives in any iteration, check to
-      see if a previous iteration matched. If so, the group has matched;
-      continue from afterwards. Otherwise it has failed; restore the previous
-      capture values before returning NOMATCH. */
+    save_offset1 = md->offset_vector[offset];
+    save_offset2 = md->offset_vector[offset+1];
+    save_offset3 = md->offset_vector[md->offset_end - number];
+    save_capture_last = md->capture_last;
 
-      for (;;) /* LOOP_COUNT: Ok */
-        {
-        md->offset_vector[md->offset_end - number] =
-          (int)(eptr - md->start_subject);
-        if (op >= OP_SBRA) md->match_function_type = MATCH_CBEGROUP;
-        RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md,
-          eptrb, RM63);
-        if (rrc == MATCH_KETRPOS)
-          {
-          offset_top = md->end_offset_top;
-          eptr = md->end_match_ptr;
-          ecode = md->start_code + code_offset;
-          save_capture_last = md->capture_last;
-          matched_once = TRUE;
-          continue;
-          }
+    DPRINTF(("saving %d %d %d\n", save_offset1, save_offset2, save_offset3));
 
-        /* See comment in the code for capturing groups above about handling
-        THEN. */
+    /* Each time round the loop, save the current subject position for use
+    when the group matches. For MATCH_MATCH, the group has matched, so we
+    restart it with a new subject starting position, remembering that we had
+    at least one match. For MATCH_NOMATCH, carry on with the alternatives, as
+    usual. If we haven't matched any alternatives in any iteration, check to
+    see if a previous iteration matched. If so, the group has matched;
+    continue from afterwards. Otherwise it has failed; restore the previous
+    capture values before returning NOMATCH. */
 
-        if (rrc == MATCH_THEN)
+    for (;;) /* LOOP_COUNT: Ok */
+      {
+      md->offset_vector[md->offset_end - number] =
+        (int)(eptr - md->start_subject);
+      if (op >= OP_SBRA) md->match_function_type = MATCH_CBEGROUP;
+      RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md,
+        eptrb, RM63);
+      if (rrc == MATCH_KETRPOS)
+        {
+        offset_top = md->end_offset_top;
+        ecode = md->start_code + code_offset;
+        save_capture_last = md->capture_last;
+        matched_once = TRUE;
+        mstart = md->start_match_ptr;    /* In case \K changed it */
+        if (eptr == md->end_match_ptr)   /* Matched an empty string */
           {
-          next = ecode + GET(ecode,1);
-          if (md->start_match_ptr < next &&
-              (*ecode == OP_ALT || *next == OP_ALT))
-            rrc = MATCH_NOMATCH;
+          do ecode += GET(ecode, 1); while (*ecode == OP_ALT); /* LOOP_COUNT: Ok */
+          break;
           }
-
-        if (rrc != MATCH_NOMATCH) RRETURN(rrc);
-        md->capture_last = save_capture_last;
-        ecode += GET(ecode, 1);
-        if (*ecode != OP_ALT) break;
+        eptr = md->end_match_ptr;
+        continue;
         }
 
-      if (!matched_once)
-        {
-        md->offset_vector[offset] = save_offset1;
-        md->offset_vector[offset+1] = save_offset2;
-        md->offset_vector[md->offset_end - number] = save_offset3;
-        }
+      /* See comment in the code for capturing groups above about handling
+      THEN. */
 
-      if (allow_zero || matched_once)
+      if (rrc == MATCH_THEN)
         {
-        ecode += 1 + LINK_SIZE;
-        break;
+        next = ecode + GET(ecode,1);
+        if (md->start_match_ptr < next &&
+            (*ecode == OP_ALT || *next == OP_ALT))
+          rrc = MATCH_NOMATCH;
         }
 
-      RRETURN(MATCH_NOMATCH);
+      if (rrc != MATCH_NOMATCH) RRETURN(rrc);
+      md->capture_last = save_capture_last;
+      ecode += GET(ecode, 1);
+      if (*ecode != OP_ALT) break;
       }
 
-    /* FALL THROUGH ... Insufficient room for saving captured contents. Treat
-    as a non-capturing bracket. */
-
-    /* VVVVVVVVVVVVVVVVVVVVVVVVV */
-    /* VVVVVVVVVVVVVVVVVVVVVVVVV */
+    if (!matched_once)
+      {
+      md->offset_vector[offset] = save_offset1;
+      md->offset_vector[offset+1] = save_offset2;
+      md->offset_vector[md->offset_end - number] = save_offset3;
+      }
 
-    DPRINTF(("insufficient capture room: treat as non-capturing\n"));
+    if (allow_zero || matched_once)
+      {
+      ecode += 1 + LINK_SIZE;
+      break;
+      }
 
-    /* VVVVVVVVVVVVVVVVVVVVVVVVV */
-    /* VVVVVVVVVVVVVVVVVVVVVVVVV */
+    RRETURN(MATCH_NOMATCH);
 
     /* Non-capturing possessive bracket with unlimited repeat. We come here
     from BRAZERO with allow_zero = TRUE. The code is similar to the above,
@@ -1334,9 +1322,15 @@ for (;;)
       if (rrc == MATCH_KETRPOS)
         {
         offset_top = md->end_offset_top;
-        eptr = md->end_match_ptr;
         ecode = md->start_code + code_offset;
         matched_once = TRUE;
+        mstart = md->start_match_ptr;   /* In case \K reset it */
+        if (eptr == md->end_match_ptr)  /* Matched an empty string */
+          {
+          do ecode += GET(ecode, 1); while (*ecode == OP_ALT); /* LOOP_COUNT: Ok */
+          break;
+          }
+        eptr = md->end_match_ptr;
         continue;
         }
 
@@ -1366,25 +1360,32 @@ for (;;)
 
     /* Control never reaches here. */
 
-    /* Conditional group: compilation checked that there are no more than
-    two branches. If the condition is false, skipping the first branch takes us
-    past the end if there is only one branch, but that's OK because that is
-    exactly what going to the ket would do. */
+    /* Conditional group: compilation checked that there are no more than two
+    branches. If the condition is false, skipping the first branch takes us
+    past the end of the item if there is only one branch, but that's exactly
+    what we want. */
 
     case OP_COND:
     case OP_SCOND:
-    codelink = GET(ecode, 1);
+
+    /* The variable codelink will be added to ecode when the condition is
+    false, to get to the second branch. Setting it to the offset to the ALT
+    or KET, then incrementing ecode achieves this effect. We now have ecode
+    pointing to the condition or callout. */
+
+    codelink = GET(ecode, 1);   /* Offset to the second branch */
+    ecode += 1 + LINK_SIZE;     /* From this opcode */
 
     /* Because of the way auto-callout works during compile, a callout item is
     inserted between OP_COND and an assertion condition. */
 
-    if (ecode[LINK_SIZE+1] == OP_CALLOUT)
+    if (*ecode == OP_CALLOUT)
       {
       if (PUBL(callout) != NULL)
         {
         PUBL(callout_block) cb;
         cb.version          = 2;   /* Version 1 of the callout block */
-        cb.callout_number   = ecode[LINK_SIZE+2];
+        cb.callout_number   = ecode[1];
         cb.offset_vector    = md->offset_vector;
 #if defined COMPILE_PCRE8
         cb.subject          = (PCRE_SPTR)md->start_subject;
@@ -1396,8 +1397,8 @@ for (;;)
         cb.subject_length   = (int)(md->end_subject - md->start_subject);
         cb.start_match      = (int)(mstart - md->start_subject);
         cb.current_position = (int)(eptr - md->start_subject);
-        cb.pattern_position = GET(ecode, LINK_SIZE + 3);
-        cb.next_item_length = GET(ecode, 3 + 2*LINK_SIZE);
+        cb.pattern_position = GET(ecode, 2);
+        cb.next_item_length = GET(ecode, 2 + LINK_SIZE);
         cb.capture_top      = offset_top/2;
         cb.capture_last     = md->capture_last & CAPLMASK;
         /* Internal change requires this for API compatibility. */
@@ -1407,213 +1408,125 @@ for (;;)
         if ((rrc = (*PUBL(callout))(&cb)) > 0) RRETURN(MATCH_NOMATCH);
         if (rrc < 0) RRETURN(rrc);
         }
+
+      /* Advance ecode past the callout, so it now points to the condition. We
+      must adjust codelink so that the value of ecode+codelink is unchanged. */
+
       ecode += PRIV(OP_lengths)[OP_CALLOUT];
       codelink -= PRIV(OP_lengths)[OP_CALLOUT];
       }
 
-    condcode = ecode[LINK_SIZE+1];
+    /* Test the various possible conditions */
 
-    /* Now see what the actual condition is */
-
-    if (condcode == OP_RREF || condcode == OP_NRREF)    /* Recursion test */
+    condition = FALSE;
+    switch(condcode = *ecode)
       {
-      if (md->recursive == NULL)                /* Not recursing => FALSE */
+      case OP_RREF:         /* Numbered group recursion test */
+      if (md->recursive != NULL)     /* Not recursing => FALSE */
         {
-        condition = FALSE;
-        ecode += GET(ecode, 1);
-        }
-      else
-        { /* LOOP_COUNT: Warning, No CHK in this block */
-        unsigned int recno = GET2(ecode, LINK_SIZE + 2);   /* Recursion group number*/
+        unsigned int recno = GET2(ecode, 1);   /* Recursion group number*/
         condition = (recno == RREF_ANY || recno == md->recursive->group_num);
+        }
+      break;
 
-        /* If the test is for recursion into a specific subpattern, and it is
-        false, but the test was set up by name, scan the table to see if the
-        name refers to any other numbers, and test them. The condition is true
-        if any one is set. */
-
-        if (!condition && condcode == OP_NRREF)
+      case OP_DNRREF:       /* Duplicate named group recursion test */
+      if (md->recursive != NULL)
+        {
+        int count = GET2(ecode, 1 + IMM2_SIZE);
+        pcre_uchar *slot = md->name_table + GET2(ecode, 1) * md->name_entry_size;
+        while (count-- > 0) /* LOOP_COUNT: COST */
           {
-          pcre_uchar *slotA = md->name_table;
-          for (i = 0; i < md->name_count; i++)/* LOOP_COUNT: COST */
-            {
-            if (GET2(slotA, 0) == recno) break;
-            slotA += md->name_entry_size;
-	    COST(1);
-            }
-
-          /* Found a name for the number - there can be only one; duplicate
-          names for different numbers are allowed, but not vice versa. First
-          scan down for duplicates. */
-
-          if (i < md->name_count)
-            {
-            pcre_uchar *slotB = slotA;
-            while (slotB > md->name_table) /* LOOP_COUNT: COST */
-              {
-              slotB -= md->name_entry_size;
-              if (STRCMP_UC_UC(slotA + IMM2_SIZE, slotB + IMM2_SIZE) == 0)
-                {
-                condition = GET2(slotB, 0) == md->recursive->group_num;
-                if (condition) break;
-                }
-              else break;
-	      COST(1);
-              }
-
-            /* Scan up for duplicates */
-
-            if (!condition)
-              {
-              slotB = slotA;
-              for (i++; i < md->name_count; i++)/* LOOP_COUNT: COST */
-                {
-                slotB += md->name_entry_size;
-                if (STRCMP_UC_UC(slotA + IMM2_SIZE, slotB + IMM2_SIZE) == 0)
-                  {
-                  condition = GET2(slotB, 0) == md->recursive->group_num;
-                  if (condition) break;
-                  }
-                else break;
-		COST(1);
-                }
-              }
-            }
+          unsigned int recno = GET2(slot, 0);
+          condition = recno == md->recursive->group_num;
+          if (condition) break;
+          slot += md->name_entry_size;
+	  COST(1);
           }
-
-        /* Chose branch according to the condition */
-
-        ecode += condition? 1 + IMM2_SIZE : GET(ecode, 1);
         }
-      }
+      break;
 
-    else if (condcode == OP_CREF || condcode == OP_NCREF)  /* Group used test */
-      {
-      offset = GET2(ecode, LINK_SIZE+2) << 1;  /* Doubled ref number */
+      case OP_CREF:         /* Numbered group used test */
+      offset = GET2(ecode, 1) << 1;  /* Doubled ref number */
       condition = offset < offset_top && md->offset_vector[offset] >= 0;
+      break;
 
-      /* If the numbered capture is unset, but the reference was by name,
-      scan the table to see if the name refers to any other numbers, and test
-      them. The condition is true if any one is set. This is tediously similar
-      to the code above, but not close enough to try to amalgamate. */
-
-      if (!condition && condcode == OP_NCREF)
+      case OP_DNCREF:      /* Duplicate named group used test */
         {
-        unsigned int refno = offset >> 1;
-        pcre_uchar *slotA = md->name_table;/* LOOP_COUNT: Warning, no CHK in this block */
-
-        for (i = 0; i < md->name_count; i++) /* LOOP_COUNT: COST */
+        int count = GET2(ecode, 1 + IMM2_SIZE);
+        pcre_uchar *slot = md->name_table + GET2(ecode, 1) * md->name_entry_size;
+        while (count-- > 0) /* LOOP_COUNT: COST */
           {
-          if (GET2(slotA, 0) == refno) break;
-          slotA += md->name_entry_size;
+          offset = GET2(slot, 0) << 1;
+          condition = offset < offset_top && md->offset_vector[offset] >= 0;
+          if (condition) break;
+          slot += md->name_entry_size;
 	  COST(1);
           }
-
-        /* Found a name for the number - there can be only one; duplicate names
-        for different numbers are allowed, but not vice versa. First scan down
-        for duplicates. */
-
-        if (i < md->name_count)
-          {
-          pcre_uchar *slotB = slotA;
-          while (slotB > md->name_table) /* LOOP_COUNT: COST */
-            {
-            slotB -= md->name_entry_size;
-            if (STRCMP_UC_UC(slotA + IMM2_SIZE, slotB + IMM2_SIZE) == 0)
-              {
-              offset = GET2(slotB, 0) << 1;
-              condition = offset < offset_top &&
-                md->offset_vector[offset] >= 0;
-              if (condition) break;
-              }
-            else break;
-	    COST(1);
-            }
-
-          /* Scan up for duplicates */
-
-          if (!condition)
-            {
-            slotB = slotA;
-            for (i++; i < md->name_count; i++) /* LOOP_COUNT: COST */
-              {
-              slotB += md->name_entry_size;
-              if (STRCMP_UC_UC(slotA + IMM2_SIZE, slotB + IMM2_SIZE) == 0)
-                {
-                offset = GET2(slotB, 0) << 1;
-                condition = offset < offset_top &&
-                  md->offset_vector[offset] >= 0;
-                if (condition) break;
-                }
-              else break;
-	      COST(1);
-              }
-            }
-          }
         }
+      break;
 
-      /* Chose branch according to the condition */
-
-      ecode += condition? 1 + IMM2_SIZE : GET(ecode, 1);
-      }
-
-    else if (condcode == OP_DEF)     /* DEFINE - always false */
-      {
-      condition = FALSE;
-      ecode += GET(ecode, 1);
-      }
+      case OP_DEF:     /* DEFINE - always false */
+      case OP_FAIL:    /* From optimized (?!) condition */
+      break;
 
-    /* The condition is an assertion. Call match() to evaluate it - setting
-    md->match_function_type to MATCH_CONDASSERT causes it to stop at the end of
-    an assertion. */
+      /* The condition is an assertion. Call match() to evaluate it - setting
+      md->match_function_type to MATCH_CONDASSERT causes it to stop at the end
+      of an assertion. */
 
-    else
-      {
+      default:
       md->match_function_type = MATCH_CONDASSERT;
-      RMATCH(eptr, ecode + 1 + LINK_SIZE, offset_top, md, NULL, RM3);
+      RMATCH(eptr, ecode, offset_top, md, NULL, RM3);
       if (rrc == MATCH_MATCH)
         {
         if (md->end_offset_top > offset_top)
           offset_top = md->end_offset_top;  /* Captures may have happened */
         condition = TRUE;
-        ecode += 1 + LINK_SIZE + GET(ecode, LINK_SIZE + 2);
+
+        /* Advance ecode past the assertion to the start of the first branch,
+        but adjust it so that the general choosing code below works. If the
+        assertion has a quantifier that allows zero repeats we must skip over
+        the BRAZERO. This is a lunatic thing to do, but somebody did! */
+
+        if (*ecode == OP_BRAZERO) ecode++;
+        ecode += GET(ecode, 1);
         while (*ecode == OP_ALT) ecode += GET(ecode, 1); /* LOOP_COUNT: Ok */
+        ecode += 1 + LINK_SIZE - PRIV(OP_lengths)[condcode];
         }
 
       /* PCRE doesn't allow the effect of (*THEN) to escape beyond an
-      assertion; it is therefore treated as NOMATCH. */
+      assertion; it is therefore treated as NOMATCH. Any other return is an
+      error. */
 
       else if (rrc != MATCH_NOMATCH && rrc != MATCH_THEN)
         {
         RRETURN(rrc);         /* Need braces because of following else */
         }
-      else
-        {
-        condition = FALSE;
-        ecode += codelink;
-        }
+      break;
       }
 
-    /* We are now at the branch that is to be obeyed. As there is only one, can
-    use tail recursion to avoid using another stack frame, except when there is
-    unlimited repeat of a possibly empty group. In the latter case, a recursive
-    call to match() is always required, unless the second alternative doesn't
-    exist, in which case we can just plough on. Note that, for compatibility
-    with Perl, the | in a conditional group is NOT treated as creating two
-    alternatives. If a THEN is encountered in the branch, it propagates out to
-    the enclosing alternative (unless nested in a deeper set of alternatives,
-    of course). */
-
-    if (condition || *ecode == OP_ALT)
+    /* Choose branch according to the condition */
+
+    ecode += condition? PRIV(OP_lengths)[condcode] : codelink;
+
+    /* We are now at the branch that is to be obeyed. As there is only one, we
+    can use tail recursion to avoid using another stack frame, except when
+    there is unlimited repeat of a possibly empty group. In the latter case, a
+    recursive call to match() is always required, unless the second alternative
+    doesn't exist, in which case we can just plough on. Note that, for
+    compatibility with Perl, the | in a conditional group is NOT treated as
+    creating two alternatives. If a THEN is encountered in the branch, it
+    propagates out to the enclosing alternative (unless nested in a deeper set
+    of alternatives, of course). */
+
+    if (condition || ecode[-(1+LINK_SIZE)] == OP_ALT)
       {
       if (op != OP_SCOND)
         {
-        ecode += 1 + LINK_SIZE;
         goto TAIL_RECURSE;
         }
 
       md->match_function_type = MATCH_CBEGROUP;
-      RMATCH(eptr, ecode + 1 + LINK_SIZE, offset_top, md, eptrb, RM49);
+      RMATCH(eptr, ecode, offset_top, md, eptrb, RM49);
       RRETURN(rrc);
       }
 
@@ -1621,7 +1534,6 @@ for (;;)
 
     else
       {
-      ecode += 1 + LINK_SIZE;
       }
     break;
 
@@ -1644,7 +1556,22 @@ for (;;)
       md->offset_vector[offset] =
         md->offset_vector[md->offset_end - number];
       md->offset_vector[offset+1] = (int)(eptr - md->start_subject);
-      if (offset_top <= offset) offset_top = offset + 2;
+
+      /* If this group is at or above the current highwater mark, ensure that
+      any groups between the current high water mark and this group are marked
+      unset and then update the high water mark. */
+
+      if (offset >= offset_top)
+        {
+        register int *iptr = md->offset_vector + offset_top;
+        register int *iend = md->offset_vector + offset;
+        if (iptr < iend)
+          {
+          COST(iend - iptr);
+          while (iptr < iend) *iptr++ = -1; /* LOOP_COUNT: COST */
+          }
+        offset_top = offset + 2;
+        }
       }
     ecode += 1 + IMM2_SIZE;
     break;
@@ -2000,7 +1927,11 @@ for (;;)
         are defined in a range that can be tested for. */
 
         if (rrc >= MATCH_BACKTRACK_MIN && rrc <= MATCH_BACKTRACK_MAX)
+          {
+          if (new_recursive.offset_save != stacksave)
+            (PUBL(free))(new_recursive.offset_save);
           RRETURN(MATCH_NOMATCH);
+          }
 
         /* Any return code other than NOMATCH is an error. */
 
@@ -2151,7 +2082,11 @@ for (;;)
           {
           register int *iptr = md->offset_vector + offset_top;
           register int *iend = md->offset_vector + offset;
-          while (iptr < iend) *iptr++ = -1; /* LOOP_COUNT: CHK */
+          if (iptr < iend)
+            {
+            COST(iend - iptr);
+            while (iptr < iend) *iptr++ = -1; /* LOOP_COUNT: COST */
+            }
           }
 
         /* Now make the extraction */
@@ -2163,6 +2098,19 @@ for (;;)
         }
       }
 
+    /* OP_KETRPOS is a possessive repeating ket. Remember the current position,
+    and return the MATCH_KETRPOS. This makes it possible to do the repeats one
+    at a time from the outer level, thus saving stack. This must precede the
+    empty string test - in this case that test is done at the outer level. */
+
+    if (*ecode == OP_KETRPOS)
+      {
+      md->start_match_ptr = mstart;    /* In case \K reset it */
+      md->end_match_ptr = eptr;
+      md->end_offset_top = offset_top;
+      RRETURN(MATCH_KETRPOS);
+      }
+
     /* For an ordinary non-repeating ket, just continue at this level. This
     also happens for a repeating ket if no characters were matched in the
     group. This is the forcible breaking of infinite loops as implemented in
@@ -2185,17 +2133,6 @@ for (;;)
       break;
       }
 
-    /* OP_KETRPOS is a possessive repeating ket. Remember the current position,
-    and return the MATCH_KETRPOS. This makes it possible to do the repeats one
-    at a time from the outer level, thus saving stack. */
-
-    if (*ecode == OP_KETRPOS)
-      {
-      md->end_match_ptr = eptr;
-      md->end_offset_top = offset_top;
-      RRETURN(MATCH_KETRPOS);
-      }
-
     /* The normal repeating kets try the rest of the pattern or restart from
     the preceding bracket, in the appropriate order. In the second case, we can
     use tail recursion to avoid using another stack frame, unless we have an
@@ -2286,7 +2223,7 @@ for (;;)
             eptr + 1 >= md->end_subject &&
             NLBLOCK->nltype == NLTYPE_FIXED &&
             NLBLOCK->nllen == 2 &&
-            RAWUCHARTEST(eptr) == NLBLOCK->nl[0])
+            UCHAR21TEST(eptr) == NLBLOCK->nl[0])
           {
           md->hitend = TRUE;
           if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL);
@@ -2330,7 +2267,7 @@ for (;;)
           eptr + 1 >= md->end_subject &&
           NLBLOCK->nltype == NLTYPE_FIXED &&
           NLBLOCK->nllen == 2 &&
-          RAWUCHARTEST(eptr) == NLBLOCK->nl[0])
+          UCHAR21TEST(eptr) == NLBLOCK->nl[0])
         {
         md->hitend = TRUE;
         if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL);
@@ -2473,7 +2410,7 @@ for (;;)
         eptr + 1 >= md->end_subject &&
         NLBLOCK->nltype == NLTYPE_FIXED &&
         NLBLOCK->nllen == 2 &&
-        RAWUCHARTEST(eptr) == NLBLOCK->nl[0])
+        UCHAR21TEST(eptr) == NLBLOCK->nl[0])
       {
       md->hitend = TRUE;
       if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL);
@@ -2627,7 +2564,7 @@ for (;;)
         {
         SCHECK_PARTIAL();
         }
-      else if (RAWUCHARTEST(eptr) == CHAR_LF) eptr++;
+      else if (UCHAR21TEST(eptr) == CHAR_LF) eptr++;
       break;
 
       case CHAR_LF:
@@ -2758,19 +2695,24 @@ for (;;)
           RRETURN(MATCH_NOMATCH);
         break;
 
-        case PT_SPACE:    /* Perl space */
-        if ((PRIV(ucp_gentype)[prop->chartype] == ucp_Z ||
-             c == CHAR_HT || c == CHAR_NL || c == CHAR_FF || c == CHAR_CR)
-               == (op == OP_NOTPROP))
-          RRETURN(MATCH_NOMATCH);
-        break;
+        /* Perl space used to exclude VT, but from Perl 5.18 it is included,
+        which means that Perl space and POSIX space are now identical. PCRE
+        was changed at release 8.34. */
 
+        case PT_SPACE:    /* Perl space */
         case PT_PXSPACE:  /* POSIX space */
-        if ((PRIV(ucp_gentype)[prop->chartype] == ucp_Z ||
-             c == CHAR_HT || c == CHAR_NL || c == CHAR_VT ||
-             c == CHAR_FF || c == CHAR_CR)
-               == (op == OP_NOTPROP))
-          RRETURN(MATCH_NOMATCH);
+        switch(c)
+          {
+          HSPACE_CASES:
+          VSPACE_CASES:
+          if (op == OP_NOTPROP) RRETURN(MATCH_NOMATCH);
+          break;
+
+          default:
+          if ((PRIV(ucp_gentype)[prop->chartype] == ucp_Z) ==
+            (op == OP_NOTPROP)) RRETURN(MATCH_NOMATCH);
+          break;
+          }
         break;
 
         case PT_WORD:
@@ -2848,15 +2790,7 @@ for (;;)
     similar code to character type repeats - written out again for speed.
     However, if the referenced string is the empty string, always treat
     it as matched, any number of times (otherwise there could be infinite
-    loops). */
-
-    case OP_REF:
-    case OP_REFI:
-    caseless = op == OP_REFI;
-    offset = GET2(ecode, 1) << 1;               /* Doubled ref number */
-    ecode += 1 + IMM2_SIZE;
-
-    /* If the reference is unset, there are two possibilities:
+    loops). If the reference is unset, there are two possibilities:
 
     (a) In the default, Perl-compatible state, set the length negative;
     this ensures that every attempt at a match fails. We can't just fail
@@ -2866,8 +2800,46 @@ for (;;)
     so that the back reference matches an empty string.
 
     Otherwise, set the length to the length of what was matched by the
-    referenced subpattern. */
+    referenced subpattern.
+
+    The OP_REF and OP_REFI opcodes are used for a reference to a numbered group
+    or to a non-duplicated named group. For a duplicated named group, OP_DNREF
+    and OP_DNREFI are used. In this case we must scan the list of groups to
+    which the name refers, and use the first one that is set. */
+
+    case OP_DNREF:
+    case OP_DNREFI:
+    caseless = op == OP_DNREFI;
+      {
+      int count = GET2(ecode, 1+IMM2_SIZE);
+      pcre_uchar *slot = md->name_table + GET2(ecode, 1) * md->name_entry_size;
+      ecode += 1 + 2*IMM2_SIZE;
+
+      /* Setting the default length first and initializing 'offset' avoids
+      compiler warnings in the REF_REPEAT code. */
+
+      length = (md->jscript_compat)? 0 : -1;
+      offset = 0;
 
+      while (count-- > 0) /* LOOP_COUNT: COST */
+        {
+        offset = GET2(slot, 0) << 1;
+        if (offset < offset_top && md->offset_vector[offset] >= 0)
+          {
+          length = md->offset_vector[offset+1] - md->offset_vector[offset];
+          break;
+          }
+        slot += md->name_entry_size;
+        }
+        COST(1);
+      }
+    goto REF_REPEAT;
+
+    case OP_REF:
+    case OP_REFI:
+    caseless = op == OP_REFI;
+    offset = GET2(ecode, 1) << 1;               /* Doubled ref number */
+    ecode += 1 + IMM2_SIZE;
     if (offset >= offset_top || md->offset_vector[offset] < 0)
       length = (md->jscript_compat)? 0 : -1;
     else
@@ -2875,6 +2847,7 @@ for (;;)
 
     /* Set up for repetition, or handle the non-repeated case */
 
+    REF_REPEAT:
     switch (*ecode)
       {
       case OP_CRSTAR:
@@ -3027,8 +3000,12 @@ for (;;)
         case OP_CRMINPLUS:
         case OP_CRQUERY:
         case OP_CRMINQUERY:
+        case OP_CRPOSSTAR:
+        case OP_CRPOSPLUS:
+        case OP_CRPOSQUERY:
         c = *ecode++ - OP_CRSTAR;
-        minimize = (c & 1) != 0;
+        if (c < OP_CRPOSSTAR - OP_CRSTAR) minimize = (c & 1) != 0;
+        else possessive = TRUE;
         min = rep_min[c];                 /* Pick up values from tables; */
         max = rep_max[c];                 /* zero for max => infinity */
         if (max == 0) max = INT_MAX;
@@ -3036,7 +3013,9 @@ for (;;)
 
         case OP_CRRANGE:
         case OP_CRMINRANGE:
+        case OP_CRPOSRANGE:
         minimize = (*ecode == OP_CRMINRANGE);
+        possessive = (*ecode == OP_CRPOSRANGE);
         min = GET2(ecode, 1);
         max = GET2(ecode, 1 + IMM2_SIZE);
         if (max == 0) max = INT_MAX;
@@ -3181,6 +3160,9 @@ for (;;)
             eptr += len;
 	    COST_CHK(1);
             }
+
+          if (possessive) continue;    /* No backtracking */
+
           for (;;) /* LOOP_COUNT: Ok */
             {
             RMATCH(eptr, ecode, offset_top, md, eptrb, RM18);
@@ -3212,7 +3194,10 @@ for (;;)
 	    COST_CHK(1);
             eptr++;
             }
-          while (eptr >= pp)  /* LOOP_COUNT: Ok */
+
+          if (possessive) continue;    /* No backtracking */
+
+          while (eptr >= pp) /* LOOP_COUNT: Ok */
             {
             RMATCH(eptr, ecode, offset_top, md, eptrb, RM19);
             if (rrc != MATCH_NOMATCH) RRETURN(rrc);
@@ -3227,9 +3212,10 @@ for (;;)
     /* Control never gets here */
 
 
-    /* Match an extended character class. This opcode is encountered only
-    when UTF-8 mode mode is supported. Nevertheless, we may not be in UTF-8
-    mode, because Unicode properties are supported in non-UTF-8 mode. */
+    /* Match an extended character class. In the 8-bit library, this opcode is
+    encountered only when UTF-8 mode mode is supported. In the 16-bit and
+    32-bit libraries, codepoints greater than 255 may be encountered even when
+    UTF is not supported. */
 
 #if defined SUPPORT_UTF || !defined COMPILE_PCRE8
     case OP_XCLASS:
@@ -3245,8 +3231,12 @@ for (;;)
         case OP_CRMINPLUS:
         case OP_CRQUERY:
         case OP_CRMINQUERY:
+        case OP_CRPOSSTAR:
+        case OP_CRPOSPLUS:
+        case OP_CRPOSQUERY:
         c = *ecode++ - OP_CRSTAR;
-        minimize = (c & 1) != 0;
+        if (c < OP_CRPOSSTAR - OP_CRSTAR) minimize = (c & 1) != 0;
+        else possessive = TRUE;
         min = rep_min[c];                 /* Pick up values from tables; */
         max = rep_max[c];                 /* zero for max => infinity */
         if (max == 0) max = INT_MAX;
@@ -3254,7 +3244,9 @@ for (;;)
 
         case OP_CRRANGE:
         case OP_CRMINRANGE:
+        case OP_CRPOSRANGE:
         minimize = (*ecode == OP_CRMINRANGE);
+        possessive = (*ecode == OP_CRPOSRANGE);
         min = GET2(ecode, 1);
         max = GET2(ecode, 1 + IMM2_SIZE);
         if (max == 0) max = INT_MAX;
@@ -3327,6 +3319,9 @@ for (;;)
           eptr += len;
 	  COST_CHK(1);
           }
+
+        if (possessive) continue;    /* No backtracking */
+
         for(;;) /* LOOP_COUNT: Ok */
           {
           RMATCH(eptr, ecode, offset_top, md, eptrb, RM21);
@@ -3357,7 +3352,7 @@ for (;;)
         CHECK_PARTIAL();             /* Not SCHECK_PARTIAL() */
         RRETURN(MATCH_NOMATCH);
         }
-      while (length-- > 0) if (*ecode++ != RAWUCHARINC(eptr)) RRETURN(MATCH_NOMATCH);  /* LOOP_COUNT: Ok */
+      while (length-- > 0) if (*ecode++ != UCHAR21INC(eptr)) RRETURN(MATCH_NOMATCH); /* LOOP_COUNT: Ok */
       }
     else
 #endif
@@ -3398,7 +3393,7 @@ for (;;)
 
       if (fc < 128)
         {
-        pcre_uint32 cc = RAWUCHAR(eptr);
+        pcre_uint32 cc = UCHAR21(eptr);
         if (md->lcc[fc] != TABLE_GET(cc, md->lcc, cc)) RRETURN(MATCH_NOMATCH);
         ecode++;
         eptr++;
@@ -3606,7 +3601,7 @@ for (;;)
           if (possessive) continue;    /* No backtracking */
           for(;;) /* LOOP_COUNT: Ok */
             {
-            if (eptr == pp) goto TAIL_RECURSE;
+            if (eptr <= pp) goto TAIL_RECURSE;
             RMATCH(eptr, ecode, offset_top, md, eptrb, RM23);
             if (rrc != MATCH_NOMATCH) RRETURN(rrc);
 #ifdef SUPPORT_UCP
@@ -3668,7 +3663,7 @@ for (;;)
           SCHECK_PARTIAL();
           RRETURN(MATCH_NOMATCH);
           }
-        cc = RAWUCHARTEST(eptr);
+        cc = UCHAR21TEST(eptr);
         if (fc != cc && foc != cc) RRETURN(MATCH_NOMATCH);
         eptr++;
 	COST_CHK(1);
@@ -3687,7 +3682,7 @@ for (;;)
             SCHECK_PARTIAL();
             RRETURN(MATCH_NOMATCH);
             }
-          cc = RAWUCHARTEST(eptr);
+          cc = UCHAR21TEST(eptr);
           if (fc != cc && foc != cc) RRETURN(MATCH_NOMATCH);
           eptr++;
           }
@@ -3704,12 +3699,11 @@ for (;;)
             SCHECK_PARTIAL();
             break;
             }
-          cc = RAWUCHARTEST(eptr);
+          cc = UCHAR21TEST(eptr);
           if (fc != cc && foc != cc) break;
           eptr++;
 	  COST_CHK(1);
           }
-
         if (possessive) continue;       /* No backtracking */
         for (;;)  /* LOOP_COUNT: Ok */
           {
@@ -3718,9 +3712,8 @@ for (;;)
           eptr--;
           if (rrc != MATCH_NOMATCH) RRETURN(rrc);
           }
-        RRETURN(MATCH_NOMATCH);
+        /* Control never gets here */
         }
-      /* Control never gets here */
       }
 
     /* Caseful comparisons (includes all multi-byte characters) */
@@ -3735,7 +3728,7 @@ for (;;)
           SCHECK_PARTIAL();
           RRETURN(MATCH_NOMATCH);
           }
-        if (fc != RAWUCHARINCTEST(eptr)) RRETURN(MATCH_NOMATCH);
+        if (fc != UCHAR21INCTEST(eptr)) RRETURN(MATCH_NOMATCH);
         }
 
       if (min == max) continue;
@@ -3752,7 +3745,7 @@ for (;;)
             SCHECK_PARTIAL();
             RRETURN(MATCH_NOMATCH);
             }
-          if (fc != RAWUCHARINCTEST(eptr)) RRETURN(MATCH_NOMATCH);
+          if (fc != UCHAR21INCTEST(eptr)) RRETURN(MATCH_NOMATCH);
           }
         /* Control never gets here */
         }
@@ -3766,7 +3759,7 @@ for (;;)
             SCHECK_PARTIAL();
             break;
             }
-          if (fc != RAWUCHARTEST(eptr)) break;
+          if (fc != UCHAR21TEST(eptr)) break;
           eptr++;
 	  COST_CHK(1);
           }
@@ -3778,7 +3771,7 @@ for (;;)
           eptr--;
           if (rrc != MATCH_NOMATCH) RRETURN(rrc);
           }
-        RRETURN(MATCH_NOMATCH);
+        /* Control never gets here */
         }
       }
     /* Control never gets here */
@@ -4036,7 +4029,7 @@ for (;;)
           if (possessive) continue;    /* No backtracking */
           for(;;)  /* LOOP_COUNT: Ok */
             {
-            if (eptr == pp) goto TAIL_RECURSE;
+            if (eptr <= pp) goto TAIL_RECURSE;
             RMATCH(eptr, ecode, offset_top, md, eptrb, RM30);
             if (rrc != MATCH_NOMATCH) RRETURN(rrc);
             eptr--;
@@ -4067,10 +4060,8 @@ for (;;)
             eptr--;
             }
           }
-
-        RRETURN(MATCH_NOMATCH);
+        /* Control never gets here */
         }
-      /* Control never gets here */
       }
 
     /* Caseful comparisons */
@@ -4098,7 +4089,7 @@ for (;;)
       /* Not UTF mode */
         {
 	COST(min);
-        for (i = 1; i <= min; i++) /* LOOP_COUNT: Ok */
+        for (i = 1; i <= min; i++) /* LOOP_COUNT: Cost */
           {
           if (eptr >= md->end_subject)
             {
@@ -4177,7 +4168,7 @@ for (;;)
           if (possessive) continue;    /* No backtracking */
           for(;;)  /* LOOP_COUNT: Ok */
             {
-            if (eptr == pp) goto TAIL_RECURSE;
+            if (eptr <= pp) goto TAIL_RECURSE;
             RMATCH(eptr, ecode, offset_top, md, eptrb, RM34);
             if (rrc != MATCH_NOMATCH) RRETURN(rrc);
             eptr--;
@@ -4208,8 +4199,7 @@ for (;;)
             eptr--;
             }
           }
-
-        RRETURN(MATCH_NOMATCH);
+        /* Control never gets here */
         }
       }
     /* Control never gets here */
@@ -4392,7 +4382,12 @@ for (;;)
             }
           break;
 
+          /* Perl space used to exclude VT, but from Perl 5.18 it is included,
+          which means that Perl space and POSIX space are now identical. PCRE
+          was changed at release 8.34. */
+
           case PT_SPACE:    /* Perl space */
+          case PT_PXSPACE:  /* POSIX space */
           for (i = 1; i <= min; i++) /* LOOP_COUNT: COST (above) */
             {
             if (eptr >= md->end_subject)
@@ -4401,26 +4396,18 @@ for (;;)
               RRETURN(MATCH_NOMATCH);
               }
             GETCHARINCTEST(c, eptr);
-            if ((UCD_CATEGORY(c) == ucp_Z || c == CHAR_HT || c == CHAR_NL ||
-                 c == CHAR_FF || c == CHAR_CR)
-                   == prop_fail_result)
-              RRETURN(MATCH_NOMATCH);
-            }
-          break;
-
-          case PT_PXSPACE:  /* POSIX space */
-          for (i = 1; i <= min; i++) /* LOOP_COUNT: COST (above) */
-            {
-            if (eptr >= md->end_subject)
+            switch(c)
               {
-              SCHECK_PARTIAL();
-              RRETURN(MATCH_NOMATCH);
+              HSPACE_CASES:
+              VSPACE_CASES:
+              if (prop_fail_result) RRETURN(MATCH_NOMATCH);
+              break;
+
+              default:
+              if ((UCD_CATEGORY(c) == ucp_Z) == prop_fail_result)
+                RRETURN(MATCH_NOMATCH);
+              break;
               }
-            GETCHARINCTEST(c, eptr);
-            if ((UCD_CATEGORY(c) == ucp_Z || c == CHAR_HT || c == CHAR_NL ||
-                 c == CHAR_VT || c == CHAR_FF || c == CHAR_CR)
-                   == prop_fail_result)
-              RRETURN(MATCH_NOMATCH);
             }
           break;
 
@@ -4543,7 +4530,7 @@ for (;;)
               eptr + 1 >= md->end_subject &&
               NLBLOCK->nltype == NLTYPE_FIXED &&
               NLBLOCK->nllen == 2 &&
-              RAWUCHAR(eptr) == NLBLOCK->nl[0])
+              UCHAR21(eptr) == NLBLOCK->nl[0])
             {
             md->hitend = TRUE;
             if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL);
@@ -4587,7 +4574,7 @@ for (;;)
             default: RRETURN(MATCH_NOMATCH);
 
             case CHAR_CR:
-            if (eptr < md->end_subject && RAWUCHAR(eptr) == CHAR_LF) eptr++;
+            if (eptr < md->end_subject && UCHAR21(eptr) == CHAR_LF) eptr++;
             break;
 
             case CHAR_LF:
@@ -4703,7 +4690,7 @@ for (;;)
             SCHECK_PARTIAL();
             RRETURN(MATCH_NOMATCH);
             }
-          cc = RAWUCHAR(eptr);
+          cc = UCHAR21(eptr);
           if (cc >= 128 || (md->ctypes[cc] & ctype_digit) == 0)
             RRETURN(MATCH_NOMATCH);
           eptr++;
@@ -4721,7 +4708,7 @@ for (;;)
             SCHECK_PARTIAL();
             RRETURN(MATCH_NOMATCH);
             }
-          cc = RAWUCHAR(eptr);
+          cc = UCHAR21(eptr);
           if (cc < 128 && (md->ctypes[cc] & ctype_space) != 0)
             RRETURN(MATCH_NOMATCH);
           eptr++;
@@ -4739,7 +4726,7 @@ for (;;)
             SCHECK_PARTIAL();
             RRETURN(MATCH_NOMATCH);
             }
-          cc = RAWUCHAR(eptr);
+          cc = UCHAR21(eptr);
           if (cc >= 128 || (md->ctypes[cc] & ctype_space) == 0)
             RRETURN(MATCH_NOMATCH);
           eptr++;
@@ -4757,7 +4744,7 @@ for (;;)
             SCHECK_PARTIAL();
             RRETURN(MATCH_NOMATCH);
             }
-          cc = RAWUCHAR(eptr);
+          cc = UCHAR21(eptr);
           if (cc < 128 && (md->ctypes[cc] & ctype_word) != 0)
             RRETURN(MATCH_NOMATCH);
           eptr++;
@@ -4775,7 +4762,7 @@ for (;;)
             SCHECK_PARTIAL();
             RRETURN(MATCH_NOMATCH);
             }
-          cc = RAWUCHAR(eptr);
+          cc = UCHAR21(eptr);
           if (cc >= 128 || (md->ctypes[cc] & ctype_word) == 0)
             RRETURN(MATCH_NOMATCH);
           eptr++;
@@ -5170,25 +5157,11 @@ for (;;)
             }
           /* Control never gets here */
 
-          case PT_SPACE:    /* Perl space */
-          for (fi = min;; fi++) /* LOOP_COUNT: Ok */
-            {
-            RMATCH(eptr, ecode, offset_top, md, eptrb, RM60);
-            if (rrc != MATCH_NOMATCH) RRETURN(rrc);
-            if (fi >= max) RRETURN(MATCH_NOMATCH);
-            if (eptr >= md->end_subject)
-              {
-              SCHECK_PARTIAL();
-              RRETURN(MATCH_NOMATCH);
-              }
-            GETCHARINCTEST(c, eptr);
-            if ((UCD_CATEGORY(c) == ucp_Z || c == CHAR_HT || c == CHAR_NL ||
-                 c == CHAR_FF || c == CHAR_CR)
-                   == prop_fail_result)
-              RRETURN(MATCH_NOMATCH);
-            }
-          /* Control never gets here */
+          /* Perl space used to exclude VT, but from Perl 5.18 it is included,
+          which means that Perl space and POSIX space are now identical. PCRE
+          was changed at release 8.34. */
 
+          case PT_SPACE:    /* Perl space */
           case PT_PXSPACE:  /* POSIX space */
           for (fi = min;; fi++) /* LOOP_COUNT: Ok */
             {
@@ -5201,10 +5174,18 @@ for (;;)
               RRETURN(MATCH_NOMATCH);
               }
             GETCHARINCTEST(c, eptr);
-            if ((UCD_CATEGORY(c) == ucp_Z || c == CHAR_HT || c == CHAR_NL ||
-                 c == CHAR_VT || c == CHAR_FF || c == CHAR_CR)
-                   == prop_fail_result)
-              RRETURN(MATCH_NOMATCH);
+            switch(c)
+              {
+              HSPACE_CASES:
+              VSPACE_CASES:
+              if (prop_fail_result) RRETURN(MATCH_NOMATCH);
+              break;
+
+              default:
+              if ((UCD_CATEGORY(c) == ucp_Z) == prop_fail_result)
+                RRETURN(MATCH_NOMATCH);
+              break;
+              }
             }
           /* Control never gets here */
 
@@ -5258,7 +5239,7 @@ for (;;)
           case PT_UCNC:
           for (fi = min;; fi++) /* LOOP_COUNT: Ok */
             {
-            RMATCH(eptr, ecode, offset_top, md, eptrb, RM68);
+            RMATCH(eptr, ecode, offset_top, md, eptrb, RM60);
             if (rrc != MATCH_NOMATCH) RRETURN(rrc);
             if (fi >= max) RRETURN(MATCH_NOMATCH);
             if (eptr >= md->end_subject)
@@ -5358,7 +5339,7 @@ for (;;)
               {
               default: RRETURN(MATCH_NOMATCH);
               case CHAR_CR:
-              if (eptr < md->end_subject && RAWUCHAR(eptr) == CHAR_LF) eptr++;
+              if (eptr < md->end_subject && UCHAR21(eptr) == CHAR_LF) eptr++;
               break;
 
               case CHAR_LF:
@@ -5698,7 +5679,12 @@ for (;;)
             }
           break;
 
+          /* Perl space used to exclude VT, but from Perl 5.18 it is included,
+          which means that Perl space and POSIX space are now identical. PCRE
+          was changed at release 8.34. */
+
           case PT_SPACE:    /* Perl space */
+          case PT_PXSPACE:  /* POSIX space */
           for (i = min; i < max; i++) /* LOOP_COUNT: CHK */
             {
             int len = 1;
@@ -5708,32 +5694,22 @@ for (;;)
               break;
               }
             GETCHARLENTEST(c, eptr, len);
-            if ((UCD_CATEGORY(c) == ucp_Z || c == CHAR_HT || c == CHAR_NL ||
-                 c == CHAR_FF || c == CHAR_CR)
-                 == prop_fail_result)
+            switch(c)
+              {
+              HSPACE_CASES:
+              VSPACE_CASES:
+              if (prop_fail_result) goto ENDLOOP99;  /* Break the loop */
               break;
-            eptr+= len;
-	    COST_CHK(1);
-            }
-          break;
 
-          case PT_PXSPACE:  /* POSIX space */
-          for (i = min; i < max; i++) /* LOOP_COUNT: CHK */
-            {
-            int len = 1;
-            if (eptr >= md->end_subject)
-              {
-              SCHECK_PARTIAL();
+              default:
+              if ((UCD_CATEGORY(c) == ucp_Z) == prop_fail_result)
+                goto ENDLOOP99;   /* Break the loop */
               break;
               }
-            GETCHARLENTEST(c, eptr, len);
-            if ((UCD_CATEGORY(c) == ucp_Z || c == CHAR_HT || c == CHAR_NL ||
-                 c == CHAR_VT || c == CHAR_FF || c == CHAR_CR)
-                 == prop_fail_result)
-              break;
             eptr+= len;
 	    COST_CHK(1);
             }
+          ENDLOOP99:
           break;
 
           case PT_WORD:
@@ -5741,7 +5717,7 @@ for (;;)
             {
             int category;
             int len = 1;
-            if (eptr >= md->end_subject) 
+            if (eptr >= md->end_subject)
               {
               SCHECK_PARTIAL();
               break;
@@ -5810,7 +5786,7 @@ for (;;)
         if (possessive) continue;    /* No backtracking */
         for(;;) /* LOOP_COUNT: Ok */
           {
-          if (eptr == pp) goto TAIL_RECURSE;
+          if (eptr <= pp) goto TAIL_RECURSE;
           RMATCH(eptr, ecode, offset_top, md, eptrb, RM44);
           if (rrc != MATCH_NOMATCH) RRETURN(rrc);
           eptr--;
@@ -5855,22 +5831,27 @@ for (;;)
         /* eptr is now past the end of the maximum run */
 
         if (possessive) continue;    /* No backtracking */
-         
+
+        /* We use <= pp rather than == pp to detect the start of the run while
+        backtracking because the use of \C in UTF mode can cause BACKCHAR to
+        move back past pp. This is just palliative; the use of \C in UTF mode
+        is fraught with danger. */
+
         for(;;) /* LOOP_COUNT: Ok */
           {
 #ifndef ERLANG_INTEGRATION
-          int lgb, rgb; 
+          int lgb, rgb;
 #endif
           PCRE_PUCHAR fptr;
-            
-          if (eptr == pp) goto TAIL_RECURSE;   /* At start of char run */
+
+          if (eptr <= pp) goto TAIL_RECURSE;   /* At start of char run */
           RMATCH(eptr, ecode, offset_top, md, eptrb, RM45);
           if (rrc != MATCH_NOMATCH) RRETURN(rrc);
 
           /* Backtracking over an extended grapheme cluster involves inspecting
           the previous two characters (if present) to see if a break is
           permitted between them. */
- 
+
           eptr--;
           if (!utf) c = *eptr; else
             {
@@ -5881,14 +5862,14 @@ for (;;)
 
           for (;;) /* LOOP_COUNT: COST */
             {
-            if (eptr == pp) goto TAIL_RECURSE;   /* At start of char run */
+            if (eptr <= pp) goto TAIL_RECURSE;   /* At start of char run */
             fptr = eptr - 1;
             if (!utf) c = *fptr; else
               {
               BACKCHAR(fptr);
               GETCHAR(c, fptr);
               }
-            lgb = UCD_GRAPHBREAK(c);        
+            lgb = UCD_GRAPHBREAK(c);
             if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break;
             eptr = fptr;
             rgb = lgb;
@@ -5906,56 +5887,26 @@ for (;;)
         switch(ctype)
           {
           case OP_ANY:
-          if (max < INT_MAX)
+          for (i = min; i < max; i++) /* LOOP_COUNT: CHK */
             {
-            for (i = min; i < max; i++) /* LOOP_COUNT: CHK */
+            if (eptr >= md->end_subject)
               {
-              if (eptr >= md->end_subject)
-                {
-                SCHECK_PARTIAL();
-                break;
-                }
-              if (IS_NEWLINE(eptr)) break;
-              if (md->partial != 0 &&    /* Take care with CRLF partial */
-                  eptr + 1 >= md->end_subject &&
-                  NLBLOCK->nltype == NLTYPE_FIXED &&
-                  NLBLOCK->nllen == 2 &&
-                  RAWUCHAR(eptr) == NLBLOCK->nl[0])
-                {
-                md->hitend = TRUE;
-                if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL);
-                }
-              eptr++;
-              ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++);
-	      COST_CHK(1);
+              SCHECK_PARTIAL();
+              break;
               }
-            }
-
-          /* Handle unlimited UTF-8 repeat */
-
-          else
-            {
-            for (i = min; i < max; i++) /* LOOP_COUNT: CHK */
+            if (IS_NEWLINE(eptr)) break;
+            if (md->partial != 0 &&    /* Take care with CRLF partial */
+                eptr + 1 >= md->end_subject &&
+                NLBLOCK->nltype == NLTYPE_FIXED &&
+                NLBLOCK->nllen == 2 &&
+                UCHAR21(eptr) == NLBLOCK->nl[0])
               {
-              if (eptr >= md->end_subject)
-                {
-                SCHECK_PARTIAL();
-                break;
-                }
-              if (IS_NEWLINE(eptr)) break;
-              if (md->partial != 0 &&    /* Take care with CRLF partial */
-                  eptr + 1 >= md->end_subject &&
-                  NLBLOCK->nltype == NLTYPE_FIXED &&
-                  NLBLOCK->nllen == 2 &&
-                  RAWUCHAR(eptr) == NLBLOCK->nl[0])
-                {
-                md->hitend = TRUE;
-                if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL);
-                }
-              eptr++;
-              ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++);
-	      COST_CHK(1);
+              md->hitend = TRUE;
+              if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL);
               }
+            eptr++;
+            ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++);
+	    COST_CHK(1);
             }
           break;
 
@@ -6006,7 +5957,7 @@ for (;;)
             if (c == CHAR_CR)
               {
               if (++eptr >= md->end_subject) break;
-              if (RAWUCHAR(eptr) == CHAR_LF) eptr++;
+              if (UCHAR21(eptr) == CHAR_LF) eptr++;
               }
             else
               {
@@ -6173,13 +6124,13 @@ for (;;)
         if (possessive) continue;    /* No backtracking */
         for(;;) /* LOOP_COUNT: Ok */
           {
-          if (eptr == pp) goto TAIL_RECURSE;
+          if (eptr <= pp) goto TAIL_RECURSE;
           RMATCH(eptr, ecode, offset_top, md, eptrb, RM46);
           if (rrc != MATCH_NOMATCH) RRETURN(rrc);
           eptr--;
           BACKCHAR(eptr);
-          if (ctype == OP_ANYNL && eptr > pp  && RAWUCHAR(eptr) == CHAR_NL &&
-              RAWUCHAR(eptr - 1) == CHAR_CR) eptr--;
+          if (ctype == OP_ANYNL && eptr > pp  && UCHAR21(eptr) == CHAR_NL &&
+              UCHAR21(eptr - 1) == CHAR_CR) eptr--;
           }
         }
       else
@@ -6438,11 +6389,8 @@ for (;;)
           }
         }
 
-      /* Get here if we can't make it match with any permitted repetitions */
-
-      RRETURN(MATCH_NOMATCH);
+      /* Control never gets here */
       }
-    /* Control never gets here */
 
     /* There's been some horrible disaster. Arrival here can only mean there is
     something seriously wrong in the code above or the OP_xxx definitions. */
@@ -6476,15 +6424,15 @@ switch (frame->Xwhere)
   LBL(53) LBL(54) LBL(55) LBL(56) LBL(57) LBL(58) LBL(63) LBL(64)
   LBL(65) LBL(66)
 #if defined SUPPORT_UTF || !defined COMPILE_PCRE8
-  LBL(21)
+  LBL(20) LBL(21)
 #endif
 #ifdef SUPPORT_UTF
-  LBL(16) LBL(18) LBL(20)
+  LBL(16) LBL(18)
   LBL(22) LBL(23) LBL(28) LBL(30)
   LBL(32) LBL(34) LBL(42) LBL(46)
 #ifdef SUPPORT_UCP
   LBL(36) LBL(37) LBL(38) LBL(39) LBL(40) LBL(41) LBL(44) LBL(45)
-  LBL(59) LBL(60) LBL(61) LBL(62) LBL(67) LBL(68)
+  LBL(59) LBL(60) LBL(61) LBL(62) LBL(67)
 #endif  /* SUPPORT_UCP */
 #endif  /* SUPPORT_UTF */
   default:
@@ -6747,7 +6695,7 @@ const pcre_uint8 *start_bits = NULL;
 PCRE_PUCHAR start_match = (PCRE_PUCHAR)subject + start_offset;
 PCRE_PUCHAR end_subject;
 PCRE_PUCHAR start_partial = NULL;
-PCRE_PUCHAR match_partial;
+PCRE_PUCHAR match_partial = NULL;
 PCRE_PUCHAR req_char_ptr = start_match - 1;
 
 const pcre_study_data *study;
@@ -6852,6 +6800,7 @@ pcre_uchar req_char;
    start_bits = NULL;
    start_match = (PCRE_PUCHAR)subject + start_offset;
    start_partial = NULL;
+   match_partial = NULL;
    req_char_ptr = start_match - 1;
    re = (REAL_PCRE *)argument_re;
 
@@ -6988,7 +6937,7 @@ tables = re->tables;
 
 if (extra_data != NULL)
   {
-  register unsigned int flags = extra_data->flags;
+  unsigned long int flags = extra_data->flags;
   if ((flags & PCRE_EXTRA_STUDY_DATA) != 0)
     study = (const pcre_study_data *)extra_data->study_data;
   if ((flags & PCRE_EXTRA_MATCH_LIMIT) != 0)
@@ -7166,7 +7115,8 @@ if (md->offset_vector != NULL)
   register int *iend = iptr - re->top_bracket;
   if (iend < md->offset_vector + 2) iend = md->offset_vector + 2;
   while (--iptr >= iend) *iptr = -1;
-  md->offset_vector[0] = md->offset_vector[1] = -1;
+  if (offsetcount > 0) md->offset_vector[0] = -1;
+  if (offsetcount > 1) md->offset_vector[1] = -1;
   }
 
 /* Set up the first character to match, if available. The first_char value is
@@ -7264,10 +7214,10 @@ for(;;)
 
       if (first_char != first_char2)
         while (start_match < end_subject &&
-          (smc = RAWUCHARTEST(start_match)) != first_char && smc != first_char2)
+          (smc = UCHAR21TEST(start_match)) != first_char && smc != first_char2)
           start_match++;
       else
-        while (start_match < end_subject && RAWUCHARTEST(start_match) != first_char)
+        while (start_match < end_subject && UCHAR21TEST(start_match) != first_char)
           start_match++;
       }
 
@@ -7299,7 +7249,7 @@ for(;;)
         if (start_match[-1] == CHAR_CR &&
              (md->nltype == NLTYPE_ANY || md->nltype == NLTYPE_ANYCRLF) &&
              start_match < end_subject &&
-             RAWUCHARTEST(start_match) == CHAR_NL)
+             UCHAR21TEST(start_match) == CHAR_NL)
           start_match++;
         }
       }
@@ -7310,33 +7260,22 @@ for(;;)
       {
       while (start_match < end_subject)
         {
-        register pcre_uint32 c = RAWUCHARTEST(start_match);
+        register pcre_uint32 c = UCHAR21TEST(start_match);
 #ifndef COMPILE_PCRE8
         if (c > 255) c = 255;
 #endif
-        if ((start_bits[c/8] & (1 << (c&7))) == 0)
-          {
-          start_match++;
-#if defined SUPPORT_UTF && defined COMPILE_PCRE8
-          /* In non 8-bit mode, the iteration will stop for
-          characters > 255 at the beginning or not stop at all. */
-          if (utf)
-            ACROSSCHAR(start_match < end_subject, *start_match,
-              start_match++);
-#endif
-          }
-#ifdef ERLANG_INTEGRATION
-	else {
-	if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) 
+        if ((start_bits[c/8] & (1 << (c&7))) != 0)
 	  {
-	  *extra_data->loop_counter_return = 
-	    (extra_data->loop_limit - md->loop_limit);
-          }
-        break;
-	}
-#else
-        else break;
+#ifdef ERLANG_INTEGRATION
+          if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) 
+            {
+            *extra_data->loop_counter_return = 
+               (extra_data->loop_limit - md->loop_limit);
+            }
 #endif
+          break;
+	  }
+        start_match++;
         }
       }
     }   /* Starting optimizations */
@@ -7396,7 +7335,7 @@ for(;;)
           {
           while (p < end_subject)
             {
-            register pcre_uint32 pp = RAWUCHARINCTEST(p);
+            register pcre_uint32 pp = UCHAR21INCTEST(p);
             if (pp == req_char || pp == req_char2) { p--; break; }
             }
           }
@@ -7404,7 +7343,7 @@ for(;;)
           {
           while (p < end_subject)
             {
-            if (RAWUCHARINCTEST(p) == req_char) { p--; break; }
+            if (UCHAR21INCTEST(p) == req_char) { p--; break; }
             }
           }
 
@@ -7684,7 +7623,7 @@ if (rc != MATCH_NOMATCH && rc != PCRE_ERROR_PARTIAL)
 
 /* Handle partial matches - disable any mark data */
 
-if (start_partial != NULL)
+if (match_partial != NULL)
   {
   DPRINTF((">>>> returning PCRE_ERROR_PARTIAL\n"));
   md->mark = NULL;
diff --git a/erts/emulator/pcre/pcre_fullinfo.c b/erts/emulator/pcre/pcre_fullinfo.c
index 1636c5978f..4e22430a1c 100644
--- a/erts/emulator/pcre/pcre_fullinfo.c
+++ b/erts/emulator/pcre/pcre_fullinfo.c
@@ -239,6 +239,10 @@ switch (what)
   *((pcre_uint32 *)where) = re->limit_recursion;
   break;
 
+  case PCRE_INFO_MATCH_EMPTY:
+  *((int *)where) = (re->flags & PCRE_MATCH_EMPTY) != 0;
+  break;
+
   default: return PCRE_ERROR_BADOPTION;
   }
 
diff --git a/erts/emulator/pcre/pcre_get.c b/erts/emulator/pcre/pcre_get.c
index c0bb21bbbf..86ab8fe5c5 100644
--- a/erts/emulator/pcre/pcre_get.c
+++ b/erts/emulator/pcre/pcre_get.c
@@ -284,6 +284,7 @@ Arguments:
   code         the compiled regex
   stringname   the name of the capturing substring
   ovector      the vector of matched substrings
+  stringcount  number of captured substrings
 
 Returns:       the number of the first that is set,
                or the number of the last one if none are set,
@@ -292,13 +293,16 @@ Returns:       the number of the first that is set,
 
 #if defined COMPILE_PCRE8
 static int
-get_first_set(const pcre *code, const char *stringname, int *ovector)
+get_first_set(const pcre *code, const char *stringname, int *ovector,
+  int stringcount)
 #elif defined COMPILE_PCRE16
 static int
-get_first_set(const pcre16 *code, PCRE_SPTR16 stringname, int *ovector)
+get_first_set(const pcre16 *code, PCRE_SPTR16 stringname, int *ovector,
+  int stringcount)
 #elif defined COMPILE_PCRE32
 static int
-get_first_set(const pcre32 *code, PCRE_SPTR32 stringname, int *ovector)
+get_first_set(const pcre32 *code, PCRE_SPTR32 stringname, int *ovector,
+  int stringcount)
 #endif
 {
 const REAL_PCRE *re = (const REAL_PCRE *)code;
@@ -335,7 +339,7 @@ if (entrysize <= 0) return entrysize;
 for (entry = (pcre_uchar *)first; entry <= (pcre_uchar *)last; entry += entrysize)
   {
   int n = GET2(entry, 0);
-  if (ovector[n*2] >= 0) return n;
+  if (n < stringcount && ovector[n*2] >= 0) return n;
   }
 return GET2(entry, 0);
 }
@@ -455,7 +459,7 @@ pcre32_copy_named_substring(const pcre32 *code, PCRE_SPTR32 subject,
   PCRE_UCHAR32 *buffer, int size)
 #endif
 {
-int n = get_first_set(code, stringname, ovector);
+int n = get_first_set(code, stringname, ovector, stringcount);
 if (n <= 0) return n;
 #if defined COMPILE_PCRE8
 #if defined(ERLANG_INTEGRATION)
@@ -520,7 +524,10 @@ pcre_uchar **stringlist;
 pcre_uchar *p;
 
 for (i = 0; i < double_count; i += 2)
-  size += sizeof(pcre_uchar *) + IN_UCHARS(ovector[i+1] - ovector[i] + 1);
+  {
+  size += sizeof(pcre_uchar *) + IN_UCHARS(1);
+  if (ovector[i+1] > ovector[i]) size += IN_UCHARS(ovector[i+1] - ovector[i]);
+  }
 
 stringlist = (pcre_uchar **)(PUBL(malloc))(size);
 if (stringlist == NULL) return PCRE_ERROR_NOMEMORY;
@@ -536,7 +543,7 @@ p = (pcre_uchar *)(stringlist + stringcount + 1);
 
 for (i = 0; i < double_count; i += 2)
   {
-  int len = ovector[i+1] - ovector[i];
+  int len = (ovector[i+1] > ovector[i])? (ovector[i+1] - ovector[i]) : 0;
   memcpy(p, subject + ovector[i], IN_UCHARS(len));
   *stringlist++ = p;
   p += len;
@@ -700,7 +707,7 @@ pcre32_get_named_substring(const pcre32 *code, PCRE_SPTR32 subject,
   PCRE_SPTR32 *stringptr)
 #endif
 {
-int n = get_first_set(code, stringname, ovector);
+int n = get_first_set(code, stringname, ovector, stringcount);
 if (n <= 0) return n;
 #if defined COMPILE_PCRE8
 #if defined(ERLANG_INTEGRATION)
diff --git a/erts/emulator/pcre/pcre_globals.c b/erts/emulator/pcre/pcre_globals.c
index ce143b8c21..4d83606a61 100644
--- a/erts/emulator/pcre/pcre_globals.c
+++ b/erts/emulator/pcre/pcre_globals.c
@@ -6,7 +6,7 @@
 and semantics are as close as possible to those of the Perl 5 language.
 
                        Written by Philip Hazel
-           Copyright (c) 1997-2012 University of Cambridge
+           Copyright (c) 1997-2014 University of Cambridge
 
 -----------------------------------------------------------------------------
 Redistribution and use in source and binary forms, with or without
@@ -74,6 +74,7 @@ PCRE_EXP_DATA_DEFN void  (*PUBL(free))(void *) = LocalPcreFree;
 PCRE_EXP_DATA_DEFN void *(*PUBL(stack_malloc))(size_t) = LocalPcreMalloc;
 PCRE_EXP_DATA_DEFN void  (*PUBL(stack_free))(void *) = LocalPcreFree;
 PCRE_EXP_DATA_DEFN int   (*PUBL(callout))(PUBL(callout_block) *) = NULL;
+PCRE_EXP_DATA_DEFN int   (*PUBL(stack_guard))(void) = NULL;
 
 #elif !defined VPCOMPAT
 PCRE_EXP_DATA_DEFN void *(*PUBL(malloc))(size_t) = malloc;
@@ -81,6 +82,7 @@ PCRE_EXP_DATA_DEFN void  (*PUBL(free))(void *) = free;
 PCRE_EXP_DATA_DEFN void *(*PUBL(stack_malloc))(size_t) = malloc;
 PCRE_EXP_DATA_DEFN void  (*PUBL(stack_free))(void *) = free;
 PCRE_EXP_DATA_DEFN int   (*PUBL(callout))(PUBL(callout_block) *) = NULL;
+PCRE_EXP_DATA_DEFN int   (*PUBL(stack_guard))(void) = NULL;
 #endif
 
 /* End of pcre_globals.c */
diff --git a/erts/emulator/pcre/pcre_internal.h b/erts/emulator/pcre/pcre_internal.h
index af436bd99b..cc4f171438 100644
--- a/erts/emulator/pcre/pcre_internal.h
+++ b/erts/emulator/pcre/pcre_internal.h
@@ -7,7 +7,7 @@
 and semantics are as close as possible to those of the Perl 5 language.
 
                        Written by Philip Hazel
-           Copyright (c) 1997-2013 University of Cambridge
+           Copyright (c) 1997-2016 University of Cambridge
 
 -----------------------------------------------------------------------------
 Redistribution and use in source and binary forms, with or without
@@ -281,7 +281,7 @@ pcre.h(.in) and disable (comment out) this message. */
 
 typedef pcre_uint16 pcre_uchar;
 #define UCHAR_SHIFT (1)
-#define IN_UCHARS(x) ((x) << UCHAR_SHIFT)
+#define IN_UCHARS(x) ((x) * 2)
 #define MAX_255(c) ((c) <= 255u)
 #define TABLE_GET(c, table, default) (MAX_255(c)? ((table)[c]):(default))
 
@@ -289,7 +289,7 @@ typedef pcre_uint16 pcre_uchar;
 
 typedef pcre_uint32 pcre_uchar;
 #define UCHAR_SHIFT (2)
-#define IN_UCHARS(x) ((x) << UCHAR_SHIFT)
+#define IN_UCHARS(x) ((x) * 4)
 #define MAX_255(c) ((c) <= 255u)
 #define TABLE_GET(c, table, default) (MAX_255(c)? ((table)[c]):(default))
 
@@ -322,8 +322,8 @@ start/end of string field names are. */
        &(NLBLOCK->nllen), utf)) \
     : \
     ((p) <= NLBLOCK->PSEND - NLBLOCK->nllen && \
-     RAWUCHARTEST(p) == NLBLOCK->nl[0] && \
-     (NLBLOCK->nllen == 1 || RAWUCHARTEST(p+1) == NLBLOCK->nl[1])       \
+     UCHAR21TEST(p) == NLBLOCK->nl[0] && \
+     (NLBLOCK->nllen == 1 || UCHAR21TEST(p+1) == NLBLOCK->nl[1])       \
     ) \
   )
 
@@ -336,8 +336,8 @@ start/end of string field names are. */
        &(NLBLOCK->nllen), utf)) \
     : \
     ((p) >= NLBLOCK->PSSTART + NLBLOCK->nllen && \
-     RAWUCHARTEST(p - NLBLOCK->nllen) == NLBLOCK->nl[0] &&              \
-     (NLBLOCK->nllen == 1 || RAWUCHARTEST(p - NLBLOCK->nllen + 1) == NLBLOCK->nl[1]) \
+     UCHAR21TEST(p - NLBLOCK->nllen) == NLBLOCK->nl[0] &&              \
+     (NLBLOCK->nllen == 1 || UCHAR21TEST(p - NLBLOCK->nllen + 1) == NLBLOCK->nl[1]) \
     ) \
   )
 
@@ -588,12 +588,27 @@ changed in future to be a fixed number of bytes or to depend on LINK_SIZE. */
 #define MAX_MARK ((1u << 8) - 1)
 #endif
 
+/* There is a proposed future special "UTF-21" mode, in which only the lowest
+21 bits of a 32-bit character are interpreted as UTF, with the remaining 11
+high-order bits available to the application for other uses. In preparation for
+the future implementation of this mode, there are macros that load a data item
+and, if in this special mode, mask it to 21 bits. These macros all have names
+starting with UCHAR21. In all other modes, including the normal 32-bit
+library, the macros all have the same simple definitions. When the new mode is
+implemented, it is expected that these definitions will be varied appropriately
+using #ifdef when compiling the library that supports the special mode. */
+
+#define UCHAR21(eptr)        (*(eptr))
+#define UCHAR21TEST(eptr)    (*(eptr))
+#define UCHAR21INC(eptr)     (*(eptr)++)
+#define UCHAR21INCTEST(eptr) (*(eptr)++)
+
 /* When UTF encoding is being used, a character is no longer just a single
-byte. The macros for character handling generate simple sequences when used in
-character-mode, and more complicated ones for UTF characters. GETCHARLENTEST
-and other macros are not used when UTF is not supported, so they are not
-defined. To make sure they can never even appear when UTF support is omitted,
-we don't even define them. */
+byte in 8-bit mode or a single short in 16-bit mode. The macros for character
+handling generate simple sequences when used in the basic mode, and more
+complicated ones for UTF characters. GETCHARLENTEST and other macros are not
+used when UTF is not supported. To make sure they can never even appear when
+UTF support is omitted, we don't even define them. */
 
 #ifndef SUPPORT_UTF
 
@@ -606,10 +621,6 @@ we don't even define them. */
 #define GETCHARINC(c, eptr) c = *eptr++;
 #define GETCHARINCTEST(c, eptr) c = *eptr++;
 #define GETCHARLEN(c, eptr, len) c = *eptr;
-#define RAWUCHAR(eptr) (*(eptr))
-#define RAWUCHARINC(eptr) (*(eptr)++)
-#define RAWUCHARTEST(eptr) (*(eptr))
-#define RAWUCHARINCTEST(eptr) (*(eptr)++)
 /* #define GETCHARLENTEST(c, eptr, len) */
 /* #define BACKCHAR(eptr) */
 /* #define FORWARDCHAR(eptr) */
@@ -782,30 +793,6 @@ do not know if we are in UTF-8 mode. */
   c = *eptr; \
   if (utf && c >= 0xc0) GETUTF8LEN(c, eptr, len);
 
-/* Returns the next uchar, not advancing the pointer. This is called when
-we know we are in UTF mode. */
-
-#define RAWUCHAR(eptr) \
-  (*(eptr))
-
-/* Returns the next uchar, advancing the pointer. This is called when
-we know we are in UTF mode. */
-
-#define RAWUCHARINC(eptr) \
-  (*((eptr)++))
-
-/* Returns the next uchar, testing for UTF mode, and not advancing the
-pointer. */
-
-#define RAWUCHARTEST(eptr) \
-  (*(eptr))
-
-/* Returns the next uchar, testing for UTF mode, advancing the
-pointer. */
-
-#define RAWUCHARINCTEST(eptr) \
-  (*((eptr)++))
-
 /* If the pointer is not at the start of a character, move it back until
 it is. This is called only in UTF-8 mode - we don't put a test within the macro
 because almost all calls are already within a block of UTF-8 only code. */
@@ -901,30 +888,6 @@ we do not know if we are in UTF-16 mode. */
   c = *eptr; \
   if (utf && (c & 0xfc00) == 0xd800) GETUTF16LEN(c, eptr, len);
 
-/* Returns the next uchar, not advancing the pointer. This is called when
-we know we are in UTF mode. */
-
-#define RAWUCHAR(eptr) \
-  (*(eptr))
-
-/* Returns the next uchar, advancing the pointer. This is called when
-we know we are in UTF mode. */
-
-#define RAWUCHARINC(eptr) \
-  (*((eptr)++))
-
-/* Returns the next uchar, testing for UTF mode, and not advancing the
-pointer. */
-
-#define RAWUCHARTEST(eptr) \
-  (*(eptr))
-
-/* Returns the next uchar, testing for UTF mode, advancing the
-pointer. */
-
-#define RAWUCHARINCTEST(eptr) \
-  (*((eptr)++))
-
 /* If the pointer is not at the start of a character, move it back until
 it is. This is called only in UTF-16 mode - we don't put a test within the
 macro because almost all calls are already within a block of UTF-16 only
@@ -986,30 +949,6 @@ This is called when we do not know if we are in UTF-32 mode. */
 #define GETCHARLENTEST(c, eptr, len) \
   GETCHARTEST(c, eptr)
 
-/* Returns the next uchar, not advancing the pointer. This is called when
-we know we are in UTF mode. */
-
-#define RAWUCHAR(eptr) \
-  (*(eptr))
-
-/* Returns the next uchar, advancing the pointer. This is called when
-we know we are in UTF mode. */
-
-#define RAWUCHARINC(eptr) \
-  (*((eptr)++))
-
-/* Returns the next uchar, testing for UTF mode, and not advancing the
-pointer. */
-
-#define RAWUCHARTEST(eptr) \
-  (*(eptr))
-
-/* Returns the next uchar, testing for UTF mode, advancing the
-pointer. */
-
-#define RAWUCHARINCTEST(eptr) \
-  (*((eptr)++))
-
 /* If the pointer is not at the start of a character, move it back until
 it is. This is called only in UTF-32 mode - we don't put a test within the
 macro because almost all calls are already within a block of UTF-32 only
@@ -1051,7 +990,7 @@ other. NOTE: The values also appear in pcre_jit_compile.c. */
 #ifndef EBCDIC
 
 #define HSPACE_LIST \
-  CHAR_HT, CHAR_SPACE, 0xa0, \
+  CHAR_HT, CHAR_SPACE, CHAR_NBSP, \
   0x1680, 0x180e, 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, \
   0x2006, 0x2007, 0x2008, 0x2009, 0x200A, 0x202f, 0x205f, 0x3000, \
   NOTACHAR
@@ -1077,7 +1016,7 @@ other. NOTE: The values also appear in pcre_jit_compile.c. */
 #define HSPACE_BYTE_CASES \
   case CHAR_HT: \
   case CHAR_SPACE: \
-  case 0xa0     /* NBSP */
+  case CHAR_NBSP
 
 #define HSPACE_CASES \
   HSPACE_BYTE_CASES: \
@@ -1104,11 +1043,12 @@ other. NOTE: The values also appear in pcre_jit_compile.c. */
 /* ------ EBCDIC environments ------ */
 
 #else
-#define HSPACE_LIST CHAR_HT, CHAR_SPACE
+#define HSPACE_LIST CHAR_HT, CHAR_SPACE, CHAR_NBSP, NOTACHAR
 
 #define HSPACE_BYTE_CASES \
   case CHAR_HT: \
-  case CHAR_SPACE
+  case CHAR_SPACE: \
+  case CHAR_NBSP
 
 #define HSPACE_CASES HSPACE_BYTE_CASES
 
@@ -1155,6 +1095,7 @@ compatibility. */
 #define PCRE_HASTHEN       0x00001000  /* pattern contains (*THEN) */
 #define PCRE_MLSET         0x00002000  /* match limit set by regex */
 #define PCRE_RLSET         0x00004000  /* recursion limit set by regex */
+#define PCRE_MATCH_EMPTY   0x00008000  /* pattern can match empty string */
 
 #if defined COMPILE_PCRE8
 #define PCRE_MODE          PCRE_MODE8
@@ -1179,7 +1120,8 @@ time, run time, or study time, respectively. */
 #define PUBLIC_COMPILE_OPTIONS \
   (PCRE_CASELESS|PCRE_EXTENDED|PCRE_ANCHORED|PCRE_MULTILINE| \
    PCRE_DOTALL|PCRE_DOLLAR_ENDONLY|PCRE_EXTRA|PCRE_UNGREEDY|PCRE_UTF8| \
-   PCRE_NO_AUTO_CAPTURE|PCRE_NO_UTF8_CHECK|PCRE_AUTO_CALLOUT|PCRE_FIRSTLINE| \
+   PCRE_NO_AUTO_CAPTURE|PCRE_NO_AUTO_POSSESS| \
+   PCRE_NO_UTF8_CHECK|PCRE_AUTO_CALLOUT|PCRE_FIRSTLINE| \
    PCRE_DUPNAMES|PCRE_NEWLINE_BITS|PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE| \
    PCRE_JAVASCRIPT_COMPAT|PCRE_UCP|PCRE_NO_START_OPTIMIZE|PCRE_NEVER_UTF)
 
@@ -1280,6 +1222,7 @@ same code point. */
 
 #define CHAR_ESC                    '\047'
 #define CHAR_DEL                    '\007'
+#define CHAR_NBSP                   '\x41'
 #define STR_ESC                     "\047"
 #define STR_DEL                     "\007"
 
@@ -1294,6 +1237,7 @@ a positive value. */
 #define CHAR_NEL                    ((unsigned char)'\x85')
 #define CHAR_ESC                    '\033'
 #define CHAR_DEL                    '\177'
+#define CHAR_NBSP                   ((unsigned char)'\xa0')
 
 #define STR_LF                      "\n"
 #define STR_NL                      STR_LF
@@ -1537,22 +1481,25 @@ a positive value. */
 #define STRING_xdigit               "xdigit"
 
 #define STRING_DEFINE               "DEFINE"
-
-#define STRING_CR_RIGHTPAR             "CR)"
-#define STRING_LF_RIGHTPAR             "LF)"
-#define STRING_CRLF_RIGHTPAR           "CRLF)"
-#define STRING_ANY_RIGHTPAR            "ANY)"
-#define STRING_ANYCRLF_RIGHTPAR        "ANYCRLF)"
-#define STRING_BSR_ANYCRLF_RIGHTPAR    "BSR_ANYCRLF)"
-#define STRING_BSR_UNICODE_RIGHTPAR    "BSR_UNICODE)"
-#define STRING_UTF8_RIGHTPAR           "UTF8)"
-#define STRING_UTF16_RIGHTPAR          "UTF16)"
-#define STRING_UTF32_RIGHTPAR          "UTF32)"
-#define STRING_UTF_RIGHTPAR            "UTF)"
-#define STRING_UCP_RIGHTPAR            "UCP)"
-#define STRING_NO_START_OPT_RIGHTPAR   "NO_START_OPT)"
-#define STRING_LIMIT_MATCH_EQ          "LIMIT_MATCH="
-#define STRING_LIMIT_RECURSION_EQ      "LIMIT_RECURSION="
+#define STRING_WEIRD_STARTWORD      "[:<:]]"
+#define STRING_WEIRD_ENDWORD        "[:>:]]"
+
+#define STRING_CR_RIGHTPAR              "CR)"
+#define STRING_LF_RIGHTPAR              "LF)"
+#define STRING_CRLF_RIGHTPAR            "CRLF)"
+#define STRING_ANY_RIGHTPAR             "ANY)"
+#define STRING_ANYCRLF_RIGHTPAR         "ANYCRLF)"
+#define STRING_BSR_ANYCRLF_RIGHTPAR     "BSR_ANYCRLF)"
+#define STRING_BSR_UNICODE_RIGHTPAR     "BSR_UNICODE)"
+#define STRING_UTF8_RIGHTPAR            "UTF8)"
+#define STRING_UTF16_RIGHTPAR           "UTF16)"
+#define STRING_UTF32_RIGHTPAR           "UTF32)"
+#define STRING_UTF_RIGHTPAR             "UTF)"
+#define STRING_UCP_RIGHTPAR             "UCP)"
+#define STRING_NO_AUTO_POSSESS_RIGHTPAR "NO_AUTO_POSSESS)"
+#define STRING_NO_START_OPT_RIGHTPAR    "NO_START_OPT)"
+#define STRING_LIMIT_MATCH_EQ           "LIMIT_MATCH="
+#define STRING_LIMIT_RECURSION_EQ       "LIMIT_RECURSION="
 
 #else  /* SUPPORT_UTF */
 
@@ -1668,6 +1615,7 @@ only. */
 #define CHAR_VERTICAL_LINE          '\174'
 #define CHAR_RIGHT_CURLY_BRACKET    '\175'
 #define CHAR_TILDE                  '\176'
+#define CHAR_NBSP                   ((unsigned char)'\xa0')
 
 #define STR_HT                      "\011"
 #define STR_VT                      "\013"
@@ -1800,27 +1748,34 @@ only. */
 #define STRING_xdigit               STR_x STR_d STR_i STR_g STR_i STR_t
 
 #define STRING_DEFINE               STR_D STR_E STR_F STR_I STR_N STR_E
-
-#define STRING_CR_RIGHTPAR             STR_C STR_R STR_RIGHT_PARENTHESIS
-#define STRING_LF_RIGHTPAR             STR_L STR_F STR_RIGHT_PARENTHESIS
-#define STRING_CRLF_RIGHTPAR           STR_C STR_R STR_L STR_F STR_RIGHT_PARENTHESIS
-#define STRING_ANY_RIGHTPAR            STR_A STR_N STR_Y STR_RIGHT_PARENTHESIS
-#define STRING_ANYCRLF_RIGHTPAR        STR_A STR_N STR_Y STR_C STR_R STR_L STR_F STR_RIGHT_PARENTHESIS
-#define STRING_BSR_ANYCRLF_RIGHTPAR    STR_B STR_S STR_R STR_UNDERSCORE STR_A STR_N STR_Y STR_C STR_R STR_L STR_F STR_RIGHT_PARENTHESIS
-#define STRING_BSR_UNICODE_RIGHTPAR    STR_B STR_S STR_R STR_UNDERSCORE STR_U STR_N STR_I STR_C STR_O STR_D STR_E STR_RIGHT_PARENTHESIS
-#define STRING_UTF8_RIGHTPAR           STR_U STR_T STR_F STR_8 STR_RIGHT_PARENTHESIS
-#define STRING_UTF16_RIGHTPAR          STR_U STR_T STR_F STR_1 STR_6 STR_RIGHT_PARENTHESIS
-#define STRING_UTF32_RIGHTPAR          STR_U STR_T STR_F STR_3 STR_2 STR_RIGHT_PARENTHESIS
-#define STRING_UTF_RIGHTPAR            STR_U STR_T STR_F STR_RIGHT_PARENTHESIS
-#define STRING_UCP_RIGHTPAR            STR_U STR_C STR_P STR_RIGHT_PARENTHESIS
-#define STRING_NO_START_OPT_RIGHTPAR   STR_N STR_O STR_UNDERSCORE STR_S STR_T STR_A STR_R STR_T STR_UNDERSCORE STR_O STR_P STR_T STR_RIGHT_PARENTHESIS
-#define STRING_LIMIT_MATCH_EQ          STR_L STR_I STR_M STR_I STR_T STR_UNDERSCORE STR_M STR_A STR_T STR_C STR_H STR_EQUALS_SIGN
-#define STRING_LIMIT_RECURSION_EQ      STR_L STR_I STR_M STR_I STR_T STR_UNDERSCORE STR_R STR_E STR_C STR_U STR_R STR_S STR_I STR_O STR_N STR_EQUALS_SIGN
+#define STRING_WEIRD_STARTWORD      STR_LEFT_SQUARE_BRACKET STR_COLON STR_LESS_THAN_SIGN STR_COLON STR_RIGHT_SQUARE_BRACKET STR_RIGHT_SQUARE_BRACKET
+#define STRING_WEIRD_ENDWORD        STR_LEFT_SQUARE_BRACKET STR_COLON STR_GREATER_THAN_SIGN STR_COLON STR_RIGHT_SQUARE_BRACKET STR_RIGHT_SQUARE_BRACKET
+
+#define STRING_CR_RIGHTPAR              STR_C STR_R STR_RIGHT_PARENTHESIS
+#define STRING_LF_RIGHTPAR              STR_L STR_F STR_RIGHT_PARENTHESIS
+#define STRING_CRLF_RIGHTPAR            STR_C STR_R STR_L STR_F STR_RIGHT_PARENTHESIS
+#define STRING_ANY_RIGHTPAR             STR_A STR_N STR_Y STR_RIGHT_PARENTHESIS
+#define STRING_ANYCRLF_RIGHTPAR         STR_A STR_N STR_Y STR_C STR_R STR_L STR_F STR_RIGHT_PARENTHESIS
+#define STRING_BSR_ANYCRLF_RIGHTPAR     STR_B STR_S STR_R STR_UNDERSCORE STR_A STR_N STR_Y STR_C STR_R STR_L STR_F STR_RIGHT_PARENTHESIS
+#define STRING_BSR_UNICODE_RIGHTPAR     STR_B STR_S STR_R STR_UNDERSCORE STR_U STR_N STR_I STR_C STR_O STR_D STR_E STR_RIGHT_PARENTHESIS
+#define STRING_UTF8_RIGHTPAR            STR_U STR_T STR_F STR_8 STR_RIGHT_PARENTHESIS
+#define STRING_UTF16_RIGHTPAR           STR_U STR_T STR_F STR_1 STR_6 STR_RIGHT_PARENTHESIS
+#define STRING_UTF32_RIGHTPAR           STR_U STR_T STR_F STR_3 STR_2 STR_RIGHT_PARENTHESIS
+#define STRING_UTF_RIGHTPAR             STR_U STR_T STR_F STR_RIGHT_PARENTHESIS
+#define STRING_UCP_RIGHTPAR             STR_U STR_C STR_P STR_RIGHT_PARENTHESIS
+#define STRING_NO_AUTO_POSSESS_RIGHTPAR STR_N STR_O STR_UNDERSCORE STR_A STR_U STR_T STR_O STR_UNDERSCORE STR_P STR_O STR_S STR_S STR_E STR_S STR_S STR_RIGHT_PARENTHESIS
+#define STRING_NO_START_OPT_RIGHTPAR    STR_N STR_O STR_UNDERSCORE STR_S STR_T STR_A STR_R STR_T STR_UNDERSCORE STR_O STR_P STR_T STR_RIGHT_PARENTHESIS
+#define STRING_LIMIT_MATCH_EQ           STR_L STR_I STR_M STR_I STR_T STR_UNDERSCORE STR_M STR_A STR_T STR_C STR_H STR_EQUALS_SIGN
+#define STRING_LIMIT_RECURSION_EQ       STR_L STR_I STR_M STR_I STR_T STR_UNDERSCORE STR_R STR_E STR_C STR_U STR_R STR_S STR_I STR_O STR_N STR_EQUALS_SIGN
 
 #endif  /* SUPPORT_UTF */
 
 /* Escape items that are just an encoding of a particular data value. */
 
+#ifndef ESC_a
+#define ESC_a CHAR_BEL
+#endif
+
 #ifndef ESC_e
 #define ESC_e CHAR_ESC
 #endif
@@ -1857,12 +1812,24 @@ only. */
 #define PT_WORD       8    /* Word - L plus N plus underscore */
 #define PT_CLIST      9    /* Pseudo-property: match character list */
 #define PT_UCNC      10    /* Universal Character nameable character */
+#define PT_TABSIZE   11    /* Size of square table for autopossessify tests */
+
+/* The following special properties are used only in XCLASS items, when POSIX
+classes are specified and PCRE_UCP is set - in other words, for Unicode
+handling of these classes. They are not available via the \p or \P escapes like
+those in the above list, and so they do not take part in the autopossessifying
+table. */
+
+#define PT_PXGRAPH   11    /* [:graph:] - characters that mark the paper */
+#define PT_PXPRINT   12    /* [:print:] - [:graph:] plus non-control spaces */
+#define PT_PXPUNCT   13    /* [:punct:] - punctuation characters */
 
 /* Flag bits and data types for the extended class (OP_XCLASS) for classes that
 contain characters with values greater than 255. */
 
-#define XCL_NOT    0x01    /* Flag: this is a negative class */
-#define XCL_MAP    0x02    /* Flag: a 32-byte map is present */
+#define XCL_NOT       0x01    /* Flag: this is a negative class */
+#define XCL_MAP       0x02    /* Flag: a 32-byte map is present */
+#define XCL_HASPROP   0x04    /* Flag: property checks are present. */
 
 #define XCL_END       0    /* Marks end of individual items */
 #define XCL_SINGLE    1    /* Single item (one multibyte char) follows */
@@ -1871,9 +1838,9 @@ contain characters with values greater than 255. */
 #define XCL_NOTPROP   4    /* Unicode inverted property (ditto) */
 
 /* These are escaped items that aren't just an encoding of a particular data
-value such as \n. They must have non-zero values, as check_escape() returns
-0 for a data character.  Also, they must appear in the same order as in the opcode
-definitions below, up to ESC_z. There's a dummy for OP_ALLANY because it
+value such as \n. They must have non-zero values, as check_escape() returns 0
+for a data character.  Also, they must appear in the same order as in the
+opcode definitions below, up to ESC_z. There's a dummy for OP_ALLANY because it
 corresponds to "." in DOTALL mode rather than an escape sequence. It is also
 used for [^] in JavaScript compatibility mode, and for \C in non-utf mode. In
 non-DOTALL mode, "." behaves like \N.
@@ -1896,12 +1863,31 @@ enum { ESC_A = 1, ESC_G, ESC_K, ESC_B, ESC_b, ESC_D, ESC_d, ESC_S, ESC_s,
        ESC_E, ESC_Q, ESC_g, ESC_k,
        ESC_DU, ESC_du, ESC_SU, ESC_su, ESC_WU, ESC_wu };
 
-/* Opcode table: Starting from 1 (i.e. after OP_END), the values up to
-OP_EOD must correspond in order to the list of escapes immediately above.
 
-*** NOTE NOTE NOTE *** Whenever this list is updated, the two macro definitions
-that follow must also be updated to match. There are also tables called
-"coptable" and "poptable" in pcre_dfa_exec.c that must be updated. */
+/********************** Opcode definitions ******************/
+
+/****** NOTE NOTE NOTE ******
+
+Starting from 1 (i.e. after OP_END), the values up to OP_EOD must correspond in
+order to the list of escapes immediately above. Furthermore, values up to
+OP_DOLLM must not be changed without adjusting the table called autoposstab in
+pcre_compile.c
+
+Whenever this list is updated, the two macro definitions that follow must be
+updated to match. The possessification table called "opcode_possessify" in
+pcre_compile.c must also be updated, and also the tables called "coptable"
+and "poptable" in pcre_dfa_exec.c.
+
+****** NOTE NOTE NOTE ******/
+
+
+/* The values between FIRST_AUTOTAB_OP and LAST_AUTOTAB_RIGHT_OP, inclusive,
+are used in a table for deciding whether a repeated character type can be
+auto-possessified. */
+
+#define FIRST_AUTOTAB_OP       OP_NOT_DIGIT
+#define LAST_AUTOTAB_LEFT_OP   OP_EXTUNI
+#define LAST_AUTOTAB_RIGHT_OP  OP_DOLLM
 
 enum {
   OP_END,            /* 0 End of pattern */
@@ -1934,10 +1920,15 @@ enum {
   OP_EODN,           /* 23 End of data or \n at end of data (\Z) */
   OP_EOD,            /* 24 End of data (\z) */
 
-  OP_CIRC,           /* 25 Start of line - not multiline */
-  OP_CIRCM,          /* 26 Start of line - multiline */
-  OP_DOLL,           /* 27 End of line - not multiline */
-  OP_DOLLM,          /* 28 End of line - multiline */
+  /* Line end assertions */
+
+  OP_DOLL,           /* 25 End of line - not multiline */
+  OP_DOLLM,          /* 26 End of line - multiline */
+  OP_CIRC,           /* 27 Start of line - not multiline */
+  OP_CIRCM,          /* 28 Start of line - multiline */
+
+  /* Single characters; caseful must precede the caseless ones */
+
   OP_CHAR,           /* 29 Match one character, casefully */
   OP_CHARI,          /* 30 Match one character, caselessly */
   OP_NOT,            /* 31 Match one character, not the given one, casefully */
@@ -1946,7 +1937,7 @@ enum {
   /* The following sets of 13 opcodes must always be kept in step because
   the offset from the first one is used to generate the others. */
 
-  /**** Single characters, caseful, must precede the caseless ones ****/
+  /* Repeated characters; caseful must precede the caseless ones */
 
   OP_STAR,           /* 33 The maximizing and minimizing versions of */
   OP_MINSTAR,        /* 34 these six opcodes must come in pairs, with */
@@ -1964,7 +1955,7 @@ enum {
   OP_POSQUERY,       /* 44 Posesssified query, caseful */
   OP_POSUPTO,        /* 45 Possessified upto, caseful */
 
-  /**** Single characters, caseless, must follow the caseful ones */
+  /* Repeated characters; caseless must follow the caseful ones */
 
   OP_STARI,          /* 46 */
   OP_MINSTARI,       /* 47 */
@@ -1982,8 +1973,8 @@ enum {
   OP_POSQUERYI,      /* 57 Posesssified query, caseless */
   OP_POSUPTOI,       /* 58 Possessified upto, caseless */
 
-  /**** The negated ones must follow the non-negated ones, and match them ****/
-  /**** Negated single character, caseful; must precede the caseless ones ****/
+  /* The negated ones must follow the non-negated ones, and match them */
+  /* Negated repeated character, caseful; must precede the caseless ones */
 
   OP_NOTSTAR,        /* 59 The maximizing and minimizing versions of */
   OP_NOTMINSTAR,     /* 60 these six opcodes must come in pairs, with */
@@ -2001,7 +1992,7 @@ enum {
   OP_NOTPOSQUERY,    /* 70 */
   OP_NOTPOSUPTO,     /* 71 */
 
-  /**** Negated single character, caseless; must follow the caseful ones ****/
+  /* Negated repeated character, caseless; must follow the caseful ones */
 
   OP_NOTSTARI,       /* 72 */
   OP_NOTMINSTARI,    /* 73 */
@@ -2019,7 +2010,7 @@ enum {
   OP_NOTPOSQUERYI,   /* 83 */
   OP_NOTPOSUPTOI,    /* 84 */
 
-  /**** Character types ****/
+  /* Character types */
 
   OP_TYPESTAR,       /* 85 The maximizing and minimizing versions of */
   OP_TYPEMINSTAR,    /* 86 these six opcodes must come in pairs, with */
@@ -2050,89 +2041,96 @@ enum {
   OP_CRRANGE,        /* 104 These are different to the three sets above. */
   OP_CRMINRANGE,     /* 105 */
 
+  OP_CRPOSSTAR,      /* 106 Possessified versions */
+  OP_CRPOSPLUS,      /* 107 */
+  OP_CRPOSQUERY,     /* 108 */
+  OP_CRPOSRANGE,     /* 109 */
+
   /* End of quantifier opcodes */
 
-  OP_CLASS,          /* 106 Match a character class, chars < 256 only */
-  OP_NCLASS,         /* 107 Same, but the bitmap was created from a negative
+  OP_CLASS,          /* 110 Match a character class, chars < 256 only */
+  OP_NCLASS,         /* 111 Same, but the bitmap was created from a negative
                               class - the difference is relevant only when a
                               character > 255 is encountered. */
-  OP_XCLASS,         /* 108 Extended class for handling > 255 chars within the
+  OP_XCLASS,         /* 112 Extended class for handling > 255 chars within the
                               class. This does both positive and negative. */
-  OP_REF,            /* 109 Match a back reference, casefully */
-  OP_REFI,           /* 110 Match a back reference, caselessly */
-  OP_RECURSE,        /* 111 Match a numbered subpattern (possibly recursive) */
-  OP_CALLOUT,        /* 112 Call out to external function if provided */
-
-  OP_ALT,            /* 113 Start of alternation */
-  OP_KET,            /* 114 End of group that doesn't have an unbounded repeat */
-  OP_KETRMAX,        /* 115 These two must remain together and in this */
-  OP_KETRMIN,        /* 116 order. They are for groups the repeat for ever. */
-  OP_KETRPOS,        /* 117 Possessive unlimited repeat. */
+  OP_REF,            /* 113 Match a back reference, casefully */
+  OP_REFI,           /* 114 Match a back reference, caselessly */
+  OP_DNREF,          /* 115 Match a duplicate name backref, casefully */
+  OP_DNREFI,         /* 116 Match a duplicate name backref, caselessly */
+  OP_RECURSE,        /* 117 Match a numbered subpattern (possibly recursive) */
+  OP_CALLOUT,        /* 118 Call out to external function if provided */
+
+  OP_ALT,            /* 119 Start of alternation */
+  OP_KET,            /* 120 End of group that doesn't have an unbounded repeat */
+  OP_KETRMAX,        /* 121 These two must remain together and in this */
+  OP_KETRMIN,        /* 122 order. They are for groups the repeat for ever. */
+  OP_KETRPOS,        /* 123 Possessive unlimited repeat. */
 
   /* The assertions must come before BRA, CBRA, ONCE, and COND, and the four
   asserts must remain in order. */
 
-  OP_REVERSE,        /* 118 Move pointer back - used in lookbehind assertions */
-  OP_ASSERT,         /* 119 Positive lookahead */
-  OP_ASSERT_NOT,     /* 120 Negative lookahead */
-  OP_ASSERTBACK,     /* 121 Positive lookbehind */
-  OP_ASSERTBACK_NOT, /* 122 Negative lookbehind */
+  OP_REVERSE,        /* 124 Move pointer back - used in lookbehind assertions */
+  OP_ASSERT,         /* 125 Positive lookahead */
+  OP_ASSERT_NOT,     /* 126 Negative lookahead */
+  OP_ASSERTBACK,     /* 127 Positive lookbehind */
+  OP_ASSERTBACK_NOT, /* 128 Negative lookbehind */
 
   /* ONCE, ONCE_NC, BRA, BRAPOS, CBRA, CBRAPOS, and COND must come immediately
   after the assertions, with ONCE first, as there's a test for >= ONCE for a
   subpattern that isn't an assertion. The POS versions must immediately follow
   the non-POS versions in each case. */
 
-  OP_ONCE,           /* 123 Atomic group, contains captures */
-  OP_ONCE_NC,        /* 124 Atomic group containing no captures */
-  OP_BRA,            /* 125 Start of non-capturing bracket */
-  OP_BRAPOS,         /* 126 Ditto, with unlimited, possessive repeat */
-  OP_CBRA,           /* 127 Start of capturing bracket */
-  OP_CBRAPOS,        /* 128 Ditto, with unlimited, possessive repeat */
-  OP_COND,           /* 129 Conditional group */
+  OP_ONCE,           /* 129 Atomic group, contains captures */
+  OP_ONCE_NC,        /* 130 Atomic group containing no captures */
+  OP_BRA,            /* 131 Start of non-capturing bracket */
+  OP_BRAPOS,         /* 132 Ditto, with unlimited, possessive repeat */
+  OP_CBRA,           /* 133 Start of capturing bracket */
+  OP_CBRAPOS,        /* 134 Ditto, with unlimited, possessive repeat */
+  OP_COND,           /* 135 Conditional group */
 
   /* These five must follow the previous five, in the same order. There's a
   check for >= SBRA to distinguish the two sets. */
 
-  OP_SBRA,           /* 130 Start of non-capturing bracket, check empty  */
-  OP_SBRAPOS,        /* 131 Ditto, with unlimited, possessive repeat */
-  OP_SCBRA,          /* 132 Start of capturing bracket, check empty */
-  OP_SCBRAPOS,       /* 133 Ditto, with unlimited, possessive repeat */
-  OP_SCOND,          /* 134 Conditional group, check empty */
+  OP_SBRA,           /* 136 Start of non-capturing bracket, check empty  */
+  OP_SBRAPOS,        /* 137 Ditto, with unlimited, possessive repeat */
+  OP_SCBRA,          /* 138 Start of capturing bracket, check empty */
+  OP_SCBRAPOS,       /* 139 Ditto, with unlimited, possessive repeat */
+  OP_SCOND,          /* 140 Conditional group, check empty */
 
   /* The next two pairs must (respectively) be kept together. */
 
-  OP_CREF,           /* 135 Used to hold a capture number as condition */
-  OP_NCREF,          /* 136 Same, but generated by a name reference*/
-  OP_RREF,           /* 137 Used to hold a recursion number as condition */
-  OP_NRREF,          /* 138 Same, but generated by a name reference*/
-  OP_DEF,            /* 139 The DEFINE condition */
+  OP_CREF,           /* 141 Used to hold a capture number as condition */
+  OP_DNCREF,         /* 142 Used to point to duplicate names as a condition */
+  OP_RREF,           /* 143 Used to hold a recursion number as condition */
+  OP_DNRREF,         /* 144 Used to point to duplicate names as a condition */
+  OP_DEF,            /* 145 The DEFINE condition */
 
-  OP_BRAZERO,        /* 140 These two must remain together and in this */
-  OP_BRAMINZERO,     /* 141 order. */
-  OP_BRAPOSZERO,     /* 142 */
+  OP_BRAZERO,        /* 146 These two must remain together and in this */
+  OP_BRAMINZERO,     /* 147 order. */
+  OP_BRAPOSZERO,     /* 148 */
 
   /* These are backtracking control verbs */
 
-  OP_MARK,           /* 143 always has an argument */
-  OP_PRUNE,          /* 144 */
-  OP_PRUNE_ARG,      /* 145 same, but with argument */
-  OP_SKIP,           /* 146 */
-  OP_SKIP_ARG,       /* 147 same, but with argument */
-  OP_THEN,           /* 148 */
-  OP_THEN_ARG,       /* 149 same, but with argument */
-  OP_COMMIT,         /* 150 */
+  OP_MARK,           /* 149 always has an argument */
+  OP_PRUNE,          /* 150 */
+  OP_PRUNE_ARG,      /* 151 same, but with argument */
+  OP_SKIP,           /* 152 */
+  OP_SKIP_ARG,       /* 153 same, but with argument */
+  OP_THEN,           /* 154 */
+  OP_THEN_ARG,       /* 155 same, but with argument */
+  OP_COMMIT,         /* 156 */
 
   /* These are forced failure and success verbs */
 
-  OP_FAIL,           /* 151 */
-  OP_ACCEPT,         /* 152 */
-  OP_ASSERT_ACCEPT,  /* 153 Used inside assertions */
-  OP_CLOSE,          /* 154 Used before OP_ACCEPT to close open captures */
+  OP_FAIL,           /* 157 */
+  OP_ACCEPT,         /* 158 */
+  OP_ASSERT_ACCEPT,  /* 159 Used inside assertions */
+  OP_CLOSE,          /* 160 Used before OP_ACCEPT to close open captures */
 
   /* This is used to skip a subpattern with a {0} quantifier */
 
-  OP_SKIPZERO,       /* 155 */
+  OP_SKIPZERO,       /* 161 */
 
   /* This is not an opcode, but is used to check that tables indexed by opcode
   are the correct length, in order to catch updating errors - there have been
@@ -2143,7 +2141,8 @@ enum {
 
 /* *** NOTE NOTE NOTE *** Whenever the list above is updated, the two macro
 definitions that follow must also be updated to match. There are also tables
-called "coptable" and "poptable" in pcre_dfa_exec.c that must be updated. */
+called "opcode_possessify" in pcre_compile.c and "coptable" and "poptable" in
+pcre_dfa_exec.c that must be updated. */
 
 
 /* This macro defines textual names for all the opcodes. These are used only
@@ -2156,7 +2155,7 @@ some cases doesn't actually use these names at all). */
   "\\S", "\\s", "\\W", "\\w", "Any", "AllAny", "Anybyte",         \
   "notprop", "prop", "\\R", "\\H", "\\h", "\\V", "\\v",           \
   "extuni",  "\\Z", "\\z",                                        \
-  "^", "^", "$", "$", "char", "chari", "not", "noti",             \
+  "$", "$", "^", "^", "char", "chari", "not", "noti",             \
   "*", "*?", "+", "+?", "?", "??",                                \
   "{", "{", "{",                                                  \
   "*+","++", "?+", "{",                                           \
@@ -2172,7 +2171,8 @@ some cases doesn't actually use these names at all). */
   "*", "*?", "+", "+?", "?", "??", "{", "{", "{",                 \
   "*+","++", "?+", "{",                                           \
   "*", "*?", "+", "+?", "?", "??", "{", "{",                      \
-  "class", "nclass", "xclass", "Ref", "Refi",                     \
+  "*+","++", "?+", "{",                                           \
+  "class", "nclass", "xclass", "Ref", "Refi", "DnRef", "DnRefi",  \
   "Recurse", "Callout",                                           \
   "Alt", "Ket", "KetRmax", "KetRmin", "KetRpos",                  \
   "Reverse", "Assert", "Assert not", "AssertB", "AssertB not",    \
@@ -2181,7 +2181,7 @@ some cases doesn't actually use these names at all). */
   "Cond",                                                         \
   "SBra", "SBraPos", "SCBra", "SCBraPos",                         \
   "SCond",                                                        \
-  "Cond ref", "Cond nref", "Cond rec", "Cond nrec", "Cond def",   \
+  "Cond ref", "Cond dnref", "Cond rec", "Cond dnrec", "Cond def", \
   "Brazero", "Braminzero", "Braposzero",                          \
   "*MARK", "*PRUNE", "*PRUNE", "*SKIP", "*SKIP",                  \
   "*THEN", "*THEN", "*COMMIT", "*FAIL",                           \
@@ -2206,7 +2206,7 @@ in UTF-8 mode. The code that uses this table must know about such things. */
   3, 3,                          /* \P, \p                                 */ \
   1, 1, 1, 1, 1,                 /* \R, \H, \h, \V, \v                     */ \
   1,                             /* \X                                     */ \
-  1, 1, 1, 1, 1, 1,              /* \Z, \z, ^, ^M, $, $M                   */ \
+  1, 1, 1, 1, 1, 1,              /* \Z, \z, $, $M ^, ^M                    */ \
   2,                             /* Char  - the minimum length             */ \
   2,                             /* Chari  - the minimum length            */ \
   2,                             /* not                                    */ \
@@ -2237,11 +2237,14 @@ in UTF-8 mode. The code that uses this table must know about such things. */
   /* Character class & ref repeats                                         */ \
   1, 1, 1, 1, 1, 1,              /* *, *?, +, +?, ?, ??                    */ \
   1+2*IMM2_SIZE, 1+2*IMM2_SIZE,  /* CRRANGE, CRMINRANGE                    */ \
+  1, 1, 1, 1+2*IMM2_SIZE,        /* Possessive *+, ++, ?+, CRPOSRANGE      */ \
   1+(32/sizeof(pcre_uchar)),     /* CLASS                                  */ \
   1+(32/sizeof(pcre_uchar)),     /* NCLASS                                 */ \
   0,                             /* XCLASS - variable length               */ \
   1+IMM2_SIZE,                   /* REF                                    */ \
   1+IMM2_SIZE,                   /* REFI                                   */ \
+  1+2*IMM2_SIZE,                 /* DNREF                                  */ \
+  1+2*IMM2_SIZE,                 /* DNREFI                                 */ \
   1+LINK_SIZE,                   /* RECURSE                                */ \
   2+2*LINK_SIZE,                 /* CALLOUT                                */ \
   1+LINK_SIZE,                   /* Alt                                    */ \
@@ -2266,8 +2269,8 @@ in UTF-8 mode. The code that uses this table must know about such things. */
   1+LINK_SIZE+IMM2_SIZE,         /* SCBRA                                  */ \
   1+LINK_SIZE+IMM2_SIZE,         /* SCBRAPOS                               */ \
   1+LINK_SIZE,                   /* SCOND                                  */ \
-  1+IMM2_SIZE, 1+IMM2_SIZE,      /* CREF, NCREF                            */ \
-  1+IMM2_SIZE, 1+IMM2_SIZE,      /* RREF, NRREF                            */ \
+  1+IMM2_SIZE, 1+2*IMM2_SIZE,    /* CREF, DNCREF                           */ \
+  1+IMM2_SIZE, 1+2*IMM2_SIZE,    /* RREF, DNRREF                           */ \
   1,                             /* DEF                                    */ \
   1, 1, 1,                       /* BRAZERO, BRAMINZERO, BRAPOSZERO        */ \
   3, 1, 3,                       /* MARK, PRUNE, PRUNE_ARG                 */ \
@@ -2276,8 +2279,7 @@ in UTF-8 mode. The code that uses this table must know about such things. */
   1, 1, 1, 1,                    /* COMMIT, FAIL, ACCEPT, ASSERT_ACCEPT    */ \
   1+IMM2_SIZE, 1                 /* CLOSE, SKIPZERO                        */
 
-/* A magic value for OP_RREF and OP_NRREF to indicate the "any recursion"
-condition. */
+/* A magic value for OP_RREF to indicate the "any recursion" condition. */
 
 #define RREF_ANY  0xffff
 
@@ -2292,9 +2294,11 @@ enum { ERR0,  ERR1,  ERR2,  ERR3,  ERR4,  ERR5,  ERR6,  ERR7,  ERR8,  ERR9,
        ERR40, ERR41, ERR42, ERR43, ERR44, ERR45, ERR46, ERR47, ERR48, ERR49,
        ERR50, ERR51, ERR52, ERR53, ERR54, ERR55, ERR56, ERR57, ERR58, ERR59,
        ERR60, ERR61, ERR62, ERR63, ERR64, ERR65, ERR66, ERR67, ERR68, ERR69,
-       ERR70, ERR71, ERR72, ERR73, ERR74, ERR75, ERR76, ERR77, ERR78, ERRCOUNT };
+       ERR70, ERR71, ERR72, ERR73, ERR74, ERR75, ERR76, ERR77, ERR78, ERR79,
+       ERR80, ERR81, ERR82, ERR83, ERR84, ERR85, ERR86, ERR87, ERRCOUNT };
 
 /* JIT compiling modes. The function list is indexed by them. */
+
 enum { JIT_COMPILE, JIT_PARTIAL_SOFT_COMPILE, JIT_PARTIAL_HARD_COMPILE,
        JIT_NUMBER_OF_COMPILE_MODES };
 
@@ -2412,6 +2416,15 @@ typedef struct open_capitem {
   pcre_uint16 flag;             /* Set TRUE if recursive back ref */
 } open_capitem;
 
+/* Structure for building a list of named groups during the first pass of
+compiling. */
+
+typedef struct named_group {
+  const pcre_uchar  *name;          /* Points to the name in the pattern */
+  int                length;        /* Length of the name */
+  pcre_uint32        number;        /* Group number */
+} named_group;
+
 /* Structure for passing "static" information around between the functions
 doing the compiling, so that they are thread-safe. */
 
@@ -2424,17 +2437,21 @@ typedef struct compile_data {
   const pcre_uchar *start_code;     /* The start of the compiled code */
   const pcre_uchar *start_pattern;  /* The start of the pattern */
   const pcre_uchar *end_pattern;    /* The end of the pattern */
-  open_capitem *open_caps;          /* Chain of open capture items */
   pcre_uchar *hwm;                  /* High watermark of workspace */
+  open_capitem *open_caps;          /* Chain of open capture items */
+  named_group *named_groups;        /* Points to vector in pre-compile */
   pcre_uchar *name_table;           /* The name/number table */
   int  names_found;                 /* Number of entries so far */
   int  name_entry_size;             /* Size of each entry */
+  int  named_group_list_size;       /* Number of entries in the list */
   int  workspace_size;              /* Size of workspace */
   unsigned int bracount;            /* Count of capturing parens as we compile */
   int  final_bracount;              /* Saved value after first pass */
   int  max_lookbehind;              /* Maximum lookbehind (characters) */
   int  top_backref;                 /* Maximum back reference */
   unsigned int backref_map;         /* Bitmap of low back refs */
+  unsigned int namedrefcount;       /* Number of backreferences by name */
+  int  parens_depth;                /* Depth of nested parentheses */
   int  assert_depth;                /* Depth of nested assertions */
   pcre_uint32 external_options;     /* External (initial) options */
   pcre_uint32 external_flags;       /* External flag bits to be set */
@@ -2442,6 +2459,9 @@ typedef struct compile_data {
   BOOL had_accept;                  /* (*ACCEPT) encountered */
   BOOL had_pruneorskip;             /* (*PRUNE) or (*SKIP) encountered */
   BOOL check_lookbehind;            /* Lookbehinds need later checking */
+  BOOL dupnames;                    /* Duplicate names exist */
+  BOOL dupgroups;                   /* Duplicate groups exist: (?| found */
+  BOOL iscondassert;                /* Next assert is a condition */
   int  nltype;                      /* Newline type */
   int  nllen;                       /* Newline string length */
   pcre_uchar nl[4];                 /* Newline string when fixed length */
@@ -2455,6 +2475,13 @@ typedef struct branch_chain {
   pcre_uchar *current_branch;
 } branch_chain;
 
+/* Structure for mutual recursion detection. */
+
+typedef struct recurse_check {
+  struct recurse_check *prev;
+  const pcre_uchar *group;
+} recurse_check;
+
 /* Structure for items in a linked list that represents an explicit recursive
 call within the pattern; used by pcre_exec(). */
 
diff --git a/erts/emulator/pcre/pcre_jit_compile.c b/erts/emulator/pcre/pcre_jit_compile.c
index fb26c62817..89400498f0 100644
--- a/erts/emulator/pcre/pcre_jit_compile.c
+++ b/erts/emulator/pcre/pcre_jit_compile.c
@@ -52,8 +52,8 @@ POSSIBILITY OF SUCH DAMAGE.
 we just include it. This way we don't need to touch the build
 system files. */
 
-#define SLJIT_MALLOC(size) (PUBL(malloc))(size)
-#define SLJIT_FREE(ptr) (PUBL(free))(ptr)
+#define SLJIT_MALLOC(size, allocator_data) (PUBL(malloc))(size)
+#define SLJIT_FREE(ptr, allocator_data) (PUBL(free))(ptr)
 #define SLJIT_CONFIG_AUTO 1
 #define SLJIT_CONFIG_STATIC 1
 #define SLJIT_VERBOSE 0
@@ -168,22 +168,23 @@ typedef struct jit_arguments {
   pcre_uchar *mark_ptr;
   void *callout_data;
   /* Everything else after. */
-  pcre_uint32 limit_match;
+  sljit_u32 limit_match;
   int real_offset_count;
   int offset_count;
-  pcre_uint8 notbol;
-  pcre_uint8 noteol;
-  pcre_uint8 notempty;
-  pcre_uint8 notempty_atstart;
+  sljit_u8 notbol;
+  sljit_u8 noteol;
+  sljit_u8 notempty;
+  sljit_u8 notempty_atstart;
 } jit_arguments;
 
 typedef struct executable_functions {
   void *executable_funcs[JIT_NUMBER_OF_COMPILE_MODES];
+  void *read_only_data_heads[JIT_NUMBER_OF_COMPILE_MODES];
+  sljit_uw executable_sizes[JIT_NUMBER_OF_COMPILE_MODES];
   PUBL(jit_callback) callback;
   void *userdata;
-  pcre_uint32 top_bracket;
-  pcre_uint32 limit_match;
-  sljit_uw executable_sizes[JIT_NUMBER_OF_COMPILE_MODES];
+  sljit_u32 top_bracket;
+  sljit_u32 limit_match;
 } executable_functions;
 
 typedef struct jump_list {
@@ -197,6 +198,12 @@ typedef struct stub_list {
   struct stub_list *next;
 } stub_list;
 
+typedef struct label_addr_list {
+  struct sljit_label *label;
+  sljit_uw *update_addr;
+  struct label_addr_list *next;
+} label_addr_list;
+
 enum frame_types {
   no_frame = -1,
   no_stack = -2
@@ -270,11 +277,25 @@ typedef struct braminzero_backtrack {
   struct sljit_label *matchingpath;
 } braminzero_backtrack;
 
-typedef struct iterator_backtrack {
+typedef struct char_iterator_backtrack {
+  backtrack_common common;
+  /* Next iteration. */
+  struct sljit_label *matchingpath;
+  union {
+    jump_list *backtracks;
+    struct {
+      unsigned int othercasebit;
+      pcre_uchar chr;
+      BOOL enabled;
+    } charpos;
+  } u;
+} char_iterator_backtrack;
+
+typedef struct ref_iterator_backtrack {
   backtrack_common common;
   /* Next iteration. */
   struct sljit_label *matchingpath;
-} iterator_backtrack;
+} ref_iterator_backtrack;
 
 typedef struct recurse_entry {
   struct recurse_entry *next;
@@ -306,7 +327,7 @@ typedef struct then_trap_backtrack {
   int framesize;
 } then_trap_backtrack;
 
-#define MAX_RANGE_SIZE 6
+#define MAX_RANGE_SIZE 4
 
 typedef struct compiler_common {
   /* The sljit ceneric compiler. */
@@ -314,64 +335,77 @@ typedef struct compiler_common {
   /* First byte code. */
   pcre_uchar *start;
   /* Maps private data offset to each opcode. */
-  sljit_si *private_data_ptrs;
+  sljit_s32 *private_data_ptrs;
+  /* Chain list of read-only data ptrs. */
+  void *read_only_data_head;
   /* Tells whether the capturing bracket is optimized. */
-  pcre_uint8 *optimized_cbracket;
+  sljit_u8 *optimized_cbracket;
   /* Tells whether the starting offset is a target of then. */
-  pcre_uint8 *then_offsets;
+  sljit_u8 *then_offsets;
   /* Current position where a THEN must jump. */
   then_trap_backtrack *then_trap;
   /* Starting offset of private data for capturing brackets. */
-  int cbra_ptr;
+  sljit_s32 cbra_ptr;
   /* Output vector starting point. Must be divisible by 2. */
-  int ovector_start;
+  sljit_s32 ovector_start;
+  /* Points to the starting character of the current match. */
+  sljit_s32 start_ptr;
   /* Last known position of the requested byte. */
-  int req_char_ptr;
+  sljit_s32 req_char_ptr;
   /* Head of the last recursion. */
-  int recursive_head_ptr;
-  /* First inspected character for partial matching. */
-  int start_used_ptr;
+  sljit_s32 recursive_head_ptr;
+  /* First inspected character for partial matching.
+     (Needed for avoiding zero length partial matches.) */
+  sljit_s32 start_used_ptr;
   /* Starting pointer for partial soft matches. */
-  int hit_start;
-  /* End pointer of the first line. */
-  int first_line_end;
+  sljit_s32 hit_start;
+  /* Pointer of the match end position. */
+  sljit_s32 match_end_ptr;
   /* Points to the marked string. */
-  int mark_ptr;
+  sljit_s32 mark_ptr;
   /* Recursive control verb management chain. */
-  int control_head_ptr;
+  sljit_s32 control_head_ptr;
   /* Points to the last matched capture block index. */
-  int capture_last_ptr;
-  /* Points to the starting position of the current match. */
-  int start_ptr;
+  sljit_s32 capture_last_ptr;
+  /* Fast forward skipping byte code pointer. */
+  pcre_uchar *fast_forward_bc_ptr;
+  /* Locals used by fast fail optimization. */
+  sljit_s32 fast_fail_start_ptr;
+  sljit_s32 fast_fail_end_ptr;
 
   /* Flipped and lower case tables. */
-  const pcre_uint8 *fcc;
+  const sljit_u8 *fcc;
   sljit_sw lcc;
   /* Mode can be PCRE_STUDY_JIT_COMPILE and others. */
   int mode;
+  /* TRUE, when minlength is greater than 0. */
+  BOOL might_be_empty;
   /* \K is found in the pattern. */
   BOOL has_set_som;
   /* (*SKIP:arg) is found in the pattern. */
   BOOL has_skip_arg;
   /* (*THEN) is found in the pattern. */
   BOOL has_then;
-  /* Needs to know the start position anytime. */
-  BOOL needs_start_ptr;
+  /* (*SKIP) or (*SKIP:arg) is found in lookbehind assertion. */
+  BOOL has_skip_in_assert_back;
   /* Currently in recurse or negative assert. */
   BOOL local_exit;
   /* Currently in a positive assert. */
   BOOL positive_assert;
   /* Newline control. */
   int nltype;
+  sljit_u32 nlmax;
+  sljit_u32 nlmin;
   int newline;
   int bsr_nltype;
+  sljit_u32 bsr_nlmax;
+  sljit_u32 bsr_nlmin;
   /* Dollar endonly. */
   int endonly;
   /* Tables. */
   sljit_sw ctypes;
-  int digits[2 + MAX_RANGE_SIZE];
   /* Named capturing brackets. */
-  sljit_uw name_table;
+  pcre_uchar *name_table;
   sljit_sw name_count;
   sljit_sw name_entry_size;
 
@@ -380,7 +414,9 @@ typedef struct compiler_common {
   struct sljit_label *quit_label;
   struct sljit_label *forced_quit_label;
   struct sljit_label *accept_label;
+  struct sljit_label *ff_newline_shortcut;
   stub_list *stubs;
+  label_addr_list *label_addrs;
   recurse_entry *entries;
   recurse_entry *currententry;
   jump_list *partialmatch;
@@ -403,17 +439,14 @@ typedef struct compiler_common {
   BOOL utf;
 #ifdef SUPPORT_UCP
   BOOL use_ucp;
-#endif
-#ifndef COMPILE_PCRE32
-  jump_list *utfreadchar;
+  jump_list *getucd;
 #endif
 #ifdef COMPILE_PCRE8
+  jump_list *utfreadchar;
+  jump_list *utfreadchar16;
   jump_list *utfreadtype8;
 #endif
 #endif /* SUPPORT_UTF */
-#ifdef SUPPORT_UCP
-  jump_list *getucd;
-#endif
 } compiler_common;
 
 /* For byte_sequence_compare. */
@@ -424,27 +457,27 @@ typedef struct compare_context {
 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
   int ucharptr;
   union {
-    sljit_si asint;
-    sljit_uh asushort;
+    sljit_s32 asint;
+    sljit_u16 asushort;
 #if defined COMPILE_PCRE8
-    sljit_ub asbyte;
-    sljit_ub asuchars[4];
+    sljit_u8 asbyte;
+    sljit_u8 asuchars[4];
 #elif defined COMPILE_PCRE16
-    sljit_uh asuchars[2];
+    sljit_u16 asuchars[2];
 #elif defined COMPILE_PCRE32
-    sljit_ui asuchars[1];
+    sljit_u32 asuchars[1];
 #endif
   } c;
   union {
-    sljit_si asint;
-    sljit_uh asushort;
+    sljit_s32 asint;
+    sljit_u16 asushort;
 #if defined COMPILE_PCRE8
-    sljit_ub asbyte;
-    sljit_ub asuchars[4];
+    sljit_u8 asbyte;
+    sljit_u8 asuchars[4];
 #elif defined COMPILE_PCRE16
-    sljit_uh asuchars[2];
+    sljit_u16 asuchars[2];
 #elif defined COMPILE_PCRE32
-    sljit_ui asuchars[1];
+    sljit_u32 asuchars[1];
 #endif
   } oc;
 #endif
@@ -456,16 +489,16 @@ typedef struct compare_context {
 /* Used for accessing the elements of the stack. */
 #define STACK(i)      ((-(i) - 1) * (int)sizeof(sljit_sw))
 
-#define TMP1          SLJIT_SCRATCH_REG1
-#define TMP2          SLJIT_SCRATCH_REG3
-#define TMP3          SLJIT_TEMPORARY_EREG2
-#define STR_PTR       SLJIT_SAVED_REG1
-#define STR_END       SLJIT_SAVED_REG2
-#define STACK_TOP     SLJIT_SCRATCH_REG2
-#define STACK_LIMIT   SLJIT_SAVED_REG3
-#define ARGUMENTS     SLJIT_SAVED_EREG1
-#define COUNT_MATCH   SLJIT_SAVED_EREG2
-#define RETURN_ADDR   SLJIT_TEMPORARY_EREG1
+#define TMP1          SLJIT_R0
+#define TMP2          SLJIT_R2
+#define TMP3          SLJIT_R3
+#define STR_PTR       SLJIT_S0
+#define STR_END       SLJIT_S1
+#define STACK_TOP     SLJIT_R1
+#define STACK_LIMIT   SLJIT_S2
+#define COUNT_MATCH   SLJIT_S3
+#define ARGUMENTS     SLJIT_S4
+#define RETURN_ADDR   SLJIT_R4
 
 /* Local space layout. */
 /* These two locals can be used by the current opcode. */
@@ -481,19 +514,19 @@ to characters. The vector data is divided into two groups: the first
 group contains the start / end character pointers, and the second is
 the start pointers when the end of the capturing group has not yet reached. */
 #define OVECTOR_START    (common->ovector_start)
-#define OVECTOR(i)       (OVECTOR_START + (i) * sizeof(sljit_sw))
-#define OVECTOR_PRIV(i)  (common->cbra_ptr + (i) * sizeof(sljit_sw))
+#define OVECTOR(i)       (OVECTOR_START + (i) * (sljit_sw)sizeof(sljit_sw))
+#define OVECTOR_PRIV(i)  (common->cbra_ptr + (i) * (sljit_sw)sizeof(sljit_sw))
 #define PRIVATE_DATA(cc) (common->private_data_ptrs[(cc) - common->start])
 
 #if defined COMPILE_PCRE8
-#define MOV_UCHAR  SLJIT_MOV_UB
-#define MOVU_UCHAR SLJIT_MOVU_UB
+#define MOV_UCHAR  SLJIT_MOV_U8
+#define MOVU_UCHAR SLJIT_MOVU_U8
 #elif defined COMPILE_PCRE16
-#define MOV_UCHAR  SLJIT_MOV_UH
-#define MOVU_UCHAR SLJIT_MOVU_UH
+#define MOV_UCHAR  SLJIT_MOV_U16
+#define MOVU_UCHAR SLJIT_MOVU_U16
 #elif defined COMPILE_PCRE32
-#define MOV_UCHAR  SLJIT_MOV_UI
-#define MOVU_UCHAR SLJIT_MOVU_UI
+#define MOV_UCHAR  SLJIT_MOV_U32
+#define MOVU_UCHAR SLJIT_MOVU_U32
 #else
 #error Unsupported compiling mode
 #endif
@@ -524,7 +557,9 @@ the start pointers when the end of the capturing group has not yet reached. */
 #define GET_LOCAL_BASE(dst, dstw, offset) \
   sljit_get_local_base(compiler, (dst), (dstw), (offset))
 
-static pcre_uchar* bracketend(pcre_uchar* cc)
+#define READ_CHAR_MAX 0x7fffffff
+
+static pcre_uchar *bracketend(pcre_uchar *cc)
 {
 SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND));
 do cc += GET(cc, 1); while (*cc == OP_ALT);
@@ -533,6 +568,20 @@ cc += 1 + LINK_SIZE;
 return cc;
 }
 
+static int no_alternatives(pcre_uchar *cc)
+{
+int count = 0;
+SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND));
+do
+  {
+  cc += GET(cc, 1);
+  count++;
+  }
+while (*cc == OP_ALT);
+SLJIT_ASSERT(*cc >= OP_KET && *cc <= OP_KETRPOS);
+return count;
+}
+
 /* Functions whose might need modification for all new supported opcodes:
  next_opcode
  check_opcode_types
@@ -585,10 +634,16 @@ switch(*cc)
   case OP_CRMINQUERY:
   case OP_CRRANGE:
   case OP_CRMINRANGE:
+  case OP_CRPOSSTAR:
+  case OP_CRPOSPLUS:
+  case OP_CRPOSQUERY:
+  case OP_CRPOSRANGE:
   case OP_CLASS:
   case OP_NCLASS:
   case OP_REF:
   case OP_REFI:
+  case OP_DNREF:
+  case OP_DNREFI:
   case OP_RECURSE:
   case OP_CALLOUT:
   case OP_ALT:
@@ -614,9 +669,9 @@ switch(*cc)
   case OP_SCBRAPOS:
   case OP_SCOND:
   case OP_CREF:
-  case OP_NCREF:
+  case OP_DNCREF:
   case OP_RREF:
-  case OP_NRREF:
+  case OP_DNRREF:
   case OP_DEF:
   case OP_BRAZERO:
   case OP_BRAMINZERO:
@@ -736,10 +791,9 @@ switch(*cc)
 
 static BOOL check_opcode_types(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend)
 {
-pcre_uchar *name;
-pcre_uchar *name2;
-unsigned int cbra_index;
-int i;
+int count;
+pcre_uchar *slot;
+pcre_uchar *assert_back_end = cc - 1;
 
 /* Calculate important variables (like stack size) and checks whether all opcodes are supported. */
 while (cc < ccend)
@@ -748,6 +802,7 @@ while (cc < ccend)
     {
     case OP_SET_SOM:
     common->has_set_som = TRUE;
+    common->might_be_empty = TRUE;
     cc += 1;
     break;
 
@@ -773,29 +828,21 @@ while (cc < ccend)
     break;
 
     case OP_CREF:
-    i = GET2(cc, 1);
-    common->optimized_cbracket[i] = 0;
+    common->optimized_cbracket[GET2(cc, 1)] = 0;
     cc += 1 + IMM2_SIZE;
     break;
 
-    case OP_NCREF:
-    cbra_index = GET2(cc, 1);
-    name = (pcre_uchar *)common->name_table;
-    name2 = name;
-    for (i = 0; i < common->name_count; i++)
-      {
-      if (GET2(name, 0) == cbra_index) break;
-      name += common->name_entry_size;
-      }
-    SLJIT_ASSERT(i != common->name_count);
-
-    for (i = 0; i < common->name_count; i++)
+    case OP_DNREF:
+    case OP_DNREFI:
+    case OP_DNCREF:
+    count = GET2(cc, 1 + IMM2_SIZE);
+    slot = common->name_table + GET2(cc, 1) * common->name_entry_size;
+    while (count-- > 0)
       {
-      if (STRCMP_UC_UC(name2 + IMM2_SIZE, name + IMM2_SIZE) == 0)
-        common->optimized_cbracket[GET2(name2, 0)] = 0;
-      name2 += common->name_entry_size;
+      common->optimized_cbracket[GET2(slot, 0)] = 0;
+      slot += common->name_entry_size;
       }
-    cc += 1 + IMM2_SIZE;
+    cc += 1 + 2 * IMM2_SIZE;
     break;
 
     case OP_RECURSE:
@@ -817,15 +864,19 @@ while (cc < ccend)
     cc += 2 + 2 * LINK_SIZE;
     break;
 
+    case OP_ASSERTBACK:
+    slot = bracketend(cc);
+    if (slot > assert_back_end)
+      assert_back_end = slot;
+    cc += 1 + LINK_SIZE;
+    break;
+
     case OP_THEN_ARG:
     common->has_then = TRUE;
     common->control_head_ptr = 1;
     /* Fall through. */
 
     case OP_PRUNE_ARG:
-    common->needs_start_ptr = TRUE;
-    /* Fall through. */
-
     case OP_MARK:
     if (common->mark_ptr == 0)
       {
@@ -838,17 +889,20 @@ while (cc < ccend)
     case OP_THEN:
     common->has_then = TRUE;
     common->control_head_ptr = 1;
-    /* Fall through. */
+    cc += 1;
+    break;
 
-    case OP_PRUNE:
     case OP_SKIP:
-    common->needs_start_ptr = TRUE;
+    if (cc < assert_back_end)
+      common->has_skip_in_assert_back = TRUE;
     cc += 1;
     break;
 
     case OP_SKIP_ARG:
     common->control_head_ptr = 1;
     common->has_skip_arg = TRUE;
+    if (cc < assert_back_end)
+      common->has_skip_in_assert_back = TRUE;
     cc += 1 + 2 + cc[1];
     break;
 
@@ -862,8 +916,189 @@ while (cc < ccend)
 return TRUE;
 }
 
+static BOOL is_accelerated_repeat(pcre_uchar *cc)
+{
+switch(*cc)
+  {
+  case OP_TYPESTAR:
+  case OP_TYPEMINSTAR:
+  case OP_TYPEPLUS:
+  case OP_TYPEMINPLUS:
+  case OP_TYPEPOSSTAR:
+  case OP_TYPEPOSPLUS:
+  return (cc[1] != OP_ANYNL && cc[1] != OP_EXTUNI);
+
+  case OP_STAR:
+  case OP_MINSTAR:
+  case OP_PLUS:
+  case OP_MINPLUS:
+  case OP_POSSTAR:
+  case OP_POSPLUS:
+
+  case OP_STARI:
+  case OP_MINSTARI:
+  case OP_PLUSI:
+  case OP_MINPLUSI:
+  case OP_POSSTARI:
+  case OP_POSPLUSI:
+
+  case OP_NOTSTAR:
+  case OP_NOTMINSTAR:
+  case OP_NOTPLUS:
+  case OP_NOTMINPLUS:
+  case OP_NOTPOSSTAR:
+  case OP_NOTPOSPLUS:
+
+  case OP_NOTSTARI:
+  case OP_NOTMINSTARI:
+  case OP_NOTPLUSI:
+  case OP_NOTMINPLUSI:
+  case OP_NOTPOSSTARI:
+  case OP_NOTPOSPLUSI:
+  return TRUE;
+
+  case OP_CLASS:
+  case OP_NCLASS:
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+  case OP_XCLASS:
+  cc += (*cc == OP_XCLASS) ? GET(cc, 1) : (int)(1 + (32 / sizeof(pcre_uchar)));
+#else
+  cc += (1 + (32 / sizeof(pcre_uchar)));
+#endif
+
+  switch(*cc)
+    {
+    case OP_CRSTAR:
+    case OP_CRMINSTAR:
+    case OP_CRPLUS:
+    case OP_CRMINPLUS:
+    case OP_CRPOSSTAR:
+    case OP_CRPOSPLUS:
+    return TRUE;
+    }
+  break;
+  }
+return FALSE;
+}
+
+static SLJIT_INLINE BOOL detect_fast_forward_skip(compiler_common *common, int *private_data_start)
+{
+pcre_uchar *cc = common->start;
+pcre_uchar *end;
+
+/* Skip not repeated brackets. */
+while (TRUE)
+  {
+  switch(*cc)
+    {
+    case OP_SOD:
+    case OP_SOM:
+    case OP_SET_SOM:
+    case OP_NOT_WORD_BOUNDARY:
+    case OP_WORD_BOUNDARY:
+    case OP_EODN:
+    case OP_EOD:
+    case OP_CIRC:
+    case OP_CIRCM:
+    case OP_DOLL:
+    case OP_DOLLM:
+    /* Zero width assertions. */
+    cc++;
+    continue;
+    }
+
+  if (*cc != OP_BRA && *cc != OP_CBRA)
+    break;
+
+  end = cc + GET(cc, 1);
+  if (*end != OP_KET || PRIVATE_DATA(end) != 0)
+    return FALSE;
+  if (*cc == OP_CBRA)
+    {
+    if (common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0)
+      return FALSE;
+    cc += IMM2_SIZE;
+    }
+  cc += 1 + LINK_SIZE;
+  }
+
+if (is_accelerated_repeat(cc))
+  {
+  common->fast_forward_bc_ptr = cc;
+  common->private_data_ptrs[(cc + 1) - common->start] = *private_data_start;
+  *private_data_start += sizeof(sljit_sw);
+  return TRUE;
+  }
+return FALSE;
+}
+
+static SLJIT_INLINE void detect_fast_fail(compiler_common *common, pcre_uchar *cc, int *private_data_start, sljit_s32 depth)
+{
+  pcre_uchar *next_alt;
+
+  SLJIT_ASSERT(*cc == OP_BRA || *cc == OP_CBRA);
+
+  if (*cc == OP_CBRA && common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0)
+    return;
+
+  next_alt = bracketend(cc) - (1 + LINK_SIZE);
+  if (*next_alt != OP_KET || PRIVATE_DATA(next_alt) != 0)
+    return;
+
+  do
+    {
+    next_alt = cc + GET(cc, 1);
+
+    cc += 1 + LINK_SIZE + ((*cc == OP_CBRA) ? IMM2_SIZE : 0);
+
+    while (TRUE)
+      {
+      switch(*cc)
+        {
+        case OP_SOD:
+        case OP_SOM:
+        case OP_SET_SOM:
+        case OP_NOT_WORD_BOUNDARY:
+        case OP_WORD_BOUNDARY:
+        case OP_EODN:
+        case OP_EOD:
+        case OP_CIRC:
+        case OP_CIRCM:
+        case OP_DOLL:
+        case OP_DOLLM:
+        /* Zero width assertions. */
+        cc++;
+        continue;
+        }
+      break;
+      }
+
+    if (depth > 0 && (*cc == OP_BRA || *cc == OP_CBRA))
+      detect_fast_fail(common, cc, private_data_start, depth - 1);
+
+    if (is_accelerated_repeat(cc))
+      {
+      common->private_data_ptrs[(cc + 1) - common->start] = *private_data_start;
+
+      if (common->fast_fail_start_ptr == 0)
+        common->fast_fail_start_ptr = *private_data_start;
+
+      *private_data_start += sizeof(sljit_sw);
+      common->fast_fail_end_ptr = *private_data_start;
+
+      if (*private_data_start > SLJIT_MAX_LOCAL_SIZE)
+        return;
+      }
+
+    cc = next_alt;
+    }
+  while (*cc == OP_ALT);
+}
+
 static int get_class_iterator_size(pcre_uchar *cc)
 {
+sljit_u32 min;
+sljit_u32 max;
 switch(*cc)
   {
   case OP_CRSTAR:
@@ -878,9 +1113,14 @@ switch(*cc)
 
   case OP_CRRANGE:
   case OP_CRMINRANGE:
-  if (GET2(cc, 1) == GET2(cc, 1 + IMM2_SIZE))
-    return 0;
-  return 2;
+  min = GET2(cc, 1);
+  max = GET2(cc, 1 + IMM2_SIZE);
+  if (max == 0)
+    return (*cc == OP_CRRANGE) ? 2 : 1;
+  max -= min;
+  if (max > 2)
+    max = 2;
+  return max;
 
   default:
   return 0;
@@ -1031,6 +1271,7 @@ pcre_uchar *alternative;
 pcre_uchar *end = NULL;
 int private_data_ptr = *private_data_start;
 int space, size, bracketlen;
+BOOL repeat_check = TRUE;
 
 while (cc < ccend)
   {
@@ -1038,9 +1279,10 @@ while (cc < ccend)
   size = 0;
   bracketlen = 0;
   if (private_data_ptr > SLJIT_MAX_LOCAL_SIZE)
-    return;
+    break;
 
-  if (*cc == OP_ONCE || *cc == OP_ONCE_NC || *cc == OP_BRA || *cc == OP_CBRA || *cc == OP_COND)
+  if (repeat_check && (*cc == OP_ONCE || *cc == OP_ONCE_NC || *cc == OP_BRA || *cc == OP_CBRA || *cc == OP_COND))
+    {
     if (detect_repeat(common, cc))
       {
       /* These brackets are converted to repeats, so no global
@@ -1048,6 +1290,8 @@ while (cc < ccend)
       if (cc >= end)
         end = bracketend(cc);
       }
+    }
+  repeat_check = TRUE;
 
   switch(*cc)
     {
@@ -1103,6 +1347,13 @@ while (cc < ccend)
     bracketlen = 1 + LINK_SIZE + IMM2_SIZE;
     break;
 
+    case OP_BRAZERO:
+    case OP_BRAMINZERO:
+    case OP_BRAPOSZERO:
+    repeat_check = FALSE;
+    size = 1;
+    break;
+
     CASE_ITERATOR_PRIVATE_DATA_1
     space = 1;
     size = -2;
@@ -1129,22 +1380,27 @@ while (cc < ccend)
     size = 1;
     break;
 
-    CASE_ITERATOR_TYPE_PRIVATE_DATA_2B
+    case OP_TYPEUPTO:
     if (cc[1 + IMM2_SIZE] != OP_ANYNL && cc[1 + IMM2_SIZE] != OP_EXTUNI)
       space = 2;
     size = 1 + IMM2_SIZE;
     break;
 
+    case OP_TYPEMINUPTO:
+    space = 2;
+    size = 1 + IMM2_SIZE;
+    break;
+
     case OP_CLASS:
     case OP_NCLASS:
-    size += 1 + 32 / sizeof(pcre_uchar);
     space = get_class_iterator_size(cc + size);
+    size = 1 + 32 / sizeof(pcre_uchar);
     break;
 
 #if defined SUPPORT_UTF || !defined COMPILE_PCRE8
     case OP_XCLASS:
-    size = GET(cc, 1);
     space = get_class_iterator_size(cc + size);
+    size = GET(cc, 1);
     break;
 #endif
 
@@ -1190,7 +1446,7 @@ while (cc < ccend)
 }
 
 /* Returns with a frame_types (always < 0) if no need for frame. */
-static int get_framesize(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend, BOOL recursive, BOOL* needs_control_head)
+static int get_framesize(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend, BOOL recursive, BOOL *needs_control_head)
 {
 int length = 0;
 int possessive = 0;
@@ -1283,6 +1539,13 @@ while (cc < ccend)
     cc += 1 + LINK_SIZE + IMM2_SIZE;
     break;
 
+    case OP_THEN:
+    stack_restore = TRUE;
+    if (common->control_head_ptr != 0)
+      *needs_control_head = TRUE;
+    cc ++;
+    break;
+
     default:
     stack_restore = TRUE;
     /* Fall through. */
@@ -1350,6 +1613,7 @@ while (cc < ccend)
     case OP_CLASS:
     case OP_NCLASS:
     case OP_XCLASS:
+    case OP_CALLOUT:
 
     cc = next_opcode(common, cc);
     SLJIT_ASSERT(cc != NULL);
@@ -1394,7 +1658,7 @@ while (cc < ccend)
     SLJIT_ASSERT(common->has_set_som);
     if (!setsom_found)
       {
-      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0));
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -OVECTOR(0));
       stackpos += (int)sizeof(sljit_sw);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
@@ -1410,7 +1674,7 @@ while (cc < ccend)
     SLJIT_ASSERT(common->mark_ptr != 0);
     if (!setmark_found)
       {
-      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->mark_ptr);
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->mark_ptr);
       stackpos += (int)sizeof(sljit_sw);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
@@ -1423,7 +1687,7 @@ while (cc < ccend)
     case OP_RECURSE:
     if (common->has_set_som && !setsom_found)
       {
-      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0));
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -OVECTOR(0));
       stackpos += (int)sizeof(sljit_sw);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
@@ -1432,7 +1696,7 @@ while (cc < ccend)
       }
     if (common->mark_ptr != 0 && !setmark_found)
       {
-      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->mark_ptr);
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->mark_ptr);
       stackpos += (int)sizeof(sljit_sw);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
@@ -1441,7 +1705,7 @@ while (cc < ccend)
       }
     if (common->capture_last_ptr != 0 && !capture_last_found)
       {
-      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->capture_last_ptr);
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->capture_last_ptr);
       stackpos += (int)sizeof(sljit_sw);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
@@ -1457,7 +1721,7 @@ while (cc < ccend)
     case OP_SCBRAPOS:
     if (common->capture_last_ptr != 0 && !capture_last_found)
       {
-      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->capture_last_ptr);
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->capture_last_ptr);
       stackpos += (int)sizeof(sljit_sw);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
@@ -1467,8 +1731,8 @@ while (cc < ccend)
     offset = (GET2(cc, 1 + LINK_SIZE)) << 1;
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, OVECTOR(offset));
     stackpos += (int)sizeof(sljit_sw);
-    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
-    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset));
+    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1));
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
     stackpos += (int)sizeof(sljit_sw);
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP2, 0);
@@ -1500,7 +1764,11 @@ while (cc < ccend)
     {
     case OP_KET:
     if (PRIVATE_DATA(cc) != 0)
+      {
       private_data_length++;
+      SLJIT_ASSERT(PRIVATE_DATA(cc + 1) != 0);
+      cc += PRIVATE_DATA(cc + 1);
+      }
     cc += 1 + LINK_SIZE;
     break;
 
@@ -1515,6 +1783,7 @@ while (cc < ccend)
     case OP_SBRAPOS:
     case OP_SCOND:
     private_data_length++;
+    SLJIT_ASSERT(PRIVATE_DATA(cc) != 0);
     cc += 1 + LINK_SIZE;
     break;
 
@@ -1677,6 +1946,8 @@ do
         {
         count = 1;
         srcw[0] = PRIVATE_DATA(cc);
+        SLJIT_ASSERT(PRIVATE_DATA(cc + 1) != 0);
+        cc += PRIVATE_DATA(cc + 1);
         }
       cc += 1 + LINK_SIZE;
       break;
@@ -1848,7 +2119,7 @@ do
           OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP1, 0);
           stackptr += sizeof(sljit_sw);
           }
-        OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), srcw[count]);
+        OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), srcw[count]);
         tmp1empty = FALSE;
         tmp1next = FALSE;
         }
@@ -1859,7 +2130,7 @@ do
           OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP2, 0);
           stackptr += sizeof(sljit_sw);
           }
-        OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), srcw[count]);
+        OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), srcw[count]);
         tmp2empty = FALSE;
         tmp1next = TRUE;
         }
@@ -1869,7 +2140,7 @@ do
       if (tmp1next)
         {
         SLJIT_ASSERT(!tmp1empty);
-        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), srcw[count], TMP1, 0);
+        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), srcw[count], TMP1, 0);
         tmp1empty = stackptr >= stacktop;
         if (!tmp1empty)
           {
@@ -1881,7 +2152,7 @@ do
       else
         {
         SLJIT_ASSERT(!tmp2empty);
-        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), srcw[count], TMP2, 0);
+        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), srcw[count], TMP2, 0);
         tmp2empty = stackptr >= stacktop;
         if (!tmp2empty)
           {
@@ -1927,7 +2198,7 @@ if (save)
 SLJIT_ASSERT(cc == ccend && stackptr == stacktop && (save || (tmp1empty && tmp2empty)));
 }
 
-static SLJIT_INLINE pcre_uchar *set_then_offsets(compiler_common *common, pcre_uchar *cc, pcre_uint8 *current_offset)
+static SLJIT_INLINE pcre_uchar *set_then_offsets(compiler_common *common, pcre_uchar *cc, sljit_u8 *current_offset)
 {
 pcre_uchar *end = bracketend(cc);
 BOOL has_alternatives = cc[GET(cc, 1)] == OP_ALT;
@@ -1983,7 +2254,7 @@ while (list)
   }
 }
 
-static SLJIT_INLINE void add_jump(struct sljit_compiler *compiler, jump_list **list, struct sljit_jump* jump)
+static SLJIT_INLINE void add_jump(struct sljit_compiler *compiler, jump_list **list, struct sljit_jump *jump)
 {
 jump_list *list_item = sljit_alloc_memory(compiler, sizeof(jump_list));
 if (list_item)
@@ -1997,7 +2268,7 @@ if (list_item)
 static void add_stub(compiler_common *common, struct sljit_jump *start)
 {
 DEFINE_COMPILER;
-stub_list* list_item = sljit_alloc_memory(compiler, sizeof(stub_list));
+stub_list *list_item = sljit_alloc_memory(compiler, sizeof(stub_list));
 
 if (list_item)
   {
@@ -2011,7 +2282,7 @@ if (list_item)
 static void flush_stubs(compiler_common *common)
 {
 DEFINE_COMPILER;
-stub_list* list_item = common->stubs;
+stub_list *list_item = common->stubs;
 
 while (list_item)
   {
@@ -2023,12 +2294,26 @@ while (list_item)
 common->stubs = NULL;
 }
 
+static void add_label_addr(compiler_common *common, sljit_uw *update_addr)
+{
+DEFINE_COMPILER;
+label_addr_list *label_addr;
+
+label_addr = sljit_alloc_memory(compiler, sizeof(label_addr_list));
+if (label_addr == NULL)
+  return;
+label_addr->label = LABEL();
+label_addr->update_addr = update_addr;
+label_addr->next = common->label_addrs;
+common->label_addrs = label_addr;
+}
+
 static SLJIT_INLINE void count_match(compiler_common *common)
 {
 DEFINE_COMPILER;
 
 OP2(SLJIT_SUB | SLJIT_SET_E, COUNT_MATCH, 0, COUNT_MATCH, 0, SLJIT_IMM, 1);
-add_jump(compiler, &common->calllimit, JUMP(SLJIT_C_ZERO));
+add_jump(compiler, &common->calllimit, JUMP(SLJIT_ZERO));
 }
 
 static SLJIT_INLINE void allocate_stack(compiler_common *common, int size)
@@ -2036,23 +2321,60 @@ static SLJIT_INLINE void allocate_stack(compiler_common *common, int size)
 /* May destroy all locals and registers except TMP2. */
 DEFINE_COMPILER;
 
+SLJIT_ASSERT(size > 0);
 OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw));
 #ifdef DESTROY_REGISTERS
 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 12345);
 OP1(SLJIT_MOV, TMP3, 0, TMP1, 0);
 OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0);
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, TMP1, 0);
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, TMP1, 0);
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, TMP1, 0);
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP1, 0);
 #endif
-add_stub(common, CMP(SLJIT_C_GREATER, STACK_TOP, 0, STACK_LIMIT, 0));
+add_stub(common, CMP(SLJIT_GREATER, STACK_TOP, 0, STACK_LIMIT, 0));
 }
 
 static SLJIT_INLINE void free_stack(compiler_common *common, int size)
 {
 DEFINE_COMPILER;
+
+SLJIT_ASSERT(size > 0);
 OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw));
 }
 
+static sljit_uw * allocate_read_only_data(compiler_common *common, sljit_uw size)
+{
+DEFINE_COMPILER;
+sljit_uw *result;
+
+if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
+  return NULL;
+
+result = (sljit_uw *)SLJIT_MALLOC(size + sizeof(sljit_uw), compiler->allocator_data);
+if (SLJIT_UNLIKELY(result == NULL))
+  {
+  sljit_set_compiler_memory_error(compiler);
+  return NULL;
+  }
+
+*(void**)result = common->read_only_data_head;
+common->read_only_data_head = (void *)result;
+return result + 1;
+}
+
+static void free_read_only_data(void *current, void *allocator_data)
+{
+void *next;
+
+SLJIT_UNUSED_ARG(allocator_data);
+
+while (current != NULL)
+  {
+  next = *(void**)current;
+  SLJIT_FREE(current, allocator_data);
+  current = next;
+  }
+}
+
 static SLJIT_INLINE void reset_ovector(compiler_common *common, int length)
 {
 DEFINE_COMPILER;
@@ -2062,23 +2384,35 @@ int i;
 /* At this point we can freely use all temporary registers. */
 SLJIT_ASSERT(length > 1);
 /* TMP1 returns with begin - 1. */
-OP2(SLJIT_SUB, SLJIT_SCRATCH_REG1, 0, SLJIT_MEM1(SLJIT_SAVED_REG1), SLJIT_OFFSETOF(jit_arguments, begin), SLJIT_IMM, IN_UCHARS(1));
+OP2(SLJIT_SUB, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_S0), SLJIT_OFFSETOF(jit_arguments, begin), SLJIT_IMM, IN_UCHARS(1));
 if (length < 8)
   {
   for (i = 1; i < length; i++)
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(i), SLJIT_SCRATCH_REG1, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(i), SLJIT_R0, 0);
   }
 else
   {
-  GET_LOCAL_BASE(SLJIT_SCRATCH_REG2, 0, OVECTOR_START);
-  OP1(SLJIT_MOV, SLJIT_SCRATCH_REG3, 0, SLJIT_IMM, length - 1);
+  GET_LOCAL_BASE(SLJIT_R1, 0, OVECTOR_START);
+  OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, length - 1);
   loop = LABEL();
-  OP1(SLJIT_MOVU, SLJIT_MEM1(SLJIT_SCRATCH_REG2), sizeof(sljit_sw), SLJIT_SCRATCH_REG1, 0);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_SCRATCH_REG3, 0, SLJIT_SCRATCH_REG3, 0, SLJIT_IMM, 1);
-  JUMPTO(SLJIT_C_NOT_ZERO, loop);
+  OP1(SLJIT_MOVU, SLJIT_MEM1(SLJIT_R1), sizeof(sljit_sw), SLJIT_R0, 0);
+  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, 1);
+  JUMPTO(SLJIT_NOT_ZERO, loop);
   }
 }
 
+static SLJIT_INLINE void reset_fast_fail(compiler_common *common)
+{
+DEFINE_COMPILER;
+sljit_s32 i;
+
+SLJIT_ASSERT(common->fast_fail_start_ptr < common->fast_fail_end_ptr);
+
+OP2(SLJIT_SUB, TMP1, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+for (i = common->fast_fail_start_ptr; i < common->fast_fail_end_ptr; i += sizeof(sljit_sw))
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), i, TMP1, 0);
+}
+
 static SLJIT_INLINE void do_reset_match(compiler_common *common, int length)
 {
 DEFINE_COMPILER;
@@ -2088,11 +2422,11 @@ int i;
 SLJIT_ASSERT(length > 1);
 /* OVECTOR(1) contains the "string begin - 1" constant. */
 if (length > 2)
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1));
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1));
 if (length < 8)
   {
   for (i = 2; i < length; i++)
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(i), TMP1, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(i), TMP1, 0);
   }
 else
   {
@@ -2101,16 +2435,16 @@ else
   loop = LABEL();
   OP1(SLJIT_MOVU, SLJIT_MEM1(TMP2), sizeof(sljit_sw), TMP1, 0);
   OP2(SLJIT_SUB | SLJIT_SET_E, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 1);
-  JUMPTO(SLJIT_C_NOT_ZERO, loop);
+  JUMPTO(SLJIT_NOT_ZERO, loop);
   }
 
 OP1(SLJIT_MOV, STACK_TOP, 0, ARGUMENTS, 0);
 if (common->mark_ptr != 0)
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->mark_ptr, SLJIT_IMM, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, SLJIT_IMM, 0);
 if (common->control_head_ptr != 0)
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, SLJIT_IMM, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0);
 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), SLJIT_OFFSETOF(jit_arguments, stack));
-OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->start_ptr);
+OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->start_ptr);
 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), SLJIT_OFFSETOF(struct sljit_stack, base));
 }
 
@@ -2132,6 +2466,7 @@ while (current != NULL)
     SLJIT_ASSERT_STOP();
     break;
     }
+  SLJIT_ASSERT(current > (sljit_sw*)current[-1]);
   current = (sljit_sw*)current[-1];
   }
 return -1;
@@ -2144,44 +2479,44 @@ struct sljit_label *loop;
 struct sljit_jump *early_quit;
 
 /* At this point we can freely use all registers. */
-OP1(SLJIT_MOV, SLJIT_SAVED_REG3, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1));
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1), STR_PTR, 0);
+OP1(SLJIT_MOV, SLJIT_S2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1));
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(1), STR_PTR, 0);
 
-OP1(SLJIT_MOV, SLJIT_SCRATCH_REG1, 0, ARGUMENTS, 0);
+OP1(SLJIT_MOV, SLJIT_R0, 0, ARGUMENTS, 0);
 if (common->mark_ptr != 0)
-  OP1(SLJIT_MOV, SLJIT_SCRATCH_REG3, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->mark_ptr);
-OP1(SLJIT_MOV_SI, SLJIT_SCRATCH_REG2, 0, SLJIT_MEM1(SLJIT_SCRATCH_REG1), SLJIT_OFFSETOF(jit_arguments, offset_count));
+  OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr);
+OP1(SLJIT_MOV_S32, SLJIT_R1, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, offset_count));
 if (common->mark_ptr != 0)
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SCRATCH_REG1), SLJIT_OFFSETOF(jit_arguments, mark_ptr), SLJIT_SCRATCH_REG3, 0);
-OP2(SLJIT_SUB, SLJIT_SCRATCH_REG3, 0, SLJIT_MEM1(SLJIT_SCRATCH_REG1), SLJIT_OFFSETOF(jit_arguments, offsets), SLJIT_IMM, sizeof(int));
-OP1(SLJIT_MOV, SLJIT_SCRATCH_REG1, 0, SLJIT_MEM1(SLJIT_SCRATCH_REG1), SLJIT_OFFSETOF(jit_arguments, begin));
-GET_LOCAL_BASE(SLJIT_SAVED_REG1, 0, OVECTOR_START);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, mark_ptr), SLJIT_R2, 0);
+OP2(SLJIT_SUB, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, offsets), SLJIT_IMM, sizeof(int));
+OP1(SLJIT_MOV, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, begin));
+GET_LOCAL_BASE(SLJIT_S0, 0, OVECTOR_START);
 /* Unlikely, but possible */
-early_quit = CMP(SLJIT_C_EQUAL, SLJIT_SCRATCH_REG2, 0, SLJIT_IMM, 0);
+early_quit = CMP(SLJIT_EQUAL, SLJIT_R1, 0, SLJIT_IMM, 0);
 loop = LABEL();
-OP2(SLJIT_SUB, SLJIT_SAVED_REG2, 0, SLJIT_MEM1(SLJIT_SAVED_REG1), 0, SLJIT_SCRATCH_REG1, 0);
-OP2(SLJIT_ADD, SLJIT_SAVED_REG1, 0, SLJIT_SAVED_REG1, 0, SLJIT_IMM, sizeof(sljit_sw));
+OP2(SLJIT_SUB, SLJIT_S1, 0, SLJIT_MEM1(SLJIT_S0), 0, SLJIT_R0, 0);
+OP2(SLJIT_ADD, SLJIT_S0, 0, SLJIT_S0, 0, SLJIT_IMM, sizeof(sljit_sw));
 /* Copy the integer value to the output buffer */
 #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32
-OP2(SLJIT_ASHR, SLJIT_SAVED_REG2, 0, SLJIT_SAVED_REG2, 0, SLJIT_IMM, UCHAR_SHIFT);
+OP2(SLJIT_ASHR, SLJIT_S1, 0, SLJIT_S1, 0, SLJIT_IMM, UCHAR_SHIFT);
 #endif
-OP1(SLJIT_MOVU_SI, SLJIT_MEM1(SLJIT_SCRATCH_REG3), sizeof(int), SLJIT_SAVED_REG2, 0);
-OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_SCRATCH_REG2, 0, SLJIT_SCRATCH_REG2, 0, SLJIT_IMM, 1);
-JUMPTO(SLJIT_C_NOT_ZERO, loop);
+OP1(SLJIT_MOVU_S32, SLJIT_MEM1(SLJIT_R2), sizeof(int), SLJIT_S1, 0);
+OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1);
+JUMPTO(SLJIT_NOT_ZERO, loop);
 JUMPHERE(early_quit);
 
 /* Calculate the return value, which is the maximum ovector value. */
 if (topbracket > 1)
   {
-  GET_LOCAL_BASE(SLJIT_SCRATCH_REG1, 0, OVECTOR_START + topbracket * 2 * sizeof(sljit_sw));
-  OP1(SLJIT_MOV, SLJIT_SCRATCH_REG2, 0, SLJIT_IMM, topbracket + 1);
+  GET_LOCAL_BASE(SLJIT_R0, 0, OVECTOR_START + topbracket * 2 * sizeof(sljit_sw));
+  OP1(SLJIT_MOV, SLJIT_R1, 0, SLJIT_IMM, topbracket + 1);
 
-  /* OVECTOR(0) is never equal to SLJIT_SAVED_REG3. */
+  /* OVECTOR(0) is never equal to SLJIT_S2. */
   loop = LABEL();
-  OP1(SLJIT_MOVU, SLJIT_SCRATCH_REG3, 0, SLJIT_MEM1(SLJIT_SCRATCH_REG1), -(2 * (sljit_sw)sizeof(sljit_sw)));
-  OP2(SLJIT_SUB, SLJIT_SCRATCH_REG2, 0, SLJIT_SCRATCH_REG2, 0, SLJIT_IMM, 1);
-  CMPTO(SLJIT_C_EQUAL, SLJIT_SCRATCH_REG3, 0, SLJIT_SAVED_REG3, 0, loop);
-  OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_SCRATCH_REG2, 0);
+  OP1(SLJIT_MOVU, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_R0), -(2 * (sljit_sw)sizeof(sljit_sw)));
+  OP2(SLJIT_SUB, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1);
+  CMPTO(SLJIT_EQUAL, SLJIT_R2, 0, SLJIT_S2, 0, loop);
+  OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_R1, 0);
   }
 else
   OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1);
@@ -2192,39 +2527,39 @@ static SLJIT_INLINE void return_with_partial_match(compiler_common *common, stru
 DEFINE_COMPILER;
 struct sljit_jump *jump;
 
-SLJIT_COMPILE_ASSERT(STR_END == SLJIT_SAVED_REG2, str_end_must_be_saved_reg2);
+SLJIT_COMPILE_ASSERT(STR_END == SLJIT_S1, str_end_must_be_saved_reg2);
 SLJIT_ASSERT(common->start_used_ptr != 0 && common->start_ptr != 0
   && (common->mode == JIT_PARTIAL_SOFT_COMPILE ? common->hit_start != 0 : common->hit_start == 0));
 
-OP1(SLJIT_MOV, SLJIT_SCRATCH_REG2, 0, ARGUMENTS, 0);
+OP1(SLJIT_MOV, SLJIT_R1, 0, ARGUMENTS, 0);
 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_PARTIAL);
-OP1(SLJIT_MOV_SI, SLJIT_SCRATCH_REG3, 0, SLJIT_MEM1(SLJIT_SCRATCH_REG2), SLJIT_OFFSETOF(jit_arguments, real_offset_count));
-CMPTO(SLJIT_C_SIG_LESS, SLJIT_SCRATCH_REG3, 0, SLJIT_IMM, 2, quit);
+OP1(SLJIT_MOV_S32, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_R1), SLJIT_OFFSETOF(jit_arguments, real_offset_count));
+CMPTO(SLJIT_SIG_LESS, SLJIT_R2, 0, SLJIT_IMM, 2, quit);
 
 /* Store match begin and end. */
-OP1(SLJIT_MOV, SLJIT_SAVED_REG1, 0, SLJIT_MEM1(SLJIT_SCRATCH_REG2), SLJIT_OFFSETOF(jit_arguments, begin));
-OP1(SLJIT_MOV, SLJIT_SCRATCH_REG2, 0, SLJIT_MEM1(SLJIT_SCRATCH_REG2), SLJIT_OFFSETOF(jit_arguments, offsets));
+OP1(SLJIT_MOV, SLJIT_S0, 0, SLJIT_MEM1(SLJIT_R1), SLJIT_OFFSETOF(jit_arguments, begin));
+OP1(SLJIT_MOV, SLJIT_R1, 0, SLJIT_MEM1(SLJIT_R1), SLJIT_OFFSETOF(jit_arguments, offsets));
 
-jump = CMP(SLJIT_C_SIG_LESS, SLJIT_SCRATCH_REG3, 0, SLJIT_IMM, 3);
-OP2(SLJIT_SUB, SLJIT_SCRATCH_REG3, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->mode == JIT_PARTIAL_HARD_COMPILE ? common->start_ptr : (common->hit_start + (int)sizeof(sljit_sw)), SLJIT_SAVED_REG1, 0);
+jump = CMP(SLJIT_SIG_LESS, SLJIT_R2, 0, SLJIT_IMM, 3);
+OP2(SLJIT_SUB, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), common->mode == JIT_PARTIAL_HARD_COMPILE ? common->start_ptr : (common->hit_start + (int)sizeof(sljit_sw)), SLJIT_S0, 0);
 #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32
-OP2(SLJIT_ASHR, SLJIT_SCRATCH_REG3, 0, SLJIT_SCRATCH_REG3, 0, SLJIT_IMM, UCHAR_SHIFT);
+OP2(SLJIT_ASHR, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, UCHAR_SHIFT);
 #endif
-OP1(SLJIT_MOV_SI, SLJIT_MEM1(SLJIT_SCRATCH_REG2), 2 * sizeof(int), SLJIT_SCRATCH_REG3, 0);
+OP1(SLJIT_MOV_S32, SLJIT_MEM1(SLJIT_R1), 2 * sizeof(int), SLJIT_R2, 0);
 JUMPHERE(jump);
 
-OP1(SLJIT_MOV, SLJIT_SCRATCH_REG3, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->mode == JIT_PARTIAL_HARD_COMPILE ? common->start_used_ptr : common->hit_start);
-OP2(SLJIT_SUB, SLJIT_SAVED_REG2, 0, STR_END, 0, SLJIT_SAVED_REG1, 0);
+OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), common->mode == JIT_PARTIAL_HARD_COMPILE ? common->start_used_ptr : common->hit_start);
+OP2(SLJIT_SUB, SLJIT_S1, 0, STR_END, 0, SLJIT_S0, 0);
 #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32
-OP2(SLJIT_ASHR, SLJIT_SAVED_REG2, 0, SLJIT_SAVED_REG2, 0, SLJIT_IMM, UCHAR_SHIFT);
+OP2(SLJIT_ASHR, SLJIT_S1, 0, SLJIT_S1, 0, SLJIT_IMM, UCHAR_SHIFT);
 #endif
-OP1(SLJIT_MOV_SI, SLJIT_MEM1(SLJIT_SCRATCH_REG2), sizeof(int), SLJIT_SAVED_REG2, 0);
+OP1(SLJIT_MOV_S32, SLJIT_MEM1(SLJIT_R1), sizeof(int), SLJIT_S1, 0);
 
-OP2(SLJIT_SUB, SLJIT_SCRATCH_REG3, 0, SLJIT_SCRATCH_REG3, 0, SLJIT_SAVED_REG1, 0);
+OP2(SLJIT_SUB, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_S0, 0);
 #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32
-OP2(SLJIT_ASHR, SLJIT_SCRATCH_REG3, 0, SLJIT_SCRATCH_REG3, 0, SLJIT_IMM, UCHAR_SHIFT);
+OP2(SLJIT_ASHR, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, UCHAR_SHIFT);
 #endif
-OP1(SLJIT_MOV_SI, SLJIT_MEM1(SLJIT_SCRATCH_REG2), 0, SLJIT_SCRATCH_REG3, 0);
+OP1(SLJIT_MOV_S32, SLJIT_MEM1(SLJIT_R1), 0, SLJIT_R2, 0);
 
 JUMPTO(SLJIT_JUMP, quit);
 }
@@ -2238,22 +2573,22 @@ struct sljit_jump *jump;
 if (common->mode == JIT_PARTIAL_SOFT_COMPILE)
   {
   /* The value of -1 must be kept for start_used_ptr! */
-  OP2(SLJIT_ADD, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->start_used_ptr, SLJIT_IMM, 1);
+  OP2(SLJIT_ADD, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, SLJIT_IMM, 1);
   /* Jumps if start_used_ptr < STR_PTR, or start_used_ptr == -1. Although overwriting
   is not necessary if start_used_ptr == STR_PTR, it does not hurt as well. */
-  jump = CMP(SLJIT_C_LESS_EQUAL, TMP1, 0, STR_PTR, 0);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->start_used_ptr, STR_PTR, 0);
+  jump = CMP(SLJIT_LESS_EQUAL, TMP1, 0, STR_PTR, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0);
   JUMPHERE(jump);
   }
 else if (common->mode == JIT_PARTIAL_HARD_COMPILE)
   {
-  jump = CMP(SLJIT_C_LESS_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), common->start_used_ptr, STR_PTR, 0);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->start_used_ptr, STR_PTR, 0);
+  jump = CMP(SLJIT_LESS_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0);
   JUMPHERE(jump);
   }
 }
 
-static SLJIT_INLINE BOOL char_has_othercase(compiler_common *common, pcre_uchar* cc)
+static SLJIT_INLINE BOOL char_has_othercase(compiler_common *common, pcre_uchar *cc)
 {
 /* Detects if the character has an othercase. */
 unsigned int c;
@@ -2296,7 +2631,7 @@ if (common->utf && c > 127)
 return TABLE_GET(c, common->fcc, c);
 }
 
-static unsigned int char_get_othercase_bit(compiler_common *common, pcre_uchar* cc)
+static unsigned int char_get_othercase_bit(compiler_common *common, pcre_uchar *cc)
 {
 /* Detects if the character and its othercase has only 1 bit difference. */
 unsigned int c, oc, bit;
@@ -2384,12 +2719,12 @@ if (common->mode == JIT_COMPILE)
   return;
 
 if (!force)
-  jump = CMP(SLJIT_C_GREATER_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), common->start_used_ptr, STR_PTR, 0);
+  jump = CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0);
 else if (common->mode == JIT_PARTIAL_SOFT_COMPILE)
-  jump = CMP(SLJIT_C_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), common->start_used_ptr, SLJIT_IMM, -1);
+  jump = CMP(SLJIT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, SLJIT_IMM, -1);
 
 if (common->mode == JIT_PARTIAL_SOFT_COMPILE)
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->hit_start, SLJIT_IMM, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, 0);
 else
   {
   if (common->partialmatchlabel != NULL)
@@ -2410,20 +2745,20 @@ struct sljit_jump *jump;
 
 if (common->mode == JIT_COMPILE)
   {
-  add_jump(compiler, end_reached, CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
+  add_jump(compiler, end_reached, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
   return;
   }
 
-jump = CMP(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0);
+jump = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0);
 if (common->mode == JIT_PARTIAL_SOFT_COMPILE)
   {
-  add_jump(compiler, end_reached, CMP(SLJIT_C_GREATER_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), common->start_used_ptr, STR_PTR, 0));
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->hit_start, SLJIT_IMM, 0);
+  add_jump(compiler, end_reached, CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0));
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, 0);
   add_jump(compiler, end_reached, JUMP(SLJIT_JUMP));
   }
 else
   {
-  add_jump(compiler, end_reached, CMP(SLJIT_C_GREATER_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), common->start_used_ptr, STR_PTR, 0));
+  add_jump(compiler, end_reached, CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0));
   if (common->partialmatchlabel != NULL)
     JUMPTO(SLJIT_JUMP, common->partialmatchlabel);
   else
@@ -2439,16 +2774,16 @@ struct sljit_jump *jump;
 
 if (common->mode == JIT_COMPILE)
   {
-  add_jump(compiler, backtracks, CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
+  add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
   return;
   }
 
 /* Partial matching mode. */
-jump = CMP(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0);
-add_jump(compiler, backtracks, CMP(SLJIT_C_GREATER_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), common->start_used_ptr, STR_PTR, 0));
+jump = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0);
+add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0));
 if (common->mode == JIT_PARTIAL_SOFT_COMPILE)
   {
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->hit_start, SLJIT_IMM, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, 0);
   add_jump(compiler, backtracks, JUMP(SLJIT_JUMP));
   }
 else
@@ -2461,107 +2796,290 @@ else
 JUMPHERE(jump);
 }
 
-static void read_char(compiler_common *common)
+static void peek_char(compiler_common *common, sljit_u32 max)
 {
-/* Reads the character into TMP1, updates STR_PTR.
+/* Reads the character into TMP1, keeps STR_PTR.
 Does not check STR_END. TMP2 Destroyed. */
 DEFINE_COMPILER;
 #if defined SUPPORT_UTF && !defined COMPILE_PCRE32
 struct sljit_jump *jump;
 #endif
 
+SLJIT_UNUSED_ARG(max);
+
 OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
-#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
 if (common->utf)
   {
-#if defined COMPILE_PCRE8
-  jump = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
-#elif defined COMPILE_PCRE16
-  jump = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xd800);
-#endif /* COMPILE_PCRE[8|16] */
+  if (max < 128) return;
+
+  jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
+  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
   add_jump(compiler, &common->utfreadchar, JUMP(SLJIT_FAST_CALL));
+  OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
   JUMPHERE(jump);
   }
 #endif /* SUPPORT_UTF && !COMPILE_PCRE32 */
-OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
-}
-
-static void peek_char(compiler_common *common)
-{
-/* Reads the character into TMP1, keeps STR_PTR.
-Does not check STR_END. TMP2 Destroyed. */
-DEFINE_COMPILER;
-#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
-struct sljit_jump *jump;
-#endif
 
-OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
-#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+#if defined SUPPORT_UTF && defined COMPILE_PCRE16
 if (common->utf)
   {
-#if defined COMPILE_PCRE8
-  jump = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
-#elif defined COMPILE_PCRE16
-  jump = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xd800);
-#endif /* COMPILE_PCRE[8|16] */
-  add_jump(compiler, &common->utfreadchar, JUMP(SLJIT_FAST_CALL));
-  OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
+  if (max < 0xd800) return;
+
+  OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800);
+  jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800 - 1);
+  /* TMP2 contains the high surrogate. */
+  OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
+  OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x40);
+  OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 10);
+  OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3ff);
+  OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
   JUMPHERE(jump);
   }
-#endif /* SUPPORT_UTF && !COMPILE_PCRE32 */
+#endif
 }
 
-static void read_char8_type(compiler_common *common)
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
+
+static BOOL is_char7_bitset(const sljit_u8 *bitset, BOOL nclass)
 {
-/* Reads the character type into TMP1, updates STR_PTR. Does not check STR_END. */
+/* Tells whether the character codes below 128 are enough
+to determine a match. */
+const sljit_u8 value = nclass ? 0xff : 0;
+const sljit_u8 *end = bitset + 32;
+
+bitset += 16;
+do
+  {
+  if (*bitset++ != value)
+    return FALSE;
+  }
+while (bitset < end);
+return TRUE;
+}
+
+static void read_char7_type(compiler_common *common, BOOL full_read)
+{
+/* Reads the precise character type of a character into TMP1, if the character
+is less than 128. Otherwise it returns with zero. Does not check STR_END. The
+full_read argument tells whether characters above max are accepted or not. */
 DEFINE_COMPILER;
-#if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32
 struct sljit_jump *jump;
+
+SLJIT_ASSERT(common->utf);
+
+OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), 0);
+OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+
+OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
+
+if (full_read)
+  {
+  jump = CMP(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0xc0);
+  OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(utf8_table4) - 0xc0);
+  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
+  JUMPHERE(jump);
+  }
+}
+
+#endif /* SUPPORT_UTF && COMPILE_PCRE8 */
+
+static void read_char_range(compiler_common *common, sljit_u32 min, sljit_u32 max, BOOL update_str_ptr)
+{
+/* Reads the precise value of a character into TMP1, if the character is
+between min and max (c >= min && c <= max). Otherwise it returns with a value
+outside the range. Does not check STR_END. */
+DEFINE_COMPILER;
+#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+struct sljit_jump *jump;
+#endif
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
+struct sljit_jump *jump2;
 #endif
 
-#ifdef SUPPORT_UTF
+SLJIT_UNUSED_ARG(update_str_ptr);
+SLJIT_UNUSED_ARG(min);
+SLJIT_UNUSED_ARG(max);
+SLJIT_ASSERT(min <= max);
+
+OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
+OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
 if (common->utf)
   {
-  OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), 0);
-  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
-#if defined COMPILE_PCRE8
-  /* This can be an extra read in some situations, but hopefully
-  it is needed in most cases. */
-  OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
-  jump = CMP(SLJIT_C_LESS, TMP2, 0, SLJIT_IMM, 0xc0);
-  add_jump(compiler, &common->utfreadtype8, JUMP(SLJIT_FAST_CALL));
-  JUMPHERE(jump);
-#elif defined COMPILE_PCRE16
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
-  jump = CMP(SLJIT_C_GREATER, TMP2, 0, SLJIT_IMM, 255);
-  OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
+  if (max < 128 && !update_str_ptr) return;
+
+  jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
+  if (min >= 0x10000)
+    {
+    OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xf0);
+    if (update_str_ptr)
+      OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0);
+    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
+    jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0x7);
+    OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
+    OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f);
+    OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
+    OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
+    OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6);
+    OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
+    OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
+    OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(2));
+    if (!update_str_ptr)
+      OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3));
+    OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6);
+    OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
+    OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
+    JUMPHERE(jump2);
+    if (update_str_ptr)
+      OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, RETURN_ADDR, 0);
+    }
+  else if (min >= 0x800 && max <= 0xffff)
+    {
+    OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xe0);
+    if (update_str_ptr)
+      OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0);
+    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
+    jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xf);
+    OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
+    OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f);
+    OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
+    OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
+    if (!update_str_ptr)
+      OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));
+    OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6);
+    OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
+    OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
+    JUMPHERE(jump2);
+    if (update_str_ptr)
+      OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, RETURN_ADDR, 0);
+    }
+  else if (max >= 0x800)
+    add_jump(compiler, (max < 0x10000) ? &common->utfreadchar16 : &common->utfreadchar, JUMP(SLJIT_FAST_CALL));
+  else if (max < 128)
+    {
+    OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0);
+    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
+    }
+  else
+    {
+    OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
+    if (!update_str_ptr)
+      OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+    else
+      OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0);
+    OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f);
+    OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6);
+    OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
+    OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
+    if (update_str_ptr)
+      OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, RETURN_ADDR, 0);
+    }
   JUMPHERE(jump);
+  }
+#endif
+
+#if defined SUPPORT_UTF && defined COMPILE_PCRE16
+if (common->utf)
+  {
+  if (max >= 0x10000)
+    {
+    OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800);
+    jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800 - 1);
+    /* TMP2 contains the high surrogate. */
+    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
+    OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x40);
+    OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 10);
+    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+    OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3ff);
+    OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
+    JUMPHERE(jump);
+    return;
+    }
+
+  if (max < 0xd800 && !update_str_ptr) return;
+
   /* Skip low surrogate if necessary. */
-  OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xfc00);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0xd800);
-  OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
-  OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
-  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
-#elif defined COMPILE_PCRE32
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
-  jump = CMP(SLJIT_C_GREATER, TMP2, 0, SLJIT_IMM, 255);
-  OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
+  OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800);
+  jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800 - 1);
+  if (update_str_ptr)
+    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+  if (max >= 0xd800)
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0x10000);
   JUMPHERE(jump);
-#endif /* COMPILE_PCRE[8|16|32] */
-  return;
   }
-#endif /* SUPPORT_UTF */
+#endif
+}
+
+static SLJIT_INLINE void read_char(compiler_common *common)
+{
+read_char_range(common, 0, READ_CHAR_MAX, TRUE);
+}
+
+static void read_char8_type(compiler_common *common, BOOL update_str_ptr)
+{
+/* Reads the character type into TMP1, updates STR_PTR. Does not check STR_END. */
+DEFINE_COMPILER;
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+struct sljit_jump *jump;
+#endif
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
+struct sljit_jump *jump2;
+#endif
+
+SLJIT_UNUSED_ARG(update_str_ptr);
+
 OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), 0);
 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
-#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32
+
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
+if (common->utf)
+  {
+  /* This can be an extra read in some situations, but hopefully
+  it is needed in most cases. */
+  OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
+  jump = CMP(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0xc0);
+  if (!update_str_ptr)
+    {
+    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
+    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+    OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
+    OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
+    OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f);
+    OP2(SLJIT_OR, TMP2, 0, TMP2, 0, TMP1, 0);
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
+    jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 255);
+    OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
+    JUMPHERE(jump2);
+    }
+  else
+    add_jump(compiler, &common->utfreadtype8, JUMP(SLJIT_FAST_CALL));
+  JUMPHERE(jump);
+  return;
+  }
+#endif /* SUPPORT_UTF && COMPILE_PCRE8 */
+
+#if !defined COMPILE_PCRE8
 /* The ctypes array contains only 256 values. */
 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
-jump = CMP(SLJIT_C_GREATER, TMP2, 0, SLJIT_IMM, 255);
+jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 255);
 #endif
-OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
-#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32
+OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
+#if !defined COMPILE_PCRE8
 JUMPHERE(jump);
 #endif
+
+#if defined SUPPORT_UTF && defined COMPILE_PCRE16
+if (common->utf && update_str_ptr)
+  {
+  /* Skip low surrogate if necessary. */
+  OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xd800);
+  jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800 - 1);
+  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+  JUMPHERE(jump);
+  }
+#endif /* SUPPORT_UTF && COMPILE_PCRE16 */
 }
 
 static void skip_char_back(compiler_common *common)
@@ -2578,7 +3096,7 @@ if (common->utf)
   OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), -IN_UCHARS(1));
   OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
   OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0);
-  CMPTO(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 0x80, label);
+  CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0x80, label);
   return;
   }
 #elif defined COMPILE_PCRE16
@@ -2589,7 +3107,7 @@ if (common->utf)
   /* Skip low surrogate if necessary. */
   OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00);
   OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xdc00);
-  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
+  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
   OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
   OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
   return;
@@ -2599,28 +3117,35 @@ if (common->utf)
 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
 }
 
-static void check_newlinechar(compiler_common *common, int nltype, jump_list **backtracks, BOOL jumpiftrue)
+static void check_newlinechar(compiler_common *common, int nltype, jump_list **backtracks, BOOL jumpifmatch)
 {
 /* Character comes in TMP1. Checks if it is a newline. TMP2 may be destroyed. */
 DEFINE_COMPILER;
+struct sljit_jump *jump;
 
 if (nltype == NLTYPE_ANY)
   {
   add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL));
-  add_jump(compiler, backtracks, JUMP(jumpiftrue ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
+  add_jump(compiler, backtracks, JUMP(jumpifmatch ? SLJIT_NOT_ZERO : SLJIT_ZERO));
   }
 else if (nltype == NLTYPE_ANYCRLF)
   {
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_CR);
-  OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL);
-  OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
-  add_jump(compiler, backtracks, JUMP(jumpiftrue ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
+  if (jumpifmatch)
+    {
+    add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR));
+    add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL));
+    }
+  else
+    {
+    jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
+    add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL));
+    JUMPHERE(jump);
+    }
   }
 else
   {
   SLJIT_ASSERT(nltype == NLTYPE_FIXED && common->newline < 256);
-  add_jump(compiler, backtracks, CMP(jumpiftrue ? SLJIT_C_EQUAL : SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline));
+  add_jump(compiler, backtracks, CMP(jumpifmatch ? SLJIT_EQUAL : SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline));
   }
 }
 
@@ -2630,58 +3155,84 @@ else
 static void do_utfreadchar(compiler_common *common)
 {
 /* Fast decoding a UTF-8 character. TMP1 contains the first byte
-of the character (>= 0xc0). Return char value in TMP1, length - 1 in TMP2. */
+of the character (>= 0xc0). Return char value in TMP1, length in TMP2. */
 DEFINE_COMPILER;
 struct sljit_jump *jump;
 
 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
-/* Searching for the first zero. */
-OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x20);
-jump = JUMP(SLJIT_C_NOT_ZERO);
-/* Two byte sequence. */
-OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
-OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
-OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1f);
+OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
+OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f);
 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6);
 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
-OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, IN_UCHARS(1));
+
+/* Searching for the first zero. */
+OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800);
+jump = JUMP(SLJIT_NOT_ZERO);
+/* Two byte sequence. */
+OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, IN_UCHARS(2));
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
-JUMPHERE(jump);
 
-OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x10);
-jump = JUMP(SLJIT_C_NOT_ZERO);
-/* Three byte sequence. */
+JUMPHERE(jump);
 OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
-OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0f);
-OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 12);
+OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x800);
+OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6);
 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
-OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
-OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(2));
+
+OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x10000);
+jump = JUMP(SLJIT_NOT_ZERO);
+/* Three byte sequence. */
 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));
-OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
-OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
-OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, IN_UCHARS(2));
+OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, IN_UCHARS(3));
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
-JUMPHERE(jump);
 
 /* Four byte sequence. */
-OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
-OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x07);
-OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 18);
+JUMPHERE(jump);
+OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(2));
+OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x10000);
+OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6);
+OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3));
 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
-OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 12);
 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
-OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(2));
+OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, IN_UCHARS(4));
+sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
+}
+
+static void do_utfreadchar16(compiler_common *common)
+{
+/* Fast decoding a UTF-8 character. TMP1 contains the first byte
+of the character (>= 0xc0). Return value in TMP1. */
+DEFINE_COMPILER;
+struct sljit_jump *jump;
+
+sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
+OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
+OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f);
+OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6);
 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
-OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
-OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(3));
-OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3));
+
+/* Searching for the first zero. */
+OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800);
+jump = JUMP(SLJIT_NOT_ZERO);
+/* Two byte sequence. */
+OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
+
+JUMPHERE(jump);
+OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x400);
+OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_NOT_ZERO);
+/* This code runs only in 8 bit mode. No need to shift the value. */
+OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
+OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
+OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x800);
+OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6);
 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
-OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, IN_UCHARS(3));
+/* Three byte sequence. */
+OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 }
 
@@ -2696,58 +3247,32 @@ struct sljit_jump *compare;
 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
 
 OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x20);
-jump = JUMP(SLJIT_C_NOT_ZERO);
+jump = JUMP(SLJIT_NOT_ZERO);
 /* Two byte sequence. */
 OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x1f);
+/* The upper 5 bits are known at this point. */
+compare = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0x3);
 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f);
 OP2(SLJIT_OR, TMP2, 0, TMP2, 0, TMP1, 0);
-compare = CMP(SLJIT_C_GREATER, TMP2, 0, SLJIT_IMM, 255);
-OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
+OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 
 JUMPHERE(compare);
 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
-JUMPHERE(jump);
 
 /* We only have types for characters less than 256. */
-OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(utf8_table4) - 0xc0);
-OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
-OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
-sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
-}
-
-#elif defined COMPILE_PCRE16
-
-static void do_utfreadchar(compiler_common *common)
-{
-/* Fast decoding a UTF-16 character. TMP1 contains the first 16 bit char
-of the character (>= 0xd800). Return char value in TMP1, length - 1 in TMP2. */
-DEFINE_COMPILER;
-struct sljit_jump *jump;
-
-sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
-jump = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xdc00);
-/* Do nothing, only return. */
-sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
-
 JUMPHERE(jump);
-/* Combine two 16 bit characters. */
-OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
-OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
-OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3ff);
-OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 10);
-OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3ff);
-OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
-OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, IN_UCHARS(1));
-OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x10000);
+OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(utf8_table4) - 0xc0);
+OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
+OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 }
 
-#endif /* COMPILE_PCRE[8|16] */
+#endif /* COMPILE_PCRE8 */
 
 #endif /* SUPPORT_UTF */
 
@@ -2767,26 +3292,26 @@ SLJIT_ASSERT(UCD_BLOCK_SIZE == 128 && sizeof(ucd_record) == 8);
 
 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
 OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT);
-OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_stage1));
+OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_stage1));
 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK);
 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCD_BLOCK_SHIFT);
 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0);
 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_stage2));
-OP1(SLJIT_MOV_UH, TMP2, 0, SLJIT_MEM2(TMP2, TMP1), 1);
+OP1(SLJIT_MOV_U16, TMP2, 0, SLJIT_MEM2(TMP2, TMP1), 1);
 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype));
-OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3);
+OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3);
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 }
 #endif
 
-static SLJIT_INLINE struct sljit_label *mainloop_entry(compiler_common *common, BOOL hascrorlf, BOOL firstline)
+static SLJIT_INLINE struct sljit_label *mainloop_entry(compiler_common *common, BOOL hascrorlf)
 {
 DEFINE_COMPILER;
 struct sljit_label *mainloop;
 struct sljit_label *newlinelabel = NULL;
 struct sljit_jump *start;
 struct sljit_jump *end = NULL;
-struct sljit_jump *nl = NULL;
+struct sljit_jump *end2 = NULL;
 #if defined SUPPORT_UTF && !defined COMPILE_PCRE32
 struct sljit_jump *singlechar;
 #endif
@@ -2794,39 +3319,38 @@ jump_list *newline = NULL;
 BOOL newlinecheck = FALSE;
 BOOL readuchar = FALSE;
 
-if (!(hascrorlf || firstline) && (common->nltype == NLTYPE_ANY ||
-    common->nltype == NLTYPE_ANYCRLF || common->newline > 255))
+if (!(hascrorlf || (common->match_end_ptr != 0)) &&
+    (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF || common->newline > 255))
   newlinecheck = TRUE;
 
-if (firstline)
+if (common->match_end_ptr != 0)
   {
   /* Search for the end of the first line. */
-  SLJIT_ASSERT(common->first_line_end != 0);
   OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0);
 
   if (common->nltype == NLTYPE_FIXED && common->newline > 255)
     {
     mainloop = LABEL();
     OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
-    end = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
+    end = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
     OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1));
     OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
-    CMPTO(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, mainloop);
-    CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, mainloop);
+    CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, mainloop);
+    CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, mainloop);
     JUMPHERE(end);
-    OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_LOCALS_REG), common->first_line_end, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+    OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
     }
   else
     {
-    end = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
+    end = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
     mainloop = LABEL();
     /* Continual stores does not cause data dependency. */
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->first_line_end, STR_PTR, 0);
-    read_char(common);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, STR_PTR, 0);
+    read_char_range(common, common->nlmin, common->nlmax, TRUE);
     check_newlinechar(common, common->nltype, &newline, TRUE);
-    CMPTO(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0, mainloop);
+    CMPTO(SLJIT_LESS, STR_PTR, 0, STR_END, 0, mainloop);
     JUMPHERE(end);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->first_line_end, STR_PTR, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, STR_PTR, 0);
     set_jumps(newline, LABEL());
     }
 
@@ -2839,15 +3363,15 @@ if (newlinecheck)
   {
   newlinelabel = LABEL();
   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
-  end = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
+  end = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
   OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
   OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, common->newline & 0xff);
-  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
+  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
 #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32
   OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT);
 #endif
   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
-  nl = JUMP(SLJIT_JUMP);
+  end2 = JUMP(SLJIT_JUMP);
   }
 
 mainloop = LABEL();
@@ -2862,25 +3386,25 @@ if (readuchar)
   OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
 
 if (newlinecheck)
-  CMPTO(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, newlinelabel);
+  CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, newlinelabel);
 
 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
 #if defined SUPPORT_UTF && !defined COMPILE_PCRE32
 #if defined COMPILE_PCRE8
 if (common->utf)
   {
-  singlechar = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
-  OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0);
+  singlechar = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
+  OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0);
   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
   JUMPHERE(singlechar);
   }
 #elif defined COMPILE_PCRE16
 if (common->utf)
   {
-  singlechar = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xd800);
+  singlechar = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800);
   OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00);
   OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800);
-  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
+  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
   OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
   JUMPHERE(singlechar);
@@ -2892,43 +3416,79 @@ JUMPHERE(start);
 if (newlinecheck)
   {
   JUMPHERE(end);
-  JUMPHERE(nl);
+  JUMPHERE(end2);
   }
 
 return mainloop;
 }
 
-#define MAX_N_CHARS 3
+#define MAX_N_CHARS 16
+#define MAX_DIFF_CHARS 6
 
-static SLJIT_INLINE BOOL fast_forward_first_n_chars(compiler_common *common, BOOL firstline)
+static SLJIT_INLINE void add_prefix_char(pcre_uchar chr, pcre_uchar *chars)
 {
-DEFINE_COMPILER;
-struct sljit_label *start;
-struct sljit_jump *quit;
-pcre_uint32 chars[MAX_N_CHARS * 2];
-pcre_uchar *cc = common->start + 1 + LINK_SIZE;
-int location = 0;
-pcre_int32 len, c, bit, caseless;
-int must_stop;
-
-/* We do not support alternatives now. */
-if (*(common->start + GET(common->start, 1)) == OP_ALT)
-  return FALSE;
+pcre_uchar i, len;
+
+len = chars[0];
+if (len == 255)
+  return;
+
+if (len == 0)
+  {
+  chars[0] = 1;
+  chars[1] = chr;
+  return;
+  }
+
+for (i = len; i > 0; i--)
+  if (chars[i] == chr)
+    return;
+
+if (len >= MAX_DIFF_CHARS - 1)
+  {
+  chars[0] = 255;
+  return;
+  }
+
+len++;
+chars[len] = chr;
+chars[0] = len;
+}
+
+static int scan_prefix(compiler_common *common, pcre_uchar *cc, pcre_uchar *chars, int max_chars, sljit_u32 *rec_count)
+{
+/* Recursive function, which scans prefix literals. */
+BOOL last, any, class, caseless;
+int len, repeat, len_save, consumed = 0;
+sljit_u32 chr; /* Any unicode character. */
+sljit_u8 *bytes, *bytes_end, byte;
+pcre_uchar *alternative, *cc_save, *oc;
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
+pcre_uchar othercase[8];
+#elif defined SUPPORT_UTF && defined COMPILE_PCRE16
+pcre_uchar othercase[2];
+#else
+pcre_uchar othercase[1];
+#endif
 
+repeat = 1;
 while (TRUE)
   {
-  caseless = 0;
-  must_stop = 1;
-  switch(*cc)
-    {
-    case OP_CHAR:
-    must_stop = 0;
-    cc++;
-    break;
+  if (*rec_count == 0)
+    return 0;
+  (*rec_count)--;
 
+  last = TRUE;
+  any = FALSE;
+  class = FALSE;
+  caseless = FALSE;
+
+  switch (*cc)
+    {
     case OP_CHARI:
-    caseless = 1;
-    must_stop = 0;
+    caseless = TRUE;
+    case OP_CHAR:
+    last = FALSE;
     cc++;
     break;
 
@@ -2947,184 +3507,1045 @@ while (TRUE)
     cc++;
     continue;
 
-    case OP_PLUS:
-    case OP_MINPLUS:
-    case OP_POSPLUS:
-    cc++;
-    break;
-
-    case OP_EXACT:
-    cc += 1 + IMM2_SIZE;
-    break;
+    case OP_ASSERT:
+    case OP_ASSERT_NOT:
+    case OP_ASSERTBACK:
+    case OP_ASSERTBACK_NOT:
+    cc = bracketend(cc);
+    continue;
 
     case OP_PLUSI:
     case OP_MINPLUSI:
     case OP_POSPLUSI:
-    caseless = 1;
+    caseless = TRUE;
+    case OP_PLUS:
+    case OP_MINPLUS:
+    case OP_POSPLUS:
     cc++;
     break;
 
     case OP_EXACTI:
-    caseless = 1;
+    caseless = TRUE;
+    case OP_EXACT:
+    repeat = GET2(cc, 1);
+    last = FALSE;
     cc += 1 + IMM2_SIZE;
     break;
 
-    default:
-    must_stop = 2;
+    case OP_QUERYI:
+    case OP_MINQUERYI:
+    case OP_POSQUERYI:
+    caseless = TRUE;
+    case OP_QUERY:
+    case OP_MINQUERY:
+    case OP_POSQUERY:
+    len = 1;
+    cc++;
+#ifdef SUPPORT_UTF
+    if (common->utf && HAS_EXTRALEN(*cc)) len += GET_EXTRALEN(*cc);
+#endif
+    max_chars = scan_prefix(common, cc + len, chars, max_chars, rec_count);
+    if (max_chars == 0)
+      return consumed;
+    last = FALSE;
+    break;
+
+    case OP_KET:
+    cc += 1 + LINK_SIZE;
+    continue;
+
+    case OP_ALT:
+    cc += GET(cc, 1);
+    continue;
+
+    case OP_ONCE:
+    case OP_ONCE_NC:
+    case OP_BRA:
+    case OP_BRAPOS:
+    case OP_CBRA:
+    case OP_CBRAPOS:
+    alternative = cc + GET(cc, 1);
+    while (*alternative == OP_ALT)
+      {
+      max_chars = scan_prefix(common, alternative + 1 + LINK_SIZE, chars, max_chars, rec_count);
+      if (max_chars == 0)
+        return consumed;
+      alternative += GET(alternative, 1);
+      }
+
+    if (*cc == OP_CBRA || *cc == OP_CBRAPOS)
+      cc += IMM2_SIZE;
+    cc += 1 + LINK_SIZE;
+    continue;
+
+    case OP_CLASS:
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
+    if (common->utf && !is_char7_bitset((const sljit_u8 *)(cc + 1), FALSE))
+      return consumed;
+#endif
+    class = TRUE;
+    break;
+
+    case OP_NCLASS:
+#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+    if (common->utf) return consumed;
+#endif
+    class = TRUE;
+    break;
+
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+    case OP_XCLASS:
+#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+    if (common->utf) return consumed;
+#endif
+    any = TRUE;
+    cc += GET(cc, 1);
+    break;
+#endif
+
+    case OP_DIGIT:
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
+    if (common->utf && !is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_digit, FALSE))
+      return consumed;
+#endif
+    any = TRUE;
+    cc++;
     break;
+
+    case OP_WHITESPACE:
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
+    if (common->utf && !is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_space, FALSE))
+      return consumed;
+#endif
+    any = TRUE;
+    cc++;
+    break;
+
+    case OP_WORDCHAR:
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
+    if (common->utf && !is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_word, FALSE))
+      return consumed;
+#endif
+    any = TRUE;
+    cc++;
+    break;
+
+    case OP_NOT:
+    case OP_NOTI:
+    cc++;
+    /* Fall through. */
+    case OP_NOT_DIGIT:
+    case OP_NOT_WHITESPACE:
+    case OP_NOT_WORDCHAR:
+    case OP_ANY:
+    case OP_ALLANY:
+#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+    if (common->utf) return consumed;
+#endif
+    any = TRUE;
+    cc++;
+    break;
+
+#ifdef SUPPORT_UTF
+    case OP_NOTPROP:
+    case OP_PROP:
+#ifndef COMPILE_PCRE32
+    if (common->utf) return consumed;
+#endif
+    any = TRUE;
+    cc += 1 + 2;
+    break;
+#endif
+
+    case OP_TYPEEXACT:
+    repeat = GET2(cc, 1);
+    cc += 1 + IMM2_SIZE;
+    continue;
+
+    case OP_NOTEXACT:
+    case OP_NOTEXACTI:
+#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+    if (common->utf) return consumed;
+#endif
+    any = TRUE;
+    repeat = GET2(cc, 1);
+    cc += 1 + IMM2_SIZE + 1;
+    break;
+
+    default:
+    return consumed;
     }
 
-  if (must_stop == 2)
+  if (any)
+    {
+    do
+      {
+      chars[0] = 255;
+
+      consumed++;
+      if (--max_chars == 0)
+        return consumed;
+      chars += MAX_DIFF_CHARS;
+      }
+    while (--repeat > 0);
+
+    repeat = 1;
+    continue;
+    }
+
+  if (class)
+    {
+    bytes = (sljit_u8*) (cc + 1);
+    cc += 1 + 32 / sizeof(pcre_uchar);
+
+    switch (*cc)
+      {
+      case OP_CRSTAR:
+      case OP_CRMINSTAR:
+      case OP_CRPOSSTAR:
+      case OP_CRQUERY:
+      case OP_CRMINQUERY:
+      case OP_CRPOSQUERY:
+      max_chars = scan_prefix(common, cc + 1, chars, max_chars, rec_count);
+      if (max_chars == 0)
+        return consumed;
       break;
 
+      default:
+      case OP_CRPLUS:
+      case OP_CRMINPLUS:
+      case OP_CRPOSPLUS:
+      break;
+
+      case OP_CRRANGE:
+      case OP_CRMINRANGE:
+      case OP_CRPOSRANGE:
+      repeat = GET2(cc, 1);
+      if (repeat <= 0)
+        return consumed;
+      break;
+      }
+
+    do
+      {
+      if (bytes[31] & 0x80)
+        chars[0] = 255;
+      else if (chars[0] != 255)
+        {
+        bytes_end = bytes + 32;
+        chr = 0;
+        do
+          {
+          byte = *bytes++;
+          SLJIT_ASSERT((chr & 0x7) == 0);
+          if (byte == 0)
+            chr += 8;
+          else
+            {
+            do
+              {
+              if ((byte & 0x1) != 0)
+                add_prefix_char(chr, chars);
+              byte >>= 1;
+              chr++;
+              }
+            while (byte != 0);
+            chr = (chr + 7) & ~7;
+            }
+          }
+        while (chars[0] != 255 && bytes < bytes_end);
+        bytes = bytes_end - 32;
+        }
+
+      consumed++;
+      if (--max_chars == 0)
+        return consumed;
+      chars += MAX_DIFF_CHARS;
+      }
+    while (--repeat > 0);
+
+    switch (*cc)
+      {
+      case OP_CRSTAR:
+      case OP_CRMINSTAR:
+      case OP_CRPOSSTAR:
+      return consumed;
+
+      case OP_CRQUERY:
+      case OP_CRMINQUERY:
+      case OP_CRPOSQUERY:
+      cc++;
+      break;
+
+      case OP_CRRANGE:
+      case OP_CRMINRANGE:
+      case OP_CRPOSRANGE:
+      if (GET2(cc, 1) != GET2(cc, 1 + IMM2_SIZE))
+        return consumed;
+      cc += 1 + 2 * IMM2_SIZE;
+      break;
+      }
+
+    repeat = 1;
+    continue;
+    }
+
   len = 1;
 #ifdef SUPPORT_UTF
-  if (common->utf && HAS_EXTRALEN(cc[0])) len += GET_EXTRALEN(cc[0]);
+  if (common->utf && HAS_EXTRALEN(*cc)) len += GET_EXTRALEN(*cc);
 #endif
 
   if (caseless && char_has_othercase(common, cc))
     {
-    caseless = char_get_othercase_bit(common, cc);
-    if (caseless == 0)
-      return FALSE;
-#ifdef COMPILE_PCRE8
-    caseless = ((caseless & 0xff) << 8) | (len - (caseless >> 8));
-#else
-    if ((caseless & 0x100) != 0)
-      caseless = ((caseless & 0xff) << 16) | (len - (caseless >> 9));
+#ifdef SUPPORT_UTF
+    if (common->utf)
+      {
+      GETCHAR(chr, cc);
+      if ((int)PRIV(ord2utf)(char_othercase(common, chr), othercase) != len)
+        return consumed;
+      }
     else
-      caseless = ((caseless & 0xff) << 8) | (len - (caseless >> 9));
 #endif
+      {
+      chr = *cc;
+      othercase[0] = TABLE_GET(chr, common->fcc, chr);
+      }
     }
   else
-    caseless = 0;
+    {
+    caseless = FALSE;
+    othercase[0] = 0; /* Stops compiler warning - PH */
+    }
 
-  while (len > 0 && location < MAX_N_CHARS * 2)
+  len_save = len;
+  cc_save = cc;
+  while (TRUE)
     {
-    c = *cc;
-    bit = 0;
-    if (len == (caseless & 0xff))
+    oc = othercase;
+    do
       {
-      bit = caseless >> 8;
-      c |= bit;
+      chr = *cc;
+      add_prefix_char(*cc, chars);
+
+      if (caseless)
+        add_prefix_char(*oc, chars);
+
+      len--;
+      consumed++;
+      if (--max_chars == 0)
+        return consumed;
+      chars += MAX_DIFF_CHARS;
+      cc++;
+      oc++;
       }
+    while (len > 0);
 
-    chars[location] = c;
-    chars[location + 1] = bit;
+    if (--repeat == 0)
+      break;
 
-    len--;
-    location += 2;
-    cc++;
+    len = len_save;
+    cc = cc_save;
     }
 
-  if (location >= MAX_N_CHARS * 2 || must_stop != 0)
-    break;
+  repeat = 1;
+  if (last)
+    return consumed;
+  }
+}
+
+#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
+
+static sljit_s32 character_to_int32(pcre_uchar chr)
+{
+sljit_s32 value = (sljit_s32)chr;
+#if defined COMPILE_PCRE8
+#define SSE2_COMPARE_TYPE_INDEX 0
+return (value << 24) | (value << 16) | (value << 8) | value;
+#elif defined COMPILE_PCRE16
+#define SSE2_COMPARE_TYPE_INDEX 1
+return (value << 16) | value;
+#elif defined COMPILE_PCRE32
+#define SSE2_COMPARE_TYPE_INDEX 2
+return value;
+#else
+#error "Unsupported unit width"
+#endif
+}
+
+static SLJIT_INLINE void fast_forward_first_char2_sse2(compiler_common *common, pcre_uchar char1, pcre_uchar char2)
+{
+DEFINE_COMPILER;
+struct sljit_label *start;
+struct sljit_jump *quit[3];
+struct sljit_jump *nomatch;
+sljit_u8 instruction[8];
+sljit_s32 tmp1_ind = sljit_get_register_index(TMP1);
+sljit_s32 tmp2_ind = sljit_get_register_index(TMP2);
+sljit_s32 str_ptr_ind = sljit_get_register_index(STR_PTR);
+BOOL load_twice = FALSE;
+pcre_uchar bit;
+
+bit = char1 ^ char2;
+if (!is_powerof2(bit))
+  bit = 0;
+
+if ((char1 != char2) && bit == 0)
+  load_twice = TRUE;
+
+quit[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
+
+/* First part (unaligned start) */
+
+OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1 | bit));
+
+SLJIT_ASSERT(tmp1_ind < 8 && tmp2_ind == 1);
+
+/* MOVD xmm, r/m32 */
+instruction[0] = 0x66;
+instruction[1] = 0x0f;
+instruction[2] = 0x6e;
+instruction[3] = 0xc0 | (2 << 3) | tmp1_ind;
+sljit_emit_op_custom(compiler, instruction, 4);
+
+if (char1 != char2)
+  {
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(bit != 0 ? bit : char2));
+
+  /* MOVD xmm, r/m32 */
+  instruction[3] = 0xc0 | (3 << 3) | tmp1_ind;
+  sljit_emit_op_custom(compiler, instruction, 4);
+  }
+
+/* PSHUFD xmm1, xmm2/m128, imm8 */
+instruction[2] = 0x70;
+instruction[3] = 0xc0 | (2 << 3) | 2;
+instruction[4] = 0;
+sljit_emit_op_custom(compiler, instruction, 5);
+
+if (char1 != char2)
+  {
+  /* PSHUFD xmm1, xmm2/m128, imm8 */
+  instruction[3] = 0xc0 | (3 << 3) | 3;
+  instruction[4] = 0;
+  sljit_emit_op_custom(compiler, instruction, 5);
+  }
+
+OP2(SLJIT_AND, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 0xf);
+OP2(SLJIT_AND, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, ~0xf);
+
+/* MOVDQA xmm1, xmm2/m128 */
+#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
+
+if (str_ptr_ind < 8)
+  {
+  instruction[2] = 0x6f;
+  instruction[3] = (0 << 3) | str_ptr_ind;
+  sljit_emit_op_custom(compiler, instruction, 4);
+
+  if (load_twice)
+    {
+    instruction[3] = (1 << 3) | str_ptr_ind;
+    sljit_emit_op_custom(compiler, instruction, 4);
+    }
+  }
+else
+  {
+  instruction[1] = 0x41;
+  instruction[2] = 0x0f;
+  instruction[3] = 0x6f;
+  instruction[4] = (0 << 3) | (str_ptr_ind & 0x7);
+  sljit_emit_op_custom(compiler, instruction, 5);
+
+  if (load_twice)
+    {
+    instruction[4] = (1 << 3) | str_ptr_ind;
+    sljit_emit_op_custom(compiler, instruction, 5);
+    }
+  instruction[1] = 0x0f;
+  }
+
+#else
+
+instruction[2] = 0x6f;
+instruction[3] = (0 << 3) | str_ptr_ind;
+sljit_emit_op_custom(compiler, instruction, 4);
+
+if (load_twice)
+  {
+  instruction[3] = (1 << 3) | str_ptr_ind;
+  sljit_emit_op_custom(compiler, instruction, 4);
+  }
+
+#endif
+
+if (bit != 0)
+  {
+  /* POR xmm1, xmm2/m128 */
+  instruction[2] = 0xeb;
+  instruction[3] = 0xc0 | (0 << 3) | 3;
+  sljit_emit_op_custom(compiler, instruction, 4);
+  }
+
+/* PCMPEQB/W/D xmm1, xmm2/m128 */
+instruction[2] = 0x74 + SSE2_COMPARE_TYPE_INDEX;
+instruction[3] = 0xc0 | (0 << 3) | 2;
+sljit_emit_op_custom(compiler, instruction, 4);
+
+if (load_twice)
+  {
+  instruction[3] = 0xc0 | (1 << 3) | 3;
+  sljit_emit_op_custom(compiler, instruction, 4);
+  }
+
+/* PMOVMSKB reg, xmm */
+instruction[2] = 0xd7;
+instruction[3] = 0xc0 | (tmp1_ind << 3) | 0;
+sljit_emit_op_custom(compiler, instruction, 4);
+
+if (load_twice)
+  {
+  OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP2, 0);
+  instruction[3] = 0xc0 | (tmp2_ind << 3) | 1;
+  sljit_emit_op_custom(compiler, instruction, 4);
+
+  OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
+  OP1(SLJIT_MOV, TMP2, 0, RETURN_ADDR, 0);
+  }
+
+OP2(SLJIT_ASHR, TMP1, 0, TMP1, 0, TMP2, 0);
+
+/* BSF r32, r/m32 */
+instruction[0] = 0x0f;
+instruction[1] = 0xbc;
+instruction[2] = 0xc0 | (tmp1_ind << 3) | tmp1_ind;
+sljit_emit_op_custom(compiler, instruction, 3);
+
+nomatch = JUMP(SLJIT_ZERO);
+
+OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
+OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
+quit[1] = JUMP(SLJIT_JUMP);
+
+JUMPHERE(nomatch);
+
+start = LABEL();
+OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 16);
+quit[2] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
+
+/* Second part (aligned) */
+
+instruction[0] = 0x66;
+instruction[1] = 0x0f;
+
+/* MOVDQA xmm1, xmm2/m128 */
+#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
+
+if (str_ptr_ind < 8)
+  {
+  instruction[2] = 0x6f;
+  instruction[3] = (0 << 3) | str_ptr_ind;
+  sljit_emit_op_custom(compiler, instruction, 4);
+
+  if (load_twice)
+    {
+    instruction[3] = (1 << 3) | str_ptr_ind;
+    sljit_emit_op_custom(compiler, instruction, 4);
+    }
+  }
+else
+  {
+  instruction[1] = 0x41;
+  instruction[2] = 0x0f;
+  instruction[3] = 0x6f;
+  instruction[4] = (0 << 3) | (str_ptr_ind & 0x7);
+  sljit_emit_op_custom(compiler, instruction, 5);
+
+  if (load_twice)
+    {
+    instruction[4] = (1 << 3) | str_ptr_ind;
+    sljit_emit_op_custom(compiler, instruction, 5);
+    }
+  instruction[1] = 0x0f;
+  }
+
+#else
+
+instruction[2] = 0x6f;
+instruction[3] = (0 << 3) | str_ptr_ind;
+sljit_emit_op_custom(compiler, instruction, 4);
+
+if (load_twice)
+  {
+  instruction[3] = (1 << 3) | str_ptr_ind;
+  sljit_emit_op_custom(compiler, instruction, 4);
+  }
+
+#endif
+
+if (bit != 0)
+  {
+  /* POR xmm1, xmm2/m128 */
+  instruction[2] = 0xeb;
+  instruction[3] = 0xc0 | (0 << 3) | 3;
+  sljit_emit_op_custom(compiler, instruction, 4);
+  }
+
+/* PCMPEQB/W/D xmm1, xmm2/m128 */
+instruction[2] = 0x74 + SSE2_COMPARE_TYPE_INDEX;
+instruction[3] = 0xc0 | (0 << 3) | 2;
+sljit_emit_op_custom(compiler, instruction, 4);
+
+if (load_twice)
+  {
+  instruction[3] = 0xc0 | (1 << 3) | 3;
+  sljit_emit_op_custom(compiler, instruction, 4);
+  }
+
+/* PMOVMSKB reg, xmm */
+instruction[2] = 0xd7;
+instruction[3] = 0xc0 | (tmp1_ind << 3) | 0;
+sljit_emit_op_custom(compiler, instruction, 4);
+
+if (load_twice)
+  {
+  instruction[3] = 0xc0 | (tmp2_ind << 3) | 1;
+  sljit_emit_op_custom(compiler, instruction, 4);
+
+  OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
+  }
+
+/* BSF r32, r/m32 */
+instruction[0] = 0x0f;
+instruction[1] = 0xbc;
+instruction[2] = 0xc0 | (tmp1_ind << 3) | tmp1_ind;
+sljit_emit_op_custom(compiler, instruction, 3);
+
+JUMPTO(SLJIT_ZERO, start);
+
+OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
+
+start = LABEL();
+SET_LABEL(quit[0], start);
+SET_LABEL(quit[1], start);
+SET_LABEL(quit[2], start);
+}
+
+#undef SSE2_COMPARE_TYPE_INDEX
+
+#endif
+
+static void fast_forward_first_char2(compiler_common *common, pcre_uchar char1, pcre_uchar char2, sljit_s32 offset)
+{
+DEFINE_COMPILER;
+struct sljit_label *start;
+struct sljit_jump *quit;
+struct sljit_jump *found;
+pcre_uchar mask;
+#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+struct sljit_label *utf_start = NULL;
+struct sljit_jump *utf_quit = NULL;
+#endif
+BOOL has_match_end = (common->match_end_ptr != 0);
+
+if (offset > 0)
+  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset));
+
+if (has_match_end)
+  {
+  OP1(SLJIT_MOV, TMP3, 0, STR_END, 0);
+
+  OP2(SLJIT_ADD, STR_END, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, SLJIT_IMM, IN_UCHARS(offset + 1));
+#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
+  if (sljit_x86_is_cmov_available())
+    {
+    OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_END, 0, TMP3, 0);
+    sljit_x86_emit_cmov(compiler, SLJIT_GREATER, STR_END, TMP3, 0);
+    }
+#endif
+    {
+    quit = CMP(SLJIT_LESS_EQUAL, STR_END, 0, TMP3, 0);
+    OP1(SLJIT_MOV, STR_END, 0, TMP3, 0);
+    JUMPHERE(quit);
+    }
+  }
+
+#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+if (common->utf && offset > 0)
+  utf_start = LABEL();
+#endif
+
+#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
+
+/* SSE2 accelerated first character search. */
+
+if (sljit_x86_is_sse2_available())
+  {
+  fast_forward_first_char2_sse2(common, char1, char2);
+
+  SLJIT_ASSERT(common->mode == JIT_COMPILE || offset == 0);
+  if (common->mode == JIT_COMPILE)
+    {
+    /* In complete mode, we don't need to run a match when STR_PTR == STR_END. */
+    SLJIT_ASSERT(common->forced_quit_label == NULL);
+    OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_NOMATCH);
+    add_jump(compiler, &common->forced_quit, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
+
+#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+    if (common->utf && offset > 0)
+      {
+      SLJIT_ASSERT(common->mode == JIT_COMPILE);
+
+      OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-offset));
+      OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+#if defined COMPILE_PCRE8
+      OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0);
+      CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0x80, utf_start);
+#elif defined COMPILE_PCRE16
+      OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00);
+      CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0xdc00, utf_start);
+#else
+#error "Unknown code width"
+#endif
+      OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+      }
+#endif
+
+    if (offset > 0)
+      OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset));
+    }
+  else if (sljit_x86_is_cmov_available())
+    {
+    OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_PTR, 0, STR_END, 0);
+    sljit_x86_emit_cmov(compiler, SLJIT_GREATER_EQUAL, STR_PTR, has_match_end ? SLJIT_MEM1(SLJIT_SP) : STR_END, has_match_end ? common->match_end_ptr : 0);
+    }
+  else
+    {
+    quit = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0);
+    OP1(SLJIT_MOV, STR_PTR, 0, has_match_end ? SLJIT_MEM1(SLJIT_SP) : STR_END, has_match_end ? common->match_end_ptr : 0);
+    JUMPHERE(quit);
+    }
+
+  if (has_match_end)
+    OP1(SLJIT_MOV, STR_END, 0, TMP3, 0);
+  return;
+  }
+
+#endif
+
+quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
+
+start = LABEL();
+OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
+
+if (char1 == char2)
+  found = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, char1);
+else
+  {
+  mask = char1 ^ char2;
+  if (is_powerof2(mask))
+    {
+    OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, mask);
+    found = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, char1 | mask);
+    }
+  else
+    {
+    OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char1);
+    OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
+    OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char2);
+    OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+    found = JUMP(SLJIT_NOT_ZERO);
+    }
   }
 
-/* At least two characters are required. */
-if (location < 2 * 2)
+OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+CMPTO(SLJIT_LESS, STR_PTR, 0, STR_END, 0, start);
+
+#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+if (common->utf && offset > 0)
+  utf_quit = JUMP(SLJIT_JUMP);
+#endif
+
+JUMPHERE(found);
+
+#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+if (common->utf && offset > 0)
+  {
+  OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-offset));
+  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+#if defined COMPILE_PCRE8
+  OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0);
+  CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0x80, utf_start);
+#elif defined COMPILE_PCRE16
+  OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00);
+  CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0xdc00, utf_start);
+#else
+#error "Unknown code width"
+#endif
+  OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+  JUMPHERE(utf_quit);
+  }
+#endif
+
+JUMPHERE(quit);
+
+if (has_match_end)
+  {
+  quit = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0);
+  OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr);
+  if (offset > 0)
+    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset));
+  JUMPHERE(quit);
+  OP1(SLJIT_MOV, STR_END, 0, TMP3, 0);
+  }
+
+if (offset > 0)
+  OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset));
+}
+
+static SLJIT_INLINE BOOL fast_forward_first_n_chars(compiler_common *common)
+{
+DEFINE_COMPILER;
+struct sljit_label *start;
+struct sljit_jump *quit;
+struct sljit_jump *match;
+/* bytes[0] represent the number of characters between 0
+and MAX_N_BYTES - 1, 255 represents any character. */
+pcre_uchar chars[MAX_N_CHARS * MAX_DIFF_CHARS];
+sljit_s32 offset;
+pcre_uchar mask;
+pcre_uchar *char_set, *char_set_end;
+int i, max, from;
+int range_right = -1, range_len;
+sljit_u8 *update_table = NULL;
+BOOL in_range;
+sljit_u32 rec_count;
+
+for (i = 0; i < MAX_N_CHARS; i++)
+  chars[i * MAX_DIFF_CHARS] = 0;
+
+rec_count = 10000;
+max = scan_prefix(common, common->start, chars, MAX_N_CHARS, &rec_count);
+
+if (max < 1)
+  return FALSE;
+
+in_range = FALSE;
+/* Prevent compiler "uninitialized" warning */
+from = 0;
+range_len = 4 /* minimum length */ - 1;
+for (i = 0; i <= max; i++)
+  {
+  if (in_range && (i - from) > range_len && (chars[(i - 1) * MAX_DIFF_CHARS] < 255))
+    {
+    range_len = i - from;
+    range_right = i - 1;
+    }
+
+  if (i < max && chars[i * MAX_DIFF_CHARS] < 255)
+    {
+    SLJIT_ASSERT(chars[i * MAX_DIFF_CHARS] > 0);
+    if (!in_range)
+      {
+      in_range = TRUE;
+      from = i;
+      }
+    }
+  else
+    in_range = FALSE;
+  }
+
+if (range_right >= 0)
+  {
+  update_table = (sljit_u8 *)allocate_read_only_data(common, 256);
+  if (update_table == NULL)
+    return TRUE;
+  memset(update_table, IN_UCHARS(range_len), 256);
+
+  for (i = 0; i < range_len; i++)
+    {
+    char_set = chars + ((range_right - i) * MAX_DIFF_CHARS);
+    SLJIT_ASSERT(char_set[0] > 0 && char_set[0] < 255);
+    char_set_end = char_set + char_set[0];
+    char_set++;
+    while (char_set <= char_set_end)
+      {
+      if (update_table[(*char_set) & 0xff] > IN_UCHARS(i))
+        update_table[(*char_set) & 0xff] = IN_UCHARS(i);
+      char_set++;
+      }
+    }
+  }
+
+offset = -1;
+/* Scan forward. */
+for (i = 0; i < max; i++)
+  {
+  if (offset == -1)
+    {
+    if (chars[i * MAX_DIFF_CHARS] <= 2)
+      offset = i;
+    }
+  else if (chars[offset * MAX_DIFF_CHARS] == 2 && chars[i * MAX_DIFF_CHARS] <= 2)
+    {
+    if (chars[i * MAX_DIFF_CHARS] == 1)
+      offset = i;
+    else
+      {
+      mask = chars[offset * MAX_DIFF_CHARS + 1] ^ chars[offset * MAX_DIFF_CHARS + 2];
+      if (!is_powerof2(mask))
+        {
+        mask = chars[i * MAX_DIFF_CHARS + 1] ^ chars[i * MAX_DIFF_CHARS + 2];
+        if (is_powerof2(mask))
+          offset = i;
+        }
+      }
+    }
+  }
+
+if (range_right < 0)
+  {
+  if (offset < 0)
     return FALSE;
+  SLJIT_ASSERT(chars[offset * MAX_DIFF_CHARS] >= 1 && chars[offset * MAX_DIFF_CHARS] <= 2);
+  /* Works regardless the value is 1 or 2. */
+  mask = chars[offset * MAX_DIFF_CHARS + chars[offset * MAX_DIFF_CHARS]];
+  fast_forward_first_char2(common, chars[offset * MAX_DIFF_CHARS + 1], mask, offset);
+  return TRUE;
+  }
+
+if (range_right == offset)
+  offset = -1;
 
-if (firstline)
+SLJIT_ASSERT(offset == -1 || (chars[offset * MAX_DIFF_CHARS] >= 1 && chars[offset * MAX_DIFF_CHARS] <= 2));
+
+max -= 1;
+SLJIT_ASSERT(max > 0);
+if (common->match_end_ptr != 0)
   {
-  SLJIT_ASSERT(common->first_line_end != 0);
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr);
   OP1(SLJIT_MOV, TMP3, 0, STR_END, 0);
-  OP2(SLJIT_SUB, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->first_line_end, SLJIT_IMM, IN_UCHARS((location >> 1) - 1));
+  OP2(SLJIT_SUB, STR_END, 0, STR_END, 0, SLJIT_IMM, IN_UCHARS(max));
+  quit = CMP(SLJIT_LESS_EQUAL, STR_END, 0, TMP1, 0);
+  OP1(SLJIT_MOV, STR_END, 0, TMP1, 0);
+  JUMPHERE(quit);
   }
 else
-  OP2(SLJIT_SUB, STR_END, 0, STR_END, 0, SLJIT_IMM, IN_UCHARS((location >> 1) - 1));
+  OP2(SLJIT_SUB, STR_END, 0, STR_END, 0, SLJIT_IMM, IN_UCHARS(max));
+
+SLJIT_ASSERT(range_right >= 0);
+
+#if !(defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
+OP1(SLJIT_MOV, RETURN_ADDR, 0, SLJIT_IMM, (sljit_sw)update_table);
+#endif
 
 start = LABEL();
-quit = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
+quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
 
-OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
-OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
-OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
-if (chars[1] != 0)
-  OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, chars[1]);
-CMPTO(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, chars[0], start);
-if (location > 2 * 2)
-  OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
-if (chars[3] != 0)
-  OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, chars[3]);
-CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, chars[2], start);
-if (location > 2 * 2)
-  {
-  if (chars[5] != 0)
-    OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, chars[5]);
-  CMPTO(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, chars[4], start);
+#if defined COMPILE_PCRE8 || (defined SLJIT_LITTLE_ENDIAN && SLJIT_LITTLE_ENDIAN)
+OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(range_right));
+#else
+OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(range_right + 1) - 1);
+#endif
+
+#if !(defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
+OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(RETURN_ADDR, TMP1), 0);
+#else
+OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)update_table);
+#endif
+OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
+CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0, start);
+
+if (offset >= 0)
+  {
+  OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(offset));
+  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+
+  if (chars[offset * MAX_DIFF_CHARS] == 1)
+    CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, chars[offset * MAX_DIFF_CHARS + 1], start);
+  else
+    {
+    mask = chars[offset * MAX_DIFF_CHARS + 1] ^ chars[offset * MAX_DIFF_CHARS + 2];
+    if (is_powerof2(mask))
+      {
+      OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, mask);
+      CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, chars[offset * MAX_DIFF_CHARS + 1] | mask, start);
+      }
+    else
+      {
+      match = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, chars[offset * MAX_DIFF_CHARS + 1]);
+      CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, chars[offset * MAX_DIFF_CHARS + 2], start);
+      JUMPHERE(match);
+      }
+    }
+  }
+
+#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+if (common->utf && offset != 0)
+  {
+  if (offset < 0)
+    {
+    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
+    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+    }
+  else
+    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1));
+#if defined COMPILE_PCRE8
+  OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0);
+  CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0x80, start);
+#elif defined COMPILE_PCRE16
+  OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00);
+  CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0xdc00, start);
+#else
+#error "Unknown code width"
+#endif
+  if (offset < 0)
+    OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
   }
-OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+#endif
+
+if (offset >= 0)
+  OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
 
 JUMPHERE(quit);
 
-if (firstline)
+if (common->match_end_ptr != 0)
+  {
+  if (range_right >= 0)
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr);
   OP1(SLJIT_MOV, STR_END, 0, TMP3, 0);
+  if (range_right >= 0)
+    {
+    quit = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP1, 0);
+    OP1(SLJIT_MOV, STR_PTR, 0, TMP1, 0);
+    JUMPHERE(quit);
+    }
+  }
 else
-  OP2(SLJIT_ADD, STR_END, 0, STR_END, 0, SLJIT_IMM, IN_UCHARS((location >> 1) - 1));
+  OP2(SLJIT_ADD, STR_END, 0, STR_END, 0, SLJIT_IMM, IN_UCHARS(max));
 return TRUE;
 }
 
 #undef MAX_N_CHARS
+#undef MAX_DIFF_CHARS
 
-static SLJIT_INLINE void fast_forward_first_char(compiler_common *common, pcre_uchar first_char, BOOL caseless, BOOL firstline)
+static SLJIT_INLINE void fast_forward_first_char(compiler_common *common, pcre_uchar first_char, BOOL caseless)
 {
-DEFINE_COMPILER;
-struct sljit_label *start;
-struct sljit_jump *quit;
-struct sljit_jump *found;
-pcre_uchar oc, bit;
-
-if (firstline)
-  {
-  SLJIT_ASSERT(common->first_line_end != 0);
-  OP1(SLJIT_MOV, TMP3, 0, STR_END, 0);
-  OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->first_line_end);
-  }
-
-start = LABEL();
-quit = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
-OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
+pcre_uchar oc;
 
 oc = first_char;
 if (caseless)
   {
   oc = TABLE_GET(first_char, common->fcc, first_char);
-#if defined SUPPORT_UCP && !(defined COMPILE_PCRE8)
+#if defined SUPPORT_UCP && !defined COMPILE_PCRE8
   if (first_char > 127 && common->utf)
     oc = UCD_OTHERCASE(first_char);
 #endif
   }
-if (first_char == oc)
-  found = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, first_char);
-else
-  {
-  bit = first_char ^ oc;
-  if (is_powerof2(bit))
-    {
-    OP2(SLJIT_OR, TMP2, 0, TMP1, 0, SLJIT_IMM, bit);
-    found = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, first_char | bit);
-    }
-  else
-    {
-    OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, first_char);
-    OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
-    OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, oc);
-    OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
-    found = JUMP(SLJIT_C_NOT_ZERO);
-    }
-  }
 
-OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
-JUMPTO(SLJIT_JUMP, start);
-JUMPHERE(found);
-JUMPHERE(quit);
-
-if (firstline)
-  OP1(SLJIT_MOV, STR_END, 0, TMP3, 0);
+fast_forward_first_char2(common, first_char, oc, 0);
 }
 
-static SLJIT_INLINE void fast_forward_newline(compiler_common *common, BOOL firstline)
+static SLJIT_INLINE void fast_forward_newline(compiler_common *common)
 {
 DEFINE_COMPILER;
 struct sljit_label *loop;
@@ -3135,24 +4556,23 @@ struct sljit_jump *foundcr = NULL;
 struct sljit_jump *notfoundnl;
 jump_list *newline = NULL;
 
-if (firstline)
+if (common->match_end_ptr != 0)
   {
-  SLJIT_ASSERT(common->first_line_end != 0);
   OP1(SLJIT_MOV, TMP3, 0, STR_END, 0);
-  OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->first_line_end);
+  OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr);
   }
 
 if (common->nltype == NLTYPE_FIXED && common->newline > 255)
   {
-  lastchar = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
+  lastchar = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
   OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
   OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
   OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
-  firstchar = CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, TMP2, 0);
+  firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0);
 
   OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(2));
   OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_PTR, 0, TMP1, 0);
-  OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_GREATER_EQUAL);
+  OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_GREATER_EQUAL);
 #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32
   OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCHAR_SHIFT);
 #endif
@@ -3160,31 +4580,33 @@ if (common->nltype == NLTYPE_FIXED && common->newline > 255)
 
   loop = LABEL();
   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
-  quit = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
+  quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
   OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2));
   OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1));
-  CMPTO(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, loop);
-  CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, loop);
+  CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, loop);
+  CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, loop);
 
   JUMPHERE(quit);
   JUMPHERE(firstchar);
   JUMPHERE(lastchar);
 
-  if (firstline)
-    OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
+  if (common->match_end_ptr != 0)
+    OP1(SLJIT_MOV, STR_END, 0, TMP3, 0);
   return;
   }
 
 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
-firstchar = CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, TMP2, 0);
+firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0);
 skip_char_back(common);
 
 loop = LABEL();
-read_char(common);
-lastchar = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
+common->ff_newline_shortcut = loop;
+
+read_char_range(common, common->nlmin, common->nlmax, TRUE);
+lastchar = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
 if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF)
-  foundcr = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
+  foundcr = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
 check_newlinechar(common, common->nltype, &newline, FALSE);
 set_jumps(newline, loop);
 
@@ -3192,10 +4614,10 @@ if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF)
   {
   quit = JUMP(SLJIT_JUMP);
   JUMPHERE(foundcr);
-  notfoundnl = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
+  notfoundnl = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
   OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
   OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL);
-  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
+  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
 #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32
   OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT);
 #endif
@@ -3206,56 +4628,50 @@ if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF)
 JUMPHERE(lastchar);
 JUMPHERE(firstchar);
 
-if (firstline)
+if (common->match_end_ptr != 0)
   OP1(SLJIT_MOV, STR_END, 0, TMP3, 0);
 }
 
-static BOOL check_class_ranges(compiler_common *common, const pcre_uint8 *bits, BOOL nclass, jump_list **backtracks);
+static BOOL check_class_ranges(compiler_common *common, const sljit_u8 *bits, BOOL nclass, BOOL invert, jump_list **backtracks);
 
-static SLJIT_INLINE void fast_forward_start_bits(compiler_common *common, sljit_uw start_bits, BOOL firstline)
+static SLJIT_INLINE void fast_forward_start_bits(compiler_common *common, const sljit_u8 *start_bits)
 {
 DEFINE_COMPILER;
 struct sljit_label *start;
 struct sljit_jump *quit;
 struct sljit_jump *found = NULL;
 jump_list *matches = NULL;
-pcre_uint8 inverted_start_bits[32];
-int i;
 #ifndef COMPILE_PCRE8
 struct sljit_jump *jump;
 #endif
 
-for (i = 0; i < 32; ++i)
-  inverted_start_bits[i] = ~(((pcre_uint8*)start_bits)[i]);
-
-if (firstline)
+if (common->match_end_ptr != 0)
   {
-  SLJIT_ASSERT(common->first_line_end != 0);
   OP1(SLJIT_MOV, RETURN_ADDR, 0, STR_END, 0);
-  OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->first_line_end);
+  OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr);
   }
 
 start = LABEL();
-quit = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
+quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
 OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
 #ifdef SUPPORT_UTF
 if (common->utf)
   OP1(SLJIT_MOV, TMP3, 0, TMP1, 0);
 #endif
 
-if (!check_class_ranges(common, inverted_start_bits, (inverted_start_bits[31] & 0x80) != 0, &matches))
+if (!check_class_ranges(common, start_bits, (start_bits[31] & 0x80) != 0, TRUE, &matches))
   {
 #ifndef COMPILE_PCRE8
-  jump = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 255);
+  jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 255);
   OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 255);
   JUMPHERE(jump);
 #endif
   OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
   OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
-  OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), start_bits);
+  OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)start_bits);
   OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
   OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
-  found = JUMP(SLJIT_C_NOT_ZERO);
+  found = JUMP(SLJIT_NOT_ZERO);
   }
 
 #ifdef SUPPORT_UTF
@@ -3267,17 +4683,17 @@ OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
 #if defined COMPILE_PCRE8
 if (common->utf)
   {
-  CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0, start);
-  OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0);
+  CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0, start);
+  OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0);
   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
   }
 #elif defined COMPILE_PCRE16
 if (common->utf)
   {
-  CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xd800, start);
+  CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800, start);
   OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00);
   OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800);
-  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
+  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
   OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
   }
@@ -3290,7 +4706,7 @@ if (matches != NULL)
   set_jumps(matches, LABEL());
 JUMPHERE(quit);
 
-if (firstline)
+if (common->match_end_ptr != 0)
   OP1(SLJIT_MOV, STR_END, 0, RETURN_ADDR, 0);
 }
 
@@ -3303,13 +4719,13 @@ struct sljit_jump *alreadyfound;
 struct sljit_jump *found;
 struct sljit_jump *foundoc = NULL;
 struct sljit_jump *notfound;
-pcre_uint32 oc, bit;
+sljit_u32 oc, bit;
 
 SLJIT_ASSERT(common->req_char_ptr != 0);
-OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->req_char_ptr);
+OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->req_char_ptr);
 OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, REQ_BYTE_MAX);
-toolong = CMP(SLJIT_C_LESS, TMP1, 0, STR_END, 0);
-alreadyfound = CMP(SLJIT_C_LESS, STR_PTR, 0, TMP2, 0);
+toolong = CMP(SLJIT_LESS, TMP1, 0, STR_END, 0);
+alreadyfound = CMP(SLJIT_LESS, STR_PTR, 0, TMP2, 0);
 
 if (has_firstchar)
   OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
@@ -3317,7 +4733,7 @@ else
   OP1(SLJIT_MOV, TMP1, 0, STR_PTR, 0);
 
 loop = LABEL();
-notfound = CMP(SLJIT_C_GREATER_EQUAL, TMP1, 0, STR_END, 0);
+notfound = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, STR_END, 0);
 
 OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(TMP1), 0);
 oc = req_char;
@@ -3330,19 +4746,19 @@ if (caseless)
 #endif
   }
 if (req_char == oc)
-  found = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, req_char);
+  found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char);
 else
   {
   bit = req_char ^ oc;
   if (is_powerof2(bit))
     {
     OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, bit);
-    found = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, req_char | bit);
+    found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char | bit);
     }
   else
     {
-    found = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, req_char);
-    foundoc = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, oc);
+    found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char);
+    foundoc = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, oc);
     }
   }
 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1));
@@ -3351,7 +4767,7 @@ JUMPTO(SLJIT_JUMP, loop);
 JUMPHERE(found);
 if (foundoc)
   JUMPHERE(foundoc);
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->req_char_ptr, TMP1, 0);
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->req_char_ptr, TMP1, 0);
 JUMPHERE(alreadyfound);
 JUMPHERE(toolong);
 return notfound;
@@ -3371,7 +4787,7 @@ GET_LOCAL_BASE(TMP3, 0, 0);
 mainloop = LABEL();
 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), 0);
 OP2(SLJIT_SUB | SLJIT_SET_S, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0);
-jump = JUMP(SLJIT_C_SIG_LESS_EQUAL);
+jump = JUMP(SLJIT_SIG_LESS_EQUAL);
 
 OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP3, 0);
 OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(TMP1), sizeof(sljit_sw));
@@ -3380,7 +4796,7 @@ OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 3 * sizeof(sljit_sw));
 JUMPTO(SLJIT_JUMP, mainloop);
 
 JUMPHERE(jump);
-jump = JUMP(SLJIT_C_SIG_LESS);
+jump = JUMP(SLJIT_SIG_LESS);
 /* End of dropping frames. */
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 
@@ -3403,12 +4819,12 @@ struct sljit_jump *jump;
 
 SLJIT_COMPILE_ASSERT(ctype_word == 0x10, ctype_word_must_be_16);
 
-sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
+sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0);
 /* Get type of the previous char, and put it to LOCALS1. */
 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, SLJIT_IMM, 0);
-skipread = CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, TMP1, 0);
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, SLJIT_IMM, 0);
+skipread = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP1, 0);
 skip_char_back(common);
 check_start_used_ptr(common);
 read_char(common);
@@ -3418,32 +4834,32 @@ read_char(common);
 if (common->use_ucp)
   {
   OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1);
-  jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE);
+  jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE);
   add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
   OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll);
   OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
-  OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_LESS_EQUAL);
+  OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL);
   OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll);
   OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd);
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_C_LESS_EQUAL);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
   JUMPHERE(jump);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, TMP2, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP2, 0);
   }
 else
 #endif
   {
 #ifndef COMPILE_PCRE8
-  jump = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
+  jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255);
 #elif defined SUPPORT_UTF
   /* Here LOCALS1 has already been zeroed. */
   jump = NULL;
   if (common->utf)
-    jump = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
+    jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255);
 #endif /* COMPILE_PCRE8 */
-  OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), common->ctypes);
+  OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), common->ctypes);
   OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 4 /* ctype_word */);
   OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, TMP1, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP1, 0);
 #ifndef COMPILE_PCRE8
   JUMPHERE(jump);
 #elif defined SUPPORT_UTF
@@ -3455,21 +4871,21 @@ JUMPHERE(skipread);
 
 OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0);
 check_str_end(common, &skipread_list);
-peek_char(common);
+peek_char(common, READ_CHAR_MAX);
 
 /* Testing char type. This is a code duplication. */
 #ifdef SUPPORT_UCP
 if (common->use_ucp)
   {
   OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1);
-  jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE);
+  jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE);
   add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
   OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll);
   OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
-  OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_LESS_EQUAL);
+  OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL);
   OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll);
   OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd);
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_C_LESS_EQUAL);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
   JUMPHERE(jump);
   }
 else
@@ -3478,14 +4894,14 @@ else
 #ifndef COMPILE_PCRE8
   /* TMP2 may be destroyed by peek_char. */
   OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0);
-  jump = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
+  jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255);
 #elif defined SUPPORT_UTF
   OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0);
   jump = NULL;
   if (common->utf)
-    jump = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
+    jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255);
 #endif
-  OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), common->ctypes);
+  OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), common->ctypes);
   OP2(SLJIT_LSHR, TMP2, 0, TMP2, 0, SLJIT_IMM, 4 /* ctype_word */);
   OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
 #ifndef COMPILE_PCRE8
@@ -3497,114 +4913,20 @@ else
   }
 set_jumps(skipread_list, LABEL());
 
-OP2(SLJIT_XOR | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1);
-sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
+OP2(SLJIT_XOR | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1);
+sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0);
 }
 
-/*
-  range format:
-
-  ranges[0] = length of the range (max MAX_RANGE_SIZE, -1 means invalid range).
-  ranges[1] = first bit (0 or 1)
-  ranges[2-length] = position of the bit change (when the current bit is not equal to the previous)
-*/
-
-static BOOL check_ranges(compiler_common *common, int *ranges, jump_list **backtracks, BOOL readch)
+static BOOL check_class_ranges(compiler_common *common, const sljit_u8 *bits, BOOL nclass, BOOL invert, jump_list **backtracks)
 {
+/* May destroy TMP1. */
 DEFINE_COMPILER;
-struct sljit_jump *jump;
-
-if (ranges[0] < 0)
-  return FALSE;
-
-switch(ranges[0])
-  {
-  case 1:
-  if (readch)
-    read_char(common);
-  add_jump(compiler, backtracks, CMP(ranges[1] == 0 ? SLJIT_C_LESS : SLJIT_C_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[2]));
-  return TRUE;
-
-  case 2:
-  if (readch)
-    read_char(common);
-  OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[2]);
-  add_jump(compiler, backtracks, CMP(ranges[1] != 0 ? SLJIT_C_LESS : SLJIT_C_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[3] - ranges[2]));
-  return TRUE;
-
-  case 4:
-  if (ranges[2] + 1 == ranges[3] && ranges[4] + 1 == ranges[5])
-    {
-    if (readch)
-      read_char(common);
-    if (ranges[1] != 0)
-      {
-      add_jump(compiler, backtracks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, ranges[2]));
-      add_jump(compiler, backtracks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, ranges[4]));
-      }
-    else
-      {
-      jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, ranges[2]);
-      add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[4]));
-      JUMPHERE(jump);
-      }
-    return TRUE;
-    }
-  if ((ranges[3] - ranges[2]) == (ranges[5] - ranges[4]) && is_powerof2(ranges[4] - ranges[2]))
-    {
-    if (readch)
-      read_char(common);
-    OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[4] - ranges[2]);
-    OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[4]);
-    add_jump(compiler, backtracks, CMP(ranges[1] != 0 ? SLJIT_C_LESS : SLJIT_C_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[5] - ranges[4]));
-    return TRUE;
-    }
-  return FALSE;
-
-  default:
-  return FALSE;
-  }
-}
-
-static void get_ctype_ranges(compiler_common *common, int flag, int *ranges)
-{
-int i, bit, length;
-const pcre_uint8 *ctypes = (const pcre_uint8*)common->ctypes;
-
-bit = ctypes[0] & flag;
-ranges[0] = -1;
-ranges[1] = bit != 0 ? 1 : 0;
-length = 0;
-
-for (i = 1; i < 256; i++)
-  if ((ctypes[i] & flag) != bit)
-    {
-    if (length >= MAX_RANGE_SIZE)
-      return;
-    ranges[2 + length] = i;
-    length++;
-    bit ^= flag;
-    }
-
-if (bit != 0)
-  {
-  if (length >= MAX_RANGE_SIZE)
-    return;
-  ranges[2 + length] = 256;
-  length++;
-  }
-ranges[0] = length;
-}
-
-static BOOL check_class_ranges(compiler_common *common, const pcre_uint8 *bits, BOOL nclass, jump_list **backtracks)
-{
-int ranges[2 + MAX_RANGE_SIZE];
-pcre_uint8 bit, cbit, all;
+int ranges[MAX_RANGE_SIZE];
+sljit_u8 bit, cbit, all;
 int i, byte, length = 0;
 
 bit = bits[0] & 0x1;
-ranges[1] = bit;
-/* Can be 0 or 255. */
+/* All bits will be zero or one (since bit is zero or one). */
 all = -bit;
 
 for (i = 0; i < 256; )
@@ -3619,7 +4941,7 @@ for (i = 0; i < 256; )
       {
       if (length >= MAX_RANGE_SIZE)
         return FALSE;
-      ranges[2 + length] = i;
+      ranges[length] = i;
       length++;
       bit = cbit;
       all = -cbit;
@@ -3632,12 +4954,119 @@ if (((bit == 0) && nclass) || ((bit == 1) && !nclass))
   {
   if (length >= MAX_RANGE_SIZE)
     return FALSE;
-  ranges[2 + length] = 256;
+  ranges[length] = 256;
   length++;
   }
-ranges[0] = length;
 
-return check_ranges(common, ranges, backtracks, FALSE);
+if (length < 0 || length > 4)
+  return FALSE;
+
+bit = bits[0] & 0x1;
+if (invert) bit ^= 0x1;
+
+/* No character is accepted. */
+if (length == 0 && bit == 0)
+  add_jump(compiler, backtracks, JUMP(SLJIT_JUMP));
+
+switch(length)
+  {
+  case 0:
+  /* When bit != 0, all characters are accepted. */
+  return TRUE;
+
+  case 1:
+  add_jump(compiler, backtracks, CMP(bit == 0 ? SLJIT_LESS : SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[0]));
+  return TRUE;
+
+  case 2:
+  if (ranges[0] + 1 != ranges[1])
+    {
+    OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[0]);
+    add_jump(compiler, backtracks, CMP(bit != 0 ? SLJIT_LESS : SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[1] - ranges[0]));
+    }
+  else
+    add_jump(compiler, backtracks, CMP(bit != 0 ? SLJIT_EQUAL : SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[0]));
+  return TRUE;
+
+  case 3:
+  if (bit != 0)
+    {
+    add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[2]));
+    if (ranges[0] + 1 != ranges[1])
+      {
+      OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[0]);
+      add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[1] - ranges[0]));
+      }
+    else
+      add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[0]));
+    return TRUE;
+    }
+
+  add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[0]));
+  if (ranges[1] + 1 != ranges[2])
+    {
+    OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[1]);
+    add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[2] - ranges[1]));
+    }
+  else
+    add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[1]));
+  return TRUE;
+
+  case 4:
+  if ((ranges[1] - ranges[0]) == (ranges[3] - ranges[2])
+      && (ranges[0] | (ranges[2] - ranges[0])) == ranges[2]
+      && (ranges[1] & (ranges[2] - ranges[0])) == 0
+      && is_powerof2(ranges[2] - ranges[0]))
+    {
+    SLJIT_ASSERT((ranges[0] & (ranges[2] - ranges[0])) == 0 && (ranges[2] & ranges[3] & (ranges[2] - ranges[0])) != 0);
+    OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[2] - ranges[0]);
+    if (ranges[2] + 1 != ranges[3])
+      {
+      OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[2]);
+      add_jump(compiler, backtracks, CMP(bit != 0 ? SLJIT_LESS : SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[3] - ranges[2]));
+      }
+    else
+      add_jump(compiler, backtracks, CMP(bit != 0 ? SLJIT_EQUAL : SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[2]));
+    return TRUE;
+    }
+
+  if (bit != 0)
+    {
+    i = 0;
+    if (ranges[0] + 1 != ranges[1])
+      {
+      OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[0]);
+      add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[1] - ranges[0]));
+      i = ranges[0];
+      }
+    else
+      add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[0]));
+
+    if (ranges[2] + 1 != ranges[3])
+      {
+      OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[2] - i);
+      add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[3] - ranges[2]));
+      }
+    else
+      add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[2] - i));
+    return TRUE;
+    }
+
+  OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[0]);
+  add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[3] - ranges[0]));
+  if (ranges[1] + 1 != ranges[2])
+    {
+    OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[1] - ranges[0]);
+    add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[2] - ranges[1]));
+    }
+  else
+    add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[1] - ranges[0]));
+  return TRUE;
+
+  default:
+  SLJIT_ASSERT_STOP();
+  return FALSE;
+  }
 }
 
 static void check_anynewline(compiler_common *common)
@@ -3649,21 +5078,21 @@ sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
 
 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a);
 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a);
-OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_LESS_EQUAL);
+OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL);
 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a);
 #if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32
 #ifdef COMPILE_PCRE8
 if (common->utf)
   {
 #endif
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
   OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1);
   OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a);
 #ifdef COMPILE_PCRE8
   }
 #endif
 #endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */
-OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
+OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 }
 
@@ -3675,33 +5104,33 @@ DEFINE_COMPILER;
 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
 
 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x09);
-OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
+OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x20);
-OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
+OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xa0);
 #if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32
 #ifdef COMPILE_PCRE8
 if (common->utf)
   {
 #endif
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
   OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x1680);
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
   OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e);
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
   OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x2000);
   OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x200A - 0x2000);
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_C_LESS_EQUAL);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
   OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x202f - 0x2000);
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
   OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x205f - 0x2000);
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
   OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x3000 - 0x2000);
 #ifdef COMPILE_PCRE8
   }
 #endif
 #endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */
-OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
+OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
 
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 }
@@ -3715,21 +5144,21 @@ sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
 
 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a);
 OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a);
-OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_LESS_EQUAL);
+OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL);
 OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a);
 #if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32
 #ifdef COMPILE_PCRE8
 if (common->utf)
   {
 #endif
-  OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
+  OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
   OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1);
   OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a);
 #ifdef COMPILE_PCRE8
   }
 #endif
 #endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */
-OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
+OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
 
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 }
@@ -3746,21 +5175,21 @@ struct sljit_label *label;
 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
 OP1(SLJIT_MOV, TMP3, 0, CHAR1, 0);
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, CHAR2, 0);
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, CHAR2, 0);
 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1));
 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
 
 label = LABEL();
 OP1(MOVU_UCHAR, CHAR1, 0, SLJIT_MEM1(TMP1), IN_UCHARS(1));
 OP1(MOVU_UCHAR, CHAR2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
-jump = CMP(SLJIT_C_NOT_EQUAL, CHAR1, 0, CHAR2, 0);
+jump = CMP(SLJIT_NOT_EQUAL, CHAR1, 0, CHAR2, 0);
 OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1));
-JUMPTO(SLJIT_C_NOT_ZERO, label);
+JUMPTO(SLJIT_NOT_ZERO, label);
 
 JUMPHERE(jump);
 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
 OP1(SLJIT_MOV, CHAR1, 0, TMP3, 0);
-OP1(SLJIT_MOV, CHAR2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
+OP1(SLJIT_MOV, CHAR2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 }
 
@@ -3776,8 +5205,8 @@ sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
 
 OP1(SLJIT_MOV, TMP3, 0, LCC_TABLE, 0);
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, CHAR1, 0);
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, CHAR2, 0);
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, CHAR1, 0);
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, CHAR2, 0);
 OP1(SLJIT_MOV, LCC_TABLE, 0, SLJIT_IMM, common->lcc);
 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1));
 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
@@ -3786,26 +5215,26 @@ label = LABEL();
 OP1(MOVU_UCHAR, CHAR1, 0, SLJIT_MEM1(TMP1), IN_UCHARS(1));
 OP1(MOVU_UCHAR, CHAR2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
 #ifndef COMPILE_PCRE8
-jump = CMP(SLJIT_C_GREATER, CHAR1, 0, SLJIT_IMM, 255);
+jump = CMP(SLJIT_GREATER, CHAR1, 0, SLJIT_IMM, 255);
 #endif
-OP1(SLJIT_MOV_UB, CHAR1, 0, SLJIT_MEM2(LCC_TABLE, CHAR1), 0);
+OP1(SLJIT_MOV_U8, CHAR1, 0, SLJIT_MEM2(LCC_TABLE, CHAR1), 0);
 #ifndef COMPILE_PCRE8
 JUMPHERE(jump);
-jump = CMP(SLJIT_C_GREATER, CHAR2, 0, SLJIT_IMM, 255);
+jump = CMP(SLJIT_GREATER, CHAR2, 0, SLJIT_IMM, 255);
 #endif
-OP1(SLJIT_MOV_UB, CHAR2, 0, SLJIT_MEM2(LCC_TABLE, CHAR2), 0);
+OP1(SLJIT_MOV_U8, CHAR2, 0, SLJIT_MEM2(LCC_TABLE, CHAR2), 0);
 #ifndef COMPILE_PCRE8
 JUMPHERE(jump);
 #endif
-jump = CMP(SLJIT_C_NOT_EQUAL, CHAR1, 0, CHAR2, 0);
+jump = CMP(SLJIT_NOT_EQUAL, CHAR1, 0, CHAR2, 0);
 OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1));
-JUMPTO(SLJIT_C_NOT_ZERO, label);
+JUMPTO(SLJIT_NOT_ZERO, label);
 
 JUMPHERE(jump);
 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
 OP1(SLJIT_MOV, LCC_TABLE, 0, TMP3, 0);
-OP1(SLJIT_MOV, CHAR1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
-OP1(SLJIT_MOV, CHAR2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1);
+OP1(SLJIT_MOV, CHAR1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
+OP1(SLJIT_MOV, CHAR2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1);
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 }
 
@@ -3818,11 +5247,11 @@ sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 static const pcre_uchar * SLJIT_CALL do_utf_caselesscmp(pcre_uchar *src1, jit_arguments *args, pcre_uchar *end1)
 {
 /* This function would be ineffective to do in JIT level. */
-pcre_uint32 c1, c2;
+sljit_u32 c1, c2;
 const pcre_uchar *src2 = args->uchar_ptr;
 const pcre_uchar *end2 = args->end;
 const ucd_record *ur;
-const pcre_uint32 *pp;
+const sljit_u32 *pp;
 
 while (src1 < end1)
   {
@@ -3847,7 +5276,7 @@ return src2;
 #endif /* SUPPORT_UTF && SUPPORT_UCP */
 
 static pcre_uchar *byte_sequence_compare(compiler_common *common, BOOL caseless, pcre_uchar *cc,
-    compare_context* context, jump_list **backtracks)
+    compare_context *context, jump_list **backtracks)
 {
 DEFINE_COMPILER;
 unsigned int othercasebit = 0;
@@ -3882,16 +5311,16 @@ if (context->sourcereg == -1)
 #if defined COMPILE_PCRE8
 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
   if (context->length >= 4)
-    OP1(SLJIT_MOV_SI, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
+    OP1(SLJIT_MOV_S32, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
   else if (context->length >= 2)
-    OP1(SLJIT_MOV_UH, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
+    OP1(SLJIT_MOV_U16, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
   else
 #endif
-    OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
+    OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
 #elif defined COMPILE_PCRE16
 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
   if (context->length >= 4)
-    OP1(SLJIT_MOV_SI, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
+    OP1(SLJIT_MOV_S32, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
   else
 #endif
     OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length);
@@ -3933,12 +5362,12 @@ do
 #endif
     {
     if (context->length >= 4)
-      OP1(SLJIT_MOV_SI, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
+      OP1(SLJIT_MOV_S32, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
     else if (context->length >= 2)
-      OP1(SLJIT_MOV_UH, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
+      OP1(SLJIT_MOV_U16, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
 #if defined COMPILE_PCRE8
     else if (context->length >= 1)
-      OP1(SLJIT_MOV_UB, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
+      OP1(SLJIT_MOV_U8, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length);
 #endif /* COMPILE_PCRE8 */
     context->sourcereg = context->sourcereg == TMP1 ? TMP2 : TMP1;
 
@@ -3947,20 +5376,20 @@ do
       case 4 / sizeof(pcre_uchar):
       if (context->oc.asint != 0)
         OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asint);
-      add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asint | context->oc.asint));
+      add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asint | context->oc.asint));
       break;
 
       case 2 / sizeof(pcre_uchar):
       if (context->oc.asushort != 0)
         OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asushort);
-      add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asushort | context->oc.asushort));
+      add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asushort | context->oc.asushort));
       break;
 
 #ifdef COMPILE_PCRE8
       case 1:
       if (context->oc.asbyte != 0)
         OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asbyte);
-      add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asbyte | context->oc.asbyte));
+      add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asbyte | context->oc.asbyte));
       break;
 #endif
 
@@ -3982,10 +5411,10 @@ do
   if (othercasebit != 0 && othercasechar == cc)
     {
     OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, othercasebit);
-    add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, *cc | othercasebit));
+    add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, *cc | othercasebit));
     }
   else
-    add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, *cc));
+    add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, *cc));
 
 #endif
 
@@ -4004,104 +5433,76 @@ return cc;
 #define SET_TYPE_OFFSET(value) \
   if ((value) != typeoffset) \
     { \
-    if ((value) > typeoffset) \
-      OP2(SLJIT_SUB, typereg, 0, typereg, 0, SLJIT_IMM, (value) - typeoffset); \
-    else \
+    if ((value) < typeoffset) \
       OP2(SLJIT_ADD, typereg, 0, typereg, 0, SLJIT_IMM, typeoffset - (value)); \
+    else \
+      OP2(SLJIT_SUB, typereg, 0, typereg, 0, SLJIT_IMM, (value) - typeoffset); \
     } \
   typeoffset = (value);
 
 #define SET_CHAR_OFFSET(value) \
   if ((value) != charoffset) \
     { \
-    if ((value) > charoffset) \
-      OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, (value) - charoffset); \
+    if ((value) < charoffset) \
+      OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(charoffset - (value))); \
     else \
-      OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, charoffset - (value)); \
+      OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)((value) - charoffset)); \
     } \
   charoffset = (value);
 
+static pcre_uchar *compile_char1_matchingpath(compiler_common *common, pcre_uchar type, pcre_uchar *cc, jump_list **backtracks, BOOL check_str_ptr);
+
 static void compile_xclass_matchingpath(compiler_common *common, pcre_uchar *cc, jump_list **backtracks)
 {
 DEFINE_COMPILER;
 jump_list *found = NULL;
-jump_list **list = (*cc & XCL_NOT) == 0 ? &found : backtracks;
-pcre_int32 c, charoffset;
-const pcre_uint32 *other_cases;
+jump_list **list = (cc[0] & XCL_NOT) == 0 ? &found : backtracks;
+sljit_uw c, charoffset, max = 256, min = READ_CHAR_MAX;
 struct sljit_jump *jump = NULL;
 pcre_uchar *ccbegin;
 int compares, invertcmp, numberofcmps;
+#if defined SUPPORT_UTF && (defined COMPILE_PCRE8 || defined COMPILE_PCRE16)
+BOOL utf = common->utf;
+#endif
+
 #ifdef SUPPORT_UCP
 BOOL needstype = FALSE, needsscript = FALSE, needschar = FALSE;
 BOOL charsaved = FALSE;
-int typereg = TMP1, scriptreg = TMP1;
-pcre_int32 typeoffset;
+int typereg = TMP1;
+const sljit_u32 *other_cases;
+sljit_uw typeoffset;
 #endif
 
-/* Although SUPPORT_UTF must be defined, we are
-   not necessary in utf mode even in 8 bit mode. */
-detect_partial_match(common, backtracks);
-read_char(common);
-
-if ((*cc++ & XCL_MAP) != 0)
+/* Scanning the necessary info. */
+cc++;
+ccbegin = cc;
+compares = 0;
+if (cc[-1] & XCL_MAP)
   {
-  OP1(SLJIT_MOV, TMP3, 0, TMP1, 0);
-#ifndef COMPILE_PCRE8
-  jump = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
-#elif defined SUPPORT_UTF
-  if (common->utf)
-    jump = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
-#endif
-
-  if (!check_class_ranges(common, (const pcre_uint8 *)cc, TRUE, list))
-    {
-    OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
-    OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
-    OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc);
-    OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
-    OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
-    add_jump(compiler, list, JUMP(SLJIT_C_NOT_ZERO));
-    }
-
-#ifndef COMPILE_PCRE8
-  JUMPHERE(jump);
-#elif defined SUPPORT_UTF
-  if (common->utf)
-    JUMPHERE(jump);
-#endif
-  OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
-#ifdef SUPPORT_UCP
-  charsaved = TRUE;
-#endif
+  min = 0;
   cc += 32 / sizeof(pcre_uchar);
   }
 
-/* Scanning the necessary info. */
-ccbegin = cc;
-compares = 0;
 while (*cc != XCL_END)
   {
   compares++;
   if (*cc == XCL_SINGLE)
     {
-    cc += 2;
-#ifdef SUPPORT_UTF
-    if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
-#endif
+    cc ++;
+    GETCHARINCTEST(c, cc);
+    if (c > max) max = c;
+    if (c < min) min = c;
 #ifdef SUPPORT_UCP
     needschar = TRUE;
 #endif
     }
   else if (*cc == XCL_RANGE)
     {
-    cc += 2;
-#ifdef SUPPORT_UTF
-    if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
-#endif
-    cc++;
-#ifdef SUPPORT_UTF
-    if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
-#endif
+    cc ++;
+    GETCHARINCTEST(c, cc);
+    if (c < min) min = c;
+    GETCHARINCTEST(c, cc);
+    if (c > max) max = c;
 #ifdef SUPPORT_UCP
     needschar = TRUE;
 #endif
@@ -4111,9 +5512,33 @@ while (*cc != XCL_END)
     {
     SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP);
     cc++;
+    if (*cc == PT_CLIST)
+      {
+      other_cases = PRIV(ucd_caseless_sets) + cc[1];
+      while (*other_cases != NOTACHAR)
+        {
+        if (*other_cases > max) max = *other_cases;
+        if (*other_cases < min) min = *other_cases;
+        other_cases++;
+        }
+      }
+    else
+      {
+      max = READ_CHAR_MAX;
+      min = 0;
+      }
+
     switch(*cc)
       {
       case PT_ANY:
+      /* Any either accepts everything or ignored. */
+      if (cc[-1] == XCL_PROP)
+        {
+        compile_char1_matchingpath(common, OP_ALLANY, cc, backtracks, FALSE);
+        if (list == backtracks)
+          add_jump(compiler, backtracks, JUMP(SLJIT_JUMP));
+        return;
+        }
       break;
 
       case PT_LAMP:
@@ -4130,66 +5555,167 @@ while (*cc != XCL_END)
       case PT_SPACE:
       case PT_PXSPACE:
       case PT_WORD:
+      case PT_PXGRAPH:
+      case PT_PXPRINT:
+      case PT_PXPUNCT:
       needstype = TRUE;
       needschar = TRUE;
       break;
 
-      case PT_CLIST:
-      case PT_UCNC:
-      needschar = TRUE;
-      break;
+      case PT_CLIST:
+      case PT_UCNC:
+      needschar = TRUE;
+      break;
+
+      default:
+      SLJIT_ASSERT_STOP();
+      break;
+      }
+    cc += 2;
+    }
+#endif
+  }
+SLJIT_ASSERT(compares > 0);
+
+/* We are not necessary in utf mode even in 8 bit mode. */
+cc = ccbegin;
+read_char_range(common, min, max, (cc[-1] & XCL_NOT) != 0);
+
+if ((cc[-1] & XCL_HASPROP) == 0)
+  {
+  if ((cc[-1] & XCL_MAP) != 0)
+    {
+    jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255);
+    if (!check_class_ranges(common, (const sljit_u8 *)cc, (((const sljit_u8 *)cc)[31] & 0x80) != 0, TRUE, &found))
+      {
+      OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
+      OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
+      OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc);
+      OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
+      OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
+      add_jump(compiler, &found, JUMP(SLJIT_NOT_ZERO));
+      }
+
+    add_jump(compiler, backtracks, JUMP(SLJIT_JUMP));
+    JUMPHERE(jump);
+
+    cc += 32 / sizeof(pcre_uchar);
+    }
+  else
+    {
+    OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, min);
+    add_jump(compiler, (cc[-1] & XCL_NOT) == 0 ? backtracks : &found, CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, max - min));
+    }
+  }
+else if ((cc[-1] & XCL_MAP) != 0)
+  {
+  OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0);
+#ifdef SUPPORT_UCP
+  charsaved = TRUE;
+#endif
+  if (!check_class_ranges(common, (const sljit_u8 *)cc, FALSE, TRUE, list))
+    {
+#ifdef COMPILE_PCRE8
+    jump = NULL;
+    if (common->utf)
+#endif
+      jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255);
+
+    OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
+    OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
+    OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc);
+    OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
+    OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
+    add_jump(compiler, list, JUMP(SLJIT_NOT_ZERO));
 
-      default:
-      SLJIT_ASSERT_STOP();
-      break;
-      }
-    cc += 2;
-    }
+#ifdef COMPILE_PCRE8
+    if (common->utf)
 #endif
+      JUMPHERE(jump);
+    }
+
+  OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0);
+  cc += 32 / sizeof(pcre_uchar);
   }
 
 #ifdef SUPPORT_UCP
-/* Simple register allocation. TMP1 is preferred if possible. */
 if (needstype || needsscript)
   {
   if (needschar && !charsaved)
-    OP1(SLJIT_MOV, TMP3, 0, TMP1, 0);
-  add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
-  if (needschar)
+    OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0);
+
+  OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT);
+  OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_stage1));
+  OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK);
+  OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCD_BLOCK_SHIFT);
+  OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0);
+  OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_stage2));
+  OP1(SLJIT_MOV_U16, TMP2, 0, SLJIT_MEM2(TMP2, TMP1), 1);
+
+  /* Before anything else, we deal with scripts. */
+  if (needsscript)
     {
-    if (needstype)
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, script));
+    OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3);
+
+    ccbegin = cc;
+
+    while (*cc != XCL_END)
       {
-      OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0);
-      typereg = RETURN_ADDR;
+      if (*cc == XCL_SINGLE)
+        {
+        cc ++;
+        GETCHARINCTEST(c, cc);
+        }
+      else if (*cc == XCL_RANGE)
+        {
+        cc ++;
+        GETCHARINCTEST(c, cc);
+        GETCHARINCTEST(c, cc);
+        }
+      else
+        {
+        SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP);
+        cc++;
+        if (*cc == PT_SC)
+          {
+          compares--;
+          invertcmp = (compares == 0 && list != backtracks);
+          if (cc[-1] == XCL_NOTPROP)
+            invertcmp ^= 0x1;
+          jump = CMP(SLJIT_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, (int)cc[1]);
+          add_jump(compiler, compares > 0 ? list : backtracks, jump);
+          }
+        cc += 2;
+        }
       }
 
-    if (needsscript)
-      scriptreg = TMP3;
-    OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
+    cc = ccbegin;
     }
-  else if (needstype && needsscript)
-    scriptreg = TMP3;
-  /* In all other cases only one of them was specified, and that can goes to TMP1. */
 
-  if (needsscript)
+  if (needschar)
+    {
+    OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0);
+    }
+
+  if (needstype)
     {
-    if (scriptreg == TMP1)
+    if (!needschar)
       {
-      OP1(SLJIT_MOV, scriptreg, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, script));
-      OP1(SLJIT_MOV_UB, scriptreg, 0, SLJIT_MEM2(scriptreg, TMP2), 3);
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype));
+      OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3);
       }
     else
       {
       OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 3);
-      OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, script));
-      OP1(SLJIT_MOV_UB, scriptreg, 0, SLJIT_MEM1(TMP2), 0);
+      OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype));
+      typereg = RETURN_ADDR;
       }
     }
   }
 #endif
 
 /* Generating code. */
-cc = ccbegin;
 charoffset = 0;
 numberofcmps = 0;
 #ifdef SUPPORT_UCP
@@ -4205,148 +5731,123 @@ while (*cc != XCL_END)
   if (*cc == XCL_SINGLE)
     {
     cc ++;
-#ifdef SUPPORT_UTF
-    if (common->utf)
-      {
-      GETCHARINC(c, cc);
-      }
-    else
-#endif
-      c = *cc++;
+    GETCHARINCTEST(c, cc);
 
     if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE))
       {
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c - charoffset);
-      OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, numberofcmps == 0 ? SLJIT_UNUSED : TMP2, 0, SLJIT_C_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
+      OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, numberofcmps == 0 ? SLJIT_UNUSED : TMP2, 0, SLJIT_EQUAL);
       numberofcmps++;
       }
     else if (numberofcmps > 0)
       {
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c - charoffset);
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
-      jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
+      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+      jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
       numberofcmps = 0;
       }
     else
       {
-      jump = CMP(SLJIT_C_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, c - charoffset);
+      jump = CMP(SLJIT_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
       numberofcmps = 0;
       }
     }
   else if (*cc == XCL_RANGE)
     {
     cc ++;
-#ifdef SUPPORT_UTF
-    if (common->utf)
-      {
-      GETCHARINC(c, cc);
-      }
-    else
-#endif
-      c = *cc++;
+    GETCHARINCTEST(c, cc);
     SET_CHAR_OFFSET(c);
-#ifdef SUPPORT_UTF
-    if (common->utf)
-      {
-      GETCHARINC(c, cc);
-      }
-    else
-#endif
-      c = *cc++;
+    GETCHARINCTEST(c, cc);
+
     if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE))
       {
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c - charoffset);
-      OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, numberofcmps == 0 ? SLJIT_UNUSED : TMP2, 0, SLJIT_C_LESS_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
+      OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, numberofcmps == 0 ? SLJIT_UNUSED : TMP2, 0, SLJIT_LESS_EQUAL);
       numberofcmps++;
       }
     else if (numberofcmps > 0)
       {
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c - charoffset);
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_C_LESS_EQUAL);
-      jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
+      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
+      jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
       numberofcmps = 0;
       }
     else
       {
-      jump = CMP(SLJIT_C_LESS_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, c - charoffset);
+      jump = CMP(SLJIT_LESS_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
       numberofcmps = 0;
       }
     }
 #ifdef SUPPORT_UCP
   else
     {
+    SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP);
     if (*cc == XCL_NOTPROP)
       invertcmp ^= 0x1;
     cc++;
     switch(*cc)
       {
       case PT_ANY:
-      if (list != backtracks)
-        {
-        if ((cc[-1] == XCL_NOTPROP && compares > 0) || (cc[-1] == XCL_PROP && compares == 0))
-          continue;
-        }
-      else if (cc[-1] == XCL_NOTPROP)
-        continue;
-      jump = JUMP(SLJIT_JUMP);
+      if (!invertcmp)
+        jump = JUMP(SLJIT_JUMP);
       break;
 
       case PT_LAMP:
       OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - typeoffset);
-      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
       OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ll - typeoffset);
-      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
+      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
       OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lt - typeoffset);
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
-      jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+      jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
       break;
 
       case PT_GC:
       c = PRIV(ucp_typerange)[(int)cc[1] * 2];
       SET_TYPE_OFFSET(c);
-      jump = CMP(SLJIT_C_LESS_EQUAL ^ invertcmp, typereg, 0, SLJIT_IMM, PRIV(ucp_typerange)[(int)cc[1] * 2 + 1] - c);
+      jump = CMP(SLJIT_LESS_EQUAL ^ invertcmp, typereg, 0, SLJIT_IMM, PRIV(ucp_typerange)[(int)cc[1] * 2 + 1] - c);
       break;
 
       case PT_PC:
-      jump = CMP(SLJIT_C_EQUAL ^ invertcmp, typereg, 0, SLJIT_IMM, (int)cc[1] - typeoffset);
+      jump = CMP(SLJIT_EQUAL ^ invertcmp, typereg, 0, SLJIT_IMM, (int)cc[1] - typeoffset);
       break;
 
       case PT_SC:
-      jump = CMP(SLJIT_C_EQUAL ^ invertcmp, scriptreg, 0, SLJIT_IMM, (int)cc[1]);
+      compares++;
+      /* Do nothing. */
       break;
 
       case PT_SPACE:
       case PT_PXSPACE:
-      if (*cc == PT_SPACE)
-        {
-        OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0);
-        jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 11 - charoffset);
-        }
       SET_CHAR_OFFSET(9);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 13 - 9);
-      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_LESS_EQUAL);
-      if (*cc == PT_SPACE)
-        JUMPHERE(jump);
+      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd - 0x9);
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL);
+
+      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x9);
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+
+      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x9);
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
 
       SET_TYPE_OFFSET(ucp_Zl);
       OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Zl);
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_C_LESS_EQUAL);
-      jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
+      jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
       break;
 
       case PT_WORD:
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE - charoffset);
-      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_UNDERSCORE - charoffset));
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
       /* Fall through. */
 
       case PT_ALNUM:
       SET_TYPE_OFFSET(ucp_Ll);
       OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
-      OP_FLAGS((*cc == PT_ALNUM) ? SLJIT_MOV : SLJIT_OR, TMP2, 0, (*cc == PT_ALNUM) ? SLJIT_UNUSED : TMP2, 0, SLJIT_C_LESS_EQUAL);
+      OP_FLAGS((*cc == PT_ALNUM) ? SLJIT_MOV : SLJIT_OR, TMP2, 0, (*cc == PT_ALNUM) ? SLJIT_UNUSED : TMP2, 0, SLJIT_LESS_EQUAL);
       SET_TYPE_OFFSET(ucp_Nd);
       OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_No - ucp_Nd);
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_C_LESS_EQUAL);
-      jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
+      jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
       break;
 
       case PT_CLIST:
@@ -4368,7 +5869,7 @@ while (*cc != XCL_END)
           OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]);
           }
         OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[1]);
-        OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
+        OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
         other_cases += 2;
         }
       else if (is_powerof2(other_cases[2] ^ other_cases[1]))
@@ -4381,145 +5882,421 @@ while (*cc != XCL_END)
           OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]);
           }
         OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[2]);
-        OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
+        OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
 
-        OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, other_cases[0] - charoffset);
-        OP_FLAGS(SLJIT_OR | ((other_cases[3] == NOTACHAR) ? SLJIT_SET_E : 0), TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
+        OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(other_cases[0] - charoffset));
+        OP_FLAGS(SLJIT_OR | ((other_cases[3] == NOTACHAR) ? SLJIT_SET_E : 0), TMP2, 0, TMP2, 0, SLJIT_EQUAL);
 
         other_cases += 3;
         }
       else
         {
-        OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, *other_cases++ - charoffset);
-        OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
+        OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset));
+        OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
         }
 
       while (*other_cases != NOTACHAR)
         {
-        OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, *other_cases++ - charoffset);
-        OP_FLAGS(SLJIT_OR | ((*other_cases == NOTACHAR) ? SLJIT_SET_E : 0), TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
+        OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset));
+        OP_FLAGS(SLJIT_OR | ((*other_cases == NOTACHAR) ? SLJIT_SET_E : 0), TMP2, 0, TMP2, 0, SLJIT_EQUAL);
         }
-      jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
+      jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
       break;
 
       case PT_UCNC:
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_DOLLAR_SIGN - charoffset);
-      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_COMMERCIAL_AT - charoffset);
-      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_GRAVE_ACCENT - charoffset);
-      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_DOLLAR_SIGN - charoffset));
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_COMMERCIAL_AT - charoffset));
+      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_GRAVE_ACCENT - charoffset));
+      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
 
       SET_CHAR_OFFSET(0xa0);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd7ff - charoffset);
-      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_C_LESS_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(0xd7ff - charoffset));
+      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
       SET_CHAR_OFFSET(0);
       OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xe000 - 0);
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_C_GREATER_EQUAL);
-      jump = JUMP(SLJIT_C_NOT_ZERO ^ invertcmp);
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_GREATER_EQUAL);
+      jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
+      break;
+
+      case PT_PXGRAPH:
+      /* C and Z groups are the farthest two groups. */
+      SET_TYPE_OFFSET(ucp_Ll);
+      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll);
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_GREATER);
+
+      jump = CMP(SLJIT_NOT_EQUAL, typereg, 0, SLJIT_IMM, ucp_Cf - ucp_Ll);
+
+      /* In case of ucp_Cf, we overwrite the result. */
+      SET_CHAR_OFFSET(0x2066);
+      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066);
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL);
+
+      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066);
+      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+
+      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x2066);
+      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+
+      JUMPHERE(jump);
+      jump = CMP(SLJIT_ZERO ^ invertcmp, TMP2, 0, SLJIT_IMM, 0);
+      break;
+
+      case PT_PXPRINT:
+      /* C and Z groups are the farthest two groups. */
+      SET_TYPE_OFFSET(ucp_Ll);
+      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll);
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_GREATER);
+
+      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Ll);
+      OP_FLAGS(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_NOT_EQUAL);
+
+      jump = CMP(SLJIT_NOT_EQUAL, typereg, 0, SLJIT_IMM, ucp_Cf - ucp_Ll);
+
+      /* In case of ucp_Cf, we overwrite the result. */
+      SET_CHAR_OFFSET(0x2066);
+      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066);
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL);
+
+      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066);
+      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+
+      JUMPHERE(jump);
+      jump = CMP(SLJIT_ZERO ^ invertcmp, TMP2, 0, SLJIT_IMM, 0);
+      break;
+
+      case PT_PXPUNCT:
+      SET_TYPE_OFFSET(ucp_Sc);
+      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Sc);
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL);
+
+      SET_CHAR_OFFSET(0);
+      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x7f);
+      OP_FLAGS(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
+
+      SET_TYPE_OFFSET(ucp_Pc);
+      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ps - ucp_Pc);
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
+      jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
+      break;
+
+      default:
+      SLJIT_ASSERT_STOP();
       break;
       }
     cc += 2;
     }
 #endif
-
-  if (jump != NULL)
-    add_jump(compiler, compares > 0 ? list : backtracks, jump);
+
+  if (jump != NULL)
+    add_jump(compiler, compares > 0 ? list : backtracks, jump);
+  }
+
+if (found != NULL)
+  set_jumps(found, LABEL());
+}
+
+#undef SET_TYPE_OFFSET
+#undef SET_CHAR_OFFSET
+
+#endif
+
+static pcre_uchar *compile_simple_assertion_matchingpath(compiler_common *common, pcre_uchar type, pcre_uchar *cc, jump_list **backtracks)
+{
+DEFINE_COMPILER;
+int length;
+struct sljit_jump *jump[4];
+#ifdef SUPPORT_UTF
+struct sljit_label *label;
+#endif /* SUPPORT_UTF */
+
+switch(type)
+  {
+  case OP_SOD:
+  OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
+  add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, TMP1, 0));
+  return cc;
+
+  case OP_SOM:
+  OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
+  add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, TMP1, 0));
+  return cc;
+
+  case OP_NOT_WORD_BOUNDARY:
+  case OP_WORD_BOUNDARY:
+  add_jump(compiler, &common->wordboundary, JUMP(SLJIT_FAST_CALL));
+  add_jump(compiler, backtracks, JUMP(type == OP_NOT_WORD_BOUNDARY ? SLJIT_NOT_ZERO : SLJIT_ZERO));
+  return cc;
+
+  case OP_EODN:
+  /* Requires rather complex checks. */
+  jump[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
+  if (common->nltype == NLTYPE_FIXED && common->newline > 255)
+    {
+    OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));
+    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
+    if (common->mode == JIT_COMPILE)
+      add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, STR_END, 0));
+    else
+      {
+      jump[1] = CMP(SLJIT_EQUAL, TMP2, 0, STR_END, 0);
+      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0);
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS);
+      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff);
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_NOT_EQUAL);
+      add_jump(compiler, backtracks, JUMP(SLJIT_NOT_EQUAL));
+      check_partial(common, TRUE);
+      add_jump(compiler, backtracks, JUMP(SLJIT_JUMP));
+      JUMPHERE(jump[1]);
+      }
+    OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
+    add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
+    add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff));
+    }
+  else if (common->nltype == NLTYPE_FIXED)
+    {
+    OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
+    add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, STR_END, 0));
+    add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline));
+    }
+  else
+    {
+    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
+    jump[1] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
+    OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));
+    OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0);
+    jump[2] = JUMP(SLJIT_GREATER);
+    add_jump(compiler, backtracks, JUMP(SLJIT_LESS));
+    /* Equal. */
+    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
+    jump[3] = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL);
+    add_jump(compiler, backtracks, JUMP(SLJIT_JUMP));
+
+    JUMPHERE(jump[1]);
+    if (common->nltype == NLTYPE_ANYCRLF)
+      {
+      OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+      add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP2, 0, STR_END, 0));
+      add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL));
+      }
+    else
+      {
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, STR_PTR, 0);
+      read_char_range(common, common->nlmin, common->nlmax, TRUE);
+      add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, STR_END, 0));
+      add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL));
+      add_jump(compiler, backtracks, JUMP(SLJIT_ZERO));
+      OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1);
+      }
+    JUMPHERE(jump[2]);
+    JUMPHERE(jump[3]);
+    }
+  JUMPHERE(jump[0]);
+  check_partial(common, FALSE);
+  return cc;
+
+  case OP_EOD:
+  add_jump(compiler, backtracks, CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0));
+  check_partial(common, FALSE);
+  return cc;
+
+  case OP_DOLL:
+  OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
+  OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, noteol));
+  add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
+
+  if (!common->endonly)
+    compile_simple_assertion_matchingpath(common, OP_EODN, cc, backtracks);
+  else
+    {
+    add_jump(compiler, backtracks, CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0));
+    check_partial(common, FALSE);
+    }
+  return cc;
+
+  case OP_DOLLM:
+  jump[1] = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0);
+  OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
+  OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, noteol));
+  add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
+  check_partial(common, FALSE);
+  jump[0] = JUMP(SLJIT_JUMP);
+  JUMPHERE(jump[1]);
+
+  if (common->nltype == NLTYPE_FIXED && common->newline > 255)
+    {
+    OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));
+    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
+    if (common->mode == JIT_COMPILE)
+      add_jump(compiler, backtracks, CMP(SLJIT_GREATER, TMP2, 0, STR_END, 0));
+    else
+      {
+      jump[1] = CMP(SLJIT_LESS_EQUAL, TMP2, 0, STR_END, 0);
+      /* STR_PTR = STR_END - IN_UCHARS(1) */
+      add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
+      check_partial(common, TRUE);
+      add_jump(compiler, backtracks, JUMP(SLJIT_JUMP));
+      JUMPHERE(jump[1]);
+      }
+
+    OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
+    add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
+    add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff));
+    }
+  else
+    {
+    peek_char(common, common->nlmax);
+    check_newlinechar(common, common->nltype, backtracks, FALSE);
+    }
+  JUMPHERE(jump[0]);
+  return cc;
+
+  case OP_CIRC:
+  OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin));
+  add_jump(compiler, backtracks, CMP(SLJIT_GREATER, STR_PTR, 0, TMP1, 0));
+  OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, notbol));
+  add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
+  return cc;
+
+  case OP_CIRCM:
+  OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin));
+  jump[1] = CMP(SLJIT_GREATER, STR_PTR, 0, TMP1, 0);
+  OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, notbol));
+  add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
+  jump[0] = JUMP(SLJIT_JUMP);
+  JUMPHERE(jump[1]);
+
+  add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
+  if (common->nltype == NLTYPE_FIXED && common->newline > 255)
+    {
+    OP2(SLJIT_SUB, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));
+    add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP2, 0, TMP1, 0));
+    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2));
+    OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1));
+    add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
+    add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff));
+    }
+  else
+    {
+    skip_char_back(common);
+    read_char_range(common, common->nlmin, common->nlmax, TRUE);
+    check_newlinechar(common, common->nltype, backtracks, FALSE);
+    }
+  JUMPHERE(jump[0]);
+  return cc;
+
+  case OP_REVERSE:
+  length = GET(cc, 0);
+  if (length == 0)
+    return cc + LINK_SIZE;
+  OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
+#ifdef SUPPORT_UTF
+  if (common->utf)
+    {
+    OP1(SLJIT_MOV, TMP3, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
+    OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, length);
+    label = LABEL();
+    add_jump(compiler, backtracks, CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP3, 0));
+    skip_char_back(common);
+    OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
+    JUMPTO(SLJIT_NOT_ZERO, label);
+    }
+  else
+#endif
+    {
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
+    OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(length));
+    add_jump(compiler, backtracks, CMP(SLJIT_LESS, STR_PTR, 0, TMP1, 0));
+    }
+  check_start_used_ptr(common);
+  return cc + LINK_SIZE;
   }
-
-if (found != NULL)
-  set_jumps(found, LABEL());
+SLJIT_ASSERT_STOP();
+return cc;
 }
 
-#undef SET_TYPE_OFFSET
-#undef SET_CHAR_OFFSET
-
-#endif
-
-static pcre_uchar *compile_char1_matchingpath(compiler_common *common, pcre_uchar type, pcre_uchar *cc, jump_list **backtracks)
+static pcre_uchar *compile_char1_matchingpath(compiler_common *common, pcre_uchar type, pcre_uchar *cc, jump_list **backtracks, BOOL check_str_ptr)
 {
 DEFINE_COMPILER;
 int length;
 unsigned int c, oc, bit;
 compare_context context;
-struct sljit_jump *jump[4];
+struct sljit_jump *jump[3];
 jump_list *end_list;
 #ifdef SUPPORT_UTF
 struct sljit_label *label;
 #ifdef SUPPORT_UCP
 pcre_uchar propdata[5];
 #endif
-#endif
+#endif /* SUPPORT_UTF */
 
 switch(type)
   {
-  case OP_SOD:
-  OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
-  add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, TMP1, 0));
-  return cc;
-
-  case OP_SOM:
-  OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
-  add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, TMP1, 0));
-  return cc;
-
-  case OP_NOT_WORD_BOUNDARY:
-  case OP_WORD_BOUNDARY:
-  add_jump(compiler, &common->wordboundary, JUMP(SLJIT_FAST_CALL));
-  add_jump(compiler, backtracks, JUMP(type == OP_NOT_WORD_BOUNDARY ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
-  return cc;
-
   case OP_NOT_DIGIT:
   case OP_DIGIT:
   /* Digits are usually 0-9, so it is worth to optimize them. */
-  if (common->digits[0] == -2)
-    get_ctype_ranges(common, ctype_digit, common->digits);
-  detect_partial_match(common, backtracks);
-  /* Flip the starting bit in the negative case. */
-  if (type == OP_NOT_DIGIT)
-    common->digits[1] ^= 1;
-  if (!check_ranges(common, common->digits, backtracks, TRUE))
-    {
-    read_char8_type(common);
-    OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_digit);
-    add_jump(compiler, backtracks, JUMP(type == OP_DIGIT ? SLJIT_C_ZERO : SLJIT_C_NOT_ZERO));
-    }
-  if (type == OP_NOT_DIGIT)
-    common->digits[1] ^= 1;
+  if (check_str_ptr)
+    detect_partial_match(common, backtracks);
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
+  if (common->utf && is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_digit, FALSE))
+    read_char7_type(common, type == OP_NOT_DIGIT);
+  else
+#endif
+    read_char8_type(common, type == OP_NOT_DIGIT);
+    /* Flip the starting bit in the negative case. */
+  OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_digit);
+  add_jump(compiler, backtracks, JUMP(type == OP_DIGIT ? SLJIT_ZERO : SLJIT_NOT_ZERO));
   return cc;
 
   case OP_NOT_WHITESPACE:
   case OP_WHITESPACE:
-  detect_partial_match(common, backtracks);
-  read_char8_type(common);
+  if (check_str_ptr)
+    detect_partial_match(common, backtracks);
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
+  if (common->utf && is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_space, FALSE))
+    read_char7_type(common, type == OP_NOT_WHITESPACE);
+  else
+#endif
+    read_char8_type(common, type == OP_NOT_WHITESPACE);
   OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_space);
-  add_jump(compiler, backtracks, JUMP(type == OP_WHITESPACE ? SLJIT_C_ZERO : SLJIT_C_NOT_ZERO));
+  add_jump(compiler, backtracks, JUMP(type == OP_WHITESPACE ? SLJIT_ZERO : SLJIT_NOT_ZERO));
   return cc;
 
   case OP_NOT_WORDCHAR:
   case OP_WORDCHAR:
-  detect_partial_match(common, backtracks);
-  read_char8_type(common);
+  if (check_str_ptr)
+    detect_partial_match(common, backtracks);
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
+  if (common->utf && is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_word, FALSE))
+    read_char7_type(common, type == OP_NOT_WORDCHAR);
+  else
+#endif
+    read_char8_type(common, type == OP_NOT_WORDCHAR);
   OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_word);
-  add_jump(compiler, backtracks, JUMP(type == OP_WORDCHAR ? SLJIT_C_ZERO : SLJIT_C_NOT_ZERO));
+  add_jump(compiler, backtracks, JUMP(type == OP_WORDCHAR ? SLJIT_ZERO : SLJIT_NOT_ZERO));
   return cc;
 
   case OP_ANY:
-  detect_partial_match(common, backtracks);
-  read_char(common);
+  if (check_str_ptr)
+    detect_partial_match(common, backtracks);
+  read_char_range(common, common->nlmin, common->nlmax, TRUE);
   if (common->nltype == NLTYPE_FIXED && common->newline > 255)
     {
-    jump[0] = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff);
+    jump[0] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff);
     end_list = NULL;
     if (common->mode != JIT_PARTIAL_HARD_COMPILE)
-      add_jump(compiler, &end_list, CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
+      add_jump(compiler, &end_list, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
     else
       check_str_end(common, &end_list);
 
     OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
-    add_jump(compiler, backtracks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, common->newline & 0xff));
+    add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline & 0xff));
     set_jumps(end_list, LABEL());
     JUMPHERE(jump[0]);
     }
@@ -4528,7 +6305,8 @@ switch(type)
   return cc;
 
   case OP_ALLANY:
-  detect_partial_match(common, backtracks);
+  if (check_str_ptr)
+    detect_partial_match(common, backtracks);
 #ifdef SUPPORT_UTF
   if (common->utf)
     {
@@ -4536,14 +6314,14 @@ switch(type)
     OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
 #if defined COMPILE_PCRE8 || defined COMPILE_PCRE16
 #if defined COMPILE_PCRE8
-    jump[0] = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
-    OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0);
+    jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
+    OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0);
     OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
 #elif defined COMPILE_PCRE16
-    jump[0] = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xd800);
+    jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800);
     OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00);
     OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800);
-    OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
+    OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
     OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
     OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
 #endif
@@ -4556,7 +6334,8 @@ switch(type)
   return cc;
 
   case OP_ANYBYTE:
-  detect_partial_match(common, backtracks);
+  if (check_str_ptr)
+    detect_partial_match(common, backtracks);
   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
   return cc;
 
@@ -4564,28 +6343,31 @@ switch(type)
 #ifdef SUPPORT_UCP
   case OP_NOTPROP:
   case OP_PROP:
-  propdata[0] = 0;
+  propdata[0] = XCL_HASPROP;
   propdata[1] = type == OP_NOTPROP ? XCL_NOTPROP : XCL_PROP;
   propdata[2] = cc[0];
   propdata[3] = cc[1];
   propdata[4] = XCL_END;
+  if (check_str_ptr)
+    detect_partial_match(common, backtracks);
   compile_xclass_matchingpath(common, propdata, backtracks);
   return cc + 2;
 #endif
 #endif
 
   case OP_ANYNL:
-  detect_partial_match(common, backtracks);
-  read_char(common);
-  jump[0] = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
+  if (check_str_ptr)
+    detect_partial_match(common, backtracks);
+  read_char_range(common, common->bsr_nlmin, common->bsr_nlmax, FALSE);
+  jump[0] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
   /* We don't need to handle soft partial matching case. */
   end_list = NULL;
   if (common->mode != JIT_PARTIAL_HARD_COMPILE)
-    add_jump(compiler, &end_list, CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
+    add_jump(compiler, &end_list, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
   else
     check_str_end(common, &end_list);
   OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
-  jump[1] = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL);
+  jump[1] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL);
   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
   jump[2] = JUMP(SLJIT_JUMP);
   JUMPHERE(jump[0]);
@@ -4597,52 +6379,55 @@ switch(type)
 
   case OP_NOT_HSPACE:
   case OP_HSPACE:
-  detect_partial_match(common, backtracks);
-  read_char(common);
+  if (check_str_ptr)
+    detect_partial_match(common, backtracks);
+  read_char_range(common, 0x9, 0x3000, type == OP_NOT_HSPACE);
   add_jump(compiler, &common->hspace, JUMP(SLJIT_FAST_CALL));
-  add_jump(compiler, backtracks, JUMP(type == OP_NOT_HSPACE ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
+  add_jump(compiler, backtracks, JUMP(type == OP_NOT_HSPACE ? SLJIT_NOT_ZERO : SLJIT_ZERO));
   return cc;
 
   case OP_NOT_VSPACE:
   case OP_VSPACE:
-  detect_partial_match(common, backtracks);
-  read_char(common);
+  if (check_str_ptr)
+    detect_partial_match(common, backtracks);
+  read_char_range(common, 0xa, 0x2029, type == OP_NOT_VSPACE);
   add_jump(compiler, &common->vspace, JUMP(SLJIT_FAST_CALL));
-  add_jump(compiler, backtracks, JUMP(type == OP_NOT_VSPACE ? SLJIT_C_NOT_ZERO : SLJIT_C_ZERO));
+  add_jump(compiler, backtracks, JUMP(type == OP_NOT_VSPACE ? SLJIT_NOT_ZERO : SLJIT_ZERO));
   return cc;
 
 #ifdef SUPPORT_UCP
   case OP_EXTUNI:
-  detect_partial_match(common, backtracks);
+  if (check_str_ptr)
+    detect_partial_match(common, backtracks);
   read_char(common);
   add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
   OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, gbprop));
   /* Optimize register allocation: use a real register. */
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, STACK_TOP, 0);
-  OP1(SLJIT_MOV_UB, STACK_TOP, 0, SLJIT_MEM2(TMP1, TMP2), 3);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0);
+  OP1(SLJIT_MOV_U8, STACK_TOP, 0, SLJIT_MEM2(TMP1, TMP2), 3);
 
   label = LABEL();
-  jump[0] = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
+  jump[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
   OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0);
   read_char(common);
   add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
   OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, gbprop));
-  OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM2(TMP1, TMP2), 3);
+  OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM2(TMP1, TMP2), 3);
 
   OP2(SLJIT_SHL, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 2);
-  OP1(SLJIT_MOV_UI, TMP1, 0, SLJIT_MEM1(STACK_TOP), (sljit_sw)PRIV(ucp_gbtable));
+  OP1(SLJIT_MOV_U32, TMP1, 0, SLJIT_MEM1(STACK_TOP), (sljit_sw)PRIV(ucp_gbtable));
   OP1(SLJIT_MOV, STACK_TOP, 0, TMP2, 0);
   OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
   OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
-  JUMPTO(SLJIT_C_NOT_ZERO, label);
+  JUMPTO(SLJIT_NOT_ZERO, label);
 
   OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0);
   JUMPHERE(jump[0]);
-  OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
+  OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
 
   if (common->mode == JIT_PARTIAL_HARD_COMPILE)
     {
-    jump[0] = CMP(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0);
+    jump[0] = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0);
     /* Since we successfully read a char above, partial matching must occure. */
     check_partial(common, TRUE);
     JUMPHERE(jump[0]);
@@ -4650,176 +6435,17 @@ switch(type)
   return cc;
 #endif
 
-  case OP_EODN:
-  /* Requires rather complex checks. */
-  jump[0] = CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
-  if (common->nltype == NLTYPE_FIXED && common->newline > 255)
-    {
-    OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));
-    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
-    if (common->mode == JIT_COMPILE)
-      add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_END, 0));
-    else
-      {
-      jump[1] = CMP(SLJIT_C_EQUAL, TMP2, 0, STR_END, 0);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0);
-      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_LESS);
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff);
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_C_NOT_EQUAL);
-      add_jump(compiler, backtracks, JUMP(SLJIT_C_NOT_EQUAL));
-      check_partial(common, TRUE);
-      add_jump(compiler, backtracks, JUMP(SLJIT_JUMP));
-      JUMPHERE(jump[1]);
-      }
-    OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
-    add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
-    add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff));
-    }
-  else if (common->nltype == NLTYPE_FIXED)
-    {
-    OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
-    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
-    add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_END, 0));
-    add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline));
-    }
-  else
-    {
-    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
-    jump[1] = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
-    OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));
-    OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0);
-    jump[2] = JUMP(SLJIT_C_GREATER);
-    add_jump(compiler, backtracks, JUMP(SLJIT_C_LESS));
-    /* Equal. */
-    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
-    jump[3] = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL);
-    add_jump(compiler, backtracks, JUMP(SLJIT_JUMP));
-
-    JUMPHERE(jump[1]);
-    if (common->nltype == NLTYPE_ANYCRLF)
-      {
-      OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
-      add_jump(compiler, backtracks, CMP(SLJIT_C_LESS, TMP2, 0, STR_END, 0));
-      add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL));
-      }
-    else
-      {
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, STR_PTR, 0);
-      read_char(common);
-      add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, STR_END, 0));
-      add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL));
-      add_jump(compiler, backtracks, JUMP(SLJIT_C_ZERO));
-      OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1);
-      }
-    JUMPHERE(jump[2]);
-    JUMPHERE(jump[3]);
-    }
-  JUMPHERE(jump[0]);
-  check_partial(common, FALSE);
-  return cc;
-
-  case OP_EOD:
-  add_jump(compiler, backtracks, CMP(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0));
-  check_partial(common, FALSE);
-  return cc;
-
-  case OP_CIRC:
-  OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin));
-  add_jump(compiler, backtracks, CMP(SLJIT_C_GREATER, STR_PTR, 0, TMP1, 0));
-  OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, notbol));
-  add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
-  return cc;
-
-  case OP_CIRCM:
-  OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin));
-  jump[1] = CMP(SLJIT_C_GREATER, STR_PTR, 0, TMP1, 0);
-  OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, notbol));
-  add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
-  jump[0] = JUMP(SLJIT_JUMP);
-  JUMPHERE(jump[1]);
-
-  add_jump(compiler, backtracks, CMP(SLJIT_C_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
-  if (common->nltype == NLTYPE_FIXED && common->newline > 255)
-    {
-    OP2(SLJIT_SUB, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));
-    add_jump(compiler, backtracks, CMP(SLJIT_C_LESS, TMP2, 0, TMP1, 0));
-    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2));
-    OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1));
-    add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
-    add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff));
-    }
-  else
-    {
-    skip_char_back(common);
-    read_char(common);
-    check_newlinechar(common, common->nltype, backtracks, FALSE);
-    }
-  JUMPHERE(jump[0]);
-  return cc;
-
-  case OP_DOLL:
-  OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
-  OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, noteol));
-  add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
-
-  if (!common->endonly)
-    compile_char1_matchingpath(common, OP_EODN, cc, backtracks);
-  else
-    {
-    add_jump(compiler, backtracks, CMP(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0));
-    check_partial(common, FALSE);
-    }
-  return cc;
-
-  case OP_DOLLM:
-  jump[1] = CMP(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0);
-  OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
-  OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, noteol));
-  add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
-  check_partial(common, FALSE);
-  jump[0] = JUMP(SLJIT_JUMP);
-  JUMPHERE(jump[1]);
-
-  if (common->nltype == NLTYPE_FIXED && common->newline > 255)
-    {
-    OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));
-    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
-    if (common->mode == JIT_COMPILE)
-      add_jump(compiler, backtracks, CMP(SLJIT_C_GREATER, TMP2, 0, STR_END, 0));
-    else
-      {
-      jump[1] = CMP(SLJIT_C_LESS_EQUAL, TMP2, 0, STR_END, 0);
-      /* STR_PTR = STR_END - IN_UCHARS(1) */
-      add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
-      check_partial(common, TRUE);
-      add_jump(compiler, backtracks, JUMP(SLJIT_JUMP));
-      JUMPHERE(jump[1]);
-      }
-
-    OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
-    add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff));
-    add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff));
-    }
-  else
-    {
-    peek_char(common);
-    check_newlinechar(common, common->nltype, backtracks, FALSE);
-    }
-  JUMPHERE(jump[0]);
-  return cc;
-
   case OP_CHAR:
   case OP_CHARI:
   length = 1;
 #ifdef SUPPORT_UTF
   if (common->utf && HAS_EXTRALEN(*cc)) length += GET_EXTRALEN(*cc);
 #endif
-  if (common->mode == JIT_COMPILE && (type == OP_CHAR || !char_has_othercase(common, cc) || char_get_othercase_bit(common, cc) != 0))
+  if (common->mode == JIT_COMPILE && check_str_ptr
+      && (type == OP_CHAR || !char_has_othercase(common, cc) || char_get_othercase_bit(common, cc) != 0))
     {
     OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(length));
-    add_jump(compiler, backtracks, CMP(SLJIT_C_GREATER, STR_PTR, 0, STR_END, 0));
+    add_jump(compiler, backtracks, CMP(SLJIT_GREATER, STR_PTR, 0, STR_END, 0));
 
     context.length = IN_UCHARS(length);
     context.sourcereg = -1;
@@ -4828,8 +6454,9 @@ switch(type)
 #endif
     return byte_sequence_compare(common, type == OP_CHARI, cc, &context, backtracks);
     }
-  detect_partial_match(common, backtracks);
-  read_char(common);
+
+  if (check_str_ptr)
+    detect_partial_match(common, backtracks);
 #ifdef SUPPORT_UTF
   if (common->utf)
     {
@@ -4838,29 +6465,31 @@ switch(type)
   else
 #endif
     c = *cc;
+
   if (type == OP_CHAR || !char_has_othercase(common, cc))
     {
-    add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, c));
+    read_char_range(common, c, c, FALSE);
+    add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, c));
     return cc + length;
     }
   oc = char_othercase(common, c);
+  read_char_range(common, c < oc ? c : oc, c > oc ? c : oc, FALSE);
   bit = c ^ oc;
   if (is_powerof2(bit))
     {
     OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, bit);
-    add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, c | bit));
+    add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, c | bit));
     return cc + length;
     }
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, c);
-  OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, oc);
-  OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
-  add_jump(compiler, backtracks, JUMP(SLJIT_C_ZERO));
+  jump[0] = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c);
+  add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, oc));
+  JUMPHERE(jump[0]);
   return cc + length;
 
   case OP_NOT:
   case OP_NOTI:
-  detect_partial_match(common, backtracks);
+  if (check_str_ptr)
+    detect_partial_match(common, backtracks);
   length = 1;
 #ifdef SUPPORT_UTF
   if (common->utf)
@@ -4869,18 +6498,18 @@ switch(type)
     c = *cc;
     if (c < 128)
       {
-      OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
+      OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
       if (type == OP_NOT || !char_has_othercase(common, cc))
-        add_jump(compiler, backtracks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c));
+        add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c));
       else
         {
         /* Since UTF8 code page is fixed, we know that c is in [a-z] or [A-Z] range. */
         OP2(SLJIT_OR, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x20);
-        add_jump(compiler, backtracks, CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, c | 0x20));
+        add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, c | 0x20));
         }
       /* Skip the variable-length character. */
       OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
-      jump[0] = CMP(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
+      jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
       OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0);
       OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
       JUMPHERE(jump[0]);
@@ -4890,101 +6519,90 @@ switch(type)
 #endif /* COMPILE_PCRE8 */
       {
       GETCHARLEN(c, cc, length);
-      read_char(common);
       }
     }
   else
 #endif /* SUPPORT_UTF */
-    {
-    read_char(common);
     c = *cc;
-    }
 
   if (type == OP_NOT || !char_has_othercase(common, cc))
-    add_jump(compiler, backtracks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c));
+    {
+    read_char_range(common, c, c, TRUE);
+    add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c));
+    }
   else
     {
     oc = char_othercase(common, c);
+    read_char_range(common, c < oc ? c : oc, c > oc ? c : oc, TRUE);
     bit = c ^ oc;
     if (is_powerof2(bit))
       {
       OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, bit);
-      add_jump(compiler, backtracks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c | bit));
+      add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c | bit));
       }
     else
       {
-      add_jump(compiler, backtracks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, c));
-      add_jump(compiler, backtracks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, oc));
+      add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c));
+      add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, oc));
       }
     }
   return cc + length;
 
   case OP_CLASS:
   case OP_NCLASS:
-  detect_partial_match(common, backtracks);
-  read_char(common);
-  if (check_class_ranges(common, (const pcre_uint8 *)cc, type == OP_NCLASS, backtracks))
+  if (check_str_ptr)
+    detect_partial_match(common, backtracks);
+
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
+  bit = (common->utf && is_char7_bitset((const sljit_u8 *)cc, type == OP_NCLASS)) ? 127 : 255;
+  read_char_range(common, 0, bit, type == OP_NCLASS);
+#else
+  read_char_range(common, 0, 255, type == OP_NCLASS);
+#endif
+
+  if (check_class_ranges(common, (const sljit_u8 *)cc, type == OP_NCLASS, FALSE, backtracks))
     return cc + 32 / sizeof(pcre_uchar);
 
-#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
   jump[0] = NULL;
-#ifdef COMPILE_PCRE8
-  /* This check only affects 8 bit mode. In other modes, we
-  always need to compare the value with 255. */
   if (common->utf)
-#endif /* COMPILE_PCRE8 */
     {
-    jump[0] = CMP(SLJIT_C_GREATER, TMP1, 0, SLJIT_IMM, 255);
+    jump[0] = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, bit);
     if (type == OP_CLASS)
       {
       add_jump(compiler, backtracks, jump[0]);
       jump[0] = NULL;
       }
     }
-#endif /* SUPPORT_UTF || !COMPILE_PCRE8 */
+#elif !defined COMPILE_PCRE8
+  jump[0] = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255);
+  if (type == OP_CLASS)
+    {
+    add_jump(compiler, backtracks, jump[0]);
+    jump[0] = NULL;
+    }
+#endif /* SUPPORT_UTF && COMPILE_PCRE8 */
+
   OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
   OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
-  OP1(SLJIT_MOV_UB, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc);
+  OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc);
   OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
   OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
-  add_jump(compiler, backtracks, JUMP(SLJIT_C_ZERO));
+  add_jump(compiler, backtracks, JUMP(SLJIT_ZERO));
+
 #if defined SUPPORT_UTF || !defined COMPILE_PCRE8
   if (jump[0] != NULL)
     JUMPHERE(jump[0]);
-#endif /* SUPPORT_UTF || !COMPILE_PCRE8 */
+#endif
   return cc + 32 / sizeof(pcre_uchar);
 
 #if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32
   case OP_XCLASS:
+  if (check_str_ptr)
+    detect_partial_match(common, backtracks);
   compile_xclass_matchingpath(common, cc + LINK_SIZE, backtracks);
   return cc + GET(cc, 0) - 1;
 #endif
-
-  case OP_REVERSE:
-  length = GET(cc, 0);
-  if (length == 0)
-    return cc + LINK_SIZE;
-  OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
-#ifdef SUPPORT_UTF
-  if (common->utf)
-    {
-    OP1(SLJIT_MOV, TMP3, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
-    OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, length);
-    label = LABEL();
-    add_jump(compiler, backtracks, CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, TMP3, 0));
-    skip_char_back(common);
-    OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
-    JUMPTO(SLJIT_C_NOT_ZERO, label);
-    }
-  else
-#endif
-    {
-    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
-    OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(length));
-    add_jump(compiler, backtracks, CMP(SLJIT_C_LESS, STR_PTR, 0, TMP1, 0));
-    }
-  check_start_used_ptr(common);
-  return cc + LINK_SIZE;
   }
 SLJIT_ASSERT_STOP();
 return cc;
@@ -5042,7 +6660,7 @@ if (context.length > 0)
   {
   /* We have a fixed-length byte sequence. */
   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, context.length);
-  add_jump(compiler, backtracks, CMP(SLJIT_C_GREATER, STR_PTR, 0, STR_END, 0));
+  add_jump(compiler, backtracks, CMP(SLJIT_GREATER, STR_PTR, 0, STR_END, 0));
 
   context.sourcereg = -1;
 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
@@ -5053,29 +6671,7 @@ if (context.length > 0)
   }
 
 /* A non-fixed length character will be checked if length == 0. */
-return compile_char1_matchingpath(common, *cc, cc + 1, backtracks);
-}
-
-static struct sljit_jump *compile_ref_checks(compiler_common *common, pcre_uchar *cc, jump_list **backtracks)
-{
-DEFINE_COMPILER;
-int offset = GET2(cc, 1) << 1;
-
-OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
-if (!common->jscript_compat)
-  {
-  if (backtracks == NULL)
-    {
-    /* OVECTOR(1) contains the "string begin - 1" constant. */
-    OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1));
-    OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_C_EQUAL);
-    OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
-    OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_C_EQUAL);
-    return JUMP(SLJIT_C_NOT_ZERO);
-    }
-  add_jump(compiler, backtracks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1)));
-  }
-return CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
+return compile_char1_matchingpath(common, *cc, cc + 1, backtracks, TRUE);
 }
 
 /* Forward definitions. */
@@ -5110,39 +6706,80 @@ static void compile_backtrackingpath(compiler_common *, struct backtrack_common
 
 #define BACKTRACK_AS(type) ((type *)backtrack)
 
-static pcre_uchar *compile_ref_matchingpath(compiler_common *common, pcre_uchar *cc, jump_list **backtracks, BOOL withchecks, BOOL emptyfail)
+static void compile_dnref_search(compiler_common *common, pcre_uchar *cc, jump_list **backtracks)
+{
+/* The OVECTOR offset goes to TMP2. */
+DEFINE_COMPILER;
+int count = GET2(cc, 1 + IMM2_SIZE);
+pcre_uchar *slot = common->name_table + GET2(cc, 1) * common->name_entry_size;
+unsigned int offset;
+jump_list *found = NULL;
+
+SLJIT_ASSERT(*cc == OP_DNREF || *cc == OP_DNREFI);
+
+OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1));
+
+count--;
+while (count-- > 0)
+  {
+  offset = GET2(slot, 0) << 1;
+  GET_LOCAL_BASE(TMP2, 0, OVECTOR(offset));
+  add_jump(compiler, &found, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0));
+  slot += common->name_entry_size;
+  }
+
+offset = GET2(slot, 0) << 1;
+GET_LOCAL_BASE(TMP2, 0, OVECTOR(offset));
+if (backtracks != NULL && !common->jscript_compat)
+  add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0));
+
+set_jumps(found, LABEL());
+}
+
+static void compile_ref_matchingpath(compiler_common *common, pcre_uchar *cc, jump_list **backtracks, BOOL withchecks, BOOL emptyfail)
 {
 DEFINE_COMPILER;
-int offset = GET2(cc, 1) << 1;
+BOOL ref = (*cc == OP_REF || *cc == OP_REFI);
+int offset = 0;
 struct sljit_jump *jump = NULL;
 struct sljit_jump *partial;
 struct sljit_jump *nopartial;
 
-OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
-/* OVECTOR(1) contains the "string begin - 1" constant. */
-if (withchecks && !common->jscript_compat)
-  add_jump(compiler, backtracks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1)));
+if (ref)
+  {
+  offset = GET2(cc, 1) << 1;
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset));
+  /* OVECTOR(1) contains the "string begin - 1" constant. */
+  if (withchecks && !common->jscript_compat)
+    add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1)));
+  }
+else
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), 0);
 
 #if defined SUPPORT_UTF && defined SUPPORT_UCP
 if (common->utf && *cc == OP_REFI)
   {
-  SLJIT_ASSERT(TMP1 == SLJIT_SCRATCH_REG1 && STACK_TOP == SLJIT_SCRATCH_REG2 && TMP2 == SLJIT_SCRATCH_REG3);
-  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
+  SLJIT_ASSERT(TMP1 == SLJIT_R0 && STACK_TOP == SLJIT_R1 && TMP2 == SLJIT_R2);
+  if (ref)
+    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1));
+  else
+    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw));
+
   if (withchecks)
-    jump = CMP(SLJIT_C_EQUAL, TMP1, 0, TMP2, 0);
+    jump = CMP(SLJIT_EQUAL, TMP1, 0, TMP2, 0);
 
   /* Needed to save important temporary registers. */
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, STACK_TOP, 0);
-  OP1(SLJIT_MOV, SLJIT_SCRATCH_REG2, 0, ARGUMENTS, 0);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SCRATCH_REG2), SLJIT_OFFSETOF(jit_arguments, uchar_ptr), STR_PTR, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0);
+  OP1(SLJIT_MOV, SLJIT_R1, 0, ARGUMENTS, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_R1), SLJIT_OFFSETOF(jit_arguments, uchar_ptr), STR_PTR, 0);
   sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_utf_caselesscmp));
-  OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
+  OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
   if (common->mode == JIT_COMPILE)
-    add_jump(compiler, backtracks, CMP(SLJIT_C_LESS_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1));
+    add_jump(compiler, backtracks, CMP(SLJIT_LESS_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1));
   else
     {
-    add_jump(compiler, backtracks, CMP(SLJIT_C_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0));
-    nopartial = CMP(SLJIT_C_NOT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1);
+    add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0));
+    nopartial = CMP(SLJIT_NOT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1);
     check_partial(common, FALSE);
     add_jump(compiler, backtracks, JUMP(SLJIT_JUMP));
     JUMPHERE(nopartial);
@@ -5152,17 +6789,21 @@ if (common->utf && *cc == OP_REFI)
 else
 #endif /* SUPPORT_UTF && SUPPORT_UCP */
   {
-  OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), TMP1, 0);
+  if (ref)
+    OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP1, 0);
+  else
+    OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw), TMP1, 0);
+
   if (withchecks)
-    jump = JUMP(SLJIT_C_ZERO);
+    jump = JUMP(SLJIT_ZERO);
 
   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
-  partial = CMP(SLJIT_C_GREATER, STR_PTR, 0, STR_END, 0);
+  partial = CMP(SLJIT_GREATER, STR_PTR, 0, STR_END, 0);
   if (common->mode == JIT_COMPILE)
     add_jump(compiler, backtracks, partial);
 
   add_jump(compiler, *cc == OP_REF ? &common->casefulcmp : &common->caselesscmp, JUMP(SLJIT_FAST_CALL));
-  add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
+  add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
 
   if (common->mode != JIT_COMPILE)
     {
@@ -5171,10 +6812,10 @@ else
     /* TMP2 -= STR_END - STR_PTR */
     OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, STR_PTR, 0);
     OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, STR_END, 0);
-    partial = CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, 0);
+    partial = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, 0);
     OP1(SLJIT_MOV, STR_PTR, 0, STR_END, 0);
     add_jump(compiler, *cc == OP_REF ? &common->casefulcmp : &common->caselesscmp, JUMP(SLJIT_FAST_CALL));
-    add_jump(compiler, backtracks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
+    add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
     JUMPHERE(partial);
     check_partial(common, FALSE);
     add_jump(compiler, backtracks, JUMP(SLJIT_JUMP));
@@ -5189,14 +6830,15 @@ if (jump != NULL)
   else
     JUMPHERE(jump);
   }
-return cc + 1 + IMM2_SIZE;
 }
 
 static SLJIT_INLINE pcre_uchar *compile_ref_iterator_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent)
 {
 DEFINE_COMPILER;
+BOOL ref = (*cc == OP_REF || *cc == OP_REFI);
 backtrack_common *backtrack;
 pcre_uchar type;
+int offset = 0;
 struct sljit_label *label;
 struct sljit_jump *zerolength;
 struct sljit_jump *jump = NULL;
@@ -5204,9 +6846,15 @@ pcre_uchar *ccbegin = cc;
 int min = 0, max = 0;
 BOOL minimize;
 
-PUSH_BACKTRACK(sizeof(iterator_backtrack), cc, NULL);
+PUSH_BACKTRACK(sizeof(ref_iterator_backtrack), cc, NULL);
 
+if (ref)
+  offset = GET2(cc, 1) << 1;
+else
+  cc += IMM2_SIZE;
 type = cc[1 + IMM2_SIZE];
+
+SLJIT_COMPILE_ASSERT((OP_CRSTAR & 0x1) == 0, crstar_opcode_must_be_even);
 minimize = (type & 0x1) != 0;
 switch(type)
   {
@@ -5244,37 +6892,64 @@ if (!minimize)
   if (min == 0)
     {
     allocate_stack(common, 2);
+    if (ref)
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset));
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0);
     /* Temporary release of STR_PTR. */
     OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
-    zerolength = compile_ref_checks(common, ccbegin, NULL);
+    /* Handles both invalid and empty cases. Since the minimum repeat,
+    is zero the invalid case is basically the same as an empty case. */
+    if (ref)
+      zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1));
+    else
+      {
+      compile_dnref_search(common, ccbegin, NULL);
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), 0);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, TMP2, 0);
+      zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw));
+      }
     /* Restore if not zero length. */
     OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
     }
   else
     {
     allocate_stack(common, 1);
+    if (ref)
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset));
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
-    zerolength = compile_ref_checks(common, ccbegin, &backtrack->topbacktracks);
+    if (ref)
+      {
+      add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1)));
+      zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1));
+      }
+    else
+      {
+      compile_dnref_search(common, ccbegin, &backtrack->topbacktracks);
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), 0);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, TMP2, 0);
+      zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw));
+      }
     }
 
   if (min > 1 || max > 1)
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0, SLJIT_IMM, 0);
 
   label = LABEL();
+  if (!ref)
+    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1);
   compile_ref_matchingpath(common, ccbegin, &backtrack->topbacktracks, FALSE, FALSE);
 
   if (min > 1 || max > 1)
     {
-    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0);
     OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, TMP1, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0, TMP1, 0);
     if (min > 1)
-      CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, min, label);
+      CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, min, label);
     if (max > 1)
       {
-      jump = CMP(SLJIT_C_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, max);
+      jump = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, max);
       allocate_stack(common, 1);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
       JUMPTO(SLJIT_JUMP, label);
@@ -5291,30 +6966,58 @@ if (!minimize)
     }
 
   JUMPHERE(zerolength);
-  BACKTRACK_AS(iterator_backtrack)->matchingpath = LABEL();
+  BACKTRACK_AS(ref_iterator_backtrack)->matchingpath = LABEL();
 
   count_match(common);
   return cc;
   }
 
-allocate_stack(common, 2);
+allocate_stack(common, ref ? 2 : 3);
+if (ref)
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset));
 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
 if (type != OP_CRMINSTAR)
   OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0);
 
 if (min == 0)
   {
-  zerolength = compile_ref_checks(common, ccbegin, NULL);
+  /* Handles both invalid and empty cases. Since the minimum repeat,
+  is zero the invalid case is basically the same as an empty case. */
+  if (ref)
+    zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1));
+  else
+    {
+    compile_dnref_search(common, ccbegin, NULL);
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), TMP2, 0);
+    zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw));
+    }
+  /* Length is non-zero, we can match real repeats. */
   OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
   jump = JUMP(SLJIT_JUMP);
   }
 else
-  zerolength = compile_ref_checks(common, ccbegin, &backtrack->topbacktracks);
+  {
+  if (ref)
+    {
+    add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1)));
+    zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1));
+    }
+  else
+    {
+    compile_dnref_search(common, ccbegin, &backtrack->topbacktracks);
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), TMP2, 0);
+    zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw));
+    }
+  }
 
-BACKTRACK_AS(iterator_backtrack)->matchingpath = LABEL();
+BACKTRACK_AS(ref_iterator_backtrack)->matchingpath = LABEL();
 if (max > 0)
-  add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_C_GREATER_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, max));
+  add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, max));
 
+if (!ref)
+  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(2));
 compile_ref_matchingpath(common, ccbegin, &backtrack->topbacktracks, TRUE, TRUE);
 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
 
@@ -5323,7 +7026,7 @@ if (min > 1)
   OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
   OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
   OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
-  CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, min, BACKTRACK_AS(iterator_backtrack)->matchingpath);
+  CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, min, BACKTRACK_AS(ref_iterator_backtrack)->matchingpath);
   }
 else if (max > 0)
   OP2(SLJIT_ADD, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1);
@@ -5383,15 +7086,15 @@ if (entry == NULL)
 
 if (common->has_set_som && common->mark_ptr != 0)
   {
-  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
+  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0));
   allocate_stack(common, 2);
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->mark_ptr);
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr);
   OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0);
   OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
   }
 else if (common->has_set_som || common->mark_ptr != 0)
   {
-  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->has_set_som ? (int)(OVECTOR(0)) : common->mark_ptr);
+  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->has_set_som ? (int)(OVECTOR(0)) : common->mark_ptr);
   allocate_stack(common, 1);
   OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0);
   }
@@ -5401,11 +7104,11 @@ if (entry->entry == NULL)
 else
   JUMPTO(SLJIT_FAST_CALL, entry->entry);
 /* Leave if the match is failed. */
-add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 0));
+add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0));
 return cc + 1 + LINK_SIZE;
 }
 
-static int SLJIT_CALL do_callout(struct jit_arguments* arguments, PUBL(callout_block) *callout_block, pcre_uchar **jit_ovector)
+static int SLJIT_CALL do_callout(struct jit_arguments *arguments, PUBL(callout_block) *callout_block, pcre_uchar **jit_ovector)
 {
 const pcre_uchar *begin = arguments->begin;
 int *offset_vector = arguments->offsets;
@@ -5465,45 +7168,71 @@ PUSH_BACKTRACK(sizeof(backtrack_common), cc, NULL);
 
 allocate_stack(common, CALLOUT_ARG_SIZE / sizeof(sljit_sw));
 
-OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->capture_last_ptr);
-OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
 SLJIT_ASSERT(common->capture_last_ptr != 0);
-OP1(SLJIT_MOV_SI, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(callout_number), SLJIT_IMM, cc[1]);
-OP1(SLJIT_MOV_SI, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(capture_last), TMP2, 0);
+OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr);
+OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
+OP1(SLJIT_MOV_S32, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(callout_number), SLJIT_IMM, cc[1]);
+OP1(SLJIT_MOV_S32, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(capture_last), TMP2, 0);
 
 /* These pointer sized fields temporarly stores internal variables. */
-OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
+OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0));
 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(offset_vector), STR_PTR, 0);
 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(subject), TMP2, 0);
 
 if (common->mark_ptr != 0)
   OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, mark_ptr));
-OP1(SLJIT_MOV_SI, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(pattern_position), SLJIT_IMM, GET(cc, 2));
-OP1(SLJIT_MOV_SI, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(next_item_length), SLJIT_IMM, GET(cc, 2 + LINK_SIZE));
+OP1(SLJIT_MOV_S32, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(pattern_position), SLJIT_IMM, GET(cc, 2));
+OP1(SLJIT_MOV_S32, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(next_item_length), SLJIT_IMM, GET(cc, 2 + LINK_SIZE));
 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(mark), (common->mark_ptr != 0) ? TMP2 : SLJIT_IMM, 0);
 
 /* Needed to save important temporary registers. */
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, STACK_TOP, 0);
-OP2(SLJIT_SUB, SLJIT_SCRATCH_REG2, 0, STACK_TOP, 0, SLJIT_IMM, CALLOUT_ARG_SIZE);
-GET_LOCAL_BASE(SLJIT_SCRATCH_REG3, 0, OVECTOR_START);
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0);
+OP2(SLJIT_SUB, SLJIT_R1, 0, STACK_TOP, 0, SLJIT_IMM, CALLOUT_ARG_SIZE);
+GET_LOCAL_BASE(SLJIT_R2, 0, OVECTOR_START);
 sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_callout));
-OP1(SLJIT_MOV_SI, SLJIT_RETURN_REG, 0, SLJIT_RETURN_REG, 0);
-OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
+OP1(SLJIT_MOV_S32, SLJIT_RETURN_REG, 0, SLJIT_RETURN_REG, 0);
+OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
 free_stack(common, CALLOUT_ARG_SIZE / sizeof(sljit_sw));
 
 /* Check return value. */
 OP2(SLJIT_SUB | SLJIT_SET_S, SLJIT_UNUSED, 0, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0);
-add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_C_SIG_GREATER));
+add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_SIG_GREATER));
 if (common->forced_quit_label == NULL)
-  add_jump(compiler, &common->forced_quit, JUMP(SLJIT_C_SIG_LESS));
+  add_jump(compiler, &common->forced_quit, JUMP(SLJIT_SIG_LESS));
 else
-  JUMPTO(SLJIT_C_SIG_LESS, common->forced_quit_label);
+  JUMPTO(SLJIT_SIG_LESS, common->forced_quit_label);
 return cc + 2 + 2 * LINK_SIZE;
 }
 
 #undef CALLOUT_ARG_SIZE
 #undef CALLOUT_ARG_OFFSET
 
+static SLJIT_INLINE BOOL assert_needs_str_ptr_saving(pcre_uchar *cc)
+{
+while (TRUE)
+  {
+  switch (*cc)
+    {
+    case OP_NOT_WORD_BOUNDARY:
+    case OP_WORD_BOUNDARY:
+    case OP_CIRC:
+    case OP_CIRCM:
+    case OP_DOLL:
+    case OP_DOLLM:
+    case OP_CALLOUT:
+    case OP_ALT:
+    cc += PRIV(OP_lengths)[*cc];
+    break;
+
+    case OP_KET:
+    return FALSE;
+
+    default:
+    return TRUE;
+    }
+  }
+}
+
 static pcre_uchar *compile_assert_matchingpath(compiler_common *common, pcre_uchar *cc, assert_backtrack *backtrack, BOOL conditional)
 {
 DEFINE_COMPILER;
@@ -5555,21 +7284,34 @@ if (bra == OP_BRAMINZERO)
   /* This is a braminzero backtrack path. */
   OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
   free_stack(common, 1);
-  brajump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
+  brajump = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
   }
 
 if (framesize < 0)
   {
-  extrasize = needs_control_head ? 2 : 1;
+  extrasize = 1;
+  if (bra == OP_BRA && !assert_needs_str_ptr_saving(ccbegin + 1 + LINK_SIZE))
+    extrasize = 0;
+
+  if (needs_control_head)
+    extrasize++;
+
   if (framesize == no_frame)
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, STACK_TOP, 0);
-  allocate_stack(common, extrasize);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0);
+
+  if (extrasize > 0)
+    allocate_stack(common, extrasize);
+
   if (needs_control_head)
-    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr);
-  OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr);
+
+  if (extrasize > 0)
+    OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
+
   if (needs_control_head)
     {
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, SLJIT_IMM, 0);
+    SLJIT_ASSERT(extrasize == 2);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0);
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
     }
   }
@@ -5577,20 +7319,23 @@ else
   {
   extrasize = needs_control_head ? 3 : 2;
   allocate_stack(common, framesize + extrasize);
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
   OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw));
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, TMP2, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP2, 0);
   if (needs_control_head)
-    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr);
+    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr);
   OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
+
   if (needs_control_head)
     {
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), TMP1, 0);
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, SLJIT_IMM, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0);
     }
   else
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0);
+
   init_frame(common, ccbegin, NULL, framesize + extrasize - 1, extrasize, FALSE);
   }
 
@@ -5614,7 +7359,7 @@ while (1)
   altbacktrack.top = NULL;
   altbacktrack.topbacktracks = NULL;
 
-  if (*ccbegin == OP_ALT)
+  if (*ccbegin == OP_ALT && extrasize > 0)
     OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
 
   altbacktrack.cc = ccbegin;
@@ -5642,26 +7387,27 @@ while (1)
   if (framesize < 0)
     {
     if (framesize == no_frame)
-      OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
-    else
+      OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
+    else if (extrasize > 0)
       free_stack(common, extrasize);
+
     if (needs_control_head)
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), 0);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), 0);
     }
   else
     {
     if ((opcode != OP_ASSERT_NOT && opcode != OP_ASSERTBACK_NOT) || conditional)
       {
       /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */
-      OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw));
+      OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw));
       if (needs_control_head)
-        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), 0);
+        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), 0);
       }
     else
       {
-      OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+      OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
       if (needs_control_head)
-        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), (framesize + 1) * sizeof(sljit_sw));
+        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), (framesize + 1) * sizeof(sljit_sw));
       add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
       }
     }
@@ -5670,7 +7416,10 @@ while (1)
     {
     /* We know that STR_PTR was stored on the top of the stack. */
     if (conditional)
-      OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), needs_control_head ? sizeof(sljit_sw) : 0);
+      {
+      if (extrasize > 0)
+        OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), needs_control_head ? sizeof(sljit_sw) : 0);
+      }
     else if (bra == OP_BRAZERO)
       {
       if (framesize < 0)
@@ -5679,7 +7428,7 @@ while (1)
         {
         OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw));
         OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (framesize + extrasize - 1) * sizeof(sljit_sw));
-        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, TMP1, 0);
+        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0);
         }
       OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
@@ -5687,7 +7436,7 @@ while (1)
     else if (framesize >= 0)
       {
       /* For OP_BRA and OP_BRAMINZERO. */
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw));
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw));
       }
     }
   add_jump(compiler, found, JUMP(SLJIT_JUMP));
@@ -5731,10 +7480,10 @@ if (common->positive_assert_quit != NULL)
   set_jumps(common->positive_assert_quit, LABEL());
   SLJIT_ASSERT(framesize != no_stack);
   if (framesize < 0)
-    OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, SLJIT_IMM, extrasize * sizeof(sljit_sw));
+    OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, extrasize * sizeof(sljit_sw));
   else
     {
-    OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+    OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
     add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
     OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw));
     }
@@ -5742,12 +7491,12 @@ if (common->positive_assert_quit != NULL)
   }
 
 if (needs_control_head)
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(1));
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(1));
 
 if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK)
   {
   /* Assert is failed. */
-  if (conditional || bra == OP_BRAZERO)
+  if ((conditional && extrasize > 0) || bra == OP_BRAZERO)
     OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
 
   if (framesize < 0)
@@ -5759,7 +7508,7 @@ if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK)
         free_stack(common, 1);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
       }
-    else
+    else if (extrasize > 0)
       free_stack(common, extrasize);
     }
   else
@@ -5773,7 +7522,7 @@ if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK)
       }
     else
       free_stack(common, framesize + extrasize);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, TMP1, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0);
     }
   jump = JUMP(SLJIT_JUMP);
   if (bra != OP_BRAZERO)
@@ -5784,7 +7533,9 @@ if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK)
   if (framesize < 0)
     {
     /* We know that STR_PTR was stored on the top of the stack. */
-    OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (extrasize - 1) * sizeof(sljit_sw));
+    if (extrasize > 0)
+      OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (extrasize - 1) * sizeof(sljit_sw));
+
     /* Keep the STR_PTR on the top of the stack. */
     if (bra == OP_BRAZERO)
       {
@@ -5803,13 +7554,13 @@ if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK)
     if (bra == OP_BRA)
       {
       /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */
-      OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw));
+      OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw));
       OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (extrasize - 2) * sizeof(sljit_sw));
       }
     else
       {
       /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */
-      OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, SLJIT_IMM, (framesize + 2) * sizeof(sljit_sw));
+      OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 2) * sizeof(sljit_sw));
       if (extrasize == 2)
         {
         OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
@@ -5835,9 +7586,9 @@ if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK)
     JUMPHERE(brajump);
     if (framesize >= 0)
       {
-      OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+      OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
       add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw));
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw));
       }
     set_jumps(backtrack->common.topbacktracks, LABEL());
     }
@@ -5847,14 +7598,16 @@ else
   /* AssertNot is successful. */
   if (framesize < 0)
     {
-    OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
+    if (extrasize > 0)
+      OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
+
     if (bra != OP_BRA)
       {
       if (extrasize == 2)
         free_stack(common, 1);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
       }
-    else
+    else if (extrasize > 0)
       free_stack(common, extrasize);
     }
   else
@@ -5869,7 +7622,7 @@ else
       }
     else
       free_stack(common, framesize + extrasize);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, TMP1, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0);
     }
 
   if (bra == OP_BRAZERO)
@@ -5902,116 +7655,6 @@ common->accept = save_accept;
 return cc + 1 + LINK_SIZE;
 }
 
-static sljit_sw SLJIT_CALL do_searchovector(sljit_uw refno, sljit_sw* locals, pcre_uchar *name_table)
-{
-int condition = FALSE;
-pcre_uchar *slotA = name_table;
-pcre_uchar *slotB;
-sljit_sw name_count = locals[LOCALS0 / sizeof(sljit_sw)];
-sljit_sw name_entry_size = locals[LOCALS1 / sizeof(sljit_sw)];
-sljit_sw no_capture;
-int i;
-
-locals += refno & 0xff;
-refno >>= 8;
-no_capture = locals[1];
-
-for (i = 0; i < name_count; i++)
-  {
-  if (GET2(slotA, 0) == refno) break;
-  slotA += name_entry_size;
-  }
-
-if (i < name_count)
-  {
-  /* Found a name for the number - there can be only one; duplicate names
-  for different numbers are allowed, but not vice versa. First scan down
-  for duplicates. */
-
-  slotB = slotA;
-  while (slotB > name_table)
-    {
-    slotB -= name_entry_size;
-    if (STRCMP_UC_UC(slotA + IMM2_SIZE, slotB + IMM2_SIZE) == 0)
-      {
-      condition = locals[GET2(slotB, 0) << 1] != no_capture;
-      if (condition) break;
-      }
-    else break;
-    }
-
-  /* Scan up for duplicates */
-  if (!condition)
-    {
-    slotB = slotA;
-    for (i++; i < name_count; i++)
-      {
-      slotB += name_entry_size;
-      if (STRCMP_UC_UC(slotA + IMM2_SIZE, slotB + IMM2_SIZE) == 0)
-        {
-        condition = locals[GET2(slotB, 0) << 1] != no_capture;
-        if (condition) break;
-        }
-      else break;
-      }
-    }
-  }
-return condition;
-}
-
-static sljit_sw SLJIT_CALL do_searchgroups(sljit_uw recno, sljit_uw* locals, pcre_uchar *name_table)
-{
-int condition = FALSE;
-pcre_uchar *slotA = name_table;
-pcre_uchar *slotB;
-sljit_uw name_count = locals[LOCALS0 / sizeof(sljit_sw)];
-sljit_uw name_entry_size = locals[LOCALS1 / sizeof(sljit_sw)];
-sljit_uw group_num = locals[POSSESSIVE0 / sizeof(sljit_sw)];
-sljit_uw i;
-
-for (i = 0; i < name_count; i++)
-  {
-  if (GET2(slotA, 0) == recno) break;
-  slotA += name_entry_size;
-  }
-
-if (i < name_count)
-  {
-  /* Found a name for the number - there can be only one; duplicate
-  names for different numbers are allowed, but not vice versa. First
-  scan down for duplicates. */
-
-  slotB = slotA;
-  while (slotB > name_table)
-    {
-    slotB -= name_entry_size;
-    if (STRCMP_UC_UC(slotA + IMM2_SIZE, slotB + IMM2_SIZE) == 0)
-      {
-      condition = GET2(slotB, 0) == group_num;
-      if (condition) break;
-      }
-    else break;
-    }
-
-  /* Scan up for duplicates */
-  if (!condition)
-    {
-    slotB = slotA;
-    for (i++; i < name_count; i++)
-      {
-      slotB += name_entry_size;
-      if (STRCMP_UC_UC(slotA + IMM2_SIZE, slotB + IMM2_SIZE) == 0)
-        {
-        condition = GET2(slotB, 0) == group_num;
-        if (condition) break;
-        }
-      else break;
-      }
-    }
-  }
-return condition;
-}
-
 static SLJIT_INLINE void match_once_common(compiler_common *common, pcre_uchar ket, int framesize, int private_data_ptr, BOOL has_alternatives, BOOL needs_control_head)
 {
 DEFINE_COMPILER;
@@ -6020,13 +7663,15 @@ int stacksize;
 if (framesize < 0)
   {
   if (framesize == no_frame)
-    OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+    OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
   else
     {
     stacksize = needs_control_head ? 1 : 0;
     if (ket != OP_KET || has_alternatives)
       stacksize++;
-    free_stack(common, stacksize);
+
+    if (stacksize > 0)
+      free_stack(common, stacksize);
     }
 
   if (needs_control_head)
@@ -6038,13 +7683,13 @@ if (framesize < 0)
   else if (ket == OP_KETRMIN)
     {
     /* Move the STR_PTR to the private_data_ptr. */
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, SLJIT_MEM1(STACK_TOP), 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), 0);
     }
   }
 else
   {
   stacksize = (ket != OP_KET || has_alternatives) ? 2 : 1;
-  OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, SLJIT_IMM, (framesize + stacksize) * sizeof(sljit_sw));
+  OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + stacksize) * sizeof(sljit_sw));
   if (needs_control_head)
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), 0);
 
@@ -6055,7 +7700,7 @@ else
     }
   }
 if (needs_control_head)
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, TMP1, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, TMP1, 0);
 }
 
 static SLJIT_INLINE int match_capture_common(compiler_common *common, int stacksize, int offset, int private_data_ptr)
@@ -6064,20 +7709,20 @@ DEFINE_COMPILER;
 
 if (common->capture_last_ptr != 0)
   {
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->capture_last_ptr);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->capture_last_ptr, SLJIT_IMM, offset >> 1);
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, SLJIT_IMM, offset >> 1);
   OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), TMP1, 0);
   stacksize++;
   }
 if (common->optimized_cbracket[offset >> 1] == 0)
   {
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
-  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset));
+  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1));
   OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), TMP1, 0);
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
   OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize + 1), TMP2, 0);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0);
   stacksize += 2;
   }
 return stacksize;
@@ -6144,11 +7789,12 @@ backtrack_common *backtrack;
 pcre_uchar opcode;
 int private_data_ptr = 0;
 int offset = 0;
-int stacksize;
+int i, stacksize;
 int repeat_ptr = 0, repeat_length = 0;
 int repeat_type = 0, repeat_count = 0;
 pcre_uchar *ccbegin;
 pcre_uchar *matchingpath;
+pcre_uchar *slot;
 pcre_uchar bra = OP_BRA;
 pcre_uchar ket;
 assert_backtrack *assert;
@@ -6198,20 +7844,8 @@ SLJIT_ASSERT(!((bra == OP_BRAZERO && ket == OP_KETRMIN) || (bra == OP_BRAMINZERO
 cc += GET(cc, 1);
 
 has_alternatives = *cc == OP_ALT;
-if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND))
-  {
-  has_alternatives = (*matchingpath == OP_RREF) ? FALSE : TRUE;
-  if (*matchingpath == OP_NRREF)
-    {
-    stacksize = GET2(matchingpath, 1);
-    if (common->currententry == NULL || stacksize == RREF_ANY)
-      has_alternatives = FALSE;
-    else if (common->currententry->start == 0)
-      has_alternatives = stacksize != 0;
-    else
-      has_alternatives = stacksize != (int)GET2(common->start, common->currententry->start + 1 + LINK_SIZE);
-    }
-  }
+if (SLJIT_UNLIKELY(opcode == OP_COND || opcode == OP_SCOND))
+  has_alternatives = (*matchingpath == OP_RREF || *matchingpath == OP_DNRREF || *matchingpath == OP_FAIL) ? FALSE : TRUE;
 
 if (SLJIT_UNLIKELY(opcode == OP_COND) && (*cc == OP_KETRMAX || *cc == OP_KETRMIN))
   opcode = OP_SCOND;
@@ -6272,13 +7906,13 @@ if (bra == OP_BRAMINZERO)
   if (ket != OP_KETRMIN)
     {
     free_stack(common, 1);
-    braminzero = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
+    braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
     }
   else
     {
     if (opcode == OP_ONCE || opcode >= OP_SBRA)
       {
-      jump = CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
+      jump = CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
       OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
       /* Nothing stored during the first run. */
       skip = JUMP(SLJIT_JUMP);
@@ -6287,19 +7921,19 @@ if (bra == OP_BRAMINZERO)
       if (opcode != OP_ONCE || BACKTRACK_AS(bracket_backtrack)->u.framesize < 0)
         {
         /* When we come from outside, private_data_ptr contains the previous STR_PTR. */
-        braminzero = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+        braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
         }
       else
         {
         /* Except when the whole stack frame must be saved. */
-        OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
-        braminzero = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (BACKTRACK_AS(bracket_backtrack)->u.framesize + 1) * sizeof(sljit_sw));
+        OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
+        braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (BACKTRACK_AS(bracket_backtrack)->u.framesize + 1) * sizeof(sljit_sw));
         }
       JUMPHERE(skip);
       }
     else
       {
-      jump = CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
+      jump = CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
       OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
       JUMPHERE(jump);
       }
@@ -6308,7 +7942,7 @@ if (bra == OP_BRAMINZERO)
 
 if (repeat_type != 0)
   {
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), repeat_ptr, SLJIT_IMM, repeat_count);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, repeat_count);
   if (repeat_type == OP_EXACT)
     rmax_label = LABEL();
   }
@@ -6329,7 +7963,7 @@ if (opcode == OP_ONCE)
   stacksize = 0;
   if (needs_control_head)
     {
-    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr);
+    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr);
     stacksize++;
     }
 
@@ -6340,12 +7974,12 @@ if (opcode == OP_ONCE)
       {
       stacksize += 2;
       if (!needs_control_head)
-        OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+        OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
       }
     else
       {
       if (BACKTRACK_AS(bracket_backtrack)->u.framesize == no_frame)
-        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, STACK_TOP, 0);
+        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0);
       if (ket == OP_KETRMAX || has_alternatives)
         stacksize++;
       }
@@ -6363,10 +7997,10 @@ if (opcode == OP_ONCE)
     if (ket == OP_KETRMIN)
       {
       if (needs_control_head)
-        OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+        OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0);
       if (BACKTRACK_AS(bracket_backtrack)->u.framesize == no_frame)
-        OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, needs_control_head ? (2 * sizeof(sljit_sw)) : sizeof(sljit_sw));
+        OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, needs_control_head ? (2 * sizeof(sljit_sw)) : sizeof(sljit_sw));
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize + 1), TMP2, 0);
       }
     else if (ket == OP_KETRMAX || has_alternatives)
@@ -6383,20 +8017,20 @@ if (opcode == OP_ONCE)
     if (needs_control_head)
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0);
 
-    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
     OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw));
 
     stacksize = needs_control_head ? 1 : 0;
     if (ket != OP_KET || has_alternatives)
       {
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0);
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, TMP2, 0);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP2, 0);
       stacksize++;
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), TMP1, 0);
       }
     else
       {
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, TMP2, 0);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP2, 0);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), TMP1, 0);
       }
     init_frame(common, ccbegin, NULL, BACKTRACK_AS(bracket_backtrack)->u.framesize + stacksize, stacksize + 1, FALSE);
@@ -6409,26 +8043,26 @@ else if (opcode == OP_CBRA || opcode == OP_SCBRA)
     {
     SLJIT_ASSERT(private_data_ptr == OVECTOR(offset));
     allocate_stack(common, 2);
-    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
-    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr + sizeof(sljit_sw));
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, STR_PTR, 0);
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
+    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr + sizeof(sljit_sw));
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STR_PTR, 0);
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0);
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0);
     }
   else
     {
-    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
     allocate_stack(common, 1);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, STR_PTR, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STR_PTR, 0);
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0);
     }
   }
 else if (opcode == OP_SBRA || opcode == OP_SCOND)
   {
   /* Saving the previous value. */
-  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
   allocate_stack(common, 1);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, STR_PTR, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STR_PTR, 0);
   OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0);
   }
 else if (has_alternatives)
@@ -6445,49 +8079,77 @@ if (opcode == OP_COND || opcode == OP_SCOND)
     {
     SLJIT_ASSERT(has_alternatives);
     add_jump(compiler, &(BACKTRACK_AS(bracket_backtrack)->u.condfailed),
-      CMP(SLJIT_C_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(GET2(matchingpath, 1) << 1), SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1)));
+      CMP(SLJIT_EQUAL, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(matchingpath, 1) << 1), SLJIT_MEM1(SLJIT_SP), OVECTOR(1)));
     matchingpath += 1 + IMM2_SIZE;
     }
-  else if (*matchingpath == OP_NCREF)
+  else if (*matchingpath == OP_DNCREF)
     {
     SLJIT_ASSERT(has_alternatives);
-    stacksize = GET2(matchingpath, 1);
-    jump = CMP(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(stacksize << 1), SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(1));
-
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STACK_TOP, 0);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, SLJIT_IMM, common->name_count);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, SLJIT_IMM, common->name_entry_size);
-    OP1(SLJIT_MOV, SLJIT_SCRATCH_REG1, 0, SLJIT_IMM, (stacksize << 8) | (common->ovector_start / sizeof(sljit_sw)));
-    GET_LOCAL_BASE(SLJIT_SCRATCH_REG2, 0, 0);
-    OP1(SLJIT_MOV, SLJIT_SCRATCH_REG3, 0, SLJIT_IMM, common->name_table);
-    sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_searchovector));
-    OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1);
-    add_jump(compiler, &(BACKTRACK_AS(bracket_backtrack)->u.condfailed), CMP(SLJIT_C_EQUAL, SLJIT_SCRATCH_REG1, 0, SLJIT_IMM, 0));
 
-    JUMPHERE(jump);
-    matchingpath += 1 + IMM2_SIZE;
+    i = GET2(matchingpath, 1 + IMM2_SIZE);
+    slot = common->name_table + GET2(matchingpath, 1) * common->name_entry_size;
+    OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0);
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1));
+    OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(slot, 0) << 1), TMP1, 0);
+    slot += common->name_entry_size;
+    i--;
+    while (i-- > 0)
+      {
+      OP2(SLJIT_SUB, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(slot, 0) << 1), TMP1, 0);
+      OP2(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, STR_PTR, 0);
+      slot += common->name_entry_size;
+      }
+    OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0);
+    add_jump(compiler, &(BACKTRACK_AS(bracket_backtrack)->u.condfailed), JUMP(SLJIT_ZERO));
+    matchingpath += 1 + 2 * IMM2_SIZE;
     }
-  else if (*matchingpath == OP_RREF || *matchingpath == OP_NRREF)
+  else if (*matchingpath == OP_RREF || *matchingpath == OP_DNRREF || *matchingpath == OP_FAIL)
     {
     /* Never has other case. */
     BACKTRACK_AS(bracket_backtrack)->u.condfailed = NULL;
+    SLJIT_ASSERT(!has_alternatives);
 
-    stacksize = GET2(matchingpath, 1);
-    if (common->currententry == NULL)
+    if (*matchingpath == OP_FAIL)
       stacksize = 0;
-    else if (stacksize == RREF_ANY)
-      stacksize = 1;
-    else if (common->currententry->start == 0)
-      stacksize = stacksize == 0;
-    else
-      stacksize = stacksize == (int)GET2(common->start, common->currententry->start + 1 + LINK_SIZE);
-
-    if (*matchingpath == OP_RREF || stacksize || common->currententry == NULL)
+    if (*matchingpath == OP_RREF)
       {
-      SLJIT_ASSERT(!has_alternatives);
+      stacksize = GET2(matchingpath, 1);
+      if (common->currententry == NULL)
+        stacksize = 0;
+      else if (stacksize == RREF_ANY)
+        stacksize = 1;
+      else if (common->currententry->start == 0)
+        stacksize = stacksize == 0;
+      else
+        stacksize = stacksize == (int)GET2(common->start, common->currententry->start + 1 + LINK_SIZE);
+
       if (stacksize != 0)
         matchingpath += 1 + IMM2_SIZE;
+      }
+    else
+      {
+      if (common->currententry == NULL || common->currententry->start == 0)
+        stacksize = 0;
       else
+        {
+        stacksize = GET2(matchingpath, 1 + IMM2_SIZE);
+        slot = common->name_table + GET2(matchingpath, 1) * common->name_entry_size;
+        i = (int)GET2(common->start, common->currententry->start + 1 + LINK_SIZE);
+        while (stacksize > 0)
+          {
+          if ((int)GET2(slot, 0) == i)
+            break;
+          slot += common->name_entry_size;
+          stacksize--;
+          }
+        }
+
+      if (stacksize != 0)
+        matchingpath += 1 + 2 * IMM2_SIZE;
+      }
+
+      /* The stacksize == 0 is a common "else" case. */
+      if (stacksize == 0)
         {
         if (*cc == OP_ALT)
           {
@@ -6497,24 +8159,6 @@ if (opcode == OP_COND || opcode == OP_SCOND)
         else
           matchingpath = cc;
         }
-      }
-    else
-      {
-      SLJIT_ASSERT(has_alternatives);
-
-      stacksize = GET2(matchingpath, 1);
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, STACK_TOP, 0);
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, SLJIT_IMM, common->name_count);
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, SLJIT_IMM, common->name_entry_size);
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, GET2(common->start, common->currententry->start + 1 + LINK_SIZE));
-      OP1(SLJIT_MOV, SLJIT_SCRATCH_REG1, 0, SLJIT_IMM, stacksize);
-      GET_LOCAL_BASE(SLJIT_SCRATCH_REG2, 0, 0);
-      OP1(SLJIT_MOV, SLJIT_SCRATCH_REG3, 0, SLJIT_IMM, common->name_table);
-      sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_searchgroups));
-      OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1);
-      add_jump(compiler, &(BACKTRACK_AS(bracket_backtrack)->u.condfailed), CMP(SLJIT_C_EQUAL, SLJIT_SCRATCH_REG1, 0, SLJIT_IMM, 0));
-      matchingpath += 1 + IMM2_SIZE;
-      }
     }
   else
     {
@@ -6541,7 +8185,7 @@ stacksize = 0;
 if (repeat_type == OP_MINUPTO)
   {
   /* We need to preserve the counter. TMP2 will be used below. */
-  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), repeat_ptr);
+  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), repeat_ptr);
   stacksize++;
   }
 if (ket != OP_KET || bra != OP_BRA)
@@ -6591,7 +8235,7 @@ if (has_alternatives)
 if (offset != 0 && common->optimized_cbracket[offset >> 1] != 0)
   {
   SLJIT_ASSERT(private_data_ptr == OVECTOR(offset + 0));
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0);
   }
 
 if (ket == OP_KETRMAX)
@@ -6600,8 +8244,8 @@ if (ket == OP_KETRMAX)
     {
     if (has_alternatives)
       BACKTRACK_AS(bracket_backtrack)->alternative_matchingpath = LABEL();
-    OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_MEM1(SLJIT_LOCALS_REG), repeat_ptr, SLJIT_MEM1(SLJIT_LOCALS_REG), repeat_ptr, SLJIT_IMM, 1);
-    JUMPTO(SLJIT_C_NOT_ZERO, rmax_label);
+    OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1);
+    JUMPTO(SLJIT_NOT_ZERO, rmax_label);
     /* Drop STR_PTR for greedy plus quantifier. */
     if (opcode != OP_ONCE)
       free_stack(common, 1);
@@ -6613,14 +8257,14 @@ if (ket == OP_KETRMAX)
     /* Checking zero-length iteration. */
     if (opcode != OP_ONCE)
       {
-      CMPTO(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, STR_PTR, 0, rmax_label);
+      CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STR_PTR, 0, rmax_label);
       /* Drop STR_PTR for greedy plus quantifier. */
       if (bra != OP_BRAZERO)
         free_stack(common, 1);
       }
     else
       /* TMP2 must contain the starting STR_PTR. */
-      CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_PTR, 0, rmax_label);
+      CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, STR_PTR, 0, rmax_label);
     }
   else
     JUMPTO(SLJIT_JUMP, rmax_label);
@@ -6629,13 +8273,14 @@ if (ket == OP_KETRMAX)
 
 if (repeat_type == OP_EXACT)
   {
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_MEM1(SLJIT_LOCALS_REG), repeat_ptr, SLJIT_MEM1(SLJIT_LOCALS_REG), repeat_ptr, SLJIT_IMM, 1);
-  JUMPTO(SLJIT_C_NOT_ZERO, rmax_label);
+  count_match(common);
+  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1);
+  JUMPTO(SLJIT_NOT_ZERO, rmax_label);
   }
 else if (repeat_type == OP_UPTO)
   {
   /* We need to preserve the counter. */
-  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), repeat_ptr);
+  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), repeat_ptr);
   allocate_stack(common, 1);
   OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0);
   }
@@ -6655,7 +8300,7 @@ if (bra == OP_BRAMINZERO)
     framesize is < 0, OP_ONCE will do the release itself. */
     if (opcode == OP_ONCE && BACKTRACK_AS(bracket_backtrack)->u.framesize >= 0)
       {
-      OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+      OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
       add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
       }
     else if (ket == OP_KETRMIN && opcode != OP_ONCE)
@@ -6672,9 +8317,13 @@ while (*cc == OP_ALT)
   cc += GET(cc, 1);
 cc += 1 + LINK_SIZE;
 
-/* Temporarily encoding the needs_control_head in framesize. */
 if (opcode == OP_ONCE)
+  {
+  /* We temporarily encode the needs_control_head in the lowest bit.
+     Note: on the target architectures of SLJIT the ((x << 1) >> 1) returns
+     the same value for small signed numbers (including negative numbers). */
   BACKTRACK_AS(bracket_backtrack)->u.framesize = (BACKTRACK_AS(bracket_backtrack)->u.framesize << 1) | (needs_control_head ? 1 : 0);
+  }
 return cc + repeat_length;
 }
 
@@ -6750,20 +8399,20 @@ if (framesize < 0)
   BACKTRACK_AS(bracketpos_backtrack)->stacksize = stacksize;
   allocate_stack(common, stacksize);
   if (framesize == no_frame)
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, STACK_TOP, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0);
 
   stack = 0;
   if (offset != 0)
     {
     stack = 2;
-    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset));
-    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1));
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset));
+    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1));
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0);
     if (common->capture_last_ptr != 0)
-      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->capture_last_ptr);
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr);
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0);
     if (needs_control_head)
-      OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr);
+      OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr);
     if (common->capture_last_ptr != 0)
       {
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), TMP1, 0);
@@ -6773,7 +8422,7 @@ if (framesize < 0)
   else
     {
     if (needs_control_head)
-      OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr);
+      OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr);
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
     stack = 1;
     }
@@ -6800,10 +8449,10 @@ else
   BACKTRACK_AS(bracketpos_backtrack)->stacksize = stacksize;
 
   allocate_stack(common, stacksize);
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
   if (needs_control_head)
-    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr);
-  OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, -STACK(stacksize - 1));
+    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr);
+  OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, -STACK(stacksize - 1));
 
   stack = 0;
   if (!zero)
@@ -6827,7 +8476,7 @@ else
   }
 
 if (offset != 0)
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr, STR_PTR, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), cbraprivptr, STR_PTR, 0);
 
 loop = LABEL();
 while (*cc != OP_KETRPOS)
@@ -6843,16 +8492,16 @@ while (*cc != OP_KETRPOS)
   if (framesize < 0)
     {
     if (framesize == no_frame)
-      OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+      OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
 
     if (offset != 0)
       {
-      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr);
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr, STR_PTR, 0);
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), cbraprivptr);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), cbraprivptr, STR_PTR, 0);
       if (common->capture_last_ptr != 0)
-        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->capture_last_ptr, SLJIT_IMM, offset >> 1);
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
+        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, SLJIT_IMM, offset >> 1);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0);
       }
     else
       {
@@ -6861,8 +8510,12 @@ while (*cc != OP_KETRPOS)
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
       }
 
+    /* Even if the match is empty, we need to reset the control head. */
+    if (needs_control_head)
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(stack));
+
     if (opcode == OP_SBRAPOS || opcode == OP_SCBRAPOS)
-      add_jump(compiler, &emptymatch, CMP(SLJIT_C_EQUAL, TMP1, 0, STR_PTR, 0));
+      add_jump(compiler, &emptymatch, CMP(SLJIT_EQUAL, TMP1, 0, STR_PTR, 0));
 
     if (!zero)
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0);
@@ -6871,25 +8524,29 @@ while (*cc != OP_KETRPOS)
     {
     if (offset != 0)
       {
-      OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, SLJIT_IMM, stacksize * sizeof(sljit_sw));
-      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr);
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr, STR_PTR, 0);
+      OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, stacksize * sizeof(sljit_sw));
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), cbraprivptr);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), cbraprivptr, STR_PTR, 0);
       if (common->capture_last_ptr != 0)
-        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->capture_last_ptr, SLJIT_IMM, offset >> 1);
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
+        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, SLJIT_IMM, offset >> 1);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0);
       }
     else
       {
-      OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+      OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
       OP2(SLJIT_ADD, STACK_TOP, 0, TMP2, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw));
       if (opcode == OP_SBRAPOS)
         OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_sw));
       OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_sw), STR_PTR, 0);
       }
 
+    /* Even if the match is empty, we need to reset the control head. */
+    if (needs_control_head)
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(stack));
+
     if (opcode == OP_SBRAPOS || opcode == OP_SCBRAPOS)
-      add_jump(compiler, &emptymatch, CMP(SLJIT_C_EQUAL, TMP1, 0, STR_PTR, 0));
+      add_jump(compiler, &emptymatch, CMP(SLJIT_EQUAL, TMP1, 0, STR_PTR, 0));
 
     if (!zero)
       {
@@ -6900,9 +8557,6 @@ while (*cc != OP_KETRPOS)
       }
     }
 
-  if (needs_control_head)
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(stack));
-
   JUMPTO(SLJIT_JUMP, loop);
   flush_stubs(common);
 
@@ -6914,7 +8568,7 @@ while (*cc != OP_KETRPOS)
   if (framesize < 0)
     {
     if (offset != 0)
-      OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr);
+      OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), cbraprivptr);
     else
       OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
     }
@@ -6924,12 +8578,12 @@ while (*cc != OP_KETRPOS)
       {
       /* Last alternative. */
       if (*cc == OP_KETRPOS)
-        OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
-      OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), cbraprivptr);
+        OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
+      OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), cbraprivptr);
       }
     else
       {
-      OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+      OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
       OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_sw));
       }
     }
@@ -6945,9 +8599,9 @@ backtrack->topbacktracks = NULL;
 if (!zero)
   {
   if (framesize < 0)
-    add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0));
+    add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0));
   else /* TMP2 is set to [private_data_ptr] above. */
-    add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(TMP2), (stacksize - 1) * sizeof(sljit_sw), SLJIT_IMM, 0));
+    add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(TMP2), (stacksize - 1) * sizeof(sljit_sw), SLJIT_IMM, 0));
   }
 
 /* None of them matched. */
@@ -6956,11 +8610,13 @@ count_match(common);
 return cc + 1 + LINK_SIZE;
 }
 
-static SLJIT_INLINE pcre_uchar *get_iterator_parameters(compiler_common *common, pcre_uchar *cc, pcre_uchar *opcode, pcre_uchar *type, int *arg1, int *arg2, pcre_uchar **end)
+static SLJIT_INLINE pcre_uchar *get_iterator_parameters(compiler_common *common, pcre_uchar *cc, pcre_uchar *opcode, pcre_uchar *type, sljit_u32 *max, sljit_u32 *exact, pcre_uchar **end)
 {
 int class_len;
 
 *opcode = *cc;
+*exact = 0;
+
 if (*opcode >= OP_STAR && *opcode <= OP_POSUPTO)
   {
   cc++;
@@ -6988,63 +8644,114 @@ else if (*opcode >= OP_TYPESTAR && *opcode <= OP_TYPEPOSUPTO)
   {
   cc++;
   *opcode -= OP_TYPESTAR - OP_STAR;
-  *type = 0;
+  *type = OP_END;
   }
 else
   {
-  SLJIT_ASSERT(*opcode >= OP_CLASS || *opcode <= OP_XCLASS);
+  SLJIT_ASSERT(*opcode == OP_CLASS || *opcode == OP_NCLASS || *opcode == OP_XCLASS);
   *type = *opcode;
   cc++;
   class_len = (*type < OP_XCLASS) ? (int)(1 + (32 / sizeof(pcre_uchar))) : GET(cc, 0);
   *opcode = cc[class_len - 1];
+
   if (*opcode >= OP_CRSTAR && *opcode <= OP_CRMINQUERY)
     {
     *opcode -= OP_CRSTAR - OP_STAR;
-    if (end != NULL)
-      *end = cc + class_len;
+    *end = cc + class_len;
+
+    if (*opcode == OP_PLUS || *opcode == OP_MINPLUS)
+      {
+      *exact = 1;
+      *opcode -= OP_PLUS - OP_STAR;
+      }
     }
-  else
+  else if (*opcode >= OP_CRPOSSTAR && *opcode <= OP_CRPOSQUERY)
     {
-    SLJIT_ASSERT(*opcode == OP_CRRANGE || *opcode == OP_CRMINRANGE);
-    *arg1 = GET2(cc, (class_len + IMM2_SIZE));
-    *arg2 = GET2(cc, class_len);
+    *opcode -= OP_CRPOSSTAR - OP_POSSTAR;
+    *end = cc + class_len;
 
-    if (*arg2 == 0)
+    if (*opcode == OP_POSPLUS)
       {
-      SLJIT_ASSERT(*arg1 != 0);
-      *opcode = (*opcode == OP_CRRANGE) ? OP_UPTO : OP_MINUPTO;
+      *exact = 1;
+      *opcode = OP_POSSTAR;
       }
-    if (*arg1 == *arg2)
-      *opcode = OP_EXACT;
+    }
+  else
+    {
+    SLJIT_ASSERT(*opcode == OP_CRRANGE || *opcode == OP_CRMINRANGE || *opcode == OP_CRPOSRANGE);
+    *max = GET2(cc, (class_len + IMM2_SIZE));
+    *exact = GET2(cc, class_len);
 
-    if (end != NULL)
-      *end = cc + class_len + 2 * IMM2_SIZE;
+    if (*max == 0)
+      {
+      if (*opcode == OP_CRPOSRANGE)
+        *opcode = OP_POSSTAR;
+      else
+        *opcode -= OP_CRRANGE - OP_STAR;
+      }
+    else
+      {
+      *max -= *exact;
+      if (*max == 0)
+        *opcode = OP_EXACT;
+      else if (*max == 1)
+        {
+        if (*opcode == OP_CRPOSRANGE)
+          *opcode = OP_POSQUERY;
+        else
+          *opcode -= OP_CRRANGE - OP_QUERY;
+        }
+      else
+        {
+        if (*opcode == OP_CRPOSRANGE)
+          *opcode = OP_POSUPTO;
+        else
+          *opcode -= OP_CRRANGE - OP_UPTO;
+        }
+      }
+    *end = cc + class_len + 2 * IMM2_SIZE;
     }
   return cc;
   }
 
-if (*opcode == OP_UPTO || *opcode == OP_MINUPTO || *opcode == OP_EXACT || *opcode == OP_POSUPTO)
+switch(*opcode)
   {
-  *arg1 = GET2(cc, 0);
+  case OP_EXACT:
+  *exact = GET2(cc, 0);
+  cc += IMM2_SIZE;
+  break;
+
+  case OP_PLUS:
+  case OP_MINPLUS:
+  *exact = 1;
+  *opcode -= OP_PLUS - OP_STAR;
+  break;
+
+  case OP_POSPLUS:
+  *exact = 1;
+  *opcode = OP_POSSTAR;
+  break;
+
+  case OP_UPTO:
+  case OP_MINUPTO:
+  case OP_POSUPTO:
+  *max = GET2(cc, 0);
   cc += IMM2_SIZE;
+  break;
   }
 
-if (*type == 0)
+if (*type == OP_END)
   {
   *type = *cc;
-  if (end != NULL)
-    *end = next_opcode(common, cc);
+  *end = next_opcode(common, cc);
   cc++;
   return cc;
   }
 
-if (end != NULL)
-  {
-  *end = cc + 1;
+*end = cc + 1;
 #ifdef SUPPORT_UTF
-  if (common->utf && HAS_EXTRALEN(*cc)) *end += GET_EXTRALEN(*cc);
+if (common->utf && HAS_EXTRALEN(*cc)) *end += GET_EXTRALEN(*cc);
 #endif
-  }
 return cc;
 }
 
@@ -7054,95 +8761,110 @@ DEFINE_COMPILER;
 backtrack_common *backtrack;
 pcre_uchar opcode;
 pcre_uchar type;
-int arg1 = -1, arg2 = -1;
-pcre_uchar* end;
-jump_list *nomatch = NULL;
+sljit_u32 max = 0, exact;
+BOOL fast_fail;
+sljit_s32 fast_str_ptr;
+BOOL charpos_enabled;
+pcre_uchar charpos_char;
+unsigned int charpos_othercasebit;
+pcre_uchar *end;
+jump_list *no_match = NULL;
+jump_list *no_char1_match = NULL;
 struct sljit_jump *jump = NULL;
 struct sljit_label *label;
 int private_data_ptr = PRIVATE_DATA(cc);
-int base = (private_data_ptr == 0) ? SLJIT_MEM1(STACK_TOP) : SLJIT_MEM1(SLJIT_LOCALS_REG);
+int base = (private_data_ptr == 0) ? SLJIT_MEM1(STACK_TOP) : SLJIT_MEM1(SLJIT_SP);
 int offset0 = (private_data_ptr == 0) ? STACK(0) : private_data_ptr;
 int offset1 = (private_data_ptr == 0) ? STACK(1) : private_data_ptr + (int)sizeof(sljit_sw);
 int tmp_base, tmp_offset;
 
-PUSH_BACKTRACK(sizeof(iterator_backtrack), cc, NULL);
+PUSH_BACKTRACK(sizeof(char_iterator_backtrack), cc, NULL);
 
-cc = get_iterator_parameters(common, cc, &opcode, &type, &arg1, &arg2, &end);
+fast_str_ptr = PRIVATE_DATA(cc + 1);
+fast_fail = TRUE;
 
-switch(type)
+SLJIT_ASSERT(common->fast_forward_bc_ptr == NULL || fast_str_ptr == 0 || cc == common->fast_forward_bc_ptr);
+
+if (cc == common->fast_forward_bc_ptr)
+  fast_fail = FALSE;
+else if (common->fast_fail_start_ptr == 0)
+  fast_str_ptr = 0;
+
+SLJIT_ASSERT(common->fast_forward_bc_ptr != NULL || fast_str_ptr == 0
+  || (fast_str_ptr >= common->fast_fail_start_ptr && fast_str_ptr <= common->fast_fail_end_ptr));
+
+cc = get_iterator_parameters(common, cc, &opcode, &type, &max, &exact, &end);
+
+if (type != OP_EXTUNI)
   {
-  case OP_NOT_DIGIT:
-  case OP_DIGIT:
-  case OP_NOT_WHITESPACE:
-  case OP_WHITESPACE:
-  case OP_NOT_WORDCHAR:
-  case OP_WORDCHAR:
-  case OP_ANY:
-  case OP_ALLANY:
-  case OP_ANYBYTE:
-  case OP_ANYNL:
-  case OP_NOT_HSPACE:
-  case OP_HSPACE:
-  case OP_NOT_VSPACE:
-  case OP_VSPACE:
-  case OP_CHAR:
-  case OP_CHARI:
-  case OP_NOT:
-  case OP_NOTI:
-  case OP_CLASS:
-  case OP_NCLASS:
   tmp_base = TMP3;
   tmp_offset = 0;
-  break;
+  }
+else
+  {
+  tmp_base = SLJIT_MEM1(SLJIT_SP);
+  tmp_offset = POSSESSIVE0;
+  }
 
-  default:
-  SLJIT_ASSERT_STOP();
-  /* Fall through. */
+if (fast_fail && fast_str_ptr != 0)
+  add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), fast_str_ptr));
 
-  case OP_EXTUNI:
-  case OP_XCLASS:
-  case OP_NOTPROP:
-  case OP_PROP:
-  tmp_base = SLJIT_MEM1(SLJIT_LOCALS_REG);
-  tmp_offset = POSSESSIVE0;
-  break;
+/* Handle fixed part first. */
+if (exact > 1)
+  {
+  SLJIT_ASSERT(fast_str_ptr == 0);
+  if (common->mode == JIT_COMPILE
+#ifdef SUPPORT_UTF
+      && !common->utf
+#endif
+      )
+    {
+    OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(exact));
+    add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_GREATER, TMP1, 0, STR_END, 0));
+    OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, exact);
+    label = LABEL();
+    compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, FALSE);
+    OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+    JUMPTO(SLJIT_NOT_ZERO, label);
+    }
+  else
+    {
+    OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, exact);
+    label = LABEL();
+    compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, TRUE);
+    OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+    JUMPTO(SLJIT_NOT_ZERO, label);
+    }
   }
+else if (exact == 1)
+  compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, TRUE);
 
 switch(opcode)
   {
   case OP_STAR:
-  case OP_PLUS:
   case OP_UPTO:
-  case OP_CRRANGE:
+  SLJIT_ASSERT(fast_str_ptr == 0 || opcode == OP_STAR);
+
   if (type == OP_ANYNL || type == OP_EXTUNI)
     {
     SLJIT_ASSERT(private_data_ptr == 0);
-    if (opcode == OP_STAR || opcode == OP_UPTO)
-      {
-      allocate_stack(common, 2);
-      OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
-      OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0);
-      }
-    else
-      {
-      allocate_stack(common, 1);
-      OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
-      }
+    SLJIT_ASSERT(fast_str_ptr == 0);
 
-    if (opcode == OP_UPTO || opcode == OP_CRRANGE)
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, SLJIT_IMM, 0);
+    allocate_stack(common, 2);
+    OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0);
+
+    if (opcode == OP_UPTO)
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0, SLJIT_IMM, max);
 
     label = LABEL();
-    compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks);
-    if (opcode == OP_UPTO || opcode == OP_CRRANGE)
+    compile_char1_matchingpath(common, type, cc, &BACKTRACK_AS(char_iterator_backtrack)->u.backtracks, TRUE);
+    if (opcode == OP_UPTO)
       {
-      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0);
-      OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
-      if (opcode == OP_CRRANGE && arg2 > 0)
-        CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg2, label);
-      if (opcode == OP_UPTO || (opcode == OP_CRRANGE && arg1 > 0))
-        jump = CMP(SLJIT_C_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, arg1);
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE0, TMP1, 0);
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0);
+      OP2(SLJIT_SUB | SLJIT_SET_E, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
+      jump = JUMP(SLJIT_ZERO);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0, TMP1, 0);
       }
 
     /* We cannot use TMP3 because of this allocate_stack. */
@@ -7154,106 +8876,268 @@ switch(opcode)
     }
   else
     {
-    if (opcode == OP_PLUS)
-      compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks);
-    if (private_data_ptr == 0)
-      allocate_stack(common, 2);
-    OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
-    if (opcode <= OP_PLUS)
+    charpos_enabled = FALSE;
+    charpos_char = 0;
+    charpos_othercasebit = 0;
+
+    if ((type != OP_CHAR && type != OP_CHARI) && (*end == OP_CHAR || *end == OP_CHARI))
+      {
+      charpos_enabled = TRUE;
+#ifdef SUPPORT_UTF
+      charpos_enabled = !common->utf || !HAS_EXTRALEN(end[1]);
+#endif
+      if (charpos_enabled && *end == OP_CHARI && char_has_othercase(common, end + 1))
+        {
+        charpos_othercasebit = char_get_othercase_bit(common, end + 1);
+        if (charpos_othercasebit == 0)
+          charpos_enabled = FALSE;
+        }
+
+      if (charpos_enabled)
+        {
+        charpos_char = end[1];
+        /* Consumpe the OP_CHAR opcode. */
+        end += 2;
+#if defined COMPILE_PCRE8
+        SLJIT_ASSERT((charpos_othercasebit >> 8) == 0);
+#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32
+        SLJIT_ASSERT((charpos_othercasebit >> 9) == 0);
+        if ((charpos_othercasebit & 0x100) != 0)
+          charpos_othercasebit = (charpos_othercasebit & 0xff) << 8;
+#endif
+        if (charpos_othercasebit != 0)
+          charpos_char |= charpos_othercasebit;
+
+        BACKTRACK_AS(char_iterator_backtrack)->u.charpos.enabled = TRUE;
+        BACKTRACK_AS(char_iterator_backtrack)->u.charpos.chr = charpos_char;
+        BACKTRACK_AS(char_iterator_backtrack)->u.charpos.othercasebit = charpos_othercasebit;
+        }
+      }
+
+    if (charpos_enabled)
+      {
+      if (opcode == OP_UPTO)
+        OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max + 1);
+
+      /* Search the first instance of charpos_char. */
+      jump = JUMP(SLJIT_JUMP);
+      label = LABEL();
+      if (opcode == OP_UPTO)
+        {
+        OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+        add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_ZERO));
+        }
+      compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, FALSE);
+      if (fast_str_ptr != 0)
+        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0);
+      JUMPHERE(jump);
+
+      detect_partial_match(common, &backtrack->topbacktracks);
+      OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
+      if (charpos_othercasebit != 0)
+        OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, charpos_othercasebit);
+      CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, charpos_char, label);
+
+      if (private_data_ptr == 0)
+        allocate_stack(common, 2);
+      OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
       OP1(SLJIT_MOV, base, offset1, STR_PTR, 0);
-    else
-      OP1(SLJIT_MOV, base, offset1, SLJIT_IMM, 1);
-    label = LABEL();
-    compile_char1_matchingpath(common, type, cc, &nomatch);
-    OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
-    if (opcode <= OP_PLUS)
-      JUMPTO(SLJIT_JUMP, label);
-    else if (opcode == OP_CRRANGE && arg1 == 0)
+      if (opcode == OP_UPTO)
+        {
+        OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+        add_jump(compiler, &no_match, JUMP(SLJIT_ZERO));
+        }
+
+      /* Search the last instance of charpos_char. */
+      label = LABEL();
+      compile_char1_matchingpath(common, type, cc, &no_match, FALSE);
+      if (fast_str_ptr != 0)
+        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0);
+      detect_partial_match(common, &no_match);
+      OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
+      if (charpos_othercasebit != 0)
+        OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, charpos_othercasebit);
+      if (opcode == OP_STAR)
+        {
+        CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, charpos_char, label);
+        OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
+        }
+      else
+        {
+        jump = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, charpos_char);
+        OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
+        JUMPHERE(jump);
+        }
+
+      if (opcode == OP_UPTO)
+        {
+        OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+        JUMPTO(SLJIT_NOT_ZERO, label);
+        }
+      else
+        JUMPTO(SLJIT_JUMP, label);
+
+      set_jumps(no_match, LABEL());
+      OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
+      OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+      OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
+      }
+#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+    else if (common->utf)
       {
-      OP2(SLJIT_ADD, base, offset1, base, offset1, SLJIT_IMM, 1);
-      JUMPTO(SLJIT_JUMP, label);
+      if (private_data_ptr == 0)
+        allocate_stack(common, 2);
+
+      OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
+      OP1(SLJIT_MOV, base, offset1, STR_PTR, 0);
+
+      if (opcode == OP_UPTO)
+        OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max);
+
+      label = LABEL();
+      compile_char1_matchingpath(common, type, cc, &no_match, TRUE);
+      OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
+
+      if (opcode == OP_UPTO)
+        {
+        OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+        JUMPTO(SLJIT_NOT_ZERO, label);
+        }
+      else
+        JUMPTO(SLJIT_JUMP, label);
+
+      set_jumps(no_match, LABEL());
+      OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
+      if (fast_str_ptr != 0)
+        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0);
       }
+#endif
     else
       {
-      OP1(SLJIT_MOV, TMP1, 0, base, offset1);
-      OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
-      OP1(SLJIT_MOV, base, offset1, TMP1, 0);
-      CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg1 + 1, label);
+      if (private_data_ptr == 0)
+        allocate_stack(common, 2);
+
+      OP1(SLJIT_MOV, base, offset1, STR_PTR, 0);
+      if (opcode == OP_UPTO)
+        OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max);
+
+      label = LABEL();
+      detect_partial_match(common, &no_match);
+      compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE);
+      if (opcode == OP_UPTO)
+        {
+        OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+        JUMPTO(SLJIT_NOT_ZERO, label);
+        OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+        }
+      else
+        JUMPTO(SLJIT_JUMP, label);
+
+      set_jumps(no_char1_match, LABEL());
+      OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+      set_jumps(no_match, LABEL());
+      OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
+      if (fast_str_ptr != 0)
+        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0);
       }
-    set_jumps(nomatch, LABEL());
-    if (opcode == OP_CRRANGE)
-      add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_C_LESS, base, offset1, SLJIT_IMM, arg2 + 1));
-    OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
     }
-  BACKTRACK_AS(iterator_backtrack)->matchingpath = LABEL();
+  BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL();
   break;
 
   case OP_MINSTAR:
-  case OP_MINPLUS:
-  if (opcode == OP_MINPLUS)
-    compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks);
   if (private_data_ptr == 0)
     allocate_stack(common, 1);
   OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
-  BACKTRACK_AS(iterator_backtrack)->matchingpath = LABEL();
+  BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL();
+  if (fast_str_ptr != 0)
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0);
   break;
 
   case OP_MINUPTO:
-  case OP_CRMINRANGE:
+  SLJIT_ASSERT(fast_str_ptr == 0);
   if (private_data_ptr == 0)
     allocate_stack(common, 2);
   OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
-  OP1(SLJIT_MOV, base, offset1, SLJIT_IMM, 1);
-  if (opcode == OP_CRMINRANGE)
-    add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_JUMP));
-  BACKTRACK_AS(iterator_backtrack)->matchingpath = LABEL();
+  OP1(SLJIT_MOV, base, offset1, SLJIT_IMM, max + 1);
+  BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL();
   break;
 
   case OP_QUERY:
   case OP_MINQUERY:
+  SLJIT_ASSERT(fast_str_ptr == 0);
   if (private_data_ptr == 0)
     allocate_stack(common, 1);
   OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
   if (opcode == OP_QUERY)
-    compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks);
-  BACKTRACK_AS(iterator_backtrack)->matchingpath = LABEL();
+    compile_char1_matchingpath(common, type, cc, &BACKTRACK_AS(char_iterator_backtrack)->u.backtracks, TRUE);
+  BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL();
   break;
 
   case OP_EXACT:
-  OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, arg1);
-  label = LABEL();
-  compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks);
-  OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
-  JUMPTO(SLJIT_C_NOT_ZERO, label);
   break;
 
   case OP_POSSTAR:
-  case OP_POSPLUS:
-  case OP_POSUPTO:
-  if (opcode == OP_POSPLUS)
-    compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks);
-  if (opcode == OP_POSUPTO)
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, SLJIT_IMM, arg1);
-  OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0);
-  label = LABEL();
-  compile_char1_matchingpath(common, type, cc, &nomatch);
-  OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0);
-  if (opcode != OP_POSUPTO)
+#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+  if (common->utf)
+    {
+    OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0);
+    label = LABEL();
+    compile_char1_matchingpath(common, type, cc, &no_match, TRUE);
+    OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0);
     JUMPTO(SLJIT_JUMP, label);
-  else
+    set_jumps(no_match, LABEL());
+    OP1(SLJIT_MOV, STR_PTR, 0, tmp_base, tmp_offset);
+    if (fast_str_ptr != 0)
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0);
+    break;
+    }
+#endif
+  label = LABEL();
+  detect_partial_match(common, &no_match);
+  compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE);
+  JUMPTO(SLJIT_JUMP, label);
+  set_jumps(no_char1_match, LABEL());
+  OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+  set_jumps(no_match, LABEL());
+  if (fast_str_ptr != 0)
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0);
+  break;
+
+  case OP_POSUPTO:
+  SLJIT_ASSERT(fast_str_ptr == 0);
+#if defined SUPPORT_UTF && !defined COMPILE_PCRE32
+  if (common->utf)
     {
-    OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, SLJIT_MEM1(SLJIT_LOCALS_REG), POSSESSIVE1, SLJIT_IMM, 1);
-    JUMPTO(SLJIT_C_NOT_ZERO, label);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, STR_PTR, 0);
+    OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max);
+    label = LABEL();
+    compile_char1_matchingpath(common, type, cc, &no_match, TRUE);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, STR_PTR, 0);
+    OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+    JUMPTO(SLJIT_NOT_ZERO, label);
+    set_jumps(no_match, LABEL());
+    OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1);
+    break;
     }
-  set_jumps(nomatch, LABEL());
-  OP1(SLJIT_MOV, STR_PTR, 0, tmp_base, tmp_offset);
+#endif
+  OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max);
+  label = LABEL();
+  detect_partial_match(common, &no_match);
+  compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE);
+  OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+  JUMPTO(SLJIT_NOT_ZERO, label);
+  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+  set_jumps(no_char1_match, LABEL());
+  OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+  set_jumps(no_match, LABEL());
   break;
 
   case OP_POSQUERY:
+  SLJIT_ASSERT(fast_str_ptr == 0);
   OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0);
-  compile_char1_matchingpath(common, type, cc, &nomatch);
+  compile_char1_matchingpath(common, type, cc, &no_match, TRUE);
   OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0);
-  set_jumps(nomatch, LABEL());
+  set_jumps(no_match, LABEL());
   OP1(SLJIT_MOV, STR_PTR, 0, tmp_base, tmp_offset);
   break;
 
@@ -7279,7 +9163,7 @@ if (*cc == OP_FAIL)
   return cc + 1;
   }
 
-if (*cc == OP_ASSERT_ACCEPT || common->currententry != NULL)
+if (*cc == OP_ASSERT_ACCEPT || common->currententry != NULL || !common->might_be_empty)
   {
   /* No need to check notempty conditions. */
   if (common->accept_label == NULL)
@@ -7290,22 +9174,22 @@ if (*cc == OP_ASSERT_ACCEPT || common->currententry != NULL)
   }
 
 if (common->accept_label == NULL)
-  add_jump(compiler, &common->accept, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0)));
+  add_jump(compiler, &common->accept, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)));
 else
-  CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), common->accept_label);
+  CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), common->accept_label);
 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
-OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty));
-add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
-OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty_atstart));
+OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty));
+add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
+OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty_atstart));
 if (common->accept_label == NULL)
-  add_jump(compiler, &common->accept, CMP(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, 0));
+  add_jump(compiler, &common->accept, CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, 0));
 else
-  CMPTO(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, 0, common->accept_label);
+  CMPTO(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, 0, common->accept_label);
 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
 if (common->accept_label == NULL)
-  add_jump(compiler, &common->accept, CMP(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_PTR, 0));
+  add_jump(compiler, &common->accept, CMP(SLJIT_NOT_EQUAL, TMP2, 0, STR_PTR, 0));
 else
-  CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_PTR, 0, common->accept_label);
+  CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, STR_PTR, 0, common->accept_label);
 add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_JUMP));
 return cc + 1;
 }
@@ -7321,11 +9205,11 @@ if (common->currententry != NULL)
   return cc + 1 + IMM2_SIZE;
 
 if (!optimized_cbracket)
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR_PRIV(offset));
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR_PRIV(offset));
 offset <<= 1;
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0);
 if (!optimized_cbracket)
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0);
 return cc + 1 + IMM2_SIZE;
 }
 
@@ -7352,7 +9236,7 @@ if (opcode == OP_PRUNE_ARG || opcode == OP_THEN_ARG)
   {
   OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
   OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)(cc + 2));
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->mark_ptr, TMP2, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, TMP2, 0);
   OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, mark_ptr), TMP2, 0);
   }
 
@@ -7377,12 +9261,12 @@ BACKTRACK_AS(then_trap_backtrack)->framesize = get_framesize(common, cc, ccend,
 size = BACKTRACK_AS(then_trap_backtrack)->framesize;
 size = 3 + (size < 0 ? 0 : size);
 
-OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr);
+OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr);
 allocate_stack(common, size);
 if (size > 3)
-  OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, STACK_TOP, 0, SLJIT_IMM, (size - 3) * sizeof(sljit_sw));
+  OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0, SLJIT_IMM, (size - 3) * sizeof(sljit_sw));
 else
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, STACK_TOP, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0);
 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(size - 1), SLJIT_IMM, BACKTRACK_AS(then_trap_backtrack)->start);
 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(size - 2), SLJIT_IMM, type_then_trap);
 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(size - 3), TMP2, 0);
@@ -7418,6 +9302,16 @@ while (cc < ccend)
     case OP_SOM:
     case OP_NOT_WORD_BOUNDARY:
     case OP_WORD_BOUNDARY:
+    case OP_EODN:
+    case OP_EOD:
+    case OP_DOLL:
+    case OP_DOLLM:
+    case OP_CIRC:
+    case OP_CIRCM:
+    case OP_REVERSE:
+    cc = compile_simple_assertion_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks);
+    break;
+
     case OP_NOT_DIGIT:
     case OP_DIGIT:
     case OP_NOT_WHITESPACE:
@@ -7435,23 +9329,16 @@ while (cc < ccend)
     case OP_NOT_VSPACE:
     case OP_VSPACE:
     case OP_EXTUNI:
-    case OP_EODN:
-    case OP_EOD:
-    case OP_CIRC:
-    case OP_CIRCM:
-    case OP_DOLL:
-    case OP_DOLLM:
     case OP_NOT:
     case OP_NOTI:
-    case OP_REVERSE:
-    cc = compile_char1_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks);
+    cc = compile_char1_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE);
     break;
 
     case OP_SET_SOM:
     PUSH_BACKTRACK_NOVALUE(sizeof(backtrack_common), cc);
-    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
+    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0));
     allocate_stack(common, 1);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), STR_PTR, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), STR_PTR, 0);
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0);
     cc++;
     break;
@@ -7461,7 +9348,7 @@ while (cc < ccend)
     if (common->mode == JIT_COMPILE)
       cc = compile_charn_matchingpath(common, cc, ccend, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks);
     else
-      cc = compile_char1_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks);
+      cc = compile_char1_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE);
     break;
 
     case OP_STAR:
@@ -7534,27 +9421,42 @@ while (cc < ccend)
 
     case OP_CLASS:
     case OP_NCLASS:
-    if (cc[1 + (32 / sizeof(pcre_uchar))] >= OP_CRSTAR && cc[1 + (32 / sizeof(pcre_uchar))] <= OP_CRMINRANGE)
+    if (cc[1 + (32 / sizeof(pcre_uchar))] >= OP_CRSTAR && cc[1 + (32 / sizeof(pcre_uchar))] <= OP_CRPOSRANGE)
       cc = compile_iterator_matchingpath(common, cc, parent);
     else
-      cc = compile_char1_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks);
+      cc = compile_char1_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE);
     break;
 
 #if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32
     case OP_XCLASS:
-    if (*(cc + GET(cc, 1)) >= OP_CRSTAR && *(cc + GET(cc, 1)) <= OP_CRMINRANGE)
+    if (*(cc + GET(cc, 1)) >= OP_CRSTAR && *(cc + GET(cc, 1)) <= OP_CRPOSRANGE)
       cc = compile_iterator_matchingpath(common, cc, parent);
     else
-      cc = compile_char1_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks);
+      cc = compile_char1_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE);
     break;
 #endif
 
     case OP_REF:
     case OP_REFI:
-    if (cc[1 + IMM2_SIZE] >= OP_CRSTAR && cc[1 + IMM2_SIZE] <= OP_CRMINRANGE)
+    if (cc[1 + IMM2_SIZE] >= OP_CRSTAR && cc[1 + IMM2_SIZE] <= OP_CRPOSRANGE)
+      cc = compile_ref_iterator_matchingpath(common, cc, parent);
+    else
+      {
+      compile_ref_matchingpath(common, cc, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE, FALSE);
+      cc += 1 + IMM2_SIZE;
+      }
+    break;
+
+    case OP_DNREF:
+    case OP_DNREFI:
+    if (cc[1 + 2 * IMM2_SIZE] >= OP_CRSTAR && cc[1 + 2 * IMM2_SIZE] <= OP_CRPOSRANGE)
       cc = compile_ref_iterator_matchingpath(common, cc, parent);
     else
-      cc = compile_ref_matchingpath(common, cc, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE, FALSE);
+      {
+      compile_dnref_search(common, cc, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks);
+      compile_ref_matchingpath(common, cc, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE, FALSE);
+      cc += 1 + 2 * IMM2_SIZE;
+      }
     break;
 
     case OP_RECURSE:
@@ -7588,8 +9490,7 @@ while (cc < ccend)
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), STR_PTR, 0);
       }
     BACKTRACK_AS(braminzero_backtrack)->matchingpath = LABEL();
-    if (cc[1] > OP_ASSERTBACK_NOT)
-      count_match(common);
+    count_match(common);
     break;
 
     case OP_ONCE:
@@ -7624,17 +9525,17 @@ while (cc < ccend)
     case OP_MARK:
     PUSH_BACKTRACK_NOVALUE(sizeof(backtrack_common), cc);
     SLJIT_ASSERT(common->mark_ptr != 0);
-    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->mark_ptr);
+    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr);
     allocate_stack(common, common->has_skip_arg ? 5 : 1);
     OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(common->has_skip_arg ? 4 : 0), TMP2, 0);
     OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)(cc + 2));
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->mark_ptr, TMP2, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, TMP2, 0);
     OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, mark_ptr), TMP2, 0);
     if (common->has_skip_arg)
       {
-      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr);
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, STACK_TOP, 0);
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, type_mark);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), SLJIT_IMM, (sljit_sw)(cc + 2));
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(3), STR_PTR, 0);
@@ -7707,95 +9608,82 @@ DEFINE_COMPILER;
 pcre_uchar *cc = current->cc;
 pcre_uchar opcode;
 pcre_uchar type;
-int arg1 = -1, arg2 = -1;
+sljit_u32 max = 0, exact;
 struct sljit_label *label = NULL;
 struct sljit_jump *jump = NULL;
 jump_list *jumplist = NULL;
+pcre_uchar *end;
 int private_data_ptr = PRIVATE_DATA(cc);
-int base = (private_data_ptr == 0) ? SLJIT_MEM1(STACK_TOP) : SLJIT_MEM1(SLJIT_LOCALS_REG);
+int base = (private_data_ptr == 0) ? SLJIT_MEM1(STACK_TOP) : SLJIT_MEM1(SLJIT_SP);
 int offset0 = (private_data_ptr == 0) ? STACK(0) : private_data_ptr;
 int offset1 = (private_data_ptr == 0) ? STACK(1) : private_data_ptr + (int)sizeof(sljit_sw);
 
-cc = get_iterator_parameters(common, cc, &opcode, &type, &arg1, &arg2, NULL);
+cc = get_iterator_parameters(common, cc, &opcode, &type, &max, &exact, &end);
 
 switch(opcode)
   {
   case OP_STAR:
-  case OP_PLUS:
   case OP_UPTO:
-  case OP_CRRANGE:
   if (type == OP_ANYNL || type == OP_EXTUNI)
     {
     SLJIT_ASSERT(private_data_ptr == 0);
-    set_jumps(current->topbacktracks, LABEL());
+    set_jumps(CURRENT_AS(char_iterator_backtrack)->u.backtracks, LABEL());
     OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
     free_stack(common, 1);
-    CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(iterator_backtrack)->matchingpath);
+    CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(char_iterator_backtrack)->matchingpath);
     }
   else
     {
-    if (opcode == OP_UPTO)
-      arg2 = 0;
-    if (opcode <= OP_PLUS)
+    if (CURRENT_AS(char_iterator_backtrack)->u.charpos.enabled)
       {
       OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
-      jump = CMP(SLJIT_C_LESS_EQUAL, STR_PTR, 0, base, offset1);
+      OP1(SLJIT_MOV, TMP2, 0, base, offset1);
+      OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+
+      jump = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0);
+      label = LABEL();
+      OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1));
+      OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
+      if (CURRENT_AS(char_iterator_backtrack)->u.charpos.othercasebit != 0)
+        OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, CURRENT_AS(char_iterator_backtrack)->u.charpos.othercasebit);
+      CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CURRENT_AS(char_iterator_backtrack)->u.charpos.chr, CURRENT_AS(char_iterator_backtrack)->matchingpath);
+      skip_char_back(common);
+      CMPTO(SLJIT_GREATER, STR_PTR, 0, TMP2, 0, label);
       }
     else
       {
-      OP1(SLJIT_MOV, TMP1, 0, base, offset1);
       OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
-      jump = CMP(SLJIT_C_LESS_EQUAL, TMP1, 0, SLJIT_IMM, arg2 + 1);
-      OP2(SLJIT_SUB, base, offset1, TMP1, 0, SLJIT_IMM, 1);
+      jump = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, base, offset1);
+      skip_char_back(common);
+      OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
+      JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath);
       }
-    skip_char_back(common);
-    OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
-    JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_backtrack)->matchingpath);
-    if (opcode == OP_CRRANGE)
-      set_jumps(current->topbacktracks, LABEL());
     JUMPHERE(jump);
     if (private_data_ptr == 0)
       free_stack(common, 2);
-    if (opcode == OP_PLUS)
-      set_jumps(current->topbacktracks, LABEL());
     }
   break;
 
   case OP_MINSTAR:
-  case OP_MINPLUS:
   OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
-  compile_char1_matchingpath(common, type, cc, &jumplist);
+  compile_char1_matchingpath(common, type, cc, &jumplist, TRUE);
   OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
-  JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_backtrack)->matchingpath);
+  JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath);
   set_jumps(jumplist, LABEL());
   if (private_data_ptr == 0)
     free_stack(common, 1);
-  if (opcode == OP_MINPLUS)
-    set_jumps(current->topbacktracks, LABEL());
   break;
 
   case OP_MINUPTO:
-  case OP_CRMINRANGE:
-  if (opcode == OP_CRMINRANGE)
-    {
-    label = LABEL();
-    set_jumps(current->topbacktracks, label);
-    }
+  OP1(SLJIT_MOV, TMP1, 0, base, offset1);
   OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
-  compile_char1_matchingpath(common, type, cc, &jumplist);
+  OP2(SLJIT_SUB | SLJIT_SET_E, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
+  add_jump(compiler, &jumplist, JUMP(SLJIT_ZERO));
 
-  OP1(SLJIT_MOV, TMP1, 0, base, offset1);
-  OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
-  OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
   OP1(SLJIT_MOV, base, offset1, TMP1, 0);
-
-  if (opcode == OP_CRMINRANGE)
-    CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg2 + 1, label);
-
-  if (opcode == OP_CRMINRANGE && arg1 == 0)
-    JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_backtrack)->matchingpath);
-  else
-    CMPTO(SLJIT_C_LESS, TMP1, 0, SLJIT_IMM, arg1 + 2, CURRENT_AS(iterator_backtrack)->matchingpath);
+  compile_char1_matchingpath(common, type, cc, &jumplist, TRUE);
+  OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
+  JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath);
 
   set_jumps(jumplist, LABEL());
   if (private_data_ptr == 0)
@@ -7805,12 +9693,12 @@ switch(opcode)
   case OP_QUERY:
   OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
   OP1(SLJIT_MOV, base, offset0, SLJIT_IMM, 0);
-  CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(iterator_backtrack)->matchingpath);
+  CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(char_iterator_backtrack)->matchingpath);
   jump = JUMP(SLJIT_JUMP);
-  set_jumps(current->topbacktracks, LABEL());
+  set_jumps(CURRENT_AS(char_iterator_backtrack)->u.backtracks, LABEL());
   OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
   OP1(SLJIT_MOV, base, offset0, SLJIT_IMM, 0);
-  JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_backtrack)->matchingpath);
+  JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath);
   JUMPHERE(jump);
   if (private_data_ptr == 0)
     free_stack(common, 1);
@@ -7819,9 +9707,9 @@ switch(opcode)
   case OP_MINQUERY:
   OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
   OP1(SLJIT_MOV, base, offset0, SLJIT_IMM, 0);
-  jump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
-  compile_char1_matchingpath(common, type, cc, &jumplist);
-  JUMPTO(SLJIT_JUMP, CURRENT_AS(iterator_backtrack)->matchingpath);
+  jump = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
+  compile_char1_matchingpath(common, type, cc, &jumplist, TRUE);
+  JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath);
   set_jumps(jumplist, LABEL());
   JUMPHERE(jump);
   if (private_data_ptr == 0)
@@ -7829,10 +9717,6 @@ switch(opcode)
   break;
 
   case OP_EXACT:
-  case OP_POSPLUS:
-  set_jumps(current->topbacktracks, LABEL());
-  break;
-
   case OP_POSSTAR:
   case OP_POSQUERY:
   case OP_POSUPTO:
@@ -7842,28 +9726,33 @@ switch(opcode)
   SLJIT_ASSERT_STOP();
   break;
   }
+
+set_jumps(current->topbacktracks, LABEL());
 }
 
 static SLJIT_INLINE void compile_ref_iterator_backtrackingpath(compiler_common *common, struct backtrack_common *current)
 {
 DEFINE_COMPILER;
 pcre_uchar *cc = current->cc;
+BOOL ref = (*cc == OP_REF || *cc == OP_REFI);
 pcre_uchar type;
 
-type = cc[1 + IMM2_SIZE];
+type = cc[ref ? 1 + IMM2_SIZE : 1 + 2 * IMM2_SIZE];
+
 if ((type & 0x1) == 0)
   {
+  /* Maximize case. */
   set_jumps(current->topbacktracks, LABEL());
   OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
   free_stack(common, 1);
-  CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(iterator_backtrack)->matchingpath);
+  CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(ref_iterator_backtrack)->matchingpath);
   return;
   }
 
 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
-CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(iterator_backtrack)->matchingpath);
+CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(ref_iterator_backtrack)->matchingpath);
 set_jumps(current->topbacktracks, LABEL());
-free_stack(common, 2);
+free_stack(common, ref ? 2 : 3);
 }
 
 static SLJIT_INLINE void compile_recurse_backtrackingpath(compiler_common *common, struct backtrack_common *current)
@@ -7881,14 +9770,14 @@ if (common->has_set_som && common->mark_ptr != 0)
   OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
   OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
   free_stack(common, 2);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), TMP2, 0);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->mark_ptr, TMP1, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), TMP2, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, TMP1, 0);
   }
 else if (common->has_set_som || common->mark_ptr != 0)
   {
   OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
   free_stack(common, 1);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->has_set_som ? (int)(OVECTOR(0)) : common->mark_ptr, TMP2, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->has_set_som ? (int)(OVECTOR(0)) : common->mark_ptr, TMP2, 0);
   }
 }
 
@@ -7919,7 +9808,7 @@ if (CURRENT_AS(assert_backtrack)->framesize < 0)
   if (bra == OP_BRAZERO)
     {
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
-    CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(assert_backtrack)->matchingpath);
+    CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(assert_backtrack)->matchingpath);
     free_stack(common, 1);
     }
   return;
@@ -7930,19 +9819,19 @@ if (bra == OP_BRAZERO)
   if (*cc == OP_ASSERT_NOT || *cc == OP_ASSERTBACK_NOT)
     {
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
-    CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(assert_backtrack)->matchingpath);
+    CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(assert_backtrack)->matchingpath);
     free_stack(common, 1);
     return;
     }
   free_stack(common, 1);
-  brajump = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
+  brajump = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
   }
 
 if (*cc == OP_ASSERT || *cc == OP_ASSERTBACK)
   {
-  OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(assert_backtrack)->private_data_ptr);
+  OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(assert_backtrack)->private_data_ptr);
   add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(assert_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(assert_backtrack)->framesize * sizeof(sljit_sw));
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(assert_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(assert_backtrack)->framesize * sizeof(sljit_sw));
 
   set_jumps(current->topbacktracks, LABEL());
   }
@@ -7962,21 +9851,22 @@ if (bra == OP_BRAZERO)
 static void compile_bracket_backtrackingpath(compiler_common *common, struct backtrack_common *current)
 {
 DEFINE_COMPILER;
-int opcode, stacksize, count;
+int opcode, stacksize, alt_count, alt_max;
 int offset = 0;
 int private_data_ptr = CURRENT_AS(bracket_backtrack)->private_data_ptr;
 int repeat_ptr = 0, repeat_type = 0, repeat_count = 0;
 pcre_uchar *cc = current->cc;
 pcre_uchar *ccbegin;
 pcre_uchar *ccprev;
-jump_list *jumplist = NULL;
-jump_list *jumplistitem = NULL;
 pcre_uchar bra = OP_BRA;
 pcre_uchar ket;
 assert_backtrack *assert;
+sljit_uw *next_update_addr = NULL;
 BOOL has_alternatives;
 BOOL needs_control_head = FALSE;
 struct sljit_jump *brazero = NULL;
+struct sljit_jump *alt1 = NULL;
+struct sljit_jump *alt2 = NULL;
 struct sljit_jump *once = NULL;
 struct sljit_jump *cond = NULL;
 struct sljit_label *rmin_label = NULL;
@@ -8014,6 +9904,8 @@ if (SLJIT_UNLIKELY(opcode == OP_COND) && (*cc == OP_KETRMAX || *cc == OP_KETRMIN
 if (SLJIT_UNLIKELY(opcode == OP_ONCE_NC))
   opcode = OP_ONCE;
 
+alt_max = has_alternatives ? no_alternatives(ccbegin) : 0;
+
 /* Decoding the needs_control_head in framesize. */
 if (opcode == OP_ONCE)
   {
@@ -8027,9 +9919,9 @@ if (ket != OP_KET && repeat_type != 0)
   OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
   free_stack(common, 1);
   if (repeat_type == OP_UPTO)
-    OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_LOCALS_REG), repeat_ptr, TMP1, 0, SLJIT_IMM, 1);
+    OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), repeat_ptr, TMP1, 0, SLJIT_IMM, 1);
   else
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), repeat_ptr, TMP1, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), repeat_ptr, TMP1, 0);
   }
 
 if (ket == OP_KETRMAX)
@@ -8038,7 +9930,7 @@ if (ket == OP_KETRMAX)
     {
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
     free_stack(common, 1);
-    brazero = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 0);
+    brazero = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0);
     }
   }
 else if (ket == OP_KETRMIN)
@@ -8049,7 +9941,7 @@ else if (ket == OP_KETRMIN)
     if (repeat_type != 0)
       {
       /* TMP1 was set a few lines above. */
-      CMPTO(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0, CURRENT_AS(bracket_backtrack)->recursive_matchingpath);
+      CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0, CURRENT_AS(bracket_backtrack)->recursive_matchingpath);
       /* Drop STR_PTR for non-greedy plus quantifier. */
       if (opcode != OP_ONCE)
         free_stack(common, 1);
@@ -8058,11 +9950,11 @@ else if (ket == OP_KETRMIN)
       {
       /* Checking zero-length iteration. */
       if (opcode != OP_ONCE || CURRENT_AS(bracket_backtrack)->u.framesize < 0)
-        CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, CURRENT_AS(bracket_backtrack)->recursive_matchingpath);
+        CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, CURRENT_AS(bracket_backtrack)->recursive_matchingpath);
       else
         {
-        OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
-        CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (CURRENT_AS(bracket_backtrack)->u.framesize + 1) * sizeof(sljit_sw), CURRENT_AS(bracket_backtrack)->recursive_matchingpath);
+        OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
+        CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (CURRENT_AS(bracket_backtrack)->u.framesize + 1) * sizeof(sljit_sw), CURRENT_AS(bracket_backtrack)->recursive_matchingpath);
         }
       /* Drop STR_PTR for non-greedy plus quantifier. */
       if (opcode != OP_ONCE)
@@ -8073,17 +9965,17 @@ else if (ket == OP_KETRMIN)
     }
   rmin_label = LABEL();
   if (repeat_type != 0)
-    OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_LOCALS_REG), repeat_ptr, SLJIT_MEM1(SLJIT_LOCALS_REG), repeat_ptr, SLJIT_IMM, 1);
+    OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1);
   }
 else if (bra == OP_BRAZERO)
   {
   OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
   free_stack(common, 1);
-  brazero = CMP(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0);
+  brazero = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0);
   }
 else if (repeat_type == OP_EXACT)
   {
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), repeat_ptr, SLJIT_IMM, 1);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1);
   exact_label = LABEL();
   }
 
@@ -8094,19 +9986,19 @@ if (offset != 0)
     SLJIT_ASSERT(common->optimized_cbracket[offset >> 1] == 0);
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
     OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->capture_last_ptr, TMP1, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, TMP1, 0);
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(2));
     free_stack(common, 3);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP2, 0);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), TMP1, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP2, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP1, 0);
     }
   else if (common->optimized_cbracket[offset >> 1] == 0)
     {
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
     OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
     free_stack(common, 2);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), TMP2, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP2, 0);
     }
   }
 
@@ -8114,7 +10006,7 @@ if (SLJIT_UNLIKELY(opcode == OP_ONCE))
   {
   if (CURRENT_AS(bracket_backtrack)->u.framesize >= 0)
     {
-    OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+    OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
     add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
     }
   once = JUMP(SLJIT_JUMP);
@@ -8127,44 +10019,30 @@ else if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND)
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
     free_stack(common, 1);
 
-    jumplistitem = sljit_alloc_memory(compiler, sizeof(jump_list));
-    if (SLJIT_UNLIKELY(!jumplistitem))
-      return;
-    jumplist = jumplistitem;
-    jumplistitem->next = NULL;
-    jumplistitem->jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, 1);
+    alt_max = 2;
+    alt1 = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, sizeof(sljit_uw));
     }
   }
-else if (*cc == OP_ALT)
+else if (has_alternatives)
   {
-  /* Build a jump list. Get the last successfully matched branch index. */
   OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
   free_stack(common, 1);
-  count = 1;
-  do
-    {
-    /* Append as the last item. */
-    if (jumplist != NULL)
-      {
-      jumplistitem->next = sljit_alloc_memory(compiler, sizeof(jump_list));
-      jumplistitem = jumplistitem->next;
-      }
-    else
-      {
-      jumplistitem = sljit_alloc_memory(compiler, sizeof(jump_list));
-      jumplist = jumplistitem;
-      }
 
-    if (SLJIT_UNLIKELY(!jumplistitem))
+  if (alt_max > 4)
+    {
+    /* Table jump if alt_max is greater than 4. */
+    next_update_addr = allocate_read_only_data(common, alt_max * sizeof(sljit_uw));
+    if (SLJIT_UNLIKELY(next_update_addr == NULL))
       return;
-
-    jumplistitem->next = NULL;
-    jumplistitem->jump = CMP(SLJIT_C_EQUAL, TMP1, 0, SLJIT_IMM, count++);
-    cc += GET(cc, 1);
+    sljit_emit_ijump(compiler, SLJIT_JUMP, SLJIT_MEM1(TMP1), (sljit_sw)next_update_addr);
+    add_label_addr(common, next_update_addr++);
+    }
+  else
+    {
+    if (alt_max == 4)
+      alt2 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_uw));
+    alt1 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, sizeof(sljit_uw));
     }
-  while (*cc == OP_ALT);
-
-  cc = ccbegin + GET(ccbegin, 1);
   }
 
 COMPILE_BACKTRACKINGPATH(current->top);
@@ -8180,9 +10058,9 @@ if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND))
     assert = CURRENT_AS(bracket_backtrack)->u.assert;
     if (assert->framesize >= 0 && (ccbegin[1 + LINK_SIZE] == OP_ASSERT || ccbegin[1 + LINK_SIZE] == OP_ASSERTBACK))
       {
-      OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), assert->private_data_ptr);
+      OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr);
       add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_sw));
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_sw));
       }
     cond = JUMP(SLJIT_JUMP);
     set_jumps(CURRENT_AS(bracket_backtrack)->u.assert->condfailed, LABEL());
@@ -8199,7 +10077,7 @@ if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND))
 
 if (has_alternatives)
   {
-  count = 1;
+  alt_count = sizeof(sljit_uw);
   do
     {
     current->top = NULL;
@@ -8215,7 +10093,7 @@ if (has_alternatives)
         if (opcode != OP_ONCE)
           {
           if (private_data_ptr != 0)
-            OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr);
+            OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
           else
             OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
           }
@@ -8236,7 +10114,7 @@ if (has_alternatives)
     if (repeat_type == OP_MINUPTO)
       {
       /* We need to preserve the counter. TMP2 will be used below. */
-      OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), repeat_ptr);
+      OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), repeat_ptr);
       stacksize++;
       }
     if (ket != OP_KET || bra != OP_BRA)
@@ -8275,22 +10153,37 @@ if (has_alternatives)
       stacksize = match_capture_common(common, stacksize, offset, private_data_ptr);
 
     if (opcode != OP_ONCE)
-      OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, count++);
+      OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, alt_count);
 
     if (offset != 0 && ket == OP_KETRMAX && common->optimized_cbracket[offset >> 1] != 0)
       {
       /* If ket is not OP_KETRMAX, this code path is executed after the jump to alternative_matchingpath. */
       SLJIT_ASSERT(private_data_ptr == OVECTOR(offset + 0));
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), STR_PTR, 0);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0);
       }
 
     JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_backtrack)->alternative_matchingpath);
 
     if (opcode != OP_ONCE)
       {
-      SLJIT_ASSERT(jumplist);
-      JUMPHERE(jumplist->jump);
-      jumplist = jumplist->next;
+      if (alt_max > 4)
+        add_label_addr(common, next_update_addr++);
+      else
+        {
+        if (alt_count != 2 * sizeof(sljit_uw))
+          {
+          JUMPHERE(alt1);
+          if (alt_max == 3 && alt_count == sizeof(sljit_uw))
+            alt2 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_uw));
+          }
+        else
+          {
+          JUMPHERE(alt2);
+          if (alt_max == 4)
+            alt1 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 3 * sizeof(sljit_uw));
+          }
+        }
+      alt_count += sizeof(sljit_uw);
       }
 
     COMPILE_BACKTRACKINGPATH(current->top);
@@ -8299,7 +10192,6 @@ if (has_alternatives)
     SLJIT_ASSERT(!current->nextbacktracks);
     }
   while (*cc == OP_ALT);
-  SLJIT_ASSERT(!jumplist);
 
   if (cond != NULL)
     {
@@ -8307,9 +10199,9 @@ if (has_alternatives)
     assert = CURRENT_AS(bracket_backtrack)->u.assert;
     if ((ccbegin[1 + LINK_SIZE] == OP_ASSERT_NOT || ccbegin[1 + LINK_SIZE] == OP_ASSERTBACK_NOT) && assert->framesize >= 0)
       {
-      OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), assert->private_data_ptr);
+      OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr);
       add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_sw));
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_sw));
       }
     JUMPHERE(cond);
     }
@@ -8327,19 +10219,19 @@ if (offset != 0)
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
     OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
     free_stack(common, 2);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), TMP2, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP2, 0);
     }
   else
     {
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
     free_stack(common, 1);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, TMP1, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0);
     }
   }
 else if (opcode == OP_SBRA || opcode == OP_SCOND)
   {
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(0));
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(0));
   free_stack(common, 1);
   }
 else if (opcode == OP_ONCE)
@@ -8357,26 +10249,28 @@ else if (opcode == OP_ONCE)
     /* The STR_PTR must be released. */
     stacksize++;
     }
-  free_stack(common, stacksize);
+
+  if (stacksize > 0)
+    free_stack(common, stacksize);
 
   JUMPHERE(once);
   /* Restore previous private_data_ptr */
   if (CURRENT_AS(bracket_backtrack)->u.framesize >= 0)
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracket_backtrack)->u.framesize * sizeof(sljit_sw));
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracket_backtrack)->u.framesize * sizeof(sljit_sw));
   else if (ket == OP_KETRMIN)
     {
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
     /* See the comment below. */
     free_stack(common, 2);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), private_data_ptr, TMP1, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0);
     }
   }
 
 if (repeat_type == OP_EXACT)
   {
-  OP2(SLJIT_ADD, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), repeat_ptr, SLJIT_IMM, 1);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), repeat_ptr, TMP1, 0);
-  CMPTO(SLJIT_C_LESS_EQUAL, TMP1, 0, SLJIT_IMM, repeat_count, exact_label);
+  OP2(SLJIT_ADD, TMP1, 0, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), repeat_ptr, TMP1, 0);
+  CMPTO(SLJIT_LESS_EQUAL, TMP1, 0, SLJIT_IMM, repeat_count, exact_label);
   }
 else if (ket == OP_KETRMAX)
   {
@@ -8384,7 +10278,7 @@ else if (ket == OP_KETRMAX)
   if (bra != OP_BRAZERO)
     free_stack(common, 1);
 
-  CMPTO(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(bracket_backtrack)->recursive_matchingpath);
+  CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(bracket_backtrack)->recursive_matchingpath);
   if (bra == OP_BRAZERO)
     {
     OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
@@ -8402,7 +10296,7 @@ else if (ket == OP_KETRMIN)
   affect badly the free_stack(2) above. */
   if (opcode != OP_ONCE)
     free_stack(common, 1);
-  CMPTO(SLJIT_C_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0, rmin_label);
+  CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0, rmin_label);
   if (opcode == OP_ONCE)
     free_stack(common, bra == OP_BRAMINZERO ? 2 : 1);
   else if (bra == OP_BRAMINZERO)
@@ -8429,19 +10323,19 @@ if (CURRENT_AS(bracketpos_backtrack)->framesize < 0)
     offset = (GET2(current->cc, 1 + LINK_SIZE)) << 1;
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
     OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset), TMP1, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0);
     if (common->capture_last_ptr != 0)
       OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(2));
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(offset + 1), TMP2, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP2, 0);
     if (common->capture_last_ptr != 0)
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->capture_last_ptr, TMP1, 0);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, TMP1, 0);
     }
   set_jumps(current->topbacktracks, LABEL());
   free_stack(common, CURRENT_AS(bracketpos_backtrack)->stacksize);
   return;
   }
 
-OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(bracketpos_backtrack)->private_data_ptr);
+OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(bracketpos_backtrack)->private_data_ptr);
 add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
 
 if (current->topbacktracks)
@@ -8452,7 +10346,7 @@ if (current->topbacktracks)
   free_stack(common, CURRENT_AS(bracketpos_backtrack)->stacksize);
   JUMPHERE(jump);
   }
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), CURRENT_AS(bracketpos_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracketpos_backtrack)->framesize * sizeof(sljit_sw));
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(bracketpos_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracketpos_backtrack)->framesize * sizeof(sljit_sw));
 }
 
 static SLJIT_INLINE void compile_braminzero_backtrackingpath(compiler_common *common, struct backtrack_common *current)
@@ -8492,7 +10386,7 @@ if (opcode == OP_THEN || opcode == OP_THEN_ARG)
     {
     SLJIT_ASSERT(common->control_head_ptr != 0);
 
-    OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr);
+    OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr);
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, type_then_trap);
     OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, common->then_trap->start);
     jump = JUMP(SLJIT_JUMP);
@@ -8500,8 +10394,8 @@ if (opcode == OP_THEN || opcode == OP_THEN_ARG)
     loop = LABEL();
     OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), -(int)sizeof(sljit_sw));
     JUMPHERE(jump);
-    CMPTO(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), -(int)(2 * sizeof(sljit_sw)), TMP1, 0, loop);
-    CMPTO(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), -(int)(3 * sizeof(sljit_sw)), TMP2, 0, loop);
+    CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), -(int)(2 * sizeof(sljit_sw)), TMP1, 0, loop);
+    CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), -(int)(3 * sizeof(sljit_sw)), TMP2, 0, loop);
     add_jump(compiler, &common->then_trap->quit, JUMP(SLJIT_JUMP));
     return;
     }
@@ -8524,14 +10418,14 @@ if (common->local_exit)
 if (opcode == OP_SKIP_ARG)
   {
   SLJIT_ASSERT(common->control_head_ptr != 0);
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0, STACK_TOP, 0);
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0);
   OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_IMM, (sljit_sw)(current->cc + 2));
   sljit_emit_ijump(compiler, SLJIT_CALL2, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_search_mark));
-  OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
+  OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
 
   OP1(SLJIT_MOV, STR_PTR, 0, TMP1, 0);
-  add_jump(compiler, &common->reset_match, CMP(SLJIT_C_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, -1));
+  add_jump(compiler, &common->reset_match, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, -1));
   return;
   }
 
@@ -8569,7 +10463,7 @@ OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
 free_stack(common, 3);
 
 JUMPHERE(jump);
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, TMP1, 0);
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, TMP1, 0);
 }
 
 static void compile_backtrackingpath(compiler_common *common, struct backtrack_common *current)
@@ -8586,7 +10480,7 @@ while (current)
     case OP_SET_SOM:
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
     free_stack(common, 1);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), TMP1, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), TMP1, 0);
     break;
 
     case OP_STAR:
@@ -8664,6 +10558,8 @@ while (current)
 
     case OP_REF:
     case OP_REFI:
+    case OP_DNREF:
+    case OP_DNREFI:
     compile_ref_iterator_backtrackingpath(common, current);
     break;
 
@@ -8713,9 +10609,9 @@ while (current)
     if (common->has_skip_arg)
       OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
     free_stack(common, common->has_skip_arg ? 5 : 1);
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->mark_ptr, TMP1, 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, TMP1, 0);
     if (common->has_skip_arg)
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, TMP2, 0);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, TMP2, 0);
     break;
 
     case OP_THEN:
@@ -8762,7 +10658,7 @@ static SLJIT_INLINE void compile_recurse(compiler_common *common)
 DEFINE_COMPILER;
 pcre_uchar *cc = common->start + common->currententry->start;
 pcre_uchar *ccbegin = cc + 1 + LINK_SIZE + (*cc == OP_BRA ? 0 : IMM2_SIZE);
-pcre_uchar *ccend = bracketend(cc);
+pcre_uchar *ccend = bracketend(cc) - (1 + LINK_SIZE);
 BOOL needs_control_head;
 int framesize = get_framesize(common, cc, NULL, TRUE, &needs_control_head);
 int private_data_size = get_private_data_copy_length(common, ccbegin, ccend, needs_control_head);
@@ -8785,12 +10681,13 @@ common->currententry->entry = LABEL();
 set_jumps(common->currententry->calls, common->currententry->entry);
 
 sljit_emit_fast_enter(compiler, TMP2, 0);
+count_match(common);
 allocate_stack(common, private_data_size + framesize + alternativesize);
 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(private_data_size + framesize + alternativesize - 1), TMP2, 0);
 copy_private_data(common, ccbegin, ccend, TRUE, private_data_size + framesize + alternativesize, framesize + alternativesize, needs_control_head);
 if (needs_control_head)
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, SLJIT_IMM, 0);
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->recursive_head_ptr, STACK_TOP, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0);
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, STACK_TOP, 0);
 if (needs_frame)
   init_frame(common, cc, NULL, framesize + alternativesize - 1, alternativesize, TRUE);
 
@@ -8837,7 +10734,7 @@ jump = JUMP(SLJIT_JUMP);
 if (common->quit != NULL)
   {
   set_jumps(common->quit, LABEL());
-  OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->recursive_head_ptr);
+  OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr);
   if (needs_frame)
     {
     OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw));
@@ -8850,7 +10747,7 @@ if (common->quit != NULL)
   }
 
 set_jumps(common->accept, LABEL());
-OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->recursive_head_ptr);
+OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr);
 if (needs_frame)
   {
   OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw));
@@ -8868,15 +10765,15 @@ if (needs_control_head)
   {
   OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), 2 * sizeof(sljit_sw));
   OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), sizeof(sljit_sw));
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->recursive_head_ptr, TMP1, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, TMP1, 0);
   OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, TMP2, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, TMP2, 0);
   }
 else
   {
   OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), sizeof(sljit_sw));
   OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->recursive_head_ptr, TMP2, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, TMP2, 0);
   }
 sljit_emit_fast_return(compiler, SLJIT_MEM1(STACK_TOP), 0);
 }
@@ -8891,23 +10788,25 @@ struct sljit_compiler *compiler;
 backtrack_common rootbacktrack;
 compiler_common common_data;
 compiler_common *common = &common_data;
-const pcre_uint8 *tables = re->tables;
+const sljit_u8 *tables = re->tables;
 pcre_study_data *study;
 int private_data_size;
 pcre_uchar *ccend;
 executable_functions *functions;
 void *executable_func;
 sljit_uw executable_size;
+sljit_uw total_length;
+label_addr_list *label_addr;
 struct sljit_label *mainloop_label = NULL;
 struct sljit_label *continue_match_label;
-struct sljit_label *empty_match_found_label;
-struct sljit_label *empty_match_backtrack_label;
+struct sljit_label *empty_match_found_label = NULL;
+struct sljit_label *empty_match_backtrack_label = NULL;
 struct sljit_label *reset_match_label;
+struct sljit_label *quit_label;
 struct sljit_jump *jump;
 struct sljit_jump *minlength_check_failed = NULL;
 struct sljit_jump *reqbyte_notfound = NULL;
-struct sljit_jump *empty_match;
-struct sljit_label *quit_label;
+struct sljit_jump *empty_match = NULL;
 
 SLJIT_ASSERT((extra->flags & PCRE_EXTRA_STUDY_DATA) != 0);
 study = extra->study_data;
@@ -8920,9 +10819,11 @@ memset(common, 0, sizeof(compiler_common));
 rootbacktrack.cc = (pcre_uchar *)re + re->name_table_offset + re->name_count * re->name_entry_size;
 
 common->start = rootbacktrack.cc;
+common->read_only_data_head = NULL;
 common->fcc = tables + fcc_offset;
 common->lcc = (sljit_sw)(tables + lcc_offset);
 common->mode = mode;
+common->might_be_empty = study->minlength == 0;
 common->nltype = NLTYPE_FIXED;
 switch(re->options & PCRE_NEWLINE_BITS)
   {
@@ -8943,6 +10844,8 @@ switch(re->options & PCRE_NEWLINE_BITS)
   case PCRE_NEWLINE_ANYCRLF: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANYCRLF; break;
   default: return;
   }
+common->nlmax = READ_CHAR_MAX;
+common->nlmin = 0;
 if ((re->options & PCRE_BSR_ANYCRLF) != 0)
   common->bsr_nltype = NLTYPE_ANYCRLF;
 else if ((re->options & PCRE_BSR_UNICODE) != 0)
@@ -8955,10 +10858,11 @@ else
   common->bsr_nltype = NLTYPE_ANY;
 #endif
   }
+common->bsr_nlmax = READ_CHAR_MAX;
+common->bsr_nlmin = 0;
 common->endonly = (re->options & PCRE_DOLLAR_ENDONLY) != 0;
 common->ctypes = (sljit_sw)(tables + ctypes_offset);
-common->digits[0] = -2;
-common->name_table = (sljit_sw)((pcre_uchar *)re + re->name_table_offset);
+common->name_table = ((pcre_uchar *)re) + re->name_table_offset;
 common->name_count = re->name_count;
 common->name_entry_size = re->name_entry_size;
 common->jscript_compat = (re->options & PCRE_JAVASCRIPT_COMPAT) != 0;
@@ -8968,12 +10872,35 @@ common->utf = (re->options & PCRE_UTF8) != 0;
 #ifdef SUPPORT_UCP
 common->use_ucp = (re->options & PCRE_UCP) != 0;
 #endif
+if (common->utf)
+  {
+  if (common->nltype == NLTYPE_ANY)
+    common->nlmax = 0x2029;
+  else if (common->nltype == NLTYPE_ANYCRLF)
+    common->nlmax = (CHAR_CR > CHAR_NL) ? CHAR_CR : CHAR_NL;
+  else
+    {
+    /* We only care about the first newline character. */
+    common->nlmax = common->newline & 0xff;
+    }
+
+  if (common->nltype == NLTYPE_FIXED)
+    common->nlmin = common->newline & 0xff;
+  else
+    common->nlmin = (CHAR_CR < CHAR_NL) ? CHAR_CR : CHAR_NL;
+
+  if (common->bsr_nltype == NLTYPE_ANY)
+    common->bsr_nlmax = 0x2029;
+  else
+    common->bsr_nlmax = (CHAR_CR > CHAR_NL) ? CHAR_CR : CHAR_NL;
+  common->bsr_nlmin = (CHAR_CR < CHAR_NL) ? CHAR_CR : CHAR_NL;
+  }
 #endif /* SUPPORT_UTF */
-ccend = bracketend(rootbacktrack.cc);
+ccend = bracketend(common->start);
 
 /* Calculate the local space size on the stack. */
 common->ovector_start = LIMIT_MATCH + sizeof(sljit_sw);
-common->optimized_cbracket = (pcre_uint8 *)SLJIT_MALLOC(re->top_bracket + 1);
+common->optimized_cbracket = (sljit_u8 *)SLJIT_MALLOC(re->top_bracket + 1, compiler->allocator_data);
 if (!common->optimized_cbracket)
   return;
 #if defined DEBUG_FORCE_UNOPTIMIZED_CBRAS && DEBUG_FORCE_UNOPTIMIZED_CBRAS == 1
@@ -8982,14 +10909,14 @@ memset(common->optimized_cbracket, 0, re->top_bracket + 1);
 memset(common->optimized_cbracket, 1, re->top_bracket + 1);
 #endif
 
-SLJIT_ASSERT(*rootbacktrack.cc == OP_BRA && ccend[-(1 + LINK_SIZE)] == OP_KET);
+SLJIT_ASSERT(*common->start == OP_BRA && ccend[-(1 + LINK_SIZE)] == OP_KET);
 #if defined DEBUG_FORCE_UNOPTIMIZED_CBRAS && DEBUG_FORCE_UNOPTIMIZED_CBRAS == 2
 common->capture_last_ptr = common->ovector_start;
 common->ovector_start += sizeof(sljit_sw);
 #endif
-if (!check_opcode_types(common, rootbacktrack.cc, ccend))
+if (!check_opcode_types(common, common->start, ccend))
   {
-  SLJIT_FREE(common->optimized_cbracket);
+  SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data);
   return;
   }
 
@@ -9008,15 +10935,10 @@ if (mode != JIT_COMPILE)
     common->hit_start = common->ovector_start;
     common->ovector_start += 2 * sizeof(sljit_sw);
     }
-  else
-    {
-    SLJIT_ASSERT(mode == JIT_PARTIAL_HARD_COMPILE);
-    common->needs_start_ptr = TRUE;
-    }
   }
 if ((re->options & PCRE_FIRSTLINE) != 0)
   {
-  common->first_line_end = common->ovector_start;
+  common->match_end_ptr = common->ovector_start;
   common->ovector_start += sizeof(sljit_sw);
   }
 #if defined DEBUG_FORCE_CONTROL_HEAD && DEBUG_FORCE_CONTROL_HEAD
@@ -9027,14 +10949,12 @@ if (common->control_head_ptr != 0)
   common->control_head_ptr = common->ovector_start;
   common->ovector_start += sizeof(sljit_sw);
   }
-if (common->needs_start_ptr && common->has_set_som)
+if (common->has_set_som)
   {
   /* Saving the real start pointer is necessary. */
   common->start_ptr = common->ovector_start;
   common->ovector_start += sizeof(sljit_sw);
   }
-else
-  common->needs_start_ptr = FALSE;
 
 /* Aligning ovector to even number of sljit words. */
 if ((common->ovector_start & sizeof(sljit_sw)) != 0)
@@ -9050,88 +10970,93 @@ if (common->capture_last_ptr != 0)
 SLJIT_ASSERT(!(common->req_char_ptr != 0 && common->start_used_ptr != 0));
 common->cbra_ptr = OVECTOR_START + (re->top_bracket + 1) * 2 * sizeof(sljit_sw);
 
-common->private_data_ptrs = (int *)SLJIT_MALLOC((ccend - rootbacktrack.cc) * sizeof(sljit_si));
+total_length = ccend - common->start;
+common->private_data_ptrs = (sljit_s32 *)SLJIT_MALLOC(total_length * (sizeof(sljit_s32) + (common->has_then ? 1 : 0)), compiler->allocator_data);
 if (!common->private_data_ptrs)
   {
-  SLJIT_FREE(common->optimized_cbracket);
+  SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data);
   return;
   }
-memset(common->private_data_ptrs, 0, (ccend - rootbacktrack.cc) * sizeof(int));
+memset(common->private_data_ptrs, 0, total_length * sizeof(sljit_s32));
 
 private_data_size = common->cbra_ptr + (re->top_bracket + 1) * sizeof(sljit_sw);
 set_private_data_ptrs(common, &private_data_size, ccend);
+if ((re->options & PCRE_ANCHORED) == 0 && (re->options & PCRE_NO_START_OPTIMIZE) == 0)
+  {
+  if (!detect_fast_forward_skip(common, &private_data_size) && !common->has_skip_in_assert_back)
+    detect_fast_fail(common, common->start, &private_data_size, 4);
+  }
+
+SLJIT_ASSERT(common->fast_fail_start_ptr <= common->fast_fail_end_ptr);
+
 if (private_data_size > SLJIT_MAX_LOCAL_SIZE)
   {
-  SLJIT_FREE(common->private_data_ptrs);
-  SLJIT_FREE(common->optimized_cbracket);
+  SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data);
+  SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data);
   return;
   }
 
 if (common->has_then)
   {
-  common->then_offsets = (pcre_uint8 *)SLJIT_MALLOC(ccend - rootbacktrack.cc);
-  if (!common->then_offsets)
-    {
-    SLJIT_FREE(common->optimized_cbracket);
-    SLJIT_FREE(common->private_data_ptrs);
-    return;
-    }
-  memset(common->then_offsets, 0, ccend - rootbacktrack.cc);
-  set_then_offsets(common, rootbacktrack.cc, NULL);
+  common->then_offsets = (sljit_u8 *)(common->private_data_ptrs + total_length);
+  memset(common->then_offsets, 0, total_length);
+  set_then_offsets(common, common->start, NULL);
   }
 
-compiler = sljit_create_compiler();
+compiler = sljit_create_compiler(NULL);
 if (!compiler)
   {
-  SLJIT_FREE(common->optimized_cbracket);
-  SLJIT_FREE(common->private_data_ptrs);
-  if (common->has_then)
-    SLJIT_FREE(common->then_offsets);
+  SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data);
+  SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data);
   return;
   }
 common->compiler = compiler;
 
 /* Main pcre_jit_exec entry. */
-sljit_emit_enter(compiler, 1, 5, 5, private_data_size);
+sljit_emit_enter(compiler, 0, 1, 5, 5, 0, 0, private_data_size);
 
 /* Register init. */
 reset_ovector(common, (re->top_bracket + 1) * 2);
 if (common->req_char_ptr != 0)
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->req_char_ptr, SLJIT_SCRATCH_REG1, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->req_char_ptr, SLJIT_R0, 0);
 
-OP1(SLJIT_MOV, ARGUMENTS, 0, SLJIT_SAVED_REG1, 0);
-OP1(SLJIT_MOV, TMP1, 0, SLJIT_SAVED_REG1, 0);
+OP1(SLJIT_MOV, ARGUMENTS, 0, SLJIT_S0, 0);
+OP1(SLJIT_MOV, TMP1, 0, SLJIT_S0, 0);
 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, end));
 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack));
-OP1(SLJIT_MOV_UI, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, limit_match));
+OP1(SLJIT_MOV_U32, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, limit_match));
 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_stack, base));
 OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_stack, limit));
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LIMIT_MATCH, TMP1, 0);
+OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LIMIT_MATCH, TMP1, 0);
+
+if (common->fast_fail_start_ptr < common->fast_fail_end_ptr)
+  reset_fast_fail(common);
 
 if (mode == JIT_PARTIAL_SOFT_COMPILE)
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->hit_start, SLJIT_IMM, -1);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, -1);
 if (common->mark_ptr != 0)
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->mark_ptr, SLJIT_IMM, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, SLJIT_IMM, 0);
 if (common->control_head_ptr != 0)
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->control_head_ptr, SLJIT_IMM, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0);
 
 /* Main part of the matching */
 if ((re->options & PCRE_ANCHORED) == 0)
   {
-  mainloop_label = mainloop_entry(common, (re->flags & PCRE_HASCRORLF) != 0, (re->options & PCRE_FIRSTLINE) != 0);
+  mainloop_label = mainloop_entry(common, (re->flags & PCRE_HASCRORLF) != 0);
   continue_match_label = LABEL();
   /* Forward search if possible. */
   if ((re->options & PCRE_NO_START_OPTIMIZE) == 0)
     {
-    if (mode == JIT_COMPILE && fast_forward_first_n_chars(common, (re->options & PCRE_FIRSTLINE) != 0))
-      { /* Do nothing */ }
+    if (mode == JIT_COMPILE && fast_forward_first_n_chars(common))
+      ;
     else if ((re->flags & PCRE_FIRSTSET) != 0)
-      fast_forward_first_char(common, (pcre_uchar)re->first_char, (re->flags & PCRE_FCH_CASELESS) != 0, (re->options & PCRE_FIRSTLINE) != 0);
+      fast_forward_first_char(common, (pcre_uchar)re->first_char, (re->flags & PCRE_FCH_CASELESS) != 0);
     else if ((re->flags & PCRE_STARTLINE) != 0)
-      fast_forward_newline(common, (re->options & PCRE_FIRSTLINE) != 0);
-    else if ((re->flags & PCRE_STARTLINE) == 0 && study != NULL && (study->flags & PCRE_STUDY_MAPPED) != 0)
-      fast_forward_start_bits(common, (sljit_uw)study->start_bits, (re->options & PCRE_FIRSTLINE) != 0);
+      fast_forward_newline(common);
+    else if (study != NULL && (study->flags & PCRE_STUDY_MAPPED) != 0)
+      fast_forward_start_bits(common, study->start_bits);
     }
   }
 else
@@ -9141,50 +11066,49 @@ if (mode == JIT_COMPILE && study->minlength > 0 && (re->options & PCRE_NO_START_
   {
   OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_NOMATCH);
   OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(study->minlength));
-  minlength_check_failed = CMP(SLJIT_C_GREATER, TMP2, 0, STR_END, 0);
+  minlength_check_failed = CMP(SLJIT_GREATER, TMP2, 0, STR_END, 0);
   }
 if (common->req_char_ptr != 0)
   reqbyte_notfound = search_requested_char(common, (pcre_uchar)re->req_char, (re->flags & PCRE_RCH_CASELESS) != 0, (re->flags & PCRE_FIRSTSET) != 0);
 
 /* Store the current STR_PTR in OVECTOR(0). */
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0), STR_PTR, 0);
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), STR_PTR, 0);
 /* Copy the limit of allowed recursions. */
-OP1(SLJIT_MOV, COUNT_MATCH, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LIMIT_MATCH);
+OP1(SLJIT_MOV, COUNT_MATCH, 0, SLJIT_MEM1(SLJIT_SP), LIMIT_MATCH);
 if (common->capture_last_ptr != 0)
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->capture_last_ptr, SLJIT_IMM, -1);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, SLJIT_IMM, -1);
+if (common->fast_forward_bc_ptr != NULL)
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), PRIVATE_DATA(common->fast_forward_bc_ptr + 1), STR_PTR, 0);
 
-if (common->needs_start_ptr)
-  {
-  SLJIT_ASSERT(common->start_ptr != OVECTOR(0));
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->start_ptr, STR_PTR, 0);
-  }
-else
-  SLJIT_ASSERT(common->start_ptr == OVECTOR(0));
+if (common->start_ptr != OVECTOR(0))
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_ptr, STR_PTR, 0);
 
 /* Copy the beginning of the string. */
 if (mode == JIT_PARTIAL_SOFT_COMPILE)
   {
-  jump = CMP(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), common->hit_start, SLJIT_IMM, -1);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->start_used_ptr, STR_PTR, 0);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->hit_start + sizeof(sljit_sw), STR_PTR, 0);
+  jump = CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, -1);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start + sizeof(sljit_sw), STR_PTR, 0);
   JUMPHERE(jump);
   }
 else if (mode == JIT_PARTIAL_HARD_COMPILE)
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->start_used_ptr, STR_PTR, 0);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0);
 
-compile_matchingpath(common, rootbacktrack.cc, ccend, &rootbacktrack);
+compile_matchingpath(common, common->start, ccend, &rootbacktrack);
 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
   {
   sljit_free_compiler(compiler);
-  SLJIT_FREE(common->optimized_cbracket);
-  SLJIT_FREE(common->private_data_ptrs);
-  if (common->has_then)
-    SLJIT_FREE(common->then_offsets);
+  SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data);
+  SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data);
+  free_read_only_data(common->read_only_data_head, compiler->allocator_data);
   return;
   }
 
-empty_match = CMP(SLJIT_C_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), OVECTOR(0));
-empty_match_found_label = LABEL();
+if (common->might_be_empty)
+  {
+  empty_match = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0));
+  empty_match_found_label = LABEL();
+  }
 
 common->accept_label = LABEL();
 if (common->accept != NULL)
@@ -9208,15 +11132,15 @@ if (mode != JIT_COMPILE)
   return_with_partial_match(common, common->quit_label);
   }
 
-empty_match_backtrack_label = LABEL();
+if (common->might_be_empty)
+  empty_match_backtrack_label = LABEL();
 compile_backtrackingpath(common, rootbacktrack.top);
 if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
   {
   sljit_free_compiler(compiler);
-  SLJIT_FREE(common->optimized_cbracket);
-  SLJIT_FREE(common->private_data_ptrs);
-  if (common->has_then)
-    SLJIT_FREE(common->then_offsets);
+  SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data);
+  SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data);
+  free_read_only_data(common->read_only_data_head, compiler->allocator_data);
   return;
   }
 
@@ -9226,28 +11150,33 @@ reset_match_label = LABEL();
 if (mode == JIT_PARTIAL_SOFT_COMPILE)
   {
   /* Update hit_start only in the first time. */
-  jump = CMP(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), common->hit_start, SLJIT_IMM, 0);
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->start_used_ptr);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->start_used_ptr, SLJIT_IMM, -1);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), common->hit_start, TMP1, 0);
+  jump = CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, 0);
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, SLJIT_IMM, -1);
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, TMP1, 0);
   JUMPHERE(jump);
   }
 
 /* Check we have remaining characters. */
 if ((re->options & PCRE_ANCHORED) == 0 && (re->options & PCRE_FIRSTLINE) != 0)
   {
-  SLJIT_ASSERT(common->first_line_end != 0);
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->first_line_end);
+  SLJIT_ASSERT(common->match_end_ptr != 0);
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr);
   }
 
-OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), common->start_ptr);
+OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP),
+    (common->fast_forward_bc_ptr != NULL) ? (PRIVATE_DATA(common->fast_forward_bc_ptr + 1)) : common->start_ptr);
 
 if ((re->options & PCRE_ANCHORED) == 0)
   {
-  if ((re->options & PCRE_FIRSTLINE) == 0)
-    CMPTO(SLJIT_C_LESS, STR_PTR, 0, STR_END, 0, mainloop_label);
+  if (common->ff_newline_shortcut != NULL)
+    {
+    if ((re->options & PCRE_FIRSTLINE) == 0)
+      CMPTO(SLJIT_LESS, STR_PTR, 0, STR_END, 0, common->ff_newline_shortcut);
+    /* There cannot be more newlines here. */
+    }
   else
-    CMPTO(SLJIT_C_LESS, STR_PTR, 0, TMP1, 0, mainloop_label);
+    CMPTO(SLJIT_LESS, STR_PTR, 0, ((re->options & PCRE_FIRSTLINE) == 0) ? STR_END : TMP1, 0, mainloop_label);
   }
 
 /* No more remaining characters. */
@@ -9255,23 +11184,29 @@ if (reqbyte_notfound != NULL)
   JUMPHERE(reqbyte_notfound);
 
 if (mode == JIT_PARTIAL_SOFT_COMPILE)
-  CMPTO(SLJIT_C_NOT_EQUAL, SLJIT_MEM1(SLJIT_LOCALS_REG), common->hit_start, SLJIT_IMM, -1, common->partialmatchlabel);
+  CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, -1, common->partialmatchlabel);
 
 OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_NOMATCH);
 JUMPTO(SLJIT_JUMP, common->quit_label);
 
 flush_stubs(common);
 
-JUMPHERE(empty_match);
-OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
-OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty));
-CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0, empty_match_backtrack_label);
-OP1(SLJIT_MOV_UB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty_atstart));
-CMPTO(SLJIT_C_EQUAL, TMP2, 0, SLJIT_IMM, 0, empty_match_found_label);
-OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
-CMPTO(SLJIT_C_NOT_EQUAL, TMP2, 0, STR_PTR, 0, empty_match_found_label);
-JUMPTO(SLJIT_JUMP, empty_match_backtrack_label);
+if (common->might_be_empty)
+  {
+  JUMPHERE(empty_match);
+  OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
+  OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty));
+  CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0, empty_match_backtrack_label);
+  OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty_atstart));
+  CMPTO(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, 0, empty_match_found_label);
+  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
+  CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, STR_PTR, 0, empty_match_found_label);
+  JUMPTO(SLJIT_JUMP, empty_match_backtrack_label);
+  }
 
+common->fast_forward_bc_ptr = NULL;
+common->fast_fail_start_ptr = 0;
+common->fast_fail_end_ptr = 0;
 common->currententry = common->entries;
 common->local_exit = TRUE;
 quit_label = common->quit_label;
@@ -9282,10 +11217,9 @@ while (common->currententry != NULL)
   if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
     {
     sljit_free_compiler(compiler);
-    SLJIT_FREE(common->optimized_cbracket);
-    SLJIT_FREE(common->private_data_ptrs);
-    if (common->has_then)
-      SLJIT_FREE(common->then_offsets);
+    SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data);
+    SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data);
+    free_read_only_data(common->read_only_data_head, compiler->allocator_data);
     return;
     }
   flush_stubs(common);
@@ -9298,21 +11232,21 @@ common->quit_label = quit_label;
 /* This is a (really) rare case. */
 set_jumps(common->stackalloc, LABEL());
 /* RETURN_ADDR is not a saved register. */
-sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1, TMP2, 0);
+sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0);
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP2, 0);
 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack));
 OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top), STACK_TOP, 0);
 OP2(SLJIT_ADD, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit), SLJIT_IMM, STACK_GROWTH_RATE);
 
 sljit_emit_ijump(compiler, SLJIT_CALL2, SLJIT_IMM, SLJIT_FUNC_OFFSET(sljit_stack_resize));
-jump = CMP(SLJIT_C_NOT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0);
+jump = CMP(SLJIT_NOT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0);
 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack));
 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top));
 OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit));
-OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS1);
-sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_LOCALS_REG), LOCALS0);
+OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1);
+sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0);
 
 /* Allocation failed. */
 JUMPHERE(jump);
@@ -9364,19 +11298,22 @@ if (common->reset_match != NULL)
   {
   set_jumps(common->reset_match, LABEL());
   do_reset_match(common, (re->top_bracket + 1) * 2);
-  CMPTO(SLJIT_C_GREATER, STR_PTR, 0, TMP1, 0, continue_match_label);
+  CMPTO(SLJIT_GREATER, STR_PTR, 0, TMP1, 0, continue_match_label);
   OP1(SLJIT_MOV, STR_PTR, 0, TMP1, 0);
   JUMPTO(SLJIT_JUMP, reset_match_label);
   }
 #ifdef SUPPORT_UTF
-#ifndef COMPILE_PCRE32
+#ifdef COMPILE_PCRE8
 if (common->utfreadchar != NULL)
   {
   set_jumps(common->utfreadchar, LABEL());
   do_utfreadchar(common);
   }
-#endif /* !COMPILE_PCRE32 */
-#ifdef COMPILE_PCRE8
+if (common->utfreadchar16 != NULL)
+  {
+  set_jumps(common->utfreadchar16, LABEL());
+  do_utfreadchar16(common);
+  }
 if (common->utfreadtype8 != NULL)
   {
   set_jumps(common->utfreadtype8, LABEL());
@@ -9392,16 +11329,23 @@ if (common->getucd != NULL)
   }
 #endif
 
-SLJIT_FREE(common->optimized_cbracket);
-SLJIT_FREE(common->private_data_ptrs);
-if (common->has_then)
-  SLJIT_FREE(common->then_offsets);
+SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data);
+SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data);
 
 executable_func = sljit_generate_code(compiler);
 executable_size = sljit_get_generated_code_size(compiler);
+label_addr = common->label_addrs;
+while (label_addr != NULL)
+  {
+  *label_addr->update_addr = sljit_get_label_addr(label_addr->label);
+  label_addr = label_addr->next;
+  }
 sljit_free_compiler(compiler);
 if (executable_func == NULL)
+  {
+  free_read_only_data(common->read_only_data_head, compiler->allocator_data);
   return;
+  }
 
 /* Reuse the function descriptor if possible. */
 if ((extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 && extra->executable_jit != NULL)
@@ -9417,12 +11361,13 @@ else
    * bit remains set, as the bit indicates that the pointer to the data
    * is valid.)
    */
-  functions = SLJIT_MALLOC(sizeof(executable_functions));
+  functions = SLJIT_MALLOC(sizeof(executable_functions), compiler->allocator_data);
   if (functions == NULL)
     {
     /* This case is highly unlikely since we just recently
-    freed a lot of memory. Although not impossible. */
+    freed a lot of memory. Not impossible though. */
     sljit_free_code(executable_func);
+    free_read_only_data(common->read_only_data_head, compiler->allocator_data);
     return;
     }
   memset(functions, 0, sizeof(executable_functions));
@@ -9433,16 +11378,17 @@ else
   }
 
 functions->executable_funcs[mode] = executable_func;
+functions->read_only_data_heads[mode] = common->read_only_data_head;
 functions->executable_sizes[mode] = executable_size;
 }
 
-static int jit_machine_stack_exec(jit_arguments *arguments, void* executable_func)
+static SLJIT_NOINLINE int jit_machine_stack_exec(jit_arguments *arguments, void *executable_func)
 {
 union {
-   void* executable_func;
+   void *executable_func;
    jit_function call_executable_func;
 } convert_executable_func;
-pcre_uint8 local_space[MACHINE_STACK_SIZE];
+sljit_u8 local_space[MACHINE_STACK_SIZE];
 struct sljit_stack local_stack;
 
 local_stack.top = (sljit_sw)&local_space;
@@ -9460,7 +11406,7 @@ PRIV(jit_exec)(const PUBL(extra) *extra_data, const pcre_uchar *subject,
 {
 executable_functions *functions = (executable_functions *)extra_data->executable_jit;
 union {
-   void* executable_func;
+   void *executable_func;
    jit_function call_executable_func;
 } convert_executable_func;
 jit_arguments arguments;
@@ -9482,7 +11428,7 @@ arguments.begin = subject;
 arguments.end = subject + length;
 arguments.mark_ptr = NULL;
 /* JIT decreases this value less frequently than the interpreter. */
-arguments.limit_match = ((extra_data->flags & PCRE_EXTRA_MATCH_LIMIT) == 0) ? MATCH_LIMIT : (pcre_uint32)(extra_data->match_limit);
+arguments.limit_match = ((extra_data->flags & PCRE_EXTRA_MATCH_LIMIT) == 0) ? MATCH_LIMIT : (sljit_u32)(extra_data->match_limit);
 if (functions->limit_match != 0 && functions->limit_match < arguments.limit_match)
   arguments.limit_match = functions->limit_match;
 arguments.notbol = (options & PCRE_NOTBOL) != 0;
@@ -9554,7 +11500,7 @@ pcre32_jit_exec(const pcre32 *argument_re, const pcre32_extra *extra_data,
 pcre_uchar *subject_ptr = (pcre_uchar *)subject;
 executable_functions *functions = (executable_functions *)extra_data->executable_jit;
 union {
-   void* executable_func;
+   void *executable_func;
    jit_function call_executable_func;
 } convert_executable_func;
 jit_arguments arguments;
@@ -9582,7 +11528,7 @@ arguments.begin = subject_ptr;
 arguments.end = subject_ptr + length;
 arguments.mark_ptr = NULL;
 /* JIT decreases this value less frequently than the interpreter. */
-arguments.limit_match = ((extra_data->flags & PCRE_EXTRA_MATCH_LIMIT) == 0) ? MATCH_LIMIT : (pcre_uint32)(extra_data->match_limit);
+arguments.limit_match = ((extra_data->flags & PCRE_EXTRA_MATCH_LIMIT) == 0) ? MATCH_LIMIT : (sljit_u32)(extra_data->match_limit);
 if (functions->limit_match != 0 && functions->limit_match < arguments.limit_match)
   arguments.limit_match = functions->limit_match;
 arguments.notbol = (options & PCRE_NOTBOL) != 0;
@@ -9626,8 +11572,9 @@ for (i = 0; i < JIT_NUMBER_OF_COMPILE_MODES; i++)
   {
   if (functions->executable_funcs[i] != NULL)
     sljit_free_code(functions->executable_funcs[i]);
+  free_read_only_data(functions->read_only_data_heads[i], NULL);
   }
-SLJIT_FREE(functions);
+SLJIT_FREE(functions, compiler->allocator_data);
 }
 
 int
@@ -9669,7 +11616,7 @@ if (startsize > maxsize)
   startsize = maxsize;
 startsize = (startsize + STACK_GROWTH_RATE - 1) & ~(STACK_GROWTH_RATE - 1);
 maxsize = (maxsize + STACK_GROWTH_RATE - 1) & ~(STACK_GROWTH_RATE - 1);
-return (PUBL(jit_stack)*)sljit_allocate_stack(startsize, maxsize);
+return (PUBL(jit_stack)*)sljit_allocate_stack(startsize, maxsize, NULL);
 }
 
 #if defined COMPILE_PCRE8
@@ -9688,7 +11635,7 @@ PCRE_EXP_DECL void
 pcre32_jit_stack_free(pcre32_jit_stack *stack)
 #endif
 {
-sljit_free_stack((struct sljit_stack *)stack);
+sljit_free_stack((struct sljit_stack *)stack, NULL);
 }
 
 #if defined COMPILE_PCRE8
@@ -9718,6 +11665,20 @@ if (extra != NULL &&
   }
 }
 
+#if defined COMPILE_PCRE8
+PCRE_EXP_DECL void
+pcre_jit_free_unused_memory(void)
+#elif defined COMPILE_PCRE16
+PCRE_EXP_DECL void
+pcre16_jit_free_unused_memory(void)
+#elif defined COMPILE_PCRE32
+PCRE_EXP_DECL void
+pcre32_jit_free_unused_memory(void)
+#endif
+{
+sljit_free_unused_memory_exec();
+}
+
 #else  /* SUPPORT_JIT */
 
 /* These are dummy functions to avoid linking errors when JIT support is not
@@ -9784,6 +11745,19 @@ pcre32_assign_jit_stack(pcre32_extra *extra, pcre32_jit_callback callback, void
 (void)userdata;
 }
 
+#if defined COMPILE_PCRE8
+PCRE_EXP_DECL void
+pcre_jit_free_unused_memory(void)
+#elif defined COMPILE_PCRE16
+PCRE_EXP_DECL void
+pcre16_jit_free_unused_memory(void)
+#elif defined COMPILE_PCRE32
+PCRE_EXP_DECL void
+pcre32_jit_free_unused_memory(void)
+#endif
+{
+}
+
 #endif
 
 /* End of pcre_jit_compile.c */
diff --git a/erts/emulator/pcre/pcre_latin_1_table.c b/erts/emulator/pcre/pcre_latin_1_table.c
index 599540723b..aa29275a94 100644
--- a/erts/emulator/pcre/pcre_latin_1_table.c
+++ b/erts/emulator/pcre/pcre_latin_1_table.c
@@ -158,7 +158,7 @@ print, punct, and cntrl. Other classes are built from combinations. */
 */
 
   0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /*   0-  7 */
-  0x00,0x01,0x01,0x00,0x01,0x01,0x00,0x00, /*   8- 15 */
+  0x00,0x01,0x01,0x01,0x01,0x01,0x00,0x00, /*   8- 15 */
   0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /*  16- 23 */
   0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /*  24- 31 */
   0x01,0x00,0x00,0x00,0x80,0x00,0x00,0x00, /*    - '  */
diff --git a/erts/emulator/pcre/pcre_maketables.c b/erts/emulator/pcre/pcre_maketables.c
index 9310d886fa..89204d1152 100644
--- a/erts/emulator/pcre/pcre_maketables.c
+++ b/erts/emulator/pcre/pcre_maketables.c
@@ -104,13 +104,17 @@ for (i = 0; i < 256; i++) *p++ = tolower(i);
 for (i = 0; i < 256; i++) *p++ = islower(i)? toupper(i) : tolower(i);
 
 /* Then the character class tables. Don't try to be clever and save effort on
-exclusive ones - in some locales things may be different. Note that the table
-for "space" includes everything "isspace" gives, including VT in the default
-locale. This makes it work for the POSIX class [:space:]. Note also that it is
-possible for a character to be alnum or alpha without being lower or upper,
-such as "male and female ordinals" (\xAA and \xBA) in the fr_FR locale (at
-least under Debian Linux's locales as of 12/2005). So we must test for alnum
-specially. */
+exclusive ones - in some locales things may be different.
+
+Note that the table for "space" includes everything "isspace" gives, including
+VT in the default locale. This makes it work for the POSIX class [:space:].
+From release 8.34 is is also correct for Perl space, because Perl added VT at
+release 5.18.
+
+Note also that it is possible for a character to be alnum or alpha without
+being lower or upper, such as "male and female ordinals" (\xAA and \xBA) in the
+fr_FR locale (at least under Debian Linux's locales as of 12/2005). So we must
+test for alnum specially. */
 
 memset(p, 0, cbit_length);
 for (i = 0; i < 256; i++)
@@ -129,14 +133,15 @@ for (i = 0; i < 256; i++)
   }
 p += cbit_length;
 
-/* Finally, the character type table. In this, we exclude VT from the white
-space chars, because Perl doesn't recognize it as such for \s and for comments
-within regexes. */
+/* Finally, the character type table. In this, we used to exclude VT from the
+white space chars, because Perl didn't recognize it as such for \s and for
+comments within regexes. However, Perl changed at release 5.18, so PCRE changed
+at release 8.34. */
 
 for (i = 0; i < 256; i++)
   {
   int x = 0;
-  if (i != CHAR_VT && isspace(i)) x += ctype_space;
+  if (isspace(i)) x += ctype_space;
   if (isalpha(i)) x += ctype_letter;
   if (isdigit(i)) x += ctype_digit;
   if (isxdigit(i)) x += ctype_xdigit;
diff --git a/erts/emulator/pcre/pcre_string_utils.c b/erts/emulator/pcre/pcre_string_utils.c
index 274070469f..abce400dc5 100644
--- a/erts/emulator/pcre/pcre_string_utils.c
+++ b/erts/emulator/pcre/pcre_string_utils.c
@@ -6,7 +6,7 @@
 and semantics are as close as possible to those of the Perl 5 language.
 
                        Written by Philip Hazel
-           Copyright (c) 1997-2013 University of Cambridge
+           Copyright (c) 1997-2014 University of Cambridge
 
 -----------------------------------------------------------------------------
 Redistribution and use in source and binary forms, with or without
@@ -91,8 +91,8 @@ pcre_uchar c2;
 
 while (*str1 != '\0' || *str2 != '\0')
   {
-  c1 = RAWUCHARINC(str1);
-  c2 = RAWUCHARINC(str2);
+  c1 = UCHAR21INC(str1);
+  c2 = UCHAR21INC(str2);
   if (c1 != c2)
     return ((c1 > c2) << 1) - 1;
   }
@@ -131,7 +131,7 @@ pcre_uchar c2;
 
 while (*str1 != '\0' || *ustr2 != '\0')
   {
-  c1 = RAWUCHARINC(str1);
+  c1 = UCHAR21INC(str1);
   c2 = (pcre_uchar)*ustr2++;
   if (c1 != c2)
     return ((c1 > c2) << 1) - 1;
diff --git a/erts/emulator/pcre/pcre_study.c b/erts/emulator/pcre/pcre_study.c
index 3d8961ffb0..91afa2e140 100644
--- a/erts/emulator/pcre/pcre_study.c
+++ b/erts/emulator/pcre/pcre_study.c
@@ -67,10 +67,12 @@ string of that length that matches. In UTF8 mode, the result is in characters
 rather than bytes.
 
 Arguments:
+  re              compiled pattern block
   code            pointer to start of group (the bracket)
-  startcode       pointer to start of the whole pattern
+  startcode       pointer to start of the whole pattern's code
   options         the compiling options
-  int             RECURSE depth
+  recurses        chain of recurse_check to catch mutual recursion
+  countptr        pointer to call count (to catch over complexity)
 
 Returns:   the minimum length
            -1 if \C in UTF-8 mode or (*ACCEPT) was encountered
@@ -79,16 +81,20 @@ Returns:   the minimum length
 */
 
 static int
-find_minlength(const pcre_uchar *code, const pcre_uchar *startcode, int options,
-  int recurse_depth)
+find_minlength(const REAL_PCRE *re, const pcre_uchar *code,
+  const pcre_uchar *startcode, int options, recurse_check *recurses,
+  int *countptr)
 {
 int length = -1;
 /* PCRE_UTF16 has the same value as PCRE_UTF8. */
 BOOL utf = (options & PCRE_UTF8) != 0;
 BOOL had_recurse = FALSE;
+recurse_check this_recurse;
 register int branchlength = 0;
 register pcre_uchar *cc = (pcre_uchar *)code + 1 + LINK_SIZE;
 
+if ((*countptr)++ > 1000) return -1;   /* too complex */
+
 if (*code == OP_CBRA || *code == OP_SCBRA ||
     *code == OP_CBRAPOS || *code == OP_SCBRAPOS) cc += IMM2_SIZE;
 
@@ -130,7 +136,7 @@ for (;;)
     case OP_SBRAPOS:
     case OP_ONCE:
     case OP_ONCE_NC:
-    d = find_minlength(cc, startcode, options, recurse_depth);
+    d = find_minlength(re, cc, startcode, options, recurses, countptr);
     if (d < 0) return d;
     branchlength += d;
     do cc += GET(cc, 1); while (*cc == OP_ALT);
@@ -176,9 +182,9 @@ for (;;)
 
     case OP_REVERSE:
     case OP_CREF:
-    case OP_NCREF:
+    case OP_DNCREF:
     case OP_RREF:
-    case OP_NRREF:
+    case OP_DNRREF:
     case OP_DEF:
     case OP_CALLOUT:
     case OP_SOD:
@@ -342,6 +348,7 @@ for (;;)
       {
       case OP_CRPLUS:
       case OP_CRMINPLUS:
+      case OP_CRPOSPLUS:
       branchlength++;
       /* Fall through */
 
@@ -349,11 +356,14 @@ for (;;)
       case OP_CRMINSTAR:
       case OP_CRQUERY:
       case OP_CRMINQUERY:
+      case OP_CRPOSSTAR:
+      case OP_CRPOSQUERY:
       cc++;
       break;
 
       case OP_CRRANGE:
       case OP_CRMINRANGE:
+      case OP_CRPOSRANGE:
       branchlength += GET2(cc,1);
       cc += 1 + 2 * IMM2_SIZE;
       break;
@@ -376,21 +386,80 @@ for (;;)
     matches an empty string (by default it causes a matching failure), so in
     that case we must set the minimum length to zero. */
 
-    case OP_REF:
+    case OP_DNREF:     /* Duplicate named pattern back reference */
+    case OP_DNREFI:
+    if ((options & PCRE_JAVASCRIPT_COMPAT) == 0)
+      {
+      int count = GET2(cc, 1+IMM2_SIZE);
+      pcre_uchar *slot = (pcre_uchar *)re +
+        re->name_table_offset + GET2(cc, 1) * re->name_entry_size;
+      d = INT_MAX;
+      while (count-- > 0)
+        {
+        ce = cs = (pcre_uchar *)PRIV(find_bracket)(startcode, utf, GET2(slot, 0));
+        if (cs == NULL) return -2;
+        do ce += GET(ce, 1); while (*ce == OP_ALT);
+        if (cc > cs && cc < ce)     /* Simple recursion */
+          {
+          d = 0;
+          had_recurse = TRUE;
+          break;
+          }
+        else
+          {
+          recurse_check *r = recurses;
+          for (r = recurses; r != NULL; r = r->prev) if (r->group == cs) break;
+          if (r != NULL)           /* Mutual recursion */
+            {
+            d = 0;
+            had_recurse = TRUE;
+            break;
+            }
+          else
+            {
+            int dd;
+            this_recurse.prev = recurses;
+            this_recurse.group = cs;
+            dd = find_minlength(re, cs, startcode, options, &this_recurse,
+              countptr);
+            if (dd < d) d = dd;
+            }
+          }
+        slot += re->name_entry_size;
+        }
+      }
+    else d = 0;
+    cc += 1 + 2*IMM2_SIZE;
+    goto REPEAT_BACK_REFERENCE;
+
+    case OP_REF:      /* Single back reference */
     case OP_REFI:
     if ((options & PCRE_JAVASCRIPT_COMPAT) == 0)
       {
       ce = cs = (pcre_uchar *)PRIV(find_bracket)(startcode, utf, GET2(cc, 1));
       if (cs == NULL) return -2;
       do ce += GET(ce, 1); while (*ce == OP_ALT);
-      if (cc > cs && cc < ce)
+      if (cc > cs && cc < ce)    /* Simple recursion */
         {
         d = 0;
         had_recurse = TRUE;
         }
       else
         {
-        d = find_minlength(cs, startcode, options, recurse_depth);
+        recurse_check *r = recurses;
+        for (r = recurses; r != NULL; r = r->prev) if (r->group == cs) break;
+        if (r != NULL)           /* Mutual recursion */
+          {
+          d = 0;
+          had_recurse = TRUE;
+          }
+        else
+          {
+          this_recurse.prev = recurses;
+          this_recurse.group = cs;
+          d = find_minlength(re, cs, startcode, options, &this_recurse,
+            countptr);
+          }
         }
       }
     else d = 0;
@@ -398,24 +467,29 @@ for (;;)
 
     /* Handle repeated back references */
 
+    REPEAT_BACK_REFERENCE:
     switch (*cc)
       {
       case OP_CRSTAR:
       case OP_CRMINSTAR:
       case OP_CRQUERY:
       case OP_CRMINQUERY:
+      case OP_CRPOSSTAR:
+      case OP_CRPOSQUERY:
       min = 0;
       cc++;
       break;
 
       case OP_CRPLUS:
       case OP_CRMINPLUS:
+      case OP_CRPOSPLUS:
       min = 1;
       cc++;
       break;
 
       case OP_CRRANGE:
       case OP_CRMINRANGE:
+      case OP_CRPOSRANGE:
       min = GET2(cc, 1);
       cc += 1 + 2 * IMM2_SIZE;
       break;
@@ -434,11 +508,21 @@ for (;;)
     case OP_RECURSE:
     cs = ce = (pcre_uchar *)startcode + GET(cc, 1);
     do ce += GET(ce, 1); while (*ce == OP_ALT);
-    if ((cc > cs && cc < ce) || recurse_depth > 10)
+    if (cc > cs && cc < ce)    /* Simple recursion */
       had_recurse = TRUE;
     else
       {
-      branchlength += find_minlength(cs, startcode, options, recurse_depth + 1);
+      recurse_check *r = recurses;
+      for (r = recurses; r != NULL; r = r->prev) if (r->group == cs) break;
+      if (r != NULL)           /* Mutual recursion */
+        had_recurse = TRUE;
+      else
+        {
+        this_recurse.prev = recurses;
+        this_recurse.group = cs;
+        branchlength += find_minlength(re, cs, startcode, options,
+          &this_recurse, countptr);
+        }
       }
     cc += 1 + LINK_SIZE;
     break;
@@ -779,6 +863,10 @@ do
       case OP_COND:
       case OP_CREF:
       case OP_DEF:
+      case OP_DNCREF:
+      case OP_DNREF:
+      case OP_DNREFI:
+      case OP_DNRREF:
       case OP_DOLL:
       case OP_DOLLM:
       case OP_END:
@@ -787,7 +875,6 @@ do
       case OP_EXTUNI:
       case OP_FAIL:
       case OP_MARK:
-      case OP_NCREF:
       case OP_NOT:
       case OP_NOTEXACT:
       case OP_NOTEXACTI:
@@ -819,8 +906,6 @@ do
       case OP_NOTUPTOI:
       case OP_NOT_HSPACE:
       case OP_NOT_VSPACE:
-      case OP_NRREF:
-      case OP_PROP:
       case OP_PRUNE:
       case OP_PRUNE_ARG:
       case OP_RECURSE:
@@ -836,11 +921,33 @@ do
       case OP_SOM:
       case OP_THEN:
       case OP_THEN_ARG:
-#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
-      case OP_XCLASS:
-#endif
       return SSB_FAIL;
 
+      /* A "real" property test implies no starting bits, but the fake property
+      PT_CLIST identifies a list of characters. These lists are short, as they
+      are used for characters with more than one "other case", so there is no
+      point in recognizing them for OP_NOTPROP. */
+
+      case OP_PROP:
+      if (tcode[1] != PT_CLIST) return SSB_FAIL;
+        {
+        const pcre_uint32 *p = PRIV(ucd_caseless_sets) + tcode[2];
+        while ((c = *p++) < NOTACHAR)
+          {
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
+          if (utf)
+            {
+            pcre_uchar buff[6];
+            (void)PRIV(ord2utf)(c, buff);
+            c = buff[0];
+            }
+#endif
+          if (c > 0xff) SET_BIT(0xff); else SET_BIT(c);
+          }
+        }
+      try_next = FALSE;
+      break;
+
       /* We can ignore word boundary tests. */
 
       case OP_WORD_BOUNDARY:
@@ -1066,24 +1173,17 @@ do
       try_next = FALSE;
       break;
 
-      /* The cbit_space table has vertical tab as whitespace; we have to
-      ensure it is set as not whitespace. Luckily, the code value is the same
-      (0x0b) in ASCII and EBCDIC, so we can just adjust the appropriate bit. */
+      /* The cbit_space table has vertical tab as whitespace; we no longer
+      have to play fancy tricks because Perl added VT to its whitespace at
+      release 5.18. PCRE added it at release 8.34. */
 
       case OP_NOT_WHITESPACE:
       set_nottype_bits(start_bits, cbit_space, table_limit, cd);
-      start_bits[1] |= 0x08;
       try_next = FALSE;
       break;
 
-      /* The cbit_space table has vertical tab as whitespace; we have to not
-      set it from the table. Luckily, the code value is the same (0x0b) in
-      ASCII and EBCDIC, so we can just adjust the appropriate bit. */
-
       case OP_WHITESPACE:
-      c = start_bits[1];    /* Save in case it was already set */
       set_type_bits(start_bits, cbit_space, table_limit, cd);
-      start_bits[1] = (start_bits[1] & ~0x08) | c;
       try_next = FALSE;
       break;
 
@@ -1184,24 +1284,16 @@ do
         set_type_bits(start_bits, cbit_digit, table_limit, cd);
         break;
 
-        /* The cbit_space table has vertical tab as whitespace; we have to
-        ensure it gets set as not whitespace. Luckily, the code value is the
-        same (0x0b) in ASCII and EBCDIC, so we can just adjust the appropriate
-        bit. */
+        /* The cbit_space table has vertical tab as whitespace; we no longer
+        have to play fancy tricks because Perl added VT to its whitespace at
+        release 5.18. PCRE added it at release 8.34. */
 
         case OP_NOT_WHITESPACE:
         set_nottype_bits(start_bits, cbit_space, table_limit, cd);
-        start_bits[1] |= 0x08;
         break;
 
-        /* The cbit_space table has vertical tab as whitespace; we have to
-        avoid setting it. Luckily, the code value is the same (0x0b) in ASCII
-        and EBCDIC, so we can just adjust the appropriate bit. */
-
         case OP_WHITESPACE:
-        c = start_bits[1];    /* Save in case it was already set */
         set_type_bits(start_bits, cbit_space, table_limit, cd);
-        start_bits[1] = (start_bits[1] & ~0x08) | c;
         break;
 
         case OP_NOT_WORDCHAR:
@@ -1222,6 +1314,16 @@ do
       with a value >= 0xc4 is a potentially valid starter because it starts a
       character with a value > 255. */
 
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+      case OP_XCLASS:
+      if ((tcode[1 + LINK_SIZE] & XCL_HASPROP) != 0)
+        return SSB_FAIL;
+      /* All bits are set. */
+      if ((tcode[1 + LINK_SIZE] & XCL_MAP) == 0 && (tcode[1 + LINK_SIZE] & XCL_NOT) != 0)
+        return SSB_FAIL;
+#endif
+      /* Fall through */
+
       case OP_NCLASS:
 #if defined SUPPORT_UTF && defined COMPILE_PCRE8
       if (utf)
@@ -1238,8 +1340,21 @@ do
       case OP_CLASS:
         {
         pcre_uint8 *map;
-        tcode++;
-        map = (pcre_uint8 *)tcode;
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+        map = NULL;
+        if (*tcode == OP_XCLASS)
+          {
+          if ((tcode[1 + LINK_SIZE] & XCL_MAP) != 0)
+            map = (pcre_uint8 *)(tcode + 1 + LINK_SIZE + 1);
+          tcode += GET(tcode, 1);
+          }
+        else
+#endif
+          {
+          tcode++;
+          map = (pcre_uint8 *)tcode;
+          tcode += 32 / sizeof(pcre_uchar);
+          }
 
         /* In UTF-8 mode, the bits in a bit map correspond to character
         values, not to byte values. However, the bit map we are constructing is
@@ -1247,42 +1362,49 @@ do
         value is > 127. In fact, there are only two possible starting bytes for
         characters in the range 128 - 255. */
 
-#if defined SUPPORT_UTF && defined COMPILE_PCRE8
-        if (utf)
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+        if (map != NULL)
+#endif
           {
-          for (c = 0; c < 16; c++) start_bits[c] |= map[c];
-          for (c = 128; c < 256; c++)
+#if defined SUPPORT_UTF && defined COMPILE_PCRE8
+          if (utf)
             {
-            if ((map[c/8] && (1 << (c&7))) != 0)
+            for (c = 0; c < 16; c++) start_bits[c] |= map[c];
+            for (c = 128; c < 256; c++)
               {
-              int d = (c >> 6) | 0xc0;            /* Set bit for this starter */
-              start_bits[d/8] |= (1 << (d&7));    /* and then skip on to the */
-              c = (c & 0xc0) + 0x40 - 1;          /* next relevant character. */
+              if ((map[c/8] & (1 << (c&7))) != 0)
+                {
+                int d = (c >> 6) | 0xc0;            /* Set bit for this starter */
+                start_bits[d/8] |= (1 << (d&7));    /* and then skip on to the */
+                c = (c & 0xc0) + 0x40 - 1;          /* next relevant character. */
+                }
               }
             }
-          }
-        else
+          else
 #endif
-          {
-          /* In non-UTF-8 mode, the two bit maps are completely compatible. */
-          for (c = 0; c < 32; c++) start_bits[c] |= map[c];
+            {
+            /* In non-UTF-8 mode, the two bit maps are completely compatible. */
+            for (c = 0; c < 32; c++) start_bits[c] |= map[c];
+            }
           }
 
         /* Advance past the bit map, and act on what follows. For a zero
         minimum repeat, continue; otherwise stop processing. */
 
-        tcode += 32 / sizeof(pcre_uchar);
         switch (*tcode)
           {
           case OP_CRSTAR:
           case OP_CRMINSTAR:
           case OP_CRQUERY:
           case OP_CRMINQUERY:
+          case OP_CRPOSSTAR:
+          case OP_CRPOSQUERY:
           tcode++;
           break;
 
           case OP_CRRANGE:
           case OP_CRMINRANGE:
+          case OP_CRPOSRANGE:
           if (GET2(tcode, 1) == 0) tcode += 1 + 2 * IMM2_SIZE;
             else try_next = FALSE;
           break;
@@ -1343,6 +1465,7 @@ pcre32_study(const pcre32 *external_re, int options, const char **errorptr)
 #endif
 {
 int min;
+int count = 0;
 BOOL bits_set = FALSE;
 pcre_uint8 start_bits[32];
 PUBL(extra) *extra = NULL;
@@ -1352,6 +1475,7 @@ pcre_uchar *code;
 compile_data compile_block;
 const REAL_PCRE *re = (const REAL_PCRE *)external_re;
 
+
 *errorptr = NULL;
 
 if (re == NULL || re->magic_number != MAGIC_NUMBER)
@@ -1434,7 +1558,7 @@ if ((re->options & PCRE_ANCHORED) == 0 &&
 
 /* Find the minimum length of subject string. */
 
-switch(min = find_minlength(code, code, re->options, 0))
+switch(min = find_minlength(re, code, code, re->options, NULL, &count))
   {
   case -2: *errorptr = "internal error: missing capturing bracket"; return NULL;
   case -3: *errorptr = "internal error: opcode not recognized"; return NULL;
diff --git a/erts/emulator/pcre/pcre_tables.c b/erts/emulator/pcre/pcre_tables.c
index 7be23babe0..2f6302e2e1 100644
--- a/erts/emulator/pcre/pcre_tables.c
+++ b/erts/emulator/pcre/pcre_tables.c
@@ -214,6 +214,7 @@ strings to make sure that UTF-8 support works on EBCDIC platforms. */
 #define STRING_Avestan0 STR_A STR_v STR_e STR_s STR_t STR_a STR_n "\0"
 #define STRING_Balinese0 STR_B STR_a STR_l STR_i STR_n STR_e STR_s STR_e "\0"
 #define STRING_Bamum0 STR_B STR_a STR_m STR_u STR_m "\0"
+#define STRING_Bassa_Vah0 STR_B STR_a STR_s STR_s STR_a STR_UNDERSCORE STR_V STR_a STR_h "\0"
 #define STRING_Batak0 STR_B STR_a STR_t STR_a STR_k "\0"
 #define STRING_Bengali0 STR_B STR_e STR_n STR_g STR_a STR_l STR_i "\0"
 #define STRING_Bopomofo0 STR_B STR_o STR_p STR_o STR_m STR_o STR_f STR_o "\0"
@@ -224,6 +225,7 @@ strings to make sure that UTF-8 support works on EBCDIC platforms. */
 #define STRING_C0 STR_C "\0"
 #define STRING_Canadian_Aboriginal0 STR_C STR_a STR_n STR_a STR_d STR_i STR_a STR_n STR_UNDERSCORE STR_A STR_b STR_o STR_r STR_i STR_g STR_i STR_n STR_a STR_l "\0"
 #define STRING_Carian0 STR_C STR_a STR_r STR_i STR_a STR_n "\0"
+#define STRING_Caucasian_Albanian0 STR_C STR_a STR_u STR_c STR_a STR_s STR_i STR_a STR_n STR_UNDERSCORE STR_A STR_l STR_b STR_a STR_n STR_i STR_a STR_n "\0"
 #define STRING_Cc0 STR_C STR_c "\0"
 #define STRING_Cf0 STR_C STR_f "\0"
 #define STRING_Chakma0 STR_C STR_h STR_a STR_k STR_m STR_a "\0"
@@ -239,11 +241,14 @@ strings to make sure that UTF-8 support works on EBCDIC platforms. */
 #define STRING_Cyrillic0 STR_C STR_y STR_r STR_i STR_l STR_l STR_i STR_c "\0"
 #define STRING_Deseret0 STR_D STR_e STR_s STR_e STR_r STR_e STR_t "\0"
 #define STRING_Devanagari0 STR_D STR_e STR_v STR_a STR_n STR_a STR_g STR_a STR_r STR_i "\0"
+#define STRING_Duployan0 STR_D STR_u STR_p STR_l STR_o STR_y STR_a STR_n "\0"
 #define STRING_Egyptian_Hieroglyphs0 STR_E STR_g STR_y STR_p STR_t STR_i STR_a STR_n STR_UNDERSCORE STR_H STR_i STR_e STR_r STR_o STR_g STR_l STR_y STR_p STR_h STR_s "\0"
+#define STRING_Elbasan0 STR_E STR_l STR_b STR_a STR_s STR_a STR_n "\0"
 #define STRING_Ethiopic0 STR_E STR_t STR_h STR_i STR_o STR_p STR_i STR_c "\0"
 #define STRING_Georgian0 STR_G STR_e STR_o STR_r STR_g STR_i STR_a STR_n "\0"
 #define STRING_Glagolitic0 STR_G STR_l STR_a STR_g STR_o STR_l STR_i STR_t STR_i STR_c "\0"
 #define STRING_Gothic0 STR_G STR_o STR_t STR_h STR_i STR_c "\0"
+#define STRING_Grantha0 STR_G STR_r STR_a STR_n STR_t STR_h STR_a "\0"
 #define STRING_Greek0 STR_G STR_r STR_e STR_e STR_k "\0"
 #define STRING_Gujarati0 STR_G STR_u STR_j STR_a STR_r STR_a STR_t STR_i "\0"
 #define STRING_Gurmukhi0 STR_G STR_u STR_r STR_m STR_u STR_k STR_h STR_i "\0"
@@ -263,12 +268,15 @@ strings to make sure that UTF-8 support works on EBCDIC platforms. */
 #define STRING_Kayah_Li0 STR_K STR_a STR_y STR_a STR_h STR_UNDERSCORE STR_L STR_i "\0"
 #define STRING_Kharoshthi0 STR_K STR_h STR_a STR_r STR_o STR_s STR_h STR_t STR_h STR_i "\0"
 #define STRING_Khmer0 STR_K STR_h STR_m STR_e STR_r "\0"
+#define STRING_Khojki0 STR_K STR_h STR_o STR_j STR_k STR_i "\0"
+#define STRING_Khudawadi0 STR_K STR_h STR_u STR_d STR_a STR_w STR_a STR_d STR_i "\0"
 #define STRING_L0 STR_L "\0"
 #define STRING_L_AMPERSAND0 STR_L STR_AMPERSAND "\0"
 #define STRING_Lao0 STR_L STR_a STR_o "\0"
 #define STRING_Latin0 STR_L STR_a STR_t STR_i STR_n "\0"
 #define STRING_Lepcha0 STR_L STR_e STR_p STR_c STR_h STR_a "\0"
 #define STRING_Limbu0 STR_L STR_i STR_m STR_b STR_u "\0"
+#define STRING_Linear_A0 STR_L STR_i STR_n STR_e STR_a STR_r STR_UNDERSCORE STR_A "\0"
 #define STRING_Linear_B0 STR_L STR_i STR_n STR_e STR_a STR_r STR_UNDERSCORE STR_B "\0"
 #define STRING_Lisu0 STR_L STR_i STR_s STR_u "\0"
 #define STRING_Ll0 STR_L STR_l "\0"
@@ -279,18 +287,24 @@ strings to make sure that UTF-8 support works on EBCDIC platforms. */
 #define STRING_Lycian0 STR_L STR_y STR_c STR_i STR_a STR_n "\0"
 #define STRING_Lydian0 STR_L STR_y STR_d STR_i STR_a STR_n "\0"
 #define STRING_M0 STR_M "\0"
+#define STRING_Mahajani0 STR_M STR_a STR_h STR_a STR_j STR_a STR_n STR_i "\0"
 #define STRING_Malayalam0 STR_M STR_a STR_l STR_a STR_y STR_a STR_l STR_a STR_m "\0"
 #define STRING_Mandaic0 STR_M STR_a STR_n STR_d STR_a STR_i STR_c "\0"
+#define STRING_Manichaean0 STR_M STR_a STR_n STR_i STR_c STR_h STR_a STR_e STR_a STR_n "\0"
 #define STRING_Mc0 STR_M STR_c "\0"
 #define STRING_Me0 STR_M STR_e "\0"
 #define STRING_Meetei_Mayek0 STR_M STR_e STR_e STR_t STR_e STR_i STR_UNDERSCORE STR_M STR_a STR_y STR_e STR_k "\0"
+#define STRING_Mende_Kikakui0 STR_M STR_e STR_n STR_d STR_e STR_UNDERSCORE STR_K STR_i STR_k STR_a STR_k STR_u STR_i "\0"
 #define STRING_Meroitic_Cursive0 STR_M STR_e STR_r STR_o STR_i STR_t STR_i STR_c STR_UNDERSCORE STR_C STR_u STR_r STR_s STR_i STR_v STR_e "\0"
 #define STRING_Meroitic_Hieroglyphs0 STR_M STR_e STR_r STR_o STR_i STR_t STR_i STR_c STR_UNDERSCORE STR_H STR_i STR_e STR_r STR_o STR_g STR_l STR_y STR_p STR_h STR_s "\0"
 #define STRING_Miao0 STR_M STR_i STR_a STR_o "\0"
 #define STRING_Mn0 STR_M STR_n "\0"
+#define STRING_Modi0 STR_M STR_o STR_d STR_i "\0"
 #define STRING_Mongolian0 STR_M STR_o STR_n STR_g STR_o STR_l STR_i STR_a STR_n "\0"
+#define STRING_Mro0 STR_M STR_r STR_o "\0"
 #define STRING_Myanmar0 STR_M STR_y STR_a STR_n STR_m STR_a STR_r "\0"
 #define STRING_N0 STR_N "\0"
+#define STRING_Nabataean0 STR_N STR_a STR_b STR_a STR_t STR_a STR_e STR_a STR_n "\0"
 #define STRING_Nd0 STR_N STR_d "\0"
 #define STRING_New_Tai_Lue0 STR_N STR_e STR_w STR_UNDERSCORE STR_T STR_a STR_i STR_UNDERSCORE STR_L STR_u STR_e "\0"
 #define STRING_Nko0 STR_N STR_k STR_o "\0"
@@ -299,12 +313,17 @@ strings to make sure that UTF-8 support works on EBCDIC platforms. */
 #define STRING_Ogham0 STR_O STR_g STR_h STR_a STR_m "\0"
 #define STRING_Ol_Chiki0 STR_O STR_l STR_UNDERSCORE STR_C STR_h STR_i STR_k STR_i "\0"
 #define STRING_Old_Italic0 STR_O STR_l STR_d STR_UNDERSCORE STR_I STR_t STR_a STR_l STR_i STR_c "\0"
+#define STRING_Old_North_Arabian0 STR_O STR_l STR_d STR_UNDERSCORE STR_N STR_o STR_r STR_t STR_h STR_UNDERSCORE STR_A STR_r STR_a STR_b STR_i STR_a STR_n "\0"
+#define STRING_Old_Permic0 STR_O STR_l STR_d STR_UNDERSCORE STR_P STR_e STR_r STR_m STR_i STR_c "\0"
 #define STRING_Old_Persian0 STR_O STR_l STR_d STR_UNDERSCORE STR_P STR_e STR_r STR_s STR_i STR_a STR_n "\0"
 #define STRING_Old_South_Arabian0 STR_O STR_l STR_d STR_UNDERSCORE STR_S STR_o STR_u STR_t STR_h STR_UNDERSCORE STR_A STR_r STR_a STR_b STR_i STR_a STR_n "\0"
 #define STRING_Old_Turkic0 STR_O STR_l STR_d STR_UNDERSCORE STR_T STR_u STR_r STR_k STR_i STR_c "\0"
 #define STRING_Oriya0 STR_O STR_r STR_i STR_y STR_a "\0"
 #define STRING_Osmanya0 STR_O STR_s STR_m STR_a STR_n STR_y STR_a "\0"
 #define STRING_P0 STR_P "\0"
+#define STRING_Pahawh_Hmong0 STR_P STR_a STR_h STR_a STR_w STR_h STR_UNDERSCORE STR_H STR_m STR_o STR_n STR_g "\0"
+#define STRING_Palmyrene0 STR_P STR_a STR_l STR_m STR_y STR_r STR_e STR_n STR_e "\0"
+#define STRING_Pau_Cin_Hau0 STR_P STR_a STR_u STR_UNDERSCORE STR_C STR_i STR_n STR_UNDERSCORE STR_H STR_a STR_u "\0"
 #define STRING_Pc0 STR_P STR_c "\0"
 #define STRING_Pd0 STR_P STR_d "\0"
 #define STRING_Pe0 STR_P STR_e "\0"
@@ -314,6 +333,7 @@ strings to make sure that UTF-8 support works on EBCDIC platforms. */
 #define STRING_Pi0 STR_P STR_i "\0"
 #define STRING_Po0 STR_P STR_o "\0"
 #define STRING_Ps0 STR_P STR_s "\0"
+#define STRING_Psalter_Pahlavi0 STR_P STR_s STR_a STR_l STR_t STR_e STR_r STR_UNDERSCORE STR_P STR_a STR_h STR_l STR_a STR_v STR_i "\0"
 #define STRING_Rejang0 STR_R STR_e STR_j STR_a STR_n STR_g "\0"
 #define STRING_Runic0 STR_R STR_u STR_n STR_i STR_c "\0"
 #define STRING_S0 STR_S "\0"
@@ -322,6 +342,7 @@ strings to make sure that UTF-8 support works on EBCDIC platforms. */
 #define STRING_Sc0 STR_S STR_c "\0"
 #define STRING_Sharada0 STR_S STR_h STR_a STR_r STR_a STR_d STR_a "\0"
 #define STRING_Shavian0 STR_S STR_h STR_a STR_v STR_i STR_a STR_n "\0"
+#define STRING_Siddham0 STR_S STR_i STR_d STR_d STR_h STR_a STR_m "\0"
 #define STRING_Sinhala0 STR_S STR_i STR_n STR_h STR_a STR_l STR_a "\0"
 #define STRING_Sk0 STR_S STR_k "\0"
 #define STRING_Sm0 STR_S STR_m "\0"
@@ -342,8 +363,10 @@ strings to make sure that UTF-8 support works on EBCDIC platforms. */
 #define STRING_Thai0 STR_T STR_h STR_a STR_i "\0"
 #define STRING_Tibetan0 STR_T STR_i STR_b STR_e STR_t STR_a STR_n "\0"
 #define STRING_Tifinagh0 STR_T STR_i STR_f STR_i STR_n STR_a STR_g STR_h "\0"
+#define STRING_Tirhuta0 STR_T STR_i STR_r STR_h STR_u STR_t STR_a "\0"
 #define STRING_Ugaritic0 STR_U STR_g STR_a STR_r STR_i STR_t STR_i STR_c "\0"
 #define STRING_Vai0 STR_V STR_a STR_i "\0"
+#define STRING_Warang_Citi0 STR_W STR_a STR_r STR_a STR_n STR_g STR_UNDERSCORE STR_C STR_i STR_t STR_i "\0"
 #define STRING_Xan0 STR_X STR_a STR_n "\0"
 #define STRING_Xps0 STR_X STR_p STR_s "\0"
 #define STRING_Xsp0 STR_X STR_s STR_p "\0"
@@ -362,6 +385,7 @@ const char PRIV(utt_names)[] =
   STRING_Avestan0
   STRING_Balinese0
   STRING_Bamum0
+  STRING_Bassa_Vah0
   STRING_Batak0
   STRING_Bengali0
   STRING_Bopomofo0
@@ -372,6 +396,7 @@ const char PRIV(utt_names)[] =
   STRING_C0
   STRING_Canadian_Aboriginal0
   STRING_Carian0
+  STRING_Caucasian_Albanian0
   STRING_Cc0
   STRING_Cf0
   STRING_Chakma0
@@ -387,11 +412,14 @@ const char PRIV(utt_names)[] =
   STRING_Cyrillic0
   STRING_Deseret0
   STRING_Devanagari0
+  STRING_Duployan0
   STRING_Egyptian_Hieroglyphs0
+  STRING_Elbasan0
   STRING_Ethiopic0
   STRING_Georgian0
   STRING_Glagolitic0
   STRING_Gothic0
+  STRING_Grantha0
   STRING_Greek0
   STRING_Gujarati0
   STRING_Gurmukhi0
@@ -411,12 +439,15 @@ const char PRIV(utt_names)[] =
   STRING_Kayah_Li0
   STRING_Kharoshthi0
   STRING_Khmer0
+  STRING_Khojki0
+  STRING_Khudawadi0
   STRING_L0
   STRING_L_AMPERSAND0
   STRING_Lao0
   STRING_Latin0
   STRING_Lepcha0
   STRING_Limbu0
+  STRING_Linear_A0
   STRING_Linear_B0
   STRING_Lisu0
   STRING_Ll0
@@ -427,18 +458,24 @@ const char PRIV(utt_names)[] =
   STRING_Lycian0
   STRING_Lydian0
   STRING_M0
+  STRING_Mahajani0
   STRING_Malayalam0
   STRING_Mandaic0
+  STRING_Manichaean0
   STRING_Mc0
   STRING_Me0
   STRING_Meetei_Mayek0
+  STRING_Mende_Kikakui0
   STRING_Meroitic_Cursive0
   STRING_Meroitic_Hieroglyphs0
   STRING_Miao0
   STRING_Mn0
+  STRING_Modi0
   STRING_Mongolian0
+  STRING_Mro0
   STRING_Myanmar0
   STRING_N0
+  STRING_Nabataean0
   STRING_Nd0
   STRING_New_Tai_Lue0
   STRING_Nko0
@@ -447,12 +484,17 @@ const char PRIV(utt_names)[] =
   STRING_Ogham0
   STRING_Ol_Chiki0
   STRING_Old_Italic0
+  STRING_Old_North_Arabian0
+  STRING_Old_Permic0
   STRING_Old_Persian0
   STRING_Old_South_Arabian0
   STRING_Old_Turkic0
   STRING_Oriya0
   STRING_Osmanya0
   STRING_P0
+  STRING_Pahawh_Hmong0
+  STRING_Palmyrene0
+  STRING_Pau_Cin_Hau0
   STRING_Pc0
   STRING_Pd0
   STRING_Pe0
@@ -462,6 +504,7 @@ const char PRIV(utt_names)[] =
   STRING_Pi0
   STRING_Po0
   STRING_Ps0
+  STRING_Psalter_Pahlavi0
   STRING_Rejang0
   STRING_Runic0
   STRING_S0
@@ -470,6 +513,7 @@ const char PRIV(utt_names)[] =
   STRING_Sc0
   STRING_Sharada0
   STRING_Shavian0
+  STRING_Siddham0
   STRING_Sinhala0
   STRING_Sk0
   STRING_Sm0
@@ -490,8 +534,10 @@ const char PRIV(utt_names)[] =
   STRING_Thai0
   STRING_Tibetan0
   STRING_Tifinagh0
+  STRING_Tirhuta0
   STRING_Ugaritic0
   STRING_Vai0
+  STRING_Warang_Citi0
   STRING_Xan0
   STRING_Xps0
   STRING_Xsp0
@@ -510,146 +556,169 @@ const ucp_type_table PRIV(utt)[] = {
   {  20, PT_SC, ucp_Avestan },
   {  28, PT_SC, ucp_Balinese },
   {  37, PT_SC, ucp_Bamum },
-  {  43, PT_SC, ucp_Batak },
-  {  49, PT_SC, ucp_Bengali },
-  {  57, PT_SC, ucp_Bopomofo },
-  {  66, PT_SC, ucp_Brahmi },
-  {  73, PT_SC, ucp_Braille },
-  {  81, PT_SC, ucp_Buginese },
-  {  90, PT_SC, ucp_Buhid },
-  {  96, PT_GC, ucp_C },
-  {  98, PT_SC, ucp_Canadian_Aboriginal },
-  { 118, PT_SC, ucp_Carian },
-  { 125, PT_PC, ucp_Cc },
-  { 128, PT_PC, ucp_Cf },
-  { 131, PT_SC, ucp_Chakma },
-  { 138, PT_SC, ucp_Cham },
-  { 143, PT_SC, ucp_Cherokee },
-  { 152, PT_PC, ucp_Cn },
-  { 155, PT_PC, ucp_Co },
-  { 158, PT_SC, ucp_Common },
-  { 165, PT_SC, ucp_Coptic },
-  { 172, PT_PC, ucp_Cs },
-  { 175, PT_SC, ucp_Cuneiform },
-  { 185, PT_SC, ucp_Cypriot },
-  { 193, PT_SC, ucp_Cyrillic },
-  { 202, PT_SC, ucp_Deseret },
-  { 210, PT_SC, ucp_Devanagari },
-  { 221, PT_SC, ucp_Egyptian_Hieroglyphs },
-  { 242, PT_SC, ucp_Ethiopic },
-  { 251, PT_SC, ucp_Georgian },
-  { 260, PT_SC, ucp_Glagolitic },
-  { 271, PT_SC, ucp_Gothic },
-  { 278, PT_SC, ucp_Greek },
-  { 284, PT_SC, ucp_Gujarati },
-  { 293, PT_SC, ucp_Gurmukhi },
-  { 302, PT_SC, ucp_Han },
-  { 306, PT_SC, ucp_Hangul },
-  { 313, PT_SC, ucp_Hanunoo },
-  { 321, PT_SC, ucp_Hebrew },
-  { 328, PT_SC, ucp_Hiragana },
-  { 337, PT_SC, ucp_Imperial_Aramaic },
-  { 354, PT_SC, ucp_Inherited },
-  { 364, PT_SC, ucp_Inscriptional_Pahlavi },
-  { 386, PT_SC, ucp_Inscriptional_Parthian },
-  { 409, PT_SC, ucp_Javanese },
-  { 418, PT_SC, ucp_Kaithi },
-  { 425, PT_SC, ucp_Kannada },
-  { 433, PT_SC, ucp_Katakana },
-  { 442, PT_SC, ucp_Kayah_Li },
-  { 451, PT_SC, ucp_Kharoshthi },
-  { 462, PT_SC, ucp_Khmer },
-  { 468, PT_GC, ucp_L },
-  { 470, PT_LAMP, 0 },
-  { 473, PT_SC, ucp_Lao },
-  { 477, PT_SC, ucp_Latin },
-  { 483, PT_SC, ucp_Lepcha },
-  { 490, PT_SC, ucp_Limbu },
-  { 496, PT_SC, ucp_Linear_B },
-  { 505, PT_SC, ucp_Lisu },
-  { 510, PT_PC, ucp_Ll },
-  { 513, PT_PC, ucp_Lm },
-  { 516, PT_PC, ucp_Lo },
-  { 519, PT_PC, ucp_Lt },
-  { 522, PT_PC, ucp_Lu },
-  { 525, PT_SC, ucp_Lycian },
-  { 532, PT_SC, ucp_Lydian },
-  { 539, PT_GC, ucp_M },
-  { 541, PT_SC, ucp_Malayalam },
-  { 551, PT_SC, ucp_Mandaic },
-  { 559, PT_PC, ucp_Mc },
-  { 562, PT_PC, ucp_Me },
-  { 565, PT_SC, ucp_Meetei_Mayek },
-  { 578, PT_SC, ucp_Meroitic_Cursive },
-  { 595, PT_SC, ucp_Meroitic_Hieroglyphs },
-  { 616, PT_SC, ucp_Miao },
-  { 621, PT_PC, ucp_Mn },
-  { 624, PT_SC, ucp_Mongolian },
-  { 634, PT_SC, ucp_Myanmar },
-  { 642, PT_GC, ucp_N },
-  { 644, PT_PC, ucp_Nd },
-  { 647, PT_SC, ucp_New_Tai_Lue },
-  { 659, PT_SC, ucp_Nko },
-  { 663, PT_PC, ucp_Nl },
-  { 666, PT_PC, ucp_No },
-  { 669, PT_SC, ucp_Ogham },
-  { 675, PT_SC, ucp_Ol_Chiki },
-  { 684, PT_SC, ucp_Old_Italic },
-  { 695, PT_SC, ucp_Old_Persian },
-  { 707, PT_SC, ucp_Old_South_Arabian },
-  { 725, PT_SC, ucp_Old_Turkic },
-  { 736, PT_SC, ucp_Oriya },
-  { 742, PT_SC, ucp_Osmanya },
-  { 750, PT_GC, ucp_P },
-  { 752, PT_PC, ucp_Pc },
-  { 755, PT_PC, ucp_Pd },
-  { 758, PT_PC, ucp_Pe },
-  { 761, PT_PC, ucp_Pf },
-  { 764, PT_SC, ucp_Phags_Pa },
-  { 773, PT_SC, ucp_Phoenician },
-  { 784, PT_PC, ucp_Pi },
-  { 787, PT_PC, ucp_Po },
-  { 790, PT_PC, ucp_Ps },
-  { 793, PT_SC, ucp_Rejang },
-  { 800, PT_SC, ucp_Runic },
-  { 806, PT_GC, ucp_S },
-  { 808, PT_SC, ucp_Samaritan },
-  { 818, PT_SC, ucp_Saurashtra },
-  { 829, PT_PC, ucp_Sc },
-  { 832, PT_SC, ucp_Sharada },
-  { 840, PT_SC, ucp_Shavian },
-  { 848, PT_SC, ucp_Sinhala },
-  { 856, PT_PC, ucp_Sk },
-  { 859, PT_PC, ucp_Sm },
-  { 862, PT_PC, ucp_So },
-  { 865, PT_SC, ucp_Sora_Sompeng },
-  { 878, PT_SC, ucp_Sundanese },
-  { 888, PT_SC, ucp_Syloti_Nagri },
-  { 901, PT_SC, ucp_Syriac },
-  { 908, PT_SC, ucp_Tagalog },
-  { 916, PT_SC, ucp_Tagbanwa },
-  { 925, PT_SC, ucp_Tai_Le },
-  { 932, PT_SC, ucp_Tai_Tham },
-  { 941, PT_SC, ucp_Tai_Viet },
-  { 950, PT_SC, ucp_Takri },
-  { 956, PT_SC, ucp_Tamil },
-  { 962, PT_SC, ucp_Telugu },
-  { 969, PT_SC, ucp_Thaana },
-  { 976, PT_SC, ucp_Thai },
-  { 981, PT_SC, ucp_Tibetan },
-  { 989, PT_SC, ucp_Tifinagh },
-  { 998, PT_SC, ucp_Ugaritic },
-  { 1007, PT_SC, ucp_Vai },
-  { 1011, PT_ALNUM, 0 },
-  { 1015, PT_PXSPACE, 0 },
-  { 1019, PT_SPACE, 0 },
-  { 1023, PT_UCNC, 0 },
-  { 1027, PT_WORD, 0 },
-  { 1031, PT_SC, ucp_Yi },
-  { 1034, PT_GC, ucp_Z },
-  { 1036, PT_PC, ucp_Zl },
-  { 1039, PT_PC, ucp_Zp },
-  { 1042, PT_PC, ucp_Zs }
+  {  43, PT_SC, ucp_Bassa_Vah },
+  {  53, PT_SC, ucp_Batak },
+  {  59, PT_SC, ucp_Bengali },
+  {  67, PT_SC, ucp_Bopomofo },
+  {  76, PT_SC, ucp_Brahmi },
+  {  83, PT_SC, ucp_Braille },
+  {  91, PT_SC, ucp_Buginese },
+  { 100, PT_SC, ucp_Buhid },
+  { 106, PT_GC, ucp_C },
+  { 108, PT_SC, ucp_Canadian_Aboriginal },
+  { 128, PT_SC, ucp_Carian },
+  { 135, PT_SC, ucp_Caucasian_Albanian },
+  { 154, PT_PC, ucp_Cc },
+  { 157, PT_PC, ucp_Cf },
+  { 160, PT_SC, ucp_Chakma },
+  { 167, PT_SC, ucp_Cham },
+  { 172, PT_SC, ucp_Cherokee },
+  { 181, PT_PC, ucp_Cn },
+  { 184, PT_PC, ucp_Co },
+  { 187, PT_SC, ucp_Common },
+  { 194, PT_SC, ucp_Coptic },
+  { 201, PT_PC, ucp_Cs },
+  { 204, PT_SC, ucp_Cuneiform },
+  { 214, PT_SC, ucp_Cypriot },
+  { 222, PT_SC, ucp_Cyrillic },
+  { 231, PT_SC, ucp_Deseret },
+  { 239, PT_SC, ucp_Devanagari },
+  { 250, PT_SC, ucp_Duployan },
+  { 259, PT_SC, ucp_Egyptian_Hieroglyphs },
+  { 280, PT_SC, ucp_Elbasan },
+  { 288, PT_SC, ucp_Ethiopic },
+  { 297, PT_SC, ucp_Georgian },
+  { 306, PT_SC, ucp_Glagolitic },
+  { 317, PT_SC, ucp_Gothic },
+  { 324, PT_SC, ucp_Grantha },
+  { 332, PT_SC, ucp_Greek },
+  { 338, PT_SC, ucp_Gujarati },
+  { 347, PT_SC, ucp_Gurmukhi },
+  { 356, PT_SC, ucp_Han },
+  { 360, PT_SC, ucp_Hangul },
+  { 367, PT_SC, ucp_Hanunoo },
+  { 375, PT_SC, ucp_Hebrew },
+  { 382, PT_SC, ucp_Hiragana },
+  { 391, PT_SC, ucp_Imperial_Aramaic },
+  { 408, PT_SC, ucp_Inherited },
+  { 418, PT_SC, ucp_Inscriptional_Pahlavi },
+  { 440, PT_SC, ucp_Inscriptional_Parthian },
+  { 463, PT_SC, ucp_Javanese },
+  { 472, PT_SC, ucp_Kaithi },
+  { 479, PT_SC, ucp_Kannada },
+  { 487, PT_SC, ucp_Katakana },
+  { 496, PT_SC, ucp_Kayah_Li },
+  { 505, PT_SC, ucp_Kharoshthi },
+  { 516, PT_SC, ucp_Khmer },
+  { 522, PT_SC, ucp_Khojki },
+  { 529, PT_SC, ucp_Khudawadi },
+  { 539, PT_GC, ucp_L },
+  { 541, PT_LAMP, 0 },
+  { 544, PT_SC, ucp_Lao },
+  { 548, PT_SC, ucp_Latin },
+  { 554, PT_SC, ucp_Lepcha },
+  { 561, PT_SC, ucp_Limbu },
+  { 567, PT_SC, ucp_Linear_A },
+  { 576, PT_SC, ucp_Linear_B },
+  { 585, PT_SC, ucp_Lisu },
+  { 590, PT_PC, ucp_Ll },
+  { 593, PT_PC, ucp_Lm },
+  { 596, PT_PC, ucp_Lo },
+  { 599, PT_PC, ucp_Lt },
+  { 602, PT_PC, ucp_Lu },
+  { 605, PT_SC, ucp_Lycian },
+  { 612, PT_SC, ucp_Lydian },
+  { 619, PT_GC, ucp_M },
+  { 621, PT_SC, ucp_Mahajani },
+  { 630, PT_SC, ucp_Malayalam },
+  { 640, PT_SC, ucp_Mandaic },
+  { 648, PT_SC, ucp_Manichaean },
+  { 659, PT_PC, ucp_Mc },
+  { 662, PT_PC, ucp_Me },
+  { 665, PT_SC, ucp_Meetei_Mayek },
+  { 678, PT_SC, ucp_Mende_Kikakui },
+  { 692, PT_SC, ucp_Meroitic_Cursive },
+  { 709, PT_SC, ucp_Meroitic_Hieroglyphs },
+  { 730, PT_SC, ucp_Miao },
+  { 735, PT_PC, ucp_Mn },
+  { 738, PT_SC, ucp_Modi },
+  { 743, PT_SC, ucp_Mongolian },
+  { 753, PT_SC, ucp_Mro },
+  { 757, PT_SC, ucp_Myanmar },
+  { 765, PT_GC, ucp_N },
+  { 767, PT_SC, ucp_Nabataean },
+  { 777, PT_PC, ucp_Nd },
+  { 780, PT_SC, ucp_New_Tai_Lue },
+  { 792, PT_SC, ucp_Nko },
+  { 796, PT_PC, ucp_Nl },
+  { 799, PT_PC, ucp_No },
+  { 802, PT_SC, ucp_Ogham },
+  { 808, PT_SC, ucp_Ol_Chiki },
+  { 817, PT_SC, ucp_Old_Italic },
+  { 828, PT_SC, ucp_Old_North_Arabian },
+  { 846, PT_SC, ucp_Old_Permic },
+  { 857, PT_SC, ucp_Old_Persian },
+  { 869, PT_SC, ucp_Old_South_Arabian },
+  { 887, PT_SC, ucp_Old_Turkic },
+  { 898, PT_SC, ucp_Oriya },
+  { 904, PT_SC, ucp_Osmanya },
+  { 912, PT_GC, ucp_P },
+  { 914, PT_SC, ucp_Pahawh_Hmong },
+  { 927, PT_SC, ucp_Palmyrene },
+  { 937, PT_SC, ucp_Pau_Cin_Hau },
+  { 949, PT_PC, ucp_Pc },
+  { 952, PT_PC, ucp_Pd },
+  { 955, PT_PC, ucp_Pe },
+  { 958, PT_PC, ucp_Pf },
+  { 961, PT_SC, ucp_Phags_Pa },
+  { 970, PT_SC, ucp_Phoenician },
+  { 981, PT_PC, ucp_Pi },
+  { 984, PT_PC, ucp_Po },
+  { 987, PT_PC, ucp_Ps },
+  { 990, PT_SC, ucp_Psalter_Pahlavi },
+  { 1006, PT_SC, ucp_Rejang },
+  { 1013, PT_SC, ucp_Runic },
+  { 1019, PT_GC, ucp_S },
+  { 1021, PT_SC, ucp_Samaritan },
+  { 1031, PT_SC, ucp_Saurashtra },
+  { 1042, PT_PC, ucp_Sc },
+  { 1045, PT_SC, ucp_Sharada },
+  { 1053, PT_SC, ucp_Shavian },
+  { 1061, PT_SC, ucp_Siddham },
+  { 1069, PT_SC, ucp_Sinhala },
+  { 1077, PT_PC, ucp_Sk },
+  { 1080, PT_PC, ucp_Sm },
+  { 1083, PT_PC, ucp_So },
+  { 1086, PT_SC, ucp_Sora_Sompeng },
+  { 1099, PT_SC, ucp_Sundanese },
+  { 1109, PT_SC, ucp_Syloti_Nagri },
+  { 1122, PT_SC, ucp_Syriac },
+  { 1129, PT_SC, ucp_Tagalog },
+  { 1137, PT_SC, ucp_Tagbanwa },
+  { 1146, PT_SC, ucp_Tai_Le },
+  { 1153, PT_SC, ucp_Tai_Tham },
+  { 1162, PT_SC, ucp_Tai_Viet },
+  { 1171, PT_SC, ucp_Takri },
+  { 1177, PT_SC, ucp_Tamil },
+  { 1183, PT_SC, ucp_Telugu },
+  { 1190, PT_SC, ucp_Thaana },
+  { 1197, PT_SC, ucp_Thai },
+  { 1202, PT_SC, ucp_Tibetan },
+  { 1210, PT_SC, ucp_Tifinagh },
+  { 1219, PT_SC, ucp_Tirhuta },
+  { 1227, PT_SC, ucp_Ugaritic },
+  { 1236, PT_SC, ucp_Vai },
+  { 1240, PT_SC, ucp_Warang_Citi },
+  { 1252, PT_ALNUM, 0 },
+  { 1256, PT_PXSPACE, 0 },
+  { 1260, PT_SPACE, 0 },
+  { 1264, PT_UCNC, 0 },
+  { 1268, PT_WORD, 0 },
+  { 1272, PT_SC, ucp_Yi },
+  { 1275, PT_GC, ucp_Z },
+  { 1277, PT_PC, ucp_Zl },
+  { 1280, PT_PC, ucp_Zp },
+  { 1283, PT_PC, ucp_Zs }
 };
 
 const int PRIV(utt_size) = sizeof(PRIV(utt)) / sizeof(ucp_type_table);
diff --git a/erts/emulator/pcre/pcre_ucd.c b/erts/emulator/pcre/pcre_ucd.c
index bdbc73c245..9b700c0785 100644
--- a/erts/emulator/pcre/pcre_ucd.c
+++ b/erts/emulator/pcre/pcre_ucd.c
@@ -20,7 +20,7 @@ needed. */
 
 /* Unicode character database. */
 /* This file was autogenerated by the MultiStage2.py script. */
-/* Total size: 65696 bytes, block size: 128. */
+/* Total size: 72576 bytes, block size: 128. */
 
 /* The tables herein are needed only when UCP support is built
 into PCRE. This module should not be referenced otherwise, so
@@ -79,7 +79,7 @@ const pcre_uint32 PRIV(ucd_caseless_sets)[] = {
 
 #ifndef PCRE_INCLUDED
 
-const ucd_record PRIV(ucd_records)[] = { /* 5024 bytes, record size 8 */
+const ucd_record PRIV(ucd_records)[] = { /* 5760 bytes, record size 8 */
   {     9,      0,      2,      0,      0, }, /*   0 */
   {     9,      0,      1,      0,      0, }, /*   1 */
   {     9,      0,      0,      0,      0, }, /*   2 */
@@ -166,548 +166,640 @@ const ucd_record PRIV(ucd_records)[] = { /* 5024 bytes, record size 8 */
   {    33,      5,     12,      0,   -205, }, /*  83 */
   {    33,      5,     12,      0,   -202, }, /*  84 */
   {    33,      5,     12,      0,   -203, }, /*  85 */
-  {    33,      5,     12,      0,   -207, }, /*  86 */
-  {    33,      5,     12,      0,  42280, }, /*  87 */
-  {    33,      5,     12,      0,  42308, }, /*  88 */
-  {    33,      5,     12,      0,   -209, }, /*  89 */
-  {    33,      5,     12,      0,   -211, }, /*  90 */
-  {    33,      5,     12,      0,  10743, }, /*  91 */
-  {    33,      5,     12,      0,  10749, }, /*  92 */
-  {    33,      5,     12,      0,   -213, }, /*  93 */
-  {    33,      5,     12,      0,   -214, }, /*  94 */
-  {    33,      5,     12,      0,  10727, }, /*  95 */
-  {    33,      5,     12,      0,   -218, }, /*  96 */
-  {    33,      5,     12,      0,    -69, }, /*  97 */
-  {    33,      5,     12,      0,   -217, }, /*  98 */
-  {    33,      5,     12,      0,    -71, }, /*  99 */
-  {    33,      5,     12,      0,   -219, }, /* 100 */
-  {    33,      6,     12,      0,      0, }, /* 101 */
-  {     9,      6,     12,      0,      0, }, /* 102 */
-  {     3,     24,     12,      0,      0, }, /* 103 */
-  {    27,     12,      3,      0,      0, }, /* 104 */
-  {    27,     12,      3,     21,    116, }, /* 105 */
-  {    19,      9,     12,      0,      1, }, /* 106 */
-  {    19,      5,     12,      0,     -1, }, /* 107 */
-  {    19,     24,     12,      0,      0, }, /* 108 */
-  {     9,      2,     12,      0,      0, }, /* 109 */
-  {    19,      6,     12,      0,      0, }, /* 110 */
-  {    19,      5,     12,      0,    130, }, /* 111 */
-  {    19,      9,     12,      0,     38, }, /* 112 */
-  {    19,      9,     12,      0,     37, }, /* 113 */
-  {    19,      9,     12,      0,     64, }, /* 114 */
-  {    19,      9,     12,      0,     63, }, /* 115 */
-  {    19,      5,     12,      0,      0, }, /* 116 */
-  {    19,      9,     12,      0,     32, }, /* 117 */
-  {    19,      9,     12,     34,     32, }, /* 118 */
-  {    19,      9,     12,     59,     32, }, /* 119 */
-  {    19,      9,     12,     38,     32, }, /* 120 */
-  {    19,      9,     12,     21,     32, }, /* 121 */
-  {    19,      9,     12,     51,     32, }, /* 122 */
-  {    19,      9,     12,     26,     32, }, /* 123 */
-  {    19,      9,     12,     47,     32, }, /* 124 */
-  {    19,      9,     12,     55,     32, }, /* 125 */
-  {    19,      9,     12,     30,     32, }, /* 126 */
-  {    19,      9,     12,     43,     32, }, /* 127 */
-  {    19,      9,     12,     67,     32, }, /* 128 */
-  {    19,      5,     12,      0,    -38, }, /* 129 */
-  {    19,      5,     12,      0,    -37, }, /* 130 */
-  {    19,      5,     12,      0,    -32, }, /* 131 */
-  {    19,      5,     12,     34,    -32, }, /* 132 */
-  {    19,      5,     12,     59,    -32, }, /* 133 */
-  {    19,      5,     12,     38,    -32, }, /* 134 */
-  {    19,      5,     12,     21,   -116, }, /* 135 */
-  {    19,      5,     12,     51,    -32, }, /* 136 */
-  {    19,      5,     12,     26,   -775, }, /* 137 */
-  {    19,      5,     12,     47,    -32, }, /* 138 */
-  {    19,      5,     12,     55,    -32, }, /* 139 */
-  {    19,      5,     12,     30,      1, }, /* 140 */
-  {    19,      5,     12,     30,    -32, }, /* 141 */
-  {    19,      5,     12,     43,    -32, }, /* 142 */
-  {    19,      5,     12,     67,    -32, }, /* 143 */
-  {    19,      5,     12,      0,    -64, }, /* 144 */
-  {    19,      5,     12,      0,    -63, }, /* 145 */
-  {    19,      9,     12,      0,      8, }, /* 146 */
-  {    19,      5,     12,     34,    -30, }, /* 147 */
-  {    19,      5,     12,     38,    -25, }, /* 148 */
-  {    19,      9,     12,      0,      0, }, /* 149 */
-  {    19,      5,     12,     43,    -15, }, /* 150 */
-  {    19,      5,     12,     47,    -22, }, /* 151 */
-  {    19,      5,     12,      0,     -8, }, /* 152 */
-  {    10,      9,     12,      0,      1, }, /* 153 */
-  {    10,      5,     12,      0,     -1, }, /* 154 */
-  {    19,      5,     12,     51,    -54, }, /* 155 */
-  {    19,      5,     12,     55,    -48, }, /* 156 */
-  {    19,      5,     12,      0,      7, }, /* 157 */
-  {    19,      9,     12,     38,    -60, }, /* 158 */
-  {    19,      5,     12,     59,    -64, }, /* 159 */
-  {    19,     25,     12,      0,      0, }, /* 160 */
-  {    19,      9,     12,      0,     -7, }, /* 161 */
-  {    19,      9,     12,      0,   -130, }, /* 162 */
-  {    12,      9,     12,      0,     80, }, /* 163 */
-  {    12,      9,     12,      0,     32, }, /* 164 */
-  {    12,      5,     12,      0,    -32, }, /* 165 */
-  {    12,      5,     12,      0,    -80, }, /* 166 */
-  {    12,      9,     12,      0,      1, }, /* 167 */
-  {    12,      5,     12,      0,     -1, }, /* 168 */
-  {    12,     26,     12,      0,      0, }, /* 169 */
-  {    12,     12,      3,      0,      0, }, /* 170 */
-  {    12,     11,      3,      0,      0, }, /* 171 */
-  {    12,      9,     12,      0,     15, }, /* 172 */
-  {    12,      5,     12,      0,    -15, }, /* 173 */
-  {     1,      9,     12,      0,     48, }, /* 174 */
-  {     1,      6,     12,      0,      0, }, /* 175 */
-  {     1,     21,     12,      0,      0, }, /* 176 */
-  {     1,      5,     12,      0,    -48, }, /* 177 */
-  {     1,      5,     12,      0,      0, }, /* 178 */
-  {     1,     17,     12,      0,      0, }, /* 179 */
-  {     1,     23,     12,      0,      0, }, /* 180 */
-  {    25,     12,      3,      0,      0, }, /* 181 */
-  {    25,     17,     12,      0,      0, }, /* 182 */
-  {    25,     21,     12,      0,      0, }, /* 183 */
-  {    25,      7,     12,      0,      0, }, /* 184 */
-  {     0,      1,      2,      0,      0, }, /* 185 */
-  {     0,     25,     12,      0,      0, }, /* 186 */
-  {     0,     21,     12,      0,      0, }, /* 187 */
-  {     0,     23,     12,      0,      0, }, /* 188 */
-  {     0,     26,     12,      0,      0, }, /* 189 */
-  {     0,     12,      3,      0,      0, }, /* 190 */
-  {     0,      7,     12,      0,      0, }, /* 191 */
-  {     0,      6,     12,      0,      0, }, /* 192 */
-  {     0,     13,     12,      0,      0, }, /* 193 */
-  {    49,     21,     12,      0,      0, }, /* 194 */
-  {    49,      1,      2,      0,      0, }, /* 195 */
-  {    49,      7,     12,      0,      0, }, /* 196 */
-  {    49,     12,      3,      0,      0, }, /* 197 */
-  {    55,      7,     12,      0,      0, }, /* 198 */
-  {    55,     12,      3,      0,      0, }, /* 199 */
-  {    63,     13,     12,      0,      0, }, /* 200 */
-  {    63,      7,     12,      0,      0, }, /* 201 */
-  {    63,     12,      3,      0,      0, }, /* 202 */
-  {    63,      6,     12,      0,      0, }, /* 203 */
-  {    63,     26,     12,      0,      0, }, /* 204 */
-  {    63,     21,     12,      0,      0, }, /* 205 */
-  {    89,      7,     12,      0,      0, }, /* 206 */
-  {    89,     12,      3,      0,      0, }, /* 207 */
-  {    89,      6,     12,      0,      0, }, /* 208 */
-  {    89,     21,     12,      0,      0, }, /* 209 */
-  {    94,      7,     12,      0,      0, }, /* 210 */
-  {    94,     12,      3,      0,      0, }, /* 211 */
-  {    94,     21,     12,      0,      0, }, /* 212 */
-  {    14,     12,      3,      0,      0, }, /* 213 */
-  {    14,     10,      5,      0,      0, }, /* 214 */
-  {    14,      7,     12,      0,      0, }, /* 215 */
-  {    14,     13,     12,      0,      0, }, /* 216 */
-  {    14,     21,     12,      0,      0, }, /* 217 */
-  {    14,      6,     12,      0,      0, }, /* 218 */
-  {     2,     12,      3,      0,      0, }, /* 219 */
-  {     2,     10,      5,      0,      0, }, /* 220 */
-  {     2,      7,     12,      0,      0, }, /* 221 */
-  {     2,     10,      3,      0,      0, }, /* 222 */
-  {     2,     13,     12,      0,      0, }, /* 223 */
-  {     2,     23,     12,      0,      0, }, /* 224 */
-  {     2,     15,     12,      0,      0, }, /* 225 */
-  {     2,     26,     12,      0,      0, }, /* 226 */
-  {    21,     12,      3,      0,      0, }, /* 227 */
-  {    21,     10,      5,      0,      0, }, /* 228 */
-  {    21,      7,     12,      0,      0, }, /* 229 */
-  {    21,     13,     12,      0,      0, }, /* 230 */
-  {    20,     12,      3,      0,      0, }, /* 231 */
-  {    20,     10,      5,      0,      0, }, /* 232 */
-  {    20,      7,     12,      0,      0, }, /* 233 */
-  {    20,     13,     12,      0,      0, }, /* 234 */
-  {    20,     21,     12,      0,      0, }, /* 235 */
-  {    20,     23,     12,      0,      0, }, /* 236 */
-  {    43,     12,      3,      0,      0, }, /* 237 */
-  {    43,     10,      5,      0,      0, }, /* 238 */
-  {    43,      7,     12,      0,      0, }, /* 239 */
-  {    43,     10,      3,      0,      0, }, /* 240 */
-  {    43,     13,     12,      0,      0, }, /* 241 */
-  {    43,     26,     12,      0,      0, }, /* 242 */
-  {    43,     15,     12,      0,      0, }, /* 243 */
-  {    53,     12,      3,      0,      0, }, /* 244 */
-  {    53,      7,     12,      0,      0, }, /* 245 */
-  {    53,     10,      3,      0,      0, }, /* 246 */
-  {    53,     10,      5,      0,      0, }, /* 247 */
-  {    53,     13,     12,      0,      0, }, /* 248 */
-  {    53,     15,     12,      0,      0, }, /* 249 */
-  {    53,     26,     12,      0,      0, }, /* 250 */
-  {    53,     23,     12,      0,      0, }, /* 251 */
-  {    54,     10,      5,      0,      0, }, /* 252 */
-  {    54,      7,     12,      0,      0, }, /* 253 */
-  {    54,     12,      3,      0,      0, }, /* 254 */
-  {    54,     13,     12,      0,      0, }, /* 255 */
-  {    54,     15,     12,      0,      0, }, /* 256 */
-  {    54,     26,     12,      0,      0, }, /* 257 */
-  {    28,     10,      5,      0,      0, }, /* 258 */
-  {    28,      7,     12,      0,      0, }, /* 259 */
-  {    28,     12,      3,      0,      0, }, /* 260 */
-  {    28,     10,      3,      0,      0, }, /* 261 */
-  {    28,     13,     12,      0,      0, }, /* 262 */
-  {    36,     10,      5,      0,      0, }, /* 263 */
-  {    36,      7,     12,      0,      0, }, /* 264 */
-  {    36,     10,      3,      0,      0, }, /* 265 */
-  {    36,     12,      3,      0,      0, }, /* 266 */
-  {    36,     13,     12,      0,      0, }, /* 267 */
-  {    36,     15,     12,      0,      0, }, /* 268 */
-  {    36,     26,     12,      0,      0, }, /* 269 */
-  {    47,     10,      5,      0,      0, }, /* 270 */
-  {    47,      7,     12,      0,      0, }, /* 271 */
-  {    47,     12,      3,      0,      0, }, /* 272 */
-  {    47,     10,      3,      0,      0, }, /* 273 */
-  {    47,     21,     12,      0,      0, }, /* 274 */
-  {    56,      7,     12,      0,      0, }, /* 275 */
-  {    56,     12,      3,      0,      0, }, /* 276 */
-  {    56,      7,      5,      0,      0, }, /* 277 */
-  {    56,      6,     12,      0,      0, }, /* 278 */
-  {    56,     21,     12,      0,      0, }, /* 279 */
-  {    56,     13,     12,      0,      0, }, /* 280 */
-  {    32,      7,     12,      0,      0, }, /* 281 */
-  {    32,     12,      3,      0,      0, }, /* 282 */
-  {    32,      7,      5,      0,      0, }, /* 283 */
-  {    32,      6,     12,      0,      0, }, /* 284 */
-  {    32,     13,     12,      0,      0, }, /* 285 */
-  {    57,      7,     12,      0,      0, }, /* 286 */
-  {    57,     26,     12,      0,      0, }, /* 287 */
-  {    57,     21,     12,      0,      0, }, /* 288 */
-  {    57,     12,      3,      0,      0, }, /* 289 */
-  {    57,     13,     12,      0,      0, }, /* 290 */
-  {    57,     15,     12,      0,      0, }, /* 291 */
-  {    57,     22,     12,      0,      0, }, /* 292 */
-  {    57,     18,     12,      0,      0, }, /* 293 */
-  {    57,     10,      5,      0,      0, }, /* 294 */
-  {    38,      7,     12,      0,      0, }, /* 295 */
-  {    38,     10,     12,      0,      0, }, /* 296 */
-  {    38,     12,      3,      0,      0, }, /* 297 */
-  {    38,     10,      5,      0,      0, }, /* 298 */
-  {    38,     13,     12,      0,      0, }, /* 299 */
-  {    38,     21,     12,      0,      0, }, /* 300 */
-  {    38,     26,     12,      0,      0, }, /* 301 */
-  {    16,      9,     12,      0,   7264, }, /* 302 */
-  {    16,      7,     12,      0,      0, }, /* 303 */
-  {    16,      6,     12,      0,      0, }, /* 304 */
-  {    23,      7,      6,      0,      0, }, /* 305 */
-  {    23,      7,      7,      0,      0, }, /* 306 */
-  {    23,      7,      8,      0,      0, }, /* 307 */
-  {    15,      7,     12,      0,      0, }, /* 308 */
-  {    15,     12,      3,      0,      0, }, /* 309 */
-  {    15,     21,     12,      0,      0, }, /* 310 */
-  {    15,     15,     12,      0,      0, }, /* 311 */
-  {    15,     26,     12,      0,      0, }, /* 312 */
-  {     8,      7,     12,      0,      0, }, /* 313 */
-  {     7,     17,     12,      0,      0, }, /* 314 */
-  {     7,      7,     12,      0,      0, }, /* 315 */
-  {     7,     21,     12,      0,      0, }, /* 316 */
-  {    40,     29,     12,      0,      0, }, /* 317 */
-  {    40,      7,     12,      0,      0, }, /* 318 */
-  {    40,     22,     12,      0,      0, }, /* 319 */
-  {    40,     18,     12,      0,      0, }, /* 320 */
-  {    45,      7,     12,      0,      0, }, /* 321 */
-  {    45,     14,     12,      0,      0, }, /* 322 */
-  {    50,      7,     12,      0,      0, }, /* 323 */
-  {    50,     12,      3,      0,      0, }, /* 324 */
-  {    24,      7,     12,      0,      0, }, /* 325 */
-  {    24,     12,      3,      0,      0, }, /* 326 */
-  {     6,      7,     12,      0,      0, }, /* 327 */
-  {     6,     12,      3,      0,      0, }, /* 328 */
-  {    51,      7,     12,      0,      0, }, /* 329 */
-  {    51,     12,      3,      0,      0, }, /* 330 */
-  {    31,      7,     12,      0,      0, }, /* 331 */
-  {    31,     12,      3,      0,      0, }, /* 332 */
-  {    31,     10,      5,      0,      0, }, /* 333 */
-  {    31,     21,     12,      0,      0, }, /* 334 */
-  {    31,      6,     12,      0,      0, }, /* 335 */
-  {    31,     23,     12,      0,      0, }, /* 336 */
-  {    31,     13,     12,      0,      0, }, /* 337 */
-  {    31,     15,     12,      0,      0, }, /* 338 */
-  {    37,     21,     12,      0,      0, }, /* 339 */
-  {    37,     17,     12,      0,      0, }, /* 340 */
-  {    37,     12,      3,      0,      0, }, /* 341 */
-  {    37,     29,     12,      0,      0, }, /* 342 */
-  {    37,     13,     12,      0,      0, }, /* 343 */
-  {    37,      7,     12,      0,      0, }, /* 344 */
-  {    37,      6,     12,      0,      0, }, /* 345 */
-  {    34,      7,     12,      0,      0, }, /* 346 */
-  {    34,     12,      3,      0,      0, }, /* 347 */
-  {    34,     10,      5,      0,      0, }, /* 348 */
-  {    34,     26,     12,      0,      0, }, /* 349 */
-  {    34,     21,     12,      0,      0, }, /* 350 */
-  {    34,     13,     12,      0,      0, }, /* 351 */
-  {    52,      7,     12,      0,      0, }, /* 352 */
-  {    39,      7,     12,      0,      0, }, /* 353 */
-  {    39,     10,     12,      0,      0, }, /* 354 */
-  {    39,     10,      5,      0,      0, }, /* 355 */
-  {    39,     13,     12,      0,      0, }, /* 356 */
-  {    39,     15,     12,      0,      0, }, /* 357 */
-  {    39,     26,     12,      0,      0, }, /* 358 */
-  {    31,     26,     12,      0,      0, }, /* 359 */
-  {     5,      7,     12,      0,      0, }, /* 360 */
-  {     5,     12,      3,      0,      0, }, /* 361 */
-  {     5,     10,      5,      0,      0, }, /* 362 */
-  {     5,     21,     12,      0,      0, }, /* 363 */
-  {    90,      7,     12,      0,      0, }, /* 364 */
-  {    90,     10,      5,      0,      0, }, /* 365 */
-  {    90,     12,      3,      0,      0, }, /* 366 */
-  {    90,     10,     12,      0,      0, }, /* 367 */
-  {    90,     13,     12,      0,      0, }, /* 368 */
-  {    90,     21,     12,      0,      0, }, /* 369 */
-  {    90,      6,     12,      0,      0, }, /* 370 */
-  {    61,     12,      3,      0,      0, }, /* 371 */
-  {    61,     10,      5,      0,      0, }, /* 372 */
-  {    61,      7,     12,      0,      0, }, /* 373 */
-  {    61,     13,     12,      0,      0, }, /* 374 */
-  {    61,     21,     12,      0,      0, }, /* 375 */
-  {    61,     26,     12,      0,      0, }, /* 376 */
-  {    75,     12,      3,      0,      0, }, /* 377 */
-  {    75,     10,      5,      0,      0, }, /* 378 */
-  {    75,      7,     12,      0,      0, }, /* 379 */
-  {    75,     13,     12,      0,      0, }, /* 380 */
-  {    92,      7,     12,      0,      0, }, /* 381 */
-  {    92,     12,      3,      0,      0, }, /* 382 */
-  {    92,     10,      5,      0,      0, }, /* 383 */
-  {    92,     21,     12,      0,      0, }, /* 384 */
-  {    69,      7,     12,      0,      0, }, /* 385 */
-  {    69,     10,      5,      0,      0, }, /* 386 */
-  {    69,     12,      3,      0,      0, }, /* 387 */
-  {    69,     21,     12,      0,      0, }, /* 388 */
-  {    69,     13,     12,      0,      0, }, /* 389 */
-  {    72,     13,     12,      0,      0, }, /* 390 */
-  {    72,      7,     12,      0,      0, }, /* 391 */
-  {    72,      6,     12,      0,      0, }, /* 392 */
-  {    72,     21,     12,      0,      0, }, /* 393 */
-  {    75,     21,     12,      0,      0, }, /* 394 */
-  {     9,     10,      5,      0,      0, }, /* 395 */
-  {     9,      7,     12,      0,      0, }, /* 396 */
-  {    12,      5,     12,      0,      0, }, /* 397 */
-  {    12,      6,     12,      0,      0, }, /* 398 */
-  {    33,      5,     12,      0,  35332, }, /* 399 */
-  {    33,      5,     12,      0,   3814, }, /* 400 */
-  {    33,      9,     12,     63,      1, }, /* 401 */
-  {    33,      5,     12,     63,     -1, }, /* 402 */
-  {    33,      5,     12,     63,    -58, }, /* 403 */
-  {    33,      9,     12,      0,  -7615, }, /* 404 */
-  {    19,      5,     12,      0,      8, }, /* 405 */
-  {    19,      9,     12,      0,     -8, }, /* 406 */
-  {    19,      5,     12,      0,     74, }, /* 407 */
-  {    19,      5,     12,      0,     86, }, /* 408 */
-  {    19,      5,     12,      0,    100, }, /* 409 */
-  {    19,      5,     12,      0,    128, }, /* 410 */
-  {    19,      5,     12,      0,    112, }, /* 411 */
-  {    19,      5,     12,      0,    126, }, /* 412 */
-  {    19,      8,     12,      0,     -8, }, /* 413 */
-  {    19,      5,     12,      0,      9, }, /* 414 */
-  {    19,      9,     12,      0,    -74, }, /* 415 */
-  {    19,      8,     12,      0,     -9, }, /* 416 */
-  {    19,      5,     12,     21,  -7173, }, /* 417 */
-  {    19,      9,     12,      0,    -86, }, /* 418 */
-  {    19,      9,     12,      0,   -100, }, /* 419 */
-  {    19,      9,     12,      0,   -112, }, /* 420 */
-  {    19,      9,     12,      0,   -128, }, /* 421 */
-  {    19,      9,     12,      0,   -126, }, /* 422 */
-  {    27,      1,      3,      0,      0, }, /* 423 */
-  {     9,     27,      2,      0,      0, }, /* 424 */
-  {     9,     28,      2,      0,      0, }, /* 425 */
-  {     9,      2,      2,      0,      0, }, /* 426 */
-  {    27,     11,      3,      0,      0, }, /* 427 */
-  {     9,      9,     12,      0,      0, }, /* 428 */
-  {     9,      5,     12,      0,      0, }, /* 429 */
-  {    19,      9,     12,     67,  -7517, }, /* 430 */
-  {    33,      9,     12,     71,  -8383, }, /* 431 */
-  {    33,      9,     12,     75,  -8262, }, /* 432 */
-  {    33,      9,     12,      0,     28, }, /* 433 */
-  {    33,      5,     12,      0,    -28, }, /* 434 */
-  {    33,     14,     12,      0,     16, }, /* 435 */
-  {    33,     14,     12,      0,    -16, }, /* 436 */
-  {    33,     14,     12,      0,      0, }, /* 437 */
-  {     9,     26,     12,      0,     26, }, /* 438 */
-  {     9,     26,     12,      0,    -26, }, /* 439 */
-  {     4,     26,     12,      0,      0, }, /* 440 */
-  {    17,      9,     12,      0,     48, }, /* 441 */
-  {    17,      5,     12,      0,    -48, }, /* 442 */
-  {    33,      9,     12,      0, -10743, }, /* 443 */
-  {    33,      9,     12,      0,  -3814, }, /* 444 */
-  {    33,      9,     12,      0, -10727, }, /* 445 */
-  {    33,      5,     12,      0, -10795, }, /* 446 */
-  {    33,      5,     12,      0, -10792, }, /* 447 */
-  {    33,      9,     12,      0, -10780, }, /* 448 */
-  {    33,      9,     12,      0, -10749, }, /* 449 */
-  {    33,      9,     12,      0, -10783, }, /* 450 */
-  {    33,      9,     12,      0, -10782, }, /* 451 */
-  {    33,      9,     12,      0, -10815, }, /* 452 */
-  {    10,      5,     12,      0,      0, }, /* 453 */
-  {    10,     26,     12,      0,      0, }, /* 454 */
-  {    10,     12,      3,      0,      0, }, /* 455 */
-  {    10,     21,     12,      0,      0, }, /* 456 */
-  {    10,     15,     12,      0,      0, }, /* 457 */
-  {    16,      5,     12,      0,  -7264, }, /* 458 */
-  {    58,      7,     12,      0,      0, }, /* 459 */
-  {    58,      6,     12,      0,      0, }, /* 460 */
-  {    58,     21,     12,      0,      0, }, /* 461 */
-  {    58,     12,      3,      0,      0, }, /* 462 */
-  {    22,     26,     12,      0,      0, }, /* 463 */
-  {    22,      6,     12,      0,      0, }, /* 464 */
-  {    22,     14,     12,      0,      0, }, /* 465 */
-  {    23,     10,      3,      0,      0, }, /* 466 */
-  {    26,      7,     12,      0,      0, }, /* 467 */
-  {    26,      6,     12,      0,      0, }, /* 468 */
-  {    29,      7,     12,      0,      0, }, /* 469 */
-  {    29,      6,     12,      0,      0, }, /* 470 */
-  {     3,      7,     12,      0,      0, }, /* 471 */
-  {    23,      7,     12,      0,      0, }, /* 472 */
-  {    23,     26,     12,      0,      0, }, /* 473 */
-  {    29,     26,     12,      0,      0, }, /* 474 */
-  {    22,      7,     12,      0,      0, }, /* 475 */
-  {    60,      7,     12,      0,      0, }, /* 476 */
-  {    60,      6,     12,      0,      0, }, /* 477 */
-  {    60,     26,     12,      0,      0, }, /* 478 */
-  {    85,      7,     12,      0,      0, }, /* 479 */
-  {    85,      6,     12,      0,      0, }, /* 480 */
-  {    85,     21,     12,      0,      0, }, /* 481 */
-  {    76,      7,     12,      0,      0, }, /* 482 */
-  {    76,      6,     12,      0,      0, }, /* 483 */
-  {    76,     21,     12,      0,      0, }, /* 484 */
-  {    76,     13,     12,      0,      0, }, /* 485 */
-  {    12,      7,     12,      0,      0, }, /* 486 */
-  {    12,     21,     12,      0,      0, }, /* 487 */
-  {    78,      7,     12,      0,      0, }, /* 488 */
-  {    78,     14,     12,      0,      0, }, /* 489 */
-  {    78,     12,      3,      0,      0, }, /* 490 */
-  {    78,     21,     12,      0,      0, }, /* 491 */
-  {    33,      9,     12,      0, -35332, }, /* 492 */
-  {    33,      9,     12,      0, -42280, }, /* 493 */
-  {    33,      9,     12,      0, -42308, }, /* 494 */
-  {    48,      7,     12,      0,      0, }, /* 495 */
-  {    48,     12,      3,      0,      0, }, /* 496 */
-  {    48,     10,      5,      0,      0, }, /* 497 */
-  {    48,     26,     12,      0,      0, }, /* 498 */
-  {    64,      7,     12,      0,      0, }, /* 499 */
-  {    64,     21,     12,      0,      0, }, /* 500 */
-  {    74,     10,      5,      0,      0, }, /* 501 */
-  {    74,      7,     12,      0,      0, }, /* 502 */
-  {    74,     12,      3,      0,      0, }, /* 503 */
-  {    74,     21,     12,      0,      0, }, /* 504 */
-  {    74,     13,     12,      0,      0, }, /* 505 */
-  {    68,     13,     12,      0,      0, }, /* 506 */
-  {    68,      7,     12,      0,      0, }, /* 507 */
-  {    68,     12,      3,      0,      0, }, /* 508 */
-  {    68,     21,     12,      0,      0, }, /* 509 */
-  {    73,      7,     12,      0,      0, }, /* 510 */
-  {    73,     12,      3,      0,      0, }, /* 511 */
-  {    73,     10,      5,      0,      0, }, /* 512 */
-  {    73,     21,     12,      0,      0, }, /* 513 */
-  {    83,     12,      3,      0,      0, }, /* 514 */
-  {    83,     10,      5,      0,      0, }, /* 515 */
-  {    83,      7,     12,      0,      0, }, /* 516 */
-  {    83,     21,     12,      0,      0, }, /* 517 */
-  {    83,      6,     12,      0,      0, }, /* 518 */
-  {    83,     13,     12,      0,      0, }, /* 519 */
-  {    67,      7,     12,      0,      0, }, /* 520 */
-  {    67,     12,      3,      0,      0, }, /* 521 */
-  {    67,     10,      5,      0,      0, }, /* 522 */
-  {    67,     13,     12,      0,      0, }, /* 523 */
-  {    67,     21,     12,      0,      0, }, /* 524 */
-  {    38,      6,     12,      0,      0, }, /* 525 */
-  {    91,      7,     12,      0,      0, }, /* 526 */
-  {    91,     12,      3,      0,      0, }, /* 527 */
-  {    91,      6,     12,      0,      0, }, /* 528 */
-  {    91,     21,     12,      0,      0, }, /* 529 */
-  {    86,      7,     12,      0,      0, }, /* 530 */
-  {    86,     10,      5,      0,      0, }, /* 531 */
-  {    86,     12,      3,      0,      0, }, /* 532 */
-  {    86,     21,     12,      0,      0, }, /* 533 */
-  {    86,      6,     12,      0,      0, }, /* 534 */
-  {    86,     13,     12,      0,      0, }, /* 535 */
-  {    23,      7,      9,      0,      0, }, /* 536 */
-  {    23,      7,     10,      0,      0, }, /* 537 */
-  {     9,      4,      2,      0,      0, }, /* 538 */
-  {     9,      3,     12,      0,      0, }, /* 539 */
-  {    25,     25,     12,      0,      0, }, /* 540 */
-  {     0,     24,     12,      0,      0, }, /* 541 */
-  {     9,      6,      3,      0,      0, }, /* 542 */
-  {    35,      7,     12,      0,      0, }, /* 543 */
-  {    19,     14,     12,      0,      0, }, /* 544 */
-  {    19,     15,     12,      0,      0, }, /* 545 */
-  {    19,     26,     12,      0,      0, }, /* 546 */
-  {    70,      7,     12,      0,      0, }, /* 547 */
-  {    66,      7,     12,      0,      0, }, /* 548 */
-  {    41,      7,     12,      0,      0, }, /* 549 */
-  {    41,     15,     12,      0,      0, }, /* 550 */
-  {    18,      7,     12,      0,      0, }, /* 551 */
-  {    18,     14,     12,      0,      0, }, /* 552 */
-  {    59,      7,     12,      0,      0, }, /* 553 */
-  {    59,     21,     12,      0,      0, }, /* 554 */
-  {    42,      7,     12,      0,      0, }, /* 555 */
-  {    42,     21,     12,      0,      0, }, /* 556 */
-  {    42,     14,     12,      0,      0, }, /* 557 */
-  {    13,      9,     12,      0,     40, }, /* 558 */
-  {    13,      5,     12,      0,    -40, }, /* 559 */
-  {    46,      7,     12,      0,      0, }, /* 560 */
-  {    44,      7,     12,      0,      0, }, /* 561 */
-  {    44,     13,     12,      0,      0, }, /* 562 */
-  {    11,      7,     12,      0,      0, }, /* 563 */
-  {    80,      7,     12,      0,      0, }, /* 564 */
-  {    80,     21,     12,      0,      0, }, /* 565 */
-  {    80,     15,     12,      0,      0, }, /* 566 */
-  {    65,      7,     12,      0,      0, }, /* 567 */
-  {    65,     15,     12,      0,      0, }, /* 568 */
-  {    65,     21,     12,      0,      0, }, /* 569 */
-  {    71,      7,     12,      0,      0, }, /* 570 */
-  {    71,     21,     12,      0,      0, }, /* 571 */
-  {    97,      7,     12,      0,      0, }, /* 572 */
-  {    96,      7,     12,      0,      0, }, /* 573 */
-  {    30,      7,     12,      0,      0, }, /* 574 */
-  {    30,     12,      3,      0,      0, }, /* 575 */
-  {    30,     15,     12,      0,      0, }, /* 576 */
-  {    30,     21,     12,      0,      0, }, /* 577 */
-  {    87,      7,     12,      0,      0, }, /* 578 */
-  {    87,     15,     12,      0,      0, }, /* 579 */
-  {    87,     21,     12,      0,      0, }, /* 580 */
-  {    77,      7,     12,      0,      0, }, /* 581 */
-  {    77,     21,     12,      0,      0, }, /* 582 */
-  {    82,      7,     12,      0,      0, }, /* 583 */
-  {    82,     15,     12,      0,      0, }, /* 584 */
-  {    81,      7,     12,      0,      0, }, /* 585 */
-  {    81,     15,     12,      0,      0, }, /* 586 */
-  {    88,      7,     12,      0,      0, }, /* 587 */
-  {     0,     15,     12,      0,      0, }, /* 588 */
-  {    93,     10,      5,      0,      0, }, /* 589 */
-  {    93,     12,      3,      0,      0, }, /* 590 */
-  {    93,      7,     12,      0,      0, }, /* 591 */
-  {    93,     21,     12,      0,      0, }, /* 592 */
-  {    93,     15,     12,      0,      0, }, /* 593 */
-  {    93,     13,     12,      0,      0, }, /* 594 */
-  {    84,     12,      3,      0,      0, }, /* 595 */
-  {    84,     10,      5,      0,      0, }, /* 596 */
-  {    84,      7,     12,      0,      0, }, /* 597 */
-  {    84,     21,     12,      0,      0, }, /* 598 */
-  {    84,      1,      2,      0,      0, }, /* 599 */
-  {   100,      7,     12,      0,      0, }, /* 600 */
-  {   100,     13,     12,      0,      0, }, /* 601 */
-  {    95,     12,      3,      0,      0, }, /* 602 */
-  {    95,      7,     12,      0,      0, }, /* 603 */
-  {    95,     10,      5,      0,      0, }, /* 604 */
-  {    95,     13,     12,      0,      0, }, /* 605 */
-  {    95,     21,     12,      0,      0, }, /* 606 */
-  {    99,     12,      3,      0,      0, }, /* 607 */
-  {    99,     10,      5,      0,      0, }, /* 608 */
-  {    99,      7,     12,      0,      0, }, /* 609 */
-  {    99,     21,     12,      0,      0, }, /* 610 */
-  {    99,     13,     12,      0,      0, }, /* 611 */
-  {   101,      7,     12,      0,      0, }, /* 612 */
-  {   101,     12,      3,      0,      0, }, /* 613 */
-  {   101,     10,      5,      0,      0, }, /* 614 */
-  {   101,     13,     12,      0,      0, }, /* 615 */
-  {    62,      7,     12,      0,      0, }, /* 616 */
-  {    62,     14,     12,      0,      0, }, /* 617 */
-  {    62,     21,     12,      0,      0, }, /* 618 */
-  {    79,      7,     12,      0,      0, }, /* 619 */
-  {    98,      7,     12,      0,      0, }, /* 620 */
-  {    98,     10,      5,      0,      0, }, /* 621 */
-  {    98,     12,      3,      0,      0, }, /* 622 */
-  {    98,      6,     12,      0,      0, }, /* 623 */
-  {     9,     10,      3,      0,      0, }, /* 624 */
-  {    19,     12,      3,      0,      0, }, /* 625 */
-  {     9,     26,     11,      0,      0, }, /* 626 */
-  {    26,     26,     12,      0,      0, }, /* 627 */
+  {    33,      5,     12,      0,  42319, }, /*  86 */
+  {    33,      5,     12,      0,  42315, }, /*  87 */
+  {    33,      5,     12,      0,   -207, }, /*  88 */
+  {    33,      5,     12,      0,  42280, }, /*  89 */
+  {    33,      5,     12,      0,  42308, }, /*  90 */
+  {    33,      5,     12,      0,   -209, }, /*  91 */
+  {    33,      5,     12,      0,   -211, }, /*  92 */
+  {    33,      5,     12,      0,  10743, }, /*  93 */
+  {    33,      5,     12,      0,  42305, }, /*  94 */
+  {    33,      5,     12,      0,  10749, }, /*  95 */
+  {    33,      5,     12,      0,   -213, }, /*  96 */
+  {    33,      5,     12,      0,   -214, }, /*  97 */
+  {    33,      5,     12,      0,  10727, }, /*  98 */
+  {    33,      5,     12,      0,   -218, }, /*  99 */
+  {    33,      5,     12,      0,  42282, }, /* 100 */
+  {    33,      5,     12,      0,    -69, }, /* 101 */
+  {    33,      5,     12,      0,   -217, }, /* 102 */
+  {    33,      5,     12,      0,    -71, }, /* 103 */
+  {    33,      5,     12,      0,   -219, }, /* 104 */
+  {    33,      5,     12,      0,  42258, }, /* 105 */
+  {    33,      6,     12,      0,      0, }, /* 106 */
+  {     9,      6,     12,      0,      0, }, /* 107 */
+  {     3,     24,     12,      0,      0, }, /* 108 */
+  {    27,     12,      3,      0,      0, }, /* 109 */
+  {    27,     12,      3,     21,    116, }, /* 110 */
+  {    19,      9,     12,      0,      1, }, /* 111 */
+  {    19,      5,     12,      0,     -1, }, /* 112 */
+  {    19,     24,     12,      0,      0, }, /* 113 */
+  {     9,      2,     12,      0,      0, }, /* 114 */
+  {    19,      6,     12,      0,      0, }, /* 115 */
+  {    19,      5,     12,      0,    130, }, /* 116 */
+  {    19,      9,     12,      0,    116, }, /* 117 */
+  {    19,      9,     12,      0,     38, }, /* 118 */
+  {    19,      9,     12,      0,     37, }, /* 119 */
+  {    19,      9,     12,      0,     64, }, /* 120 */
+  {    19,      9,     12,      0,     63, }, /* 121 */
+  {    19,      5,     12,      0,      0, }, /* 122 */
+  {    19,      9,     12,      0,     32, }, /* 123 */
+  {    19,      9,     12,     34,     32, }, /* 124 */
+  {    19,      9,     12,     59,     32, }, /* 125 */
+  {    19,      9,     12,     38,     32, }, /* 126 */
+  {    19,      9,     12,     21,     32, }, /* 127 */
+  {    19,      9,     12,     51,     32, }, /* 128 */
+  {    19,      9,     12,     26,     32, }, /* 129 */
+  {    19,      9,     12,     47,     32, }, /* 130 */
+  {    19,      9,     12,     55,     32, }, /* 131 */
+  {    19,      9,     12,     30,     32, }, /* 132 */
+  {    19,      9,     12,     43,     32, }, /* 133 */
+  {    19,      9,     12,     67,     32, }, /* 134 */
+  {    19,      5,     12,      0,    -38, }, /* 135 */
+  {    19,      5,     12,      0,    -37, }, /* 136 */
+  {    19,      5,     12,      0,    -32, }, /* 137 */
+  {    19,      5,     12,     34,    -32, }, /* 138 */
+  {    19,      5,     12,     59,    -32, }, /* 139 */
+  {    19,      5,     12,     38,    -32, }, /* 140 */
+  {    19,      5,     12,     21,   -116, }, /* 141 */
+  {    19,      5,     12,     51,    -32, }, /* 142 */
+  {    19,      5,     12,     26,   -775, }, /* 143 */
+  {    19,      5,     12,     47,    -32, }, /* 144 */
+  {    19,      5,     12,     55,    -32, }, /* 145 */
+  {    19,      5,     12,     30,      1, }, /* 146 */
+  {    19,      5,     12,     30,    -32, }, /* 147 */
+  {    19,      5,     12,     43,    -32, }, /* 148 */
+  {    19,      5,     12,     67,    -32, }, /* 149 */
+  {    19,      5,     12,      0,    -64, }, /* 150 */
+  {    19,      5,     12,      0,    -63, }, /* 151 */
+  {    19,      9,     12,      0,      8, }, /* 152 */
+  {    19,      5,     12,     34,    -30, }, /* 153 */
+  {    19,      5,     12,     38,    -25, }, /* 154 */
+  {    19,      9,     12,      0,      0, }, /* 155 */
+  {    19,      5,     12,     43,    -15, }, /* 156 */
+  {    19,      5,     12,     47,    -22, }, /* 157 */
+  {    19,      5,     12,      0,     -8, }, /* 158 */
+  {    10,      9,     12,      0,      1, }, /* 159 */
+  {    10,      5,     12,      0,     -1, }, /* 160 */
+  {    19,      5,     12,     51,    -54, }, /* 161 */
+  {    19,      5,     12,     55,    -48, }, /* 162 */
+  {    19,      5,     12,      0,      7, }, /* 163 */
+  {    19,      5,     12,      0,   -116, }, /* 164 */
+  {    19,      9,     12,     38,    -60, }, /* 165 */
+  {    19,      5,     12,     59,    -64, }, /* 166 */
+  {    19,     25,     12,      0,      0, }, /* 167 */
+  {    19,      9,     12,      0,     -7, }, /* 168 */
+  {    19,      9,     12,      0,   -130, }, /* 169 */
+  {    12,      9,     12,      0,     80, }, /* 170 */
+  {    12,      9,     12,      0,     32, }, /* 171 */
+  {    12,      5,     12,      0,    -32, }, /* 172 */
+  {    12,      5,     12,      0,    -80, }, /* 173 */
+  {    12,      9,     12,      0,      1, }, /* 174 */
+  {    12,      5,     12,      0,     -1, }, /* 175 */
+  {    12,     26,     12,      0,      0, }, /* 176 */
+  {    12,     12,      3,      0,      0, }, /* 177 */
+  {    12,     11,      3,      0,      0, }, /* 178 */
+  {    12,      9,     12,      0,     15, }, /* 179 */
+  {    12,      5,     12,      0,    -15, }, /* 180 */
+  {     1,      9,     12,      0,     48, }, /* 181 */
+  {     1,      6,     12,      0,      0, }, /* 182 */
+  {     1,     21,     12,      0,      0, }, /* 183 */
+  {     1,      5,     12,      0,    -48, }, /* 184 */
+  {     1,      5,     12,      0,      0, }, /* 185 */
+  {     1,     17,     12,      0,      0, }, /* 186 */
+  {     1,     26,     12,      0,      0, }, /* 187 */
+  {     1,     23,     12,      0,      0, }, /* 188 */
+  {    25,     12,      3,      0,      0, }, /* 189 */
+  {    25,     17,     12,      0,      0, }, /* 190 */
+  {    25,     21,     12,      0,      0, }, /* 191 */
+  {    25,      7,     12,      0,      0, }, /* 192 */
+  {     0,      1,      2,      0,      0, }, /* 193 */
+  {     0,     25,     12,      0,      0, }, /* 194 */
+  {     0,     21,     12,      0,      0, }, /* 195 */
+  {     0,     23,     12,      0,      0, }, /* 196 */
+  {     0,     26,     12,      0,      0, }, /* 197 */
+  {     0,     12,      3,      0,      0, }, /* 198 */
+  {     0,      7,     12,      0,      0, }, /* 199 */
+  {     0,      6,     12,      0,      0, }, /* 200 */
+  {     0,     13,     12,      0,      0, }, /* 201 */
+  {    49,     21,     12,      0,      0, }, /* 202 */
+  {    49,      1,      2,      0,      0, }, /* 203 */
+  {    49,      7,     12,      0,      0, }, /* 204 */
+  {    49,     12,      3,      0,      0, }, /* 205 */
+  {    55,      7,     12,      0,      0, }, /* 206 */
+  {    55,     12,      3,      0,      0, }, /* 207 */
+  {    63,     13,     12,      0,      0, }, /* 208 */
+  {    63,      7,     12,      0,      0, }, /* 209 */
+  {    63,     12,      3,      0,      0, }, /* 210 */
+  {    63,      6,     12,      0,      0, }, /* 211 */
+  {    63,     26,     12,      0,      0, }, /* 212 */
+  {    63,     21,     12,      0,      0, }, /* 213 */
+  {    89,      7,     12,      0,      0, }, /* 214 */
+  {    89,     12,      3,      0,      0, }, /* 215 */
+  {    89,      6,     12,      0,      0, }, /* 216 */
+  {    89,     21,     12,      0,      0, }, /* 217 */
+  {    94,      7,     12,      0,      0, }, /* 218 */
+  {    94,     12,      3,      0,      0, }, /* 219 */
+  {    94,     21,     12,      0,      0, }, /* 220 */
+  {    14,     12,      3,      0,      0, }, /* 221 */
+  {    14,     10,      5,      0,      0, }, /* 222 */
+  {    14,      7,     12,      0,      0, }, /* 223 */
+  {    14,     13,     12,      0,      0, }, /* 224 */
+  {    14,     21,     12,      0,      0, }, /* 225 */
+  {    14,      6,     12,      0,      0, }, /* 226 */
+  {     2,      7,     12,      0,      0, }, /* 227 */
+  {     2,     12,      3,      0,      0, }, /* 228 */
+  {     2,     10,      5,      0,      0, }, /* 229 */
+  {     2,     10,      3,      0,      0, }, /* 230 */
+  {     2,     13,     12,      0,      0, }, /* 231 */
+  {     2,     23,     12,      0,      0, }, /* 232 */
+  {     2,     15,     12,      0,      0, }, /* 233 */
+  {     2,     26,     12,      0,      0, }, /* 234 */
+  {    21,     12,      3,      0,      0, }, /* 235 */
+  {    21,     10,      5,      0,      0, }, /* 236 */
+  {    21,      7,     12,      0,      0, }, /* 237 */
+  {    21,     13,     12,      0,      0, }, /* 238 */
+  {    20,     12,      3,      0,      0, }, /* 239 */
+  {    20,     10,      5,      0,      0, }, /* 240 */
+  {    20,      7,     12,      0,      0, }, /* 241 */
+  {    20,     13,     12,      0,      0, }, /* 242 */
+  {    20,     21,     12,      0,      0, }, /* 243 */
+  {    20,     23,     12,      0,      0, }, /* 244 */
+  {    43,     12,      3,      0,      0, }, /* 245 */
+  {    43,     10,      5,      0,      0, }, /* 246 */
+  {    43,      7,     12,      0,      0, }, /* 247 */
+  {    43,     10,      3,      0,      0, }, /* 248 */
+  {    43,     13,     12,      0,      0, }, /* 249 */
+  {    43,     26,     12,      0,      0, }, /* 250 */
+  {    43,     15,     12,      0,      0, }, /* 251 */
+  {    53,     12,      3,      0,      0, }, /* 252 */
+  {    53,      7,     12,      0,      0, }, /* 253 */
+  {    53,     10,      3,      0,      0, }, /* 254 */
+  {    53,     10,      5,      0,      0, }, /* 255 */
+  {    53,     13,     12,      0,      0, }, /* 256 */
+  {    53,     15,     12,      0,      0, }, /* 257 */
+  {    53,     26,     12,      0,      0, }, /* 258 */
+  {    53,     23,     12,      0,      0, }, /* 259 */
+  {    54,     12,      3,      0,      0, }, /* 260 */
+  {    54,     10,      5,      0,      0, }, /* 261 */
+  {    54,      7,     12,      0,      0, }, /* 262 */
+  {    54,     13,     12,      0,      0, }, /* 263 */
+  {    54,     15,     12,      0,      0, }, /* 264 */
+  {    54,     26,     12,      0,      0, }, /* 265 */
+  {    28,     12,      3,      0,      0, }, /* 266 */
+  {    28,     10,      5,      0,      0, }, /* 267 */
+  {    28,      7,     12,      0,      0, }, /* 268 */
+  {    28,     10,      3,      0,      0, }, /* 269 */
+  {    28,     13,     12,      0,      0, }, /* 270 */
+  {    36,     12,      3,      0,      0, }, /* 271 */
+  {    36,     10,      5,      0,      0, }, /* 272 */
+  {    36,      7,     12,      0,      0, }, /* 273 */
+  {    36,     10,      3,      0,      0, }, /* 274 */
+  {    36,     13,     12,      0,      0, }, /* 275 */
+  {    36,     15,     12,      0,      0, }, /* 276 */
+  {    36,     26,     12,      0,      0, }, /* 277 */
+  {    47,     10,      5,      0,      0, }, /* 278 */
+  {    47,      7,     12,      0,      0, }, /* 279 */
+  {    47,     12,      3,      0,      0, }, /* 280 */
+  {    47,     10,      3,      0,      0, }, /* 281 */
+  {    47,     13,     12,      0,      0, }, /* 282 */
+  {    47,     21,     12,      0,      0, }, /* 283 */
+  {    56,      7,     12,      0,      0, }, /* 284 */
+  {    56,     12,      3,      0,      0, }, /* 285 */
+  {    56,      7,      5,      0,      0, }, /* 286 */
+  {    56,      6,     12,      0,      0, }, /* 287 */
+  {    56,     21,     12,      0,      0, }, /* 288 */
+  {    56,     13,     12,      0,      0, }, /* 289 */
+  {    32,      7,     12,      0,      0, }, /* 290 */
+  {    32,     12,      3,      0,      0, }, /* 291 */
+  {    32,      7,      5,      0,      0, }, /* 292 */
+  {    32,      6,     12,      0,      0, }, /* 293 */
+  {    32,     13,     12,      0,      0, }, /* 294 */
+  {    57,      7,     12,      0,      0, }, /* 295 */
+  {    57,     26,     12,      0,      0, }, /* 296 */
+  {    57,     21,     12,      0,      0, }, /* 297 */
+  {    57,     12,      3,      0,      0, }, /* 298 */
+  {    57,     13,     12,      0,      0, }, /* 299 */
+  {    57,     15,     12,      0,      0, }, /* 300 */
+  {    57,     22,     12,      0,      0, }, /* 301 */
+  {    57,     18,     12,      0,      0, }, /* 302 */
+  {    57,     10,      5,      0,      0, }, /* 303 */
+  {    38,      7,     12,      0,      0, }, /* 304 */
+  {    38,     10,     12,      0,      0, }, /* 305 */
+  {    38,     12,      3,      0,      0, }, /* 306 */
+  {    38,     10,      5,      0,      0, }, /* 307 */
+  {    38,     13,     12,      0,      0, }, /* 308 */
+  {    38,     21,     12,      0,      0, }, /* 309 */
+  {    38,     26,     12,      0,      0, }, /* 310 */
+  {    16,      9,     12,      0,   7264, }, /* 311 */
+  {    16,      7,     12,      0,      0, }, /* 312 */
+  {    16,      6,     12,      0,      0, }, /* 313 */
+  {    23,      7,      6,      0,      0, }, /* 314 */
+  {    23,      7,      7,      0,      0, }, /* 315 */
+  {    23,      7,      8,      0,      0, }, /* 316 */
+  {    15,      7,     12,      0,      0, }, /* 317 */
+  {    15,     12,      3,      0,      0, }, /* 318 */
+  {    15,     21,     12,      0,      0, }, /* 319 */
+  {    15,     15,     12,      0,      0, }, /* 320 */
+  {    15,     26,     12,      0,      0, }, /* 321 */
+  {     8,      7,     12,      0,      0, }, /* 322 */
+  {     7,     17,     12,      0,      0, }, /* 323 */
+  {     7,      7,     12,      0,      0, }, /* 324 */
+  {     7,     21,     12,      0,      0, }, /* 325 */
+  {    40,     29,     12,      0,      0, }, /* 326 */
+  {    40,      7,     12,      0,      0, }, /* 327 */
+  {    40,     22,     12,      0,      0, }, /* 328 */
+  {    40,     18,     12,      0,      0, }, /* 329 */
+  {    45,      7,     12,      0,      0, }, /* 330 */
+  {    45,     14,     12,      0,      0, }, /* 331 */
+  {    50,      7,     12,      0,      0, }, /* 332 */
+  {    50,     12,      3,      0,      0, }, /* 333 */
+  {    24,      7,     12,      0,      0, }, /* 334 */
+  {    24,     12,      3,      0,      0, }, /* 335 */
+  {     6,      7,     12,      0,      0, }, /* 336 */
+  {     6,     12,      3,      0,      0, }, /* 337 */
+  {    51,      7,     12,      0,      0, }, /* 338 */
+  {    51,     12,      3,      0,      0, }, /* 339 */
+  {    31,      7,     12,      0,      0, }, /* 340 */
+  {    31,     12,      3,      0,      0, }, /* 341 */
+  {    31,     10,      5,      0,      0, }, /* 342 */
+  {    31,     21,     12,      0,      0, }, /* 343 */
+  {    31,      6,     12,      0,      0, }, /* 344 */
+  {    31,     23,     12,      0,      0, }, /* 345 */
+  {    31,     13,     12,      0,      0, }, /* 346 */
+  {    31,     15,     12,      0,      0, }, /* 347 */
+  {    37,     21,     12,      0,      0, }, /* 348 */
+  {    37,     17,     12,      0,      0, }, /* 349 */
+  {    37,     12,      3,      0,      0, }, /* 350 */
+  {    37,      1,      2,      0,      0, }, /* 351 */
+  {    37,     13,     12,      0,      0, }, /* 352 */
+  {    37,      7,     12,      0,      0, }, /* 353 */
+  {    37,      6,     12,      0,      0, }, /* 354 */
+  {    34,      7,     12,      0,      0, }, /* 355 */
+  {    34,     12,      3,      0,      0, }, /* 356 */
+  {    34,     10,      5,      0,      0, }, /* 357 */
+  {    34,     26,     12,      0,      0, }, /* 358 */
+  {    34,     21,     12,      0,      0, }, /* 359 */
+  {    34,     13,     12,      0,      0, }, /* 360 */
+  {    52,      7,     12,      0,      0, }, /* 361 */
+  {    39,      7,     12,      0,      0, }, /* 362 */
+  {    39,     10,     12,      0,      0, }, /* 363 */
+  {    39,     10,      5,      0,      0, }, /* 364 */
+  {    39,     13,     12,      0,      0, }, /* 365 */
+  {    39,     15,     12,      0,      0, }, /* 366 */
+  {    39,     26,     12,      0,      0, }, /* 367 */
+  {    31,     26,     12,      0,      0, }, /* 368 */
+  {     5,      7,     12,      0,      0, }, /* 369 */
+  {     5,     12,      3,      0,      0, }, /* 370 */
+  {     5,     10,      5,      0,      0, }, /* 371 */
+  {     5,     21,     12,      0,      0, }, /* 372 */
+  {    90,      7,     12,      0,      0, }, /* 373 */
+  {    90,     10,      5,      0,      0, }, /* 374 */
+  {    90,     12,      3,      0,      0, }, /* 375 */
+  {    90,     10,     12,      0,      0, }, /* 376 */
+  {    90,     13,     12,      0,      0, }, /* 377 */
+  {    90,     21,     12,      0,      0, }, /* 378 */
+  {    90,      6,     12,      0,      0, }, /* 379 */
+  {    27,     11,      3,      0,      0, }, /* 380 */
+  {    61,     12,      3,      0,      0, }, /* 381 */
+  {    61,     10,      5,      0,      0, }, /* 382 */
+  {    61,      7,     12,      0,      0, }, /* 383 */
+  {    61,     13,     12,      0,      0, }, /* 384 */
+  {    61,     21,     12,      0,      0, }, /* 385 */
+  {    61,     26,     12,      0,      0, }, /* 386 */
+  {    75,     12,      3,      0,      0, }, /* 387 */
+  {    75,     10,      5,      0,      0, }, /* 388 */
+  {    75,      7,     12,      0,      0, }, /* 389 */
+  {    75,     13,     12,      0,      0, }, /* 390 */
+  {    92,      7,     12,      0,      0, }, /* 391 */
+  {    92,     12,      3,      0,      0, }, /* 392 */
+  {    92,     10,      5,      0,      0, }, /* 393 */
+  {    92,     21,     12,      0,      0, }, /* 394 */
+  {    69,      7,     12,      0,      0, }, /* 395 */
+  {    69,     10,      5,      0,      0, }, /* 396 */
+  {    69,     12,      3,      0,      0, }, /* 397 */
+  {    69,     21,     12,      0,      0, }, /* 398 */
+  {    69,     13,     12,      0,      0, }, /* 399 */
+  {    72,     13,     12,      0,      0, }, /* 400 */
+  {    72,      7,     12,      0,      0, }, /* 401 */
+  {    72,      6,     12,      0,      0, }, /* 402 */
+  {    72,     21,     12,      0,      0, }, /* 403 */
+  {    75,     21,     12,      0,      0, }, /* 404 */
+  {     9,     10,      5,      0,      0, }, /* 405 */
+  {     9,      7,     12,      0,      0, }, /* 406 */
+  {    12,      5,     12,      0,      0, }, /* 407 */
+  {    12,      6,     12,      0,      0, }, /* 408 */
+  {    33,      5,     12,      0,  35332, }, /* 409 */
+  {    33,      5,     12,      0,   3814, }, /* 410 */
+  {    33,      9,     12,     63,      1, }, /* 411 */
+  {    33,      5,     12,     63,     -1, }, /* 412 */
+  {    33,      5,     12,     63,    -58, }, /* 413 */
+  {    33,      9,     12,      0,  -7615, }, /* 414 */
+  {    19,      5,     12,      0,      8, }, /* 415 */
+  {    19,      9,     12,      0,     -8, }, /* 416 */
+  {    19,      5,     12,      0,     74, }, /* 417 */
+  {    19,      5,     12,      0,     86, }, /* 418 */
+  {    19,      5,     12,      0,    100, }, /* 419 */
+  {    19,      5,     12,      0,    128, }, /* 420 */
+  {    19,      5,     12,      0,    112, }, /* 421 */
+  {    19,      5,     12,      0,    126, }, /* 422 */
+  {    19,      8,     12,      0,     -8, }, /* 423 */
+  {    19,      5,     12,      0,      9, }, /* 424 */
+  {    19,      9,     12,      0,    -74, }, /* 425 */
+  {    19,      8,     12,      0,     -9, }, /* 426 */
+  {    19,      5,     12,     21,  -7173, }, /* 427 */
+  {    19,      9,     12,      0,    -86, }, /* 428 */
+  {    19,      9,     12,      0,   -100, }, /* 429 */
+  {    19,      9,     12,      0,   -112, }, /* 430 */
+  {    19,      9,     12,      0,   -128, }, /* 431 */
+  {    19,      9,     12,      0,   -126, }, /* 432 */
+  {    27,      1,      3,      0,      0, }, /* 433 */
+  {     9,     27,      2,      0,      0, }, /* 434 */
+  {     9,     28,      2,      0,      0, }, /* 435 */
+  {     9,      2,      2,      0,      0, }, /* 436 */
+  {     9,      9,     12,      0,      0, }, /* 437 */
+  {     9,      5,     12,      0,      0, }, /* 438 */
+  {    19,      9,     12,     67,  -7517, }, /* 439 */
+  {    33,      9,     12,     71,  -8383, }, /* 440 */
+  {    33,      9,     12,     75,  -8262, }, /* 441 */
+  {    33,      9,     12,      0,     28, }, /* 442 */
+  {    33,      5,     12,      0,    -28, }, /* 443 */
+  {    33,     14,     12,      0,     16, }, /* 444 */
+  {    33,     14,     12,      0,    -16, }, /* 445 */
+  {    33,     14,     12,      0,      0, }, /* 446 */
+  {     9,     26,     12,      0,     26, }, /* 447 */
+  {     9,     26,     12,      0,    -26, }, /* 448 */
+  {     4,     26,     12,      0,      0, }, /* 449 */
+  {    17,      9,     12,      0,     48, }, /* 450 */
+  {    17,      5,     12,      0,    -48, }, /* 451 */
+  {    33,      9,     12,      0, -10743, }, /* 452 */
+  {    33,      9,     12,      0,  -3814, }, /* 453 */
+  {    33,      9,     12,      0, -10727, }, /* 454 */
+  {    33,      5,     12,      0, -10795, }, /* 455 */
+  {    33,      5,     12,      0, -10792, }, /* 456 */
+  {    33,      9,     12,      0, -10780, }, /* 457 */
+  {    33,      9,     12,      0, -10749, }, /* 458 */
+  {    33,      9,     12,      0, -10783, }, /* 459 */
+  {    33,      9,     12,      0, -10782, }, /* 460 */
+  {    33,      9,     12,      0, -10815, }, /* 461 */
+  {    10,      5,     12,      0,      0, }, /* 462 */
+  {    10,     26,     12,      0,      0, }, /* 463 */
+  {    10,     12,      3,      0,      0, }, /* 464 */
+  {    10,     21,     12,      0,      0, }, /* 465 */
+  {    10,     15,     12,      0,      0, }, /* 466 */
+  {    16,      5,     12,      0,  -7264, }, /* 467 */
+  {    58,      7,     12,      0,      0, }, /* 468 */
+  {    58,      6,     12,      0,      0, }, /* 469 */
+  {    58,     21,     12,      0,      0, }, /* 470 */
+  {    58,     12,      3,      0,      0, }, /* 471 */
+  {    22,     26,     12,      0,      0, }, /* 472 */
+  {    22,      6,     12,      0,      0, }, /* 473 */
+  {    22,     14,     12,      0,      0, }, /* 474 */
+  {    23,     10,      3,      0,      0, }, /* 475 */
+  {    26,      7,     12,      0,      0, }, /* 476 */
+  {    26,      6,     12,      0,      0, }, /* 477 */
+  {    29,      7,     12,      0,      0, }, /* 478 */
+  {    29,      6,     12,      0,      0, }, /* 479 */
+  {     3,      7,     12,      0,      0, }, /* 480 */
+  {    23,      7,     12,      0,      0, }, /* 481 */
+  {    23,     26,     12,      0,      0, }, /* 482 */
+  {    29,     26,     12,      0,      0, }, /* 483 */
+  {    22,      7,     12,      0,      0, }, /* 484 */
+  {    60,      7,     12,      0,      0, }, /* 485 */
+  {    60,      6,     12,      0,      0, }, /* 486 */
+  {    60,     26,     12,      0,      0, }, /* 487 */
+  {    85,      7,     12,      0,      0, }, /* 488 */
+  {    85,      6,     12,      0,      0, }, /* 489 */
+  {    85,     21,     12,      0,      0, }, /* 490 */
+  {    76,      7,     12,      0,      0, }, /* 491 */
+  {    76,      6,     12,      0,      0, }, /* 492 */
+  {    76,     21,     12,      0,      0, }, /* 493 */
+  {    76,     13,     12,      0,      0, }, /* 494 */
+  {    12,      7,     12,      0,      0, }, /* 495 */
+  {    12,     21,     12,      0,      0, }, /* 496 */
+  {    78,      7,     12,      0,      0, }, /* 497 */
+  {    78,     14,     12,      0,      0, }, /* 498 */
+  {    78,     12,      3,      0,      0, }, /* 499 */
+  {    78,     21,     12,      0,      0, }, /* 500 */
+  {    33,      9,     12,      0, -35332, }, /* 501 */
+  {    33,      9,     12,      0, -42280, }, /* 502 */
+  {    33,      9,     12,      0, -42308, }, /* 503 */
+  {    33,      9,     12,      0, -42319, }, /* 504 */
+  {    33,      9,     12,      0, -42315, }, /* 505 */
+  {    33,      9,     12,      0, -42305, }, /* 506 */
+  {    33,      9,     12,      0, -42258, }, /* 507 */
+  {    33,      9,     12,      0, -42282, }, /* 508 */
+  {    48,      7,     12,      0,      0, }, /* 509 */
+  {    48,     12,      3,      0,      0, }, /* 510 */
+  {    48,     10,      5,      0,      0, }, /* 511 */
+  {    48,     26,     12,      0,      0, }, /* 512 */
+  {    64,      7,     12,      0,      0, }, /* 513 */
+  {    64,     21,     12,      0,      0, }, /* 514 */
+  {    74,     10,      5,      0,      0, }, /* 515 */
+  {    74,      7,     12,      0,      0, }, /* 516 */
+  {    74,     12,      3,      0,      0, }, /* 517 */
+  {    74,     21,     12,      0,      0, }, /* 518 */
+  {    74,     13,     12,      0,      0, }, /* 519 */
+  {    68,     13,     12,      0,      0, }, /* 520 */
+  {    68,      7,     12,      0,      0, }, /* 521 */
+  {    68,     12,      3,      0,      0, }, /* 522 */
+  {    68,     21,     12,      0,      0, }, /* 523 */
+  {    73,      7,     12,      0,      0, }, /* 524 */
+  {    73,     12,      3,      0,      0, }, /* 525 */
+  {    73,     10,      5,      0,      0, }, /* 526 */
+  {    73,     21,     12,      0,      0, }, /* 527 */
+  {    83,     12,      3,      0,      0, }, /* 528 */
+  {    83,     10,      5,      0,      0, }, /* 529 */
+  {    83,      7,     12,      0,      0, }, /* 530 */
+  {    83,     21,     12,      0,      0, }, /* 531 */
+  {    83,     13,     12,      0,      0, }, /* 532 */
+  {    38,      6,     12,      0,      0, }, /* 533 */
+  {    67,      7,     12,      0,      0, }, /* 534 */
+  {    67,     12,      3,      0,      0, }, /* 535 */
+  {    67,     10,      5,      0,      0, }, /* 536 */
+  {    67,     13,     12,      0,      0, }, /* 537 */
+  {    67,     21,     12,      0,      0, }, /* 538 */
+  {    91,      7,     12,      0,      0, }, /* 539 */
+  {    91,     12,      3,      0,      0, }, /* 540 */
+  {    91,      6,     12,      0,      0, }, /* 541 */
+  {    91,     21,     12,      0,      0, }, /* 542 */
+  {    86,      7,     12,      0,      0, }, /* 543 */
+  {    86,     10,      5,      0,      0, }, /* 544 */
+  {    86,     12,      3,      0,      0, }, /* 545 */
+  {    86,     21,     12,      0,      0, }, /* 546 */
+  {    86,      6,     12,      0,      0, }, /* 547 */
+  {    86,     13,     12,      0,      0, }, /* 548 */
+  {    23,      7,      9,      0,      0, }, /* 549 */
+  {    23,      7,     10,      0,      0, }, /* 550 */
+  {     9,      4,      2,      0,      0, }, /* 551 */
+  {     9,      3,     12,      0,      0, }, /* 552 */
+  {    25,     25,     12,      0,      0, }, /* 553 */
+  {     0,     24,     12,      0,      0, }, /* 554 */
+  {     9,      6,      3,      0,      0, }, /* 555 */
+  {    35,      7,     12,      0,      0, }, /* 556 */
+  {    19,     14,     12,      0,      0, }, /* 557 */
+  {    19,     15,     12,      0,      0, }, /* 558 */
+  {    19,     26,     12,      0,      0, }, /* 559 */
+  {    70,      7,     12,      0,      0, }, /* 560 */
+  {    66,      7,     12,      0,      0, }, /* 561 */
+  {    41,      7,     12,      0,      0, }, /* 562 */
+  {    41,     15,     12,      0,      0, }, /* 563 */
+  {    18,      7,     12,      0,      0, }, /* 564 */
+  {    18,     14,     12,      0,      0, }, /* 565 */
+  {   117,      7,     12,      0,      0, }, /* 566 */
+  {   117,     12,      3,      0,      0, }, /* 567 */
+  {    59,      7,     12,      0,      0, }, /* 568 */
+  {    59,     21,     12,      0,      0, }, /* 569 */
+  {    42,      7,     12,      0,      0, }, /* 570 */
+  {    42,     21,     12,      0,      0, }, /* 571 */
+  {    42,     14,     12,      0,      0, }, /* 572 */
+  {    13,      9,     12,      0,     40, }, /* 573 */
+  {    13,      5,     12,      0,    -40, }, /* 574 */
+  {    46,      7,     12,      0,      0, }, /* 575 */
+  {    44,      7,     12,      0,      0, }, /* 576 */
+  {    44,     13,     12,      0,      0, }, /* 577 */
+  {   105,      7,     12,      0,      0, }, /* 578 */
+  {   103,      7,     12,      0,      0, }, /* 579 */
+  {   103,     21,     12,      0,      0, }, /* 580 */
+  {   109,      7,     12,      0,      0, }, /* 581 */
+  {    11,      7,     12,      0,      0, }, /* 582 */
+  {    80,      7,     12,      0,      0, }, /* 583 */
+  {    80,     21,     12,      0,      0, }, /* 584 */
+  {    80,     15,     12,      0,      0, }, /* 585 */
+  {   119,      7,     12,      0,      0, }, /* 586 */
+  {   119,     26,     12,      0,      0, }, /* 587 */
+  {   119,     15,     12,      0,      0, }, /* 588 */
+  {   115,      7,     12,      0,      0, }, /* 589 */
+  {   115,     15,     12,      0,      0, }, /* 590 */
+  {    65,      7,     12,      0,      0, }, /* 591 */
+  {    65,     15,     12,      0,      0, }, /* 592 */
+  {    65,     21,     12,      0,      0, }, /* 593 */
+  {    71,      7,     12,      0,      0, }, /* 594 */
+  {    71,     21,     12,      0,      0, }, /* 595 */
+  {    97,      7,     12,      0,      0, }, /* 596 */
+  {    96,      7,     12,      0,      0, }, /* 597 */
+  {    30,      7,     12,      0,      0, }, /* 598 */
+  {    30,     12,      3,      0,      0, }, /* 599 */
+  {    30,     15,     12,      0,      0, }, /* 600 */
+  {    30,     21,     12,      0,      0, }, /* 601 */
+  {    87,      7,     12,      0,      0, }, /* 602 */
+  {    87,     15,     12,      0,      0, }, /* 603 */
+  {    87,     21,     12,      0,      0, }, /* 604 */
+  {   116,      7,     12,      0,      0, }, /* 605 */
+  {   116,     15,     12,      0,      0, }, /* 606 */
+  {   111,      7,     12,      0,      0, }, /* 607 */
+  {   111,     26,     12,      0,      0, }, /* 608 */
+  {   111,     12,      3,      0,      0, }, /* 609 */
+  {   111,     15,     12,      0,      0, }, /* 610 */
+  {   111,     21,     12,      0,      0, }, /* 611 */
+  {    77,      7,     12,      0,      0, }, /* 612 */
+  {    77,     21,     12,      0,      0, }, /* 613 */
+  {    82,      7,     12,      0,      0, }, /* 614 */
+  {    82,     15,     12,      0,      0, }, /* 615 */
+  {    81,      7,     12,      0,      0, }, /* 616 */
+  {    81,     15,     12,      0,      0, }, /* 617 */
+  {   120,      7,     12,      0,      0, }, /* 618 */
+  {   120,     21,     12,      0,      0, }, /* 619 */
+  {   120,     15,     12,      0,      0, }, /* 620 */
+  {    88,      7,     12,      0,      0, }, /* 621 */
+  {     0,     15,     12,      0,      0, }, /* 622 */
+  {    93,     10,      5,      0,      0, }, /* 623 */
+  {    93,     12,      3,      0,      0, }, /* 624 */
+  {    93,      7,     12,      0,      0, }, /* 625 */
+  {    93,     21,     12,      0,      0, }, /* 626 */
+  {    93,     15,     12,      0,      0, }, /* 627 */
+  {    93,     13,     12,      0,      0, }, /* 628 */
+  {    84,     12,      3,      0,      0, }, /* 629 */
+  {    84,     10,      5,      0,      0, }, /* 630 */
+  {    84,      7,     12,      0,      0, }, /* 631 */
+  {    84,     21,     12,      0,      0, }, /* 632 */
+  {    84,      1,      2,      0,      0, }, /* 633 */
+  {   100,      7,     12,      0,      0, }, /* 634 */
+  {   100,     13,     12,      0,      0, }, /* 635 */
+  {    95,     12,      3,      0,      0, }, /* 636 */
+  {    95,      7,     12,      0,      0, }, /* 637 */
+  {    95,     10,      5,      0,      0, }, /* 638 */
+  {    95,     13,     12,      0,      0, }, /* 639 */
+  {    95,     21,     12,      0,      0, }, /* 640 */
+  {   110,      7,     12,      0,      0, }, /* 641 */
+  {   110,     12,      3,      0,      0, }, /* 642 */
+  {   110,     21,     12,      0,      0, }, /* 643 */
+  {    99,     12,      3,      0,      0, }, /* 644 */
+  {    99,     10,      5,      0,      0, }, /* 645 */
+  {    99,      7,     12,      0,      0, }, /* 646 */
+  {    99,     21,     12,      0,      0, }, /* 647 */
+  {    99,     13,     12,      0,      0, }, /* 648 */
+  {    47,     15,     12,      0,      0, }, /* 649 */
+  {   107,      7,     12,      0,      0, }, /* 650 */
+  {   107,     10,      5,      0,      0, }, /* 651 */
+  {   107,     12,      3,      0,      0, }, /* 652 */
+  {   107,     21,     12,      0,      0, }, /* 653 */
+  {   108,      7,     12,      0,      0, }, /* 654 */
+  {   108,     12,      3,      0,      0, }, /* 655 */
+  {   108,     10,      5,      0,      0, }, /* 656 */
+  {   108,     13,     12,      0,      0, }, /* 657 */
+  {   106,     12,      3,      0,      0, }, /* 658 */
+  {   106,     10,      5,      0,      0, }, /* 659 */
+  {   106,      7,     12,      0,      0, }, /* 660 */
+  {   106,     10,      3,      0,      0, }, /* 661 */
+  {   123,      7,     12,      0,      0, }, /* 662 */
+  {   123,     10,      3,      0,      0, }, /* 663 */
+  {   123,     10,      5,      0,      0, }, /* 664 */
+  {   123,     12,      3,      0,      0, }, /* 665 */
+  {   123,     21,     12,      0,      0, }, /* 666 */
+  {   123,     13,     12,      0,      0, }, /* 667 */
+  {   122,      7,     12,      0,      0, }, /* 668 */
+  {   122,     10,      3,      0,      0, }, /* 669 */
+  {   122,     10,      5,      0,      0, }, /* 670 */
+  {   122,     12,      3,      0,      0, }, /* 671 */
+  {   122,     21,     12,      0,      0, }, /* 672 */
+  {   113,      7,     12,      0,      0, }, /* 673 */
+  {   113,     10,      5,      0,      0, }, /* 674 */
+  {   113,     12,      3,      0,      0, }, /* 675 */
+  {   113,     21,     12,      0,      0, }, /* 676 */
+  {   113,     13,     12,      0,      0, }, /* 677 */
+  {   101,      7,     12,      0,      0, }, /* 678 */
+  {   101,     12,      3,      0,      0, }, /* 679 */
+  {   101,     10,      5,      0,      0, }, /* 680 */
+  {   101,     13,     12,      0,      0, }, /* 681 */
+  {   124,      9,     12,      0,     32, }, /* 682 */
+  {   124,      5,     12,      0,    -32, }, /* 683 */
+  {   124,     13,     12,      0,      0, }, /* 684 */
+  {   124,     15,     12,      0,      0, }, /* 685 */
+  {   124,      7,     12,      0,      0, }, /* 686 */
+  {   121,      7,     12,      0,      0, }, /* 687 */
+  {    62,      7,     12,      0,      0, }, /* 688 */
+  {    62,     14,     12,      0,      0, }, /* 689 */
+  {    62,     21,     12,      0,      0, }, /* 690 */
+  {    79,      7,     12,      0,      0, }, /* 691 */
+  {   114,      7,     12,      0,      0, }, /* 692 */
+  {   114,     13,     12,      0,      0, }, /* 693 */
+  {   114,     21,     12,      0,      0, }, /* 694 */
+  {   102,      7,     12,      0,      0, }, /* 695 */
+  {   102,     12,      3,      0,      0, }, /* 696 */
+  {   102,     21,     12,      0,      0, }, /* 697 */
+  {   118,      7,     12,      0,      0, }, /* 698 */
+  {   118,     12,      3,      0,      0, }, /* 699 */
+  {   118,     21,     12,      0,      0, }, /* 700 */
+  {   118,     26,     12,      0,      0, }, /* 701 */
+  {   118,      6,     12,      0,      0, }, /* 702 */
+  {   118,     13,     12,      0,      0, }, /* 703 */
+  {   118,     15,     12,      0,      0, }, /* 704 */
+  {    98,      7,     12,      0,      0, }, /* 705 */
+  {    98,     10,      5,      0,      0, }, /* 706 */
+  {    98,     12,      3,      0,      0, }, /* 707 */
+  {    98,      6,     12,      0,      0, }, /* 708 */
+  {   104,      7,     12,      0,      0, }, /* 709 */
+  {   104,     26,     12,      0,      0, }, /* 710 */
+  {   104,     12,      3,      0,      0, }, /* 711 */
+  {   104,     21,     12,      0,      0, }, /* 712 */
+  {     9,     10,      3,      0,      0, }, /* 713 */
+  {    19,     12,      3,      0,      0, }, /* 714 */
+  {   112,      7,     12,      0,      0, }, /* 715 */
+  {   112,     15,     12,      0,      0, }, /* 716 */
+  {   112,     12,      3,      0,      0, }, /* 717 */
+  {     9,     26,     11,      0,      0, }, /* 718 */
+  {    26,     26,     12,      0,      0, }, /* 719 */
 };
 
 const pcre_uint8 PRIV(ucd_stage1)[] = { /* 8704 bytes */
@@ -743,38 +835,38 @@ const pcre_uint8 PRIV(ucd_stage1)[] = { /* 8704 bytes */
 123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,123, /* U+E800 */
 123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,123, /* U+F000 */
 123,123, 95, 95,124,125,126,127,128,128,129,130,131,132,133,134, /* U+F800 */
-135,136,137,138, 79,139,140,141,142,143, 79, 79, 79, 79, 79, 79, /* U+10000 */
-144, 79,145,146,147, 79,148, 79,149, 79, 79, 79,150, 79, 79, 79, /* U+10800 */
-151,152,153,154, 79, 79, 79, 79, 79, 79, 79, 79, 79,155, 79, 79, /* U+11000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+11800 */
-156,156,156,156,156,156,157, 79,158, 79, 79, 79, 79, 79, 79, 79, /* U+12000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+12800 */
-159,159,159,159,159,159,159,159,160, 79, 79, 79, 79, 79, 79, 79, /* U+13000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+13800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+14000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+14800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+15000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+15800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+16000 */
-161,161,161,161,162, 79, 79, 79, 79, 79, 79, 79, 79, 79,163,164, /* U+16800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+17000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+17800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+18000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+18800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+19000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+19800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+1A000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+1A800 */
-165, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+1B000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+1B800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+1C000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+1C800 */
- 71,166,167,168,169, 79,170, 79,171,172,173,174,175,176,177,178, /* U+1D000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+1D800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+1E000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79,179,180, 79, 79, /* U+1E800 */
-181,182,183,184,185, 79,186,187,188,189,190,191,192,193,194, 79, /* U+1F000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+1F800 */
+135,136,137,138,139,140,141,142,143,144,145,139,146,146,147,139, /* U+10000 */
+148,149,150,151,152,153,154,155,156,139,139,139,157,139,139,139, /* U+10800 */
+158,159,160,161,162,163,164,139,139,165,139,166,167,168,139,139, /* U+11000 */
+139,169,139,139,139,170,139,139,139,139,139,139,139,139,139,139, /* U+11800 */
+171,171,171,171,171,171,171,172,173,139,139,139,139,139,139,139, /* U+12000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+12800 */
+174,174,174,174,174,174,174,174,175,139,139,139,139,139,139,139, /* U+13000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+13800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+14000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+14800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+15000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+15800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+16000 */
+176,176,176,176,177,178,179,180,139,139,139,139,139,139,181,182, /* U+16800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+17000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+17800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+18000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+18800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+19000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+19800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+1A000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+1A800 */
+183,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+1B000 */
+139,139,139,139,139,139,139,139,184,185,139,139,139,139,139,139, /* U+1B800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+1C000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+1C800 */
+ 71,186,187,188,189,139,190,139,191,192,193,194,195,196,197,198, /* U+1D000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+1D800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+1E000 */
+199,200,139,139,139,139,139,139,139,139,139,139,201,202,139,139, /* U+1E800 */
+203,204,205,206,207,139,208,209, 71,210,211,212,213,214,215,216, /* U+1F000 */
+217,218,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+1F800 */
  95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, /* U+20000 */
  95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, /* U+20800 */
  95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, /* U+21000 */
@@ -795,402 +887,402 @@ const pcre_uint8 PRIV(ucd_stage1)[] = { /* 8704 bytes */
  95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, /* U+28800 */
  95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, /* U+29000 */
  95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, /* U+29800 */
- 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95,195, 95, 95, /* U+2A000 */
+ 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95,219, 95, 95, /* U+2A000 */
  95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, /* U+2A800 */
- 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95,196, 95, /* U+2B000 */
-197, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+2B800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+2C000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+2C800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+2D000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+2D800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+2E000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+2E800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+2F000 */
- 95, 95, 95, 95,197, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+2F800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+30000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+30800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+31000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+31800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+32000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+32800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+33000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+33800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+34000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+34800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+35000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+35800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+36000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+36800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+37000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+37800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+38000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+38800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+39000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+39800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+3A000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+3A800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+3B000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+3B800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+3C000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+3C800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+3D000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+3D800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+3E000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+3E800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+3F000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+3F800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+40000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+40800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+41000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+41800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+42000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+42800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+43000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+43800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+44000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+44800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+45000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+45800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+46000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+46800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+47000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+47800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+48000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+48800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+49000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+49800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+4A000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+4A800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+4B000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+4B800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+4C000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+4C800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+4D000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+4D800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+4E000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+4E800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+4F000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+4F800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+50000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+50800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+51000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+51800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+52000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+52800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+53000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+53800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+54000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+54800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+55000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+55800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+56000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+56800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+57000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+57800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+58000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+58800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+59000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+59800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+5A000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+5A800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+5B000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+5B800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+5C000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+5C800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+5D000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+5D800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+5E000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+5E800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+5F000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+5F800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+60000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+60800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+61000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+61800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+62000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+62800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+63000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+63800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+64000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+64800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+65000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+65800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+66000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+66800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+67000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+67800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+68000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+68800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+69000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+69800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+6A000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+6A800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+6B000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+6B800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+6C000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+6C800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+6D000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+6D800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+6E000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+6E800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+6F000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+6F800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+70000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+70800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+71000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+71800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+72000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+72800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+73000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+73800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+74000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+74800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+75000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+75800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+76000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+76800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+77000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+77800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+78000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+78800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+79000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+79800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+7A000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+7A800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+7B000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+7B800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+7C000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+7C800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+7D000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+7D800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+7E000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+7E800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+7F000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+7F800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+80000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+80800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+81000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+81800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+82000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+82800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+83000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+83800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+84000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+84800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+85000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+85800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+86000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+86800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+87000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+87800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+88000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+88800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+89000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+89800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+8A000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+8A800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+8B000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+8B800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+8C000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+8C800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+8D000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+8D800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+8E000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+8E800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+8F000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+8F800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+90000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+90800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+91000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+91800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+92000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+92800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+93000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+93800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+94000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+94800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+95000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+95800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+96000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+96800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+97000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+97800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+98000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+98800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+99000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+99800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+9A000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+9A800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+9B000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+9B800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+9C000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+9C800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+9D000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+9D800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+9E000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+9E800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+9F000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+9F800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A0000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A0800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A1000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A1800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A2000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A2800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A3000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A3800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A4000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A4800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A5000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A5800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A6000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A6800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A7000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A7800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A8000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A8800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A9000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+A9800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+AA000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+AA800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+AB000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+AB800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+AC000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+AC800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+AD000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+AD800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+AE000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+AE800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+AF000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+AF800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B0000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B0800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B1000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B1800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B2000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B2800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B3000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B3800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B4000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B4800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B5000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B5800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B6000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B6800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B7000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B7800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B8000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B8800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B9000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+B9800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+BA000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+BA800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+BB000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+BB800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+BC000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+BC800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+BD000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+BD800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+BE000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+BE800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+BF000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+BF800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C0000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C0800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C1000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C1800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C2000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C2800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C3000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C3800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C4000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C4800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C5000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C5800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C6000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C6800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C7000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C7800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C8000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C8800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C9000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+C9800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+CA000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+CA800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+CB000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+CB800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+CC000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+CC800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+CD000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+CD800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+CE000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+CE800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+CF000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+CF800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D0000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D0800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D1000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D1800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D2000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D2800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D3000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D3800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D4000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D4800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D5000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D5800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D6000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D6800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D7000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D7800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D8000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D8800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D9000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+D9800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+DA000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+DA800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+DB000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+DB800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+DC000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+DC800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+DD000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+DD800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+DE000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+DE800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+DF000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+DF800 */
-198,199,200,201,199,199,199,199,199,199,199,199,199,199,199,199, /* U+E0000 */
-199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199, /* U+E0800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E1000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E1800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E2000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E2800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E3000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E3800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E4000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E4800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E5000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E5800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E6000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E6800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E7000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E7800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E8000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E8800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E9000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+E9800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+EA000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+EA800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+EB000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+EB800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+EC000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+EC800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+ED000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+ED800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+EE000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+EE800 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+EF000 */
- 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, /* U+EF800 */
+ 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95,220, 95, /* U+2B000 */
+221,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+2B800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+2C000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+2C800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+2D000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+2D800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+2E000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+2E800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+2F000 */
+ 95, 95, 95, 95,221,139,139,139,139,139,139,139,139,139,139,139, /* U+2F800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+30000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+30800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+31000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+31800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+32000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+32800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+33000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+33800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+34000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+34800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+35000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+35800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+36000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+36800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+37000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+37800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+38000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+38800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+39000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+39800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+3A000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+3A800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+3B000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+3B800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+3C000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+3C800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+3D000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+3D800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+3E000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+3E800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+3F000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+3F800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+40000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+40800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+41000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+41800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+42000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+42800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+43000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+43800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+44000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+44800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+45000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+45800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+46000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+46800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+47000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+47800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+48000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+48800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+49000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+49800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+4A000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+4A800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+4B000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+4B800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+4C000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+4C800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+4D000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+4D800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+4E000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+4E800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+4F000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+4F800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+50000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+50800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+51000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+51800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+52000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+52800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+53000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+53800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+54000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+54800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+55000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+55800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+56000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+56800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+57000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+57800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+58000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+58800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+59000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+59800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+5A000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+5A800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+5B000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+5B800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+5C000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+5C800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+5D000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+5D800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+5E000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+5E800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+5F000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+5F800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+60000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+60800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+61000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+61800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+62000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+62800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+63000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+63800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+64000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+64800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+65000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+65800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+66000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+66800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+67000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+67800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+68000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+68800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+69000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+69800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+6A000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+6A800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+6B000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+6B800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+6C000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+6C800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+6D000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+6D800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+6E000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+6E800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+6F000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+6F800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+70000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+70800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+71000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+71800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+72000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+72800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+73000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+73800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+74000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+74800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+75000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+75800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+76000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+76800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+77000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+77800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+78000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+78800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+79000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+79800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+7A000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+7A800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+7B000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+7B800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+7C000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+7C800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+7D000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+7D800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+7E000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+7E800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+7F000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+7F800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+80000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+80800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+81000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+81800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+82000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+82800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+83000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+83800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+84000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+84800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+85000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+85800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+86000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+86800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+87000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+87800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+88000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+88800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+89000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+89800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+8A000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+8A800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+8B000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+8B800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+8C000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+8C800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+8D000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+8D800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+8E000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+8E800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+8F000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+8F800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+90000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+90800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+91000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+91800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+92000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+92800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+93000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+93800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+94000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+94800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+95000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+95800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+96000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+96800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+97000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+97800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+98000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+98800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+99000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+99800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+9A000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+9A800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+9B000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+9B800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+9C000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+9C800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+9D000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+9D800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+9E000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+9E800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+9F000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+9F800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A0000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A0800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A1000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A1800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A2000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A2800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A3000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A3800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A4000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A4800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A5000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A5800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A6000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A6800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A7000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A7800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A8000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A8800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A9000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+A9800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+AA000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+AA800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+AB000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+AB800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+AC000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+AC800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+AD000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+AD800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+AE000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+AE800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+AF000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+AF800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B0000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B0800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B1000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B1800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B2000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B2800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B3000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B3800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B4000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B4800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B5000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B5800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B6000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B6800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B7000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B7800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B8000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B8800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B9000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+B9800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+BA000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+BA800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+BB000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+BB800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+BC000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+BC800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+BD000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+BD800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+BE000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+BE800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+BF000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+BF800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C0000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C0800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C1000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C1800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C2000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C2800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C3000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C3800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C4000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C4800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C5000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C5800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C6000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C6800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C7000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C7800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C8000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C8800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C9000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+C9800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+CA000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+CA800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+CB000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+CB800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+CC000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+CC800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+CD000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+CD800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+CE000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+CE800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+CF000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+CF800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D0000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D0800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D1000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D1800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D2000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D2800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D3000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D3800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D4000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D4800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D5000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D5800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D6000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D6800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D7000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D7800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D8000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D8800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D9000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+D9800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+DA000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+DA800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+DB000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+DB800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+DC000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+DC800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+DD000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+DD800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+DE000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+DE800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+DF000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+DF800 */
+222,223,224,225,223,223,223,223,223,223,223,223,223,223,223,223, /* U+E0000 */
+223,223,223,223,223,223,223,223,223,223,223,223,223,223,223,223, /* U+E0800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E1000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E1800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E2000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E2800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E3000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E3800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E4000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E4800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E5000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E5800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E6000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E6800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E7000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E7800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E8000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E8800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E9000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+E9800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+EA000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+EA800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+EB000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+EB800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+EC000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+EC800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+ED000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+ED800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+EE000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+EE800 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+EF000 */
+139,139,139,139,139,139,139,139,139,139,139,139,139,139,139,139, /* U+EF800 */
 123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,123, /* U+F0000 */
 123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,123, /* U+F0800 */
 123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,123, /* U+F1000 */
@@ -1222,7 +1314,7 @@ const pcre_uint8 PRIV(ucd_stage1)[] = { /* 8704 bytes */
 123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,123, /* U+FE000 */
 123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,123, /* U+FE800 */
 123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,123, /* U+FF000 */
-123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,202, /* U+FF800 */
+123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,226, /* U+FF800 */
 123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,123, /* U+100000 */
 123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,123, /* U+100800 */
 123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,123, /* U+101000 */
@@ -1254,10 +1346,10 @@ const pcre_uint8 PRIV(ucd_stage1)[] = { /* 8704 bytes */
 123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,123, /* U+10E000 */
 123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,123, /* U+10E800 */
 123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,123, /* U+10F000 */
-123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,202, /* U+10F800 */
+123,123,123,123,123,123,123,123,123,123,123,123,123,123,123,226, /* U+10F800 */
 };
 
-const pcre_uint16 PRIV(ucd_stage2)[] = { /* 51968 bytes, block = 128 */
+const pcre_uint16 PRIV(ucd_stage2)[] = { /* 58112 bytes, block = 128 */
 /* block 0 */
   0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  0,  0,  2,  0,  0,
   0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
@@ -1304,539 +1396,539 @@ const pcre_uint16 PRIV(ucd_stage2)[] = { /* 51968 bytes, block = 128 */
  70, 33, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
  30, 31, 30, 31, 33, 33, 33, 33, 33, 33, 71, 30, 31, 72, 73, 74,
  74, 30, 31, 75, 76, 77, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
- 78, 79, 80, 81, 82, 33, 83, 83, 33, 84, 33, 85, 33, 33, 33, 33,
- 83, 33, 33, 86, 33, 87, 88, 33, 89, 90, 33, 91, 33, 33, 33, 90,
- 33, 92, 93, 33, 33, 94, 33, 33, 33, 33, 33, 33, 33, 95, 33, 33,
+ 78, 79, 80, 81, 82, 33, 83, 83, 33, 84, 33, 85, 86, 33, 33, 33,
+ 83, 87, 33, 88, 33, 89, 90, 33, 91, 92, 33, 93, 94, 33, 33, 92,
+ 33, 95, 96, 33, 33, 97, 33, 33, 33, 33, 33, 33, 33, 98, 33, 33,
 
 /* block 5 */
- 96, 33, 33, 96, 33, 33, 33, 33, 96, 97, 98, 98, 99, 33, 33, 33,
- 33, 33,100, 33, 20, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33,
+ 99, 33, 33, 99, 33, 33, 33,100, 99,101,102,102,103, 33, 33, 33,
+ 33, 33,104, 33, 20, 33, 33, 33, 33, 33, 33, 33, 33, 33,105, 33,
  33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33,
-101,101,101,101,101,101,101,101,101,102,102,102,102,102,102,102,
-102,102, 14, 14, 14, 14,102,102,102,102,102,102,102,102,102,102,
-102,102, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
-101,101,101,101,101, 14, 14, 14, 14, 14,103,103,102, 14,102, 14,
+106,106,106,106,106,106,106,106,106,107,107,107,107,107,107,107,
+107,107, 14, 14, 14, 14,107,107,107,107,107,107,107,107,107,107,
+107,107, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+106,106,106,106,106, 14, 14, 14, 14, 14,108,108,107, 14,107, 14,
  14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
 
 /* block 6 */
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,105,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-106,107,106,107,102,108,106,107,109,109,110,111,111,111,  4,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,110,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+111,112,111,112,107,113,111,112,114,114,115,116,116,116,  4,117,
 
 /* block 7 */
-109,109,109,109,108, 14,112,  4,113,113,113,109,114,109,115,115,
-116,117,118,117,117,119,117,117,120,121,122,117,123,117,117,117,
-124,125,109,126,117,117,127,117,117,128,117,117,129,130,130,130,
-116,131,132,131,131,133,131,131,134,135,136,131,137,131,131,131,
-138,139,140,141,131,131,142,131,131,143,131,131,144,145,145,146,
-147,148,149,149,149,150,151,152,106,107,106,107,106,107,106,107,
-106,107,153,154,153,154,153,154,153,154,153,154,153,154,153,154,
-155,156,157,116,158,159,160,106,107,161,106,107,116,162,162,162,
+114,114,114,114,113, 14,118,  4,119,119,119,114,120,114,121,121,
+122,123,124,123,123,125,123,123,126,127,128,123,129,123,123,123,
+130,131,114,132,123,123,133,123,123,134,123,123,135,136,136,136,
+122,137,138,137,137,139,137,137,140,141,142,137,143,137,137,137,
+144,145,146,147,137,137,148,137,137,149,137,137,150,151,151,152,
+153,154,155,155,155,156,157,158,111,112,111,112,111,112,111,112,
+111,112,159,160,159,160,159,160,159,160,159,160,159,160,159,160,
+161,162,163,164,165,166,167,111,112,168,111,112,122,169,169,169,
 
 /* block 8 */
-163,163,163,163,163,163,163,163,163,163,163,163,163,163,163,163,
-164,164,164,164,164,164,164,164,164,164,164,164,164,164,164,164,
-164,164,164,164,164,164,164,164,164,164,164,164,164,164,164,164,
-165,165,165,165,165,165,165,165,165,165,165,165,165,165,165,165,
-165,165,165,165,165,165,165,165,165,165,165,165,165,165,165,165,
-166,166,166,166,166,166,166,166,166,166,166,166,166,166,166,166,
-167,168,167,168,167,168,167,168,167,168,167,168,167,168,167,168,
-167,168,167,168,167,168,167,168,167,168,167,168,167,168,167,168,
+170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,
+171,171,171,171,171,171,171,171,171,171,171,171,171,171,171,171,
+171,171,171,171,171,171,171,171,171,171,171,171,171,171,171,171,
+172,172,172,172,172,172,172,172,172,172,172,172,172,172,172,172,
+172,172,172,172,172,172,172,172,172,172,172,172,172,172,172,172,
+173,173,173,173,173,173,173,173,173,173,173,173,173,173,173,173,
+174,175,174,175,174,175,174,175,174,175,174,175,174,175,174,175,
+174,175,174,175,174,175,174,175,174,175,174,175,174,175,174,175,
 
 /* block 9 */
-167,168,169,170,170,104,104,170,171,171,167,168,167,168,167,168,
-167,168,167,168,167,168,167,168,167,168,167,168,167,168,167,168,
-167,168,167,168,167,168,167,168,167,168,167,168,167,168,167,168,
-167,168,167,168,167,168,167,168,167,168,167,168,167,168,167,168,
-172,167,168,167,168,167,168,167,168,167,168,167,168,167,168,173,
-167,168,167,168,167,168,167,168,167,168,167,168,167,168,167,168,
-167,168,167,168,167,168,167,168,167,168,167,168,167,168,167,168,
-167,168,167,168,167,168,167,168,167,168,167,168,167,168,167,168,
+174,175,176,177,177,109,109,177,178,178,174,175,174,175,174,175,
+174,175,174,175,174,175,174,175,174,175,174,175,174,175,174,175,
+174,175,174,175,174,175,174,175,174,175,174,175,174,175,174,175,
+174,175,174,175,174,175,174,175,174,175,174,175,174,175,174,175,
+179,174,175,174,175,174,175,174,175,174,175,174,175,174,175,180,
+174,175,174,175,174,175,174,175,174,175,174,175,174,175,174,175,
+174,175,174,175,174,175,174,175,174,175,174,175,174,175,174,175,
+174,175,174,175,174,175,174,175,174,175,174,175,174,175,174,175,
 
 /* block 10 */
-167,168,167,168,167,168,167,168,167,168,167,168,167,168,167,168,
-167,168,167,168,167,168,167,168,167,168,167,168,167,168,167,168,
-167,168,167,168,167,168,167,168,109,109,109,109,109,109,109,109,
-109,174,174,174,174,174,174,174,174,174,174,174,174,174,174,174,
-174,174,174,174,174,174,174,174,174,174,174,174,174,174,174,174,
-174,174,174,174,174,174,174,109,109,175,176,176,176,176,176,176,
-109,177,177,177,177,177,177,177,177,177,177,177,177,177,177,177,
-177,177,177,177,177,177,177,177,177,177,177,177,177,177,177,177,
-
-/* block 11 */
-177,177,177,177,177,177,177,178,109,  4,179,109,109,109,109,180,
-109,181,181,181,181,181,181,181,181,181,181,181,181,181,181,181,
+174,175,174,175,174,175,174,175,174,175,174,175,174,175,174,175,
+174,175,174,175,174,175,174,175,174,175,174,175,174,175,174,175,
+174,175,174,175,174,175,174,175,174,175,174,175,174,175,174,175,
+114,181,181,181,181,181,181,181,181,181,181,181,181,181,181,181,
 181,181,181,181,181,181,181,181,181,181,181,181,181,181,181,181,
-181,181,181,181,181,181,181,181,181,181,181,181,181,181,182,181,
-183,181,181,183,181,181,183,181,109,109,109,109,109,109,109,109,
+181,181,181,181,181,181,181,114,114,182,183,183,183,183,183,183,
+114,184,184,184,184,184,184,184,184,184,184,184,184,184,184,184,
 184,184,184,184,184,184,184,184,184,184,184,184,184,184,184,184,
-184,184,184,184,184,184,184,184,184,184,184,109,109,109,109,109,
-184,184,184,183,183,109,109,109,109,109,109,109,109,109,109,109,
+
+/* block 11 */
+184,184,184,184,184,184,184,185,114,  4,186,114,114,187,187,188,
+114,189,189,189,189,189,189,189,189,189,189,189,189,189,189,189,
+189,189,189,189,189,189,189,189,189,189,189,189,189,189,189,189,
+189,189,189,189,189,189,189,189,189,189,189,189,189,189,190,189,
+191,189,189,191,189,189,191,189,114,114,114,114,114,114,114,114,
+192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,
+192,192,192,192,192,192,192,192,192,192,192,114,114,114,114,114,
+192,192,192,191,191,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 12 */
-185,185,185,185,185,109,186,186,186,187,187,188,  4,187,189,189,
-190,190,190,190,190,190,190,190,190,190,190,  4,109,109,187,  4,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-102,191,191,191,191,191,191,191,191,191,191,104,104,104,104,104,
-104,104,104,104,104,104,190,190,190,190,190,190,190,190,190,190,
- 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,187,187,187,187,191,191,
-104,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
+193,193,193,193,193, 22,194,194,194,195,195,196,  4,195,197,197,
+198,198,198,198,198,198,198,198,198,198,198,  4, 22,114,195,  4,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+107,199,199,199,199,199,199,199,199,199,199,109,109,109,109,109,
+109,109,109,109,109,109,198,198,198,198,198,198,198,198,198,198,
+ 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,195,195,195,195,199,199,
+109,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
 
 /* block 13 */
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,187,191,190,190,190,190,190,190,190, 22,189,190,
-190,190,190,190,190,192,192,190,190,189,190,190,190,190,191,191,
-193,193,193,193,193,193,193,193,193,193,191,191,191,189,189,191,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,195,199,198,198,198,198,198,198,198, 22,197,198,
+198,198,198,198,198,200,200,198,198,197,198,198,198,198,199,199,
+201,201,201,201,201,201,201,201,201,201,199,199,199,197,197,199,
 
 /* block 14 */
-194,194,194,194,194,194,194,194,194,194,194,194,194,194,109,195,
-196,197,196,196,196,196,196,196,196,196,196,196,196,196,196,196,
-196,196,196,196,196,196,196,196,196,196,196,196,196,196,196,196,
-197,197,197,197,197,197,197,197,197,197,197,197,197,197,197,197,
-197,197,197,197,197,197,197,197,197,197,197,109,109,196,196,196,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
+202,202,202,202,202,202,202,202,202,202,202,202,202,202,114,203,
+204,205,204,204,204,204,204,204,204,204,204,204,204,204,204,204,
+204,204,204,204,204,204,204,204,204,204,204,204,204,204,204,204,
+205,205,205,205,205,205,205,205,205,205,205,205,205,205,205,205,
+205,205,205,205,205,205,205,205,205,205,205,114,114,204,204,204,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
 
 /* block 15 */
-198,198,198,198,198,198,198,198,198,198,198,198,198,198,198,198,
-198,198,198,198,198,198,198,198,198,198,198,198,198,198,198,198,
-198,198,198,198,198,198,199,199,199,199,199,199,199,199,199,199,
-199,198,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-200,200,200,200,200,200,200,200,200,200,201,201,201,201,201,201,
-201,201,201,201,201,201,201,201,201,201,201,201,201,201,201,201,
-201,201,201,201,201,201,201,201,201,201,201,202,202,202,202,202,
-202,202,202,202,203,203,204,205,205,205,203,109,109,109,109,109,
+206,206,206,206,206,206,206,206,206,206,206,206,206,206,206,206,
+206,206,206,206,206,206,206,206,206,206,206,206,206,206,206,206,
+206,206,206,206,206,206,207,207,207,207,207,207,207,207,207,207,
+207,206,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+208,208,208,208,208,208,208,208,208,208,209,209,209,209,209,209,
+209,209,209,209,209,209,209,209,209,209,209,209,209,209,209,209,
+209,209,209,209,209,209,209,209,209,209,209,210,210,210,210,210,
+210,210,210,210,211,211,212,213,213,213,211,114,114,114,114,114,
 
 /* block 16 */
-206,206,206,206,206,206,206,206,206,206,206,206,206,206,206,206,
-206,206,206,206,206,206,207,207,207,207,208,207,207,207,207,207,
-207,207,207,207,208,207,207,207,208,207,207,207,207,207,109,109,
-209,209,209,209,209,209,209,209,209,209,209,209,209,209,209,109,
-210,210,210,210,210,210,210,210,210,210,210,210,210,210,210,210,
-210,210,210,210,210,210,210,210,210,211,211,211,109,109,212,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+214,214,214,214,214,214,214,214,214,214,214,214,214,214,214,214,
+214,214,214,214,214,214,215,215,215,215,216,215,215,215,215,215,
+215,215,215,215,216,215,215,215,216,215,215,215,215,215,114,114,
+217,217,217,217,217,217,217,217,217,217,217,217,217,217,217,114,
+218,218,218,218,218,218,218,218,218,218,218,218,218,218,218,218,
+218,218,218,218,218,218,218,218,218,219,219,219,114,114,220,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 17 */
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-191,109,191,191,191,191,191,191,191,191,191,191,191,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,190,190,190,190,190,190,190,190,190,190,190,190,
-190,190,190,190,190,190,190,190,190,190,190,190,190,190,190,109,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,198,198,198,198,198,198,198,198,198,198,198,198,
+198,198,198,198,198,198,198,198,198,198,198,198,198,198,198,198,
 
 /* block 18 */
-213,213,213,214,215,215,215,215,215,215,215,215,215,215,215,215,
-215,215,215,215,215,215,215,215,215,215,215,215,215,215,215,215,
-215,215,215,215,215,215,215,215,215,215,215,215,215,215,215,215,
-215,215,215,215,215,215,215,215,215,215,213,214,213,215,214,214,
-214,213,213,213,213,213,213,213,213,214,214,214,214,213,214,214,
-215,104,104,213,213,213,213,213,215,215,215,215,215,215,215,215,
-215,215,213,213,  4,  4,216,216,216,216,216,216,216,216,216,216,
-217,218,215,215,215,215,215,215,109,215,215,215,215,215,215,215,
+221,221,221,222,223,223,223,223,223,223,223,223,223,223,223,223,
+223,223,223,223,223,223,223,223,223,223,223,223,223,223,223,223,
+223,223,223,223,223,223,223,223,223,223,223,223,223,223,223,223,
+223,223,223,223,223,223,223,223,223,223,221,222,221,223,222,222,
+222,221,221,221,221,221,221,221,221,222,222,222,222,221,222,222,
+223,109,109,221,221,221,221,221,223,223,223,223,223,223,223,223,
+223,223,221,221,  4,  4,224,224,224,224,224,224,224,224,224,224,
+225,226,223,223,223,223,223,223,223,223,223,223,223,223,223,223,
 
 /* block 19 */
-109,219,220,220,109,221,221,221,221,221,221,221,221,109,109,221,
-221,109,109,221,221,221,221,221,221,221,221,221,221,221,221,221,
-221,221,221,221,221,221,221,221,221,109,221,221,221,221,221,221,
-221,109,221,109,109,109,221,221,221,221,109,109,219,221,222,220,
-220,219,219,219,219,109,109,220,220,109,109,220,220,219,221,109,
-109,109,109,109,109,109,109,222,109,109,109,109,221,221,109,221,
-221,221,219,219,109,109,223,223,223,223,223,223,223,223,223,223,
-221,221,224,224,225,225,225,225,225,225,226,224,109,109,109,109,
+227,228,229,229,114,227,227,227,227,227,227,227,227,114,114,227,
+227,114,114,227,227,227,227,227,227,227,227,227,227,227,227,227,
+227,227,227,227,227,227,227,227,227,114,227,227,227,227,227,227,
+227,114,227,114,114,114,227,227,227,227,114,114,228,227,230,229,
+229,228,228,228,228,114,114,229,229,114,114,229,229,228,227,114,
+114,114,114,114,114,114,114,230,114,114,114,114,227,227,114,227,
+227,227,228,228,114,114,231,231,231,231,231,231,231,231,231,231,
+227,227,232,232,233,233,233,233,233,233,234,232,114,114,114,114,
 
 /* block 20 */
-109,227,227,228,109,229,229,229,229,229,229,109,109,109,109,229,
-229,109,109,229,229,229,229,229,229,229,229,229,229,229,229,229,
-229,229,229,229,229,229,229,229,229,109,229,229,229,229,229,229,
-229,109,229,229,109,229,229,109,229,229,109,109,227,109,228,228,
-228,227,227,109,109,109,109,227,227,109,109,227,227,227,109,109,
-109,227,109,109,109,109,109,109,109,229,229,229,229,109,229,109,
-109,109,109,109,109,109,230,230,230,230,230,230,230,230,230,230,
-227,227,229,229,229,227,109,109,109,109,109,109,109,109,109,109,
+114,235,235,236,114,237,237,237,237,237,237,114,114,114,114,237,
+237,114,114,237,237,237,237,237,237,237,237,237,237,237,237,237,
+237,237,237,237,237,237,237,237,237,114,237,237,237,237,237,237,
+237,114,237,237,114,237,237,114,237,237,114,114,235,114,236,236,
+236,235,235,114,114,114,114,235,235,114,114,235,235,235,114,114,
+114,235,114,114,114,114,114,114,114,237,237,237,237,114,237,114,
+114,114,114,114,114,114,238,238,238,238,238,238,238,238,238,238,
+235,235,237,237,237,235,114,114,114,114,114,114,114,114,114,114,
 
 /* block 21 */
-109,231,231,232,109,233,233,233,233,233,233,233,233,233,109,233,
-233,233,109,233,233,233,233,233,233,233,233,233,233,233,233,233,
-233,233,233,233,233,233,233,233,233,109,233,233,233,233,233,233,
-233,109,233,233,109,233,233,233,233,233,109,109,231,233,232,232,
-232,231,231,231,231,231,109,231,231,232,109,232,232,231,109,109,
-233,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-233,233,231,231,109,109,234,234,234,234,234,234,234,234,234,234,
-235,236,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+114,239,239,240,114,241,241,241,241,241,241,241,241,241,114,241,
+241,241,114,241,241,241,241,241,241,241,241,241,241,241,241,241,
+241,241,241,241,241,241,241,241,241,114,241,241,241,241,241,241,
+241,114,241,241,114,241,241,241,241,241,114,114,239,241,240,240,
+240,239,239,239,239,239,114,239,239,240,114,240,240,239,114,114,
+241,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+241,241,239,239,114,114,242,242,242,242,242,242,242,242,242,242,
+243,244,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 22 */
-109,237,238,238,109,239,239,239,239,239,239,239,239,109,109,239,
-239,109,109,239,239,239,239,239,239,239,239,239,239,239,239,239,
-239,239,239,239,239,239,239,239,239,109,239,239,239,239,239,239,
-239,109,239,239,109,239,239,239,239,239,109,109,237,239,240,237,
-238,237,237,237,237,109,109,238,238,109,109,238,238,237,109,109,
-109,109,109,109,109,109,237,240,109,109,109,109,239,239,109,239,
-239,239,237,237,109,109,241,241,241,241,241,241,241,241,241,241,
-242,239,243,243,243,243,243,243,109,109,109,109,109,109,109,109,
+114,245,246,246,114,247,247,247,247,247,247,247,247,114,114,247,
+247,114,114,247,247,247,247,247,247,247,247,247,247,247,247,247,
+247,247,247,247,247,247,247,247,247,114,247,247,247,247,247,247,
+247,114,247,247,114,247,247,247,247,247,114,114,245,247,248,245,
+246,245,245,245,245,114,114,246,246,114,114,246,246,245,114,114,
+114,114,114,114,114,114,245,248,114,114,114,114,247,247,114,247,
+247,247,245,245,114,114,249,249,249,249,249,249,249,249,249,249,
+250,247,251,251,251,251,251,251,114,114,114,114,114,114,114,114,
 
 /* block 23 */
-109,109,244,245,109,245,245,245,245,245,245,109,109,109,245,245,
-245,109,245,245,245,245,109,109,109,245,245,109,245,109,245,245,
-109,109,109,245,245,109,109,109,245,245,245,109,109,109,245,245,
-245,245,245,245,245,245,245,245,245,245,109,109,109,109,246,247,
-244,247,247,109,109,109,247,247,247,109,247,247,247,244,109,109,
-245,109,109,109,109,109,109,246,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,248,248,248,248,248,248,248,248,248,248,
-249,249,249,250,250,250,250,250,250,251,250,109,109,109,109,109,
+114,114,252,253,114,253,253,253,253,253,253,114,114,114,253,253,
+253,114,253,253,253,253,114,114,114,253,253,114,253,114,253,253,
+114,114,114,253,253,114,114,114,253,253,253,114,114,114,253,253,
+253,253,253,253,253,253,253,253,253,253,114,114,114,114,254,255,
+252,255,255,114,114,114,255,255,255,114,255,255,255,252,114,114,
+253,114,114,114,114,114,114,254,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,256,256,256,256,256,256,256,256,256,256,
+257,257,257,258,258,258,258,258,258,259,258,114,114,114,114,114,
 
 /* block 24 */
-109,252,252,252,109,253,253,253,253,253,253,253,253,109,253,253,
-253,109,253,253,253,253,253,253,253,253,253,253,253,253,253,253,
-253,253,253,253,253,253,253,253,253,109,253,253,253,253,253,253,
-253,253,253,253,109,253,253,253,253,253,109,109,109,253,254,254,
-254,252,252,252,252,109,254,254,254,109,254,254,254,254,109,109,
-109,109,109,109,109,254,254,109,253,253,109,109,109,109,109,109,
-253,253,254,254,109,109,255,255,255,255,255,255,255,255,255,255,
-109,109,109,109,109,109,109,109,256,256,256,256,256,256,256,257,
+260,261,261,261,114,262,262,262,262,262,262,262,262,114,262,262,
+262,114,262,262,262,262,262,262,262,262,262,262,262,262,262,262,
+262,262,262,262,262,262,262,262,262,114,262,262,262,262,262,262,
+262,262,262,262,262,262,262,262,262,262,114,114,114,262,260,260,
+260,261,261,261,261,114,260,260,260,114,260,260,260,260,114,114,
+114,114,114,114,114,260,260,114,262,262,114,114,114,114,114,114,
+262,262,260,260,114,114,263,263,263,263,263,263,263,263,263,263,
+114,114,114,114,114,114,114,114,264,264,264,264,264,264,264,265,
 
 /* block 25 */
-109,109,258,258,109,259,259,259,259,259,259,259,259,109,259,259,
-259,109,259,259,259,259,259,259,259,259,259,259,259,259,259,259,
-259,259,259,259,259,259,259,259,259,109,259,259,259,259,259,259,
-259,259,259,259,109,259,259,259,259,259,109,109,260,259,258,260,
-258,258,261,258,258,109,260,258,258,109,258,258,260,260,109,109,
-109,109,109,109,109,261,261,109,109,109,109,109,109,109,259,109,
-259,259,260,260,109,109,262,262,262,262,262,262,262,262,262,262,
-109,259,259,109,109,109,109,109,109,109,109,109,109,109,109,109,
+114,266,267,267,114,268,268,268,268,268,268,268,268,114,268,268,
+268,114,268,268,268,268,268,268,268,268,268,268,268,268,268,268,
+268,268,268,268,268,268,268,268,268,114,268,268,268,268,268,268,
+268,268,268,268,114,268,268,268,268,268,114,114,266,268,267,266,
+267,267,269,267,267,114,266,267,267,114,267,267,266,266,114,114,
+114,114,114,114,114,269,269,114,114,114,114,114,114,114,268,114,
+268,268,266,266,114,114,270,270,270,270,270,270,270,270,270,270,
+114,268,268,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 26 */
-109,109,263,263,109,264,264,264,264,264,264,264,264,109,264,264,
-264,109,264,264,264,264,264,264,264,264,264,264,264,264,264,264,
-264,264,264,264,264,264,264,264,264,264,264,264,264,264,264,264,
-264,264,264,264,264,264,264,264,264,264,264,109,109,264,265,263,
-263,266,266,266,266,109,263,263,263,109,263,263,263,266,264,109,
-109,109,109,109,109,109,109,265,109,109,109,109,109,109,109,109,
-264,264,266,266,109,109,267,267,267,267,267,267,267,267,267,267,
-268,268,268,268,268,268,109,109,109,269,264,264,264,264,264,264,
+114,271,272,272,114,273,273,273,273,273,273,273,273,114,273,273,
+273,114,273,273,273,273,273,273,273,273,273,273,273,273,273,273,
+273,273,273,273,273,273,273,273,273,273,273,273,273,273,273,273,
+273,273,273,273,273,273,273,273,273,273,273,114,114,273,274,272,
+272,271,271,271,271,114,272,272,272,114,272,272,272,271,273,114,
+114,114,114,114,114,114,114,274,114,114,114,114,114,114,114,114,
+273,273,271,271,114,114,275,275,275,275,275,275,275,275,275,275,
+276,276,276,276,276,276,114,114,114,277,273,273,273,273,273,273,
 
 /* block 27 */
-109,109,270,270,109,271,271,271,271,271,271,271,271,271,271,271,
-271,271,271,271,271,271,271,109,109,109,271,271,271,271,271,271,
-271,271,271,271,271,271,271,271,271,271,271,271,271,271,271,271,
-271,271,109,271,271,271,271,271,271,271,271,271,109,271,109,109,
-271,271,271,271,271,271,271,109,109,109,272,109,109,109,109,273,
-270,270,272,272,272,109,272,109,270,270,270,270,270,270,270,273,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,270,270,274,109,109,109,109,109,109,109,109,109,109,109,
+114,114,278,278,114,279,279,279,279,279,279,279,279,279,279,279,
+279,279,279,279,279,279,279,114,114,114,279,279,279,279,279,279,
+279,279,279,279,279,279,279,279,279,279,279,279,279,279,279,279,
+279,279,114,279,279,279,279,279,279,279,279,279,114,279,114,114,
+279,279,279,279,279,279,279,114,114,114,280,114,114,114,114,281,
+278,278,280,280,280,114,280,114,278,278,278,278,278,278,278,281,
+114,114,114,114,114,114,282,282,282,282,282,282,282,282,282,282,
+114,114,278,278,283,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 28 */
-109,275,275,275,275,275,275,275,275,275,275,275,275,275,275,275,
-275,275,275,275,275,275,275,275,275,275,275,275,275,275,275,275,
-275,275,275,275,275,275,275,275,275,275,275,275,275,275,275,275,
-275,276,275,277,276,276,276,276,276,276,276,109,109,109,109,  5,
-275,275,275,275,275,275,278,276,276,276,276,276,276,276,276,279,
-280,280,280,280,280,280,280,280,280,280,279,279,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+114,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,
+284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,
+284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,
+284,285,284,286,285,285,285,285,285,285,285,114,114,114,114,  5,
+284,284,284,284,284,284,287,285,285,285,285,285,285,285,285,288,
+289,289,289,289,289,289,289,289,289,289,288,288,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 29 */
-109,281,281,109,281,109,109,281,281,109,281,109,109,281,109,109,
-109,109,109,109,281,281,281,281,109,281,281,281,281,281,281,281,
-109,281,281,281,109,281,109,281,109,109,281,281,109,281,281,281,
-281,282,281,283,282,282,282,282,282,282,109,282,282,281,109,109,
-281,281,281,281,281,109,284,109,282,282,282,282,282,282,109,109,
-285,285,285,285,285,285,285,285,285,285,109,109,281,281,281,281,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+114,290,290,114,290,114,114,290,290,114,290,114,114,290,114,114,
+114,114,114,114,290,290,290,290,114,290,290,290,290,290,290,290,
+114,290,290,290,114,290,114,290,114,114,290,290,114,290,290,290,
+290,291,290,292,291,291,291,291,291,291,114,291,291,290,114,114,
+290,290,290,290,290,114,293,114,291,291,291,291,291,291,114,114,
+294,294,294,294,294,294,294,294,294,294,114,114,290,290,290,290,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 30 */
-286,287,287,287,288,288,288,288,288,288,288,288,288,288,288,288,
-288,288,288,287,288,287,287,287,289,289,287,287,287,287,287,287,
-290,290,290,290,290,290,290,290,290,290,291,291,291,291,291,291,
-291,291,291,291,287,289,287,289,287,289,292,293,292,293,294,294,
-286,286,286,286,286,286,286,286,109,286,286,286,286,286,286,286,
-286,286,286,286,286,286,286,286,286,286,286,286,286,286,286,286,
-286,286,286,286,286,286,286,286,286,286,286,286,286,109,109,109,
-109,289,289,289,289,289,289,289,289,289,289,289,289,289,289,294,
+295,296,296,296,297,297,297,297,297,297,297,297,297,297,297,297,
+297,297,297,296,297,296,296,296,298,298,296,296,296,296,296,296,
+299,299,299,299,299,299,299,299,299,299,300,300,300,300,300,300,
+300,300,300,300,296,298,296,298,296,298,301,302,301,302,303,303,
+295,295,295,295,295,295,295,295,114,295,295,295,295,295,295,295,
+295,295,295,295,295,295,295,295,295,295,295,295,295,295,295,295,
+295,295,295,295,295,295,295,295,295,295,295,295,295,114,114,114,
+114,298,298,298,298,298,298,298,298,298,298,298,298,298,298,303,
 
 /* block 31 */
-289,289,289,289,289,288,289,289,286,286,286,286,286,289,289,289,
-289,289,289,289,289,289,289,289,109,289,289,289,289,289,289,289,
-289,289,289,289,289,289,289,289,289,289,289,289,289,289,289,289,
-289,289,289,289,289,289,289,289,289,289,289,289,289,109,287,287,
-287,287,287,287,287,287,289,287,287,287,287,287,287,109,287,287,
-288,288,288,288,288, 19, 19, 19, 19,288,288,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+298,298,298,298,298,297,298,298,295,295,295,295,295,298,298,298,
+298,298,298,298,298,298,298,298,114,298,298,298,298,298,298,298,
+298,298,298,298,298,298,298,298,298,298,298,298,298,298,298,298,
+298,298,298,298,298,298,298,298,298,298,298,298,298,114,296,296,
+296,296,296,296,296,296,298,296,296,296,296,296,296,114,296,296,
+297,297,297,297,297, 19, 19, 19, 19,297,297,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 32 */
-295,295,295,295,295,295,295,295,295,295,295,295,295,295,295,295,
-295,295,295,295,295,295,295,295,295,295,295,295,295,295,295,295,
-295,295,295,295,295,295,295,295,295,295,295,296,296,297,297,297,
-297,298,297,297,297,297,297,297,296,297,297,298,298,297,297,295,
-299,299,299,299,299,299,299,299,299,299,300,300,300,300,300,300,
-295,295,295,295,295,295,298,298,297,297,295,295,295,295,297,297,
-297,295,296,296,296,295,295,296,296,296,296,296,296,296,295,295,
-295,297,297,297,297,295,295,295,295,295,295,295,295,295,295,295,
+304,304,304,304,304,304,304,304,304,304,304,304,304,304,304,304,
+304,304,304,304,304,304,304,304,304,304,304,304,304,304,304,304,
+304,304,304,304,304,304,304,304,304,304,304,305,305,306,306,306,
+306,307,306,306,306,306,306,306,305,306,306,307,307,306,306,304,
+308,308,308,308,308,308,308,308,308,308,309,309,309,309,309,309,
+304,304,304,304,304,304,307,307,306,306,304,304,304,304,306,306,
+306,304,305,305,305,304,304,305,305,305,305,305,305,305,304,304,
+304,306,306,306,306,304,304,304,304,304,304,304,304,304,304,304,
 
 /* block 33 */
-295,295,297,296,298,297,297,296,296,296,296,296,296,297,295,296,
-299,299,299,299,299,299,299,299,299,299,296,296,296,297,301,301,
-302,302,302,302,302,302,302,302,302,302,302,302,302,302,302,302,
-302,302,302,302,302,302,302,302,302,302,302,302,302,302,302,302,
-302,302,302,302,302,302,109,302,109,109,109,109,109,302,109,109,
-303,303,303,303,303,303,303,303,303,303,303,303,303,303,303,303,
-303,303,303,303,303,303,303,303,303,303,303,303,303,303,303,303,
-303,303,303,303,303,303,303,303,303,303,303,  4,304,303,303,303,
+304,304,306,305,307,306,306,305,305,305,305,305,305,306,304,305,
+308,308,308,308,308,308,308,308,308,308,305,305,305,306,310,310,
+311,311,311,311,311,311,311,311,311,311,311,311,311,311,311,311,
+311,311,311,311,311,311,311,311,311,311,311,311,311,311,311,311,
+311,311,311,311,311,311,114,311,114,114,114,114,114,311,114,114,
+312,312,312,312,312,312,312,312,312,312,312,312,312,312,312,312,
+312,312,312,312,312,312,312,312,312,312,312,312,312,312,312,312,
+312,312,312,312,312,312,312,312,312,312,312,  4,313,312,312,312,
 
 /* block 34 */
-305,305,305,305,305,305,305,305,305,305,305,305,305,305,305,305,
-305,305,305,305,305,305,305,305,305,305,305,305,305,305,305,305,
-305,305,305,305,305,305,305,305,305,305,305,305,305,305,305,305,
-305,305,305,305,305,305,305,305,305,305,305,305,305,305,305,305,
-305,305,305,305,305,305,305,305,305,305,305,305,305,305,305,305,
-305,305,305,305,305,305,305,305,305,305,305,305,305,305,305,305,
-306,306,306,306,306,306,306,306,306,306,306,306,306,306,306,306,
-306,306,306,306,306,306,306,306,306,306,306,306,306,306,306,306,
+314,314,314,314,314,314,314,314,314,314,314,314,314,314,314,314,
+314,314,314,314,314,314,314,314,314,314,314,314,314,314,314,314,
+314,314,314,314,314,314,314,314,314,314,314,314,314,314,314,314,
+314,314,314,314,314,314,314,314,314,314,314,314,314,314,314,314,
+314,314,314,314,314,314,314,314,314,314,314,314,314,314,314,314,
+314,314,314,314,314,314,314,314,314,314,314,314,314,314,314,314,
+315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
+315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
 
 /* block 35 */
-306,306,306,306,306,306,306,306,306,306,306,306,306,306,306,306,
-306,306,306,306,306,306,306,306,306,306,306,306,306,306,306,306,
-306,306,306,306,306,306,306,306,307,307,307,307,307,307,307,307,
-307,307,307,307,307,307,307,307,307,307,307,307,307,307,307,307,
-307,307,307,307,307,307,307,307,307,307,307,307,307,307,307,307,
-307,307,307,307,307,307,307,307,307,307,307,307,307,307,307,307,
-307,307,307,307,307,307,307,307,307,307,307,307,307,307,307,307,
-307,307,307,307,307,307,307,307,307,307,307,307,307,307,307,307,
+315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
+315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
+315,315,315,315,315,315,315,315,316,316,316,316,316,316,316,316,
+316,316,316,316,316,316,316,316,316,316,316,316,316,316,316,316,
+316,316,316,316,316,316,316,316,316,316,316,316,316,316,316,316,
+316,316,316,316,316,316,316,316,316,316,316,316,316,316,316,316,
+316,316,316,316,316,316,316,316,316,316,316,316,316,316,316,316,
+316,316,316,316,316,316,316,316,316,316,316,316,316,316,316,316,
 
 /* block 36 */
-308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,
-308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,
-308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,
-308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,
-308,308,308,308,308,308,308,308,308,109,308,308,308,308,109,109,
-308,308,308,308,308,308,308,109,308,109,308,308,308,308,109,109,
-308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,
-308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,
+317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,
+317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,
+317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,
+317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,
+317,317,317,317,317,317,317,317,317,114,317,317,317,317,114,114,
+317,317,317,317,317,317,317,114,317,114,317,317,317,317,114,114,
+317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,
+317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,
 
 /* block 37 */
-308,308,308,308,308,308,308,308,308,109,308,308,308,308,109,109,
-308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,
-308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,
-308,109,308,308,308,308,109,109,308,308,308,308,308,308,308,109,
-308,109,308,308,308,308,109,109,308,308,308,308,308,308,308,308,
-308,308,308,308,308,308,308,109,308,308,308,308,308,308,308,308,
-308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,
-308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,
+317,317,317,317,317,317,317,317,317,114,317,317,317,317,114,114,
+317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,
+317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,
+317,114,317,317,317,317,114,114,317,317,317,317,317,317,317,114,
+317,114,317,317,317,317,114,114,317,317,317,317,317,317,317,317,
+317,317,317,317,317,317,317,114,317,317,317,317,317,317,317,317,
+317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,
+317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,
 
 /* block 38 */
-308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,
-308,109,308,308,308,308,109,109,308,308,308,308,308,308,308,308,
-308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,
-308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,
-308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,
-308,308,308,308,308,308,308,308,308,308,308,109,109,309,309,309,
-310,310,310,310,310,310,310,310,310,311,311,311,311,311,311,311,
-311,311,311,311,311,311,311,311,311,311,311,311,311,109,109,109,
+317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,
+317,114,317,317,317,317,114,114,317,317,317,317,317,317,317,317,
+317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,
+317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,
+317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,
+317,317,317,317,317,317,317,317,317,317,317,114,114,318,318,318,
+319,319,319,319,319,319,319,319,319,320,320,320,320,320,320,320,
+320,320,320,320,320,320,320,320,320,320,320,320,320,114,114,114,
 
 /* block 39 */
-308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,
-312,312,312,312,312,312,312,312,312,312,109,109,109,109,109,109,
-313,313,313,313,313,313,313,313,313,313,313,313,313,313,313,313,
-313,313,313,313,313,313,313,313,313,313,313,313,313,313,313,313,
-313,313,313,313,313,313,313,313,313,313,313,313,313,313,313,313,
-313,313,313,313,313,313,313,313,313,313,313,313,313,313,313,313,
-313,313,313,313,313,313,313,313,313,313,313,313,313,313,313,313,
-313,313,313,313,313,109,109,109,109,109,109,109,109,109,109,109,
+317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,
+321,321,321,321,321,321,321,321,321,321,114,114,114,114,114,114,
+322,322,322,322,322,322,322,322,322,322,322,322,322,322,322,322,
+322,322,322,322,322,322,322,322,322,322,322,322,322,322,322,322,
+322,322,322,322,322,322,322,322,322,322,322,322,322,322,322,322,
+322,322,322,322,322,322,322,322,322,322,322,322,322,322,322,322,
+322,322,322,322,322,322,322,322,322,322,322,322,322,322,322,322,
+322,322,322,322,322,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 40 */
-314,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
+323,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
 
 /* block 41 */
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
 
 /* block 42 */
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,316,316,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,325,325,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
 
 /* block 43 */
-317,318,318,318,318,318,318,318,318,318,318,318,318,318,318,318,
-318,318,318,318,318,318,318,318,318,318,318,319,320,109,109,109,
-321,321,321,321,321,321,321,321,321,321,321,321,321,321,321,321,
-321,321,321,321,321,321,321,321,321,321,321,321,321,321,321,321,
-321,321,321,321,321,321,321,321,321,321,321,321,321,321,321,321,
-321,321,321,321,321,321,321,321,321,321,321,321,321,321,321,321,
-321,321,321,321,321,321,321,321,321,321,321,  4,  4,  4,322,322,
-322,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+326,327,327,327,327,327,327,327,327,327,327,327,327,327,327,327,
+327,327,327,327,327,327,327,327,327,327,327,328,329,114,114,114,
+330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,
+330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,
+330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,
+330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,
+330,330,330,330,330,330,330,330,330,330,330,  4,  4,  4,331,331,
+331,330,330,330,330,330,330,330,330,114,114,114,114,114,114,114,
 
 /* block 44 */
-323,323,323,323,323,323,323,323,323,323,323,323,323,109,323,323,
-323,323,324,324,324,109,109,109,109,109,109,109,109,109,109,109,
-325,325,325,325,325,325,325,325,325,325,325,325,325,325,325,325,
-325,325,326,326,326,  4,  4,109,109,109,109,109,109,109,109,109,
-327,327,327,327,327,327,327,327,327,327,327,327,327,327,327,327,
-327,327,328,328,109,109,109,109,109,109,109,109,109,109,109,109,
-329,329,329,329,329,329,329,329,329,329,329,329,329,109,329,329,
-329,109,330,330,109,109,109,109,109,109,109,109,109,109,109,109,
+332,332,332,332,332,332,332,332,332,332,332,332,332,114,332,332,
+332,332,333,333,333,114,114,114,114,114,114,114,114,114,114,114,
+334,334,334,334,334,334,334,334,334,334,334,334,334,334,334,334,
+334,334,335,335,335,  4,  4,114,114,114,114,114,114,114,114,114,
+336,336,336,336,336,336,336,336,336,336,336,336,336,336,336,336,
+336,336,337,337,114,114,114,114,114,114,114,114,114,114,114,114,
+338,338,338,338,338,338,338,338,338,338,338,338,338,114,338,338,
+338,114,339,339,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 45 */
-331,331,331,331,331,331,331,331,331,331,331,331,331,331,331,331,
-331,331,331,331,331,331,331,331,331,331,331,331,331,331,331,331,
-331,331,331,331,331,331,331,331,331,331,331,331,331,331,331,331,
-331,331,331,331,332,332,333,332,332,332,332,332,332,332,333,333,
-333,333,333,333,333,333,332,333,333,332,332,332,332,332,332,332,
-332,332,332,332,334,334,334,335,334,334,334,336,331,332,109,109,
-337,337,337,337,337,337,337,337,337,337,109,109,109,109,109,109,
-338,338,338,338,338,338,338,338,338,338,109,109,109,109,109,109,
+340,340,340,340,340,340,340,340,340,340,340,340,340,340,340,340,
+340,340,340,340,340,340,340,340,340,340,340,340,340,340,340,340,
+340,340,340,340,340,340,340,340,340,340,340,340,340,340,340,340,
+340,340,340,340,341,341,342,341,341,341,341,341,341,341,342,342,
+342,342,342,342,342,342,341,342,342,341,341,341,341,341,341,341,
+341,341,341,341,343,343,343,344,343,343,343,345,340,341,114,114,
+346,346,346,346,346,346,346,346,346,346,114,114,114,114,114,114,
+347,347,347,347,347,347,347,347,347,347,114,114,114,114,114,114,
 
 /* block 46 */
-339,339,  4,  4,339,  4,340,339,339,339,339,341,341,341,342,109,
-343,343,343,343,343,343,343,343,343,343,109,109,109,109,109,109,
-344,344,344,344,344,344,344,344,344,344,344,344,344,344,344,344,
-344,344,344,344,344,344,344,344,344,344,344,344,344,344,344,344,
-344,344,344,345,344,344,344,344,344,344,344,344,344,344,344,344,
-344,344,344,344,344,344,344,344,344,344,344,344,344,344,344,344,
-344,344,344,344,344,344,344,344,344,344,344,344,344,344,344,344,
-344,344,344,344,344,344,344,344,109,109,109,109,109,109,109,109,
+348,348,  4,  4,348,  4,349,348,348,348,348,350,350,350,351,114,
+352,352,352,352,352,352,352,352,352,352,114,114,114,114,114,114,
+353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,
+353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,
+353,353,353,354,353,353,353,353,353,353,353,353,353,353,353,353,
+353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,
+353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,
+353,353,353,353,353,353,353,353,114,114,114,114,114,114,114,114,
 
 /* block 47 */
-344,344,344,344,344,344,344,344,344,344,344,344,344,344,344,344,
-344,344,344,344,344,344,344,344,344,344,344,344,344,344,344,344,
-344,344,344,344,344,344,344,344,344,341,344,109,109,109,109,109,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
-315,315,315,315,315,315,109,109,109,109,109,109,109,109,109,109,
+353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,
+353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,
+353,353,353,353,353,353,353,353,353,350,353,114,114,114,114,114,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,
+324,324,324,324,324,324,114,114,114,114,114,114,114,114,114,114,
 
 /* block 48 */
-346,346,346,346,346,346,346,346,346,346,346,346,346,346,346,346,
-346,346,346,346,346,346,346,346,346,346,346,346,346,109,109,109,
-347,347,347,348,348,348,348,347,347,348,348,348,109,109,109,109,
-348,348,347,348,348,348,348,348,348,347,347,347,109,109,109,109,
-349,109,109,109,350,350,351,351,351,351,351,351,351,351,351,351,
-352,352,352,352,352,352,352,352,352,352,352,352,352,352,352,352,
-352,352,352,352,352,352,352,352,352,352,352,352,352,352,109,109,
-352,352,352,352,352,109,109,109,109,109,109,109,109,109,109,109,
+355,355,355,355,355,355,355,355,355,355,355,355,355,355,355,355,
+355,355,355,355,355,355,355,355,355,355,355,355,355,355,355,114,
+356,356,356,357,357,357,357,356,356,357,357,357,114,114,114,114,
+357,357,356,357,357,357,357,357,357,356,356,356,114,114,114,114,
+358,114,114,114,359,359,360,360,360,360,360,360,360,360,360,360,
+361,361,361,361,361,361,361,361,361,361,361,361,361,361,361,361,
+361,361,361,361,361,361,361,361,361,361,361,361,361,361,114,114,
+361,361,361,361,361,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 49 */
-353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,
-353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,
-353,353,353,353,353,353,353,353,353,353,353,353,109,109,109,109,
-354,354,354,354,354,355,355,355,354,354,355,354,354,354,354,354,
-354,353,353,353,353,353,353,353,354,354,109,109,109,109,109,109,
-356,356,356,356,356,356,356,356,356,356,357,109,109,109,358,358,
-359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,
-359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,
+362,362,362,362,362,362,362,362,362,362,362,362,362,362,362,362,
+362,362,362,362,362,362,362,362,362,362,362,362,362,362,362,362,
+362,362,362,362,362,362,362,362,362,362,362,362,114,114,114,114,
+363,363,363,363,363,364,364,364,363,363,364,363,363,363,363,363,
+363,362,362,362,362,362,362,362,363,363,114,114,114,114,114,114,
+365,365,365,365,365,365,365,365,365,365,366,114,114,114,367,367,
+368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,
+368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,
 
 /* block 50 */
-360,360,360,360,360,360,360,360,360,360,360,360,360,360,360,360,
-360,360,360,360,360,360,360,361,361,362,362,362,109,109,363,363,
-364,364,364,364,364,364,364,364,364,364,364,364,364,364,364,364,
-364,364,364,364,364,364,364,364,364,364,364,364,364,364,364,364,
-364,364,364,364,364,364,364,364,364,364,364,364,364,364,364,364,
-364,364,364,364,364,365,366,365,366,366,366,366,366,366,366,109,
-366,367,366,367,367,366,366,366,366,366,366,366,366,365,365,365,
-365,365,365,366,366,366,366,366,366,366,366,366,366,109,109,366,
+369,369,369,369,369,369,369,369,369,369,369,369,369,369,369,369,
+369,369,369,369,369,369,369,370,370,371,371,370,114,114,372,372,
+373,373,373,373,373,373,373,373,373,373,373,373,373,373,373,373,
+373,373,373,373,373,373,373,373,373,373,373,373,373,373,373,373,
+373,373,373,373,373,373,373,373,373,373,373,373,373,373,373,373,
+373,373,373,373,373,374,375,374,375,375,375,375,375,375,375,114,
+375,376,375,376,376,375,375,375,375,375,375,375,375,374,374,374,
+374,374,374,375,375,375,375,375,375,375,375,375,375,114,114,375,
 
 /* block 51 */
-368,368,368,368,368,368,368,368,368,368,109,109,109,109,109,109,
-368,368,368,368,368,368,368,368,368,368,109,109,109,109,109,109,
-369,369,369,369,369,369,369,370,369,369,369,369,369,369,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+377,377,377,377,377,377,377,377,377,377,114,114,114,114,114,114,
+377,377,377,377,377,377,377,377,377,377,114,114,114,114,114,114,
+378,378,378,378,378,378,378,379,378,378,378,378,378,378,114,114,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,380,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 52 */
-371,371,371,371,372,373,373,373,373,373,373,373,373,373,373,373,
-373,373,373,373,373,373,373,373,373,373,373,373,373,373,373,373,
-373,373,373,373,373,373,373,373,373,373,373,373,373,373,373,373,
-373,373,373,373,371,372,371,371,371,371,371,372,371,372,372,372,
-372,372,371,372,372,373,373,373,373,373,373,373,109,109,109,109,
-374,374,374,374,374,374,374,374,374,374,375,375,375,375,375,375,
-375,376,376,376,376,376,376,376,376,376,376,371,371,371,371,371,
-371,371,371,371,376,376,376,376,376,376,376,376,376,109,109,109,
+381,381,381,381,382,383,383,383,383,383,383,383,383,383,383,383,
+383,383,383,383,383,383,383,383,383,383,383,383,383,383,383,383,
+383,383,383,383,383,383,383,383,383,383,383,383,383,383,383,383,
+383,383,383,383,381,382,381,381,381,381,381,382,381,382,382,382,
+382,382,381,382,382,383,383,383,383,383,383,383,114,114,114,114,
+384,384,384,384,384,384,384,384,384,384,385,385,385,385,385,385,
+385,386,386,386,386,386,386,386,386,386,386,381,381,381,381,381,
+381,381,381,381,386,386,386,386,386,386,386,386,386,114,114,114,
 
 /* block 53 */
-377,377,378,379,379,379,379,379,379,379,379,379,379,379,379,379,
-379,379,379,379,379,379,379,379,379,379,379,379,379,379,379,379,
-379,378,377,377,377,377,378,378,377,377,378,377,378,378,379,379,
-380,380,380,380,380,380,380,380,380,380,379,379,379,379,379,379,
-381,381,381,381,381,381,381,381,381,381,381,381,381,381,381,381,
-381,381,381,381,381,381,381,381,381,381,381,381,381,381,381,381,
-381,381,381,381,381,381,382,383,382,382,383,383,383,382,383,382,
-382,382,383,383,109,109,109,109,109,109,109,109,384,384,384,384,
+387,387,388,389,389,389,389,389,389,389,389,389,389,389,389,389,
+389,389,389,389,389,389,389,389,389,389,389,389,389,389,389,389,
+389,388,387,387,387,387,388,388,387,387,388,387,387,387,389,389,
+390,390,390,390,390,390,390,390,390,390,389,389,389,389,389,389,
+391,391,391,391,391,391,391,391,391,391,391,391,391,391,391,391,
+391,391,391,391,391,391,391,391,391,391,391,391,391,391,391,391,
+391,391,391,391,391,391,392,393,392,392,393,393,393,392,393,392,
+392,392,393,393,114,114,114,114,114,114,114,114,394,394,394,394,
 
 /* block 54 */
-385,385,385,385,385,385,385,385,385,385,385,385,385,385,385,385,
-385,385,385,385,385,385,385,385,385,385,385,385,385,385,385,385,
-385,385,385,385,386,386,386,386,386,386,386,386,387,387,387,387,
-387,387,387,387,386,386,387,387,109,109,109,388,388,388,388,388,
-389,389,389,389,389,389,389,389,389,389,109,109,109,385,385,385,
-390,390,390,390,390,390,390,390,390,390,391,391,391,391,391,391,
-391,391,391,391,391,391,391,391,391,391,391,391,391,391,391,391,
-391,391,391,391,391,391,391,391,392,392,392,392,392,392,393,393,
+395,395,395,395,395,395,395,395,395,395,395,395,395,395,395,395,
+395,395,395,395,395,395,395,395,395,395,395,395,395,395,395,395,
+395,395,395,395,396,396,396,396,396,396,396,396,397,397,397,397,
+397,397,397,397,396,396,397,397,114,114,114,398,398,398,398,398,
+399,399,399,399,399,399,399,399,399,399,114,114,114,395,395,395,
+400,400,400,400,400,400,400,400,400,400,401,401,401,401,401,401,
+401,401,401,401,401,401,401,401,401,401,401,401,401,401,401,401,
+401,401,401,401,401,401,401,401,402,402,402,402,402,402,403,403,
 
 /* block 55 */
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-394,394,394,394,394,394,394,394,109,109,109,109,109,109,109,109,
-104,104,104,  4,104,104,104,104,104,104,104,104,104,104,104,104,
-104,395,104,104,104,104,104,104,104,396,396,396,396,104,396,396,
-396,396,395,395,104,396,396,109,109,109,109,109,109,109,109,109,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+404,404,404,404,404,404,404,404,114,114,114,114,114,114,114,114,
+109,109,109,  4,109,109,109,109,109,109,109,109,109,109,109,109,
+109,405,109,109,109,109,109,109,109,406,406,406,406,109,406,406,
+406,406,405,405,109,406,406,114,109,109,114,114,114,114,114,114,
 
 /* block 56 */
  33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33,
  33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33,
- 33, 33, 33, 33, 33, 33,116,116,116,116,116,397,101,101,101,101,
-101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,
-101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,
-101,101,101,101,101,101,101,101,101,101,101,101,101,110,110,110,
-110,110,101,101,101,101,110,110,110,110,110, 33, 33, 33, 33, 33,
- 33, 33, 33, 33, 33, 33, 33, 33,398,399, 33, 33, 33,400, 33, 33,
+ 33, 33, 33, 33, 33, 33,122,122,122,122,122,407,106,106,106,106,
+106,106,106,106,106,106,106,106,106,106,106,106,106,106,106,106,
+106,106,106,106,106,106,106,106,106,106,106,106,106,106,106,106,
+106,106,106,106,106,106,106,106,106,106,106,106,106,115,115,115,
+115,115,106,106,106,106,115,115,115,115,115, 33, 33, 33, 33, 33,
+ 33, 33, 33, 33, 33, 33, 33, 33,408,409, 33, 33, 33,410, 33, 33,
 
 /* block 57 */
  33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33,
- 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33,101,101,101,101,101,
-101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,
-101,101,101,101,101,101,101,101,101,101,101,101,101,101,101,110,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,104,104,104,104,
+ 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33,106,106,106,106,106,
+106,106,106,106,106,106,106,106,106,106,106,106,106,106,106,106,
+106,106,106,106,106,106,106,106,106,106,106,106,106,106,106,115,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,114,114,114,114,114,114,109,109,109,109,
 
 /* block 58 */
  30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
@@ -1845,12 +1937,12 @@ const pcre_uint16 PRIV(ucd_stage2)[] = { /* 51968 bytes, block = 128 */
  30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
  30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
  30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
-401,402, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
+411,412, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
  30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
 
 /* block 59 */
  30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
- 30, 31, 30, 31, 30, 31, 33, 33, 33, 33, 33,403, 33, 33,404, 33,
+ 30, 31, 30, 31, 30, 31, 33, 33, 33, 33, 33,413, 33, 33,414, 33,
  30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
  30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
  30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
@@ -1859,57 +1951,57 @@ const pcre_uint16 PRIV(ucd_stage2)[] = { /* 51968 bytes, block = 128 */
  30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
 
 /* block 60 */
-405,405,405,405,405,405,405,405,406,406,406,406,406,406,406,406,
-405,405,405,405,405,405,109,109,406,406,406,406,406,406,109,109,
-405,405,405,405,405,405,405,405,406,406,406,406,406,406,406,406,
-405,405,405,405,405,405,405,405,406,406,406,406,406,406,406,406,
-405,405,405,405,405,405,109,109,406,406,406,406,406,406,109,109,
-116,405,116,405,116,405,116,405,109,406,109,406,109,406,109,406,
-405,405,405,405,405,405,405,405,406,406,406,406,406,406,406,406,
-407,407,408,408,408,408,409,409,410,410,411,411,412,412,109,109,
+415,415,415,415,415,415,415,415,416,416,416,416,416,416,416,416,
+415,415,415,415,415,415,114,114,416,416,416,416,416,416,114,114,
+415,415,415,415,415,415,415,415,416,416,416,416,416,416,416,416,
+415,415,415,415,415,415,415,415,416,416,416,416,416,416,416,416,
+415,415,415,415,415,415,114,114,416,416,416,416,416,416,114,114,
+122,415,122,415,122,415,122,415,114,416,114,416,114,416,114,416,
+415,415,415,415,415,415,415,415,416,416,416,416,416,416,416,416,
+417,417,418,418,418,418,419,419,420,420,421,421,422,422,114,114,
 
 /* block 61 */
-405,405,405,405,405,405,405,405,413,413,413,413,413,413,413,413,
-405,405,405,405,405,405,405,405,413,413,413,413,413,413,413,413,
-405,405,405,405,405,405,405,405,413,413,413,413,413,413,413,413,
-405,405,116,414,116,109,116,116,406,406,415,415,416,108,417,108,
-108,108,116,414,116,109,116,116,418,418,418,418,416,108,108,108,
-405,405,116,116,109,109,116,116,406,406,419,419,109,108,108,108,
-405,405,116,116,116,157,116,116,406,406,420,420,161,108,108,108,
-109,109,116,414,116,109,116,116,421,421,422,422,416,108,108,109,
+415,415,415,415,415,415,415,415,423,423,423,423,423,423,423,423,
+415,415,415,415,415,415,415,415,423,423,423,423,423,423,423,423,
+415,415,415,415,415,415,415,415,423,423,423,423,423,423,423,423,
+415,415,122,424,122,114,122,122,416,416,425,425,426,113,427,113,
+113,113,122,424,122,114,122,122,428,428,428,428,426,113,113,113,
+415,415,122,122,114,114,122,122,416,416,429,429,114,113,113,113,
+415,415,122,122,122,163,122,122,416,416,430,430,168,113,113,113,
+114,114,122,424,122,114,122,122,431,431,432,432,426,113,113,114,
 
 /* block 62 */
-  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3, 22,423,423, 22, 22,
+  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3, 22,433,433, 22, 22,
   9,  9,  9,  9,  9,  9,  4,  4, 21, 25,  6, 21, 21, 25,  6, 21,
-  4,  4,  4,  4,  4,  4,  4,  4,424,425, 22, 22, 22, 22, 22,  3,
+  4,  4,  4,  4,  4,  4,  4,  4,434,435, 22, 22, 22, 22, 22,  3,
   4,  4,  4,  4,  4,  4,  4,  4,  4, 21, 25,  4,  4,  4,  4, 15,
  15,  4,  4,  4,  8,  6,  7,  4,  4,  4,  4,  4,  4,  4,  4,  4,
   4,  4,  8,  4, 15,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  3,
- 22, 22, 22, 22, 22,426,426,426,426,426, 22, 22, 22, 22, 22, 22,
- 23,101,109,109, 23, 23, 23, 23, 23, 23,  8,  8,  8,  6,  7,101,
+ 22, 22, 22, 22, 22,436, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 23,106,114,114, 23, 23, 23, 23, 23, 23,  8,  8,  8,  6,  7,106,
 
 /* block 63 */
- 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,  8,  8,  8,  6,  7,109,
-101,101,101,101,101,101,101,101,101,101,101,101,101,109,109,109,
+ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,  8,  8,  8,  6,  7,114,
+106,106,106,106,106,106,106,106,106,106,106,106,106,114,114,114,
   5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,
-  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-104,104,104,104,104,104,104,104,104,104,104,104,104,427,427,427,
-427,104,427,427,427,104,104,104,104,104,104,104,104,104,104,104,
-104,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+109,109,109,109,109,109,109,109,109,109,109,109,109,380,380,380,
+380,109,380,380,380,109,109,109,109,109,109,109,109,109,109,109,
+109,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 64 */
- 19, 19,428, 19, 19, 19, 19,428, 19, 19,429,428,428,428,429,429,
-428,428,428,429, 19,428, 19, 19,  8,428,428,428,428,428, 19, 19,
- 19, 19, 19, 19,428, 19,430, 19,428, 19,431,432,428,428, 19,429,
-428,428,433,428,429,396,396,396,396,429, 19, 19,429,429,428,428,
-  8,  8,  8,  8,  8,428,429,429,429,429, 19,  8, 19, 19,434, 19,
+ 19, 19,437, 19, 19, 19, 19,437, 19, 19,438,437,437,437,438,438,
+437,437,437,438, 19,437, 19, 19,  8,437,437,437,437,437, 19, 19,
+ 19, 19, 19, 19,437, 19,439, 19,437, 19,440,441,437,437, 19,438,
+437,437,442,437,438,406,406,406,406,438, 19, 19,438,438,437,437,
+  8,  8,  8,  8,  8,437,438,438,438,438, 19,  8, 19, 19,443, 19,
  23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
-435,435,435,435,435,435,435,435,435,435,435,435,435,435,435,435,
-436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,
+444,444,444,444,444,444,444,444,444,444,444,444,444,444,444,444,
+445,445,445,445,445,445,445,445,445,445,445,445,445,445,445,445,
 
 /* block 65 */
-437,437,437, 30, 31,437,437,437,437, 23,109,109,109,109,109,109,
+446,446,446, 30, 31,446,446,446,446, 23,114,114,114,114,114,114,
   8,  8,  8,  8,  8, 19, 19, 19, 19, 19,  8,  8, 19, 19, 19, 19,
   8, 19, 19,  8, 19, 19,  8, 19, 19, 19, 19, 19, 19, 19,  8, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
@@ -1929,7 +2021,7 @@ const pcre_uint16 PRIV(ucd_stage2)[] = { /* 51968 bytes, block = 128 */
   8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,
 
 /* block 67 */
- 19, 19, 19, 19, 19, 19, 19, 19,  8,  8,  8,  8, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19,  6,  7,  6,  7, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
   8,  8, 19, 19, 19, 19, 19, 19, 19,  6,  7, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
@@ -1946,15 +2038,15 @@ const pcre_uint16 PRIV(ucd_stage2)[] = { /* 51968 bytes, block = 128 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,  8,  8,  8,  8,
   8,  8, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19,109,109,109,109,109,109,109,109,109,109,109,109,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,114,114,
 
 /* block 69 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19, 19,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
- 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+ 19, 19, 19, 19, 19, 19, 19,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
  23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
  23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
 
@@ -1962,10 +2054,10 @@ const pcre_uint16 PRIV(ucd_stage2)[] = { /* 51968 bytes, block = 128 */
  23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
  23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19,438,438,438,438,438,438,438,438,438,438,
-438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,
-439,439,439,439,439,439,439,439,439,439,439,439,439,439,439,439,
-439,439,439,439,439,439,439,439,439,439, 23, 23, 23, 23, 23, 23,
+ 19, 19, 19, 19, 19, 19,447,447,447,447,447,447,447,447,447,447,
+447,447,447,447,447,447,447,447,447,447,447,447,447,447,447,447,
+448,448,448,448,448,448,448,448,448,448,448,448,448,448,448,448,
+448,448,448,448,448,448,448,448,448,448, 23, 23, 23, 23, 23, 23,
  23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
 
 /* block 71 */
@@ -1999,7 +2091,7 @@ const pcre_uint16 PRIV(ucd_stage2)[] = { /* 51968 bytes, block = 128 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
 
 /* block 74 */
-109, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
@@ -2019,14 +2111,14 @@ const pcre_uint16 PRIV(ucd_stage2)[] = { /* 51968 bytes, block = 128 */
   8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,
 
 /* block 76 */
-440,440,440,440,440,440,440,440,440,440,440,440,440,440,440,440,
-440,440,440,440,440,440,440,440,440,440,440,440,440,440,440,440,
-440,440,440,440,440,440,440,440,440,440,440,440,440,440,440,440,
-440,440,440,440,440,440,440,440,440,440,440,440,440,440,440,440,
-440,440,440,440,440,440,440,440,440,440,440,440,440,440,440,440,
-440,440,440,440,440,440,440,440,440,440,440,440,440,440,440,440,
-440,440,440,440,440,440,440,440,440,440,440,440,440,440,440,440,
-440,440,440,440,440,440,440,440,440,440,440,440,440,440,440,440,
+449,449,449,449,449,449,449,449,449,449,449,449,449,449,449,449,
+449,449,449,449,449,449,449,449,449,449,449,449,449,449,449,449,
+449,449,449,449,449,449,449,449,449,449,449,449,449,449,449,449,
+449,449,449,449,449,449,449,449,449,449,449,449,449,449,449,449,
+449,449,449,449,449,449,449,449,449,449,449,449,449,449,449,449,
+449,449,449,449,449,449,449,449,449,449,449,449,449,449,449,449,
+449,449,449,449,449,449,449,449,449,449,449,449,449,449,449,449,
+449,449,449,449,449,449,449,449,449,449,449,449,449,449,449,449,
 
 /* block 77 */
   8,  8,  8,  6,  7,  6,  7,  6,  7,  6,  7,  6,  7,  6,  7,  6,
@@ -2043,150 +2135,150 @@ const pcre_uint16 PRIV(ucd_stage2)[] = { /* 51968 bytes, block = 128 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
   8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,
-  8,  8,  8,  8,  8, 19, 19,  8,  8,  8,  8,  8,  8,109,109,109,
- 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+  8,  8,  8,  8,  8, 19, 19,  8,  8,  8,  8,  8,  8, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19,114,114, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
 
 /* block 79 */
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19,114,114, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,114, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19,114, 19, 19, 19, 19, 19, 19,
+ 19, 19,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 80 */
-441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,
-441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,
-441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,109,
-442,442,442,442,442,442,442,442,442,442,442,442,442,442,442,442,
-442,442,442,442,442,442,442,442,442,442,442,442,442,442,442,442,
-442,442,442,442,442,442,442,442,442,442,442,442,442,442,442,109,
- 30, 31,443,444,445,446,447, 30, 31, 30, 31, 30, 31,448,449,450,
-451, 33, 30, 31, 33, 30, 31, 33, 33, 33, 33, 33,101,101,452,452,
+450,450,450,450,450,450,450,450,450,450,450,450,450,450,450,450,
+450,450,450,450,450,450,450,450,450,450,450,450,450,450,450,450,
+450,450,450,450,450,450,450,450,450,450,450,450,450,450,450,114,
+451,451,451,451,451,451,451,451,451,451,451,451,451,451,451,451,
+451,451,451,451,451,451,451,451,451,451,451,451,451,451,451,451,
+451,451,451,451,451,451,451,451,451,451,451,451,451,451,451,114,
+ 30, 31,452,453,454,455,456, 30, 31, 30, 31, 30, 31,457,458,459,
+460, 33, 30, 31, 33, 30, 31, 33, 33, 33, 33, 33,106,106,461,461,
 
 /* block 81 */
-153,154,153,154,153,154,153,154,153,154,153,154,153,154,153,154,
-153,154,153,154,153,154,153,154,153,154,153,154,153,154,153,154,
-153,154,153,154,153,154,153,154,153,154,153,154,153,154,153,154,
-153,154,153,154,153,154,153,154,153,154,153,154,153,154,153,154,
-153,154,153,154,153,154,153,154,153,154,153,154,153,154,153,154,
-153,154,153,154,153,154,153,154,153,154,153,154,153,154,153,154,
-153,154,153,154,453,454,454,454,454,454,454,153,154,153,154,455,
-455,455,153,154,109,109,109,109,109,456,456,456,456,457,456,456,
+159,160,159,160,159,160,159,160,159,160,159,160,159,160,159,160,
+159,160,159,160,159,160,159,160,159,160,159,160,159,160,159,160,
+159,160,159,160,159,160,159,160,159,160,159,160,159,160,159,160,
+159,160,159,160,159,160,159,160,159,160,159,160,159,160,159,160,
+159,160,159,160,159,160,159,160,159,160,159,160,159,160,159,160,
+159,160,159,160,159,160,159,160,159,160,159,160,159,160,159,160,
+159,160,159,160,462,463,463,463,463,463,463,159,160,159,160,464,
+464,464,159,160,114,114,114,114,114,465,465,465,465,466,465,465,
 
 /* block 82 */
-458,458,458,458,458,458,458,458,458,458,458,458,458,458,458,458,
-458,458,458,458,458,458,458,458,458,458,458,458,458,458,458,458,
-458,458,458,458,458,458,109,458,109,109,109,109,109,458,109,109,
-459,459,459,459,459,459,459,459,459,459,459,459,459,459,459,459,
-459,459,459,459,459,459,459,459,459,459,459,459,459,459,459,459,
-459,459,459,459,459,459,459,459,459,459,459,459,459,459,459,459,
-459,459,459,459,459,459,459,459,109,109,109,109,109,109,109,460,
-461,109,109,109,109,109,109,109,109,109,109,109,109,109,109,462,
+467,467,467,467,467,467,467,467,467,467,467,467,467,467,467,467,
+467,467,467,467,467,467,467,467,467,467,467,467,467,467,467,467,
+467,467,467,467,467,467,114,467,114,114,114,114,114,467,114,114,
+468,468,468,468,468,468,468,468,468,468,468,468,468,468,468,468,
+468,468,468,468,468,468,468,468,468,468,468,468,468,468,468,468,
+468,468,468,468,468,468,468,468,468,468,468,468,468,468,468,468,
+468,468,468,468,468,468,468,468,114,114,114,114,114,114,114,469,
+470,114,114,114,114,114,114,114,114,114,114,114,114,114,114,471,
 
 /* block 83 */
-308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,308,
-308,308,308,308,308,308,308,109,109,109,109,109,109,109,109,109,
-308,308,308,308,308,308,308,109,308,308,308,308,308,308,308,109,
-308,308,308,308,308,308,308,109,308,308,308,308,308,308,308,109,
-308,308,308,308,308,308,308,109,308,308,308,308,308,308,308,109,
-308,308,308,308,308,308,308,109,308,308,308,308,308,308,308,109,
-170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,
-170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,
+317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,317,
+317,317,317,317,317,317,317,114,114,114,114,114,114,114,114,114,
+317,317,317,317,317,317,317,114,317,317,317,317,317,317,317,114,
+317,317,317,317,317,317,317,114,317,317,317,317,317,317,317,114,
+317,317,317,317,317,317,317,114,317,317,317,317,317,317,317,114,
+317,317,317,317,317,317,317,114,317,317,317,317,317,317,317,114,
+177,177,177,177,177,177,177,177,177,177,177,177,177,177,177,177,
+177,177,177,177,177,177,177,177,177,177,177,177,177,177,177,177,
 
 /* block 84 */
   4,  4, 21, 25, 21, 25,  4,  4,  4, 21, 25,  4, 21, 25,  4,  4,
   4,  4,  4,  4,  4,  4,  4,  9,  4,  4,  9,  4, 21, 25,  4,  4,
- 21, 25,  6,  7,  6,  7,  6,  7,  6,  7,  4,  4,  4,  4,  4,102,
-  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  9,  9,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+ 21, 25,  6,  7,  6,  7,  6,  7,  6,  7,  4,  4,  4,  4,  4,107,
+  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  9,  9,  4,  4,  4,  4,
+  9,  4,  6,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 85 */
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,109,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,109,109,109,109,109,109,109,109,109,109,109,109,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,114,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 86 */
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
 
 /* block 87 */
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,463,
-463,463,463,463,463,463,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
- 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,109,109,109,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+472,472,472,472,472,472,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,114,
 
 /* block 88 */
-  3,  4,  4,  4, 19,464,396,465,  6,  7,  6,  7,  6,  7,  6,  7,
+  3,  4,  4,  4, 19,473,406,474,  6,  7,  6,  7,  6,  7,  6,  7,
   6,  7, 19, 19,  6,  7,  6,  7,  6,  7,  6,  7,  9,  6,  7,  7,
- 19,465,465,465,465,465,465,465,465,465,104,104,104,104,466,466,
-  9,102,102,102,102,102, 19, 19,465,465,465,464,396,  4, 19, 19,
-109,467,467,467,467,467,467,467,467,467,467,467,467,467,467,467,
-467,467,467,467,467,467,467,467,467,467,467,467,467,467,467,467,
-467,467,467,467,467,467,467,467,467,467,467,467,467,467,467,467,
-467,467,467,467,467,467,467,467,467,467,467,467,467,467,467,467,
+ 19,474,474,474,474,474,474,474,474,474,109,109,109,109,475,475,
+  9,107,107,107,107,107, 19, 19,474,474,474,473,406,  4, 19, 19,
+114,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
+476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
+476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
+476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
 
 /* block 89 */
-467,467,467,467,467,467,467,467,467,467,467,467,467,467,467,467,
-467,467,467,467,467,467,467,109,109,104,104, 14, 14,468,468,467,
-  9,469,469,469,469,469,469,469,469,469,469,469,469,469,469,469,
-469,469,469,469,469,469,469,469,469,469,469,469,469,469,469,469,
-469,469,469,469,469,469,469,469,469,469,469,469,469,469,469,469,
-469,469,469,469,469,469,469,469,469,469,469,469,469,469,469,469,
-469,469,469,469,469,469,469,469,469,469,469,469,469,469,469,469,
-469,469,469,469,469,469,469,469,469,469,469,  4,102,470,470,469,
+476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
+476,476,476,476,476,476,476,114,114,109,109, 14, 14,477,477,476,
+  9,478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,
+478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,
+478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,
+478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,
+478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,
+478,478,478,478,478,478,478,478,478,478,478,  4,107,479,479,478,
 
 /* block 90 */
-109,109,109,109,109,471,471,471,471,471,471,471,471,471,471,471,
-471,471,471,471,471,471,471,471,471,471,471,471,471,471,471,471,
-471,471,471,471,471,471,471,471,471,471,471,471,471,471,109,109,
-109,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
-472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
-472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
-472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
-472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
+114,114,114,114,114,480,480,480,480,480,480,480,480,480,480,480,
+480,480,480,480,480,480,480,480,480,480,480,480,480,480,480,480,
+480,480,480,480,480,480,480,480,480,480,480,480,480,480,114,114,
+114,481,481,481,481,481,481,481,481,481,481,481,481,481,481,481,
+481,481,481,481,481,481,481,481,481,481,481,481,481,481,481,481,
+481,481,481,481,481,481,481,481,481,481,481,481,481,481,481,481,
+481,481,481,481,481,481,481,481,481,481,481,481,481,481,481,481,
+481,481,481,481,481,481,481,481,481,481,481,481,481,481,481,481,
 
 /* block 91 */
-472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,109,
+481,481,481,481,481,481,481,481,481,481,481,481,481,481,481,114,
  19, 19, 23, 23, 23, 23, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
-471,471,471,471,471,471,471,471,471,471,471,471,471,471,471,471,
-471,471,471,471,471,471,471,471,471,471,471,109,109,109,109,109,
+480,480,480,480,480,480,480,480,480,480,480,480,480,480,480,480,
+480,480,480,480,480,480,480,480,480,480,480,114,114,114,114,114,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19,109,109,109,109,109,109,109,109,109,109,109,109,
-469,469,469,469,469,469,469,469,469,469,469,469,469,469,469,469,
+ 19, 19, 19, 19,114,114,114,114,114,114,114,114,114,114,114,114,
+478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,
 
 /* block 92 */
-473,473,473,473,473,473,473,473,473,473,473,473,473,473,473,473,
-473,473,473,473,473,473,473,473,473,473,473,473,473,473,473,109,
+482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,
+482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,114,
  23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 23, 23, 23, 23, 23, 23, 23, 23,
  19, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
-473,473,473,473,473,473,473,473,473,473,473,473,473,473,473,473,
-473,473,473,473,473,473,473,473,473,473,473,473,473,473,473, 19,
+482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,
+482,482,482,482,482,482,482,482,482,482,482,482,482,482,482, 19,
 
 /* block 93 */
  23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 19, 19, 19, 19, 19, 19,
@@ -2194,731 +2286,931 @@ const pcre_uint16 PRIV(ucd_stage2)[] = { /* 51968 bytes, block = 128 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
-474,474,474,474,474,474,474,474,474,474,474,474,474,474,474,474,
-474,474,474,474,474,474,474,474,474,474,474,474,474,474,474,474,
-474,474,474,474,474,474,474,474,474,474,474,474,474,474,474,109,
+483,483,483,483,483,483,483,483,483,483,483,483,483,483,483,483,
+483,483,483,483,483,483,483,483,483,483,483,483,483,483,483,483,
+483,483,483,483,483,483,483,483,483,483,483,483,483,483,483,114,
 
 /* block 94 */
-474,474,474,474,474,474,474,474,474,474,474,474,474,474,474,474,
-474,474,474,474,474,474,474,474,474,474,474,474,474,474,474,474,
-474,474,474,474,474,474,474,474,474,474,474,474,474,474,474,474,
-474,474,474,474,474,474,474,474,474,474,474,474,474,474,474,474,
-474,474,474,474,474,474,474,474,474,474,474,474,474,474,474,474,
-474,474,474,474,474,474,474,474, 19, 19, 19, 19, 19, 19, 19, 19,
+483,483,483,483,483,483,483,483,483,483,483,483,483,483,483,483,
+483,483,483,483,483,483,483,483,483,483,483,483,483,483,483,483,
+483,483,483,483,483,483,483,483,483,483,483,483,483,483,483,483,
+483,483,483,483,483,483,483,483,483,483,483,483,483,483,483,483,
+483,483,483,483,483,483,483,483,483,483,483,483,483,483,483,483,
+483,483,483,483,483,483,483,483, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
 
 /* block 95 */
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
 
 /* block 96 */
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,109,109,109,109,109,109,109,109,109,109,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,114,114,114,114,114,114,114,114,114,114,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
 
 /* block 97 */
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 98 */
-476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
-476,476,476,476,476,477,476,476,476,476,476,476,476,476,476,476,
-476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
-476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
-476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
-476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
-476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
-476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
+485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,
+485,485,485,485,485,486,485,485,485,485,485,485,485,485,485,485,
+485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,
+485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,
+485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,
+485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,
+485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,
+485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,
 
 /* block 99 */
-476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
-476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
-476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
-476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
-476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
-476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
-476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
-476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,476,
+485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,
+485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,
+485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,
+485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,
+485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,
+485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,
+485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,
+485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,485,
 
 /* block 100 */
-476,476,476,476,476,476,476,476,476,476,476,476,476,109,109,109,
-478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,
-478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,
-478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,
-478,478,478,478,478,478,478,109,109,109,109,109,109,109,109,109,
-479,479,479,479,479,479,479,479,479,479,479,479,479,479,479,479,
-479,479,479,479,479,479,479,479,479,479,479,479,479,479,479,479,
-479,479,479,479,479,479,479,479,480,480,480,480,480,480,481,481,
+485,485,485,485,485,485,485,485,485,485,485,485,485,114,114,114,
+487,487,487,487,487,487,487,487,487,487,487,487,487,487,487,487,
+487,487,487,487,487,487,487,487,487,487,487,487,487,487,487,487,
+487,487,487,487,487,487,487,487,487,487,487,487,487,487,487,487,
+487,487,487,487,487,487,487,114,114,114,114,114,114,114,114,114,
+488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
+488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
+488,488,488,488,488,488,488,488,489,489,489,489,489,489,490,490,
 
 /* block 101 */
-482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,
-482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,
-482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,
-482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,
-482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,
-482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,
-482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,
-482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,
+491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,
+491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,
+491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,
+491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,
+491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,
+491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,
+491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,
+491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,
 
 /* block 102 */
-482,482,482,482,482,482,482,482,482,482,482,482,483,484,484,484,
-482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,482,
-485,485,485,485,485,485,485,485,485,485,482,482,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-167,168,167,168,167,168,167,168,167,168,167,168,167,168,167,168,
-167,168,167,168,167,168,167,168,167,168,167,168,167,168,167,168,
-167,168,167,168,167,168,167,168,167,168,167,168,167,168,486,170,
-171,171,171,487,170,170,170,170,170,170,170,170,170,170,487,398,
+491,491,491,491,491,491,491,491,491,491,491,491,492,493,493,493,
+491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,491,
+494,494,494,494,494,494,494,494,494,494,491,491,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+174,175,174,175,174,175,174,175,174,175,174,175,174,175,174,175,
+174,175,174,175,174,175,174,175,174,175,174,175,174,175,174,175,
+174,175,174,175,174,175,174,175,174,175,174,175,174,175,495,177,
+178,178,178,496,177,177,177,177,177,177,177,177,177,177,496,408,
 
 /* block 103 */
-167,168,167,168,167,168,167,168,167,168,167,168,167,168,167,168,
-167,168,167,168,167,168,167,168,109,109,109,109,109,109,109,170,
-488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
-488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
-488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
-488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
-488,488,488,488,488,488,489,489,489,489,489,489,489,489,489,489,
-490,490,491,491,491,491,491,491,109,109,109,109,109,109,109,109,
+174,175,174,175,174,175,174,175,174,175,174,175,174,175,174,175,
+174,175,174,175,174,175,174,175,174,175,174,175,408,408,114,177,
+497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,
+497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,
+497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,
+497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,
+497,497,497,497,497,497,498,498,498,498,498,498,498,498,498,498,
+499,499,500,500,500,500,500,500,114,114,114,114,114,114,114,114,
 
 /* block 104 */
  14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
- 14, 14, 14, 14, 14, 14, 14,102,102,102,102,102,102,102,102,102,
+ 14, 14, 14, 14, 14, 14, 14,107,107,107,107,107,107,107,107,107,
  14, 14, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
  33, 33, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
  30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
  30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
  30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
-101, 33, 33, 33, 33, 33, 33, 33, 33, 30, 31, 30, 31,492, 30, 31,
+106, 33, 33, 33, 33, 33, 33, 33, 33, 30, 31, 30, 31,501, 30, 31,
 
 /* block 105 */
- 30, 31, 30, 31, 30, 31, 30, 31,102, 14, 14, 30, 31,493, 33,109,
- 30, 31, 30, 31,109,109,109,109,109,109,109,109,109,109,109,109,
- 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,494,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,101,101, 33, 20, 20, 20, 20, 20,
+ 30, 31, 30, 31, 30, 31, 30, 31,107, 14, 14, 30, 31,502, 33,114,
+ 30, 31, 30, 31, 33, 33, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,
+ 30, 31, 30, 31, 30, 31, 30, 31, 30, 31,503,504,505,506,114,114,
+507,508,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114, 20,106,106, 33, 20, 20, 20, 20, 20,
 
 /* block 106 */
-495,495,496,495,495,495,496,495,495,495,495,496,495,495,495,495,
-495,495,495,495,495,495,495,495,495,495,495,495,495,495,495,495,
-495,495,495,497,497,496,496,497,498,498,498,498,109,109,109,109,
- 23, 23, 23, 23, 23, 23, 19, 19,  5, 19,109,109,109,109,109,109,
-499,499,499,499,499,499,499,499,499,499,499,499,499,499,499,499,
-499,499,499,499,499,499,499,499,499,499,499,499,499,499,499,499,
-499,499,499,499,499,499,499,499,499,499,499,499,499,499,499,499,
-499,499,499,499,500,500,500,500,109,109,109,109,109,109,109,109,
+509,509,510,509,509,509,510,509,509,509,509,510,509,509,509,509,
+509,509,509,509,509,509,509,509,509,509,509,509,509,509,509,509,
+509,509,509,511,511,510,510,511,512,512,512,512,114,114,114,114,
+ 23, 23, 23, 23, 23, 23, 19, 19,  5, 19,114,114,114,114,114,114,
+513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,
+513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,
+513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,
+513,513,513,513,514,514,514,514,114,114,114,114,114,114,114,114,
 
 /* block 107 */
-501,501,502,502,502,502,502,502,502,502,502,502,502,502,502,502,
-502,502,502,502,502,502,502,502,502,502,502,502,502,502,502,502,
-502,502,502,502,502,502,502,502,502,502,502,502,502,502,502,502,
-502,502,502,502,501,501,501,501,501,501,501,501,501,501,501,501,
-501,501,501,501,503,109,109,109,109,109,109,109,109,109,504,504,
-505,505,505,505,505,505,505,505,505,505,109,109,109,109,109,109,
-213,213,213,213,213,213,213,213,213,213,213,213,213,213,213,213,
-213,213,215,215,215,215,215,215,217,217,217,215,109,109,109,109,
+515,515,516,516,516,516,516,516,516,516,516,516,516,516,516,516,
+516,516,516,516,516,516,516,516,516,516,516,516,516,516,516,516,
+516,516,516,516,516,516,516,516,516,516,516,516,516,516,516,516,
+516,516,516,516,515,515,515,515,515,515,515,515,515,515,515,515,
+515,515,515,515,517,114,114,114,114,114,114,114,114,114,518,518,
+519,519,519,519,519,519,519,519,519,519,114,114,114,114,114,114,
+221,221,221,221,221,221,221,221,221,221,221,221,221,221,221,221,
+221,221,223,223,223,223,223,223,225,225,225,223,114,114,114,114,
 
 /* block 108 */
-506,506,506,506,506,506,506,506,506,506,507,507,507,507,507,507,
-507,507,507,507,507,507,507,507,507,507,507,507,507,507,507,507,
-507,507,507,507,507,507,508,508,508,508,508,508,508,508,509,509,
-510,510,510,510,510,510,510,510,510,510,510,510,510,510,510,510,
-510,510,510,510,510,510,510,511,511,511,511,511,511,511,511,511,
-511,511,512,512,109,109,109,109,109,109,109,109,109,109,109,513,
-305,305,305,305,305,305,305,305,305,305,305,305,305,305,305,305,
-305,305,305,305,305,305,305,305,305,305,305,305,305,109,109,109,
+520,520,520,520,520,520,520,520,520,520,521,521,521,521,521,521,
+521,521,521,521,521,521,521,521,521,521,521,521,521,521,521,521,
+521,521,521,521,521,521,522,522,522,522,522,522,522,522,  4,523,
+524,524,524,524,524,524,524,524,524,524,524,524,524,524,524,524,
+524,524,524,524,524,524,524,525,525,525,525,525,525,525,525,525,
+525,525,526,526,114,114,114,114,114,114,114,114,114,114,114,527,
+314,314,314,314,314,314,314,314,314,314,314,314,314,314,314,314,
+314,314,314,314,314,314,314,314,314,314,314,314,314,114,114,114,
 
 /* block 109 */
-514,514,514,515,516,516,516,516,516,516,516,516,516,516,516,516,
-516,516,516,516,516,516,516,516,516,516,516,516,516,516,516,516,
-516,516,516,516,516,516,516,516,516,516,516,516,516,516,516,516,
-516,516,516,514,515,515,514,514,514,514,515,515,514,515,515,515,
-515,517,517,517,517,517,517,517,517,517,517,517,517,517,109,518,
-519,519,519,519,519,519,519,519,519,519,109,109,109,109,517,517,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+528,528,528,529,530,530,530,530,530,530,530,530,530,530,530,530,
+530,530,530,530,530,530,530,530,530,530,530,530,530,530,530,530,
+530,530,530,530,530,530,530,530,530,530,530,530,530,530,530,530,
+530,530,530,528,529,529,528,528,528,528,529,529,528,529,529,529,
+529,531,531,531,531,531,531,531,531,531,531,531,531,531,114,107,
+532,532,532,532,532,532,532,532,532,532,114,114,114,114,531,531,
+304,304,304,304,304,306,533,304,304,304,304,304,304,304,304,304,
+308,308,308,308,308,308,308,308,308,308,304,304,304,304,304,114,
 
 /* block 110 */
-520,520,520,520,520,520,520,520,520,520,520,520,520,520,520,520,
-520,520,520,520,520,520,520,520,520,520,520,520,520,520,520,520,
-520,520,520,520,520,520,520,520,520,521,521,521,521,521,521,522,
-522,521,521,522,522,521,521,109,109,109,109,109,109,109,109,109,
-520,520,520,521,520,520,520,520,520,520,520,520,521,522,109,109,
-523,523,523,523,523,523,523,523,523,523,109,109,524,524,524,524,
-295,295,295,295,295,295,295,295,295,295,295,295,295,295,295,295,
-525,295,295,295,295,295,295,301,301,301,295,296,109,109,109,109,
+534,534,534,534,534,534,534,534,534,534,534,534,534,534,534,534,
+534,534,534,534,534,534,534,534,534,534,534,534,534,534,534,534,
+534,534,534,534,534,534,534,534,534,535,535,535,535,535,535,536,
+536,535,535,536,536,535,535,114,114,114,114,114,114,114,114,114,
+534,534,534,535,534,534,534,534,534,534,534,534,535,536,114,114,
+537,537,537,537,537,537,537,537,537,537,114,114,538,538,538,538,
+304,304,304,304,304,304,304,304,304,304,304,304,304,304,304,304,
+533,304,304,304,304,304,304,310,310,310,304,305,306,305,304,304,
 
 /* block 111 */
-526,526,526,526,526,526,526,526,526,526,526,526,526,526,526,526,
-526,526,526,526,526,526,526,526,526,526,526,526,526,526,526,526,
-526,526,526,526,526,526,526,526,526,526,526,526,526,526,526,526,
-527,526,527,527,527,526,526,527,527,526,526,526,526,526,527,527,
-526,527,526,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,526,526,528,529,529,
-530,530,530,530,530,530,530,530,530,530,530,531,532,532,531,531,
-533,533,530,534,534,531,532,109,109,109,109,109,109,109,109,109,
+539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
+539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
+539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
+540,539,540,540,540,539,539,540,540,539,539,539,539,539,540,540,
+539,540,539,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,539,539,541,542,542,
+543,543,543,543,543,543,543,543,543,543,543,544,545,545,544,544,
+546,546,543,547,547,544,545,114,114,114,114,114,114,114,114,114,
 
 /* block 112 */
-109,308,308,308,308,308,308,109,109,308,308,308,308,308,308,109,
-109,308,308,308,308,308,308,109,109,109,109,109,109,109,109,109,
-308,308,308,308,308,308,308,109,308,308,308,308,308,308,308,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+114,317,317,317,317,317,317,114,114,317,317,317,317,317,317,114,
+114,317,317,317,317,317,317,114,114,114,114,114,114,114,114,114,
+317,317,317,317,317,317,317,114,317,317,317,317,317,317,317,114,
+ 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33,
+ 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33,
+ 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 14,106,106,106,106,
+114,114,114,114, 33,122,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 113 */
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-530,530,530,530,530,530,530,530,530,530,530,530,530,530,530,530,
-530,530,530,530,530,530,530,530,530,530,530,530,530,530,530,530,
-530,530,530,531,531,532,531,531,532,531,531,533,531,532,109,109,
-535,535,535,535,535,535,535,535,535,535,109,109,109,109,109,109,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,
+543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,
+543,543,543,544,544,545,544,544,545,544,544,546,544,545,114,114,
+548,548,548,548,548,548,548,548,548,548,114,114,114,114,114,114,
 
 /* block 114 */
-536,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,536,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,536,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,536,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-536,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
+549,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,549,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,549,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,549,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+549,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
 
 /* block 115 */
-537,537,537,537,537,537,537,537,537,537,537,537,536,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,536,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,536,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-536,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,536,537,537,537,
+550,550,550,550,550,550,550,550,550,550,550,550,549,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,549,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,549,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+549,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,549,550,550,550,
 
 /* block 116 */
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,536,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,536,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-536,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,536,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,549,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,549,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+549,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,549,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
 
 /* block 117 */
-537,537,537,537,537,537,537,537,536,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,536,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-536,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,536,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,536,537,537,537,537,537,537,537,
+550,550,550,550,550,550,550,550,549,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,549,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+549,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,549,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,549,550,550,550,550,550,550,550,
 
 /* block 118 */
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,536,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-536,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,536,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,536,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,549,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+549,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,549,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,549,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
 
 /* block 119 */
-537,537,537,537,536,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-536,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,536,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,536,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,536,537,537,537,537,537,537,537,537,537,537,537,
+550,550,550,550,549,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+549,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,549,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,549,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,549,550,550,550,550,550,550,550,550,550,550,550,
 
 /* block 120 */
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-536,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,536,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,536,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,536,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+549,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,549,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,549,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,549,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
 
 /* block 121 */
-537,537,537,537,537,537,537,537,536,537,537,537,537,537,537,537,
-537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,537,
-537,537,537,537,109,109,109,109,109,109,109,109,109,109,109,109,
-306,306,306,306,306,306,306,306,306,306,306,306,306,306,306,306,
-306,306,306,306,306,306,306,109,109,109,109,307,307,307,307,307,
-307,307,307,307,307,307,307,307,307,307,307,307,307,307,307,307,
-307,307,307,307,307,307,307,307,307,307,307,307,307,307,307,307,
-307,307,307,307,307,307,307,307,307,307,307,307,109,109,109,109,
+550,550,550,550,550,550,550,550,549,550,550,550,550,550,550,550,
+550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,550,
+550,550,550,550,114,114,114,114,114,114,114,114,114,114,114,114,
+315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,315,
+315,315,315,315,315,315,315,114,114,114,114,316,316,316,316,316,
+316,316,316,316,316,316,316,316,316,316,316,316,316,316,316,316,
+316,316,316,316,316,316,316,316,316,316,316,316,316,316,316,316,
+316,316,316,316,316,316,316,316,316,316,316,316,114,114,114,114,
 
 /* block 122 */
-538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,
-538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,
-538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,
-538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,
-538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,
-538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,
-538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,
-538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,
+551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,
+551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,
+551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,
+551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,
+551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,
+551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,
+551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,
+551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,
 
 /* block 123 */
-539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
-539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
-539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
-539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
-539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
-539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
-539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
-539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
+552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,
+552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,
+552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,
+552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,
+552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,
+552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,
+552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,
+552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,
 
 /* block 124 */
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,109,109,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,114,114,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
 
 /* block 125 */
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 126 */
- 33, 33, 33, 33, 33, 33, 33,109,109,109,109,109,109,109,109,109,
-109,109,109,178,178,178,178,178,109,109,109,109,109,184,181,184,
-184,184,184,184,184,184,184,184,184,540,184,184,184,184,184,184,
-184,184,184,184,184,184,184,109,184,184,184,184,184,109,184,109,
-184,184,109,184,184,109,184,184,184,184,184,184,184,184,184,184,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
+ 33, 33, 33, 33, 33, 33, 33,114,114,114,114,114,114,114,114,114,
+114,114,114,185,185,185,185,185,114,114,114,114,114,192,189,192,
+192,192,192,192,192,192,192,192,192,553,192,192,192,192,192,192,
+192,192,192,192,192,192,192,114,192,192,192,192,192,114,192,114,
+192,192,114,192,192,114,192,192,192,192,192,192,192,192,192,192,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
 
 /* block 127 */
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,541,541,541,541,541,541,541,541,541,541,541,541,541,541,
-541,541,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,554,554,554,554,554,554,554,554,554,554,554,554,554,554,
+554,554,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
 
 /* block 128 */
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
 
 /* block 129 */
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,  6,  7,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,  7,  6,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
 
 /* block 130 */
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-109,109,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-191,191,191,191,191,191,191,191,191,191,191,191,188, 19,109,109,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+114,114,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+199,199,199,199,199,199,199,199,199,199,199,199,196,197,114,114,
 
 /* block 131 */
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-  4,  4,  4,  4,  4,  4,  4,  6,  7,  4,109,109,109,109,109,109,
-104,104,104,104,104,104,104,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+  4,  4,  4,  4,  4,  4,  4,  6,  7,  4,114,114,114,114,114,114,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,114,114,
   4,  9,  9, 15, 15,  6,  7,  6,  7,  6,  7,  6,  7,  6,  7,  6,
   7,  6,  7,  6,  7,  4,  4,  6,  7,  4,  4,  4,  4, 15, 15, 15,
-  4,  4,  4,109,  4,  4,  4,  4,  9,  6,  7,  6,  7,  6,  7,  4,
-  4,  4,  8,  9,  8,  8,  8,109,  4,  5,  4,  4,109,109,109,109,
-191,191,191,191,191,109,191,191,191,191,191,191,191,191,191,191,
+  4,  4,  4,114,  4,  4,  4,  4,  9,  6,  7,  6,  7,  6,  7,  4,
+  4,  4,  8,  9,  8,  8,  8,114,  4,  5,  4,  4,114,114,114,114,
+199,199,199,199,199,114,199,199,199,199,199,199,199,199,199,199,
 
 /* block 132 */
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,109,109, 22,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,114,114, 22,
 
 /* block 133 */
-109,  4,  4,  4,  5,  4,  4,  4,  6,  7,  4,  8,  4,  9,  4,  4,
+114,  4,  4,  4,  5,  4,  4,  4,  6,  7,  4,  8,  4,  9,  4,  4,
  10, 10, 10, 10, 10, 10, 10, 10, 10, 10,  4,  4,  8,  8,  8,  4,
   4, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
  11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,  6,  4,  7, 14, 15,
  14, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
  16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,  6,  8,  7,  8,  6,
-  7,  4,  6,  7,  4,  4,469,469,469,469,469,469,469,469,469,469,
-102,469,469,469,469,469,469,469,469,469,469,469,469,469,469,469,
+  7,  4,  6,  7,  4,  4,478,478,478,478,478,478,478,478,478,478,
+107,478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,
 
 /* block 134 */
-469,469,469,469,469,469,469,469,469,469,469,469,469,469,469,469,
-469,469,469,469,469,469,469,469,469,469,469,469,469,469,542,542,
-472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,
-472,472,472,472,472,472,472,472,472,472,472,472,472,472,472,109,
-109,109,472,472,472,472,472,472,109,109,472,472,472,472,472,472,
-109,109,472,472,472,472,472,472,109,109,472,472,472,109,109,109,
-  5,  5,  8, 14, 19,  5,  5,109, 19,  8,  8,  8,  8, 19, 19,109,
-426,426,426,426,426,426,426,426,426, 22, 22, 22, 19, 19,109,109,
+478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,478,
+478,478,478,478,478,478,478,478,478,478,478,478,478,478,555,555,
+481,481,481,481,481,481,481,481,481,481,481,481,481,481,481,481,
+481,481,481,481,481,481,481,481,481,481,481,481,481,481,481,114,
+114,114,481,481,481,481,481,481,114,114,481,481,481,481,481,481,
+114,114,481,481,481,481,481,481,114,114,481,481,481,114,114,114,
+  5,  5,  8, 14, 19,  5,  5,114, 19,  8,  8,  8,  8, 19, 19,114,
+436,436,436,436,436,436,436,436,436, 22, 22, 22, 19, 19,114,114,
 
 /* block 135 */
-543,543,543,543,543,543,543,543,543,543,543,543,109,543,543,543,
-543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,
-543,543,543,543,543,543,543,109,543,543,543,543,543,543,543,543,
-543,543,543,543,543,543,543,543,543,543,543,109,543,543,109,543,
-543,543,543,543,543,543,543,543,543,543,543,543,543,543,109,109,
-543,543,543,543,543,543,543,543,543,543,543,543,543,543,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+556,556,556,556,556,556,556,556,556,556,556,556,114,556,556,556,
+556,556,556,556,556,556,556,556,556,556,556,556,556,556,556,556,
+556,556,556,556,556,556,556,114,556,556,556,556,556,556,556,556,
+556,556,556,556,556,556,556,556,556,556,556,114,556,556,114,556,
+556,556,556,556,556,556,556,556,556,556,556,556,556,556,114,114,
+556,556,556,556,556,556,556,556,556,556,556,556,556,556,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 136 */
-543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,
-543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,
-543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,
-543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,
-543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,
-543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,
-543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,
-543,543,543,543,543,543,543,543,543,543,543,109,109,109,109,109,
+556,556,556,556,556,556,556,556,556,556,556,556,556,556,556,556,
+556,556,556,556,556,556,556,556,556,556,556,556,556,556,556,556,
+556,556,556,556,556,556,556,556,556,556,556,556,556,556,556,556,
+556,556,556,556,556,556,556,556,556,556,556,556,556,556,556,556,
+556,556,556,556,556,556,556,556,556,556,556,556,556,556,556,556,
+556,556,556,556,556,556,556,556,556,556,556,556,556,556,556,556,
+556,556,556,556,556,556,556,556,556,556,556,556,556,556,556,556,
+556,556,556,556,556,556,556,556,556,556,556,114,114,114,114,114,
 
 /* block 137 */
-  4,  4,  4,109,109,109,109, 23, 23, 23, 23, 23, 23, 23, 23, 23,
+  4,  4,  4,114,114,114,114, 23, 23, 23, 23, 23, 23, 23, 23, 23,
  23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
  23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
- 23, 23, 23, 23,109,109,109, 19, 19, 19, 19, 19, 19, 19, 19, 19,
-544,544,544,544,544,544,544,544,544,544,544,544,544,544,544,544,
-544,544,544,544,544,544,544,544,544,544,544,544,544,544,544,544,
-544,544,544,544,544,544,544,544,544,544,544,544,544,544,544,544,
-544,544,544,544,544,545,545,545,545,546,546,546,546,546,546,546,
+ 23, 23, 23, 23,114,114,114, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+557,557,557,557,557,557,557,557,557,557,557,557,557,557,557,557,
+557,557,557,557,557,557,557,557,557,557,557,557,557,557,557,557,
+557,557,557,557,557,557,557,557,557,557,557,557,557,557,557,557,
+557,557,557,557,557,558,558,558,558,559,559,559,559,559,559,559,
 
 /* block 138 */
-546,546,546,546,546,546,546,546,546,546,545,109,109,109,109,109,
- 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+559,559,559,559,559,559,559,559,559,559,558,558,559,114,114,114,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,114,
+559,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,104,109,109,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,114,114,
 
 /* block 139 */
-547,547,547,547,547,547,547,547,547,547,547,547,547,547,547,547,
-547,547,547,547,547,547,547,547,547,547,547,547,547,109,109,109,
-548,548,548,548,548,548,548,548,548,548,548,548,548,548,548,548,
-548,548,548,548,548,548,548,548,548,548,548,548,548,548,548,548,
-548,548,548,548,548,548,548,548,548,548,548,548,548,548,548,548,
-548,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 140 */
-549,549,549,549,549,549,549,549,549,549,549,549,549,549,549,549,
-549,549,549,549,549,549,549,549,549,549,549,549,549,549,549,109,
-550,550,550,550,109,109,109,109,109,109,109,109,109,109,109,109,
-551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,551,
-551,552,551,551,551,551,551,551,551,551,552,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+560,560,560,560,560,560,560,560,560,560,560,560,560,560,560,560,
+560,560,560,560,560,560,560,560,560,560,560,560,560,114,114,114,
+561,561,561,561,561,561,561,561,561,561,561,561,561,561,561,561,
+561,561,561,561,561,561,561,561,561,561,561,561,561,561,561,561,
+561,561,561,561,561,561,561,561,561,561,561,561,561,561,561,561,
+561,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+109, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
+ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,114,114,114,114,
 
 /* block 141 */
-553,553,553,553,553,553,553,553,553,553,553,553,553,553,553,553,
-553,553,553,553,553,553,553,553,553,553,553,553,553,553,109,554,
-555,555,555,555,555,555,555,555,555,555,555,555,555,555,555,555,
-555,555,555,555,555,555,555,555,555,555,555,555,555,555,555,555,
-555,555,555,555,109,109,109,109,555,555,555,555,555,555,555,555,
-556,557,557,557,557,557,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+562,562,562,562,562,562,562,562,562,562,562,562,562,562,562,562,
+562,562,562,562,562,562,562,562,562,562,562,562,562,562,562,562,
+563,563,563,563,114,114,114,114,114,114,114,114,114,114,114,114,
+564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,
+564,565,564,564,564,564,564,564,564,564,565,114,114,114,114,114,
+566,566,566,566,566,566,566,566,566,566,566,566,566,566,566,566,
+566,566,566,566,566,566,566,566,566,566,566,566,566,566,566,566,
+566,566,566,566,566,566,567,567,567,567,567,114,114,114,114,114,
 
 /* block 142 */
-558,558,558,558,558,558,558,558,558,558,558,558,558,558,558,558,
-558,558,558,558,558,558,558,558,558,558,558,558,558,558,558,558,
-558,558,558,558,558,558,558,558,559,559,559,559,559,559,559,559,
-559,559,559,559,559,559,559,559,559,559,559,559,559,559,559,559,
-559,559,559,559,559,559,559,559,559,559,559,559,559,559,559,559,
-560,560,560,560,560,560,560,560,560,560,560,560,560,560,560,560,
-560,560,560,560,560,560,560,560,560,560,560,560,560,560,560,560,
-560,560,560,560,560,560,560,560,560,560,560,560,560,560,560,560,
+568,568,568,568,568,568,568,568,568,568,568,568,568,568,568,568,
+568,568,568,568,568,568,568,568,568,568,568,568,568,568,114,569,
+570,570,570,570,570,570,570,570,570,570,570,570,570,570,570,570,
+570,570,570,570,570,570,570,570,570,570,570,570,570,570,570,570,
+570,570,570,570,114,114,114,114,570,570,570,570,570,570,570,570,
+571,572,572,572,572,572,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 143 */
-561,561,561,561,561,561,561,561,561,561,561,561,561,561,561,561,
-561,561,561,561,561,561,561,561,561,561,561,561,561,561,109,109,
-562,562,562,562,562,562,562,562,562,562,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,
+573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,
+573,573,573,573,573,573,573,573,574,574,574,574,574,574,574,574,
+574,574,574,574,574,574,574,574,574,574,574,574,574,574,574,574,
+574,574,574,574,574,574,574,574,574,574,574,574,574,574,574,574,
+575,575,575,575,575,575,575,575,575,575,575,575,575,575,575,575,
+575,575,575,575,575,575,575,575,575,575,575,575,575,575,575,575,
+575,575,575,575,575,575,575,575,575,575,575,575,575,575,575,575,
 
 /* block 144 */
-563,563,563,563,563,563,109,109,563,109,563,563,563,563,563,563,
-563,563,563,563,563,563,563,563,563,563,563,563,563,563,563,563,
-563,563,563,563,563,563,563,563,563,563,563,563,563,563,563,563,
-563,563,563,563,563,563,109,563,563,109,109,109,563,109,109,563,
-564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,
-564,564,564,564,564,564,109,565,566,566,566,566,566,566,566,566,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+576,576,576,576,576,576,576,576,576,576,576,576,576,576,576,576,
+576,576,576,576,576,576,576,576,576,576,576,576,576,576,114,114,
+577,577,577,577,577,577,577,577,577,577,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 145 */
-567,567,567,567,567,567,567,567,567,567,567,567,567,567,567,567,
-567,567,567,567,567,567,568,568,568,568,568,568,109,109,109,569,
-570,570,570,570,570,570,570,570,570,570,570,570,570,570,570,570,
-570,570,570,570,570,570,570,570,570,570,109,109,109,109,109,571,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,
+578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,
+578,578,578,578,578,578,578,578,114,114,114,114,114,114,114,114,
+579,579,579,579,579,579,579,579,579,579,579,579,579,579,579,579,
+579,579,579,579,579,579,579,579,579,579,579,579,579,579,579,579,
+579,579,579,579,579,579,579,579,579,579,579,579,579,579,579,579,
+579,579,579,579,114,114,114,114,114,114,114,114,114,114,114,580,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 146 */
-572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,
-572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,
-573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,
-573,573,573,573,573,573,573,573,109,109,109,109,109,109,573,573,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,
+581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,
+581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,
+581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,
+581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,
+581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,
+581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,
+581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,
 
 /* block 147 */
-574,575,575,575,109,575,575,109,109,109,109,109,575,575,575,575,
-574,574,574,574,109,574,574,574,109,574,574,574,574,574,574,574,
-574,574,574,574,574,574,574,574,574,574,574,574,574,574,574,574,
-574,574,574,574,109,109,109,109,575,575,575,109,109,109,109,575,
-576,576,576,576,576,576,576,576,109,109,109,109,109,109,109,109,
-577,577,577,577,577,577,577,577,577,109,109,109,109,109,109,109,
-578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,
-578,578,578,578,578,578,578,578,578,578,578,578,578,579,579,580,
-
-/* block 148 */
 581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,
 581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,
 581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,
-581,581,581,581,581,581,109,109,109,582,582,582,582,582,582,582,
+581,581,581,581,581,581,581,114,114,114,114,114,114,114,114,114,
+581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,
+581,581,581,581,581,581,114,114,114,114,114,114,114,114,114,114,
+581,581,581,581,581,581,581,581,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+
+/* block 148 */
+582,582,582,582,582,582,114,114,582,114,582,582,582,582,582,582,
+582,582,582,582,582,582,582,582,582,582,582,582,582,582,582,582,
+582,582,582,582,582,582,582,582,582,582,582,582,582,582,582,582,
+582,582,582,582,582,582,114,582,582,114,114,114,582,114,114,582,
 583,583,583,583,583,583,583,583,583,583,583,583,583,583,583,583,
-583,583,583,583,583,583,109,109,584,584,584,584,584,584,584,584,
-585,585,585,585,585,585,585,585,585,585,585,585,585,585,585,585,
-585,585,585,109,109,109,109,109,586,586,586,586,586,586,586,586,
+583,583,583,583,583,583,114,584,585,585,585,585,585,585,585,585,
+586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,
+586,586,586,586,586,586,586,587,587,588,588,588,588,588,588,588,
 
 /* block 149 */
-587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,
-587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,
-587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,
-587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,
-587,587,587,587,587,587,587,587,587,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+589,589,589,589,589,589,589,589,589,589,589,589,589,589,589,589,
+589,589,589,589,589,589,589,589,589,589,589,589,589,589,589,114,
+114,114,114,114,114,114,114,590,590,590,590,590,590,590,590,590,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 150 */
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-588,588,588,588,588,588,588,588,588,588,588,588,588,588,588,588,
-588,588,588,588,588,588,588,588,588,588,588,588,588,588,588,109,
+591,591,591,591,591,591,591,591,591,591,591,591,591,591,591,591,
+591,591,591,591,591,591,592,592,592,592,592,592,114,114,114,593,
+594,594,594,594,594,594,594,594,594,594,594,594,594,594,594,594,
+594,594,594,594,594,594,594,594,594,594,114,114,114,114,114,595,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 151 */
-589,590,589,591,591,591,591,591,591,591,591,591,591,591,591,591,
-591,591,591,591,591,591,591,591,591,591,591,591,591,591,591,591,
-591,591,591,591,591,591,591,591,591,591,591,591,591,591,591,591,
-591,591,591,591,591,591,591,591,590,590,590,590,590,590,590,590,
-590,590,590,590,590,590,590,592,592,592,592,592,592,592,109,109,
-109,109,593,593,593,593,593,593,593,593,593,593,593,593,593,593,
-593,593,593,593,593,593,594,594,594,594,594,594,594,594,594,594,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+596,596,596,596,596,596,596,596,596,596,596,596,596,596,596,596,
+596,596,596,596,596,596,596,596,596,596,596,596,596,596,596,596,
+597,597,597,597,597,597,597,597,597,597,597,597,597,597,597,597,
+597,597,597,597,597,597,597,597,114,114,114,114,114,114,597,597,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 152 */
-595,595,596,597,597,597,597,597,597,597,597,597,597,597,597,597,
-597,597,597,597,597,597,597,597,597,597,597,597,597,597,597,597,
-597,597,597,597,597,597,597,597,597,597,597,597,597,597,597,597,
-596,596,596,595,595,595,595,596,596,595,595,598,598,599,598,598,
-598,598,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-600,600,600,600,600,600,600,600,600,600,600,600,600,600,600,600,
-600,600,600,600,600,600,600,600,600,109,109,109,109,109,109,109,
-601,601,601,601,601,601,601,601,601,601,109,109,109,109,109,109,
+598,599,599,599,114,599,599,114,114,114,114,114,599,599,599,599,
+598,598,598,598,114,598,598,598,114,598,598,598,598,598,598,598,
+598,598,598,598,598,598,598,598,598,598,598,598,598,598,598,598,
+598,598,598,598,114,114,114,114,599,599,599,114,114,114,114,599,
+600,600,600,600,600,600,600,600,114,114,114,114,114,114,114,114,
+601,601,601,601,601,601,601,601,601,114,114,114,114,114,114,114,
+602,602,602,602,602,602,602,602,602,602,602,602,602,602,602,602,
+602,602,602,602,602,602,602,602,602,602,602,602,602,603,603,604,
 
 /* block 153 */
-602,602,602,603,603,603,603,603,603,603,603,603,603,603,603,603,
-603,603,603,603,603,603,603,603,603,603,603,603,603,603,603,603,
-603,603,603,603,603,603,603,602,602,602,602,602,604,602,602,602,
-602,602,602,602,602,109,605,605,605,605,605,605,605,605,605,605,
-606,606,606,606,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+605,605,605,605,605,605,605,605,605,605,605,605,605,605,605,605,
+605,605,605,605,605,605,605,605,605,605,605,605,605,606,606,606,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+607,607,607,607,607,607,607,607,608,607,607,607,607,607,607,607,
+607,607,607,607,607,607,607,607,607,607,607,607,607,607,607,607,
+607,607,607,607,607,609,609,114,114,114,114,610,610,610,610,610,
+611,611,611,611,611,611,611,114,114,114,114,114,114,114,114,114,
 
 /* block 154 */
-607,607,608,609,609,609,609,609,609,609,609,609,609,609,609,609,
-609,609,609,609,609,609,609,609,609,609,609,609,609,609,609,609,
-609,609,609,609,609,609,609,609,609,609,609,609,609,609,609,609,
-609,609,609,608,608,608,607,607,607,607,607,607,607,607,607,608,
-608,609,609,609,609,610,610,610,610,109,109,109,109,109,109,109,
-611,611,611,611,611,611,611,611,611,611,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-
-/* block 155 */
 612,612,612,612,612,612,612,612,612,612,612,612,612,612,612,612,
 612,612,612,612,612,612,612,612,612,612,612,612,612,612,612,612,
-612,612,612,612,612,612,612,612,612,612,612,613,614,613,614,614,
-613,613,613,613,613,613,614,613,109,109,109,109,109,109,109,109,
-615,615,615,615,615,615,615,615,615,615,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+612,612,612,612,612,612,612,612,612,612,612,612,612,612,612,612,
+612,612,612,612,612,612,114,114,114,613,613,613,613,613,613,613,
+614,614,614,614,614,614,614,614,614,614,614,614,614,614,614,614,
+614,614,614,614,614,614,114,114,615,615,615,615,615,615,615,615,
+616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,
+616,616,616,114,114,114,114,114,617,617,617,617,617,617,617,617,
+
+/* block 155 */
+618,618,618,618,618,618,618,618,618,618,618,618,618,618,618,618,
+618,618,114,114,114,114,114,114,114,619,619,619,619,114,114,114,
+114,114,114,114,114,114,114,114,114,620,620,620,620,620,620,620,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 156 */
-616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,
-616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,
-616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,
-616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,
-616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,
-616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,
-616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,
-616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,
+621,621,621,621,621,621,621,621,621,621,621,621,621,621,621,621,
+621,621,621,621,621,621,621,621,621,621,621,621,621,621,621,621,
+621,621,621,621,621,621,621,621,621,621,621,621,621,621,621,621,
+621,621,621,621,621,621,621,621,621,621,621,621,621,621,621,621,
+621,621,621,621,621,621,621,621,621,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 157 */
-616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,
-616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,
-616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,
-616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,
-616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,
-616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,
-616,616,616,616,616,616,616,616,616,616,616,616,616,616,616,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+622,622,622,622,622,622,622,622,622,622,622,622,622,622,622,622,
+622,622,622,622,622,622,622,622,622,622,622,622,622,622,622,114,
 
 /* block 158 */
-617,617,617,617,617,617,617,617,617,617,617,617,617,617,617,617,
-617,617,617,617,617,617,617,617,617,617,617,617,617,617,617,617,
-617,617,617,617,617,617,617,617,617,617,617,617,617,617,617,617,
-617,617,617,617,617,617,617,617,617,617,617,617,617,617,617,617,
-617,617,617,617,617,617,617,617,617,617,617,617,617,617,617,617,
-617,617,617,617,617,617,617,617,617,617,617,617,617,617,617,617,
-617,617,617,109,109,109,109,109,109,109,109,109,109,109,109,109,
-618,618,618,618,109,109,109,109,109,109,109,109,109,109,109,109,
+623,624,623,625,625,625,625,625,625,625,625,625,625,625,625,625,
+625,625,625,625,625,625,625,625,625,625,625,625,625,625,625,625,
+625,625,625,625,625,625,625,625,625,625,625,625,625,625,625,625,
+625,625,625,625,625,625,625,625,624,624,624,624,624,624,624,624,
+624,624,624,624,624,624,624,626,626,626,626,626,626,626,114,114,
+114,114,627,627,627,627,627,627,627,627,627,627,627,627,627,627,
+627,627,627,627,627,627,628,628,628,628,628,628,628,628,628,628,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,624,
 
 /* block 159 */
-619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,
-619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,
-619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,
-619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,
-619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,
-619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,
-619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,
-619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,
+629,629,630,631,631,631,631,631,631,631,631,631,631,631,631,631,
+631,631,631,631,631,631,631,631,631,631,631,631,631,631,631,631,
+631,631,631,631,631,631,631,631,631,631,631,631,631,631,631,631,
+630,630,630,629,629,629,629,630,630,629,629,632,632,633,632,632,
+632,632,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+634,634,634,634,634,634,634,634,634,634,634,634,634,634,634,634,
+634,634,634,634,634,634,634,634,634,114,114,114,114,114,114,114,
+635,635,635,635,635,635,635,635,635,635,114,114,114,114,114,114,
 
 /* block 160 */
-619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,
-619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,
-619,619,619,619,619,619,619,619,619,619,619,619,619,619,619,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+636,636,636,637,637,637,637,637,637,637,637,637,637,637,637,637,
+637,637,637,637,637,637,637,637,637,637,637,637,637,637,637,637,
+637,637,637,637,637,637,637,636,636,636,636,636,638,636,636,636,
+636,636,636,636,636,114,639,639,639,639,639,639,639,639,639,639,
+640,640,640,640,114,114,114,114,114,114,114,114,114,114,114,114,
+641,641,641,641,641,641,641,641,641,641,641,641,641,641,641,641,
+641,641,641,641,641,641,641,641,641,641,641,641,641,641,641,641,
+641,641,641,642,643,643,641,114,114,114,114,114,114,114,114,114,
 
 /* block 161 */
-488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
-488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
-488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
-488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
-488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
-488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
-488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
-488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
+644,644,645,646,646,646,646,646,646,646,646,646,646,646,646,646,
+646,646,646,646,646,646,646,646,646,646,646,646,646,646,646,646,
+646,646,646,646,646,646,646,646,646,646,646,646,646,646,646,646,
+646,646,646,645,645,645,644,644,644,644,644,644,644,644,644,645,
+645,646,646,646,646,647,647,647,647,114,114,114,114,647,114,114,
+648,648,648,648,648,648,648,648,648,648,646,114,114,114,114,114,
+114,649,649,649,649,649,649,649,649,649,649,649,649,649,649,649,
+649,649,649,649,649,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 162 */
-488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
-488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
-488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,488,
-488,488,488,488,488,488,488,488,488,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+650,650,650,650,650,650,650,650,650,650,650,650,650,650,650,650,
+650,650,114,650,650,650,650,650,650,650,650,650,650,650,650,650,
+650,650,650,650,650,650,650,650,650,650,650,650,651,651,651,652,
+652,652,651,651,652,651,652,652,653,653,653,653,653,653,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 163 */
-620,620,620,620,620,620,620,620,620,620,620,620,620,620,620,620,
-620,620,620,620,620,620,620,620,620,620,620,620,620,620,620,620,
-620,620,620,620,620,620,620,620,620,620,620,620,620,620,620,620,
-620,620,620,620,620,620,620,620,620,620,620,620,620,620,620,620,
-620,620,620,620,620,109,109,109,109,109,109,109,109,109,109,109,
-620,621,621,621,621,621,621,621,621,621,621,621,621,621,621,621,
-621,621,621,621,621,621,621,621,621,621,621,621,621,621,621,621,
-621,621,621,621,621,621,621,621,621,621,621,621,621,621,621,109,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+654,654,654,654,654,654,654,654,654,654,654,654,654,654,654,654,
+654,654,654,654,654,654,654,654,654,654,654,654,654,654,654,654,
+654,654,654,654,654,654,654,654,654,654,654,654,654,654,654,655,
+656,656,656,655,655,655,655,655,655,655,655,114,114,114,114,114,
+657,657,657,657,657,657,657,657,657,657,114,114,114,114,114,114,
 
 /* block 164 */
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,622,
-622,622,622,623,623,623,623,623,623,623,623,623,623,623,623,623,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+114,658,659,659,114,660,660,660,660,660,660,660,660,114,114,660,
+660,114,114,660,660,660,660,660,660,660,660,660,660,660,660,660,
+660,660,660,660,660,660,660,660,660,114,660,660,660,660,660,660,
+660,114,660,660,114,660,660,660,660,660,114,114,658,660,661,659,
+658,659,659,659,659,114,114,659,659,114,114,659,659,659,114,114,
+114,114,114,114,114,114,114,661,114,114,114,114,114,660,660,660,
+660,660,659,659,114,114,658,658,658,658,658,658,658,114,114,114,
+658,658,658,658,658,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 165 */
-469,467,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+662,662,662,662,662,662,662,662,662,662,662,662,662,662,662,662,
+662,662,662,662,662,662,662,662,662,662,662,662,662,662,662,662,
+662,662,662,662,662,662,662,662,662,662,662,662,662,662,662,662,
+663,664,664,665,665,665,665,665,665,664,665,664,664,663,664,665,
+665,664,665,665,662,662,666,662,114,114,114,114,114,114,114,114,
+667,667,667,667,667,667,667,667,667,667,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
 /* block 166 */
+668,668,668,668,668,668,668,668,668,668,668,668,668,668,668,668,
+668,668,668,668,668,668,668,668,668,668,668,668,668,668,668,668,
+668,668,668,668,668,668,668,668,668,668,668,668,668,668,668,669,
+670,670,671,671,671,671,114,114,670,670,670,670,671,671,670,671,
+671,672,672,672,672,672,672,672,672,672,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+
+/* block 167 */
+673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,
+673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,
+673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,
+674,674,674,675,675,675,675,675,675,675,675,674,674,675,674,675,
+675,676,676,676,673,114,114,114,114,114,114,114,114,114,114,114,
+677,677,677,677,677,677,677,677,677,677,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+
+/* block 168 */
+678,678,678,678,678,678,678,678,678,678,678,678,678,678,678,678,
+678,678,678,678,678,678,678,678,678,678,678,678,678,678,678,678,
+678,678,678,678,678,678,678,678,678,678,678,679,680,679,680,680,
+679,679,679,679,679,679,680,679,114,114,114,114,114,114,114,114,
+681,681,681,681,681,681,681,681,681,681,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+
+/* block 169 */
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+682,682,682,682,682,682,682,682,682,682,682,682,682,682,682,682,
+682,682,682,682,682,682,682,682,682,682,682,682,682,682,682,682,
+683,683,683,683,683,683,683,683,683,683,683,683,683,683,683,683,
+683,683,683,683,683,683,683,683,683,683,683,683,683,683,683,683,
+684,684,684,684,684,684,684,684,684,684,685,685,685,685,685,685,
+685,685,685,114,114,114,114,114,114,114,114,114,114,114,114,686,
+
+/* block 170 */
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+687,687,687,687,687,687,687,687,687,687,687,687,687,687,687,687,
+687,687,687,687,687,687,687,687,687,687,687,687,687,687,687,687,
+687,687,687,687,687,687,687,687,687,687,687,687,687,687,687,687,
+687,687,687,687,687,687,687,687,687,114,114,114,114,114,114,114,
+
+/* block 171 */
+688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,
+688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,
+688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,
+688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,
+688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,
+688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,
+688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,
+688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,
+
+/* block 172 */
+688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,
+688,688,688,688,688,688,688,688,688,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+
+/* block 173 */
+689,689,689,689,689,689,689,689,689,689,689,689,689,689,689,689,
+689,689,689,689,689,689,689,689,689,689,689,689,689,689,689,689,
+689,689,689,689,689,689,689,689,689,689,689,689,689,689,689,689,
+689,689,689,689,689,689,689,689,689,689,689,689,689,689,689,689,
+689,689,689,689,689,689,689,689,689,689,689,689,689,689,689,689,
+689,689,689,689,689,689,689,689,689,689,689,689,689,689,689,689,
+689,689,689,689,689,689,689,689,689,689,689,689,689,689,689,114,
+690,690,690,690,690,114,114,114,114,114,114,114,114,114,114,114,
+
+/* block 174 */
+691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,
+691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,
+691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,
+691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,
+691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,
+691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,
+691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,
+691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,
+
+/* block 175 */
+691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,
+691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,
+691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+
+/* block 176 */
+497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,
+497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,
+497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,
+497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,
+497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,
+497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,
+497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,
+497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,
+
+/* block 177 */
+497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,
+497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,
+497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,497,
+497,497,497,497,497,497,497,497,497,114,114,114,114,114,114,114,
+692,692,692,692,692,692,692,692,692,692,692,692,692,692,692,692,
+692,692,692,692,692,692,692,692,692,692,692,692,692,692,692,114,
+693,693,693,693,693,693,693,693,693,693,114,114,114,114,694,694,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+
+/* block 178 */
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+695,695,695,695,695,695,695,695,695,695,695,695,695,695,695,695,
+695,695,695,695,695,695,695,695,695,695,695,695,695,695,114,114,
+696,696,696,696,696,697,114,114,114,114,114,114,114,114,114,114,
+
+/* block 179 */
+698,698,698,698,698,698,698,698,698,698,698,698,698,698,698,698,
+698,698,698,698,698,698,698,698,698,698,698,698,698,698,698,698,
+698,698,698,698,698,698,698,698,698,698,698,698,698,698,698,698,
+699,699,699,699,699,699,699,700,700,700,700,700,701,701,701,701,
+702,702,702,702,700,701,114,114,114,114,114,114,114,114,114,114,
+703,703,703,703,703,703,703,703,703,703,114,704,704,704,704,704,
+704,704,114,698,698,698,698,698,698,698,698,698,698,698,698,698,
+698,698,698,698,698,698,698,698,114,114,114,114,114,698,698,698,
+
+/* block 180 */
+698,698,698,698,698,698,698,698,698,698,698,698,698,698,698,698,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+
+/* block 181 */
+705,705,705,705,705,705,705,705,705,705,705,705,705,705,705,705,
+705,705,705,705,705,705,705,705,705,705,705,705,705,705,705,705,
+705,705,705,705,705,705,705,705,705,705,705,705,705,705,705,705,
+705,705,705,705,705,705,705,705,705,705,705,705,705,705,705,705,
+705,705,705,705,705,114,114,114,114,114,114,114,114,114,114,114,
+705,706,706,706,706,706,706,706,706,706,706,706,706,706,706,706,
+706,706,706,706,706,706,706,706,706,706,706,706,706,706,706,706,
+706,706,706,706,706,706,706,706,706,706,706,706,706,706,706,114,
+
+/* block 182 */
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,707,
+707,707,707,708,708,708,708,708,708,708,708,708,708,708,708,708,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+
+/* block 183 */
+478,476,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+
+/* block 184 */
+709,709,709,709,709,709,709,709,709,709,709,709,709,709,709,709,
+709,709,709,709,709,709,709,709,709,709,709,709,709,709,709,709,
+709,709,709,709,709,709,709,709,709,709,709,709,709,709,709,709,
+709,709,709,709,709,709,709,709,709,709,709,709,709,709,709,709,
+709,709,709,709,709,709,709,709,709,709,709,709,709,709,709,709,
+709,709,709,709,709,709,709,709,709,709,709,709,709,709,709,709,
+709,709,709,709,709,709,709,709,709,709,709,114,114,114,114,114,
+709,709,709,709,709,709,709,709,709,709,709,709,709,114,114,114,
+
+/* block 185 */
+709,709,709,709,709,709,709,709,709,114,114,114,114,114,114,114,
+709,709,709,709,709,709,709,709,709,709,114,114,710,711,711,712,
+ 22, 22, 22, 22,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+
+/* block 186 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
@@ -2926,321 +3218,361 @@ const pcre_uint16 PRIV(ucd_stage2)[] = { /* 51968 bytes, block = 128 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19,109,109,109,109,109,109,109,109,109,109,
+ 19, 19, 19, 19, 19, 19,114,114,114,114,114,114,114,114,114,114,
 
-/* block 167 */
+/* block 187 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19, 19,109,109, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19,114,114, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19,624,395,104,104,104, 19, 19, 19,395,624,624,
-624,624,624, 22, 22, 22, 22, 22, 22, 22, 22,104,104,104,104,104,
+ 19, 19, 19, 19, 19,713,405,109,109,109, 19, 19, 19,405,713,713,
+713,713,713, 22, 22, 22, 22, 22, 22, 22, 22,109,109,109,109,109,
 
-/* block 168 */
-104,104,104, 19, 19,104,104,104,104,104,104,104, 19, 19, 19, 19,
+/* block 188 */
+109,109,109, 19, 19,109,109,109,109,109,109,109, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,104,104,104,104, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,109,109,109, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
-/* block 169 */
-546,546,546,546,546,546,546,546,546,546,546,546,546,546,546,546,
-546,546,546,546,546,546,546,546,546,546,546,546,546,546,546,546,
-546,546,546,546,546,546,546,546,546,546,546,546,546,546,546,546,
-546,546,546,546,546,546,546,546,546,546,546,546,546,546,546,546,
-546,546,625,625,625,546,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+/* block 189 */
+559,559,559,559,559,559,559,559,559,559,559,559,559,559,559,559,
+559,559,559,559,559,559,559,559,559,559,559,559,559,559,559,559,
+559,559,559,559,559,559,559,559,559,559,559,559,559,559,559,559,
+559,559,559,559,559,559,559,559,559,559,559,559,559,559,559,559,
+559,559,714,714,714,559,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
-/* block 170 */
+/* block 190 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19, 19,109,109,109,109,109,109,109,109,109,
+ 19, 19, 19, 19, 19, 19, 19,114,114,114,114,114,114,114,114,114,
  23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
- 23, 23,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+ 23, 23,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
-/* block 171 */
-428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,
-428,428,428,428,428,428,428,428,428,428,429,429,429,429,429,429,
-429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,
-429,429,429,429,428,428,428,428,428,428,428,428,428,428,428,428,
-428,428,428,428,428,428,428,428,428,428,428,428,428,428,429,429,
-429,429,429,429,429,109,429,429,429,429,429,429,429,429,429,429,
-429,429,429,429,429,429,429,429,428,428,428,428,428,428,428,428,
-428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,
+/* block 191 */
+437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,
+437,437,437,437,437,437,437,437,437,437,438,438,438,438,438,438,
+438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,
+438,438,438,438,437,437,437,437,437,437,437,437,437,437,437,437,
+437,437,437,437,437,437,437,437,437,437,437,437,437,437,438,438,
+438,438,438,438,438,114,438,438,438,438,438,438,438,438,438,438,
+438,438,438,438,438,438,438,438,437,437,437,437,437,437,437,437,
+437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,
 
-/* block 172 */
-428,428,429,429,429,429,429,429,429,429,429,429,429,429,429,429,
-429,429,429,429,429,429,429,429,429,429,429,429,428,109,428,428,
-109,109,428,109,109,428,428,109,109,428,428,428,428,109,428,428,
-428,428,428,428,428,428,429,429,429,429,109,429,109,429,429,429,
-429,429,429,429,109,429,429,429,429,429,429,429,429,429,429,429,
-428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,
-428,428,428,428,428,428,428,428,428,428,429,429,429,429,429,429,
-429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,
+/* block 192 */
+437,437,438,438,438,438,438,438,438,438,438,438,438,438,438,438,
+438,438,438,438,438,438,438,438,438,438,438,438,437,114,437,437,
+114,114,437,114,114,437,437,114,114,437,437,437,437,114,437,437,
+437,437,437,437,437,437,438,438,438,438,114,438,114,438,438,438,
+438,438,438,438,114,438,438,438,438,438,438,438,438,438,438,438,
+437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,
+437,437,437,437,437,437,437,437,437,437,438,438,438,438,438,438,
+438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,
 
-/* block 173 */
-429,429,429,429,428,428,109,428,428,428,428,109,109,428,428,428,
-428,428,428,428,428,109,428,428,428,428,428,428,428,109,429,429,
-429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,
-429,429,429,429,429,429,429,429,428,428,109,428,428,428,428,109,
-428,428,428,428,428,109,428,109,109,109,428,428,428,428,428,428,
-428,109,429,429,429,429,429,429,429,429,429,429,429,429,429,429,
-429,429,429,429,429,429,429,429,429,429,429,429,428,428,428,428,
-428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,
+/* block 193 */
+438,438,438,438,437,437,114,437,437,437,437,114,114,437,437,437,
+437,437,437,437,437,114,437,437,437,437,437,437,437,114,438,438,
+438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,
+438,438,438,438,438,438,438,438,437,437,114,437,437,437,437,114,
+437,437,437,437,437,114,437,114,114,114,437,437,437,437,437,437,
+437,114,438,438,438,438,438,438,438,438,438,438,438,438,438,438,
+438,438,438,438,438,438,438,438,438,438,438,438,437,437,437,437,
+437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,
 
-/* block 174 */
-428,428,428,428,428,428,429,429,429,429,429,429,429,429,429,429,
-429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,
-428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,
-428,428,428,428,428,428,428,428,428,428,429,429,429,429,429,429,
-429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,
-429,429,429,429,428,428,428,428,428,428,428,428,428,428,428,428,
-428,428,428,428,428,428,428,428,428,428,428,428,428,428,429,429,
-429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,
+/* block 194 */
+437,437,437,437,437,437,438,438,438,438,438,438,438,438,438,438,
+438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,
+437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,
+437,437,437,437,437,437,437,437,437,437,438,438,438,438,438,438,
+438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,
+438,438,438,438,437,437,437,437,437,437,437,437,437,437,437,437,
+437,437,437,437,437,437,437,437,437,437,437,437,437,437,438,438,
+438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,
 
-/* block 175 */
-429,429,429,429,429,429,429,429,428,428,428,428,428,428,428,428,
-428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,
-428,428,429,429,429,429,429,429,429,429,429,429,429,429,429,429,
-429,429,429,429,429,429,429,429,429,429,429,429,428,428,428,428,
-428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,
-428,428,428,428,428,428,429,429,429,429,429,429,429,429,429,429,
-429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,
-428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,
+/* block 195 */
+438,438,438,438,438,438,438,438,437,437,437,437,437,437,437,437,
+437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,
+437,437,438,438,438,438,438,438,438,438,438,438,438,438,438,438,
+438,438,438,438,438,438,438,438,438,438,438,438,437,437,437,437,
+437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,
+437,437,437,437,437,437,438,438,438,438,438,438,438,438,438,438,
+438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,
+437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,
 
-/* block 176 */
-428,428,428,428,428,428,428,428,428,428,429,429,429,429,429,429,
-429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,
-429,429,429,429,429,429,109,109,428,428,428,428,428,428,428,428,
-428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,
-428,  8,429,429,429,429,429,429,429,429,429,429,429,429,429,429,
-429,429,429,429,429,429,429,429,429,429,429,  8,429,429,429,429,
-429,429,428,428,428,428,428,428,428,428,428,428,428,428,428,428,
-428,428,428,428,428,428,428,428,428,428,428,  8,429,429,429,429,
+/* block 196 */
+437,437,437,437,437,437,437,437,437,437,438,438,438,438,438,438,
+438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,
+438,438,438,438,438,438,114,114,437,437,437,437,437,437,437,437,
+437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,
+437,  8,438,438,438,438,438,438,438,438,438,438,438,438,438,438,
+438,438,438,438,438,438,438,438,438,438,438,  8,438,438,438,438,
+438,438,437,437,437,437,437,437,437,437,437,437,437,437,437,437,
+437,437,437,437,437,437,437,437,437,437,437,  8,438,438,438,438,
 
-/* block 177 */
-429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,
-429,429,429,429,429,  8,429,429,429,429,429,429,428,428,428,428,
-428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,
-428,428,428,428,428,  8,429,429,429,429,429,429,429,429,429,429,
-429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,  8,
-429,429,429,429,429,429,428,428,428,428,428,428,428,428,428,428,
-428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,  8,
-429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,
+/* block 197 */
+438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,
+438,438,438,438,438,  8,438,438,438,438,438,438,437,437,437,437,
+437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,
+437,437,437,437,437,  8,438,438,438,438,438,438,438,438,438,438,
+438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,  8,
+438,438,438,438,438,438,437,437,437,437,437,437,437,437,437,437,
+437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,  8,
+438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,
 
-/* block 178 */
-429,429,429,429,429,429,429,429,429,  8,429,429,429,429,429,429,
-428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,
-428,428,428,428,428,428,428,428,428,  8,429,429,429,429,429,429,
-429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,429,
-429,429,429,  8,429,429,429,429,429,429,428,429,109,109, 10, 10,
+/* block 198 */
+438,438,438,438,438,438,438,438,438,  8,438,438,438,438,438,438,
+437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,
+437,437,437,437,437,437,437,437,437,  8,438,438,438,438,438,438,
+438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,
+438,438,438,  8,438,438,438,438,438,438,437,438,114,114, 10, 10,
  10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
  10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
  10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
 
-/* block 179 */
-191,191,191,191,109,191,191,191,191,191,191,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,
-109,191,191,109,191,109,109,191,109,191,191,191,191,191,191,191,
-191,191,191,109,191,191,191,191,109,191,109,191,109,109,109,109,
-109,109,191,109,109,109,109,191,109,191,109,191,109,191,191,191,
-109,191,191,109,191,109,109,191,109,191,109,191,109,191,109,191,
-109,191,191,109,191,109,109,191,191,191,191,109,191,191,191,191,
-191,191,191,109,191,191,191,191,109,191,191,191,191,109,191,109,
+/* block 199 */
+715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,
+715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,
+715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,
+715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,
+715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,
+715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,
+715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,
+715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,
 
-/* block 180 */
-191,191,191,191,191,191,191,191,191,191,109,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,109,109,109,109,
-109,191,191,191,109,191,191,191,191,191,109,191,191,191,191,191,
-191,191,191,191,191,191,191,191,191,191,191,191,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-186,186,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+/* block 200 */
+715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,
+715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,
+715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,
+715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,
+715,715,715,715,715,114,114,716,716,716,716,716,716,716,716,716,
+717,717,717,717,717,717,717,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
-/* block 181 */
+/* block 201 */
+199,199,199,199,114,199,199,199,199,199,199,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,199,
+114,199,199,114,199,114,114,199,114,199,199,199,199,199,199,199,
+199,199,199,114,199,199,199,199,114,199,114,199,114,114,114,114,
+114,114,199,114,114,114,114,199,114,199,114,199,114,199,199,199,
+114,199,199,114,199,114,114,199,114,199,114,199,114,199,114,199,
+114,199,199,114,199,114,114,199,199,199,199,114,199,199,199,199,
+199,199,199,114,199,199,199,199,114,199,199,199,199,114,199,114,
+
+/* block 202 */
+199,199,199,199,199,199,199,199,199,199,114,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,114,114,114,114,
+114,199,199,199,114,199,199,199,199,199,114,199,199,199,199,199,
+199,199,199,199,199,199,199,199,199,199,199,199,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+194,194,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+
+/* block 203 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,109,109,109,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,114,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
 
-/* block 182 */
+/* block 204 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19,109,109,109,109,109,109,109,109,109,109,109,109,
- 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,
-109, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,
-109, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
-109, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+ 19, 19, 19, 19,114,114,114,114,114,114,114,114,114,114,114,114,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,
+114, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+114, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+114, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19,114,114,114,114,114,114,114,114,114,114,
 
-/* block 183 */
- 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,109,109,109,109,109,
+/* block 205 */
+ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,114,114,114,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,109,109,109,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,114,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
 
-/* block 184 */
+/* block 206 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,626,626,626,626,626,626,626,626,626,626,
-626,626,626,626,626,626,626,626,626,626,626,626,626,626,626,626,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,718,718,718,718,718,718,718,718,718,718,
+718,718,718,718,718,718,718,718,718,718,718,718,718,718,718,718,
 
-/* block 185 */
-627, 19, 19,109,109,109,109,109,109,109,109,109,109,109,109,109,
+/* block 207 */
+719, 19, 19,114,114,114,114,114,114,114,114,114,114,114,114,114,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,109,109,109,109,
- 19, 19, 19, 19, 19, 19, 19, 19, 19,109,109,109,109,109,109,109,
- 19, 19,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,114,114,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,114,114,114,114,
+ 19, 19,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
-/* block 186 */
+/* block 208 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
- 19, 19, 19, 19, 19, 19,109, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,109,109,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,
 
-/* block 187 */
+/* block 209 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19,109,109,109,109,109,109,109,109,109,109,109,109,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19,109, 19, 19, 19, 19, 19,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,
+114,114,114,114, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,114,114,114,114,114,
 
-/* block 188 */
+/* block 210 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,
- 19,109, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
-
-/* block 189 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,
+
+/* block 211 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,114,114,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19, 19, 19,109, 19, 19, 19, 19,109,109,109,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114, 19, 19, 19, 19, 19,
 
-/* block 190 */
+/* block 212 */
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19,114, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,109,109,
- 19, 19, 19, 19,109,109,109,109,109,109,109,109,109,109,109,109,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19, 19, 19,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
 
-/* block 191 */
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109, 19, 19, 19, 19, 19,
+/* block 213 */
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19,114,114, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
 
-/* block 192 */
+/* block 214 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19,109,109,109,109, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,
+ 19, 19, 19, 19,114,114,114,114,114,114,114,114,114,114,114,114,
 
-/* block 193 */
+/* block 215 */
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19, 19,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19,114,114,114,114,114,114,114,114,114,114,114,114,
 
-/* block 194 */
+/* block 216 */
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+
+/* block 217 */
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,114,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,114,114,114,114,114,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,114,114,114,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
- 19, 19, 19, 19,109,109,109,109,109,109,109,109,109,109,109,109,
 
-/* block 195 */
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+/* block 218 */
+ 19, 19, 19, 19, 19, 19, 19, 19,114,114,114,114,114,114,114,114,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
-/* block 196 */
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,109,109,109,109,109,109,109,109,109,109,109,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
+/* block 219 */
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
 
-/* block 197 */
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,475,
-475,475,475,475,475,475,475,475,475,475,475,475,475,475,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
-109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+/* block 220 */
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,114,114,114,114,114,114,114,114,114,114,114,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
 
-/* block 198 */
-426, 22,426,426,426,426,426,426,426,426,426,426,426,426,426,426,
-426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,
+/* block 221 */
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,484,
+484,484,484,484,484,484,484,484,484,484,484,484,484,484,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,114,
+
+/* block 222 */
+436, 22,436,436,436,436,436,436,436,436,436,436,436,436,436,436,
+436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,
  22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
  22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
  22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
@@ -3248,45 +3580,45 @@ const pcre_uint16 PRIV(ucd_stage2)[] = { /* 51968 bytes, block = 128 */
  22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
  22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
 
-/* block 199 */
-426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,
-426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,
-426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,
-426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,
-426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,
-426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,
-426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,
-426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,
+/* block 223 */
+436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,
+436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,
+436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,
+436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,
+436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,
+436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,
+436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,
+436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,
 
-/* block 200 */
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
+/* block 224 */
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
 
-/* block 201 */
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,104,
-426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,
+/* block 225 */
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,
+436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,436,
 
-/* block 202 */
-539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
-539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
-539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
-539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
-539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
-539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
-539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,
-539,539,539,539,539,539,539,539,539,539,539,539,539,539,109,109,
+/* block 226 */
+552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,
+552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,
+552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,
+552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,
+552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,
+552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,
+552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,
+552,552,552,552,552,552,552,552,552,552,552,552,552,552,114,114,
 
 };
 
diff --git a/erts/emulator/pcre/pcre_xclass.c b/erts/emulator/pcre/pcre_xclass.c
index cf07451a10..67095fa18b 100644
--- a/erts/emulator/pcre/pcre_xclass.c
+++ b/erts/emulator/pcre/pcre_xclass.c
@@ -82,6 +82,11 @@ additional data. */
 
 if (c < 256)
   {
+  if ((*data & XCL_HASPROP) == 0)
+    {
+    if ((*data & XCL_MAP) == 0) return negated;
+    return (((pcre_uint8 *)(data + 1))[c/8] & (1 << (c&7))) != 0;
+    }
   if ((*data & XCL_MAP) != 0 &&
     (((pcre_uint8 *)(data + 1))[c/8] & (1 << (c&7))) != 0)
     return !negated; /* char found */
@@ -129,55 +134,62 @@ while ((t = *data++) != XCL_END)
   else  /* XCL_PROP & XCL_NOTPROP */
     {
     const ucd_record *prop = GET_UCD(c);
+    BOOL isprop = t == XCL_PROP;
 
     switch(*data)
       {
       case PT_ANY:
-      if (t == XCL_PROP) return !negated;
+      if (isprop) return !negated;
       break;
 
       case PT_LAMP:
       if ((prop->chartype == ucp_Lu || prop->chartype == ucp_Ll ||
-           prop->chartype == ucp_Lt) == (t == XCL_PROP)) return !negated;
+           prop->chartype == ucp_Lt) == isprop) return !negated;
       break;
 
       case PT_GC:
-      if ((data[1] == PRIV(ucp_gentype)[prop->chartype]) == (t == XCL_PROP))
+      if ((data[1] == PRIV(ucp_gentype)[prop->chartype]) == isprop)
         return !negated;
       break;
 
       case PT_PC:
-      if ((data[1] == prop->chartype) == (t == XCL_PROP)) return !negated;
+      if ((data[1] == prop->chartype) == isprop) return !negated;
       break;
 
       case PT_SC:
-      if ((data[1] == prop->script) == (t == XCL_PROP)) return !negated;
+      if ((data[1] == prop->script) == isprop) return !negated;
       break;
 
       case PT_ALNUM:
       if ((PRIV(ucp_gentype)[prop->chartype] == ucp_L ||
-           PRIV(ucp_gentype)[prop->chartype] == ucp_N) == (t == XCL_PROP))
+           PRIV(ucp_gentype)[prop->chartype] == ucp_N) == isprop)
         return !negated;
       break;
 
-      case PT_SPACE:    /* Perl space */
-      if ((PRIV(ucp_gentype)[prop->chartype] == ucp_Z ||
-           c == CHAR_HT || c == CHAR_NL || c == CHAR_FF || c == CHAR_CR)
-             == (t == XCL_PROP))
-        return !negated;
-      break;
+      /* Perl space used to exclude VT, but from Perl 5.18 it is included,
+      which means that Perl space and POSIX space are now identical. PCRE
+      was changed at release 8.34. */
 
+      case PT_SPACE:    /* Perl space */
       case PT_PXSPACE:  /* POSIX space */
-      if ((PRIV(ucp_gentype)[prop->chartype] == ucp_Z ||
-           c == CHAR_HT || c == CHAR_NL || c == CHAR_VT ||
-           c == CHAR_FF || c == CHAR_CR) == (t == XCL_PROP))
-        return !negated;
+      switch(c)
+        {
+        HSPACE_CASES:
+        VSPACE_CASES:
+        if (isprop) return !negated;
+        break;
+
+        default:
+        if ((PRIV(ucp_gentype)[prop->chartype] == ucp_Z) == isprop)
+          return !negated;
+        break;
+        }
       break;
 
       case PT_WORD:
       if ((PRIV(ucp_gentype)[prop->chartype] == ucp_L ||
            PRIV(ucp_gentype)[prop->chartype] == ucp_N || c == CHAR_UNDERSCORE)
-             == (t == XCL_PROP))
+             == isprop)
         return !negated;
       break;
 
@@ -185,16 +197,60 @@ while ((t = *data++) != XCL_END)
       if (c < 0xa0)
         {
         if ((c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT ||
-             c == CHAR_GRAVE_ACCENT) == (t == XCL_PROP))
+             c == CHAR_GRAVE_ACCENT) == isprop)
           return !negated;
         }
       else
         {
-        if ((c < 0xd800 || c > 0xdfff) == (t == XCL_PROP))
+        if ((c < 0xd800 || c > 0xdfff) == isprop)
           return !negated;
         }
       break;
 
+      /* The following three properties can occur only in an XCLASS, as there
+      is no \p or \P coding for them. */
+
+      /* Graphic character. Implement this as not Z (space or separator) and
+      not C (other), except for Cf (format) with a few exceptions. This seems
+      to be what Perl does. The exceptional characters are:
+
+      U+061C           Arabic Letter Mark
+      U+180E           Mongolian Vowel Separator
+      U+2066 - U+2069  Various "isolate"s
+      */
+
+      case PT_PXGRAPH:
+      if ((PRIV(ucp_gentype)[prop->chartype] != ucp_Z &&
+            (PRIV(ucp_gentype)[prop->chartype] != ucp_C ||
+              (prop->chartype == ucp_Cf &&
+                c != 0x061c && c != 0x180e && (c < 0x2066 || c > 0x2069))
+         )) == isprop)
+        return !negated;
+      break;
+
+      /* Printable character: same as graphic, with the addition of Zs, i.e.
+      not Zl and not Zp, and U+180E. */
+
+      case PT_PXPRINT:
+      if ((prop->chartype != ucp_Zl &&
+           prop->chartype != ucp_Zp &&
+            (PRIV(ucp_gentype)[prop->chartype] != ucp_C ||
+              (prop->chartype == ucp_Cf &&
+                c != 0x061c && (c < 0x2066 || c > 0x2069))
+         )) == isprop)
+        return !negated;
+      break;
+
+      /* Punctuation: all Unicode punctuation, plus ASCII characters that
+      Unicode treats as symbols rather than punctuation, for Perl
+      compatibility (these are $+<=>^`|~). */
+
+      case PT_PXPUNCT:
+      if ((PRIV(ucp_gentype)[prop->chartype] == ucp_P ||
+            (c < 128 && PRIV(ucp_gentype)[prop->chartype] == ucp_S)) == isprop)
+        return !negated;
+      break;
+
       /* This should never occur, but compilers may mutter if there is no
       default. */
 
diff --git a/erts/emulator/pcre/ucp.h b/erts/emulator/pcre/ucp.h
index bbfe0f3ecb..7073bb9806 100644
--- a/erts/emulator/pcre/ucp.h
+++ b/erts/emulator/pcre/ucp.h
@@ -13,7 +13,10 @@ should always be at the end of each enum, for backwards compatibility.
 
 IMPORTANT: Note also that the specific numeric values of the enums have to be
 the same as the values that are generated by the maint/MultiStage2.py script,
-where the equivalent property descriptive names are listed in vectors. */
+where the equivalent property descriptive names are listed in vectors.
+
+ALSO: The specific values of the first two enums are assumed for the table
+called catposstab in pcre_compile.c. */
 
 /* These are the general character categories. */
 
@@ -191,7 +194,31 @@ enum {
   ucp_Miao,
   ucp_Sharada,
   ucp_Sora_Sompeng,
-  ucp_Takri
+  ucp_Takri,
+  /* New for Unicode 7.0.0: */
+  ucp_Bassa_Vah,
+  ucp_Caucasian_Albanian,
+  ucp_Duployan,
+  ucp_Elbasan,
+  ucp_Grantha,
+  ucp_Khojki,
+  ucp_Khudawadi,
+  ucp_Linear_A,
+  ucp_Mahajani,
+  ucp_Manichaean,
+  ucp_Mende_Kikakui,
+  ucp_Modi,
+  ucp_Mro,
+  ucp_Nabataean,
+  ucp_Old_North_Arabian,
+  ucp_Old_Permic,
+  ucp_Pahawh_Hmong,
+  ucp_Palmyrene,
+  ucp_Psalter_Pahlavi,
+  ucp_Pau_Cin_Hau,
+  ucp_Siddham,
+  ucp_Tirhuta,
+  ucp_Warang_Citi
 };
 
 #endif
-- 
cgit v1.2.3


From 9c4a807cc1b83a6918c0ccf19cdc3837d746f53b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 5 Apr 2017 14:11:56 +0200
Subject: erts: Remove hipe_bifs:remove_refs_from/1

which serves no purpose after all the
hipe load&purge fixes merged at
32729cab75325de58bf127e6e8836348071b8682
---
 erts/emulator/hipe/hipe_bif0.c   | 40 ----------------------------------------
 erts/emulator/hipe/hipe_bif0.tab |  1 -
 2 files changed, 41 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index e51e9f2733..9076c65c2a 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1616,46 +1616,6 @@ static void purge_mfa(struct hipe_mfa_info* p)
     erts_free(ERTS_ALC_T_HIPE_LL, p);
 }
 
-/* Called by init:restart after unloading all hipe compiled modules
- * to work around old bug that caused execution of deallocated beam code.
- * Can be removed now when delete/purge of native modules works better.
- * Test: Do init:restart in debug compiled vm with hipe compiled kernel. 
- */
-static void hipe_purge_all_refs(void)
-{
-    struct hipe_mfa_info **bucket;
-    unsigned int i, nrbuckets;
-
-    hipe_mfa_info_table_rwlock();
-
-    ASSERT(hipe_mfa_info_table.used == 0);
-    bucket = hipe_mfa_info_table.bucket;
-    nrbuckets = 1 << hipe_mfa_info_table.log2size;
-    for (i = 0; i < nrbuckets; ++i) {	
-        ASSERT(bucket[i] == NULL);
-	while (bucket[i] != NULL) {
-	    struct hipe_mfa_info* mfa = bucket[i];
-	    bucket[i] = mfa->bucket.next;
-
-            hash_erase(&mod2mfa_tab, mfa);
-            erts_free(ERTS_ALC_T_HIPE_LL, mfa);
-	}
-    }
-    hipe_mfa_info_table.used = 0;
-    hipe_mfa_info_table_rwunlock();
-}
-
-BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1)
-{
-    if (BIF_ARG_1 == am_all) {
-	hipe_purge_all_refs();
-	BIF_RET(am_ok);
-    }
-
-    ASSERT(!"hipe_bifs_remove_refs_from_1() called");
-    BIF_ERROR(BIF_P, BADARG);
-}
-
 int hipe_purge_need_blocking(Module* modp)
 {
     /* SVERK: Verify if this is really necessary */
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 264ea2c34a..078ebc40b7 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -82,7 +82,6 @@ bif hipe_bifs:patch_insn/3
 bif hipe_bifs:patch_call/3
 
 bif hipe_bifs:add_ref/2
-bif hipe_bifs:remove_refs_from/1
 
 bif hipe_bifs:alloc_loader_state/1
 
-- 
cgit v1.2.3


From 647984f14188ba2cf73128c09e888d4bcf733a9f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 5 Apr 2017 14:35:56 +0200
Subject: erts: Remove deliberate leak of hipe fun entries

should be fixed since
32729cab75325de58bf127e6e8836348071b8682
---
 erts/emulator/beam/beam_load.c | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index ae98b37956..baa8a183d3 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -6269,16 +6269,7 @@ patch_funentries(Eterm Patchlist)
     fe = erts_get_fun_entry(Mod, uniq, index);
     fe->native_address = (Uint *)native_address;
 
-    /* Deliberate MEMORY LEAK of native fun entries!!!
-     *
-     * Uncomment line below when hipe code upgrade and purging works correctly.
-     * Today we may get cases when old (leaked) native code of a purged module
-     * gets called and tries to create instances of a deleted fun entry.
-     *
-     * Reproduced on a debug emulator with stdlib_test/qlc_SUITE:join_merge
-     *
-     * erts_smp_refc_dec(&fe->refc, 1);
-     */
+    erts_smp_refc_dec(&fe->refc, 1);
 
     if (!patch(Addresses, (Uint) fe))
       return 0;
-- 
cgit v1.2.3


From bb89d065dbe47554c0cb929d244e0d38cad52e65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 7 Apr 2017 12:04:11 +0200
Subject: beam_emu: Slightly optimize the bxor/2 operator

bxor is used in the rand module, so even small optimizations
could be worthwile.

Suggested by Raimo Niskanen.
---
 erts/emulator/beam/beam_emu.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 9a91fdce08..6010c17c17 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3044,10 +3044,12 @@ do {						\
      GetArg2(2, Op1, Op2);
      if (is_both_small(Op1, Op2)) {
 	 /*
-	  * We could extract the tag from one argument, but a tag extraction
-	  * could mean a shift.  Therefore, play it safe here.
+          * TAG ^ TAG == 0.
+          *
+          * Therefore, we perform the XOR operation on the tagged values,
+          * and OR in the tag bits.
 	  */
-	 Eterm result = make_small(signed_val(Op1) ^ signed_val(Op2));
+	 Eterm result = (Op1 ^ Op2) | make_small(0);
 	 StoreBifResult(4, result);
      }
      DO_OUTLINED_ARITH_2(bxor, Op1, Op2);
-- 
cgit v1.2.3


From 8f452530e61b299d4d48f82f41ab5364723607ae Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 5 Apr 2017 14:32:16 +0200
Subject: Stack guard for PCRE

---
 erts/emulator/beam/beam_debug.c         | 46 +++++++++++++++++++++++++++++
 erts/emulator/beam/erl_bif_info.c       | 15 ++++++++++
 erts/emulator/beam/erl_bif_re.c         | 35 ++++++++++++++++++++++
 erts/emulator/beam/erl_init.c           | 52 +++++++++++++++++++++++----------
 erts/emulator/beam/global.h             | 15 ++++++++++
 erts/emulator/beam/sys.h                |  2 ++
 erts/emulator/beam/utils.c              | 47 +++++++++++++++++++++++++++++
 erts/emulator/pcre/local_config.h       |  2 +-
 erts/emulator/sys/unix/sys.c            | 12 ++++++++
 erts/emulator/sys/win32/sys.c           |  6 ++++
 erts/emulator/test/erts_debug_SUITE.erl | 14 +++++++--
 11 files changed, 228 insertions(+), 18 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 21d336049f..d708cf89a7 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -731,3 +731,49 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 
     return size;
 }
+
+
+#ifdef ERTS_SMP
+#  define ERTS_STACK_LIMIT ((char *) ethr_get_stacklimit())
+#else
+#  define ERTS_STACK_LIMIT ((char *) erts_scheduler_stack_limit)
+#endif
+
+/*
+ * The below functions is for testing of the stack
+ * limit functionality. They are intentionally
+ * written body recursive in order to prevent
+ * last call optimization...
+ */
+
+UWord
+erts_check_stack_recursion_downwards(char *start_c)
+{
+    char *limit = ERTS_STACK_LIMIT;
+    char c;
+    UWord res;
+    if (erts_check_below_limit(&c, limit + 1024))
+        return (char *) erts_ptr_id(start_c) - (char *) erts_ptr_id(&c);
+    res = erts_check_stack_recursion_downwards(start_c);
+    erts_ptr_id(&c);
+    return res;
+}
+
+UWord
+erts_check_stack_recursion_upwards(char *start_c)
+{
+    char *limit = ERTS_STACK_LIMIT;
+    char c;
+    UWord res;
+    if (erts_check_above_limit(&c, limit - 1024))
+        return (char *) erts_ptr_id(&c) - (char *) erts_ptr_id(start_c);
+    res = erts_check_stack_recursion_upwards(start_c);
+    erts_ptr_id(&c);
+    return res;
+}
+
+int
+erts_is_above_stack_limit(char *ptr)
+{
+    return (char *) ptr > ERTS_STACK_LIMIT;
+}
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 7bd45916f5..dd1299ca57 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3674,6 +3674,21 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 		BIF_RET(erts_sint64_to_big(value, &hp));
 	    }
 	}
+        else if (ERTS_IS_ATOM_STR("stack_check", BIF_ARG_1)) {
+            UWord size;
+            char c;
+            if (erts_is_above_stack_limit(&c))
+                size = erts_check_stack_recursion_downwards(&c);
+            else
+                size = erts_check_stack_recursion_upwards(&c);
+	    if (IS_SSMALL(size))
+		BIF_RET(make_small(size));
+	    else {
+		Uint hsz = BIG_UWORD_HEAP_SIZE(size);
+		Eterm *hp = HAlloc(BIF_P, hsz);
+		BIF_RET(uword_to_big(size, hp));
+	    }
+        }
     }
     else if (is_tuple(BIF_ARG_1)) {
 	Eterm* tp = tuple_val(BIF_ARG_1);
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index ff7746ce1d..35b196743f 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -64,12 +64,47 @@ static void erts_erts_pcre_stack_free(void *ptr) {
     erts_free(ERTS_ALC_T_RE_STACK,ptr);
 }
 
+#define ERTS_PCRE_STACK_MARGIN (10*1024)
+
+#ifdef ERTS_SMP
+#  define ERTS_STACK_LIMIT ((char *) ethr_get_stacklimit())
+#else
+#  define ERTS_STACK_LIMIT ((char *) erts_scheduler_stack_limit)
+#endif
+
+static int
+stack_guard_downwards(void)
+{
+    char *limit = ERTS_STACK_LIMIT;
+    char c;
+
+    ASSERT(limit);
+
+    return erts_check_below_limit(&c, limit + ERTS_PCRE_STACK_MARGIN);
+}
+
+static int
+stack_guard_upwards(void)
+{
+    char *limit = ERTS_STACK_LIMIT;
+    char c;
+
+    ASSERT(limit);
+
+    return erts_check_above_limit(&c, limit - ERTS_PCRE_STACK_MARGIN);
+}
+
 void erts_init_bif_re(void)
 {
+    char c;
     erts_pcre_malloc = &erts_erts_pcre_malloc;
     erts_pcre_free = &erts_erts_pcre_free;
     erts_pcre_stack_malloc = &erts_erts_pcre_stack_malloc;
     erts_pcre_stack_free = &erts_erts_pcre_stack_free;
+    if ((char *) erts_ptr_id(&c) > ERTS_STACK_LIMIT)
+        erts_pcre_stack_guard = stack_guard_downwards;
+    else
+        erts_pcre_stack_guard = stack_guard_upwards;
     default_table = NULL; /* ISO8859-1 default, forced into pcre */
     max_loop_limit = CONTEXT_REDS * LOOP_FACTOR;
 
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index e8b8739852..4e408e8305 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -61,6 +61,9 @@
 
 #define ERTS_DEFAULT_NO_ASYNC_THREADS	10
 
+#define ERTS_DEFAULT_SCHED_STACK_SIZE   256
+#define ERTS_MIN_SCHED_STACK_SIZE       20
+
 /*
  * The variables below (prefixed with etp_) are for erts/etc/unix/etp-commands
  * only. Do not remove even though they aren't used elsewhere in the emulator!
@@ -123,6 +126,8 @@ const Eterm etp_hole_marker = ERTS_HOLE_MARKER;
 const Eterm etp_hole_marker = 0;
 #endif
 
+static int modified_sched_thread_suggested_stack_size = 0;
+
 /*
  * Note about VxWorks: All variables must be initialized by executable code,
  * not by an initializer. Otherwise a new instance of the emulator will
@@ -1231,24 +1236,38 @@ early_init(int *argc, char **argv) /*
 }
 
 #ifndef ERTS_SMP
+
+void *erts_scheduler_stack_limit;
+
+
 static void set_main_stack_size(void)
 {
-    if (erts_sched_thread_suggested_stack_size > 0) {
+    char c;
+    UWord stacksize;
 # if HAVE_DECL_GETRLIMIT && HAVE_DECL_SETRLIMIT && HAVE_DECL_RLIMIT_STACK
-	struct rlimit rl;
-	int bytes = erts_sched_thread_suggested_stack_size * sizeof(Uint) * 1024;
-	if (getrlimit(RLIMIT_STACK, &rl) != 0 ||
-	    (rl.rlim_cur = bytes, setrlimit(RLIMIT_STACK, &rl) != 0)) {
-	    erts_fprintf(stderr, "failed to set stack size for scheduler "
-				 "thread to %d bytes\n", bytes);
-	    erts_usage();
-	}	    
+    struct rlimit rl;
+    int bytes;
+    stacksize = erts_sched_thread_suggested_stack_size * sizeof(Uint) * 1024;
+    /* Add some extra pages... neede by some systems... */
+    bytes = (int) stacksize + 3*erts_sys_get_page_size();
+    if (getrlimit(RLIMIT_STACK, &rl) != 0 ||
+        (rl.rlim_cur = bytes, setrlimit(RLIMIT_STACK, &rl) != 0)) {
+        erts_fprintf(stderr, "failed to set stack size for scheduler "
+                     "thread to %d bytes\n", bytes);
+        erts_usage();
+    }
 # else
+    if (modified_sched_thread_suggested_stack_size) {
 	erts_fprintf(stderr, "no OS support for dynamic stack size limit\n");
-	erts_usage();    
-# endif
+	erts_usage();
     }
+    /* Be conservative and hope it is not more than 64 kWords... */
+    stacksize = 64*1024*sizeof(void *);
+# endif
+
+    erts_scheduler_stack_limit = erts_calc_stacklimit(&c, stacksize);
 }
+
 #endif
 
 void
@@ -1293,12 +1312,11 @@ erl_start(int argc, char **argv)
 	port_tab_sz_ignore_files = 1;
     }
 
-#if (defined(__APPLE__) && defined(__MACH__)) || defined(__DARWIN__)
     /*
-     * The default stack size on MacOS X is too small for pcre.
+     * A default stack size suitable for pcre which might use quite
+     * a lot of stack.
      */
-    erts_sched_thread_suggested_stack_size = 256;
-#endif
+    erts_sched_thread_suggested_stack_size = ERTS_DEFAULT_SCHED_STACK_SIZE;
 
 #ifdef DEBUG
     verbose = DEBUG_DEFAULT;
@@ -1921,6 +1939,7 @@ erl_start(int argc, char **argv)
 		/* suggested stack size (Kilo Words) for scheduler threads */
 		arg = get_arg(sub_param+2, argv[i+1], &i);
 		erts_sched_thread_suggested_stack_size = atoi(arg);
+                modified_sched_thread_suggested_stack_size = 1;
 
 		if ((erts_sched_thread_suggested_stack_size
 		     < ERTS_SCHED_THREAD_MIN_STACK_SIZE)
@@ -2230,6 +2249,9 @@ erl_start(int argc, char **argv)
     boot_argc = argc - i;  /* Number of arguments to init */
     boot_argv = &argv[i];
 
+    if (erts_sched_thread_suggested_stack_size < ERTS_MIN_SCHED_STACK_SIZE)
+        erts_sched_thread_suggested_stack_size = ERTS_MIN_SCHED_STACK_SIZE;
+
     erl_init(ncpu,
 	     proc_tab_sz,
 	     legacy_proc_tab,
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 19286e1310..45c01673d8 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1218,6 +1218,11 @@ void erts_short_init(void);
 void erl_start(int, char**);
 void erts_usage(void);
 Eterm erts_preloaded(Process* p);
+
+#ifndef ERTS_SMP
+extern void *erts_scheduler_stack_limit;
+#endif
+
 /* erl_md5.c */
 
 typedef struct {
@@ -1278,6 +1283,11 @@ Uint64 erts_timestamp_millis(void);
 
 Export* erts_find_function(Eterm, Eterm, unsigned int, ErtsCodeIndex);
 
+void *erts_calc_stacklimit(char *prev_c, UWord stacksize);
+int erts_check_below_limit(char *ptr, char *limit);
+int erts_check_above_limit(char *ptr, char *limit);
+void *erts_ptr_id(void *ptr);
+
 Eterm store_external_or_ref_in_proc_(Process *, Eterm);
 Eterm store_external_or_ref_(Uint **, ErlOffHeap*, Eterm);
 
@@ -1313,6 +1323,11 @@ void erts_init_external(void);
 /* erl_map.c */
 void erts_init_map(void);
 
+/* beam_debug.c */
+UWord erts_check_stack_recursion_downwards(char *start_c);
+UWord erts_check_stack_recursion_upwards(char *start_c);
+int erts_is_above_stack_limit(char *ptr);
+
 /* erl_unicode.c */
 void erts_init_unicode(void);
 Sint erts_unicode_set_loop_limit(Sint limit);
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index d49edad6dc..41ba51d037 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -604,6 +604,8 @@ __decl_noreturn void __noreturn erts_exit(int n, char*, ...);
 
 Eterm erts_check_io_info(void *p);
 
+UWord erts_sys_get_page_size(void);
+
 /* Size of misc memory allocated from system dependent code */
 Uint erts_sys_misc_mem_sz(void);
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index d90c282c7e..d91ae1ffe6 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -5031,6 +5031,53 @@ Uint64 erts_timestamp_millis(void)
 #endif
 }
 
+void *
+erts_calc_stacklimit(char *prev_c, UWord stacksize)
+{
+    /*
+     * We *don't* want this function inlined, i.e., it is
+     * risky to call this function from another function
+     * in utils.c
+     */
+
+    UWord pagesize = erts_sys_get_page_size();
+    char c;
+    char *start;
+    if (&c > prev_c) {
+        start = (char *) ((((UWord) prev_c) / pagesize) * pagesize);
+        return (void *) (start + stacksize);
+    }
+    else {
+        start = (char *) (((((UWord) prev_c) - 1) / pagesize + 1) * pagesize);
+        return (void *) (start - stacksize);
+    }
+}
+
+/*
+ * erts_check_below_limit() and
+ * erts_check_above_limit() are put
+ * in utils.c in order to prevent
+ * inlining.
+ */
+
+int
+erts_check_below_limit(char *ptr, char *limit)
+{
+    return ptr < limit;
+}
+
+int
+erts_check_above_limit(char *ptr, char *limit)
+{
+    return ptr > limit;
+}
+
+void *
+erts_ptr_id(void *ptr)
+{
+    return ptr;
+}
+
 #ifdef DEBUG
 /*
  * Handy functions when using a debugger - don't use in the code!
diff --git a/erts/emulator/pcre/local_config.h b/erts/emulator/pcre/local_config.h
index 6a0b7b4d4d..e90f4dcada 100644
--- a/erts/emulator/pcre/local_config.h
+++ b/erts/emulator/pcre/local_config.h
@@ -58,7 +58,7 @@
 /* The value of PARENS_NEST_LIMIT specifies the maximum depth of nested
    parentheses (of any kind) in a pattern. This limits the amount of system
    stack that is used while compiling a pattern. */
-#define PARENS_NEST_LIMIT 250
+#define PARENS_NEST_LIMIT 10000
 
 /* Define if linking statically (TODO: make nice with Libtool) */
 #define PCRE_STATIC 1
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 6459fa064b..0c0acbf90c 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -275,6 +275,18 @@ erts_sys_schedule_interrupt_timed(int set, ErtsMonotonicTime timeout_time)
 }
 #endif
 
+UWord
+erts_sys_get_page_size(void)
+{
+#if defined(_SC_PAGESIZE)
+    return (UWord) sysconf(_SC_PAGESIZE);
+#elif defined(HAVE_GETPAGESIZE)
+    return (UWord) getpagesize();
+#else
+    return (UWord) 4*1024; /* Guess 4 KB */
+#endif
+}
+
 Uint
 erts_sys_misc_mem_sz(void)
 {
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index f3881e0736..2cd88b503e 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -186,6 +186,12 @@ void sys_primitive_init(HMODULE beam)
     beam_module = (HMODULE) beam;
 }
 
+UWord
+erts_sys_get_page_size(void)
+{
+    return (UWord) 4*1024; /* Guess 4 KB */
+}
+
 Uint
 erts_sys_misc_mem_sz(void)
 {
diff --git a/erts/emulator/test/erts_debug_SUITE.erl b/erts/emulator/test/erts_debug_SUITE.erl
index 23871585f7..c9c664de38 100644
--- a/erts/emulator/test/erts_debug_SUITE.erl
+++ b/erts/emulator/test/erts_debug_SUITE.erl
@@ -23,14 +23,15 @@
 
 -export([all/0, suite/0,
 	 test_size/1,flat_size_big/1,df/1,term_type/1,
-	 instructions/1]).
+	 instructions/1, stack_check/1]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
      {timetrap, {minutes, 2}}].
 
 all() -> 
-    [test_size, flat_size_big, df, instructions, term_type].
+    [test_size, flat_size_big, df, instructions, term_type,
+     stack_check].
 
 test_size(Config) when is_list(Config) ->
     ConsCell1 = id([a|b]),
@@ -181,6 +182,15 @@ df(Config) when is_list(Config) ->
     true = (P0 == pps()),
     ok.
 
+stack_check(Config) when is_list(Config) ->
+    erts_debug:set_internal_state(available_internal_state,true),
+    %% Recurses on the C stack until stacklimit is reached. That
+    %% is, tests that the stack limit functionality works (used
+    %% by PCRE). VM will crash if it doesn't work...
+    Size = erts_debug:get_internal_state(stack_check),
+    erts_debug:set_internal_state(available_internal_state,false),
+    {comment, "Stack size: "++integer_to_list(Size)++" bytes"}.
+
 df_smoke([M|Ms]) ->
     io:format("~p", [M]),
     erts_debug:df(M),
-- 
cgit v1.2.3


From b12f49679b8f0541edd93c2da40866e256cb7c95 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 29 Mar 2017 16:59:03 +0200
Subject: Update README.pcre_update.md

---
 erts/emulator/pcre/README.pcre_update.md | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/pcre/README.pcre_update.md b/erts/emulator/pcre/README.pcre_update.md
index 5f2414f0d9..8caf575d31 100644
--- a/erts/emulator/pcre/README.pcre_update.md
+++ b/erts/emulator/pcre/README.pcre_update.md
@@ -243,7 +243,8 @@ To begin with you will need a default table for Latin-1 characters, so:
 	 ~/tmp/pcre/pcre-8.33> LANG=sv_SE ./dftables -L ../epcre-8.33/pcre_latin_1_table.c
 
 Compare it to the pcre\_latin\_1\_table.c in the old version, they
-should not differ in any significant way.
+should not differ in any significant way. If they do, it might be
+that you do not have the sv_SE locale installed on your machine.
 
 A good starting point is then to try to find all files in the new
 version of the library that have (probably) the same names as the
@@ -685,10 +686,23 @@ generation that you do not get to many of the "Fishy character"
 messages, if they are more than, say 20, you will probably need to
 address the UTF8 issues in the Perl execution. As it is now, we skip
 non latin1 characters in this test. You will need to run iconv on the
-generated module to make it UTF-8 before running tests.
+generated module to make it UTF-8 before running tests. Try to use a
+perl version that is as new as possible.
 
 The exact same procedure goes for the re\_testoutput1\_split\_test.erl. 
 
+Make a note about perl version used in the commit updating the replace
+and split test files.
+
+Note that the perl version you are using may not be completely
+compatible with the PCRE version you are upgrading to. If this is the
+case you might get failures when running the replace and split tests.
+If you get failures, you need to inspect the failures and decide what
+to do. If there are only a small amount of failures you will probably
+end up preferring the behavior of PCRE, and manually changing these
+tests. Do these changes in a separate commit so it is easy to see
+what differed.
+
 Also add copyright headers to the files after converting them to UTF-8.
 
 After ironing out the rest of the bugs, you should be done with the
-- 
cgit v1.2.3


From 5e8f498b810aeae9cb4c9c3715fe62a33b84cdde Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 11 Apr 2017 15:30:11 +0200
Subject: erts: Init refc=1 in erts_bin_nrml_alloc

Only term_to_binary needed some extra attention
as it used to initialize refc as 0 instead of 1.
---
 erts/emulator/beam/beam_emu.c        |  2 --
 erts/emulator/beam/binary.c          |  2 --
 erts/emulator/beam/erl_bif_binary.c  |  1 -
 erts/emulator/beam/erl_binary.h      |  1 +
 erts/emulator/beam/erl_bits.c        |  3 ---
 erts/emulator/beam/erl_trace.c       |  1 -
 erts/emulator/beam/external.c        | 15 +++++----------
 erts/emulator/beam/io.c              |  2 --
 erts/emulator/hipe/hipe_native_bif.c |  1 -
 9 files changed, 6 insertions(+), 22 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 3291f92b71..321008616d 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3885,7 +3885,6 @@ do {						\
 	  * Allocate the binary struct itself.
 	  */
 	 bptr = erts_bin_nrml_alloc(num_bytes);
-	 erts_refc_init(&bptr->refc, 1);
 	 erts_current_bin = (byte *) bptr->orig_bytes;
 
 	 /*
@@ -3980,7 +3979,6 @@ do {						\
 	  * Allocate the binary struct itself.
 	  */
 	 bptr = erts_bin_nrml_alloc(BsOp1);
-	 erts_refc_init(&bptr->refc, 1);
 	 erts_current_bin = (byte *) bptr->orig_bytes;
 
 	 /*
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 4dd8316dad..0df6bbb289 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -84,7 +84,6 @@ new_binary(Process *p, byte *buf, Uint len)
      * Allocate the binary struct itself.
      */
     bptr = erts_bin_nrml_alloc(len);
-    erts_refc_init(&bptr->refc, 1);
     if (buf != NULL) {
 	sys_memcpy(bptr->orig_bytes, buf, len);
     }
@@ -121,7 +120,6 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, Uint len)
      * Allocate the binary struct itself.
      */
     bptr = erts_bin_nrml_alloc(len);
-    erts_refc_init(&bptr->refc, 1);
     if (buf != NULL) {
 	sys_memcpy(bptr->orig_bytes, buf, len);
     }
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 62a752d854..f79b5b6843 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -2669,7 +2669,6 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en)
 	}
 	cbs->result = erts_bin_nrml_alloc(target_size); /* Always offheap
 							   if trapping */
-	erts_refc_init(&(cbs->result->refc), 1);
 	t = (byte *) cbs->result->orig_bytes; /* No offset or anything */
 	pos = 0;
 	i = 0;
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 6ff71ec6d1..cb26ee3407 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -411,6 +411,7 @@ erts_bin_nrml_alloc(Uint size)
     ERTS_CHK_BIN_ALIGNMENT(res);
     res->orig_size = size;
     res->flags = 0;
+    erts_refc_init(&res->refc, 1);
     return res;
 }
 
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 885e955332..de4fa3594d 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1404,7 +1404,6 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term,
 	 * Allocate the binary data struct itself.
 	 */
 	bptr = erts_bin_nrml_alloc(bin_size);
-	erts_refc_init(&bptr->refc, 1);
 	erts_current_bin = (byte *) bptr->orig_bytes;
 
 	/*
@@ -1518,7 +1517,6 @@ erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit)
 	     * binary and copy the contents of the old binary into it.
 	     */
 	    Binary* bptr = erts_bin_nrml_alloc(new_size);
-	    erts_refc_init(&bptr->refc, 1);
 	    sys_memcpy(bptr->orig_bytes, binp->orig_bytes, binp->orig_size);
 	    pb->flags |= PB_IS_WRITABLE | PB_ACTIVE_WRITER;
 	    pb->val = bptr;
@@ -1565,7 +1563,6 @@ erts_bs_init_writable(Process* p, Eterm sz)
      * Allocate the binary data struct itself.
      */
     bptr = erts_bin_nrml_alloc(bin_size);
-    erts_refc_init(&bptr->refc, 1);
     
     /*
      * Now allocate the ProcBin on the heap.
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 04f3160d42..85b63208a3 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1875,7 +1875,6 @@ trace_port_tmp_binary(char *bin, Sint sz, Binary **bptrp, Eterm **hp)
     } else {
         ProcBin* pb = (ProcBin *)*hp;
         Binary *bptr = erts_bin_nrml_alloc(sz);
-        erts_refc_init(&bptr->refc, 1);
         sys_memcpy(bptr->orig_bytes, bin, sz);
         pb->thing_word = HEADER_PROC_BIN;
         pb->size = sz;
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 205a7711ec..873945c3b5 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1821,7 +1821,7 @@ static int ttb_context_destructor(Binary *context_bin)
 	case TTBEncode:
 	    DESTROY_SAVED_WSTACK(&context->s.ec.wstack);
 	    if (context->s.ec.result_bin != NULL) { /* Set to NULL if ever made alive! */
-		ASSERT(erts_refc_read(&(context->s.ec.result_bin->refc),0) == 0);
+		ASSERT(erts_refc_read(&(context->s.ec.result_bin->refc),1));
 		erts_bin_free(context->s.ec.result_bin);
 		context->s.ec.result_bin = NULL;
 	    }
@@ -1830,13 +1830,13 @@ static int ttb_context_destructor(Binary *context_bin)
 	    erl_zlib_deflate_finish(&(context->s.cc.stream));
 
 	    if (context->s.cc.destination_bin != NULL) { /* Set to NULL if ever made alive! */
-		ASSERT(erts_refc_read(&(context->s.cc.destination_bin->refc),0) == 0);
+		ASSERT(erts_refc_read(&(context->s.cc.destination_bin->refc),1));
 		erts_bin_free(context->s.cc.destination_bin);
 		context->s.cc.destination_bin = NULL;
 	    }
 	    
 	    if (context->s.cc.result_bin != NULL) { /* Set to NULL if ever made alive! */
-		ASSERT(erts_refc_read(&(context->s.cc.result_bin->refc),0) == 0);
+		ASSERT(erts_refc_read(&(context->s.cc.result_bin->refc),1));
 		erts_bin_free(context->s.cc.result_bin);
 		context->s.cc.result_bin = NULL;
 	    }
@@ -1920,7 +1920,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		}
 
 		result_bin = erts_bin_nrml_alloc(size);
-		erts_refc_init(&result_bin->refc, 0);
 		result_bin->orig_bytes[0] = VERSION_MAGIC;
 		/* Next state immediately, no need to export context */
 		context->state = TTBEncode;
@@ -1960,7 +1959,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		    pb->bytes = (byte*) result_bin->orig_bytes;
 		    pb->flags = 0;
 		    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
-		    erts_refc_inc(&result_bin->refc, 1);
 		    if (context_b && erts_refc_read(&context_b->refc,0) == 0) {
 			erts_bin_free(context_b);
 		    }
@@ -1980,7 +1978,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		context->s.cc.result_bin = result_bin;
 
 		result_bin = erts_bin_nrml_alloc(real_size);
-		erts_refc_init(&result_bin->refc, 0);
 		result_bin->orig_bytes[0] = VERSION_MAGIC;
 
 		context->s.cc.destination_bin = result_bin;
@@ -2028,10 +2025,10 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 			pb->next = MSO(p).first;
 			MSO(p).first = (struct erl_off_heap_header*)pb;
 			pb->val = result_bin;
+			ASSERT(erts_refc_read(&result_bin->refc, 1));
 			pb->bytes = (byte*) result_bin->orig_bytes;
 			pb->flags = 0;
 			OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
-			erts_refc_inc(&result_bin->refc, 1);
 			erts_bin_free(context->s.cc.result_bin);
 			context->s.cc.result_bin = NULL;
 			context->alive = 0;
@@ -2055,7 +2052,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		    pb->bytes = (byte*) result_bin->orig_bytes;
 		    pb->flags = 0;
 		    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
-		    erts_refc_inc(&result_bin->refc, 1);
+		    ASSERT(erts_refc_read(&result_bin->refc, 1));
 		    erl_zlib_deflate_finish(&(context->s.cc.stream));
 		    erts_bin_free(context->s.cc.destination_bin);
 		    context->s.cc.destination_bin = NULL;
@@ -3536,7 +3533,6 @@ dec_term_atom_common:
 		} else {
 		    Binary* dbin = erts_bin_nrml_alloc(n);
 		    ProcBin* pb;
-		    erts_refc_init(&dbin->refc, 1);
 		    pb = (ProcBin *) hp;
 		    hp += PROC_BIN_SIZE;
 		    pb->thing_word = HEADER_PROC_BIN;
@@ -3589,7 +3585,6 @@ dec_term_atom_common:
 		    Binary* dbin = erts_bin_nrml_alloc(n);
 		    ProcBin* pb;
 
-		    erts_refc_init(&dbin->refc, 1);
 		    pb = (ProcBin *) hp;
 		    pb->thing_word = HEADER_PROC_BIN;
 		    pb->size = n;
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index ddff862607..c85f900de5 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -3791,7 +3791,6 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
 	Binary* bptr;
 
 	bptr = erts_bin_nrml_alloc(len);
-	erts_refc_init(&bptr->refc, 1);
 	sys_memcpy(bptr->orig_bytes, buf, len);
 
 	pb = (ProcBin *) hp;
@@ -6400,7 +6399,6 @@ driver_deliver_term(Port *prt, Eterm to, ErlDrvTermData* data, int len)
 		ProcBin* pbp;
 		Binary* bp = erts_bin_nrml_alloc(size);
 		ASSERT(bufp);
-		erts_refc_init(&bp->refc, 1);
 		sys_memcpy((void *) bp->orig_bytes, (void *) bufp, size);
 		pbp = (ProcBin *) erts_produce_heap(&factory,
 						    PROC_BIN_SIZE, HEAP_EXTRA);
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index e581f07f56..342407ef62 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -323,7 +323,6 @@ char *hipe_bs_allocate(int len)
     Binary *bptr;
 
     bptr = erts_bin_nrml_alloc(len);
-    erts_refc_init(&bptr->refc, 1);
     return bptr->orig_bytes;
 }
 
-- 
cgit v1.2.3


From 37bdfb3b5b56c2311db1780b052b7e2c0f139cef Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 11 Apr 2017 16:11:37 +0200
Subject: erts: Init refc=1 in erts_bin_drv_alloc*

---
 erts/emulator/beam/dist.c       | 1 -
 erts/emulator/beam/erl_binary.h | 2 ++
 erts/emulator/beam/erl_nif.c    | 1 -
 erts/emulator/beam/io.c         | 1 -
 4 files changed, 2 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index d1c2da9074..49a48ac02e 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -629,7 +629,6 @@ alloc_dist_obuf(Uint size)
     ErtsDistOutputBuf *obuf;
     Uint obuf_size = sizeof(ErtsDistOutputBuf)+sizeof(byte)*(size-1);
     Binary *bin = erts_bin_drv_alloc(obuf_size);
-    erts_refc_init(&bin->refc, 1);
     obuf = (ErtsDistOutputBuf *) &bin->orig_bytes[0];
 #ifdef DEBUG
     obuf->dbg_pattern = ERTS_DIST_OUTPUT_BUF_DBG_PATTERN;
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index cb26ee3407..51072f4b0f 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -375,6 +375,7 @@ erts_bin_drv_alloc_fnf(Uint size)
     if (res) {
 	res->orig_size = size;
 	res->flags = BIN_FLAG_DRV;
+        erts_refc_init(&res->refc, 1);
     }
     return res;
 }
@@ -393,6 +394,7 @@ erts_bin_drv_alloc(Uint size)
     ERTS_CHK_BIN_ALIGNMENT(res);
     res->orig_size = size;
     res->flags = BIN_FLAG_DRV;
+    erts_refc_init(&res->refc, 1);
     return res;
 }
 
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index deed74ee49..94d25ba85a 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1074,7 +1074,6 @@ int enif_alloc_binary(size_t size, ErlNifBinary* bin)
     if (refbin == NULL) {
 	return 0; /* The NIF must take action */
     }
-    erts_refc_init(&refbin->refc, 1);
 
     bin->size = size;
     bin->data = (unsigned char*) refbin->orig_bytes;
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index c85f900de5..36b8ee6860 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -6938,7 +6938,6 @@ driver_alloc_binary(ErlDrvSizeT size)
     bin = erts_bin_drv_alloc_fnf((Uint) size);
     if (!bin)
 	return NULL; /* The driver write must take action */
-    erts_refc_init(&bin->refc, 1);
     return Binary2ErlDrvBinary(bin);
 }
 
-- 
cgit v1.2.3


From 486a758ef245effab01d880493a0274de49c1797 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 11 Apr 2017 16:28:12 +0200
Subject: erts: Introduce erts_bin_release

---
 erts/emulator/beam/beam_load.c   |  8 ++------
 erts/emulator/beam/dist.c        |  3 +--
 erts/emulator/beam/erl_binary.h  |  9 +++++++++
 erts/emulator/beam/erl_bits.c    |  4 +---
 erts/emulator/beam/erl_db.c      | 17 ++++++-----------
 erts/emulator/beam/erl_db_util.c |  7 ++-----
 erts/emulator/beam/erl_gc.c      |  7 ++-----
 erts/emulator/beam/erl_message.c |  7 ++-----
 erts/emulator/beam/erl_nif.c     | 12 +++---------
 erts/emulator/beam/erl_trace.c   |  8 ++++----
 erts/emulator/beam/global.h      |  4 ++--
 erts/emulator/beam/io.c          |  6 ++----
 12 files changed, 36 insertions(+), 56 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index baa8a183d3..3a3d1f57ce 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -996,9 +996,7 @@ static void
 free_loader_state(Binary* magic)
 {
     loader_state_dtor(magic);
-    if (erts_refc_dectest(&magic->refc, 0) == 0) {
-	erts_bin_free(magic);
-    }
+    erts_bin_release(magic);
 }
 
 static ErlHeapFragment* new_literal_fragment(Uint size)
@@ -5672,9 +5670,7 @@ erts_release_literal_area(ErtsLiteralArea* literal_area)
 	Binary* bptr;
 	ASSERT(thing_subtag(oh->thing_word) == REFC_BINARY_SUBTAG);
 	bptr = ((ProcBin*)oh)->val;
-	if (erts_refc_dectest(&bptr->refc, 0) == 0) {
-	    erts_bin_free(bptr);
-	}
+        erts_bin_release(bptr);
 	oh = oh->next;
     }
     erts_free(ERTS_ALC_T_LITERAL, literal_area);
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 49a48ac02e..a1581908e5 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -642,8 +642,7 @@ free_dist_obuf(ErtsDistOutputBuf *obuf)
 {
     Binary *bin = ErtsDistOutputBuf2Binary(obuf);
     ASSERT(obuf->dbg_pattern == ERTS_DIST_OUTPUT_BUF_DBG_PATTERN);
-    if (erts_refc_dectest(&bin->refc, 0) == 0)
-	erts_bin_free(bin);
+    erts_bin_release(bin);
 }
 
 static ERTS_INLINE Sint
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 51072f4b0f..eb3f6dfb3b 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -316,6 +316,7 @@ ERTS_GLB_INLINE Binary *erts_bin_nrml_alloc(Uint size);
 ERTS_GLB_INLINE Binary *erts_bin_realloc_fnf(Binary *bp, Uint size);
 ERTS_GLB_INLINE Binary *erts_bin_realloc(Binary *bp, Uint size);
 ERTS_GLB_INLINE void erts_bin_free(Binary *bp);
+ERTS_GLB_INLINE void erts_bin_release(Binary *bp);
 ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size,
                                                   int (*destructor)(Binary *),
                                                    ErtsAlcType_t alloc_type,
@@ -469,6 +470,14 @@ erts_bin_free(Binary *bp)
 	erts_free(ERTS_ALC_T_BINARY, (void *) bp);
 }
 
+ERTS_GLB_INLINE void
+erts_bin_release(Binary *bp)
+{
+    if (erts_refc_dectest(&bp->refc, 0) == 0) {
+        erts_bin_free(bp);
+    }
+}
+
 ERTS_GLB_INLINE Binary *
 erts_create_magic_binary_x(Uint size, int (*destructor)(Binary *),
                            ErtsAlcType_t alloc_type,
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index de4fa3594d..df3f6ad557 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1521,9 +1521,7 @@ erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit)
 	    pb->flags |= PB_IS_WRITABLE | PB_ACTIVE_WRITER;
 	    pb->val = bptr;
 	    pb->bytes = (byte *) bptr->orig_bytes;
-	    if (erts_refc_dectest(&binp->refc, 0) == 0) {
-		erts_bin_free(binp);
-	    }
+            erts_bin_release(binp);
 	}
     }
     erts_current_bin = pb->bytes;
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 378328856d..e5601e7571 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -402,8 +402,8 @@ free_dbtable(void *vtb)
 #endif
 	ASSERT(is_immed(tb->common.heir_data));
 
-        if (tb->common.btid && erts_refc_dectest(&tb->common.btid->refc, 0) == 0)
-            erts_bin_free(tb->common.btid);
+        if (tb->common.btid)
+            erts_bin_release(tb->common.btid);
 
 	erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));
 }
@@ -3616,9 +3616,7 @@ static SWord proc_cleanup_fixed_table(Process* p, DbFixation* fix)
 	ASSERT(fix->counter == 0);
     }
 
-    if (erts_refc_dectest(&fix->tabs.btid->refc, 0) == 0) {
-	erts_bin_free(fix->tabs.btid);
-    }
+    erts_bin_release(fix->tabs.btid);
     erts_free(ERTS_ALC_T_DB_FIXATION, fix);
     ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation));
     ++work;
@@ -3889,9 +3887,7 @@ static void free_fixations_op(DbFixation* fix, void* vctx)
     {
         fixed_tabs_delete(fix->procs.p, fix);
 
-        if (erts_refc_dectest(&fix->tabs.btid->refc, 0) == 0) {
-            erts_bin_free(fix->tabs.btid);
-        }
+        erts_bin_release(fix->tabs.btid);
 
         erts_db_free(ERTS_ALC_T_DB_FIXATION,
 		     ctx->tb, (void *) fix, sizeof(DbFixation));
@@ -3906,9 +3902,8 @@ int erts_db_execute_free_fixation(Process* p, DbFixation* fix)
     ASSERT(fix->counter == 0);
     fixed_tabs_delete(p, fix);
 
-    if (erts_refc_dectest(&fix->tabs.btid->refc, 0) == 0) {
-        erts_bin_free(fix->tabs.btid);
-    }
+    erts_bin_release(fix->tabs.btid);
+
     erts_free(ERTS_ALC_T_DB_FIXATION, fix);
     ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation));
     return 1;
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 03cc11bdc4..24b22eafb8 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -3265,9 +3265,7 @@ void db_cleanup_offheap_comp(DbTerm* obj)
 	}
 	switch (thing_subtag(u.hdr->thing_word)) {
 	case REFC_BINARY_SUBTAG:
-	    if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) {
-		erts_bin_free(u.pb->val);
-	    }
+            erts_bin_release(u.pb->val);
 	    break;
 	case FUN_SUBTAG:
 	    ASSERT(u.pb != &tmp);
@@ -3277,8 +3275,7 @@ void db_cleanup_offheap_comp(DbTerm* obj)
 	    break;
 	case REF_SUBTAG:
 	    ASSERT(is_magic_ref_thing(u.hdr));
-	    if (erts_refc_dectest(&u.mref->mb->refc, 0) == 0)
-		erts_bin_free((Binary *)u.mref->mb);		    
+            erts_bin_release((Binary *)u.mref->mb);
 	    break;
 	default:
 	    ASSERT(is_external_header(u.hdr->thing_word));
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 50805d9cd9..4398da36d8 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2879,9 +2879,7 @@ sweep_off_heap(Process *p, int fullsweep)
 	    case REFC_BINARY_SUBTAG:
 		{
 		    Binary* bptr = ((ProcBin*)ptr)->val;	
-		    if (erts_refc_dectest(&bptr->refc, 0) == 0) {
-			erts_bin_free(bptr);
-		    }
+                    erts_bin_release(bptr);
 		    break;
 		}
 	    case FUN_SUBTAG:
@@ -2897,8 +2895,7 @@ sweep_off_heap(Process *p, int fullsweep)
 		    ErtsMagicBinary *bptr;
 		    ASSERT(is_magic_ref_thing(ptr));
 		    bptr = ((ErtsMRefThing *) ptr)->mb;
-		    if (erts_refc_dectest(&bptr->refc, 0) == 0)
-			erts_bin_free((Binary *) bptr);		    
+                    erts_bin_release((Binary *) bptr);
 		    break;
 		}
 	    default:
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index f181c1e3cb..17982a2d14 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -167,9 +167,7 @@ erts_cleanup_offheap(ErlOffHeap *offheap)
     for (u.hdr = offheap->first; u.hdr; u.hdr = u.hdr->next) {
 	switch (thing_subtag(u.hdr->thing_word)) {
 	case REFC_BINARY_SUBTAG:
-	    if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) {
-		erts_bin_free(u.pb->val);
-	    }
+            erts_bin_release(u.pb->val);
 	    break;
 	case FUN_SUBTAG:
 	    if (erts_smp_refc_dectest(&u.fun->fe->refc, 0) == 0) {
@@ -178,8 +176,7 @@ erts_cleanup_offheap(ErlOffHeap *offheap)
 	    break;
 	case REF_SUBTAG:
 	    ASSERT(is_magic_ref_thing(u.hdr));
-	    if (erts_refc_dectest(&u.mref->mb->refc, 0) == 0)
-		erts_bin_free((Binary *)u.mref->mb);		    
+            erts_bin_release((Binary *)u.mref->mb);
 	    break;
 	default:
 	    ASSERT(is_external_header(u.hdr->thing_word));
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 94d25ba85a..e76226c5eb 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1112,9 +1112,7 @@ void enif_release_binary(ErlNifBinary* bin)
     if (bin->ref_bin != NULL) {
 	Binary* refbin = bin->ref_bin;
 	ASSERT(bin->bin_term == THE_NON_VALUE);
-	if (erts_refc_dectest(&refbin->refc, 0) == 0) {
-	    erts_bin_free(refbin);
-	}
+        erts_bin_release(refbin);
     }
 #ifdef DEBUG
     bin->data = NULL;
@@ -2347,9 +2345,7 @@ void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref)
         resource->type->down(&msg_env.env, resource->data, &nif_pid, &nif_monitor);
         post_nif_noproc(&msg_env);
 
-        if (erts_refc_dectest(&bin->binary.refc, 0) == 0) {
-            erts_bin_free(&bin->binary);
-        }
+        erts_bin_release(&bin->binary);
     }
     erts_destroy_monitor(rmon);
 }
@@ -2407,9 +2403,7 @@ void enif_release_resource(void* obj)
 #ifdef DEBUG
     erts_refc_dec(&resource->nif_refc, 0);
 #endif
-    if (erts_refc_dectest(&bin->binary.refc, 0) == 0) {
-	erts_bin_free(&bin->binary);
-    }
+    erts_bin_release(&bin->binary);
 }
 
 void enif_keep_resource(void* obj)
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 85b63208a3..870f1f142d 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1999,8 +1999,8 @@ trace_port_receive(Port *t_p, Eterm caller, Eterm what, ...)
                            TRACE_FUN_T_RECEIVE,
                            am_receive, data, THE_NON_VALUE, am_true);
 
-        if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0)
-            erts_bin_free(bptr);
+        if (bptr)
+            erts_bin_release(bptr);
 
         if (orig_hp)
             erts_free(ERTS_ALC_T_TMP, orig_hp);
@@ -2050,8 +2050,8 @@ void trace_port_send_binary(Port *t_p, Eterm to, Eterm what, char *bin, Sint sz)
 
         send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_SEND,
                            am_send, msg, to, am_true);
-        if (bptr && erts_refc_dectest(&bptr->refc, 1) == 0)
-            erts_bin_free(bptr);
+        if (bptr)
+            erts_bin_release(bptr);
 
         UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 #undef LOCAL_HEAP_SIZE
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index c4c848f49f..4d864d1402 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1416,8 +1416,8 @@ do {						\
 
 #define MatchSetUnref(MPSP)					\
 do {								\
-    if (((MPSP) != NULL) && erts_refc_dectest(&(MPSP)->refc, 0) <= 0) { \
-	erts_bin_free(MPSP);					\
+    if (((MPSP) != NULL)) {                                     \
+	erts_bin_release(MPSP);					\
     }								\
 } while(0)
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 36b8ee6860..a1d3f8e29f 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -4557,8 +4557,7 @@ static void
 cleanup_scheduled_control(Binary *binp, char *bufp)
 {
     if (binp) {
-	if (erts_refc_dectest(&binp->refc, 0) == 0)
-	    erts_bin_free(binp);
+        erts_bin_release(binp);
     }
     else {
 	if (bufp)
@@ -6967,8 +6966,7 @@ void driver_free_binary(ErlDrvBinary* dbin)
 	return;
 
     bin = ErlDrvBinary2Binary(dbin);
-    if (erts_refc_dectest(&bin->refc, 0) == 0)
-	erts_bin_free(bin);
+    erts_bin_release(bin);
 }
 
 
-- 
cgit v1.2.3


From e2abc143613c83e00c7c2e22721d30fd2f6309a3 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 11 Apr 2017 17:25:49 +0200
Subject: Fix +SDPcpu

---
 erts/emulator/beam/erl_init.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 56c0f19230..541bfec532 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -1138,8 +1138,12 @@ early_init(int *argc, char **argv) /*
 	}
 	if (dirty_cpu_scheds > schdlrs)
 	    dirty_cpu_scheds = schdlrs;
+        if (dirty_cpu_scheds < 1)
+            dirty_cpu_scheds = 1;
 	if (dirty_cpu_scheds_online > schdlrs_onln)
 	    dirty_cpu_scheds_online = schdlrs_onln;
+	if (dirty_cpu_scheds_online < 1)
+	    dirty_cpu_scheds_online = 1;
 #endif
     }
 
-- 
cgit v1.2.3


From 53f11c147d7c7a1557e12a7c262c2289d89047aa Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 11 Apr 2017 17:50:10 +0200
Subject: Fix multi-scheduling with only one normal scheduler

---
 erts/emulator/beam/bif.c         | 66 +++++++++++++++++++---------------------
 erts/emulator/beam/erl_process.c |  7 +++--
 2 files changed, 35 insertions(+), 38 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 2d37f977c0..214de3652f 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4506,41 +4506,37 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 			 || BIF_ARG_2 == am_block_normal);
 	    int normal = (BIF_ARG_2 == am_block_normal
 			  || BIF_ARG_2 == am_unblock_normal);
-	    if (erts_no_schedulers == 1)
-		BIF_RET(am_disabled);
-	    else {
-		switch (erts_block_multi_scheduling(BIF_P,
-						    ERTS_PROC_LOCK_MAIN,
-						    block,
-						    normal,
-						    0)) {
-		case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED:
-		    BIF_RET(am_blocked);
-		case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED:
-		    BIF_RET(am_blocked_normal);
-		case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED:
-		    ERTS_BIF_YIELD_RETURN_X(BIF_P, am_blocked,
-					    am_multi_scheduling);
-		case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED:
-		    ERTS_BIF_YIELD_RETURN_X(BIF_P, am_blocked_normal,
-					    am_multi_scheduling);
-		case ERTS_SCHDLR_SSPND_DONE:
-		    BIF_RET(am_enabled);
-		case ERTS_SCHDLR_SSPND_YIELD_RESTART:
-		    ERTS_VBUMP_ALL_REDS(BIF_P);
-		    BIF_TRAP2(bif_export[BIF_system_flag_2],
-			      BIF_P, BIF_ARG_1, BIF_ARG_2);
-		case ERTS_SCHDLR_SSPND_YIELD_DONE:
-		    ERTS_BIF_YIELD_RETURN_X(BIF_P, am_enabled,
-					    am_multi_scheduling);
-		case ERTS_SCHDLR_SSPND_EINVAL:
-		    goto error;
-		default:
-		    ASSERT(0);
-		    BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
-		    break;
-		}
-	    }
+            switch (erts_block_multi_scheduling(BIF_P,
+                                                ERTS_PROC_LOCK_MAIN,
+                                                block,
+                                                normal,
+                                                0)) {
+            case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED:
+                BIF_RET(am_blocked);
+            case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED:
+                BIF_RET(am_blocked_normal);
+            case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED:
+                ERTS_BIF_YIELD_RETURN_X(BIF_P, am_blocked,
+                                        am_multi_scheduling);
+            case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED:
+                ERTS_BIF_YIELD_RETURN_X(BIF_P, am_blocked_normal,
+                                        am_multi_scheduling);
+            case ERTS_SCHDLR_SSPND_DONE:
+                BIF_RET(am_enabled);
+            case ERTS_SCHDLR_SSPND_YIELD_RESTART:
+                ERTS_VBUMP_ALL_REDS(BIF_P);
+                BIF_TRAP2(bif_export[BIF_system_flag_2],
+                          BIF_P, BIF_ARG_1, BIF_ARG_2);
+            case ERTS_SCHDLR_SSPND_YIELD_DONE:
+                ERTS_BIF_YIELD_RETURN_X(BIF_P, am_enabled,
+                                        am_multi_scheduling);
+            case ERTS_SCHDLR_SSPND_EINVAL:
+                goto error;
+            default:
+                ASSERT(0);
+                BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
+                break;
+            }
 #endif
 	}
     } else if (BIF_ARG_1 == am_schedulers_online) {
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 88fae30845..f0fd3e3dc9 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -8515,8 +8515,9 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 	    p->flags |= have_blckd_flg;
 	    goto wait_until_msb;
 	}
-	else if (msbp->blckrs) {
-	    ASSERT(msbp->ongoing);
+	else if (msbp->blckrs || (normal && erts_no_schedulers == 1)) {
+	    ASSERT(!msbp->blckrs || msbp->ongoing);
+	    msbp->ongoing = 1;
 	    plp = proclist_create(p);
 	    erts_proclist_store_last(&msbp->blckrs, plp);
 	    p->flags |= have_blckd_flg;
@@ -8530,7 +8531,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 	    else
 		res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED;
 	}
-	else {
+        else {
 	    int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
 							ERTS_SCHED_NORMAL);
 	    ASSERT(!msbp->ongoing);
-- 
cgit v1.2.3


From b5cd902c38687b9578d2222ace7956fdb26c9401 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 1 Mar 2017 17:05:53 +0100
Subject: Fix dirty GC implementation

---
 erts/emulator/beam/erl_gc.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 50805d9cd9..d51d4fff45 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -479,9 +479,15 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int
 	p->live_hf_end = live_hf_end;
     }
 
-    if (need == 0)
+    if (need == 0) {
+#ifdef ERTS_DIRTY_SCHEDULERS
+        if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) {
+            ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p)));
+            goto force_reschedule;
+        }
+#endif
 	return 1;
-
+    }
     /*
      * Satisfy need in a heap fragment...
      */
@@ -534,6 +540,10 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int
     p->heap_hfrag = hfrag;
 #endif
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+force_reschedule:
+#endif
+
     /* Make sure that we do a proper GC as soon as possible... */
     p->flags |= F_FORCE_GC;
     reds_left = ERTS_REDS_LEFT(p, fcalls);
-- 
cgit v1.2.3


From f0c4d095e0d9be054181785f5c9ca34b52aa2995 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 11 Apr 2017 19:15:05 +0200
Subject: Suggested stack size options for dirty schedulers

---
 erts/emulator/beam/erl_init.c    | 72 ++++++++++++++++++++++++++++++++++++----
 erts/emulator/beam/erl_process.c |  7 +++-
 erts/emulator/beam/erl_process.h |  6 +++-
 3 files changed, 77 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 56c0f19230..9fe8a7d361 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -61,8 +61,9 @@
 
 #define ERTS_DEFAULT_NO_ASYNC_THREADS	10
 
-#define ERTS_DEFAULT_SCHED_STACK_SIZE   256
-#define ERTS_MIN_SCHED_STACK_SIZE       20
+#define ERTS_DEFAULT_SCHED_STACK_SIZE   128
+#define ERTS_DEFAULT_DCPU_SCHED_STACK_SIZE 40
+#define ERTS_DEFAULT_DIO_SCHED_STACK_SIZE 40
 
 /*
  * The variables below (prefixed with etp_) are for erts/etc/unix/etp-commands
@@ -641,9 +642,22 @@ void erts_usage(void)
     erts_fprintf(stderr, "-swt val       set scheduler wakeup threshold, valid values are:\n");
     erts_fprintf(stderr, "               very_low|low|medium|high|very_high.\n");
     erts_fprintf(stderr, "-sss size      suggested stack size in kilo words for scheduler threads,\n");
-    erts_fprintf(stderr, "               valid range is [%d-%d]\n",
+    erts_fprintf(stderr, "               valid range is [%d-%d] (default %d)\n",
+		 ERTS_SCHED_THREAD_MIN_STACK_SIZE,
+		 ERTS_SCHED_THREAD_MAX_STACK_SIZE,
+                 ERTS_DEFAULT_SCHED_STACK_SIZE);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_fprintf(stderr, "-sssdcpu size  suggested stack size in kilo words for dirty CPU scheduler\n");
+    erts_fprintf(stderr, "               threads, valid range is [%d-%d] (default %d)\n",
+		 ERTS_SCHED_THREAD_MIN_STACK_SIZE,
+		 ERTS_SCHED_THREAD_MAX_STACK_SIZE,
+                 ERTS_DEFAULT_DCPU_SCHED_STACK_SIZE);
+    erts_fprintf(stderr, "-sssdio size   suggested stack size in kilo words for dirty IO scheduler\n");
+    erts_fprintf(stderr, "               threads, valid range is [%d-%d] (default %d)\n",
 		 ERTS_SCHED_THREAD_MIN_STACK_SIZE,
-		 ERTS_SCHED_THREAD_MAX_STACK_SIZE);
+		 ERTS_SCHED_THREAD_MAX_STACK_SIZE,
+                 ERTS_DEFAULT_DIO_SCHED_STACK_SIZE);
+#endif
     erts_fprintf(stderr, "-spp Bool      set port parallelism scheduling hint\n");
     erts_fprintf(stderr, "-S n1:n2       set number of schedulers (n1), and number of\n");
     erts_fprintf(stderr, "               schedulers online (n2), maximum for both\n");
@@ -1323,6 +1337,10 @@ erl_start(int argc, char **argv)
      * a lot of stack.
      */
     erts_sched_thread_suggested_stack_size = ERTS_DEFAULT_SCHED_STACK_SIZE;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_dcpu_sched_thread_suggested_stack_size = ERTS_DEFAULT_DCPU_SCHED_STACK_SIZE;
+    erts_dio_sched_thread_suggested_stack_size = ERTS_DEFAULT_DIO_SCHED_STACK_SIZE;
+#endif
 
 #ifdef DEBUG
     verbose = DEBUG_DEFAULT;
@@ -1941,6 +1959,42 @@ erl_start(int argc, char **argv)
 		VERBOSE(DEBUG_SYSTEM,
 			("scheduler wakeup threshold: %s\n", arg));
 	    }
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    else if (has_prefix("ssdcpu", sub_param)) {
+		/* suggested stack size (Kilo Words) for dirty CPU scheduler threads */
+		arg = get_arg(sub_param+6, argv[i+1], &i);
+		erts_dcpu_sched_thread_suggested_stack_size = atoi(arg);
+
+		if ((erts_dcpu_sched_thread_suggested_stack_size
+		     < ERTS_SCHED_THREAD_MIN_STACK_SIZE)
+		    || (erts_dcpu_sched_thread_suggested_stack_size >
+			ERTS_SCHED_THREAD_MAX_STACK_SIZE)) {
+		    erts_fprintf(stderr, "bad stack size for dirty CPU scheduler threads %s\n",
+				 arg);
+		    erts_usage();
+		}
+		VERBOSE(DEBUG_SYSTEM,
+			("suggested dirty CPU scheduler thread stack size %d kilo words\n",
+			 erts_dcpu_sched_thread_suggested_stack_size));
+	    }
+	    else if (has_prefix("ssdio", sub_param)) {
+		/* suggested stack size (Kilo Words) for dirty IO scheduler threads */
+		arg = get_arg(sub_param+5, argv[i+1], &i);
+		erts_dio_sched_thread_suggested_stack_size = atoi(arg);
+
+		if ((erts_dio_sched_thread_suggested_stack_size
+		     < ERTS_SCHED_THREAD_MIN_STACK_SIZE)
+		    || (erts_dio_sched_thread_suggested_stack_size >
+			ERTS_SCHED_THREAD_MAX_STACK_SIZE)) {
+		    erts_fprintf(stderr, "bad stack size for dirty IO scheduler threads %s\n",
+				 arg);
+		    erts_usage();
+		}
+		VERBOSE(DEBUG_SYSTEM,
+			("suggested dirty IO scheduler thread stack size %d kilo words\n",
+			 erts_dio_sched_thread_suggested_stack_size));
+	    }
+#endif
 	    else if (has_prefix("ss", sub_param)) {
 		/* suggested stack size (Kilo Words) for scheduler threads */
 		arg = get_arg(sub_param+2, argv[i+1], &i);
@@ -2255,8 +2309,14 @@ erl_start(int argc, char **argv)
     boot_argc = argc - i;  /* Number of arguments to init */
     boot_argv = &argv[i];
 
-    if (erts_sched_thread_suggested_stack_size < ERTS_MIN_SCHED_STACK_SIZE)
-        erts_sched_thread_suggested_stack_size = ERTS_MIN_SCHED_STACK_SIZE;
+    if (erts_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE)
+        erts_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (erts_dcpu_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE)
+        erts_dcpu_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE;
+    if (erts_dio_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE)
+        erts_dio_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE;
+#endif
 
     erl_init(ncpu,
 	     proc_tab_sz,
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 11a1ce625b..2d80617445 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -194,7 +194,10 @@ static UWord thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_O
 ErtsPTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE);
 
 int erts_sched_thread_suggested_stack_size = -1;
-
+#ifdef ERTS_DIRTY_SCHEDULERS
+int erts_dcpu_sched_thread_suggested_stack_size = -1;
+int erts_dio_sched_thread_suggested_stack_size = -1;
+#endif
 #ifdef ERTS_ENABLE_LOCK_CHECK
 ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];
 #endif
@@ -8976,6 +8979,7 @@ erts_start_schedulers(void)
 	for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
 	    ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
 	    erts_snprintf(opts.name, 16, "%d_dirty_cpu_scheduler", ix + 1);
+            opts.suggested_stack_size = erts_dcpu_sched_thread_suggested_stack_size;
 	    res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts);
 	    if (res != 0)
 		erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d\n", ix);
@@ -8983,6 +8987,7 @@ erts_start_schedulers(void)
 	for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
 	    ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
 	    erts_snprintf(opts.name, 16, "%d_dirty_io_scheduler", ix + 1);
+            opts.suggested_stack_size = erts_dio_sched_thread_suggested_stack_size;
 	    res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts);
 	    if (res != 0)
 		erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d\n", ix);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 883d9f2a4c..408e9c4c45 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -117,7 +117,11 @@ extern Uint erts_no_dirty_io_schedulers;
 #endif
 extern Uint erts_no_run_queues;
 extern int erts_sched_thread_suggested_stack_size;
-#define ERTS_SCHED_THREAD_MIN_STACK_SIZE 4	/* Kilo words */
+#ifdef ERTS_DIRTY_SCHEDULERS
+extern int erts_dcpu_sched_thread_suggested_stack_size;
+extern int erts_dio_sched_thread_suggested_stack_size;
+#endif
+#define ERTS_SCHED_THREAD_MIN_STACK_SIZE 20	/* Kilo words */
 #define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */
 
 #ifdef ERTS_SMP
-- 
cgit v1.2.3


From 56c4aee677f305f2ed9ca877a39e9c3c4f266f4b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 12 Apr 2017 19:50:01 +0200
Subject: erts: Introduce struct binary_internals

to replace macro ERTS_INTERNAL_BINARY_FIELDS
as header in Binary and friends.
---
 erts/emulator/beam/beam_bif_load.c    |  4 +--
 erts/emulator/beam/beam_load.c        |  2 +-
 erts/emulator/beam/break.c            |  2 +-
 erts/emulator/beam/copy.c             | 16 +++++------
 erts/emulator/beam/erl_bif_info.c     |  6 ++---
 erts/emulator/beam/erl_bif_unique.c   |  2 +-
 erts/emulator/beam/erl_bif_unique.h   | 10 +++----
 erts/emulator/beam/erl_binary.h       | 50 ++++++++++++++++++-----------------
 erts/emulator/beam/erl_db.c           |  6 ++---
 erts/emulator/beam/erl_db_hash.c      |  4 +--
 erts/emulator/beam/erl_db_tree.c      | 12 ++++-----
 erts/emulator/beam/erl_db_util.h      |  6 ++---
 erts/emulator/beam/erl_gc.c           |  6 ++---
 erts/emulator/beam/erl_nif.c          | 16 +++++------
 erts/emulator/beam/erl_process_dump.c |  6 ++---
 erts/emulator/beam/external.c         | 22 +++++++--------
 erts/emulator/beam/global.h           |  2 +-
 erts/emulator/beam/io.c               |  8 +++---
 erts/emulator/beam/utils.c            |  2 +-
 erts/emulator/hipe/hipe_bif0.c        |  2 +-
 erts/emulator/hipe/hipe_load.c        |  2 +-
 21 files changed, 94 insertions(+), 92 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 4ba8c2a669..769fe89219 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -233,7 +233,7 @@ prepare_loading_2(BIF_ALIST_2)
     }
     hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
     res = erts_mk_magic_ref(&hp, &MSO(BIF_P), magic);
-    erts_refc_dec(&magic->refc, 1);
+    erts_refc_dec(&magic->intern.refc, 1);
     BIF_RET(res);
 }
 
@@ -435,7 +435,7 @@ finish_loading_1(BIF_ALIST_1)
 	    Eterm mod;
 	    Eterm retval;
 
-	    erts_refc_inc(&p[i].code->refc, 1);
+	    erts_refc_inc(&p[i].code->intern.refc, 1);
 	    retval = erts_finish_loading(p[i].code, BIF_P, 0, &mod);
 	    ASSERT(retval == NIL || retval == am_on_load);
 	    if (retval == am_on_load) {
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 3a3d1f57ce..7b79e6303b 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -913,7 +913,7 @@ erts_alloc_loader_state(void)
 
     magic = erts_create_magic_binary(sizeof(LoaderState),
 				     loader_state_dtor);
-    erts_refc_inc(&magic->refc, 1);
+    erts_refc_inc(&magic->intern.refc, 1);
     stp = ERTS_MAGIC_BIN_DATA(magic);
     stp->bin = NULL;
     stp->function = THE_NON_VALUE; /* Function not known yet */
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 0b40d70cb7..4703fb0933 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -660,7 +660,7 @@ bin_check(void)
 		erts_printf("%p orig_size: %bpd, norefs = %bpd\n",
 			    bp->val, 
 			    bp->val->orig_size, 
-			    erts_refc_read(&bp->val->refc, 1));
+			    erts_refc_read(&bp->val->intern.refc, 1));
 	    }
 	}
 	if (printed) {
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 264ba89e8b..62c3f9520d 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -761,7 +761,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 		    }
 		    *argp = make_binary(hbot);
 		    pb = (ProcBin*) hbot;
-		    erts_refc_inc(&pb->val->refc, 2);
+		    erts_refc_inc(&pb->val->intern.refc, 2);
 		    pb->next = off_heap->first;
 		    pb->flags = 0;
 		    off_heap->first = (struct erl_off_heap_header*) pb;
@@ -809,7 +809,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 			to->thing_word = HEADER_PROC_BIN;
 			to->size = real_size;
 			to->val = from->val;
-			erts_refc_inc(&to->val->refc, 2);
+			erts_refc_inc(&to->val->intern.refc, 2);
 			to->bytes = from->bytes + offset;
 			to->next = off_heap->first;
 			to->flags = 0;
@@ -901,7 +901,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 	    case REF_SUBTAG:
 		if (is_magic_ref_thing(objp)) {
 		    ErtsMRefThing *mreft = (ErtsMRefThing *) objp;
-		    erts_refc_inc(&mreft->mb->refc, 2);
+		    erts_refc_inc(&mreft->mb->intern.refc, 2);
 		    goto L_off_heap_node_container_common;
 		}
 		/* Fall through... */
@@ -1585,7 +1585,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 		while (sz-- > 0) {
 		    *hp++ = *ptr++;
 		}
-		erts_refc_inc(&pb->val->refc, 2);
+		erts_refc_inc(&pb->val->intern.refc, 2);
 		pb->next = off_heap->first;
 		pb->flags = 0;
 		off_heap->first = (struct erl_off_heap_header*) pb;
@@ -1644,7 +1644,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 		    to->thing_word = HEADER_PROC_BIN;
 		    to->size = real_size;
 		    to->val = from->val;
-		    erts_refc_inc(&to->val->refc, 2);
+		    erts_refc_inc(&to->val->intern.refc, 2);
 		    to->bytes = from->bytes + offset;
 		    to->next = off_heap->first;
 		    to->flags = 0;
@@ -1678,7 +1678,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 	    case REF_SUBTAG:
 		if (is_magic_ref_thing(ptr)) {
 		    ErtsMRefThing *mreft = (ErtsMRefThing *) ptr;
-		    erts_refc_inc(&mreft->mb->refc, 2);
+		    erts_refc_inc(&mreft->mb->intern.refc, 2);
 		    goto off_heap_node_container_common;
 		}
 		/* Fall through... */
@@ -1847,7 +1847,7 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    case REFC_BINARY_SUBTAG:
 		{
 		    ProcBin* pb = (ProcBin *) (tp-1);
-		    erts_refc_inc(&pb->val->refc, 2);
+		    erts_refc_inc(&pb->val->intern.refc, 2);
 		    OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
 		}
 		goto off_heap_common;
@@ -1882,7 +1882,7 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		ErtsRefThing *rtp = (ErtsRefThing *) (tp - 1);
 		if (is_magic_ref_thing(rtp)) {
 		    ErtsMRefThing *mreft = (ErtsMRefThing *) rtp;
-		    erts_refc_inc(&mreft->mb->refc, 2);
+		    erts_refc_inc(&mreft->mb->intern.refc, 2);
 		    goto off_heap_common;
 		}
 		/* Fall through... */
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 025f99330a..6d10ced87a 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -177,7 +177,7 @@ bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh)
 	    if (szp)
 		*szp += 4+2;
 	    if (hpp) {
-		Uint refc = (Uint) erts_refc_read(&pb->val->refc, 1);
+		Uint refc = (Uint) erts_refc_read(&pb->val->intern.refc, 1);
 		tuple = TUPLE3(*hpp, val, orig_size, make_small(refc));
 		res = CONS(*hpp + 4, tuple, res);
 		*hpp += 4+2;
@@ -203,7 +203,7 @@ bld_magic_ref_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh)
 	    if (szp)
 		*szp += 4+2;
 	    if (hpp) {
-		Uint refc = (Uint) erts_refc_read(&mrtp->mb->refc, 1);
+		Uint refc = (Uint) erts_refc_read(&mrtp->mb->intern.refc, 1);
 		tuple = TUPLE3(*hpp, val, orig_size, make_small(refc));
 		res = CONS(*hpp + 4, tuple, res);
 		*hpp += 4+2;
@@ -3969,7 +3969,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
                     BIF_RET(am_false);
                 }
                 bin = erts_magic_ref2bin(tp[2]);
-                refc = erts_refc_read(&bin->refc, 1);
+                refc = erts_refc_read(&bin->intern.refc, 1);
                 bin_addr = (UWord) bin;
                 sz = 4;
                 erts_bld_uword(NULL, &sz, bin_addr);
diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c
index 3fd4c87094..7f438daec0 100644
--- a/erts/emulator/beam/erl_bif_unique.c
+++ b/erts/emulator/beam/erl_bif_unique.c
@@ -197,7 +197,7 @@ erts_magic_ref_lookup_bin__(Uint32 refn[ERTS_REF_NUMBERS])
     else {
 	erts_aint_t refc;
 	mb = tep->mb;
-        refc = erts_refc_inc_unless(&mb->refc, 0, 0);
+        refc = erts_refc_inc_unless(&mb->intern.refc, 0, 0);
         if (refc == 0)
             mb = NULL;
     }
diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h
index 27c2a15a5e..0f3f794878 100644
--- a/erts/emulator/beam/erl_bif_unique.h
+++ b/erts/emulator/beam/erl_bif_unique.h
@@ -178,10 +178,10 @@ ERTS_GLB_INLINE Eterm
 erts_mk_magic_ref(Eterm **hpp, ErlOffHeap *ohp, Binary *bp)
 {
     Eterm *hp = *hpp;
-    ASSERT(bp->flags & BIN_FLAG_MAGIC);
+    ASSERT(bp->intern.flags & BIN_FLAG_MAGIC);
     write_magic_ref_thing(hp, ohp, (ErtsMagicBinary *) bp);
     *hpp += ERTS_MAGIC_REF_THING_SIZE;
-    erts_refc_inc(&bp->refc, 1);
+    erts_refc_inc(&bp->intern.refc, 1);
     OH_OVERHEAD(ohp, bp->orig_size / sizeof(Eterm));
     return make_internal_ref(hp);
 }
@@ -297,14 +297,14 @@ erts_iref_storage_save(ErtsIRefStorage *iref, Eterm ref)
 	ASSERT(is_magic_ref_thing(hp));
 	iref->is_magic = 1;
 	iref->u.mb = mrtp->mb;
-	erts_refc_inc(&mrtp->mb->refc, 1);
+	erts_refc_inc(&mrtp->mb->intern.refc, 1);
     }
 }
 
 ERTS_GLB_INLINE void
 erts_iref_storage_clean(ErtsIRefStorage *iref)
 {
-    if (iref->is_magic && erts_refc_dectest(&iref->u.mb->refc, 0) == 0)
+    if (iref->is_magic && erts_refc_dectest(&iref->u.mb->intern.refc, 0) == 0)
 	erts_ref_bin_free(iref->u.mb);
 #ifdef DEBUG
     memset((void *) iref, 0xf, sizeof(ErtsIRefStorage));
@@ -337,7 +337,7 @@ erts_iref_storage_make_ref(ErtsIRefStorage *iref,
 	 * refc increment of the cleaned storage...
 	 */
 	if (!clean_storage)
-	    erts_refc_inc(&iref->u.mb->refc, 1);
+	    erts_refc_inc(&iref->u.mb->intern.refc, 1);
     }
 
 #ifdef DEBUG
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index eb3f6dfb3b..de7dbf4e20 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -42,14 +42,16 @@
 #define ERTS_BINARY_STRUCT_ALIGNMENT
 #endif
 
-/* Add fields in ERTS_INTERNAL_BINARY_FIELDS, otherwise the drivers crash */
-#define ERTS_INTERNAL_BINARY_FIELDS				\
-    UWord flags;						\
-    erts_refc_t refc;						\
+/* Add fields in binary_internals, otherwise the drivers crash */
+struct binary_internals {
+    UWord flags;
+    erts_refc_t refc;
     ERTS_BINARY_STRUCT_ALIGNMENT
+};
+
 
 typedef struct binary {
-    ERTS_INTERNAL_BINARY_FIELDS
+    struct binary_internals intern;
     SWord orig_size;
     char orig_bytes[1]; /* to be continued */
 } Binary;
@@ -63,7 +65,7 @@ typedef struct binary {
 
 typedef struct magic_binary ErtsMagicBinary;
 struct magic_binary {
-    ERTS_INTERNAL_BINARY_FIELDS
+    struct binary_internals intern;
     SWord orig_size;
     int (*destructor)(Binary *);
     Uint32 refn[ERTS_REF_NUMBERS];
@@ -87,7 +89,7 @@ typedef union {
     Binary binary;
     ErtsMagicBinary magic_binary;
     struct {
-	ERTS_INTERNAL_BINARY_FIELDS
+	struct binary_internals intern;
 	ErlDrvBinary binary;
     } driver;
 } ErtsBinary;
@@ -375,8 +377,8 @@ erts_bin_drv_alloc_fnf(Uint size)
     ERTS_CHK_BIN_ALIGNMENT(res);
     if (res) {
 	res->orig_size = size;
-	res->flags = BIN_FLAG_DRV;
-        erts_refc_init(&res->refc, 1);
+	res->intern.flags = BIN_FLAG_DRV;
+        erts_refc_init(&res->intern.refc, 1);
     }
     return res;
 }
@@ -394,8 +396,8 @@ erts_bin_drv_alloc(Uint size)
     res = erts_alloc(ERTS_ALC_T_DRV_BINARY, bsize);
     ERTS_CHK_BIN_ALIGNMENT(res);
     res->orig_size = size;
-    res->flags = BIN_FLAG_DRV;
-    erts_refc_init(&res->refc, 1);
+    res->intern.flags = BIN_FLAG_DRV;
+    erts_refc_init(&res->intern.refc, 1);
     return res;
 }
 
@@ -413,8 +415,8 @@ erts_bin_nrml_alloc(Uint size)
     res = erts_alloc(ERTS_ALC_T_BINARY, bsize);
     ERTS_CHK_BIN_ALIGNMENT(res);
     res->orig_size = size;
-    res->flags = 0;
-    erts_refc_init(&res->refc, 1);
+    res->intern.flags = 0;
+    erts_refc_init(&res->intern.refc, 1);
     return res;
 }
 
@@ -423,9 +425,9 @@ erts_bin_realloc_fnf(Binary *bp, Uint size)
 {
     Binary *nbp;
     Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
-    ErtsAlcType_t type = (bp->flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY
+    ErtsAlcType_t type = (bp->intern.flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY
 	                                            : ERTS_ALC_T_BINARY;
-    ASSERT((bp->flags & BIN_FLAG_MAGIC) == 0);
+    ASSERT((bp->intern.flags & BIN_FLAG_MAGIC) == 0);
     if (bsize < size) /* overflow */
 	return NULL;
     nbp = erts_realloc_fnf(type, (void *) bp, bsize);
@@ -440,9 +442,9 @@ erts_bin_realloc(Binary *bp, Uint size)
 {
     Binary *nbp;
     Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
-    ErtsAlcType_t type = (bp->flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY
+    ErtsAlcType_t type = (bp->intern.flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY
 	                                            : ERTS_ALC_T_BINARY;
-    ASSERT((bp->flags & BIN_FLAG_MAGIC) == 0);
+    ASSERT((bp->intern.flags & BIN_FLAG_MAGIC) == 0);
     if (bsize < size) /* overflow */
 	erts_realloc_enomem(type, bp, size);
     nbp = erts_realloc_fnf(type, (void *) bp, bsize);
@@ -456,7 +458,7 @@ erts_bin_realloc(Binary *bp, Uint size)
 ERTS_GLB_INLINE void
 erts_bin_free(Binary *bp)
 {
-    if (bp->flags & BIN_FLAG_MAGIC) {
+    if (bp->intern.flags & BIN_FLAG_MAGIC) {
         if (!ERTS_MAGIC_BIN_DESTRUCTOR(bp)(bp)) {
             /* Destructor took control of the deallocation */
             return;
@@ -464,7 +466,7 @@ erts_bin_free(Binary *bp)
 	erts_magic_ref_remove_bin(ERTS_MAGIC_BIN_REFN(bp));
         erts_free(ERTS_MAGIC_BIN_ATYPE(bp), (void *) bp);
     }
-    else if (bp->flags & BIN_FLAG_DRV)
+    else if (bp->intern.flags & BIN_FLAG_DRV)
 	erts_free(ERTS_ALC_T_DRV_BINARY, (void *) bp);
     else
 	erts_free(ERTS_ALC_T_BINARY, (void *) bp);
@@ -473,7 +475,7 @@ erts_bin_free(Binary *bp)
 ERTS_GLB_INLINE void
 erts_bin_release(Binary *bp)
 {
-    if (erts_refc_dectest(&bp->refc, 0) == 0) {
+    if (erts_refc_dectest(&bp->intern.refc, 0) == 0) {
         erts_bin_free(bp);
     }
 }
@@ -490,10 +492,10 @@ erts_create_magic_binary_x(Uint size, int (*destructor)(Binary *),
     if (!bptr)
 	erts_alloc_n_enomem(ERTS_ALC_T2N(alloc_type), bsize);
     ERTS_CHK_BIN_ALIGNMENT(bptr);
-    bptr->flags = BIN_FLAG_MAGIC;
+    bptr->intern.flags = BIN_FLAG_MAGIC;
     bptr->orig_size = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(size)
                                 : ERTS_MAGIC_BIN_ORIG_SIZE(size);
-    erts_refc_init(&bptr->refc, 0);
+    erts_refc_init(&bptr->intern.refc, 0);
     ERTS_MAGIC_BIN_DESTRUCTOR(bptr) = destructor;
     ERTS_MAGIC_BIN_ATYPE(bptr) = alloc_type;
     erts_make_magic_ref_in_array(ERTS_MAGIC_BIN_REFN(bptr));
@@ -521,7 +523,7 @@ ERTS_GLB_INLINE erts_smp_atomic_t *
 erts_smp_binary_to_magic_indirection(Binary *bp)
 {
     ErtsMagicIndirectionWord *mip;
-    ASSERT(bp->flags & BIN_FLAG_MAGIC);
+    ASSERT(bp->intern.flags & BIN_FLAG_MAGIC);
     ASSERT(ERTS_MAGIC_BIN_ATYPE(bp) == ERTS_ALC_T_MINDIRECTION);
     mip = ERTS_MAGIC_BIN_UNALIGNED_DATA(bp);
     return &mip->smp_atomic_word;
@@ -531,7 +533,7 @@ ERTS_GLB_INLINE erts_atomic_t *
 erts_binary_to_magic_indirection(Binary *bp)
 {
     ErtsMagicIndirectionWord *mip;
-    ASSERT(bp->flags & BIN_FLAG_MAGIC);
+    ASSERT(bp->intern.flags & BIN_FLAG_MAGIC);
     ASSERT(ERTS_MAGIC_BIN_ATYPE(bp) == ERTS_ALC_T_MINDIRECTION);
     mip = ERTS_MAGIC_BIN_UNALIGNED_DATA(bp);
     return &mip->atomic_word;
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index e5601e7571..98c689f13f 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -217,7 +217,7 @@ make_btid(DbTable *tb)
      * and table is refered once by being alive...
      */
     erts_smp_refc_init(&tb->common.refc, 2);
-    erts_refc_inc(&btid->refc, 1);
+    erts_refc_inc(&btid->intern.refc, 1);
 }
 
 static ERTS_INLINE DbTable* btid2tab(Binary* btid)
@@ -3781,7 +3781,7 @@ static void fix_table_locked(Process* p, DbTable* tb)
 				       tb, sizeof(DbFixation));
     ERTS_ETS_MISC_MEM_ADD(sizeof(DbFixation));
     fix->tabs.btid = tb->common.btid;
-    erts_refc_inc(&fix->tabs.btid->refc, 2);
+    erts_refc_inc(&fix->tabs.btid->intern.refc, 2);
     fix->procs.p = p;
     fix->counter = 1;
     fixing_procs_rbt_insert(&tb->common.fixing_procs, fix);
@@ -3817,7 +3817,7 @@ static void unfix_table_locked(Process* p,  DbTable* tb,
 #endif
 	    fixed_tabs_delete(p, fix);
 
-	    erts_refc_dec(&fix->tabs.btid->refc, 1);
+	    erts_refc_dec(&fix->tabs.btid->intern.refc, 1);
 
 	    erts_db_free(ERTS_ALC_T_DB_FIXATION,
 			 tb, (void *) fix, sizeof(DbFixation));
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 7ab27df00c..80c4824eeb 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1258,7 +1258,7 @@ static int match_traverse(Process* p, DbTableHash* tb,
     }
 
     if (mpi.all_objects) {
-        mpi.mp->flags |= BIN_FLAG_ALL_OBJECTS;
+        mpi.mp->intern.flags |= BIN_FLAG_ALL_OBJECTS;
     }
 
     /*
@@ -1383,7 +1383,7 @@ static int match_traverse_continue(Process* p, DbTableHash* tb,
                                    void* context_ptr, /* For callbacks */
                                    Eterm* ret)
 {
-    int all_objects = (*mpp)->flags & BIN_FLAG_ALL_OBJECTS;
+    int all_objects = (*mpp)->intern.flags & BIN_FLAG_ALL_OBJECTS;
     HashDbTerm** current_ptr;  /* Refers to either the bucket pointer or
                                        * the 'next' pointer in the previous term
                                        */
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index ab8da6ccf6..f6918b8ec4 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -995,7 +995,7 @@ static int db_select_continue_tree(Process *p,
     sc.lastobj = NULL;
     sc.max = 1000;
     sc.keypos = tb->common.keypos;
-    sc.all_objects = mp->flags & BIN_FLAG_ALL_OBJECTS;
+    sc.all_objects = mp->intern.flags & BIN_FLAG_ALL_OBJECTS;
     sc.chunk_size = chunk_size;
     reverse = unsigned_val(tptr[7]);
     sc.got = signed_val(tptr[8]);
@@ -1187,7 +1187,7 @@ static int db_select_tree(Process *p, DbTable *tbl, Eterm tid,
     hp = HAlloc(p, 9 + sz + ERTS_MAGIC_REF_THING_SIZE);
     key = copy_struct(key, sz, &hp, &MSO(p));
     if (mpi.all_objects)
-	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
+	(mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS;
     mpb= erts_db_make_match_prog_ref(p,mpi.mp,&hp);
 	    
     continuation = TUPLE8
@@ -1385,7 +1385,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, Eterm tid,
     }
     key = copy_struct(key, sz, &hp, &MSO(p));
     if (mpi.all_objects)
-	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
+	(mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS;
     mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
 	    
     continuation = TUPLE5
@@ -1510,7 +1510,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid,
 	hp = HAlloc(p, 9 + sz + ERTS_MAGIC_REF_THING_SIZE);
 	key = copy_struct(key, sz, &hp, &MSO(p));
 	if (mpi.all_objects)
-	    (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
+	    (mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS;
 	mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
 	
 	continuation = TUPLE8
@@ -1536,7 +1536,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid,
     key = copy_struct(key, sz, &hp, &MSO(p));
 
     if (mpi.all_objects)
-	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
+	(mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS;
     mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
     continuation = TUPLE8
 	(hp,
@@ -1932,7 +1932,7 @@ static int db_select_replace_tree(Process *p, DbTable *tbl, Eterm tid,
     }
     key = copy_struct(key, sz, &hp, &MSO(p));
     if (mpi.all_objects)
-        (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
+        (mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS;
     mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
 
     continuation = TUPLE5
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 9be77fcefa..ed7b9c8618 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -504,7 +504,7 @@ ERTS_GLB_INLINE Binary *
 erts_db_get_match_prog_binary_unchecked(Eterm term)
 {
     Binary *bp = erts_magic_ref2bin(term);
-    ASSERT(bp->flags & BIN_FLAG_MAGIC);
+    ASSERT(bp->intern.flags & BIN_FLAG_MAGIC);
     ASSERT((ERTS_MAGIC_BIN_DESTRUCTOR(bp) == erts_db_match_prog_destructor));
     return bp;
 }
@@ -516,7 +516,7 @@ erts_db_get_match_prog_binary(Eterm term)
     if (!is_internal_magic_ref(term))
 	return NULL;
     bp = erts_magic_ref2bin(term);
-    ASSERT(bp->flags & BIN_FLAG_MAGIC);
+    ASSERT(bp->intern.flags & BIN_FLAG_MAGIC);
     if (ERTS_MAGIC_BIN_DESTRUCTOR(bp) != erts_db_match_prog_destructor)
 	return NULL;
     return bp;
@@ -528,7 +528,7 @@ erts_db_get_match_prog_binary(Eterm term)
 ** Convenience when compiling into Binary structures
 */
 #define IsMatchProgBinary(BP) \
-  (((BP)->flags & BIN_FLAG_MAGIC) \
+  (((BP)->intern.flags & BIN_FLAG_MAGIC) \
    && ERTS_MAGIC_BIN_DESTRUCTOR((BP)) == erts_db_match_prog_destructor)
 
 #define Binary2MatchProg(BP) \
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 4398da36d8..8605b8a22a 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1242,7 +1242,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
 	     * link it into the MSO list for the process.
 	     */
 
-	    erts_refc_inc(&bptr->refc, 1);
+	    erts_refc_inc(&bptr->intern.refc, 1);
 	    *prev = ptr;
 	    prev = &ptr->next;
 	}
@@ -3596,7 +3596,7 @@ erts_check_off_heap2(Process *p, Eterm *htop)
 	erts_aint_t refc;
 	switch (thing_subtag(u.hdr->thing_word)) {
 	case REFC_BINARY_SUBTAG:
-	    refc = erts_refc_read(&u.pb->val->refc, 1);		
+	    refc = erts_refc_read(&u.pb->val->intern.refc, 1);
 	    break;
 	case FUN_SUBTAG:
 	    refc = erts_smp_refc_read(&u.fun->fe->refc, 1);
@@ -3608,7 +3608,7 @@ erts_check_off_heap2(Process *p, Eterm *htop)
 	    break;
 	case REF_SUBTAG:
 	    ASSERT(is_magic_ref_thing(u.hdr));
-	    refc = erts_refc_read(&u.mref->mb->refc, 1);
+	    refc = erts_refc_read(&u.mref->mb->intern.refc, 1);
 	    break;
 	default:
 	    ASSERT(!"erts_check_off_heap2: Invalid thing_word");
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index e76226c5eb..635b2ab90d 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1276,7 +1276,7 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin)
 	
 	OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm));
 	bin_term = make_binary(pb);	
-	if (erts_refc_read(&bptr->refc, 1) == 1) {
+	if (erts_refc_read(&bptr->intern.refc, 1) == 1) {
 	    /* Total ownership transfer */
 	    bin->ref_bin = NULL;
 	    bin->bin_term = bin_term;
@@ -2246,7 +2246,7 @@ static int nif_resource_dtor(Binary* bin)
 
         ASSERT(type->down);
         erts_smp_mtx_lock(&rm->lock);
-        ASSERT(erts_refc_read(&bin->refc, 0) == 0);
+        ASSERT(erts_refc_read(&bin->intern.refc, 0) == 0);
         if (rm->root) {
             ASSERT(!rm->is_dying);
             destroy_all_monitors(rm->root, resource);
@@ -2320,13 +2320,13 @@ void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref)
         erts_smp_mtx_unlock(&rmp->lock);
 
         if (free_me) {
-            ASSERT(erts_refc_read(&bin->binary.refc, 0) == 0);
+            ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) == 0);
             erts_bin_free(&bin->binary);
         }
         return;
     }
     ASSERT(!rmp->is_dying);
-    if (erts_refc_inc_unless(&bin->binary.refc, 0, 0) == 0) {
+    if (erts_refc_inc_unless(&bin->binary.intern.refc, 0, 0) == 0) {
         /*
          * Racing resource destruction. 
          * To avoid a more complex refc-dance with destructing thread
@@ -2374,7 +2374,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t data_sz)
 
     ASSERT(type->owner && type->next && type->prev); /* not allowed in load/upgrade */
     resource->type = type;
-    erts_refc_inc(&bin->refc, 1);
+    erts_refc_inc(&bin->intern.refc, 1);
 #ifdef DEBUG
     erts_refc_init(&resource->nif_refc, 1);
 #endif
@@ -2416,7 +2416,7 @@ void enif_keep_resource(void* obj)
 #ifdef DEBUG
     erts_refc_inc(&resource->nif_refc, 1);
 #endif
-    erts_refc_inc(&bin->binary.refc, 2);
+    erts_refc_inc(&bin->binary.intern.refc, 2);
 }
 
 Eterm erts_bld_resource_ref(Eterm** hpp, ErlOffHeap* oh, ErtsResource* resource)
@@ -2453,7 +2453,7 @@ ERL_NIF_TERM enif_make_resource_binary(ErlNifEnv* env, void* obj,
     pb->flags = 0;
 
     OH_OVERHEAD(ohp, size / sizeof(Eterm));
-    erts_refc_inc(&bin->binary.refc, 1);
+    erts_refc_inc(&bin->binary.intern.refc, 1);
 
     return make_binary(hp);
 }
@@ -2478,7 +2478,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ
         }
         */
         mbin = ((ProcBin *) hp)->val;
-        if (!(mbin->flags & BIN_FLAG_MAGIC))
+        if (!(mbin->intern.flags & BIN_FLAG_MAGIC))
             return 0;
     }
     resource = (ErtsResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin);
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 77c7c5e73c..fbf14df92b 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -447,8 +447,8 @@ heap_dump(fmtfn_t to, void *to_arg, Eterm x)
 			ProcBin* pb = (ProcBin *) binary_val(x);
 			Binary* val = pb->val;
 
-			if (erts_atomic_xchg_nob(&val->refc, 0) != 0) {
-			    val->flags = (UWord) all_binaries;
+			if (erts_atomic_xchg_nob(&val->intern.refc, 0) != 0) {
+			    val->intern.flags = (UWord) all_binaries;
 			    all_binaries = val;
 			}
 			erts_print(to, to_arg,
@@ -529,7 +529,7 @@ dump_binaries(fmtfn_t to, void *to_arg, Binary* current)
 	    erts_print(to, to_arg, "%02X", bytes[i]);
 	}
 	erts_putc(to, to_arg, '\n');
-	current = (Binary *) current->flags;
+	current = (Binary *) current->intern.flags;
     }
 }
 
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 873945c3b5..e5358c2c3f 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1821,7 +1821,7 @@ static int ttb_context_destructor(Binary *context_bin)
 	case TTBEncode:
 	    DESTROY_SAVED_WSTACK(&context->s.ec.wstack);
 	    if (context->s.ec.result_bin != NULL) { /* Set to NULL if ever made alive! */
-		ASSERT(erts_refc_read(&(context->s.ec.result_bin->refc),1));
+		ASSERT(erts_refc_read(&(context->s.ec.result_bin->intern.refc),1));
 		erts_bin_free(context->s.ec.result_bin);
 		context->s.ec.result_bin = NULL;
 	    }
@@ -1830,13 +1830,13 @@ static int ttb_context_destructor(Binary *context_bin)
 	    erl_zlib_deflate_finish(&(context->s.cc.stream));
 
 	    if (context->s.cc.destination_bin != NULL) { /* Set to NULL if ever made alive! */
-		ASSERT(erts_refc_read(&(context->s.cc.destination_bin->refc),1));
+		ASSERT(erts_refc_read(&(context->s.cc.destination_bin->intern.refc),1));
 		erts_bin_free(context->s.cc.destination_bin);
 		context->s.cc.destination_bin = NULL;
 	    }
 	    
 	    if (context->s.cc.result_bin != NULL) { /* Set to NULL if ever made alive! */
-		ASSERT(erts_refc_read(&(context->s.cc.result_bin->refc),1));
+		ASSERT(erts_refc_read(&(context->s.cc.result_bin->intern.refc),1));
 		erts_bin_free(context->s.cc.result_bin);
 		context->s.cc.result_bin = NULL;
 	    }
@@ -1959,7 +1959,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		    pb->bytes = (byte*) result_bin->orig_bytes;
 		    pb->flags = 0;
 		    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
-		    if (context_b && erts_refc_read(&context_b->refc,0) == 0) {
+		    if (context_b && erts_refc_read(&context_b->intern.refc,0) == 0) {
 			erts_bin_free(context_b);
 		    }
 		    return make_binary(pb);
@@ -2025,7 +2025,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 			pb->next = MSO(p).first;
 			MSO(p).first = (struct erl_off_heap_header*)pb;
 			pb->val = result_bin;
-			ASSERT(erts_refc_read(&result_bin->refc, 1));
+			ASSERT(erts_refc_read(&result_bin->intern.refc, 1));
 			pb->bytes = (byte*) result_bin->orig_bytes;
 			pb->flags = 0;
 			OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
@@ -2033,7 +2033,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 			context->s.cc.result_bin = NULL;
 			context->alive = 0;
 			BUMP_REDS(p, (this_time * CONTEXT_REDS) / TERM_TO_BINARY_COMPRESS_CHUNK);
-			if (context_b && erts_refc_read(&context_b->refc,0) == 0) {
+			if (context_b && erts_refc_read(&context_b->intern.refc,0) == 0) {
 			    erts_bin_free(context_b);
 			}
 			return make_binary(pb);
@@ -2052,13 +2052,13 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		    pb->bytes = (byte*) result_bin->orig_bytes;
 		    pb->flags = 0;
 		    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
-		    ASSERT(erts_refc_read(&result_bin->refc, 1));
+		    ASSERT(erts_refc_read(&result_bin->intern.refc, 1));
 		    erl_zlib_deflate_finish(&(context->s.cc.stream));
 		    erts_bin_free(context->s.cc.destination_bin);
 		    context->s.cc.destination_bin = NULL;
 		    context->alive = 0;
 		    BUMP_REDS(p, (this_time * CONTEXT_REDS) / TERM_TO_BINARY_COMPRESS_CHUNK);
-		    if (context_b && erts_refc_read(&context_b->refc,0) == 0) {
+		    if (context_b && erts_refc_read(&context_b->intern.refc,0) == 0) {
 			erts_bin_free(context_b);
 		    }
 		    return make_binary(pb);
@@ -2773,7 +2773,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 			    erts_emasculate_writable_binary(pb);
 			    bytes += (pb->val->orig_bytes - before_realloc);
 			}
-			erts_refc_inc(&pb->val->refc, 2);
+			erts_refc_inc(&pb->val->intern.refc, 2);
 
 			sys_memcpy(&tmp, pb, sizeof(ProcBin));
 			tmp.next = *off_heap;
@@ -3887,7 +3887,7 @@ dec_term_atom_common:
 		sys_memcpy(pb, ep, sizeof(ProcBin));
 		ep += sizeof(ProcBin);
 
-		erts_refc_inc(&pb->val->refc, 1);
+		erts_refc_inc(&pb->val->intern.refc, 1);
 		hp += PROC_BIN_SIZE;
 		pb->next = factory->off_heap->first;
 		factory->off_heap->first = (struct erl_off_heap_header*)pb;
@@ -3905,7 +3905,7 @@ dec_term_atom_common:
 		sys_memcpy(pb, ep, sizeof(ProcBin));
 		ep += sizeof(ProcBin);
 
-		erts_refc_inc(&pb->val->refc, 1);
+		erts_refc_inc(&pb->val->intern.refc, 1);
 		hp += PROC_BIN_SIZE;
 		pb->next = factory->off_heap->first;
 		factory->off_heap->first = (struct erl_off_heap_header*)pb;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 4d864d1402..d765547e89 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1410,7 +1410,7 @@ Eterm erts_msacc_request(Process *c_p, int action, Eterm *threads);
 #define MatchSetRef(MPSP) 			\
 do {						\
     if ((MPSP) != NULL) {			\
-	erts_refc_inc(&(MPSP)->refc, 1);	\
+	erts_refc_inc(&(MPSP)->intern.refc, 1);	\
     }						\
 } while (0)
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index a1d3f8e29f..2f3117223f 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -4901,7 +4901,7 @@ erts_port_control(Process* c_p,
 	    ASSERT(bufp <= bufp + size);
 	    ASSERT(binp->orig_bytes <= bufp
 		   && bufp + size <= binp->orig_bytes + binp->orig_size);
-	    erts_refc_inc(&binp->refc, 1);
+	    erts_refc_inc(&binp->intern.refc, 1);
 	}
     }
 
@@ -6907,21 +6907,21 @@ ErlDrvSInt
 driver_binary_get_refc(ErlDrvBinary *dbp)
 {
     Binary* bp = ErlDrvBinary2Binary(dbp);
-    return (ErlDrvSInt) erts_refc_read(&bp->refc, 1);
+    return (ErlDrvSInt) erts_refc_read(&bp->intern.refc, 1);
 }
 
 ErlDrvSInt
 driver_binary_inc_refc(ErlDrvBinary *dbp)
 {
     Binary* bp = ErlDrvBinary2Binary(dbp);
-    return (ErlDrvSInt) erts_refc_inctest(&bp->refc, 2);
+    return (ErlDrvSInt) erts_refc_inctest(&bp->intern.refc, 2);
 }
 
 ErlDrvSInt
 driver_binary_dec_refc(ErlDrvBinary *dbp)
 {
     Binary* bp = ErlDrvBinary2Binary(dbp);
-    return (ErlDrvSInt) erts_refc_dectest(&bp->refc, 1);
+    return (ErlDrvSInt) erts_refc_dectest(&bp->intern.refc, 1);
 }
 
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 092a5320ba..e2b5434284 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3594,7 +3594,7 @@ store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns)
 	ErtsMRefThing *mreft = (ErtsMRefThing *) from_hp;
         ErtsMagicBinary *mb = mreft->mb;
 	ASSERT(is_magic_ref_thing(from_hp));
-	erts_refc_inc(&mb->refc, 2);
+	erts_refc_inc(&mb->intern.refc, 2);
         OH_OVERHEAD(oh, mb->orig_size / sizeof(Eterm));
     }
 
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 9076c65c2a..072ca19eae 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1936,6 +1936,6 @@ BIF_RETTYPE hipe_bifs_alloc_loader_state_1(BIF_ALIST_1)
 
     hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
     res = erts_mk_magic_ref(&hp, &MSO(BIF_P), magic);
-    erts_refc_dec(&magic->refc, 1);
+    erts_refc_dec(&magic->intern.refc, 1);
     BIF_RET(res);
 }
diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
index 0b53880628..9a9e3c6b12 100644
--- a/erts/emulator/hipe/hipe_load.c
+++ b/erts/emulator/hipe/hipe_load.c
@@ -81,7 +81,7 @@ Binary *hipe_alloc_loader_state(Eterm module)
 
     magic = erts_create_magic_binary(sizeof(HipeLoaderState),
 				     hipe_loader_state_dtor);
-    erts_refc_inc(&magic->refc, 1);
+    erts_refc_inc(&magic->intern.refc, 1);
     stp = ERTS_MAGIC_BIN_DATA(magic);
 
     stp->module = module;
-- 
cgit v1.2.3


From b50650b4c56de5279395ea195e9e68b077a1d3de Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 8 Feb 2017 17:57:52 +0100
Subject: Fix aux-work timer implementation

---
 erts/emulator/beam/erl_init.c    |  1 +
 erts/emulator/beam/erl_process.c | 32 ++++++++++++++++++++++++++------
 erts/emulator/beam/erl_process.h |  1 +
 3 files changed, 28 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 541bfec532..ac0324d846 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -2334,6 +2334,7 @@ erl_start(int argc, char **argv)
 	set_main_stack_size();
 	erts_sched_init_time_sup(esdp);
         erts_ets_sched_spec_data_init(esdp);
+        erts_aux_work_timeout_late_init(esdp);
 	process_main(esdp->x_reg_array, esdp->f_reg_array);
     }
 #endif
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 6b5b64993f..e2764cf86f 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -560,7 +560,6 @@ static int stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg);
 
 static void aux_work_timeout(void *unused);
 static void aux_work_timeout_early_init(int no_schedulers);
-static void aux_work_timeout_late_init(void);
 static void setup_aux_work_timer(ErtsSchedulerData *esdp);
 
 static int execute_sys_tasks(Process *c_p,
@@ -2790,6 +2789,9 @@ typedef struct {
 
     int initialized;
     erts_atomic32_t refc;
+#ifdef DEBUG
+    erts_atomic32_t used;
+#endif
     erts_atomic32_t type[1];
 } ErtsAuxWorkTmo;
 
@@ -2799,6 +2801,13 @@ static ERTS_INLINE void
 start_aux_work_timer(ErtsSchedulerData *esdp)
 {
     ErtsMonotonicTime tmo = erts_get_monotonic_time(esdp);
+#ifdef DEBUG
+    Uint no = (Uint) erts_atomic32_xchg_mb(&aux_work_tmo->used,
+                                           (erts_aint32_t) esdp->no);
+    ASSERT(esdp->type == ERTS_SCHED_NORMAL);
+    ASSERT(!no);
+#endif
+
     tmo = ERTS_MONOTONIC_TO_CLKTCKS(tmo-1);
     tmo += ERTS_MSEC_TO_CLKTCKS(1000) + 1;
     erts_twheel_init_timer(&aux_work_tmo->timer.data);
@@ -2835,16 +2844,19 @@ aux_work_timeout_early_init(int no_schedulers)
     aux_work_tmo = (ErtsAuxWorkTmo *) p;
     aux_work_tmo->initialized = 0;
     erts_atomic32_init_nob(&aux_work_tmo->refc, 0);
+#ifdef DEBUG
+    erts_atomic32_init_nob(&aux_work_tmo->used, 0);
+#endif
     for (i = 0; i <= no_schedulers; i++)
 	erts_atomic32_init_nob(&aux_work_tmo->type[i], 0);
 }
 
 void
-aux_work_timeout_late_init(void)
+erts_aux_work_timeout_late_init(ErtsSchedulerData *esdp)
 {
     aux_work_tmo->initialized = 1;
-    if (erts_atomic32_read_nob(&aux_work_tmo->refc))
-	start_aux_work_timer(erts_get_scheduler_data());
+    if (erts_atomic32_read_acqb(&aux_work_tmo->refc))
+	start_aux_work_timer(esdp);
 }
 
 static void
@@ -2852,6 +2864,13 @@ aux_work_timeout(void *vesdp)
 {
     erts_aint32_t refc;
     int i;
+#ifdef DEBUG
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    Uint no = (Uint) erts_atomic32_xchg_mb(&aux_work_tmo->used, 0);
+    ASSERT(no == esdp->no);
+    ASSERT(esdp == (ErtsSchedulerData *) vesdp);
+#endif
+
 #ifdef ERTS_SMP
     i = 0;
 #else
@@ -6465,8 +6484,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     /* init port tasks */
     erts_port_task_init();
 
-    aux_work_timeout_late_init();
-
 #ifndef ERTS_SMP
 #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
     erts_scheduler_data->verify_unused_temp_alloc
@@ -8752,6 +8769,9 @@ sched_thread_func(void *vesdp)
 
     erts_sched_init_time_sup(esdp);
 
+    if (no == 1)
+        erts_aux_work_timeout_late_init(esdp);
+
     (void) ERTS_RUNQ_FLGS_SET_NOB(esdp->run_queue,
 				  ERTS_RUNQ_FLG_EXEC);
 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 883d9f2a4c..f6683b6e7b 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1830,6 +1830,7 @@ void erts_schedule_multi_misc_aux_work(int ignore_self,
 				       void (*func)(void *),
 				       void *arg);
 erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int);
+void erts_aux_work_timeout_late_init(ErtsSchedulerData *esdp);
 void erts_sched_notify_check_cpu_bind(void);
 Uint erts_active_schedulers(void);
 void erts_init_process(int, int, int);
-- 
cgit v1.2.3


From cb7514ab298c715af435d9a8e5a019e91105cb98 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 30 Jan 2017 15:25:57 +0100
Subject: Remove accessor BIF timer implementation

---
 erts/emulator/beam/erl_alloc.c     |  10 --
 erts/emulator/beam/erl_alloc.types |   1 -
 erts/emulator/beam/erl_hl_timer.c  | 206 +------------------------------------
 erts/emulator/beam/erl_process.c   |  22 ----
 erts/emulator/beam/erl_process.h   |   3 -
 5 files changed, 3 insertions(+), 239 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index a374593c5d..71957b2259 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -678,10 +678,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	= erts_timer_type_size(ERTS_ALC_T_HL_PTIMER);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_BIF_TIMER)]
 	= erts_timer_type_size(ERTS_ALC_T_BIF_TIMER);
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_ABIF_TIMER)]
-	= erts_timer_type_size(ERTS_ALC_T_ABIF_TIMER);
-#endif
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_EXP_TRACE)]
 	= sizeof(NifExportTrace);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MREF_NSCHED_ENT)]
@@ -2440,12 +2436,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 		       &size.processes_used,
 		       fi,
 		       ERTS_ALC_T_BIF_TIMER);
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-	add_fix_values(&size.processes,
-		       &size.processes_used,
-		       fi,
-		       ERTS_ALC_T_ABIF_TIMER);
-#endif
 	add_fix_values(&size.processes,
 		       &size.processes_used,
 		       fi,
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 43f43f9034..f296a98125 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -168,7 +168,6 @@ type	TIMER_SERVICE	LONG_LIVED	SYSTEM		timer_service
 type    LL_PTIMER	FIXED_SIZE	PROCESSES	ll_ptimer
 type    HL_PTIMER	FIXED_SIZE	PROCESSES	hl_ptimer
 type    BIF_TIMER	FIXED_SIZE	PROCESSES	bif_timer
-# type    ABIF_TIMER	FIXED_SIZE	PROCESSES	accessor_bif_timer
 type    TIMER_REQUEST	SHORT_LIVED	PROCESSES	timer_request
 type    BTM_YIELD_STATE	SHORT_LIVED	PROCESSES	btm_yield_state
 type	REG_TABLE	STANDARD	SYSTEM		reg_tab
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 26be8c7edf..cfa1709518 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -106,9 +106,6 @@ typedef enum {
 #define ERTS_TMR_ROFLG_PROC		(((Uint32) 1) << 14)
 #define ERTS_TMR_ROFLG_PORT		(((Uint32) 1) << 15)
 #define ERTS_TMR_ROFLG_CALLBACK		(((Uint32) 1) << 16)
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-#define ERTS_TMR_ROFLG_ABIF_TMR		(((Uint32) 1) << 17)
-#endif
 
 #define ERTS_TMR_ROFLG_SID_MASK	\
     (ERTS_TMR_ROFLG_HLT - (Uint32) 1)
@@ -185,21 +182,10 @@ struct ErtsHLTimer_ {
 	Eterm message;
 	ErlHeapFragment *bp;
     } btm;
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-    struct {
-	Eterm accessor;
-	ErtsHLTimerTree tree;
-    } abtm;
-#endif
 };
 
 #define ERTS_HL_PTIMER_SIZE offsetof(ErtsHLTimer, btm)
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-#define ERTS_BIF_TIMER_SIZE offsetof(ErtsHLTimer, abtm)
-#define ERTS_ABIF_TIMER_SIZE sizeof(ErtsHLTimer)
-#else
 #define ERTS_BIF_TIMER_SIZE sizeof(ErtsHLTimer)
-#endif
 
 typedef struct {
     ErtsTmrHead head; /* NEED to be first! */
@@ -606,60 +592,6 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x)
 
 #include "erl_rbtree.h"
 
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-
-#define ERTS_RBT_PREFIX abtm
-#define ERTS_RBT_T ErtsHLTimer
-#define ERTS_RBT_KEY_T Uint32 *
-#define ERTS_RBT_FLAGS_T UWord
-#define ERTS_RBT_INIT_EMPTY_TNODE(T)					\
-    do {								\
-	(T)->abtm.tree.parent = (UWord) NULL;				\
-	(T)->abtm.tree.right = NULL;					\
-	(T)->abtm.tree.left = NULL;					\
-    } while (0)
-#define ERTS_RBT_IS_RED(T)						\
-    ((int) ((T)->abtm.tree.parent & ERTS_HLT_PFLG_RED))
-#define ERTS_RBT_SET_RED(T)						\
-    ((T)->abtm.tree.parent |= ERTS_HLT_PFLG_RED)
-#define ERTS_RBT_IS_BLACK(T)						\
-    (!ERTS_RBT_IS_RED((T)))
-#define ERTS_RBT_SET_BLACK(T)						\
-    ((T)->abtm.tree.parent &= ~ERTS_HLT_PFLG_RED)
-#define ERTS_RBT_GET_FLAGS(T)						\
-    ((T)->abtm.tree.parent & ERTS_HLT_PFLGS_MASK)
-#define ERTS_RBT_SET_FLAGS(T, F)					\
-    do {								\
-	ERTS_HLT_ASSERT((((UWord) (F)) & ~ERTS_HLT_PFLGS_MASK) == 0);	\
-	(T)->abtm.tree.parent &= ~ERTS_HLT_PFLGS_MASK;			\
-	(T)->abtm.tree.parent |= (F);					\
-    } while (0)
-#define ERTS_RBT_GET_PARENT(T)						\
-    ((ErtsHLTimer *) ((T)->abtm.tree.parent & ~ERTS_HLT_PFLGS_MASK))
-#define ERTS_RBT_SET_PARENT(T, P)					\
-    do {								\
-	ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0);	\
-	(T)->abtm.tree.parent &= ERTS_HLT_PFLGS_MASK;			\
-	(T)->abtm.tree.parent |= (UWord) (P);				\
-    } while (0)
-#define ERTS_RBT_GET_RIGHT(T) ((T)->abtm.tree.right)
-#define ERTS_RBT_SET_RIGHT(T, R) ((T)->abtm.tree.right = (R))
-#define ERTS_RBT_GET_LEFT(T) ((T)->abtm.tree.left)
-#define ERTS_RBT_SET_LEFT(T, L) ((T)->abtm.tree.left = (L))
-#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn)
-#define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY))
-#define ERTS_RBT_IS_EQ(KX, KY) \
-    (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2]))
-#define ERTS_RBT_WANT_DELETE
-#define ERTS_RBT_WANT_INSERT
-#define ERTS_RBT_WANT_LOOKUP
-#define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING
-#define ERTS_RBT_UNDEF
-
-#include "erl_rbtree.h"
-
-#endif /* ERTS_BTM_ACCESSOR_SUPPORT */
-
 #ifdef ERTS_SMP
 static void init_canceled_queue(ErtsHLTCncldTmrQ *cq);
 #endif
@@ -699,9 +631,6 @@ erts_timer_type_size(ErtsAlcType_t type)
     case ERTS_ALC_T_LL_PTIMER: return sizeof(ErtsTWTimer);
     case ERTS_ALC_T_HL_PTIMER: return ERTS_HL_PTIMER_SIZE;
     case ERTS_ALC_T_BIF_TIMER: return ERTS_BIF_TIMER_SIZE;
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-    case ERTS_ALC_T_ABIF_TIMER: return ERTS_ABIF_TIMER_SIZE;
-#endif
     default: ERTS_INTERNAL_ERROR("Unknown type");
     }
     return 0;
@@ -920,10 +849,6 @@ hl_timer_destroy(ErtsHLTimer *tmr)
     else {
 	if (roflgs & ERTS_TMR_ROFLG_PRE_ALC)
 	    bif_timer_pre_free(tmr);
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-	else if (roflgs & ERTS_TMR_ROFLG_ABIF_TMR)
-	    erts_free(ERTS_ALC_T_ABIF_TIMER, tmr);
-#endif
 	else
 	    erts_free(ERTS_ALC_T_BIF_TIMER, tmr);
     }
@@ -1018,33 +943,6 @@ check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv)
 #endif
 }
 
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-
-static void
-hlt_delete_abtm(ErtsHLTimer *tmr)
-{
-    Process *proc;
-
-    ERTS_HLT_ASSERT(tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR);
-
-    proc = erts_proc_lookup(tmr->abtm.accessor);
-
-    if (proc) {
-	int deref = 0;
-	erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM);
-	if (tmr->abtm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
-	    abtm_rbt_delete(&proc->accessor_bif_timers, tmr);
-	    deref = 1;
-	    tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
-	}
-	erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
-	if (deref)
-	    hl_timer_pre_dec_refc(tmr);
-    }
-}
-
-#endif
-
 static ErtsHLTimer *
 create_hl_timer(ErtsSchedulerData *esdp,
 		ErtsMonotonicTime timeout_pos,
@@ -1106,9 +1004,6 @@ create_hl_timer(ErtsSchedulerData *esdp,
     }
     else { /* ERTS_TMR_BIF */
 	Uint hsz;
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-	int is_abif_tmr = is_value(acsr) && acsr != rcvr;
-#endif
 
 	if (short_time) {
 	    tmr = bif_timer_pre_alloc();
@@ -1118,14 +1013,8 @@ create_hl_timer(ErtsSchedulerData *esdp,
 	}
 	else {
 	alloc_bif_timer:
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-	    if (is_abif_tmr)
-		tmr = erts_alloc(ERTS_ALC_T_ABIF_TIMER,
-				 ERTS_ABIF_TIMER_SIZE);
-	    else
-#endif
-		tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER,
-				 ERTS_BIF_TIMER_SIZE);
+            tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER,
+                             ERTS_BIF_TIMER_SIZE);
 	}
 
 	tmr->timeout = timeout_pos;
@@ -1160,23 +1049,6 @@ create_hl_timer(ErtsSchedulerData *esdp,
 
 	tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
 
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-	if (is_abif_tmr) {
-	    Process *aproc;
-	    roflgs |= ERTS_TMR_ROFLG_ABIF_TMR;
-	    tmr->abtm.accessor = acsr;
-	    aproc = erts_proc_lookup(acsr);
-	    if (!aproc)
-		tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
-	    else {
-		refc++;
-		erts_smp_proc_lock(aproc, ERTS_PROC_LOCK_BTM);
-		abtm_rbt_insert(&aproc->accessor_bif_timers, tmr);
-		erts_smp_proc_unlock(aproc, ERTS_PROC_LOCK_BTM);
-	    }
-	}
-#endif
-
 	btm_rbt_insert(&srv->btm_tree, tmr);
     }
 
@@ -1222,11 +1094,6 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs)
     Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME);
     ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR);
 
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-    if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR)
-	hlt_delete_abtm(tmr);
-#endif
-
     if (is_reg_name) {
 	Eterm pid;
 	ERTS_HLT_ASSERT(is_atom(tmr->receiver.name));
@@ -1409,10 +1276,6 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr)
 	    tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
 	}
 
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-	if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR)
-	    hlt_delete_abtm(tmr);
-#endif
     }
 
     if (tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE) {
@@ -2032,15 +1895,10 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
 
     /*
      * Check if the timer is aimed at current
-     * process of if this process is an accessor
-     * of the timer...
+     * process...
      */
     erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_BTM);
     tmr = proc_btm_rbt_lookup(c_p->bif_timers, trefn);
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-    if (!tmr)
-	tmr = abtm_rbt_lookup(c_p->accessor_bif_timers, trefn);
-#endif
     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_BTM);
     if (!tmr)
 	return 0;
@@ -2290,13 +2148,6 @@ parse_bif_timer_options(Eterm option_list, int *async, int *info,
 	    if (!abs || !bool_arg(tp[2], abs))
 		return 0;
 	    break;
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-	case am_accessor:
-	    if (!accessor || is_not_internal_pid(tp[2]))
-		return 0;
-	    *accessor = tp[2];
-	    break;
-#endif
 	default:
 	    return 0;
 	}
@@ -2358,9 +2209,6 @@ typedef struct {
     ErtsBifTimers *bif_timers;
     union {
 	proc_btm_rbt_yield_state_t proc_btm_yield_state;
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-	abtm_rbt_yield_state_t abtm_yield_state;
-#endif
     } u;
 } ErtsBifTimerYieldState;
 
@@ -2401,54 +2249,6 @@ int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp)
     return res;
 }
 
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-
-static void
-detach_bif_timer(ErtsHLTimer *tmr, void *vesdp)
-{
-    tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
-    hl_timer_dec_refc(tmr, tmr->head.roflgs);
-}
-
-int erts_detach_accessor_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp)
-{
-    ErtsSchedulerData *esdp = erts_proc_sched_data(p);
-    ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}};
-    ErtsBifTimerYieldState *ysp;
-    int res;
-
-    ysp = (ErtsBifTimerYieldState *) *vyspp;
-    if (!ysp)
-	ysp = &ys;
-
-    res = abtm_rbt_foreach_destroy_yielding(&ysp->bif_timers,
-					    detach_bif_timer,
-					    (void *) esdp,
-					    &ysp->u.abtm_yield_state,
-					    ERTS_BTM_MAX_DESTROY_LIMIT);
-
-    if (res == 0) {
-	if (ysp != &ys)
-	    erts_free(ERTS_ALC_T_BTM_YIELD_STATE, ysp);
-	*vyspp = NULL;
-    }
-    else {
-
-	if (ysp == &ys) {
-	    ysp = erts_alloc(ERTS_ALC_T_BTM_YIELD_STATE,
-			     sizeof(ErtsBifTimerYieldState));
-	    sys_memcpy((void *) ysp, (void *) &ys,
-		       sizeof(ErtsBifTimerYieldState));
-	}
-
-	*vyspp = (void *) ysp;
-    }
-
-    return res;
-}
-
-#endif /* ERTS_BTM_ACCESSOR_SUPPORT */
-
 static ERTS_INLINE int
 parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg,
 		  ErtsMonotonicTime *conv_arg, int abs,
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index e2764cf86f..cb95eab67e 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -12447,9 +12447,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->msg_inq.len = 0;
 #endif
     p->bif_timers = NULL;
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-    p->accessor_bif_timers = NULL;
-#endif
     p->mbuf = NULL;
     p->msg_frag = NULL;
     p->mbuf_sz = 0;
@@ -12648,9 +12645,6 @@ void erts_init_empty_process(Process *p)
     p->msg.save = &p->msg.first;
     p->msg.len = 0;
     p->bif_timers = NULL;
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-    p->accessor_bif_timers = NULL;
-#endif
     p->dictionary = NULL;
     p->seq_trace_clock = 0;
     p->seq_trace_lastcnt = 0;
@@ -12751,9 +12745,6 @@ erts_debug_verify_clean_empty_process(Process* p)
     ASSERT(p->msg.first == NULL);
     ASSERT(p->msg.len == 0);
     ASSERT(p->bif_timers == NULL);
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-    ASSERT(p->accessor_bif_timers == NULL);
-#endif
     ASSERT(p->dictionary == NULL);
     ASSERT(p->catches == 0);
     ASSERT(p->cp == NULL);
@@ -13827,19 +13818,6 @@ erts_continue_exit_process(Process *p)
 	p->bif_timers = NULL;
     }
 
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-    if (p->accessor_bif_timers) {
-	if (erts_detach_accessor_bif_timers(p,
-					    p->accessor_bif_timers,
-					    &p->u.terminate)) {
-	    ASSERT(erts_proc_read_refc(p) > 0);
-	    goto yield;
-	}
-	ASSERT(erts_proc_read_refc(p) > 0);
-	p->accessor_bif_timers = NULL;
-    }
-#endif
-
 #ifdef ERTS_SMP
     if (p->flags & F_SCHDLR_ONLN_WAITQ)
 	abort_sched_onln_chng_waitq(p);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index f6683b6e7b..b21597d63b 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1026,9 +1026,6 @@ struct process {
     ErlMessageQueue msg;	/* Message queue */
 
     ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
-#ifdef ERTS_BTM_ACCESSOR_SUPPORT
-    ErtsBifTimers *accessor_bif_timers;	/* Accessor bif timers */
-#endif
 
     ProcDict  *dictionary;       /* Process dictionary, may be NULL */
 
-- 
cgit v1.2.3


From e46671ce6ea9429fe3d6723b4f93c08c7fa85cd0 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 31 Jan 2017 13:30:49 +0100
Subject: Use magic refs for BIF timers

---
 erts/emulator/beam/erl_hl_timer.c | 757 +++++++++++++++++++++++++++-----------
 erts/emulator/beam/erl_hl_timer.h |   2 +-
 erts/emulator/beam/erl_process.c  |   2 +-
 3 files changed, 553 insertions(+), 208 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index cfa1709518..6d570d28ec 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -29,6 +29,8 @@
 #  include "config.h"
 #endif
 
+#define ERTS_MAGIC_REF_BIF_TIMERS
+
 #include "sys.h"
 #include "global.h"
 #include "bif.h"
@@ -36,6 +38,9 @@
 #define ERTS_WANT_TIMER_WHEEL_API
 #include "erl_time.h"
 #include "erl_hl_timer.h"
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+#include "erl_binary.h"
+#endif
 
 #define ERTS_TMR_CHECK_CANCEL_ON_CREATE 0
 
@@ -124,6 +129,11 @@ typedef struct ErtsHLTimer_ ErtsHLTimer;
 
 #define ERTS_HLT_PFIELD_NOT_IN_TABLE	(~((UWord) 0))
 
+typedef struct {
+    ErtsHLTimer *next;
+    ErtsHLTimer *prev;    
+} ErtsHLTimerList;
+
 typedef struct {
     UWord parent; /* parent pointer and flags... */
     union {
@@ -140,11 +150,6 @@ typedef struct {
 } ErtsHLTimerTimeTree;
 
 typedef struct {
-    UWord parent; /* parent pointer and flags... */
-    ErtsHLTimer *right;
-    ErtsHLTimer *left;
-} ErtsHLTimerTree;
-
 typedef struct {
     Uint32 roflgs;
     erts_smp_atomic32_t refc;
@@ -176,9 +181,14 @@ struct ErtsHLTimer_ {
 
     /* BIF timer only fields follow... */
     struct {
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+        ErtsMagicBinary *mbin;
+        ErtsHLTimerList proc_list;
+#else
 	Uint32 refn[ERTS_REF_NUMBERS];
 	ErtsHLTimerTree proc_tree;
 	ErtsHLTimerTree tree;
+#endif
 	Eterm message;
 	ErlHeapFragment *bp;
     } btm;
@@ -282,12 +292,16 @@ struct ErtsHLTimerService_ {
     ErtsHLTCncldTmrQ canceled_queue;
 #endif
     ErtsHLTimer *time_tree;
+#ifndef ERTS_MAGIC_REF_BIF_TIMERS
     ErtsHLTimer *btm_tree;
+#endif
     ErtsHLTimer *next_timeout;
     ErtsYieldingTimeoutState yield;
     ErtsTWheelTimer service_timer;
 };
 
+#ifndef ERTS_MAGIC_REF_BIF_TIMERS
+
 static ERTS_INLINE int
 refn_is_lt(Uint32 *x, Uint32 *y)
 {
@@ -303,6 +317,14 @@ refn_is_lt(Uint32 *x, Uint32 *y)
     return x[0] < y[0];
 }
 
+static ERTS_INLINE int
+refn_is_eq(Uint32 *x, Uint32 *y)
+{
+    return (x[0] == y[0]) & (x[1] == y[1]) & (x[2] == y[2]);
+}
+
+#endif
+
 #define ERTS_RBT_PREFIX time
 #define ERTS_RBT_T ErtsHLTimer
 #define ERTS_RBT_KEY_T ErtsMonotonicTime
@@ -492,6 +514,14 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x)
 
 #endif /* ERTS_HLT_HARD_DEBUG */
 
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+#define ERTS_BTM_HLT2REFN(T) ((T)->btm.mbin->refn)
+#else
+#define ERTS_BTM_HLT2REFN(T) ((T)->btm.refn)
+#endif
+
+#ifndef ERTS_MAGIC_REF_BIF_TIMERS
+
 #define ERTS_RBT_PREFIX btm
 #define ERTS_RBT_T ErtsHLTimer
 #define ERTS_RBT_KEY_T Uint32 *
@@ -530,18 +560,92 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x)
 #define ERTS_RBT_SET_RIGHT(T, R) ((T)->btm.tree.right = (R))
 #define ERTS_RBT_GET_LEFT(T) ((T)->btm.tree.left)
 #define ERTS_RBT_SET_LEFT(T, L) ((T)->btm.tree.left = (L))
-#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn)
+#define ERTS_RBT_GET_KEY(T) ERTS_BTM_HLT2REFN((T))
 #define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY))
-#define ERTS_RBT_IS_EQ(KX, KY) \
-    (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2]))
+#define ERTS_RBT_IS_EQ(KX, KY) refn_is_eq((KX), (KY))
 #define ERTS_RBT_WANT_DELETE
 #define ERTS_RBT_WANT_INSERT
+#ifndef ERTS_MAGIC_REF_BIF_TIMERS
 #define ERTS_RBT_WANT_LOOKUP
+#endif
 #define ERTS_RBT_WANT_FOREACH
 #define ERTS_RBT_UNDEF
 
 #include "erl_rbtree.h"
 
+#endif /* !ERTS_MAGIC_REF_BIF_TIMERS */
+
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+
+static ERTS_INLINE void
+proc_btm_list_insert(ErtsHLTimer **list, ErtsHLTimer *x)
+{
+    ErtsHLTimer *y = *list;
+    if (!y) {
+        x->btm.proc_list.next = x;
+        x->btm.proc_list.prev = x;
+        *list = x;
+    }
+    else {
+        ERTS_HLT_ASSERT(y->btm.proc_list.prev->btm.proc_list.next == y);
+        x->btm.proc_list.next = y;
+        x->btm.proc_list.prev = y->btm.proc_list.prev;
+        y->btm.proc_list.prev->btm.proc_list.next = x;
+        y->btm.proc_list.prev = x;
+    }
+}
+
+static ERTS_INLINE void
+proc_btm_list_delete(ErtsHLTimer **list, ErtsHLTimer *x)
+{
+    ErtsHLTimer *y = *list;
+    if (y == x && x->btm.proc_list.next == x) {
+        ERTS_HLT_ASSERT(x->btm.proc_list.prev == x);
+        *list = NULL;
+    }
+    else {
+        if (y == x)
+            *list = x->btm.proc_list.next;
+        ERTS_HLT_ASSERT(x->btm.proc_list.prev->btm.proc_list.next == x);
+        ERTS_HLT_ASSERT(x->btm.proc_list.next->btm.proc_list.prev == x);
+        x->btm.proc_list.prev->btm.proc_list.next = x->btm.proc_list.next;
+        x->btm.proc_list.next->btm.proc_list.prev = x->btm.proc_list.prev;
+    }
+    x->btm.proc_list.next = NULL;
+}
+
+static ERTS_INLINE int
+proc_btm_list_foreach_destroy_yielding(ErtsHLTimer **list,
+                                       void (*destroy)(ErtsHLTimer *, void *),
+                                       void *arg,
+                                       int limit)
+{
+    int i;
+    ErtsHLTimer *first, *last;
+
+    first = *list;
+    if (!first)
+        return 0;
+
+    last = first->btm.proc_list.prev;
+    for (i = 0; i < limit; i++) {
+        ErtsHLTimer *x = last;
+        last = last->btm.proc_list.prev;
+        (*destroy)(x, arg);
+        x->btm.proc_list.next = NULL;
+        if (x == first) {
+            *list = NULL;
+            return 0;
+        }
+    }
+
+    last->btm.proc_list.next = first;
+    first->btm.proc_list.prev = last;
+    return 1;
+}
+
+#else /* !ERTS_MAGIC_REF_BIF_TIMERS */
+
 #define ERTS_RBT_PREFIX proc_btm
 #define ERTS_RBT_T ErtsHLTimer
 #define ERTS_RBT_KEY_T Uint32 *
@@ -580,18 +684,21 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x)
 #define ERTS_RBT_SET_RIGHT(T, R) ((T)->btm.proc_tree.right = (R))
 #define ERTS_RBT_GET_LEFT(T) ((T)->btm.proc_tree.left)
 #define ERTS_RBT_SET_LEFT(T, L) ((T)->btm.proc_tree.left = (L))
-#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn)
+#define ERTS_RBT_GET_KEY(T) ERTS_BTM_HLT2REFN((T))
 #define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY))
-#define ERTS_RBT_IS_EQ(KX, KY) \
-    (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2]))
+#define ERTS_RBT_IS_EQ(KX, KY) refn_is_eq((KX), (KY))
 #define ERTS_RBT_WANT_DELETE
 #define ERTS_RBT_WANT_INSERT
+#ifndef ERTS_MAGIC_REF_BIF_TIMERS
 #define ERTS_RBT_WANT_LOOKUP
+#endif
 #define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING
 #define ERTS_RBT_UNDEF
 
 #include "erl_rbtree.h"
 
+#endif /* !ERTS_MAGIC_REF_BIF_TIMERS */
+
 #ifdef ERTS_SMP
 static void init_canceled_queue(ErtsHLTCncldTmrQ *cq);
 #endif
@@ -612,7 +719,9 @@ erts_create_timer_service(void)
     srv = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_SERVICE,
 					     sizeof(ErtsHLTimerService));
     srv->time_tree = NULL;
+#ifndef ERTS_MAGIC_REF_BIF_TIMERS
     srv->btm_tree = NULL;
+#endif
     srv->next_timeout = NULL;
     srv->yield = init_yield;
     erts_twheel_init_timer(&srv->service_timer);
@@ -847,6 +956,11 @@ hl_timer_destroy(ErtsHLTimer *tmr)
     if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR))
 	erts_free(ERTS_ALC_T_HL_PTIMER, tmr);
     else {
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+        Binary *bp = (Binary *) tmr->btm.mbin;
+        if (erts_refc_dectest(&bp->refc, 0) == 0)
+            erts_bin_free(bp);
+#endif
 	if (roflgs & ERTS_TMR_ROFLG_PRE_ALC)
 	    bif_timer_pre_free(tmr);
 	else
@@ -892,6 +1006,7 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs)
 	 * once...
 	 */
 	size = sizeof(ErtsHLTimer);
+
     }
 
     erts_schedule_thr_prgr_later_cleanup_op(
@@ -943,12 +1058,64 @@ check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv)
 #endif
 }
 
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+
+static erts_smp_atomic_t *
+mbin_to_btmref__(ErtsMagicBinary *mbin)
+{
+    return erts_smp_binary_to_magic_indirection((Binary *) mbin);
+}
+
+static ERTS_INLINE void
+magic_binary_init(ErtsMagicBinary *mbin, ErtsHLTimer *tmr)
+{
+    erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin);
+    erts_smp_atomic_init_nob(aptr, (erts_aint_t) tmr);
+}
+
+static ERTS_INLINE ErtsHLTimer *
+magic_binary_to_btm(ErtsMagicBinary *mbin)
+{
+    erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin);
+    ErtsHLTimer *tmr = (ErtsHLTimer *) erts_smp_atomic_read_nob(aptr);
+    ERTS_HLT_ASSERT(!tmr || tmr->btm.mbin == mbin);
+    return tmr;
+}
+
+static ERTS_INLINE void
+btm_clear_magic_binary(ErtsHLTimer *tmr)
+{
+    erts_smp_atomic_t *aptr = mbin_to_btmref__(tmr->btm.mbin);
+#ifdef ERTS_HLT_DEBUG
+    erts_aint_t tval = erts_smp_atomic_xchg_nob(aptr,
+                                                (erts_aint_t) NULL);
+    ERTS_HLT_ASSERT(tval == (erts_aint_t) tmr);
+#else
+    erts_smp_atomic_set_nob(aptr, (erts_aint_t) NULL);
+#endif
+    hl_timer_dec_refc(tmr, tmr->head.roflgs);
+}
+
+static int
+bif_timer_ref_destructor(Binary *unused)
+{
+    return 1;
+}
+
+#endif
+
+
 static ErtsHLTimer *
 create_hl_timer(ErtsSchedulerData *esdp,
 		ErtsMonotonicTime timeout_pos,
 		int short_time, ErtsTmrType type,
 		void *rcvrp, Eterm rcvr, Eterm acsr,
-		Eterm msg, Uint32 *refn,
+		Eterm msg,
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+                ErtsMagicBinary *mbin,
+#else
+                Uint32 *refn,
+#endif
 		void (*callback)(void *), void *arg)
 {
     ErtsHLTimerService *srv = esdp->timer_service;
@@ -1043,6 +1210,13 @@ create_hl_timer(ErtsSchedulerData *esdp,
 	    tmr->btm.message = copy_struct(msg, hsz, &hp, &bp->off_heap);
 	    tmr->btm.bp = bp;
 	}
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+        tmr->btm.mbin = mbin;
+        erts_refc_inc(&mbin->refc, 1);
+        magic_binary_init(mbin, tmr);
+        refc += 1; /* from magic binary... */
+	tmr->btm.proc_list.next = NULL;
+#else
 	tmr->btm.refn[0] = refn[0];
 	tmr->btm.refn[1] = refn[1];
 	tmr->btm.refn[2] = refn[2];
@@ -1050,6 +1224,7 @@ create_hl_timer(ErtsSchedulerData *esdp,
 	tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
 
 	btm_rbt_insert(&srv->btm_tree, tmr);
+#endif
     }
 
     tmr->head.roflgs = roflgs;
@@ -1094,6 +1269,10 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs)
     Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME);
     ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR);
 
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+    btm_clear_magic_binary(tmr);
+#endif
+
     if (is_reg_name) {
 	Eterm pid;
 	ERTS_HLT_ASSERT(is_atom(tmr->receiver.name));
@@ -1123,11 +1302,18 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs)
 	    queued_message = 1;
 	    proc_locks &= ~ERTS_PROC_LOCKS_MSG_SEND;
 	    tmr->btm.bp = NULL;
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+            if (tmr->btm.proc_list.next) {
+                proc_btm_list_delete(&proc->bif_timers, tmr);
+		dec_refc = 1;
+            }
+#else
 	    if (tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
 		proc_btm_rbt_delete(&proc->bif_timers, tmr);
 		tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
 		dec_refc = 1;
 	    }
+#endif
 	}
 	if (proc_locks)
 	    erts_smp_proc_unlock(proc, proc_locks);
@@ -1154,7 +1340,9 @@ hlt_port_timeout(ErtsHLTimer *tmr)
 
 static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv)
 {
+#if !defined(ERTS_MAGIC_REF_BIF_TIMERS) || defined(ERTS_HLT_HARD_DEBUG)
     ErtsHLTimerService *srv = (ErtsHLTimerService *) vsrv;
+#endif
     Uint32 roflgs;
     erts_aint32_t state;
 
@@ -1186,11 +1374,14 @@ static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv)
     }
 
     tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+
+#ifndef ERTS_MAGIC_REF_BIF_TIMERS
     if ((roflgs & ERTS_TMR_ROFLG_BIF_TMR)
 	&& tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
 	btm_rbt_delete(&srv->btm_tree, tmr);
 	tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
     }
+#endif
 
     ERTS_HLT_HDBG_CHK_SRV(srv);
 
@@ -1269,6 +1460,7 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr)
 
     ERTS_HLT_HDBG_CHK_SRV(srv);
 
+#ifndef ERTS_MAGIC_REF_BIF_TIMERS
     if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR) {
 
 	if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
@@ -1277,6 +1469,7 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr)
 	}
 
     }
+#endif
 
     if (tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE) {
 	/* Already removed... */
@@ -1613,6 +1806,7 @@ continue_cancel_ptimer(ErtsSchedulerData *esdp, ErtsTimer *tmr)
  * BIF timer specific
  */
 
+
 Uint erts_bif_timer_memory_size(void)
 {
     return (Uint) 0;
@@ -1627,41 +1821,59 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos,
     Eterm ref, tmo_msg, *hp;
     ErtsHLTimer *tmr;
     ErtsSchedulerData *esdp;
-    DeclareTmpHeap(tmp_hp, 4, c_p);
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+    Binary *mbin;
+#endif
+    Eterm tmp_hp[4];
 
     if (is_not_internal_pid(rcvr) && is_not_atom(rcvr))
 	goto badarg;
 
     esdp = erts_proc_sched_data(c_p);
 
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+    mbin = erts_create_magic_indirection(bif_timer_ref_destructor);
+    hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE);
+    ref = erts_mk_magic_ref(&hp, &c_p->off_heap, mbin);
+    ASSERT(erts_get_ref_numbers_thr_id(((ErtsMagicBinary *)mbin)->refn)
+           == (Uint32) esdp->no);
+#else
     hp = HAlloc(c_p, ERTS_REF_THING_SIZE);
     ref = erts_sched_make_ref_in_buffer(esdp, hp);
-
-    ASSERT(erts_get_ref_numbers_thr_id(
-	       internal_ref_numbers(ref)) == (Uint32) esdp->no);
-
-    UseTmpHeap(4, c_p);
+    ASSERT(erts_get_ref_numbers_thr_id(internal_ordinary_ref_numbers(ref))
+           == (Uint32) esdp->no);
+#endif
 
     tmo_msg = wrap ? TUPLE3(tmp_hp, am_timeout, ref, msg) : msg;
 
     tmr = create_hl_timer(esdp, timeout_pos, short_time,
 			  ERTS_TMR_BIF, NULL,  rcvr, acsr, tmo_msg,
-			  internal_ref_numbers(ref), NULL, NULL);
-
-    UnUseTmpHeap(4, c_p);
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+                          (ErtsMagicBinary *) mbin,
+#else
+                          internal_ordinary_ref_numbers(ref),
+#endif
+                          NULL, NULL);
 
     if (is_internal_pid(rcvr)) {
 	Process *proc = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
 					  rcvr, ERTS_PROC_LOCK_BTM,
 					  ERTS_P2P_FLG_INC_REFC);
 	if (!proc) {
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+            btm_clear_magic_binary(tmr);
+#endif
 	    if (tmr->btm.bp)
 		free_message_buffer(tmr->btm.bp);
 	    hlt_delete_timer(esdp, tmr);
 	    hl_timer_destroy(tmr);
 	}
 	else {
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+            proc_btm_list_insert(&proc->bif_timers, tmr);
+#else
 	    proc_btm_rbt_insert(&proc->bif_timers, tmr);
+#endif
 	    erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
 	    tmr->receiver.proc = proc;
 	}
@@ -1689,6 +1901,10 @@ cancel_bif_timer(ErtsHLTimer *tmr)
     if (state != ERTS_TMR_STATE_ACTIVE)
 	return 0;
 
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+    btm_clear_magic_binary(tmr);
+#endif
+
     if (tmr->btm.bp)
 	free_message_buffer(tmr->btm.bp);
 
@@ -1705,25 +1921,215 @@ cancel_bif_timer(ErtsHLTimer *tmr)
 	 * the btm tree by itself (it may be in
 	 * the middle of tree destruction).
 	 */
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+        if (!ERTS_PROC_IS_EXITING(proc) && tmr->btm.proc_list.next) {
+            proc_btm_list_delete(&proc->bif_timers, tmr);
+            res = 1;
+        }
+#else
 	if (!ERTS_PROC_IS_EXITING(proc)
 	    && tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
 	    proc_btm_rbt_delete(&proc->bif_timers, tmr);
 	    tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
 	    res = 1;
 	}
+#endif
 	erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
     }
 
     return res;
 }
 
+static ERTS_INLINE Sint64
+access_btm(ErtsHLTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel)
+{
+    int cncl_res;
+    Sint64 time_left;
+
+    if (!tmr)
+        return -1;
+
+    if (!cancel) {
+        erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state);
+        if (state == ERTS_TMR_STATE_ACTIVE)
+            return get_time_left(esdp, tmr->timeout);
+        return -1;
+    }
+
+    cncl_res = cancel_bif_timer(tmr);
+    if (!cncl_res)
+        return -1;
+
+    time_left = get_time_left(esdp, tmr->timeout);
+
+    if (sid != (Uint32) esdp->no) {
+        if (cncl_res > 0)
+            queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr);
+    }
+    else {
+        if (cncl_res > 0)
+            hl_timer_dec_refc(tmr, tmr->head.roflgs);
+
+        hlt_delete_timer(esdp, tmr);
+    }
+
+    return time_left;
+}
+
+static ERTS_INLINE Eterm
+return_info(Process *c_p, Sint64 time_left)
+{
+    Uint hsz;
+    Eterm *hp;
+
+    if (time_left < 0)
+        return am_false;
+
+    if (time_left <= (Sint64) MAX_SMALL)
+        return make_small((Sint) time_left);
+
+    hsz = ERTS_SINT64_HEAP_SIZE(time_left);
+    hp = HAlloc(c_p, hsz);
+    return erts_sint64_to_big(time_left, &hp);
+}
+
+static ERTS_INLINE Eterm
+send_async_info(Process *proc, ErtsProcLocks initial_locks,
+                Eterm tref, int cancel, Sint64 time_left)
+{
+    ErtsProcLocks locks = initial_locks;
+    ErtsMessage *mp;
+    Eterm tag, res, msg, ref;
+    Uint hsz;
+    Eterm *hp;
+    ErlOffHeap *ohp;
+
+    hsz = 4;
+    hsz += NC_HEAP_SIZE(tref);
+
+    if (time_left > (Sint64) MAX_SMALL)
+        hsz += ERTS_SINT64_HEAP_SIZE(time_left);
+
+    mp = erts_alloc_message_heap(proc, &locks, hsz, &hp, &ohp);
+
+    if (cancel)
+        tag = am_cancel_timer;
+    else
+        tag = am_read_timer;
+
+    ref = STORE_NC(&hp, ohp, tref);
+
+    if (time_left < 0)
+        res = am_false;
+    else if (time_left <= (Sint64) MAX_SMALL)
+        res = make_small((Sint) time_left);
+    else
+        res = erts_sint64_to_big(time_left, &hp);
+
+    msg = TUPLE3(hp, tag, ref, res);
+
+    erts_queue_message(proc, locks, mp, msg, am_clock_service);
+
+    locks &= ~initial_locks;
+    if (locks)
+        erts_smp_proc_unlock(proc, locks);
+
+    return am_ok;
+}
+
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+
+static BIF_RETTYPE
+access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info)
+{
+    BIF_RETTYPE ret;
+    Eterm res;
+    Sint64 time_left;
+
+    if (!is_internal_magic_ref(tref)) {
+	if (is_not_ref(tref)) {
+            ERTS_BIF_PREP_ERROR(ret, c_p, BADARG);
+            return ret;
+        }
+        time_left = -1;
+    }
+    else {
+        ErtsMagicBinary *mbin;
+        mbin = (ErtsMagicBinary *) erts_magic_ref2bin(tref);
+        if (mbin->destructor != bif_timer_ref_destructor)
+            time_left = -1;
+        else {
+            ErtsHLTimer *tmr;
+            Uint32 sid;
+            tmr = magic_binary_to_btm(mbin);
+            sid = erts_get_ref_numbers_thr_id(internal_magic_ref_numbers(tref));
+            ASSERT(1 <= sid && sid <= erts_no_schedulers);
+            time_left = access_btm(tmr, sid, erts_proc_sched_data(c_p), cancel);
+        }
+    }
+
+    if (!info)
+        res = am_ok;
+    else if (!async)
+        res = return_info(c_p, time_left);
+    else
+        res = send_async_info(c_p, ERTS_PROC_LOCK_MAIN,
+                              tref, cancel, time_left);
+
+    ERTS_BIF_PREP_RET(ret, res);
+
+    return ret;
+}
+
+#else /* !ERTS_MAGIC_REF_BIF_TIMERS */
+
+static ERTS_INLINE Eterm
+send_sync_info(Process *proc, ErtsProcLocks initial_locks,
+               Uint32 *refn, int cancel, Sint64 time_left)
+{
+    ErtsProcLocks locks = initial_locks;
+    ErtsMessage *mp;
+    Eterm res, msg, ref;
+    Uint hsz;
+    Eterm *hp;
+    ErlOffHeap *ohp;
+
+    hsz = 3 + ERTS_REF_THING_SIZE;
+
+    if (time_left > (Sint64) MAX_SMALL)
+        hsz += ERTS_SINT64_HEAP_SIZE(time_left);
+
+    mp = erts_alloc_message_heap(proc, &locks, hsz, &hp, &ohp);
+
+    write_ref_thing(hp, refn[0], refn[1], refn[2]);
+    ref = make_internal_ref(hp);
+    hp += ERTS_REF_THING_SIZE;
+
+    if (time_left < 0)
+        res = am_false;
+    else if (time_left <= (Sint64) MAX_SMALL)
+        res = make_small((Sint) time_left);
+    else
+        res = erts_sint64_to_big(time_left, &hp);
+
+    msg = TUPLE2(hp, ref, res);
+
+    erts_queue_message(proc, locks, mp, msg, am_clock_service);
+
+    locks &= ~initial_locks;
+    if (locks)
+        erts_smp_proc_unlock(proc, locks);
+
+    return am_ok;
+}
+
 static ERTS_INLINE Eterm
 access_sched_local_btm(Process *c_p, Eterm pid,
-		       Eterm tref, Uint32 *trefn,
-		       Uint32 *rrefn,
-		       int async, int cancel,
-		       int return_res,
-		       int info)
+                       Eterm tref, Uint32 *trefn,
+                       Uint32 *rrefn,
+                       int async, int cancel,
+                       int return_res,
+                       int info)
 {
     ErtsSchedulerData *esdp;
     ErtsHLTimerService *srv;
@@ -1747,111 +2153,40 @@ access_sched_local_btm(Process *c_p, Eterm pid,
     srv = esdp->timer_service;
 
     tmr = btm_rbt_lookup(srv->btm_tree, trefn);
-    if (tmr) {
-	if (!cancel) {
-	    erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state);
-	    if (state == ERTS_TMR_STATE_ACTIVE)
-		time_left = get_time_left(esdp, tmr->timeout);
-	}
-	else {
-	    int cncl_res = cancel_bif_timer(tmr);
-	    if (cncl_res) {
-
-		time_left = get_time_left(esdp, tmr->timeout);
 
-		if (cncl_res > 0)
-		    hl_timer_dec_refc(tmr, tmr->head.roflgs);
-
-		hlt_delete_timer(esdp, tmr);
-	    }
-	}
-    }
+    time_left = access_btm(tmr, (Uint32) esdp->no, esdp, cancel);
 
     if (!info)
-	return am_ok;
-
-    if (return_res) {
-	ERTS_HLT_ASSERT(c_p);
-	if (time_left < 0)
-	    return am_false;
-	else if (time_left <= (Sint64) MAX_SMALL)
-	    return make_small((Sint) time_left);
-	else {
-	    Uint hsz = ERTS_SINT64_HEAP_SIZE(time_left);
-	    Eterm *hp = HAlloc(c_p, hsz);
-	    return erts_sint64_to_big(time_left, &hp);
-	}
-    }
+        return am_ok;
 
     if (c_p) {
-	proc = c_p;
-	proc_locks = ERTS_PROC_LOCK_MAIN;
+        proc = c_p;
+        proc_locks = ERTS_PROC_LOCK_MAIN;
     }
     else {
-	proc = erts_proc_lookup(pid);
-	proc_locks = 0;
+        proc = erts_proc_lookup(pid);
+        proc_locks = 0;
     }
 
-    if (proc) {
-	Uint hsz;
-	ErtsMessage *mp;
-	Eterm *hp, msg, ref, result;
-	ErlOffHeap *ohp;
-	Uint32 *refn;
-#ifdef ERTS_HLT_DEBUG
-	Eterm *hp_end;
-#endif
-
-	hsz = ERTS_REF_THING_SIZE;
-	if (async) {
-	    refn = trefn; /* timer ref */
-	    hsz += 4; /* 3-tuple */
-	}
-	else {
-	    refn = rrefn; /* request ref */
-	    hsz += 3; /* 2-tuple */
-	}
-
-	ERTS_HLT_ASSERT(refn);
-
-	if (time_left > (Sint64) MAX_SMALL)
-	    hsz += ERTS_SINT64_HEAP_SIZE(time_left);
-
-	mp = erts_alloc_message_heap(proc, &proc_locks,
-				     hsz, &hp, &ohp);
-
-#ifdef ERTS_HLT_DEBUG
-	hp_end = hp + hsz;
-#endif
-
-	if (time_left < 0)
-	    result = am_false;
-	else if (time_left <= (Sint64) MAX_SMALL)
-	    result = make_small((Sint) time_left);
-	else
-	    result = erts_sint64_to_big(time_left, &hp);
-
-	write_ref_thing(hp,
-			refn[0],
-			refn[1],
-			refn[2]);
-	ref = make_internal_ref(hp);
-	hp += ERTS_REF_THING_SIZE;
-
-	msg = (async
-	       ? TUPLE3(hp, (cancel
-			     ? am_cancel_timer
-			     : am_read_timer), ref, result)
-	       : TUPLE2(hp, ref, result));
-
-	ERTS_HLT_ASSERT(hp + (async ? 4 : 3) == hp_end);
-
-	erts_queue_message(proc, proc_locks, mp, msg, am_clock_service);
-
-	if (c_p)
-	    proc_locks &= ~ERTS_PROC_LOCK_MAIN;
-	if (proc_locks)
-	    erts_smp_proc_unlock(proc, proc_locks);
+    if (!async) {
+        if (c_p)
+            return return_info(c_p, time_left);
+
+        if (proc)
+            return send_sync_info(proc, proc_locks,
+                                  rrefn, cancel, time_left);
+    }
+    else if (proc) {
+        Eterm ref;
+        Eterm heap[ERTS_REF_THING_SIZE];
+        if (is_value(tref))
+            ref = tref;
+        else {
+            write_ref_thing(&heap[0], trefn[0], trefn[1], trefn[2]);
+            ref = make_internal_ref(&heap[0]);
+        }
+        return send_async_info(proc, proc_locks,
+                               ref, cancel, time_left);
     }
 
     return am_ok;
@@ -1884,7 +2219,7 @@ bif_timer_access_request(void *vreq)
 static int
 try_access_sched_remote_btm(ErtsSchedulerData *esdp,
 			    Process *c_p, Uint32 sid,
-			    Uint32 *trefn,
+			    Eterm tref, Uint32 *trefn,
 			    int async, int cancel,
 			    int info, Eterm *resp)
 {
@@ -1903,84 +2238,45 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
     if (!tmr)
 	return 0;
 
-    if (!cancel) {
-	erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state);
-	if (state == ERTS_TMR_STATE_ACTIVE)
-	    time_left = get_time_left(esdp, tmr->timeout);
-	else
-	    time_left = -1;
-    }
-    else {
-	int cncl_res = cancel_bif_timer(tmr);
-	if (!cncl_res)
-	    time_left = -1;
-	else {
-	    time_left = get_time_left(esdp, tmr->timeout);
-	    if (cncl_res > 0)
-		queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr);
-	}
-    }
+    time_left = access_btm(tmr, sid, esdp, cancel);
 
-    if (!info) {
+    if (!info)
 	*resp = am_ok;
-	return 1;
-    }
-
-    if (!async) {
-	if (time_left < 0)
-	    *resp = am_false;
-	else if (time_left <= (Sint64) MAX_SMALL)
-	    *resp = make_small((Sint) time_left);
-	else {
-	    Uint hsz = ERTS_SINT64_HEAP_SIZE(time_left);
-	    Eterm *hp = HAlloc(c_p, hsz);
-	    *resp = erts_sint64_to_big(time_left, &hp);
-	}
-    }
-    else {
-	ErtsMessage *mp;
-	Eterm tag, res, msg, tref;
-	Uint hsz;
-	Eterm *hp;
-	ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN;
-	ErlOffHeap *ohp;
-
-	hsz = 4 + ERTS_REF_THING_SIZE;
-	if (time_left > (Sint64) MAX_SMALL)
-	    hsz += ERTS_SINT64_HEAP_SIZE(time_left);
-
-	mp = erts_alloc_message_heap(c_p, &proc_locks,
-				     hsz, &hp, &ohp);
-	if (cancel)
-	    tag = am_cancel_timer;
-	else
-	    tag = am_read_timer;
-
-	write_ref_thing(hp,
-			trefn[0],
-			trefn[1],
-			trefn[2]);
-	tref = make_internal_ref(hp);
-	hp += ERTS_REF_THING_SIZE;
-
-	if (time_left < 0)
-	    res = am_false;
-	else if (time_left <= (Sint64) MAX_SMALL)
-	    res = make_small((Sint) time_left);
-	else
-	    res = erts_sint64_to_big(time_left, &hp);
-
-	msg = TUPLE3(hp, tag, tref, res);
+    else if (!async)
+        *resp = return_info(c_p, time_left);
+    else
+        *resp = send_async_info(c_p, ERTS_PROC_LOCK_MAIN,
+                                tref, cancel, time_left);
 
-        erts_queue_message(c_p, proc_locks, mp, msg, am_clock_service);
+    return 1;
+}
 
-	proc_locks &= ~ERTS_PROC_LOCK_MAIN;
-	if (proc_locks)
-	    erts_smp_proc_unlock(c_p, proc_locks);
+static Eterm
+no_timer_result(Process *c_p, Eterm tref, int cancel, int async, int info)
+{
+    ErtsMessage *mp;
+    Uint hsz;
+    Eterm *hp, msg, ref, tag;
+    ErlOffHeap *ohp;
+    ErtsProcLocks locks;
 
-	*resp = am_ok;
-    }
-    return 1;
+    if (!async)
+        return am_false;
+    if (!info)
+        return am_ok;
+
+    hsz = 4;
+    hsz += NC_HEAP_SIZE(tref);
+    locks = ERTS_PROC_LOCK_MAIN;
+    mp = erts_alloc_message_heap(c_p, &locks, hsz, &hp, &ohp);
+    ref = STORE_NC(&hp, ohp, tref);
+    tag = cancel ? am_cancel_timer : am_read_timer;
+    msg = TUPLE3(hp, tag, ref, am_false);
+    erts_queue_message(c_p, locks, mp, msg, am_clock_service);
+    locks &= ~ERTS_PROC_LOCK_MAIN;
+    if (locks)
+        erts_smp_proc_unlock(c_p, locks);
+    return am_ok;
 }
 
 static BIF_RETTYPE
@@ -2014,7 +2310,7 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info)
 	ERTS_BIF_PREP_RET(ret, res);
     }
     else if (try_access_sched_remote_btm(esdp, c_p,
-					 sid, trefn,
+					 sid, tref, trefn,
 					 async, cancel,
 					 info, &res)) {
 	ERTS_BIF_PREP_RET(ret, res);
@@ -2093,11 +2389,11 @@ badarg:
     return ret;
 
 no_timer:
-    ERTS_BIF_PREP_RET(ret, am_false);
-    return ret;
-    
+    return no_timer_result(c_p, tref, cancel, async, info);
 }
 
+#endif /* !ERTS_MAGIC_REF_BIF_TIMERS */
+
 static ERTS_INLINE int
 bool_arg(Eterm val, int *argp)
 {
@@ -2174,14 +2470,20 @@ exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp)
     roflgs = tmr->head.roflgs;
     sid = roflgs & ERTS_TMR_ROFLG_SID_MASK;
 
-    ERTS_HLT_ASSERT(sid == erts_get_ref_numbers_thr_id(tmr->btm.refn));
+    ERTS_HLT_ASSERT(sid == erts_get_ref_numbers_thr_id(ERTS_BTM_HLT2REFN(tmr)));
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+    ERTS_HLT_ASSERT(tmr->btm.proc_list.next);
+#else
     ERTS_HLT_ASSERT(tmr->btm.proc_tree.parent
 		    != ERTS_HLT_PFIELD_NOT_IN_TABLE);
-
     tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+#endif
 
     if (sid == (Uint32) esdp->no) {
 	if (state == ERTS_TMR_STATE_ACTIVE) {
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+            btm_clear_magic_binary(tmr);
+#endif
 	    if (tmr->btm.bp)
 		free_message_buffer(tmr->btm.bp);
 	    hlt_delete_timer(esdp, tmr);
@@ -2190,6 +2492,9 @@ exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp)
     }
     else {
 	if (state == ERTS_TMR_STATE_ACTIVE) {
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+            btm_clear_magic_binary(tmr);
+#endif
 	    if (tmr->btm.bp)
 		free_message_buffer(tmr->btm.bp);
 	    queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr);
@@ -2205,17 +2510,29 @@ exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp)
 #  define ERTS_BTM_MAX_DESTROY_LIMIT 50
 #endif
 
+#ifndef ERTS_MAGIC_REF_BIF_TIMERS
 typedef struct {
     ErtsBifTimers *bif_timers;
     union {
 	proc_btm_rbt_yield_state_t proc_btm_yield_state;
     } u;
 } ErtsBifTimerYieldState;
+#endif
 
-int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp)
+int erts_cancel_bif_timers(Process *p, ErtsBifTimers **btm, void **vyspp)
 {
     ErtsSchedulerData *esdp = erts_proc_sched_data(p);
-    ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}};
+
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+
+    return proc_btm_list_foreach_destroy_yielding(btm,
+                                                  exit_cancel_bif_timer,
+                                                  (void *) esdp,
+                                                  ERTS_BTM_MAX_DESTROY_LIMIT);
+
+#else /* !ERTS_MAGIC_REF_BIF_TIMERS */
+
+    ErtsBifTimerYieldState ys = {*btm, {ERTS_RBT_YIELD_STAT_INITER}};
     ErtsBifTimerYieldState *ysp;
     int res;
 
@@ -2247,6 +2564,8 @@ int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp)
     }
 
     return res;
+
+#endif /* !ERTS_MAGIC_REF_BIF_TIMERS */
 }
 
 static ERTS_INLINE int
@@ -2663,6 +2982,11 @@ btm_print(ErtsHLTimer *tmr, void *vbtmp)
     ErtsMonotonicTime left;
     Eterm receiver;
 
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+    if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR))
+        return;
+#endif
+
     if (tmr->timeout <= btmp->now)
 	left = 0;
     else
@@ -2698,7 +3022,11 @@ erts_print_bif_timer_info(fmtfn_t to, void *to_arg)
     for (six = 0; six < erts_no_schedulers; six++) {
 	ErtsHLTimerService *srv =
 	    erts_aligned_scheduler_data[six].esd.timer_service;
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+	time_rbt_foreach(srv->time_tree, btm_print, (void *) &btmp);
+#else
 	btm_rbt_foreach(srv->btm_tree, btm_print, (void *) &btmp);
+#endif
     }
 }
 
@@ -2713,6 +3041,10 @@ typedef struct {
 static void
 debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd)
 {
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+    if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR))
+        return;
+#endif
     if (erts_smp_atomic32_read_nob(&tmr->state) == ERTS_TMR_STATE_ACTIVE) {
 	ErtsBTMForeachDebug *btmfd = (ErtsBTMForeachDebug *) vbtmfd;
 	(*btmfd->func)(((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
@@ -2743,9 +3075,16 @@ erts_debug_bif_timer_foreach(void (*func)(Eterm,
     for (six = 0; six < erts_no_schedulers; six++) {
 	ErtsHLTimerService *srv =
 	    erts_aligned_scheduler_data[six].esd.timer_service;
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+        time_rbt_foreach(srv->time_tree,
+                         debug_btm_foreach,
+                         (void *) &btmfd);
+            
+#else
 	btm_rbt_foreach(srv->btm_tree,
 			debug_btm_foreach,
 			(void *) &btmfd);
+#endif
     }
 }
 
@@ -2868,7 +3207,9 @@ st_hdbg_func(ErtsHLTimer *tmr, void *vhdbg)
     }
     ERTS_HLT_ASSERT(tmr->time.tree.u.l.next->time.tree.u.l.prev == tmr);
     ERTS_HLT_ASSERT(tmr->time.tree.u.l.prev->time.tree.u.l.next == tmr);
-    ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr);
+#ifndef ERTS_MAGIC_REF_BIF_TIMERS
+    ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, ERTS_BTM_HLT2REFN(tmr)) == tmr);
+#endif
 }
 
 static void
@@ -2897,8 +3238,10 @@ tt_hdbg_func(ErtsHLTimer *tmr, void *vhdbg)
 				& ~ERTS_HLT_PFLGS_MASK);
 	ERTS_HLT_ASSERT(tmr == prnt);
     }
+#ifndef ERTS_MAGIC_REF_BIF_TIMERS
     if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)
-	ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr);
+	ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, ERTS_BTM_HLT2REFN(tmr)) == tmr);
+#endif
     if (tmr->time.tree.same_time) {
 	ErtsHdbgHLT st_hdbg;
 	st_hdbg.srv = hdbg->srv;
@@ -2964,6 +3307,7 @@ hdbg_chk_srv(ErtsHLTimerService *srv)
 	time_rbt_foreach(srv->time_tree, tt_hdbg_func, (void *) &hdbg);
 	ERTS_HLT_ASSERT(hdbg.found_root);
     }
+#ifndef ERTS_MAGIC_REF_BIF_TIMERS
     if (srv->btm_tree) {
 	ErtsHdbgHLT hdbg;
 	hdbg.srv = srv;
@@ -2972,6 +3316,7 @@ hdbg_chk_srv(ErtsHLTimerService *srv)
 	btm_rbt_foreach(srv->btm_tree, bt_hdbg_func, (void *) &hdbg);
 	ERTS_HLT_ASSERT(hdbg.found_root);
     }
+#endif
 }
 
 #endif /* ERTS_HLT_HARD_DEBUG */
diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h
index 9cdcd581a0..33cdf217d7 100644
--- a/erts/emulator/beam/erl_hl_timer.h
+++ b/erts/emulator/beam/erl_hl_timer.h
@@ -56,7 +56,7 @@ void erts_cancel_proc_timer(Process *);
 void erts_set_port_timer(Port *, Sint64);
 void erts_cancel_port_timer(Port *);
 Sint64 erts_read_port_timer(Port *);
-int erts_cancel_bif_timers(Process *, ErtsBifTimers *, void **);
+int erts_cancel_bif_timers(Process *, ErtsBifTimers **, void **);
 int erts_detach_accessor_bif_timers(Process *, ErtsBifTimers *, void **);
 ErtsHLTimerService *erts_create_timer_service(void);
 void erts_hl_timer_init(void);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index cb95eab67e..5baf533acd 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -13810,7 +13810,7 @@ erts_continue_exit_process(Process *p)
 
     ASSERT(erts_proc_read_refc(p) > 0);
     if (p->bif_timers) {
-	if (erts_cancel_bif_timers(p, p->bif_timers, &p->u.terminate)) {
+	if (erts_cancel_bif_timers(p, &p->bif_timers, &p->u.terminate)) {
 	    ASSERT(erts_proc_read_refc(p) > 0);
 	    goto yield;
 	}
-- 
cgit v1.2.3


From a09f867979d8039434271a2177c90aabced3a255 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 3 Feb 2017 03:00:37 +0100
Subject: Use timer wheel for short BIF timers

---
 erts/emulator/beam/erl_hl_timer.c | 1055 +++++++++++++++++++++++--------------
 erts/emulator/beam/erl_hl_timer.h |    2 +-
 2 files changed, 662 insertions(+), 395 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 6d570d28ec..102a739b13 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -29,7 +29,7 @@
 #  include "config.h"
 #endif
 
-#define ERTS_MAGIC_REF_BIF_TIMERS
+/* #define ERTS_MAGIC_REF_BIF_TIMERS */
 
 #include "sys.h"
 #include "global.h"
@@ -129,10 +129,12 @@ typedef struct ErtsHLTimer_ ErtsHLTimer;
 
 #define ERTS_HLT_PFIELD_NOT_IN_TABLE	(~((UWord) 0))
 
+typedef struct ErtsBifTimer_ ErtsBifTimer;
+
 typedef struct {
-    ErtsHLTimer *next;
-    ErtsHLTimer *prev;    
-} ErtsHLTimerList;
+    ErtsBifTimer *next;
+    ErtsBifTimer *prev;    
+} ErtsBifTimerList;
 
 typedef struct {
     UWord parent; /* parent pointer and flags... */
@@ -150,6 +152,11 @@ typedef struct {
 } ErtsHLTimerTimeTree;
 
 typedef struct {
+    UWord parent; /* parent pointer and flags... */
+    ErtsBifTimer *right;
+    ErtsBifTimer *left;
+} ErtsBifTimerTree;
+
 typedef struct {
     Uint32 roflgs;
     erts_smp_atomic32_t refc;
@@ -176,42 +183,59 @@ struct ErtsHLTimer_ {
 #ifdef ERTS_HLT_HARD_DEBUG
     int pending_timeout;
 #endif
+};
 
-    erts_smp_atomic32_t state;
+typedef struct {
+    ErtsTmrHead head; /* NEED to be first! */
+    union {
+        Eterm name;
+	Process *proc;
+        Port *port;
+	void (*callback)(void *);
+    } receiver;
+    ErtsTWheelTimer tw_tmr;
+} ErtsTWTimer;
 
-    /* BIF timer only fields follow... */
+struct ErtsBifTimer_ {
+    union {
+        ErtsTmrHead head;
+        ErtsHLTimer hlt;
+        ErtsTWTimer twt;
+    } type;
     struct {
+        erts_smp_atomic32_t state;
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
         ErtsMagicBinary *mbin;
         ErtsHLTimerList proc_list;
 #else
 	Uint32 refn[ERTS_REF_NUMBERS];
-	ErtsHLTimerTree proc_tree;
-	ErtsHLTimerTree tree;
+	ErtsBifTimerTree proc_tree;
+	ErtsBifTimerTree tree;
 #endif
-	Eterm message;
-	ErlHeapFragment *bp;
+        Eterm message;
+        ErlHeapFragment *bp;
     } btm;
 };
 
-#define ERTS_HL_PTIMER_SIZE offsetof(ErtsHLTimer, btm)
-#define ERTS_BIF_TIMER_SIZE sizeof(ErtsHLTimer)
-
-typedef struct {
-    ErtsTmrHead head; /* NEED to be first! */
-    union {
-	void *p;
-	void (*callback)(void *);
-    } u;
-    ErtsTWheelTimer tw_tmr;
-} ErtsTWTimer;
-
 typedef union {
     ErtsTmrHead head;
     ErtsHLTimer hlt;
     ErtsTWTimer twt;
+    ErtsBifTimer btm;
 } ErtsTimer;
 
+typedef ErtsTimer *(*ErtsCreateTimerFunc)(ErtsSchedulerData *esdp,
+                                          ErtsMonotonicTime timeout_pos,
+                                          int short_time, ErtsTmrType type,
+                                          void *rcvrp, Eterm rcvr,
+                                          Eterm msg,
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+                                          ErtsMagicBinary *mbin,
+#else
+                                          Uint32 *refn,
+#endif
+                                          void (*callback)(void *), void *arg);
+
 #ifdef SMALL_MEMORY
 #define BIF_TIMER_PREALC_SZ	10
 #define PTIMER_PREALC_SZ	10
@@ -221,7 +245,7 @@ typedef union {
 #endif
 
 ERTS_SCHED_PREF_PALLOC_IMPL(bif_timer_pre,
-			    ErtsHLTimer,
+			    ErtsBifTimer,
 			    BIF_TIMER_PREALC_SZ)
 
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(tw_timer,
@@ -293,7 +317,7 @@ struct ErtsHLTimerService_ {
 #endif
     ErtsHLTimer *time_tree;
 #ifndef ERTS_MAGIC_REF_BIF_TIMERS
-    ErtsHLTimer *btm_tree;
+    ErtsBifTimer *btm_tree;
 #endif
     ErtsHLTimer *next_timeout;
     ErtsYieldingTimeoutState yield;
@@ -523,7 +547,7 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x)
 #ifndef ERTS_MAGIC_REF_BIF_TIMERS
 
 #define ERTS_RBT_PREFIX btm
-#define ERTS_RBT_T ErtsHLTimer
+#define ERTS_RBT_T ErtsBifTimer
 #define ERTS_RBT_KEY_T Uint32 *
 #define ERTS_RBT_FLAGS_T UWord
 #define ERTS_RBT_INIT_EMPTY_TNODE(T)					\
@@ -549,7 +573,7 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x)
 	(T)->btm.tree.parent |= (F);					\
     } while (0)
 #define ERTS_RBT_GET_PARENT(T)						\
-    ((ErtsHLTimer *) ((T)->btm.tree.parent & ~ERTS_HLT_PFLGS_MASK))
+    ((ErtsBifTimer *) ((T)->btm.tree.parent & ~ERTS_HLT_PFLGS_MASK))
 #define ERTS_RBT_SET_PARENT(T, P)					\
     do {								\
 	ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0);	\
@@ -578,9 +602,9 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x)
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
 
 static ERTS_INLINE void
-proc_btm_list_insert(ErtsHLTimer **list, ErtsHLTimer *x)
+proc_btm_list_insert(ErtsBifTimer **list, ErtsBifTimer *x)
 {
-    ErtsHLTimer *y = *list;
+    ErtsBifTimer *y = *list;
     if (!y) {
         x->btm.proc_list.next = x;
         x->btm.proc_list.prev = x;
@@ -596,9 +620,9 @@ proc_btm_list_insert(ErtsHLTimer **list, ErtsHLTimer *x)
 }
 
 static ERTS_INLINE void
-proc_btm_list_delete(ErtsHLTimer **list, ErtsHLTimer *x)
+proc_btm_list_delete(ErtsBifTimer **list, ErtsBifTimer *x)
 {
-    ErtsHLTimer *y = *list;
+    ErtsBifTimer *y = *list;
     if (y == x && x->btm.proc_list.next == x) {
         ERTS_HLT_ASSERT(x->btm.proc_list.prev == x);
         *list = NULL;
@@ -615,13 +639,13 @@ proc_btm_list_delete(ErtsHLTimer **list, ErtsHLTimer *x)
 }
 
 static ERTS_INLINE int
-proc_btm_list_foreach_destroy_yielding(ErtsHLTimer **list,
-                                       void (*destroy)(ErtsHLTimer *, void *),
+proc_btm_list_foreach_destroy_yielding(ErtsBifTimer **list,
+                                       void (*destroy)(ErtsBifTimer *, void *),
                                        void *arg,
                                        int limit)
 {
     int i;
-    ErtsHLTimer *first, *last;
+    ErtsBifTimer *first, *last;
 
     first = *list;
     if (!first)
@@ -629,7 +653,7 @@ proc_btm_list_foreach_destroy_yielding(ErtsHLTimer **list,
 
     last = first->btm.proc_list.prev;
     for (i = 0; i < limit; i++) {
-        ErtsHLTimer *x = last;
+        ErtsBifTimer *x = last;
         last = last->btm.proc_list.prev;
         (*destroy)(x, arg);
         x->btm.proc_list.next = NULL;
@@ -647,7 +671,7 @@ proc_btm_list_foreach_destroy_yielding(ErtsHLTimer **list,
 #else /* !ERTS_MAGIC_REF_BIF_TIMERS */
 
 #define ERTS_RBT_PREFIX proc_btm
-#define ERTS_RBT_T ErtsHLTimer
+#define ERTS_RBT_T ErtsBifTimer
 #define ERTS_RBT_KEY_T Uint32 *
 #define ERTS_RBT_FLAGS_T UWord
 #define ERTS_RBT_INIT_EMPTY_TNODE(T)					\
@@ -673,7 +697,7 @@ proc_btm_list_foreach_destroy_yielding(ErtsHLTimer **list,
 	(T)->btm.proc_tree.parent |= (F);				\
     } while (0)
 #define ERTS_RBT_GET_PARENT(T)						\
-    ((ErtsHLTimer *) ((T)->btm.proc_tree.parent & ~ERTS_HLT_PFLGS_MASK))
+    ((ErtsBifTimer *) ((T)->btm.proc_tree.parent & ~ERTS_HLT_PFLGS_MASK))
 #define ERTS_RBT_SET_PARENT(T, P)					\
     do {								\
 	ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0);	\
@@ -738,8 +762,8 @@ erts_timer_type_size(ErtsAlcType_t type)
 {
     switch (type) {
     case ERTS_ALC_T_LL_PTIMER: return sizeof(ErtsTWTimer);
-    case ERTS_ALC_T_HL_PTIMER: return ERTS_HL_PTIMER_SIZE;
-    case ERTS_ALC_T_BIF_TIMER: return ERTS_BIF_TIMER_SIZE;
+    case ERTS_ALC_T_HL_PTIMER: return sizeof(ErtsHLTimer);
+    case ERTS_ALC_T_BIF_TIMER: return sizeof(ErtsBifTimer);
     default: ERTS_INTERNAL_ERROR("Unknown type");
     }
     return 0;
@@ -798,6 +822,111 @@ port_timeout_common(Port *port, void *tmr)
     return 0;
 }
 
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+
+static erts_smp_atomic_t *
+mbin_to_btmref__(ErtsMagicBinary *mbin)
+{
+    return erts_smp_binary_to_magic_indirection((Binary *) mbin);
+}
+
+static ERTS_INLINE void
+magic_binary_init(ErtsMagicBinary *mbin, ErtsBifTimer *tmr)
+{
+    erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin);
+    erts_smp_atomic_init_nob(aptr, (erts_aint_t) tmr);
+}
+
+static ERTS_INLINE ErtsBifTimer *
+magic_binary_to_btm(ErtsMagicBinary *mbin)
+{
+    erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin);
+    ErtsBifTimer *tmr = (ErtsBifTimer *) erts_smp_atomic_read_nob(aptr);
+    ERTS_HLT_ASSERT(!tmr || tmr->btm.mbin == mbin);
+    return tmr;
+}
+
+#endif /* ERTS_MAGIC_REF_BIF_TIMERS */
+
+static ERTS_INLINE erts_aint_t
+init_btm_specifics(ErtsSchedulerData *esdp,
+                   ErtsBifTimer *tmr, Eterm msg,
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+                   ErtsMagicBinary *mbin
+#else
+                   Uint32 *refn
+#endif
+    )
+{
+    Uint hsz = is_immed(msg) ? ((Uint) 0) : size_object(msg);
+    int refc;
+    if (!hsz) {
+        tmr->btm.message = msg;
+        tmr->btm.bp = NULL;
+    }
+    else {
+        ErlHeapFragment *bp = new_message_buffer(hsz);
+        Eterm *hp = bp->mem;
+        tmr->btm.message = copy_struct(msg, hsz, &hp, &bp->off_heap);
+        tmr->btm.bp = bp;
+    }
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+    refc = 1;
+    tmr->btm.mbin = mbin;
+    erts_refc_inc(&mbin->refc, 1);
+    magic_binary_init(mbin, tmr);
+    tmr->btm.proc_list.next = NULL;
+#else
+    refc = 0;
+    tmr->btm.refn[0] = refn[0];
+    tmr->btm.refn[1] = refn[1];
+    tmr->btm.refn[2] = refn[2];
+
+    tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+
+    btm_rbt_insert(&esdp->timer_service->btm_tree, tmr);
+#endif
+
+    erts_smp_atomic32_init_nob(&tmr->btm.state, ERTS_TMR_STATE_ACTIVE);
+    return refc; /* refc from magic binary... */
+}
+
+static void tw_bif_timer_timeout(void *vbtmp);
+
+static ERTS_INLINE void
+timer_destroy(ErtsTimer *tmr, int twt, int btm)
+{
+    if (!btm) {
+        if (twt)
+            tw_timer_free(&tmr->twt);
+        else
+            erts_free(ERTS_ALC_T_HL_PTIMER, tmr);
+    }
+    else {
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+        Binary *bp = (Binary *) tmr->btm.btm.mbin;
+        if (erts_refc_dectest(&bp->refc, 0) == 0)
+            erts_bin_free(bp);
+#endif
+	if (tmr->head.roflgs & ERTS_TMR_ROFLG_PRE_ALC)
+	    bif_timer_pre_free(&tmr->btm);
+	else
+	    erts_free(ERTS_ALC_T_BIF_TIMER, &tmr->btm);
+    }
+}
+
+static ERTS_INLINE void
+timer_pre_dec_refc(ErtsTimer *tmr)
+{
+#ifdef ERTS_HLT_DEBUG
+    erts_aint_t refc;
+    refc = erts_smp_atomic32_dec_read_nob(&tmr->head.refc);
+    ERTS_HLT_ASSERT(refc > 0);
+#else
+    erts_smp_atomic32_dec_nob(&tmr->head.refc);
+#endif
+}
+
 /*
  * Basic timer wheel timer stuff
  */
@@ -805,26 +934,39 @@ port_timeout_common(Port *port, void *tmr)
 static void
 scheduled_tw_timer_destroy(void *vtmr)
 {
-    tw_timer_free((ErtsTWTimer *) vtmr);
+    ErtsTimer * tmr = (ErtsTimer *) vtmr;
+    int btm = !!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR);
+    timer_destroy((ErtsTimer *) vtmr, 1, btm);
 }
 
 static void
 schedule_tw_timer_destroy(ErtsTWTimer *tmr)
 {
+    Uint size;
     /*
      * Reference to process/port can be
      * dropped at once...
      */
     if (tmr->head.roflgs & ERTS_TMR_ROFLG_PROC)
-	erts_proc_dec_refc((Process *) tmr->u.p);
+	erts_proc_dec_refc(tmr->receiver.proc);
     else if (tmr->head.roflgs & ERTS_TMR_ROFLG_PORT)
-	erts_port_dec_refc((Port *) tmr->u.p);
+	erts_port_dec_refc(tmr->receiver.port);
+
+    if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR))
+	size = sizeof(ErtsHLTimer);
+    else {
+	/* Message buffer already dropped... */
+	size = sizeof(ErtsBifTimer);
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+        size += sizeof(ErtsMagicIndirectionWord);
+#endif
+    }
 
     erts_schedule_thr_prgr_later_cleanup_op(
 	scheduled_tw_timer_destroy,
 	(void *) tmr,
 	&tmr->tw_tmr.u.cleanup,
-	sizeof(ErtsTWTimer));
+	size);
 }
 
 static ERTS_INLINE void
@@ -840,7 +982,7 @@ static void
 tw_proc_timeout(void *vtwtp)
 {
     ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp;
-    Process *proc = (Process *) twtp->u.p;
+    Process *proc = twtp->receiver.proc;
     if (proc_timeout_common(proc, vtwtp))
 	tw_timer_dec_refc(twtp);
     tw_timer_dec_refc(twtp);
@@ -850,7 +992,7 @@ static void
 tw_port_timeout(void *vtwtp)
 {
     ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp;
-    Port *port = (Port *) twtp->u.p;
+    Port *port = twtp->receiver.port;
     if (port_timeout_common(port, vtwtp))
 	tw_timer_dec_refc(twtp);
     tw_timer_dec_refc(twtp);
@@ -874,53 +1016,78 @@ static void
 tw_callback_timeout(void *vtwtp)
 {
     ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp;
-    void (*callback)(void *) = twtp->u.callback;
+    void (*callback)(void *) = twtp->receiver.callback;
     void *arg = twtp->head.u.arg;
     tw_timer_dec_refc(twtp);
     (*callback)(arg);
 }
 
-static ErtsTWTimer *
-create_tw_timer(ErtsSchedulerData *esdp, 
-		ErtsTmrType type, void *p,
-		void (*callback)(void *), void *arg,
-		ErtsMonotonicTime timeout_pos)
+static ErtsTimer *
+create_tw_timer(ErtsSchedulerData *esdp,
+		ErtsMonotonicTime timeout_pos,
+		int short_time, ErtsTmrType type,
+		void *rcvrp, Eterm rcvr,
+		Eterm msg,
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+                ErtsMagicBinary *mbin,
+#else
+                Uint32 *refn,
+#endif
+		void (*callback)(void *), void *arg)
 {
     ErtsTWTimer *tmr;
     void (*timeout_func)(void *);
     void (*cancel_func)(void *);
     erts_aint32_t refc;
 
-    tmr = tw_timer_alloc();
-    erts_twheel_init_timer(&tmr->tw_tmr);
+    if (type != ERTS_TMR_BIF) {
+        tmr = tw_timer_alloc();
+        tmr->head.roflgs = 0;
+    }
+    else {
+        if (short_time) {
+            tmr = (ErtsTWTimer *) bif_timer_pre_alloc();
+            if (!tmr)
+                goto alloc_bif_timer;
+            tmr->head.roflgs = (ERTS_TMR_ROFLG_BIF_TMR
+                                | ERTS_TMR_ROFLG_PRE_ALC);
+        }
+        else {
+        alloc_bif_timer:
+            tmr = (ErtsTWTimer *) erts_alloc(ERTS_ALC_T_BIF_TIMER,
+                                             sizeof(ErtsBifTimer));
+            tmr->head.roflgs = ERTS_TMR_ROFLG_BIF_TMR;
+        }
+    }
 
-    tmr->head.roflgs = (Uint32) esdp->no;
-    ERTS_HLT_ASSERT((tmr->head.roflgs
-		     & ~ERTS_TMR_ROFLG_SID_MASK) == 0);
+    erts_twheel_init_timer(&tmr->tw_tmr);
+    tmr->head.roflgs |= (Uint32) esdp->no;
+    ERTS_HLT_ASSERT((((Uint32) esdp->no)
+                     & ~ERTS_TMR_ROFLG_SID_MASK) == 0);
 
     switch (type) {
 
     case ERTS_TMR_PROC:
-	tmr->u.p = p;
+	tmr->receiver.proc = (Process *) rcvrp;
 	tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC;
 	timeout_func = tw_proc_timeout;
 	cancel_func = tw_ptimer_cancel;
-	erts_proc_inc_refc((Process *) p);
+	erts_proc_inc_refc((Process *) rcvrp);
 	refc = 2;
 	break;
 
     case ERTS_TMR_PORT:
-	tmr->u.p = p;
+	tmr->receiver.port = (Port *) rcvrp;
 	tmr->head.roflgs |= ERTS_TMR_ROFLG_PORT;
 	timeout_func = tw_port_timeout;
 	cancel_func = tw_ptimer_cancel;
-	erts_port_inc_refc((Port *) p);
+	erts_port_inc_refc((Port *) rcvrp);
 	refc = 2;
 	break;
 
     case ERTS_TMR_CALLBACK:
 	tmr->head.u.arg = arg;
-	tmr->u.callback = callback;
+	tmr->receiver.callback = callback;
 
 	tmr->head.roflgs |= ERTS_TMR_ROFLG_CALLBACK;
 	timeout_func = tw_callback_timeout;
@@ -928,6 +1095,33 @@ create_tw_timer(ErtsSchedulerData *esdp,
 	refc = 1;
 	break;
 
+    case ERTS_TMR_BIF:
+
+	timeout_func = tw_bif_timer_timeout;
+	cancel_func = NULL;
+	if (is_internal_pid(rcvr)) {
+	    tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC;
+	    tmr->receiver.proc = (Process *) rcvrp;
+	    refc = 2;
+	}
+	else {
+	    ERTS_HLT_ASSERT(is_atom(rcvr));
+	    tmr->head.roflgs |= ERTS_TMR_ROFLG_REG_NAME;
+	    tmr->receiver.name = (Eterm) rcvr;
+	    refc = 1;
+	}
+
+        refc += init_btm_specifics(esdp,
+                                   (ErtsBifTimer *) tmr,
+                                   msg,
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+                                   mbin
+#else
+                                   refn
+#endif
+            );
+        break;
+
     default:
 	ERTS_INTERNAL_ERROR("Unsupported timer type");
 	return NULL;
@@ -942,36 +1136,19 @@ create_tw_timer(ErtsSchedulerData *esdp,
 			  tmr,
 			  timeout_pos);
 
-    return tmr;
+    return (ErtsTimer *) tmr;
 }
 
 /*
  * Basic high level timer stuff
  */
 
-static ERTS_INLINE void
-hl_timer_destroy(ErtsHLTimer *tmr)
-{
-    Uint32 roflgs = tmr->head.roflgs;
-    if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR))
-	erts_free(ERTS_ALC_T_HL_PTIMER, tmr);
-    else {
-#ifdef ERTS_MAGIC_REF_BIF_TIMERS
-        Binary *bp = (Binary *) tmr->btm.mbin;
-        if (erts_refc_dectest(&bp->refc, 0) == 0)
-            erts_bin_free(bp);
-#endif
-	if (roflgs & ERTS_TMR_ROFLG_PRE_ALC)
-	    bif_timer_pre_free(tmr);
-	else
-	    erts_free(ERTS_ALC_T_BIF_TIMER, tmr);
-    }
-}
-
 static void
 scheduled_hl_timer_destroy(void *vtmr)
 {
-    hl_timer_destroy((ErtsHLTimer *) vtmr);
+    ErtsTimer * tmr = (ErtsTimer *) vtmr;
+    int btm = !!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR);
+    timer_destroy((ErtsTimer *) vtmr, 0, btm);
 }
 
 static void
@@ -999,14 +1176,13 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs)
     }
 
     if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR))
-	size = ERTS_HL_PTIMER_SIZE;
-    else {
-	/*
-	 * Message buffer can be dropped at
-	 * once...
-	 */
 	size = sizeof(ErtsHLTimer);
-
+    else {
+	/* Message buffer already dropped... */
+	size = sizeof(ErtsBifTimer);
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+        size += sizeof(ErtsMagicIndirectionWord);
+#endif
     }
 
     erts_schedule_thr_prgr_later_cleanup_op(
@@ -1014,18 +1190,6 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs)
 	&tmr->time.cleanup, size);
 }
 
-static ERTS_INLINE void
-hl_timer_pre_dec_refc(ErtsHLTimer *tmr)
-{
-#ifdef ERTS_HLT_DEBUG
-    erts_aint_t refc;
-    refc = erts_smp_atomic32_dec_read_nob(&tmr->head.refc);
-    ERTS_HLT_ASSERT(refc > 0);
-#else
-    erts_smp_atomic32_dec_nob(&tmr->head.refc);
-#endif
-}
-
 static ERTS_INLINE void
 hl_timer_dec_refc(ErtsHLTimer *tmr, Uint32 roflgs)
 {
@@ -1060,32 +1224,17 @@ check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv)
 
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
 
-static erts_smp_atomic_t *
-mbin_to_btmref__(ErtsMagicBinary *mbin)
-{
-    return erts_smp_binary_to_magic_indirection((Binary *) mbin);
-}
-
-static ERTS_INLINE void
-magic_binary_init(ErtsMagicBinary *mbin, ErtsHLTimer *tmr)
-{
-    erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin);
-    erts_smp_atomic_init_nob(aptr, (erts_aint_t) tmr);
-}
-
-static ERTS_INLINE ErtsHLTimer *
-magic_binary_to_btm(ErtsMagicBinary *mbin)
+static int
+bif_timer_ref_destructor(Binary *unused)
 {
-    erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin);
-    ErtsHLTimer *tmr = (ErtsHLTimer *) erts_smp_atomic_read_nob(aptr);
-    ERTS_HLT_ASSERT(!tmr || tmr->btm.mbin == mbin);
-    return tmr;
+    return 1;
 }
 
 static ERTS_INLINE void
-btm_clear_magic_binary(ErtsHLTimer *tmr)
+btm_clear_magic_binary(ErtsBifTimer *tmr)
 {
     erts_smp_atomic_t *aptr = mbin_to_btmref__(tmr->btm.mbin);
+    Uint32 roflgs = tmr->type.head.roflgs;
 #ifdef ERTS_HLT_DEBUG
     erts_aint_t tval = erts_smp_atomic_xchg_nob(aptr,
                                                 (erts_aint_t) NULL);
@@ -1093,23 +1242,117 @@ btm_clear_magic_binary(ErtsHLTimer *tmr)
 #else
     erts_smp_atomic_set_nob(aptr, (erts_aint_t) NULL);
 #endif
-    hl_timer_dec_refc(tmr, tmr->head.roflgs);
+    if (roflgs & ERTS_TMR_ROFLG_HLT)
+        hl_timer_dec_refc(&tmr->type.hlt, roflgs);
+    else
+        tw_timer_dec_refc(&tmr->type.twt);
 }
 
-static int
-bif_timer_ref_destructor(Binary *unused)
+#endif /* ERTS_MAGIC_REF_BIF_TIMERS */
+
+static ERTS_INLINE void
+bif_timer_timeout(ErtsHLTimerService *srv, ErtsBifTimer *tmr,
+                  Uint32 roflgs, int is_hlt)
 {
-    return 1;
-}
+    erts_aint32_t state;
+
+    ERTS_HLT_ASSERT(tmr->type.head.roflgs == roflgs);    
+    ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR);
+    ERTS_HLT_ASSERT(!!is_hlt == !!(roflgs & ERTS_TMR_ROFLG_HLT));
+
+    state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state,
+					   ERTS_TMR_STATE_TIMED_OUT,
+					   ERTS_TMR_STATE_ACTIVE);
 
+    ERTS_HLT_ASSERT(state == ERTS_TMR_STATE_CANCELED
+		    || state == ERTS_TMR_STATE_ACTIVE);
+
+    if (state == ERTS_TMR_STATE_ACTIVE) {
+        Process *proc;
+        int dec_refc = 0;
+        Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME);
+
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+        btm_clear_magic_binary(tmr);
 #endif
 
+        if (is_reg_name) {
+            Eterm term;
+            if (is_hlt)
+                term = tmr->type.hlt.receiver.name;
+            else
+                term = tmr->type.twt.receiver.name;
+            ERTS_HLT_ASSERT(is_atom(term));
+            term = erts_whereis_name_to_id(NULL, term);
+            proc = erts_proc_lookup(term);
+        }
+        else {
+            ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PROC);
+            if (is_hlt)
+                proc = tmr->type.hlt.receiver.proc;
+            else
+                proc = tmr->type.twt.receiver.proc;
+            ERTS_HLT_ASSERT(proc);
+        }
+        if (proc) {
+            if (!ERTS_PROC_IS_EXITING(proc)) {
+                ErtsMessage *mp;
+                erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM);
+                mp = erts_alloc_message(0, NULL);
+                mp->data.heap_frag = tmr->btm.bp;
+                erts_queue_message(proc, ERTS_PROC_LOCK_BTM, mp,
+                                   tmr->btm.message, am_clock_service);
+                tmr->btm.bp = NULL;
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+                if (tmr->btm.proc_list.next) {
+                    proc_btm_list_delete(&proc->bif_timers, tmr);
+                    dec_refc = 1;
+                }
+#else
+                if (tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
+                    proc_btm_rbt_delete(&proc->bif_timers, tmr);
+                    tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+                    dec_refc = 1;
+                }
+#endif
+                erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
+            }
+            if (dec_refc)
+                timer_pre_dec_refc((ErtsTimer *) tmr);
+        }
+        if (tmr->btm.bp)
+            free_message_buffer(tmr->btm.bp);
+    }
+
+#ifndef ERTS_MAGIC_REF_BIF_TIMERS
+    if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
+	btm_rbt_delete(&srv->btm_tree, tmr);
+	tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+    }
+#endif
+
+
+}
+
+static void
+tw_bif_timer_timeout(void *vbtmp)
+{
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+    ErtsHLTimerService *srv = NULL;
+#else
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    ErtsHLTimerService *srv = esdp->timer_service;
+#endif
+    ErtsBifTimer *btmp = (ErtsBifTimer *) vbtmp;
+    bif_timer_timeout(srv, btmp, btmp->type.head.roflgs, 0);
+    tw_timer_dec_refc(&btmp->type.twt);
+}
 
-static ErtsHLTimer *
+static ErtsTimer *
 create_hl_timer(ErtsSchedulerData *esdp,
 		ErtsMonotonicTime timeout_pos,
 		int short_time, ErtsTmrType type,
-		void *rcvrp, Eterm rcvr, Eterm acsr,
+		void *rcvrp, Eterm rcvr,
 		Eterm msg,
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
                 ErtsMagicBinary *mbin,
@@ -1134,7 +1377,7 @@ create_hl_timer(ErtsSchedulerData *esdp,
     if (type != ERTS_TMR_BIF) {
 
 	tmr = erts_alloc(ERTS_ALC_T_HL_PTIMER,
-			 ERTS_HL_PTIMER_SIZE);
+                         sizeof(ErtsHLTimer));
 	tmr->timeout = timeout_pos;
 
 	switch (type) {
@@ -1170,19 +1413,18 @@ create_hl_timer(ErtsSchedulerData *esdp,
 
     }
     else { /* ERTS_TMR_BIF */
-	Uint hsz;
 
 	if (short_time) {
-	    tmr = bif_timer_pre_alloc();
+	    tmr = (ErtsHLTimer *) bif_timer_pre_alloc();
 	    if (!tmr)
 		goto alloc_bif_timer;
 	    roflgs |= ERTS_TMR_ROFLG_PRE_ALC;
 	}
 	else {
 	alloc_bif_timer:
-            tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER,
-                             ERTS_BIF_TIMER_SIZE);
-	}
+            tmr = (ErtsHLTimer *) erts_alloc(ERTS_ALC_T_BIF_TIMER,
+                                             sizeof(ErtsBifTimer));
+        }
 
 	tmr->timeout = timeout_pos;
 
@@ -1199,37 +1441,19 @@ create_hl_timer(ErtsSchedulerData *esdp,
 	    refc = 1;
 	}
 
-	hsz = is_immed(msg) ? ((Uint) 0) : size_object(msg);
-	if (!hsz) {
-	    tmr->btm.message = msg;
-	    tmr->btm.bp = NULL;
-	}
-	else {
-	    ErlHeapFragment *bp = new_message_buffer(hsz);
-	    Eterm *hp = bp->mem;
-	    tmr->btm.message = copy_struct(msg, hsz, &hp, &bp->off_heap);
-	    tmr->btm.bp = bp;
-	}
+        refc += init_btm_specifics(esdp,
+                                   (ErtsBifTimer *) tmr,
+                                   msg,
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
-        tmr->btm.mbin = mbin;
-        erts_refc_inc(&mbin->refc, 1);
-        magic_binary_init(mbin, tmr);
-        refc += 1; /* from magic binary... */
-	tmr->btm.proc_list.next = NULL;
+                                   mbin
 #else
-	tmr->btm.refn[0] = refn[0];
-	tmr->btm.refn[1] = refn[1];
-	tmr->btm.refn[2] = refn[2];
-
-	tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
-
-	btm_rbt_insert(&srv->btm_tree, tmr);
+                                   refn
 #endif
+            );
     }
 
     tmr->head.roflgs = roflgs;
     erts_smp_atomic32_init_nob(&tmr->head.refc, refc);
-    erts_smp_atomic32_init_nob(&tmr->state, ERTS_TMR_STATE_ACTIVE);
 
     if (!srv->next_timeout
 	|| tmr->timeout < srv->next_timeout->timeout) {
@@ -1256,72 +1480,7 @@ create_hl_timer(ErtsSchedulerData *esdp,
 
     ERTS_HLT_HDBG_CHK_SRV(srv);
 
-    return tmr;
-}
-
-static ERTS_INLINE void
-hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs)
-{
-    ErtsProcLocks proc_locks = ERTS_PROC_LOCKS_MSG_SEND;
-    Process *proc;
-    int queued_message = 0;
-    int dec_refc = 0;
-    Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME);
-    ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR);
-
-#ifdef ERTS_MAGIC_REF_BIF_TIMERS
-    btm_clear_magic_binary(tmr);
-#endif
-
-    if (is_reg_name) {
-	Eterm pid;
-	ERTS_HLT_ASSERT(is_atom(tmr->receiver.name));
-	pid = erts_whereis_name_to_id(NULL, tmr->receiver.name);
-	proc = erts_proc_lookup(pid);
-    }
-    else {
-	ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PROC);
-	ERTS_HLT_ASSERT(tmr->receiver.proc);
-
-	proc = tmr->receiver.proc;
-	proc_locks |= ERTS_PROC_LOCK_BTM;
-    }
-    if (proc) {
-	erts_smp_proc_lock(proc, proc_locks);
-	/*
-	 * If process is exiting, let it clean up
-	 * the btm tree by itself (it may be in
-	 * the middle of tree destruction).
-	 */
-	if (!ERTS_PROC_IS_EXITING(proc)) {
-	    ErtsMessage *mp = erts_alloc_message(0, NULL);
-	    mp->data.heap_frag = tmr->btm.bp;
-	    erts_queue_message(proc, proc_locks, mp,
-			       tmr->btm.message, am_clock_service);
-	    erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_SEND);
-	    queued_message = 1;
-	    proc_locks &= ~ERTS_PROC_LOCKS_MSG_SEND;
-	    tmr->btm.bp = NULL;
-#ifdef ERTS_MAGIC_REF_BIF_TIMERS
-            if (tmr->btm.proc_list.next) {
-                proc_btm_list_delete(&proc->bif_timers, tmr);
-		dec_refc = 1;
-            }
-#else
-	    if (tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
-		proc_btm_rbt_delete(&proc->bif_timers, tmr);
-		tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
-		dec_refc = 1;
-	    }
-#endif
-	}
-	if (proc_locks)
-	    erts_smp_proc_unlock(proc, proc_locks);
-	if (dec_refc)
-	    hl_timer_pre_dec_refc(tmr);
-    }
-    if (!queued_message && tmr->btm.bp)
-	free_message_buffer(tmr->btm.bp);
+    return (ErtsTimer *) tmr;
 }
 
 static ERTS_INLINE void
@@ -1340,49 +1499,27 @@ hlt_port_timeout(ErtsHLTimer *tmr)
 
 static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv)
 {
-#if !defined(ERTS_MAGIC_REF_BIF_TIMERS) || defined(ERTS_HLT_HARD_DEBUG)
     ErtsHLTimerService *srv = (ErtsHLTimerService *) vsrv;
-#endif
     Uint32 roflgs;
-    erts_aint32_t state;
 
     ERTS_HLT_HDBG_CHK_SRV(srv);
 
     roflgs = tmr->head.roflgs;
     ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_HLT);
 
-    state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state,
-					   ERTS_TMR_STATE_TIMED_OUT,
-					   ERTS_TMR_STATE_ACTIVE);
-
-    ERTS_HLT_ASSERT(state == ERTS_TMR_STATE_CANCELED
-		    || state == ERTS_TMR_STATE_ACTIVE);
-
-    if (state == ERTS_TMR_STATE_ACTIVE) {
-
-	if (roflgs & ERTS_TMR_ROFLG_BIF_TMR)
-	    hlt_bif_timer_timeout(tmr, roflgs);
-	else if (roflgs & ERTS_TMR_ROFLG_PROC)
-	    hlt_proc_timeout(tmr);
-	else if (roflgs & ERTS_TMR_ROFLG_PORT)
-	    hlt_port_timeout(tmr);
-	else {
-	    ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_CALLBACK);
-	    (*tmr->receiver.callback)(tmr->head.u.arg);
-	}
-
+    if (roflgs & ERTS_TMR_ROFLG_BIF_TMR)
+        bif_timer_timeout(srv, (ErtsBifTimer *) tmr, roflgs, 1);
+    else if (roflgs & ERTS_TMR_ROFLG_PROC)
+        hlt_proc_timeout(tmr);
+    else if (roflgs & ERTS_TMR_ROFLG_PORT)
+        hlt_port_timeout(tmr);
+    else {
+        ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_CALLBACK);
+        (*tmr->receiver.callback)(tmr->head.u.arg);
     }
 
     tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
 
-#ifndef ERTS_MAGIC_REF_BIF_TIMERS
-    if ((roflgs & ERTS_TMR_ROFLG_BIF_TMR)
-	&& tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
-	btm_rbt_delete(&srv->btm_tree, tmr);
-	tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
-    }
-#endif
-
     ERTS_HLT_HDBG_CHK_SRV(srv);
 
     hl_timer_dec_refc(tmr, roflgs);
@@ -1460,17 +1597,6 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr)
 
     ERTS_HLT_HDBG_CHK_SRV(srv);
 
-#ifndef ERTS_MAGIC_REF_BIF_TIMERS
-    if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR) {
-
-	if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
-	    btm_rbt_delete(&srv->btm_tree, tmr);
-	    tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
-	}
-
-    }
-#endif
-
     if (tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE) {
 	/* Already removed... */
 	ERTS_HLT_HDBG_CHK_SRV(srv);
@@ -1541,6 +1667,17 @@ cleanup_sched_local_canceled_timer(ErtsSchedulerData *esdp,
     ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data());
     ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK)
 		    == (Uint32) esdp->no);
+
+#ifndef ERTS_MAGIC_REF_BIF_TIMERS
+    if (roflgs & ERTS_TMR_ROFLG_BIF_TMR) {
+        ErtsBifTimer *btm = (ErtsBifTimer *) tmr;
+	if (btm->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
+	    btm_rbt_delete(&esdp->timer_service->btm_tree, btm);
+	    btm->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+	}
+    }
+#endif
+
     if (roflgs & ERTS_TMR_ROFLG_HLT) {
 	hlt_delete_timer(esdp, &tmr->hlt);
 	hl_timer_dec_refc(&tmr->hlt, roflgs);
@@ -1813,18 +1950,18 @@ Uint erts_bif_timer_memory_size(void)
 }
 
 static BIF_RETTYPE
-setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos,
-		int short_time, Eterm rcvr, Eterm acsr,
-		Eterm msg, int wrap)
+setup_bif_timer(Process *c_p, int twheel, ErtsMonotonicTime timeout_pos,
+		int short_time, Eterm rcvr, Eterm msg, int wrap)
 {
     BIF_RETTYPE ret;
     Eterm ref, tmo_msg, *hp;
-    ErtsHLTimer *tmr;
+    ErtsBifTimer *tmr;
     ErtsSchedulerData *esdp;
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
     Binary *mbin;
 #endif
     Eterm tmp_hp[4];
+    ErtsCreateTimerFunc create_timer;
 
     if (is_not_internal_pid(rcvr) && is_not_atom(rcvr))
 	goto badarg;
@@ -1846,14 +1983,16 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos,
 
     tmo_msg = wrap ? TUPLE3(tmp_hp, am_timeout, ref, msg) : msg;
 
-    tmr = create_hl_timer(esdp, timeout_pos, short_time,
-			  ERTS_TMR_BIF, NULL,  rcvr, acsr, tmo_msg,
+    create_timer = twheel ? create_tw_timer : create_hl_timer;
+    tmr = (ErtsBifTimer *) create_timer(esdp, timeout_pos,
+                                        short_time, ERTS_TMR_BIF,
+                                        NULL, rcvr, tmo_msg,
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
-                          (ErtsMagicBinary *) mbin,
+                                        (ErtsMagicBinary *) mbin,
 #else
-                          internal_ordinary_ref_numbers(ref),
+                                        internal_ordinary_ref_numbers(ref),
 #endif
-                          NULL, NULL);
+                                        NULL, NULL);
 
     if (is_internal_pid(rcvr)) {
 	Process *proc = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
@@ -1862,11 +2001,19 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos,
 	if (!proc) {
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
             btm_clear_magic_binary(tmr);
+#else
+            if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
+                btm_rbt_delete(&esdp->timer_service->btm_tree, tmr);
+                tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+            }
 #endif
 	    if (tmr->btm.bp)
 		free_message_buffer(tmr->btm.bp);
-	    hlt_delete_timer(esdp, tmr);
-	    hl_timer_destroy(tmr);
+            if (twheel)
+                cancel_tw_timer(esdp, &tmr->type.twt);
+            else
+                hlt_delete_timer(esdp, &tmr->type.hlt);
+            timer_destroy((ErtsTimer *) tmr, twheel, 1);
 	}
 	else {
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
@@ -1875,7 +2022,10 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos,
 	    proc_btm_rbt_insert(&proc->bif_timers, tmr);
 #endif
 	    erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
-	    tmr->receiver.proc = proc;
+            if (twheel)
+                tmr->type.twt.receiver.proc = proc;
+            else
+                tmr->type.hlt.receiver.proc = proc;
 	}
     }
 
@@ -1889,13 +2039,13 @@ badarg:
 }
 
 static int
-cancel_bif_timer(ErtsHLTimer *tmr)
+cancel_bif_timer(ErtsBifTimer *tmr)
 {
     erts_aint_t state;
     Uint32 roflgs;
     int res;
 
-    state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state,
+    state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state,
 					   ERTS_TMR_STATE_CANCELED,
 					   ERTS_TMR_STATE_ACTIVE);
     if (state != ERTS_TMR_STATE_ACTIVE)
@@ -1910,10 +2060,14 @@ cancel_bif_timer(ErtsHLTimer *tmr)
 
     res = -1;
 
-    roflgs = tmr->head.roflgs;
+    roflgs = tmr->type.head.roflgs;
     if (roflgs & ERTS_TMR_ROFLG_PROC) {
-	Process *proc = tmr->receiver.proc;
-	ERTS_HLT_ASSERT(!(tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME));
+	Process *proc;
+
+        proc = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT)
+                ? tmr->type.hlt.receiver.proc
+                : tmr->type.twt.receiver.proc);
+	ERTS_HLT_ASSERT(!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME));
 
 	erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM);
 	/*
@@ -1941,18 +2095,25 @@ cancel_bif_timer(ErtsHLTimer *tmr)
 }
 
 static ERTS_INLINE Sint64
-access_btm(ErtsHLTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel)
+access_btm(ErtsBifTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel)
 {
     int cncl_res;
     Sint64 time_left;
+    ErtsMonotonicTime timeout;
+    int is_hlt;
 
     if (!tmr)
         return -1;
 
+    is_hlt = !!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT);
+    timeout = (is_hlt
+               ? tmr->type.hlt.timeout
+               : tmr->type.twt.tw_tmr.timeout_pos);
+
     if (!cancel) {
-        erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state);
+        erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->btm.state);
         if (state == ERTS_TMR_STATE_ACTIVE)
-            return get_time_left(esdp, tmr->timeout);
+            return get_time_left(esdp, timeout);
         return -1;
     }
 
@@ -1960,17 +2121,29 @@ access_btm(ErtsHLTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel)
     if (!cncl_res)
         return -1;
 
-    time_left = get_time_left(esdp, tmr->timeout);
+    time_left = get_time_left(esdp, timeout);
 
     if (sid != (Uint32) esdp->no) {
         if (cncl_res > 0)
             queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr);
     }
     else {
-        if (cncl_res > 0)
-            hl_timer_dec_refc(tmr, tmr->head.roflgs);
-
-        hlt_delete_timer(esdp, tmr);
+#ifndef ERTS_MAGIC_REF_BIF_TIMERS
+        if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
+	    btm_rbt_delete(&esdp->timer_service->btm_tree, tmr);
+	    tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+	}
+#endif
+        if (is_hlt) {
+            if (cncl_res > 0)
+                hl_timer_dec_refc(&tmr->type.hlt, tmr->type.hlt.head.roflgs);
+            hlt_delete_timer(esdp, &tmr->type.hlt);
+        }
+        else {
+            if (cncl_res > 0)
+                tw_timer_dec_refc(&tmr->type.twt);
+            cancel_tw_timer(esdp, &tmr->type.twt);
+        }
     }
 
     return time_left;
@@ -2059,7 +2232,7 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info)
         if (mbin->destructor != bif_timer_ref_destructor)
             time_left = -1;
         else {
-            ErtsHLTimer *tmr;
+            ErtsBifTimer *tmr;
             Uint32 sid;
             tmr = magic_binary_to_btm(mbin);
             sid = erts_get_ref_numbers_thr_id(internal_magic_ref_numbers(tref));
@@ -2133,7 +2306,7 @@ access_sched_local_btm(Process *c_p, Eterm pid,
 {
     ErtsSchedulerData *esdp;
     ErtsHLTimerService *srv;
-    ErtsHLTimer *tmr;
+    ErtsBifTimer *tmr;
     Sint64 time_left;
     Process *proc;
     ErtsProcLocks proc_locks;
@@ -2223,7 +2396,7 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
 			    int async, int cancel,
 			    int info, Eterm *resp)
 {
-    ErtsHLTimer *tmr;
+    ErtsBifTimer *tmr;
     Sint64 time_left;
 
     ERTS_HLT_ASSERT(c_p);
@@ -2405,8 +2578,8 @@ bool_arg(Eterm val, int *argp)
 }
 
 static ERTS_INLINE int
-parse_bif_timer_options(Eterm option_list, int *async, int *info,
-			int *abs, Eterm *accessor)
+parse_bif_timer_options(Eterm option_list, int *async,
+                        int *info, int *abs)
 {
     Eterm list = option_list;
 
@@ -2416,8 +2589,6 @@ parse_bif_timer_options(Eterm option_list, int *async, int *info,
 	*info = 1;
     if (abs)
 	*abs = 0;
-    if (accessor)
-	*accessor = THE_NON_VALUE;
 
     while (is_list(list)) {
 	Eterm *consp, *tp, opt;
@@ -2457,18 +2628,20 @@ parse_bif_timer_options(Eterm option_list, int *async, int *info,
 }
 
 static void
-exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp)
+exit_cancel_bif_timer(ErtsBifTimer *tmr, void *vesdp)
 {
     ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
     Uint32 sid, roflgs;
     erts_aint_t state;
+    int is_hlt;
 
-    state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state,
+    state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state,
 					   ERTS_TMR_STATE_CANCELED,
 					   ERTS_TMR_STATE_ACTIVE);
 
-    roflgs = tmr->head.roflgs;
+    roflgs = tmr->type.head.roflgs;
     sid = roflgs & ERTS_TMR_ROFLG_SID_MASK;
+    is_hlt = !!(roflgs & ERTS_TMR_ROFLG_HLT);
 
     ERTS_HLT_ASSERT(sid == erts_get_ref_numbers_thr_id(ERTS_BTM_HLT2REFN(tmr)));
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
@@ -2479,29 +2652,33 @@ exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp)
     tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
 #endif
 
-    if (sid == (Uint32) esdp->no) {
-	if (state == ERTS_TMR_STATE_ACTIVE) {
+    if (state == ERTS_TMR_STATE_ACTIVE) {
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
-            btm_clear_magic_binary(tmr);
+        btm_clear_magic_binary(tmr);
 #endif
-	    if (tmr->btm.bp)
-		free_message_buffer(tmr->btm.bp);
-	    hlt_delete_timer(esdp, tmr);
+        if (tmr->btm.bp)
+            free_message_buffer(tmr->btm.bp);
+
+        if (sid != (Uint32) esdp->no) {
+            queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr);
+            return;
+        }
+
+#ifndef ERTS_MAGIC_REF_BIF_TIMERS
+        if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) {
+	    btm_rbt_delete(&esdp->timer_service->btm_tree, tmr);
+	    tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
 	}
-	hl_timer_dec_refc(tmr, roflgs);
-    }
-    else {
-	if (state == ERTS_TMR_STATE_ACTIVE) {
-#ifdef ERTS_MAGIC_REF_BIF_TIMERS
-            btm_clear_magic_binary(tmr);
 #endif
-	    if (tmr->btm.bp)
-		free_message_buffer(tmr->btm.bp);
-	    queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr);
-	}
-	else
-	    hl_timer_dec_refc(tmr, roflgs);
+        if (is_hlt)
+            hlt_delete_timer(esdp, &tmr->type.hlt);
+        else
+            cancel_tw_timer(esdp, &tmr->type.twt);
     }
+    if (is_hlt)
+        hl_timer_dec_refc(&tmr->type.hlt, roflgs);
+    else
+        tw_timer_dec_refc(&tmr->type.twt);
 }
 
 #ifdef ERTS_HLT_DEBUG
@@ -2571,10 +2748,11 @@ int erts_cancel_bif_timers(Process *p, ErtsBifTimers **btm, void **vyspp)
 static ERTS_INLINE int
 parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg,
 		  ErtsMonotonicTime *conv_arg, int abs,
-		  ErtsMonotonicTime *tposp, int *stimep)
+		  ErtsMonotonicTime *tposp, int *stimep,
+                  ErtsMonotonicTime *msp)
 {
-    ErtsMonotonicTime t;
-
+    ErtsMonotonicTime t, now;
+    
     if (!term_to_Sint64(arg, &t)) {
 	ERTS_HLT_ASSERT(!is_small(arg));
 	if (!is_big(arg))
@@ -2589,22 +2767,30 @@ parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg,
     if (conv_arg)
 	*conv_arg = t;
 
+    now = erts_get_monotonic_time(esdp);
+
     if (abs) {
 	t += -1*ERTS_MONOTONIC_OFFSET_MSEC; /* external to internal */
 	if (t < ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_BEGIN))
 	    return 1;
 	if (t > ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_END))
 	    return 1;
+        if (msp)
+            *msp = t - ERTS_MONOTONIC_TO_MSEC(now);
+
 	*stimep = (t - ERTS_MONOTONIC_TO_MSEC(esdp->last_monotonic_time)
 		   < ERTS_BIF_TIMER_SHORT_TIME);
 	*tposp = ERTS_MSEC_TO_CLKTCKS(t);
     }
     else {
-	ErtsMonotonicTime now, ticks;
+	ErtsMonotonicTime ticks;
 	
 	if (t < 0)
 	    return -1;
 
+        if (msp)
+            *msp = t;
+
 	ticks = ERTS_MSEC_TO_CLKTCKS(t);
 
 	if (ERTS_CLKTCK_RESOLUTION > 1000 && ticks < 0)
@@ -2612,7 +2798,6 @@ parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg,
 
 	ERTS_HLT_ASSERT(ticks >= 0);
 
-	now = erts_get_monotonic_time(esdp);
 	ticks += ERTS_MONOTONIC_TO_CLKTCKS(now-1);
 	ticks += 1;
 
@@ -2635,66 +2820,68 @@ parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg,
 
 BIF_RETTYPE send_after_3(BIF_ALIST_3)
 {
-    ErtsMonotonicTime timeout_pos;
+    ErtsMonotonicTime timeout_pos, tmo;
     int short_time, tres;
 
-    tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL,
-			     0, &timeout_pos, &short_time);
+    tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1,
+			     NULL, 0, &timeout_pos, &short_time, &tmo);
     if (tres != 0)
 	BIF_ERROR(BIF_P, BADARG);
 
-    return setup_bif_timer(BIF_P, timeout_pos, short_time,
-			   BIF_ARG_2, BIF_ARG_2, BIF_ARG_3, 0);
+    return setup_bif_timer(BIF_P, tmo < ERTS_TIMER_WHEEL_MSEC,
+                           timeout_pos, short_time, BIF_ARG_2,
+                           BIF_ARG_3, 0);
 }
 
 BIF_RETTYPE send_after_4(BIF_ALIST_4)
 {
-    ErtsMonotonicTime timeout_pos;
-    Eterm accessor;
+    ErtsMonotonicTime timeout_pos, tmo;
     int short_time, abs, tres;
 
-    if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor))
+    if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs))
 	BIF_ERROR(BIF_P, BADARG);
     
     tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL,
-			     abs, &timeout_pos, &short_time);
+			     abs, &timeout_pos, &short_time, &tmo);
     if (tres != 0)
 	BIF_ERROR(BIF_P, BADARG);
 
-    return setup_bif_timer(BIF_P, timeout_pos, short_time,
-			   BIF_ARG_2, accessor, BIF_ARG_3, 0);
+    return setup_bif_timer(BIF_P, tmo < ERTS_TIMER_WHEEL_MSEC,
+                           timeout_pos, short_time, BIF_ARG_2,
+                           BIF_ARG_3, 0);
 }
 
 BIF_RETTYPE start_timer_3(BIF_ALIST_3)
 {
-    ErtsMonotonicTime timeout_pos;
+    ErtsMonotonicTime timeout_pos, tmo;
     int short_time, tres;
 
     tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL,
-			     0, &timeout_pos, &short_time);
+			     0, &timeout_pos, &short_time, &tmo);
     if (tres != 0)
 	BIF_ERROR(BIF_P, BADARG);
 
-    return setup_bif_timer(BIF_P, timeout_pos, short_time,
-			   BIF_ARG_2, BIF_ARG_2, BIF_ARG_3, !0);
+    return setup_bif_timer(BIF_P, tmo < ERTS_TIMER_WHEEL_MSEC,
+                           timeout_pos, short_time, BIF_ARG_2,
+                           BIF_ARG_3, !0);
 }
 
 BIF_RETTYPE start_timer_4(BIF_ALIST_4)
 {
-    ErtsMonotonicTime timeout_pos;
-    Eterm accessor;
+    ErtsMonotonicTime timeout_pos, tmo;
     int short_time, abs, tres;
 
-    if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor))
+    if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs))
 	BIF_ERROR(BIF_P, BADARG);
 
     tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL,
-			     abs, &timeout_pos, &short_time);
+			     abs, &timeout_pos, &short_time, &tmo);
     if (tres != 0)
 	BIF_ERROR(BIF_P, BADARG);
 
-    return setup_bif_timer(BIF_P, timeout_pos, short_time,
-			   BIF_ARG_2, accessor, BIF_ARG_3, !0);
+    return setup_bif_timer(BIF_P, tmo < ERTS_TIMER_WHEEL_MSEC,
+                           timeout_pos, short_time, BIF_ARG_2,
+                           BIF_ARG_3, !0);
 }
 
 BIF_RETTYPE cancel_timer_1(BIF_ALIST_1)
@@ -2707,7 +2894,7 @@ BIF_RETTYPE cancel_timer_2(BIF_ALIST_2)
     BIF_RETTYPE ret;
     int async, info;
 
-    if (parse_bif_timer_options(BIF_ARG_2, &async, &info, NULL, NULL))
+    if (parse_bif_timer_options(BIF_ARG_2, &async, &info, NULL))
 	return access_bif_timer(BIF_P, BIF_ARG_1, 1, async, info);
 
     ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
@@ -2724,7 +2911,7 @@ BIF_RETTYPE read_timer_2(BIF_ALIST_2)
     BIF_RETTYPE ret;
     int async;
 
-    if (parse_bif_timer_options(BIF_ARG_2, &async, NULL, NULL, NULL))
+    if (parse_bif_timer_options(BIF_ARG_2, &async, NULL, NULL))
 	return access_bif_timer(BIF_P, BIF_ARG_1, 0, async, 1);
 
     ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
@@ -2739,14 +2926,13 @@ start_callback_timer(ErtsSchedulerData *esdp,
 		     void *arg)
 		     
 {
-    if (twt)
-	create_tw_timer(esdp, ERTS_TMR_CALLBACK, NULL,
-			callback, arg, timeout_pos);
-    else
-	create_hl_timer(esdp, timeout_pos, 0,
-			ERTS_TMR_CALLBACK, NULL,
-			NIL, THE_NON_VALUE, NIL,
-			NULL, callback, arg);
+    ErtsCreateTimerFunc create_timer = (twt
+                                        ? create_tw_timer
+                                        : create_hl_timer);
+    (void) create_timer(esdp, timeout_pos, 0,
+                        ERTS_TMR_CALLBACK, NULL,
+                        NIL, THE_NON_VALUE, NULL,
+                        callback, arg);
 }
 
 typedef struct {
@@ -2823,18 +3009,18 @@ set_proc_timer_common(Process *c_p, ErtsSchedulerData *esdp, Sint64 tmo,
     if (tmo == 0)
 	c_p->flags |= F_TIMO;
     else {
+        ErtsCreateTimerFunc create_timer;
 
 	c_p->flags |= F_INSLPQUEUE;
 	c_p->flags &= ~F_TIMO;
 
-	if (tmo < ERTS_TIMER_WHEEL_MSEC)
-	    tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PROC, (void *) c_p,
-					   NULL, NULL, timeout_pos);
-	else
-	    tmr = (void *) create_hl_timer(esdp, timeout_pos, short_time,
-					   ERTS_TMR_PROC, (void *) c_p,
-					   c_p->common.id, THE_NON_VALUE,
-					   NIL, NULL, NULL, NULL);
+        create_timer = (tmo < ERTS_TIMER_WHEEL_MSEC
+                        ? create_tw_timer
+                        : create_hl_timer);
+        tmr = (void *) create_timer(esdp, timeout_pos, short_time,
+                                    ERTS_TMR_PROC, (void *) c_p,
+                                    c_p->common.id, THE_NON_VALUE,
+                                    NULL, NULL, NULL);
 	erts_smp_atomic_set_relb(&c_p->common.timer, (erts_aint_t) tmr);
     }
 }
@@ -2850,7 +3036,7 @@ erts_set_proc_timer_term(Process *c_p, Eterm etmo)
 		    == ERTS_PTMR_NONE);
 
     tres = parse_timeout_pos(esdp, etmo, &tmo, 0,
-			     &timeout_pos, &short_time);
+			     &timeout_pos, &short_time, NULL);
     if (tres != 0)
 	return tres;
 
@@ -2908,6 +3094,7 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo)
     void *tmr;
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
     ErtsMonotonicTime timeout_pos;
+    ErtsCreateTimerFunc create_timer;
 
     if (erts_smp_atomic_read_nob(&c_prt->common.timer) != ERTS_PTMR_NONE)
 	erts_cancel_port_timer(c_prt);
@@ -2916,13 +3103,12 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo)
 
     timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), tmo);
 
-    if (tmo < ERTS_TIMER_WHEEL_MSEC)
-	tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PORT, (void *) c_prt,
-				       NULL, NULL, timeout_pos);
-    else
-	tmr = (void *) create_hl_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT,
-				       (void *) c_prt, c_prt->common.id,
-				       THE_NON_VALUE, NIL, NULL, NULL, NULL);
+    create_timer = (tmo < ERTS_TIMER_WHEEL_MSEC
+                    ? create_tw_timer
+                    : create_hl_timer);
+    tmr = (void *) create_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT,
+                                (void *) c_prt, c_prt->common.id,
+                                THE_NON_VALUE, NULL, NULL, NULL);
     erts_smp_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr);
 }
 
@@ -2976,25 +3162,39 @@ typedef struct {
 } ErtsBTMPrint;
 
 static void
-btm_print(ErtsHLTimer *tmr, void *vbtmp)
+btm_print(ErtsBifTimer *tmr, void *vbtmp, ErtsMonotonicTime tpos, int is_hlt)
 {
     ErtsBTMPrint *btmp = (ErtsBTMPrint *) vbtmp;
     ErtsMonotonicTime left;
     Eterm receiver;
 
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
-    if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR))
+    if (!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_BIF_TMR))
         return;
 #endif
 
-    if (tmr->timeout <= btmp->now)
-	left = 0;
-    else
-        left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now);
+    if (is_hlt) {
+        ERTS_HLT_ASSERT(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT);
+        if (tmr->type.hlt.timeout <= btmp->now)
+            left = 0;
+        else
+            left = ERTS_CLKTCKS_TO_MSEC(tmr->type.hlt.timeout - btmp->now);
 
-    receiver = ((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
-		? tmr->receiver.name
-		: tmr->receiver.proc->common.id);
+        receiver = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
+                    ? tmr->type.hlt.receiver.name
+                    : tmr->type.hlt.receiver.proc->common.id);
+    }
+    else {
+        ERTS_HLT_ASSERT(!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT));
+        if (tpos <= btmp->now)
+            left = 0;
+        else
+            left = ERTS_CLKTCKS_TO_MSEC(tpos - btmp->now);
+
+        receiver = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
+                    ? tmr->type.twt.receiver.name
+                    : tmr->type.twt.receiver.proc->common.id);
+    }
 
     erts_print(btmp->to, btmp->to_arg,
 	       "=timer:%T\n"
@@ -3005,6 +3205,36 @@ btm_print(ErtsHLTimer *tmr, void *vbtmp)
 	       (Sint64) left);
 }
 
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+
+static void
+hlt_btm_print(ErtsHLTimer *tmr, void *vbtmp)
+{
+    btm_print((ErtsBifTimer *) tmr, vbtmp, 0, 1);
+}
+
+static void
+twt_btm_print(void *vbtmp, ErtsMonotonicTime tpos, void *vtwtp)
+{
+    btm_print((ErtsBifTimer *) vtwtp, vbtmp, tpos, 0);
+}
+
+#else
+
+static void
+btm_tree_print(ErtsBifTimer *tmr, void *vbtmp)
+{
+    int is_hlt = !!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT);
+    ErtsMonotonicTime tpos;
+    if (is_hlt)
+        tpos = 0;
+    else
+        tpos = tmr->type.twt.tw_tmr.timeout_pos;
+    btm_print(tmr, vbtmp, tpos, is_hlt);
+}
+
+#endif
+
 void
 erts_print_bif_timer_info(fmtfn_t to, void *to_arg)
 {
@@ -3023,9 +3253,13 @@ erts_print_bif_timer_info(fmtfn_t to, void *to_arg)
 	ErtsHLTimerService *srv =
 	    erts_aligned_scheduler_data[six].esd.timer_service;
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
-	time_rbt_foreach(srv->time_tree, btm_print, (void *) &btmp);
+	ErtsTimerWheel *twheel =
+	    erts_aligned_scheduler_data[six].esd.timer_wheel;
+	erts_twheel_debug_foreach(twheel, tw_bif_timer_timeout,
+				  twt_btm_print, (void *) &btmp);
+	time_rbt_foreach(srv->time_tree, hlt_btm_print, (void *) &btmp);
 #else
-	btm_rbt_foreach(srv->btm_tree, btm_print, (void *) &btmp);
+	btm_rbt_foreach(srv->btm_tree, btm_tree_print, (void *) &btmp);
 #endif
     }
 }
@@ -3039,23 +3273,52 @@ typedef struct {
 } ErtsBTMForeachDebug;
 
 static void
-debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd)
+debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd, int is_hlt)
 {
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
-    if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR))
+    if (!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_BIF_TMR))
         return;
 #endif
-    if (erts_smp_atomic32_read_nob(&tmr->state) == ERTS_TMR_STATE_ACTIVE) {
+    if (erts_smp_atomic32_read_nob(&tmr->btm.state) == ERTS_TMR_STATE_ACTIVE) {
 	ErtsBTMForeachDebug *btmfd = (ErtsBTMForeachDebug *) vbtmfd;
-	(*btmfd->func)(((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
-			? tmr->receiver.name
-			: tmr->receiver.proc->common.id),
-		       tmr->btm.message,
-		       tmr->btm.bp,
-		       btmfd->arg);
+        Eterm id;
+        if (is_hlt)
+            id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
+                  ? tmr->type.hlt.receiver.name
+                  : tmr->type.hlt.receiver.proc->common.id);
+        else
+            id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
+                  ? tmr->type.twt.receiver.name
+                  : tmr->type.twt.receiver.proc->common.id);
+	(*btmfd->func)(id, tmr->btm.message, tmr->btm.bp, btmfd->arg);
     }
 }
 
+#ifdef ERTS_MAGIC_REF_BIF_TIMERS
+
+static void
+hlt_debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd)
+{
+    debug_btm_foreach((ErtsBifTimer *) tmr, vbtmfd, 1);
+}
+
+static void
+twt_debug_btm_foreach(void *vbtmfd, ErtsMonotonicTime tpos, void *vtwtp)
+{
+    debug_btm_foreach((ErtsBifTimer *) vtwtp, vbtmfd, 0);    
+}
+
+#else
+
+static void
+debug_btm_tree_foreach(ErtsBifTimer *tmr, void *vbtmfd)
+{
+    debug_btm_foreach(tmr, vbtmfd, !!(tmr->type.head.roflgs
+                                      & ERTS_TMR_ROFLG_HLT));
+}
+
+#endif
+
 void
 erts_debug_bif_timer_foreach(void (*func)(Eterm,
 					  Eterm,
@@ -3076,13 +3339,17 @@ erts_debug_bif_timer_foreach(void (*func)(Eterm,
 	ErtsHLTimerService *srv =
 	    erts_aligned_scheduler_data[six].esd.timer_service;
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
+	ErtsTimerWheel *twheel =
+	    erts_aligned_scheduler_data[six].esd.timer_wheel;            
+	erts_twheel_debug_foreach(twheel, tw_bif_timer_timeout,
+				  twt_debug_btm_foreach,
+                                  (void *) &btmfd);
         time_rbt_foreach(srv->time_tree,
-                         debug_btm_foreach,
+                         hlt_debug_btm_foreach,
                          (void *) &btmfd);
-            
 #else
 	btm_rbt_foreach(srv->btm_tree,
-			debug_btm_foreach,
+			debug_btm_tree_foreach,
 			(void *) &btmfd);
 #endif
     }
@@ -3136,7 +3403,7 @@ debug_tw_callback_timer(void *vdfct,
     ErtsDebugForeachCallbackTimer *dfct
 	= (ErtsDebugForeachCallbackTimer *) vdfct;
 
-    if (twtp->u.callback == dfct->tclbk)
+    if (twtp->receiver.callback == dfct->tclbk)
 	(*dfct->func)(dfct->arg,
 		      timeout_pos,
 		      twtp->head.u.arg);
diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h
index 33cdf217d7..f70fcdd1c0 100644
--- a/erts/emulator/beam/erl_hl_timer.h
+++ b/erts/emulator/beam/erl_hl_timer.h
@@ -21,7 +21,7 @@
 #ifndef ERL_HL_TIMER_H__
 #define ERL_HL_TIMER_H__
 
-typedef struct ErtsHLTimer_ ErtsBifTimers;
+typedef struct ErtsBifTimer_ ErtsBifTimers;
 typedef struct ErtsHLTimerService_ ErtsHLTimerService;
 
 #include "sys.h"
-- 
cgit v1.2.3


From 79ac974710ecab6dfe37b23b4fd3b52e760b3504 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 3 Feb 2017 15:50:51 +0100
Subject: Rearrange timer struct fields in order to simplify

---
 erts/emulator/beam/erl_hl_timer.c | 177 +++++++++++++++-----------------------
 erts/emulator/beam/erl_time.h     |  20 +++--
 erts/emulator/beam/time.c         |  32 +++----
 3 files changed, 98 insertions(+), 131 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 102a739b13..8e47fdd674 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -164,21 +164,21 @@ typedef struct {
 	void *arg;
 	erts_atomic_t next;
     } u;
+    union {
+	Process *proc;
+	Port *port;
+	Eterm name;
+	void (*callback)(void *);
+    } receiver;
 } ErtsTmrHead;
 
 struct ErtsHLTimer_ {
     ErtsTmrHead head; /* NEED to be first! */
+    ErtsMonotonicTime timeout;
     union {
 	ErtsThrPrgrLaterOp cleanup;
 	ErtsHLTimerTimeTree tree;
     } time;
-    ErtsMonotonicTime timeout;
-    union {
-	Process *proc;
-	Port *port;
-	Eterm name;
-	void (*callback)(void *);
-    } receiver;
 
 #ifdef ERTS_HLT_HARD_DEBUG
     int pending_timeout;
@@ -188,12 +188,9 @@ struct ErtsHLTimer_ {
 typedef struct {
     ErtsTmrHead head; /* NEED to be first! */
     union {
-        Eterm name;
-	Process *proc;
-        Port *port;
-	void (*callback)(void *);
-    } receiver;
-    ErtsTWheelTimer tw_tmr;
+        ErtsTWheelTimer tw_tmr;
+	ErtsThrPrgrLaterOp cleanup;
+    } u;
 } ErtsTWTimer;
 
 struct ErtsBifTimer_ {
@@ -948,9 +945,9 @@ schedule_tw_timer_destroy(ErtsTWTimer *tmr)
      * dropped at once...
      */
     if (tmr->head.roflgs & ERTS_TMR_ROFLG_PROC)
-	erts_proc_dec_refc(tmr->receiver.proc);
+	erts_proc_dec_refc(tmr->head.receiver.proc);
     else if (tmr->head.roflgs & ERTS_TMR_ROFLG_PORT)
-	erts_port_dec_refc(tmr->receiver.port);
+	erts_port_dec_refc(tmr->head.receiver.port);
 
     if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR))
 	size = sizeof(ErtsHLTimer);
@@ -965,7 +962,7 @@ schedule_tw_timer_destroy(ErtsTWTimer *tmr)
     erts_schedule_thr_prgr_later_cleanup_op(
 	scheduled_tw_timer_destroy,
 	(void *) tmr,
-	&tmr->tw_tmr.u.cleanup,
+	&tmr->u.cleanup,
 	size);
 }
 
@@ -982,7 +979,7 @@ static void
 tw_proc_timeout(void *vtwtp)
 {
     ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp;
-    Process *proc = twtp->receiver.proc;
+    Process *proc = twtp->head.receiver.proc;
     if (proc_timeout_common(proc, vtwtp))
 	tw_timer_dec_refc(twtp);
     tw_timer_dec_refc(twtp);
@@ -992,7 +989,7 @@ static void
 tw_port_timeout(void *vtwtp)
 {
     ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp;
-    Port *port = twtp->receiver.port;
+    Port *port = twtp->head.receiver.port;
     if (port_timeout_common(port, vtwtp))
 	tw_timer_dec_refc(twtp);
     tw_timer_dec_refc(twtp);
@@ -1009,14 +1006,14 @@ cancel_tw_timer(ErtsSchedulerData *esdp, ErtsTWTimer *tmr)
 {
     ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK)
 		    == (Uint32) esdp->no);
-    erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->tw_tmr);
+    erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->u.tw_tmr);
 }
 
 static void
 tw_callback_timeout(void *vtwtp)
 {
     ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp;
-    void (*callback)(void *) = twtp->receiver.callback;
+    void (*callback)(void *) = twtp->head.receiver.callback;
     void *arg = twtp->head.u.arg;
     tw_timer_dec_refc(twtp);
     (*callback)(arg);
@@ -1060,7 +1057,7 @@ create_tw_timer(ErtsSchedulerData *esdp,
         }
     }
 
-    erts_twheel_init_timer(&tmr->tw_tmr);
+    erts_twheel_init_timer(&tmr->u.tw_tmr);
     tmr->head.roflgs |= (Uint32) esdp->no;
     ERTS_HLT_ASSERT((((Uint32) esdp->no)
                      & ~ERTS_TMR_ROFLG_SID_MASK) == 0);
@@ -1068,7 +1065,7 @@ create_tw_timer(ErtsSchedulerData *esdp,
     switch (type) {
 
     case ERTS_TMR_PROC:
-	tmr->receiver.proc = (Process *) rcvrp;
+	tmr->head.receiver.proc = (Process *) rcvrp;
 	tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC;
 	timeout_func = tw_proc_timeout;
 	cancel_func = tw_ptimer_cancel;
@@ -1077,7 +1074,7 @@ create_tw_timer(ErtsSchedulerData *esdp,
 	break;
 
     case ERTS_TMR_PORT:
-	tmr->receiver.port = (Port *) rcvrp;
+	tmr->head.receiver.port = (Port *) rcvrp;
 	tmr->head.roflgs |= ERTS_TMR_ROFLG_PORT;
 	timeout_func = tw_port_timeout;
 	cancel_func = tw_ptimer_cancel;
@@ -1087,7 +1084,7 @@ create_tw_timer(ErtsSchedulerData *esdp,
 
     case ERTS_TMR_CALLBACK:
 	tmr->head.u.arg = arg;
-	tmr->receiver.callback = callback;
+	tmr->head.receiver.callback = callback;
 
 	tmr->head.roflgs |= ERTS_TMR_ROFLG_CALLBACK;
 	timeout_func = tw_callback_timeout;
@@ -1101,13 +1098,13 @@ create_tw_timer(ErtsSchedulerData *esdp,
 	cancel_func = NULL;
 	if (is_internal_pid(rcvr)) {
 	    tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC;
-	    tmr->receiver.proc = (Process *) rcvrp;
+	    tmr->head.receiver.proc = (Process *) rcvrp;
 	    refc = 2;
 	}
 	else {
 	    ERTS_HLT_ASSERT(is_atom(rcvr));
 	    tmr->head.roflgs |= ERTS_TMR_ROFLG_REG_NAME;
-	    tmr->receiver.name = (Eterm) rcvr;
+	    tmr->head.receiver.name = (Eterm) rcvr;
 	    refc = 1;
 	}
 
@@ -1130,7 +1127,7 @@ create_tw_timer(ErtsSchedulerData *esdp,
     erts_smp_atomic32_init_nob(&tmr->head.refc, refc);
 
     erts_twheel_set_timer(esdp->timer_wheel,
-			  &tmr->tw_tmr,
+			  &tmr->u.tw_tmr,
 			  timeout_func,
 			  cancel_func,
 			  tmr,
@@ -1164,15 +1161,15 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs)
     ERTS_HLT_ASSERT(erts_smp_atomic32_read_nob(&tmr->head.refc) == 0);
 
     if (roflgs & ERTS_TMR_ROFLG_REG_NAME) {
-	ERTS_HLT_ASSERT(is_atom(tmr->receiver.name));
+	ERTS_HLT_ASSERT(is_atom(tmr->head.receiver.name));
     }
     else if (roflgs & ERTS_TMR_ROFLG_PROC) {
-	ERTS_HLT_ASSERT(tmr->receiver.proc);
-	erts_proc_dec_refc(tmr->receiver.proc);
+	ERTS_HLT_ASSERT(tmr->head.receiver.proc);
+	erts_proc_dec_refc(tmr->head.receiver.proc);
     }
     else if (roflgs & ERTS_TMR_ROFLG_PORT) {
-	ERTS_HLT_ASSERT(tmr->receiver.port);
-	erts_port_dec_refc(tmr->receiver.port);
+	ERTS_HLT_ASSERT(tmr->head.receiver.port);
+	erts_port_dec_refc(tmr->head.receiver.port);
     }
 
     if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR))
@@ -1251,14 +1248,14 @@ btm_clear_magic_binary(ErtsBifTimer *tmr)
 #endif /* ERTS_MAGIC_REF_BIF_TIMERS */
 
 static ERTS_INLINE void
-bif_timer_timeout(ErtsHLTimerService *srv, ErtsBifTimer *tmr,
-                  Uint32 roflgs, int is_hlt)
+bif_timer_timeout(ErtsHLTimerService *srv,
+                  ErtsBifTimer *tmr,
+                  Uint32 roflgs)
 {
     erts_aint32_t state;
 
     ERTS_HLT_ASSERT(tmr->type.head.roflgs == roflgs);    
     ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR);
-    ERTS_HLT_ASSERT(!!is_hlt == !!(roflgs & ERTS_TMR_ROFLG_HLT));
 
     state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state,
 					   ERTS_TMR_STATE_TIMED_OUT,
@@ -1269,40 +1266,32 @@ bif_timer_timeout(ErtsHLTimerService *srv, ErtsBifTimer *tmr,
 
     if (state == ERTS_TMR_STATE_ACTIVE) {
         Process *proc;
-        int dec_refc = 0;
-        Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME);
 
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
         btm_clear_magic_binary(tmr);
 #endif
 
-        if (is_reg_name) {
+        if (roflgs & ERTS_TMR_ROFLG_REG_NAME) {
             Eterm term;
-            if (is_hlt)
-                term = tmr->type.hlt.receiver.name;
-            else
-                term = tmr->type.twt.receiver.name;
+            term = tmr->type.head.receiver.name;
             ERTS_HLT_ASSERT(is_atom(term));
             term = erts_whereis_name_to_id(NULL, term);
             proc = erts_proc_lookup(term);
         }
         else {
             ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PROC);
-            if (is_hlt)
-                proc = tmr->type.hlt.receiver.proc;
-            else
-                proc = tmr->type.twt.receiver.proc;
+            proc = tmr->type.head.receiver.proc;
             ERTS_HLT_ASSERT(proc);
         }
         if (proc) {
             if (!ERTS_PROC_IS_EXITING(proc)) {
-                ErtsMessage *mp;
-                erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM);
-                mp = erts_alloc_message(0, NULL);
+                int dec_refc = 0;
+                ErtsMessage *mp = erts_alloc_message(0, NULL);
                 mp->data.heap_frag = tmr->btm.bp;
-                erts_queue_message(proc, ERTS_PROC_LOCK_BTM, mp,
-                                   tmr->btm.message, am_clock_service);
                 tmr->btm.bp = NULL;
+                erts_queue_message(proc, 0, mp, tmr->btm.message,
+                                   am_clock_service);
+                erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM);
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
                 if (tmr->btm.proc_list.next) {
                     proc_btm_list_delete(&proc->bif_timers, tmr);
@@ -1316,9 +1305,9 @@ bif_timer_timeout(ErtsHLTimerService *srv, ErtsBifTimer *tmr,
                 }
 #endif
                 erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
+                if (dec_refc)
+                    timer_pre_dec_refc((ErtsTimer *) tmr);
             }
-            if (dec_refc)
-                timer_pre_dec_refc((ErtsTimer *) tmr);
         }
         if (tmr->btm.bp)
             free_message_buffer(tmr->btm.bp);
@@ -1344,7 +1333,7 @@ tw_bif_timer_timeout(void *vbtmp)
     ErtsHLTimerService *srv = esdp->timer_service;
 #endif
     ErtsBifTimer *btmp = (ErtsBifTimer *) vbtmp;
-    bif_timer_timeout(srv, btmp, btmp->type.head.roflgs, 0);
+    bif_timer_timeout(srv, btmp, btmp->type.head.roflgs);
     tw_timer_dec_refc(&btmp->type.twt);
 }
 
@@ -1386,7 +1375,7 @@ create_hl_timer(ErtsSchedulerData *esdp,
 	    ERTS_HLT_ASSERT(is_internal_pid(rcvr));
 
 	    erts_proc_inc_refc((Process *) rcvrp);
-	    tmr->receiver.proc = (Process *) rcvrp;
+	    tmr->head.receiver.proc = (Process *) rcvrp;
 	    roflgs |= ERTS_TMR_ROFLG_PROC;
 	    refc = 2;
 	    break;
@@ -1394,14 +1383,14 @@ create_hl_timer(ErtsSchedulerData *esdp,
 	case ERTS_TMR_PORT:
 	    ERTS_HLT_ASSERT(is_internal_port(rcvr));
 	    erts_port_inc_refc((Port *) rcvrp);
-	    tmr->receiver.port = (Port *) rcvrp;
+	    tmr->head.receiver.port = (Port *) rcvrp;
 	    roflgs |= ERTS_TMR_ROFLG_PORT;
 	    refc = 2;
 	    break;
 
 	case ERTS_TMR_CALLBACK:
 	    roflgs |= ERTS_TMR_ROFLG_CALLBACK;
-	    tmr->receiver.callback = callback;
+	    tmr->head.receiver.callback = callback;
 	    tmr->head.u.arg = arg;
 	    refc = 1;
 	    break;
@@ -1431,13 +1420,13 @@ create_hl_timer(ErtsSchedulerData *esdp,
 	roflgs |= ERTS_TMR_ROFLG_BIF_TMR;
 	if (is_internal_pid(rcvr)) {
 	    roflgs |= ERTS_TMR_ROFLG_PROC;
-	    tmr->receiver.proc = (Process *) rcvrp;
+	    tmr->head.receiver.proc = (Process *) rcvrp;
 	    refc = 2;
 	}
 	else {
 	    ERTS_HLT_ASSERT(is_atom(rcvr));
 	    roflgs |= ERTS_TMR_ROFLG_REG_NAME;
-	    tmr->receiver.name = rcvr;
+	    tmr->head.receiver.name = rcvr;
 	    refc = 1;
 	}
 
@@ -1486,14 +1475,14 @@ create_hl_timer(ErtsSchedulerData *esdp,
 static ERTS_INLINE void
 hlt_proc_timeout(ErtsHLTimer *tmr)
 {
-    if (proc_timeout_common(tmr->receiver.proc, (void *) tmr))
+    if (proc_timeout_common(tmr->head.receiver.proc, (void *) tmr))
 	hl_timer_dec_refc(tmr, tmr->head.roflgs);
 }
 
 static ERTS_INLINE void
 hlt_port_timeout(ErtsHLTimer *tmr)
 {
-    if (port_timeout_common(tmr->receiver.port, (void *) tmr))
+    if (port_timeout_common(tmr->head.receiver.port, (void *) tmr))
 	hl_timer_dec_refc(tmr, tmr->head.roflgs);
 }
 
@@ -1508,14 +1497,14 @@ static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv)
     ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_HLT);
 
     if (roflgs & ERTS_TMR_ROFLG_BIF_TMR)
-        bif_timer_timeout(srv, (ErtsBifTimer *) tmr, roflgs, 1);
+        bif_timer_timeout(srv, (ErtsBifTimer *) tmr, roflgs);
     else if (roflgs & ERTS_TMR_ROFLG_PROC)
         hlt_proc_timeout(tmr);
     else if (roflgs & ERTS_TMR_ROFLG_PORT)
         hlt_port_timeout(tmr);
     else {
         ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_CALLBACK);
-        (*tmr->receiver.callback)(tmr->head.u.arg);
+        (*tmr->head.receiver.callback)(tmr->head.u.arg);
     }
 
     tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
@@ -2022,10 +2011,7 @@ setup_bif_timer(Process *c_p, int twheel, ErtsMonotonicTime timeout_pos,
 	    proc_btm_rbt_insert(&proc->bif_timers, tmr);
 #endif
 	    erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
-            if (twheel)
-                tmr->type.twt.receiver.proc = proc;
-            else
-                tmr->type.hlt.receiver.proc = proc;
+            tmr->type.head.receiver.proc = proc;
 	}
     }
 
@@ -2064,9 +2050,7 @@ cancel_bif_timer(ErtsBifTimer *tmr)
     if (roflgs & ERTS_TMR_ROFLG_PROC) {
 	Process *proc;
 
-        proc = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT)
-                ? tmr->type.hlt.receiver.proc
-                : tmr->type.twt.receiver.proc);
+        proc = tmr->type.head.receiver.proc;
 	ERTS_HLT_ASSERT(!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME));
 
 	erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM);
@@ -2108,7 +2092,7 @@ access_btm(ErtsBifTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel)
     is_hlt = !!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT);
     timeout = (is_hlt
                ? tmr->type.hlt.timeout
-               : tmr->type.twt.tw_tmr.timeout_pos);
+               : erts_tweel_read_timeout(&tmr->type.twt.u.tw_tmr));
 
     if (!cancel) {
         erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->btm.state);
@@ -3147,7 +3131,7 @@ erts_read_port_timer(Port *c_prt)
     if (tmr->head.roflgs & ERTS_TMR_ROFLG_HLT)
 	timeout_pos = tmr->hlt.timeout;
     else
-	timeout_pos = tmr->twt.tw_tmr.timeout_pos;
+	timeout_pos = erts_tweel_read_timeout(&tmr->twt.u.tw_tmr);
     return get_time_left(NULL, timeout_pos);
 }
 
@@ -3179,10 +3163,6 @@ btm_print(ErtsBifTimer *tmr, void *vbtmp, ErtsMonotonicTime tpos, int is_hlt)
             left = 0;
         else
             left = ERTS_CLKTCKS_TO_MSEC(tmr->type.hlt.timeout - btmp->now);
-
-        receiver = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
-                    ? tmr->type.hlt.receiver.name
-                    : tmr->type.hlt.receiver.proc->common.id);
     }
     else {
         ERTS_HLT_ASSERT(!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT));
@@ -3190,12 +3170,12 @@ btm_print(ErtsBifTimer *tmr, void *vbtmp, ErtsMonotonicTime tpos, int is_hlt)
             left = 0;
         else
             left = ERTS_CLKTCKS_TO_MSEC(tpos - btmp->now);
-
-        receiver = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
-                    ? tmr->type.twt.receiver.name
-                    : tmr->type.twt.receiver.proc->common.id);
     }
 
+    receiver = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
+                ? tmr->type.head.receiver.name
+                : tmr->type.head.receiver.proc->common.id);
+
     erts_print(btmp->to, btmp->to_arg,
 	       "=timer:%T\n"
 	       "Message: %T\n"
@@ -3229,7 +3209,7 @@ btm_tree_print(ErtsBifTimer *tmr, void *vbtmp)
     if (is_hlt)
         tpos = 0;
     else
-        tpos = tmr->type.twt.tw_tmr.timeout_pos;
+        tpos = erts_tweel_read_timeout(&tmr->type.twt.u.tw_tmr);
     btm_print(tmr, vbtmp, tpos, is_hlt);
 }
 
@@ -3273,7 +3253,7 @@ typedef struct {
 } ErtsBTMForeachDebug;
 
 static void
-debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd, int is_hlt)
+debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd)
 {
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
     if (!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_BIF_TMR))
@@ -3281,15 +3261,9 @@ debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd, int is_hlt)
 #endif
     if (erts_smp_atomic32_read_nob(&tmr->btm.state) == ERTS_TMR_STATE_ACTIVE) {
 	ErtsBTMForeachDebug *btmfd = (ErtsBTMForeachDebug *) vbtmfd;
-        Eterm id;
-        if (is_hlt)
-            id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
-                  ? tmr->type.hlt.receiver.name
-                  : tmr->type.hlt.receiver.proc->common.id);
-        else
-            id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
-                  ? tmr->type.twt.receiver.name
-                  : tmr->type.twt.receiver.proc->common.id);
+        Eterm id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
+                    ? tmr->type.head.receiver.name
+                    : tmr->type.head.receiver.proc->common.id);
 	(*btmfd->func)(id, tmr->btm.message, tmr->btm.bp, btmfd->arg);
     }
 }
@@ -3299,22 +3273,13 @@ debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd, int is_hlt)
 static void
 hlt_debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd)
 {
-    debug_btm_foreach((ErtsBifTimer *) tmr, vbtmfd, 1);
+    debug_btm_foreach((ErtsBifTimer *) tmr, vbtmfd);
 }
 
 static void
 twt_debug_btm_foreach(void *vbtmfd, ErtsMonotonicTime tpos, void *vtwtp)
 {
-    debug_btm_foreach((ErtsBifTimer *) vtwtp, vbtmfd, 0);    
-}
-
-#else
-
-static void
-debug_btm_tree_foreach(ErtsBifTimer *tmr, void *vbtmfd)
-{
-    debug_btm_foreach(tmr, vbtmfd, !!(tmr->type.head.roflgs
-                                      & ERTS_TMR_ROFLG_HLT));
+    debug_btm_foreach((ErtsBifTimer *) vtwtp, vbtmfd);
 }
 
 #endif
@@ -3349,7 +3314,7 @@ erts_debug_bif_timer_foreach(void (*func)(Eterm,
                          (void *) &btmfd);
 #else
 	btm_rbt_foreach(srv->btm_tree,
-			debug_btm_tree_foreach,
+			debug_btm_foreach,
 			(void *) &btmfd);
 #endif
     }
@@ -3370,7 +3335,7 @@ debug_callback_timer_foreach_list(ErtsHLTimer *tmr, void *vdfct)
 	= (ErtsDebugForeachCallbackTimer *) vdfct;
 
     if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK)
-	&& (tmr->receiver.callback == dfct->tclbk))
+	&& (tmr->head.receiver.callback == dfct->tclbk))
 	(*dfct->func)(dfct->arg,
 		      tmr->timeout,
 		      tmr->head.u.arg);
@@ -3388,7 +3353,7 @@ debug_callback_timer_foreach(ErtsHLTimer *tmr, void *vdfct)
 			       vdfct);
 
     if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK)
-	&& (tmr->receiver.callback == dfct->tclbk))
+	&& (tmr->head.receiver.callback == dfct->tclbk))
 	(*dfct->func)(dfct->arg,
 		      tmr->timeout,
 		      tmr->head.u.arg);
@@ -3403,7 +3368,7 @@ debug_tw_callback_timer(void *vdfct,
     ErtsDebugForeachCallbackTimer *dfct
 	= (ErtsDebugForeachCallbackTimer *) vdfct;
 
-    if (twtp->receiver.callback == dfct->tclbk)
+    if (twtp->head.receiver.callback == dfct->tclbk)
 	(*dfct->func)(dfct->arg,
 		      timeout_pos,
 		      twtp->head.u.arg);
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index a1c4220633..2fb4b754d7 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -419,17 +419,12 @@ void erts_sched_init_time_sup(ErtsSchedulerData *esdp);
 ** Timer entry:
 */
 typedef struct erl_timer {
+    ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */
     struct erl_timer* next;	/* next entry tiw slot or chain */
     struct erl_timer* prev;	/* prev entry tiw slot or chain */
-    union {
-	struct {
-	    void (*timeout)(void*); /* called when timeout */
-	    void (*cancel)(void*);  /* called when cancel (may be NULL) */
-	    void* arg;              /* argument to timeout/cancel procs */
-	} func;
-	ErtsThrPrgrLaterOp cleanup;
-    } u;
-    ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */
+    void (*timeout)(void*); /* called when timeout */
+    void (*cancel)(void*);  /* called when cancel (may be NULL) */
+    void* arg;              /* argument to timeout/cancel procs */
     int slot;
 } ErtsTWheelTimer;
 
@@ -447,6 +442,7 @@ ErtsMonotonicTime erts_check_next_timeout_time(ErtsSchedulerData *);
 
 ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p);
 ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef);
+ERTS_GLB_INLINE ErtsMonotonicTime erts_tweel_read_timeout(ErtsTWheelTimer *twt);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
@@ -460,6 +456,12 @@ ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_
     return *((ErtsMonotonicTime *) nxt_tmo_ref);
 }
 
+ERTS_GLB_INLINE ErtsMonotonicTime
+erts_tweel_read_timeout(ErtsTWheelTimer *twt)
+{
+    return twt->timeout_pos;
+}
+
 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 void
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 6f15082130..fdf74498c9 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -330,8 +330,8 @@ timeout_timer(ErtsTWheelTimer *p)
     ErlTimeoutProc timeout;
     void *arg;
     p->slot = ERTS_TWHEEL_SLOT_INACTIVE;
-    timeout = p->u.func.timeout;
-    arg = p->u.func.arg;
+    timeout = p->timeout;
+    arg = p->arg;
     (*timeout)(arg);
     ASSERT_NO_LOCKED_LOCKS;
 }
@@ -542,9 +542,9 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp)
     tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY;
     tiw->sentinel.next = &tiw->sentinel;
     tiw->sentinel.prev = &tiw->sentinel;
-    tiw->sentinel.u.func.timeout = NULL;
-    tiw->sentinel.u.func.cancel = NULL;
-    tiw->sentinel.u.func.arg = NULL;
+    tiw->sentinel.timeout = NULL;
+    tiw->sentinel.cancel = NULL;
+    tiw->sentinel.arg = NULL;
     return tiw;
 }
 
@@ -583,9 +583,9 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw,
     ErtsMonotonicTime timeout_time;
     ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
 
-    p->u.func.timeout = timeout;
-    p->u.func.cancel = cancel;
-    p->u.func.arg = arg;
+    p->timeout = timeout;
+    p->cancel = cancel;
+    p->arg = arg;
 
     ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE);
 
@@ -636,8 +636,8 @@ erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
 	void *arg;
         ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
 	remove_timer(tiw, p);
-	cancel = p->u.func.cancel;
-	arg = p->u.func.arg;
+	cancel = p->cancel;
+	arg = p->arg;
 	if (cancel)
 	    (*cancel)(arg);
         ERTS_MSACC_POP_STATE_M_X();
@@ -657,22 +657,22 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw,
 
     tmr = tiw->sentinel.next;
     while (tmr != &tiw->sentinel) {
-	if (tmr->u.func.timeout == tclbk)
-	    (*func)(arg, tmr->timeout_pos, tmr->u.func.arg);
+	if (tmr->timeout == tclbk)
+	    (*func)(arg, tmr->timeout_pos, tmr->arg);
 	tmr = tmr->next;
     }
 
     for (tmr = tiw->at_once.head; tmr; tmr = tmr->next) {
-	if (tmr->u.func.timeout == tclbk)
-	    (*func)(arg, tmr->timeout_pos, tmr->u.func.arg);
+	if (tmr->timeout == tclbk)
+	    (*func)(arg, tmr->timeout_pos, tmr->arg);
     }
 
     for (ix = 0; ix < ERTS_TIW_SIZE; ix++) {
 	tmr = tiw->w[ix];
 	if (tmr) {
 	    do {
-		if (tmr->u.func.timeout == tclbk)
-		    (*func)(arg, tmr->timeout_pos, tmr->u.func.arg);
+		if (tmr->timeout == tclbk)
+		    (*func)(arg, tmr->timeout_pos, tmr->arg);
 		tmr = tmr->next;
 	    } while (tmr != tiw->w[ix]);
 	}
-- 
cgit v1.2.3


From d37df80764c9c2e517d7c9a90291d75975a5e9bb Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 3 Feb 2017 16:08:12 +0100
Subject: Remove unnecessary cancel callback from timer-wheel timers

---
 erts/emulator/beam/erl_hl_timer.c | 16 +---------------
 erts/emulator/beam/erl_process.c  |  1 -
 erts/emulator/beam/erl_time.h     |  7 ++-----
 erts/emulator/beam/erl_time_sup.c |  3 ---
 erts/emulator/beam/time.c         | 11 +----------
 5 files changed, 4 insertions(+), 34 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 8e47fdd674..13d6136672 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -995,18 +995,13 @@ tw_port_timeout(void *vtwtp)
     tw_timer_dec_refc(twtp);
 }
 
-static void
-tw_ptimer_cancel(void *vtwtp)
-{
-    tw_timer_dec_refc((ErtsTWTimer *) vtwtp);    
-}
-
 static void
 cancel_tw_timer(ErtsSchedulerData *esdp, ErtsTWTimer *tmr)
 {
     ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK)
 		    == (Uint32) esdp->no);
     erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->u.tw_tmr);
+    tw_timer_dec_refc(tmr);
 }
 
 static void
@@ -1034,7 +1029,6 @@ create_tw_timer(ErtsSchedulerData *esdp,
 {
     ErtsTWTimer *tmr;
     void (*timeout_func)(void *);
-    void (*cancel_func)(void *);
     erts_aint32_t refc;
 
     if (type != ERTS_TMR_BIF) {
@@ -1068,7 +1062,6 @@ create_tw_timer(ErtsSchedulerData *esdp,
 	tmr->head.receiver.proc = (Process *) rcvrp;
 	tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC;
 	timeout_func = tw_proc_timeout;
-	cancel_func = tw_ptimer_cancel;
 	erts_proc_inc_refc((Process *) rcvrp);
 	refc = 2;
 	break;
@@ -1077,7 +1070,6 @@ create_tw_timer(ErtsSchedulerData *esdp,
 	tmr->head.receiver.port = (Port *) rcvrp;
 	tmr->head.roflgs |= ERTS_TMR_ROFLG_PORT;
 	timeout_func = tw_port_timeout;
-	cancel_func = tw_ptimer_cancel;
 	erts_port_inc_refc((Port *) rcvrp);
 	refc = 2;
 	break;
@@ -1088,14 +1080,12 @@ create_tw_timer(ErtsSchedulerData *esdp,
 
 	tmr->head.roflgs |= ERTS_TMR_ROFLG_CALLBACK;
 	timeout_func = tw_callback_timeout;
-	cancel_func = NULL;
 	refc = 1;
 	break;
 
     case ERTS_TMR_BIF:
 
 	timeout_func = tw_bif_timer_timeout;
-	cancel_func = NULL;
 	if (is_internal_pid(rcvr)) {
 	    tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC;
 	    tmr->head.receiver.proc = (Process *) rcvrp;
@@ -1129,7 +1119,6 @@ create_tw_timer(ErtsSchedulerData *esdp,
     erts_twheel_set_timer(esdp->timer_wheel,
 			  &tmr->u.tw_tmr,
 			  timeout_func,
-			  cancel_func,
 			  tmr,
 			  timeout_pos);
 
@@ -1452,7 +1441,6 @@ create_hl_timer(ErtsSchedulerData *esdp,
 	erts_twheel_set_timer(esdp->timer_wheel,
 			      &srv->service_timer,
 			      hlt_service_timeout,
-			      NULL,
 			      (void *) esdp,
 			      tmr->timeout);
 	srv->next_timeout = tmr;
@@ -1574,7 +1562,6 @@ hlt_service_timeout(void *vesdp)
 	erts_twheel_set_timer(esdp->timer_wheel,
 			      &srv->service_timer,
 			      hlt_service_timeout,
-			      NULL,
 			      vesdp,
 			      tmr->timeout);
 }
@@ -1631,7 +1618,6 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr)
 		erts_twheel_set_timer(esdp->timer_wheel,
 				      &srv->service_timer,
 				      hlt_service_timeout,
-				      NULL,
 				      (void *) esdp,
 				      smlst->timeout);
 	    }
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 5baf533acd..267ca7e8a6 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2815,7 +2815,6 @@ start_aux_work_timer(ErtsSchedulerData *esdp)
     erts_twheel_set_timer(esdp->timer_wheel,
 			  &aux_work_tmo->timer.data,
 			  aux_work_timeout,
-			  NULL,
 			  (void *) esdp,
 			  tmo);
 }
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 2fb4b754d7..00b4a84b4a 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -420,21 +420,18 @@ void erts_sched_init_time_sup(ErtsSchedulerData *esdp);
 */
 typedef struct erl_timer {
     ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */
-    struct erl_timer* next;	/* next entry tiw slot or chain */
+    struct erl_timer* next;     /* next entry tiw slot or chain */
     struct erl_timer* prev;	/* prev entry tiw slot or chain */
     void (*timeout)(void*); /* called when timeout */
-    void (*cancel)(void*);  /* called when cancel (may be NULL) */
     void* arg;              /* argument to timeout/cancel procs */
     int slot;
 } ErtsTWheelTimer;
 
 typedef void (*ErlTimeoutProc)(void*);
-typedef void (*ErlCancelProc)(void*);
 
 void erts_twheel_set_timer(ErtsTimerWheel *tiw,
 			   ErtsTWheelTimer *p, ErlTimeoutProc timeout,
-			   ErlCancelProc cancel, void *arg,
-			   ErtsMonotonicTime timeout_pos);
+			   void *arg, ErtsMonotonicTime timeout_pos);
 void erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p);
 ErtsTimerWheel *erts_create_timer_wheel(ErtsSchedulerData *esdp);
 
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index cf9d3adc86..c69fec3c80 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -678,7 +678,6 @@ check_time_correction(void *vesdp)
     erts_twheel_set_timer(esdp->timer_wheel,
 			  &time_sup.inf.c.parmon.timer,
 			  check_time_correction,
-			  NULL,
 			  (void *) esdp,
 			  timeout_pos);
 }
@@ -729,7 +728,6 @@ check_time_offset(void *vesdp)
     erts_twheel_set_timer(esdp->timer_wheel,
 			  &time_sup.inf.c.parmon.timer,
 			  check_time_offset,
-			  NULL,
 			  vesdp,
 			  timeout_pos);
 }
@@ -836,7 +834,6 @@ late_init_time_correction(ErtsSchedulerData *esdp)
     erts_twheel_set_timer(esdp->timer_wheel,
 			  &time_sup.inf.c.parmon.timer,
 			  check_func,
-			  NULL,
 			  (quick_init_drift_adj
 			   ? NULL
 			   : esdp),
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index fdf74498c9..53896be19e 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -543,7 +543,6 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp)
     tiw->sentinel.next = &tiw->sentinel;
     tiw->sentinel.prev = &tiw->sentinel;
     tiw->sentinel.timeout = NULL;
-    tiw->sentinel.cancel = NULL;
     tiw->sentinel.arg = NULL;
     return tiw;
 }
@@ -577,14 +576,12 @@ erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode)
 void
 erts_twheel_set_timer(ErtsTimerWheel *tiw,
 		      ErtsTWheelTimer *p, ErlTimeoutProc timeout,
-		      ErlCancelProc cancel, void *arg,
-		      ErtsMonotonicTime timeout_pos)
+		      void *arg, ErtsMonotonicTime timeout_pos)
 {
     ErtsMonotonicTime timeout_time;
     ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
 
     p->timeout = timeout;
-    p->cancel = cancel;
     p->arg = arg;
 
     ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE);
@@ -632,14 +629,8 @@ void
 erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
 {
     if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) {
-	ErlCancelProc cancel;
-	void *arg;
         ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
 	remove_timer(tiw, p);
-	cancel = p->cancel;
-	arg = p->arg;
-	if (cancel)
-	    (*cancel)(arg);
         ERTS_MSACC_POP_STATE_M_X();
     }
 }
-- 
cgit v1.2.3


From 0981bfd7afee64c3c1f48db134d81e57c30e31c7 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 6 Feb 2017 20:39:02 +0100
Subject: Timer wheel divided into a "soon wheel" and a "later wheel"

The old single wheel implementation handled about 65 seconds. The
new dual wheel implementation handles more than 37 hours. The dual
wheels have also been shrunk compared to the single wheel, so the
total memory consumption for timer wheels have been cut in half.
---
 erts/emulator/beam/break.c    |   2 +-
 erts/emulator/beam/erl_time.h |  50 ++-
 erts/emulator/beam/time.c     | 810 ++++++++++++++++++++++++++++++++----------
 3 files changed, 671 insertions(+), 191 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 0b40d70cb7..538ab0d947 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -590,7 +590,7 @@ do_break(void)
 #endif
 #ifdef DEBUG
 	case 't':
-	    erts_p_slpq();
+	    /* erts_p_slpq(); */
 	    return;
 	case 'b':
 	    bin_check();
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 00b4a84b4a..699e69e991 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -21,19 +21,52 @@
 #ifndef ERL_TIME_H__
 #define ERL_TIME_H__
 
-/* timer wheel size NEED to be a power of 2 */
-#ifdef SMALL_MEMORY
-#define ERTS_TIW_SIZE (1 << 13)
-#else
-#define ERTS_TIW_SIZE (1 << 16)
+#if 0
+#  define ERTS_TW_DEBUG
+#endif
+#if defined(DEBUG) && !defined(ERTS_TW_DEBUG)
+#  define ERTS_TW_DEBUG
 #endif
 
-#if defined(DEBUG) || 0
+#if defined(ERTS_TW_DEBUG)
 #define ERTS_TIME_ASSERT(B) ERTS_ASSERT(B)
 #else
 #define ERTS_TIME_ASSERT(B) ((void) 1)
 #endif
 
+#ifdef ERTS_TW_DEBUG
+/*
+ * Soon wheel will handle about 1 seconds
+ * Later wheel will handle about 8 minutes
+ */
+#  define ERTS_TW_SOON_WHEEL_BITS 10
+#  define ERTS_TW_LATER_WHEEL_BITS 10
+#else
+#  ifdef SMALL_MEMORY
+/*
+ * Soon wheel will handle about 4 seconds
+ * Later wheel will handle about 2 hours and 19 minutes
+ */
+#    define ERTS_TW_SOON_WHEEL_BITS 12
+#    define ERTS_TW_LATER_WHEEL_BITS 12
+#  else
+/*
+ * Soon wheel will handle about 16 seconds
+ * Later wheel will handle about 37 hours and 16 minutes
+ */
+#    define ERTS_TW_SOON_WHEEL_BITS 14
+#    define ERTS_TW_LATER_WHEEL_BITS 14
+#  endif
+#endif
+
+/*
+ * Number of slots in each timer wheel...
+ *
+ * These *need* to be a power of 2
+ */
+#define ERTS_TW_SOON_WHEEL_SIZE (1 << ERTS_TW_SOON_WHEEL_BITS)
+#define ERTS_TW_LATER_WHEEL_SIZE (1 << ERTS_TW_LATER_WHEEL_BITS)
+
 typedef enum {
     ERTS_NO_TIME_WARP_MODE,
     ERTS_SINGLE_TIME_WARP_MODE,
@@ -103,7 +136,10 @@ Eterm erts_system_time_source(struct process*c_p);
 #define ERTS_CLKTCK_RESOLUTION (erts_time_sup__.r.o.clktck_resolution)
 #endif
 
-#define ERTS_TIMER_WHEEL_MSEC (ERTS_TIW_SIZE/(ERTS_CLKTCK_RESOLUTION/1000))
+#define ERTS_TW_SOON_WHEEL_MSEC (ERTS_TW_SOON_WHEEL_SIZE/(ERTS_CLKTCK_RESOLUTION/1000))
+#define ERTS_TW_LATER_WHEEL_MSEC (ERTS_TW_LATER_WHEEL_SIZE*ERTS_TW_SOON_WHEEL_MSEC/2)
+
+#define ERTS_TIMER_WHEEL_MSEC ERTS_TW_LATER_WHEEL_MSEC
 
 struct erts_time_sup_read_only__ {
     ErtsMonotonicTime monotonic_time_unit;
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 53896be19e..7b5ff7b992 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -17,57 +17,124 @@
  * 
  * %CopyrightEnd%
  */
+ 
 
 /*
- * TIMING WHEEL
+ * TIMER WHEEL
+ *
+ *
+ * The time scale used for timers is Erlang monotonic time. The
+ * time unit used is ERTS specific clock ticks. A clock tick is
+ * currently defined to 1 millisecond. That is, the resolution of
+ * timers triggered by the runtime system is 1 millisecond.
  * 
- * Timeouts kept in an wheel. A timeout is measured relative to the
- * current slot (tiw_pos) in the wheel, and inserted at slot 
- * (tiw_pos + timeout) % TIW_SIZE. Each timeout also has a count
- * equal to timeout/TIW_SIZE, which is needed since the time axis
- * is wrapped arount the wheel. 
+ * When a timer is set, it is determined at what Erlang monotonic
+ * time, in clock ticks, it should be triggered.
  *
- * Several slots may be processed in one operation. If the number of
- * slots is greater that the wheel size, the wheel is only traversed
- * once,
+ * The 'pos' field of the wheel corresponds to current time of
+ * the wheel. That is, it corresponds to Erlang monotonic time in
+ * clock tick time unit. The 'pos' field of the wheel is
+ * monotonically increased when erts_bump_timers() is called. All
+ * timers in the wheel that have a time less than or equal to
+ * 'pos' are triggered by the bump operation. The bump operation
+ * may however be spread over multiple calls to erts_bump_timers()
+ * if there are a lots of timers to trigger.
+ *
+ * Each scheduler thread maintains its own timer wheel. The timer
+ * wheel of a scheduler, however, actually consists of two wheels.
+ * A soon wheel and a later wheel.
+ *
+ *
+ * -- The Soon Wheel --
+ *
+ * The soon wheel contain timers that should be triggered soon.
+ * That is, they are soon to be triggered. Each slot in the soon
+ * wheel is 1 clock tick wide. The number of slots in the soon
+ * wheel is currently 2¹⁴. That is, it contains timers in the
+ * range ('pos', 'pos' + 2¹⁴] which corresponds to a bit more
+ * than 16 seconds.
+ *
+ * When the bump operation is started, 'pos' is moved forward to a
+ * position that corresponds to current Erlang monotonic time. Then
+ * all timers that are in the range (old 'pos', new 'pos'] are
+ * triggered. During a bump operation, the soon wheel may contain
+ * timers in the two, possibly overlapping, ranges (old 'pos',
+ * old 'pos' + 2¹⁴], and (new 'pos', new 'pos' + 2¹⁴]. This may
+ * occur even if the bump operation doesn't yield, due to timeout
+ * callbacks inserting new timers.
+ *
+ *
+ * -- The Later Wheel --
+ *
+ * The later wheel contain timers that are further away from 'pos'
+ * than the width of the soon timer wheel. That is, currently
+ * timers further away from 'pos' than 2¹⁴ clock ticks. The width
+ * of each slot in the later wheel is half the width of the soon
+ * wheel. That is, each slot is currently 2¹³ clock ticks wide
+ * which corresponds to about 8 seconds. If three timers of the
+ * times 'pos' + 17000, 'pos' + 18000, and 'pos' + 19000 are
+ * inserted, they will all end up in the same slot in the later
+ * wheel.
+ *
+ * The number of slots in the later wheel is currently the same as
+ * in the soon wheel, i.e. 2¹⁴. That is, one revolution of the later
+ * wheel currently corresponds to 2¹⁴×2¹³ clock ticks which is
+ * almost 37 ½ hour. Timers even further away than that are put in
+ * the later slot identified by their time modulo the size of the later
+ * wheel. Such timers are however very uncommon. Most timers used
+ * by the runtime system will utilize the high level timer API.
+ * The high level timer implementation will not insert timers
+ * further away then one revolution into the later wheel. It will
+ * instead keep such timers in a tree of very long timers. The
+ * high level timer implementation utilize one timer wheel timer
+ * for the management of this tree of timers. This timer is set to
+ * the closest timeout in the tree. This timer may however be
+ * further away than one revolution in the later wheel.
+ *
+ * The 'later.pos' field identifies next position in the later wheel.
+ * 'later.pos' is always increased by the width of a later wheel slot.
+ * That is, currently 2¹³ clock ticks. When 'pos' is moved (during
+ * a bump operation) closer to 'later.pos' than the width of a later
+ * wheel slot, i.e. currently when 'pos' + 2¹³ ≥ 'later.pos', we
+ * inspect the slot identified by 'later.pos' and then move 'later.pos'
+ * forward. When inspecting the later slot we move all timers in the
+ * slot, that are in the soon wheel range, from the later wheel to
+ * the soon wheel. Timers one or more revolutions of the later wheel
+ * away are kept in the slot.
+ *
+ * During normal operation, timers originally located in the later
+ * wheel will currently be moved into the soon wheel about 8 to
+ * 16 seconds before they should be triggered. During extremely
+ * heavy load, the scheduler might however be heavily delayed, so
+ * the code must be prepared for situations where time for
+ * triggering the timer has passed when we inspect the later wheel
+ * slot, and then trigger the timer immediately. We must also be
+ * prepared to inspect multiple later wheel slots at once due to the
+ * delay.
+ *
+ *
+ * -- Slot Management --
+ *
+ * All timers of a slot are placed in a circular double linked
+ * list. This makes insertion and removal of a timer O(1).
+ *
+ * While bumping timers in a slot, we move the circular list
+ * away from the slot, and refer to it from the 'sentinel'
+ * field. The list will stay there until we are done with it
+ * even if the bump operation should yield. The cancel operation
+ * can remove the timer from this position as well as from the
+ * slot position by just removing it from the circular double
+ * linked list that it is in.
+ *
+ * -- At Once List --
+ *
+ * If a timer is set that has a time earlier or equal to 'pos',
+ * it is not inserted into the wheel. It is instead inserted,
+ * into a list referred to by the 'at_once' field. When the
+ * bump operation is performed thise timers will be triggered
+ * at once.
  *
- * The following example shows a time axis where there is one timeout
- * at each "tick", and where 1, 2, 3 ... wheel slots are released in
- * one operation. The notation "<x" means "release all items with
- * counts less than x". 
  *
- * Size of wheel: 4
- * 
- *   --|----|----|----|----|----|----|----|----|----|----|----|----|----
- *    0.0  0.1  0.2  0.3  1.0  1.1  1.2  1.3  2.0  2.1  2.2  2.3  3.0
- * 
- * 1   [    )
- *     <1  0.1  0.2  0.3  0.0  1.1  1.2  1.3  1.0  2.1  2.2  2.3  2.0
- * 
- * 2   [         )
- *     <1   <1  0.2  0.3  0.0  0.1  1.2  1.3  1.0  1.1  2.2  2.3  2.0
- * 
- * 3   [              )
- *     <1   <1   <1  0.3  0.0  0.1  0.2  1.3  1.0  1.1  1.2  2.3  2.0
- * 
- * 4   [                   )
- *     <1   <1   <1   <1  0.0  0.1  0.2  0.3  1.0  1.1  1.2  1.3  2.0
- * 
- * 5   [                        )
- *     <2   <1   <1   <1.      0.1  0.2  0.3  0.0  1.1  1.2  1.3  1.0
- * 
- * 6   [                             )
- *     <2   <2   <1   <1.           0.2  0.3  0.0  0.1  1.2  1.3  1.0
- * 
- * 7   [                                  )
- *     <2   <2   <2   <1.                0.3  0.0  0.1  0.2  1.3  1.0
- * 
- * 8   [                                       )   
- *     <2   <2   <2   <2.                     0.0  0.1  0.2  0.3  1.0
- * 
- * 9   [                                            )
- *     <3   <2   <2   <2.                          0.1  0.2  0.3  0.0
- * 
  */
 
 #ifdef HAVE_CONFIG_H
@@ -90,6 +157,10 @@
 #endif
 
 #if 0
+#  define ERTS_TW_HARD_DEBUG
+#endif
+
+#if defined(ERTS_TW_HARD_DEBUG) && !defined(ERTS_TW_DEBUG)
 #  define ERTS_TW_DEBUG
 #endif
 #if defined(DEBUG) && !defined(ERTS_TW_DEBUG)
@@ -97,17 +168,48 @@
 #endif
 
 #undef ERTS_TW_ASSERT
-#if defined(ERTS_TW_DEBUG)
+#if defined(ERTS_TW_DEBUG) 
 #  define ERTS_TW_ASSERT(E) ERTS_ASSERT(E)
 #else
 #  define ERTS_TW_ASSERT(E) ((void) 1)
 #endif
 
 #ifdef ERTS_TW_DEBUG
-#  define ERTS_TWHEEL_BUMP_YIELD_LIMIT 5
+#  define ERTS_TWHEEL_BUMP_YIELD_LIMIT        500
 #else
-#  define ERTS_TWHEEL_BUMP_YIELD_LIMIT 100
+#  define ERTS_TWHEEL_BUMP_YIELD_LIMIT        10000
 #endif
+#define ERTS_TW_COST_SLOT                     1
+#define ERTS_TW_COST_SLOT_MOVE                5
+#define ERTS_TW_COST_TIMEOUT                  100
+
+/*
+ * Every slot in the soon wheel is a clock tick (as defined
+ * by ERTS) wide. A clock tick is currently 1 milli second.
+ */
+
+#define ERTS_TW_SOON_WHEEL_FIRST_SLOT 0
+#define ERTS_TW_SOON_WHEEL_END_SLOT \
+    (ERTS_TW_SOON_WHEEL_FIRST_SLOT + ERTS_TW_SOON_WHEEL_SIZE)
+
+#define ERTS_TW_SOON_WHEEL_MASK (ERTS_TW_SOON_WHEEL_SIZE-1)
+
+/*
+ * Every slot in the later wheel is as wide as half the size
+ * of the soon wheel.
+ */
+
+#define ERTS_TW_LATER_WHEEL_SHIFT (ERTS_TW_SOON_WHEEL_BITS - 1)
+#define ERTS_TW_LATER_WHEEL_SLOT_SIZE \
+    ((ErtsMonotonicTime) (1 << ERTS_TW_LATER_WHEEL_SHIFT))
+#define ERTS_TW_LATER_WHEEL_POS_MASK \
+    (~((ErtsMonotonicTime) (1 << ERTS_TW_LATER_WHEEL_SHIFT)-1))
+
+#define ERTS_TW_LATER_WHEEL_FIRST_SLOT ERTS_TW_SOON_WHEEL_SIZE
+#define ERTS_TW_LATER_WHEEL_END_SLOT \
+    (ERTS_TW_LATER_WHEEL_FIRST_SLOT + ERTS_TW_LATER_WHEEL_SIZE)
+
+#define ERTS_TW_LATER_WHEEL_MASK (ERTS_TW_LATER_WHEEL_SIZE-1)
 
 /* Actual interval time chosen by sys_init_time() */
 
@@ -120,7 +222,7 @@ static int tiw_itime; /* Constant after init */
 #endif
 
 struct ErtsTimerWheel_ {
-    ErtsTWheelTimer *w[ERTS_TIW_SIZE];
+    ErtsTWheelTimer *w[ERTS_TW_SOON_WHEEL_SIZE + ERTS_TW_LATER_WHEEL_SIZE];
     ErtsMonotonicTime pos;
     Uint nto;
     struct {
@@ -128,14 +230,58 @@ struct ErtsTimerWheel_ {
 	ErtsTWheelTimer *tail;
 	Uint nto;
     } at_once;
+    struct {
+        Uint nto;
+    } soon;
+    struct {
+        ErtsMonotonicTime pos;
+    } later;
     int yield_slot;
     int yield_slots_left;
-    int yield_start_pos;
     ErtsTWheelTimer sentinel;
     int true_next_timeout_time;
     ErtsMonotonicTime next_timeout_time;
 };
 
+#define ERTS_TW_BUMP_LATER_WHEEL(TIW) \
+    ((tiw)->pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE >= (TIW)->later.pos)
+
+static int bump_later_wheel(ErtsTimerWheel *tiw, int *yield_count_p);
+
+static ERTS_INLINE int
+soon_slot(ErtsMonotonicTime soon_pos)
+{
+    ErtsMonotonicTime slot = soon_pos;
+    slot &= ERTS_TW_SOON_WHEEL_MASK;
+
+    ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= slot);
+    ERTS_TW_ASSERT(slot < ERTS_TW_SOON_WHEEL_END_SLOT);
+
+    return (int) slot;
+}
+
+static ERTS_INLINE int
+later_slot(ErtsMonotonicTime later_pos)
+{
+    ErtsMonotonicTime slot = later_pos;
+    slot >>= ERTS_TW_LATER_WHEEL_SHIFT;
+    slot &= ERTS_TW_LATER_WHEEL_MASK;
+    slot += ERTS_TW_LATER_WHEEL_FIRST_SLOT;
+
+    ERTS_TW_ASSERT(ERTS_TW_LATER_WHEEL_FIRST_SLOT <= slot);
+    ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT);
+
+    return (int) slot;
+}
+
+#ifdef ERTS_TW_HARD_DEBUG
+#define ERTS_HARD_DBG_CHK_WHEELS(TIW, CHK_MIN_TPOS) \
+    hrd_dbg_check_wheels((TIW), (CHK_MIN_TPOS))
+static void hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos);
+#else
+#define ERTS_HARD_DBG_CHK_WHEELS(TIW, CHK_MIN_TPOS)
+#endif
+
 static ERTS_INLINE ErtsMonotonicTime
 find_next_timeout(ErtsSchedulerData *esdp,
 		  ErtsTimerWheel *tiw,
@@ -143,11 +289,14 @@ find_next_timeout(ErtsSchedulerData *esdp,
 		  ErtsMonotonicTime curr_time,       /* When !search_all */
 		  ErtsMonotonicTime max_search_time) /* When !search_all */
 {
-    int start_ix, tiw_pos_ix;
-    ErtsTWheelTimer *p;
+    int start_ix, tiw_pos_ix, pos_inc, wheel_start_ix, wheel_end_ix;
     int true_min_timeout = 0;
     ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos;
 
+    ERTS_HARD_DBG_CHK_WHEELS(tiw, 0);
+
+    ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TWHEEL_SLOT_INACTIVE);
+
     if (tiw->nto == 0) { /* no timeouts in wheel */
 	if (!search_all)
 	    min_timeout_pos = tiw->pos;
@@ -156,58 +305,115 @@ find_next_timeout(ErtsSchedulerData *esdp,
 	    tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
 	}
 	min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY);
-	goto found_next;
+	goto done;
     }
 
-    slot_timeout_pos = min_timeout_pos = tiw->pos;
     if (search_all)
-	min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY);
+        min_timeout_pos = tiw->pos + ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY);
     else
-	min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time);
+        min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time);
+
+    if (!tiw->soon.nto) {
+        /* Select later wheel... */
+        slot_timeout_pos = tiw->later.pos;
+        start_ix = tiw_pos_ix = later_slot(slot_timeout_pos);
+        pos_inc = ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+        wheel_start_ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT;
+        wheel_end_ix = ERTS_TW_LATER_WHEEL_END_SLOT;
+        /* Pre-timeout for move from later to soon wheel... */
+        slot_timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+    }
+    else {
+        /* Select soon wheel... */
+
+        /*
+         * Besides inspecting the soon wheel we
+         * may also have to inspect two slots in the
+         * later wheel which potentially can trigger
+         * timeouts before timeouts in soon wheel...
+         */
+        slot_timeout_pos = tiw->later.pos;
+        slot_timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+        if (slot_timeout_pos < min_timeout_pos) {
+            int fslot = later_slot(tiw->later.pos);
+            if (tiw->w[fslot]) {
+                min_timeout_pos = slot_timeout_pos;
+                true_min_timeout = 1;
+            }
+            else {
+                slot_timeout_pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+                if (slot_timeout_pos < min_timeout_pos) {
+                    fslot++;
+                    if (fslot == ERTS_TW_LATER_WHEEL_END_SLOT)
+                        fslot = ERTS_TW_LATER_WHEEL_FIRST_SLOT;
+                    if (tiw->w[fslot]) {
+                        min_timeout_pos = slot_timeout_pos;
+                        true_min_timeout = 1;
+                    }
+                }
+            }
+        }
 
-    start_ix = tiw_pos_ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1));
+        slot_timeout_pos = tiw->pos;
+        start_ix = tiw_pos_ix = soon_slot(tiw->pos);
+        pos_inc = 1;
+        wheel_start_ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
+        wheel_end_ix = ERTS_TW_SOON_WHEEL_END_SLOT;
+    }
 
+    /*
+     * Scan selected wheel...
+     */
     do {
-	if (++slot_timeout_pos >= min_timeout_pos)
-	    break;
-
-	p = tiw->w[tiw_pos_ix];
-
-	if (p) {
-	    ErtsTWheelTimer *end = p;
-
-	    do  {
-		ErtsMonotonicTime timeout_pos;
-		timeout_pos = p->timeout_pos;
-		if (min_timeout_pos > timeout_pos) {
-		    true_min_timeout = 1;
-		    min_timeout_pos = timeout_pos;
-		    if (min_timeout_pos <= slot_timeout_pos)
-			goto found_next;
-		}
-		p = p->next;
-	    } while (p != end);
-	}
-
-	tiw_pos_ix++;
-	if (tiw_pos_ix == ERTS_TIW_SIZE)
-	    tiw_pos_ix = 0;
+        if (tiw->w[tiw_pos_ix]) {
+            min_timeout_pos = slot_timeout_pos;
+            true_min_timeout = 1;
+            break;
+        }
+
+        slot_timeout_pos += pos_inc;
+        if (slot_timeout_pos >= min_timeout_pos)
+            break;
+
+        tiw_pos_ix++;
+        if (tiw_pos_ix == wheel_end_ix)
+            tiw_pos_ix = wheel_start_ix;
     } while (start_ix != tiw_pos_ix);
 
-found_next:
+done:
 
     min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos);
     tiw->next_timeout_time = min_timeout;
     tiw->true_next_timeout_time = true_min_timeout;
 
+    ERTS_HARD_DBG_CHK_WHEELS(tiw, 1);
+
     return min_timeout;
 }
 
+static ERTS_INLINE void
+insert_timer_into_at_once_list(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
+{
+    tiw->at_once.nto++;
+    p->next = NULL;
+    p->prev = tiw->at_once.tail;
+    if (tiw->at_once.tail) {
+        ERTS_TW_ASSERT(tiw->at_once.head);
+        tiw->at_once.tail->next = p;
+    }
+    else {	
+        ERTS_TW_ASSERT(!tiw->at_once.head);
+        tiw->at_once.head = p;
+    }
+    tiw->at_once.tail = p;
+    p->slot = ERTS_TWHEEL_SLOT_AT_ONCE;
+}
+
 static ERTS_INLINE void
 insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p)
 {
-    ERTS_TW_ASSERT(slot >= 0);
-    ERTS_TW_ASSERT(slot < ERTS_TIW_SIZE);
+    ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= slot);
+    ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT);
     p->slot = slot;
     if (!tiw->w[slot]) {
 	tiw->w[slot] = p;
@@ -223,6 +429,10 @@ insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p)
 	prev->next = p;
 	next->prev = p;
     }
+    if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) {
+        ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
+        tiw->soon.nto++;
+    }
 }
 
 static ERTS_INLINE void
@@ -231,13 +441,13 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
     int slot = p->slot;
     ERTS_TW_ASSERT(slot != ERTS_TWHEEL_SLOT_INACTIVE);
 
-    if (slot >= 0) {
+    if (slot >= ERTS_TW_SOON_WHEEL_FIRST_SLOT) {
 	/*
 	 * Timer in wheel or in circular
 	 * list of timers currently beeing
 	 * triggered (referred by sentinel).
 	 */
-	ERTS_TW_ASSERT(slot < ERTS_TIW_SIZE);
+	ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT);
 
 	if (p->next == p) {
 	    ERTS_TW_ASSERT(tiw->w[slot] == p);
@@ -249,6 +459,8 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
 	    p->prev->next = p->next;
 	    p->next->prev = p->prev;
 	}
+        if (slot < ERTS_TW_SOON_WHEEL_END_SLOT)
+            tiw->soon.nto--;
     }
     else {
 	/* Timer in "at once" queue... */
@@ -270,8 +482,6 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
     }
 
     p->slot = ERTS_TWHEEL_SLOT_INACTIVE;
-
-    tiw->nto--;
 }
 
 ErtsMonotonicTime
@@ -299,13 +509,13 @@ debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos)
     ErtsTWheelTimer *tmr;
     ErtsMonotonicTime tmp;
 
-    ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1));
+    ix = soon_slot(tiw->pos);
     tmp = skip_to_pos - tiw->pos;
     ERTS_TW_ASSERT(tmp >= 0);
-    if (tmp < (ErtsMonotonicTime) ERTS_TIW_SIZE)
+    if (tmp < (ErtsMonotonicTime) ERTS_TW_SOON_WHEEL_SIZE)
 	slots = (int) tmp;
     else
-	slots = ERTS_TIW_SIZE;
+	slots = ERTS_TW_SOON_WHEEL_SIZE;
 
      while (slots > 0) {
 	 tmr = tiw->w[ix];
@@ -317,10 +527,41 @@ debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos)
 	     } while (tmr != end);
 	 }
 	 ix++;
-	 if (ix == ERTS_TIW_SIZE)
-	     ix = 0;
+	 if (ix == ERTS_TW_SOON_WHEEL_END_SLOT)
+	     ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
 	 slots--;
     }
+
+    ix = later_slot(tiw->later.pos);
+    tmp = skip_to_pos;
+    tmp &= ERTS_TW_LATER_WHEEL_POS_MASK;
+    if (tmp > tiw->later.pos) {
+        tmp -= tiw->later.pos;
+        tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+        ERTS_TW_ASSERT(tmp >= 0);
+        if (tmp < (ErtsMonotonicTime) ERTS_TW_LATER_WHEEL_SIZE)
+            slots = (int) tmp;
+        else
+            slots = ERTS_TW_LATER_WHEEL_SIZE;
+
+        while (slots > 0) {
+            tmr = tiw->w[ix];
+            if (tmr) {
+                ErtsMonotonicTime tpos = tmr->timeout_pos;
+                ErtsTWheelTimer *end = tmr;
+                do {
+                    tpos &= ERTS_TW_LATER_WHEEL_POS_MASK;
+                    tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+                    ERTS_TW_ASSERT(tpos > skip_to_pos);
+                    tmr = tmr->next;
+                } while (tmr != end);
+            }
+            ix++;
+            if (ix == ERTS_TW_LATER_WHEEL_END_SLOT)
+                ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT;
+            slots--;
+        }
+    }
 }
 #endif
 
@@ -339,8 +580,8 @@ timeout_timer(ErtsTWheelTimer *p)
 void
 erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 {
-    int tiw_pos_ix, slots, yielded_slot_restarted, yield_count;
-    ErtsMonotonicTime bump_to, tmp_slots, old_pos;
+    int tiw_pos_ix, yielded_slot_restarted, yield_count, slots;
+    ErtsMonotonicTime bump_to;
     ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
 
     yield_count = ERTS_TWHEEL_BUMP_YIELD_LIMIT;
@@ -350,13 +591,16 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
      * where we left off when restarting after a yield.
      */
 
-    if (tiw->yield_slot >= 0) {
+    if (tiw->yield_slot >= ERTS_TW_SOON_WHEEL_FIRST_SLOT) {
 	yielded_slot_restarted = 1;
+	bump_to = tiw->pos;
+        if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT)
+            goto restart_yielded_later_slot;
 	tiw_pos_ix = tiw->yield_slot;
 	slots = tiw->yield_slots_left;
-	bump_to = tiw->pos;
-	old_pos = tiw->yield_start_pos;
-	goto restart_yielded_slot;
+        ASSERT(0 <= slots && slots <= ERTS_TW_SOON_WHEEL_SIZE);
+        tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
+        goto restart_yielded_soon_slot;
     }
 
     do {
@@ -368,8 +612,6 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 	while (1) {
 	    ErtsTWheelTimer *p;
 
-	    old_pos = tiw->pos;
-
 	    if (tiw->nto == 0) {
 	    empty_wheel:
 		ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to);
@@ -383,12 +625,12 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 
 	    p = tiw->at_once.head;
 	    while (p) {
-		if (--yield_count <= 0) {
+		if (yield_count <= 0) {
 		    ERTS_TW_ASSERT(tiw->nto > 0);
 		    ERTS_TW_ASSERT(tiw->at_once.nto > 0);
 		    tiw->yield_slot = ERTS_TWHEEL_SLOT_AT_ONCE;
 		    tiw->true_next_timeout_time = 1;
-		    tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos);
+		    tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to);
                     ERTS_MSACC_POP_STATE_M_X();
 		    return;
 		}
@@ -405,6 +647,8 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 
 		timeout_timer(p);
 
+                yield_count -= ERTS_TW_COST_TIMEOUT;
+
 		p = tiw->at_once.head;
 	    }
 
@@ -433,22 +677,34 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 		    ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, skip_until_pos);
 
 		    tiw->pos = skip_until_pos;
+
+                    skip_until_pos++;
+                    skip_until_pos &= ERTS_TW_LATER_WHEEL_POS_MASK;
+                    if (tiw->later.pos < skip_until_pos)
+                        tiw->later.pos = skip_until_pos;
 		}
 	    }
 
-	    tiw_pos_ix = (int) ((tiw->pos+1) & (ERTS_TIW_SIZE-1));
-	    tmp_slots = (bump_to - tiw->pos);
-	    if (tmp_slots < (ErtsMonotonicTime) ERTS_TIW_SIZE)
-		slots = (int) tmp_slots;
-	    else
-		slots = ERTS_TIW_SIZE;
+            {
+                ErtsMonotonicTime tmp_slots = bump_to - tiw->pos;
+                tmp_slots = (bump_to - tiw->pos);
+                if (tmp_slots < ERTS_TW_SOON_WHEEL_SIZE)
+                    slots = (int) tmp_slots;
+                else
+                    slots = ERTS_TW_SOON_WHEEL_SIZE;
+            }
 
+            tiw_pos_ix = soon_slot(tiw->pos+1);
 	    tiw->pos = bump_to;
 
-	    while (slots > 0) {
+            /* Timeout timers in soon wheel */
+	    while (slots) {
+
+                yield_count -= ERTS_TW_COST_SLOT;
 
 		p = tiw->w[tiw_pos_ix];
 		if (p) {
+                    /* timeout callback need tiw->pos to be up to date */
 		    if (p->next == p) {
 			ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel);
 			ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel);
@@ -463,18 +719,21 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 
 		    while (1) {
 
-			if (p->timeout_pos > bump_to) {
-			    /* Very unusual case... */
-			    ++yield_count;
-			    insert_timer_into_slot(tiw, tiw_pos_ix, p);
-			}
-			else {
-			    /* Normal case... */
-			    timeout_timer(p);
-			    tiw->nto--;
-			}
-
-		    restart_yielded_slot:
+                        ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= p->slot
+                                       && p->slot < ERTS_TW_SOON_WHEEL_END_SLOT);
+                        tiw->soon.nto--;
+                        if (p->timeout_pos <= bump_to) {
+                            timeout_timer(p);
+                            tiw->nto--;
+                            yield_count -= ERTS_TW_COST_TIMEOUT;
+                        }
+                        else {
+                            /* uncommon case */
+                            insert_timer_into_slot(tiw, tiw_pos_ix, p);
+                            yield_count -= ERTS_TW_COST_SLOT_MOVE;
+                        }
+
+		    restart_yielded_soon_slot:
 
 			p = tiw->sentinel.next;
 			if (p == &tiw->sentinel) {
@@ -482,12 +741,11 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 			    break;
 			}
 
-			if (--yield_count <= 0) {
+			if (yield_count <= 0) {
 			    tiw->true_next_timeout_time = 1;
-			    tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos);
+			    tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to);
 			    tiw->yield_slot = tiw_pos_ix;
 			    tiw->yield_slots_left = slots;
-			    tiw->yield_start_pos = old_pos;
                             ERTS_MSACC_POP_STATE_M_X();
 			    return; /* Yield! */
 			}
@@ -496,16 +754,22 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 			p->next->prev = &tiw->sentinel;
 		    }
 		}
-		tiw_pos_ix++;
-		if (tiw_pos_ix == ERTS_TIW_SIZE)
-		    tiw_pos_ix = 0;
-		slots--;
+
+                tiw_pos_ix++;
+                if (tiw_pos_ix == ERTS_TW_SOON_WHEEL_END_SLOT)
+                    tiw_pos_ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
+                slots--;
 	    }
+
+            if (ERTS_TW_BUMP_LATER_WHEEL(tiw)) {
+            restart_yielded_later_slot:
+                if (bump_later_wheel(tiw, &yield_count))
+                    return; /* Yield! */
+            }
 	}
 
     } while (yielded_slot_restarted);
 
-    tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
     tiw->true_next_timeout_time = 0;
     tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY;
 
@@ -514,6 +778,119 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
     ERTS_MSACC_POP_STATE_M_X();
 }
 
+static int
+bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
+{
+    ErtsMonotonicTime cpos = tiw->pos;
+    ErtsMonotonicTime later_pos = tiw->later.pos;
+    int ycount = *ycount_p;
+    int slots, fslot;
+
+    ERTS_HARD_DBG_CHK_WHEELS(tiw, 0);
+
+    if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) {
+        fslot = tiw->yield_slot;
+        slots = tiw->yield_slots_left;
+        ASSERT(0 <= slots && slots <= ERTS_TW_LATER_WHEEL_SIZE);
+        tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
+        goto restart_yielded_slot;
+    }
+    else {
+        ErtsMonotonicTime tmp_slots = cpos;
+        tmp_slots += ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+        tmp_slots -= later_pos;
+        tmp_slots /= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+        if (tmp_slots < ERTS_TW_LATER_WHEEL_SIZE)
+            slots = (int) tmp_slots;
+        else
+            slots = ERTS_TW_LATER_WHEEL_SIZE;
+    }
+
+    while (slots >= 0) {
+        ErtsTWheelTimer *p;
+
+        fslot = later_slot(later_pos);
+
+        ycount -= ERTS_TW_COST_SLOT;
+
+        p = tiw->w[fslot];
+
+        if (p) {
+
+            if (p->next == p) {
+                ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel);
+                ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel);
+            }
+            else {
+                tiw->sentinel.next = p->next;
+                tiw->sentinel.prev = p->prev;
+                tiw->sentinel.next->prev = &tiw->sentinel;
+                tiw->sentinel.prev->next = &tiw->sentinel;
+            }
+            tiw->w[fslot] = NULL;
+
+            while (1) {
+                ErtsMonotonicTime tpos = p->timeout_pos;
+
+                ERTS_TW_ASSERT(tpos >= later_pos);
+                ERTS_TW_ASSERT(p->slot == fslot);
+
+                if (tpos >= later_pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE) {
+                    /* keep in later slot; very uncommon... */
+                    insert_timer_into_slot(tiw, fslot, p);
+                    ycount -= ERTS_TW_COST_SLOT_MOVE;
+                }
+                else {
+                    ERTS_TW_ASSERT(tpos < cpos + ERTS_TW_SOON_WHEEL_SIZE);
+                    if (tpos > cpos) {
+                        /* move into soon wheel */
+                        insert_timer_into_slot(tiw, soon_slot(tpos), p);
+                        ycount -= ERTS_TW_COST_SLOT_MOVE;
+                    }
+                    else {
+                        /* trigger at once */
+                        timeout_timer(p);
+                        tiw->nto--;
+                        ycount -= ERTS_TW_COST_TIMEOUT;
+                    }
+                }
+
+            restart_yielded_slot:
+
+                p = tiw->sentinel.next;
+                if (p == &tiw->sentinel) {
+                    ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel);
+                    break;
+                }
+
+                if (ycount < 0) {
+                    tiw->later.pos = later_pos;
+		    tiw->true_next_timeout_time = 1;
+		    tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(cpos);
+                    tiw->yield_slot = fslot;
+                    tiw->yield_slots_left = slots;
+                    *ycount_p = 0;
+                    ERTS_HARD_DBG_CHK_WHEELS(tiw, 0);
+                    return 1; /* Yield! */
+                }
+
+                tiw->sentinel.next = p->next;
+                p->next->prev = &tiw->sentinel;
+            }
+        }
+        slots--;
+        later_pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+    }
+
+    ERTS_HARD_DBG_CHK_WHEELS(tiw, 0);
+
+    tiw->later.pos = later_pos;
+
+    *ycount_p = ycount;
+
+    return 0;
+}
+
 Uint
 erts_timer_wheel_memory_size(void)
 {
@@ -524,11 +901,11 @@ ErtsTimerWheel *
 erts_create_timer_wheel(ErtsSchedulerData *esdp)
 {
     ErtsMonotonicTime mtime;
-    int i;
+    int i = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
     ErtsTimerWheel *tiw;
     tiw = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_WHEEL,
 					     sizeof(ErtsTimerWheel));
-    for(i = 0; i < ERTS_TIW_SIZE; i++)
+    for(; i < ERTS_TW_LATER_WHEEL_END_SLOT; i++)
 	tiw->w[i] = NULL;
 
     mtime = erts_get_monotonic_time(esdp);
@@ -537,6 +914,10 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp)
     tiw->at_once.head = NULL;
     tiw->at_once.tail = NULL;
     tiw->at_once.nto = 0;
+    tiw->soon.nto = 0;
+    tiw->later.pos = tiw->pos;
+    tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK;
+    tiw->later.pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE;
     tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
     tiw->true_next_timeout_time = 0;
     tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY;
@@ -586,36 +967,40 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw,
 
     ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE);
 
+    tiw->nto++;
     if (timeout_pos <= tiw->pos) {
-	tiw->nto++;
-	tiw->at_once.nto++;
-	p->next = NULL;
-	p->prev = tiw->at_once.tail;
-	if (tiw->at_once.tail) {
-	    ERTS_TW_ASSERT(tiw->at_once.head);
-	    tiw->at_once.tail->next = p;
-	}
-	else {	
-	    ERTS_TW_ASSERT(!tiw->at_once.head);
-	    tiw->at_once.head = p;
-	}
-	tiw->at_once.tail = p;
-	p->timeout_pos = tiw->pos;
-	p->slot = ERTS_TWHEEL_SLOT_AT_ONCE;
+        p->timeout_pos = tiw->pos;
+        insert_timer_into_at_once_list(tiw, p);
 	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->pos);
     }
     else {
 	int slot;
 
+	p->timeout_pos = timeout_pos;
+
 	/* calculate slot */
-	slot = (int) (timeout_pos & (ERTS_TIW_SIZE-1));
+        if (timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE) {
+            /* soon wheel */
+            slot = soon_slot(timeout_pos);
+        }
+        else {
+            /* later wheel */
+            slot = later_slot(timeout_pos);
+
+            /*
+             * Next timeout due to this timeout
+             * should be in good time before the
+             * actual timeout (one later wheel slot
+             * size). This, in order to move it
+             * from the later wheel to the soon
+             * wheel.
+             */
+            timeout_pos &= ERTS_TW_LATER_WHEEL_POS_MASK;
+            timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+        }
 
 	insert_timer_into_slot(tiw, slot, p);
-
-	tiw->nto++;
-
-	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
-	p->timeout_pos = timeout_pos;
+        timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
     }
 
     if (timeout_time < tiw->next_timeout_time) {
@@ -631,6 +1016,7 @@ erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
     if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) {
         ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
 	remove_timer(tiw, p);
+        tiw->nto--;
         ERTS_MSACC_POP_STATE_M_X();
     }
 }
@@ -658,7 +1044,9 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw,
 	    (*func)(arg, tmr->timeout_pos, tmr->arg);
     }
 
-    for (ix = 0; ix < ERTS_TIW_SIZE; ix++) {
+    for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
+         ix < ERTS_TW_LATER_WHEEL_END_SLOT;
+         ix++) {
 	tmr = tiw->w[ix];
 	if (tmr) {
 	    do {
@@ -670,36 +1058,92 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw,
     }
 }
 
-#ifdef ERTS_TW_DEBUG
-void erts_p_slpq(void)
+#ifdef ERTS_TW_HARD_DEBUG
+
+static void
+hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos)
 {
-    erts_printf("Not yet implemented...\n");
-#if 0
-    ErtsMonotonicTime current_time = erts_get_monotonic_time(NULL);
-    int i;
-    ErtsTWheelTimer* p;
-  
-    /* print the whole wheel, starting at the current position */
-    erts_printf("\ncurrent time = %bps tiw_pos = %d tiw_nto %d\n",
-		current_time, tiw->pos, tiw->nto);
-    i = tiw->pos;
-    if (tiw->w[i] != NULL) {
-	erts_printf("%d:\n", i);
-	for(p = tiw->w[i]; p != NULL; p = p->next) {
-	    erts_printf(" (timeout time %bps, slot %d)\n",
-			ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos),
-			p->slot);
-	}
+    int ix, soon_tmo, later_tmo, at_once_tmo;
+    ErtsMonotonicTime min_tpos;
+
+    min_tpos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time);
+
+    soon_tmo = 0;
+    for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
+         ix < ERTS_TW_SOON_WHEEL_END_SLOT;
+         ix++) {
+        ErtsTWheelTimer *p;
+
+        p = tiw->w[ix];
+        if (p) {
+            ErtsTWheelTimer *first = p;
+            do {
+                ErtsMonotonicTime tpos = p->timeout_pos;
+                ERTS_TW_ASSERT(p->slot == ix);
+                soon_tmo++;
+                ERTS_TW_ASSERT(ix == soon_slot(tpos));
+                ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
+                ERTS_TW_ASSERT(!check_min_tpos || tpos >= min_tpos);
+                ERTS_TW_ASSERT(p->next->prev == p);
+                p = p->next;
+            } while (p != first);
+        }
     }
-    for(i = ((i+1) & (ERTS_TIW_SIZE-1)); i != (tiw->pos & (ERTS_TIW_SIZE-1)); i = ((i+1) & (ERTS_TIW_SIZE-1))) {
-	if (tiw->w[i] != NULL) {
-	    erts_printf("%d:\n", i);
-	    for(p = tiw->w[i]; p != NULL; p = p->next) {
-		erts_printf(" (timeout time %bps, slot %d)\n",
-			    ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos), p->slot);
-	    }
-	}
+
+    later_tmo = 0;
+    for (ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT;
+         ix < ERTS_TW_LATER_WHEEL_END_SLOT;
+         ix++) {
+        ErtsTWheelTimer *p;
+
+        p = tiw->w[ix];
+        if (p) {
+            ErtsTWheelTimer *first = p;
+            do {
+                ErtsMonotonicTime tpos = p->timeout_pos;
+                ERTS_TW_ASSERT(p->slot == ix);
+                later_tmo++;
+                ERTS_TW_ASSERT(later_slot(tpos) == ix);
+                tpos &= ERTS_TW_LATER_WHEEL_POS_MASK;
+                tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+                ERTS_TW_ASSERT(!check_min_tpos || tpos >= min_tpos);
+                ERTS_TW_ASSERT(p->next->prev == p);
+                p = p->next;
+            } while (p != first);
+        }
     }
-#endif
+
+    if (tiw->yield_slot >= 0) {
+        ErtsTWheelTimer *p = tiw->sentinel.next;
+        while (p != &tiw->sentinel) {
+            ErtsMonotonicTime tpos = p->timeout_pos;
+            if (tiw->yield_slot >= ERTS_TW_SOON_WHEEL_SIZE) {
+                later_tmo++;
+                ERTS_TW_ASSERT(p->slot == later_slot(tpos));
+            }
+            else {
+                soon_tmo++;
+                ERTS_TW_ASSERT(p->slot == (tpos & ERTS_TW_SOON_WHEEL_MASK));
+                ERTS_TW_ASSERT(tpos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
+            }
+            p = p->next;
+        }
+    }
+
+    at_once_tmo = 0;
+    if (tiw->at_once.head) {
+        ErtsTWheelTimer *p = tiw->at_once.head;
+        while (p) {
+            ErtsMonotonicTime tpos = p->timeout_pos;
+            at_once_tmo++;
+            ERTS_TW_ASSERT(tpos <= tiw->pos);
+            p = p->next;
+        }
+    }
+
+    ERTS_TW_ASSERT(tiw->at_once.nto == at_once_tmo);
+    ERTS_TW_ASSERT(tiw->soon.nto == soon_tmo);
+    ERTS_TW_ASSERT(tiw->nto == soon_tmo + later_tmo + at_once_tmo);
 }
-#endif /* ERTS_TW_DEBUG */
+
+#endif
-- 
cgit v1.2.3


From 2ed143ac11d858db728bfa69f14fb34d7f449305 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 10 Feb 2017 17:16:51 +0100
Subject: Introduce timer slot range counters

Timer counters for ranges of slots in order to be able
to avoid inspecting large ranges of slots without timers.
---
 erts/emulator/beam/time.c | 374 +++++++++++++++++++++++++++++++++++++---------
 1 file changed, 301 insertions(+), 73 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 7b5ff7b992..1b6ca2b269 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -18,7 +18,6 @@
  * %CopyrightEnd%
  */
  
-
 /*
  * TIMER WHEEL
  *
@@ -134,6 +133,32 @@
  * bump operation is performed thise timers will be triggered
  * at once.
  *
+ * -- Searching for Next Timeout --
+ *
+ * In order to limit the amount of work needed in order to find
+ * next timeout, we keep track of total amount of timers in the
+ * wheels, total amount of timers in soon wheel, and the total
+ * amount of timers in each range of slots. Each slot range
+ * currently contain 512 slots.
+ *
+ * When next timeout is less than half the soon wheel width away
+ * we determine the exact timeout. Due to the timer counts of
+ * slot ranges, we currently at most need to search 1024 slots
+ * in the soon wheel. This besides inspecting slot range counts
+ * and two slots in the later wheel which potentially might trigger
+ * timeouts for moving timers from the later wheel to the soon wheel
+ * earlier than timeouts in the soon wheel.
+ *
+ * When next timeout is further away than half the soon wheel
+ * width we settle for the earliest possible timeout in the first
+ * non-empty slot range. The further away the next timeout is, the
+ * more likely it is that the next timeout change before we
+ * actually get there. That is, a change due to another timer is
+ * set to an earlier time and/or the timer is cancelled. It is
+ * therefore in this case no point determining next timeout
+ * exactly. If the state should not change, we will wake up a bit
+ * early and do a recalculation of next timeout and eventually
+ * we will be so close to it that we determine it exactly.
  *
  */
 
@@ -147,8 +172,8 @@
 #define ERTS_WANT_TIMER_WHEEL_API
 #include "erl_time.h"
 
-#define ERTS_MONOTONIC_DAY ERTS_SEC_TO_MONOTONIC(60*60*24)
-#define ERTS_CLKTCKS_DAY ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY)
+#define ERTS_MONOTONIC_WEEK ERTS_SEC_TO_MONOTONIC(7*60*60*24)
+#define ERTS_CLKTCKS_WEEK ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK)
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
 #define ASSERT_NO_LOCKED_LOCKS		erts_lc_check_exact(NULL, 0)
@@ -211,6 +236,21 @@
 
 #define ERTS_TW_LATER_WHEEL_MASK (ERTS_TW_LATER_WHEEL_SIZE-1)
 
+#define ERTS_TW_SCNT_BITS 9
+#define ERTS_TW_SCNT_SHIFT 
+#define ERTS_TW_SCNT_SIZE \
+    ((ERTS_TW_SOON_WHEEL_SIZE + ERTS_TW_LATER_WHEEL_SIZE) \
+     >> ERTS_TW_SCNT_BITS)
+
+#ifdef __GNUC__
+#if ERTS_TW_SOON_WHEEL_BITS < ERTS_TW_SCNT_BITS
+#  warning Consider larger soon timer wheel
+#endif
+#if ERTS_TW_SOON_WHEEL_BITS < ERTS_TW_SCNT_BITS
+#  warning Consider larger later timer wheel
+#endif
+#endif
+
 /* Actual interval time chosen by sys_init_time() */
 
 #if SYS_CLOCK_RESOLUTION == 1
@@ -223,6 +263,8 @@ static int tiw_itime; /* Constant after init */
 
 struct ErtsTimerWheel_ {
     ErtsTWheelTimer *w[ERTS_TW_SOON_WHEEL_SIZE + ERTS_TW_LATER_WHEEL_SIZE];
+    Sint scnt[ERTS_TW_SCNT_SIZE];
+    Sint bump_scnt[ERTS_TW_SCNT_SIZE];
     ErtsMonotonicTime pos;
     Uint nto;
     struct {
@@ -248,6 +290,108 @@ struct ErtsTimerWheel_ {
 
 static int bump_later_wheel(ErtsTimerWheel *tiw, int *yield_count_p);
 
+static ERTS_INLINE int
+scnt_get_ix(int slot)
+{
+    return slot >> ERTS_TW_SCNT_BITS;
+}
+
+static ERTS_INLINE void
+scnt_inc(Sint *scnt, int slot)
+{
+    scnt[slot >> ERTS_TW_SCNT_BITS]++;
+}
+
+#ifdef ERTS_TW_HARD_DEBUG
+
+static ERTS_INLINE void
+scnt_ix_inc(Sint *scnt, int six)
+{
+    scnt[six]++;
+}
+
+#endif
+
+static ERTS_INLINE void
+scnt_dec(Sint *scnt, int slot)
+{
+    scnt[slot >> ERTS_TW_SCNT_BITS]--;
+    ERTS_TW_ASSERT(scnt[slot >> ERTS_TW_SCNT_BITS] >= 0);
+}
+
+static ERTS_INLINE void
+scnt_ix_dec(Sint *scnt, int six)
+{
+    scnt[six]--;
+    ERTS_TW_ASSERT(scnt[six] >= 0);
+}
+
+static ERTS_INLINE void
+scnt_wheel_next(int *slotp, int *leftp, ErtsMonotonicTime *posp,
+                int *sixp, Sint *scnt, int first_slot,
+                int end_slot, ErtsMonotonicTime slot_sz)
+{
+    int slot = *slotp;
+    int left = *leftp;
+    int ix;
+
+    ERTS_TW_ASSERT(*leftp >= 0);
+
+    left--;
+    slot++;
+    if (slot == end_slot)
+        slot = first_slot;
+    ix = slot >> ERTS_TW_SCNT_BITS;
+
+    while (!scnt[ix] && left > 0) {
+        int diff, old_slot = slot;
+        ix++;
+        slot = (ix << ERTS_TW_SCNT_BITS);
+        diff = slot - old_slot;
+        if (left < diff) {
+            slot = old_slot + left;
+            diff = left;
+        }
+        if (slot < end_slot)
+            left -= diff;
+        else {
+            left -= end_slot - old_slot;
+            slot = first_slot;
+            ix = slot >> ERTS_TW_SCNT_BITS;
+        }
+    }
+
+    ERTS_TW_ASSERT(left >= -1);
+
+    if (posp)
+        *posp += slot_sz * ((ErtsMonotonicTime) (*leftp - left));
+    if (sixp)
+        *sixp = slot >> ERTS_TW_SCNT_BITS;
+    *leftp = left;
+    *slotp = slot;
+}
+
+
+static ERTS_INLINE void
+scnt_soon_wheel_next(int *slotp, int *leftp, ErtsMonotonicTime *posp,
+                    int *sixp, Sint *scnt)
+{
+    scnt_wheel_next(slotp, leftp, posp, sixp, scnt,
+                    ERTS_TW_SOON_WHEEL_FIRST_SLOT,
+                    ERTS_TW_SOON_WHEEL_END_SLOT, 1);
+}
+
+static ERTS_INLINE void
+scnt_later_wheel_next(int *slotp, int *leftp, ErtsMonotonicTime *posp,
+                    int *sixp, Sint *scnt)
+{
+    scnt_wheel_next(slotp, leftp, posp, sixp, scnt,
+                    ERTS_TW_LATER_WHEEL_FIRST_SLOT,
+                    ERTS_TW_LATER_WHEEL_END_SLOT,
+                    ERTS_TW_LATER_WHEEL_SLOT_SIZE);
+}
+
+
 static ERTS_INLINE int
 soon_slot(ErtsMonotonicTime soon_pos)
 {
@@ -282,14 +426,10 @@ static void hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos);
 #define ERTS_HARD_DBG_CHK_WHEELS(TIW, CHK_MIN_TPOS)
 #endif
 
-static ERTS_INLINE ErtsMonotonicTime
-find_next_timeout(ErtsSchedulerData *esdp,
-		  ErtsTimerWheel *tiw,
-		  int search_all,
-		  ErtsMonotonicTime curr_time,       /* When !search_all */
-		  ErtsMonotonicTime max_search_time) /* When !search_all */
+static ErtsMonotonicTime
+find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, int thorough)
 {
-    int start_ix, tiw_pos_ix, pos_inc, wheel_start_ix, wheel_end_ix;
+    int slot, slots;
     int true_min_timeout = 0;
     ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos;
 
@@ -298,32 +438,51 @@ find_next_timeout(ErtsSchedulerData *esdp,
     ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TWHEEL_SLOT_INACTIVE);
 
     if (tiw->nto == 0) { /* no timeouts in wheel */
-	if (!search_all)
+	if (!thorough)
 	    min_timeout_pos = tiw->pos;
 	else {
-	    curr_time = erts_get_monotonic_time(esdp);
+	    ErtsMonotonicTime curr_time = erts_get_monotonic_time(esdp);
 	    tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
 	}
-	min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY);
+	min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK);
 	goto done;
     }
 
-    if (search_all)
-        min_timeout_pos = tiw->pos + ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY);
+    min_timeout_pos = tiw->pos;
+    if (thorough)
+        min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK);
     else
-        min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time);
+        min_timeout_pos += ERTS_TW_SOON_WHEEL_SIZE/2;
 
     if (!tiw->soon.nto) {
+        ErtsMonotonicTime tmp;
         /* Select later wheel... */
         slot_timeout_pos = tiw->later.pos;
-        start_ix = tiw_pos_ix = later_slot(slot_timeout_pos);
-        pos_inc = ERTS_TW_LATER_WHEEL_SLOT_SIZE;
-        wheel_start_ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT;
-        wheel_end_ix = ERTS_TW_LATER_WHEEL_END_SLOT;
+        slot = later_slot(slot_timeout_pos);
         /* Pre-timeout for move from later to soon wheel... */
         slot_timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+        tmp = min_timeout_pos - slot_timeout_pos;
+        tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+        tmp++;
+        if (tmp > ERTS_TW_LATER_WHEEL_SIZE)
+            slots = ERTS_TW_LATER_WHEEL_SIZE;
+        else
+            slots = (int) tmp;
+
+        /*
+         * We never search for an exact timeout in the
+         * later wheel, but instead settle for the first
+         * scnt range used.
+         */
+        if (tiw->w[slot])
+            true_min_timeout = 1;
+        else
+            scnt_later_wheel_next(&slot, &slots, &slot_timeout_pos,
+                                NULL, tiw->scnt);
+        min_timeout_pos = slot_timeout_pos;
     }
     else {
+        ErtsMonotonicTime tmp;
         /* Select soon wheel... */
 
         /*
@@ -355,30 +514,31 @@ find_next_timeout(ErtsSchedulerData *esdp,
         }
 
         slot_timeout_pos = tiw->pos;
-        start_ix = tiw_pos_ix = soon_slot(tiw->pos);
-        pos_inc = 1;
-        wheel_start_ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
-        wheel_end_ix = ERTS_TW_SOON_WHEEL_END_SLOT;
-    }
+        slot = soon_slot(tiw->pos);
+        tmp = min_timeout_pos - slot_timeout_pos;
+        if (tmp > ERTS_TW_SOON_WHEEL_SIZE)
+            slots = ERTS_TW_SOON_WHEEL_SIZE;
+        else
+            slots = (int) tmp;
 
-    /*
-     * Scan selected wheel...
-     */
-    do {
-        if (tiw->w[tiw_pos_ix]) {
-            min_timeout_pos = slot_timeout_pos;
-            true_min_timeout = 1;
-            break;
+        while (slot_timeout_pos < min_timeout_pos) {
+            if (tiw->w[slot]) {
+                ERTS_TW_ASSERT(tiw->w[slot]->timeout_pos == slot_timeout_pos);
+                min_timeout_pos = slot_timeout_pos;
+                true_min_timeout = 1;
+                break;
+            }
+            scnt_soon_wheel_next(&slot, &slots, &slot_timeout_pos,
+                                 NULL, tiw->scnt);
         }
 
-        slot_timeout_pos += pos_inc;
-        if (slot_timeout_pos >= min_timeout_pos)
-            break;
-
-        tiw_pos_ix++;
-        if (tiw_pos_ix == wheel_end_ix)
-            tiw_pos_ix = wheel_start_ix;
-    } while (start_ix != tiw_pos_ix);
+        if (slots > 0 && !true_min_timeout) {
+            /* Find first non-empty range... */
+            scnt_soon_wheel_next(&slot, &slots, &slot_timeout_pos,
+                                 NULL, tiw->scnt);
+            min_timeout_pos = slot_timeout_pos;
+        }
+    }
 
 done:
 
@@ -433,6 +593,7 @@ insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p)
         ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
         tiw->soon.nto++;
     }
+    scnt_inc(tiw->scnt, slot);
 }
 
 static ERTS_INLINE void
@@ -461,6 +622,7 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
 	}
         if (slot < ERTS_TW_SOON_WHEEL_END_SLOT)
             tiw->soon.nto--;
+        scnt_dec(tiw->scnt, slot);
     }
     else {
 	/* Timer in "at once" queue... */
@@ -480,7 +642,6 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
 	ERTS_TW_ASSERT(tiw->at_once.nto > 0);
 	tiw->at_once.nto--;
     }
-
     p->slot = ERTS_TWHEEL_SLOT_INACTIVE;
 }
 
@@ -493,7 +654,7 @@ erts_check_next_timeout_time(ErtsSchedulerData *esdp)
     if (tiw->true_next_timeout_time)
 	return tiw->next_timeout_time;
     ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_X(ERTS_MSACC_STATE_TIMERS);
-    time = find_next_timeout(esdp, tiw, 1, 0, 0);
+    time = find_next_timeout(esdp, tiw, 1);
     ERTS_MSACC_POP_STATE_M_X();
     return time;
 }
@@ -535,9 +696,10 @@ debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos)
     ix = later_slot(tiw->later.pos);
     tmp = skip_to_pos;
     tmp &= ERTS_TW_LATER_WHEEL_POS_MASK;
-    if (tmp > tiw->later.pos) {
+    if (tmp >= tiw->later.pos) {
         tmp -= tiw->later.pos;
         tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+        tmp++;
         ERTS_TW_ASSERT(tmp >= 0);
         if (tmp < (ErtsMonotonicTime) ERTS_TW_LATER_WHEEL_SIZE)
             slots = (int) tmp;
@@ -580,12 +742,16 @@ timeout_timer(ErtsTWheelTimer *p)
 void
 erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 {
-    int tiw_pos_ix, yielded_slot_restarted, yield_count, slots;
+    int slot, yielded_slot_restarted, yield_count, slots, scnt_ix;
     ErtsMonotonicTime bump_to;
+    Sint *scnt, *bump_scnt;
     ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
 
     yield_count = ERTS_TWHEEL_BUMP_YIELD_LIMIT;
 
+    scnt = &tiw->scnt[0];
+    bump_scnt = &tiw->bump_scnt[0];
+
     /*
      * In order to be fair we always continue with work
      * where we left off when restarting after a yield.
@@ -596,7 +762,8 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 	bump_to = tiw->pos;
         if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT)
             goto restart_yielded_later_slot;
-	tiw_pos_ix = tiw->yield_slot;
+	slot = tiw->yield_slot;
+        scnt_ix = scnt_get_ix(slot);
 	slots = tiw->yield_slots_left;
         ASSERT(0 <= slots && slots <= ERTS_TW_SOON_WHEEL_SIZE);
         tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
@@ -616,7 +783,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 	    empty_wheel:
 		ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to);
 		tiw->true_next_timeout_time = 0;
-		tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY;
+		tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK;
 		tiw->pos = bump_to;
 		tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
                 ERTS_MSACC_POP_STATE_M_X();
@@ -660,6 +827,22 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 	    if (tiw->nto == 0)
 		goto empty_wheel;
 
+            /*
+             * Save slot counts in bump operation local
+             * array.
+             *
+             * The amount of timers to trigger (or move)
+             * will only decrease from now until we have
+             * completed this bump operation (even if we
+             * yield in the middle of it).
+             *
+             * The amount of timers in the wheels may
+             * however increase due to timers being set
+             * by timeout callbacks.
+             */
+            sys_memcpy((void *) bump_scnt, (void *) scnt,
+                       sizeof(Sint) * ERTS_TW_SCNT_SIZE);
+
 	    if (tiw->true_next_timeout_time) {
 		ErtsMonotonicTime skip_until_pos;
 		/*
@@ -694,15 +877,17 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
                     slots = ERTS_TW_SOON_WHEEL_SIZE;
             }
 
-            tiw_pos_ix = soon_slot(tiw->pos+1);
+            slot = soon_slot(tiw->pos+1);
 	    tiw->pos = bump_to;
 
+            scnt_ix = scnt_get_ix(slot);
+
             /* Timeout timers in soon wheel */
-	    while (slots) {
+	    while (slots > 0) {
 
                 yield_count -= ERTS_TW_COST_SLOT;
 
-		p = tiw->w[tiw_pos_ix];
+		p = tiw->w[slot];
 		if (p) {
                     /* timeout callback need tiw->pos to be up to date */
 		    if (p->next == p) {
@@ -715,21 +900,23 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 			tiw->sentinel.next->prev = &tiw->sentinel;
 			tiw->sentinel.prev->next = &tiw->sentinel;
 		    }
-		    tiw->w[tiw_pos_ix] = NULL;
+		    tiw->w[slot] = NULL;
 
 		    while (1) {
 
                         ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= p->slot
                                        && p->slot < ERTS_TW_SOON_WHEEL_END_SLOT);
                         tiw->soon.nto--;
+                        scnt_ix_dec(scnt, scnt_ix);
                         if (p->timeout_pos <= bump_to) {
                             timeout_timer(p);
                             tiw->nto--;
+                            scnt_ix_dec(bump_scnt, scnt_ix);
                             yield_count -= ERTS_TW_COST_TIMEOUT;
                         }
                         else {
                             /* uncommon case */
-                            insert_timer_into_slot(tiw, tiw_pos_ix, p);
+                            insert_timer_into_slot(tiw, slot, p);
                             yield_count -= ERTS_TW_COST_SLOT_MOVE;
                         }
 
@@ -744,7 +931,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 			if (yield_count <= 0) {
 			    tiw->true_next_timeout_time = 1;
 			    tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to);
-			    tiw->yield_slot = tiw_pos_ix;
+			    tiw->yield_slot = slot;
 			    tiw->yield_slots_left = slots;
                             ERTS_MSACC_POP_STATE_M_X();
 			    return; /* Yield! */
@@ -755,10 +942,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 		    }
 		}
 
-                tiw_pos_ix++;
-                if (tiw_pos_ix == ERTS_TW_SOON_WHEEL_END_SLOT)
-                    tiw_pos_ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
-                slots--;
+                scnt_soon_wheel_next(&slot, &slots, NULL, &scnt_ix, bump_scnt);
 	    }
 
             if (ERTS_TW_BUMP_LATER_WHEEL(tiw)) {
@@ -771,10 +955,9 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
     } while (yielded_slot_restarted);
 
     tiw->true_next_timeout_time = 0;
-    tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY;
+    tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK;
 
-    /* Search at most two seconds ahead... */
-    (void) find_next_timeout(NULL, tiw, 0, curr_time, ERTS_SEC_TO_MONOTONIC(2));
+    (void) find_next_timeout(NULL, tiw, 0);
     ERTS_MSACC_POP_STATE_M_X();
 }
 
@@ -784,12 +967,17 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
     ErtsMonotonicTime cpos = tiw->pos;
     ErtsMonotonicTime later_pos = tiw->later.pos;
     int ycount = *ycount_p;
-    int slots, fslot;
+    int slots, fslot, scnt_ix;
+    Sint *scnt, *bump_scnt;
+
+    scnt = &tiw->scnt[0];
+    bump_scnt = &tiw->bump_scnt[0];
 
     ERTS_HARD_DBG_CHK_WHEELS(tiw, 0);
 
     if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) {
         fslot = tiw->yield_slot;
+        scnt_ix = scnt_get_ix(fslot);
         slots = tiw->yield_slots_left;
         ASSERT(0 <= slots && slots <= ERTS_TW_LATER_WHEEL_SIZE);
         tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
@@ -800,17 +988,18 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
         tmp_slots += ERTS_TW_LATER_WHEEL_SLOT_SIZE;
         tmp_slots -= later_pos;
         tmp_slots /= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+        tmp_slots++;
         if (tmp_slots < ERTS_TW_LATER_WHEEL_SIZE)
             slots = (int) tmp_slots;
         else
             slots = ERTS_TW_LATER_WHEEL_SIZE;
+        fslot = later_slot(later_pos);
+        scnt_ix = scnt_get_ix(fslot);
     }
 
-    while (slots >= 0) {
+    while (slots > 0) {
         ErtsTWheelTimer *p;
 
-        fslot = later_slot(later_pos);
-
         ycount -= ERTS_TW_COST_SLOT;
 
         p = tiw->w[fslot];
@@ -835,12 +1024,15 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
                 ERTS_TW_ASSERT(tpos >= later_pos);
                 ERTS_TW_ASSERT(p->slot == fslot);
 
+                scnt_ix_dec(scnt, scnt_ix);
+
                 if (tpos >= later_pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE) {
                     /* keep in later slot; very uncommon... */
                     insert_timer_into_slot(tiw, fslot, p);
                     ycount -= ERTS_TW_COST_SLOT_MOVE;
                 }
                 else {
+                    scnt_ix_dec(bump_scnt, scnt_ix);
                     ERTS_TW_ASSERT(tpos < cpos + ERTS_TW_SOON_WHEEL_SIZE);
                     if (tpos > cpos) {
                         /* move into soon wheel */
@@ -878,8 +1070,8 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
                 p->next->prev = &tiw->sentinel;
             }
         }
-        slots--;
-        later_pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+
+        scnt_later_wheel_next(&fslot, &slots, &later_pos, &scnt_ix, bump_scnt);
     }
 
     ERTS_HARD_DBG_CHK_WHEELS(tiw, 0);
@@ -908,6 +1100,9 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp)
     for(; i < ERTS_TW_LATER_WHEEL_END_SLOT; i++)
 	tiw->w[i] = NULL;
 
+    for (i = 0; i < ERTS_TW_SCNT_SIZE; i++)
+        tiw->scnt[i] = 0;
+
     mtime = erts_get_monotonic_time(esdp);
     tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime);
     tiw->nto = 0;
@@ -920,7 +1115,7 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp)
     tiw->later.pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE;
     tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
     tiw->true_next_timeout_time = 0;
-    tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY;
+    tiw->next_timeout_time = mtime + ERTS_MONOTONIC_WEEK;
     tiw->sentinel.next = &tiw->sentinel;
     tiw->sentinel.prev = &tiw->sentinel;
     tiw->sentinel.timeout = NULL;
@@ -1063,24 +1258,37 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw,
 static void
 hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos)
 {
-    int ix, soon_tmo, later_tmo, at_once_tmo;
+    int ix, six, soon_tmo, later_tmo, at_once_tmo,
+        scnt_slot, scnt_slots, scnt_six;
     ErtsMonotonicTime min_tpos;
+    Sint scnt[ERTS_TW_SCNT_SIZE];
+
+    for (six = 0; six < ERTS_TW_SCNT_SIZE; six++)
+        scnt[six] = 0;
 
     min_tpos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time);
 
     soon_tmo = 0;
+    scnt_slot = ERTS_TW_SOON_WHEEL_END_SLOT-1;
+    scnt_slots = ERTS_TW_SOON_WHEEL_SIZE;
+    scnt_six = 0;
+    scnt_soon_wheel_next(&scnt_slot, &scnt_slots,
+                         NULL, &scnt_six, tiw->scnt);
     for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
          ix < ERTS_TW_SOON_WHEEL_END_SLOT;
          ix++) {
         ErtsTWheelTimer *p;
 
         p = tiw->w[ix];
+        six = scnt_get_ix(ix);
+        ERTS_TW_ASSERT(!p || six == scnt_six);
         if (p) {
             ErtsTWheelTimer *first = p;
             do {
                 ErtsMonotonicTime tpos = p->timeout_pos;
-                ERTS_TW_ASSERT(p->slot == ix);
                 soon_tmo++;
+                scnt_ix_inc(scnt, six);
+                ERTS_TW_ASSERT(p->slot == ix);
                 ERTS_TW_ASSERT(ix == soon_slot(tpos));
                 ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
                 ERTS_TW_ASSERT(!check_min_tpos || tpos >= min_tpos);
@@ -1088,21 +1296,33 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos)
                 p = p->next;
             } while (p != first);
         }
+        if (ix == scnt_slot)
+            scnt_soon_wheel_next(&scnt_slot, &scnt_slots,
+                                 NULL, &scnt_six, tiw->scnt);
     }
 
     later_tmo = 0;
+    scnt_slot = ERTS_TW_SOON_WHEEL_END_SLOT-1;
+    scnt_slots = ERTS_TW_SOON_WHEEL_SIZE;
+    scnt_six = 0;
+    scnt_later_wheel_next(&scnt_slot, &scnt_slots,
+                         NULL, &scnt_six, tiw->scnt);
     for (ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT;
          ix < ERTS_TW_LATER_WHEEL_END_SLOT;
          ix++) {
         ErtsTWheelTimer *p;
 
         p = tiw->w[ix];
+        six = scnt_get_ix(ix);
+        ERTS_TW_ASSERT(!p || six == scnt_six);
         if (p) {
             ErtsTWheelTimer *first = p;
+            six = scnt_get_ix(ix);
             do {
                 ErtsMonotonicTime tpos = p->timeout_pos;
-                ERTS_TW_ASSERT(p->slot == ix);
                 later_tmo++;
+                scnt_ix_inc(scnt, six);
+                ERTS_TW_ASSERT(p->slot == ix);
                 ERTS_TW_ASSERT(later_slot(tpos) == ix);
                 tpos &= ERTS_TW_LATER_WHEEL_POS_MASK;
                 tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
@@ -1111,19 +1331,24 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos)
                 p = p->next;
             } while (p != first);
         }
+        if (ix == scnt_slot)
+            scnt_later_wheel_next(&scnt_slot, &scnt_slots,
+                                NULL, &scnt_six, tiw->scnt);
     }
 
     if (tiw->yield_slot >= 0) {
         ErtsTWheelTimer *p = tiw->sentinel.next;
+        ix = tiw->yield_slot;
         while (p != &tiw->sentinel) {
             ErtsMonotonicTime tpos = p->timeout_pos;
-            if (tiw->yield_slot >= ERTS_TW_SOON_WHEEL_SIZE) {
+            scnt_inc(scnt, ix);
+            if (ix >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) {
                 later_tmo++;
-                ERTS_TW_ASSERT(p->slot == later_slot(tpos));
+                ERTS_TW_ASSERT(ix == later_slot(tpos));
             }
             else {
                 soon_tmo++;
-                ERTS_TW_ASSERT(p->slot == (tpos & ERTS_TW_SOON_WHEEL_MASK));
+                ERTS_TW_ASSERT(ix == (tpos & ERTS_TW_SOON_WHEEL_MASK));
                 ERTS_TW_ASSERT(tpos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
             }
             p = p->next;
@@ -1144,6 +1369,9 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos)
     ERTS_TW_ASSERT(tiw->at_once.nto == at_once_tmo);
     ERTS_TW_ASSERT(tiw->soon.nto == soon_tmo);
     ERTS_TW_ASSERT(tiw->nto == soon_tmo + later_tmo + at_once_tmo);
+
+    for (six = 0; six < ERTS_TW_SCNT_SIZE; six++)
+        ERTS_TW_ASSERT(scnt[six] == tiw->scnt[six]);
 }
 
 #endif
-- 
cgit v1.2.3


From 25e9f3abd395c26eccb7962acc47dffe90beec81 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Sun, 12 Feb 2017 04:03:55 +0100
Subject: Manage timers to trigger at once in a slot similar to other timers

---
 erts/emulator/beam/erl_time.h |   5 +-
 erts/emulator/beam/time.c     | 365 ++++++++++++++++++++++--------------------
 2 files changed, 193 insertions(+), 177 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 699e69e991..46d6da6448 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -448,8 +448,7 @@ erts_time_unit_conversion(Uint64 value,
 void erts_sched_init_time_sup(ErtsSchedulerData *esdp);
 
 
-#define ERTS_TWHEEL_SLOT_AT_ONCE -1
-#define ERTS_TWHEEL_SLOT_INACTIVE -2
+#define ERTS_TW_SLOT_INACTIVE (-2)
 
 /*
 ** Timer entry:
@@ -481,7 +480,7 @@ ERTS_GLB_INLINE ErtsMonotonicTime erts_tweel_read_timeout(ErtsTWheelTimer *twt);
 
 ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p)
 {
-    p->slot = ERTS_TWHEEL_SLOT_INACTIVE;
+    p->slot = ERTS_TW_SLOT_INACTIVE;
 }
 
 ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_tmo_ref)
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 1b6ca2b269..16e4328a05 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -125,13 +125,17 @@
  * slot position by just removing it from the circular double
  * linked list that it is in.
  *
- * -- At Once List --
+ * -- At Once Slot --
  *
  * If a timer is set that has a time earlier or equal to 'pos',
  * it is not inserted into the wheel. It is instead inserted,
- * into a list referred to by the 'at_once' field. When the
- * bump operation is performed thise timers will be triggered
- * at once.
+ * into a circular double linked list referred to by the "at
+ * once" slot. When the bump operation is performed these timers
+ * will be triggered at once. The circular list of the slot will
+ * be moved to the 'sentinel' field while bumping these timers
+ * as when bumping an ordinary wheel slot. A yielding bump
+ * operation and cancelation of timers is handled the same way
+ * as if the timer was in a wheel slot.
  *
  * -- Searching for Next Timeout --
  *
@@ -262,14 +266,15 @@ static int tiw_itime; /* Constant after init */
 #endif
 
 struct ErtsTimerWheel_ {
-    ErtsTWheelTimer *w[ERTS_TW_SOON_WHEEL_SIZE + ERTS_TW_LATER_WHEEL_SIZE];
+    ErtsTWheelTimer *slots[1                            /* At Once Slot */
+                           + ERTS_TW_SOON_WHEEL_SIZE    /* Soon Wheel Slots */
+                           + ERTS_TW_LATER_WHEEL_SIZE];   /* Later Wheel Slots */
+    ErtsTWheelTimer **w;
     Sint scnt[ERTS_TW_SCNT_SIZE];
     Sint bump_scnt[ERTS_TW_SCNT_SIZE];
     ErtsMonotonicTime pos;
     Uint nto;
     struct {
-	ErtsTWheelTimer *head;
-	ErtsTWheelTimer *tail;
 	Uint nto;
     } at_once;
     struct {
@@ -285,6 +290,8 @@ struct ErtsTimerWheel_ {
     ErtsMonotonicTime next_timeout_time;
 };
 
+#define ERTS_TW_SLOT_AT_ONCE (-1)
+
 #define ERTS_TW_BUMP_LATER_WHEEL(TIW) \
     ((tiw)->pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE >= (TIW)->later.pos)
 
@@ -435,7 +442,7 @@ find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, int thorough)
 
     ERTS_HARD_DBG_CHK_WHEELS(tiw, 0);
 
-    ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TWHEEL_SLOT_INACTIVE);
+    ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TW_SLOT_INACTIVE);
 
     if (tiw->nto == 0) { /* no timeouts in wheel */
 	if (!thorough)
@@ -551,29 +558,11 @@ done:
     return min_timeout;
 }
 
-static ERTS_INLINE void
-insert_timer_into_at_once_list(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
-{
-    tiw->at_once.nto++;
-    p->next = NULL;
-    p->prev = tiw->at_once.tail;
-    if (tiw->at_once.tail) {
-        ERTS_TW_ASSERT(tiw->at_once.head);
-        tiw->at_once.tail->next = p;
-    }
-    else {	
-        ERTS_TW_ASSERT(!tiw->at_once.head);
-        tiw->at_once.head = p;
-    }
-    tiw->at_once.tail = p;
-    p->slot = ERTS_TWHEEL_SLOT_AT_ONCE;
-}
-
 static ERTS_INLINE void
 insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p)
 {
-    ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= slot);
-    ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT);
+    ERTS_TW_ASSERT(ERTS_TW_SLOT_AT_ONCE <= slot
+                   && slot < ERTS_TW_LATER_WHEEL_END_SLOT);
     p->slot = slot;
     if (!tiw->w[slot]) {
 	tiw->w[slot] = p;
@@ -589,60 +578,53 @@ insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p)
 	prev->next = p;
 	next->prev = p;
     }
-    if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) {
-        ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
-        tiw->soon.nto++;
+    if (slot == ERTS_TW_SLOT_AT_ONCE)
+	tiw->at_once.nto++;
+    else {
+        if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) {
+            ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
+            tiw->soon.nto++;
+        }
+        scnt_inc(tiw->scnt, slot);
     }
-    scnt_inc(tiw->scnt, slot);
 }
 
 static ERTS_INLINE void
 remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
 {
     int slot = p->slot;
-    ERTS_TW_ASSERT(slot != ERTS_TWHEEL_SLOT_INACTIVE);
-
-    if (slot >= ERTS_TW_SOON_WHEEL_FIRST_SLOT) {
-	/*
-	 * Timer in wheel or in circular
-	 * list of timers currently beeing
-	 * triggered (referred by sentinel).
-	 */
-	ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT);
-
-	if (p->next == p) {
-	    ERTS_TW_ASSERT(tiw->w[slot] == p);
-	    tiw->w[slot] = NULL;
-	}
-	else {
-	    if (tiw->w[slot] == p)
-		tiw->w[slot] = p->next;
-	    p->prev->next = p->next;
-	    p->next->prev = p->prev;
-	}
-        if (slot < ERTS_TW_SOON_WHEEL_END_SLOT)
-            tiw->soon.nto--;
-        scnt_dec(tiw->scnt, slot);
+    ERTS_TW_ASSERT(slot != ERTS_TW_SLOT_INACTIVE);
+
+    /*
+     * Timer is in circular list either referred to
+     * by at once slot, slot in soon wheel, slot
+     * in later wheel, or by sentinel (timers currently
+     * being triggered).
+     */
+    ERTS_TW_ASSERT(ERTS_TW_SLOT_AT_ONCE <= slot
+                   && slot < ERTS_TW_LATER_WHEEL_END_SLOT);
+
+    if (p->next == p) {
+        /* Cannot be referred by sentinel, i.e. must be referred by slot... */
+        ERTS_TW_ASSERT(tiw->w[slot] == p);
+        tiw->w[slot] = NULL;
     }
     else {
-	/* Timer in "at once" queue... */
-	ERTS_TW_ASSERT(slot == ERTS_TWHEEL_SLOT_AT_ONCE);
-	if (p->prev)
-	    p->prev->next = p->next;
-	else {
-	    ERTS_TW_ASSERT(tiw->at_once.head == p);
-	    tiw->at_once.head = p->next;
-	}
-	if (p->next)
-	    p->next->prev = p->prev;
-	else {
-	    ERTS_TW_ASSERT(tiw->at_once.tail == p);
-	    tiw->at_once.tail = p->prev;
-	}
+        if (tiw->w[slot] == p)
+            tiw->w[slot] = p->next;
+        p->prev->next = p->next;
+        p->next->prev = p->prev;
+    }
+    if (slot == ERTS_TW_SLOT_AT_ONCE) {
 	ERTS_TW_ASSERT(tiw->at_once.nto > 0);
 	tiw->at_once.nto--;
     }
-    p->slot = ERTS_TWHEEL_SLOT_INACTIVE;
+    else {
+        if (slot < ERTS_TW_SOON_WHEEL_END_SLOT)
+            tiw->soon.nto--;
+        scnt_dec(tiw->scnt, slot);
+    }
+    p->slot = ERTS_TW_SLOT_INACTIVE;
 }
 
 ErtsMonotonicTime
@@ -732,7 +714,7 @@ timeout_timer(ErtsTWheelTimer *p)
 {
     ErlTimeoutProc timeout;
     void *arg;
-    p->slot = ERTS_TWHEEL_SLOT_INACTIVE;
+    p->slot = ERTS_TW_SLOT_INACTIVE;
     timeout = p->timeout;
     arg = p->arg;
     (*timeout)(arg);
@@ -742,7 +724,7 @@ timeout_timer(ErtsTWheelTimer *p)
 void
 erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 {
-    int slot, yielded_slot_restarted, yield_count, slots, scnt_ix;
+    int slot, restarted, yield_count, slots, scnt_ix;
     ErtsMonotonicTime bump_to;
     Sint *scnt, *bump_scnt;
     ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
@@ -757,23 +739,24 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
      * where we left off when restarting after a yield.
      */
 
-    if (tiw->yield_slot >= ERTS_TW_SOON_WHEEL_FIRST_SLOT) {
-	yielded_slot_restarted = 1;
+    slot = tiw->yield_slot;
+    restarted = slot != ERTS_TW_SLOT_INACTIVE;
+    if (restarted) {
 	bump_to = tiw->pos;
-        if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT)
+        if (slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT)
             goto restart_yielded_later_slot;
-	slot = tiw->yield_slot;
+        tiw->yield_slot = ERTS_TW_SLOT_INACTIVE;
+        if (slot == ERTS_TW_SLOT_AT_ONCE)
+            goto restart_yielded_at_once_slot;
         scnt_ix = scnt_get_ix(slot);
 	slots = tiw->yield_slots_left;
         ASSERT(0 <= slots && slots <= ERTS_TW_SOON_WHEEL_SIZE);
-        tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
         goto restart_yielded_soon_slot;
     }
 
     do {
 
-	yielded_slot_restarted = 0;
-
+        restarted = 0;
 	bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
 
 	while (1) {
@@ -785,38 +768,59 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 		tiw->true_next_timeout_time = 0;
 		tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK;
 		tiw->pos = bump_to;
-		tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
+		tiw->yield_slot = ERTS_TW_SLOT_INACTIVE;
                 ERTS_MSACC_POP_STATE_M_X();
 		return;
 	    }
 
-	    p = tiw->at_once.head;
-	    while (p) {
-		if (yield_count <= 0) {
-		    ERTS_TW_ASSERT(tiw->nto > 0);
-		    ERTS_TW_ASSERT(tiw->at_once.nto > 0);
-		    tiw->yield_slot = ERTS_TWHEEL_SLOT_AT_ONCE;
-		    tiw->true_next_timeout_time = 1;
-		    tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to);
-                    ERTS_MSACC_POP_STATE_M_X();
-		    return;
-		}
+	    p = tiw->w[ERTS_TW_SLOT_AT_ONCE];
 
-		ERTS_TW_ASSERT(tiw->nto > 0);
-		ERTS_TW_ASSERT(tiw->at_once.nto > 0);
-		tiw->nto--;
-		tiw->at_once.nto--;
-		tiw->at_once.head = p->next;
-		if (p->next)
-		    p->next->prev = NULL;
-		else
-		    tiw->at_once.tail = NULL;
+	    if (p) {
 
-		timeout_timer(p);
+                if (p->next == p) {
+                    ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel);
+                    ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel);
+                }
+                else {
+                    tiw->sentinel.next = p->next;
+                    tiw->sentinel.prev = p->prev;
+                    tiw->sentinel.next->prev = &tiw->sentinel;
+                    tiw->sentinel.prev->next = &tiw->sentinel;
+                }
+                tiw->w[ERTS_TW_SLOT_AT_ONCE] = NULL;
+
+                while (1) {
+                    ERTS_TW_ASSERT(tiw->nto > 0);
+                    ERTS_TW_ASSERT(tiw->at_once.nto > 0);
+                    tiw->nto--;
+                    tiw->at_once.nto--;
+
+                    timeout_timer(p);
+
+                    yield_count -= ERTS_TW_COST_TIMEOUT;
+
+                restart_yielded_at_once_slot:
 
-                yield_count -= ERTS_TW_COST_TIMEOUT;
+                    p = tiw->sentinel.next;
+                    if (p == &tiw->sentinel) {
+                        ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel);
+                        break;
+                    }
+
+                    if (yield_count <= 0) {
+                        ERTS_TW_ASSERT(tiw->nto > 0);
+                        ERTS_TW_ASSERT(tiw->at_once.nto > 0);
+                        tiw->yield_slot = ERTS_TW_SLOT_AT_ONCE;
+                        tiw->true_next_timeout_time = 1;
+                        tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to);
+                        ERTS_MSACC_POP_STATE_M_X();
+                        return; /* Yield! */
+                    }
+
+                    tiw->sentinel.next = p->next;
+                    p->next->prev = &tiw->sentinel;
+                }
 
-		p = tiw->at_once.head;
 	    }
 
 	    if (tiw->pos >= bump_to) {
@@ -952,7 +956,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
             }
 	}
 
-    } while (yielded_slot_restarted);
+    } while (restarted);
 
     tiw->true_next_timeout_time = 0;
     tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK;
@@ -980,7 +984,7 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
         scnt_ix = scnt_get_ix(fslot);
         slots = tiw->yield_slots_left;
         ASSERT(0 <= slots && slots <= ERTS_TW_LATER_WHEEL_SIZE);
-        tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
+        tiw->yield_slot = ERTS_TW_SLOT_INACTIVE;
         goto restart_yielded_slot;
     }
     else {
@@ -1093,11 +1097,29 @@ ErtsTimerWheel *
 erts_create_timer_wheel(ErtsSchedulerData *esdp)
 {
     ErtsMonotonicTime mtime;
-    int i = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
+    int i;
     ErtsTimerWheel *tiw;
+
+    /* Some compile time sanity checks... */
+
+    /* Slots... */
+    ERTS_CT_ASSERT(ERTS_TW_SLOT_AT_ONCE == -1);
+    ERTS_CT_ASSERT(ERTS_TW_SLOT_INACTIVE < ERTS_TW_SLOT_AT_ONCE);
+    ERTS_CT_ASSERT(ERTS_TW_SLOT_AT_ONCE + 1 == ERTS_TW_SOON_WHEEL_FIRST_SLOT);
+    ERTS_CT_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT < ERTS_TW_SOON_WHEEL_END_SLOT);
+    ERTS_CT_ASSERT(ERTS_TW_SOON_WHEEL_END_SLOT == ERTS_TW_LATER_WHEEL_FIRST_SLOT);    
+    ERTS_CT_ASSERT(ERTS_TW_LATER_WHEEL_FIRST_SLOT < ERTS_TW_LATER_WHEEL_END_SLOT);
+
+    /* Both wheel sizes should be a powers of 2 */
+    ERTS_CT_ASSERT(ERTS_TW_SOON_WHEEL_SIZE
+                   && !(ERTS_TW_SOON_WHEEL_SIZE & (ERTS_TW_SOON_WHEEL_SIZE-1)));
+    ERTS_CT_ASSERT(ERTS_TW_LATER_WHEEL_SIZE
+                   && !(ERTS_TW_LATER_WHEEL_SIZE & (ERTS_TW_LATER_WHEEL_SIZE-1)));
+
     tiw = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_WHEEL,
 					     sizeof(ErtsTimerWheel));
-    for(; i < ERTS_TW_LATER_WHEEL_END_SLOT; i++)
+    tiw->w = &tiw->slots[1];
+    for(i = ERTS_TW_SLOT_AT_ONCE; i < ERTS_TW_LATER_WHEEL_END_SLOT; i++)
 	tiw->w[i] = NULL;
 
     for (i = 0; i < ERTS_TW_SCNT_SIZE; i++)
@@ -1106,14 +1128,12 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp)
     mtime = erts_get_monotonic_time(esdp);
     tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime);
     tiw->nto = 0;
-    tiw->at_once.head = NULL;
-    tiw->at_once.tail = NULL;
     tiw->at_once.nto = 0;
     tiw->soon.nto = 0;
     tiw->later.pos = tiw->pos;
     tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK;
     tiw->later.pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE;
-    tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
+    tiw->yield_slot = ERTS_TW_SLOT_INACTIVE;
     tiw->true_next_timeout_time = 0;
     tiw->next_timeout_time = mtime + ERTS_MONOTONIC_WEEK;
     tiw->sentinel.next = &tiw->sentinel;
@@ -1154,50 +1174,48 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw,
 		      ErtsTWheelTimer *p, ErlTimeoutProc timeout,
 		      void *arg, ErtsMonotonicTime timeout_pos)
 {
+    int slot;
     ErtsMonotonicTime timeout_time;
     ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
 
     p->timeout = timeout;
     p->arg = arg;
 
-    ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE);
+    ERTS_TW_ASSERT(p->slot == ERTS_TW_SLOT_INACTIVE);
 
     tiw->nto++;
+
+    /* calculate slot */
     if (timeout_pos <= tiw->pos) {
-        p->timeout_pos = tiw->pos;
-        insert_timer_into_at_once_list(tiw, p);
-	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->pos);
+        /* at once */
+        p->timeout_pos = timeout_pos = tiw->pos;
+        slot = ERTS_TW_SLOT_AT_ONCE;
+    }
+    else if (timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE) {
+        /* soon wheel */
+        p->timeout_pos = timeout_pos;
+        slot = soon_slot(timeout_pos);
     }
     else {
-	int slot;
-
-	p->timeout_pos = timeout_pos;
-
-	/* calculate slot */
-        if (timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE) {
-            /* soon wheel */
-            slot = soon_slot(timeout_pos);
-        }
-        else {
-            /* later wheel */
-            slot = later_slot(timeout_pos);
-
-            /*
-             * Next timeout due to this timeout
-             * should be in good time before the
-             * actual timeout (one later wheel slot
-             * size). This, in order to move it
-             * from the later wheel to the soon
-             * wheel.
-             */
-            timeout_pos &= ERTS_TW_LATER_WHEEL_POS_MASK;
-            timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
-        }
+        /* later wheel */
+        p->timeout_pos = timeout_pos;
+        slot = later_slot(timeout_pos);
 
-	insert_timer_into_slot(tiw, slot, p);
-        timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
+        /*
+         * Next timeout due to this timeout
+         * should be in good time before the
+         * actual timeout (one later wheel slot
+         * size). This, in order to move it
+         * from the later wheel to the soon
+         * wheel.
+         */
+        timeout_pos &= ERTS_TW_LATER_WHEEL_POS_MASK;
+        timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
     }
 
+    insert_timer_into_slot(tiw, slot, p);
+    timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
+
     if (timeout_time < tiw->next_timeout_time) {
 	tiw->true_next_timeout_time = 1;
 	tiw->next_timeout_time = timeout_time;
@@ -1208,7 +1226,7 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw,
 void
 erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
 {
-    if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) {
+    if (p->slot != ERTS_TW_SLOT_INACTIVE) {
         ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
 	remove_timer(tiw, p);
         tiw->nto--;
@@ -1234,14 +1252,7 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw,
 	tmr = tmr->next;
     }
 
-    for (tmr = tiw->at_once.head; tmr; tmr = tmr->next) {
-	if (tmr->timeout == tclbk)
-	    (*func)(arg, tmr->timeout_pos, tmr->arg);
-    }
-
-    for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
-         ix < ERTS_TW_LATER_WHEEL_END_SLOT;
-         ix++) {
+    for (ix = ERTS_TW_SLOT_AT_ONCE; ix < ERTS_TW_LATER_WHEEL_END_SLOT; ix++) {
 	tmr = tiw->w[ix];
 	if (tmr) {
 	    do {
@@ -1262,12 +1273,27 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos)
         scnt_slot, scnt_slots, scnt_six;
     ErtsMonotonicTime min_tpos;
     Sint scnt[ERTS_TW_SCNT_SIZE];
+    ErtsTWheelTimer *p;
 
     for (six = 0; six < ERTS_TW_SCNT_SIZE; six++)
         scnt[six] = 0;
 
     min_tpos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time);
 
+    at_once_tmo = 0;
+    p = tiw->w[ERTS_TW_SLOT_AT_ONCE];
+    if (p) {
+        ErtsTWheelTimer *first = p;
+        do {
+            at_once_tmo++;
+            ERTS_TW_ASSERT(p->slot == ERTS_TW_SLOT_AT_ONCE);
+            ERTS_TW_ASSERT(p->timeout_pos <= tiw->pos);
+            ERTS_TW_ASSERT(!check_min_tpos || tiw->pos >= min_tpos);
+            ERTS_TW_ASSERT(p->next->prev == p);
+            p = p->next;
+        } while (p != first);
+    }
+
     soon_tmo = 0;
     scnt_slot = ERTS_TW_SOON_WHEEL_END_SLOT-1;
     scnt_slots = ERTS_TW_SOON_WHEEL_SIZE;
@@ -1277,8 +1303,6 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos)
     for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
          ix < ERTS_TW_SOON_WHEEL_END_SLOT;
          ix++) {
-        ErtsTWheelTimer *p;
-
         p = tiw->w[ix];
         six = scnt_get_ix(ix);
         ERTS_TW_ASSERT(!p || six == scnt_six);
@@ -1310,8 +1334,6 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos)
     for (ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT;
          ix < ERTS_TW_LATER_WHEEL_END_SLOT;
          ix++) {
-        ErtsTWheelTimer *p;
-
         p = tiw->w[ix];
         six = scnt_get_ix(ix);
         ERTS_TW_ASSERT(!p || six == scnt_six);
@@ -1336,35 +1358,30 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos)
                                 NULL, &scnt_six, tiw->scnt);
     }
 
-    if (tiw->yield_slot >= 0) {
-        ErtsTWheelTimer *p = tiw->sentinel.next;
+    if (tiw->yield_slot != ERTS_TW_SLOT_INACTIVE) {
+        p = tiw->sentinel.next;
         ix = tiw->yield_slot;
         while (p != &tiw->sentinel) {
             ErtsMonotonicTime tpos = p->timeout_pos;
-            scnt_inc(scnt, ix);
-            if (ix >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) {
-                later_tmo++;
-                ERTS_TW_ASSERT(ix == later_slot(tpos));
-            }
+            ERTS_TW_ASSERT(ix == p->slot);
+            if (ix == ERTS_TW_SLOT_AT_ONCE)
+                at_once_tmo++;
             else {
-                soon_tmo++;
-                ERTS_TW_ASSERT(ix == (tpos & ERTS_TW_SOON_WHEEL_MASK));
-                ERTS_TW_ASSERT(tpos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
+                scnt_inc(scnt, ix);
+                if (ix >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) {
+                    later_tmo++;
+                    ERTS_TW_ASSERT(ix == later_slot(tpos));
+                }
+                else {
+                    soon_tmo++;
+                    ERTS_TW_ASSERT(ix == (tpos & ERTS_TW_SOON_WHEEL_MASK));
+                    ERTS_TW_ASSERT(tpos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
+                }
+                p = p->next;
             }
-            p = p->next;
         }
     }
 
-    at_once_tmo = 0;
-    if (tiw->at_once.head) {
-        ErtsTWheelTimer *p = tiw->at_once.head;
-        while (p) {
-            ErtsMonotonicTime tpos = p->timeout_pos;
-            at_once_tmo++;
-            ERTS_TW_ASSERT(tpos <= tiw->pos);
-            p = p->next;
-        }
-    }
 
     ERTS_TW_ASSERT(tiw->at_once.nto == at_once_tmo);
     ERTS_TW_ASSERT(tiw->soon.nto == soon_tmo);
-- 
cgit v1.2.3


From cfd009221d53d2f19dfa2852962f48b85b34d485 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Sun, 12 Feb 2017 23:25:39 +0100
Subject: Minimum timeout position in each timer wheel

Minimum known timeout position is saved in bot far and near
wheel. This information is used to avoid scanning from current
position in the cases were we know the minimum timeout position.
---
 erts/emulator/beam/time.c | 499 +++++++++++++++++++++++++++++-----------------
 1 file changed, 315 insertions(+), 184 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 16e4328a05..f530ee5de0 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -141,20 +141,24 @@
  *
  * In order to limit the amount of work needed in order to find
  * next timeout, we keep track of total amount of timers in the
- * wheels, total amount of timers in soon wheel, and the total
- * amount of timers in each range of slots. Each slot range
- * currently contain 512 slots.
+ * wheels, total amount of timers in the later wheel, total amount
+ * of timers in soon wheel, and the total amount of timers in
+ * each range of slots. Each slot range currently contain 512
+ * slots.
  *
- * When next timeout is less than half the soon wheel width away
- * we determine the exact timeout. Due to the timer counts of
+ * When next timeout is less than the soon wheel width away we
+ * determine the exact timeout. Due to the timer counts of
  * slot ranges, we currently at most need to search 1024 slots
  * in the soon wheel. This besides inspecting slot range counts
  * and two slots in the later wheel which potentially might trigger
  * timeouts for moving timers from the later wheel to the soon wheel
- * earlier than timeouts in the soon wheel.
+ * earlier than timeouts in the soon wheel. We also keep track
+ * of latest known minimum timeout position in each wheel which
+ * makes it possible to avoid scanning from current position
+ * each time.
  *
- * When next timeout is further away than half the soon wheel
- * width we settle for the earliest possible timeout in the first
+ * When next timeout is further away than the soon wheel width
+ * we settle for the earliest possible timeout in the first
  * non-empty slot range. The further away the next timeout is, the
  * more likely it is that the next timeout change before we
  * actually get there. That is, a change due to another timer is
@@ -176,8 +180,11 @@
 #define ERTS_WANT_TIMER_WHEEL_API
 #include "erl_time.h"
 
-#define ERTS_MONOTONIC_WEEK ERTS_SEC_TO_MONOTONIC(7*60*60*24)
-#define ERTS_CLKTCKS_WEEK ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK)
+#define ERTS_MAX_CLKTCKS \
+    ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_TIME_MAX)
+
+#define ERTS_CLKTCKS_WEEK \
+    ERTS_MONOTONIC_TO_CLKTCKS(ERTS_SEC_TO_MONOTONIC(7*60*60*24))
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
 #define ASSERT_NO_LOCKED_LOCKS		erts_lc_check_exact(NULL, 0)
@@ -265,10 +272,21 @@ static int tiw_itime; /* Constant after init */
 #  define TIW_ITIME tiw_itime
 #endif
 
+const int etp_tw_soon_wheel_size = ERTS_TW_SOON_WHEEL_SIZE;
+const ErtsMonotonicTime etp_tw_soon_wheel_mask = ERTS_TW_SOON_WHEEL_MASK;
+const int etp_tw_soon_wheel_first_slot = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
+
+const int etp_tw_later_wheel_size = ERTS_TW_LATER_WHEEL_SIZE;
+const ErtsMonotonicTime etp_tw_later_wheel_slot_size = ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+const int etp_tw_later_wheel_shift = ERTS_TW_LATER_WHEEL_SHIFT;
+const ErtsMonotonicTime etp_tw_later_wheel_mask = ERTS_TW_LATER_WHEEL_MASK;
+const ErtsMonotonicTime etp_tw_later_wheel_pos_mask = ERTS_TW_LATER_WHEEL_POS_MASK;
+const int etp_tw_later_wheel_first_slot = ERTS_TW_LATER_WHEEL_FIRST_SLOT;
+
 struct ErtsTimerWheel_ {
     ErtsTWheelTimer *slots[1                            /* At Once Slot */
                            + ERTS_TW_SOON_WHEEL_SIZE    /* Soon Wheel Slots */
-                           + ERTS_TW_LATER_WHEEL_SIZE];   /* Later Wheel Slots */
+                           + ERTS_TW_LATER_WHEEL_SIZE]; /* Later Wheel Slots */
     ErtsTWheelTimer **w;
     Sint scnt[ERTS_TW_SCNT_SIZE];
     Sint bump_scnt[ERTS_TW_SCNT_SIZE];
@@ -278,15 +296,20 @@ struct ErtsTimerWheel_ {
 	Uint nto;
     } at_once;
     struct {
+        ErtsMonotonicTime min_tpos;
         Uint nto;
     } soon;
     struct {
+        ErtsMonotonicTime min_tpos;
+        int min_tpos_slot;
         ErtsMonotonicTime pos;
+        Uint nto;
     } later;
     int yield_slot;
     int yield_slots_left;
     ErtsTWheelTimer sentinel;
     int true_next_timeout_time;
+    ErtsMonotonicTime next_timeout_pos;
     ErtsMonotonicTime next_timeout_time;
 };
 
@@ -297,6 +320,18 @@ struct ErtsTimerWheel_ {
 
 static int bump_later_wheel(ErtsTimerWheel *tiw, int *yield_count_p);
 
+#ifdef ERTS_TW_DEBUG
+#define ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(TIW, TO_POS) \
+    dbg_verify_empty_soon_slots((TIW), (TO_POS))
+#define ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(TIW, TO_POS) \
+    dbg_verify_empty_later_slots((TIW), (TO_POS))
+void dbg_verify_empty_soon_slots(ErtsTimerWheel *, ErtsMonotonicTime);
+void dbg_verify_empty_later_slots(ErtsTimerWheel *, ErtsMonotonicTime);
+#else
+#define ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(TIW, TO_POS)
+#define ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(TIW, TO_POS)
+#endif
+
 static ERTS_INLINE int
 scnt_get_ix(int slot)
 {
@@ -434,47 +469,59 @@ static void hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos);
 #endif
 
 static ErtsMonotonicTime
-find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, int thorough)
+find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw)
 {
     int slot, slots;
     int true_min_timeout = 0;
-    ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos;
+    ErtsMonotonicTime min_timeout_pos;
+
+    ERTS_TW_ASSERT(tiw->pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE < tiw->later.pos
+                   && tiw->later.pos <= tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
 
     ERTS_HARD_DBG_CHK_WHEELS(tiw, 0);
 
     ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TW_SLOT_INACTIVE);
 
     if (tiw->nto == 0) { /* no timeouts in wheel */
-	if (!thorough)
-	    min_timeout_pos = tiw->pos;
-	else {
-	    ErtsMonotonicTime curr_time = erts_get_monotonic_time(esdp);
-	    tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
-	}
-	min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK);
+        ErtsMonotonicTime curr_time = erts_get_monotonic_time(esdp);
+        tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
+        tiw->later.pos = min_timeout_pos + ERTS_TW_SOON_WHEEL_SIZE;
+        tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK;
+	min_timeout_pos += ERTS_CLKTCKS_WEEK;
 	goto done;
     }
 
-    min_timeout_pos = tiw->pos;
-    if (thorough)
-        min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK);
-    else
-        min_timeout_pos += ERTS_TW_SOON_WHEEL_SIZE/2;
+    ERTS_TW_ASSERT(tiw->soon.nto || tiw->later.nto);
 
     if (!tiw->soon.nto) {
-        ErtsMonotonicTime tmp;
-        /* Select later wheel... */
-        slot_timeout_pos = tiw->later.pos;
-        slot = later_slot(slot_timeout_pos);
-        /* Pre-timeout for move from later to soon wheel... */
-        slot_timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
-        tmp = min_timeout_pos - slot_timeout_pos;
-        tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
-        tmp++;
-        if (tmp > ERTS_TW_LATER_WHEEL_SIZE)
+        ErtsMonotonicTime tpos, min_tpos;
+
+        /* Search later wheel... */
+
+        min_tpos = tiw->later.min_tpos & ERTS_TW_LATER_WHEEL_POS_MASK;
+
+        if (min_tpos <= tiw->later.pos) {
+            tpos = tiw->later.pos;
             slots = ERTS_TW_LATER_WHEEL_SIZE;
-        else
-            slots = (int) tmp;
+        }
+        else {
+            ErtsMonotonicTime tmp;
+            /* Don't inspect slots we know are empty... */
+            tmp = min_tpos - tiw->later.pos;
+            tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+            if (tmp >= ERTS_TW_LATER_WHEEL_SIZE) {
+                /* Timeout more than one revolution ahead... */
+
+                /* Pre-timeout for move from later to soon wheel... */
+                min_timeout_pos = min_tpos - ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+                goto done;
+            }
+            tpos = min_tpos;
+            ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, min_tpos);
+            slots = ERTS_TW_LATER_WHEEL_SIZE - ((int) tmp);
+        }
+
+        slot = later_slot(tpos);
 
         /*
          * We never search for an exact timeout in the
@@ -484,13 +531,21 @@ find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, int thorough)
         if (tiw->w[slot])
             true_min_timeout = 1;
         else
-            scnt_later_wheel_next(&slot, &slots, &slot_timeout_pos,
-                                NULL, tiw->scnt);
-        min_timeout_pos = slot_timeout_pos;
+            scnt_later_wheel_next(&slot, &slots, &tpos, NULL, tiw->scnt);
+
+        tiw->later.min_tpos = tpos;
+        tiw->later.min_tpos_slot = slot;
+        ERTS_TW_ASSERT(slot == later_slot(tpos));
+
+        /* Pre-timeout for move from later to soon wheel... */
+        tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+        min_timeout_pos = tpos;
     }
     else {
-        ErtsMonotonicTime tmp;
-        /* Select soon wheel... */
+        ErtsMonotonicTime tpos;
+        /* Search soon wheel... */
+
+        min_timeout_pos = tiw->pos + ERTS_TW_SOON_WHEEL_SIZE;
 
         /*
          * Besides inspecting the soon wheel we
@@ -498,64 +553,72 @@ find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, int thorough)
          * later wheel which potentially can trigger
          * timeouts before timeouts in soon wheel...
          */
-        slot_timeout_pos = tiw->later.pos;
-        slot_timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
-        if (slot_timeout_pos < min_timeout_pos) {
-            int fslot = later_slot(tiw->later.pos);
-            if (tiw->w[fslot]) {
-                min_timeout_pos = slot_timeout_pos;
-                true_min_timeout = 1;
-            }
+        if (tiw->later.min_tpos > (tiw->later.pos
+                                   + 2*ERTS_TW_LATER_WHEEL_SLOT_SIZE)) {
+            ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(
+                tiw, 2*ERTS_TW_LATER_WHEEL_SLOT_SIZE);
+        }
+        else {
+            int fslot;
+            tpos = tiw->later.pos;
+            tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+            fslot = later_slot(tiw->later.pos);
+            if (tiw->w[fslot])
+                min_timeout_pos = tpos;
             else {
-                slot_timeout_pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE;
-                if (slot_timeout_pos < min_timeout_pos) {
+                tpos += ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+                if (tpos < min_timeout_pos) {
                     fslot++;
                     if (fslot == ERTS_TW_LATER_WHEEL_END_SLOT)
                         fslot = ERTS_TW_LATER_WHEEL_FIRST_SLOT;
-                    if (tiw->w[fslot]) {
-                        min_timeout_pos = slot_timeout_pos;
-                        true_min_timeout = 1;
-                    }
+                    if (tiw->w[fslot])
+                        min_timeout_pos = tpos;
                 }
             }
         }
 
-        slot_timeout_pos = tiw->pos;
-        slot = soon_slot(tiw->pos);
-        tmp = min_timeout_pos - slot_timeout_pos;
-        if (tmp > ERTS_TW_SOON_WHEEL_SIZE)
+        if (tiw->soon.min_tpos <= tiw->pos) {
+            tpos = tiw->pos;
             slots = ERTS_TW_SOON_WHEEL_SIZE;
-        else
-            slots = (int) tmp;
+        }
+        else {
+            ErtsMonotonicTime tmp;
+            /* Don't inspect slots we know are empty... */
+            tmp = tiw->soon.min_tpos - tiw->pos;
+            ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_SIZE > tmp);
+            ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(tiw, tiw->soon.min_tpos);
+            slots = ERTS_TW_SOON_WHEEL_SIZE - ((int) tmp);
+            tpos = tiw->soon.min_tpos;
+        }
 
-        while (slot_timeout_pos < min_timeout_pos) {
+        slot = soon_slot(tpos);
+
+        /* find next non-empty slot */
+        while (tpos < min_timeout_pos) {
             if (tiw->w[slot]) {
-                ERTS_TW_ASSERT(tiw->w[slot]->timeout_pos == slot_timeout_pos);
-                min_timeout_pos = slot_timeout_pos;
-                true_min_timeout = 1;
+                ERTS_TW_ASSERT(tiw->w[slot]->timeout_pos == tpos);
+                min_timeout_pos = tpos;
                 break;
             }
-            scnt_soon_wheel_next(&slot, &slots, &slot_timeout_pos,
-                                 NULL, tiw->scnt);
+            scnt_soon_wheel_next(&slot, &slots, &tpos, NULL, tiw->scnt);
         }
 
-        if (slots > 0 && !true_min_timeout) {
-            /* Find first non-empty range... */
-            scnt_soon_wheel_next(&slot, &slots, &slot_timeout_pos,
-                                 NULL, tiw->scnt);
-            min_timeout_pos = slot_timeout_pos;
-        }
+        tiw->soon.min_tpos = min_timeout_pos;
+        true_min_timeout = 1;
     }
 
-done:
+done: {
+        ErtsMonotonicTime min_timeout;
 
-    min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos);
-    tiw->next_timeout_time = min_timeout;
-    tiw->true_next_timeout_time = true_min_timeout;
+        min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos);
+        tiw->next_timeout_pos = min_timeout_pos;
+        tiw->next_timeout_time = min_timeout;
+        tiw->true_next_timeout_time = true_min_timeout;
 
-    ERTS_HARD_DBG_CHK_WHEELS(tiw, 1);
+        ERTS_HARD_DBG_CHK_WHEELS(tiw, 1);
 
-    return min_timeout;
+        return min_timeout;
+    }
 }
 
 static ERTS_INLINE void
@@ -581,9 +644,20 @@ insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p)
     if (slot == ERTS_TW_SLOT_AT_ONCE)
 	tiw->at_once.nto++;
     else {
+        ErtsMonotonicTime tpos = p->timeout_pos;
         if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) {
             ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
             tiw->soon.nto++;
+            if (tiw->soon.min_tpos > tpos)
+                tiw->soon.min_tpos = tpos;
+        }
+        else {
+            ERTS_TW_ASSERT(p->timeout_pos >= tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
+            tiw->later.nto++;
+            if (tiw->later.min_tpos > tpos) {
+                tiw->later.min_tpos = tpos;
+                tiw->later.min_tpos_slot = slot;
+            }
         }
         scnt_inc(tiw->scnt, slot);
     }
@@ -593,6 +667,7 @@ static ERTS_INLINE void
 remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
 {
     int slot = p->slot;
+    int empty_slot;
     ERTS_TW_ASSERT(slot != ERTS_TW_SLOT_INACTIVE);
 
     /*
@@ -608,21 +683,45 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
         /* Cannot be referred by sentinel, i.e. must be referred by slot... */
         ERTS_TW_ASSERT(tiw->w[slot] == p);
         tiw->w[slot] = NULL;
+        empty_slot = 1;
     }
     else {
         if (tiw->w[slot] == p)
             tiw->w[slot] = p->next;
         p->prev->next = p->next;
         p->next->prev = p->prev;
+        empty_slot = 0;
     }
     if (slot == ERTS_TW_SLOT_AT_ONCE) {
 	ERTS_TW_ASSERT(tiw->at_once.nto > 0);
 	tiw->at_once.nto--;
     }
     else {
-        if (slot < ERTS_TW_SOON_WHEEL_END_SLOT)
-            tiw->soon.nto--;
         scnt_dec(tiw->scnt, slot);
+        if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) {
+            if (empty_slot
+                && tiw->true_next_timeout_time
+                && p->timeout_pos == tiw->next_timeout_pos) {
+                tiw->true_next_timeout_time = 0;
+            }
+            if (--tiw->soon.nto == 0)
+                tiw->soon.min_tpos = ERTS_MAX_CLKTCKS;
+        }
+        else {
+            if (empty_slot
+                && tiw->true_next_timeout_time
+                && tiw->later.min_tpos_slot == slot) {
+                ErtsMonotonicTime tpos = tiw->later.min_tpos;
+                tpos &= ERTS_TW_LATER_WHEEL_POS_MASK;
+                tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+                if (tpos == tiw->next_timeout_pos)
+                    tiw->true_next_timeout_time = 0;
+            }
+            if (--tiw->later.nto == 0) {
+                tiw->later.min_tpos = ERTS_MAX_CLKTCKS;
+                tiw->later.min_tpos_slot = ERTS_TW_LATER_WHEEL_END_SLOT;
+            }
+        }
     }
     p->slot = ERTS_TW_SLOT_INACTIVE;
 }
@@ -633,82 +732,18 @@ erts_check_next_timeout_time(ErtsSchedulerData *esdp)
     ErtsTimerWheel *tiw = esdp->timer_wheel;
     ErtsMonotonicTime time;
     ERTS_MSACC_DECLARE_CACHE_X();
+    ERTS_TW_ASSERT(tiw->next_timeout_time
+                   == ERTS_CLKTCKS_TO_MONOTONIC(tiw->next_timeout_pos));
     if (tiw->true_next_timeout_time)
-	return tiw->next_timeout_time;
+	return tiw->next_timeout_time; /* known timeout... */
+    if (tiw->next_timeout_pos > tiw->pos + ERTS_TW_SOON_WHEEL_SIZE)
+        return tiw->next_timeout_time; /* sufficiently later away... */
     ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_X(ERTS_MSACC_STATE_TIMERS);
-    time = find_next_timeout(esdp, tiw, 1);
+    time = find_next_timeout(esdp, tiw);
     ERTS_MSACC_POP_STATE_M_X();
     return time;
 }
 
-#ifndef ERTS_TW_DEBUG
-#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) ((void) 0)
-#else
-#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) debug_check_safe_to_skip_to((TIW), (TO))
-static void
-debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos)
-{
-    int slots, ix;
-    ErtsTWheelTimer *tmr;
-    ErtsMonotonicTime tmp;
-
-    ix = soon_slot(tiw->pos);
-    tmp = skip_to_pos - tiw->pos;
-    ERTS_TW_ASSERT(tmp >= 0);
-    if (tmp < (ErtsMonotonicTime) ERTS_TW_SOON_WHEEL_SIZE)
-	slots = (int) tmp;
-    else
-	slots = ERTS_TW_SOON_WHEEL_SIZE;
-
-     while (slots > 0) {
-	 tmr = tiw->w[ix];
-	 if (tmr) {
-	     ErtsTWheelTimer *end = tmr;
-	     do {
-		 ERTS_TW_ASSERT(tmr->timeout_pos > skip_to_pos);
-		 tmr = tmr->next;
-	     } while (tmr != end);
-	 }
-	 ix++;
-	 if (ix == ERTS_TW_SOON_WHEEL_END_SLOT)
-	     ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
-	 slots--;
-    }
-
-    ix = later_slot(tiw->later.pos);
-    tmp = skip_to_pos;
-    tmp &= ERTS_TW_LATER_WHEEL_POS_MASK;
-    if (tmp >= tiw->later.pos) {
-        tmp -= tiw->later.pos;
-        tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
-        tmp++;
-        ERTS_TW_ASSERT(tmp >= 0);
-        if (tmp < (ErtsMonotonicTime) ERTS_TW_LATER_WHEEL_SIZE)
-            slots = (int) tmp;
-        else
-            slots = ERTS_TW_LATER_WHEEL_SIZE;
-
-        while (slots > 0) {
-            tmr = tiw->w[ix];
-            if (tmr) {
-                ErtsMonotonicTime tpos = tmr->timeout_pos;
-                ErtsTWheelTimer *end = tmr;
-                do {
-                    tpos &= ERTS_TW_LATER_WHEEL_POS_MASK;
-                    tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
-                    ERTS_TW_ASSERT(tpos > skip_to_pos);
-                    tmr = tmr->next;
-                } while (tmr != end);
-            }
-            ix++;
-            if (ix == ERTS_TW_LATER_WHEEL_END_SLOT)
-                ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT;
-            slots--;
-        }
-    }
-}
-#endif
-
 static ERTS_INLINE void
 timeout_timer(ErtsTWheelTimer *p)
 {
@@ -758,16 +793,23 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 
         restarted = 0;
 	bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
+        tiw->true_next_timeout_time = 1;
+        tiw->next_timeout_pos = bump_to;
+        tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to);
 
 	while (1) {
 	    ErtsTWheelTimer *p;
 
 	    if (tiw->nto == 0) {
 	    empty_wheel:
-		ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to);
+                ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(tiw, bump_to);
+                ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, bump_to);
 		tiw->true_next_timeout_time = 0;
-		tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK;
+                tiw->next_timeout_pos = bump_to + ERTS_CLKTCKS_WEEK;
+		tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->next_timeout_pos);;
 		tiw->pos = bump_to;
+                tiw->later.pos = bump_to + ERTS_TW_SOON_WHEEL_SIZE;
+                tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK;
 		tiw->yield_slot = ERTS_TW_SLOT_INACTIVE;
                 ERTS_MSACC_POP_STATE_M_X();
 		return;
@@ -811,8 +853,6 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
                         ERTS_TW_ASSERT(tiw->nto > 0);
                         ERTS_TW_ASSERT(tiw->at_once.nto > 0);
                         tiw->yield_slot = ERTS_TW_SLOT_AT_ONCE;
-                        tiw->true_next_timeout_time = 1;
-                        tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to);
                         ERTS_MSACC_POP_STATE_M_X();
                         return; /* Yield! */
                     }
@@ -847,28 +887,22 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
             sys_memcpy((void *) bump_scnt, (void *) scnt,
                        sizeof(Sint) * ERTS_TW_SCNT_SIZE);
 
-	    if (tiw->true_next_timeout_time) {
-		ErtsMonotonicTime skip_until_pos;
+	    if (tiw->soon.min_tpos > tiw->pos) {
+		ErtsMonotonicTime skip_until_pos = tiw->soon.min_tpos;
+
 		/*
 		 * No need inspecting slots where we know no timeouts
 		 * to trigger should reside.
 		 */
 
-		skip_until_pos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time);
 		if (skip_until_pos > bump_to)
 		    skip_until_pos = bump_to;
 
 		skip_until_pos--;
 
 		if (skip_until_pos > tiw->pos) {
-		    ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, skip_until_pos);
-
+                    ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(tiw, skip_until_pos);
 		    tiw->pos = skip_until_pos;
-
-                    skip_until_pos++;
-                    skip_until_pos &= ERTS_TW_LATER_WHEEL_POS_MASK;
-                    if (tiw->later.pos < skip_until_pos)
-                        tiw->later.pos = skip_until_pos;
 		}
 	    }
 
@@ -884,6 +918,9 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
             slot = soon_slot(tiw->pos+1);
 	    tiw->pos = bump_to;
 
+            tiw->next_timeout_pos = bump_to;
+            tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to);
+
             scnt_ix = scnt_get_ix(slot);
 
             /* Timeout timers in soon wheel */
@@ -910,7 +947,8 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 
                         ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= p->slot
                                        && p->slot < ERTS_TW_SOON_WHEEL_END_SLOT);
-                        tiw->soon.nto--;
+                        if (--tiw->soon.nto == 0)
+                            tiw->soon.min_tpos = ERTS_MAX_CLKTCKS;
                         scnt_ix_dec(scnt, scnt_ix);
                         if (p->timeout_pos <= bump_to) {
                             timeout_timer(p);
@@ -933,8 +971,6 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 			}
 
 			if (yield_count <= 0) {
-			    tiw->true_next_timeout_time = 1;
-			    tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to);
 			    tiw->yield_slot = slot;
 			    tiw->yield_slots_left = slots;
                             ERTS_MSACC_POP_STATE_M_X();
@@ -959,9 +995,9 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
     } while (restarted);
 
     tiw->true_next_timeout_time = 0;
-    tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK;
+    ERTS_TW_ASSERT(tiw->next_timeout_pos == bump_to);
 
-    (void) find_next_timeout(NULL, tiw, 0);
+    (void) find_next_timeout(NULL, tiw);
     ERTS_MSACC_POP_STATE_M_X();
 }
 
@@ -988,15 +1024,31 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
         goto restart_yielded_slot;
     }
     else {
-        ErtsMonotonicTime tmp_slots = cpos;
-        tmp_slots += ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+        ErtsMonotonicTime end_later_pos, tmp_slots, min_tpos;
+
+        min_tpos = tiw->later.min_tpos & ERTS_TW_LATER_WHEEL_POS_MASK;
+        end_later_pos = cpos + ERTS_TW_SOON_WHEEL_SIZE;
+        end_later_pos &= ERTS_TW_LATER_WHEEL_POS_MASK;
+
+        /* Skip known empty slots... */
+        if (min_tpos > later_pos) {
+            if (min_tpos > end_later_pos) {
+                later_pos = end_later_pos;
+                ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, later_pos);
+                goto done;
+            }
+            later_pos = min_tpos;
+            ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, later_pos);
+        }
+
+        tmp_slots = end_later_pos;
         tmp_slots -= later_pos;
         tmp_slots /= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
-        tmp_slots++;
         if (tmp_slots < ERTS_TW_LATER_WHEEL_SIZE)
             slots = (int) tmp_slots;
         else
             slots = ERTS_TW_LATER_WHEEL_SIZE;
+
         fslot = later_slot(later_pos);
         scnt_ix = scnt_get_ix(fslot);
     }
@@ -1028,6 +1080,10 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
                 ERTS_TW_ASSERT(tpos >= later_pos);
                 ERTS_TW_ASSERT(p->slot == fslot);
 
+                if (--tiw->later.nto == 0) {
+                    tiw->later.min_tpos = ERTS_MAX_CLKTCKS;
+                    tiw->later.min_tpos_slot = ERTS_TW_LATER_WHEEL_END_SLOT;
+                }
                 scnt_ix_dec(scnt, scnt_ix);
 
                 if (tpos >= later_pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE) {
@@ -1061,8 +1117,6 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
 
                 if (ycount < 0) {
                     tiw->later.pos = later_pos;
-		    tiw->true_next_timeout_time = 1;
-		    tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(cpos);
                     tiw->yield_slot = fslot;
                     tiw->yield_slots_left = slots;
                     *ycount_p = 0;
@@ -1078,6 +1132,8 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
         scnt_later_wheel_next(&fslot, &slots, &later_pos, &scnt_ix, bump_scnt);
     }
 
+done:
+
     ERTS_HARD_DBG_CHK_WHEELS(tiw, 0);
 
     tiw->later.pos = later_pos;
@@ -1129,13 +1185,17 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp)
     tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime);
     tiw->nto = 0;
     tiw->at_once.nto = 0;
+    tiw->soon.min_tpos = ERTS_MAX_CLKTCKS;
     tiw->soon.nto = 0;
-    tiw->later.pos = tiw->pos;
+    tiw->later.min_tpos = ERTS_MAX_CLKTCKS;
+    tiw->later.min_tpos_slot = ERTS_TW_LATER_WHEEL_END_SLOT;
+    tiw->later.pos = tiw->pos + ERTS_TW_SOON_WHEEL_SIZE;
     tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK;
-    tiw->later.pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+    tiw->later.nto = 0;
     tiw->yield_slot = ERTS_TW_SLOT_INACTIVE;
     tiw->true_next_timeout_time = 0;
-    tiw->next_timeout_time = mtime + ERTS_MONOTONIC_WEEK;
+    tiw->next_timeout_pos = tiw->pos + ERTS_CLKTCKS_WEEK;
+    tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->next_timeout_pos);
     tiw->sentinel.next = &tiw->sentinel;
     tiw->sentinel.prev = &tiw->sentinel;
     tiw->sentinel.timeout = NULL;
@@ -1175,7 +1235,6 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw,
 		      void *arg, ErtsMonotonicTime timeout_pos)
 {
     int slot;
-    ErtsMonotonicTime timeout_time;
     ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
 
     p->timeout = timeout;
@@ -1195,6 +1254,8 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw,
         /* soon wheel */
         p->timeout_pos = timeout_pos;
         slot = soon_slot(timeout_pos);
+        if (tiw->soon.min_tpos > timeout_pos)
+            tiw->soon.min_tpos = timeout_pos;
     }
     else {
         /* later wheel */
@@ -1214,11 +1275,13 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw,
     }
 
     insert_timer_into_slot(tiw, slot, p);
-    timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
 
-    if (timeout_time < tiw->next_timeout_time) {
+    if (timeout_pos <= tiw->next_timeout_pos) {
 	tiw->true_next_timeout_time = 1;
-	tiw->next_timeout_time = timeout_time;
+        if (timeout_pos < tiw->next_timeout_pos) {
+            tiw->next_timeout_pos = timeout_pos;
+            tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
+        }
     }
     ERTS_MSACC_POP_STATE_M_X();
 }
@@ -1264,6 +1327,73 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw,
     }
 }
 
+#ifdef ERTS_TW_DEBUG
+
+void
+dbg_verify_empty_soon_slots(ErtsTimerWheel *tiw, ErtsMonotonicTime to_pos)
+{
+    int ix;
+    ErtsMonotonicTime tmp;
+
+    ix = soon_slot(tiw->pos);
+    tmp = to_pos;
+    if (tmp > tiw->pos) {
+        int slots;
+        tmp -= tiw->pos;
+        ERTS_TW_ASSERT(tmp > 0);
+        if (tmp < (ErtsMonotonicTime) ERTS_TW_SOON_WHEEL_SIZE)
+            slots = (int) tmp;
+        else
+            slots = ERTS_TW_SOON_WHEEL_SIZE;
+
+        while (slots > 0) {
+            ERTS_TW_ASSERT(!tiw->w[ix]);
+            ix++;
+            if (ix == ERTS_TW_SOON_WHEEL_END_SLOT)
+                ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
+            slots--;
+        }
+    }    
+}
+
+void
+dbg_verify_empty_later_slots(ErtsTimerWheel *tiw, ErtsMonotonicTime to_pos)
+{
+    int ix;
+    ErtsMonotonicTime tmp;
+
+    ix = later_slot(tiw->later.pos);
+    tmp = to_pos;
+    tmp &= ERTS_TW_LATER_WHEEL_POS_MASK;
+    if (tmp > tiw->later.pos) {
+        int slots;
+        tmp -= tiw->later.pos;
+        tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
+        ERTS_TW_ASSERT(tmp > 0);
+        if (tmp < (ErtsMonotonicTime) ERTS_TW_LATER_WHEEL_SIZE)
+            slots = (int) tmp;
+        else
+            slots = ERTS_TW_LATER_WHEEL_SIZE;
+
+        while (slots > 0) {
+            ErtsTWheelTimer *tmr = tiw->w[ix];
+            if (tmr) {
+                ErtsTWheelTimer *end = tmr;
+                do {
+                    ERTS_TW_ASSERT(tmr->timeout_pos > to_pos);
+                    tmr = tmr->next;
+                } while (tmr != end);
+            }
+            ix++;
+            if (ix == ERTS_TW_LATER_WHEEL_END_SLOT)
+                ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT;
+            slots--;
+        }
+    }    
+}
+
+#endif /* ERTS_TW_DEBUG */
+
 #ifdef ERTS_TW_HARD_DEBUG
 
 static void
@@ -1385,10 +1515,11 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos)
 
     ERTS_TW_ASSERT(tiw->at_once.nto == at_once_tmo);
     ERTS_TW_ASSERT(tiw->soon.nto == soon_tmo);
+    ERTS_TW_ASSERT(tiw->later.nto == later_tmo);
     ERTS_TW_ASSERT(tiw->nto == soon_tmo + later_tmo + at_once_tmo);
 
     for (six = 0; six < ERTS_TW_SCNT_SIZE; six++)
         ERTS_TW_ASSERT(scnt[six] == tiw->scnt[six]);
 }
 
-#endif
+#endif /* ERTS_TW_HARD_DEBUG */
-- 
cgit v1.2.3


From 02da78a8cce79a07f00ff0816967e84ead8a3bfa Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 18 Apr 2017 16:37:44 +0200
Subject: Fix of later timer wheel

---
 erts/emulator/beam/time.c | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index f530ee5de0..cee3cb619f 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -1033,8 +1033,8 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
         /* Skip known empty slots... */
         if (min_tpos > later_pos) {
             if (min_tpos > end_later_pos) {
-                later_pos = end_later_pos;
-                ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, later_pos);
+                ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, end_later_pos);
+                tiw->later.pos = end_later_pos;
                 goto done;
             }
             later_pos = min_tpos;
@@ -1051,6 +1051,8 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
 
         fslot = later_slot(later_pos);
         scnt_ix = scnt_get_ix(fslot);
+
+        tiw->later.pos = end_later_pos;
     }
 
     while (slots > 0) {
@@ -1077,7 +1079,6 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
             while (1) {
                 ErtsMonotonicTime tpos = p->timeout_pos;
 
-                ERTS_TW_ASSERT(tpos >= later_pos);
                 ERTS_TW_ASSERT(p->slot == fslot);
 
                 if (--tiw->later.nto == 0) {
@@ -1086,7 +1087,7 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
                 }
                 scnt_ix_dec(scnt, scnt_ix);
 
-                if (tpos >= later_pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE) {
+                if (tpos >= tiw->later.pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE) {
                     /* keep in later slot; very uncommon... */
                     insert_timer_into_slot(tiw, fslot, p);
                     ycount -= ERTS_TW_COST_SLOT_MOVE;
@@ -1116,7 +1117,6 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
                 }
 
                 if (ycount < 0) {
-                    tiw->later.pos = later_pos;
                     tiw->yield_slot = fslot;
                     tiw->yield_slots_left = slots;
                     *ycount_p = 0;
@@ -1129,15 +1129,13 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
             }
         }
 
-        scnt_later_wheel_next(&fslot, &slots, &later_pos, &scnt_ix, bump_scnt);
+        scnt_later_wheel_next(&fslot, &slots, NULL, &scnt_ix, bump_scnt);
     }
 
 done:
 
     ERTS_HARD_DBG_CHK_WHEELS(tiw, 0);
 
-    tiw->later.pos = later_pos;
-
     *ycount_p = ycount;
 
     return 0;
-- 
cgit v1.2.3


From 0835f40ae25f97360dc393928796387d3cd6b16e Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Wed, 19 Apr 2017 00:43:37 +0100
Subject: erts: Add enif_phash2 and enif_phash2_ranged

These allow one to hash VM terms from NIF code.
---
 erts/emulator/beam/bif.c                      | 17 ++----
 erts/emulator/beam/erl_nif.c                  | 20 +++++++
 erts/emulator/beam/erl_nif_api_funcs.h        |  4 ++
 erts/emulator/beam/erl_utils.h                |  1 +
 erts/emulator/beam/utils.c                    | 10 ++++
 erts/emulator/test/nif_SUITE.erl              | 85 ++++++++++++++++++++++++++-
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 19 ++++++
 7 files changed, 143 insertions(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 214de3652f..d59adc18d6 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4888,7 +4888,6 @@ BIF_RETTYPE phash2_1(BIF_ALIST_1)
 BIF_RETTYPE phash2_2(BIF_ALIST_2)
 {
     Uint32 hash;
-    Uint32 final_hash;
     Uint32 range;
 
     /* Check for special case 2^32 */
@@ -4901,23 +4900,19 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2)
 	}
 	range = (Uint32) u;
     }
-    hash = make_hash2(BIF_ARG_1);
-    if (range) {
-	final_hash = hash % range; /* [0..range-1] */
-    } else {
-	final_hash = hash;
-    }
+    hash = make_hash2_within_range(BIF_ARG_1, range);
+
     /*
      * Return either a small or a big. Use the heap for bigs if there is room.
      */
 #if defined(ARCH_64)
-    BIF_RET(make_small(final_hash));
+    BIF_RET(make_small(hash));
 #else
-    if (IS_USMALL(0, final_hash)) {
-	BIF_RET(make_small(final_hash));
+    if (IS_USMALL(0, hash)) {
+	BIF_RET(make_small(hash));
     } else {
 	Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE);
-	BIF_RET(uint_to_big(final_hash, hp));
+	BIF_RET(uint_to_big(hash, hp));
     }
 #endif
 }
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index f86b9739fa..14abf461a3 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -55,6 +55,7 @@
 #include "dtrace-wrapper.h"
 #include "erl_process.h"
 #include "erl_bif_unique.h"
+#include "erl_utils.h"
 #undef ERTS_WANT_NFUNC_SCHED_INTERNALS__
 #define ERTS_WANT_NFUNC_SCHED_INTERNALS__
 #include "erl_nfunc_sched.h"
@@ -1213,6 +1214,25 @@ int enif_compare(Eterm lhs, Eterm rhs)
     return result;
 }
 
+#if SIZEOF_LONG < 4
+/* This *really* shouldn't happen */
+#  error Incompatible long word size
+#endif
+
+unsigned long enif_phash2(Eterm term)
+{
+    return make_hash2(term) & ((1 << 27) - 1);
+}
+
+unsigned long enif_phash2_ranged(Eterm term, unsigned long range)
+{
+#if SIZEOF_LONG > 4
+    if (range > (unsigned long) UINT32_MAX)
+        range = 0;
+#endif
+    return make_hash2_within_range(term, range);
+}
+
 int enif_get_tuple(ErlNifEnv* env, Eterm tpl, int* arity, const Eterm** array)
 {
     Eterm* ptr;
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 01d9e386ed..8a3f9165d9 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -47,6 +47,8 @@ ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term,
 ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array));
 ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
 ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
+ERL_NIF_API_FUNC_DECL(unsigned long,enif_phash2,(ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(unsigned long,enif_phash2_ranged,(ERL_NIF_TERM term, unsigned long));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i));
@@ -208,6 +210,8 @@ ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const Erl
 #  define enif_get_list_cell ERL_NIF_API_FUNC_MACRO(enif_get_list_cell)
 #  define enif_is_identical ERL_NIF_API_FUNC_MACRO(enif_is_identical)
 #  define enif_compare ERL_NIF_API_FUNC_MACRO(enif_compare)
+#  define enif_phash2 ERL_NIF_API_FUNC_MACRO(enif_phash2)
+#  define enif_phash2_ranged ERL_NIF_API_FUNC_MACRO(enif_phash2_ranged)
 
 #  define enif_make_binary ERL_NIF_API_FUNC_MACRO(enif_make_binary)
 #  define enif_make_badarg ERL_NIF_API_FUNC_MACRO(enif_make_badarg)
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 47289a0af1..75578c26d6 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -119,6 +119,7 @@ Sint erts_list_length(Eterm);
 int erts_is_builtin(Eterm, Eterm, int);
 Uint32 block_hash(byte *, unsigned, Uint32);
 Uint32 make_hash2(Eterm);
+Uint32 make_hash2_within_range(Eterm, Uint32);
 Uint32 make_hash(Eterm);
 Uint32 make_internal_hash(Eterm);
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 8f3f48f38f..72c59b33c6 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1552,6 +1552,16 @@ make_hash2(Eterm term)
     }
 }
 
+Uint32
+make_hash2_within_range(Eterm term, Uint32 range)
+{
+    Uint32 hash = make_hash2(term);
+    if (range)
+        return hash % range; /* [0..range-1] */
+    else
+        return hash; /* Special case: 2**32 */
+}
+
 /* Term hash function for internal use.
  *
  * Limitation #1: Is not "portable" in any way between different VM instances.
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 693db42e58..780fe0a5ee 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -56,7 +56,9 @@
          nif_is_process_alive/1, nif_is_port_alive/1,
          nif_term_to_binary/1, nif_binary_to_term/1,
          nif_port_command/1,
-         nif_snprintf/1
+         nif_snprintf/1,
+         nif_phash2/1,
+         nif_phash2_ranged/1
 	]).
 
 -export([many_args_100/100]).
@@ -90,7 +92,9 @@ all() ->
      nif_is_process_alive, nif_is_port_alive,
      nif_term_to_binary, nif_binary_to_term,
      nif_port_command,
-     nif_snprintf].
+     nif_snprintf,
+     nif_phash2,
+     nif_phash2_ranged].
 
 groups() ->
     [{G, [], api_repeaters()} || G <- api_groups()]
@@ -2610,6 +2614,81 @@ nif_snprintf(Config) ->
     <<"{{hello,world,-33},",0>> = format_term_nif(20,{{hello,world, -33}, 3.14, self()}),
     ok.
 
+nif_phash2(Config) ->
+    ensure_lib_loaded(Config),
+    Terms =
+        [random_term() || _ <- lists:seq(1, 1000)],
+
+    lists:foreach(
+      fun (Term) ->
+              HashValue = erlang:phash2(Term),
+              NifHashValue = phash2_nif(Term),
+              (HashValue =:= NifHashValue
+               orelse ct:fail("Expected: ~p\nActual:   ~p",
+                              [HashValue, NifHashValue]))
+      end,
+      Terms).
+
+nif_phash2_ranged(Config) ->
+    ensure_lib_loaded(Config),
+    RandomRangedTerms =
+        [{random_term(), rand:uniform((1 bsl 32) - 1)}
+         || _ <- lists:seq(1, 1000)],
+
+    lists:foreach(
+      fun ({Term, Range}) ->
+              HashValue = erlang:phash2(Term, Range),
+              NifHashValue = phash2_ranged_nif(Term, Range),
+              (HashValue =:= NifHashValue
+               orelse ct:fail("Expected: ~p\nActual:   ~p",
+                              [HashValue, NifHashValue]))
+      end,
+      RandomRangedTerms),
+
+    EdgeCaseTerm = random_term(),
+    EdgeCaseHashValue = erlang:phash2(EdgeCaseTerm, 1 bsl 32),
+    EdgeCaseNifHashValue = phash2_ranged_nif(EdgeCaseTerm, 0),
+    (EdgeCaseHashValue =:= EdgeCaseNifHashValue
+     orelse ct:fail("Expected: ~p\nActual:   ~p",
+                    [EdgeCaseHashValue, EdgeCaseNifHashValue])).
+
+-define(HALF_DBL_EPSILON, 1.1102230246251565e-16). % math:pow(2, -53)
+
+random_term() ->
+    case rand:uniform(6) of
+        1 -> rand:uniform(1 bsl 27) - 1; % small
+        2 -> (1 bsl 27) + rand:uniform(1 bsl 128); % big
+        3 -> random_sign() * (rand:uniform() * ?HALF_DBL_EPSILON); % float
+        4 -> random_binary();
+        5 -> random_pid();
+        6 ->
+            Length = rand:uniform(10),
+            List = [random_term() || _ <- lists:seq(1, Length)],
+            case rand:uniform(2) of
+                1 ->
+                   List;
+                2 ->
+                   list_to_tuple(List)
+            end
+    end.
+
+random_sign() ->
+    case rand:uniform(2) of
+        1 -> -1.0;
+        2 -> 1.0
+    end.
+
+random_binary() ->
+    list_to_binary(random_bytes(rand:uniform(32) - 1)).
+
+random_bytes(0) ->
+    [];
+random_bytes(N) when N > 0 ->
+    [rand:uniform(256) - 1 | random_bytes(N - 1)].
+
+random_pid() ->
+    Processes = erlang:processes(),
+    lists:nth(rand:uniform(length(Processes)), Processes).
 
 %% The NIFs:
 lib_version() -> undefined.
@@ -2621,6 +2700,8 @@ type_test() -> ?nif_stub.
 tuple_2_list(_) -> ?nif_stub.    
 is_identical(_,_) -> ?nif_stub.
 compare(_,_) -> ?nif_stub.
+phash2_nif(_) -> ?nif_stub.
+phash2_ranged_nif(_, _) -> ?nif_stub.
 many_args_100(_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_) -> ?nif_stub.
 clone_bin(_) -> ?nif_stub.
 make_sub_bin(_,_,_) -> ?nif_stub.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 8fe5ee809a..3bdd4d93c7 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -687,6 +687,23 @@ static ERL_NIF_TERM compare(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     return enif_make_int(env, enif_compare(argv[0],argv[1]));
 }
 
+static ERL_NIF_TERM phash2_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    if (argc != 1) {
+        return enif_make_badarg(env);
+    }
+    return enif_make_ulong(env, enif_phash2(argv[0]));
+}
+
+static ERL_NIF_TERM phash2_ranged_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    unsigned long range;
+    if (argc != 2 || !enif_get_ulong(env, argv[1], &range)) {
+        return enif_make_badarg(env);
+    }
+    return enif_make_ulong(env, enif_phash2_ranged(argv[0], range));
+}
+
 static ERL_NIF_TERM many_args_100(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     int i, k;
@@ -2864,6 +2881,8 @@ static ErlNifFunc nif_funcs[] =
     {"tuple_2_list", 1, tuple_2_list},
     {"is_identical",2,is_identical},
     {"compare",2,compare},
+    {"phash2_nif",1,phash2_nif},
+    {"phash2_ranged_nif",2,phash2_ranged_nif},
     {"many_args_100", 100, many_args_100},
     {"clone_bin", 1, clone_bin},
     {"make_sub_bin", 3, make_sub_bin},
-- 
cgit v1.2.3


From 9e4c07da93f8fd9788c10995c98bcfd1a0c2edda Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 30 Mar 2017 10:35:04 +0200
Subject: erts: Fix new gcc warning in check io

---
 erts/emulator/sys/common/erl_check_io.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 00c70268df..b6d71b3ec5 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -2306,7 +2306,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
         case ERTS_EV_TYPE_NIF: { /* Requested via enif_select()... */
             struct erts_nif_select_event in = {NIL};
             struct erts_nif_select_event out = {NIL};
-            ErtsResource* resource;
+            ErtsResource* resource = NULL;
             ErtsPollEvents revents = pollres[i].events;
 
             if (revents & ERTS_POLL_EV_ERR) {
-- 
cgit v1.2.3


From 0326b22639d0c8c5f915aed34ad3e099e4ee4642 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 19 Apr 2017 12:46:44 +0200
Subject: Do atom roundtripping with an R16B node

It is no longer possible to communicate with R15B nodes.
---
 erts/emulator/test/distribution_SUITE.erl | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index 6994bfef83..4f98cd6cee 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -19,7 +19,7 @@
 %%
 
 -module(distribution_SUITE).
--compile(r15).
+-compile(r16).
 
 -define(VERSION_MAGIC,       131).
 
@@ -48,7 +48,7 @@
          dist_parallel_send/1,
          atom_roundtrip/1,
          unicode_atom_roundtrip/1,
-         atom_roundtrip_r15b/1,
+         atom_roundtrip_r16b/1,
          contended_atom_cache_entry/1,
          contended_unicode_atom_cache_entry/1,
          bad_dist_structure/1,
@@ -78,7 +78,8 @@ all() ->
      link_to_dead_new_node, applied_monitor_node,
      ref_port_roundtrip, nil_roundtrip, stop_dist,
      {group, trap_bif}, {group, dist_auto_connect},
-     dist_parallel_send, atom_roundtrip, unicode_atom_roundtrip, atom_roundtrip_r15b,
+     dist_parallel_send, atom_roundtrip, unicode_atom_roundtrip,
+     atom_roundtrip_r16b,
      contended_atom_cache_entry, contended_unicode_atom_cache_entry,
      bad_dist_structure, {group, bad_dist_ext},
      start_epmd_false, epmd_module].
@@ -1032,21 +1033,21 @@ atom_roundtrip(Config) when is_list(Config) ->
     stop_node(Node),
     ok.
 
-atom_roundtrip_r15b(Config) when is_list(Config) ->
-    case test_server:is_release_available("r15b") of
+atom_roundtrip_r16b(Config) when is_list(Config) ->
+    case test_server:is_release_available("r16b") of
         true ->
             ct:timetrap({minutes, 6}),
-            AtomData = atom_data(),
+            AtomData = unicode_atom_data(),
             verify_atom_data(AtomData),
-            case start_node(Config, [], "r15b") of
+            case start_node(Config, [], "r16b") of
                 {ok, Node} ->
                     do_atom_roundtrip(Node, AtomData),
                     stop_node(Node);
                 {error, timeout} ->
-                    {skip,"Unable to start OTP R15B release"}
+                    {skip,"Unable to start OTP R16B release"}
             end;
         false ->
-            {skip,"No OTP R15B available"}
+            {skip,"No OTP R16B available"}
     end.
 
 unicode_atom_roundtrip(Config) when is_list(Config) ->
-- 
cgit v1.2.3


From 8121b6af55ba485fe58a9dbfb0f1180393b86a3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 19 Apr 2017 12:54:56 +0200
Subject: Remove test case for testing compatibility with R9B

It is no longer possible to commuicate with R9B nodes.
---
 erts/emulator/test/Makefile                 |  1 -
 erts/emulator/test/node_container_SUITE.erl | 32 +------------------
 erts/emulator/test/old_mod.erl              | 48 -----------------------------
 3 files changed, 1 insertion(+), 80 deletions(-)
 delete mode 100644 erts/emulator/test/old_mod.erl

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index 8c8c73aa3e..ca81c9915e 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -121,7 +121,6 @@ MODULES= \
 	port_trace_SUITE \
 	unique_SUITE \
 	z_SUITE \
-	old_mod \
 	long_timers_test \
 	ignore_cores \
 	dgawd_handler \
diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl
index 291d3ee30d..30518a5b27 100644
--- a/erts/emulator/test/node_container_SUITE.erl
+++ b/erts/emulator/test/node_container_SUITE.erl
@@ -45,7 +45,6 @@
          ets_refc/1,
          match_spec_refc/1,
          timer_refc/1,
-         otp_4715/1,
          pid_wrap/1,
          port_wrap/1,
          bad_nc/1,
@@ -62,7 +61,7 @@ all() ->
     [term_to_binary_to_term_eq, round_trip_eq, cmp, ref_eq,
      node_table_gc, dist_link_refc, dist_monitor_refc,
      node_controller_refc, ets_refc, match_spec_refc,
-     timer_refc, otp_4715, pid_wrap, port_wrap, bad_nc,
+     timer_refc, pid_wrap, port_wrap, bad_nc,
      unique_pid, iter_max_procs, magic_ref].
 
 init_per_suite(Config) ->
@@ -684,35 +683,6 @@ timer_refc(Config) when is_list(Config) ->
     nc_refc_check(node()),
     ok.
 
-otp_4715(Config) when is_list(Config) ->
-    case test_server:is_release_available("r9b") of
-        true -> otp_4715_1(Config);
-        false -> {skip,"No R9B found"}
-    end.
-
-otp_4715_1(Config) ->
-    case erlang:system_info(compat_rel) of
-        9 ->
-            run_otp_4715(Config);
-        _ ->
-            Pa = filename:dirname(code:which(?MODULE)),
-            test_server:run_on_shielded_node(fun () ->
-                                                     run_otp_4715(Config)
-                                             end,
-                                             "+R9 -pa " ++ Pa)
-    end.
-
-run_otp_4715(Config) when is_list(Config) ->
-    erts_debug:set_internal_state(available_internal_state, true),
-    PidList = [mk_pid({a@b, 1}, 4710, 2),
-               mk_pid({a@b, 1}, 4712, 1),
-               mk_pid({c@b, 1}, 4711, 1),
-               mk_pid({b@b, 3}, 4711, 1),
-               mk_pid({b@b, 2}, 4711, 1)],
-
-    R9Sorted = old_mod:sort_on_old_node(PidList),
-    R9Sorted = lists:sort(PidList).
-
 pid_wrap(Config) when is_list(Config) -> pp_wrap(pid).
 
 port_wrap(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/old_mod.erl b/erts/emulator/test/old_mod.erl
deleted file mode 100644
index 866aba79bb..0000000000
--- a/erts/emulator/test/old_mod.erl
+++ /dev/null
@@ -1,48 +0,0 @@
-%%
-%% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 2003-2016. All Rights Reserved.
-%% 
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%% 
-%% %CopyrightEnd%
-%%
-
--module(old_mod).
--compile(r10).
-
--export([sort_on_old_node/1, sorter/3]).
-
--include_lib("common_test/include/ct.hrl").
-
-sorter(Receiver, Ref, List) ->
-    Receiver ! {Ref, lists:sort(List)}.
-
-sort_on_old_node(List) when is_list(List) ->
-    OldVersion = "r10",
-    Pa = filename:dirname(code:which(?MODULE)),
-    {X, Y, Z} = now(),
-    NodeName = list_to_atom(OldVersion 
-                            ++ "_"
-                            ++ integer_to_list(X)
-                            ++ integer_to_list(Y)
-                            ++ integer_to_list(Z)),
-    {ok, Node} = test_server:start_node(NodeName,
-                                        peer,
-                                        [{args, " -pa " ++ Pa},
-                                         {erl, [{release, OldVersion++"b_patched"}]}]),
-    Ref = make_ref(),
-    spawn_link(Node, ?MODULE, sorter, [self(), Ref, List]),
-    SortedPids = receive {Ref, SP} -> SP end,
-    true = test_server:stop_node(Node),
-    SortedPids.
-- 
cgit v1.2.3


From f8622f94739bb4d240cc3965cc67b8096feab503 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 20 Apr 2017 14:32:36 +0200
Subject: erts: Initialize esdp->type in non-smp to normal

The previous code worked through luck because the memory
returned from erts_alloc was zero:ed and SCHED_NORMAL
is 0. But if you run with +Meamin that is not always the
case which is how this error was detected.
---
 erts/emulator/beam/erl_process.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 894e0ee582..c8e217d114 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -6073,6 +6073,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     runq->scheduler = esdp;
     esdp->run_queue = runq;
     esdp->no = (Uint) num;
+    esdp->type = ERTS_SCHED_NORMAL;
 #endif
 
     esdp->ssi = ssi;
-- 
cgit v1.2.3


From 308841d8c99907a364d6876ed9375507153729eb Mon Sep 17 00:00:00 2001
From: Andreas Schultz <aschultz@tpip.net>
Date: Thu, 2 Feb 2017 11:17:17 +0100
Subject: implement SO_BINDTODEVICE for inet protocols

bind to device is needed to properly support VRF-Lite under
Linux (see [1] for details).

[1]: https://www.kernel.org/doc/Documentation/networking/vrf.txt
---
 erts/emulator/drivers/common/inet_drv.c | 97 ++++++++++++++++++++++++++++++++-
 1 file changed, 96 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 0fe5183b42..90114b77f5 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -591,7 +591,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 			     (((unsigned char*) (s))[1] << 8) | \
 			     (((unsigned char*) (s))[0]))
 
-#ifdef HAVE_SYS_UN_H
+#if defined(HAVE_SYS_UN_H) || defined(SO_BINDTODEVICE)
 
 /* strnlen doesn't exist everywhere */
 static size_t my_strnlen(const char *s, size_t maxlen)
@@ -602,6 +602,10 @@ static size_t my_strnlen(const char *s, size_t maxlen)
     return i;
 }
 
+#endif
+
+#if defined(HAVE_SYS_UN_H)
+
 /* Check that some character in the buffer != '\0' */
 static int is_nonzero(const char *s, size_t n)
 {
@@ -778,6 +782,7 @@ static int is_nonzero(const char *s, size_t n)
 #define INET_LOPT_TCP_SHOW_ECONNRESET 39  /* tell user about incoming RST */
 #define INET_LOPT_LINE_DELIM        40  /* Line delimiting char */
 #define INET_OPT_TCLASS             41  /* IPv6 transport class */
+#define INET_OPT_BIND_TO_DEVICE     42  /* get/set network device the socket is bound to */
 /* SCTP options: a separate range, from 100: */
 #define SCTP_OPT_RTOINFO		100
 #define SCTP_OPT_ASSOCINFO		101
@@ -1334,6 +1339,7 @@ static ErlDrvTermData am_tos;
 static ErlDrvTermData am_tclass;
 static ErlDrvTermData am_ipv6_v6only;
 static ErlDrvTermData am_netns;
+static ErlDrvTermData am_bind_to_device;
 #endif
 
 static char str_eafnosupport[] = "eafnosupport";
@@ -3725,6 +3731,7 @@ static void inet_init_sctp(void) {
     INIT_ATOM(tclass);
     INIT_ATOM(ipv6_v6only);
     INIT_ATOM(netns);
+    INIT_ATOM(bind_to_device);
     
     /* Option names */
     INIT_ATOM(sctp_rtoinfo);
@@ -5946,6 +5953,9 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
     int ival;
     char* arg_ptr;
     int arg_sz;
+#ifdef SO_BINDTODEVICE
+    char ifname[IFNAMSIZ];
+#endif
     enum PacketParseType old_htype = desc->htype;
     int old_active = desc->active;
     int propagate; /* Set to 1 if failure to set this option
@@ -6331,6 +6341,29 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    len -= arg_sz;
 	    break;
 
+#ifdef SO_BINDTODEVICE
+	case INET_OPT_BIND_TO_DEVICE:
+	    if (ival < 0) return -1;
+	    if (len < ival) return -1;
+	    if (ival > sizeof(ifname)) {
+		return -1;
+	    }
+	    memcpy(ifname, ptr, ival);
+	    ifname[ival] = '\0';
+	    ptr += ival;
+	    len -= ival;
+
+	    proto = SOL_SOCKET;
+	    type = SO_BINDTODEVICE;
+	    arg_ptr = (char*)&ifname;
+	    arg_sz = sizeof(ifname);
+	    propagate = 1; /* We do want to know if this fails */
+
+	    DEBUGF(("inet_set_opts(%ld): s=%d, SO_BINDTODEVICE=%s\r\n",
+		    (long)desc->port, desc->s, ifname));
+	    break;
+#endif
+
 	default:
 	    return -1;
 	}
@@ -6462,6 +6495,9 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	struct sctp_event_subscribe es;
 #	ifdef SCTP_DELAYED_ACK_TIME
 	struct sctp_assoc_value     av; /* Not in SOLARIS10 */
+#	endif
+#	ifdef SO_BINDTODEVICE
+	char ifname[IFNAMSIZ];
 #	endif
     }
     arg;
@@ -6702,6 +6738,23 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    continue; /* Option not supported -- ignore it */
 #       endif
 
+#ifdef SO_BINDTODEVICE
+	case INET_OPT_BIND_TO_DEVICE:
+	    arg_sz = get_int32(curr);			curr += 4;
+	    CHKLEN(curr, arg_sz);
+	    if (arg_sz >= sizeof(arg.ifname))
+		return -1;
+	    memcpy(arg.ifname, curr, arg_sz);
+	    arg.ifname[arg_sz] = '\0';
+	    curr += arg_sz;
+
+	    proto   = SOL_SOCKET;
+	    type    = SO_BINDTODEVICE;
+	    arg_ptr = (char*) (&arg.ifname);
+	    arg_sz  = sizeof  ( arg.ifname);
+	    break;
+#endif
+
 	case SCTP_OPT_AUTOCLOSE:
 	{
 	    arg.ival= get_int32 (curr);	  curr += 4;
@@ -6967,6 +7020,9 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc,
     ErlDrvSizeT dest_used = 0;
     ErlDrvSizeT dest_allocated = destlen;
     char *orig_dest = *dest;
+#ifdef SO_BINDTODEVICE
+    char ifname[IFNAMSIZ];
+#endif
 
     /* Ptr is a name parameter */ 
 #define RETURN_ERROR()				\
@@ -7302,6 +7358,26 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc,
 		put_int32(arg_sz,ptr);
 		continue;
 	    }
+
+#ifdef SO_BINDTODEVICE
+	case INET_OPT_BIND_TO_DEVICE:
+	    arg_sz = sizeof(ifname);
+	    TRUNCATE_TO(0,ptr);
+	    PLACE_FOR(5 + arg_sz,ptr);
+	    arg_ptr = ptr + 5;
+	    if (IS_SOCKET_ERROR(sock_getopt(desc->s,SOL_SOCKET,SO_BINDTODEVICE,
+						arg_ptr,&arg_sz))) {
+		    TRUNCATE_TO(0,ptr);
+		    continue;
+		}
+	    arg_sz = my_strnlen(arg_ptr, arg_sz);
+	    TRUNCATE_TO(arg_sz + 5,ptr);
+	    *ptr++ = opt;
+	    put_int32(arg_sz,ptr);
+	    ptr += arg_sz;
+	    continue;
+#endif
+
 	default:
 	    RETURN_ERROR();
 	}
@@ -7583,6 +7659,25 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc,
 	    i = LOAD_TUPLE	(spec, i, 2);
 	    break;
 	}
+
+#ifdef SO_BINDTODEVICE
+	/* The following option returns a binary:   */
+	case INET_OPT_BIND_TO_DEVICE: {
+	    char ifname[IFNAMSIZ];
+	    unsigned int  sz = sizeof(ifname);
+
+	    if (sock_getopt(desc->s, SOL_SOCKET, SO_BINDTODEVICE,
+			    &ifname, &sz) < 0) continue;
+	    /* Fill in the response: */
+	    PLACE_FOR(spec, i,
+		      LOAD_ATOM_CNT + LOAD_BUF2BINARY_CNT + LOAD_TUPLE_CNT);
+	    i = LOAD_ATOM (spec, i, am_bind_to_device);
+	    i = LOAD_BUF2BINARY(spec, i, ifname, my_strnlen(ifname, sz));
+	    i = LOAD_TUPLE (spec, i, 2);
+	    break;
+	}
+#endif
+
 	/* The following options just return an integer value: */
 	case INET_OPT_RCVBUF   :
 	case INET_OPT_SNDBUF   :
-- 
cgit v1.2.3


From 1b37d0b010ea31b04b9d0a15d0bec9c75a013dc9 Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Thu, 20 Apr 2017 21:35:54 +0100
Subject: erts: Remove enif_phash2_ranged

---
 erts/emulator/beam/bif.c                      | 17 +++++++++------
 erts/emulator/beam/erl_nif.c                  | 14 ++-----------
 erts/emulator/beam/erl_nif_api_funcs.h        |  2 --
 erts/emulator/beam/erl_utils.h                |  1 -
 erts/emulator/beam/utils.c                    | 10 ---------
 erts/emulator/test/nif_SUITE.erl              | 30 ++-------------------------
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 10 ---------
 7 files changed, 15 insertions(+), 69 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index d59adc18d6..214de3652f 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4888,6 +4888,7 @@ BIF_RETTYPE phash2_1(BIF_ALIST_1)
 BIF_RETTYPE phash2_2(BIF_ALIST_2)
 {
     Uint32 hash;
+    Uint32 final_hash;
     Uint32 range;
 
     /* Check for special case 2^32 */
@@ -4900,19 +4901,23 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2)
 	}
 	range = (Uint32) u;
     }
-    hash = make_hash2_within_range(BIF_ARG_1, range);
-
+    hash = make_hash2(BIF_ARG_1);
+    if (range) {
+	final_hash = hash % range; /* [0..range-1] */
+    } else {
+	final_hash = hash;
+    }
     /*
      * Return either a small or a big. Use the heap for bigs if there is room.
      */
 #if defined(ARCH_64)
-    BIF_RET(make_small(hash));
+    BIF_RET(make_small(final_hash));
 #else
-    if (IS_USMALL(0, hash)) {
-	BIF_RET(make_small(hash));
+    if (IS_USMALL(0, final_hash)) {
+	BIF_RET(make_small(final_hash));
     } else {
 	Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE);
-	BIF_RET(uint_to_big(hash, hp));
+	BIF_RET(uint_to_big(final_hash, hp));
     }
 #endif
 }
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 14abf461a3..f7777c8b92 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1214,25 +1214,15 @@ int enif_compare(Eterm lhs, Eterm rhs)
     return result;
 }
 
+unsigned long enif_phash2(Eterm term)
+{
 #if SIZEOF_LONG < 4
 /* This *really* shouldn't happen */
 #  error Incompatible long word size
 #endif
-
-unsigned long enif_phash2(Eterm term)
-{
     return make_hash2(term) & ((1 << 27) - 1);
 }
 
-unsigned long enif_phash2_ranged(Eterm term, unsigned long range)
-{
-#if SIZEOF_LONG > 4
-    if (range > (unsigned long) UINT32_MAX)
-        range = 0;
-#endif
-    return make_hash2_within_range(term, range);
-}
-
 int enif_get_tuple(ErlNifEnv* env, Eterm tpl, int* arity, const Eterm** array)
 {
     Eterm* ptr;
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 8a3f9165d9..c096b18f91 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -48,7 +48,6 @@ ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int*
 ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
 ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
 ERL_NIF_API_FUNC_DECL(unsigned long,enif_phash2,(ERL_NIF_TERM term));
-ERL_NIF_API_FUNC_DECL(unsigned long,enif_phash2_ranged,(ERL_NIF_TERM term, unsigned long));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i));
@@ -211,7 +210,6 @@ ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const Erl
 #  define enif_is_identical ERL_NIF_API_FUNC_MACRO(enif_is_identical)
 #  define enif_compare ERL_NIF_API_FUNC_MACRO(enif_compare)
 #  define enif_phash2 ERL_NIF_API_FUNC_MACRO(enif_phash2)
-#  define enif_phash2_ranged ERL_NIF_API_FUNC_MACRO(enif_phash2_ranged)
 
 #  define enif_make_binary ERL_NIF_API_FUNC_MACRO(enif_make_binary)
 #  define enif_make_badarg ERL_NIF_API_FUNC_MACRO(enif_make_badarg)
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 75578c26d6..47289a0af1 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -119,7 +119,6 @@ Sint erts_list_length(Eterm);
 int erts_is_builtin(Eterm, Eterm, int);
 Uint32 block_hash(byte *, unsigned, Uint32);
 Uint32 make_hash2(Eterm);
-Uint32 make_hash2_within_range(Eterm, Uint32);
 Uint32 make_hash(Eterm);
 Uint32 make_internal_hash(Eterm);
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 72c59b33c6..8f3f48f38f 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1552,16 +1552,6 @@ make_hash2(Eterm term)
     }
 }
 
-Uint32
-make_hash2_within_range(Eterm term, Uint32 range)
-{
-    Uint32 hash = make_hash2(term);
-    if (range)
-        return hash % range; /* [0..range-1] */
-    else
-        return hash; /* Special case: 2**32 */
-}
-
 /* Term hash function for internal use.
  *
  * Limitation #1: Is not "portable" in any way between different VM instances.
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 780fe0a5ee..f868b018c9 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -57,8 +57,7 @@
          nif_term_to_binary/1, nif_binary_to_term/1,
          nif_port_command/1,
          nif_snprintf/1,
-         nif_phash2/1,
-         nif_phash2_ranged/1
+         nif_phash2/1
 	]).
 
 -export([many_args_100/100]).
@@ -93,8 +92,7 @@ all() ->
      nif_term_to_binary, nif_binary_to_term,
      nif_port_command,
      nif_snprintf,
-     nif_phash2,
-     nif_phash2_ranged].
+     nif_phash2].
 
 groups() ->
     [{G, [], api_repeaters()} || G <- api_groups()]
@@ -2629,29 +2627,6 @@ nif_phash2(Config) ->
       end,
       Terms).
 
-nif_phash2_ranged(Config) ->
-    ensure_lib_loaded(Config),
-    RandomRangedTerms =
-        [{random_term(), rand:uniform((1 bsl 32) - 1)}
-         || _ <- lists:seq(1, 1000)],
-
-    lists:foreach(
-      fun ({Term, Range}) ->
-              HashValue = erlang:phash2(Term, Range),
-              NifHashValue = phash2_ranged_nif(Term, Range),
-              (HashValue =:= NifHashValue
-               orelse ct:fail("Expected: ~p\nActual:   ~p",
-                              [HashValue, NifHashValue]))
-      end,
-      RandomRangedTerms),
-
-    EdgeCaseTerm = random_term(),
-    EdgeCaseHashValue = erlang:phash2(EdgeCaseTerm, 1 bsl 32),
-    EdgeCaseNifHashValue = phash2_ranged_nif(EdgeCaseTerm, 0),
-    (EdgeCaseHashValue =:= EdgeCaseNifHashValue
-     orelse ct:fail("Expected: ~p\nActual:   ~p",
-                    [EdgeCaseHashValue, EdgeCaseNifHashValue])).
-
 -define(HALF_DBL_EPSILON, 1.1102230246251565e-16). % math:pow(2, -53)
 
 random_term() ->
@@ -2701,7 +2676,6 @@ tuple_2_list(_) -> ?nif_stub.
 is_identical(_,_) -> ?nif_stub.
 compare(_,_) -> ?nif_stub.
 phash2_nif(_) -> ?nif_stub.
-phash2_ranged_nif(_, _) -> ?nif_stub.
 many_args_100(_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_) -> ?nif_stub.
 clone_bin(_) -> ?nif_stub.
 make_sub_bin(_,_,_) -> ?nif_stub.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 3bdd4d93c7..dac6d02e9d 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -695,15 +695,6 @@ static ERL_NIF_TERM phash2_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
     return enif_make_ulong(env, enif_phash2(argv[0]));
 }
 
-static ERL_NIF_TERM phash2_ranged_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
-{
-    unsigned long range;
-    if (argc != 2 || !enif_get_ulong(env, argv[1], &range)) {
-        return enif_make_badarg(env);
-    }
-    return enif_make_ulong(env, enif_phash2_ranged(argv[0], range));
-}
-
 static ERL_NIF_TERM many_args_100(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     int i, k;
@@ -2882,7 +2873,6 @@ static ErlNifFunc nif_funcs[] =
     {"is_identical",2,is_identical},
     {"compare",2,compare},
     {"phash2_nif",1,phash2_nif},
-    {"phash2_ranged_nif",2,phash2_ranged_nif},
     {"many_args_100", 100, many_args_100},
     {"clone_bin", 1, clone_bin},
     {"make_sub_bin", 3, make_sub_bin},
-- 
cgit v1.2.3


From 5ad95cca3f3c6802cd71fdd35dd397c3a37dc239 Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Thu, 20 Apr 2017 21:53:54 +0100
Subject: erts: Fix random floats in enif_phash2 tests

---
 erts/emulator/test/nif_SUITE.erl | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index f868b018c9..9127a5eae9 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -2627,13 +2627,11 @@ nif_phash2(Config) ->
       end,
       Terms).
 
--define(HALF_DBL_EPSILON, 1.1102230246251565e-16). % math:pow(2, -53)
-
 random_term() ->
     case rand:uniform(6) of
         1 -> rand:uniform(1 bsl 27) - 1; % small
         2 -> (1 bsl 27) + rand:uniform(1 bsl 128); % big
-        3 -> random_sign() * (rand:uniform() * ?HALF_DBL_EPSILON); % float
+        3 -> random_sign() * (rand:uniform() * (1 bsl 53)); % float
         4 -> random_binary();
         5 -> random_pid();
         6 ->
-- 
cgit v1.2.3


From d8756f8665a42effa2f3515369058cff4441abeb Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Thu, 20 Apr 2017 22:44:04 +0100
Subject: erts: Refactor enif_phash2 into enif_hash

A more generic hashing function which can also hash terms based on
`make_internal_hash'.
---
 erts/emulator/beam/erl_nif.c                  | 11 +++++--
 erts/emulator/beam/erl_nif.h                  |  5 +++
 erts/emulator/beam/erl_nif_api_funcs.h        |  4 +--
 erts/emulator/test/nif_SUITE.erl              | 47 +++++++++++++++++++++++++--
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 20 +++++++++---
 5 files changed, 76 insertions(+), 11 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index f7777c8b92..3470cf3fd5 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1214,13 +1214,20 @@ int enif_compare(Eterm lhs, Eterm rhs)
     return result;
 }
 
-unsigned long enif_phash2(Eterm term)
+unsigned long enif_hash(ErlNifHash type, Eterm term)
 {
 #if SIZEOF_LONG < 4
 /* This *really* shouldn't happen */
 #  error Incompatible long word size
 #endif
-    return make_hash2(term) & ((1 << 27) - 1);
+    switch (type) {
+        case ERL_NIF_INTERNAL_HASH:
+            return make_internal_hash(term);
+        case ERL_NIF_PHASH2:
+            return make_hash2(term) & ((1 << 27) - 1);
+        default:
+            return 0;
+    }
 }
 
 int enif_get_tuple(ErlNifEnv* env, Eterm tpl, int* arity, const Eterm** array)
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index ac45f3ac81..5a81f5fbbb 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -236,6 +236,11 @@ typedef enum {
     ERL_NIF_BIN2TERM_SAFE = 0x20000000
 } ErlNifBinaryToTerm;
 
+typedef enum {
+    ERL_NIF_INTERNAL_HASH = 1,
+    ERL_NIF_PHASH2 = 2
+} ErlNifHash;
+
 /*
  * Return values from enif_thread_type(). Negative values
  * reserved for specific types of non-scheduler threads.
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index c096b18f91..f52c841ed7 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -47,7 +47,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term,
 ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array));
 ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
 ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
-ERL_NIF_API_FUNC_DECL(unsigned long,enif_phash2,(ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(unsigned long,enif_hash,(ErlNifHash type, ERL_NIF_TERM term));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i));
@@ -209,7 +209,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const Erl
 #  define enif_get_list_cell ERL_NIF_API_FUNC_MACRO(enif_get_list_cell)
 #  define enif_is_identical ERL_NIF_API_FUNC_MACRO(enif_is_identical)
 #  define enif_compare ERL_NIF_API_FUNC_MACRO(enif_compare)
-#  define enif_phash2 ERL_NIF_API_FUNC_MACRO(enif_phash2)
+#  define enif_hash ERL_NIF_API_FUNC_MACRO(enif_hash)
 
 #  define enif_make_binary ERL_NIF_API_FUNC_MACRO(enif_make_binary)
 #  define enif_make_badarg ERL_NIF_API_FUNC_MACRO(enif_make_badarg)
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 9127a5eae9..e177fa4963 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -57,6 +57,7 @@
          nif_term_to_binary/1, nif_binary_to_term/1,
          nif_port_command/1,
          nif_snprintf/1,
+         nif_internal_hash/1,
          nif_phash2/1
 	]).
 
@@ -92,6 +93,7 @@ all() ->
      nif_term_to_binary, nif_binary_to_term,
      nif_port_command,
      nif_snprintf,
+     nif_internal_hash,
      nif_phash2].
 
 groups() ->
@@ -2612,15 +2614,54 @@ nif_snprintf(Config) ->
     <<"{{hello,world,-33},",0>> = format_term_nif(20,{{hello,world, -33}, 3.14, self()}),
     ok.
 
+nif_internal_hash(Config) ->
+    ensure_lib_loaded(Config),
+    Terms =
+        [random_term() || _ <- lists:seq(1, 5000)],
+
+    % Unlike the phash2 hash, in this case we
+    % have nothing to compare to, so let's try
+    % and at least make sure the distribution
+    % isn't outright wrong, statistical nuances
+    % aside.
+
+    OnesPerBit =
+        lists:foldl(
+          fun (Term, Acc) ->
+                  NifHashValue = hash_nif(internal, Term),
+                  lists:foldl(
+                    fun (BitIndex, AccB) ->
+                            BitValue = (NifHashValue band (1 bsl BitIndex)) bsr BitIndex,
+                            dict:update_counter(BitIndex, BitValue, AccB)
+                    end,
+                    Acc,
+                    lists:seq(0, 31))
+          end,
+          dict:new(),
+          Terms),
+
+    ExpectedNrOfOnes = length(Terms) div 2,
+    dict:fold(
+      fun (BitIndex, NrOfOnes, Acc) ->
+              RelativeDeviation = abs(NrOfOnes - ExpectedNrOfOnes) / ExpectedNrOfOnes,
+              (RelativeDeviation < 0.10
+               orelse ct:fail("Unreasonable deviation on number of set bits (i=~p): "
+                              "expected ~p, got ~p (relative dev. ~.3f)",
+                              [BitIndex, ExpectedNrOfOnes, NrOfOnes, RelativeDeviation])),
+              Acc
+      end,
+      ok,
+      OnesPerBit).
+
 nif_phash2(Config) ->
     ensure_lib_loaded(Config),
     Terms =
-        [random_term() || _ <- lists:seq(1, 1000)],
+        [random_term() || _ <- lists:seq(1, 5000)],
 
     lists:foreach(
       fun (Term) ->
               HashValue = erlang:phash2(Term),
-              NifHashValue = phash2_nif(Term),
+              NifHashValue = hash_nif(phash2, Term),
               (HashValue =:= NifHashValue
                orelse ct:fail("Expected: ~p\nActual:   ~p",
                               [HashValue, NifHashValue]))
@@ -2673,7 +2714,7 @@ type_test() -> ?nif_stub.
 tuple_2_list(_) -> ?nif_stub.    
 is_identical(_,_) -> ?nif_stub.
 compare(_,_) -> ?nif_stub.
-phash2_nif(_) -> ?nif_stub.
+hash_nif(_, _) -> ?nif_stub.
 many_args_100(_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_) -> ?nif_stub.
 clone_bin(_) -> ?nif_stub.
 make_sub_bin(_,_,_) -> ?nif_stub.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index dac6d02e9d..9d65b9c09b 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -687,12 +687,24 @@ static ERL_NIF_TERM compare(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     return enif_make_int(env, enif_compare(argv[0],argv[1]));
 }
 
-static ERL_NIF_TERM phash2_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+static ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    if (argc != 1) {
+    if (argc != 2) {
+        return enif_make_badarg(env);
+    }
+
+    ErlNifHash type;
+    if (enif_is_identical(argv[0], enif_make_atom(env, "internal"))) {
+        type = ERL_NIF_INTERNAL_HASH;
+    }
+    else if (enif_is_identical(argv[0], enif_make_atom(env, "phash2"))) {
+        type = ERL_NIF_PHASH2;
+    }
+    else {
         return enif_make_badarg(env);
     }
-    return enif_make_ulong(env, enif_phash2(argv[0]));
+
+    return enif_make_ulong(env, enif_hash(type, argv[1]));
 }
 
 static ERL_NIF_TERM many_args_100(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
@@ -2872,7 +2884,7 @@ static ErlNifFunc nif_funcs[] =
     {"tuple_2_list", 1, tuple_2_list},
     {"is_identical",2,is_identical},
     {"compare",2,compare},
-    {"phash2_nif",1,phash2_nif},
+    {"hash_nif",2,hash_nif},
     {"many_args_100", 100, many_args_100},
     {"clone_bin", 1, clone_bin},
     {"make_sub_bin", 3, make_sub_bin},
-- 
cgit v1.2.3


From 59c67fd2fb8a83efadbcd3c88db0128c968ddca5 Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikpelinux@gmail.com>
Date: Fri, 21 Apr 2017 14:10:22 +0200
Subject: erl_mseg.c: don't use invalid indices in - > cache_powered_node[]

---
 erts/emulator/sys/common/erl_mseg.c | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index 882c93a83c..968f71211c 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -87,6 +87,7 @@ static const int debruijn[32] = {
 
 #define CACHE_AREAS      (32 - MSEG_ALIGN_BITS)
 
+/* FIXME: segment sizes > 2 GB result in bogus negative indices */
 #define SIZE_TO_CACHE_AREA_IDX(S)   (LOG2((S)) - MSEG_ALIGN_BITS)
 #define MAX_CACHE_SIZE   (30)
 
@@ -396,6 +397,9 @@ static ERTS_INLINE int cache_bless_segment(ErtsMsegAllctr_t *ma, void *seg, UWor
 	if (MSEG_FLG_IS_2POW(flags)) {
 	    int ix = SIZE_TO_CACHE_AREA_IDX(size);
 
+            if (ix < 0)
+                return 0;
+
 	    ASSERT(ix < CACHE_AREAS);
 	    ASSERT((1 << (ix + MSEG_ALIGN_BITS)) == size);
 
@@ -471,6 +475,9 @@ static ERTS_INLINE void *cache_get_segment(ErtsMsegAllctr_t *ma, UWord *size_p,
 
 	ASSERT(IS_2POW(size));
 
+        if (ix < 0)
+            return NULL;
+
 	for( i = ix; i < CACHE_AREAS; i++) {
 
 	    if (erts_circleq_is_empty(&(ma->cache_powered_node[i])))
-- 
cgit v1.2.3


From f30d131bd979e29b68fb7d9ff515c61a246201f4 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Tue, 21 Mar 2017 13:40:08 +0100
Subject: erts: Deprecate the non-smp emulators

---
 erts/emulator/Makefile.in               | 35 +++++++++++++++++----------------
 erts/emulator/test/smoke_test_SUITE.erl |  7 -------
 2 files changed, 18 insertions(+), 24 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 7ea0111e59..227aabcf1e 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -175,7 +175,23 @@ endif
 # NOTE: When adding a new type update ERL_BUILD_TYPE_MARKER in sys/unix/sys.c
 #
 
-ifeq ($(FLAVOR),smp)
+FLAVOR=$(DEFAULT_FLAVOR)
+
+ifeq ($(FLAVOR),plain)
+
+DS_SUPPORT=no
+DS_TEST=no
+
+FLAVOR_MARKER=
+FLAVOR_FLAGS=
+ENABLE_ALLOC_TYPE_VARS += nofrag
+M4FLAGS +=
+
+else # FLAVOR
+
+# If flavor isn't one of the above, it *is* smp flavor...
+override FLAVOR=smp
+
 FLAVOR_MARKER=.smp
 FLAVOR_FLAGS=-DERTS_SMP
 ENABLE_ALLOC_TYPE_VARS += smp nofrag
@@ -196,30 +212,15 @@ DS_SUPPORT=no
 DS_TEST=no
 endif # DIRTY_SCHEDULER_SUPPORT
 
-else # FLAVOR
-
-DS_SUPPORT=no
-DS_TEST=no
-
-# If flavor isn't one of the above, it *is* plain flavor...
-override FLAVOR=plain
-FLAVOR_MARKER=
-FLAVOR_FLAGS=
-ENABLE_ALLOC_TYPE_VARS += nofrag
-M4FLAGS +=
-endif
+endif # FLAVOR
 
 TF_MARKER=$(TYPEMARKER)$(FLAVOR_MARKER)
 
-ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no)
-VOID_EMULATOR = '*** SMP emulator disabled by configure'
-else
 ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no)
 VOID_EMULATOR = '*** valgrind emulator disabled by configure'
 else
 VOID_EMULATOR =
 endif
-endif
 
 OPSYS=@OPSYS@
 sol2CFLAGS=
diff --git a/erts/emulator/test/smoke_test_SUITE.erl b/erts/emulator/test/smoke_test_SUITE.erl
index 5eccdc562b..45b28b28a5 100644
--- a/erts/emulator/test/smoke_test_SUITE.erl
+++ b/erts/emulator/test/smoke_test_SUITE.erl
@@ -66,17 +66,10 @@ boot_combo(Config) when is_list(Config) ->
 			  ok
 		  end
 	  end,
-    SMPDisable = fun () -> false = erlang:system_info(smp_support) end,
     try
 	chk_boot(Config, "+Ktrue", NOOP),
 	chk_boot(Config, "+A42", A42),
-	chk_boot(Config, "-smp disable", SMPDisable),
 	chk_boot(Config, "+Ktrue +A42", A42),
-	chk_boot(Config, "-smp disable +A42",
-		 fun () -> SMPDisable(), A42() end),
-	chk_boot(Config, "-smp disable +Ktrue", SMPDisable),
-	chk_boot(Config, "-smp disable +Ktrue +A42",
-		 fun () -> SMPDisable(), A42() end),
 	%% A lot more combos could be implemented...
 	ok
     after
-- 
cgit v1.2.3


From 8434b76e48b4ad1fe47b1d38dc3cac0b50d7d52d Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 10 Apr 2017 09:41:46 +0200
Subject: erts: Fix testcases for smp +S 1:1

---
 erts/emulator/test/scheduler_SUITE.erl  | 16 ++++------------
 erts/emulator/test/statistics_SUITE.erl |  4 ++--
 2 files changed, 6 insertions(+), 14 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl
index 885298ce34..64e51e7d7c 100644
--- a/erts/emulator/test/scheduler_SUITE.erl
+++ b/erts/emulator/test/scheduler_SUITE.erl
@@ -1087,12 +1087,8 @@ scheduler_threads(Config) when is_list(Config) ->
     {Sched, SchedOnln, _} = get_sstate(Config, ""),
     %% Configure half the number of both the scheduler threads and
     %% the scheduler threads online.
-    {HalfSched, HalfSchedOnln} = case SmpSupport of
-                                     false -> {1,1};
-                                     true ->
-                                         {Sched div 2,
-                                          SchedOnln div 2}
-                                 end,
+    {HalfSched, HalfSchedOnln} = {lists:max([1,Sched div 2]),
+                                  lists:max([1,SchedOnln div 2])},
     {HalfSched, HalfSchedOnln, _} = get_sstate(Config, "+SP 50:50"),
     %% Use +S to configure 4x the number of scheduler threads and
     %% 4x the number of scheduler threads online, but alter that
@@ -1149,12 +1145,8 @@ dirty_scheduler_threads(Config) when is_list(Config) ->
 dirty_scheduler_threads_test(Config) ->
     SmpSupport = erlang:system_info(smp_support),
     {Sched, SchedOnln, _} = get_dsstate(Config, ""),
-    {HalfSched, HalfSchedOnln} = case SmpSupport of
-                                     false -> {1,1};
-                                     true ->
-                                         {Sched div 2,
-                                          SchedOnln div 2}
-                                 end,
+    {HalfSched, HalfSchedOnln} = {lists:max([1,Sched div 2]),
+                                  lists:max([1,SchedOnln div 2])},
     Cmd1 = "+SDcpu "++integer_to_list(HalfSched)++":"++
 	integer_to_list(HalfSchedOnln),
     {HalfSched, HalfSchedOnln, _} = get_dsstate(Config, Cmd1),
diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl
index f51244485b..729e86cb4f 100644
--- a/erts/emulator/test/statistics_SUITE.erl
+++ b/erts/emulator/test/statistics_SUITE.erl
@@ -329,9 +329,9 @@ scheduler_wall_time_test(Type) ->
         %% 50% load
         HalfHogs = [StartHog() || _ <- lists:seq(1, (Schedulers-1) div 2)],
         HalfDirtyCPUHogs = [StartDirtyHog(dirty_cpu)
-                            || _ <- lists:seq(1, DirtyCPUSchedulers div 2)],
+                            || _ <- lists:seq(1, lists:max([1,DirtyCPUSchedulers div 2]))],
         HalfDirtyIOHogs = [StartDirtyHog(dirty_io)
-                           || _ <- lists:seq(1, DirtyIOSchedulers div 2)],
+                           || _ <- lists:seq(1, lists:max([1,DirtyIOSchedulers div 2]))],
         HalfLoad = lists:sum(get_load(Type)) div TotLoadSchedulers,
         if Schedulers < 2, HalfLoad > 80 -> ok; %% Ok only one scheduler online and one hog
            %% We want roughly 50% load
-- 
cgit v1.2.3


From 4c408d98127c3a44f47b86e89d2a0bd5567b1e70 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 21 Apr 2017 17:16:13 +0200
Subject: Do not ignore SIGTERM when VM has been started with +Bi

---
 erts/emulator/sys/unix/erl_unix_sys.h |  1 +
 erts/emulator/sys/unix/sys.c          | 19 ++++++++++++++-----
 erts/emulator/sys/unix/sys_drivers.c  |  1 +
 3 files changed, 16 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index b64b0d87f6..054a4678c8 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -322,6 +322,7 @@ extern SIGFUNC sys_signal(int, SIGFUNC);
 extern void sys_sigrelease(int);
 extern void sys_sigblock(int);
 extern void sys_init_suspend_handler(void);
+extern void erts_sys_unix_later_init(void);
 
 /*
  * Handling of floating point exceptions.
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 55411f83d0..3010af44be 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -791,10 +791,14 @@ static RETSIGTYPE do_quit(int signum)
 
 /* Disable break */
 void erts_set_ignore_break(void) {
-    sys_signal(SIGINT,  SIG_IGN);
-    sys_signal(SIGTERM, SIG_IGN);
-    sys_signal(SIGQUIT, SIG_IGN);
-    sys_signal(SIGTSTP, SIG_IGN);
+    /*
+     * Ignore signals that can be sent to the VM by
+     * typing certain key combinations at the
+     * controlling terminal...
+     */
+    sys_signal(SIGINT,  SIG_IGN);       /* Ctrl-C */
+    sys_signal(SIGQUIT, SIG_IGN);       /* Ctrl-\ */
+    sys_signal(SIGTSTP, SIG_IGN);       /* Ctrl-Z */
 }
 
 /* Don't use ctrl-c for break handler but let it be 
@@ -818,7 +822,6 @@ void erts_replace_intr(void) {
 void init_break_handler(void)
 {
    sys_signal(SIGINT, request_break);
-   sys_signal(SIGTERM, request_stop);
 #ifndef ETHR_UNUSABLE_SIGUSRX
    sys_signal(SIGUSR1, user_signal1);
 #endif /* #ifndef ETHR_UNUSABLE_SIGUSRX */
@@ -832,6 +835,12 @@ void sys_init_suspend_handler(void)
 #endif
 }
 
+void
+erts_sys_unix_later_init(void)
+{
+    sys_signal(SIGTERM, request_stop);
+}
+
 int sys_max_files(void)
 {
    return(max_files);
diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c
index 400f163652..93cf64719a 100644
--- a/erts/emulator/sys/unix/sys_drivers.c
+++ b/erts/emulator/sys/unix/sys_drivers.c
@@ -204,6 +204,7 @@ erl_sys_late_init(void)
 #ifdef ERTS_SMP
     erts_mtx_unlock(port->lock);
 #endif
+    erts_sys_unix_later_init(); /* Need to be called after forker has been started */
 }
 
 /* II. Prototypes */
-- 
cgit v1.2.3


From 21e1d879faaae2278a68200fe1085d7e73791fa0 Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Sat, 22 Apr 2017 17:22:32 +0100
Subject: erts: Support custom salt in enif_hash

---
 erts/emulator/beam/bif.c                      |  4 +--
 erts/emulator/beam/erl_bif_info.c             |  2 +-
 erts/emulator/beam/erl_db_hash.c              |  2 +-
 erts/emulator/beam/erl_map.h                  |  2 +-
 erts/emulator/beam/erl_nif.c                  |  6 ++--
 erts/emulator/beam/erl_nif_api_funcs.h        |  2 +-
 erts/emulator/beam/erl_process_dict.c         |  2 +-
 erts/emulator/beam/erl_trace.c                |  2 +-
 erts/emulator/beam/erl_utils.h                |  4 +--
 erts/emulator/beam/utils.c                    | 40 +++++++++++++--------------
 erts/emulator/hipe/hipe_bif0.c                |  2 +-
 erts/emulator/test/nif_SUITE.erl              |  6 ++--
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 11 ++++++--
 13 files changed, 45 insertions(+), 40 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 214de3652f..7473a8e43f 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4881,7 +4881,7 @@ BIF_RETTYPE phash2_1(BIF_ALIST_1)
 {
     Uint32 hash;
 
-    hash = make_hash2(BIF_ARG_1);
+    hash = make_hash2(BIF_ARG_1, 0);
     BIF_RET(make_small(hash & ((1L << 27) - 1)));
 }
 
@@ -4901,7 +4901,7 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2)
 	}
 	range = (Uint32) u;
     }
-    hash = make_hash2(BIF_ARG_1);
+    hash = make_hash2(BIF_ARG_1, 0);
     if (range) {
 	final_hash = hash % range; /* [0..range-1] */
     } else {
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 3a8687dc59..f2863f5d5b 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3950,7 +3950,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 		BIF_RET(erts_debug_reader_groups_map(BIF_P, (int) groups));
 	    }
 	    else if (ERTS_IS_ATOM_STR("internal_hash", tp[1])) {
-		Uint hash = (Uint) make_internal_hash(tp[2]);
+		Uint hash = (Uint) make_internal_hash(tp[2], 0);
 		Uint hsz = 0;
 		Eterm* hp;
 		erts_bld_uint(NULL, &hsz, hash);
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 7ab27df00c..07f6da7b6d 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -189,7 +189,7 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix,
 /* optimised version of make_hash (normal case? atomic key) */
 #define MAKE_HASH(term) \
     ((is_atom(term) ? (atom_tab(atom_val(term))->slot.bucket.hvalue) : \
-      make_internal_hash(term)) % MAX_HASH)
+      make_internal_hash(term, 0)) % MAX_HASH)
 
 #ifdef ERTS_SMP
 #  define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1)
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 61a841f7f0..f7d0413685 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -57,7 +57,7 @@ typedef struct flatmap_s {
 
 
 #define hashmap_size(x)               (((hashmap_head_t*) hashmap_val(x))->size)
-#define hashmap_make_hash(Key)        make_internal_hash(Key)
+#define hashmap_make_hash(Key)        make_internal_hash(Key, 0)
 
 #define hashmap_restore_hash(Heap,Lvl,Key) \
     (((Lvl) < 8) ? hashmap_make_hash(Key) >> (4*(Lvl)) : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), (Key))) >> (4*((Lvl) & 7)))
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 3470cf3fd5..bdfb7f3b8e 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1214,7 +1214,7 @@ int enif_compare(Eterm lhs, Eterm rhs)
     return result;
 }
 
-unsigned long enif_hash(ErlNifHash type, Eterm term)
+unsigned long enif_hash(ErlNifHash type, Eterm term, unsigned long salt)
 {
 #if SIZEOF_LONG < 4
 /* This *really* shouldn't happen */
@@ -1222,9 +1222,9 @@ unsigned long enif_hash(ErlNifHash type, Eterm term)
 #endif
     switch (type) {
         case ERL_NIF_INTERNAL_HASH:
-            return make_internal_hash(term);
+            return make_internal_hash(term, salt);
         case ERL_NIF_PHASH2:
-            return make_hash2(term) & ((1 << 27) - 1);
+            return make_hash2(term, salt) & ((1 << 27) - 1);
         default:
             return 0;
     }
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index f52c841ed7..6b9f78d46b 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -47,7 +47,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term,
 ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array));
 ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
 ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
-ERL_NIF_API_FUNC_DECL(unsigned long,enif_hash,(ErlNifHash type, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(unsigned long,enif_hash,(ErlNifHash type, ERL_NIF_TERM term, unsigned long salt));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i));
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 7cfdf20341..01e240c65d 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -57,7 +57,7 @@
     ((is_small(Term)) ? (Uint32) unsigned_val(Term) :  \
      ((is_atom(Term)) ?                                \
       (Uint32) atom_val(Term) :                        \
-      make_internal_hash(Term)))
+      make_internal_hash(Term, 0)))
 
 #define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm))
 
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 04f3160d42..8855449ca1 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -3230,7 +3230,7 @@ static int tracer_cmp_fun(void* a, void* b)
 
 static HashValue tracer_hash_fun(void* obj)
 {
-    return make_internal_hash(((ErtsTracerNif*)obj)->module);
+    return make_internal_hash(((ErtsTracerNif*)obj)->module, 0);
 }
 
 static void *tracer_alloc_fun(void* tmpl)
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 47289a0af1..b38beaf93d 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -118,9 +118,9 @@ int erts_fit_in_bits_uint(Uint);
 Sint erts_list_length(Eterm);
 int erts_is_builtin(Eterm, Eterm, int);
 Uint32 block_hash(byte *, unsigned, Uint32);
-Uint32 make_hash2(Eterm);
+Uint32 make_hash2(Eterm, Uint32 salt);
 Uint32 make_hash(Eterm);
-Uint32 make_internal_hash(Eterm);
+Uint32 make_internal_hash(Eterm, Uint32 salt);
 
 void erts_save_emu_args(int argc, char **argv);
 Eterm erts_get_emu_args(struct process *c_p);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 8f3f48f38f..2378c057dd 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1003,7 +1003,7 @@ tail_recur:
 	    break;
 	}
     case MAP_DEF:
-        hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term);
+        hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term, 0);
         break;
     case TUPLE_DEF:
 	{
@@ -1109,7 +1109,7 @@ block_hash(byte *k, unsigned length, Uint32 initval)
 }
 
 Uint32
-make_hash2(Eterm term)
+make_hash2(Eterm term, Uint32 salt)
 {
     Uint32 hash;
     Uint32 hash_xor_pairs;
@@ -1179,7 +1179,7 @@ make_hash2(Eterm term)
 	    switch (term & _TAG_IMMED2_MASK) {
 	    case _TAG_IMMED2_ATOM:
 		/* Fast, but the poor hash value should be mixed. */
-		return atom_tab(atom_val(term))->slot.bucket.hvalue;
+		return atom_tab(atom_val(term))->slot.bucket.hvalue ^ salt;
 	    }
 	    break;
 	case _TAG_IMMED1_SMALL:
@@ -1190,7 +1190,7 @@ make_hash2(Eterm term)
 		  term = small_to_big(x, tmp_big);
 		  break;
 	      }
-	      hash = 0;
+	      hash = salt;
 	      SINT32_HASH(x, HCONST);
 	      return hash;
 	  }
@@ -1201,7 +1201,7 @@ make_hash2(Eterm term)
     DECLARE_ESTACK(s);
 
     UseTmpHeapNoproc(2);
-    hash = 0;
+    hash = salt;
     for (;;) {
 	switch (primary_tag(term)) {
 	case TAG_PRIMARY_LIST:
@@ -1277,7 +1277,7 @@ make_hash2(Eterm term)
                     ESTACK_PUSH(s, hash_xor_pairs);
                     ESTACK_PUSH(s, hash);
                     ESTACK_PUSH(s, HASH_MAP_TAIL);
-                    hash = 0;
+                    hash = salt;
                     hash_xor_pairs = 0;
                     for (i = size - 1; i >= 0; i--) {
                         ESTACK_PUSH(s, HASH_MAP_PAIR);
@@ -1296,7 +1296,7 @@ make_hash2(Eterm term)
                     ESTACK_PUSH(s, hash_xor_pairs);
                     ESTACK_PUSH(s, hash);
                     ESTACK_PUSH(s, HASH_MAP_TAIL);
-                    hash = 0;
+                    hash = salt;
                     hash_xor_pairs = 0;
                 }
                 switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
@@ -1471,7 +1471,7 @@ make_hash2(Eterm term)
 	    break;
 
 	    default:
-		erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
+		erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X, %lu)\n", term, salt);
 	    }
 	}
 	break;
@@ -1488,21 +1488,21 @@ make_hash2(Eterm term)
 	    case _TAG_IMMED1_IMMED2:
 		switch (term & _TAG_IMMED2_MASK) {
 		case _TAG_IMMED2_ATOM:
-		    if (hash == 0)
+		    if (hash == salt)
 			/* Fast, but the poor hash value should be mixed. */
-			hash = atom_tab(atom_val(term))->slot.bucket.hvalue;
+			hash = atom_tab(atom_val(term))->slot.bucket.hvalue ^ salt;
 		    else
 			UINT32_HASH(atom_tab(atom_val(term))->slot.bucket.hvalue,
 				    HCONST_3);
 		    goto hash2_common;
 		case _TAG_IMMED2_NIL:
-		    if (hash == 0)
-			hash = 3468870702UL;
+		    if (hash == salt)
+			hash = 3468870702UL ^ salt;
 		    else
 			UINT32_HASH(NIL_DEF, HCONST_2);
 		    goto hash2_common;
 		default:
-		    erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
+		    erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X, %lu)\n", term, salt);
 		}
 	    case _TAG_IMMED1_SMALL:
 	      {
@@ -1518,7 +1518,7 @@ make_hash2(Eterm term)
 	    }
 	    break;
 	default:
-	    erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
+	    erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X, %lu)\n", term, salt);
 	hash2_common:
 
 	    /* Uint32 hash always has the hash value of the previous term,
@@ -1542,7 +1542,7 @@ make_hash2(Eterm term)
 		}
 		case HASH_MAP_PAIR:
 		    hash_xor_pairs ^= hash;
-                    hash = 0;
+                    hash = salt;
 		    goto hash2_common;
 		default:
 		    break;
@@ -1578,7 +1578,7 @@ do {  /* Lightweight mixing of constant (type info) */  \
 } while (0)
 
 Uint32
-make_internal_hash(Eterm term)
+make_internal_hash(Eterm term, Uint32 salt)
 {
     Uint32 hash;
     Uint32 hash_xor_pairs;
@@ -1587,7 +1587,7 @@ make_internal_hash(Eterm term)
 
     /* Optimization. Simple cases before declaration of estack. */
     if (primary_tag(term) == TAG_PRIMARY_IMMED1) {
-        hash = 0;
+        hash = salt;
     #if ERTS_SIZEOF_ETERM == 8
         UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST);
     #elif ERTS_SIZEOF_ETERM == 4
@@ -1601,7 +1601,7 @@ make_internal_hash(Eterm term)
     Eterm tmp;
     DECLARE_ESTACK(s);
 
-    hash = 0;
+    hash = salt;
     for (;;) {
 	switch (primary_tag(term)) {
 	case TAG_PRIMARY_LIST:
@@ -1894,7 +1894,7 @@ make_internal_hash(Eterm term)
 		goto pop_next;
 	    }
 	    default:
-		erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
+		erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_internal_hash(0x%X, %lu)\n", term, salt);
 	    }
 	}
 	break;
@@ -1907,7 +1907,7 @@ make_internal_hash(Eterm term)
             goto pop_next;
 
 	default:
-	    erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
+	    erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_internal_hash(0x%X, %lu)\n", term, salt);
 
 	pop_next:
 	    if (ESTACK_ISEMPTY(s)) {
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 8b420b9e9b..b7297043e5 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -496,7 +496,7 @@ static ErlOffHeap const_term_table_off_heap;
 
 static HashValue const_term_hash(void *tmpl)
 {
-    return make_hash2((Eterm)tmpl);
+    return make_hash2((Eterm)tmpl, 0);
 }
 
 static int const_term_cmp(void *tmpl, void *bucket)
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index e177fa4963..36af68be60 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -2628,7 +2628,7 @@ nif_internal_hash(Config) ->
     OnesPerBit =
         lists:foldl(
           fun (Term, Acc) ->
-                  NifHashValue = hash_nif(internal, Term),
+                  NifHashValue = hash_nif(internal, Term, 0),
                   lists:foldl(
                     fun (BitIndex, AccB) ->
                             BitValue = (NifHashValue band (1 bsl BitIndex)) bsr BitIndex,
@@ -2661,7 +2661,7 @@ nif_phash2(Config) ->
     lists:foreach(
       fun (Term) ->
               HashValue = erlang:phash2(Term),
-              NifHashValue = hash_nif(phash2, Term),
+              NifHashValue = hash_nif(phash2, Term, 0),
               (HashValue =:= NifHashValue
                orelse ct:fail("Expected: ~p\nActual:   ~p",
                               [HashValue, NifHashValue]))
@@ -2714,7 +2714,7 @@ type_test() -> ?nif_stub.
 tuple_2_list(_) -> ?nif_stub.    
 is_identical(_,_) -> ?nif_stub.
 compare(_,_) -> ?nif_stub.
-hash_nif(_, _) -> ?nif_stub.
+hash_nif(_Type, _Term, _Salt) -> ?nif_stub.
 many_args_100(_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_) -> ?nif_stub.
 clone_bin(_) -> ?nif_stub.
 make_sub_bin(_,_,_) -> ?nif_stub.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 9d65b9c09b..2a80d48b62 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -689,7 +689,7 @@ static ERL_NIF_TERM compare(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 
 static ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    if (argc != 2) {
+    if (argc != 3) {
         return enif_make_badarg(env);
     }
 
@@ -704,7 +704,12 @@ static ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]
         return enif_make_badarg(env);
     }
 
-    return enif_make_ulong(env, enif_hash(type, argv[1]));
+    unsigned long salt;
+    if (! enif_get_ulong(env, argv[2], &salt)) {
+        return enif_make_badarg(env);
+    }
+
+    return enif_make_ulong(env, enif_hash(type, argv[1], salt));
 }
 
 static ERL_NIF_TERM many_args_100(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
@@ -2884,7 +2889,7 @@ static ErlNifFunc nif_funcs[] =
     {"tuple_2_list", 1, tuple_2_list},
     {"is_identical",2,is_identical},
     {"compare",2,compare},
-    {"hash_nif",2,hash_nif},
+    {"hash_nif",3,hash_nif},
     {"many_args_100", 100, many_args_100},
     {"clone_bin", 1, clone_bin},
     {"make_sub_bin", 3, make_sub_bin},
-- 
cgit v1.2.3


From c94506ccc003016db49fa2dbe5741196e071d45a Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Sat, 22 Apr 2017 18:40:26 +0100
Subject: erts: Add test cases for salted enif_hash calls

---
 erts/emulator/test/nif_SUITE.erl | 148 ++++++++++++++++++++++++++++-----------
 1 file changed, 107 insertions(+), 41 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 36af68be60..6e4866bfd8 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -58,7 +58,9 @@
          nif_port_command/1,
          nif_snprintf/1,
          nif_internal_hash/1,
-         nif_phash2/1
+         nif_internal_hash_salted/1,
+         nif_phash2/1,
+         nif_phash2_salted/1
 	]).
 
 -export([many_args_100/100]).
@@ -94,7 +96,9 @@ all() ->
      nif_port_command,
      nif_snprintf,
      nif_internal_hash,
-     nif_phash2].
+     nif_internal_hash_salted,
+     nif_phash2,
+     nif_phash2_salted].
 
 groups() ->
     [{G, [], api_repeaters()} || G <- api_groups()]
@@ -2616,57 +2620,119 @@ nif_snprintf(Config) ->
 
 nif_internal_hash(Config) ->
     ensure_lib_loaded(Config),
-    Terms =
-        [random_term() || _ <- lists:seq(1, 5000)],
+    HashValueBitSize = nif_hash_result_bitsize(internal),
+    Terms = unique([random_term() || _ <- lists:seq(1, 5000)]),
+    HashValues = [hash_nif(internal, Term, 0) || Term <- Terms],
+    test_bit_distribution_fitness(HashValues, HashValueBitSize, 0.05).
 
-    % Unlike the phash2 hash, in this case we
-    % have nothing to compare to, so let's try
-    % and at least make sure the distribution
-    % isn't outright wrong, statistical nuances
-    % aside.
+nif_internal_hash_salted(Config) ->
+    ensure_lib_loaded(Config),
+    test_salted_nif_hash(internal).
+
+nif_phash2(Config) ->
+    ensure_lib_loaded(Config),
+    HashValueBitSize = nif_hash_result_bitsize(phash2),
+    Terms = unique([random_term() || _ <- lists:seq(1, 5000)]),
+    HashValues =
+        lists:map(
+          fun (Term) ->
+                  HashValue = erlang:phash2(Term),
+                  NifHashValue = hash_nif(phash2, Term, 0),
+                  (HashValue =:= NifHashValue
+                   orelse ct:fail("Expected: ~p\nActual:   ~p",
+                                  [HashValue, NifHashValue])),
+                  HashValue
+          end,
+          Terms),
+    test_bit_distribution_fitness(HashValues, HashValueBitSize, 0.05).
+
+nif_phash2_salted(Config) ->
+    ensure_lib_loaded(Config),
+    test_salted_nif_hash(phash2).
+
+test_salted_nif_hash(HashType) ->
+    HashValueBitSize = nif_hash_result_bitsize(HashType),
+    Terms = unique([random_term() || _ <- lists:seq(1, 5000)]),
+    Salts = unique([random_uint32() || _ <- lists:seq(1, 100)]),
+    {HashValuesPerSalt, HashValuesPerTerm} =
+        lists:mapfoldl(
+          fun (Salt, Acc) ->
+                  {HashValues, NewAcc} =
+                    lists:mapfoldl(
+                      fun (Term, AccB) ->
+                              HashValue = hash_nif(HashType, Term, Salt),
+                              NewAccB = dict:append(Term, HashValue, AccB),
+                              {HashValue, NewAccB}
+                      end,
+                      Acc,
+                      Terms),
+                  {{Salt, HashValues}, NewAcc}
+          end,
+          dict:new(),
+          Salts),
+
+    % Test per-salt hash distribution of different terms
+    lists:foreach(
+      fun ({_Salt, HashValues}) ->
+              test_bit_distribution_fitness(HashValues, HashValueBitSize, 0.05)
+      end,
+      HashValuesPerSalt),
 
+    % Test per-term hash distribution of different salts
+    dict:fold(
+      fun (_Term, HashValues, Acc) ->
+              % Be more tolerant of relative deviation,
+              % as there's fewer hash values here.
+              test_bit_distribution_fitness(HashValues, HashValueBitSize, 0.30),
+              Acc
+      end,
+      ok,
+      HashValuesPerTerm).
+
+test_bit_distribution_fitness(Integers, BitSize, MaxRelativeDeviation) ->
+    MaxInteger = (1 bsl BitSize) - 1,
     OnesPerBit =
         lists:foldl(
-          fun (Term, Acc) ->
-                  NifHashValue = hash_nif(internal, Term, 0),
+          fun (Integer, Acc) when Integer >= 0, Integer =< MaxInteger ->
                   lists:foldl(
                     fun (BitIndex, AccB) ->
-                            BitValue = (NifHashValue band (1 bsl BitIndex)) bsr BitIndex,
-                            dict:update_counter(BitIndex, BitValue, AccB)
+                            BitValue = (Integer band (1 bsl BitIndex)) bsr BitIndex,
+                            orddict:update_counter(BitIndex, BitValue, AccB)
                     end,
                     Acc,
-                    lists:seq(0, 31))
+                    lists:seq(0, BitSize - 1))
           end,
-          dict:new(),
-          Terms),
+          orddict:new(),
+          Integers),
+
+    ExpectedNrOfOnes = length(Integers) div 2,
+    FailureText =
+        orddict:fold(
+          fun (BitIndex, NrOfOnes, Acc) ->
+                  RelativeDeviation = abs(NrOfOnes - ExpectedNrOfOnes) / length(Integers),
+                  case RelativeDeviation >= MaxRelativeDeviation of
+                      false -> Acc;
+                      true ->
+                          [Acc,
+                           io_lib:format(
+                             "Unreasonable deviation on number of set bits (i=~p): "
+                             "expected ~p, got ~p (relative dev. ~.3f)~n",
+                             [BitIndex, ExpectedNrOfOnes, NrOfOnes, RelativeDeviation])]
+                  end
+          end,
+          [],
+          OnesPerBit),
 
-    ExpectedNrOfOnes = length(Terms) div 2,
-    dict:fold(
-      fun (BitIndex, NrOfOnes, Acc) ->
-              RelativeDeviation = abs(NrOfOnes - ExpectedNrOfOnes) / ExpectedNrOfOnes,
-              (RelativeDeviation < 0.10
-               orelse ct:fail("Unreasonable deviation on number of set bits (i=~p): "
-                              "expected ~p, got ~p (relative dev. ~.3f)",
-                              [BitIndex, ExpectedNrOfOnes, NrOfOnes, RelativeDeviation])),
-              Acc
-      end,
-      ok,
-      OnesPerBit).
+    (FailureText =:= [] orelse ct:fail(FailureText)).
 
-nif_phash2(Config) ->
-    ensure_lib_loaded(Config),
-    Terms =
-        [random_term() || _ <- lists:seq(1, 5000)],
+nif_hash_result_bitsize(internal) -> 32;
+nif_hash_result_bitsize(phash2) -> 27.
 
-    lists:foreach(
-      fun (Term) ->
-              HashValue = erlang:phash2(Term),
-              NifHashValue = hash_nif(phash2, Term, 0),
-              (HashValue =:= NifHashValue
-               orelse ct:fail("Expected: ~p\nActual:   ~p",
-                              [HashValue, NifHashValue]))
-      end,
-      Terms).
+unique(List) ->
+    lists:usort(List).
+
+random_uint32() ->
+    rand:uniform(1 bsl 32) - 1.
 
 random_term() ->
     case rand:uniform(6) of
-- 
cgit v1.2.3


From da9abd24a93ae8fe174cdd38fc9699bbc45fdf56 Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Sat, 22 Apr 2017 20:00:28 +0100
Subject: erts: Allow for easier future enif_hash expansion

Allow for expanding support to 64-bit hashes without breaking the
interface.
---
 erts/emulator/beam/erl_nif.c                  | 10 +++-------
 erts/emulator/beam/erl_nif_api_funcs.h        |  2 +-
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c |  6 +++---
 3 files changed, 7 insertions(+), 11 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index bdfb7f3b8e..3ff6c8d3ed 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1214,17 +1214,13 @@ int enif_compare(Eterm lhs, Eterm rhs)
     return result;
 }
 
-unsigned long enif_hash(ErlNifHash type, Eterm term, unsigned long salt)
+ErlNifUInt64 enif_hash(ErlNifHash type, Eterm term, ErlNifUInt64 salt)
 {
-#if SIZEOF_LONG < 4
-/* This *really* shouldn't happen */
-#  error Incompatible long word size
-#endif
     switch (type) {
         case ERL_NIF_INTERNAL_HASH:
-            return make_internal_hash(term, salt);
+            return make_internal_hash(term, (Uint32) salt);
         case ERL_NIF_PHASH2:
-            return make_hash2(term, salt) & ((1 << 27) - 1);
+            return make_hash2(term, (Uint32) salt) & ((1 << 27) - 1);
         default:
             return 0;
     }
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 6b9f78d46b..1a91a04f32 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -47,7 +47,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term,
 ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array));
 ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
 ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
-ERL_NIF_API_FUNC_DECL(unsigned long,enif_hash,(ErlNifHash type, ERL_NIF_TERM term, unsigned long salt));
+ERL_NIF_API_FUNC_DECL(ErlNifUInt64,enif_hash,(ErlNifHash type, ERL_NIF_TERM term, ErlNifUInt64 salt));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i));
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 2a80d48b62..a255c9f096 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -704,12 +704,12 @@ static ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]
         return enif_make_badarg(env);
     }
 
-    unsigned long salt;
-    if (! enif_get_ulong(env, argv[2], &salt)) {
+    ErlNifUInt64 salt;
+    if (! enif_get_uint64(env, argv[2], &salt)) {
         return enif_make_badarg(env);
     }
 
-    return enif_make_ulong(env, enif_hash(type, argv[1], salt));
+    return enif_make_uint64(env, enif_hash(type, argv[1], salt));
 }
 
 static ERL_NIF_TERM many_args_100(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
-- 
cgit v1.2.3


From a9d0d119837fb0bc52d2bb3d48a47568de9100b4 Mon Sep 17 00:00:00 2001
From: Dan Gudmundsson <dgud@erlang.org>
Date: Tue, 28 Feb 2017 15:23:11 +0100
Subject: Handle chardata in string:to_float and string:to_list

---
 erts/emulator/beam/bif.c              | 4 ++--
 erts/emulator/beam/bif.tab            | 4 ++--
 erts/emulator/test/list_bif_SUITE.erl | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 214de3652f..075909a881 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3109,7 +3109,7 @@ BIF_RETTYPE integer_to_list_1(BIF_ALIST_1)
  * On error returns: {error,not_a_list}, or {error, no_integer}
  */
 
-BIF_RETTYPE string_to_integer_1(BIF_ALIST_1)
+BIF_RETTYPE string_list_to_integer_1(BIF_ALIST_1)
 {
      Eterm res;
      Eterm tail;
@@ -3295,7 +3295,7 @@ BIF_RETTYPE float_to_binary_2(BIF_ALIST_2)
 #define LOAD_E(xi,xim,xl,xlm) ((xi)=(xim), (xl)=(xlm))
 
 #define STRING_TO_FLOAT_BUF_INC_SZ (128)
-BIF_RETTYPE string_to_float_1(BIF_ALIST_1)
+BIF_RETTYPE string_list_to_float_1(BIF_ALIST_1)
 {
     Eterm orig = BIF_ARG_1;
     Eterm list = orig;
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 66e5dc2988..5aee795088 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -460,8 +460,8 @@ bif error_logger:warning_map/0
 bif erlang:get_module_info/1
 bif erlang:get_module_info/2
 ubif erlang:is_boolean/1
-bif string:to_integer/1
-bif string:to_float/1
+bif string:list_to_integer/1
+bif string:list_to_float/1
 bif erlang:make_fun/3
 bif erlang:iolist_size/1
 bif erlang:iolist_to_binary/1
diff --git a/erts/emulator/test/list_bif_SUITE.erl b/erts/emulator/test/list_bif_SUITE.erl
index 514dd2f412..9c40929926 100644
--- a/erts/emulator/test/list_bif_SUITE.erl
+++ b/erts/emulator/test/list_bif_SUITE.erl
@@ -47,9 +47,9 @@ t_list_to_integer(Config) when is_list(Config) ->
     {12373281903728109372810937209817320981321,"ABC"} = string:to_integer("12373281903728109372810937209817320981321ABC"),
     {-12373281903728109372810937209817320981321,"ABC"} = string:to_integer("-12373281903728109372810937209817320981321ABC"),
     {12,[345]} = string:to_integer([$1,$2,345]),
-    {12,[a]} = string:to_integer([$1,$2,a]),
+    {error, badarg} = string:to_integer([$1,$2,a]),
     {error,no_integer} = string:to_integer([$A]),
-    {error,not_a_list} = string:to_integer($A),
+    {error,badarg} = string:to_integer($A),
     ok.
 
 %% Test hd/1 with correct and incorrect arguments.
-- 
cgit v1.2.3


From bf2e5951dbd6398b71e32fd1cfb72b2870280280 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 24 Apr 2017 17:18:53 +0200
Subject: Fix assert in dbg_verify_empty_later_slots()

---
 erts/emulator/beam/time.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index cee3cb619f..a7e5a64b22 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -1364,21 +1364,29 @@ dbg_verify_empty_later_slots(ErtsTimerWheel *tiw, ErtsMonotonicTime to_pos)
     tmp = to_pos;
     tmp &= ERTS_TW_LATER_WHEEL_POS_MASK;
     if (tmp > tiw->later.pos) {
+        ErtsMonotonicTime pos_min;
         int slots;
         tmp -= tiw->later.pos;
         tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
         ERTS_TW_ASSERT(tmp > 0);
+
+        pos_min = tiw->later.pos;
+
         if (tmp < (ErtsMonotonicTime) ERTS_TW_LATER_WHEEL_SIZE)
             slots = (int) tmp;
-        else
+        else {
+            pos_min += ((tmp / ERTS_TW_LATER_WHEEL_SIZE)
+                        * ERTS_TW_LATER_WHEEL_SLOT_SIZE);
             slots = ERTS_TW_LATER_WHEEL_SIZE;
+        }
 
         while (slots > 0) {
             ErtsTWheelTimer *tmr = tiw->w[ix];
+            pos_min += ERTS_TW_LATER_WHEEL_SLOT_SIZE;
             if (tmr) {
                 ErtsTWheelTimer *end = tmr;
                 do {
-                    ERTS_TW_ASSERT(tmr->timeout_pos > to_pos);
+                    ERTS_TW_ASSERT(tmr->timeout_pos >= pos_min);
                     tmr = tmr->next;
                 } while (tmr != end);
             }
-- 
cgit v1.2.3


From 3d6b8e7fb68543713f1620a45b8a590ef4ed88a5 Mon Sep 17 00:00:00 2001
From: Guilherme Andrade <g@gandrade.net>
Date: Mon, 24 Apr 2017 22:51:49 +0100
Subject: erts: Discontinue salted use of enif_hash/phash2

---
 erts/emulator/beam/bif.c         |  4 ++--
 erts/emulator/beam/erl_nif.c     |  6 +++++-
 erts/emulator/beam/erl_utils.h   |  2 +-
 erts/emulator/beam/utils.c       | 30 +++++++++++++++---------------
 erts/emulator/hipe/hipe_bif0.c   |  2 +-
 erts/emulator/test/nif_SUITE.erl | 13 ++++---------
 6 files changed, 28 insertions(+), 29 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 7473a8e43f..214de3652f 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4881,7 +4881,7 @@ BIF_RETTYPE phash2_1(BIF_ALIST_1)
 {
     Uint32 hash;
 
-    hash = make_hash2(BIF_ARG_1, 0);
+    hash = make_hash2(BIF_ARG_1);
     BIF_RET(make_small(hash & ((1L << 27) - 1)));
 }
 
@@ -4901,7 +4901,7 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2)
 	}
 	range = (Uint32) u;
     }
-    hash = make_hash2(BIF_ARG_1, 0);
+    hash = make_hash2(BIF_ARG_1);
     if (range) {
 	final_hash = hash % range; /* [0..range-1] */
     } else {
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 3ff6c8d3ed..e101747011 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1220,7 +1220,11 @@ ErlNifUInt64 enif_hash(ErlNifHash type, Eterm term, ErlNifUInt64 salt)
         case ERL_NIF_INTERNAL_HASH:
             return make_internal_hash(term, (Uint32) salt);
         case ERL_NIF_PHASH2:
-            return make_hash2(term, (Uint32) salt) & ((1 << 27) - 1);
+            /* It appears that make_hash2 doesn't always react to seasoning
+             * as well as it should. Therefore, let's make it ignore the salt
+             * value and declare salted uses of phash2 as unsupported.
+             */
+            return make_hash2(term) & ((1 << 27) - 1);
         default:
             return 0;
     }
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index b38beaf93d..75d7e47239 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -118,7 +118,7 @@ int erts_fit_in_bits_uint(Uint);
 Sint erts_list_length(Eterm);
 int erts_is_builtin(Eterm, Eterm, int);
 Uint32 block_hash(byte *, unsigned, Uint32);
-Uint32 make_hash2(Eterm, Uint32 salt);
+Uint32 make_hash2(Eterm);
 Uint32 make_hash(Eterm);
 Uint32 make_internal_hash(Eterm, Uint32 salt);
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 2378c057dd..9d140470ac 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1003,7 +1003,7 @@ tail_recur:
 	    break;
 	}
     case MAP_DEF:
-        hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term, 0);
+        hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term);
         break;
     case TUPLE_DEF:
 	{
@@ -1109,7 +1109,7 @@ block_hash(byte *k, unsigned length, Uint32 initval)
 }
 
 Uint32
-make_hash2(Eterm term, Uint32 salt)
+make_hash2(Eterm term)
 {
     Uint32 hash;
     Uint32 hash_xor_pairs;
@@ -1179,7 +1179,7 @@ make_hash2(Eterm term, Uint32 salt)
 	    switch (term & _TAG_IMMED2_MASK) {
 	    case _TAG_IMMED2_ATOM:
 		/* Fast, but the poor hash value should be mixed. */
-		return atom_tab(atom_val(term))->slot.bucket.hvalue ^ salt;
+		return atom_tab(atom_val(term))->slot.bucket.hvalue;
 	    }
 	    break;
 	case _TAG_IMMED1_SMALL:
@@ -1190,7 +1190,7 @@ make_hash2(Eterm term, Uint32 salt)
 		  term = small_to_big(x, tmp_big);
 		  break;
 	      }
-	      hash = salt;
+	      hash = 0;
 	      SINT32_HASH(x, HCONST);
 	      return hash;
 	  }
@@ -1201,7 +1201,7 @@ make_hash2(Eterm term, Uint32 salt)
     DECLARE_ESTACK(s);
 
     UseTmpHeapNoproc(2);
-    hash = salt;
+    hash = 0;
     for (;;) {
 	switch (primary_tag(term)) {
 	case TAG_PRIMARY_LIST:
@@ -1277,7 +1277,7 @@ make_hash2(Eterm term, Uint32 salt)
                     ESTACK_PUSH(s, hash_xor_pairs);
                     ESTACK_PUSH(s, hash);
                     ESTACK_PUSH(s, HASH_MAP_TAIL);
-                    hash = salt;
+                    hash = 0;
                     hash_xor_pairs = 0;
                     for (i = size - 1; i >= 0; i--) {
                         ESTACK_PUSH(s, HASH_MAP_PAIR);
@@ -1296,7 +1296,7 @@ make_hash2(Eterm term, Uint32 salt)
                     ESTACK_PUSH(s, hash_xor_pairs);
                     ESTACK_PUSH(s, hash);
                     ESTACK_PUSH(s, HASH_MAP_TAIL);
-                    hash = salt;
+                    hash = 0;
                     hash_xor_pairs = 0;
                 }
                 switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
@@ -1471,7 +1471,7 @@ make_hash2(Eterm term, Uint32 salt)
 	    break;
 
 	    default:
-		erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X, %lu)\n", term, salt);
+		erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
 	    }
 	}
 	break;
@@ -1488,21 +1488,21 @@ make_hash2(Eterm term, Uint32 salt)
 	    case _TAG_IMMED1_IMMED2:
 		switch (term & _TAG_IMMED2_MASK) {
 		case _TAG_IMMED2_ATOM:
-		    if (hash == salt)
+		    if (hash == 0)
 			/* Fast, but the poor hash value should be mixed. */
-			hash = atom_tab(atom_val(term))->slot.bucket.hvalue ^ salt;
+			hash = atom_tab(atom_val(term))->slot.bucket.hvalue;
 		    else
 			UINT32_HASH(atom_tab(atom_val(term))->slot.bucket.hvalue,
 				    HCONST_3);
 		    goto hash2_common;
 		case _TAG_IMMED2_NIL:
-		    if (hash == salt)
-			hash = 3468870702UL ^ salt;
+		    if (hash == 0)
+			hash = 3468870702UL;
 		    else
 			UINT32_HASH(NIL_DEF, HCONST_2);
 		    goto hash2_common;
 		default:
-		    erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X, %lu)\n", term, salt);
+		    erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
 		}
 	    case _TAG_IMMED1_SMALL:
 	      {
@@ -1518,7 +1518,7 @@ make_hash2(Eterm term, Uint32 salt)
 	    }
 	    break;
 	default:
-	    erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X, %lu)\n", term, salt);
+	    erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
 	hash2_common:
 
 	    /* Uint32 hash always has the hash value of the previous term,
@@ -1542,7 +1542,7 @@ make_hash2(Eterm term, Uint32 salt)
 		}
 		case HASH_MAP_PAIR:
 		    hash_xor_pairs ^= hash;
-                    hash = salt;
+                    hash = 0;
 		    goto hash2_common;
 		default:
 		    break;
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index b7297043e5..8b420b9e9b 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -496,7 +496,7 @@ static ErlOffHeap const_term_table_off_heap;
 
 static HashValue const_term_hash(void *tmpl)
 {
-    return make_hash2((Eterm)tmpl, 0);
+    return make_hash2((Eterm)tmpl);
 }
 
 static int const_term_cmp(void *tmpl, void *bucket)
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 6e4866bfd8..8ad11d3bf3 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -59,8 +59,7 @@
          nif_snprintf/1,
          nif_internal_hash/1,
          nif_internal_hash_salted/1,
-         nif_phash2/1,
-         nif_phash2_salted/1
+         nif_phash2/1
 	]).
 
 -export([many_args_100/100]).
@@ -97,8 +96,7 @@ all() ->
      nif_snprintf,
      nif_internal_hash,
      nif_internal_hash_salted,
-     nif_phash2,
-     nif_phash2_salted].
+     nif_phash2].
 
 groups() ->
     [{G, [], api_repeaters()} || G <- api_groups()]
@@ -2637,7 +2635,8 @@ nif_phash2(Config) ->
         lists:map(
           fun (Term) ->
                   HashValue = erlang:phash2(Term),
-                  NifHashValue = hash_nif(phash2, Term, 0),
+                  Salt = random_uint32(), % phash2 should ignore salt
+                  NifHashValue = hash_nif(phash2, Term, Salt),
                   (HashValue =:= NifHashValue
                    orelse ct:fail("Expected: ~p\nActual:   ~p",
                                   [HashValue, NifHashValue])),
@@ -2646,10 +2645,6 @@ nif_phash2(Config) ->
           Terms),
     test_bit_distribution_fitness(HashValues, HashValueBitSize, 0.05).
 
-nif_phash2_salted(Config) ->
-    ensure_lib_loaded(Config),
-    test_salted_nif_hash(phash2).
-
 test_salted_nif_hash(HashType) ->
     HashValueBitSize = nif_hash_result_bitsize(HashType),
     Terms = unique([random_term() || _ <- lists:seq(1, 5000)]),
-- 
cgit v1.2.3


From eec41eeed17682a4e1f8ca15d92a2b469f0c39b8 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 21 Apr 2017 14:54:33 +0200
Subject: erts: Add erlang:list_to_port/1 debug bif

---
 erts/emulator/beam/bif.c              | 69 +++++++++++++++++++++++++++++++++++
 erts/emulator/beam/bif.tab            |  1 +
 erts/emulator/test/list_bif_SUITE.erl | 21 +++++++++--
 3 files changed, 88 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 214de3652f..d7a18e8d88 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4254,6 +4254,75 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1)
     BIF_ERROR(BIF_P, BADARG);
 }
 
+BIF_RETTYPE list_to_port_1(BIF_ALIST_1)
+{
+    /*
+     * A valid port identifier is on the format
+     * "#Port<N.P>" where N is node and P is
+     * the port id. Both N and P are of type Uint32.
+     */
+    Uint32 n, p;
+    char* cp;
+    int i;
+    DistEntry *dep = NULL;
+    char buf[6 /* #Port< */
+             + (2)*(10 + 1) /* N.P> */
+             + 1 /* \0 */];
+
+    /* walk down the list and create a C string */
+    if ((i = intlist_to_buf(BIF_ARG_1, buf, sizeof(buf)-1)) < 0)
+	goto bad;
+
+    buf[i] = '\0';		/* null terminal */
+
+    cp = &buf[0];
+    if (strncmp("#Port<", cp, 6) != 0)
+        goto bad;
+
+    cp += 6; /* strlen("#Port<") */
+
+    if (sscanf(cp, "%u.%u>", &n, &p) < 2)
+        goto bad;
+
+    if (p > ERTS_MAX_PORT_NUMBER)
+	goto bad;
+
+    dep = erts_channel_no_to_dist_entry(n);
+
+    if (!dep)
+	goto bad;
+
+    if(dep == erts_this_dist_entry) {
+	erts_deref_dist_entry(dep);
+	BIF_RET(make_internal_port(p));
+    }
+    else {
+      ExternalThing *etp;
+      ErlNode *enp;
+
+      if (is_nil(dep->cid))
+	  goto bad;
+
+      enp = erts_find_or_insert_node(dep->sysname, dep->creation);
+      ASSERT(enp != erts_this_node);
+
+      etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1);
+      etp->header = make_external_port_header(1);
+      etp->next = MSO(BIF_P).first;
+      etp->node = enp;
+      etp->data.ui[0] = p;
+
+      MSO(BIF_P).first = (struct erl_off_heap_header*) etp;
+      erts_deref_dist_entry(dep);
+      BIF_RET(make_external_port(etp));
+    }
+
+ bad:
+    if (dep)
+	erts_deref_dist_entry(dep);
+    BIF_ERROR(BIF_P, BADARG);
+}
+
 BIF_RETTYPE list_to_ref_1(BIF_ALIST_1)
 {
     /*
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 66e5dc2988..56d656a173 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -88,6 +88,7 @@ bif erlang:list_to_binary/1
 bif erlang:list_to_float/1
 bif erlang:list_to_integer/1
 bif erlang:list_to_pid/1
+bif erlang:list_to_port/1
 bif erlang:list_to_ref/1
 bif erlang:list_to_tuple/1
 bif erlang:loaded/0
diff --git a/erts/emulator/test/list_bif_SUITE.erl b/erts/emulator/test/list_bif_SUITE.erl
index 514dd2f412..9e1e351cd2 100644
--- a/erts/emulator/test/list_bif_SUITE.erl
+++ b/erts/emulator/test/list_bif_SUITE.erl
@@ -23,7 +23,7 @@
 
 -export([all/0, suite/0]).
 -export([hd_test/1,tl_test/1,t_length/1,t_list_to_pid/1,
-	 t_list_to_float/1,t_list_to_integer/1]).
+         t_list_to_port/1,t_list_to_float/1,t_list_to_integer/1]).
 
 
 suite() ->
@@ -32,7 +32,7 @@ suite() ->
 
 
 all() -> 
-    [hd_test, tl_test, t_length, t_list_to_pid,
+    [hd_test, tl_test, t_length, t_list_to_pid, t_list_to_port,
      t_list_to_float, t_list_to_integer].
 
 %% Tests list_to_integer and string:to_integer
@@ -106,10 +106,25 @@ t_list_to_pid(Config) when is_list(Config) ->
         {'EXIT', {badarg, _}} ->
             ok;
         Res ->
-            ct:fail("list_to_pid/1 with incorrect arg succeeded.~nResult: ~p", [Res])
+            ct:fail("list_to_pid/1 with incorrect arg succeeded.~n"
+                    "Result: ~p", [Res])
     end,
     ok.
 
+%% Test list_to_port/1 with correct and incorrect arguments.
+
+t_list_to_port(Config) when is_list(Config) ->
+    Me = hd(erlang:ports()),
+    MyListedPid = port_to_list(Me),
+    Me = list_to_port(MyListedPid),
+    case catch list_to_port(id("Incorrect list")) of
+        {'EXIT', {badarg, _}} ->
+            ok;
+        Res ->
+            ct:fail("list_to_port/1 with incorrect arg succeeded.~n"
+                    "Result: ~p", [Res])
+    end,
+    ok.
 
 %% Test list_to_float/1 with correct and incorrect arguments.
 
-- 
cgit v1.2.3


From 5f8a362c11a29c635131a7af4c3d7211da6c9bcf Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikpelinux@gmail.com>
Date: Fri, 21 Apr 2017 14:10:22 +0200
Subject: erl_mseg.c: don't use invalid indices in - > cache_powered_node[]

---
 erts/emulator/sys/common/erl_mseg.c | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index 882c93a83c..968f71211c 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -87,6 +87,7 @@ static const int debruijn[32] = {
 
 #define CACHE_AREAS      (32 - MSEG_ALIGN_BITS)
 
+/* FIXME: segment sizes > 2 GB result in bogus negative indices */
 #define SIZE_TO_CACHE_AREA_IDX(S)   (LOG2((S)) - MSEG_ALIGN_BITS)
 #define MAX_CACHE_SIZE   (30)
 
@@ -396,6 +397,9 @@ static ERTS_INLINE int cache_bless_segment(ErtsMsegAllctr_t *ma, void *seg, UWor
 	if (MSEG_FLG_IS_2POW(flags)) {
 	    int ix = SIZE_TO_CACHE_AREA_IDX(size);
 
+            if (ix < 0)
+                return 0;
+
 	    ASSERT(ix < CACHE_AREAS);
 	    ASSERT((1 << (ix + MSEG_ALIGN_BITS)) == size);
 
@@ -471,6 +475,9 @@ static ERTS_INLINE void *cache_get_segment(ErtsMsegAllctr_t *ma, UWord *size_p,
 
 	ASSERT(IS_2POW(size));
 
+        if (ix < 0)
+            return NULL;
+
 	for( i = ix; i < CACHE_AREAS; i++) {
 
 	    if (erts_circleq_is_empty(&(ma->cache_powered_node[i])))
-- 
cgit v1.2.3


From e40ec046a2a1037b2f87b657503c5f21c5de4e2a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 27 Apr 2017 12:18:57 +0200
Subject: Move enif_hash last to not break Windows

---
 erts/emulator/beam/erl_nif_api_funcs.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 1a91a04f32..b34058f303 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -47,7 +47,6 @@ ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term,
 ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array));
 ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
 ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
-ERL_NIF_API_FUNC_DECL(ErlNifUInt64,enif_hash,(ErlNifHash type, ERL_NIF_TERM term, ErlNifUInt64 salt));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i));
@@ -181,6 +180,7 @@ ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type_x,(ErlNifEnv*,
 ERL_NIF_API_FUNC_DECL(int, enif_monitor_process,(ErlNifEnv*,void* obj,const ErlNifPid*,ErlDrvMonitor *monitor));
 ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const ErlDrvMonitor *monitor));
 ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const ErlNifMonitor*));
+ERL_NIF_API_FUNC_DECL(ErlNifUInt64,enif_hash,(ErlNifHash type, ERL_NIF_TERM term, ErlNifUInt64 salt));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
@@ -209,7 +209,6 @@ ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const Erl
 #  define enif_get_list_cell ERL_NIF_API_FUNC_MACRO(enif_get_list_cell)
 #  define enif_is_identical ERL_NIF_API_FUNC_MACRO(enif_is_identical)
 #  define enif_compare ERL_NIF_API_FUNC_MACRO(enif_compare)
-#  define enif_hash ERL_NIF_API_FUNC_MACRO(enif_hash)
 
 #  define enif_make_binary ERL_NIF_API_FUNC_MACRO(enif_make_binary)
 #  define enif_make_badarg ERL_NIF_API_FUNC_MACRO(enif_make_badarg)
@@ -344,6 +343,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const Erl
 #  define enif_monitor_process ERL_NIF_API_FUNC_MACRO(enif_monitor_process)
 #  define enif_demonitor_process ERL_NIF_API_FUNC_MACRO(enif_demonitor_process)
 #  define enif_compare_monitors ERL_NIF_API_FUNC_MACRO(enif_compare_monitors)
+#  define enif_hash ERL_NIF_API_FUNC_MACRO(enif_hash)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
-- 
cgit v1.2.3


From 282a7cab9989abb7a1b2daca7aec907d53163ff4 Mon Sep 17 00:00:00 2001
From: Richard Carlsson <richardc@klarna.com>
Date: Thu, 27 Apr 2017 14:44:10 +0200
Subject: Rename macro and add clarifying comment

The macro previously named 'LOG2' is in fact computing the LSB (not the MSB)
of its input, and can only be used to compute log2(n) when n is is a power
of 2, that is, when its LSB is also its MSB.
---
 erts/emulator/sys/common/erl_mseg.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index d1895f3793..1e99078906 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -83,12 +83,13 @@ static const int debruijn[32] = {
     31, 27, 13, 23, 21, 19, 16,  7, 26, 12, 18,  6, 11,  5, 10,  9
 };
 
-#define LOG2(X) (debruijn[((Uint32)(((X) & -(X)) * 0x077CB531U)) >> 27])
+#define LSB(X) (debruijn[((Uint32)(((X) & -(X)) * 0x077CB531U)) >> 27])
 
 #define CACHE_AREAS      (32 - MSEG_ALIGN_BITS)
 
 /* FIXME: segment sizes > 2 GB result in bogus negative indices */
-#define SIZE_TO_CACHE_AREA_IDX(S)   (LOG2((S)) - MSEG_ALIGN_BITS)
+/* NOTE: using LSB instead of proper log2 only works if S is a power of 2 */
+#define SIZE_TO_CACHE_AREA_IDX(S)   (LSB((S)) - MSEG_ALIGN_BITS)
 #define MAX_CACHE_SIZE   (30)
 
 #define MSEG_FLG_IS_2POW(X)    ((X) & ERTS_MSEG_FLG_2POW)
-- 
cgit v1.2.3


From 3684673aab9006d114db4298ab882805bf9bf657 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 27 Apr 2017 19:27:36 +0200
Subject: erts: Optimize make_internal_hash for maps

No need for the xor-trick, to be order independent, as both
flatmaps and hashmaps have a constant key-value order internally.
---
 erts/emulator/beam/utils.c | 43 ++++++-------------------------------------
 1 file changed, 6 insertions(+), 37 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 9263798a28..cdf766b206 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1581,9 +1581,6 @@ Uint32
 make_internal_hash(Eterm term)
 {
     Uint32 hash;
-    Uint32 hash_xor_pairs;
-
-    ERTS_UNDEF(hash_xor_pairs, 0);
 
     /* Optimization. Simple cases before declaration of estack. */
     if (primary_tag(term) == TAG_PRIMARY_IMMED1) {
@@ -1664,6 +1661,11 @@ make_internal_hash(Eterm term)
                 Eterm* ptr = boxed_val(term) + 1;
                 Uint size;
                 int i;
+
+                /*
+                 * We rely on key-value iteration order being constant
+                 * for identical maps (in this VM instance).
+                 */
                 switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
                 case HAMT_SUBTAG_HEAD_FLATMAP:
                 {
@@ -1675,26 +1677,10 @@ make_internal_hash(Eterm term)
                     if (size == 0)
                         goto pop_next;
 
-                    /* We want a hash function that is *independent* of
-                     * the order in which keys and values are encountered.
-                     * We therefore calculate context independent hashes for all    				      .
-                     * key-value pairs and then xor them together.
-                     *
-                     * We *do* need to use an initial seed for each pair, i.e. the
-                     * hash value, so the hash value is reset for each pair with 'hash'.
-                     * If we don't, no additional entropy is given to the system and the
-                     * hash collision resolution in maps:from_list/1 would fail.
-                     */
-                    ESTACK_PUSH(s, hash_xor_pairs);
-                    ESTACK_PUSH(s, hash);
-                    ESTACK_PUSH(s, HASH_MAP_TAIL);
                     for (i = size - 1; i >= 0; i--) {
-                        ESTACK_PUSH(s, hash); /* initial seed for all pairs */
-                        ESTACK_PUSH(s, HASH_MAP_PAIR);
                         ESTACK_PUSH(s, vs[i]);
                         ESTACK_PUSH(s, ks[i]);
                     }
-                    hash_xor_pairs = 0;
                     goto pop_next;
                 }
                 case HAMT_SUBTAG_HEAD_ARRAY:
@@ -1703,10 +1689,6 @@ make_internal_hash(Eterm term)
                     UINT32_HASH(size, HCONST_16);
                     if (size == 0)
                         goto pop_next;
-                    ESTACK_PUSH(s, hash_xor_pairs);
-                    ESTACK_PUSH(s, hash);
-                    ESTACK_PUSH(s, HASH_MAP_TAIL);
-                    hash_xor_pairs = 0;
                 }
                 switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
                 case HAMT_SUBTAG_HEAD_ARRAY:
@@ -1722,8 +1704,6 @@ make_internal_hash(Eterm term)
                 while (i) {
                     if (is_list(*ptr)) {
                         Eterm* cons = list_val(*ptr);
-                        ESTACK_PUSH(s, hash); /* initial seed for all pairs */
-                        ESTACK_PUSH(s, HASH_MAP_PAIR);
                         ESTACK_PUSH(s, CDR(cons));
                         ESTACK_PUSH(s, CAR(cons));
                     }
@@ -1739,7 +1719,7 @@ make_internal_hash(Eterm term)
 	    case EXPORT_SUBTAG:
 	    {
 		Export* ep = *((Export **) (export_val(term) + 1));
-                /* Assumes Export entries never moves */
+                /* Assumes Export entries never move */
                 POINTER_HASH(ep, HCONST_14);
 		goto pop_next;
 	    }
@@ -1919,17 +1899,6 @@ make_internal_hash(Eterm term)
 	    term = ESTACK_POP(s);
 
 	    switch (term) {
-		case HASH_MAP_TAIL: {
-		    hash = (Uint32) ESTACK_POP(s);
-                    UINT32_HASH(hash_xor_pairs, HCONST_19);
-		    hash_xor_pairs = (Uint32) ESTACK_POP(s);
-		    goto pop_next;
-		}
-		case HASH_MAP_PAIR:
-		    hash_xor_pairs ^= hash;
-		    hash = (Uint32) ESTACK_POP(s); /* initial seed for all pairs */
-		    goto pop_next;
-
 	        case HASH_CDR:
 		    CONST_HASH(HCONST_18);   /* Hash CDR i cons cell */
 		    goto pop_next;
-- 
cgit v1.2.3


From 7ce5c7a0d1da593d7718131580b4ce2e73de48c6 Mon Sep 17 00:00:00 2001
From: Salikhov Dinislam <dinislam.salikhov@gmail.com>
Date: Sun, 30 Apr 2017 08:28:20 +0300
Subject: Make a function static

---
 erts/emulator/beam/erl_port.h | 10 ----------
 erts/emulator/beam/io.c       |  2 +-
 2 files changed, 1 insertion(+), 11 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index c59b42cdae..1c676c7c4f 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -988,16 +988,6 @@ typedef enum {
     ERTS_PORT_OP_DONE
 } ErtsPortOpResult;
 
-ErtsPortOpResult
-erts_schedule_proc2port_signal(Process *,
-			       Port *,
-			       Eterm,
-			       Eterm *,
-			       ErtsProc2PortSigData *,
-			       int,
-			       ErtsPortTaskHandle *,
-			       ErtsProc2PortSigCallback);
-
 int erts_deliver_port_exit(Port *, Eterm, Eterm, int, int);
 
 /*
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index c3eb610fdc..f672444836 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1511,7 +1511,7 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg, Port* prt)
 }
 
 
-ErtsPortOpResult
+static ErtsPortOpResult
 erts_schedule_proc2port_signal(Process *c_p,
 			       Port *prt,
 			       Eterm caller,
-- 
cgit v1.2.3


From 83e20c62057ebc1d8064bf57b01be560cd244e1d Mon Sep 17 00:00:00 2001
From: Raimo Niskanen <raimo@erlang.org>
Date: Thu, 4 May 2017 15:42:21 +0200
Subject: Update copyright year

---
 erts/emulator/Makefile.in                                         | 2 +-
 erts/emulator/beam/atom.names                                     | 2 +-
 erts/emulator/beam/beam_bif_load.c                                | 2 +-
 erts/emulator/beam/beam_bp.c                                      | 2 +-
 erts/emulator/beam/beam_bp.h                                      | 2 +-
 erts/emulator/beam/beam_debug.c                                   | 2 +-
 erts/emulator/beam/beam_emu.c                                     | 2 +-
 erts/emulator/beam/beam_load.c                                    | 2 +-
 erts/emulator/beam/bif.c                                          | 2 +-
 erts/emulator/beam/bif.tab                                        | 2 +-
 erts/emulator/beam/binary.c                                       | 2 +-
 erts/emulator/beam/break.c                                        | 2 +-
 erts/emulator/beam/code_ix.h                                      | 2 +-
 erts/emulator/beam/copy.c                                         | 2 +-
 erts/emulator/beam/dist.c                                         | 2 +-
 erts/emulator/beam/dist.h                                         | 2 +-
 erts/emulator/beam/dtrace-wrapper.h                               | 2 +-
 erts/emulator/beam/erl_alloc.c                                    | 2 +-
 erts/emulator/beam/erl_alloc.types                                | 2 +-
 erts/emulator/beam/erl_alloc_util.c                               | 2 +-
 erts/emulator/beam/erl_async.h                                    | 2 +-
 erts/emulator/beam/erl_bif_binary.c                               | 2 +-
 erts/emulator/beam/erl_bif_ddll.c                                 | 2 +-
 erts/emulator/beam/erl_bif_guard.c                                | 2 +-
 erts/emulator/beam/erl_bif_info.c                                 | 2 +-
 erts/emulator/beam/erl_bif_os.c                                   | 2 +-
 erts/emulator/beam/erl_bif_port.c                                 | 2 +-
 erts/emulator/beam/erl_bif_re.c                                   | 2 +-
 erts/emulator/beam/erl_bif_trace.c                                | 2 +-
 erts/emulator/beam/erl_bif_unique.c                               | 2 +-
 erts/emulator/beam/erl_bif_unique.h                               | 2 +-
 erts/emulator/beam/erl_binary.h                                   | 2 +-
 erts/emulator/beam/erl_bits.c                                     | 2 +-
 erts/emulator/beam/erl_bits.h                                     | 2 +-
 erts/emulator/beam/erl_db.c                                       | 2 +-
 erts/emulator/beam/erl_db_hash.c                                  | 2 +-
 erts/emulator/beam/erl_db_hash.h                                  | 2 +-
 erts/emulator/beam/erl_db_tree.c                                  | 2 +-
 erts/emulator/beam/erl_db_util.c                                  | 2 +-
 erts/emulator/beam/erl_db_util.h                                  | 2 +-
 erts/emulator/beam/erl_debug.c                                    | 2 +-
 erts/emulator/beam/erl_driver.h                                   | 2 +-
 erts/emulator/beam/erl_drv_nif.h                                  | 2 +-
 erts/emulator/beam/erl_fun.c                                      | 2 +-
 erts/emulator/beam/erl_fun.h                                      | 2 +-
 erts/emulator/beam/erl_gc.c                                       | 2 +-
 erts/emulator/beam/erl_gc.h                                       | 2 +-
 erts/emulator/beam/erl_hl_timer.c                                 | 2 +-
 erts/emulator/beam/erl_hl_timer.h                                 | 2 +-
 erts/emulator/beam/erl_init.c                                     | 2 +-
 erts/emulator/beam/erl_lock_check.c                               | 2 +-
 erts/emulator/beam/erl_lock_count.h                               | 2 +-
 erts/emulator/beam/erl_map.c                                      | 2 +-
 erts/emulator/beam/erl_map.h                                      | 2 +-
 erts/emulator/beam/erl_message.c                                  | 2 +-
 erts/emulator/beam/erl_monitors.c                                 | 2 +-
 erts/emulator/beam/erl_monitors.h                                 | 2 +-
 erts/emulator/beam/erl_msacc.c                                    | 2 +-
 erts/emulator/beam/erl_mtrace.c                                   | 2 +-
 erts/emulator/beam/erl_nif.c                                      | 2 +-
 erts/emulator/beam/erl_nif.h                                      | 2 +-
 erts/emulator/beam/erl_nif_api_funcs.h                            | 2 +-
 erts/emulator/beam/erl_node_container_utils.h                     | 2 +-
 erts/emulator/beam/erl_node_tables.c                              | 2 +-
 erts/emulator/beam/erl_node_tables.h                              | 2 +-
 erts/emulator/beam/erl_printf_term.c                              | 2 +-
 erts/emulator/beam/erl_process.c                                  | 2 +-
 erts/emulator/beam/erl_process.h                                  | 2 +-
 erts/emulator/beam/erl_process_dict.c                             | 2 +-
 erts/emulator/beam/erl_process_dump.c                             | 2 +-
 erts/emulator/beam/erl_process_lock.c                             | 2 +-
 erts/emulator/beam/erl_process_lock.h                             | 2 +-
 erts/emulator/beam/erl_ptab.c                                     | 2 +-
 erts/emulator/beam/erl_rbtree.h                                   | 2 +-
 erts/emulator/beam/erl_term.c                                     | 2 +-
 erts/emulator/beam/erl_term.h                                     | 2 +-
 erts/emulator/beam/erl_time.h                                     | 2 +-
 erts/emulator/beam/erl_time_sup.c                                 | 2 +-
 erts/emulator/beam/erl_trace.c                                    | 2 +-
 erts/emulator/beam/erl_unicode.c                                  | 2 +-
 erts/emulator/beam/erl_utils.h                                    | 2 +-
 erts/emulator/beam/erl_vm.h                                       | 2 +-
 erts/emulator/beam/export.c                                       | 2 +-
 erts/emulator/beam/external.c                                     | 2 +-
 erts/emulator/beam/global.h                                       | 2 +-
 erts/emulator/beam/index.c                                        | 2 +-
 erts/emulator/beam/index.h                                        | 2 +-
 erts/emulator/beam/io.c                                           | 2 +-
 erts/emulator/beam/sys.h                                          | 2 +-
 erts/emulator/beam/time.c                                         | 2 +-
 erts/emulator/beam/utils.c                                        | 2 +-
 erts/emulator/drivers/common/efile_drv.c                          | 2 +-
 erts/emulator/drivers/common/inet_drv.c                           | 2 +-
 erts/emulator/drivers/common/zlib_drv.c                           | 2 +-
 erts/emulator/drivers/unix/sig_drv.c                              | 2 +-
 erts/emulator/hipe/elf64ppc.x                                     | 2 +-
 erts/emulator/hipe/hipe_bif0.c                                    | 2 +-
 erts/emulator/hipe/hipe_bif0.h                                    | 2 +-
 erts/emulator/hipe/hipe_bif0.tab                                  | 2 +-
 erts/emulator/hipe/hipe_bif1.c                                    | 2 +-
 erts/emulator/hipe/hipe_gc.c                                      | 2 +-
 erts/emulator/hipe/hipe_load.c                                    | 2 +-
 erts/emulator/hipe/hipe_mode_switch.c                             | 2 +-
 erts/emulator/hipe/hipe_mode_switch.h                             | 2 +-
 erts/emulator/hipe/hipe_module.c                                  | 2 +-
 erts/emulator/hipe/hipe_native_bif.c                              | 2 +-
 erts/emulator/hipe/hipe_process.h                                 | 2 +-
 erts/emulator/hipe/hipe_stack.c                                   | 2 +-
 erts/emulator/hipe/hipe_x86_signal.c                              | 2 +-
 erts/emulator/sys/common/erl_check_io.c                           | 2 +-
 erts/emulator/sys/common/erl_check_io.h                           | 2 +-
 erts/emulator/sys/common/erl_mseg.c                               | 2 +-
 erts/emulator/sys/unix/erl_child_setup.h                          | 2 +-
 erts/emulator/sys/unix/erl_unix_sys.h                             | 2 +-
 erts/emulator/sys/unix/sys.c                                      | 2 +-
 erts/emulator/sys/unix/sys_drivers.c                              | 2 +-
 erts/emulator/sys/win32/erl_poll.c                                | 2 +-
 erts/emulator/sys/win32/sys.c                                     | 2 +-
 erts/emulator/test/Makefile                                       | 2 +-
 erts/emulator/test/binary_SUITE.erl                               | 2 +-
 erts/emulator/test/bs_match_int_SUITE.erl                         | 2 +-
 erts/emulator/test/call_trace_SUITE.erl                           | 2 +-
 erts/emulator/test/ddll_SUITE.erl                                 | 2 +-
 erts/emulator/test/dirty_bif_SUITE.erl                            | 2 +-
 erts/emulator/test/dirty_nif_SUITE.erl                            | 2 +-
 erts/emulator/test/distribution_SUITE.erl                         | 2 +-
 erts/emulator/test/erl_link_SUITE.erl                             | 2 +-
 erts/emulator/test/erts_debug_SUITE.erl                           | 2 +-
 erts/emulator/test/estone_SUITE.erl                               | 2 +-
 erts/emulator/test/float_SUITE.erl                                | 2 +-
 erts/emulator/test/guard_SUITE.erl                                | 2 +-
 erts/emulator/test/hibernate_SUITE.erl                            | 2 +-
 erts/emulator/test/list_bif_SUITE.erl                             | 2 +-
 erts/emulator/test/match_spec_SUITE.erl                           | 2 +-
 erts/emulator/test/nif_SUITE.erl                                  | 2 +-
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c                     | 2 +-
 erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_drv_nif.h       | 2 +-
 erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif.h           | 2 +-
 erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif_api_funcs.h | 2 +-
 erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_drv_nif.h       | 2 +-
 erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif.h           | 2 +-
 erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif_api_funcs.h | 2 +-
 erts/emulator/test/nif_SUITE_data/nif_mod.c                       | 2 +-
 erts/emulator/test/node_container_SUITE.erl                       | 2 +-
 erts/emulator/test/num_bif_SUITE.erl                              | 2 +-
 erts/emulator/test/port_SUITE.erl                                 | 2 +-
 erts/emulator/test/scheduler_SUITE.erl                            | 2 +-
 erts/emulator/test/smoke_test_SUITE.erl                           | 2 +-
 erts/emulator/test/statistics_SUITE.erl                           | 2 +-
 erts/emulator/test/system_info_SUITE.erl                          | 2 +-
 erts/emulator/test/time_SUITE.erl                                 | 2 +-
 erts/emulator/test/z_SUITE.erl                                    | 2 +-
 erts/emulator/utils/make_preload                                  | 2 +-
 153 files changed, 153 insertions(+), 153 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 227aabcf1e..254e59f2c7 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1996-2016. All Rights Reserved.
+# Copyright Ericsson AB 1996-2017. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index df2866b40e..477a7676d6 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1996-2016. All Rights Reserved.
+# Copyright Ericsson AB 1996-2017. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 769fe89219..5aceae8ffe 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 1efe7536d6..92ede09102 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index 3dba3cc1b5..56fa82b912 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 3f4a1f9e80..a2060c80de 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 3d36094e07..311bb94242 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 7b79e6303b..a0351746b9 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 84c9f9d7c4..7134d2da56 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 3f6d82d65c..a8bbf5f8c1 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1996-2016. All Rights Reserved.
+# Copyright Ericsson AB 1996-2017. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 0df6bbb289..ca3e48e205 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index e23fdb83d9..76a0c5c716 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h
index e802ad5dd7..a28b0cd36e 100644
--- a/erts/emulator/beam/code_ix.h
+++ b/erts/emulator/beam/code_ix.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 62c3f9520d..fefde256d7 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index a1581908e5..982f1066df 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 6ed36a478e..93a651b24d 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/dtrace-wrapper.h b/erts/emulator/beam/dtrace-wrapper.h
index f6fc28b801..15ea182976 100644
--- a/erts/emulator/beam/dtrace-wrapper.h
+++ b/erts/emulator/beam/dtrace-wrapper.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2016.
+ * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2017.
  * All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 71957b2259..b2c1133ded 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index f296a98125..8a23a1526e 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 2003-2016. All Rights Reserved.
+# Copyright Ericsson AB 2003-2017. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index a6bac06478..4889fac923 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_async.h b/erts/emulator/beam/erl_async.h
index c884a5040d..4b470e7679 100644
--- a/erts/emulator/beam/erl_async.h
+++ b/erts/emulator/beam/erl_async.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index f79b5b6843..756c7dce05 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2010-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 2ce872d2bc..e9bfb39035 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c
index cdac4c1d11..8a5c6ada6c 100644
--- a/erts/emulator/beam/erl_bif_guard.c
+++ b/erts/emulator/beam/erl_bif_guard.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2006-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 1a680b127c..5fc70dfc02 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c
index edc3c82b23..5cd172c26f 100644
--- a/erts/emulator/beam/erl_bif_os.c
+++ b/erts/emulator/beam/erl_bif_os.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 09d2c5376b..ff03151619 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index a72697eb80..ad124fd979 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 023bfca797..45159c4392 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c
index 7f438daec0..fc6fb5f868 100644
--- a/erts/emulator/beam/erl_bif_unique.c
+++ b/erts/emulator/beam/erl_bif_unique.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2014-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h
index 0f3f794878..9aa631fde9 100644
--- a/erts/emulator/beam/erl_bif_unique.h
+++ b/erts/emulator/beam/erl_bif_unique.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2014-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index de7dbf4e20..b036b28dbf 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index df3f6ad557..eea55b3239 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h
index 1b4546722f..632855255e 100644
--- a/erts/emulator/beam/erl_bits.h
+++ b/erts/emulator/beam/erl_bits.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 98c689f13f..17e0f2aeec 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 9009c00833..08a0f0e83b 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index c340c72311..f491c85d95 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1998-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index f6918b8ec4..d7deadacf0 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 24b22eafb8..91ac53ad0b 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index ed7b9c8618..19055c6110 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index 517dad9cbb..bf8244564a 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index b386b68cff..0e8ebf0c98 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h
index 2d21dac87a..f88138063e 100644
--- a/erts/emulator/beam/erl_drv_nif.h
+++ b/erts/emulator/beam/erl_drv_nif.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2010-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index cedc0517e1..d18016c42e 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h
index e3073eb874..289d0d0b28 100644
--- a/erts/emulator/beam/erl_fun.h
+++ b/erts/emulator/beam/erl_fun.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index a991c2c164..a7a8da4ed8 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 414aff1e06..6a529b8443 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2007-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 13d6136672..99995be464 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2015-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2015-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h
index f70fcdd1c0..ff31f04cb9 100644
--- a/erts/emulator/beam/erl_hl_timer.h
+++ b/erts/emulator/beam/erl_hl_timer.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2015-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2015-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index eaaf5c911a..fb38327d02 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index da73469516..f270d8baef 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index 71cd73ee27..3041474382 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index a80f5d6a16..f0c54e05f7 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2014-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index f7d0413685..c3ccf80b85 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2014-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 17982a2d14..c1af70592a 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
index 6dee1d5ef3..3994800ba7 100644
--- a/erts/emulator/beam/erl_monitors.c
+++ b/erts/emulator/beam/erl_monitors.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2004-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h
index f659829e6c..1cacecd7e9 100644
--- a/erts/emulator/beam/erl_monitors.h
+++ b/erts/emulator/beam/erl_monitors.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2004-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index 1c3160efaf..2d70f0d874 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2014-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c
index bb6f8660f1..19bb7d5b31 100644
--- a/erts/emulator/beam/erl_mtrace.c
+++ b/erts/emulator/beam/erl_mtrace.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2003-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index e041fd7b83..ea835d1b64 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 5a81f5fbbb..b0d5c39798 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index b34058f303..bbdadafade 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h
index e27f1d121a..6ec428e282 100644
--- a/erts/emulator/beam/erl_node_container_utils.h
+++ b/erts/emulator/beam/erl_node_container_utils.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 89b627aaf5..3c5945d48d 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 35051173d0..489da1ba17 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index 6b64bbd2f1..e6f8460164 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 9947e33f47..a56ef4aedb 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 5b35dc3c78..6113c8aa05 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 01e240c65d..3c80f0e0f6 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index fbf14df92b..b826e6c5d3 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2003-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index a93f1755c8..c0e7380ed0 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2007-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 46a72fcb0c..6e704b185d 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2007-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index abf2fe44f3..c3d59cb3a8 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_rbtree.h b/erts/emulator/beam/erl_rbtree.h
index 6a42853957..e59d6900b0 100644
--- a/erts/emulator/beam/erl_rbtree.h
+++ b/erts/emulator/beam/erl_rbtree.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2015. All Rights Reserved.
+ * Copyright Ericsson AB 2015-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index 6b25728af7..d904e35e40 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 097d580d99..842802f8d9 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 46d6da6448..ccc5526664 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index c69fec3c80..3084a8db75 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index a5fc3a2477..4b06c55770 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 01629db9ad..2d1d1443a7 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 75d7e47239..07cf4f6903 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 5366ee3644..0b8d78c469 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index ecfef28c57..57f5ba5436 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index f5a5da981c..1190d90b8e 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index bb4d442240..e3be6a5a22 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c
index cd834e2c12..a1f6f54543 100644
--- a/erts/emulator/beam/index.c
+++ b/erts/emulator/beam/index.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h
index 10f5d1eb39..6c07571df6 100644
--- a/erts/emulator/beam/index.h
+++ b/erts/emulator/beam/index.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 2f3117223f..bf89740fda 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 24de35696c..d752ea4330 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index a7e5a64b22..a3069e419a 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 96a7bfe8ac..457cada745 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c
index 173a39533d..1538191d67 100644
--- a/erts/emulator/drivers/common/efile_drv.c
+++ b/erts/emulator/drivers/common/efile_drv.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 90114b77f5..2de908ffb3 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/drivers/common/zlib_drv.c b/erts/emulator/drivers/common/zlib_drv.c
index e8afddb01b..e342e414b5 100644
--- a/erts/emulator/drivers/common/zlib_drv.c
+++ b/erts/emulator/drivers/common/zlib_drv.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2003-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/drivers/unix/sig_drv.c b/erts/emulator/drivers/unix/sig_drv.c
index 18f2038431..03d87b289b 100644
--- a/erts/emulator/drivers/unix/sig_drv.c
+++ b/erts/emulator/drivers/unix/sig_drv.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/elf64ppc.x b/erts/emulator/hipe/elf64ppc.x
index bb14a6cd29..f9b3e6795d 100644
--- a/erts/emulator/hipe/elf64ppc.x
+++ b/erts/emulator/hipe/elf64ppc.x
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 072ca19eae..0225f17613 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h
index d6be473ff5..ee97e6fc64 100644
--- a/erts/emulator/hipe/hipe_bif0.h
+++ b/erts/emulator/hipe/hipe_bif0.h
@@ -2,7 +2,7 @@
  * %CopyrightBegin%
 
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 078ebc40b7..4038ca7ef8 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 2001-2016. All Rights Reserved.
+# Copyright Ericsson AB 2001-2017. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c
index 0d4c1539b6..3d3df4fd48 100644
--- a/erts/emulator/hipe/hipe_bif1.c
+++ b/erts/emulator/hipe/hipe_bif1.c
@@ -2,7 +2,7 @@
  * %CopyrightBegin%
 
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c
index 2311beb34a..1a4a4c7952 100644
--- a/erts/emulator/hipe/hipe_gc.c
+++ b/erts/emulator/hipe/hipe_gc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2004-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
index 9a9e3c6b12..27d7bb9dee 100644
--- a/erts/emulator/hipe/hipe_load.c
+++ b/erts/emulator/hipe/hipe_load.c
@@ -2,7 +2,7 @@
  * %CopyrightBegin%
 
  *
- * Copyright Ericsson AB 2016. All Rights Reserved.
+ * Copyright Ericsson AB 2016-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index 1270a94986..ba7ae1e6a8 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -2,7 +2,7 @@
  * %CopyrightBegin%
 
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h
index 7b896872a6..3c042913f6 100644
--- a/erts/emulator/hipe/hipe_mode_switch.h
+++ b/erts/emulator/hipe/hipe_mode_switch.h
@@ -2,7 +2,7 @@
  * %CopyrightBegin%
 
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_module.c b/erts/emulator/hipe/hipe_module.c
index 2e99a30556..45d47272ed 100644
--- a/erts/emulator/hipe/hipe_module.c
+++ b/erts/emulator/hipe/hipe_module.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2016. All Rights Reserved.
+ * Copyright Ericsson AB 2016-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index 342407ef62..d8044fe6da 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_process.h b/erts/emulator/hipe/hipe_process.h
index 36b6ffc021..cc92bf653c 100644
--- a/erts/emulator/hipe/hipe_process.h
+++ b/erts/emulator/hipe/hipe_process.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c
index d0f0407489..4ee06690cf 100644
--- a/erts/emulator/hipe/hipe_stack.c
+++ b/erts/emulator/hipe/hipe_stack.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2003-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_x86_signal.c b/erts/emulator/hipe/hipe_x86_signal.c
index b24b9148a2..be68d7d463 100644
--- a/erts/emulator/hipe/hipe_x86_signal.c
+++ b/erts/emulator/hipe/hipe_x86_signal.c
@@ -2,7 +2,7 @@
  * %CopyrightBegin%
 
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index b6d71b3ec5..ad580e7d52 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index f02d6c1f62..ee4abeece9 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index 1e99078906..b8f0bb7150 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/unix/erl_child_setup.h b/erts/emulator/sys/unix/erl_child_setup.h
index b61e557ddf..0058b92344 100644
--- a/erts/emulator/sys/unix/erl_child_setup.h
+++ b/erts/emulator/sys/unix/erl_child_setup.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2015-2015. All Rights Reserved.
+ * Copyright Ericsson AB 2015-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index aa373e76bc..22059d21d5 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 3752edc480..0079912b10 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c
index 93cf64719a..834706d86f 100644
--- a/erts/emulator/sys/unix/sys_drivers.c
+++ b/erts/emulator/sys/unix/sys_drivers.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index 93013a412b..b10fc1e430 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2007-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index fa9ed308a2..28019e306c 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index ca81c9915e..2479ccc01f 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1997-2016. All Rights Reserved.
+# Copyright Ericsson AB 1997-2017. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index 324730d562..355be7a36d 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/bs_match_int_SUITE.erl b/erts/emulator/test/bs_match_int_SUITE.erl
index 32bfa2f1fa..e913dc98b0 100644
--- a/erts/emulator/test/bs_match_int_SUITE.erl
+++ b/erts/emulator/test/bs_match_int_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl
index e74631e916..1216863c51 100644
--- a/erts/emulator/test/call_trace_SUITE.erl
+++ b/erts/emulator/test/call_trace_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/ddll_SUITE.erl b/erts/emulator/test/ddll_SUITE.erl
index e7e518f82b..0b9f76a892 100644
--- a/erts/emulator/test/ddll_SUITE.erl
+++ b/erts/emulator/test/ddll_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/dirty_bif_SUITE.erl b/erts/emulator/test/dirty_bif_SUITE.erl
index 308323594d..b8361690e6 100644
--- a/erts/emulator/test/dirty_bif_SUITE.erl
+++ b/erts/emulator/test/dirty_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2010-2014. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl
index 5ba0d85ff3..f62f1e9dce 100644
--- a/erts/emulator/test/dirty_nif_SUITE.erl
+++ b/erts/emulator/test/dirty_nif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index 4f98cd6cee..434e729310 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/erl_link_SUITE.erl b/erts/emulator/test/erl_link_SUITE.erl
index 9258897764..5622cce980 100644
--- a/erts/emulator/test/erl_link_SUITE.erl
+++ b/erts/emulator/test/erl_link_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2001-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2001-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/erts_debug_SUITE.erl b/erts/emulator/test/erts_debug_SUITE.erl
index c9c664de38..6aa7a445b5 100644
--- a/erts/emulator/test/erts_debug_SUITE.erl
+++ b/erts/emulator/test/erts_debug_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2005-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/estone_SUITE.erl b/erts/emulator/test/estone_SUITE.erl
index 35f695ffe5..8b336b366d 100644
--- a/erts/emulator/test/estone_SUITE.erl
+++ b/erts/emulator/test/estone_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2002-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/float_SUITE.erl b/erts/emulator/test/float_SUITE.erl
index ad33ad705b..4098aa9c6a 100644
--- a/erts/emulator/test/float_SUITE.erl
+++ b/erts/emulator/test/float_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/guard_SUITE.erl b/erts/emulator/test/guard_SUITE.erl
index 54ee710363..1a93a9f5c2 100644
--- a/erts/emulator/test/guard_SUITE.erl
+++ b/erts/emulator/test/guard_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/hibernate_SUITE.erl b/erts/emulator/test/hibernate_SUITE.erl
index 90693595a7..a20f306e04 100644
--- a/erts/emulator/test/hibernate_SUITE.erl
+++ b/erts/emulator/test/hibernate_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2003-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2003-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/list_bif_SUITE.erl b/erts/emulator/test/list_bif_SUITE.erl
index 08574bff71..f95251943d 100644
--- a/erts/emulator/test/list_bif_SUITE.erl
+++ b/erts/emulator/test/list_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl
index 971a047309..eb189c2c33 100644
--- a/erts/emulator/test/match_spec_SUITE.erl
+++ b/erts/emulator/test/match_spec_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 8ad11d3bf3..1eb58699b2 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index a255c9f096..3747291e7e 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_drv_nif.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_drv_nif.h
index ea013a49a3..3e5435e353 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_drv_nif.h
+++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_drv_nif.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2017. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif.h
index 936f03bce1..4b2b7550e5 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif.h
+++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2017. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif_api_funcs.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif_api_funcs.h
index ef4e9580b0..302973fcca 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif_api_funcs.h
+++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif_api_funcs.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2017. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_drv_nif.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_drv_nif.h
index ea013a49a3..3e5435e353 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_drv_nif.h
+++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_drv_nif.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2017. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif.h
index 8006741a63..c3013b6b74 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif.h
+++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2017. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif_api_funcs.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif_api_funcs.h
index 2f841645e1..92954403f3 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif_api_funcs.h
+++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif_api_funcs.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2017. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.c b/erts/emulator/test/nif_SUITE_data/nif_mod.c
index 04699d3327..885b8ebaf8 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_mod.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl
index 30518a5b27..8e9e3cb05a 100644
--- a/erts/emulator/test/node_container_SUITE.erl
+++ b/erts/emulator/test/node_container_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2002-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl
index f62eb0b430..1c76eb8019 100644
--- a/erts/emulator/test/num_bif_SUITE.erl
+++ b/erts/emulator/test/num_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index c117554f90..94ee9851dd 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl
index 64e51e7d7c..8d71df65e7 100644
--- a/erts/emulator/test/scheduler_SUITE.erl
+++ b/erts/emulator/test/scheduler_SUITE.erl
@@ -1,7 +1,7 @@
 %% 
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2008-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2008-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/smoke_test_SUITE.erl b/erts/emulator/test/smoke_test_SUITE.erl
index 45b28b28a5..41bb07b84c 100644
--- a/erts/emulator/test/smoke_test_SUITE.erl
+++ b/erts/emulator/test/smoke_test_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2011-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2011-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl
index 729e86cb4f..3057905f4c 100644
--- a/erts/emulator/test/statistics_SUITE.erl
+++ b/erts/emulator/test/statistics_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl
index 6bd1eb1e1e..9c71f20279 100644
--- a/erts/emulator/test/system_info_SUITE.erl
+++ b/erts/emulator/test/system_info_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2005-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/time_SUITE.erl b/erts/emulator/test/time_SUITE.erl
index c13d03bcc4..214a549a9d 100644
--- a/erts/emulator/test/time_SUITE.erl
+++ b/erts/emulator/test/time_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl
index a90701c5d2..a2b267543f 100644
--- a/erts/emulator/test/z_SUITE.erl
+++ b/erts/emulator/test/z_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/utils/make_preload b/erts/emulator/utils/make_preload
index bcb2e42614..0cd3509b62 100755
--- a/erts/emulator/utils/make_preload
+++ b/erts/emulator/utils/make_preload
@@ -2,7 +2,7 @@
 #
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 1999-2016. All Rights Reserved.
+# Copyright Ericsson AB 1999-2017. All Rights Reserved.
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
-- 
cgit v1.2.3


From e01b87f45486330561d5673fd109a6b6e40828b2 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 3 May 2017 20:27:01 +0200
Subject: erts: Move and rename erts_is_native_break()

---
 erts/emulator/beam/beam_bp.c   | 21 +++++----------------
 erts/emulator/beam/beam_load.c | 12 ++++++++++++
 erts/emulator/beam/beam_load.h |  1 +
 3 files changed, 18 insertions(+), 16 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 1efe7536d6..b01527fa52 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -211,7 +211,7 @@ erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified)
 
 	    ci = code_hdr->functions[fi];
 	    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
-	    if (erts_is_native_break(ci)) {
+	    if (erts_is_function_native(ci)) {
 		continue;
 	    }
 	    if (is_nil(ci->mfa.module)) { /* Ignore BIF stub */
@@ -277,7 +277,7 @@ erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified)
 		continue;
 	    }
 	    ASSERT(*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint));
-	} else if (erts_is_native_break(erts_code_to_codeinfo(ep->addressv[code_ix]))) {
+	} else if (erts_is_function_native(erts_code_to_codeinfo(ep->addressv[code_ix]))) {
 	    continue;
 	}
 
@@ -611,7 +611,7 @@ erts_clear_module_break(Module *modp) {
     n = (Uint)(UWord) code_hdr->num_functions;
     for (i = 0; i < n; ++i) {
 	ErtsCodeInfo *ci = code_hdr->functions[i];
-	if (erts_is_native_break(ci))
+	if (erts_is_function_native(ci))
 	    continue;
 	clear_function_break(ci, ERTS_BPF_ALL);
     }
@@ -620,7 +620,7 @@ erts_clear_module_break(Module *modp) {
 
     for (i = 0; i < n; ++i) {
 	ErtsCodeInfo *ci = code_hdr->functions[i];
-	if (erts_is_native_break(ci))
+	if (erts_is_function_native(ci))
 	    continue;
 	uninstall_breakpoint(ci);
 	consolidate_bp_data(modp, ci, 1);
@@ -1212,17 +1212,6 @@ erts_is_mtrace_break(ErtsCodeInfo *ci, Binary **match_spec_ret,
     return 0;
 }
 
-int
-erts_is_native_break(ErtsCodeInfo *ci) {
-#ifdef HIPE
-    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
-    return erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call)
-	|| erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure);
-#else
-    return 0;
-#endif
-}
-
 int 
 erts_is_count_break(ErtsCodeInfo *ci, Uint *count_ret)
 {
@@ -1731,7 +1720,7 @@ check_break(ErtsCodeInfo *ci, Uint break_flags)
     GenericBp* g = ci->u.gen_bp;
 
     ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
-    if (erts_is_native_break(ci)) {
+    if (erts_is_function_native(ci)) {
 	return 0;
     }
     if (g) {
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 7b79e6303b..2763d8a38e 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -5696,6 +5696,18 @@ erts_is_module_native(BeamCodeHeader* code_hdr)
     return 0;
 }
 
+int
+erts_is_function_native(ErtsCodeInfo *ci)
+{
+#ifdef HIPE
+    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+    return erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call)
+	|| erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure);
+#else
+    return 0;
+#endif
+}
+
 /*
  * Builds a list of all functions including native addresses.
  *     [{Name,Arity,NativeAddress},...]
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 659b9c303f..b8d8634e28 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -119,6 +119,7 @@ typedef struct beam_code_header {
 
 void erts_release_literal_area(struct ErtsLiteralArea_* literal_area);
 int erts_is_module_native(BeamCodeHeader* code);
+int erts_is_function_native(ErtsCodeInfo*);
 void erts_beam_bif_load_init(void);
 struct erl_fun_entry;
 void erts_purge_state_add_fun(struct erl_fun_entry *fe);
-- 
cgit v1.2.3


From f95094d67e56a51bbdaadaf18625bb01c584bf2d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 3 May 2017 20:33:13 +0200
Subject: erts: Fix code:is_module_native for local trace

Local trace on first function in module
made code:is_module_native/1 return true.

Use new erts_is_function_native() to make a proper check.
---
 erts/emulator/beam/beam_load.c           | 9 ++++-----
 erts/emulator/test/trace_local_SUITE.erl | 1 +
 2 files changed, 5 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 2763d8a38e..4f5c68d29a 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -5685,12 +5685,11 @@ erts_is_module_native(BeamCodeHeader* code_hdr)
     if (code_hdr != NULL) {
         num_functions = code_hdr->num_functions;
         for (i=0; i<num_functions; i++) {
-            BeamInstr* func_info = (BeamInstr *) code_hdr->functions[i];
-            Eterm name = (Eterm) func_info[3];
-            if (is_atom(name)) {
-                return func_info[1] != 0;
+            ErtsCodeInfo* ci = code_hdr->functions[i];
+            if (is_atom(ci->mfa.function)) {
+                return erts_is_function_native(ci);
             }
-            else ASSERT(is_nil(name)); /* ignore BIF stubs */
+            else ASSERT(is_nil(ci->mfa.function)); /* ignore BIF stubs */
         }
     }
     return 0;
diff --git a/erts/emulator/test/trace_local_SUITE.erl b/erts/emulator/test/trace_local_SUITE.erl
index 5b65889f4a..1cbe6201c3 100644
--- a/erts/emulator/test/trace_local_SUITE.erl
+++ b/erts/emulator/test/trace_local_SUITE.erl
@@ -298,6 +298,7 @@ basic_test() ->
     setup([call]),
     NumMatches = erlang:trace_pattern({?MODULE,'_','_'},[],[local]),
     NumMatches = erlang:trace_pattern({?MODULE,'_','_'},[],[local]),
+    false = code:is_module_native(?MODULE), % got fooled by local trace
     erlang:trace_pattern({?MODULE,slave,'_'},false,[local]),
     [1,1,1,997] = apply_slave(?MODULE,exported_wrap,[1]),
     ?CT(?MODULE,exported_wrap,[1]),
-- 
cgit v1.2.3


From 6962098f2ff7dcff5ce46364898815c685d0f0b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 4 May 2017 15:35:06 +0200
Subject: Extend hipe_SUITE to test exceptions and try/catch

---
 erts/emulator/test/hipe_SUITE.erl                 | 68 +++++++++++++++++++++++
 erts/emulator/test/hipe_SUITE_data/trycatch_1.erl | 14 +++++
 erts/emulator/test/hipe_SUITE_data/trycatch_2.erl | 10 ++++
 erts/emulator/test/hipe_SUITE_data/trycatch_3.erl |  9 +++
 4 files changed, 101 insertions(+)
 create mode 100644 erts/emulator/test/hipe_SUITE_data/trycatch_1.erl
 create mode 100644 erts/emulator/test/hipe_SUITE_data/trycatch_2.erl
 create mode 100644 erts/emulator/test/hipe_SUITE_data/trycatch_3.erl

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/hipe_SUITE.erl b/erts/emulator/test/hipe_SUITE.erl
index 0b44dd7fb7..5083f01f34 100644
--- a/erts/emulator/test/hipe_SUITE.erl
+++ b/erts/emulator/test/hipe_SUITE.erl
@@ -22,6 +22,7 @@
 -export([all/0
 	,t_copy_literals/1
 	,t_purge/1
+        ,t_trycatch/1
 	]).
 
 all() ->
@@ -29,6 +30,7 @@ all() ->
 	undefined -> {skip, "HiPE is disabled"};
 	_ -> [t_copy_literals
 	     ,t_purge
+             ,t_trycatch
 	     ]
     end.
 
@@ -118,3 +120,69 @@ t_purge(Config) when is_list(Config) ->
 call(Pid, Call) ->
     Pid ! {Call, self()},
     receive {Pid, Res} -> Res end.
+
+t_trycatch(Config) ->
+    DataDir = proplists:get_value(data_dir, Config),
+    Files = ["trycatch_1.erl","trycatch_2.erl","trycatch_3.erl"],
+    Sources0 = [filename:join(DataDir, Src) || Src <- Files],
+    Sources = trycatch_combine(Sources0),
+    t_trycatch_1(Sources).
+
+t_trycatch_1([S|Ss]) ->
+    io:format("~p", [S]),
+    compile_and_load(S),
+    call_trycatch(try_catch),
+    call_trycatch(plain_catch),
+    io:nl(),
+    t_trycatch_1(Ss);
+t_trycatch_1([]) ->
+    ok.
+
+trycatch_combine([N|Ns]) ->
+    Combined = trycatch_combine(Ns),
+    lists:append([[[{N,[]}|C],[{N,[native]},C]] || C <- Combined]);
+trycatch_combine([]) ->
+    [[]].
+
+call_trycatch(Func) ->
+    case do_call_trycatch(error, Func, {error,whatever}) of
+        {error,whatever,[{trycatch_3,three,1,_}|_]} ->
+            ok
+    end,
+    case do_call_trycatch(error, Func, fc) of
+        {error,function_clause,[{trycatch_3,three,[fc],_}|_]} ->
+            ok;
+        {error,function_clause,[{trycatch_3,three,1,_}|_]} ->
+            ok
+    end,
+    case do_call_trycatch(throw, Func, {throw,{a,b}}) of
+        {throw,{a,b},[{trycatch_3,three,1,_}|_]} ->
+            ok
+    end,
+    case do_call_trycatch(exit, Func, {exit,{a,b,c}}) of
+        {exit,{a,b,c},[{trycatch_3,three,1,_}|_]} ->
+            ok
+    end,
+    ok.
+
+do_call_trycatch(_Class, try_catch, Argument) ->
+    trycatch_1:one_try_catch(Argument);
+do_call_trycatch(error, plain_catch, Argument) ->
+    {{'EXIT',{Reason,Stk}},Stk} = trycatch_1:one_plain_catch(Argument),
+    {error,Reason,Stk};
+do_call_trycatch(throw, plain_catch, Argument) ->
+    {Reason,Stk} = trycatch_1:one_plain_catch(Argument),
+    {throw,Reason,Stk};
+do_call_trycatch(exit, plain_catch, Argument) ->
+    {{'EXIT',Reason},Stk} = trycatch_1:one_plain_catch(Argument),
+    {exit,Reason,Stk}.
+
+compile_and_load(Sources) ->
+    _ = [begin
+             {ok,Mod,Bin} = compile:file(Src, [binary,report|Opts]),
+             code:purge(Mod),
+             code:delete(Mod),
+             code:purge(Mod),
+             {module,Mod} = code:load_binary(Mod, atom_to_list(Mod), Bin)
+         end || {Src,Opts} <- Sources],
+    ok.
diff --git a/erts/emulator/test/hipe_SUITE_data/trycatch_1.erl b/erts/emulator/test/hipe_SUITE_data/trycatch_1.erl
new file mode 100644
index 0000000000..702b14b5b9
--- /dev/null
+++ b/erts/emulator/test/hipe_SUITE_data/trycatch_1.erl
@@ -0,0 +1,14 @@
+-module(trycatch_1).
+-export([one_try_catch/1,one_plain_catch/1]).
+
+one_try_catch(Term) ->
+    try
+        trycatch_2:two(Term)
+    catch
+        C:R ->
+            Stk = erlang:get_stacktrace(),
+            {C,R,Stk}
+    end.
+
+one_plain_catch(Term) ->
+    {catch trycatch_2:two(Term),erlang:get_stacktrace()}.
diff --git a/erts/emulator/test/hipe_SUITE_data/trycatch_2.erl b/erts/emulator/test/hipe_SUITE_data/trycatch_2.erl
new file mode 100644
index 0000000000..ffac420197
--- /dev/null
+++ b/erts/emulator/test/hipe_SUITE_data/trycatch_2.erl
@@ -0,0 +1,10 @@
+-module(trycatch_2).
+-export([two/1]).
+
+two(Term) ->
+    Res = trycatch_3:three(Term),
+    foo(),
+    Res.
+
+foo() ->
+    ok.
diff --git a/erts/emulator/test/hipe_SUITE_data/trycatch_3.erl b/erts/emulator/test/hipe_SUITE_data/trycatch_3.erl
new file mode 100644
index 0000000000..578fa0e87e
--- /dev/null
+++ b/erts/emulator/test/hipe_SUITE_data/trycatch_3.erl
@@ -0,0 +1,9 @@
+-module(trycatch_3).
+-export([three/1]).
+
+three({error,Term}) ->
+    error(Term);
+three({throw,Term}) ->
+    throw(Term);
+three({exit,Term}) ->
+    exit(Term).
-- 
cgit v1.2.3


From ff94c59ca65ab3f4d9a1660e64ae31ed2232a5c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 5 May 2017 05:23:54 +0200
Subject: Clear p->fvalue when handling a try/catch

p->fvalue will be set by BIFs that generate exceptions
(such as throw/1), and it will not be cleared before another
exception is generated. Potentially, p->fvalue may contain a
huge term (e.g. after throw(HugeTerm)) which will be kept in
the heap.

We can shorten the lifetime of f->value by clearing it in
the instructions that handle catches: catch_end and try_end.
That is safe because BEAM code will never access p->fvalue.
If BEAM code needs to rethrow an exception it will use a
reference to the value passed in an X register.

The reason that p->fvalue must not be cleared already in
handle_error() is that native code trap handlers will use
it. (See the comment before handle_error() and the comment
at the end of handle_error() in beam_emu.c for some more
information about exception handling.)
---
 erts/emulator/beam/beam_emu.c | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 311bb94242..a90e6a0ba8 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1814,6 +1814,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
      c_p->catches--;
      make_blank(yb(Arg(0)));
      if (is_non_value(r(0))) {
+         c_p->fvalue = NIL;
 	 if (x(1) == am_throw) {
 	     r(0) = x(2);
 	 } else {
@@ -1843,6 +1844,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
      c_p->catches--;
      make_blank(yb(Arg(0)));
      if (is_non_value(r(0))) {
+         c_p->fvalue = NIL;
 	 r(0) = x(1);
 	 x(1) = x(2);
 	 x(2) = x(3);
-- 
cgit v1.2.3


From 900affc2420905ec70f3cf906324093fc0c2e8a8 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 5 May 2017 14:55:36 +0200
Subject: erts: the esdp is not always available in matchspec

This happens for instance when a receive_trace is run in the
sys_msg_dispatcher thread when the trace_delivered trace message
is traced.
---
 erts/emulator/beam/erl_db_util.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 91ac53ad0b..16142e69fb 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2006,7 +2006,8 @@ restart:
     do_catch = 0;
     fail_label = -1;
     build_proc = psp;
-    esdp->current_process = psp;
+    if (esdp)
+        esdp->current_process = psp;
 
 #ifdef DEBUG
     ASSERT(variables == mpsp->u.variables);
@@ -2675,7 +2676,8 @@ restart:
 	    do_catch = 1;
 	    if (in_flags & ERTS_PAM_COPY_RESULT) {
 		build_proc = c_p;
-		esdp->current_process = c_p;
+                if (esdp)
+                    esdp->current_process = c_p;
 	    }
 	    break;
 	case matchHalt:
-- 
cgit v1.2.3


From 3f0e95a08394e92a58f99f99a94f9349e35842dd Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 5 May 2017 19:47:07 +0200
Subject: erts: Unbreak --enable-lock-checking --enable-lock-counter

both together.

Broken in OTP-20.0-rc1 by 7d161f5b475575bd79bd90977b3a79334a8ec658.
---
 erts/emulator/beam/erl_lock_count.c | 8 ++++++--
 erts/emulator/beam/erl_lock_count.h | 2 ++
 erts/emulator/beam/erl_port_task.h  | 8 +++++---
 erts/emulator/beam/erl_smp.h        | 2 +-
 erts/emulator/beam/erl_threads.h    | 6 +++---
 erts/emulator/beam/io.c             | 8 +++++---
 6 files changed, 22 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index 6354fc8663..49b1f08b44 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -358,8 +358,11 @@ void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) {
 
 void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) {
     int i;
-    if (name == NULL) { ERTS_LCNT_CLEAR_FLAG(lock); return; }
-    lcnt_lock();
+
+    if (flag & ERTS_LCNT_LT_DISABLE) {
+        ERTS_LCNT_CLEAR_FLAG(lock);
+        return;
+    }
 
     lock->next = NULL;
     lock->prev = NULL;
@@ -379,6 +382,7 @@ void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eter
         lcnt_clear_stats(&lock->stats[i]);
     }
 
+    lcnt_lock();
     erts_lcnt_list_insert(erts_lcnt_data->current_locks, lock);
     lcnt_unlock();
 }
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index 3041474382..be601a26fc 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -95,6 +95,8 @@
 #define ERTS_LCNT_LO_READ       (((Uint16) 1) << 6)
 #define ERTS_LCNT_LO_WRITE      (((Uint16) 1) << 7)
 
+#define ERTS_LCNT_LT_DISABLE    (((Uint16) 1) << 8)
+
 #define ERTS_LCNT_LO_READ_WRITE ( ERTS_LCNT_LO_READ  \
                                 | ERTS_LCNT_LO_WRITE )
 
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index e3550e878e..858ddfe029 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -189,11 +189,13 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id)
     erts_smp_atomic32_init_nob(&ptsp->flags, 0);
 #ifdef ERTS_SMP
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK))
-        lock_str = NULL;
-#endif
+    erts_mtx_init_x_opt(&ptsp->mtx, lock_str, instr_id,
+			((erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)
+			 ? 0 : ERTS_LCNT_LT_DISABLE));
+#else
     erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id);
 #endif
+#endif
 }
 
 ERTS_GLB_INLINE void
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index 14be511f86..55ba943bdd 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -1073,7 +1073,7 @@ ERTS_GLB_INLINE void
 erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
 {
 #ifdef ERTS_SMP
-    erts_mtx_init_locked_x(mtx, name, extra);
+    erts_mtx_init_locked_x_opt(mtx, name, extra, 0);
 #endif
 }
 
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 9e75f6fee5..28ff5d3a42 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -481,7 +481,7 @@ ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len);
 ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y);
 ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra);
 ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt);
-ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra);
+ERTS_GLB_INLINE void erts_mtx_init_locked_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt);
 ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx);
@@ -2192,7 +2192,7 @@ erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt)
 
 
 ERTS_GLB_INLINE void
-erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra)
+erts_mtx_init_locked_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt)
 {
 #ifdef USE_THREADS
     int res = ethr_mutex_init(&mtx->mtx);
@@ -2202,7 +2202,7 @@ erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra)
     erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
+    erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra);
 #endif
     ethr_mutex_lock(&mtx->mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index bf89740fda..93a5ed4397 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -260,10 +260,12 @@ static ERTS_INLINE void port_init_instr(Port *prt
     if (!prt->drv_ptr->lock) {
 	char *lock_str = "port_lock";
 #ifdef ERTS_ENABLE_LOCK_COUNT
-        if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK))
-            lock_str = NULL;
+	Uint16 opt = ((erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)
+			    ? 0 : ERTS_LCNT_LT_DISABLE);
+#else
+	Uint16 opt = 0;
 #endif
-	erts_mtx_init_locked_x(prt->lock, lock_str, id);
+	erts_mtx_init_locked_x_opt(prt->lock, lock_str, id, opt);
     }
 #endif
     erts_port_task_init_sched(&prt->sched, id);
-- 
cgit v1.2.3


From eb9ffb378b9eb3b9f4a0f898e60ade01d3d48283 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 5 May 2017 20:29:58 +0200
Subject: erts: Add minor lock optimization for lcnt

---
 erts/emulator/beam/erl_lock_count.c | 21 ++++++++++-----------
 erts/emulator/beam/erl_lock_count.h |  1 -
 2 files changed, 10 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index 49b1f08b44..aee9796171 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -299,22 +299,16 @@ erts_lcnt_lock_list_t *erts_lcnt_list_init(void) {
     return list;
 }
 
-/* only do this on the list with the deleted locks! */
-void erts_lcnt_list_clear(erts_lcnt_lock_list_t *list) {
-    erts_lcnt_lock_t *lock = NULL,
-                     *next = NULL;
+static void lcnt_list_free(erts_lcnt_lock_t *head) {
+    erts_lcnt_lock_t *lock, *next;
 
-    lock = list->head;
+    lock = head;
 
     while(lock != NULL) {
         next = lock->next;
         free(lock);
         lock = next;
     }
-
-    list->head = NULL;
-    list->tail = NULL;
-    list->n    = 0;
 }
 
 void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) {
@@ -679,12 +673,17 @@ void erts_lcnt_clear_counters(void) {
         lock->n_stats = 1;
     }
 
-    /* empty deleted locks in lock list */
-    erts_lcnt_list_clear(erts_lcnt_data->deleted_locks);
+    lock = erts_lcnt_data->deleted_locks->head;
+    erts_lcnt_data->deleted_locks->head = NULL;
+    erts_lcnt_data->deleted_locks->tail = NULL;
+    erts_lcnt_data->deleted_locks->n = 0;
 
     lcnt_time(&timer_start);
 
     lcnt_unlock();
+
+    /* free deleted locks */
+    lcnt_list_free(lock);
 }
 
 erts_lcnt_data_t *erts_lcnt_get_data(void) {
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index be601a26fc..6caffbfe86 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -206,7 +206,6 @@ void erts_lcnt_thread_exit_handler(void);
 /* list operations (local)  */
 erts_lcnt_lock_list_t *erts_lcnt_list_init(void);
 
-void erts_lcnt_list_clear( erts_lcnt_lock_list_t *list);
 void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock);
 void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock);
 
-- 
cgit v1.2.3


From 95dec7497abbdb06defcb23e99fa31e06597496e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 3 May 2017 18:22:45 +0200
Subject: erts: Fix ets:select_replace with {const, NewTuple}

Enable ets:select_replace to do a generic single object
compare-and-swap operation of any ets-tuple using
a matchspec like this:

[{Old, [], [{const, New}]}]

The only exception when this does not work is if the key
contains maps or atoms looking like variables (like '$1').
---
 erts/emulator/beam/erl_db_util.c | 41 +++++++++++++++++++++++++---------------
 1 file changed, 26 insertions(+), 15 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 24b22eafb8..d1130fac87 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1125,12 +1125,11 @@ error:
  * Returns true if 'b' is guaranteed to always construct
  * the same term as 'a' has matched.
  */
-static int db_match_eq_body(Eterm a, Eterm b)
+static int db_match_eq_body(Eterm a, Eterm b, int const_mode)
 {
     DECLARE_ESTACK(s);
     Uint arity;
     Eterm *ap, *bp;
-    int const_mode = 0;
     const Eterm CONST_MODE_OFF = THE_NON_VALUE;
 
     while (1) {
@@ -1224,6 +1223,7 @@ int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body)
     Eterm single_body_subterm;
     Eterm single_body_subterm_key;
     Eterm* single_body_subterm_key_tpl;
+    int const_mode;
 
     if (!is_list(body)) {
         return 0;
@@ -1244,42 +1244,53 @@ int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body)
         return 0;
     }
 
-    single_body_term_tpl = tuple_val(single_body_term);
-    if (arityval(*single_body_term_tpl) != 1) {
-        // not the 1-element tuple we're expecting
-        return 0;
-    }
-
     match_key = db_getkey(keypos, match);
     if (!is_value(match_key)) {
         // can't get key out of match
         return 0;
     }
 
-    single_body_subterm = single_body_term_tpl[1];
+    single_body_term_tpl = tuple_val(single_body_term);
+    if (single_body_term_tpl[0] == make_arityval(2) &&
+        single_body_term_tpl[1] == am_const) {
+        /* {const, {"ets-tuple constant"}} */
+        single_body_subterm = single_body_term_tpl[2];
+        const_mode = 1;
+    }
+    else if (*single_body_term_tpl == make_arityval(1)) {
+        /* {{"ets-tuple construction"}} */
+        single_body_subterm = single_body_term_tpl[1];
+        const_mode = 0;
+    }
+    else {
+        /* not a tuple construction */
+        return 0;
+    }
+
     single_body_subterm_key = db_getkey(keypos, single_body_subterm);
     if (!is_value(single_body_subterm_key)) {
         // can't get key out of single body subterm
         return 0;
     }
 
-    if (db_match_eq_body(match_key, single_body_subterm_key)) {
+    if (db_match_eq_body(match_key, single_body_subterm_key, const_mode)) {
         /* tuple with same key is returned */
         return 1;
     }
 
-    if (!is_tuple(single_body_subterm_key)) {
-        /* can't possibly be an element instruction */
+    if (const_mode) {
+        /* constant key did not match */
         return 0;
     }
 
-    single_body_subterm_key_tpl = tuple_val(single_body_subterm_key);
-    if (arityval(*single_body_subterm_key_tpl) != 3) {
+    if (!is_tuple(single_body_subterm_key)) {
         /* can't possibly be an element instruction */
         return 0;
     }
 
-    if (single_body_subterm_key_tpl[1] == am_element &&
+    single_body_subterm_key_tpl = tuple_val(single_body_subterm_key);
+    if (single_body_subterm_key_tpl[0] == make_arityval(3) &&
+        single_body_subterm_key_tpl[1] == am_element &&
         single_body_subterm_key_tpl[3] == am_DollarUnderscore &&
         single_body_subterm_key_tpl[2] == make_small(keypos))
     {
-- 
cgit v1.2.3


From 2746827fe38db572bb26b280287da22b958d89c3 Mon Sep 17 00:00:00 2001
From: Daniel Goertzen <daniel.goertzen@gmail.com>
Date: Fri, 12 May 2017 14:05:05 -0500
Subject: use ErlNifMonitor instead of ErlDrvMonitor for enif_monitor/demonitor

---
 erts/emulator/beam/erl_nif_api_funcs.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index bbdadafade..c305732d63 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -177,8 +177,8 @@ ERL_NIF_API_FUNC_DECL(int,enif_thread_type,(void));
 ERL_NIF_API_FUNC_DECL(int,enif_snprintf,(char * buffer, size_t size, const char *format, ...));
 ERL_NIF_API_FUNC_DECL(int,enif_select,(ErlNifEnv* env, ErlNifEvent e, enum ErlNifSelectFlags flags, void* obj, const ErlNifPid* pid, ERL_NIF_TERM ref));
 ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type_x,(ErlNifEnv*, const char* name_str, const ErlNifResourceTypeInit*, ErlNifResourceFlags flags, ErlNifResourceFlags* tried));
-ERL_NIF_API_FUNC_DECL(int, enif_monitor_process,(ErlNifEnv*,void* obj,const ErlNifPid*,ErlDrvMonitor *monitor));
-ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const ErlDrvMonitor *monitor));
+ERL_NIF_API_FUNC_DECL(int, enif_monitor_process,(ErlNifEnv*,void* obj,const ErlNifPid*,ErlNifMonitor *monitor));
+ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const ErlNifMonitor *monitor));
 ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const ErlNifMonitor*));
 ERL_NIF_API_FUNC_DECL(ErlNifUInt64,enif_hash,(ErlNifHash type, ERL_NIF_TERM term, ErlNifUInt64 salt));
 
-- 
cgit v1.2.3


From e6437e926340c3024449b83826f8013d187caaed Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 3 May 2017 17:18:44 +0200
Subject: erts: Remove old unused functions

The functions have been found using: https://github.com/caolanm/callcatcher
---
 erts/emulator/beam/big.c              | 25 ----------------------
 erts/emulator/beam/big.h              |  2 --
 erts/emulator/beam/erl_alloc.c        | 22 --------------------
 erts/emulator/beam/erl_alloc.h        |  7 -------
 erts/emulator/beam/erl_cpu_topology.c | 11 ----------
 erts/emulator/beam/erl_cpu_topology.h |  8 -------
 erts/emulator/beam/erl_db_util.c      |  5 -----
 erts/emulator/beam/erl_gc.c           |  2 +-
 erts/emulator/beam/erl_init.c         | 24 ---------------------
 erts/emulator/beam/erl_port_task.c    |  7 -------
 erts/emulator/beam/erl_port_task.h    |  1 -
 erts/emulator/beam/erl_process.c      | 39 -----------------------------------
 erts/emulator/beam/erl_process.h      |  7 -------
 erts/emulator/beam/global.h           |  2 --
 14 files changed, 1 insertion(+), 161 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 4baee7900b..1f6feade1c 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -2265,21 +2265,6 @@ Eterm big_minus(Eterm x, Eterm y, Eterm *r)
 			BIG_V(yp),BIG_SIZE(yp),(short) !BIG_SIGN(yp), r);
 }
 
-/*
-** Subtract a digit from big number
-*/
-Eterm big_minus_small(Eterm x, Eterm y, Eterm *r)
-{
-    Eterm* xp = big_val(x);
-
-    if (BIG_SIGN(xp))
-	return big_norm(r, D_add(BIG_V(xp),BIG_SIZE(xp), (ErtsDigit) y, BIG_V(r)), 
-			(short) BIG_SIGN(xp));
-    else
-	return big_norm(r, D_sub(BIG_V(xp),BIG_SIZE(xp), (ErtsDigit) y, BIG_V(r)), 
-			(short) BIG_SIGN(xp));
-}
-
 /*
 ** Multiply smallnums
 */
@@ -2412,16 +2397,6 @@ Eterm big_rem(Eterm x, Eterm y, Eterm *r)
     }
 }
 
-Eterm big_neg(Eterm x, Eterm *r)
-{
-    Eterm* xp = big_val(x);
-    dsize_t xsz = BIG_SIZE(xp);
-    short xsgn = BIG_SIGN(xp);
-    
-    MOVE_DIGITS(BIG_V(r), BIG_V(xp), xsz);
-    return big_norm(r, xsz, (short) !xsgn);
-}
-
 Eterm big_band(Eterm x, Eterm y, Eterm *r)
 {
     Eterm* xp = big_val(x);
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index 4a96d971c3..258038a157 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -118,9 +118,7 @@ Eterm big_minus(Eterm, Eterm, Eterm*);
 Eterm big_times(Eterm, Eterm, Eterm*);
 Eterm big_div(Eterm, Eterm, Eterm*);
 Eterm big_rem(Eterm, Eterm, Eterm*);
-Eterm big_neg(Eterm, Eterm*);
 
-Eterm big_minus_small(Eterm, Uint, Eterm*);
 Eterm big_plus_small(Eterm, Uint, Eterm*);
 Eterm big_times_small(Eterm, Uint, Eterm*);
 
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 71957b2259..5aea917ddc 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -3512,28 +3512,6 @@ void erts_allctr_wrapper_pre_unlock(void)
 }
 
 
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- * Deprecated functions                                                    *
- *                                                                         *
- * These functions are still defined since "non-OTP linked in drivers" may *
- * contain (illegal) calls to them.                                        *
-\*                                                                         */
-
-/* --- DO *NOT* USE THESE FUNCTIONS --- */
-
-void *sys_alloc(Uint sz)
-{ return erts_alloc_fnf(ERTS_ALC_T_UNDEF, sz); }
-void *sys_realloc(void *ptr, Uint sz)
-{ return erts_realloc_fnf(ERTS_ALC_T_UNDEF, ptr, sz); }
-void sys_free(void *ptr)
-{ erts_free(ERTS_ALC_T_UNDEF, ptr); }
-void *safe_alloc(Uint sz)
-{ return erts_alloc(ERTS_ALC_T_UNDEF, sz); }
-void *safe_realloc(void *ptr, Uint sz)
-{ return erts_realloc(ERTS_ALC_T_UNDEF, ptr, sz); }
-
-
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
  * NOTE: erts_alc_test() is only supposed to be used for testing.            *
  *                                                                           *
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 56a3b73bf9..758d529f87 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -173,13 +173,6 @@ __decl_noreturn void erts_realloc_n_enomem(ErtsAlcType_t,void*,Uint)
 __decl_noreturn void erts_alc_fatal_error(int,int,ErtsAlcType_t,...)	
      __noreturn;
 
-/* --- DO *NOT* USE THESE DEPRECATED FUNCTIONS ---    Instead use:       */
-void *safe_alloc(Uint)               __deprecated; /* erts_alloc()       */
-void *safe_realloc(void *, Uint)     __deprecated; /* erts_realloc()     */
-void  sys_free(void *)               __deprecated; /* erts_free()        */
-void *sys_alloc(Uint )               __deprecated; /* erts_alloc_fnf()   */
-void *sys_realloc(void *, Uint)      __deprecated; /* erts_realloc_fnf() */
-
 #undef ERTS_HAVE_IS_IN_LITERAL_RANGE
 #if defined(ARCH_32) || defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
 #  define ERTS_HAVE_IS_IN_LITERAL_RANGE
diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
index 28aaeeb479..4347f9f2b7 100644
--- a/erts/emulator/beam/erl_cpu_topology.c
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -827,17 +827,6 @@ erts_sched_bind_atfork_child(int unbind)
     return 0;
 }
 
-char *
-erts_sched_bind_atvfork_child(int unbind)
-{
-    if (unbind) {
-	ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx)
-			   || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
-	return erts_get_unbind_from_cpu_str(cpuinfo);
-    }
-    return "false";
-}
-
 void
 erts_sched_bind_atfork_parent(int unbind)
 {
diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h
index 45324ac4a0..cf139d95a9 100644
--- a/erts/emulator/beam/erl_cpu_topology.h
+++ b/erts/emulator/beam/erl_cpu_topology.h
@@ -85,22 +85,14 @@ void erts_sched_bind_atthrcreate_parent(int unbind);
 
 int erts_sched_bind_atfork_prepare(void);
 int erts_sched_bind_atfork_child(int unbind);
-char *erts_sched_bind_atvfork_child(int unbind);
 void erts_sched_bind_atfork_parent(int unbind);
 
 Eterm erts_fake_scheduler_bindings(Process *p, Eterm how);
 Eterm erts_debug_cpu_groups_map(Process *c_p, int groups);
 
-
 typedef void (*erts_cpu_groups_callback_t)(int,
 					   ErtsSchedulerData *,
 					   int,
 					   void *);
 
-void erts_add_cpu_groups(int groups,
-			 erts_cpu_groups_callback_t callback,
-			 void *arg);
-void erts_remove_cpu_groups(erts_cpu_groups_callback_t callback,
-			    void *arg);
-
 #endif
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 24b22eafb8..d69035cf64 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1290,11 +1290,6 @@ int db_match_keeps_key(int keypos, Eterm match, Eterm guard, Eterm body)
     return 0;
 }
 
-/* This is used when tracing */
-Eterm erts_match_set_lint(Process *p, Eterm matchexpr) {
-    return db_match_set_lint(p, matchexpr, DCOMP_TRACE);
-}
-
 Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags) 
 {
     Eterm l;
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index a991c2c164..6ec6be5a08 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -439,7 +439,7 @@ Eterm
 erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity)
 {
     return erts_gc_after_bif_call_lhf(p, ERTS_INVALID_HFRAG_PTR,
-				      result, regs, arity);
+                                      result, regs, arity);
 }
 
 static ERTS_INLINE void reset_active_writer(Process *p)
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index eaaf5c911a..2527732eb3 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -309,30 +309,6 @@ void erl_error(char *fmt, va_list args)
 
 static int early_init(int *argc, char **argv);
 
-void
-erts_short_init(void)
-{
-    
-    int ncpu;
-    int time_correction;
-    ErtsTimeWarpMode time_warp_mode;
-
-    set_default_time_adj(&time_correction,
-			 &time_warp_mode);
-    ncpu = early_init(NULL, NULL);
-    erl_init(ncpu,
-	     ERTS_DEFAULT_MAX_PROCESSES,
-	     0,
-	     ERTS_DEFAULT_MAX_PORTS,
-	     0,
-	     0,
-	     time_correction,
-	     time_warp_mode,
-	     ERTS_NODE_TAB_DELAY_GC_DEFAULT,
-	     ERTS_DB_SPNCNT_NORMAL);
-    erts_initialized = 1;
-}
-
 static void
 erl_init(int ncpu,
 	 int proc_tab_sz,
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 4836b9e2d3..55526e1d5e 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -2161,13 +2161,6 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
 #endif
 }
 
-int
-erts_port_is_scheduled(Port *pp)
-{
-    erts_aint32_t flags = erts_smp_atomic32_read_acqb(&pp->sched.flags);
-    return (flags & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC)) != 0;
-}
-
 #ifdef ERTS_SMP
 
 void
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index e3550e878e..a48b492ba7 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -265,7 +265,6 @@ int erts_port_task_schedule(Eterm,
 			    ErtsPortTaskType,
 			    ...);
 void erts_port_task_free_port(Port *);
-int erts_port_is_scheduled(Port *);
 ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data(void);
 ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data_extra(size_t extra, void **extra_ptr);
 void erts_port_task_free_p2p_sig_data(ErtsProc2PortSigData *sigdp);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 9947e33f47..a594578793 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1537,12 +1537,6 @@ proclist_destroy(ErtsProcList *plp)
     proclist_free(plp);
 }
 
-ErtsProcList *
-erts_proclist_copy(ErtsProcList *plp)
-{
-    return proclist_copy(plp);
-}
-
 ErtsProcList *
 erts_proclist_create(Process *p)
 {
@@ -3275,13 +3269,6 @@ thr_prgr_fin_wait(void *vssi)
 
 static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp);
 
-void
-erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time)
-{
-    /* TODO only poke when needed (based on timeout_time) */
-    erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(-1));
-}
-
 static void *
 aux_thread(void *unused)
 {
@@ -9315,17 +9302,6 @@ erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
     return pid2proc_not_running(c_p, c_p_locks, pid, pid_locks, 0);
 }
 
-/*
- * Like erts_pid2proc_not_running(), but hands over the process
- * in a suspended state unless (c_p is looked up).
- */
-Process *
-erts_pid2proc_suspend(Process *c_p, ErtsProcLocks c_p_locks,
-		      Eterm pid, ErtsProcLocks pid_locks)
-{
-    return pid2proc_not_running(c_p, c_p_locks, pid, pid_locks, 1);
-}
-
 /*
  * erts_pid2proc_nropt() is normally the same as
  * erts_pid2proc_not_running(). However it is only
@@ -9444,21 +9420,6 @@ handle_pend_bif_async_suspend(Process *suspendee,
     }
 }
 
-#else
-
-/*
- * Non-smp version of erts_pid2proc_suspend().
- */
-Process *
-erts_pid2proc_suspend(Process *c_p, ErtsProcLocks c_p_locks,
-		      Eterm pid, ErtsProcLocks pid_locks)
-{
-    Process *rp = erts_pid2proc(c_p, c_p_locks, pid, pid_locks);
-    if (rp)
-	erts_suspend(rp, pid_locks, NULL);
-    return rp;
-}
-
 #endif /* ERTS_SMP */
 
 /*
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 5b35dc3c78..d6d7750a33 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1606,7 +1606,6 @@ Uint64 erts_ensure_later_proc_interval(Uint64);
 Uint64 erts_step_proc_interval(void);
 
 ErtsProcList *erts_proclist_create(Process *);
-ErtsProcList *erts_proclist_copy(ErtsProcList *);
 void erts_proclist_destroy(ErtsProcList *);
 
 ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *);
@@ -2556,10 +2555,6 @@ ERTS_TIME2REDS_IMPL__(ErtsMonotonicTime start, ErtsMonotonicTime end)
 }
 #endif
 
-Process *erts_pid2proc_suspend(Process *,
-			       ErtsProcLocks,
-			       Eterm,
-			       ErtsProcLocks);
 #ifdef ERTS_SMP
 
 Process *erts_pid2proc_not_running(Process *,
@@ -2601,8 +2596,6 @@ extern int erts_disable_proc_not_running_opt;
 
 void erts_smp_notify_inc_runq(ErtsRunQueue *runq);
 
-void erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time);
-
 #ifdef ERTS_SMP
 void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
 ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index bb4d442240..d6a66ce7cb 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1132,7 +1132,6 @@ extern erts_tid_t erts_main_thread;
 #endif
 extern int erts_compat_rel;
 extern int erts_use_sender_punish;
-void erts_short_init(void);
 void erl_start(int, char**);
 void erts_usage(void);
 Eterm erts_preloaded(Process* p);
@@ -1439,7 +1438,6 @@ do {								\
 #define MatchSetGetSource(MPSP) erts_match_set_get_source(MPSP)
 
 extern Binary *erts_match_set_compile(Process *p, Eterm matchexpr, Eterm MFA);
-Eterm erts_match_set_lint(Process *p, Eterm matchexpr); 
 extern void erts_match_set_release_result(Process* p);
 ERTS_GLB_INLINE void erts_match_set_release_result_trace(Process* p, Eterm);
 
-- 
cgit v1.2.3


From a1956797340a09cdf35d60cc6ccf255c9173d3f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 17 May 2017 11:43:11 +0200
Subject: Eliminate warning for variable 'Config' being unused

---
 erts/emulator/test/call_trace_SUITE.erl | 2 +-
 erts/emulator/test/driver_SUITE.erl     | 2 +-
 erts/emulator/test/mtx_SUITE.erl        | 2 +-
 erts/emulator/test/signal_SUITE.erl     | 2 +-
 erts/emulator/test/statistics_SUITE.erl | 4 ++--
 erts/emulator/test/trace_SUITE.erl      | 4 ++--
 erts/emulator/test/trace_meta_SUITE.erl | 2 +-
 7 files changed, 9 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl
index 1216863c51..95171d04ce 100644
--- a/erts/emulator/test/call_trace_SUITE.erl
+++ b/erts/emulator/test/call_trace_SUITE.erl
@@ -60,7 +60,7 @@ all() ->
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Config.
 
-end_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, _Config) ->
     %% Reloading the module will clear all trace patterns, and
     %% in a debug-compiled emulator run assertions of the counters
     %% for the number of traced exported functions in this module.
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 2fbf6eae61..e854a5f945 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -127,7 +127,7 @@ init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) ->
     0 = element(1, erts_debug:get_internal_state(check_io_debug)),
     [{testcase, Case}|Config].
 
-end_per_testcase(Case, Config) ->
+end_per_testcase(Case, _Config) ->
     erlang:display({end_per_testcase, Case}),
     0 = element(1, erts_debug:get_internal_state(check_io_debug)),
     ok.
diff --git a/erts/emulator/test/mtx_SUITE.erl b/erts/emulator/test/mtx_SUITE.erl
index 12928ed6d8..0d6ab5cdb2 100644
--- a/erts/emulator/test/mtx_SUITE.erl
+++ b/erts/emulator/test/mtx_SUITE.erl
@@ -97,7 +97,7 @@ init_per_testcase(_Case, Config) ->
     wait_deallocations(),
     Config.
 
-end_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, _Config) ->
     ok.
 
 wait_deallocations() ->
diff --git a/erts/emulator/test/signal_SUITE.erl b/erts/emulator/test/signal_SUITE.erl
index 7e516176f7..d788360812 100644
--- a/erts/emulator/test/signal_SUITE.erl
+++ b/erts/emulator/test/signal_SUITE.erl
@@ -53,7 +53,7 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     available_internal_state(true),
     [{testcase, Func}|Config].
 
-end_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, _Config) ->
     ok.
 
 init_per_suite(Config) ->
diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl
index 3057905f4c..0cea941687 100644
--- a/erts/emulator/test/statistics_SUITE.erl
+++ b/erts/emulator/test/statistics_SUITE.erl
@@ -396,7 +396,7 @@ msb_swt_hog(false) ->
     count(1000000),
     msb_swt_hog(false).
 
-msb_scheduler_wall_time(Config) ->
+msb_scheduler_wall_time(_Config) ->
     erlang:system_flag(scheduler_wall_time, true),
     Dirty = erlang:system_info(dirty_cpu_schedulers) /= 0,
     Hogs = lists:map(fun (_) ->
@@ -493,7 +493,7 @@ badarg(Config) when is_list(Config) ->
 tok_loop() ->
     tok_loop().
 
-run_queues_lengths_active_tasks(Config) ->
+run_queues_lengths_active_tasks(_Config) ->
     TokLoops = lists:map(fun (_) ->
                                  spawn_opt(fun () ->
                                                    tok_loop()
diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl
index f846b0f4b9..643c2e0472 100644
--- a/erts/emulator/test/trace_SUITE.erl
+++ b/erts/emulator/test/trace_SUITE.erl
@@ -733,7 +733,7 @@ set_on_first_spawn(Config) when is_list(Config) ->
 
 %% Tests trace(Pid, How, [set_on_link]).
 
-set_on_link(Config) ->
+set_on_link(_Config) ->
     Listener = fun_spawn(fun process/0),
 
     %% Create and trace a process with the set_on_link flag.
@@ -756,7 +756,7 @@ set_on_link(Config) ->
 
 %% Tests trace(Pid, How, [set_on_first_spawn]).
 
-set_on_first_link(Config) ->
+set_on_first_link(_Config) ->
     ct:timetrap({seconds, 10}),
     Listener = fun_spawn(fun process/0),
 
diff --git a/erts/emulator/test/trace_meta_SUITE.erl b/erts/emulator/test/trace_meta_SUITE.erl
index b6a6fd5404..ef58978749 100644
--- a/erts/emulator/test/trace_meta_SUITE.erl
+++ b/erts/emulator/test/trace_meta_SUITE.erl
@@ -74,7 +74,7 @@ config(priv_dir,_) ->
 init_per_testcase(_Case, Config) ->
     Config.
 
-end_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, _Config) ->
     shutdown(),
     ok.
 
-- 
cgit v1.2.3


From caf36b503f6c43494a89e196a59aa69ad3156087 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 17 May 2017 11:48:07 +0200
Subject: code_SUITE: Remove unused functions

---
 erts/emulator/test/code_SUITE.erl | 21 ---------------------
 1 file changed, 21 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index d07166ed98..35f7baf2cf 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -229,27 +229,6 @@ multi_proc_purge(Config) when is_list(Config) ->
 		   Pid9, Pid10, Pid11, Pid12, Pid13, Pid14, Pid15, Pid16]),
     ok.
 
-body(F, Fakes) ->
-    receive
-        jog ->
-            40 = F(3),
-            erlang:garbage_collect(),
-            body(F, Fakes);
-        drop_funs ->
-            dropped_body()
-    end.
-
-dropped_body() ->
-    receive
-        X -> exit(X)
-    end.
-
-gc() ->
-    erlang:garbage_collect(),
-    gc1().
-gc1() -> ok.
-
-
 %% Test the erlang:check_old_code/1 BIF.
 t_check_old_code(Config) when is_list(Config) ->
     Data = proplists:get_value(data_dir, Config),
-- 
cgit v1.2.3


From a3407eaa2104d634e4400c0c805202da462ee66f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 18 May 2017 07:17:22 +0200
Subject: Eliminate the -gen_dest macro flag

Instructions that take a 'd' argument needs a -gen_dest flag in their
macros. For example:

  %macro:put_list PutList -pack -gen_dest
  put_list s s d

-gen_dest was needed when x(0) was stored in a register, since it is
not possible to take the address of a register. Now that x(0) is stored
in memory and we can take the address, we can eliminate gen_dest.
---
 erts/emulator/beam/beam_emu.c    | 74 +++++++++++++++++++---------------------
 erts/emulator/beam/ops.tab       | 12 +++----
 erts/emulator/utils/beam_makeops | 15 ++------
 3 files changed, 44 insertions(+), 57 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index a90e6a0ba8..79d751d13e 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -158,7 +158,9 @@ do {                                     \
 /*
  * Register target (X or Y register).
  */
-#define REG_TARGET(Target) (*(((Target) & 1) ? &yb(Target-1) : &xb(Target)))
+
+#define REG_TARGET_PTR(Target) (((Target) & 1) ? &yb(Target-1) : &xb(Target))
+#define REG_TARGET(Target) (*REG_TARGET_PTR(Target))
 
 /*
  * Store a result into a register given a destination descriptor.
@@ -172,8 +174,6 @@ do {                                     \
     REG_TARGET(stb_reg) = (Result);		\
   } while (0)
 
-#define StoreSimpleDest(Src, Dest) Dest = (Src)
-
 /*
  * Store a result into a register and execute the next instruction.
  * Dst points to the word with a destination descriptor, which MUST
@@ -420,7 +420,7 @@ void** beam_ops;
 #define TestHeapPutList(Need, Reg)		\
   do {						\
      TestHeap((Need), 1);			\
-     PutList(Reg, r(0), r(0), StoreSimpleDest);	\
+     PutList(Reg, r(0), r(0));                  \
      CHECK_TERM(r(0));				\
   } while (0)
 
@@ -547,11 +547,11 @@ void** beam_ops;
      GetR((N)+1, Dst2);            \
    } while (0)
 
-#define PutList(H, T, Dst, Store)  \
-  do {                             \
-   HTOP[0] = (H); HTOP[1] = (T);   \
-   Store(make_list(HTOP), Dst);    \
-   HTOP += 2;                      \
+#define PutList(H, T, Dst)                      \
+  do {                                          \
+   HTOP[0] = (H); HTOP[1] = (T);                \
+   Dst = make_list(HTOP);                       \
+   HTOP += 2;                                   \
   } while (0)
 
 #define Swap(R1, R2)				\
@@ -568,11 +568,7 @@ void** beam_ops;
     R2 = Tmp = V;				\
   } while (0)
 
-#define Move(Src, Dst, Store)      \
-   do {                            \
-       Eterm term = (Src);         \
-       Store(term, Dst);           \
-   } while (0)
+#define Move(Src, Dst) Dst = (Src)
 
 #define Move2Par(S1, D1, S2, D2)		\
   do {						\
@@ -911,7 +907,7 @@ do {                                            \
       Target = _uint_size * Unit;				\
    } while (0)
 
-#define BsGetFloat2(Ms, Live, Sz, Flags, Dst, Store, Fail)		\
+#define BsGetFloat2(Ms, Live, Sz, Flags, Dst, Fail)                     \
  do {									\
    ErlBinMatchBuffer *_mb;						\
    Eterm _result; Sint _size;						\
@@ -922,12 +918,12 @@ do {                                            \
    LIGHT_SWAPOUT;							\
    _result = erts_bs_get_float_2(c_p, _size, (Flags), _mb);		\
    LIGHT_SWAPIN;							\
-   HEAP_SPACE_VERIFIED(0);                                          \
+   HEAP_SPACE_VERIFIED(0);                                              \
    if (is_non_value(_result)) { Fail; }					\
-   else { Store(_result, Dst); }					\
+   else { Dst = _result; }                                              \
  } while (0)
 
-#define BsGetBinaryImm_2(Ms, Live, Sz, Flags, Dst, Store, Fail)	\
+#define BsGetBinaryImm_2(Ms, Live, Sz, Flags, Dst, Fail)	\
   do {								\
     ErlBinMatchBuffer *_mb;					\
     Eterm _result;						\
@@ -936,12 +932,12 @@ do {                                            \
     LIGHT_SWAPOUT;						\
     _result = erts_bs_get_binary_2(c_p, (Sz), (Flags), _mb);	\
     LIGHT_SWAPIN;						\
-    HEAP_SPACE_VERIFIED(0);                                 \
+    HEAP_SPACE_VERIFIED(0);                                     \
     if (is_non_value(_result)) { Fail; }			\
-    else { Store(_result, Dst); }				\
+    else { Dst = _result; }                                     \
   } while (0)
 
-#define BsGetBinary_2(Ms, Live, Sz, Flags, Dst, Store, Fail)	\
+#define BsGetBinary_2(Ms, Live, Sz, Flags, Dst, Fail)           \
   do {								\
     ErlBinMatchBuffer *_mb;					\
     Eterm _result; Uint _size;					\
@@ -951,27 +947,27 @@ do {                                            \
     LIGHT_SWAPOUT;						\
     _result = erts_bs_get_binary_2(c_p, _size, (Flags), _mb);	\
     LIGHT_SWAPIN;						\
-    HEAP_SPACE_VERIFIED(0);                                 \
+    HEAP_SPACE_VERIFIED(0);                                     \
     if (is_non_value(_result)) { Fail; }			\
-    else { Store(_result, Dst); }				\
+    else { Dst = _result; }                                     \
   } while (0)
 
-#define BsGetBinaryAll_2(Ms, Live, Unit, Dst, Store, Fail)	\
-  do {								\
-    ErlBinMatchBuffer *_mb;					\
-    Eterm _result;						\
-    TestHeap(ERL_SUB_BIN_SIZE, Live);				\
-    _mb = ms_matchbuffer(Ms);					\
-    if (((_mb->size - _mb->offset) % Unit) == 0) {		\
-      LIGHT_SWAPOUT;						\
-      _result = erts_bs_get_binary_all_2(c_p, _mb);		\
-      LIGHT_SWAPIN;						\
-      HEAP_SPACE_VERIFIED(0);                               \
-      ASSERT(is_value(_result));				\
-      Store(_result, Dst);					\
-    } else {                                                    \
-	HEAP_SPACE_VERIFIED(0);                             \
-	Fail; }						        \
+#define BsGetBinaryAll_2(Ms, Live, Unit, Dst, Fail)	\
+  do {                                                  \
+    ErlBinMatchBuffer *_mb;                             \
+    Eterm _result;                                      \
+    TestHeap(ERL_SUB_BIN_SIZE, Live);                   \
+    _mb = ms_matchbuffer(Ms);                           \
+    if (((_mb->size - _mb->offset) % Unit) == 0) {      \
+      LIGHT_SWAPOUT;                                    \
+      _result = erts_bs_get_binary_all_2(c_p, _mb);     \
+      LIGHT_SWAPIN;                                     \
+      HEAP_SPACE_VERIFIED(0);                           \
+      ASSERT(is_value(_result));                        \
+      Dst = _result;					\
+    } else {                                            \
+	HEAP_SPACE_VERIFIED(0);                         \
+	Fail; }                                         \
  } while (0)
 
 #define BsSkipBits2(Ms, Bits, Unit, Fail)			\
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 9b5bd7a749..896ab82a50 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -409,7 +409,7 @@ move3 x x x x x x
 move S=n D=y => init D
 move S=c D=y => move S x | move x D
 
-%macro:move Move -pack -gen_dest
+%macro:move Move -pack
 move x x
 move x y
 move y x
@@ -538,7 +538,7 @@ i_put_tuple y I
 #
 put_list Const=c n Dst => move Const x | put_list x n Dst
 
-%macro:put_list PutList -pack -gen_dest
+%macro:put_list PutList -pack
 
 put_list x n x
 put_list y n x
@@ -1209,9 +1209,9 @@ i_bs_get_integer_32 x f I d
 bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \
 			gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
 
-%macro: i_bs_get_binary_imm2 BsGetBinaryImm_2 -fail_action -gen_dest
-%macro: i_bs_get_binary2 BsGetBinary_2 -fail_action -gen_dest
-%macro: i_bs_get_binary_all2 BsGetBinaryAll_2 -fail_action -gen_dest
+%macro: i_bs_get_binary_imm2 BsGetBinaryImm_2 -fail_action
+%macro: i_bs_get_binary2 BsGetBinary_2 -fail_action
+%macro: i_bs_get_binary_all2 BsGetBinaryAll_2 -fail_action
 
 i_bs_get_binary_imm2 f x I I I d
 i_bs_get_binary2 f x I s I d
@@ -1224,7 +1224,7 @@ bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \
 
 bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail
 
-%macro: i_bs_get_float2 BsGetFloat2 -fail_action -gen_dest
+%macro: i_bs_get_float2 BsGetFloat2 -fail_action
 i_bs_get_float2 f x I s I d
 
 # Miscellanous
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 9813142585..b2bc9a17a8 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -928,7 +928,6 @@ sub basic_generator {
     my($tmp_arg_num) = 1;
     my($pack_spec) = '';
     my($var_decls) = '';
-    my($gen_dest_arg) = 'StoreSimpleDest';
     my($i);
     my($no_prefetch) = 0;
 
@@ -994,11 +993,11 @@ sub basic_generator {
 			    push(@f_types, $_);
 			    $prefix .= "GetR($size, $tmp);\n";
 			    last SWITCH; };
-	    /d/    and do { $var_decls .= "Eterm dst; ";
-			    push(@f, "dst");
+	    /d/    and do { $var_decls .= "Eterm dst; Eterm* dst_ptr; ";
+			    push(@f, "*dst_ptr");
 			    push(@f_types, $_);
 			    $prefix .= "dst = Arg($size);\n";
-			    $gen_dest_arg = 'StoreResult';
+			    $prefix .= "dst_ptr = REG_TARGET_PTR(dst);\n";
 			    last SWITCH;
 			};
 	    defined($incl_arg{$_})
@@ -1016,14 +1015,6 @@ sub basic_generator {
 	$size += $this_size;
     }
 
-    #
-    # If requested, pass a pointer to the destination register.
-    # The destination must be the last operand.
-    #
-    if ($flags =~ /-gen_dest/) {
-	push(@f, $gen_dest_arg);
-    }
-
     #
     # Add a fail action macro if requested.
     #
-- 
cgit v1.2.3


From 027b178bdc20f4a4b813d030bc7cb7f834d2980e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 18 May 2017 10:26:40 +0200
Subject: Modernize subroutine calls by removing '&'

In Perl 5, '&' on direct subroutine calls are optional.
---
 erts/emulator/utils/beam_makeops | 110 +++++++++++++++++++--------------------
 1 file changed, 55 insertions(+), 55 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index b2bc9a17a8..bfa5236764 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -332,9 +332,9 @@ while (<>) {
     if (/^\%macro:(.*)/) {
 	my($op, $macro, @flags) = split(' ', $1);
 	defined($macro) and $macro =~ /^-/ and
-	    &error("A macro must not start with a hyphen");
+	    error("A macro must not start with a hyphen");
 	foreach (@flags) {
-	    /^-/ or &error("Flags for macros should start with a hyphen");
+	    /^-/ or error("Flags for macros should start with a hyphen");
 	}
 	error("Macro for '$op' is already defined")
 	    if defined $macro{$op};
@@ -347,7 +347,7 @@ while (<>) {
     # Handle transformations.
     #
     if (/=>/) {
-	&parse_transformation($_);
+	parse_transformation($_);
 	next;
     }
 
@@ -357,8 +357,8 @@ while (<>) {
     $op_num = undef;
     if (s/^(\d+):\s*//) {
 	$op_num = $1;
-	$op_num != 0 or &error("Opcode 0 invalid");
-	&error("Opcode $op_num already defined")
+	$op_num != 0 or error("Opcode 0 invalid");
+	error("Opcode $op_num already defined")
 	    if defined $gen_opname[$op_num];
     }
 
@@ -369,11 +369,11 @@ while (<>) {
 	my($obsolete) = $1;
 	my($name) = $2;
 	my($arity) = $3;
-	$name =~ /^[a-z]/ or &error("Opname must start with a lowercase letter");
+	$name =~ /^[a-z]/ or error("Opname must start with a lowercase letter");
 	defined $gen_arity{$name} and $gen_arity{$name} != $arity and
-	    &error("Opname $name already defined with arity $gen_arity{$name}");
+	    error("Opname $name already defined with arity $gen_arity{$name}");
 	defined $unnumbered{$name,$arity} and
-	    &error("Opname $name already defined with arity $gen_arity{$name}");
+	    error("Opname $name already defined with arity $gen_arity{$name}");
 	
 	if (defined $op_num) {	# Numbered generic operation
 	    $gen_opname[$op_num] = $name;
@@ -395,16 +395,16 @@ while (<>) {
     #    Name Arg1 Arg2...
     #
     my($name, @args) = split;
-    &error("too many operands")
+    error("too many operands")
 	if @args > $max_spec_operands;
-    &syntax_check($name, @args);
+    syntax_check($name, @args);
     my $arity = @args;
     if (defined $gen_opnum{$name,$arity} and $obsolete[$gen_opnum{$name,$arity}]) {
 	error("specific instructions may not be specified for obsolete instructions");
     }
     push(@{$specific_op{"$name/$arity"}}, [$name, $hot, @args]);
     if (defined $op_num) {
-	&error("specific instructions must not be numbered");
+	error("specific instructions must not be numbered");
     } elsif (!defined($gen_arity{$name}) && !defined($unnumbered{$name,$arity})) {
 	#
 	# Create an unumbered generic instruction too.
@@ -446,7 +446,7 @@ $num_file_opcodes = @gen_opname;
 # Produce output for the chosen target.
 #
 
-&$target;
+&$target();
 
 #
 # Produce output needed by the emulator/loader.
@@ -462,7 +462,7 @@ sub emulator_output {
     #
     $name = "$outdir/beam_opcodes.c";
     open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
-    &comment('C');
+    comment('C');
     print "#ifdef HAVE_CONFIG_H\n";
     print "#  include \"config.h\"\n";
     print "#endif\n\n";
@@ -525,7 +525,7 @@ sub emulator_output {
 	    # Call a generator to calculate size and generate macros
 	    # for the emulator.
 	    #
-	    my($size, $code, $pack) = &basic_generator($name, $hot, @args);
+	    my($size, $code, $pack) = basic_generator($name, $hot, @args);
 
 	    #
 	    # Save the generated $code for later.
@@ -587,7 +587,7 @@ sub emulator_output {
     # Generate transformations.
     #
 
-    &tr_gen(@transformations);
+    tr_gen(@transformations);
 
     #
     # Print the generic instruction table.
@@ -602,7 +602,7 @@ sub emulator_output {
 	my($arity) = $gen_arity[$i];
 	printf "/* %3d */  ", $i;
 	if (!defined $name) {
-	    &init_item("", 0, 0, 0, -1);
+	    init_item("", 0, 0, 0, -1);
 	} else {
 	    my($key) = "$name/$arity";
 	    my($tr) = defined $gen_transform_offset{$key} ?
@@ -614,7 +614,7 @@ sub emulator_output {
 		$is_transformed{$name,$arity} or
 		error("instruction $key has no specific instruction");
 	    $spec_op = -1 unless defined $spec_op;
-	    &init_item($name, $arity, $spec_op, $num_specific, $tr);
+	    init_item($name, $arity, $spec_op, $num_specific, $tr);
 	}
     }
     print "};\n";
@@ -624,7 +624,7 @@ sub emulator_output {
     #
     $name = "$outdir/beam_opcodes.h";
     open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
-    &comment('C');
+    comment('C');
     print "#ifndef __OPCODES_H__\n";
     print "#define __OPCODES_H__\n\n";
 
@@ -662,14 +662,14 @@ sub emulator_output {
     my $letter;
     my $tag_num = 0;
 
-    &comment('C', "The following operand types for generic instructions",
+    comment('C', "The following operand types for generic instructions",
 	     "occur in beam files.");
     foreach $letter (split('', $compiler_types)) {
 	print "#define TAG_$letter $tag_num\n";
 	$tag_num++;
     }
     print "\n";
-    &comment('C', "The following operand types are only used in the loader.");
+    comment('C', "The following operand types are only used in the loader.");
     foreach $letter (split('', $loader_types)) {
 	print "#define TAG_$letter $tag_num\n";
 	$tag_num++;
@@ -736,26 +736,26 @@ sub emulator_output {
 
     $name = "$outdir/beam_tr_funcs.h";
     open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
-    &comment('C');
-    &tr_gen_call(@call_table);
+    comment('C');
+    tr_gen_call(@call_table);
 
     $name = "$outdir/beam_pred_funcs.h";
     open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
-    &comment('C');
-    &tr_gen_call(@pred_table);
+    comment('C');
+    tr_gen_call(@pred_table);
 
     #
     # Implementation of operations for emulator.
     #
     $name = "$outdir/beam_hot.h";
     open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
-    &comment('C');
-    &print_code(\%hot_code);
+    comment('C');
+    print_code(\%hot_code);
 
     $name = "$outdir/beam_cold.h";
     open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
-    &comment('C');
-    &print_code(\%cold_code);
+    comment('C');
+    print_code(\%cold_code);
 
 }
 
@@ -818,7 +818,7 @@ sub compiler_output {
 
     open(STDOUT, ">$outdir/$name") || die "Failed to open $name for writing: $!\n";
     print "-module($module).\n";
-    &comment('erlang');
+    comment('erlang');
 
     print "-export([format_number/0]).\n";
     print "-export([opcode/2,opname/1]).\n";
@@ -830,7 +830,7 @@ sub compiler_output {
     for ($i = 0; $i < @gen_opname; $i++) {
 	next unless defined $gen_opname[$i];
 	print "%%" if $obsolete[$i];
-	print "opcode(", &quote($gen_opname[$i]), ", $gen_arity[$i]) -> $i;\n";
+	print "opcode(", quote($gen_opname[$i]), ", $gen_arity[$i]) -> $i;\n";
     }
     print "opcode(Name, Arity) -> erlang:error(badarg, [Name,Arity]).\n\n";
 
@@ -838,7 +838,7 @@ sub compiler_output {
     for ($i = 0; $i < @gen_opname; $i++) {
 	next unless defined $gen_opname[$i];
 	print "opname($i) -> {",
-	&quote($gen_opname[$i]), ",$gen_arity[$i]};\n";
+	quote($gen_opname[$i]), ",$gen_arity[$i]};\n";
     }
     print "opname(Number) -> erlang:error(badarg, [Number]).\n";
 
@@ -847,7 +847,7 @@ sub compiler_output {
     #
     my($hrl_name) = "$outdir/${module}.hrl";
     open(STDOUT, ">$hrl_name") || die "Failed to open $hrl_name for writing: $!\n";
-    &comment('erlang');
+    comment('erlang');
 
     for ($i = 0; $i < @tag_type && $i < 8; $i++) {
 	print "-define(tag_$tag_type[$i], $i).\n";
@@ -863,10 +863,10 @@ sub syntax_check {
     my($name, @args) = @_;
     my($i);
 
-    &error("Bad opcode name '$name'")
+    error("Bad opcode name '$name'")
 	unless $name =~ /^[a-z][\w\d_]*$/;
     for ($i = 0; $i < @args; $i++) {
-	&error("Argument " . ($i+1) . ": invalid type '$args[$i]'")
+	error("Argument " . ($i+1) . ": invalid type '$args[$i]'")
 	    unless defined $arg_size{$args[$i]};
     }
 }
@@ -963,7 +963,7 @@ sub basic_generator {
     #
 
     if ($flags =~ /-pack/ && $hot) {
-        ($prefix, $pack_spec, @args) = &do_pack(@args);
+        ($prefix, $pack_spec, @args) = do_pack(@args);
     }
 
     #
@@ -1186,7 +1186,7 @@ sub do_pack {
 	    }
 
 	    $down = "$instr[$ap]$down";
-	    my($unpack) = &make_unpack($tmpnum, $shift[$ap], $mask[$ap]);
+	    my($unpack) = make_unpack($tmpnum, $shift[$ap], $mask[$ap]);
 	    $args[$i] = "pack:$this_size:$reg" . "b($unpack)";
 
 	    if (++$ap == $args_per_word) {
@@ -1250,7 +1250,7 @@ sub parse_transformation {
     foreach (@from) {
 	if (/^(\w+)\((.*?)\)/) {
 	    my($name, $arglist) = ($1, $2);
-	    $_ = (&compile_transform_function($name, split(/\s*,\s*/, $arglist)));
+	    $_ = (compile_transform_function($name, split(/\s*,\s*/, $arglist)));
 	} else {
 	    (@op) = split;
 	    ($rest_var,$_) = compile_transform(1, $rest_var, @op);
@@ -1266,7 +1266,7 @@ sub parse_transformation {
     my @to;
     if ($to =~ /^(\w+)\((.*?)\)/) {
 	my($name, $arglist) = ($1, $2);
-	@to = (&compile_transform_function($name, split(/\s*,\s*/, $arglist)));
+	@to = (compile_transform_function($name, split(/\s*,\s*/, $arglist)));
     } else {
 	@to = split(/\s*\|\s*/, $to);
 	foreach (@to) {
@@ -1288,7 +1288,7 @@ sub compile_transform {
     my $arity = 0;
 
     foreach (@ops) {
-	my(@list) = &tr_parse_op($src, $_);
+	my(@list) = tr_parse_op($src, $_);
 	if ($list[1] eq '*') {
 	    $rest_var = $list[0];
 	} elsif (defined $rest_var and $list[0] eq $rest_var) {
@@ -1325,7 +1325,7 @@ sub tr_parse_op {
     if (/^([A-Z]\w*)(.*)/) {
 	$var = $1;
 	$_ = $2;
-	&error("garbage after variable")
+	error("garbage after variable")
 	    unless /^=(.*)/ or /^(\s*)$/;
 	$_ = $1;
     }
@@ -1336,7 +1336,7 @@ sub tr_parse_op {
 	$type = $1;
 	$_ = $2;
 	foreach (split('', $type)) {
-	    &error("bad type in $op")
+	    error("bad type in $op")
 		unless defined $type_bit{$_} or $type eq '*';
 	    $_ eq 'r' and
 		error("$op: 'r' is not allowed in transformations")
@@ -1383,7 +1383,7 @@ sub tr_parse_op {
 
     # Nothing more is allowed after the command.
 
-    &error("garbage '$_' after operand: $op")
+    error("garbage '$_' after operand: $op")
 	unless /^\s*$/;
 
     # Test that destination has no conditions.
@@ -1510,7 +1510,7 @@ sub tr_gen_from {
 	# Check that $name/$arity refers to a valid generic instruction.
 	#
 
-	&error($where, "invalid generic op $name/$arity")
+	error($where, "invalid generic op $name/$arity")
 	    unless defined $gen_opnum{$name,$arity};
 	$opnum = $gen_opnum{$name,$arity};
 
@@ -1538,11 +1538,11 @@ sub tr_gen_from {
 			$type_mask |= $type_bit{$_};
 		    }
 		    if ($cond ne 'is_eq') {
-			push(@code, &make_op($types, 'is_type', $type_mask));
+			push(@code, make_op($types, 'is_type', $type_mask));
 		    } else {
 			$cond = '';
-			push(@code, &make_op("$types== $val", 'is_type_eq',
-					     $type_mask, $val));
+			push(@code, make_op("$types== $val", 'is_type_eq',
+                                            $type_mask, $val));
 		    }
 		}
 	    }
@@ -1551,12 +1551,12 @@ sub tr_gen_from {
 		my($m, $f, $a) = split(/:/, $val);
 		$ignored_var = '';
 		$may_fail = 1;
-		push(@code, &make_op('', "$cond", "am_$m",
+		push(@code, make_op('', "$cond", "am_$m",
 				     "am_$f", $a));
 	    } elsif ($cond ne '') {
 		$ignored_var = '';
 		$may_fail = 1;
-		push(@code, &make_op('', "$cond", $val));
+		push(@code, make_op('', "$cond", $val));
 	    }
 
 	    if ($var ne '') {
@@ -1582,7 +1582,7 @@ sub tr_gen_from {
 		    $var_type{$var} = 'scalar';
 		    $var{$var} = $var_num;
 		    $var_num++;
-		    push(@code, &make_op($var, 'set_var', $var{$var}));
+		    push(@code, make_op($var, 'set_var', $var{$var}));
 		}
 	    }
 	    if (is_instr($code[$#code], 'set_var')) {
@@ -1591,7 +1591,7 @@ sub tr_gen_from {
 		my $var = $ref->[1][1];
 		push(@code, make_op($comment, 'set_var_next_arg', $var));
 	    } else {
-		push(@code, &make_op($ignored_var, 'next_arg'));
+		push(@code, make_op($ignored_var, 'next_arg'));
 	    }
 	}
 
@@ -1636,7 +1636,7 @@ sub tr_gen_to {
 	    my(@args);
 
 	    foreach $var (@ops) {
-		&error($where, "variable '$var' unbound")
+		error($where, "variable '$var' unbound")
 		    unless defined $var{$var};
 		if ($var_type{$var} eq 'scalar') {
 		    push(@args, "var[$var{$var}]");
@@ -1659,7 +1659,7 @@ sub tr_gen_to {
 	#
 
 	my($key) = "$name/$arity";
-	&error($where, "invalid generic op $name/$arity")
+	error($where, "invalid generic op $name/$arity")
 	    unless defined $gen_opnum{$name,$arity};
 	my $opnum = $gen_opnum{$name,$arity};
 
@@ -1674,15 +1674,15 @@ sub tr_gen_to {
 	    if ($type eq '*') {
 		push(@code, make_op($var, 'store_rest_args'));
 	    } elsif ($var ne '') {
-		&error($where, "variable '$var' unbound")
+		error($where, "variable '$var' unbound")
 		    unless defined $var{$var};
 		my $op = make_op($var, 'store_var_next_arg', $var{$var});
 		op_slot_usage($op, $var{$var});
 		push(@code, $op);
 	    } elsif ($type ne '') {
-		push(@code, &make_op('', 'store_type', "TAG_$type"));
+		push(@code, make_op('', 'store_type', "TAG_$type"));
 		if ($type_val) {
-		    push(@code, &make_op('', 'store_val', $type_val));
+		    push(@code, make_op('', 'store_val', $type_val));
 		}
 		push(@code, make_op('', 'next_arg'));
 	    }
-- 
cgit v1.2.3


From 29aea1a92e25ee47e55b4a3c48edfc62e8aa4a42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 18 May 2017 12:24:07 +0200
Subject: Allow multiple types per argument for specific instructions

Inroduce syntactic sugar so that we can write:

get_list xy xy xy

instead of:

get_list x x x
get_list x x y
get_list x y x
get_list x y y
get_list y x x
get_list y x y
get_list y y x
get_list y y y
---
 erts/emulator/beam/ops.tab       | 116 +++++++++++----------------------------
 erts/emulator/utils/beam_makeops |  28 +++++++++-
 2 files changed, 57 insertions(+), 87 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 896ab82a50..8abe871c14 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -182,15 +182,7 @@ i_jump_on_val x f I I
 i_jump_on_val y f I I
 
 %macro: get_list GetList -pack
-get_list x x x
-get_list x x y
-get_list x y x
-get_list x y y
-
-get_list y x x
-get_list y x y
-get_list y y x
-get_list y y y
+get_list xy xy xy
 
 # The following get_list instructions using x(0) are frequently used.
 get_list r x x
@@ -218,12 +210,10 @@ set_tuple_element s d P
 # Get tuple element
 
 %macro: i_get_tuple_element GetTupleElement -pack
-i_get_tuple_element x P x
-i_get_tuple_element y P x
+i_get_tuple_element xy P x
 
 %cold
-i_get_tuple_element x P y
-i_get_tuple_element y P y
+i_get_tuple_element xy P y
 %hot
 
 %macro: i_get_tuple_element2 GetTupleElement2 -pack
@@ -270,11 +260,7 @@ system_limit j
 move C=cxy x==0 | jump Lbl => move_jump Lbl C
 
 %macro: move_jump MoveJump -nonext
-move_jump f n
-move_jump f c
-move_jump f x
-move_jump f y
-
+move_jump f ncxy
 
 # Movement to and from the stack is common
 # Try to pack as much as we can into one instruction
@@ -326,12 +312,10 @@ swap_temp R1 R2 Tmp | line Loc | call_ext_last Live Addr D | \
   is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_last Live Addr D
 
 %macro: swap_temp SwapTemp -pack
-swap_temp x x x
-swap_temp x y x
+swap_temp x xy x
 
 %macro: swap Swap -pack
-swap x x
-swap x y
+swap x xy
 
 move Src=x D1=x | move Src=x D2=x => move_dup Src D1 D2
 move Src=x SD=x | move SD=x D=x   => move_dup Src SD D
@@ -381,10 +365,7 @@ move_shift x y x
 move_shift x x y
 
 %macro: move_dup MoveDup -pack
-move_dup x x x
-move_dup x x y
-move_dup y x x
-move_dup y x y
+move_dup xy x xy
 
 %macro: move2_par Move2Par -pack
 
@@ -483,8 +464,7 @@ i_is_ne_exact_literal f y c
 
 is_eq_exact Lbl Y=y X=x => is_eq_exact Lbl X Y
 %macro: is_eq_exact EqualExact -fail_action -pack
-is_eq_exact f x x
-is_eq_exact f x y
+is_eq_exact f x xy
 is_eq_exact f s s
 
 %macro: is_lt IsLessThan -fail_action
@@ -630,9 +610,7 @@ is_tagged_tuple Fail=f c Arity Atom  => jump Fail
 
 %macro:is_tagged_tuple IsTaggedTuple -fail_action
 
-is_tagged_tuple f r A a
-is_tagged_tuple f x A a
-is_tagged_tuple f y A a
+is_tagged_tuple f rxy A a
 
 # Test tuple & arity (head)
 
@@ -642,21 +620,16 @@ is_tuple Fail=f S=xy | test_arity Fail=f S=xy Arity => is_tuple_of_arity Fail S
 
 %macro:is_tuple_of_arity IsTupleOfArity -fail_action
 
-is_tuple_of_arity f r A
-is_tuple_of_arity f x A
-is_tuple_of_arity f y A
+is_tuple_of_arity f rxy A
 
 %macro: is_tuple IsTuple -fail_action
-is_tuple f r
-is_tuple f x
-is_tuple f y
+is_tuple f rxy
 
 test_arity Fail Literal=q Arity => move Literal x | test_arity Fail x Arity
 test_arity Fail=f c Arity => jump Fail
 
 %macro: test_arity IsArity -fail_action
-test_arity f x A
-test_arity f y A
+test_arity f xy A
 
 get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \
    get_tuple_element Reg=x P3 D3=x | \
@@ -681,8 +654,7 @@ is_integer Fail=f S=x | allocate Need Regs => is_integer_allocate Fail S Need Re
 is_integer_allocate f x I I
 
 %macro: is_integer IsInteger -fail_action
-is_integer f x
-is_integer f y
+is_integer f xy
 
 is_list Fail=f n =>
 is_list Fail Literal=q => move Literal x | is_list Fail x
@@ -696,8 +668,7 @@ is_list f y
 is_nonempty_list Fail=f S=x | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs
 
 %macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -pack
-is_nonempty_list_allocate f r I t
-is_nonempty_list_allocate f x I t
+is_nonempty_list_allocate f rx I t
 
 is_nonempty_list F=f x==0 | test_heap I1 I2 => is_non_empty_list_test_heap F I1 I2
 
@@ -708,12 +679,10 @@ is_nonempty_list Fail=f S=x | get_list S D1=x D2=x => \
   is_nonempty_list_get_list Fail S D1 D2
 
 %macro: is_nonempty_list_get_list IsNonemptyListGetList -fail_action -pack
-is_nonempty_list_get_list f r x x
-is_nonempty_list_get_list f x x x
+is_nonempty_list_get_list f rx x x
 
 %macro: is_nonempty_list IsNonemptyList -fail_action
-is_nonempty_list f x
-is_nonempty_list f y
+is_nonempty_list f xy
 
 %macro: is_atom IsAtom -fail_action
 is_atom f x
@@ -735,8 +704,7 @@ is_nil Fail=f n =>
 is_nil Fail=f qia => jump Fail
 
 %macro: is_nil IsNil -fail_action
-is_nil f x
-is_nil f y
+is_nil f xy
 
 is_binary Fail Literal=q => move Literal x | is_binary Fail x
 is_binary Fail=f c => jump Fail
@@ -784,8 +752,7 @@ is_boolean Fail=f ac => jump Fail
 
 %cold
 %macro: is_boolean IsBoolean -fail_action
-is_boolean f x
-is_boolean f y
+is_boolean f xy
 %hot
 
 is_function2 Fail=f acq Arity => jump Fail
@@ -1079,8 +1046,7 @@ i_get_hash c I d
 i_get s d
 
 %macro: self Self
-self x
-self y
+self xy
 
 %macro: node Node
 node x
@@ -1091,8 +1057,7 @@ node y
 i_fast_element j x I d
 i_fast_element j y I d
 
-i_element j x s d
-i_element j y s d
+i_element j xy s d
 
 bif1 f b s d
 bif1_body b s d
@@ -1111,8 +1076,7 @@ i_move_call c f
 %macro:move_call MoveCall -arg_f -size -nonext
 move_call/2
 
-move_call x f
-move_call y f
+move_call xy f
 
 move S=c x==0 | call_last Ar P=f D => i_move_call_last P D S
 move S x==0 | call_last Ar P=f D => move_call_last S P D
@@ -1122,8 +1086,7 @@ i_move_call_last f P c
 %macro:move_call_last MoveCallLast -arg_f -nonext -pack
 
 move_call_last/3
-move_call_last x f Q
-move_call_last y f Q
+move_call_last xy f Q
 
 move S=c x==0 | call_only Ar P=f => i_move_call_only P S
 move S=x x==0 | call_only Ar P=f => move_call_only S P
@@ -1167,8 +1130,7 @@ i_make_fun I t
 %hot
 
 %macro: is_function IsFunction -fail_action
-is_function f x
-is_function f y
+is_function f xy
 is_function Fail=f c => jump Fail
 
 func_info M F A => i_func_info u M F A
@@ -1180,8 +1142,7 @@ func_info M F A => i_func_info u M F A
 %cold
 bs_start_match2 Fail=f ica X Y D => jump Fail
 bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D
-i_bs_start_match2 x f I I d
-i_bs_start_match2 y f I I d
+i_bs_start_match2 xy f I I d
 
 bs_save2 Reg Index => gen_bs_save(Reg, Index)
 i_bs_save2 x I
@@ -1236,8 +1197,7 @@ bs_skip_bits2 Fail=f Ms=x Sz=sq Unit=u Flags=u => \
 i_bs_skip_bits_imm2 f x I
 
 %macro: i_bs_skip_bits2 BsSkipBits2 -fail_action
-i_bs_skip_bits2 f x x I
-i_bs_skip_bits2 f x y I
+i_bs_skip_bits2 f x xy I
 
 %macro: i_bs_skip_bits_all2 BsSkipBitsAll2 -fail_action
 i_bs_skip_bits_all2 f x I 
@@ -1302,8 +1262,7 @@ bs_init2 Fail Sz Words=u==0 Regs Flags Dst => \
 bs_init2 Fail Sz Words Regs Flags Dst => \
   i_bs_init_fail_heap Sz Words Fail Regs Dst
 
-i_bs_init_fail x j I d
-i_bs_init_fail y j I d
+i_bs_init_fail xy j I d
 
 i_bs_init_fail_heap s I j I d
 
@@ -1324,8 +1283,7 @@ bs_init_bits Fail Sz Words=u==0 Regs Flags Dst => \
 bs_init_bits Fail Sz Words Regs Flags Dst => \
   i_bs_init_bits_fail_heap Sz Words Fail Regs Dst
 
-i_bs_init_bits_fail x j I d
-i_bs_init_bits_fail y j I d
+i_bs_init_bits_fail xy j I d
 
 i_bs_init_bits_fail_heap s I j I d
 
@@ -1493,8 +1451,7 @@ is_map Fail Lit=q | literal_is_map(Lit) =>
 is_map Fail cq => jump Fail
 
 %macro: is_map IsMap -fail_action
-is_map f x
-is_map f y
+is_map f xy
 
 ## Transform has_map_fields #{ K1 := _, K2 := _ } to has_map_elements
 
@@ -1514,16 +1471,10 @@ i_get_map_element Fail Src=xy Key=y Dst => \
     move Key x | i_get_map_element Fail Src x Dst
 
 %macro: i_get_map_element_hash GetMapElementHash -fail_action
-i_get_map_element_hash f x c I x
-i_get_map_element_hash f y c I x
-i_get_map_element_hash f x c I y
-i_get_map_element_hash f y c I y
+i_get_map_element_hash f xy c I xy
 
 %macro: i_get_map_element GetMapElement -fail_action
-i_get_map_element f x x x
-i_get_map_element f y x x
-i_get_map_element f x x y
-i_get_map_element f y x y
+i_get_map_element f xy x xy
 
 #
 # Convert the plus operations to a generic plus instruction.
@@ -1589,12 +1540,9 @@ gc_bif2 Fail Live u$bif:erlang:bxor/2 S1 S2 Dst => \
 
 gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst
 
-i_increment r I I d
-i_increment x I I d
-i_increment y I I d
+i_increment rxy I I d
 
-i_plus j I x x d
-i_plus j I x y d
+i_plus j I x xy d
 i_plus j I s s d
 
 i_minus j I x x d
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index bfa5236764..05cd48d434 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -402,7 +402,7 @@ while (<>) {
     if (defined $gen_opnum{$name,$arity} and $obsolete[$gen_opnum{$name,$arity}]) {
 	error("specific instructions may not be specified for obsolete instructions");
     }
-    push(@{$specific_op{"$name/$arity"}}, [$name, $hot, @args]);
+    save_specific_ops($name, $arity, $hot, @args);
     if (defined $op_num) {
 	error("specific instructions must not be numbered");
     } elsif (!defined($gen_arity{$name}) && !defined($unnumbered{$name,$arity})) {
@@ -866,8 +866,30 @@ sub syntax_check {
     error("Bad opcode name '$name'")
 	unless $name =~ /^[a-z][\w\d_]*$/;
     for ($i = 0; $i < @args; $i++) {
-	error("Argument " . ($i+1) . ": invalid type '$args[$i]'")
-	    unless defined $arg_size{$args[$i]};
+        foreach my $type (split(//, $args[$i])) {
+            error("Argument " . ($i+1) . ": invalid type '$type'")
+                unless defined $arg_size{$type};
+        }
+    }
+}
+
+sub save_specific_ops {
+    my($name,$arity,$hot,@args) = @_;
+    my(@res) = ("");
+
+    foreach my $arg (@args) {
+        my @new_res = ();
+        foreach my $type (split(//, $arg)) {
+            foreach my $args (@res) {
+                push @new_res, "$args$type";
+            }
+        }
+        @res = @new_res;
+    }
+    my $key = "$name/$arity";
+    foreach my $args (@res) {
+        @args = split //, $args;
+        push @{$specific_op{$key}}, [$name,$hot,@args];
     }
 }
 
-- 
cgit v1.2.3


From 9f0e2acbdc7105f02a8cac2aa11cf9259dca34ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 18 May 2017 23:05:02 +0200
Subject: Make lock counter info independent of the locks being counted

This allows us to enable/disable lock counting at will, and greatly improves
the performance of erts_debug:lock_counters/1 since we no longer have to
worry about the lock counters "dying" while we're enumerating them.

OTP-14412
---
 erts/emulator/Makefile.in             |   1 +
 erts/emulator/beam/erl_alloc.types    |   7 +
 erts/emulator/beam/erl_bif_info.c     | 317 +++++++-----
 erts/emulator/beam/erl_dirty_bif.tab  |   2 +
 erts/emulator/beam/erl_drv_thread.c   |  14 +-
 erts/emulator/beam/erl_lock_count.c   | 832 +++++++++++++-----------------
 erts/emulator/beam/erl_lock_count.h   | 937 +++++++++++++++++++++++++++++-----
 erts/emulator/beam/erl_process_lock.c | 126 ++---
 erts/emulator/beam/erl_process_lock.h | 177 ++++++-
 erts/emulator/beam/erl_thr_progress.c |  33 +-
 erts/emulator/beam/erl_threads.h      |  58 ++-
 erts/emulator/beam/io.c               |  53 +-
 erts/emulator/beam/utils.c            |   1 +
 erts/emulator/sys/unix/sys.c          |  12 +-
 erts/emulator/sys/win32/sys.c         |  15 +-
 15 files changed, 1690 insertions(+), 895 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 254e59f2c7..ecbf76595b 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -116,6 +116,7 @@ ifeq ($(TYPE),lcnt)
 PURIFY =
 TYPEMARKER = .lcnt
 TYPE_FLAGS = @CFLAGS@ -DERTS_ENABLE_LOCK_COUNT
+ENABLE_ALLOC_TYPE_VARS += lcnt
 else
 
 ifeq ($(TYPE),frmptr)
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 8a23a1526e..c155630765 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -368,6 +368,13 @@ type	SSB		SHORT_LIVED	PROCESSES	ssb
 
 +endif
 
++if lcnt
+
+type    LCNT_CARRIER   STANDARD    SYSTEM  lcnt_lock_info_carrier
+type    LCNT_VECTOR SHORT_LIVED SYSTEM lcnt_lock_info_vector
+
++endif
+
 type	DEBUG	        SHORT_LIVED	SYSTEM          debugging
 
 type	DDLL_PROCESS	STANDARD	SYSTEM		ddll_processes
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 5fc70dfc02..8f7de32f12 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -4430,48 +4430,108 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 }
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
+
+typedef struct lcnt_lock_info_vector_ {
+    erts_lcnt_lock_info_t **elements;
+    size_t size;
+} lcnt_lock_info_vector_t;
+
+static lcnt_lock_info_vector_t lcnt_build_lock_info_vector(erts_lcnt_lock_info_list_t *list) {
+    erts_lcnt_lock_info_t *iterator;
+    lcnt_lock_info_vector_t result;
+    size_t allocated_entries;
+
+    allocated_entries = 64;
+    result.size = 0;
+
+    result.elements = erts_alloc(ERTS_ALC_T_LCNT_VECTOR,
+        allocated_entries * sizeof(erts_lcnt_lock_info_t*));
+
+    iterator = NULL;
+    while(erts_lcnt_iterate_list(list, &iterator)) {
+        erts_lcnt_retain_lock_info(iterator);
+
+        result.elements[result.size++] = iterator;
+
+        if(result.size >= allocated_entries) {
+            allocated_entries *= 2;
+
+            result.elements = erts_realloc(ERTS_ALC_T_LCNT_VECTOR, result.elements,
+                allocated_entries * sizeof(erts_lcnt_lock_info_t*));
+        }
+    }
+
+    return result;
+}
+
+static void lcnt_destroy_lock_info_vector(lcnt_lock_info_vector_t *vector) {
+    size_t i;
+
+    for(i = 0; i < vector->size; i++) {
+        erts_lcnt_release_lock_info(vector->elements[i]);
+    }
+
+    erts_free(ERTS_ALC_T_LCNT_VECTOR, vector->elements);
+}
+
+/* The size of an integer is not guaranteed to be constant since we're walking
+ * over live data, and may cross over into bignum territory between size calc
+ * and the actual build. This takes care of that through always assuming the
+ * worst, but needs to be fixed up with HRelease once the final term has been
+ * built. */
+static ERTS_INLINE Eterm bld_unstable_uint64(Uint **hpp, Uint *szp, Uint64 ui) {
+    Eterm res = THE_NON_VALUE;
+
+    if(szp) {
+        *szp += ERTS_UINT64_HEAP_SIZE(~((Uint64) 0));
+    }
+
+    if(hpp) {
+        if (IS_USMALL(0, ui)) {
+            res = make_small(ui);
+        } else {
+            res = erts_uint64_to_big(ui, hpp);
+        }
+    }
+
+    return res;
+}
+
 static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_stats_t *stats, Eterm res) {
-    Uint tries = 0, colls = 0;
-    unsigned long timer_s = 0, timer_ns = 0, timer_n = 0;
-    unsigned int  line = 0;
     unsigned int  i;
-    
+    const char *file;
+
     Eterm af, uil;
     Eterm uit, uic;
     Eterm uits, uitns, uitn;
     Eterm tt, tstat, tloc, t;
     Eterm thist, vhist[ERTS_LCNT_HISTOGRAM_SLOT_SIZE];
-	
+
     /* term:
-     * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}},
-     *   { .. histogram .. }]
-     */
+        * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}},
+        *   { .. histogram .. }]
+        */
 
-    tries = (Uint) ethr_atomic_read(&stats->tries);
-    colls = (Uint) ethr_atomic_read(&stats->colls);
-   
-    line     = stats->line; 
-    timer_s  = stats->timer.s;
-    timer_ns = stats->timer.ns;
-    timer_n  = stats->timer_n;
-   
-    af    = erts_atom_put((byte *)stats->file, strlen(stats->file), ERTS_ATOM_ENC_LATIN1, 1);
-    uil   = erts_bld_uint( hpp, szp, line);
+    file = stats->file ? stats->file : "undefined";
+
+    af    = erts_atom_put((byte *)file, strlen(file), ERTS_ATOM_ENC_LATIN1, 1);
+    uil   = erts_bld_uint( hpp, szp, stats->line);
     tloc  = erts_bld_tuple(hpp, szp, 2, af, uil);
-    
-    uit   = erts_bld_uint( hpp, szp, tries);
-    uic   = erts_bld_uint( hpp, szp, colls);
 
-    uits  = erts_bld_uint( hpp, szp, timer_s);
-    uitns = erts_bld_uint( hpp, szp, timer_ns);
-    uitn  = erts_bld_uint( hpp, szp, timer_n);
+    uit   = bld_unstable_uint64(hpp, szp, (Uint)ethr_atomic_read(&stats->attempts));
+    uic   = bld_unstable_uint64(hpp, szp, (Uint)ethr_atomic_read(&stats->collisions));
+
+    uits  = bld_unstable_uint64(hpp, szp, stats->total_time_waited.s);
+    uitns = bld_unstable_uint64(hpp, szp, stats->total_time_waited.ns);
+    uitn  = bld_unstable_uint64(hpp, szp, stats->times_waited);
     tt    = erts_bld_tuple(hpp, szp, 3, uits, uitns, uitn);
 
     tstat = erts_bld_tuple(hpp, szp, 3, uit, uic, tt);
 
     for(i = 0; i < ERTS_LCNT_HISTOGRAM_SLOT_SIZE; i++) {
-	vhist[i] = erts_bld_uint(hpp, szp, stats->hist.ns[i]);
+        vhist[i] = bld_unstable_uint64(hpp, szp, stats->wait_time_histogram.ns[i]);
     }
+
     thist  = erts_bld_tuplev(hpp, szp, ERTS_LCNT_HISTOGRAM_SLOT_SIZE, vhist);
 
     t   = erts_bld_tuple(hpp, szp, 3, tloc, tstat, thist);
@@ -4480,150 +4540,165 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s
     return res;
 }
 
-static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_t *lock, Eterm res) {
+static Eterm lcnt_pretty_print_lock_id(erts_lcnt_lock_info_t *info) {
+    Eterm id = info->id;
+
+    if(info->flag & ERTS_LCNT_LT_ALLOC) {
+        /* Use allocator type names as id's for allocator locks. This blatantly
+         * assumes that all locks in this category are identified by their
+         * allocator number. */
+        const char *ltype = (const char*)ERTS_ALC_A2AD(signed_val(info->id));
+        id = erts_atom_put((byte*)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1);
+    } else if(info->flag & ERTS_LCNT_LT_PROCLOCK) {
+        /* Use registered names as id's for process locks if available. Thread
+         * progress is delayed since we may be running on a dirty scheduler. */
+        ErtsThrPrgrDelayHandle delay_handle;
+        Process *process;
+
+        delay_handle = erts_thr_progress_unmanaged_delay();
+
+        process = erts_proc_lookup(info->id);
+        if (process && process->common.u.alive.reg) {
+            id = process->common.u.alive.reg->name;
+        }
+
+        erts_thr_progress_unmanaged_continue(delay_handle);
+    }
+
+    return id;
+}
+
+static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_info_t *info, Eterm res) {
     Eterm name, type, id, stats = NIL, t;
-    Process *proc = NULL;
-    char *ltype;
+    const char *ltype;
     int i;
     
     /* term:
      * [{name, id, type, stats()}] 
      */
-	
-    ASSERT(lock->name);
+
+    ASSERT(info->name);
     
-    ltype = erts_lcnt_lock_type(lock->flag);
+    ltype = erts_lcnt_lock_type(info->flag);
     
     ASSERT(ltype);
     
-    type  = erts_atom_put((byte *)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1);
-    name  = erts_atom_put((byte *)lock->name, strlen(lock->name), ERTS_ATOM_ENC_LATIN1, 1);
-
-    if (lock->flag & ERTS_LCNT_LT_ALLOC) {
-	/* use allocator types names as id's for allocator locks */
-	ltype = (char *) ERTS_ALC_A2AD(signed_val(lock->id));
-	id    = erts_atom_put((byte *)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1);
-    } else if (lock->flag & ERTS_LCNT_LT_PROCLOCK) {
-	/* use registered names as id's for process locks if available */
-	proc  = erts_proc_lookup(lock->id);
-	if (proc && proc->common.u.alive.reg) {
-	    id = proc->common.u.alive.reg->name;
-	} else {
-	    /* otherwise use process id */
-	    id = lock->id;
-	}
+    type  = erts_atom_put((byte*)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1);
+    name  = erts_atom_put((byte*)info->name, strlen(info->name), ERTS_ATOM_ENC_LATIN1, 1);
+
+    /* Only attempt to resolve ids when actually emitting the term. This ought
+     * to be safe since all immediates are the same size. */
+    if(hpp != NULL) {
+        id = lcnt_pretty_print_lock_id(info);
     } else {
-	id = lock->id;
+        id = NIL;
     }
 
-    for (i = 0; i < lock->n_stats; i++) {
-	stats = lcnt_build_lock_stats_term(hpp, szp, &(lock->stats[i]), stats);
+    /* info->location_count is ignored since it can be changed at any time. */
+    for(i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) {
+        stats = lcnt_build_lock_stats_term(hpp, szp, &(info->location_stats[i]), stats);
     }
 
     t   = erts_bld_tuple(hpp, szp, 4, name, id, type, stats);
-    res = erts_bld_cons( hpp, szp, t, res);
+    res = erts_bld_cons(hpp, szp, t, res);
 
     return res;
 }
 
-static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_data_t *data, Eterm res) {
+static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_time_t *duration,
+                                    lcnt_lock_info_vector_t *current_locks,
+                                    lcnt_lock_info_vector_t *deleted_locks, Eterm res) {
     Eterm dts, dtns, tdt, adur, tdur, aloc, lloc = NIL, tloc;
-    erts_lcnt_lock_t *lock = NULL;
+
+    size_t i;
+
     char *str_duration = "duration";
     char *str_locks    = "locks";
-    
+
     /* term:
      * [{'duration', {seconds, nanoseconds}}, {'locks', locks()}]
      */
-   
+
     /* duration tuple */ 
-    dts  = erts_bld_uint( hpp, szp, data->duration.s);
-    dtns = erts_bld_uint( hpp, szp, data->duration.ns);
+    dts  = bld_unstable_uint64(hpp, szp, duration->s);
+    dtns = bld_unstable_uint64(hpp, szp, duration->ns);
     tdt  = erts_bld_tuple(hpp, szp, 2, dts, dtns);
-    
+
     adur = erts_atom_put((byte *)str_duration, strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1);
     tdur = erts_bld_tuple(hpp, szp, 2, adur, tdt);
    
     /* lock tuple */
     
     aloc = erts_atom_put((byte *)str_locks, strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1);
-    	
-    for (lock = data->current_locks->head; lock != NULL ; lock = lock->next ) {
-	lloc = lcnt_build_lock_term(hpp, szp, lock, lloc);
+
+    for(i = 0; i < current_locks->size; i++) {
+        lloc = lcnt_build_lock_term(hpp, szp, current_locks->elements[i], lloc);
     }
-    
-    for (lock = data->deleted_locks->head; lock != NULL ; lock = lock->next ) {
-	lloc = lcnt_build_lock_term(hpp, szp, lock, lloc);
+
+    for(i = 0; i < deleted_locks->size; i++) {
+        lloc = lcnt_build_lock_term(hpp, szp, deleted_locks->elements[i], lloc);
     }
-    
+
     tloc = erts_bld_tuple(hpp, szp, 2, aloc, lloc);
-    
-    res  = erts_bld_cons( hpp, szp, tloc, res);          
-    res  = erts_bld_cons( hpp, szp, tdur, res);          
+
+    res  = erts_bld_cons(hpp, szp, tloc, res);
+    res  = erts_bld_cons(hpp, szp, tdur, res);
 
     return res;
-}    
+}
+
 #endif
 
 BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1)
 {
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    Eterm res = NIL;
-#endif
-
-
+#ifndef ERTS_ENABLE_LOCK_COUNT
     if (BIF_ARG_1 == am_enabled) {
-#ifdef ERTS_ENABLE_LOCK_COUNT
-	BIF_RET(am_true);
-#else
-	BIF_RET(am_false);
-#endif
+        BIF_RET(am_false);
     }
-#ifdef ERTS_ENABLE_LOCK_COUNT
 
-    else if (BIF_ARG_1 == am_info) {
-	erts_lcnt_data_t *data; 
-	Uint hsize = 0;
-	Uint *szp;
-    	Eterm* hp;
+    BIF_ERROR(BIF_P, BADARG);
+#else
 
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_smp_thr_progress_block();
+    if (BIF_ARG_1 == am_enabled) {
+       BIF_RET(am_true);
+    } else if (BIF_ARG_1 == am_info) {
+        lcnt_lock_info_vector_t current_locks, deleted_locks;
+        erts_lcnt_data_t data;
 
-	erts_lcnt_set_rt_opt(ERTS_LCNT_OPT_SUSPEND);
-	data = erts_lcnt_get_data();
+        Eterm *term_heap_start, *term_heap_end;
+        Uint term_heap_size = 0;
+        Eterm result;
 
-	/* calculate size */
+        data = erts_lcnt_get_data();
 
-	szp = &hsize;
-	lcnt_build_result_term(NULL, szp, data, NIL);
+        current_locks = lcnt_build_lock_info_vector(data.current_locks);
+        deleted_locks = lcnt_build_lock_info_vector(data.deleted_locks);
 
-	/* alloc and build */
+        lcnt_build_result_term(NULL, &term_heap_size, &data.duration,
+            &current_locks, &deleted_locks, NIL);
 
-	hp = HAlloc(BIF_P, hsize);
+        term_heap_start = HAlloc(BIF_P, term_heap_size);
+        term_heap_end = term_heap_start;
 
-	res = lcnt_build_result_term(&hp, NULL, data, res);
-	
-	erts_lcnt_clear_rt_opt(ERTS_LCNT_OPT_SUSPEND);
+        result = lcnt_build_result_term(&term_heap_end, NULL,
+            &data.duration, &current_locks, &deleted_locks, NIL);
 
-	erts_smp_thr_progress_unblock();
-	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	
-	BIF_RET(res);
-    } else if (BIF_ARG_1 == am_clear) {
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_smp_thr_progress_block();
+        HRelease(BIF_P, term_heap_start + term_heap_size, term_heap_end);
 
-	erts_lcnt_clear_counters();
+        lcnt_destroy_lock_info_vector(&current_locks);
+        lcnt_destroy_lock_info_vector(&deleted_locks);
 
-	erts_smp_thr_progress_unblock();
-	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+        BIF_RET(result);
+    } else if (BIF_ARG_1 == am_clear) {
+        erts_lcnt_clear_counters();
 
-	BIF_RET(am_ok);
+        BIF_RET(am_ok);
     } else if (is_tuple(BIF_ARG_1)) {
         Eterm* ptr = tuple_val(BIF_ARG_1);
 
         if ((arityval(ptr[0]) == 2) && (ptr[2] == am_false || ptr[2] == am_true)) {
+           Eterm res;
+
             int lock_opt = 0, enable = (ptr[2] == am_true) ? 1 : 0;
             if (ERTS_IS_ATOM_STR("copy_save", ptr[1])) {
                 lock_opt = ERTS_LCNT_OPT_COPYSAVE;
@@ -4631,35 +4706,41 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1)
                 lock_opt = ERTS_LCNT_OPT_PROCLOCK;
             } else if (ERTS_IS_ATOM_STR("port_locks", ptr[1])) {
                 lock_opt = ERTS_LCNT_OPT_PORTLOCK;
-            } else if (ERTS_IS_ATOM_STR("suspend", ptr[1])) {
-                lock_opt = ERTS_LCNT_OPT_SUSPEND;
             } else if (ERTS_IS_ATOM_STR("location", ptr[1])) {
                 lock_opt = ERTS_LCNT_OPT_LOCATION;
             } else {
                 BIF_ERROR(BIF_P, BADARG);
             }
 
+            if (enable) {
+                res = erts_lcnt_set_rt_opt(lock_opt) ? am_true : am_false;
+            } else {
+                res = erts_lcnt_clear_rt_opt(lock_opt) ? am_true : am_false;
+            }
+
+#ifdef ERTS_SMP
+            /* FIXME: make this non-blocking. */
             erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
             erts_smp_thr_progress_block();
+#endif
 
-            if (enable) res = erts_lcnt_set_rt_opt(lock_opt) ? am_true : am_false;
-            else res = erts_lcnt_clear_rt_opt(lock_opt) ? am_true : am_false;
-            
-#ifdef ERTS_SMP
             if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PORTLOCK) {
                 erts_lcnt_enable_io_lock_count(enable);
             } else if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PROCLOCK) {
                 erts_lcnt_enable_proc_lock_count(enable);
             }
-#endif
-            erts_smp_thr_progress_unblock();
+
+#ifdef ERTS_SMP
             erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+            erts_smp_thr_progress_unblock();
+#endif
+
             BIF_RET(res);
         }
-    } 
+    }
 
-#endif 
     BIF_ERROR(BIF_P, BADARG);
+#endif
 }
 
 static void os_info_init(void)
diff --git a/erts/emulator/beam/erl_dirty_bif.tab b/erts/emulator/beam/erl_dirty_bif.tab
index 69421dcfcc..94d8589779 100644
--- a/erts/emulator/beam/erl_dirty_bif.tab
+++ b/erts/emulator/beam/erl_dirty_bif.tab
@@ -46,6 +46,8 @@
 dirty-cpu erts_debug:dirty_cpu/2
 dirty-io erts_debug:dirty_io/2
 
+dirty-cpu erts_debug:lock_counters/1
+
 # --- TEST of Dirty BIF functionality ---
 #  Functions below will execute on dirty schedulers when emulator has
 #  been configured for testing dirty schedulers. This is used for test
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 0e6aadf568..3b68abe5d7 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -55,7 +55,7 @@ fatal_error(int err, char *func)
 struct ErlDrvMutex_ {
     ethr_mutex mtx;
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_t lcnt;
+    erts_lcnt_ref_t lcnt;
 #endif
     char *name;
 };
@@ -68,7 +68,7 @@ struct ErlDrvCond_ {
 struct ErlDrvRWLock_ {
     ethr_rwmutex rwmtx;
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_t lcnt;
+    erts_lcnt_ref_t lcnt;
 #endif
     char *name;
 };
@@ -176,7 +176,8 @@ erl_drv_mutex_create(char *name)
 	    dmtx->name = no_name;
 	}
 #ifdef ERTS_ENABLE_LOCK_COUNT
-        erts_lcnt_init_lock(&dmtx->lcnt, dmtx->name, ERTS_LCNT_LT_MUTEX);
+    erts_lcnt_init_ref(&dmtx->lcnt);
+    erts_lcnt_install_new_lock_info(&dmtx->lcnt, dmtx->name, ERTS_LCNT_LT_MUTEX);
 #endif
     }
     return dmtx;
@@ -191,7 +192,7 @@ erl_drv_mutex_destroy(ErlDrvMutex *dmtx)
 #ifdef USE_THREADS
     int res;
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_destroy_lock(&dmtx->lcnt);
+    erts_lcnt_uninstall(&dmtx->lcnt);
 #endif
     res = dmtx ? ethr_mutex_destroy(&dmtx->mtx) : EINVAL;
     if (res != 0)
@@ -368,7 +369,8 @@ erl_drv_rwlock_create(char *name)
 	    drwlck->name = no_name;
 	}
 #ifdef ERTS_ENABLE_LOCK_COUNT
-        erts_lcnt_init_lock(&drwlck->lcnt, drwlck->name, ERTS_LCNT_LT_RWMUTEX);
+    erts_lcnt_init_ref(&drwlck->lcnt);
+    erts_lcnt_install_new_lock_info(&drwlck->lcnt, drwlck->name, ERTS_LCNT_LT_RWMUTEX);
 #endif
     }
     return drwlck;
@@ -383,7 +385,7 @@ erl_drv_rwlock_destroy(ErlDrvRWLock *drwlck)
 #ifdef USE_THREADS
     int res;
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_destroy_lock(&drwlck->lcnt);
+    erts_lcnt_uninstall(&drwlck->lcnt);
 #endif
     res = drwlck ? ethr_rwmutex_destroy(&drwlck->rwmtx) : EINVAL;
     if (res != 0)
diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index aee9796171..2cf59aa367 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,51 +18,30 @@
  * %CopyrightEnd%
  */
 
-/*
- * Description:	Statistics for locks.
- *
- * Author:	Björn-Egil Dahlberg
- * Date:	2008-07-03
- */
-
 #ifdef HAVE_CONFIG_H
 #  include "config.h"
 #endif
 
-/* Needed for VxWorks va_arg */
-#include "sys.h"
-
 #ifdef ERTS_ENABLE_LOCK_COUNT
 
+#include "sys.h"
+
 #include "erl_lock_count.h"
-#include "ethread.h"
-#include "erl_term.h"
-#include "atom.h"
-#include <stdio.h>
+#include "erl_thr_progress.h"
 
-/* globals, dont access these without locks or blocks */
+#define LCNT_MAX_CARRIER_ENTRIES 255
 
-ethr_mutex lcnt_data_lock;
-erts_lcnt_data_t *erts_lcnt_data;
-Uint16 erts_lcnt_rt_options;
-erts_lcnt_time_t timer_start;
-const char *str_undefined = "undefined";
+/* - Exported global - */
 
-static ethr_tsd_key lcnt_thr_data_key;
-static int lcnt_n_thr;
-static erts_lcnt_thread_data_t *lcnt_thread_data[2048];
+Uint16 erts_lcnt_rt_options = ERTS_LCNT_OPT_PROCLOCK | ERTS_LCNT_OPT_LOCATION;
 
-/* local functions */
+/* - Locals that are shared with the header implementation - */
 
-static ERTS_INLINE void lcnt_lock(void) {
-    ethr_mutex_lock(&lcnt_data_lock);
-}
+int lcnt_initialization_completed__ = 0;
 
-static ERTS_INLINE void lcnt_unlock(void) {
-    ethr_mutex_unlock(&lcnt_data_lock);
-}
+ethr_tsd_key lcnt_thr_data_key__;
 
-const int log2_tab64[64] = {
+const int lcnt_log2_tab64__[64] = {
     63,  0, 58,  1, 59, 47, 53,  2,
     60, 39, 48, 27, 54, 33, 42,  3,
     61, 51, 37, 40, 49, 18, 28, 20,
@@ -72,99 +51,52 @@ const int log2_tab64[64] = {
     56, 45, 25, 31, 35, 16,  9, 12,
     44, 24, 15,  8, 23,  7,  6,  5};
 
-static ERTS_INLINE int lcnt_log2(Uint64 v) {
-    v |= v >> 1;
-    v |= v >> 2;
-    v |= v >> 4;
-    v |= v >> 8;
-    v |= v >> 16;
-    v |= v >> 32;
-    return log2_tab64[((Uint64)((v - (v >> 1))*0x07EDD5E59A4E28C2)) >> 58];
-}
-
-static char* lcnt_lock_type(Uint16 flag) {
-    switch(flag & ERTS_LCNT_LT_ALL) {
-        case ERTS_LCNT_LT_SPINLOCK:   return "spinlock";
-        case ERTS_LCNT_LT_RWSPINLOCK: return "rw_spinlock";
-        case ERTS_LCNT_LT_MUTEX:      return "mutex";
-        case ERTS_LCNT_LT_RWMUTEX:    return "rw_mutex";
-        case ERTS_LCNT_LT_PROCLOCK:   return "proclock";
-        default:                      return "";
-    }
-}
+/* - Local variables - */
 
-static void lcnt_clear_stats(erts_lcnt_lock_stats_t *stats) {
-    ethr_atomic_set(&stats->tries, 0);
-    ethr_atomic_set(&stats->colls, 0);
-    stats->timer.s  = 0;
-    stats->timer.ns = 0;
-    stats->timer_n  = 0;
-    stats->file     = (char *)str_undefined;
-    stats->line     = 0;
-    sys_memzero(stats->hist.ns, sizeof(stats->hist.ns));
-}
+static erts_lcnt_lock_info_list_t lcnt_current_lock_list;
+static erts_lcnt_lock_info_list_t lcnt_deleted_lock_list;
 
-static void lcnt_time(erts_lcnt_time_t *time) {
-    /*
-     * erts_sys_hrtime() is the highest resolution
-     * we could find, it may or may not be monotonic...
-     */
-    ErtsMonotonicTime mtime = erts_sys_hrtime();
-    time->s  = (unsigned long) (mtime / 1000000000LL);
-    time->ns = (unsigned long) (mtime - 1000000000LL*time->s);
-}
+static erts_lcnt_time_t lcnt_timer_start;
 
-static void lcnt_time_diff(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0) {
-    long ds;
-    long dns;
+/* local functions */
 
-    ds  = t1->s  - t0->s;
-    dns = t1->ns - t0->ns;
+static void lcnt_clear_stats(erts_lcnt_lock_info_t *info) {
+    size_t i;
 
-    /* the difference should not be able to get bigger than 1 sec in ns*/
+    for(i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) {
+        erts_lcnt_lock_stats_t *stats = &info->location_stats[i];
 
-    if (dns < 0) {
-        ds  -= 1;
-        dns += 1000000000LL;
-    }
+        sys_memzero(&stats->wait_time_histogram, sizeof(stats->wait_time_histogram));
 
-    ASSERT(ds >= 0);
+        stats->total_time_waited.s = 0;
+        stats->total_time_waited.ns = 0;
 
-    d->s  = ds;
-    d->ns = dns;
-}
+        stats->times_waited = 0;
 
-/* difference d must be non-negative */
+        stats->file = NULL;
+        stats->line = 0;
 
-static void lcnt_time_add(erts_lcnt_time_t *t, erts_lcnt_time_t *d) {
-    t->s  += d->s;
-    t->ns += d->ns;
+        ethr_atomic_set(&stats->attempts, 0);
+        ethr_atomic_set(&stats->collisions, 0);
+    }
 
-    t->s  += t->ns / 1000000000LL;
-    t->ns  = t->ns % 1000000000LL;
+    info->location_count = 1;
 }
 
-static erts_lcnt_thread_data_t *lcnt_thread_data_alloc(void) {
-    erts_lcnt_thread_data_t *eltd;
+static lcnt_thread_data_t__ *lcnt_thread_data_alloc(void) {
+    lcnt_thread_data_t__ *eltd =
+        (lcnt_thread_data_t__*)malloc(sizeof(lcnt_thread_data_t__));
 
-    eltd = (erts_lcnt_thread_data_t*)malloc(sizeof(erts_lcnt_thread_data_t));
-    if (!eltd) {
-        ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!");
+    if(!eltd) {
+        ERTS_INTERNAL_ERROR("Failed to allocate lcnt thread data.");
     }
+
     eltd->timer_set = 0;
     eltd->lock_in_conflict = 0;
 
-    eltd->id = lcnt_n_thr++;
-    /* set thread data to array */
-    lcnt_thread_data[eltd->id] = eltd;
-
     return eltd;
 }
 
-static erts_lcnt_thread_data_t *lcnt_get_thread_data(void) {
-    return (erts_lcnt_thread_data_t *)ethr_tsd_get(lcnt_thr_data_key);
-}
-
 /* debug */
 
 #if 0
@@ -175,472 +107,391 @@ static char* lock_opt(Uint16 flag) {
     return "--";
 }
 
-static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action) {
-    erts_aint_t w_state, r_state;
+static void print_lock_x(erts_lcnt_lock_info_t *info, Uint16 flag, char *action) {
+    ethr_sint_t w_state, r_state;
     char *type;
 
-    if (strcmp(lock->name, "run_queue") != 0) return;
-    type = lcnt_lock_type(lock->flag);
-    r_state = ethr_atomic_read(&lock->r_state);
-    w_state = ethr_atomic_read(&lock->w_state);
+    if (strcmp(info->name, "run_queue") != 0) return;
+    type = erts_lcnt_lock_type(info->flag);
+    r_state = ethr_atomic_read(&info->r_state);
+    w_state = ethr_atomic_read(&info->w_state);
 
-    if (lock->flag & flag) {
+    if (info->flag & flag) {
         erts_fprintf(stderr,"%10s [%24s] [r/w state %4ld/%4ld] %2s id %T\r\n",
                 action,
-                lock->name,
+                info->name,
                 r_state,
                 w_state,
                 type,
-                lock->id);
+                info->id);
     }
 }
 #endif
 
-static erts_lcnt_lock_stats_t *lcnt_get_lock_stats(erts_lcnt_lock_t *lock, char *file, unsigned int line) {
-    unsigned int i;
-    erts_lcnt_lock_stats_t *stats = NULL;
-
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_LOCATION) {
-        for (i = 0; i < lock->n_stats; i++) {
-            if ((lock->stats[i].file == file) && (lock->stats[i].line == line)) {
-                return &(lock->stats[i]);
-            }
-        }
-        if (lock->n_stats < ERTS_LCNT_MAX_LOCK_LOCATIONS) {
-            stats = &lock->stats[lock->n_stats];
-            lock->n_stats++;
-            stats->file = file;
-            stats->line = line;
-            return stats;
-        }
-    }
-    return &lock->stats[0];
+/* - List operations -
+ *
+ * Info entries are kept in a doubly linked list where each entry is locked
+ * with its neighbors rather than a global lock. Deletion is rather quick, but
+ * insertion is still serial since the head becomes a de facto global lock.
+ *
+ * We rely on ad-hoc spinlocks to avoid "recursing" into this module. */
+
+#define LCNT_SPINLOCK_YIELD_ITERATIONS 50
+
+#define LCNT_SPINLOCK_HELPER_INIT \
+    Uint failed_spin_count = 0;
+
+#define LCNT_SPINLOCK_HELPER_YIELD \
+    do { \
+        failed_spin_count++; \
+        if(!(failed_spin_count % LCNT_SPINLOCK_YIELD_ITERATIONS)) { \
+            erts_thr_yield(); \
+        } else { \
+            ERTS_SPIN_BODY; \
+        } \
+    } while(0)
+
+static void lcnt_unlock_list_entry(erts_lcnt_lock_info_t *info) {
+    ethr_atomic32_set_relb(&info->lock, 0);
 }
 
-static void lcnt_update_stats_hist(erts_lcnt_hist_t *hist, erts_lcnt_time_t *time_wait) {
-    int idx;
-    unsigned long r;
+static int lcnt_try_lock_list_entry(erts_lcnt_lock_info_t *info) {
+    return ethr_atomic32_cmpxchg_acqb(&info->lock, 1, 0) == 0;
+}
 
-    if (time_wait->s > 0 || time_wait->ns > ERTS_LCNT_HISTOGRAM_MAX_NS) {
-        idx = ERTS_LCNT_HISTOGRAM_SLOT_SIZE - 1;
-    } else {
-        r = time_wait->ns >> ERTS_LCNT_HISTOGRAM_RSHIFT;
-        if (r) idx = lcnt_log2(r);
-        else idx = 0;
+static void lcnt_lock_list_entry(erts_lcnt_lock_info_t *info) {
+    LCNT_SPINLOCK_HELPER_INIT;
+
+    while(!lcnt_try_lock_list_entry(info)) {
+        LCNT_SPINLOCK_HELPER_YIELD;
     }
-    hist->ns[idx]++;
 }
 
-static void lcnt_update_stats(erts_lcnt_lock_stats_t *stats, int lock_in_conflict,
-        erts_lcnt_time_t *time_wait) {
+static void lcnt_lock_list_entry_with_neighbors(erts_lcnt_lock_info_t *info) {
+    LCNT_SPINLOCK_HELPER_INIT;
 
-    ethr_atomic_inc(&stats->tries);
+    for(;;) {
+        if(!lcnt_try_lock_list_entry(info))
+            goto retry_after_entry_failed;
+        if(!lcnt_try_lock_list_entry(info->next))
+            goto retry_after_next_failed;
+        if(!lcnt_try_lock_list_entry(info->prev))
+            goto retry_after_prev_failed;
 
-    if (lock_in_conflict)
-        ethr_atomic_inc(&stats->colls);
+        return;
 
-    if (time_wait) {
-        lcnt_time_add(&(stats->timer), time_wait);
-        stats->timer_n++;
-        lcnt_update_stats_hist(&stats->hist,time_wait);
+    retry_after_prev_failed:
+        lcnt_unlock_list_entry(info->next);
+    retry_after_next_failed:
+        lcnt_unlock_list_entry(info);
+    retry_after_entry_failed:
+        LCNT_SPINLOCK_HELPER_YIELD;
     }
 }
 
-/* interface */
+static void lcnt_unlock_list_entry_with_neighbors(erts_lcnt_lock_info_t *info) {
+    lcnt_unlock_list_entry(info->prev);
+    lcnt_unlock_list_entry(info->next);
+    lcnt_unlock_list_entry(info);
+}
 
-void erts_lcnt_init() {
-    erts_lcnt_thread_data_t *eltd = NULL;
+static void lcnt_insert_list_entry(erts_lcnt_lock_info_list_t *list, erts_lcnt_lock_info_t *info) {
+    erts_lcnt_lock_info_t *next, *prev;
 
-    /* init lock */
-    if (ethr_mutex_init(&lcnt_data_lock) != 0) abort();
+    prev = &list->head;
 
-    /* init tsd */
-    lcnt_n_thr = 0;
-    ethr_tsd_key_create(&lcnt_thr_data_key, "lcnt_data");
+    lcnt_lock_list_entry(prev);
 
-    lcnt_lock();
+    next = prev->next;
 
-    erts_lcnt_rt_options = ERTS_LCNT_OPT_LOCATION | ERTS_LCNT_OPT_PROCLOCK;
-    eltd = lcnt_thread_data_alloc();
-    ethr_tsd_set(lcnt_thr_data_key, eltd);
+    lcnt_lock_list_entry(next);
 
-    /* init lcnt structure */
-    erts_lcnt_data = (erts_lcnt_data_t*)malloc(sizeof(erts_lcnt_data_t));
-    if (!erts_lcnt_data) {
-        ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!");
-    }
-    erts_lcnt_data->current_locks = erts_lcnt_list_init();
-    erts_lcnt_data->deleted_locks = erts_lcnt_list_init();
+    info->next = next;
+    info->prev = prev;
 
-    lcnt_unlock();
+    prev->next = info;
+    next->prev = info;
 
+    lcnt_unlock_list_entry(next);
+    lcnt_unlock_list_entry(prev);
 }
 
-void erts_lcnt_late_init() {
-    /* set start timer and zero statistics */
-    erts_lcnt_clear_counters();
-    erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler);
-}
+static void lcnt_insert_list_carrier(erts_lcnt_lock_info_list_t *list,
+                                     erts_lcnt_lock_info_carrier_t *carrier) {
+    erts_lcnt_lock_info_t *next, *prev;
+    size_t i;
 
-/* list operations */
+    for(i = 0; i < carrier->entry_count; i++) {
+        erts_lcnt_lock_info_t *info = &carrier->entries[i];
 
-/* BEGIN ASSUMPTION: lcnt_data_lock taken */
+        info->prev = &carrier->entries[i - 1];
+        info->next = &carrier->entries[i + 1];
+    }
 
-erts_lcnt_lock_list_t *erts_lcnt_list_init(void) {
-    erts_lcnt_lock_list_t *list;
+    prev = &list->head;
 
-    list = (erts_lcnt_lock_list_t*)malloc(sizeof(erts_lcnt_lock_list_t));
-    if (!list) {
-        ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!");
-    }
-    list->head = NULL;
-    list->tail = NULL;
-    list->n    = 0;
-    return list;
-}
+    lcnt_lock_list_entry(prev);
 
-static void lcnt_list_free(erts_lcnt_lock_t *head) {
-    erts_lcnt_lock_t *lock, *next;
+    next = prev->next;
 
-    lock = head;
+    lcnt_lock_list_entry(next);
 
-    while(lock != NULL) {
-        next = lock->next;
-        free(lock);
-        lock = next;
-    }
+    next->prev = &carrier->entries[carrier->entry_count - 1];
+    carrier->entries[carrier->entry_count - 1].next = next;
+
+    prev->next = &carrier->entries[0];
+    carrier->entries[0].prev = prev;
+
+    lcnt_unlock_list_entry(next);
+    lcnt_unlock_list_entry(prev);
 }
 
-void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) {
-    erts_lcnt_lock_t *tail = NULL;
+static void lcnt_init_list(erts_lcnt_lock_info_list_t *list) {
+    /* Ensure that ref_count operations explode when touching the sentinels in
+     * DEBUG mode. */
+    ethr_atomic_init(&(list->head.ref_count), -1);
+    ethr_atomic_init(&(list->tail.ref_count), -1);
 
-    tail = list->tail;
-    if (tail) {
-        tail->next = lock;
-        lock->prev = tail;
-    } else {
-        list->head = lock;
-        lock->prev = NULL;
-        ASSERT(!lock->next);
-    }
-    lock->next = NULL;
-    list->tail = lock;
+    ethr_atomic32_init(&(list->head.lock), 0);
+    (list->head).next = &list->tail;
+    (list->head).prev = &list->tail;
 
-    list->n++;
+    ethr_atomic32_init(&(list->tail.lock), 0);
+    (list->tail).next = &list->head;
+    (list->tail).prev = &list->head;
 }
 
-void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) {
-    if (lock->next) lock->next->prev = lock->prev;
-    if (lock->prev) lock->prev->next = lock->next;
-    if (list->head == lock) list->head = lock->next;
-    if (list->tail == lock) list->tail = lock->prev;
+/* - Carrier operations - */
 
-    lock->prev = NULL;
-    lock->next = NULL;
-    list->n--;
+int lcnt_thr_progress_unmanaged_delay__(void) {
+    return erts_thr_progress_unmanaged_delay();
 }
-/* END ASSUMPTION: lcnt_data_lock taken */
 
+void lcnt_thr_progress_unmanaged_continue__(int handle) {
+    return erts_thr_progress_unmanaged_continue(handle);
+}
 
-/* lock operations */
+static void lcnt_deallocate_carrier_malloc(erts_lcnt_lock_info_carrier_t *carrier) {
+    ASSERT(ethr_atomic_read(&carrier->ref_count) == 0);
+    free(carrier);
+}
 
-/* interface to erl_threads.h */
-/* only lock on init and destroy, all others should use atomics */
-void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) {
-    erts_lcnt_init_lock_x(lock, name, flag, NIL);
+static void lcnt_deallocate_carrier_erts(erts_lcnt_lock_info_carrier_t *carrier) {
+    ASSERT(ethr_atomic_read(&carrier->ref_count) == 0);
+    erts_free(ERTS_ALC_T_LCNT_CARRIER, (void*)carrier);
 }
 
-void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) {
-    int i;
+static void lcnt_thr_prg_cleanup_carrier(void *data) {
+    erts_lcnt_lock_info_carrier_t *carrier = data;
+    size_t entry_count, i;
 
-    if (flag & ERTS_LCNT_LT_DISABLE) {
-        ERTS_LCNT_CLEAR_FLAG(lock);
-        return;
+    /* carrier->entry_count will be replaced with garbage if it's deallocated
+     * on the final iteration, so we'll tuck it away to get a clean exit. */
+    entry_count = carrier->entry_count;
+
+    for(i = 0; i < entry_count; i++) {
+        ASSERT(ethr_atomic_read(&carrier->ref_count) >= (entry_count - i));
+
+        erts_lcnt_release_lock_info(&carrier->entries[i]);
     }
+}
 
-    lock->next = NULL;
-    lock->prev = NULL;
-    lock->flag = flag;
-    lock->name = name;
-    lock->id   = id;
+static void lcnt_schedule_carrier_cleanup(void *data) {
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
 
-    ethr_atomic_init(&lock->r_state, 0);
-    ethr_atomic_init(&lock->w_state, 0);
-#ifdef DEBUG
-    ethr_atomic_init(&lock->flowstate, 0);
-#endif
+    /* We can't issue cleanup jobs on anything other than normal schedulers, so
+     * we move to the first scheduler if required. */
+
+    if(!esdp || esdp->type != ERTS_SCHED_NORMAL) {
+        erts_schedule_misc_aux_work(1, &lcnt_schedule_carrier_cleanup, data);
+    } else {
+        erts_lcnt_lock_info_carrier_t *carrier = data;
+        size_t carrier_size;
 
-    lock->n_stats = 1;
+        carrier_size = sizeof(erts_lcnt_lock_info_carrier_t) +
+                       sizeof(erts_lcnt_lock_info_t) * carrier->entry_count;
 
-    for (i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) {
-        lcnt_clear_stats(&lock->stats[i]);
+        erts_schedule_thr_prgr_later_cleanup_op(&lcnt_thr_prg_cleanup_carrier,
+            data, (ErtsThrPrgrLaterOp*)&carrier->release_entries, carrier_size);
     }
+}
 
-    lcnt_lock();
-    erts_lcnt_list_insert(erts_lcnt_data->current_locks, lock);
-    lcnt_unlock();
+static void lcnt_info_deallocate(erts_lcnt_lock_info_t *info) {
+    lcnt_release_carrier__(info->carrier);
 }
 
-/* init empty, instead of zero struct
- * used by process locks probes
- */
-void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock) {
-    lock->next = NULL;
-    lock->prev = NULL;
-    lock->flag = 0;
-    lock->name = NULL;
-    lock->id   = NIL;
-    ethr_atomic_init(&lock->r_state, 0);
-    ethr_atomic_init(&lock->w_state, 0);
-#ifdef DEBUG
-    ethr_atomic_init(&lock->flowstate, 0);
-#endif
-    lock->n_stats = 0;
-    sys_memzero(lock->stats, sizeof(lock->stats));
-}
-/* destroy lock */
-void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) {
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
-    lcnt_lock();
-
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_COPYSAVE) {
-        erts_lcnt_lock_t *deleted_lock;
-        /* copy structure and insert the copy */
-        deleted_lock = (erts_lcnt_lock_t*)malloc(sizeof(erts_lcnt_lock_t));
-        if (!deleted_lock) {
-            ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!");
-        }
-        memcpy(deleted_lock, lock, sizeof(erts_lcnt_lock_t));
-        deleted_lock->next = NULL;
-        deleted_lock->prev = NULL;
-        erts_lcnt_list_insert(erts_lcnt_data->deleted_locks, deleted_lock);
-    }
-    /* delete original */
-    erts_lcnt_list_delete(erts_lcnt_data->current_locks, lock);
-    ERTS_LCNT_CLEAR_FLAG(lock);
+static void lcnt_info_dispose(erts_lcnt_lock_info_t *info) {
+    ASSERT(ethr_atomic_read(&info->ref_count) == 0);
 
-    lcnt_unlock();
+    if(erts_lcnt_rt_options & ERTS_LCNT_OPT_COPYSAVE) {
+        ethr_atomic_set(&info->ref_count, 1);
+
+        /* Move straight to deallocation the next time around. */
+        info->dispose = &lcnt_info_deallocate;
+
+        lcnt_insert_list_entry(&lcnt_deleted_lock_list, info);
+    } else {
+        lcnt_info_deallocate(info);
+    }
 }
 
-/* lock */
+static void lcnt_lock_info_init_helper(erts_lcnt_lock_info_t *info) {
+#ifdef DEBUG
+    ethr_atomic_init(&info->flowstate, 0);
+#endif
 
-void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
-    erts_aint_t r_state = 0, w_state = 0;
-    erts_lcnt_thread_data_t *eltd;
+    ethr_atomic_init(&info->ref_count, 1);
+    ethr_atomic32_init(&info->lock, 0);
 
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
+    ethr_atomic_init(&info->r_state, 0);
+    ethr_atomic_init(&info->w_state, 0);
 
-    eltd = lcnt_get_thread_data();
-    ASSERT(eltd);
+    info->dispose = &lcnt_info_dispose;
 
-    w_state = ethr_atomic_read(&lock->w_state);
+    lcnt_clear_stats(info);
+}
 
-    if (option & ERTS_LCNT_LO_WRITE) {
-        r_state = ethr_atomic_read(&lock->r_state);
-        ethr_atomic_inc( &lock->w_state);
-    }
-    if (option & ERTS_LCNT_LO_READ) {
-        ethr_atomic_inc( &lock->r_state);
-    }
+erts_lcnt_lock_info_carrier_t *erts_lcnt_create_lock_info_carrier(int entry_count) {
+    erts_lcnt_lock_info_carrier_t *result;
+    size_t carrier_size, i;
+
+    ASSERT(entry_count > 0 && entry_count <= LCNT_MAX_CARRIER_ENTRIES);
 
-    /* we cannot acquire w_lock if either w or r are taken */
-    /* we cannot acquire r_lock if w_lock is taken */
+    carrier_size = sizeof(erts_lcnt_lock_info_carrier_t) +
+                   sizeof(erts_lcnt_lock_info_t) * entry_count;
 
-    if ((w_state > 0) || (r_state > 0)) {
-        eltd->lock_in_conflict = 1;
-        if (eltd->timer_set == 0) {
-            lcnt_time(&eltd->timer);
-        }
-        eltd->timer_set++;
+    if(lcnt_initialization_completed__) {
+        result = (erts_lcnt_lock_info_carrier_t*)erts_alloc(ERTS_ALC_T_LCNT_CARRIER, carrier_size);
+        result->deallocate = &lcnt_deallocate_carrier_erts;
     } else {
-        eltd->lock_in_conflict = 0;
+        result = (erts_lcnt_lock_info_carrier_t*)malloc(carrier_size);
+        result->deallocate = &lcnt_deallocate_carrier_malloc;
     }
-}
 
-void erts_lcnt_lock(erts_lcnt_lock_t *lock) {
-    erts_aint_t w_state;
-    erts_lcnt_thread_data_t *eltd;
+    ethr_atomic_init(&result->ref_count, entry_count);
 
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
+    result->entry_count = entry_count;
 
-    w_state = ethr_atomic_read(&lock->w_state);
-    ethr_atomic_inc(&lock->w_state);
-    eltd = lcnt_get_thread_data();
+    for(i = 0; i < entry_count; i++) {
+        erts_lcnt_lock_info_t *info = &result->entries[i];
 
-    ASSERT(eltd);
+        lcnt_lock_info_init_helper(info);
 
-    if (w_state > 0) {
-        eltd->lock_in_conflict = 1;
-        /* only set the timer if nobody else has it
-         * This should only happen when proc_locks aquires several locks
-         * 'atomicly'. All other locks will block the thread if w_state > 0
-         * i.e. locked.
-         */
-        if (eltd->timer_set == 0) {
-            lcnt_time(&eltd->timer);
-        }
-        eltd->timer_set++;
-    } else {
-        eltd->lock_in_conflict = 0;
+        info->carrier = result;
     }
-}
-
-/* if a lock wasn't really a lock operation, bad bad process locks */
-
-void erts_lcnt_lock_unaquire(erts_lcnt_lock_t *lock) {
-    /* should check if this thread was "waiting" */
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
 
-    ethr_atomic_dec(&lock->w_state);
+    return result;
 }
 
-/*
- * erts_lcnt_lock_post
- *
- * Used when we get a lock (i.e. directly after a lock operation)
- * if the timer was set then we had to wait for the lock
- * lock_post will calculate the wait time.
- */
+void erts_lcnt_install(erts_lcnt_ref_t *ref, erts_lcnt_lock_info_carrier_t *carrier) {
+    ethr_sint_t swapped_carrier;
 
-void erts_lcnt_lock_post(erts_lcnt_lock_t *lock) {
-    erts_lcnt_lock_post_x(lock, (char*)str_undefined, 0);
-}
+    swapped_carrier = ethr_atomic_cmpxchg_mb(ref, (ethr_sint_t)carrier, (ethr_sint_t)NULL);
 
-void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line) {
-    erts_lcnt_thread_data_t *eltd;
-    erts_lcnt_time_t timer;
-    erts_lcnt_time_t time_wait;
-    erts_lcnt_lock_stats_t *stats;
+    if(swapped_carrier != (ethr_sint_t)NULL) {
 #ifdef DEBUG
-    erts_aint_t flowstate;
+        ASSERT(ethr_atomic_read(&carrier->ref_count) == carrier->entry_count);
+        ethr_atomic_set(&carrier->ref_count, 0);
 #endif
 
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
+        carrier->deallocate(carrier);
+    } else {
+        lcnt_insert_list_carrier(&lcnt_current_lock_list, carrier);
+    }
+}
+
+void erts_lcnt_uninstall(erts_lcnt_ref_t *ref) {
+    ethr_sint_t previous_carrier, swapped_carrier;
 
-#ifdef DEBUG
-    if (!(lock->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) {
-        flowstate = ethr_atomic_read(&lock->flowstate);
-        ASSERT(flowstate == 0);
-        ethr_atomic_inc(&lock->flowstate);
+    previous_carrier = ethr_atomic_read(ref);
+    swapped_carrier = ethr_atomic_cmpxchg_mb(ref, (ethr_sint_t)NULL, previous_carrier);
+
+    if(previous_carrier && previous_carrier == swapped_carrier) {
+        lcnt_schedule_carrier_cleanup((void*)previous_carrier);
     }
-#endif
+}
 
-    eltd = lcnt_get_thread_data();
+/* - Initialization - */
 
-    ASSERT(eltd);
+void erts_lcnt_pre_thr_init() {
+    /* Ensure that the dependency hack mentioned in the header doesn't
+     * explode at runtime. */
+    ERTS_CT_ASSERT(sizeof(LcntThrPrgrLaterOp) >= sizeof(ErtsThrPrgrLaterOp));
+    ERTS_CT_ASSERT(ERTS_THR_PRGR_DHANDLE_MANAGED == 
+        (ErtsThrPrgrDelayHandle)LCNT_THR_PRGR_DHANDLE_MANAGED);
 
-    /* if lock was in conflict, time it */
-    stats = lcnt_get_lock_stats(lock, file, line);
-    if (eltd->timer_set) {
-        lcnt_time(&timer);
+    lcnt_init_list(&lcnt_current_lock_list);
+    lcnt_init_list(&lcnt_deleted_lock_list);
+}
 
-        lcnt_time_diff(&time_wait, &timer, &(eltd->timer));
-        lcnt_update_stats(stats, eltd->lock_in_conflict, &time_wait);
-        eltd->timer_set--;
-        ASSERT(eltd->timer_set >= 0);
-    } else {
-        lcnt_update_stats(stats, eltd->lock_in_conflict, NULL);
-    }
+void erts_lcnt_post_thr_init() {
+    /* ASSUMPTION: this is safe since it runs prior to the creation of other
+     * threads (Directly after ethread init). */
 
+    ethr_tsd_key_create(&lcnt_thr_data_key__, "lcnt_data");
+
+    erts_lcnt_thread_setup();
 }
 
-/* unlock */
+void erts_lcnt_late_init() {
+    /* Set start timer and zero all statistics */
+    erts_lcnt_clear_counters();
+    erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler);
 
-void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
-    if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_dec(&lock->w_state);
-    if (option & ERTS_LCNT_LO_READ ) ethr_atomic_dec(&lock->r_state);
+    /* It's safe to use erts_alloc and thread progress past this point. */
+    lcnt_initialization_completed__ = 1;
 }
 
-void erts_lcnt_unlock(erts_lcnt_lock_t *lock) {
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
-#ifdef DEBUG
-    {
-        erts_aint_t w_state;
-        erts_aint_t flowstate;
-
-        /* flowstate */
-        flowstate = ethr_atomic_read(&lock->flowstate);
-        ASSERT(flowstate == 1);
-        ethr_atomic_dec(&lock->flowstate);
-
-        /* write state */
-        w_state = ethr_atomic_read(&lock->w_state);
-        ASSERT(w_state > 0);
-    }
-#endif
-    ethr_atomic_dec(&lock->w_state);
+void erts_lcnt_thread_setup() {
+    lcnt_thread_data_t__ *eltd = lcnt_thread_data_alloc();
+
+    ASSERT(eltd);
+
+    ethr_tsd_set(lcnt_thr_data_key__, eltd);
 }
 
-/* trylock */
+void erts_lcnt_thread_exit_handler() {
+    lcnt_thread_data_t__ *eltd = lcnt_get_thread_data__();
 
-void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option) {
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
-    /* Determine lock_state via res instead of state */
-    if (res != EBUSY) {
-        if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_inc(&lock->w_state);
-        if (option & ERTS_LCNT_LO_READ ) ethr_atomic_inc(&lock->r_state);
-        lcnt_update_stats(&(lock->stats[0]), 0, NULL);
-    } else {
-        ethr_atomic_inc(&lock->stats[0].tries);
-        ethr_atomic_inc(&lock->stats[0].colls);
+    if (eltd) {
+        free(eltd);
     }
 }
 
+/* - BIF interface - */
 
-void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) {
-    /* Determine lock_state via res instead of state */
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
-    if (res != EBUSY) {
+void erts_lcnt_retain_lock_info(erts_lcnt_lock_info_t *info) {
 #ifdef DEBUG
-        {
-            erts_aint_t flowstate;
-            flowstate = ethr_atomic_read(&lock->flowstate);
-            ASSERT(flowstate == 0);
-            ethr_atomic_inc( &lock->flowstate);
-        }
+    ASSERT(ethr_atomic_inc_read_acqb(&info->ref_count) >= 2);
+#else
+    ethr_atomic_inc_acqb(&info->ref_count);
 #endif
-        ethr_atomic_inc(&lock->w_state);
-        lcnt_update_stats(&(lock->stats[0]), 0, NULL);
-    } else {
-        ethr_atomic_inc(&lock->stats[0].tries);
-        ethr_atomic_inc(&lock->stats[0].colls);
-    }
 }
 
-/* thread operations */
+void erts_lcnt_release_lock_info(erts_lcnt_lock_info_t *info) {
+    ethr_sint_t count;
 
-void erts_lcnt_thread_setup(void) {
-    erts_lcnt_thread_data_t *eltd;
+    /* We need to acquire the lock before decrementing ref_count to avoid
+     * racing with list iteration; there's a short window between reading the
+     * reference to info and increasing its ref_count. */
+    lcnt_lock_list_entry_with_neighbors(info);
 
-    lcnt_lock();
-    /* lock for thread id global update */
-    eltd = lcnt_thread_data_alloc();
-    lcnt_unlock();
-    ASSERT(eltd);
-    ethr_tsd_set(lcnt_thr_data_key, eltd);
-}
+    count = ethr_atomic_dec_read(&info->ref_count);
 
-void erts_lcnt_thread_exit_handler() {
-    erts_lcnt_thread_data_t *eltd;
+    ASSERT(count >= 0);
 
-    eltd = ethr_tsd_get(lcnt_thr_data_key);
+    if(count > 0) {
+        lcnt_unlock_list_entry_with_neighbors(info);
+    } else {
+        (info->next)->prev = info->prev;
+        (info->prev)->next = info->next;
 
-    if (eltd) {
-        free(eltd);
+        lcnt_unlock_list_entry_with_neighbors(info);
+
+        info->dispose(info);
     }
 }
 
-/* bindings for bifs */
-
 Uint16 erts_lcnt_set_rt_opt(Uint16 opt) {
     Uint16 prev;
     prev = (erts_lcnt_rt_options & opt);
@@ -656,51 +507,72 @@ Uint16 erts_lcnt_clear_rt_opt(Uint16 opt) {
 }
 
 void erts_lcnt_clear_counters(void) {
-    erts_lcnt_lock_t *lock;
-    erts_lcnt_lock_list_t *list;
-    erts_lcnt_lock_stats_t *stats;
-    int i;
+    erts_lcnt_lock_info_t *iterator;
 
-    lcnt_lock();
+    lcnt_time__(&lcnt_timer_start);
 
-    list = erts_lcnt_data->current_locks;
+    iterator = NULL;
+    while(erts_lcnt_iterate_list(&lcnt_current_lock_list, &iterator)) {
+        lcnt_clear_stats(iterator);
+    }
 
-    for (lock = list->head; lock != NULL; lock = lock->next) {
-        for( i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) {
-            stats = &lock->stats[i];
-            lcnt_clear_stats(stats);
-        }
-        lock->n_stats = 1;
+    iterator = NULL;
+    while(erts_lcnt_iterate_list(&lcnt_deleted_lock_list, &iterator)) {
+        erts_lcnt_release_lock_info(iterator);
     }
+}
+
+erts_lcnt_data_t erts_lcnt_get_data(void) {
+    erts_lcnt_time_t timer_stop;
+    erts_lcnt_data_t result;
 
-    lock = erts_lcnt_data->deleted_locks->head;
-    erts_lcnt_data->deleted_locks->head = NULL;
-    erts_lcnt_data->deleted_locks->tail = NULL;
-    erts_lcnt_data->deleted_locks->n = 0;
+    lcnt_time__(&timer_stop);
 
-    lcnt_time(&timer_start);
+    result.timer_start = lcnt_timer_start;
 
-    lcnt_unlock();
+    result.current_locks = &lcnt_current_lock_list;
+    result.deleted_locks = &lcnt_deleted_lock_list;
 
-    /* free deleted locks */
-    lcnt_list_free(lock);
+    lcnt_time_diff__(&result.duration, &timer_stop, &result.timer_start);
+
+    return result;
 }
 
-erts_lcnt_data_t *erts_lcnt_get_data(void) {
-    erts_lcnt_time_t timer_stop;
+const char *erts_lcnt_lock_type(Uint16 type) {
+    switch(type & ERTS_LCNT_LT_ALL) {
+        case ERTS_LCNT_LT_SPINLOCK:   return "spinlock";
+        case ERTS_LCNT_LT_RWSPINLOCK: return "rw_spinlock";
+        case ERTS_LCNT_LT_MUTEX:      return "mutex";
+        case ERTS_LCNT_LT_RWMUTEX:    return "rw_mutex";
+        case ERTS_LCNT_LT_PROCLOCK:   return "proclock";
+        default:                      return "";
+    }
+}
 
-    lcnt_lock();
+int erts_lcnt_iterate_list(erts_lcnt_lock_info_list_t *list, erts_lcnt_lock_info_t **iterator) {
+    erts_lcnt_lock_info_t *current, *next;
 
-    lcnt_time(&timer_stop);
-    lcnt_time_diff(&(erts_lcnt_data->duration), &timer_stop, &timer_start);
+    current = *iterator ? *iterator : &list->head;
 
-    lcnt_unlock();
+    ASSERT(current != &list->tail);
 
-    return erts_lcnt_data;
-}
+    lcnt_lock_list_entry(current);
+
+    next = current->next;
+
+    if(next != &list->tail) {
+        erts_lcnt_retain_lock_info(next);
+    }
+
+    lcnt_unlock_list_entry(current);
+
+    if(current != &list->head) {
+        erts_lcnt_release_lock_info(current);
+    }
+
+    *iterator = next;
 
-char *erts_lcnt_lock_type(Uint16 type) {
-    return lcnt_lock_type(type);
+    return next != &list->tail;
 }
 
-#endif /* ifdef ERTS_ENABLE_LOCK_COUNT */
+#endif /* #ifdef ERTS_ENABLE_LOCK_COUNT */
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index 6caffbfe86..c89dfcba47 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -18,60 +18,56 @@
  * %CopyrightEnd%
  */
 
-/*
- * Description:	Statistics for locks.
- *
- * Author:	Björn-Egil Dahlberg
- * Date:	2008-07-03
- * Abstract:
- * 	Locks statistics internal representation.
- * 	
- * 	Conceptual representation, 
- * 	- set name
- * 	| - id (the unique lock)
- * 	| | - lock type
- * 	| | - statistics
- * 	| | | - location (file and line number)
- * 	| | | - tries
- * 	| | | - collisions (including trylock busy)
- * 	| | | - timer (time spent in waiting for lock)
- * 	| | | - n_timer (collisions excluding trylock busy)
- * 	| | | - histogram
- * 	| | | | - # 0 = log2(lock wait_time ns)
- * 	| | | | - ...
- * 	| | | | - # n = log2(lock wait_time ns)
- *
- * 	Each instance of a lock is the unique lock, i.e. set and id in that set.
- * 	For each lock there is a set of statistics with where and what impact
- * 	the lock aqusition had.
- *
- * 	Runtime options
- * 	- suspend, used when internal lock-counting can't be applied. For instance
- * 	  when allocating a term for the outside and halloc needs to be used.
- * 	  Default: off.
- * 	- location, reserved and not used.
- * 	- proclock, disable proclock counting. Used when performance might be an
- * 	  issue. Accessible from erts_debug:lock_counters({process_locks, bool()}).
- * 	  Default: off.
- * 	- copysave, enable saving of destroyed locks (and thereby its statistics).
- * 	  If memory constraints is an issue this need to be disabled.
- * 	  Accessible from erts_debug:lock_counters({copy_save, bool()}).
- * 	  Default: off.
+/**
+ * @description Statistics for locks.
+ * @file erl_lock_count.h
+ *
+ * @author Björn-Egil Dahlberg
+ * @author John Högberg
+ *
+ * Conceptual representation:
  *
+ *     - set name
+ *     | - id (the unique lock)
+ *     | | - lock type
+ *     | | - statistics
+ *     | | | - location (file and line number)
+ *     | | | - attempts
+ *     | | | - collisions (including trylock busy)
+ *     | | | - timer (time spent in waiting for lock)
+ *     | | | - n_timer (collisions excluding trylock busy)
+ *     | | | - histogram
+ *     | | | | - # 0 = log2(lock wait_time ns)
+ *     | | | | - ...
+ *     | | | | - # n = log2(lock wait_time ns)
+ *
+ * Each instance of a lock is the unique lock, i.e. set and id in that set.
+ * For each lock there is a set of statistics with where and what impact
+ * the lock aqusition had.
+ *
+ * Runtime options:
+ * - location, reserved and not used.
+ * - proclock, disable proclock counting. Used when performance might be an
+ *   issue. Accessible from erts_debug:lock_counters({process_locks, bool()}).
+ *   Default: off.
+ * - copysave, enable saving of destroyed locks (and thereby its statistics).
+ *   If memory constraints is an issue this need to be disabled.
+ *   Accessible from erts_debug:lock_counters({copy_save, bool()}).
+ *   Default: off.
  */
 
-#include "sys.h"
-
 #ifndef ERTS_LOCK_COUNT_H__
 #define ERTS_LOCK_COUNT_H__
 
 #ifdef  ERTS_ENABLE_LOCK_COUNT
 #ifndef ERTS_ENABLE_LOCK_POSITION
-/* Enable in order for _x variants of mtx functions to be used. */
+/** @brief Controls whether _x variants of mtx functions are used. */
 #define ERTS_ENABLE_LOCK_POSITION 1
 #endif
 
+#include "sys.h"
 #include "ethread.h"
+#include "erl_term.h"
 
 #define ERTS_LCNT_MAX_LOCK_LOCATIONS  (10)
 
@@ -106,13 +102,8 @@
                                 | ERTS_LCNT_LT_RWMUTEX    \
                                 | ERTS_LCNT_LT_PROCLOCK   )
 
-#define ERTS_LCNT_LOCK_TYPE(lock)       ((lock)->flag & ERTS_LCNT_LT_ALL)
-#define ERTS_LCNT_IS_LOCK_INVALID(lock) (!((lock)->flag & ERTS_LCNT_LT_ALL))
-#define ERTS_LCNT_CLEAR_FLAG(lock)      ((lock)->flag = 0)
-
-/* runtime options */
+/* Runtime options */
 
-#define ERTS_LCNT_OPT_SUSPEND  (((Uint16) 1) << 0)
 #define ERTS_LCNT_OPT_LOCATION (((Uint16) 1) << 1)
 #define ERTS_LCNT_OPT_PROCLOCK (((Uint16) 1) << 2)
 #define ERTS_LCNT_OPT_PORTLOCK (((Uint16) 1) << 3)
@@ -123,116 +114,818 @@ typedef struct {
     unsigned long ns;
 } erts_lcnt_time_t;
 
-extern erts_lcnt_time_t timer_start;
-
 typedef struct {
-    Uint32 ns[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; /* log2 array of nano seconds occurences */
+    /* @brief log2 array of nano seconds occurences */
+    Uint32 ns[ERTS_LCNT_HISTOGRAM_SLOT_SIZE];
 } erts_lcnt_hist_t;
 
-typedef struct erts_lcnt_lock_stats_s {
-    /* "tries" and "colls" needs to be atomic since
-     * trylock busy does not acquire a lock and there
-     * is no post action to rectify the situation
-     */
+typedef struct {
+    /** @brief In which file the lock was taken. May be NULL. */
+    const char *file;
+    /** @brief Line number in \c file */
+    unsigned int line;
+
+    /* "attempts" and "collisions" need to be atomic since try_lock busy does
+     * not acquire a lock and there is no post action to rectify the
+     * situation. */
 
-    char         *file;       /* which file the lock was taken */
-    unsigned int  line;       /* line number in file */
+    ethr_atomic_t attempts;
+    ethr_atomic_t collisions;
 
-    ethr_atomic_t tries;      /* n tries to get lock */
-    ethr_atomic_t colls;      /* n collisions of tries to get lock */
+    erts_lcnt_time_t total_time_waited;
+    Uint64 times_waited;
 
-    unsigned long timer_n;    /* #times waited for lock */
-    erts_lcnt_time_t timer;   /* total wait time for lock */
-    erts_lcnt_hist_t hist;
+    erts_lcnt_hist_t wait_time_histogram;
 } erts_lcnt_lock_stats_t;
 
-/* rw locks uses both states, other locks only uses w_state */
-typedef struct erts_lcnt_lock_s {
-    char *name;            /* lock name */
-    Uint16 flag;           /* lock type */
-    Eterm id;              /* id if possible */
+typedef struct lcnt_lock_info_t_ {
+    const char *name; /**< Lock name */
+    Uint16 flag;      /**< Lock type */
+    Eterm id;         /**< Id if possible, must be an immediate */
 
-#ifdef DEBUG
+    /* The first entry is reserved as a fallback for when location information
+     * is missing, and when the lock is used in more than (MAX_LOCK_LOCATIONS
+     * - 1) different places. */
+    erts_lcnt_lock_stats_t location_stats[ERTS_LCNT_MAX_LOCK_LOCATIONS];
+    unsigned int location_count;
+
+    /* -- Everything below is internal to this module ---------------------- */
+
+    /* Lock states; rw locks uses both states, other locks only uses w_state */
+
+    /** @brief Write state. 0 = not taken, otherwise n threads waiting */
+    ethr_atomic_t w_state;
+    /** @brief Read state. 0 = not taken, > 0 -> writes will wait */
+    ethr_atomic_t r_state;
+
+#ifdef LCNT_DEBUG_LOCK_FLOW
+    /** @brief Tracks lock/unlock operations. This will explode if the lock is
+     * held at the time lock counting is installed.
+     *
+     * Avoid enabling existing locks at runtime while running in this
+     * configuration. */
     ethr_atomic_t flowstate;
 #endif
 
-    /* lock states */
-    ethr_atomic_t w_state; /* 0 not taken, otherwise n threads waiting */
-    ethr_atomic_t r_state; /* 0 not taken, > 0 -> writes will wait */
+    struct lcnt_lock_info_t_ *prev;
+    struct lcnt_lock_info_t_ *next;
 
-    /* statistics */
-    unsigned int n_stats;
-    erts_lcnt_lock_stats_t stats[ERTS_LCNT_MAX_LOCK_LOCATIONS]; /* first entry is "undefined"*/
+    /** @brief Used in place of erts_refc_t to avoid a circular dependency. */
+    ethr_atomic_t ref_count;
+    ethr_atomic32_t lock;
 
-    /* chains for list handling  */
-    /* data is hold by lcnt_lock */
-    struct erts_lcnt_lock_s *prev;
-    struct erts_lcnt_lock_s *next;
-} erts_lcnt_lock_t;
+    /** @brief Deletion hook called once \c ref_count reaches 0; may defer
+     * deletion by modifying \c ref_count. */
+    void (*dispose)(struct lcnt_lock_info_t_ *);
 
-typedef struct {
-    erts_lcnt_lock_t *head;
-    erts_lcnt_lock_t *tail;
-    unsigned long n;
-} erts_lcnt_lock_list_t;
+    struct lcnt_lock_info_carrier_ *carrier;
+} erts_lcnt_lock_info_t;
+
+typedef struct lcnt_lock_info_list_ {
+    erts_lcnt_lock_info_t head;
+    erts_lcnt_lock_info_t tail;
+} erts_lcnt_lock_info_list_t;
 
 typedef struct {
-    erts_lcnt_time_t duration;			/* time since last clear */
-    erts_lcnt_lock_list_t *current_locks;
-    erts_lcnt_lock_list_t *deleted_locks;
+    erts_lcnt_time_t timer_start; /**< Time of last clear */
+    erts_lcnt_time_t duration; /**< Time since last clear */
+
+    erts_lcnt_lock_info_list_t *current_locks;
+    erts_lcnt_lock_info_list_t *deleted_locks;
 } erts_lcnt_data_t;
 
-typedef struct {
-    int id;
+typedef struct lcnt_lock_info_carrier_ erts_lcnt_lock_info_carrier_t;
 
-    erts_lcnt_time_t timer;	/* timer         */
-    int timer_set;		/* bool          */
-    int lock_in_conflict;       /* bool          */
-} erts_lcnt_thread_data_t;
+typedef ethr_atomic_t erts_lcnt_ref_t;
 
-/* globals */
+/* -- Globals -------------------------------------------------------------- */
 
 extern Uint16 erts_lcnt_rt_options;
 
-/* function declerations */
-
-void erts_lcnt_init(void);
+/* -- Lock operations ------------------------------------------------------
+ *
+ * All of these will nop if there's nothing "installed" on the given reference,
+ * in order to transparently support enable/disable at runtime. */
+
+/** @brief Records that a lock is being acquired. */
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_lock(erts_lcnt_ref_t *ref);
+
+/** @copydoc erts_lcnt_lock
+ * @param option Notes whether the lock is a read or write lock. */
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_lock_opt(erts_lcnt_ref_t *ref, Uint16 option);
+
+/** @brief Records that a lock has been acquired. */
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_lock_post(erts_lcnt_ref_t *ref);
+
+/** @copydoc erts_lcnt_lock_post.
+ * @param file The name of the file where the lock was acquired.
+ * @param line The line at which the lock was acquired. */
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_lock_post_x(erts_lcnt_ref_t *ref, char *file, unsigned int line);
+
+/** @brief Records that a lock has been released. */
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_unlock(erts_lcnt_ref_t *ref);
+
+/** @copydoc erts_lcnt_unlock_opt
+ * @param option Whether the lock is a read or write lock. */
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_unlock_opt(erts_lcnt_ref_t *ref, Uint16 option);
+
+/** @brief Rectifies the case where a lock wasn't actually a lock operation.
+ *
+ * Only used for process locks at the moment. */
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_lock_unacquire(erts_lcnt_ref_t *ref);
+
+/** @brief Records the result of a trylock, placing the queried lock status in
+ * \c result. */
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_trylock(erts_lcnt_ref_t *ref, int result);
+
+/** @copydoc erts_lcnt_trylock
+ * @param option Whether the lock is a read or write lock. */
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_trylock_opt(erts_lcnt_ref_t *ref, int result, Uint16 option);
+
+/* Indexed variants of the standard lock operations, for use when a single
+ * reference contains many counters (eg. process locks).
+ *
+ * erts_lcnt_open_ref must be used to safely extract the installed carrier,
+ * which must released with erts_lcnt_close_reference on success.
+ *
+ * Refer to \c erts_lcnt_lock for example usage. */
+
+ERTS_GLB_FORCE_INLINE
+int erts_lcnt_open_ref(erts_lcnt_ref_t *ref, int *handle, erts_lcnt_lock_info_carrier_t **result);
+
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_close_ref(int handle, erts_lcnt_lock_info_carrier_t *carrier);
+
+ERTS_GLB_INLINE
+void erts_lcnt_lock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index);
+ERTS_GLB_INLINE
+void erts_lcnt_lock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, Uint16 option);
+
+ERTS_GLB_INLINE
+void erts_lcnt_lock_post_idx(erts_lcnt_lock_info_carrier_t *carrier, int index);
+ERTS_GLB_INLINE
+void erts_lcnt_lock_post_x_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, char *file, unsigned int line);
+
+ERTS_GLB_INLINE
+void erts_lcnt_lock_unacquire_idx(erts_lcnt_lock_info_carrier_t *carrier, int index);
+
+ERTS_GLB_INLINE
+void erts_lcnt_unlock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index);
+ERTS_GLB_INLINE
+void erts_lcnt_unlock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, Uint16 option);
+
+ERTS_GLB_INLINE
+void erts_lcnt_trylock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result);
+ERTS_GLB_INLINE
+void erts_lcnt_trylock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result, Uint16 option);
+
+/* -- Reference operations ------------------------------------------------- */
+
+erts_lcnt_lock_info_carrier_t *erts_lcnt_create_lock_info_carrier(int count);
+
+/** @brief Fills in the name and lock type of the given index. */
+#define erts_lcnt_init_lock_info_idx(carrier, index, name, flag) \
+    erts_lcnt_init_lock_info_x_idx(carrier, index, name, flag, NIL)
+
+/** @copydoc erts_lcnt_install_new_lock_info
+ * @param id An immediate erlang term with whatever extra data you want to
+ * identify this lock with. */
+ERTS_GLB_INLINE
+void erts_lcnt_init_lock_info_x_idx(erts_lcnt_lock_info_carrier_t *carrier, int index,
+    const char *name, Uint16 flag, Eterm id);
+
+/** @brief Initializes a lock counter reference; this must be called prior to
+ * using any other functions in this module. */
+#define erts_lcnt_init_ref(ref) ethr_atomic_init(ref, (ethr_sint_t)NULL);
+
+/** @brief Atomically installs the given lock counters. Nops (and releases the
+ * provided carrier) if something was already installed. */
+void erts_lcnt_install(erts_lcnt_ref_t *ref, erts_lcnt_lock_info_carrier_t *carrier);
+
+/** @brief Atomically removes the currently installed lock counters. Nops if
+ * nothing was installed. */
+void erts_lcnt_uninstall(erts_lcnt_ref_t *ref);
+
+/** @brief Convenience macro to install a single lock counter of the given
+ * name and type. */
+#define erts_lcnt_install_new_lock_info(reference, name, flag) \
+    erts_lcnt_install_new_lock_info_x(reference, name, flag, NIL)
+
+/** @copydoc erts_lcnt_install_new_lock_info
+ * @param id An immediate erlang term with whatever extra data you want to
+ * identify this lock with. */
+#define erts_lcnt_install_new_lock_info_x(reference, name, flag, id) \
+    do { \
+        erts_lcnt_lock_info_carrier_t *__carrier; \
+        __carrier = erts_lcnt_create_lock_info_carrier(1); \
+        erts_lcnt_init_lock_info_x_idx(__carrier, 0, name, flag, id); \
+        erts_lcnt_install(reference, __carrier); \
+    } while(0)
+
+/* -- Module initialization ------------------------------------------------ */
+
+void erts_lcnt_pre_thr_init(void);
+void erts_lcnt_post_thr_init(void);
 void erts_lcnt_late_init(void);
 
-/* thread operations */
 void erts_lcnt_thread_setup(void);
 void erts_lcnt_thread_exit_handler(void);
 
-/* list operations (local)  */
-erts_lcnt_lock_list_t *erts_lcnt_list_init(void);
+/* -- BIF interface -------------------------------------------------------- */
 
-void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock);
-void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock);
-
-/* lock operations (global) */
-void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag);
-void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id);
-void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock);
-void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock);
+/** @brief Safely iterates through all entries in the given list.
+ *
+ * The referenced item will be valid until the next call to
+ * \c erts_lcnt_iterate_list after which point it may be destroyed; call
+ * erts_lcnt_retain_lock_info if you wish to hang on to it beyond that point.
+ *
+ * Iteration can be cancelled by calling erts_lcnt_release_lock_info on the
+ * iterator and breaking out of the loop.
+ *
+ * @param iterator The iteration variable; set the pointee to NULL to start
+ * iteration.
+ * @return 1 while the iterator is valid, 0 at the end of the list. */
+int erts_lcnt_iterate_list(erts_lcnt_lock_info_list_t *list, erts_lcnt_lock_info_t **iterator);
 
-void erts_lcnt_lock(erts_lcnt_lock_t *lock);
-void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option);
-void erts_lcnt_lock_post(erts_lcnt_lock_t *lock);
-void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line);
-void erts_lcnt_lock_unaquire(erts_lcnt_lock_t *lock);
+/** @brief Clears the counter state of all locks, and releases all locks
+ * preserved through ERTS_LCNT_OPT_COPYSAVE (if any). */
+void erts_lcnt_clear_counters(void);
 
-void erts_lcnt_unlock(erts_lcnt_lock_t *lock);
-void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option);
+/** @brief Retrieves the global lock counter state.
+ *
+ * Note that the lists may be modified while you're mucking around with them.
+ * Always use \c erts_lcnt_iterate_list to enumerate them. */
+erts_lcnt_data_t erts_lcnt_get_data(void);
 
-void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option);
-void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res);
+void erts_lcnt_retain_lock_info(erts_lcnt_lock_info_t *info);
+void erts_lcnt_release_lock_info(erts_lcnt_lock_info_t *info);
 
-/* bif interface */
 Uint16 erts_lcnt_set_rt_opt(Uint16 opt);
 Uint16 erts_lcnt_clear_rt_opt(Uint16 opt);
-void   erts_lcnt_clear_counters(void);
-char  *erts_lcnt_lock_type(Uint16 type);
-erts_lcnt_data_t *erts_lcnt_get_data(void);
+
+const char *erts_lcnt_lock_type(Uint16 type);
+
+/* -- Inline implementation ------------------------------------------------ */
+
+/* The following is a hack to get the things we need from erl_thr_progress.h,
+ * which we can't #include without dependency hell breaking loose.
+ *
+ * The size of LcntThrPrgrLaterOp and value of the constant are verified at
+ * compile-time in erts_lcnt_pre_thr_init. */
+
+int lcnt_thr_progress_unmanaged_delay__(void);
+void lcnt_thr_progress_unmanaged_continue__(int handle);
+typedef struct { Uint64 _[4]; } LcntThrPrgrLaterOp;
+#define LCNT_THR_PRGR_DHANDLE_MANAGED -1
+
+struct lcnt_lock_info_carrier_ {
+    ethr_atomic_t ref_count;
+
+    void (*deallocate)(struct lcnt_lock_info_carrier_ *);
+
+    LcntThrPrgrLaterOp release_entries;
+
+    unsigned char entry_count;
+    erts_lcnt_lock_info_t entries[];
+};
+
+typedef struct {
+    erts_lcnt_time_t timer; /* timer */
+    int timer_set;          /* bool */
+    int lock_in_conflict;   /* bool */
+} lcnt_thread_data_t__;
+
+extern ethr_tsd_key lcnt_thr_data_key__;
+
+/** @brief Some operations (eg erts_alloc or erts_thr_progress_unmanaged_delay)
+ * are unsafe in the early stages of initialization, so we're using this flag
+ * to know when we can move over to normal operation. */
+extern int lcnt_initialization_completed__;
+
+extern const int lcnt_log2_tab64__[];
+
+ERTS_GLB_INLINE
+int lcnt_log2__(Uint64 v);
+
+ERTS_GLB_INLINE
+void lcnt_update_wait_histogram__(erts_lcnt_hist_t *hist, erts_lcnt_time_t *time_waited);
+
+ERTS_GLB_INLINE
+void lcnt_update_stats__(erts_lcnt_lock_stats_t *stats, int lock_in_conflict, erts_lcnt_time_t *time_waited);
+
+ERTS_GLB_INLINE
+erts_lcnt_lock_stats_t *lcnt_get_lock_stats__(erts_lcnt_lock_info_t *info, char *file, unsigned int line);
+
+ERTS_GLB_INLINE
+void lcnt_dec_lock_state__(ethr_atomic_t *l_state);
+
+ERTS_GLB_INLINE
+void lcnt_time__(erts_lcnt_time_t *time);
+
+ERTS_GLB_INLINE
+void lcnt_time_add__(erts_lcnt_time_t *t, erts_lcnt_time_t *d);
+
+ERTS_GLB_INLINE
+void lcnt_time_diff__(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0);
+
+ERTS_GLB_INLINE
+void lcnt_retain_carrier__(erts_lcnt_lock_info_carrier_t *carrier);
+
+ERTS_GLB_INLINE
+void lcnt_release_carrier__(erts_lcnt_lock_info_carrier_t *carrier);
+
+ERTS_GLB_INLINE
+lcnt_thread_data_t__ *lcnt_get_thread_data__(void);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE
+void lcnt_time__(erts_lcnt_time_t *time) {
+    /*
+     * erts_sys_hrtime() is the highest resolution
+     * we could find, it may or may not be monotonic...
+     */
+    ErtsMonotonicTime mtime = erts_sys_hrtime();
+    time->s  = (unsigned long) (mtime / 1000000000LL);
+    time->ns = (unsigned long) (mtime - 1000000000LL*time->s);
+}
+
+/* difference d must be non-negative */
+
+ERTS_GLB_INLINE
+void lcnt_time_add__(erts_lcnt_time_t *t, erts_lcnt_time_t *d) {
+    t->s  += d->s;
+    t->ns += d->ns;
+
+    t->s  += t->ns / 1000000000LL;
+    t->ns  = t->ns % 1000000000LL;
+}
+
+ERTS_GLB_INLINE
+void lcnt_time_diff__(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0) {
+    long ds;
+    long dns;
+
+    ds  = t1->s  - t0->s;
+    dns = t1->ns - t0->ns;
+
+    /* the difference should not be able to get bigger than 1 sec in ns*/
+
+    if (dns < 0) {
+        ds  -= 1;
+        dns += 1000000000LL;
+    }
+
+    ASSERT(ds >= 0);
+
+    d->s  = ds;
+    d->ns = dns;
+}
+
+ERTS_GLB_INLINE
+int lcnt_log2__(Uint64 v) {
+    v |= v >> 1;
+    v |= v >> 2;
+    v |= v >> 4;
+    v |= v >> 8;
+    v |= v >> 16;
+    v |= v >> 32;
+
+    return lcnt_log2_tab64__[((Uint64)((v - (v >> 1))*0x07EDD5E59A4E28C2)) >> 58];
+}
+
+ERTS_GLB_INLINE
+void lcnt_update_wait_histogram__(erts_lcnt_hist_t *hist, erts_lcnt_time_t *time_waited) {
+    int idx;
+
+    if(time_waited->s > 0 || time_waited->ns > ERTS_LCNT_HISTOGRAM_MAX_NS) {
+        idx = ERTS_LCNT_HISTOGRAM_SLOT_SIZE - 1;
+    } else {
+        unsigned long r = time_waited->ns >> ERTS_LCNT_HISTOGRAM_RSHIFT;
+
+        idx = r ? lcnt_log2__(r) : 0;
+    }
+
+    hist->ns[idx]++;
+}
+
+ERTS_GLB_INLINE
+void lcnt_update_stats__(erts_lcnt_lock_stats_t *stats, int lock_in_conflict, erts_lcnt_time_t *time_waited) {
+    ethr_atomic_inc(&stats->attempts);
+
+    if(lock_in_conflict) {
+        ethr_atomic_inc(&stats->collisions);
+    }
+
+    if(time_waited) {
+        stats->times_waited++;
+
+        lcnt_time_add__(&stats->total_time_waited, time_waited);
+        lcnt_update_wait_histogram__(&stats->wait_time_histogram, time_waited);
+    }
+}
+
+/* If we were installed while the lock was held, r/w_state will be 0 and we
+ * can't tell which unlock or unacquire operation was the last. To get around
+ * this we assume that all excess operations go *towards* zero rather than down
+ * to zero, eventually becoming consistent with the actual state once the lock
+ * is fully released.
+ *
+ * Conflicts might not be counted until the recorded state is fully consistent
+ * with the actual state, but there should be no other ill effects. */
+
+ERTS_GLB_INLINE
+void lcnt_dec_lock_state__(ethr_atomic_t *l_state) {
+    ethr_sint_t state = ethr_atomic_dec_read_acqb(l_state);
+
+    /* We can not assume that state is >= -1 here; unlock and unacquire might
+     * bring it below -1 and race to increment it back. */
+
+    if(state < 0) {
+        ethr_atomic_inc_acqb(l_state);
+    }
+}
+
+ERTS_GLB_INLINE
+erts_lcnt_lock_stats_t *lcnt_get_lock_stats__(erts_lcnt_lock_info_t *info, char *file, unsigned int line) {
+    ASSERT(info->location_count >= 1 && info->location_count <= ERTS_LCNT_MAX_LOCK_LOCATIONS);
+
+    if(erts_lcnt_rt_options & ERTS_LCNT_OPT_LOCATION) {
+        unsigned int i;
+
+        for(i = 0; i < info->location_count; i++) {
+            erts_lcnt_lock_stats_t *stats = &info->location_stats[i];
+
+            if(stats->file == file && stats->line == line) {
+                return stats;
+            }
+        }
+
+        if(info->location_count < ERTS_LCNT_MAX_LOCK_LOCATIONS) {
+            erts_lcnt_lock_stats_t *stats = &info->location_stats[info->location_count];
+
+            stats->file = file;
+            stats->line = line;
+
+            info->location_count++;
+
+            return stats;
+        }
+    }
+
+    return &info->location_stats[0];
+}
+
+ERTS_GLB_INLINE
+lcnt_thread_data_t__ *lcnt_get_thread_data__(void) {
+    lcnt_thread_data_t__ *eltd = (lcnt_thread_data_t__ *)ethr_tsd_get(lcnt_thr_data_key__);
+
+    ASSERT(eltd);
+
+    return eltd;
+}
+
+ERTS_GLB_FORCE_INLINE
+int erts_lcnt_open_ref(erts_lcnt_ref_t *ref, int *handle, erts_lcnt_lock_info_carrier_t **result) {
+    if(!*ethr_atomic_addr(ref) || !lcnt_initialization_completed__) {
+        return 0;
+    }
+
+    (*handle) = lcnt_thr_progress_unmanaged_delay__();
+    (*result) = (erts_lcnt_lock_info_carrier_t*)ethr_atomic_read(ref);
+
+    if(*result) {
+        if(*handle != LCNT_THR_PRGR_DHANDLE_MANAGED) {
+            lcnt_retain_carrier__(*result);
+            lcnt_thr_progress_unmanaged_continue__(*handle);
+        }
+
+        return 1;
+    } else if(*handle != LCNT_THR_PRGR_DHANDLE_MANAGED) {
+        lcnt_thr_progress_unmanaged_continue__(*handle);
+    }
+
+    return 0;
+}
+
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_close_ref(int handle, erts_lcnt_lock_info_carrier_t *carrier) {
+    if(handle != LCNT_THR_PRGR_DHANDLE_MANAGED) {
+        lcnt_release_carrier__(carrier);
+    }
+}
+
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_lock(erts_lcnt_ref_t *ref) {
+    erts_lcnt_lock_info_carrier_t *carrier;
+    int handle;
+
+    if(erts_lcnt_open_ref(ref, &handle, &carrier)) {
+        erts_lcnt_lock_idx(carrier, 0);
+
+        erts_lcnt_close_ref(handle, carrier);
+    }
+}
+
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_lock_opt(erts_lcnt_ref_t *ref, Uint16 option) {
+    erts_lcnt_lock_info_carrier_t *carrier;
+    int handle;
+
+    if(erts_lcnt_open_ref(ref, &handle, &carrier)) {
+        erts_lcnt_lock_opt_idx(carrier, 0, option);
+
+        erts_lcnt_close_ref(handle, carrier);
+    }
+}
+
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_lock_post(erts_lcnt_ref_t *ref) {
+    erts_lcnt_lock_info_carrier_t *carrier;
+    int handle;
+
+    if(erts_lcnt_open_ref(ref, &handle, &carrier)) {
+        erts_lcnt_lock_post_idx(carrier, 0);
+
+        erts_lcnt_close_ref(handle, carrier);
+    }
+}
+
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_lock_post_x(erts_lcnt_ref_t *ref, char *file, unsigned int line) {
+    erts_lcnt_lock_info_carrier_t *carrier;
+    int handle;
+
+    if(erts_lcnt_open_ref(ref, &handle, &carrier)) {
+        erts_lcnt_lock_post_x_idx(carrier, 0, file, line);
+
+        erts_lcnt_close_ref(handle, carrier);
+    }
+}
+
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_lock_unacquire(erts_lcnt_ref_t *ref) {
+    erts_lcnt_lock_info_carrier_t *carrier;
+    int handle;
+
+    if(erts_lcnt_open_ref(ref, &handle, &carrier)) {
+        erts_lcnt_lock_unacquire_idx(carrier, 0);
+
+        erts_lcnt_close_ref(handle, carrier);
+    }
+}
+
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_unlock(erts_lcnt_ref_t *ref) {
+    erts_lcnt_lock_info_carrier_t *carrier;
+    int handle;
+
+    if(erts_lcnt_open_ref(ref, &handle, &carrier)) {
+        erts_lcnt_unlock_idx(carrier, 0);
+
+        erts_lcnt_close_ref(handle, carrier);
+    }
+}
+
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_unlock_opt(erts_lcnt_ref_t *ref, Uint16 option) {
+    erts_lcnt_lock_info_carrier_t *carrier;
+    int handle;
+
+    if(erts_lcnt_open_ref(ref, &handle, &carrier)) {
+        erts_lcnt_unlock_opt_idx(carrier, 0, option);
+
+        erts_lcnt_close_ref(handle, carrier);
+    }
+}
+
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_trylock(erts_lcnt_ref_t *ref, int result) {
+    erts_lcnt_lock_info_carrier_t *carrier;
+    int handle;
+
+    if(erts_lcnt_open_ref(ref, &handle, &carrier)) {
+        erts_lcnt_trylock_idx(carrier, 0, result);
+
+        erts_lcnt_close_ref(handle, carrier);
+    }
+}
+
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_trylock_opt(erts_lcnt_ref_t *ref, int result, Uint16 option) {
+    erts_lcnt_lock_info_carrier_t *carrier;
+    int handle;
+
+    if(erts_lcnt_open_ref(ref, &handle, &carrier)) {
+        erts_lcnt_trylock_opt_idx(carrier, 0, result, option);
+
+        erts_lcnt_close_ref(handle, carrier);
+    }
+}
+
+ERTS_GLB_INLINE
+void erts_lcnt_lock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index) {
+    erts_lcnt_lock_opt_idx(carrier, index, ERTS_LCNT_LO_WRITE);
+}
+
+ERTS_GLB_INLINE
+void erts_lcnt_lock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, Uint16 option) {
+    erts_lcnt_lock_info_t *info = &carrier->entries[index];
+
+    lcnt_thread_data_t__ *eltd = lcnt_get_thread_data__();
+
+    ASSERT(index < carrier->entry_count);
+
+    ASSERT((option & ERTS_LCNT_LO_READ) || (option & ERTS_LCNT_LO_WRITE));
+
+    if(option & ERTS_LCNT_LO_WRITE) {
+        ethr_sint_t w_state, r_state;
+
+        w_state = ethr_atomic_inc_read(&info->w_state) - 1;
+        r_state = ethr_atomic_read(&info->r_state);
+
+        /* We cannot acquire w_lock if either w or r are taken */
+        eltd->lock_in_conflict = (w_state > 0) || (r_state > 0);
+    } else {
+        ethr_sint_t w_state = ethr_atomic_read(&info->w_state);
+
+        /* We cannot acquire r_lock if w_lock is taken */
+        eltd->lock_in_conflict = (w_state > 0);
+    }
+
+    if(option & ERTS_LCNT_LO_READ) {
+        ethr_atomic_inc(&info->r_state);
+    }
+
+    if(eltd->lock_in_conflict) {
+        /* Only set the timer if nobody else has it. This should only happen
+         * when proc_locks acquires several locks "atomically." All other locks
+         * will block the thread when locked (w_state > 0) */
+        if(eltd->timer_set == 0) {
+            lcnt_time__(&eltd->timer);
+        }
+
+        eltd->timer_set++;
+    }
+}
+
+ERTS_GLB_INLINE
+void erts_lcnt_lock_post_idx(erts_lcnt_lock_info_carrier_t *carrier, int index) {
+    erts_lcnt_lock_post_x_idx(carrier, index, NULL, 0);
+}
+
+ERTS_GLB_INLINE
+void erts_lcnt_lock_post_x_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, char *file, unsigned int line) {
+    erts_lcnt_lock_info_t *info = &carrier->entries[index];
+
+    lcnt_thread_data_t__ *eltd = lcnt_get_thread_data__();
+    erts_lcnt_lock_stats_t *stats;
+
+    ASSERT(index < carrier->entry_count);
+
+#ifdef LCNT_DEBUG_LOCK_FLOW
+    if(!(info->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) {
+        ASSERT(ethr_atomic_inc_read(&info->flowstate) == 1);
+    }
+#endif
+
+    /* If the lock was in conflict, update the time spent waiting. */
+    stats = lcnt_get_lock_stats__(info, file, line);
+    if(eltd->timer_set) {
+        erts_lcnt_time_t time_wait;
+        erts_lcnt_time_t timer;
+
+        lcnt_time__(&timer);
+
+        lcnt_time_diff__(&time_wait, &timer, &eltd->timer);
+        lcnt_update_stats__(stats, eltd->lock_in_conflict, &time_wait);
+
+        eltd->timer_set--;
+
+        ASSERT(eltd->timer_set >= 0);
+    } else {
+        lcnt_update_stats__(stats, eltd->lock_in_conflict, NULL);
+    }
+}
+
+ERTS_GLB_INLINE
+void erts_lcnt_unlock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index) {
+#ifdef LCNT_DEBUG_LOCK_FLOW
+    erts_lcnt_lock_info_t *info = &carrier->entries[index];
+
+    ASSERT(ethr_atomic_dec_read(&info->flowstate) == 0);
+#endif
+
+    ASSERT(index < carrier->entry_count);
+
+    erts_lcnt_unlock_opt_idx(carrier, index, ERTS_LCNT_LO_WRITE);
+}
+
+ERTS_GLB_INLINE
+void erts_lcnt_unlock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, Uint16 option) {
+    erts_lcnt_lock_info_t *info = &carrier->entries[index];
+
+    ASSERT(index < carrier->entry_count);
+
+    ASSERT((option & ERTS_LCNT_LO_READ) || (option & ERTS_LCNT_LO_WRITE));
+
+    if(option & ERTS_LCNT_LO_WRITE) {
+        lcnt_dec_lock_state__(&info->w_state);
+    }
+
+    if(option & ERTS_LCNT_LO_READ) {
+        lcnt_dec_lock_state__(&info->r_state);
+    }
+}
+
+ERTS_GLB_INLINE
+void erts_lcnt_lock_unacquire_idx(erts_lcnt_lock_info_carrier_t *carrier, int index) {
+    erts_lcnt_lock_info_t *info = &carrier->entries[index];
+
+    ASSERT(index < carrier->entry_count);
+
+    lcnt_dec_lock_state__(&info->w_state);
+}
+
+ERTS_GLB_INLINE
+void erts_lcnt_trylock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result) {
+#ifdef LCNT_DEBUG_LOCK_FLOW
+    erts_lcnt_lock_info_t *info = &carrier->entries[index];
+
+    ASSERT(result == EBUSY || ethr_atomic_inc_read(&info->flowstate) == 1);
+#endif
+
+    ASSERT(index < carrier->entry_count);
+
+    erts_lcnt_trylock_opt_idx(carrier, index, result, ERTS_LCNT_LO_WRITE);
+}
+
+ERTS_GLB_INLINE
+void erts_lcnt_trylock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result, Uint16 option) {
+    erts_lcnt_lock_info_t *info = &carrier->entries[index];
+
+    ASSERT(index < carrier->entry_count);
+
+    ASSERT((option & ERTS_LCNT_LO_READ) || (option & ERTS_LCNT_LO_WRITE));
+
+    if(result != EBUSY) {
+        if(option & ERTS_LCNT_LO_WRITE) {
+            ethr_atomic_inc(&info->w_state);
+        }
+
+        if(option & ERTS_LCNT_LO_READ) {
+            ethr_atomic_inc(&info->r_state);
+        }
+
+        lcnt_update_stats__(&info->location_stats[0], 0, NULL);
+    } else {
+        ethr_atomic_inc(&info->location_stats[0].attempts);
+        ethr_atomic_inc(&info->location_stats[0].collisions);
+    }
+}
+
+ERTS_GLB_INLINE
+void erts_lcnt_init_lock_info_x_idx(erts_lcnt_lock_info_carrier_t *carrier, int index,
+                                    const char *name, Uint16 flag, Eterm id) {
+    erts_lcnt_lock_info_t *info = &carrier->entries[index];
+
+    ASSERT(is_immed(id));
+
+    info->flag = flag;
+    info->name = name;
+    info->id = id;
+}
+
+ERTS_GLB_INLINE
+void lcnt_retain_carrier__(erts_lcnt_lock_info_carrier_t *carrier) {
+#ifdef DEBUG
+    ASSERT(ethr_atomic_inc_read_acqb(&carrier->ref_count) >= 2);
+#else
+    ethr_atomic_inc_acqb(&carrier->ref_count);
+#endif
+}
+
+ERTS_GLB_INLINE
+void lcnt_release_carrier__(erts_lcnt_lock_info_carrier_t *carrier) {
+    ethr_sint_t count = ethr_atomic_dec_read_relb(&carrier->ref_count);
+
+    ASSERT(count >= 0);
+
+    if(count == 0) {
+        carrier->deallocate(carrier);
+    }
+}
+
+#endif
 
 #endif /* ifdef  ERTS_ENABLE_LOCK_COUNT  */
 #endif /* ifndef ERTS_LOCK_COUNT_H__     */
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index c0e7380ed0..59577bf422 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -944,7 +944,7 @@ erts_pid2proc_opt(Process *c_p,
 			erts_proc_inc_refc(proc);
 
 #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
-		    erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks);
+		    erts_lcnt_proc_lock_unacquire(&proc->lock, lcnt_locks);
 #endif
 
 		    managed = dhndl == ERTS_THR_PRGR_DHANDLE_MANAGED;
@@ -1127,114 +1127,52 @@ erts_proc_lock_fin(Process *p)
 void erts_lcnt_enable_proc_lock_count(int enable) {
     int ix, max = erts_ptab_max(&erts_proc);
     Process *proc = NULL;
-    for (ix = 0; ix < max; ++ix) {
-	if ((proc = erts_pix2proc(ix)) != NULL)
+    for(ix = 0; ix < max; ++ix) {
+        if((proc = erts_pix2proc(ix)) != NULL) {
             lcnt_enable_proc_lock_count(proc, enable);
+        }
     } /* for all processes */
 }
 
 void erts_lcnt_proc_lock_init(Process *p) {
-    if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) {
-        erts_lcnt_init_lock_empty(&(p->lock.lcnt_main));
-        erts_lcnt_init_lock_empty(&(p->lock.lcnt_link));
-        erts_lcnt_init_lock_empty(&(p->lock.lcnt_msgq));
-        erts_lcnt_init_lock_empty(&(p->lock.lcnt_btm));
-        erts_lcnt_init_lock_empty(&(p->lock.lcnt_status));
-        erts_lcnt_init_lock_empty(&(p->lock.lcnt_trace));
-    } else { /* now the common case */
-        Eterm pid = (p->common.id != ERTS_INVALID_PID) ? p->common.id : NIL;
-        erts_lcnt_init_lock_x(&(p->lock.lcnt_main),  "proc_main",  ERTS_LCNT_LT_PROCLOCK, pid);
-        erts_lcnt_init_lock_x(&(p->lock.lcnt_link),  "proc_link",  ERTS_LCNT_LT_PROCLOCK, pid);
-        erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq),  "proc_msgq",  ERTS_LCNT_LT_PROCLOCK, pid);
-        erts_lcnt_init_lock_x(&(p->lock.lcnt_btm),   "proc_btm",   ERTS_LCNT_LT_PROCLOCK, pid);
-        erts_lcnt_init_lock_x(&(p->lock.lcnt_status),"proc_status",ERTS_LCNT_LT_PROCLOCK, pid);
-        erts_lcnt_init_lock_x(&(p->lock.lcnt_trace), "proc_trace", ERTS_LCNT_LT_PROCLOCK, pid);
-    } /* the lock names should really be aligned to four characters */
+    erts_lcnt_init_ref(&p->lock.lcnt_carrier);
+
+    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) {
+        lcnt_enable_proc_lock_count(p, 1);
+    }
 } /* logic reversed */
 
 void erts_lcnt_proc_lock_destroy(Process *p) {
-    erts_lcnt_destroy_lock(&(p->lock.lcnt_main));
-    erts_lcnt_destroy_lock(&(p->lock.lcnt_link));
-    erts_lcnt_destroy_lock(&(p->lock.lcnt_msgq));
-    erts_lcnt_destroy_lock(&(p->lock.lcnt_btm));
-    erts_lcnt_destroy_lock(&(p->lock.lcnt_status));
-    erts_lcnt_destroy_lock(&(p->lock.lcnt_trace));
+    erts_lcnt_uninstall(&p->lock.lcnt_carrier);
 }
 
 static void lcnt_enable_proc_lock_count(Process *proc, int enable) {
     if (enable) {
-        if (!ERTS_LCNT_LOCK_TYPE(&(proc->lock.lcnt_main))) {
-            erts_lcnt_proc_lock_init(proc);
-        }
-    }
-    else {
-        if (ERTS_LCNT_LOCK_TYPE(&(proc->lock.lcnt_main))) {
-            erts_lcnt_proc_lock_destroy(proc);
-        }
-    }
-}
-
-void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) {
-    if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
-    if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock(&(lock->lcnt_main)); }
-    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock(&(lock->lcnt_link)); }
-    if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock(&(lock->lcnt_msgq)); }
-    if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock(&(lock->lcnt_btm)); }
-    if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock(&(lock->lcnt_status)); }
-    if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_lock(&(lock->lcnt_trace)); }
-}
-
-void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks,
-                                char *file, unsigned int line) {
-    if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
-    if (locks & ERTS_PROC_LOCK_MAIN) {
-	erts_lcnt_lock_post_x(&(lock->lcnt_main), file, line);
-    }
-    if (locks & ERTS_PROC_LOCK_LINK) {
-	erts_lcnt_lock_post_x(&(lock->lcnt_link), file, line);
-    }
-    if (locks & ERTS_PROC_LOCK_MSGQ) {
-        erts_lcnt_lock_post_x(&(lock->lcnt_msgq), file, line);
-    }
-    if (locks & ERTS_PROC_LOCK_BTM) {
-        erts_lcnt_lock_post_x(&(lock->lcnt_btm), file, line);
-    }
-    if (locks & ERTS_PROC_LOCK_STATUS) {
-	erts_lcnt_lock_post_x(&(lock->lcnt_status), file, line);
-    }
-    if (locks & ERTS_PROC_LOCK_TRACE) {
-        erts_lcnt_lock_post_x(&(lock->lcnt_trace), file, line);
+        erts_lcnt_lock_info_carrier_t *carrier;
+        Eterm pid;
+
+        carrier = erts_lcnt_create_lock_info_carrier(ERTS_LCNT_PROCLOCK_COUNT);
+        pid = (proc->common.id != ERTS_INVALID_PID) ? proc->common.id : NIL;
+
+        erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN,
+            "proc_main", ERTS_LCNT_LT_PROCLOCK, pid);
+        erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK,
+            "proc_link", ERTS_LCNT_LT_PROCLOCK, pid);
+        erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ,
+            "proc_msgq", ERTS_LCNT_LT_PROCLOCK, pid);
+        erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM,
+            "proc_btm", ERTS_LCNT_LT_PROCLOCK, pid);
+        erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS,
+            "proc_status",ERTS_LCNT_LT_PROCLOCK, pid);
+        erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE,
+            "proc_trace", ERTS_LCNT_LT_PROCLOCK, pid);
+
+        erts_lcnt_install(&proc->lock.lcnt_carrier, carrier);
+    } else {
+        erts_lcnt_proc_lock_destroy(proc);
     }
 }
 
-void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks) {
-    if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
-    if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_unaquire(&(lock->lcnt_main)); }
-    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock_unaquire(&(lock->lcnt_link)); }
-    if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock_unaquire(&(lock->lcnt_msgq)); }
-    if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock_unaquire(&(lock->lcnt_btm)); }
-    if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock_unaquire(&(lock->lcnt_status)); }
-    if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_lock_unaquire(&(lock->lcnt_trace)); }
-}
-
-void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks) {
-    if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
-    if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_unlock(&(lock->lcnt_main)); }
-    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_unlock(&(lock->lcnt_link)); }
-    if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_unlock(&(lock->lcnt_msgq)); }
-    if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_unlock(&(lock->lcnt_btm)); }
-    if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_unlock(&(lock->lcnt_status)); }
-    if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_unlock(&(lock->lcnt_trace)); }
-}
-void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res) {
-    if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return;
-    if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_trylock(&(lock->lcnt_main), res); }
-    if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_trylock(&(lock->lcnt_link), res); }
-    if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_trylock(&(lock->lcnt_msgq), res); }
-    if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_trylock(&(lock->lcnt_btm), res); }
-    if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_trylock(&(lock->lcnt_status), res); }
-    if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_trylock(&(lock->lcnt_trace), res); }
-} /* reversed logic */
 #endif /* ERTS_ENABLE_LOCK_COUNT */
 
 
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 6e704b185d..b4a5ef3f28 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -78,13 +78,19 @@ typedef struct erts_proc_lock_t_ {
     ErtsProcLocks flags;
 #endif
     erts_tse_t *queue[ERTS_PROC_LOCK_MAX_BIT+1];
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_t lcnt_main;
-    erts_lcnt_lock_t lcnt_link;
-    erts_lcnt_lock_t lcnt_msgq;
-    erts_lcnt_lock_t lcnt_btm;
-    erts_lcnt_lock_t lcnt_status;
-    erts_lcnt_lock_t lcnt_trace;
+#if defined(ERTS_ENABLE_LOCK_COUNT) && !ERTS_PROC_LOCK_RAW_MUTEX_IMPL
+    /* Each erts_mtx_t has its own lock counter ^ */
+
+    #define ERTS_LCNT_PROCLOCK_IDX_MAIN 0
+    #define ERTS_LCNT_PROCLOCK_IDX_LINK 1
+    #define ERTS_LCNT_PROCLOCK_IDX_MSGQ 2
+    #define ERTS_LCNT_PROCLOCK_IDX_BTM 3
+    #define ERTS_LCNT_PROCLOCK_IDX_STATUS 4
+    #define ERTS_LCNT_PROCLOCK_IDX_TRACE 5
+
+    #define ERTS_LCNT_PROCLOCK_COUNT 6
+
+    erts_lcnt_ref_t lcnt_carrier;
 #endif
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
     erts_mtx_t main;
@@ -245,14 +251,169 @@ typedef struct erts_proc_lock_t_ {
 
 void erts_lcnt_proc_lock_init(Process *p);
 void erts_lcnt_proc_lock_destroy(Process *p);
+
+ERTS_GLB_INLINE
 void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks);
+ERTS_GLB_INLINE
 void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, char *file, unsigned int line);
-void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks);
+ERTS_GLB_INLINE
+void erts_lcnt_proc_lock_unacquire(erts_proc_lock_t *lock, ErtsProcLocks locks);
+ERTS_GLB_INLINE
 void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks);
+ERTS_GLB_INLINE
 void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res);
 
 void erts_lcnt_enable_proc_lock_count(int enable);
 
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE
+void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) {
+    erts_lcnt_lock_info_carrier_t *carrier;
+    int handle;
+
+    if(erts_lcnt_open_ref(&lock->lcnt_carrier, &handle, &carrier)) {
+        if (locks & ERTS_PROC_LOCK_MAIN) {
+            erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN);
+        }
+        if (locks & ERTS_PROC_LOCK_LINK) {
+            erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK);
+        }
+        if (locks & ERTS_PROC_LOCK_MSGQ) {
+            erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ);
+        }
+        if (locks & ERTS_PROC_LOCK_BTM) {
+            erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM);
+        }
+        if (locks & ERTS_PROC_LOCK_STATUS) {
+            erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS);
+        }
+        if (locks & ERTS_PROC_LOCK_TRACE) {
+            erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE);
+        }
+
+        erts_lcnt_close_ref(handle, carrier);
+    }
+}
+
+ERTS_GLB_INLINE
+void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks,
+                                char *file, unsigned int line) {
+    erts_lcnt_lock_info_carrier_t *carrier;
+    int handle;
+
+    if(erts_lcnt_open_ref(&lock->lcnt_carrier, &handle, &carrier)) {
+        if (locks & ERTS_PROC_LOCK_MAIN) {
+            erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN, file, line);
+        }
+        if (locks & ERTS_PROC_LOCK_LINK) {
+            erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK, file, line);
+        }
+        if (locks & ERTS_PROC_LOCK_MSGQ) {
+            erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ, file, line);
+        }
+        if (locks & ERTS_PROC_LOCK_BTM) {
+            erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM, file, line);
+        }
+        if (locks & ERTS_PROC_LOCK_STATUS) {
+            erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS, file, line);
+        }
+        if (locks & ERTS_PROC_LOCK_TRACE) {
+            erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE, file, line);
+        }
+
+        erts_lcnt_close_ref(handle, carrier);
+    }
+}
+
+ERTS_GLB_INLINE
+void erts_lcnt_proc_lock_unacquire(erts_proc_lock_t *lock, ErtsProcLocks locks) {
+    erts_lcnt_lock_info_carrier_t *carrier;
+    int handle;
+
+    if(erts_lcnt_open_ref(&lock->lcnt_carrier, &handle, &carrier)) {
+        if (locks & ERTS_PROC_LOCK_MAIN) {
+            erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN);
+        }
+        if (locks & ERTS_PROC_LOCK_LINK) {
+            erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK);
+        }
+        if (locks & ERTS_PROC_LOCK_MSGQ) {
+            erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ);
+        }
+        if (locks & ERTS_PROC_LOCK_BTM) {
+            erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM);
+        }
+        if (locks & ERTS_PROC_LOCK_STATUS) {
+            erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS);
+        }
+        if (locks & ERTS_PROC_LOCK_TRACE) {
+            erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE);
+        }
+
+        erts_lcnt_close_ref(handle, carrier);
+    }
+}
+
+ERTS_GLB_INLINE
+void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks) {
+    erts_lcnt_lock_info_carrier_t *carrier;
+    int handle;
+
+    if(erts_lcnt_open_ref(&lock->lcnt_carrier, &handle, &carrier)) {
+        if (locks & ERTS_PROC_LOCK_MAIN) {
+            erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN);
+        }
+        if (locks & ERTS_PROC_LOCK_LINK) {
+            erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK);
+        }
+        if (locks & ERTS_PROC_LOCK_MSGQ) {
+            erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ);
+        }
+        if (locks & ERTS_PROC_LOCK_BTM) {
+            erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM);
+        }
+        if (locks & ERTS_PROC_LOCK_STATUS) {
+            erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS);
+        }
+        if (locks & ERTS_PROC_LOCK_TRACE) {
+            erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE);
+        }
+
+        erts_lcnt_close_ref(handle, carrier);
+    }
+}
+
+ERTS_GLB_INLINE
+void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res) {
+    erts_lcnt_lock_info_carrier_t *carrier;
+    int handle;
+
+    if(erts_lcnt_open_ref(&lock->lcnt_carrier, &handle, &carrier)) {
+        if (locks & ERTS_PROC_LOCK_MAIN) {
+            erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN, res);
+        }
+        if (locks & ERTS_PROC_LOCK_LINK) {
+            erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK, res);
+        }
+        if (locks & ERTS_PROC_LOCK_MSGQ) {
+            erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ, res);
+        }
+        if (locks & ERTS_PROC_LOCK_BTM) {
+            erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM, res);
+        }
+        if (locks & ERTS_PROC_LOCK_STATUS) {
+            erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS, res);
+        }
+        if (locks & ERTS_PROC_LOCK_TRACE) {
+            erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE, res);
+        }
+
+        erts_lcnt_close_ref(handle, carrier);
+    }
+} /* reversed logic */
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 #endif /* ERTS_ENABLE_LOCK_COUNT*/
 
 
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 700ed90def..2a9f276e02 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -321,13 +321,23 @@ tmp_thr_prgr_data(ErtsSchedulerData *esdp)
     ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(esdp);
 
     if (!tpd) {
-	/*
-	 * We only allocate the part up to the wakeup_request field
-	 * which is the first field only used by registered threads
-	 */
-	tpd = erts_alloc(ERTS_ALC_T_T_THR_PRGR_DATA,
-			 offsetof(ErtsThrPrgrData, wakeup_request));
-	init_tmp_thr_prgr_data(tpd);
+        /*
+         * We only allocate the part up to the wakeup_request field which is
+         * the first field only used by registered threads
+         */
+        size_t alloc_size = offsetof(ErtsThrPrgrData, wakeup_request);
+
+        /* We may land here as a result of unmanaged_delay being called from
+         * the lock counting module, which in turn might be called from within
+         * the allocator, so we use plain malloc to avoid deadlocks. */
+        tpd =
+#ifdef ERTS_ENABLE_LOCK_COUNT
+            malloc(alloc_size);
+#else
+            erts_alloc(ERTS_ALC_T_T_THR_PRGR_DATA, alloc_size);
+#endif
+
+        init_tmp_thr_prgr_data(tpd);
     }
 
     return tpd;
@@ -337,8 +347,13 @@ static ERTS_INLINE void
 return_tmp_thr_prgr_data(ErtsThrPrgrData *tpd)
 {
     if (tpd->is_temporary) {
-	erts_tsd_set(erts_thr_prgr_data_key__, NULL);
-	erts_free(ERTS_ALC_T_T_THR_PRGR_DATA, tpd);
+        erts_tsd_set(erts_thr_prgr_data_key__, NULL);
+
+#ifdef ERTS_ENABLE_LOCK_COUNT
+        free(tpd);
+#else
+        erts_free(ERTS_ALC_T_T_THR_PRGR_DATA, tpd);
+#endif
     }
 }
 
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 28ff5d3a42..f7b53f5ef3 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -307,7 +307,7 @@ typedef struct {
     erts_lc_lock_t lc;
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_t lcnt;
+    erts_lcnt_ref_t lcnt;
 #endif
 
 } erts_mtx_t;
@@ -320,7 +320,7 @@ typedef struct {
     erts_lc_lock_t lc;
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_t lcnt;
+    erts_lcnt_ref_t lcnt;
 #endif
 } erts_rwmtx_t;
 
@@ -365,7 +365,7 @@ typedef struct {
     erts_lc_lock_t lc;
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_t lcnt;
+    erts_lcnt_ref_t lcnt;
 #endif
 } erts_spinlock_t;
 
@@ -376,7 +376,7 @@ typedef struct {
     erts_lc_lock_t lc;
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_t lcnt;
+    erts_lcnt_ref_t lcnt;
 #endif
 } erts_rwlock_t;
 
@@ -2169,7 +2169,8 @@ erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra)
     erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
+    erts_lcnt_init_ref(&mtx->lcnt);
+    erts_lcnt_install_new_lock_info_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
 #endif
 #endif
 }
@@ -2185,7 +2186,10 @@ erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt)
     erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra);
+    erts_lcnt_init_ref(&mtx->lcnt);
+    if(!(opt & ERTS_LCNT_LT_DISABLE)) { // Incorrect, but preserves the old interface.
+        erts_lcnt_install_new_lock_info_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra);
+    }
 #endif
 #endif
 }
@@ -2202,7 +2206,10 @@ erts_mtx_init_locked_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt)
     erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra);
+    erts_lcnt_init_ref(&mtx->lcnt);
+    if(!(opt & ERTS_LCNT_LT_DISABLE)) { // Incorrect, but preserves the old interface.
+        erts_lcnt_install_new_lock_info_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
+    }
 #endif
     ethr_mutex_lock(&mtx->mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -2225,7 +2232,8 @@ erts_mtx_init(erts_mtx_t *mtx, char *name)
     erts_lc_init_lock(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX);
+    erts_lcnt_init_ref(&mtx->lcnt);
+    erts_lcnt_install_new_lock_info(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX);
 #endif
 #endif
 }
@@ -2241,7 +2249,8 @@ erts_mtx_init_locked(erts_mtx_t *mtx, char *name)
     erts_lc_init_lock(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX);
+    erts_lcnt_init_ref(&mtx->lcnt);
+    erts_lcnt_install_new_lock_info(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX);
 #endif
     ethr_mutex_lock(&mtx->mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -2262,7 +2271,7 @@ erts_mtx_destroy(erts_mtx_t *mtx)
     erts_lc_destroy_lock(&mtx->lc);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_destroy_lock(&mtx->lcnt);
+    erts_lcnt_uninstall(&mtx->lcnt);
 #endif
     res = ethr_mutex_destroy(&mtx->mtx);
     if (res != 0) {
@@ -2481,10 +2490,11 @@ erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx,
     erts_lc_init_lock_x(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_init_ref(&rwmtx->lcnt);
     if (name && name[0] == '\0')
-      erts_lcnt_init_lock_x(&rwmtx->lcnt, NULL, ERTS_LCNT_LT_RWMUTEX, extra);
+        erts_lcnt_install_new_lock_info_x(&rwmtx->lcnt, NULL, ERTS_LCNT_LT_RWMUTEX, extra);
     else
-      erts_lcnt_init_lock_x(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX, extra);
+        erts_lcnt_install_new_lock_info_x(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX, extra);
 #endif
 #endif
 }
@@ -2510,7 +2520,8 @@ erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
     erts_lc_init_lock(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_lock(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX);
+    erts_lcnt_init_ref(&rwmtx->lcnt);
+    erts_lcnt_install_new_lock_info(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX);
 #endif
 #endif
 }
@@ -2530,7 +2541,7 @@ erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
     erts_lc_destroy_lock(&rwmtx->lc);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_destroy_lock(&rwmtx->lcnt);
+    erts_lcnt_uninstall(&rwmtx->lcnt);
 #endif
     res = ethr_rwmutex_destroy(&rwmtx->rwmtx);
     if (res != 0) {
@@ -3085,7 +3096,8 @@ erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra)
     erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK, extra);
+    erts_lcnt_init_ref(&lock->lcnt);
+    erts_lcnt_install_new_lock_info_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK, extra);
 #endif
 #else
     (void)lock;
@@ -3104,7 +3116,8 @@ erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra,
     erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK|opt, extra);
+    erts_lcnt_init_ref(&lock->lcnt);
+    erts_lcnt_install_new_lock_info_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK | opt, extra);
 #endif
 #else
     (void)lock;
@@ -3123,7 +3136,8 @@ erts_spinlock_init(erts_spinlock_t *lock, char *name)
     erts_lc_init_lock(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_lock(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK);
+    erts_lcnt_init_ref(&lock->lcnt);
+    erts_lcnt_install_new_lock_info(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK);
 #endif
 #else
     (void)lock;
@@ -3139,7 +3153,7 @@ erts_spinlock_destroy(erts_spinlock_t *lock)
     erts_lc_destroy_lock(&lock->lc);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_destroy_lock(&lock->lcnt);
+    erts_lcnt_uninstall(&lock->lcnt);
 #endif
     res = ethr_spinlock_destroy(&lock->slck);
     if (res != 0) {
@@ -3228,7 +3242,8 @@ erts_rwlock_init_x(erts_rwlock_t *lock, char *name, Eterm extra)
     erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK, extra);
+    erts_lcnt_init_ref(&lock->lcnt);
+    erts_lcnt_install_new_lock_info_x(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK, extra);
 #endif
 #else
     (void)lock;
@@ -3246,7 +3261,8 @@ erts_rwlock_init(erts_rwlock_t *lock, char *name)
     erts_lc_init_lock(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_lock(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK);
+    erts_lcnt_init_ref(&lock->lcnt);
+    erts_lcnt_install_new_lock_info(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK);
 #endif
 #else
     (void)lock;
@@ -3262,7 +3278,7 @@ erts_rwlock_destroy(erts_rwlock_t *lock)
     erts_lc_destroy_lock(&lock->lc);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_destroy_lock(&lock->lcnt);
+    erts_lcnt_uninstall(&lock->lcnt);
 #endif
     res = ethr_rwlock_destroy(&lock->rwlck);
     if (res != 0) {
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index d25e53ada0..11cb2dfddf 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -3461,39 +3461,34 @@ void erts_init_io(int port_tab_size,
 static ERTS_INLINE void lcnt_enable_drv_lock_count(erts_driver_t *dp, int enable)
 {
     if (dp->lock) {
-	if (enable)
-	    erts_lcnt_init_lock_x(&dp->lock->lcnt,
-				  "driver_lock",
-				  ERTS_LCNT_LT_MUTEX,
-				  erts_atom_put((byte*)dp->name,
-						sys_strlen(dp->name),
-						ERTS_ATOM_ENC_LATIN1,
-						1));
-
-	else
-	    erts_lcnt_destroy_lock(&dp->lock->lcnt);
-
-    }
+	if (enable) {
+			Eterm name_as_atom = erts_atom_put((byte*)dp->name, sys_strlen(dp->name),
+											   ERTS_ATOM_ENC_LATIN1, 1);
+			erts_lcnt_install_new_lock_info_x(&dp->lock->lcnt,
+				"driver_lock", ERTS_LCNT_LT_MUTEX, name_as_atom);
+		} else {
+			erts_lcnt_uninstall(&dp->lock->lcnt);
+		}
+	}
 }
 
 static ERTS_INLINE void lcnt_enable_port_lock_count(Port *prt, int enable)
 {
     erts_aint32_t state = erts_atomic32_read_nob(&prt->state);
-    if (!enable) {
-	erts_lcnt_destroy_lock(&prt->sched.mtx.lcnt);
-	if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK)
-	    erts_lcnt_destroy_lock(&prt->lock->lcnt);
-    }
-    else {
-	erts_lcnt_init_lock_x(&prt->sched.mtx.lcnt,
-			      "port_sched_lock",
-			      ERTS_LCNT_LT_MUTEX,
-			      prt->common.id);
-	if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK)
-	    erts_lcnt_init_lock_x(&prt->lock->lcnt,
-				  "port_lock",
-				  ERTS_LCNT_LT_MUTEX,
-				  prt->common.id);
+
+    if(enable) {
+		erts_lcnt_install_new_lock_info_x(&prt->sched.mtx.lcnt,
+			"port_sched_lock", ERTS_LCNT_LT_MUTEX,prt->common.id);
+
+		if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) {
+			erts_lcnt_install_new_lock_info_x(&prt->lock->lcnt,
+				"port_lock", ERTS_LCNT_LT_MUTEX, prt->common.id);
+		}
+    } else {
+		erts_lcnt_uninstall(&prt->sched.mtx.lcnt);
+		if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) {
+			erts_lcnt_uninstall(&prt->lock->lcnt);
+		}
     }
 }
 
@@ -3701,7 +3696,7 @@ deliver_result(Port *prt, Eterm sender, Eterm pid, Eterm res)
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
     ASSERT(!prt || prt->common.id == sender);
-#ifdef ERTS_SMP
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
     ASSERT(!prt || erts_lc_is_port_locked(prt));
 #endif
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 457cada745..0fb25c2082 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -42,6 +42,7 @@
 #include "dist.h"
 #include "erl_printf.h"
 #include "erl_threads.h"
+#include "erl_lock_count.h"
 #include "erl_smp.h"
 #include "erl_time.h"
 #include "erl_thr_progress.h"
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 0079912b10..e86258b5e0 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -438,14 +438,18 @@ erts_sys_pre_init(void)
     /* After creation in parent */
     eid.thread_create_parent_func = thr_create_cleanup,
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_pre_thr_init();
+#endif
+
     erts_thr_init(&eid);
 
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init();
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_post_thr_init();
 #endif
 
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init();
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    erts_lc_init();
 #endif
 
 #endif /* USE_THREADS */
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index 28019e306c..15c59109b1 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -3195,15 +3195,22 @@ erts_sys_pre_init(void)
     /* After creation in parent */
     eid.thread_create_parent_func = thr_create_cleanup;
 
-    erts_thr_init(&eid);
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init();
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_pre_thr_init();
 #endif
+
+    erts_thr_init(&eid);
+
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init();
+    erts_lcnt_post_thr_init();
 #endif
+
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    erts_lc_init();
 #endif
 
+#endif /* USE_THREADS */
+
     erts_init_sys_time_sup();
 
     erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0);
-- 
cgit v1.2.3


From 81b62880e3768dcd161f107f766f3e6e89d59446 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 19 May 2017 09:44:11 +0200
Subject: erts: Add undocumented option to do default SIGTERM

This is a stopgap measure before the release of OTP-20 where
that makes it possible for the user to make the vm not do
anything with SIGTERM and instead rely on the OS default.

To enable this behaviour the user should set the environment
variable ERL_ZZ_SIGTERM_KILL="true".
---
 erts/emulator/sys/unix/sys.c | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 3010af44be..de8481b206 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -838,6 +838,11 @@ void sys_init_suspend_handler(void)
 void
 erts_sys_unix_later_init(void)
 {
+    char env[4];
+    size_t envsz = sizeof(env);
+    if (erts_sys_getenv_raw("ERL_ZZ_SIGTERM_KILL", env, &envsz) == 0)
+        if (envsz == 4 && sys_strncmp("true",env,4) == 0)
+            return;
     sys_signal(SIGTERM, request_stop);
 }
 
-- 
cgit v1.2.3


From 32ea8ba368c455afba07afd85bed6fb57879f56d Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 18 May 2017 20:42:04 +0200
Subject: Make statistics/1 aware of dirty run-queues and tasks

---
 erts/emulator/beam/atom.names           |   4 +
 erts/emulator/beam/erl_bif_info.c       |  34 ++++--
 erts/emulator/beam/erl_process.c        | 188 +++++++++++++++++++++++---------
 erts/emulator/beam/erl_process.h        |  23 ++--
 erts/emulator/test/statistics_SUITE.erl |  25 ++++-
 5 files changed, 202 insertions(+), 72 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 477a7676d6..a44d23b181 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -76,6 +76,7 @@ atom ac
 atom accessor
 atom active
 atom active_tasks
+atom active_tasks_all
 atom alive
 atom all
 atom all_but_first
@@ -567,6 +568,7 @@ atom return_to
 atom return_trace
 atom run_queue
 atom run_queue_lengths
+atom run_queue_lengths_all
 atom runnable
 atom runnable_ports
 atom runnable_procs
@@ -656,8 +658,10 @@ atom Times='*'
 atom timestamp
 atom total
 atom total_active_tasks
+atom total_active_tasks_all
 atom total_heap_size
 atom total_run_queue_lengths
+atom total_run_queue_lengths_all
 atom tpkt
 atom trace trace_ts traced
 atom trace_control_word
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 5fc70dfc02..e2773475b0 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3450,9 +3450,15 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
 	if (is_non_value(res))
 	    BIF_RET(am_undefined);
 	BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res);
-    } else if (BIF_ARG_1 == am_total_active_tasks
-	       || BIF_ARG_1 == am_total_run_queue_lengths) {
-	Uint no = erts_run_queues_len(NULL, 0, BIF_ARG_1 == am_total_active_tasks);
+    } else if ((BIF_ARG_1 == am_total_active_tasks)
+	       | (BIF_ARG_1 == am_total_run_queue_lengths)
+               | (BIF_ARG_1 == am_total_active_tasks_all)
+	       | (BIF_ARG_1 == am_total_run_queue_lengths_all)) {
+	Uint no = erts_run_queues_len(NULL, 0,
+                                      ((BIF_ARG_1 == am_total_active_tasks)
+                                       | (BIF_ARG_1 == am_total_active_tasks_all)),
+                                      ((BIF_ARG_1 == am_total_active_tasks_all)
+                                       | (BIF_ARG_1 == am_total_run_queue_lengths_all)));
 	if (IS_USMALL(0, no))
 	    res = make_small(no);
 	else {
@@ -3460,13 +3466,21 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
 	    res = uint_to_big(no, hp);
 	}
 	BIF_RET(res);
-    } else if (BIF_ARG_1 == am_active_tasks
-	       || BIF_ARG_1 == am_run_queue_lengths) {
+    } else if ((BIF_ARG_1 == am_active_tasks)
+           | (BIF_ARG_1 == am_run_queue_lengths)
+           | (BIF_ARG_1 == am_active_tasks_all)
+           | (BIF_ARG_1 == am_run_queue_lengths_all)) {
 	Eterm res, *hp, **hpp;
 	Uint sz, *szp;
-	int no_qs = erts_no_run_queues;
+        int incl_dirty_io = ((BIF_ARG_1 == am_active_tasks_all)
+                             | (BIF_ARG_1 == am_run_queue_lengths_all));
+        int no_qs = (erts_no_run_queues + ERTS_NUM_DIRTY_CPU_RUNQS +
+                     (incl_dirty_io ? ERTS_NUM_DIRTY_IO_RUNQS : 0));
 	Uint *qszs = erts_alloc(ERTS_ALC_T_TMP,sizeof(Uint)*no_qs*2);
-	(void) erts_run_queues_len(qszs, 0, BIF_ARG_1 == am_active_tasks);
+        (void) erts_run_queues_len(qszs, 0,
+                                   ((BIF_ARG_1 == am_active_tasks)
+                                    | (BIF_ARG_1 == am_active_tasks_all)),
+                                   incl_dirty_io);
 	sz = 0;
 	szp = &sz;
 	hpp = NULL;
@@ -3539,7 +3553,7 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
 	res = TUPLE2(hp, b1, b2);
 	BIF_RET(res);
     } else if (BIF_ARG_1 ==  am_run_queue) {
-	res = erts_run_queues_len(NULL, 1, 0);
+	res = erts_run_queues_len(NULL, 1, 0, 0);
 	BIF_RET(make_small(res));
     } else if (BIF_ARG_1 == am_wall_clock) {
 	UWord w1, w2;
@@ -3557,9 +3571,9 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
     else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) {
 	Eterm res, *hp, **hpp;
 	Uint sz, *szp;
-	int no_qs = erts_no_run_queues;
+	int no_qs = erts_no_run_queues + ERTS_NUM_DIRTY_RUNQS;
 	Uint *qszs = erts_alloc(ERTS_ALC_T_TMP,sizeof(Uint)*no_qs*2);
-	(void) erts_run_queues_len(qszs, 0, 0);
+	(void) erts_run_queues_len(qszs, 0, 0, 1);
 	sz = 0;
 	szp = &sz;
 	hpp = NULL;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index a56ef4aedb..6f96833b21 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -405,13 +405,59 @@ static erts_atomic_t runq_supervisor_sleeping;
 ErtsSchedulerData *erts_scheduler_data;
 #endif
 
-ErtsAlignedRunQueue *erts_aligned_run_queues;
-Uint erts_no_run_queues;
+ErtsAlignedRunQueue * ERTS_WRITE_UNLIKELY(erts_aligned_run_queues);
+Uint ERTS_WRITE_UNLIKELY(erts_no_run_queues);
 
-ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
 #ifdef ERTS_DIRTY_SCHEDULERS
-ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data;
-ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
+
+struct {
+    union {
+        erts_smp_atomic32_t active;
+        char align__[ERTS_CACHE_LINE_SIZE];
+    } cpu;
+    union {
+        erts_smp_atomic32_t active;
+        char align__[ERTS_CACHE_LINE_SIZE];
+    } io;
+} dirty_count erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+
+#endif
+
+static ERTS_INLINE void
+dirty_active(ErtsSchedulerData *esdp, erts_aint32_t add)
+{
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_aint32_t val;
+    erts_smp_atomic32_t *ap;
+    switch (esdp->type) {
+    case ERTS_SCHED_DIRTY_CPU:
+        ap = &dirty_count.cpu.active;
+        break;
+    case ERTS_SCHED_DIRTY_IO:
+        ap = &dirty_count.io.active;
+        break;
+    default:
+        ap = NULL;
+        ERTS_INTERNAL_ERROR("Not a dirty scheduler");
+        break;
+    }
+
+    /*
+     * All updates done under run-queue lock, so
+     * no inc or dec needed...
+     */
+    ERTS_SMP_ASSERT(erts_smp_lc_runq_is_locked(esdp->run_queue));
+
+    val = erts_smp_atomic32_read_nob(ap);
+    val += add;
+    erts_smp_atomic32_set_nob(ap, val);
+#endif
+}
+
+ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_scheduler_data);
+#ifdef ERTS_DIRTY_SCHEDULERS
+ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_cpu_scheduler_data);
+ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_io_scheduler_data);
 typedef union {
     Process dsp;
     char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(Process))];
@@ -539,22 +585,28 @@ do {									\
     }									\
 } while (0)
 
-#define ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, DO, DOX)			\
+#define ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, NRQS, DO, DOX)		\
 do {									\
     ErtsRunQueue *RQVAR;						\
+    int nrqs = (NRQS);                                                  \
     int ix__;								\
-    for (ix__ = 0; ix__ < erts_no_run_queues; ix__++) {			\
+    for (ix__ = 0; ix__ < nrqs; ix__++) {                               \
 	RQVAR = ERTS_RUNQ_IX(ix__);					\
 	erts_smp_runq_lock(RQVAR);					\
 	{ DO; }								\
     }									\
     { DOX; }								\
-    for (ix__ = 0; ix__ < erts_no_run_queues; ix__++)			\
+    for (ix__ = 0; ix__ < nrqs; ix__++)                                 \
 	erts_smp_runq_unlock(ERTS_RUNQ_IX(ix__));			\
 } while (0)
 
-#define ERTS_ATOMIC_FOREACH_RUNQ(RQVAR, DO) \
-  ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, DO, )
+#define ERTS_ATOMIC_FOREACH_RUNQ(RQVAR, DO)                             \
+  ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, erts_no_run_queues + ERTS_NUM_DIRTY_RUNQS, DO, )
+
+#define ERTS_ATOMIC_FOREACH_NORMAL_RUNQ(RQVAR, DO)                      \
+    ERTS_ATOMIC_FOREACH_RUNQ_X(RQVAR, erts_no_run_queues, DO, )
+
+
 /*
  * Local functions.
  */
@@ -2977,7 +3029,7 @@ erts_active_schedulers(void)
 {
     Uint as = erts_no_schedulers;
 
-    ERTS_ATOMIC_FOREACH_RUNQ(rq, as -= abs(rq->waiting));
+    ERTS_ATOMIC_FOREACH_NORMAL_RUNQ(rq, as -= abs(rq->waiting));
 
     return as;
 }
@@ -3396,6 +3448,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    rq->sleepers.list->prev = ssi;
 	rq->sleepers.list = ssi;
 	erts_smp_spin_unlock(&rq->sleepers.lock);
+        dirty_active(esdp, -1);
     }
 #endif
 
@@ -3737,6 +3790,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	sched_active_sys(esdp->no, rq);
     }
 
+    if (ERTS_SCHEDULER_IS_DIRTY(esdp))
+        dirty_active(esdp, 1);
+
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 }
 
@@ -6136,7 +6192,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 #endif
 		     )
 {
-    int ix, n, no_ssi;
+    int ix, n, no_ssi, tot_rqs;
     char *daww_ptr;
     size_t daww_sz;
     size_t size_runqs;
@@ -6169,26 +6225,19 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     /* Create and initialize run queues */
 
     n = no_schedulers;
-    size_runqs = sizeof(ErtsAlignedRunQueue) * (n + ERTS_NUM_DIRTY_RUNQS);
+    tot_rqs = (n + ERTS_NUM_DIRTY_RUNQS);
+    size_runqs = sizeof(ErtsAlignedRunQueue) * tot_rqs;
     erts_aligned_run_queues =
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, size_runqs);
 #ifdef ERTS_SMP
-#ifdef ERTS_DIRTY_SCHEDULERS
-    erts_aligned_run_queues += ERTS_NUM_DIRTY_RUNQS;
-#endif
     erts_smp_atomic32_init_nob(&no_empty_run_queues, 0);
 #endif
 
     erts_no_run_queues = n;
 
-    for (ix = -(ERTS_NUM_DIRTY_RUNQS); ix < n; ix++) {
+    for (ix = 0; ix < tot_rqs; ix++) {
 	int pix, rix;
-#ifdef ERTS_DIRTY_SCHEDULERS
-	ErtsRunQueue *rq = ERTS_RUNQ_IX_IS_DIRTY(ix) ?
-	    ERTS_DIRTY_RUNQ_IX(ix) : ERTS_RUNQ_IX(ix);
-#else
 	ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
-#endif
 
 	rq->ix = ix;
 
@@ -6461,6 +6510,11 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 			     ERTS_SCHED_DIRTY_IO,
 			     no_dirty_io_schedulers);
 
+    erts_smp_atomic32_init_nob(&dirty_count.cpu.active,
+                               (erts_aint32_t) no_dirty_cpu_schedulers);
+    erts_smp_atomic32_init_nob(&dirty_count.io.active,
+                               (erts_aint32_t) no_dirty_io_schedulers);
+
 #endif
 
     if (set_schdlr_sspnd_change_flags)
@@ -7835,6 +7889,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 #endif
 
     if (sched_type != ERTS_SCHED_NORMAL) {
+        dirty_active(esdp, -1);
 	erts_smp_runq_unlock(esdp->run_queue);
         dirty_sched_wall_time_change(esdp, 0);
     }
@@ -8143,7 +8198,9 @@ suspend_scheduler(ErtsSchedulerData *esdp)
     erts_smp_runq_lock(esdp->run_queue);
     non_empty_runq(esdp->run_queue);
 
-    if (sched_type == ERTS_SCHED_NORMAL) {
+    if (sched_type != ERTS_SCHED_NORMAL)
+        dirty_active(esdp, 1);
+    else {
 	schedule_bound_processes(esdp->run_queue, &sbp);
 
 	erts_sched_check_cpu_bind_post_suspend(esdp);
@@ -9803,38 +9860,69 @@ erts_internal_is_process_executing_dirty_1(BIF_ALIST_1)
     BIF_RET(am_false);
 }
 
+static ERTS_INLINE void
+run_queues_len_aux(ErtsRunQueue *rq, Uint *tot_len, Uint *qlen, int *ip, int incl_active_sched, int locked)
+{
+    Sint rq_len;
+
+    if (locked)
+        rq_len = (Sint) erts_smp_atomic32_read_dirty(&rq->len);
+    else
+        rq_len = (Sint) erts_smp_atomic32_read_nob(&rq->len);
+    ASSERT(rq_len >= 0);
+
+    if (incl_active_sched) {
+#ifdef ERTS_DIRTY_SCHEDULERS
+        if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) {
+            erts_aint32_t dcnt;
+            if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(rq)) {
+                dcnt = erts_smp_atomic32_read_nob(&dirty_count.cpu.active);
+                ASSERT(0 <= dcnt && dcnt <= erts_no_dirty_cpu_schedulers);
+            }
+            else {
+                ASSERT(ERTS_RUNQ_IS_DIRTY_IO_RUNQ(rq));
+                dcnt = erts_smp_atomic32_read_nob(&dirty_count.io.active);
+                ASSERT(0 <= dcnt && dcnt <= erts_no_dirty_io_schedulers);
+            }
+            rq_len += (Sint) dcnt;
+        }
+        else
+#endif
+        {
+            if (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)
+                rq_len++;
+        }
+    }
+    if (qlen)
+        qlen[(*ip)++] = rq_len;
+    *tot_len += (Uint) rq_len;
+}
 
 Uint
-erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched)
+erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched,
+                    int incl_dirty_io)
 {
-    int i = 0;
+    int i = 0, j = 0;
     Uint len = 0;
-    if (atomic_queues_read)
-	ERTS_ATOMIC_FOREACH_RUNQ(rq,
-	 {
-	     Sint rq_len = (Sint) erts_smp_atomic32_read_dirty(&rq->len);
-	     ASSERT(rq_len >= 0);
-	     if (incl_active_sched
-		 && (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)) {
-		 rq_len++;
-	     }
-	     if (qlen)
-		 qlen[i++] = rq_len;
-	     len += (Uint) rq_len;
-	 }
-	    );
+    int no_rqs = erts_no_run_queues;
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (incl_dirty_io)
+        no_rqs += ERTS_NUM_DIRTY_RUNQS;
+    else
+        no_rqs += ERTS_NUM_DIRTY_CPU_RUNQS;
+#endif
+
+    if (atomic_queues_read) {
+        ERTS_ATOMIC_FOREACH_RUNQ_X(rq, no_rqs,
+                                   run_queues_len_aux(rq, &len, qlen, &j,
+                                                      incl_active_sched, 1),
+                                   /* Nothing... */);
+    }
     else {
-	for (i = 0; i < erts_no_run_queues; i++) {
+	for (i = 0; i < no_rqs; i++) {
 	    ErtsRunQueue *rq = ERTS_RUNQ_IX(i);
-	    Sint rq_len = (Sint) erts_smp_atomic32_read_nob(&rq->len);
-	    ASSERT(rq_len >= 0);
-	     if (incl_active_sched
-		 && (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)) {
-		 rq_len++;
-	     }
-	    if (qlen)
-		qlen[i] = rq_len;
-	    len += (Uint) rq_len;
+            run_queues_len_aux(rq, &len, qlen, &j, incl_active_sched, 0);
 	}
 
     }
@@ -12136,6 +12224,8 @@ erts_get_total_reductions(Uint *redsp, Uint *diffp)
     Uint reds = 0;
     ERTS_ATOMIC_FOREACH_RUNQ_X(rq,
 
+                               erts_no_run_queues + ERTS_NUM_DIRTY_RUNQS,
+
 			       reds += rq->procs.reductions,
 
 			       if (redsp) *redsp = reds;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 6113c8aa05..aa88ff26e8 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1541,24 +1541,29 @@ extern int erts_system_profile_ts_type;
     } while (0)
 
 #if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
-#define ERTS_NUM_DIRTY_RUNQS 2
+#define ERTS_NUM_DIRTY_CPU_RUNQS 1
+#define ERTS_NUM_DIRTY_IO_RUNQS 1
 #else
-#define ERTS_NUM_DIRTY_RUNQS 0
+#define ERTS_NUM_DIRTY_CPU_RUNQS 0
+#define ERTS_NUM_DIRTY_IO_RUNQS 0
 #endif
 
+#define ERTS_NUM_DIRTY_RUNQS (ERTS_NUM_DIRTY_CPU_RUNQS+ERTS_NUM_DIRTY_IO_RUNQS)
+
 #define ERTS_RUNQ_IX(IX)						\
-  (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues),			\
+  (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues+ERTS_NUM_DIRTY_RUNQS), \
    &erts_aligned_run_queues[(IX)].runq)
 #ifdef ERTS_DIRTY_SCHEDULERS
 #define ERTS_RUNQ_IX_IS_DIRTY(IX)					\
-  (-(ERTS_NUM_DIRTY_RUNQS) <= (IX) && (IX) < 0)
+  (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues+ERTS_NUM_DIRTY_RUNQS), \
+   (erts_no_run_queues <= (IX)))
 #define ERTS_DIRTY_RUNQ_IX(IX)						\
   (ASSERT(ERTS_RUNQ_IX_IS_DIRTY(IX)),					\
    &erts_aligned_run_queues[(IX)].runq)
-#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[-1].runq)
-#define ERTS_DIRTY_IO_RUNQ  (&erts_aligned_run_queues[-2].runq)
-#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ)->ix == -1)
-#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ)->ix == -2)
+#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[erts_no_run_queues].runq)
+#define ERTS_DIRTY_IO_RUNQ  (&erts_aligned_run_queues[erts_no_run_queues+1].runq)
+#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ) == ERTS_DIRTY_CPU_RUNQ)
+#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ) == ERTS_DIRTY_IO_RUNQ)
 #else
 #define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
 #endif
@@ -1837,7 +1842,7 @@ Uint erts_active_schedulers(void);
 void erts_init_process(int, int, int);
 Eterm erts_process_state2status(erts_aint32_t);
 Eterm erts_process_status(Process *, Eterm);
-Uint erts_run_queues_len(Uint *, int, int);
+Uint erts_run_queues_len(Uint *, int, int, int);
 void erts_add_to_runq(Process *);
 Eterm erts_bound_schedulers_term(Process *c_p);
 Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which);
diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl
index 3057905f4c..65188b6e26 100644
--- a/erts/emulator/test/statistics_SUITE.erl
+++ b/erts/emulator/test/statistics_SUITE.erl
@@ -502,20 +502,37 @@ run_queues_lengths_active_tasks(Config) ->
                          end,
                          lists:seq(1,10)),
 
+                   
+
     TRQLs0 = statistics(total_run_queue_lengths),
+    TRQLAs0 = statistics(total_run_queue_lengths_all),
     TATs0 = statistics(total_active_tasks),
+    TATAs0 = statistics(total_active_tasks_all),
     true = is_integer(TRQLs0),
     true = is_integer(TATs0),
     true = TRQLs0 >= 0,
+    true = TRQLAs0 >= 0,
     true = TATs0 >= 11,
+    true = TATAs0 >= 11,
 
     NoScheds = erlang:system_info(schedulers),
+    {DefRqs,
+     AllRqs} = case erlang:system_info(dirty_cpu_schedulers) of
+                   0 -> {NoScheds, NoScheds};
+                   _ -> {NoScheds+1, NoScheds+2}
+               end,
     RQLs0 = statistics(run_queue_lengths),
+    RQLAs0 = statistics(run_queue_lengths_all),
     ATs0 = statistics(active_tasks),
-    NoScheds = length(RQLs0),
-    NoScheds = length(ATs0),
+    ATAs0 = statistics(active_tasks_all),
+    DefRqs = length(RQLs0),
+    AllRqs = length(RQLAs0),
+    DefRqs = length(ATs0),
+    AllRqs = length(ATAs0),
     true = lists:sum(RQLs0) >= 0,
+    true = lists:sum(RQLAs0) >= 0,
     true = lists:sum(ATs0) >= 11,
+    true = lists:sum(ATAs0) >= 11,
 
     SO = erlang:system_flag(schedulers_online, 1),
 
@@ -531,8 +548,8 @@ run_queues_lengths_active_tasks(Config) ->
 
     RQLs1 = statistics(run_queue_lengths),
     ATs1 = statistics(active_tasks),
-    NoScheds = length(RQLs1),
-    NoScheds = length(ATs1),
+    DefRqs = length(RQLs1),
+    DefRqs = length(ATs1),
     TRQLs2 = lists:sum(RQLs1),
     TATs2 = lists:sum(ATs1),
     true = TRQLs2 >= 10,
-- 
cgit v1.2.3


From a4905863ec2ed082c67330384c182da0ed4b87b8 Mon Sep 17 00:00:00 2001
From: Raimo Niskanen <raimo@erlang.org>
Date: Tue, 9 May 2017 16:39:09 +0200
Subject: Do not zero terminate Linux abstract addresses

---
 erts/emulator/drivers/common/inet_drv.c | 75 ++++++++++++++++++---------------
 1 file changed, 42 insertions(+), 33 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 2de908ffb3..bfebff5706 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -604,18 +604,6 @@ static size_t my_strnlen(const char *s, size_t maxlen)
 
 #endif
 
-#if defined(HAVE_SYS_UN_H)
-
-/* Check that some character in the buffer != '\0' */
-static int is_nonzero(const char *s, size_t n)
-{
-    size_t i;
-    for (i = 0;  i < n;  i++) if (s[i] != '\0') return !0;
-    return 0;
-}
-
-#endif
-
 #ifdef VALGRIND
 #  include <valgrind/memcheck.h>   
 #else
@@ -4025,13 +4013,30 @@ static char* inet_set_address(int family, inet_address* dst,
         int n;
         if (*len == 0) return str_einval;
 	n = *((unsigned char*)(*src)); /* Length field */
-	if ((*len < 1+n) || (sizeof(dst->sal.sun_path) < n+1)) {
+	if (*len < 1+n) return str_einval;
+	if (n +
+#ifdef __linux__
+            /* Make sure the address gets zero terminated
+             * except when the first byte is \0 because then it is
+             * sort of zero terminated although the zero termination
+             * comes before the address...
+             * This fix handles Linux's nonportable
+             * abstract socket address extension.
+             */
+            ((*len) > 1 && (*src)[1] == '\0' ? 0 : 1)
+#else
+            1
+#endif
+            > sizeof(dst->sal.sun_path)) {
 	    return str_einval;
 	}
 	sys_memzero((char*)dst, sizeof(struct sockaddr_un));
 	dst->sal.sun_family = family;
 	sys_memcpy(dst->sal.sun_path, (*src)+1, n);
 	*len = offsetof(struct sockaddr_un, sun_path) + n;
+#ifndef NO_SA_LEN
+        dst->sal.sun_len = *len;
+#endif
 	*src += 1 + n;
 	return NULL;
     }
@@ -4176,15 +4181,16 @@ static int inet_get_address(char* dst, inet_address* src, unsigned int* len)
         if (*len < offsetof(struct sockaddr_un, sun_path)) return -1;
         n = *len - offsetof(struct sockaddr_un, sun_path);
         if (255 < n) return -1;
-	/* Portability fix: Assume that the address is a zero terminated
-	 * string, except when the first byte is \0 i.e the
-	 * string length is 0.  Then use the reported length instead.
-	 * This fix handles Linux's abstract socket address
-	 * nonportable extension.
-	 */
         m = my_strnlen(src->sal.sun_path, n);
-	if ((m == 0) && is_nonzero(src->sal.sun_path, n))
-	    m = n;
+#ifdef __linux__
+	/* Assume that the address is a zero terminated string,
+         * except when the first byte is \0 i.e the string length is 0,
+         * then use the reported length instead.
+	 * This fix handles Linux's nonportable
+         * abstract socket address extension.
+	 */
+	if (m == 0)  m = n;
+#endif
         dst[0] = INET_AF_LOCAL;
         dst[1] = (char) ((unsigned char) m);
         sys_memcpy(dst+2, src->sal.sun_path, m);
@@ -4241,15 +4247,16 @@ inet_address_to_erlang(char *dst, inet_address **src, SOCKLEN_T sz) {
 	if (sz < offsetof(struct sockaddr_un, sun_path)) return -1;
 	n = sz - offsetof(struct sockaddr_un, sun_path);
 	if (255 < n) return -1;
-	/* Portability fix: Assume that the address is a zero terminated
-	 * string, except when the first byte is \0 i.e the
-	 * string length is 0.  Then use the reported length instead.
-	 * This fix handles Linux's abstract socket address
-	 * nonportable extension.
-	 */
         m = my_strnlen((*src)->sal.sun_path, n);
-	if ((m == 0) && is_nonzero((*src)->sal.sun_path, n))
-	    m = n;
+#ifdef __linux__
+	/* Assume that the address is a zero terminated string,
+         * except when the first byte is \0 i.e the string length is 0,
+         * Then use the reported length instead.
+	 * This fix handles Linux's nonportable
+         * abstract socket address extension.
+	 */
+	if (m == 0)  m = n;
+#endif
 	if (dst) {
 	    dst[0] = INET_AF_LOCAL;
 	    dst[1] = (char) ((unsigned char) m);
@@ -8680,6 +8687,7 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf,
 	else {
 	    ptr = &peer;
             sz = sizeof(peer);
+            sys_memzero((char *) &peer, sz);
 	    if (IS_SOCKET_ERROR
 		(sock_peer
 		 (desc->s, (struct sockaddr*)ptr, &sz)))
@@ -10830,10 +10838,11 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event)
 
 #ifndef SO_ERROR
 	{
-	    int sz = sizeof(desc->inet.remote);
-	    int code = sock_peer(desc->inet.s,
-				 (struct sockaddr*) &desc->inet.remote, &sz);
-
+	    int sz, code;
+            sz = sizeof(desc->inet.remote);
+            sys_memzero((char *) &desc->inet.remote, sz);
+	    code = sock_peer(desc->inet.s,
+                             (struct sockaddr*) &desc->inet.remote, &sz);
 	    if (IS_SOCKET_ERROR(code)) {
 		desc->inet.state = INET_STATE_OPEN;  /* restore state */
 		ret =  async_error(INETP(desc), sock_errno());
-- 
cgit v1.2.3


From ad5799f22d5091581a7faa457819e9c5ecb759fa Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 19 May 2017 16:07:35 +0200
Subject: Restore sighup behaviour

---
 erts/emulator/sys/unix/sys.c | 1 -
 1 file changed, 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 0079912b10..b1bea3a960 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -885,7 +885,6 @@ void erts_replace_intr(void) {
 void init_break_handler(void)
 {
    sys_signal(SIGINT,  request_break);
-   sys_signal(SIGHUP,  generic_signal_handler);
 #ifndef ETHR_UNUSABLE_SIGUSRX
    sys_signal(SIGUSR1, generic_signal_handler);
 #endif /* #ifndef ETHR_UNUSABLE_SIGUSRX */
-- 
cgit v1.2.3


From 13fe88d704d2cab0f4c369447117d552ae15a7c4 Mon Sep 17 00:00:00 2001
From: Salikhov Dinislam <dinislam.salikhov@gmail.com>
Date: Mon, 8 May 2017 15:39:30 +0300
Subject: erts: Remove unused functions from erl_cpu_topology

---
 erts/emulator/beam/erl_cpu_topology.c | 57 -----------------------------------
 1 file changed, 57 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
index 4347f9f2b7..d0fd763798 100644
--- a/erts/emulator/beam/erl_cpu_topology.c
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -2243,45 +2243,6 @@ add_cpu_groups(int groups,
     return cgm;
 }
 
-static void
-remove_cpu_groups(erts_cpu_groups_callback_t callback, void *arg)
-{
-    erts_cpu_groups_map_t *prev_cgm, *cgm;
-    erts_cpu_groups_callback_list_t *prev_cgcl, *cgcl;
-
-    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
-
-    no_cpu_groups_callbacks--;
-
-    prev_cgm = NULL;
-    for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) {
-	prev_cgcl = NULL;
-	for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) {
-	    if (cgcl->callback == callback && cgcl->arg == arg) {
-		if (prev_cgcl)
-		    prev_cgcl->next = cgcl->next;
-		else
-		    cgm->callback_list = cgcl->next;
-		erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgcl);
-		if (!cgm->callback_list) {
-		    if (prev_cgm)
-			prev_cgm->next = cgm->next;
-		    else
-			cpu_groups_maps = cgm->next;
-		    if (cgm->array)
-			erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgm->array);
-		    erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgm);
-		}
-		return;
-	    }
-	    prev_cgcl = cgcl;
-	}
-	prev_cgm = cgm;
-    }
-
-    erts_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n");
-}
-
 static int
 cpu_groups_lookup(erts_cpu_groups_map_t *map,
 		  ErtsSchedulerData *esdp)
@@ -2321,21 +2282,3 @@ update_cpu_groups_maps(void)
     for (cgm = cpu_groups_maps; cgm; cgm = cgm->next)
 	make_cpu_groups_map(cgm, 0);
 }
-
-void
-erts_add_cpu_groups(int groups,
-		    erts_cpu_groups_callback_t callback,
-		    void *arg)
-{
-    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
-    add_cpu_groups(groups, callback, arg);
-    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
-}
-
-void erts_remove_cpu_groups(erts_cpu_groups_callback_t callback,
-			    void *arg)
-{
-    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
-    remove_cpu_groups(callback, arg);
-    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
-}
-- 
cgit v1.2.3


From 51e4cc4097c1937873fcaf5968ddfa37952ba863 Mon Sep 17 00:00:00 2001
From: Salikhov Dinislam <dinislam.salikhov@gmail.com>
Date: Mon, 8 May 2017 15:13:59 +0300
Subject: erts: Make allocator functions static

---
 erts/emulator/beam/erl_alloc_util.c | 12 ++++++------
 erts/emulator/beam/erl_alloc_util.h |  7 -------
 2 files changed, 6 insertions(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 4889fac923..6fddba4b34 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -821,7 +821,7 @@ static void clear_literal_range(void* start, Uint size)
 
 #if HAVE_ERTS_MSEG
 
-void*
+static void*
 erts_alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
 {
     void *res;
@@ -832,7 +832,7 @@ erts_alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
     return res;
 }
 
-void*
+static void*
 erts_alcu_mseg_realloc(Allctr_t *allctr, void *seg,
                        Uint old_size, Uint *new_size_p)
 {
@@ -845,7 +845,7 @@ erts_alcu_mseg_realloc(Allctr_t *allctr, void *seg,
     return res;
 }
 
-void
+static void
 erts_alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags)
 {
     erts_mseg_dealloc_opt(allctr->alloc_no, seg, (UWord) size, flags, &allctr->mseg_opt);
@@ -996,7 +996,7 @@ erts_alcu_exec_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags)
 
 #endif /* HAVE_ERTS_MSEG */
 
-void*
+static void*
 erts_alcu_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign)
 {
     void *res;
@@ -1013,7 +1013,7 @@ erts_alcu_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign)
     return res;
 }
 
-void*
+static void*
 erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign)
 {
     void *res;
@@ -1035,7 +1035,7 @@ erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size,
     return res;
 }
 
-void
+static void
 erts_alcu_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign)
 {
 #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index f570703f25..7a96bd0319 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -195,10 +195,6 @@ extern UWord erts_literal_vspace_map[];
 # define ERTS_VSPACE_WORD_BITS (sizeof(UWord)*8)
 #endif
 
-void* erts_alcu_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
-void* erts_alcu_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
-void  erts_alcu_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
-
 #if HAVE_ERTS_MSEG
 # if defined(ARCH_32)
 void* erts_alcu_literal_32_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
@@ -218,9 +214,6 @@ void  erts_alcu_exec_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
 # endif
 #endif /* HAVE_ERTS_MSEG */
 
-void* erts_alcu_sys_alloc(Allctr_t*, Uint *size_p, int superalign);
-void* erts_alcu_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign);
-void  erts_alcu_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign);
 #ifdef ARCH_32
 void* erts_alcu_literal_32_sys_alloc(Allctr_t*, Uint *size_p, int superalign);
 void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign);
-- 
cgit v1.2.3


From 2084cb04283e11409dfe2b2b26cdf53355c5c117 Mon Sep 17 00:00:00 2001
From: Salikhov Dinislam <dinislam.salikhov@gmail.com>
Date: Mon, 8 May 2017 15:32:54 +0300
Subject: erts: Make erts_align_utf8_bytes() static

---
 erts/emulator/beam/erl_bits.c | 2 +-
 erts/emulator/beam/erl_bits.h | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index eea55b3239..71c64997c1 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1636,7 +1636,7 @@ erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb)
     return LSB[0] | (LSB[1]<<8) | (LSB[2]<<16) | (LSB[3]<<24);
 }
 
-void
+static void
 erts_align_utf8_bytes(ErlBinMatchBuffer* mb, byte* buf)
 {
     Uint bits = mb->size - mb->offset;
diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h
index 632855255e..5da2b28a89 100644
--- a/erts/emulator/beam/erl_bits.h
+++ b/erts/emulator/beam/erl_bits.h
@@ -202,7 +202,6 @@ void erts_new_bs_put_string(ERL_BITS_PROTO_2(byte* iptr, Uint num_bytes));
 
 Uint erts_bits_bufs_size(void);
 Uint32 erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb);
-void erts_align_utf8_bytes(ErlBinMatchBuffer* mb, byte* buf);
 Eterm erts_bs_get_utf8(ErlBinMatchBuffer* mb);
 Eterm erts_bs_get_utf16(ErlBinMatchBuffer* mb, Uint flags);
 Eterm erts_bs_append(Process* p, Eterm* reg, Uint live, Eterm build_size_term,
-- 
cgit v1.2.3


From bc91e7e9c000145d5feec6877c3abe5e6747351b Mon Sep 17 00:00:00 2001
From: Salikhov Dinislam <dinislam.salikhov@gmail.com>
Date: Thu, 11 May 2017 00:38:25 +0300
Subject: erts: Make bif's do_send() static

---
 erts/emulator/beam/bif.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 7134d2da56..40dd4129d2 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2047,8 +2047,6 @@ ebif_bang_2(BIF_ALIST_2)
 #define SEND_YIELD_CONTINUE     (-8)
 
 
-Sint do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext*);
-
 static Sint remote_send(Process *p, DistEntry *dep,
 			Eterm to, Eterm full_to, Eterm msg,
 			ErtsSendContext* ctx)
@@ -2102,8 +2100,8 @@ static Sint remote_send(Process *p, DistEntry *dep,
     return res;
 }
 
-Sint
-do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx)
+static Sint
+do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx)
 {
     Eterm portid;
     Port *pt;
-- 
cgit v1.2.3


From e606eae1325a9f73140ab309d5dbbb7cdb589a04 Mon Sep 17 00:00:00 2001
From: Ted Burghart <ted@tedb.net>
Date: Mon, 22 May 2017 15:36:09 -0400
Subject: Add enif_whereis_...() functions

#### Why do we need this new feature?

There are cases when a NIF needs to send a message, using `enif_send()`, to a long-lived process with a registered name.

A common use-case is logging, where asynchronous fire-and-forget messages are the norm.

There can also be cases where a yielding or dirty NIF or background thread may request a callback from a service with additional information it needs to complete its operation, yielding or waiting (with suitable timeouts, etc) until its state has been updated through the NIF module's API.

NIFs can only send messages to pids, and the lack of name resolution leaves a complicated dance between separate monitoring processes and the NIF as the only way to keep a NIF informed of the whereabouts of such long-lived processes.

Providing a reliable, built-in facility for NIFs to resolve process (or port) names simplifies these use cases considerably.

#### Risks or uncertain artifacts?

Testing has not exposed any significant risk.

The implementation behaves as expected on regular and dirty scheduler threads as well as non-scheduler threads.

By constraining the `enif_whereis_...()` functions to their minimal scopes and using patterns consistent with related functions, the implementation, testing, and maintenance burden is low.

The API and behavior of existing functions is unchanged.

#### How did you solve it?

While extending `enif_send()` to operate on a pid or an atom (as `erlang:send/2` does) was attractive, it would have entailed changing the type of its `to_pid` parameter and thereby breaking backward compatibility.

The same consideration applies to `enif_port_command()`.

That leaves a choice between 1, 2, or 3 new functions:

 1. `enif_whereis()`
 2. `enif_whereis_pid()` and `enif_whereis_port()`
 3. All of the above.

While option (1), directly mimicking the behavior of `erlang:whereis/1`, is appealing, it poses potential problems if `pid()` or `port()` are subsequently implemented as non-integral types that must be bound to an owning `ErlNifEnv` instance.

Therefore, option (2) has been chosen to use `ErlNifPid`/`ErlNifPort` structures in the API to maintain proper term ownership semantics.
---
 erts/emulator/beam/erl_nif.c                       |  58 +++++
 erts/emulator/beam/erl_nif_api_funcs.h             |   4 +
 erts/emulator/test/dirty_nif_SUITE.erl             | 140 ++++++++++-
 .../test/dirty_nif_SUITE_data/Makefile.src         |   2 +-
 .../test/dirty_nif_SUITE_data/dirty_nif_SUITE.c    | 172 +++++++++++++-
 erts/emulator/test/dirty_nif_SUITE_data/echo_drv.c |  62 +++++
 erts/emulator/test/nif_SUITE.erl                   | 173 +++++++++++++-
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c      | 262 ++++++++++++++++++++-
 8 files changed, 866 insertions(+), 7 deletions(-)
 create mode 100644 erts/emulator/test/dirty_nif_SUITE_data/echo_drv.c

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index ea835d1b64..4815e5e7bb 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -879,6 +879,64 @@ enif_port_command(ErlNifEnv *env, const ErlNifPort* to_port,
     return res;
 }
 
+/*
+ *  env must be the caller's environment in a scheduler or NULL in a
+ *      non-scheduler thread.
+ *  name must be an atom - anything else will just waste time.
+ */
+static Eterm call_whereis(ErlNifEnv *env, Eterm name)
+{
+    Process *c_p;
+    Eterm res;
+    int scheduler;
+    int unlock;
+
+    execution_state(env, &c_p, &scheduler);
+    ASSERT((c_p && scheduler) || (!c_p && !scheduler));
+
+    unlock = 0;
+    if (scheduler < 0) {
+        /* dirty scheduler */
+        if (ERTS_PROC_IS_EXITING(c_p))
+            return 0;
+
+        if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
+            erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+            unlock = 1;
+        }
+    }
+    res = erts_whereis_name_to_id(c_p, name);
+
+    if (unlock)
+        erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+
+    return res;
+}
+
+int enif_whereis_pid(ErlNifEnv *env, ERL_NIF_TERM name, ErlNifPid *pid)
+{
+    Eterm res;
+
+    if (is_not_atom(name))
+        return 0;
+
+    res = call_whereis(env, name);
+    /* enif_get_local_ functions check the type */
+    return enif_get_local_pid(env, res, pid);
+}
+
+int enif_whereis_port(ErlNifEnv *env, ERL_NIF_TERM name, ErlNifPort *port)
+{
+    Eterm res;
+
+    if (is_not_atom(name))
+        return 0;
+
+    res = call_whereis(env, name);
+    /* enif_get_local_ functions check the type */
+    return enif_get_local_port(env, res, port);
+}
+
 ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)
 {
     Uint sz;
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index c305732d63..94c04cd126 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -181,6 +181,8 @@ ERL_NIF_API_FUNC_DECL(int, enif_monitor_process,(ErlNifEnv*,void* obj,const ErlN
 ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const ErlNifMonitor *monitor));
 ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const ErlNifMonitor*));
 ERL_NIF_API_FUNC_DECL(ErlNifUInt64,enif_hash,(ErlNifHash type, ERL_NIF_TERM term, ErlNifUInt64 salt));
+ERL_NIF_API_FUNC_DECL(int, enif_whereis_pid, (ErlNifEnv *env, ERL_NIF_TERM name, ErlNifPid *pid));
+ERL_NIF_API_FUNC_DECL(int, enif_whereis_port, (ErlNifEnv *env, ERL_NIF_TERM name, ErlNifPort *port));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
@@ -344,6 +346,8 @@ ERL_NIF_API_FUNC_DECL(ErlNifUInt64,enif_hash,(ErlNifHash type, ERL_NIF_TERM term
 #  define enif_demonitor_process ERL_NIF_API_FUNC_MACRO(enif_demonitor_process)
 #  define enif_compare_monitors ERL_NIF_API_FUNC_MACRO(enif_compare_monitors)
 #  define enif_hash ERL_NIF_API_FUNC_MACRO(enif_hash)
+#  define enif_whereis_pid ERL_NIF_API_FUNC_MACRO(enif_whereis_pid)
+#  define enif_whereis_port ERL_NIF_API_FUNC_MACRO(enif_whereis_port)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl
index f62f1e9dce..13806fd5c4 100644
--- a/erts/emulator/test/dirty_nif_SUITE.erl
+++ b/erts/emulator/test/dirty_nif_SUITE.erl
@@ -34,7 +34,8 @@
 	 dirty_scheduler_exit/1, dirty_call_while_terminated/1,
 	 dirty_heap_access/1, dirty_process_info/1,
 	 dirty_process_register/1, dirty_process_trace/1,
-	 code_purge/1, dirty_nif_send_traced/1]).
+	 code_purge/1, dirty_nif_send_traced/1,
+	 nif_whereis/1, nif_whereis_parallel/1, nif_whereis_proxy/1]).
 
 -define(nif_stub,nif_stub_error(?LINE)).
 
@@ -51,7 +52,9 @@ all() ->
      dirty_process_register,
      dirty_process_trace,
      code_purge,
-     dirty_nif_send_traced].
+     dirty_nif_send_traced,
+     nif_whereis,
+     nif_whereis_parallel].
 
 init_per_suite(Config) ->
     case erlang:system_info(dirty_cpu_schedulers) of
@@ -531,6 +534,137 @@ mcall(Node, Funs) ->
                       end
               end, Refs).
 
+%% Test enif_whereis_...
+%% These tests are mostly identical to their counterparts in nif_SUITE.erl,
+%% with just name and count changes in the first few lines.
+
+nif_whereis(Config) when is_list(Config) ->
+    erl_ddll:try_load(?config(data_dir, Config), echo_drv, []),
+
+    RegName = dirty_nif_whereis_test_thing,
+    undefined = erlang:whereis(RegName),
+    false = whereis_term(pid, RegName),
+
+    Mgr = self(),
+    Ref = make_ref(),
+    ProcMsg = {Ref, ?LINE},
+    PortMsg = ?MODULE_STRING " whereis hello\n",
+
+    {Pid, Mon} = spawn_monitor(?MODULE, nif_whereis_proxy, [Ref]),
+    true = register(RegName, Pid),
+    Pid = erlang:whereis(RegName),
+    Pid = whereis_term(pid, RegName),
+    false = whereis_term(port, RegName),
+    false = whereis_term(pid, [RegName]),
+
+    ok = whereis_send(pid, RegName, {forward, Mgr, ProcMsg}),
+    ok = receive ProcMsg -> ok end,
+
+    Pid ! {Ref, quit},
+    ok = receive {'DOWN', Mon, process, Pid, normal} -> ok end,
+    undefined = erlang:whereis(RegName),
+    false = whereis_term(pid, RegName),
+
+    Port = open_port({spawn, echo_drv}, [eof]),
+    true = register(RegName, Port),
+    Port = erlang:whereis(RegName),
+    Port = whereis_term(port, RegName),
+    false = whereis_term(pid, RegName),
+    false = whereis_term(port, [RegName]),
+
+    ok = whereis_send(port, RegName, PortMsg),
+    ok = receive {Port, {data, PortMsg}} -> ok end,
+
+    port_close(Port),
+    undefined = erlang:whereis(RegName),
+    false = whereis_term(port, RegName),
+    ok.
+
+nif_whereis_parallel(Config) when is_list(Config) ->
+
+    %% try to be at least a little asymetric
+    NProcs = trunc(3.5 * erlang:system_info(schedulers)),
+    NSeq = lists:seq(1, NProcs),
+    Names = [list_to_atom("dirty_nif_whereis_proc_" ++ integer_to_list(N))
+            || N <- NSeq],
+    Mgr = self(),
+    Ref = make_ref(),
+
+    NotReg = fun(Name) ->
+        erlang:whereis(Name) == undefined
+    end,
+    PidReg = fun({Name, Pid, _Mon}) ->
+        erlang:whereis(Name) == Pid andalso whereis_term(pid, Name) == Pid
+    end,
+    RecvDown = fun({_Name, Pid, Mon}) ->
+        receive {'DOWN', Mon, process, Pid, normal} -> true
+        after   1500 -> false end
+    end,
+    RecvNum = fun(N) ->
+        receive {N, Ref} -> true
+        after   1500 -> false end
+    end,
+
+    true = lists:all(NotReg, Names),
+
+    %% {Name, Pid, Mon}
+    Procs = lists:map(
+        fun(N) ->
+            Name = lists:nth(N, Names),
+            Prev = lists:nth((if N == 1 -> NProcs; true -> (N - 1) end), Names),
+            Next = lists:nth((if N == NProcs -> 1; true -> (N + 1) end), Names),
+            {Pid, Mon} = spawn_monitor(
+                ?MODULE, nif_whereis_proxy, [{N, Ref, Mgr, [Prev, Next]}]),
+            true = register(Name, Pid),
+            {Name, Pid, Mon}
+        end, NSeq),
+
+    true = lists:all(PidReg, Procs),
+
+    %% tell them all to 'fire' as fast as we can
+    [P ! {Ref, send_proc} || {_, P, _} <- Procs],
+
+    %% each gets forwarded through two processes
+    true = lists:all(RecvNum, NSeq),
+    true = lists:all(RecvNum, NSeq),
+
+    %% tell them all to 'quit' by name
+    [N ! {Ref, quit} || {N, _, _} <- Procs],
+    true = lists:all(RecvDown, Procs),
+    true = lists:all(NotReg, Names),
+    ok.
+
+%% exported to be spawned by MFA by whereis tests
+nif_whereis_proxy({N, Ref, Mgr, Targets} = Args) ->
+    receive
+        {forward, To, Data} ->
+            To ! Data,
+            nif_whereis_proxy(Args);
+        {Ref, quit} ->
+            ok;
+        {Ref, send_port} ->
+            Msg = ?MODULE_STRING " whereis " ++ integer_to_list(N) ++ "\n",
+            lists:foreach(
+                fun(T) ->
+                    ok = whereis_send(port, T, Msg)
+                end, Targets),
+            nif_whereis_proxy(Args);
+        {Ref, send_proc} ->
+            lists:foreach(
+                fun(T) ->
+                    ok = whereis_send(pid, T, {forward, Mgr, {N, Ref}})
+                end, Targets),
+            nif_whereis_proxy(Args)
+    end;
+nif_whereis_proxy(Ref) ->
+    receive
+        {forward, To, Data} ->
+            To ! Data,
+            nif_whereis_proxy(Ref);
+        {Ref, quit} ->
+            ok
+    end.
+
 %% The NIFs:
 lib_loaded() -> false.
 call_dirty_nif(_,_,_) -> ?nif_stub.
@@ -542,6 +676,8 @@ dirty_call_while_terminated_nif(_) -> ?nif_stub.
 dirty_sleeper() -> ?nif_stub.
 dirty_sleeper(_) -> ?nif_stub.
 dirty_heap_access_nif(_) -> ?nif_stub.
+whereis_term(_Type,_Name) -> ?nif_stub.
+whereis_send(_Type,_Name,_Msg) -> ?nif_stub.
 
 nif_stub_error(Line) ->
     exit({nif_not_loaded,module,?MODULE,line,Line}).
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/Makefile.src b/erts/emulator/test/dirty_nif_SUITE_data/Makefile.src
index e9301753b0..4462afd815 100644
--- a/erts/emulator/test/dirty_nif_SUITE_data/Makefile.src
+++ b/erts/emulator/test/dirty_nif_SUITE_data/Makefile.src
@@ -1,6 +1,6 @@
 
 NIF_LIBS = dirty_nif_SUITE@dll@
 
-all: $(NIF_LIBS)
+all: $(NIF_LIBS) echo_drv@dll@
 
 @SHLIB_RULES@
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
index caf99c952f..1ab39466db 100644
--- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
+++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
@@ -25,8 +25,35 @@
 #include <unistd.h>
 #endif
 
+/*
+ * Hack to get around this function missing from the NIF API.
+ * TODO: Add this function/macro in the appropriate place, probably with
+ *       enif_make_pid() in erl_nif_api_funcs.h
+ */
+#ifndef enif_make_port
+#define enif_make_port(ENV, PORT) ((void)(ENV),(const ERL_NIF_TERM)((PORT)->port_id))
+#endif
+
+static ERL_NIF_TERM atom_badarg;
+static ERL_NIF_TERM atom_error;
+static ERL_NIF_TERM atom_false;
+static ERL_NIF_TERM atom_lookup;
+static ERL_NIF_TERM atom_ok;
+static ERL_NIF_TERM atom_pid;
+static ERL_NIF_TERM atom_port;
+static ERL_NIF_TERM atom_send;
+
 static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
 {
+    atom_badarg = enif_make_atom(env, "badarg");
+    atom_error = enif_make_atom(env, "error");
+    atom_false = enif_make_atom(env,"false");
+    atom_lookup = enif_make_atom(env, "lookup");
+    atom_ok = enif_make_atom(env,"ok");
+    atom_pid = enif_make_atom(env, "pid");
+    atom_port = enif_make_atom(env, "port");
+    atom_send = enif_make_atom(env, "send");
+
     return 0;
 }
 
@@ -257,6 +284,147 @@ static ERL_NIF_TERM dirty_heap_access_nif(ErlNifEnv* env, int argc, const ERL_NI
     return res;
 }
 
+/*
+ * enif_whereis_... tests
+ * subset of the functions in nif_SUITE.c
+ */
+
+enum {
+    /* results */
+    WHEREIS_SUCCESS,
+    WHEREIS_ERROR_TYPE,
+    WHEREIS_ERROR_LOOKUP,
+    WHEREIS_ERROR_SEND,
+    /* types */
+    WHEREIS_LOOKUP_PID,     /* enif_whereis_pid() */
+    WHEREIS_LOOKUP_PORT     /* enif_whereis_port() */
+};
+
+typedef union {
+    ErlNifPid   pid;
+    ErlNifPort  port;
+} whereis_term_data_t;
+
+static int whereis_type(ERL_NIF_TERM type)
+{
+    if (enif_is_identical(type, atom_pid))
+        return WHEREIS_LOOKUP_PID;
+
+    if (enif_is_identical(type, atom_port))
+        return WHEREIS_LOOKUP_PORT;
+
+    return WHEREIS_ERROR_TYPE;
+}
+
+static int whereis_lookup_internal(
+    ErlNifEnv* env, int type, ERL_NIF_TERM name, whereis_term_data_t* out)
+{
+    if (type == WHEREIS_LOOKUP_PID)
+        return enif_whereis_pid(env, name, & out->pid)
+            ? WHEREIS_SUCCESS : WHEREIS_ERROR_LOOKUP;
+
+    if (type == WHEREIS_LOOKUP_PORT)
+        return enif_whereis_port(env, name, & out->port)
+            ? WHEREIS_SUCCESS : WHEREIS_ERROR_LOOKUP;
+
+    return WHEREIS_ERROR_TYPE;
+}
+
+static int whereis_send_internal(
+    ErlNifEnv* env, int type, whereis_term_data_t* to, ERL_NIF_TERM msg)
+{
+    if (type == WHEREIS_LOOKUP_PID)
+        return enif_send(env, & to->pid, NULL, msg)
+            ? WHEREIS_SUCCESS : WHEREIS_ERROR_SEND;
+
+    if (type == WHEREIS_LOOKUP_PORT)
+        return enif_port_command(env, & to->port, NULL, msg)
+            ? WHEREIS_SUCCESS : WHEREIS_ERROR_SEND;
+
+    return WHEREIS_ERROR_TYPE;
+}
+
+static int whereis_lookup_term(
+    ErlNifEnv* env, int type, ERL_NIF_TERM name, ERL_NIF_TERM* out)
+{
+    whereis_term_data_t res;
+    int rc = whereis_lookup_internal(env, type, name, &res);
+    if (rc == WHEREIS_SUCCESS) {
+        switch (type) {
+            case WHEREIS_LOOKUP_PID:
+                *out = enif_make_pid(env, & res.pid);
+                break;
+            case WHEREIS_LOOKUP_PORT:
+                *out = enif_make_port(env, & res.port);
+                break;
+            default:
+                rc = WHEREIS_ERROR_TYPE;
+                break;
+        }
+    }
+    return rc;
+}
+
+static ERL_NIF_TERM whereis_result_term(ErlNifEnv* env, int result)
+{
+    ERL_NIF_TERM err;
+    switch (result)
+    {
+        case WHEREIS_SUCCESS:
+            return atom_ok;
+        case WHEREIS_ERROR_LOOKUP:
+            err = atom_lookup;
+            break;
+        case WHEREIS_ERROR_SEND:
+            err = atom_send;
+            break;
+        case WHEREIS_ERROR_TYPE:
+            err = atom_badarg;
+            break;
+        default:
+            err = enif_make_int(env, -result);
+            break;
+    }
+    return enif_make_tuple2(env, atom_error, err);
+}
+
+/* whereis_term(Type, Name) -> pid() | port() | false */
+static ERL_NIF_TERM
+whereis_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ERL_NIF_TERM ret;
+    int type, rc;
+
+    if (argc != 2)  /* allow non-atom name for testing */
+        return enif_make_badarg(env);
+
+    if ((type = whereis_type(argv[0])) == WHEREIS_ERROR_TYPE)
+        return enif_make_badarg(env);
+
+    rc = whereis_lookup_term(env, type, argv[1], &ret);
+    return (rc == WHEREIS_SUCCESS) ? ret : atom_false;
+}
+
+/* whereis_send(Type, Name, Message) -> ok | {error, Reason} */
+static ERL_NIF_TERM
+whereis_send(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    whereis_term_data_t to;
+    int type, rc;
+
+    if (argc != 3 || !enif_is_atom(env, argv[1]))
+        return enif_make_badarg(env);
+
+    if ((type = whereis_type(argv[0])) == WHEREIS_ERROR_TYPE)
+        return enif_make_badarg(env);
+
+    rc = whereis_lookup_internal(env, type, argv[1], & to);
+    if (rc == WHEREIS_SUCCESS)
+        rc = whereis_send_internal(env, type, & to, argv[2]);
+
+    return whereis_result_term(env, rc);
+}
+
 
 static ErlNifFunc nif_funcs[] =
 {
@@ -269,7 +437,9 @@ static ErlNifFunc nif_funcs[] =
     {"dirty_sleeper", 0, dirty_sleeper, ERL_NIF_DIRTY_JOB_IO_BOUND},
     {"dirty_sleeper", 1, dirty_sleeper, ERL_NIF_DIRTY_JOB_CPU_BOUND},
     {"dirty_call_while_terminated_nif", 1, dirty_call_while_terminated_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
-    {"dirty_heap_access_nif", 1, dirty_heap_access_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}
+    {"dirty_heap_access_nif", 1, dirty_heap_access_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
+    {"whereis_send", 3, whereis_send, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"whereis_term", 2, whereis_term, ERL_NIF_DIRTY_JOB_CPU_BOUND}
 };
 
 ERL_NIF_INIT(dirty_nif_SUITE,nif_funcs,load,NULL,NULL,NULL)
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/echo_drv.c b/erts/emulator/test/dirty_nif_SUITE_data/echo_drv.c
new file mode 100644
index 0000000000..2b3510c641
--- /dev/null
+++ b/erts/emulator/test/dirty_nif_SUITE_data/echo_drv.c
@@ -0,0 +1,62 @@
+#include <stdio.h>
+#include "erl_driver.h"
+
+static ErlDrvPort erlang_port;
+static ErlDrvData echo_start(ErlDrvPort, char *);
+static void from_erlang(ErlDrvData, char*, ErlDrvSizeT);
+static ErlDrvSSizeT echo_call(ErlDrvData drv_data, unsigned int command,
+			      char *buf, ErlDrvSizeT len,
+			      char **rbuf, ErlDrvSizeT rlen, unsigned *ret_flags);
+static ErlDrvEntry echo_driver_entry = { 
+    NULL,			/* Init */
+    echo_start,
+    NULL,			/* Stop */
+    from_erlang,
+    NULL,			/* Ready input */
+    NULL,			/* Ready output */
+    "echo_drv",
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    echo_call,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
+    NULL
+};
+
+DRIVER_INIT(echo_drv)
+{
+    return &echo_driver_entry;
+}
+
+static ErlDrvData
+echo_start(ErlDrvPort port, char *buf)
+{
+    return (ErlDrvData) port;
+}
+
+static void
+from_erlang(ErlDrvData data, char *buf, ErlDrvSizeT count)
+{
+    driver_output((ErlDrvPort) data, buf, count);
+}
+
+static ErlDrvSSizeT
+echo_call(ErlDrvData drv_data, unsigned int command,
+	  char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen,
+	  unsigned *ret_flags)
+{
+    *rbuf = buf;
+    *ret_flags |= DRIVER_CALL_KEEP_BUFFER;
+    return len;
+}
+
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 1eb58699b2..c8bd9f81f8 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -59,7 +59,9 @@
          nif_snprintf/1,
          nif_internal_hash/1,
          nif_internal_hash_salted/1,
-         nif_phash2/1
+         nif_phash2/1,
+         nif_whereis/1, nif_whereis_parallel/1,
+         nif_whereis_threaded/1, nif_whereis_proxy/1
 	]).
 
 -export([many_args_100/100]).
@@ -96,7 +98,8 @@ all() ->
      nif_snprintf,
      nif_internal_hash,
      nif_internal_hash_salted,
-     nif_phash2].
+     nif_phash2,
+     nif_whereis, nif_whereis_parallel, nif_whereis_threaded].
 
 groups() ->
     [{G, [], api_repeaters()} || G <- api_groups()]
@@ -134,6 +137,11 @@ init_per_testcase(hipe, Config) ->
 	undefined -> {skip, "HiPE is disabled"};
 	_ -> Config
     end;
+init_per_testcase(nif_whereis_threaded, Config) ->
+    case erlang:system_info(threads) of
+        true -> Config;
+        false -> {skip, "No thread support"}
+    end;
 init_per_testcase(select, Config) ->
     case os:type() of
         {win32,_} ->
@@ -2765,6 +2773,161 @@ random_pid() ->
     Processes = erlang:processes(),
     lists:nth(rand:uniform(length(Processes)), Processes).
 
+%% Test enif_whereis_...
+
+nif_whereis(Config) when is_list(Config) ->
+    ensure_lib_loaded(Config),
+
+    RegName = nif_whereis_test_thing,
+    undefined = erlang:whereis(RegName),
+    false = whereis_term(pid, RegName),
+
+    Mgr = self(),
+    Ref = make_ref(),
+    ProcMsg = {Ref, ?LINE},
+    PortMsg = ?MODULE_STRING " whereis hello\n",
+
+    {Pid, Mon} = spawn_monitor(?MODULE, nif_whereis_proxy, [Ref]),
+    true = register(RegName, Pid),
+    Pid = erlang:whereis(RegName),
+    Pid = whereis_term(pid, RegName),
+    false = whereis_term(port, RegName),
+    false = whereis_term(pid, [RegName]),
+
+    ok = whereis_send(pid, RegName, {forward, Mgr, ProcMsg}),
+    ok = receive ProcMsg -> ok end,
+
+    Pid ! {Ref, quit},
+    ok = receive {'DOWN', Mon, process, Pid, normal} -> ok end,
+    undefined = erlang:whereis(RegName),
+    false = whereis_term(pid, RegName),
+
+    Port = open_port({spawn, echo_drv}, [eof]),
+    true = register(RegName, Port),
+    Port = erlang:whereis(RegName),
+    Port = whereis_term(port, RegName),
+    false = whereis_term(pid, RegName),
+    false = whereis_term(port, [RegName]),
+
+    ok = whereis_send(port, RegName, PortMsg),
+    ok = receive {Port, {data, PortMsg}} -> ok end,
+
+    port_close(Port),
+    undefined = erlang:whereis(RegName),
+    false = whereis_term(port, RegName),
+    ok.
+
+nif_whereis_parallel(Config) when is_list(Config) ->
+    ensure_lib_loaded(Config),
+
+    %% try to be at least a little asymetric
+    NProcs = trunc(3.7 * erlang:system_info(schedulers)),
+    NSeq = lists:seq(1, NProcs),
+    Names = [list_to_atom("nif_whereis_proc_" ++ integer_to_list(N))
+            || N <- NSeq],
+    Mgr = self(),
+    Ref = make_ref(),
+
+    NotReg = fun(Name) ->
+        erlang:whereis(Name) == undefined
+    end,
+    PidReg = fun({Name, Pid, _Mon}) ->
+        erlang:whereis(Name) == Pid andalso whereis_term(pid, Name) == Pid
+    end,
+    RecvDown = fun({_Name, Pid, Mon}) ->
+        receive {'DOWN', Mon, process, Pid, normal} -> true
+        after   1500 -> false end
+    end,
+    RecvNum = fun(N) ->
+        receive {N, Ref} -> true
+        after   1500 -> false end
+    end,
+
+    true = lists:all(NotReg, Names),
+
+    %% {Name, Pid, Mon}
+    Procs = lists:map(
+        fun(N) ->
+            Name = lists:nth(N, Names),
+            Prev = lists:nth((if N == 1 -> NProcs; true -> (N - 1) end), Names),
+            Next = lists:nth((if N == NProcs -> 1; true -> (N + 1) end), Names),
+            {Pid, Mon} = spawn_monitor(
+                ?MODULE, nif_whereis_proxy, [{N, Ref, Mgr, [Prev, Next]}]),
+            true = register(Name, Pid),
+            {Name, Pid, Mon}
+        end, NSeq),
+
+    true = lists:all(PidReg, Procs),
+
+    %% tell them all to 'fire' as fast as we can
+    [P ! {Ref, send_proc} || {_, P, _} <- Procs],
+
+    %% each gets forwarded through two processes
+    true = lists:all(RecvNum, NSeq),
+    true = lists:all(RecvNum, NSeq),
+
+    %% tell them all to 'quit' by name
+    [N ! {Ref, quit} || {N, _, _} <- Procs],
+    true = lists:all(RecvDown, Procs),
+    true = lists:all(NotReg, Names),
+    ok.
+
+nif_whereis_threaded(Config) when is_list(Config) ->
+    ensure_lib_loaded(Config),
+
+    RegName = nif_whereis_test_threaded,
+    undefined = erlang:whereis(RegName),
+
+    Ref = make_ref(),
+    {Pid, Mon} = spawn_monitor(?MODULE, nif_whereis_proxy, [Ref]),
+    true = register(RegName, Pid),
+
+    {ok, ProcThr} = whereis_thd_lookup(pid, RegName),
+    {ok, Pid} = whereis_thd_result(ProcThr),
+
+    Pid ! {Ref, quit},
+    ok = receive {'DOWN', Mon, process, Pid, normal} -> ok end,
+
+    Port = open_port({spawn, echo_drv}, [eof]),
+    true = register(RegName, Port),
+
+    {ok, PortThr} = whereis_thd_lookup(port, RegName),
+    {ok, Port} = whereis_thd_result(PortThr),
+
+    port_close(Port),
+    ok.
+
+%% exported to be spawned by MFA by whereis tests
+nif_whereis_proxy({N, Ref, Mgr, Targets} = Args) ->
+    receive
+        {forward, To, Data} ->
+            To ! Data,
+            nif_whereis_proxy(Args);
+        {Ref, quit} ->
+            ok;
+        {Ref, send_port} ->
+            Msg = ?MODULE_STRING " whereis " ++ integer_to_list(N) ++ "\n",
+            lists:foreach(
+                fun(T) ->
+                    ok = whereis_send(port, T, Msg)
+                end, Targets),
+            nif_whereis_proxy(Args);
+        {Ref, send_proc} ->
+            lists:foreach(
+                fun(T) ->
+                    ok = whereis_send(pid, T, {forward, Mgr, {N, Ref}})
+                end, Targets),
+            nif_whereis_proxy(Args)
+    end;
+nif_whereis_proxy(Ref) ->
+    receive
+        {forward, To, Data} ->
+            To ! Data,
+            nif_whereis_proxy(Ref);
+        {Ref, quit} ->
+            ok
+    end.
+
 %% The NIFs:
 lib_version() -> undefined.
 call_history() -> ?nif_stub.
@@ -2839,6 +3002,12 @@ demonitor_process_nif(_,_) -> ?nif_stub.
 compare_monitors_nif(_,_) -> ?nif_stub.
 monitor_frenzy_nif(_,_,_,_) -> ?nif_stub.
 
+%% whereis
+whereis_send(_Type,_Name,_Msg) -> ?nif_stub.
+whereis_term(_Type,_Name) -> ?nif_stub.
+whereis_thd_lookup(_Type,_Name) -> ?nif_stub.
+whereis_thd_result(_Thd) -> ?nif_stub.
+
 %% maps
 is_map_nif(_) -> ?nif_stub.
 get_map_size_nif(_) -> ?nif_stub.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 3747291e7e..003a0f2929 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -52,6 +52,15 @@ static ErlNifMutex* dbg_trace_lock;
 #define DBG_TRACE4(FMT, A, B, C, D)
 #endif
 
+/*
+ * Hack to get around this function missing from the NIF API.
+ * TODO: Add this function/macro in the appropriate place, probably with
+ *       enif_make_pid() in erl_nif_api_funcs.h
+ */
+#ifndef enif_make_port
+#define enif_make_port(ENV, PORT) ((void)(ENV),(const ERL_NIF_TERM)((PORT)->port_id))
+#endif
+
 static int static_cntA; /* zero by default */
 static int static_cntB = NIF_SUITE_LIB_VER * 100;
 
@@ -76,6 +85,11 @@ static ERL_NIF_TERM atom_stats;
 static ERL_NIF_TERM atom_done;
 static ERL_NIF_TERM atom_stop;
 static ERL_NIF_TERM atom_null;
+static ERL_NIF_TERM atom_pid;
+static ERL_NIF_TERM atom_port;
+static ERL_NIF_TERM atom_send;
+static ERL_NIF_TERM atom_lookup;
+static ERL_NIF_TERM atom_badarg;
 
 typedef struct
 {
@@ -170,6 +184,9 @@ static ErlNifResourceTypeInit frenzy_rt_init = {
     frenzy_resource_down
 };
 
+static ErlNifResourceType* whereis_resource_type;
+static void whereis_thread_resource_dtor(ErlNifEnv* env, void* obj);
+
 static int get_pointer(ErlNifEnv* env, ERL_NIF_TERM term, void** pp)
 {
     ErlNifBinary bin;
@@ -223,6 +240,9 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
 						      &frenzy_rt_init,
 						      ERL_NIF_RT_CREATE, NULL);
 
+    whereis_resource_type = enif_open_resource_type(env, NULL, "nif_SUITE.whereis",
+                            whereis_thread_resource_dtor, ERL_NIF_RT_CREATE, NULL);
+
     atom_false = enif_make_atom(env,"false");
     atom_true = enif_make_atom(env,"true");
     atom_self = enif_make_atom(env,"self");
@@ -244,6 +264,11 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
     atom_done = enif_make_atom(env,"done");
     atom_stop = enif_make_atom(env,"stop");
     atom_null = enif_make_atom(env,"null");
+    atom_pid = enif_make_atom(env, "pid");
+    atom_port = enif_make_atom(env, "port");
+    atom_send = enif_make_atom(env, "send");
+    atom_lookup = enif_make_atom(env, "lookup");
+    atom_badarg = enif_make_atom(env, "badarg");
 
     *priv_data = data;
     return 0;
@@ -1137,6 +1162,237 @@ static void fill(void* dst, unsigned bytes, int seed)
     }
 }
 
+/* enif_whereis_... tests */
+
+enum {
+    /* results */
+    WHEREIS_SUCCESS,
+    WHEREIS_ERROR_TYPE,
+    WHEREIS_ERROR_LOOKUP,
+    WHEREIS_ERROR_SEND,
+    /* types */
+    WHEREIS_LOOKUP_PID,     /* enif_whereis_pid() */
+    WHEREIS_LOOKUP_PORT     /* enif_whereis_port() */
+};
+
+typedef union {
+    ErlNifPid   pid;
+    ErlNifPort  port;
+} whereis_term_data_t;
+
+/* single use, no cross-thread access/serialization */
+typedef struct {
+    ErlNifEnv* env;
+    ERL_NIF_TERM name;
+    whereis_term_data_t res;
+    ErlNifTid tid;
+    int type;
+} whereis_thread_resource_t;
+
+static whereis_thread_resource_t* whereis_thread_resource_create(void)
+{
+    whereis_thread_resource_t* rp = (whereis_thread_resource_t*)
+        enif_alloc_resource(whereis_resource_type, sizeof(*rp));
+    memset(rp, 0, sizeof(*rp));
+    rp->env = enif_alloc_env();
+
+    return rp;
+}
+
+static void whereis_thread_resource_dtor(ErlNifEnv* env, void* obj)
+{
+    whereis_thread_resource_t* rp = (whereis_thread_resource_t*) obj;
+    enif_free_env(rp->env);
+}
+
+static int whereis_type(ERL_NIF_TERM type)
+{
+    if (enif_is_identical(type, atom_pid))
+        return WHEREIS_LOOKUP_PID;
+
+    if (enif_is_identical(type, atom_port))
+        return WHEREIS_LOOKUP_PORT;
+
+    return WHEREIS_ERROR_TYPE;
+}
+
+static int whereis_lookup_internal(
+    ErlNifEnv* env, int type, ERL_NIF_TERM name, whereis_term_data_t* out)
+{
+    if (type == WHEREIS_LOOKUP_PID)
+        return enif_whereis_pid(env, name, & out->pid)
+            ? WHEREIS_SUCCESS : WHEREIS_ERROR_LOOKUP;
+
+    if (type == WHEREIS_LOOKUP_PORT)
+        return enif_whereis_port(env, name, & out->port)
+            ? WHEREIS_SUCCESS : WHEREIS_ERROR_LOOKUP;
+
+    return WHEREIS_ERROR_TYPE;
+}
+
+static int whereis_send_internal(
+    ErlNifEnv* env, int type, whereis_term_data_t* to, ERL_NIF_TERM msg)
+{
+    if (type == WHEREIS_LOOKUP_PID)
+        return enif_send(env, & to->pid, NULL, msg)
+            ? WHEREIS_SUCCESS : WHEREIS_ERROR_SEND;
+
+    if (type == WHEREIS_LOOKUP_PORT)
+        return enif_port_command(env, & to->port, NULL, msg)
+            ? WHEREIS_SUCCESS : WHEREIS_ERROR_SEND;
+
+    return WHEREIS_ERROR_TYPE;
+}
+
+static int whereis_resolved_term(
+    ErlNifEnv* env, int type, whereis_term_data_t* res, ERL_NIF_TERM* out)
+{
+    switch (type) {
+        case WHEREIS_LOOKUP_PID:
+            *out = enif_make_pid(env, & res->pid);
+            break;
+        case WHEREIS_LOOKUP_PORT:
+            *out = enif_make_port(env, & res->port);
+            break;
+        default:
+            return WHEREIS_ERROR_TYPE;
+    }
+    return WHEREIS_SUCCESS;
+}
+
+static ERL_NIF_TERM whereis_result_term(ErlNifEnv* env, int result)
+{
+    ERL_NIF_TERM err;
+    switch (result)
+    {
+        case WHEREIS_SUCCESS:
+            return atom_ok;
+        case WHEREIS_ERROR_LOOKUP:
+            err = atom_lookup;
+            break;
+        case WHEREIS_ERROR_SEND:
+            err = atom_send;
+            break;
+        case WHEREIS_ERROR_TYPE:
+            err = atom_badarg;
+            break;
+        default:
+            err = enif_make_int(env, -result);
+            break;
+    }
+    return enif_make_tuple2(env, atom_error, err);
+}
+
+static void* whereis_lookup_thread(void* arg)
+{
+    whereis_thread_resource_t* rp = (whereis_thread_resource_t*) arg;
+    int rc;
+
+    /* enif_whereis_xxx should work with allocated or null env */
+    rc = whereis_lookup_internal(
+        ((rp->type == WHEREIS_LOOKUP_PID) ? NULL : rp->env),
+        rp->type, rp->name, & rp->res);
+
+    return (((char*) NULL) + rc);
+}
+
+/* whereis_term(Type, Name) -> pid() | port() | false */
+static ERL_NIF_TERM
+whereis_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    whereis_term_data_t res;
+    ERL_NIF_TERM ret;
+    int type, rc;
+
+    if (argc != 2)  /* allow non-atom name for testing */
+        return enif_make_badarg(env);
+
+    if ((type = whereis_type(argv[0])) == WHEREIS_ERROR_TYPE)
+        return enif_make_badarg(env);
+
+    rc = whereis_lookup_internal(env, type, argv[1], & res);
+    if (rc == WHEREIS_SUCCESS) {
+        rc = whereis_resolved_term(env, type, & res, & ret);
+    }
+    return (rc == WHEREIS_SUCCESS) ? ret : atom_false;
+}
+
+/* whereis_send(Type, Name, Message) -> ok | {error, Reason} */
+static ERL_NIF_TERM
+whereis_send(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    whereis_term_data_t to;
+    int type, rc;
+
+    if (argc != 3 || !enif_is_atom(env, argv[1]))
+        return enif_make_badarg(env);
+
+    if ((type = whereis_type(argv[0])) == WHEREIS_ERROR_TYPE)
+        return enif_make_badarg(env);
+
+    rc = whereis_lookup_internal(env, type, argv[1], & to);
+    if (rc == WHEREIS_SUCCESS)
+        rc = whereis_send_internal(env, type, & to, argv[2]);
+
+    return whereis_result_term(env, rc);
+}
+
+/* whereis_thd_lookup(Type, Name) -> {ok, Resource} | {error, SysErrno} */
+static ERL_NIF_TERM
+whereis_thd_lookup(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    whereis_thread_resource_t* rp;
+    int type, rc;
+
+    if (argc != 2 || !enif_is_atom(env, argv[1]))
+        return enif_make_badarg(env);
+
+    if ((type = whereis_type(argv[0])) == WHEREIS_ERROR_TYPE)
+        return enif_make_badarg(env);
+
+    rp = whereis_thread_resource_create();
+    rp->type = type;
+    rp->name = enif_make_copy(rp->env, argv[1]);
+
+    rc = enif_thread_create(
+        "nif_SUITE:whereis_thd", & rp->tid, whereis_lookup_thread, rp, NULL);
+
+    if (rc == 0) {
+        return enif_make_tuple2(env, atom_ok, enif_make_resource(env, rp));
+    }
+    else {
+        enif_release_resource(rp);
+        return enif_make_tuple2(env, atom_error, enif_make_int(env, rc));
+    }
+}
+
+/* whereis_thd_result(Resource) -> {ok, pid() | port()} | {error, ErrNum} */
+static ERL_NIF_TERM
+whereis_thd_result(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    whereis_thread_resource_t* rp;
+    ERL_NIF_TERM ret;
+    char* thdret; /* so we can keep compilers happy converting to int */
+    int rc;
+
+    if (argc != 1
+    || !enif_get_resource(env, argv[0], whereis_resource_type, (void**) & rp))
+        return enif_make_badarg(env);
+
+    if ((rc = enif_thread_join(rp->tid, (void**) & thdret)) != 0)
+        return enif_make_tuple2(env, atom_error, enif_make_int(env, rc));
+    
+    rc = (int)(thdret - ((char*) NULL));
+    if (rc == WHEREIS_SUCCESS) {
+        rc = whereis_resolved_term(env, rp->type, & rp->res, & ret);
+    }
+    ret = (rc == WHEREIS_SUCCESS)
+        ? enif_make_tuple2(env, atom_ok, ret) : whereis_result_term(env, rc);
+    
+    enif_release_resource(rp);
+    return ret;
+}
+
 #define MAKE_TERM_REUSE_LEN 16
 struct make_term_info
 {
@@ -2968,7 +3224,11 @@ static ErlNifFunc nif_funcs[] =
     {"monitor_process_nif", 4, monitor_process_nif},
     {"demonitor_process_nif", 2, demonitor_process_nif},
     {"compare_monitors_nif", 2, compare_monitors_nif},
-    {"monitor_frenzy_nif", 4, monitor_frenzy_nif}
+    {"monitor_frenzy_nif", 4, monitor_frenzy_nif},
+    {"whereis_send", 3, whereis_send},
+    {"whereis_term", 2, whereis_term},
+    {"whereis_thd_lookup", 2, whereis_thd_lookup},
+    {"whereis_thd_result", 1, whereis_thd_result}
 };
 
 ERL_NIF_INIT(nif_SUITE,nif_funcs,load,NULL,upgrade,unload)
-- 
cgit v1.2.3


From 8543e7e6e92e0e15843f2ce0ea70513d421cd8ae Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 10 May 2017 20:38:32 +0200
Subject: erts: Test monitor resource destructor from thread

which is scheduled on non-smp VM.
---
 erts/emulator/test/nif_SUITE.erl              | 15 ++++++++++++++-
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 25 +++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 1eb58699b2..0ae8e02011 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -629,6 +629,15 @@ monitor_process_a(Config) ->
 monitor_process_b(Config) ->
     ensure_lib_loaded(Config),
 
+    monitor_process_b_do(false),
+    case erlang:system_info(threads) of
+        true ->  monitor_process_b_do(true);
+        false -> ok
+    end,
+    ok.
+
+
+monitor_process_b_do(FromThread) ->
     Pid = spawn_link(fun() ->
                              receive
                                  return -> ok
@@ -637,7 +646,10 @@ monitor_process_b(Config) ->
     R_ptr = alloc_monitor_resource_nif(),
     {0,_} = monitor_process_nif(R_ptr, Pid, true, self()),
     [R_ptr] = monitored_by(Pid),
-    ok = release_resource(R_ptr),
+    case FromThread of
+        false -> ok = release_resource(R_ptr);
+        true -> ok = release_resource_from_thread(R_ptr)
+    end,
     [] = flush(),
     {R_ptr, _, 1} = last_resource_dtor_call(),
     [] = monitored_by(Pid),
@@ -2789,6 +2801,7 @@ alloc_resource(_,_) -> ?nif_stub.
 make_resource(_) -> ?nif_stub.
 get_resource(_,_) -> ?nif_stub.
 release_resource(_) -> ?nif_stub.
+release_resource_from_thread(_) -> ?nif_stub.
 last_resource_dtor_call() -> ?nif_stub.
 make_new_resource(_,_) -> ?nif_stub.
 check_is(_,_,_,_,_,_,_,_,_,_,_) -> ?nif_stub.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 3747291e7e..4f0400c960 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -972,6 +972,30 @@ static ERL_NIF_TERM release_resource(ErlNifEnv* env, int argc, const ERL_NIF_TER
     return enif_make_atom(env,"ok");
 }
 
+static void* threaded_release_resource(void* resource)
+{
+    enif_release_resource(resource);
+}
+
+static ERL_NIF_TERM release_resource_from_thread(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    void* resource;
+    ErlNifTid tid;
+    int err;
+
+    if (!get_pointer(env, argv[0], &resource)) {
+        return enif_make_badarg(env);
+    }
+    if (enif_thread_create("nif_SUITE:release_resource_from_thread", &tid,
+                           threaded_release_resource, resource, NULL) != 0) {
+        return enif_make_badarg(env);
+    }
+    err = enif_thread_join(tid, NULL);
+    assert(err == 0);
+    return atom_ok;
+}
+
+
 /*
  * argv[0] an atom
  * argv[1] a binary
@@ -2903,6 +2927,7 @@ static ErlNifFunc nif_funcs[] =
     {"make_resource", 1, make_resource},
     {"get_resource", 2, get_resource},
     {"release_resource", 1, release_resource},
+    {"release_resource_from_thread", 1, release_resource_from_thread},
     {"last_resource_dtor_call", 0, last_resource_dtor_call},
     {"make_new_resource", 2, make_new_resource},
     {"check_is", 11, check_is},
-- 
cgit v1.2.3


From fb943b6efa883991377f475f4e24ce16bf948021 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 17 May 2017 19:21:20 +0200
Subject: erts: Improve message flush in nif_SUITE

Pass number of expected messages in order to
wait longer for messages (for slow machines)
while not wasting test time waiting for nothing.
---
 erts/emulator/test/nif_SUITE.erl | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 0ae8e02011..f9ee96c9ca 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -488,7 +488,7 @@ select(Config) when is_list(Config) ->
     %% Wait for read
     eagain = read_nif(R, 3),
     0 = select_nif(R,?ERL_NIF_SELECT_READ,R,null,Ref),
-    [] = flush(),
+    [] = flush(0),
     ok = write_nif(W, <<"hej">>),
     [{select, R, Ref, ready_input}] = flush(),
     0 = select_nif(R,?ERL_NIF_SELECT_READ,R,self(),Ref2),
@@ -505,7 +505,7 @@ select(Config) when is_list(Config) ->
     %% Wait for write
     Written = write_full(W, $a),
     0 = select_nif(W,?ERL_NIF_SELECT_WRITE,W,self(),Ref),
-    [] = flush(),
+    [] = flush(0),
     Written = read_nif(R,byte_size(Written)),
     [{select, W, Ref, ready_output}] = flush(),
 
@@ -515,7 +515,7 @@ select(Config) when is_list(Config) ->
     [{fd_resource_stop, W_ptr, _}] = flush(),
     {1, {W_ptr,_}} = last_fd_stop_call(),
     true = is_closed_nif(W),
-    [] = flush(),
+    [] = flush(0),
     0 = select_nif(R,?ERL_NIF_SELECT_READ,R,self(),Ref),
     [{select, R, Ref, ready_input}] = flush(),
     eof = read_nif(R,1),
@@ -540,7 +540,7 @@ select_2(Config) ->
     0 = select_nif(R,?ERL_NIF_SELECT_READ,R,null,Ref1),
     0 = select_nif(R,?ERL_NIF_SELECT_READ,R,self(),Ref2),
 
-    [] = flush(),
+    [] = flush(0),
     ok = write_nif(W, <<"hej">>),
     [{select, R, Ref2, ready_input}] = flush(),
     <<"hej">> = read_nif(R, 3),
@@ -551,7 +551,7 @@ select_2(Config) ->
     Papa = self(),
     spawn_link(fun() ->
                        0 = select_nif(R,?ERL_NIF_SELECT_READ,R,null,Ref1),
-                       [] = flush(),
+                       [] = flush(0),
                        Papa ! sync,
                        [{select, R, Ref1, ready_input}] = flush(),
                        <<"hej">> = read_nif(R, 3),
@@ -560,7 +560,7 @@ select_2(Config) ->
     sync = receive_any(),
     ok = write_nif(W, <<"hej">>),
     done = receive_any(),
-    [] = flush(),
+    [] = flush(0),
 
     check_stop_ret(select_nif(R,?ERL_NIF_SELECT_STOP,R,null,Ref1)),
     [{fd_resource_stop, R_ptr, _}] = flush(),
@@ -650,7 +650,7 @@ monitor_process_b_do(FromThread) ->
         false -> ok = release_resource(R_ptr);
         true -> ok = release_resource_from_thread(R_ptr)
     end,
-    [] = flush(),
+    [] = flush(0),
     {R_ptr, _, 1} = last_resource_dtor_call(),
     [] = monitored_by(Pid),
     Pid ! return,
@@ -672,7 +672,7 @@ monitor_process_c(Config) ->
                              exit
                      end),
     [{Pid, done, R_ptr, Mon1},
-     {monitor_resource_down, R_ptr, Pid, Mon2}] = flush(),
+     {monitor_resource_down, R_ptr, Pid, Mon2}] = flush(2),
     compare_monitors_nif(Mon1, Mon2),
     {R_ptr, _, 1} = last_resource_dtor_call(),
     ok.
@@ -720,7 +720,7 @@ demonitor_process(Config) ->
     1 = demonitor_process_nif(R_ptr, MonBin2),
 
     ok = release_resource(R_ptr),
-    [] = flush(),
+    [] = flush(0),
     {R_ptr, _, 1} = last_resource_dtor_call(),
     [] = monitored_by(Pid),
     Pid ! return,
@@ -2319,10 +2319,16 @@ receive_any(Timeout) ->
     after Timeout -> timeout end.
 
 flush() ->
-    flush(10).
-flush(Timeout) ->
+    flush(1).
+
+flush(0) ->
+    flush(0, 10);  % don't waste too much time waiting for nothing
+flush(N) ->
+    flush(N, 1000).
+
+flush(N, Timeout) ->
     receive M ->
-            [M | flush(Timeout)]
+            [M | flush(N-1)]
     after Timeout ->
             []
     end.
-- 
cgit v1.2.3


From d05821528fa012b9eaf01dd79d79f4f34f2e86ef Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 22 May 2017 18:23:45 +0200
Subject: erts: Fix nif_SUITE:monitor_frenzy for threadless

---
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 4f0400c960..15d31162ed 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -2561,6 +2561,7 @@ static ERL_NIF_TERM monitor_frenzy_nif(ErlNifEnv* env, int argc, const ERL_NIF_T
     static unsigned long spawn_cnt = 0;
     static unsigned long kill_cnt = 0;
     static unsigned long proc_histogram[FRENZY_PROCS_MAX];
+    static int initialized = 0;
 
     static const unsigned int primes[] = {7, 13, 17, 19};
 
@@ -2580,7 +2581,7 @@ static ERL_NIF_TERM monitor_frenzy_nif(ErlNifEnv* env, int argc, const ERL_NIF_T
 
     if (enif_is_atom(env, Op)) {
         if (Op == atom_init) {
-            if (procs_lock || !enif_get_uint(env, Rnd, &frenzy_rand_bits_max))
+            if (initialized || !enif_get_uint(env, Rnd, &frenzy_rand_bits_max))
                 return enif_make_badarg(env);
 
             procs_lock = enif_mutex_create("nif_SUITE:monitor_frenzy.procs");
@@ -2607,6 +2608,7 @@ static ERL_NIF_TERM monitor_frenzy_nif(ErlNifEnv* env, int argc, const ERL_NIF_T
 
             spawn_cnt = 1;
             kill_cnt = 0;
+            initialized = 1;
             return enif_make_uint(env, 0);  /* SelfPix */
         }
         else if (Op == atom_stats) {
@@ -2637,7 +2639,7 @@ static ERL_NIF_TERM monitor_frenzy_nif(ErlNifEnv* env, int argc, const ERL_NIF_T
                                              enif_make_ulong(env, res_dtor_cnt)));
 
         }
-        else if (Op == atom_stop && procs_lock) {  /* stop all */
+        else if (Op == atom_stop && initialized) {  /* stop all */
 
             /* Release all resources */
             for (rix = 0; rix < FRENZY_RESOURCES_MAX; rix++) {
-- 
cgit v1.2.3


From 3907fdbb277efdfd9e0675569fed92d78fc88d73 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 23 May 2017 17:32:47 +0200
Subject: erts: Reduce runtime for nif_SUITE hash tests

and base allowed hash deviation on nr of standard deviations
to make it easier to fiddle with the work load.
---
 erts/emulator/test/nif_SUITE.erl | 42 ++++++++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 17 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index f9ee96c9ca..bcea9e3539 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -2637,9 +2637,9 @@ nif_snprintf(Config) ->
 nif_internal_hash(Config) ->
     ensure_lib_loaded(Config),
     HashValueBitSize = nif_hash_result_bitsize(internal),
-    Terms = unique([random_term() || _ <- lists:seq(1, 5000)]),
+    Terms = unique([random_term() || _ <- lists:seq(1, 500)]),
     HashValues = [hash_nif(internal, Term, 0) || Term <- Terms],
-    test_bit_distribution_fitness(HashValues, HashValueBitSize, 0.05).
+    test_bit_distribution_fitness(HashValues, HashValueBitSize).
 
 nif_internal_hash_salted(Config) ->
     ensure_lib_loaded(Config),
@@ -2648,7 +2648,7 @@ nif_internal_hash_salted(Config) ->
 nif_phash2(Config) ->
     ensure_lib_loaded(Config),
     HashValueBitSize = nif_hash_result_bitsize(phash2),
-    Terms = unique([random_term() || _ <- lists:seq(1, 5000)]),
+    Terms = unique([random_term() || _ <- lists:seq(1, 500)]),
     HashValues =
         lists:map(
           fun (Term) ->
@@ -2661,12 +2661,12 @@ nif_phash2(Config) ->
                   HashValue
           end,
           Terms),
-    test_bit_distribution_fitness(HashValues, HashValueBitSize, 0.05).
+    test_bit_distribution_fitness(HashValues, HashValueBitSize).
 
 test_salted_nif_hash(HashType) ->
     HashValueBitSize = nif_hash_result_bitsize(HashType),
-    Terms = unique([random_term() || _ <- lists:seq(1, 5000)]),
-    Salts = unique([random_uint32() || _ <- lists:seq(1, 100)]),
+    Terms = unique([random_term() || _ <- lists:seq(1, 500)]),
+    Salts = unique([random_uint32() || _ <- lists:seq(1, 50)]),
     {HashValuesPerSalt, HashValuesPerTerm} =
         lists:mapfoldl(
           fun (Salt, Acc) ->
@@ -2687,22 +2687,20 @@ test_salted_nif_hash(HashType) ->
     % Test per-salt hash distribution of different terms
     lists:foreach(
       fun ({_Salt, HashValues}) ->
-              test_bit_distribution_fitness(HashValues, HashValueBitSize, 0.05)
+              test_bit_distribution_fitness(HashValues, HashValueBitSize)
       end,
       HashValuesPerSalt),
 
     % Test per-term hash distribution of different salts
     dict:fold(
       fun (_Term, HashValues, Acc) ->
-              % Be more tolerant of relative deviation,
-              % as there's fewer hash values here.
-              test_bit_distribution_fitness(HashValues, HashValueBitSize, 0.30),
+              test_bit_distribution_fitness(HashValues, HashValueBitSize),
               Acc
       end,
       ok,
       HashValuesPerTerm).
 
-test_bit_distribution_fitness(Integers, BitSize, MaxRelativeDeviation) ->
+test_bit_distribution_fitness(Integers, BitSize) ->
     MaxInteger = (1 bsl BitSize) - 1,
     OnesPerBit =
         lists:foldl(
@@ -2718,19 +2716,29 @@ test_bit_distribution_fitness(Integers, BitSize, MaxRelativeDeviation) ->
           orddict:new(),
           Integers),
 
-    ExpectedNrOfOnes = length(Integers) div 2,
+    N = length(Integers),
+    ExpectedNrOfOnes = N div 2,
+    %% ExpectedNrOfOnes should have a binomial distribution
+    %% with a standard deviation as:
+    ExpectedStdDev = math:sqrt(N) / 2,
+    %% which can be approximated as a normal distribution
+    %% where we allow a deviation of 6 std.devs
+    %% for a fail probability of 0.000000002:
+    MaxStdDevs = 6,
+
     FailureText =
         orddict:fold(
           fun (BitIndex, NrOfOnes, Acc) ->
-                  RelativeDeviation = abs(NrOfOnes - ExpectedNrOfOnes) / length(Integers),
-                  case RelativeDeviation >= MaxRelativeDeviation of
-                      false -> Acc;
+                  Deviation = abs(NrOfOnes - ExpectedNrOfOnes) / ExpectedStdDev,
+                  case Deviation >= MaxStdDevs of
+                      false ->
+                          Acc;
                       true ->
                           [Acc,
                            io_lib:format(
                              "Unreasonable deviation on number of set bits (i=~p): "
-                             "expected ~p, got ~p (relative dev. ~.3f)~n",
-                             [BitIndex, ExpectedNrOfOnes, NrOfOnes, RelativeDeviation])]
+                             "expected ~p, got ~p (# std.dev ~.3f > ~p)~n",
+                             [BitIndex, ExpectedNrOfOnes, NrOfOnes, Deviation, MaxStdDevs])]
                   end
           end,
           [],
-- 
cgit v1.2.3


From 666de16fbea2e06b76e1850296ee42eda563a4c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 24 May 2017 14:42:39 +0200
Subject: bs_construct_SUITE: Correct calculation of free memory

free_mem/0 returned the number of free Kb, but the caller
assumed that it was in Mb.

Also add another clause to further scale down the size of
the binaries created.
---
 erts/emulator/test/bs_construct_SUITE.erl | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl
index 95042ac802..ed03284a5b 100644
--- a/erts/emulator/test/bs_construct_SUITE.erl
+++ b/erts/emulator/test/bs_construct_SUITE.erl
@@ -537,6 +537,8 @@ huge_binary(Config) when is_list(Config) ->
     ct:timetrap({seconds, 60}),
     16777216 = size(<<0:(id(1 bsl 26)),(-1):(id(1 bsl 26))>>),
     garbage_collect(),
+    FreeMem = free_mem(),
+    io:format("Free memory (Mb): ~p\n", [FreeMem]),
     {Shift,Return} = case free_mem() of
 			 undefined ->
 			     %% This test has to be inlined inside the case to
@@ -552,10 +554,14 @@ huge_binary(Config) when is_list(Config) ->
 			     garbage_collect(),
 			     id(<<0:((1 bsl 31)-1)>>),
 			     {31,"Limit huge binaries to 256 Mb"};
-			 _ ->
+			 Mb when Mb > 200 ->
 			     garbage_collect(),
 			     id(<<0:((1 bsl 30)-1)>>),
-			     {30,"Limit huge binary to 128 Mb"}
+			     {30,"Limit huge binary to 128 Mb"};
+			 _ ->
+			     garbage_collect(),
+			     id(<<0:((1 bsl 29)-1)>>),
+                             {29,"Limit huge binary to 64 Mb"}
 		     end,
     garbage_collect(),
     id(<<0:((1 bsl Shift)-1)>>),
@@ -567,13 +573,14 @@ huge_binary(Config) when is_list(Config) ->
 	Comment -> {comment, Comment}
     end.
 
+%% Return the amount of free memory in Mb.
 free_mem() ->
     {ok,Apps} = application:ensure_all_started(os_mon),
     Mem = memsup:get_system_memory_data(),
     [ok = application:stop(App)||App <- Apps],
     case proplists:get_value(free_memory,Mem) of
         undefined -> undefined;
-        Val -> Val div 1024
+        Val -> Val div (1024*1024)
     end.
 
 system_limit(Config) when is_list(Config) ->
-- 
cgit v1.2.3


From a261e4f4eda13f6edb37349b18e096694a86e29a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 24 May 2017 15:47:37 +0200
Subject: Identify db_hash_slot locks by their table name instead of slot index

---
 erts/emulator/beam/erl_db_hash.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 390369fdb9..036cc9d9e9 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -684,7 +684,7 @@ int db_create_hash(Process *p, DbTable *tbl)
 							      sizeof(DbTableHashFineLocks));	    	    
 	for (i=0; i<DB_HASH_LOCK_CNT; ++i) {
 	    erts_smp_rwmtx_init_opt_x(&tb->locks->lck_vec[i].lck, &rwmtx_opt,
-				      "db_hash_slot", make_small(i));
+				      "db_hash_slot", tb->common.the_name);
 	}
 	/* This important property is needed to guarantee that the buckets
     	 * involved in a grow/shrink operation it protected by the same lock:
-- 
cgit v1.2.3


From 23f132d9ab776ab94c29f1faed76c9fde8f4a73f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 29 May 2017 14:41:14 +0200
Subject: Make sure that asynchronous replies are not lost

If an synchronous GC was requested by calling:

     erlang:garbage_collect(Pid, [{async,Ref}])

the reply message could in certain circumstances be
lost.

The problem is in cleanup_sys_tasks() in erl_process.c.
If there were at least one dirty task, only the first
dirty task would be cleaned up. All other systems tasks
would not be cleaned up (that is, no replies would be
sent to other processes waiting for the tasks to finish).
---
 erts/emulator/beam/erl_process.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 7952e3031d..0575729bbd 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11331,7 +11331,9 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
     erts_aint32_t state = in_state;
     int max_reds = in_reds;
     int reds = 0;
-    int qmask = 0;
+    int qmask = 1;          /* Set to 1 to force looping as long as there
+                             * are dirty tasks.
+                             */
 
     ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
 
-- 
cgit v1.2.3


From c819af0538083d861e8bd8ba66768f5e5a633311 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 29 May 2017 14:51:01 +0200
Subject: erl_process.c: Add more assertions in process termination

Before terminating a process, add assertions to make sure that
all queues for system tasks have been emptied.
---
 erts/emulator/beam/erl_process.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 0575729bbd..d4385e3987 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -13937,6 +13937,16 @@ erts_continue_exit_process(Process *p)
 	    goto yield;
     }
 
+#ifdef DEBUG
+    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+    ASSERT(p->sys_task_qs == NULL);
+    ASSERT(ERTS_PROC_GET_DELAYED_GC_TASK_QS(p) == NULL);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(p->dirty_sys_tasks == NULL);
+#endif
+    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+#endif
+
     if (p->flags & F_USING_DDLL) {
 	erts_ddll_proc_dead(p, ERTS_PROC_LOCK_MAIN);
 	p->flags &= ~F_USING_DDLL;
-- 
cgit v1.2.3


From 6934f9779d9c81648c75721e751ef37d34fbe944 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 22 May 2017 12:29:17 +0200
Subject: hipe: Fix hipe_mkliterals make recipe

If a link dependency is added in LDFLAGS it should be resolved
in LIBS, so we have to use LIBS and not TYPE_LIBS.
---
 erts/emulator/Makefile.in | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 254e59f2c7..61c1e14741 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -149,6 +149,8 @@ endif
 
 LIBS += $(TYPE_LIBS)
 
+ORIG_LIBS:= $(LIBS)
+
 comma:=,
 space:=
 space+=
@@ -965,7 +967,7 @@ $(OBJDIR)/%.o: hipe/%.c
 	$(V_CC) $(subst O2,O3, $(CFLAGS)) $(INCLUDES) -c $< -o $@
 
 $(BINDIR)/hipe_mkliterals$(TF_MARKER):	$(OBJDIR)/hipe_mkliterals.o
-	$(ld_verbose)$(CC) $(LDFLAGS) -o $@ $< $(TYPE_LIBS)
+	$(ld_verbose)$(CC) $(LDFLAGS) -o $@ $< $(ORIG_LIBS)
 
 $(OBJDIR)/hipe_mkliterals.o:	$(HIPE_ASM) $(TTF_DIR)/erl_alloc_types.h $(DTRACE_HEADERS) \
 	$(TTF_DIR)/OPCODES-GENERATED $(TTF_DIR)/TABLES-GENERATED
-- 
cgit v1.2.3


From 132fb25718f46553c538d5a26c9cab2a761515a7 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 29 May 2017 15:10:05 +0200
Subject: Fix WX lock check assertion on MacOSX

The fix avoids registering the main thread as an emulator thread
on MacOSX. This since WX steals the main thread for its own usage
on MacOSX, and use the thread as an ordinary driver thread.
---
 erts/emulator/sys/unix/sys.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index b1bea3a960..5cf0a49972 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -1456,12 +1456,12 @@ erts_sys_main_thread(void)
     erts_thread_disable_fpe();
 #ifdef __DARWIN__
     initialize_darwin_main_thread_pipes();
-#endif
+#else
     /* Become signal receiver thread... */
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_set_thread_name("signal_receiver");
 #endif
-
+#endif
     smp_sig_notify(0); /* Notify initialized */
 
     /* Wait for a signal to arrive... */
-- 
cgit v1.2.3


From 8fe44f0f81603697174fcf6ad20e7acbb22fea5f Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 29 May 2017 17:19:18 +0200
Subject: Enable off-heap message queue for code purger, et. al.

---
 erts/emulator/beam/erl_init.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index ac99f043e6..6172595552 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -431,7 +431,7 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char**
 }
 
 static Eterm
-erl_system_process_otp(Eterm parent_pid, char* modname)
+erl_system_process_otp(Eterm parent_pid, char* modname, int off_heap_msgq)
 {
     Eterm start_mod;
     Process* parent;
@@ -447,6 +447,8 @@ erl_system_process_otp(Eterm parent_pid, char* modname)
     parent = erts_pid2proc(NULL, 0, parent_pid, ERTS_PROC_LOCK_MAIN);
 
     so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC;
+    if (off_heap_msgq)
+        so.flags |= SPO_OFF_HEAP_MSGQ;
     res = erl_create_process(parent, start_mod, am_start, NIL, &so);
     erts_smp_proc_unlock(parent, ERTS_PROC_LOCK_MAIN);
     return res;
@@ -2326,14 +2328,14 @@ erl_start(int argc, char **argv)
 	 */
 	Eterm pid;
 
-	pid = erl_system_process_otp(otp_ring0_pid, "erts_code_purger");
+	pid = erl_system_process_otp(otp_ring0_pid, "erts_code_purger", !0);
 	erts_code_purger
 	    = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc,
 						    internal_pid_index(pid));
 	ASSERT(erts_code_purger && erts_code_purger->common.id == pid);
 	erts_proc_inc_refc(erts_code_purger); 
 
-	pid = erl_system_process_otp(otp_ring0_pid, "erts_literal_area_collector");
+	pid = erl_system_process_otp(otp_ring0_pid, "erts_literal_area_collector", !0);
 	erts_literal_area_collector
 	    = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc,
 						    internal_pid_index(pid));
@@ -2342,7 +2344,7 @@ erl_start(int argc, char **argv)
 	erts_proc_inc_refc(erts_literal_area_collector);
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-	pid = erl_system_process_otp(otp_ring0_pid, "erts_dirty_process_code_checker");
+	pid = erl_system_process_otp(otp_ring0_pid, "erts_dirty_process_code_checker", !0);
 	erts_dirty_process_code_checker
 	    = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc,
 						    internal_pid_index(pid));
-- 
cgit v1.2.3


From 0ddedf0680ae23ead8ebf7999af846ce5d722d36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 24 May 2017 15:32:31 +0200
Subject: Fix a race condition that consistently affected lc+lcnt builds

The garbage_collect message could be received while waiting for trace messages,
causing the test to crash erroneously.
---
 erts/emulator/test/gc_SUITE.erl | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/gc_SUITE.erl b/erts/emulator/test/gc_SUITE.erl
index 35dd147550..02b1927621 100644
--- a/erts/emulator/test/gc_SUITE.erl
+++ b/erts/emulator/test/gc_SUITE.erl
@@ -268,24 +268,30 @@ minor_major_gc_option_async(_Config) ->
     erlang:trace(P4, true, [garbage_collection]),
     ?assertEqual(async,
                  erlang:garbage_collect(P4, [{type, minor}, {async, Ref}])),
+    receive
+        {garbage_collect, Ref, true} ->
+            ok
+    after 10000 ->
+            ct:pal("Did not receive an async GC notification", []),
+            ?assert(false)
+    end,
     expect_trace_messages(P4, [gc_minor_start, gc_minor_end]),
     erlang:trace(P4, false, [garbage_collection]),
-    receive {garbage_collect, Ref, true} -> ok;
-        Other4 -> ct:pal("Unexpected message: ~p~n"
-                         ++ "while waiting for async gc result", [Other4])
-    after 2000 -> ?assert(false)
-    end,
     erlang:exit(P4, kill).
 
-%% Given a list of atoms, trace tags - receives messages and checks if they are
-%% trace events, and if the tag matches. Else will crash failing the test.
-expect_trace_messages(_Pid, []) -> ok;
-expect_trace_messages(Pid, [Tag | TraceTags]) ->
+%% Ensures that trace messages with the provided tags have all been received.
+%% The test fails if there's any unmatched messages left in the mailbox after
+%% all tags have been matched.
+expect_trace_messages(_Pid, []) ->
     receive
-        {trace, Pid, Tag, _Data} -> ok;
-        AnythingElse ->
-            ct:pal("Unexpected message: ~p~nWhile expected {trace, _, ~p, _}",
-                   [AnythingElse, Tag]),
+        Anything ->
+            ct:pal("Unexpected message: ~p", [Anything]),
             ?assert(false)
+    after 0 ->
+        ok
+    end;
+expect_trace_messages(Pid, [Tag | TraceTags]) ->
+    receive
+        {trace, Pid, Tag, _Data} -> ok
     end,
     expect_trace_messages(Pid, TraceTags).
-- 
cgit v1.2.3


From acd8571b35756c3fdc389807c34d80c9d44435e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 31 May 2017 12:55:40 +0200
Subject: Make sure that asynchronous replies are not lost

(First attempt to fix in 23f132d9ab776a.)

If an synchronous GC was requested by calling:

    erlang:garbage_collect(Pid, [{async,Ref}])

the reply message could in certain circumstances be
lost.

The problem is that cleanup_sys_tasks() is never called
if there are dirty tasks, but no other active system tasks.

Also shorten the long waiting times in the test case
binary_SUITE:trapping/1 to make it much more likely that
the GC have not already finished when the process is killed.
---
 erts/emulator/beam/erl_process.c    |  6 +++++-
 erts/emulator/test/binary_SUITE.erl | 14 ++++++++------
 2 files changed, 13 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index d4385e3987..285fa05d63 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -13932,7 +13932,11 @@ erts_continue_exit_process(Process *p)
 
     erts_set_gc_state(p, 1);
     state = erts_smp_atomic32_read_acqb(&p->state);
-    if (state & ERTS_PSFLG_ACTIVE_SYS) {
+    if (state & ERTS_PSFLG_ACTIVE_SYS
+#ifdef ERTS_DIRTY_SCHEDULERS
+        || p->dirty_sys_tasks
+#endif
+        ) {
 	if (cleanup_sys_tasks(p, state, CONTEXT_REDS) >= CONTEXT_REDS/2)
 	    goto yield;
     }
diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index 355be7a36d..4d17276e5c 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -1358,17 +1358,19 @@ do_trapping(N, Bif, ArgFun) ->
     io:format("N=~p: Do ~p ~s gc.\n", [N, Bif, case N rem 2 of 0 -> "with"; 1 -> "without" end]),
     Pid = spawn(?MODULE,trapping_loop,[Bif, ArgFun, 1000, self()]),
     receive ok -> ok end,
-    receive after 100 -> ok end,
     Ref = make_ref(),
     case N rem 2 of
-	0 -> erlang:garbage_collect(Pid, [{async,Ref}]),
-	     receive after 100 -> ok end;
+	0 ->
+            erlang:garbage_collect(Pid, [{async,Ref}]),
+            receive after 1 -> ok end;
 	1 -> void
     end,
-    exit(Pid,kill),
+    exit(Pid, kill),
     case N rem 2 of
-	0 -> receive {garbage_collect, Ref, _} -> ok end;
-	1 -> void
+	0 ->
+            receive {garbage_collect, Ref, _} -> ok end;
+	1 ->
+            void
     end,
     receive after 1 -> ok end,
     do_trapping(N-1, Bif, ArgFun).
-- 
cgit v1.2.3


From e86537cf39a6abe5b68bd418f3cc92b03c68b40f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 31 May 2017 12:40:19 +0200
Subject: Break out a lot of duplicated code in minor_major_gc_option_ZZZ

---
 erts/emulator/test/gc_SUITE.erl | 151 +++++++++++++++++++---------------------
 1 file changed, 73 insertions(+), 78 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/gc_SUITE.erl b/erts/emulator/test/gc_SUITE.erl
index 02b1927621..2c2cb9c32d 100644
--- a/erts/emulator/test/gc_SUITE.erl
+++ b/erts/emulator/test/gc_SUITE.erl
@@ -203,95 +203,90 @@ long_receive() ->
     end.
 
 minor_major_gc_option_self(_Config) ->
-    Endless = fun Endless() ->
-                receive
-                    {gc, Type} -> erlang:garbage_collect(self(), [{type, Type}])
-                    after 100 -> ok end,
-                Endless()
-              end,
-
-    %% Try as major, a test process will self-trigger GC
-    P1 = spawn(Endless),
-    erlang:garbage_collect(P1, []),
-    erlang:trace(P1, true, [garbage_collection]),
-    P1 ! {gc, major},
-    expect_trace_messages(P1, [gc_major_start, gc_major_end]),
-    erlang:trace(P1, false, [garbage_collection]),
-    erlang:exit(P1, kill),
-
-    %% Try as minor, a test process will self-trigger GC
-    P2 = spawn(Endless),
-    erlang:garbage_collect(P2, []),
-    erlang:trace(P2, true, [garbage_collection]),
-    P2 ! {gc, minor},
-    expect_trace_messages(P2, [gc_minor_start, gc_minor_end]),
-    erlang:trace(P2, false, [garbage_collection]),
-    erlang:exit(P2, kill).
+    %% Try as major, the test process will self-trigger GC
+    check_gc_tracing_around(
+        fun(Pid, Ref) ->
+            Pid ! {gc, Ref, major}
+        end, [gc_major_start, gc_major_end]),
+
+    %% Try as minor, the test process will self-trigger GC
+    check_gc_tracing_around(
+        fun(Pid, Ref) ->
+            Pid ! {gc, Ref, minor}
+        end, [gc_minor_start, gc_minor_end]).
 
 minor_major_gc_option_async(_Config) ->
-    Endless = fun Endless() ->
-                  receive after 100 -> ok end,
-                  Endless()
-              end,
-
-    %% Try with default option, must be major gc
-    P1 = spawn(Endless),
-    erlang:garbage_collect(P1, []),
-    erlang:trace(P1, true, [garbage_collection]),
-    erlang:garbage_collect(P1, []),
-    expect_trace_messages(P1, [gc_major_start, gc_major_end]),
-    erlang:trace(P1, false, [garbage_collection]),
-    erlang:exit(P1, kill),
+    %% Try with default option, must be major GC
+    check_gc_tracing_around(
+        fun(Pid, _Ref) ->
+            erlang:garbage_collect(Pid, [])
+        end, [gc_major_start, gc_major_end]),
 
     %% Try with the 'major' type
-    P2 = spawn(Endless),
-    erlang:garbage_collect(P2, []),
-    erlang:trace(P2, true, [garbage_collection]),
-    erlang:garbage_collect(P2, [{type, major}]),
-    expect_trace_messages(P2, [gc_major_start, gc_major_end]),
-    erlang:trace(P2, false, [garbage_collection]),
-    erlang:exit(P2, kill),
+    check_gc_tracing_around(
+        fun(Pid, _Ref) ->
+            erlang:garbage_collect(Pid, [{type, major}])
+        end, [gc_major_start, gc_major_end]),
 
     %% Try with 'minor' option, once
-    P3 = spawn(Endless),
-    erlang:garbage_collect(P3, []),
-    erlang:trace(P3, true, [garbage_collection]),
-    erlang:garbage_collect(P3, [{type, minor}]),
-    expect_trace_messages(P3, [gc_minor_start, gc_minor_end]),
-    erlang:trace(P3, false, [garbage_collection]),
-    erlang:exit(P3, kill),
+    check_gc_tracing_around(
+        fun(Pid, _Ref) ->
+            erlang:garbage_collect(Pid, [{type, minor}])
+        end, [gc_minor_start, gc_minor_end]),
 
     %% Try with 'minor' option, once, async
-    P4 = spawn(Endless),
+    check_gc_tracing_around(
+        fun(Pid, Ref) ->
+            ?assertEqual(async,
+                erlang:garbage_collect(Pid, [{type, minor}, {async, Ref}])),
+
+            receive
+                {garbage_collect, Ref, true} ->
+                    ok
+            after 10000 ->
+                ct:fail("Did not receive a completion notification on async GC")
+            end
+        end, [gc_minor_start, gc_minor_end]).
+
+%% Traces garbage collection around the given operation, and fails the test if
+%% it results in any unexpected messages or if the expected trace tags are not
+%% received.
+check_gc_tracing_around(Fun, ExpectedTraceTags) ->
     Ref = erlang:make_ref(),
-    erlang:garbage_collect(P4, []),
-    erlang:trace(P4, true, [garbage_collection]),
-    ?assertEqual(async,
-                 erlang:garbage_collect(P4, [{type, minor}, {async, Ref}])),
-    receive
-        {garbage_collect, Ref, true} ->
-            ok
-    after 10000 ->
-            ct:pal("Did not receive an async GC notification", []),
-            ?assert(false)
-    end,
-    expect_trace_messages(P4, [gc_minor_start, gc_minor_end]),
-    erlang:trace(P4, false, [garbage_collection]),
-    erlang:exit(P4, kill).
-
-%% Ensures that trace messages with the provided tags have all been received.
-%% The test fails if there's any unmatched messages left in the mailbox after
-%% all tags have been matched.
+    Pid = spawn(
+        fun Endless() ->
+            receive
+                {gc, Ref, Type} ->
+                    erlang:garbage_collect(self(), [{type, Type}])
+            after 100 ->
+                ok
+            end,
+            Endless()
+        end),
+    erlang:garbage_collect(Pid, []),
+    erlang:trace(Pid, true, [garbage_collection]),
+    Fun(Pid, Ref),
+    expect_trace_messages(Pid, ExpectedTraceTags),
+    erlang:trace(Pid, false, [garbage_collection]),
+    erlang:exit(Pid, kill),
+    check_no_unexpected_messages().
+
+%% Ensures that trace messages with the provided tags have all been received
+%% within a reasonable timeframe.
 expect_trace_messages(_Pid, []) ->
+    ok;
+expect_trace_messages(Pid, [Tag | TraceTags]) ->
+    receive
+        {trace, Pid, Tag, _Data} ->
+            expect_trace_messages(Pid, TraceTags)
+    after 4000 ->
+        ct:fail("Didn't receive tag ~p within 4000ms", [Tag])
+    end.
+
+check_no_unexpected_messages() ->
     receive
         Anything ->
-            ct:pal("Unexpected message: ~p", [Anything]),
-            ?assert(false)
+            ct:fail("Unexpected message: ~p", [Anything])
     after 0 ->
         ok
-    end;
-expect_trace_messages(Pid, [Tag | TraceTags]) ->
-    receive
-        {trace, Pid, Tag, _Data} -> ok
-    end,
-    expect_trace_messages(Pid, TraceTags).
+    end.
-- 
cgit v1.2.3


From ae8d3bc9b32e95ce6f9a2da3b6ae961f04071c36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 1 Jun 2017 10:13:51 +0200
Subject: Unconditionally clear IO buffers on send/shutdown errors

This fixes a bug where a send/shutdown error on an active-mode socket results
in the port never being properly closed.
---
 erts/emulator/drivers/common/inet_drv.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 1885338ce5..ebd13e6f05 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -10386,6 +10386,9 @@ static int tcp_send_or_shutdown_error(tcp_descriptor* desc, int err)
 	set_busy_port(desc->inet.port, 0);
     }
 
+    tcp_clear_output(desc);
+    tcp_clear_input(desc);
+
     /*
      * We used to handle "expected errors" differently from unexpected ones.
      * Now we handle all errors in the same way (unless the show_econnreset
@@ -10408,8 +10411,6 @@ static int tcp_send_or_shutdown_error(tcp_descriptor* desc, int err)
 	else
 	    desc_close(INETP(desc));
     } else {
-	tcp_clear_output(desc);
-	tcp_clear_input(desc);
 	tcp_close_check(desc);
 	erl_inet_close(INETP(desc));
 
-- 
cgit v1.2.3


From f4836a7883842f0e2ae20285ccedf9856c66b24a Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 1 Jun 2017 19:24:27 +0200
Subject: Fix check_process_code() on hibernated process

---
 erts/emulator/beam/beam_bif_load.c | 13 ++++++++++++-
 erts/emulator/beam/erl_gc.c        |  5 ++++-
 erts/emulator/beam/erl_process.h   |  1 +
 3 files changed, 17 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 0e192b1ebd..6072eaa8eb 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -808,6 +808,10 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 		if (done_gc) {
 		    return am_true;
 		} else {
+		    if (rp->flags & F_HIBERNATED) {
+			/* GC wont help; everything on heap is live... */
+			return am_true;
+		    }
 		    if (!allow_gc)
 			return am_aborted;
 		    /*
@@ -870,6 +874,8 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 	if (done_gc) {
 	    return am_true;
 	} else {
+	    int hibernated = !!(rp->flags & F_HIBERNATED);
+	    int gc_cost;
 	    Eterm* literals;
 	    Uint lit_size;
 	    struct erl_off_heap_header* oh;
@@ -886,13 +892,18 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 	    rp->ftrace = NIL;
 	    done_gc = 1;
 	    FLAGS(rp) |= F_NEED_FULLSWEEP;
-	    *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
+	    gc_cost = erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
+	    *redsp += gc_cost;
 	    literals = (Eterm *) modp->old.code[MI_LITERALS_START];
 	    lit_size = (Eterm *) modp->old.code[MI_LITERALS_END] - literals;
 	    oh = (struct erl_off_heap_header *)
 		modp->old.code[MI_LITERALS_OFF_HEAP];
 	    *redsp += lit_size / 10; /* Need, better value... */
 	    erts_garbage_collect_literals(rp, literals, lit_size, oh);
+	    if (hibernated) {
+		erts_garbage_collect_hibernate(rp);
+		*redsp += gc_cost;
+	    }
 	}
     }
     return am_false;
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 2f21111a2e..c22577a254 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -490,7 +490,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     esdp->gc_info.garbage_cols++;
     esdp->gc_info.reclaimed += reclaimed_now;
     
-    FLAGS(p) &= ~F_FORCE_GC;
+    FLAGS(p) &= ~(F_FORCE_GC|F_HIBERNATED);
 
 #ifdef CHECK_FOR_HOLES
     /*
@@ -658,6 +658,8 @@ erts_garbage_collect_hibernate(Process* p)
 
     ErtsGcQuickSanityCheck(p);
 
+    p->flags |= F_HIBERNATED;
+
     erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
 }
 
@@ -830,6 +832,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
     /*
      * Restore status.
      */
+    p->flags &= ~F_HIBERNATED;
     erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
 }
 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 10c6fa4a67..41e5f55476 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1292,6 +1292,7 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
 #define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
 #define F_DISABLE_GC         (1 << 11) /* Disable GC */
+#define F_HIBERNATED         (1 << 12) /* Hibernated */
 
 /* process trace_flags */
 #define F_SENSITIVE          (1 << 0)
-- 
cgit v1.2.3


From c8f1d8218a2aa01a2b8d4f65af6e07e5ca2b631e Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 1 Jun 2017 19:29:36 +0200
Subject: Do not GC hibernated process from other processes

---
 erts/emulator/beam/erl_process.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index d583118e7b..bce4e7fff3 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10059,7 +10059,8 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 		reds++;
 	    }
 	    else {
-		if (!garbage_collected) {
+		if (!garbage_collected
+		    && !(c_p->flags & F_HIBERNATED)) {
 		    FLAGS(c_p) |= F_NEED_FULLSWEEP;
 		    reds += erts_garbage_collect(c_p,
 						 0,
-- 
cgit v1.2.3


From de2685850a498a3d968622fd9ab23b8aca2047c7 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 1 Jun 2017 21:09:24 +0200
Subject: Update testcase to check that purge handle hibernated process correct

---
 erts/emulator/test/code_SUITE.erl | 40 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 39 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index 9f318a38be..fcb6485e7e 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -492,7 +492,38 @@ constant_pools(Config) when is_list(Config) ->
     receive
 	{'EXIT',OldHeap,{A,B,C,[1,2,3|_]=Seq}} when length(Seq) =:= 16 ->
 	    ok
-    end.
+    end,
+
+    {module,literals} = erlang:load_module(literals, Code),
+    %% Have a hibernated process that references the literals
+    %% in the 'literals' module.
+    {Hib, Mon} = spawn_monitor(fun() -> hibernated(Self) end),
+    receive go -> ok end,
+    [{heap_size,OldHeapSz},
+     {total_heap_size,OldTotHeapSz}] = process_info(Hib, [heap_size,
+							  total_heap_size]),
+    OldHeapSz = OldTotHeapSz,
+    io:format("OldHeapSz=~p OldTotHeapSz=~p~n", [OldHeapSz, OldTotHeapSz]),
+    true = erlang:delete_module(literals),
+    false = erlang:check_process_code(Hib, literals),
+    erlang:check_process_code(self(), literals),
+    erlang:purge_module(literals),
+    receive after 1000 -> ok end,
+    [{heap_size,HeapSz},
+     {total_heap_size,TotHeapSz}] = process_info(Hib, [heap_size,
+						       total_heap_size]),
+    io:format("HeapSz=~p TotHeapSz=~p~n", [HeapSz, TotHeapSz]),
+    Hib ! hej,
+    receive
+	{'DOWN', Mon, process, Hib, Reason} ->
+	    {undef, [{no_module,
+		      no_function,
+		      [{A,B,C,[1,2,3|_]=Seq}], _}]} = Reason,
+	    16 = length(Seq)
+    end,
+    HeapSz = TotHeapSz, %% Ensure restored to hibernated state...
+    true = HeapSz > OldHeapSz,
+    ok.
 
 no_old_heap(Parent) ->
     A = literals:a(),
@@ -515,6 +546,13 @@ old_heap(Parent) ->
 	    exit(Res)
     end.
 
+hibernated(Parent) ->
+    A = literals:a(),
+    B = literals:b(),
+    Res = {A,B,literals:huge_bignum(),lists:seq(1, 16)},
+    Parent ! go,
+    erlang:hibernate(no_module, no_function, [Res]).
+
 create_old_heap() ->
     case process_info(self(), [heap_size,total_heap_size]) of
 	[{heap_size,Sz},{total_heap_size,Total}] when Sz < Total ->
-- 
cgit v1.2.3


From 8f87dd1eaa80bdad501ad6f7767f14b4e0cb01d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 30 May 2017 09:40:43 +0200
Subject: Fix process_SUITE:low_prio/1

It is expected that low-priority processes should be allowed
to run about 1/8 of the time of normal-priority processes.

The proportion calculated was usually considerably lower than
1/8. It seems that the main reason is the punishement in
reductions when sending to a process that alreday has many
messages in the message queue. I have verified that by running
the emulator with the +snsp to turn off the send punishment.

To avoid that punishment, let the server process run at high
priority so that it can keep its message queue short.

It seems that the proportion is now very close to 1/8. Therefore,
we can tighten the margins for the proportion.
---
 erts/emulator/test/process_SUITE.erl | 50 ++++++++++++++++++++++--------------
 1 file changed, 31 insertions(+), 19 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index e14185e881..a65c3f0cad 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -1024,36 +1024,48 @@ bump_big(Prev, Limit) ->
 %% Priority 'low' should be mixed with 'normal' using a factor of
 %% about 8. (OTP-2644)
 low_prio(Config) when is_list(Config) ->
-    case erlang:system_info(schedulers_online) of
-	1 ->
-	    ok = low_prio_test(Config);
-	_ -> 
-	    erlang:system_flag(multi_scheduling, block_normal),
-	    ok = low_prio_test(Config),
-	    erlang:system_flag(multi_scheduling, unblock_normal),
-	    {comment,
-		   "Test not written for SMP runtime system. "
-		   "Multi scheduling blocked during test."}
-    end.
+    erlang:system_flag(multi_scheduling, block_normal),
+    Prop = low_prio_test(Config),
+    erlang:system_flag(multi_scheduling, unblock_normal),
+    Str = lists:flatten(io_lib:format("Low/high proportion is ~.3f",
+                                      [Prop])),
+    {comment,Str}.
 
 low_prio_test(Config) when is_list(Config) ->
     process_flag(trap_exit, true),
-    S = spawn_link(?MODULE, prio_server, [0, 0]),
+
+    %% Spawn the server running with high priority. The server must
+    %% not run at normal priority as that would skew the results for
+    %% two reasons:
+    %%
+    %% 1. There would be one more normal-priority processes than
+    %% low-priority processes.
+    %%
+    %% 2. The receive queue would grow faster than the server process
+    %% could process it. That would in turn trigger the reduction
+    %% punishment for the clients.
+    S = spawn_opt(?MODULE, prio_server, [0, 0], [link,{priority,high}]),
+
+    %% Spawn the clients and let them run for a while.
     PCs = spawn_prio_clients(S, erlang:system_info(schedulers_online)),
-    ct:sleep({seconds,3}),
+    ct:sleep({seconds,2}),
     lists:foreach(fun (P) -> exit(P, kill) end, PCs),
+
+    %% Stop the server and retrieve the result.
     S ! exit,
-    receive {'EXIT', S, {A, B}} -> check_prio(A, B) end,
-    ok.
+    receive
+        {'EXIT', S, {A, B}} ->
+            check_prio(A, B)
+    end.
 
 check_prio(A, B) ->
     Prop = A/B,
     ok = io:format("Low=~p, High=~p, Prop=~p\n", [A, B, Prop]),
 
-    %% It isn't 1/8, it's more like 0.3, but let's check that
-    %% the low-prio processes get some little chance to run at all.
-    true = (Prop < 1.0),
-    true = (Prop > 1/32).
+    %% Prop is expected to be appr. 1/8. Allow a reasonable margin.
+    true = Prop < 1/4,
+    true = Prop > 1/16,
+    Prop.
 
 prio_server(A, B) ->
     receive
-- 
cgit v1.2.3


From 8381b753ce94769691a6dc499f4d40373cb0504c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 30 May 2017 14:05:35 +0200
Subject: Lengthen too short timetraps

c062dfc485a added timetraps to test suites that had no timetraps.

Some of added timetraps are only 5 or 10 seconds, which is can be
too short time on some slow computers. Change those times to one
or two minutes.
---
 erts/emulator/test/bs_construct_SUITE.erl | 2 +-
 erts/emulator/test/call_trace_SUITE.erl   | 2 +-
 erts/emulator/test/ddll_SUITE.erl         | 2 +-
 erts/emulator/test/evil_SUITE.erl         | 2 +-
 erts/emulator/test/exception_SUITE.erl    | 2 +-
 erts/emulator/test/lttng_SUITE.erl        | 2 +-
 erts/emulator/test/match_spec_SUITE.erl   | 2 +-
 erts/emulator/test/nested_SUITE.erl       | 2 +-
 erts/emulator/test/port_SUITE.erl         | 2 +-
 erts/emulator/test/port_trace_SUITE.erl   | 2 +-
 erts/emulator/test/trace_SUITE.erl        | 2 +-
 erts/emulator/test/trace_port_SUITE.erl   | 2 +-
 12 files changed, 12 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl
index ed03284a5b..779d81daa5 100644
--- a/erts/emulator/test/bs_construct_SUITE.erl
+++ b/erts/emulator/test/bs_construct_SUITE.erl
@@ -35,7 +35,7 @@
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
-     {timetrap, {seconds, 10}}].
+     {timetrap, {minutes, 1}}].
 
 all() -> 
     [test1, test2, test3, test4, test5, testf, not_used,
diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl
index 95171d04ce..1251d644ae 100644
--- a/erts/emulator/test/call_trace_SUITE.erl
+++ b/erts/emulator/test/call_trace_SUITE.erl
@@ -43,7 +43,7 @@
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
-     {timetrap, {seconds, 30}}].
+     {timetrap, {minutes, 2}}].
 
 all() ->
     Common = [errors, on_load],
diff --git a/erts/emulator/test/ddll_SUITE.erl b/erts/emulator/test/ddll_SUITE.erl
index 0b9f76a892..031b05790d 100644
--- a/erts/emulator/test/ddll_SUITE.erl
+++ b/erts/emulator/test/ddll_SUITE.erl
@@ -55,7 +55,7 @@
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
-     {timetrap, {seconds, 10}}].
+     {timetrap, {minutes, 1}}].
 
 all() -> 
     [ddll_test, errors, reference_count, kill_port,
diff --git a/erts/emulator/test/evil_SUITE.erl b/erts/emulator/test/evil_SUITE.erl
index 9416ac7a02..fb1954ce37 100644
--- a/erts/emulator/test/evil_SUITE.erl
+++ b/erts/emulator/test/evil_SUITE.erl
@@ -34,7 +34,7 @@
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
-     {timetrap, {seconds, 30}}].
+     {timetrap, {minutes, 1}}].
 
 all() -> 
     [heap_frag, encode_decode_ext, decode_integer_ext,
diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl
index 76e3556bc4..ad6d8c890f 100644
--- a/erts/emulator/test/exception_SUITE.erl
+++ b/erts/emulator/test/exception_SUITE.erl
@@ -33,7 +33,7 @@
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
-     {timetrap, {seconds, 10}}].
+     {timetrap, {minutes, 1}}].
 
 all() -> 
     [badmatch, pending_errors, nil_arith, stacktrace,
diff --git a/erts/emulator/test/lttng_SUITE.erl b/erts/emulator/test/lttng_SUITE.erl
index c12f63706a..1c1952f912 100644
--- a/erts/emulator/test/lttng_SUITE.erl
+++ b/erts/emulator/test/lttng_SUITE.erl
@@ -40,7 +40,7 @@
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
-     {timetrap, {seconds, 10}}].
+     {timetrap, {minutes, 1}}].
 
 all() -> 
     [t_lttng_list,
diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl
index eb189c2c33..92ddc23592 100644
--- a/erts/emulator/test/match_spec_SUITE.erl
+++ b/erts/emulator/test/match_spec_SUITE.erl
@@ -42,7 +42,7 @@
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
-     {timetrap, {seconds, 30}}].
+     {timetrap, {minutes, 1}}].
 
 all() -> 
     case test_server:is_native(match_spec_SUITE) of
diff --git a/erts/emulator/test/nested_SUITE.erl b/erts/emulator/test/nested_SUITE.erl
index 7af2873ce2..5059317172 100644
--- a/erts/emulator/test/nested_SUITE.erl
+++ b/erts/emulator/test/nested_SUITE.erl
@@ -27,7 +27,7 @@
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
-     {timetrap, {seconds, 10}}].
+     {timetrap, {minutes, 1}}].
 
 all() -> 
     [case_in_case, case_in_after, catch_in_catch,
diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index 94ee9851dd..fccbaf13ee 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -153,7 +153,7 @@
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
-     {timetrap, {seconds, 10}}].
+     {timetrap, {minutes, 1}}].
 
 all() ->
     [otp_6224, {group, stream}, basic_ping, slow_writes,
diff --git a/erts/emulator/test/port_trace_SUITE.erl b/erts/emulator/test/port_trace_SUITE.erl
index 03efdc15db..bfc3c8cb51 100644
--- a/erts/emulator/test/port_trace_SUITE.erl
+++ b/erts/emulator/test/port_trace_SUITE.erl
@@ -52,7 +52,7 @@
 -define(ECHO_DRV_REMOTE_SEND_TERM, 15).
 
 suite() -> [{ct_hooks,[ts_install_cth]},
-            {timetrap, {seconds, 30}}].
+            {timetrap, {minutes, 2}}].
 
 all() ->
     [port_specs, ports, open_close,
diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl
index 643c2e0472..53249229c9 100644
--- a/erts/emulator/test/trace_SUITE.erl
+++ b/erts/emulator/test/trace_SUITE.erl
@@ -46,7 +46,7 @@
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
-     {timetrap, {seconds, 5}}].
+     {timetrap, {minutes, 1}}].
 
 all() -> 
     [cpu_timestamp, receive_trace, link_receive_call_correlation,
diff --git a/erts/emulator/test/trace_port_SUITE.erl b/erts/emulator/test/trace_port_SUITE.erl
index e4db368ea1..5eb27a7b68 100644
--- a/erts/emulator/test/trace_port_SUITE.erl
+++ b/erts/emulator/test/trace_port_SUITE.erl
@@ -37,7 +37,7 @@
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
-     {timetrap, {seconds, 30}}].
+     {timetrap, {minutes, 2}}].
 
 all() ->
     [call_trace, return_trace, send, receive_trace,
-- 
cgit v1.2.3


From 98fde55efd0f8c3f5013329cf3d1acfe7516ab38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 30 May 2017 14:25:34 +0200
Subject: receive_SUITE: Remove ?line macros

---
 erts/emulator/test/trace_SUITE.erl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl
index 53249229c9..bd0ea22de9 100644
--- a/erts/emulator/test/trace_SUITE.erl
+++ b/erts/emulator/test/trace_SUITE.erl
@@ -184,10 +184,10 @@ receive_trace(Config) when is_list(Config) ->
     {'EXIT', Intruder, {badarg, _}} = receive_first(),
 
     %% Untrace the process; we should not receive anything.
-    ?line 1 = erlang:trace(Receiver, false, ['receive']),
-    ?line Receiver ! {hello, there},
-    ?line Receiver ! any_garbage,
-    ?line receive_nothing(),
+    1 = erlang:trace(Receiver, false, ['receive']),
+    Receiver ! {hello, there},
+    Receiver ! any_garbage,
+    receive_nothing(),
 
     %% Verify restrictions in matchspec for 'receive'
     F3 = fun (Pat) -> {'EXIT', {badarg,_}} = (catch erlang:trace_pattern('receive', Pat, [])) end,
-- 
cgit v1.2.3


From f5c6195f7cfaea66ab3137ba5fa30f90626c8213 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 30 May 2017 16:14:32 +0200
Subject: distribution_SUITE: Refactor bulk_send_bigbig/1

Refactor bulk_send_bigbig/1 to make it somewhat easier to follow.
---
 erts/emulator/test/distribution_SUITE.erl | 122 +++++++++++++++---------------
 1 file changed, 59 insertions(+), 63 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index 434e729310..78d67ef2ae 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -62,8 +62,7 @@
 -export([sender/3, receiver2/2, dummy_waiter/0, dead_process/0,
          roundtrip/1, bounce/1, do_dist_auto_connect/1, inet_rpc_server/1,
          dist_parallel_sender/3, dist_parallel_receiver/0,
-         dist_evil_parallel_receiver/0,
-         sendersender/4, sendersender2/4]).
+         dist_evil_parallel_receiver/0]).
 
 %% epmd_module exports
 -export([start_link/0, register_node/2, register_node/3, port_please/2]).
@@ -129,33 +128,43 @@ bulk_send_small(Config) when is_list(Config) ->
 bulk_send_big(Config) when is_list(Config) ->
     bulk_send(32, 64).
 
-bulk_send_bigbig(Config) when is_list(Config) ->
-    bulk_sendsend(32*5, 4).
-
 bulk_send(Terms, BinSize) ->
     ct:timetrap({seconds, 30}),
 
     io:format("Sending ~w binaries, each of size ~w K", [Terms, BinSize]),
     {ok, Node} = start_node(bulk_receiver),
     Recv = spawn(Node, erlang, apply, [fun receiver/2, [0, 0]]),
-    Bin = list_to_binary(lists:duplicate(BinSize*1024, 253)),
+    Bin = binary:copy(<<253>>, BinSize*1024),
     Size = Terms*size(Bin),
     {Elapsed, {Terms, Size}} = test_server:timecall(?MODULE, sender,
                                                     [Recv, Bin, Terms]),
     stop_node(Node),
-    {comment, integer_to_list(trunc(Size/1024/max(1,Elapsed)+0.5)) ++ " K/s"}.
+    {comment, integer_to_list(round(Size/1024/max(1,Elapsed))) ++ " K/s"}.
+
+sender(To, _Bin, 0) ->
+    To ! {done, self()},
+    receive
+        Any ->
+            Any
+    end;
+sender(To, Bin, Left) ->
+    To ! {term, Bin},
+    sender(To, Bin, Left-1).
 
-bulk_sendsend(Terms, BinSize) ->
+bulk_send_bigbig(Config) when is_list(Config) ->
+    Terms = 32*5,
+    BinSize = 4,
     {Rate1, MonitorCount1} = bulk_sendsend2(Terms, BinSize,   5),
     {Rate2, MonitorCount2} = bulk_sendsend2(Terms, BinSize, 995),
     Ratio = if MonitorCount2 == 0 -> MonitorCount1 / 1.0;
                true               -> MonitorCount1 / MonitorCount2
             end,
-    Comment = integer_to_list(Rate1) ++ " K/s, " ++
-    integer_to_list(Rate2) ++ " K/s, " ++
-    integer_to_list(MonitorCount1) ++ " monitor msgs, " ++
-    integer_to_list(MonitorCount2) ++ " monitor msgs, " ++
-    float_to_list(Ratio) ++ " monitor ratio",
+    Comment0 = io_lib:format("~p K/s, ~p K/s, "
+                             "~p monitor msgs, ~p monitor msgs, "
+                             "~.1f monitor ratio",
+                             [Rate1,Rate2,MonitorCount1,
+                              MonitorCount2,Ratio]),
+    Comment = lists:flatten(Comment0),
     if
         %% A somewhat arbitrary ratio, but hopefully one that will
         %% accommodate a wide range of CPU speeds.
@@ -169,12 +178,11 @@ bulk_sendsend(Terms, BinSize) ->
 bulk_sendsend2(Terms, BinSize, BusyBufSize) ->
     ct:timetrap({seconds, 30}),
 
-    io:format("Sending ~w binaries, each of size ~w K",
+    io:format("\nSending ~w binaries, each of size ~w K",
               [Terms, BinSize]),
     {ok, NodeRecv} = start_node(bulk_receiver),
     Recv = spawn(NodeRecv, erlang, apply, [fun receiver/2, [0, 0]]),
-    Bin = list_to_binary(lists:duplicate(BinSize*1024, 253)),
-    %%Size = Terms*size(Bin),
+    Bin = binary:copy(<<253>>, BinSize*1024),
 
     %% SLF LEFT OFF HERE.
     %% When the caller uses small hunks, like 4k via
@@ -185,74 +193,62 @@ bulk_sendsend2(Terms, BinSize, BusyBufSize) ->
     %% default busy size and "+zdbbl 5", and if the 5 case gets
     %% "many many more" monitor messages, then we know we're working.
 
-    {ok, NodeSend} = start_node(bulk_sender, "+zdbbl " ++ integer_to_list(BusyBufSize)),
-    _Send = spawn(NodeSend, erlang, apply, [fun sendersender/4, [self(), Recv, Bin, Terms]]),
+    {ok, NodeSend} = start_node(bulk_sender, "+zdbbl " ++
+                                    integer_to_list(BusyBufSize)),
+    _Send = spawn(NodeSend, erlang, apply,
+                  [fun sendersender/4, [self(), Recv, Bin, Terms]]),
     {Elapsed, {_TermsN, SizeN}, MonitorCount} =
-    receive
-        %% On some platforms (windows), the time taken is 0 so we
-        %% simulate that some little time has passed.
-        {sendersender, {0.0,T,MC}} ->
-            {0.0015, T, MC};
-        {sendersender, BigRes} ->
-            BigRes
-    end,
+        receive
+            %% On some platforms (Windows), the time taken is 0 so we
+            %% simulate that some little time has passed.
+            {sendersender, {0.0,T,MC}} ->
+                {0.0015, T, MC};
+            {sendersender, BigRes} ->
+                BigRes
+        end,
     stop_node(NodeRecv),
     stop_node(NodeSend),
-    {trunc(SizeN/1024/Elapsed+0.5), MonitorCount}.
-
-sender(To, _Bin, 0) ->
-    To ! {done, self()},
-    receive
-        Any ->
-            Any
-    end;
-sender(To, Bin, Left) ->
-    To ! {term, Bin},
-    sender(To, Bin, Left-1).
+    {round(SizeN/1024/Elapsed), MonitorCount}.
 
 %% Sender process to be run on a slave node
 
 sendersender(Parent, To, Bin, Left) ->
     erlang:system_monitor(self(), [busy_dist_port]),
-    [spawn(fun() -> sendersender2(To, Bin, Left, false) end) ||
-     _ <- lists:seq(1,1)],
+    _ = spawn(fun() ->
+                      sendersender_send(To, Bin, Left),
+                      exit(normal)
+              end),
     {USec, {Res, MonitorCount}} =
-    timer:tc(?MODULE, sendersender2, [To, Bin, Left, true]),
+        timer:tc(fun() ->
+                         sendersender_send(To, Bin, Left),
+                         To ! {done, self()},
+                         count_monitors(0)
+                 end),
     Parent ! {sendersender, {USec/1000000, Res, MonitorCount}}.
 
-sendersender2(To, Bin, Left, SendDone) ->
-    sendersender3(To, Bin, Left, SendDone, 0).
+sendersender_send(_To, _Bin, 0) ->
+    ok;
+sendersender_send(To, Bin, Left) ->
+    To ! {term, Bin},
+    sendersender_send(To, Bin, Left-1).
 
-sendersender3(To, _Bin, 0, SendDone, MonitorCount) ->
-    if SendDone ->
-           To ! {done, self()};
-       true ->
-           ok
-    end,
+count_monitors(MonitorCount) ->
     receive
         {monitor, _Pid, _Type, _Info} ->
-            sendersender3(To, _Bin, 0, SendDone, MonitorCount + 1)
+            count_monitors(MonitorCount + 1)
     after 0 ->
-              if SendDone ->
-                     receive
-                         Any when is_tuple(Any), size(Any) == 2 ->
-                             {Any, MonitorCount}
-                     end;
-                 true ->
-                     exit(normal)
-              end
-    end;
-sendersender3(To, Bin, Left, SendDone, MonitorCount) ->
-    To ! {term, Bin},
-    %%timer:sleep(50),
-    sendersender3(To, Bin, Left-1, SendDone, MonitorCount).
+            receive
+                {_,_}=Any ->
+                    {Any,MonitorCount}
+            end
+    end.
 
 %% Receiver process to be run on a slave node.
 
 receiver(Terms, Size) ->
     receive
         {term, Bin} ->
-            receiver(Terms+1, Size+size(Bin));
+            receiver(Terms+1, Size+byte_size(Bin));
         {done, ReplyTo} ->
             ReplyTo ! {Terms, Size}
     end.
-- 
cgit v1.2.3


From 94e8b18640304bb20dc591a459802ba99dbbd39e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 30 May 2017 16:18:10 +0200
Subject: Make bulk_send_bigbig/1 purely informational

bulk_send_bigbig/1 sometimes fails even if there is nothing wrong.

Stop testing the ratio, and just return information about the speed
and number of monitor messages. The testcase will still serve as a
smoke test of the +zdbbl option.
---
 erts/emulator/test/distribution_SUITE.erl | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index 78d67ef2ae..dc7afa381b 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -165,15 +165,7 @@ bulk_send_bigbig(Config) when is_list(Config) ->
                              [Rate1,Rate2,MonitorCount1,
                               MonitorCount2,Ratio]),
     Comment = lists:flatten(Comment0),
-    if
-        %% A somewhat arbitrary ratio, but hopefully one that will
-        %% accommodate a wide range of CPU speeds.
-        Ratio > 8.0 ->
-            {comment,Comment};
-        true ->
-            io:put_chars(Comment),
-            ct:fail(ratio_too_low)
-    end.
+    {comment,Comment}.
 
 bulk_sendsend2(Terms, BinSize, BusyBufSize) ->
     ct:timetrap({seconds, 30}),
-- 
cgit v1.2.3


From 1bb171a7c27fcb79435a915bac834a4013b5f0dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 1 Jun 2017 07:29:23 +0200
Subject: Contain damage cause by spawn_opt_max_heap_size failing

If process_SUITE:spawn_opt_max_heap_size/1 failed, the default
value for the maximum heap size could have been changed. That
would cause other test cases that spawned huge processes to
fail.

Contain the damage by always restoring the default value
for max_heap_size in end_per_testcase/2.
---
 erts/emulator/test/process_SUITE.erl | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index a65c3f0cad..180311202f 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -134,6 +134,11 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     [{testcase, Func}|Config].
 
 end_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
+    %% Restore max_heap_size to default value.
+    erlang:system_flag(max_heap_size,
+                       #{size => 0,
+                         kill => true,
+                         error_logger => true}),
     ok.
 
 fun_spawn(Fun) ->
-- 
cgit v1.2.3


From ff1234e3eba1889a9b8cd81cc03f38cac78a67fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 1 Jun 2017 10:32:13 +0200
Subject: Robustify process_SUITE:spawn_opt_max_heap_size/1

process_SUITE starts os_mon in init_per_suite/1. Therefore,
there may be occasional alarm info messages received. Make
sure that they are ignored and don't cause the test fail to
fail.

Also, get rid of the flush/0 function that discards all
messages in the message queue. Instead, be more selective
and only discard {error, ...} messages.
---
 erts/emulator/test/process_SUITE.erl | 28 ++++++++++++++++++++--------
 1 file changed, 20 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index 180311202f..4204d12eb3 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -2074,6 +2074,7 @@ max_heap_size_test(Option, Size, Kill, ErrorLogger) ->
     end,
     if ErrorLogger ->
             receive
+                %% There must be at least one error message.
                 {error, _, {emulator, _, [Pid|_]}} ->
                     ok
             end;
@@ -2086,22 +2087,33 @@ max_heap_size_test(Option, Size, Kill, ErrorLogger) ->
                 {'DOWN', Ref, process, Pid, die} ->
                     ok
             end,
-            flush();
+            %% If the process was not killed, the limit may have
+            %% been reached more than once and there may be
+            %% more {error, ...} messages left.
+            receive_error_messages(Pid);
        true ->
             ok
     end,
+
+    %% Make sure that there are no unexpected messages.
+    receive_unexpected().
+
+receive_error_messages(Pid) ->
     receive
-        M ->
-            ct:fail({unexpected_message, M})
-    after 10 ->
+        {error, _, {emulator, _, [Pid|_]}} ->
+            receive_error_messages(Pid)
+    after 1000 ->
             ok
     end.
 
-flush() ->
+receive_unexpected() ->
     receive
-        _M ->
-            flush()
-    after 1000 ->
+        {info_report, _, _} ->
+            %% May be an alarm message from os_mon. Ignore.
+            receive_unexpected();
+        M ->
+            ct:fail({unexpected_message, M})
+    after 10 ->
             ok
     end.
 
-- 
cgit v1.2.3


From 6a18795542432fb3f8ba1d4306ec28d6a9250f3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 1 Jun 2017 14:51:20 +0200
Subject: Remove obsolete old_scheduler_SUITE

old_scheduler_SUITE was written before the SMP emulator.
It does not test anything that scheduler_SUITE does not
test. Keeping it wastes time when running the emulator
test suite, and does not help us find any bugs that
scheduler_SUITE would find.
---
 erts/emulator/test/Makefile                |   1 -
 erts/emulator/test/old_scheduler_SUITE.erl | 384 -----------------------------
 2 files changed, 385 deletions(-)
 delete mode 100644 erts/emulator/test/old_scheduler_SUITE.erl

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index 2479ccc01f..fcd7244ae9 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -117,7 +117,6 @@ MODULES= \
 	tracer_SUITE \
 	tracer_test \
 	scheduler_SUITE \
-	old_scheduler_SUITE \
 	port_trace_SUITE \
 	unique_SUITE \
 	z_SUITE \
diff --git a/erts/emulator/test/old_scheduler_SUITE.erl b/erts/emulator/test/old_scheduler_SUITE.erl
deleted file mode 100644
index 8515a87df8..0000000000
--- a/erts/emulator/test/old_scheduler_SUITE.erl
+++ /dev/null
@@ -1,384 +0,0 @@
-%%
-%% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 2004-2016. All Rights Reserved.
-%% 
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%% 
-%% %CopyrightEnd%
-%%
-
--module(old_scheduler_SUITE).
-
--include_lib("common_test/include/ct.hrl").
-
--export([all/0, suite/0,
-         init_per_testcase/2, end_per_testcase/2]).
--export([equal/1, many_low/1, few_low/1, max/1, high/1]).
-
-suite() ->
-    [{ct_hooks,[ts_install_cth]},
-     {timetrap, {minutes, 11}}].
-
-all() -> 
-    case catch erlang:system_info(modified_timing_level) of
-        Level when is_integer(Level) ->
-            {skipped,
-             "Modified timing (level " ++
-             integer_to_list(Level) ++
-             ") is enabled. Testcases gets messed "
-             "up by modfied timing."};
-        _ -> [equal, many_low, few_low, max, high]
-    end.
-
-
-%%-----------------------------------------------------------------------------------
-%% TEST SUITE DESCRIPTION
-%%
-%% The test case function spawns two controlling processes: Starter and Receiver.
-%% Starter spawns a number of prio A and a number of prio B test processes. Each
-%% test process loops for a number of times, sends a report to the Receiver, then
-%% loops again. For each report, the Receiver increases a counter that corresponds
-%% to the priority of the sender. After a certain amount of time, the Receiver
-%% sends the collected data to the main test process and waits for the test case
-%% to terminate. From this data, it's possible to calculate the average run time
-%% relationship between the prio A and B test processes.
-%%
-%% Note that in order to be able to run tests with high or max prio test processes, 
-%% the main test process and the Receiver needs to run at max prio, or they will
-%% be starved by the test processes. The controlling processes must not wait for
-%% messages from a normal (or low) prio process while max or high prio test processes
-%% are running (which happens e.g. if an io function is called).
-%%-----------------------------------------------------------------------------------
-
-init_per_testcase(_Case, Config) ->
-    %% main test process needs max prio
-    Prio = process_flag(priority, max),
-    MS = erlang:system_flag(multi_scheduling, block_normal),
-    [{prio,Prio},{multi_scheduling, MS}|Config].
-
-end_per_testcase(_Case, Config) ->
-    erlang:system_flag(multi_scheduling, unblock_normal),
-    Prio=proplists:get_value(prio, Config),
-    process_flag(priority, Prio),
-    ok.
-
-ok(Config) when is_list(Config) ->
-    case proplists:get_value(multi_scheduling, Config) of
-        blocked ->
-            {comment,
-             "Multi-scheduling blocked during test. This testcase was not "
-             "written to work with multiple schedulers."};
-        _ -> ok
-    end.
-
-%% Run equal number of low and normal prio processes.
-
-equal(Config) when is_list(Config) ->
-    Self = self(),
-
-    %% specify number of test processes to run
-    Normal = {normal,500},
-    Low    = {low,500},
-
-    %% specify time of test (in seconds)
-    Time = 30,
-
-    %% start controllers
-    Receiver = 
-    spawn(fun() -> receiver(erlang:monotonic_time(), Time, Self, Normal, Low) end),
-    Starter =
-    spawn(fun() -> starter(Normal, Low, Receiver) end),
-
-    %% receive test data from Receiver
-    {NRs,NAvg,LRs,LAvg,Ratio} = 
-    receive
-        {Receiver,Res} -> Res
-    end,
-
-    %% stop controllers and test processes
-    exit(Starter, kill),
-    exit(Receiver, kill),
-
-    io:format("Reports: ~w normal (~w/proc), ~w low (~w/proc). Ratio: ~w~n", 
-              [NRs,NAvg,LRs,LAvg,Ratio]),
-
-    %% runtime ratio between normal and low should be ~8
-    if Ratio < 7.5 ; Ratio > 8.5 ->	
-           ct:fail({bad_ratio,Ratio});
-       true ->
-           ok(Config)
-    end.
-
-
-%% Run many low and few normal prio processes.
-
-many_low(Config) when is_list(Config) ->
-    Self = self(),
-    Normal = {normal,1},
-    Low    = {low,1000},
-
-    %% specify time of test (in seconds)
-    Time = 30,
-
-    Receiver = 
-    spawn(fun() -> receiver(erlang:monotonic_time(), Time, Self, Normal, Low) end),
-    Starter =
-    spawn(fun() -> starter(Normal, Low, Receiver) end),
-    {NRs,NAvg,LRs,LAvg,Ratio} = 
-    receive
-        {Receiver,Res} -> Res
-    end,
-    exit(Starter, kill),
-    exit(Receiver, kill),
-    io:format("Reports: ~w normal (~w/proc), ~w low (~w/proc). Ratio: ~w~n", 
-              [NRs,NAvg,LRs,LAvg,Ratio]),
-    if Ratio < 7.5 ; Ratio > 8.5 ->
-           ct:fail({bad_ratio,Ratio});
-       true ->
-           ok(Config)
-    end.
-
-
-%% Run few low and many normal prio processes.
-
-few_low(Config) when is_list(Config) ->
-    Self = self(),
-    Normal = {normal,1000},
-    Low    = {low,1},
-
-    %% specify time of test (in seconds)
-    Time = 30,
-
-    Receiver = 
-    spawn(fun() -> receiver(erlang:monotonic_time(), Time, Self, Normal, Low) end),
-    Starter =
-    spawn(fun() -> starter(Normal, Low, Receiver) end),
-    {NRs,NAvg,LRs,LAvg,Ratio} = 
-    receive
-        {Receiver,Res} -> Res
-    end,
-    exit(Starter, kill),
-    exit(Receiver, kill),
-    io:format("Reports: ~w normal (~w/proc), ~w low (~w/proc). Ratio: ~w~n", 
-              [NRs,NAvg,LRs,LAvg,Ratio]),
-    if Ratio < 7.0 ; Ratio > 8.5 ->
-           ct:fail({bad_ratio,Ratio});
-       true ->
-           ok(Config)
-    end.
-
-
-%% Run max prio processes and verify they get at least as much 
-%% runtime as high, normal and low.
-
-max(Config) when is_list(Config) ->
-    max = process_flag(priority, max),		% should already be max (init_per_tc)
-    Self = self(),
-    Max    = {max,2},
-    High   = {high,2},
-    Normal = {normal,100},
-    Low    = {low,100},
-
-    %% specify time of test (in seconds)
-    Time = 30,
-
-    Receiver1 = 
-    spawn(fun() -> receiver(erlang:monotonic_time(), Time, Self, Max, High) end),
-    Starter1 =
-    spawn(fun() -> starter(Max, High, Receiver1) end),
-    {M1Rs,M1Avg,HRs,HAvg,Ratio1} = 
-    receive
-        {Receiver1,Res1} -> Res1
-    end,
-    exit(Starter1, kill),
-    exit(Receiver1, kill),
-    io:format("Reports: ~w max (~w/proc), ~w high (~w/proc). Ratio: ~w~n", 
-              [M1Rs,M1Avg,HRs,HAvg,Ratio1]),
-    if Ratio1 < 1.0 ->
-           ct:fail({bad_ratio,Ratio1});
-       true ->
-           ok(Config)
-    end,
-
-    Receiver2 = 
-    spawn(fun() -> receiver(erlang:monotonic_time(), Time, Self, Max, Normal) end),
-    Starter2 =
-    spawn(fun() -> starter(Max, Normal, Receiver2) end),
-    {M2Rs,M2Avg,NRs,NAvg,Ratio2} = 
-    receive
-        {Receiver2,Res2} -> Res2
-    end,
-    exit(Starter2, kill),
-    exit(Receiver2, kill),
-    io:format("Reports: ~w max (~w/proc), ~w normal (~w/proc). Ratio: ~w~n", 
-              [M2Rs,M2Avg,NRs,NAvg,Ratio2]),
-    if Ratio2 < 1.0 ->
-           ct:fail({bad_ratio,Ratio2});
-       true ->
-           ok
-    end,
-
-    Receiver3 = 
-    spawn(fun() -> receiver(erlang:monotonic_time(), Time, Self, Max, Low) end),
-    Starter3 =
-    spawn(fun() -> starter(Max, Low, Receiver3) end),
-    {M3Rs,M3Avg,LRs,LAvg,Ratio3} = 
-    receive
-        {Receiver3,Res3} -> Res3
-    end,
-    exit(Starter3, kill),
-    exit(Receiver3, kill),
-    io:format("Reports: ~w max (~w/proc), ~w low (~w/proc). Ratio: ~w~n", 
-              [M3Rs,M3Avg,LRs,LAvg,Ratio3]),
-    if Ratio3 < 1.0 ->
-           ct:fail({bad_ratio,Ratio3});
-       true ->
-           ok(Config)
-    end.
-
-
-%% Run high prio processes and verify they get at least as much 
-%% runtime as normal and low.
-
-high(Config) when is_list(Config) ->
-    max = process_flag(priority, max),		% should already be max (init_per_tc)
-    Self = self(),
-    High   = {high,2},
-    Normal = {normal,100},
-    Low    = {low,100},
-
-    %% specify time of test (in seconds)
-    Time = 30,
-
-    Receiver1 = 
-    spawn(fun() -> receiver(erlang:monotonic_time(), Time, Self, High, Normal) end),
-    Starter1 =
-    spawn(fun() -> starter(High, Normal, Receiver1) end),
-    {H1Rs,H1Avg,NRs,NAvg,Ratio1} = 
-    receive
-        {Receiver1,Res1} -> Res1
-    end,
-    exit(Starter1, kill),
-    exit(Receiver1, kill),
-    io:format("Reports: ~w high (~w/proc), ~w normal (~w/proc). Ratio: ~w~n", 
-              [H1Rs,H1Avg,NRs,NAvg,Ratio1]),
-    if Ratio1 < 1.0 ->
-           ct:fail({bad_ratio,Ratio1});
-       true ->
-           ok
-    end,
-
-    Receiver2 = 
-    spawn(fun() -> receiver(erlang:monotonic_time(), Time, Self, High, Low) end),
-    Starter2 =
-    spawn(fun() -> starter(High, Low, Receiver2) end),
-    {H2Rs,H2Avg,LRs,LAvg,Ratio2} = 
-    receive
-        {Receiver2,Res2} -> Res2
-    end,
-    exit(Starter2, kill),
-    exit(Receiver2, kill),
-    io:format("Reports: ~w high (~w/proc), ~w low (~w/proc). Ratio: ~w~n", 
-              [H2Rs,H2Avg,LRs,LAvg,Ratio2]),
-    if Ratio2 < 1.0 ->
-           ct:fail({bad_ratio,Ratio2});
-       true ->
-           ok(Config)
-    end.
-
-
-%%-----------------------------------------------------------------------------------
-%% Controller processes and help functions
-%%-----------------------------------------------------------------------------------
-
-receiver(T0, TimeSec, Main, {P1,P1N}, {P2,P2N}) ->
-    %% prio should be max so that mailbox doesn't overflow
-    process_flag(priority, max),
-    receiver(T0, TimeSec*1000, Main, P1,P1N,0, P2,P2N,0, 100000).
-
-%% uncomment lines below to get life sign (debug)
-receiver(T0, Time, Main, P1,P1N,P1Rs, P2,P2N,P2Rs, 0) ->
-    %    T = erlang:convert_time_unit(erlang:monotonic_time() - T0, native, millisecond),
-    %    erlang:display({round(T/1000),P1Rs,P2Rs}),
-    receiver(T0, Time, Main, P1,P1N,P1Rs, P2,P2N,P2Rs, 100000);
-
-receiver(T0, Time, Main, P1,P1N,P1Rs, P2,P2N,P2Rs, C) ->
-    Remain = Time - erlang:convert_time_unit(erlang:monotonic_time() - T0,
-                                             native, millisecond), % test time remaining
-    Remain1 = if Remain < 0 ->
-                     0;
-                 true ->
-                     Remain
-              end,
-    {P1Rs1,P2Rs1} = 
-    receive
-        {_Pid,P1} ->			% report from a P1 process
-            {P1Rs+1,P2Rs};
-        {_Pid,P2} ->			% report from a P2 process
-            {P1Rs,P2Rs+1}
-    after Remain1 ->
-              {P1Rs,P2Rs}
-    end,
-    if Remain > 0 ->				% keep going
-           receiver(T0, Time, Main, P1,P1N,P1Rs1, P2,P2N,P2Rs1, C-1);
-       true ->					% finish
-           %% calculate results and send to main test process
-           P1Avg = P1Rs1/P1N,
-           P2Avg = P2Rs1/P2N,
-           Ratio = if P2Avg < 1.0 -> P1Avg;
-                      true -> P1Avg/P2Avg
-                   end,
-           Main ! {self(),{P1Rs1,round(P1Avg),P2Rs1,round(P2Avg),Ratio}},
-           flush_loop()
-    end.
-
-starter({P1,P1N}, {P2,P2N}, Receiver) ->
-    %% start N1 processes with prio P1
-    start_p(P1, P1N, Receiver),
-    %% start N2 processes with prio P2
-    start_p(P2, P2N, Receiver),
-    erlang:display({started,P1N+P2N}),
-    flush_loop().
-
-start_p(_, 0, _) ->
-    ok;
-start_p(Prio, N, Receiver) ->
-    spawn_link(fun() -> p(Prio, Receiver) end),
-    start_p(Prio, N-1, Receiver).
-
-p(Prio, Receiver) ->
-    %% set process priority
-    process_flag(priority, Prio),
-    p_loop(0, Prio, Receiver).
-
-p_loop(100, Prio, Receiver) ->
-    receive after 0 -> ok end,
-    %% if Receiver gone, we're done
-    case is_process_alive(Receiver) of
-        false -> exit(bye);
-        true -> ok
-    end,
-    %% send report
-    Receiver ! {self(),Prio},
-    p_loop(0, Prio, Receiver);
-
-p_loop(N, Prio, Receiver) ->
-    p_loop(N+1, Prio, Receiver).
-
-
-flush_loop() ->
-    receive _ ->
-                ok
-    end,
-    flush_loop().
-- 
cgit v1.2.3


From 7aa347e1f911f1bcef026c08e32cc4eed280ce38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 1 Jun 2017 15:16:00 +0200
Subject: same_time_yielding/1: Avoid failing if there are many schedulers

On a computer with 32 schedulers, there would be 9632 (301*32)
'timeout' message in the receive queue. Receiving them with a
selective receive (matching on the timer ID) is quite slow.

Change the test case to read out the queue in the order the
messages are stored in the queue.
---
 erts/emulator/test/timer_bif_SUITE.erl | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/timer_bif_SUITE.erl b/erts/emulator/test/timer_bif_SUITE.erl
index 7cbd93a0f3..a977eb41c4 100644
--- a/erts/emulator/test/timer_bif_SUITE.erl
+++ b/erts/emulator/test/timer_bif_SUITE.erl
@@ -488,24 +488,40 @@ registered_process(Config) when is_list(Config) ->
 
 same_time_yielding(Config) when is_list(Config) ->
     Mem = mem(),
+    Ref = make_ref(),
     SchdlrsOnln = erlang:system_info(schedulers_online),
     Tmo = erlang:monotonic_time(millisecond) + 3000,
     Tmrs = lists:map(fun (I) ->
                              process_flag(scheduler, (I rem SchdlrsOnln) + 1),
-                             erlang:start_timer(Tmo, self(), hej, [{abs, true}])
+                             erlang:start_timer(Tmo, self(), Ref, [{abs, true}])
                      end,
                      lists:seq(1, (?TIMEOUT_YIELD_LIMIT*3+1)*SchdlrsOnln)),
     true = mem_larger_than(Mem),
-    lists:foreach(fun (Tmr) -> receive {timeout, Tmr, hej} -> ok end end, Tmrs),
+    receive_all_timeouts(length(Tmrs), Ref),
     Done = erlang:monotonic_time(millisecond),
     true = Done >= Tmo,
+    MsAfterTmo = Done - Tmo,
+    io:format("Done ~p ms after Tmo\n", [MsAfterTmo]),
     case erlang:system_info(build_type) of
-        opt -> true = Done < Tmo + 200;
-        _ -> true = Done < Tmo + 1000
+        opt ->
+            true = MsAfterTmo < 200;
+        _ ->
+            true = MsAfterTmo < 1000
     end,
     Mem = mem(),
     ok.
 
+%% Read out all timeouts in receive queue order. This is efficient
+%% even if there are very many messages.
+
+receive_all_timeouts(0, _Ref) ->
+    ok;
+receive_all_timeouts(N, Ref) ->
+    receive
+        {timeout, _Tmr, Ref} ->
+            receive_all_timeouts(N-1, Ref)
+    end.
+
 same_time_yielding_with_cancel(Config) when is_list(Config) ->
     same_time_yielding_with_cancel_test(false, false).
 
-- 
cgit v1.2.3


From 46b62dea54a230e46bbadfec7f119ff5ab513e6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 2 Jun 2017 11:41:48 +0200
Subject: Stabilize call_with_huge_message_queue/1

Time measurements are always tricky, particulary on virtual hosts.
On one particular virtual host, the measured times were often 0.

Measuring the time for 100 calls instead of 10 calls helps, but
0 can still be returned, so we will also need to discard
measurements that return 0 and try again.
---
 erts/emulator/test/receive_SUITE.erl | 55 +++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 20 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/receive_SUITE.erl b/erts/emulator/test/receive_SUITE.erl
index 83653a7a36..c7d5a3f5a0 100644
--- a/erts/emulator/test/receive_SUITE.erl
+++ b/erts/emulator/test/receive_SUITE.erl
@@ -39,25 +39,43 @@ groups() ->
 
 call_with_huge_message_queue(Config) when is_list(Config) ->
     Pid = spawn_link(fun echo_loop/0),
-
-    {Time,ok} = tc(fun() -> calls(10, Pid) end),
-
-    [self() ! {msg,N} || N <- lists:seq(1, 500000)],
+    _WarmUpTime = time_calls(Pid),
+    Time = time_calls(Pid),
+    _ = [self() ! {msg,N} || N <- lists:seq(1, 500000)],
+    io:format("Time for empty message queue: ~p", [Time]),
     erlang:garbage_collect(),
-    {NewTime1,ok} = tc(fun() -> calls(10, Pid) end),
-    {NewTime2,ok} = tc(fun() -> calls(10, Pid) end),
+    call_with_huge_message_queue_1(Pid, Time, 5).
+
+call_with_huge_message_queue_1(_Pid, _Time, 0) ->
+    ct:fail(bad_ratio);
+call_with_huge_message_queue_1(Pid, Time, NumTries) ->
+    HugeTime = time_calls(Pid),
+    io:format("Time for huge message queue: ~p", [HugeTime]),
+
+    case (HugeTime+1) / (Time+1) of
+        Q when Q < 10 ->
+            ok;
+        Q ->
+            io:format("Too high ratio: ~p\n", [Q]),
+            call_with_huge_message_queue_1(Pid, Time, NumTries-1)
+    end.
 
-    io:format("Time for empty message queue: ~p", [Time]),
-    io:format("Time1 for huge message queue: ~p", [NewTime1]),
-    io:format("Time2 for huge message queue: ~p", [NewTime2]),
-
-    case hd(lists:sort([(NewTime1+1) / (Time+1), (NewTime2+1) / (Time+1)])) of
-	Q when Q < 10 ->
-	    ok;
-	Q ->
-	    ct:fail("Best Q = ~p", [Q])
-    end,
-    ok.
+%% Time a number calls. Try to avoid returning a zero time.
+time_calls(Pid) ->
+    time_calls(Pid, 10).
+
+time_calls(_Pid, 0) ->
+    0;
+time_calls(Pid, NumTries) ->
+    case timer:tc(fun() -> calls(Pid) end) of
+        {0,ok} ->
+            time_calls(Pid, NumTries-1);
+        {Time,ok} ->
+            Time
+    end.
+
+calls(Pid) ->
+    calls(100, Pid).
 
 calls(0, _) -> ok;
 calls(N, Pid) ->
@@ -108,6 +126,3 @@ echo_loop() ->
 	    Pid ! {Ref,Msg},
 	    echo_loop()
     end.
-
-tc(Fun) ->
-    timer:tc(erlang, apply, [Fun,[]]).
-- 
cgit v1.2.3


From 8c807c89fd93ac60137c72ce597759b697ac0ec7 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 5 Jun 2017 15:55:31 +0200
Subject: erts: Fix so that 81b628 (sigterm=kill) works

OTP-14451
---
 erts/emulator/sys/unix/sys.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index de8481b206..7f738751ff 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -838,11 +838,13 @@ void sys_init_suspend_handler(void)
 void
 erts_sys_unix_later_init(void)
 {
-    char env[4];
+    char env[5];
     size_t envsz = sizeof(env);
+
     if (erts_sys_getenv_raw("ERL_ZZ_SIGTERM_KILL", env, &envsz) == 0)
-        if (envsz == 4 && sys_strncmp("true",env,4) == 0)
+        if (envsz == 4 && sys_strncmp("true", env, 4) == 0)
             return;
+
     sys_signal(SIGTERM, request_stop);
 }
 
-- 
cgit v1.2.3


From 4e38e959a1ecfaa5fa5f3571599a60ab0a88c712 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 22 May 2017 11:25:20 +0200
Subject: erts: Fix sendfile closeduring scenario on sunos

On Solaris, giving a too long sfv_len results in an
EINVAL error, but data is still transmitted and len is
correctly. So we translate this to a success with that
amount of data sent. This may hide some other errors
that causes EINVAL, but it is the best we can do for now.
---
 erts/emulator/drivers/unix/unix_efile.c | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c
index 0acc2432a7..f8341f788a 100644
--- a/erts/emulator/drivers/unix/unix_efile.c
+++ b/erts/emulator/drivers/unix/unix_efile.c
@@ -969,17 +969,21 @@ efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd,
 	fdrec.sfv_len = SENDFILE_CHUNK_SIZE;
       else
 	fdrec.sfv_len = *nbytes;
+
       retval = sendfilev(out_fd, &fdrec, 1, &len);
 
-      /* Sometimes sendfilev can return -1 and still send data. 
-         When that happens we just pretend that no error happend. */
-      if (retval != -1 || errno == EAGAIN || errno == EINTR ||
-	  len != 0) {
+      if (retval == -1 && errno == EINVAL) {
+          /* On some solaris versions (I've seen it on SunOS 5.10),
+             using a sfv_len larger then a filesize will result in
+             a -1 && errno == EINVAL return. We translate this so
+             a successful send of the data.*/
+          retval = len;
+      }
+
+      if (retval != -1 || errno == EAGAIN || errno == EINTR) {
         *offset += len;
 	*nbytes -= len;
 	written += len;
-	if (errno != EAGAIN && errno != EINTR && len != 0)
-	  retval = len;
       }
     } while (len == SENDFILE_CHUNK_SIZE);
 #elif defined(__DARWIN__)
-- 
cgit v1.2.3


From e7377ce9f0a6a37973baca55d3e4ee27380b9313 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 12 Jun 2017 16:21:18 +0200
Subject: after_SUITE: Don't leave a process running

---
 erts/emulator/test/after_SUITE.erl | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/after_SUITE.erl b/erts/emulator/test/after_SUITE.erl
index b1f7e06bf5..6409f0b336 100644
--- a/erts/emulator/test/after_SUITE.erl
+++ b/erts/emulator/test/after_SUITE.erl
@@ -45,13 +45,15 @@ all() ->
 
 %% Tests for an old round-off error in 'receive after'."
 t_after(Config) when is_list(Config) ->
-    spawn(fun frequent_process/0),
+    Frequent = spawn_link(fun frequent_process/0),
     Period = test_server:minutes(1),
     Before = erlang:monotonic_time(),
     receive
     after Period ->
-                After = erlang:monotonic_time(),
-                report(Period, Before, After)
+            After = erlang:monotonic_time(),
+            unlink(Frequent),
+            exit(Frequent, die),
+            report(Period, Before, After)
     end.
 
 report(Period, Before, After) ->
-- 
cgit v1.2.3


From 6b25d2451a870a13965e0b21dd99efe115db4c8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 12 Jun 2017 15:49:52 +0200
Subject: busy_port_SUITE: Eliminate 'export_all'

---
 erts/emulator/test/busy_port_SUITE.erl | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/busy_port_SUITE.erl b/erts/emulator/test/busy_port_SUITE.erl
index 7094cee992..ce643872e5 100644
--- a/erts/emulator/test/busy_port_SUITE.erl
+++ b/erts/emulator/test/busy_port_SUITE.erl
@@ -24,14 +24,15 @@
 	 io_to_busy/1, message_order/1, send_3/1, 
 	 system_monitor/1, no_trap_exit/1,
 	 no_trap_exit_unlinked/1, trap_exit/1, multiple_writers/1,
-	 hard_busy_driver/1, soft_busy_driver/1]).
-
--compile(export_all).
+	 hard_busy_driver/1, soft_busy_driver/1,
+         scheduling_delay_busy/1,
+         scheduling_delay_busy_nosuspend/1,
+         scheduling_busy_link/1]).
 
 -include_lib("common_test/include/ct.hrl").
 
 %% Internal exports.
--export([init/2]).
+-export([init/2,process_init/2,ack/2,call/2,cast/2]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
-- 
cgit v1.2.3


From c7b5246935a924db24610edf8201c5a553206f87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 12 Jun 2017 15:51:37 +0200
Subject: busy_port_SUITE: Eliminate warnings for unused variables

---
 erts/emulator/test/busy_port_SUITE.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/busy_port_SUITE.erl b/erts/emulator/test/busy_port_SUITE.erl
index ce643872e5..b1600c31bb 100644
--- a/erts/emulator/test/busy_port_SUITE.erl
+++ b/erts/emulator/test/busy_port_SUITE.erl
@@ -135,7 +135,7 @@ message_order(Config) when is_list(Config) ->
     ok.
 
 send_to_busy_1(Parent) ->
-    {Owner, Slave} = get_slave(),
+    {_Owner, Slave} = get_slave(),
     (catch port_command(Slave, "set_me_busy")),
     (catch port_command(Slave, "hello")),
     (catch port_command(Slave, "hello again")),
@@ -344,7 +344,7 @@ multiple_writers(Config) when is_list(Config) ->
     ok.
 
 quick_writer() ->
-    {Owner, Port} = get_slave(),
+    {_Owner, Port} = get_slave(),
     (catch port_command(Port, "port to busy")),
     (catch port_command(Port, "lock me")),
     ok.
-- 
cgit v1.2.3


From c75bc0ef433caa42cebcd8e3c8dd7a5f0c4fd408 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 12 Jun 2017 16:52:10 +0200
Subject: busy_port_SUITE: Ensure that all created procesesses are killed

---
 erts/emulator/test/busy_port_SUITE.erl | 42 ++++++++++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/busy_port_SUITE.erl b/erts/emulator/test/busy_port_SUITE.erl
index b1600c31bb..644848daf5 100644
--- a/erts/emulator/test/busy_port_SUITE.erl
+++ b/erts/emulator/test/busy_port_SUITE.erl
@@ -20,7 +20,7 @@
 
 -module(busy_port_SUITE).
 
--export([all/0, suite/0, end_per_testcase/2,
+-export([all/0, suite/0, init_per_testcase/2, end_per_testcase/2,
 	 io_to_busy/1, message_order/1, send_3/1, 
 	 system_monitor/1, no_trap_exit/1,
 	 no_trap_exit_unlinked/1, trap_exit/1, multiple_writers/1,
@@ -45,6 +45,11 @@ all() ->
      scheduling_delay_busy,scheduling_delay_busy_nosuspend,
      scheduling_busy_link].
 
+init_per_testcase(_Case, Config) when is_list(Config) ->
+    Killer = spawn(fun() -> killer_loop([]) end),
+    register(killer_process, Killer),
+    Config.
+
 end_per_testcase(_Case, Config) when is_list(Config) ->
     case whereis(busy_drv_server) of
 	undefined ->
@@ -58,8 +63,38 @@ end_per_testcase(_Case, Config) when is_list(Config) ->
 		    ok
 	    end
     end,
+    kill_processes(),
     Config.
 
+kill_processes() ->
+    killer_process ! {get_pids,self()},
+    receive
+        {pids_to_kill,Pids} -> ok
+    end,
+    _ = [begin
+             case erlang:is_process_alive(P) of
+                 true ->
+                     io:format("Killing ~p\n", [P]);
+                 false ->
+                     ok
+             end,
+             unlink(P),
+             exit(P, kill)
+         end || P <- Pids],
+    ok.
+
+killer_loop(Pids) ->
+    receive
+        {add_pid,Pid} ->
+            killer_loop([Pid|Pids]);
+        {get_pids,To} ->
+            To ! {pids_to_kill,Pids}
+    end.
+
+kill_me(Pid) ->
+    killer_process ! {add_pid,Pid},
+    Pid.
+
 %% Tests I/O operations to a busy port, to make sure a suspended send
 %% operation is correctly restarted.  This used to crash Beam.
 
@@ -713,6 +748,7 @@ run_scenario([],Vars) ->
 
 run_command(_M,spawn,{Args,Opts}) ->
     Pid = spawn_opt(fun() -> apply(?MODULE,process_init,Args) end,[link|Opts]),
+    kill_me(Pid),
     pal("spawn(~p): ~p",[Args,Pid]),
     Pid;
 run_command(M,spawn,Args) ->
@@ -808,7 +844,9 @@ fun_spawn(Fun) ->
     fun_spawn(Fun, []).
 
 fun_spawn(Fun, Args) ->
-    spawn_link(erlang, apply, [Fun, Args]).
+    Pid = spawn_link(erlang, apply, [Fun, Args]),
+    kill_me(Pid),
+    Pid.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% These routines provide a port which will become busy when the
-- 
cgit v1.2.3


From 0725fe8296204a61f3dbc57142bfd458a45dd547 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 13 Jun 2017 07:10:39 +0200
Subject: Add informational test case z_SUITE:leaked_processes/1

Add z_SUITE:leaked_processes/1 to print the process information for
all new processes created during execution of the emulator test
suite. Test cases are not supposed leak processes, because that
could disturb later test cases.
---
 erts/emulator/test/a_SUITE.erl | 48 ++++++++++++++++++++++++++++++++++--------
 erts/emulator/test/z_SUITE.erl | 33 +++++++++++++++++++++++++++--
 2 files changed, 70 insertions(+), 11 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/a_SUITE.erl b/erts/emulator/test/a_SUITE.erl
index 880c5a5821..07922f9942 100644
--- a/erts/emulator/test/a_SUITE.erl
+++ b/erts/emulator/test/a_SUITE.erl
@@ -29,25 +29,55 @@
 
 -include_lib("common_test/include/ct.hrl").
 
--export([all/0, suite/0,
-	 long_timers/1, pollset_size/1]).
+-export([all/0, suite/0, init_per_suite/1, end_per_suite/1,
+	 leaked_processes/1, long_timers/1, pollset_size/1]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]}].
 
-all() -> 
-    [long_timers, pollset_size].
+all() ->
+    [leaked_processes, long_timers, pollset_size].
+
+%% Start some system servers now to avoid having them
+%% reported as leaks.
+
+init_per_suite(Config) when is_list(Config) ->
+    %% Ensure inet_gethost_native port program started, in order to
+    %% allow other suites to use it...
+    inet_gethost_native:gethostbyname("localhost"),
+
+    %% Start the timer server.
+    timer:start(),
+
+    Config.
+
+end_per_suite(Config) when is_list(Config) ->
+    Config.
+
+leaked_processes(Config) when is_list(Config) ->
+    Parent = self(),
+    Go = make_ref(),
+    spawn(fun () ->
+                  Name = leaked_processes__process_holder,
+                  true = register(Name, self()),
+                  Ps = processes(),
+                  Parent ! Go,
+                  receive
+                      {get_initial_processes, Pid} ->
+                          Pid ! {initial_processes, Ps}
+                  end
+          end),
+    receive Go -> ok end,
+    {comment, "Testcase started! This test will run in parallel with the "
+     "erts testsuite and ends in the z_SUITE:leaked_processes/1 testcase."}.
 
 long_timers(Config) when is_list(Config) ->
     Dir = proplists:get_value(data_dir, Config),
     long_timers_test:start(Dir),
     {comment, "Testcase started! This test will run in parallel with the "
-              "erts testsuite and ends in the z_SUITE:long_timers testcase."}.
+     "erts testsuite and ends in the z_SUITE:long_timers/1 testcase."}.
 
 pollset_size(Config) when is_list(Config) ->
-    %% Ensure inet_gethost_native port program started, in order to
-    %% allow other suites to use it...
-    inet_gethost_native:gethostbyname("localhost"),
     Parent = self(),
     Go = make_ref(),
     spawn(fun () ->
@@ -63,7 +93,7 @@ pollset_size(Config) when is_list(Config) ->
           end),
     receive Go -> ok end,
     {comment, "Testcase started! This test will run in parallel with the "
-              "erts testsuite and ends in the z_SUITE:pollset_size testcase."}.
+     "erts testsuite and ends in the z_SUITE:pollset_size/1 testcase."}.
 
 %%
 %% Internal functions...
diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl
index a2b267543f..feea7432a9 100644
--- a/erts/emulator/test/z_SUITE.erl
+++ b/erts/emulator/test/z_SUITE.erl
@@ -36,7 +36,8 @@
 
 -export([schedulers_alive/1, node_container_refc_check/1,
 	 long_timers/1, pollset_size/1,
-	 check_io_debug/1, get_check_io_info/0]).
+	 check_io_debug/1, get_check_io_info/0,
+         leaked_processes/1]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
@@ -44,7 +45,10 @@ suite() ->
 
 all() -> 
     [schedulers_alive, node_container_refc_check,
-     long_timers, pollset_size, check_io_debug].
+     long_timers, pollset_size, check_io_debug,
+     %% Make sure that the leaked_processes/1 is always
+     %% run last.
+     leaked_processes].
 
 %%%
 %%% The test cases -------------------------------------------------------------
@@ -285,6 +289,31 @@ has_gethost([P|T]) ->
 has_gethost([]) ->
     false.
 
+leaked_processes(Config) when is_list(Config) ->
+    %% Replace the defualt timetrap with a timetrap with
+    %% known pid.
+    test_server:timetrap_cancel(),
+    Dog = test_server:timetrap(test_server:minutes(5)),
+
+    Name = leaked_processes__process_holder,
+    Name ! {get_initial_processes, self()},
+    receive
+        {initial_processes, Initial0} -> ok
+    end,
+    Initial = ordsets:from_list(Initial0),
+
+    KnownPids = ordsets:from_list([self(),Dog]),
+    Now0 = ordsets:from_list(processes()),
+    Now = ordsets:subtract(Now0, KnownPids),
+    Leaked = ordsets:subtract(Now, Initial),
+
+    _ = [begin
+             Info = process_info(P) ++ process_info(P, [current_stacktrace]),
+             io:format("~p: ~p\n", [P,Info])
+         end || P <- Leaked],
+    Comment = lists:flatten(io_lib:format("~p process(es)",
+                                          [length(Leaked)])),
+    {comment, Comment}.
 
 %%
 %% Internal functions...
-- 
cgit v1.2.3


From a54014638c6a1cd0a3e7b1ee36998756a475684b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 13 Jun 2017 10:00:15 +0200
Subject: message_queue_data_SUITE: Don't leave processes running

---
 erts/emulator/test/message_queue_data_SUITE.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/message_queue_data_SUITE.erl b/erts/emulator/test/message_queue_data_SUITE.erl
index e084b9482d..0fd86155fb 100644
--- a/erts/emulator/test/message_queue_data_SUITE.erl
+++ b/erts/emulator/test/message_queue_data_SUITE.erl
@@ -169,7 +169,7 @@ total_heap_size(_Config) ->
     Fun = fun F() -> receive Pid when is_pid(Pid) -> Pid ! ok,F() end end,
 
     %% Test that on_heap messages grow the heap even if they are not received
-    OnPid = spawn_opt(Fun, [{message_queue_data, on_heap}]),
+    OnPid = spawn_opt(Fun, [{message_queue_data, on_heap},link]),
     {total_heap_size, OnSize} = erlang:process_info(OnPid, total_heap_size),
     [OnPid ! lists:duplicate(N,N) || N <- lists:seq(1,100)],
     OnPid ! self(), receive ok -> ok end,
@@ -178,7 +178,7 @@ total_heap_size(_Config) ->
     true = OnSize < OnSizeAfter,
 
     %% Test that off_heap messages do not grow the heap if they are not received
-    OffPid = spawn_opt(Fun, [{message_queue_data, off_heap}]),
+    OffPid = spawn_opt(Fun, [{message_queue_data, off_heap},link]),
     {total_heap_size, OffSize} = erlang:process_info(OffPid, total_heap_size),
     [OffPid ! lists:duplicate(N,N) || N <- lists:seq(1,100)],
     OffPid ! self(), receive ok -> ok end,
-- 
cgit v1.2.3


From 23c436e47f3f28b8797626a113361611631e7bf0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 13 Jun 2017 10:04:10 +0200
Subject: trace_SUITE: Don't leave processes running

---
 erts/emulator/test/trace_SUITE.erl | 31 ++++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl
index bd0ea22de9..9ddc8628b6 100644
--- a/erts/emulator/test/trace_SUITE.erl
+++ b/erts/emulator/test/trace_SUITE.erl
@@ -24,7 +24,8 @@
 %%% Tests the trace BIF.
 %%%
 
--export([all/0, suite/0, link_receive_call_correlation/0,
+-export([all/0, suite/0, init_per_testcase/2, end_per_testcase/2,
+         link_receive_call_correlation/0,
          receive_trace/1, link_receive_call_correlation/1, self_send/1,
 	 timeout_trace/1, send_trace/1,
 	 procs_trace/1, dist_procs_trace/1, procs_new_trace/1,
@@ -62,6 +63,14 @@ all() ->
      system_monitor_long_schedule,
      system_monitor_large_heap_2, bad_flag, trace_delivered].
 
+init_per_testcase(_Case, Config) ->
+    [{receiver,spawn(fun receiver/0)}|Config].
+
+end_per_testcase(_Case, Config) ->
+    Receiver = proplists:get_value(receiver, Config),
+    unlink(Receiver),
+    exit(Receiver, die),
+    ok.
 
 %% No longer testing anything, just reporting whether cpu_timestamp
 %% is enabled or not.
@@ -83,7 +92,7 @@ cpu_timestamp(Config) when is_list(Config) ->
 %% Tests that trace(Pid, How, ['receive']) works.
 
 receive_trace(Config) when is_list(Config) ->
-    Receiver = fun_spawn(fun receiver/0),
+    Receiver = proplists:get_value(receiver, Config),
 
     %% Trace the process; make sure that we receive the trace messages.
     1 = erlang:trace(Receiver, true, ['receive']),
@@ -353,7 +362,7 @@ timeout_trace(Config) when is_list(Config) ->
 send_trace(Config) when is_list(Config) ->
     process_flag(trap_exit, true),
     Sender = fun_spawn(fun sender/0),
-    Receiver = fun_spawn(fun receiver/0),
+    Receiver = proplists:get_value(receiver, Config),
 
     %% Check that a message sent to another process is traced.
     1 = erlang:trace(Sender, true, [send]),
@@ -1604,7 +1613,8 @@ suspend_waiting(Config) when is_list(Config) ->
 
 %% Test that erlang:trace(new, true, ...) is cleared when tracer dies.
 new_clear(Config) when is_list(Config) ->
-    Tracer = spawn(fun receiver/0),
+    Tracer = proplists:get_value(receiver, Config),
+
     0 = erlang:trace(new, true, [send, {tracer, Tracer}]),
     {flags, [send]} = erlang:trace_info(new, flags),
     {tracer, Tracer} = erlang:trace_info(new, tracer),
@@ -1623,7 +1633,7 @@ new_clear(Config) when is_list(Config) ->
 existing_clear(Config) when is_list(Config) ->
     Self = self(),
 
-    Tracer = fun_spawn(fun receiver/0),
+    Tracer = proplists:get_value(receiver, Config),
     N = erlang:trace(existing, true, [send, {tracer, Tracer}]),
     {flags, [send]} = erlang:trace_info(Self, flags),
     {tracer, Tracer} = erlang:trace_info(Self, tracer),
@@ -1639,27 +1649,30 @@ existing_clear(Config) when is_list(Config) ->
 %% Test that erlang:trace/3 can be called on processes where the
 %% tracer has died. OTP-13928
 tracer_die(Config) when is_list(Config) ->
-    Proc = spawn(fun receiver/0),
+    Proc = spawn_link(fun receiver/0),
 
-    Tracer = spawn(fun receiver/0),
+    Tracer = spawn_link(fun receiver/0),
     timer:sleep(1),
     N = erlang:trace(existing, true, [send, {tracer, Tracer}]),
     {flags, [send]} = erlang:trace_info(Proc, flags),
     {tracer, Tracer} = erlang:trace_info(Proc, tracer),
+    unlink(Tracer),
     exit(Tracer, die),
 
-    Tracer2 = spawn(fun receiver/0),
+    Tracer2 = spawn_link(fun receiver/0),
     timer:sleep(1),
     N = erlang:trace(existing, true, [send, {tracer, Tracer2}]),
     {flags, [send]} = erlang:trace_info(Proc, flags),
     {tracer, Tracer2} = erlang:trace_info(Proc, tracer),
+    unlink(Tracer2),
     exit(Tracer2, die),
 
-    Tracer3 = spawn(fun receiver/0),
+    Tracer3 = spawn_link(fun receiver/0),
     timer:sleep(1),
     1 = erlang:trace(Proc, true, [send, {tracer, Tracer3}]),
     {flags, [send]} = erlang:trace_info(Proc, flags),
     {tracer, Tracer3} = erlang:trace_info(Proc, tracer),
+    unlink(Tracer3),
     exit(Tracer3, die),
 
     ok.
-- 
cgit v1.2.3


From 23e3855559445500f109798373312830d6e746f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 13 Jun 2017 10:07:46 +0200
Subject: trace_bif_SUITE: Don't leave processes running

---
 erts/emulator/test/trace_bif_SUITE.erl | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/trace_bif_SUITE.erl b/erts/emulator/test/trace_bif_SUITE.erl
index f60c777ba1..ce9c60d9d9 100644
--- a/erts/emulator/test/trace_bif_SUITE.erl
+++ b/erts/emulator/test/trace_bif_SUITE.erl
@@ -47,7 +47,7 @@ not_run(Config) when is_list(Config) ->
 
 %% Tests switching tracing on and off.
 trace_on_and_off(Config) when is_list(Config) ->
-    Pid = spawn(?MODULE, bif_process, []),
+    Pid = spawn_link(?MODULE, bif_process, []),
     Self = self(),
     1 = erlang:trace(Pid, true, [call,timestamp]),
     {flags, Flags} = erlang:trace_info(Pid,flags),
@@ -59,6 +59,7 @@ trace_on_and_off(Config) when is_list(Config) ->
     1 = erlang:trace(Pid, false, [call]),
     {flags,[]} =  erlang:trace_info(Pid,flags),
     {tracer, []} = erlang:trace_info(Pid,tracer),
+    unlink(Pid),
     exit(Pid,kill),
     ok.
 
@@ -71,7 +72,7 @@ trace_bif_local(Config) when is_list(Config) ->
     do_trace_bif([local]).
 
 do_trace_bif(Flags) ->
-    Pid = spawn(?MODULE, bif_process, []),
+    Pid = spawn_link(?MODULE, bif_process, []),
     1 = erlang:trace(Pid, true, [call]),
     erlang:trace_pattern({erlang,'_','_'}, [], Flags),
     Pid ! {do_bif, time, []},
@@ -90,6 +91,7 @@ do_trace_bif(Flags) ->
 
     1 = erlang:trace(Pid, false, [call]),
     erlang:trace_pattern({erlang,'_','_'}, false, Flags),
+    unlink(Pid),
     exit(Pid, die),
     ok.
 
@@ -121,7 +123,7 @@ trace_bif_timestamp_local(Config) when is_list(Config) ->
 
 do_trace_bif_timestamp(Flags, TsType, TsFlags) ->
     io:format("Testing with TsType=~p TsFlags=~p~n", [TsType, TsFlags]),
-    Pid=spawn(?MODULE, bif_process, []),
+    Pid = spawn_link(?MODULE, bif_process, []),
     1 = erlang:trace(Pid, true, [call]++TsFlags),
     erlang:trace_pattern({erlang,'_','_'}, [], Flags),
 
@@ -161,6 +163,7 @@ do_trace_bif_timestamp(Flags, TsType, TsFlags) ->
     1 = erlang:trace(Pid, false, [call]),
     erlang:trace_pattern({erlang,'_','_'}, false, Flags),
 
+    unlink(Pid),
     exit(Pid, die),
     ok.
 
@@ -179,7 +182,7 @@ trace_bif_return(Config) when is_list(Config) ->
 
 do_trace_bif_return(TsType, TsFlags) ->
     io:format("Testing with TsType=~p TsFlags=~p~n", [TsType, TsFlags]),
-    Pid=spawn(?MODULE, bif_process, []),
+    Pid = spawn_link(?MODULE, bif_process, []),
     1 = erlang:trace(Pid, true, [call,return_to]++TsFlags),
     erlang:trace_pattern({erlang,'_','_'}, [{'_',[],[{return_trace}]}], 
                          [local]),
-- 
cgit v1.2.3


From 66bfe2f5d977bdac5809a273bd7b30f42063548f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 13 Jun 2017 10:09:31 +0200
Subject: trace_nif_SUITE: Don't leave processes running

---
 erts/emulator/test/trace_nif_SUITE.erl | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/trace_nif_SUITE.erl b/erts/emulator/test/trace_nif_SUITE.erl
index 7ac6fce234..5c35a8a83f 100644
--- a/erts/emulator/test/trace_nif_SUITE.erl
+++ b/erts/emulator/test/trace_nif_SUITE.erl
@@ -80,7 +80,7 @@ trace_nif_meta(Config) when is_list(Config) ->
                           {?MODULE,nif, ["Arg1"]}}),
     ok.
 do_trace_nif(Flags) ->
-    Pid = spawn(?MODULE, nif_process, []),
+    Pid = spawn_link(?MODULE, nif_process, []),
     1 = erlang:trace(Pid, true, [call]),
     erlang:trace_pattern({?MODULE,nif,'_'}, [], Flags),
     Pid ! {apply_nif, nif, []},
@@ -123,6 +123,8 @@ do_trace_nif(Flags) ->
 
     1 = erlang:trace(Pid, false, [call]),
     erlang:trace_pattern({?MODULE,nif,'_'}, false, Flags),
+
+    unlink(Pid),
     exit(Pid, die),
     ok.
 
@@ -137,7 +139,7 @@ trace_nif_timestamp_local(Config) when is_list(Config) ->
     do_trace_nif_timestamp([local]).
 
 do_trace_nif_timestamp(Flags) ->
-    Pid=spawn(?MODULE, nif_process, []),
+    Pid = spawn_link(?MODULE, nif_process, []),
     1 = erlang:trace(Pid, true, [call,timestamp]),
     erlang:trace_pattern({?MODULE,nif,'_'}, [], Flags),
 
@@ -170,6 +172,7 @@ do_trace_nif_timestamp(Flags) ->
     1 = erlang:trace(Pid, false, [call]),
     erlang:trace_pattern({erlang,'_','_'}, false, Flags),
 
+    unlink(Pid),
     exit(Pid, die),
     ok.
 
@@ -177,7 +180,7 @@ do_trace_nif_timestamp(Flags) ->
 trace_nif_return(Config) when is_list(Config) ->
     load_nif(Config),
 
-    Pid=spawn(?MODULE, nif_process, []),
+    Pid = spawn_link(?MODULE, nif_process, []),
     1 = erlang:trace(Pid, true, [call,timestamp,return_to]),
     erlang:trace_pattern({?MODULE,nif,'_'}, [{'_',[],[{return_trace}]}], 
                          [local]),
-- 
cgit v1.2.3


From 11879f00ffddef3c2c118b733e91690ff178a695 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 13 Jun 2017 10:24:25 +0200
Subject: tracer_SUITE: Don't leave processes running

---
 erts/emulator/test/tracer_SUITE.erl | 44 +++++++++++++++++++++++++++++++++----
 1 file changed, 40 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/tracer_SUITE.erl b/erts/emulator/test/tracer_SUITE.erl
index 730c43d8c2..c2fa3afd9b 100644
--- a/erts/emulator/test/tracer_SUITE.erl
+++ b/erts/emulator/test/tracer_SUITE.erl
@@ -70,7 +70,7 @@ init_per_testcase(TC, Config) when TC =:= load; TC =:= reload ->
                                end
                        end),
     register(tracer_test_config, Pid),
-    Config;
+    common_init_per_testcase(Config);
 init_per_testcase(_, Config) ->
     DataDir = proplists:get_value(data_dir, Config),
     case catch tracer_test:enabled(trace_status, self(), self()) of
@@ -79,16 +79,52 @@ init_per_testcase(_, Config) ->
         _ ->
             tracer_test:load(DataDir)
     end,
+    common_init_per_testcase(Config).
+
+common_init_per_testcase(Config) ->
+    Killer = erlang:spawn(fun() -> killer_loop([]) end),
+    register(killer_process, Killer),
     Config.
 
 end_per_testcase(TC, _Config) when TC =:= load; TC =:= reload ->
     purge(),
     exit(whereis(tracer_test_config), kill),
-    ok;
+    kill_processes();
 end_per_testcase(_, _Config) ->
     purge(),
+    kill_processes().
+
+kill_processes() ->
+    killer_process ! {get_pids,self()},
+    receive
+        {pids_to_kill,Pids} -> ok
+    end,
+    _ = [begin
+             case erlang:is_process_alive(P) of
+                 true ->
+                     io:format("Killing ~p\n", [P]);
+                 false ->
+                     ok
+             end,
+             erlang:unlink(P),
+             exit(P, kill)
+         end || P <- Pids],
     ok.
 
+killer_loop(Pids) ->
+    receive
+        {add_pid,Pid} ->
+            killer_loop([Pid|Pids]);
+        {get_pids,To} ->
+            To ! {pids_to_kill,Pids}
+    end.
+
+kill_me(Pid) ->
+    killer_process ! {add_pid,Pid},
+    Pid.
+
+%%% Test cases follow.
+
 load(_Config) ->
     purge(),
     1 = erlang:trace(self(), true, [{tracer, tracer_test, []}, call]),
@@ -113,7 +149,6 @@ unload(_Config) ->
 
     Pid = erlang:spawn_link(fun() -> ServerFun(0, undefined) end),
 
-
     Tc = fun(N) ->
                  Pid ! {N, self()},
                  receive done -> ok after 1000 -> ct:fail(timeout) end,
@@ -295,7 +330,7 @@ call_test(Arg) ->
 spawn(_Config) ->
 
     Tc = fun(Pid) ->
-                 Pid ! fun() -> erlang:spawn(lists,seq,[1,10]), ok end
+                 Pid ! fun() -> kill_me(erlang:spawn(lists,seq,[1,10])), ok end
          end,
 
     Expect =
@@ -355,6 +390,7 @@ unlink(_Config) ->
                                SPid = erlang:spawn(fun() -> receive _ -> ok end end),
                                erlang:link(SPid),
                                erlang:unlink(SPid),
+                               kill_me(SPid),
                                ok
                        end
          end,
-- 
cgit v1.2.3


From 8322886e6ac13476d01d6db96d5b50d376cc123e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 13 Jun 2017 10:27:22 +0200
Subject: trace_port_SUITE: Don't leave processes running

---
 erts/emulator/test/trace_port_SUITE.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/trace_port_SUITE.erl b/erts/emulator/test/trace_port_SUITE.erl
index 5eb27a7b68..140a59782f 100644
--- a/erts/emulator/test/trace_port_SUITE.erl
+++ b/erts/emulator/test/trace_port_SUITE.erl
@@ -190,7 +190,7 @@ receive_trace(Config) when is_list(Config) ->
 receive_trace_non_scheduler(Config) when is_list(Config) ->
     start_tracer(Config),
     S = self(),
-    Receiver = spawn(
+    Receiver = spawn_link(
                  fun() ->
                          receive
                              go ->
-- 
cgit v1.2.3


From a27c69e55f6f10d63135b6323060cf76b41509da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 13 Jun 2017 10:29:06 +0200
Subject: process_SUITE: Don't leave processes running

---
 erts/emulator/test/process_SUITE.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index 4204d12eb3..b993bf5c69 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -677,7 +677,7 @@ chk_pi_order([{Arg, _}| Values], [Arg|Args]) ->
     chk_pi_order(Values, Args).
 
 process_info_2_list(Config) when is_list(Config) ->
-    Proc = spawn(fun () -> receive after infinity -> ok end end),
+    Proc = spawn_link(fun () -> receive after infinity -> ok end end),
     register(process_SUITE_process_info_2_list1, self()),
     register(process_SUITE_process_info_2_list2, Proc),
     erts_debug:set_internal_state(available_internal_state,true),
-- 
cgit v1.2.3


From f529828f540ad50cc157755fbd19d8b595ee07ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 13 Jun 2017 14:14:26 +0200
Subject: Remove unused functions in test emulator test suites

---
 erts/emulator/test/a_SUITE.erl                  | 10 ------
 erts/emulator/test/dirty_bif_SUITE.erl          |  3 --
 erts/emulator/test/distribution_SUITE.erl       | 46 -------------------------
 erts/emulator/test/driver_SUITE.erl             |  6 ----
 erts/emulator/test/message_queue_data_SUITE.erl |  2 --
 erts/emulator/test/monitor_SUITE.erl            |  3 --
 erts/emulator/test/receive_SUITE.erl            |  3 --
 erts/emulator/test/system_info_SUITE.erl        |  5 ---
 erts/emulator/test/time_SUITE.erl               |  3 --
 erts/emulator/test/trace_port_SUITE.erl         |  9 -----
 erts/emulator/test/unique_SUITE.erl             |  7 ----
 11 files changed, 97 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/a_SUITE.erl b/erts/emulator/test/a_SUITE.erl
index 07922f9942..9cc86014ec 100644
--- a/erts/emulator/test/a_SUITE.erl
+++ b/erts/emulator/test/a_SUITE.erl
@@ -99,15 +99,5 @@ pollset_size(Config) when is_list(Config) ->
 %% Internal functions...
 %%
 
-display_check_io(ChkIo) ->
-    catch erlang:display('--- CHECK IO INFO ---'),
-    catch erlang:display(ChkIo),
-    catch erts_debug:set_internal_state(available_internal_state, true),
-    NoOfErrorFds = (catch element(1, erts_debug:get_internal_state(check_io_debug))),
-    catch erlang:display({'NoOfErrorFds', NoOfErrorFds}),
-    catch erts_debug:set_internal_state(available_internal_state, false),
-    catch erlang:display('--- CHECK IO INFO ---'),
-    ok.
-
 get_check_io_info() ->
     z_SUITE:get_check_io_info().
diff --git a/erts/emulator/test/dirty_bif_SUITE.erl b/erts/emulator/test/dirty_bif_SUITE.erl
index b8361690e6..981ec4d48d 100644
--- a/erts/emulator/test/dirty_bif_SUITE.erl
+++ b/erts/emulator/test/dirty_bif_SUITE.erl
@@ -543,9 +543,6 @@ test_dirty_process_access(Start, Test, Finish) ->
 	 end,
     ok = Finish(BifPid).
 
-receive_any() ->
-    receive M -> M end.
-
 start_node(Config) ->
     start_node(Config, "").
 
diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index dc7afa381b..b4ec99f902 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -2255,52 +2255,6 @@ string_to_utf8_list([CP|CPs]) when is_integer(CP),
      16#80 bor (16#3F band CP)
      | string_to_utf8_list(CPs)].
 
-utf8_list_to_string([]) ->
-    [];
-utf8_list_to_string([B|Bs]) when is_integer(B),
-                                 0 =< B,
-                                 B =< 16#7F ->
-    [B | utf8_list_to_string(Bs)];
-utf8_list_to_string([B0, B1 | Bs]) when is_integer(B0),
-                                        16#C0 =< B0,
-                                        B0 =< 16#DF,
-                                        is_integer(B1),
-                                        16#80 =< B1,
-                                        B1 =< 16#BF ->
-    [(((B0 band 16#1F) bsl 6)
-      bor (B1 band 16#3F))
-     | utf8_list_to_string(Bs)];
-utf8_list_to_string([B0, B1, B2 | Bs]) when is_integer(B0),
-                                            16#E0 =< B0,
-                                            B0 =< 16#EF,
-                                            is_integer(B1),
-                                            16#80 =< B1,
-                                            B1 =< 16#BF,
-                                            is_integer(B2),
-                                            16#80 =< B2,
-                                            B2 =< 16#BF ->
-    [(((B0 band 16#F) bsl 12)
-      bor ((B1 band 16#3F) bsl 6)
-      bor (B2 band 16#3F))
-     | utf8_list_to_string(Bs)];
-utf8_list_to_string([B0, B1, B2, B3 | Bs]) when is_integer(B0),
-                                                16#F0 =< B0,
-                                                B0 =< 16#F7,
-                                                is_integer(B1),
-                                                16#80 =< B1,
-                                                B1 =< 16#BF,
-                                                is_integer(B2),
-                                                16#80 =< B2,
-                                                B2 =< 16#BF,
-                                                is_integer(B3),
-                                                16#80 =< B3,
-                                                B3 =< 16#BF ->
-    [(((B0 band 16#7) bsl 18)
-      bor ((B1 band 16#3F) bsl 12)
-      bor ((B2 band 16#3F) bsl 6)
-      bor (B3 band 16#3F))
-     | utf8_list_to_string(Bs)].
-
 mk_pid({NodeName, Creation}, Number, Serial) when is_atom(NodeName) ->
     <<?VERSION_MAGIC, NodeNameExt/binary>> = term_to_binary(NodeName),
     mk_pid({NodeNameExt, Creation}, Number, Serial);
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index e854a5f945..954f3b137f 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -1750,12 +1750,6 @@ thread_mseg_alloc_cache_clean(Config) when is_list(Config) ->
             ok
     end.
 
-mseg_alloc_cci(MsegAllocInfo) ->
-    {value,{options, OL}}
-    = lists:keysearch(options, 1, MsegAllocInfo),
-    {value,{cci,CCI}} = lists:keysearch(cci,1,OL),
-    CCI.
-
 mseg_alloc_ccc() ->
     mseg_alloc_ccc(mseg_inst_info(0)).
 
diff --git a/erts/emulator/test/message_queue_data_SUITE.erl b/erts/emulator/test/message_queue_data_SUITE.erl
index 0fd86155fb..0d5f4aa0af 100644
--- a/erts/emulator/test/message_queue_data_SUITE.erl
+++ b/erts/emulator/test/message_queue_data_SUITE.erl
@@ -192,8 +192,6 @@ total_heap_size(_Config) ->
 %%
 %%
 
-start_node(Config) ->
-    start_node(Config, []).
 start_node(Config, Opts) when is_list(Config), is_list(Opts) ->
     Pa = filename:dirname(code:which(?MODULE)),
     Name = list_to_atom(atom_to_list(?MODULE)
diff --git a/erts/emulator/test/monitor_SUITE.erl b/erts/emulator/test/monitor_SUITE.erl
index 827ed817cc..ed3ab23738 100644
--- a/erts/emulator/test/monitor_SUITE.erl
+++ b/erts/emulator/test/monitor_SUITE.erl
@@ -974,9 +974,6 @@ generate(_Fun, 0) ->
 generate(Fun, N) ->
     [Fun() | generate(Fun, N-1)].
 
-start_node(Config) ->
-    start_node(Config, "").
-
 start_node(Config, Args) ->
     TestCase = proplists:get_value(testcase, Config),
     PA = filename:dirname(code:which(?MODULE)),
diff --git a/erts/emulator/test/receive_SUITE.erl b/erts/emulator/test/receive_SUITE.erl
index c7d5a3f5a0..26eddfd03c 100644
--- a/erts/emulator/test/receive_SUITE.erl
+++ b/erts/emulator/test/receive_SUITE.erl
@@ -34,9 +34,6 @@ suite() ->
 all() -> 
     [call_with_huge_message_queue, receive_in_between].
 
-groups() -> 
-    [].
-
 call_with_huge_message_queue(Config) when is_list(Config) ->
     Pid = spawn_link(fun echo_loop/0),
     _WarmUpTime = time_calls(Pid),
diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl
index 9c71f20279..56522039da 100644
--- a/erts/emulator/test/system_info_SUITE.erl
+++ b/erts/emulator/test/system_info_SUITE.erl
@@ -363,11 +363,6 @@ mem_workers_call(MWs, Fun, Args) ->
                       end
               end, MWs).
 
-mem_workers_cast(MWs, Fun, Args) ->
-    lists:foreach(fun (MW) ->
-			  MW ! {cast, self(), Fun, Args}
-		  end, MWs).
-
 spawn_mem_workers() ->
     spawn_mem_workers(erlang:system_info(schedulers_online)).
 
diff --git a/erts/emulator/test/time_SUITE.erl b/erts/emulator/test/time_SUITE.erl
index 214a549a9d..6b9a358496 100644
--- a/erts/emulator/test/time_SUITE.erl
+++ b/erts/emulator/test/time_SUITE.erl
@@ -993,9 +993,6 @@ bad_dates() ->
      {{1996, 4, 30}, {12, 0, -1}},		% Sec
      {{1996, 4, 30}, {12, 0, 60}}].
 
-start_node(Config) ->
-    start_node(Config, "").
-
 start_node(Config, Args) ->
     TestCase = proplists:get_value(testcase, Config),
     PA = filename:dirname(code:which(?MODULE)),
diff --git a/erts/emulator/test/trace_port_SUITE.erl b/erts/emulator/test/trace_port_SUITE.erl
index 140a59782f..fa6460073a 100644
--- a/erts/emulator/test/trace_port_SUITE.erl
+++ b/erts/emulator/test/trace_port_SUITE.erl
@@ -349,15 +349,6 @@ huge_data(N) ->
     P = huge_data(N div 2),
     [16#1234566,P|P].
 
-expect() ->
-    receive
-        Other ->
-            ok = io:format("Unexpected; got ~p", [Other]),
-            ct:fail({unexpected, Other})
-    after 200 ->
-              ok
-    end.
-
 expect({trace_ts,E1,E2,info,ts}=Message) ->
     receive
         {trace_ts,E1,E2,_Info,_Ts}=MessageTs ->
diff --git a/erts/emulator/test/unique_SUITE.erl b/erts/emulator/test/unique_SUITE.erl
index c5aa80c7b4..3b972b87b5 100644
--- a/erts/emulator/test/unique_SUITE.erl
+++ b/erts/emulator/test/unique_SUITE.erl
@@ -302,13 +302,6 @@ smaller_valid_uniqint(Int, UinqintInfo) ->
             smaller_valid_uniqint(Cand, UinqintInfo)
     end.
 
-int32_to_bigendian_list(Int) ->
-    0 = Int bsr 32,
-    [(Int bsr 24) band 16#ff,
-     (Int bsr 16) band 16#ff,
-     (Int bsr 8) band 16#ff,
-     Int band 16#ff].
-
 mk_uniqint(Int, #uniqint_info {min_int = MinInt,
                                sched_bits = SchedBits} = _UinqintInfo) ->
     Int1 = Int - MinInt,
-- 
cgit v1.2.3


From 3ea80da9e908e0802b78e19bd29f4ccc2c7be703 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 13 Jun 2017 14:24:41 +0200
Subject: Eliminate warnings for unused variables

---
 erts/emulator/test/efile_SUITE.erl          | 1 -
 erts/emulator/test/long_timers_test.erl     | 2 +-
 erts/emulator/test/port_SUITE.erl           | 2 +-
 erts/emulator/test/scheduler_SUITE.erl      | 5 ++---
 erts/emulator/test/system_profile_SUITE.erl | 6 ++++--
 erts/emulator/test/time_SUITE.erl           | 2 +-
 erts/emulator/test/unique_SUITE.erl         | 1 -
 7 files changed, 9 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl
index 6bb8487c4e..eaf6b58c33 100644
--- a/erts/emulator/test/efile_SUITE.erl
+++ b/erts/emulator/test/efile_SUITE.erl
@@ -74,7 +74,6 @@ file_keys(Dir,Num,FdList,FnList) ->
 %% Check that the distribution of files over async threads is fair
 async_dist(Config) when is_list(Config) ->
     DataDir = proplists:get_value(data_dir,Config),
-    TestFile = filename:join(DataDir, "existing_file"),
     Dir = filename:dirname(code:which(?MODULE)),
     AsyncSizes = [7,10,100,255,256,64,63,65],
     Max = 0.5,
diff --git a/erts/emulator/test/long_timers_test.erl b/erts/emulator/test/long_timers_test.erl
index c9a380a229..896ac5e1f6 100644
--- a/erts/emulator/test/long_timers_test.erl
+++ b/erts/emulator/test/long_timers_test.erl
@@ -175,7 +175,7 @@ had_high_cpu_util(StartTime,
     ActTo = TargetTo + TimeoutDiff,
     hcpu(ActTo, TargetTo, UtilData).
 
-hcpu(_ActTo, _TargetTo, [{UT, 0} | _] = UD) ->
+hcpu(_ActTo, _TargetTo, [{_UT, 0} | _]) ->
     missing; %% Util is the integer zero when not supported...
 %% UT2 =:= UT1
 hcpu(ActTo, TargetTo, [{UT, _}, {UT, _} | _] = UD) ->
diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index fccbaf13ee..f512fa3a57 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -1042,7 +1042,7 @@ pipe_limit_env_do(Bytes, Cmd, CmdSize) ->
 
 %% environ format: KEY=VALUE\0
 env_of_bytes(Bytes) when Bytes > 3 ->
-    Env = [{"X",lists:duplicate(Bytes-3, $x)}];
+    [{"X",lists:duplicate(Bytes-3, $x)}];
 env_of_bytes(_) -> [].
 
 %% White box assumption about payload written to pipe
diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl
index 8d71df65e7..af33de237c 100644
--- a/erts/emulator/test/scheduler_SUITE.erl
+++ b/erts/emulator/test/scheduler_SUITE.erl
@@ -799,7 +799,7 @@ update_cpu_info(Config) when is_list(Config) ->
 		      unchanged -> ok;
 		      changed -> ok
 		  end;
-	      {Avail, _} ->
+	      {_Avail, _} ->
 		  try
 		      adjust_schedulers_online(),
 		      case erlang:system_info(schedulers_online) of
@@ -848,7 +848,7 @@ update_cpu_info(Config) when is_list(Config) ->
 bits_in_mask(Mask) ->
     bits_in_mask(Mask, 0, 0).
 
-bits_in_mask(0, Shift, N) ->
+bits_in_mask(0, _Shift, N) ->
     N;
 bits_in_mask(Mask, Shift, N) ->
     case Mask band (1 bsl Shift) of
@@ -1143,7 +1143,6 @@ dirty_scheduler_threads(Config) when is_list(Config) ->
     end.
 
 dirty_scheduler_threads_test(Config) ->
-    SmpSupport = erlang:system_info(smp_support),
     {Sched, SchedOnln, _} = get_dsstate(Config, ""),
     {HalfSched, HalfSchedOnln} = {lists:max([1,Sched div 2]),
                                   lists:max([1,SchedOnln div 2])},
diff --git a/erts/emulator/test/system_profile_SUITE.erl b/erts/emulator/test/system_profile_SUITE.erl
index 2e359b11ce..3a8f862b05 100644
--- a/erts/emulator/test/system_profile_SUITE.erl
+++ b/erts/emulator/test/system_profile_SUITE.erl
@@ -542,8 +542,10 @@ has_runnable_event(TsType, Events) ->
 	    end
         end, Events).
 
-has_profiler_pid_event([], _) -> false;
-has_profiler_pid_event([{profile, Pid, _Activity, _MFA, _TS}|Events], Pid) -> true;
+has_profiler_pid_event([], _) ->
+    false;
+has_profiler_pid_event([{profile, Pid, _Activity, _MFA, _TS}|_Events], Pid) ->
+    true;
 has_profiler_pid_event([_|Events], Pid) ->
     has_profiler_pid_event(Events, Pid).
 
diff --git a/erts/emulator/test/time_SUITE.erl b/erts/emulator/test/time_SUITE.erl
index 6b9a358496..e01efac86b 100644
--- a/erts/emulator/test/time_SUITE.erl
+++ b/erts/emulator/test/time_SUITE.erl
@@ -300,7 +300,7 @@ os_system_time_offset() ->
 had_time_warp(Secs) ->
     had_time_warp(os_system_time_offset(), Secs).
 
-had_time_warp(OrigOffs, 0) ->
+had_time_warp(_OrigOffs, 0) ->
     false;
 had_time_warp(OrigOffs, N) ->
     receive after 1000 -> ok end,
diff --git a/erts/emulator/test/unique_SUITE.erl b/erts/emulator/test/unique_SUITE.erl
index 3b972b87b5..ab02531de3 100644
--- a/erts/emulator/test/unique_SUITE.erl
+++ b/erts/emulator/test/unique_SUITE.erl
@@ -308,7 +308,6 @@ mk_uniqint(Int, #uniqint_info {min_int = MinInt,
     ThrId = Int1 band ((1 bsl SchedBits) - 1),
     Value = (Int1 bsr SchedBits) band ((1 bsl 64) - 1),
     0 = Int1 bsr (SchedBits + 64),
-    NodeName = atom_to_list(node()),
     Make = {make_unique_integer, ThrId, Value},
     %% erlang:display(Make),
     Res = erts_debug:get_internal_state(Make),
-- 
cgit v1.2.3


From f1c69ee583dcd1f525562cf6adc382b8464b1578 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 13 Jun 2017 15:51:42 +0200
Subject: Revert "erts: Do not generate atoms on old latin1 external format"

This reverts commit 65b04e233e09e3cc2e0fda3c28e155b95c5a4baf.
---
 erts/emulator/beam/dist.h     |  1 -
 erts/emulator/beam/external.c | 39 ++++++++++++++++++++++++++++++++++++---
 2 files changed, 36 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 93a651b24d..3e17645997 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -53,7 +53,6 @@
 			       | DFLAG_EXPORT_PTR_TAG		\
 			       | DFLAG_BIT_BINARIES             \
 			       | DFLAG_MAP_TAG                  \
-                               | DFLAG_UTF8_ATOMS               \
                                | DFLAG_BIG_CREATION)
 
 /* opcodes used in distribution messages */
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 1190d90b8e..06213daa67 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2090,6 +2090,7 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags)
 {
     int iix;
     int len;
+    int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS);
 
     ASSERT(is_atom(atom));
 
@@ -2118,8 +2119,8 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags)
     if (iix < 0) {
 	Atom *a = atom_tab(atom_val(atom));
 	len = a->len;
-        {
-            if (len > 255) {
+	if (utf8_atoms || a->latin1_chars < 0) {
+	    if (len > 255) {
 		*ep++ = ATOM_UTF8_EXT;
 		put_int16(len, ep);
 		ep += 2;
@@ -2131,6 +2132,32 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags)
 	    }
 	    sys_memcpy((char *) ep, (char *) a->name, len);
 	}
+	else {
+	    if (a->latin1_chars <= 255 && (dflags & DFLAG_SMALL_ATOM_TAGS)) {
+		*ep++ = SMALL_ATOM_EXT;
+		if (len == a->latin1_chars) {
+		    sys_memcpy(ep+1, a->name, len);
+		}
+		else {
+		    len = erts_utf8_to_latin1(ep+1, a->name, len);
+		    ASSERT(len == a->latin1_chars);
+		}
+		put_int8(len, ep);
+		ep++;
+	    }
+	    else {
+		*ep++ = ATOM_EXT;
+		if (len == a->latin1_chars) {
+		    sys_memcpy(ep+2, a->name, len);
+		}
+		else {
+		    len = erts_utf8_to_latin1(ep+2, a->name, len);
+		    ASSERT(len == a->latin1_chars);
+		}
+		put_int16(len, ep);
+		ep += 2;
+	    }	    
+	}
 	ep += len;
 	return ep;
     }
@@ -4053,13 +4080,19 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 	    else {
 		Atom *a = atom_tab(atom_val(obj));
 		int alen;
-                {
+		if ((dflags & DFLAG_UTF8_ATOMS) || a->latin1_chars < 0) {
 		    alen = a->len;
 		    result += 1 + 1 + alen;
 		    if (alen > 255) {
 			result++; /* ATOM_UTF8_EXT (not small) */
 		    }
 		}
+		else {
+		    alen = a->latin1_chars;
+		    result += 1 + 1 + alen;
+		    if (alen > 255 || !(dflags & DFLAG_SMALL_ATOM_TAGS))
+			result++; /* ATOM_EXT (not small) */
+		}
 		insert_acache_map(acmp, obj, dflags);
 	    }
 	    break;
-- 
cgit v1.2.3


From e2915bedad8579cb4f2e52db01bc7adc1477ba69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 13 Jun 2017 14:26:45 +0200
Subject: Assert that enqueue_later_op must not run on a dirty scheduler.

You're not supposed to do this, but it behaved correctly even with a debug
build aside from degraded performance (The scheduler started to busy-wait
for new operations rather than sleep).
---
 erts/emulator/beam/erl_process.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index fc2b34e70f..2a988b9d2d 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2371,7 +2371,7 @@ enqueue_later_op(ErtsSchedulerData *esdp,
 		 ErtsThrPrgrLaterOp *lop)
 {
     ErtsThrPrgrVal later = erts_thr_progress_later(esdp);
-    ASSERT(esdp);
+    ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp));
 
     lop->func = later_func;
     lop->data = later_data;
-- 
cgit v1.2.3


From 7567c97c4391c61199e81b48f584815284aef289 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 14 Jun 2017 08:45:13 +0200
Subject: Remove noisy and redundant #ifdefs

The relevant macros are already guarded by said #ifdefs.
---
 erts/emulator/beam/erl_process.c | 68 +++++++++++++++-------------------------
 1 file changed, 25 insertions(+), 43 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 2a988b9d2d..ae598e9663 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1289,10 +1289,8 @@ reply_sched_wall_time(void *vswtrp)
     ErlOffHeap *ohp = NULL;
     ErtsMessage *mp = NULL;
 
-    ASSERT(esdp);
-#ifdef ERTS_DIRTY_SCHEDULERS
-    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
-#endif
+    ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp));
+
     if (swtrp->set) {
 	if (!swtrp->enable && esdp->sched_wall_time.enabled) {
 	    esdp->sched_wall_time.u.need = erts_sched_balance_util;
@@ -1458,11 +1456,10 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable,
     ErtsSchedWallTimeReq *swtrp;
     Eterm *hp;
 
+    ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp));
+
     if (!set && !esdp->sched_wall_time.enabled)
 	return THE_NON_VALUE;
-#ifdef ERTS_DIRTY_SCHEDULERS
-    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
-#endif
 
     swtrp = swtreq_alloc();
     ref = erts_make_ref(c_p);
@@ -1509,10 +1506,7 @@ reply_system_check(void *vscrp)
     ErlOffHeap *ohp = NULL;
     ErtsMessage *mp = NULL;
 
-    ASSERT(esdp);
-#ifdef ERTS_DIRTY_SCHEDULERS
-    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
-#endif
+    ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp));
 
     sz = ERTS_REF_THING_SIZE;
     mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
@@ -1778,9 +1772,9 @@ static ERTS_INLINE void
 haw_thr_prgr_current_check_progress(ErtsAuxWorkData *awdp)
 {
     ErtsThrPrgrVal current = awdp->current_thr_prgr;
-#ifdef ERTS_DIRTY_SCHEDULERS
+
     ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
-#endif
+
     if (current != ERTS_THR_PRGR_INVALID
 	&& !erts_thr_progress_equal(current, erts_thr_progress_current())) {
 	/*
@@ -1797,9 +1791,7 @@ handle_delayed_aux_work_wakeup(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, in
 {
     int jix, max_jix;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
-#endif
 
     ASSERT(awdp->delayed_wakeup.next != ERTS_DELAYED_WAKEUP_INFINITY);
 
@@ -1956,9 +1948,8 @@ handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp,
 			      erts_aint32_t aux_work,
 			      int waiting)
 {
-#ifdef ERTS_DIRTY_SCHEDULERS
     ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
-#endif
+
     if (!erts_thr_progress_has_reached_this(haw_thr_prgr_current(awdp),
 					    awdp->misc.thr_prgr))
 	return aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
@@ -2011,9 +2002,9 @@ erts_schedule_multi_misc_aux_work(int ignore_self,
 
     if (ignore_self) {
 	ErtsSchedulerData *esdp = erts_get_scheduler_data();
-#ifdef ERTS_DIRTY_SCHEDULERS
+
 	ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
-#endif
+
 	if (esdp)
 	    self = (int) esdp->no;
     }
@@ -2043,9 +2034,9 @@ handle_async_ready(ErtsAuxWorkData *awdp,
 		   int waiting)
 {
     ErtsSchedulerSleepInfo *ssi = awdp->ssi;
-#ifdef ERTS_DIRTY_SCHEDULERS
+
     ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
-#endif
+
     unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY);
     if (erts_check_async_ready(awdp->async_ready.queue)) {
 	if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY)
@@ -2070,9 +2061,8 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp,
 {
     void *thr_prgr_p;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
-#endif
+
 #ifdef ERTS_SMP
     if (awdp->async_ready.need_thr_prgr
 	&& !erts_thr_progress_has_reached_this(haw_thr_prgr_current(awdp),
@@ -2110,9 +2100,8 @@ handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
     ErtsSchedulerSleepInfo *ssi = awdp->ssi;
     erts_aint32_t res;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
-#endif
+
     unset_aux_work_flags(ssi, (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
 			       | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC));
     aux_work &= ~(ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
@@ -2160,9 +2149,9 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin
     ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
     int more_work = 0;
     ERTS_MSACC_PUSH_STATE_M_X();
-#ifdef ERTS_DIRTY_SCHEDULERS
+
     ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
-#endif
+
     unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD);
     ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ALLOC);
     erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
@@ -2199,9 +2188,8 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i
     ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
     ErtsThrPrgrVal current = haw_thr_prgr_current(awdp);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
-#endif
+
     if (!erts_thr_progress_has_reached_this(current, awdp->dd.thr_prgr))
 	return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR;
 
@@ -2258,9 +2246,8 @@ handle_canceled_timers(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin
     ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
     int more_work = 0;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
-#endif
+
     unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS);
     erts_handle_canceled_timers((void *) awdp->esdp,
 				&need_thr_progress,
@@ -2294,9 +2281,8 @@ handle_canceled_timers_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i
     ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
     ErtsThrPrgrVal current = haw_thr_prgr_current(awdp);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
-#endif
+
     if (!erts_thr_progress_has_reached_this(current, awdp->cncld_tmrs.thr_prgr))
 	return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR;
 
@@ -2339,9 +2325,8 @@ handle_thr_prgr_later_op(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int wait
     int lops;
     ErtsThrPrgrVal current = haw_thr_prgr_current(awdp);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
-#endif
+
     for (lops = 0; lops < ERTS_MAX_THR_PRGR_LATER_OPS; lops++) {
 	ErtsThrPrgrLaterOp *lop = awdp->later_op.first;
 
@@ -2424,9 +2409,7 @@ handle_debug_wait_completed(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int w
     ErtsSchedulerSleepInfo *ssi = awdp->ssi;
     erts_aint32_t saved_aux_work, flags;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
-#endif
 
     flags = awdp->debug.wait_completed.flags;
 
@@ -3008,9 +2991,9 @@ static ERTS_INLINE void
 sched_active_sys(Uint no, ErtsRunQueue *rq)
 {
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
-#ifdef ERTS_DIRTY_SCHEDULERS
+
     ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
-#endif
+
     ASSERT(rq->waiting < 0);
     rq->waiting *= -1;
     rq->waiting--;
@@ -3075,9 +3058,9 @@ static ERTS_INLINE void
 sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq)
 {
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
-#ifdef ERTS_DIRTY_SCHEDULERS
+
     ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
-#endif
+
     ASSERT(rq->waiting < 0);
     rq->waiting *= -1;
 }
@@ -3586,9 +3569,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	erts_smp_atomic32_set_relb(&function_calls, 0);
 	*fcalls = 0;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 	ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
-#endif
+
 	sched_waiting_sys(esdp->no, rq);
 
 	erts_smp_runq_unlock(rq);
-- 
cgit v1.2.3


From 635b35ef4d0403a9bbf149e076aed6fca84af383 Mon Sep 17 00:00:00 2001
From: kvakvs <dmytro.lytovchenko@gmail.com>
Date: Wed, 3 Aug 2016 14:39:00 +0200
Subject: Mark socket disconnected on tcp_send_or_shutdown_error

The socket left lingering due to {exit_on_close, false} will accept writes
in a confusing way, returning either enotconn or blocking. This fix allows
socket to know that it has been closed recently, and new writes won't pass.
---
 erts/emulator/drivers/common/inet_drv.c | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 2d2bd80783..1649b07ad9 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -4378,6 +4378,12 @@ static void desc_close(inet_descriptor* desc)
 	desc->event = INVALID_EVENT; /* closed by stop_select callback */
 	desc->s = INVALID_SOCKET;
 	desc->event_mask = 0;
+
+	/* mark as disconnected in case when socket is left lingering due to
+	 * {exit_on_close, false} option in gen_tcp socket creation. Next
+	 * write to socket should produce {error, enotconn} and send a
+	 * message {tcp_error,#Port<>,econnreset} */
+	desc->state &= ~INET_STATE_CONNECTED;
     }
 }
 
-- 
cgit v1.2.3


From 22ecae32fb0a0a9777cb00599a3b6440adb7bd83 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 14 Jun 2017 10:47:54 +0200
Subject: erts: Cleanup dropped port tasks correctly

Before this fix, the extra data attached to a port task had to
be cleaned up by the calling function. This caused problems
because the outputv call, co-allocates the extra data with the
port task. So in rare circumstances the port task would be
free'd before the extra data was free'd which led to segfault
when looking at the port task.

This has been fixed by the generic PORT_TASK_ABORT behaviour
being used even for tasks dropped in the erts_schedule_proc2port_signal
API.
---
 erts/emulator/beam/erl_port_task.c | 26 ++++++++++++----
 erts/emulator/beam/io.c            | 62 ++++++++++++--------------------------
 2 files changed, 40 insertions(+), 48 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 3102e44c11..8d78b22228 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -852,10 +852,11 @@ schedule_port_task_handle_list_free(ErtsPortTaskHandleList *pthlp)
 }
 
 static ERTS_INLINE void
-abort_nosuspend_task(Port *pp,
-		     ErtsPortTaskType type,
-		     ErtsPortTaskTypeData *tdp,
-		     int bpq_data)
+abort_signal_task(Port *pp,
+                  int abort_type,
+                  ErtsPortTaskType type,
+                  ErtsPortTaskTypeData *tdp,
+                  int bpq_data)
 {
 
     ASSERT(type == ERTS_PORT_TASK_PROC_SIG);
@@ -863,18 +864,28 @@ abort_nosuspend_task(Port *pp,
     if (!bpq_data)
 	tdp->psig.callback(NULL,
 			   ERTS_PORT_SFLG_INVALID,
-			   ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND,
+			   abort_type,
 			   &tdp->psig.data);
     else {
 	ErlDrvSizeT size = erts_proc2port_sig_command_data_size(&tdp->psig.data);
 	tdp->psig.callback(NULL,
 			   ERTS_PORT_SFLG_INVALID,
-			   ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND,
+			   abort_type,
 			   &tdp->psig.data);
 	aborted_proc2port_data(pp, size);
     }
 }
 
+
+static ERTS_INLINE void
+abort_nosuspend_task(Port *pp,
+		     ErtsPortTaskType type,
+		     ErtsPortTaskTypeData *tdp,
+		     int bpq_data)
+{
+    abort_signal_task(pp, ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND, type, tdp, bpq_data);
+}
+
 static ErtsPortTaskHandleList *
 get_free_nosuspend_handles(Port *pp)
 {
@@ -1625,6 +1636,9 @@ fail:
 	erts_port_dec_refc(pp);
 #endif
 
+    abort_signal_task(pp, ERTS_PROC2PORT_SIG_ABORT,
+                      ptp->type, &ptp->u.alive.td, 0);
+
     if (ns_pthlp)
 	erts_free(ERTS_ALC_T_PT_HNDL_LIST, ns_pthlp);
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index c3eb610fdc..caf8bfc294 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1968,7 +1968,6 @@ int
 erts_port_output_async(Port *prt, Eterm from, Eterm list)
 {
 
-    ErtsPortOpResult res;
     ErtsProc2PortSigData *sigdp;
     erts_driver_t *drv = prt->drv_ptr;
     size_t size;
@@ -2102,26 +2101,18 @@ erts_port_output_async(Port *prt, Eterm from, Eterm list)
 	sigdp->u.output.size = size;
 	port_sig_callback = port_sig_output;
     }
-    sigdp->flags = 0;
     ns_pthp = NULL;
     task_flags = 0;
 
-    res = erts_schedule_proc2port_signal(NULL,
-					 prt,
-					 ERTS_INVALID_PID,
-					 NULL,
-					 sigdp,
-					 task_flags,
-					 ns_pthp,
-					 port_sig_callback);
+    erts_schedule_proc2port_signal(NULL,
+                                   prt,
+                                   ERTS_INVALID_PID,
+                                   NULL,
+                                   sigdp,
+                                   task_flags,
+                                   ns_pthp,
+                                   port_sig_callback);
 
-    if (res != ERTS_PORT_OP_SCHEDULED) {
-	if (drv->outputv)
-	    cleanup_scheduled_outputv(evp, cbin);
-	else
-	    cleanup_scheduled_output(buf);
-	return 1;
-    }
     return 1;
 
 bad_value:
@@ -2554,10 +2545,6 @@ erts_port_output(Process *c_p,
 					 port_sig_callback);
 
     if (res != ERTS_PORT_OP_SCHEDULED) {
-	if (drv->outputv)
-	    cleanup_scheduled_outputv(evp, cbin);
-	else
-	    cleanup_scheduled_output(buf);
 	return res;
     }
 
@@ -2736,21 +2723,14 @@ erts_port_exit(Process *c_p,
 					   &bp->off_heap);
     }
 
-    res = erts_schedule_proc2port_signal(c_p,
-					 prt,
-					 c_p ? c_p->common.id : from,
-					 refp,
-					 sigdp,
-					 0,
-					 NULL,
-					 port_sig_exit);
-
-    if (res == ERTS_PORT_OP_DROPPED) {
-	if (bp)
-	    free_message_buffer(bp);
-    }
-
-    return res;
+    return erts_schedule_proc2port_signal(c_p,
+                                          prt,
+                                          c_p ? c_p->common.id : from,
+                                          refp,
+                                          sigdp,
+                                          0,
+                                          NULL,
+                                          port_sig_exit);
 }
 
 static ErtsPortOpResult
@@ -4932,10 +4912,9 @@ erts_port_control(Process* c_p,
 					 0,
 					 NULL,
 					 port_sig_control);
-    if (res != ERTS_PORT_OP_SCHEDULED) {
-	cleanup_scheduled_control(binp, bufp);
+    if (res != ERTS_PORT_OP_SCHEDULED)
 	return ERTS_PORT_OP_BADARG;
-    }
+
     return res;
 }
 
@@ -5225,10 +5204,9 @@ erts_port_call(Process* c_p,
 					 0,
 					 NULL,
 					 port_sig_call);
-    if (res != ERTS_PORT_OP_SCHEDULED) {
-	cleanup_scheduled_call(bufp);
+    if (res != ERTS_PORT_OP_SCHEDULED)
 	return ERTS_PORT_OP_BADARG;
-    }
+
     return res;
 }
 
-- 
cgit v1.2.3


From bdc0f3504fb13f777a7cc826caa5fd10dc6fc291 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 14 Jun 2017 13:23:34 +0200
Subject: Introduce minor vsn 2 in term_to_binary/2

---
 erts/emulator/beam/external.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 06213daa67..95275847f8 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1129,8 +1129,10 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
 		case 0:
 		    flags = TERM_TO_BINARY_DFLAGS & ~DFLAG_NEW_FLOATS;
 		    break;
-		case 1:
+		case 1: /* Current default... */
 		    flags = TERM_TO_BINARY_DFLAGS;
+                case 2:
+                    flags = TERM_TO_BINARY_DFLAGS | DFLAG_UTF8_ATOMS;
 		    break;
 		default:
 		    goto error;
-- 
cgit v1.2.3


From 43718d3b81d7f3d08e25047e22d579801bbe5044 Mon Sep 17 00:00:00 2001
From: Hans Nilsson <hans@erlang.org>
Date: Wed, 14 Jun 2017 15:36:21 +0200
Subject: Update copyright year

---
 erts/emulator/beam/beam_load.h                            | 2 +-
 erts/emulator/beam/big.c                                  | 2 +-
 erts/emulator/beam/big.h                                  | 2 +-
 erts/emulator/beam/erl_alloc.h                            | 2 +-
 erts/emulator/beam/erl_alloc_util.h                       | 2 +-
 erts/emulator/beam/erl_cpu_topology.c                     | 2 +-
 erts/emulator/beam/erl_cpu_topology.h                     | 2 +-
 erts/emulator/beam/erl_lock_count.c                       | 2 +-
 erts/emulator/beam/erl_port.h                             | 2 +-
 erts/emulator/beam/erl_port_task.c                        | 2 +-
 erts/emulator/beam/erl_port_task.h                        | 2 +-
 erts/emulator/beam/erl_smp.h                              | 2 +-
 erts/emulator/beam/erl_threads.h                          | 2 +-
 erts/emulator/beam/ops.tab                                | 2 +-
 erts/emulator/test/a_SUITE.erl                            | 2 +-
 erts/emulator/test/after_SUITE.erl                        | 2 +-
 erts/emulator/test/bs_construct_SUITE.erl                 | 2 +-
 erts/emulator/test/busy_port_SUITE.erl                    | 2 +-
 erts/emulator/test/code_SUITE.erl                         | 2 +-
 erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c | 2 +-
 erts/emulator/test/driver_SUITE.erl                       | 2 +-
 erts/emulator/test/efile_SUITE.erl                        | 2 +-
 erts/emulator/test/evil_SUITE.erl                         | 2 +-
 erts/emulator/test/exception_SUITE.erl                    | 2 +-
 erts/emulator/test/gc_SUITE.erl                           | 2 +-
 erts/emulator/test/hipe_SUITE.erl                         | 2 +-
 erts/emulator/test/long_timers_test.erl                   | 2 +-
 erts/emulator/test/lttng_SUITE.erl                        | 2 +-
 erts/emulator/test/message_queue_data_SUITE.erl           | 2 +-
 erts/emulator/test/monitor_SUITE.erl                      | 2 +-
 erts/emulator/test/mtx_SUITE.erl                          | 2 +-
 erts/emulator/test/nested_SUITE.erl                       | 2 +-
 erts/emulator/test/port_trace_SUITE.erl                   | 2 +-
 erts/emulator/test/process_SUITE.erl                      | 2 +-
 erts/emulator/test/receive_SUITE.erl                      | 2 +-
 erts/emulator/test/signal_SUITE.erl                       | 2 +-
 erts/emulator/test/system_profile_SUITE.erl               | 2 +-
 erts/emulator/test/timer_bif_SUITE.erl                    | 2 +-
 erts/emulator/test/trace_SUITE.erl                        | 2 +-
 erts/emulator/test/trace_bif_SUITE.erl                    | 2 +-
 erts/emulator/test/trace_local_SUITE.erl                  | 2 +-
 erts/emulator/test/trace_meta_SUITE.erl                   | 2 +-
 erts/emulator/test/trace_nif_SUITE.erl                    | 2 +-
 erts/emulator/test/trace_port_SUITE.erl                   | 2 +-
 erts/emulator/test/tracer_SUITE.erl                       | 2 +-
 erts/emulator/test/unique_SUITE.erl                       | 2 +-
 erts/emulator/utils/beam_makeops                          | 2 +-
 47 files changed, 47 insertions(+), 47 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index b8d8634e28..c088bdb751 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 1f6feade1c..7128b8ed23 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index 258038a157..48efce20e7 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 758d529f87..7b5cbe2178 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 7a96bd0319..e889980fa4 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
index d0fd763798..50f33b2014 100644
--- a/erts/emulator/beam/erl_cpu_topology.c
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2010-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h
index cf139d95a9..c922214702 100644
--- a/erts/emulator/beam/erl_cpu_topology.h
+++ b/erts/emulator/beam/erl_cpu_topology.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2010-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index aee9796171..678bc43f04 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index 5c947ad1c0..6a3213ec52 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 55526e1d5e..a044de3fee 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2006-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index ab536c6f27..9cca62ffaf 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2017. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index 55ba943bdd..181736b009 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 28ff5d3a42..9612b70469 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 8abe871c14..cdf9cb58b9 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1997-2016. All Rights Reserved.
+# Copyright Ericsson AB 1997-2017. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/a_SUITE.erl b/erts/emulator/test/a_SUITE.erl
index 9cc86014ec..5b04a15b85 100644
--- a/erts/emulator/test/a_SUITE.erl
+++ b/erts/emulator/test/a_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/after_SUITE.erl b/erts/emulator/test/after_SUITE.erl
index 6409f0b336..8a34195e8d 100644
--- a/erts/emulator/test/after_SUITE.erl
+++ b/erts/emulator/test/after_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl
index 779d81daa5..b79f4b995d 100644
--- a/erts/emulator/test/bs_construct_SUITE.erl
+++ b/erts/emulator/test/bs_construct_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/busy_port_SUITE.erl b/erts/emulator/test/busy_port_SUITE.erl
index 644848daf5..4e7004a424 100644
--- a/erts/emulator/test/busy_port_SUITE.erl
+++ b/erts/emulator/test/busy_port_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index ab0fc0d42c..77321aa50f 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
index 1ab39466db..0321b9898f 100644
--- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
+++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 954f3b137f..6810729285 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl
index eaf6b58c33..f0e1bcf04b 100644
--- a/erts/emulator/test/efile_SUITE.erl
+++ b/erts/emulator/test/efile_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/evil_SUITE.erl b/erts/emulator/test/evil_SUITE.erl
index fb1954ce37..fc4ac037ac 100644
--- a/erts/emulator/test/evil_SUITE.erl
+++ b/erts/emulator/test/evil_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2002-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl
index ad6d8c890f..aaca522da6 100644
--- a/erts/emulator/test/exception_SUITE.erl
+++ b/erts/emulator/test/exception_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/gc_SUITE.erl b/erts/emulator/test/gc_SUITE.erl
index 2c2cb9c32d..f3942ef416 100644
--- a/erts/emulator/test/gc_SUITE.erl
+++ b/erts/emulator/test/gc_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/hipe_SUITE.erl b/erts/emulator/test/hipe_SUITE.erl
index 5083f01f34..e62d4260f6 100644
--- a/erts/emulator/test/hipe_SUITE.erl
+++ b/erts/emulator/test/hipe_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2016. All Rights Reserved.
+%% Copyright Ericsson AB 2016-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/long_timers_test.erl b/erts/emulator/test/long_timers_test.erl
index 896ac5e1f6..de1a6e6d32 100644
--- a/erts/emulator/test/long_timers_test.erl
+++ b/erts/emulator/test/long_timers_test.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/lttng_SUITE.erl b/erts/emulator/test/lttng_SUITE.erl
index 1c1952f912..a012fa1da2 100644
--- a/erts/emulator/test/lttng_SUITE.erl
+++ b/erts/emulator/test/lttng_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/message_queue_data_SUITE.erl b/erts/emulator/test/message_queue_data_SUITE.erl
index 0d5f4aa0af..7f0cbdd885 100644
--- a/erts/emulator/test/message_queue_data_SUITE.erl
+++ b/erts/emulator/test/message_queue_data_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2014-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2014-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/monitor_SUITE.erl b/erts/emulator/test/monitor_SUITE.erl
index ed3ab23738..9d772480d9 100644
--- a/erts/emulator/test/monitor_SUITE.erl
+++ b/erts/emulator/test/monitor_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/mtx_SUITE.erl b/erts/emulator/test/mtx_SUITE.erl
index 0d6ab5cdb2..843e917dfc 100644
--- a/erts/emulator/test/mtx_SUITE.erl
+++ b/erts/emulator/test/mtx_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/nested_SUITE.erl b/erts/emulator/test/nested_SUITE.erl
index 5059317172..f1b4c03ae4 100644
--- a/erts/emulator/test/nested_SUITE.erl
+++ b/erts/emulator/test/nested_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/port_trace_SUITE.erl b/erts/emulator/test/port_trace_SUITE.erl
index bfc3c8cb51..c78dc754a9 100644
--- a/erts/emulator/test/port_trace_SUITE.erl
+++ b/erts/emulator/test/port_trace_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index b993bf5c69..6ded7ff1c9 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/receive_SUITE.erl b/erts/emulator/test/receive_SUITE.erl
index 26eddfd03c..a12019ec83 100644
--- a/erts/emulator/test/receive_SUITE.erl
+++ b/erts/emulator/test/receive_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/signal_SUITE.erl b/erts/emulator/test/signal_SUITE.erl
index d788360812..f1d11d1814 100644
--- a/erts/emulator/test/signal_SUITE.erl
+++ b/erts/emulator/test/signal_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/system_profile_SUITE.erl b/erts/emulator/test/system_profile_SUITE.erl
index 3a8f862b05..9b678fcff9 100644
--- a/erts/emulator/test/system_profile_SUITE.erl
+++ b/erts/emulator/test/system_profile_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2007-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2007-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/timer_bif_SUITE.erl b/erts/emulator/test/timer_bif_SUITE.erl
index a977eb41c4..fc11a04a31 100644
--- a/erts/emulator/test/timer_bif_SUITE.erl
+++ b/erts/emulator/test/timer_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1998-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl
index 9ddc8628b6..72acd33033 100644
--- a/erts/emulator/test/trace_SUITE.erl
+++ b/erts/emulator/test/trace_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/trace_bif_SUITE.erl b/erts/emulator/test/trace_bif_SUITE.erl
index ce9c60d9d9..f12c359874 100644
--- a/erts/emulator/test/trace_bif_SUITE.erl
+++ b/erts/emulator/test/trace_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1998-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/trace_local_SUITE.erl b/erts/emulator/test/trace_local_SUITE.erl
index 1cbe6201c3..253d5fed23 100644
--- a/erts/emulator/test/trace_local_SUITE.erl
+++ b/erts/emulator/test/trace_local_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2000-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2000-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/trace_meta_SUITE.erl b/erts/emulator/test/trace_meta_SUITE.erl
index ef58978749..f157a6c9eb 100644
--- a/erts/emulator/test/trace_meta_SUITE.erl
+++ b/erts/emulator/test/trace_meta_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2002-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/trace_nif_SUITE.erl b/erts/emulator/test/trace_nif_SUITE.erl
index 5c35a8a83f..f796b9d667 100644
--- a/erts/emulator/test/trace_nif_SUITE.erl
+++ b/erts/emulator/test/trace_nif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/trace_port_SUITE.erl b/erts/emulator/test/trace_port_SUITE.erl
index fa6460073a..c85a77536e 100644
--- a/erts/emulator/test/trace_port_SUITE.erl
+++ b/erts/emulator/test/trace_port_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/tracer_SUITE.erl b/erts/emulator/test/tracer_SUITE.erl
index c2fa3afd9b..ab7d047bc3 100644
--- a/erts/emulator/test/tracer_SUITE.erl
+++ b/erts/emulator/test/tracer_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/unique_SUITE.erl b/erts/emulator/test/unique_SUITE.erl
index ab02531de3..cfc37bd44f 100644
--- a/erts/emulator/test/unique_SUITE.erl
+++ b/erts/emulator/test/unique_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2014-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2014-2017. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 05cd48d434..0a30553f71 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -2,7 +2,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1998-2016. All Rights Reserved.
+# Copyright Ericsson AB 1998-2017. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
-- 
cgit v1.2.3


From 5337eff8e86e352eb5ca63cc07bb9880a4f5c9ec Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 14 Jun 2017 16:21:56 +0200
Subject: Fix minor vsn 1 in term_to_binary/2 broken in this branch

---
 erts/emulator/beam/external.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 95275847f8..1560844521 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1131,6 +1131,7 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
 		    break;
 		case 1: /* Current default... */
 		    flags = TERM_TO_BINARY_DFLAGS;
+                    break;
                 case 2:
                     flags = TERM_TO_BINARY_DFLAGS | DFLAG_UTF8_ATOMS;
 		    break;
-- 
cgit v1.2.3


From 5a42fa760c2d908c0724a6e49a1f25a82b5b7792 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 14 Jun 2017 17:22:13 +0200
Subject: erts: Add tests to detect port close race

---
 erts/emulator/test/port_SUITE.erl             | 40 +++++++++++++++++++++++++++
 erts/emulator/test/port_SUITE_data/echo_drv.c | 31 ++++++++++++++++++---
 2 files changed, 67 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index 23594aa8c4..3a2243f553 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -86,6 +86,7 @@
     cd_relative/1,
     close_deaf_port/1,
     count_fds/1,
+    dropped_commands/1,
     dying_port/1,
     env/1,
     eof/1,
@@ -548,6 +549,45 @@ make_dying_port(Config) when is_list(Config) ->
     Command = lists:concat([PortTest, " -h0 -d -q"]),
     open_port({spawn, Command}, [stream]).
 
+%% Test that dropped port_commands work correctly.
+%% This used to cause a segfault.
+%%
+%% This testcase creates a port and then lets many processes
+%% do parallel commands to it. After a while it closes the
+%% port and we are trying to catch the race when doing a
+%% command while the port is closing.
+dropped_commands(Config) ->
+    %% Test with output callback
+    dropped_commands(Config, false, {self(), {command, "1"}}),
+    %% Test with outputv callback
+    dropped_commands(Config, true, {self(), {command, "1"}}).
+
+dropped_commands(Config, Outputv, Cmd) ->
+    Path = proplists:get_value(data_dir, Config),
+    os:putenv("ECHO_DRV_USE_OUTPUTV", atom_to_list(Outputv)),
+    ok = load_driver(Path, "echo_drv"),
+    [dropped_commands_test(Cmd) || _ <- lists:seq(1, 100)],
+    timer:sleep(100),
+    erl_ddll:unload_driver("echo_drv"),
+    ok.
+
+dropped_commands_test(Cmd) ->
+    Port = erlang:open_port({spawn_driver, "echo_drv"}, [{parallelism, true}]),
+    spawn_monitor(
+      fun() ->
+              [spawn_link(fun() -> spin(Port, Cmd) end) || _ <- lists:seq(1,8)],
+              timer:sleep(5),
+              port_close(Port),
+              timer:sleep(5),
+              exit(nok)
+      end),
+    receive _M -> timer:sleep(5) end.
+
+spin(P, Cmd) ->
+    P ! Cmd,
+    spin(P, Cmd).
+
+
 %% Tests that port program with complete path (but without any
 %% .exe extension) can be started, even if there is a file with
 %% the same name but without the extension in the same directory.
diff --git a/erts/emulator/test/port_SUITE_data/echo_drv.c b/erts/emulator/test/port_SUITE_data/echo_drv.c
index 1d39c6a00c..b4370f6455 100644
--- a/erts/emulator/test/port_SUITE_data/echo_drv.c
+++ b/erts/emulator/test/port_SUITE_data/echo_drv.c
@@ -18,8 +18,11 @@ typedef struct _erl_drv_data EchoDrvData;
 
 static EchoDrvData *echo_drv_start(ErlDrvPort port, char *command);
 static void         echo_drv_stop(EchoDrvData *data_p);
-static void         echo_drv_output(ErlDrvData drv_data, char *buf,
-				    ErlDrvSizeT len);
+static void         echo_drv_output(ErlDrvData drv_data, char *buf, ErlDrvSizeT len);
+static ErlDrvSSizeT echo_control(ErlDrvData drv_data,
+                                 unsigned int command, char *buf,
+                                 ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen);
+static void         echo_outputv(ErlDrvData drv_data, ErlIOVec *ev);
 static void         echo_drv_finish(void);
 
 static ErlDrvEntry echo_drv_entry = { 
@@ -32,9 +35,9 @@ static ErlDrvEntry echo_drv_entry = {
     "echo_drv",
     echo_drv_finish,
     NULL, /* handle */
-    NULL, /* control */
+    echo_control, /* control */
     NULL, /* timeout */
-    NULL, /* outputv */
+    echo_outputv, /* outputv */
     NULL, /* ready_async */
     NULL,
     NULL,
@@ -56,6 +59,14 @@ static ErlDrvEntry echo_drv_entry = {
 
 DRIVER_INIT(echo_drv)
 {
+    char buf[10];
+    size_t bufsz = sizeof(buf);
+    char *use_outputv;
+    use_outputv = (erl_drv_getenv("ECHO_DRV_USE_OUTPUTV", buf, &bufsz) == 0
+		   ? buf
+		   : "false");
+    if (strcmp(use_outputv, "true") != 0)
+	echo_drv_entry.outputv = NULL;
     return &echo_drv_entry;
 }
 
@@ -87,3 +98,15 @@ static void echo_drv_output(ErlDrvData drv_data, char *buf, ErlDrvSizeT len) {
 
 static void echo_drv_finish() {
 }
+
+static ErlDrvSSizeT echo_control(ErlDrvData drv_data,
+                                 unsigned int command, char *buf,
+                                 ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen)
+{
+    return 0;
+}
+
+static void echo_outputv(ErlDrvData drv_data, ErlIOVec *ev)
+{
+    return;
+}
-- 
cgit v1.2.3


From 37faaedc1d7aedf66c29130794e8339b96c051d8 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 14 Jun 2017 17:30:02 +0200
Subject: fixup! erts: Cleanup dropped port tasks correctly

---
 erts/emulator/beam/erl_port_task.c | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 8d78b22228..2083a43f69 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -1624,8 +1624,9 @@ abort_nosuspend:
 
     ASSERT(ns_pthlp);
     erts_free(ERTS_ALC_T_PT_HNDL_LIST, ns_pthlp);
-    if (ptp)
-	port_task_free(ptp);
+
+    ASSERT(ptp);
+    port_task_free(ptp);
 
     return 0;
 
@@ -1636,15 +1637,15 @@ fail:
 	erts_port_dec_refc(pp);
 #endif
 
-    abort_signal_task(pp, ERTS_PROC2PORT_SIG_ABORT,
-                      ptp->type, &ptp->u.alive.td, 0);
+    if (ptp) {
+        abort_signal_task(pp, ERTS_PROC2PORT_SIG_ABORT,
+                          ptp->type, &ptp->u.alive.td, 0);
+	port_task_free(ptp);
+    }
 
     if (ns_pthlp)
 	erts_free(ERTS_ALC_T_PT_HNDL_LIST, ns_pthlp);
 
-    if (ptp)
-	port_task_free(ptp);
-
     return -1;
 }
 
-- 
cgit v1.2.3


From b2f7b51f420a233a41c885d8b44919d88f1e7ba5 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 14 Jun 2017 11:57:13 +0200
Subject: erts: Must have main lock when flushing trace messages

If we don't have the main lock, multiple threads may come in and
do the flush at the same time which will lead to all kinds of
strang problems.
---
 erts/emulator/beam/erl_nif.c     |  4 ++++
 erts/emulator/beam/erl_process.c | 18 +++++++++---------
 2 files changed, 13 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 4815e5e7bb..8ed429b214 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -588,6 +588,10 @@ int erts_flush_trace_messages(Process *c_p, ErtsProcLocks c_p_locks)
     ErlTraceMessageQueue *msgq, **last_msgq;
     int reds = 0;
 
+    /* Only one thread at a time is allowed to flush trace messages,
+       so we require the main lock to be held when doing the flush */
+    ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p);
+
     erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_TRACE);
 
     msgq = c_p->trace_msg_q;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index fc2b34e70f..0ff9484daf 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11768,9 +11768,11 @@ flush_dirty_trace_messages(void *vpid)
     erts_free(ERTS_ALC_T_DIRTY_SL, vpid);
 #endif
 
-    proc = erts_proc_lookup(pid);
-    if (proc)
-	(void) erts_flush_trace_messages(proc, 0);
+    proc = erts_pid2proc_opt(NULL, 0, pid, ERTS_PROC_LOCK_MAIN, 0);
+    if (proc) {
+	(void) erts_flush_trace_messages(proc, ERTS_PROC_LOCK_MAIN);
+        erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
+    }
 }
 
 #endif /* ERTS_DIRTY_SCHEDULERS */
@@ -14102,8 +14104,11 @@ erts_continue_exit_process(Process *p)
 								 have none here */
     }
 
+    erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+    ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
+
 #ifdef ERTS_SMP
-    erts_flush_trace_messages(p, 0);
+    erts_flush_trace_messages(p, ERTS_PROC_LOCK_MAIN);
 #endif
 
     ERTS_TRACER_CLEAR(&ERTS_TRACER(p));
@@ -14111,11 +14116,6 @@ erts_continue_exit_process(Process *p)
     if (!delay_del_proc)
 	delete_process(p);
 
-#ifdef ERTS_SMP
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
-    ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
-#endif
-
     return;
 
  yield:
-- 
cgit v1.2.3


From 1f1c7a90ff5bacf19f0437e6f54cfe97e2d25e97 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 22 Jun 2017 16:11:52 +0200
Subject: Fix statistics(wall_clock) and statistics(runtime) implementation

---
 erts/emulator/beam/erl_bif_info.c | 26 +++++++++++++-------
 erts/emulator/beam/erl_time_sup.c | 50 ++++++++++++++++++++++-----------------
 erts/emulator/beam/erl_utils.h    |  1 +
 erts/emulator/beam/sys.h          |  8 +++----
 4 files changed, 50 insertions(+), 35 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 7bd45916f5..7791231d56 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3468,24 +3468,32 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
 	res = TUPLE2(hp, b1, b2); 
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_runtime) {
-	UWord u1, u2, dummy;
+	ErtsMonotonicTime u1, u2;
 	Eterm b1, b2;
-	elapsed_time_both(&u1,&dummy,&u2,&dummy);
-	b1 = erts_make_integer(u1,BIF_P);
-	b2 = erts_make_integer(u2,BIF_P);
-	hp = HAlloc(BIF_P,3);
+        Uint hsz;
+	elapsed_time_both(&u1, NULL, &u2, NULL);
+        hsz = 3; /* 2-tuple */
+        (void) erts_bld_monotonic_time(NULL, &hsz, u1);
+        (void) erts_bld_monotonic_time(NULL, &hsz, u2);
+	hp = HAlloc(BIF_P, hsz);
+        b1 = erts_bld_monotonic_time(&hp, NULL, u1);
+        b2 = erts_bld_monotonic_time(&hp, NULL, u2);
 	res = TUPLE2(hp, b1, b2);
 	BIF_RET(res);
     } else if (BIF_ARG_1 ==  am_run_queue) {
 	res = erts_run_queues_len(NULL, 1, 0);
 	BIF_RET(make_small(res));
     } else if (BIF_ARG_1 == am_wall_clock) {
-	UWord w1, w2;
+	ErtsMonotonicTime w1, w2;
 	Eterm b1, b2;
+        Uint hsz;
 	wall_clock_elapsed_time_both(&w1, &w2);
-	b1 = erts_make_integer((Uint) w1,BIF_P);
-	b2 = erts_make_integer((Uint) w2,BIF_P);
-	hp = HAlloc(BIF_P,3);
+        hsz = 3; /* 2-tuple */
+        (void) erts_bld_monotonic_time(NULL, &hsz, w1);
+        (void) erts_bld_monotonic_time(NULL, &hsz, w2);
+	hp = HAlloc(BIF_P, hsz);
+        b1 = erts_bld_monotonic_time(&hp, NULL, w1);
+        b2 = erts_bld_monotonic_time(&hp, NULL, w2);
 	res = TUPLE2(hp, b1, b2);
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_io) {
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 6aa2a7500f..1ef8c1b73a 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1289,56 +1289,62 @@ erts_finalize_time_offset(void)
 /* info functions */
 
 void 
-elapsed_time_both(UWord *ms_user, UWord *ms_sys, 
-		  UWord *ms_user_diff, UWord *ms_sys_diff)
+elapsed_time_both(ErtsMonotonicTime *ms_user, ErtsMonotonicTime *ms_sys, 
+		  ErtsMonotonicTime *ms_user_diff, ErtsMonotonicTime *ms_sys_diff)
 {
-    UWord prev_total_user, prev_total_sys;
-    UWord total_user, total_sys;
+    ErtsMonotonicTime prev_total_user, prev_total_sys;
+    ErtsMonotonicTime total_user, total_sys;
     SysTimes now;
 
     sys_times(&now);
-    total_user = (now.tms_utime * 1000) / SYS_CLK_TCK;
-    total_sys = (now.tms_stime * 1000) / SYS_CLK_TCK;
+    total_user = (ErtsMonotonicTime) ((now.tms_utime * 1000) / SYS_CLK_TCK);
+    total_sys = (ErtsMonotonicTime) ((now.tms_stime * 1000) / SYS_CLK_TCK);
 
     if (ms_user != NULL)
 	*ms_user = total_user;
     if (ms_sys != NULL)
 	*ms_sys = total_sys;
 
-    erts_smp_mtx_lock(&erts_timeofday_mtx);
+    if (ms_user_diff || ms_sys_diff) {
+        erts_smp_mtx_lock(&erts_timeofday_mtx);
     
-    prev_total_user = (t_start.tms_utime * 1000) / SYS_CLK_TCK;
-    prev_total_sys = (t_start.tms_stime * 1000) / SYS_CLK_TCK;
-    t_start = now;
+        prev_total_user = (ErtsMonotonicTime) ((t_start.tms_utime * 1000) / SYS_CLK_TCK);
+        prev_total_sys = (ErtsMonotonicTime) ((t_start.tms_stime * 1000) / SYS_CLK_TCK);
+        t_start = now;
     
-    erts_smp_mtx_unlock(&erts_timeofday_mtx);
+        erts_smp_mtx_unlock(&erts_timeofday_mtx);
 
-    if (ms_user_diff != NULL)
-	*ms_user_diff = total_user - prev_total_user;
+        if (ms_user_diff != NULL)
+            *ms_user_diff = total_user - prev_total_user;
 	  
-    if (ms_sys_diff != NULL)
-	*ms_sys_diff = total_sys - prev_total_sys;
+        if (ms_sys_diff != NULL)
+            *ms_sys_diff = total_sys - prev_total_sys;
+    }
 }
 
 
 /* wall clock routines */
 
 void 
-wall_clock_elapsed_time_both(UWord *ms_total, UWord *ms_diff)
+wall_clock_elapsed_time_both(ErtsMonotonicTime *ms_total, ErtsMonotonicTime *ms_diff)
 {
     ErtsMonotonicTime now, elapsed;
 
-    erts_smp_mtx_lock(&erts_timeofday_mtx);
-
     now = time_sup.r.o.get_time();
     update_last_mtime(NULL, now);
 
     elapsed = ERTS_MONOTONIC_TO_MSEC(now);
-    *ms_total = (UWord) elapsed;
-    *ms_diff = (UWord) (elapsed - prev_wall_clock_elapsed);
-    prev_wall_clock_elapsed = elapsed;
 
-    erts_smp_mtx_unlock(&erts_timeofday_mtx);
+    *ms_total = elapsed;
+
+    if (ms_diff) {
+        erts_smp_mtx_lock(&erts_timeofday_mtx);
+
+        *ms_diff = elapsed - prev_wall_clock_elapsed;
+        prev_wall_clock_elapsed = elapsed;
+
+        erts_smp_mtx_unlock(&erts_timeofday_mtx);
+    }
 }
 
 /* get current time */
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 81800752f0..b0912a346d 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -132,6 +132,7 @@ Eterm erts_bld_uint(Uint **hpp, Uint *szp, Uint ui);
 Eterm erts_bld_uword(Uint **hpp, Uint *szp, UWord uw);
 Eterm erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64);
 Eterm erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64);
+#define erts_bld_monotonic_time erts_bld_sint64
 Eterm erts_bld_cons(Uint **hpp, Uint *szp, Eterm car, Eterm cdr);
 Eterm erts_bld_tuple(Uint **hpp, Uint *szp, Uint arity, ...);
 #define erts_bld_tuple2(H,S,E1,E2) erts_bld_tuple(H,S,2,E1,E2)
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index dd4f05686b..950583bbfa 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -790,10 +790,10 @@ Preload* sys_preloaded(void);
 unsigned char* sys_preload_begin(Preload*);
 void sys_preload_end(Preload*);
 int sys_get_key(int);
-void elapsed_time_both(UWord *ms_user, UWord *ms_sys, 
-		       UWord *ms_user_diff, UWord *ms_sys_diff);
-void wall_clock_elapsed_time_both(UWord *ms_total, 
-				  UWord *ms_diff);
+void elapsed_time_both(ErtsMonotonicTime *ms_user, ErtsMonotonicTime *ms_sys, 
+		       ErtsMonotonicTime *ms_user_diff, ErtsMonotonicTime *ms_sys_diff);
+void wall_clock_elapsed_time_both(ErtsMonotonicTime *ms_total, 
+				  ErtsMonotonicTime *ms_diff);
 void get_time(int *hour, int *minute, int *second);
 void get_date(int *year, int *month, int *day);
 void get_localtime(int *year, int *month, int *day, 
-- 
cgit v1.2.3


From 42955cbdf3b4e2f378602e16cb373970d2ac5749 Mon Sep 17 00:00:00 2001
From: Boris Bochkarev <Boris-Bochkaryov@yandex.ru>
Date: Fri, 23 Jun 2017 16:01:05 +0300
Subject: Support e2k architecture

---
 erts/emulator/beam/sys.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index d752ea4330..b87446b2fb 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -21,7 +21,7 @@
 #ifndef __SYS_H__
 #define __SYS_H__
 
-#if !defined(__GNUC__)
+#if !defined(__GNUC__) || defined(__e2k__)
 #  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
 #elif !defined(__GNUC_MINOR__)
 #  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
-- 
cgit v1.2.3


From f405b07bc6136c76a016da67dc65e5c7ec9d8f9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 26 Jun 2017 10:30:54 +0200
Subject: Eliminate potential unsafe use of general destination

a3407eaa2104d6 eliminated the -gen_dest flag for macros in ops.tab.

It turns out that the new implementation (taking the address of the
X or X destination register) is unsafe if the destination is a Y
register and there can be a GC. The problem is that the address to
the Y register will change if there is a GC.

Fortunately, the few instructions in OTP 20 that have a general
destinations are safe. The put_list_ssd instruction never does a GC.
The bit syntax instructions that may do a GC will always store the
result to an X register.

To be completely sure, rewrite the destination register from 'd' to
'x' for the bit syntax instructions. That means that a bit syntax
instruction with a Y register destionation will abort the loading
if it is encountered.
---
 erts/emulator/beam/ops.tab | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index cdf9cb58b9..44613c7d85 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1174,9 +1174,9 @@ bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \
 %macro: i_bs_get_binary2 BsGetBinary_2 -fail_action
 %macro: i_bs_get_binary_all2 BsGetBinaryAll_2 -fail_action
 
-i_bs_get_binary_imm2 f x I I I d
-i_bs_get_binary2 f x I s I d
-i_bs_get_binary_all2 f x I I d
+i_bs_get_binary_imm2 f x I I I x
+i_bs_get_binary2 f x I s I x
+i_bs_get_binary_all2 f x I I x
 i_bs_get_binary_all_reuse x f I
 
 # Fetching float from binaries.
@@ -1186,7 +1186,7 @@ bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \
 bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail
 
 %macro: i_bs_get_float2 BsGetFloat2 -fail_action
-i_bs_get_float2 f x I s I d
+i_bs_get_float2 f x I s I x
 
 # Miscellanous
 
-- 
cgit v1.2.3


From 86bcdcb3db79ac2d1faa387bbe194924e966dd69 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 27 Jun 2017 17:08:26 +0200
Subject: erts: Make apply throw 'badarg' if Args is not a list

instead of a strange 'undef' exception.
---
 erts/emulator/beam/beam_emu.c    |  2 +-
 erts/emulator/test/fun_SUITE.erl | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 79d751d13e..bc83699951 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6829,7 +6829,7 @@ apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg)
     }
 
     if (is_not_nil(tmp)) {	/* Must be well-formed list */
-	p->freason = EXC_UNDEF;
+	p->freason = EXC_BADARG;
 	return NULL;
     }
     reg[arity] = fun;
diff --git a/erts/emulator/test/fun_SUITE.erl b/erts/emulator/test/fun_SUITE.erl
index e4640909aa..7d29ebec52 100644
--- a/erts/emulator/test/fun_SUITE.erl
+++ b/erts/emulator/test/fun_SUITE.erl
@@ -22,6 +22,7 @@
 
 -export([all/0, suite/0,
 	 bad_apply/1,bad_fun_call/1,badarity/1,ext_badarity/1,
+         bad_arglist/1,
 	 equality/1,ordering/1,
 	 fun_to_port/1,t_phash/1,t_phash2/1,md5/1,
 	 refc/1,refc_ets/1,refc_dist/1,
@@ -39,6 +40,7 @@ suite() ->
 
 all() ->
     [bad_apply, bad_fun_call, badarity, ext_badarity,
+     bad_arglist,
      equality, ordering, fun_to_port, t_phash,
      t_phash2, md5, refc, refc_ets, refc_dist,
      const_propagation, t_arity, t_is_function2, t_fun_info,
@@ -107,6 +109,18 @@ bad_call_fc(Fun) ->
 	    ct:fail({bad_result,Other})
     end.
 
+% Test erlang:apply with non-proper arg-list
+bad_arglist(Config) when is_list(Config) ->
+    Fun = fun(A,B) -> A+B end,
+    {'EXIT', {badarg,_}} = (catch apply(Fun, 17)),
+    {'EXIT', {badarg,_}} = (catch apply(Fun, [17|18])),
+    {'EXIT', {badarg,_}} = (catch apply(Fun, [17,18|19])),
+    {'EXIT', {badarg,_}} = (catch apply(lists,seq, 17)),
+    {'EXIT', {badarg,_}} = (catch apply(lists,seq, [17|18])),
+    {'EXIT', {badarg,_}} = (catch apply(lists,seq, [17,18|19])),
+    ok.
+
+
 %% Call and apply valid funs with wrong number of arguments.
 
 badarity(Config) when is_list(Config) ->
-- 
cgit v1.2.3


From 59433fce199cf65fdbde72e419adae159ef72935 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 26 Jun 2017 11:52:12 +0200
Subject: ops.tab: Use combined operands

---
 erts/emulator/beam/ops.tab | 46 ++++++++++++++--------------------------------
 1 file changed, 14 insertions(+), 32 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 44613c7d85..8d13b9073e 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -160,26 +160,19 @@ is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \
 select_tuple_arity S=d Fail=f Size=u Rest=* => \
   gen_select_tuple_arity(S, Fail, Size, Rest)
 
-i_select_val_bins x f I
-i_select_val_bins y f I
+i_select_val_bins xy f I
 
-i_select_val_lins x f I
-i_select_val_lins y f I
+i_select_val_lins xy f I
 
-i_select_val2 x f c c f f
-i_select_val2 y f c c f f
+i_select_val2 xy f c c f f
 
-i_select_tuple_arity x f I
-i_select_tuple_arity y f I
+i_select_tuple_arity xy f I
 
-i_select_tuple_arity2 x f A A f f
-i_select_tuple_arity2 y f A A f f
+i_select_tuple_arity2 xy f A A f f
 
-i_jump_on_val_zero x f I
-i_jump_on_val_zero y f I
+i_jump_on_val_zero xy f I
 
-i_jump_on_val x f I I
-i_jump_on_val y f I I
+i_jump_on_val xy f I I
 
 %macro: get_list GetList -pack
 get_list xy xy xy
@@ -448,19 +441,14 @@ is_ne_exact Lbl R=xy C=ian => i_is_ne_exact_immed Lbl R C
 is_ne_exact Lbl R=xy C=q => i_is_ne_exact_literal Lbl R C
 
 %macro: i_is_eq_exact_immed EqualImmed -fail_action
-i_is_eq_exact_immed f r c
-i_is_eq_exact_immed f x c
-i_is_eq_exact_immed f y c
 
-i_is_eq_exact_literal f x c
-i_is_eq_exact_literal f y c
+i_is_eq_exact_immed f rxy c
+i_is_eq_exact_literal f xy c
 
 %macro: i_is_ne_exact_immed NotEqualImmed -fail_action
-i_is_ne_exact_immed f x c
-i_is_ne_exact_immed f y c
+i_is_ne_exact_immed f xy c
 
-i_is_ne_exact_literal f x c
-i_is_ne_exact_literal f y c
+i_is_ne_exact_literal f xy c
 
 is_eq_exact Lbl Y=y X=x => is_eq_exact Lbl X Y
 %macro: is_eq_exact EqualExact -fail_action -pack
@@ -508,8 +496,7 @@ i_put_tuple Dst Arity Puts=* | put S => \
 i_put_tuple/2
 
 %macro:i_put_tuple PutTuple -pack -goto:do_put_tuple
-i_put_tuple x I
-i_put_tuple y I
+i_put_tuple xy I
 
 #
 # The instruction "put_list Const [] Dst" were generated in rare
@@ -578,17 +565,12 @@ return_trace
 move S x==0 | return => move_return S
 
 %macro: move_return MoveReturn -nonext
-move_return x
-move_return c
-move_return n
+move_return xcn
 
 move S x==0 | deallocate D | return => move_deallocate_return S D
 
 %macro: move_deallocate_return MoveDeallocateReturn -pack -nonext
-move_deallocate_return x Q
-move_deallocate_return y Q
-move_deallocate_return c Q
-move_deallocate_return n Q
+move_deallocate_return xycn Q
 
 deallocate D | return => deallocate_return D
 
-- 
cgit v1.2.3


From a5848c2c1eff90e7310627ec3d164db37c5e712a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 26 Jun 2017 13:32:21 +0200
Subject: Reduce C code size for bit syntax instructions

Bit syntax instructions never store their result in a Y register.

Therefore, change the bit syntax instructions to use 'x' as the
destination instead of 'd'. That will simplify the code that stores
the result, and will be a slight reduction in code size and execution
time.
---
 erts/emulator/beam/beam_emu.c | 103 ++++++++++++++++++++++++------------------
 erts/emulator/beam/ops.tab    |  48 ++++++++++----------
 2 files changed, 83 insertions(+), 68 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 79d751d13e..e49a408af8 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3768,33 +3768,33 @@ do {						\
      Uint alloc;
      Uint num_bytes;
 
-     OpCase(i_bs_init_bits_heap_IIId): {
+     OpCase(i_bs_init_bits_heap_IIIx): {
 	 num_bits = Arg(0);
 	 alloc = Arg(1);
 	 I++;
 	 goto do_bs_init_bits_known;
      }
      
-     OpCase(i_bs_init_bits_IId): {
+     OpCase(i_bs_init_bits_IIx): {
 	 num_bits = Arg(0);
 	 alloc = 0;
 	 goto do_bs_init_bits_known;
      }
 
-     OpCase(i_bs_init_bits_fail_heap_sIjId): {
+     OpCase(i_bs_init_bits_fail_heap_sIjIx): {
 	 GetArg1(0, num_bits_term);
 	 alloc = Arg(1);
 	 I += 2;
 	 goto do_bs_init_bits;
      }
 
-     OpCase(i_bs_init_bits_fail_yjId): {
+     OpCase(i_bs_init_bits_fail_yjIx): {
 	 num_bits_term = yb(Arg(0));
 	 I++;
 	 alloc = 0;
 	 goto do_bs_init_bits;
      }
-     OpCase(i_bs_init_bits_fail_xjId): {
+     OpCase(i_bs_init_bits_fail_xjIx): {
 	 num_bits_term = xb(Arg(0));
 	 I++;
 	 alloc = 0;
@@ -3873,7 +3873,8 @@ do {						\
 	     new_binary = make_binary(sb);
 	 }
 	 HEAP_SPACE_VERIFIED(0);
-	 StoreBifResult(2, new_binary);
+         xb(Arg(2)) = new_binary;
+         Next(3);
      } else {
 	 Binary* bptr;
 	 ProcBin* pb;
@@ -3908,21 +3909,21 @@ do {						\
  {
      Eterm BsOp1, BsOp2;
 
-     OpCase(i_bs_init_fail_heap_sIjId): {
+     OpCase(i_bs_init_fail_heap_sIjIx): {
 	 GetArg1(0, BsOp1);
 	 BsOp2 = Arg(1);
 	 I += 2;
 	 goto do_bs_init;
      }
 
-     OpCase(i_bs_init_fail_yjId): {
+     OpCase(i_bs_init_fail_yjIx): {
 	 BsOp1 = yb(Arg(0));
 	 BsOp2 = 0;
 	 I++;
 	 goto do_bs_init;
      }
 
-     OpCase(i_bs_init_fail_xjId): {
+     OpCase(i_bs_init_fail_xjIx): {
 	 BsOp1 = xb(Arg(0));
 	 BsOp2 = 0;
 	 I++;
@@ -3954,14 +3955,14 @@ do {						\
 	 }
 
 
-     OpCase(i_bs_init_heap_IIId): {
+     OpCase(i_bs_init_heap_IIIx): {
 	 BsOp1 = Arg(0);
 	 BsOp2 = Arg(1);
 	 I++;
 	 goto do_proc_bin_alloc;
      }
 
-     OpCase(i_bs_init_IId): {
+     OpCase(i_bs_init_IIx): {
 	 BsOp1 = Arg(0);
 	 BsOp2 = 0;
      }
@@ -3996,17 +3997,18 @@ do {						\
 	 
 	 OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm));
 
-	 StoreBifResult(2, make_binary(pb));
+	 xb(Arg(2)) = make_binary(pb);
+         Next(3);
      }
 
-     OpCase(i_bs_init_heap_bin_heap_IIId): {
+     OpCase(i_bs_init_heap_bin_heap_IIIx): {
 	 BsOp1 = Arg(0);
 	 BsOp2 = Arg(1);
 	 I++;
 	 goto do_heap_bin_alloc;
      }
 
-     OpCase(i_bs_init_heap_bin_IId): {
+     OpCase(i_bs_init_heap_bin_IIx): {
 	 BsOp1 = Arg(0);
 	 BsOp2 = 0;
      }
@@ -4026,11 +4028,12 @@ do {						\
 	     hb->size = BsOp1;
 	     erts_current_bin = (byte *) hb->data;
 	     BsOp1 = make_binary(hb);
-	     StoreBifResult(2, BsOp1);
+             xb(Arg(2)) = BsOp1;
+             Next(3);
 	 }
  }
 
- OpCase(bs_add_jssId): {
+ OpCase(bs_add_jssIx): {
      Eterm Op1, Op2;
      Uint Unit = Arg(3);
 
@@ -4060,7 +4063,8 @@ do {						\
 		 Op1 = erts_make_integer(Op1, c_p);
 		 HTOP = HEAP_TOP(c_p);
 	     }
-	     StoreBifResult(4, Op1);
+             xb(Arg(4)) = Op1;
+             Next(5);
 	 }
 	 goto badarg;
      } else {
@@ -4129,7 +4133,7 @@ do {						\
   * Operands: Fail ExtraHeap Live Unit Size Dst
   */
 
- OpCase(i_bs_append_jIIIsd): {
+ OpCase(i_bs_append_jIIIsx): {
      Uint live = Arg(2);
      Uint res;
      Eterm Size;
@@ -4143,13 +4147,14 @@ do {						\
 	 /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */
 	 goto lb_Cl_error;
      }
-     StoreBifResult(5, res);
+     xb(Arg(5)) = res;
+     Next(6);
  }
 
  /*
   * Operands: Fail Size Src Unit Dst
   */
- OpCase(i_bs_private_append_jIssd): {
+ OpCase(i_bs_private_append_jIssx): {
      Eterm res;
      Eterm Size, Src;
 
@@ -4159,7 +4164,8 @@ do {						\
 	 /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */
 	 goto lb_Cl_error;
      }
-     StoreBifResult(4, res);
+     xb(Arg(4)) = res;
+     Next(5);
  }
 
  OpCase(bs_init_writable): {
@@ -4176,7 +4182,7 @@ do {						\
   * can get away with that because we KNOW that bs_put_utf8 will do
   * full error checking.
   */
- OpCase(i_bs_utf8_size_sd): {
+ OpCase(i_bs_utf8_size_sx): {
      Eterm arg;
      Eterm result;
 
@@ -4190,7 +4196,8 @@ do {						\
      } else {
 	 result = make_small(4);
      }
-     StoreBifResult(1, result);
+     xb(Arg(1)) = result;
+     Next(2);
  }
 
  OpCase(i_bs_put_utf8_js): {
@@ -4211,7 +4218,7 @@ do {						\
   * full error checking.
   */
 
- OpCase(i_bs_utf16_size_sd): {
+ OpCase(i_bs_utf16_size_sx): {
      Eterm arg;
      Eterm result = make_small(2);
 
@@ -4219,7 +4226,8 @@ do {						\
      if (arg >= make_small(0x10000UL)) {
 	 result = make_small(4);
      }
-     StoreBifResult(1, result);
+     xb(Arg(1)) = result;
+     Next(2);
  }
 
  OpCase(bs_put_utf16_jIs): {
@@ -4313,7 +4321,7 @@ do {						\
 		 *HTOP = HEADER_BIN_MATCHSTATE(slots);
 		 HTOP += wordsneeded;
 		 HEAP_SPACE_VERIFIED(0);
-		 StoreResult(make_matchstate(dst), Arg(3));
+                 xb(Arg(3)) = make_matchstate(dst);
 	     }
 	 } else if (is_binary_header(header)) {
 	     Eterm result;
@@ -4330,19 +4338,19 @@ do {						\
 	     if (is_non_value(result)) {
 		 ClauseFail();
 	     } else {
-		 StoreResult(result, Arg(3));
+                 xb(Arg(3)) = result;
 	     }
 	 } else {
 	     ClauseFail();
 	 }
 	 NextPF(4, next);
 
-     OpCase(i_bs_start_match2_xfIId): {
+     OpCase(i_bs_start_match2_xfIIx): {
 	 context = xb(Arg(0));
 	 I++;
 	 goto do_start_match;
      }
-     OpCase(i_bs_start_match2_yfIId): {
+     OpCase(i_bs_start_match2_yfIIx): {
 	 context = yb(Arg(0));
 	 I++;
 	 goto do_start_match;
@@ -4397,7 +4405,7 @@ do {						\
  {
      Eterm bs_get_integer8_context;
 
- OpCase(i_bs_get_integer_8_xfd): {
+ OpCase(i_bs_get_integer_8_xfx): {
 	 ErlBinMatchBuffer *_mb;
 	 Eterm _result;
 	 bs_get_integer8_context = xb(Arg(0));
@@ -4412,14 +4420,15 @@ do {						\
 	     _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]);
 	     _mb->offset += 8;
 	 }
-	 StoreBifResult(1, _result);
+         xb(Arg(1)) = _result;
+         Next(2);
      }
  }
 
  {
      Eterm bs_get_integer_16_context;
 
- OpCase(i_bs_get_integer_16_xfd):
+ OpCase(i_bs_get_integer_16_xfx):
      bs_get_integer_16_context = xb(Arg(0));
      I++;
 
@@ -4436,14 +4445,15 @@ do {						\
 	     _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset)));
 	     _mb->offset += 16;
 	 }
-	 StoreBifResult(1, _result);
+         xb(Arg(1)) = _result;
+         Next(2);
      }
  }
 
  {
      Eterm bs_get_integer_32_context;
 
- OpCase(i_bs_get_integer_32_xfId):
+ OpCase(i_bs_get_integer_32_xfIx):
      bs_get_integer_32_context = xb(Arg(0));
      I++;
 
@@ -4471,7 +4481,8 @@ do {						\
 	     HEAP_SPACE_VERIFIED(0);
 	 }
 #endif
-	 StoreBifResult(2, _result);
+         xb(Arg(2)) = _result;
+         Next(3);
      }
  }
 
@@ -4479,7 +4490,7 @@ do {						\
      Eterm Ms, Sz;
 
      /* Operands: x(Reg) Size Live Fail Flags Dst */
- OpCase(i_bs_get_integer_imm_xIIfId): {
+ OpCase(i_bs_get_integer_imm_xIIfIx): {
 	 Uint wordsneeded;
 	 Ms = xb(Arg(0));
 	 Sz = Arg(1);
@@ -4491,7 +4502,7 @@ do {						\
      }
 
      /* Operands: x(Reg) Size Fail Flags Dst */
- OpCase(i_bs_get_integer_small_imm_xIfId): {
+ OpCase(i_bs_get_integer_small_imm_xIfIx): {
 	 Ms = xb(Arg(0));
 	 Sz = Arg(1);
 	 I += 2;
@@ -4516,14 +4527,15 @@ do {						\
 	 if (is_non_value(result)) {
 	     ClauseFail();
 	 }
-	 StoreBifResult(2, result);
+         xb(Arg(2)) = result;
+         Next(3);
      }
  }
 
  /*
   * Operands: Fail Live FlagsAndUnit Ms Sz Dst
   */
- OpCase(i_bs_get_integer_fIIssd): {
+ OpCase(i_bs_get_integer_fIIssx): {
      Uint flags;
      Uint size;
      Eterm Ms;
@@ -4558,14 +4570,15 @@ do {						\
      if (is_non_value(result)) {
 	 ClauseFail();
      }
-     StoreBifResult(5, result);
+     xb(Arg(5)) = result;
+     Next(6);
  }
 
  {
      Eterm get_utf8_context;
 
      /* Operands: MatchContext Fail Dst */
- OpCase(i_bs_get_utf8_xfd): {
+ OpCase(i_bs_get_utf8_xfx): {
 	 get_utf8_context = xb(Arg(0));
 	 I++;
      }
@@ -4580,7 +4593,8 @@ do {						\
 	 if (is_non_value(result)) {
 	     ClauseFail();
 	 }
-	 StoreBifResult(1, result);
+         xb(Arg(1)) = result;
+         Next(2);
      }
  }
 
@@ -4588,7 +4602,7 @@ do {						\
      Eterm get_utf16_context;
 
      /* Operands: MatchContext Fail Flags Dst */
-     OpCase(i_bs_get_utf16_xfId): {
+     OpCase(i_bs_get_utf16_xfIx): {
 	 get_utf16_context = xb(Arg(0));
 	 I++;
      }
@@ -4603,7 +4617,8 @@ do {						\
 	 if (is_non_value(result)) {
 	     ClauseFail();
 	 }
-	 StoreBifResult(2, result);
+         xb(Arg(2)) = result;
+         Next(3);
      }
  }
 
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 8d13b9073e..4ce86ce949 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1124,7 +1124,7 @@ func_info M F A => i_func_info u M F A
 %cold
 bs_start_match2 Fail=f ica X Y D => jump Fail
 bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D
-i_bs_start_match2 xy f I I d
+i_bs_start_match2 xy f I I x
 
 bs_save2 Reg Index => gen_bs_save(Reg, Index)
 i_bs_save2 x I
@@ -1141,12 +1141,12 @@ i_bs_match_string x f I I
 bs_get_integer2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \
 			gen_get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
 
-i_bs_get_integer_small_imm x I f I d
-i_bs_get_integer_imm x I I f I d
-i_bs_get_integer f I I s s d
-i_bs_get_integer_8 x f d
-i_bs_get_integer_16 x f d
-i_bs_get_integer_32 x f I d
+i_bs_get_integer_small_imm x I f I x
+i_bs_get_integer_imm x I I f I x
+i_bs_get_integer f I I s s x
+i_bs_get_integer_8 x f x
+i_bs_get_integer_16 x f x
+i_bs_get_integer_32 x f I x
 
 # Fetching binaries from binaries.
 bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \
@@ -1204,14 +1204,14 @@ bs_context_to_binary x
 # Utf8/utf16/utf32 support. (R12B-5)
 #
 bs_get_utf8 Fail=f Ms=x u u Dst=d => i_bs_get_utf8 Ms Fail Dst
-i_bs_get_utf8 x f d
+i_bs_get_utf8 x f x
 
 bs_skip_utf8 Fail=f Ms=x u u => i_bs_get_utf8 Ms Fail x
 
 bs_get_utf16 Fail=f Ms=x u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst
 bs_skip_utf16 Fail=f Ms=x u Flags=u => i_bs_get_utf16 Ms Fail Flags x
 
-i_bs_get_utf16 x f I d
+i_bs_get_utf16 x f I x
 
 bs_get_utf32 Fail=f Ms=x Live=u Flags=u Dst=d => \
 	bs_get_integer2 Fail Ms Live i=32 u=1 Flags Dst | \
@@ -1244,15 +1244,15 @@ bs_init2 Fail Sz Words=u==0 Regs Flags Dst => \
 bs_init2 Fail Sz Words Regs Flags Dst => \
   i_bs_init_fail_heap Sz Words Fail Regs Dst
 
-i_bs_init_fail xy j I d
+i_bs_init_fail xy j I x
 
-i_bs_init_fail_heap s I j I d
+i_bs_init_fail_heap s I j I x
 
-i_bs_init I I d
-i_bs_init_heap_bin I I d
+i_bs_init I I x
+i_bs_init_heap_bin I I x
 
-i_bs_init_heap I I I d
-i_bs_init_heap_bin_heap I I I d
+i_bs_init_heap I I I x
+i_bs_init_heap_bin_heap I I I x
 
 
 bs_init_bits Fail Sz=o Words Regs Flags Dst => system_limit Fail
@@ -1265,16 +1265,16 @@ bs_init_bits Fail Sz Words=u==0 Regs Flags Dst => \
 bs_init_bits Fail Sz Words Regs Flags Dst => \
   i_bs_init_bits_fail_heap Sz Words Fail Regs Dst
 
-i_bs_init_bits_fail xy j I d
+i_bs_init_bits_fail xy j I x
 
-i_bs_init_bits_fail_heap s I j I d
+i_bs_init_bits_fail_heap s I j I x
 
-i_bs_init_bits I I d
-i_bs_init_bits_heap I I I d
+i_bs_init_bits I I x
+i_bs_init_bits_heap I I I x
 
 bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D
 
-bs_add j s s I d
+bs_add j s s I x
 
 bs_append Fail Size Extra Live Unit Bin Flags Dst => \
   move Bin x | i_bs_append Fail Extra Live Unit Size Dst
@@ -1284,8 +1284,8 @@ bs_private_append Fail Size Unit Bin Flags Dst => \
 
 bs_init_writable
 
-i_bs_append j I I I s d
-i_bs_private_append j I s s d
+i_bs_append j I I I s x
+i_bs_private_append j I s s x
 
 #
 # Storing integers into binaries.
@@ -1306,11 +1306,11 @@ i_new_bs_put_integer_imm j I I s
 
 bs_utf8_size j Src=s Dst=d => i_bs_utf8_size Src Dst
 
-i_bs_utf8_size s d
+i_bs_utf8_size s x
 
 bs_utf16_size j Src=s Dst=d => i_bs_utf16_size Src Dst
 
-i_bs_utf16_size s d
+i_bs_utf16_size s x
 
 bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src
 
-- 
cgit v1.2.3


From dbbf643c632a175a28ea39ba780947ff13eca039 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 29 Jun 2017 23:30:23 +0200
Subject: erts: Fix bug in quick alloc

The effect of the race is that a pre-allocated memory block
is inserted last without updating tail.data.last, which will cause
all subsequent insertions to also fail to update tail.data.last.

Hence all pre-allocation for this quick alloc instance is leaked
for this thread and will fallback on erts_alloc.
---
 erts/emulator/beam/erl_sched_spec_pre_alloc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
index caec24bc03..ca2d834381 100644
--- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
@@ -161,7 +161,7 @@ enqueue_remote_managed_thread(erts_sspa_chunk_header_t *chdr,
 	    if ((i & 1) == 0)
 		itmp = itmp2;
 	    else {
-		enq = (erts_sspa_blk_t *) itmp;
+		enq = (erts_sspa_blk_t *) itmp2;
 		itmp = erts_atomic_read_acqb(&enq->next_atmc);
 		ASSERT(itmp != ERTS_AINT_NULL);
 	    }
-- 
cgit v1.2.3


From 28b43fc941659a8283693b0f5de107610ad0dc1b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 30 Jun 2017 17:29:36 +0200
Subject: erts: Increase pre-allocated blocks #ifdef DEBUG

Choose a "lagom" low value to provoke both
fallback on erts_alloc
AND thread racing in lockless deallocation queue.
---
 erts/emulator/beam/erl_alloc.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index f540bae20d..89a7052894 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -407,7 +407,7 @@ NAME##_free(TYPE *p)							\
 }
 
 #ifdef DEBUG
-#define ERTS_PRE_ALLOC_SIZE(SZ) 2
+#define ERTS_PRE_ALLOC_SIZE(SZ) ((SZ) < 1000 ? (SZ)/10 + 10 : 100)
 #define ERTS_PRE_ALLOC_CLOBBER(P, T) memset((void *) (P), 0xfd, sizeof(T))
 #else
 #define ERTS_PRE_ALLOC_SIZE(SZ) ((SZ) > 1 ? (SZ) : 1)
-- 
cgit v1.2.3


From f2b332186a8cdb90b199b7121881b2ae8a454b8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Valim?= <jose.valim@plataformatec.com.br>
Date: Mon, 3 Jul 2017 14:16:13 +0200
Subject: Make tuple calls opt-in

Tuple calls is the ability to invoke a function on a tuple
as first argument:

    1> Var = dict:new().
    {dict,0,16,16,8,80,48,
          {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},
          {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]}}}
    2> Var:size().
    0

This behaviour is considered by most to be undesired and confusing,
especially when it comes to errors. For example, imagine you invoke
"Mod:new()" where a Mod is an atom and you accidentally pass {ok, dict}.
It raises:

    {undef,[{ok,new,[{ok,dict}],[]},...]}

As it attempts to invoke ok:new/1, which is really hard to debug
as there is no call to new/1 on the source code.

Furthemore, this behaviour is implemented at the VM level, which
imposes such semantics on all languages running on BEAM.

Since we cannot remove the behaviour above, this proposal makes the
behaviour opt-in with a compiler flag:

    -compile(tuple_calls).

This means that, if a codebase relies on this functionality, they
can keep compatibility by adding configuring their build tool to
always use the 'tuple_calls' flag or explicitly on each module.

As long as the compile attribute above is listed, the codebase will
work on old and new Erlang versions alike. The only downside of the
current implementation is that modules compiled on OTP 20 that rely
on 'tuple_calls' will have to be recompiled to run with 'tuple_calls'
on OTP 21+.
---
 erts/emulator/beam/beam_emu.c | 38 +++++---------------------------------
 1 file changed, 5 insertions(+), 33 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 79d751d13e..d96d496cc2 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6354,7 +6354,7 @@ BeamInstr *I, Uint stack_offset)
 {
     int arity;
     Export* ep;
-    Eterm tmp, this;
+    Eterm tmp;
 
     /*
      * Check the arguments which should be of the form apply(Module,
@@ -6377,20 +6377,8 @@ BeamInstr *I, Uint stack_offset)
 
     while (1) {
 	Eterm m, f, a;
-	/* The module argument may be either an atom or an abstract module
-	 * (currently implemented using tuples, but this might change).
-	 */
-	this = THE_NON_VALUE;
-	if (is_not_atom(module)) {
-	    Eterm* tp;
-
-	    if (is_not_tuple(module)) goto error;
-	    tp = tuple_val(module);
-	    if (arityval(tp[0]) < 1) goto error;
-	    this = module;
-	    module = tp[1];
-	    if (is_not_atom(module)) goto error;
-	}
+
+	if (is_not_atom(module)) goto error;
 
 	if (module != am_erlang || function != am_apply)
 	    break;
@@ -6425,9 +6413,7 @@ BeamInstr *I, Uint stack_offset)
     }
     /*
      * Walk down the 3rd parameter of apply (the argument list) and copy
-     * the parameters to the x registers (reg[]). If the module argument
-     * was an abstract module, add 1 to the function arity and put the
-     * module argument in the n+1st x register as a THIS reference.
+     * the parameters to the x registers (reg[]).
      */
 
     tmp = args;
@@ -6444,9 +6430,6 @@ BeamInstr *I, Uint stack_offset)
     if (is_not_nil(tmp)) {	/* Must be well-formed list */
 	goto error;
     }
-    if (this != THE_NON_VALUE) {
-        reg[arity++] = this;
-    }
 
     /*
      * Get the index into the export table, or failing that the export
@@ -6485,18 +6468,7 @@ fixed_apply(Process* p, Eterm* reg, Uint arity,
 	return 0;
     }
 
-    /* The module argument may be either an atom or an abstract module
-     * (currently implemented using tuples, but this might change).
-     */
-    if (is_not_atom(module)) {
-	Eterm* tp;
-        if (is_not_tuple(module)) goto error;
-        tp = tuple_val(module);
-        if (arityval(tp[0]) < 1) goto error;
-        module = tp[1];
-        if (is_not_atom(module)) goto error;
-        ++arity;
-    }
+    if (is_not_atom(module)) goto error;
 
     /* Handle apply of apply/3... */
     if (module == am_erlang && function == am_apply && arity == 3)
-- 
cgit v1.2.3


From 6b267b203c950db2879f254b6a9d3b7591115f9d Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 19 Jun 2017 17:06:25 +0200
Subject: Change some dist-entry types

---
 erts/emulator/beam/dist.c            | 71 +++++++++++++++++++++---------------
 erts/emulator/beam/dist.h            |  7 +---
 erts/emulator/beam/erl_node_tables.c |  4 +-
 erts/emulator/beam/erl_node_tables.h |  4 +-
 4 files changed, 48 insertions(+), 38 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 982f1066df..d9feec4722 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -176,11 +176,13 @@ Uint erts_dist_cache_size(void)
 }
 
 static ErtsProcList *
-get_suspended_on_de(DistEntry *dep, Uint32 unset_qflgs)
+get_suspended_on_de(DistEntry *dep, erts_aint32_t unset_qflgs)
 {
+    erts_aint32_t qflgs;
     ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&dep->qlock));
-    dep->qflgs &= ~unset_qflgs;
-    if (dep->qflgs & ERTS_DE_QFLG_EXIT) {
+    qflgs = erts_smp_atomic32_read_band_acqb(&dep->qflgs, ~unset_qflgs);
+    qflgs &= ~unset_qflgs;
+    if (qflgs & ERTS_DE_QFLG_EXIT) {
 	/* No resume when exit has been scheduled */
 	return NULL;
     }
@@ -544,16 +546,14 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 
 	if (dep->status & ERTS_DE_SFLG_EXITING) {
 #ifdef DEBUG
-	    erts_smp_mtx_lock(&dep->qlock);
-	    ASSERT(dep->qflgs & ERTS_DE_QFLG_EXIT);
-	    erts_smp_mtx_unlock(&dep->qlock);
+	    ASSERT(erts_smp_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT);
 #endif
 	}
 	else {
 	    dep->status |= ERTS_DE_SFLG_EXITING;
 	    erts_smp_mtx_lock(&dep->qlock);
-	    ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT));
-	    dep->qflgs |= ERTS_DE_QFLG_EXIT;
+	    ASSERT(!(erts_smp_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT));
+	    erts_smp_atomic32_read_bor_relb(&dep->qflgs, ERTS_DE_QFLG_EXIT);
 	    erts_smp_mtx_unlock(&dep->qlock);
 	}
 
@@ -706,8 +706,9 @@ static void clear_dist_entry(DistEntry *dep)
 
     if (obufsize) {
 	erts_smp_mtx_lock(&dep->qlock);
-	ASSERT(dep->qsize >= obufsize);
-	dep->qsize -= obufsize;
+	ASSERT(erts_smp_atomic_read_nob(&dep->qsize) >= obufsize);
+        erts_smp_atomic_add_nob(&dep->qsize,
+                                (erts_aint_t) -obufsize);
 	erts_smp_mtx_unlock(&dep->qlock);
     }
 }
@@ -1861,12 +1862,18 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 		free_dist_obuf(ctx->obuf);
 	    }
 	    else {
+                Sint qsize;
+                erts_aint32_t qflgs;
 		ErtsProcList *plp = NULL;
 		erts_smp_mtx_lock(&dep->qlock);
-		dep->qsize += size_obuf(ctx->obuf);
-		if (dep->qsize >= erts_dist_buf_busy_limit)
-		    dep->qflgs |= ERTS_DE_QFLG_BUSY;
-		if (!ctx->force_busy && (dep->qflgs & ERTS_DE_QFLG_BUSY)) {
+		qsize = erts_smp_atomic_add_read_nob(&dep->qsize,
+                                                     (erts_aint_t) size_obuf(ctx->obuf));
+                qflgs = erts_smp_atomic32_read_nob(&dep->qflgs);
+		if (!(qflgs & ERTS_DE_QFLG_BUSY) && qsize >= erts_dist_buf_busy_limit) {
+		    erts_smp_atomic32_read_bor_relb(&dep->qflgs, ERTS_DE_QFLG_BUSY);
+                    qflgs |= ERTS_DE_QFLG_BUSY;
+                }
+		if (!ctx->force_busy && (qflgs & ERTS_DE_QFLG_BUSY)) {
 		    erts_smp_mtx_unlock(&dep->qlock);
 
 		    plp = erts_proclist_create(ctx->c_p);
@@ -1883,7 +1890,8 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 		dep->out_queue.last = ctx->obuf;
 
 		if (!ctx->force_busy) {
-		    if (!(dep->qflgs & ERTS_DE_QFLG_BUSY)) {
+                    qflgs = erts_smp_atomic32_read_nob(&dep->qflgs);
+		    if (!(qflgs & ERTS_DE_QFLG_BUSY)) {
 			if (suspended)
 			    resume = 1; /* was busy when we started, but isn't now */
     #ifdef USE_VM_PROBES
@@ -2074,7 +2082,7 @@ erts_dist_command(Port *prt, int reds_limit)
     Sint reds = ERTS_PORT_REDS_DIST_CMD_START;
     Uint32 status;
     Uint32 flags;
-    Sint obufsize = 0;
+    Sint qsize, obufsize = 0;
     ErtsDistOutputQueue oq, foq;
     DistEntry *dep = prt->dist_entry;
     Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf);
@@ -2204,6 +2212,7 @@ erts_dist_command(Port *prt, int reds_limit)
 	}
     }
     else {
+        int de_busy;
 	int preempt = 0;
 	while (oq.first && !preempt) {
 	    ErtsDistOutputBuf *fob;
@@ -2257,12 +2266,13 @@ erts_dist_command(Port *prt, int reds_limit)
 	 * processes.
 	 */
 	erts_smp_mtx_lock(&dep->qlock);
-	ASSERT(dep->qsize >= obufsize);
-	dep->qsize -= obufsize;
+        de_busy = !!(erts_smp_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_BUSY);
+        qsize = (Sint) erts_smp_atomic_add_read_nob(&dep->qsize,
+                                                    (erts_aint_t) -obufsize);
+	ASSERT(qsize >= 0);
 	obufsize = 0;
 	if (!(sched_flags & ERTS_PTS_FLG_BUSY_PORT)
-	    && (dep->qflgs & ERTS_DE_QFLG_BUSY)
-	    && dep->qsize < erts_dist_buf_busy_limit) {
+	    && de_busy && qsize < erts_dist_buf_busy_limit) {
 	    ErtsProcList *suspendees;
 	    int resumed;
 	    suspendees = get_suspended_on_de(dep, ERTS_DE_QFLG_BUSY);
@@ -2282,8 +2292,13 @@ erts_dist_command(Port *prt, int reds_limit)
     if (obufsize != 0) {
 	ASSERT(obufsize > 0);
 	erts_smp_mtx_lock(&dep->qlock);
-	ASSERT(dep->qsize >= obufsize);
-	dep->qsize -= obufsize;
+#ifdef DEBUG
+        qsize = (Sint) erts_smp_atomic_add_read_nob(&dep->qsize,
+                                                    (erts_aint_t) -obufsize);
+	ASSERT(qsize >= 0);
+#else
+        erts_smp_atomic_add_nob(&dep->qsize, (erts_aint_t) -obufsize);
+#endif
 	erts_smp_mtx_unlock(&dep->qlock);
     }
 
@@ -2339,7 +2354,7 @@ erts_dist_command(Port *prt, int reds_limit)
 
 #ifdef DEBUG
 	erts_smp_mtx_lock(&dep->qlock);
-	ASSERT(dep->qsize == obufsize);
+	ASSERT(erts_smp_atomic_read_nob(&dep->qsize) == obufsize);
 	erts_smp_mtx_unlock(&dep->qlock);
 #endif
     }
@@ -2350,7 +2365,7 @@ erts_dist_command(Port *prt, int reds_limit)
 	     * in out_queue.
 	     */
 	    erts_smp_mtx_lock(&dep->qlock);
-	    dep->qsize -= obufsize;
+	    erts_smp_atomic_add_nob(&dep->qsize, -obufsize);
 	    obufsize = 0;
 	    oq.last->next = dep->out_queue.first;
 	    dep->out_queue.first = oq.first;
@@ -2394,8 +2409,8 @@ erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id)
 	dep->status |= ERTS_DE_SFLG_EXITING;
 
 	erts_smp_mtx_lock(&dep->qlock);
-	ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT));
-	dep->qflgs |= ERTS_DE_QFLG_EXIT;
+	ASSERT(!(erts_smp_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT));
+	erts_smp_atomic32_read_bor_nob(&dep->qflgs, ERTS_DE_QFLG_EXIT);
 	erts_smp_mtx_unlock(&dep->qlock);
 
 	erts_schedule_dist_command(NULL, dep);
@@ -2808,9 +2823,7 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     ASSERT(dep->send);
 
 #ifdef DEBUG
-    erts_smp_mtx_lock(&dep->qlock);
-    ASSERT(dep->qsize == 0);
-    erts_smp_mtx_unlock(&dep->qlock);
+    ASSERT(erts_smp_atomic_read_nob(&dep->qsize) == 0);
 #endif
 
     erts_set_dist_entry_connected(dep, BIF_ARG_2, flags);
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 3e17645997..dcd5846ca5 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -161,13 +161,10 @@ erts_dsig_prepare(ErtsDSigData *dsdp,
 	goto fail;
     }
     if (no_suspend) {
-	failure = ERTS_DSIG_PREP_CONNECTED;
-	erts_smp_mtx_lock(&dep->qlock);
-	if (dep->qflgs & ERTS_DE_QFLG_BUSY)
+        if (erts_smp_atomic32_read_acqb(&dep->qflgs) & ERTS_DE_QFLG_BUSY) {
 	    failure = ERTS_DSIG_PREP_WOULD_SUSPEND;
-	erts_smp_mtx_unlock(&dep->qlock);
-	if (failure == ERTS_DSIG_PREP_WOULD_SUSPEND)
 	    goto fail;
+        }
     }
     dsdp->proc = proc;
     dsdp->dep = dep;
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 3c5945d48d..c1796a8894 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -113,8 +113,8 @@ dist_table_alloc(void *dep_tmpl)
     dep->monitors			= NULL;
 
     erts_smp_mtx_init_x(&dep->qlock, "dist_entry_out_queue", chnl_nr);
-    dep->qflgs				= 0;
-    dep->qsize				= 0;
+    erts_smp_atomic32_init_nob(&dep->qflgs, 0);
+    erts_smp_atomic_init_nob(&dep->qsize, 0);
     dep->out_queue.first		= NULL;
     dep->out_queue.last			= NULL;
     dep->suspended			= NULL;
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 489da1ba17..04acfe41b1 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -133,8 +133,8 @@ typedef struct dist_entry_ {
     ErtsMonitor *monitors;      /* Monitor tree */
 
     erts_smp_mtx_t qlock;       /* Protects qflgs and out_queue */
-    Uint32 qflgs;
-    Sint qsize;
+    erts_smp_atomic32_t qflgs;
+    erts_smp_atomic_t qsize;
     ErtsDistOutputQueue out_queue;
     struct ErtsProcList_ *suspended;
 
-- 
cgit v1.2.3


From 90d2a1c4b85ce8dab16c7000cb02ab947337a64f Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 5 Jul 2017 11:15:44 +0200
Subject: erts: Add HRelease endp assert

This assert makes sure that endp is correct and that
no other HAlloc has been done inbetween HAlloc and HRelease.
---
 erts/emulator/beam/erl_vm.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 0b8d78c469..d077d3dea2 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -102,9 +102,11 @@
   if ((ptr) == (endp)) {					\
      ;								\
   } else if (HEAP_START(p) <= (ptr) && (ptr) < HEAP_TOP(p)) {	\
+     ASSERT(HEAP_TOP(p) == (endp));                             \
      HEAP_TOP(p) = (ptr);					\
   } else {							\
-     erts_heap_frag_shrink(p, ptr);					\
+     ASSERT(MBUF(p)->mem + MBUF(p)->used_size == (endp));       \
+     erts_heap_frag_shrink(p, ptr);                             \
   }
 
 #define HeapWordsLeft(p) (HEAP_LIMIT(p) - HEAP_TOP(p))
-- 
cgit v1.2.3


From 245aa4ed9b41aa601784598b0b3286d29b6d0085 Mon Sep 17 00:00:00 2001
From: Michal Muskala <michal@muskala.eu>
Date: Sat, 1 Jul 2017 17:19:52 +0200
Subject: Introduce new_small_map_lit op

Take advantage of the fact that small maps have a tuple for keys.
When new map is constructed and all keys are literals, we can construct
the entire keys tuple as a literal.

This should reduce the memory of maps created with literal keys almost by half,
since they all can share the same keys tuple.
---
 erts/emulator/beam/beam_debug.c |  21 ++++++++
 erts/emulator/beam/beam_emu.c   |  50 +++++++++++++++++++
 erts/emulator/beam/beam_load.c  | 105 ++++++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/ops.tab      |   4 ++
 4 files changed, 180 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index a2060c80de..74e6ab8bc5 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -718,6 +718,27 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	    }
 	}
 	break;
+    case op_i_new_small_map_lit_dIq:
+        {
+            Eterm *tp = tuple_val(unpacked[-1]);
+            int n = arityval(*tp);
+
+            while (n > 0) {
+                switch (loader_tag(ap[0])) {
+                case LOADER_X_REG:
+                    erts_print(to, to_arg, " x(%d)", loader_x_reg_index(ap[0]));
+                    break;
+                case LOADER_Y_REG:
+		    erts_print(to, to_arg, " x(%d)", loader_y_reg_index(ap[0]));
+		    break;
+                default:
+		    erts_print(to, to_arg, " %T", (Eterm) ap[0]);
+		    break;
+                }
+                ap++, size++, n--;
+            }
+        }
+        break;
     case op_i_get_map_elements_fsI:
 	{
 	    int n = unpacked[-1];
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 27d19d2504..039ce3fa24 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1080,6 +1080,7 @@ static BeamInstr* apply_fun(Process* p, Eterm fun,
 static Eterm new_fun(Process* p, Eterm* reg,
 		     ErlFunEntry* fe, int num_free) NOINLINE;
 static Eterm new_map(Process* p, Eterm* reg, BeamInstr* I) NOINLINE;
+static Eterm new_small_map_lit(Process* p, Eterm* reg, Uint* n_exp, BeamInstr* I) NOINLINE;
 static Eterm update_map_assoc(Process* p, Eterm* reg,
 			      Eterm map, BeamInstr* I) NOINLINE;
 static Eterm update_map_exact(Process* p, Eterm* reg,
@@ -2461,6 +2462,17 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
      Next(3+Arg(2));
  }
 
+ OpCase(i_new_small_map_lit_dIq): {
+     Eterm res;
+     Uint n;
+
+     HEAVY_SWAPOUT;
+     res = new_small_map_lit(c_p, reg, &n, I-1);
+     HEAVY_SWAPIN;
+     StoreResult(res, Arg(0));
+     Next(3+n);
+ }
+
 #define PUT_TERM_REG(term, desc)		\
 do {						\
     switch (loader_tag(desc)) {			\
@@ -7034,6 +7046,44 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
     return make_flatmap(mp);
 }
 
+static Eterm
+new_small_map_lit(Process* p, Eterm* reg, Uint* n_exp, BeamInstr* I)
+{
+    Eterm* keys = tuple_val(Arg(3));
+    Uint n = arityval(*keys);
+    Uint need = n + 1 /* hdr */ + 1 /*size*/ + 1 /* ptr */ + 1 /* arity */;
+    Uint i;
+    BeamInstr *ptr;
+    flatmap_t *mp;
+    Eterm *mhp;
+    Eterm *E;
+
+    *n_exp = n;
+    ptr = &Arg(4);
+
+    ASSERT(n <= MAP_SMALL_MAP_LIMIT);
+
+    if (HeapWordsLeft(p) < need) {
+        erts_garbage_collect(p, need, reg, Arg(2));
+    }
+
+    mhp = p->htop;
+    E   = p->stop;
+
+    mp = (flatmap_t *)mhp; mhp += MAP_HEADER_FLATMAP_SZ;
+    mp->thing_word = MAP_HEADER_FLATMAP;
+    mp->size = n;
+    mp->keys = Arg(3);
+
+    for (i = 0; i < n; i++) {
+        GET_TERM(*ptr++, *mhp++);
+    }
+
+    p->htop = mhp;
+
+    return make_flatmap(mp);
+}
+
 static Eterm
 update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 {
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 23258dbe9c..71637cf4d6 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -537,6 +537,7 @@ static int get_tag_and_value(LoaderState* stp, Uint len_code,
 static int new_label(LoaderState* stp);
 static void new_literal_patch(LoaderState* stp, int pos);
 static void new_string_patch(LoaderState* stp, int pos);
+static int find_literal(LoaderState* stp, Eterm needle, Uint *idx);
 static Uint new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size);
 static int genopargcompare(GenOpArg* a, GenOpArg* b);
 static Eterm get_module_info(Process* p, ErtsCodeIndex code_ix,
@@ -4221,6 +4222,92 @@ literal_is_map(LoaderState* stp, GenOpArg Lit)
     return is_map(term);
 }
 
+/*
+ * Predicate to test whether all of the given new small map keys are literals
+ */
+static int
+is_small_map_literal_keys(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
+{
+    if (Size.val > MAP_SMALL_MAP_LIMIT) {
+        return 0;
+    }
+
+    /*
+     * Operations with non-literals have always only one key.
+     */
+    if (Size.val != 2) {
+        return 1;
+    }
+
+    switch (Rest[0].type) {
+    case TAG_a:
+    case TAG_i:
+    case TAG_n:
+    case TAG_q:
+        return 1;
+    default:
+        return 0;
+    }
+}
+
+static GenOp*
+gen_new_small_map_lit(LoaderState* stp, GenOpArg Dst, GenOpArg Live,
+                      GenOpArg Size, GenOpArg* Rest)
+{
+    unsigned size = Size.val;
+    Uint lit;
+    unsigned i;
+    GenOp* op;
+    GenOpArg* dst;
+    Eterm* hp;
+    Eterm* tmp;
+    Eterm* thp;
+    Eterm keys;
+
+    NEW_GENOP(stp, op);
+    GENOP_ARITY(op, 3 + size/2);
+    op->next = NULL;
+    op->op = genop_i_new_small_map_lit_3;
+
+    tmp = thp = erts_alloc(ERTS_ALC_T_LOADER_TMP, (1 + size/2) * sizeof(*tmp));
+    keys = make_tuple(thp);
+    *thp++ = make_arityval(size/2);
+
+    dst = op->a+3;
+
+    for (i = 0; i < size; i += 2) {
+        switch (Rest[i].type) {
+        case TAG_a:
+            *thp++ = Rest[i].val;
+            ASSERT(is_atom(Rest[i].val));
+            break;
+        case TAG_i:
+            *thp++ = make_small(Rest[i].val);
+            break;
+        case TAG_n:
+            *thp++ = NIL;
+            break;
+        case TAG_q:
+            *thp++ = stp->literals[Rest[i].val].term;
+            break;
+        }
+        *dst++ = Rest[i + 1];
+    }
+
+    if (!find_literal(stp, keys, &lit)) {
+        lit = new_literal(stp, &hp, 1 + size/2);
+        sys_memcpy(hp, tmp, (1 + size/2) * sizeof(*tmp));
+    }
+    erts_free(ERTS_ALC_T_LOADER_TMP, tmp);
+
+    op->a[0] = Dst;
+    op->a[1] = Live;
+    op->a[2].type = TAG_q;
+    op->a[2].val = lit;
+
+    return op;
+}
+
 /*
  * Predicate to test whether the given literal is an empty map.
  */
@@ -5509,6 +5596,24 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size)
     return stp->num_literals++;
 }
 
+static int
+find_literal(LoaderState* stp, Eterm needle, Uint *idx)
+{
+    int i;
+
+    /*
+     * The search is done backwards since the most recent literals
+     * allocated by the loader itself will be placed at the end
+     */
+    for (i = stp->num_literals - 1; i >= 0; i--) {
+        if (EQ(needle, stp->literals[i].term)) {
+            *idx = (Uint) i;
+            return 1;
+        }
+    }
+    return 0;
+}
+
 Eterm
 erts_module_info_0(Process* p, Eterm module)
 {
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 4ce86ce949..ed856b760b 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1425,7 +1425,11 @@ sorted_put_map_exact F Src=s Dst Live Size Rest=* => \
 sorted_put_map_exact F Src Dst Live Size Rest=* => \
 	      move Src x | update_map_exact F x Dst Live Size Rest
 
+new_map Dst Live Size Rest=* | is_small_map_literal_keys(Size, Rest) => \
+   gen_new_small_map_lit(Dst, Live, Size, Rest)
+
 new_map d I I
+i_new_small_map_lit d I q
 update_map_assoc j s d I I
 update_map_exact j s d I I
 
-- 
cgit v1.2.3


From df3442b0eb1ec4c6f8f4547cc20e5ad0824e7cd5 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 5 Jul 2017 16:08:59 +0200
Subject: erts: Refactor erts_unicode_list_to_buf

to get bytes written when truncated.
---
 erts/emulator/beam/bif.c    | 17 +++++++------
 erts/emulator/beam/global.h |  2 +-
 erts/emulator/beam/utils.c  | 60 +++++++++++++++++++++++++--------------------
 3 files changed, 44 insertions(+), 35 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 40dd4129d2..8380efaf13 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3021,17 +3021,17 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1)
 {
     Eterm res;
     byte *buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_SZ_LIMIT);
-    Sint i = erts_unicode_list_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS);
-
+    Sint written;
+    int i = erts_unicode_list_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS,
+                                     &written);
     if (i < 0) {
 	erts_free(ERTS_ALC_T_TMP, (void *) buf);
-	i = erts_list_length(BIF_ARG_1);
-	if (i > MAX_ATOM_CHARACTERS) {
+	if (i == -2) {
 	    BIF_ERROR(BIF_P, SYSTEM_LIMIT);
 	}
 	BIF_ERROR(BIF_P, BADARG);
     }
-    res = erts_atom_put(buf, i, ERTS_ATOM_ENC_UTF8, 1);
+    res = erts_atom_put(buf, written, ERTS_ATOM_ENC_UTF8, 1);
     ASSERT(is_atom(res));
     erts_free(ERTS_ALC_T_TMP, (void *) buf);
     BIF_RET(res);
@@ -3042,8 +3042,9 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1)
 BIF_RETTYPE list_to_existing_atom_1(BIF_ALIST_1)
 {
     byte *buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_SZ_LIMIT);
-    Sint i = erts_unicode_list_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS);
-
+    Sint written;
+    int i = erts_unicode_list_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS,
+                                     &written);
     if (i < 0) {
     error:
 	erts_free(ERTS_ALC_T_TMP, (void *) buf);
@@ -3051,7 +3052,7 @@ BIF_RETTYPE list_to_existing_atom_1(BIF_ALIST_1)
     } else {
 	Eterm a;
 	
-	if (erts_atom_get((char *) buf, i, &a, ERTS_ATOM_ENC_UTF8)) {
+	if (erts_atom_get((char *) buf, written, &a, ERTS_ATOM_ENC_UTF8)) {
 	    erts_free(ERTS_ALC_T_TMP, (void *) buf);
 	    BIF_RET(a);
 	} else {
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index fc95535ec3..fcb88712e9 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1287,7 +1287,7 @@ int erts_utf8_to_latin1(byte* dest, const byte* source, int slen);
 
 void bin_write(fmtfn_t, void*, byte*, size_t);
 Sint intlist_to_buf(Eterm, char*, Sint); /* most callers pass plain char*'s */
-Sint erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len);
+int erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len, Sint* written);
 
 struct Sint_buf {
 #if defined(ARCH_64)
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 457cada745..ba7596cde4 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3634,30 +3634,40 @@ intlist_to_buf(Eterm list, char *buf, Sint len)
     return -2;			/* not enough space */
 }
 
-/* Fill buf with the contents of the unicode list.
- * Return the number of bytes in the buffer,
- * or -1 for type error,
- * or -2 for not enough buffer space (buffer contains truncated result).
+/** @brief Fill buf with the UTF8 contents of the unicode list
+ * @param len Max number of characters to write.
+ * @param written NULL or bytes written.
+ * @return 0 ok,
+ *        -1 type error,
+ *        -2 list too long, only \c len characters written
  */
-Sint
-erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len)
+int
+erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len, Sint* written)
 {
     Eterm* listptr;
     Sint sz = 0;
+    Sint val;
+    int res;
 
-    if (is_nil(list)) {
-	return 0;
-    }
-    if (is_not_list(list)) {
-	return -1;
-    }
-    listptr = list_val(list);
+    while (1) {
+        if (is_nil(list)) {
+            res = 0;
+            break;
+        }
+        if (is_not_list(list)) {
+            res = -1;
+            break;
+        }
+        listptr = list_val(list);
 
-    while (len-- > 0) {
-	Sint val;
+        if (len-- <= 0) {
+            res = -2;
+            break;
+        }
 
 	if (is_not_small(CAR(listptr))) {
-	    return -1;
+	    res = -1;
+            break;
 	}
 	val = signed_val(CAR(listptr));
 	if (0 <= val && val < 0x80) {
@@ -3669,7 +3679,8 @@ erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len)
 	    sz += 2;
 	} else if (val < 0x10000UL) {
 	    if (0xD800 <= val && val <= 0xDFFF) {
-		return -1;
+		res = -1;
+                break;
 	    }
 	    buf[sz+0] = 0xE0 | (val >> 12);
 	    buf[sz+1] = 0x80 | ((val >> 6) & 0x3F);
@@ -3682,18 +3693,15 @@ erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len)
 	    buf[sz+3] = 0x80 | (val & 0x3F);
 	    sz += 4;
 	} else {
-	    return -1;
+            res = -1;
+            break;
 	}
 	list = CDR(listptr);
-	if (is_nil(list)) {
-	    return sz;
-	}
-	if (is_not_list(list)) {
-	    return -1;
-	}
-	listptr = list_val(list);
     }
-    return -2;			/* not enough space */
+
+    if (written)
+        *written = sz;
+    return res;
 }
 
 /*
-- 
cgit v1.2.3


From 29c9a1fc01d64a88cb8e441a63779cfdabefb457 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 23 May 2017 18:08:34 +0200
Subject: Enable register_SUITE for lcnt builds

It was disabled for performance reasons, and the new implementation handles
it just fine (roughly half as fast as without lcnt).
---
 erts/emulator/test/register_SUITE.erl | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/register_SUITE.erl b/erts/emulator/test/register_SUITE.erl
index 43ae749498..49da94a775 100644
--- a/erts/emulator/test/register_SUITE.erl
+++ b/erts/emulator/test/register_SUITE.erl
@@ -44,14 +44,7 @@ all() ->
 -define(OTP_8099_NAME, otp_8099_reg_proc).
 
 otp_8099(Config) when is_list(Config) ->
-    case catch erlang:system_info(lock_counting) of
-	true -> {skipped,
-		 "Lock counting enabled. Current lock counting "
-		 "implementation cannot handle this many "
-		 "processes."};
-	_ ->
-	    otp_8099_test(1000000)
-    end.
+    otp_8099_test(1000000).
 
 otp_8099_test(0) ->
     ok;
-- 
cgit v1.2.3


From e9bb277cc5cc8326c3c90ed82954f2ee14c1844b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 13 Jun 2017 10:37:12 +0200
Subject: Move lock flags to a common header

---
 erts/emulator/Makefile.in           |  2 +-
 erts/emulator/beam/erl_lock_flags.c | 46 +++++++++++++++++++++++
 erts/emulator/beam/erl_lock_flags.h | 74 +++++++++++++++++++++++++++++++++++++
 3 files changed, 121 insertions(+), 1 deletion(-)
 create mode 100644 erts/emulator/beam/erl_lock_flags.c
 create mode 100644 erts/emulator/beam/erl_lock_flags.h

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index ecbf76595b..8b2dd5dbb9 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -813,7 +813,7 @@ RUN_OBJS = \
 	$(OBJDIR)/erl_bif_binary.o      $(OBJDIR)/erl_ao_firstfit_alloc.o \
 	$(OBJDIR)/erl_thr_queue.o	$(OBJDIR)/erl_sched_spec_pre_alloc.o \
 	$(OBJDIR)/erl_ptab.o		$(OBJDIR)/erl_map.o \
-	$(OBJDIR)/erl_msacc.o
+	$(OBJDIR)/erl_msacc.o		$(OBJDIR)/erl_lock_flags.o
 
 LTTNG_OBJS = $(OBJDIR)/erlang_lttng.o
 NIF_OBJS = $(OBJDIR)/erl_tracer_nif.o
diff --git a/erts/emulator/beam/erl_lock_flags.c b/erts/emulator/beam/erl_lock_flags.c
new file mode 100644
index 0000000000..ba6a7217f2
--- /dev/null
+++ b/erts/emulator/beam/erl_lock_flags.c
@@ -0,0 +1,46 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2017. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "erl_lock_flags.h"
+
+const char *erts_lock_flags_get_type_name(erts_lock_flags_t flags) {
+    switch(flags & ERTS_LOCK_FLAGS_MASK_TYPE) {
+        case ERTS_LOCK_FLAGS_TYPE_PROCLOCK:
+            return "proclock";
+        case ERTS_LOCK_FLAGS_TYPE_MUTEX:
+            if(flags & ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE) {
+                return "rw_mutex";
+            }
+
+            return "mutex";
+        case ERTS_LOCK_FLAGS_TYPE_SPINLOCK:
+            if(flags & ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE) {
+                return "rw_spinlock";
+            }
+
+            return "spinlock";
+        default:
+            return "garbage";
+    }
+}
diff --git a/erts/emulator/beam/erl_lock_flags.h b/erts/emulator/beam/erl_lock_flags.h
new file mode 100644
index 0000000000..b66c160af5
--- /dev/null
+++ b/erts/emulator/beam/erl_lock_flags.h
@@ -0,0 +1,74 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2017. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifndef ERTS_LOCK_FLAGS_H__
+#define ERTS_LOCK_FLAGS_H__
+
+#define ERTS_LOCK_OPTION_READ  (1 << 1)
+#define ERTS_LOCK_OPTION_WRITE (1 << 2)
+
+#define ERTS_LOCK_OPTION_RDWR (ERTS_LOCK_OPTION_READ | ERTS_LOCK_OPTION_WRITE)
+
+/* Property/category are bitfields to simplify their use in masks. */
+#define ERTS_LOCK_FLAGS_MASK_CATEGORY (0xFFC0)
+#define ERTS_LOCK_FLAGS_MASK_PROPERTY (0x0030)
+
+/* Type is a plain number. */
+#define ERTS_LOCK_FLAGS_MASK_TYPE     (0x000F)
+
+#define ERTS_LOCK_FLAGS_TYPE_SPINLOCK (1)
+#define ERTS_LOCK_FLAGS_TYPE_MUTEX    (2)
+#define ERTS_LOCK_FLAGS_TYPE_PROCLOCK (3)
+
+/* "Static" guarantees that the lock will never be destroyed once created. */
+#define ERTS_LOCK_FLAGS_PROPERTY_STATIC     (1 << 4)
+#define ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE (1 << 5)
+
+#define ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR    (1 << 6)
+#define ERTS_LOCK_FLAGS_CATEGORY_PROCESS      (1 << 7)
+#define ERTS_LOCK_FLAGS_CATEGORY_IO           (1 << 8)
+#define ERTS_LOCK_FLAGS_CATEGORY_DB           (1 << 9)
+#define ERTS_LOCK_FLAGS_CATEGORY_DEBUG        (1 << 10)
+#define ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER    (1 << 11)
+#define ERTS_LOCK_FLAGS_CATEGORY_GENERIC      (1 << 12)
+#define ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION (1 << 13)
+
+#define ERTS_LOCK_TYPE_SPINLOCK \
+    (ERTS_LOCK_FLAGS_TYPE_SPINLOCK)
+#define ERTS_LOCK_TYPE_RWSPINLOCK \
+    (ERTS_LOCK_TYPE_SPINLOCK | \
+     ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE)
+#define ERTS_LOCK_TYPE_MUTEX \
+    (ERTS_LOCK_FLAGS_TYPE_MUTEX)
+#define ERTS_LOCK_TYPE_RWMUTEX \
+    (ERTS_LOCK_TYPE_MUTEX | \
+     ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE)
+#define ERTS_LOCK_TYPE_PROCLOCK \
+    (ERTS_LOCK_FLAGS_CATEGORY_PROCESS | \
+     ERTS_LOCK_FLAGS_TYPE_PROCLOCK)
+
+/* -- -- */
+
+typedef unsigned short erts_lock_flags_t;
+
+/* @brief Gets the type name of the lock, honoring the RW flag if supplied. */
+const char *erts_lock_flags_get_type_name(erts_lock_flags_t flags);
+
+#endif /* ERTS_LOCK_FLAGS_H__ */
-- 
cgit v1.2.3


From f3fa8288287072195baa791ee11d5480d3cd45ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 22 Jun 2017 13:22:13 +0200
Subject: Allow toggling lock counting at runtime

The implementation is still hidden behind ERTS_ENABLE_LOCK_COUNT, and
all categories are still enabled by default, but the actual counting can be
toggled at will.

OTP-13170
---
 erts/emulator/beam/atom.c                |   3 +-
 erts/emulator/beam/beam_bif_load.c       |   7 +-
 erts/emulator/beam/beam_bp.c             |   3 +-
 erts/emulator/beam/bif.c                 |   6 -
 erts/emulator/beam/code_ix.c             |   3 +-
 erts/emulator/beam/dist.c                |   3 +-
 erts/emulator/beam/erl_alloc.c           |   3 +-
 erts/emulator/beam/erl_alloc.h           |   9 +-
 erts/emulator/beam/erl_alloc.types       |   2 +-
 erts/emulator/beam/erl_alloc_util.c      |  57 ++++-
 erts/emulator/beam/erl_alloc_util.h      |   5 +-
 erts/emulator/beam/erl_async.c           |   6 +-
 erts/emulator/beam/erl_bif_info.c        | 181 ++++++++--------
 erts/emulator/beam/erl_bif_unique.c      |   6 +-
 erts/emulator/beam/erl_cpu_topology.c    |   3 +-
 erts/emulator/beam/erl_db.c              |  63 +++++-
 erts/emulator/beam/erl_db.h              |   5 +
 erts/emulator/beam/erl_db_hash.c         |  24 ++-
 erts/emulator/beam/erl_db_hash.h         |   4 +
 erts/emulator/beam/erl_drv_thread.c      |  23 +-
 erts/emulator/beam/erl_fun.c             |   3 +-
 erts/emulator/beam/erl_gc.c              |   3 +-
 erts/emulator/beam/erl_init.c            |  11 +-
 erts/emulator/beam/erl_instrument.c      |   6 +-
 erts/emulator/beam/erl_lock_check.c      |   5 +-
 erts/emulator/beam/erl_lock_count.c      | 259 ++++++++++++++++-------
 erts/emulator/beam/erl_lock_count.h      | 287 +++++++++++++------------
 erts/emulator/beam/erl_msacc.c           |   6 +-
 erts/emulator/beam/erl_mtrace.c          |   6 +-
 erts/emulator/beam/erl_nif.c             |   3 +-
 erts/emulator/beam/erl_node_tables.c     |  44 +++-
 erts/emulator/beam/erl_node_tables.h     |   4 +
 erts/emulator/beam/erl_port_task.h       |   8 +-
 erts/emulator/beam/erl_process.c         |  33 +--
 erts/emulator/beam/erl_process_lock.c    | 107 +++++-----
 erts/emulator/beam/erl_process_lock.h    |   3 +-
 erts/emulator/beam/erl_ptab.c            |   3 +-
 erts/emulator/beam/erl_smp.h             | 133 ++++--------
 erts/emulator/beam/erl_threads.h         | 348 ++++++++++---------------------
 erts/emulator/beam/erl_time_sup.c        |  11 +-
 erts/emulator/beam/erl_trace.c           |  10 +-
 erts/emulator/beam/export.c              |   3 +-
 erts/emulator/beam/global.h              |   3 +-
 erts/emulator/beam/io.c                  | 152 ++++++++------
 erts/emulator/beam/module.c              |   3 +-
 erts/emulator/beam/register.c            |   3 +-
 erts/emulator/beam/safe_hash.c           |  23 +-
 erts/emulator/beam/safe_hash.h           |   7 +-
 erts/emulator/drivers/common/efile_drv.c |   3 +-
 erts/emulator/hipe/hipe_bif0.c           |   3 +-
 erts/emulator/sys/common/erl_check_io.c  |  36 ++--
 erts/emulator/sys/common/erl_check_io.h  |   9 +
 erts/emulator/sys/common/erl_mmap.c      |   9 +-
 erts/emulator/sys/common/erl_mseg.c      |   6 +-
 erts/emulator/sys/common/erl_poll.c      |  38 +++-
 erts/emulator/sys/common/erl_poll.h      |   4 +
 erts/emulator/sys/unix/sys.c             |   3 +-
 erts/emulator/sys/unix/sys_time.c        |   4 +-
 erts/emulator/sys/win32/erl_poll.c       |  10 +-
 erts/emulator/sys/win32/sys_env.c        |   3 +-
 erts/emulator/sys/win32/sys_time.c       |   4 +-
 61 files changed, 1137 insertions(+), 898 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index 2055c29190..c2d78aaccb 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -442,7 +442,8 @@ init_atom_table(void)
     erts_smp_atomic_init_nob(&atom_put_ops, 0);
 #endif
 
-    erts_smp_rwmtx_init_opt(&atom_table_lock, &rwmtx_opt, "atom_tab");
+    erts_smp_rwmtx_init_opt(&atom_table_lock, &rwmtx_opt, "atom_tab", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
     f.hash = (H_FUN) atom_hash;
     f.cmp  = (HCMP_FUN) atom_cmp;
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 5aceae8ffe..ffa33d6303 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -97,7 +97,8 @@ init_purge_state(void)
 {
     purge_state.module = THE_NON_VALUE;
 
-    erts_smp_mtx_init(&purge_state.mtx, "purge_state");
+    erts_smp_mtx_init(&purge_state.mtx, "purge_state", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
     purge_state.pending_purge_lambda =
 	erts_export_put(am_erts_code_purger, am_pending_purge_lambda, 3);
@@ -118,7 +119,9 @@ init_purge_state(void)
 void
 erts_beam_bif_load_init(void)
 {
-    erts_smp_mtx_init(&release_literal_areas.mtx, "release_literal_areas");
+    erts_smp_mtx_init(&release_literal_areas.mtx, "release_literal_areas", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
+
     release_literal_areas.first = NULL;
     release_literal_areas.last = NULL;
     erts_smp_atomic_init_nob(&erts_copy_literal_area__,
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index b9453c1d9a..950639f7ae 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -165,7 +165,8 @@ erts_bp_init(void) {
     erts_smp_atomic32_init_nob(&erts_active_bp_index, 0);
     erts_smp_atomic32_init_nob(&erts_staging_bp_index, 1);
 #ifdef ERTS_DIRTY_SCHEDULERS
-    erts_smp_mtx_init(&erts_dirty_bp_ix_mtx, "dirty_break_point_index");
+    erts_smp_mtx_init(&erts_dirty_bp_ix_mtx, "dirty_break_point_index", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
 #endif
 }
 
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 7134d2da56..b9384b28a1 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -62,9 +62,6 @@ static erts_smp_atomic32_t msacc;
 static Export *await_sched_wall_time_mod_trap;
 static erts_smp_atomic32_t sched_wall_time;
 
-static erts_smp_mtx_t ports_snapshot_mtx;
-erts_smp_atomic_t erts_dead_ports_ptr; /* To store dying ports during snapshot */
-
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
 /*
@@ -5140,9 +5137,6 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a,
 
 void erts_init_bif(void)
 {
-    erts_smp_mtx_init(&ports_snapshot_mtx, "ports_snapshot");
-    erts_smp_atomic_init_nob(&erts_dead_ports_ptr, (erts_aint_t) NULL);
-
     /*
      * bif_return_trap/2 is a hidden BIF that bifs that need to
      * yield the calling process traps to.
diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c
index ec6267711b..8a3d1b20b4 100644
--- a/erts/emulator/beam/code_ix.c
+++ b/erts/emulator/beam/code_ix.c
@@ -57,7 +57,8 @@ void erts_code_ix_init(void)
      */
     erts_smp_atomic32_init_nob(&the_active_code_index, 0);
     erts_smp_atomic32_init_nob(&the_staging_code_index, 0);
-    erts_smp_mtx_init(&code_write_permission_mtx, "code_write_permission");
+    erts_smp_mtx_init(&code_write_permission_mtx, "code_write_permission", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_tsd_key_create(&has_code_write_permission,
 			"erts_has_code_write_permission");
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 982f1066df..09fdb897f5 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -3223,7 +3223,8 @@ static ErtsNodesMonitor *nodes_monitors_end;
 static void
 init_nodes_monitors(void)
 {
-    erts_smp_mtx_init(&nodes_monitors_mtx, "nodes_monitors");
+    erts_smp_mtx_init(&nodes_monitors_mtx, "nodes_monitors", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
     nodes_monitors = NULL;
     nodes_monitors_end = NULL;
 }
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 169e1e423d..c7ab444c96 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -3819,7 +3819,8 @@ hdbg_init(void)
     hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK-1].next = NULL;
     free_hdbg_mblks = &hdbg_mblks[0];
     used_hdbg_mblks = NULL;
-    erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug");
+    erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR);
 }
 
 static void *check_memory_fence(void *ptr,
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 758d529f87..d743c62a02 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -344,7 +344,8 @@ ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT,				\
 #define ERTS_SMP_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)			\
 static erts_smp_spinlock_t NAME##_lck;					\
 ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT,				\
-		      erts_smp_spinlock_init(&NAME##_lck, #NAME "_alloc_lock"),\
+		      erts_smp_spinlock_init(&NAME##_lck, #NAME "_alloc_lock", NIL, \
+		          ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR),\
 		      erts_smp_spin_lock(&NAME##_lck),			\
 		      erts_smp_spin_unlock(&NAME##_lck))
 
@@ -358,7 +359,8 @@ ERTS_SMP_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)
 #define ERTS_TS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)			\
 static erts_mtx_t NAME##_lck;						\
 ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT,				\
-		      erts_mtx_init(NAME##_lck, #NAME "_alloc_lock"),	\
+		      erts_mtx_init(NAME##_lck, #NAME "_alloc_lock", NIL, \
+		          ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR),\
 		      erts_mtx_lock(&NAME##_lck),			\
 		      erts_mtx_unlock(&NAME##_lck))
 
@@ -371,7 +373,8 @@ ERTS_PRE_ALLOC_IMPL(NAME, TYPE, PASZ, (void) 0, (void) 0, (void) 0)
 #define ERTS_TS_PALLOC_IMPL(NAME, TYPE, PASZ)				\
 static erts_spinlock_t NAME##_lck;					\
 ERTS_PRE_ALLOC_IMPL(NAME, TYPE, PASZ,					\
-		    erts_spinlock_init(&NAME##_lck, #NAME "_alloc_lock"),\
+		    erts_spinlock_init(&NAME##_lck, #NAME "_alloc_lock", NIL, \
+		          ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR),\
 		    erts_spin_lock(&NAME##_lck),			\
 		    erts_spin_unlock(&NAME##_lck))
 
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index c155630765..50a1d97dd5 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -371,7 +371,7 @@ type	SSB		SHORT_LIVED	PROCESSES	ssb
 +if lcnt
 
 type    LCNT_CARRIER   STANDARD    SYSTEM  lcnt_lock_info_carrier
-type    LCNT_VECTOR SHORT_LIVED SYSTEM lcnt_lock_info_vector
+type    LCNT_VECTOR SHORT_LIVED SYSTEM lcnt_sample_vector
 
 +endif
 
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 4889fac923..c246143b3f 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -6135,16 +6135,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
     if (init->ts) {
 	allctr->thread_safe = 1;
 
-#ifdef ERTS_ENABLE_LOCK_COUNT
-	erts_mtx_init_x_opt(&allctr->mutex,
-			    "alcu_allocator",
-			    make_small(allctr->alloc_no),
-			    ERTS_LCNT_LT_ALLOC);
-#else
-	erts_mtx_init_x(&allctr->mutex,
-			"alcu_allocator",
-			make_small(allctr->alloc_no));
-#endif /*ERTS_ENABLE_LOCK_COUNT*/
+        erts_mtx_init(&allctr->mutex, "alcu_allocator", make_small(allctr->alloc_no),
+            ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR);
 
 #ifdef DEBUG
 	allctr->debug.saved_tid = 0;
@@ -6324,7 +6316,8 @@ erts_alcu_init(AlcUInit_t *init)
     carrier_alignment = sizeof(Unit_t);
 #endif
 
-    erts_mtx_init(&init_atoms_mtx, "alcu_init_atoms");
+    erts_mtx_init(&init_atoms_mtx, "alcu_init_atoms", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR);
 
     atoms_initialized = 0;
     initialized = 1;
@@ -6592,3 +6585,45 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
 
 #endif /* ERTS_ALLOC_UTIL_HARD_DEBUG */
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+
+static void lcnt_enable_allocator_lock_count(Allctr_t *allocator, int enable) {
+    if(!allocator->thread_safe) {
+        return;
+    }
+
+    if(enable) {
+        erts_lcnt_install_new_lock_info(&allocator->mutex.lcnt,
+            "alcu_allocator", make_small(allocator->alloc_no),
+            ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR);
+    } else {
+        erts_lcnt_uninstall(&allocator->mutex.lcnt);
+    }
+}
+
+static void lcnt_update_thread_spec_locks(ErtsAllocatorThrSpec_t *tspec, int enable) {
+    if(tspec->enabled) {
+        int i;
+
+        for(i = 0; i < tspec->size; i++) {
+            lcnt_enable_allocator_lock_count(tspec->allctr[i], enable);
+        }
+    }
+}
+
+void erts_lcnt_update_allocator_locks(int enable) {
+    int i;
+
+    for(i = ERTS_ALC_A_MIN; i < ERTS_ALC_A_MAX; i++) {
+        ErtsAllocatorInfo_t *ai = &erts_allctrs_info[i];
+
+        if(ai->enabled && ai->alloc_util) {
+            if(ai->thr_spec) {
+                lcnt_update_thread_spec_locks((ErtsAllocatorThrSpec_t*)ai->extra, enable);
+            } else {
+                lcnt_enable_allocator_lock_count((Allctr_t*)ai->extra, enable);
+            }
+        }
+    }
+}
+#endif /* ERTS_ENABLE_LOCK_COUNT */
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index f570703f25..431ce430be 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -227,6 +227,10 @@ void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint
 void  erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign);
 #endif
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void erts_lcnt_update_allocator_locks(int enable);
+#endif
+
 #endif /* !ERL_ALLOC_UTIL__ */
 
 #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__)
@@ -680,7 +684,6 @@ void erts_alcu_assert_failed(char* expr, char* file, int line, char *func);
 int is_sbc_blk(Block_t*);
 #endif
 
-
 #endif /* #if defined(GET_ERL_ALLOC_UTIL_IMPL)
 	      && !defined(ERL_ALLOC_UTIL_IMPL__) */
 
diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index 84254af0c2..9a93034fcb 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -194,7 +194,8 @@ erts_init_async(void)
 	ptr += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData));
 
 	async->init.data.no_initialized = 0;
-	erts_mtx_init(&async->init.data.mtx, "async_init_mtx");
+        erts_mtx_init(&async->init.data.mtx, "async_init_mtx", NIL,
+            ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
 	erts_cnd_init(&async->init.data.cnd);
 	erts_atomic_init_nob(&async->init.data.id, 0);
 
@@ -213,7 +214,8 @@ erts_init_async(void)
 	for (i = 1; i <= erts_no_schedulers; i++) {
 	    ErtsAsyncReadyQ *arq = async_ready_q(i);
 #if ERTS_USE_ASYNC_READY_ENQ_MTX
-	    erts_mtx_init(&arq->x.data.enq_mtx, "async_enq_mtx");
+            erts_mtx_init(&arq->x.data.enq_mtx, "async_enq_mtx", make_small(i),
+                ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
 #endif
 	    erts_thr_q_finalize_dequeue_state_init(&arq->fin_deq);
 	    qinit.arg = (void *) (SWord) i;
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 8f7de32f12..02e34014e3 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -4431,44 +4431,56 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
 
-typedef struct lcnt_lock_info_vector_ {
-    erts_lcnt_lock_info_t **elements;
+typedef struct {
+    /* info->location_count may increase between size calculation and term
+     * building, so we cap it at the value sampled in lcnt_build_result_vector.
+     *
+     * Shrinking is safe though. */
+    int max_location_count;
+    erts_lcnt_lock_info_t *info;
+} lcnt_sample_t;
+
+typedef struct lcnt_sample_vector_ {
+    lcnt_sample_t *elements;
     size_t size;
-} lcnt_lock_info_vector_t;
+} lcnt_sample_vector_t;
 
-static lcnt_lock_info_vector_t lcnt_build_lock_info_vector(erts_lcnt_lock_info_list_t *list) {
+static lcnt_sample_vector_t lcnt_build_sample_vector(erts_lcnt_lock_info_list_t *list) {
     erts_lcnt_lock_info_t *iterator;
-    lcnt_lock_info_vector_t result;
+    lcnt_sample_vector_t result;
     size_t allocated_entries;
 
     allocated_entries = 64;
     result.size = 0;
 
     result.elements = erts_alloc(ERTS_ALC_T_LCNT_VECTOR,
-        allocated_entries * sizeof(erts_lcnt_lock_info_t*));
+        allocated_entries * sizeof(lcnt_sample_t));
 
     iterator = NULL;
     while(erts_lcnt_iterate_list(list, &iterator)) {
         erts_lcnt_retain_lock_info(iterator);
 
-        result.elements[result.size++] = iterator;
+        result.elements[result.size].max_location_count = iterator->location_count;
+        result.elements[result.size].info = iterator;
+
+        result.size++;
 
         if(result.size >= allocated_entries) {
             allocated_entries *= 2;
 
             result.elements = erts_realloc(ERTS_ALC_T_LCNT_VECTOR, result.elements,
-                allocated_entries * sizeof(erts_lcnt_lock_info_t*));
+                allocated_entries * sizeof(lcnt_sample_t));
         }
     }
 
     return result;
 }
 
-static void lcnt_destroy_lock_info_vector(lcnt_lock_info_vector_t *vector) {
+static void lcnt_destroy_sample_vector(lcnt_sample_vector_t *vector) {
     size_t i;
 
     for(i = 0; i < vector->size; i++) {
-        erts_lcnt_release_lock_info(vector->elements[i]);
+        erts_lcnt_release_lock_info(vector->elements[i].info);
     }
 
     erts_free(ERTS_ALC_T_LCNT_VECTOR, vector->elements);
@@ -4508,9 +4520,9 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s
     Eterm thist, vhist[ERTS_LCNT_HISTOGRAM_SLOT_SIZE];
 
     /* term:
-        * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}},
-        *   { .. histogram .. }]
-        */
+     *  [{{file, line},
+         {tries, colls, {seconds, nanoseconds, n_blocks}},
+     *   { .. histogram .. }] */
 
     file = stats->file ? stats->file : "undefined";
 
@@ -4543,13 +4555,7 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s
 static Eterm lcnt_pretty_print_lock_id(erts_lcnt_lock_info_t *info) {
     Eterm id = info->id;
 
-    if(info->flag & ERTS_LCNT_LT_ALLOC) {
-        /* Use allocator type names as id's for allocator locks. This blatantly
-         * assumes that all locks in this category are identified by their
-         * allocator number. */
-        const char *ltype = (const char*)ERTS_ALC_A2AD(signed_val(info->id));
-        id = erts_atom_put((byte*)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1);
-    } else if(info->flag & ERTS_LCNT_LT_PROCLOCK) {
+    if((info->flags & ERTS_LOCK_FLAGS_MASK_TYPE) == ERTS_LOCK_TYPE_PROCLOCK) {
         /* Use registered names as id's for process locks if available. Thread
          * progress is delayed since we may be running on a dirty scheduler. */
         ErtsThrPrgrDelayHandle delay_handle;
@@ -4563,27 +4569,30 @@ static Eterm lcnt_pretty_print_lock_id(erts_lcnt_lock_info_t *info) {
         }
 
         erts_thr_progress_unmanaged_continue(delay_handle);
+    } else if(info->flags & ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR) {
+        if(is_small(id) && !sys_strcmp(info->name, "alcu_allocator")) {
+            const char *name = (const char*)ERTS_ALC_A2AD(signed_val(id));
+            id = erts_atom_put((byte*)name, strlen(name), ERTS_ATOM_ENC_LATIN1, 1);
+        }
     }
 
     return id;
 }
 
-static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_info_t *info, Eterm res) {
+static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, lcnt_sample_t *sample, Eterm res) {
+    erts_lcnt_lock_info_t *info = sample->info;
+
     Eterm name, type, id, stats = NIL, t;
-    const char *ltype;
+    const char *lock_desc;
     int i;
-    
-    /* term:
-     * [{name, id, type, stats()}] 
-     */
+
+    /* term: [{name, id, type, stats()}] */
 
     ASSERT(info->name);
     
-    ltype = erts_lcnt_lock_type(info->flag);
-    
-    ASSERT(ltype);
-    
-    type  = erts_atom_put((byte*)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1);
+    lock_desc = erts_lock_flags_get_type_name(info->flags);
+
+    type  = erts_atom_put((byte*)lock_desc, strlen(lock_desc), ERTS_ATOM_ENC_LATIN1, 1);
     name  = erts_atom_put((byte*)info->name, strlen(info->name), ERTS_ATOM_ENC_LATIN1, 1);
 
     /* Only attempt to resolve ids when actually emitting the term. This ought
@@ -4594,8 +4603,7 @@ static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_info_t
         id = NIL;
     }
 
-    /* info->location_count is ignored since it can be changed at any time. */
-    for(i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) {
+    for(i = 0; i < MIN(info->location_count, sample->max_location_count); i++) {
         stats = lcnt_build_lock_stats_term(hpp, szp, &(info->location_stats[i]), stats);
     }
 
@@ -4606,18 +4614,15 @@ static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_info_t
 }
 
 static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_time_t *duration,
-                                    lcnt_lock_info_vector_t *current_locks,
-                                    lcnt_lock_info_vector_t *deleted_locks, Eterm res) {
-    Eterm dts, dtns, tdt, adur, tdur, aloc, lloc = NIL, tloc;
+                                    lcnt_sample_vector_t *current_locks,
+                                    lcnt_sample_vector_t *deleted_locks, Eterm res) {
+    const char *str_duration = "duration";
+    const char *str_locks = "locks";
 
+    Eterm dts, dtns, tdt, adur, tdur, aloc, lloc = NIL, tloc;
     size_t i;
 
-    char *str_duration = "duration";
-    char *str_locks    = "locks";
-
-    /* term:
-     * [{'duration', {seconds, nanoseconds}}, {'locks', locks()}]
-     */
+    /* term: [{'duration', {seconds, nanoseconds}}, {'locks', locks()}] */
 
     /* duration tuple */ 
     dts  = bld_unstable_uint64(hpp, szp, duration->s);
@@ -4628,15 +4633,14 @@ static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_time_t *du
     tdur = erts_bld_tuple(hpp, szp, 2, adur, tdt);
    
     /* lock tuple */
-    
     aloc = erts_atom_put((byte *)str_locks, strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1);
 
     for(i = 0; i < current_locks->size; i++) {
-        lloc = lcnt_build_lock_term(hpp, szp, current_locks->elements[i], lloc);
+        lloc = lcnt_build_lock_term(hpp, szp, &current_locks->elements[i], lloc);
     }
 
     for(i = 0; i < deleted_locks->size; i++) {
-        lloc = lcnt_build_lock_term(hpp, szp, deleted_locks->elements[i], lloc);
+        lloc = lcnt_build_lock_term(hpp, szp, &deleted_locks->elements[i], lloc);
     }
 
     tloc = erts_bld_tuple(hpp, szp, 2, aloc, lloc);
@@ -4647,6 +4651,26 @@ static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_time_t *du
     return res;
 }
 
+static erts_lock_flags_t lcnt_atom_to_lock_category(Eterm atom) {
+    if(ERTS_IS_ATOM_STR("generic", atom)) {
+        return ERTS_LOCK_FLAGS_CATEGORY_GENERIC;
+    } else if(ERTS_IS_ATOM_STR("allocator", atom)) {
+        return ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR;
+    } else if(ERTS_IS_ATOM_STR("scheduler", atom)) {
+        return ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER;
+    } else if(ERTS_IS_ATOM_STR("process", atom)) {
+        return ERTS_LOCK_FLAGS_CATEGORY_PROCESS;
+    } else if(ERTS_IS_ATOM_STR("db", atom)) {
+        return ERTS_LOCK_FLAGS_CATEGORY_DB;
+    } else if(ERTS_IS_ATOM_STR("io", atom)) {
+        return ERTS_LOCK_FLAGS_CATEGORY_IO;
+    } else if(ERTS_IS_ATOM_STR("debug", atom)) {
+        return ERTS_LOCK_FLAGS_CATEGORY_DEBUG;
+    }
+
+    return 0;
+}
+
 #endif
 
 BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1)
@@ -4662,7 +4686,7 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1)
     if (BIF_ARG_1 == am_enabled) {
        BIF_RET(am_true);
     } else if (BIF_ARG_1 == am_info) {
-        lcnt_lock_info_vector_t current_locks, deleted_locks;
+        lcnt_sample_vector_t current_locks, deleted_locks;
         erts_lcnt_data_t data;
 
         Eterm *term_heap_start, *term_heap_end;
@@ -4671,8 +4695,8 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1)
 
         data = erts_lcnt_get_data();
 
-        current_locks = lcnt_build_lock_info_vector(data.current_locks);
-        deleted_locks = lcnt_build_lock_info_vector(data.deleted_locks);
+        current_locks = lcnt_build_sample_vector(data.current_locks);
+        deleted_locks = lcnt_build_sample_vector(data.deleted_locks);
 
         lcnt_build_result_term(NULL, &term_heap_size, &data.duration,
             &current_locks, &deleted_locks, NIL);
@@ -4685,8 +4709,8 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1)
 
         HRelease(BIF_P, term_heap_start + term_heap_size, term_heap_end);
 
-        lcnt_destroy_lock_info_vector(&current_locks);
-        lcnt_destroy_lock_info_vector(&deleted_locks);
+        lcnt_destroy_sample_vector(&current_locks);
+        lcnt_destroy_sample_vector(&deleted_locks);
 
         BIF_RET(result);
     } else if (BIF_ARG_1 == am_clear) {
@@ -4696,46 +4720,39 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1)
     } else if (is_tuple(BIF_ARG_1)) {
         Eterm* ptr = tuple_val(BIF_ARG_1);
 
-        if ((arityval(ptr[0]) == 2) && (ptr[2] == am_false || ptr[2] == am_true)) {
-           Eterm res;
-
-            int lock_opt = 0, enable = (ptr[2] == am_true) ? 1 : 0;
-            if (ERTS_IS_ATOM_STR("copy_save", ptr[1])) {
-                lock_opt = ERTS_LCNT_OPT_COPYSAVE;
-            } else if (ERTS_IS_ATOM_STR("process_locks", ptr[1])) {
-                lock_opt = ERTS_LCNT_OPT_PROCLOCK;
-            } else if (ERTS_IS_ATOM_STR("port_locks", ptr[1])) {
-                lock_opt = ERTS_LCNT_OPT_PORTLOCK;
-            } else if (ERTS_IS_ATOM_STR("location", ptr[1])) {
-                lock_opt = ERTS_LCNT_OPT_LOCATION;
-            } else {
+        if(arityval(ptr[0]) != 2) {
+            BIF_ERROR(BIF_P, BADARG);
+        } else if(ERTS_IS_ATOM_STR("mask", ptr[1])) {
+            erts_lock_flags_t category_mask = 0;
+            Eterm categories = ptr[2];
+
+            if(!(is_list(categories) || is_nil(categories))) {
                 BIF_ERROR(BIF_P, BADARG);
             }
 
-            if (enable) {
-                res = erts_lcnt_set_rt_opt(lock_opt) ? am_true : am_false;
-            } else {
-                res = erts_lcnt_clear_rt_opt(lock_opt) ? am_true : am_false;
-            }
+            while(is_list(categories)) {
+                Eterm *cell = list_val(categories);
+                erts_lock_flags_t category;
 
-#ifdef ERTS_SMP
-            /* FIXME: make this non-blocking. */
-            erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-            erts_smp_thr_progress_block();
-#endif
+                category = lcnt_atom_to_lock_category(CAR(cell));
+
+                if(!category) {
+                    /* Return {error, bad_category, Category}? Or leave that to
+                     * the lcnt module? */
+                    BIF_ERROR(BIF_P, BADARG);
+                }
 
-            if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PORTLOCK) {
-                erts_lcnt_enable_io_lock_count(enable);
-            } else if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PROCLOCK) {
-                erts_lcnt_enable_proc_lock_count(enable);
+                category_mask |= category;
+                categories = CDR(cell);
             }
 
-#ifdef ERTS_SMP
-            erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
-            erts_smp_thr_progress_unblock();
-#endif
+            erts_lcnt_set_category_mask(category_mask);
+
+            BIF_RET(am_ok);
+        } else if(ERTS_IS_ATOM_STR("copy_save", ptr[1]) &&
+            (ptr[2] == am_true || ptr[2] == am_false)) {
 
-            BIF_RET(res);
+            erts_lcnt_set_preserve_info(ptr[2] == am_true);
         }
     }
 
diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c
index fc6fb5f868..2f8adc87d5 100644
--- a/erts/emulator/beam/erl_bif_unique.c
+++ b/erts/emulator/beam/erl_bif_unique.c
@@ -392,7 +392,8 @@ init_magic_ref_tables(void)
     erts_snprintf(&tblp->name[0], sizeof(tblp->name),
 		  "magic_ref_table_0");
     hash_init(0, &tblp->hash, &tblp->name[0], 1, hash_funcs);
-    erts_rwmtx_init(&tblp->rwmtx, "magic_ref_table");
+    erts_rwmtx_init(&tblp->rwmtx, "magic_ref_table", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
     hash_funcs.hash = nsched_mreft_hash;
     hash_funcs.cmp = nsched_mreft_cmp;
@@ -402,7 +403,8 @@ init_magic_ref_tables(void)
 	erts_snprintf(&tblp->name[0], sizeof(tblp->name),
 		      "magic_ref_table_%d", i);
 	hash_init(0, &tblp->hash, &tblp->name[0], 1, hash_funcs);
-	erts_rwmtx_init(&tblp->rwmtx, "magic_ref_table");
+	erts_rwmtx_init(&tblp->rwmtx, "magic_ref_table", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
     }
 }
 
diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
index 4347f9f2b7..9f3a96fa1e 100644
--- a/erts/emulator/beam/erl_cpu_topology.c
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -1706,7 +1706,8 @@ erts_init_cpu_topology(void)
 {
     int ix;
 
-    erts_smp_rwmtx_init(&cpuinfo_rwmtx, "cpu_info");
+    erts_smp_rwmtx_init(&cpuinfo_rwmtx, "cpu_info", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
     erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
 
     scheduler2cpu_map = erts_alloc(ERTS_ALC_T_CPUDATA,
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 17e0f2aeec..b83134c79c 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -575,9 +575,7 @@ delete_owned_table(Process *p, DbTable *tb)
     table_dec_refc(tb, 1);
 }
 
-
-static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock,
-				     char *rwname, char* fixname)
+static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock)
 {
 #ifdef ERTS_SMP
     erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
@@ -587,9 +585,10 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock,
 	rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count;
 #endif
 #ifdef ERTS_SMP
-    erts_smp_rwmtx_init_opt_x(&tb->common.rwlock, &rwmtx_opt,
-			      rwname, tb->common.the_name);
-    erts_smp_mtx_init_x(&tb->common.fixlock, fixname, tb->common.the_name);
+    erts_smp_rwmtx_init_opt(&tb->common.rwlock, &rwmtx_opt, "db_tab",
+        tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
+    erts_smp_mtx_init(&tb->common.fixlock, "db_tab_fix",
+        tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
     tb->common.is_thread_safe = !(tb->common.status & DB_FINE_LOCKED);
 #endif
 }
@@ -1753,8 +1752,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     /* Note, 'type' is *read only* from now on... */
 #endif
     erts_smp_refc_init(&tb->common.fix_count, 0);
-    db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ),
-		 "db_tab", "db_tab_fix");
+    db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ));
     tb->common.keypos = keypos;
     tb->common.owner = BIF_P->common.id;
     set_heir(BIF_P, tb, heir, heir_data);
@@ -3391,8 +3389,9 @@ void init_db(ErtsDbSpinCount db_spin_count)
 	rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count;
 
     for (i=0; i<META_NAME_TAB_LOCK_CNT; i++) {
-	erts_smp_rwmtx_init_opt_x(&meta_name_tab_rwlocks[i].lck, &rwmtx_opt,
-				  "meta_name_tab", make_small(i));
+        erts_smp_rwmtx_init_opt(&meta_name_tab_rwlocks[i].lck, &rwmtx_opt,
+            "meta_name_tab", make_small(i),
+            ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DB);
     }
 #endif
 
@@ -4334,3 +4333,47 @@ erts_ets_colliding_names(Process* p, Eterm name, Uint cnt)
     return list;
 }
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+
+void erts_lcnt_enable_db_lock_count(DbTable *tb, int enable) {
+    if(enable) {
+        erts_lcnt_install_new_lock_info(&tb->common.rwlock.lcnt, "db_tab",
+            tb->common.the_name, ERTS_LOCK_TYPE_RWMUTEX | ERTS_LOCK_FLAGS_CATEGORY_DB);
+        erts_lcnt_install_new_lock_info(&tb->common.fixlock.lcnt, "db_tab_fix",
+            tb->common.the_name, ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_DB);
+    } else {
+        erts_lcnt_uninstall(&tb->common.rwlock.lcnt);
+        erts_lcnt_uninstall(&tb->common.fixlock.lcnt);
+    }
+
+    if(IS_HASH_TABLE(tb->common.status)) {
+        erts_lcnt_enable_db_hash_lock_count(&tb->hash, enable);
+    }
+}
+
+static void lcnt_update_db_locks_per_sched(void *enable) {
+    ErtsSchedulerData *esdp;
+    DbTable *head;
+
+    esdp = erts_get_scheduler_data();
+    head = esdp->ets_tables.clist;
+
+    if(head) {
+        DbTable *iterator = head;
+
+        do {
+            if(is_table_alive(iterator)) {
+                erts_lcnt_enable_db_lock_count(iterator, !!enable);
+            }
+
+            iterator = iterator->common.all.next;
+        } while (iterator != head);
+    }
+}
+
+void erts_lcnt_update_db_locks(int enable) {
+    erts_schedule_multi_misc_aux_work(0, erts_no_schedulers,
+        &lcnt_update_db_locks_per_sched, (void*)(UWord)enable);
+}
+
+#endif /* ERTS_ENABLE_LOCK_COUNT */
\ No newline at end of file
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index 4ff9f224e8..d83126b3a2 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -129,6 +129,11 @@ extern erts_smp_atomic_t erts_ets_misc_mem_size;
 Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt);
 Uint erts_db_get_max_tabs(void);
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void erts_lcnt_enable_db_lock_count(DbTable *tb, int enable);
+void erts_lcnt_update_db_locks(int enable);
+#endif
+
 #endif /* ERL_DB_H__ */
 
 #if defined(ERTS_WANT_DB_INTERNAL__) && !defined(ERTS_HAVE_DB_INTERNAL__)
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 08a0f0e83b..ae9322dfd3 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -675,8 +675,8 @@ int db_create_hash(Process *p, DbTable *tbl)
 							      (DbTable *) tb,
 							      sizeof(DbTableHashFineLocks));	    	    
 	for (i=0; i<DB_HASH_LOCK_CNT; ++i) {
-	    erts_smp_rwmtx_init_opt_x(&tb->locks->lck_vec[i].lck, &rwmtx_opt,
-				      "db_hash_slot", make_small(i));
+            erts_smp_rwmtx_init_opt(&tb->locks->lck_vec[i].lck, &rwmtx_opt,
+                "db_hash_slot", tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
 	}
 	/* This important property is needed to guarantee the two buckets
     	 * involved in a grow/shrink operation it protected by the same lock:
@@ -3206,3 +3206,23 @@ Eterm erts_ets_hash_sizeof_ext_segtab(void)
     return make_small(((SIZEOF_EXT_SEGTAB(0)-1) / sizeof(UWord)) + 1);
 }
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void erts_lcnt_enable_db_hash_lock_count(DbTableHash *tb, int enable) {
+    int i;
+
+    if(tb->locks == NULL) {
+        return;
+    }
+
+    for(i = 0; i < DB_HASH_LOCK_CNT; i++) {
+        erts_lcnt_ref_t *ref = &tb->locks->lck_vec[i].lck.lcnt;
+
+        if(enable) {
+            erts_lcnt_install_new_lock_info(ref, "db_hash_slot", tb->common.the_name,
+                ERTS_LOCK_TYPE_RWMUTEX | ERTS_LOCK_FLAGS_CATEGORY_DB);
+        } else {
+            erts_lcnt_uninstall(ref);
+        }
+    }
+}
+#endif /* ERTS_ENABLE_LOCK_COUNT */
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index f491c85d95..523ed7860e 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -103,4 +103,8 @@ typedef struct {
 void db_calc_stats_hash(DbTableHash* tb, DbHashStats*);
 Eterm erts_ets_hash_sizeof_ext_segtab(void);
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void erts_lcnt_enable_db_hash_lock_count(DbTableHash *tb, int enable);
+#endif
+
 #endif /* _DB_HASH_H */
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 3b68abe5d7..49bbab55a8 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -146,7 +146,8 @@ void erl_drv_thr_init(void)
 			       sizeof(char *)*ERL_DRV_TSD_KEYS_INC);
     for (i = 0; i < ERL_DRV_TSD_KEYS_INC; i++)
 	used_tsd_keys[i] = NULL;
-    erts_mtx_init(&tsd_mtx, "drv_tsd");
+    erts_mtx_init(&tsd_mtx, "drv_tsd", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
 }
 
 /*
@@ -176,8 +177,8 @@ erl_drv_mutex_create(char *name)
 	    dmtx->name = no_name;
 	}
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_ref(&dmtx->lcnt);
-    erts_lcnt_install_new_lock_info(&dmtx->lcnt, dmtx->name, ERTS_LCNT_LT_MUTEX);
+    erts_lcnt_init_ref_x(&dmtx->lcnt, dmtx->name, NIL,
+        ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO);
 #endif
     }
     return dmtx;
@@ -369,8 +370,8 @@ erl_drv_rwlock_create(char *name)
 	    drwlck->name = no_name;
 	}
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_ref(&drwlck->lcnt);
-    erts_lcnt_install_new_lock_info(&drwlck->lcnt, drwlck->name, ERTS_LCNT_LT_RWMUTEX);
+    erts_lcnt_init_ref_x(&drwlck->lcnt, drwlck->name, NIL,
+        ERTS_LOCK_TYPE_RWMUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO);
 #endif
     }
     return drwlck;
@@ -413,7 +414,7 @@ erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_tryrlock()");
     res = ethr_rwmutex_tryrlock(&drwlck->rwmtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LCNT_LO_READ);
+    erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LOCK_OPTION_READ);
 #endif
     return res;
 #else
@@ -428,7 +429,7 @@ erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck)
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_rlock()");
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ);
+    erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LOCK_OPTION_READ);
 #endif
     ethr_rwmutex_rlock(&drwlck->rwmtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -444,7 +445,7 @@ erl_drv_rwlock_runlock(ErlDrvRWLock *drwlck)
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_runlock()");
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ);
+    erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LOCK_OPTION_READ);
 #endif
     ethr_rwmutex_runlock(&drwlck->rwmtx);
 #endif
@@ -459,7 +460,7 @@ erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_tryrwlock()");
     res = ethr_rwmutex_tryrwlock(&drwlck->rwmtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LCNT_LO_READ_WRITE);
+    erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LOCK_OPTION_RDWR);
 #endif
     return res;
 #else
@@ -474,7 +475,7 @@ erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck)
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_rwlock()");
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ_WRITE);
+    erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LOCK_OPTION_RDWR);
 #endif
     ethr_rwmutex_rwlock(&drwlck->rwmtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -490,7 +491,7 @@ erl_drv_rwlock_rwunlock(ErlDrvRWLock *drwlck)
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_rwunlock()");
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ_WRITE);
+    erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LOCK_OPTION_RDWR);
 #endif
     ethr_rwmutex_rwunlock(&drwlck->rwmtx);
 #endif
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index d18016c42e..535f677bb3 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -63,7 +63,8 @@ erts_init_fun_table(void)
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
-    erts_smp_rwmtx_init_opt(&erts_fun_table_lock, &rwmtx_opt, "fun_tab");
+    erts_smp_rwmtx_init_opt(&erts_fun_table_lock, &rwmtx_opt, "fun_tab", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
     f.hash = (H_FUN) fun_hash;
     f.cmp  = (HCMP_FUN) fun_cmp;
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 2ff49c97b3..23c5b56067 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -274,7 +274,8 @@ erts_init_gc(void)
     }
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-    erts_smp_mtx_init(&dirty_gc.mtx, "dirty_gc_info");
+    erts_smp_mtx_init(&dirty_gc.mtx, "dirty_gc_info", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
     init_gc_info(&dirty_gc.info);
 #endif
 
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index ac99f043e6..aaecde4d78 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -2355,8 +2355,12 @@ erl_start(int argc, char **argv)
 
 #ifdef ERTS_SMP
     erts_start_schedulers();
-    /* Let system specific code decide what to do with the main thread... */
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_post_startup();
+#endif
+
+    /* Let system specific code decide what to do with the main thread... */
     erts_sys_main_thread(); /* May or may not return! */
 #else
     {
@@ -2371,6 +2375,11 @@ erl_start(int argc, char **argv)
 	erts_sched_init_time_sup(esdp);
         erts_ets_sched_spec_data_init(esdp);
         erts_aux_work_timeout_late_init(esdp);
+
+#ifdef ERTS_ENABLE_LOCK_COUNT
+        erts_lcnt_post_startup();
+#endif
+
 	process_main(esdp->x_reg_array, esdp->f_reg_array);
     }
 #endif
diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c
index 4d4defd8b5..634509f880 100644
--- a/erts/emulator/beam/erl_instrument.c
+++ b/erts/emulator/beam/erl_instrument.c
@@ -1200,7 +1200,8 @@ erts_instr_init(int stat, int map_stat)
 
     stats = erts_alloc(ERTS_ALC_T_INSTR_INFO, sizeof(struct stats_));
 
-    erts_mtx_init(&instr_mutex, "instr");
+    erts_mtx_init(&instr_mutex, "instr", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
 
     mem_anchor = NULL;
 
@@ -1223,7 +1224,8 @@ erts_instr_init(int stat, int map_stat)
 
     if (map_stat) {
 
-	erts_mtx_init(&instr_x_mutex, "instr_x");
+	erts_mtx_init(&instr_x_mutex, "instr_x", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
 
 	erts_instr_memory_map = 1;
 	erts_instr_stat = 1;
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index f270d8baef..85ee703c99 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -104,7 +104,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"db_tab",				"address"		},
     {	"proc_status",				"pid"			},
     {	"proc_trace",				"pid"			},
-    {   "ports_snapshot",                       NULL                    },
     {	"db_tab_fix",				"address"		},
     {	"db_hash_slot",				"address"		},
     {	"node_table",				NULL			},
@@ -161,6 +160,9 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"mtrace_op",				NULL			},
     {	"instr_x",				NULL			},
     {	"instr",				NULL			},
+#ifdef ERTS_SMP
+    {	"pollsets_lock",			NULL			},
+#endif
     {	"alcu_allocator",			"index"			},
     {	"mseg",					NULL			},
 #ifdef ERTS_SMP
@@ -173,7 +175,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"get_time",				NULL			},
     {	"get_corrected_time",			NULL			},
     {	"breakpoints",				NULL			},
-    {	"pollsets_lock",			NULL			},
     {	"pix_lock",				"address"		},
     {	"run_queues_lists",			NULL			},
     {	"sched_stat",				NULL			},
diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index 2cf59aa367..d2e8f47d59 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -26,19 +26,26 @@
 
 #include "sys.h"
 
+#include "global.h"
+
 #include "erl_lock_count.h"
 #include "erl_thr_progress.h"
 
-#define LCNT_MAX_CARRIER_ENTRIES 255
-
-/* - Exported global - */
+#include "erl_node_tables.h"
+#include "erl_alloc_util.h"
+#include "erl_check_io.h"
+#include "erl_poll.h"
+#include "erl_db.h"
 
-Uint16 erts_lcnt_rt_options = ERTS_LCNT_OPT_PROCLOCK | ERTS_LCNT_OPT_LOCATION;
+#define LCNT_MAX_CARRIER_ENTRIES 255
 
 /* - Locals that are shared with the header implementation - */
 
-int lcnt_initialization_completed__ = 0;
+#ifdef DEBUG
+int lcnt_initialization_completed__;
+#endif
 
+erts_lock_flags_t lcnt_category_mask__;
 ethr_tsd_key lcnt_thr_data_key__;
 
 const int lcnt_log2_tab64__[64] = {
@@ -53,11 +60,25 @@ const int lcnt_log2_tab64__[64] = {
 
 /* - Local variables - */
 
+typedef struct lcnt_static_lock_ref_ {
+    erts_lcnt_ref_t *reference;
+
+    erts_lock_flags_t flags;
+    const char *name;
+    Eterm id;
+
+    struct lcnt_static_lock_ref_ *next;
+} lcnt_static_lock_ref_t;
+
+static ethr_atomic_t lcnt_static_lock_registry;
+
 static erts_lcnt_lock_info_list_t lcnt_current_lock_list;
 static erts_lcnt_lock_info_list_t lcnt_deleted_lock_list;
 
 static erts_lcnt_time_t lcnt_timer_start;
 
+static int lcnt_preserve_info;
+
 /* local functions */
 
 static void lcnt_clear_stats(erts_lcnt_lock_info_t *info) {
@@ -97,37 +118,6 @@ static lcnt_thread_data_t__ *lcnt_thread_data_alloc(void) {
     return eltd;
 }
 
-/* debug */
-
-#if 0
-static char* lock_opt(Uint16 flag) {
-    if ((flag & ERTS_LCNT_LO_WRITE) && (flag & ERTS_LCNT_LO_READ)) return "rw";
-    if  (flag & ERTS_LCNT_LO_READ )                                return "r ";
-    if  (flag & ERTS_LCNT_LO_WRITE)                                return " w";
-    return "--";
-}
-
-static void print_lock_x(erts_lcnt_lock_info_t *info, Uint16 flag, char *action) {
-    ethr_sint_t w_state, r_state;
-    char *type;
-
-    if (strcmp(info->name, "run_queue") != 0) return;
-    type = erts_lcnt_lock_type(info->flag);
-    r_state = ethr_atomic_read(&info->r_state);
-    w_state = ethr_atomic_read(&info->w_state);
-
-    if (info->flag & flag) {
-        erts_fprintf(stderr,"%10s [%24s] [r/w state %4ld/%4ld] %2s id %T\r\n",
-                action,
-                info->name,
-                r_state,
-                w_state,
-                type,
-                info->id);
-    }
-}
-#endif
-
 /* - List operations -
  *
  * Info entries are kept in a doubly linked list where each entry is locked
@@ -271,12 +261,7 @@ void lcnt_thr_progress_unmanaged_continue__(int handle) {
     return erts_thr_progress_unmanaged_continue(handle);
 }
 
-static void lcnt_deallocate_carrier_malloc(erts_lcnt_lock_info_carrier_t *carrier) {
-    ASSERT(ethr_atomic_read(&carrier->ref_count) == 0);
-    free(carrier);
-}
-
-static void lcnt_deallocate_carrier_erts(erts_lcnt_lock_info_carrier_t *carrier) {
+void lcnt_deallocate_carrier__(erts_lcnt_lock_info_carrier_t *carrier) {
     ASSERT(ethr_atomic_read(&carrier->ref_count) == 0);
     erts_free(ERTS_ALC_T_LCNT_CARRIER, (void*)carrier);
 }
@@ -323,7 +308,7 @@ static void lcnt_info_deallocate(erts_lcnt_lock_info_t *info) {
 static void lcnt_info_dispose(erts_lcnt_lock_info_t *info) {
     ASSERT(ethr_atomic_read(&info->ref_count) == 0);
 
-    if(erts_lcnt_rt_options & ERTS_LCNT_OPT_COPYSAVE) {
+    if(lcnt_preserve_info) {
         ethr_atomic_set(&info->ref_count, 1);
 
         /* Move straight to deallocation the next time around. */
@@ -336,10 +321,6 @@ static void lcnt_info_dispose(erts_lcnt_lock_info_t *info) {
 }
 
 static void lcnt_lock_info_init_helper(erts_lcnt_lock_info_t *info) {
-#ifdef DEBUG
-    ethr_atomic_init(&info->flowstate, 0);
-#endif
-
     ethr_atomic_init(&info->ref_count, 1);
     ethr_atomic32_init(&info->lock, 0);
 
@@ -356,22 +337,16 @@ erts_lcnt_lock_info_carrier_t *erts_lcnt_create_lock_info_carrier(int entry_coun
     size_t carrier_size, i;
 
     ASSERT(entry_count > 0 && entry_count <= LCNT_MAX_CARRIER_ENTRIES);
+    ASSERT(lcnt_initialization_completed__);
 
     carrier_size = sizeof(erts_lcnt_lock_info_carrier_t) +
                    sizeof(erts_lcnt_lock_info_t) * entry_count;
 
-    if(lcnt_initialization_completed__) {
-        result = (erts_lcnt_lock_info_carrier_t*)erts_alloc(ERTS_ALC_T_LCNT_CARRIER, carrier_size);
-        result->deallocate = &lcnt_deallocate_carrier_erts;
-    } else {
-        result = (erts_lcnt_lock_info_carrier_t*)malloc(carrier_size);
-        result->deallocate = &lcnt_deallocate_carrier_malloc;
-    }
+    result = (erts_lcnt_lock_info_carrier_t*)erts_alloc(ERTS_ALC_T_LCNT_CARRIER, carrier_size);
+    result->entry_count = entry_count;
 
     ethr_atomic_init(&result->ref_count, entry_count);
 
-    result->entry_count = entry_count;
-
     for(i = 0; i < entry_count; i++) {
         erts_lcnt_lock_info_t *info = &result->entries[i];
 
@@ -386,6 +361,24 @@ erts_lcnt_lock_info_carrier_t *erts_lcnt_create_lock_info_carrier(int entry_coun
 void erts_lcnt_install(erts_lcnt_ref_t *ref, erts_lcnt_lock_info_carrier_t *carrier) {
     ethr_sint_t swapped_carrier;
 
+#ifdef DEBUG
+    int i;
+
+    /* Verify that all locks share the same categories/static property; all
+     * other flags are fair game. */
+    for(i = 1; i < carrier->entry_count; i++) {
+        const erts_lock_flags_t SIGNIFICANT_DIFF_MASK =
+            ERTS_LOCK_FLAGS_MASK_CATEGORY | ERTS_LOCK_FLAGS_PROPERTY_STATIC;
+
+        erts_lcnt_lock_info_t *previous, *current;
+
+        previous = &carrier->entries[i - 1];
+        current = &carrier->entries[i];
+
+        ASSERT(!((previous->flags ^ current->flags) & SIGNIFICANT_DIFF_MASK));
+    }
+#endif
+
     swapped_carrier = ethr_atomic_cmpxchg_mb(ref, (ethr_sint_t)carrier, (ethr_sint_t)NULL);
 
     if(swapped_carrier != (ethr_sint_t)NULL) {
@@ -394,7 +387,7 @@ void erts_lcnt_install(erts_lcnt_ref_t *ref, erts_lcnt_lock_info_carrier_t *carr
         ethr_atomic_set(&carrier->ref_count, 0);
 #endif
 
-        carrier->deallocate(carrier);
+        lcnt_deallocate_carrier__(carrier);
     } else {
         lcnt_insert_list_carrier(&lcnt_current_lock_list, carrier);
     }
@@ -411,6 +404,61 @@ void erts_lcnt_uninstall(erts_lcnt_ref_t *ref) {
     }
 }
 
+/* - Static lock registry -
+ *
+ * Since static locks can be trusted to never disappear, we can track them
+ * pretty cheaply and won't need to bother writing an "erts_lcnt_update_xx"
+ * variant. */
+
+static void lcnt_init_static_lock_registry(void) {
+    ethr_atomic_init(&lcnt_static_lock_registry, (ethr_sint_t)NULL);
+}
+
+static void lcnt_update_static_locks(void) {
+    lcnt_static_lock_ref_t *iterator =
+        (lcnt_static_lock_ref_t*)ethr_atomic_read(&lcnt_static_lock_registry);
+
+    while(iterator != NULL) {
+        if(!erts_lcnt_check_enabled(iterator->flags)) {
+            erts_lcnt_uninstall(iterator->reference);
+        } else if(!erts_lcnt_check_ref_installed(iterator->reference)) {
+            erts_lcnt_lock_info_carrier_t *carrier = erts_lcnt_create_lock_info_carrier(1);
+
+            erts_lcnt_init_lock_info_idx(carrier, 0, iterator->name, iterator->id, iterator->flags);
+
+            erts_lcnt_install(iterator->reference, carrier);
+        }
+
+        iterator = iterator->next;
+    }
+}
+
+void lcnt_register_static_lock__(erts_lcnt_ref_t *reference, const char *name, Eterm id,
+                                 erts_lock_flags_t flags) {
+    lcnt_static_lock_ref_t *lock = malloc(sizeof(lcnt_static_lock_ref_t));
+    int retry_insertion;
+
+    ASSERT(flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC);
+
+    lock->reference = reference;
+    lock->flags = flags;
+    lock->name = name;
+    lock->id = id;
+
+    do {
+        ethr_sint_t swapped_head;
+
+        lock->next = (lcnt_static_lock_ref_t*)ethr_atomic_read(&lcnt_static_lock_registry);
+
+        swapped_head = ethr_atomic_cmpxchg_acqb(
+            &lcnt_static_lock_registry,
+            (ethr_sint_t)lock,
+            (ethr_sint_t)lock->next);
+
+        retry_insertion = (swapped_head != (ethr_sint_t)lock->next);
+    } while(retry_insertion);
+}
+
 /* - Initialization - */
 
 void erts_lcnt_pre_thr_init() {
@@ -422,6 +470,8 @@ void erts_lcnt_pre_thr_init() {
 
     lcnt_init_list(&lcnt_current_lock_list);
     lcnt_init_list(&lcnt_deleted_lock_list);
+
+    lcnt_init_static_lock_registry();
 }
 
 void erts_lcnt_post_thr_init() {
@@ -438,8 +488,16 @@ void erts_lcnt_late_init() {
     erts_lcnt_clear_counters();
     erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler);
 
+#ifdef DEBUG
     /* It's safe to use erts_alloc and thread progress past this point. */
     lcnt_initialization_completed__ = 1;
+#endif
+}
+
+void erts_lcnt_post_startup(void) {
+    /* Default to capturing everything to match the behavior of the old lock
+     * counter build. */
+    erts_lcnt_set_category_mask(ERTS_LOCK_FLAGS_MASK_CATEGORY);
 }
 
 void erts_lcnt_thread_setup() {
@@ -492,18 +550,74 @@ void erts_lcnt_release_lock_info(erts_lcnt_lock_info_t *info) {
     }
 }
 
-Uint16 erts_lcnt_set_rt_opt(Uint16 opt) {
-    Uint16 prev;
-    prev = (erts_lcnt_rt_options & opt);
-    erts_lcnt_rt_options |= opt;
-    return prev;
+erts_lock_flags_t erts_lcnt_get_category_mask() {
+    return lcnt_category_mask__;
 }
 
-Uint16 erts_lcnt_clear_rt_opt(Uint16 opt) {
-    Uint16 prev;
-    prev = (erts_lcnt_rt_options & opt);
-    erts_lcnt_rt_options &= ~opt;
-    return prev;
+#ifdef ERTS_ENABLE_KERNEL_POLL
+/* erl_poll/erl_check_io only exports one of these variants at a time, and we
+ * may need to use either one depending on emulator startup flags. */
+void erts_lcnt_update_pollset_locks_nkp(int);
+void erts_lcnt_update_pollset_locks_kp(int);
+
+void erts_lcnt_update_cio_locks_nkp(int);
+void erts_lcnt_update_cio_locks_kp(int);
+#endif
+
+void erts_lcnt_set_category_mask(erts_lock_flags_t mask) {
+    erts_lock_flags_t changed_categories;
+
+    ASSERT(!(mask & ~ERTS_LOCK_FLAGS_MASK_CATEGORY));
+    ASSERT(lcnt_initialization_completed__);
+
+    changed_categories = (lcnt_category_mask__ ^ mask);
+    lcnt_category_mask__ = mask;
+
+    if(changed_categories) {
+        lcnt_update_static_locks();
+    }
+
+    if(changed_categories & ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION) {
+        erts_lcnt_update_distribution_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
+    }
+
+    if(changed_categories & ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR) {
+        erts_lcnt_update_allocator_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR);
+    }
+
+    if(changed_categories & ERTS_LOCK_FLAGS_CATEGORY_PROCESS) {
+        erts_lcnt_update_process_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_PROCESS);
+    }
+
+    if(changed_categories & ERTS_LOCK_FLAGS_CATEGORY_IO) {
+#ifdef ERTS_ENABLE_KERNEL_POLL
+        if(erts_use_kernel_poll) {
+            erts_lcnt_update_pollset_locks_kp(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
+            erts_lcnt_update_cio_locks_kp(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
+        } else {
+            erts_lcnt_update_pollset_locks_nkp(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
+            erts_lcnt_update_cio_locks_nkp(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
+        }
+#else
+        erts_lcnt_update_pollset_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
+        erts_lcnt_update_cio_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
+#endif
+
+        erts_lcnt_update_driver_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
+        erts_lcnt_update_port_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
+    }
+
+    if(changed_categories & ERTS_LOCK_FLAGS_CATEGORY_DB) {
+        erts_lcnt_update_db_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_DB);
+    }
+}
+
+void erts_lcnt_set_preserve_info(int enable) {
+    lcnt_preserve_info = enable;
+}
+
+int erts_lcnt_get_preserve_info() {
+    return lcnt_preserve_info;
 }
 
 void erts_lcnt_clear_counters(void) {
@@ -538,17 +652,6 @@ erts_lcnt_data_t erts_lcnt_get_data(void) {
     return result;
 }
 
-const char *erts_lcnt_lock_type(Uint16 type) {
-    switch(type & ERTS_LCNT_LT_ALL) {
-        case ERTS_LCNT_LT_SPINLOCK:   return "spinlock";
-        case ERTS_LCNT_LT_RWSPINLOCK: return "rw_spinlock";
-        case ERTS_LCNT_LT_MUTEX:      return "mutex";
-        case ERTS_LCNT_LT_RWMUTEX:    return "rw_mutex";
-        case ERTS_LCNT_LT_PROCLOCK:   return "proclock";
-        default:                      return "";
-    }
-}
-
 int erts_lcnt_iterate_list(erts_lcnt_lock_info_list_t *list, erts_lcnt_lock_info_t **iterator) {
     erts_lcnt_lock_info_t *current, *next;
 
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index c89dfcba47..3181dbcad4 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -43,17 +43,7 @@
  *
  * Each instance of a lock is the unique lock, i.e. set and id in that set.
  * For each lock there is a set of statistics with where and what impact
- * the lock aqusition had.
- *
- * Runtime options:
- * - location, reserved and not used.
- * - proclock, disable proclock counting. Used when performance might be an
- *   issue. Accessible from erts_debug:lock_counters({process_locks, bool()}).
- *   Default: off.
- * - copysave, enable saving of destroyed locks (and thereby its statistics).
- *   If memory constraints is an issue this need to be disabled.
- *   Accessible from erts_debug:lock_counters({copy_save, bool()}).
- *   Default: off.
+ * the lock acquisition had.
  */
 
 #ifndef ERTS_LOCK_COUNT_H__
@@ -67,11 +57,12 @@
 
 #include "sys.h"
 #include "ethread.h"
+
 #include "erl_term.h"
+#include "erl_lock_flags.h"
 
-#define ERTS_LCNT_MAX_LOCK_LOCATIONS  (10)
+#define ERTS_LCNT_MAX_LOCK_LOCATIONS  (5)
 
-/* histogram */
 #define ERTS_LCNT_HISTOGRAM_MAX_NS    (((unsigned long)1LL << 28) - 1)
 #if 0 || defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT)
 #define ERTS_LCNT_HISTOGRAM_SLOT_SIZE (30)
@@ -81,34 +72,6 @@
 #define ERTS_LCNT_HISTOGRAM_RSHIFT    (10)
 #endif
 
-#define ERTS_LCNT_LT_SPINLOCK   (((Uint16) 1) << 0)
-#define ERTS_LCNT_LT_RWSPINLOCK (((Uint16) 1) << 1)
-#define ERTS_LCNT_LT_MUTEX      (((Uint16) 1) << 2)
-#define ERTS_LCNT_LT_RWMUTEX    (((Uint16) 1) << 3)
-#define ERTS_LCNT_LT_PROCLOCK   (((Uint16) 1) << 4)
-#define ERTS_LCNT_LT_ALLOC      (((Uint16) 1) << 5)
-
-#define ERTS_LCNT_LO_READ       (((Uint16) 1) << 6)
-#define ERTS_LCNT_LO_WRITE      (((Uint16) 1) << 7)
-
-#define ERTS_LCNT_LT_DISABLE    (((Uint16) 1) << 8)
-
-#define ERTS_LCNT_LO_READ_WRITE ( ERTS_LCNT_LO_READ  \
-                                | ERTS_LCNT_LO_WRITE )
-
-#define ERTS_LCNT_LT_ALL        ( ERTS_LCNT_LT_SPINLOCK   \
-                                | ERTS_LCNT_LT_RWSPINLOCK \
-                                | ERTS_LCNT_LT_MUTEX      \
-                                | ERTS_LCNT_LT_RWMUTEX    \
-                                | ERTS_LCNT_LT_PROCLOCK   )
-
-/* Runtime options */
-
-#define ERTS_LCNT_OPT_LOCATION (((Uint16) 1) << 1)
-#define ERTS_LCNT_OPT_PROCLOCK (((Uint16) 1) << 2)
-#define ERTS_LCNT_OPT_PORTLOCK (((Uint16) 1) << 3)
-#define ERTS_LCNT_OPT_COPYSAVE (((Uint16) 1) << 4)
-
 typedef struct {
     unsigned long s;
     unsigned long ns;
@@ -139,9 +102,10 @@ typedef struct {
 } erts_lcnt_lock_stats_t;
 
 typedef struct lcnt_lock_info_t_ {
-    const char *name; /**< Lock name */
-    Uint16 flag;      /**< Lock type */
-    Eterm id;         /**< Id if possible, must be an immediate */
+    erts_lock_flags_t flags;
+    const char *name;
+    /** @brief Id if possible, must be an immediate */
+    Eterm id;
 
     /* The first entry is reserved as a fallback for when location information
      * is missing, and when the lock is used in more than (MAX_LOCK_LOCATIONS
@@ -158,15 +122,6 @@ typedef struct lcnt_lock_info_t_ {
     /** @brief Read state. 0 = not taken, > 0 -> writes will wait */
     ethr_atomic_t r_state;
 
-#ifdef LCNT_DEBUG_LOCK_FLOW
-    /** @brief Tracks lock/unlock operations. This will explode if the lock is
-     * held at the time lock counting is installed.
-     *
-     * Avoid enabling existing locks at runtime while running in this
-     * configuration. */
-    ethr_atomic_t flowstate;
-#endif
-
     struct lcnt_lock_info_t_ *prev;
     struct lcnt_lock_info_t_ *next;
 
@@ -200,7 +155,10 @@ typedef ethr_atomic_t erts_lcnt_ref_t;
 
 /* -- Globals -------------------------------------------------------------- */
 
-extern Uint16 erts_lcnt_rt_options;
+/** @brief Checks whether counting is enabled for any of the given
+ * categories. */
+#define erts_lcnt_check_enabled(flags) \
+    (lcnt_category_mask__ & flags)
 
 /* -- Lock operations ------------------------------------------------------
  *
@@ -259,12 +217,6 @@ void erts_lcnt_trylock_opt(erts_lcnt_ref_t *ref, int result, Uint16 option);
  *
  * Refer to \c erts_lcnt_lock for example usage. */
 
-ERTS_GLB_FORCE_INLINE
-int erts_lcnt_open_ref(erts_lcnt_ref_t *ref, int *handle, erts_lcnt_lock_info_carrier_t **result);
-
-ERTS_GLB_FORCE_INLINE
-void erts_lcnt_close_ref(int handle, erts_lcnt_lock_info_carrier_t *carrier);
-
 ERTS_GLB_INLINE
 void erts_lcnt_lock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index);
 ERTS_GLB_INLINE
@@ -290,22 +242,43 @@ void erts_lcnt_trylock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index
 
 /* -- Reference operations ------------------------------------------------- */
 
-erts_lcnt_lock_info_carrier_t *erts_lcnt_create_lock_info_carrier(int count);
-
-/** @brief Fills in the name and lock type of the given index. */
-#define erts_lcnt_init_lock_info_idx(carrier, index, name, flag) \
-    erts_lcnt_init_lock_info_x_idx(carrier, index, name, flag, NIL)
+/** @brief Registers a lock counter reference; this must be called prior to
+ * using any other functions in this module. */
+ERTS_GLB_INLINE
+void erts_lcnt_init_ref(erts_lcnt_ref_t *ref);
 
-/** @copydoc erts_lcnt_install_new_lock_info
+/** @brief As \c erts_lcnt_init_ref, but also enables lock counting right
+ * away if appropriate to reduce noise.
  * @param id An immediate erlang term with whatever extra data you want to
  * identify this lock with. */
 ERTS_GLB_INLINE
-void erts_lcnt_init_lock_info_x_idx(erts_lcnt_lock_info_carrier_t *carrier, int index,
-    const char *name, Uint16 flag, Eterm id);
+void erts_lcnt_init_ref_x(erts_lcnt_ref_t *ref, const char *name,
+    Eterm id, erts_lock_flags_t flags);
 
-/** @brief Initializes a lock counter reference; this must be called prior to
- * using any other functions in this module. */
-#define erts_lcnt_init_ref(ref) ethr_atomic_init(ref, (ethr_sint_t)NULL);
+/** @brief Checks whether counting is enabled on the given reference. */
+ERTS_GLB_FORCE_INLINE
+int erts_lcnt_check_ref_installed(erts_lcnt_ref_t *ref);
+
+/** @brief Convenience macro to re/enable counting on an already initialized
+ * reference. Don't forget to specify the lock type in \c flags! */
+#define erts_lcnt_install_new_lock_info(ref, name, id, flags) \
+    if(!erts_lcnt_check_ref_installed(ref)) { \
+        erts_lcnt_lock_info_carrier_t *__carrier; \
+        __carrier = erts_lcnt_create_lock_info_carrier(1);\
+        erts_lcnt_init_lock_info_idx(__carrier, 0, name, id, flags); \
+        erts_lcnt_install(ref, __carrier);\
+    } while(0)
+
+erts_lcnt_lock_info_carrier_t *erts_lcnt_create_lock_info_carrier(int count);
+
+/* @brief Initializes the lock info at the given index.
+ * @param id An immediate erlang term with whatever extra data you want to
+ * identify this lock with.
+ * @param flags The flags the lock itself was initialized with. Keep in mind
+ * that all locks in a carrier must share the same category/static property. */
+ERTS_GLB_INLINE
+void erts_lcnt_init_lock_info_idx(erts_lcnt_lock_info_carrier_t *carrier, int index,
+    const char *name, Eterm id, erts_lock_flags_t flags);
 
 /** @brief Atomically installs the given lock counters. Nops (and releases the
  * provided carrier) if something was already installed. */
@@ -315,21 +288,11 @@ void erts_lcnt_install(erts_lcnt_ref_t *ref, erts_lcnt_lock_info_carrier_t *carr
  * nothing was installed. */
 void erts_lcnt_uninstall(erts_lcnt_ref_t *ref);
 
-/** @brief Convenience macro to install a single lock counter of the given
- * name and type. */
-#define erts_lcnt_install_new_lock_info(reference, name, flag) \
-    erts_lcnt_install_new_lock_info_x(reference, name, flag, NIL)
+ERTS_GLB_FORCE_INLINE
+int erts_lcnt_open_ref(erts_lcnt_ref_t *ref, int *handle, erts_lcnt_lock_info_carrier_t **result);
 
-/** @copydoc erts_lcnt_install_new_lock_info
- * @param id An immediate erlang term with whatever extra data you want to
- * identify this lock with. */
-#define erts_lcnt_install_new_lock_info_x(reference, name, flag, id) \
-    do { \
-        erts_lcnt_lock_info_carrier_t *__carrier; \
-        __carrier = erts_lcnt_create_lock_info_carrier(1); \
-        erts_lcnt_init_lock_info_x_idx(__carrier, 0, name, flag, id); \
-        erts_lcnt_install(reference, __carrier); \
-    } while(0)
+ERTS_GLB_FORCE_INLINE
+void erts_lcnt_close_ref(int handle, erts_lcnt_lock_info_carrier_t *carrier);
 
 /* -- Module initialization ------------------------------------------------ */
 
@@ -337,6 +300,11 @@ void erts_lcnt_pre_thr_init(void);
 void erts_lcnt_post_thr_init(void);
 void erts_lcnt_late_init(void);
 
+/* @brief Called after everything in the system has been initialized, including
+ * the schedulers. This is mainly a backwards compatibility shim for matching
+ * the old lcnt behavior where all lock counting was enabled by default. */
+void erts_lcnt_post_startup(void);
+
 void erts_lcnt_thread_setup(void);
 void erts_lcnt_thread_exit_handler(void);
 
@@ -357,7 +325,7 @@ void erts_lcnt_thread_exit_handler(void);
 int erts_lcnt_iterate_list(erts_lcnt_lock_info_list_t *list, erts_lcnt_lock_info_t **iterator);
 
 /** @brief Clears the counter state of all locks, and releases all locks
- * preserved through ERTS_LCNT_OPT_COPYSAVE (if any). */
+ * preserved through erts_lcnt_set_preserve_info (if any). */
 void erts_lcnt_clear_counters(void);
 
 /** @brief Retrieves the global lock counter state.
@@ -369,10 +337,24 @@ erts_lcnt_data_t erts_lcnt_get_data(void);
 void erts_lcnt_retain_lock_info(erts_lcnt_lock_info_t *info);
 void erts_lcnt_release_lock_info(erts_lcnt_lock_info_t *info);
 
-Uint16 erts_lcnt_set_rt_opt(Uint16 opt);
-Uint16 erts_lcnt_clear_rt_opt(Uint16 opt);
+/** @brief Sets whether to preserve the info of destroyed/uninstalled locks.
+ *
+ * This option makes no distinction whether the lock was destroyed or if lock
+ * counting was simply disabled, so erts_lcnt_set_category_mask must not be
+ * used while this option is active. */
+void erts_lcnt_set_preserve_info(int enable);
 
-const char *erts_lcnt_lock_type(Uint16 type);
+int erts_lcnt_get_preserve_info(void);
+
+/** @brief Updates the category mask, enabling or disabling counting on the
+ * affected locks as necessary.
+ *
+ * This is not guaranteed to find all existing locks; only those that are
+ * flagged as static locks and those reachable through other means can be
+ * altered. */
+void erts_lcnt_set_category_mask(erts_lock_flags_t mask);
+
+erts_lock_flags_t erts_lcnt_get_category_mask(void);
 
 /* -- Inline implementation ------------------------------------------------ */
 
@@ -390,8 +372,6 @@ typedef struct { Uint64 _[4]; } LcntThrPrgrLaterOp;
 struct lcnt_lock_info_carrier_ {
     ethr_atomic_t ref_count;
 
-    void (*deallocate)(struct lcnt_lock_info_carrier_ *);
-
     LcntThrPrgrLaterOp release_entries;
 
     unsigned char entry_count;
@@ -404,14 +384,19 @@ typedef struct {
     int lock_in_conflict;   /* bool */
 } lcnt_thread_data_t__;
 
+extern const int lcnt_log2_tab64__[];
+
 extern ethr_tsd_key lcnt_thr_data_key__;
+extern erts_lock_flags_t lcnt_category_mask__;
 
-/** @brief Some operations (eg erts_alloc or erts_thr_progress_unmanaged_delay)
- * are unsafe in the early stages of initialization, so we're using this flag
- * to know when we can move over to normal operation. */
+#ifdef DEBUG
 extern int lcnt_initialization_completed__;
+#endif
 
-extern const int lcnt_log2_tab64__[];
+void lcnt_register_static_lock__(erts_lcnt_ref_t *reference, const char *name, Eterm id,
+    erts_lock_flags_t flags);
+
+void lcnt_deallocate_carrier__(erts_lcnt_lock_info_carrier_t *carrier);
 
 ERTS_GLB_INLINE
 int lcnt_log2__(Uint64 v);
@@ -557,29 +542,27 @@ void lcnt_dec_lock_state__(ethr_atomic_t *l_state) {
 
 ERTS_GLB_INLINE
 erts_lcnt_lock_stats_t *lcnt_get_lock_stats__(erts_lcnt_lock_info_t *info, char *file, unsigned int line) {
-    ASSERT(info->location_count >= 1 && info->location_count <= ERTS_LCNT_MAX_LOCK_LOCATIONS);
+    unsigned int i;
 
-    if(erts_lcnt_rt_options & ERTS_LCNT_OPT_LOCATION) {
-        unsigned int i;
+    ASSERT(info->location_count >= 1 && info->location_count <= ERTS_LCNT_MAX_LOCK_LOCATIONS);
 
-        for(i = 0; i < info->location_count; i++) {
-            erts_lcnt_lock_stats_t *stats = &info->location_stats[i];
+    for(i = 0; i < info->location_count; i++) {
+        erts_lcnt_lock_stats_t *stats = &info->location_stats[i];
 
-            if(stats->file == file && stats->line == line) {
-                return stats;
-            }
+        if(stats->file == file && stats->line == line) {
+            return stats;
         }
+    }
 
-        if(info->location_count < ERTS_LCNT_MAX_LOCK_LOCATIONS) {
-            erts_lcnt_lock_stats_t *stats = &info->location_stats[info->location_count];
+    if(info->location_count < ERTS_LCNT_MAX_LOCK_LOCATIONS) {
+        erts_lcnt_lock_stats_t *stats = &info->location_stats[info->location_count];
 
-            stats->file = file;
-            stats->line = line;
+        stats->file = file;
+        stats->line = line;
 
-            info->location_count++;
+        info->location_count++;
 
-            return stats;
-        }
+        return stats;
     }
 
     return &info->location_stats[0];
@@ -596,10 +579,12 @@ lcnt_thread_data_t__ *lcnt_get_thread_data__(void) {
 
 ERTS_GLB_FORCE_INLINE
 int erts_lcnt_open_ref(erts_lcnt_ref_t *ref, int *handle, erts_lcnt_lock_info_carrier_t **result) {
-    if(!*ethr_atomic_addr(ref) || !lcnt_initialization_completed__) {
+    if(ERTS_LIKELY(!erts_lcnt_check_ref_installed(ref))) {
         return 0;
     }
 
+    ASSERT(lcnt_initialization_completed__);
+
     (*handle) = lcnt_thr_progress_unmanaged_delay__();
     (*result) = (erts_lcnt_lock_info_carrier_t*)ethr_atomic_read(ref);
 
@@ -624,6 +609,30 @@ void erts_lcnt_close_ref(int handle, erts_lcnt_lock_info_carrier_t *carrier) {
     }
 }
 
+ERTS_GLB_INLINE
+void erts_lcnt_init_ref(erts_lcnt_ref_t *ref) {
+    ethr_atomic_init(ref, (ethr_sint_t)NULL);
+}
+
+ERTS_GLB_INLINE
+void erts_lcnt_init_ref_x(erts_lcnt_ref_t *ref, const char *name,
+                          Eterm id, erts_lock_flags_t flags) {
+    erts_lcnt_init_ref(ref);
+
+    if(flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC) {
+        lcnt_register_static_lock__(ref, name, id, flags);
+    }
+
+    if(erts_lcnt_check_enabled(flags)) {
+        erts_lcnt_install_new_lock_info(ref, name, id, flags);
+    }
+}
+
+ERTS_GLB_FORCE_INLINE
+int erts_lcnt_check_ref_installed(erts_lcnt_ref_t *ref) {
+    return (!!*ethr_atomic_addr(ref));
+}
+
 ERTS_GLB_FORCE_INLINE
 void erts_lcnt_lock(erts_lcnt_ref_t *ref) {
     erts_lcnt_lock_info_carrier_t *carrier;
@@ -734,7 +743,7 @@ void erts_lcnt_trylock_opt(erts_lcnt_ref_t *ref, int result, Uint16 option) {
 
 ERTS_GLB_INLINE
 void erts_lcnt_lock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index) {
-    erts_lcnt_lock_opt_idx(carrier, index, ERTS_LCNT_LO_WRITE);
+    erts_lcnt_lock_opt_idx(carrier, index, ERTS_LOCK_OPTION_WRITE);
 }
 
 ERTS_GLB_INLINE
@@ -745,9 +754,9 @@ void erts_lcnt_lock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, U
 
     ASSERT(index < carrier->entry_count);
 
-    ASSERT((option & ERTS_LCNT_LO_READ) || (option & ERTS_LCNT_LO_WRITE));
+    ASSERT((option & ERTS_LOCK_OPTION_READ) || (option & ERTS_LOCK_OPTION_WRITE));
 
-    if(option & ERTS_LCNT_LO_WRITE) {
+    if(option & ERTS_LOCK_OPTION_WRITE) {
         ethr_sint_t w_state, r_state;
 
         w_state = ethr_atomic_inc_read(&info->w_state) - 1;
@@ -762,7 +771,8 @@ void erts_lcnt_lock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, U
         eltd->lock_in_conflict = (w_state > 0);
     }
 
-    if(option & ERTS_LCNT_LO_READ) {
+    if(option & ERTS_LOCK_OPTION_READ) {
+        ASSERT(info->flags & ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE);
         ethr_atomic_inc(&info->r_state);
     }
 
@@ -792,12 +802,6 @@ void erts_lcnt_lock_post_x_idx(erts_lcnt_lock_info_carrier_t *carrier, int index
 
     ASSERT(index < carrier->entry_count);
 
-#ifdef LCNT_DEBUG_LOCK_FLOW
-    if(!(info->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) {
-        ASSERT(ethr_atomic_inc_read(&info->flowstate) == 1);
-    }
-#endif
-
     /* If the lock was in conflict, update the time spent waiting. */
     stats = lcnt_get_lock_stats__(info, file, line);
     if(eltd->timer_set) {
@@ -819,15 +823,9 @@ void erts_lcnt_lock_post_x_idx(erts_lcnt_lock_info_carrier_t *carrier, int index
 
 ERTS_GLB_INLINE
 void erts_lcnt_unlock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index) {
-#ifdef LCNT_DEBUG_LOCK_FLOW
-    erts_lcnt_lock_info_t *info = &carrier->entries[index];
-
-    ASSERT(ethr_atomic_dec_read(&info->flowstate) == 0);
-#endif
-
     ASSERT(index < carrier->entry_count);
 
-    erts_lcnt_unlock_opt_idx(carrier, index, ERTS_LCNT_LO_WRITE);
+    erts_lcnt_unlock_opt_idx(carrier, index, ERTS_LOCK_OPTION_WRITE);
 }
 
 ERTS_GLB_INLINE
@@ -836,13 +834,14 @@ void erts_lcnt_unlock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index,
 
     ASSERT(index < carrier->entry_count);
 
-    ASSERT((option & ERTS_LCNT_LO_READ) || (option & ERTS_LCNT_LO_WRITE));
+    ASSERT((option & ERTS_LOCK_OPTION_READ) || (option & ERTS_LOCK_OPTION_WRITE));
 
-    if(option & ERTS_LCNT_LO_WRITE) {
+    if(option & ERTS_LOCK_OPTION_WRITE) {
         lcnt_dec_lock_state__(&info->w_state);
     }
 
-    if(option & ERTS_LCNT_LO_READ) {
+    if(option & ERTS_LOCK_OPTION_READ) {
+        ASSERT(info->flags & ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE);
         lcnt_dec_lock_state__(&info->r_state);
     }
 }
@@ -858,15 +857,9 @@ void erts_lcnt_lock_unacquire_idx(erts_lcnt_lock_info_carrier_t *carrier, int in
 
 ERTS_GLB_INLINE
 void erts_lcnt_trylock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result) {
-#ifdef LCNT_DEBUG_LOCK_FLOW
-    erts_lcnt_lock_info_t *info = &carrier->entries[index];
-
-    ASSERT(result == EBUSY || ethr_atomic_inc_read(&info->flowstate) == 1);
-#endif
-
     ASSERT(index < carrier->entry_count);
 
-    erts_lcnt_trylock_opt_idx(carrier, index, result, ERTS_LCNT_LO_WRITE);
+    erts_lcnt_trylock_opt_idx(carrier, index, result, ERTS_LOCK_OPTION_WRITE);
 }
 
 ERTS_GLB_INLINE
@@ -875,14 +868,15 @@ void erts_lcnt_trylock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index
 
     ASSERT(index < carrier->entry_count);
 
-    ASSERT((option & ERTS_LCNT_LO_READ) || (option & ERTS_LCNT_LO_WRITE));
+    ASSERT((option & ERTS_LOCK_OPTION_READ) || (option & ERTS_LOCK_OPTION_WRITE));
 
     if(result != EBUSY) {
-        if(option & ERTS_LCNT_LO_WRITE) {
+        if(option & ERTS_LOCK_OPTION_WRITE) {
             ethr_atomic_inc(&info->w_state);
         }
 
-        if(option & ERTS_LCNT_LO_READ) {
+        if(option & ERTS_LOCK_OPTION_READ) {
+            ASSERT(info->flags & ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE);
             ethr_atomic_inc(&info->r_state);
         }
 
@@ -894,13 +888,16 @@ void erts_lcnt_trylock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index
 }
 
 ERTS_GLB_INLINE
-void erts_lcnt_init_lock_info_x_idx(erts_lcnt_lock_info_carrier_t *carrier, int index,
-                                    const char *name, Uint16 flag, Eterm id) {
+void erts_lcnt_init_lock_info_idx(erts_lcnt_lock_info_carrier_t *carrier, int index,
+                                  const char *name, Eterm id, erts_lock_flags_t flags) {
     erts_lcnt_lock_info_t *info = &carrier->entries[index];
 
     ASSERT(is_immed(id));
 
-    info->flag = flag;
+    ASSERT(flags & ERTS_LOCK_FLAGS_MASK_TYPE);
+    ASSERT(flags & ERTS_LOCK_FLAGS_MASK_CATEGORY);
+
+    info->flags = flags;
     info->name = name;
     info->id = id;
 }
@@ -921,7 +918,7 @@ void lcnt_release_carrier__(erts_lcnt_lock_info_carrier_t *carrier) {
     ASSERT(count >= 0);
 
     if(count == 0) {
-        carrier->deallocate(carrier);
+        lcnt_deallocate_carrier__(carrier);
     }
 }
 
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index 2d70f0d874..6c477be615 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -76,7 +76,8 @@ void erts_msacc_early_init(void) {
 #ifndef ERTS_MSACC_ALWAYS_ON
     erts_msacc_enabled = 0;
 #endif
-    erts_rwmtx_init(&msacc_mutex,"msacc_list_mutex");
+    erts_rwmtx_init(&msacc_mutex, "msacc_list_mutex", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
 #ifdef USE_THREADS
     erts_tsd_key_create(&erts_msacc_key,"erts_msacc_key");
 #else
@@ -109,7 +110,8 @@ void erts_msacc_init_thread(char *type, int id, int managed) {
 #ifdef USE_THREADS
     erts_rwmtx_rwlock(&msacc_mutex);
     if (!managed) {
-        erts_mtx_init(&msacc->mtx,"msacc_unmanaged_mutex");
+        erts_mtx_init(&msacc->mtx, "msacc_unmanaged_mutex", NIL,
+            ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
         msacc->next = msacc_unmanaged;
         msacc_unmanaged = msacc;
         msacc_unmanaged_count++;
diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c
index 19bb7d5b31..f2a660f085 100644
--- a/erts/emulator/beam/erl_mtrace.c
+++ b/erts/emulator/beam/erl_mtrace.c
@@ -583,8 +583,10 @@ void erts_mtrace_init(char *receiver, char *nodename)
 	byte ip_addr[4];
 	Uint16 port;
 
-	erts_mtx_init(&mtrace_buf_mutex, "mtrace_buf");
-	erts_mtx_init(&mtrace_op_mutex, "mtrace_op");
+        erts_mtx_init(&mtrace_buf_mutex, "mtrace_buf", NIL,
+            ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
+        erts_mtx_init(&mtrace_op_mutex, "mtrace_op", NIL,
+            ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
 
 	socket_desc = erts_sock_open();
 	if (socket_desc == ERTS_SOCK_INVALID_SOCKET) {
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index ea835d1b64..399626de74 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2398,7 +2398,8 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t data_sz)
     erts_refc_inc(&resource->type->refc, 2);
     if (type->down) {
         resource->monitors = (ErtsResourceMonitors*) (resource->data + monitors_offs);
-        erts_smp_mtx_init(&resource->monitors->lock, "resource_monitors");
+        erts_smp_mtx_init(&resource->monitors->lock, "resource_monitors", NIL,
+            ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
         resource->monitors->root = NULL;
         resource->monitors->pending_failed_fire = 0;
         resource->monitors->is_dying = 0;
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 3c5945d48d..deadf435e9 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -85,21 +85,20 @@ dist_table_cmp(void *dep1, void *dep2)
 static void*
 dist_table_alloc(void *dep_tmpl)
 {
-    Eterm chnl_nr;
     Eterm sysname;
     DistEntry *dep;
     erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
 
     sysname = ((DistEntry *) dep_tmpl)->sysname;
-    chnl_nr = make_small((Uint) atom_val(sysname));
     dep = (DistEntry *) erts_alloc(ERTS_ALC_T_DIST_ENTRY, sizeof(DistEntry));
 
     dist_entries++;
 
     dep->prev				= NULL;
     erts_smp_refc_init(&dep->refc, -1);
-    erts_smp_rwmtx_init_opt_x(&dep->rwmtx, &rwmtx_opt, "dist_entry", chnl_nr);
+    erts_smp_rwmtx_init_opt(&dep->rwmtx, &rwmtx_opt, "dist_entry", sysname,
+        ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
     dep->sysname			= sysname;
     dep->cid				= NIL;
     dep->connection_id			= 0;
@@ -107,12 +106,14 @@ dist_table_alloc(void *dep_tmpl)
     dep->flags				= 0;
     dep->version			= 0;
 
-    erts_smp_mtx_init_x(&dep->lnk_mtx, "dist_entry_links", chnl_nr);
+    erts_smp_mtx_init(&dep->lnk_mtx, "dist_entry_links", sysname,
+        ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
     dep->node_links			= NULL;
     dep->nlinks				= NULL;
     dep->monitors			= NULL;
 
-    erts_smp_mtx_init_x(&dep->qlock, "dist_entry_out_queue", chnl_nr);
+    erts_smp_mtx_init(&dep->qlock, "dist_entry_out_queue", sysname,
+        ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
     dep->qflgs				= 0;
     dep->qsize				= 0;
     dep->out_queue.first		= NULL;
@@ -760,8 +761,10 @@ void erts_init_node_tables(int dd_sec)
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
-    erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table");
-    erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table");
+    erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
+    erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
 
     f.hash       = (H_FUN)		dist_table_hash;
     f.cmp        = (HCMP_FUN)		dist_table_cmp;
@@ -818,6 +821,33 @@ int erts_lc_is_de_rlocked(DistEntry *dep)
 #endif
 #endif
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+
+static void erts_lcnt_enable_dist_lock_count(void *dep_raw, void *enable) {
+    DistEntry *dep = (DistEntry*)dep_raw;
+
+    if(enable) {
+        erts_lcnt_install_new_lock_info(&dep->rwmtx.lcnt, "dist_entry", dep->sysname, 
+            ERTS_LOCK_TYPE_RWMUTEX | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
+        erts_lcnt_install_new_lock_info(&dep->lnk_mtx.lcnt, "dist_entry_links", dep->sysname,
+            ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
+        erts_lcnt_install_new_lock_info(&dep->qlock.lcnt, "dist_entry_out_queue", dep->sysname,
+            ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
+    } else {
+        erts_lcnt_uninstall(&dep->rwmtx.lcnt);
+        erts_lcnt_uninstall(&dep->lnk_mtx.lcnt);
+        erts_lcnt_uninstall(&dep->qlock.lcnt);
+    }
+}
+
+void erts_lcnt_update_distribution_locks(int enable) {
+    erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
+    hash_foreach(&erts_dist_table, erts_lcnt_enable_dist_lock_count,
+        (void*)(UWord)enable);
+    erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
+}
+#endif
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
  * The following is only supposed to be used for testing, and debugging.     *
  *                                                                           *
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 489da1ba17..91bcb4fce1 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -195,6 +195,10 @@ int erts_lc_is_de_rwlocked(DistEntry *);
 int erts_lc_is_de_rlocked(DistEntry *);
 #endif
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void erts_lcnt_update_distribution_locks(int enable);
+#endif
+
 ERTS_GLB_INLINE void erts_deref_dist_entry(DistEntry *dep);
 ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np);
 ERTS_GLB_INLINE void erts_smp_de_rlock(DistEntry *dep);
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index ab536c6f27..4e2e168320 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -188,13 +188,7 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id)
     ptsp->taskq.in.last = NULL;
     erts_smp_atomic32_init_nob(&ptsp->flags, 0);
 #ifdef ERTS_SMP
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_mtx_init_x_opt(&ptsp->mtx, lock_str, instr_id,
-			((erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)
-			 ? 0 : ERTS_LCNT_LT_DISABLE));
-#else
-    erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id);
-#endif
+    erts_mtx_init(&ptsp->mtx, lock_str, instr_id, ERTS_LOCK_FLAGS_CATEGORY_IO);
 #endif
 }
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index da27c7e7c6..f18537bcf1 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -789,7 +789,9 @@ erts_late_init_process(void)
 {
     int ix;
 
-    erts_smp_spinlock_init(&erts_sched_stat.lock, "sched_stat");
+    erts_smp_spinlock_init(&erts_sched_stat.lock, "sched_stat", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
+
     for (ix = 0; ix < ERTS_NO_PRIO_LEVELS; ix++) {
 	Eterm atom;
 	char *atom_str;
@@ -1958,12 +1960,13 @@ erts_schedule_multi_misc_aux_work(int ignore_self,
     int id, self = 0;
 
     if (ignore_self) {
-	ErtsSchedulerData *esdp = erts_get_scheduler_data();
-#ifdef ERTS_DIRTY_SCHEDULERS
-	ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
-#endif
-	if (esdp)
-	    self = (int) esdp->no;
+        ErtsSchedulerData *esdp = erts_get_scheduler_data();
+
+        /* ignore_self is meaningless on dirty schedulers since aux work can
+         * only run on normal schedulers, and their ids do not translate. */
+        if(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+            self = (int)esdp->no;
+        }
     }
 
     ASSERT(0 < max_sched && max_sched <= erts_no_schedulers);
@@ -6183,13 +6186,17 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	 * id if the esdp->no <-> ix+1 mapping change.
 	 */
 
-	erts_smp_mtx_init_x(&rq->mtx, "run_queue", make_small(ix + 1));
+	erts_smp_mtx_init(&rq->mtx, "run_queue", make_small(ix + 1),
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
 	erts_smp_cnd_init(&rq->cnd);
 
 #ifdef ERTS_DIRTY_SCHEDULERS
 #ifdef ERTS_SMP
-	if (ERTS_RUNQ_IX_IS_DIRTY(ix))
-	    erts_smp_spinlock_init(&rq->sleepers.lock, "dirty_run_queue_sleep_list");
+        if (ERTS_RUNQ_IX_IS_DIRTY(ix)) {
+            erts_smp_spinlock_init(&rq->sleepers.lock, "dirty_run_queue_sleep_list",
+                make_small(ix + 1),
+                ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
+        }
 	rq->sleepers.list = NULL;
 #endif
 #endif
@@ -6382,7 +6389,8 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 
     init_no_runqs(no_schedulers_online, no_schedulers_online);
     balance_info.last_active_runqs = no_schedulers;
-    erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update");
+    erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
     balance_info.forced_check_balance = 0;
     balance_info.halftime = 1;
     balance_info.full_reds_history_index = 0;
@@ -7439,7 +7447,8 @@ sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi)
 static void
 init_scheduler_suspend(void)
 {
-    erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd");
+    erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
     schdlr_sspnd.online.normal = 1;
     schdlr_sspnd.curr_online.normal = 1;
     schdlr_sspnd.active.normal = 1;
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index 59577bf422..23c7414901 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -112,21 +112,13 @@ static struct {
 
 erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS];
 
-#ifdef ERTS_ENABLE_LOCK_COUNT
-static void lcnt_enable_proc_lock_count(Process *proc, int enable);
-#endif
-
 void
 erts_init_proc_lock(int cpus)
 {
     int i;
     for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) {
-#ifdef ERTS_ENABLE_LOCK_COUNT
-	erts_mtx_init_x(&erts_pix_locks[i].u.mtx,
-			"pix_lock", make_small(i));
-#else
-	erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock");
-#endif
+        erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock", make_small(i),
+            ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_PROCESS);
     }
 #if ERTS_PROC_LOCK_OWN_IMPL
     erts_thr_install_exit_handler(cleanup_tse);
@@ -1062,32 +1054,38 @@ erts_proc_lock_init(Process *p)
 #endif
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
 
-    erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id);
+    erts_mtx_init(&p->lock.main, "proc_main", p->common.id,
+        ERTS_LOCK_FLAGS_CATEGORY_PROCESS);
     ethr_mutex_lock(&p->lock.main.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.main.lc);
 #endif
-    erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id);
+    erts_mtx_init(&p->lock.link, "proc_link", p->common.id,
+        ERTS_LOCK_FLAGS_CATEGORY_PROCESS);
     ethr_mutex_lock(&p->lock.link.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.link.lc);
 #endif
-    erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id);
+    erts_mtx_init(&p->lock.msgq, "proc_msgq", p->common.id,
+        ERTS_LOCK_FLAGS_CATEGORY_PROCESS);
     ethr_mutex_lock(&p->lock.msgq.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.msgq.lc);
 #endif
-    erts_mtx_init_x(&p->lock.btm, "proc_btm", p->common.id);
+    erts_mtx_init(&p->lock.btm, "proc_btm", p->common.id,
+        ERTS_LOCK_FLAGS_CATEGORY_PROCESS);
     ethr_mutex_lock(&p->lock.btm.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.btm.lc);
 #endif
-    erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id);
+    erts_mtx_init(&p->lock.status, "proc_status", p->common.id,
+        ERTS_LOCK_FLAGS_CATEGORY_PROCESS);
     ethr_mutex_lock(&p->lock.status.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.status.lc);
 #endif
-    erts_mtx_init_x(&p->lock.trace, "proc_trace", p->common.id);
+    erts_mtx_init(&p->lock.trace, "proc_trace", p->common.id,
+        ERTS_LOCK_FLAGS_CATEGORY_PROCESS);
     ethr_mutex_lock(&p->lock.trace.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.trace.lc);
@@ -1124,21 +1122,11 @@ erts_proc_lock_fin(Process *p)
 
 #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
 
-void erts_lcnt_enable_proc_lock_count(int enable) {
-    int ix, max = erts_ptab_max(&erts_proc);
-    Process *proc = NULL;
-    for(ix = 0; ix < max; ++ix) {
-        if((proc = erts_pix2proc(ix)) != NULL) {
-            lcnt_enable_proc_lock_count(proc, enable);
-        }
-    } /* for all processes */
-}
-
 void erts_lcnt_proc_lock_init(Process *p) {
     erts_lcnt_init_ref(&p->lock.lcnt_carrier);
 
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) {
-        lcnt_enable_proc_lock_count(p, 1);
+    if(erts_lcnt_check_enabled(ERTS_LOCK_FLAGS_CATEGORY_PROCESS)) {
+        erts_lcnt_enable_proc_lock_count(p, 1);
     }
 } /* logic reversed */
 
@@ -1146,30 +1134,55 @@ void erts_lcnt_proc_lock_destroy(Process *p) {
     erts_lcnt_uninstall(&p->lock.lcnt_carrier);
 }
 
-static void lcnt_enable_proc_lock_count(Process *proc, int enable) {
-    if (enable) {
+void erts_lcnt_enable_proc_lock_count(Process *proc, int enable) {
+    if(proc->common.id == ERTS_INVALID_PID) {
+        /* Locks without an id are more trouble than they're worth; there's no
+         * way to look them up and we can't track them with _STATIC since it's
+         * too early to tell whether we're a system process (proc->static_flags
+         * hasn't been not set yet). */
+    } else if(!enable) {
+        erts_lcnt_proc_lock_destroy(proc);
+    } else if(!erts_lcnt_check_ref_installed(&proc->lock.lcnt_carrier)) {
         erts_lcnt_lock_info_carrier_t *carrier;
-        Eterm pid;
 
         carrier = erts_lcnt_create_lock_info_carrier(ERTS_LCNT_PROCLOCK_COUNT);
-        pid = (proc->common.id != ERTS_INVALID_PID) ? proc->common.id : NIL;
-
-        erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN,
-            "proc_main", ERTS_LCNT_LT_PROCLOCK, pid);
-        erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK,
-            "proc_link", ERTS_LCNT_LT_PROCLOCK, pid);
-        erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ,
-            "proc_msgq", ERTS_LCNT_LT_PROCLOCK, pid);
-        erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM,
-            "proc_btm", ERTS_LCNT_LT_PROCLOCK, pid);
-        erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS,
-            "proc_status",ERTS_LCNT_LT_PROCLOCK, pid);
-        erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE,
-            "proc_trace", ERTS_LCNT_LT_PROCLOCK, pid);
+
+        erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN,
+            "proc_main", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK);
+        erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK,
+            "proc_link", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK);
+        erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ,
+            "proc_msgq", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK);
+        erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM,
+            "proc_btm", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK);
+        erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS,
+            "proc_status",proc->common.id, ERTS_LOCK_TYPE_PROCLOCK);
+        erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE,
+            "proc_trace", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK);
 
         erts_lcnt_install(&proc->lock.lcnt_carrier, carrier);
-    } else {
-        erts_lcnt_proc_lock_destroy(proc);
+    }
+}
+
+void erts_lcnt_update_process_locks(int enable) {
+    int i, max;
+
+    max = erts_ptab_max(&erts_proc);
+
+    for(i = 0; i < max; i++) {
+        int delay_handle;
+        Process *proc;
+
+        delay_handle = erts_thr_progress_unmanaged_delay();
+        proc = erts_pix2proc(i);
+
+        if(proc != NULL) {
+            erts_lcnt_enable_proc_lock_count(proc, enable);
+        }
+
+        if(delay_handle != ERTS_THR_PRGR_DHANDLE_MANAGED) {
+            erts_thr_progress_unmanaged_continue(delay_handle);
+        }
     }
 }
 
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index b4a5ef3f28..023ba4d4ae 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -263,7 +263,8 @@ void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks);
 ERTS_GLB_INLINE
 void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res);
 
-void erts_lcnt_enable_proc_lock_count(int enable);
+void erts_lcnt_enable_proc_lock_count(Process *proc, int enable);
+void erts_lcnt_update_process_locks(int enable);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index c3d59cb3a8..b3bcb3af3f 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -372,7 +372,8 @@ erts_ptab_init_table(ErtsPTab *ptab,
     rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
     rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
-    erts_smp_rwmtx_init_opt(&ptab->list.data.rwmtx, &rwmtx_opts, name);
+    erts_smp_rwmtx_init_opt(&ptab->list.data.rwmtx, &rwmtx_opts, name, NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
     erts_smp_atomic32_init_nob(&ptab->vola.tile.count, 0);
     last_data_init_nob(ptab, ~((Uint64) 0));
 
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index 55ba943bdd..3502de5135 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -128,14 +128,14 @@ ERTS_GLB_INLINE int erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y);
 #define ERTS_SMP_HAVE_REC_MTX_INIT 1
 ERTS_GLB_INLINE void erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx);
 #endif
-ERTS_GLB_INLINE void erts_smp_mtx_init_x(erts_smp_mtx_t *mtx,
-					 char *name,
-					 Eterm extra);
-ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx,
-						char *name,
-						Eterm extra);
-ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name);
-ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name);
+ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx,
+                                       char *name,
+                                       Eterm extra,
+                                       erts_lock_flags_t flags);
+ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx,
+                                               char *name,
+                                               Eterm extra,
+                                               erts_lock_flags_t flags);
 ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx);
 #ifdef ERTS_ENABLE_LOCK_POSITION
 ERTS_GLB_INLINE int erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
@@ -153,18 +153,15 @@ ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd,
 ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd);
 ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd);
 ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no);
-ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
-					       erts_smp_rwmtx_opt_t *opt,
-					       char *name,
-					       Eterm extra);
-ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx,
-					   char *name,
-					   Eterm extra);
 ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
-					     erts_smp_rwmtx_opt_t *opt,
-					     char *name);
+                                             erts_smp_rwmtx_opt_t *opt,
+                                             char *name,
+                                             Eterm extra,
+                                             erts_lock_flags_t flags);
 ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx,
-					 char *name);
+                                         char *name,
+                                         Eterm extra,
+                                         erts_lock_flags_t flags);
 ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx);
 #ifdef ERTS_ENABLE_LOCK_POSITION
 ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
@@ -181,11 +178,10 @@ ERTS_GLB_INLINE void erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx);
 ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx);
-ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock,
-					      char *name,
-					      Eterm extra);
 ERTS_GLB_INLINE void erts_smp_spinlock_init(erts_smp_spinlock_t *lock,
-					    char *name);
+                                            char *name,
+                                            Eterm extra,
+                                            erts_lock_flags_t flags);
 ERTS_GLB_INLINE void erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock);
 ERTS_GLB_INLINE void erts_smp_spin_unlock(erts_smp_spinlock_t *lock);
 #ifdef ERTS_ENABLE_LOCK_POSITION
@@ -194,11 +190,10 @@ ERTS_GLB_INLINE void erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file,
 ERTS_GLB_INLINE void erts_smp_spin_lock(erts_smp_spinlock_t *lock);
 #endif
 ERTS_GLB_INLINE int erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock);
-ERTS_GLB_INLINE void erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock,
-					    char *name,
-					    Eterm extra);
 ERTS_GLB_INLINE void erts_smp_rwlock_init(erts_smp_rwlock_t *lock,
-					  char *name);
+                                          char *name,
+                                          Eterm extra,
+                                          erts_lock_flags_t flags);
 ERTS_GLB_INLINE void erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock);
 ERTS_GLB_INLINE void erts_smp_read_unlock(erts_smp_rwlock_t *lock);
 #ifdef ERTS_ENABLE_LOCK_POSITION
@@ -1062,34 +1057,18 @@ erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx)
 #endif
 
 ERTS_GLB_INLINE void
-erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
+erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags)
 {
 #ifdef ERTS_SMP
-    erts_mtx_init_x(mtx, name, extra);
+    erts_mtx_init(mtx, name, extra, flags);
 #endif
 }
 
 ERTS_GLB_INLINE void
-erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
+erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags)
 {
 #ifdef ERTS_SMP
-    erts_mtx_init_locked_x_opt(mtx, name, extra, 0);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name)
-{
-#ifdef ERTS_SMP
-    erts_mtx_init(mtx, name);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name)
-{
-#ifdef ERTS_SMP
-    erts_mtx_init_locked(mtx, name);
+    erts_mtx_init_locked(mtx, name, extra, flags);
 #endif
 }
 
@@ -1211,39 +1190,25 @@ erts_smp_rwmtx_set_reader_group(int no)
 }
 
 ERTS_GLB_INLINE void
-erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
-			  erts_smp_rwmtx_opt_t *opt,
-			  char *name,
-			  Eterm extra)
-{
-#ifdef ERTS_SMP
-    erts_rwmtx_init_opt_x(rwmtx, opt, name, extra);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra)
+erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx,
+                    char *name,
+                    Eterm extra,
+                    erts_lock_flags_t flags)
 {
 #ifdef ERTS_SMP
-    erts_rwmtx_init_x(rwmtx, name, extra);
+    erts_smp_rwmtx_init_opt(rwmtx, NULL, name, extra, flags);
 #endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
-			erts_smp_rwmtx_opt_t *opt,
-			char *name)
-{
-#ifdef ERTS_SMP
-    erts_rwmtx_init_opt(rwmtx, opt, name);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name)
+                        erts_smp_rwmtx_opt_t *opt,
+                        char *name,
+                        Eterm extra,
+                        erts_lock_flags_t flags)
 {
 #ifdef ERTS_SMP
-    erts_rwmtx_init(rwmtx, name);
+    erts_rwmtx_init_opt(rwmtx, opt, name, extra, flags);
 #endif
 }
 
@@ -1379,20 +1344,10 @@ erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx)
 }
 
 ERTS_GLB_INLINE void
-erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra)
-{
-#ifdef ERTS_SMP
-    erts_spinlock_init_x(lock, name, extra);
-#else
-    (void)lock;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_spinlock_init(erts_smp_spinlock_t *lock, char *name)
+erts_smp_spinlock_init(erts_smp_spinlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags)
 {
 #ifdef ERTS_SMP
-    erts_spinlock_init(lock, name);
+    erts_spinlock_init(lock, name, extra, flags);
 #else
     (void)lock;
 #endif
@@ -1445,20 +1400,10 @@ erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock)
 }
 
 ERTS_GLB_INLINE void
-erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock, char *name, Eterm extra)
-{
-#ifdef ERTS_SMP
-    erts_rwlock_init_x(lock, name, extra);
-#else
-    (void)lock;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_rwlock_init(erts_smp_rwlock_t *lock, char *name)
+erts_smp_rwlock_init(erts_smp_rwlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags)
 {
 #ifdef ERTS_SMP
-    erts_rwlock_init(lock, name);
+    erts_rwlock_init(lock, name, extra, flags);
 #else
     (void)lock;
 #endif
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index f7b53f5ef3..3b13defdc6 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -259,13 +259,16 @@
 
 #include "sys.h"
 
+#include "erl_lock_flags.h"
+#include "erl_term.h"
+
 #ifdef USE_THREADS
 
 #define ETHR_TRY_INLINE_FUNCS
 #include "ethread.h"
+
 #include "erl_lock_check.h"
 #include "erl_lock_count.h"
-#include "erl_term.h"
 
 #if defined(__GLIBC__) && (__GLIBC__ << 16) + __GLIBC_MINOR__ < (2 << 16) + 4
 /*
@@ -309,7 +312,9 @@ typedef struct {
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_ref_t lcnt;
 #endif
-
+#ifdef DEBUG
+    erts_lock_flags_t flags;
+#endif
 } erts_mtx_t;
 typedef ethr_cond erts_cnd_t;
 
@@ -322,6 +327,9 @@ typedef struct {
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_ref_t lcnt;
 #endif
+#ifdef DEBUG
+    erts_lock_flags_t flags;
+#endif
 } erts_rwmtx_t;
 
 #define ERTS_MTX_OPT_DEFAULT_INITER ETHR_MUTEX_OPT_DEFAULT_INITER
@@ -367,6 +375,9 @@ typedef struct {
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_ref_t lcnt;
 #endif
+#ifdef DEBUG
+    erts_lock_flags_t flags;
+#endif
 } erts_spinlock_t;
 
 /* rwlock */
@@ -378,6 +389,9 @@ typedef struct {
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_ref_t lcnt;
 #endif
+#ifdef DEBUG
+    erts_lock_flags_t flags;
+#endif
 } erts_rwlock_t;
 
 __decl_noreturn void  __noreturn erts_thr_fatal_error(int, char *); 
@@ -479,11 +493,14 @@ ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void));
 ERTS_GLB_INLINE erts_tid_t erts_thr_self(void);
 ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len);
 ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y);
-ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra);
-ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt);
-ERTS_GLB_INLINE void erts_mtx_init_locked_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt);
-ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name);
-ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name);
+ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx,
+                                   char *name,
+                                   Eterm extra,
+                                   erts_lock_flags_t flags);
+ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx,
+                                          char *name,
+                                          Eterm extra,
+                                          erts_lock_flags_t flags);
 ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx);
 #ifdef ERTS_ENABLE_LOCK_POSITION
 ERTS_GLB_INLINE int erts_mtx_trylock_x(erts_mtx_t *mtx, char *file,
@@ -502,18 +519,15 @@ ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx);
 ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd);
 ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd);
 ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no);
-ERTS_GLB_INLINE void erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx,
-					   erts_rwmtx_opt_t *opt,
-					   char *name,
-					   Eterm extra);
-ERTS_GLB_INLINE void erts_rwmtx_init_x(erts_rwmtx_t *rwmtx,
-				       char *name,
-				       Eterm extra);
 ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
-					 erts_rwmtx_opt_t *opt,
-					 char *name);
+                                         erts_rwmtx_opt_t *opt,
+                                         char *name,
+                                         Eterm extra,
+                                         erts_lock_flags_t flags);
 ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx,
-					 char *name);
+                                     char *name,
+                                     Eterm extra,
+                                     erts_lock_flags_t flags);
 ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx);
 #ifdef ERTS_ENABLE_LOCK_POSITION
 ERTS_GLB_INLINE int erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
@@ -603,16 +617,10 @@ ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp
 ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
 							 erts_aint64_t mask,
 							 erts_aint64_t set);
-
-ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock,
-					      char *name,
-					      Eterm extra,
-					      Uint16 opt);
-ERTS_GLB_INLINE void erts_spinlock_init_x(erts_spinlock_t *lock,
-					  char *name,
-					  Eterm extra);
 ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock,
-					char *name);
+                                        char *name,
+                                        Eterm extra,
+                                        erts_lock_flags_t flags);
 ERTS_GLB_INLINE void erts_spinlock_destroy(erts_spinlock_t *lock);
 ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock);
 #ifdef ERTS_ENABLE_LOCK_POSITION
@@ -621,11 +629,10 @@ ERTS_GLB_INLINE void erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigne
 ERTS_GLB_INLINE void erts_spin_lock(erts_spinlock_t *lock);
 #endif
 ERTS_GLB_INLINE int erts_lc_spinlock_is_locked(erts_spinlock_t *lock);
-ERTS_GLB_INLINE void erts_rwlock_init_x(erts_rwlock_t *lock,
-					char *name,
-					Eterm extra);
 ERTS_GLB_INLINE void erts_rwlock_init(erts_rwlock_t *lock,
-				      char *name);
+                                      char *name,
+                                      Eterm extra,
+                                      erts_lock_flags_t flags);
 ERTS_GLB_INLINE void erts_rwlock_destroy(erts_rwlock_t *lock);
 ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock);
 #ifdef ERTS_ENABLE_LOCK_POSITION
@@ -2159,106 +2166,41 @@ erts_equal_tids(erts_tid_t x, erts_tid_t y)
 }
 
 ERTS_GLB_INLINE void
-erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra)
-{
-#ifdef USE_THREADS
-    int res = ethr_mutex_init(&mtx->mtx);
-    if (res)
-	erts_thr_fatal_error(res, "initialize mutex");
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
-#endif
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_ref(&mtx->lcnt);
-    erts_lcnt_install_new_lock_info_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
-#endif
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt)
+erts_mtx_init(erts_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags)
 {
 #ifdef USE_THREADS
     int res = ethr_mutex_init(&mtx->mtx);
-    if (res)
-	erts_thr_fatal_error(res, "initialize mutex");
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
-#endif
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_ref(&mtx->lcnt);
-    if(!(opt & ERTS_LCNT_LT_DISABLE)) { // Incorrect, but preserves the old interface.
-        erts_lcnt_install_new_lock_info_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra);
+    if (res) {
+        erts_thr_fatal_error(res, "initialize mutex");
     }
-#endif
-#endif
-}
 
+    flags |= ERTS_LOCK_TYPE_MUTEX;
+#ifdef DEBUG
+    mtx->flags = flags;
+#endif
 
-ERTS_GLB_INLINE void
-erts_mtx_init_locked_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt)
-{
-#ifdef USE_THREADS
-    int res = ethr_mutex_init(&mtx->mtx);
-    if (res)
-	erts_thr_fatal_error(res, "initialize mutex");
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_ref(&mtx->lcnt);
-    if(!(opt & ERTS_LCNT_LT_DISABLE)) { // Incorrect, but preserves the old interface.
-        erts_lcnt_install_new_lock_info_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
-    }
-#endif
-    ethr_mutex_lock(&mtx->mtx);
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_trylock(1, &mtx->lc);
-#endif
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_trylock(&mtx->lcnt, 1);
-#endif
+    erts_lcnt_init_ref_x(&mtx->lcnt, name, extra, flags);
 #endif
+#endif /* USE_THREADS */
 }
 
 ERTS_GLB_INLINE void
-erts_mtx_init(erts_mtx_t *mtx, char *name)
+erts_mtx_init_locked(erts_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags)
 {
 #ifdef USE_THREADS
-    int res = ethr_mutex_init(&mtx->mtx);
-    if (res)
-	erts_thr_fatal_error(res, "initialize mutex");
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init_lock(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX);
-#endif
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_ref(&mtx->lcnt);
-    erts_lcnt_install_new_lock_info(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX);
-#endif
-#endif
-}
+    erts_mtx_init(mtx, name, extra, flags);
 
-ERTS_GLB_INLINE void
-erts_mtx_init_locked(erts_mtx_t *mtx, char *name)
-{
-#ifdef USE_THREADS
-    int res = ethr_mutex_init(&mtx->mtx);
-    if (res)
-	erts_thr_fatal_error(res, "initialize mutex");
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init_lock(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX);
-#endif
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_ref(&mtx->lcnt);
-    erts_lcnt_install_new_lock_info(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX);
-#endif
     ethr_mutex_lock(&mtx->mtx);
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_trylock(1, &mtx->lc);
-#endif
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_trylock(&mtx->lcnt, 1);
-#endif
+    #ifdef ERTS_ENABLE_LOCK_CHECK
+        erts_lc_trylock(1, &mtx->lc);
+    #endif
+    #ifdef ERTS_ENABLE_LOCK_COUNT
+        erts_lcnt_trylock(&mtx->lcnt, 1);
+    #endif
 #endif
 }
 
@@ -2267,6 +2209,9 @@ erts_mtx_destroy(erts_mtx_t *mtx)
 {
 #ifdef USE_THREADS
     int res;
+
+    ASSERT(!(mtx->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC));
+
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_destroy_lock(&mtx->lc);
 #endif
@@ -2477,59 +2422,32 @@ erts_rwmtx_set_reader_group(int no)
 }
 
 ERTS_GLB_INLINE void
-erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx,
-		      erts_rwmtx_opt_t *opt,
-		      char *name,
-		      Eterm extra)
-{
+erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, erts_rwmtx_opt_t *opt,
+                    char *name, Eterm extra, erts_lock_flags_t flags) {
 #ifdef USE_THREADS
     int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt);
-    if (res != 0)
-	erts_thr_fatal_error(res, "initialize rwmutex");
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init_lock_x(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX, extra);
-#endif
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_ref(&rwmtx->lcnt);
-    if (name && name[0] == '\0')
-        erts_lcnt_install_new_lock_info_x(&rwmtx->lcnt, NULL, ERTS_LCNT_LT_RWMUTEX, extra);
-    else
-        erts_lcnt_install_new_lock_info_x(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX, extra);
-#endif
-#endif
-}
+    if (res != 0) {
+        erts_thr_fatal_error(res, "initialize rwmutex");
+    }
 
-ERTS_GLB_INLINE void
-erts_rwmtx_init_x(erts_rwmtx_t *rwmtx,
-		  char *name,
-		  Eterm extra)
-{
-    erts_rwmtx_init_opt_x(rwmtx, NULL, name, extra);
-}
+    flags |= ERTS_LOCK_TYPE_RWMUTEX;
+#ifdef DEBUG
+    rwmtx->flags = flags;
+#endif
 
-ERTS_GLB_INLINE void
-erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
-		    erts_rwmtx_opt_t *opt,
-		    char *name)
-{
-#ifdef USE_THREADS
-    int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt);
-    if (res != 0)
-	erts_thr_fatal_error(res, "initialize rwmutex");
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init_lock(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX);
+    erts_lc_init_lock_x(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_ref(&rwmtx->lcnt);
-    erts_lcnt_install_new_lock_info(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX);
-#endif
+    erts_lcnt_init_ref_x(&rwmtx->lcnt, name, extra, flags);
 #endif
+#endif /* USE_THREADS */
 }
 
 ERTS_GLB_INLINE void
-erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name)
-{
-    erts_rwmtx_init_opt(rwmtx, NULL, name);
+erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name, Eterm extra,
+                erts_lock_flags_t flags) {
+    erts_rwmtx_init_opt(rwmtx, NULL, name, extra, flags);
 }
 
 ERTS_GLB_INLINE void
@@ -2537,6 +2455,9 @@ erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
 {
 #ifdef USE_THREADS
     int res;
+
+    ASSERT(!(rwmtx->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC));
+
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_destroy_lock(&rwmtx->lc);
 #endif
@@ -2584,7 +2505,7 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx)
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ);
+    erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LOCK_OPTION_READ);
 #endif
     
     return res;
@@ -2609,7 +2530,7 @@ erts_rwmtx_rlock(erts_rwmtx_t *rwmtx)
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ);
+    erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTION_READ);
 #endif
     ethr_rwmutex_rlock(&rwmtx->rwmtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -2626,7 +2547,7 @@ erts_rwmtx_runlock(erts_rwmtx_t *rwmtx)
     erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ);
+    erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTION_READ);
 #endif
     ethr_rwmutex_runlock(&rwmtx->rwmtx);
 #endif
@@ -2659,7 +2580,7 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx)
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ_WRITE);
+    erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LOCK_OPTION_RDWR);
 #endif
     
     return res;
@@ -2684,7 +2605,7 @@ erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx)
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE);
+    erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTION_RDWR);
 #endif
     ethr_rwmutex_rwlock(&rwmtx->rwmtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -2701,7 +2622,7 @@ erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx)
     erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE);
+    erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTION_RDWR);
 #endif
     ethr_rwmutex_rwunlock(&rwmtx->rwmtx);
 #endif
@@ -3086,62 +3007,26 @@ erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
 /* spinlock */
 
 ERTS_GLB_INLINE void
-erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra)
+erts_spinlock_init(erts_spinlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags)
 {
 #ifdef USE_THREADS
     int res = ethr_spinlock_init(&lock->slck);
-    if (res)
-	erts_thr_fatal_error(res, "init spinlock");
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra);
-#endif
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_ref(&lock->lcnt);
-    erts_lcnt_install_new_lock_info_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK, extra);
-#endif
-#else
-    (void)lock;
-#endif
-}
+    if (res) {
+        erts_thr_fatal_error(res, "init spinlock");
+    }
 
-ERTS_GLB_INLINE void
-erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra,
-			 Uint16 opt)
-{
-#ifdef USE_THREADS
-    int res = ethr_spinlock_init(&lock->slck);
-    if (res)
-	erts_thr_fatal_error(res, "init spinlock");
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra);
-#endif
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_ref(&lock->lcnt);
-    erts_lcnt_install_new_lock_info_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK | opt, extra);
+    flags |= ERTS_LOCK_TYPE_SPINLOCK;
+#ifdef DEBUG
+    lock->flags = flags;
 #endif
-#else
-    (void)lock;
-#endif
-}
 
-
-ERTS_GLB_INLINE void
-erts_spinlock_init(erts_spinlock_t *lock, char *name)
-{
-#ifdef USE_THREADS
-    int res = ethr_spinlock_init(&lock->slck);
-    if (res)
-	erts_thr_fatal_error(res, "init spinlock");
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init_lock(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK);
+    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_ref(&lock->lcnt);
-    erts_lcnt_install_new_lock_info(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK);
-#endif
-#else
-    (void)lock;
+    erts_lcnt_init_ref_x(&lock->lcnt, name, extra, flags);
 #endif
+#endif /* USE_THREADS */
 }
 
 ERTS_GLB_INLINE void
@@ -3149,6 +3034,9 @@ erts_spinlock_destroy(erts_spinlock_t *lock)
 {
 #ifdef USE_THREADS
     int res;
+
+    ASSERT(!(lock->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC));
+
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_destroy_lock(&lock->lc);
 #endif
@@ -3232,41 +3120,26 @@ erts_lc_spinlock_is_locked(erts_spinlock_t *lock)
 /* rwspinlock */
 
 ERTS_GLB_INLINE void
-erts_rwlock_init_x(erts_rwlock_t *lock, char *name, Eterm extra)
+erts_rwlock_init(erts_rwlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags)
 {
 #ifdef USE_THREADS
     int res = ethr_rwlock_init(&lock->rwlck);
-    if (res)
-	erts_thr_fatal_error(res, "init rwlock");
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK, extra);
-#endif
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_ref(&lock->lcnt);
-    erts_lcnt_install_new_lock_info_x(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK, extra);
-#endif
-#else
-    (void)lock;
+    if (res) {
+        erts_thr_fatal_error(res, "init rwlock");
+    }
+
+    flags |= ERTS_LOCK_TYPE_RWSPINLOCK;
+#ifdef DEBUG
+    lock->flags = flags;
 #endif
-}
 
-ERTS_GLB_INLINE void
-erts_rwlock_init(erts_rwlock_t *lock, char *name)
-{
-#ifdef USE_THREADS
-    int res = ethr_rwlock_init(&lock->rwlck);
-    if (res)
-	erts_thr_fatal_error(res, "init rwlock");
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init_lock(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK);
+    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_init_ref(&lock->lcnt);
-    erts_lcnt_install_new_lock_info(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK);
-#endif
-#else
-    (void)lock;
+    erts_lcnt_init_ref_x(&lock->lcnt, name, extra, flags);
 #endif
+#endif /* USE_THREADS */
 }
 
 ERTS_GLB_INLINE void
@@ -3274,6 +3147,9 @@ erts_rwlock_destroy(erts_rwlock_t *lock)
 {
 #ifdef USE_THREADS
     int res;
+
+    ASSERT(!(lock->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC));
+
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_destroy_lock(&lock->lc);
 #endif
@@ -3305,7 +3181,7 @@ erts_read_unlock(erts_rwlock_t *lock)
     erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ);
+    erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LOCK_OPTION_READ);
 #endif
     ethr_read_unlock(&lock->rwlck);
 #else
@@ -3329,7 +3205,7 @@ erts_read_lock(erts_rwlock_t *lock)
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ);
+    erts_lcnt_lock_opt(&lock->lcnt, ERTS_LOCK_OPTION_READ);
 #endif
     ethr_read_lock(&lock->rwlck);
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -3348,7 +3224,7 @@ erts_write_unlock(erts_rwlock_t *lock)
     erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE);
+    erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LOCK_OPTION_RDWR);
 #endif
     ethr_write_unlock(&lock->rwlck);
 #else
@@ -3372,7 +3248,7 @@ erts_write_lock(erts_rwlock_t *lock)
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE);
+    erts_lcnt_lock_opt(&lock->lcnt, ERTS_LOCK_OPTION_RDWR);
 #endif
     ethr_write_lock(&lock->rwlck);
 #ifdef ERTS_ENABLE_LOCK_COUNT
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 3084a8db75..6ddf97b463 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -954,8 +954,10 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 
     ASSERT(ERTS_MONOTONIC_TIME_MIN < ERTS_MONOTONIC_TIME_MAX);
 
-    erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday");
-    erts_smp_mtx_init(&erts_get_time_mtx, "get_time");
+    erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
+    erts_smp_mtx_init(&erts_get_time_mtx, "get_time", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
     time_sup.r.o.correction = time_correction;
     time_sup.r.o.warp_mode = time_warp_mode;
@@ -1120,8 +1122,9 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 	rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
 	rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
-	erts_smp_rwmtx_init_opt(&time_sup.inf.c.parmon.rwmtx,
-				&rwmtx_opts, "get_corrected_time");
+        erts_smp_rwmtx_init_opt(&time_sup.inf.c.parmon.rwmtx, &rwmtx_opts,
+            "get_corrected_time", NIL,
+            ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
 	cdatap = &time_sup.inf.c.parmon.cdata;
     
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 4b06c55770..db7d0ac449 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -336,7 +336,8 @@ void erts_init_trace(void) {
     rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
     rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
-    erts_smp_rwmtx_init_opt(&sys_trace_rwmtx, &rwmtx_opts, "sys_tracers");
+    erts_smp_rwmtx_init_opt(&sys_trace_rwmtx, &rwmtx_opts, "sys_tracers", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
 
 #ifdef HAVE_ERTS_NOW_CPU
     erts_cpu_timestamp = 0;
@@ -2625,7 +2626,8 @@ init_sys_msg_dispatcher(void)
     sys_message_queue = NULL;
     sys_message_queue_end = NULL;
     erts_smp_cnd_init(&smq_cnd);
-    erts_smp_mtx_init(&smq_mtx, "sys_msg_q");
+    erts_smp_mtx_init(&smq_mtx, "sys_msg_q", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
     erts_smp_thr_create(&sys_msg_dispatcher_tid,
 			sys_msg_dispatcher_func,
 			NULL,
@@ -3185,7 +3187,9 @@ static void init_tracer_nif()
     erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
-    erts_smp_rwmtx_init_opt(&tracer_mtx, &rwmtx_opt, "tracer_mtx");
+
+    erts_smp_rwmtx_init_opt(&tracer_mtx, &rwmtx_opt, "tracer_mtx", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
 
     erts_tracer_nif_clear();
 
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 57f5ba5436..828c833ffc 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -182,7 +182,8 @@ init_export_table(void)
     HashFunctions f;
     int i;
 
-    erts_smp_mtx_init(&export_staging_lock, "export_tab");
+    erts_smp_mtx_init(&export_staging_lock, "export_tab", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
     erts_smp_atomic_init_nob(&total_entries_bytes, 0);
 
     f.hash = (H_FUN) export_hash;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index fc95535ec3..2105ee7a6c 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1183,7 +1183,8 @@ void erts_ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon);
 Eterm erts_driver_monitor_to_ref(Eterm* hp, const ErlDrvMonitor *mon);
 
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT)
-void erts_lcnt_enable_io_lock_count(int enable);
+void erts_lcnt_update_driver_locks(int enable);
+void erts_lcnt_update_port_locks(int enable);
 #endif
 
 /* driver_tab.c */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 11cb2dfddf..135beab6e1 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -258,14 +258,7 @@ static ERTS_INLINE void port_init_instr(Port *prt
 #ifdef ERTS_SMP
     ASSERT(prt->drv_ptr && prt->lock);
     if (!prt->drv_ptr->lock) {
-	char *lock_str = "port_lock";
-#ifdef ERTS_ENABLE_LOCK_COUNT
-	Uint16 opt = ((erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)
-			    ? 0 : ERTS_LCNT_LT_DISABLE);
-#else
-	Uint16 opt = 0;
-#endif
-	erts_mtx_init_locked_x_opt(prt->lock, lock_str, id, opt);
+        erts_mtx_init_locked(prt->lock, "port_lock", id, ERTS_LOCK_FLAGS_CATEGORY_IO);
     }
 #endif
     erts_port_task_init_sched(&prt->sched, id);
@@ -3419,9 +3412,8 @@ void erts_init_io(int port_tab_size,
     else if (port_tab_size < ERTS_MIN_PORTS)
 	port_tab_size = ERTS_MIN_PORTS;
 
-    erts_smp_rwmtx_init_opt(&erts_driver_list_lock,
-			    &drv_list_rwmtx_opts,
-			    "driver_list");
+    erts_smp_rwmtx_init_opt(&erts_driver_list_lock, &drv_list_rwmtx_opts, "driver_list", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
     driver_list = NULL;
     erts_smp_tsd_key_create(&driver_list_lock_status_key,
 			    "erts_driver_list_lock_status_key");
@@ -3458,62 +3450,94 @@ void erts_init_io(int port_tab_size,
 }
 
 #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP)
-static ERTS_INLINE void lcnt_enable_drv_lock_count(erts_driver_t *dp, int enable)
+static void lcnt_enable_driver_lock_count(erts_driver_t *dp, int enable)
 {
     if (dp->lock) {
-	if (enable) {
-			Eterm name_as_atom = erts_atom_put((byte*)dp->name, sys_strlen(dp->name),
-											   ERTS_ATOM_ENC_LATIN1, 1);
-			erts_lcnt_install_new_lock_info_x(&dp->lock->lcnt,
-				"driver_lock", ERTS_LCNT_LT_MUTEX, name_as_atom);
-		} else {
-			erts_lcnt_uninstall(&dp->lock->lcnt);
-		}
-	}
+        if (enable) {
+            Eterm name_as_atom = erts_atom_put((byte*)dp->name, sys_strlen(dp->name),
+                                                ERTS_ATOM_ENC_LATIN1, 1);
+
+            erts_lcnt_install_new_lock_info(&dp->lock->lcnt, "driver_lock", name_as_atom,
+                ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO);
+        } else {
+            erts_lcnt_uninstall(&dp->lock->lcnt);
+        }
+    }
 }
 
-static ERTS_INLINE void lcnt_enable_port_lock_count(Port *prt, int enable)
+static void lcnt_enable_port_lock_count(Port *prt, int enable)
 {
     erts_aint32_t state = erts_atomic32_read_nob(&prt->state);
 
     if(enable) {
-		erts_lcnt_install_new_lock_info_x(&prt->sched.mtx.lcnt,
-			"port_sched_lock", ERTS_LCNT_LT_MUTEX,prt->common.id);
+        ErlDrvPDL pdl = prt->port_data_lock;
 
-		if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) {
-			erts_lcnt_install_new_lock_info_x(&prt->lock->lcnt,
-				"port_lock", ERTS_LCNT_LT_MUTEX, prt->common.id);
-		}
+        erts_lcnt_install_new_lock_info(&prt->sched.mtx.lcnt, "port_sched_lock",
+            prt->common.id, ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO);
+
+        if(pdl) {
+            erts_lcnt_install_new_lock_info(&pdl->mtx.lcnt, "port_data_lock",
+                prt->common.id, ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO);
+        }
+
+        if(state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) {
+            erts_lcnt_install_new_lock_info(&prt->lock->lcnt, "port_lock",
+                prt->common.id, ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO);
+        }
     } else {
-		erts_lcnt_uninstall(&prt->sched.mtx.lcnt);
-		if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) {
-			erts_lcnt_uninstall(&prt->lock->lcnt);
-		}
-    }
-}
+        erts_lcnt_uninstall(&prt->sched.mtx.lcnt);
 
-void erts_lcnt_enable_io_lock_count(int enable) {
-    erts_driver_t *dp;
-    int ix, max = erts_ptab_max(&erts_port);
-    Port *prt;
+        if(prt->port_data_lock) {
+            erts_lcnt_uninstall(&prt->port_data_lock->mtx.lcnt);
+        }
 
-    for (ix = 0; ix < max; ix++) {
-	if ((prt = erts_pix2port(ix)) != NULL) {
-	    lcnt_enable_port_lock_count(prt, enable);
+        if(state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) {
+            erts_lcnt_uninstall(&prt->lock->lcnt);
         }
-    } /* for all ports */
+    }
+}
 
-    lcnt_enable_drv_lock_count(&vanilla_driver, enable);
-    lcnt_enable_drv_lock_count(&spawn_driver, enable);
+void erts_lcnt_update_driver_locks(int enable) {
+    erts_driver_t *driver;
+
+    lcnt_enable_driver_lock_count(&vanilla_driver, enable);
+    lcnt_enable_driver_lock_count(&spawn_driver, enable);
 #ifndef __WIN32__
-    lcnt_enable_drv_lock_count(&forker_driver, enable);
+    lcnt_enable_driver_lock_count(&forker_driver, enable);
 #endif
-    lcnt_enable_drv_lock_count(&fd_driver, enable);
-    /* enable lock counting in all drivers */
-    for (dp = driver_list; dp; dp = dp->next) {
-	lcnt_enable_drv_lock_count(dp, enable);
+    lcnt_enable_driver_lock_count(&fd_driver, enable);
+
+    erts_rwmtx_rlock(&erts_driver_list_lock);
+
+    for (driver = driver_list; driver; driver = driver->next) {
+        lcnt_enable_driver_lock_count(driver, enable);
+    }
+
+    erts_rwmtx_runlock(&erts_driver_list_lock);
+}
+
+void erts_lcnt_update_port_locks(int enable) {
+    int i, max;
+
+    max = erts_ptab_max(&erts_port);
+
+    for(i = 0; i < max; i++) {
+        int delay_handle;
+        Port *port;
+
+        delay_handle = erts_thr_progress_unmanaged_delay();
+        port = erts_pix2port(i);
+
+        if(port != NULL) {
+            lcnt_enable_port_lock_count(port, enable);
+        }
+
+        if(delay_handle != ERTS_THR_PRGR_DHANDLE_MANAGED) {
+            erts_thr_progress_unmanaged_continue(delay_handle);
+        }
     }
-} /* enable/disable lock counting of ports */
+}
+
 #endif /* defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) */
 /*
  * Buffering of data when using line oriented I/O on ports
@@ -7088,7 +7112,7 @@ driver_pdl_create(ErlDrvPort dp)
 	return NULL;
     pdl = erts_alloc(ERTS_ALC_T_PORT_DATA_LOCK,
 		     sizeof(struct erl_drv_port_data_lock));
-    erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id);
+    erts_mtx_init(&pdl->mtx, "port_data_lock", pp->common.id, ERTS_LOCK_FLAGS_CATEGORY_IO);
     pdl_init_refc(pdl);
     erts_port_inc_refc(pp);
     pdl->prt = pp;
@@ -8255,22 +8279,16 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
     drv->flags = de->driver_flags;
     drv->handle = handle;
 #ifdef ERTS_SMP
-    if (drv->flags & ERL_DRV_FLAG_USE_PORT_LOCKING)
-	drv->lock = NULL;
-    else {
-	drv->lock = erts_alloc(ERTS_ALC_T_DRIVER_LOCK,
-			       sizeof(erts_mtx_t));
-	erts_mtx_init_x(drv->lock,
-			"driver_lock",
-#if defined(ERTS_ENABLE_LOCK_CHECK) || defined(ERTS_ENABLE_LOCK_COUNT)
-                        erts_atom_put((byte *) drv->name,
-				       sys_strlen(drv->name),
-                                       ERTS_ATOM_ENC_LATIN1,
-                                       1)
-#else
-			NIL
-#endif
-	    );
+    if (drv->flags & ERL_DRV_FLAG_USE_PORT_LOCKING) {
+        drv->lock = NULL;
+    } else {
+        Eterm driver_id = erts_atom_put((byte *) drv->name,
+                                        sys_strlen(drv->name),
+                                        ERTS_ATOM_ENC_LATIN1, 1);
+
+        drv->lock = erts_alloc(ERTS_ALC_T_DRIVER_LOCK, sizeof(erts_mtx_t));
+
+        erts_mtx_init(drv->lock, "driver_lock", driver_id, ERTS_LOCK_FLAGS_CATEGORY_IO);
     }
 #endif
     drv->entry = de;
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index 8ab6c713d6..7987cb2eb5 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -120,7 +120,8 @@ void init_module_table(void)
     }
 
     for (i=0; i<ERTS_NUM_CODE_IX; i++) {
-	erts_smp_rwmtx_init_x(&the_old_code_rwlocks[i], "old_code", make_small(i));
+        erts_smp_rwmtx_init(&the_old_code_rwlocks[i], "old_code", make_small(i),
+            ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
     }
     erts_smp_atomic_init_nob(&tot_module_bytes, 0);
 }
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index 7f60710124..bf3267cff1 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -145,7 +145,8 @@ void init_register_table(void)
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
-    erts_smp_rwmtx_init_opt(&regtab_rwmtx, &rwmtx_opt, "reg_tab");
+    erts_smp_rwmtx_init_opt(&regtab_rwmtx, &rwmtx_opt, "reg_tab", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
     f.hash = (H_FUN) reg_hash;
     f.cmp  = (HCMP_FUN) reg_cmp;
diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c
index 30b26a7296..527c9efeca 100644
--- a/erts/emulator/beam/safe_hash.c
+++ b/erts/emulator/beam/safe_hash.c
@@ -155,7 +155,8 @@ int safe_hash_table_sz(SafeHash *h)
 ** Init a pre allocated or static hash structure
 ** and allocate buckets. NOT SAFE
 */
-SafeHash* safe_hash_init(ErtsAlcType_t type, SafeHash* h, char* name, int size, SafeHashFunctions fun)
+SafeHash* safe_hash_init(ErtsAlcType_t type, SafeHash* h, char* name, erts_lock_flags_t flags,
+    int size, SafeHashFunctions fun)
 {
     int i, bytes;
 
@@ -170,7 +171,8 @@ SafeHash* safe_hash_init(ErtsAlcType_t type, SafeHash* h, char* name, int size,
     erts_smp_atomic_init_nob(&h->is_rehashing, 0);
     erts_smp_atomic_init_nob(&h->nitems, 0);
     for (i=0; i<SAFE_HASH_LOCK_CNT; i++) {
-	erts_smp_mtx_init(&h->lock_vec[i].mtx,"safe_hash");
+        erts_smp_mtx_init(&h->lock_vec[i].mtx, "safe_hash", NIL,
+            flags);
     }
     return h;
 }
@@ -273,5 +275,22 @@ void safe_hash_for_each(SafeHash* h, void (*func)(void *, void *), void *func_ar
     }
 }
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void erts_lcnt_enable_hash_lock_count(SafeHash *h, erts_lock_flags_t flags, int enable) {
+    int i;
+
+    for(i = 0; i < SAFE_HASH_LOCK_CNT; i++) {
+        erts_smp_mtx_t *lock = &h->lock_vec[i].mtx;
+
+        if(enable) {
+            erts_lcnt_install_new_lock_info(&lock->lcnt, "safe_hash", NIL,
+                ERTS_LOCK_TYPE_MUTEX | flags);
+        } else {
+            erts_lcnt_uninstall(&lock->lcnt);
+        }
+    }
+}
+#endif /* ERTS_ENABLE_LOCK_COUNT */
+
 #endif /* !ERTS_SYS_CONTINOUS_FD_NUMBERS */
 
diff --git a/erts/emulator/beam/safe_hash.h b/erts/emulator/beam/safe_hash.h
index 285103cb17..dde48a6de8 100644
--- a/erts/emulator/beam/safe_hash.h
+++ b/erts/emulator/beam/safe_hash.h
@@ -28,6 +28,7 @@
 
 #include "sys.h"
 #include "erl_alloc.h"
+#include "erl_lock_flags.h"
 
 typedef unsigned long SafeHashValue;
 
@@ -85,7 +86,7 @@ typedef struct
     /* A: Lockless atomics */
 } SafeHash;
 
-SafeHash* safe_hash_init(ErtsAlcType_t, SafeHash*, char*, int, SafeHashFunctions);
+SafeHash* safe_hash_init(ErtsAlcType_t, SafeHash*, char*, erts_lock_flags_t, int, SafeHashFunctions);
 
 void  safe_hash_get_info(SafeHashInfo*, SafeHash*);
 int   safe_hash_table_sz(SafeHash *);
@@ -96,5 +97,9 @@ void* safe_hash_erase(SafeHash*, void*);
 
 void  safe_hash_for_each(SafeHash*, void (*func)(void *, void *), void *);
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void erts_lcnt_enable_hash_lock_count(SafeHash*, erts_lock_flags_t, int);
+#endif
+
 #endif /* __SAFE_HASH_H__ */
 
diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c
index 1538191d67..3a7b3bb50c 100644
--- a/erts/emulator/drivers/common/efile_drv.c
+++ b/erts/emulator/drivers/common/efile_drv.c
@@ -742,7 +742,8 @@ file_init(void)
     efile_init();
 
 #ifdef  USE_VM_PROBES
-    erts_mtx_init(&dt_driver_mutex, "efile_drv dtrace mutex");
+    erts_mtx_init(&dt_driver_mutex, "efile_drv dtrace mutex", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
     pthread_key_create(&dt_driver_key, NULL);
 #endif  /* USE_VM_PROBES */
 
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 0225f17613..94bc563fda 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1129,7 +1129,8 @@ struct hipe_ref {
 
 static inline void hipe_mfa_info_table_init_lock(void)
 {
-    erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock");
+    erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 }
 
 static inline void hipe_mfa_info_table_rlock(void)
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index ad580e7d52..799f67fc45 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -2564,24 +2564,22 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
 #ifdef ERTS_SMP
     init_removed_fd_alloc();
     pollset.removed_list = NULL;
-    erts_smp_spinlock_init(&pollset.removed_list_lock,
-			   "pollset_rm_list");
+    erts_smp_spinlock_init(&pollset.removed_list_lock, "pollset_rm_list", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
     {
-	int i;
-	for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) {
-#ifdef ERTS_ENABLE_LOCK_COUNT
-	    erts_smp_mtx_init_x(&drv_ev_state_locks[i].lck, "drv_ev_state", make_small(i));
-#else
-	    erts_smp_mtx_init(&drv_ev_state_locks[i].lck, "drv_ev_state");
-#endif
-	}
+        int i;
+        for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) {
+            erts_smp_mtx_init(&drv_ev_state_locks[i].lck, "drv_ev_state", make_small(i),
+                ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
+        }
     }
 #endif
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     max_fds = ERTS_CIO_POLL_MAX_FDS();
     erts_smp_atomic_init_nob(&drv_ev_state_len, 0);
     drv_ev_state = NULL;
-    erts_smp_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow");
+    erts_smp_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
 #else
     {
 	SafeHashFunctions hf;
@@ -2591,10 +2589,11 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
 	hf.free = &drv_ev_state_free;
 	num_state_prealloc = 0;
 	state_prealloc_first = NULL;
-	erts_smp_spinlock_init(&state_prealloc_lock,"state_prealloc");
+	erts_smp_spinlock_init(&state_prealloc_lock,"state_prealloc", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
 
-	safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state_tab, "drv_ev_state_tab", 
-		       DRV_EV_STATE_HTAB_SIZE, hf);
+	safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state_tab, "drv_ev_state_tab",
+            ERTS_LOCK_FLAGS_CATEGORY_IO, DRV_EV_STATE_HTAB_SIZE, hf);
     }
 #endif
 }
@@ -3130,3 +3129,12 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
     return counters.num_errors;
 }
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks)(int enable) {
+#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
+    erts_lcnt_enable_hash_lock_count(&drv_ev_state_tab, ERTS_LOCK_FLAGS_CATEGORY_IO, enable);
+#else
+    (void)enable;
+#endif
+}
+#endif /* ERTS_ENABLE_LOCK_COUNT */
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index ee4abeece9..2d3bb98afa 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -59,6 +59,11 @@ void erts_init_check_io_nkp(void);
 int erts_check_io_debug_kp(ErtsCheckIoDebugInfo *);
 int erts_check_io_debug_nkp(ErtsCheckIoDebugInfo *);
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void erts_lcnt_update_cio_locks_kp(int enable);
+void erts_lcnt_update_cio_locks_nkp(int enable);
+#endif
+
 #else /* !ERTS_ENABLE_KERNEL_POLL */
 
 Uint erts_check_io_size(void);
@@ -72,6 +77,10 @@ void erts_check_io_interrupt_timed(int, ErtsMonotonicTime);
 void erts_check_io(int);
 void erts_init_check_io(void);
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void erts_lcnt_update_cio_locks(int enable);
+#endif
+
 #endif
 
 extern erts_smp_atomic_t erts_check_io_time;
diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index bb930ff03b..214ed01c82 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -2212,9 +2212,11 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable)
 	erts_exit(1, "erts_mmap: Failed to open /dev/zero\n");
 #endif
 
-    erts_smp_mtx_init(&mm->mtx, "erts_mmap");
+    erts_smp_mtx_init(&mm->mtx, "erts_mmap", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
     if (is_first_call) {
-        erts_mtx_init(&am.init_mutex, "mmap_init_atoms");
+        erts_mtx_init(&am.init_mutex, "mmap_init_atoms", NIL,
+            ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
     }
 
 #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
@@ -2816,7 +2818,8 @@ static void hard_dbg_mseg_init(void)
 {
     ErtsFreeSegDesc_fake* p;
 
-    erts_mtx_init(&hard_dbg_mseg_mtx, "hard_dbg_mseg");
+    erts_mtx_init(&hard_dbg_mseg_mtx, "hard_dbg_mseg", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
     hard_dbg_mseg_tree.root = NULL;
     hard_dbg_mseg_tree.order = ADDR_ORDER;
 
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index b8f0bb7150..d69a79dc2a 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -1420,7 +1420,8 @@ erts_mseg_init(ErtsMsegInit_t *init)
 
     atoms_initialized = 0;
 
-    erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms");
+    erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
 #ifdef ERTS_HAVE_EXEC_MMAPPER
     /* Initialize erts_exec_mapper *FIRST*, to increase probability
@@ -1449,7 +1450,8 @@ erts_mseg_init(ErtsMsegInit_t *init)
 	    ma->is_thread_safe = 0;
 	else {
 	    ma->is_thread_safe = 1;
-	    erts_mtx_init(&ma->mtx, "mseg");
+            erts_mtx_init(&ma->mtx, "mseg", make_small(i),
+                ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR);
 	}
 
 	ma->is_cache_check_scheduled = 0;
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index 5e7ae8953a..52a8b6a53f 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -336,7 +336,7 @@ static void fatal_error_async_signal_safe(char *error_str);
 
 static int max_fds = -1;
 static ErtsPollSet pollsets;
-static erts_smp_spinlock_t pollsets_lock;
+static erts_smp_mtx_t pollsets_lock;
 
 #if ERTS_POLL_USE_POLL
 
@@ -2583,7 +2583,8 @@ ERTS_POLL_EXPORT(erts_poll_max_fds)(void)
 void
 ERTS_POLL_EXPORT(erts_poll_init)(void)
 {
-    erts_smp_spinlock_init(&pollsets_lock, "pollsets_lock");
+    erts_smp_mtx_init(&pollsets_lock, "pollsets_lock", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
     pollsets = NULL;
 
     errno = 0;
@@ -2689,7 +2690,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
 #endif
 #ifdef ERTS_SMP
     erts_atomic32_init_nob(&ps->polled, 0);
-    erts_smp_mtx_init(&ps->mtx, "pollset");
+    erts_smp_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO);
 #endif
 #if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
     erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0);
@@ -2731,10 +2732,10 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
 #endif
     erts_smp_atomic_set_nob(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */
 
-    erts_smp_spin_lock(&pollsets_lock);
+    erts_smp_mtx_lock(&pollsets_lock);
     ps->next = pollsets;
     pollsets = ps;
-    erts_smp_spin_unlock(&pollsets_lock);
+    erts_smp_mtx_unlock(&pollsets_lock);
 
     return ps;
 }
@@ -2795,7 +2796,7 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps)
         close(ps->timer_fd);
 #endif
 
-    erts_smp_spin_lock(&pollsets_lock);
+    erts_smp_mtx_lock(&pollsets_lock);
     if (ps == pollsets)
 	pollsets = pollsets->next;
     else {
@@ -2805,7 +2806,7 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps)
 	ASSERT(ps == prev_ps->next);
 	prev_ps->next = ps->next;
     }
-    erts_smp_spin_unlock(&pollsets_lock);
+    erts_smp_mtx_unlock(&pollsets_lock);
 
     erts_free(ERTS_ALC_T_POLLSET, (void *) ps);
 }
@@ -3148,3 +3149,26 @@ print_misc_debug_info(void)
 }
     
 #endif
+
+#ifdef ERTS_ENABLE_LOCK_COUNT
+static void erts_lcnt_enable_pollset_lock_count(ErtsPollSet pollset, int enable) {
+    if(enable) {
+        erts_lcnt_install_new_lock_info(&pollset->mtx.lcnt, "pollset_rm", NIL,
+            ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO);
+    } else {
+        erts_lcnt_uninstall(&pollset->mtx.lcnt);
+    }
+}
+
+void ERTS_POLL_EXPORT(erts_lcnt_update_pollset_locks)(int enable) {
+    ErtsPollSet iterator;
+
+    erts_smp_mtx_lock(&pollsets_lock);
+
+    for(iterator = pollsets; iterator != NULL; iterator = iterator->next) {
+        erts_lcnt_enable_pollset_lock_count(iterator, enable);
+    }
+
+    erts_smp_mtx_unlock(&pollsets_lock);
+}
+#endif
diff --git a/erts/emulator/sys/common/erl_poll.h b/erts/emulator/sys/common/erl_poll.h
index c16122610d..b3b4d79984 100644
--- a/erts/emulator/sys/common/erl_poll.h
+++ b/erts/emulator/sys/common/erl_poll.h
@@ -260,4 +260,8 @@ void		ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet,
 
 int erts_poll_new_table_len(int old_len, int need_len);
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void ERTS_POLL_EXPORT(erts_lcnt_update_pollset_locks)(int enable);
+#endif
+
 #endif /* #ifndef ERL_POLL_H__ */
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index e86258b5e0..de4ec61c47 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -1550,7 +1550,8 @@ erl_sys_args(int* argc, char** argv)
 {
     int i, j;
 
-    erts_smp_rwmtx_init(&environ_rwmtx, "environ");
+    erts_smp_rwmtx_init(&environ_rwmtx, "environ", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
     i = 1;
 
diff --git a/erts/emulator/sys/unix/sys_time.c b/erts/emulator/sys/unix/sys_time.c
index 4f26639703..102ef7bebf 100644
--- a/erts/emulator/sys/unix/sys_time.c
+++ b/erts/emulator/sys/unix/sys_time.c
@@ -304,8 +304,8 @@ sys_init_time(ErtsSysInitTimeResult *init_resp)
 	    erts_sys_time_data__.r.o.os_times =
 		clock_gettime_times_verified;
 #endif
-	    erts_smp_mtx_init(&internal_state.w.f.mtx,
-			      "os_monotonic_time");
+            erts_smp_mtx_init(&internal_state.w.f.mtx, "os_monotonic_time", NIL,
+                ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
 	    internal_state.w.f.last_delivered
 		= clock_gettime_monotonic();
 	    init_resp->os_monotonic_time_info.locked_use = 1;
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index b10fc1e430..8743f83a50 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -142,7 +142,8 @@ static erts_mtx_t save_ops_mtx;
 
 static void poll_debug_init(void)
 {
-    erts_mtx_init(&save_ops_mtx, "save_ops_lock");
+    erts_mtx_init(&save_ops_mtx, "save_ops_lock", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
 }
 
 void poll_debug_set_active_fd(ErtsSysFdType fd)
@@ -677,7 +678,7 @@ static void new_waiter(ErtsPollSet ps)
     w->active_events = 1;
     w->highwater = 1;
     w->total_events = 1;
-    erts_mtx_init(&w->mtx, "pollwaiter");
+    erts_mtx_init(&w->mtx, "pollwaiter", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO);
 
 
     /*
@@ -1359,7 +1360,7 @@ ErtsPollSet erts_poll_create_pollset(void)
 
     erts_atomic32_init_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
 #ifdef ERTS_SMP
-    erts_smp_mtx_init(&ps->mtx, "pollset");
+    erts_smp_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO);
 #endif
     init_timeout_time(ps);
 
@@ -1411,7 +1412,8 @@ void  erts_poll_init(void)
     HARDTRACEF(("In erts_poll_init"));
     erts_sys_break_event = CreateManualEvent(FALSE);
 
-    erts_mtx_init(&break_waiter_lock,"break_waiter_lock");
+    erts_mtx_init(&break_waiter_lock, "break_waiter_lock", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
     break_happened_event = CreateManualEvent(FALSE);
     erts_atomic32_init_nob(&break_waiter_state, 0); 
 
diff --git a/erts/emulator/sys/win32/sys_env.c b/erts/emulator/sys/win32/sys_env.c
index 21ef71ad9a..8fcee1cbb6 100644
--- a/erts/emulator/sys/win32/sys_env.c
+++ b/erts/emulator/sys/win32/sys_env.c
@@ -37,7 +37,8 @@ static erts_smp_rwmtx_t environ_rwmtx;
 void
 erts_sys_env_init(void)
 {
-    erts_smp_rwmtx_init(&environ_rwmtx, "environ");
+    erts_smp_rwmtx_init(&environ_rwmtx, "environ", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 }
 
 int
diff --git a/erts/emulator/sys/win32/sys_time.c b/erts/emulator/sys/win32/sys_time.c
index e8c67b3928..88131aaa6a 100644
--- a/erts/emulator/sys/win32/sys_time.c
+++ b/erts/emulator/sys/win32/sys_time.c
@@ -300,8 +300,8 @@ sys_init_time(ErtsSysInitTimeResult *init_resp)
     module = GetModuleHandle(kernel_dll_name);
     if (!module) {
     get_tick_count:
-	erts_smp_mtx_init(&internal_state.w.f.mtime_mtx,
-			  "os_monotonic_time");
+        erts_smp_mtx_init(&internal_state.w.f.mtime_mtx, "os_monotonic_time", NIL,
+            ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 	internal_state.w.f.wrap = 0;
 	internal_state.w.f.last_tick_count = 0;
 
-- 
cgit v1.2.3


From 33a1962290ba6a6116cefe9fabb2e6cef823b33b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Fri, 23 Jun 2017 13:15:22 +0200
Subject: Break erts_debug:lock_counters/1 into separate BIFs

---
 erts/emulator/beam/bif.tab           |   5 +-
 erts/emulator/beam/erl_bif_info.c    | 197 ++++++++++++++++++++++-------------
 erts/emulator/beam/erl_dirty_bif.tab |   5 +-
 3 files changed, 134 insertions(+), 73 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index a8bbf5f8c1..962b00ae7b 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -437,7 +437,10 @@ bif erts_debug:dump_links/1
 #
 # Lock counter bif's
 #
-bif erts_debug:lock_counters/1
+bif erts_debug:lcnt_control/2
+bif erts_debug:lcnt_control/1
+bif erts_debug:lcnt_collect/0
+bif erts_debug:lcnt_clear/0
 
 #
 # New Bifs in R8.
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 02e34014e3..249cb1fdb6 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -4651,113 +4651,168 @@ static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_time_t *du
     return res;
 }
 
+static struct {
+    const char *name;
+    erts_lock_flags_t flag;
+} lcnt_category_map[] = {
+        {"allocator", ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR},
+        {"db", ERTS_LOCK_FLAGS_CATEGORY_DB},
+        {"debug", ERTS_LOCK_FLAGS_CATEGORY_DEBUG},
+        {"distribution", ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION},
+        {"generic", ERTS_LOCK_FLAGS_CATEGORY_GENERIC},
+        {"io", ERTS_LOCK_FLAGS_CATEGORY_IO},
+        {"process", ERTS_LOCK_FLAGS_CATEGORY_PROCESS},
+        {"scheduler", ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER},
+        {NULL, 0}
+    };
+
 static erts_lock_flags_t lcnt_atom_to_lock_category(Eterm atom) {
-    if(ERTS_IS_ATOM_STR("generic", atom)) {
-        return ERTS_LOCK_FLAGS_CATEGORY_GENERIC;
-    } else if(ERTS_IS_ATOM_STR("allocator", atom)) {
-        return ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR;
-    } else if(ERTS_IS_ATOM_STR("scheduler", atom)) {
-        return ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER;
-    } else if(ERTS_IS_ATOM_STR("process", atom)) {
-        return ERTS_LOCK_FLAGS_CATEGORY_PROCESS;
-    } else if(ERTS_IS_ATOM_STR("db", atom)) {
-        return ERTS_LOCK_FLAGS_CATEGORY_DB;
-    } else if(ERTS_IS_ATOM_STR("io", atom)) {
-        return ERTS_LOCK_FLAGS_CATEGORY_IO;
-    } else if(ERTS_IS_ATOM_STR("debug", atom)) {
-        return ERTS_LOCK_FLAGS_CATEGORY_DEBUG;
+    int i = 0;
+
+    for(i = 0; lcnt_category_map[i].name != NULL; i++) {
+        if(erts_is_atom_str(lcnt_category_map[i].name, atom, 0)) {
+            return lcnt_category_map[i].flag;
+        }
     }
 
     return 0;
 }
 
+static Eterm lcnt_build_category_list(Eterm **hpp, Uint *szp, erts_lock_flags_t mask) {
+    Eterm res;
+    int i;
+
+    res = NIL;
+
+    for(i = 0; lcnt_category_map[i].name != NULL; i++) {
+        if(mask & lcnt_category_map[i].flag) {
+            Eterm category = erts_atom_put((byte*)lcnt_category_map[i].name,
+                                           strlen(lcnt_category_map[i].name),
+                                           ERTS_ATOM_ENC_UTF8, 0);
+
+            res = erts_bld_cons(hpp, szp, category, res);
+        }
+    }
+
+    return res;
+}
+
 #endif
 
-BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1)
+BIF_RETTYPE erts_debug_lcnt_clear_0(BIF_ALIST_0)
 {
 #ifndef ERTS_ENABLE_LOCK_COUNT
-    if (BIF_ARG_1 == am_enabled) {
-        BIF_RET(am_false);
-    }
+    BIF_RET(am_error);
+#else
+    erts_lcnt_clear_counters();
 
-    BIF_ERROR(BIF_P, BADARG);
+    BIF_RET(am_ok);
+#endif
+}
+
+BIF_RETTYPE erts_debug_lcnt_collect_0(BIF_ALIST_0)
+{
+#ifndef ERTS_ENABLE_LOCK_COUNT
+    BIF_RET(am_error);
 #else
+    lcnt_sample_vector_t current_locks, deleted_locks;
+    erts_lcnt_data_t data;
 
-    if (BIF_ARG_1 == am_enabled) {
-       BIF_RET(am_true);
-    } else if (BIF_ARG_1 == am_info) {
-        lcnt_sample_vector_t current_locks, deleted_locks;
-        erts_lcnt_data_t data;
+    Eterm *term_heap_start, *term_heap_end;
+    Uint term_heap_size = 0;
+    Eterm result;
 
-        Eterm *term_heap_start, *term_heap_end;
-        Uint term_heap_size = 0;
-        Eterm result;
+    data = erts_lcnt_get_data();
 
-        data = erts_lcnt_get_data();
+    current_locks = lcnt_build_sample_vector(data.current_locks);
+    deleted_locks = lcnt_build_sample_vector(data.deleted_locks);
 
-        current_locks = lcnt_build_sample_vector(data.current_locks);
-        deleted_locks = lcnt_build_sample_vector(data.deleted_locks);
+    lcnt_build_result_term(NULL, &term_heap_size, &data.duration,
+        &current_locks, &deleted_locks, NIL);
 
-        lcnt_build_result_term(NULL, &term_heap_size, &data.duration,
-            &current_locks, &deleted_locks, NIL);
+    term_heap_start = HAlloc(BIF_P, term_heap_size);
+    term_heap_end = term_heap_start;
 
-        term_heap_start = HAlloc(BIF_P, term_heap_size);
-        term_heap_end = term_heap_start;
+    result = lcnt_build_result_term(&term_heap_end, NULL,
+        &data.duration, &current_locks, &deleted_locks, NIL);
 
-        result = lcnt_build_result_term(&term_heap_end, NULL,
-            &data.duration, &current_locks, &deleted_locks, NIL);
+    HRelease(BIF_P, term_heap_start + term_heap_size, term_heap_end);
 
-        HRelease(BIF_P, term_heap_start + term_heap_size, term_heap_end);
+    lcnt_destroy_sample_vector(&current_locks);
+    lcnt_destroy_sample_vector(&deleted_locks);
 
-        lcnt_destroy_sample_vector(&current_locks);
-        lcnt_destroy_sample_vector(&deleted_locks);
+    BIF_RET(result);
+#endif
+}
 
-        BIF_RET(result);
-    } else if (BIF_ARG_1 == am_clear) {
-        erts_lcnt_clear_counters();
+BIF_RETTYPE erts_debug_lcnt_control_1(BIF_ALIST_1)
+{
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    if(ERTS_IS_ATOM_STR("mask", BIF_ARG_1)) {
+        erts_lock_flags_t mask;
+        Eterm *term_heap_block;
+        Uint term_heap_size;
 
-        BIF_RET(am_ok);
-    } else if (is_tuple(BIF_ARG_1)) {
-        Eterm* ptr = tuple_val(BIF_ARG_1);
+        mask = erts_lcnt_get_category_mask();
+        term_heap_size = 0;
 
-        if(arityval(ptr[0]) != 2) {
-            BIF_ERROR(BIF_P, BADARG);
-        } else if(ERTS_IS_ATOM_STR("mask", ptr[1])) {
-            erts_lock_flags_t category_mask = 0;
-            Eterm categories = ptr[2];
+        lcnt_build_category_list(NULL, &term_heap_size, mask);
 
-            if(!(is_list(categories) || is_nil(categories))) {
-                BIF_ERROR(BIF_P, BADARG);
-            }
+        term_heap_block = HAlloc(BIF_P, term_heap_size);
 
-            while(is_list(categories)) {
-                Eterm *cell = list_val(categories);
-                erts_lock_flags_t category;
+        BIF_RET(lcnt_build_category_list(&term_heap_block, NULL, mask));
+    } else if(ERTS_IS_ATOM_STR("copy_save", BIF_ARG_1)) {
+        if(erts_lcnt_get_preserve_info()) {
+            BIF_RET(am_true);
+        }
 
-                category = lcnt_atom_to_lock_category(CAR(cell));
+        BIF_RET(am_false);
+    }
+#endif
+    BIF_ERROR(BIF_P, BADARG);
+}
 
-                if(!category) {
-                    /* Return {error, bad_category, Category}? Or leave that to
-                     * the lcnt module? */
-                    BIF_ERROR(BIF_P, BADARG);
-                }
+BIF_RETTYPE erts_debug_lcnt_control_2(BIF_ALIST_2)
+{
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    if(ERTS_IS_ATOM_STR("mask", BIF_ARG_1)) {
+        erts_lock_flags_t category_mask = 0;
+        Eterm categories = BIF_ARG_2;
 
-                category_mask |= category;
-                categories = CDR(cell);
+        if(!(is_list(categories) || is_nil(categories))) {
+            BIF_ERROR(BIF_P, BADARG);
+        }
+
+        while(is_list(categories)) {
+            Eterm *cell = list_val(categories);
+            erts_lock_flags_t category;
+
+            category = lcnt_atom_to_lock_category(CAR(cell));
+
+            if(!category) {
+                Eterm *hp = HAlloc(BIF_P, 4);
+
+                BIF_RET(TUPLE3(hp, am_error, am_badarg, CAR(cell)));
             }
 
-            erts_lcnt_set_category_mask(category_mask);
+            category_mask |= category;
+            categories = CDR(cell);
+        }
 
-            BIF_RET(am_ok);
-        } else if(ERTS_IS_ATOM_STR("copy_save", ptr[1]) &&
-            (ptr[2] == am_true || ptr[2] == am_false)) {
+        erts_lcnt_set_category_mask(category_mask);
 
-            erts_lcnt_set_preserve_info(ptr[2] == am_true);
+        BIF_RET(am_ok);
+    } else if(BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) {
+        int enabled = (BIF_ARG_2 == am_true);
+
+        if(ERTS_IS_ATOM_STR("copy_save", BIF_ARG_1)) {
+            erts_lcnt_set_preserve_info(enabled);
+
+            BIF_RET(am_ok);
         }
     }
-
-    BIF_ERROR(BIF_P, BADARG);
 #endif
+    BIF_ERROR(BIF_P, BADARG);
 }
 
 static void os_info_init(void)
diff --git a/erts/emulator/beam/erl_dirty_bif.tab b/erts/emulator/beam/erl_dirty_bif.tab
index 94d8589779..10c76d2579 100644
--- a/erts/emulator/beam/erl_dirty_bif.tab
+++ b/erts/emulator/beam/erl_dirty_bif.tab
@@ -46,7 +46,10 @@
 dirty-cpu erts_debug:dirty_cpu/2
 dirty-io erts_debug:dirty_io/2
 
-dirty-cpu erts_debug:lock_counters/1
+# lcnt_control/1 doesn't need to be dirty.
+dirty-cpu erts_debug:lcnt_control/2
+dirty-cpu erts_debug:lcnt_collect/0
+dirty-cpu erts_debug:lcnt_clear/0
 
 # --- TEST of Dirty BIF functionality ---
 #  Functions below will execute on dirty schedulers when emulator has
-- 
cgit v1.2.3


From fcd8e103006767622cf2ab61a4c85683d863db74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 29 Jun 2017 18:00:46 +0200
Subject: Add an emulator test suite for lock counting

OTP-14413
---
 erts/emulator/test/Makefile       |   1 +
 erts/emulator/test/lcnt_SUITE.erl | 156 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 157 insertions(+)
 create mode 100644 erts/emulator/test/lcnt_SUITE.erl

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index 2479ccc01f..627ca13887 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -73,6 +73,7 @@ MODULES= \
 	hipe_SUITE \
 	list_bif_SUITE \
 	lttng_SUITE \
+	lcnt_SUITE \
 	map_SUITE \
 	match_spec_SUITE \
 	module_info_SUITE \
diff --git a/erts/emulator/test/lcnt_SUITE.erl b/erts/emulator/test/lcnt_SUITE.erl
new file mode 100644
index 0000000000..504b9b54cf
--- /dev/null
+++ b/erts/emulator/test/lcnt_SUITE.erl
@@ -0,0 +1,156 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(lcnt_SUITE).
+
+-include_lib("common_test/include/ct.hrl").
+
+-export(
+    [all/0, suite/0,
+     init_per_suite/1, end_per_suite/1,
+     init_per_testcase/2, end_per_testcase/2]).
+
+-export(
+    [toggle_lock_counting/1, error_on_invalid_category/1, preserve_locks/1]).
+
+suite() ->
+    [{ct_hooks,[ts_install_cth]},
+     {timetrap, {seconds, 10}}].
+
+all() ->
+    [toggle_lock_counting, error_on_invalid_category, preserve_locks].
+
+init_per_suite(Config) ->
+    case erlang:system_info(lock_counting) of
+        true ->
+            %% The tests will run straight over these properties, so we have to
+            %% preserve them to avoid tainting the other tests.
+            OldCopySave = erts_debug:lcnt_control(copy_save),
+            OldMask = erts_debug:lcnt_control(mask),
+            [{lcnt_SUITE, {OldCopySave, OldMask}} | Config];
+        _ ->
+            {skip, "Lock counting is not enabled"}
+    end.
+
+end_per_suite(Config) ->
+    {OldCopySave, OldMask} = proplists:get_value(lcnt_SUITE, Config),
+
+    erts_debug:lcnt_control(copy_save, OldCopySave),
+    OldCopySave = erts_debug:lcnt_control(copy_save),
+
+    erts_debug:lcnt_control(mask, OldMask),
+    OldMask = erts_debug:lcnt_control(mask),
+
+    erts_debug:lcnt_clear(),
+    ok.
+
+init_per_testcase(_Case, Config) ->
+    disable_lock_counting(),
+    Config.
+
+end_per_testcase(_Case, _Config) ->
+    ok.
+
+disable_lock_counting() ->
+    ok = erts_debug:lcnt_control(copy_save, false),
+    ok = erts_debug:lcnt_control(mask, []),
+    ok = erts_debug:lcnt_clear(),
+
+    %% Sanity check.
+    false = erts_debug:lcnt_control(copy_save),
+    [] = erts_debug:lcnt_control(mask),
+
+    %% The above commands rely on some lazy operations, so we'll have to wait
+    %% for the list to clear.
+    ok = wait_for_empty_lock_list().
+
+wait_for_empty_lock_list() ->
+    wait_for_empty_lock_list(10).
+wait_for_empty_lock_list(Tries) when Tries > 0 ->
+    try_flush_cleanup_ops(),
+    case erts_debug:lcnt_collect() of
+        [{duration, _}, {locks, []}] ->
+            ok;
+        _ ->
+            timer:sleep(50),
+            wait_for_empty_lock_list(Tries - 1)
+    end;
+wait_for_empty_lock_list(0) ->
+    ct:fail("Lock list failed to clear after disabling lock counting.").
+
+%% Queue up a lot of thread progress cleanup ops in a vain attempt to
+%% flush the lock list.
+try_flush_cleanup_ops() ->
+    false = lists:member(process, erts_debug:lcnt_control(mask)),
+    [spawn(fun() -> ok end) || _ <- lists:seq(1, 1000)].
+
+%%
+%% Test cases
+%%
+
+toggle_lock_counting(Config) when is_list(Config) ->
+    Categories =
+        [allocator, db, debug, distribution, generic, io, process, scheduler],
+    lists:foreach(
+        fun(Category) ->
+            Locks = get_lock_info_for(Category),
+            if
+                Locks =/= [] ->
+                    disable_lock_counting();
+                Locks =:= [] ->
+                    ct:fail("Failed to toggle ~p locks.", [Category])
+            end
+        end, Categories).
+
+get_lock_info_for(Categories) when is_list(Categories) ->
+    ok = erts_debug:lcnt_control(mask, Categories),
+    [{duration, _}, {locks, Locks}] = erts_debug:lcnt_collect(),
+    Locks;
+
+get_lock_info_for(Category) when is_atom(Category) ->
+    get_lock_info_for([Category]).
+
+preserve_locks(Config) when is_list(Config) ->
+    erts_debug:lcnt_control(mask, [process]),
+
+    erts_debug:lcnt_control(copy_save, true),
+    [spawn(fun() -> ok end) || _ <- lists:seq(1, 1000)],
+
+    %% Wait for the processes to be fully destroyed before disabling copy_save,
+    %% then remove all active locks from the list. (There's no foolproof method
+    %% to do this; sleeping before/after is the best way we have)
+    timer:sleep(500),
+
+    erts_debug:lcnt_control(copy_save, false),
+    erts_debug:lcnt_control(mask, []),
+
+    try_flush_cleanup_ops(),
+    timer:sleep(500),
+
+    case erts_debug:lcnt_collect() of
+        [{duration, _}, {locks, Locks}] when length(Locks) > 0 ->
+            ct:pal("Preserved ~p locks.", [length(Locks)]);
+        [{duration, _}, {locks, []}] ->
+            ct:fail("copy_save didn't preserve any locks.")
+    end.
+
+error_on_invalid_category(Config) when is_list(Config) ->
+    {error, badarg, q_invalid} = erts_debug:lcnt_control(mask, [q_invalid]),
+    ok.
-- 
cgit v1.2.3


From f0715ae80670b12f4120f2b9d3b7bd51be690ea8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 6 Jul 2017 15:07:34 +0200
Subject: erts: Generate crash_dump slogan string as UTF8

the same as atoms.
---
 erts/emulator/beam/bif.c | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 8380efaf13..8bf7e3926e 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3971,9 +3971,6 @@ BIF_RETTYPE display_nl_0(BIF_ALIST_0)
 /**********************************************************************/
 
 
-#define HALT_MSG_SIZE	200
-static char halt_msg[HALT_MSG_SIZE+1];
-
 /* stop the system with exit code and flags */
 BIF_RETTYPE halt_2(BIF_ALIST_2)
 {
@@ -4024,15 +4021,16 @@ BIF_RETTYPE halt_2(BIF_ALIST_2)
 	erts_exit(ERTS_ABORT_EXIT, "");
     }
     else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) {
-	Sint i;
+#       define HALT_MSG_SIZE 200
+        static byte halt_msg[4*HALT_MSG_SIZE+1];
+        Sint written;
 
-        if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE)) == -1) {
+        if (erts_unicode_list_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE,
+                                     &written) == -1 ) {
             goto error;
         }
-        if (i == -2) /* truncated string */
-            i = HALT_MSG_SIZE;
-        ASSERT(i >= 0 && i <= HALT_MSG_SIZE);
-	halt_msg[i] = '\0';
+        ASSERT(written >= 0 && written < sizeof(halt_msg));
+	halt_msg[written] = '\0';
 	VERBOSE(DEBUG_SYSTEM,
 		("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2));
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-- 
cgit v1.2.3


From 125b98ecc43a90a3a3fbe0a79c12ef798893e9cf Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikpelinux@gmail.com>
Date: Fri, 7 Jul 2017 10:59:20 +0200
Subject: erts: ensure alignment of VM core types

---
 erts/emulator/beam/sys.h | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index e8615d1140..1e09752d87 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -327,9 +327,9 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f
 #endif
 
 #if SIZEOF_VOID_P == SIZEOF_LONG
-typedef unsigned long Eterm;
-typedef unsigned long Uint;
-typedef long          Sint;
+typedef unsigned long Eterm erts_align_attribute(sizeof(long));
+typedef unsigned long Uint  erts_align_attribute(sizeof(long));
+typedef long          Sint  erts_align_attribute(sizeof(long));
 #define SWORD_CONSTANT(Const) Const##L
 #define UWORD_CONSTANT(Const) Const##UL
 #define ERTS_UWORD_MAX ULONG_MAX
@@ -337,9 +337,9 @@ typedef long          Sint;
 #define ERTS_SIZEOF_ETERM SIZEOF_LONG
 #define ErtsStrToSint strtol
 #elif SIZEOF_VOID_P == SIZEOF_INT
-typedef unsigned int Eterm;
-typedef unsigned int Uint;
-typedef int          Sint;
+typedef unsigned int Eterm erts_align_attribute(sizeof(int));
+typedef unsigned int Uint  erts_align_attribute(sizeof(int));
+typedef int          Sint  erts_align_attribute(sizeof(int));
 #define SWORD_CONSTANT(Const) Const
 #define UWORD_CONSTANT(Const) Const##U
 #define ERTS_UWORD_MAX UINT_MAX
@@ -347,9 +347,9 @@ typedef int          Sint;
 #define ERTS_SIZEOF_ETERM SIZEOF_INT
 #define ErtsStrToSint strtol
 #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
-typedef unsigned long long Eterm;
-typedef unsigned long long Uint;
-typedef long long          Sint;
+typedef unsigned long long Eterm erts_align_attribute(sizeof(long long));
+typedef unsigned long long Uint  erts_align_attribute(sizeof(long long));
+typedef long long          Sint  erts_align_attribute(sizeof(long long));
 #define SWORD_CONSTANT(Const) Const##LL
 #define UWORD_CONSTANT(Const) Const##ULL
 #define ERTS_UWORD_MAX ULLONG_MAX
-- 
cgit v1.2.3


From e8d45ae14c6c3bdfcbbc7964228b004ef4f11ea6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Valim?= <jose.valim@plataformatec.com.br>
Date: Wed, 24 May 2017 12:59:14 +0200
Subject: Add stacktrace entries to BIF calls from emulator

The goal of this change is to improve debugging of
emulator calls. For example, the following code

    rem(1, y)

will error with atom `badarith` when y is 0 and the
stacktrace has no entry for `erlang:rem/2`, making
such cases very hard to debug. This patch makes it
so the stacktrace includes `erlang:rem(1, 0)`.

The following emulator BIFs have been changed:

  * band/2
  * bnot/1
  * bor/2
  * bsl/2
  * bsr/2
  * bxor/2
  * div/2
  * element/2
  * int_div/2
  * rem/2
  * sminus/2
  * splus/2
  * stimes/2
---
 erts/emulator/beam/beam_emu.c          | 66 ++++++++++++++++-----------
 erts/emulator/test/exception_SUITE.erl | 82 ++++++++++++++++++++++++++++------
 2 files changed, 109 insertions(+), 39 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 79d751d13e..1997c74697 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -204,6 +204,21 @@ do {                                     \
 
 #define ISCATCHEND(instr) ((Eterm *) *(instr) == OpCode(catch_end_y))
 
+#define BIF_ERROR_ARITY_1(Op1, BIF)				\
+  if (Arg(0) != 0) goto jump_f;					\
+  reg[0] = Op1;							\
+  SWAPOUT;							\
+  I = handle_error(c_p, I, reg, &bif_export[BIF]->info.mfa);	\
+  goto post_error_handling
+
+#define BIF_ERROR_ARITY_2(Op1, Op2, BIF)			\
+  if (Arg(0) != 0) goto jump_f;					\
+  reg[0] = Op1;							\
+  reg[1] = Op2;							\
+  SWAPOUT;							\
+  I = handle_error(c_p, I, reg, &bif_export[BIF]->info.mfa);	\
+  goto post_error_handling
+
 /*
  * Special Beam instructions.
  */
@@ -1485,7 +1500,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 	    goto find_func_info;
 	}
 
-#define DO_OUTLINED_ARITH_2(name, Op1, Op2)	\
+#define DO_OUTLINED_ARITH_2(name, Op1, Op2, BIF)\
  do {						\
      Eterm result;				\
      Uint live = Arg(1);			\
@@ -1499,7 +1514,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
      if (is_value(result)) {			\
 	 StoreBifResult(4, result);		\
      }						\
-     goto lb_Cl_error;				\
+     BIF_ERROR_ARITY_2(reg[live], reg[live+1], BIF);\
  } while (0)
 
  {
@@ -1529,7 +1544,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
              StoreBifResult(4, result);
 	 }
      }
-     DO_OUTLINED_ARITH_2(mixed_plus, PlusOp1, PlusOp2);
+     DO_OUTLINED_ARITH_2(mixed_plus, PlusOp1, PlusOp2, BIF_splus_2);
  }
 
  {
@@ -1554,7 +1569,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
              StoreBifResult(4, result);
 	 }
      }
-     DO_OUTLINED_ARITH_2(mixed_minus, MinusOp1, MinusOp2);
+     DO_OUTLINED_ARITH_2(mixed_minus, MinusOp1, MinusOp2, BIF_sminus_2);
  }
 
     {
@@ -1770,8 +1785,9 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 		StoreBifResult(3, result);
 	    }
 	}
+	c_p->freason = BADARG;
+	BIF_ERROR_ARITY_2(element_index, element_tuple, BIF_element_2);
     }
- /* Fall through */
 
  OpCase(badarg_j):
  badarg:
@@ -1798,7 +1814,8 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 		StoreBifResult(3, result);
 	    }
 	}
-     goto badarg;
+	c_p->freason = BADARG;
+	BIF_ERROR_ARITY_2(make_small(Arg(2)), fast_element_tuple, BIF_element_2);
  }
 
  OpCase(catch_yf):
@@ -2929,14 +2946,14 @@ do {						\
  {
      Eterm Op1, Op2;
      GetArg2(2, Op1, Op2);
-     DO_OUTLINED_ARITH_2(mixed_times, Op1, Op2);
+     DO_OUTLINED_ARITH_2(mixed_times, Op1, Op2, BIF_stimes_2);
  }
 
  OpCase(i_m_div_jIssd):
  {
      Eterm Op1, Op2;
      GetArg2(2, Op1, Op2);
-     DO_OUTLINED_ARITH_2(mixed_div, Op1, Op2);
+     DO_OUTLINED_ARITH_2(mixed_div, Op1, Op2, BIF_div_2);
  }
 
  OpCase(i_int_div_jIssd):
@@ -2945,7 +2962,8 @@ do {						\
 
      GetArg2(2, Op1, Op2);
      if (Op2 == SMALL_ZERO) {
-	 goto badarith;
+	 c_p->freason = BADARITH;
+	 BIF_ERROR_ARITY_2(Op1, Op2, BIF_intdiv_2);
      } else if (is_both_small(Op1, Op2)) {
 	 Sint ires = signed_val(Op1) / signed_val(Op2);
 	 if (MY_IS_SSMALL(ires)) {
@@ -2953,7 +2971,7 @@ do {						\
 	     StoreBifResult(4, result);
 	 }
      }
-     DO_OUTLINED_ARITH_2(int_div, Op1, Op2);
+     DO_OUTLINED_ARITH_2(int_div, Op1, Op2, BIF_intdiv_2);
  }
 
  {
@@ -2970,12 +2988,13 @@ do {						\
 
  do_rem:
      if (RemOp2 == SMALL_ZERO) {
-         goto badarith;
+	 c_p->freason = BADARITH;
+	 BIF_ERROR_ARITY_2(RemOp1, RemOp2, BIF_rem_2);
      } else if (is_both_small(RemOp1, RemOp2)) {
          Eterm result = make_small(signed_val(RemOp1) % signed_val(RemOp2));
          StoreBifResult(4, result);
      } else {
-	 DO_OUTLINED_ARITH_2(int_rem, RemOp1, RemOp2);
+	 DO_OUTLINED_ARITH_2(int_rem, RemOp1, RemOp2, BIF_rem_2);
      }
  }
 
@@ -2999,7 +3018,7 @@ do {						\
          Eterm result = BandOp1 & BandOp2;
          StoreBifResult(4, result);
      }
-     DO_OUTLINED_ARITH_2(band, BandOp1, BandOp2);
+     DO_OUTLINED_ARITH_2(band, BandOp1, BandOp2, BIF_band_2);
  }
 
  /*
@@ -3032,7 +3051,7 @@ do {						\
 	 Eterm result = Op1 | Op2;
 	 StoreBifResult(4, result);
      }
-     DO_OUTLINED_ARITH_2(bor, Op1, Op2);
+     DO_OUTLINED_ARITH_2(bor, Op1, Op2, BIF_bor_2);
  }
 
  OpCase(i_bxor_jIssd):
@@ -3050,7 +3069,7 @@ do {						\
 	 Eterm result = (Op1 ^ Op2) | make_small(0);
 	 StoreBifResult(4, result);
      }
-     DO_OUTLINED_ARITH_2(bxor, Op1, Op2);
+     DO_OUTLINED_ARITH_2(bxor, Op1, Op2, BIF_bxor_2);
  }
 
  {
@@ -3081,8 +3100,9 @@ do {						\
 	     Op2 = make_small(bignum_header_is_neg(*big_val(Op2)) ?
 				   MAX_SMALL : MIN_SMALL);
 	     goto do_bsl;
-	}
-     goto badarith;
+	 }
+	 c_p->freason = BADARITH;
+	 BIF_ERROR_ARITY_2(Op1, Op2, BIF_bsr_2);
      
      OpCase(i_bsl_jIssd):
          GetArg2(2, Op1, Op2);
@@ -3182,10 +3202,8 @@ do {						\
 	     }
 	     /* Fall through if the left argument is not an integer. */
 	 }
-     /*
-      * One or more non-integer arguments.
-      */
-     goto badarith;
+	 c_p->freason = BADARITH;
+	 BIF_ERROR_ARITY_2(Op1, Op2, BIF_bsl_2);
  }
 
  OpCase(i_int_bnot_jsId):
@@ -3203,16 +3221,12 @@ do {						\
 	 HEAVY_SWAPIN;
 	 ERTS_HOLE_CHECK(c_p);
 	 if (is_nil(bnot_val)) {
-	     goto lb_Cl_error;
+	     BIF_ERROR_ARITY_1(reg[live], BIF_bnot_1);
 	 }
      }
      StoreBifResult(3, bnot_val);
  }
 
- badarith:
-    c_p->freason = BADARITH;
-    goto lb_Cl_error;
-
  OpCase(i_apply): {
      BeamInstr *next;
      HEAVY_SWAPOUT;
diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl
index 76e3556bc4..8cba69582c 100644
--- a/erts/emulator/test/exception_SUITE.erl
+++ b/erts/emulator/test/exception_SUITE.erl
@@ -21,7 +21,7 @@
 -module(exception_SUITE).
 
 -export([all/0, suite/0,
-         badmatch/1, pending_errors/1, nil_arith/1,
+         badmatch/1, pending_errors/1, nil_arith/1, top_of_stacktrace/1,
          stacktrace/1, nested_stacktrace/1, raise/1, gunilla/1, per/1,
          exception_with_heap_frag/1, line_numbers/1]).
 
@@ -36,8 +36,8 @@ suite() ->
      {timetrap, {seconds, 10}}].
 
 all() -> 
-    [badmatch, pending_errors, nil_arith, stacktrace,
-     nested_stacktrace, raise, gunilla, per,
+    [badmatch, pending_errors, nil_arith, top_of_stacktrace,
+     stacktrace, nested_stacktrace, raise, gunilla, per,
      exception_with_heap_frag, line_numbers].
 
 -define(try_match(E),
@@ -241,7 +241,54 @@ ba_bnot(A) ->
     io:format("bnot ~p", [A]),
     {'EXIT', {badarith, _}} = (catch bnot A).
 
+%% Test that BIFs are added to the top of the stacktrace.
+
+top_of_stacktrace(Conf) when is_list(Conf) ->
+    %% Arithmetic operators
+    {'EXIT', {badarith, [{erlang, '+', [1, ok], _} | _]}} = (catch my_add(1, ok)),
+    {'EXIT', {badarith, [{erlang, '-', [1, ok], _} | _]}} = (catch my_minus(1, ok)),
+    {'EXIT', {badarith, [{erlang, '*', [1, ok], _} | _]}} = (catch my_times(1, ok)),
+    {'EXIT', {badarith, [{erlang, 'div', [1, ok], _} | _]}} = (catch my_div(1, ok)),
+    {'EXIT', {badarith, [{erlang, 'div', [1, 0], _} | _]}} = (catch my_div(1, 0)),
+    {'EXIT', {badarith, [{erlang, 'rem', [1, ok], _} | _]}} = (catch my_rem(1, ok)),
+    {'EXIT', {badarith, [{erlang, 'rem', [1, 0], _} | _]}} = (catch my_rem(1, 0)),
+
+    %% Bit operators
+    {'EXIT', {badarith, [{erlang, 'band', [1, ok], _} | _]}} = (catch my_band(1, ok)),
+    {'EXIT', {badarith, [{erlang, 'bor', [1, ok], _} | _]}} = (catch my_bor(1, ok)),
+    {'EXIT', {badarith, [{erlang, 'bsl', [1, ok], _} | _]}} = (catch my_bsl(1, ok)),
+    {'EXIT', {badarith, [{erlang, 'bsr', [1, ok], _} | _]}} = (catch my_bsr(1, ok)),
+    {'EXIT', {badarith, [{erlang, 'bxor', [1, ok], _} | _]}} = (catch my_bxor(1, ok)),
+    {'EXIT', {badarith, [{erlang, 'bnot', [ok], _} | _]}} = (catch my_bnot(ok)),
+
+    %% Tuples
+    {'EXIT', {badarg, [{erlang, element, [1, ok], _} | _]}} = (catch my_element(1, ok)),
+    {'EXIT', {badarg, [{erlang, element, [ok, {}], _} | _]}} = (catch my_element(ok, {})),
+    {'EXIT', {badarg, [{erlang, element, [1, {}], _} | _]}} = (catch my_element(1, {})),
+    {'EXIT', {badarg, [{erlang, element, [1, {}], _} | _]}} = (catch element(1, erlang:make_tuple(0, ok))),
+
+    %% System limits
+    Maxbig = maxbig(),
+    MinusMaxbig = -Maxbig,
+    {'EXIT', {system_limit, [{erlang, '+', [Maxbig, 1], _} | _]}} = (catch my_add(Maxbig, 1)),
+    {'EXIT', {system_limit, [{erlang, '+', [Maxbig, 1], _} | _]}} = (catch my_add(maxbig_gc(), 1)),
+    {'EXIT', {system_limit, [{erlang, '-', [MinusMaxbig, 1], _} | _]}} = (catch my_minus(-Maxbig, 1)),
+    {'EXIT', {system_limit, [{erlang, '-', [MinusMaxbig, 1], _} | _]}} = (catch my_minus(-maxbig_gc(), 1)),
+    {'EXIT', {system_limit, [{erlang, '*', [Maxbig, 2], _} | _]}} = (catch my_times(Maxbig, 2)),
+    {'EXIT', {system_limit, [{erlang, '*', [Maxbig, 2], _} | _]}} = (catch my_times(maxbig_gc(), 2)),
+    {'EXIT', {system_limit, [{erlang, 'bnot', [Maxbig], _} | _]}} = (catch my_bnot(Maxbig)),
+    {'EXIT', {system_limit, [{erlang, 'bnot', [Maxbig], _} | _]}} = (catch my_bnot(maxbig_gc())),
+    ok.
+
+maxbig() ->
+    %% We assume that the maximum arity is (1 bsl 19) - 1.
+    Ws = erlang:system_info(wordsize),
+    (((1 bsl ((16777184 * (Ws div 4))-1)) - 1) bsl 1) + 1.
 
+maxbig_gc() ->
+    Maxbig = maxbig(),
+    erlang:garbage_collect(),
+    Maxbig.
 
 stacktrace(Conf) when is_list(Conf) ->
     Tag = make_ref(),
@@ -253,9 +300,9 @@ stacktrace(Conf) when is_list(Conf) ->
     St1 = erase(stacktrace1),
     St1 = erase(stacktrace2),
     St1 = erlang:get_stacktrace(),
-    {caught2,{error,badarith},[{?MODULE,my_add,2,_}|_]=St2} =
+    {caught2,{error,badarith},[{erlang,'+',[0,a],_},{?MODULE,my_add,2,_}|_]=St2} =
     stacktrace_1({'div',{1,0}}, error, {'add',{0,a}}),
-    [{?MODULE,my_div,2,_}|_] = erase(stacktrace1),
+    [{erlang,'div',[1,0],_},{?MODULE,my_div,2,_}|_] = erase(stacktrace1),
     St2 = erase(stacktrace2),
     St2 = erlang:get_stacktrace(),
     {caught2,{error,{try_clause,V}},[{?MODULE,stacktrace_1,3,_}|_]=St3} =
@@ -308,13 +355,13 @@ nested_stacktrace(Conf) when is_list(Conf) ->
     nested_stacktrace_1({{value,{V,x1}},void,{V,x1}},
                         {void,void,void}),
     {caught1,
-     [{?MODULE,my_add,2,_}|_],
+     [{erlang,'+',[V,x1],_},{?MODULE,my_add,2,_}|_],
      value2,
-     [{?MODULE,my_add,2,_}|_]} =
+     [{erlang,'+',[V,x1],_},{?MODULE,my_add,2,_}|_]} =
     nested_stacktrace_1({{'add',{V,x1}},error,badarith},
                         {{value,{V,x2}},void,{V,x2}}),
     {caught1,
-     [{?MODULE,my_add,2,_}|_],
+     [{erlang,'+',[V,x1],_},{?MODULE,my_add,2,_}|_],
      {caught2,[{erlang,abs,[V],_}|_]},
      [{erlang,abs,[V],_}|_]} =
     nested_stacktrace_1({{'add',{V,x1}},error,badarith},
@@ -355,7 +402,7 @@ raise(Conf) when is_list(Conf) ->
     end,
     A = erlang:get_stacktrace(),
     A = get(raise),
-    [{?MODULE,my_div,2,_}|_] = A,
+    [{erlang,'div',[1, 0], _},{?MODULE,my_div,2,_}|_] = A,
     %%
     N = 8, % Must be even
     N = erlang:system_flag(backtrace_depth, N),
@@ -404,11 +451,20 @@ foo({raise,{Class,Reason,Stacktrace}}) ->
     erlang:raise(Class, Reason, Stacktrace).
 %%foo(function_clause) -> % must not be defined!
 
-my_div(A, B) ->
-    A div B.
+my_add(A, B) -> A + B.
+my_minus(A, B) -> A - B.
+my_times(A, B) -> A * B.
+my_div(A, B) -> A div B.
+my_rem(A, B) -> A rem B.
+
+my_band(A, B) -> A band B.
+my_bor(A, B) -> A bor B.
+my_bsl(A, B) -> A bsl B.
+my_bsr(A, B) -> A bsr B.
+my_bxor(A, B) -> A bxor B.
+my_bnot(A) -> bnot A.
 
-my_add(A, B) ->
-    A + B.
+my_element(A, B) -> element(A, B).
 
 my_abs(X) -> abs(X).
 
-- 
cgit v1.2.3


From f4661e4e1d234fdbcd37bbc4a8a9bfdf2cfaa586 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Fri, 7 Jul 2017 16:40:30 +0200
Subject: Add a proper type for lock options

---
 erts/emulator/beam/erl_drv_thread.c | 12 +++++-----
 erts/emulator/beam/erl_lock_count.h | 48 ++++++++++++++++++-------------------
 erts/emulator/beam/erl_lock_flags.c | 13 ++++++++++
 erts/emulator/beam/erl_lock_flags.h | 10 +++++---
 erts/emulator/beam/erl_threads.h    | 20 ++++++++--------
 5 files changed, 60 insertions(+), 43 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 49bbab55a8..742c428f2a 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -414,7 +414,7 @@ erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_tryrlock()");
     res = ethr_rwmutex_tryrlock(&drwlck->rwmtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LOCK_OPTION_READ);
+    erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LOCK_OPTIONS_READ);
 #endif
     return res;
 #else
@@ -429,7 +429,7 @@ erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck)
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_rlock()");
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LOCK_OPTION_READ);
+    erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LOCK_OPTIONS_READ);
 #endif
     ethr_rwmutex_rlock(&drwlck->rwmtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -445,7 +445,7 @@ erl_drv_rwlock_runlock(ErlDrvRWLock *drwlck)
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_runlock()");
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LOCK_OPTION_READ);
+    erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LOCK_OPTIONS_READ);
 #endif
     ethr_rwmutex_runlock(&drwlck->rwmtx);
 #endif
@@ -460,7 +460,7 @@ erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_tryrwlock()");
     res = ethr_rwmutex_tryrwlock(&drwlck->rwmtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LOCK_OPTION_RDWR);
+    erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LOCK_OPTIONS_RDWR);
 #endif
     return res;
 #else
@@ -475,7 +475,7 @@ erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck)
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_rwlock()");
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LOCK_OPTION_RDWR);
+    erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LOCK_OPTIONS_RDWR);
 #endif
     ethr_rwmutex_rwlock(&drwlck->rwmtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -491,7 +491,7 @@ erl_drv_rwlock_rwunlock(ErlDrvRWLock *drwlck)
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_rwunlock()");
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LOCK_OPTION_RDWR);
+    erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LOCK_OPTIONS_RDWR);
 #endif
     ethr_rwmutex_rwunlock(&drwlck->rwmtx);
 #endif
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index 3181dbcad4..89d95a73cf 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -172,7 +172,7 @@ void erts_lcnt_lock(erts_lcnt_ref_t *ref);
 /** @copydoc erts_lcnt_lock
  * @param option Notes whether the lock is a read or write lock. */
 ERTS_GLB_FORCE_INLINE
-void erts_lcnt_lock_opt(erts_lcnt_ref_t *ref, Uint16 option);
+void erts_lcnt_lock_opt(erts_lcnt_ref_t *ref, erts_lock_options_t option);
 
 /** @brief Records that a lock has been acquired. */
 ERTS_GLB_FORCE_INLINE
@@ -191,7 +191,7 @@ void erts_lcnt_unlock(erts_lcnt_ref_t *ref);
 /** @copydoc erts_lcnt_unlock_opt
  * @param option Whether the lock is a read or write lock. */
 ERTS_GLB_FORCE_INLINE
-void erts_lcnt_unlock_opt(erts_lcnt_ref_t *ref, Uint16 option);
+void erts_lcnt_unlock_opt(erts_lcnt_ref_t *ref, erts_lock_options_t option);
 
 /** @brief Rectifies the case where a lock wasn't actually a lock operation.
  *
@@ -207,7 +207,7 @@ void erts_lcnt_trylock(erts_lcnt_ref_t *ref, int result);
 /** @copydoc erts_lcnt_trylock
  * @param option Whether the lock is a read or write lock. */
 ERTS_GLB_FORCE_INLINE
-void erts_lcnt_trylock_opt(erts_lcnt_ref_t *ref, int result, Uint16 option);
+void erts_lcnt_trylock_opt(erts_lcnt_ref_t *ref, int result, erts_lock_options_t option);
 
 /* Indexed variants of the standard lock operations, for use when a single
  * reference contains many counters (eg. process locks).
@@ -220,7 +220,7 @@ void erts_lcnt_trylock_opt(erts_lcnt_ref_t *ref, int result, Uint16 option);
 ERTS_GLB_INLINE
 void erts_lcnt_lock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index);
 ERTS_GLB_INLINE
-void erts_lcnt_lock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, Uint16 option);
+void erts_lcnt_lock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, erts_lock_options_t option);
 
 ERTS_GLB_INLINE
 void erts_lcnt_lock_post_idx(erts_lcnt_lock_info_carrier_t *carrier, int index);
@@ -233,12 +233,12 @@ void erts_lcnt_lock_unacquire_idx(erts_lcnt_lock_info_carrier_t *carrier, int in
 ERTS_GLB_INLINE
 void erts_lcnt_unlock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index);
 ERTS_GLB_INLINE
-void erts_lcnt_unlock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, Uint16 option);
+void erts_lcnt_unlock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, erts_lock_options_t option);
 
 ERTS_GLB_INLINE
 void erts_lcnt_trylock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result);
 ERTS_GLB_INLINE
-void erts_lcnt_trylock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result, Uint16 option);
+void erts_lcnt_trylock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result, erts_lock_options_t option);
 
 /* -- Reference operations ------------------------------------------------- */
 
@@ -646,7 +646,7 @@ void erts_lcnt_lock(erts_lcnt_ref_t *ref) {
 }
 
 ERTS_GLB_FORCE_INLINE
-void erts_lcnt_lock_opt(erts_lcnt_ref_t *ref, Uint16 option) {
+void erts_lcnt_lock_opt(erts_lcnt_ref_t *ref, erts_lock_options_t option) {
     erts_lcnt_lock_info_carrier_t *carrier;
     int handle;
 
@@ -706,7 +706,7 @@ void erts_lcnt_unlock(erts_lcnt_ref_t *ref) {
 }
 
 ERTS_GLB_FORCE_INLINE
-void erts_lcnt_unlock_opt(erts_lcnt_ref_t *ref, Uint16 option) {
+void erts_lcnt_unlock_opt(erts_lcnt_ref_t *ref, erts_lock_options_t option) {
     erts_lcnt_lock_info_carrier_t *carrier;
     int handle;
 
@@ -730,7 +730,7 @@ void erts_lcnt_trylock(erts_lcnt_ref_t *ref, int result) {
 }
 
 ERTS_GLB_FORCE_INLINE
-void erts_lcnt_trylock_opt(erts_lcnt_ref_t *ref, int result, Uint16 option) {
+void erts_lcnt_trylock_opt(erts_lcnt_ref_t *ref, int result, erts_lock_options_t option) {
     erts_lcnt_lock_info_carrier_t *carrier;
     int handle;
 
@@ -743,20 +743,20 @@ void erts_lcnt_trylock_opt(erts_lcnt_ref_t *ref, int result, Uint16 option) {
 
 ERTS_GLB_INLINE
 void erts_lcnt_lock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index) {
-    erts_lcnt_lock_opt_idx(carrier, index, ERTS_LOCK_OPTION_WRITE);
+    erts_lcnt_lock_opt_idx(carrier, index, ERTS_LOCK_OPTIONS_WRITE);
 }
 
 ERTS_GLB_INLINE
-void erts_lcnt_lock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, Uint16 option) {
+void erts_lcnt_lock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, erts_lock_options_t option) {
     erts_lcnt_lock_info_t *info = &carrier->entries[index];
 
     lcnt_thread_data_t__ *eltd = lcnt_get_thread_data__();
 
     ASSERT(index < carrier->entry_count);
 
-    ASSERT((option & ERTS_LOCK_OPTION_READ) || (option & ERTS_LOCK_OPTION_WRITE));
+    ASSERT((option & ERTS_LOCK_OPTIONS_READ) || (option & ERTS_LOCK_OPTIONS_WRITE));
 
-    if(option & ERTS_LOCK_OPTION_WRITE) {
+    if(option & ERTS_LOCK_OPTIONS_WRITE) {
         ethr_sint_t w_state, r_state;
 
         w_state = ethr_atomic_inc_read(&info->w_state) - 1;
@@ -771,7 +771,7 @@ void erts_lcnt_lock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, U
         eltd->lock_in_conflict = (w_state > 0);
     }
 
-    if(option & ERTS_LOCK_OPTION_READ) {
+    if(option & ERTS_LOCK_OPTIONS_READ) {
         ASSERT(info->flags & ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE);
         ethr_atomic_inc(&info->r_state);
     }
@@ -825,22 +825,22 @@ ERTS_GLB_INLINE
 void erts_lcnt_unlock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index) {
     ASSERT(index < carrier->entry_count);
 
-    erts_lcnt_unlock_opt_idx(carrier, index, ERTS_LOCK_OPTION_WRITE);
+    erts_lcnt_unlock_opt_idx(carrier, index, ERTS_LOCK_OPTIONS_WRITE);
 }
 
 ERTS_GLB_INLINE
-void erts_lcnt_unlock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, Uint16 option) {
+void erts_lcnt_unlock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, erts_lock_options_t option) {
     erts_lcnt_lock_info_t *info = &carrier->entries[index];
 
     ASSERT(index < carrier->entry_count);
 
-    ASSERT((option & ERTS_LOCK_OPTION_READ) || (option & ERTS_LOCK_OPTION_WRITE));
+    ASSERT((option & ERTS_LOCK_OPTIONS_READ) || (option & ERTS_LOCK_OPTIONS_WRITE));
 
-    if(option & ERTS_LOCK_OPTION_WRITE) {
+    if(option & ERTS_LOCK_OPTIONS_WRITE) {
         lcnt_dec_lock_state__(&info->w_state);
     }
 
-    if(option & ERTS_LOCK_OPTION_READ) {
+    if(option & ERTS_LOCK_OPTIONS_READ) {
         ASSERT(info->flags & ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE);
         lcnt_dec_lock_state__(&info->r_state);
     }
@@ -859,23 +859,23 @@ ERTS_GLB_INLINE
 void erts_lcnt_trylock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result) {
     ASSERT(index < carrier->entry_count);
 
-    erts_lcnt_trylock_opt_idx(carrier, index, result, ERTS_LOCK_OPTION_WRITE);
+    erts_lcnt_trylock_opt_idx(carrier, index, result, ERTS_LOCK_OPTIONS_WRITE);
 }
 
 ERTS_GLB_INLINE
-void erts_lcnt_trylock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result, Uint16 option) {
+void erts_lcnt_trylock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result, erts_lock_options_t option) {
     erts_lcnt_lock_info_t *info = &carrier->entries[index];
 
     ASSERT(index < carrier->entry_count);
 
-    ASSERT((option & ERTS_LOCK_OPTION_READ) || (option & ERTS_LOCK_OPTION_WRITE));
+    ASSERT((option & ERTS_LOCK_OPTIONS_READ) || (option & ERTS_LOCK_OPTIONS_WRITE));
 
     if(result != EBUSY) {
-        if(option & ERTS_LOCK_OPTION_WRITE) {
+        if(option & ERTS_LOCK_OPTIONS_WRITE) {
             ethr_atomic_inc(&info->w_state);
         }
 
-        if(option & ERTS_LOCK_OPTION_READ) {
+        if(option & ERTS_LOCK_OPTIONS_READ) {
             ASSERT(info->flags & ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE);
             ethr_atomic_inc(&info->r_state);
         }
diff --git a/erts/emulator/beam/erl_lock_flags.c b/erts/emulator/beam/erl_lock_flags.c
index ba6a7217f2..e0a0e95c09 100644
--- a/erts/emulator/beam/erl_lock_flags.c
+++ b/erts/emulator/beam/erl_lock_flags.c
@@ -44,3 +44,16 @@ const char *erts_lock_flags_get_type_name(erts_lock_flags_t flags) {
             return "garbage";
     }
 }
+
+const char *erts_lock_options_get_short_desc(erts_lock_options_t options) {
+    switch(options) {
+        case ERTS_LOCK_OPTIONS_RDWR:
+            return "rw";
+        case ERTS_LOCK_OPTIONS_READ:
+            return "r";
+        case ERTS_LOCK_OPTIONS_WRITE:
+            return "w";
+        default:
+            return "none";
+    }
+}
diff --git a/erts/emulator/beam/erl_lock_flags.h b/erts/emulator/beam/erl_lock_flags.h
index b66c160af5..d711f69456 100644
--- a/erts/emulator/beam/erl_lock_flags.h
+++ b/erts/emulator/beam/erl_lock_flags.h
@@ -21,10 +21,10 @@
 #ifndef ERTS_LOCK_FLAGS_H__
 #define ERTS_LOCK_FLAGS_H__
 
-#define ERTS_LOCK_OPTION_READ  (1 << 1)
-#define ERTS_LOCK_OPTION_WRITE (1 << 2)
+#define ERTS_LOCK_OPTIONS_READ  (1 << 1)
+#define ERTS_LOCK_OPTIONS_WRITE (1 << 2)
 
-#define ERTS_LOCK_OPTION_RDWR (ERTS_LOCK_OPTION_READ | ERTS_LOCK_OPTION_WRITE)
+#define ERTS_LOCK_OPTIONS_RDWR (ERTS_LOCK_OPTIONS_READ | ERTS_LOCK_OPTIONS_WRITE)
 
 /* Property/category are bitfields to simplify their use in masks. */
 #define ERTS_LOCK_FLAGS_MASK_CATEGORY (0xFFC0)
@@ -67,8 +67,12 @@
 /* -- -- */
 
 typedef unsigned short erts_lock_flags_t;
+typedef unsigned short erts_lock_options_t;
 
 /* @brief Gets the type name of the lock, honoring the RW flag if supplied. */
 const char *erts_lock_flags_get_type_name(erts_lock_flags_t flags);
 
+/* @brief Gets a short-form description of the given lock options. (rw/r/w) */
+const char *erts_lock_options_get_short_desc(erts_lock_options_t options);
+
 #endif /* ERTS_LOCK_FLAGS_H__ */
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 3fdf29d678..6e5a8210cb 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -2505,7 +2505,7 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx)
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LOCK_OPTION_READ);
+    erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LOCK_OPTIONS_READ);
 #endif
     
     return res;
@@ -2530,7 +2530,7 @@ erts_rwmtx_rlock(erts_rwmtx_t *rwmtx)
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTION_READ);
+    erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_READ);
 #endif
     ethr_rwmutex_rlock(&rwmtx->rwmtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -2547,7 +2547,7 @@ erts_rwmtx_runlock(erts_rwmtx_t *rwmtx)
     erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTION_READ);
+    erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_READ);
 #endif
     ethr_rwmutex_runlock(&rwmtx->rwmtx);
 #endif
@@ -2580,7 +2580,7 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx)
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LOCK_OPTION_RDWR);
+    erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LOCK_OPTIONS_RDWR);
 #endif
     
     return res;
@@ -2605,7 +2605,7 @@ erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx)
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTION_RDWR);
+    erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_RDWR);
 #endif
     ethr_rwmutex_rwlock(&rwmtx->rwmtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -2622,7 +2622,7 @@ erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx)
     erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTION_RDWR);
+    erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_RDWR);
 #endif
     ethr_rwmutex_rwunlock(&rwmtx->rwmtx);
 #endif
@@ -3181,7 +3181,7 @@ erts_read_unlock(erts_rwlock_t *lock)
     erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LOCK_OPTION_READ);
+    erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_READ);
 #endif
     ethr_read_unlock(&lock->rwlck);
 #else
@@ -3205,7 +3205,7 @@ erts_read_lock(erts_rwlock_t *lock)
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_opt(&lock->lcnt, ERTS_LOCK_OPTION_READ);
+    erts_lcnt_lock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_READ);
 #endif
     ethr_read_lock(&lock->rwlck);
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -3224,7 +3224,7 @@ erts_write_unlock(erts_rwlock_t *lock)
     erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LOCK_OPTION_RDWR);
+    erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_RDWR);
 #endif
     ethr_write_unlock(&lock->rwlck);
 #else
@@ -3248,7 +3248,7 @@ erts_write_lock(erts_rwlock_t *lock)
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_lcnt_lock_opt(&lock->lcnt, ERTS_LOCK_OPTION_RDWR);
+    erts_lcnt_lock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_RDWR);
 #endif
     ethr_write_lock(&lock->rwlck);
 #ifdef ERTS_ENABLE_LOCK_COUNT
-- 
cgit v1.2.3


From 9b24da12c766d15493d9c99be18f5a14498ec558 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 10 Jul 2017 08:26:55 +0200
Subject: Close TCP ports properly on send timeout

---
 erts/emulator/drivers/common/inet_drv.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index ebd13e6f05..d87fa3f268 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -9516,6 +9516,7 @@ static void tcp_inet_timeout(ErlDrvData e)
 	    set_busy_port(desc->inet.port, 0);
 	    inet_reply_error_am(INETP(desc), am_timeout);
 	    if (desc->send_timeout_close) {
+                tcp_clear_output(desc);
 		erl_inet_close(INETP(desc));
 	    }
 	}
-- 
cgit v1.2.3


From 518bfe032f201bd236ac1c876de1c4b9321f73dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 10 Jul 2017 08:28:09 +0200
Subject: Add a dedicated close function for TCP ports to prevent issues like
 ERL-430/448

---
 erts/emulator/drivers/common/inet_drv.c | 46 +++++++++++++++++++++------------
 1 file changed, 29 insertions(+), 17 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index d87fa3f268..2a833ca83e 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -1256,6 +1256,8 @@ static int tcp_shutdown_error(tcp_descriptor* desc, int err);
 static int tcp_inet_output(tcp_descriptor* desc, HANDLE event);
 static int tcp_inet_input(tcp_descriptor* desc, HANDLE event);
 
+static void tcp_desc_close(tcp_descriptor*);
+
 #ifdef HAVE_UDP
 typedef struct {
     inet_descriptor inet;   /* common data structure (DON'T MOVE) */
@@ -9138,16 +9140,31 @@ static void tcp_inet_stop(ErlDrvData e)
     tcp_descriptor* desc = (tcp_descriptor*)e;
     DEBUGF(("tcp_inet_stop(%ld) {s=%d\r\n", 
 	    (long)desc->inet.port, desc->inet.s));
+
     tcp_close_check(desc);
-    /* free input buffer & output buffer */
-    if (desc->i_buf != NULL)
-	release_buffer(desc->i_buf);
-    desc->i_buf = NULL; /* net_mess2 may call this function recursively when 
-			   faulty messages arrive on dist ports*/
+    tcp_clear_input(desc);
+
     DEBUGF(("tcp_inet_stop(%ld) }\r\n", (long)desc->inet.port));
     inet_stop(INETP(desc));
 }
 
+/* Closes a tcp descriptor without leaving things hanging; the VM keeps trying
+ * to flush IO queues as long as it contains anything even after the port has
+ * been closed from the erlang side, which is desired behavior (Think escripts
+ * writing to files) but pretty hopeless if the underlying fd has been set to
+ * INVALID_SOCKET through desc_close.
+ *
+ * This function should be used in place of desc_close/erl_inet_close in all
+ * TCP-related operations. Note that this only closes the desc cleanly; it
+ * will be freed through tcp_inet_stop later on. */
+static void tcp_desc_close(tcp_descriptor* desc)
+{
+    tcp_clear_input(desc);
+    tcp_clear_output(desc);
+
+    erl_inet_close(INETP(desc));
+}
+
 /* TCP requests from Erlang */
 static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd,
 				 char* buf, ErlDrvSizeT len,
@@ -9392,7 +9409,7 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd,
     case INET_REQ_CLOSE:
 	DEBUGF(("tcp_inet_ctl(%ld): CLOSE\r\n", (long)desc->inet.port)); 
 	tcp_close_check(desc);
-	erl_inet_close(INETP(desc));
+	tcp_desc_close(desc);
 	return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
 
 
@@ -9516,8 +9533,7 @@ static void tcp_inet_timeout(ErlDrvData e)
 	    set_busy_port(desc->inet.port, 0);
 	    inet_reply_error_am(INETP(desc), am_timeout);
 	    if (desc->send_timeout_close) {
-                tcp_clear_output(desc);
-		erl_inet_close(INETP(desc));
+		tcp_desc_close(desc);
 	    }
 	}
 	else {
@@ -9531,7 +9547,7 @@ static void tcp_inet_timeout(ErlDrvData e)
     else if ((state & INET_STATE_CONNECTING) == INET_STATE_CONNECTING) {
 	/* assume connect timeout */
 	/* close the socket since it's not usable (see man pages) */
-	erl_inet_close(INETP(desc));
+	tcp_desc_close(desc);
 	async_error_am(INETP(desc), am_timeout);
     }
     else if ((state & INET_STATE_ACCEPTING) == INET_STATE_ACCEPTING) {
@@ -9694,8 +9710,7 @@ static int tcp_recv_closed(tcp_descriptor* desc)
 	/* passive mode do not terminate port ! */
 	tcp_clear_input(desc);
 	if (desc->inet.exitf) {
-	    tcp_clear_output(desc);
-	    desc_close(INETP(desc));
+	    tcp_desc_close(desc);
 	} else {
 	    desc_close_read(INETP(desc));
 	}
@@ -9738,7 +9753,7 @@ static int tcp_recv_error(tcp_descriptor* desc, int err)
 	    driver_cancel_timer(desc->inet.port);
 	    tcp_clear_input(desc);
 	    if (desc->inet.exitf) {
-		desc_close(INETP(desc));
+		tcp_desc_close(desc);
 	    } else {
 		desc_close_read(INETP(desc));
 	    }
@@ -10387,9 +10402,6 @@ static int tcp_send_or_shutdown_error(tcp_descriptor* desc, int err)
 	set_busy_port(desc->inet.port, 0);
     }
 
-    tcp_clear_output(desc);
-    tcp_clear_input(desc);
-
     /*
      * We used to handle "expected errors" differently from unexpected ones.
      * Now we handle all errors in the same way (unless the show_econnreset
@@ -10410,10 +10422,10 @@ static int tcp_send_or_shutdown_error(tcp_descriptor* desc, int err)
 	if (desc->inet.exitf)
 	    driver_exit(desc->inet.port, 0);
 	else
-	    desc_close(INETP(desc));
+	    tcp_desc_close(desc);
     } else {
 	tcp_close_check(desc);
-	erl_inet_close(INETP(desc));
+	tcp_desc_close(desc);
 
 	if (desc->inet.caller) {
 	    if (show_econnreset)
-- 
cgit v1.2.3


From 37074ef6833feb2991ff5c07eeb9f8ad015c7df7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Fri, 7 Jul 2017 16:41:29 +0200
Subject: Use erl_lock_flags in lock checking

---
 erts/emulator/beam/erl_lock_check.c   | 280 +++++++++++++++++++---------------
 erts/emulator/beam/erl_lock_check.h   |  50 ++----
 erts/emulator/beam/erl_process_lock.c |  28 ++--
 erts/emulator/beam/erl_threads.h      |  64 ++++----
 4 files changed, 218 insertions(+), 204 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 85ee703c99..cf091ee43f 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -200,41 +200,20 @@ static erts_lc_lock_order_t erts_lock_order[] = {
 #define ERTS_LOCK_ORDER_SIZE \
   (sizeof(erts_lock_order)/sizeof(erts_lc_lock_order_t))
 
-#define LOCK_IS_TYPE_ORDER_VIOLATION(LCK_FLG, LCKD_FLG)			\
-  (((LCKD_FLG) & (ERTS_LC_FLG_LT_SPINLOCK|ERTS_LC_FLG_LT_RWSPINLOCK))	\
-   && ((LCK_FLG)							\
-       & ERTS_LC_FLG_LT_ALL						\
-       & ~(ERTS_LC_FLG_LT_SPINLOCK|ERTS_LC_FLG_LT_RWSPINLOCK)))
+#define LOCK_IS_TYPE_ORDER_VIOLATION(LCK_FLG, LCKD_FLG) \
+    (((LCKD_FLG) & ERTS_LOCK_FLAGS_MASK_TYPE) == ERTS_LOCK_FLAGS_TYPE_SPINLOCK \
+        && \
+     ((LCK_FLG) & ERTS_LOCK_FLAGS_MASK_TYPE) != ERTS_LOCK_FLAGS_TYPE_SPINLOCK)
 
 static __decl_noreturn void  __noreturn lc_abort(void);
 
-static char *
-lock_type(Uint16 flags)
+static const char *rw_op_str(erts_lock_options_t options)
 {
-    switch (flags & ERTS_LC_FLG_LT_ALL) {
-    case ERTS_LC_FLG_LT_SPINLOCK:	return "[spinlock]";
-    case ERTS_LC_FLG_LT_RWSPINLOCK:	return "[rw(spin)lock]";
-    case ERTS_LC_FLG_LT_MUTEX:		return "[mutex]";
-    case ERTS_LC_FLG_LT_RWMUTEX:	return "[rwmutex]";
-    case ERTS_LC_FLG_LT_PROCLOCK:	return "[proclock]";
-    default:				return "";
+    if(options == ERTS_LOCK_OPTIONS_WRITE) {
+        ERTS_INTERNAL_ERROR("Only write flag present");
     }
-}
 
-static char *
-rw_op_str(Uint16 flags)
-{
-    switch (flags & ERTS_LC_FLG_LO_READ_WRITE) {
-    case ERTS_LC_FLG_LO_READ_WRITE:
-	return " (rw)";
-    case ERTS_LC_FLG_LO_READ:
-	return " (r)";
-    case ERTS_LC_FLG_LO_WRITE:
-	ERTS_INTERNAL_ERROR("Only write flag present");
-    default:
-	break;
-    }
-    return "";
+    return erts_lock_options_get_short_desc(options);
 }
 
 typedef struct erts_lc_locked_lock_t_ erts_lc_locked_lock_t;
@@ -245,7 +224,8 @@ struct erts_lc_locked_lock_t_ {
     Sint16 id;
     char *file;
     unsigned int line;
-    Uint16 flags;
+    erts_lock_flags_t flags;
+    erts_lock_options_t taken_options;
 };
 
 typedef struct {
@@ -432,7 +412,7 @@ make_my_locked_locks(void)
 }
 
 static ERTS_INLINE erts_lc_locked_lock_t *
-new_locked_lock(erts_lc_lock_t *lck, Uint16 op_flags,
+new_locked_lock(erts_lc_lock_t *lck, erts_lock_options_t options,
 		char *file, unsigned int line)
 {
     erts_lc_locked_lock_t *l_lck = (erts_lc_locked_lock_t *) lc_alloc();
@@ -442,12 +422,13 @@ new_locked_lock(erts_lc_lock_t *lck, Uint16 op_flags,
     l_lck->extra = lck->extra;
     l_lck->file = file;
     l_lck->line = line;
-    l_lck->flags = lck->flags | op_flags;
+    l_lck->flags = lck->flags;
+    l_lck->taken_options = options;
     return l_lck;
 }
 
 static void
-raw_print_lock(char *prefix, Sint16 id, Wterm extra, Uint16 flags,
+raw_print_lock(char *prefix, Sint16 id, Wterm extra, erts_lock_flags_t flags,
 	       char* file, unsigned int line, char *suffix)
 {
     char *lname = (0 <= id && id < ERTS_LOCK_ORDER_SIZE
@@ -459,16 +440,16 @@ raw_print_lock(char *prefix, Sint16 id, Wterm extra, Uint16 flags,
       erts_fprintf(stderr,"%p",_unchecked_boxed_val(extra));
     else
       erts_fprintf(stderr,"%T",extra);
-    erts_fprintf(stderr,"%s",lock_type(flags));
+    erts_fprintf(stderr,"[%s]",erts_lock_flags_get_type_name(flags));
 
     if (file)
       erts_fprintf(stderr,"(%s:%d)",file,line);
 
-    erts_fprintf(stderr,"'%s%s",rw_op_str(flags),suffix);
+    erts_fprintf(stderr,"'(%s)%s",rw_op_str(flags),suffix);
 }
 
 static void
-print_lock2(char *prefix, Sint16 id, Wterm extra, Uint16 flags, char *suffix)
+print_lock2(char *prefix, Sint16 id, Wterm extra, erts_lock_flags_t flags, char *suffix)
 {
   raw_print_lock(prefix, id, extra, flags, NULL, 0, suffix);
 }
@@ -523,9 +504,9 @@ uninitialized_lock(void)
 
 static void
 lock_twice(char *prefix, erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck,
-	   Uint16 op_flags)
+	   erts_lock_options_t options)
 {
-    erts_fprintf(stderr, "%s%s", prefix, rw_op_str(op_flags));
+    erts_fprintf(stderr, "%s (%s)", prefix, rw_op_str(options));
     print_lock(" ", lck, " lock which is already locked by thread!\n");
     print_curr_locks(l_lcks);
     lc_abort();
@@ -533,9 +514,9 @@ lock_twice(char *prefix, erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck,
 
 static void
 unlock_op_mismatch(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck,
-		   Uint16 op_flags)
+		   erts_lock_options_t options)
 {
-    erts_fprintf(stderr, "Unlocking%s ", rw_op_str(op_flags));
+    erts_fprintf(stderr, "Unlocking (%s) ", rw_op_str(options));
     print_lock("", lck, " lock which mismatch previous lock operation!\n");
     print_curr_locks(l_lcks);
     lc_abort();
@@ -746,84 +727,128 @@ erts_lc_get_lock_order_id(char *name)
     return (Sint16) -1;
 }
 
+static int compare_locked_by_id(erts_lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand)
+{
+    if(locked_lock->id < comparand->id) {
+        return -1;
+    } else if(locked_lock->id > comparand->id) {
+        return 1;
+    }
 
-static int
-find_lock(erts_lc_locked_lock_t **l_lcks, erts_lc_lock_t *lck)
+    return 0;
+}
+
+static int compare_locked_by_id_extra(erts_lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand)
 {
-    erts_lc_locked_lock_t *l_lck = *l_lcks;
+    int order = compare_locked_by_id(locked_lock, comparand);
+
+    if(order) {
+        return order;
+    } else if(locked_lock->extra < comparand->extra) {
+        return -1;
+    } else if(locked_lock->extra > comparand->extra) {
+        return 1;
+    }
 
-    if (l_lck) {
-	if (l_lck->id == lck->id && l_lck->extra == lck->extra) {
-	    if ((l_lck->flags & lck->flags) == lck->flags)
-		return 1;
-	    return 0;
-	}
-	else if (l_lck->id < lck->id
-		 || (l_lck->id == lck->id
-		     && l_lck->extra < lck->extra)) {
-	    for (l_lck = l_lck->next; l_lck; l_lck = l_lck->next) {
-		if (l_lck->id > lck->id
-		    || (l_lck->id == lck->id
-			&& l_lck->extra >= lck->extra)) {
-		    *l_lcks = l_lck;
-		    if (l_lck->id == lck->id
-			&& l_lck->extra == lck->extra
-			&& ((l_lck->flags & lck->flags) == lck->flags))
-			    return 1;
-		    return 0;
-		}
-	    }
-	}
-	else {
-	    for (l_lck = l_lck->prev; l_lck; l_lck = l_lck->prev) {
-		if (l_lck->id < lck->id
-		    || (l_lck->id == lck->id
-			&& l_lck->extra <= lck->extra)) {
-		    *l_lcks = l_lck;
-		    if (l_lck->id == lck->id
-			&& l_lck->extra == lck->extra
-			&& ((l_lck->flags & lck->flags) == lck->flags))
-			return 1;
-		    return 0;
-		}
-	    }
-	}
+    return 0;
+}
+
+typedef int (*locked_compare_func)(erts_lc_locked_lock_t *, erts_lc_lock_t *);
+
+/* Searches through a list of taken locks, bailing when it hits an entry whose
+ * order relative to the search template is the opposite of the one at the
+ * start of the search. (*closest_neighbor) is either set to the exact match,
+ * or the one closest to it in the sort order. */
+static int search_locked_list(locked_compare_func compare,
+                              erts_lc_locked_lock_t *locked_locks,
+                              erts_lc_lock_t *search_template,
+                              erts_lc_locked_lock_t **closest_neighbor)
+{
+    erts_lc_locked_lock_t *iterator = locked_locks;
+
+    (*closest_neighbor) = iterator;
+
+    if(iterator) {
+        int relative_order = compare(iterator, search_template);
+
+        if(relative_order < 0) {
+            while((iterator = iterator->next) != NULL) {
+                relative_order = compare(iterator, search_template);
+
+                if(relative_order >= 0) {
+                    (*closest_neighbor) = iterator;
+                    break;
+                }
+            }
+        } else if(relative_order > 0) {
+            while((iterator = iterator->prev) != NULL) {
+                relative_order = compare(iterator, search_template);
+
+                if(relative_order <= 0) {
+                    (*closest_neighbor) = iterator;
+                    break;
+                }
+            }
+        }
+
+        return relative_order == 0;
     }
+
     return 0;
 }
 
+/* Searches for a lock in the given list that matches search_template, and sets
+ * (*locked_locks) to the closest lock in the sort order. */
 static int
-find_id(erts_lc_locked_lock_t **l_lcks, Sint16 id)
-{
-    erts_lc_locked_lock_t *l_lck = *l_lcks;
-
-    if (l_lck) {
-	if (l_lck->id == id)
-	    return 1;
-	else if (l_lck->id < id) {
-	    for (l_lck = l_lck->next; l_lck; l_lck = l_lck->next) {
-		if (l_lck->id >= id) {
-		    *l_lcks = l_lck;
-		    if (l_lck->id == id)
-			return 1;
-		    return 0;
-		}
-	    }
-	}
-	else {
-	    for (l_lck = l_lck->prev; l_lck; l_lck = l_lck->prev) {
-		if (l_lck->id <= id) {
-		    *l_lcks = l_lck;
-		    if (l_lck->id == id)
-			return 1;
-		    return 0;
-		}
-	    }
-	}
+find_lock(erts_lc_locked_lock_t **locked_locks, erts_lc_lock_t *search_template)
+{
+    erts_lc_locked_lock_t *closest_neighbor;
+    int found_lock;
+
+    found_lock = search_locked_list(compare_locked_by_id_extra,
+                                    (*locked_locks),
+                                    search_template,
+                                    &closest_neighbor);
+
+    (*locked_locks) = closest_neighbor;
+
+    if(found_lock) {
+        erts_lock_options_t relevant_options;
+        erts_lock_flags_t relevant_flags;
+
+        /* We only care about the options and flags that are set in the
+         * template. */
+        relevant_options = (closest_neighbor->taken_options & search_template->taken_options);
+        relevant_flags = (closest_neighbor->flags & search_template->flags);
+
+        return search_template->taken_options == relevant_options &&
+               search_template->flags == relevant_flags;
     }
+
     return 0;
 }
 
+/* Searches for a lock in the given list by id, and sets (*locked_locks) to the
+ * closest lock in the sort order. */
+static int
+find_id(erts_lc_locked_lock_t **locked_locks, Sint16 id)
+{
+    erts_lc_locked_lock_t *closest_neighbor;
+    erts_lc_lock_t search_template;
+    int found_lock;
+
+    search_template.id = id;
+
+    found_lock = search_locked_list(compare_locked_by_id,
+                                    (*locked_locks),
+                                    &search_template,
+                                    &closest_neighbor);
+
+    (*locked_locks) = closest_neighbor;
+
+    return found_lock;
+}
+
 void
 erts_lc_have_locks(int *resv, erts_lc_lock_t *locks, int len)
 {
@@ -919,17 +944,17 @@ erts_lc_check_exact(erts_lc_lock_t *have, int have_len)
 }
 
 void
-erts_lc_check_no_locked_of_type(Uint16 flags)
+erts_lc_check_no_locked_of_type(erts_lock_flags_t type)
 {
     erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
     if (l_lcks) {
 	erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
 	for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next) {
-	    if (l_lck->flags & flags) {
+	    if ((l_lck->flags & ERTS_LOCK_FLAGS_MASK_TYPE) == type) {
 		erts_fprintf(stderr,
 			     "Locked lock of type %s found which isn't "
 			     "allowed here!\n",
-			     lock_type(l_lck->flags));
+			     erts_lock_flags_get_type_name(l_lck->flags));
 		print_curr_locks(l_lcks);
 		lc_abort();
 	    }
@@ -938,7 +963,7 @@ erts_lc_check_no_locked_of_type(Uint16 flags)
 }
 
 int
-erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags)
+erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
 {
 #ifdef ERTS_LC_DO_NOT_FORCE_BUSY_TRYLOCK_ON_LOCK_ORDER_VIOLATION
     return 0;
@@ -987,7 +1012,7 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags)
 	    if (tl_lck->id < lck->id
 		|| (tl_lck->id == lck->id && tl_lck->extra <= lck->extra)) {
 		if (tl_lck->id == lck->id && tl_lck->extra == lck->extra)
-		    lock_twice("Trylocking", l_lcks, lck, op_flags);
+		    lock_twice("Trylocking", l_lcks, lck, options);
 		break;
 	    }
 	}
@@ -1009,7 +1034,7 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags)
 #endif
 }
 
-void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, Uint16 op_flags,
+void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, erts_lock_options_t options,
 			   char *file, unsigned int line)
 {
     erts_lc_locked_locks_t *l_lcks;
@@ -1022,7 +1047,7 @@ void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, Uint16 op_flags,
 	return;
 
     l_lcks = make_my_locked_locks();
-    l_lck = locked ? new_locked_lock(lck, op_flags, file, line) : NULL;
+    l_lck = locked ? new_locked_lock(lck, options, file, line) : NULL;
 
     if (!l_lcks->locked.last) {
 	ASSERT(!l_lcks->locked.first);
@@ -1040,7 +1065,7 @@ void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, Uint16 op_flags,
 	    if (tl_lck->id < lck->id
 		|| (tl_lck->id == lck->id && tl_lck->extra <= lck->extra)) {
 		if (tl_lck->id == lck->id && tl_lck->extra == lck->extra)
-		    lock_twice("Trylocking", l_lcks, lck, op_flags);
+		    lock_twice("Trylocking", l_lcks, lck, options);
 		if (locked) {
 		    l_lck->next = tl_lck->next;
 		    l_lck->prev = tl_lck;
@@ -1063,14 +1088,14 @@ void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, Uint16 op_flags,
 
 }
 
-void erts_lc_require_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags,
+void erts_lc_require_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options,
 			      char *file, unsigned int line)
 {
     erts_lc_locked_locks_t *l_lcks = make_my_locked_locks();
     erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
     if (!find_lock(&l_lck, lck))
 	required_not_locked(l_lcks, lck);
-    l_lck = new_locked_lock(lck, op_flags, file, line);
+    l_lck = new_locked_lock(lck, options, file, line);
     if (!l_lcks->required.last) {
 	ASSERT(!l_lcks->required.first);
 	l_lck->next = l_lck->prev = NULL;
@@ -1110,7 +1135,7 @@ void erts_lc_require_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags,
     }
 }
 
-void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags)
+void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
 {
     erts_lc_locked_locks_t *l_lcks = make_my_locked_locks();
     erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
@@ -1138,7 +1163,7 @@ void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags)
     lc_free((void *) l_lck);
 }
 
-void erts_lc_lock_flg_x(erts_lc_lock_t *lck, Uint16 op_flags,
+void erts_lc_lock_flg_x(erts_lc_lock_t *lck, erts_lock_options_t options,
 			char *file, unsigned int line)
 {
     erts_lc_locked_locks_t *l_lcks;
@@ -1151,7 +1176,7 @@ void erts_lc_lock_flg_x(erts_lc_lock_t *lck, Uint16 op_flags,
 	return;
 
     l_lcks = make_my_locked_locks();
-    l_lck = new_locked_lock(lck, op_flags, file, line);
+    l_lck = new_locked_lock(lck, options, file, line);
 
     if (!l_lcks->locked.last) {
 	ASSERT(!l_lcks->locked.first);
@@ -1167,12 +1192,12 @@ void erts_lc_lock_flg_x(erts_lc_lock_t *lck, Uint16 op_flags,
 	l_lcks->locked.last = l_lck;
     }
     else if (l_lcks->locked.last->id == lck->id && l_lcks->locked.last->extra == lck->extra)
-	lock_twice("Locking", l_lcks, lck, op_flags);
+	lock_twice("Locking", l_lcks, lck, options);
     else
 	lock_order_violation(l_lcks, lck);
 }
 
-void erts_lc_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags)
+void erts_lc_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
 {
     erts_lc_locked_locks_t *l_lcks;
     erts_lc_locked_lock_t *l_lck;
@@ -1193,8 +1218,8 @@ void erts_lc_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags)
 
     for (l_lck = l_lcks ? l_lcks->locked.last : NULL; l_lck; l_lck = l_lck->prev) {
 	if (l_lck->id == lck->id && l_lck->extra == lck->extra) {
-	    if ((l_lck->flags & ERTS_LC_FLG_LO_ALL) != op_flags)
-		unlock_op_mismatch(l_lcks, lck, op_flags);
+	    if ((l_lck->taken_options & ERTS_LOCK_OPTIONS_RDWR) != options)
+		unlock_op_mismatch(l_lcks, lck, options);
 	    if (l_lck->prev)
 		l_lck->prev->next = l_lck->next;
 	    else
@@ -1211,7 +1236,7 @@ void erts_lc_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags)
     unlock_of_not_locked(l_lcks, lck);
 }
 
-void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags)
+void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
 {
     erts_lc_locked_locks_t *l_lcks;
     erts_lc_locked_lock_t *l_lck;
@@ -1275,23 +1300,25 @@ void erts_lc_unrequire_lock(erts_lc_lock_t *lck)
 }
 
 void
-erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags)
+erts_lc_init_lock(erts_lc_lock_t *lck, char *name, erts_lock_flags_t flags)
 {
     lck->id = erts_lc_get_lock_order_id(name);
 
     lck->extra = (UWord) &lck->extra;
     ASSERT(is_not_immed(lck->extra));
     lck->flags = flags;
+    lck->taken_options = 0;
     lck->inited = ERTS_LC_INITITALIZED;
 }
 
 void
-erts_lc_init_lock_x(erts_lc_lock_t *lck, char *name, Uint16 flags, Eterm extra)
+erts_lc_init_lock_x(erts_lc_lock_t *lck, char *name, erts_lock_flags_t flags, Eterm extra)
 {
     lck->id = erts_lc_get_lock_order_id(name);
     lck->extra = extra;
     ASSERT(is_immed(lck->extra));
     lck->flags = flags;
+    lck->taken_options = 0;
     lck->inited = ERTS_LC_INITITALIZED;
 }
 
@@ -1305,6 +1332,7 @@ erts_lc_destroy_lock(erts_lc_lock_t *lck)
     lck->id = -1;
     lck->extra = THE_NON_VALUE;
     lck->flags = 0;
+    lck->taken_options = 0;
 }
 
 void
diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h
index 18296d1fec..8c754a8dfa 100644
--- a/erts/emulator/beam/erl_lock_check.h
+++ b/erts/emulator/beam/erl_lock_check.h
@@ -36,6 +36,8 @@
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
 
+#include "erl_lock_flags.h"
+
 #ifndef ERTS_ENABLE_LOCK_POSITION
 /* Enable in order for _x variants of mtx functions to be used. */
 #define ERTS_ENABLE_LOCK_POSITION 1
@@ -44,36 +46,14 @@
 typedef struct {
     int inited;
     Sint16 id;
-    Uint16 flags;
+    erts_lock_flags_t flags;
+    erts_lock_options_t taken_options;
     UWord extra;
 } erts_lc_lock_t;
 
 #define ERTS_LC_INITITALIZED 0x7f7f7f7f
 
-
-#define ERTS_LC_FLG_LT_SPINLOCK		(((Uint16) 1) << 0)
-#define ERTS_LC_FLG_LT_RWSPINLOCK	(((Uint16) 1) << 1)
-#define ERTS_LC_FLG_LT_MUTEX		(((Uint16) 1) << 2)
-#define ERTS_LC_FLG_LT_RWMUTEX		(((Uint16) 1) << 3)
-#define ERTS_LC_FLG_LT_PROCLOCK		(((Uint16) 1) << 4)
-
-#define ERTS_LC_FLG_LO_READ		(((Uint16) 1) << 5)
-#define ERTS_LC_FLG_LO_WRITE		(((Uint16) 1) << 6)
-
-#define ERTS_LC_FLG_LO_READ_WRITE	(ERTS_LC_FLG_LO_READ		\
-					 | ERTS_LC_FLG_LO_WRITE)
-
-#define ERTS_LC_FLG_LT_ALL		(ERTS_LC_FLG_LT_SPINLOCK	\
-					 | ERTS_LC_FLG_LT_RWSPINLOCK	\
-					 | ERTS_LC_FLG_LT_MUTEX		\
-					 | ERTS_LC_FLG_LT_RWMUTEX	\
-					 | ERTS_LC_FLG_LT_PROCLOCK)
-
-#define ERTS_LC_FLG_LO_ALL		(ERTS_LC_FLG_LO_READ		\
-					 | ERTS_LC_FLG_LO_WRITE)
-
-
-#define ERTS_LC_LOCK_INIT(ID, X, F) {ERTS_LC_INITITALIZED, (ID), (F), (X)}
+#define ERTS_LC_LOCK_INIT(ID, X, F) {ERTS_LC_INITITALIZED, (ID), (F), 0, (X)}
 
 void erts_lc_init(void);
 void erts_lc_late_init(void);
@@ -83,31 +63,31 @@ void erts_lc_check(erts_lc_lock_t *have, int have_len,
 void erts_lc_check_exact(erts_lc_lock_t *have, int have_len);
 void erts_lc_have_locks(int *resv, erts_lc_lock_t *lcks, int len);
 void erts_lc_have_lock_ids(int *resv, int *ids, int len);
-void erts_lc_check_no_locked_of_type(Uint16 flags);
-int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags);
-void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, Uint16 op_flags,
+void erts_lc_check_no_locked_of_type(erts_lock_flags_t flags);
+int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options);
+void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, erts_lock_options_t options,
 			   char *file, unsigned int line);
-void erts_lc_lock_flg_x(erts_lc_lock_t *lck, Uint16 op_flags,
+void erts_lc_lock_flg_x(erts_lc_lock_t *lck, erts_lock_options_t options,
 			char *file, unsigned int line);
-void erts_lc_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags);
-void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags);
+void erts_lc_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options);
+void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options);
 int erts_lc_trylock_force_busy(erts_lc_lock_t *lck);
 void erts_lc_trylock_x(int locked, erts_lc_lock_t *lck,
 		     char* file, unsigned int line);
 void erts_lc_lock_x(erts_lc_lock_t *lck, char* file, unsigned int line);
 void erts_lc_unlock(erts_lc_lock_t *lck);
 void erts_lc_might_unlock(erts_lc_lock_t *lck);
-void erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags);
-void erts_lc_init_lock_x(erts_lc_lock_t *lck, char *name, Uint16 flags, Eterm extra);
+void erts_lc_init_lock(erts_lc_lock_t *lck, char *name, erts_lock_flags_t flags);
+void erts_lc_init_lock_x(erts_lc_lock_t *lck, char *name, erts_lock_flags_t flags, Eterm extra);
 void erts_lc_destroy_lock(erts_lc_lock_t *lck);
 void erts_lc_fail(char *fmt, ...);
 int erts_lc_assert_failed(char *file, int line, char *assertion);
 void erts_lc_set_thread_name(char *thread_name);
 void erts_lc_pll(void);
 
-void erts_lc_require_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags,
+void erts_lc_require_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options,
 			      char *file, unsigned int line);
-void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags);
+void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options);
 
 void erts_lc_require_lock(erts_lc_lock_t *lck, char *file, unsigned int line);
 void erts_lc_unrequire_lock(erts_lc_lock_t *lck);
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index 23c7414901..ff124d5ba7 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -1200,7 +1200,7 @@ erts_proc_lc_lock(Process *p, ErtsProcLocks locks, char *file, unsigned int line
 {
     erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
 					   p->common.id,
-					   ERTS_LC_FLG_LT_PROCLOCK);
+					   ERTS_LOCK_TYPE_PROCLOCK);
     if (locks & ERTS_PROC_LOCK_MAIN) {
 	lck.id = lc_id.proc_lock_main;
 	erts_lc_lock_x(&lck,file,line);
@@ -1233,7 +1233,7 @@ erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked,
 {
     erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
 					   p->common.id,
-					   ERTS_LC_FLG_LT_PROCLOCK);
+					   ERTS_LOCK_TYPE_PROCLOCK);
     if (locks & ERTS_PROC_LOCK_MAIN) {
 	lck.id = lc_id.proc_lock_main;
 	erts_lc_trylock_x(locked, &lck, file, line);
@@ -1265,7 +1265,7 @@ erts_proc_lc_unlock(Process *p, ErtsProcLocks locks)
 {
     erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
 					   p->common.id,
-					   ERTS_LC_FLG_LT_PROCLOCK);
+					   ERTS_LOCK_TYPE_PROCLOCK);
     if (locks & ERTS_PROC_LOCK_TRACE) {
 	lck.id = lc_id.proc_lock_trace;
 	erts_lc_unlock(&lck);
@@ -1300,7 +1300,7 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks)
 #if ERTS_PROC_LOCK_OWN_IMPL
     erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
 					   p->common.id,
-					   ERTS_LC_FLG_LT_PROCLOCK);
+					   ERTS_LOCK_TYPE_PROCLOCK);
     if (locks & ERTS_PROC_LOCK_TRACE) {
 	lck.id = lc_id.proc_lock_trace;
 	erts_lc_might_unlock(&lck);
@@ -1348,7 +1348,7 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file,
 #if ERTS_PROC_LOCK_OWN_IMPL
     erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
 					   p->common.id,
-					   ERTS_LC_FLG_LT_PROCLOCK);
+					   ERTS_LOCK_TYPE_PROCLOCK);
     if (locks & ERTS_PROC_LOCK_MAIN) {
 	lck.id = lc_id.proc_lock_main;
 	erts_lc_require_lock(&lck, file, line);
@@ -1395,7 +1395,7 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks)
 #if ERTS_PROC_LOCK_OWN_IMPL
     erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
 					   p->common.id,
-					   ERTS_LC_FLG_LT_PROCLOCK);
+					   ERTS_LOCK_TYPE_PROCLOCK);
     if (locks & ERTS_PROC_LOCK_TRACE) {
 	lck.id = lc_id.proc_lock_trace;
 	erts_lc_unrequire_lock(&lck);
@@ -1444,7 +1444,7 @@ erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks)
     if (locks & ERTS_PROC_LOCKS_ALL) {
 	erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
 					       p->common.id,
-					       ERTS_LC_FLG_LT_PROCLOCK);
+					       ERTS_LOCK_TYPE_PROCLOCK);
 
 	if (locks & ERTS_PROC_LOCK_MAIN)
 	    lck.id = lc_id.proc_lock_main;
@@ -1475,7 +1475,7 @@ void erts_proc_lc_chk_only_proc_main(Process *p)
 
 #if ERTS_PROC_LOCK_OWN_IMPL
 #define ERTS_PROC_LC_EMPTY_LOCK_INIT \
-  ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LC_FLG_LT_PROCLOCK)
+  ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LOCK_TYPE_PROCLOCK)
 #endif /* ERTS_PROC_LOCK_OWN_IMPL */
 
 void erts_proc_lc_chk_only_proc(Process *p, ErtsProcLocks locks)
@@ -1690,22 +1690,22 @@ erts_proc_lc_my_proc_locks(Process *p)
 #if ERTS_PROC_LOCK_OWN_IMPL
     erts_lc_lock_t locks[6] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main,
 						 p->common.id,
-						 ERTS_LC_FLG_LT_PROCLOCK),
+						 ERTS_LOCK_TYPE_PROCLOCK),
 			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_link,
 						 p->common.id,
-						 ERTS_LC_FLG_LT_PROCLOCK),
+						 ERTS_LOCK_TYPE_PROCLOCK),
 			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_msgq,
 						 p->common.id,
-						 ERTS_LC_FLG_LT_PROCLOCK),
+						 ERTS_LOCK_TYPE_PROCLOCK),
 			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_btm,
 						 p->common.id,
-						 ERTS_LC_FLG_LT_PROCLOCK),
+						 ERTS_LOCK_TYPE_PROCLOCK),
 			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_status,
 						 p->common.id,
-						 ERTS_LC_FLG_LT_PROCLOCK),
+						 ERTS_LOCK_TYPE_PROCLOCK),
 			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_trace,
 						 p->common.id,
-						 ERTS_LC_FLG_LT_PROCLOCK)};
+						 ERTS_LOCK_TYPE_PROCLOCK)};
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
     erts_lc_lock_t locks[6] = {p->lock.main.lc,
 			       p->lock.link.lc,
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 6e5a8210cb..8b5c17d739 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -2180,7 +2180,7 @@ erts_mtx_init(erts_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags)
 #endif
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
+    erts_lc_init_lock_x(&mtx->lc, name, flags, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_init_ref_x(&mtx->lcnt, name, extra, flags);
@@ -2313,7 +2313,8 @@ erts_lc_mtx_is_locked(erts_mtx_t *mtx)
 #if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
     int res;
     erts_lc_lock_t lc = mtx->lc;
-    lc.flags = 0;
+    lc.flags = ERTS_LOCK_FLAGS_TYPE_MUTEX;
+    lc.taken_options = 0;
     erts_lc_have_locks(&res, &lc, 1);
     return res;
 #else
@@ -2413,7 +2414,7 @@ erts_rwmtx_set_reader_group(int no)
 #ifdef USE_THREADS
     int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_check_no_locked_of_type(ERTS_LC_FLG_LT_RWMUTEX);
+    erts_lc_check_no_locked_of_type(ERTS_LOCK_TYPE_RWMUTEX);
 #endif
     res = ethr_rwmutex_set_reader_group(no);
     if (res != 0)
@@ -2436,7 +2437,7 @@ erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, erts_rwmtx_opt_t *opt,
 #endif
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init_lock_x(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX, extra);
+    erts_lc_init_lock_x(&rwmtx->lc, name, flags, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_init_ref_x(&rwmtx->lcnt, name, extra, flags);
@@ -2490,7 +2491,7 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx)
     int res;
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ))
+    if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_READ))
 	return EBUSY; /* Make sure caller can handle the situation without
 			 causing a lock order violation */
 #endif
@@ -2499,9 +2500,9 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx)
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
 #ifdef ERTS_ENABLE_LOCK_POSITION
-    erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line);
+    erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LOCK_OPTIONS_READ,file,line);
 #else
-    erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ);
+    erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LOCK_OPTIONS_READ);
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -2524,9 +2525,9 @@ erts_rwmtx_rlock(erts_rwmtx_t *rwmtx)
 #ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
 #ifdef ERTS_ENABLE_LOCK_POSITION
-    erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line);
+    erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LOCK_OPTIONS_READ,file,line);
 #else
-    erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
+    erts_lc_lock_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_READ);
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -2544,7 +2545,7 @@ erts_rwmtx_runlock(erts_rwmtx_t *rwmtx)
 {
 #ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
+    erts_lc_unlock_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_READ);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_READ);
@@ -2565,7 +2566,7 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx)
     int res;
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE))
+    if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR))
 	return EBUSY; /* Make sure caller can handle the situation without
 			 causing a lock order violation */
 #endif
@@ -2574,9 +2575,9 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx)
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
 #ifdef ERTS_ENABLE_LOCK_POSITION
-    erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
+    erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR,file,line);
 #else
-    erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
+    erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR);
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -2599,9 +2600,9 @@ erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx)
 #ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
 #ifdef ERTS_ENABLE_LOCK_POSITION
-    erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
+    erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR,file,line);
 #else
-    erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
+    erts_lc_lock_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR);
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -2619,7 +2620,7 @@ erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx)
 {
 #ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
+    erts_lc_unlock_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_RDWR);
@@ -2660,7 +2661,8 @@ erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx)
 #if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
     int res;
     erts_lc_lock_t lc = mtx->lc;
-    lc.flags = ERTS_LC_FLG_LO_READ;
+    lc.flags = ERTS_LOCK_TYPE_RWMUTEX;
+    lc.taken_options = ERTS_LOCK_OPTIONS_READ;
     erts_lc_have_locks(&res, &lc, 1);
     return res;
 #else
@@ -2674,7 +2676,8 @@ erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx)
 #if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
     int res;
     erts_lc_lock_t lc = mtx->lc;
-    lc.flags = ERTS_LC_FLG_LO_READ|ERTS_LC_FLG_LO_WRITE;
+    lc.flags = ERTS_LOCK_TYPE_RWMUTEX;
+    lc.taken_options = ERTS_LOCK_OPTIONS_RDWR;
     erts_lc_have_locks(&res, &lc, 1);
     return res;
 #else
@@ -3021,7 +3024,7 @@ erts_spinlock_init(erts_spinlock_t *lock, char *name, Eterm extra, erts_lock_fla
 #endif
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra);
+    erts_lc_init_lock_x(&lock->lc, name, flags, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_init_ref_x(&lock->lcnt, name, extra, flags);
@@ -3109,7 +3112,8 @@ erts_lc_spinlock_is_locked(erts_spinlock_t *lock)
 #if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
     int res;
     erts_lc_lock_t lc = lock->lc;
-    lc.flags = 0;
+    lc.flags = ERTS_LOCK_TYPE_SPINLOCK;
+    lc.taken_options = 0;
     erts_lc_have_locks(&res, &lc, 1);
     return res;
 #else
@@ -3134,7 +3138,7 @@ erts_rwlock_init(erts_rwlock_t *lock, char *name, Eterm extra, erts_lock_flags_t
 #endif
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK, extra);
+    erts_lc_init_lock_x(&lock->lc, name, flags, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_init_ref_x(&lock->lcnt, name, extra, flags);
@@ -3178,7 +3182,7 @@ erts_read_unlock(erts_rwlock_t *lock)
 {
 #ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
+    erts_lc_unlock_flg(&lock->lc, ERTS_LOCK_OPTIONS_READ);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_READ);
@@ -3199,9 +3203,9 @@ erts_read_lock(erts_rwlock_t *lock)
 #ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
 #ifdef ERTS_ENABLE_LOCK_POSITION
-    erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ,file,line);
+    erts_lc_lock_flg_x(&lock->lc, ERTS_LOCK_OPTIONS_READ,file,line);
 #else
-    erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
+    erts_lc_lock_flg(&lock->lc, ERTS_LOCK_OPTIONS_READ);
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -3221,7 +3225,7 @@ erts_write_unlock(erts_rwlock_t *lock)
 {
 #ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
+    erts_lc_unlock_flg(&lock->lc, ERTS_LOCK_OPTIONS_RDWR);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_RDWR);
@@ -3242,9 +3246,9 @@ erts_write_lock(erts_rwlock_t *lock)
 #ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
 #ifdef ERTS_ENABLE_LOCK_POSITION
-    erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
+    erts_lc_lock_flg_x(&lock->lc, ERTS_LOCK_OPTIONS_RDWR,file,line);
 #else
-    erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
+    erts_lc_lock_flg(&lock->lc, ERTS_LOCK_OPTIONS_RDWR);
 #endif
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -3265,7 +3269,8 @@ erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock)
 #if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
     int res;
     erts_lc_lock_t lc = lock->lc;
-    lc.flags = ERTS_LC_FLG_LO_READ;
+    lc.flags = ERTS_LOCK_TYPE_RWSPINLOCK;
+    lc.taken_options = ERTS_LOCK_OPTIONS_READ;
     erts_lc_have_locks(&res, &lc, 1);
     return res;
 #else
@@ -3279,7 +3284,8 @@ erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock)
 #if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
     int res;
     erts_lc_lock_t lc = lock->lc;
-    lc.flags = ERTS_LC_FLG_LO_READ|ERTS_LC_FLG_LO_WRITE;
+    lc.flags = ERTS_LOCK_TYPE_RWSPINLOCK;
+    lc.taken_options = ERTS_LOCK_OPTIONS_RDWR;
     erts_lc_have_locks(&res, &lc, 1);
     return res;
 #else
-- 
cgit v1.2.3


From 5e8f74d6c2d98f22e5f32e866064974de6ee4e33 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 11 Jul 2017 19:33:19 +0200
Subject: erts: Fix bug in enif_whereis_pid/port

that could cause heap corruption if
whereis lookup conflicts with other register updater
AND other thread sends on-heap message while main lock is released.

Also improved enif_whereis from dirty nifs by passing c_p as NULL.
---
 erts/emulator/beam/erl_nif.c     | 19 ++++++++++---------
 erts/emulator/test/nif_SUITE.erl | 10 +++++++---
 2 files changed, 17 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 4815e5e7bb..cdce6abafd 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -889,26 +889,27 @@ static Eterm call_whereis(ErlNifEnv *env, Eterm name)
     Process *c_p;
     Eterm res;
     int scheduler;
-    int unlock;
 
     execution_state(env, &c_p, &scheduler);
     ASSERT((c_p && scheduler) || (!c_p && !scheduler));
 
-    unlock = 0;
     if (scheduler < 0) {
         /* dirty scheduler */
         if (ERTS_PROC_IS_EXITING(c_p))
             return 0;
 
-        if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
-            erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
-            unlock = 1;
-        }
+        if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)
+            c_p = NULL; /* as we don't have main lock */
     }
-    res = erts_whereis_name_to_id(c_p, name);
 
-    if (unlock)
-        erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+
+    if (c_p) {
+         /* main lock may be released below and c_p->htop updated by others */
+        flush_env(env);
+    }
+    res = erts_whereis_name_to_id(c_p, name);
+    if (c_p)
+        cache_env(env);
 
     return res;
 }
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 05c250125d..0337274178 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -2886,11 +2886,15 @@ nif_whereis_parallel(Config) when is_list(Config) ->
     true = lists:all(PidReg, Procs),
 
     %% tell them all to 'fire' as fast as we can
-    [P ! {Ref, send_proc} || {_, P, _} <- Procs],
+    repeat(10, fun(_) ->
+                       [P ! {Ref, send_proc} || {_, P, _} <- Procs]
+               end, void),
 
     %% each gets forwarded through two processes
-    true = lists:all(RecvNum, NSeq),
-    true = lists:all(RecvNum, NSeq),
+    repeat(10, fun(_) ->
+                       true = lists:all(RecvNum, NSeq),
+                       true = lists:all(RecvNum, NSeq)
+               end, void),
 
     %% tell them all to 'quit' by name
     [N ! {Ref, quit} || {N, _, _} <- Procs],
-- 
cgit v1.2.3


From abc4fd372d476821448dfb949bea4e28ab82ac26 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 12 Jul 2017 17:09:10 +0200
Subject: erts: Fix bug in bxor of a big negative number

Wrong result for

(X bsl WS) bxor Y.

where
X is any negative integer
Y is any integer that does not require more words than X
WS is erlang:system_info(wordsize) or larger


Fix: The subtraction of 1 (for 2-complement conversion)
must be carried along all the way to the last words.
---
 erts/emulator/beam/big.c                      | 11 ++++--
 erts/emulator/test/big_SUITE.erl              | 53 +++++++++++++++++++++++++++
 erts/emulator/test/big_SUITE_data/borders.dat | 35 ++++++++++++++++++
 3 files changed, 96 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index d1e46e3063..323cd6c518 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1273,8 +1273,11 @@ static dsize_t I_bxor(ErtsDigit* x, dsize_t xl, short xsgn,
 		*r++ = ~c ^ *y++;
 		x++;
 	    }
-	    while(xl--)
-		*r++ = ~*x++;
+	    while(xl--) {
+		DSUBb(*x,0,b,c);
+		*r++ = ~c;
+		x++;
+	    }
 	}
 	else {
 	    ErtsDigit b1, b2;
@@ -1292,7 +1295,9 @@ static dsize_t I_bxor(ErtsDigit* x, dsize_t xl, short xsgn,
 		x++; y++;
 	    }
 	    while(xl--) {
-		*r++ = *x++;
+                DSUBb(*x,0,b1,c1);
+		*r++ = c1;
+                x++;
 	    }
 	}
     }
diff --git a/erts/emulator/test/big_SUITE.erl b/erts/emulator/test/big_SUITE.erl
index 3193d56e2a..c5bb228bc3 100644
--- a/erts/emulator/test/big_SUITE.erl
+++ b/erts/emulator/test/big_SUITE.erl
@@ -23,6 +23,7 @@
 	 init_per_group/2,end_per_group/2]).
 -export([t_div/1, eq_28/1, eq_32/1, eq_big/1, eq_math/1, big_literals/1,
 	 borders/1, negative/1, big_float_1/1, big_float_2/1,
+         bxor_2pow/1,
 	 shift_limit_1/1, powmod/1, system_limit/1, toobig/1, otp_6692/1]).
 
 %% Internal exports.
@@ -40,6 +41,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}].
 all() -> 
     [t_div, eq_28, eq_32, eq_big, eq_math, big_literals,
      borders, negative, {group, big_float}, shift_limit_1,
+     bxor_2pow,
      powmod, system_limit, toobig, otp_6692].
 
 groups() -> 
@@ -423,3 +425,54 @@ loop2(X,Y,N,M) ->
     end,
     loop2(X,Y,N+1,M).
     
+
+%% ERL-450
+bxor_2pow(_Config) ->
+    IL = lists:seq(8*3, 8*16, 4),
+    JL = lists:seq(0, 64),
+    [bxor_2pow_1((1 bsl I), (1 bsl J))
+     || I <- IL, J <- JL],
+    ok.
+
+bxor_2pow_1(A, B) ->
+    for(-1,1, fun(Ad) ->
+                      for(-1,1, fun(Bd) ->
+                                        bxor_2pow_2(A+Ad, B+Bd),
+                                        bxor_2pow_2(-A+Ad, B+Bd),
+                                        bxor_2pow_2(A+Ad, -B+Bd),
+                                        bxor_2pow_2(-A+Ad, -B+Bd)
+                                end)
+              end).
+
+for(From, To, _Fun) when From > To ->
+    ok;
+for(From, To, Fun) ->
+    Fun(From),
+    for(From+1, To, Fun).
+
+bxor_2pow_2(A, B) ->
+    Correct = my_bxor(A, B),
+    case A bxor B of
+        Correct -> ok;
+        Wrong ->
+            io:format("~.16b bxor ~.16b\n", [A,B]),
+            io:format("Expected ~.16b\n", [Correct]),
+            io:format("Got      ~.16b\n", [Wrong]),
+            ct:fail({failed, 'bxor'})
+
+    end.
+
+%% Implement bxor without bxor
+my_bxor(A, B) ->
+    my_bxor(A, B, 0, 0).
+
+my_bxor(0, 0, _, Acc) -> Acc;
+my_bxor(-1, -1, _, Acc) -> Acc;
+my_bxor(-1, 0, N, Acc) -> (-1 bsl N) bor Acc; % sign extension
+my_bxor(0, -1, N, Acc) -> (-1 bsl N) bor Acc; % sign extension
+my_bxor(A, B, N, Acc0) ->
+    Acc1 = case (A band 1) =:= (B band 1) of
+               true -> Acc0;
+               false -> Acc0 bor (1 bsl N)
+          end,
+    my_bxor(A bsr 1, B bsr 1, N+1, Acc1).
diff --git a/erts/emulator/test/big_SUITE_data/borders.dat b/erts/emulator/test/big_SUITE_data/borders.dat
index 52e4f35861..c38ff93383 100644
--- a/erts/emulator/test/big_SUITE_data/borders.dat
+++ b/erts/emulator/test/big_SUITE_data/borders.dat
@@ -1114,3 +1114,38 @@
 1 = 16#800000000000001 rem (-16#800000000000000).
 0 = 16#FFFFFFFFFFFFFFF800000000 rem 16#FFFFFFFFFFFFFFF80.
 
+% ERL-450 bxor of big negative 2-pow
+-(1 bsl 8) bxor -1 = 16#ff.
+-(1 bsl 16) bxor -1 = 16#ffff.
+-(1 bsl 24) bxor -1 = 16#ffffff.
+-(1 bsl 32) bxor -1 = 16#ffffffff.
+-(1 bsl 40) bxor -1 = 16#ffffffffff.
+-(1 bsl 48) bxor -1 = 16#ffffffffffff.
+-(1 bsl 56) bxor -1 = 16#ffffffffffffff.
+-(1 bsl 64) bxor -1 = 16#ffffffffffffffff.
+-(1 bsl 72) bxor -1 = 16#ffffffffffffffffff.
+-(1 bsl 80) bxor -1 = 16#ffffffffffffffffffff.
+-(1 bsl 88) bxor -1 = 16#ffffffffffffffffffffff.
+-(1 bsl 96) bxor -1 = 16#ffffffffffffffffffffffff.
+-(1 bsl 104) bxor -1 = 16#ffffffffffffffffffffffffff.
+-(1 bsl 112) bxor -1 = 16#ffffffffffffffffffffffffffff.
+-(1 bsl 120) bxor -1 = 16#ffffffffffffffffffffffffffffff.
+-(1 bsl 128) bxor -1 = 16#ffffffffffffffffffffffffffffffff.
+-(1 bsl 136) bxor -1 = 16#ffffffffffffffffffffffffffffffffff.
+-(1 bsl 8) bxor 1 = -16#ff.
+-(1 bsl 16) bxor 1 = -16#ffff.
+-(1 bsl 24) bxor 1 = -16#ffffff.
+-(1 bsl 32) bxor 1 = -16#ffffffff.
+-(1 bsl 40) bxor 1 = -16#ffffffffff.
+-(1 bsl 48) bxor 1 = -16#ffffffffffff.
+-(1 bsl 56) bxor 1 = -16#ffffffffffffff.
+-(1 bsl 64) bxor 1 = -16#ffffffffffffffff.
+-(1 bsl 72) bxor 1 = -16#ffffffffffffffffff.
+-(1 bsl 80) bxor 1 = -16#ffffffffffffffffffff.
+-(1 bsl 88) bxor 1 = -16#ffffffffffffffffffffff.
+-(1 bsl 96) bxor 1 = -16#ffffffffffffffffffffffff.
+-(1 bsl 104) bxor 1 = -16#ffffffffffffffffffffffffff.
+-(1 bsl 112) bxor 1 = -16#ffffffffffffffffffffffffffff.
+-(1 bsl 120) bxor 1 = -16#ffffffffffffffffffffffffffffff.
+-(1 bsl 128) bxor 1 = -16#ffffffffffffffffffffffffffffffff.
+-(1 bsl 136) bxor 1 = -16#ffffffffffffffffffffffffffffffffff.
-- 
cgit v1.2.3


From 8e57a504030e9a5531e6cbed2af9c3013fb351ef Mon Sep 17 00:00:00 2001
From: Alexandre Manuel <alexandre-tsu-manuel@users.noreply.github.com>
Date: Sun, 16 Jul 2017 08:14:57 +0200
Subject: Fix a typo in comment in beam_emu : purpouses -> purposes

---
 erts/emulator/beam/beam_emu.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index c7503aff71..377b8e651d 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1371,7 +1371,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
     start_time = 0;
 #ifdef DEBUG
-    pid = c_p->common.id; /* Save for debugging purpouses */
+    pid = c_p->common.id; /* Save for debugging purposes */
 #endif
     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
     PROCESS_MAIN_CHK_LOCKS(c_p);
-- 
cgit v1.2.3


From a69339d9f8c73d2a0a2369289b464db00c479302 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 12 Jul 2017 14:56:09 +0200
Subject: erts: Remove ERTS_SMP and USE_THREAD defines

This refactor was done using the unifdef tool like this:
    for file in $(find erts/ -name *.[ch]); do unifdef -t -f defile -o $file $file; done

where defile contained:
    #define ERTS_SMP 1
    #define USE_THREADS 1
    #define DDLL_SMP 1
    #define ERTS_HAVE_SMP_EMU 1
    #define SMP 1
    #define ERL_BITS_REENTRANT 1
    #define ERTS_USE_ASYNC_READY_Q 1
    #define FDBLOCK 1
    #undef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
    #define ERTS_POLL_ASYNC_INTERRUPT_SUPPORT 0
    #define ERTS_POLL_USE_WAKEUP_PIPE 1
    #define ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE 1
    #undef ERTS_HAVE_PLAIN_EMU
    #undef ERTS_SIGNAL_STATE
---
 erts/emulator/beam/atom.c                          |  12 -
 erts/emulator/beam/beam_bif_load.c                 |  31 --
 erts/emulator/beam/beam_debug.c                    |   4 -
 erts/emulator/beam/beam_emu.c                      |  54 --
 erts/emulator/beam/beam_load.c                     |  14 -
 erts/emulator/beam/bif.c                           |  68 ---
 erts/emulator/beam/break.c                         |  16 -
 erts/emulator/beam/code_ix.c                       |   2 -
 erts/emulator/beam/dist.c                          |   6 -
 erts/emulator/beam/erl_alloc.c                     |  88 ---
 erts/emulator/beam/erl_alloc.h                     |  34 --
 erts/emulator/beam/erl_alloc_util.c                | 180 ------
 erts/emulator/beam/erl_alloc_util.h                |  22 -
 erts/emulator/beam/erl_async.c                     |  75 ---
 erts/emulator/beam/erl_async.h                     |  18 -
 erts/emulator/beam/erl_bif_ddll.c                  |  64 ---
 erts/emulator/beam/erl_bif_info.c                  |  88 ---
 erts/emulator/beam/erl_bif_port.c                  |   8 -
 erts/emulator/beam/erl_bif_re.c                    |   4 -
 erts/emulator/beam/erl_bif_trace.c                 |  48 --
 erts/emulator/beam/erl_bits.c                      |  18 -
 erts/emulator/beam/erl_bits.h                      |  36 --
 erts/emulator/beam/erl_cpu_topology.c              |   6 -
 erts/emulator/beam/erl_cpu_topology.h              |   2 -
 erts/emulator/beam/erl_db.c                        |  78 ---
 erts/emulator/beam/erl_db_hash.c                   |  59 --
 erts/emulator/beam/erl_db_hash.h                   |   2 -
 erts/emulator/beam/erl_db_util.c                   |  15 -
 erts/emulator/beam/erl_db_util.h                   |   2 -
 erts/emulator/beam/erl_drv_thread.c                | 105 ----
 erts/emulator/beam/erl_fun.c                       |   2 -
 erts/emulator/beam/erl_gc.c                        |   5 -
 erts/emulator/beam/erl_hl_timer.c                  |  22 -
 erts/emulator/beam/erl_hl_timer.h                  |   2 -
 erts/emulator/beam/erl_init.c                      | 119 ----
 erts/emulator/beam/erl_lock_check.c                |  14 -
 erts/emulator/beam/erl_lock_check.h                |   4 -
 erts/emulator/beam/erl_message.c                   |  34 --
 erts/emulator/beam/erl_message.h                   |  14 -
 erts/emulator/beam/erl_msacc.c                     |  22 -
 erts/emulator/beam/erl_msacc.h                     |   9 -
 erts/emulator/beam/erl_nif.c                       |  89 ---
 erts/emulator/beam/erl_node_tables.c               |   8 -
 erts/emulator/beam/erl_port.h                      |  49 --
 erts/emulator/beam/erl_port_task.c                 |  64 ---
 erts/emulator/beam/erl_port_task.h                 |  14 -
 erts/emulator/beam/erl_process.c                   | 416 --------------
 erts/emulator/beam/erl_process.h                   | 106 ----
 erts/emulator/beam/erl_process_lock.c              |   6 -
 erts/emulator/beam/erl_process_lock.h              |  50 +-
 erts/emulator/beam/erl_ptab.h                      |   2 -
 erts/emulator/beam/erl_sched_spec_pre_alloc.c      |   2 -
 erts/emulator/beam/erl_sched_spec_pre_alloc.h      |   2 -
 erts/emulator/beam/erl_smp.h                       | 540 +-----------------
 erts/emulator/beam/erl_thr_progress.c              |   2 -
 erts/emulator/beam/erl_thr_progress.h              |  12 -
 erts/emulator/beam/erl_thr_queue.c                 | 123 -----
 erts/emulator/beam/erl_thr_queue.h                 |  25 -
 erts/emulator/beam/erl_threads.h                   | 605 ---------------------
 erts/emulator/beam/erl_trace.c                     | 225 --------
 erts/emulator/beam/erl_trace.h                     |   6 -
 erts/emulator/beam/erl_utils.h                     |   8 -
 erts/emulator/beam/export.c                        |   4 -
 erts/emulator/beam/global.h                        |   8 -
 erts/emulator/beam/io.c                            |  79 ---
 erts/emulator/beam/register.c                      |  34 --
 erts/emulator/beam/sys.h                           |  27 -
 erts/emulator/beam/utils.c                         |  55 --
 erts/emulator/drivers/common/efile_drv.c           |   9 -
 erts/emulator/hipe/hipe_mkliterals.c               |   6 -
 erts/emulator/hipe/hipe_mode_switch.c              |  26 -
 erts/emulator/hipe/hipe_native_bif.c               |  12 -
 erts/emulator/hipe/hipe_native_bif.h               |   6 -
 erts/emulator/hipe/hipe_primops.h                  |   2 -
 erts/emulator/hipe/hipe_process.h                  |   2 -
 erts/emulator/hipe/hipe_stack.h                    |   5 -
 erts/emulator/hipe/hipe_x86_signal.c               |   8 -
 erts/emulator/sys/common/erl_check_io.c            |  34 --
 erts/emulator/sys/common/erl_check_io.h            |   7 -
 erts/emulator/sys/common/erl_mseg.c                |  17 -
 .../sys/common/erl_os_monotonic_time_extender.c    |  10 -
 .../sys/common/erl_os_monotonic_time_extender.h    |  19 -
 erts/emulator/sys/common/erl_poll.c                | 184 +------
 erts/emulator/sys/common/erl_poll.h                |   3 -
 erts/emulator/sys/unix/erl_unix_sys.h              |  10 -
 erts/emulator/sys/unix/sys.c                       | 126 -----
 erts/emulator/sys/unix/sys_drivers.c               |  24 -
 erts/emulator/sys/unix/sys_float.c                 |   8 -
 erts/emulator/sys/win32/erl_poll.c                 |  23 -
 erts/emulator/sys/win32/erl_win_sys.h              |   2 -
 erts/emulator/sys/win32/sys.c                      |  47 --
 erts/emulator/sys/win32/sys_interrupt.c            |   6 -
 92 files changed, 15 insertions(+), 4548 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index c2d78aaccb..8023414821 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -359,32 +359,24 @@ am_atom_put(const char* name, int len)
 int atom_table_size(void)
 {
     int ret;
-#ifdef ERTS_SMP
     int lock = !ERTS_IS_CRASH_DUMPING;
     if (lock)
 	atom_read_lock();
-#endif
     ret = erts_atom_table.entries;
-#ifdef ERTS_SMP
     if (lock)
 	atom_read_unlock();
-#endif
     return ret;
 }
 
 int atom_table_sz(void)
 {
     int ret;
-#ifdef ERTS_SMP
     int lock = !ERTS_IS_CRASH_DUMPING;
     if (lock)
 	atom_read_lock();
-#endif
     ret = index_table_sz(&erts_atom_table);
-#ifdef ERTS_SMP
     if (lock)
 	atom_read_unlock();
-#endif
     return ret;
 }
 
@@ -412,19 +404,15 @@ erts_atom_get(const char *name, int len, Eterm* ap, ErtsAtomEncoding enc)
 void
 erts_atom_get_text_space_sizes(Uint *reserved, Uint *used)
 {
-#ifdef ERTS_SMP
     int lock = !ERTS_IS_CRASH_DUMPING;
     if (lock)
 	atom_read_lock();
-#endif
     if (reserved)
 	*reserved = reserved_atom_space;
     if (used)
 	*used = atom_space;
-#ifdef ERTS_SMP
     if (lock)
 	atom_read_unlock();
-#endif
 }
 
 void
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 14ddb74324..fb22d28af0 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -265,7 +265,6 @@ struct m {
 };
 
 static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int, int);
-#ifdef ERTS_SMP
 static void smp_code_ix_commiter(void*);
 
 static struct /* Protected by code_write_permission */
@@ -273,7 +272,6 @@ static struct /* Protected by code_write_permission */
     Process* stager;
     ErtsThrPrgrLaterOp lop;
 } committer_state;
-#endif
 
 static Eterm
 exception_list(Process* p, Eterm tag, struct m* mp, Sint exceptions)
@@ -465,9 +463,7 @@ static Eterm
 staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
 		 struct m* mods, int nmods, int free_mods)
 {    
-#ifdef ERTS_SMP
     if (is_blocking || !commit)
-#endif
     {
 	if (commit) {
 	    int i;
@@ -496,7 +492,6 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
 	erts_release_code_write_permission();
 	return res;
     }
-#ifdef ERTS_SMP
     else {
 	ASSERT(is_value(res));
 
@@ -521,11 +516,9 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
 	 */
 	ERTS_BIF_YIELD_RETURN(c_p, res);
     }
-#endif
 }
 
 
-#ifdef ERTS_SMP
 static void smp_code_ix_commiter(void* null)
 {
     Process* p = committer_state.stager;
@@ -542,7 +535,6 @@ static void smp_code_ix_commiter(void* null)
     erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
     erts_proc_dec_refc(p);
 }
-#endif /* ERTS_SMP */
 
 
 
@@ -1312,7 +1304,6 @@ hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp,
     }
 }
 
-#ifdef ERTS_SMP
 
 ErtsThrPrgrLaterOp later_literal_area_switch;
 
@@ -1340,7 +1331,6 @@ complete_literal_area_switch(void *literal_area)
     if (literal_area)
 	erts_release_literal_area((ErtsLiteralArea *) literal_area);
 }
-#endif
 
 BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0)
 {
@@ -1366,7 +1356,6 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0)
     if (!la_ref) {
 	ERTS_SET_COPY_LITERAL_AREA(NULL);
 	if (unused_la) {
-#ifdef ERTS_SMP
 	    ErtsLaterReleasLiteralArea *lrlap;
 	    lrlap = erts_alloc(ERTS_ALC_T_RELEASE_LAREA,
 			       sizeof(ErtsLaterReleasLiteralArea));
@@ -1380,9 +1369,6 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0)
 		 + ((unused_la->end
 		     - &unused_la->start[0])
 		    - 1)*(sizeof(Eterm))));
-#else
-	    erts_release_literal_area(unused_la);
-#endif
 	}
 	BIF_RET(am_false);
     }
@@ -1391,16 +1377,11 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0)
 
     erts_free(ERTS_ALC_T_LITERAL_REF, la_ref);
 
-#ifdef ERTS_SMP
     erts_schedule_thr_prgr_later_op(complete_literal_area_switch,
 				    unused_la,
 				    &later_literal_area_switch);
     erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
     ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
-#else
-    erts_release_literal_area(unused_la);
-    BIF_RET(am_true);
-#endif
 
 }
 
@@ -1506,7 +1487,6 @@ finalize_purge_operation(Process *c_p, int succeded)
     purge_state.fe_ix = 0;
 }
 
-#ifdef ERTS_SMP
 
 static ErtsThrPrgrLaterOp purger_lop_data;
 
@@ -1529,7 +1509,6 @@ finalize_purge_abort(void *unused)
     resume_purger(NULL);
 }
 
-#endif /* ERTS_SMP */
 
 BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 {
@@ -1604,9 +1583,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 	    }
 	}
 	
-#ifndef ERTS_SMP
-	BIF_RET(res);
-#else
 	if (res != am_true)
 	    BIF_RET(res);
 	else {
@@ -1625,7 +1601,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 	    erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
 	    ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
 	}
-#endif
     }
 
     case am_abort: {
@@ -1639,11 +1614,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 
 	erts_fun_purge_abort_prepare(purge_state.funs, purge_state.fe_ix);
 
-#ifndef ERTS_SMP
-	erts_fun_purge_abort_finalize(purge_state.funs, purge_state.fe_ix);
-	finalize_purge_operation(BIF_P, 0);
-	BIF_RET(am_false);
-#else
 	/*
 	 * We need to restore the code addresses of the funs in
 	 * two stages in order to ensure that we do not get any
@@ -1659,7 +1629,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 					&purger_lop_data);
 	erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
 	ERTS_BIF_YIELD_RETURN(BIF_P, am_false);
-#endif
     }
 
     case am_complete: {
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 74e6ab8bc5..5f47a7c99f 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -1148,11 +1148,7 @@ ms_wait(Process *c_p, Eterm etimeout, int busy)
 
 #endif /* ERTS_DIRTY_SCHEDULERS */
 
-#ifdef ERTS_SMP
 #  define ERTS_STACK_LIMIT ((char *) ethr_get_stacklimit())
-#else
-#  define ERTS_STACK_LIMIT ((char *) erts_scheduler_stack_limit)
-#endif
 
 /*
  * The below functions is for testing of the stack
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 377b8e651d..0b4a69411a 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -62,7 +62,6 @@
 #endif
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
-#  ifdef ERTS_SMP
 #    define PROCESS_MAIN_CHK_LOCKS(P)					\
 do {									\
     if ((P))								\
@@ -80,11 +79,6 @@ do {									\
     if ((P))								\
 	erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN);		\
 } while (0)
-#  else
-#    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)
-#    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P)
-#    define PROCESS_MAIN_CHK_LOCKS(P) erts_lc_check_exact(NULL, 0)
-#  endif
 #else
 #  define PROCESS_MAIN_CHK_LOCKS(P)
 #  define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)
@@ -1943,7 +1937,6 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
      msgp = PEEK_MESSAGE(c_p);
 
      if (!msgp) {
-#ifdef ERTS_SMP
 	 erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	 /* Make sure messages wont pass exit signals... */
 	 if (ERTS_PROC_PENDING_EXIT(c_p)) {
@@ -1958,7 +1951,6 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 	 if (msgp)
 	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	 else
-#endif
 	 {
 	     c_p->flags &= ~F_DELAY_GC;
 	     SET_I((BeamInstr *) Arg(0));
@@ -2180,23 +2172,6 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 	 OpCase(wait_f):
 
 	 wait2: {
-#ifndef ERTS_SMP
-	     if (ERTS_PROC_IS_EXITING(c_p)) {
-		 /*
-		  * I non smp case:
-		  *
-		  * Currently executing process might be sent an exit
-		  * signal if it is traced by a port that it also is
-		  * linked to, and the port terminates during the
-		  * trace. In this case we do *not* want to clear
-		  * the active flag, which will make the process hang
-		  * in limbo forever.
-		  */
-		 SWAPOUT;
-                 c_p->arity = 0;
-		 goto do_schedule;
-	     }
-#endif
 	     c_p->i = (BeamInstr *) Arg(0); /* L1 */
 	     SWAPOUT;
 	     c_p->arity = 0;
@@ -3622,9 +3597,7 @@ do {						\
 		typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
 		NifF* fp = vbf = (NifF*) I[1];
 		struct enif_environment_t env;
-#ifdef ERTS_SMP
 		ASSERT(c_p->scheduler_data);
-#endif
 		live_hf_end = c_p->mbuf;
 		ERTS_CHK_MBUF_SZ(c_p);
 		erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
@@ -6539,22 +6512,6 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
     int arity;
     Eterm tmp;
 
-#ifndef ERTS_SMP
-    if (ERTS_PROC_IS_EXITING(c_p)) {
-	/*
-	 * I non smp case:
-	 *
-	 * Currently executing process might be sent an exit
-	 * signal if it is traced by a port that it also is
-	 * linked to, and the port terminates during the
-	 * trace. In this case we do *not* want to clear
-	 * the active flag, which will make the process hang
-	 * in limbo forever. Get out of here and terminate
-	 * the process...
-	 */
-	return -1;
-    }
-#endif
 
     if (is_not_atom(module) || is_not_atom(function)) {
 	/*
@@ -6632,19 +6589,8 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
-#ifndef ERTS_SMP
-	if (ERTS_PROC_IS_EXITING(c_p)) {
-	    /*
-	     * See comment in the beginning of the function...
-	     *
-	     * This second test is needed since gc might be traced.
-	     */
-	    return -1;
-	}
-#else /* ERTS_SMP */
         ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
 	if (!c_p->msg.len)
-#endif
 	    erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
     }
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 71637cf4d6..9bb4525306 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -86,11 +86,9 @@ typedef struct {
     Sint patches;		/* Index (into code buffer) to first location
 				 * which must be patched with the value of this label.
 				 */
-#ifdef ERTS_SMP
     Uint looprec_targeted;	/* Non-zero if this label is the target of a loop_rec
 				 * instruction.
 				 */
-#endif
 } Label;
 
 /*
@@ -1875,9 +1873,7 @@ read_code_header(LoaderState* stp)
     for (i = 0; i < stp->num_labels; i++) {
 	stp->labels[i].value = 0;
 	stp->labels[i].patches = -1;
-#ifdef ERTS_SMP
 	stp->labels[i].looprec_targeted = 0;
-#endif
     }
 
     stp->catches = 0;
@@ -3389,11 +3385,7 @@ negation_is_small(LoaderState* stp, GenOpArg Int)
 static int
 smp(LoaderState* stp)
 {
-#ifdef ERTS_SMP
     return 1;
-#else
-    return 0;
-#endif
 }
 
 /*
@@ -3402,10 +3394,8 @@ smp(LoaderState* stp)
 static int
 smp_mark_target_label(LoaderState* stp, GenOpArg L)
 {
-#ifdef ERTS_SMP
     ASSERT(L.type == TAG_f);
     stp->labels[L.val].looprec_targeted = 1;
-#endif
     return 1;
 }
 
@@ -3416,12 +3406,8 @@ smp_mark_target_label(LoaderState* stp, GenOpArg L)
 static int
 smp_already_locked(LoaderState* stp, GenOpArg L)
 {
-#ifdef ERTS_SMP
     ASSERT(L.type == TAG_u);
     return stp->labels[L.val].looprec_targeted;
-#else
-    return 0;
-#endif
 }
 
 /*
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 890277a3ba..dbd09ef126 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -98,14 +98,12 @@ static int insert_internal_link(Process* p, Eterm rpid)
 
     ASSERT(is_internal_pid(rpid));
 
-#ifdef ERTS_SMP
     if (IS_TRACED(p)
 	&& (ERTS_TRACE_FLAGS(p) & (F_TRACE_SOL|F_TRACE_SOL1))) {
 	rp_locks = ERTS_PROC_LOCKS_ALL;
     }
 
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_LINK);
-#endif
 
     /* get a pointer to the process struct of the linked process */
     rp = erts_pid2proc_opt(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK,
@@ -290,9 +288,6 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
     ErtsMonitor *mon;
     int code;
     Eterm res = am_false;
-#ifndef ERTS_SMP
-    int stale_mon = 0;
-#endif
 
     ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK)
 		       == erts_proc_lc_my_proc_locks(c_p));
@@ -301,13 +296,6 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
     switch (code) {
     case ERTS_DSIG_PREP_NOT_ALIVE:
     case ERTS_DSIG_PREP_NOT_CONNECTED:
-#ifndef ERTS_SMP
-	/* XXX Is this possible? Shouldn't this link
-	   previously have been removed if the node
-	   had previously been disconnected. */
-	ASSERT(0);
-	stale_mon = 1;
-#endif
 	/*
 	 * In the smp case this is possible if the node goes
 	 * down just before the call to demonitor.
@@ -335,13 +323,6 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
 	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_LINK);
 
 	if (!dmon) {
-#ifndef ERTS_SMP
-	    /* XXX How is this possible? Shouldn't this link
-	       previously have been removed when the distributed
-	       end was removed. */
-	    ASSERT(0);
-	    stale_mon = 1;
-#endif
 	    /*
 	     * This is possible when smp support is enabled.
 	     * 'DOWN' message just arrived.
@@ -370,18 +351,6 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
         return am_internal_error;
     }
 
-#ifndef ERTS_SMP
-    if (stale_mon) {
-	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	erts_dsprintf(dsbufp, "Stale process monitor %T to ", ref);
-	if (is_atom(to))
-	    erts_dsprintf(dsbufp, "{%T, %T}", to, dep->sysname);
-	else
-	    erts_dsprintf(dsbufp, "%T", to);
-	erts_dsprintf(dsbufp, " found\n");
-	erts_send_error_to_logger(c_p->group_leader, dsbufp);
-    }
-#endif
 
     /*
      * We aren't allowed to destroy 'mon' until now, since 'to'
@@ -405,13 +374,9 @@ demonitor_local_process(Process *c_p, Eterm ref, Eterm to, Eterm *res)
                            ERTS_P2P_FLG_ALLOW_OTHER_X);
     ErtsMonitor *mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref);
 
-#ifndef ERTS_SMP
-    ASSERT(mon);
-#else
     if (!mon)
         *res = am_false;
     else
-#endif
     {
         *res = am_true;
         erts_destroy_monitor(mon);
@@ -1151,10 +1116,8 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 
     if (is_internal_port(BIF_ARG_1)) {
 	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
-#ifdef ERTS_SMP
 	if (ERTS_PROC_PENDING_EXIT(BIF_P))
 	    goto handle_pending_exit;
-#endif
 
 	l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1);
 
@@ -1201,10 +1164,8 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 	   us in a state where monitors might be inconsistent, but the dist
 	   code should take care of it. */
 	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
-#ifdef ERTS_SMP
 	if (ERTS_PROC_PENDING_EXIT(BIF_P))
 	    goto handle_pending_exit;
-#endif
 	l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1);
 
 	erts_smp_proc_unlock(BIF_P,
@@ -1258,13 +1219,11 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 			   BIF_ARG_1, ERTS_PROC_LOCK_LINK,
 			   ERTS_P2P_FLG_ALLOW_OTHER_X);
 
-#ifdef ERTS_SMP
     if (ERTS_PROC_PENDING_EXIT(BIF_P)) {
 	if (rp && rp != BIF_P)
 	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 	goto handle_pending_exit;
     }
-#endif
 
     /* unlink and ignore errors */
     l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1);
@@ -1294,7 +1253,6 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 
     BIF_RET(am_true);
 
-#ifdef ERTS_SMP
  handle_pending_exit:
     erts_handle_pending_exit(BIF_P, (ERTS_PROC_LOCK_MAIN
 				     | ERTS_PROC_LOCK_LINK
@@ -1302,7 +1260,6 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
     ASSERT(ERTS_PROC_IS_EXITING(BIF_P)); 
     erts_smp_proc_unlock(BIF_P,	ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
     ERTS_BIF_EXITED(BIF_P);
-#endif
 }
 
 BIF_RETTYPE hibernate_3(BIF_ALIST_3)
@@ -1657,12 +1614,10 @@ BIF_RETTYPE exit_2(BIF_ALIST_2)
 			       NIL,
 			       NULL,
 			       BIF_P == rp ? ERTS_XSIG_FLG_NO_IGN_NORMAL : 0);
-#ifdef ERTS_SMP
 	 if (rp == BIF_P)
 	     rp_locks &= ~ERTS_PROC_LOCK_MAIN;
 	 if (rp_locks)
 	     erts_smp_proc_unlock(rp, rp_locks);
-#endif
 	 /*
 	  * We may have exited ourselves and may have to take action.
 	  */
@@ -1783,12 +1738,10 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
 						  ~ERTS_PSFLG_TRAP_EXIT);
        erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND);
 
-#ifdef ERTS_SMP
        if (state & ERTS_PSFLG_PENDING_EXIT) {
 	   erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
 	   ERTS_BIF_EXITED(BIF_P);
        }
-#endif
 
        old_value = (state & ERTS_PSFLG_TRAP_EXIT) ? am_true : am_false;
        BIF_RET(old_value);
@@ -1811,9 +1764,7 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
        }
        else {
 	   new = erts_schedid2runq(sched);
-#ifdef ERTS_SMP
 	   erts_atomic_set_nob(&BIF_P->run_queue, (erts_aint_t) new);
-#endif
 	   state = erts_smp_atomic32_read_bor_mb(&BIF_P->state,
 						 ERTS_PSFLG_BOUND);
        }
@@ -1946,15 +1897,11 @@ BIF_RETTYPE process_flag_3(BIF_ALIST_3)
    Process *rp;
    Eterm res;
 
-#ifdef ERTS_SMP
    rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN,
 				  BIF_ARG_1, ERTS_PROC_LOCK_MAIN);
    if (rp == ERTS_PROC_LOCK_BUSY)
        ERTS_BIF_YIELD3(bif_export[BIF_process_flag_3], BIF_P,
 		       BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
-#else
-   rp = erts_proc_lookup(BIF_ARG_1);
-#endif
 
    if (!rp)
        BIF_ERROR(BIF_P, BADARG);
@@ -2296,10 +2243,8 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx)
  send_message: {
 	ErtsProcLocks rp_locks = 0;
 	Sint res;
-#ifdef ERTS_SMP
 	if (p == rp)
 	    rp_locks |= ERTS_PROC_LOCK_MAIN;
-#endif
 	/* send to local process */
 	res = erts_send_message(p, rp, &rp_locks, msg, 0);
 	if (erts_use_sender_punish)
@@ -4563,9 +4508,6 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
     if (BIF_ARG_1 == am_multi_scheduling) {
 	if (BIF_ARG_2 == am_block || BIF_ARG_2 == am_unblock
 	    || BIF_ARG_2 == am_block_normal || BIF_ARG_2 == am_unblock_normal) {
-#ifndef ERTS_SMP
-	    BIF_RET(am_disabled);
-#else
 	    int block = (BIF_ARG_2 == am_block
 			 || BIF_ARG_2 == am_block_normal);
 	    int normal = (BIF_ARG_2 == am_block_normal
@@ -4601,15 +4543,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
                 BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
                 break;
             }
-#endif
 	}
     } else if (BIF_ARG_1 == am_schedulers_online) {
-#ifndef ERTS_SMP
-	if (BIF_ARG_2 != make_small(1))
-	    goto error;
-	else
-	    BIF_RET(make_small(1));
-#else
 	Sint old_no;
 	if (!is_small(BIF_ARG_2))
 	    goto error;
@@ -4633,7 +4568,6 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	    BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
 	    break;
 	}
-#endif
     } else if (BIF_ARG_1 == am_fullsweep_after) {
 	Uint16 nval;
 	Uint oval;
@@ -5019,7 +4953,6 @@ static BIF_RETTYPE bif_return_trap(BIF_ALIST_2)
     Eterm res = BIF_ARG_1;
 
     switch (BIF_ARG_2) {
-#ifdef ERTS_SMP
     case am_multi_scheduling: {
 	int msb = erts_is_multi_scheduling_blocked();
 	if (msb > 0)
@@ -5030,7 +4963,6 @@ static BIF_RETTYPE bif_return_trap(BIF_ALIST_2)
 	    ERTS_INTERNAL_ERROR("Unexpected multi scheduling block state");
 	break;
     }
-#endif
     default:
 	break;
     }
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 76a0c5c716..9bcf50653d 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -344,9 +344,7 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p)
 	erts_program_counter_info(to, to_arg, p);
     } else {
 	erts_print(to, to_arg, "Stack dump:\n");
-#ifdef ERTS_SMP
 	if (!garbing)
-#endif
 	    erts_stack_dump(to, to_arg, p);
     }
 
@@ -358,11 +356,9 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p)
 static void
 print_garb_info(fmtfn_t to, void *to_arg, Process* p)
 {
-#ifdef ERTS_SMP
     /* ERTS_SMP: A scheduler is probably concurrently doing gc... */
     if (!ERTS_IS_CRASH_DUMPING)
       return;
-#endif
     erts_print(to, to_arg, "New heap start: %bpX\n", p->heap);
     erts_print(to, to_arg, "New heap top: %bpX\n", p->htop);
     erts_print(to, to_arg, "Stack top: %bpX\n", p->stop);
@@ -698,9 +694,7 @@ static int crash_dump_limited_writer(void* vfdp, char* buf, size_t len)
 void
 erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
 {
-#ifdef ERTS_SMP
     ErtsThrPrgrData tpd_buf; /* in case we aren't a managed thread... */
-#endif
     int fd;
     size_t envsz;
     time_t now;
@@ -717,7 +711,6 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     if (ERTS_SOMEONE_IS_CRASH_DUMPING)
 	return;
 
-#ifdef ERTS_SMP
     /* Order all managed threads to block, this has to be done
        first to guarantee that this is the only thread to generate
        crash dump. */
@@ -744,9 +737,6 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     erts_smp_atomic32_set_mb(&erts_writing_erl_crash_dump, 1);
     erts_smp_tsd_set(erts_is_crash_dumping_key, (void *) 1);
 
-#else /* !ERTS_SMP */
-    erts_writing_erl_crash_dump = 1;
-#endif /* ERTS_SMP */
 
     envsz = sizeof(env);
     /* ERL_CRASH_DUMP_SECONDS not set
@@ -841,7 +831,6 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     erts_print_nif_taints(to, to_arg);
     erts_cbprintf(to, to_arg, "Atoms: %d\n", atom_table_size());
 
-#ifdef USE_THREADS
     /* We want to note which thread it was that called erts_exit */
     if (erts_get_scheduler_data()) {
         erts_cbprintf(to, to_arg, "Calling Thread: scheduler:%d\n",
@@ -852,9 +841,6 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
         else
             erts_cbprintf(to, to_arg, "Calling Thread: %p\n", erts_thr_self());
     }
-#else
-    erts_cbprintf(to, to_arg, "Calling Thread: scheduler:1\n");
-#endif
 
 #if defined(ERTS_HAVE_TRY_CATCH)
 
@@ -873,7 +859,6 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     }
 #endif
 
-#ifdef ERTS_SMP
 
 #ifdef ERTS_SYS_SUSPEND_SIGNAL
 
@@ -895,7 +880,6 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
      */
     erts_thr_progress_fatal_error_wait(60000);
     /* Either worked or not... */
-#endif
 
 #ifndef ERTS_HAVE_TRY_CATCH
     /* This is safe to call here, as all schedulers are blocked */
diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c
index 8a3d1b20b4..ba68d612e3 100644
--- a/erts/emulator/beam/code_ix.c
+++ b/erts/emulator/beam/code_ix.c
@@ -115,9 +115,7 @@ void erts_abort_staging_code_ix(void)
 int erts_try_seize_code_write_permission(Process* c_p)
 {
     int success;
-#ifdef ERTS_SMP
     ASSERT(!erts_smp_thr_progress_is_blocking()); /* to avoid deadlock */
-#endif
     ASSERT(c_p != NULL);
 
     erts_smp_mtx_lock(&code_write_permission_mtx);
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 09fdb897f5..e19ff2be68 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -283,10 +283,8 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp)
 	    watched = (is_atom(rmon->name)
 		       ? TUPLE2(lhp, rmon->name, dep->sysname)
 		       : rmon->u.pid);
-#ifdef ERTS_SMP
 	    rp_locks |= ERTS_PROC_LOCKS_MSG_SEND;
 	    erts_smp_proc_lock(rp, ERTS_PROC_LOCKS_MSG_SEND);
-#endif
 	    erts_queue_monitor_message(rp, &rp_locks, mon->ref, am_process, 
 				       watched, am_noconnection);
 	    erts_destroy_monitor(rmon);
@@ -2902,10 +2900,8 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3)
 				     NIL,
 				     NULL,
 				     0);
-#ifdef ERTS_SMP
 	if (lp == BIF_P)
 	    lp_locks &= ~ERTS_PROC_LOCK_MAIN;
-#endif
 	erts_smp_proc_unlock(lp, lp_locks);
 	if (lp == BIF_P) {
 	    erts_aint32_t state = erts_smp_atomic32_read_acqb(&BIF_P->state);
@@ -2913,10 +2909,8 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3)
 	     * We may have exited current process and may have to take action.
 	     */
 	    if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
-#ifdef ERTS_SMP
 		if (state & ERTS_PSFLG_PENDING_EXIT)
 		    erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
-#endif
 		ERTS_BIF_EXITED(BIF_P);
 	    }
 	}
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index c7ab444c96..bf7f71ebde 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -83,14 +83,6 @@
 #define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC
 #define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC
 
-#ifndef ERTS_SMP
-#  undef ERTS_ALC_DEFAULT_ACUL
-#  define ERTS_ALC_DEFAULT_ACUL 0
-#  undef ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC
-#  define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC 0
-#  undef ERTS_ALC_DEFAULT_ACUL_LL_ALLOC
-#  define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC 0
-#endif
 
 #ifdef DEBUG
 static Uint install_debug_functions(void);
@@ -528,7 +520,6 @@ set_default_test_alloc_opts(struct au_init *ip)
 }
 
 
-#ifdef ERTS_SMP
 
 static void
 adjust_tpref(struct au_init *ip, int no_sched)
@@ -551,7 +542,6 @@ adjust_tpref(struct au_init *ip, int no_sched)
     }
 }
 
-#endif
 
 static void handle_args(int *, char **, erts_alc_hndl_args_init_t *);
 
@@ -580,7 +570,6 @@ static void adjust_fix_alloc_sizes(UWord extra_block_size)
     if (extra_block_size && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled) {
 	int j;
 
-#ifdef ERTS_SMP
 	if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec) {
 	    int i;
 	    ErtsAllocatorThrSpec_t* tspec;
@@ -596,7 +585,6 @@ static void adjust_fix_alloc_sizes(UWord extra_block_size)
 	    }
 	}
 	else
-#endif
 	{
 	    Allctr_t* allctr = erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra;
 	    for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) {
@@ -619,7 +607,6 @@ strategy_support_carrier_migration(struct au_init *auip)
 static ERTS_INLINE void
 adjust_carrier_migration_support(struct au_init *auip)
 {
-#ifdef ERTS_SMP
     if (auip->init.util.acul) {
 	auip->thr_spec = -1; /* Need thread preferred */
 
@@ -633,9 +620,6 @@ adjust_carrier_migration_support(struct au_init *auip)
 	    auip->init.aoff.flavor = AOFF_BF;
 	}
     }
-#else
-    auip->init.util.acul = 0;
-#endif
 }
 
 void
@@ -668,10 +652,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
         = sizeof(ErtsNifSelectDataState);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)]
 	= sizeof(ErtsMessageRef);
-#ifdef ERTS_SMP
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_THR_Q_EL_SL)]
 	= sizeof(ErtsThrQElement_t);
-#endif
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_LL_PTIMER)]
 	= erts_timer_type_size(ERTS_ALC_T_LL_PTIMER);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_HL_PTIMER)]
@@ -734,20 +716,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 #endif
     }
 
-#ifndef ERTS_SMP
-    init.sl_alloc.thr_spec = 0;
-    init.std_alloc.thr_spec = 0;
-    init.ll_alloc.thr_spec = 0;
-    init.eheap_alloc.thr_spec = 0;
-    init.binary_alloc.thr_spec = 0;
-    init.ets_alloc.thr_spec = 0;
-    init.driver_alloc.thr_spec = 0;
-    init.fix_alloc.thr_spec = 0;
-    init.literal_alloc.thr_spec = 0;
-#ifdef ERTS_ALC_A_EXEC
-    init.exec_alloc.thr_spec = 0;
-#endif
-#endif
 
     /* Make adjustments for carrier migration support */
     init.temp_alloc.init.util.acul = 0;
@@ -798,7 +766,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 #endif
     }
 
-#ifdef ERTS_SMP
     /* Only temp_alloc can use thread specific interface */
     if (init.temp_alloc.thr_spec)
 	init.temp_alloc.thr_spec = erts_no_schedulers;
@@ -817,10 +784,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     adjust_tpref(&init.exec_alloc, erts_no_schedulers);
 #endif
 
-#else
-    /* No thread specific if not smp */
-    init.temp_alloc.thr_spec = 0;
-#endif
 
     /*
      * The following allocators cannot be run with afit strategy.
@@ -839,10 +802,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     refuse_af_strategy(&init.exec_alloc);
 #endif
 
-#ifdef ERTS_SMP 
     if (!init.temp_alloc.thr_spec)
 	refuse_af_strategy(&init.temp_alloc);
-#endif
 
     erts_mtrace_pre_init();
 #if HAVE_ERTS_MSEG
@@ -1006,8 +967,6 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu)
 	return;
     }
 
-#ifdef USE_THREADS
-#ifdef ERTS_SMP
     if (init->thr_spec) {
 	if (init->thr_spec > 0) {
 	    af->alloc = erts_alcu_alloc_thr_spec;
@@ -1037,7 +996,6 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu)
 	ai->thr_spec	= tspec->size;
     }
     else
-#endif
 	if (init->init.util.ts) {
 	af->alloc = erts_alcu_alloc_ts;
 	if (init->init.util.fix_type_size)
@@ -1049,21 +1007,9 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu)
 	af->free = erts_alcu_free_ts;
     }
     else
-#endif
     {
-#ifdef ERTS_SMP
         erts_exit(ERTS_ABORT_EXIT, "%salloc is not thread safe\n",
                  init->init.util.name_prefix);
-#else
-	af->alloc = erts_alcu_alloc;
-	if (init->init.util.fix_type_size)
-	    af->realloc = erts_realloc_fixed_size;
-	else if (init->init.util.ramv)
-	    af->realloc = erts_alcu_realloc_mv;
-	else
-	    af->realloc = erts_alcu_realloc;
-	af->free = erts_alcu_free;
-#endif
     }
     af->extra	= NULL;
     ai->alloc_util	= 1;
@@ -1915,7 +1861,6 @@ erts_alloc_register_scheduler(void *vesdp)
     }
 }
 
-#ifdef ERTS_SMP
 void
 erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
 					    int *need_thr_progress,
@@ -1944,12 +1889,10 @@ erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
 	}
     }
 }
-#endif
 
 erts_aint32_t
 erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs)
 {
-#ifdef ERTS_SMP
     ErtsAllocatorThrSpec_t *tspec;
     tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE];
     if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec && tspec->enabled)
@@ -1957,11 +1900,6 @@ erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs)
     if (ix == 0 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra)
 	return erts_alcu_fix_alloc_shrink(
 	    erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs);
-#else
-    if (ix == 1 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra)
-	return erts_alcu_fix_alloc_shrink(
-	    erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs);
-#endif
     return 0;
 }
 
@@ -2845,11 +2783,7 @@ erts_allocator_info(fmtfn_t to, void *arg)
 #if HAVE_ERTS_MSEG
     {
 	struct erts_mmap_info_struct emis;
-#ifdef ERTS_SMP
 	int max = (int) erts_no_schedulers;
-#else
-	int max = 0;
-#endif
 	int i;
 	for (i = 0; i <= max; i++) {
 	    erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i);
@@ -3451,13 +3385,11 @@ erts_request_alloc_info(struct process *c_p,
 
     erts_proc_add_refc(c_p, (Sint) erts_no_schedulers);
 
-#ifdef ERTS_SMP
     if (erts_no_schedulers > 1)
 	erts_schedule_multi_misc_aux_work(1,
 					  erts_no_schedulers,
 					  reply_alloc_info,
 					  (void *) air);
-#endif
 
     reply_alloc_info((void *) air);
 
@@ -3532,35 +3464,29 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
     case 0xf:
 	switch (op) {
 	case 0xf00:
-#ifdef USE_THREADS
 	    if (((Allctr_t *) a1)->thread_safe)
 		return (UWord) erts_alcu_alloc_ts(ERTS_ALC_T_UNDEF,
 							  (void *) a1,
 							  (Uint) a2);
 	    else
-#endif
 		return (UWord) erts_alcu_alloc(ERTS_ALC_T_UNDEF,
 						       (void *) a1,
 						       (Uint) a2);
 	case 0xf01:
-#ifdef USE_THREADS
 	    if (((Allctr_t *) a1)->thread_safe)
 		return (UWord) erts_alcu_realloc_ts(ERTS_ALC_T_UNDEF,
 							    (void *) a1,
 							    (void *) a2,
 							    (Uint) a3);
 	    else
-#endif
 		return (UWord) erts_alcu_realloc(ERTS_ALC_T_UNDEF,
 							 (void *) a1,
 							 (void *) a2,
 							 (Uint) a3);
 	case 0xf02:
-#ifdef USE_THREADS
 	    if (((Allctr_t *) a1)->thread_safe)
 		erts_alcu_free_ts(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2);
 	    else
-#endif
 		erts_alcu_free(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2);
 	    return 0;
 	case 0xf03: {
@@ -3571,11 +3497,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
 	    init.enable = 1;
 	    init.atype = GOODFIT;
 	    init.init.util.name_prefix = (char *) a1;
-#ifdef ERTS_SMP
 	    init.init.util.ts = 1;
-#else
-            init.init.util.ts = a2 ? 1 : 0;
-#endif
 	    if ((char **) a3) {
 		char **argv = (char **) a3;
 		int i = 0;
@@ -3630,7 +3552,6 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
 	    erts_alcu_stop((Allctr_t *) a1);
 	    erts_free(ERTS_ALC_T_UNDEF, (void *) a1);
 	    break;
-#ifdef USE_THREADS
 	case 0xf05: return (UWord) 1;
 	case 0xf06: return (UWord) ((Allctr_t *) a1)->thread_safe;
 #ifdef ETHR_NO_FORKSAFETY
@@ -3700,12 +3621,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
 	    ethr_thr_exit((void *) a1);
 	    ERTS_ALC_TEST_ABORT;
 	    break;
-#endif /* #ifdef USE_THREADS */
-#ifdef ERTS_SMP
 	case 0xf13: return (UWord) 1;
-#else
-	case 0xf13: return (UWord) 0;
-#endif
 	case 0xf14: return (UWord) erts_alloc(ERTS_ALC_T_TEST, (Uint)a1);
 
 	case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0;
@@ -3909,10 +3825,8 @@ void check_allocators(void)
 	    ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) erts_allctrs[i].extra;
 	    Allctr_t *allctr = real_af->extra;
 	    Carrier_t *ct;
-#ifdef USE_THREADS
 	if (allctr->thread_safe)
 	    erts_mtx_lock(&allctr->mutex);
-#endif
 
 	    if (allctr->check_mbc) {
 		for (ct = allctr->mbc_list.first; ct; ct = ct->next) {
@@ -3920,10 +3834,8 @@ void check_allocators(void)
 		    allctr->check_mbc(allctr,ct);
 		}
 	    }
-#ifdef USE_THREADS
 	if (allctr->thread_safe)
 	    erts_mtx_unlock(&allctr->mutex);
-#endif
 	}
     }
 }
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 97a1cf1308..ed5ff4a2ff 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -27,9 +27,7 @@
 #include "erl_thr_progress.h"
 #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
 #include "erl_alloc_util.h"
-#ifdef USE_THREADS
 #include "erl_threads.h"
-#endif
 #include "erl_mmap.h"
 
 #ifdef DEBUG
@@ -154,12 +152,10 @@ void erts_allctr_wrapper_pre_lock(void);
 void erts_allctr_wrapper_pre_unlock(void);
 
 void erts_alloc_register_scheduler(void *vesdp);
-#ifdef ERTS_SMP
 void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
 						 int *need_thr_progress,
 						 ErtsThrPrgrVal *thr_prgr_p,
 						 int *more_work);
-#endif
 erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs);
 
 __decl_noreturn void erts_alloc_enomem(ErtsAlcType_t,Uint)		
@@ -349,23 +345,10 @@ ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT,				\
 		      erts_smp_spin_lock(&NAME##_lck),			\
 		      erts_smp_spin_unlock(&NAME##_lck))
 
-#ifdef ERTS_SMP
 
 #define ERTS_TS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)			\
 ERTS_SMP_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)
 
-#else /* !ERTS_SMP */
-
-#define ERTS_TS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)			\
-static erts_mtx_t NAME##_lck;						\
-ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT,				\
-		      erts_mtx_init(NAME##_lck, #NAME "_alloc_lock", NIL, \
-		          ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR),\
-		      erts_mtx_lock(&NAME##_lck),			\
-		      erts_mtx_unlock(&NAME##_lck))
-
-
-#endif
 
 #define ERTS_PALLOC_IMPL(NAME, TYPE, PASZ)				\
 ERTS_PRE_ALLOC_IMPL(NAME, TYPE, PASZ, (void) 0, (void) 0, (void) 0)
@@ -378,17 +361,10 @@ ERTS_PRE_ALLOC_IMPL(NAME, TYPE, PASZ,					\
 		    erts_spin_lock(&NAME##_lck),			\
 		    erts_spin_unlock(&NAME##_lck))
 
-#ifdef ERTS_SMP
 
 #define ERTS_SMP_PALLOC_IMPL(NAME, TYPE, PASZ)				\
   ERTS_TS_PALLOC_IMPL(NAME, TYPE, PASZ)
 
-#else /* !ERTS_SMP */
-
-#define ERTS_SMP_PALLOC_IMPL(NAME, TYPE, PASZ)				\
-  ERTS_PALLOC_IMPL(NAME, TYPE, PASZ)
-
-#endif
 
 #define ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT, ILCK, LCK, ULCK)	\
 ERTS_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ, ILCK, LCK, ULCK)		\
@@ -412,21 +388,11 @@ NAME##_free(TYPE *p)							\
 	erts_free(ALCT, (void *) p);					\
 }
 
-#ifdef ERTS_SMP
 #define ERTS_SCHED_PREF_PALLOC_IMPL(NAME, TYPE, PASZ)			\
   ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ)
-#else
-#define ERTS_SCHED_PREF_PALLOC_IMPL(NAME, TYPE, PASZ)			\
-  ERTS_PRE_ALLOC_IMPL(NAME, TYPE, PASZ, (void) 0, (void) 0, (void) 0)
-#endif
 
-#ifdef ERTS_SMP
 #define ERTS_SCHED_PREF_AUX(NAME, TYPE, PASZ)				\
 ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ)
-#else
-#define ERTS_SCHED_PREF_AUX(NAME, TYPE, PASZ)				\
-ERTS_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ, (void) 0, (void) 0, (void) 0)
-#endif
 
 #define ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT)	\
 ERTS_SCHED_PREF_AUX(NAME, TYPE, PASZ)					\
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index af86ad0548..f4c88438f5 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -305,9 +305,6 @@ MBC after deallocating first block:
 #  define ERTS_ALC_CPOOL_DEBUG
 #endif
 
-#ifndef ERTS_SMP
-#  undef ERTS_ALC_CPOOL_DEBUG
-#endif
 
 #ifdef ERTS_ALC_CPOOL_DEBUG
 #  define ERTS_ALC_CPOOL_ASSERT(A)				\
@@ -322,13 +319,8 @@ MBC after deallocating first block:
 #  define ERTS_ALC_CPOOL_ASSERT(A) ((void) 1)
 #endif
 
-#ifdef ERTS_SMP
 #define ERTS_ALC_IS_CPOOL_ENABLED(A)	((A)->cpool.util_limit)
-#else
-#define ERTS_ALC_IS_CPOOL_ENABLED(A)	(0)
-#endif
 
-#ifdef ERTS_SMP
 
 #define ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON		1000
 #define ERTS_ALC_CPOOL_ALLOC_OP_INC			8
@@ -367,28 +359,16 @@ do {										\
     }										\
 } while (0)
 
-#else
-#define ERTS_ALC_CPOOL_ALLOC_OP(A)
-#define ERTS_ALC_CPOOL_REALLOC_OP(A)
-#define ERTS_ALC_CPOOL_FREE_OP(A)
-#endif
 
 #define ERTS_CRR_ALCTR_FLG_IN_POOL	(((erts_aint_t) 1) << 0)
 #define ERTS_CRR_ALCTR_FLG_BUSY		(((erts_aint_t) 1) << 1)
 #define ERTS_CRR_ALCTR_FLG_MASK (ERTS_CRR_ALCTR_FLG_IN_POOL | \
                                  ERTS_CRR_ALCTR_FLG_BUSY)
 
-#ifdef ERTS_SMP
 #define SBC_HEADER_SIZE	   						\
     (UNIT_CEILING(offsetof(Carrier_t, cpool)                            \
 	          + ABLK_HDR_SZ)	                                \
      - ABLK_HDR_SZ)
-#else
-#define SBC_HEADER_SIZE	   						\
-    (UNIT_CEILING(sizeof(Carrier_t)					\
-		  + ABLK_HDR_SZ)					\
-     - ABLK_HDR_SZ)
-#endif
 #define MBC_HEADER_SIZE(AP) ((AP)->mbc_header_size)
 
 
@@ -598,15 +578,11 @@ do {									\
     (AP)->mbcs.blocks.curr.size -= (CRR)->cpool.blocks_size;		\
 } while (0)
 
-#ifdef ERTS_SMP
 #define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ)				\
 do {									\
     (CRR)->cpool.blocks++;						\
     (CRR)->cpool.blocks_size += (BSZ);					\
 } while (0)
-#else
-#define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) ((void) (CRR)) /* Get rid of warning */
-#endif
 
 #define STAT_MBC_BLK_ALLOC(AP, CRR, BSZ, FLGS)	       			\
 do {									\
@@ -626,7 +602,6 @@ stat_cpool_mbc_blk_free(Allctr_t *allctr,
 			Carrier_t **busy_pcrr_pp,
 			UWord blksz)
 {
-#ifdef ERTS_SMP
 
     ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks > 0);
     crr->cpool.blocks--;
@@ -651,9 +626,6 @@ stat_cpool_mbc_blk_free(Allctr_t *allctr,
 #endif
 
     return 1;
-#else
-    return 0;
-#endif
 }
 
 #define STAT_MBC_BLK_FREE(AP, CRR, BPCRRPP, BSZ, FLGS)			\
@@ -689,12 +661,7 @@ do {									\
 #endif
 
 #ifdef DEBUG
-#ifdef USE_THREADS
-# ifdef ERTS_SMP
 #  define IS_ACTUALLY_BLOCKING (erts_thr_progress_is_blocking())
-# else
-#  define IS_ACTUALLY_BLOCKING 0
-# endif
 #define ERTS_ALCU_DBG_CHK_THR_ACCESS(A)					\
 do {									\
     if (!(A)->thread_safe && !IS_ACTUALLY_BLOCKING) {                   \
@@ -711,9 +678,6 @@ do {									\
 #else
 #define ERTS_ALCU_DBG_CHK_THR_ACCESS(A)
 #endif
-#else
-#define ERTS_ALCU_DBG_CHK_THR_ACCESS(A)
-#endif
 
 static void make_name_atoms(Allctr_t *allctr);
 
@@ -1191,7 +1155,6 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr)
     }
 }
 
-#ifdef ERTS_SMP
 
 #ifdef DEBUG
 static int is_in_list(ErtsDoubleLink_t* sentinel, ErtsDoubleLink_t* node)
@@ -1301,7 +1264,6 @@ clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr)
     }
 }
 
-#endif /* ERTS_SMP */
 
 #if 0
 #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B)			\
@@ -1325,12 +1287,10 @@ chk_fix_list(Allctr_t *allctr, ErtsAlcFixList_t *fix, int ix, int before)
 
 static void *mbc_alloc(Allctr_t *allctr, Uint size);
 
-#ifdef ERTS_SMP
 typedef struct {
     ErtsAllctrDDBlock_t ddblock__; /* must be first */
     ErtsAlcType_t fix_type;
 } ErtsAllctrFixDDBlock_t;
-#endif
 
 #define ERTS_ALC_FIX_NO_UNUSE (((ErtsAlcType_t) 1) << ERTS_ALC_N_BITS)
 
@@ -1341,11 +1301,9 @@ dealloc_fix_block(Allctr_t *allctr,
 		  ErtsAlcFixList_t *fix,
 		  int dec_cc_on_redirect)
 {
-#ifdef ERTS_SMP
     /* May be redirected... */
     ASSERT((type & ERTS_ALC_FIX_NO_UNUSE) == 0);
     ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type | ERTS_ALC_FIX_NO_UNUSE;
-#endif
     dealloc_block(allctr, ptr, fix, dec_cc_on_redirect);
 }
 
@@ -1379,12 +1337,10 @@ fix_cpool_check_shrink(Allctr_t *allctr,
 	    fix->u.cpool.shrink_list = 0;
 	else {
 	    void *p;
-#ifdef ERTS_SMP
 	    if (busy_pcrr_pp) {
 		clear_busy_pool_carrier(allctr, *busy_pcrr_pp);
 		*busy_pcrr_pp = NULL;
 	    }
-#endif
 	    fix->u.cpool.shrink_list--;
 	    p = fix->list;
 	    fix->list = *((void **) p);
@@ -1477,10 +1433,8 @@ fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
     int ix, o;
     int flush = flgs == 0;
 
-#ifdef USE_THREADS
     if (allctr->thread_safe)
 	erts_mtx_lock(&allctr->mutex);
-#endif
 
     for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) {
 	ErtsAlcFixList_t *fix = &allctr->fix[ix];
@@ -1520,10 +1474,8 @@ fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
     if (all_empty)
 	sched_fix_shrink(allctr, 0);
 
-#ifdef USE_THREADS
     if (allctr->thread_safe)
 	erts_mtx_unlock(&allctr->mutex);
-#endif
 
     return res;
 }
@@ -1635,10 +1587,8 @@ fix_nocpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
     int ix, o;
     int flush = flgs == 0;
 
-#ifdef USE_THREADS
     if (allctr->thread_safe)
 	erts_mtx_lock(&allctr->mutex);
-#endif
 
     for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) {
 	ErtsAlcFixList_t *fix = &allctr->fix[ix];
@@ -1680,10 +1630,8 @@ fix_nocpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
     if (all_empty)
 	sched_fix_shrink(allctr, 0);
 
-#ifdef USE_THREADS
     if (allctr->thread_safe)
 	erts_mtx_unlock(&allctr->mutex);
-#endif
 
     return res;
 }
@@ -1709,7 +1657,6 @@ dealloc_mbc(Allctr_t *allctr, Carrier_t *crr)
     dealloc_carrier(allctr, crr, 1);
 }
 
-#ifdef ERTS_SMP
 
 static ERTS_INLINE Allctr_t*
 get_pref_allctr(void *extra)
@@ -2201,9 +2148,7 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type,
 	erts_alloc_notify_delayed_dealloc(allctr->ix);
 }
 
-#endif
 
-#ifdef ERTS_SMP
 static void
 set_new_allctr_abandon_limit(Allctr_t *allctr);
 static void
@@ -2265,7 +2210,6 @@ erts_alcu_check_delayed_dealloc(Allctr_t *allctr,
 			   thr_prgr_p,
 			   more_work);
 }
-#endif
 
 #define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked)			\
     handle_delayed_dealloc((Allctr), (Locked), 1, 			\
@@ -2281,19 +2225,13 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_
 
     if (IS_SBC_BLK(blk)) {
 	destroy_carrier(allctr, blk, NULL);
-#ifdef ERTS_SMP
 	if (fix && ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
 	    ErtsAlcType_t type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type;
 	    if (!(type & ERTS_ALC_FIX_NO_UNUSE))
 		fix->u.cpool.used--;
 	    fix->u.cpool.allocated--;
 	}
-#endif
     }
-#ifndef ERTS_SMP
-    else
-	mbc_free(allctr, ptr, NULL);
-#else
     else if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
 	mbc_free(allctr, ptr, NULL);
     else {
@@ -2323,7 +2261,6 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_
 		erts_alloc_notify_delayed_dealloc(used_allctr->ix);
 	}
     }
-#endif
 }
 
 /* Multi block carrier alloc/realloc/free ... */
@@ -2571,9 +2508,7 @@ mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp)
     else {
 	(*allctr->link_free_block)(allctr, blk);
 	HARD_CHECK_BLK_CARRIER(allctr, blk);
-#ifdef ERTS_SMP
 	check_abandon_carrier(allctr, blk, busy_pcrr_pp);
-#endif
     }
 }
 
@@ -2607,10 +2542,8 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs,
 	return NULL;
 #else /* !MBC_REALLOC_ALWAYS_MOVES */
 
-#ifdef ERTS_SMP
     if (busy_pcrr_pp && *busy_pcrr_pp)
 	goto realloc_move; /* Don't want to use carrier in pool */
-#endif
 
     get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size);
 
@@ -2731,9 +2664,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs,
 		    
 	HARD_CHECK_BLK_CARRIER(allctr, blk);
 
-#ifdef ERTS_SMP
 	check_abandon_carrier(allctr, nxt_blk, NULL);
-#endif
 
 	return p;
     }
@@ -2845,9 +2776,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs,
 
     if (cand_blk_sz < get_blk_sz) {
 	/* We wont fit in cand_blk get a new one */
-#ifdef ERTS_SMP
     realloc_move:
-#endif
 #endif /* !MBC_REALLOC_ALWAYS_MOVES */
 
 	new_p = mbc_alloc(allctr, size);
@@ -2949,7 +2878,6 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs,
 #endif /* !MBC_REALLOC_ALWAYS_MOVES */
 }
 
-#ifdef ERTS_SMP
 
 #define ERTS_ALC_MAX_DEALLOC_CARRIER		10
 #define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT	20
@@ -3735,7 +3663,6 @@ cpool_read_stat(Allctr_t *allctr, UWord *nocp, UWord *cszp, UWord *nobp, UWord *
 }
 
 
-#endif /* ERTS_SMP */
 
 #ifdef DEBUG
 
@@ -3836,7 +3763,6 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
         blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz);
     }
 
-#ifdef ERTS_SMP
     allctr->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON;
 
     if ((flags & (CFLG_MBC|CFLG_NO_CPOOL)) == CFLG_MBC
@@ -3852,7 +3778,6 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 	    return blk;
 	}
     }
-#endif
 
 #if HAVE_ERTS_MSEG
 
@@ -3982,9 +3907,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 	    allctr->main_carrier = crr;
 	}
 
-#ifdef ERTS_SMP
 	cpool_init_carrier_data(allctr, crr);
-#endif
 
 	link_carrier(&allctr->mbc_list, crr);
 
@@ -4204,7 +4127,6 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
 	}
 #endif
 
-#ifdef ERTS_SMP
 	if (busy_pcrr_pp && *busy_pcrr_pp) {
 	    ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr);
 	    *busy_pcrr_pp = NULL;
@@ -4216,7 +4138,6 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
 	    cpool_delete(allctr, allctr, crr);
 	}
 	else
-#endif
 	{
 	    unlink_carrier(&allctr->mbc_list, crr);
 #if HAVE_ERTS_MSEG
@@ -4247,11 +4168,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
         }
 #endif
 
-#ifdef ERTS_SMP
 	schedule_dealloc_carrier(allctr, crr);
-#else
-	dealloc_mbc(allctr, crr);
-#endif
     }
 }
 
@@ -4294,9 +4211,7 @@ static struct {
     Eterm fix_types;
 
     Eterm mbcs;
-#ifdef ERTS_SMP
     Eterm mbcs_pool;
-#endif
     Eterm sbcs;
 
     Eterm sys_alloc_carriers_size;
@@ -4384,9 +4299,7 @@ init_atoms(Allctr_t *allctr)
 	AM_INIT(fix_types);
 
 	AM_INIT(mbcs);
-#ifdef ERTS_SMP
 	AM_INIT(mbcs_pool);
-#endif
 	AM_INIT(sbcs);
 
 	AM_INIT(sys_alloc_carriers_size);
@@ -4636,7 +4549,6 @@ sz_info_carriers(Allctr_t *allctr,
     return res;
 }
 
-#ifdef ERTS_SMP
 
 static Eterm
 info_cpool(Allctr_t *allctr,
@@ -4690,7 +4602,6 @@ info_cpool(Allctr_t *allctr,
     return res;
 }
 
-#endif /* ERTS_SMP */
 
 static Eterm
 info_carriers(Allctr_t *allctr,
@@ -4945,11 +4856,7 @@ info_options(Allctr_t *allctr,
 	return res;
     }
 
-#ifdef ERTS_SMP
     acul = allctr->cpool.util_limit;
-#else
-    acul = 0;
-#endif
 
     if (print_to_p) {
 	char topt[21]; /* Enough for any 64-bit integer */
@@ -5132,19 +5039,15 @@ erts_alcu_info_options(Allctr_t *allctr,
     if (hpp || szp)
 	ensure_atoms_initialized(allctr);
 
-#ifdef USE_THREADS
     if (allctr->thread_safe) {
 	erts_allctr_wrapper_pre_lock();
 	erts_mtx_lock(&allctr->mutex);
     }
-#endif
     res = info_options(allctr, print_to_p, print_to_arg, hpp, szp);
-#ifdef USE_THREADS
     if (allctr->thread_safe) { 
 	erts_mtx_unlock(&allctr->mutex);
 	erts_allctr_wrapper_pre_unlock();
     }
-#endif
     return res;
 }
 
@@ -5160,9 +5063,7 @@ erts_alcu_sz_info(Allctr_t *allctr,
 		  Uint *szp)
 {
     Eterm res, mbcs, sbcs, fix = THE_NON_VALUE;
-#ifdef ERTS_SMP
     Eterm mbcs_pool;
-#endif
 
     res  = THE_NON_VALUE;
 
@@ -5177,12 +5078,10 @@ erts_alcu_sz_info(Allctr_t *allctr,
     if (hpp || szp)
 	ensure_atoms_initialized(allctr);
 
-#ifdef USE_THREADS
     if (allctr->thread_safe) {
 	erts_allctr_wrapper_pre_lock();
 	erts_mtx_lock(&allctr->mutex);
     }
-#endif
 
     ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
 
@@ -5198,23 +5097,19 @@ erts_alcu_sz_info(Allctr_t *allctr,
 	fix = sz_info_fix(allctr, internal, print_to_p, print_to_arg, hpp, szp);
     mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p,
 			    print_to_arg, hpp, szp);
-#ifdef ERTS_SMP
     if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
 	mbcs_pool = info_cpool(allctr, 1, "mbcs_pool ", print_to_p,
 			       print_to_arg, hpp, szp);
     else
 	mbcs_pool = THE_NON_VALUE; /* shut up annoying warning... */
-#endif
     sbcs = sz_info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p,
 			    print_to_arg, hpp, szp);
 
     if (hpp || szp) {
 	res = NIL;
 	add_2tup(hpp, szp, &res, am.sbcs, sbcs);
-#ifdef ERTS_SMP
 	if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
 	    add_2tup(hpp, szp, &res, am.mbcs_pool, mbcs_pool);
-#endif
 	add_2tup(hpp, szp, &res, am.mbcs, mbcs);
 	add_fix_types(allctr, internal, hpp, szp, &res, fix);
     }
@@ -5225,12 +5120,10 @@ erts_alcu_sz_info(Allctr_t *allctr,
     }
 
 
-#ifdef USE_THREADS
     if (allctr->thread_safe) {
 	erts_mtx_unlock(&allctr->mutex);
 	erts_allctr_wrapper_pre_unlock();
     }
-#endif
 
     return res;
 }
@@ -5246,9 +5139,7 @@ erts_alcu_info(Allctr_t *allctr,
 	       Uint *szp)
 {
     Eterm res, sett, mbcs, sbcs, calls, fix = THE_NON_VALUE;
-#ifdef ERTS_SMP
     Eterm mbcs_pool;
-#endif
 
     res  = THE_NON_VALUE;
 
@@ -5263,12 +5154,10 @@ erts_alcu_info(Allctr_t *allctr,
     if (hpp || szp)
 	ensure_atoms_initialized(allctr);
 
-#ifdef USE_THREADS
     if (allctr->thread_safe) {
 	erts_allctr_wrapper_pre_lock();
 	erts_mtx_lock(&allctr->mutex);
     }
-#endif
 
     ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
 
@@ -5293,13 +5182,11 @@ erts_alcu_info(Allctr_t *allctr,
 	fix = sz_info_fix(allctr, internal, print_to_p, print_to_arg, hpp, szp);
     mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p,
 			 print_to_arg, hpp, szp);
-#ifdef ERTS_SMP
     if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
 	mbcs_pool = info_cpool(allctr, 0, "mbcs_pool ", print_to_p,
 			       print_to_arg, hpp, szp);
     else
 	mbcs_pool = THE_NON_VALUE; /* shut up annoying warning... */
-#endif
     sbcs = info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p,
 			 print_to_arg, hpp, szp);
     calls = info_calls(allctr, print_to_p, print_to_arg, hpp, szp);
@@ -5309,10 +5196,8 @@ erts_alcu_info(Allctr_t *allctr,
 
 	add_2tup(hpp, szp, &res, am.calls, calls);
 	add_2tup(hpp, szp, &res, am.sbcs, sbcs);
-#ifdef ERTS_SMP
 	if (ERTS_ALC_IS_CPOOL_ENABLED(allctr))
 	    add_2tup(hpp, szp, &res, am.mbcs_pool, mbcs_pool);
-#endif
 	add_2tup(hpp, szp, &res, am.mbcs, mbcs);
 	add_fix_types(allctr, internal, hpp, szp, &res, fix);
 	add_2tup(hpp, szp, &res, am.options, sett);
@@ -5328,12 +5213,10 @@ erts_alcu_info(Allctr_t *allctr,
     }
 
 
-#ifdef USE_THREADS
     if (allctr->thread_safe) {
 	erts_mtx_unlock(&allctr->mutex);
 	erts_allctr_wrapper_pre_unlock();
     }
-#endif
 
     return res;
 }
@@ -5343,10 +5226,8 @@ void
 erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t *fi, int fisz)
 {
 
-#ifdef USE_THREADS
     if (allctr->thread_safe)
 	erts_mtx_lock(&allctr->mutex);
-#endif
 
     size->carriers = allctr->mbcs.curr.norm.mseg.size;
     size->carriers += allctr->mbcs.curr.norm.sys_alloc.size;
@@ -5356,14 +5237,12 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t *
     size->blocks = allctr->mbcs.blocks.curr.size;
     size->blocks += allctr->sbcs.blocks.curr.size;
 
-#ifdef ERTS_SMP
     if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
 	UWord csz, bsz;
 	cpool_read_stat(allctr, NULL, &csz, NULL, &bsz);
 	size->blocks += bsz;
 	size->carriers += csz;
     }
-#endif
 
     if (fi) {
 	int ix;
@@ -5385,10 +5264,8 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t *
 	}
     }
 
-#ifdef USE_THREADS
     if (allctr->thread_safe)
 	erts_mtx_unlock(&allctr->mutex);
-#endif
 }
 
 /* ----------------------------------------------------------------------- */
@@ -5436,18 +5313,13 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
 void *erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
 {
     void *res;
-#ifdef ERTS_SMP
     ASSERT(!"This is not thread safe");
-#elif defined(USE_THREADS)
-    ASSERT(erts_equal_tids(erts_main_thread, erts_thr_self()));
-#endif
     res = do_erts_alcu_alloc(type, extra, size);
     DEBUG_CHECK_ALIGNMENT(res);
     return res;
 }
 
 
-#ifdef USE_THREADS
 
 void *
 erts_alcu_alloc_ts(ErtsAlcType_t type, void *extra, Uint size)
@@ -5463,7 +5335,6 @@ erts_alcu_alloc_ts(ErtsAlcType_t type, void *extra, Uint size)
     return res;
 }
 
-#ifdef ERTS_SMP
 
 void *
 erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size)
@@ -5503,21 +5374,17 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size)
     if (pref_allctr->thread_safe)
 	erts_mtx_lock(&pref_allctr->mutex);
 
-#ifdef ERTS_SMP
     ASSERT(pref_allctr->dd.use);
     ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1);
-#endif
 
     ERTS_ALCU_DBG_CHK_THR_ACCESS(pref_allctr);
 
     res = do_erts_alcu_alloc(type, pref_allctr, size);
 
-#ifdef ERTS_SMP
     if (!res && ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1)) {
 	/* Cleaned up a bit more; try one more time... */
 	res = do_erts_alcu_alloc(type, pref_allctr, size);
     }
-#endif
 
     if (pref_allctr->thread_safe)
 	erts_mtx_unlock(&pref_allctr->mutex);
@@ -5528,9 +5395,7 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size)
     return res;
 }
 
-#endif
 
-#endif
 
 /* ------------------------------------------------------------------------- */
 
@@ -5573,7 +5438,6 @@ void erts_alcu_free(ErtsAlcType_t type, void *extra, void *p)
     do_erts_alcu_free(type, extra, p, NULL);
 }
 
-#ifdef USE_THREADS
 
 void
 erts_alcu_free_ts(ErtsAlcType_t type, void *extra, void *p)
@@ -5584,7 +5448,6 @@ erts_alcu_free_ts(ErtsAlcType_t type, void *extra, void *p)
     erts_mtx_unlock(&allctr->mutex);
 }
 
-#ifdef ERTS_SMP
 
 void
 erts_alcu_free_thr_spec(ErtsAlcType_t type, void *extra, void *p)
@@ -5634,9 +5497,7 @@ erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p)
     }
 }
 
-#endif
 
-#endif
 
 /* ------------------------------------------------------------------------- */
 
@@ -5785,7 +5646,6 @@ erts_alcu_realloc_mv(ErtsAlcType_t type, void *extra, void *p, Uint size)
 }
 
 
-#ifdef USE_THREADS
 
 void *
 erts_alcu_realloc_ts(ErtsAlcType_t type, void *extra, void *ptr, Uint size)
@@ -5824,7 +5684,6 @@ erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size)
     return res;
 }
 
-#ifdef ERTS_SMP
 
 void *
 erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra,
@@ -5905,9 +5764,7 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size,
     Allctr_t *pref_allctr, *used_allctr;
     UWord old_user_size;
     Carrier_t *busy_pcrr_p;
-#ifdef ERTS_SMP
     int retried;
-#endif
 
     if (!p)
 	return erts_alcu_alloc_thr_pref(type, extra, size);
@@ -5917,12 +5774,10 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size,
     if (pref_allctr->thread_safe)
 	erts_mtx_lock(&pref_allctr->mutex);
 
-#ifdef ERTS_SMP
     ASSERT(pref_allctr->dd.use);
     ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1);
     retried = 0;
 restart:
-#endif
 
     used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_NO,
 				  p, &old_user_size, &busy_pcrr_p);
@@ -5938,13 +5793,11 @@ restart:
 				   0,
 				   &busy_pcrr_p);
 	clear_busy_pool_carrier(used_allctr, busy_pcrr_p);
-#ifdef ERTS_SMP
 	if (!res && !retried && ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1)) {
 	    /* Cleaned up a bit more; try one more time... */
 	    retried = 1;
 	    goto restart;
 	}
-#endif	    
 	if (pref_allctr->thread_safe)
 	    erts_mtx_unlock(&pref_allctr->mutex);
     }
@@ -5999,9 +5852,7 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra,
     return realloc_thr_pref(type, extra, p, size, 1);
 }
 
-#endif
 
-#endif
 
 /* ------------------------------------------------------------------------- */
 
@@ -6022,10 +5873,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
     sys_memcpy((void *) &allctr->mseg_opt,
 	       (void *) &erts_mseg_default_opt,
 	       sizeof(ErtsMsegOpt_t));
-#ifdef ERTS_SMP
     if (init->tspec || init->tpref)
 	allctr->mseg_opt.sched_spec = 1;
-#endif /* ERTS_SMP */
 #endif /* HAVE_ERTS_MSEG */
 
     allctr->name_prefix			= init->name_prefix;
@@ -6083,7 +5932,6 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 	goto error;
     allctr->min_block_size		= UNIT_CEILING(allctr->min_block_size
 						       + sizeof(FreeBlkFtr_t));
-#ifdef ERTS_SMP
     if (init->tpref) {
 	Uint sz = ABLK_HDR_SZ;
 	sz += (init->fix ? 
@@ -6107,7 +5955,6 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
     erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0);
     allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT;
     allctr->cpool.util_limit = init->ts ? 0 : init->acul;
-#endif
 
     allctr->sbc_threshold		= init->sbct;
 #ifndef ARCH_64
@@ -6131,7 +5978,6 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 	allctr->mseg_opt.abs_shrink_th = ~((UWord) 0) / 100;
 #endif
 
-#ifdef USE_THREADS
     if (init->ts) {
 	allctr->thread_safe = 1;
 
@@ -6142,7 +5988,6 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 	allctr->debug.saved_tid = 0;
 #endif
     }
-#endif
 
     if(!allctr->get_free_block
        || !allctr->link_free_block
@@ -6155,14 +6000,12 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 
     if (allctr->mbc_header_size < sizeof(Carrier_t))
 	goto error;
-#ifdef ERTS_SMP
     allctr->dd.use = 0;
     if (init->tpref) {
 	allctr->dd.use = 1;
 	init_dd_queue(&allctr->dd.q);
 	allctr->dd.ix = init->ix;
     }
-#endif
     allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size
 					    + ABLK_HDR_SZ)
 			       - ABLK_HDR_SZ);
@@ -6216,10 +6059,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 			     | CFLG_NO_CPOOL
 			     | CFLG_MAIN_CARRIER);
 	if (!blk) {
-#ifdef USE_THREADS
 	  if (allctr->thread_safe)
 	    erts_mtx_destroy(&allctr->mutex);
-#endif
 	  erts_exit(ERTS_ABORT_EXIT,
 	    "Failed to create main carrier for %salloc\n",
 	    init->name_prefix);
@@ -6239,9 +6080,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 	    allctr->fix[i].type_size = init->fix_type_size[i];
 	    allctr->fix[i].list_size = 0;
 	    allctr->fix[i].list = NULL;
-#ifdef ERTS_SMP
 	    ASSERT(allctr->fix[i].type_size >= sizeof(ErtsAllctrFixDDBlock_t));
-#endif
 	    if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
 		allctr->fix[i].u.cpool.min_list_size = 0;
 		allctr->fix[i].u.cpool.shrink_list = 0;
@@ -6261,10 +6100,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 
  error:
 
-#ifdef USE_THREADS
     if (allctr->thread_safe)
 	erts_mtx_destroy(&allctr->mutex);
-#endif
 
     return 0;
 
@@ -6282,10 +6119,8 @@ erts_alcu_stop(Allctr_t *allctr)
     while (allctr->mbc_list.first)
 	destroy_carrier(allctr, MBC_TO_FIRST_BLK(allctr, allctr->mbc_list.first), NULL);
 
-#ifdef USE_THREADS
     if (allctr->thread_safe)
 	erts_mtx_destroy(&allctr->mutex);
-#endif
 
 }
 
@@ -6294,14 +6129,12 @@ erts_alcu_stop(Allctr_t *allctr)
 void
 erts_alcu_init(AlcUInit_t *init)
 {
-#ifdef ERTS_SMP
     int i;
     for (i = 0; i <= ERTS_ALC_A_MAX; i++) {
 	ErtsAlcCPoolData_t *sentinel = &carrier_pool[i].sentinel;
 	erts_atomic_init_nob(&sentinel->next, (erts_aint_t) sentinel);
 	erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel);
     }
-#endif
     ERTS_CT_ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */
 #if HAVE_ERTS_MSEG
     ASSERT(erts_mseg_unit_size() == ERTS_SACRR_UNIT_SZ);
@@ -6369,7 +6202,6 @@ erts_alcu_test(UWord op, UWord a1, UWord a2)
     case 0x01c: return (UWord) BLK_TO_MBC((Block_t*) a1);
     case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break;
     case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break;
-#ifdef ERTS_SMP
     case 0x01f: return (UWord) sizeof(ErtsAlcCrrPool_t);
     case 0x020:
 	SET_CARRIER_HDR((Carrier_t *) a2, 0, SCH_SYS_ALLOC|SCH_MBC, (Allctr_t *) a1);
@@ -6383,14 +6215,6 @@ erts_alcu_test(UWord op, UWord a1, UWord a2)
 	return (UWord) a2;
     case 0x023: return (UWord) cpool_is_empty((Allctr_t *) a1);
     case 0x024: return (UWord) cpool_dbg_is_in_pool((Allctr_t *) a1, (Carrier_t *) a2);
-#else
-    case 0x01f: return (UWord) 0;
-    case 0x020: return (UWord) 0;
-    case 0x021: return (UWord) 0;
-    case 0x022: return (UWord) 0;
-    case 0x023: return (UWord) 0;
-    case 0x024: return (UWord) 0;
-#endif
     case 0x025: /* UMEM2BLK_TEST*/
 #ifdef DEBUG
 # ifdef HARD_DEBUG
@@ -6448,13 +6272,9 @@ erts_alcu_verify_unused(Allctr_t *allctr)
 void
 erts_alcu_verify_unused_ts(Allctr_t *allctr)
 {
-#ifdef USE_THREADS
     erts_mtx_lock(&allctr->mutex);
-#endif
     erts_alcu_verify_unused(allctr);
-#ifdef USE_THREADS
     erts_mtx_unlock(&allctr->mutex);
-#endif
 }
 
 #ifdef DEBUG
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 73c467aa0a..0cce4c9125 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -24,10 +24,8 @@
 #define ERTS_ALCU_VSN_STR "3.0"
 
 #include "erl_alloc_types.h"
-#ifdef USE_THREADS
 #define ERL_THREADS_EMU_INTERNAL__
 #include "erl_threads.h"
-#endif
 
 #include "erl_mseg.h"
 #include "lttng-wrapper.h"
@@ -162,12 +160,10 @@ void *	erts_alcu_alloc(ErtsAlcType_t, void *, Uint);
 void *	erts_alcu_realloc(ErtsAlcType_t, void *, void *, Uint);
 void *	erts_alcu_realloc_mv(ErtsAlcType_t, void *, void *, Uint);
 void	erts_alcu_free(ErtsAlcType_t, void *, void *);
-#ifdef USE_THREADS
 void *	erts_alcu_alloc_ts(ErtsAlcType_t, void *, Uint);
 void *	erts_alcu_realloc_ts(ErtsAlcType_t, void *, void *, Uint);
 void *	erts_alcu_realloc_mv_ts(ErtsAlcType_t, void *, void *, Uint);
 void	erts_alcu_free_ts(ErtsAlcType_t, void *, void *);
-#ifdef ERTS_SMP
 void *	erts_alcu_alloc_thr_spec(ErtsAlcType_t, void *, Uint);
 void *	erts_alcu_realloc_thr_spec(ErtsAlcType_t, void *, void *, Uint);
 void *	erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t, void *, void *, Uint);
@@ -176,8 +172,6 @@ void *	erts_alcu_alloc_thr_pref(ErtsAlcType_t, void *, Uint);
 void *	erts_alcu_realloc_thr_pref(ErtsAlcType_t, void *, void *, Uint);
 void *	erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t, void *, void *, Uint);
 void	erts_alcu_free_thr_pref(ErtsAlcType_t, void *, void *);
-#endif
-#endif
 Eterm	erts_alcu_au_info_options(fmtfn_t *, void *, Uint **, Uint *);
 Eterm	erts_alcu_info_options(Allctr_t *, fmtfn_t *, void *, Uint **, Uint *);
 Eterm	erts_alcu_sz_info(Allctr_t *, int, int, fmtfn_t *, void *, Uint **, Uint *);
@@ -185,9 +179,7 @@ Eterm	erts_alcu_info(Allctr_t *, int, int, fmtfn_t *, void *, Uint **, Uint *);
 void	erts_alcu_init(AlcUInit_t *);
 void    erts_alcu_current_size(Allctr_t *, AllctrSize_t *,
 			       ErtsAlcUFixInfo_t *, int);
-#ifdef ERTS_SMP
 void    erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal *, int *);
-#endif
 erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
 
 #ifdef ARCH_32
@@ -308,7 +300,6 @@ void erts_lcnt_update_allocator_locks(int enable);
 
 typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t;
 
-#ifdef ERTS_SMP
 
 typedef struct ErtsDoubleLink_t_ {
     struct ErtsDoubleLink_t_ *next;
@@ -327,7 +318,6 @@ typedef struct {
     ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */
 } ErtsAlcCPoolData_t;
 
-#endif
 
 typedef struct Carrier_t_ Carrier_t;
 struct Carrier_t_ {
@@ -335,9 +325,7 @@ struct Carrier_t_ {
     Carrier_t *next;
     Carrier_t *prev;
     erts_smp_atomic_t allctr;
-#ifdef ERTS_SMP
     ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */
-#endif
 };
 
 #define ERTS_ALC_CARRIER_TO_ALLCTR(C) \
@@ -434,7 +422,6 @@ typedef struct {
     } while (0)
 #endif
 
-#ifdef ERTS_SMP
 
 typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t;
 
@@ -477,7 +464,6 @@ typedef struct {
     } head;
 } ErtsAllctrDDQueue_t;
 
-#endif
 
 typedef struct {
     size_t type_size;
@@ -500,7 +486,6 @@ typedef struct {
 } ErtsAlcFixList_t;
 
 struct Allctr_t_ {
-#ifdef ERTS_SMP
     struct {
 	/*
 	 * We want the queue at the beginning of
@@ -511,7 +496,6 @@ struct Allctr_t_ {
 	int		use;
 	int		ix;
     } dd;
-#endif
 
     /* Allocator name prefix */
     char *		name_prefix;
@@ -560,7 +544,6 @@ struct Allctr_t_ {
     /* Carriers */
     CarrierList_t	mbc_list;
     CarrierList_t	sbc_list;
-#ifdef ERTS_SMP
     struct {
 	/* pooled_list, traitor list and dc_list contain only
            carriers _created_ by this allocator */
@@ -579,7 +562,6 @@ struct Allctr_t_ {
 	    erts_atomic_t	no_carriers;
 	} stat;
     } cpool;
-#endif
 
     /* Main carrier (if there is one) */
     Carrier_t *		main_carrier;
@@ -622,7 +604,6 @@ struct Allctr_t_ {
     int			fix_shrink_scheduled;
     ErtsAlcFixList_t	*fix;
 
-#ifdef USE_THREADS
     /* Mutex for this allocator */
     erts_mtx_t		mutex;
     int			thread_safe;
@@ -631,7 +612,6 @@ struct Allctr_t_ {
 	Allctr_t	*next;
     } ts_list;
 
-#endif
 
     int			atoms_initialized;
 
@@ -654,13 +634,11 @@ struct Allctr_t_ {
     CarriersStats_t	mbcs;
     
 #ifdef DEBUG
-#ifdef USE_THREADS
     struct {
 	int saved_tid;
 	erts_tid_t tid;
     } debug;
 #endif
-#endif
 };
 
 int	erts_alcu_start(Allctr_t *, AllctrInit_t *);
diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index 9a93034fcb..75e88afc5e 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -34,9 +34,6 @@
 
 #define ERTS_ASYNC_PRINT_JOB 0
 
-#if !defined(ERTS_SMP) && defined(USE_THREADS) && !ERTS_USE_ASYNC_READY_Q
-#  error "Need async ready queue in non-smp case"
-#endif
 
 typedef struct _erl_async {
     DE_Handle*         hndl;   /* The DE_Handle is needed when port is gone */
@@ -46,16 +43,13 @@ typedef struct _erl_async {
     ErlDrvPDL          pdl;
     void (*async_invoke)(void*);
     void (*async_free)(void*);
-#if ERTS_USE_ASYNC_READY_Q
     Uint               sched_id;
     union {
 	ErtsThrQPrepEnQ_t *prep_enq;
 	ErtsThrQFinDeQ_t   fin_deq;
     } q;
-#endif
 } ErtsAsync;
 
-#if ERTS_USE_ASYNC_READY_Q
 
 /*
  * We can do without the enqueue mutex since it isn't needed for
@@ -94,7 +88,6 @@ typedef union {
     char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncReadyQ))];
 } ErtsAlgndAsyncReadyQ;
 
-#endif /* ERTS_USE_ASYNC_READY_Q */
 
 typedef struct {
     ErtsThrQ_t thr_q;
@@ -119,9 +112,7 @@ typedef struct {
 	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncInit))];
     } init;
     ErtsAlgndAsyncQ *queue;
-#if ERTS_USE_ASYNC_READY_Q
     ErtsAlgndAsyncReadyQ *ready_queue;
-#endif
 } ErtsAsyncData;
 
 #if defined(USE_THREADS) && defined(USE_VM_PROBES)
@@ -140,15 +131,6 @@ int erts_async_thread_suggested_stack_size; /* Initialized by erl_init.c */
 
 static ErtsAsyncData *async;
 
-#ifndef USE_THREADS
-
-void
-erts_init_async(void)
-{
-
-}
-
-#else
 
 static void *async_main(void *);
 
@@ -158,7 +140,6 @@ async_q(int i)
     return &async->queue[i].aq;
 }
 
-#if ERTS_USE_ASYNC_READY_Q
 
 static ERTS_INLINE ErtsAsyncReadyQ *
 async_ready_q(Uint sched_id)
@@ -166,16 +147,13 @@ async_ready_q(Uint sched_id)
     return &async->ready_queue[((int)sched_id)-1].arq;
 }
 
-#endif
 
 void
 erts_init_async(void)
 {
     async = NULL;
     if (erts_async_max_threads > 0) {
-#if ERTS_USE_ASYNC_READY_Q
 	ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT;
-#endif
 	erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER;
 	char *ptr, thr_name[16];
 	size_t tot_size = 0;
@@ -183,9 +161,7 @@ erts_init_async(void)
 
 	tot_size += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData));
 	tot_size += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads;
-#if ERTS_USE_ASYNC_READY_Q
 	tot_size += sizeof(ErtsAlgndAsyncReadyQ)*erts_no_schedulers;
-#endif
 
 	ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_ASYNC_DATA,
 						 tot_size);
@@ -202,7 +178,6 @@ erts_init_async(void)
 	async->queue = (ErtsAlgndAsyncQ *) ptr;
 	ptr += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads;
 
-#if ERTS_USE_ASYNC_READY_Q
 
 	qinit.live.queue = ERTS_THR_Q_LIVE_LONG;
 	qinit.live.objects = ERTS_THR_Q_LIVE_SHORT;
@@ -222,7 +197,6 @@ erts_init_async(void)
 	    erts_thr_q_initialize(&arq->thr_q, &qinit);
 	}
 
-#endif
 
 	/* Create async threads... */
 
@@ -253,7 +227,6 @@ erts_init_async(void)
     }
 }
 
-#if ERTS_USE_ASYNC_READY_Q
 
 void *
 erts_get_async_ready_queue(Uint sched_id)
@@ -261,7 +234,6 @@ erts_get_async_ready_queue(Uint sched_id)
     return (void *) async ? async_ready_q(sched_id) : NULL;
 }
 
-#endif
 
 static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q)
 {
@@ -270,10 +242,8 @@ static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q)
 #endif
 
     if (is_internal_port(a->port)) {
-#if ERTS_USE_ASYNC_READY_Q
 	ErtsAsyncReadyQ *arq = async_ready_q(a->sched_id);
 	a->q.prep_enq = erts_thr_q_prepare_enqueue(&arq->thr_q);
-#endif
 	/* make sure the driver will stay around */
 	if (a->hndl)
 	    erts_ddll_reference_referenced_driver(a->hndl);
@@ -309,10 +279,8 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q,
 					erts_tse_t *tse,
 					ErtsThrQPrepEnQ_t **prep_enq)
 {
-#if ERTS_USE_ASYNC_READY_Q
     int saved_fin_deq = 0;
     ErtsThrQFinDeQ_t fin_deq;
-#endif
 #ifdef USE_VM_PROBES
     int len;
 #endif
@@ -321,12 +289,10 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q,
 	ErtsAsync *a = (ErtsAsync *) erts_thr_q_dequeue(q);
 	if (a) {
 
-#if ERTS_USE_ASYNC_READY_Q
 	    *prep_enq = a->q.prep_enq;
 	    erts_thr_q_get_finalize_dequeue_data(q, &a->q.fin_deq);
 	    if (saved_fin_deq)
 		erts_thr_q_append_finalize_dequeue_data(&a->q.fin_deq, &fin_deq);
-#endif
 #ifdef USE_LTTNG_VM_TRACEPOINTS
             if (LTTNG_ENABLED(aio_pool_get)) {
                 lttng_decl_portbuf(port_str);
@@ -354,7 +320,6 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q,
 
 	    erts_tse_reset(tse);
 
-#if ERTS_USE_ASYNC_READY_Q
 	chk_fin_deq:
 	    if (erts_thr_q_get_finalize_dequeue_data(q, &tmp_fin_deq)) {
 		if (!saved_fin_deq) {
@@ -364,13 +329,11 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q,
 		erts_thr_q_append_finalize_dequeue_data(&fin_deq,
 							&tmp_fin_deq);
 	    }
-#endif
 
 	    switch (erts_thr_q_inspect(q, 1)) {
 	    case ERTS_THR_Q_DIRTY:
 		break;
 	    case ERTS_THR_Q_NEED_THR_PRGR:
-#ifdef ERTS_SMP
 	    {
 		ErtsThrPrgrVal prgr = erts_thr_q_need_thr_progress(q);
 		erts_thr_progress_wakeup(NULL, prgr);
@@ -382,17 +345,14 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q,
 		erts_tse_wait(tse);
 		break;
 	    }
-#endif
 	    case ERTS_THR_Q_CLEAN:
 
-#if ERTS_USE_ASYNC_READY_Q
 		if (saved_fin_deq) {
 		    if (erts_thr_q_finalize_dequeue(&fin_deq))
 			goto chk_fin_deq;
 		    else
 			saved_fin_deq = 0;
 		}
-#endif
 
 		erts_tse_wait(tse);
 		break;
@@ -408,15 +368,10 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q,
 
 static ERTS_INLINE void call_async_ready(ErtsAsync *a)
 {
-#if ERTS_USE_ASYNC_READY_Q
     Port *p = erts_id2port_sflgs(a->port,
 				 NULL,
 				 0,
 				 ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP);
-#else
-    Port *p = erts_thr_id2port_sflgs(a->port,
-				     ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP);
-#endif
     if (!p) {
 	if (a->async_free) {
             ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_PORT);
@@ -432,11 +387,7 @@ static ERTS_INLINE void call_async_ready(ErtsAsync *a)
                 ERTS_MSACC_POP_STATE();
             }
 	}
-#if ERTS_USE_ASYNC_READY_Q
 	erts_port_release(p);
-#else
-	erts_thr_port_release(p);
-#endif
     }
     if (a->pdl)
 	driver_pdl_dec_refc(a->pdl);
@@ -446,7 +397,6 @@ static ERTS_INLINE void call_async_ready(ErtsAsync *a)
 
 static ERTS_INLINE void async_reply(ErtsAsync *a, ErtsThrQPrepEnQ_t *prep_enq)
 {
-#if ERTS_USE_ASYNC_READY_Q
     ErtsAsyncReadyQ *arq;
 
 #if ERTS_ASYNC_PRINT_JOB
@@ -465,12 +415,6 @@ static ERTS_INLINE void async_reply(ErtsAsync *a, ErtsThrQPrepEnQ_t *prep_enq)
 	erts_mtx_unlock(&arq->x.data.enq_mtx);
 #endif
 
-#else /* ERTS_USE_ASYNC_READY_Q */
-
-	call_async_ready(a);
-	erts_free(ERTS_ALC_T_ASYNC, (void *) a);
-
-#endif /* ERTS_USE_ASYNC_READY_Q */
 }
 
 
@@ -486,7 +430,6 @@ static erts_tse_t *async_thread_init(ErtsAsyncQ *aq)
     erts_tse_t *tse = erts_tse_fetch();
     ERTS_DECLARE_DUMMY(Uint no);
 
-#ifdef ERTS_SMP
     ErtsThrPrgrCallbacks callbacks;
 
     callbacks.arg = (void *) tse;
@@ -495,15 +438,12 @@ static erts_tse_t *async_thread_init(ErtsAsyncQ *aq)
     callbacks.wait = NULL;
 
     erts_thr_progress_register_unmanaged_thread(&callbacks);
-#endif
 
     qinit.live.queue = ERTS_THR_Q_LIVE_LONG;
     qinit.live.objects = ERTS_THR_Q_LIVE_SHORT;
     qinit.arg = (void *) tse;
     qinit.notify = async_wakeup;
-#if ERTS_USE_ASYNC_READY_Q
     qinit.auto_finalize_dequeue = 0;
-#endif
 
     erts_thr_q_initialize(&aq->thr_q, &qinit);
 
@@ -545,12 +485,10 @@ static void *async_main(void* arg)
     return NULL;
 }
 
-#endif /* USE_THREADS */
 
 void
 erts_exit_flush_async(void)
 {
-#ifdef USE_THREADS
     int i;
     ErtsAsync a;
     a.port = NIL;
@@ -564,10 +502,8 @@ erts_exit_flush_async(void)
 	async_add(&a, async_q(i));
     for (i = 0; i < erts_async_max_threads; i++)
 	erts_thr_join(async->queue[i].aq.thr_id, NULL);
-#endif
 }
 
-#if defined(USE_THREADS) && ERTS_USE_ASYNC_READY_Q
 
 int erts_check_async_ready(void *varq)
 {
@@ -609,18 +545,15 @@ int erts_async_ready_clean(void *varq, void *val)
     case ERTS_THR_Q_DIRTY:
 	return ERTS_ASYNC_READY_DIRTY;
     case ERTS_THR_Q_NEED_THR_PRGR:
-#ifdef ERTS_SMP
 	*((ErtsThrPrgrVal *) val)
 	    = erts_thr_q_need_thr_progress(&arq->thr_q);
 	return ERTS_ASYNC_READY_NEED_THR_PRGR;
-#endif
     case ERTS_THR_Q_CLEAN:
 	break;
     }
     return ERTS_ASYNC_READY_CLEAN;
 }
 
-#endif
 
 /*
 ** Generate a fair async key prom an ErlDrvPort
@@ -658,16 +591,12 @@ long driver_async(ErlDrvPort ix, unsigned int* key,
     Port* prt;
     long id;
     unsigned int qix;
-#if ERTS_USE_ASYNC_READY_Q
     Uint sched_id;
     ERTS_MSACC_PUSH_STATE();
 
     sched_id = erts_get_scheduler_id();
     if (!sched_id)
 	sched_id = 1;
-#else
-    ERTS_MSACC_PUSH_STATE();
-#endif
 
     prt = erts_drvport2port(ix);
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
@@ -677,9 +606,7 @@ long driver_async(ErlDrvPort ix, unsigned int* key,
 
     a = (ErtsAsync*) erts_alloc(ERTS_ALC_T_ASYNC, sizeof(ErtsAsync));
 
-#if ERTS_USE_ASYNC_READY_Q
     a->sched_id = sched_id;
-#endif
     a->hndl = (DE_Handle*)prt->drv_ptr->handle;
     a->port = prt->common.id;
     a->pdl = NULL;
@@ -709,7 +636,6 @@ long driver_async(ErlDrvPort ix, unsigned int* key,
 	    (*key % erts_async_max_threads) : 0;
 	*key = qix;
     }
-#ifdef USE_THREADS
     if (erts_async_max_threads > 0) {
 	if (prt->port_data_lock) {
 	    driver_pdl_inc_refc(prt->port_data_lock);
@@ -718,7 +644,6 @@ long driver_async(ErlDrvPort ix, unsigned int* key,
 	async_add(a, async_q(qix));
 	return id;
     }
-#endif
     
     ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_PORT);
     (*a->async_invoke)(a->async_data);
diff --git a/erts/emulator/beam/erl_async.h b/erts/emulator/beam/erl_async.h
index 4b470e7679..76fc61b560 100644
--- a/erts/emulator/beam/erl_async.h
+++ b/erts/emulator/beam/erl_async.h
@@ -28,38 +28,20 @@ extern int erts_async_max_threads;
 extern int erts_async_thread_suggested_stack_size;
 
 
-#ifdef ERTS_SMP
 /*
  * With smp support we can choose to have, or not to
  * have an async ready queue.
  */
 #define ERTS_USE_ASYNC_READY_Q 1
-#endif
 
-#ifndef ERTS_SMP
-/* In non-smp case we *need* the async ready queue */
-#  undef ERTS_USE_ASYNC_READY_Q
-#  define ERTS_USE_ASYNC_READY_Q 1
-#endif
 
-#ifndef ERTS_USE_ASYNC_READY_Q
-#  define ERTS_USE_ASYNC_READY_Q 0
-#endif
 
-#ifndef USE_THREADS
-#  undef ERTS_USE_ASYNC_READY_Q
-#  define ERTS_USE_ASYNC_READY_Q 0
-#endif /* !USE_THREADS */
-#if ERTS_USE_ASYNC_READY_Q
 int erts_check_async_ready(void *);
 int erts_async_ready_clean(void *, void *);
 void *erts_get_async_ready_queue(Uint sched_id);
 #define ERTS_ASYNC_READY_CLEAN 0
 #define ERTS_ASYNC_READY_DIRTY 1
-#ifdef ERTS_SMP
 #define ERTS_ASYNC_READY_NEED_THR_PRGR 2
-#endif
-#endif /* ERTS_USE_ASYNC_READY_Q */
 
 void erts_init_async(void);
 void erts_exit_flush_async(void);
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index e9bfb39035..cea8b2200c 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -50,11 +50,7 @@
 #include "dtrace-wrapper.h"
 #include "lttng-wrapper.h"
 
-#ifdef ERTS_SMP
 #define DDLL_SMP 1
-#else
-#define DDLL_SMP 0
-#endif
 
 
 /*
@@ -280,10 +276,8 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
     path[path_len++] = '/';
     sys_strcpy(path+path_len,name);
 
-#if DDLL_SMP
     erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
     lock_drv_list();
-#endif
     if ((drv = lookup_driver(name)) != NULL) {
 	if (drv->handle == NULL) {
 	    /* static_driver */
@@ -404,24 +398,18 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
 	erts_ddll_reference_driver(dh);
 	ASSERT(dh->status == ERL_DE_RELOAD);
 	dh->status = ERL_DE_FORCE_RELOAD;
-#if DDLL_SMP
 	unlock_drv_list();
-#endif
 	kill_ports_driver_unloaded(dh);
 	/* Dereference, eventually causing driver destruction */
-#if DDLL_SMP
 	lock_drv_list(); 
-#endif
 	erts_ddll_dereference_driver(dh);
     } 
 
-#if DDLL_SMP
     erts_ddll_reference_driver(dh);
     unlock_drv_list();
     erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     lock_drv_list();
     erts_ddll_dereference_driver(dh);
-#endif
 
     BIF_P->flags |= F_USING_DDLL;
     if (monitor) {
@@ -432,18 +420,14 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
 	hp = HAlloc(BIF_P, 3);
 	t = TUPLE2(hp, am_ok, ok_term);
     }
-#if DDLL_SMP
     unlock_drv_list();
-#endif
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path);
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
     BIF_RET(t);
  soft_error:
-#if DDLL_SMP
     unlock_drv_list();
     erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
-#endif
     if (do_build_load_error) {
 	soft_error_term = build_load_error(BIF_P, build_this_load_error);
     }
@@ -551,9 +535,7 @@ Eterm erl_ddll_try_unload_2(BIF_ALIST_2)
 	goto error;
     }
 
-#if DDLL_SMP
     lock_drv_list();
-#endif
 
     if ((drv = lookup_driver(name)) == NULL) {
 	soft_error_term = am_not_loaded;
@@ -608,23 +590,17 @@ done:
 	/* Avoid closing the driver by referencing it */
 	erts_ddll_reference_driver(dh);
 	dh->status = ERL_DE_FORCE_UNLOAD;
-#if DDLL_SMP
 	unlock_drv_list();
-#endif
 	kill_ports_driver_unloaded(dh);
-#if DDLL_SMP
 	lock_drv_list(); 
-#endif
 	erts_ddll_dereference_driver(dh);
     } 
 
-#if DDLL_SMP
     erts_ddll_reference_driver(dh);
     unlock_drv_list();
     erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     lock_drv_list();
     erts_ddll_dereference_driver(dh);
-#endif
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
     BIF_P->flags |= F_USING_DDLL;
     if (monitor > 0) {
@@ -638,15 +614,11 @@ done:
     if (kill_ports > 1) {
 	ERTS_BIF_CHK_EXITED(BIF_P); /* May be exited by port killing */
     }
-#if DDLL_SMP
     unlock_drv_list();
-#endif
     BIF_RET(t);
  
 soft_error:
-#if DDLL_SMP
     unlock_drv_list();
-#endif
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
     erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     hp = HAlloc(BIF_P, 3);
@@ -697,9 +669,7 @@ BIF_RETTYPE erl_ddll_loaded_drivers_0(BIF_ALIST_0)
     int need = 3;
     Eterm res = NIL;
     erts_driver_t *drv;
-#if DDLL_SMP
     lock_drv_list();
-#endif
     for (drv = driver_list; drv; drv = drv->next) {
 	need += sys_strlen(drv->name)*2+2;
     }
@@ -712,9 +682,7 @@ BIF_RETTYPE erl_ddll_loaded_drivers_0(BIF_ALIST_0)
     }
     res = TUPLE2(hp,am_ok,res);
     /* hp += 3 */
-#if DDLL_SMP
     unlock_drv_list();
-#endif
     BIF_RET(res);
 }
 
@@ -736,9 +704,7 @@ BIF_RETTYPE erl_ddll_info_2(BIF_ALIST_2)
     Eterm *hp;
     int i;
     Uint filter;
-#if DDLL_SMP
     int have_lock = 0;
-#endif
 
     if ((name = pick_list_or_atom(name_term)) == NULL) {
 	goto error;
@@ -748,10 +714,8 @@ BIF_RETTYPE erl_ddll_info_2(BIF_ALIST_2)
 	goto error;
     }
 
-#if DDLL_SMP 
     lock_drv_list();
     have_lock = 1;
-#endif
     if ((drv = lookup_driver(name)) == NULL) {
 	goto error;
     }
@@ -827,9 +791,7 @@ BIF_RETTYPE erl_ddll_info_2(BIF_ALIST_2)
 	hp += 2;
     }    
  done:    
-#if DDLL_SMP
     unlock_drv_list();
-#endif
     if (pei)
 	erts_free(ERTS_ALC_T_DDLL_TMP_BUF, pei);
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
@@ -838,11 +800,9 @@ BIF_RETTYPE erl_ddll_info_2(BIF_ALIST_2)
     if (name != NULL) {
 	erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
     }
-#if DDLL_SMP
     if (have_lock) {
 	unlock_drv_list();
     }
-#endif
     BIF_ERROR(p,BADARG);
 }
 
@@ -899,13 +859,9 @@ BIF_RETTYPE erl_ddll_format_error_int_1(BIF_ALIST_1)
 	if (errdesc_to_code(code_term,&errint) != 0) {
 	    goto error;
 	}
-#if DDLL_SMP 
 	lock_drv_list();
-#endif
 	errstring = erts_ddll_error(errint);
-#if DDLL_SMP
 	unlock_drv_list();
-#endif
 	break;
     }
     if (errstring == NULL) {
@@ -1045,13 +1001,9 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks)
 		    DE_Handle *dh = drv->handle;
 		    erts_ddll_reference_driver(dh);
 		    dh->status = ERL_DE_FORCE_UNLOAD;
-#if DDLL_SMP
 		    unlock_drv_list();
-#endif
 		    kill_ports_driver_unloaded(dh);
-#if DDLL_SMP
 		    lock_drv_list(); /* Needed for future list operations */
-#endif
 		    drv = drv->next; /* before allowing destruction */
 		    erts_ddll_dereference_driver(dh);
 		} else {
@@ -1282,9 +1234,7 @@ static Eterm notify_when_loaded(Process *p, Eterm name_term, char *name, ErtsPro
     erts_driver_t *drv;
 
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & plocks);
-#if DDLL_SMP 
     lock_drv_list();
-#endif
     if ((drv = lookup_driver(name)) == NULL) {
 	immediate_tag = am_unloaded;
 	immediate_type = am_DOWN;
@@ -1314,20 +1264,14 @@ static Eterm notify_when_loaded(Process *p, Eterm name_term, char *name, ErtsPro
     }
     p->flags |= F_USING_DDLL;
     r = add_monitor(p, drv->handle, ERL_DE_PROC_AWAIT_LOAD);
-#if DDLL_SMP
     unlock_drv_list();
-#endif
     BIF_RET(r);
  immediate:
     r =  erts_make_ref(p);
-#if DDLL_SMP 
     erts_smp_proc_unlock(p, plocks);
-#endif
     notify_proc(p, r, name_term, immediate_type, immediate_tag, 0);
-#if DDLL_SMP
     unlock_drv_list();
     erts_smp_proc_lock(p, plocks);
-#endif
     BIF_RET(r);
 }
 
@@ -1339,9 +1283,7 @@ static Eterm notify_when_unloaded(Process *p, Eterm name_term, char *name, ErtsP
     erts_driver_t *drv;
 
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & plocks);
-#if DDLL_SMP 
     lock_drv_list();
-#endif
     if ((drv = lookup_driver(name)) == NULL) {
 	immediate_tag = am_unloaded;
 	immediate_type = am_DOWN;
@@ -1355,20 +1297,14 @@ static Eterm notify_when_unloaded(Process *p, Eterm name_term, char *name, ErtsP
 
     p->flags |= F_USING_DDLL;
     r = add_monitor(p, drv->handle, flag);
-#if DDLL_SMP
     unlock_drv_list();
-#endif
     BIF_RET(r);
  immediate:
     r =  erts_make_ref(p);
-#if DDLL_SMP 
     erts_smp_proc_unlock(p, plocks);
-#endif
     notify_proc(p, r, name_term, immediate_type, immediate_tag, 0);
-#if DDLL_SMP
     unlock_drv_list();
     erts_smp_proc_lock(p, plocks);
-#endif
     BIF_RET(r);
 }
 
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index e5d7efcc72..df02554d83 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -87,10 +87,7 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
 #ifdef ARCH_64
 				     " [64-bit]"
 #endif
-#ifdef ERTS_SMP
 				     " [smp:%beu:%beu]"
-#endif
-#ifdef USE_THREADS
 #if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
 				     " [ds:%beu:%beu:%beu]"
 #endif
@@ -98,7 +95,6 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
 				     " [dirty-schedulers-TEST]"
 #endif
 				     " [async-threads:%d]"
-#endif
 #ifdef HIPE
 				     " [hipe]"
 #endif	
@@ -353,14 +349,12 @@ erts_print_system_version(fmtfn_t to, void *arg, Process *c_p)
     char *rc_str = "";
     char rc_buf[100];
     char *ov = otp_version;
-#ifdef ERTS_SMP
     Uint total, online, active;
     Uint dirty_cpu, dirty_cpu_onln, dirty_io;
 
     erts_schedulers_state(&total, &online, &active,
 			  &dirty_cpu, &dirty_cpu_onln, NULL,
 			  &dirty_io, NULL);
-#endif
     for (i = 0; i < sizeof(otp_version)-4; i++) {
 	if (ov[i] == '-' && ov[i+1] == 'r' && ov[i+2] == 'c')
 	    rc = atoi(&ov[i+3]);
@@ -375,15 +369,11 @@ erts_print_system_version(fmtfn_t to, void *arg, Process *c_p)
     }
     return erts_print(to, arg, erts_system_version,
 		      rc_str
-#ifdef ERTS_SMP
 		      , total, online
 #ifdef ERTS_DIRTY_SCHEDULERS
 		      , dirty_cpu, dirty_cpu_onln, dirty_io
 #endif
-#endif
-#ifdef USE_THREADS
 		      , erts_async_max_threads
-#endif
 #ifdef ERTS_ENABLE_KERNEL_POLL
 		      , erts_use_kernel_poll ? "true" : "false"
 #endif
@@ -763,7 +753,6 @@ process_info_init(void)
 static ERTS_INLINE Process *
 pi_pid2proc(Process *c_p, Eterm pid, ErtsProcLocks info_locks)
 {
-#ifdef ERTS_SMP
     /*
      * If the main lock is needed, we use erts_pid2proc_not_running()
      * instead of erts_pid2proc() for two reasons:
@@ -781,7 +770,6 @@ pi_pid2proc(Process *c_p, Eterm pid, ErtsProcLocks info_locks)
 	return erts_pid2proc_not_running(c_p, ERTS_PROC_LOCK_MAIN,
 					 pid, info_locks);
     else
-#endif
 	return erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN,
 			     pid, info_locks);
 }
@@ -1086,12 +1074,10 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2)
     }
     ASSERT(is_value(res));
 
-#ifdef ERTS_SMP
     if (BIF_P == rp)
 	info_locks &= ~ERTS_PROC_LOCK_MAIN;
     if (rp && info_locks)
 	erts_smp_proc_unlock(rp, info_locks);
-#endif
 
     ASSERT(!(BIF_P->flags & F_P2PNR_RESCHED));
     BIF_RET(res);
@@ -2144,9 +2130,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	ASSERT(erts_compat_rel > 0);
 	BIF_RET(make_small(erts_compat_rel));
     } else if (BIF_ARG_1 == am_multi_scheduling) {
-#ifndef ERTS_SMP
-	BIF_RET(am_disabled);
-#else
 #ifndef ERTS_DIRTY_SCHEDULERS
 	if (erts_no_schedulers == 1)
 	    BIF_RET(am_disabled);
@@ -2160,7 +2143,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 		       ? am_blocked
 		       : am_blocked_normal));
 	}
-#endif
     } else if (BIF_ARG_1 == am_build_type) {
 #if defined(DEBUG)
 	ERTS_DECL_AM(debug);
@@ -2392,16 +2374,10 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(erts_allocator_options((void *) BIF_P));
     }
     else if (BIF_ARG_1 == am_thread_pool_size) {
-#ifdef USE_THREADS
 	extern int erts_async_max_threads;
-#endif
 	int n;
 	
-#ifdef USE_THREADS
 	n = erts_async_max_threads;
-#else
-	n = 0;
-#endif
 	BIF_RET(make_small(n));
     }
     else if (BIF_ARG_1 == am_alloc_util_allocators) {
@@ -2549,11 +2525,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(res);
 #endif
     } else if (BIF_ARG_1 == am_threads) {
-#ifdef USE_THREADS
 	return am_true;
-#else
-	return am_false;
-#endif
     } else if (BIF_ARG_1 == am_creation) {
 	return make_small(erts_this_node->creation);
     } else if (BIF_ARG_1 == am_break_ignored) {
@@ -2612,11 +2584,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	hp = HAlloc(BIF_P, 2*n);
 	BIF_RET(buf_to_intlist(&hp, buf, n, NIL));
     } else if (ERTS_IS_ATOM_STR("smp_support", BIF_ARG_1)) {
-#ifdef ERTS_SMP
 	BIF_RET(am_true);
-#else
-	BIF_RET(am_false);
-#endif
     } else if (ERTS_IS_ATOM_STR("scheduler_bind_type", BIF_ARG_1)) {
 	BIF_RET(erts_bound_schedulers_term(BIF_P));
     } else if (ERTS_IS_ATOM_STR("scheduler_bindings", BIF_ARG_1)) {
@@ -2628,11 +2596,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	res = make_small(erts_no_schedulers);
 	BIF_RET(res);
     } else if (ERTS_IS_ATOM_STR("schedulers_state", BIF_ARG_1)) {
-#ifndef ERTS_SMP
-	Eterm *hp = HAlloc(BIF_P, 4);
-	res = TUPLE3(hp, make_small(1), make_small(1), make_small(1));
-	BIF_RET(res);
-#else
 	Eterm *hp;
 	Uint total, online, active;
 	erts_schedulers_state(&total, &online, &active,
@@ -2643,13 +2606,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 		     make_small(online),
 		     make_small(active));
 	BIF_RET(res);
-#endif
     } else if (ERTS_IS_ATOM_STR("schedulers_state", BIF_ARG_1)) {
-#ifndef ERTS_SMP
-	Eterm *hp = HAlloc(BIF_P, 4);
-	res = TUPLE3(hp, make_small(1), make_small(1), make_small(1));
-	BIF_RET(res);
-#else
 	Eterm *hp;
 	Uint total, online, active;
 	erts_schedulers_state(&total, &online, &active,
@@ -2660,19 +2617,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 		     make_small(online),
 		     make_small(active));
 	BIF_RET(res);
-#endif
     } else if (ERTS_IS_ATOM_STR("all_schedulers_state", BIF_ARG_1)) {
-#ifndef ERTS_SMP
-	Eterm *hp = HAlloc(BIF_P, 2+5);
-	res = CONS(hp+5,
-		   TUPLE4(hp,
-			  am_normal,
-			  make_small(1),
-			  make_small(1),
-			  make_small(1)),
-		   NIL);
-	BIF_RET(res);
-#else
 	Eterm *hp, tpl;
 	Uint sz, total, online, active,
 	    dirty_cpu_total, dirty_cpu_online, dirty_cpu_active,
@@ -2718,23 +2663,14 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	hp += 5;
 	res = CONS(hp, tpl, res);
 	BIF_RET(res);
-#endif
     } else if (ERTS_IS_ATOM_STR("schedulers_online", BIF_ARG_1)) {
-#ifndef ERTS_SMP
-	BIF_RET(make_small(1));
-#else
 	Uint online;
 	erts_schedulers_state(NULL, &online, NULL, NULL, NULL, NULL, NULL, NULL);
 	BIF_RET(make_small(online));
-#endif
     } else if (ERTS_IS_ATOM_STR("schedulers_active", BIF_ARG_1)) {
-#ifndef ERTS_SMP
-	BIF_RET(make_small(1));
-#else
 	Uint active;
 	erts_schedulers_state(NULL, NULL, &active, NULL, NULL, NULL, NULL, NULL);
 	BIF_RET(make_small(active));
-#endif
     } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers", BIF_ARG_1)) {
 	Uint dirty_cpu;
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -2804,23 +2740,15 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     } else if (ERTS_IS_ATOM_STR("check_io", BIF_ARG_1)) {
 	BIF_RET(erts_check_io_info(BIF_P));
     } else if (ERTS_IS_ATOM_STR("multi_scheduling_blockers", BIF_ARG_1)) {
-#ifndef ERTS_SMP
-	BIF_RET(NIL);
-#else
 	if (erts_no_schedulers == 1)
 	    BIF_RET(NIL);
 	else
 	    BIF_RET(erts_multi_scheduling_blockers(BIF_P, 0));
-#endif
     } else if (ERTS_IS_ATOM_STR("normal_multi_scheduling_blockers", BIF_ARG_1)) {
-#ifndef ERTS_SMP
-	BIF_RET(NIL);
-#else
 	if (erts_no_schedulers == 1)
 	    BIF_RET(NIL);
 	else
 	    BIF_RET(erts_multi_scheduling_blockers(BIF_P, 1));
-#endif
     } else if (ERTS_IS_ATOM_STR("modified_timing_level", BIF_ARG_1)) {
 	BIF_RET(ERTS_USE_MODIFIED_TIMING()
 		? make_small(erts_modified_timing_level)
@@ -2883,12 +2811,10 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(am_false);
 #endif	
     }
-#ifdef ERTS_SMP
     else if (ERTS_IS_ATOM_STR("thread_progress", BIF_ARG_1)) {
 	erts_thr_progress_dbg_print_state();
 	BIF_RET(am_true);
     }
-#endif
     else if (BIF_ARG_1 == am_message_queue_data) {
 	switch (erts_default_spo_flags & (SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ)) {
 	case SPO_OFF_HEAP_MSGQ:
@@ -3171,9 +3097,6 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt,
     }
     else if (ERTS_IS_ATOM_STR("locking", item)) {
 	if (hpp) {
-#ifndef ERTS_SMP
-	    res = am_false;
-#else
 	    if (erts_atomic32_read_nob(&prt->state)
 		& ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) {
 		DECL_AM(port_level);
@@ -3187,7 +3110,6 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt,
 			 & ERL_DRV_FLAG_USE_PORT_LOCKING));
 		res = AM_driver_level;
 	    }
-#endif
 	}
 	if (szp) {
 	    res = am_true;
@@ -3425,11 +3347,9 @@ BIF_RETTYPE process_display_2(BIF_ALIST_2)
 				   2);
    }
    erts_stack_dump(ERTS_PRINT_STDERR, NULL, rp);
-#ifdef ERTS_SMP
    erts_smp_proc_unlock(rp, (BIF_P == rp
 			     ? ERTS_PROC_LOCKS_ALL_MINOR
 			     : ERTS_PROC_LOCKS_ALL));
-#endif
    BIF_RET(am_true);
 }
 
@@ -4239,10 +4159,8 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 			BIF_RET(AM_dead);
 		    }
 
-#ifdef ERTS_SMP
 		    if (BIF_P == rp)
 			rp_locks |= ERTS_PROC_LOCK_MAIN;
-#endif
 		    xres = erts_send_exit_signal(NULL, /* NULL in order to
 							  force a pending exit
 							  when we send to our
@@ -4254,10 +4172,8 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 						 NIL,
 						 NULL,
 						 0);
-#ifdef ERTS_SMP
 		    if (BIF_P == rp)
 			rp_locks &= ~ERTS_PROC_LOCK_MAIN;
-#endif
 		    erts_smp_proc_unlock(rp, rp_locks);
 		    if (xres > 1) {
 			DECL_AM(message);
@@ -4364,7 +4280,6 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	    }
 	}
 	else if (ERTS_IS_ATOM_STR("not_running_optimization", BIF_ARG_1)) {
-#ifdef ERTS_SMP
 	    int old_use_opt, use_opt;
 	    switch (BIF_ARG_2) {
 	    case am_true:
@@ -4384,9 +4299,6 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	    erts_smp_thr_progress_unblock();
 	    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	    BIF_RET(old_use_opt ? am_true : am_false);
-#else
-	    BIF_ERROR(BIF_P,  EXC_NOTSUP);
-#endif
 	}
 	else if (ERTS_IS_ATOM_STR("wait", BIF_ARG_1)) {
 	    if (ERTS_IS_ATOM_STR("deallocations", BIF_ARG_2)) {
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index ff03151619..106f18b747 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -273,10 +273,8 @@ BIF_RETTYPE erts_internal_port_call_3(BIF_ALIST_3)
 
     state = erts_smp_atomic32_read_acqb(&BIF_P->state);
     if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
-#ifdef ERTS_SMP
 	if (state & ERTS_PSFLG_PENDING_EXIT)
 	    erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
-#endif
 	ERTS_BIF_EXITED(BIF_P);
     }
 
@@ -323,10 +321,8 @@ BIF_RETTYPE erts_internal_port_control_3(BIF_ALIST_3)
 
     state = erts_smp_atomic32_read_acqb(&BIF_P->state);
     if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
-#ifdef ERTS_SMP
 	if (state & ERTS_PSFLG_PENDING_EXIT)
 	    erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
-#endif
 	ERTS_BIF_EXITED(BIF_P);
     }
 
@@ -511,7 +507,6 @@ cleanup_old_port_data(erts_aint_t data)
 	ASSERT(is_immed((Eterm) data));
     }
     else {
-#ifdef ERTS_SMP
 	ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data;
 	size_t size;
 	ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
@@ -520,9 +515,6 @@ cleanup_old_port_data(erts_aint_t data)
 						(void *) pdhp,
 						&pdhp->later_op,
 						size);
-#else
-	free_port_data_heap((void *) data);
-#endif
     }
 }
 
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index ad124fd979..bc819505e7 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -66,11 +66,7 @@ static void erts_erts_pcre_stack_free(void *ptr) {
 
 #define ERTS_PCRE_STACK_MARGIN (10*1024)
 
-#ifdef ERTS_SMP
 #  define ERTS_STACK_LIMIT ((char *) ethr_get_stacklimit())
-#else
-#  define ERTS_STACK_LIMIT ((char *) erts_scheduler_stack_limit)
-#endif
 
 static int
 stack_guard_downwards(void)
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 45159c4392..c476e18c33 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -60,10 +60,8 @@ static struct {			/* Protected by code write permission */
     int local;
     BpFunctions f;		/* Local functions */
     BpFunctions e;		/* Export entries */
-#ifdef ERTS_SMP
     Process* stager;
     ErtsThrPrgrLaterOp lop;
-#endif
 } finish_bp;
 
 static Eterm
@@ -71,9 +69,7 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist);
 static int
 erts_set_tracing_event_pattern(Eterm event, Binary*, int on);
 
-#ifdef ERTS_SMP
 static void smp_bp_finisher(void* arg);
-#endif
 static BIF_RETTYPE
 system_monitor(Process *p, Eterm monitor_pid, Eterm list);
 
@@ -345,7 +341,6 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 
     ERTS_TRACER_CLEAR(&meta_tracer);
 
-#ifdef ERTS_SMP
     if (finish_bp.current >= 0) {
 	ASSERT(matches >= 0);
 	ASSERT(finish_bp.stager == NULL);
@@ -355,7 +350,6 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 	erts_suspend(p, ERTS_PROC_LOCK_MAIN, NULL);
 	ERTS_BIF_YIELD_RETURN(p, make_small(matches));
     }
-#endif
 
     erts_release_code_write_permission();
 
@@ -367,7 +361,6 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
     }
 }
 
-#ifdef ERTS_SMP
 static void smp_bp_finisher(void* null)
 {
     if (erts_finish_breakpointing()) { /* Not done */
@@ -388,7 +381,6 @@ static void smp_bp_finisher(void* null)
 	erts_proc_dec_refc(p);
     }
 }
-#endif /* ERTS_SMP */
 
 void
 erts_get_default_trace_pattern(int *trace_pattern_is_on,
@@ -543,9 +535,7 @@ Eterm erts_internal_trace_3(BIF_ALIST_3)
     int matches = 0;
     Uint mask = 0;
     int cpu_ts = 0;
-#ifdef ERTS_SMP
     int system_blocked = 0;
-#endif
 
     if (! erts_trace_flags(list, &mask, &tracer, &cpu_ts)) {
 	BIF_ERROR(p, BADARG);
@@ -699,11 +689,9 @@ Eterm erts_internal_trace_3(BIF_ALIST_3)
                     mods = 1;
             }
 
-#ifdef ERTS_SMP
 	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
 	    erts_smp_thr_progress_block();
 	    system_blocked = 1;
-#endif
 
 	    ok = 1;
 	    if (procs || mods) {
@@ -766,12 +754,10 @@ Eterm erts_internal_trace_3(BIF_ALIST_3)
 	    goto error;
     }
 
-#ifdef ERTS_SMP
     if (system_blocked) {
 	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     }
-#endif
     erts_release_code_write_permission();
     ERTS_TRACER_CLEAR(&tracer);
 
@@ -785,12 +771,10 @@ Eterm erts_internal_trace_3(BIF_ALIST_3)
 
     ERTS_TRACER_CLEAR(&tracer);
 
-#ifdef ERTS_SMP
     if (system_blocked) {
 	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     }
-#endif
     erts_release_code_write_permission();
 
     BIF_ERROR(p, BADARG);
@@ -1055,12 +1039,10 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
     mfa[1] = tp[2];
     mfa[2] = signed_val(tp[3]);
 
-#ifdef ERTS_SMP
     if ( (key == am_call_time) || (key == am_all)) {
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
 	erts_smp_thr_progress_block();
     }
-#endif
 #ifdef ERTS_DIRTY_SCHEDULERS
     erts_smp_mtx_lock(&erts_dirty_bp_ix_mtx);
 #endif
@@ -1071,12 +1053,10 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 #ifdef ERTS_DIRTY_SCHEDULERS
     erts_smp_mtx_unlock(&erts_dirty_bp_ix_mtx);
 #endif
-#ifdef ERTS_SMP
     if ( (key == am_call_time) || (key == am_all)) {
 	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     }
-#endif
 
     switch (r) {
     case FUNC_TRACE_NOEXIST:
@@ -1526,17 +1506,13 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
     finish_bp.install = on;
     finish_bp.local = flags.breakpoint;
 
-#ifdef ERTS_SMP
     if (is_blocking) {
 	ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
-#endif
 	while (erts_finish_breakpointing()) {
 	    /* Empty loop body */
 	}
-#ifdef ERTS_SMP
 	finish_bp.current = -1;
     }
-#endif
 
     if (flags.breakpoint) {
 	matches += finish_bp.f.matched;
@@ -1571,11 +1547,6 @@ erts_set_tracing_event_pattern(Eterm event, Binary* match_spec, int on)
     finish_bp.f.matched = 0;
     finish_bp.f.matching = NULL;
 
-#ifndef ERTS_SMP
-    while (erts_finish_breakpointing()) {
-        /* Empty loop body */
-    }
-#endif
     return 1;
 }
 
@@ -2015,24 +1986,20 @@ BIF_RETTYPE seq_trace_print_2(BIF_ALIST_2)
 }
 
 void erts_system_monitor_clear(Process *c_p) {
-#ifdef ERTS_SMP
     if (c_p) {
 	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
 	erts_smp_thr_progress_block();
     }
-#endif
     erts_set_system_monitor(NIL);
     erts_system_monitor_long_gc = 0;
     erts_system_monitor_long_schedule = 0;
     erts_system_monitor_large_heap = 0;
     erts_system_monitor_flags.busy_port = 0;
     erts_system_monitor_flags.busy_dist_port = 0;
-#ifdef ERTS_SMP
     if (c_p) {
 	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
     }
-#endif
 }
 
 
@@ -2200,23 +2167,19 @@ system_monitor(Process *p, Eterm monitor_pid, Eterm list)
 /* Begin: Trace for System Profiling */
 
 void erts_system_profile_clear(Process *c_p) {
-#ifdef ERTS_SMP
     if (c_p) {
 	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
 	erts_smp_thr_progress_block();
     }
-#endif
     erts_set_system_profile(NIL);
     erts_system_profile_flags.scheduler = 0;
     erts_system_profile_flags.runnable_procs = 0;
     erts_system_profile_flags.runnable_ports = 0;
     erts_system_profile_flags.exclusive = 0;
-#ifdef ERTS_SMP
     if (c_p) {
 	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
     }
-#endif
 }
 
 static Eterm system_profile_get(Process *p) {
@@ -2378,26 +2341,15 @@ reply_trace_delivered_all(void *vtdarp)
         Process *rp = tdarp->proc;
         Eterm *hp = NULL;
         ErlOffHeap *ohp;
-#ifdef ERTS_SMP
         ErlHeapFragment *bp;
         bp = new_message_buffer(4 + NC_HEAP_SIZE(tdarp->ref));
         hp = &bp->mem[0];
         ohp = &bp->off_heap;
-#else
-        ErtsProcLocks rp_locks = 0;
-        ErtsMessage *mp;
-        mp = erts_alloc_message_heap(
-            rp, &rp_locks, 4 + NC_HEAP_SIZE(tdarp->ref), &hp, &ohp);
-#endif
 
         ref_copy = STORE_NC(&hp, ohp, tdarp->ref);
         msg = TUPLE3(hp, am_trace_delivered, tdarp->target, ref_copy);
 
-#ifdef ERTS_SMP
         erts_send_sys_msg_proc(rp->common.id, rp->common.id, msg, bp);
-#else
-        erts_queue_message(rp, rp_locks, mp, msg, am_system);
-#endif
 
 	erts_free(ERTS_ALC_T_MISC_AUX_WORK, vtdarp);
         erts_proc_dec_refc(rp);
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 71c64997c1..e52b5b4bc5 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -64,15 +64,7 @@
 
 static byte get_bit(byte b, size_t a_offs); 
 
-#if defined(ERTS_SMP)
 /* the state resides in the current process' scheduler data */
-#elif defined(ERL_BITS_REENTRANT)
-/* reentrant API but with a hidden single global state, for testing only */
-struct erl_bits_state ErlBitsState_;
-#else
-/* non-reentrant API with a single global state */
-struct erl_bits_state ErlBitsState;
-#endif
 
 #define byte_buf	(ErlBitsState.byte_buf_)
 #define byte_buf_len	(ErlBitsState.byte_buf_len_)
@@ -85,9 +77,6 @@ erts_bits_bufs_size(void)
     return (Uint) erts_smp_atomic_read_nob(&bits_bufs_size);
 }
 
-#if !defined(ERTS_SMP)
-static
-#endif
 void
 erts_bits_init_state(ERL_BITS_PROTO_0)
 {
@@ -97,13 +86,11 @@ erts_bits_init_state(ERL_BITS_PROTO_0)
     erts_bin_offset = 0;
 }
 
-#if defined(ERTS_SMP)
 void
 erts_bits_destroy_state(ERL_BITS_PROTO_0)
 {
     erts_free(ERTS_ALC_T_BITS_BUF, byte_buf);
 }
-#endif
 
 void
 erts_init_bits(void)
@@ -114,12 +101,7 @@ erts_init_bits(void)
                 == offsetof(Binary,orig_bytes));
 
     erts_smp_atomic_init_nob(&bits_bufs_size, 0);
-#if defined(ERTS_SMP)
     /* erl_process.c calls erts_bits_init_state() on all state instances */
-#else
-    ERL_BITS_DECLARE_STATEP;
-    erts_bits_init_state(ERL_BITS_ARGS_0);
-#endif
 }
 
 /*****************************************************************
diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h
index 5da2b28a89..1445a38772 100644
--- a/erts/emulator/beam/erl_bits.h
+++ b/erts/emulator/beam/erl_bits.h
@@ -84,31 +84,17 @@ typedef struct erl_bin_match_struct{
 #define ms_matchbuffer(_Ms) &(((ErlBinMatchState*) boxed_val(_Ms))->mb)
 
 
-#if defined(ERTS_SMP)
 #define ERL_BITS_REENTRANT
-#else
-/* uncomment to test the reentrant API in the non-SMP runtime system */
-/* #define ERL_BITS_REENTRANT */
-#endif
 
-#ifdef ERL_BITS_REENTRANT
 
 /*
  * Reentrant API with the state passed as a parameter.
  * (Except when the current Process* already is a parameter.)
  */
-#ifdef ERTS_SMP
 /* the state resides in the current process' scheduler data */
 #define ERL_BITS_DECLARE_STATEP			struct erl_bits_state *EBS
 #define ERL_BITS_RELOAD_STATEP(P)		do{EBS = &erts_proc_sched_data((P))->erl_bits_state;}while(0)
 #define ERL_BITS_DEFINE_STATEP(P)		struct erl_bits_state *EBS = &erts_proc_sched_data((P))->erl_bits_state
-#else
-/* reentrant API but with a hidden single global state, for testing only */
-extern struct erl_bits_state ErlBitsState_;
-#define ERL_BITS_DECLARE_STATEP			struct erl_bits_state *EBS = &ErlBitsState_
-#define ERL_BITS_RELOAD_STATEP(P)		do{}while(0)
-#define ERL_BITS_DEFINE_STATEP(P)		ERL_BITS_DECLARE_STATEP
-#endif
 #define ErlBitsState				(*EBS)
 
 #define ERL_BITS_PROTO_0			struct erl_bits_state *EBS
@@ -120,26 +106,6 @@ extern struct erl_bits_state ErlBitsState_;
 #define ERL_BITS_ARGS_2(ARG1,ARG2)		EBS, ARG1, ARG2
 #define ERL_BITS_ARGS_3(ARG1,ARG2,ARG3)		EBS, ARG1, ARG2, ARG3
 
-#else	/* ERL_BITS_REENTRANT */
-
-/*
- * Non-reentrant API with a single global state.
- */
-extern struct erl_bits_state ErlBitsState;
-#define ERL_BITS_DECLARE_STATEP			/*empty*/
-#define ERL_BITS_RELOAD_STATEP(P)		do{}while(0)
-#define ERL_BITS_DEFINE_STATEP(P)		/*empty*/
-
-#define ERL_BITS_PROTO_0			void
-#define ERL_BITS_PROTO_1(PARM1)			PARM1
-#define ERL_BITS_PROTO_2(PARM1,PARM2)		PARM1, PARM2
-#define ERL_BITS_PROTO_3(PARM1,PARM2,PARM3)	PARM1, PARM2, PARM3
-#define ERL_BITS_ARGS_0				/*empty*/
-#define ERL_BITS_ARGS_1(ARG1)			ARG1
-#define ERL_BITS_ARGS_2(ARG1,ARG2)		ARG1, ARG2
-#define ERL_BITS_ARGS_3(ARG1,ARG2,ARG3)		ARG1, ARG2, ARG3
-
-#endif	/* ERL_BITS_REENTRANT */
 
 #define erts_bin_offset		(ErlBitsState.erts_bin_offset_)
 #define erts_current_bin	(ErlBitsState.erts_current_bin_)
@@ -158,10 +124,8 @@ extern struct erl_bits_state ErlBitsState;
   }  while (0)
 
 void erts_init_bits(void);	/* Initialization once. */
-#ifdef ERTS_SMP
 void erts_bits_init_state(ERL_BITS_PROTO_0);
 void erts_bits_destroy_state(ERL_BITS_PROTO_0);
-#endif
 
 
 /*
diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
index f8b2fa744f..48b02c839e 100644
--- a/erts/emulator/beam/erl_cpu_topology.c
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -131,13 +131,11 @@ static erts_cpu_groups_map_t *reader_groups_map;
 
 #define ERTS_MAX_CPU_TOPOLOGY_ID ((int) 0xffff)
 
-#ifdef ERTS_SMP
 static void cpu_bind_order_sort(erts_cpu_topology_t *cpudata,
 				int size,
 				ErtsCpuBindOrder bind_order,
 				int mk_seq);
 static void write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size);
-#endif
 
 static void reader_groups_callback(int, ErtsSchedulerData *, int, void *);
 static erts_cpu_groups_map_t *add_cpu_groups(int groups,
@@ -434,7 +432,6 @@ processor_order_cmp(const void *vx, const void *vy)
     return 0;
 }
 
-#ifdef ERTS_SMP
 void
 erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp)
 {
@@ -490,7 +487,6 @@ erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp)
     (void) ERTS_RUNQ_FLGS_SET(esdp->run_queue, ERTS_RUNQ_FLG_CHK_CPU_BIND);
 }
 
-#endif
 
 void
 erts_sched_check_cpu_bind(ErtsSchedulerData *esdp)
@@ -556,7 +552,6 @@ erts_sched_check_cpu_bind(ErtsSchedulerData *esdp)
     erts_smp_runq_lock(esdp->run_queue);
 }
 
-#ifdef ERTS_SMP
 void
 erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp)
 {
@@ -594,7 +589,6 @@ erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp)
     if (esdp->no <= max_main_threads)
 	erts_thr_set_main_status(1, (int) esdp->no);
 }
-#endif
 
 static void
 write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size)
diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h
index c922214702..88bcad79ab 100644
--- a/erts/emulator/beam/erl_cpu_topology.h
+++ b/erts/emulator/beam/erl_cpu_topology.h
@@ -60,11 +60,9 @@ int erts_init_scheduler_bind_type_string(char *how);
 int erts_init_cpu_topology_string(char *topology_str);
 
 void erts_sched_check_cpu_bind(ErtsSchedulerData *esdp);
-#ifdef ERTS_SMP
 void erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp);
 void erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp);
 void erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp);
-#endif
 
 int erts_update_cpu_info(void);
 
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index b83134c79c..2e5d401ca4 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -61,15 +61,9 @@ enum DbIterSafety {
     ITER_SAFE_LOCKED, /* Safe while table is locked, not between trap calls */
     ITER_SAFE         /* No need to fixate at all */
 };
-#ifdef ERTS_SMP
 #  define ITERATION_SAFETY(Proc,Tab) \
     ((IS_TREE_TABLE((Tab)->common.status) || ONLY_WRITER(Proc,Tab)) ? ITER_SAFE \
      : (((Tab)->common.status & DB_FINE_LOCKED) ? ITER_UNSAFE : ITER_SAFE_LOCKED))
-#else
-#  define ITERATION_SAFETY(Proc,Tab) \
-    ((IS_TREE_TABLE((Tab)->common.status) || ONLY_WRITER(Proc,Tab)) \
-     ? ITER_SAFE : ITER_SAFE_LOCKED)
-#endif
 
 #define DID_TRAP(P,Ret) (!is_value(Ret) && ((P)->freason == TRAP))
 
@@ -264,11 +258,7 @@ is_table_alive(DbTable *tb)
 static ERTS_INLINE int
 is_table_named(DbTable *tb)
 {
-#ifdef ERTS_SMP
     return tb->common.type & DB_NAMED_TABLE;
-#else
-    return tb->common.status & DB_NAMED_TABLE;
-#endif
 }
 
 
@@ -297,13 +287,11 @@ make_tid(Process *c_p, DbTable *tb)
 /* 
 ** The meta hash table of all NAMED ets tables
 */
-#ifdef ERTS_SMP
 #  define META_NAME_TAB_LOCK_CNT 16
 union {
     erts_smp_rwmtx_t lck;
     byte _cache_line_alignment[64];
 }meta_name_tab_rwlocks[META_NAME_TAB_LOCK_CNT];
-#endif
 static struct meta_name_tab_entry {
     union {
 	Eterm name_atom;
@@ -323,9 +311,7 @@ struct meta_name_tab_entry* meta_name_tab_bucket(Eterm name,
 {
     unsigned bix = atom_val(name) & meta_name_tab_mask;
     struct meta_name_tab_entry* bucket = &meta_name_tab[bix];
-#ifdef ERTS_SMP
     *lockp = &meta_name_tab_rwlocks[bix % META_NAME_TAB_LOCK_CNT].lck;
-#endif
     return bucket;
 }    
 
@@ -396,10 +382,8 @@ free_dbtable(void *vtb)
 			 tb->common.fixations);
 	}
 #endif
-#ifdef ERTS_SMP
 	erts_smp_rwmtx_destroy(&tb->common.rwlock);
 	erts_smp_mtx_destroy(&tb->common.fixlock);
-#endif
 	ASSERT(is_immed(tb->common.heir_data));
 
         if (tb->common.btid)
@@ -577,25 +561,20 @@ delete_owned_table(Process *p, DbTable *tb)
 
 static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock)
 {
-#ifdef ERTS_SMP
     erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     if (use_frequent_read_lock)
 	rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     if (erts_ets_rwmtx_spin_count >= 0)
 	rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count;
-#endif
-#ifdef ERTS_SMP
     erts_smp_rwmtx_init_opt(&tb->common.rwlock, &rwmtx_opt, "db_tab",
         tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
     erts_smp_mtx_init(&tb->common.fixlock, "db_tab_fix",
         tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
     tb->common.is_thread_safe = !(tb->common.status & DB_FINE_LOCKED);
-#endif
 }
 
 static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind)
 {
-#ifdef ERTS_SMP
     if (tb->common.type & DB_FINE_LOCKED) {
 	if (kind == LCK_WRITE) {	   
 	    erts_smp_rwmtx_rwlock(&tb->common.rwlock);
@@ -617,7 +596,6 @@ static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind)
 	}
 	ASSERT(tb->common.is_thread_safe);
     }
-#endif
 }
 
 static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
@@ -627,7 +605,6 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
      * DbTable structure. That is, ONLY the SMP case is allowed
      * to follow the tb pointer!
      */
-#ifdef ERTS_SMP
     if (tb->common.type & DB_FINE_LOCKED) {
 	if (kind == LCK_WRITE) {
 	    ASSERT(tb->common.is_thread_safe);
@@ -650,7 +627,6 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
 	    erts_smp_rwmtx_runlock(&tb->common.rwlock);
 	}
     }
-#endif
 }
 
 static ERTS_INLINE
@@ -790,13 +766,11 @@ static int remove_named_tab(DbTable *tb, int have_lock)
     struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom,
 							      &rwlock);
     ASSERT(is_table_named(tb));
-#ifdef ERTS_SMP
     if (!have_lock && erts_smp_rwmtx_tryrwlock(rwlock) == EBUSY) {
 	db_unlock(tb, LCK_WRITE);
 	erts_smp_rwmtx_rwlock(rwlock);
 	db_lock(tb, LCK_WRITE);
     }
-#endif
 
     ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(rwlock));
 
@@ -1598,9 +1572,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     Uint32 status;
     Sint keypos;
     int is_named, is_compressed;
-#ifdef ERTS_SMP
     int is_fine_locked, frequent_read;
-#endif
 #ifdef DEBUG
     int cret;
 #endif
@@ -1616,10 +1588,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     status = DB_SET | DB_PROTECTED;
     keypos = 1;
     is_named = 0;
-#ifdef ERTS_SMP
     is_fine_locked = 0;
     frequent_read = 0;
-#endif
     heir = am_none;
     heir_data = (UWord) am_undefined;
     is_compressed = erts_ets_always_compress;
@@ -1647,30 +1617,18 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 		    keypos = signed_val(tp[2]);
 		}		
 		else if (tp[1] == am_write_concurrency) {
-#ifdef ERTS_SMP
 		    if (tp[2] == am_true) {
 			is_fine_locked = 1;
 		    } else if (tp[2] == am_false) {
 			is_fine_locked = 0;
 		    } else break;
-#else
-		    if ((tp[2] != am_true) &&  (tp[2] != am_false)) {
-			break;
-		    }
-#endif
 		}
 		else if (tp[1] == am_read_concurrency) {
-#ifdef ERTS_SMP
 		    if (tp[2] == am_true) {
 			frequent_read = 1;
 		    } else if (tp[2] == am_false) {
 			frequent_read = 0;
 		    } else break;
-#else
-		    if ((tp[2] != am_true) &&  (tp[2] != am_false)) {
-			break;
-		    }
-#endif
 		    
 		}
 		else if (tp[1] == am_heir && tp[2] == am_none) {
@@ -1712,11 +1670,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     }
     if (IS_HASH_TABLE(status)) {
 	meth = &db_hash;
-#ifdef ERTS_SMP
 	if (is_fine_locked && !(status & DB_PRIVATE)) {
 	    status |= DB_FINE_LOCKED;
 	}
-#endif
     }
     else if (IS_TREE_TABLE(status)) {
 	meth = &db_tree;
@@ -1725,10 +1681,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 	BIF_ERROR(BIF_P, BADARG);
     }
 
-#ifdef ERTS_SMP
     if (frequent_read && !(status & DB_PRIVATE))
 	status |= DB_FREQ_READ;
-#endif
 
     /* we create table outside any table lock
      * and take the unusal cost of destroy table if it
@@ -1747,10 +1701,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     tb->common.meth = meth;
     tb->common.the_name = BIF_ARG_1;
     tb->common.status = status;    
-#ifdef ERTS_SMP
     tb->common.type = status & ERTS_ETS_TABLE_TYPES;
     /* Note, 'type' is *read only* from now on... */
-#endif
     erts_smp_refc_init(&tb->common.fix_count, 0);
     db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ));
     tb->common.keypos = keypos;
@@ -2522,13 +2474,11 @@ BIF_RETTYPE ets_internal_request_all_0(BIF_ALIST_0)
     req->proc = BIF_P;
     erts_proc_add_refc(BIF_P, (Sint) erts_no_schedulers);
 
-#ifdef ERTS_SMP
     if (erts_no_schedulers > 1)
 	erts_schedule_multi_misc_aux_work(1,
 					  erts_no_schedulers,
                                           handle_ets_all_request,
 					  (void *) req);
-#endif
 
     handle_ets_all_request((void *) req);
     BIF_RET(ref);
@@ -3345,7 +3295,6 @@ void init_db(ErtsDbSpinCount db_spin_count)
     unsigned bits;
     size_t size;
 
-#ifdef ERTS_SMP
     int max_spin_count = (1 << 15) - 1; /* internal limit */
     erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
@@ -3393,7 +3342,6 @@ void init_db(ErtsDbSpinCount db_spin_count)
             "meta_name_tab", make_small(i),
             ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DB);
     }
-#endif
 
     erts_smp_atomic_init_nob(&erts_ets_misc_mem_size, 0);
     db_initialize_util();
@@ -3584,9 +3532,7 @@ static SWord proc_cleanup_fixed_table(Process* p, DbFixation* fix)
 	db_lock(tb, LCK_WRITE_REC);
 	if (!(tb->common.status & DB_DELETE)) {
 	    erts_aint_t diff;
-    #ifdef ERTS_SMP
 	    erts_smp_mtx_lock(&tb->common.fixlock);
-    #endif
 
 	    ASSERT(fixing_procs_rbt_lookup(tb->common.fixing_procs, p));
 
@@ -3596,9 +3542,7 @@ static SWord proc_cleanup_fixed_table(Process* p, DbFixation* fix)
 
 	    fixing_procs_rbt_delete(&tb->common.fixing_procs, fix);
 
-    #ifdef ERTS_SMP
 	    erts_smp_mtx_unlock(&tb->common.fixlock);
-    #endif
 	    if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)) {
 		work += db_unfix_table_hash(&(tb->hash));
 	    }
@@ -3754,9 +3698,7 @@ static void fix_table_locked(Process* p, DbTable* tb)
 {
     DbFixation *fix;
 
-#ifdef ERTS_SMP
     erts_smp_mtx_lock(&tb->common.fixlock);
-#endif
     erts_smp_refc_inc(&tb->common.fix_count,1);
     fix = tb->common.fixing_procs;
     if (fix == NULL) {
@@ -3770,9 +3712,7 @@ static void fix_table_locked(Process* p, DbTable* tb)
 	    ASSERT(fixed_tabs_find(NULL, fix));
 	    ++(fix->counter);
 
-#ifdef ERTS_SMP
 	    erts_smp_mtx_unlock(&tb->common.fixlock);
-#endif
 	    return;
 	}
     }
@@ -3785,9 +3725,7 @@ static void fix_table_locked(Process* p, DbTable* tb)
     fix->counter = 1;
     fixing_procs_rbt_insert(&tb->common.fixing_procs, fix);
 
-#ifdef ERTS_SMP
     erts_smp_mtx_unlock(&tb->common.fixlock);
-#endif
     p->flags |= F_USING_DB;
 
     fixed_tabs_insert(p, fix);
@@ -3800,9 +3738,7 @@ static void unfix_table_locked(Process* p,  DbTable* tb,
 {
     DbFixation* fix;
 
-#ifdef ERTS_SMP
     erts_smp_mtx_lock(&tb->common.fixlock);
-#endif
     fix = fixing_procs_rbt_lookup(tb->common.fixing_procs, p);
 
     if (fix) {
@@ -3811,9 +3747,7 @@ static void unfix_table_locked(Process* p,  DbTable* tb,
 	ASSERT(fix->counter >= 0);
 	if (fix->counter == 0) {
 	    fixing_procs_rbt_delete(&tb->common.fixing_procs, fix);
-#ifdef ERTS_SMP
 	    erts_smp_mtx_unlock(&tb->common.fixlock);
-#endif
 	    fixed_tabs_delete(p, fix);
 
 	    erts_refc_dec(&fix->tabs.btid->intern.refc, 1);
@@ -3824,14 +3758,11 @@ static void unfix_table_locked(Process* p,  DbTable* tb,
 	    goto unlocked;
 	}
     }
-#ifdef ERTS_SMP
     erts_smp_mtx_unlock(&tb->common.fixlock);
-#endif
 unlocked:
 
     if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)
 	&& erts_smp_atomic_read_nob(&tb->hash.fixdel) != (erts_aint_t)NULL) {
-#ifdef ERTS_SMP
 	if (*kind_p == LCK_READ && tb->common.is_thread_safe) {
 	    /* Must have write lock while purging pseudo-deleted (OTP-8166) */
 	    erts_smp_rwmtx_runlock(&tb->common.rwlock);
@@ -3839,7 +3770,6 @@ unlocked:
 	    *kind_p = LCK_WRITE;
 	    if (tb->common.status & DB_DELETE) return;
 	}
-#endif
 	db_unfix_table_hash(&(tb->hash));
     }
 }
@@ -3866,7 +3796,6 @@ static void free_fixations_op(DbFixation* fix, void* vctx)
     diff = -((erts_aint_t) fix->counter);
     erts_smp_refc_add(&ctx->tb->common.fix_count, diff, 0);
 
-#ifdef ERTS_SMP
     if (fix->procs.p != ctx->p) { /* Fixated by other process */
         fix->counter = 0;
 
@@ -3882,7 +3811,6 @@ static void free_fixations_op(DbFixation* fix, void* vctx)
          */
     }
     else
-#endif
     {
         fixed_tabs_delete(fix->procs.p, fix);
 
@@ -3895,7 +3823,6 @@ static void free_fixations_op(DbFixation* fix, void* vctx)
     ctx->cnt++;
 }
 
-#ifdef ERTS_SMP
 int erts_db_execute_free_fixation(Process* p, DbFixation* fix)
 {
     ASSERT(fix->counter == 0);
@@ -3907,7 +3834,6 @@ int erts_db_execute_free_fixation(Process* p, DbFixation* fix)
     ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation));
     return 1;
 }
-#endif
 
 static SWord free_fixations_locked(Process* p, DbTable *tb)
 {
@@ -4115,9 +4041,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 		= ERTS_IS_ATOM_STR("safe_fixed_monotonic_time",
 				   What))
 	       || ERTS_IS_ATOM_STR("safe_fixed", What)) {
-#ifdef ERTS_SMP
 	erts_smp_mtx_lock(&tb->common.fixlock);
-#endif
 	if (IS_FIXED(tb)) {
 	    Uint need;
 	    Eterm *hp;
@@ -4159,9 +4083,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	} else {
 	    ret = am_false;
 	}
-#ifdef ERTS_SMP
 	erts_smp_mtx_unlock(&tb->common.fixlock);
-#endif
     } else if (What == am_atom_put("stats",5)) {
 	if (IS_HASH_TABLE(tb->common.status)) {
 	    FloatDef f;
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index ae9322dfd3..2945afe275 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -109,11 +109,7 @@
 #define NSEG_2   256 /* Size of second segment table */
 #define NSEG_INC 128 /* Number of segments to grow after that */
 
-#ifdef ERTS_SMP
 #  define DB_USING_FINE_LOCKING(TB) (((TB))->common.type & DB_FINE_LOCKED)
-#else
-#  define DB_USING_FINE_LOCKING(TB) 0
-#endif
 
 #ifdef ETHR_ORDERED_READ_DEPEND
 #define SEGTAB(tb) ((struct segment**) erts_smp_atomic_read_nob(&(tb)->segtab))
@@ -191,7 +187,6 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix,
     ((is_atom(term) ? (atom_tab(atom_val(term))->slot.bucket.hvalue) : \
       make_internal_hash(term, 0)) % MAX_HASH)
 
-#ifdef ERTS_SMP
 #  define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1)
 #  define GET_LOCK(tb,hval) (&(tb)->locks->lck_vec[(hval) & DB_HASH_LOCK_MASK].lck)
 #  define GET_LOCK_MAYBE(tb,hval) ((tb)->common.is_thread_safe ? NULL : GET_LOCK(tb,hval))
@@ -234,12 +229,6 @@ static ERTS_INLINE void WUNLOCK_HASH(erts_smp_rwmtx_t* lck)
 	erts_smp_rwmtx_rwunlock(lck);
     }
 }
-#else /* ERTS_SMP */
-# define RLOCK_HASH(tb,hval) NULL 
-# define WLOCK_HASH(tb,hval) NULL
-# define RUNLOCK_HASH(lck) ((void)lck) 
-# define WUNLOCK_HASH(lck) ((void)lck)
-#endif /* ERTS_SMP */
 
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -261,31 +250,23 @@ static ERTS_INLINE void WUNLOCK_HASH(erts_smp_rwmtx_t* lck)
 static ERTS_INLINE Sint next_slot(DbTableHash* tb, Uint ix,
 				  erts_smp_rwmtx_t** lck_ptr)
 {
-#ifdef ERTS_SMP
     ix += DB_HASH_LOCK_CNT;
     if (ix < NACTIVE(tb)) return ix;
     RUNLOCK_HASH(*lck_ptr);
     ix = (ix + 1) & DB_HASH_LOCK_MASK;
     if (ix != 0) *lck_ptr = RLOCK_HASH(tb,ix);
     return ix;
-#else
-    return (++ix < NACTIVE(tb)) ? ix : 0;
-#endif
 }
 /* Same as next_slot but with WRITE locking */
 static ERTS_INLINE Sint next_slot_w(DbTableHash* tb, Uint ix,
 				    erts_smp_rwmtx_t** lck_ptr)
 {
-#ifdef ERTS_SMP
     ix += DB_HASH_LOCK_CNT;
     if (ix < NACTIVE(tb)) return ix;
     WUNLOCK_HASH(*lck_ptr);
     ix = (ix + 1) & DB_HASH_LOCK_MASK;
     if (ix != 0) *lck_ptr = WLOCK_HASH(tb,ix);
     return ix;
-#else
-    return next_slot(tb,ix,lck_ptr);
-#endif
 }
 
 #ifndef MIN
@@ -334,9 +315,7 @@ struct segment {
 
 /* An extended segment table */
 struct ext_segtab {
-#ifdef ERTS_SMP
     ErtsThrPrgrLaterOp lop;
-#endif
     struct segment** prev_segtab;  /* Used when table is shrinking */
     int prev_nsegs;                /* Size of prev_segtab */
     int nsegs;                     /* Size of this segtab */
@@ -662,7 +641,6 @@ int db_create_hash(Process *p, DbTable *tbl)
                                                           SIZEOF_SEGMENT(FIRST_SEGSZ));
     sys_memset(tb->first_segtab[0], 0, SIZEOF_SEGMENT(FIRST_SEGSZ));
 
-#ifdef ERTS_SMP
     erts_smp_atomic_init_nob(&tb->is_resizing, 0);
     if (tb->common.type & DB_FINE_LOCKED) {
 	erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
@@ -687,7 +665,6 @@ int db_create_hash(Process *p, DbTable *tbl)
 	tb->locks = NULL;
     }
     ERTS_THR_MEMORY_BARRIER;
-#endif /* ERST_SMP */
     return DB_ERROR_NONE;
 }
 
@@ -1232,15 +1209,10 @@ static int match_traverse(Process* p, DbTableHash* tb,
     Sint got = 0;                  /* Matched terms counter */
     erts_smp_rwmtx_t* lck;         /* Slot lock */
     int ret_value;
-#ifdef ERTS_SMP
     erts_smp_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue)
         = (lock_for_write ? WLOCK_HASH : RLOCK_HASH);
     void (*unlock_hash_function)(erts_smp_rwmtx_t*)
         = (lock_for_write ? WUNLOCK_HASH : RUNLOCK_HASH);
-#else
-    #define lock_hash_function(tb, hval) NULL
-    #define unlock_hash_function(lck) ((void)lck)
-#endif
     Sint (*next_slot_function)(DbTableHash*, Uint, erts_smp_rwmtx_t**)
         = (lock_for_write ? next_slot_w : next_slot);
 
@@ -1359,10 +1331,6 @@ done:
     }
     return ret_value;
 
-#ifndef SMP
-#undef lock_hash_function
-#undef unlock_hash_function
-#endif
 }
 
 /*
@@ -1391,15 +1359,10 @@ static int match_traverse_continue(Process* p, DbTableHash* tb,
     Eterm match_res;
     erts_smp_rwmtx_t* lck;
     int ret_value;
-#ifdef ERTS_SMP
     erts_smp_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue)
         = (lock_for_write ? WLOCK_HASH : RLOCK_HASH);
     void (*unlock_hash_function)(erts_smp_rwmtx_t*)
         = (lock_for_write ? WUNLOCK_HASH : RUNLOCK_HASH);
-#else
-    #define lock_hash_function(tb, hval) NULL
-    #define unlock_hash_function(lck) ((void)lck)
-#endif
     Sint (*next_slot_function)(DbTableHash* tb, Uint ix, erts_smp_rwmtx_t** lck_ptr)
         = (lock_for_write ? next_slot_w : next_slot);
 
@@ -1475,10 +1438,6 @@ done:
      */
     return ret_value;
 
-#ifndef SMP
-#undef lock_hash_function
-#undef unlock_hash_function
-#endif
 }
 
 
@@ -2051,11 +2010,7 @@ static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, Eterm patt
     sd_context.tid = tid;
     sd_context.hp = NULL;
     sd_context.prev_continuation_tptr = NULL;
-#ifdef ERTS_SMP
     sd_context.fixated_by_me = sd_context.tb->common.is_thread_safe ? 0 : 1; /* TODO: something nicer */
-#else
-    sd_context.fixated_by_me = 0;
-#endif
     sd_context.last_pseudo_delete = (Uint) -1;
 
     return match_traverse(
@@ -2330,7 +2285,6 @@ static void db_print_hash(fmtfn_t to, void *to_arg, int show, DbTable *tbl)
 
     erts_print(to, to_arg, "Buckets: %d\n", NACTIVE(tb));
 
-#ifdef ERTS_SMP
     i = tbl->common.is_thread_safe;
     /* If crash dumping we set table to thread safe in order to
        avoid taking any locks */
@@ -2340,9 +2294,6 @@ static void db_print_hash(fmtfn_t to, void *to_arg, int show, DbTable *tbl)
     db_calc_stats_hash(&tbl->hash, &stats);
 
     tbl->common.is_thread_safe = i;
-#else
-    db_calc_stats_hash(&tbl->hash, &stats);
-#endif
 
     erts_print(to, to_arg, "Chain Length Avg: %f\n", stats.avg_chain_len);
     erts_print(to, to_arg, "Chain Length Max: %d\n", stats.max_chain_len);
@@ -2423,7 +2374,6 @@ static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds)
 	    return reds;	/* Not done */
 	}
     }
-#ifdef ERTS_SMP
     if (tb->locks != NULL) {
 	int i;
 	for (i=0; i<DB_HASH_LOCK_CNT; ++i) {
@@ -2433,7 +2383,6 @@ static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds)
 		     (void*)tb->locks, sizeof(DbTableHashFineLocks));
 	tb->locks = NULL;
     }
-#endif    
     ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size) == sizeof(DbTable));
     return reds;			/* Done */
 }
@@ -2637,14 +2586,12 @@ static void alloc_seg(DbTableHash *tb)
     tb->nslots += EXT_SEGSZ;
 }
 
-#ifdef ERTS_SMP
 static void dealloc_ext_segtab(void* lop_data)
 {
     struct ext_segtab* est = (struct ext_segtab*) lop_data;
 
     erts_free(ERTS_ALC_T_DB_SEG, est);
 }
-#endif
 
 /* Shrink table by freeing the top segment
 ** free_records: 1=free any records in segment, 0=assume segment is empty 
@@ -2683,7 +2630,6 @@ static int free_seg(DbTableHash *tb, int free_records)
             SET_SEGTAB(tb, est->prev_segtab);
             tb->nsegs = est->prev_nsegs;
 
-#ifdef ERTS_SMP
             if (!tb->common.is_thread_safe) {
                 /*
                  * Table is doing a graceful shrink operation and we must avoid
@@ -2701,7 +2647,6 @@ static int free_seg(DbTableHash *tb, int free_records)
                                                         sz);
             }
             else
-#endif
                 erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable*)tb, est,
                              SIZEOF_EXT_SEGTAB(est->nsegs));
         }
@@ -2762,22 +2707,18 @@ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2,
 static ERTS_INLINE int
 begin_resizing(DbTableHash* tb)
 {
-#ifdef ERTS_SMP
     if (DB_USING_FINE_LOCKING(tb))
 	return !erts_atomic_xchg_acqb(&tb->is_resizing, 1);
     else
         ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock));
-#endif
     return 1;
 }
 
 static ERTS_INLINE void
 done_resizing(DbTableHash* tb)
 {
-#ifdef ERTS_SMP
     if (DB_USING_FINE_LOCKING(tb))
 	erts_atomic_set_relb(&tb->is_resizing, 0);
-#endif
 }
 
 /* Grow table with one or more new buckets.
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index 523ed7860e..319d563859 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -63,10 +63,8 @@ typedef struct db_table_hash {
 
     /* List of slots where elements have been deleted while table was fixed */
     erts_smp_atomic_t fixdel;  /* (FixedDeletion*) */	
-#ifdef ERTS_SMP
     erts_smp_atomic_t is_resizing; /* grow/shrink in progress */
     DbTableHashFineLocks* locks;
-#endif
 } DbTableHash;
 
 
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 13eacaa8a9..7310694e63 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -361,11 +361,7 @@ typedef struct {
 } ErtsMatchPseudoProcess;
 
 
-#ifdef ERTS_SMP
 static erts_smp_tsd_key_t match_pseudo_process_key;
-#else
-static ErtsMatchPseudoProcess *match_pseudo_process;
-#endif
 
 static ERTS_INLINE void
 cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap)
@@ -414,7 +410,6 @@ static ERTS_INLINE ErtsMatchPseudoProcess *
 get_match_pseudo_process(Process *c_p, Uint heap_size)
 {
     ErtsMatchPseudoProcess *mpsp;
-#ifdef ERTS_SMP
     ErtsSchedulerData *esdp;
 
     esdp = c_p ? c_p->scheduler_data : erts_get_scheduler_data();
@@ -436,10 +431,6 @@ get_match_pseudo_process(Process *c_p, Uint heap_size)
         mpsp->process.scheduler_data = esdp;
 	erts_smp_tsd_set(match_pseudo_process_key, (void *) mpsp);
     }
-#else
-    mpsp = match_pseudo_process;
-    cleanup_match_pseudo_process(mpsp, 0);
-#endif
     if (heap_size > ERTS_DEFAULT_MS_HEAP_SIZE*sizeof(Eterm)) {
 	mpsp->u.heap = (Eterm*) erts_alloc(ERTS_ALC_T_DB_MS_RUN_HEAP, heap_size);
     }
@@ -449,7 +440,6 @@ get_match_pseudo_process(Process *c_p, Uint heap_size)
     return mpsp;
 }
 
-#ifdef ERTS_SMP
 static void
 destroy_match_pseudo_process(void)
 {
@@ -461,19 +451,14 @@ destroy_match_pseudo_process(void)
 	erts_smp_tsd_set(match_pseudo_process_key, (void *) NULL);
     }
 }
-#endif
 
 static
 void
 match_pseudo_process_init(void)
 {
-#ifdef ERTS_SMP
     erts_smp_tsd_key_create(&match_pseudo_process_key,
 			    "erts_match_pseudo_process_key");
     erts_smp_install_exit_handler(destroy_match_pseudo_process);
-#else
-    match_pseudo_process = create_match_pseudo_process();
-#endif
 }
 
 void
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 19055c6110..fb9db3b010 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -241,12 +241,10 @@ typedef struct db_table_common {
     erts_smp_refc_t fix_count;/* fixation counter */
     DbTableList all;
     DbTableList owned;
-#ifdef ERTS_SMP
     erts_smp_rwmtx_t rwlock;  /* rw lock on table */
     erts_smp_mtx_t fixlock;   /* Protects fixing_procs and time */
     int is_thread_safe;       /* No fine locking inside table needed */
     Uint32 type;              /* table type, *read only* after creation */
-#endif
     Eterm owner;              /* Pid of the creator */
     Eterm heir;               /* Pid of the heir */
     UWord heir_data;          /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 49bbab55a8..4238d161ec 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -50,7 +50,6 @@ fatal_error(int err, char *func)
 #define ERL_DRV_TSD_EXTRA 10
 #define ERL_DRV_INVALID_TSD_KEY INT_MAX
 
-#ifdef USE_THREADS
 
 struct ErlDrvMutex_ {
     ethr_mutex mtx;
@@ -85,10 +84,6 @@ struct ErlDrvTid_ {
 
 static ethr_tsd_key tid_key;
 
-#else /* USE_THREADS */
-static Uint tsd_len;
-static void **tsd;
-#endif
 
 static ErlDrvTSDKey next_tsd_key;
 static ErlDrvTSDKey max_used_tsd_key;
@@ -97,7 +92,6 @@ static char **used_tsd_keys;
 static erts_mtx_t tsd_mtx;
 static char *no_name;
 
-#ifdef USE_THREADS
 
 static void
 thread_exit_handler(void)
@@ -122,21 +116,15 @@ erl_drv_thread_wrapper(void *vdtid)
     return (*dtid->func)(dtid->arg);
 }
 
-#endif
 
 void erl_drv_thr_init(void)
 {
     int i;
-#ifdef USE_THREADS
     int res = ethr_tsd_key_create(&tid_key,"erts_tid_key");
     if (res == 0)
 	res = ethr_install_exit_handler(thread_exit_handler);
     if (res != 0)
 	fatal_error(res, "erl_drv_thr_init()");
-#else
-    tsd_len = 0;
-    tsd = NULL;
-#endif
 
     no_name = "unknown";
     next_tsd_key = 0;
@@ -159,7 +147,6 @@ void erl_drv_thr_init(void)
 ErlDrvMutex *
 erl_drv_mutex_create(char *name)
 {
-#ifdef USE_THREADS
     ErlDrvMutex *dmtx = erts_alloc_fnf(ERTS_ALC_T_DRV_MTX,
 				       (sizeof(ErlDrvMutex)
 					+ (name ? sys_strlen(name) + 1 : 0)));
@@ -182,15 +169,11 @@ erl_drv_mutex_create(char *name)
 #endif
     }
     return dmtx;
-#else
-    return (ErlDrvMutex *) NULL;
-#endif
 }
 
 void
 erl_drv_mutex_destroy(ErlDrvMutex *dmtx)
 {
-#ifdef USE_THREADS
     int res;
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_uninstall(&dmtx->lcnt);
@@ -199,24 +182,18 @@ erl_drv_mutex_destroy(ErlDrvMutex *dmtx)
     if (res != 0)
 	fatal_error(res, "erl_drv_mutex_destroy()");
     erts_free(ERTS_ALC_T_DRV_MTX, (void *) dmtx);
-#endif
 }
 
 
 char *
 erl_drv_mutex_name(ErlDrvMutex *dmtx)
 {
-#ifdef USE_THREADS
     return dmtx ? dmtx->name : NULL;
-#else
-    return NULL;
-#endif
 }
 
 int
 erl_drv_mutex_trylock(ErlDrvMutex *dmtx)
 {
-#ifdef USE_THREADS
     int res;
     if (!dmtx)
 	fatal_error(EINVAL, "erl_drv_mutex_trylock()");
@@ -225,22 +202,17 @@ erl_drv_mutex_trylock(ErlDrvMutex *dmtx)
     erts_lcnt_trylock(&dmtx->lcnt, res);
 #endif
     return res;
-#else
-    return 0;
-#endif
 }
 
 void
 erl_drv_mutex_lock(ErlDrvMutex *dmtx)
 {
-#ifdef USE_THREADS
     if (!dmtx)
 	fatal_error(EINVAL, "erl_drv_mutex_lock()");
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock(&dmtx->lcnt);
 #endif
     ethr_mutex_lock(&dmtx->mtx);
-#endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post(&dmtx->lcnt);
 #endif
@@ -249,20 +221,17 @@ erl_drv_mutex_lock(ErlDrvMutex *dmtx)
 void
 erl_drv_mutex_unlock(ErlDrvMutex *dmtx)
 {
-#ifdef USE_THREADS
     if (!dmtx)
 	fatal_error(EINVAL, "erl_drv_mutex_unlock()");
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock(&dmtx->lcnt);
 #endif
     ethr_mutex_unlock(&dmtx->mtx);
-#endif
 }
 
 ErlDrvCond *
 erl_drv_cond_create(char *name)
 {
-#ifdef USE_THREADS
     ErlDrvCond *dcnd = erts_alloc_fnf(ERTS_ALC_T_DRV_CND,
 				      (sizeof(ErlDrvCond)
 				       + (name ? sys_strlen(name) + 1 : 0)));
@@ -281,57 +250,43 @@ erl_drv_cond_create(char *name)
 	}
     }
     return dcnd;
-#else
-    return (ErlDrvCond *) NULL;
-#endif
 }
 
 void
 erl_drv_cond_destroy(ErlDrvCond *dcnd)
 {
-#ifdef USE_THREADS
     int res = dcnd ? ethr_cond_destroy(&dcnd->cnd) : EINVAL;
     if (res != 0)
 	fatal_error(res, "erl_drv_cond_destroy()");
     erts_free(ERTS_ALC_T_DRV_CND, (void *) dcnd);
-#endif
 }
 
 char *
 erl_drv_cond_name(ErlDrvCond *dcnd)
 {
-#ifdef USE_THREADS
     return dcnd ? dcnd->name : NULL;
-#else
-    return NULL;
-#endif
 }
 
 void
 erl_drv_cond_signal(ErlDrvCond *dcnd)
 {
-#ifdef USE_THREADS
     if (!dcnd)
 	fatal_error(EINVAL, "erl_drv_cond_signal()");
     ethr_cond_signal(&dcnd->cnd);
-#endif
 }
 
 void
 erl_drv_cond_broadcast(ErlDrvCond *dcnd)
 {
-#ifdef USE_THREADS
     if (!dcnd)
 	fatal_error(EINVAL, "erl_drv_cond_broadcast()");
     ethr_cond_broadcast(&dcnd->cnd);
-#endif
 }
 
 
 void
 erl_drv_cond_wait(ErlDrvCond *dcnd, ErlDrvMutex *dmtx)
 {
-#ifdef USE_THREADS
     if (!dcnd || !dmtx) {
 	fatal_error(EINVAL, "erl_drv_cond_wait()");
     }
@@ -348,13 +303,11 @@ erl_drv_cond_wait(ErlDrvCond *dcnd, ErlDrvMutex *dmtx)
 	    break;
         }
     }
-#endif
 }
 
 ErlDrvRWLock *
 erl_drv_rwlock_create(char *name)
 {
-#ifdef USE_THREADS
     ErlDrvRWLock *drwlck = erts_alloc_fnf(ERTS_ALC_T_DRV_RWLCK,
 					  (sizeof(ErlDrvRWLock)
 					   + (name ? sys_strlen(name) + 1 : 0)));
@@ -375,15 +328,11 @@ erl_drv_rwlock_create(char *name)
 #endif
     }
     return drwlck;
-#else
-    return (ErlDrvRWLock *) NULL;
-#endif
 }
 
 void
 erl_drv_rwlock_destroy(ErlDrvRWLock *drwlck)
 {
-#ifdef USE_THREADS
     int res;
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_uninstall(&drwlck->lcnt);
@@ -392,23 +341,17 @@ erl_drv_rwlock_destroy(ErlDrvRWLock *drwlck)
     if (res != 0)
 	fatal_error(res, "erl_drv_rwlock_destroy()");
     erts_free(ERTS_ALC_T_DRV_RWLCK, (void *) drwlck);
-#endif
 }
 
 char *
 erl_drv_rwlock_name(ErlDrvRWLock *drwlck)
 {
-#ifdef USE_THREADS
     return drwlck ? drwlck->name : NULL;
-#else
-    return NULL;
-#endif
 }
 
 int
 erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck)
 {
-#ifdef USE_THREADS
     int res;
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_tryrlock()");
@@ -417,15 +360,11 @@ erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck)
     erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LOCK_OPTION_READ);
 #endif
     return res;
-#else
-    return 0;
-#endif
 }
 
 void
 erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck)
 {
-#ifdef USE_THREADS
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_rlock()");
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -435,26 +374,22 @@ erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post(&drwlck->lcnt);
 #endif
-#endif
 }
 
 void
 erl_drv_rwlock_runlock(ErlDrvRWLock *drwlck)
 {
-#ifdef USE_THREADS
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_runlock()");
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LOCK_OPTION_READ);
 #endif
     ethr_rwmutex_runlock(&drwlck->rwmtx);
-#endif
 }
 
 int
 erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck)
 {
-#ifdef USE_THREADS
     int res;
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_tryrwlock()");
@@ -463,15 +398,11 @@ erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck)
     erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LOCK_OPTION_RDWR);
 #endif
     return res;
-#else
-    return 0;
-#endif
 }
 
 void
 erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck)
 {
-#ifdef USE_THREADS
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_rwlock()");
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -481,20 +412,17 @@ erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post(&drwlck->lcnt);
 #endif
-#endif
 }
 
 void
 erl_drv_rwlock_rwunlock(ErlDrvRWLock *drwlck)
 {
-#ifdef USE_THREADS
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_rwunlock()");
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LOCK_OPTION_RDWR);
 #endif
     ethr_rwmutex_rwunlock(&drwlck->rwmtx);
-#endif
 }
 
 int
@@ -588,20 +516,13 @@ erl_drv_tsd_key_destroy(ErlDrvTSDKey key)
 }
 
 
-#ifdef USE_THREADS
 #define ERL_DRV_TSD__ (dtid->tsd)
 #define ERL_DRV_TSD_LEN__ (dtid->tsd_len)
-#else
-#define ERL_DRV_TSD__ (tsd)
-#define ERL_DRV_TSD_LEN__ (tsd_len)
-#endif
 
 void
 erl_drv_tsd_set(ErlDrvTSDKey key, void *data)
 {
-#ifdef USE_THREADS
     struct ErlDrvTid_ *dtid = (struct ErlDrvTid_ *) erl_drv_thread_self();
-#endif
 
     if (key < 0 || max_used_tsd_key < key || !used_tsd_keys[key])
 	fatal_error(EINVAL, "erl_drv_tsd_set()");
@@ -629,15 +550,11 @@ erl_drv_tsd_set(ErlDrvTSDKey key, void *data)
 void *
 erl_drv_tsd_get(ErlDrvTSDKey key)
 {
-#ifdef USE_THREADS
     struct ErlDrvTid_ *dtid = ethr_tsd_get(tid_key);
-#endif
     if (key < 0 || max_used_tsd_key < key || !used_tsd_keys[key])
 	fatal_error(EINVAL, "erl_drv_tsd_get()");
-#ifdef USE_THREADS
     if (!dtid)
 	return NULL;
-#endif
     if (ERL_DRV_TSD_LEN__ <= key)
 	return NULL;
     return ERL_DRV_TSD__[key];
@@ -672,7 +589,6 @@ erl_drv_thread_create(char *name,
 		      void* arg,
 		      ErlDrvThreadOpts *opts)
 {
-#ifdef USE_THREADS
     int res;
     struct ErlDrvTid_ *dtid;
     ethr_thr_opts ethr_opts = ETHR_THR_OPTS_DEFAULT_INITER;
@@ -714,27 +630,19 @@ erl_drv_thread_create(char *name,
 
     *tid = (ErlDrvTid) dtid;
     return 0;
-#else
-    return ENOTSUP;
-#endif
 }
 
 char *
 erl_drv_thread_name(ErlDrvTid tid)
 {
-#ifdef USE_THREADS
     struct ErlDrvTid_ *dtid = (struct ErlDrvTid_ *) tid;
     return dtid ? dtid->name : NULL;
-#else
-    return NULL;
-#endif
 }
 
 
 ErlDrvTid
 erl_drv_thread_self(void)
 {
-#ifdef USE_THREADS
     struct ErlDrvTid_ *dtid = ethr_tsd_get(tid_key);
     if (!dtid) {
 	int res;
@@ -753,15 +661,11 @@ erl_drv_thread_self(void)
 	    fatal_error(res, "erl_drv_thread_self()");
     }
     return (ErlDrvTid) dtid;
-#else
-    return (ErlDrvTid) NULL;
-#endif
 }
 
 int
 erl_drv_equal_tids(ErlDrvTid tid1, ErlDrvTid tid2)
 {   
-#ifdef USE_THREADS
     int res;
     struct ErlDrvTid_ *dtid1 = (struct ErlDrvTid_ *) tid1;
     struct ErlDrvTid_ *dtid2 = (struct ErlDrvTid_ *) tid2;
@@ -775,28 +679,22 @@ erl_drv_equal_tids(ErlDrvTid tid1, ErlDrvTid tid2)
 	   : !ethr_equal_tids(dtid1->tid, dtid2->tid));
 
     return res;
-#else
-    return 1;
-#endif
 }
 
 void
 erl_drv_thread_exit(void *res)
 {
-#ifdef USE_THREADS
     struct ErlDrvTid_ *dtid = ethr_tsd_get(tid_key);
     if (dtid && dtid->drv_thr) {
 	ethr_thr_exit(res);
 	fatal_error(0, "erl_drv_thread_exit()");
     }
-#endif
     fatal_error(EACCES, "erl_drv_thread_exit()");
 }
 
 int
 erl_drv_thread_join(ErlDrvTid tid, void **respp)
 {
-#ifdef USE_THREADS
     int res;
     struct ErlDrvTid_ *dtid = (struct ErlDrvTid_ *) tid;
 
@@ -809,9 +707,6 @@ erl_drv_thread_join(ErlDrvTid tid, void **respp)
     if (res == 0)
 	erts_free(ERTS_ALC_T_DRV_TID, dtid);
     return res;
-#else
-    return ENOTSUP;
-#endif
 }
 
 #if defined(__DARWIN__) && defined(USE_THREADS) && defined(ERTS_SMP)
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index 535f677bb3..bfecb7a85f 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -183,13 +183,11 @@ void
 erts_erase_fun_entry(ErlFunEntry* fe)
 {
     erts_fun_write_lock();
-#ifdef ERTS_SMP
     /*
      * We have to check refc again since someone might have looked up
      * the fun entry and incremented refc after last check.
      */
     if (erts_smp_refc_dectest(&fe->refc, -1) <= 0)
-#endif
     {
 	if (fe->address != unloaded_fun)
 	    erts_exit(ERTS_ERROR_EXIT,
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 8cb977a7f3..d49564731b 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -60,9 +60,6 @@
 #  define ERTS_GC_ASSERT(B) ((void) 1)
 #endif
 
-#if defined(DEBUG) && 0
-#  define HARDDEBUG 1
-#endif
 
 /*
  * Returns number of elements in an array.
@@ -3335,13 +3332,11 @@ erts_gc_info_request(Process *c_p)
 
     erts_proc_add_refc(c_p, (Sint) erts_no_schedulers);
 
-#ifdef ERTS_SMP
     if (erts_no_schedulers > 1)
 	erts_schedule_multi_misc_aux_work(1,
 					  erts_no_schedulers,
 					  reply_gc_info,
 					  (void *) gcirp);
-#endif
 
     reply_gc_info((void *) gcirp);
 
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 99995be464..5e9b7fe42e 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -96,12 +96,8 @@ typedef enum {
 
 #define ERTS_BIF_TIMER_SHORT_TIME 5000
 
-#ifdef ERTS_SMP
 #  define ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore \
     ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore)
-#else
-#  define ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore
-#endif
 
 /* Bit 0 to 9 contains scheduler id (see mask below) */
 #define ERTS_TMR_ROFLG_HLT		(((Uint32) 1) << 10)
@@ -269,7 +265,6 @@ typedef struct {
     erts_atomic_t last;
 } ErtsHLTCncldTmrQTail;
 
-#ifdef ERTS_SMP
 
 typedef struct {
     /*
@@ -301,7 +296,6 @@ typedef struct {
     } head;
 } ErtsHLTCncldTmrQ;
 
-#endif /* ERTS_SMP */
 
 typedef struct {
     ErtsHLTimer *root;
@@ -309,9 +303,7 @@ typedef struct {
 } ErtsYieldingTimeoutState;
 
 struct ErtsHLTimerService_ {
-#ifdef ERTS_SMP
     ErtsHLTCncldTmrQ canceled_queue;
-#endif
     ErtsHLTimer *time_tree;
 #ifndef ERTS_MAGIC_REF_BIF_TIMERS
     ErtsBifTimer *btm_tree;
@@ -720,9 +712,7 @@ proc_btm_list_foreach_destroy_yielding(ErtsBifTimer **list,
 
 #endif /* !ERTS_MAGIC_REF_BIF_TIMERS */
 
-#ifdef ERTS_SMP
 static void init_canceled_queue(ErtsHLTCncldTmrQ *cq);
-#endif
 
 void
 erts_hl_timer_init(void)
@@ -747,9 +737,7 @@ erts_create_timer_service(void)
     srv->yield = init_yield;
     erts_twheel_init_timer(&srv->service_timer);
 
-#ifdef ERTS_SMP
     init_canceled_queue(&srv->canceled_queue);
-#endif
 
     return srv;
 }
@@ -1186,7 +1174,6 @@ hl_timer_dec_refc(ErtsHLTimer *tmr, Uint32 roflgs)
 }
 
 static void hlt_service_timeout(void *vesdp);
-#ifdef ERTS_SMP
 static void handle_canceled_queue(ErtsSchedulerData *esdp,
 				  ErtsHLTCncldTmrQ *cq,
 				  int use_limit,
@@ -1194,7 +1181,6 @@ static void handle_canceled_queue(ErtsSchedulerData *esdp,
 				  int *need_thr_progress,
 				  ErtsThrPrgrVal *thr_prgr_p,
 				  int *need_more_work);
-#endif
 
 static ERTS_INLINE void
 check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv)
@@ -1663,7 +1649,6 @@ cleanup_sched_local_canceled_timer(ErtsSchedulerData *esdp,
     }
 }
 
-#ifdef ERTS_SMP
 
 static void
 init_canceled_queue(ErtsHLTCncldTmrQ *cq)
@@ -1886,31 +1871,24 @@ erts_handle_canceled_timers(void *vesdp,
 			  need_more_work);
 }
 
-#endif /* ERTS_SMP */
 
 static void
 queue_canceled_timer(ErtsSchedulerData *esdp, int rsched_id, ErtsTimer *tmr)
 {
-#ifdef ERTS_SMP
     ErtsHLTCncldTmrQ *cq;
     cq = &ERTS_SCHEDULER_IX(rsched_id-1)->timer_service->canceled_queue;
     if (cq_enqueue(cq, tmr, rsched_id - (int) esdp->no))
 	erts_notify_canceled_timer(esdp, rsched_id);
-#else
-    ERTS_INTERNAL_ERROR("Unexpected enqueue of canceled timer");
-#endif
 }
 
 static void
 continue_cancel_ptimer(ErtsSchedulerData *esdp, ErtsTimer *tmr)
 {
-#ifdef ERTS_SMP
     Uint32 sid = (tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK);
 
     if (esdp->no != sid)
 	queue_canceled_timer(esdp, sid, tmr);
     else
-#endif
 	cleanup_sched_local_canceled_timer(esdp, tmr);
 }
 
diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h
index ff31f04cb9..852bf88174 100644
--- a/erts/emulator/beam/erl_hl_timer.h
+++ b/erts/emulator/beam/erl_hl_timer.h
@@ -63,13 +63,11 @@ void erts_hl_timer_init(void);
 void erts_start_timer_callback(ErtsMonotonicTime,
 			       void (*)(void *),
 			       void *);
-#ifdef ERTS_SMP
 void
 erts_handle_canceled_timers(void *vesdp,
 			    int *need_thr_progress,
 			    ErtsThrPrgrVal *thr_prgr_p,
 			    int *need_more_work);
-#endif
 
 Uint erts_bif_timer_memory_size(void);
 void erts_print_bif_timer_info(fmtfn_t to, void *to_arg);
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 5206d7564f..fe7f09e5fa 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -69,16 +69,8 @@
  * The variables below (prefixed with etp_) are for erts/etc/unix/etp-commands
  * only. Do not remove even though they aren't used elsewhere in the emulator!
  */
-#ifdef ERTS_SMP
 const int etp_smp_compiled = 1;
-#else
-const int etp_smp_compiled = 0;
-#endif
-#ifdef USE_THREADS
 const int etp_thread_compiled = 1;
-#else
-const int etp_thread_compiled = 0;
-#endif
 const char etp_erts_version[] = ERLANG_VERSION;
 const char etp_otp_release[] = ERLANG_OTP_RELEASE;
 const char etp_compile_date[] = ERLANG_COMPILE_DATE;
@@ -156,17 +148,10 @@ static void erl_init(int ncpu,
 
 static erts_atomic_t exiting;
 
-#ifdef ERTS_SMP
 erts_smp_atomic32_t erts_writing_erl_crash_dump;
 erts_tsd_key_t erts_is_crash_dumping_key;
-#else
-volatile int erts_writing_erl_crash_dump = 0;
-#endif
 int erts_initialized = 0;
 
-#if defined(USE_THREADS) && !defined(ERTS_SMP)
-erts_tid_t erts_main_thread;
-#endif
 
 int erts_use_sender_punish;
 
@@ -682,7 +667,6 @@ void erts_usage(void)
     erts_exit(1, "");
 }
 
-#ifdef USE_THREADS
 /*
  * allocators for thread lib
  */
@@ -724,7 +708,6 @@ static void ethr_ll_free(void *ptr)
     erts_free(ERTS_ALC_T_ETHR_LL, ptr);
 }
 
-#endif
 
 static int
 early_init(int *argc, char **argv) /*
@@ -754,9 +737,6 @@ early_init(int *argc, char **argv) /*
     char envbuf[21]; /* enough for any 64-bit integer */
     size_t envbufsz;
 
-#if defined(USE_THREADS) && !defined(ERTS_SMP)
-    erts_main_thread = erts_thr_self();
-#endif
 
     erts_save_emu_args(*argc, argv);
 
@@ -781,11 +761,6 @@ early_init(int *argc, char **argv) /*
 				     &ncpu,
 				     &ncpuonln,
 				     &ncpuavail);
-#ifndef ERTS_SMP
-    ncpu = 1;
-    ncpuonln = 1;
-    ncpuavail = 1;
-#endif
 
     ignore_break = 0;
     replace_intr = 0;
@@ -797,16 +772,10 @@ early_init(int *argc, char **argv) /*
 
     erts_sys_pre_init();
     erts_atomic_init_nob(&exiting, 0);
-#ifdef ERTS_SMP
     erts_thr_progress_pre_init();
-#endif
 
-#ifdef ERTS_SMP
     erts_smp_atomic32_init_nob(&erts_writing_erl_crash_dump, 0L);
     erts_tsd_key_create(&erts_is_crash_dumping_key,"erts_is_crash_dumping_key");
-#else
-    erts_writing_erl_crash_dump = 0;
-#endif
 
     erts_smp_atomic32_init_nob(&erts_max_gen_gcs,
 			       (erts_aint32_t) ((Uint16) -1));
@@ -1093,7 +1062,6 @@ early_init(int *argc, char **argv) /*
 	    i++;
 	}
 
-#ifdef ERTS_SMP
 	/* apply any scheduler percentages */
 	if (schdlrs_percentage != 100 || schdlrs_onln_percentage != 100) {
 	    schdlrs = schdlrs * schdlrs_percentage / 100;
@@ -1117,11 +1085,6 @@ early_init(int *argc, char **argv) /*
 		erts_usage();
 	    }
 	}
-#else
-	/* Silence gcc warnings */
-	(void)schdlrs_percentage;
-	(void)schdlrs_onln_percentage;
-#endif
 #ifdef ERTS_DIRTY_SCHEDULERS
 	/* apply any dirty scheduler precentages */
 	if (dirty_cpu_scheds_pctg != 100 || dirty_cpu_scheds_onln_pctg != 100) {
@@ -1139,18 +1102,11 @@ early_init(int *argc, char **argv) /*
 #endif
     }
 
-#ifndef USE_THREADS
-    erts_async_max_threads = 0;
-#endif
 
-#ifdef ERTS_SMP
     no_schedulers = schdlrs;
     no_schedulers_online = schdlrs_onln;
 
     erts_no_schedulers = (Uint) no_schedulers;
-#else
-    erts_no_schedulers = 1;
-#endif
 #ifdef ERTS_DIRTY_SCHEDULERS
     erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers = dirty_cpu_scheds;
     no_dirty_cpu_schedulers_online = dirty_cpu_scheds_online;
@@ -1162,7 +1118,6 @@ early_init(int *argc, char **argv) /*
     erts_alloc_init(argc, argv, &alloc_opts); /* Handles (and removes)
 						 -M flags. */
     /* Require allocators */
-#ifdef ERTS_SMP
     /*
      * Thread progress management:
      *
@@ -1185,7 +1140,6 @@ early_init(int *argc, char **argv) /*
 			   erts_no_dirty_io_schedulers
 #endif
 			   );
-#endif
     erts_thr_q_init();
     erts_init_utils();
     erts_early_init_cpu_topology(no_schedulers,
@@ -1193,7 +1147,6 @@ early_init(int *argc, char **argv) /*
 				 max_reader_groups,
 				 &reader_groups);
 
-#ifdef USE_THREADS
     {
 	erts_thr_late_init_data_t elid = ERTS_THR_LATE_INIT_DATA_DEF_INITER;
 	elid.mem.std.alloc = ethr_std_alloc;
@@ -1210,7 +1163,6 @@ early_init(int *argc, char **argv) /*
 
 	erts_thr_late_init(&elid);
     }
-#endif
     erts_msacc_early_init();
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -1237,40 +1189,6 @@ early_init(int *argc, char **argv) /*
     return ncpu;
 }
 
-#ifndef ERTS_SMP
-
-void *erts_scheduler_stack_limit;
-
-
-static void set_main_stack_size(void)
-{
-    char c;
-    UWord stacksize;
-# if HAVE_DECL_GETRLIMIT && HAVE_DECL_SETRLIMIT && HAVE_DECL_RLIMIT_STACK
-    struct rlimit rl;
-    int bytes;
-    stacksize = erts_sched_thread_suggested_stack_size * sizeof(Uint) * 1024;
-    /* Add some extra pages... neede by some systems... */
-    bytes = (int) stacksize + 3*erts_sys_get_page_size();
-    if (getrlimit(RLIMIT_STACK, &rl) != 0 ||
-        (rl.rlim_cur = bytes, setrlimit(RLIMIT_STACK, &rl) != 0)) {
-        erts_fprintf(stderr, "failed to set stack size for scheduler "
-                     "thread to %d bytes\n", bytes);
-        erts_usage();
-    }
-# else
-    if (modified_sched_thread_suggested_stack_size) {
-	erts_fprintf(stderr, "no OS support for dynamic stack size limit\n");
-	erts_usage();
-    }
-    /* Be conservative and hope it is not more than 64 kWords... */
-    stacksize = 64*1024*sizeof(void *);
-# endif
-
-    erts_scheduler_stack_limit = erts_calc_stacklimit(&c, stacksize);
-}
-
-#endif
 
 void
 erl_start(int argc, char **argv)
@@ -1490,12 +1408,8 @@ erl_start(int argc, char **argv)
 #ifdef DEBUG
 		strcat(tmp, ",DEBUG");
 #endif
-#ifdef ERTS_SMP
 		strcat(tmp, ",SMP");
-#endif
-#ifdef USE_THREADS
 		strcat(tmp, ",ASYNC_THREADS");
-#endif
 #ifdef HIPE
 		strcat(tmp, ",HIPE");
 #endif
@@ -2007,9 +1921,7 @@ erl_start(int argc, char **argv)
 				 arg);
 		    erts_usage();
 		}
-#ifdef ERTS_SMP
 		erts_runq_supervision_interval = val;
-#endif
 	    }
 	    else {
 		erts_fprintf(stderr, "bad scheduling option %s\n", argv[i]);
@@ -2355,7 +2267,6 @@ erl_start(int argc, char **argv)
 
     }
 
-#ifdef ERTS_SMP
     erts_start_schedulers();
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
@@ -2364,31 +2275,9 @@ erl_start(int argc, char **argv)
 
     /* Let system specific code decide what to do with the main thread... */
     erts_sys_main_thread(); /* May or may not return! */
-#else
-    {
-	ErtsSchedulerData *esdp = erts_get_scheduler_data();
-        erts_msacc_init_thread("scheduler", 1, 1);
-	erts_thr_set_main_status(1, 1);
-#if ERTS_USE_ASYNC_READY_Q
-	esdp->aux_work_data.async_ready.queue
-	    = erts_get_async_ready_queue(1);
-#endif
-	set_main_stack_size();
-	erts_sched_init_time_sup(esdp);
-        erts_ets_sched_spec_data_init(esdp);
-        erts_aux_work_timeout_late_init(esdp);
-
-#ifdef ERTS_ENABLE_LOCK_COUNT
-        erts_lcnt_post_startup();
-#endif
-
-	process_main(esdp->x_reg_array, esdp->f_reg_array);
-    }
-#endif
 }
 
 
-#ifdef USE_THREADS
 
 __decl_noreturn void erts_thr_fatal_error(int err, char *what)
 {
@@ -2402,7 +2291,6 @@ __decl_noreturn void erts_thr_fatal_error(int err, char *what)
     abort();
 }
 
-#endif
 
 static void
 system_cleanup(int flush_async)
@@ -2415,7 +2303,6 @@ system_cleanup(int flush_async)
 	 * Another thread is currently exiting the system;
 	 * wait for it to do its job.
 	 */
-#ifdef ERTS_SMP
 	if (erts_thr_progress_is_managed_thread()) {
 	    /*
 	     * The exiting thread might be waiting for
@@ -2424,7 +2311,6 @@ system_cleanup(int flush_async)
 	    erts_thr_progress_active(NULL, 0);
 	    erts_thr_progress_prepare_wait(NULL);
 	}
-#endif
 	/* Wait forever... */
 	while (1)
 	    erts_milli_sleep(10000000);
@@ -2439,16 +2325,11 @@ system_cleanup(int flush_async)
 
     if (!flush_async
 	|| !erts_initialized
-#if defined(USE_THREADS) && !defined(ERTS_SMP)
-	|| !erts_equal_tids(erts_main_thread, erts_thr_self())
-#endif
 	)
 	return;
 
-#ifdef ERTS_SMP
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_check_exact(NULL, 0);
-#endif
 #endif
 
     erts_exit_flush_async();
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 85ee703c99..35f259f7c8 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -75,12 +75,9 @@ static erts_lc_lock_order_t erts_lock_order[] = {
      *						 if only one lock use
      *						 the lock name)"
      */
-#ifdef ERTS_SMP
     {	"driver_lock",				"driver_name"		},
     {	"port_lock",				"port_id"		},
-#endif
     {	"port_data_lock",			"address"		},
-#ifdef ERTS_SMP
     {	"bif_timers",				NULL			},
     {	"reg_tab",				NULL			},
     {	"proc_main",				"pid"			},
@@ -89,9 +86,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"hipe_mfait_lock",			NULL			},
 #endif
     {	"nodes_monitors",			NULL			},
-#ifdef ERTS_SMP
     {	"resource_monitors",			"address"	        },
-#endif
     {   "driver_list",                          NULL                    },
     {	"proc_link",				"pid"			},
     {	"proc_msgq",				"pid"			},
@@ -114,7 +109,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"fun_tab",				NULL			},
     {	"environ",				NULL			},
     {	"release_literal_areas",		NULL			},
-#endif
     {	"efile_drv",				"address"		},
     {	"drv_ev_state_grow",			NULL,   		},
     {	"drv_ev_state",				"address"		},
@@ -144,7 +138,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"async_enq_mtx",			NULL			},
     {   "msacc_list_mutex",                     NULL                    },
     {   "msacc_unmanaged_mutex",                NULL                    },
-#ifdef ERTS_SMP
     {	"atom_tab",				NULL			},
     {	"misc_op_list_pre_alloc_lock",		"address"		},
     {	"message_pre_alloc_lock",		"address"		},
@@ -155,17 +148,13 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"sys_msg_q", 				NULL			},
     {	"tracer_mtx", 				NULL			},
     {   "port_table",                           NULL                    },
-#endif
     {	"magic_ref_table",			"address"		},
     {	"mtrace_op",				NULL			},
     {	"instr_x",				NULL			},
     {	"instr",				NULL			},
-#ifdef ERTS_SMP
     {	"pollsets_lock",			NULL			},
-#endif
     {	"alcu_allocator",			"index"			},
     {	"mseg",					NULL			},
-#ifdef ERTS_SMP
     {	"port_task_pre_alloc_lock",		"address"		},
     {	"proclist_pre_alloc_lock",		"address"		},
     {	"xports_list_pre_alloc_lock",		"address"		},
@@ -178,7 +167,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"pix_lock",				"address"		},
     {	"run_queues_lists",			NULL			},
     {	"sched_stat",				NULL			},
-#endif
     {	"async_init_mtx",			NULL			},
 #ifdef __WIN32__
 #ifdef DEBUG
@@ -189,9 +177,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {   "efile_drv dtrace mutex",               NULL                    },
 #endif
     {	"mtrace_buf",				NULL			},
-#ifdef ERTS_SMP
     {	"os_monotonic_time",			NULL			},
-#endif
     {	"erts_alloc_hard_debug",		NULL			},
     {	"hard_dbg_mseg",		        NULL	                },
     {	"erts_mmap",				NULL			}
diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h
index 18296d1fec..ca5a72b183 100644
--- a/erts/emulator/beam/erl_lock_check.h
+++ b/erts/emulator/beam/erl_lock_check.h
@@ -116,11 +116,7 @@ int erts_lc_is_emu_thr(void);
 
 #define ERTS_LC_ASSERT(A) \
     ((void) (((A) || ERTS_SOMEONE_IS_CRASH_DUMPING) ? 1 : erts_lc_assert_failed(__FILE__, __LINE__, #A)))
-#ifdef ERTS_SMP
 #define ERTS_SMP_LC_ASSERT(A) ERTS_LC_ASSERT(A)
-#else
-#define ERTS_SMP_LC_ASSERT(A) ((void) 1)
-#endif
 #else /* #ifdef ERTS_ENABLE_LOCK_CHECK */
 #define ERTS_SMP_LC_ASSERT(A) ((void) 1)
 #define ERTS_LC_ASSERT(A) ((void) 1)
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index c1af70592a..b35c14bfc8 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -39,11 +39,7 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message_ref,
 				 ERL_MESSAGE_BUF_SZ,
 				 ERTS_ALC_T_MSG_REF)
 
-#if defined(DEBUG) && 0
-#define HARD_DEBUG
-#else
 #undef HARD_DEBUG
-#endif
 
 void
 init_message(void)
@@ -265,9 +261,7 @@ erts_queue_dist_message(Process *rcvr,
     Sint tok_lastcnt = 0;
     Sint tok_serial = 0;
 #endif
-#ifdef ERTS_SMP
     erts_aint_t state;
-#endif
 
     ERTS_SMP_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
 
@@ -283,7 +277,6 @@ erts_queue_dist_message(Process *rcvr,
 #endif
 	ERL_MESSAGE_TOKEN(mp) = token;
 
-#ifdef ERTS_SMP
     if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
 	if (erts_smp_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
 	    ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
@@ -305,7 +298,6 @@ erts_queue_dist_message(Process *rcvr,
 	erts_cleanup_messages(mp);
     }
     else
-#endif
     if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) {
         if (from == am_Empty)
             from = dist_ext->dep->sysname;
@@ -364,11 +356,7 @@ erts_queue_dist_message(Process *rcvr,
 	    erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
 
 	erts_proc_notify_new_message(rcvr,
-#ifdef ERTS_SMP
 				     rcvr_locks
-#else
-				     0
-#endif
 	    );
     }
 }
@@ -393,7 +381,6 @@ queue_messages(Process* receiver,
            ERL_MESSAGE_TOKEN(first) == NIL ||
            is_tuple(ERL_MESSAGE_TOKEN(first)));
 
-#ifdef ERTS_SMP
 #ifdef ERTS_ENABLE_LOCK_CHECK
     ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(receiver) < ERTS_PROC_LOCK_MSGQ ||
                        receiver_locks == erts_proc_lc_my_proc_locks(receiver));
@@ -420,14 +407,11 @@ queue_messages(Process* receiver,
 	locked_msgq = 1;
     }
 
-#endif
 
     state = erts_smp_atomic32_read_nob(&receiver->state);
 
     if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
-#ifdef ERTS_SMP
     exiting:
-#endif
 	/* Drop message if receiver is exiting or has a pending exit... */
 	if (locked_msgq)
 	    erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
@@ -436,7 +420,6 @@ queue_messages(Process* receiver,
     }
 
     res = receiver->msg.len;
-#ifdef ERTS_SMP
     if (receiver_locks & ERTS_PROC_LOCK_MAIN) {
 	/*
 	 * We move 'in queue' to 'private queue' and place
@@ -451,7 +434,6 @@ queue_messages(Process* receiver,
         LINK_MESSAGE_PRIVQ(receiver, first, last, len);
     }
     else
-#endif
     {
 	LINK_MESSAGE(receiver, first, last, len);
     }
@@ -492,11 +474,7 @@ queue_messages(Process* receiver,
 	erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
     }
 
-#ifdef ERTS_SMP
     erts_proc_notify_new_message(receiver, receiver_locks);
-#else
-    erts_proc_notify_new_message(receiver, 0);
-#endif
     return res;
 }
 
@@ -597,9 +575,7 @@ erts_try_alloc_message_on_heap(Process *pp,
 			       ErlOffHeap **ohpp,
 			       int *on_heap_p)
 {
-#ifdef ERTS_SMP
     int locked_main = 0;
-#endif
     ErtsMessage *mp;
 
     ASSERT(!(*psp & ERTS_PSFLG_OFF_HEAP_MSGQ));
@@ -607,15 +583,9 @@ erts_try_alloc_message_on_heap(Process *pp,
     if ((*psp) & ERTS_PSFLGS_VOLATILE_HEAP)
 	goto in_message_fragment;
     else if (
-#if defined(ERTS_SMP)
 	*plp & ERTS_PROC_LOCK_MAIN
-#else
-	pp
-#endif
 	) {
-#ifdef ERTS_SMP
     try_on_heap:
-#endif
 	if (((*psp) & ERTS_PSFLGS_VOLATILE_HEAP)
 	    || (pp->flags & F_DISABLE_GC)
 	    || HEAP_LIMIT(pp) - HEAP_TOP(pp) <= sz) {
@@ -623,12 +593,10 @@ erts_try_alloc_message_on_heap(Process *pp,
 	     * The heap is either potentially in an inconsistent
 	     * state, or not large enough.
 	     */
-#ifdef ERTS_SMP
 	    if (locked_main) {
 		*plp &= ~ERTS_PROC_LOCK_MAIN;
 		erts_smp_proc_unlock(pp, ERTS_PROC_LOCK_MAIN);
 	    }
-#endif
 	    goto in_message_fragment;
 	}
 
@@ -639,14 +607,12 @@ erts_try_alloc_message_on_heap(Process *pp,
 	mp->data.attached = NULL;
 	*on_heap_p = !0;
     }
-#ifdef ERTS_SMP
     else if (pp && erts_smp_proc_trylock(pp, ERTS_PROC_LOCK_MAIN) == 0) {
 	locked_main = 1;
 	*psp = erts_smp_atomic32_read_nob(&pp->state);
 	*plp |= ERTS_PROC_LOCK_MAIN;
 	goto try_on_heap;
     }
-#endif
     else {
     in_message_fragment:
 	if (!((*psp) & ERTS_PSFLG_ON_HEAP_MSGQ)) {
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 42ed14e69c..1f95ffaa5a 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -174,7 +174,6 @@ typedef struct {
     ErtsMessage** saved_last;	/* saved last pointer */
 } ErlMessageQueue;
 
-#ifdef ERTS_SMP
 
 typedef struct {
     ErtsMessage* first;
@@ -190,7 +189,6 @@ typedef struct erl_trace_message_queue__ {
     Sint len;            /* queue length */
 } ErlTraceMessageQueue;
 
-#endif
 
 /* Get "current" message */
 #define PEEK_MESSAGE(p)  (*(p)->msg.save)
@@ -207,7 +205,6 @@ typedef struct erl_trace_message_queue__ {
         (p)->where.len += (num_msgs);                                   \
     } while(0)
 
-#ifdef ERTS_SMP
 
 /* Add message last in private message queue */
 #define LINK_MESSAGE_PRIVQ(p, first_msg, last_msg, len)                 \
@@ -231,17 +228,6 @@ typedef struct erl_trace_message_queue__ {
         }                                               \
     } while (0)
 
-#else
-
-#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p)
-
-/* Add message last_msg in message queue */
-#define LINK_MESSAGE(p, first_msg, last_msg, len)                       \
-    do {                                                                \
-        LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg);            \
-    } while(0)
-
-#endif
 
 /* Unlink current message */
 #define UNLINK_MESSAGE(p,msgp) do { \
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index 6c477be615..957358bcf5 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -48,11 +48,7 @@ static Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, ErtsHeapFactory *factory)
 static void erts_msacc_reset(ErtsMsAcc *msacc);
 static ErtsMsAcc* get_msacc(void);
 
-#ifdef USE_THREADS
 erts_tsd_key_t ERTS_WRITE_UNLIKELY(erts_msacc_key);
-#else
-ErtsMsAcc *ERTS_WRITE_UNLIKELY(erts_msacc) = NULL;
-#endif
 #ifndef ERTS_MSACC_ALWAYS_ON
 int ERTS_WRITE_UNLIKELY(erts_msacc_enabled);
 #endif
@@ -60,10 +56,8 @@ int ERTS_WRITE_UNLIKELY(erts_msacc_enabled);
 static Eterm *erts_msacc_state_atoms = NULL;
 static erts_rwmtx_t msacc_mutex;
 static ErtsMsAcc *msacc_managed = NULL;
-#ifdef USE_THREADS
 static ErtsMsAcc *msacc_unmanaged = NULL;
 static Uint msacc_unmanaged_count = 0;
-#endif
 
 #if ERTS_MSACC_STATE_COUNT < MAP_SMALL_MAP_LIMIT
 #define DEFAULT_MSACC_MSG_SIZE (3 + 1 + ERTS_MSACC_STATE_COUNT * 2 + 3 + ERTS_REF_THING_SIZE)
@@ -78,11 +72,7 @@ void erts_msacc_early_init(void) {
 #endif
     erts_rwmtx_init(&msacc_mutex, "msacc_list_mutex", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
-#ifdef USE_THREADS
     erts_tsd_key_create(&erts_msacc_key,"erts_msacc_key");
-#else
-    erts_msacc = NULL;
-#endif
 }
 
 void erts_msacc_init(void) {
@@ -107,7 +97,6 @@ void erts_msacc_init_thread(char *type, int id, int managed) {
     msacc->tid = erts_thr_self();
     msacc->perf_counter = 0;
 
-#ifdef USE_THREADS
     erts_rwmtx_rwlock(&msacc_mutex);
     if (!managed) {
         erts_mtx_init(&msacc->mtx, "msacc_unmanaged_mutex", NIL,
@@ -121,9 +110,6 @@ void erts_msacc_init_thread(char *type, int id, int managed) {
         msacc_managed = msacc;
     }
     erts_rwmtx_rwunlock(&msacc_mutex);
-#else
-    msacc_managed = msacc;
-#endif
 
     erts_msacc_reset(msacc);
 
@@ -370,12 +356,8 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads)
     msaccrp->ref = STORE_NC(&hp, NULL, ref);
     msaccrp->req_sched = esdp->no;
 
-#ifdef ERTS_SMP
     *threads = erts_no_schedulers;
     *threads += 1; /* aux thread */
-#else
-    *threads = 1;
-#endif
 
     erts_smp_atomic32_init_nob(&msaccrp->refc,(erts_aint32_t)*threads);
 
@@ -386,12 +368,9 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads)
                                           erts_no_schedulers,
                                           reply_msacc,
                                           (void *) msaccrp);
-#ifdef ERTS_SMP
     /* aux thread */
     erts_schedule_misc_aux_work(0, reply_msacc, (void *) msaccrp);
-#endif
 
-#ifdef USE_THREADS
     /* Manage unmanaged threads */
     switch (action) {
     case ERTS_MSACC_GATHER: {
@@ -468,7 +447,6 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads)
     default: { ASSERT(0); }
     }
 
-#endif
 
     *threads = make_small(*threads);
 
diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h
index d64ef8c8b9..8349a7e297 100644
--- a/erts/emulator/beam/erl_msacc.h
+++ b/erts/emulator/beam/erl_msacc.h
@@ -159,11 +159,7 @@ struct erl_msacc_t_ {
 
 #ifdef ERTS_ENABLE_MSACC
 
-#ifdef USE_THREADS
 extern erts_tsd_key_t erts_msacc_key;
-#else
-extern ErtsMsAcc *erts_msacc;
-#endif
 
 #ifdef ERTS_MSACC_ALWAYS_ON
 #define erts_msacc_enabled 1
@@ -171,13 +167,8 @@ extern ErtsMsAcc *erts_msacc;
 extern int erts_msacc_enabled;
 #endif
 
-#ifdef USE_THREADS
 #define ERTS_MSACC_TSD_GET() erts_tsd_get(erts_msacc_key)
 #define ERTS_MSACC_TSD_SET(tsd) erts_tsd_set(erts_msacc_key,tsd)
-#else
-#define ERTS_MSACC_TSD_GET() erts_msacc
-#define ERTS_MSACC_TSD_SET(tsd) erts_msacc = tsd
-#endif
 
 void erts_msacc_early_init(void);
 void erts_msacc_init(void);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 848e116621..b586a4636a 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -564,7 +564,6 @@ void enif_clear_env(ErlNifEnv* env)
     free_tmp_objs(env);
 }
 
-#ifdef ERTS_SMP
 #ifdef DEBUG
 static int enif_send_delay = 0;
 #define ERTS_FORCE_ENIF_SEND_DELAY() (enif_send_delay++ % 2 == 0)
@@ -648,16 +647,13 @@ error:
     return reds;
 }
 
-#endif
 
 int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 	      ErlNifEnv* msg_env, ERL_NIF_TERM msg)
 {
     struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)msg_env;
     ErtsProcLocks rp_locks = 0;
-#ifdef ERTS_SMP
     ErtsProcLocks lc_locks = 0;
-#endif
     Process* rp;
     Process* c_p;
     ErtsMessage *mp;
@@ -666,13 +662,6 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 
     execution_state(env, &c_p, &scheduler);
 
-#ifndef ERTS_SMP
-    if (!scheduler) {
-	erts_exit(ERTS_ABORT_EXIT,
-		  "enif_send: called from non-scheduler thread on non-SMP VM");
-	return 0;
-   }
-#endif
 
     if (scheduler > 0) { /* Normal scheduler */
 	rp = erts_proc_lookup(receiver);
@@ -756,7 +745,6 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 	    full_cache_env(env);
 	}
     }
-#ifdef ERTS_SMP
     else {
         /* This clause is taken when the nif is called in the context
            of a traced process. We do not know which locks we have
@@ -814,12 +802,10 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
             rp_locks |= ERTS_PROC_LOCK_MSGQ;
         }
     }
-#endif /* ERTS_SMP */
 
     erts_queue_message(rp, rp_locks, mp, msg,
                        c_p ? c_p->common.id : am_undefined);
 
-#ifdef ERTS_SMP
 done:
     if (c_p == rp)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -827,7 +813,6 @@ done:
 	erts_smp_proc_unlock(rp, rp_locks & ~lc_locks);
     if (c_p && (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC))
 	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
-#endif
     if (scheduler <= 0)
 	erts_proc_dec_refc(rp);
 
@@ -857,15 +842,9 @@ enif_port_command(ErlNifEnv *env, const ErlNifPort* to_port,
     if (scheduler > 0)
 	prt = erts_port_lookup(to_port->port_id, iflags);
     else {
-#ifdef ERTS_SMP
 	if (ERTS_PROC_IS_EXITING(c_p))
 	    return 0;
 	prt = erts_thr_port_lookup(to_port->port_id, iflags);
-#else
-        erts_exit(ERTS_ABORT_EXIT,
-		  "enif_port_command: called from non-scheduler "
-                  "thread on non-SMP VM");
-#endif
     }
 
     if (!prt)
@@ -1847,18 +1826,11 @@ int enif_is_process_alive(ErlNifEnv* env, ErlNifPid *proc)
     if (scheduler > 0)
 	return !!erts_proc_lookup(proc->pid);
     else {
-#ifdef ERTS_SMP
 	Process* rp = erts_pid2proc_opt(NULL, 0, proc->pid, 0,
 					ERTS_P2P_FLG_INC_REFC);
 	if (rp)
 	    erts_proc_dec_refc(rp);
 	return !!rp;
-#else
-	erts_exit(ERTS_ABORT_EXIT, "enif_is_process_alive: "
-		  "called from non-scheduler thread "
-                  "in non-smp emulator");
-	return 0;
-#endif
     }
 }
 
@@ -1874,17 +1846,10 @@ int enif_is_port_alive(ErlNifEnv *env, ErlNifPort *port)
     if (scheduler > 0)
 	return !!erts_port_lookup(port->port_id, iflags);
     else {
-#ifdef ERTS_SMP
 	Port *prt = erts_thr_port_lookup(port->port_id, iflags);
 	if (prt)
 	    erts_port_dec_refc(prt);
 	return !!prt;
-#else
-	erts_exit(ERTS_ABORT_EXIT, "enif_is_port_alive: "
-		  "called from non-scheduler thread "
-                  "in non-smp emulator");
-	return 0;
-#endif
     }
 }
 
@@ -2231,12 +2196,7 @@ static void destroy_one_monitor(ErtsMonitor* mon, void* context)
         rp = erts_proc_lookup(mon->u.pid);
     }
     else {
-#ifdef ERTS_SMP
         rp = erts_proc_lookup_inc_refc(mon->u.pid);
-#else
-        ASSERT(!"nif monitor destruction in non-scheduler thread");
-        rp = NULL;
-#endif
     }
 
     if (!rp) {
@@ -2252,10 +2212,8 @@ static void destroy_one_monitor(ErtsMonitor* mon, void* context)
             is_exiting = 0;
         }
         erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-#ifdef ERTS_SMP
         if (ctx->scheduler <= 0)
             erts_proc_dec_refc(rp);
-#endif
     }
     if (is_exiting) {
         ctx->resource->monitors->pending_failed_fire++;
@@ -2281,34 +2239,7 @@ static void destroy_all_monitors(ErtsMonitor* monitors, ErtsResource* resource)
 }
 
 
-#ifdef ERTS_SMP
 #  define NIF_RESOURCE_DTOR &nif_resource_dtor
-#else
-#  define NIF_RESOURCE_DTOR &nosmp_nif_resource_dtor_prologue
-
-/*
- * NO-SMP: Always run resource destructor on scheduler thread 
- *         as we may have to remove process monitors.
- */
-static int nif_resource_dtor(Binary*);
-
-static void nosmp_nif_resource_dtor_scheduled(void* vbin)
-{
-    erts_bin_free((Binary*)vbin);
-}
-
-static int nosmp_nif_resource_dtor_prologue(Binary* bin)
-{
-    if (is_scheduler()) {
-        return nif_resource_dtor(bin);
-    }
-    else {
-        erts_schedule_misc_aux_work(1, nosmp_nif_resource_dtor_scheduled, bin);
-        return 0; /* do not free */
-    }
-}
-
-#endif /* !ERTS_SMP */
 
 static int nif_resource_dtor(Binary* bin)
 {
@@ -3216,18 +3147,10 @@ int enif_monitor_process(ErlNifEnv* env, void* obj, const ErlNifPid* target_pid,
 
     execution_state(env, NULL, &scheduler);
 
-#ifdef ERTS_SMP
     if (scheduler > 0) /* Normal scheduler */
         rp = erts_proc_lookup_raw(target_pid->pid);
     else
         rp = erts_proc_lookup_raw_inc_refc(target_pid->pid);
-#else
-    if (scheduler <= 0) {
-        erts_exit(ERTS_ABORT_EXIT, "enif_monitor_process: called from "
-                  "non-scheduler thread on non-SMP VM");
-    }
-    rp = erts_proc_lookup(target_pid->pid);
-#endif
 
     if (!rp)
         return 1;
@@ -3247,10 +3170,8 @@ int enif_monitor_process(ErlNifEnv* env, void* obj, const ErlNifPid* target_pid,
     erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
     erts_smp_mtx_unlock(&rsrc->monitors->lock);
 
-#ifdef ERTS_SMP
     if (scheduler <= 0)
         erts_proc_dec_refc(rp);
-#endif
     if (monitor)
         erts_ref_to_driver_monitor(ref,monitor);
 
@@ -3289,18 +3210,10 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit
     ASSERT(mon->type == MON_ORIGIN);
     ASSERT(is_internal_pid(mon->u.pid));
 
-#ifdef ERTS_SMP
     if (scheduler > 0) /* Normal scheduler */
         rp = erts_proc_lookup(mon->u.pid);
     else
         rp = erts_proc_lookup_inc_refc(mon->u.pid);
-#else
-    if (scheduler <= 0) {
-        erts_exit(ERTS_ABORT_EXIT, "enif_demonitor_process: called from "
-                  "non-scheduler thread on non-SMP VM");
-    }
-    rp = erts_proc_lookup(mon->u.pid);
-#endif
 
     if (!rp) {
         is_exiting = 1;
@@ -3316,10 +3229,8 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit
         }
         erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 
-#ifdef ERTS_SMP
         if (scheduler <= 0)
             erts_proc_dec_refc(rp);
-#endif
     }
     if (is_exiting) {
         rsrc->monitors->pending_failed_fire++;
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index deadf435e9..c12f231a3b 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -808,7 +808,6 @@ void erts_init_node_tables(int dd_sec)
     references_atoms_need_init = 1;
 }
 
-#ifdef ERTS_SMP
 #ifdef ERTS_ENABLE_LOCK_CHECK
 int erts_lc_is_de_rwlocked(DistEntry *dep)
 {
@@ -819,7 +818,6 @@ int erts_lc_is_de_rlocked(DistEntry *dep)
     return erts_smp_lc_rwmtx_is_rlocked(&dep->rwmtx);
 }
 #endif
-#endif
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
 
@@ -1292,13 +1290,11 @@ init_referred_dist(void *dist, void *unused)
     no_referred_dists++;
 }
 
-#ifdef ERTS_SMP
 static void
 insert_sys_msg(Eterm from, Eterm to, Eterm msg, ErlHeapFragment *bp)
 {
     insert_offheap(&bp->off_heap, HEAP_REF, to);
 }
-#endif
 
 static void
 insert_delayed_delete_node(void *state,
@@ -1381,9 +1377,7 @@ setup_reference_table(void)
 	    int mli;
 	    ErtsMessage *msg_list[] = {
 		proc->msg.first,
-#ifdef ERTS_SMP
 		proc->msg_inq.first,
-#endif
 		proc->msg_frag};
 
 	    /* Insert Heap */
@@ -1430,9 +1424,7 @@ setup_reference_table(void)
 	}
     }
     
-#ifdef ERTS_SMP
     erts_foreach_sys_msg_in_q(insert_sys_msg);
-#endif
 
     /* Insert all ports */
     max = erts_ptab_max(&erts_port);
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index 6a3213ec52..4c8e4197aa 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -131,9 +131,7 @@ typedef struct {
     void *data[ERTS_PRTSD_SIZE];
 } ErtsPrtSD;
 
-#ifdef ERTS_SMP
 typedef struct ErtsXPortsList_ ErtsXPortsList;
-#endif
 
 /*
  * Port locking:
@@ -158,14 +156,9 @@ struct _erl_drv_port {
 
     ErtsPortTaskSched sched;
     ErtsPortTaskHandle timeout_task;
-#ifdef ERTS_SMP
     erts_mtx_t *lock;
     ErtsXPortsList *xports;
     erts_smp_atomic_t run_queue;
-#else
-    erts_atomic32_t refc;
-    int cleanup;
-#endif
     erts_atomic_t connected;	/* A connected process */
     Eterm caller;		/* Current caller. */
     erts_smp_atomic_t data;	/* Data associated with port. */
@@ -221,7 +214,6 @@ ERTS_GLB_INLINE ErtsRunQueue *erts_port_runq(Port *prt);
 ERTS_GLB_INLINE ErtsRunQueue *
 erts_port_runq(Port *prt)
 {
-#ifdef ERTS_SMP
     ErtsRunQueue *rq1, *rq2;
     rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
     if (!rq1)
@@ -236,9 +228,6 @@ erts_port_runq(Port *prt)
 	if (!rq1)
 	    return NULL;
     }
-#else
-    return ERTS_RUNQ_IX(0);
-#endif
 }
 
 #endif
@@ -269,12 +258,10 @@ erts_prtsd_set(Port *prt, int ix, void *data)
     psd = (ErtsPrtSD *) erts_smp_atomic_read_nob(&prt->psd);
 
     if (psd) {
-#ifdef ERTS_SMP
 #ifdef ETHR_ORDERED_READ_DEPEND
 	ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
 #else
 	ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreStore);
-#endif
 #endif
 	old = psd->data[ix];
 	psd->data[ix] = data;
@@ -371,13 +358,8 @@ Eterm erts_request_io_bytes(Process *c_p);
 
 void print_port_info(Port *, fmtfn_t, void *);
 void erts_port_free(Port *);
-#ifndef ERTS_SMP
-void erts_port_cleanup(Port *);
-#endif
 void erts_fire_port_monitor(Port *prt, Eterm ref);
-#ifdef ERTS_SMP
 int erts_port_handle_xports(Port *);
-#endif
 
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
 int erts_lc_is_port_locked(Port *);
@@ -421,33 +403,25 @@ ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt)
 ERTS_GLB_INLINE int
 erts_smp_port_trylock(Port *prt)
 {
-#ifdef ERTS_SMP
     /* *Need* to be a managed thread */
     ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
     return erts_mtx_trylock(prt->lock);
-#else
-    return 0;
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_port_lock(Port *prt)
 {
-#ifdef ERTS_SMP
     /* *Need* to be a managed thread */
     ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
     erts_mtx_lock(prt->lock);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_port_unlock(Port *prt)
 {
-#ifdef ERTS_SMP
     /* *Need* to be a managed thread */
     ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
     erts_mtx_unlock(prt->lock);
-#endif
 }
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
@@ -478,9 +452,7 @@ extern const Port erts_invalid_port;
 int erts_is_port_ioq_empty(Port *);
 void erts_terminate_port(Port *);
 
-#ifdef ERTS_SMP
 Port *erts_de2port(DistEntry *, Process *, ErtsProcLocks);
-#endif
 
 ERTS_GLB_INLINE Port *erts_pix2port(int);
 ERTS_GLB_INLINE Port *erts_port_lookup_raw(Eterm);
@@ -488,11 +460,9 @@ ERTS_GLB_INLINE Port *erts_port_lookup(Eterm, Uint32);
 ERTS_GLB_INLINE Port*erts_id2port(Eterm id);
 ERTS_GLB_INLINE Port *erts_id2port_sflgs(Eterm, Process *, ErtsProcLocks, Uint32);
 ERTS_GLB_INLINE void erts_port_release(Port *);
-#ifdef ERTS_SMP
 ERTS_GLB_INLINE Port *erts_thr_port_lookup(Eterm id, Uint32 invalid_sflgs);
 ERTS_GLB_INLINE Port *erts_thr_id2port_sflgs(Eterm id, Uint32 invalid_sflgs);
 ERTS_GLB_INLINE void erts_thr_port_release(Port *prt);
-#endif
 ERTS_GLB_INLINE Port *erts_thr_drvport2port(ErlDrvPort, int);
 ERTS_GLB_INLINE Port *erts_drvport2port_state(ErlDrvPort, erts_aint32_t *);
 ERTS_GLB_INLINE Eterm erts_drvport2id(ErlDrvPort);
@@ -574,9 +544,7 @@ erts_id2port_sflgs(Eterm id,
 		   Process *c_p, ErtsProcLocks c_p_locks,
 		   Uint32 invalid_sflgs)
 {
-#ifdef ERTS_SMP
     int no_proc_locks = !c_p || !c_p_locks;
-#endif
     erts_aint32_t state;
     Port *prt;
 
@@ -592,7 +560,6 @@ erts_id2port_sflgs(Eterm id,
     if (!prt || prt->common.id != id)
 	return NULL;
 
-#ifdef ERTS_SMP
     if (no_proc_locks)
 	erts_smp_port_lock(prt);
     else if (erts_smp_port_trylock(prt) == EBUSY) {
@@ -601,12 +568,9 @@ erts_id2port_sflgs(Eterm id,
 	erts_smp_port_lock(prt);
 	erts_smp_proc_lock(c_p, c_p_locks);
     }
-#endif
     state = erts_atomic32_read_nob(&prt->state);
     if (state & invalid_sflgs) {
-#ifdef ERTS_SMP
 	erts_smp_port_unlock(prt);
-#endif
 	return NULL;
     }
 
@@ -618,17 +582,9 @@ erts_port_release(Port *prt)
 {
     /* Only allowed to be called from managed threads */
     ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
-#ifdef ERTS_SMP
     erts_smp_port_unlock(prt);
-#else
-    if (prt->cleanup) {
-	prt->cleanup = 0;
-	erts_port_cleanup(prt);
-    }
-#endif
 }
 
-#ifdef ERTS_SMP
 /*
  * erts_thr_id2port_sflgs() and erts_port_dec_refc(prt) can
  * be used by unmanaged threads in the SMP case.
@@ -714,13 +670,10 @@ ERTS_GLB_INLINE void
 erts_thr_port_release(Port *prt)
 {
     erts_mtx_unlock(prt->lock);
-#ifdef ERTS_SMP
     if (!erts_thr_progress_is_managed_thread())
 	erts_port_dec_refc(prt);
-#endif
 }
 
-#endif
 
 ERTS_GLB_INLINE Port *
 erts_thr_drvport2port(ErlDrvPort drvport, int lock_pdl)
@@ -832,13 +785,11 @@ erts_port_driver_callback_epilogue(Port *prt, erts_aint32_t *statep)
 	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
     }
 
-#ifdef ERTS_SMP
     if (prt->xports) {
 	reds += erts_port_handle_xports(prt);
 	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 	ASSERT(!prt->xports);
     }
-#endif
 
     if (statep)
 	*statep = state;
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 1ab1e47254..5f333ea8d8 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -43,11 +43,7 @@
  */
 #define ERTS_PORT_CALLBACK_VREDS (CONTEXT_REDS/20)
 
-#if defined(DEBUG) && 0
-#define ERTS_HARD_DEBUG_TASK_QUEUES
-#else
 #undef ERTS_HARD_DEBUG_TASK_QUEUES
-#endif
 
 #ifdef ERTS_HARD_DEBUG_TASK_QUEUES
 static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_queue);
@@ -126,9 +122,7 @@ struct ErtsPortTaskHandleList_ {
     ErtsPortTaskHandle handle;
     union {
 	ErtsPortTaskHandleList *next;
-#ifdef ERTS_SMP
 	ErtsThrPrgrLaterOp release;
-#endif
     } u;
 };
 
@@ -161,25 +155,19 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(busy_caller_table,
 				 50,
 				 ERTS_ALC_T_BUSY_CALLER_TAB)
 
-#ifdef ERTS_SMP
 static void
 call_port_task_free(void *vptp)
 {
     port_task_free((ErtsPortTask *) vptp);
 }
-#endif
 
 static ERTS_INLINE void
 schedule_port_task_free(ErtsPortTask *ptp)
 {
-#ifdef ERTS_SMP
     erts_schedule_thr_prgr_later_cleanup_op(call_port_task_free,
 					    (void *) ptp,
 					    &ptp->u.release,
 					    sizeof(ErtsPortTask));
-#else
-    port_task_free(ptp);
-#endif
 }
 
 static ERTS_INLINE ErtsPortTask *
@@ -830,25 +818,19 @@ erl_drv_busy_msgq_limits(ErlDrvPort dport, ErlDrvSizeT *lowp, ErlDrvSizeT *highp
  * No-suspend handles.
  */
 
-#ifdef ERTS_SMP
 static void
 free_port_task_handle_list(void *vpthlp)
 {
     erts_free(ERTS_ALC_T_PT_HNDL_LIST, vpthlp);
 }
-#endif
 
 static void
 schedule_port_task_handle_list_free(ErtsPortTaskHandleList *pthlp)
 {
-#ifdef ERTS_SMP
     erts_schedule_thr_prgr_later_cleanup_op(free_port_task_handle_list,
 					    (void *) pthlp,
 					    &pthlp->u.release,
 					    sizeof(ErtsPortTaskHandleList));
-#else
-    erts_free(ERTS_ALC_T_PT_HNDL_LIST, pthlp);
-#endif
 }
 
 static ERTS_INLINE void
@@ -946,10 +928,8 @@ enqueue_port(ErtsRunQueue *runq, Port *pp)
 
     erts_smp_inc_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL);
 
-#ifdef ERTS_SMP
     if (ERTS_RUNQ_FLGS_GET_NOB(runq) & ERTS_RUNQ_FLG_HALTING)
 	erts_non_empty_runq(runq);
-#endif
 }
 
 static ERTS_INLINE Port *
@@ -1301,9 +1281,7 @@ erts_port_task_abort(ErtsPortTaskHandle *pthp)
 {
     int res;
     ErtsPortTask *ptp;
-#ifdef ERTS_SMP
     ErtsThrPrgrDelayHandle dhndl = erts_thr_progress_unmanaged_delay();
-#endif
 
     ptp = handle2task(pthp);
     if (!ptp)
@@ -1345,9 +1323,7 @@ erts_port_task_abort(ErtsPortTaskHandle *pthp)
 	}
     }
 
-#ifdef ERTS_SMP
     erts_thr_progress_unmanaged_continue(dhndl);
-#endif
 
     return res;
 }
@@ -1356,9 +1332,7 @@ void
 erts_port_task_abort_nosuspend_tasks(Port *pp)
 {
     ErtsPortTaskHandleList *abort_list;
-#ifdef ERTS_SMP
     ErtsThrPrgrDelayHandle dhndl = ERTS_THR_PRGR_DHANDLE_INVALID;
-#endif
 
     erts_port_task_sched_lock(&pp->sched);
     erts_smp_atomic32_read_band_nob(&pp->sched.flags,
@@ -1381,18 +1355,14 @@ erts_port_task_abort_nosuspend_tasks(Port *pp)
 	pthlp = abort_list;
 	abort_list = pthlp->u.next;
 
-#ifdef ERTS_SMP
 	if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
 	    dhndl = erts_thr_progress_unmanaged_delay();
-#endif
 
 	pthp = &pthlp->handle;
 	ptp = handle2task(pthp);
 	if (!ptp) {
-#ifdef ERTS_SMP
 	    if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
 		erts_thr_progress_unmanaged_continue(dhndl);
-#endif
 	    schedule_port_task_handle_list_free(pthlp);
 	    continue;
 	}
@@ -1411,10 +1381,8 @@ erts_port_task_abort_nosuspend_tasks(Port *pp)
 						  ERTS_PT_STATE_SCHEDULED);
 	if (old_state != ERTS_PT_STATE_SCHEDULED) {
 	    /* Task already aborted, executing, or executed */
-#ifdef ERTS_SMP
 	    if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
 		erts_thr_progress_unmanaged_continue(dhndl);
-#endif
 	    schedule_port_task_handle_list_free(pthlp);
 	    continue;
 	}
@@ -1424,10 +1392,8 @@ erts_port_task_abort_nosuspend_tasks(Port *pp)
 	type = ptp->type;
 	td = ptp->u.alive.td;
 
-#ifdef ERTS_SMP
 	if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
 	    erts_thr_progress_unmanaged_continue(dhndl);
-#endif
 	schedule_port_task_handle_list_free(pthlp);
 
 	abort_nosuspend_task(pp, type, &td, pp->sched.taskq.bpq != NULL);
@@ -1446,10 +1412,8 @@ erts_port_task_schedule(Eterm id,
 {
     ErtsProc2PortSigData *sigdp = NULL;
     ErtsPortTaskHandleList *ns_pthlp = NULL;
-#ifdef ERTS_SMP
     ErtsRunQueue *xrunq;
     ErtsThrPrgrDelayHandle dhndl;
-#endif
     ErtsRunQueue *runq;
     Port *pp;
     ErtsPortTask *ptp = NULL;
@@ -1460,19 +1424,15 @@ erts_port_task_schedule(Eterm id,
 
     ASSERT(is_internal_port(id));
 
-#ifdef ERTS_SMP
     dhndl = erts_thr_progress_unmanaged_delay();
-#endif
 
     pp = erts_port_lookup_raw(id);
 
-#ifdef ERTS_SMP
     if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) {
 	if (pp)
 	    erts_port_inc_refc(pp);
 	erts_thr_progress_unmanaged_continue(dhndl);
     }
-#endif
 
     if (!pp)
 	goto fail;
@@ -1581,7 +1541,6 @@ erts_port_task_schedule(Eterm id,
     if (!runq)
 	ERTS_INTERNAL_ERROR("Missing run-queue");
 
-#ifdef ERTS_SMP
     xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL);
     ERTS_SMP_LC_ASSERT(runq != xrunq);
     ERTS_SMP_LC_VERIFY_RQ(runq, pp);
@@ -1593,7 +1552,6 @@ erts_port_task_schedule(Eterm id,
 	if (!runq)
 	    ERTS_INTERNAL_ERROR("Missing run-queue");
     }
-#endif
 
     enqueue_port(runq, pp);
 
@@ -1606,19 +1564,15 @@ done:
     if (prof_runnable_ports)
 	erts_port_task_sched_unlock(&pp->sched);
 
-#ifdef ERTS_SMP
     if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
 	erts_port_dec_refc(pp);
-#endif
 
     return 0;
 
 abort_nosuspend:
 
-#ifdef ERTS_SMP
     if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
 	erts_port_dec_refc(pp);
-#endif
 
     abort_nosuspend_task(pp, ptp->type, &ptp->u.alive.td, 0);
 
@@ -1632,10 +1586,8 @@ abort_nosuspend:
 
 fail:
 
-#ifdef ERTS_SMP
     if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
 	erts_port_dec_refc(pp);
-#endif
 
     if (ptp) {
         abort_signal_task(pp, ERTS_PROC2PORT_SIG_ABORT,
@@ -1892,9 +1844,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 				 -1*io_tasks_executed);
     }
 
-#ifdef ERTS_SMP
     ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue));
-#endif
 
     active = finalize_exec(pp, &execq, processing_busy_q);
 
@@ -1907,21 +1857,16 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     erts_smp_runq_lock(runq);
  
     if (active) {
-#ifdef ERTS_SMP
 	ErtsRunQueue *xrunq;
-#endif
 
 	ASSERT(!(erts_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD));
 
-#ifdef ERTS_SMP
 	xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL);
 	ERTS_SMP_LC_ASSERT(runq != xrunq);
 	ERTS_SMP_LC_VERIFY_RQ(runq, pp);
 	if (!xrunq) {
-#endif
 	    enqueue_port(runq, pp);
 	    /* No need to notify ourselves about inc in runq. */
-#ifdef ERTS_SMP
 	}
 	else {
 	    /* Emigrate port... */
@@ -1936,7 +1881,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 
 	    erts_smp_runq_lock(runq);
 	}
-#endif
     }
 
  done:
@@ -1951,7 +1895,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     return res;
 }
 
-#ifdef ERTS_SMP
 static void
 release_port(void *vport)
 {
@@ -1967,7 +1910,6 @@ schedule_release_port(void *vport) {
 				  &pp->common.u.release);
 }
 
-#endif
 
 static void
 begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
@@ -2161,7 +2103,6 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
     /*
      * Schedule cleanup of port structure...
      */
-#ifdef ERTS_SMP
     /* We might not be a scheduler, eg. traceing to port we are sys_msg_dispatcher */
     if (!erts_get_scheduler_data()) {
       erts_schedule_misc_aux_work(1, schedule_release_port, (void*)pp);
@@ -2171,12 +2112,8 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
 				      (void *) pp,
 				      &pp->common.u.release);
     }
-#else
-    pp->cleanup = 1;
-#endif
 }
 
-#ifdef ERTS_SMP
 
 void
 erts_enqueue_port(ErtsRunQueue *rq, Port *pp)
@@ -2200,7 +2137,6 @@ erts_dequeue_port(ErtsRunQueue *rq)
     return pp;
 }
 
-#endif
 
 /*
  * Initialize the module.
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index 39f403b443..14db641bb6 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -125,9 +125,7 @@ typedef struct {
 	ErtsPortTaskBusyPortQ *bpq;
     } taskq;
     erts_smp_atomic32_t flags;
-#ifdef ERTS_SMP
     erts_mtx_t mtx;
-#endif
 } ErtsPortTaskSched;
 
 ERTS_GLB_INLINE void erts_port_task_handle_init(ErtsPortTaskHandle *pthp);
@@ -175,9 +173,7 @@ ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp,
 ERTS_GLB_INLINE void
 erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id)
 {
-#ifdef ERTS_SMP
     char *lock_str = "port_sched_lock";
-#endif
     ptsp->next = NULL;
     ptsp->taskq.local.busy.first = NULL;
     ptsp->taskq.local.busy.last = NULL;
@@ -187,25 +183,19 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id)
     ptsp->taskq.in.first = NULL;
     ptsp->taskq.in.last = NULL;
     erts_smp_atomic32_init_nob(&ptsp->flags, 0);
-#ifdef ERTS_SMP
     erts_mtx_init(&ptsp->mtx, lock_str, instr_id, ERTS_LOCK_FLAGS_CATEGORY_IO);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_port_task_sched_lock(ErtsPortTaskSched *ptsp)
 {
-#ifdef ERTS_SMP
     erts_mtx_lock(&ptsp->mtx);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp)
 {
-#ifdef ERTS_SMP
     erts_mtx_unlock(&ptsp->mtx);
-#endif
 }
 
 ERTS_GLB_INLINE int
@@ -222,9 +212,7 @@ erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp)
 ERTS_GLB_INLINE void
 erts_port_task_fini_sched(ErtsPortTaskSched *ptsp)
 {
-#ifdef ERTS_SMP
     erts_mtx_destroy(&ptsp->mtx);
-#endif
 }
 
 ERTS_GLB_INLINE void
@@ -265,10 +253,8 @@ ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data(void);
 ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data_extra(size_t extra, void **extra_ptr);
 void erts_port_task_free_p2p_sig_data(ErtsProc2PortSigData *sigdp);
 
-#ifdef ERTS_SMP
 void erts_enqueue_port(ErtsRunQueue *rq, Port *pp);
 Port *erts_dequeue_port(ErtsRunQueue *rq);
-#endif
 #undef ERTS_INCLUDE_SCHEDULER_INTERNALS
 #endif /* ERL_PORT_TASK_H__ */
 #endif /* ERTS_PORT_TASK_ONLY_BASIC_TYPES__ */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index d58203e9e1..ba6d56269d 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -82,11 +82,7 @@
 #define ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA
 #endif
 
-#if defined(DEBUG) && 0
-#define HARDDEBUG
-#else
 #undef HARDDEBUG
-#endif
 
 #ifdef HARDDEBUG
 #define HARDDEBUG_RUNQS
@@ -127,13 +123,11 @@ runq_got_work_to_execute_flags(Uint32 flags)
     return !ERTS_IS_RUNQ_EMPTY_FLGS(flags);
 }
 
-#ifdef ERTS_SMP
 static ERTS_INLINE int
 runq_got_work_to_execute(ErtsRunQueue *rq)
 {
     return runq_got_work_to_execute_flags(ERTS_RUNQ_FLGS_GET_NOB(rq));
 }
-#endif
 
 #undef RUNQ_READ_RQ
 #undef RUNQ_SET_RQ
@@ -208,7 +202,6 @@ static struct {
     int sys_schedule;
 } sched_busy_wait;
 
-#ifdef ERTS_SMP
 int erts_disable_proc_not_running_opt;
 
 static ErtsAuxWorkData *aux_thread_aux_work_data;
@@ -384,26 +377,19 @@ do {							\
     balance_info.prev_rise.reds = (REDS);		\
 } while (0)
 
-#endif
 
 erts_sched_stat_t erts_sched_stat;
 
-#ifdef USE_THREADS
 static erts_tsd_key_t ERTS_WRITE_UNLIKELY(sched_data_key);
-#endif
 
 static erts_smp_atomic32_t function_calls;
 
-#ifdef ERTS_SMP
 static erts_smp_atomic32_t doing_sys_schedule;
 static erts_smp_atomic32_t no_empty_run_queues;
 long erts_runq_supervision_interval = 0;
 static ethr_event runq_supervision_event;
 static erts_tid_t runq_supervisor_tid;
 static erts_atomic_t runq_supervisor_sleeping;
-#else /* !ERTS_SMP */
-ErtsSchedulerData *erts_scheduler_data;
-#endif
 
 ErtsAlignedRunQueue * ERTS_WRITE_UNLIKELY(erts_aligned_run_queues);
 Uint ERTS_WRITE_UNLIKELY(erts_no_run_queues);
@@ -471,11 +457,9 @@ typedef union {
 
 static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info;
 #ifdef ERTS_DIRTY_SCHEDULERS
-#ifdef ERTS_SMP
 static ErtsAlignedSchedulerSleepInfo *aligned_dirty_cpu_sched_sleep_info;
 static ErtsAlignedSchedulerSleepInfo *aligned_dirty_io_sched_sleep_info;
 #endif
-#endif
 
 static Uint last_reductions;
 static Uint last_exact_reductions;
@@ -638,11 +622,8 @@ dbg_chk_aux_work_val(erts_aint32_t value)
     valid |= ERTS_SSI_AUX_WORK_MISC;
     valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM;
     valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC;
-#if ERTS_USE_ASYNC_READY_Q
     valid |= ERTS_SSI_AUX_WORK_ASYNC_READY;
     valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
-#endif
-#ifdef ERTS_SMP
     valid |= ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP;
     valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
     valid |= ERTS_SSI_AUX_WORK_DD;
@@ -651,7 +632,6 @@ dbg_chk_aux_work_val(erts_aint32_t value)
     valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR;
     valid |= ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP;
     valid |= ERTS_SSI_AUX_WORK_PENDING_EXITERS;
-#endif
 #if HAVE_ERTS_MSEG
     valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK;
 #endif
@@ -673,10 +653,8 @@ dbg_chk_aux_work_val(erts_aint32_t value)
 #define ERTS_DBG_CHK_SSI_AUX_WORK(SSI)
 #endif
 
-#ifdef ERTS_SMP
 static void do_handle_pending_exiters(ErtsProcList *);
 static void wake_scheduler(ErtsRunQueue *rq);
-#endif
 
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
 int
@@ -714,9 +692,7 @@ erts_step_proc_interval(void)
 void
 erts_pre_init_process(void)
 {
-#ifdef USE_THREADS
     erts_tsd_key_create(&sched_data_key, "erts_sched_data_key");
-#endif
 
     erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX]
 	= "DELAYED_AW_WAKEUP";
@@ -810,10 +786,8 @@ void
 erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab)
 {
 
-#ifdef ERTS_SMP
     erts_disable_proc_not_running_opt = 0;
     erts_init_proc_lock(ncpu);
-#endif
 
     init_proclist_alloc();
 
@@ -825,11 +799,7 @@ erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab)
 			 sizeof(Process),
 			 "process_table",
 			 legacy_proc_tab,
-#ifdef ERTS_SMP
 			 1
-#else
-			 0
-#endif
 	);
 
     last_reductions = 0;
@@ -1145,7 +1115,6 @@ read_dirty_sched_wall_time(ErtsSchedulerData *esdp, ErtsDirtySchedWallTime *info
 
 #endif
 
-#ifdef ERTS_SMP
 
 static void
 dirty_sched_wall_time_change(ErtsSchedulerData *esdp, int working)
@@ -1202,7 +1171,6 @@ dirty_sched_wall_time_change(ErtsSchedulerData *esdp, int working)
 #endif
 }
 
-#endif /* ERTS_SMP */
 
 static void
 sched_wall_time_change(ErtsSchedulerData *esdp, int working)
@@ -1481,13 +1449,11 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable,
 
     erts_proc_add_refc(c_p, (Sint32) erts_no_schedulers);
 
-#ifdef ERTS_SMP
     if (erts_no_schedulers > 1)
 	erts_schedule_multi_misc_aux_work(1,
 					  erts_no_schedulers,
 					  reply_sched_wall_time,
 					  (void *) swtrp);
-#endif
 
     reply_sched_wall_time((void *) swtrp);
 
@@ -1547,13 +1513,11 @@ Eterm erts_system_check_request(Process *c_p) {
 
     erts_proc_add_refc(c_p, (Sint) erts_no_schedulers);
 
-#ifdef ERTS_SMP
     if (erts_no_schedulers > 1)
 	erts_schedule_multi_misc_aux_work(1,
 					  erts_no_schedulers,
 					  reply_system_check,
 					  (void *) scrp);
-#endif
 
     reply_system_check((void *) scrp);
 
@@ -1620,7 +1584,6 @@ erts_psd_set_init(Process *p, int ix, void *data)
     return old;
 }
 
-#ifdef ERTS_SMP
 
 void
 erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags)
@@ -1648,7 +1611,6 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags)
     }
 }
 
-#endif
 
 static ERTS_INLINE void
 set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi,
@@ -1664,11 +1626,7 @@ set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi,
 	old_flgs = erts_atomic32_read_bor_nob(&ssi->aux_work, flgs);
 
 	if ((old_flgs & flgs) != flgs) {
-#ifdef ERTS_SMP
 	    erts_sched_poke(ssi);
-#else
-	    erts_sys_schedule_interrupt(1);
-#endif
 	}
     }
 }
@@ -1684,11 +1642,7 @@ set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi,
     old_flgs = erts_atomic32_read_bor_relb(&ssi->aux_work, flgs);
 
     if ((old_flgs & flgs) != flgs) {
-#ifdef ERTS_SMP
 	erts_sched_poke(ssi);
-#else
-	erts_sys_schedule_interrupt(1);
-#endif
     }
 }
 
@@ -1704,7 +1658,6 @@ unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs)
     return erts_atomic32_read_band_nob(&ssi->aux_work, ~flgs);
 }
 
-#ifdef ERTS_SMP
 
 static ERTS_INLINE void
 haw_chk_later_cleanup_op_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val)
@@ -1851,7 +1804,6 @@ schedule_aux_work_wakeup(ErtsAuxWorkData *awdp,
     }
 }
 
-#endif
 
 typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t;
 struct erts_misc_aux_work_t_ {
@@ -1892,11 +1844,7 @@ init_misc_aux_work(void)
 					   sizeof(erts_algnd_misc_aux_work_q_t)
 					   * (erts_no_schedulers+1));
 
-#ifdef ERTS_SMP
     ix = 0; /* aux_thread + schedulers */
-#else
-    ix = 1; /* scheduler only */
-#endif
 
     for (; ix <= erts_no_schedulers; ix++) {
 	qinit.arg = (void *) ERTS_SCHED_SLEEP_INFO_IX(ix-1);
@@ -1914,10 +1862,8 @@ misc_aux_work_clean(ErtsThrQ_t *q,
 	set_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC);
 	return aux_work | ERTS_SSI_AUX_WORK_MISC;
     case ERTS_THR_Q_NEED_THR_PRGR:
-#ifdef ERTS_SMP
 	set_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR);
 	haw_thr_prgr_soft_wakeup(awdp, erts_thr_q_need_thr_progress(q));
-#endif
     case ERTS_THR_Q_CLEAN:
 	break;
     }
@@ -1943,7 +1889,6 @@ handle_misc_aux_work(ErtsAuxWorkData *awdp,
     return misc_aux_work_clean(q, awdp, aux_work & ~ERTS_SSI_AUX_WORK_MISC);
 }
 
-#ifdef ERTS_SMP
 
 static ERTS_INLINE erts_aint32_t
 handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp,
@@ -1963,7 +1908,6 @@ handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp,
 			       aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR);
 }
 
-#endif
 
 static ERTS_INLINE void
 schedule_misc_aux_work(int sched_id,
@@ -1973,11 +1917,7 @@ schedule_misc_aux_work(int sched_id,
     ErtsThrQ_t *q;
     erts_misc_aux_work_t *mawp;
 
-#ifdef ERTS_SMP
     ASSERT(0 <= sched_id && sched_id <= erts_no_schedulers);
-#else
-    ASSERT(sched_id == 1);
-#endif
 
     q = &misc_aux_work_queues[sched_id].q;
     mawp = misc_aux_work_alloc();
@@ -2021,7 +1961,6 @@ erts_schedule_multi_misc_aux_work(int ignore_self,
    }
 }
 
-#if ERTS_USE_ASYNC_READY_Q
 
 void
 erts_notify_check_async_ready_queue(void *vno)
@@ -2049,9 +1988,7 @@ handle_async_ready(ErtsAuxWorkData *awdp,
 	}
 	return aux_work;
     }
-#ifdef ERTS_SMP
     awdp->async_ready.need_thr_prgr = 0;
-#endif
     set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN);
     return ((aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY)
 	    | ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN);
@@ -2066,7 +2003,6 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp,
 
     ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
 
-#ifdef ERTS_SMP
     if (awdp->async_ready.need_thr_prgr
 	&& !erts_thr_progress_has_reached_this(haw_thr_prgr_current(awdp),
 					       awdp->async_ready.thr_prgr)) {
@@ -2075,26 +2011,20 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp,
 
     awdp->async_ready.need_thr_prgr = 0;
     thr_prgr_p = (void *) &awdp->async_ready.thr_prgr;
-#else
-    thr_prgr_p = NULL;
-#endif
 
     switch (erts_async_ready_clean(awdp->async_ready.queue, thr_prgr_p)) {
     case ERTS_ASYNC_READY_CLEAN:
 	unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN);
 	return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
-#ifdef ERTS_SMP
     case ERTS_ASYNC_READY_NEED_THR_PRGR:
 	haw_thr_prgr_soft_wakeup(awdp, awdp->async_ready.thr_prgr);
 	awdp->async_ready.need_thr_prgr = 1;
 	return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
-#endif
     default:
 	return aux_work;
     }
 }
 
-#endif /* ERTS_USE_ASYNC_READY_Q */
 
 
 static ERTS_INLINE erts_aint32_t
@@ -2118,7 +2048,6 @@ handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
     return aux_work;
 }
 
-#ifdef ERTS_SMP
 
 void
 erts_alloc_notify_delayed_dealloc(int ix)
@@ -2375,20 +2304,15 @@ enqueue_later_op(ErtsSchedulerData *esdp,
     return later;
 }
 
-#endif /* ERTS_SMP */
 
 void
 erts_schedule_thr_prgr_later_op(void (*later_func)(void *),
 				void *later_data,
 				ErtsThrPrgrLaterOp *lop)
 {
-#ifndef ERTS_SMP
-    later_func(later_data);
-#else
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
     ErtsThrPrgrVal later = enqueue_later_op(esdp, later_func, later_data, lop);
     haw_thr_prgr_wakeup(&esdp->aux_work_data, later);
-#endif
 }
 
 void
@@ -2397,13 +2321,9 @@ erts_schedule_thr_prgr_later_cleanup_op(void (*later_func)(void *),
 					ErtsThrPrgrLaterOp *lop,
 					UWord size)
 {
-#ifndef ERTS_SMP
-    later_func(later_data);
-#else
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
     ErtsThrPrgrVal later = enqueue_later_op(esdp, later_func, later_data, lop);
     haw_thr_prgr_later_cleanup_op_wakeup(&esdp->aux_work_data, later, size);
-#endif
 }
 
 static ERTS_INLINE erts_aint32_t
@@ -2453,11 +2373,7 @@ setup_thr_debug_wait_completed(void *vproc)
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
     ErtsAuxWorkData *awdp;
     erts_aint32_t wait_flags, aux_work_flags;
-#ifdef ERTS_SMP
     awdp = esdp ? &esdp->aux_work_data : aux_thread_aux_work_data;
-#else
-    awdp = &esdp->aux_work_data;
-#endif
 
     wait_flags = 0;
     aux_work_flags = ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
@@ -2466,18 +2382,14 @@ setup_thr_debug_wait_completed(void *vproc)
 	erts_alloc_fix_alloc_shrink(awdp->sched_id, 0);
 	wait_flags |= (ERTS_SSI_AUX_WORK_DD
 		       | ERTS_SSI_AUX_WORK_DD_THR_PRGR);
-#ifdef ERTS_SMP
 	aux_work_flags |= ERTS_SSI_AUX_WORK_DD;
-#endif
     }
 
     if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS) {
 	wait_flags |= (ERTS_SSI_AUX_WORK_CNCLD_TMRS
 		       | ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR);
-#ifdef ERTS_SMP
 	if (awdp->esdp && !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp))
 	    aux_work_flags |= ERTS_SSI_AUX_WORK_CNCLD_TMRS;
-#endif
     }
 
     set_aux_work_flags_wakeup_nob(awdp->ssi, aux_work_flags);
@@ -2496,21 +2408,17 @@ static void later_thr_debug_wait_completed(void *vlop)
 {
     struct debug_lop *lop = vlop;
     erts_aint32_t count = (erts_aint32_t) erts_no_schedulers;
-#ifdef ERTS_SMP
     count += 1; /* aux thread */
-#endif
     if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == count) {
         /* scheduler threads */
         erts_schedule_multi_misc_aux_work(0,
                                           erts_no_schedulers,
                                           setup_thr_debug_wait_completed,
                                           lop->proc);
-#ifdef ERTS_SMP
         /* aux_thread */
         erts_schedule_misc_aux_work(0,
                                     setup_thr_debug_wait_completed,
                                     lop->proc);
-#endif
     }
     erts_free(ERTS_ALC_T_DEBUG, lop);
 }
@@ -2531,9 +2439,7 @@ erts_debug_wait_completed(Process *c_p, int flags)
 {
     /* Only one process at a time can do this */
     erts_aint32_t count = (erts_aint32_t) (2*erts_no_schedulers);
-#ifdef ERTS_SMP
     count += 1; /* aux thread */
-#endif
     if (0 == erts_atomic32_cmpxchg_mb(&debug_wait_completed_count,
 				      count,
 				      0)) {
@@ -2661,7 +2567,6 @@ handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiti
 
 #endif
 
-#ifdef ERTS_SMP
 
 static ERTS_INLINE erts_aint32_t
 handle_pending_exiters(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
@@ -2683,7 +2588,6 @@ handle_pending_exiters(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin
     return aux_work & ~ERTS_SSI_AUX_WORK_PENDING_EXITERS;
 }
 
-#endif
 
 static ERTS_INLINE erts_aint32_t
 handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
@@ -2715,9 +2619,7 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
     ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_AUX);
 
     ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
-#ifdef ERTS_SMP
     haw_thr_prgr_current_reset(awdp);
-#endif
 
     ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
     ASSERT(aux_work);
@@ -2736,7 +2638,6 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
      * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative
      * eachother. Most frequent first.
      */
-#ifdef ERTS_SMP
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP,
 		    handle_delayed_aux_work_wakeup);
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD,
@@ -2744,13 +2645,11 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
     /* DD must be before DD_THR_PRGR */
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD_THR_PRGR,
 		    handle_delayed_dealloc_thr_prgr);
-#endif
 
     HANDLE_AUX_WORK((ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
 		     | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
 		    handle_fix_alloc);
 
-#ifdef ERTS_SMP
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP,
 		    handle_thr_prgr_later_op);
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CNCLD_TMRS,
@@ -2758,28 +2657,21 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
     /* CNCLD_TMRS must be before CNCLD_TMRS_THR_PRGR */
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR,
 		    handle_canceled_timers_thr_prgr);
-#endif
 
-#if ERTS_USE_ASYNC_READY_Q
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY,
 		    handle_async_ready);
     /* ASYNC_READY must be before ASYNC_READY_CLEAN */
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN,
 		    handle_async_ready_clean);
-#endif
 
-#ifdef ERTS_SMP
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC_THR_PRGR,
 		    handle_misc_aux_work_thr_prgr);
-#endif
     /* MISC_THR_PRGR must be before MISC */
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC,
 		    handle_misc_aux_work);
 
-#ifdef ERTS_SMP
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_PENDING_EXITERS,
 		    handle_pending_exiters);
-#endif
 
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO,
 		    handle_setup_aux_work_timer);
@@ -2805,10 +2697,8 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
 
     ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
 
-#ifdef ERTS_SMP
     if (waiting && !aux_work)
 	haw_thr_prgr_current_check_progress(awdp);
-#endif
 
     ERTS_MSACC_UPDATE_CACHE();
     ERTS_MSACC_POP_STATE_M();
@@ -2907,11 +2797,7 @@ aux_work_timeout(void *vesdp)
     ASSERT(esdp == (ErtsSchedulerData *) vesdp);
 #endif
 
-#ifdef ERTS_SMP
     i = 0;
-#else
-    i = 1;
-#endif
 
     for (; i <= erts_no_schedulers; i++) {
 	erts_aint32_t type;
@@ -2945,9 +2831,6 @@ erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable)
 {
     erts_aint32_t old, refc;
 
-#ifndef ERTS_SMP
-    ix = 1;
-#endif
 
     ERTS_DBG_CHK_AUX_WORK_VAL(type);
     ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix]));
@@ -3014,7 +2897,6 @@ erts_active_schedulers(void)
     return as;
 }
 
-#ifdef ERTS_SMP
 
 static ERTS_INLINE void
 clear_sys_scheduling(void)
@@ -3028,20 +2910,14 @@ try_set_sys_scheduling(void)
     return 0 == erts_smp_atomic32_cmpxchg_acqb(&doing_sys_schedule, 1, 0);
 }
 
-#endif
 
 static ERTS_INLINE int
 prepare_for_sys_schedule(int non_blocking)
 {
     if (non_blocking && erts_eager_check_io) {
-#ifdef ERTS_SMP
 	return try_set_sys_scheduling();
-#else
-	return 1;
-#endif
     }
     else {
-#ifdef ERTS_SMP
 	while (!erts_port_task_have_outstanding_io_tasks()
 	       && try_set_sys_scheduling()) {
 	    if (!erts_port_task_have_outstanding_io_tasks())
@@ -3049,13 +2925,9 @@ prepare_for_sys_schedule(int non_blocking)
 	    clear_sys_scheduling();
 	}
 	return 0;
-#else
-	return !erts_port_task_have_outstanding_io_tasks();
-#endif
     }
 }
 
-#ifdef ERTS_SMP
 
 static ERTS_INLINE void
 sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq)
@@ -3381,7 +3253,6 @@ aux_thread(void *unused)
 
 static void suspend_scheduler(ErtsSchedulerData *esdp);
 
-#endif /* ERTS_SMP */
 
 static void
 scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
@@ -3390,12 +3261,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
     ErtsSchedulerSleepInfo *ssi = esdp->ssi;
     int spincount;
     erts_aint32_t aux_work = 0;
-#ifdef ERTS_SMP
     int thr_prgr_active = 1;
     erts_aint32_t flgs;
-#endif
     ERTS_MSACC_PUSH_STATE_M();
-#ifdef ERTS_SMP
 
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
@@ -3566,7 +3434,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
     }
     else
-#endif
     {
 
 	erts_smp_atomic32_set_relb(&function_calls, 0);
@@ -3608,30 +3475,19 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    }
 
 	sys_aux_work:
-#ifndef ERTS_SMP
-	    erts_sys_schedule_interrupt(0);
-#endif
 
 	    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 	    if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) {
 		if (!working)
 		    sched_wall_time_change(esdp, working = 1);
-#ifdef ERTS_SMP
 		if (!thr_prgr_active)
 		    erts_thr_progress_active(esdp, thr_prgr_active = 1);
-#endif
 		aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1);
                 ERTS_MSACC_UPDATE_CACHE();
-#ifdef ERTS_SMP
 		if (aux_work && erts_thr_progress_update(esdp))
 		    erts_thr_progress_leader_update(esdp);
-#endif
 	    }
 
-#ifndef ERTS_SMP
-	    if (erts_smp_atomic32_read_dirty(&rq->len) != 0 || rq->misc.start)
-		goto sys_woken;
-#else
 	    flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
 	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
 		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
@@ -3653,12 +3509,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		    goto tse_wait;
 		}
 	    }
-#endif
 	}
 
 	erts_smp_runq_lock(rq);
 
-#ifdef ERTS_SMP
 	/*
 	 * If we got new I/O tasks we aren't allowed to
 	 * sleep in erl_sys_schedule().
@@ -3681,12 +3535,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		goto tse_wait;
 	    }
 	}
-#endif
 	if (aux_work) {
 	    erts_smp_runq_unlock(rq);
 	    goto sys_poll_aux_work;
 	}
-#ifdef ERTS_SMP
 	flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING);
 	if (!(flgs & ERTS_SSI_FLG_SLEEPING)) {
 	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
@@ -3705,17 +3557,14 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING);
 	ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-#endif
 
 	erts_smp_runq_unlock(rq);
 
 	if (working)
 	    sched_wall_time_change(esdp, working = 0);
 
-#ifdef ERTS_SMP
 	if (thr_prgr_active)
 	    erts_thr_progress_active(esdp, thr_prgr_active = 0);
-#endif
 
 	ASSERT(!erts_port_task_have_outstanding_io_tasks());
 
@@ -3732,11 +3581,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		erts_bump_timers(esdp->timer_wheel, current_time);
 	}
 
-#ifndef ERTS_SMP
-	if (erts_smp_atomic32_read_dirty(&rq->len) == 0 && !rq->misc.start)
-	    goto sys_aux_work;
-    sys_woken:
-#else
 	flgs = sched_prep_cont_spin_wait(ssi);
 	if (flgs & ERTS_SSI_FLG_WAITING)
 	    goto sys_aux_work;
@@ -3756,7 +3600,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    erts_smp_atomic32_read_band_nob(&ssi->flags,
                                             (ERTS_SSI_FLG_SUSPENDED
                                              | ERTS_SSI_FLG_MSB_EXEC));
-#endif
 	if (!working)
 	    sched_wall_time_change(esdp, working = 1);
 	sched_active_sys(esdp->no, rq);
@@ -3768,7 +3611,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 }
 
-#ifdef ERTS_SMP
 
 static ERTS_INLINE erts_aint32_t
 ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi)
@@ -4025,12 +3867,10 @@ wake_scheduler_on_empty_runq(ErtsRunQueue *crq)
     }
 }
 
-#endif /* ERTS_SMP */
 
 static ERTS_INLINE void
 smp_notify_inc_runq(ErtsRunQueue *runq)
 {
-#ifdef ERTS_SMP
     if (runq) {
 #ifdef ERTS_DIRTY_SCHEDULERS
 	if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix))
@@ -4039,7 +3879,6 @@ smp_notify_inc_runq(ErtsRunQueue *runq)
 #endif
 	    wake_scheduler(runq);
     }
-#endif
 }
 
 void
@@ -4051,16 +3890,12 @@ erts_smp_notify_inc_runq(ErtsRunQueue *runq)
 void
 erts_sched_notify_check_cpu_bind(void)
 {
-#ifdef ERTS_SMP
     int ix;
     for (ix = 0; ix < erts_no_run_queues; ix++) {
 	ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
 	(void) ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND);
 	wake_scheduler(rq);
     }
-#else
-    erts_sched_check_cpu_bind(erts_get_scheduler_data());
-#endif
 }
 
 
@@ -4174,7 +4009,6 @@ free_proxy_proc(Process *proxy)
     erts_free(ERTS_ALC_T_PROC, proxy);
 }
 
-#ifdef ERTS_SMP
 
 static ErtsRunQueue *
 check_immigration_need(ErtsRunQueue *c_rq, ErtsMigrationPath *mp, int prio)
@@ -5615,20 +5449,15 @@ change_no_used_runqs(int used)
 }
 
 
-#endif /* #ifdef ERTS_SMP */
 
 Uint
 erts_debug_nbalance(void)
 {
-#ifdef ERTS_SMP
     Uint n;
     erts_smp_mtx_lock(&balance_info.update_mtx);
     n = balance_info.n;
     erts_smp_mtx_unlock(&balance_info.update_mtx);
     return n;
-#else
-    return 0;
-#endif
 }
 
 /* Wakeup other schedulers */
@@ -5674,7 +5503,6 @@ typedef enum {
 #define ERTS_WAKEUP_OTHER_DEC_LEGACY 10
 #define ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY (CONTEXT_REDS/10)
 
-#ifdef ERTS_SMP
 
 static struct {
     ErtsSchedWakeupOtherThreshold threshold;
@@ -5868,16 +5696,13 @@ runq_supervisor(void *unused)
     return NULL;
 }
 
-#endif
 
 void
 erts_early_init_scheduling(int no_schedulers)
 {
     aux_work_timeout_early_init(no_schedulers);
-#ifdef ERTS_SMP
     wakeup_other.threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM;
     wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT;
-#endif
     sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM;
     sched_busy_wait.tse = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM
 			   * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT);
@@ -5888,7 +5713,6 @@ erts_early_init_scheduling(int no_schedulers)
 int
 erts_sched_set_wakeup_other_thresold(char *str)
 {
-#ifdef ERTS_SMP
     ErtsSchedWakeupOtherThreshold threshold;
     if (sys_strcmp(str, "very_high") == 0)
 	threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH;
@@ -5905,20 +5729,11 @@ erts_sched_set_wakeup_other_thresold(char *str)
     wakeup_other.threshold = threshold;
     set_wakeup_other_data();
     return 0;
-#else
-    if (sys_strcmp(str, "very_high") == 0 || sys_strcmp(str, "high") == 0 ||
-	sys_strcmp(str, "medium") == 0 || sys_strcmp(str, "low") == 0 ||
-	sys_strcmp(str, "very_low") == 0) {
-	return 0;
-    } 
-    return EINVAL;
-#endif
 }
 
 int
 erts_sched_set_wakeup_other_type(char *str)
 {
-#ifdef ERTS_SMP
     ErtsSchedWakeupOtherType type;
     if (sys_strcmp(str, "default") == 0)
 	type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT;
@@ -5928,12 +5743,6 @@ erts_sched_set_wakeup_other_type(char *str)
 	return EINVAL;
     wakeup_other.type = type;
     return 0;
-#else
-    if (sys_strcmp(str, "default") == 0 || sys_strcmp(str, "legacy") == 0) {
-	return 0;
-    } 
-    return EINVAL;
-#endif
 }
 
 int
@@ -6024,7 +5833,6 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
     awdp->sched_id = id;
     awdp->esdp = esdp;
     awdp->ssi = esdp ? esdp->ssi : NULL;
-#ifdef ERTS_SMP
     awdp->latest_wakeup = ERTS_THR_PRGR_VAL_FIRST;
     awdp->misc.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
     awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
@@ -6033,15 +5841,9 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
     awdp->later_op.size = 0;
     awdp->later_op.first = NULL;
     awdp->later_op.last = NULL;
-#endif
-#ifdef ERTS_USE_ASYNC_READY_Q
-#ifdef ERTS_SMP
     awdp->async_ready.need_thr_prgr = 0;
     awdp->async_ready.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
-#endif
     awdp->async_ready.queue = NULL;
-#endif
-#ifdef ERTS_SMP
     awdp->delayed_wakeup.next = ERTS_DELAYED_WAKEUP_INFINITY;
     if (!dawwp) {
 	awdp->delayed_wakeup.job = NULL;
@@ -6057,7 +5859,6 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
 	for (i = 0; i <= erts_no_schedulers; i++)
 	    awdp->delayed_wakeup.sched2jix[i] = -1;
     }
-#endif
     awdp->debug.wait_completed.flags = 0;
     awdp->debug.wait_completed.callback = NULL;
     awdp->debug.wait_completed.arg = NULL;
@@ -6072,11 +5873,9 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
                     Uint64 time_stamp)
 {
     esdp->timer_wheel = NULL;
-#ifdef ERTS_SMP
     erts_bits_init_state(&esdp->erl_bits_state);
     esdp->match_pseudo_process = NULL;
     esdp->free_process = NULL;
-#endif
     esdp->x_reg_array =
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER,
 					   ERTS_X_REGS_ALLOCATED *
@@ -6145,9 +5944,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 
     if (daww_ptr) {
 	init_aux_work_data(&esdp->aux_work_data, esdp, *daww_ptr);
-#ifdef ERTS_SMP
 	*daww_ptr += daww_sz;
-#endif
     }
 
     esdp->reductions = 0;
@@ -6168,16 +5965,12 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     char *daww_ptr;
     size_t daww_sz;
     size_t size_runqs;
-#ifdef ERTS_SMP
     erts_aint32_t set_schdlr_sspnd_change_flags;
-#endif
 
     init_misc_op_list_alloc();
     init_proc_sys_task_queues_alloc();
 
-#ifdef ERTS_SMP
     set_wakeup_other_data();
-#endif
 
 #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
     if (erts_sched_balance_util)
@@ -6201,9 +5994,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     size_runqs = sizeof(ErtsAlignedRunQueue) * tot_rqs;
     erts_aligned_run_queues =
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, size_runqs);
-#ifdef ERTS_SMP
     erts_smp_atomic32_init_nob(&no_empty_run_queues, 0);
-#endif
 
     erts_no_run_queues = n;
 
@@ -6222,14 +6013,12 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	erts_smp_cnd_init(&rq->cnd);
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-#ifdef ERTS_SMP
         if (ERTS_RUNQ_IX_IS_DIRTY(ix)) {
             erts_smp_spinlock_init(&rq->sleepers.lock, "dirty_run_queue_sleep_list",
                 make_small(ix + 1),
                 ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
         }
 	rq->sleepers.list = NULL;
-#endif
 #endif
 
 	rq->waiting = 0;
@@ -6276,7 +6065,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 
     }
 
-#ifdef ERTS_SMP
 
     if (erts_no_run_queues != 1) {
 	run_queue_info = erts_alloc(ERTS_ALC_T_RUNQ_BLNS,
@@ -6287,7 +6075,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 					* erts_no_run_queues));
     }
 
-#endif
 
     n = (int) no_schedulers;
     erts_no_schedulers = n;
@@ -6300,29 +6087,22 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 #endif
 
     /* Create and initialize scheduler sleep info */
-#ifdef ERTS_SMP
     no_ssi = n+1;
-#else
-    no_ssi = 1;
-#endif
     aligned_sched_sleep_info =
 	erts_alloc_permanent_cache_aligned(
 	    ERTS_ALC_T_SCHDLR_SLP_INFO,
 	    no_ssi*sizeof(ErtsAlignedSchedulerSleepInfo));
     for (ix = 0; ix < no_ssi; ix++) {
 	ErtsSchedulerSleepInfo *ssi = &aligned_sched_sleep_info[ix].ssi;
-#ifdef ERTS_SMP
 #if 0 /* no need to initialize these... */
 	ssi->next = NULL;
 	ssi->prev = NULL;
 #endif
 	erts_smp_atomic32_init_nob(&ssi->flags, 0);
 	ssi->event = NULL; /* initialized in sched_thread_func */
-#endif
 	erts_atomic32_init_nob(&ssi->aux_work, 0);
     }
 
-#ifdef ERTS_SMP
     aligned_sched_sleep_info++;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -6346,20 +6126,14 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	ssi->event = NULL; /* initialized in sched_dirty_io_thread_func */
 	erts_atomic32_init_nob(&ssi->aux_work, 0);
     }
-#endif
 #endif
 
     /* Create and initialize scheduler specific data */
 
-#ifdef ERTS_SMP
     daww_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE((sizeof(ErtsDelayedAuxWorkWakeupJob)
 					      + sizeof(int))*(n+1));
     daww_ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
 						  daww_sz*n);
-#else
-    daww_sz = 0;
-    daww_ptr = NULL;
-#endif
 
     erts_aligned_scheduler_data = 
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
@@ -6412,7 +6186,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */
     debug_wait_completed_flags = 0;
 
-#ifdef ERTS_SMP
 
     aux_thread_aux_work_data =
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
@@ -6502,39 +6275,16 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 
     init_misc_aux_work();
 
-#else /* !ERTS_SMP */
-    {
-	ErtsSchedulerData *esdp;
-	esdp = ERTS_SCHEDULER_IX(0);
-	erts_scheduler_data = esdp;
-#ifdef USE_THREADS
-	erts_tsd_set(sched_data_key, (void *) esdp);
-#endif
-    }
-    erts_no_dirty_cpu_schedulers = 0;
-    erts_no_dirty_io_schedulers = 0;
-#endif
 
     erts_smp_atomic32_init_nob(&function_calls, 0);
 
     /* init port tasks */
     erts_port_task_init();
 
-#ifndef ERTS_SMP
-#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
-    erts_scheduler_data->verify_unused_temp_alloc
-	= erts_alloc_get_verify_unused_temp_alloc(
-	    &erts_scheduler_data->verify_unused_temp_alloc_data);
-    ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL);
-#endif
-#endif
 
     erts_smp_atomic32_init_relb(&erts_halt_progress, -1);
     erts_halt_code = 0;
 
-#if !defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
-    erts_lc_set_thread_name("scheduler 1");
-#endif
 
 }
 
@@ -6547,7 +6297,6 @@ erts_schedid2runq(Uint id)
     return ERTS_RUNQ_IX(ix);
 }
 
-#ifdef USE_THREADS
 
 ErtsSchedulerData *
 erts_get_scheduler_data(void)
@@ -6555,16 +6304,13 @@ erts_get_scheduler_data(void)
     return (ErtsSchedulerData *) erts_tsd_get(sched_data_key);
 }
 
-#endif
 
 static Process *
 make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
 {
     erts_aint32_t state;
     Process *proxy;
-#ifdef ERTS_SMP
     ErtsRunQueue *rq = RUNQ_READ_RQ(&proc->run_queue);
-#endif
 
     state = (ERTS_PSFLG_PROXY
 	     | ERTS_PSFLG_IN_RUNQ
@@ -6577,9 +6323,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
 	proxy = prev_proxy;
 	ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY);
 	erts_smp_atomic32_set_nob(&proxy->state, state);
-#ifdef ERTS_SMP
 	RUNQ_SET_RQ(&proc->run_queue, rq);
-#endif
     }
     else {
 	proxy = erts_alloc(ERTS_ALC_T_PROC, sizeof(Process));
@@ -6592,10 +6336,8 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
 	}
 #endif
 	erts_smp_atomic32_init_nob(&proxy->state, state);
-#ifdef ERTS_SMP
 	erts_smp_atomic_init_nob(&proxy->run_queue,
 				 erts_smp_atomic_read_nob(&proc->run_queue));
-#endif
     }
 
     proxy->common.id = proc->common.id;
@@ -6798,7 +6540,6 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st
 
 	runq = erts_get_runq_proc(p);
 
-#ifdef ERTS_SMP
 	if (!(ERTS_PSFLG_BOUND & state)) {
 	    ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio);
 	    if (new_runq) {
@@ -6806,7 +6547,6 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st
 		runq = new_runq;
 	    }
 	}
-#endif
 
 	ASSERT(runq);
 
@@ -6895,13 +6635,8 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
 
 	erts_smp_runq_lock(c_rq);
 
-#if !defined(ERTS_SMP)
-        /* Decrement refc if process struct is free... */
-        return !!(n & ERTS_PSFLG_FREE);
-#else
         /* Decrement refc if scheduled out from dirty scheduler... */
 	return !is_normal_sched;
-#endif
     }
     else {
 	Process* sched_p;
@@ -7363,7 +7098,6 @@ resume_process(Process *p, ErtsProcLocks locks)
     add2runq(enqueue, enq_prio, p, state, NULL);
 }
 
-#ifdef ERTS_SMP
 
 static ERTS_INLINE void
 sched_resume_wake__(ErtsSchedulerSleepInfo *ssi)
@@ -8803,9 +8537,7 @@ sched_thread_func(void *vesdp)
     ErtsThrPrgrCallbacks callbacks;
     ErtsSchedulerData *esdp = vesdp;
     Uint no = esdp->no;
-#ifdef ERTS_SMP
     erts_tse_t *tse;
-#endif
 
     erts_sched_init_time_sup(esdp);
 
@@ -8815,7 +8547,6 @@ sched_thread_func(void *vesdp)
     (void) ERTS_RUNQ_FLGS_SET_NOB(esdp->run_queue,
 				  ERTS_RUNQ_FLG_EXEC);
 
-#ifdef ERTS_SMP
     tse = erts_tse_fetch();
     erts_tse_prepare_timed(tse);
     ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = tse;
@@ -8829,7 +8560,6 @@ sched_thread_func(void *vesdp)
 
     erts_thr_progress_register_managed_thread(esdp, &callbacks, 0);
     erts_alloc_register_scheduler(vesdp);
-#endif
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
 	char buf[31];
@@ -8838,18 +8568,14 @@ sched_thread_func(void *vesdp)
     }
 #endif
     erts_tsd_set(sched_data_key, vesdp);
-#ifdef ERTS_SMP
 #if HAVE_ERTS_MSEG
     erts_mseg_late_init();
 #endif
-#if ERTS_USE_ASYNC_READY_Q
     esdp->aux_work_data.async_ready.queue = erts_get_async_ready_queue(no);
-#endif
 
     erts_sched_init_check_cpu_bind(esdp);
 
     erts_proc_lock_prepare_proc_lock_waiter();
-#endif
 
 #ifdef HIPE
     hipe_thread_signal_init();
@@ -8875,7 +8601,6 @@ sched_thread_func(void *vesdp)
 }
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-#ifdef ERTS_SMP
 static void*
 sched_dirty_cpu_thread_func(void *vesdp)
 {
@@ -8905,9 +8630,7 @@ sched_dirty_cpu_thread_func(void *vesdp)
     }
 #endif
     erts_tsd_set(sched_data_key, vesdp);
-#if ERTS_USE_ASYNC_READY_Q
     esdp->aux_work_data.async_ready.queue = NULL;
-#endif
 
     erts_proc_lock_prepare_proc_lock_waiter();
 
@@ -8953,9 +8676,7 @@ sched_dirty_io_thread_func(void *vesdp)
     }
 #endif
     erts_tsd_set(sched_data_key, vesdp);
-#if ERTS_USE_ASYNC_READY_Q
     esdp->aux_work_data.async_ready.queue = NULL;
-#endif
 
     erts_proc_lock_prepare_proc_lock_waiter();
 
@@ -8972,7 +8693,6 @@ sched_dirty_io_thread_func(void *vesdp)
     return NULL;
 }
 #endif
-#endif
 
 static ethr_tid aux_tid;
 
@@ -8990,7 +8710,6 @@ erts_start_schedulers(void)
 
     opts.name = name;
 
-#ifdef ERTS_SMP
     if (erts_runq_supervision_interval) {
 	opts.suggested_stack_size = 16;
         erts_snprintf(opts.name, 16, "runq_supervisor");
@@ -9004,7 +8723,6 @@ erts_start_schedulers(void)
 	    erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision thread\n");
 
     }
-#endif
 
     opts.suggested_stack_size = erts_sched_thread_suggested_stack_size;
 
@@ -9031,7 +8749,6 @@ erts_start_schedulers(void)
     erts_no_schedulers = actual;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-#ifdef ERTS_SMP
     {
 	int ix;
 	for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
@@ -9051,7 +8768,6 @@ erts_start_schedulers(void)
 		erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d\n", ix);
 	}
     }
-#endif
 #endif
 
     ERTS_THR_MEMORY_BARRIER;
@@ -9079,9 +8795,7 @@ erts_start_schedulers(void)
     }
 }
 
-#endif /* ERTS_SMP */
 
-#ifdef ERTS_SMP
 
 static void
 add_pend_suspend(Process *suspendee,
@@ -9468,7 +9182,6 @@ handle_pend_bif_async_suspend(Process *suspendee,
     }
 }
 
-#endif /* ERTS_SMP */
 
 /*
  * The erlang:suspend_process/2 BIF
@@ -9684,20 +9397,16 @@ suspend_process_2(BIF_ALIST_2)
     goto do_return;
 
  no_suspendee:
-#ifdef ERTS_SMP
     BIF_P->suspendee = NIL;
-#endif
     erts_delete_suspend_monitor(&BIF_P->suspend_monitors, BIF_ARG_1);
 
  badarg:
     ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
-#ifdef ERTS_SMP
     goto do_return;
 
  yield:
     ERTS_BIF_PREP_YIELD2(res, bif_export[BIF_suspend_process_2],
 			 BIF_P, BIF_ARG_1, BIF_ARG_2);
-#endif
 
  do_return:
     if (suspendee)
@@ -9920,7 +9629,6 @@ erts_process_status(Process *rp, Eterm rpid)
 	erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
 	res = erts_process_state2status(state);
     }
-#ifdef ERTS_SMP
     else {
 	int i;
 	ErtsSchedulerData *esdp;
@@ -9937,7 +9645,6 @@ erts_process_status(Process *rp, Eterm rpid)
 	    erts_smp_runq_unlock(esdp->run_queue);
 	}
     }
-#endif
     return res;
 }
 
@@ -10218,7 +9925,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	actual_reds = reds = 0;
 	erts_smp_runq_lock(rq);
     } else {
-#ifdef ERTS_SMP
 #ifdef ERTS_DIRTY_SCHEDULERS
 	is_normal_sched = !esdp;
 	if (is_normal_sched) {
@@ -10234,11 +9940,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 #endif
 	ASSERT(esdp->current_process == p
 	       || esdp->free_process == p);
-#else
-	esdp = erts_scheduler_data;
-	ASSERT(esdp->current_process == p);
-	is_normal_sched = 1;
-#endif
 
     sched_out_proc:
 
@@ -10279,18 +9980,15 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
 
-#ifdef ERTS_SMP
         if (p->trace_msg_q) {
 	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
             erts_schedule_flush_trace_messages(p, 1);
 	    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
 	}
-#endif
 
         /* have to re-read state after taking lock */
         state = erts_smp_atomic32_read_nob(&p->state);
 
-#ifdef ERTS_SMP
 	if (is_normal_sched && (state & ERTS_PSFLG_PENDING_EXIT))
 	    erts_handle_pending_exit(p, (ERTS_PROC_LOCK_MAIN
 					 | ERTS_PROC_LOCK_TRACE
@@ -10299,7 +9997,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    handle_pending_suspend(p, (ERTS_PROC_LOCK_MAIN
 				       | ERTS_PROC_LOCK_TRACE
 				       | ERTS_PROC_LOCK_STATUS));
-#endif
 
 	esdp->reductions += reds;
 
@@ -10317,10 +10014,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
                                           actual_reds);
 
             esdp->current_process = NULL;
-#ifdef ERTS_SMP
             if (is_normal_sched)
                 p->scheduler_data = NULL;
-#endif
 
             erts_smp_proc_unlock(p, (ERTS_PROC_LOCK_MAIN
                                      | ERTS_PROC_LOCK_STATUS
@@ -10328,7 +10023,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
             ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER);
 
-#ifdef ERTS_SMP
             if (state & ERTS_PSFLG_FREE) {
                 if (!is_normal_sched) {
                     ASSERT(p->flags & F_DELAYED_DEL_PROC);
@@ -10338,15 +10032,12 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
                     esdp->free_process = NULL;
                 }
             }
-#endif
 
             if (dec_refc)
                 erts_proc_dec_refc(p);
         }
 
-#ifdef ERTS_SMP
 	ASSERT(!esdp->free_process);
-#endif
 	ASSERT(!esdp->current_process);
 
 	ERTS_SMP_CHK_NO_PROC_LOCKS;
@@ -10367,7 +10058,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
  check_activities_to_run: {
 	erts_aint32_t psflg_running, psflg_running_sys;
-#ifdef ERTS_SMP
 	ErtsMigrationPaths *mps;
 	ErtsMigrationPath *mp;
 
@@ -10433,14 +10123,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	}
 	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
-#else /* ERTS_SMP */
-	{
-	    erts_aint32_t aux_work;
-	    aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work);
-	    if (aux_work)
-		handle_aux_work(&esdp->aux_work_data, aux_work, 0);
-	}
-#endif /* ERTS_SMP */
 
 	flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
 
@@ -10451,7 +10133,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	}
 	else if (!runq_got_work_to_execute_flags(flags)) {
 	    /* Prepare for scheduler wait */
-#ifdef ERTS_SMP
 	    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
 	    rq->wakeup_other = 0;
@@ -10502,16 +10183,13 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		}
 		empty_runq(rq);
 	    }
-#endif
 
 	    (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_EXEC);
 	    scheduler_wait(&fcalls, esdp, rq);
 	    flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC);
 	    flags |= ERTS_RUNQ_FLG_EXEC;
             ERTS_MSACC_UPDATE_CACHE();
-#ifdef ERTS_SMP
 	    non_empty_runq(rq);
-#endif
 
 	    goto check_activities_to_run;
 	}
@@ -10543,21 +10221,15 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
 		erts_bump_timers(esdp->timer_wheel, current_time);
 
-#ifdef ERTS_SMP
 	    erts_smp_runq_lock(rq);
 	    clear_sys_scheduling();
 	    goto continue_check_activities_to_run;
-#else
-	    goto check_activities_to_run;
-#endif
 	}
 
         if (flags & ERTS_RUNQ_FLG_MISC_OP)
 	    exec_misc_ops(rq);
 
-#ifdef ERTS_SMP
 	wakeup_other.check(rq, flags);
-#endif
 
 	/*
 	 * Find a new port to run.
@@ -10744,7 +10416,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
         ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_EMULATOR);
 
-#ifdef ERTS_SMP
 
 	if (flags & ERTS_RUNQ_FLG_PROTECTED)
 	    (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
@@ -10823,7 +10494,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    state = erts_smp_atomic32_read_nob(&p->state);
 	}
 
-#endif /* ERTS_SMP */
 
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 
@@ -10932,9 +10602,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	if (!(state & ERTS_PSFLG_EXITING) && ERTS_PTMR_IS_TIMED_OUT(p)) {
 	    BeamInstr** pi;
-#ifdef ERTS_SMP
 	    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
-#endif
 	    pi = (BeamInstr **) p->def_arg_reg;
 	    p->i = *pi;
 	    p->flags &= ~F_INSLPQUEUE;
@@ -11188,10 +10856,8 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	Eterm st_res;
 
 	if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
-#ifdef ERTS_SMP
 	    if (state & ERTS_PSFLG_PENDING_EXIT)
 		erts_handle_pending_exit(c_p, ERTS_PROC_LOCK_MAIN);
-#endif
 	    ASSERT(ERTS_PROC_IS_EXITING(c_p));
 	    break;
 	}
@@ -11286,18 +10952,14 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	    reds -= erts_complete_off_heap_message_queue_change(c_p);
 	    st_res = am_true;
 	    break;
-#ifdef ERTS_SMP
         case ERTS_PSTT_FTMQ:
 	    reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN);
 	    st_res = am_true;
 	    break;
-#endif
-#ifdef ERTS_SMP
         case ERTS_PSTT_ETS_FREE_FIXATION:
 	    reds -= erts_db_execute_free_fixation(c_p, (DbFixation*)st->arg[0]);
 	    st_res = am_true;
 	    break;
-#endif
 	default:
 	    ERTS_INTERNAL_ERROR("Invalid process sys task type");
 	    st_res = am_false;
@@ -11358,12 +11020,10 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
 	case ERTS_PSTT_CLA:
 	    st_res = am_ok;
 	    break;
-#ifdef ERTS_SMP
         case ERTS_PSTT_FTMQ:
 	    reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN);
 	    st_res = am_true;
 	    break;
-#endif
 	default:
 	    ERTS_INTERNAL_ERROR("Invalid process sys task type");
 	    st_res = am_false;
@@ -11769,9 +11429,7 @@ flush_dirty_trace_messages(void *vpid)
 void
 erts_schedule_flush_trace_messages(Process *proc, int force_on_proc)
 {
-#ifdef ERTS_SMP
     ErtsThrPrgrDelayHandle dhndl;
-#endif
     Eterm pid = proc->common.id;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -11786,15 +11444,11 @@ erts_schedule_flush_trace_messages(Process *proc, int force_on_proc)
     }
 #endif
 
-#ifdef ERTS_SMP
     dhndl = erts_thr_progress_unmanaged_delay();
-#endif
 
     erts_schedule_generic_sys_task(pid, ERTS_PSTT_FTMQ, NULL);
 
-#ifdef ERTS_SMP
     erts_thr_progress_unmanaged_continue(dhndl);
-#endif
 
 #ifdef ERTS_DIRTY_SCHEDULERS
     if (!force_on_proc) {
@@ -12091,7 +11745,6 @@ erts_schedule_misc_op(void (*func)(void *), void *arg)
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
     ErtsRunQueue *rq = esdp ? esdp->run_queue : ERTS_RUNQ_IX(0);
     ErtsMiscOpList *molp = misc_op_list_alloc();
-#ifdef ERTS_SMP
     ErtsMigrationPaths *mpaths = erts_get_migration_paths();
 
     if (!mpaths)
@@ -12101,7 +11754,6 @@ erts_schedule_misc_op(void (*func)(void *), void *arg)
 	if (erq)
 	    rq = erq;
     }
-#endif
 
     erts_smp_runq_lock(rq);
 
@@ -12114,9 +11766,7 @@ erts_schedule_misc_op(void (*func)(void *), void *arg)
 	rq->misc.start = molp;
     rq->misc.end = molp;
 
-#ifdef ERTS_SMP
     non_empty_runq(rq);
-#endif
 
     ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP);
 
@@ -12215,9 +11865,7 @@ static void delete_process(Process* p);
 void
 erts_free_proc(Process *p)
 {
-#ifdef ERTS_SMP
     erts_proc_lock_fin(p);
-#endif
     ASSERT(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_FREE);
     ASSERT(0 == erts_proc_read_refc(p));
     if (p->flags & F_DELAYED_DEL_PROC)
@@ -12243,11 +11891,9 @@ static void early_init_process_struct(void *varg, Eterm data)
 #endif
     erts_smp_atomic32_init_relb(&proc->state, arg->state);
 
-#ifdef ERTS_SMP
     RUNQ_SET_RQ(&proc->run_queue, arg->run_queue);
 
     erts_proc_lock_init(proc); /* All locks locked */
-#endif
 
 }
 
@@ -12422,9 +12068,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 
 #ifdef HIPE
     hipe_init_process(&p->hipe);
-#ifdef ERTS_SMP
     hipe_init_process_smp(&p->hipe_smp);
-#endif
 #endif
     p->heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*sz);
     p->old_hend = p->old_htop = p->old_heap = NULL;
@@ -12493,11 +12137,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->msg.last = &p->msg.first;
     p->msg.save = &p->msg.first;
     p->msg.len = 0;
-#ifdef ERTS_SMP
     p->msg_inq.first = NULL;
     p->msg_inq.last = &p->msg_inq.first;
     p->msg_inq.len = 0;
-#endif
     p->bif_timers = NULL;
     p->mbuf = NULL;
     p->msg_frag = NULL;
@@ -12520,14 +12162,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->last_old_htop = NULL;
 #endif
 
-#ifdef ERTS_SMP
     p->trace_msg_q = NULL;
     p->scheduler_data = NULL;
     p->suspendee = NIL;
     p->pending_suspenders = NULL;
     p->pending_exit.reason = THE_NON_VALUE;
     p->pending_exit.bp = NULL;
-#endif
 
 #if !defined(NO_FPE_SIGNALS) || defined(HIPE)
     p->fp_exception = 0;
@@ -12730,9 +12370,7 @@ void erts_init_empty_process(Process *p)
 
 #ifdef HIPE
     hipe_init_process(&p->hipe);
-#ifdef ERTS_SMP
     hipe_init_process_smp(&p->hipe_smp);
-#endif
 #endif
 
     INIT_HOLE_CHECK(p);
@@ -12746,7 +12384,6 @@ void erts_init_empty_process(Process *p)
 #endif
     erts_smp_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL);
 
-#ifdef ERTS_SMP
     p->scheduler_data = NULL;
     p->msg_inq.first = NULL;
     p->msg_inq.last = &p->msg_inq.first;
@@ -12758,7 +12395,6 @@ void erts_init_empty_process(Process *p)
     erts_proc_lock_init(p);
     erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
     RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0));
-#endif
 
 #if !defined(NO_FPE_SIGNALS) || defined(HIPE)
     p->fp_exception = 0;
@@ -12805,14 +12441,12 @@ erts_debug_verify_clean_empty_process(Process* p)
 
     ASSERT(p->parent == NIL);
 
-#ifdef ERTS_SMP
     ASSERT(p->msg_inq.first == NULL);
     ASSERT(p->msg_inq.len == 0);
     ASSERT(p->suspendee == NIL);
     ASSERT(p->pending_suspenders == NULL);
     ASSERT(p->pending_exit.reason == THE_NON_VALUE);
     ASSERT(p->pending_exit.bp == NULL);
-#endif
 
     /* Thing that erts_cleanup_empty_process() cleans up */
 
@@ -12837,9 +12471,7 @@ erts_cleanup_empty_process(Process* p)
 	free_message_buffer(p->mbuf);
 	p->mbuf = NULL;
     }
-#ifdef ERTS_SMP
     erts_proc_lock_fin(p);
-#endif
 #ifdef DEBUG
     erts_debug_verify_clean_empty_process(p);
 #endif
@@ -12955,22 +12587,6 @@ set_proc_exiting(Process *p,
     KILL_CATCHES(p);
     p->i = (BeamInstr *) beam_exit;
 
-#ifndef ERTS_SMP
-    if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)
-        && !(state & ERTS_PSFLG_GC)) {
-	/*
-	 * I non smp case:
-	 *
-	 * Currently executing process might be sent an exit
-	 * signal if it is traced by a port that it also is
-	 * linked to, and the port terminates during the
-	 * trace. In this case we want schedule out the
-	 * process as quickly as possible in order to detect
-	 * the event as fast as possible.
-	 */
-	ERTS_VBUMP_ALL_REDS(p);
-    }
-#endif
 
     add2runq(enqueue, enq_prio, p, state, NULL);
 }
@@ -13005,7 +12621,6 @@ set_proc_self_exiting(Process *c_p)
     return state;
 }
 
-#ifdef ERTS_SMP
 
 void
 erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks)
@@ -13120,7 +12735,6 @@ save_pending_exiter(Process *p, ErtsProcList *plp)
     set_aux_work_flags_wakeup_nob(ssi, ERTS_SSI_AUX_WORK_PENDING_EXITERS);
 }
 
-#endif
 
 /*
  * This function delivers an EXIT message to a process
@@ -13292,7 +12906,6 @@ send_exit_signal(Process *c_p,		/* current process if and only
 	return 1; /* Receiver will get a message */
     }
     else if (reason != am_normal || (flags & ERTS_XSIG_FLG_NO_IGN_NORMAL)) {
-#ifdef ERTS_SMP
 	if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))) {
 	    ASSERT(!rp->pending_exit.bp);
 
@@ -13419,17 +13032,6 @@ send_exit_signal(Process *c_p,		/* current process if and only
 	 *          that the receiver *will* exit; either on the pending
 	 *          exit or by itself before seeing the pending exit.
 	 */
-#else /* !ERTS_SMP */
-	erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state);
-	if (!(state & ERTS_PSFLG_EXITING)) {
-	    set_proc_exiting(rp,
-			     state,
-			     (is_immed(rsn) || c_p == rp
-			      ? rsn
-			      : copy_object(rsn, rp)),
-			     NULL);
-	}
-#endif
 	return -1; /* Receiver will exit */
     }
 
@@ -13783,18 +13385,13 @@ erts_do_exit_process(Process* p, Eterm reason)
 	erts_exit(ERTS_DUMP_EXIT, "System process %T terminated: %T\n",
                  p->common.id, reason);
 
-#ifdef ERTS_SMP
     ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
     /* By locking all locks (main lock is already locked) when going
        to exiting state (ERTS_PSFLG_EXITING), it is enough to take any lock when
        looking up a process (erts_pid2proc()) to prevent the looked up
        process from exiting until the lock has been released. */
     erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
-#endif
 
-#ifndef ERTS_SMP
-    set_proc_self_exiting(p);
-#else
     if (ERTS_PSFLG_PENDING_EXIT & set_proc_self_exiting(p)) {
 	/* Process exited before pending exit was received... */
 	p->pending_exit.reason = THE_NON_VALUE;
@@ -13807,7 +13404,6 @@ erts_do_exit_process(Process* p, Eterm reason)
     cancel_suspend_of_suspendee(p, ERTS_PROC_LOCKS_ALL); 
 
     ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
-#endif
 
     if (IS_TRACED(p)) {
 	if (IS_TRACED_FL(p, F_TRACE_CALLS))
@@ -13870,7 +13466,6 @@ erts_continue_exit_process(Process *p)
 	p->bif_timers = NULL;
     }
 
-#ifdef ERTS_SMP
     if (p->flags & F_SCHDLR_ONLN_WAITQ)
 	abort_sched_onln_chng_waitq(p);
 
@@ -13914,7 +13509,6 @@ erts_continue_exit_process(Process *p)
 		     __FILE__, __LINE__, (int) ssr);
 	}
     }
-#endif
 
     if (p->flags & F_USING_DB) {
 	if (erts_db_process_exiting(p, ERTS_PROC_LOCK_MAIN))
@@ -13998,16 +13592,12 @@ erts_continue_exit_process(Process *p)
 
 	erts_smp_runq_lock(rq);
 
-#ifdef ERTS_SMP
 	ASSERT(p->scheduler_data);
 	ASSERT(p->scheduler_data->current_process == p);
 	ASSERT(p->scheduler_data->free_process == NULL);
 
 	p->scheduler_data->current_process = NULL;
 	p->scheduler_data->free_process = p;
-#else
-	erts_proc_inc_refc(p); /* Decremented in schedule() */
-#endif
 
 	/* Time of death! */
 	erts_ptab_delete_element(&erts_proc, &p->common);
@@ -14093,19 +13683,15 @@ erts_continue_exit_process(Process *p)
 								 have none here */
     }
 
-#ifdef ERTS_SMP
     erts_flush_trace_messages(p, 0);
-#endif
 
     ERTS_TRACER_CLEAR(&ERTS_TRACER(p));
 
     if (!delay_del_proc)
 	delete_process(p);
 
-#ifdef ERTS_SMP
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
-#endif
 
     return;
 
@@ -14245,7 +13831,6 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
 
     erts_print(to, to_arg, "=scheduler:%u\n", esdp->no);
 
-#ifdef ERTS_SMP
     flg = erts_smp_atomic32_read_dirty(&esdp->ssi->flags);
     erts_print(to, to_arg, "Scheduler Sleep Info Flags: ");
     for (i = 0; i < ERTS_SSI_FLGS_MAX && flg; i++) {
@@ -14273,7 +13858,6 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
         }
     }
     erts_print(to, to_arg, "\n");
-#endif
 
     flg = erts_atomic32_read_dirty(&esdp->ssi->aux_work);
     erts_print(to, to_arg, "Scheduler Sleep Info Aux Work: ");
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 9d7ba27c50..5ed1b4e975 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -124,9 +124,7 @@ extern int erts_dio_sched_thread_suggested_stack_size;
 #define ERTS_SCHED_THREAD_MIN_STACK_SIZE 20	/* Kilo words */
 #define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */
 
-#ifdef ERTS_SMP
 #include "erl_bits.h"
-#endif
 
 /* process priorities */
 #define PRIORITY_MAX          0
@@ -373,12 +371,10 @@ typedef struct {
 #endif
 
 struct ErtsSchedulerSleepInfo_ {
-#ifdef ERTS_SMP
     ErtsSchedulerSleepInfo *next;
     ErtsSchedulerSleepInfo *prev;
     erts_smp_atomic32_t flags;
     erts_tse_t *event;
-#endif
     erts_atomic32_t aux_work;
 };
 
@@ -433,7 +429,6 @@ typedef struct {
 #  define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1
 #endif
 
-#ifdef ERTS_SMP
 
 #undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
 #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
@@ -476,7 +471,6 @@ struct ErtsMigrationPaths_ {
     ErtsMigrationPath mpath[1];
 };
 
-#endif /* ERTS_SMP */
 
 struct ErtsRunQueue_ {
     int ix;
@@ -485,9 +479,7 @@ struct ErtsRunQueue_ {
     erts_smp_cnd_t cnd;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-#ifdef ERTS_SMP
     ErtsSchedulerSleepList sleepers;
-#endif
 #endif
 
     ErtsSchedulerData *scheduler;
@@ -531,9 +523,7 @@ struct ErtsRunQueue_ {
 #endif
 };
 
-#ifdef ERTS_SMP
 extern long erts_runq_supervision_interval;
-#endif
 
 typedef union {
     ErtsRunQueue runq;
@@ -581,17 +571,12 @@ typedef struct {
     int sched_id;
     ErtsSchedulerData *esdp;
     ErtsSchedulerSleepInfo *ssi;
-#ifdef ERTS_SMP
     ErtsThrPrgrVal current_thr_prgr;
     ErtsThrPrgrVal latest_wakeup;
-#endif
     struct {
 	int ix;
-#ifdef ERTS_SMP
 	ErtsThrPrgrVal thr_prgr;
-#endif
     } misc;
-#ifdef ERTS_SMP
     struct {
 	ErtsThrPrgrVal thr_prgr;
     } dd;
@@ -604,24 +589,17 @@ typedef struct {
 	ErtsThrPrgrLaterOp *first;
 	ErtsThrPrgrLaterOp *last;
     } later_op;
-#endif
-#ifdef ERTS_USE_ASYNC_READY_Q
     struct {
-#ifdef ERTS_SMP
 	int need_thr_prgr;
 	ErtsThrPrgrVal thr_prgr;
-#endif
 	void *queue;
     } async_ready;
-#endif
-#ifdef ERTS_SMP
     struct {
 	Uint64 next;
 	int *sched2jix;
 	int jix;
 	ErtsDelayedAuxWorkWakeupJob *job;
     } delayed_wakeup;
-#endif
     struct {
         ErtsEtsAllYieldData ets_all;
         /* Other yielding operations... */
@@ -659,13 +637,11 @@ struct ErtsSchedulerData_ {
     ErtsTimerWheel *timer_wheel;
     ErtsNextTimeoutRef next_tmo_ref;
     ErtsHLTimerService *timer_service;
-#ifdef ERTS_SMP
     ethr_tid tid;		/* Thread id */
     struct erl_bits_state erl_bits_state; /* erl_bits.c state */
     void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
     Process *free_process;
     ErtsThrPrgrData thr_progress_data;
-#endif
     ErtsSchedulerSleepInfo *ssi;
     Process *current_process;
     ErtsSchedType type;
@@ -717,9 +693,6 @@ extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data;
 extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
 #endif
 
-#ifndef ERTS_SMP
-extern ErtsSchedulerData *erts_scheduler_data;
-#endif
 
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
 int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
@@ -727,10 +700,8 @@ int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
 
 #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
 
-#ifdef ERTS_SMP
 void erts_empty_runq(ErtsRunQueue *rq);
 void erts_non_empty_runq(ErtsRunQueue *rq);
-#endif
 
 
 /*
@@ -753,10 +724,8 @@ erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
 
     len = erts_smp_atomic32_read_dirty(&rq->len);
 
-#ifdef ERTS_SMP
     if (len == 0)
 	erts_non_empty_runq(rq);
-#endif
     len++;
     if (rq->max_len < len)
 	rq->max_len = len;
@@ -911,7 +880,6 @@ typedef struct {
 typedef struct ErtsProcSysTask_ ErtsProcSysTask;
 typedef struct ErtsProcSysTaskQs_ ErtsProcSysTaskQs;
 
-#ifdef ERTS_SMP
 
 typedef struct ErtsPendingSuspend_ ErtsPendingSuspend;
 struct ErtsPendingSuspend_ {
@@ -924,7 +892,6 @@ struct ErtsPendingSuspend_ {
 			Eterm pid);
 };
 
-#endif
 
 
 /* Defines to ease the change of memory architecture */
@@ -1092,7 +1059,6 @@ struct process {
     erts_smp_atomic32_t dirty_state; /* Process dirty state flags (see ERTS_PDSFLG_*) */
 #endif
 
-#ifdef ERTS_SMP
     ErlMessageInQueue msg_inq;
     ErlTraceMessageQueue *trace_msg_q;
     ErtsPendExit pending_exit;
@@ -1104,7 +1070,6 @@ struct process {
 #ifdef HIPE
     struct hipe_process_state_smp hipe_smp;
 #endif
-#endif
 
 #ifdef CHECK_FOR_HOLES
     Eterm* last_htop;		/* No need to scan the heap below this point. */
@@ -1801,11 +1766,8 @@ int erts_flush_trace_messages(Process *c_p, ErtsProcLocks locks);
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
 int erts_dbg_check_halloc_lock(Process *p);
 #endif
-#if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS)
 void
 erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, Uint *);
-#endif
-#ifdef ERTS_SMP
 ErtsSchedSuspendResult
 erts_set_schedulers_online(Process *p,
 			   ErtsProcLocks plocks,
@@ -1820,14 +1782,9 @@ void erts_start_schedulers(void);
 void erts_alloc_notify_delayed_dealloc(int);
 void erts_alloc_ensure_handle_delayed_dealloc_call(int);
 void erts_notify_canceled_timer(ErtsSchedulerData *, int);
-#endif
-#if ERTS_USE_ASYNC_READY_Q
 void erts_notify_check_async_ready_queue(void *);
-#endif
-#ifdef ERTS_SMP
 void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later);
 void erts_notify_finish_breakpointing(Process* p);
-#endif
 void erts_schedule_misc_aux_work(int sched_id,
 				 void (*func)(void *),
 				 void *arg);
@@ -1896,13 +1853,9 @@ int erts_send_exit_signal(Process *,
 			  Eterm,
 			  Process *,
 			  Uint32);
-#ifdef ERTS_SMP
 void erts_handle_pending_exit(Process *, ErtsProcLocks);
 #define ERTS_PROC_PENDING_EXIT(P) \
     (ERTS_PSFLG_PENDING_EXIT & erts_smp_atomic32_read_acqb(&(P)->state))
-#else
-#define ERTS_PROC_PENDING_EXIT(P) 0
-#endif
 
 void erts_deep_process_dump(fmtfn_t, void *);
 
@@ -1932,19 +1885,7 @@ do {										\
 #  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP)
 #endif
 
-#if defined(ERTS_SMP) || defined(USE_THREADS)
 ErtsSchedulerData *erts_get_scheduler_data(void);
-#else
-ERTS_GLB_INLINE ErtsSchedulerData *erts_get_scheduler_data(void);
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-
-ERTS_GLB_INLINE
-ErtsSchedulerData *erts_get_scheduler_data(void)
-{
-    return erts_scheduler_data;
-}
-#endif
-#endif
 
 void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks);
 
@@ -2060,12 +2001,10 @@ erts_psd_set(Process *p, int ix, void *data)
     ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
     if (psd) {
 	void *old;
-#ifdef ERTS_SMP
 #ifdef ETHR_ORDERED_READ_DEPEND
 	ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
 #else
 	ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreStore);
-#endif
 #endif
 	old = psd->data[ix];
 	psd->data[ix] = data;
@@ -2146,7 +2085,6 @@ erts_proc_set_error_handler(Process *p, Eterm handler)
 
 #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
 
-#ifdef ERTS_SMP
 
 #include "erl_thr_progress.h"
 
@@ -2248,7 +2186,6 @@ erts_check_emigration_need(ErtsRunQueue *c_rq, int prio)
 
 #endif
 
-#endif
 
 #endif
 
@@ -2290,9 +2227,6 @@ ErtsSchedulerData *erts_proc_sched_data(Process *c_p)
 {
     ErtsSchedulerData *esdp;
     ASSERT(c_p);
-#if !defined(ERTS_SMP)
-    esdp = erts_get_scheduler_data();
-#else
     esdp = c_p->scheduler_data;
 #  if defined(ERTS_DIRTY_SCHEDULERS)
     if (esdp) {
@@ -2305,7 +2239,6 @@ ErtsSchedulerData *erts_proc_sched_data(Process *c_p)
 	ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp));
     }
 #  endif
-#endif
     ASSERT(esdp);
     return esdp;
 }
@@ -2336,7 +2269,6 @@ Eterm erts_get_current_pid(void)
 ERTS_GLB_INLINE
 Uint erts_get_scheduler_id(void)
 {
-#ifdef ERTS_SMP
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
 #ifdef ERTS_DIRTY_SCHEDULERS
     if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp))
@@ -2344,65 +2276,45 @@ Uint erts_get_scheduler_id(void)
     else
 #endif
 	return esdp ? esdp->no : (Uint) 0;
-#else
-    return erts_get_scheduler_data() ? (Uint) 1 : (Uint) 0;
-#endif
 }
 
 ERTS_GLB_INLINE ErtsRunQueue *
 erts_get_runq_proc(Process *p)
 {
-#ifdef ERTS_SMP
     ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue));
     return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue);
-#else
-    return ERTS_RUNQ_IX(0);
-#endif
 }
 
 ERTS_GLB_INLINE ErtsRunQueue *
 erts_get_runq_current(ErtsSchedulerData *esdp)
 {
     ASSERT(!esdp || esdp == erts_get_scheduler_data());
-#ifdef ERTS_SMP
     if (!esdp)
 	esdp = erts_get_scheduler_data();
     return esdp->run_queue;
-#else
-    return ERTS_RUNQ_IX(0);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_runq_lock(ErtsRunQueue *rq)
 {
-#ifdef ERTS_SMP
     erts_smp_mtx_lock(&rq->mtx);
-#endif
 }
 
 ERTS_GLB_INLINE int
 erts_smp_runq_trylock(ErtsRunQueue *rq)
 {
-#ifdef ERTS_SMP
     return erts_smp_mtx_trylock(&rq->mtx);
-#else
-    return 0;
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_runq_unlock(ErtsRunQueue *rq)
 {
-#ifdef ERTS_SMP
     erts_smp_mtx_unlock(&rq->mtx);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
 {
-#ifdef ERTS_SMP
     ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx));
     if (xrq != rq) {
 	if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) {
@@ -2415,22 +2327,18 @@ erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
 	    }
 	}
     }
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
 {
-#ifdef ERTS_SMP
     if (xrq != rq)
 	erts_smp_mtx_unlock(&xrq->mtx);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
 {
-#ifdef ERTS_SMP
     ASSERT(rq1 && rq2);
     if (rq1 == rq2)
 	erts_smp_mtx_lock(&rq1->mtx);
@@ -2442,18 +2350,15 @@ erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
 	erts_smp_mtx_lock(&rq2->mtx);
 	erts_smp_mtx_lock(&rq1->mtx);	
     }
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
 {
-#ifdef ERTS_SMP
     ASSERT(rq1 && rq2);
     erts_smp_mtx_unlock(&rq1->mtx);
     if (rq1 != rq2)
 	erts_smp_mtx_unlock(&rq2->mtx);
-#endif
 }
 
 ERTS_GLB_INLINE ErtsMessage *
@@ -2561,7 +2466,6 @@ ERTS_TIME2REDS_IMPL__(ErtsMonotonicTime start, ErtsMonotonicTime end)
 }
 #endif
 
-#ifdef ERTS_SMP
 
 Process *erts_pid2proc_not_running(Process *,
 				   ErtsProcLocks,
@@ -2580,14 +2484,6 @@ extern int erts_disable_proc_not_running_opt;
 #define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)
 #endif
 
-#else /* !ERTS_SMP */
-
-#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)
-
-#define erts_pid2proc_not_running erts_pid2proc
-#define erts_pid2proc_nropt erts_pid2proc
-
-#endif
 
 #define ERTS_PROC_IS_EXITING(P) \
     (ERTS_PSFLG_EXITING & erts_smp_atomic32_read_acqb(&(P)->state))
@@ -2602,7 +2498,6 @@ extern int erts_disable_proc_not_running_opt;
 
 void erts_smp_notify_inc_runq(ErtsRunQueue *runq);
 
-#ifdef ERTS_SMP
 void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
 ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);
 
@@ -2623,7 +2518,6 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-#endif /* #ifdef ERTS_SMP */
 
 #include "erl_process_lock.h"
 
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index 23c7414901..c824724cc9 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -69,7 +69,6 @@
 #include "erl_process.h"
 #include "erl_thr_progress.h"
 
-#ifdef ERTS_SMP
 
 #if ERTS_PROC_LOCK_OWN_IMPL
 
@@ -1002,11 +1001,9 @@ static ERTS_INLINE
 Process *proc_lookup_inc_refc(Eterm pid, int allow_exit)
 {
     Process *proc;
-#ifdef ERTS_SMP
     ErtsThrPrgrDelayHandle dhndl;
 
     dhndl = erts_thr_progress_unmanaged_delay();
-#endif
 
     proc = erts_proc_lookup_raw(pid);
     if (proc) {
@@ -1016,9 +1013,7 @@ Process *proc_lookup_inc_refc(Eterm pid, int allow_exit)
             erts_proc_inc_refc(proc);
     }
 
-#ifdef ERTS_SMP
     erts_thr_progress_unmanaged_continue(dhndl);
-#endif
 
     return proc;
 }
@@ -1785,4 +1780,3 @@ check_queue(erts_proc_lock_t *lck)
 }
 #endif
 
-#endif /* ERTS_SMP */
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 023ba4d4ae..16cdebb791 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -455,7 +455,6 @@ void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks);
 #ifndef ERTS_PROCESS_LOCK_H__
 #define ERTS_PROCESS_LOCK_H__
 
-#ifdef ERTS_SMP
 
 typedef struct {
     union {
@@ -947,7 +946,6 @@ erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked)
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-#endif /* ERTS_SMP */
 
 #ifdef ERTS_ENABLE_LOCK_POSITION
 ERTS_GLB_INLINE void erts_smp_proc_lock_x(Process *, ErtsProcLocks, char *file, unsigned int line);
@@ -979,7 +977,7 @@ erts_smp_proc_lock(Process *p, ErtsProcLocks locks)
 			 ERTS_PID2PIXLOCK(p->common.id),
 #endif /*ERTS_PROC_LOCK_ATOMIC_IMPL*/
 			 locks, file, line);
-#elif defined(ERTS_SMP)
+#else
     erts_smp_proc_lock__(p,
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
 			 NULL,
@@ -993,7 +991,6 @@ erts_smp_proc_lock(Process *p, ErtsProcLocks locks)
 ERTS_GLB_INLINE void
 erts_smp_proc_unlock(Process *p, ErtsProcLocks locks)
 {
-#ifdef ERTS_SMP
     erts_smp_proc_unlock__(p,
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
 			   NULL,
@@ -1001,15 +998,11 @@ erts_smp_proc_unlock(Process *p, ErtsProcLocks locks)
 			   ERTS_PID2PIXLOCK(p->common.id),
 #endif
 			   locks);
-#endif
 }
 
 ERTS_GLB_INLINE int
 erts_smp_proc_trylock(Process *p, ErtsProcLocks locks)
 {
-#ifndef ERTS_SMP
-    return 0;
-#else
     return erts_smp_proc_trylock__(p,
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
 				   NULL,
@@ -1017,28 +1010,19 @@ erts_smp_proc_trylock(Process *p, ErtsProcLocks locks)
 				   ERTS_PID2PIXLOCK(p->common.id),
 #endif
 				   locks);
-#endif
 }
 
 ERTS_GLB_INLINE void erts_proc_inc_refc(Process *p)
 {
     ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
-#ifdef ERTS_SMP
     erts_ptab_atmc_inc_refc(&p->common);
-#else
-    erts_ptab_inc_refc(&p->common);
-#endif
 }
 
 ERTS_GLB_INLINE void erts_proc_dec_refc(Process *p)
 {
     Sint referred;
     ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
-#ifdef ERTS_SMP
     referred = erts_ptab_atmc_dec_test_refc(&p->common);
-#else
-    referred = erts_ptab_dec_test_refc(&p->common);
-#endif
     if (!referred) {
 	ASSERT(ERTS_PROC_IS_EXITING(p));
 	erts_free_proc(p);
@@ -1049,11 +1033,7 @@ ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc)
 {
     Sint referred;
     ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
-#ifdef ERTS_SMP
     referred = erts_ptab_atmc_add_test_refc(&p->common, add_refc);
-#else
-    referred = erts_ptab_add_test_refc(&p->common, add_refc);
-#endif
     if (!referred) {
 	ASSERT(ERTS_PROC_IS_EXITING(p));
 	erts_free_proc(p);
@@ -1063,16 +1043,11 @@ ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc)
 ERTS_GLB_INLINE Sint erts_proc_read_refc(Process *p)
 {
     ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
-#ifdef ERTS_SMP
     return erts_ptab_atmc_read_refc(&p->common);
-#else
-    return erts_ptab_read_refc(&p->common);
-#endif
 }
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-#ifdef ERTS_SMP
 void erts_proc_lock_init(Process *);
 void erts_proc_lock_fin(Process *);
 void erts_proc_safelock(Process *a_proc,
@@ -1081,7 +1056,6 @@ void erts_proc_safelock(Process *a_proc,
 			Process *b_proc,
 			ErtsProcLocks b_have_locks,
 			ErtsProcLocks b_need_locks);
-#endif
 
 /*
  * --- Process table lookup ------------------------------------------------
@@ -1113,9 +1087,6 @@ ERTS_GLB_INLINE Process *erts_pix2proc(int ix);
 ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid);
 ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid);
 
-#ifndef ERTS_SMP
-ERTS_GLB_INLINE
-#endif
 Process *erts_pid2proc_opt(Process *, ErtsProcLocks, Eterm, ErtsProcLocks, int);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -1152,25 +1123,6 @@ ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid)
     return proc;
 }
 
-#ifndef ERTS_SMP
-ERTS_GLB_INLINE Process *
-erts_pid2proc_opt(Process *c_p_unused,
-		  ErtsProcLocks c_p_have_locks_unused,
-		  Eterm pid,
-		  ErtsProcLocks pid_need_locks_unused,
-		  int flags)
-{
-    Process *proc = erts_proc_lookup_raw(pid);
-    if (!proc)
-	return NULL;
-    if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
-	&& ERTS_PROC_IS_EXITING(proc))
-	return NULL;
-    if (flags & ERTS_P2P_FLG_INC_REFC)
-	erts_proc_inc_refc(proc);
-    return proc;
-}
-#endif /* !ERTS_SMP */
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h
index fecfd96ab0..285d325cc4 100644
--- a/erts/emulator/beam/erl_ptab.h
+++ b/erts/emulator/beam/erl_ptab.h
@@ -387,10 +387,8 @@ ERTS_GLB_INLINE Sint erts_ptab_atmc_dec_test_refc(ErtsPTabElementCommon *ptab_el
 {
     erts_aint_t refc = erts_atomic_dec_read_relb(&ptab_el->refc.atmc);
     ERTS_SMP_LC_ASSERT(refc >= 0);
-#ifdef ERTS_SMP
     if (refc == 0)
 	ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
-#endif
     return (Sint) refc;
 }
 
diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
index 96238318c9..6cb7ccab8d 100644
--- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
@@ -32,7 +32,6 @@
 #  include "config.h"
 #endif
 
-#ifdef ERTS_SMP
 
 #include "erl_process.h"
 #include "erl_thr_progress.h"
@@ -325,4 +324,3 @@ erts_sspa_process_remote_frees(erts_sspa_chunk_header_t *chdr,
     return res;
 }
 
-#endif /* ERTS_SMP */
diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.h b/erts/emulator/beam/erl_sched_spec_pre_alloc.h
index 7808d7d438..1307e65962 100644
--- a/erts/emulator/beam/erl_sched_spec_pre_alloc.h
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.h
@@ -31,7 +31,6 @@
 #ifndef ERTS_SCHED_SPEC_PRE_ALLOC_H__
 #define ERTS_SCHED_SPEC_PRE_ALLOC_H__
 
-#ifdef ERTS_SMP
 
 #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
 #define ERL_THR_PROGRESS_TSD_TYPE_ONLY
@@ -236,6 +235,5 @@ erts_sspa_free(erts_sspa_data_t *data, int cix, char *cblk)
 
 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-#endif /* ERTS_SMP */
 
 #endif /* ERTS_SCHED_SPEC_PRE_ALLOC_H__ */
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index 696bdbdaf1..c1f0fa18cc 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -40,7 +40,6 @@
 #endif
 
 
-#ifdef ERTS_SMP
 #define ERTS_SMP_THR_OPTS_DEFAULT_INITER ERTS_THR_OPTS_DEFAULT_INITER
 typedef erts_thr_opts_t erts_smp_thr_opts_t;
 typedef erts_thr_init_data_t erts_smp_thr_init_data_t;
@@ -71,47 +70,6 @@ void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */
 #define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER
 #define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
 
-#else /* #ifdef ERTS_SMP */
-
-#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0}
-typedef int erts_smp_thr_opts_t;
-typedef int erts_smp_thr_init_data_t;
-typedef int erts_smp_tid_t;
-typedef int erts_smp_mtx_t;
-typedef int erts_smp_cnd_t;
-#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER {0}
-#define ERTS_SMP_RWMTX_TYPE_NORMAL 0
-#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ 0
-#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
-#define ERTS_SMP_RWMTX_LONG_LIVED 0
-#define ERTS_SMP_RWMTX_SHORT_LIVED 0
-#define ERTS_SMP_RWMTX_UNKNOWN_LIVED 0
-typedef struct {
-    char type;
-    char lived;
-    int main_spincount;
-    int aux_spincount;
-} erts_smp_rwmtx_opt_t;
-typedef int erts_smp_rwmtx_t;
-typedef int erts_smp_tsd_key_t;
-#define erts_smp_dw_atomic_t erts_no_dw_atomic_t
-#define erts_smp_atomic_t erts_no_atomic_t
-#define erts_smp_atomic32_t erts_no_atomic32_t
-#define erts_smp_atomic64_t erts_no_atomic64_t
-#if __GNUC__ > 2
-typedef struct { } erts_smp_spinlock_t;
-typedef struct { } erts_smp_rwlock_t;
-#else
-typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t;
-typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t;
-#endif
-
-#define ERTS_SMP_MEMORY_BARRIER
-#define ERTS_SMP_WRITE_MEMORY_BARRIER
-#define ERTS_SMP_READ_MEMORY_BARRIER
-#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER
-
-#endif /* #ifdef ERTS_SMP */
 
 ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id);
 ERTS_GLB_INLINE void erts_smp_thr_create(erts_smp_tid_t *tid,
@@ -225,7 +183,6 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
  * of erl_threads.h for info on atomics.
  */
 
-#ifdef ERTS_SMP
 
 /* Double word size atomics */
 
@@ -597,452 +554,59 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic64_set_dirty erts_atomic64_set_dirty
 #define erts_smp_atomic64_read_dirty erts_atomic64_read_dirty
 
-#else /* !ERTS_SMP */
-
-/* Double word size atomics */
-
-#define erts_smp_dw_atomic_init_nob erts_no_dw_atomic_set
-#define erts_smp_dw_atomic_set_nob erts_no_dw_atomic_set
-#define erts_smp_dw_atomic_read_nob erts_no_dw_atomic_read
-#define erts_smp_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg
-
-#define erts_smp_dw_atomic_init_mb erts_no_dw_atomic_init
-#define erts_smp_dw_atomic_set_mb erts_no_dw_atomic_set
-#define erts_smp_dw_atomic_read_mb erts_no_dw_atomic_read
-#define erts_smp_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg
-
-#define erts_smp_dw_atomic_init_acqb erts_no_dw_atomic_init
-#define erts_smp_dw_atomic_set_acqb erts_no_dw_atomic_set
-#define erts_smp_dw_atomic_read_acqb erts_no_dw_atomic_read
-#define erts_smp_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg
-
-#define erts_smp_dw_atomic_init_relb erts_no_dw_atomic_init
-#define erts_smp_dw_atomic_set_relb erts_no_dw_atomic_set
-#define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read
-#define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg
-
-#define erts_smp_dw_atomic_init_ddrb erts_no_dw_atomic_init
-#define erts_smp_dw_atomic_set_ddrb erts_no_dw_atomic_set
-#define erts_smp_dw_atomic_read_ddrb erts_no_dw_atomic_read
-#define erts_smp_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg
-
-#define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init
-#define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set
-#define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read
-#define erts_smp_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg
-
-#define erts_smp_dw_atomic_init_wb erts_no_dw_atomic_init
-#define erts_smp_dw_atomic_set_wb erts_no_dw_atomic_set
-#define erts_smp_dw_atomic_read_wb erts_no_dw_atomic_read
-#define erts_smp_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg
-
-#define erts_smp_dw_atomic_set_dirty erts_no_dw_atomic_set
-#define erts_smp_dw_atomic_read_dirty erts_no_dw_atomic_read
-
-/* Word size atomics */
-
-#define erts_smp_atomic_init_nob erts_no_atomic_set
-#define erts_smp_atomic_set_nob erts_no_atomic_set
-#define erts_smp_atomic_read_nob erts_no_atomic_read
-#define erts_smp_atomic_inc_read_nob erts_no_atomic_inc_read
-#define erts_smp_atomic_dec_read_nob erts_no_atomic_dec_read
-#define erts_smp_atomic_inc_nob erts_no_atomic_inc
-#define erts_smp_atomic_dec_nob erts_no_atomic_dec
-#define erts_smp_atomic_add_read_nob erts_no_atomic_add_read
-#define erts_smp_atomic_add_nob erts_no_atomic_add
-#define erts_smp_atomic_read_bor_nob erts_no_atomic_read_bor
-#define erts_smp_atomic_read_band_nob erts_no_atomic_read_band
-#define erts_smp_atomic_xchg_nob erts_no_atomic_xchg
-#define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
-#define erts_smp_atomic_read_bset_nob erts_no_atomic_read_bset
-
-#define erts_smp_atomic_init_mb erts_no_atomic_set
-#define erts_smp_atomic_set_mb erts_no_atomic_set
-#define erts_smp_atomic_read_mb erts_no_atomic_read
-#define erts_smp_atomic_inc_read_mb erts_no_atomic_inc_read
-#define erts_smp_atomic_dec_read_mb erts_no_atomic_dec_read
-#define erts_smp_atomic_inc_mb erts_no_atomic_inc
-#define erts_smp_atomic_dec_mb erts_no_atomic_dec
-#define erts_smp_atomic_add_read_mb erts_no_atomic_add_read
-#define erts_smp_atomic_add_mb erts_no_atomic_add
-#define erts_smp_atomic_read_bor_mb erts_no_atomic_read_bor
-#define erts_smp_atomic_read_band_mb erts_no_atomic_read_band
-#define erts_smp_atomic_xchg_mb erts_no_atomic_xchg
-#define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
-#define erts_smp_atomic_read_bset_mb erts_no_atomic_read_bset
-
-#define erts_smp_atomic_init_acqb erts_no_atomic_set
-#define erts_smp_atomic_set_acqb erts_no_atomic_set
-#define erts_smp_atomic_read_acqb erts_no_atomic_read
-#define erts_smp_atomic_inc_read_acqb erts_no_atomic_inc_read
-#define erts_smp_atomic_dec_read_acqb erts_no_atomic_dec_read
-#define erts_smp_atomic_inc_acqb erts_no_atomic_inc
-#define erts_smp_atomic_dec_acqb erts_no_atomic_dec
-#define erts_smp_atomic_add_read_acqb erts_no_atomic_add_read
-#define erts_smp_atomic_add_acqb erts_no_atomic_add
-#define erts_smp_atomic_read_bor_acqb erts_no_atomic_read_bor
-#define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band
-#define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg
-#define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
-#define erts_smp_atomic_read_bset_acqb erts_no_atomic_read_bset
-
-#define erts_smp_atomic_init_relb erts_no_atomic_set
-#define erts_smp_atomic_set_relb erts_no_atomic_set
-#define erts_smp_atomic_read_relb erts_no_atomic_read
-#define erts_smp_atomic_inc_read_relb erts_no_atomic_inc_read
-#define erts_smp_atomic_dec_read_relb erts_no_atomic_dec_read
-#define erts_smp_atomic_inc_relb erts_no_atomic_inc
-#define erts_smp_atomic_dec_relb erts_no_atomic_dec
-#define erts_smp_atomic_add_read_relb erts_no_atomic_add_read
-#define erts_smp_atomic_add_relb erts_no_atomic_add
-#define erts_smp_atomic_read_bor_relb erts_no_atomic_read_bor
-#define erts_smp_atomic_read_band_relb erts_no_atomic_read_band
-#define erts_smp_atomic_xchg_relb erts_no_atomic_xchg
-#define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
-#define erts_smp_atomic_read_bset_relb erts_no_atomic_read_bset
-
-#define erts_smp_atomic_init_ddrb erts_no_atomic_set
-#define erts_smp_atomic_set_ddrb erts_no_atomic_set
-#define erts_smp_atomic_read_ddrb erts_no_atomic_read
-#define erts_smp_atomic_inc_read_ddrb erts_no_atomic_inc_read
-#define erts_smp_atomic_dec_read_ddrb erts_no_atomic_dec_read
-#define erts_smp_atomic_inc_ddrb erts_no_atomic_inc
-#define erts_smp_atomic_dec_ddrb erts_no_atomic_dec
-#define erts_smp_atomic_add_read_ddrb erts_no_atomic_add_read
-#define erts_smp_atomic_add_ddrb erts_no_atomic_add
-#define erts_smp_atomic_read_bor_ddrb erts_no_atomic_read_bor
-#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band
-#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg
-#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
-#define erts_smp_atomic_read_bset_ddrb erts_no_atomic_read_bset
-
-#define erts_smp_atomic_init_rb erts_no_atomic_set
-#define erts_smp_atomic_set_rb erts_no_atomic_set
-#define erts_smp_atomic_read_rb erts_no_atomic_read
-#define erts_smp_atomic_inc_read_rb erts_no_atomic_inc_read
-#define erts_smp_atomic_dec_read_rb erts_no_atomic_dec_read
-#define erts_smp_atomic_inc_rb erts_no_atomic_inc
-#define erts_smp_atomic_dec_rb erts_no_atomic_dec
-#define erts_smp_atomic_add_read_rb erts_no_atomic_add_read
-#define erts_smp_atomic_add_rb erts_no_atomic_add
-#define erts_smp_atomic_read_bor_rb erts_no_atomic_read_bor
-#define erts_smp_atomic_read_band_rb erts_no_atomic_read_band
-#define erts_smp_atomic_xchg_rb erts_no_atomic_xchg
-#define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
-#define erts_smp_atomic_read_bset_rb erts_no_atomic_read_bset
-
-#define erts_smp_atomic_init_wb erts_no_atomic_set
-#define erts_smp_atomic_set_wb erts_no_atomic_set
-#define erts_smp_atomic_read_wb erts_no_atomic_read
-#define erts_smp_atomic_inc_read_wb erts_no_atomic_inc_read
-#define erts_smp_atomic_dec_read_wb erts_no_atomic_dec_read
-#define erts_smp_atomic_inc_wb erts_no_atomic_inc
-#define erts_smp_atomic_dec_wb erts_no_atomic_dec
-#define erts_smp_atomic_add_read_wb erts_no_atomic_add_read
-#define erts_smp_atomic_add_wb erts_no_atomic_add
-#define erts_smp_atomic_read_bor_wb erts_no_atomic_read_bor
-#define erts_smp_atomic_read_band_wb erts_no_atomic_read_band
-#define erts_smp_atomic_xchg_wb erts_no_atomic_xchg
-#define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
-#define erts_smp_atomic_read_bset_wb erts_no_atomic_read_bset
-
-#define erts_smp_atomic_set_dirty erts_no_atomic_set
-#define erts_smp_atomic_read_dirty erts_no_atomic_read
-
-/* 32-bit atomics */
-
-#define erts_smp_atomic32_init_nob erts_no_atomic32_set
-#define erts_smp_atomic32_set_nob erts_no_atomic32_set
-#define erts_smp_atomic32_read_nob erts_no_atomic32_read
-#define erts_smp_atomic32_inc_read_nob erts_no_atomic32_inc_read
-#define erts_smp_atomic32_dec_read_nob erts_no_atomic32_dec_read
-#define erts_smp_atomic32_inc_nob erts_no_atomic32_inc
-#define erts_smp_atomic32_dec_nob erts_no_atomic32_dec
-#define erts_smp_atomic32_add_read_nob erts_no_atomic32_add_read
-#define erts_smp_atomic32_add_nob erts_no_atomic32_add
-#define erts_smp_atomic32_read_bor_nob erts_no_atomic32_read_bor
-#define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band
-#define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg
-#define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
-#define erts_smp_atomic32_read_bset_nob erts_no_atomic32_read_bset
-
-#define erts_smp_atomic32_init_mb erts_no_atomic32_set
-#define erts_smp_atomic32_set_mb erts_no_atomic32_set
-#define erts_smp_atomic32_read_mb erts_no_atomic32_read
-#define erts_smp_atomic32_inc_read_mb erts_no_atomic32_inc_read
-#define erts_smp_atomic32_dec_read_mb erts_no_atomic32_dec_read
-#define erts_smp_atomic32_inc_mb erts_no_atomic32_inc
-#define erts_smp_atomic32_dec_mb erts_no_atomic32_dec
-#define erts_smp_atomic32_add_read_mb erts_no_atomic32_add_read
-#define erts_smp_atomic32_add_mb erts_no_atomic32_add
-#define erts_smp_atomic32_read_bor_mb erts_no_atomic32_read_bor
-#define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band
-#define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg
-#define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
-#define erts_smp_atomic32_read_bset_mb erts_no_atomic32_read_bset
-
-#define erts_smp_atomic32_init_acqb erts_no_atomic32_set
-#define erts_smp_atomic32_set_acqb erts_no_atomic32_set
-#define erts_smp_atomic32_read_acqb erts_no_atomic32_read
-#define erts_smp_atomic32_inc_read_acqb erts_no_atomic32_inc_read
-#define erts_smp_atomic32_dec_read_acqb erts_no_atomic32_dec_read
-#define erts_smp_atomic32_inc_acqb erts_no_atomic32_inc
-#define erts_smp_atomic32_dec_acqb erts_no_atomic32_dec
-#define erts_smp_atomic32_add_read_acqb erts_no_atomic32_add_read
-#define erts_smp_atomic32_add_acqb erts_no_atomic32_add
-#define erts_smp_atomic32_read_bor_acqb erts_no_atomic32_read_bor
-#define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band
-#define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg
-#define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
-#define erts_smp_atomic32_read_bset_acqb erts_no_atomic32_read_bset
-
-#define erts_smp_atomic32_init_relb erts_no_atomic32_set
-#define erts_smp_atomic32_set_relb erts_no_atomic32_set
-#define erts_smp_atomic32_read_relb erts_no_atomic32_read
-#define erts_smp_atomic32_inc_read_relb erts_no_atomic32_inc_read
-#define erts_smp_atomic32_dec_read_relb erts_no_atomic32_dec_read
-#define erts_smp_atomic32_inc_relb erts_no_atomic32_inc
-#define erts_smp_atomic32_dec_relb erts_no_atomic32_dec
-#define erts_smp_atomic32_add_read_relb erts_no_atomic32_add_read
-#define erts_smp_atomic32_add_relb erts_no_atomic32_add
-#define erts_smp_atomic32_read_bor_relb erts_no_atomic32_read_bor
-#define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band
-#define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg
-#define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
-#define erts_smp_atomic32_read_bset_relb erts_no_atomic32_read_bset
-
-#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set
-#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set
-#define erts_smp_atomic32_read_ddrb erts_no_atomic32_read
-#define erts_smp_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
-#define erts_smp_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
-#define erts_smp_atomic32_inc_ddrb erts_no_atomic32_inc
-#define erts_smp_atomic32_dec_ddrb erts_no_atomic32_dec
-#define erts_smp_atomic32_add_read_ddrb erts_no_atomic32_add_read
-#define erts_smp_atomic32_add_ddrb erts_no_atomic32_add
-#define erts_smp_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
-#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band
-#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg
-#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
-#define erts_smp_atomic32_read_bset_ddrb erts_no_atomic32_read_bset
-
-#define erts_smp_atomic32_init_rb erts_no_atomic32_set
-#define erts_smp_atomic32_set_rb erts_no_atomic32_set
-#define erts_smp_atomic32_read_rb erts_no_atomic32_read
-#define erts_smp_atomic32_inc_read_rb erts_no_atomic32_inc_read
-#define erts_smp_atomic32_dec_read_rb erts_no_atomic32_dec_read
-#define erts_smp_atomic32_inc_rb erts_no_atomic32_inc
-#define erts_smp_atomic32_dec_rb erts_no_atomic32_dec
-#define erts_smp_atomic32_add_read_rb erts_no_atomic32_add_read
-#define erts_smp_atomic32_add_rb erts_no_atomic32_add
-#define erts_smp_atomic32_read_bor_rb erts_no_atomic32_read_bor
-#define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band
-#define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg
-#define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
-#define erts_smp_atomic32_read_bset_rb erts_no_atomic32_read_bset
-
-#define erts_smp_atomic32_init_wb erts_no_atomic32_set
-#define erts_smp_atomic32_set_wb erts_no_atomic32_set
-#define erts_smp_atomic32_read_wb erts_no_atomic32_read
-#define erts_smp_atomic32_inc_read_wb erts_no_atomic32_inc_read
-#define erts_smp_atomic32_dec_read_wb erts_no_atomic32_dec_read
-#define erts_smp_atomic32_inc_wb erts_no_atomic32_inc
-#define erts_smp_atomic32_dec_wb erts_no_atomic32_dec
-#define erts_smp_atomic32_add_read_wb erts_no_atomic32_add_read
-#define erts_smp_atomic32_add_wb erts_no_atomic32_add
-#define erts_smp_atomic32_read_bor_wb erts_no_atomic32_read_bor
-#define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band
-#define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg
-#define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
-#define erts_smp_atomic32_read_bset_wb erts_no_atomic32_read_bset
-
-#define erts_smp_atomic32_set_dirty erts_no_atomic32_set
-#define erts_smp_atomic32_read_dirty erts_no_atomic32_read
-
-/* 64-bit atomics */
-
-#define erts_smp_atomic64_init_nob erts_no_atomic64_set
-#define erts_smp_atomic64_set_nob erts_no_atomic64_set
-#define erts_smp_atomic64_read_nob erts_no_atomic64_read
-#define erts_smp_atomic64_inc_read_nob erts_no_atomic64_inc_read
-#define erts_smp_atomic64_dec_read_nob erts_no_atomic64_dec_read
-#define erts_smp_atomic64_inc_nob erts_no_atomic64_inc
-#define erts_smp_atomic64_dec_nob erts_no_atomic64_dec
-#define erts_smp_atomic64_add_read_nob erts_no_atomic64_add_read
-#define erts_smp_atomic64_add_nob erts_no_atomic64_add
-#define erts_smp_atomic64_read_bor_nob erts_no_atomic64_read_bor
-#define erts_smp_atomic64_read_band_nob erts_no_atomic64_read_band
-#define erts_smp_atomic64_xchg_nob erts_no_atomic64_xchg
-#define erts_smp_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
-#define erts_smp_atomic64_read_bset_nob erts_no_atomic64_read_bset
-
-#define erts_smp_atomic64_init_mb erts_no_atomic64_set
-#define erts_smp_atomic64_set_mb erts_no_atomic64_set
-#define erts_smp_atomic64_read_mb erts_no_atomic64_read
-#define erts_smp_atomic64_inc_read_mb erts_no_atomic64_inc_read
-#define erts_smp_atomic64_dec_read_mb erts_no_atomic64_dec_read
-#define erts_smp_atomic64_inc_mb erts_no_atomic64_inc
-#define erts_smp_atomic64_dec_mb erts_no_atomic64_dec
-#define erts_smp_atomic64_add_read_mb erts_no_atomic64_add_read
-#define erts_smp_atomic64_add_mb erts_no_atomic64_add
-#define erts_smp_atomic64_read_bor_mb erts_no_atomic64_read_bor
-#define erts_smp_atomic64_read_band_mb erts_no_atomic64_read_band
-#define erts_smp_atomic64_xchg_mb erts_no_atomic64_xchg
-#define erts_smp_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
-#define erts_smp_atomic64_read_bset_mb erts_no_atomic64_read_bset
-
-#define erts_smp_atomic64_init_acqb erts_no_atomic64_set
-#define erts_smp_atomic64_set_acqb erts_no_atomic64_set
-#define erts_smp_atomic64_read_acqb erts_no_atomic64_read
-#define erts_smp_atomic64_inc_read_acqb erts_no_atomic64_inc_read
-#define erts_smp_atomic64_dec_read_acqb erts_no_atomic64_dec_read
-#define erts_smp_atomic64_inc_acqb erts_no_atomic64_inc
-#define erts_smp_atomic64_dec_acqb erts_no_atomic64_dec
-#define erts_smp_atomic64_add_read_acqb erts_no_atomic64_add_read
-#define erts_smp_atomic64_add_acqb erts_no_atomic64_add
-#define erts_smp_atomic64_read_bor_acqb erts_no_atomic64_read_bor
-#define erts_smp_atomic64_read_band_acqb erts_no_atomic64_read_band
-#define erts_smp_atomic64_xchg_acqb erts_no_atomic64_xchg
-#define erts_smp_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
-#define erts_smp_atomic64_read_bset_acqb erts_no_atomic64_read_bset
-
-#define erts_smp_atomic64_init_relb erts_no_atomic64_set
-#define erts_smp_atomic64_set_relb erts_no_atomic64_set
-#define erts_smp_atomic64_read_relb erts_no_atomic64_read
-#define erts_smp_atomic64_inc_read_relb erts_no_atomic64_inc_read
-#define erts_smp_atomic64_dec_read_relb erts_no_atomic64_dec_read
-#define erts_smp_atomic64_inc_relb erts_no_atomic64_inc
-#define erts_smp_atomic64_dec_relb erts_no_atomic64_dec
-#define erts_smp_atomic64_add_read_relb erts_no_atomic64_add_read
-#define erts_smp_atomic64_add_relb erts_no_atomic64_add
-#define erts_smp_atomic64_read_bor_relb erts_no_atomic64_read_bor
-#define erts_smp_atomic64_read_band_relb erts_no_atomic64_read_band
-#define erts_smp_atomic64_xchg_relb erts_no_atomic64_xchg
-#define erts_smp_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
-#define erts_smp_atomic64_read_bset_relb erts_no_atomic64_read_bset
-
-#define erts_smp_atomic64_init_ddrb erts_no_atomic64_set
-#define erts_smp_atomic64_set_ddrb erts_no_atomic64_set
-#define erts_smp_atomic64_read_ddrb erts_no_atomic64_read
-#define erts_smp_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
-#define erts_smp_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
-#define erts_smp_atomic64_inc_ddrb erts_no_atomic64_inc
-#define erts_smp_atomic64_dec_ddrb erts_no_atomic64_dec
-#define erts_smp_atomic64_add_read_ddrb erts_no_atomic64_add_read
-#define erts_smp_atomic64_add_ddrb erts_no_atomic64_add
-#define erts_smp_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
-#define erts_smp_atomic64_read_band_ddrb erts_no_atomic64_read_band
-#define erts_smp_atomic64_xchg_ddrb erts_no_atomic64_xchg
-#define erts_smp_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
-#define erts_smp_atomic64_read_bset_ddrb erts_no_atomic64_read_bset
-
-#define erts_smp_atomic64_init_rb erts_no_atomic64_set
-#define erts_smp_atomic64_set_rb erts_no_atomic64_set
-#define erts_smp_atomic64_read_rb erts_no_atomic64_read
-#define erts_smp_atomic64_inc_read_rb erts_no_atomic64_inc_read
-#define erts_smp_atomic64_dec_read_rb erts_no_atomic64_dec_read
-#define erts_smp_atomic64_inc_rb erts_no_atomic64_inc
-#define erts_smp_atomic64_dec_rb erts_no_atomic64_dec
-#define erts_smp_atomic64_add_read_rb erts_no_atomic64_add_read
-#define erts_smp_atomic64_add_rb erts_no_atomic64_add
-#define erts_smp_atomic64_read_bor_rb erts_no_atomic64_read_bor
-#define erts_smp_atomic64_read_band_rb erts_no_atomic64_read_band
-#define erts_smp_atomic64_xchg_rb erts_no_atomic64_xchg
-#define erts_smp_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
-#define erts_smp_atomic64_read_bset_rb erts_no_atomic64_read_bset
-
-#define erts_smp_atomic64_init_wb erts_no_atomic64_set
-#define erts_smp_atomic64_set_wb erts_no_atomic64_set
-#define erts_smp_atomic64_read_wb erts_no_atomic64_read
-#define erts_smp_atomic64_inc_read_wb erts_no_atomic64_inc_read
-#define erts_smp_atomic64_dec_read_wb erts_no_atomic64_dec_read
-#define erts_smp_atomic64_inc_wb erts_no_atomic64_inc
-#define erts_smp_atomic64_dec_wb erts_no_atomic64_dec
-#define erts_smp_atomic64_add_read_wb erts_no_atomic64_add_read
-#define erts_smp_atomic64_add_wb erts_no_atomic64_add
-#define erts_smp_atomic64_read_bor_wb erts_no_atomic64_read_bor
-#define erts_smp_atomic64_read_band_wb erts_no_atomic64_read_band
-#define erts_smp_atomic64_xchg_wb erts_no_atomic64_xchg
-#define erts_smp_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
-#define erts_smp_atomic64_read_bset_wb erts_no_atomic64_read_bset
-
-#define erts_smp_atomic64_set_dirty erts_no_atomic64_set
-#define erts_smp_atomic64_read_dirty erts_no_atomic64_read
-
-#endif /* !ERTS_SMP */
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_INLINE void
 erts_smp_thr_init(erts_smp_thr_init_data_t *id)
 {
-#ifdef ERTS_SMP
     erts_thr_init(id);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_thr_create(erts_smp_tid_t *tid, void * (*func)(void *), void *arg,
 		    erts_smp_thr_opts_t *opts)
 {
-#ifdef ERTS_SMP
     erts_thr_create(tid, func, arg, opts);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res)
 {
-#ifdef ERTS_SMP
     erts_thr_join(tid, thr_res);
-#endif
 }
 
 
 ERTS_GLB_INLINE void
 erts_smp_thr_detach(erts_smp_tid_t tid)
 {
-#ifdef ERTS_SMP
     erts_thr_detach(tid);
-#endif
 }
 
 
 ERTS_GLB_INLINE void
 erts_smp_thr_exit(void *res)
 {
-#ifdef ERTS_SMP
     erts_thr_exit(res);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_install_exit_handler(void (*exit_handler)(void))
 {
-#ifdef ERTS_SMP
     erts_thr_install_exit_handler(exit_handler);
-#endif
 }
 
 ERTS_GLB_INLINE erts_smp_tid_t
 erts_smp_thr_self(void)
 {
-#ifdef ERTS_SMP
     return erts_thr_self();
-#else
-    return 0;
-#endif
 }
 
 
 ERTS_GLB_INLINE int
 erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y)
 {
-#ifdef ERTS_SMP
     return erts_equal_tids(x, y);
-#else
-    return 1;
-#endif
 }
 
 
@@ -1050,34 +614,26 @@ erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y)
 ERTS_GLB_INLINE void
 erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx)
 {
-#ifdef ERTS_SMP
     erts_rec_mtx_init(mtx);
-#endif
 }
 #endif
 
 ERTS_GLB_INLINE void
 erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags)
 {
-#ifdef ERTS_SMP
     erts_mtx_init(mtx, name, extra, flags);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags)
 {
-#ifdef ERTS_SMP
     erts_mtx_init_locked(mtx, name, extra, flags);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_mtx_destroy(erts_smp_mtx_t *mtx)
 {
-#ifdef ERTS_SMP
     erts_mtx_destroy(mtx);
-#endif
 }
 
 ERTS_GLB_INLINE int
@@ -1089,10 +645,8 @@ erts_smp_mtx_trylock(erts_smp_mtx_t *mtx)
 {
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
     return erts_mtx_trylock_x(mtx,file,line);
-#elif defined(ERTS_SMP)
-    return erts_mtx_trylock(mtx);
 #else
-    return 0;
+    return erts_mtx_trylock(mtx);
 #endif
 
 }
@@ -1107,7 +661,7 @@ erts_smp_mtx_lock(erts_smp_mtx_t *mtx)
 {
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
     erts_mtx_lock_x(mtx, file, line);
-#elif defined(ERTS_SMP)
+#else
     erts_mtx_lock(mtx);
 #endif
 }
@@ -1115,9 +669,7 @@ erts_smp_mtx_lock(erts_smp_mtx_t *mtx)
 ERTS_GLB_INLINE void
 erts_smp_mtx_unlock(erts_smp_mtx_t *mtx)
 {
-#ifdef ERTS_SMP
     erts_mtx_unlock(mtx);
-#endif
 }
 
 ERTS_GLB_INLINE int
@@ -1133,25 +685,19 @@ erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx)
 ERTS_GLB_INLINE void
 erts_smp_cnd_init(erts_smp_cnd_t *cnd)
 {
-#ifdef ERTS_SMP
     erts_cnd_init(cnd);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_cnd_destroy(erts_smp_cnd_t *cnd)
 {
-#ifdef ERTS_SMP
     erts_cnd_destroy(cnd);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx)
 {
-#ifdef ERTS_SMP
     erts_cnd_wait(cnd, mtx);
-#endif
 }
 
 /*
@@ -1167,26 +713,20 @@ erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx)
 ERTS_GLB_INLINE void
 erts_smp_cnd_signal(erts_smp_cnd_t *cnd)
 {
-#ifdef ERTS_SMP
     erts_cnd_signal(cnd);
-#endif
 }
 
 
 ERTS_GLB_INLINE void
 erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd)
 {
-#ifdef ERTS_SMP
     erts_cnd_broadcast(cnd);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_rwmtx_set_reader_group(int no)
 {
-#ifdef ERTS_SMP
     erts_rwmtx_set_reader_group(no);
-#endif
 }
 
 ERTS_GLB_INLINE void
@@ -1195,9 +735,7 @@ erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx,
                     Eterm extra,
                     erts_lock_flags_t flags)
 {
-#ifdef ERTS_SMP
     erts_smp_rwmtx_init_opt(rwmtx, NULL, name, extra, flags);
-#endif
 }
 
 ERTS_GLB_INLINE void
@@ -1207,17 +745,13 @@ erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
                         Eterm extra,
                         erts_lock_flags_t flags)
 {
-#ifdef ERTS_SMP
     erts_rwmtx_init_opt(rwmtx, opt, name, extra, flags);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx)
 {
-#ifdef ERTS_SMP
     erts_rwmtx_destroy(rwmtx);
-#endif
 }
 
 ERTS_GLB_INLINE int
@@ -1229,10 +763,8 @@ erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx)
 {
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
     return erts_rwmtx_tryrlock_x(rwmtx, file, line);
-#elif defined(ERTS_SMP)
-    return erts_rwmtx_tryrlock(rwmtx);
 #else
-    return 0;
+    return erts_rwmtx_tryrlock(rwmtx);
 #endif
 }
 
@@ -1245,7 +777,7 @@ erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx)
 {
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
     erts_rwmtx_rlock_x(rwmtx, file, line);
-#elif defined(ERTS_SMP)
+#else
     erts_rwmtx_rlock(rwmtx);
 #endif
 }
@@ -1253,9 +785,7 @@ erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx)
 ERTS_GLB_INLINE void
 erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx)
 {
-#ifdef ERTS_SMP
     erts_rwmtx_runlock(rwmtx);
-#endif
 }
 
 
@@ -1268,10 +798,8 @@ erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx)
 {
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
     return erts_rwmtx_tryrwlock_x(rwmtx, file, line);
-#elif defined(ERTS_SMP)
-    return erts_rwmtx_tryrwlock(rwmtx);
 #else
-    return 0;
+    return erts_rwmtx_tryrwlock(rwmtx);
 #endif
 }
 
@@ -1284,7 +812,7 @@ erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx)
 {
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
     erts_rwmtx_rwlock_x(rwmtx, file, line);
-#elif defined(ERTS_SMP)
+#else
     erts_rwmtx_rwlock(rwmtx);
 #endif
 }
@@ -1292,9 +820,7 @@ erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx)
 ERTS_GLB_INLINE void
 erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx)
 {
-#ifdef ERTS_SMP
     erts_rwmtx_rwunlock(rwmtx);
-#endif
 }
 
 #if 0 /* The following rwmtx function names are
@@ -1346,31 +872,19 @@ erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx)
 ERTS_GLB_INLINE void
 erts_smp_spinlock_init(erts_smp_spinlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags)
 {
-#ifdef ERTS_SMP
     erts_spinlock_init(lock, name, extra, flags);
-#else
-    (void)lock;
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock)
 {
-#ifdef ERTS_SMP
     erts_spinlock_destroy(lock);
-#else
-    (void)lock;
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_spin_unlock(erts_smp_spinlock_t *lock)
 {
-#ifdef ERTS_SMP
     erts_spin_unlock(lock);
-#else
-    (void)lock;
-#endif
 }
 
 ERTS_GLB_INLINE void
@@ -1382,10 +896,8 @@ erts_smp_spin_lock(erts_smp_spinlock_t *lock)
 {
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
     erts_spin_lock_x(lock, file, line);
-#elif defined(ERTS_SMP)
-    erts_spin_lock(lock);
 #else
-    (void)lock;
+    erts_spin_lock(lock);
 #endif
 }
 
@@ -1402,31 +914,19 @@ erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock)
 ERTS_GLB_INLINE void
 erts_smp_rwlock_init(erts_smp_rwlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags)
 {
-#ifdef ERTS_SMP
     erts_rwlock_init(lock, name, extra, flags);
-#else
-    (void)lock;
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock)
 {
-#ifdef ERTS_SMP
     erts_rwlock_destroy(lock);
-#else
-    (void)lock;
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_read_unlock(erts_smp_rwlock_t *lock)
 {
-#ifdef ERTS_SMP
     erts_read_unlock(lock);
-#else
-    (void)lock;
-#endif
 }
 
 ERTS_GLB_INLINE void
@@ -1438,21 +938,15 @@ erts_smp_read_lock(erts_smp_rwlock_t *lock)
 {
 #if defined(ERTS_ENABLE_LOCK_POSITION) && defined(ERTS_SMP)
     erts_read_lock_x(lock, file, line);
-#elif defined(ERTS_SMP)
-    erts_read_lock(lock);
 #else
-    (void)lock;
+    erts_read_lock(lock);
 #endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_write_unlock(erts_smp_rwlock_t *lock)
 {
-#ifdef ERTS_SMP
     erts_write_unlock(lock);
-#else
-    (void)lock;
-#endif
 }
 
 ERTS_GLB_INLINE void
@@ -1464,10 +958,8 @@ erts_smp_write_lock(erts_smp_rwlock_t *lock)
 {
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
     erts_write_lock_x(lock, file, line);
-#elif defined(ERTS_SMP)
-    erts_write_lock(lock);
 #else
-    (void)lock;
+    erts_write_lock(lock);
 #endif
 }
 
@@ -1494,35 +986,25 @@ erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock)
 ERTS_GLB_INLINE void
 erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp, char* keyname)
 {
-#ifdef ERTS_SMP
     erts_tsd_key_create(keyp,keyname);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_tsd_key_delete(erts_smp_tsd_key_t key)
 {
-#ifdef ERTS_SMP
     erts_tsd_key_delete(key);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value)
 {
-#ifdef ERTS_SMP
     erts_tsd_set(key, value);
-#endif
 }
 
 ERTS_GLB_INLINE void *
 erts_smp_tsd_get(erts_smp_tsd_key_t key)
 {
-#ifdef ERTS_SMP
     return erts_tsd_get(key);
-#else
-    return NULL;
-#endif
 }
 
 #ifdef ERTS_THR_HAVE_SIG_FUNCS
@@ -1531,17 +1013,13 @@ erts_smp_tsd_get(erts_smp_tsd_key_t key)
 ERTS_GLB_INLINE void
 erts_smp_thr_sigmask(int how, const sigset_t *set, sigset_t *oset)
 {
-#ifdef ERTS_SMP
     erts_thr_sigmask(how, set, oset);
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_smp_thr_sigwait(const sigset_t *set, int *sig)
 {
-#ifdef ERTS_SMP
     erts_thr_sigwait(set, sig);
-#endif
 }
 
 #endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 2a9f276e02..96824dc06e 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -80,7 +80,6 @@
 #include "erl_thr_progress.h"
 #include "global.h"
 
-#ifdef ERTS_SMP
 
 #define ERTS_THR_PRGR_DBG_CHK_WAKEUP_REQUEST_VALUE 0
 
@@ -1513,4 +1512,3 @@ void erts_thr_progress_dbg_print_state(void)
 
 }
 
-#endif
diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h
index b2894ba1fe..80ede05d73 100644
--- a/erts/emulator/beam/erl_thr_progress.h
+++ b/erts/emulator/beam/erl_thr_progress.h
@@ -33,13 +33,6 @@
 
 #include "sys.h"
 
-#ifndef ERTS_SMP
-
-#define erts_smp_thr_progress_block() ((void) 0)
-#define erts_smp_thr_progress_unblock() ((void) 0)
-#define erts_smp_thr_progress_is_blocking() 1
-
-#else /* ERTS_SMP */
 
 #define erts_smp_thr_progress_block erts_thr_progress_block
 #define erts_smp_thr_progress_unblock erts_thr_progress_unblock
@@ -87,13 +80,10 @@ typedef struct {
 int erts_thr_progress_fatal_error_block(ErtsThrPrgrData *tmp_tpd_bufp);
 void erts_thr_progress_fatal_error_wait(SWord timeout);
 
-#endif /* ERTS_SMP */
 
 typedef struct ErtsThrPrgrLaterOp_ ErtsThrPrgrLaterOp;
 struct ErtsThrPrgrLaterOp_ {
-#ifdef ERTS_SMP
     ErtsThrPrgrVal later;
-#endif
     void (*func)(void *);
     void *data;
     ErtsThrPrgrLaterOp *next;
@@ -107,7 +97,6 @@ struct ErtsThrPrgrLaterOp_ {
 #include "erl_threads.h"
 #include "erl_process.h"
 
-#ifdef ERTS_SMP
 
 /* ERTS_THR_PRGR_VAL_FIRST should only be used when initializing... */
 #define ERTS_THR_PRGR_VAL_FIRST ((ErtsThrPrgrVal) 0)
@@ -324,6 +313,5 @@ erts_thr_progress_has_reached(ErtsThrPrgrVal val)
 
 #endif
 
-#endif /* ERTS_SMP */
 
 #endif
diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c
index f56d0828dd..66566edec5 100644
--- a/erts/emulator/beam/erl_thr_queue.c
+++ b/erts/emulator/beam/erl_thr_queue.c
@@ -87,32 +87,10 @@
 
 #define ERTS_THR_Q_MAX_FINI_DEQ_OPS 50
 
-#ifdef ERTS_SMP
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(sl_element,
 				 ErtsThrQElement_t,
 				 1000,
 				 ERTS_ALC_T_THR_Q_EL_SL)
-#else
-
-static void
-init_sl_element_alloc(void)
-{
-}
-
-static ErtsThrQElement_t *
-sl_element_alloc(void)
-{
-    return erts_alloc(ERTS_ALC_T_THR_Q_EL_SL,
-		      sizeof(ErtsThrQElement_t));
-}
-
-static void
-sl_element_free(ErtsThrQElement_t *p)
-{
-    erts_free(ERTS_ALC_T_THR_Q_EL_SL, p);
-}
-
-#endif
 
 #define ErtsThrQDirtyReadEl(A) \
     ((ErtsThrQElement_t *) erts_atomic_read_dirty((A)))
@@ -135,14 +113,6 @@ static void noop_callback(void *arg) { }
 void
 erts_thr_q_initialize(ErtsThrQ_t *q, ErtsThrQInit_t *qi)
 {
-#ifndef USE_THREADS
-    q->init = *qi;
-    if (!q->init.notify)
-	q->init.notify = noop_callback;
-    q->first = NULL;
-    q->last = NULL;
-    q->q.blk = NULL;
-#else
     erts_atomic_init_nob(&q->tail.data.marker.next, ERTS_AINT_NULL);
     q->tail.data.marker.data.ptr = NULL;
     erts_atomic_init_nob(&q->tail.data.last,
@@ -164,10 +134,8 @@ erts_thr_q_initialize(ErtsThrQ_t *q, ErtsThrQInit_t *qi)
     q->head.deq_fini.automatic = qi->auto_finalize_dequeue;
     q->head.deq_fini.start = NULL;
     q->head.deq_fini.end = NULL;
-#ifdef ERTS_SMP
     q->head.next.thr_progress = erts_thr_progress_current();
     q->head.next.thr_progress_reached = 1;
-#endif
     q->head.next.um_refc_ix = 1;
     q->head.next.unref_end = &q->tail.data.marker;
     q->head.used_marker = 1;
@@ -176,15 +144,12 @@ erts_thr_q_initialize(ErtsThrQ_t *q, ErtsThrQInit_t *qi)
     q->q.finalizing = 0;
     q->q.live = qi->live.queue;
     q->q.blk = NULL;
-#endif
 }
 
 ErtsThrQCleanState_t
 erts_thr_q_finalize(ErtsThrQ_t *q)
 {
-#ifdef USE_THREADS
     q->q.finalizing = 1;
-#endif
     while (erts_thr_q_dequeue(q));
     return erts_thr_q_clean(q);
 }
@@ -229,7 +194,6 @@ erts_thr_q_destroy(ErtsThrQ_t *q)
     return erts_thr_q_finalize(q);
 }
 
-#ifdef USE_THREADS
 
 static void
 destroy(ErtsThrQ_t *q)
@@ -249,7 +213,6 @@ destroy(ErtsThrQ_t *q)
     erts_free(atype, q->q.blk);
 }
 
-#endif
 
 static ERTS_INLINE ErtsThrQElement_t *
 element_live_alloc(ErtsThrQLive_t live)
@@ -267,11 +230,7 @@ static ERTS_INLINE ErtsThrQElement_t *
 element_alloc(ErtsThrQ_t *q)
 {
     ErtsThrQLive_t live;
-#ifdef USE_THREADS
     live = q->tail.data.live;
-#else
-    live = q->init.live.objects;
-#endif
     return element_live_alloc(live);
 }
 
@@ -291,15 +250,10 @@ static ERTS_INLINE void
 element_free(ErtsThrQ_t *q, ErtsThrQElement_t *el)
 {
     ErtsThrQLive_t live;
-#ifdef USE_THREADS
     live = q->head.live;
-#else
-    live = q->init.live.objects;
-#endif
     element_live_free(live, el);
 }
 
-#ifdef USE_THREADS
 
 static ERTS_INLINE ErtsThrQElement_t *
 enqueue_managed(ErtsThrQ_t *q, ErtsThrQElement_t *this)
@@ -423,11 +377,9 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify)
 	return ERTS_THR_Q_CLEAN;
     }
 
-#ifdef ERTS_SMP
     if (q->head.next.thr_progress_reached
 	|| erts_thr_progress_has_reached(q->head.next.thr_progress)) {
 	q->head.next.thr_progress_reached = 1;
-#endif
 	um_refc_ix = q->head.next.um_refc_ix;
 	if (erts_atomic_read_acqb(&q->tail.data.um_refc[um_refc_ix]) == 0) {
 	    /* Move unreferenced end pointer forward... */
@@ -442,20 +394,14 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify)
 		ERTS_SMP_MEMORY_BARRIER;
 	    else {
 		q->head.next.unref_end = (ErtsThrQElement_t *) ilast;
-#ifdef ERTS_SMP
 		q->head.next.thr_progress = erts_thr_progress_later(NULL);
-#endif
 		erts_atomic32_set_relb(&q->tail.data.um_refc_ix,
 				       um_refc_ix);
 		q->head.next.um_refc_ix = um_refc_ix == 0 ? 1 : 0;
-#ifdef ERTS_SMP
 		q->head.next.thr_progress_reached = 0;
-#endif
 	    }
 	}
-#ifdef ERTS_SMP
     }
-#endif
 
     head = ErtsThrQDirtyReadEl(&q->head.head);
     if (q->head.first == head) {
@@ -489,9 +435,7 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify)
 
 check_thr_progress:
 
-#ifdef ERTS_SMP
     if (q->head.next.thr_progress_reached)
-#endif
     {
 	int um_refc_ix = q->head.next.um_refc_ix;
 	if (erts_atomic_read_acqb(&q->tail.data.um_refc[um_refc_ix]) == 0) {
@@ -505,24 +449,16 @@ check_thr_progress:
     return ERTS_THR_Q_NEED_THR_PRGR;
 }
 
-#endif
 
 ErtsThrQCleanState_t
 erts_thr_q_clean(ErtsThrQ_t *q)
 {
-#ifdef USE_THREADS
     return clean(q, ERTS_THR_Q_MAX_SCHED_CLEAN_OPS, 0);
-#else
-    return ERTS_THR_Q_CLEAN;
-#endif
 }
 
 ErtsThrQCleanState_t
 erts_thr_q_inspect(ErtsThrQ_t *q, int ensure_empty)
 {
-#ifndef USE_THREADS
-    return ERTS_THR_Q_CLEAN;
-#else
     ErtsThrQElement_t *head = ErtsThrQDirtyReadEl(&q->head.head);
     if (ensure_empty) {
 	erts_aint_t inext;
@@ -553,39 +489,21 @@ erts_thr_q_inspect(ErtsThrQ_t *q, int ensure_empty)
     if (q->head.first != q->head.unref_end)
 	return ERTS_THR_Q_DIRTY;
 
-#ifdef ERTS_SMP
     if (q->head.next.thr_progress_reached)
-#endif
     {
 	int um_refc_ix = q->head.next.um_refc_ix;
 	if (erts_atomic_read_acqb(&q->tail.data.um_refc[um_refc_ix]) == 0)
 	    return ERTS_THR_Q_DIRTY;
     }
     return ERTS_THR_Q_NEED_THR_PRGR;
-#endif
 }
 
 static void
 enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this)
 {
-#ifndef USE_THREADS
-    ASSERT(data);
-
-    this->next = NULL;
-    this->data.ptr = data;
-
-    if (q->last)
-	q->last->next = this;
-    else {
-	q->first = q->last = this;
-	q->init.notify(q->init.arg);
-    }
-#else
     int notify;
     int um_refc_ix = 0;
-#ifdef ERTS_SMP
     int unmanaged_thread;
-#endif
 
 #if ERTS_THR_Q_DBG_CHK_DATA
     if (!data)
@@ -596,10 +514,8 @@ enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this)
 
     this->data.ptr = data;
 
-#ifdef ERTS_SMP
     unmanaged_thread = !erts_thr_progress_is_managed_thread();
     if (unmanaged_thread)
-#endif
     {
 	um_refc_ix = erts_atomic32_read_acqb(&q->tail.data.um_refc_ix);
 	while (1) {
@@ -616,9 +532,7 @@ enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this)
     notify = this == enqueue_managed(q, this);
 	
 
-#ifdef ERTS_SMP
     if (unmanaged_thread)
-#endif
     {
 	if (notify)
 	    erts_atomic_dec_relb(&q->tail.data.um_refc[um_refc_ix]);
@@ -627,7 +541,6 @@ enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this)
     }
     if (notify)
 	q->tail.data.notify(q->tail.data.arg);
-#endif
 }
 
 void
@@ -645,9 +558,6 @@ erts_thr_q_prepare_enqueue(ErtsThrQ_t *q)
 int
 erts_thr_q_get_finalize_dequeue_data(ErtsThrQ_t *q, ErtsThrQFinDeQ_t *fdp)
 {
-#ifndef USE_THREADS
-    return 0;
-#else
 #ifdef DEBUG
     if (!q->head.deq_fini.start) {
 	ASSERT(!q->head.deq_fini.end);
@@ -670,14 +580,12 @@ erts_thr_q_get_finalize_dequeue_data(ErtsThrQ_t *q, ErtsThrQFinDeQ_t *fdp)
     q->head.deq_fini.start = NULL;
     q->head.deq_fini.end = NULL;
     return fdp->start != NULL;
-#endif
 }
 
 void
 erts_thr_q_append_finalize_dequeue_data(ErtsThrQFinDeQ_t *fdp0,
 					ErtsThrQFinDeQ_t *fdp1)
 {
-#ifdef USE_THREADS
     if (fdp1->start) {
 	if (fdp0->end)
 	    ErtsThrQDirtySetEl(&fdp0->end->next, fdp1->start);
@@ -685,13 +593,11 @@ erts_thr_q_append_finalize_dequeue_data(ErtsThrQFinDeQ_t *fdp0,
 	    fdp0->start = fdp1->start;
 	fdp0->end = fdp1->end;
     }
-#endif
 }
 
 
 int erts_thr_q_finalize_dequeue(ErtsThrQFinDeQ_t *state)
 {
-#ifdef USE_THREADS
     ErtsThrQElement_t *start = state->start;
     if (start) {
 	ErtsThrQLive_t live;
@@ -710,17 +616,14 @@ int erts_thr_q_finalize_dequeue(ErtsThrQFinDeQ_t *state)
 	    return 1; /* More to do */
 	state->end = NULL;
     }
-#endif
     return 0;
 }
 
 void
 erts_thr_q_finalize_dequeue_state_init(ErtsThrQFinDeQ_t *state)
 {
-#ifdef USE_THREADS
     state->start = NULL;
     state->end = NULL;
-#endif
 }
 
 
@@ -734,22 +637,6 @@ erts_thr_q_enqueue_prepared(ErtsThrQ_t *q, void *data, ErtsThrQPrepEnQ_t *prep)
 void *
 erts_thr_q_dequeue(ErtsThrQ_t *q)
 {
-#ifndef USE_THREADS
-    void *res;
-    ErtsThrQElement_t *tmp;
-
-    if (!q->first)
-	return NULL;
-    tmp = q->first;
-    res = tmp->data.ptr;
-    q->first = tmp->next;
-    if (!q->first)
-	q->last = NULL;
-
-    element_free(q, tmp);
-
-    return res;
-#else
     ErtsThrQElement_t *head;
     erts_aint_t inext;
     void *res;
@@ -778,7 +665,6 @@ erts_thr_q_dequeue(ErtsThrQ_t *q)
 	   ? ERTS_THR_Q_MAX_DEQUEUE_CLEAN_OPS
 	   : ERTS_THR_Q_MAX_SCHED_CLEAN_OPS), 1);
     return res;
-#endif
 }
 
 #ifdef USE_LTTNG_VM_TRACEPOINTS
@@ -786,14 +672,6 @@ int
 erts_thr_q_length_dirty(ErtsThrQ_t *q)
 {
     int n = 0;
-#ifndef USE_THREADS
-    void *res;
-    ErtsThrQElement_t *tmp;
-
-    for (tmp = q->first; tmp != NULL; tmp = tmp->next) {
-        n++;
-    }
-#else
     ErtsThrQElement_t *e;
     erts_aint_t inext;
 
@@ -808,7 +686,6 @@ erts_thr_q_length_dirty(ErtsThrQ_t *q)
         }
         inext = erts_atomic_read_acqb(&e->next);
     }
-#endif
     return n;
 }
 #endif
diff --git a/erts/emulator/beam/erl_thr_queue.h b/erts/emulator/beam/erl_thr_queue.h
index 705a67af4c..163a25318d 100644
--- a/erts/emulator/beam/erl_thr_queue.h
+++ b/erts/emulator/beam/erl_thr_queue.h
@@ -78,11 +78,7 @@ typedef struct ErtsThrQElement_t_ ErtsThrQElement_t;
 typedef struct ErtsThrQElement_t ErtsThrQPrepEnQ_t;
 
 struct ErtsThrQElement_t_ {
-#ifdef USE_THREADS
     erts_atomic_t next;
-#else
-    ErtsThrQElement_t *next;
-#endif
     union {
 	erts_atomic_t atmc;
 	void *ptr;
@@ -100,7 +96,6 @@ typedef enum {
     ERTS_THR_Q_DIRTY,
 } ErtsThrQCleanState_t;
 
-#ifdef USE_THREADS
 
 typedef struct {
     ErtsThrQElement_t marker;
@@ -108,9 +103,7 @@ typedef struct {
     erts_atomic_t um_refc[2];
     erts_atomic32_t um_refc_ix;
     ErtsThrQLive_t live;
-#ifdef ERTS_SMP
     erts_atomic32_t thr_prgr_clean_scheduled;
-#endif
     void *arg;
     void (*notify)(void *);
 } ErtsThrQTail_t;
@@ -141,10 +134,8 @@ struct ErtsThrQ_t_ {
 	    ErtsThrQElement_t *end;
 	} deq_fini;
 	struct {
-#ifdef ERTS_SMP
 	    ErtsThrPrgrVal thr_progress;
 	    int thr_progress_reached;
-#endif
 	    int um_refc_ix;
 	    ErtsThrQElement_t *unref_end;
 	} next;
@@ -159,18 +150,6 @@ struct ErtsThrQ_t_ {
     } q;
 };
 
-#else /* !USE_THREADS */
-
-struct ErtsThrQ_t_ {
-    ErtsThrQInit_t init;
-    ErtsThrQElement_t *first;
-    ErtsThrQElement_t *last;
-    struct {
-	void *blk;
-    } q;
-};
-
-#endif
 
 void erts_thr_q_init(void);
 void erts_thr_q_initialize(ErtsThrQ_t *, ErtsThrQInit_t *);
@@ -194,19 +173,15 @@ void erts_thr_q_finalize_dequeue_state_init(ErtsThrQFinDeQ_t *);
 int erts_thr_q_length_dirty(ErtsThrQ_t *);
 #endif
 
-#ifdef ERTS_SMP
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_q_need_thr_progress(ErtsThrQ_t *q);
-#endif
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-#ifdef ERTS_SMP
 ERTS_GLB_INLINE ErtsThrPrgrVal
 erts_thr_q_need_thr_progress(ErtsThrQ_t *q)
 {
     return q->head.next.thr_progress;
 }
-#endif
 
 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 3fdf29d678..55e8254de2 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -262,7 +262,6 @@
 #include "erl_lock_flags.h"
 #include "erl_term.h"
 
-#ifdef USE_THREADS
 
 #define ETHR_TRY_INLINE_FUNCS
 #include "ethread.h"
@@ -405,70 +404,6 @@ __decl_noreturn void  __noreturn erts_thr_fatal_error(int, char *);
 #  define ERTS_HAVE_REC_MTX_INIT	ETHR_HAVE_ETHR_REC_MUTEX_INIT
 #endif
 
-#else /* #ifdef USE_THREADS */
-
-#define ERTS_THR_MEMORY_BARRIER
-#define ERTS_THR_WRITE_MEMORY_BARRIER
-#define ERTS_THR_READ_MEMORY_BARRIER
-#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
-
-#define ERTS_THR_OPTS_DEFAULT_INITER 0
-typedef int erts_thr_opts_t;
-typedef int erts_thr_init_data_t;
-typedef int erts_thr_late_init_data_t;
-typedef int erts_tid_t;
-typedef int erts_mtx_t;
-typedef int erts_cnd_t;
-#define ERTS_RWMTX_OPT_DEFAULT_INITER {0}
-#define ERTS_RWMTX_TYPE_NORMAL 0
-#define ERTS_RWMTX_TYPE_FREQUENT_READ 0
-#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
-#define ERTS_RWMTX_LONG_LIVED 0
-#define ERTS_RWMTX_SHORT_LIVED 0
-#define ERTS_RWMTX_UNKNOWN_LIVED 0
-typedef struct {
-    char type;
-    char lived;
-    int main_spincount;
-    int aux_spincount;
-} erts_rwmtx_opt_t;
-typedef int erts_rwmtx_t;
-typedef int erts_tsd_key_t;
-typedef int erts_tse_t;
-
-typedef struct { SWord sint[2]; } erts_dw_aint_t;
-typedef SWord erts_aint_t;
-typedef Sint32 erts_aint32_t;
-typedef Sint64 erts_aint64_t;
-
-#define erts_dw_atomic_t erts_dw_aint_t
-#define erts_atomic_t erts_aint_t
-#define erts_atomic32_t erts_aint32_t
-#define erts_atomic64_t erts_aint64_t
-
-#if __GNUC__ > 2
-typedef struct { } erts_spinlock_t;
-typedef struct { } erts_rwlock_t;
-#else
-typedef struct { int gcc_is_buggy; } erts_spinlock_t;
-typedef struct { int gcc_is_buggy; } erts_rwlock_t;
-#endif
-
-#ifdef WORDS_BIGENDIAN
-#define ERTS_DW_AINT_LOW_WORD 1
-#define ERTS_DW_AINT_HIGH_WORD 0
-#else
-#define ERTS_DW_AINT_LOW_WORD 0
-#define ERTS_DW_AINT_HIGH_WORD 1
-#endif
-
-#define ERTS_MTX_INITER			0
-#define ERTS_CND_INITER			0
-#define ERTS_THR_INIT_DATA_DEF_INITER	0
-
-#define ERTS_HAVE_REC_MTX_INIT		1
-
-#endif /* #ifdef USE_THREADS */
 
 #define erts_no_dw_atomic_t erts_dw_aint_t
 #define erts_no_atomic_t erts_aint_t
@@ -670,13 +605,10 @@ ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set,
 				      sigset_t *oset);
 ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
 
-#ifdef USE_THREADS
 ERTS_GLB_INLINE void erts_thr_kill(erts_tid_t tid, int sig);
-#endif
 
 #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
 
-#ifdef USE_THREADS
 
 ERTS_GLB_INLINE erts_aint_t
 erts_atomic_read_bset_nob(erts_atomic_t *var,
@@ -1684,379 +1616,6 @@ erts_atomic64_read_dirty(erts_atomic64_t *var)
 
 #endif /* ARCH_32 */
 
-#else /* !USE_THREADS */
-
-/* Double word size atomics */
-
-#define erts_dw_atomic_init_nob erts_no_dw_atomic_set
-#define erts_dw_atomic_set_nob erts_no_dw_atomic_set
-#define erts_dw_atomic_read_nob erts_no_dw_atomic_read
-#define erts_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg
-
-#define erts_dw_atomic_init_mb erts_no_dw_atomic_init
-#define erts_dw_atomic_set_mb erts_no_dw_atomic_set
-#define erts_dw_atomic_read_mb erts_no_dw_atomic_read
-#define erts_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg
-
-#define erts_dw_atomic_init_acqb erts_no_dw_atomic_init
-#define erts_dw_atomic_set_acqb erts_no_dw_atomic_set
-#define erts_dw_atomic_read_acqb erts_no_dw_atomic_read
-#define erts_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg
-
-#define erts_dw_atomic_init_relb erts_no_dw_atomic_init
-#define erts_dw_atomic_set_relb erts_no_dw_atomic_set
-#define erts_dw_atomic_read_relb erts_no_dw_atomic_read
-#define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg
-
-#define erts_dw_atomic_init_ddrb erts_no_dw_atomic_init
-#define erts_dw_atomic_set_ddrb erts_no_dw_atomic_set
-#define erts_dw_atomic_read_ddrb erts_no_dw_atomic_read
-#define erts_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg
-
-#define erts_dw_atomic_init_rb erts_no_dw_atomic_init
-#define erts_dw_atomic_set_rb erts_no_dw_atomic_set
-#define erts_dw_atomic_read_rb erts_no_dw_atomic_read
-#define erts_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg
-
-#define erts_dw_atomic_init_wb erts_no_dw_atomic_init
-#define erts_dw_atomic_set_wb erts_no_dw_atomic_set
-#define erts_dw_atomic_read_wb erts_no_dw_atomic_read
-#define erts_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg
-
-#define erts_dw_atomic_set_dirty erts_no_dw_atomic_set
-#define erts_dw_atomic_read_dirty erts_no_dw_atomic_read
-
-/* Word size atomics */
-
-#define erts_atomic_init_nob erts_no_atomic_set
-#define erts_atomic_set_nob erts_no_atomic_set
-#define erts_atomic_read_nob erts_no_atomic_read
-#define erts_atomic_inc_read_nob erts_no_atomic_inc_read
-#define erts_atomic_dec_read_nob erts_no_atomic_dec_read
-#define erts_atomic_inc_nob erts_no_atomic_inc
-#define erts_atomic_dec_nob erts_no_atomic_dec
-#define erts_atomic_add_read_nob erts_no_atomic_add_read
-#define erts_atomic_add_nob erts_no_atomic_add
-#define erts_atomic_read_bor_nob erts_no_atomic_read_bor
-#define erts_atomic_read_band_nob erts_no_atomic_read_band
-#define erts_atomic_xchg_nob erts_no_atomic_xchg
-#define erts_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
-#define erts_atomic_read_bset_nob erts_no_atomic_read_bset
-
-#define erts_atomic_init_mb erts_no_atomic_set
-#define erts_atomic_set_mb erts_no_atomic_set
-#define erts_atomic_read_mb erts_no_atomic_read
-#define erts_atomic_inc_read_mb erts_no_atomic_inc_read
-#define erts_atomic_dec_read_mb erts_no_atomic_dec_read
-#define erts_atomic_inc_mb erts_no_atomic_inc
-#define erts_atomic_dec_mb erts_no_atomic_dec
-#define erts_atomic_add_read_mb erts_no_atomic_add_read
-#define erts_atomic_add_mb erts_no_atomic_add
-#define erts_atomic_read_bor_mb erts_no_atomic_read_bor
-#define erts_atomic_read_band_mb erts_no_atomic_read_band
-#define erts_atomic_xchg_mb erts_no_atomic_xchg
-#define erts_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
-#define erts_atomic_read_bset_mb erts_no_atomic_read_bset
-
-#define erts_atomic_init_acqb erts_no_atomic_set
-#define erts_atomic_set_acqb erts_no_atomic_set
-#define erts_atomic_read_acqb erts_no_atomic_read
-#define erts_atomic_inc_read_acqb erts_no_atomic_inc_read
-#define erts_atomic_dec_read_acqb erts_no_atomic_dec_read
-#define erts_atomic_inc_acqb erts_no_atomic_inc
-#define erts_atomic_dec_acqb erts_no_atomic_dec
-#define erts_atomic_add_read_acqb erts_no_atomic_add_read
-#define erts_atomic_add_acqb erts_no_atomic_add
-#define erts_atomic_read_bor_acqb erts_no_atomic_read_bor
-#define erts_atomic_read_band_acqb erts_no_atomic_read_band
-#define erts_atomic_xchg_acqb erts_no_atomic_xchg
-#define erts_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
-#define erts_atomic_read_bset_acqb erts_no_atomic_read_bset
-
-#define erts_atomic_init_relb erts_no_atomic_set
-#define erts_atomic_set_relb erts_no_atomic_set
-#define erts_atomic_read_relb erts_no_atomic_read
-#define erts_atomic_inc_read_relb erts_no_atomic_inc_read
-#define erts_atomic_dec_read_relb erts_no_atomic_dec_read
-#define erts_atomic_inc_relb erts_no_atomic_inc
-#define erts_atomic_dec_relb erts_no_atomic_dec
-#define erts_atomic_add_read_relb erts_no_atomic_add_read
-#define erts_atomic_add_relb erts_no_atomic_add
-#define erts_atomic_read_bor_relb erts_no_atomic_read_bor
-#define erts_atomic_read_band_relb erts_no_atomic_read_band
-#define erts_atomic_xchg_relb erts_no_atomic_xchg
-#define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
-#define erts_atomic_read_bset_relb erts_no_atomic_read_bset
-
-#define erts_atomic_init_ddrb erts_no_atomic_set
-#define erts_atomic_set_ddrb erts_no_atomic_set
-#define erts_atomic_read_ddrb erts_no_atomic_read
-#define erts_atomic_inc_read_ddrb erts_no_atomic_inc_read
-#define erts_atomic_dec_read_ddrb erts_no_atomic_dec_read
-#define erts_atomic_inc_ddrb erts_no_atomic_inc
-#define erts_atomic_dec_ddrb erts_no_atomic_dec
-#define erts_atomic_add_read_ddrb erts_no_atomic_add_read
-#define erts_atomic_add_ddrb erts_no_atomic_add
-#define erts_atomic_read_bor_ddrb erts_no_atomic_read_bor
-#define erts_atomic_read_band_ddrb erts_no_atomic_read_band
-#define erts_atomic_xchg_ddrb erts_no_atomic_xchg
-#define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
-#define erts_atomic_read_bset_ddrb erts_no_atomic_read_bset
-
-#define erts_atomic_init_rb erts_no_atomic_set
-#define erts_atomic_set_rb erts_no_atomic_set
-#define erts_atomic_read_rb erts_no_atomic_read
-#define erts_atomic_inc_read_rb erts_no_atomic_inc_read
-#define erts_atomic_dec_read_rb erts_no_atomic_dec_read
-#define erts_atomic_inc_rb erts_no_atomic_inc
-#define erts_atomic_dec_rb erts_no_atomic_dec
-#define erts_atomic_add_read_rb erts_no_atomic_add_read
-#define erts_atomic_add_rb erts_no_atomic_add
-#define erts_atomic_read_bor_rb erts_no_atomic_read_bor
-#define erts_atomic_read_band_rb erts_no_atomic_read_band
-#define erts_atomic_xchg_rb erts_no_atomic_xchg
-#define erts_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
-#define erts_atomic_read_bset_rb erts_no_atomic_read_bset
-
-#define erts_atomic_init_wb erts_no_atomic_set
-#define erts_atomic_set_wb erts_no_atomic_set
-#define erts_atomic_read_wb erts_no_atomic_read
-#define erts_atomic_inc_read_wb erts_no_atomic_inc_read
-#define erts_atomic_dec_read_wb erts_no_atomic_dec_read
-#define erts_atomic_inc_wb erts_no_atomic_inc
-#define erts_atomic_dec_wb erts_no_atomic_dec
-#define erts_atomic_add_read_wb erts_no_atomic_add_read
-#define erts_atomic_add_wb erts_no_atomic_add
-#define erts_atomic_read_bor_wb erts_no_atomic_read_bor
-#define erts_atomic_read_band_wb erts_no_atomic_read_band
-#define erts_atomic_xchg_wb erts_no_atomic_xchg
-#define erts_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
-#define erts_atomic_read_bset_wb erts_no_atomic_read_bset
-
-#define erts_atomic_set_dirty erts_no_atomic_set
-#define erts_atomic_read_dirty erts_no_atomic_read
-
-/* 32-bit atomics */
-
-#define erts_atomic32_init_nob erts_no_atomic32_set
-#define erts_atomic32_set_nob erts_no_atomic32_set
-#define erts_atomic32_read_nob erts_no_atomic32_read
-#define erts_atomic32_inc_read_nob erts_no_atomic32_inc_read
-#define erts_atomic32_dec_read_nob erts_no_atomic32_dec_read
-#define erts_atomic32_inc_nob erts_no_atomic32_inc
-#define erts_atomic32_dec_nob erts_no_atomic32_dec
-#define erts_atomic32_add_read_nob erts_no_atomic32_add_read
-#define erts_atomic32_add_nob erts_no_atomic32_add
-#define erts_atomic32_read_bor_nob erts_no_atomic32_read_bor
-#define erts_atomic32_read_band_nob erts_no_atomic32_read_band
-#define erts_atomic32_xchg_nob erts_no_atomic32_xchg
-#define erts_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
-#define erts_atomic32_read_bset_nob erts_no_atomic32_read_bset
-
-#define erts_atomic32_init_mb erts_no_atomic32_set
-#define erts_atomic32_set_mb erts_no_atomic32_set
-#define erts_atomic32_read_mb erts_no_atomic32_read
-#define erts_atomic32_inc_read_mb erts_no_atomic32_inc_read
-#define erts_atomic32_dec_read_mb erts_no_atomic32_dec_read
-#define erts_atomic32_inc_mb erts_no_atomic32_inc
-#define erts_atomic32_dec_mb erts_no_atomic32_dec
-#define erts_atomic32_add_read_mb erts_no_atomic32_add_read
-#define erts_atomic32_add_mb erts_no_atomic32_add
-#define erts_atomic32_read_bor_mb erts_no_atomic32_read_bor
-#define erts_atomic32_read_band_mb erts_no_atomic32_read_band
-#define erts_atomic32_xchg_mb erts_no_atomic32_xchg
-#define erts_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
-#define erts_atomic32_read_bset_mb erts_no_atomic32_read_bset
-
-#define erts_atomic32_init_acqb erts_no_atomic32_set
-#define erts_atomic32_set_acqb erts_no_atomic32_set
-#define erts_atomic32_read_acqb erts_no_atomic32_read
-#define erts_atomic32_inc_read_acqb erts_no_atomic32_inc_read
-#define erts_atomic32_dec_read_acqb erts_no_atomic32_dec_read
-#define erts_atomic32_inc_acqb erts_no_atomic32_inc
-#define erts_atomic32_dec_acqb erts_no_atomic32_dec
-#define erts_atomic32_add_read_acqb erts_no_atomic32_add_read
-#define erts_atomic32_add_acqb erts_no_atomic32_add
-#define erts_atomic32_read_bor_acqb erts_no_atomic32_read_bor
-#define erts_atomic32_read_band_acqb erts_no_atomic32_read_band
-#define erts_atomic32_xchg_acqb erts_no_atomic32_xchg
-#define erts_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
-#define erts_atomic32_read_bset_acqb erts_no_atomic32_read_bset
-
-#define erts_atomic32_init_relb erts_no_atomic32_set
-#define erts_atomic32_set_relb erts_no_atomic32_set
-#define erts_atomic32_read_relb erts_no_atomic32_read
-#define erts_atomic32_inc_read_relb erts_no_atomic32_inc_read
-#define erts_atomic32_dec_read_relb erts_no_atomic32_dec_read
-#define erts_atomic32_inc_relb erts_no_atomic32_inc
-#define erts_atomic32_dec_relb erts_no_atomic32_dec
-#define erts_atomic32_add_read_relb erts_no_atomic32_add_read
-#define erts_atomic32_add_relb erts_no_atomic32_add
-#define erts_atomic32_read_bor_relb erts_no_atomic32_read_bor
-#define erts_atomic32_read_band_relb erts_no_atomic32_read_band
-#define erts_atomic32_xchg_relb erts_no_atomic32_xchg
-#define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
-#define erts_atomic32_read_bset_relb erts_no_atomic32_read_bset
-
-#define erts_atomic32_init_ddrb erts_no_atomic32_set
-#define erts_atomic32_set_ddrb erts_no_atomic32_set
-#define erts_atomic32_read_ddrb erts_no_atomic32_read
-#define erts_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
-#define erts_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
-#define erts_atomic32_inc_ddrb erts_no_atomic32_inc
-#define erts_atomic32_dec_ddrb erts_no_atomic32_dec
-#define erts_atomic32_add_read_ddrb erts_no_atomic32_add_read
-#define erts_atomic32_add_ddrb erts_no_atomic32_add
-#define erts_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
-#define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band
-#define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg
-#define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
-#define erts_atomic32_read_bset_ddrb erts_no_atomic32_read_bset
-
-#define erts_atomic32_init_rb erts_no_atomic32_set
-#define erts_atomic32_set_rb erts_no_atomic32_set
-#define erts_atomic32_read_rb erts_no_atomic32_read
-#define erts_atomic32_inc_read_rb erts_no_atomic32_inc_read
-#define erts_atomic32_dec_read_rb erts_no_atomic32_dec_read
-#define erts_atomic32_inc_rb erts_no_atomic32_inc
-#define erts_atomic32_dec_rb erts_no_atomic32_dec
-#define erts_atomic32_add_read_rb erts_no_atomic32_add_read
-#define erts_atomic32_add_rb erts_no_atomic32_add
-#define erts_atomic32_read_bor_rb erts_no_atomic32_read_bor
-#define erts_atomic32_read_band_rb erts_no_atomic32_read_band
-#define erts_atomic32_xchg_rb erts_no_atomic32_xchg
-#define erts_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
-#define erts_atomic32_read_bset_rb erts_no_atomic32_read_bset
-
-#define erts_atomic32_init_wb erts_no_atomic32_set
-#define erts_atomic32_set_wb erts_no_atomic32_set
-#define erts_atomic32_read_wb erts_no_atomic32_read
-#define erts_atomic32_inc_read_wb erts_no_atomic32_inc_read
-#define erts_atomic32_dec_read_wb erts_no_atomic32_dec_read
-#define erts_atomic32_inc_wb erts_no_atomic32_inc
-#define erts_atomic32_dec_wb erts_no_atomic32_dec
-#define erts_atomic32_add_read_wb erts_no_atomic32_add_read
-#define erts_atomic32_add_wb erts_no_atomic32_add
-#define erts_atomic32_read_bor_wb erts_no_atomic32_read_bor
-#define erts_atomic32_read_band_wb erts_no_atomic32_read_band
-#define erts_atomic32_xchg_wb erts_no_atomic32_xchg
-#define erts_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
-#define erts_atomic32_read_bset_wb erts_no_atomic32_read_bset
-
-#define erts_atomic32_set_dirty erts_no_atomic32_set
-#define erts_atomic32_read_dirty erts_no_atomic32_read
-
-/* 64-bit atomics */
-
-#define erts_atomic64_init_nob erts_no_atomic64_set
-#define erts_atomic64_set_nob erts_no_atomic64_set
-#define erts_atomic64_read_nob erts_no_atomic64_read
-#define erts_atomic64_inc_read_nob erts_no_atomic64_inc_read
-#define erts_atomic64_dec_read_nob erts_no_atomic64_dec_read
-#define erts_atomic64_inc_nob erts_no_atomic64_inc
-#define erts_atomic64_dec_nob erts_no_atomic64_dec
-#define erts_atomic64_add_read_nob erts_no_atomic64_add_read
-#define erts_atomic64_add_nob erts_no_atomic64_add
-#define erts_atomic64_read_bor_nob erts_no_atomic64_read_bor
-#define erts_atomic64_read_band_nob erts_no_atomic64_read_band
-#define erts_atomic64_xchg_nob erts_no_atomic64_xchg
-#define erts_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
-#define erts_atomic64_read_bset_nob erts_no_atomic64_read_bset
-
-#define erts_atomic64_init_mb erts_no_atomic64_set
-#define erts_atomic64_set_mb erts_no_atomic64_set
-#define erts_atomic64_read_mb erts_no_atomic64_read
-#define erts_atomic64_inc_read_mb erts_no_atomic64_inc_read
-#define erts_atomic64_dec_read_mb erts_no_atomic64_dec_read
-#define erts_atomic64_inc_mb erts_no_atomic64_inc
-#define erts_atomic64_dec_mb erts_no_atomic64_dec
-#define erts_atomic64_add_read_mb erts_no_atomic64_add_read
-#define erts_atomic64_add_mb erts_no_atomic64_add
-#define erts_atomic64_read_bor_mb erts_no_atomic64_read_bor
-#define erts_atomic64_read_band_mb erts_no_atomic64_read_band
-#define erts_atomic64_xchg_mb erts_no_atomic64_xchg
-#define erts_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
-#define erts_atomic64_read_bset_mb erts_no_atomic64_read_bset
-
-#define erts_atomic64_init_acqb erts_no_atomic64_set
-#define erts_atomic64_set_acqb erts_no_atomic64_set
-#define erts_atomic64_read_acqb erts_no_atomic64_read
-#define erts_atomic64_inc_read_acqb erts_no_atomic64_inc_read
-#define erts_atomic64_dec_read_acqb erts_no_atomic64_dec_read
-#define erts_atomic64_inc_acqb erts_no_atomic64_inc
-#define erts_atomic64_dec_acqb erts_no_atomic64_dec
-#define erts_atomic64_add_read_acqb erts_no_atomic64_add_read
-#define erts_atomic64_add_acqb erts_no_atomic64_add
-#define erts_atomic64_read_bor_acqb erts_no_atomic64_read_bor
-#define erts_atomic64_read_band_acqb erts_no_atomic64_read_band
-#define erts_atomic64_xchg_acqb erts_no_atomic64_xchg
-#define erts_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
-#define erts_atomic64_read_bset_acqb erts_no_atomic64_read_bset
-
-#define erts_atomic64_init_relb erts_no_atomic64_set
-#define erts_atomic64_set_relb erts_no_atomic64_set
-#define erts_atomic64_read_relb erts_no_atomic64_read
-#define erts_atomic64_inc_read_relb erts_no_atomic64_inc_read
-#define erts_atomic64_dec_read_relb erts_no_atomic64_dec_read
-#define erts_atomic64_inc_relb erts_no_atomic64_inc
-#define erts_atomic64_dec_relb erts_no_atomic64_dec
-#define erts_atomic64_add_read_relb erts_no_atomic64_add_read
-#define erts_atomic64_add_relb erts_no_atomic64_add
-#define erts_atomic64_read_bor_relb erts_no_atomic64_read_bor
-#define erts_atomic64_read_band_relb erts_no_atomic64_read_band
-#define erts_atomic64_xchg_relb erts_no_atomic64_xchg
-#define erts_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
-#define erts_atomic64_read_bset_relb erts_no_atomic64_read_bset
-
-#define erts_atomic64_init_ddrb erts_no_atomic64_set
-#define erts_atomic64_set_ddrb erts_no_atomic64_set
-#define erts_atomic64_read_ddrb erts_no_atomic64_read
-#define erts_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
-#define erts_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
-#define erts_atomic64_inc_ddrb erts_no_atomic64_inc
-#define erts_atomic64_dec_ddrb erts_no_atomic64_dec
-#define erts_atomic64_add_read_ddrb erts_no_atomic64_add_read
-#define erts_atomic64_add_ddrb erts_no_atomic64_add
-#define erts_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
-#define erts_atomic64_read_band_ddrb erts_no_atomic64_read_band
-#define erts_atomic64_xchg_ddrb erts_no_atomic64_xchg
-#define erts_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
-#define erts_atomic64_read_bset_ddrb erts_no_atomic64_read_bset
-
-#define erts_atomic64_init_rb erts_no_atomic64_set
-#define erts_atomic64_set_rb erts_no_atomic64_set
-#define erts_atomic64_read_rb erts_no_atomic64_read
-#define erts_atomic64_inc_read_rb erts_no_atomic64_inc_read
-#define erts_atomic64_dec_read_rb erts_no_atomic64_dec_read
-#define erts_atomic64_inc_rb erts_no_atomic64_inc
-#define erts_atomic64_dec_rb erts_no_atomic64_dec
-#define erts_atomic64_add_read_rb erts_no_atomic64_add_read
-#define erts_atomic64_add_rb erts_no_atomic64_add
-#define erts_atomic64_read_bor_rb erts_no_atomic64_read_bor
-#define erts_atomic64_read_band_rb erts_no_atomic64_read_band
-#define erts_atomic64_xchg_rb erts_no_atomic64_xchg
-#define erts_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
-#define erts_atomic64_read_bset_rb erts_no_atomic64_read_bset
-
-#define erts_atomic64_init_wb erts_no_atomic64_set
-#define erts_atomic64_set_wb erts_no_atomic64_set
-#define erts_atomic64_read_wb erts_no_atomic64_read
-#define erts_atomic64_inc_read_wb erts_no_atomic64_inc_read
-#define erts_atomic64_dec_read_wb erts_no_atomic64_dec_read
-#define erts_atomic64_inc_wb erts_no_atomic64_inc
-#define erts_atomic64_dec_wb erts_no_atomic64_dec
-#define erts_atomic64_add_read_wb erts_no_atomic64_add_read
-#define erts_atomic64_add_wb erts_no_atomic64_add
-#define erts_atomic64_read_bor_wb erts_no_atomic64_read_bor
-#define erts_atomic64_read_band_wb erts_no_atomic64_read_band
-#define erts_atomic64_xchg_wb erts_no_atomic64_xchg
-#define erts_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
-#define erts_atomic64_read_bset_wb erts_no_atomic64_read_bset
-
-#define erts_atomic64_set_dirty erts_no_atomic64_set
-#define erts_atomic64_read_dirty erts_no_atomic64_read
-
-#endif /* !USE_THREADS */
 
 #include "erl_msacc.h"
 
@@ -2065,110 +1624,83 @@ erts_atomic64_read_dirty(erts_atomic64_t *var)
 ERTS_GLB_INLINE void
 erts_thr_init(erts_thr_init_data_t *id)
 {
-#ifdef USE_THREADS
     int res = ethr_init(id);
     if (res)
 	erts_thr_fatal_error(res, "initialize thread library");
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_thr_late_init(erts_thr_late_init_data_t *id)
 {
-#ifdef USE_THREADS
     int res = ethr_late_init(id);
     if (res)
 	erts_thr_fatal_error(res, "complete initialization of thread library");
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg,
 		erts_thr_opts_t *opts)
 {
-#ifdef USE_THREADS
     int res = ethr_thr_create(tid, func, arg, opts);
     if (res)
 	erts_thr_fatal_error(res, "create thread");
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_thr_join(erts_tid_t tid, void **thr_res)
 {
-#ifdef USE_THREADS
     int res = ethr_thr_join(tid, thr_res);
     if (res)
 	erts_thr_fatal_error(res, "join thread");
-#endif
 }
 
 
 ERTS_GLB_INLINE void
 erts_thr_detach(erts_tid_t tid)
 {
-#ifdef USE_THREADS
     int res = ethr_thr_detach(tid);
     if (res)
 	erts_thr_fatal_error(res, "detach thread");
-#endif
 }
 
 
 ERTS_GLB_INLINE void
 erts_thr_exit(void *res)
 {
-#ifdef USE_THREADS
     ethr_thr_exit(res);
     erts_thr_fatal_error(0, "terminate thread");
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_thr_install_exit_handler(void (*exit_handler)(void))
 {
-#ifdef USE_THREADS
     int res = ethr_install_exit_handler(exit_handler);
     if (res != 0)
 	erts_thr_fatal_error(res, "install thread exit handler");
-#endif
 }
 
 ERTS_GLB_INLINE erts_tid_t
 erts_thr_self(void)
 {
-#ifdef USE_THREADS
     return ethr_self();
-#else
-    return 0;
-#endif
 }
 
 ERTS_GLB_INLINE int
 erts_thr_getname(erts_tid_t tid, char *buf, size_t len)
 {
-#ifdef USE_THREADS
     return ethr_getname(tid, buf, len);
-#else
-    return -1;
-#endif
 }
 
 
 ERTS_GLB_INLINE int
 erts_equal_tids(erts_tid_t x, erts_tid_t y)
 {
-#ifdef USE_THREADS
     return ethr_equal_tids(x, y);
-#else
-    return 1;
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_mtx_init(erts_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags)
 {
-#ifdef USE_THREADS
     int res = ethr_mutex_init(&mtx->mtx);
     if (res) {
         erts_thr_fatal_error(res, "initialize mutex");
@@ -2185,13 +1717,11 @@ erts_mtx_init(erts_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_init_ref_x(&mtx->lcnt, name, extra, flags);
 #endif
-#endif /* USE_THREADS */
 }
 
 ERTS_GLB_INLINE void
 erts_mtx_init_locked(erts_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags)
 {
-#ifdef USE_THREADS
     erts_mtx_init(mtx, name, extra, flags);
 
     ethr_mutex_lock(&mtx->mtx);
@@ -2201,13 +1731,11 @@ erts_mtx_init_locked(erts_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t
     #ifdef ERTS_ENABLE_LOCK_COUNT
         erts_lcnt_trylock(&mtx->lcnt, 1);
     #endif
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_mtx_destroy(erts_mtx_t *mtx)
 {
-#ifdef USE_THREADS
     int res;
 
     ASSERT(!(mtx->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC));
@@ -2230,7 +1758,6 @@ erts_mtx_destroy(erts_mtx_t *mtx)
 #endif
 	    erts_thr_fatal_error(res, "destroy mutex");
     }
-#endif
 }
 
 ERTS_GLB_INLINE int
@@ -2240,7 +1767,6 @@ erts_mtx_trylock_x(erts_mtx_t *mtx, char *file, unsigned int line)
 erts_mtx_trylock(erts_mtx_t *mtx)
 #endif
 {
-#ifdef USE_THREADS
     int res;
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -2262,9 +1788,6 @@ erts_mtx_trylock(erts_mtx_t *mtx)
     erts_lcnt_trylock(&mtx->lcnt, res);
 #endif    
     return res;
-#else
-    return 0;
-#endif
 
 }
 
@@ -2275,7 +1798,6 @@ erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line)
 erts_mtx_lock(erts_mtx_t *mtx)
 #endif
 {
-#ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
 #ifdef ERTS_ENABLE_LOCK_POSITION
     erts_lc_lock_x(&mtx->lc, file, line);
@@ -2290,13 +1812,11 @@ erts_mtx_lock(erts_mtx_t *mtx)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&mtx->lcnt, file, line);
 #endif
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_mtx_unlock(erts_mtx_t *mtx)
 {
-#ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock(&mtx->lc);
 #endif
@@ -2304,7 +1824,6 @@ erts_mtx_unlock(erts_mtx_t *mtx)
     erts_lcnt_unlock(&mtx->lcnt);
 #endif
     ethr_mutex_unlock(&mtx->mtx);
-#endif
 }
 
 ERTS_GLB_INLINE int
@@ -2324,17 +1843,14 @@ erts_lc_mtx_is_locked(erts_mtx_t *mtx)
 ERTS_GLB_INLINE void
 erts_cnd_init(erts_cnd_t *cnd)
 {
-#ifdef USE_THREADS
     int res = ethr_cond_init(cnd);
     if (res)
 	erts_thr_fatal_error(res, "initialize condition variable");
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_cnd_destroy(erts_cnd_t *cnd)
 {
-#ifdef USE_THREADS
     int res = ethr_cond_destroy(cnd);
     if (res != 0) {
 #ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
@@ -2347,13 +1863,11 @@ erts_cnd_destroy(erts_cnd_t *cnd)
 #endif
 	    erts_thr_fatal_error(res, "destroy condition variable");
     }
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx)
 {
-#ifdef USE_THREADS
     int res;
     ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP);
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -2375,7 +1889,6 @@ erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx)
     if (res != 0 && res != EINTR)
 	erts_thr_fatal_error(res, "wait on condition variable");
     ERTS_MSACC_POP_STATE();
-#endif
 }
 
 /*
@@ -2391,18 +1904,14 @@ erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx)
 ERTS_GLB_INLINE void
 erts_cnd_signal(erts_cnd_t *cnd)
 {
-#ifdef USE_THREADS
     ethr_cond_signal(cnd);
-#endif
 }
 
 
 ERTS_GLB_INLINE void
 erts_cnd_broadcast(erts_cnd_t *cnd)
 {
-#ifdef USE_THREADS
     ethr_cond_broadcast(cnd);
-#endif
 }
 
 /* rwmutex */
@@ -2410,7 +1919,6 @@ erts_cnd_broadcast(erts_cnd_t *cnd)
 ERTS_GLB_INLINE void
 erts_rwmtx_set_reader_group(int no)
 {
-#ifdef USE_THREADS
     int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_check_no_locked_of_type(ERTS_LC_FLG_LT_RWMUTEX);
@@ -2418,13 +1926,11 @@ erts_rwmtx_set_reader_group(int no)
     res = ethr_rwmutex_set_reader_group(no);
     if (res != 0)
 	erts_thr_fatal_error(res, "set reader group");
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, erts_rwmtx_opt_t *opt,
                     char *name, Eterm extra, erts_lock_flags_t flags) {
-#ifdef USE_THREADS
     int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt);
     if (res != 0) {
         erts_thr_fatal_error(res, "initialize rwmutex");
@@ -2441,7 +1947,6 @@ erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, erts_rwmtx_opt_t *opt,
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_init_ref_x(&rwmtx->lcnt, name, extra, flags);
 #endif
-#endif /* USE_THREADS */
 }
 
 ERTS_GLB_INLINE void
@@ -2453,7 +1958,6 @@ erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name, Eterm extra,
 ERTS_GLB_INLINE void
 erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
 {
-#ifdef USE_THREADS
     int res;
 
     ASSERT(!(rwmtx->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC));
@@ -2476,7 +1980,6 @@ erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
 #endif
 	    erts_thr_fatal_error(res, "destroy rwmutex");
     }
-#endif
 }
 
 ERTS_GLB_INLINE int
@@ -2486,7 +1989,6 @@ erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
 erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx)
 #endif
 {
-#ifdef USE_THREADS
     int res;
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -2509,9 +2011,6 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx)
 #endif
     
     return res;
-#else
-    return 0;
-#endif
 }
 
 ERTS_GLB_INLINE void
@@ -2521,7 +2020,6 @@ erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
 erts_rwmtx_rlock(erts_rwmtx_t *rwmtx)
 #endif
 {
-#ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
 #ifdef ERTS_ENABLE_LOCK_POSITION
     erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line);
@@ -2536,13 +2034,11 @@ erts_rwmtx_rlock(erts_rwmtx_t *rwmtx)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line);
 #endif
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_rwmtx_runlock(erts_rwmtx_t *rwmtx)
 {
-#ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
 #endif
@@ -2550,7 +2046,6 @@ erts_rwmtx_runlock(erts_rwmtx_t *rwmtx)
     erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTION_READ);
 #endif
     ethr_rwmutex_runlock(&rwmtx->rwmtx);
-#endif
 }
 
 
@@ -2561,7 +2056,6 @@ erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
 erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx)
 #endif
 {
-#ifdef USE_THREADS
     int res;
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -2584,9 +2078,6 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx)
 #endif
     
     return res;
-#else
-    return 0;
-#endif
 }
 
 ERTS_GLB_INLINE void
@@ -2596,7 +2087,6 @@ erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
 erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx)
 #endif
 {
-#ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
 #ifdef ERTS_ENABLE_LOCK_POSITION
     erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
@@ -2611,13 +2101,11 @@ erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line);
 #endif
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx)
 {
-#ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
@@ -2625,7 +2113,6 @@ erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx)
     erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTION_RDWR);
 #endif
     ethr_rwmutex_rwunlock(&rwmtx->rwmtx);
-#endif
 }
 
 #if 0 /* The following rwmtx function names are
@@ -3009,7 +2496,6 @@ erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
 ERTS_GLB_INLINE void
 erts_spinlock_init(erts_spinlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags)
 {
-#ifdef USE_THREADS
     int res = ethr_spinlock_init(&lock->slck);
     if (res) {
         erts_thr_fatal_error(res, "init spinlock");
@@ -3026,13 +2512,11 @@ erts_spinlock_init(erts_spinlock_t *lock, char *name, Eterm extra, erts_lock_fla
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_init_ref_x(&lock->lcnt, name, extra, flags);
 #endif
-#endif /* USE_THREADS */
 }
 
 ERTS_GLB_INLINE void
 erts_spinlock_destroy(erts_spinlock_t *lock)
 {
-#ifdef USE_THREADS
     int res;
 
     ASSERT(!(lock->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC));
@@ -3055,15 +2539,11 @@ erts_spinlock_destroy(erts_spinlock_t *lock)
 #endif
 	    erts_thr_fatal_error(res, "destroy rwlock");
     }
-#else
-    (void)lock;
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_spin_unlock(erts_spinlock_t *lock)
 {
-#ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock(&lock->lc);
 #endif
@@ -3071,9 +2551,6 @@ erts_spin_unlock(erts_spinlock_t *lock)
     erts_lcnt_unlock(&lock->lcnt);
 #endif
     ethr_spin_unlock(&lock->slck);
-#else
-    (void)lock;
-#endif
 }
 
 ERTS_GLB_INLINE void
@@ -3083,7 +2560,6 @@ erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigned int line)
 erts_spin_lock(erts_spinlock_t *lock)
 #endif
 {
-#ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
 #ifdef ERTS_ENABLE_LOCK_POSITION
     erts_lc_lock_x(&lock->lc,file,line);
@@ -3098,9 +2574,6 @@ erts_spin_lock(erts_spinlock_t *lock)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&lock->lcnt, file, line);
 #endif
-#else
-    (void)lock;
-#endif
 }
 
 ERTS_GLB_INLINE int
@@ -3122,7 +2595,6 @@ erts_lc_spinlock_is_locked(erts_spinlock_t *lock)
 ERTS_GLB_INLINE void
 erts_rwlock_init(erts_rwlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags)
 {
-#ifdef USE_THREADS
     int res = ethr_rwlock_init(&lock->rwlck);
     if (res) {
         erts_thr_fatal_error(res, "init rwlock");
@@ -3139,13 +2611,11 @@ erts_rwlock_init(erts_rwlock_t *lock, char *name, Eterm extra, erts_lock_flags_t
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_init_ref_x(&lock->lcnt, name, extra, flags);
 #endif
-#endif /* USE_THREADS */
 }
 
 ERTS_GLB_INLINE void
 erts_rwlock_destroy(erts_rwlock_t *lock)
 {
-#ifdef USE_THREADS
     int res;
 
     ASSERT(!(lock->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC));
@@ -3168,15 +2638,11 @@ erts_rwlock_destroy(erts_rwlock_t *lock)
 #endif
 	    erts_thr_fatal_error(res, "destroy rwlock");
     }
-#else
-    (void)lock;
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_read_unlock(erts_rwlock_t *lock)
 {
-#ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
 #endif
@@ -3184,9 +2650,6 @@ erts_read_unlock(erts_rwlock_t *lock)
     erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LOCK_OPTION_READ);
 #endif
     ethr_read_unlock(&lock->rwlck);
-#else
-    (void)lock;
-#endif
 }
 
 ERTS_GLB_INLINE void
@@ -3196,7 +2659,6 @@ erts_read_lock_x(erts_rwlock_t *lock, char *file, unsigned int line)
 erts_read_lock(erts_rwlock_t *lock)
 #endif
 {
-#ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
 #ifdef ERTS_ENABLE_LOCK_POSITION
     erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ,file,line);
@@ -3211,15 +2673,11 @@ erts_read_lock(erts_rwlock_t *lock)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&lock->lcnt, file, line);
 #endif
-#else
-    (void)lock;
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_write_unlock(erts_rwlock_t *lock)
 {
-#ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
@@ -3227,9 +2685,6 @@ erts_write_unlock(erts_rwlock_t *lock)
     erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LOCK_OPTION_RDWR);
 #endif
     ethr_write_unlock(&lock->rwlck);
-#else
-    (void)lock;
-#endif
 }
 
 ERTS_GLB_INLINE void
@@ -3239,7 +2694,6 @@ erts_write_lock_x(erts_rwlock_t *lock, char *file, unsigned int line)
 erts_write_lock(erts_rwlock_t *lock)
 #endif
 {
-#ifdef USE_THREADS
 #ifdef ERTS_ENABLE_LOCK_CHECK
 #ifdef ERTS_ENABLE_LOCK_POSITION
     erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
@@ -3254,9 +2708,6 @@ erts_write_lock(erts_rwlock_t *lock)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&lock->lcnt, file, line);
 #endif
-#else
-    (void)lock;
-#endif
 }
 
 ERTS_GLB_INLINE int
@@ -3290,125 +2741,90 @@ erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock)
 ERTS_GLB_INLINE void
 erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname)
 {
-#ifdef USE_THREADS
     int res = ethr_tsd_key_create(keyp, keyname);
     if (res)
 	erts_thr_fatal_error(res, "create thread specific data key");
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_tsd_key_delete(erts_tsd_key_t key)
 {
-#ifdef USE_THREADS
     int res = ethr_tsd_key_delete(key);
     if (res)
 	erts_thr_fatal_error(res, "delete thread specific data key");
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_tsd_set(erts_tsd_key_t key, void *value)
 {
-#ifdef USE_THREADS
     int res = ethr_tsd_set(key, value);
     if (res)
 	erts_thr_fatal_error(res, "set thread specific data");
-#endif
 }
 
 ERTS_GLB_INLINE void *
 erts_tsd_get(erts_tsd_key_t key)
 {
-#ifdef USE_THREADS
     return ethr_tsd_get(key);
-#else
-    return NULL;
-#endif
 }
 
 ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void)
 {
-#ifdef USE_THREADS
     return (erts_tse_t *) ethr_get_ts_event();
-#else
-    return (erts_tse_t *) NULL;
-#endif
 }
 
 ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep)
 {
-#ifdef USE_THREADS
     ethr_leave_ts_event(ep);
-#endif
 }
 
 ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep)
 {
-#ifdef USE_THREADS
     int res = ethr_event_prepare_timed(&((ethr_ts_event *) ep)->event);
     if (res != 0)
 	erts_thr_fatal_error(res, "prepare timed");
-#endif
 }
 
 ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep)
 {
-#ifdef USE_THREADS
     ethr_event_set(&((ethr_ts_event *) ep)->event);
-#endif
 }
 
 ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep)
 {
-#ifdef USE_THREADS
     ethr_event_reset(&((ethr_ts_event *) ep)->event);
-#endif
 }
 
 ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep)
 {
-#ifdef USE_THREADS
     int res;
     ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP);
     res = ethr_event_wait(&((ethr_ts_event *) ep)->event);
     ERTS_MSACC_POP_STATE();
     return res;
-#else
-    return ENOTSUP;
-#endif
 }
 
 ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount)
 {
-#ifdef USE_THREADS
     int res;
     ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP);
     res = ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount);
     ERTS_MSACC_POP_STATE();
     return res;
-#else
-    return ENOTSUP;
-#endif
 }
 
 ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo)
 {
-#ifdef USE_THREADS
     int res;
     ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP);
     res = ethr_event_twait(&((ethr_ts_event *) ep)->event,
                            (ethr_sint64_t) tmo);
     ERTS_MSACC_POP_STATE();
     return res;
-#else
-    return ENOTSUP;
-#endif
 }
 
 ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo)
 {
-#ifdef USE_THREADS
     int res;
     ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP);
     res = ethr_event_stwait(&((ethr_ts_event *) ep)->event,
@@ -3416,49 +2832,34 @@ ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo)
                             (ethr_sint64_t) tmo);
     ERTS_MSACC_POP_STATE();
     return res;
-#else
-    return ENOTSUP;
-#endif
 }
 
 ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep)
 {
-#ifdef USE_THREADS
     return (ep->iflgs & ETHR_TS_EV_TMP) == ETHR_TS_EV_TMP;
-#else
-    return 0;
-#endif
 }
 
 ERTS_GLB_INLINE void erts_thr_set_main_status(int on, int no)
 {
-#ifdef USE_THREADS
     int res = ethr_set_main_thr_status(on, no);
     if (res != 0)
 	erts_thr_fatal_error(res, "set thread main status");
-#endif
 }
 
 ERTS_GLB_INLINE int erts_thr_get_main_status(void)
 {
-#ifdef USE_THREADS
     int main_status;
     int res = ethr_get_main_thr_status(&main_status);
     if (res != 0)
 	erts_thr_fatal_error(res, "get thread main status");
     return main_status;
-#else
-    return 1;
-#endif
 }
 
 ERTS_GLB_INLINE void erts_thr_yield(void)
 {
-#ifdef USE_THREADS
     int res = ETHR_YIELD();
     if (res != 0)
 	erts_thr_fatal_error(res, "yield");
-#endif    
 }
 
 
@@ -3466,34 +2867,28 @@ ERTS_GLB_INLINE void erts_thr_yield(void)
 
 ERTS_GLB_INLINE void
 erts_thr_kill(erts_tid_t tid, int sig) {
-#ifdef USE_THREADS
   int res = ethr_kill((ethr_tid)tid, sig);
   if (res)
     erts_thr_fatal_error(res, "killing thread");
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_thr_sigmask(int how, const sigset_t *set, sigset_t *oset)
 {
-#ifdef USE_THREADS
     int res = ethr_sigmask(how, set, oset);
     if (res)
 	erts_thr_fatal_error(res, "get or set signal mask");
-#endif
 }
 
 ERTS_GLB_INLINE void
 erts_thr_sigwait(const sigset_t *set, int *sig)
 {
-#ifdef USE_THREADS
     int res;
     do {
 	res = ethr_sigwait(set, sig);
     } while (res == EINTR);
     if (res)
 	erts_thr_fatal_error(res, "to wait for signal");
-#endif
 }
 
 #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index db7d0ac449..efb3de2942 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -237,7 +237,6 @@ write_timestamp(ErtsTraceTimeStamp *tsp, Eterm **hpp)
     }
 }
 
-#ifdef ERTS_SMP
 
 static ERTS_INLINE Uint
 patch_ts_size(int ts_type)
@@ -257,7 +256,6 @@ patch_ts_size(int ts_type)
 	return 0;
     }
 }
-#endif /* ERTS_SMP */
 
 /*
  * Write a timestamp. The timestamp MUST be the last
@@ -298,18 +296,11 @@ write_ts(int ts_type, Eterm *hp, ErlHeapFragment *bp, Process *tracer)
     if (shrink) {
 	if (bp)
 	    bp->used_size -= shrink;
-#ifndef ERTS_SMP
-	else if (tracer) {
-	    Eterm *endp = ts_hp + shrink;
-	    HRelease(tracer, endp, ts_hp);
-	}
-#endif
     }
 
     return res;
 }
 
-#ifdef ERTS_SMP
 static void enqueue_sys_msg_unlocked(enum ErtsSysMsgType type,
 				     Eterm from,
 				     Eterm to,
@@ -321,7 +312,6 @@ static void enqueue_sys_msg(enum ErtsSysMsgType type,
 			    Eterm msg,
 			    ErlHeapFragment *bp);
 static void init_sys_msg_dispatcher(void);
-#endif
 
 static void init_tracer_nif(void);
 static int tracer_cmp_fun(void*, void*);
@@ -350,9 +340,7 @@ void erts_init_trace(void) {
     default_port_trace_flags = F_INITIAL_TRACE_FLAGS;
     default_port_tracer = erts_tracer_nil;
     system_seq_tracer = erts_tracer_nil;
-#ifdef ERTS_SMP
     init_sys_msg_dispatcher();
-#endif
     init_tracer_nif();
 }
 
@@ -421,20 +409,12 @@ exiting_reset(Eterm exiting)
 {
     erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
     if (exiting == system_monitor) {
-#ifdef ERTS_SMP
 	system_monitor = NIL;
 	/* Let the trace message dispatcher clear flags, etc */
-#else
-	erts_system_monitor_clear(NULL);
-#endif
     }
     if (exiting == system_profile) {
-#ifdef ERTS_SMP
 	system_profile = NIL;
 	/* Let the trace message dispatcher clear flags, etc */
-#else
-	erts_system_profile_clear(NULL);
-#endif
     }
     erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
 }
@@ -679,71 +659,11 @@ write_sys_msg_to_port(Eterm unused_to,
 	erts_exit(ERTS_ERROR_EXIT, "Internal error in do_send_to_port: %d\n", ptr-buffer);
     }
 
-#ifndef ERTS_SMP
-    if (!INVALID_TRACER_PORT(trace_port, trace_port->common.id))
-#endif
 	erts_raw_port_command(trace_port, buffer, ptr-buffer);
 
     erts_free(ERTS_ALC_T_TMP, (void *) buffer);
 }
 
-#ifndef ERTS_SMP
-/* Profile send
- * Checks if profiler is port or process
- * Eterm msg is local, need copying.
- */
-
-static void 
-profile_send(Eterm from, Eterm message) {
-    Uint sz = 0;
-    Uint *hp = NULL;
-    Eterm msg = NIL;
-    Process *profile_p = NULL;
-
-    Eterm profiler = erts_get_system_profile();
-
-    /* do not profile profiler pid */
-    if (from == profiler) return;
-
-    if (is_internal_port(profiler)) {
-    	Port *profiler_port = NULL;
-
-	/* not smp */
-
-	profiler_port = erts_id2port_sflgs(profiler,
-					   NULL,
-					   0,
-					   ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
-	if (profiler_port) {
-	    write_sys_msg_to_port(profiler,
-                                  profiler_port,
-                                  NIL, /* or current process->common.id */
-                                  SYS_MSG_TYPE_SYSPROF,
-                                  message);
-	    erts_port_release(profiler_port);
-	}
-    	
-    } else {
-	ErtsMessage *mp;
-	ASSERT(is_internal_pid(profiler));
-        
-	profile_p = erts_proc_lookup(profiler);
-
-	if (!profile_p)
-	    return;
-
-	sz = size_object(message);
-	mp = erts_alloc_message(sz, &hp);
-	if (sz == 0)
-	    msg = message;
-	else
-	    msg = copy_struct(message, sz, &hp, &mp->hfrag.off_heap);
-
-        erts_queue_message(profile_p, 0, mp, msg, from);
-    }
-}
-
-#endif
 
 static void
 trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what)
@@ -815,9 +735,7 @@ trace_send(Process *p, Eterm to, Eterm msg)
     ErtsTracerNif *tnif = NULL;
     ErtsTracingEvent* te;
     Eterm pam_result;
-#ifdef ERTS_SMP
     ErtsThrPrgrDelayHandle dhndl;
-#endif
 
     ASSERT(ARE_TRACE_FLAGS_ON(p, F_TRACE_SEND));
 
@@ -842,9 +760,7 @@ trace_send(Process *p, Eterm to, Eterm msg)
     } else
         pam_result = am_true;
 
-#ifdef ERTS_SMP
     dhndl = erts_thr_progress_unmanaged_delay();
-#endif
 
     if (is_internal_pid(to)) {
 	if (!erts_proc_lookup(to))
@@ -862,9 +778,7 @@ trace_send(Process *p, Eterm to, Eterm msg)
                            operation, msg, to, pam_result);
     }
 
-#ifdef ERTS_SMP
     erts_thr_progress_unmanaged_continue(dhndl);
-#endif
 
     erts_match_set_release_result_trace(p, pam_result);
 }
@@ -1467,21 +1381,11 @@ monitor_long_schedule_proc(Process *p, ErtsCodeMFA *in_fp,
 {
     ErlHeapFragment *bp;
     ErlOffHeap *off_heap;
-#ifndef ERTS_SMP
-    Process *monitor_p;
-#endif
     Uint hsz;
     Eterm *hp, list, in_mfa = am_undefined, out_mfa = am_undefined;
     Eterm in_tpl, out_tpl, tmo_tpl, tmo, msg;
  
 
-#ifndef ERTS_SMP
-    ASSERT(is_internal_pid(system_monitor));
-    monitor_p = erts_proc_lookup(system_monitor);
-    if (!monitor_p || p == monitor_p) {
-	return;
-    }
-#endif
     /* 
      * Size: {monitor, pid, long_schedule, [{timeout, T}, {in, {M,F,A}},{out,{M,F,A}}]} ->
      * 5 (top tuple of 4), (3 (elements) * 2 (cons)) + 3 (timeout tuple of 2) + size of Timeout +
@@ -1517,36 +1421,18 @@ monitor_long_schedule_proc(Process *p, ErtsCodeMFA *in_fp,
     hp += 2;
     msg = TUPLE4(hp, am_monitor, p->common.id, am_long_schedule, list);
     hp += 5;
-#ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp);
-#else
-    {
-	ErtsMessage *mp = erts_alloc_message(0, NULL);
-	mp->data.heap_frag = bp;
-	erts_queue_message(monitor_p, 0, mp, msg, am_system);
-    }
-#endif
 }
 void 
 monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time)
 {
     ErlHeapFragment *bp;
     ErlOffHeap *off_heap;
-#ifndef ERTS_SMP
-    Process *monitor_p;
-#endif
     Uint hsz;
     Eterm *hp, list, op;
     Eterm op_tpl, tmo_tpl, tmo, msg;
  
 
-#ifndef ERTS_SMP
-    ASSERT(is_internal_pid(system_monitor));
-    monitor_p = erts_proc_lookup(system_monitor);
-    if (!monitor_p) {
-	return;
-    }
-#endif
     /* 
      * Size: {monitor, port, long_schedule, [{timeout, T}, {op, Operation}]} ->
      * 5 (top tuple of 4), (2 (elements) * 2 (cons)) + 3 (timeout tuple of 2) 
@@ -1582,24 +1468,13 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time)
     hp += 2;
     msg = TUPLE4(hp, am_monitor, pp->common.id, am_long_schedule, list);
     hp += 5;
-#ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, pp->common.id, NIL, msg, bp);
-#else
-    {
-	ErtsMessage *mp = erts_alloc_message(0, NULL);
-	mp->data.heap_frag = bp;
-	erts_queue_message(monitor_p, 0, mp, msg, am_system);
-    }
-#endif
 }
 
 void
 monitor_long_gc(Process *p, Uint time) {
     ErlHeapFragment *bp;
     ErlOffHeap *off_heap;
-#ifndef ERTS_SMP
-    Process *monitor_p;
-#endif
     Uint hsz;
     Eterm *hp, list, msg;
     Eterm tags[] = {
@@ -1624,12 +1499,6 @@ monitor_long_gc(Process *p, Uint time) {
     Eterm *hp_end;
 #endif
 	
-#ifndef ERTS_SMP
-    ASSERT(is_internal_pid(system_monitor));
-    monitor_p = erts_proc_lookup(system_monitor);
-    if (!monitor_p || p == monitor_p)
-	return;
-#endif
 
     hsz = 0;
     (void) erts_bld_atom_uword_2tup_list(NULL,
@@ -1657,24 +1526,13 @@ monitor_long_gc(Process *p, Uint time) {
     ASSERT(hp == hp_end);
 #endif
 
-#ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp);
-#else
-    {
-	ErtsMessage *mp = erts_alloc_message(0, NULL);
-	mp->data.heap_frag = bp;
-	erts_queue_message(monitor_p, 0, mp, msg, am_system);
-    }
-#endif
 }
 
 void
 monitor_large_heap(Process *p) {
     ErlHeapFragment *bp;
     ErlOffHeap *off_heap;
-#ifndef ERTS_SMP
-    Process *monitor_p;
-#endif
     Uint hsz;
     Eterm *hp, list, msg;
     Eterm tags[] = {
@@ -1698,13 +1556,6 @@ monitor_large_heap(Process *p) {
 #endif
 
 
-#ifndef ERTS_SMP 
-    ASSERT(is_internal_pid(system_monitor));
-    monitor_p = erts_proc_lookup(system_monitor);
-    if (!monitor_p || p == monitor_p) {
-	return;
-    }
-#endif
 
     hsz = 0;
     (void) erts_bld_atom_uword_2tup_list(NULL,
@@ -1732,47 +1583,22 @@ monitor_large_heap(Process *p) {
     ASSERT(hp == hp_end);
 #endif
 
-#ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp);
-#else
-    {
-	ErtsMessage *mp = erts_alloc_message(0, NULL);
-	mp->data.heap_frag = bp;
-	erts_queue_message(monitor_p, 0, mp, msg, am_system);
-    }
-#endif
 }
 
 void
 monitor_generic(Process *p, Eterm type, Eterm spec) {
     ErlHeapFragment *bp;
     ErlOffHeap *off_heap;
-#ifndef ERTS_SMP
-    Process *monitor_p;
-#endif
     Eterm *hp, msg;
 
-#ifndef ERTS_SMP
-    ASSERT(is_internal_pid(system_monitor));
-    monitor_p = erts_proc_lookup(system_monitor);
-    if (!monitor_p || p == monitor_p)
-	return;
-#endif
 
     hp = ERTS_ALLOC_SYSMSG_HEAP(5, &bp, &off_heap, monitor_p);
 
     msg = TUPLE4(hp, am_monitor, p->common.id, type, spec); 
     hp += 5;
 
-#ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp);
-#else
-    {
-	ErtsMessage *mp = erts_alloc_message(0, NULL);
-	mp->data.heap_frag = bp;
-	erts_queue_message(monitor_p, 0, mp, msg, am_system);
-    }
-#endif
 
 }
 
@@ -1785,19 +1611,12 @@ profile_scheduler(Eterm scheduler_id, Eterm state) {
     Eterm *hp, msg;
     ErlHeapFragment *bp = NULL;
 
-#ifndef ERTS_SMP
-#define LOCAL_HEAP_SIZE (7 + ERTS_TRACE_PATCH_TS_MAX_SIZE)
-    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-    hp = local_heap;
-#else
     Uint hsz;
 
     hsz = 7 + patch_ts_size(erts_system_profile_ts_type)-1;
 	
     bp = new_message_buffer(hsz);
     hp = bp->mem;
-#endif
 
     erts_smp_mtx_lock(&smq_mtx);
 
@@ -1821,13 +1640,7 @@ profile_scheduler(Eterm scheduler_id, Eterm state) {
     /* Write timestamp in element 6 of the 'msg' tuple */
     hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL);
 
-#ifndef ERTS_SMP
-    profile_send(NIL, msg);
-    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-#else
     enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
-#endif
     erts_smp_mtx_unlock(&smq_mtx);
 
 }
@@ -2092,22 +1905,12 @@ profile_runnable_port(Port *p, Eterm status) {
     ErlHeapFragment *bp = NULL;
     Eterm count = make_small(0);
 
-#ifndef ERTS_SMP
-#define LOCAL_HEAP_SIZE (6 + ERTS_TRACE_PATCH_TS_MAX_SIZE)
-
-    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-
-    hp = local_heap;
-    
-#else
     Uint hsz;
 
     hsz = 6 + patch_ts_size(erts_system_profile_ts_type)-1;
 
     bp = new_message_buffer(hsz);
     hp = bp->mem;
-#endif
 
     erts_smp_mtx_lock(&smq_mtx);
 
@@ -2118,13 +1921,7 @@ profile_runnable_port(Port *p, Eterm status) {
     /* Write timestamp in element 5 of the 'msg' tuple */
     hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL);
 
-#ifndef ERTS_SMP
-    profile_send(p->common.id, msg);
-    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-#else
     enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, p->common.id, NIL, msg, bp);
-#endif
     erts_smp_mtx_unlock(&smq_mtx);
 }
 
@@ -2136,23 +1933,13 @@ profile_runnable_proc(Process *p, Eterm status){
     ErlHeapFragment *bp = NULL;
     ErtsCodeMFA *cmfa = NULL;
 
-#ifndef ERTS_SMP
-#define LOCAL_HEAP_SIZE (4 + 6 + ERTS_TRACE_PATCH_TS_MAX_SIZE)
-    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
-    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-
-    hp = local_heap;
-#else
     ErtsThrPrgrDelayHandle dhndl;
     Uint hsz = 4 + 6 + patch_ts_size(erts_system_profile_ts_type)-1;
-#endif
     /* Assumptions:
      * We possibly don't have the MAIN_LOCK for the process p here.
      * We assume that we can read from p->current and p->i atomically
      */
-#ifdef ERTS_SMP
     dhndl = erts_thr_progress_unmanaged_delay(); /* suspend purge operations */
-#endif
 
     if (!ERTS_PROC_IS_EXITING(p)) {
         if (p->current) {
@@ -2162,14 +1949,12 @@ profile_runnable_proc(Process *p, Eterm status){
         }
     }
 
-#ifdef ERTS_SMP
     if (!cmfa) {
 	hsz -= 4;
     }
 
     bp = new_message_buffer(hsz);
     hp = bp->mem;
-#endif
 
     if (cmfa) {
 	where = TUPLE3(hp, cmfa->module, cmfa->function,
@@ -2179,9 +1964,7 @@ profile_runnable_proc(Process *p, Eterm status){
 	where = make_small(0);
     }
 
-#ifdef ERTS_SMP
     erts_thr_progress_unmanaged_continue(dhndl);
-#endif
 	
     erts_smp_mtx_lock(&smq_mtx);
 
@@ -2192,20 +1975,13 @@ profile_runnable_proc(Process *p, Eterm status){
     /* Write timestamp in element 5 of the 'msg' tuple */
     hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL);
 
-#ifndef ERTS_SMP
-    profile_send(p->common.id, msg);
-    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
-#undef LOCAL_HEAP_SIZE
-#else
     enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, p->common.id, NIL, msg, bp);
-#endif
     erts_smp_mtx_unlock(&smq_mtx);
 }
 /* End system_profile tracing */
 
 
 
-#ifdef ERTS_SMP
 
 typedef struct ErtsSysMsgQ_ ErtsSysMsgQ;
 struct  ErtsSysMsgQ_ {
@@ -2634,7 +2410,6 @@ init_sys_msg_dispatcher(void)
 			&thr_opts);
 }
 
-#endif
 
 #include "erl_nif.h"
 
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 01fe1e5e23..7bbd93713d 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -87,7 +87,6 @@ void erts_set_system_monitor(Eterm monitor);
 Eterm erts_get_system_monitor(void);
 int erts_is_tracer_valid(Process* p);
 
-#ifdef ERTS_SMP
 void erts_check_my_tracer_proc(Process *);
 void erts_block_sys_msg_dispatcher(void);
 void erts_release_sys_msg_dispatcher(void);
@@ -97,7 +96,6 @@ void erts_foreach_sys_msg_in_q(void (*func)(Eterm,
 					    ErlHeapFragment *));
 void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *);
 void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *);
-#endif
 
 void trace_send(Process*, Eterm, Eterm);
 void trace_receive(Process*, Eterm, Eterm, ErtsTracingEvent*);
@@ -149,16 +147,12 @@ erts_bif_trace_epilogue(Process *p, Eterm result, int applying,
 			Uint32 flags_meta, BeamInstr* I,
 			ErtsTracer meta_tracer);
 
-#ifdef ERTS_SMP
 void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp);
 #define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP)				\
 do {									\
     if ((ESDP)->pending_trace_msgs)					\
 	erts_send_pending_trace_msgs((ESDP));				\
 } while (0)
-#else
-#define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP)
-#endif
 
 #define seq_trace_output(token, msg, type, receiver, process) \
 seq_trace_output_generic((token), (msg), (type), (receiver), (process), NIL)
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 3d28b05752..b1ba47df7b 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -86,22 +86,14 @@ ERTS_GLB_INLINE Uint64
 erts_smp_current_interval_nob(erts_interval_t *icp)
 {
     ASSERT(icp->smp_api);
-#ifdef ERTS_SMP
     return erts_current_interval_nob__(icp);
-#else
-    return icp->counter.not_atomic;
-#endif
 }
 
 ERTS_GLB_INLINE Uint64
 erts_smp_current_interval_acqb(erts_interval_t *icp)
 {
     ASSERT(icp->smp_api);
-#ifdef ERTS_SMP
     return erts_current_interval_acqb__(icp);
-#else
-    return icp->counter.not_atomic;
-#endif
 }
 
 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 828c833ffc..2524ad40cb 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -85,17 +85,13 @@ static struct export_blob* entry_to_blob(struct export_entry* ee)
 void
 export_info(fmtfn_t to, void *to_arg)
 {
-#ifdef ERTS_SMP
     int lock = !ERTS_IS_CRASH_DUMPING;
     if (lock)
 	export_staging_lock();
-#endif
     index_info(to, to_arg, &export_tables[erts_active_code_ix()]);
     hash_info(to, to_arg, &export_tables[erts_staging_code_ix()].htable);
-#ifdef ERTS_SMP
     if (lock)
 	export_staging_unlock();
-#endif
 }
 
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 2105ee7a6c..a598709984 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -209,9 +209,7 @@ struct erts_driver_t_ {
     } version;
     int flags;
     DE_Handle *handle;
-#ifdef ERTS_SMP
     erts_smp_mtx_t *lock;
-#endif
     ErlDrvEntry *entry;
     ErlDrvData (*start)(ErlDrvPort port, char *command, SysDriverOpts* opts);
     void (*stop)(ErlDrvData drv_data);
@@ -1127,18 +1125,12 @@ extern ErtsModifiedTimings erts_modified_timings[];
 extern int erts_no_line_info;
 extern Eterm erts_error_logger_warnings;
 extern int erts_initialized;
-#if defined(USE_THREADS) && !defined(ERTS_SMP)
-extern erts_tid_t erts_main_thread;
-#endif
 extern int erts_compat_rel;
 extern int erts_use_sender_punish;
 void erl_start(int, char**);
 void erts_usage(void);
 Eterm erts_preloaded(Process* p);
 
-#ifndef ERTS_SMP
-extern void *erts_scheduler_stack_limit;
-#endif
 
 /* erl_md5.c */
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index b609f6de39..a65ff03fd8 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -93,17 +93,11 @@ static int init_driver(erts_driver_t *, ErlDrvEntry *, DE_Handle *);
 static void terminate_port(Port *p);
 static void pdl_init(void);
 static int driver_failure_term(ErlDrvPort ix, Eterm term, int eof);
-#ifdef ERTS_SMP
 static void driver_monitor_lock_pdl(Port *p);
 static void driver_monitor_unlock_pdl(Port *p);
 #define DRV_MONITOR_LOOKUP_PORT_LOCK_PDL(Port) erts_thr_drvport2port((Port), 1)
 #define DRV_MONITOR_LOCK_PDL(Port) driver_monitor_lock_pdl(Port)
 #define DRV_MONITOR_UNLOCK_PDL(Port) driver_monitor_unlock_pdl(Port)
-#else
-#define DRV_MONITOR_LOOKUP_PORT_LOCK_PDL(Port) erts_thr_drvport2port((Port), 0)
-#define DRV_MONITOR_LOCK_PDL(Port) /* nothing */
-#define DRV_MONITOR_UNLOCK_PDL(Port) /* nothing */
-#endif
 
 #define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT)
 #define SMALL_WRITE_VEC  16
@@ -218,7 +212,6 @@ kill_port(Port *pp)
     /* In non-smp case the port structure may have been deallocated now */
 }
 
-#ifdef ERTS_SMP
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
 int
@@ -231,7 +224,6 @@ erts_lc_is_port_locked(Port *prt)
 }
 #endif
 
-#endif /* #ifdef ERTS_SMP */
 
 static void initq(Port* prt);
 
@@ -255,25 +247,21 @@ static ERTS_INLINE void port_init_instr(Port *prt
      * Stuff that need to be initialized with the port id
      * in the instrumented case, but not in the normal case.
      */
-#ifdef ERTS_SMP
     ASSERT(prt->drv_ptr && prt->lock);
     if (!prt->drv_ptr->lock) {
         erts_mtx_init_locked(prt->lock, "port_lock", id, ERTS_LOCK_FLAGS_CATEGORY_IO);
     }
-#endif
     erts_port_task_init_sched(&prt->sched, id);
 }
 
 #if !ERTS_PORT_INIT_INSTR_NEED_ID
 static ERTS_INLINE void port_init_instr_abort(Port *prt)
 {
-#ifdef ERTS_SMP
     ASSERT(prt->drv_ptr && prt->lock);
     if (!prt->drv_ptr->lock) {
 	erts_mtx_unlock(prt->lock);
 	erts_mtx_destroy(prt->lock);
     }
-#endif
     erts_port_task_fini_sched(&prt->sched);
 }
 #endif
@@ -309,7 +297,6 @@ static Port *create_port(char *name,
     erts_aint32_t state = ERTS_PORT_SFLG_CONNECTED;
     erts_aint32_t x_pts_flgs = 0;
 
-#ifdef ERTS_SMP
     ErtsRunQueue *runq;
     if (!driver_lock) {
 	/* Align size for mutex following port struct */
@@ -317,7 +304,6 @@ static Port *create_port(char *name,
 	size += sizeof(erts_mtx_t);
     }
     else
-#endif
 	port_size = size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port));
 
 #ifdef DEBUG
@@ -351,7 +337,6 @@ static Port *create_port(char *name,
 	p += busy_port_queue_size;
     }
 
-#ifdef ERTS_SMP
     if (driver_lock) {
 	prt->lock = driver_lock;
 	erts_mtx_lock(driver_lock);
@@ -368,10 +353,6 @@ static Port *create_port(char *name,
     erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq);
 
     prt->xports = NULL;
-#else
-    erts_atomic32_init_nob(&prt->refc, 1);
-    prt->cleanup = 0;
-#endif
     
     erts_port_task_pre_init_sched(&prt->sched, busy_port_queue);
 
@@ -419,10 +400,8 @@ static Port *create_port(char *name,
 #if !ERTS_PORT_INIT_INSTR_NEED_ID
 	port_init_instr_abort(prt);
 #endif
-#ifdef ERTS_SMP
 	if (driver_lock)
 	    erts_mtx_unlock(driver_lock);
-#endif
 	if (enop)
 	    *enop = 0;
 	erts_free(ERTS_ALC_T_PORT, prt);
@@ -450,23 +429,11 @@ static Port *create_port(char *name,
     return prt;
 }
 
-#ifndef ERTS_SMP
-void
-erts_port_cleanup(Port *prt)
-{
-    if (prt->drv_ptr && prt->drv_ptr->handle)
-	erts_ddll_dereference_driver(prt->drv_ptr->handle);
-    prt->drv_ptr = NULL;
-    erts_port_dec_refc(prt);
-}
-#endif
 
 void
 erts_port_free(Port *prt)
 {
-#if defined(ERTS_SMP) || defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
     erts_aint32_t state = erts_atomic32_read_nob(&prt->state);
-#endif
     ERTS_LC_ASSERT(state & (ERTS_PORT_SFLG_INITIALIZING
 			    | ERTS_PORT_SFLG_FREE));
     ASSERT(state & ERTS_PORT_SFLG_PORT_DEBUG);
@@ -480,7 +447,6 @@ erts_port_free(Port *prt)
         prt->async_open_port = NULL;
     }
 
-#ifdef ERTS_SMP
     ASSERT(prt->lock);
     if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK)
 	erts_mtx_destroy(prt->lock);
@@ -497,7 +463,6 @@ erts_port_free(Port *prt)
      */
     if (prt->drv_ptr->handle)
 	erts_ddll_dereference_driver(prt->drv_ptr->handle);
-#endif
     erts_free(ERTS_ALC_T_PORT, prt);
 }
 
@@ -658,9 +623,7 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 	ERTS_OPEN_DRIVER_RET(NULL, -3, BADARG);
     }
 
-#ifdef ERTS_SMP
     driver_lock = driver->lock;
-#endif
 
     if (driver->handle != NULL) {
 	erts_ddll_increment_port_count(driver->handle);
@@ -749,11 +712,9 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 	if (IS_TRACED_FL(port, F_TRACE_SCHED_PORTS)) {
 	    trace_sched_ports_where(port, am_out, am_open);
 	}
-#ifdef ERTS_SMP
 	if (port->xports)
 	    erts_port_handle_xports(port);
 	ASSERT(!port->xports);
-#endif
     }
 
     if (error_type) {
@@ -782,7 +743,6 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 #undef ERTS_OPEN_DRIVER_RET
 }
 
-#ifdef ERTS_SMP
 
 struct ErtsXPortsList_ {
     ErtsXPortsList *next;
@@ -791,7 +751,6 @@ struct ErtsXPortsList_ {
 
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(xports_list, ErtsXPortsList, 50, ERTS_ALC_T_XPORTS_LIST)
 
-#endif
 
 /*
  * Driver function to create new instances of a driver
@@ -839,9 +798,7 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
 	erts_ddll_reference_referenced_driver(driver->handle);
     }
 
-#ifdef ERTS_SMP
     driver_lock = driver->lock;
-#endif
 
     erts_smp_rwmtx_runlock(&erts_driver_list_lock);
 
@@ -878,21 +835,18 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
     erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, port->common.id);
     erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 
-#ifdef ERTS_SMP
     if (!driver_lock) {
 	ErtsXPortsList *xplp = xports_list_alloc();
 	xplp->port = port;
 	xplp->next = creator_port->xports;
 	creator_port->xports = xplp;
     }
-#endif
 
     port->drv_data = (UWord) drv_data;
 
     return ERTS_Port2ErlDrvPort(port);
 }
 
-#ifdef ERTS_SMP
 int erts_port_handle_xports(Port *prt)
 {
     int reds = 0;
@@ -921,7 +875,6 @@ int erts_port_handle_xports(Port *prt)
     prt->xports = NULL;
     return reds;
 }
-#endif
 
 /* Fills a possibly deep list of chars and binaries into vec
 ** Small characters are first stored in the buffer buf of length ln
@@ -3373,11 +3326,9 @@ void erts_init_io(int port_tab_size,
     common_element_size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port));
     common_element_size += ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErtsPortTaskBusyPortQ));
     common_element_size += 10; /* name */
-#ifdef ERTS_SMP
     common_element_size += sizeof(erts_mtx_t);
 
     init_xports_list_alloc();
-#endif
 
     pdl_init();
 
@@ -4068,11 +4019,9 @@ static void flush_port(Port *p)
         if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) {
 	    trace_sched_ports_where(p, am_out, am_flush);
 	}
-#ifdef ERTS_SMP
 	if (p->xports)
 	    erts_port_handle_xports(p);
 	ASSERT(!p->xports);
-#endif
     }
     if ((erts_atomic32_read_nob(&p->state) & ERTS_PORT_SFLGS_DEAD) == 0
 	&& is_port_ioq_empty(p)) {
@@ -4133,11 +4082,9 @@ terminate_port(Port *prt)
 	(*drv->stop)((ErlDrvData)prt->drv_data);
 	erts_unblock_fpe(fpe_was_unmasked);
 	ERTS_MSACC_POP_STATE_M();
-#ifdef ERTS_SMP
 	if (prt->xports)
 	    erts_port_handle_xports(prt);
 	ASSERT(!prt->xports);
-#endif
     }
 
     if (is_internal_port(send_closed_port_id)
@@ -5474,13 +5421,11 @@ erts_request_io_bytes(Process *c_p)
     erts_smp_atomic32_init_nob(&req->refc,
 			       (erts_aint32_t) erts_no_schedulers);
 
-#ifdef ERTS_SMP
     if (erts_no_schedulers > 1)
 	erts_schedule_multi_misc_aux_work(1,
 					  erts_no_schedulers,
 					  reply_io_bytes,
 					  (void *) req);
-#endif
 
     reply_io_bytes((void *) req);
 
@@ -6595,9 +6540,7 @@ static ERTS_INLINE int
 deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p,
                         Port **trace_prt)
 {
-#ifdef ERTS_SMP
     ErtsThrPrgrDelayHandle dhndl = erts_thr_progress_unmanaged_delay();
-#endif
     erts_aint32_t state;
     int res = 1;
     Port *prt = erts_port_lookup_raw((Eterm) port_id);
@@ -6615,22 +6558,18 @@ deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p,
 	goto done;
     }
     if (connected_p) {
-#ifdef ERTS_SMP
 	if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
 	    ETHR_MEMBAR(ETHR_LoadLoad);
-#endif
 	*connected_p = ERTS_PORT_GET_CONNECTED(prt);
     }
 
 done:
 
-#ifdef ERTS_SMP
     if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) {
 	ERTS_SMP_LC_ASSERT(!prt || !erts_lc_is_port_locked(prt));
 	erts_thr_progress_unmanaged_continue(dhndl);
 	ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
     } else
-#endif
     if (res == 1) {
 	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
         *trace_prt = prt;
@@ -6704,9 +6643,7 @@ driver_send_term(ErlDrvPort drvport,
      */
     Port* prt = NULL;
     ERTS_SMP_CHK_NO_PROC_LOCKS;
-#ifdef ERTS_SMP
     if (erts_thr_progress_is_managed_thread())
-#endif
     {
 	erts_aint32_t state;
 	prt = erts_drvport2port_state(drvport, &state);
@@ -7051,7 +6988,6 @@ static ERTS_INLINE void pdl_destroy(ErlDrvPDL pdl)
     erts_free(ERTS_ALC_T_PORT_DATA_LOCK, pdl);
 }
 
-#ifdef ERTS_SMP
 
 static void driver_monitor_lock_pdl(Port *p) {
     if (p->port_data_lock) {
@@ -7075,7 +7011,6 @@ static void driver_monitor_unlock_pdl(Port *p) {
     }
 }
 
-#endif
 
 /*
  * exported driver_pdl_* functions ...
@@ -7898,10 +7833,8 @@ int driver_exit(ErlDrvPort ix, int err)
 
     lnk = erts_remove_link(&ERTS_P_LINKS(prt), connected);
 
-#ifdef ERTS_SMP
     if (rp)
 	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-#endif
 
     if (rlnk != NULL) {
 	erts_destroy_link(rlnk);
@@ -8134,18 +8067,10 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size)
 	sip->erts_version = ERLANG_VERSION;
 	sip->otp_release = ERLANG_OTP_RELEASE;
 	sip->thread_support = 
-#ifdef USE_THREADS
 	    1
-#else
-	    0
-#endif
 	    ;
 	sip->smp_support = 
-#ifdef ERTS_SMP
 	    1
-#else
-	    0
-#endif
 	    ;
 
     }
@@ -8256,7 +8181,6 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
     drv->version.minor = de->minor_version;
     drv->flags = de->driver_flags;
     drv->handle = handle;
-#ifdef ERTS_SMP
     if (drv->flags & ERL_DRV_FLAG_USE_PORT_LOCKING) {
         drv->lock = NULL;
     } else {
@@ -8268,7 +8192,6 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
 
         erts_mtx_init(drv->lock, "driver_lock", driver_id, ERTS_LOCK_FLAGS_CATEGORY_IO);
     }
-#endif
     drv->entry = de;
 
     drv->start = de->start;
@@ -8311,12 +8234,10 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
 void
 erts_destroy_driver(erts_driver_t *drv)
 {
-#ifdef ERTS_SMP
     if (drv->lock) {
 	erts_smp_mtx_destroy(drv->lock);
 	erts_free(ERTS_ALC_T_DRIVER_LOCK, drv->lock);
     }
-#endif    
     erts_free(ERTS_ALC_T_DRIVER, drv);
 }
 
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index bf3267cff1..aec8777e8b 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -47,7 +47,6 @@ static erts_smp_rwmtx_t regtab_rwmtx;
 #define reg_read_unlock()		erts_smp_rwmtx_runlock(&regtab_rwmtx)
 #define reg_write_unlock()		erts_smp_rwmtx_rwunlock(&regtab_rwmtx)
 
-#ifdef ERTS_SMP
 static ERTS_INLINE void
 reg_safe_read_lock(Process *c_p, ErtsProcLocks *c_p_locks)
 {
@@ -94,7 +93,6 @@ reg_safe_write_lock(Process *c_p, ErtsProcLocks *c_p_locks)
     reg_write_lock();
 }
 
-#endif
 
 static ERTS_INLINE int
 is_proc_alive(Process *p)
@@ -193,7 +191,6 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id)
 	}
     }
 
-#ifdef ERTS_SMP
     {
 	ErtsProcLocks proc_locks = proc ? ERTS_PROC_LOCK_MAIN : 0;
 	reg_safe_write_lock(proc, &proc_locks);
@@ -201,7 +198,6 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id)
 	if (proc && !proc_locks)
 	    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
     }
-#endif
 
     if (is_internal_pid(id)) {
 	if (!proc)
@@ -272,7 +268,6 @@ erts_whereis_name_to_id(Process *c_p, Eterm name)
     HashValue hval;
     int ix;
     HashBucket* b;
-#ifdef ERTS_SMP
     ErtsProcLocks c_p_locks = 0;
     if (c_p) {
         c_p_locks = ERTS_PROC_LOCK_MAIN;
@@ -282,7 +277,6 @@ erts_whereis_name_to_id(Process *c_p, Eterm name)
 
     if (c_p && !c_p_locks)
         erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
-#endif
 
     hval = REG_HASH(name);
     ix = hval % process_reg.size;
@@ -331,7 +325,6 @@ erts_whereis_name(Process *c_p,
     HashValue hval;
     int ix;
     HashBucket* b;
-#ifdef ERTS_SMP
     ErtsProcLocks current_c_p_locks;
     Port *pending_port = NULL;
 
@@ -348,7 +341,6 @@ erts_whereis_name(Process *c_p,
      * - read reg lock
      * - current_c_p_locks (either c_p_locks or 0) on c_p
      */
-#endif
 
     hval = REG_HASH(name);
     ix = hval % process_reg.size;
@@ -370,7 +362,6 @@ erts_whereis_name(Process *c_p,
 	if (!rp)
 	    *proc = NULL;
 	else {
-#ifdef ERTS_SMP
 	    if (!rp->p)
 		*proc = NULL;
 	    else {
@@ -391,13 +382,6 @@ erts_whereis_name(Process *c_p,
 		    *proc = NULL;
 		}
 	    }
-#else
-	    if (rp->p
-		&& ((flags & ERTS_P2P_FLG_ALLOW_OTHER_X) || is_proc_alive(rp->p)))
-		*proc = rp->p;
-	    else
-		*proc = NULL;
-#endif
 	    if (*proc && (flags & ERTS_P2P_FLG_INC_REFC))
 		erts_proc_inc_refc(*proc);
 	}
@@ -407,7 +391,6 @@ erts_whereis_name(Process *c_p,
 	if (!rp || !rp->pt)
 	    *port = NULL;
 	else {
-#ifdef ERTS_SMP
             if (lock_port) {
                 if (pending_port == rp->pt)
                     pending_port = NULL;
@@ -433,17 +416,14 @@ erts_whereis_name(Process *c_p,
                 }
                 ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(rp->pt));
             }
-#endif
 	    *port = rp->pt;
 	}
     }
 
-#ifdef ERTS_SMP
     if (c_p && !current_c_p_locks)
 	erts_smp_proc_lock(c_p, c_p_locks);
     if (pending_port)
 	erts_port_release(pending_port);
-#endif
 
     reg_read_unlock();
 }
@@ -476,7 +456,6 @@ int erts_unregister_name(Process *c_p,
     RegProc r, *rp;
     Port *port = c_prt;
     ErtsProcLocks current_c_p_locks = 0;
-#ifdef ERTS_SMP
 
     /*
      * SMP note: If 'c_prt != NULL' and 'c_prt->reg->name == name',
@@ -492,18 +471,15 @@ int erts_unregister_name(Process *c_p,
  restart:
 
     reg_safe_write_lock(c_p, &current_c_p_locks);
-#endif
 
     r.name = name;
     if (is_non_value(name)) {
 	/* Unregister current process name */
 	ASSERT(c_p);
-#ifdef ERTS_SMP
 	if (current_c_p_locks != c_p_locks) {
 	    erts_smp_proc_lock(c_p, c_p_locks);
 	    current_c_p_locks = c_p_locks;
 	}
-#endif
 	if (c_p->common.u.alive.reg) {
 	    r.name = c_p->common.u.alive.reg->name;
 	} else {
@@ -516,7 +492,6 @@ int erts_unregister_name(Process *c_p,
     if ((rp = (RegProc*) hash_get(&process_reg, (void*) &r)) != NULL) {
 	if (rp->pt) {
 	    if (port != rp->pt) {
-#ifdef ERTS_SMP
 		if (port) {
 		    ASSERT(port != c_prt);
 		    erts_port_release(port);
@@ -534,7 +509,6 @@ int erts_unregister_name(Process *c_p,
 		    port = erts_id2port(id);
 		    goto restart;
 		}
-#endif
 		port = rp->pt;
 	    }
 
@@ -553,7 +527,6 @@ int erts_unregister_name(Process *c_p,
 
 	} else if (rp->p) {
 
-#ifdef ERTS_SMP
 	    erts_proc_safelock(c_p,
 			       current_c_p_locks,
 			       c_p_locks,
@@ -561,17 +534,14 @@ int erts_unregister_name(Process *c_p,
 			       (c_p == rp->p) ?  current_c_p_locks : 0,
 			       ERTS_PROC_LOCK_MAIN);
 	    current_c_p_locks = c_p_locks;
-#endif
 	    rp->p->common.u.alive.reg = NULL;
 	    if (IS_TRACED_FL(rp->p, F_TRACE_PROCS)) {
                 trace_proc(rp->p, (c_p == rp->p) ? c_p_locks : ERTS_PROC_LOCK_MAIN,
                            rp->p, am_unregister, r.name);
 	    }
-#ifdef ERTS_SMP
 	    if (rp->p != c_p) {
 		erts_smp_proc_unlock(rp->p, ERTS_PROC_LOCK_MAIN);
 	    }
-#endif
 	}
 	hash_erase(&process_reg, (void*) &r);
 	res = 1;
@@ -588,11 +558,9 @@ int erts_unregister_name(Process *c_p,
 	    erts_smp_port_lock(c_prt);
 	}
     }
-#ifdef ERTS_SMP
     if (c_p && !current_c_p_locks) {
 	erts_smp_proc_lock(c_p, c_p_locks);
     }
-#endif
     return res;
 }
 
@@ -633,14 +601,12 @@ BIF_RETTYPE registered_0(BIF_ALIST_0)
     Uint need;
     Eterm* hp;
     HashBucket **bucket;
-#ifdef ERTS_SMP
     ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN;
 
     ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(BIF_P);
     reg_safe_read_lock(BIF_P, &proc_locks);
     if (!proc_locks)
 	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
-#endif
 
     bucket = process_reg.bucket;
 
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 1e09752d87..848f104871 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -34,9 +34,6 @@
   (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
 #endif
 
-#if defined(ERTS_DIRTY_SCHEDULERS) && !defined(ERTS_SMP)
-# error "Dirty schedulers not supported without smp support"
-#endif
 
 #ifdef ERTS_INLINE
 #  ifndef ERTS_CAN_INLINE
@@ -221,11 +218,7 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f
 #  define ASSERT(e) ((void) 1)
 #endif
 
-#ifdef ERTS_SMP
 #  define ERTS_SMP_ASSERT(e) ASSERT(e)
-#else
-#  define ERTS_SMP_ASSERT(e) ((void)1)
-#endif
 
 /* ERTS_UNDEF can be used to silence false warnings about
  * "variable may be used uninitialized" while keeping the variable
@@ -476,35 +469,19 @@ int erts_send_warning_to_logger_str_nogl(char *);
 #include "erl_smp.h"
 
 #ifdef ERTS_WANT_BREAK_HANDLING
-#  ifdef ERTS_SMP
 extern erts_smp_atomic32_t erts_break_requested;
 #    define ERTS_BREAK_REQUESTED \
   ((int) erts_smp_atomic32_read_nob(&erts_break_requested))
-#  else
-extern volatile int erts_break_requested;
-#    define ERTS_BREAK_REQUESTED erts_break_requested
-#  endif
 void erts_do_break_handling(void);
 #endif
 
-#if !defined(ERTS_SMP) && !defined(__WIN32__)
-extern volatile Uint erts_signal_state;
-#define ERTS_SIGNAL_STATE erts_signal_state
-void erts_handle_signal_state(void);
-#endif
 
-#ifdef ERTS_SMP
 extern erts_smp_atomic32_t erts_writing_erl_crash_dump;
 extern erts_tsd_key_t erts_is_crash_dumping_key;
 #define ERTS_SOMEONE_IS_CRASH_DUMPING \
   ((int) erts_smp_atomic32_read_mb(&erts_writing_erl_crash_dump))
 #define ERTS_IS_CRASH_DUMPING \
   ((int) (SWord) erts_tsd_get(erts_is_crash_dumping_key))
-#else
-extern volatile int erts_writing_erl_crash_dump;
-#define ERTS_SOMEONE_IS_CRASH_DUMPING erts_writing_erl_crash_dump
-#define ERTS_IS_CRASH_DUMPING erts_writing_erl_crash_dump
-#endif
 
 /* Deal with memcpy() vs bcopy() etc. We want to use the mem*() functions,
    but be able to fall back on bcopy() etc on systems that don't have
@@ -765,10 +742,8 @@ extern char *erts_sys_ddll_error(int code);
  * System interfaces for startup.
  */
 void erts_sys_schedule_interrupt(int set);
-#ifdef ERTS_SMP
 void erts_sys_schedule_interrupt_timed(int, ErtsMonotonicTime);
 void erts_sys_main_thread(void);
-#endif
 
 extern int erts_sys_prepare_crash_dump(int secs);
 extern void erts_sys_pre_init(void);
@@ -859,13 +834,11 @@ int erts_sys_unsetenv(char *key);
 char *erts_read_env(char *key);
 void erts_free_read_env(void *value);
 
-#if defined(ERTS_SMP)
 #if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX)
 extern void sys_thr_resume(erts_tid_t tid);
 extern void sys_thr_suspend(erts_tid_t tid);
 #define ERTS_SYS_SUSPEND_SIGNAL SIGUSR2
 #endif
-#endif
 
 /* utils.c */
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 0fb25c2082..3713b756e8 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1931,27 +1931,6 @@ do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp,
     gl_sz = IS_CONST(gleader) ? 0 : size_object(gleader);
     sz = sz + gl_sz;
 
-#ifndef ERTS_SMP
-#ifdef USE_THREADS
-    if (!erts_get_scheduler_data()) /* Must be scheduler thread */
-	*p = NULL;
-    else
-#endif
-    {
-	*p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0);
-	if (*p) {
-	    erts_aint32_t state = erts_smp_atomic32_read_acqb(&(*p)->state);
-	    if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))
-		*p = NULL;
-	}
-    }
-
-    if (!*p) {
-	return NIL;
-    }
-
-    /* So we have an error logger, lets build the message */
-#endif
     *bp = new_message_buffer(sz);
     *ohp = &(*bp)->off_heap;
     *hp = (*bp)->mem;
@@ -1969,20 +1948,12 @@ static void do_send_logger_message(Eterm *hp, ErlOffHeap *ohp, ErlHeapFragment *
 #ifdef HARDDEBUG
     erts_fprintf(stderr, "%T\n", message);
 #endif
-#ifdef ERTS_SMP
     {
 	Eterm from = erts_get_current_pid();
 	if (is_not_internal_pid(from))
 	    from = NIL;
 	erts_queue_error_logger_message(from, message, bp);
     }
-#else
-    {
-	ErtsMessage *mp = erts_alloc_message(0, NULL);
-	mp->data.heap_frag = bp;
-	erts_queue_message(p, 0, mp, message, am_system);
-    }
-#endif
 }
 
 /* error_logger !
@@ -4665,11 +4636,7 @@ erts_interval_init(erts_interval_t *icp)
 void
 erts_smp_interval_init(erts_interval_t *icp)
 {
-#ifdef ERTS_SMP
     erts_interval_init(icp);
-#else
-    icp->counter.not_atomic = 0;
-#endif
 #ifdef DEBUG
     icp->smp_api = 1;
 #endif
@@ -4727,22 +4694,14 @@ Uint64
 erts_smp_step_interval_nob(erts_interval_t *icp)
 {
     ASSERT(icp->smp_api);
-#ifdef ERTS_SMP
     return step_interval_nob(icp);
-#else
-    return ++icp->counter.not_atomic;
-#endif
 }
 
 Uint64
 erts_smp_step_interval_relb(erts_interval_t *icp)
 {
     ASSERT(icp->smp_api);
-#ifdef ERTS_SMP
     return step_interval_relb(icp);
-#else
-    return ++icp->counter.not_atomic;
-#endif
 }
 
 Uint64
@@ -4763,28 +4722,14 @@ Uint64
 erts_smp_ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic)
 {
     ASSERT(icp->smp_api);
-#ifdef ERTS_SMP
     return ensure_later_interval_nob(icp, ic);
-#else
-    if (icp->counter.not_atomic > ic)
-	return icp->counter.not_atomic;
-    else
-	return ++icp->counter.not_atomic;
-#endif
 }
 
 Uint64
 erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
 {
     ASSERT(icp->smp_api);
-#ifdef ERTS_SMP
     return ensure_later_interval_acqb(icp, ic);
-#else
-    if (icp->counter.not_atomic > ic)
-	return icp->counter.not_atomic;
-    else
-	return ++icp->counter.not_atomic;
-#endif
 }
 
 /*
diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c
index 3a7b3bb50c..a1b15a2199 100644
--- a/erts/emulator/drivers/common/efile_drv.c
+++ b/erts/emulator/drivers/common/efile_drv.c
@@ -167,7 +167,6 @@ dt_private *get_dt_private(int);
 #endif
 
 
-#ifdef USE_THREADS
 #define THRDS_AVAILABLE (sys_info.async_threads > 0)
 #ifdef HARDDEBUG /* HARDDEBUG in io.c is expected too */
 #define TRACE_DRIVER fprintf(stderr, "Efile: ")
@@ -177,12 +176,6 @@ dt_private *get_dt_private(int);
 #define MUTEX_INIT(m, p) do { IF_THRDS { TRACE_DRIVER; (m = driver_pdl_create(p)); } } while (0)
 #define MUTEX_LOCK(m)    do { IF_THRDS { TRACE_DRIVER; driver_pdl_lock(m);   } } while (0)
 #define MUTEX_UNLOCK(m)  do { IF_THRDS { TRACE_DRIVER; driver_pdl_unlock(m); } } while (0)
-#else
-#define THRDS_AVAILABLE (0)
-#define MUTEX_INIT(m, p)
-#define MUTEX_LOCK(m)
-#define MUTEX_UNLOCK(m)
-#endif
 #define IF_THRDS if (THRDS_AVAILABLE)
 
 
@@ -2715,7 +2708,6 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count)
 	}
 
     case FILE_READDIR: 
-#ifdef USE_THREADS
 	if (sys_info.async_threads > 0)
 	{
 	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + 
@@ -2736,7 +2728,6 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count)
 	    goto done;
 	}
 	else   
-#endif
 	{
 	    size_t resbufsize;
 	    size_t n = 0, total = 0;
diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c
index 4573980e1e..2e9d29eede 100644
--- a/erts/emulator/hipe/hipe_mkliterals.c
+++ b/erts/emulator/hipe/hipe_mkliterals.c
@@ -441,9 +441,7 @@ static const struct rts_param rts_params[] = {
     { 11, "ERL_FUN_SIZE", 1, ERL_FUN_SIZE },
 
     { 12, "P_SCHED_DATA",
-#ifdef ERTS_SMP
       1, offsetof(struct process, scheduler_data)
-#endif
     },
     { 14, "P_FP_EXCEPTION",
 #if !defined(NO_FPE_SIGNALS) || defined(HIPE)
@@ -453,11 +451,7 @@ static const struct rts_param rts_params[] = {
     /* This flag is always defined, but its value is configuration-dependent. */
     { 15, "ERTS_IS_SMP",
       1,
-#if defined(ERTS_SMP)
       1
-#else
-      0
-#endif
     },
     /* This flag is always defined, but its value is configuration-dependent. */
     { 16, "ERTS_NO_FPE_SIGNALS",
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index ba7ae1e6a8..5775f0730f 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -490,14 +490,12 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
       case HIPE_MODE_SWITCH_RES_WAIT:
       case HIPE_MODE_SWITCH_RES_WAIT_TIMEOUT: {
 	  /* same semantics, different debug trace messages */
-#ifdef ERTS_SMP
 	  /* XXX: BEAM has different entries for the locked and unlocked
 	     cases. HiPE doesn't, so we must check dynamically. */
 	  if (p->hipe_smp.have_receive_locks)
 	      p->hipe_smp.have_receive_locks = 0;
 	  else
 	      erts_smp_proc_lock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-#endif
 	  p->i = hipe_beam_pc_resume;
 	  p->arity = 0;
 	  erts_smp_atomic32_read_band_relb(&p->state,
@@ -524,10 +522,8 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
 	      p = erts_schedule(NULL, p, reds_in - p->fcalls);
 	      ERTS_SMP_REQ_PROC_MAIN_LOCK(p);
 	      ASSERT(!(p->flags & F_HIPE_MODE));
-#ifdef ERTS_SMP
 	      p->hipe_smp.have_receive_locks = 0;
 	      reg = p->scheduler_data->x_reg_array;
-#endif
 	  }
 	  {
 	      Eterm *argp;
@@ -637,28 +633,6 @@ static unsigned hipe_next_nstack_size(unsigned size)
     return size ? size * 2 : HIPE_INITIAL_NSTACK_SIZE;
 }
 
-#if 0 && defined(HIPE_NSTACK_GROWS_UP)
-void hipe_inc_nstack(Process *p)
-{
-    Eterm *old_nstack = p->hipe.nstack;
-    unsigned old_size = p->hipe.nstend - old_nstack;
-    unsigned new_size = hipe_next_nstack_size(old_size);
-    Eterm *new_nstack = erts_realloc(ERTS_ALC_T_HIPE,
-				     (char *) old_nstack,
-				     new_size*sizeof(Eterm));
-    p->hipe.nstend = new_nstack + new_size;
-    if (new_nstack != old_nstack) {
-	p->hipe.nsp = new_nstack + (p->hipe.nsp - old_nstack);
-	p->hipe.nstack = new_nstack;
-	if (p->hipe.nstgraylim)
-	    p->hipe.nstgraylim = 
-		new_nstack + (p->hipe.nstgraylim - old_nstack);
-	if (p->hipe.nstblacklim)
-	    p->hipe.nstblacklim = 
-		new_nstack + (p->hipe.nstblacklim - old_nstack);
-    }
-}
-#endif
 
 #if defined(HIPE_NSTACK_GROWS_DOWN)
 void hipe_inc_nstack(Process *p)
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index d8044fe6da..f6ecf841ba 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -143,12 +143,10 @@ BIF_RETTYPE nbif_impl_hipe_set_timeout(NBIF_ALIST_1)
     else {
 	int tres = erts_set_proc_timer_term(p, timeout_value);
 	if (tres != 0) { /* Wrong time */
-#ifdef ERTS_SMP
 	    if (p->hipe_smp.have_receive_locks) {
 		p->hipe_smp.have_receive_locks = 0;
 		erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	    }
-#endif
 	    BIF_ERROR(p, EXC_TIMEOUT_VALUE);
 	}
     }
@@ -335,9 +333,7 @@ Binary *hipe_bs_reallocate(Binary* oldbptr, int newsize)
 }
 
 int hipe_bs_put_big_integer(
-#ifdef ERTS_SMP
     Process *p,
-#endif
     Eterm arg, Uint num_bits, byte* base, unsigned offset, unsigned flags)
 {
     byte *save_bin_buf;
@@ -530,7 +526,6 @@ Eterm hipe_check_get_msg(Process *c_p)
     msgp = PEEK_MESSAGE(c_p);
 
     if (!msgp) {
-#ifdef ERTS_SMP
 	erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	/* Make sure messages wont pass exit signals... */
 	if (ERTS_PROC_PENDING_EXIT(c_p)) {
@@ -544,12 +539,9 @@ Eterm hipe_check_get_msg(Process *c_p)
 	else {
 	    /* XXX: BEAM doesn't need this */
 	    c_p->hipe_smp.have_receive_locks = 1;
-#endif
 	    c_p->flags &= ~F_DELAY_GC;
 	    return THE_NON_VALUE;
-#ifdef ERTS_SMP
 	}
-#endif
     }
 
     if (is_non_value(ERL_MESSAGE_TERM(msgp))
@@ -573,7 +565,6 @@ Eterm hipe_check_get_msg(Process *c_p)
 /*
  * SMP-specific stuff
  */
-#ifdef ERTS_SMP
 
 /*
  * This is like the timeout BEAM instruction.
@@ -584,14 +575,12 @@ void hipe_clear_timeout(Process *c_p)
      * A timeout has occurred.  Reset the save pointer so that the next
      * receive statement will examine the first message first.
      */
-#ifdef ERTS_SMP
     /* XXX: BEAM has different entries for the locked and unlocked
        cases. HiPE doesn't, so we must check dynamically. */
     if (c_p->hipe_smp.have_receive_locks) {
 	c_p->hipe_smp.have_receive_locks = 0;
 	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
     }
-#endif
     if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) {
 	trace_receive(c_p, am_clock_service, am_timeout, NULL);
     }
@@ -604,4 +593,3 @@ void hipe_atomic_inc(int *counter)
     erts_smp_atomic_inc_nob((erts_smp_atomic_t*)counter);
 }
 
-#endif
diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h
index 38f874888b..cbc7ab8dc6 100644
--- a/erts/emulator/hipe/hipe_native_bif.h
+++ b/erts/emulator/hipe/hipe_native_bif.h
@@ -107,11 +107,7 @@ void hipe_emasculate_binary(Eterm);
 /*
  * Stuff that is different in SMP and non-SMP.
  */
-#ifdef ERTS_SMP
 int hipe_bs_put_big_integer(Process*, Eterm, Uint, byte*, unsigned, unsigned);
-#else
-int hipe_bs_put_big_integer(Eterm, Uint, byte*, unsigned, unsigned);
-#endif
 
 AEXTERN(Eterm,nbif_check_get_msg,(Process*));
 Eterm hipe_check_get_msg(Process*);
@@ -122,12 +118,10 @@ BIF_RETTYPE hipe_bifs_debug_native_called_2(BIF_ALIST_2);
 /*
  * SMP-specific stuff
  */
-#ifdef ERTS_SMP
 AEXTERN(void,nbif_atomic_inc,(void));
 AEXTERN(void,nbif_clear_timeout,(Process*));
 void hipe_atomic_inc(int*);
 void hipe_clear_timeout(Process*);
-#endif
 
 #define BIF_LIST(M,F,A,B,C,I)	AEXTERN(Eterm,nbif_##C,(void));
 #include "erl_bif_list.h"
diff --git a/erts/emulator/hipe/hipe_primops.h b/erts/emulator/hipe/hipe_primops.h
index 4fcbc9df38..6aac5e6205 100644
--- a/erts/emulator/hipe/hipe_primops.h
+++ b/erts/emulator/hipe/hipe_primops.h
@@ -41,10 +41,8 @@ PRIMOP_LIST(am_bnot, &nbif_bnot_1)
 
 PRIMOP_LIST(am_gc_1, &nbif_gc_1)
 PRIMOP_LIST(am_check_get_msg, &nbif_check_get_msg)
-#ifdef ERTS_SMP
 PRIMOP_LIST(am_atomic_inc, &nbif_atomic_inc)
 PRIMOP_LIST(am_clear_timeout, &nbif_clear_timeout)
-#endif
 PRIMOP_LIST(am_select_msg, &nbif_select_msg)
 PRIMOP_LIST(am_set_timeout, &nbif_set_timeout)
 PRIMOP_LIST(am_rethrow, &nbif_rethrow)
diff --git a/erts/emulator/hipe/hipe_process.h b/erts/emulator/hipe/hipe_process.h
index cc92bf653c..2cc7fd5f4e 100644
--- a/erts/emulator/hipe/hipe_process.h
+++ b/erts/emulator/hipe/hipe_process.h
@@ -82,7 +82,6 @@ static __inline__ void hipe_delete_process(struct hipe_process_state *p)
 	erts_free(ERTS_ALC_T_HIPE_STK, (void*)p->nstack);
 }
 
-#ifdef ERTS_SMP
 struct hipe_process_state_smp {
     int have_receive_locks;
 };
@@ -91,6 +90,5 @@ static __inline__ void hipe_init_process_smp(struct hipe_process_state_smp *p)
 {
     p->have_receive_locks = 0;
 }
-#endif
 
 #endif /* HIPE_PROCESS_H */
diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h
index 7e30358767..0d3019cfd0 100644
--- a/erts/emulator/hipe/hipe_stack.h
+++ b/erts/emulator/hipe/hipe_stack.h
@@ -110,11 +110,6 @@ extern void (*hipe_handle_stack_trap(Process*))(void);
 extern void hipe_update_stack_trap(Process*, const struct hipe_sdesc*);
 extern int hipe_fill_stacktrace(Process*, int, Eterm**);
 
-#if 0 && defined(HIPE_NSTACK_GROWS_UP)
-#define hipe_nstack_start(p)	((p)->hipe.nstack)
-#define hipe_nstack_used(p)	((p)->hipe.nsp - (p)->hipe.nstack)
-#define hipe_nstack_avail(p)	((p)->hipe.nstend - (p)->hipe.nsp)
-#endif
 #if defined(HIPE_NSTACK_GROWS_DOWN)
 #define hipe_nstack_start(p)	((p)->hipe.nsp)
 #define hipe_nstack_used(p)	((p)->hipe.nstend - (p)->hipe.nsp)
diff --git a/erts/emulator/hipe/hipe_x86_signal.c b/erts/emulator/hipe/hipe_x86_signal.c
index be68d7d463..d3b6933155 100644
--- a/erts/emulator/hipe/hipe_x86_signal.c
+++ b/erts/emulator/hipe/hipe_x86_signal.c
@@ -45,10 +45,8 @@
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
-#ifdef ERTS_SMP
 #include "sys.h"
 #include "erl_alloc.h"
-#endif
 #include "hipe_signal.h"
 
 #if defined(__GLIBC__) && __GLIBC__ == 2 && (__GLIBC_MINOR__ >= 3)
@@ -259,7 +257,6 @@ static void hipe_sigaltstack(void *ss_sp)
     }
 }
 
-#ifdef ERTS_SMP
 /*
  * Set up alternate signal stack for an Erlang process scheduler thread.
  */
@@ -269,7 +266,6 @@ void hipe_thread_signal_init(void)
        We use it to suppress false leak report from valgrind */
     hipe_sigaltstack(erts_alloc_permanent_cache_aligned(ERTS_ALC_T_HIPE_LL, SIGSTKSZ));
 }
-#endif
 
 /*
  * Set up alternate signal stack for the main thread,
@@ -277,10 +273,6 @@ void hipe_thread_signal_init(void)
  */
 static void hipe_sigaltstack_init(void)
 {
-#if !defined(ERTS_SMP)
-    static unsigned long my_sigstack[SIGSTKSZ/sizeof(long)];
-    hipe_sigaltstack(my_sigstack);
-#endif
 }
 
 /*
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 799f67fc45..ce9f6e3f06 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -84,9 +84,6 @@ typedef char EventStateFlags;
 #define ERTS_CIO_POLL_CTL	ERTS_POLL_EXPORT(erts_poll_control)
 #define ERTS_CIO_POLL_CTLV	ERTS_POLL_EXPORT(erts_poll_controlv)
 #define ERTS_CIO_POLL_WAIT	ERTS_POLL_EXPORT(erts_poll_wait)
-#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
-#define ERTS_CIO_POLL_AS_INTR 	ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt)
-#endif
 #define ERTS_CIO_POLL_INTR 	ERTS_POLL_EXPORT(erts_poll_interrupt)
 #define ERTS_CIO_POLL_INTR_TMD	ERTS_POLL_EXPORT(erts_poll_interrupt_timed)
 #define ERTS_CIO_NEW_POLLSET 	ERTS_POLL_EXPORT(erts_poll_create_pollset)
@@ -108,10 +105,8 @@ static struct pollset_info
 	int size;
 	ErtsSysFdType *array;
     } active_fd;
-#ifdef ERTS_SMP
     struct removed_fd* removed_list;       /* list of deselected fd's*/
     erts_smp_spinlock_t removed_list_lock;
-#endif
 }pollset;
 #define NUM_OF_POLLSETS 1
 
@@ -137,7 +132,6 @@ typedef struct {
     EventStateFlags flags;
 } ErtsDrvEventState;
 
-#ifdef ERTS_SMP
 struct removed_fd {
     struct removed_fd *next;
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
@@ -150,7 +144,6 @@ struct removed_fd {
 #endif
 
 };
-#endif
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
 static int max_fds = -1;
@@ -161,7 +154,6 @@ static union {
     byte _cache_line_alignment[64];
 }drv_ev_state_locks[DRV_EV_STATE_LOCK_CNT];
 
-#ifdef ERTS_SMP
 static ERTS_INLINE erts_smp_mtx_t* fd_mtx(ErtsSysFdType fd)
 {
     int hash = (int)fd;
@@ -170,9 +162,6 @@ static ERTS_INLINE erts_smp_mtx_t* fd_mtx(ErtsSysFdType fd)
 # endif
     return &drv_ev_state_locks[hash % DRV_EV_STATE_LOCK_CNT].lck;
 }
-#else
-#  define fd_mtx(fd) NULL
-#endif
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
 
@@ -249,9 +238,7 @@ static void
 steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource*,
                        ErtsDrvEventState *state, int mode, int on);
 
-#ifdef ERTS_SMP
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(removed_fd, struct removed_fd, 64, ERTS_ALC_T_FD_LIST)
-#endif
 
 static ERTS_INLINE void
 init_iotask(ErtsIoTask *io_task)
@@ -337,7 +324,6 @@ free_drv_event_data(ErtsDrvEventDataState *dep)
 static ERTS_INLINE void
 remember_removed(ErtsDrvEventState *state, struct pollset_info* psi)
 {
-#ifdef ERTS_SMP
     struct removed_fd *fdlp;
     ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(fd_mtx(state->fd)));
     if (erts_smp_atomic_read_nob(&psi->in_poll_wait)) {
@@ -355,29 +341,23 @@ remember_removed(ErtsDrvEventState *state, struct pollset_info* psi)
 	psi->removed_list = fdlp;
 	erts_smp_spin_unlock(&psi->removed_list_lock);
     }
-#endif
 }
 
 
 static ERTS_INLINE int
 is_removed(ErtsDrvEventState *state)
 {
-#ifdef ERTS_SMP
     /* Note that there is a possible race here, where an fd is removed
        (increasing remove_cnt) and then added again just before erts_poll_wait
        is called by erts_check_io. Any polled event on the re-added fd will then
        be falsely ignored. But that does not matter, as the event will trigger
        again next time erl_check_io is called. */
     return state->remove_cnt > 0;
-#else
-    return 0;
-#endif
 }
 
 static void
 forget_removed(struct pollset_info* psi)
 {
-#ifdef ERTS_SMP
     struct removed_fd* fdlp;
     struct removed_fd* tofree;
 
@@ -460,7 +440,6 @@ forget_removed(struct pollset_info* psi)
 	fdlp = fdlp->next;
 	removed_fd_free(tofree);
     }
-#endif /* ERTS_SMP */
 }
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
@@ -2118,13 +2097,6 @@ eready(Eterm id, ErtsDrvEventState *state, ErlDrvEventData event_data,
 
 static void bad_fd_in_pollset(ErtsDrvEventState *, Eterm inport, Eterm outport);
 
-#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
-void
-ERTS_CIO_EXPORT(erts_check_io_async_sig_interrupt)(void)
-{
-    ERTS_CIO_POLL_AS_INTR(pollset.ps);
-}
-#endif
 
 void
 ERTS_CIO_EXPORT(erts_check_io_interrupt)(int set)
@@ -2355,9 +2327,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
                 add_active_fd(state->fd);
             }
 
-#ifdef ERTS_SMP
             erts_smp_mtx_unlock(fd_mtx(fd));
-#endif
             if (is_not_nil(in.pid)) {
                 send_event_tuple(&in, resource, am_ready_input);
             }
@@ -2404,9 +2374,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 	}
 
 	next_pollres:;
-#ifdef ERTS_SMP
 	erts_smp_mtx_unlock(fd_mtx(fd));
-#endif
         next_pollres_unlocked:;
     }
 
@@ -2561,7 +2529,6 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
 #endif
 
 
-#ifdef ERTS_SMP
     init_removed_fd_alloc();
     pollset.removed_list = NULL;
     erts_smp_spinlock_init(&pollset.removed_list_lock, "pollset_rm_list", NIL,
@@ -2573,7 +2540,6 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
                 ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
         }
     }
-#endif
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     max_fds = ERTS_CIO_POLL_MAX_FDS();
     erts_smp_atomic_init_nob(&drv_ev_state_len, 0);
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index 2d3bb98afa..d74eb764d2 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -44,10 +44,6 @@ Eterm erts_check_io_info_kp(void *);
 Eterm erts_check_io_info_nkp(void *);
 int erts_check_io_max_files_kp(void);
 int erts_check_io_max_files_nkp(void);
-#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
-void erts_check_io_async_sig_interrupt_kp(void);
-void erts_check_io_async_sig_interrupt_nkp(void);
-#endif
 void erts_check_io_interrupt_kp(int);
 void erts_check_io_interrupt_nkp(int);
 void erts_check_io_interrupt_timed_kp(int, ErtsMonotonicTime);
@@ -69,9 +65,6 @@ void erts_lcnt_update_cio_locks_nkp(int enable);
 Uint erts_check_io_size(void);
 Eterm erts_check_io_info(void *);
 int erts_check_io_max_files(void);
-#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
-void erts_check_io_async_sig_interrupt(void);
-#endif
 void erts_check_io_interrupt(int);
 void erts_check_io_interrupt_timed(int, ErtsMonotonicTime);
 void erts_check_io(int);
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index d69a79dc2a..d5b2fa87e4 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -188,7 +188,6 @@ typedef union {
 static int no_mseg_allocators;
 static ErtsAlgndMsegAllctr_t *aligned_mseg_allctr;
 
-#ifdef ERTS_SMP
 
 #define ERTS_MSEG_ALLCTR_IX(IX) \
   (&aligned_mseg_allctr[(IX)].mseg_alloc)
@@ -199,18 +198,6 @@ static ErtsAlgndMsegAllctr_t *aligned_mseg_allctr;
 #define ERTS_MSEG_ALLCTR_OPT(OPT) \
   ((OPT)->sched_spec ? ERTS_MSEG_ALLCTR_SS() : ERTS_MSEG_ALLCTR_IX(0))
 
-#else
-
-#define ERTS_MSEG_ALLCTR_IX(IX) \
-  (&aligned_mseg_allctr[0].mseg_alloc)
-
-#define ERTS_MSEG_ALLCTR_SS() \
-  (&aligned_mseg_allctr[0].mseg_alloc)
-
-#define ERTS_MSEG_ALLCTR_OPT(OPT) \
-  (&aligned_mseg_allctr[0].mseg_alloc)
-
-#endif
 
 #define ERTS_MSEG_LOCK(MA)		\
 do {					\
@@ -1404,11 +1391,7 @@ erts_mseg_init(ErtsMsegInit_t *init)
     int i;
     UWord x;
 
-#ifdef ERTS_SMP
     no_mseg_allocators = init->nos + 1;
-#else
-    no_mseg_allocators = 1;
-#endif
 
     x = (UWord) malloc(sizeof(ErtsAlgndMsegAllctr_t)
 		       *no_mseg_allocators
diff --git a/erts/emulator/sys/common/erl_os_monotonic_time_extender.c b/erts/emulator/sys/common/erl_os_monotonic_time_extender.c
index d53190fdd5..341845cc2a 100644
--- a/erts/emulator/sys/common/erl_os_monotonic_time_extender.c
+++ b/erts/emulator/sys/common/erl_os_monotonic_time_extender.c
@@ -23,7 +23,6 @@
 #endif
 #include "erl_os_monotonic_time_extender.h"
 
-#ifdef USE_THREADS
 
 static void *os_monotonic_time_extender(void *vstatep)
 {
@@ -49,30 +48,22 @@ static void *os_monotonic_time_extender(void *vstatep)
 }
 
 static erts_tid_t os_monotonic_extender_tid;
-#endif
 
 void
 erts_init_os_monotonic_time_extender(ErtsOsMonotonicTimeExtendState *statep,
 				     Uint32 (*raw_os_monotonic_time)(void),
 				     int check_seconds)
 {
-#ifdef USE_THREADS
     statep->raw_os_monotonic_time = raw_os_monotonic_time;
     erts_atomic32_init_nob(&statep->extend[0], (erts_aint32_t) 0);
     erts_atomic32_init_nob(&statep->extend[1], (erts_aint32_t) 0);
     statep->check_interval = check_seconds;
 
-#else
-    statep->extend[0] = (Uint32) 0;
-    statep->extend[1] = (Uint32) 0;
-    statep->last_msb = (ErtsMonotonicTime) 0;
-#endif
 }
 
 void
 erts_late_init_os_monotonic_time_extender(ErtsOsMonotonicTimeExtendState *statep)
 {
-#ifdef USE_THREADS
     erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER;
     thr_opts.detached = 1;
     thr_opts.suggested_stack_size = 4;
@@ -85,5 +76,4 @@ erts_late_init_os_monotonic_time_extender(ErtsOsMonotonicTimeExtendState *statep
 		    os_monotonic_time_extender,
 		    (void*) statep,
 		    &thr_opts);
-#endif
 }
diff --git a/erts/emulator/sys/common/erl_os_monotonic_time_extender.h b/erts/emulator/sys/common/erl_os_monotonic_time_extender.h
index 8089c9aed9..2266f26e24 100644
--- a/erts/emulator/sys/common/erl_os_monotonic_time_extender.h
+++ b/erts/emulator/sys/common/erl_os_monotonic_time_extender.h
@@ -25,36 +25,17 @@
 #include "erl_threads.h"
 
 typedef struct {
-#ifdef USE_THREADS
     Uint32 (*raw_os_monotonic_time)(void);
     erts_atomic32_t extend[2];
     int check_interval;
-#else
-    Uint32 extend[2];
-    ErtsMonotonicTime last_msb;
-#endif
 } ErtsOsMonotonicTimeExtendState;
 
-#ifdef USE_THREADS
 #  define ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(S, RT) ((void) 1)
 #  define ERTS_EXTEND_OS_MONOTONIC_TIME(S, RT)				\
     ((((ErtsMonotonicTime)						\
        erts_atomic32_read_nob(&((S)->extend[((int) ((RT) >> 31)) & 1]))) \
       << 32)								\
      + (RT))
-#else
-#  define ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(S, RT)			\
-    do {								\
-	Uint32 msb__ = (RT) & (((Uint32) 1) << 31);			\
-	if (msb__ != (S)->last_msb) {					\
-	    int ix__ = ((int) ((S)->last_msb >> 31)) & 1;		\
-	    (S)->extend[ix__]++;					\
-	    (S)->last_msb = msb;					\
-	}								\
-    } while (0)
-#  define ERTS_EXTEND_OS_MONOTONIC_TIME(S, RT)				\
-    ((((ErtsMonotonicTime) (S)->extend[((int) ((RT) >> 31)) & 1]) << 32) + (RT))
-#endif
 
 void
 erts_init_os_monotonic_time_extender(ErtsOsMonotonicTimeExtendState *statep,
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index 52a8b6a53f..3a6ae85b42 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -95,9 +95,6 @@
 #define ERTS_POLL_DEBUG_PRINT
 #endif
 
-#if defined(DEBUG) && 0
-#define HARD_DEBUG
-#endif
 
 #ifdef _DARWIN_UNLIMITED_SELECT
 typedef struct {
@@ -155,16 +152,11 @@ int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds,
 
 #define ERTS_POLL_COALESCE_KP_RES (ERTS_POLL_USE_KQUEUE || ERTS_POLL_USE_EPOLL)
 
-#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
-#  define ERTS_POLL_ASYNC_INTERRUPT_SUPPORT 1
-#else
 #  define ERTS_POLL_ASYNC_INTERRUPT_SUPPORT 0
-#endif
 
 #define ERTS_POLL_USE_WAKEUP_PIPE \
    (ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(USE_THREADS))
 
-#ifdef ERTS_SMP
 
 #define ERTS_POLLSET_LOCK(PS) \
   erts_smp_mtx_lock(&(PS)->mtx)
@@ -178,28 +170,13 @@ int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds,
 #define ERTS_POLLSET_IS_POLLED(PS) \
   ((int) erts_atomic32_read_nob(&(PS)->polled))
 
-#else
-
-#define ERTS_POLLSET_LOCK(PS)
-#define ERTS_POLLSET_UNLOCK(PS)
-#define ERTS_POLLSET_SET_POLLED_CHK(PS) 0
-#define ERTS_POLLSET_UNSET_POLLED(PS)
-#define ERTS_POLLSET_IS_POLLED(PS) 0
 
-#endif
-
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
 #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) \
   erts_smp_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 1)
 #define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) \
   erts_smp_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 0)
 #define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) \
   ((int) erts_smp_atomic32_read_nob(&(PS)->have_update_requests))
-#else
-#define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS)
-#define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS)
-#define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) 0
-#endif
 
 #if ERTS_POLL_USE_FALLBACK
 #  if ERTS_POLL_USE_POLL
@@ -212,7 +189,6 @@ int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds,
  * --- Data types ------------------------------------------------------------
  */
 
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
 #define ERTS_POLLSET_UPDATE_REQ_BLOCK_SIZE 128
 
 typedef struct ErtsPollSetUpdateRequestsBlock_ ErtsPollSetUpdateRequestsBlock;
@@ -222,19 +198,14 @@ struct ErtsPollSetUpdateRequestsBlock_ {
     int fds[ERTS_POLLSET_UPDATE_REQ_BLOCK_SIZE];
 };
 
-#endif
 
 
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
 #  define ERTS_POLL_FD_FLG_INURQ	(((unsigned short) 1) << 0)
-#endif
 #if ERTS_POLL_USE_FALLBACK
 #  define ERTS_POLL_FD_FLG_INFLBCK	(((unsigned short) 1) << 1)
 #  define ERTS_POLL_FD_FLG_USEFLBCK	(((unsigned short) 1) << 2)
 #endif
-#if ERTS_POLL_USE_KERNEL_POLL || defined(ERTS_SMP)
 #  define ERTS_POLL_FD_FLG_RST		(((unsigned short) 1) << 3)
-#endif
 typedef struct {
 #if ERTS_POLL_USE_POLL
     int pix;
@@ -244,9 +215,7 @@ typedef struct {
 #if ERTS_POLL_COALESCE_KP_RES
     unsigned short res_ev_ix;
 #endif
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE || ERTS_POLL_USE_FALLBACK
     unsigned short flags;
-#endif
 
 } ErtsFdStatus;
 
@@ -301,27 +270,19 @@ struct ErtsPollSet_ {
     ERTS_fd_set output_fds;
     ERTS_fd_set res_output_fds;
 #endif
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
     ErtsPollSetUpdateRequestsBlock update_requests;
     ErtsPollSetUpdateRequestsBlock *curr_upd_req_block;
     erts_smp_atomic32_t have_update_requests;
-#endif
-#ifdef ERTS_SMP
     erts_atomic32_t polled;
     erts_smp_mtx_t mtx;
-#endif
-#if ERTS_POLL_USE_WAKEUP_PIPE
     int wake_fds[2];
-#endif
 #if ERTS_POLL_USE_TIMERFD
     int timer_fd;
 #endif
 #if ERTS_POLL_USE_FALLBACK
     int fallback_used;
 #endif
-#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
     erts_atomic32_t wakeup_state;
-#endif
     erts_atomic64_t timeout_time;
 #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
     erts_smp_atomic_t no_avoided_wakeups;
@@ -413,50 +374,37 @@ get_timeout_time(ErtsPollSet ps)
 static ERTS_INLINE void
 reset_wakeup_state(ErtsPollSet ps)
 {
-#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
     erts_atomic32_set_mb(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
-#endif
 }
 
 static ERTS_INLINE int
 is_woken(ErtsPollSet ps)
 {
-#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
     return erts_atomic32_read_acqb(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN;
-#else
-    return 0;
-#endif
 }
 
 static ERTS_INLINE int
 is_interrupted_reset(ErtsPollSet ps)
 {
-#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
     return (erts_atomic32_xchg_acqb(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN)
 	    == ERTS_POLL_WOKEN_INTR);
-#else
-    return 0;
-#endif
 }
 
 static ERTS_INLINE void
 woke_up(ErtsPollSet ps)
 {
-#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
     erts_aint32_t wakeup_state = erts_atomic32_read_acqb(&ps->wakeup_state);
     if (wakeup_state == ERTS_POLL_NOT_WOKEN)
 	(void) erts_atomic32_cmpxchg_nob(&ps->wakeup_state,
 					 ERTS_POLL_WOKEN,
 					 ERTS_POLL_NOT_WOKEN);
     ASSERT(erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN);
-#endif
 }
 
 /*
  * --- Wakeup pipe -----------------------------------------------------------
  */
 
-#if ERTS_POLL_USE_WAKEUP_PIPE
 
 static ERTS_INLINE void
 wake_poller(ErtsPollSet ps, int interrupted, int async_signal_safe)
@@ -507,18 +455,11 @@ wake_poller(ErtsPollSet ps, int interrupted, int async_signal_safe)
 static ERTS_INLINE void
 cleanup_wakeup_pipe(ErtsPollSet ps)
 {
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
-    int intr = 0;
-#endif
     int fd = ps->wake_fds[0];
     int res;
     do {
 	char buf[32];
 	res = read(fd, buf, sizeof(buf));
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
-	if (res > 0)
-	    intr = 1;
-#endif
     } while (res > 0 || (res < 0 && errno == EINTR));
     if (res < 0 && errno != ERRNO_BLOCK) {
 	fatal_error("%s:%d:cleanup_wakeup_pipe(): "
@@ -528,10 +469,6 @@ cleanup_wakeup_pipe(ErtsPollSet ps)
 		    fd,
 		    erl_errno_id(errno), errno);
     }
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
-    if (intr)
-	erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_INTR);
-#endif
 }
 
 static void
@@ -574,7 +511,6 @@ create_wakeup_pipe(ErtsPollSet ps)
     ps->wake_fds[1] = wake_fds[1];
 }
 
-#endif /* ERTS_POLL_USE_WAKEUP_PIPE */
 
 /*
  * --- timer fd -----------------------------------------------------------
@@ -648,7 +584,6 @@ timerfd_clear(ErtsPollSet ps, int res, int max_res) {
 /*
  * --- Poll set update requests ----------------------------------------------
  */
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
 
 static ERTS_INLINE void
 enqueue_update_request(ErtsPollSet ps, int fd)
@@ -691,7 +626,6 @@ free_update_requests_block(ErtsPollSet ps,
     }
 }
 
-#endif /* ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE */
 
 /*
  * --- Growing poll set structures -------------------------------------------
@@ -819,9 +753,7 @@ grow_fds_status(ErtsPollSet ps, int min_fd)
 #if ERTS_POLL_COALESCE_KP_RES
 	ps->fds_status[i].res_ev_ix = (unsigned short) ERTS_POLL_MAX_RES;
 #endif
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE || ERTS_POLL_USE_FALLBACK
 	ps->fds_status[i].flags = (unsigned short) 0;
-#endif
     }
     ps->fds_status_len = new_len;
 }
@@ -849,7 +781,7 @@ need_update(ErtsPollSet ps, int fd)
 	ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST;
 	reset = 0;
     }
-#elif defined(ERTS_SMP)
+#else
     ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST;
 #endif
 
@@ -1518,7 +1450,6 @@ static int update_pollset(ErtsPollSet ps, int fd)
 
 #endif /* ERTS_POLL_USE_POLL || ERTS_POLL_USE_SELECT || ERTS_POLL_USE_FALLBACK */
 
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
 
 static void
 handle_update_requests(ErtsPollSet ps)
@@ -1565,7 +1496,6 @@ handle_update_requests(ErtsPollSet ps)
     ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(ps);
 }
 
-#endif /* ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE */
 
 static ERTS_INLINE ErtsPollEvents
 poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake)
@@ -1583,12 +1513,10 @@ poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake
 	    goto done;
 	}
 #endif
-#if ERTS_POLL_USE_WAKEUP_PIPE
 	if (fd == ps->wake_fds[0] || fd == ps->wake_fds[1]) {
 	    new_events = ERTS_POLL_EV_NVAL;
 	    goto done;
 	}
-#endif
 #if ERTS_POLL_USE_TIMERFD
 	if (fd == ps->timer_fd) {
 	    new_events = ERTS_POLL_EV_NVAL;
@@ -1615,9 +1543,7 @@ poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake
 	new_events &= ~events;
 
     if (new_events == (ErtsPollEvents) 0) {
-#if ERTS_POLL_USE_KERNEL_POLL || defined(ERTS_SMP)
 	ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_RST;
-#endif
 #if ERTS_POLL_USE_FALLBACK
 	ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_USEFLBCK;
 #endif
@@ -1626,18 +1552,12 @@ poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake
     ps->fds_status[fd].events = new_events;
 
     if (new_events == ps->fds_status[fd].used_events
-#if ERTS_POLL_USE_KERNEL_POLL || defined(ERTS_SMP)
 	&& !(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST)
-#endif
 	) {
 	*do_wake = 0;
 	goto done;
     }
 
-#if !ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
-    if (update_pollset(ps, fd) != 0)
-	new_events = ERTS_POLL_EV_ERR;
-#else /* ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE */
 
 #if ERTS_POLL_USE_CONCURRENT_UPDATE
     if (ERTS_POLLSET_IS_POLLED(ps)) {
@@ -1652,7 +1572,6 @@ poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake
 
     enqueue_update_request(ps, fd);
 	
-#ifdef ERTS_SMP
     /*
      * If new events have been added, we need to wake up the
      * polling thread, but if events have been removed we don't.
@@ -1660,9 +1579,7 @@ poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake
     if ((new_events && (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST))
 	|| (~ps->fds_status[fd].used_events & new_events))
 	*do_wake = 1;
-#endif /* ERTS_SMP */
 
-#endif /* ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE */
 
  done:
 #ifdef ERTS_POLL_DEBUG_PRINT
@@ -1695,10 +1612,8 @@ ERTS_POLL_EXPORT(erts_poll_controlv)(ErtsPollSet ps,
 
     ERTS_POLLSET_UNLOCK(ps);
 
-#ifdef ERTS_SMP
     if (final_do_wake)
 	wake_poller(ps, 0, 0);
-#endif /* ERTS_SMP */
 
 }
 
@@ -1718,11 +1633,9 @@ ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet ps,
 
     ERTS_POLLSET_UNLOCK(ps);
 
-#ifdef ERTS_SMP
     if (*do_wake) {
 	wake_poller(ps, 0, 0);
     }
-#endif /* ERTS_SMP */
 
     return res;
 }
@@ -1739,9 +1652,7 @@ save_kp_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, int chk_fds_res)
     int res = 0;
     int i;
     int n = chk_fds_res < max_res ? chk_fds_res : max_res;
-#if ERTS_POLL_USE_WAKEUP_PIPE
     int wake_fd = ps->wake_fds[0];
-#endif
 #if ERTS_POLL_USE_TIMERFD
     int timer_fd = ps->timer_fd;
 #endif
@@ -1754,12 +1665,10 @@ save_kp_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, int chk_fds_res)
 	    int fd = ps->res_events[i].data.fd;
 	    int ix;
 	    ErtsPollEvents revents;
-#if ERTS_POLL_USE_WAKEUP_PIPE
 	    if (fd == wake_fd) {
 		cleanup_wakeup_pipe(ps);
 		continue;
 	    }
-#endif
 #if ERTS_POLL_USE_TIMERFD
             if (fd == timer_fd) {
                 continue;
@@ -1805,12 +1714,10 @@ save_kp_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, int chk_fds_res)
 	}
 
 	if (ev->filter == EVFILT_READ) {
-#if ERTS_POLL_USE_WAKEUP_PIPE
 	    if (fd == wake_fd) {
 		cleanup_wakeup_pipe(ps);
 		continue;
 	    }
-#endif
 	    pr[ix].events |= ERTS_POLL_EV_IN;
 	}
 	else if (ev->filter == EVFILT_WRITE)
@@ -1833,12 +1740,10 @@ save_kp_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, int chk_fds_res)
 	if (ps->res_events[i].revents) {
 	    int fd = ps->res_events[i].fd;
 	    ErtsPollEvents revents;
-#if ERTS_POLL_USE_WAKEUP_PIPE
 	    if (fd == wake_fd) {
 		cleanup_wakeup_pipe(ps);
 		continue;
 	    }
-#endif
 #if ERTS_POLL_USE_TIMERFD
 	    if (fd == timer_fd) {
 		continue;
@@ -1938,7 +1843,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res,
 			i++;
 			continue;
 		    }
-#elif ERTS_POLL_USE_WAKEUP_PIPE
+#else
 		    if (fd == wake_fd) {
 			cleanup_wakeup_pipe(ps);
 			i++;
@@ -1988,7 +1893,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res,
 			    fd++;
 			    continue;
 			}
-#elif ERTS_POLL_USE_WAKEUP_PIPE
+#else
 			if (fd == wake_fd) {
 			    cleanup_wakeup_pipe(ps);
 			    fd++;
@@ -2051,7 +1956,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res,
 				fd++;
 				continue;
 			    }
-#elif ERTS_POLL_USE_WAKEUP_PIPE
+#else
 			    if (fd == wake_fd) {
 				cleanup_wakeup_pipe(ps);
 				fd++;
@@ -2073,7 +1978,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res,
 				    fd++;
 				    continue;
 				}
-#elif ERTS_POLL_USE_WAKEUP_PIPE
+#else
 				if (fd == wake_fd) {
 				    cleanup_wakeup_pipe(ps);
 				    fd++;
@@ -2261,9 +2166,7 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res)
                 struct itimerspec its;
                 timeout = get_timeout_itimerspec(ps, &its, timeout_time);
                 if (timeout) {
-#ifdef ERTS_SMP
                     erts_thr_progress_prepare_wait(NULL);
-#endif
                     ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
                     timerfd_set(ps, &its);
                     res = epoll_wait(ps->kp_fd, ps->res_events, max_res, -1);
@@ -2275,9 +2178,7 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res)
 #else /* !ERTS_POLL_USE_TIMERFD */
 	    timeout = (int) get_timeout(ps, 1000, timeout_time);
             if (timeout) {
-#ifdef ERTS_SMP
 		erts_thr_progress_prepare_wait(NULL);
-#endif
                 ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
             }
 	    res = epoll_wait(ps->kp_fd, ps->res_events, max_res, timeout);
@@ -2288,9 +2189,7 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res)
 		grow_res_events(ps, max_res);
 	    timeout = get_timeout_timespec(ps, &ts, timeout_time);
             if (timeout) {
-#ifdef ERTS_SMP
 		erts_thr_progress_prepare_wait(NULL);
-#endif
 		ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
             }
 	    res = kevent(ps->kp_fd, NULL, 0, ps->res_events, max_res, &ts);
@@ -2308,18 +2207,14 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res)
 	     */
 	    struct dvpoll poll_res;
 	    int nfds = (int) erts_smp_atomic_read_nob(&ps->no_of_user_fds);
-#if ERTS_POLL_USE_WAKEUP_PIPE
 	    nfds++; /* Wakeup pipe */
-#endif
 	    timeout = (int) get_timeout(ps, 1000, timeout_time);
 	    poll_res.dp_nfds = nfds < max_res ? nfds : max_res;
 	    if (poll_res.dp_nfds > ps->res_events_len)
 		grow_res_events(ps, poll_res.dp_nfds);
 	    poll_res.dp_fds = ps->res_events;
 	    if (timeout) {
-#ifdef ERTS_SMP
 		erts_thr_progress_prepare_wait(NULL);
-#endif
                 ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
             }
 	    poll_res.dp_timeout = timeout;
@@ -2328,9 +2223,7 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res)
             struct timespec ts;
 	    timeout = get_timeout_timespec(ps, &ts, timeout_time);
             if (timeout) {
-#ifdef ERTS_SMP
 		erts_thr_progress_prepare_wait(NULL);
-#endif
 		ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
             }
             res = ppoll(ps->poll_fds, ps->no_poll_fds, &ts, NULL);
@@ -2338,9 +2231,7 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res)
 	    timeout = (int) get_timeout(ps, 1000, timeout_time);
 
 	    if (timeout) {
-#ifdef ERTS_SMP
 		erts_thr_progress_prepare_wait(NULL);
-#endif
 		ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
             }
 	    res = poll(ps->poll_fds, ps->no_poll_fds, timeout);
@@ -2352,9 +2243,7 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res)
 	    ERTS_FD_COPY(&ps->output_fds, &ps->res_output_fds);
 
 	    if (timeout) {
-#ifdef ERTS_SMP
 		erts_thr_progress_prepare_wait(NULL);
-#endif
 		ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
 	    }
 	    res = ERTS_SELECT(ps->max_fd + 1,
@@ -2362,7 +2251,6 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res)
 			      &ps->res_output_fds,
 			      NULL,
 			      &to);
-#ifdef ERTS_SMP
 	    if (timeout) {
 		erts_thr_progress_finalize_wait(NULL);
 		ERTS_MSACC_POP_STATE_M();
@@ -2397,14 +2285,11 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res)
 		    res = -1;
 		}
 	    }
-#endif /* ERTS_SMP */
 	    return res;
 #endif				/* ----------------------------------------- */
 	}
 	if (timeout) {
-#ifdef ERTS_SMP
 	    erts_thr_progress_finalize_wait(NULL);
-#endif
 	    ERTS_MSACC_POP_STATE_M();
 	}
 	return res;
@@ -2420,9 +2305,7 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
     ErtsMonotonicTime to;
     int res, no_fds;
     int ebadf = 0;
-#ifdef ERTS_SMP
     int ps_locked = 0;
-#endif
 
     no_fds = *len;
 #ifdef ERTS_POLL_MAX_RES
@@ -2447,13 +2330,11 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
 	  ? ERTS_POLL_NO_TIMEOUT /* Use zero timeout */
 	  : timeout_time);
 
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
     if (ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) {
 	ERTS_POLLSET_LOCK(ps);
 	handle_update_requests(ps);
 	ERTS_POLLSET_UNLOCK(ps);
     }
-#endif
 
     while (1) {
 	res = check_fd_events(ps, to, no_fds);
@@ -2484,10 +2365,8 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
     save_results:
 #endif
 
-#ifdef ERTS_SMP
 	ps_locked = 1;
 	ERTS_POLLSET_LOCK(ps);
-#endif
 
 	no_fds = save_poll_result(ps, pr, no_fds, res, ebadf);
 
@@ -2499,11 +2378,9 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
 	*len = no_fds;
     }
 
-#ifdef ERTS_SMP
     if (ps_locked)
 	ERTS_POLLSET_UNLOCK(ps);
     ERTS_POLLSET_UNSET_POLLED(ps);
-#endif
 
  done:
     set_timeout_time(ps, ERTS_MONOTONIC_TIME_MAX);
@@ -2522,25 +2399,12 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
 void
 ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet ps, int set)
 {
-#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
     if (!set)
 	reset_wakeup_state(ps);
     else
 	wake_poller(ps, 1, 0);
-#endif
 }
 
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
-void
-ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt)(ErtsPollSet ps)
-{
-    /*
-     * NOTE: This function is called from signal handlers, it,
-     *       therefore, it has to be async-signal safe.
-     */
-    wake_poller(ps, 1, 1);
-}
-#endif
 
 /*
  * erts_poll_interrupt_timed():
@@ -2552,7 +2416,6 @@ ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps,
 					    int set,
 					    ErtsMonotonicTime timeout_time)
 {
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(ERTS_SMP)
     if (!set)
 	reset_wakeup_state(ps);
     else {
@@ -2568,7 +2431,6 @@ ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps,
 	erts_smp_atomic_inc_nob(&ps->no_interrupt_timed);
 #endif
     }
-#endif
 }
 
 int
@@ -2682,22 +2544,14 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
     ERTS_FD_ZERO(&ps->res_output_fds);
 #endif
 #endif
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
     ps->update_requests.next = NULL;
     ps->update_requests.len = 0;
     ps->curr_upd_req_block = &ps->update_requests;
     erts_smp_atomic32_init_nob(&ps->have_update_requests, 0);
-#endif
-#ifdef ERTS_SMP
     erts_atomic32_init_nob(&ps->polled, 0);
     erts_smp_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO);
-#endif
-#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
     erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0);
-#endif
-#if ERTS_POLL_USE_WAKEUP_PIPE
     create_wakeup_pipe(ps);
-#endif
 #if ERTS_POLL_USE_TIMERFD
     create_timerfd(ps);
 #endif
@@ -2724,9 +2578,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
     erts_smp_atomic_init_nob(&ps->no_avoided_interrupts, 0);
     erts_smp_atomic_init_nob(&ps->no_interrupt_timed, 0);
 #endif
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
     handle_update_requests(ps);
-#endif
 #if ERTS_POLL_USE_FALLBACK
     ps->fallback_used = 0;
 #endif
@@ -2772,7 +2624,6 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps)
 	erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->res_output_fds.ptr);
 #endif
 #endif
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
     {
 	ErtsPollSetUpdateRequestsBlock *urqbp = ps->update_requests.next;
 	while (urqbp) {
@@ -2781,16 +2632,11 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps)
 	    free_update_requests_block(ps, free_urqbp);
 	}
     }
-#endif
-#ifdef ERTS_SMP
     erts_smp_mtx_destroy(&ps->mtx);
-#endif
-#if ERTS_POLL_USE_WAKEUP_PIPE
     if (ps->wake_fds[0] >= 0)
 	close(ps->wake_fds[0]);
     if (ps->wake_fds[1] >= 0)
 	close(ps->wake_fds[1]);
-#endif
 #if ERTS_POLL_USE_TIMERFD
     if (ps->timer_fd >= 0)
         close(ps->timer_fd);
@@ -2818,9 +2664,7 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps)
 void
 ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
 {
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
     int pending_updates;
-#endif
     Uint size = 0;
 
     ERTS_POLLSET_LOCK(ps);
@@ -2845,7 +2689,6 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
 #endif
 #endif
 
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
     {
 	ErtsPollSetUpdateRequestsBlock *urqbp = ps->update_requests.next;
 	pending_updates = ps->update_requests.len;
@@ -2855,7 +2698,6 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
 	    urqbp = urqbp->next;
 	}
     }
-#endif
 
     pip->primary = 
 #if ERTS_POLL_USE_KQUEUE
@@ -2896,9 +2738,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
     pip->memory_size = size;
 
     pip->poll_set_size = (int) erts_smp_atomic_read_nob(&ps->no_of_user_fds);
-#if ERTS_POLL_USE_WAKEUP_PIPE
     pip->poll_set_size++; /* Wakeup pipe */
-#endif
 #if ERTS_POLL_USE_TIMERFD
     pip->poll_set_size++; /* timerfd */
 #endif
@@ -2922,19 +2762,11 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
 #endif
 
     pip->lazy_updates =
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
 	1
-#else
-	0
-#endif
 	;
 
     pip->pending_updates = 
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
 	pending_updates
-#else
-	0
-#endif
 	;
 
     pip->batch_updates = 
@@ -3031,9 +2863,7 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet ps,
 	else {
 	    ev[fd] = ps->fds_status[fd].events;
             if (
-#if ERTS_POLL_USE_WAKEUP_PIPE
                 fd == ps->wake_fds[0] || fd == ps->wake_fds[1] ||
-#endif
 #if ERTS_POLL_USE_TIMERFD
                 fd == ps->timer_fd ||
 #endif
@@ -3121,11 +2951,7 @@ print_misc_debug_info(void)
 		"select"
 #endif
 		,
-#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
 		"true"
-#else
-		"false"
-#endif
 		,
 #if ERTS_POLL_USE_BATCH_UPDATE_POLLSET
 		"true"
diff --git a/erts/emulator/sys/common/erl_poll.h b/erts/emulator/sys/common/erl_poll.h
index b3b4d79984..a57dc51e5b 100644
--- a/erts/emulator/sys/common/erl_poll.h
+++ b/erts/emulator/sys/common/erl_poll.h
@@ -227,9 +227,6 @@ typedef struct {
 #endif
 } ErtsPollInfo;
 
-#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
-void		ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt)(ErtsPollSet);
-#endif
 void		ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet,
 						      int);
 void		ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet,
diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index 22059d21d5..cbbd0e6f40 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -128,10 +128,6 @@
 /* File descriptors are numbers anc consecutively allocated on Unix */
 #define  ERTS_SYS_CONTINOUS_FD_NUMBERS
 
-#ifndef ERTS_SMP
-#  undef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
-#  define ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
-#endif
 
 typedef void *GETENV_STATE;
 
@@ -354,9 +350,7 @@ extern void erts_sys_unix_later_init(void);
 #ifdef NO_FPE_SIGNALS
 
 #define erts_get_current_fp_exception() NULL
-#ifdef ERTS_SMP
 #define erts_thread_init_fp_exception() do{}while(0)
-#endif
 #  define __ERTS_FP_CHECK_INIT(fpexnp) do {} while (0)
 #  define __ERTS_FP_ERROR(fpexnp, f, Action) if (!isfinite(f)) { Action; } else {}
 #  define __ERTS_FP_ERROR_THOROUGH(fpexnp, f, Action) __ERTS_FP_ERROR(fpexnp, f, Action)
@@ -369,9 +363,7 @@ extern void erts_sys_unix_later_init(void);
 #else /* !NO_FPE_SIGNALS */
 
 extern volatile unsigned long *erts_get_current_fp_exception(void);
-#ifdef ERTS_SMP
 extern void erts_thread_init_fp_exception(void);
-#endif
 #  if (defined(__i386__) || defined(__x86_64__)) && defined(__GNUC__)
 #    define erts_fwait(fpexnp,f) \
 	__asm__ __volatile__("fwait" : "=m"(*(fpexnp)) : "m"(f))
@@ -438,10 +430,8 @@ void erts_sys_unblock_fpe(int);
 
 
 /* Threads */
-#ifdef USE_THREADS
 extern int init_async(int);
 extern int exit_async(void);
-#endif
 
 #define ERTS_EXIT_AFTER_DUMP _exit
 
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 50d8a35217..83edca1c2f 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -58,9 +58,7 @@
 #define __DARWIN__ 1
 #endif
 
-#ifdef USE_THREADS
 #include "erl_threads.h"
-#endif
 
 #include "erl_mseg.h"
 
@@ -94,19 +92,12 @@ extern void erts_sys_init_float(void);
 static int debug_log = 0;
 #endif
 
-#ifdef ERTS_SMP
 static erts_smp_atomic32_t have_prepared_crash_dump;
 #define ERTS_PREPARED_CRASH_DUMP \
   ((int) erts_smp_atomic32_xchg_nob(&have_prepared_crash_dump, 1))
-#else
-static volatile int have_prepared_crash_dump;
-#define ERTS_PREPARED_CRASH_DUMP \
-  (have_prepared_crash_dump++)
-#endif
 
 erts_smp_atomic_t sys_misc_mem_sz;
 
-#if defined(ERTS_SMP)
 static void smp_sig_notify(int signum);
 static int sig_notify_fds[2] = {-1, -1};
 
@@ -114,7 +105,6 @@ static int sig_notify_fds[2] = {-1, -1};
 static int sig_suspend_fds[2] = {-1, -1};
 #endif
 
-#endif
 
 jmp_buf erts_sys_sigsegv_jmp;
 
@@ -128,38 +118,12 @@ static int max_files = -1;
 /* 
  * a few variables used by the break handler 
  */
-#ifdef ERTS_SMP
 erts_smp_atomic32_t erts_break_requested;
 #define ERTS_SET_BREAK_REQUESTED \
   erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1)
 #define ERTS_UNSET_BREAK_REQUESTED \
   erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0)
-#else
-volatile int erts_break_requested = 0;
-#define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1)
-#define ERTS_UNSET_BREAK_REQUESTED (erts_break_requested = 0)
-#endif
 
-#ifndef ERTS_SMP
-static Eterm signalstate_sigterm[] = {
-    am_sigint,   /* 0 */
-    am_sighup,   /* 1 */
-    am_sigquit,  /* 2 */
-    am_sigabrt,  /* 3 */
-    am_sigalrm,  /* 4 */
-    am_sigterm,  /* 5 */
-    am_sigusr1,  /* 6 */
-    am_sigusr2,  /* 7 */
-    am_sigchld,  /* 8 */
-    am_sigstop,  /* 9 */
-    am_sigtstp   /* 10 */
-};
-
-volatile Uint erts_signal_state = 0;
-#define ERTS_SET_SIGNAL_STATE(S) (erts_signal_state |= signum_to_signalstate(S))
-#define ERTS_CLEAR_SIGNAL_STATE (erts_signal_state = 0)
-static ERTS_INLINE Uint signum_to_signalstate(int signum);
-#endif
 
 /* set early so the break handler has access to initial mode */
 static struct termios initial_tty_mode;
@@ -223,9 +187,6 @@ init_check_io(void)
 	io_func.select			= driver_select_kp;
         io_func.enif_select		= enif_select_kp;
 	io_func.event			= driver_event_kp;
-#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
-	io_func.check_io_as_interrupt	= erts_check_io_async_sig_interrupt_kp;
-#endif
 	io_func.check_io_interrupt	= erts_check_io_interrupt_kp;
 	io_func.check_io_interrupt_tmd	= erts_check_io_interrupt_timed_kp;
 	io_func.check_io		= erts_check_io_kp;
@@ -239,9 +200,6 @@ init_check_io(void)
 	io_func.select			= driver_select_nkp;
         io_func.enif_select		= enif_select_nkp;
 	io_func.event			= driver_event_nkp;
-#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
-	io_func.check_io_as_interrupt	= erts_check_io_async_sig_interrupt_nkp;
-#endif
 	io_func.check_io_interrupt	= erts_check_io_interrupt_nkp;
 	io_func.check_io_interrupt_tmd	= erts_check_io_interrupt_timed_nkp;
 	io_func.check_io		= erts_check_io_nkp;
@@ -253,11 +211,7 @@ init_check_io(void)
     }
 }
 
-#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
-#define ERTS_CHK_IO_AS_INTR()	(*io_func.check_io_as_interrupt)()
-#else
 #define ERTS_CHK_IO_AS_INTR()	(*io_func.check_io_interrupt)(1)
-#endif
 #define ERTS_CHK_IO_INTR	(*io_func.check_io_interrupt)
 #define ERTS_CHK_IO_INTR_TMD	(*io_func.check_io_interrupt_tmd)
 #define ERTS_CHK_IO		(*io_func.check_io)
@@ -272,11 +226,7 @@ init_check_io(void)
     max_files = erts_check_io_max_files();
 }
 
-#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
-#define ERTS_CHK_IO_AS_INTR()	erts_check_io_async_sig_interrupt()
-#else
 #define ERTS_CHK_IO_AS_INTR()	erts_check_io_interrupt(1)
-#endif
 #define ERTS_CHK_IO_INTR	erts_check_io_interrupt
 #define ERTS_CHK_IO_INTR_TMD	erts_check_io_interrupt_timed
 #define ERTS_CHK_IO		erts_check_io
@@ -290,13 +240,11 @@ erts_sys_schedule_interrupt(int set)
     ERTS_CHK_IO_INTR(set);
 }
 
-#ifdef ERTS_SMP
 void
 erts_sys_schedule_interrupt_timed(int set, ErtsMonotonicTime timeout_time)
 {
     ERTS_CHK_IO_INTR_TMD(set, timeout_time);
 }
-#endif
 
 UWord
 erts_sys_get_page_size(void)
@@ -339,7 +287,6 @@ MALLOC_USE_HASH(1);
 #endif
 #endif
 
-#ifdef USE_THREADS
 
 #ifdef ERTS_THR_HAVE_SIG_FUNCS
 
@@ -418,19 +365,15 @@ thr_create_prepare_child(void *vtcdp)
     erts_sched_bind_atthrcreate_child(tcdp->sched_bind_data);
 }
 
-#endif /* #ifdef USE_THREADS */
 
 void
 erts_sys_pre_init(void)
 {
-#ifdef USE_THREADS
     erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER;
-#endif
 
     erts_printf_add_cr_to_stdout = 1;
     erts_printf_add_cr_to_stderr = 1;
 
-#ifdef USE_THREADS
 
     eid.thread_create_child_func = thr_create_prepare_child;
     /* Before creation in parent */
@@ -452,21 +395,13 @@ erts_sys_pre_init(void)
     erts_lc_init();
 #endif
 
-#endif /* USE_THREADS */
 
     erts_init_sys_time_sup();
 
-#ifdef USE_THREADS
 
-#ifdef ERTS_SMP
     erts_smp_atomic32_init_nob(&erts_break_requested, 0);
     erts_smp_atomic32_init_nob(&have_prepared_crash_dump, 0);
-#else
-    erts_break_requested = 0;
-    have_prepared_crash_dump = 0;
-#endif
 
-#endif /* USE_THREADS */
 
     erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0);
 
@@ -531,10 +466,8 @@ SIGFUNC sys_signal(int sig, SIGFUNC func)
     return(oact.sa_handler);
 }
 
-#ifdef USE_THREADS
 #undef  sigprocmask
 #define sigprocmask erts_thr_sigmask
-#endif
 
 void sys_sigblock(int sig)
 {
@@ -697,11 +630,7 @@ break_requested(void)
 
 static RETSIGTYPE request_break(int signum)
 {
-#ifdef ERTS_SMP
     smp_sig_notify(signum);
-#else
-    break_requested();
-#endif
 }
 
 #ifdef ETHR_UNUSABLE_SIGUSRX
@@ -810,35 +739,10 @@ signum_to_signalterm(int signum)
     }
 }
 
-#ifndef ERTS_SMP
-static ERTS_INLINE Uint
-signum_to_signalstate(int signum)
-{
-    switch (signum) {
-    case SIGINT:  return (1 <<  0);
-    case SIGHUP:  return (1 <<  1);
-    case SIGQUIT: return (1 <<  2);
-    case SIGABRT: return (1 <<  3);
-    case SIGALRM: return (1 <<  4);
-    case SIGTERM: return (1 <<  5);
-    case SIGUSR1: return (1 <<  6);
-    case SIGUSR2: return (1 <<  7);
-    case SIGCHLD: return (1 <<  8);
-    case SIGSTOP: return (1 <<  9);
-    case SIGTSTP: return (1 << 10);
-    default:      return 0;
-    }
-}
-#endif
 
 static RETSIGTYPE generic_signal_handler(int signum)
 {
-#ifdef ERTS_SMP
     smp_sig_notify(signum);
-#else
-    ERTS_SET_SIGNAL_STATE(signum);
-    ERTS_CHK_IO_AS_INTR(); /* Make sure we don't sleep in poll */
-#endif
 }
 
 int erts_set_signal(Eterm signal, Eterm type) {
@@ -1032,22 +936,6 @@ void erts_do_break_handling(void)
     erts_smp_thr_progress_unblock();
 }
 
-#ifdef ERTS_SIGNAL_STATE
-void erts_handle_signal_state(void) {
-    Uint signal_state = ERTS_SIGNAL_STATE;
-    Uint i = 0;
-
-    ERTS_CLEAR_SIGNAL_STATE;
-
-    while (signal_state) {
-        if (signal_state & 0x1) {
-            signal_notify_requested(signalstate_sigterm[i]);
-        }
-        i++;
-        signal_state = signal_state >> 1;
-    }
-}
-#endif
 
 /* Fills in the systems representation of the jam/beam process identifier.
 ** The Pid is put in STRING representation in the supplied buffer,
@@ -1282,16 +1170,6 @@ erl_assert_error(const char* expr, const char* func, const char* file, int line)
     fprintf(stderr, "%s:%d:%s() Assertion failed: %s\n",
             file, line, func, expr);
     fflush(stderr);
-#if !defined(ERTS_SMP) && 0
-    /* Writing a crashdump from a failed assertion when smp support
-     * is enabled almost a guaranteed deadlocking, don't even bother.
-     *
-     * It could maybe be useful (but I'm not convinced) to write the
-     * crashdump if smp support is disabled...
-     */
-    if (erts_initialized)
-	erl_crash_dump(file, line, "Assertion failed: %s\n", expr);
-#endif
     abort();
 }
 
@@ -1326,7 +1204,6 @@ erl_sys_schedule(int runnable)
 }
 
 
-#ifdef ERTS_SMP
 
 static erts_smp_tid_t sig_dispatcher_tid;
 
@@ -1515,7 +1392,6 @@ erts_sys_main_thread(void)
     }
 }
 
-#endif /* ERTS_SMP */
 
 #ifdef ERTS_ENABLE_KERNEL_POLL /* get_value() is currently only used when
 				  kernel-poll is enabled */
@@ -1602,10 +1478,8 @@ erl_sys_args(int* argc, char** argv)
 
     init_check_io();
 
-#ifdef ERTS_SMP
     init_smp_sig_notify();
     init_smp_sig_suspend();
-#endif
 
     /* Handled arguments have been marked with NULL. Slide arguments
        not handled towards the beginning of argv. */
diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c
index 834706d86f..03d948c6e9 100644
--- a/erts/emulator/sys/unix/sys_drivers.c
+++ b/erts/emulator/sys/unix/sys_drivers.c
@@ -53,9 +53,7 @@
 #define WANT_NONBLOCKING    /* must define this to pull in defs from sys.h */
 #include "sys.h"
 
-#ifdef USE_THREADS
 #include "erl_threads.h"
-#endif
 
 extern char **environ;
 extern erts_smp_rwmtx_t environ_rwmtx;
@@ -86,11 +84,7 @@ static Eterm forker_port;
 #define MAXIOV 16
 #endif
 
-#ifdef USE_THREADS
 #  define FDBLOCK 1
-#else
-#  define FDBLOCK 0
-#endif
 
 /* Used by the fd driver iff the fd could not be set to non-blocking */
 typedef struct ErtsSysBlocking_ {
@@ -178,9 +172,7 @@ void
 erl_sys_late_init(void)
 {
     SysDriverOpts opts;
-#ifdef ERTS_SMP
     Port *port;
-#endif
 
     sys_signal(SIGPIPE, SIG_IGN); /* Ignore - we'll handle the write failure */
 
@@ -197,13 +189,9 @@ erl_sys_late_init(void)
     opts.argv = NULL;
     opts.parallelism = erts_port_parallelism;
 
-#ifdef ERTS_SMP
     port =
-#endif
         erts_open_driver(&forker_driver, make_internal_pid(0), "forker", &opts, NULL, NULL);
-#ifdef ERTS_SMP
     erts_mtx_unlock(port->lock);
-#endif
     erts_sys_unix_later_init(); /* Need to be called after forker has been started */
 }
 
@@ -220,10 +208,8 @@ static ErlDrvData vanilla_start(ErlDrvPort, char*, SysDriverOpts*);
 
 /* II.III FD prototypes */
 static ErlDrvData fd_start(ErlDrvPort, char*, SysDriverOpts*);
-#if FDBLOCK
 static void fd_async(void *);
 static void fd_ready_async(ErlDrvData drv_data, ErlDrvThreadData thread_data);
-#endif
 static ErlDrvSSizeT fd_control(ErlDrvData, unsigned int, char *, ErlDrvSizeT,
 			       char **, ErlDrvSizeT);
 static void fd_stop(ErlDrvData);
@@ -287,11 +273,7 @@ struct erl_drv_entry fd_driver_entry = {
     fd_control,
     NULL,
     outputv,
-#if FDBLOCK
     fd_ready_async, /* ready_async */
-#else
-    NULL,
-#endif
     fd_flush, /* flush */
     NULL, /* call */
     NULL, /* event */
@@ -1092,13 +1074,11 @@ static void fd_stop(ErlDrvData ev)  /* Does not close the fds */
     ErlDrvPort prt = dd->port_num;
     int sz = sizeof(ErtsSysDriverData);
 
-#if FDBLOCK
     if (dd->blocking) {
         erts_free(ERTS_ALC_T_SYS_BLOCKING, dd->blocking);
         dd->blocking = NULL;
         sz += sizeof(ErtsSysBlocking);
     }
-#endif
 
     if (dd->ifd) {
         sz += sizeof(ErtsSysFdData);
@@ -1220,7 +1200,6 @@ static void outputv(ErlDrvData e, ErlIOVec* ev)
 	driver_enqv(ix, ev, n);  /* n is the skip value */
 	driver_select(ix, ofd, ERL_DRV_WRITE|ERL_DRV_USE, 1);
     }
-#if FDBLOCK
     else {
         if (ev->size != 0) {
             driver_enqv(ix, ev, 0);
@@ -1231,7 +1210,6 @@ static void outputv(ErlDrvData e, ErlIOVec* ev)
             driver_pdl_unlock(dd->blocking->pdl);
         }
     }
-#endif
     /* return 0;*/
 }
 
@@ -1579,7 +1557,6 @@ static void stop_select(ErlDrvEvent fd, void* _)
     close((int)fd);
 }
 
-#if FDBLOCK
 
 static void
 fd_async(void *async_data)
@@ -1658,7 +1635,6 @@ void fd_ready_async(ErlDrvData drv_data,
     return; /* 0; */
 }
 
-#endif
 
 /* Forker driver */
 
diff --git a/erts/emulator/sys/unix/sys_float.c b/erts/emulator/sys/unix/sys_float.c
index 6435da086f..a82c15bd32 100644
--- a/erts/emulator/sys/unix/sys_float.c
+++ b/erts/emulator/sys/unix/sys_float.c
@@ -39,7 +39,6 @@ erts_sys_init_float(void)
 
 #else  /* !NO_FPE_SIGNALS */
 
-#ifdef ERTS_SMP
 static erts_tsd_key_t fpe_key;
 
 /* once-only initialisation early in the main thread (via erts_sys_init_float()) */
@@ -61,11 +60,6 @@ static ERTS_INLINE volatile unsigned long *erts_thread_get_fp_exception(void)
 {
     return (volatile unsigned long*)erts_tsd_get(fpe_key);
 }
-#else /* !SMP */
-#define erts_init_fp_exception()	/*empty*/
-static volatile unsigned long fp_exception;
-#define erts_thread_get_fp_exception()	(&fp_exception)
-#endif /* SMP */
 
 volatile unsigned long *erts_get_current_fp_exception(void)
 {
@@ -659,11 +653,9 @@ void erts_sys_init_float(void)
 
 void erts_thread_init_float(void)
 {
-#ifdef ERTS_SMP
     /* This allows Erlang schedulers to leave Erlang-process context
        and still have working FP exceptions. XXX: is this needed? */
     erts_thread_init_fp_exception();
-#endif
 
 #ifndef NO_FPE_SIGNALS
     /* NOTE:
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index 8743f83a50..41ff186c44 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -286,41 +286,26 @@ struct ErtsPollSet_ {
     CRITICAL_SECTION standby_crit;     /* CS to guard the counter */
     HANDLE standby_wait_event;         /* Event signalled when counte == 0 */
     erts_atomic32_t wakeup_state;
-#ifdef ERTS_SMP
     erts_smp_mtx_t mtx;
-#endif
     erts_atomic64_t timeout_time;
 };
 
-#ifdef ERTS_SMP
 
 #define ERTS_POLLSET_LOCK(PS) \
   erts_smp_mtx_lock(&(PS)->mtx)
 #define ERTS_POLLSET_UNLOCK(PS) \
   erts_smp_mtx_unlock(&(PS)->mtx)
 
-#else
-
-#define ERTS_POLLSET_LOCK(PS)
-#define ERTS_POLLSET_UNLOCK(PS)
-
-#endif
 
 /*
  * Communication with sys_interrupt
  */
 
-#ifdef ERTS_SMP
 extern erts_smp_atomic32_t erts_break_requested;
 #define ERTS_SET_BREAK_REQUESTED \
   erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1)
 #define ERTS_UNSET_BREAK_REQUESTED \
   erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0)
-#else
-extern volatile int erts_break_requested;
-#define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1)
-#define ERTS_UNSET_BREAK_REQUESTED (erts_break_requested = 0)
-#endif
 
 static erts_mtx_t break_waiter_lock;
 static HANDLE break_happened_event;
@@ -1193,14 +1178,10 @@ int erts_poll_wait(ErtsPollSet ps,
 
 	HARDDEBUGF(("Start waiting %d [%d]",num_h, (int) timeout));
 	ERTS_POLLSET_UNLOCK(ps);
-#ifdef ERTS_SMP
 	erts_thr_progress_prepare_wait(NULL);
-#endif
         ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
 	WaitForMultipleObjects(num_h, harr, FALSE, timeout);
-#ifdef ERTS_SMP
 	erts_thr_progress_finalize_wait(NULL);
-#endif
         ERTS_MSACC_POP_STATE_M();
 	ERTS_POLLSET_LOCK(ps);
 	HARDDEBUGF(("Stop waiting %d [%d]",num_h, (int) timeout));
@@ -1359,9 +1340,7 @@ ErtsPollSet erts_poll_create_pollset(void)
     ps->restore_events = 0;
 
     erts_atomic32_init_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
-#ifdef ERTS_SMP
     erts_smp_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO);
-#endif
     init_timeout_time(ps);
 
     HARDTRACEF(("Out erts_poll_create_pollset"));
@@ -1391,9 +1370,7 @@ void erts_poll_destroy_pollset(ErtsPollSet ps)
     CloseHandle(ps->event_io_ready);
     CloseHandle(ps->standby_wait_event);
     ERTS_POLLSET_UNLOCK(ps);
-#ifdef ERTS_SMP
     erts_smp_mtx_destroy(&ps->mtx);
-#endif
     SEL_FREE(ERTS_ALC_T_POLLSET, (void *) ps);
     HARDTRACEF(("Out erts_poll_destroy_pollset"));
 }
diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h
index 78005aada9..1f53452d17 100644
--- a/erts/emulator/sys/win32/erl_win_sys.h
+++ b/erts/emulator/sys/win32/erl_win_sys.h
@@ -311,10 +311,8 @@ typedef long ssize_t;
 #endif
 
 /* Threads */
-#ifdef USE_THREADS
 int init_async(int);
 int exit_async(void);
-#endif
 
 #define ERTS_HAVE_TRY_CATCH 1
 
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index 15c59109b1..7fddc9fa99 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -80,9 +80,7 @@ static int application_type(const wchar_t* originalName, wchar_t fullPath[MAX_PA
 
 HANDLE erts_service_event;
 
-#ifdef ERTS_SMP
 static erts_smp_tsd_key_t win32_errstr_key;
-#endif
 
 static erts_smp_atomic_t pipe_creation_counter;
 
@@ -94,10 +92,8 @@ static erts_smp_atomic_t pipe_creation_counter;
 
 static int driver_write(long, HANDLE, byte*, int);
 static int create_file_thread(struct async_io* aio, int mode);
-#ifdef ERTS_SMP
 static void close_active_handle(DriverData *, HANDLE handle);
 static DWORD WINAPI threaded_handle_closer(LPVOID param);
-#endif
 static DWORD WINAPI threaded_reader(LPVOID param);
 static DWORD WINAPI threaded_writer(LPVOID param);
 static DWORD WINAPI threaded_exiter(LPVOID param);
@@ -450,9 +446,7 @@ typedef struct async_io {
 				 * the console for Windows NT).
 				 */
   HANDLE fd;			/* Handle for file or pipe. */
-#ifdef ERTS_SMP
   int async_io_active;          /* if true, a close of the file will signal the event in ov */
-#endif
   OVERLAPPED ov;		/* Control structure for overlapped reading.
 				 * When overlapped reading is simulated with
 				 * a thread, the fields are used as follows:
@@ -691,7 +685,6 @@ buf_alloc_error:
 static void
 release_driver_data(DriverData* dp)
 {
-#ifdef ERTS_SMP
 #ifdef USE_CANCELIOEX
     if (fpCancelIoEx != NULL) {
 	if (dp->in.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) {
@@ -734,14 +727,6 @@ release_driver_data(DriverData* dp)
 	    DEBUGF(("...done\n"));
 	}
     }
-#else
-	if (dp->in.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) {
-	     CancelIo(dp->in.fd); 
-	}
-	if (dp->out.thread == (HANDLE) -1 && dp->out.fd != INVALID_HANDLE_VALUE) {
-	    CancelIo(dp->out.fd); 
-	}
-#endif
 
     if (dp->inbuf != NULL) {
 	ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize);
@@ -777,7 +762,6 @@ release_driver_data(DriverData* dp)
     unrefer_driver_data(dp);
 }
 
-#ifdef ERTS_SMP
 
 struct handles_to_be_closed {
     HANDLE handles[MAXIMUM_WAIT_OBJECTS];
@@ -870,7 +854,6 @@ threaded_handle_closer(LPVOID param)
     DEBUGF(("threaded_handle_closer %p terminating\r\n", htbc));
     return 0;
 }
-#endif /* ERTS_SMP */
 
 /*
  * Stores input and output file descriptors in the DriverData structure,
@@ -946,9 +929,7 @@ init_async_io(DriverData *dp, AsyncIo* aio, int use_threads)
     aio->flushReplyEvent = NULL;
     aio->pendingError = 0;
     aio->bytesTransferred = 0;
-#ifdef ERTS_SMP
     aio->async_io_active = 0;
-#endif
     aio->ov.hEvent = CreateManualEvent(FALSE);
     if (aio->ov.hEvent == NULL)
 	return -1;
@@ -1029,9 +1010,7 @@ async_read_file(AsyncIo* aio, LPVOID buf, DWORD numToRead)
 	ResetEvent(aio->ov.hEvent);
 	SetEvent(aio->ioAllowed);
     } else {
-#ifdef ERTS_SMP
 	aio->async_io_active = 1; /* Will get 0 when the event actually happened */
-#endif
 	if (ReadFile(aio->fd, buf, numToRead,
 		     &aio->bytesTransferred, &aio->ov)) {
 	    DEBUGF(("async_read_file: ReadFile() suceeded: %d bytes\n",
@@ -1079,16 +1058,12 @@ async_write_file(AsyncIo* aio,		/* Pointer to async control block. */
 	ResetEvent(aio->ov.hEvent);
 	SetEvent(aio->ioAllowed);
     } else {
-#ifdef ERTS_SMP
 	aio->async_io_active = 1; /* Will get 0 when the event actually happened */
-#endif
 	if (WriteFile(aio->fd, buf, numToWrite,
 		      &aio->bytesTransferred, &aio->ov)) {
 	    DEBUGF(("async_write_file: WriteFile() suceeded: %d bytes\n",
 		    aio->bytesTransferred));
-#ifdef ERTS_SMP
 	    aio->async_io_active = 0; /* The event will not be signalled */
-#endif
 	    ResetEvent(aio->ov.hEvent);
 	    return TRUE;
 	} else {
@@ -2511,11 +2486,9 @@ ready_input(ErlDrvData drv_data, ErlDrvEvent ready_event)
     int pb;
 
     pb = dp->packet_bytes;
-#ifdef ERTS_SMP
     if(dp->in.thread == (HANDLE) -1) {
 	dp->in.async_io_active = 0;
     }
-#endif
     DEBUGF(("ready_input: dp %p, event 0x%x\n", dp, ready_event));
 
     /*
@@ -2680,11 +2653,9 @@ ready_output(ErlDrvData drv_data, ErlDrvEvent ready_event)
     DriverData *dp = (DriverData *) drv_data;
     int error;
 
-#ifdef ERTS_SMP
     if(dp->out.thread == (HANDLE) -1) {
 	dp->out.async_io_active = 0;
     }
-#endif
     DEBUGF(("ready_output(%p, 0x%x)\n", drv_data, ready_event));
     set_busy_port(dp->port_num, 0);
     if (!(dp->outbuf)) {
@@ -2743,7 +2714,6 @@ sys_init_io(void)
     max_files = 2*erts_ptab_max(&erts_port);
 }
 
-#ifdef ERTS_SMP
 void
 erts_sys_main_thread(void)
 {
@@ -2756,7 +2726,6 @@ erts_sys_main_thread(void)
 	WaitForSingleObject(dummy, INFINITE);
     }
 }
-#endif
 
 void erts_sys_alloc_init(void)
 {
@@ -2938,11 +2907,7 @@ sys_get_key(int fd)
 
 char* win32_errorstr(int error)
 {
-#ifdef SMP
   LPTSTR lpBufPtr = erts_smp_tsd_get(win32_errstr_key);
-#else
-  static LPTSTR lpBufPtr = NULL;
-#endif
   if (lpBufPtr) {
       LocalFree(lpBufPtr);
   }
@@ -2956,9 +2921,7 @@ char* win32_errorstr(int error)
 		0,
 		NULL);
   SetLastError(error);
-#ifdef ERTS_SMP
   erts_smp_tsd_set(win32_errstr_key,lpBufPtr);
-#endif
   return lpBufPtr;
 }
 
@@ -3131,7 +3094,6 @@ check_supported_os_version(void)
 #endif
 }
 
-#ifdef USE_THREADS
 
 typedef struct {
     int sched_bind_data;
@@ -3176,19 +3138,15 @@ thr_create_prepare_child(void *vtcdp)
     erts_sched_bind_atthrcreate_child(tcdp->sched_bind_data);
 }
 
-#endif /* USE_THREADS */
 
 void
 erts_sys_pre_init(void)
 {
-#ifdef USE_THREADS
     erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER;
-#endif
     int_os_version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
     GetVersionEx(&int_os_version);
     check_supported_os_version();
 
-#ifdef USE_THREADS
     eid.thread_create_child_func = thr_create_prepare_child;
     /* Before creation in parent */
     eid.thread_create_prepare_func = thr_create_prepare;
@@ -3209,7 +3167,6 @@ erts_sys_pre_init(void)
     erts_lc_init();
 #endif
 
-#endif /* USE_THREADS */
 
     erts_init_sys_time_sup();
 
@@ -3233,10 +3190,8 @@ void erl_sys_init(void)
     noinherit_std_handle(STD_INPUT_HANDLE);
     noinherit_std_handle(STD_ERROR_HANDLE);
 
-#ifdef ERTS_SMP
     erts_smp_tsd_key_create(&win32_errstr_key,"win32_errstr_key");
     InitializeCriticalSection(&htbc_lock);
-#endif
     erts_smp_atomic_init_nob(&pipe_creation_counter,0);
     /*
      * Test if we have named pipes or not.
@@ -3299,13 +3254,11 @@ erts_sys_schedule_interrupt(int set)
     erts_check_io_interrupt(set);
 }
 
-#ifdef ERTS_SMP
 void
 erts_sys_schedule_interrupt_timed(int set, ErtsMonotonicTime timeout_time)
 {
     erts_check_io_interrupt_timed(set, timeout_time);
 }
-#endif
 
 /*
  * Called from schedule() when it runs out of runnable processes,
diff --git a/erts/emulator/sys/win32/sys_interrupt.c b/erts/emulator/sys/win32/sys_interrupt.c
index df838960eb..4632ab52b1 100644
--- a/erts/emulator/sys/win32/sys_interrupt.c
+++ b/erts/emulator/sys/win32/sys_interrupt.c
@@ -35,17 +35,11 @@
 #  define WIN_SYS_INLINE __forceinline
 #endif
 
-#ifdef ERTS_SMP
 erts_smp_atomic32_t erts_break_requested;
 #define ERTS_SET_BREAK_REQUESTED \
   erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1)
 #define ERTS_UNSET_BREAK_REQUESTED \
   erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0)
-#else
-volatile int erts_break_requested = 0;
-#define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1)
-#define ERTS_UNSET_BREAK_REQUESTED (erts_break_requested = 0)
-#endif
 
 extern int nohup;
 HANDLE erts_sys_break_event = NULL;
-- 
cgit v1.2.3


From 241f27c4942d6b765abcec9d5a9712e07861bc13 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 12 Jul 2017 10:58:19 +0200
Subject: erts: Cleanup removal of non-smp emulators

---
 erts/emulator/beam/beam_bp.c                       |   2 +-
 erts/emulator/beam/bif.c                           |   4 +-
 erts/emulator/beam/dist.c                          |   2 +-
 erts/emulator/beam/erl_alloc.c                     |   2 +-
 erts/emulator/beam/erl_async.c                     |   3 +-
 erts/emulator/beam/erl_async.h                     |   9 -
 erts/emulator/beam/erl_bif_ddll.c                  |   3 -
 erts/emulator/beam/erl_bif_info.c                  |   4 +-
 erts/emulator/beam/erl_bits.h                      |   3 -
 erts/emulator/beam/erl_drv_thread.c                |   2 +-
 erts/emulator/beam/erl_gc.c                        |   3 +
 erts/emulator/beam/erl_hl_timer.c                  |  11 +-
 erts/emulator/beam/erl_message.c                   |   4 +
 erts/emulator/beam/erl_node_tables.h               |   2 +-
 erts/emulator/beam/erl_port.h                      |   2 +-
 erts/emulator/beam/erl_port_task.c                 |   4 +
 erts/emulator/beam/erl_port_task.h                 |   2 +-
 erts/emulator/beam/erl_process.c                   |  28 +-
 erts/emulator/beam/erl_process.h                   |  14 +-
 erts/emulator/beam/erl_process_lock.c              |   2 +-
 erts/emulator/beam/erl_process_lock.h              |   8 +-
 erts/emulator/beam/erl_smp.h                       |  30 +-
 erts/emulator/beam/erl_threads.h                   | 368 +--------------------
 erts/emulator/beam/erl_trace.c                     |   6 +-
 erts/emulator/beam/global.h                        |   2 +-
 erts/emulator/beam/io.c                            |  23 +-
 erts/emulator/hipe/hipe_amd64_bifs.m4              |   2 +-
 erts/emulator/hipe/hipe_arm_bifs.m4                |   2 +-
 erts/emulator/hipe/hipe_bif2.c                     |   4 +-
 erts/emulator/hipe/hipe_mkliterals.c               |   2 +-
 erts/emulator/hipe/hipe_mode_switch.c              |  24 +-
 erts/emulator/hipe/hipe_ppc_bifs.m4                |   2 +-
 erts/emulator/hipe/hipe_process.h                  |   2 +-
 erts/emulator/hipe/hipe_signal.h                   |   6 +-
 erts/emulator/hipe/hipe_sparc_bifs.m4              |   2 +-
 erts/emulator/hipe/hipe_stack.h                    |   5 +
 erts/emulator/hipe/hipe_x86_bifs.m4                |   2 +-
 erts/emulator/sys/common/erl_check_io.c            |   2 +-
 .../sys/common/erl_os_monotonic_time_extender.h    |   1 -
 erts/emulator/sys/common/erl_poll.c                |  15 +-
 erts/emulator/sys/unix/sys_drivers.c               |  12 +-
 erts/emulator/sys/unix/sys_time.c                  |   2 -
 erts/emulator/sys/win32/sys.c                      |   4 +-
 erts/emulator/sys/win32/sys_time.c                 |   6 -
 44 files changed, 128 insertions(+), 510 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 950639f7ae..5b24db2e24 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -46,7 +46,7 @@
 #define ReAlloc(P, SIZ)		erts_realloc(ERTS_ALC_T_BPD, (P), (SZ))
 #define Free(P)			erts_free(ERTS_ALC_T_BPD, (P))
 
-#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
 #  define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
       if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\
 					 __FILE__, __LINE__)
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index dbd09ef126..e9a0668e2d 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4729,7 +4729,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 		      ref,
 		      old ? am_true : am_false);
 	}
-#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS)
+#if defined(ERTS_DIRTY_SCHEDULERS)
     } else if (BIF_ARG_1 == am_dirty_cpu_schedulers_online) {
 	Sint old_no;
 	if (!is_small(BIF_ARG_2))
@@ -4987,7 +4987,7 @@ static ERTS_INLINE int
 skip_current_msgq(Process *c_p)
 {
     int res;
-#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     erts_proc_lc_chk_only_proc_main(c_p);
 #endif
 
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index e19ff2be68..2be05eed29 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -3547,7 +3547,7 @@ remove_nodes_monitors(Process *c_p, Uint32 opts, int all)
 void
 erts_delete_nodes_monitors(Process *c_p, ErtsProcLocks locks)
 {
-#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     if (c_p) {
 	ErtsProcLocks might_unlock = locks & ~ERTS_PROC_LOCK_MAIN;
 	if (might_unlock)
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index bf7f71ebde..80ee3d7f33 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -3244,7 +3244,7 @@ reply_alloc_info(void *vair)
 	    case ERTS_ALC_INFO_A_DISABLED_EXEC:
 		break;
 	    case ERTS_ALC_INFO_A_MSEG_ALLOC:
-#if HAVE_ERTS_MSEG && defined(ERTS_SMP)
+#if HAVE_ERTS_MSEG
 		alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
 		ainfo = erts_mseg_info(sched_id, NULL, NULL,
                                        hpp != NULL, air->only_sz, hpp, szp);
diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index 75e88afc5e..eafa7be738 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -115,7 +115,7 @@ typedef struct {
     ErtsAlgndAsyncReadyQ *ready_queue;
 } ErtsAsyncData;
 
-#if defined(USE_THREADS) && defined(USE_VM_PROBES)
+#if defined(USE_VM_PROBES)
 
 /*
  * Some compilers, e.g. GCC 4.2.1 and -O3, will optimize away DTrace
@@ -504,7 +504,6 @@ erts_exit_flush_async(void)
 	erts_thr_join(async->queue[i].aq.thr_id, NULL);
 }
 
-
 int erts_check_async_ready(void *varq)
 {
     ErtsAsyncReadyQ *arq = (ErtsAsyncReadyQ *) varq;
diff --git a/erts/emulator/beam/erl_async.h b/erts/emulator/beam/erl_async.h
index 76fc61b560..70ef247e0a 100644
--- a/erts/emulator/beam/erl_async.h
+++ b/erts/emulator/beam/erl_async.h
@@ -27,15 +27,6 @@ extern int erts_async_max_threads;
 #define ERTS_ASYNC_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */
 extern int erts_async_thread_suggested_stack_size;
 
-
-/*
- * With smp support we can choose to have, or not to
- * have an async ready queue.
- */
-#define ERTS_USE_ASYNC_READY_Q 1
-
-
-
 int erts_check_async_ready(void *);
 int erts_async_ready_clean(void *, void *);
 void *erts_get_async_ready_queue(Uint sched_id);
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index cea8b2200c..fa41505a83 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -50,9 +50,6 @@
 #include "dtrace-wrapper.h"
 #include "lttng-wrapper.h"
 
-#define DDLL_SMP 1
-
-
 /*
  * Local types
  */
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index df02554d83..dcbef9ce2d 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -88,7 +88,7 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
 				     " [64-bit]"
 #endif
 				     " [smp:%beu:%beu]"
-#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
+#if defined(ERTS_DIRTY_SCHEDULERS)
 				     " [ds:%beu:%beu:%beu]"
 #endif
 #if defined(ERTS_DIRTY_SCHEDULERS_TEST)
@@ -2445,7 +2445,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 #endif
 
 	BIF_RET(res);
-#endif /* #ifndef ERTS_SMP */
+#endif /* #ifndef ERTS_OPCODE_COUNTER_SUPPORT */
     } else if (BIF_ARG_1 == am_wordsize) {
 	return make_small(sizeof(Eterm));
     } else if (BIF_ARG_1 == am_endian) {
diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h
index 1445a38772..b9d141d585 100644
--- a/erts/emulator/beam/erl_bits.h
+++ b/erts/emulator/beam/erl_bits.h
@@ -84,9 +84,6 @@ typedef struct erl_bin_match_struct{
 #define ms_matchbuffer(_Ms) &(((ErlBinMatchState*) boxed_val(_Ms))->mb)
 
 
-#define ERL_BITS_REENTRANT
-
-
 /*
  * Reentrant API with the state passed as a parameter.
  * (Except when the current Process* already is a parameter.)
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 4238d161ec..7b19724814 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -709,7 +709,7 @@ erl_drv_thread_join(ErlDrvTid tid, void **respp)
     return res;
 }
 
-#if defined(__DARWIN__) && defined(USE_THREADS) && defined(ERTS_SMP)
+#if defined(__DARWIN__)
 extern int erts_darwin_main_thread_pipe[2];
 extern int erts_darwin_main_thread_result_pipe[2];
 
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index d49564731b..14eeeaf70a 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -60,6 +60,9 @@
 #  define ERTS_GC_ASSERT(B) ((void) 1)
 #endif
 
+#if defined(DEBUG) && 0
+#  define HARDDEBUG 1
+#endif
 
 /*
  * Returns number of elements in an array.
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 5e9b7fe42e..80f3aa04ab 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -96,9 +96,6 @@ typedef enum {
 
 #define ERTS_BIF_TIMER_SHORT_TIME 5000
 
-#  define ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore \
-    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore)
-
 /* Bit 0 to 9 contains scheduler id (see mask below) */
 #define ERTS_TMR_ROFLG_HLT		(((Uint32) 1) << 10)
 #define ERTS_TMR_ROFLG_BIF_TMR		(((Uint32) 1) << 11)
@@ -958,7 +955,7 @@ static ERTS_INLINE void
 tw_timer_dec_refc(ErtsTWTimer *tmr)
 {
     if (erts_smp_atomic32_dec_read_relb(&tmr->head.refc) == 0) {
-	ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore;
+        ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
 	schedule_tw_timer_destroy(tmr);
     }
 }
@@ -1168,7 +1165,7 @@ static ERTS_INLINE void
 hl_timer_dec_refc(ErtsHLTimer *tmr, Uint32 roflgs)
 {
     if (erts_smp_atomic32_dec_read_relb(&tmr->head.refc) == 0) {
-	ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore;
+        ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
 	schedule_hl_timer_destroy(tmr, roflgs);
     }
 }
@@ -1185,7 +1182,7 @@ static void handle_canceled_queue(ErtsSchedulerData *esdp,
 static ERTS_INLINE void
 check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv)
 {
-#if defined(ERTS_SMP) && ERTS_TMR_CHECK_CANCEL_ON_CREATE
+#if ERTS_TMR_CHECK_CANCEL_ON_CREATE
     ErtsHLTCncldTmrQ *cq = &srv->canceled_queue;
     if (cq->head.first != cq->head.unref_end)
 	handle_canceled_queue(esdp, cq, 1,
@@ -1778,7 +1775,7 @@ cq_check_incoming(ErtsSchedulerData *esdp, ErtsHLTCncldTmrQ *cq)
 	cq->head.next.thr_progress_reached = 1;
 	/* Move unreferenced end pointer forward... */
 
-	ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore;
+        ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
 
 	cq->head.unref_end = cq->head.next.unref_end;
 
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index b35c14bfc8..d4e9ff7d18 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -39,7 +39,11 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message_ref,
 				 ERL_MESSAGE_BUF_SZ,
 				 ERTS_ALC_T_MSG_REF)
 
+#if defined(DEBUG) && 0
+#define HARD_DEBUG
+#else
 #undef HARD_DEBUG
+#endif
 
 void
 init_message(void)
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 91bcb4fce1..b036a55609 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -190,7 +190,7 @@ void erts_init_node_tables(int);
 void erts_node_table_info(fmtfn_t, void *);
 void erts_print_node_info(fmtfn_t, void *, Eterm, int*, int*);
 Eterm erts_get_node_and_dist_references(struct process *);
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
 int erts_lc_is_de_rwlocked(DistEntry *);
 int erts_lc_is_de_rlocked(DistEntry *);
 #endif
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index 4c8e4197aa..8adf56e9fe 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -361,7 +361,7 @@ void erts_port_free(Port *);
 void erts_fire_port_monitor(Port *prt, Eterm ref);
 int erts_port_handle_xports(Port *);
 
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
 int erts_lc_is_port_locked(Port *);
 #endif
 
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 5f333ea8d8..b13d43af91 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -43,7 +43,11 @@
  */
 #define ERTS_PORT_CALLBACK_VREDS (CONTEXT_REDS/20)
 
+#if defined(DEBUG) && 0
+#define ERTS_HARD_DEBUG_TASK_QUEUES
+#else
 #undef ERTS_HARD_DEBUG_TASK_QUEUES
+#endif
 
 #ifdef ERTS_HARD_DEBUG_TASK_QUEUES
 static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_queue);
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index 14db641bb6..1a06041d8e 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -201,7 +201,7 @@ erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp)
 ERTS_GLB_INLINE int
 erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp)
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     return erts_lc_mtx_is_locked(&ptsp->mtx);
 #else
     return 0;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index ba6d56269d..8a218d9d69 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -82,7 +82,11 @@
 #define ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA
 #endif
 
+#if defined(DEBUG) && 0
+#define HARDDEBUG
+#else
 #undef HARDDEBUG
+#endif
 
 #ifdef HARDDEBUG
 #define HARDDEBUG_RUNQS
@@ -656,7 +660,7 @@ dbg_chk_aux_work_val(erts_aint32_t value)
 static void do_handle_pending_exiters(ErtsProcList *);
 static void wake_scheduler(ErtsRunQueue *rq);
 
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
 int
 erts_smp_lc_runq_is_locked(ErtsRunQueue *runq)
 {
@@ -9241,25 +9245,6 @@ suspend_process_2(BIF_ALIST_2)
 
     smon = erts_add_or_lookup_suspend_monitor(&BIF_P->suspend_monitors,
 					      BIF_ARG_1);
-#ifndef ERTS_SMP /* no ERTS_SMP */
-
-    /* This is really a piece of cake without SMP support... */
-    if (!smon->active) {
-	erts_smp_atomic32_read_bor_nob(&suspendee->state, ERTS_PSFLG_SUSPENDED);
-	suspend_process(BIF_P, suspendee);
-	smon->active++;
-	res = am_true;
-    }
-    else if (unless_suspending)
-	res = am_false;
-    else if (smon->active == INT_MAX)
-	goto system_limit;
-    else {
-	smon->active++;
-	res = am_true;
-    }
-
-#else /* ERTS_SMP */
 
     /* ... but a little trickier with SMP support ... */
 
@@ -9377,7 +9362,6 @@ suspend_process_2(BIF_ALIST_2)
 	/* --- Synchronous suspend end ------------------------------------- */
     }
 
-#endif /* ERTS_SMP */
 #ifdef DEBUG
     {
 	erts_aint32_t state = erts_smp_atomic32_read_acqb(&suspendee->state);
@@ -14039,7 +14023,7 @@ void erts_halt(int code)
     }
 }
 
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
 int
 erts_dbg_check_halloc_lock(Process *p)
 {
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 5ed1b4e975..513397ef3f 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -694,7 +694,7 @@ extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
 #endif
 
 
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
 int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
 #endif
 
@@ -1506,7 +1506,7 @@ extern int erts_system_profile_ts_type;
 	}						\
     } while (0)
 
-#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
+#if defined(ERTS_DIRTY_SCHEDULERS)
 #define ERTS_NUM_DIRTY_CPU_RUNQS 1
 #define ERTS_NUM_DIRTY_IO_RUNQS 1
 #else
@@ -1763,7 +1763,7 @@ void erts_schedule_ets_free_fixation(Eterm pid, struct db_fixation*);
 void erts_schedule_flush_trace_messages(Process *proc, int force_on_proc);
 int erts_flush_trace_messages(Process *c_p, ErtsProcLocks locks);
 
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
 int erts_dbg_check_halloc_lock(Process *p);
 #endif
 void
@@ -1931,7 +1931,7 @@ erts_schedule_dirty_sys_execution(Process *c_p)
 
 #endif
 
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
 
 #define ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__
 #include "erl_process_lock.h"
@@ -1961,7 +1961,7 @@ ERTS_GLB_INLINE void *
 erts_psd_get(Process *p, int ix)
 {
     ErtsPSD *psd;
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
     if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks)
 	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
@@ -1984,7 +1984,7 @@ ERTS_GLB_INLINE void *
 erts_psd_set(Process *p, int ix, void *data)
 {
     ErtsPSD *psd;
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
     erts_aint32_t state = state = erts_smp_atomic32_read_nob(&p->state);
     if (!(state & ERTS_PSFLG_FREE)) {
@@ -2491,7 +2491,7 @@ extern int erts_disable_proc_not_running_opt;
 
 /* Minimum NUMBER of processes for a small system to start */
 #define ERTS_MIN_PROCESSES		1024
-#if defined(ERTS_SMP) && ERTS_MIN_PROCESSES < ERTS_NO_OF_PIX_LOCKS
+#if ERTS_MIN_PROCESSES < ERTS_NO_OF_PIX_LOCKS
 #undef ERTS_MIN_PROCESSES
 #define ERTS_MIN_PROCESSES		ERTS_NO_OF_PIX_LOCKS
 #endif
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index c824724cc9..0894c5233a 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -1108,7 +1108,7 @@ erts_proc_lock_fin(Process *p)
     erts_mtx_destroy(&p->lock.status);
     erts_mtx_destroy(&p->lock.trace);
 #endif
-#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP)
+#if defined(ERTS_ENABLE_LOCK_COUNT)
     erts_lcnt_proc_lock_destroy(p);
 #endif
 }
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 16cdebb791..062dbbe2a6 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -247,7 +247,7 @@ typedef struct erts_proc_lock_t_ {
 #define erts_smp_proc_lock(P,L) erts_smp_proc_lock_x(P,L,__FILE__,__LINE__)
 #endif
 
-#if defined(ERTS_SMP) && defined (ERTS_ENABLE_LOCK_COUNT)
+#if defined (ERTS_ENABLE_LOCK_COUNT)
 
 void erts_lcnt_proc_lock_init(Process *p);
 void erts_lcnt_proc_lock_destroy(Process *p);
@@ -421,7 +421,7 @@ void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res
 
 /* --- Process lock checking ----------------------------------------------- */
 
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
 #define ERTS_SMP_CHK_NO_PROC_LOCKS \
   erts_proc_lc_chk_no_proc_locks(__FILE__, __LINE__)
 #define ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(P) \
@@ -969,7 +969,7 @@ erts_smp_proc_lock_x(Process *p, ErtsProcLocks locks, char *file, unsigned int l
 erts_smp_proc_lock(Process *p, ErtsProcLocks locks)
 #endif 
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
+#if defined(ERTS_ENABLE_LOCK_POSITION)
     erts_smp_proc_lock_x__(p,
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
 			 NULL,
@@ -985,7 +985,7 @@ erts_smp_proc_lock(Process *p, ErtsProcLocks locks)
 			 ERTS_PID2PIXLOCK(p->common.id),
 #endif /*ERTS_PROC_LOCK_ATOMIC_IMPL*/
 			 locks);
-#endif /*ERTS_SMP*/
+#endif /*ERTS_ENABLE_LOCK_POSITION*/
 }
 
 ERTS_GLB_INLINE void
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index c1f0fa18cc..dabf8702c8 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -643,7 +643,7 @@ erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
 erts_smp_mtx_trylock(erts_smp_mtx_t *mtx)
 #endif
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
+#if defined(ERTS_ENABLE_LOCK_POSITION)
     return erts_mtx_trylock_x(mtx,file,line);
 #else
     return erts_mtx_trylock(mtx);
@@ -659,7 +659,7 @@ erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
 erts_smp_mtx_lock(erts_smp_mtx_t *mtx)
 #endif
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
+#if defined(ERTS_ENABLE_LOCK_POSITION)
     erts_mtx_lock_x(mtx, file, line);
 #else
     erts_mtx_lock(mtx);
@@ -675,7 +675,7 @@ erts_smp_mtx_unlock(erts_smp_mtx_t *mtx)
 ERTS_GLB_INLINE int
 erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx)
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     return erts_lc_mtx_is_locked(mtx);
 #else
     return 0;
@@ -761,7 +761,7 @@ erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line
 erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx)
 #endif
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
+#if defined(ERTS_ENABLE_LOCK_POSITION)
     return erts_rwmtx_tryrlock_x(rwmtx, file, line);
 #else
     return erts_rwmtx_tryrlock(rwmtx);
@@ -775,7 +775,7 @@ erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
 erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx)
 #endif
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
+#if defined(ERTS_ENABLE_LOCK_POSITION)
     erts_rwmtx_rlock_x(rwmtx, file, line);
 #else
     erts_rwmtx_rlock(rwmtx);
@@ -796,7 +796,7 @@ erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int lin
 erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx)
 #endif
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
+#if defined(ERTS_ENABLE_LOCK_POSITION)
     return erts_rwmtx_tryrwlock_x(rwmtx, file, line);
 #else
     return erts_rwmtx_tryrwlock(rwmtx);
@@ -810,7 +810,7 @@ erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
 erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx)
 #endif
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
+#if defined(ERTS_ENABLE_LOCK_POSITION)
     erts_rwmtx_rwlock_x(rwmtx, file, line);
 #else
     erts_rwmtx_rwlock(rwmtx);
@@ -852,7 +852,7 @@ erts_smp_rwmtx_wunlock(erts_smp_rwmtx_t *rwmtx)
 ERTS_GLB_INLINE int
 erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx)
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     return erts_lc_rwmtx_is_rlocked(mtx);
 #else
     return 0;
@@ -862,7 +862,7 @@ erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx)
 ERTS_GLB_INLINE int
 erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx)
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     return erts_lc_rwmtx_is_rwlocked(mtx);
 #else
     return 0;
@@ -894,7 +894,7 @@ erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line)
 erts_smp_spin_lock(erts_smp_spinlock_t *lock)
 #endif
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
+#if defined(ERTS_ENABLE_LOCK_POSITION)
     erts_spin_lock_x(lock, file, line);
 #else
     erts_spin_lock(lock);
@@ -904,7 +904,7 @@ erts_smp_spin_lock(erts_smp_spinlock_t *lock)
 ERTS_GLB_INLINE int
 erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock)
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     return erts_lc_spinlock_is_locked(lock);
 #else
     return 0;
@@ -936,7 +936,7 @@ erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
 erts_smp_read_lock(erts_smp_rwlock_t *lock)
 #endif 
 {
-#if defined(ERTS_ENABLE_LOCK_POSITION) && defined(ERTS_SMP)
+#if defined(ERTS_ENABLE_LOCK_POSITION)
     erts_read_lock_x(lock, file, line);
 #else
     erts_read_lock(lock);
@@ -956,7 +956,7 @@ erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
 erts_smp_write_lock(erts_smp_rwlock_t *lock)
 #endif 
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
+#if defined(ERTS_ENABLE_LOCK_POSITION)
     erts_write_lock_x(lock, file, line);
 #else
     erts_write_lock(lock);
@@ -966,7 +966,7 @@ erts_smp_write_lock(erts_smp_rwlock_t *lock)
 ERTS_GLB_INLINE int
 erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock)
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     return erts_lc_rwlock_is_rlocked(lock);
 #else
     return 0;
@@ -976,7 +976,7 @@ erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock)
 ERTS_GLB_INLINE int
 erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock)
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     return erts_lc_rwlock_is_rwlocked(lock);
 #else
     return 0;
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 55e8254de2..22a359554e 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -1829,7 +1829,7 @@ erts_mtx_unlock(erts_mtx_t *mtx)
 ERTS_GLB_INLINE int
 erts_lc_mtx_is_locked(erts_mtx_t *mtx)
 {
-#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     int res;
     erts_lc_lock_t lc = mtx->lc;
     lc.flags = 0;
@@ -2144,7 +2144,7 @@ erts_rwmtx_wunlock(erts_rwmtx_t *rwmtx)
 ERTS_GLB_INLINE int
 erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx)
 {
-#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     int res;
     erts_lc_lock_t lc = mtx->lc;
     lc.flags = ERTS_LC_FLG_LO_READ;
@@ -2158,7 +2158,7 @@ erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx)
 ERTS_GLB_INLINE int
 erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx)
 {
-#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     int res;
     erts_lc_lock_t lc = mtx->lc;
     lc.flags = ERTS_LC_FLG_LO_READ|ERTS_LC_FLG_LO_WRITE;
@@ -2169,328 +2169,6 @@ erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx)
 #endif
 }
 
-/* No atomic ops */
-
-ERTS_GLB_INLINE void
-erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val)
-{
-    var->sint[0] = val->sint[0];
-    var->sint[1] = val->sint[1];
-}
-
-ERTS_GLB_INLINE void
-erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val)
-{
-    val->sint[0] = var->sint[0];
-    val->sint[1] = var->sint[1];
-}
-
-ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var,
-					      erts_no_dw_atomic_t *new_val,
-					      erts_no_dw_atomic_t *old_val)
-{
-    if (var->sint[0] != old_val->sint[0] || var->sint[1] != old_val->sint[1]) {
-	erts_no_dw_atomic_read(var, old_val);
-	return 0;
-    }
-    else {
-	erts_no_dw_atomic_set(var, new_val);
-	return !0;
-    }
-}
-
-ERTS_GLB_INLINE void
-erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i)
-{
-    *var = i;
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_no_atomic_read(erts_no_atomic_t *var)
-{
-    return *var;
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_no_atomic_inc_read(erts_no_atomic_t *incp)
-{
-    return ++(*incp);
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_no_atomic_dec_read(erts_no_atomic_t *decp)
-{
-    return --(*decp);
-}
-
-ERTS_GLB_INLINE void
-erts_no_atomic_inc(erts_no_atomic_t *incp)
-{
-    ++(*incp);
-}
-
-ERTS_GLB_INLINE void
-erts_no_atomic_dec(erts_no_atomic_t *decp)
-{
-    --(*decp);
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_no_atomic_add_read(erts_no_atomic_t *addp, erts_aint_t i)
-{
-    return *addp += i;
-}
-
-ERTS_GLB_INLINE void
-erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i)
-{
-    *addp += i;
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_no_atomic_read_bor(erts_no_atomic_t *var, erts_aint_t mask)
-{
-    erts_aint_t old;
-    old = *var;
-    *var |= mask;
-    return old;
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_no_atomic_read_band(erts_no_atomic_t *var, erts_aint_t mask)
-{
-    erts_aint_t old;
-    old = *var;
-    *var &= mask;
-    return old;
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_no_atomic_xchg(erts_no_atomic_t *xchgp, erts_aint_t new)
-{
-    erts_aint_t old = *xchgp;
-    *xchgp = new;
-    return old;
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
-		       erts_aint_t new,
-		       erts_aint_t expected)
-{
-    erts_aint_t old = *xchgp;
-    if (old == expected)
-        *xchgp = new;
-    return old;
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_no_atomic_read_bset(erts_no_atomic_t *var,
-			 erts_aint_t mask,
-			 erts_aint_t set)
-{
-    erts_aint_t old = *var;
-    *var &= ~mask;
-    *var |= (mask & set);
-    return old;
-}
-
-/* atomic32 */
-
-ERTS_GLB_INLINE void
-erts_no_atomic32_set(erts_no_atomic32_t *var, erts_aint32_t i)
-{
-    *var = i;
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_no_atomic32_read(erts_no_atomic32_t *var)
-{
-    return *var;
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_no_atomic32_inc_read(erts_no_atomic32_t *incp)
-{
-    return ++(*incp);
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_no_atomic32_dec_read(erts_no_atomic32_t *decp)
-{
-    return --(*decp);
-}
-
-ERTS_GLB_INLINE void
-erts_no_atomic32_inc(erts_no_atomic32_t *incp)
-{
-    ++(*incp);
-}
-
-ERTS_GLB_INLINE void
-erts_no_atomic32_dec(erts_no_atomic32_t *decp)
-{
-    --(*decp);
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_no_atomic32_add_read(erts_no_atomic32_t *addp, erts_aint32_t i)
-{
-    return *addp += i;
-}
-
-ERTS_GLB_INLINE void
-erts_no_atomic32_add(erts_no_atomic32_t *addp, erts_aint32_t i)
-{
-    *addp += i;
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_no_atomic32_read_bor(erts_no_atomic32_t *var, erts_aint32_t mask)
-{
-    erts_aint32_t old;
-    old = *var;
-    *var |= mask;
-    return old;
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_no_atomic32_read_band(erts_no_atomic32_t *var, erts_aint32_t mask)
-{
-    erts_aint32_t old;
-    old = *var;
-    *var &= mask;
-    return old;
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp, erts_aint32_t new)
-{
-    erts_aint32_t old = *xchgp;
-    *xchgp = new;
-    return old;
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
-			 erts_aint32_t new,
-			 erts_aint32_t expected)
-{
-    erts_aint32_t old = *xchgp;
-    if (old == expected)
-        *xchgp = new;
-    return old;
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_no_atomic32_read_bset(erts_no_atomic32_t *var,
-			   erts_aint32_t mask,
-			   erts_aint32_t set)
-{
-    erts_aint32_t old = *var;
-    *var &= ~mask;
-    *var |= (mask & set);
-    return old;
-}
-
-/* atomic64 */
-
-ERTS_GLB_INLINE void
-erts_no_atomic64_set(erts_no_atomic64_t *var, erts_aint64_t i)
-{
-    *var = i;
-}
-
-ERTS_GLB_INLINE erts_aint64_t
-erts_no_atomic64_read(erts_no_atomic64_t *var)
-{
-    return *var;
-}
-
-ERTS_GLB_INLINE erts_aint64_t
-erts_no_atomic64_inc_read(erts_no_atomic64_t *incp)
-{
-    return ++(*incp);
-}
-
-ERTS_GLB_INLINE erts_aint64_t
-erts_no_atomic64_dec_read(erts_no_atomic64_t *decp)
-{
-    return --(*decp);
-}
-
-ERTS_GLB_INLINE void
-erts_no_atomic64_inc(erts_no_atomic64_t *incp)
-{
-    ++(*incp);
-}
-
-ERTS_GLB_INLINE void
-erts_no_atomic64_dec(erts_no_atomic64_t *decp)
-{
-    --(*decp);
-}
-
-ERTS_GLB_INLINE erts_aint64_t
-erts_no_atomic64_add_read(erts_no_atomic64_t *addp, erts_aint64_t i)
-{
-    return *addp += i;
-}
-
-ERTS_GLB_INLINE void
-erts_no_atomic64_add(erts_no_atomic64_t *addp, erts_aint64_t i)
-{
-    *addp += i;
-}
-
-ERTS_GLB_INLINE erts_aint64_t
-erts_no_atomic64_read_bor(erts_no_atomic64_t *var, erts_aint64_t mask)
-{
-    erts_aint64_t old;
-    old = *var;
-    *var |= mask;
-    return old;
-}
-
-ERTS_GLB_INLINE erts_aint64_t
-erts_no_atomic64_read_band(erts_no_atomic64_t *var, erts_aint64_t mask)
-{
-    erts_aint64_t old;
-    old = *var;
-    *var &= mask;
-    return old;
-}
-
-ERTS_GLB_INLINE erts_aint64_t
-erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, erts_aint64_t new)
-{
-    erts_aint64_t old = *xchgp;
-    *xchgp = new;
-    return old;
-}
-
-ERTS_GLB_INLINE erts_aint64_t
-erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp,
-			 erts_aint64_t new,
-			 erts_aint64_t expected)
-{
-    erts_aint64_t old = *xchgp;
-    if (old == expected)
-        *xchgp = new;
-    return old;
-}
-
-ERTS_GLB_INLINE erts_aint64_t
-erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
-			   erts_aint64_t mask,
-			   erts_aint64_t set)
-{
-    erts_aint64_t old = *var;
-    *var &= ~mask;
-    *var |= (mask & set);
-    return old;
-}
-
 /* spinlock */
 
 ERTS_GLB_INLINE void
@@ -2579,7 +2257,7 @@ erts_spin_lock(erts_spinlock_t *lock)
 ERTS_GLB_INLINE int
 erts_lc_spinlock_is_locked(erts_spinlock_t *lock)
 {
-#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     int res;
     erts_lc_lock_t lc = lock->lc;
     lc.flags = 0;
@@ -2713,7 +2391,7 @@ erts_write_lock(erts_rwlock_t *lock)
 ERTS_GLB_INLINE int
 erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock)
 {
-#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     int res;
     erts_lc_lock_t lc = lock->lc;
     lc.flags = ERTS_LC_FLG_LO_READ;
@@ -2727,7 +2405,7 @@ erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock)
 ERTS_GLB_INLINE int
 erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock)
 {
-#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     int res;
     erts_lc_lock_t lc = lock->lc;
     lc.flags = ERTS_LC_FLG_LO_READ|ERTS_LC_FLG_LO_WRITE;
@@ -2896,37 +2574,3 @@ erts_thr_sigwait(const sigset_t *set, int *sig)
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 #endif /* #ifndef ERL_THREAD_H__ */
-
-#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS
-
-/* Deprecated functions to replace */
-
-#undef erts_atomic_init
-#undef erts_atomic_set
-#undef erts_atomic_read
-#undef erts_atomic_inctest
-#undef erts_atomic_dectest
-#undef erts_atomic_inc
-#undef erts_atomic_dec
-#undef erts_atomic_addtest
-#undef erts_atomic_add
-#undef erts_atomic_xchg
-#undef erts_atomic_cmpxchg
-#undef erts_atomic_bor
-#undef erts_atomic_band
-
-#undef erts_atomic32_init
-#undef erts_atomic32_set
-#undef erts_atomic32_read
-#undef erts_atomic32_inctest
-#undef erts_atomic32_dectest
-#undef erts_atomic32_inc
-#undef erts_atomic32_dec
-#undef erts_atomic32_addtest
-#undef erts_atomic32_add
-#undef erts_atomic32_xchg
-#undef erts_atomic32_cmpxchg
-#undef erts_atomic32_bor
-#undef erts_atomic32_band
-
-#endif
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index efb3de2942..069cc27d1b 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -2703,7 +2703,7 @@ send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p,
                    Eterm t_p_id, ErtsTracerNif *tnif, enum ErtsTracerOpt topt,
                    Eterm tag, Eterm msg, Eterm extra, Eterm pam_result)
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     if (c_p) {
         /* We have to hold the main lock of the currently executing process */
         erts_proc_lc_chk_have_proc_locks(c_p, ERTS_PROC_LOCK_MAIN);
@@ -2756,7 +2756,7 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks,
                   enum ErtsTracerOpt topt, Eterm tag) {
     Eterm nif_result;
 
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     if (c_p)
         ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == c_p_locks
                            || erts_thr_progress_is_blocking());
@@ -2841,7 +2841,7 @@ int erts_is_tracer_proc_enabled_send(Process* c_p, ErtsProcLocks c_p_locks,
 
 void erts_tracer_replace(ErtsPTabElementCommon *t_p, const ErtsTracer tracer)
 {
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     if (is_internal_pid(t_p->id) && !erts_thr_progress_is_blocking()) {
         erts_proc_lc_chk_have_proc_locks((Process*)t_p, ERTS_PROC_LOCKS_ALL);
     } else if (is_internal_port(t_p->id)) {
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index a598709984..97054b2ee4 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1174,7 +1174,7 @@ void erts_emergency_close_ports(void);
 void erts_ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon);
 Eterm erts_driver_monitor_to_ref(Eterm* hp, const ErlDrvMonitor *mon);
 
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT)
+#if defined(ERTS_ENABLE_LOCK_COUNT)
 void erts_lcnt_update_driver_locks(int enable);
 void erts_lcnt_update_port_locks(int enable);
 #endif
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index a65ff03fd8..cc7c717e6d 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -3380,7 +3380,7 @@ void erts_init_io(int port_tab_size,
     erts_smp_rwmtx_rwunlock(&erts_driver_list_lock);
 }
 
-#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP)
+#if defined(ERTS_ENABLE_LOCK_COUNT)
 static void lcnt_enable_driver_lock_count(erts_driver_t *dp, int enable)
 {
     if (dp->lock) {
@@ -3469,7 +3469,8 @@ void erts_lcnt_update_port_locks(int enable) {
     }
 }
 
-#endif /* defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) */
+#endif /* defined(ERTS_ENABLE_LOCK_COUNT) */
+
 /*
  * Buffering of data when using line oriented I/O on ports
  */
@@ -3651,9 +3652,7 @@ deliver_result(Port *prt, Eterm sender, Eterm pid, Eterm res)
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
     ASSERT(!prt || prt->common.id == sender);
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
-    ASSERT(!prt || erts_lc_is_port_locked(prt));
-#endif
+    ERTS_LC_ASSERT(!prt || erts_lc_is_port_locked(prt));
 
     ASSERT(is_internal_port(sender) && is_internal_pid(pid));
 
@@ -7604,7 +7603,7 @@ int driver_monitor_process(ErlDrvPort drvport,
 {
     Port *prt;
     int ret;
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     ErtsSchedulerData *sched = erts_get_scheduler_data();
 #endif
 
@@ -7662,7 +7661,7 @@ int driver_demonitor_process(ErlDrvPort drvport,
 {
     Port *prt;
     int ret;
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     ErtsSchedulerData *sched = erts_get_scheduler_data();
 #endif
 
@@ -7703,7 +7702,7 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport,
 {
     Port *prt;
     ErlDrvTermData ret;
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     ErtsSchedulerData *sched = erts_get_scheduler_data();
 #endif
 
@@ -8066,12 +8065,8 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size)
 	sip->driver_minor_version = ERL_DRV_EXTENDED_MINOR_VERSION;
 	sip->erts_version = ERLANG_VERSION;
 	sip->otp_release = ERLANG_OTP_RELEASE;
-	sip->thread_support = 
-	    1
-	    ;
-	sip->smp_support = 
-	    1
-	    ;
+	sip->thread_support = 1;
+	sip->smp_support = 1;
 
     }
 
diff --git a/erts/emulator/hipe/hipe_amd64_bifs.m4 b/erts/emulator/hipe/hipe_amd64_bifs.m4
index dca3887564..b3c9a460bb 100644
--- a/erts/emulator/hipe/hipe_amd64_bifs.m4
+++ b/erts/emulator/hipe/hipe_amd64_bifs.m4
@@ -39,7 +39,7 @@ define(HANDLE_GOT_MBUF,`
 3:	call nbif_$1_gc_after_bif	/* `HANDLE_GOT_MBUF' */
 	jmp 2b')
 
-`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+`#if defined(ERTS_ENABLE_LOCK_CHECK)
 #  define CALL_BIF(F) \
 		movq CSYM(nbif_impl_##F)@GOTPCREL(%rip), %r11; \
 		movq %r11, P_BIF_CALLEE(P); \
diff --git a/erts/emulator/hipe/hipe_arm_bifs.m4 b/erts/emulator/hipe/hipe_arm_bifs.m4
index a9097dabde..554faa2567 100644
--- a/erts/emulator/hipe/hipe_arm_bifs.m4
+++ b/erts/emulator/hipe/hipe_arm_bifs.m4
@@ -29,7 +29,7 @@ include(`hipe/hipe_arm_asm.m4')
 	.p2align 2
 	.arm
 
-`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+`#if defined(ERTS_ENABLE_LOCK_CHECK)
 #  define CALL_BIF(F)	ldr r14, =nbif_impl_##F; str r14, [r0, #P_BIF_CALLEE]; bl hipe_debug_bif_wrapper
 #else
 #  define CALL_BIF(F)	bl	nbif_impl_##F
diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c
index e04d3d32d1..9cac733db5 100644
--- a/erts/emulator/hipe/hipe_bif2.c
+++ b/erts/emulator/hipe/hipe_bif2.c
@@ -153,7 +153,7 @@ BIF_RETTYPE hipe_bifs_modeswitch_debug_off_0(BIF_ALIST_0)
     BIF_RET(am_true);
 }
 
-#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
 
 BIF_RETTYPE hipe_debug_bif_wrapper(NBIF_ALIST_1);
 
@@ -174,7 +174,7 @@ BIF_RETTYPE hipe_debug_bif_wrapper(NBIF_ALIST_1)
     return res;
 }
 
-#endif /* ERTS_ENABLE_LOCK_CHECK && ERTS_SMP */
+#endif /* ERTS_ENABLE_LOCK_CHECK*/
 
 
 BIF_RETTYPE hipe_bifs_debug_native_called_2(BIF_ALIST_2)
diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c
index 2e9d29eede..6ea120c65c 100644
--- a/erts/emulator/hipe/hipe_mkliterals.c
+++ b/erts/emulator/hipe/hipe_mkliterals.c
@@ -507,7 +507,7 @@ static const struct rts_param rts_params[] = {
 #endif
     },
     { 48, "P_BIF_CALLEE",
-#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
 	1, offsetof(struct process, hipe.bif_callee)
 #endif
     },
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index 5775f0730f..fd90c46fa8 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -36,7 +36,7 @@
 #include "hipe_stack.h"
 #include "hipe_bif0.h"	/* hipe_mfa_info_table_init() */
 
-#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
 #    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
         if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,	\
 					   __FILE__, __LINE__)
@@ -633,6 +633,28 @@ static unsigned hipe_next_nstack_size(unsigned size)
     return size ? size * 2 : HIPE_INITIAL_NSTACK_SIZE;
 }
 
+#if 0 && defined(HIPE_NSTACK_GROWS_UP)
+void hipe_inc_nstack(Process *p)
+{
+    Eterm *old_nstack = p->hipe.nstack;
+    unsigned old_size = p->hipe.nstend - old_nstack;
+    unsigned new_size = hipe_next_nstack_size(old_size);
+    Eterm *new_nstack = erts_realloc(ERTS_ALC_T_HIPE,
+				     (char *) old_nstack,
+				     new_size*sizeof(Eterm));
+    p->hipe.nstend = new_nstack + new_size;
+    if (new_nstack != old_nstack) {
+	p->hipe.nsp = new_nstack + (p->hipe.nsp - old_nstack);
+	p->hipe.nstack = new_nstack;
+	if (p->hipe.nstgraylim)
+	    p->hipe.nstgraylim =
+		new_nstack + (p->hipe.nstgraylim - old_nstack);
+	if (p->hipe.nstblacklim)
+	    p->hipe.nstblacklim =
+		new_nstack + (p->hipe.nstblacklim - old_nstack);
+    }
+}
+#endif
 
 #if defined(HIPE_NSTACK_GROWS_DOWN)
 void hipe_inc_nstack(Process *p)
diff --git a/erts/emulator/hipe/hipe_ppc_bifs.m4 b/erts/emulator/hipe/hipe_ppc_bifs.m4
index 79a8bef77d..283fbbb200 100644
--- a/erts/emulator/hipe/hipe_ppc_bifs.m4
+++ b/erts/emulator/hipe/hipe_ppc_bifs.m4
@@ -25,7 +25,7 @@ include(`hipe/hipe_ppc_asm.m4')
 #`include' "config.h"
 #`include' "hipe_literals.h"
 
-`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+`#if defined(ERTS_ENABLE_LOCK_CHECK)
 #  define CALL_BIF(F)	STORE_IA(CSYM(nbif_impl_##F), P_BIF_CALLEE(P), r29); bl CSYM(hipe_debug_bif_wrapper) 
 #else
 #  define CALL_BIF(F)	bl	CSYM(nbif_impl_##F)
diff --git a/erts/emulator/hipe/hipe_process.h b/erts/emulator/hipe/hipe_process.h
index 2cc7fd5f4e..ef14c75f6c 100644
--- a/erts/emulator/hipe/hipe_process.h
+++ b/erts/emulator/hipe/hipe_process.h
@@ -49,7 +49,7 @@ struct hipe_process_state {
 #ifdef NO_FPE_SIGNALS
     double float_result;        /* to be checked for inf/NaN by hipe_emulate_fpe */ 
 #endif
-#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     void (*bif_callee)(void);   /* When calling BIF's via debug wrapper */
 #endif
 #ifdef DEBUG
diff --git a/erts/emulator/hipe/hipe_signal.h b/erts/emulator/hipe/hipe_signal.h
index 5d8621135b..524def11a4 100644
--- a/erts/emulator/hipe/hipe_signal.h
+++ b/erts/emulator/hipe/hipe_signal.h
@@ -27,13 +27,9 @@
 
 #if defined(__i386__) || defined(__x86_64__)
 extern void hipe_signal_init(void);
-#else
-static __inline__ void hipe_signal_init(void) { }
-#endif
-
-#if defined(ERTS_SMP) && (defined(__i386__) || defined(__x86_64__))
 extern void hipe_thread_signal_init(void);
 #else
+static __inline__ void hipe_signal_init(void) { }
 static __inline__ void hipe_thread_signal_init(void) { }
 #endif
 
diff --git a/erts/emulator/hipe/hipe_sparc_bifs.m4 b/erts/emulator/hipe/hipe_sparc_bifs.m4
index 14330c2f1c..1b49fa57fd 100644
--- a/erts/emulator/hipe/hipe_sparc_bifs.m4
+++ b/erts/emulator/hipe/hipe_sparc_bifs.m4
@@ -28,7 +28,7 @@ include(`hipe/hipe_sparc_asm.m4')
 	.section ".text"
 	.align	4
 
-`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+`#if defined(ERTS_ENABLE_LOCK_CHECK)
 #  define CALL_BIF(F)	set nbif_impl_##F, %o7; st %o7, [%o0+P_BIF_CALLEE]; call hipe_debug_bif_wrapper 
 #else
 #  define CALL_BIF(F)	call	nbif_impl_##F
diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h
index 0d3019cfd0..7e30358767 100644
--- a/erts/emulator/hipe/hipe_stack.h
+++ b/erts/emulator/hipe/hipe_stack.h
@@ -110,6 +110,11 @@ extern void (*hipe_handle_stack_trap(Process*))(void);
 extern void hipe_update_stack_trap(Process*, const struct hipe_sdesc*);
 extern int hipe_fill_stacktrace(Process*, int, Eterm**);
 
+#if 0 && defined(HIPE_NSTACK_GROWS_UP)
+#define hipe_nstack_start(p)	((p)->hipe.nstack)
+#define hipe_nstack_used(p)	((p)->hipe.nsp - (p)->hipe.nstack)
+#define hipe_nstack_avail(p)	((p)->hipe.nstend - (p)->hipe.nsp)
+#endif
 #if defined(HIPE_NSTACK_GROWS_DOWN)
 #define hipe_nstack_start(p)	((p)->hipe.nsp)
 #define hipe_nstack_used(p)	((p)->hipe.nstend - (p)->hipe.nsp)
diff --git a/erts/emulator/hipe/hipe_x86_bifs.m4 b/erts/emulator/hipe/hipe_x86_bifs.m4
index aecf67dc1b..9cb343d067 100644
--- a/erts/emulator/hipe/hipe_x86_bifs.m4
+++ b/erts/emulator/hipe/hipe_x86_bifs.m4
@@ -31,7 +31,7 @@ include(`hipe/hipe_x86_asm.m4')
 #define TEST_GOT_EXN	cmpl	$THE_NON_VALUE,%eax
 #endif'
 
-`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+`#if defined(ERTS_ENABLE_LOCK_CHECK)
 #  define CALL_BIF(F)	movl $CSYM(nbif_impl_##F), P_BIF_CALLEE(P); call CSYM(hipe_debug_bif_wrapper) 
 #else
 #  define CALL_BIF(F)	call	CSYM(nbif_impl_##F)
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index ce9f6e3f06..3e029303a7 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -3042,7 +3042,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
 
     erts_printf("--- fds in pollset --------------------------------------\n");
 
-#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+#if defined(ERTS_ENABLE_LOCK_CHECK)
     erts_lc_check_exact(NULL, 0); /* No locks should be locked */
 #endif
 
diff --git a/erts/emulator/sys/common/erl_os_monotonic_time_extender.h b/erts/emulator/sys/common/erl_os_monotonic_time_extender.h
index 2266f26e24..53c32579d5 100644
--- a/erts/emulator/sys/common/erl_os_monotonic_time_extender.h
+++ b/erts/emulator/sys/common/erl_os_monotonic_time_extender.h
@@ -30,7 +30,6 @@ typedef struct {
     int check_interval;
 } ErtsOsMonotonicTimeExtendState;
 
-#  define ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(S, RT) ((void) 1)
 #  define ERTS_EXTEND_OS_MONOTONIC_TIME(S, RT)				\
     ((((ErtsMonotonicTime)						\
        erts_atomic32_read_nob(&((S)->extend[((int) ((RT) >> 31)) & 1]))) \
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index 3a6ae85b42..4171770f02 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -144,20 +144,11 @@ int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds,
 
 #define ERTS_POLL_USE_BATCH_UPDATE_POLLSET (ERTS_POLL_USE_DEVPOLL \
 					    || ERTS_POLL_USE_KQUEUE)
-#define ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE \
-   (defined(ERTS_SMP) || ERTS_POLL_USE_KERNEL_POLL || ERTS_POLL_USE_POLL)
 
-#define ERTS_POLL_USE_CONCURRENT_UPDATE \
-   (defined(ERTS_SMP) && ERTS_POLL_USE_EPOLL)
+#define ERTS_POLL_USE_CONCURRENT_UPDATE ERTS_POLL_USE_EPOLL
 
 #define ERTS_POLL_COALESCE_KP_RES (ERTS_POLL_USE_KQUEUE || ERTS_POLL_USE_EPOLL)
 
-#  define ERTS_POLL_ASYNC_INTERRUPT_SUPPORT 0
-
-#define ERTS_POLL_USE_WAKEUP_PIPE \
-   (ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(USE_THREADS))
-
-
 #define ERTS_POLLSET_LOCK(PS) \
   erts_smp_mtx_lock(&(PS)->mtx)
 #define ERTS_POLLSET_UNLOCK(PS) \
@@ -1818,7 +1809,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res,
 
 #if ERTS_POLL_USE_POLL	/* --- poll -------------------------------- */
 	int res = 0;
-#if ERTS_POLL_USE_WAKEUP_PIPE && !ERTS_POLL_USE_FALLBACK 
+#if !ERTS_POLL_USE_FALLBACK
 	int wake_fd = ps->wake_fds[0];
 #endif
 	int i, first_ix, end_ix;
@@ -1869,7 +1860,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res,
 
 #elif ERTS_POLL_USE_SELECT	/* --- select ------------------------------ */
 	int res = 0;
-#if ERTS_POLL_USE_WAKEUP_PIPE && !ERTS_POLL_USE_FALLBACK 
+#if !ERTS_POLL_USE_FALLBACK
 	int wake_fd = ps->wake_fds[0];
 #endif
 	int fd, first_fd, end_fd;
diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c
index 03d948c6e9..c451a0a674 100644
--- a/erts/emulator/sys/unix/sys_drivers.c
+++ b/erts/emulator/sys/unix/sys_drivers.c
@@ -84,8 +84,6 @@ static Eterm forker_port;
 #define MAXIOV 16
 #endif
 
-#  define FDBLOCK 1
-
 /* Used by the fd driver iff the fd could not be set to non-blocking */
 typedef struct ErtsSysBlocking_ {
     ErlDrvPDL pdl;
@@ -423,7 +421,7 @@ create_driver_data(ErlDrvPort port_num,
             data += sizeof(*driver_data->ofd);
             init_fd_data(driver_data->ofd, ofd);
         }
-        if (is_blocking && FDBLOCK)
+        if (is_blocking)
             if (!set_blocking_data(driver_data)) {
                 erts_free(ERTS_ALC_T_DRV_TAB, driver_data);
                 return NULL;
@@ -1171,19 +1169,19 @@ static void outputv(ErlDrvData e, ErlIOVec* ev)
     ev->iov[0].iov_len = pb;
     ev->size += pb;
 
-    if (dd->blocking && FDBLOCK)
+    if (dd->blocking)
         driver_pdl_lock(dd->blocking->pdl);
 
     if ((sz = driver_sizeq(ix)) > 0) {
 	driver_enqv(ix, ev, 0);
 
-        if (dd->blocking && FDBLOCK)
+        if (dd->blocking)
             driver_pdl_unlock(dd->blocking->pdl);
 
 	if (sz + ev->size >= (1 << 13))
 	    set_busy_port(ix, 1);
     }
-    else if (!dd->blocking || !FDBLOCK) {
+    else if (!dd->blocking) {
         /* We try to write directly if the fd in non-blocking */
 	int vsize = ev->vsize > MAX_VSIZE ? MAX_VSIZE : ev->vsize;
 
@@ -1281,7 +1279,7 @@ static int port_inp_failure(ErtsSysDriverData *dd, int res)
         clear_fd_data(dd->ifd);
     }
 
-    if (dd->blocking && FDBLOCK) {
+    if (dd->blocking) {
         driver_pdl_lock(dd->blocking->pdl);
         if (driver_sizeq(dd->port_num) > 0) {
             driver_pdl_unlock(dd->blocking->pdl);
diff --git a/erts/emulator/sys/unix/sys_time.c b/erts/emulator/sys/unix/sys_time.c
index 102ef7bebf..60afb3b672 100644
--- a/erts/emulator/sys/unix/sys_time.c
+++ b/erts/emulator/sys/unix/sys_time.c
@@ -878,8 +878,6 @@ ErtsMonotonicTime
 erts_os_monotonic_time(void)
 {
     Uint32 ticks = get_tick_count();
-    ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd,
-				      ticks);
     return ERTS_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd,
 					 ticks) << internal_state.r.o.times_shift;
 }
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index 7fddc9fa99..4c06535e4e 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -132,7 +132,7 @@ static OSVERSIONINFO int_os_version;	/* Version information for Win32. */
     Disabled the use of CancelIoEx as its been seen to cause problem with some
     drivers. Not sure what to blame; faulty drivers or some form of invalid use.
 */
-#if defined(ERTS_SMP) && defined(USE_CANCELIOEX)
+#if defined(USE_CANCELIOEX)
 static BOOL (WINAPI *fpCancelIoEx)(HANDLE,LPOVERLAPPED);
 #endif
 
@@ -1165,7 +1165,7 @@ static int
 spawn_init(void)
 {
     int i;
-#if defined(ERTS_SMP) && defined(USE_CANCELIOEX)
+#if defined(USE_CANCELIOEX)
     HMODULE module = GetModuleHandle("kernel32");
     fpCancelIoEx = (BOOL (WINAPI *)(HANDLE,LPOVERLAPPED))
 	((module != NULL) ? GetProcAddress(module,"CancelIoEx") : NULL);
diff --git a/erts/emulator/sys/win32/sys_time.c b/erts/emulator/sys/win32/sys_time.c
index 88131aaa6a..359010e9f1 100644
--- a/erts/emulator/sys/win32/sys_time.c
+++ b/erts/emulator/sys/win32/sys_time.c
@@ -187,8 +187,6 @@ os_monotonic_time_gtc32(void)
 {
     ErtsMonotonicTime mtime;
     Uint32 ticks = (Uint32) GetTickCount();
-    ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd,
-				      ticks);
     mtime = ERTS_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd,
 					  ticks);
     mtime <<= ERTS_GET_TICK_COUNT_TIME_UNIT_SHIFT;
@@ -205,8 +203,6 @@ os_times_gtc32(ErtsMonotonicTime *mtimep, ErtsSystemTime *stimep)
     ticks = (Uint32) GetTickCount();
     GetSystemTime(&st);
 
-    ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd,
-				      ticks);
     mtime = ERTS_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd,
 					  ticks);
     mtime <<= ERTS_GET_TICK_COUNT_TIME_UNIT_SHIFT;
@@ -265,8 +261,6 @@ sys_hrtime_gtc32(void)
 {
     ErtsSysHrTime time;
     Uint32 ticks = (Uint32) GetTickCount();
-    ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd,
-				      tick_count);
     time = (ErtsSysHrTime) ERTS_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd,
 							 ticks);
     time *= (ErtsSysHrTime) (1000 * 1000);
-- 
cgit v1.2.3


From ee297c32a768ec333e2a8a3ef829a7690e91d306 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 13 Jul 2017 10:43:33 +0200
Subject: erts: Cleanup configure and makefiles after non-smp removal

---
 erts/emulator/Makefile.in           | 19 -------------------
 erts/emulator/hipe/hipe_bif_list.m4 | 13 +------------
 2 files changed, 1 insertion(+), 31 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 1ae905276d..88ac3f652c 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -178,27 +178,10 @@ endif
 # NOTE: When adding a new type update ERL_BUILD_TYPE_MARKER in sys/unix/sys.c
 #
 
-FLAVOR=$(DEFAULT_FLAVOR)
-
-ifeq ($(FLAVOR),plain)
-
-DS_SUPPORT=no
-DS_TEST=no
-
-FLAVOR_MARKER=
-FLAVOR_FLAGS=
-ENABLE_ALLOC_TYPE_VARS += nofrag
-M4FLAGS +=
-
-else # FLAVOR
-
-# If flavor isn't one of the above, it *is* smp flavor...
 override FLAVOR=smp
 
 FLAVOR_MARKER=.smp
-FLAVOR_FLAGS=-DERTS_SMP
 ENABLE_ALLOC_TYPE_VARS += smp nofrag
-M4FLAGS += -DERTS_SMP=1
 ifeq ($(DIRTY_SCHEDULER_SUPPORT),yes)
 THR_DEFS += -DERTS_DIRTY_SCHEDULERS
 DS_SUPPORT=yes
@@ -215,8 +198,6 @@ DS_SUPPORT=no
 DS_TEST=no
 endif # DIRTY_SCHEDULER_SUPPORT
 
-endif # FLAVOR
-
 TF_MARKER=$(TYPEMARKER)$(FLAVOR_MARKER)
 
 ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no)
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index f034c4700c..bebe20a18e 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -262,18 +262,12 @@ noproc_primop_interface_2(nbif_bs_get_utf16, erts_bs_get_utf16)
 noproc_primop_interface_2(nbif_bs_validate_unicode_retract, hipe_bs_validate_unicode_retract)
 
 /*
- * Bit-syntax primops. The ERTS_SMP runtime system requires P,
+ * Bit-syntax primops. The runtime system requires P,
  * hence the use of nocons_nofail_primop_interface_N().
- * When ERTS_SMP is disabled, noproc_primop_interface_N()
- * should be used instead.
  */
 nocons_nofail_primop_interface_5(nbif_bs_put_small_float, hipe_bs_put_small_float)
 noproc_primop_interface_5(nbif_bs_put_bits, hipe_bs_put_bits)
-ifelse(ERTS_SMP,1,`
 nocons_nofail_primop_interface_5(nbif_bs_put_big_integer, hipe_bs_put_big_integer)
-',`
-noproc_primop_interface_5(nbif_bs_put_big_integer, hipe_bs_put_big_integer)
-')dnl
 
 nofail_primop_interface_0(nbif_check_get_msg, hipe_check_get_msg)
 
@@ -283,13 +277,8 @@ nocons_nofail_primop_interface_0(nbif_emulate_fpe, hipe_emulate_fpe)
 
 noproc_primop_interface_1(nbif_emasculate_binary, hipe_emasculate_binary)
 
-/*
- * SMP-specific stuff
- */
-ifelse(ERTS_SMP,1,`
 nocons_nofail_primop_interface_0(nbif_clear_timeout, hipe_clear_timeout)
 noproc_primop_interface_1(nbif_atomic_inc, hipe_atomic_inc)
-',)dnl
 
 /*
  * BIFs that disable GC while trapping are called via a wrapper
-- 
cgit v1.2.3


From 0a0d797a4b7609e52ce5dba6fcb19e315588cb9f Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 13 Jul 2017 10:50:46 +0200
Subject: erts: Cleanup alloc.types after non-smp removal

---
 erts/emulator/Makefile.in          |  2 +-
 erts/emulator/beam/erl_alloc.types | 62 --------------------------------------
 2 files changed, 1 insertion(+), 63 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 88ac3f652c..521fc46b47 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -181,7 +181,7 @@ endif
 override FLAVOR=smp
 
 FLAVOR_MARKER=.smp
-ENABLE_ALLOC_TYPE_VARS += smp nofrag
+ENABLE_ALLOC_TYPE_VARS += nofrag
 ifeq ($(DIRTY_SCHEDULER_SUPPORT),yes)
 THR_DEFS += -DERTS_DIRTY_SCHEDULERS
 DS_SUPPORT=yes
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 50a1d97dd5..8142ea8893 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -53,15 +53,6 @@
 #
 # IMPORTANT! Only use 7-bit ascii text in this file!
 
-+if smp
-+disable threads_no_smp
-+else
-+if threads
-+enable threads_no_smp
-+else
-+disable threads_no_smp
-+endif
-+endif
 
 # --- Allocator declarations -------------------------------------------------
 #
@@ -77,8 +68,6 @@
 
 allocator	SYSTEM		true		sys_alloc
 
-+if smp
-
 allocator	TEMPORARY	true		temp_alloc
 allocator	SHORT_LIVED	true		sl_alloc
 allocator	STANDARD	true		std_alloc
@@ -91,22 +80,6 @@ allocator	LITERAL 	true		literal_alloc
 allocator	EXEC     	true	        exec_alloc
 +endif
 
-+else # Non smp build
-
-allocator	TEMPORARY	false		temp_alloc
-allocator	SHORT_LIVED	false		sl_alloc
-allocator	STANDARD	false		std_alloc
-allocator	LONG_LIVED	false		ll_alloc
-allocator	EHEAP		false		eheap_alloc
-allocator	ETS		false		ets_alloc
-allocator	FIXED_SIZE	false		fix_alloc
-allocator	LITERAL 	false		literal_alloc
-+if exec_alloc
-allocator	EXEC     	false	        exec_alloc
-+endif
-
-+endif
-
 allocator	BINARY		true		binary_alloc
 allocator	DRIVER		true		driver_alloc
 
@@ -286,31 +259,16 @@ type	MREF_TAB_BKTS	STANDARD	SYSTEM		magic_ref_table_buckets
 type	MREF_TAB	LONG_LIVED	SYSTEM		magic_ref_table
 type	MINDIRECTION	FIXED_SIZE	SYSTEM		magic_indirection
 
-+if threads_no_smp
-# Need thread safe allocs, but std_alloc and fix_alloc are not;
-# use driver_alloc which is...
-type	THR_Q_EL	DRIVER 	   	SYSTEM		thr_q_element
-type	THR_Q_EL_SL	DRIVER		SYSTEM		sl_thr_q_element
-type	MISC_AUX_WORK	DRIVER		SYSTEM		misc_aux_work
-+else
 type	THR_Q_EL	STANDARD   	SYSTEM		thr_q_element
 type	THR_Q_EL_SL	FIXED_SIZE	SYSTEM		sl_thr_q_element
 type	MISC_AUX_WORK	SHORT_LIVED	SYSTEM		misc_aux_work
-+endif
 type	THR_Q		STANDARD	SYSTEM		thr_queue
 type	THR_Q_SL	SHORT_LIVED	SYSTEM		short_lived_thr_queue
 type	THR_Q_LL	LONG_LIVED	SYSTEM		long_lived_thr_queue
 
-+if smp
 type	ASYNC		SHORT_LIVED	SYSTEM		async
 type	ZLIB		STANDARD	SYSTEM		zlib
-+else
-# sl/std_alloc is not thread safe in non smp build; therefore, we use driver_alloc
-type	ZLIB		DRIVER   	SYSTEM		zlib
-type	ASYNC		DRIVER		SYSTEM		async
-+endif
 
-+if smp
 type	PORT_LOCK	STANDARD	SYSTEM		port_lock
 type	DRIVER_LOCK	STANDARD	SYSTEM		driver_lock
 type	XPORTS_LIST	SHORT_LIVED	SYSTEM		extra_port_list
@@ -320,33 +278,19 @@ type	THR_PRGR_IDATA	LONG_LIVED	SYSTEM		thr_prgr_internal_data
 type	THR_PRGR_DATA	LONG_LIVED	SYSTEM		thr_prgr_data
 type	T_THR_PRGR_DATA	SHORT_LIVED	SYSTEM		temp_thr_prgr_data
 type	RELEASE_LAREA	SHORT_LIVED	SYSTEM		release_literal_area
-+endif
 
 #
 # Types used for special emulators
 #
 
-+if threads
-
 type	ETHR_STD	STANDARD	SYSTEM		ethread_standard
 type	ETHR_SL		SHORT_LIVED	SYSTEM		ethread_short_lived
 type	ETHR_LL		LONG_LIVED	SYSTEM		ethread_long_lived
 
-+endif
-
-+if shared_heap
-
-type	STACK		STANDARD	PROCESSES	stack
-type	ACTIVE_PROCS	STANDARD	PROCESSES	active_procs
-
-+endif
-
-+if smp
 type	SYS_MSG_Q	SHORT_LIVED	PROCESSES	system_messages_queue
 type	FP_EXCEPTION	LONG_LIVED	SYSTEM		fp_exception
 type	LL_MPATHS	LONG_LIVED	SYSTEM		ll_migration_paths
 type	SL_MPATHS	SHORT_LIVED	SYSTEM		sl_migration_paths
-+endif
 
 +if hipe
 
@@ -360,8 +304,6 @@ type	HIPE_EXEC	EXEC            CODE		hipe_code
 
 +endif
 
-
-
 +if heap_frag_elim_test
 
 type	SSB		SHORT_LIVED	PROCESSES	ssb
@@ -431,11 +373,7 @@ type	PUTENV_STR	SYSTEM		SYSTEM		putenv_string
 type	PRT_REP_EXIT	STANDARD	SYSTEM		port_report_exit
 type	SYS_BLOCKING	STANDARD	SYSTEM		sys_blocking
 
-+if smp
 type	SYS_WRITE_BUF	TEMPORARY	SYSTEM		sys_write_buf
-+else
-type	SYS_WRITE_BUF	BINARY		SYSTEM		sys_write_buf
-+endif
 
 +endif
 
-- 
cgit v1.2.3


From 2a84c3d7f6822f1293ad860cf094b549fd3fe634 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 13 Jul 2017 12:28:49 +0200
Subject: Fix testcases after removal of non-smp emulator

---
 erts/emulator/test/alloc_SUITE.erl          |   7 +-
 erts/emulator/test/driver_SUITE.erl         |  81 ++++++++----------
 erts/emulator/test/nif_SUITE.erl            |  11 +--
 erts/emulator/test/port_trace_SUITE.erl     |   7 --
 erts/emulator/test/scheduler_SUITE.erl      |  13 +--
 erts/emulator/test/signal_SUITE.erl         | 123 +++++++++++++---------------
 erts/emulator/test/smoke_test_SUITE.erl     |   6 +-
 erts/emulator/test/statistics_SUITE.erl     |   4 +-
 erts/emulator/test/system_profile_SUITE.erl |   5 +-
 9 files changed, 104 insertions(+), 153 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl
index 3a721095e2..f0871ead7d 100644
--- a/erts/emulator/test/alloc_SUITE.erl
+++ b/erts/emulator/test/alloc_SUITE.erl
@@ -65,12 +65,7 @@ mseg_clear_cache(Cfg) -> drv_case(Cfg).
 cpool(Cfg) -> drv_case(Cfg).
 
 migration(Cfg) ->
-    case erlang:system_info(smp_support) of
-	true ->
-	    drv_case(Cfg, concurrent, "+MZe true");
-	false ->
-	    {skipped, "No smp"}
-    end.
+    drv_case(Cfg, concurrent, "+MZe true").
 
 erts_mmap(Config) when is_list(Config) ->
     case {os:type(), mmsc_flags()} of
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 6810729285..5fca191679 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -1144,8 +1144,6 @@ check_si_res(["thread", "false"]) ->
     false = erlang:system_info(threads);
 check_si_res(["smp", "true"]) ->
     true = erlang:system_info(smp_support);
-check_si_res(["smp", "false"]) ->
-    false = erlang:system_info(smp_support);
 
 %% Data added in second version of driver_system_info() (driver version 1.1)
 check_si_res(["async_thrs", Value]) ->
@@ -1944,44 +1942,39 @@ thr_msg_blast_receiver_proc(Port, Max, Parent, Done) ->
     end.
 
 thr_msg_blast(Config) when is_list(Config) ->
-    case erlang:system_info(smp_support) of
-        false ->
-            {skipped, "Non-SMP emulator; nothing to test..."};
-        true ->
-            Path = proplists:get_value(data_dir, Config),
-            erl_ddll:start(),
-            ok = load_driver(Path, thr_msg_blast_drv),
-            MemBefore = driver_alloc_size(),
-            Start = os:timestamp(),
-            Port = open_port({spawn, thr_msg_blast_drv}, []),
-            true = is_port(Port),
-            Done = make_ref(),
-            Me = self(),
-            spawn(fun () ->
-                          thr_msg_blast_receiver_proc(Port, 1, Me, Done)
-                  end),
-            receive
-                Done -> ok
-            end,
-            ok = thr_msg_blast_receiver(Port, 0, 32*10000),
-            port_close(Port),
-            End = os:timestamp(),
-            receive
-                Garbage ->
-                    ct:fail({received_garbage, Port, Garbage})
-            after 2000 ->
-                      ok
-            end,
-            MemAfter = driver_alloc_size(),
-            io:format("MemBefore=~p, MemAfter=~p~n",
-                      [MemBefore, MemAfter]),
-            ThrMsgBlastTime = timer:now_diff(End,Start)/1000000,
-            io:format("ThrMsgBlastTime=~p~n", [ThrMsgBlastTime]),
-            MemBefore = MemAfter,
-            Res = {thr_msg_blast_time, ThrMsgBlastTime},
-            erlang:display(Res),
-            Res
-    end.
+    Path = proplists:get_value(data_dir, Config),
+    erl_ddll:start(),
+    ok = load_driver(Path, thr_msg_blast_drv),
+    MemBefore = driver_alloc_size(),
+    Start = os:timestamp(),
+    Port = open_port({spawn, thr_msg_blast_drv}, []),
+    true = is_port(Port),
+    Done = make_ref(),
+    Me = self(),
+    spawn(fun () ->
+                  thr_msg_blast_receiver_proc(Port, 1, Me, Done)
+          end),
+    receive
+        Done -> ok
+    end,
+    ok = thr_msg_blast_receiver(Port, 0, 32*10000),
+    port_close(Port),
+    End = os:timestamp(),
+    receive
+        Garbage ->
+            ct:fail({received_garbage, Port, Garbage})
+    after 2000 ->
+            ok
+    end,
+    MemAfter = driver_alloc_size(),
+    io:format("MemBefore=~p, MemAfter=~p~n",
+              [MemBefore, MemAfter]),
+    ThrMsgBlastTime = timer:now_diff(End,Start)/1000000,
+    io:format("ThrMsgBlastTime=~p~n", [ThrMsgBlastTime]),
+    MemBefore = MemAfter,
+    Res = {thr_msg_blast_time, ThrMsgBlastTime},
+    erlang:display(Res),
+    Res.
 
 -define(IN_RANGE(LoW_, VaLuE_, HiGh_),
         case in_range(LoW_, VaLuE_, HiGh_) of
@@ -2488,14 +2481,6 @@ wait_deallocations() ->
     end.
 
 driver_alloc_size() ->
-    case erlang:system_info(smp_support) of
-        true ->
-            ok;
-        false ->
-            %% driver_alloc also used by elements in lock-free queues,
-            %% give these some time to be deallocated...
-            receive after 100 -> ok end
-    end,
     wait_deallocations(),
     case erlang:system_info({allocator_sizes, driver_alloc}) of
         false ->
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 05c250125d..47f9a6e712 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -1721,14 +1721,9 @@ send2(Config) when is_list(Config) ->
 
 %% Send msg from user thread
 send_threaded(Config) when is_list(Config) ->
-    case erlang:system_info(smp_support) of
-	true ->
-	    send2_do1(fun(ME,To) -> send_blob_thread_dbg(ME,To,join) end),
-	    send2_do1(fun(ME,To) -> send_blob_thread_and_join(ME,To) end),
-	    ok;
-	false ->
-	    {skipped,"No threaded send on non-SMP"}
-    end.
+    send2_do1(fun(ME,To) -> send_blob_thread_dbg(ME,To,join) end),
+    send2_do1(fun(ME,To) -> send_blob_thread_and_join(ME,To) end),
+    ok.
 
 
 send2_do1(SendBlobF) ->
diff --git a/erts/emulator/test/port_trace_SUITE.erl b/erts/emulator/test/port_trace_SUITE.erl
index c78dc754a9..a1986397a8 100644
--- a/erts/emulator/test/port_trace_SUITE.erl
+++ b/erts/emulator/test/port_trace_SUITE.erl
@@ -78,13 +78,6 @@ end_per_group(_GroupName, Config) ->
     Config.
 
 
-init_per_testcase(driver_remote_send_term, Config) ->
-    case erlang:system_info(smp_support) of
-        false ->
-            {skip,"Only supported on smp systems"};
-        true ->
-            init_per_testcase(driver_remote_send_term_smp, Config)
-    end;
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     erlang:trace(all, false, [all]),
     os:unsetenv("OUTPUTV"),
diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl
index af33de237c..12e26671c2 100644
--- a/erts/emulator/test/scheduler_SUITE.erl
+++ b/erts/emulator/test/scheduler_SUITE.erl
@@ -1083,7 +1083,6 @@ sbt_test(Config, CpuTCmd, ClBt, Bt, LP) ->
     ok.
 
 scheduler_threads(Config) when is_list(Config) ->
-    SmpSupport = erlang:system_info(smp_support),
     {Sched, SchedOnln, _} = get_sstate(Config, ""),
     %% Configure half the number of both the scheduler threads and
     %% the scheduler threads online.
@@ -1095,10 +1094,7 @@ scheduler_threads(Config) when is_list(Config) ->
     %% setting using +SP to 50% scheduler threads and 25% scheduler
     %% threads online. The result should be 2x scheduler threads and
     %% 1x scheduler threads online.
-    TwiceSched = case SmpSupport of
-                     false -> 1;
-                     true -> Sched*2
-                 end,
+    TwiceSched = Sched*2,
     FourSched = integer_to_list(Sched*4),
     FourSchedOnln = integer_to_list(SchedOnln*4),
     CombinedCmd1 = "+S "++FourSched++":"++FourSchedOnln++" +SP50:25",
@@ -1121,8 +1117,8 @@ scheduler_threads(Config) when is_list(Config) ->
 	    ResetCmd = "+S "++FourSched++":"++FourSchedOnln++" +S 0:0",
 	    {LProc, LProcAvail, _} = get_sstate(Config, ResetCmd),
 	    %% Test negative +S settings, but only for SMP-enabled emulators
-	    case {SmpSupport, LProc > 1, LProcAvail > 1} of
-		{true, true, true} ->
+	    case {LProc > 1, LProcAvail > 1} of
+		{true, true} ->
 		    SchedMinus1 = LProc-1,
 		    SchedOnlnMinus1 = LProcAvail-1,
 		    {SchedMinus1, SchedOnlnMinus1, _} = get_sstate(Config, "+S -1"),
@@ -1157,9 +1153,6 @@ dirty_scheduler_threads_test(Config) ->
     ok.
 
 dirty_schedulers_online_test() ->
-    dirty_schedulers_online_test(erlang:system_info(smp_support)).
-dirty_schedulers_online_test(false) -> ok;
-dirty_schedulers_online_test(true) ->
     dirty_schedulers_online_smp_test(erlang:system_info(schedulers_online)).
 dirty_schedulers_online_smp_test(SchedOnln) when SchedOnln < 4 -> ok;
 dirty_schedulers_online_smp_test(SchedOnln) ->
diff --git a/erts/emulator/test/signal_SUITE.erl b/erts/emulator/test/signal_SUITE.erl
index f1d11d1814..61a8617165 100644
--- a/erts/emulator/test/signal_SUITE.erl
+++ b/erts/emulator/test/signal_SUITE.erl
@@ -139,71 +139,66 @@ pending_exit_gc(Config) when is_list(Config) ->
     pending_exit_test(self(), gc).
 
 pending_exit_test(From, Type) ->
-    case catch erlang:system_info(smp_support) of
-        true ->
-            OTE = process_flag(trap_exit, true),
-            Ref = make_ref(),
-            Master = self(),
-            ExitBySignal = case Type of
-                               gc ->
-                                   lists:duplicate(10000,
-                                                   exit_by_signal);
-                               _ ->
-                                   exit_by_signal
-                           end,
-            Pid = spawn_link(
-                    fun () ->
-                            receive go -> ok end,
-                            false = have_pending_exit(),
-                            exit = fake_exit(From,
-                                             self(),
-                                             ExitBySignal),
-                            true = have_pending_exit(),
-                            Master ! {self(), Ref, Type},
-                            case Type of
-                                gc ->
-                                    force_gc(),
-                                    erlang:yield();
-                                unlink ->
-                                    unlink(From);
-                                trap_exit ->
-                                    process_flag(trap_exit, true);
-                                'receive' ->
-                                    receive _ -> ok
-                                    after 0 -> ok
-                                    end;
-                                exit ->
-                                    ok
-                            end,
-                            exit(exit_by_myself)
-                    end),
-            Mon = erlang:monitor(process, Pid),
-            Pid ! go,
-            Reason = receive
-                         {'DOWN', Mon, process, Pid, R} ->
-                             receive
-                                 {Pid, Ref, Type} ->
-                                     ok
-                             after 0 ->
-                                       ct:fail(premature_exit)
-                             end,
-                             case Type of
-                                 exit ->
-                                     exit_by_myself = R;
-                                 _ ->
-                                     ExitBySignal = R
-                             end
+    OTE = process_flag(trap_exit, true),
+    Ref = make_ref(),
+    Master = self(),
+    ExitBySignal = case Type of
+                       gc ->
+                           lists:duplicate(10000,
+                                           exit_by_signal);
+                       _ ->
+                           exit_by_signal
+                   end,
+    Pid = spawn_link(
+            fun () ->
+                    receive go -> ok end,
+                    false = have_pending_exit(),
+                    exit = fake_exit(From,
+                                     self(),
+                                     ExitBySignal),
+                    true = have_pending_exit(),
+                    Master ! {self(), Ref, Type},
+                    case Type of
+                        gc ->
+                            force_gc(),
+                            erlang:yield();
+                        unlink ->
+                            unlink(From);
+                        trap_exit ->
+                            process_flag(trap_exit, true);
+                        'receive' ->
+                            receive _ -> ok
+                            after 0 -> ok
+                            end;
+                        exit ->
+                            ok
+                    end,
+                    exit(exit_by_myself)
+            end),
+    Mon = erlang:monitor(process, Pid),
+    Pid ! go,
+    Reason = receive
+                 {'DOWN', Mon, process, Pid, R} ->
+                     receive
+                         {Pid, Ref, Type} ->
+                             ok
+                     after 0 ->
+                             ct:fail(premature_exit)
                      end,
-            receive
-                {'EXIT', Pid, R2} ->
-                    Reason = R2
-            end,
-            process_flag(trap_exit, OTE),
-            ok,
-            {comment, "Test only valid with current SMP emulator."};
-        _ ->
-            {skipped, "SMP support not enabled. Test only valid with current SMP emulator."}
-    end.
+                     case Type of
+                         exit ->
+                             exit_by_myself = R;
+                         _ ->
+                             ExitBySignal = R
+                     end
+             end,
+    receive
+        {'EXIT', Pid, R2} ->
+            Reason = R2
+    end,
+    process_flag(trap_exit, OTE),
+    ok,
+    {comment, "Test only valid with current SMP emulator."}.
 
 
 
diff --git a/erts/emulator/test/smoke_test_SUITE.erl b/erts/emulator/test/smoke_test_SUITE.erl
index 41bb07b84c..adc6f56c06 100644
--- a/erts/emulator/test/smoke_test_SUITE.erl
+++ b/erts/emulator/test/smoke_test_SUITE.erl
@@ -88,11 +88,9 @@ native_atomics(Config) when is_list(Config) ->
     {value,{NA32Key, NA32, _}} = lists:keysearch(NA32Key, 1, EthreadInfo),
     {value,{NA64Key, NA64, _}} = lists:keysearch(NA64Key, 1, EthreadInfo),
     {value,{DWNAKey, DWNA, _}} = lists:keysearch(DWNAKey, 1, EthreadInfo),
-    case {erlang:system_info(build_type), erlang:system_info(smp_support), NA32, NA64, DWNA} of
-	{opt, true, "no", "no", _} ->
+    case {erlang:system_info(build_type), NA32, NA64, DWNA} of
+	{opt, "no", "no", _} ->
 	    ct:fail(optimized_smp_runtime_without_native_atomics);
-	{_, false, "no", "no", _} ->
-	    {comment, "No native atomics"};
 	_ ->
 	    {comment,
 	     NA32 ++ " 32-bit, "
diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl
index 7690557fda..6c01bfd45c 100644
--- a/erts/emulator/test/statistics_SUITE.erl
+++ b/erts/emulator/test/statistics_SUITE.erl
@@ -610,9 +610,7 @@ msacc(Config) ->
                         (aux, 0) ->
                              %% aux will be zero if we do not have smp support
                              %% or no async threads
-                             case erlang:system_info(smp_support) orelse
-                                  erlang:system_info(thread_pool_size) > 0
-                             of
+                             case erlang:system_info(thread_pool_size) > 0 of
                                  false ->
                                      ok;
                                  true ->
diff --git a/erts/emulator/test/system_profile_SUITE.erl b/erts/emulator/test/system_profile_SUITE.erl
index 9b678fcff9..c9be54f668 100644
--- a/erts/emulator/test/system_profile_SUITE.erl
+++ b/erts/emulator/test/system_profile_SUITE.erl
@@ -146,9 +146,8 @@ do_runnable_ports({TsType, TsTypeFlag}, Config) ->
 
 %% Tests system_profiling with scheduler.
 scheduler(Config) when is_list(Config) ->
-    case {erlang:system_info(smp_support), erlang:system_info(schedulers_online)} of
-	{false,_} -> {skipped, "No need for scheduler test when smp support is disabled."};
-	{_,    1} -> {skipped, "No need for scheduler test when only one scheduler online."};
+    case erlang:system_info(schedulers_online) of
+	1 -> {skipped, "No need for scheduler test when only one scheduler online."};
 	_ ->
 	    Nodes = 10,
 	    lists:foreach(fun (TsType) ->
-- 
cgit v1.2.3


From a497237907f2e4112f0c765718975165b6554795 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 13 Jul 2017 09:55:54 +0200
Subject: erts: Replace usage of all erts_smp prefixes to just erts

---
 erts/emulator/beam/atom.c                      |   30 +-
 erts/emulator/beam/beam_bif_load.c             |  102 +--
 erts/emulator/beam/beam_bp.c                   |  104 +--
 erts/emulator/beam/beam_bp.h                   |   20 +-
 erts/emulator/beam/beam_debug.c                |   10 +-
 erts/emulator/beam/beam_emu.c                  |  112 +--
 erts/emulator/beam/beam_load.c                 |   10 +-
 erts/emulator/beam/beam_ranges.c               |   32 +-
 erts/emulator/beam/bif.c                       |  250 +++---
 erts/emulator/beam/bif.h                       |    2 +-
 erts/emulator/beam/break.c                     |   16 +-
 erts/emulator/beam/code_ix.c                   |   32 +-
 erts/emulator/beam/code_ix.h                   |    8 +-
 erts/emulator/beam/copy.c                      |   12 +-
 erts/emulator/beam/dist.c                      |  318 +++----
 erts/emulator/beam/dist.h                      |   26 +-
 erts/emulator/beam/erl_alloc.c                 |   24 +-
 erts/emulator/beam/erl_alloc.h                 |   20 +-
 erts/emulator/beam/erl_alloc_util.c            |   66 +-
 erts/emulator/beam/erl_alloc_util.h            |    4 +-
 erts/emulator/beam/erl_async.c                 |    2 +-
 erts/emulator/beam/erl_bif_ddll.c              |  108 +--
 erts/emulator/beam/erl_bif_info.c              |  134 +--
 erts/emulator/beam/erl_bif_port.c              |   36 +-
 erts/emulator/beam/erl_bif_trace.c             |   92 +-
 erts/emulator/beam/erl_bif_unique.c            |    4 +-
 erts/emulator/beam/erl_binary.h                |   16 +-
 erts/emulator/beam/erl_bits.c                  |    8 +-
 erts/emulator/beam/erl_cpu_topology.c          |   86 +-
 erts/emulator/beam/erl_db.c                    |  230 ++---
 erts/emulator/beam/erl_db.h                    |    8 +-
 erts/emulator/beam/erl_db_hash.c               |  198 ++---
 erts/emulator/beam/erl_db_hash.h               |   14 +-
 erts/emulator/beam/erl_db_tree.c               |   26 +-
 erts/emulator/beam/erl_db_tree.h               |    2 +-
 erts/emulator/beam/erl_db_util.c               |   66 +-
 erts/emulator/beam/erl_db_util.h               |   14 +-
 erts/emulator/beam/erl_drv_thread.c            |    2 +-
 erts/emulator/beam/erl_fun.c                   |   43 +-
 erts/emulator/beam/erl_fun.h                   |    4 +-
 erts/emulator/beam/erl_gc.c                    |   36 +-
 erts/emulator/beam/erl_hl_timer.c              |  108 +--
 erts/emulator/beam/erl_hl_timer.h              |    8 +-
 erts/emulator/beam/erl_init.c                  |   16 +-
 erts/emulator/beam/erl_lock_check.h            |    2 -
 erts/emulator/beam/erl_message.c               |   76 +-
 erts/emulator/beam/erl_message.h               |    2 +-
 erts/emulator/beam/erl_monitors.c              |   32 +-
 erts/emulator/beam/erl_msacc.c                 |    8 +-
 erts/emulator/beam/erl_nfunc_sched.c           |    2 +-
 erts/emulator/beam/erl_nfunc_sched.h           |   10 +-
 erts/emulator/beam/erl_nif.c                   |  114 +--
 erts/emulator/beam/erl_node_tables.c           |  170 ++--
 erts/emulator/beam/erl_node_tables.h           |   57 +-
 erts/emulator/beam/erl_port.h                  |   76 +-
 erts/emulator/beam/erl_port_task.c             |  218 ++---
 erts/emulator/beam/erl_port_task.h             |   26 +-
 erts/emulator/beam/erl_process.c               | 1112 ++++++++++++------------
 erts/emulator/beam/erl_process.h               |  209 +++--
 erts/emulator/beam/erl_process_dump.c          |    8 +-
 erts/emulator/beam/erl_process_lock.c          |   24 +-
 erts/emulator/beam/erl_process_lock.h          |   94 +-
 erts/emulator/beam/erl_ptab.c                  |  126 +--
 erts/emulator/beam/erl_ptab.h                  |   58 +-
 erts/emulator/beam/erl_smp.h                   | 1063 ----------------------
 erts/emulator/beam/erl_thr_progress.h          |    5 -
 erts/emulator/beam/erl_thr_queue.c             |    2 +-
 erts/emulator/beam/erl_threads.h               |   19 -
 erts/emulator/beam/erl_time_sup.c              |   94 +-
 erts/emulator/beam/erl_trace.c                 |  218 ++---
 erts/emulator/beam/erl_trace.h                 |    2 +-
 erts/emulator/beam/erl_utils.h                 |   45 +-
 erts/emulator/beam/erl_vm.h                    |    2 +-
 erts/emulator/beam/export.c                    |   16 +-
 erts/emulator/beam/export.h                    |    6 +-
 erts/emulator/beam/external.c                  |    6 +-
 erts/emulator/beam/global.h                    |   16 +-
 erts/emulator/beam/index.c                     |    2 +-
 erts/emulator/beam/index.h                     |    2 +-
 erts/emulator/beam/io.c                        |  378 ++++----
 erts/emulator/beam/module.c                    |   24 +-
 erts/emulator/beam/module.h                    |   12 +-
 erts/emulator/beam/register.c                  |   72 +-
 erts/emulator/beam/safe_hash.c                 |   50 +-
 erts/emulator/beam/safe_hash.h                 |    6 +-
 erts/emulator/beam/sys.h                       |  150 +---
 erts/emulator/beam/utils.c                     |   47 +-
 erts/emulator/hipe/hipe_bif0.c                 |   28 +-
 erts/emulator/hipe/hipe_bif2.c                 |   10 +-
 erts/emulator/hipe/hipe_mode_switch.c          |   22 +-
 erts/emulator/hipe/hipe_native_bif.c           |   14 +-
 erts/emulator/sys/common/erl_check_io.c        |  172 ++--
 erts/emulator/sys/common/erl_check_io.h        |    8 +-
 erts/emulator/sys/common/erl_mmap.c            |   51 +-
 erts/emulator/sys/common/erl_mseg.c            |    4 +-
 erts/emulator/sys/common/erl_poll.c            |   98 +--
 erts/emulator/sys/common/erl_sys_common_misc.c |    2 +-
 erts/emulator/sys/unix/sys.c                   |   60 +-
 erts/emulator/sys/unix/sys_drivers.c           |   34 +-
 erts/emulator/sys/unix/sys_time.c              |   12 +-
 erts/emulator/sys/win32/erl_poll.c             |   16 +-
 erts/emulator/sys/win32/sys.c                  |   50 +-
 erts/emulator/sys/win32/sys_env.c              |   34 +-
 erts/emulator/sys/win32/sys_interrupt.c        |   10 +-
 erts/emulator/sys/win32/sys_time.c             |    4 +-
 105 files changed, 3202 insertions(+), 4547 deletions(-)
 delete mode 100644 erts/emulator/beam/erl_smp.h

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index 8023414821..1b691386eb 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -34,20 +34,18 @@
 
 IndexTable erts_atom_table;	/* The index table */
 
-#include "erl_smp.h"
+static erts_rwmtx_t atom_table_lock;
 
-static erts_smp_rwmtx_t atom_table_lock;
-
-#define atom_read_lock()	erts_smp_rwmtx_rlock(&atom_table_lock)
-#define atom_read_unlock()	erts_smp_rwmtx_runlock(&atom_table_lock)
-#define atom_write_lock()	erts_smp_rwmtx_rwlock(&atom_table_lock)
-#define atom_write_unlock()	erts_smp_rwmtx_rwunlock(&atom_table_lock)
+#define atom_read_lock()	erts_rwmtx_rlock(&atom_table_lock)
+#define atom_read_unlock()	erts_rwmtx_runlock(&atom_table_lock)
+#define atom_write_lock()	erts_rwmtx_rwlock(&atom_table_lock)
+#define atom_write_unlock()	erts_rwmtx_rwunlock(&atom_table_lock)
 
 #if 0
 #define ERTS_ATOM_PUT_OPS_STAT
 #endif
 #ifdef ERTS_ATOM_PUT_OPS_STAT
-static erts_smp_atomic_t atom_put_ops;
+static erts_atomic_t atom_put_ops;
 #endif
 
 /* Functions for allocating space for the ext of atoms. We do not
@@ -76,7 +74,7 @@ void atom_info(fmtfn_t to, void *to_arg)
     index_info(to, to_arg, &erts_atom_table);
 #ifdef ERTS_ATOM_PUT_OPS_STAT
     erts_print(to, to_arg, "atom_put_ops: %ld\n",
-	       erts_smp_atomic_read_nob(&atom_put_ops));
+	       erts_atomic_read_nob(&atom_put_ops));
 #endif
 
     if (lock)
@@ -246,7 +244,7 @@ erts_atom_put_index(const byte *name, int len, ErtsAtomEncoding enc, int trunc)
     int aix;
 
 #ifdef ERTS_ATOM_PUT_OPS_STAT
-    erts_smp_atomic_inc_nob(&atom_put_ops);
+    erts_atomic_inc_nob(&atom_put_ops);
 #endif
 
     if (tlen < 0) {
@@ -421,16 +419,16 @@ init_atom_table(void)
     HashFunctions f;
     int i;
     Atom a;
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER;
 
-    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
-    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+    rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_RWMTX_LONG_LIVED;
 
 #ifdef ERTS_ATOM_PUT_OPS_STAT
-    erts_smp_atomic_init_nob(&atom_put_ops, 0);
+    erts_atomic_init_nob(&atom_put_ops, 0);
 #endif
 
-    erts_smp_rwmtx_init_opt(&atom_table_lock, &rwmtx_opt, "atom_tab", NIL,
+    erts_rwmtx_init_opt(&atom_table_lock, &rwmtx_opt, "atom_tab", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
     f.hash = (H_FUN) atom_hash;
@@ -493,4 +491,4 @@ Uint
 erts_get_atom_limit(void)
 {
     return erts_atom_table.limit;
-}
\ No newline at end of file
+}
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index fb22d28af0..b78f617560 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -50,7 +50,7 @@
 
 static struct {
     Eterm module;
-    erts_smp_mtx_t mtx;
+    erts_mtx_t mtx;
     Export *pending_purge_lambda;
     Eterm *sprocs;
     Eterm def_sprocs[10];
@@ -68,9 +68,9 @@ Process *erts_code_purger = NULL;
 #ifdef ERTS_DIRTY_SCHEDULERS
 Process *erts_dirty_process_code_checker;
 #endif
-erts_smp_atomic_t erts_copy_literal_area__;
+erts_atomic_t erts_copy_literal_area__;
 #define ERTS_SET_COPY_LITERAL_AREA(LA)			\
-    erts_smp_atomic_set_nob(&erts_copy_literal_area__,	\
+    erts_atomic_set_nob(&erts_copy_literal_area__,	\
 			    (erts_aint_t) (LA))
 Process *erts_literal_area_collector = NULL;
 
@@ -81,7 +81,7 @@ struct ErtsLiteralAreaRef_ {
 };
 
 struct {
-    erts_smp_mtx_t mtx;
+    erts_mtx_t mtx;
     ErtsLiteralAreaRef *first;
     ErtsLiteralAreaRef *last;
 } release_literal_areas;
@@ -97,7 +97,7 @@ init_purge_state(void)
 {
     purge_state.module = THE_NON_VALUE;
 
-    erts_smp_mtx_init(&purge_state.mtx, "purge_state", NIL,
+    erts_mtx_init(&purge_state.mtx, "purge_state", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
     purge_state.pending_purge_lambda =
@@ -119,12 +119,12 @@ init_purge_state(void)
 void
 erts_beam_bif_load_init(void)
 {
-    erts_smp_mtx_init(&release_literal_areas.mtx, "release_literal_areas", NIL,
+    erts_mtx_init(&release_literal_areas.mtx, "release_literal_areas", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
     release_literal_areas.first = NULL;
     release_literal_areas.last = NULL;
-    erts_smp_atomic_init_nob(&erts_copy_literal_area__,
+    erts_atomic_init_nob(&erts_copy_literal_area__,
 			     (erts_aint_t) NULL);
 
     init_purge_state();
@@ -172,8 +172,8 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
 			BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
     }
 
-    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-    erts_smp_thr_progress_block();
+    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_thr_progress_block();
 
     modp = erts_get_module(mod, erts_active_code_ix());
 
@@ -197,8 +197,8 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
     else {
 	erts_abort_staging_code_ix();
     }
-    erts_smp_thr_progress_unblock();
-    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_thr_progress_unblock();
+    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     erts_release_code_write_permission();
     return res;
 #endif
@@ -399,8 +399,8 @@ finish_loading_1(BIF_ALIST_1)
 	    erts_is_default_trace_enabled() ||
 	    IF_HIPE(hipe_need_blocking(p[i].modp))) {
 	    /* tracing or hipe need thread blocking */
-	    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	    erts_smp_thr_progress_block();
+	    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	    erts_thr_progress_block();
 	    is_blocking = 1;
 	    break;
 	}
@@ -486,8 +486,8 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
 	    erts_free(ERTS_ALC_T_LOADER_TMP, mods);
 	}
 	if (is_blocking) {
-	    erts_smp_thr_progress_unblock();
-	    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+	    erts_thr_progress_unblock();
+	    erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 	}
 	erts_release_code_write_permission();
 	return res;
@@ -528,11 +528,11 @@ static void smp_code_ix_commiter(void* null)
     committer_state.stager = NULL;
 #endif
     erts_release_code_write_permission();
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
     if (!ERTS_PROC_IS_EXITING(p)) {
 	erts_resume(p, ERTS_PROC_LOCK_STATUS);
     }
-    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
     erts_proc_dec_refc(p);
 }
 
@@ -631,7 +631,7 @@ BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2)
     res = erts_check_process_code(rp, BIF_ARG_2, &reds, BIF_P->fcalls);
 
     if (BIF_P != rp)
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
+	erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
 
     ASSERT(is_value(res));
 
@@ -674,8 +674,8 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
 		modp->curr.num_traced_exports > 0 ||
 		IF_HIPE(hipe_need_blocking(modp))) {
 		/* tracing or hipe need to go single threaded */
-		erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-		erts_smp_thr_progress_block();
+		erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+		erts_thr_progress_block();
 		is_blocking = 1;
 		if (modp->curr.num_breakpoints) {
 		    erts_clear_module_break(modp);
@@ -779,16 +779,16 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 
     /* ToDo: Use code_ix staging instead of thread blocking */
 
-    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-    erts_smp_thr_progress_block();
+    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_thr_progress_block();
 
     code_ix = erts_active_code_ix();
     modp = erts_get_module(BIF_ARG_1, code_ix);
 
     if (!modp || !modp->on_load || !modp->on_load->code_hdr) {
     error:
-	erts_smp_thr_progress_unblock();
-        erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_unblock();
+        erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	erts_release_code_write_permission();
 	BIF_ERROR(BIF_P, BADARG);
     }
@@ -859,8 +859,8 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	    ep->beam[1] = 0;
 	}
     }
-    erts_smp_thr_progress_unblock();
-    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_thr_progress_unblock();
+    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     erts_release_code_write_permission();
     BIF_RET(am_true);
 }
@@ -921,9 +921,9 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed
      * any other heap than the message it self.
      */
 
-    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
-    ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
-    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
+    erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
+    ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
+    erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
 
     for (msgp = c_p->msg.first; msgp; msgp = msgp->next) {
 	ErlHeapFragment *hf;
@@ -1325,9 +1325,9 @@ static void
 complete_literal_area_switch(void *literal_area)
 {
     Process *p = erts_literal_area_collector;
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
     erts_resume(p, ERTS_PROC_LOCK_STATUS);
-    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
     if (literal_area)
 	erts_release_literal_area((ErtsLiteralArea *) literal_area);
 }
@@ -1340,7 +1340,7 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0)
     if (BIF_P != erts_literal_area_collector)
 	BIF_ERROR(BIF_P, EXC_NOTSUP);
 
-    erts_smp_mtx_lock(&release_literal_areas.mtx);
+    erts_mtx_lock(&release_literal_areas.mtx);
 
     la_ref = release_literal_areas.first;
     if (la_ref) {
@@ -1349,7 +1349,7 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0)
 	    release_literal_areas.last = NULL;
     }
 
-    erts_smp_mtx_unlock(&release_literal_areas.mtx);
+    erts_mtx_unlock(&release_literal_areas.mtx);
 
     unused_la = ERTS_COPY_LITERAL_AREA();
 
@@ -1407,7 +1407,7 @@ erts_purge_state_add_fun(ErlFunEntry *fe)
 Export *
 erts_suspend_process_on_pending_purge_lambda(Process *c_p, ErlFunEntry* fe)
 {
-    erts_smp_mtx_lock(&purge_state.mtx);
+    erts_mtx_lock(&purge_state.mtx);
     if (purge_state.module == fe->module) {
 	/*
 	 * The process c_p is about to call a fun in the code
@@ -1433,7 +1433,7 @@ erts_suspend_process_on_pending_purge_lambda(Process *c_p, ErlFunEntry* fe)
 	erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
 	ERTS_VBUMP_ALL_REDS(c_p);
     }
-    erts_smp_mtx_unlock(&purge_state.mtx);
+    erts_mtx_unlock(&purge_state.mtx);
     return purge_state.pending_purge_lambda;
 }
 
@@ -1443,9 +1443,9 @@ finalize_purge_operation(Process *c_p, int succeded)
     Uint ix;
 
     if (c_p)
-	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+	erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
 
-    erts_smp_mtx_lock(&purge_state.mtx);
+    erts_mtx_lock(&purge_state.mtx);
 
     ASSERT(purge_state.module != THE_NON_VALUE);
 
@@ -1461,14 +1461,14 @@ finalize_purge_operation(Process *c_p, int succeded)
 				    ERTS_PROC_LOCK_STATUS);
 	if (rp) {
 	    erts_resume(rp, ERTS_PROC_LOCK_STATUS);
-	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+	    erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
 	}
     }
 
-    erts_smp_mtx_unlock(&purge_state.mtx);
+    erts_mtx_unlock(&purge_state.mtx);
 
     if (c_p)
-	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+	erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 
     if (purge_state.sprocs != &purge_state.def_sprocs[0]) {
 	erts_free(ERTS_ALC_T_PURGE_DATA, purge_state.sprocs);
@@ -1494,9 +1494,9 @@ static void
 resume_purger(void *unused)
 {
     Process *p = erts_code_purger;
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
     erts_resume(p, ERTS_PROC_LOCK_STATUS);
-    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 }
 
 static void
@@ -1567,9 +1567,9 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 	    else {
 		BeamInstr* code;
 		BeamInstr* end;
-		erts_smp_mtx_lock(&purge_state.mtx);
+		erts_mtx_lock(&purge_state.mtx);
 		purge_state.module = BIF_ARG_1;
-		erts_smp_mtx_unlock(&purge_state.mtx);
+		erts_mtx_unlock(&purge_state.mtx);
 		res = am_true;
 		code = (BeamInstr*) modp->old.code_hdr;
 		end = (BeamInstr *)((char *)code + modp->old.code_length);
@@ -1680,8 +1680,8 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 		    || IF_HIPE(hipe_purge_need_blocking(modp))) {
 		    /* ToDo: Do unload nif without blocking */
 		    erts_rwunlock_old_code(code_ix);
-		    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-		    erts_smp_thr_progress_block();
+		    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+		    erts_thr_progress_block();
 		    is_blocking = 1;
 		    erts_rwlock_old_code(code_ix);
 		    if (modp->old.nif) {
@@ -1719,8 +1719,8 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 	    erts_rwunlock_old_code(code_ix);
 	}
 	if (is_blocking) {
-	    erts_smp_thr_progress_unblock();
-	    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	    erts_thr_progress_unblock();
+	    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	}
 
 	erts_release_code_write_permission();
@@ -1733,7 +1733,7 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 			     sizeof(ErtsLiteralAreaRef));
 	    ref->literal_area = literals;
 	    ref->next = NULL;
-	    erts_smp_mtx_lock(&release_literal_areas.mtx);
+	    erts_mtx_lock(&release_literal_areas.mtx);
 	    if (release_literal_areas.last) {
 		release_literal_areas.last->next = ref;
 		release_literal_areas.last = ref;
@@ -1742,7 +1742,7 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 		release_literal_areas.first = ref;
 		release_literal_areas.last = ref;
 	    }
-	    erts_smp_mtx_unlock(&release_literal_areas.mtx);
+	    erts_mtx_unlock(&release_literal_areas.mtx);
 	    erts_queue_message(erts_literal_area_collector,
 			       0,
 			       erts_alloc_message(0, NULL),
@@ -1779,7 +1779,7 @@ delete_code(Module* modp)
 		}
 		else if (ep->beam[0] ==
 			 (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
-		    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+		    ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
 		    ASSERT(modp->curr.num_traced_exports > 0);
 		    DBG_TRACE_MFA_P(&ep->info.mfa,
 				  "export trace cleared, code_ix=%d", code_ix);
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 5b24db2e24..79a75f6698 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -47,14 +47,14 @@
 #define Free(P)			erts_free(ERTS_ALC_T_BPD, (P))
 
 #if defined(ERTS_ENABLE_LOCK_CHECK)
-#  define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
+#  define ERTS_REQ_PROC_MAIN_LOCK(P) \
       if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\
 					 __FILE__, __LINE__)
-#  define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
+#  define ERTS_UNREQ_PROC_MAIN_LOCK(P) \
       if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
 #else
-#  define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)
-#  define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P)
+#  define ERTS_REQ_PROC_MAIN_LOCK(P)
+#  define ERTS_UNREQ_PROC_MAIN_LOCK(P)
 #endif
 
 #define ERTS_BPF_LOCAL_TRACE       0x01
@@ -73,10 +73,10 @@ extern BeamInstr beam_return_trace[1];      /* OpCode(i_return_trace) */
 extern BeamInstr beam_exception_trace[1];   /* OpCode(i_exception_trace) */
 extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */
 
-erts_smp_atomic32_t erts_active_bp_index;
-erts_smp_atomic32_t erts_staging_bp_index;
+erts_atomic32_t erts_active_bp_index;
+erts_atomic32_t erts_staging_bp_index;
 #ifdef ERTS_DIRTY_SCHEDULERS
-erts_smp_mtx_t erts_dirty_bp_ix_mtx;
+erts_mtx_t erts_dirty_bp_ix_mtx;
 #endif
 
 /*
@@ -96,7 +96,7 @@ acquire_bp_sched_ix(Process *c_p)
     ASSERT(esdp);
 #ifdef ERTS_DIRTY_SCHEDULERS
     if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-	erts_smp_mtx_lock(&erts_dirty_bp_ix_mtx);
+	erts_mtx_lock(&erts_dirty_bp_ix_mtx);
         return (Uint32) erts_no_schedulers;
     }
 #endif
@@ -108,7 +108,7 @@ release_bp_sched_ix(Uint32 ix)
 {
 #ifdef ERTS_DIRTY_SCHEDULERS
     if (ix == (Uint32) erts_no_schedulers)
-        erts_smp_mtx_unlock(&erts_dirty_bp_ix_mtx);
+        erts_mtx_unlock(&erts_dirty_bp_ix_mtx);
 #endif
 }
 
@@ -162,10 +162,10 @@ static void bp_hash_delete(bp_time_hash_t *hash);
 
 void 
 erts_bp_init(void) {
-    erts_smp_atomic32_init_nob(&erts_active_bp_index, 0);
-    erts_smp_atomic32_init_nob(&erts_staging_bp_index, 1);
+    erts_atomic32_init_nob(&erts_active_bp_index, 0);
+    erts_atomic32_init_nob(&erts_staging_bp_index, 1);
 #ifdef ERTS_DIRTY_SCHEDULERS
-    erts_smp_mtx_init(&erts_dirty_bp_ix_mtx, "dirty_break_point_index", NIL,
+    erts_mtx_init(&erts_dirty_bp_ix_mtx, "dirty_break_point_index", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
 #endif
 }
@@ -306,7 +306,7 @@ erts_consolidate_bp_data(BpFunctions* f, int local)
     Uint i;
     Uint n = f->matched;
 
-    ERTS_SMP_LC_ASSERT(erts_has_code_write_permission());
+    ERTS_LC_ASSERT(erts_has_code_write_permission());
 
     for (i = 0; i < n; i++) {
 	consolidate_bp_data(fs[i].mod, fs[i].ci, local);
@@ -318,7 +318,7 @@ erts_consolidate_bif_bp_data(void)
 {
     int i;
 
-    ERTS_SMP_LC_ASSERT(erts_has_code_write_permission());
+    ERTS_LC_ASSERT(erts_has_code_write_permission());
     for (i = 0; i < BIF_SIZE; i++) {
 	Export *ep = bif_export[i];
 	consolidate_bp_data(0, &ep->info, 0);
@@ -393,17 +393,17 @@ consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local)
     }
     if (flags & ERTS_BPF_META_TRACE) {
 	dst->meta_tracer = src->meta_tracer;
-	erts_smp_refc_inc(&dst->meta_tracer->refc, 1);
+	erts_refc_inc(&dst->meta_tracer->refc, 1);
 	dst->meta_ms = src->meta_ms;
 	MatchSetRef(dst->meta_ms);
     }
     if (flags & ERTS_BPF_COUNT) {
 	dst->count = src->count;
-	erts_smp_refc_inc(&dst->count->refc, 1);
+	erts_refc_inc(&dst->count->refc, 1);
     }
     if (flags & ERTS_BPF_TIME_TRACE) {
 	dst->time = src->time;
-	erts_smp_refc_inc(&dst->time->refc, 1);
+	erts_refc_inc(&dst->time->refc, 1);
 	ASSERT(dst->time->hash);
     }
 }
@@ -414,8 +414,8 @@ erts_commit_staged_bp(void)
     ErtsBpIndex staging = erts_staging_bp_ix();
     ErtsBpIndex active = erts_active_bp_ix();
 
-    erts_smp_atomic32_set_nob(&erts_active_bp_index, staging);
-    erts_smp_atomic32_set_nob(&erts_staging_bp_index, active);
+    erts_atomic32_set_nob(&erts_active_bp_index, staging);
+    erts_atomic32_set_nob(&erts_staging_bp_index, active);
 }
 
 void
@@ -575,7 +575,7 @@ erts_clear_mtrace_bif(ErtsCodeInfo *ci)
 void
 erts_clear_debug_break(BpFunctions* f)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
     clear_break(f, ERTS_BPF_DEBUG);
 }
 
@@ -603,7 +603,7 @@ erts_clear_module_break(Module *modp) {
     Uint n;
     Uint i;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
     ASSERT(modp);
     code_hdr = modp->curr.code_hdr;
     if (!code_hdr) {
@@ -633,7 +633,7 @@ erts_clear_module_break(Module *modp) {
 void
 erts_clear_export_break(Module* modp, ErtsCodeInfo *ci)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
 
     clear_function_break(ci, ERTS_BPF_ALL);
     erts_commit_staged_bp();
@@ -679,12 +679,12 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg)
     if (bp_flags & ERTS_BPF_META_TRACE) {
 	ErtsTracer old_tracer, new_tracer;
 
-	old_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer);
+	old_tracer = erts_atomic_read_nob(&bp->meta_tracer->tracer);
 
 	new_tracer = do_call_trace(c_p, info, reg, 1, bp->meta_ms, old_tracer);
 
 	if (!ERTS_TRACER_COMPARE(new_tracer, old_tracer)) {
-            if (old_tracer == erts_smp_atomic_cmpxchg_acqb(
+            if (old_tracer == erts_atomic_cmpxchg_acqb(
                     &bp->meta_tracer->tracer,
                     (erts_aint_t)new_tracer,
                     (erts_aint_t)old_tracer)) {
@@ -696,7 +696,7 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg)
     }
 
     if (bp_flags & ERTS_BPF_COUNT_ACTIVE) {
-	erts_smp_atomic_inc_nob(&bp->count->acount);
+	erts_atomic_inc_nob(&bp->count->acount);
     }
 
     if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE) {
@@ -753,7 +753,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
     GenericBpData* bp = NULL;
     Uint bp_flags = 0;
 
-    ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
+    ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
     g = ep->info.u.gen_bp;
     if (g) {
@@ -777,7 +777,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
     if (bp_flags & ERTS_BPF_META_TRACE) {
 	ErtsTracer old_tracer;
 
-        meta_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer);
+        meta_tracer = erts_atomic_read_nob(&bp->meta_tracer->tracer);
         old_tracer = meta_tracer;
 	flags_meta = erts_call_trace(p, &ep->info, bp->meta_ms, args,
 				     0, &meta_tracer);
@@ -785,7 +785,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 	if (!ERTS_TRACER_COMPARE(old_tracer, meta_tracer)) {
             ErtsTracer new_tracer = erts_tracer_nil;
             erts_tracer_update(&new_tracer, meta_tracer);
-	    if (old_tracer == erts_smp_atomic_cmpxchg_acqb(
+	    if (old_tracer == erts_atomic_cmpxchg_acqb(
                     &bp->meta_tracer->tracer,
                     (erts_aint_t)new_tracer,
                     (erts_aint_t)old_tracer)) {
@@ -912,9 +912,9 @@ erts_bif_trace_epilogue(Process *p, Eterm result, int applying,
 		}
 	    }
 	    if ((flags_meta|flags) & MATCH_SET_EXCEPTION_TRACE) {
-		erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+		erts_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
 		ERTS_TRACE_FLAGS(p) |= F_EXCEPTION_TRACE;
-		erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+		erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
 	    }
 	}
     } else {
@@ -937,7 +937,7 @@ erts_bif_trace_epilogue(Process *p, Eterm result, int applying,
 	    }
 	}
     }
-    ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
+    ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
     return result;
 }
 
@@ -982,9 +982,9 @@ do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg,
 	c_p->cp = (BeamInstr *) cp_val(*cpp);
 	ASSERT(is_CP(*cpp));
     }
-    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
     flags = erts_call_trace(c_p, info, ms, reg, local, &tracer);
-    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
     if (cpp) {
 	c_p->cp = cp_save;
     }
@@ -1024,9 +1024,9 @@ do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg,
 				  the funcinfo is above i. */
 	c_p->cp = (flags & MATCH_SET_EXCEPTION_TRACE) ?
 	    beam_exception_trace : beam_return_trace;
-	erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+	erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 	ERTS_TRACE_FLAGS(c_p) |= F_EXCEPTION_TRACE;
-	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+	erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
     } else
         c_p->stop = E;
     return tracer;
@@ -1043,7 +1043,7 @@ erts_trace_time_call(Process* c_p, ErtsCodeInfo *info, BpDataTime* bdt)
     Uint32 six = acquire_bp_sched_ix(c_p);
 
     ASSERT(c_p);
-    ASSERT(erts_smp_atomic32_read_acqb(&c_p->state) & (ERTS_PSFLG_RUNNING
+    ASSERT(erts_atomic32_read_acqb(&c_p->state) & (ERTS_PSFLG_RUNNING
 						       | ERTS_PSFLG_DIRTY_RUNNING));
 
     /* get previous timestamp and breakpoint
@@ -1124,7 +1124,7 @@ erts_trace_time_return(Process *p, ErtsCodeInfo *ci)
     Uint32 six = acquire_bp_sched_ix(p);
 
     ASSERT(p);
-    ASSERT(erts_smp_atomic32_read_acqb(&p->state) & (ERTS_PSFLG_RUNNING
+    ASSERT(erts_atomic32_read_acqb(&p->state) & (ERTS_PSFLG_RUNNING
 						     | ERTS_PSFLG_DIRTY_RUNNING));
 
     /* get previous timestamp and breakpoint
@@ -1206,7 +1206,7 @@ erts_is_mtrace_break(ErtsCodeInfo *ci, Binary **match_spec_ret,
 	    *match_spec_ret = bp->meta_ms;
 	}
 	if (tracer_ret) {
-            *tracer_ret = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer);
+            *tracer_ret = erts_atomic_read_nob(&bp->meta_tracer->tracer);
 	}
 	return 1;
     }
@@ -1220,7 +1220,7 @@ erts_is_count_break(ErtsCodeInfo *ci, Uint *count_ret)
     
     if (bp) {
 	if (count_ret) {
-	    *count_ret = (Uint) erts_smp_atomic_read_nob(&bp->count->acount);
+	    *count_ret = (Uint) erts_atomic_read_nob(&bp->count->acount);
 	}
 	return 1;
     }
@@ -1500,7 +1500,7 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags,
     Uint common;
     ErtsBpIndex ix = erts_staging_bp_ix();
 
-    ERTS_SMP_LC_ASSERT(erts_has_code_write_permission());
+    ERTS_LC_ASSERT(erts_has_code_write_permission());
     g = ci->u.gen_bp;
     if (g == 0) {
 	int i;
@@ -1532,7 +1532,7 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags,
 	    bp->flags &= ~ERTS_BPF_COUNT_ACTIVE;
 	} else {
 	    bp->flags |= ERTS_BPF_COUNT_ACTIVE;
-	    erts_smp_atomic_set_nob(&bp->count->acount, 0);
+	    erts_atomic_set_nob(&bp->count->acount, 0);
 	}
 	ASSERT((bp->flags & ~ERTS_BPF_ALL) == 0);
 	return;
@@ -1566,17 +1566,17 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags,
 	MatchSetRef(match_spec);
 	bp->meta_ms = match_spec;
 	bmt = Alloc(sizeof(BpMetaTracer));
-	erts_smp_refc_init(&bmt->refc, 1);
+	erts_refc_init(&bmt->refc, 1);
         erts_tracer_update(&meta_tracer, tracer); /* copy tracer */
-	erts_smp_atomic_init_nob(&bmt->tracer, (erts_aint_t)meta_tracer);
+	erts_atomic_init_nob(&bmt->tracer, (erts_aint_t)meta_tracer);
 	bp->meta_tracer = bmt;
     } else if (break_flags & ERTS_BPF_COUNT) {
 	BpCount* bcp;
 
 	ASSERT((bp->flags & ERTS_BPF_COUNT) == 0);
 	bcp = Alloc(sizeof(BpCount));
-	erts_smp_refc_init(&bcp->refc, 1);
-	erts_smp_atomic_init_nob(&bcp->acount, 0);
+	erts_refc_init(&bcp->refc, 1);
+	erts_atomic_init_nob(&bcp->acount, 0);
 	bp->count = bcp;
     } else if (break_flags & ERTS_BPF_TIME_TRACE) {
 	BpDataTime* bdt;
@@ -1584,7 +1584,7 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags,
 
 	ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0);
 	bdt = Alloc(sizeof(BpDataTime));
-	erts_smp_refc_init(&bdt->refc, 1);
+	erts_refc_init(&bdt->refc, 1);
 #ifdef ERTS_DIRTY_SCHEDULERS
 	bdt->n = erts_no_schedulers + 1;
 #else
@@ -1621,7 +1621,7 @@ clear_function_break(ErtsCodeInfo *ci, Uint break_flags)
     Uint common;
     ErtsBpIndex ix = erts_staging_bp_ix();
 
-    ERTS_SMP_LC_ASSERT(erts_has_code_write_permission());
+    ERTS_LC_ASSERT(erts_has_code_write_permission());
 
     if ((g = ci->u.gen_bp) == NULL) {
 	return 1;
@@ -1654,8 +1654,8 @@ clear_function_break(ErtsCodeInfo *ci, Uint break_flags)
 static void
 bp_meta_unref(BpMetaTracer* bmt)
 {
-    if (erts_smp_refc_dectest(&bmt->refc, 0) <= 0) {
-        ErtsTracer trc = erts_smp_atomic_read_nob(&bmt->tracer);
+    if (erts_refc_dectest(&bmt->refc, 0) <= 0) {
+        ErtsTracer trc = erts_atomic_read_nob(&bmt->tracer);
         ERTS_TRACER_CLEAR(&trc);
 	Free(bmt);
     }
@@ -1664,7 +1664,7 @@ bp_meta_unref(BpMetaTracer* bmt)
 static void
 bp_count_unref(BpCount* bcp)
 {
-    if (erts_smp_refc_dectest(&bcp->refc, 0) <= 0) {
+    if (erts_refc_dectest(&bcp->refc, 0) <= 0) {
 	Free(bcp);
     }
 }
@@ -1672,7 +1672,7 @@ bp_count_unref(BpCount* bcp)
 static void
 bp_time_unref(BpDataTime* bdt)
 {
-    if (erts_smp_refc_dectest(&bdt->refc, 0) <= 0) {
+    if (erts_refc_dectest(&bdt->refc, 0) <= 0) {
 	Uint i = 0;
 	Uint j = 0;
 	Process *h_p = NULL;
@@ -1696,7 +1696,7 @@ bp_time_unref(BpDataTime* bdt)
 			    if (pbt) {
 				Free(pbt);
 			    }
-			    erts_smp_proc_unlock(h_p, ERTS_PROC_LOCK_MAIN);
+			    erts_proc_unlock(h_p, ERTS_PROC_LOCK_MAIN);
 			}
 		    }
 		}
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index 56fa82b912..1e1f6a7534 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -41,7 +41,7 @@ typedef struct {
 typedef struct bp_data_time {     /* Call time */
     Uint n;
     bp_time_hash_t *hash;
-    erts_smp_refc_t refc;
+    erts_refc_t refc;
 } BpDataTime;
 
 typedef struct {
@@ -50,13 +50,13 @@ typedef struct {
 } process_breakpoint_time_t; /* used within psd */
 
 typedef struct {
-    erts_smp_atomic_t acount;
-    erts_smp_refc_t refc;
+    erts_atomic_t acount;
+    erts_refc_t refc;
 } BpCount;
 
 typedef struct {
-    erts_smp_atomic_t tracer;
-    erts_smp_refc_t refc;
+    erts_atomic_t tracer;
+    erts_refc_t refc;
 } BpMetaTracer;
 
 typedef struct generic_bp_data {
@@ -80,7 +80,7 @@ typedef struct generic_bp {
 #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2)
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-extern erts_smp_mtx_t erts_dirty_bp_ix_mtx;
+extern erts_mtx_t erts_dirty_bp_ix_mtx;
 #endif
 
 enum erts_break_op{
@@ -173,17 +173,17 @@ ErtsCodeInfo *erts_find_local_func(ErtsCodeMFA *mfa);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-extern erts_smp_atomic32_t erts_active_bp_index;
-extern erts_smp_atomic32_t erts_staging_bp_index;
+extern erts_atomic32_t erts_active_bp_index;
+extern erts_atomic32_t erts_staging_bp_index;
 
 ERTS_GLB_INLINE ErtsBpIndex erts_active_bp_ix(void)
 {
-    return erts_smp_atomic32_read_nob(&erts_active_bp_index);
+    return erts_atomic32_read_nob(&erts_active_bp_index);
 }
 
 ERTS_GLB_INLINE ErtsBpIndex erts_staging_bp_ix(void)
 {
-    return erts_smp_atomic32_read_nob(&erts_staging_bp_index);
+    return erts_atomic32_read_nob(&erts_staging_bp_index);
 }
 #endif
 
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 5f47a7c99f..fa912e52e9 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -157,8 +157,8 @@ erts_debug_breakpoint_2(BIF_ALIST_2)
 	ERTS_BIF_YIELD2(bif_export[BIF_erts_debug_breakpoint_2],
 			BIF_P, BIF_ARG_1, BIF_ARG_2);
     }
-    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
-    erts_smp_thr_progress_block();
+    erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
+    erts_thr_progress_block();
 
     erts_bp_match_functions(&f, &mfa, specified);
     if (boolean == am_true) {
@@ -174,8 +174,8 @@ erts_debug_breakpoint_2(BIF_ALIST_2)
     res = make_small(f.matched);
     erts_bp_free_matched_functions(&f);
 
-    erts_smp_thr_progress_unblock();
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+    erts_thr_progress_unblock();
+    erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     erts_release_code_write_permission();
     return res;
 
@@ -1096,7 +1096,7 @@ dirty_send_message(Process *c_p, Eterm to, Eterm tag)
     if (rp == real_c_p)
 	rp_locks &= ~c_p_locks;
     if (rp_locks)
-	erts_smp_proc_unlock(rp, rp_locks);
+	erts_proc_unlock(rp, rp_locks);
 
     erts_proc_dec_refc(rp);
 
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 0b4a69411a..25e16764ab 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -66,23 +66,23 @@
 do {									\
     if ((P))								\
 	erts_proc_lc_chk_only_proc_main((P));				\
-    ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());		\
+    ERTS_LC_ASSERT(!erts_thr_progress_is_blocking());		\
 } while (0)
-#    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)				\
+#    define ERTS_REQ_PROC_MAIN_LOCK(P)				\
 do {									\
     if ((P))								\
 	erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,		\
 				  __FILE__, __LINE__);			\
 } while (0)
-#    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P)				\
+#    define ERTS_UNREQ_PROC_MAIN_LOCK(P)				\
 do {									\
     if ((P))								\
 	erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN);		\
 } while (0)
 #else
 #  define PROCESS_MAIN_CHK_LOCKS(P)
-#  define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)
-#  define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P)
+#  define ERTS_REQ_PROC_MAIN_LOCK(P)
+#  define ERTS_UNREQ_PROC_MAIN_LOCK(P)
 #endif
 
 /*
@@ -294,7 +294,7 @@ void** beam_ops;
      HEAP_TOP((P)) = HTOP;  						\
      (P)->stop = E;  							\
      PROCESS_MAIN_CHK_LOCKS((P));					\
-     ERTS_SMP_UNREQ_PROC_MAIN_LOCK((P))
+     ERTS_UNREQ_PROC_MAIN_LOCK((P))
 
 #define db(N) (N)
 #define tb(N) (N)
@@ -1358,7 +1358,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
     }
 
     PROCESS_MAIN_CHK_LOCKS(c_p);
-    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
     c_p = erts_schedule(NULL, c_p, reds_used);
     ASSERT(!(c_p->flags & F_HIPE_MODE));
@@ -1367,7 +1367,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 #ifdef DEBUG
     pid = c_p->common.id; /* Save for debugging purposes */
 #endif
-    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
     PROCESS_MAIN_CHK_LOCKS(c_p);
 
     ERTS_MSACC_UPDATE_CACHE_X();
@@ -1740,7 +1740,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
      result = erl_send(c_p, r(0), x(1));
      PreFetch(0, next);
      ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+     ERTS_REQ_PROC_MAIN_LOCK(c_p);
      PROCESS_MAIN_CHK_LOCKS(c_p);
      HTOP = HEAP_TOP(c_p);
      FCALLS = c_p->fcalls;
@@ -1937,19 +1937,19 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
      msgp = PEEK_MESSAGE(c_p);
 
      if (!msgp) {
-	 erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	 erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	 /* Make sure messages wont pass exit signals... */
 	 if (ERTS_PROC_PENDING_EXIT(c_p)) {
-	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	     erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	     SWAPOUT;
 	     c_p->flags &= ~F_DELAY_GC;
              c_p->arity = 0;
 	     goto do_schedule; /* Will be rescheduled for exit */
 	 }
-	 ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
+	 ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
 	 msgp = PEEK_MESSAGE(c_p);
 	 if (msgp)
-	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	     erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	 else
 	 {
 	     c_p->flags &= ~F_DELAY_GC;
@@ -2115,7 +2115,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 
 
  OpCase(i_wait_timeout_fs): {
-     erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+     erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 
      /* Fall through */
  }
@@ -2148,7 +2148,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 	     }
 	     else { /* Wrong time */
 	     OpCase(i_wait_error_locked): {
-		     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+		     erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 		     /* Fall through */
 		 }
 	     OpCase(i_wait_error): {
@@ -2177,24 +2177,24 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 	     c_p->arity = 0;
 
 	     if (!ERTS_PTMR_IS_TIMED_OUT(c_p))
-		 erts_smp_atomic32_read_band_relb(&c_p->state,
+		 erts_atomic32_read_band_relb(&c_p->state,
 						  ~ERTS_PSFLG_ACTIVE);
 	     ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	     erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	     c_p->current = NULL;
 	     goto do_schedule;
 	 }
 	 OpCase(wait_unlocked_f): {
-	     erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	     erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	     goto wait2;
 	 }
      }
-     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+     erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
      Next(2);
  }
 
  OpCase(i_wait_timeout_fI): {
-     erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+     erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
  }
 
  OpCase(i_wait_timeout_locked_fI):
@@ -2216,7 +2216,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
      * receive statement will examine the first message first.
      */
  OpCase(timeout_locked): {
-     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+     erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
  }
 
  OpCase(timeout): {
@@ -2672,12 +2672,12 @@ do {						\
 	c_p->fcalls = FCALLS;
 	SWAPOUT;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
 	ERTS_CHK_MBUF_SZ(c_p);
 	result = (*bf)(c_p, reg, live);
 	ERTS_CHK_MBUF_SZ(c_p);
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	ERTS_REQ_PROC_MAIN_LOCK(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	SWAPIN;
 	ERTS_HOLE_CHECK(c_p);
@@ -2715,12 +2715,12 @@ do {						\
 	c_p->fcalls = FCALLS;
 	SWAPOUT;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
 	ERTS_CHK_MBUF_SZ(c_p);
 	result = (*bf)(c_p, reg, live);
 	ERTS_CHK_MBUF_SZ(c_p);
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	ERTS_REQ_PROC_MAIN_LOCK(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	SWAPIN;
 	ERTS_HOLE_CHECK(c_p);
@@ -2760,12 +2760,12 @@ do {						\
 	c_p->fcalls = FCALLS;
 	SWAPOUT;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
 	ERTS_CHK_MBUF_SZ(c_p);
 	result = (*bf)(c_p, reg, live);
 	ERTS_CHK_MBUF_SZ(c_p);
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	ERTS_REQ_PROC_MAIN_LOCK(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	SWAPIN;
 	ERTS_HOLE_CHECK(c_p);
@@ -2890,7 +2890,7 @@ do {						\
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	ERTS_HOLE_CHECK(c_p);
-	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	ERTS_REQ_PROC_MAIN_LOCK(c_p);
 	if (ERTS_IS_GC_DESIRED(c_p)) {
 	    Uint arity = GET_BIF_ARITY(export);
 	    result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity);
@@ -3400,7 +3400,7 @@ do {						\
      Eterm* argp;
      int i;
 
-     if (erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_EXITING) {
+     if (erts_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_EXITING) {
          c_p->i = beam_exit;
          c_p->arity = 0;
          c_p->current = NULL;
@@ -3476,16 +3476,16 @@ do {						\
      SWAPOUT;
      c_p->freason = EXC_NORMAL;
      c_p->arity = 0;		/* In case this process will never be garbed again. */
-     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+     ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
      erts_do_exit_process(c_p, am_normal);
-     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+     ERTS_REQ_PROC_MAIN_LOCK(c_p);
      goto do_schedule;
  }
 
  OpCase(continue_exit): {
-     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+     ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
      erts_continue_exit_process(c_p);
-     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+     ERTS_REQ_PROC_MAIN_LOCK(c_p);
      goto do_schedule;
  }
 
@@ -3590,7 +3590,7 @@ do {						\
 
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
 	    bif_nif_arity = codemfa->arity;
-	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
 
 	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	    {
@@ -3655,7 +3655,7 @@ do {						\
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
 	    bif_nif_arity = codemfa->arity;
             ASSERT(bif_nif_arity <= 4);
-	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
 	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	    {
 		ErtsBifFunc bf = vbf;
@@ -3678,7 +3678,7 @@ do {						\
 	    DTRACE_BIF_RETURN(c_p, codemfa);
 
 	apply_bif_or_nif_epilogue:
-	    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	    ERTS_REQ_PROC_MAIN_LOCK(c_p);
 	    ERTS_HOLE_CHECK(c_p);
 	    if (ERTS_IS_GC_DESIRED(c_p)) {
 		nif_bif_result = erts_gc_after_bif_call_lhf(c_p, live_hf_end,
@@ -4751,9 +4751,9 @@ do {						\
      ErtsCodeMFA* mfa = (ErtsCodeMFA *)(E[0]);
      
      SWAPOUT;		/* Needed for shared heap */
-     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+     ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
      erts_trace_return(c_p, mfa, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */);
-     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+     ERTS_REQ_PROC_MAIN_LOCK(c_p);
      SWAPIN;
      c_p->cp = NULL;
      SET_I((BeamInstr *) cp_val(E[2]));
@@ -4794,9 +4794,9 @@ do {						\
 	     } else break;
 	 }
 	 SWAPOUT;		/* Needed for shared heap */
-	 ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	 ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
 	 erts_trace_return_to(c_p, cp_val(*cpp));
-	 ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	 ERTS_REQ_PROC_MAIN_LOCK(c_p);
 	 SWAPIN;
      }
      c_p->cp = NULL;
@@ -5331,7 +5331,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 	}
 
 	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	c_p = erts_schedule(esdp, c_p, reds_used);
 
@@ -5345,7 +5345,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 #ifdef DEBUG
     pid = c_p->common.id; /* Save for debugging purposes */
 #endif
-    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
     PROCESS_MAIN_CHK_LOCKS(c_p);
 
     ASSERT(!(c_p->flags & F_HIPE_MODE));
@@ -5360,7 +5360,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
     else
 	c_p->fcalls = CONTEXT_REDS;
 
-    if (erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_DIRTY_RUNNING_SYS) {
+    if (erts_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_DIRTY_RUNNING_SYS) {
 	erts_execute_dirty_system_task(c_p);
 	goto do_dirty_schedule;
     }
@@ -5431,7 +5431,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 	c_p->current = codemfa;
 	SWAPOUT;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
 
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	if (em_apply_bif == (BeamInstr *) *I) {
@@ -5445,7 +5445,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 	ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
 
 	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	ERTS_REQ_PROC_MAIN_LOCK(c_p);
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
 	if (exiting)
@@ -5624,9 +5624,9 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, ErtsCodeMFA *bif_mfa)
 	}
 	if (c_p->catches > 0) erts_exit(ERTS_ERROR_EXIT, "Catch not found");
     }
-    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
     terminate_proc(c_p, Value);
-    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
     return NULL;
 }
 
@@ -6579,22 +6579,22 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
      * If there are no waiting messages, garbage collect and
      * shrink the heap. 
      */
-    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
-    ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
+    erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
+    ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
     if (!c_p->msg.len) {
-	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
 	c_p->fvalue = NIL;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	erts_garbage_collect_hibernate(c_p);
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
-	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
-        ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
+	erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
+        ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
 	if (!c_p->msg.len)
-	    erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE);
+	    erts_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
     }
-    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
+    erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
     c_p->current = &bif_export[BIF_hibernate_3]->info.mfa;
     c_p->flags |= F_HIBERNATE_SCHED; /* Needed also when woken! */
     return 1;
@@ -6684,7 +6684,7 @@ call_fun(Process* p,		/* Current process. */
 
 		module = fe->module;
 
-		ERTS_SMP_READ_MEMORY_BARRIER;
+		ERTS_THR_READ_MEMORY_BARRIER;
 		if (fe->pend_purge_address) {
 		    /*
 		     * The system is currently trying to purge the
@@ -6815,7 +6815,7 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
     p->htop = hp + needed;
     funp = (ErlFunThing *) hp;
     hp = funp->env;
-    erts_smp_refc_inc(&fe->refc, 2);
+    erts_refc_inc(&fe->refc, 2);
     funp->thing_word = HEADER_FUN;
     funp->next = MSO(p).first;
     MSO(p).first = (struct erl_off_heap_header*) funp;
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 9bb4525306..5429a61d7b 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -794,8 +794,8 @@ erts_finish_loading(Binary* magic, Process* c_p,
      * table which is not protected by any locks.
      */
 
-    ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_has_code_write_permission() ||
-		       erts_smp_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(erts_initialized == 0 || erts_has_code_write_permission() ||
+		       erts_thr_progress_is_blocking());
     /*
      * Make current code for the module old and insert the new code
      * as current.  This will fail if there already exists old code
@@ -830,7 +830,7 @@ erts_finish_loading(Binary* magic, Process* c_p,
 		    continue;
 		} else if (ep->beam[0] ==
 			   (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
-		    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+		    ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
 		    ASSERT(mod_tab_p->curr.num_traced_exports > 0);
 		    erts_clear_export_break(mod_tab_p, &ep->info);
 		    ep->addressv[code_ix] = (BeamInstr *) ep->beam[1];
@@ -4934,7 +4934,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
 		/*
 		 * We are hiding a pointer into older code.
 		 */
-		erts_smp_refc_dec(&fe->refc, 1);
+		erts_refc_dec(&fe->refc, 1);
 	    }
 	    fe->address = code_ptr;
 #ifdef HIPE
@@ -6367,7 +6367,7 @@ patch_funentries(Eterm Patchlist)
     fe = erts_get_fun_entry(Mod, uniq, index);
     fe->native_address = (Uint *)native_address;
 
-    erts_smp_refc_dec(&fe->refc, 1);
+    erts_refc_dec(&fe->refc, 1);
 
     if (!patch(Addresses, (Uint) fe))
       return 0;
diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c
index 9b0335e83d..6e373a3480 100644
--- a/erts/emulator/beam/beam_ranges.c
+++ b/erts/emulator/beam/beam_ranges.c
@@ -29,12 +29,12 @@
 
 typedef struct {
     BeamInstr* start;		/* Pointer to start of module. */
-    erts_smp_atomic_t end; /* (BeamInstr*) Points one word beyond last function in module. */
+    erts_atomic_t end; /* (BeamInstr*) Points one word beyond last function in module. */
 } Range;
 
 /* Range 'end' needs to be atomic as we purge module
     by setting end=start in active code_ix */
-#define RANGE_END(R) ((BeamInstr*)erts_smp_atomic_read_nob(&(R)->end))
+#define RANGE_END(R) ((BeamInstr*)erts_atomic_read_nob(&(R)->end))
 
 static Range* find_range(BeamInstr* pc);
 static void lookup_loc(FunctionInfo* fi, const BeamInstr* pc,
@@ -49,10 +49,10 @@ struct ranges {
     Range* modules;	       /* Sorted lists of module addresses. */
     Sint n;		       /* Number of range entries. */
     Sint allocated;	       /* Number of allocated entries. */
-    erts_smp_atomic_t mid;     /* Cached search start point */
+    erts_atomic_t mid;     /* Cached search start point */
 };
 static struct ranges r[ERTS_NUM_CODE_IX];
-static erts_smp_atomic_t mem_used;
+static erts_atomic_t mem_used;
 static Range* write_ptr;
 
 #ifdef HARD_DEBUG
@@ -90,12 +90,12 @@ erts_init_ranges(void)
 {
     Sint i;
 
-    erts_smp_atomic_init_nob(&mem_used, 0);
+    erts_atomic_init_nob(&mem_used, 0);
     for (i = 0; i < ERTS_NUM_CODE_IX; i++) {
 	r[i].modules = 0;
 	r[i].n = 0;
 	r[i].allocated = 0;
-	erts_smp_atomic_init_nob(&r[i].mid, 0);
+	erts_atomic_init_nob(&r[i].mid, 0);
     }
 }
 
@@ -107,12 +107,12 @@ erts_start_staging_ranges(int num_new)
     Sint need;
 
     if (r[dst].modules) {
-	erts_smp_atomic_add_nob(&mem_used, -r[dst].allocated);
+	erts_atomic_add_nob(&mem_used, -r[dst].allocated);
 	erts_free(ERTS_ALC_T_MODULE_REFS, r[dst].modules);
     }
 
     need = r[dst].allocated = r[src].n + num_new;
-    erts_smp_atomic_add_nob(&mem_used, need);
+    erts_atomic_add_nob(&mem_used, need);
     write_ptr = erts_alloc(ERTS_ALC_T_MODULE_REFS,
 			   need * sizeof(Range));
     r[dst].modules = write_ptr;
@@ -135,7 +135,7 @@ erts_end_staging_ranges(int commit)
 	    if (rp->start < RANGE_END(rp)) {
 		/* Only insert a module that has not been purged. */
 		write_ptr->start = rp->start;
-		erts_smp_atomic_init_nob(&write_ptr->end,
+		erts_atomic_init_nob(&write_ptr->end,
 					 (erts_aint_t)(RANGE_END(rp)));
 		write_ptr++;
 	    }
@@ -161,7 +161,7 @@ erts_end_staging_ranges(int commit)
 	}
 	r[dst].modules = mp;
 	CHECK(&r[dst]);
-	erts_smp_atomic_set_nob(&r[dst].mid,
+	erts_atomic_set_nob(&r[dst].mid,
 				(erts_aint_t) (r[dst].modules +
 					       r[dst].n / 2));
     }
@@ -182,7 +182,7 @@ erts_update_ranges(BeamInstr* code, Uint size)
 	 */
 	if (r[dst].modules == NULL) {
 	    Sint need = 128;
-	    erts_smp_atomic_add_nob(&mem_used, need);
+	    erts_atomic_add_nob(&mem_used, need);
 	    r[dst].modules = erts_alloc(ERTS_ALC_T_MODULE_REFS,
 					need * sizeof(Range));
 	    r[dst].allocated = need;
@@ -192,7 +192,7 @@ erts_update_ranges(BeamInstr* code, Uint size)
 
     ASSERT(r[dst].modules);
     write_ptr->start = code;
-    erts_smp_atomic_init_nob(&(write_ptr->end),
+    erts_atomic_init_nob(&(write_ptr->end),
 			     (erts_aint_t)(((byte *)code) + size));
     write_ptr++;
 }
@@ -201,13 +201,13 @@ void
 erts_remove_from_ranges(BeamInstr* code)
 {
     Range* rp = find_range(code);
-    erts_smp_atomic_set_nob(&rp->end, (erts_aint_t)rp->start);
+    erts_atomic_set_nob(&rp->end, (erts_aint_t)rp->start);
 }
 
 UWord
 erts_ranges_sz(void)
 {
-    return erts_smp_atomic_read_nob(&mem_used) * sizeof(Range);
+    return erts_atomic_read_nob(&mem_used) * sizeof(Range);
 }
 
 /*
@@ -262,7 +262,7 @@ find_range(BeamInstr* pc)
     ErtsCodeIndex active = erts_active_code_ix();
     Range* low = r[active].modules;
     Range* high = low + r[active].n;
-    Range* mid = (Range *) erts_smp_atomic_read_nob(&r[active].mid);
+    Range* mid = (Range *) erts_atomic_read_nob(&r[active].mid);
 
     CHECK(&r[active]);
     while (low < high) {
@@ -271,7 +271,7 @@ find_range(BeamInstr* pc)
 	} else if (pc >= RANGE_END(mid)) {
 	    low = mid + 1;
 	} else {
-	    erts_smp_atomic_set_nob(&r[active].mid, (erts_aint_t) mid);
+	    erts_atomic_set_nob(&r[active].mid, (erts_aint_t) mid);
 	    return mid;
 	}
 	mid = low + (high-low) / 2;
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index e9a0668e2d..4b45e98685 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -57,10 +57,10 @@ static Export dsend_continue_trap_export;
 Export *erts_convert_time_unit_trap = NULL;
 
 static Export *await_msacc_mod_trap = NULL;
-static erts_smp_atomic32_t msacc;
+static erts_atomic32_t msacc;
 
 static Export *await_sched_wall_time_mod_trap;
-static erts_smp_atomic32_t sched_wall_time;
+static erts_atomic32_t sched_wall_time;
 
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
@@ -103,7 +103,7 @@ static int insert_internal_link(Process* p, Eterm rpid)
 	rp_locks = ERTS_PROC_LOCKS_ALL;
     }
 
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_LINK);
+    erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
 
     /* get a pointer to the process struct of the linked process */
     rp = erts_pid2proc_opt(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK,
@@ -111,7 +111,7 @@ static int insert_internal_link(Process* p, Eterm rpid)
 			   ERTS_P2P_FLG_ALLOW_OTHER_X);
 
     if (!rp) {
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 	return 0;
     }
 
@@ -137,10 +137,10 @@ static int insert_internal_link(Process* p, Eterm rpid)
                    rp, am_getting_linked, p->common.id);
 
     if (p == rp)
-	erts_smp_proc_unlock(p, rp_locks & ~ERTS_PROC_LOCK_MAIN);
+	erts_proc_unlock(p, rp_locks & ~ERTS_PROC_LOCK_MAIN);
     else {
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
-	erts_smp_proc_unlock(rp, rp_locks);
+	erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(rp, rp_locks);
     }
 
     return 1;
@@ -176,13 +176,13 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
 	    goto res_no_proc;
 	}
 
-	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
+	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
 	
 	if (erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, BIF_ARG_1) >= 0)
 	    send_link_signal = 1;
 	/* else: already linked */
 
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
 
 	if (send_link_signal) {
 	    Eterm ref;
@@ -210,11 +210,11 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
 
     if (is_external_pid(BIF_ARG_1)) {
 
-	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
+	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
 
 	/* We may earn time by checking first that we're not linked already */
 	if (erts_lookup_link(ERTS_P_LINKS(BIF_P), BIF_ARG_1) != NULL) {
-	    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
+	    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
 	    BIF_RET(am_true);
 	}
 	else {
@@ -223,7 +223,7 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
 	    ErtsDSigData dsd;
 	    dep = external_pid_dist_entry(BIF_ARG_1);
 	    if (dep == erts_this_dist_entry) {
-		erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
+		erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
 		goto res_no_proc;
 	    }
 
@@ -232,13 +232,13 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
 	    case ERTS_DSIG_PREP_NOT_ALIVE:
 		/* Let the dlink trap handle it */
 	    case ERTS_DSIG_PREP_NOT_CONNECTED:
-		erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
+		erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
 		BIF_TRAP1(dlink_trap, BIF_P, BIF_ARG_1);
 
 	    case ERTS_DSIG_PREP_CONNECTED:
 		/* We are connected. Setup link and send link signal */
 
-		erts_smp_de_links_lock(dep);
+		erts_de_links_lock(dep);
 
 		erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, BIF_ARG_1);
 		lnk = erts_add_or_lookup_link(&(dep->nlinks),
@@ -247,9 +247,9 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
 		ASSERT(lnk != NULL);
 		erts_add_link(&ERTS_LINK_ROOT(lnk), LINK_PID, BIF_ARG_1);
 
-		erts_smp_de_links_unlock(dep);
-		erts_smp_de_runlock(dep);
-		erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
+		erts_de_links_unlock(dep);
+		erts_de_runlock(dep);
+		erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
 
 		code = erts_dsig_send_link(&dsd, BIF_P->common.id, BIF_ARG_1);
 		if (code == ERTS_DSIG_SEND_YIELD)
@@ -265,11 +265,11 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
     BIF_ERROR(BIF_P, BADARG);
 
 res_no_proc: {
-	erts_aint32_t state = erts_smp_atomic32_read_nob(&BIF_P->state);
+	erts_aint32_t state = erts_atomic32_read_nob(&BIF_P->state);
 	if (state & ERTS_PSFLG_TRAP_EXIT) {
 	    ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN;
 	    erts_deliver_exit_message(BIF_ARG_1, BIF_P, &locks, am_noproc, NIL);
-	    erts_smp_proc_unlock(BIF_P, ~ERTS_PROC_LOCK_MAIN & locks);
+	    erts_proc_unlock(BIF_P, ~ERTS_PROC_LOCK_MAIN & locks);
 	    BIF_RET(am_true);
 	}
 	else
@@ -289,7 +289,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
     int code;
     Eterm res = am_false;
 
-    ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK)
+    ERTS_LC_ASSERT((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK)
 		       == erts_proc_lc_my_proc_locks(c_p));
 
     code = erts_dsig_prepare(&dsd, dep, c_p, ERTS_DSP_RLOCK, 0);
@@ -301,26 +301,26 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
 	 * down just before the call to demonitor.
 	 */
 	if (dep) {
-	    erts_smp_de_links_lock(dep);
+	    erts_de_links_lock(dep);
 	    dmon = erts_remove_monitor(&dep->monitors, ref);
-	    erts_smp_de_links_unlock(dep);
+	    erts_de_links_unlock(dep);
 	    if (dmon)
 		erts_destroy_monitor(dmon);
 	}
 	mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref);
-	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(c_p, ERTS_PROC_LOCK_LINK);
 
         res = am_true;
 	break;
 
     case ERTS_DSIG_PREP_CONNECTED:
 
-	erts_smp_de_links_lock(dep);
+	erts_de_links_lock(dep);
 	mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref);
 	dmon = erts_remove_monitor(&dep->monitors, ref);
-	erts_smp_de_links_unlock(dep);
-	erts_smp_de_runlock(dep);
-	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_LINK);
+	erts_de_links_unlock(dep);
+	erts_de_runlock(dep);
+	erts_proc_unlock(c_p, ERTS_PROC_LOCK_LINK);
 
 	if (!dmon) {
 	    /*
@@ -360,7 +360,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
 		    lookup and remove */
     erts_destroy_monitor(mon);
 
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
     return res;
 }
 
@@ -385,12 +385,12 @@ demonitor_local_process(Process *c_p, Eterm ref, Eterm to, Eterm *res)
         ErtsMonitor *rmon;
         rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref);
         if (rp != c_p)
-            erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+            erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
         if (rmon != NULL)
             erts_destroy_monitor(rmon);
     }
     else {
-        ERTS_SMP_ASSERT_IS_NOT_EXITING(c_p);
+        ERTS_ASSERT_IS_NOT_EXITING(c_p);
     }
 }
 
@@ -403,7 +403,7 @@ demonitor_local_port(Process *origin, Eterm ref, Eterm target)
     if (!port) {
         BIF_ERROR(origin, BADARG);
     }
-    erts_smp_proc_unlock(origin, ERTS_PROC_LOCK_LINK);
+    erts_proc_unlock(origin, ERTS_PROC_LOCK_LINK);
 
     if (port) {
         Eterm trap_ref;
@@ -423,7 +423,7 @@ demonitor_local_port(Process *origin, Eterm ref, Eterm target)
         }
     }
     else {
-        ERTS_SMP_ASSERT_IS_NOT_EXITING(origin);
+        ERTS_ASSERT_IS_NOT_EXITING(origin);
     }
     BIF_RET(res);
 }
@@ -441,7 +441,7 @@ BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip)
    BIF_RETTYPE  res = am_false;
    int          unlock_link = 1;
 
-   erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_LINK);
+   erts_proc_lock(c_p, ERTS_PROC_LOCK_LINK);
 
    if (is_not_internal_ref(ref)) {
        res = am_badarg;
@@ -503,14 +503,14 @@ badarg:
 done:
 
    if (unlock_link)
-       erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_LINK);
+       erts_proc_unlock(c_p, ERTS_PROC_LOCK_LINK);
 
    if (deref_de) {
        ASSERT(dep);
        erts_deref_dist_entry(dep);
    }
 
-   ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
+   ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
    BIF_RET(res);
 }
 
@@ -633,12 +633,12 @@ local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int boolean)
 	return ret;
     }
 
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_LINK);
+    erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
     rp = erts_pid2proc_opt(p, p_locks,
 			   target, ERTS_PROC_LOCK_LINK,
 			   ERTS_P2P_FLG_ALLOW_OTHER_X);
     if (!rp) {
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 	p_locks &= ~ERTS_PROC_LOCK_LINK;
 	if (boolean)
 	    ret = am_false;
@@ -655,10 +655,10 @@ local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int boolean)
 	erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, target, NIL);
 	erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, mon_ref, p->common.id, NIL);
 
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
     }
 
-    erts_smp_proc_unlock(p, p_locks & ~ERTS_PROC_LOCK_MAIN);
+    erts_proc_unlock(p, p_locks & ~ERTS_PROC_LOCK_MAIN);
 
     return ret;
 }
@@ -692,7 +692,7 @@ res_no_proc:
             break;
         }
     }
-    erts_smp_proc_unlock(origin, p_locks & ~ERTS_PROC_LOCK_MAIN);
+    erts_proc_unlock(origin, p_locks & ~ERTS_PROC_LOCK_MAIN);
     BIF_RET(ref);
 }
 
@@ -707,7 +707,7 @@ local_name_monitor(Process *self, Eterm type, Eterm target_name)
     Process     *proc = NULL;
     Port        *port = NULL;
 
-    erts_smp_proc_lock(self, ERTS_PROC_LOCK_LINK);
+    erts_proc_lock(self, ERTS_PROC_LOCK_LINK);
 
     erts_whereis_name(self, p_locks, target_name,
                       &proc, ERTS_PROC_LOCK_LINK,
@@ -726,7 +726,7 @@ local_name_monitor(Process *self, Eterm type, Eterm target_name)
 	Eterm item;
         UseTmpHeap(3,self);
 
-        erts_smp_proc_unlock(self, ERTS_PROC_LOCK_LINK);
+        erts_proc_unlock(self, ERTS_PROC_LOCK_LINK);
 	p_locks &= ~ERTS_PROC_LOCK_LINK;
 
 	item = TUPLE2(lhp, target_name, erts_this_dist_entry->sysname);
@@ -737,7 +737,7 @@ local_name_monitor(Process *self, Eterm type, Eterm target_name)
         UnUseTmpHeap(3,self);
     }
     else if (port) {
-        erts_smp_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN);
+        erts_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN);
         p_locks &= ~ERTS_PROC_LOCK_MAIN;
 
         switch (erts_port_monitor(self, port, target_name, &ret)) {
@@ -758,16 +758,16 @@ local_name_monitor(Process *self, Eterm type, Eterm target_name)
                          proc->common.id, target_name);
         erts_add_monitor(&ERTS_P_MONITORS(proc), MON_TARGET, ret,
                          self->common.id, target_name);
-        erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_LINK);
+        erts_proc_unlock(proc, ERTS_PROC_LOCK_LINK);
     }
 
     if (p_locks) {
-        erts_smp_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN);
+        erts_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN);
     }
     BIF_RET(ret);
 badarg:
     if (p_locks) {
-        erts_smp_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN);
+        erts_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN);
     }
     BIF_ERROR(self, BADARG);
 }
@@ -780,20 +780,20 @@ remote_monitor(Process *p, Eterm bifarg1, Eterm bifarg2,
     BIF_RETTYPE ret;
     int code;
 
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_LINK);
+    erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
     code = erts_dsig_prepare(&dsd, dep, p, ERTS_DSP_RLOCK, 0);
     switch (code) {
     case ERTS_DSIG_PREP_NOT_ALIVE:
 	/* Let the dmonitor_p trap handle it */
     case ERTS_DSIG_PREP_NOT_CONNECTED:
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 	ERTS_BIF_PREP_TRAP2(ret, dmonitor_p_trap, p, bifarg1, bifarg2);
 	break;
     case ERTS_DSIG_PREP_CONNECTED:
 	if (!(dep->flags & DFLAG_DIST_MONITOR)
 	    || (byname && !(dep->flags & DFLAG_DIST_MONITOR_NAME))) {
-	    erts_smp_de_runlock(dep);
-	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+	    erts_de_runlock(dep);
+	    erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 	    ERTS_BIF_PREP_ERROR(ret, p, BADARG);
 	}
 	else {
@@ -812,16 +812,16 @@ remote_monitor(Process *p, Eterm bifarg1, Eterm bifarg2,
 		d_name = NIL;
 	    }
 
-	    erts_smp_de_links_lock(dep);
+	    erts_de_links_lock(dep);
 
 	    erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, p_trgt,
 			     p_name);
 	    erts_add_monitor(&(dep->monitors), MON_TARGET, mon_ref, p->common.id,
 			     d_name);
 
-	    erts_smp_de_links_unlock(dep);
-	    erts_smp_de_runlock(dep);
-	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+	    erts_de_links_unlock(dep);
+	    erts_de_runlock(dep);
+	    erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 
 	    code = erts_dsig_send_monitor(&dsd, p->common.id, target, mon_ref);
 	    if (code == ERTS_DSIG_SEND_YIELD)
@@ -854,10 +854,10 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2)
             goto badarg;
         }
 	ref = erts_make_ref(BIF_P);
-	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
+	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
 	erts_add_monitor(&ERTS_P_MONITORS(BIF_P), MON_TIME_OFFSET,
 			 ref, am_clock_service, NIL);
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
 	erts_monitor_time_offset(BIF_P->common.id, ref);
 	BIF_RET(ref);
     }
@@ -974,7 +974,7 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1)
     so.max_heap_size  = H_MAX_SIZE;
     so.max_heap_flags = H_MAX_FLAGS;
     so.priority       = PRIORITY_NORMAL;
-    so.max_gen_gcs    = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
+    so.max_gen_gcs    = (Uint16) erts_atomic32_read_nob(&erts_max_gen_gcs);
     so.scheduler      = 0;
 
     /*
@@ -1115,13 +1115,13 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
     }
 
     if (is_internal_port(BIF_ARG_1)) {
-	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
+	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
 	if (ERTS_PROC_PENDING_EXIT(BIF_P))
 	    goto handle_pending_exit;
 
 	l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1);
 
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
 
 	if (l) {
 	    Port *prt;
@@ -1163,12 +1163,12 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 	/* Blind removal, we might have trapped or anything, this leaves
 	   us in a state where monitors might be inconsistent, but the dist
 	   code should take care of it. */
-	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
+	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
 	if (ERTS_PROC_PENDING_EXIT(BIF_P))
 	    goto handle_pending_exit;
 	l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1);
 
-	erts_smp_proc_unlock(BIF_P,
+	erts_proc_unlock(BIF_P,
 			     ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
 
 	if (l)
@@ -1210,7 +1210,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 
     /* Internal pid... */
 
-    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
+    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
 
     cp_locks |= ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS;
 
@@ -1221,7 +1221,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 
     if (ERTS_PROC_PENDING_EXIT(BIF_P)) {
 	if (rp && rp != BIF_P)
-	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+	    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 	goto handle_pending_exit;
     }
 
@@ -1231,7 +1231,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 	erts_destroy_link(l);
 
     if (!rp) {
-	ERTS_SMP_ASSERT_IS_NOT_EXITING(BIF_P);
+	ERTS_ASSERT_IS_NOT_EXITING(BIF_P);
     }
     else {
 	rl = erts_remove_link(&ERTS_P_LINKS(rp), BIF_P->common.id);
@@ -1239,17 +1239,17 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 	    erts_destroy_link(rl);
 
 	if (IS_TRACED_FL(rp, F_TRACE_PROCS) && rl != NULL) {
-            erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS);
+            erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS);
             cp_locks &= ~ERTS_PROC_LOCK_STATUS;
 	    trace_proc(BIF_P, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK),
                        rp, am_getting_unlinked, BIF_P->common.id);
 	}
 
 	if (rp != BIF_P)
-	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+	    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
     }
  
-    erts_smp_proc_unlock(BIF_P, cp_locks & ~ERTS_PROC_LOCK_MAIN);
+    erts_proc_unlock(BIF_P, cp_locks & ~ERTS_PROC_LOCK_MAIN);
 
     BIF_RET(am_true);
 
@@ -1258,7 +1258,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 				     | ERTS_PROC_LOCK_LINK
 				     | ERTS_PROC_LOCK_STATUS));
     ASSERT(ERTS_PROC_IS_EXITING(BIF_P)); 
-    erts_smp_proc_unlock(BIF_P,	ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
+    erts_proc_unlock(BIF_P,	ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
     ERTS_BIF_EXITED(BIF_P);
 }
 
@@ -1592,7 +1592,7 @@ BIF_RETTYPE exit_2(BIF_ALIST_2)
 	 if (BIF_ARG_1 == BIF_P->common.id) {
 	     rp_locks = ERTS_PROC_LOCKS_ALL;
 	     rp = BIF_P;
-	     erts_smp_proc_lock(rp, ERTS_PROC_LOCKS_ALL_MINOR);
+	     erts_proc_lock(rp, ERTS_PROC_LOCKS_ALL_MINOR);
 	 }
 	 else {
 	     rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
@@ -1617,7 +1617,7 @@ BIF_RETTYPE exit_2(BIF_ALIST_2)
 	 if (rp == BIF_P)
 	     rp_locks &= ~ERTS_PROC_LOCK_MAIN;
 	 if (rp_locks)
-	     erts_smp_proc_unlock(rp, rp_locks);
+	     erts_proc_unlock(rp, rp_locks);
 	 /*
 	  * We may have exited ourselves and may have to take action.
 	  */
@@ -1729,14 +1729,14 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
 	*       true. For more info, see implementation of
 	*       erts_send_exit_signal().
 	*/
-       erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND);
+       erts_proc_lock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND);
        if (trap_exit)
-	   state = erts_smp_atomic32_read_bor_mb(&BIF_P->state,
+	   state = erts_atomic32_read_bor_mb(&BIF_P->state,
 						 ERTS_PSFLG_TRAP_EXIT);
        else
-	   state = erts_smp_atomic32_read_band_mb(&BIF_P->state,
+	   state = erts_atomic32_read_band_mb(&BIF_P->state,
 						  ~ERTS_PSFLG_TRAP_EXIT);
-       erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND);
+       erts_proc_unlock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND);
 
        if (state & ERTS_PSFLG_PENDING_EXIT) {
 	   erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
@@ -1759,13 +1759,13 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
 
        if (sched == 0) {
 	   new = NULL;
-	   state = erts_smp_atomic32_read_band_mb(&BIF_P->state,
+	   state = erts_atomic32_read_band_mb(&BIF_P->state,
 						  ~ERTS_PSFLG_BOUND);
        }
        else {
 	   new = erts_schedid2runq(sched);
 	   erts_atomic_set_nob(&BIF_P->run_queue, (erts_aint_t) new);
-	   state = erts_smp_atomic32_read_bor_mb(&BIF_P->state,
+	   state = erts_atomic32_read_bor_mb(&BIF_P->state,
 						 ERTS_PSFLG_BOUND);
        }
 
@@ -1846,7 +1846,7 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
        } else {
 	   goto error;
        }
-       erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR);
+       erts_proc_lock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR);
        old_value = (ERTS_TRACE_FLAGS(BIF_P) & F_SENSITIVE
 		    ? am_true
 		    : am_false);
@@ -1855,7 +1855,7 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
        } else {
 	   ERTS_TRACE_FLAGS(BIF_P) &= ~F_SENSITIVE;
        }
-       erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR);
+       erts_proc_unlock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR);
        /* make sure to bump all reds so that we get
           rescheduled immediately so setting takes effect */
        BIF_RET2(old_value, CONTEXT_REDS);
@@ -1909,7 +1909,7 @@ BIF_RETTYPE process_flag_3(BIF_ALIST_3)
    res = process_flag_aux(BIF_P, rp, BIF_ARG_2, BIF_ARG_3);
 
    if (rp != BIF_P)
-       erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
+       erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
 
    return res;
 }
@@ -2251,7 +2251,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx)
 	    res *= 4;
 	else
 	    res = 0;
-	erts_smp_proc_unlock(rp,
+	erts_proc_unlock(rp,
 			     p == rp
 			     ? (rp_locks & ~ERTS_PROC_LOCK_MAIN)
 			     : rp_locks);
@@ -3954,14 +3954,14 @@ BIF_RETTYPE halt_2(BIF_ALIST_2)
 	    ERTS_BIF_YIELD2(bif_export[BIF_halt_2], BIF_P, am_undefined, am_undefined);
 	}
 	else {
-	    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
             erts_exit(pos_int_code, "");
 	}
     }
     else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) {
 	VERBOSE(DEBUG_SYSTEM,
 		("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2));
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	erts_exit(ERTS_ABORT_EXIT, "");
     }
     else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) {
@@ -3976,7 +3976,7 @@ BIF_RETTYPE halt_2(BIF_ALIST_2)
 	halt_msg[i] = '\0';
 	VERBOSE(DEBUG_SYSTEM,
 		("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2));
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg);
     }
     else
@@ -4450,9 +4450,9 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2)
 		new_member->group_leader = BIF_ARG_1;
 	    else {
 		locks &= ~ERTS_PROC_LOCK_STATUS;
-		erts_smp_proc_unlock(new_member, ERTS_PROC_LOCK_STATUS);
+		erts_proc_unlock(new_member, ERTS_PROC_LOCK_STATUS);
 		if (new_member == BIF_P
-		    || !(erts_smp_atomic32_read_nob(&new_member->state)
+		    || !(erts_atomic32_read_nob(&new_member->state)
 			 & ERTS_PSFLG_DIRTY_RUNNING)) {
 		    new_member->group_leader = STORE_NC_IN_PROC(new_member,
 								BIF_ARG_1);
@@ -4481,7 +4481,7 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2)
 	if (new_member == BIF_P)
 	    locks &= ~ERTS_PROC_LOCK_MAIN;
 	if (locks)
-	    erts_smp_proc_unlock(new_member, locks);
+	    erts_proc_unlock(new_member, locks);
 
 	if (await_x) {
 	    /* Wait for new_member to terminate; then badarg */
@@ -4575,7 +4575,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	    goto error;
 	}
 	nval = (n > (Sint) ((Uint16) -1)) ? ((Uint16) -1) : ((Uint16) n);
-	oval = (Uint) erts_smp_atomic32_xchg_nob(&erts_max_gen_gcs,
+	oval = (Uint) erts_atomic32_xchg_nob(&erts_max_gen_gcs,
 						 (erts_aint32_t) nval);
 	BIF_RET(make_small(oval));
     } else if (BIF_ARG_1 == am_min_heap_size) {
@@ -4585,13 +4585,13 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	    goto error;
 	}
 
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_smp_thr_progress_block();
+	erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_block();
 
 	H_MIN_SIZE = erts_next_heap_size(n, 0);
 
-	erts_smp_thr_progress_unblock();
-	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_unblock();
+	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
 	BIF_RET(make_small(oval));
     } else if (BIF_ARG_1 == am_min_bin_vheap_size) {
@@ -4601,13 +4601,13 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	    goto error;
 	}
 
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_smp_thr_progress_block();
+	erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_block();
 
 	BIN_VH_MIN_SIZE = erts_next_heap_size(n, 0);
 
-	erts_smp_thr_progress_unblock();
-	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_unblock();
+	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
 	BIF_RET(make_small(oval));
     } else if (BIF_ARG_1 == am_max_heap_size) {
@@ -4625,14 +4625,14 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
         hp = HAlloc(BIF_P, sz);
         old_value = erts_max_heap_size_map(H_MAX_SIZE, H_MAX_FLAGS, &hp, NULL);
 
-        erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-        erts_smp_thr_progress_block();
+        erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+        erts_thr_progress_block();
 
         H_MAX_SIZE = max_heap_size;
         H_MAX_FLAGS = max_heap_flags;
 
-        erts_smp_thr_progress_unblock();
-        erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+        erts_thr_progress_unblock();
+        erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
         BIF_RET(old_value);
     } else if (BIF_ARG_1 == am_display_items) {
@@ -4686,8 +4686,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
     } else if (BIF_ARG_1 == make_small(1)) {
 	int i, max;
 	ErtsMessage* mp;
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_smp_thr_progress_block();
+	erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_block();
 
 	max = erts_ptab_max(&erts_proc);
 	for (i = 0; i < max; i++) {
@@ -4700,7 +4700,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 #endif
 		p->seq_trace_clock = 0;
 		p->seq_trace_lastcnt = 0;
-		ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
+		ERTS_MSGQ_MV_INQ2PRIVQ(p);
 		mp = p->msg.first;
 		while(mp != NULL) {
 #ifdef USE_VM_PROBES
@@ -4713,14 +4713,14 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	    }
 	}
 
-	erts_smp_thr_progress_unblock();
-	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_unblock();
+	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
 	BIF_RET(am_true);
     } else if (BIF_ARG_1 == am_scheduler_wall_time) {
 	if (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) {
 	    erts_aint32_t new = BIF_ARG_2 == am_true ? 1 : 0;
-	    erts_aint32_t old = erts_smp_atomic32_xchg_nob(&sched_wall_time,
+	    erts_aint32_t old = erts_atomic32_xchg_nob(&sched_wall_time,
 							   new);
 	    Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new, 0, 0);
 	    ASSERT(is_value(ref));
@@ -4759,9 +4759,9 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
     } else if (BIF_ARG_1 == am_time_offset
 	       && ERTS_IS_ATOM_STR("finalize", BIF_ARG_2)) {
 	ErtsTimeOffsetState res;
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	res = erts_finalize_time_offset();
-        erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+        erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	switch (res) {
 	case ERTS_TIME_OFFSET_PRELIMINARY: {
 	    DECL_AM(preliminary);
@@ -4783,7 +4783,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
       Eterm threads;
       if (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) {
         erts_aint32_t new = BIF_ARG_2 == am_true ? ERTS_MSACC_ENABLE : ERTS_MSACC_DISABLE;
-	erts_aint32_t old = erts_smp_atomic32_xchg_nob(&msacc, new);
+	erts_aint32_t old = erts_atomic32_xchg_nob(&msacc, new);
 	Eterm ref = erts_msacc_request(BIF_P, new, &threads);
         if (is_non_value(ref))
             BIF_RET(old ? am_true : am_false);
@@ -4794,7 +4794,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 		  threads);
       } else if (BIF_ARG_2 == am_reset) {
 	Eterm ref = erts_msacc_request(BIF_P, ERTS_MSACC_RESET, &threads);
-	erts_aint32_t old = erts_smp_atomic32_read_nob(&msacc);
+	erts_aint32_t old = erts_atomic32_read_nob(&msacc);
 	ASSERT(is_value(ref));
 	BIF_TRAP3(await_msacc_mod_trap,
 		  BIF_P,
@@ -4813,9 +4813,9 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	    what = ERTS_SCHED_STAT_MODIFY_CLEAR;
 	else
 	    goto error;
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	erts_sched_stat_modify(what);
-	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	BIF_RET(am_true);
     } else if (ERTS_IS_ATOM_STR("internal_cpu_topology", BIF_ARG_1)) {
 	Eterm res = erts_set_cpu_topology(BIF_P, BIF_ARG_2);
@@ -4991,18 +4991,18 @@ skip_current_msgq(Process *c_p)
     erts_proc_lc_chk_only_proc_main(c_p);
 #endif
 
-    erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+    erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
     if (ERTS_PROC_PENDING_EXIT(c_p)) {
 	KILL_CATCHES(c_p);
 	c_p->freason = EXC_EXIT;
 	res = 0;
     }
     else {
-        ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
+        ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
 	c_p->msg.save = c_p->msg.last;
 	res = 1;
     }
-    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
     return res;
 }
 
@@ -5105,8 +5105,8 @@ void erts_init_bif(void)
     await_msacc_mod_trap
 	= erts_export_put(am_erts_internal, am_await_microstate_accounting_modifications, 3);
 
-    erts_smp_atomic32_init_nob(&sched_wall_time, 0);
-    erts_smp_atomic32_init_nob(&msacc, ERTS_MSACC_IS_ENABLED());
+    erts_atomic32_init_nob(&sched_wall_time, 0);
+    erts_atomic32_init_nob(&msacc, ERTS_MSACC_IS_ENABLED());
 }
 
 /*
@@ -5124,7 +5124,7 @@ schedule(Process *c_p, Process *dirty_shadow_proc,
 	 Eterm module, Eterm function,
 	 int argc, Eterm *argv)
 {
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
     (void) erts_nif_export_schedule(c_p, dirty_shadow_proc,
 				    mfa, pc, (BeamInstr) em_apply_bif,
 				    dfunc, ifunc,
@@ -5198,7 +5198,7 @@ erts_schedule_bif(Process *proc,
 	dirty_shadow_proc = proc;
 	c_p = proc->next;
 	ASSERT(c_p->common.id == dirty_shadow_proc->common.id);
-	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+	erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
     }
     else
 #endif
@@ -5240,7 +5240,7 @@ erts_schedule_bif(Process *proc,
 	    break;
 	}
 
-	(void) erts_smp_atomic32_read_bset_nob(&c_p->state, mask, set);
+	(void) erts_atomic32_read_bset_nob(&c_p->state, mask, set);
 #else
 	dbif = call_bif;
 	ibif = bif;
@@ -5285,7 +5285,7 @@ erts_schedule_bif(Process *proc,
     }
 
     if (dirty_shadow_proc)
-	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+	erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
 
     return THE_NON_VALUE;
 }
@@ -5335,7 +5335,7 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *
     erts_aint32_t state;
 
     ASSERT(!c_p->scheduler_data);
-    state = erts_smp_atomic32_read_nob(&c_p->state);
+    state = erts_atomic32_read_nob(&c_p->state);
     ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING)
 	   && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)));
     ASSERT(esdp);
@@ -5349,7 +5349,7 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *
 
     bf = (ErtsBifFunc) I[1];
 
-    erts_smp_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC
+    erts_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC
 						  | ERTS_PSFLG_DIRTY_IO_PROC));
 
     dirty_shadow_proc = erts_make_dirty_shadow_proc(esdp, c_p);
@@ -5364,11 +5364,11 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *
     c_p_htop = c_p->htop;
 #endif
 
-    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+    erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
 
     result = (*bf)(dirty_shadow_proc, reg, I);
 
-    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+    erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 
     ASSERT(c_p_htop == c_p->htop);
     ASSERT(dirty_shadow_proc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
@@ -5391,7 +5391,7 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *
 	}
 	else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) {
 	    /* Dirty BIF did an ordinary trap... */
-	    ASSERT(!(erts_smp_atomic32_read_nob(&c_p->state)
+	    ASSERT(!(erts_atomic32_read_nob(&c_p->state)
 		     & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)));
 	    schedule(c_p, dirty_shadow_proc, NULL, NULL,
 		     dirty_bif_trap, (void *) dirty_shadow_proc->i,
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 01cca90a7a..9b0870dee2 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -93,7 +93,7 @@ do {									\
 
 #define BUMP_REDS(p, gc) do {			   \
      ASSERT(p);		 			   \
-     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));\
+     ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));\
      (p)->fcalls -= (gc); 			   \
      if ((p)->fcalls < 0) { 			   \
 	if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p)))   \
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 9bcf50653d..35b2365655 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -109,8 +109,8 @@ process_killer(void)
 		    ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
 		    erts_aint32_t state;
 		    erts_proc_inc_refc(rp);
-		    erts_smp_proc_lock(rp, rp_locks);
-		    state = erts_smp_atomic32_read_acqb(&rp->state);
+		    erts_proc_lock(rp, rp_locks);
+		    state = erts_atomic32_read_acqb(&rp->state);
 		    if (state & (ERTS_PSFLG_FREE
 				 | ERTS_PSFLG_EXITING
 				 | ERTS_PSFLG_ACTIVE
@@ -132,7 +132,7 @@ process_killer(void)
 						     NULL,
 						     0);
 		    }
-		    erts_smp_proc_unlock(rp, rp_locks);
+		    erts_proc_unlock(rp, rp_locks);
 		    erts_proc_dec_refc(rp);
 		}
 		case 'n': br = 1; break;
@@ -219,7 +219,7 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p)
     /* Display the state */
     erts_print(to, to_arg, "State: ");
 
-    state = erts_smp_atomic32_read_acqb(&p->state);
+    state = erts_atomic32_read_acqb(&p->state);
     erts_dump_process_state(to, to_arg, state);
     if (state & ERTS_PSFLG_GC) {
         garbing = 1;
@@ -258,7 +258,7 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p)
     erts_print(to, to_arg, "Spawned by: %T\n", p->parent);
     approx_started = (time_t) p->approx_started;
     erts_print(to, to_arg, "Started: %s", ctime(&approx_started));
-    ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
+    ERTS_MSGQ_MV_INQ2PRIVQ(p);
     erts_print(to, to_arg, "Message queue length: %d\n", p->msg.len);
 
     /* display the message queue only if there is anything in it */
@@ -508,7 +508,7 @@ do_break(void)
     erts_free_read_env(mode);
 #endif /* __WIN32__ */
 
-    ASSERT(erts_smp_thr_progress_is_blocking());
+    ASSERT(erts_thr_progress_is_blocking());
 
     erts_printf("\n"
 		"BREAK: (a)bort (c)ontinue (p)roc info (i)nfo (l)oaded\n"
@@ -734,8 +734,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
 #endif
 
     /* Allow us to pass certain places without locking... */
-    erts_smp_atomic32_set_mb(&erts_writing_erl_crash_dump, 1);
-    erts_smp_tsd_set(erts_is_crash_dumping_key, (void *) 1);
+    erts_atomic32_set_mb(&erts_writing_erl_crash_dump, 1);
+    erts_tsd_set(erts_is_crash_dumping_key, (void *) 1);
 
 
     envsz = sizeof(env);
diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c
index ba68d612e3..34e46f5f33 100644
--- a/erts/emulator/beam/code_ix.c
+++ b/erts/emulator/beam/code_ix.c
@@ -34,8 +34,8 @@
 # define CIX_TRACE(text)
 #endif
 
-erts_smp_atomic32_t the_active_code_index;
-erts_smp_atomic32_t the_staging_code_index;
+erts_atomic32_t the_active_code_index;
+erts_atomic32_t the_staging_code_index;
 
 static Process* code_writing_process = NULL;
 struct code_write_queue_item {
@@ -43,7 +43,7 @@ struct code_write_queue_item {
     struct code_write_queue_item* next;
 };
 static struct code_write_queue_item* code_write_queue = NULL;
-static erts_smp_mtx_t code_write_permission_mtx;
+static erts_mtx_t code_write_permission_mtx;
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
 static erts_tsd_key_t has_code_write_permission;
@@ -55,9 +55,9 @@ void erts_code_ix_init(void)
      * single threaded with active and staging set both to zero.
      * Preloading is finished by a commit that will set things straight.
      */
-    erts_smp_atomic32_init_nob(&the_active_code_index, 0);
-    erts_smp_atomic32_init_nob(&the_staging_code_index, 0);
-    erts_smp_mtx_init(&code_write_permission_mtx, "code_write_permission", NIL,
+    erts_atomic32_init_nob(&the_active_code_index, 0);
+    erts_atomic32_init_nob(&the_staging_code_index, 0);
+    erts_mtx_init(&code_write_permission_mtx, "code_write_permission", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_tsd_key_create(&has_code_write_permission,
@@ -91,9 +91,9 @@ void erts_commit_staging_code_ix(void)
     /* We need to this lock as we are now making the staging export table active */
     export_staging_lock();
     ix = erts_staging_code_ix();
-    erts_smp_atomic32_set_nob(&the_active_code_index, ix);
+    erts_atomic32_set_nob(&the_active_code_index, ix);
     ix = (ix + 1) % ERTS_NUM_CODE_IX;
-    erts_smp_atomic32_set_nob(&the_staging_code_index, ix);
+    erts_atomic32_set_nob(&the_staging_code_index, ix);
     export_staging_unlock();
     erts_tracer_nif_clear();
     CIX_TRACE("activate");
@@ -115,10 +115,10 @@ void erts_abort_staging_code_ix(void)
 int erts_try_seize_code_write_permission(Process* c_p)
 {
     int success;
-    ASSERT(!erts_smp_thr_progress_is_blocking()); /* to avoid deadlock */
+    ASSERT(!erts_thr_progress_is_blocking()); /* to avoid deadlock */
     ASSERT(c_p != NULL);
 
-    erts_smp_mtx_lock(&code_write_permission_mtx);
+    erts_mtx_lock(&code_write_permission_mtx);
     success = (code_writing_process == NULL);
     if (success) {
 	code_writing_process = c_p;
@@ -136,21 +136,21 @@ int erts_try_seize_code_write_permission(Process* c_p)
 	code_write_queue = qitem;
 	erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
     }
-   erts_smp_mtx_unlock(&code_write_permission_mtx);
+   erts_mtx_unlock(&code_write_permission_mtx);
    return success;
 }
 
 void erts_release_code_write_permission(void)
 {
-    erts_smp_mtx_lock(&code_write_permission_mtx);
-    ERTS_SMP_LC_ASSERT(erts_has_code_write_permission());
+    erts_mtx_lock(&code_write_permission_mtx);
+    ERTS_LC_ASSERT(erts_has_code_write_permission());
     while (code_write_queue != NULL) { /* unleash the entire herd */
 	struct code_write_queue_item* qitem = code_write_queue;
-	erts_smp_proc_lock(qitem->p, ERTS_PROC_LOCK_STATUS);
+	erts_proc_lock(qitem->p, ERTS_PROC_LOCK_STATUS);
 	if (!ERTS_PROC_IS_EXITING(qitem->p)) {
 	    erts_resume(qitem->p, ERTS_PROC_LOCK_STATUS);
 	}
-	erts_smp_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS);
 	code_write_queue = qitem->next;
 	erts_proc_dec_refc(qitem->p);
 	erts_free(ERTS_ALC_T_CODE_IX_LOCK_Q, qitem);
@@ -159,7 +159,7 @@ void erts_release_code_write_permission(void)
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_tsd_set(has_code_write_permission, (void *) 0);
 #endif
-    erts_smp_mtx_unlock(&code_write_permission_mtx);
+    erts_mtx_unlock(&code_write_permission_mtx);
 }
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h
index a28b0cd36e..9e3280cd98 100644
--- a/erts/emulator/beam/code_ix.h
+++ b/erts/emulator/beam/code_ix.h
@@ -205,16 +205,16 @@ ErtsCodeMFA *erts_code_to_codemfa(BeamInstr *I)
     return mfa;
 }
 
-extern erts_smp_atomic32_t the_active_code_index;
-extern erts_smp_atomic32_t the_staging_code_index;
+extern erts_atomic32_t the_active_code_index;
+extern erts_atomic32_t the_staging_code_index;
 
 ERTS_GLB_INLINE ErtsCodeIndex erts_active_code_ix(void)
 {
-    return erts_smp_atomic32_read_nob(&the_active_code_index);
+    return erts_atomic32_read_nob(&the_active_code_index);
 }
 ERTS_GLB_INLINE ErtsCodeIndex erts_staging_code_ix(void)
 {
-    return erts_smp_atomic32_read_nob(&the_staging_code_index);
+    return erts_atomic32_read_nob(&the_staging_code_index);
 }
 
 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index fefde256d7..10bf197405 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -845,7 +845,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 		    funp = (ErlFunThing *) tp;
 		    funp->next = off_heap->first;
 		    off_heap->first = (struct erl_off_heap_header*) funp;
-		    erts_smp_refc_inc(&funp->fe->refc, 2);
+		    erts_refc_inc(&funp->fe->refc, 2);
 		    *argp = make_fun(tp);
 		}
 		break;
@@ -854,7 +854,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
 	    case EXTERNAL_REF_SUBTAG:
 		{
 		  ExternalThing *etp = (ExternalThing *) objp;
-		  erts_smp_refc_inc(&etp->node->refc, 2);
+		  erts_refc_inc(&etp->node->refc, 2);
 		}
 	    L_off_heap_node_container_common:
 		{
@@ -1531,7 +1531,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 		}
 		funp->next = off_heap->first;
 		off_heap->first = (struct erl_off_heap_header*) funp;
-		erts_smp_refc_inc(&funp->fe->refc, 2);
+		erts_refc_inc(&funp->fe->refc, 2);
 		goto cleanup_next;
 	    }
 	    case MAP_SUBTAG:
@@ -1658,7 +1658,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 	    case EXTERNAL_REF_SUBTAG:
 	    {
 		ExternalThing *etp = (ExternalThing *) ptr;
-		erts_smp_refc_inc(&etp->node->refc, 2);
+		erts_refc_inc(&etp->node->refc, 2);
 	    }
 	  off_heap_node_container_common:
 	    {
@@ -1855,7 +1855,7 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    case FUN_SUBTAG:
 		{
 		    ErlFunThing* funp = (ErlFunThing *) (tp-1);
-		    erts_smp_refc_inc(&funp->fe->refc, 2);
+		    erts_refc_inc(&funp->fe->refc, 2);
 		}
 		goto off_heap_common;
 	    case EXTERNAL_PID_SUBTAG:
@@ -1863,7 +1863,7 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    case EXTERNAL_REF_SUBTAG:
 		{
 		    ExternalThing* etp = (ExternalThing *) (tp-1);
-		    erts_smp_refc_inc(&etp->node->refc, 2);
+		    erts_refc_inc(&etp->node->refc, 2);
 		}
 	    off_heap_common:
 		{
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 2be05eed29..491c4d378e 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -130,8 +130,8 @@ static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy);
 static void send_nodes_mon_msgs(Process *, Eterm, Eterm, Eterm, Eterm);
 static void init_nodes_monitors(void);
 
-static erts_smp_atomic_t no_caches;
-static erts_smp_atomic_t no_nodes;
+static erts_atomic_t no_caches;
+static erts_atomic_t no_nodes;
 
 struct {
     Eterm reason;
@@ -144,8 +144,8 @@ delete_cache(ErtsAtomCache *cache)
 {
     if (cache) {
 	erts_free(ERTS_ALC_T_DCACHE, (void *) cache);
-	ASSERT(erts_smp_atomic_read_nob(&no_caches) > 0);
-	erts_smp_atomic_dec_nob(&no_caches);
+	ASSERT(erts_atomic_read_nob(&no_caches) > 0);
+	erts_atomic_dec_nob(&no_caches);
     }
 }
 
@@ -156,14 +156,14 @@ create_cache(DistEntry *dep)
     int i;
     ErtsAtomCache *cp;
 
-    ERTS_SMP_LC_ASSERT(
+    ERTS_LC_ASSERT(
 	is_internal_port(dep->cid)
 	&& erts_lc_is_port_locked(erts_port_lookup_raw(dep->cid)));
     ASSERT(!dep->cache);
 
     dep->cache = cp = (ErtsAtomCache*) erts_alloc(ERTS_ALC_T_DCACHE,
 						  sizeof(ErtsAtomCache));
-    erts_smp_atomic_inc_nob(&no_caches);
+    erts_atomic_inc_nob(&no_caches);
     for (i = 0; i < sizeof(cp->in_arr)/sizeof(cp->in_arr[0]); i++) {
 	cp->in_arr[i] = THE_NON_VALUE;
 	cp->out_arr[i] = THE_NON_VALUE;
@@ -172,13 +172,13 @@ create_cache(DistEntry *dep)
 
 Uint erts_dist_cache_size(void)
 {
-    return (Uint) erts_smp_atomic_read_mb(&no_caches)*sizeof(ErtsAtomCache);
+    return (Uint) erts_atomic_read_mb(&no_caches)*sizeof(ErtsAtomCache);
 }
 
 static ErtsProcList *
 get_suspended_on_de(DistEntry *dep, Uint32 unset_qflgs)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&dep->qlock));
+    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&dep->qlock));
     dep->qflgs &= ~unset_qflgs;
     if (dep->qflgs & ERTS_DE_QFLG_EXIT) {
 	/* No resume when exit has been scheduled */
@@ -284,14 +284,14 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp)
 		       ? TUPLE2(lhp, rmon->name, dep->sysname)
 		       : rmon->u.pid);
 	    rp_locks |= ERTS_PROC_LOCKS_MSG_SEND;
-	    erts_smp_proc_lock(rp, ERTS_PROC_LOCKS_MSG_SEND);
+	    erts_proc_lock(rp, ERTS_PROC_LOCKS_MSG_SEND);
 	    erts_queue_monitor_message(rp, &rp_locks, mon->ref, am_process, 
 				       watched, am_noconnection);
 	    erts_destroy_monitor(rmon);
 	}
 	UnUseTmpHeapNoproc(3);
     }
-    erts_smp_proc_unlock(rp, rp_locks);
+    erts_proc_unlock(rp, rp_locks);
  done:
     erts_destroy_monitor(mon);
 }
@@ -340,7 +340,7 @@ static void doit_link_net_exits_sub(ErtsLink *sublnk, void *vlnecp)
 		trace_proc(NULL, 0, rp, am_getting_unlinked, sublnk->pid);
 	    }
 	}
-	erts_smp_proc_unlock(rp, rp_locks);
+	erts_proc_unlock(rp, rp_locks);
     }
  done:
     erts_destroy_link(sublnk);
@@ -382,7 +382,7 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp)
 	rp = erts_proc_lookup(lnk->pid);
 	if (!rp)
 	    goto done;
-	erts_smp_proc_lock(rp, rp_locks);
+	erts_proc_lock(rp, rp_locks);
 	rlnk = erts_remove_link(&ERTS_P_LINKS(rp), name);
 	if (rlnk != NULL) {
 	    ASSERT(is_atom(rlnk->pid) && (rlnk->type == LINK_NODE));
@@ -399,7 +399,7 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp)
 	    tup = TUPLE2(hp, am_nodedown, name);
 	    erts_queue_message(rp, rp_locks, msgp, tup, am_system);
 	}
-	erts_smp_proc_unlock(rp, rp_locks);
+	erts_proc_unlock(rp, rp_locks);
     }
  done:
     erts_destroy_link(lnk);
@@ -411,16 +411,16 @@ set_node_not_alive(void *unused)
     ErlHeapFragment *bp;
     Eterm nodename = erts_this_dist_entry->sysname;
 
-    ASSERT(erts_smp_atomic_read_nob(&no_nodes) == 0);
+    ASSERT(erts_atomic_read_nob(&no_nodes) == 0);
 
-    erts_smp_thr_progress_block();
+    erts_thr_progress_block();
     erts_set_this_node(am_Noname, 0);
     erts_is_alive = 0;
     send_nodes_mon_msgs(NULL, am_nodedown, nodename, am_visible, nodedown.reason);
     nodedown.reason = NIL;
     bp = nodedown.bp;
     nodedown.bp = NULL;
-    erts_smp_thr_progress_unblock();
+    erts_thr_progress_unblock();
     if (bp)
 	free_message_buffer(bp);
 }
@@ -428,7 +428,7 @@ set_node_not_alive(void *unused)
 static ERTS_INLINE void
 dec_no_nodes(void)
 {
-    erts_aint_t no = erts_smp_atomic_dec_read_mb(&no_nodes);
+    erts_aint_t no = erts_atomic_dec_read_mb(&no_nodes);
     ASSERT(no >= 0);
     ASSERT(erts_get_scheduler_id()); /* Need to be a scheduler */
     if (no == 0)
@@ -441,10 +441,10 @@ static ERTS_INLINE void
 inc_no_nodes(void)
 {
 #ifdef DEBUG
-    erts_aint_t no = erts_smp_atomic_read_nob(&no_nodes);
+    erts_aint_t no = erts_atomic_read_nob(&no_nodes);
     ASSERT(erts_is_alive ? no > 0 : no == 0);
 #endif
-    erts_smp_atomic_inc_mb(&no_nodes);
+    erts_atomic_inc_mb(&no_nodes);
 }
 	
 /*
@@ -460,7 +460,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 	Eterm nd_reason = (reason == am_no_network
 			   ? am_no_network
 			   : am_net_kernel_terminated);
-	erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
+	erts_rwmtx_rlock(&erts_dist_table_rwmtx);
 
 	for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next)
 	    no_dist_port++;
@@ -469,7 +469,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 
 	/* KILL all port controllers */
 	if (no_dist_port == 0)
-	    erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
+	    erts_rwmtx_runlock(&erts_dist_table_rwmtx);
 	else {
 	    Eterm def_buf[128];
 	    int i = 0;
@@ -488,7 +488,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 		ASSERT(is_internal_port(tdep->cid));
 		dist_port[i++] = tdep->cid;
 	    }
-	    erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
+	    erts_rwmtx_runlock(&erts_dist_table_rwmtx);
 
 	    for (i = 0; i < no_dist_port; i++) {
 		Port *prt = erts_port_lookup(dist_port[i],
@@ -531,10 +531,10 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 	ErtsMonitor *monitors;
 	Uint32 flags;
 
-	erts_smp_atomic_set_mb(&dep->dist_cmd_scheduled, 1);
-	erts_smp_de_rwlock(dep);
+	erts_atomic_set_mb(&dep->dist_cmd_scheduled, 1);
+	erts_de_rwlock(dep);
 
-	ERTS_SMP_LC_ASSERT(is_internal_port(dep->cid)
+	ERTS_LC_ASSERT(is_internal_port(dep->cid)
 			   && erts_lc_is_port_locked(erts_port_lookup_raw(dep->cid)));
 
 	if (erts_port_task_is_scheduled(&dep->dist_cmd))
@@ -542,34 +542,34 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 
 	if (dep->status & ERTS_DE_SFLG_EXITING) {
 #ifdef DEBUG
-	    erts_smp_mtx_lock(&dep->qlock);
+	    erts_mtx_lock(&dep->qlock);
 	    ASSERT(dep->qflgs & ERTS_DE_QFLG_EXIT);
-	    erts_smp_mtx_unlock(&dep->qlock);
+	    erts_mtx_unlock(&dep->qlock);
 #endif
 	}
 	else {
 	    dep->status |= ERTS_DE_SFLG_EXITING;
-	    erts_smp_mtx_lock(&dep->qlock);
+	    erts_mtx_lock(&dep->qlock);
 	    ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT));
 	    dep->qflgs |= ERTS_DE_QFLG_EXIT;
-	    erts_smp_mtx_unlock(&dep->qlock);
+	    erts_mtx_unlock(&dep->qlock);
 	}
 
-	erts_smp_de_links_lock(dep);
+	erts_de_links_lock(dep);
 	monitors	= dep->monitors;
         nlinks		= dep->nlinks;
 	node_links	= dep->node_links;
 	dep->monitors	= NULL;
         dep->nlinks	= NULL;
 	dep->node_links	= NULL;
-	erts_smp_de_links_unlock(dep);
+	erts_de_links_unlock(dep);
 
 	nodename = dep->sysname;
 	flags = dep->flags;
 
 	erts_set_dist_entry_not_connected(dep);
 
-	erts_smp_de_rwunlock(dep);
+	erts_de_rwunlock(dep);
 
 	erts_sweep_monitors(monitors, &doit_monitor_net_exits, (void *) &nec);
 	erts_sweep_links(nlinks, &doit_link_net_exits, (void *) &nec);
@@ -603,8 +603,8 @@ void init_dist(void)
     nodedown.reason = NIL;
     nodedown.bp = NULL;
 
-    erts_smp_atomic_init_nob(&no_nodes, 0);
-    erts_smp_atomic_init_nob(&no_caches, 0);
+    erts_atomic_init_nob(&no_nodes, 0);
+    erts_atomic_init_nob(&no_caches, 0);
 
     /* Lookup/Install all references to trap functions */
     dsend2_trap = trap_function(am_dsend,2);
@@ -657,19 +657,19 @@ static void clear_dist_entry(DistEntry *dep)
     ErtsProcList *suspendees;
     ErtsDistOutputBuf *obuf;
 
-    erts_smp_de_rwlock(dep);
+    erts_de_rwlock(dep);
     cache = dep->cache;
     dep->cache = NULL;
 
 #ifdef DEBUG
-    erts_smp_de_links_lock(dep);
+    erts_de_links_lock(dep);
     ASSERT(!dep->nlinks);
     ASSERT(!dep->node_links);
     ASSERT(!dep->monitors);
-    erts_smp_de_links_unlock(dep);
+    erts_de_links_unlock(dep);
 #endif
 
-    erts_smp_mtx_lock(&dep->qlock);
+    erts_mtx_lock(&dep->qlock);
 
     if (!dep->out_queue.last)
 	obuf = dep->finalized_out_queue.first;
@@ -685,10 +685,10 @@ static void clear_dist_entry(DistEntry *dep)
     dep->status = 0;
     suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL);
 
-    erts_smp_mtx_unlock(&dep->qlock);
-    erts_smp_atomic_set_nob(&dep->dist_cmd_scheduled, 0);
+    erts_mtx_unlock(&dep->qlock);
+    erts_atomic_set_nob(&dep->dist_cmd_scheduled, 0);
     dep->send = NULL;
-    erts_smp_de_rwunlock(dep);
+    erts_de_rwunlock(dep);
 
     erts_resume_processes(suspendees);
 
@@ -703,10 +703,10 @@ static void clear_dist_entry(DistEntry *dep)
     }
 
     if (obufsize) {
-	erts_smp_mtx_lock(&dep->qlock);
+	erts_mtx_lock(&dep->qlock);
 	ASSERT(dep->qsize >= obufsize);
 	dep->qsize -= obufsize;
-	erts_smp_mtx_unlock(&dep->qlock);
+	erts_mtx_unlock(&dep->qlock);
     }
 }
 
@@ -811,9 +811,9 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
 		 watched, watcher, ref, reason);
 
 #ifdef DEBUG
-    erts_smp_de_links_lock(dsdp->dep);
+    erts_de_links_lock(dsdp->dep);
     ASSERT(!erts_lookup_monitor(dsdp->dep->monitors, ref));
-    erts_smp_de_links_unlock(dsdp->dep);
+    erts_de_links_unlock(dsdp->dep);
 #endif
 
     res = dsig_send_ctl(dsdp, ctl, 1);
@@ -1151,9 +1151,9 @@ int erts_net_message(Port *prt,
 
     UseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
     if (!erts_is_alive) {
 	UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
@@ -1261,23 +1261,23 @@ int erts_net_message(Port *prt,
 	    break;
 	}
 
-	erts_smp_de_links_lock(dep);
+	erts_de_links_lock(dep);
 	res = erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, from);
 
 	if (res < 0) {
 	    /* It was already there! Lets skip the rest... */
-	    erts_smp_de_links_unlock(dep);
-	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+	    erts_de_links_unlock(dep);
+	    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 	    break;
 	}
 	lnk = erts_add_or_lookup_link(&(dep->nlinks), LINK_PID, rp->common.id);
 	erts_add_link(&(ERTS_LINK_ROOT(lnk)), LINK_PID, from);
-	erts_smp_de_links_unlock(dep);
+	erts_de_links_unlock(dep);
 
 	if (IS_TRACED_FL(rp, F_TRACE_PROCS))
 	    trace_proc(NULL, 0, rp, am_getting_linked, from);
 
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 	break;
 
     case DOP_UNLINK: {
@@ -1303,7 +1303,7 @@ int erts_net_message(Port *prt,
 	    trace_proc(NULL, 0, rp, am_getting_unlinked, from);
 	}
 
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 
 	erts_remove_dist_link(&dld, to, from, dep);
 	erts_destroy_dist_link(&dld);
@@ -1355,11 +1355,11 @@ int erts_net_message(Port *prt,
 	else {
 	    if (is_atom(watched))
 		watched = rp->common.id;
-	    erts_smp_de_links_lock(dep);
+	    erts_de_links_lock(dep);
 	    erts_add_monitor(&(dep->monitors), MON_ORIGIN, ref, watched, name);
 	    erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, ref, watcher, name);
-	    erts_smp_de_links_unlock(dep);
-	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+	    erts_de_links_unlock(dep);
+	    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 	}
 
 	break;
@@ -1381,9 +1381,9 @@ int erts_net_message(Port *prt,
 	    goto invalid_message;
 	}
 
-	erts_smp_de_links_lock(dep);
+	erts_de_links_lock(dep);
 	mon = erts_remove_monitor(&(dep->monitors),ref);
-	erts_smp_de_links_unlock(dep);
+	erts_de_links_unlock(dep);
 	/* ASSERT(mon != NULL); can happen in case of broken dist message */
 	if (mon == NULL) {
 	    break;
@@ -1397,7 +1397,7 @@ int erts_net_message(Port *prt,
 	    break;
 	}
 	mon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref);
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 	ASSERT(mon != NULL);
 	if (mon == NULL) {
 	    break;
@@ -1458,7 +1458,7 @@ int erts_net_message(Port *prt,
 
 	    erts_queue_dist_message(rp, locks, ede_copy, token, from);
 	    if (locks)
-		erts_smp_proc_unlock(rp, locks);
+		erts_proc_unlock(rp, locks);
 	}
 	break;
 
@@ -1507,7 +1507,7 @@ int erts_net_message(Port *prt,
 
 	    erts_queue_dist_message(rp, locks, ede_copy, token, tuple[2]);
 	    if (locks)
-		erts_smp_proc_unlock(rp, locks);
+		erts_proc_unlock(rp, locks);
 	}
 	break;
 
@@ -1533,7 +1533,7 @@ int erts_net_message(Port *prt,
 	    goto invalid_message;
 	}
 
-	erts_smp_de_links_lock(dep);
+	erts_de_links_lock(dep);
 	sysname = dep->sysname;
 	mon = erts_remove_monitor(&(dep->monitors), ref);
 	/*
@@ -1542,7 +1542,7 @@ int erts_net_message(Port *prt,
 	 * removed info about monitor. In this case, do nothing
 	 * and everything will be as it should.
 	 */
-	erts_smp_de_links_unlock(dep);
+	erts_de_links_unlock(dep);
 	if (mon == NULL) {
 	    break;
 	}
@@ -1556,7 +1556,7 @@ int erts_net_message(Port *prt,
 	mon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref);
 
 	if (mon == NULL) {
-	    erts_smp_proc_unlock(rp, rp_locks);
+	    erts_proc_unlock(rp, rp_locks);
 	    break;
 	}
 	UseTmpHeapNoproc(3);
@@ -1567,7 +1567,7 @@ int erts_net_message(Port *prt,
 	
 	erts_queue_monitor_message(rp, &rp_locks,
 				   ref, am_process, watched, reason);
-	erts_smp_proc_unlock(rp, rp_locks);
+	erts_proc_unlock(rp, rp_locks);
 	erts_destroy_monitor(mon);
 	UnUseTmpHeapNoproc(3);
 	break;
@@ -1629,13 +1629,13 @@ int erts_net_message(Port *prt,
 		if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) {
 		    /* We didn't exit the process and it is traced */
                     if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) {
-                        erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND);
+                        erts_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND);
                         rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND;
                     }
 		    trace_proc(NULL, 0, rp, am_getting_unlinked, from);
 		}
 	    }
-	    erts_smp_proc_unlock(rp, rp_locks);
+	    erts_proc_unlock(rp, rp_locks);
 	}
 	erts_remove_dist_link(&dld, to, from, dep);
 	if (lnk)
@@ -1677,7 +1677,7 @@ int erts_net_message(Port *prt,
 					 token,
 					 NULL,
 					 0);
-	    erts_smp_proc_unlock(rp, rp_locks);
+	    erts_proc_unlock(rp, rp_locks);
 	}
 	break;
     }
@@ -1695,7 +1695,7 @@ int erts_net_message(Port *prt,
 	if (!rp)
 	    break;
 	rp->group_leader = STORE_NC_IN_PROC(rp, from);
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
+	erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
 	break;
 
     default: 
@@ -1707,7 +1707,7 @@ int erts_net_message(Port *prt,
 	erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl);
     }
     UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
     return 0;
  invalid_message:
     {
@@ -1724,7 +1724,7 @@ decode_error:
 data_error:
     UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
     erts_deliver_port_exit(prt, dep->cid, am_killed, 0, 1);
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
     return -1;
 }
 
@@ -1760,7 +1760,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 	    if (!ctx->c_p || dsdp->no_suspend)
 		ctx->force_busy = 1;
 
-	    ERTS_SMP_LC_ASSERT(!ctx->c_p
+	    ERTS_LC_ASSERT(!ctx->c_p
 			       || (ERTS_PROC_LOCK_MAIN
 				   == erts_proc_lc_my_proc_locks(ctx->c_p)));
 
@@ -1849,28 +1849,28 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 	     * and if so enqueue the signal and schedule it for send.
 	     */
 	    ctx->obuf->next = NULL;
-	    erts_smp_de_rlock(dep);
+	    erts_de_rlock(dep);
 	    cid = dep->cid;
 	    if (cid != dsdp->cid
 		|| dep->connection_id != dsdp->connection_id
 		|| dep->status & ERTS_DE_SFLG_EXITING) {
 		/* Not the same connection as when we started; drop message... */
-		erts_smp_de_runlock(dep);
+		erts_de_runlock(dep);
 		free_dist_obuf(ctx->obuf);
 	    }
 	    else {
 		ErtsProcList *plp = NULL;
-		erts_smp_mtx_lock(&dep->qlock);
+		erts_mtx_lock(&dep->qlock);
 		dep->qsize += size_obuf(ctx->obuf);
 		if (dep->qsize >= erts_dist_buf_busy_limit)
 		    dep->qflgs |= ERTS_DE_QFLG_BUSY;
 		if (!ctx->force_busy && (dep->qflgs & ERTS_DE_QFLG_BUSY)) {
-		    erts_smp_mtx_unlock(&dep->qlock);
+		    erts_mtx_unlock(&dep->qlock);
 
 		    plp = erts_proclist_create(ctx->c_p);
 		    erts_suspend(ctx->c_p, ERTS_PROC_LOCK_MAIN, NULL);
 		    suspended = 1;
-		    erts_smp_mtx_lock(&dep->qlock);
+		    erts_mtx_lock(&dep->qlock);
 		}
 
 		/* Enqueue obuf on dist entry */
@@ -1905,9 +1905,9 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 		    }
 		}
 
-		erts_smp_mtx_unlock(&dep->qlock);
+		erts_mtx_unlock(&dep->qlock);
 		erts_schedule_dist_command(NULL, dep);
-		erts_smp_de_runlock(dep);
+		erts_de_runlock(dep);
 
 		if (resume) {
 		    erts_resume(ctx->c_p, ERTS_PROC_LOCK_MAIN);
@@ -1963,8 +1963,8 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf)
     int fpe_was_unmasked;
     Uint size = obuf->ext_endp - obuf->extp;
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_CHK_NO_PROC_LOCKS;
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
     if (size > (Uint) INT_MAX)
 	erts_exit(ERTS_DUMP_EXIT,
@@ -2003,8 +2003,8 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf)
     ErlDrvBinary* bv[2];
     ErlIOVec eiov;
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_CHK_NO_PROC_LOCKS;
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
     if (size > (Uint) INT_MAX)
 	erts_exit(ERTS_DUMP_EXIT,
@@ -2079,18 +2079,18 @@ erts_dist_command(Port *prt, int reds_limit)
     erts_aint32_t sched_flags;
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
-    erts_smp_refc_inc(&dep->refc, 1); /* Otherwise dist_entry might be
+    erts_refc_inc(&dep->refc, 1); /* Otherwise dist_entry might be
 				     removed if port command fails */
 
-    erts_smp_atomic_set_mb(&dep->dist_cmd_scheduled, 0);
+    erts_atomic_set_mb(&dep->dist_cmd_scheduled, 0);
 
-    erts_smp_de_rlock(dep);
+    erts_de_rlock(dep);
     flags = dep->flags;
     status = dep->status;
     send = dep->send;
-    erts_smp_de_runlock(dep);
+    erts_de_runlock(dep);
 
     if (status & ERTS_DE_SFLG_EXITING) {
 	erts_deliver_port_exit(prt, prt->common.id, am_killed, 0, 1);
@@ -2108,19 +2108,19 @@ erts_dist_command(Port *prt, int reds_limit)
      * a mess.
      */
 
-    erts_smp_mtx_lock(&dep->qlock);
+    erts_mtx_lock(&dep->qlock);
     oq.first = dep->out_queue.first;
     oq.last = dep->out_queue.last;
     dep->out_queue.first = NULL;
     dep->out_queue.last = NULL;
-    erts_smp_mtx_unlock(&dep->qlock);
+    erts_mtx_unlock(&dep->qlock);
 
     foq.first = dep->finalized_out_queue.first;
     foq.last = dep->finalized_out_queue.last;
     dep->finalized_out_queue.first = NULL;
     dep->finalized_out_queue.last = NULL;
 
-    sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
+    sched_flags = erts_atomic32_read_nob(&prt->sched.flags);
 
     if (reds > reds_limit)
 	goto preempted;
@@ -2142,7 +2142,7 @@ erts_dist_command(Port *prt, int reds_limit)
 	    obufsize += size_obuf(fob);
 	    foq.first = foq.first->next;
 	    free_dist_obuf(fob);
-	    sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
+	    sched_flags = erts_atomic32_read_nob(&prt->sched.flags);
 	    preempt = reds > reds_limit || (sched_flags & ERTS_PTS_FLG_EXIT);
 	    if (sched_flags & ERTS_PTS_FLG_BUSY_PORT)
 		break;
@@ -2227,7 +2227,7 @@ erts_dist_command(Port *prt, int reds_limit)
 	    obufsize += size_obuf(fob);
 	    oq.first = oq.first->next;
 	    free_dist_obuf(fob);
-	    sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
+	    sched_flags = erts_atomic32_read_nob(&prt->sched.flags);
 	    preempt = reds > reds_limit || (sched_flags & ERTS_PTS_FLG_EXIT);
 	    if ((sched_flags & ERTS_PTS_FLG_BUSY_PORT) && oq.first && !preempt)
 		goto finalize_only;
@@ -2254,7 +2254,7 @@ erts_dist_command(Port *prt, int reds_limit)
 	 * dist entry in a non-busy state and resume suspended
 	 * processes.
 	 */
-	erts_smp_mtx_lock(&dep->qlock);
+	erts_mtx_lock(&dep->qlock);
 	ASSERT(dep->qsize >= obufsize);
 	dep->qsize -= obufsize;
 	obufsize = 0;
@@ -2264,13 +2264,13 @@ erts_dist_command(Port *prt, int reds_limit)
 	    ErtsProcList *suspendees;
 	    int resumed;
 	    suspendees = get_suspended_on_de(dep, ERTS_DE_QFLG_BUSY);
-	    erts_smp_mtx_unlock(&dep->qlock);
+	    erts_mtx_unlock(&dep->qlock);
 
 	    resumed = erts_resume_processes(suspendees);
 	    reds += resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED;
 	}
 	else
-	    erts_smp_mtx_unlock(&dep->qlock);
+	    erts_mtx_unlock(&dep->qlock);
     }
 
     ASSERT(!oq.first && !oq.last);
@@ -2279,10 +2279,10 @@ erts_dist_command(Port *prt, int reds_limit)
 
     if (obufsize != 0) {
 	ASSERT(obufsize > 0);
-	erts_smp_mtx_lock(&dep->qlock);
+	erts_mtx_lock(&dep->qlock);
 	ASSERT(dep->qsize >= obufsize);
 	dep->qsize -= obufsize;
-	erts_smp_mtx_unlock(&dep->qlock);
+	erts_mtx_unlock(&dep->qlock);
     }
 
     ASSERT(foq.first || !foq.last);
@@ -2336,9 +2336,9 @@ erts_dist_command(Port *prt, int reds_limit)
 	foq.last = NULL;
 
 #ifdef DEBUG
-	erts_smp_mtx_lock(&dep->qlock);
+	erts_mtx_lock(&dep->qlock);
 	ASSERT(dep->qsize == obufsize);
-	erts_smp_mtx_unlock(&dep->qlock);
+	erts_mtx_unlock(&dep->qlock);
 #endif
     }
     else {
@@ -2347,14 +2347,14 @@ erts_dist_command(Port *prt, int reds_limit)
 	     * Unhandle buffers need to be put back first
 	     * in out_queue.
 	     */
-	    erts_smp_mtx_lock(&dep->qlock);
+	    erts_mtx_lock(&dep->qlock);
 	    dep->qsize -= obufsize;
 	    obufsize = 0;
 	    oq.last->next = dep->out_queue.first;
 	    dep->out_queue.first = oq.first;
 	    if (!dep->out_queue.last)
 		dep->out_queue.last = oq.last;
-	    erts_smp_mtx_unlock(&dep->qlock);
+	    erts_mtx_unlock(&dep->qlock);
 	}
 
 	erts_schedule_dist_command(prt, NULL);
@@ -2384,21 +2384,21 @@ erts_dist_port_not_busy(Port *prt)
 void
 erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id)
 {
-    erts_smp_de_rwlock(dep);
+    erts_de_rwlock(dep);
     if (is_internal_port(dep->cid)
 	&& connection_id == dep->connection_id
 	&& !(dep->status & ERTS_DE_SFLG_EXITING)) {
 
 	dep->status |= ERTS_DE_SFLG_EXITING;
 
-	erts_smp_mtx_lock(&dep->qlock);
+	erts_mtx_lock(&dep->qlock);
 	ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT));
 	dep->qflgs |= ERTS_DE_QFLG_EXIT;
-	erts_smp_mtx_unlock(&dep->qlock);
+	erts_mtx_unlock(&dep->qlock);
 
 	erts_schedule_dist_command(NULL, dep);
     }
-    erts_smp_de_rwunlock(dep);
+    erts_de_rwunlock(dep);
 }
 
 struct print_to_data {
@@ -2514,7 +2514,7 @@ info_dist_entry(fmtfn_t to, void *arg, DistEntry *dep, int visible, int connecte
 
   erts_print(to, arg, "Name: %T", dep->sysname);
 #ifdef DEBUG
-  erts_print(to, arg, " (refc=%d)", erts_smp_refc_read(&dep->refc, 0));
+  erts_print(to, arg, " (refc=%d)", erts_refc_read(&dep->refc, 0));
 #endif
   erts_print(to, arg, "\n");
   if (!connected && is_nil(dep->cid)) {
@@ -2645,22 +2645,22 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2)
     net_kernel->flags |= F_DISTRIBUTION;
 
     if (net_kernel != BIF_P)
-	erts_smp_proc_unlock(net_kernel, ERTS_PROC_LOCK_MAIN);
+	erts_proc_unlock(net_kernel, ERTS_PROC_LOCK_MAIN);
 
 #ifdef DEBUG
-    erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
+    erts_rwmtx_rlock(&erts_dist_table_rwmtx);
     ASSERT(!erts_visible_dist_entries && !erts_hidden_dist_entries);
-    erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
+    erts_rwmtx_runlock(&erts_dist_table_rwmtx);
 #endif
 
-    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-    erts_smp_thr_progress_block();
+    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_thr_progress_block();
     inc_no_nodes();
     erts_set_this_node(BIF_ARG_1, (Uint32) creation);
     erts_is_alive = 1;
     send_nodes_mon_msgs(NULL, am_nodeup, BIF_ARG_1, am_visible, NIL);
-    erts_smp_thr_progress_unblock();
-    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_thr_progress_unblock();
+    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
     BIF_RET(am_true);
 
@@ -2750,7 +2750,7 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
 			    BIF_P,
 			    ERTS_PROC_LOCK_MAIN,
 			    ERTS_PORT_SFLGS_INVALID_LOOKUP);
-    erts_smp_de_rwlock(dep);
+    erts_de_rwlock(dep);
 
     if (!pp || (erts_atomic32_read_nob(&pp->state)
 		& ERTS_PORT_SFLG_EXITING))
@@ -2767,9 +2767,9 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
 	ErtsProcList *plp = erts_proclist_create(BIF_P);
 	plp->next = NULL;
 	erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
-	erts_smp_mtx_lock(&dep->qlock);
+	erts_mtx_lock(&dep->qlock);
 	erts_proclist_store_last(&dep->suspended, plp);
-	erts_smp_mtx_unlock(&dep->qlock);
+	erts_mtx_unlock(&dep->qlock);
 	goto yield;
     }
 
@@ -2806,9 +2806,9 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     ASSERT(dep->send);
 
 #ifdef DEBUG
-    erts_smp_mtx_lock(&dep->qlock);
+    erts_mtx_lock(&dep->qlock);
     ASSERT(dep->qsize == 0);
-    erts_smp_mtx_unlock(&dep->qlock);
+    erts_mtx_unlock(&dep->qlock);
 #endif
 
     erts_set_dist_entry_connected(dep, BIF_ARG_2, flags);
@@ -2816,7 +2816,7 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     if (flags & DFLAG_DIST_HDR_ATOM_CACHE)
 	create_cache(dep);
 
-    erts_smp_de_rwunlock(dep);
+    erts_de_rwunlock(dep);
     dep = NULL; /* inc of refc transferred to port (dist_entry field) */
 
     inc_no_nodes();
@@ -2829,7 +2829,7 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
  done:
 
     if (dep && dep != erts_this_dist_entry) {
-	erts_smp_de_rwunlock(dep);
+	erts_de_rwunlock(dep);
 	erts_deref_dist_entry(dep);
     }
 
@@ -2881,7 +2881,7 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3)
 	if (BIF_P->common.id == local) {
 	    lp_locks = ERTS_PROC_LOCKS_ALL;
 	    lp = BIF_P;
-	    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR);
+	    erts_proc_lock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR);
 	}
 	else {
 	    lp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
@@ -2902,9 +2902,9 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3)
 				     0);
 	if (lp == BIF_P)
 	    lp_locks &= ~ERTS_PROC_LOCK_MAIN;
-	erts_smp_proc_unlock(lp, lp_locks);
+	erts_proc_unlock(lp, lp_locks);
 	if (lp == BIF_P) {
-	    erts_aint32_t state = erts_smp_atomic32_read_acqb(&BIF_P->state);
+	    erts_aint32_t state = erts_atomic32_read_acqb(&BIF_P->state);
 	    /*
 	     * We may have exited current process and may have to take action.
 	     */
@@ -2996,7 +2996,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
 
     length = 0;
 
-    erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
+    erts_rwmtx_rlock(&erts_dist_table_rwmtx);
 
     ASSERT(erts_no_of_not_connected_dist_entries > 0);
     ASSERT(erts_no_of_hidden_dist_entries >= 0);
@@ -3013,7 +3013,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
     result = NIL;
 
     if (length == 0) {
-	erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
+	erts_rwmtx_runlock(&erts_dist_table_rwmtx);
 	goto done;
     }
 
@@ -3044,7 +3044,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
 	hp += 2;
     }
     ASSERT(endp == hp);
-    erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
+    erts_rwmtx_runlock(&erts_dist_table_rwmtx);
 
 done:
     UnUseTmpHeap(2,BIF_P);
@@ -3099,15 +3099,15 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options)
     if (dep == erts_this_dist_entry)
 	goto done;
 
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_LINK);
-    erts_smp_de_rlock(dep);
+    erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
+    erts_de_rlock(dep);
     if (ERTS_DE_IS_NOT_CONNECTED(dep)) {
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
-	erts_smp_de_runlock(dep);
+	erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+	erts_de_runlock(dep);
 	goto do_trap;
     }
-    erts_smp_de_links_lock(dep);
-    erts_smp_de_runlock(dep);
+    erts_de_links_lock(dep);
+    erts_de_runlock(dep);
 
     if (Bool == am_true) {
 	ASSERT(dep->cid != NIL);
@@ -3134,8 +3134,8 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options)
 	}
     }
 
-    erts_smp_de_links_unlock(dep);
-    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+    erts_de_links_unlock(dep);
+    erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 
  done:
     erts_deref_dist_entry(dep);
@@ -3167,9 +3167,9 @@ BIF_RETTYPE net_kernel_dflag_unicode_io_1(BIF_ALIST_1)
     if (de == erts_this_dist_entry) {
 	BIF_RET(am_true);
     }
-    erts_smp_de_rlock(de);
+    erts_de_rlock(de);
     f = de->flags;
-    erts_smp_de_runlock(de);
+    erts_de_runlock(de);
     BIF_RET(((f & DFLAG_UNICODE_IO) ? am_true : am_false));
 }
     
@@ -3199,7 +3199,7 @@ struct ErtsNodesMonitor_ {
     Uint16 no;
 };
 
-static erts_smp_mtx_t nodes_monitors_mtx;
+static erts_mtx_t nodes_monitors_mtx;
 static ErtsNodesMonitor *nodes_monitors;
 static ErtsNodesMonitor *nodes_monitors_end;
 
@@ -3217,7 +3217,7 @@ static ErtsNodesMonitor *nodes_monitors_end;
 static void
 init_nodes_monitors(void)
 {
-    erts_smp_mtx_init(&nodes_monitors_mtx, "nodes_monitors", NIL,
+    erts_mtx_init(&nodes_monitors_mtx, "nodes_monitors", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
     nodes_monitors = NULL;
     nodes_monitors_end = NULL;
@@ -3343,10 +3343,10 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas
     }
 #endif
 
-    ERTS_SMP_LC_ASSERT(!c_p
+    ERTS_LC_ASSERT(!c_p
 		       || (erts_proc_lc_my_proc_locks(c_p)
 			   == ERTS_PROC_LOCK_MAIN));
-    erts_smp_mtx_lock(&nodes_monitors_mtx);
+    erts_mtx_lock(&nodes_monitors_mtx);
 
     for (nmp = nodes_monitors; nmp; nmp = nmp->next) {
 	int i;
@@ -3379,7 +3379,7 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas
 	    if (rp) {
 		if (rp == c_p)
 		    rp_locks &= ~ERTS_PROC_LOCK_MAIN;
-		erts_smp_proc_unlock(rp, rp_locks);
+		erts_proc_unlock(rp, rp_locks);
 	    }
 
 	    rp = nmp->proc;
@@ -3406,10 +3406,10 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas
     if (rp) {
 	if (rp == c_p)
 	    rp_locks &= ~ERTS_PROC_LOCK_MAIN;
-	erts_smp_proc_unlock(rp, rp_locks);
+	erts_proc_unlock(rp, rp_locks);
     }
 
-    erts_smp_mtx_unlock(&nodes_monitors_mtx);
+    erts_mtx_unlock(&nodes_monitors_mtx);
 }
 
 static Eterm
@@ -3419,8 +3419,8 @@ insert_nodes_monitor(Process *c_p, Uint32 opts)
     Eterm res = am_false;
     ErtsNodesMonitor *xnmp, *nmp;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&nodes_monitors_mtx));
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN);
+    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&nodes_monitors_mtx));
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN);
 
     xnmp = c_p->nodes_monitors;
     if (xnmp) {
@@ -3504,8 +3504,8 @@ remove_nodes_monitors(Process *c_p, Uint32 opts, int all)
     Eterm res = am_false;
     ErtsNodesMonitor *nmp;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&nodes_monitors_mtx));
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN);
+    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&nodes_monitors_mtx));
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN);
 
     nmp = c_p->nodes_monitors;
     ASSERT(!nmp || !nmp->prev || nmp->prev->proc != c_p);
@@ -3554,16 +3554,16 @@ erts_delete_nodes_monitors(Process *c_p, ErtsProcLocks locks)
 	    erts_proc_lc_might_unlock(c_p, might_unlock);
     }
 #endif
-    if (erts_smp_mtx_trylock(&nodes_monitors_mtx) == EBUSY) {
+    if (erts_mtx_trylock(&nodes_monitors_mtx) == EBUSY) {
 	ErtsProcLocks unlock_locks = locks & ~ERTS_PROC_LOCK_MAIN;
 	if (c_p && unlock_locks)
-	    erts_smp_proc_unlock(c_p, unlock_locks);
-	erts_smp_mtx_lock(&nodes_monitors_mtx);
+	    erts_proc_unlock(c_p, unlock_locks);
+	erts_mtx_lock(&nodes_monitors_mtx);
 	if (c_p && unlock_locks)
-	    erts_smp_proc_lock(c_p, unlock_locks);
+	    erts_proc_lock(c_p, unlock_locks);
     }
     remove_nodes_monitors(c_p, 0, 1);
-    erts_smp_mtx_unlock(&nodes_monitors_mtx);
+    erts_mtx_unlock(&nodes_monitors_mtx);
 }
 
 Eterm
@@ -3574,7 +3574,7 @@ erts_monitor_nodes(Process *c_p, Eterm on, Eterm olist)
     Uint16 opts = (Uint16) 0;
 
     ASSERT(c_p);
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
 
     if (on != am_true && on != am_false)
 	return THE_NON_VALUE;
@@ -3630,14 +3630,14 @@ erts_monitor_nodes(Process *c_p, Eterm on, Eterm olist)
 	    return THE_NON_VALUE;
     }
 
-    erts_smp_mtx_lock(&nodes_monitors_mtx);
+    erts_mtx_lock(&nodes_monitors_mtx);
 
     if (on == am_true)
 	res = insert_nodes_monitor(c_p, opts);
     else
 	res = remove_nodes_monitors(c_p, opts, 0);
 
-    erts_smp_mtx_unlock(&nodes_monitors_mtx);
+    erts_mtx_unlock(&nodes_monitors_mtx);
 
     return res;
 }
@@ -3660,8 +3660,8 @@ erts_processes_monitoring_nodes(Process *c_p)
 #endif
 
     ASSERT(c_p);
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
-    erts_smp_mtx_lock(&nodes_monitors_mtx);
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
+    erts_mtx_lock(&nodes_monitors_mtx);
 
     sz = 0;
     szp = &sz;
@@ -3710,7 +3710,7 @@ erts_processes_monitoring_nodes(Process *c_p)
 
     ASSERT(hp == hend);
 
-    erts_smp_mtx_unlock(&nodes_monitors_mtx);
+    erts_mtx_unlock(&nodes_monitors_mtx);
 
     return res;
 }
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 3e17645997..05016cafc5 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -100,7 +100,7 @@ typedef struct {
 } ErtsDSigData;
 
 #define ERTS_DE_IS_NOT_CONNECTED(DEP) \
-  (ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&(DEP)->rwmtx) \
+  (ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&(DEP)->rwmtx) \
 		      || erts_lc_rwmtx_is_rwlocked(&(DEP)->rwmtx)), \
    (is_nil((DEP)->cid) || ((DEP)->status & ERTS_DE_SFLG_EXITING)))
 
@@ -153,19 +153,19 @@ erts_dsig_prepare(ErtsDSigData *dsdp,
     if (!dep)
 	return ERTS_DSIG_PREP_NOT_CONNECTED;
     if (dspl == ERTS_DSP_RWLOCK)
-	erts_smp_de_rwlock(dep);
+	erts_de_rwlock(dep);
     else
-	erts_smp_de_rlock(dep);
+	erts_de_rlock(dep);
     if (ERTS_DE_IS_NOT_CONNECTED(dep)) {
 	failure = ERTS_DSIG_PREP_NOT_CONNECTED;
 	goto fail;
     }
     if (no_suspend) {
 	failure = ERTS_DSIG_PREP_CONNECTED;
-	erts_smp_mtx_lock(&dep->qlock);
+	erts_mtx_lock(&dep->qlock);
 	if (dep->qflgs & ERTS_DE_QFLG_BUSY)
 	    failure = ERTS_DSIG_PREP_WOULD_SUSPEND;
-	erts_smp_mtx_unlock(&dep->qlock);
+	erts_mtx_unlock(&dep->qlock);
 	if (failure == ERTS_DSIG_PREP_WOULD_SUSPEND)
 	    goto fail;
     }
@@ -175,14 +175,14 @@ erts_dsig_prepare(ErtsDSigData *dsdp,
     dsdp->connection_id = dep->connection_id;
     dsdp->no_suspend = no_suspend;
     if (dspl == ERTS_DSP_NO_LOCK)
-	erts_smp_de_runlock(dep);
+	erts_de_runlock(dep);
     return ERTS_DSIG_PREP_CONNECTED;
 
  fail:
     if (dspl == ERTS_DSP_RWLOCK)
-	erts_smp_de_rwunlock(dep);
+	erts_de_rwunlock(dep);
     else
-	erts_smp_de_runlock(dep);
+	erts_de_runlock(dep);
     return failure;
 
 }
@@ -194,7 +194,7 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry)
     Eterm id;
 
     if (prt) {
-	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+	ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 	ASSERT((erts_atomic32_read_nob(&prt->state)
 		& ERTS_PORT_SFLGS_DEAD) == 0);
 	ASSERT(prt->dist_entry);
@@ -204,7 +204,7 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry)
     }
     else {
 	ASSERT(dist_entry);
-	ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&dist_entry->rwmtx)
+	ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&dist_entry->rwmtx)
 			   || erts_lc_rwmtx_is_rwlocked(&dist_entry->rwmtx));
 	ASSERT(is_internal_port(dist_entry->cid));
 
@@ -212,7 +212,7 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry)
 	id = dep->cid;
     }
 
-    if (!erts_smp_atomic_xchg_mb(&dep->dist_cmd_scheduled, 1))
+    if (!erts_atomic_xchg_mb(&dep->dist_cmd_scheduled, 1))
 	erts_port_task_schedule(id, &dep->dist_cmd, ERTS_PORT_TASK_DIST_CMD);
 }
 
@@ -238,7 +238,7 @@ erts_remove_dist_link(ErtsDistLinkData *dldp,
 		      Eterm rid,
 		      DistEntry *dep)
 {
-    erts_smp_de_links_lock(dep);
+    erts_de_links_lock(dep);
     dldp->d_lnk = erts_lookup_link(dep->nlinks, lid);
     if (!dldp->d_lnk)
 	dldp->d_sub_lnk = NULL;
@@ -248,7 +248,7 @@ erts_remove_dist_link(ErtsDistLinkData *dldp,
 		       ? NULL
 		       : erts_remove_link(&dep->nlinks, lid));
     }
-    erts_smp_de_links_unlock(dep);
+    erts_de_links_unlock(dep);
 }
 
 ERTS_GLB_INLINE int
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 80ee3d7f33..aee54ad0a8 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -140,7 +140,7 @@ enum {
 };
 
 typedef struct {
-    erts_smp_atomic32_t refc;
+    erts_atomic32_t refc;
     int only_sz;
     int internal;
     Uint req_sched;
@@ -2103,7 +2103,7 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
     int only_one_value = 0;
     ErtsAlcUFixInfo_t fi[ERTS_ALC_NO_FIXED_SIZES] = {{0,0}};
 
-    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
 
     /* Figure out whats wanted... */
 
@@ -2276,10 +2276,10 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 
 
     if (proc) {
-	ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN
+	ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN
 			   == erts_proc_lc_my_proc_locks(proc));
 	/* We'll need locks early in the lock order */
-	erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
+	erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
     }
 
     /* Calculate values needed... */
@@ -2437,7 +2437,7 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	Uint *hp;
 	Uint hsz;
 
-	erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
+	erts_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
 
 	if (only_one_value) {
 	    ASSERT(length == 1);
@@ -2486,11 +2486,11 @@ erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc)
     Uint reserved_atom_space, atom_space;
 
     if (proc) {
-	ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN
+	ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN
 			   == erts_proc_lc_my_proc_locks(proc));
 
 	/* We'll need locks early in the lock order */
-	erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
+	erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
     }
 
     i = 0;
@@ -2642,7 +2642,7 @@ erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc)
 	Uint hsz;
 	Uint *hszp;
 
-	erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
+	erts_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
 
 	hpp = NULL;
 	hsz = 0;
@@ -2730,7 +2730,7 @@ erts_allocator_info(fmtfn_t to, void *arg)
 {
     ErtsAlcType_t a;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
 
     for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) {
 	int ai;
@@ -3298,10 +3298,10 @@ reply_alloc_info(void *vair)
     if (air->req_sched == sched_id)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
  
-    erts_smp_proc_unlock(rp, rp_locks);
+    erts_proc_unlock(rp, rp_locks);
     erts_proc_dec_refc(rp);
 
-    if (erts_smp_atomic32_dec_read_nob(&air->refc) == 0) {
+    if (erts_atomic32_dec_read_nob(&air->refc) == 0) {
         erts_iref_storage_clean(&air->iref);
 	aireq_free(air);
     }
@@ -3380,7 +3380,7 @@ erts_request_alloc_info(struct process *c_p,
 
     air->allocs[airix] = ERTS_ALC_A_INVALID;
 
-    erts_smp_atomic32_init_nob(&air->refc,
+    erts_atomic32_init_nob(&air->refc,
 			       (erts_aint32_t) erts_no_schedulers);
 
     erts_proc_add_refc(c_p, (Sint) erts_no_schedulers);
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index ed5ff4a2ff..c661d0b226 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -334,24 +334,10 @@ erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr);
   (((((SZ) - 1) / ERTS_CACHE_LINE_SIZE) + 1) * ERTS_CACHE_LINE_SIZE)
 
 #define ERTS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)			\
-ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT,				\
-		      (void) 0, (void) 0, (void) 0)
-
-#define ERTS_SMP_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)			\
-static erts_smp_spinlock_t NAME##_lck;					\
-ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT,				\
-		      erts_smp_spinlock_init(&NAME##_lck, #NAME "_alloc_lock", NIL, \
-		          ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR),\
-		      erts_smp_spin_lock(&NAME##_lck),			\
-		      erts_smp_spin_unlock(&NAME##_lck))
-
+    ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT, (void) 0, (void) 0, (void) 0)
 
 #define ERTS_TS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)			\
-ERTS_SMP_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)
-
-
-#define ERTS_PALLOC_IMPL(NAME, TYPE, PASZ)				\
-ERTS_PRE_ALLOC_IMPL(NAME, TYPE, PASZ, (void) 0, (void) 0, (void) 0)
+ERTS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)
 
 #define ERTS_TS_PALLOC_IMPL(NAME, TYPE, PASZ)				\
 static erts_spinlock_t NAME##_lck;					\
@@ -362,7 +348,7 @@ ERTS_PRE_ALLOC_IMPL(NAME, TYPE, PASZ,					\
 		    erts_spin_unlock(&NAME##_lck))
 
 
-#define ERTS_SMP_PALLOC_IMPL(NAME, TYPE, PASZ)				\
+#define ERTS_PALLOC_IMPL(NAME, TYPE, PASZ)				\
   ERTS_TS_PALLOC_IMPL(NAME, TYPE, PASZ)
 
 
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index f4c88438f5..4d4bddb93f 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -382,7 +382,7 @@ do {										\
 
 #define SET_CARRIER_HDR(C, Sz, F, AP) \
   (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), \
-   erts_smp_atomic_init_nob(&(C)->allctr, (erts_aint_t) (AP)))
+   erts_atomic_init_nob(&(C)->allctr, (erts_aint_t) (AP)))
 
 #define BLK_TO_SBC(B) \
   ((Carrier_t *) (((char *) (B)) - SBC_HEADER_SIZE))
@@ -670,7 +670,7 @@ do {									\
 	    (A)->debug.saved_tid = 1;					\
 	}								\
 	else {								\
-	    ERTS_SMP_LC_ASSERT(						\
+	    ERTS_LC_ASSERT(						\
 		ethr_equal_tids((A)->debug.tid, erts_thr_self()));	\
 	}								\
     }									\
@@ -826,7 +826,7 @@ erts_alcu_literal_32_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
     Uint sz = ERTS_SUPERALIGNED_CEILING(*size_p);
     ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
                    allctr->t == 0);
-    ERTS_SMP_LC_ASSERT(allctr->thread_safe);
+    ERTS_LC_ASSERT(allctr->thread_safe);
 
     res = erts_alcu_mseg_alloc(allctr, &sz, flags);
     if (res) {
@@ -844,7 +844,7 @@ erts_alcu_literal_32_mseg_realloc(Allctr_t *allctr, void *seg,
     Uint new_sz = ERTS_SUPERALIGNED_CEILING(*new_size_p);
     ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
                    allctr->t == 0);
-    ERTS_SMP_LC_ASSERT(allctr->thread_safe);
+    ERTS_LC_ASSERT(allctr->thread_safe);
 
     if (seg && old_size)
         clear_literal_range(seg, old_size);
@@ -862,7 +862,7 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
 {
     ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
                    allctr->t == 0);
-    ERTS_SMP_LC_ASSERT(allctr->thread_safe);
+    ERTS_LC_ASSERT(allctr->thread_safe);
 
     erts_alcu_mseg_dealloc(allctr, seg, size, flags);
 
@@ -1022,7 +1022,7 @@ erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign)
     Uint size = ERTS_SUPERALIGNED_CEILING(*size_p);
     ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
                    allctr->t == 0);
-    ERTS_SMP_LC_ASSERT(allctr->thread_safe);
+    ERTS_LC_ASSERT(allctr->thread_safe);
 
     res = erts_alcu_sys_alloc(allctr, &size, 1);
     if (res) {
@@ -1040,7 +1040,7 @@ erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint* size_p, Uint
 
     ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
                    allctr->t == 0);
-    ERTS_SMP_LC_ASSERT(allctr->thread_safe);
+    ERTS_LC_ASSERT(allctr->thread_safe);
 
     if (ptr && old_size)
         clear_literal_range(ptr, old_size);
@@ -1057,7 +1057,7 @@ erts_alcu_literal_32_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int sup
 {
     ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
                    allctr->t == 0);
-    ERTS_SMP_LC_ASSERT(allctr->thread_safe);
+    ERTS_LC_ASSERT(allctr->thread_safe);
 
     erts_alcu_sys_dealloc(allctr, ptr, size, 1);
 
@@ -1255,11 +1255,11 @@ clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr)
 	    erts_aint_t old_val = new_val|ERTS_CRR_ALCTR_FLG_BUSY;
 
 	    ERTS_ALC_CPOOL_ASSERT(old_val
-				  == erts_smp_atomic_xchg_relb(&crr->allctr,
+				  == erts_atomic_xchg_relb(&crr->allctr,
 							       new_val));
 	}
 #else
-	erts_smp_atomic_set_relb(&crr->allctr, new_val);
+	erts_atomic_set_relb(&crr->allctr, new_val);
 #endif
     }
 }
@@ -1697,7 +1697,7 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep,
 	crr = BLK_TO_SBC(blk);
 	if (sizep)
 	    *sizep = SBC_BLK_SZ(blk) - ABLK_HDR_SZ;  
-	iallctr = erts_smp_atomic_read_dirty(&crr->allctr);
+	iallctr = erts_atomic_read_dirty(&crr->allctr);
     }
     else {
 	crr = ABLK_TO_MBC(blk);
@@ -1705,10 +1705,10 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep,
 	if (sizep)
 	    *sizep = MBC_ABLK_SZ(blk) - ABLK_HDR_SZ;
 	if (!ERTS_ALC_IS_CPOOL_ENABLED(pref_allctr))
-	    iallctr = erts_smp_atomic_read_dirty(&crr->allctr);
+	    iallctr = erts_atomic_read_dirty(&crr->allctr);
 	else {
 	    int locked_pref_allctr = 0;
-	    iallctr = erts_smp_atomic_read_ddrb(&crr->allctr);
+	    iallctr = erts_atomic_read_ddrb(&crr->allctr);
 
 	    if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock
 		&& pref_allctr->thread_safe) {
@@ -1724,7 +1724,7 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep,
 		erts_aint_t act;
 
 		ERTS_ALC_CPOOL_ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY));
-		act = erts_smp_atomic_cmpxchg_ddrb(&crr->allctr,
+		act = erts_atomic_cmpxchg_ddrb(&crr->allctr,
 						   iallctr|ERTS_CRR_ALCTR_FLG_BUSY,
 						   iallctr);
 		if (act == iallctr) {
@@ -2099,10 +2099,10 @@ handle_delayed_dealloc(Allctr_t *allctr,
 	    ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr));
 	    ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr);
 	    ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr)
-				  != (erts_smp_atomic_read_nob(&crr->allctr)
+				  != (erts_atomic_read_nob(&crr->allctr)
 				      & ~ERTS_CRR_ALCTR_FLG_MASK));
 
-	    erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
+	    erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
 
 	    schedule_dealloc_carrier(allctr, crr);
 	}
@@ -2220,7 +2220,7 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_
 {
     Block_t *blk = UMEM2BLK(ptr);
 
-    ERTS_SMP_LC_ASSERT(!allctr->thread_safe
+    ERTS_LC_ASSERT(!allctr->thread_safe
 		       || erts_lc_mtx_is_locked(&allctr->mutex));
 
     if (IS_SBC_BLK(blk)) {
@@ -3048,7 +3048,7 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr)
 
     ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */
 			  || erts_thr_progress_is_managed_thread());
-    ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr)
+    ERTS_ALC_CPOOL_ASSERT(erts_atomic_read_nob(&crr->allctr)
 			  == (erts_aint_t) allctr);
 
     erts_atomic_add_nob(&allctr->cpool.stat.blocks_size,
@@ -3118,7 +3118,7 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr)
 			 (erts_aint_t) &crr->cpool,
 			 (erts_aint_t) cpd1p);
 
-    erts_smp_atomic_set_wb(&crr->allctr,
+    erts_atomic_set_wb(&crr->allctr,
 			   ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL);
     LTTNG3(carrier_pool_put, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, CARRIER_SZ(crr));
 }
@@ -3250,11 +3250,11 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 	ASSERT(!is_in_list(&allctr->cpool.traitor_list, dl));
 	ASSERT(crr->cpool.orig_allctr == allctr);
 	dl = dl->next;
-	exp = erts_smp_atomic_read_rb(&crr->allctr);
+	exp = erts_atomic_read_rb(&crr->allctr);
 	if ((exp & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL
 	    && erts_atomic_read_nob(&crr->cpool.max_size) >= size) {
 	    /* Try to fetch it... */
-	    act = erts_smp_atomic_cmpxchg_mb(&crr->allctr,
+	    act = erts_atomic_cmpxchg_mb(&crr->allctr,
 					     (erts_aint_t) allctr,
 					     exp);
 	    if (act == exp) {
@@ -3296,12 +3296,12 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 	ASSERT(dl != &allctr->cpool.pooled_list);
 	ASSERT(crr->cpool.orig_allctr == allctr);
 	dl = dl->next;
-	exp = erts_smp_atomic_read_rb(&crr->allctr);
+	exp = erts_atomic_read_rb(&crr->allctr);
 	if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) {
 	    if (!(exp & ERTS_CRR_ALCTR_FLG_BUSY)
 		&& erts_atomic_read_nob(&crr->cpool.max_size) >= size) {
 		/* Try to fetch it... */
-		act = erts_smp_atomic_cmpxchg_mb(&crr->allctr,
+		act = erts_atomic_cmpxchg_mb(&crr->allctr,
 						 (erts_aint_t) allctr,
 						 exp);
 		if (act == exp) {
@@ -3377,12 +3377,12 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 	    has_passed_sentinel = 1;
 	}
 	crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool));
-	exp = erts_smp_atomic_read_rb(&crr->allctr);
+	exp = erts_atomic_read_rb(&crr->allctr);
 	if (((exp & (ERTS_CRR_ALCTR_FLG_MASK)) == ERTS_CRR_ALCTR_FLG_IN_POOL)
 	    && (erts_atomic_read_nob(&cpdp->max_size) >= size)) {
 	    erts_aint_t act;
 	    /* Try to fetch it... */
-	    act = erts_smp_atomic_cmpxchg_mb(&crr->allctr,
+	    act = erts_atomic_cmpxchg_mb(&crr->allctr,
 					     (erts_aint_t) allctr,
 					     exp);
 	    if (act == exp) {
@@ -3405,11 +3405,11 @@ check_dc_list:
 	    Block_t* blk;
 	    unlink_carrier(&allctr->cpool.dc_list, crr);
 #ifdef ERTS_ALC_CPOOL_DEBUG
-	    ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr,
+	    ERTS_ALC_CPOOL_ASSERT(erts_atomic_xchg_nob(&crr->allctr,
 							   ((erts_aint_t) allctr))
 				  == (((erts_aint_t) allctr) & ~ERTS_CRR_ALCTR_FLG_MASK));
 #else
-	    erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
+	    erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
 #endif
 	    blk = MBC_TO_FIRST_BLK(allctr, crr);
 	    ASSERT(FBLK_TO_MBC(blk) == crr);
@@ -3512,7 +3512,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
 	ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(blk));
 	ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, blk));
 	ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr)
-			      == (erts_smp_atomic_read_nob(&crr->allctr)
+			      == (erts_atomic_read_nob(&crr->allctr)
 				  & ~ERTS_CRR_ALCTR_FLG_MASK));
 
 	if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(blk), cinit))
@@ -4130,11 +4130,11 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
 	if (busy_pcrr_pp && *busy_pcrr_pp) {
 	    ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr);
 	    *busy_pcrr_pp = NULL;
-	    ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr)
+	    ERTS_ALC_CPOOL_ASSERT(erts_atomic_read_nob(&crr->allctr)
 				  == (((erts_aint_t) allctr)
 				      | ERTS_CRR_ALCTR_FLG_IN_POOL
 				      | ERTS_CRR_ALCTR_FLG_BUSY));
-	    erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
+	    erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
 	    cpool_delete(allctr, allctr, crr);
 	}
 	else
@@ -5280,7 +5280,7 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
 
     ASSERT(allctr);
 
-    ERTS_SMP_LC_ASSERT(!allctr->thread_safe
+    ERTS_LC_ASSERT(!allctr->thread_safe
 		       || erts_lc_mtx_is_locked(&allctr->mutex));
 
     ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
@@ -5408,7 +5408,7 @@ do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p,
 
     ASSERT(allctr);
 
-    ERTS_SMP_LC_ASSERT(!allctr->thread_safe
+    ERTS_LC_ASSERT(!allctr->thread_safe
 		       || erts_lc_mtx_is_locked(&allctr->mutex));
 
     ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
@@ -5517,7 +5517,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
 
     ASSERT(allctr);
 
-    ERTS_SMP_LC_ASSERT(!allctr->thread_safe
+    ERTS_LC_ASSERT(!allctr->thread_safe
 		       || erts_lc_mtx_is_locked(&allctr->mutex));
 
     ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 0cce4c9125..30d7baf769 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -324,12 +324,12 @@ struct Carrier_t_ {
     UWord chdr;
     Carrier_t *next;
     Carrier_t *prev;
-    erts_smp_atomic_t allctr;
+    erts_atomic_t allctr;
     ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */
 };
 
 #define ERTS_ALC_CARRIER_TO_ALLCTR(C) \
-  ((Allctr_t *) (erts_smp_atomic_read_nob(&(C)->allctr) & ~FLG_MASK))
+  ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~FLG_MASK))
 
 typedef struct {
     Carrier_t *first;
diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index eafa7be738..3ceb2fd368 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -601,7 +601,7 @@ long driver_async(ErlDrvPort ix, unsigned int* key,
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return -1;
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
     a = (ErtsAsync*) erts_alloc(ERTS_ALC_T_ASYNC, sizeof(ErtsAsync));
 
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index fa41505a83..f673ef3194 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -100,18 +100,18 @@ static void dereference_all_processes(DE_Handle *dh);
 static void restore_process_references(DE_Handle *dh);
 static void ddll_no_more_references(void *vdh);
 
-#define lock_drv_list() erts_smp_rwmtx_rwlock(&erts_driver_list_lock)
-#define unlock_drv_list() erts_smp_rwmtx_rwunlock(&erts_driver_list_lock)
+#define lock_drv_list() erts_rwmtx_rwlock(&erts_driver_list_lock)
+#define unlock_drv_list() erts_rwmtx_rwunlock(&erts_driver_list_lock)
 #define assert_drv_list_locked() \
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rwlocked(&erts_driver_list_lock) \
-		       || erts_smp_lc_rwmtx_is_rlocked(&erts_driver_list_lock))
+    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_driver_list_lock) \
+		       || erts_lc_rwmtx_is_rlocked(&erts_driver_list_lock))
 #define assert_drv_list_rwlocked() \
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rwlocked(&erts_driver_list_lock))
+    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_driver_list_lock))
 #define assert_drv_list_rlocked() \
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&erts_driver_list_lock))
+    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&erts_driver_list_lock))
 #define assert_drv_list_not_locked() \
-    ERTS_SMP_LC_ASSERT(!erts_smp_lc_rwmtx_is_rwlocked(&erts_driver_list_lock) \
-		       && !erts_smp_lc_rwmtx_is_rlocked(&erts_driver_list_lock))
+    ERTS_LC_ASSERT(!erts_lc_rwmtx_is_rwlocked(&erts_driver_list_lock) \
+		       && !erts_lc_rwmtx_is_rlocked(&erts_driver_list_lock))
 
 
 #define FREE_PORT_FLAGS (ERTS_PORT_SFLGS_DEAD & (~ERTS_PORT_SFLG_INITIALIZING))
@@ -127,13 +127,13 @@ kill_ports_driver_unloaded(DE_Handle *dh)
 	if (!prt)
 	    continue;
 
-	ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
+	ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
 
 	state = erts_atomic32_read_nob(&prt->state);
 	if (state & FREE_PORT_FLAGS)
 	    continue;
 
-	erts_smp_port_lock(prt);
+	erts_port_lock(prt);
 
 	state = erts_atomic32_read_nob(&prt->state);
 	if (!(state & ERTS_PORT_SFLGS_DEAD) && prt->drv_ptr->handle == dh)
@@ -273,7 +273,7 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
     path[path_len++] = '/';
     sys_strcpy(path+path_len,name);
 
-    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
     lock_drv_list();
     if ((drv = lookup_driver(name)) != NULL) {
 	if (drv->handle == NULL) {
@@ -404,7 +404,7 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
 
     erts_ddll_reference_driver(dh);
     unlock_drv_list();
-    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     lock_drv_list();
     erts_ddll_dereference_driver(dh);
 
@@ -420,11 +420,11 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
     unlock_drv_list();
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path);
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
     BIF_RET(t);
  soft_error:
     unlock_drv_list();
-    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     if (do_build_load_error) {
 	soft_error_term = build_load_error(BIF_P, build_this_load_error);
     }
@@ -433,11 +433,11 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
     t = TUPLE2(hp, am_error, soft_error_term);
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path);
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
     BIF_RET(t);
  error:
     assert_drv_list_not_locked();
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
     if (path != NULL) {
 	erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path);
     }
@@ -499,7 +499,7 @@ Eterm erl_ddll_try_unload_2(BIF_ALIST_2)
     Eterm l;
     int kill_ports = 0;
 
-    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
     for(l = options; is_list(l); l =  CDR(list_val(l))) {
 	Eterm opt = CAR(list_val(l));
@@ -576,7 +576,7 @@ Eterm erl_ddll_try_unload_2(BIF_ALIST_2)
 	dh->reload_full_path = dh->reload_driver_name = NULL; 
 	dh->reload_flags = 0;
     } 
-    if (erts_smp_atomic32_read_nob(&dh->port_count) > 0) {
+    if (erts_atomic32_read_nob(&dh->port_count) > 0) {
 	++kill_ports;
     }
     dh->status = ERL_DE_UNLOAD;
@@ -595,7 +595,7 @@ done:
 
     erts_ddll_reference_driver(dh);
     unlock_drv_list();
-    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     lock_drv_list();
     erts_ddll_dereference_driver(dh);
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
@@ -617,7 +617,7 @@ done:
 soft_error:
     unlock_drv_list();
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
-    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     hp = HAlloc(BIF_P, 3);
     t = TUPLE2(hp, am_error, soft_error_term);
     BIF_RET(t);
@@ -627,7 +627,7 @@ soft_error:
     if (name != NULL) {
 	erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
     }
-    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     BIF_ERROR(BIF_P, BADARG);
 }
 
@@ -742,7 +742,7 @@ BIF_RETTYPE erl_ddll_info_2(BIF_ALIST_2)
 	} else if (drv->handle->status == ERL_DE_PERMANENT) {
 	    res = am_permanent;
 	} else {
-	    res = make_small(erts_smp_atomic32_read_nob(&drv->handle->port_count));
+	    res = make_small(erts_atomic32_read_nob(&drv->handle->port_count));
 	}
 	goto done;
     case am_linked_in_driver:
@@ -921,7 +921,7 @@ Eterm erts_ddll_monitor_driver(Process *p,
 void erts_ddll_remove_monitor(Process *p, Eterm ref, ErtsProcLocks plocks)
 { 
     erts_driver_t *drv;
-    erts_smp_proc_unlock(p, plocks);
+    erts_proc_unlock(p, plocks);
     lock_drv_list();
     drv = driver_list;
     while (drv != NULL) {
@@ -946,7 +946,7 @@ void erts_ddll_remove_monitor(Process *p, Eterm ref, ErtsProcLocks plocks)
     }
  done:
     unlock_drv_list();
-    erts_smp_proc_lock(p, plocks);
+    erts_proc_lock(p, plocks);
 }
 
 /* 
@@ -955,7 +955,7 @@ void erts_ddll_remove_monitor(Process *p, Eterm ref, ErtsProcLocks plocks)
 void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks) 
 {
     erts_driver_t *drv;
-    erts_smp_proc_unlock(p, plocks);
+    erts_proc_unlock(p, plocks);
     lock_drv_list();
     drv = driver_list;
     while (drv != NULL) {
@@ -993,7 +993,7 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks)
 		dh->status = ERL_DE_UNLOAD;
 	    }
 	    if (!left
-		&& erts_smp_atomic32_read_nob(&drv->handle->port_count) > 0) {
+		&& erts_atomic32_read_nob(&drv->handle->port_count) > 0) {
 		if (kill_ports) {
 		    DE_Handle *dh = drv->handle;
 		    erts_ddll_reference_driver(dh);
@@ -1014,7 +1014,7 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks)
 	}
     }
     unlock_drv_list();
-    erts_smp_proc_lock(p, plocks);
+    erts_proc_lock(p, plocks);
 }
 void erts_ddll_lock_driver(DE_Handle *dh, char *name)
 {
@@ -1042,41 +1042,41 @@ void erts_ddll_lock_driver(DE_Handle *dh, char *name)
 void erts_ddll_increment_port_count(DE_Handle *dh) 
 {
     assert_drv_list_locked();
-    erts_smp_atomic32_inc_nob(&dh->port_count);
+    erts_atomic32_inc_nob(&dh->port_count);
 }
 
 void erts_ddll_decrement_port_count(DE_Handle *dh)
 {
     assert_drv_list_locked();
 #ifdef DEBUG
-    ASSERT(erts_smp_atomic32_dec_read_nob(&dh->port_count) >= 0);
+    ASSERT(erts_atomic32_dec_read_nob(&dh->port_count) >= 0);
 #else
-    erts_smp_atomic32_dec_nob(&dh->port_count);
+    erts_atomic32_dec_nob(&dh->port_count);
 #endif
 }
 
 static void first_ddll_reference(DE_Handle *dh) 
 {
     assert_drv_list_rwlocked();
-    erts_smp_refc_init(&(dh->refc),1);
+    erts_refc_init(&(dh->refc),1);
 }
 
 void erts_ddll_reference_driver(DE_Handle *dh)
 {
     assert_drv_list_locked();
-    if (erts_smp_refc_inctest(&(dh->refc),1) == 1) {
-	erts_smp_refc_inc(&(dh->refc),2); /* add a reference for the scheduled operation */
+    if (erts_refc_inctest(&(dh->refc),1) == 1) {
+	erts_refc_inc(&(dh->refc),2); /* add a reference for the scheduled operation */
     }
 }
 
 void erts_ddll_reference_referenced_driver(DE_Handle *dh)
 {
-    erts_smp_refc_inc(&(dh->refc),2);
+    erts_refc_inc(&(dh->refc),2);
 }
 
 void erts_ddll_dereference_driver(DE_Handle *dh)
 {
-    if (erts_smp_refc_dectest(&(dh->refc),0) == 0) {
+    if (erts_refc_dectest(&(dh->refc),0) == 0) {
 	/* No lock here, but if the driver is referenced again,
 	   the scheduled deletion is added as a reference too, see above */
 	erts_schedule_misc_op(ddll_no_more_references, (void *) dh);
@@ -1099,11 +1099,11 @@ static void restore_process_references(DE_Handle *dh)
 {
     DE_ProcEntry *p;
     assert_drv_list_rwlocked();
-    ASSERT(erts_smp_refc_read(&(dh->refc),0) == 0);
+    ASSERT(erts_refc_read(&(dh->refc),0) == 0);
     for(p  = dh->procs;p != NULL; p = p->next) {
 	if (p->awaiting_status == ERL_DE_PROC_LOADED) {
 	    ASSERT(p->flags & ERL_DE_FL_DEREFERENCED);
-	    erts_smp_refc_inc(&(dh->refc),1);
+	    erts_refc_inc(&(dh->refc),1);
 	    p->flags &= ~ERL_DE_FL_DEREFERENCED;
 	}
     }
@@ -1125,9 +1125,9 @@ static void ddll_no_more_references(void *vdh)
 
     lock_drv_list();
 
-    x = erts_smp_refc_read(&(dh->refc),0);
+    x = erts_refc_read(&(dh->refc),0);
     if (x > 0) {
-	x = erts_smp_refc_dectest(&(dh->refc),0); /* delete the reference added for me */
+	x = erts_refc_dectest(&(dh->refc),0); /* delete the reference added for me */
     }
 
 
@@ -1230,7 +1230,7 @@ static Eterm notify_when_loaded(Process *p, Eterm name_term, char *name, ErtsPro
     Eterm immediate_type = NIL;
     erts_driver_t *drv;
 
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & plocks);
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & plocks);
     lock_drv_list();
     if ((drv = lookup_driver(name)) == NULL) {
 	immediate_tag = am_unloaded;
@@ -1265,10 +1265,10 @@ static Eterm notify_when_loaded(Process *p, Eterm name_term, char *name, ErtsPro
     BIF_RET(r);
  immediate:
     r =  erts_make_ref(p);
-    erts_smp_proc_unlock(p, plocks);
+    erts_proc_unlock(p, plocks);
     notify_proc(p, r, name_term, immediate_type, immediate_tag, 0);
     unlock_drv_list();
-    erts_smp_proc_lock(p, plocks);
+    erts_proc_lock(p, plocks);
     BIF_RET(r);
 }
 
@@ -1279,7 +1279,7 @@ static Eterm notify_when_unloaded(Process *p, Eterm name_term, char *name, ErtsP
     Eterm immediate_type = NIL;
     erts_driver_t *drv;
 
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & plocks);
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & plocks);
     lock_drv_list();
     if ((drv = lookup_driver(name)) == NULL) {
 	immediate_tag = am_unloaded;
@@ -1298,10 +1298,10 @@ static Eterm notify_when_unloaded(Process *p, Eterm name_term, char *name, ErtsP
     BIF_RET(r);
  immediate:
     r =  erts_make_ref(p);
-    erts_smp_proc_unlock(p, plocks);
+    erts_proc_unlock(p, plocks);
     notify_proc(p, r, name_term, immediate_type, immediate_tag, 0);
     unlock_drv_list();
-    erts_smp_proc_lock(p, plocks);
+    erts_proc_lock(p, plocks);
     BIF_RET(r);
 }
 
@@ -1505,8 +1505,8 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name)
 	res = ERL_DE_LOAD_ERROR_BAD_NAME;
 	goto error;
     }
-    erts_smp_atomic_init_nob(&(dh->refc), (erts_aint_t) 0);
-    erts_smp_atomic32_init_nob(&dh->port_count, 0);
+    erts_atomic_init_nob(&(dh->refc), (erts_aint_t) 0);
+    erts_atomic32_init_nob(&dh->port_count, 0);
     dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1);
     sys_strcpy(dh->full_path, path);
     dh->flags = 0;
@@ -1577,8 +1577,8 @@ static int load_driver_entry(DE_Handle **dhp, char *path, char *name)
 
     dh->handle = NULL;
     dh->procs = NULL;
-    erts_smp_atomic32_init_nob(&dh->port_count, 0);
-    erts_smp_refc_init(&(dh->refc), (erts_aint_t) 0);
+    erts_atomic32_init_nob(&dh->port_count, 0);
+    erts_refc_init(&(dh->refc), (erts_aint_t) 0);
     dh->status = -1;
     dh->reload_full_path = NULL;
     dh->reload_driver_name = NULL;
@@ -1616,7 +1616,7 @@ static int reload_driver_entry(DE_Handle *dh)
     dh->reload_full_path = NULL;
     dh->reload_driver_name = NULL;
 
-    ASSERT(erts_smp_refc_read(&(dh->refc),0) == 0);
+    ASSERT(erts_refc_read(&(dh->refc),0) == 0);
     ASSERT(dh->full_path != NULL);
     erts_free(ERTS_ALC_T_DDLL_HANDLE, (void *) dh->full_path);
     dh->full_path = NULL;
@@ -1647,7 +1647,7 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type,
     ErtsMessage *mp;
     ErtsProcLocks rp_locks = 0;
     ErlOffHeap *ohp;
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
 
     assert_drv_list_rwlocked();
     if (errcode != 0) {
@@ -1673,8 +1673,8 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type,
 	mess = TUPLE5(hp,type,r,am_driver,driver_name,tag);
     }
     erts_queue_message(proc, rp_locks, mp, mess, am_system);
-    erts_smp_proc_unlock(proc, rp_locks);
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    erts_proc_unlock(proc, rp_locks);
+    ERTS_CHK_NO_PROC_LOCKS;
 }
 
 static void notify_all(DE_Handle *dh, char *name, Uint awaiting, Eterm type, Eterm tag)
@@ -1746,7 +1746,7 @@ static Eterm build_load_error(Process *p, int code)
 {
     int need = load_error_need(code);
     Eterm *hp = NULL;
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));
     if (need) {
 	hp = HAlloc(p,need);
     }
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index dcbef9ce2d..2ff95a3338 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -887,13 +887,13 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap,
 	     * is being inspected...
 	     */
 	    ASSERT(locks & ERTS_PROC_LOCK_MAIN);
-	    ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp);
+	    ERTS_MSGQ_MV_INQ2PRIVQ(rp);
 	    locks &= ~ERTS_PROC_LOCK_MSGQ;
 	    unlock_locks |= ERTS_PROC_LOCK_MSGQ;
 	}
 
 	if (unlock_locks)
-	    erts_smp_proc_unlock(rp, unlock_locks);
+	    erts_proc_unlock(rp, unlock_locks);
 
     }
 
@@ -951,7 +951,7 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap,
     if (c_p == rp)
 	locks &= ~ERTS_PROC_LOCK_MAIN;
     if (locks && rp)
-	erts_smp_proc_unlock(rp, locks);
+	erts_proc_unlock(rp, locks);
 
     if (res_elem_ix != &def_res_elem_ix_buf[0])
 	erts_free(ERTS_ALC_T_TMP, res_elem_ix);
@@ -1042,7 +1042,7 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2)
 	ERTS_BIF_YIELD2(bif_export[BIF_process_info_2], BIF_P,
 			BIF_ARG_1, BIF_ARG_2);
     else if (rp != BIF_P && ERTS_PROC_PENDING_EXIT(rp)) {
-	erts_smp_proc_unlock(rp, info_locks|ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(rp, info_locks|ERTS_PROC_LOCK_STATUS);
 	ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined);
     }
     else {
@@ -1062,13 +1062,13 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2)
 	     * is being inspected...
 	     */
 	    ASSERT(info_locks & ERTS_PROC_LOCK_MAIN);
-	    ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp);
+	    ERTS_MSGQ_MV_INQ2PRIVQ(rp);
 	    info_locks &= ~ERTS_PROC_LOCK_MSGQ;
 	    unlock_locks |= ERTS_PROC_LOCK_MSGQ;
 	}
 
 	if (unlock_locks)
-	    erts_smp_proc_unlock(rp, unlock_locks);
+	    erts_proc_unlock(rp, unlock_locks);
 
 	res = process_info_aux(BIF_P, rp, info_locks, pid, BIF_ARG_2, 0);
     }
@@ -1077,7 +1077,7 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2)
     if (BIF_P == rp)
 	info_locks &= ~ERTS_PROC_LOCK_MAIN;
     if (rp && info_locks)
-	erts_smp_proc_unlock(rp, info_locks);
+	erts_proc_unlock(rp, info_locks);
 
     ASSERT(!(BIF_P->flags & F_P2PNR_RESCHED));
     BIF_RET(res);
@@ -1364,7 +1364,7 @@ process_info_aux(Process *BIF_P,
 	break;
 
     case am_trap_exit: {
-	erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state);
+	erts_aint32_t state = erts_atomic32_read_nob(&rp->state);
 	hp = HAlloc(BIF_P, 3);
 	if (state & ERTS_PSFLG_TRAP_EXIT)
 	    res = am_true;
@@ -2252,7 +2252,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	res = TUPLE2(hp, am_sequential_tracer, val);
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_garbage_collection){
-	Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
+	Uint val = (Uint) erts_atomic32_read_nob(&erts_max_gen_gcs);
 	Eterm tup;
 	hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2 + 3+2);
 
@@ -2270,7 +2270,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_fullsweep_after){
-	Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
+	Uint val = (Uint) erts_atomic32_read_nob(&erts_max_gen_gcs);
 	hp = HAlloc(BIF_P, 3);
 	res = TUPLE2(hp, am_fullsweep_after, make_small(val));
 	BIF_RET(res);
@@ -2303,8 +2303,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	erts_dsprintf_buf_t *dsbufp = erts_create_info_dsbuf(0);
 
 	/* Need to be the only thread running... */
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_smp_thr_progress_block();
+	erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_block();
 
 	if (BIF_ARG_1 == am_info)
 	    info(ERTS_PRINT_DSBUF, (void *) dsbufp);
@@ -2315,8 +2315,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	else
 	    distribution_info(ERTS_PRINT_DSBUF, (void *) dsbufp);
 
-	erts_smp_thr_progress_unblock();
-	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_unblock();
+	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
 	ASSERT(dsbufp && dsbufp->str);
 	res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len);
@@ -2325,7 +2325,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     } else if (ERTS_IS_ATOM_STR("dist_ctrl", BIF_ARG_1)) {
 	DistEntry *dep;
 	i = 0;
-        erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
+        erts_rwmtx_rlock(&erts_dist_table_rwmtx);
 	for (dep = erts_visible_dist_entries; dep; dep = dep->next) 
 	    ++i;
 	for (dep = erts_hidden_dist_entries; dep; dep = dep->next)
@@ -2348,7 +2348,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	    res = CONS(hp, tpl, res);
 	    hp += 2;
 	}
-        erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
+        erts_rwmtx_runlock(&erts_dist_table_rwmtx);
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_system_version) {
 	erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0);
@@ -2906,7 +2906,7 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt,
 {
     Eterm res = THE_NON_VALUE;
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
     if (item == am_id) {
 	if (hpp)
@@ -3122,7 +3122,7 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt,
 	    goto done;
 	}
 	res = ((ERTS_PTS_FLG_PARALLELISM &
-		erts_smp_atomic32_read_nob(&prt->sched.flags))
+		erts_atomic32_read_nob(&prt->sched.flags))
 	       ? am_true
 	       : am_false);
     }
@@ -3198,7 +3198,7 @@ fun_info_2(BIF_ALIST_2)
 	    }
 	    break;
 	case am_refc:
-	    val = erts_make_integer(erts_smp_atomic_read_nob(&funp->fe->refc), p);
+	    val = erts_make_integer(erts_atomic_read_nob(&funp->fe->refc), p);
 	    hp = HAlloc(p, 3);
 	    break;
 	case am_arity:
@@ -3303,7 +3303,7 @@ BIF_RETTYPE is_process_alive_1(BIF_ALIST_1)
 	   BIF_RET(am_false);
        }
        else {
-	   if (erts_smp_atomic32_read_acqb(&rp->state)
+	   if (erts_atomic32_read_acqb(&rp->state)
 		 & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING))
 	       ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_false);
 	   else
@@ -3338,7 +3338,7 @@ BIF_RETTYPE process_display_2(BIF_ALIST_2)
 		       BIF_ARG_1, BIF_ARG_2);
    if (rp != BIF_P && ERTS_PROC_PENDING_EXIT(rp)) {
        Eterm args[2] = {BIF_ARG_1, BIF_ARG_2};
-       erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL);
+       erts_proc_unlock(rp, ERTS_PROC_LOCKS_ALL);
        ERTS_BIF_AWAIT_X_APPLY_TRAP(BIF_P,
 				   BIF_ARG_1,
 				   am_erlang,
@@ -3347,7 +3347,7 @@ BIF_RETTYPE process_display_2(BIF_ALIST_2)
 				   2);
    }
    erts_stack_dump(ERTS_PRINT_STDERR, NULL, rp);
-   erts_smp_proc_unlock(rp, (BIF_P == rp
+   erts_proc_unlock(rp, (BIF_P == rp
 			     ? ERTS_PROC_LOCKS_ALL_MINOR
 			     : ERTS_PROC_LOCKS_ALL));
    BIF_RET(am_true);
@@ -3527,7 +3527,7 @@ BIF_RETTYPE error_logger_warning_map_0(BIF_ALIST_0)
     BIF_RET(erts_error_logger_warnings);
 }
 
-static erts_smp_atomic_t available_internal_state;
+static erts_atomic_t available_internal_state;
 
 static int empty_magic_ref_destructor(Binary *bin)
 {
@@ -3540,7 +3540,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
      * NOTE: Only supposed to be used for testing, and debugging.
      */
 
-    if (!erts_smp_atomic_read_nob(&available_internal_state)) {
+    if (!erts_atomic_read_nob(&available_internal_state)) {
 	BIF_ERROR(BIF_P, EXC_UNDEF);
     }
 
@@ -3583,9 +3583,9 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    int no_errors;
 	    ErtsCheckIoDebugInfo ciodi = {0};
 #ifdef HAVE_ERTS_CHECK_IO_DEBUG
-	    erts_smp_proc_unlock(BIF_P,ERTS_PROC_LOCK_MAIN);
+	    erts_proc_unlock(BIF_P,ERTS_PROC_LOCK_MAIN);
 	    no_errors = erts_check_io_debug(&ciodi);
-	    erts_smp_proc_lock(BIF_P,ERTS_PROC_LOCK_MAIN);
+	    erts_proc_lock(BIF_P,ERTS_PROC_LOCK_MAIN);
 #else
 	    no_errors = 0;
 #endif
@@ -3635,9 +3635,9 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	}
 	else if (ERTS_IS_ATOM_STR("nbalance", BIF_ARG_1)) {
 	    Uint n;
-	    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	    n = erts_debug_nbalance();
-	    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	    BIF_RET(erts_make_integer(n, BIF_P));
 	}
 	else if (ERTS_IS_ATOM_STR("available_internal_state", BIF_ARG_1)) {
@@ -3652,11 +3652,11 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	}
 	else if (ERTS_IS_ATOM_STR("memory", BIF_ARG_1)) {
 	    Eterm res;
-	    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	    erts_smp_thr_progress_block();
-	    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	    erts_thr_progress_block();
+	    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	    res = erts_memory(NULL, NULL, BIF_P, THE_NON_VALUE);
-	    erts_smp_thr_progress_unblock();
+	    erts_thr_progress_unblock();
 	    BIF_RET(res);
 	}
         else if (ERTS_IS_ATOM_STR("mmap", BIF_ARG_1)) {
@@ -3723,11 +3723,11 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 				      tp[2],
 				      ERTS_PROC_LOCK_LINK);
 		    if (!p) {
-			ERTS_SMP_ASSERT_IS_NOT_EXITING(BIF_P);
+			ERTS_ASSERT_IS_NOT_EXITING(BIF_P);
 			BIF_RET(am_undefined);
 		    }
 		    res = make_link_list(BIF_P, ERTS_P_LINKS(p), NIL);
-		    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+		    erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 		    BIF_RET(res);
 		}
 		else if(is_internal_port(tp[2])) {
@@ -3746,10 +3746,10 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 		    DistEntry *dep = erts_find_dist_entry(tp[2]);
 		    if(dep) {
 			Eterm subres;
-			erts_smp_de_links_lock(dep);
+			erts_de_links_lock(dep);
 			subres = make_link_list(BIF_P, dep->nlinks, NIL);
 			subres = make_link_list(BIF_P, dep->node_links, subres);
-			erts_smp_de_links_unlock(dep);
+			erts_de_links_unlock(dep);
 			erts_deref_dist_entry(dep);
 			BIF_RET(subres);
 		    } else {
@@ -3768,19 +3768,19 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 				      tp[2],
 				      ERTS_PROC_LOCK_LINK);
 		    if (!p) {
-			ERTS_SMP_ASSERT_IS_NOT_EXITING(BIF_P);
+			ERTS_ASSERT_IS_NOT_EXITING(BIF_P);
 			BIF_RET(am_undefined);
 		    }
 		    res = make_monitor_list(BIF_P, ERTS_P_MONITORS(p));
-		    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+		    erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 		    BIF_RET(res);
 		} else if(is_node_name_atom(tp[2])) {
 		    DistEntry *dep = erts_find_dist_entry(tp[2]);
 		    if(dep) {
 			Eterm ml;
-			erts_smp_de_links_lock(dep);
+			erts_de_links_lock(dep);
 			ml = make_monitor_list(BIF_P, dep->monitors);
-			erts_smp_de_links_unlock(dep);
+			erts_de_links_unlock(dep);
 			erts_deref_dist_entry(dep);
 			BIF_RET(ml);
 		    } else {
@@ -3808,7 +3808,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 		}
 		else {
 		    Eterm res = ERTS_PROC_PENDING_EXIT(rp) ? am_true : am_false;
-		    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+		    erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
 		    BIF_RET(res);
 		}
 	    }
@@ -3862,10 +3862,10 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 		Eterm res = am_undefined;
 		DistEntry *dep = erts_sysname_to_connected_dist_entry(tp[2]);
 		if (dep) {
-		    erts_smp_de_rlock(dep);
+		    erts_de_rlock(dep);
 		    if (is_internal_port(dep->cid))
 			res = dep->cid;
-		    erts_smp_de_runlock(dep);
+		    erts_de_runlock(dep);
 		    erts_deref_dist_entry(dep);
 		}
 		BIF_RET(res);
@@ -4010,7 +4010,7 @@ BIF_RETTYPE erts_internal_system_check_1(BIF_ALIST_1)
     BIF_ERROR(BIF_P, BADARG);
 }
 
-static erts_smp_atomic_t hipe_test_reschedule_flag;
+static erts_atomic_t hipe_test_reschedule_flag;
 
 #if defined(VALGRIND) && defined(__GNUC__)
 /* Force noinline for valgrind suppression */
@@ -4034,7 +4034,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
     if (ERTS_IS_ATOM_STR("available_internal_state", BIF_ARG_1)
 	&& (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false)) {
 	erts_aint_t on = (erts_aint_t) (BIF_ARG_2 == am_true);
-	erts_aint_t prev_on = erts_smp_atomic_xchg_nob(&available_internal_state, on);
+	erts_aint_t prev_on = erts_atomic_xchg_nob(&available_internal_state, on);
 	if (on) {
 	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
 	    erts_dsprintf(dsbufp, "Process %T ", BIF_P->common.id);
@@ -4050,7 +4050,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	BIF_RET(prev_on ? am_true : am_false);
     }
 
-    if (!erts_smp_atomic_read_nob(&available_internal_state)) {
+    if (!erts_atomic_read_nob(&available_internal_state)) {
 	BIF_ERROR(BIF_P, EXC_UNDEF);
     }
 
@@ -4074,13 +4074,13 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	    Sint ms;
 	    if (term_to_Sint(BIF_ARG_2, &ms) != 0) {
 		if (ms > 0) {
-		    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+		    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 		    if (block)
-			erts_smp_thr_progress_block();
+			erts_thr_progress_block();
 		    while (erts_milli_sleep((long) ms) != 0);
 		    if (block)
-			erts_smp_thr_progress_unblock();
-		    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+			erts_thr_progress_unblock();
+		    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 		}
 		BIF_RET(am_true);
 	    }
@@ -4089,9 +4089,9 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	    Sint ms;
 	    if (term_to_Sint(BIF_ARG_2, &ms) != 0) {
 		if (ms > 0) {
-		    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+		    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 		    while (erts_milli_sleep((long) ms) != 0);
-		    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+		    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 		}
 		BIF_RET(am_true);
 	    }
@@ -4174,7 +4174,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 						 0);
 		    if (BIF_P == rp)
 			rp_locks &= ~ERTS_PROC_LOCK_MAIN;
-		    erts_smp_proc_unlock(rp, rp_locks);
+		    erts_proc_unlock(rp, rp_locks);
 		    if (xres > 1) {
 			DECL_AM(message);
 			BIF_RET(AM_message);
@@ -4236,14 +4236,14 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	}
 	else if (ERTS_IS_ATOM_STR("hipe_test_reschedule_suspend", BIF_ARG_1)) {
 	    /* Used by hipe test suites */
-	    erts_aint_t flag = erts_smp_atomic_read_nob(&hipe_test_reschedule_flag);
+	    erts_aint_t flag = erts_atomic_read_nob(&hipe_test_reschedule_flag);
 	    if (!flag && BIF_ARG_2 != am_false) {
-		erts_smp_atomic_set_nob(&hipe_test_reschedule_flag, 1);
+		erts_atomic_set_nob(&hipe_test_reschedule_flag, 1);
 		erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
 		ERTS_BIF_YIELD2(bif_export[BIF_erts_debug_set_internal_state_2],
 				BIF_P, BIF_ARG_1, BIF_ARG_2);
 	    }
-	    erts_smp_atomic_set_nob(&hipe_test_reschedule_flag, !flag);
+	    erts_atomic_set_nob(&hipe_test_reschedule_flag, !flag);
 	    BIF_RET(NIL);
 	}
 	else if (ERTS_IS_ATOM_STR("hipe_test_reschedule_resume", BIF_ARG_1)) {
@@ -4254,7 +4254,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	    if (rp) {
 		erts_resume(rp, ERTS_PROC_LOCK_STATUS);
 		res = am_true;
-		erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+		erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
 	    }
 	    BIF_RET(res);
 	}
@@ -4271,9 +4271,9 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 		BIF_RET(am_false);
 	    else {
 		Uint32 con_id;
-		erts_smp_de_rlock(dep);
+		erts_de_rlock(dep);
 		con_id = dep->connection_id;
-		erts_smp_de_runlock(dep);
+		erts_de_runlock(dep);
 		erts_kill_dist_connection(dep, con_id);
 		erts_deref_dist_entry(dep);
 		BIF_RET(am_true);
@@ -4292,12 +4292,12 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 		BIF_ERROR(BIF_P, BADARG);
 	    }
 
-	    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	    erts_smp_thr_progress_block();
+	    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	    erts_thr_progress_block();
 	    old_use_opt = !erts_disable_proc_not_running_opt;
 	    erts_disable_proc_not_running_opt = !use_opt;
-	    erts_smp_thr_progress_unblock();
-	    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	    erts_thr_progress_unblock();
+	    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	    BIF_RET(old_use_opt ? am_true : am_false);
 	}
 	else if (ERTS_IS_ATOM_STR("wait", BIF_ARG_1)) {
@@ -4326,9 +4326,9 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	    Sint64 msecs;
 	    if (term_to_Sint64(BIF_ARG_2, &msecs)) {
 		/* Negative value restore original value... */
-		erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+		erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 		erts_debug_test_node_tab_delayed_delete(msecs);
-		erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+		erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 		BIF_RET(am_ok);
 	    }
 	}
@@ -4776,8 +4776,8 @@ static void os_info_init(void)
 void
 erts_bif_info_init(void)
 {
-    erts_smp_atomic_init_nob(&available_internal_state, 0);
-    erts_smp_atomic_init_nob(&hipe_test_reschedule_flag, 0);
+    erts_atomic_init_nob(&available_internal_state, 0);
+    erts_atomic_init_nob(&hipe_test_reschedule_flag, 0);
 
     alloc_info_trap = erts_export_put(am_erlang, am_alloc_info, 1);
     alloc_sizes_trap = erts_export_put(am_erlang, am_alloc_sizes, 1);
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 106f18b747..4b73be55c6 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -86,25 +86,25 @@ BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2)
         erts_make_ref_in_array(port->async_open_port->ref);
         port->async_open_port->to = BIF_P->common.id;
 
-        erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK);
+        erts_proc_lock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK);
         if (ERTS_PROC_PENDING_EXIT(BIF_P)) {
 	    /* need to exit caller instead */
-	    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK);
+	    erts_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK);
 	    KILL_CATCHES(BIF_P);
 	    BIF_P->freason = EXC_EXIT;
             erts_port_release(port);
             BIF_RET(am_badarg);
 	}
 
-	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(BIF_P);
+	ERTS_MSGQ_MV_INQ2PRIVQ(BIF_P);
 	BIF_P->msg.save = BIF_P->msg.last;
 
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	erts_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE);
 
         res = erts_proc_store_ref(BIF_P, port->async_open_port->ref);
     } else {
         res = port->common.id;
-        erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
+        erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
     }
 
     erts_add_link(&ERTS_P_LINKS(port), LINK_PID, BIF_P->common.id);
@@ -114,7 +114,7 @@ BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2)
         trace_proc(BIF_P, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, BIF_P,
                    am_link, port->common.id);
 
-    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
+    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
 
     erts_port_release(port);
 
@@ -271,7 +271,7 @@ BIF_RETTYPE erts_internal_port_call_3(BIF_ALIST_3)
 	break;
     }
 
-    state = erts_smp_atomic32_read_acqb(&BIF_P->state);
+    state = erts_atomic32_read_acqb(&BIF_P->state);
     if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
 	if (state & ERTS_PSFLG_PENDING_EXIT)
 	    erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
@@ -319,7 +319,7 @@ BIF_RETTYPE erts_internal_port_control_3(BIF_ALIST_3)
 	break;
     }
 
-    state = erts_smp_atomic32_read_acqb(&BIF_P->state);
+    state = erts_atomic32_read_acqb(&BIF_P->state);
     if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
 	if (state & ERTS_PSFLG_PENDING_EXIT)
 	    erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
@@ -509,7 +509,7 @@ cleanup_old_port_data(erts_aint_t data)
     else {
 	ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data;
 	size_t size;
-	ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
+	ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
 	size = sizeof(ErtsPortDataHeap) + (pdhp->hsize-1)*sizeof(Eterm);
 	erts_schedule_thr_prgr_later_cleanup_op(free_port_data_heap,
 						(void *) pdhp,
@@ -521,21 +521,21 @@ cleanup_old_port_data(erts_aint_t data)
 void
 erts_init_port_data(Port *prt)
 {
-    erts_smp_atomic_init_nob(&prt->data, (erts_aint_t) am_undefined);
+    erts_atomic_init_nob(&prt->data, (erts_aint_t) am_undefined);
 }
 
 void
 erts_cleanup_port_data(Port *prt)
 {
     ASSERT(erts_atomic32_read_nob(&prt->state) & ERTS_PORT_SFLGS_INVALID_LOOKUP);
-    cleanup_old_port_data(erts_smp_atomic_xchg_nob(&prt->data,
+    cleanup_old_port_data(erts_atomic_xchg_nob(&prt->data,
 						   (erts_aint_t) NULL));
 }
 
 Uint
 erts_port_data_size(Port *prt)
 {
-    erts_aint_t data = erts_smp_atomic_read_ddrb(&prt->data);
+    erts_aint_t data = erts_atomic_read_ddrb(&prt->data);
 
     if ((data & 0x3) != 0) {
 	ASSERT(is_immed((Eterm) (UWord) data));
@@ -550,7 +550,7 @@ erts_port_data_size(Port *prt)
 ErlOffHeap *
 erts_port_data_offheap(Port *prt)
 {
-    erts_aint_t data = erts_smp_atomic_read_ddrb(&prt->data);
+    erts_aint_t data = erts_atomic_read_ddrb(&prt->data);
 
     if ((data & 0x3) != 0) {
 	ASSERT(is_immed((Eterm) (UWord) data));
@@ -595,11 +595,11 @@ BIF_RETTYPE port_set_data_2(BIF_ALIST_2)
 	ASSERT((data & 0x3) == 0);
     }
 
-    data = erts_smp_atomic_xchg_wb(&prt->data, data);
+    data = erts_atomic_xchg_wb(&prt->data, data);
 
     if (data == (erts_aint_t)NULL) {
 	/* Port terminated by racing thread */
-	data = erts_smp_atomic_xchg_wb(&prt->data, data);
+	data = erts_atomic_xchg_wb(&prt->data, data);
 	ASSERT(data != (erts_aint_t)NULL);
 	cleanup_old_port_data(data);
 	BIF_ERROR(BIF_P, BADARG);
@@ -622,7 +622,7 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1)
     if (!prt)
         BIF_ERROR(BIF_P, BADARG);
 
-    data = erts_smp_atomic_read_ddrb(&prt->data);
+    data = erts_atomic_read_ddrb(&prt->data);
     if (data == (erts_aint_t)NULL)
         BIF_ERROR(BIF_P, BADARG);  /* Port terminated by racing thread */
 
@@ -917,7 +917,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
     }
     
 
-    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
+    erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
 
     port = erts_open_driver(driver, p->common.id, name_buf, &opts, err_typep, err_nump);
 #ifdef USE_VM_PROBES
@@ -934,7 +934,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
     if (port && IS_TRACED_FL(port, F_TRACE_PORTS))
         trace_port(port, am_getting_linked, p->common.id);
 
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+    erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
 
     if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) {
         trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in);
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index c476e18c33..b02f966558 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -373,11 +373,11 @@ static void smp_bp_finisher(void* null)
 	finish_bp.stager = NULL;
 #endif
 	erts_release_code_write_permission();
-	erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+	erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
 	if (!ERTS_PROC_IS_EXITING(p)) {
 	    erts_resume(p, ERTS_PROC_LOCK_STATUS);
 	}
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 	erts_proc_dec_refc(p);
     }
 }
@@ -389,8 +389,8 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on,
 			       struct trace_pattern_flags *trace_pattern_flags,
 			       ErtsTracer *meta_tracer)
 {
-    ERTS_SMP_LC_ASSERT(erts_has_code_write_permission() ||
-		       erts_smp_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(erts_has_code_write_permission() ||
+		       erts_thr_progress_is_blocking());
     if (trace_pattern_is_on)
 	*trace_pattern_is_on = erts_default_trace_pattern_is_on;
     if (match_spec)
@@ -405,8 +405,8 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on,
 
 int erts_is_default_trace_enabled(void)
 {
-    ERTS_SMP_LC_ASSERT(erts_has_code_write_permission() ||
-		       erts_smp_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(erts_has_code_write_permission() ||
+		       erts_thr_progress_is_blocking());
     return erts_default_trace_pattern_is_on;
 }
 
@@ -610,13 +610,13 @@ Eterm erts_internal_trace_3(BIF_ALIST_3)
 	    goto error;
 
         if (start_trace(tracee_p, tracer, &tracee_p->common, on, mask)) {
-	    erts_smp_proc_unlock(tracee_p,
+	    erts_proc_unlock(tracee_p,
 				 (tracee_p == p
 				  ? ERTS_PROC_LOCKS_ALL_MINOR
 				  : ERTS_PROC_LOCKS_ALL));
 	    goto already_traced;
         }
-        erts_smp_proc_unlock(tracee_p,
+        erts_proc_unlock(tracee_p,
 			     (tracee_p == p
 			      ? ERTS_PROC_LOCKS_ALL_MINOR
 			      : ERTS_PROC_LOCKS_ALL));
@@ -689,8 +689,8 @@ Eterm erts_internal_trace_3(BIF_ALIST_3)
                     mods = 1;
             }
 
-	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
-	    erts_smp_thr_progress_block();
+	    erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
+	    erts_thr_progress_block();
 	    system_blocked = 1;
 
 	    ok = 1;
@@ -755,8 +755,8 @@ Eterm erts_internal_trace_3(BIF_ALIST_3)
     }
 
     if (system_blocked) {
-	erts_smp_thr_progress_unblock();
-	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_unblock();
+	erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     }
     erts_release_code_write_permission();
     ERTS_TRACER_CLEAR(&tracer);
@@ -772,8 +772,8 @@ Eterm erts_internal_trace_3(BIF_ALIST_3)
     ERTS_TRACER_CLEAR(&tracer);
 
     if (system_blocked) {
-	erts_smp_thr_progress_unblock();
-	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_unblock();
+	erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     }
     erts_release_code_write_permission();
 
@@ -862,7 +862,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
         trace_flags = ERTS_TRACE_FLAGS(tracee);
 
 	if (tracee != p)
-	    erts_smp_proc_unlock(tracee, ERTS_PROC_LOCK_MAIN);
+	    erts_proc_unlock(tracee, ERTS_PROC_LOCK_MAIN);
     } else if (is_external_pid(pid_spec)
 	       && external_pid_dist_entry(pid_spec) == erts_this_dist_entry) {
 	    return am_undefined;
@@ -1040,22 +1040,22 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
     mfa[2] = signed_val(tp[3]);
 
     if ( (key == am_call_time) || (key == am_all)) {
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
-	erts_smp_thr_progress_block();
+	erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_block();
     }
 #ifdef ERTS_DIRTY_SCHEDULERS
-    erts_smp_mtx_lock(&erts_dirty_bp_ix_mtx);
+    erts_mtx_lock(&erts_dirty_bp_ix_mtx);
 #endif
 
 
     r = function_is_traced(p, mfa, &ms, &ms_meta, &meta, &count, &call_time);
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-    erts_smp_mtx_unlock(&erts_dirty_bp_ix_mtx);
+    erts_mtx_unlock(&erts_dirty_bp_ix_mtx);
 #endif
     if ( (key == am_call_time) || (key == am_all)) {
-	erts_smp_thr_progress_unblock();
-	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_unblock();
+	erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     }
 
     switch (r) {
@@ -1507,7 +1507,7 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
     finish_bp.local = flags.breakpoint;
 
     if (is_blocking) {
-	ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+	ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
 	while (erts_finish_breakpointing()) {
 	    /* Empty loop body */
 	}
@@ -1565,7 +1565,7 @@ consolidate_event_tracing(ErtsTracingEvent te[])
 int
 erts_finish_breakpointing(void)
 {
-    ERTS_SMP_LC_ASSERT(erts_has_code_write_permission());
+    ERTS_LC_ASSERT(erts_has_code_write_permission());
 
     /*
      * Memory barriers will be issued for all schedulers *before*
@@ -1987,8 +1987,8 @@ BIF_RETTYPE seq_trace_print_2(BIF_ALIST_2)
 
 void erts_system_monitor_clear(Process *c_p) {
     if (c_p) {
-	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
-	erts_smp_thr_progress_block();
+	erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_block();
     }
     erts_set_system_monitor(NIL);
     erts_system_monitor_long_gc = 0;
@@ -1997,8 +1997,8 @@ void erts_system_monitor_clear(Process *c_p) {
     erts_system_monitor_flags.busy_port = 0;
     erts_system_monitor_flags.busy_dist_port = 0;
     if (c_p) {
-	erts_smp_thr_progress_unblock();
-	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_unblock();
+	erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
     }
 }
 
@@ -2109,8 +2109,8 @@ system_monitor(Process *p, Eterm monitor_pid, Eterm list)
 	int busy_port, busy_dist_port;
 
 	system_blocked = 1;
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
-	erts_smp_thr_progress_block();
+	erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_block();
 
 	if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, monitor_pid, 0))
 	    goto error;
@@ -2149,16 +2149,16 @@ system_monitor(Process *p, Eterm monitor_pid, Eterm list)
 	erts_system_monitor_flags.busy_port = !!busy_port;
 	erts_system_monitor_flags.busy_dist_port = !!busy_dist_port;
 
-	erts_smp_thr_progress_unblock();
-	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_unblock();
+	erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
 	BIF_RET(prev);
     }
 
  error:
 
     if (system_blocked) {
-	erts_smp_thr_progress_unblock();
-	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_unblock();
+	erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     }
 
     BIF_ERROR(p, BADARG);
@@ -2168,8 +2168,8 @@ system_monitor(Process *p, Eterm monitor_pid, Eterm list)
 
 void erts_system_profile_clear(Process *c_p) {
     if (c_p) {
-	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
-	erts_smp_thr_progress_block();
+	erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_block();
     }
     erts_set_system_profile(NIL);
     erts_system_profile_flags.scheduler = 0;
@@ -2177,8 +2177,8 @@ void erts_system_profile_clear(Process *c_p) {
     erts_system_profile_flags.runnable_ports = 0;
     erts_system_profile_flags.exclusive = 0;
     if (c_p) {
-	erts_smp_thr_progress_unblock();
-	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_unblock();
+	erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
     }
 }
 
@@ -2241,8 +2241,8 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2)
 	int scheduler, runnable_procs, runnable_ports, exclusive;
 	system_blocked = 1;
 	
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
-	erts_smp_thr_progress_block();
+	erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_block();
 
 	/* Check if valid process, no locks are taken */
 
@@ -2293,8 +2293,8 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2)
 	erts_system_profile_flags.runnable_procs = !!runnable_procs;
 	erts_system_profile_flags.exclusive = !!exclusive;
 	erts_system_profile_ts_type = ts;
-	erts_smp_thr_progress_unblock();
-	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+	erts_thr_progress_unblock();
+	erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
 	
 	BIF_RET(prev);
 		
@@ -2302,8 +2302,8 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2)
 
     error:
 	if (system_blocked) {
-	    erts_smp_thr_progress_unblock();
-	    erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+	    erts_thr_progress_unblock();
+	    erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     	}
 
     BIF_ERROR(p, BADARG);
@@ -2328,7 +2328,7 @@ typedef struct {
     Eterm ref;
     Eterm ref_heap[ERTS_REF_THING_SIZE];
     Eterm target;
-    erts_smp_atomic32_t refc;
+    erts_atomic32_t refc;
 } ErtsTraceDeliveredAll;
 
 static void
@@ -2336,7 +2336,7 @@ reply_trace_delivered_all(void *vtdarp)
 {
     ErtsTraceDeliveredAll *tdarp = (ErtsTraceDeliveredAll *) vtdarp;
 
-    if (erts_smp_atomic32_dec_read_nob(&tdarp->refc) == 0) {
+    if (erts_atomic32_dec_read_nob(&tdarp->refc) == 0) {
         Eterm ref_copy, msg;
         Process *rp = tdarp->proc;
         Eterm *hp = NULL;
@@ -2370,7 +2370,7 @@ trace_delivered_1(BIF_ALIST_1)
         hp = &tdarp->ref_heap[0];
         tdarp->ref = STORE_NC(&hp, NULL, ref);
         tdarp->target = BIF_ARG_1;
-        erts_smp_atomic32_init_nob(&tdarp->refc,
+        erts_atomic32_init_nob(&tdarp->refc,
                                    (erts_aint32_t) erts_no_schedulers);
         erts_proc_add_refc(BIF_P, 1);
         erts_schedule_multi_misc_aux_work(0,
diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c
index 2f8adc87d5..aa79503819 100644
--- a/erts/emulator/beam/erl_bif_unique.c
+++ b/erts/emulator/beam/erl_bif_unique.c
@@ -136,7 +136,7 @@ Eterm erts_make_ref(Process *c_p)
     Eterm* hp;
     Uint32 ref[ERTS_REF_NUMBERS];
 
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
 
     hp = HAlloc(c_p, ERTS_REF_THING_SIZE);
 
@@ -803,7 +803,7 @@ BIF_RETTYPE make_ref_0(BIF_ALIST_0)
     BIF_RETTYPE res;
     Eterm* hp;
 
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
 
     hp = HAlloc(BIF_P, ERTS_REF_THING_SIZE);
 
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index b036b28dbf..05007e864e 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -291,7 +291,7 @@ typedef union {
      * atomics are used they might
      * differ in size.
      */
-    erts_smp_atomic_t smp_atomic_word;
+    erts_atomic_t smp_atomic_word;
     erts_atomic_t atomic_word;
 } ErtsMagicIndirectionWord;
 
@@ -326,7 +326,7 @@ ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size,
 ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size,
 						 int (*destructor)(Binary *));
 ERTS_GLB_INLINE Binary *erts_create_magic_indirection(int (*destructor)(Binary *));
-ERTS_GLB_INLINE erts_smp_atomic_t *erts_smp_binary_to_magic_indirection(Binary *bp);
+ERTS_GLB_INLINE erts_atomic_t *erts_binary_to_magic_indirection(Binary *bp);
 ERTS_GLB_INLINE erts_atomic_t *erts_binary_to_magic_indirection(Binary *bp);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -519,16 +519,6 @@ erts_create_magic_indirection(int (*destructor)(Binary *))
                                              but word aligned */
 }
 
-ERTS_GLB_INLINE erts_smp_atomic_t *
-erts_smp_binary_to_magic_indirection(Binary *bp)
-{
-    ErtsMagicIndirectionWord *mip;
-    ASSERT(bp->intern.flags & BIN_FLAG_MAGIC);
-    ASSERT(ERTS_MAGIC_BIN_ATYPE(bp) == ERTS_ALC_T_MINDIRECTION);
-    mip = ERTS_MAGIC_BIN_UNALIGNED_DATA(bp);
-    return &mip->smp_atomic_word;
-}
-
 ERTS_GLB_INLINE erts_atomic_t *
 erts_binary_to_magic_indirection(Binary *bp)
 {
@@ -536,7 +526,7 @@ erts_binary_to_magic_indirection(Binary *bp)
     ASSERT(bp->intern.flags & BIN_FLAG_MAGIC);
     ASSERT(ERTS_MAGIC_BIN_ATYPE(bp) == ERTS_ALC_T_MINDIRECTION);
     mip = ERTS_MAGIC_BIN_UNALIGNED_DATA(bp);
-    return &mip->atomic_word;
+    return &mip->smp_atomic_word;
 }
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index e52b5b4bc5..2035b56eb5 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -69,12 +69,12 @@ static byte get_bit(byte b, size_t a_offs);
 #define byte_buf	(ErlBitsState.byte_buf_)
 #define byte_buf_len	(ErlBitsState.byte_buf_len_)
 
-static erts_smp_atomic_t bits_bufs_size;
+static erts_atomic_t bits_bufs_size;
 
 Uint
 erts_bits_bufs_size(void)
 {
-    return (Uint) erts_smp_atomic_read_nob(&bits_bufs_size);
+    return (Uint) erts_atomic_read_nob(&bits_bufs_size);
 }
 
 void
@@ -100,7 +100,7 @@ erts_init_bits(void)
     ERTS_CT_ASSERT(offsetof(ErtsBinary,driver.binary.orig_bytes)
                 == offsetof(Binary,orig_bytes));
 
-    erts_smp_atomic_init_nob(&bits_bufs_size, 0);
+    erts_atomic_init_nob(&bits_bufs_size, 0);
     /* erl_process.c calls erts_bits_init_state() on all state instances */
 }
 
@@ -735,7 +735,7 @@ static void
 ERTS_INLINE need_byte_buf(ERL_BITS_PROTO_1(int need))
 {
     if (byte_buf_len < need) {
-	erts_smp_atomic_add_nob(&bits_bufs_size, need - byte_buf_len);
+	erts_atomic_add_nob(&bits_bufs_size, need - byte_buf_len);
 	byte_buf_len = need;
 	byte_buf = erts_realloc(ERTS_ALC_T_BITS_BUF, byte_buf, byte_buf_len);
     }
diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
index 48b02c839e..49f9beb19f 100644
--- a/erts/emulator/beam/erl_cpu_topology.c
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -60,7 +60,7 @@ static int max_main_threads;
 static int reader_groups;
 
 static ErtsCpuBindData *scheduler2cpu_map;
-static erts_smp_rwmtx_t cpuinfo_rwmtx;
+static erts_rwmtx_t cpuinfo_rwmtx;
 
 typedef enum {
     ERTS_CPU_BIND_UNDEFINED,
@@ -441,7 +441,7 @@ erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp)
     int cgcc_ix;
 
     /* Unbind from cpu */
-    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rwlock(&cpuinfo_rwmtx);
     if (scheduler2cpu_map[esdp->no].bound_id >= 0
 	&& erts_unbind_from_cpu(cpuinfo) == 0) {
 	esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1;
@@ -460,7 +460,7 @@ erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp)
 	}
     }
     ASSERT(no_cpu_groups_callbacks == cgcc_ix);
-    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rwunlock(&cpuinfo_rwmtx);
 
     for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++)
 	cgcc[cgcc_ix].callback(1,
@@ -478,7 +478,7 @@ erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp)
 void
 erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(esdp->run_queue));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(esdp->run_queue));
 
     if (esdp->no <= max_main_threads)
 	erts_thr_set_main_status(1, (int) esdp->no);
@@ -495,8 +495,8 @@ erts_sched_check_cpu_bind(ErtsSchedulerData *esdp)
     erts_cpu_groups_map_t *cgm;
     erts_cpu_groups_callback_list_t *cgcl;
     erts_cpu_groups_callback_call_t *cgcc;
-    erts_smp_runq_unlock(esdp->run_queue);
-    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    erts_runq_unlock(esdp->run_queue);
+    erts_rwmtx_rwlock(&cpuinfo_rwmtx);
     cpu_id = scheduler2cpu_map[esdp->no].bind_id;
     if (cpu_id >= 0 && cpu_id != scheduler2cpu_map[esdp->no].bound_id) {
 	res = erts_bind_to_cpu(cpuinfo, cpu_id);
@@ -539,7 +539,7 @@ erts_sched_check_cpu_bind(ErtsSchedulerData *esdp)
     }
 
     ASSERT(no_cpu_groups_callbacks == cgcc_ix);
-    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rwunlock(&cpuinfo_rwmtx);
 
     for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++)
 	cgcc[cgcc_ix].callback(0,
@@ -549,7 +549,7 @@ erts_sched_check_cpu_bind(ErtsSchedulerData *esdp)
 
     erts_free(ERTS_ALC_T_TMP, cgcc);
 
-    erts_smp_runq_lock(esdp->run_queue);
+    erts_runq_lock(esdp->run_queue);
 }
 
 void
@@ -560,7 +560,7 @@ erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp)
     erts_cpu_groups_callback_list_t *cgcl;
     erts_cpu_groups_callback_call_t *cgcc;
 
-    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rlock(&cpuinfo_rwmtx);
 
     cgcc = erts_alloc(ERTS_ALC_T_TMP,
 		      (no_cpu_groups_callbacks
@@ -576,7 +576,7 @@ erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp)
     }
 
     ASSERT(no_cpu_groups_callbacks == cgcc_ix);
-    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    erts_rwmtx_runlock(&cpuinfo_rwmtx);
 
     for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++)
 	cgcc[cgcc_ix].callback(0,
@@ -596,7 +596,7 @@ write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size)
     int s_ix = 1;
     int cpu_ix;
 
-    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
 
     if (cpu_bind_order != ERTS_CPU_BIND_NONE && size) {
 
@@ -696,9 +696,9 @@ Eterm
 erts_bound_schedulers_term(Process *c_p)
 {
     ErtsCpuBindOrder order;
-    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rlock(&cpuinfo_rwmtx);
     order = cpu_bind_order;
-    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    erts_rwmtx_runlock(&cpuinfo_rwmtx);
     return bound_schedulers_term(order);
 }
 
@@ -711,7 +711,7 @@ erts_bind_schedulers(Process *c_p, Eterm how)
     int cpudata_size;
     ErtsCpuBindOrder old_cpu_bind_order;
 
-    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rwlock(&cpuinfo_rwmtx);
 
     if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP) {
 	if (cpu_bind_order == ERTS_CPU_BIND_NONE
@@ -767,7 +767,7 @@ erts_bind_schedulers(Process *c_p, Eterm how)
 
  done:
 
-    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rwunlock(&cpuinfo_rwmtx);
 
     if (notify)
 	erts_sched_notify_check_cpu_bind();
@@ -787,9 +787,9 @@ erts_sched_bind_atthrcreate_child(int unbind)
 {
     int res = 0;
     if (unbind) {
-	erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+	erts_rwmtx_rlock(&cpuinfo_rwmtx);
 	res = erts_unbind_from_cpu(cpuinfo);
-	erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+	erts_rwmtx_runlock(&cpuinfo_rwmtx);
     }
     return res;
 }
@@ -806,7 +806,7 @@ erts_sched_bind_atfork_prepare(void)
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
     int unbind = esdp != NULL && erts_is_scheduler_bound(esdp);
     if (unbind)
-	erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+	erts_rwmtx_rlock(&cpuinfo_rwmtx);
     return unbind;
 }
 
@@ -814,7 +814,7 @@ int
 erts_sched_bind_atfork_child(int unbind)
 {
     if (unbind) {
-	ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx)
+	ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx)
 			   || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
 	return erts_unbind_from_cpu(cpuinfo);
     }
@@ -825,7 +825,7 @@ void
 erts_sched_bind_atfork_parent(int unbind)
 {
     if (unbind)
-	erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+	erts_rwmtx_runlock(&cpuinfo_rwmtx);
 }
 
 Eterm
@@ -859,9 +859,9 @@ erts_fake_scheduler_bindings(Process *p, Eterm how)
 	return res;
     }
 
-    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rlock(&cpuinfo_rwmtx);
     create_tmp_cpu_topology_copy(&cpudata, &cpudata_size);
-    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    erts_rwmtx_runlock(&cpuinfo_rwmtx);
 
     if (!cpudata || fake_cpu_bind_order == ERTS_CPU_BIND_NONE)
 	ERTS_BIF_PREP_RET(res, am_false);
@@ -924,12 +924,12 @@ erts_get_schedulers_binds(Process *c_p)
     Eterm res = make_tuple(hp);
 
     *(hp++) = make_arityval(erts_no_schedulers);
-    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rlock(&cpuinfo_rwmtx);
     for (ix = 1; ix <= erts_no_schedulers; ix++)
 	*(hp++) = (scheduler2cpu_map[ix].bound_id >= 0
 		   ? make_small(scheduler2cpu_map[ix].bound_id)
 		   : AM_unbound);
-    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    erts_rwmtx_runlock(&cpuinfo_rwmtx);
     return res;
 }
 
@@ -1340,7 +1340,7 @@ erts_set_cpu_topology(Process *c_p, Eterm term)
     int cpudata_size = 0;
     Eterm res;
 
-    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rwlock(&cpuinfo_rwmtx);
     res = get_cpu_topology_term(c_p, ERTS_GET_USED_CPU_TOPOLOGY);
     if (term == am_undefined) {
 	if (user_cpudata)
@@ -1361,7 +1361,7 @@ erts_set_cpu_topology(Process *c_p, Eterm term)
     }
     else if (is_not_list(term)) {
     error:
-	erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+	erts_rwmtx_rwunlock(&cpuinfo_rwmtx);
 	res = THE_NON_VALUE;
 	goto done;
     }
@@ -1455,7 +1455,7 @@ erts_set_cpu_topology(Process *c_p, Eterm term)
 
     write_schedulers_bind_change(cpudata, cpudata_size);
 
-    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rwunlock(&cpuinfo_rwmtx);
     erts_sched_notify_check_cpu_bind();
 
  done:
@@ -1609,7 +1609,7 @@ erts_get_cpu_topology_term(Process *c_p, Eterm which)
 {
     Eterm res;
     int type;
-    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rlock(&cpuinfo_rwmtx);
     if (ERTS_IS_ATOM_STR("used", which))
 	type = ERTS_GET_USED_CPU_TOPOLOGY;
     else if (ERTS_IS_ATOM_STR("detected", which))
@@ -1622,7 +1622,7 @@ erts_get_cpu_topology_term(Process *c_p, Eterm which)
 	res = THE_NON_VALUE;
     else
 	res = get_cpu_topology_term(c_p, type);
-    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    erts_rwmtx_runlock(&cpuinfo_rwmtx);
     return res;
 }
 
@@ -1640,9 +1640,9 @@ get_logical_processors(int *conf, int *onln, int *avail)
 void
 erts_get_logical_processors(int *conf, int *onln, int *avail)
 {
-    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rlock(&cpuinfo_rwmtx);
     get_logical_processors(conf, onln, avail);
-    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    erts_rwmtx_runlock(&cpuinfo_rwmtx);
 }
 
 void
@@ -1700,9 +1700,9 @@ erts_init_cpu_topology(void)
 {
     int ix;
 
-    erts_smp_rwmtx_init(&cpuinfo_rwmtx, "cpu_info", NIL,
+    erts_rwmtx_init(&cpuinfo_rwmtx, "cpu_info", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
-    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rwlock(&cpuinfo_rwmtx);
 
     scheduler2cpu_map = erts_alloc(ERTS_ALC_T_CPUDATA,
 				   (sizeof(ErtsCpuBindData)
@@ -1720,13 +1720,13 @@ erts_init_cpu_topology(void)
 				       NULL);
 
     if (cpu_bind_order == ERTS_CPU_BIND_NONE)
-	erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+	erts_rwmtx_rwunlock(&cpuinfo_rwmtx);
     else {
 	erts_cpu_topology_t *cpudata;
 	int cpudata_size;
 	create_tmp_cpu_topology_copy(&cpudata, &cpudata_size);
 	write_schedulers_bind_change(cpudata, cpudata_size);
-	erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+	erts_rwmtx_rwunlock(&cpuinfo_rwmtx);
 	erts_sched_notify_check_cpu_bind();
 	destroy_tmp_cpu_topology_copy(cpudata);
     }
@@ -1736,7 +1736,7 @@ int
 erts_update_cpu_info(void)
 {
     int changed;
-    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rwlock(&cpuinfo_rwmtx);
     changed = erts_cpu_info_update(cpuinfo);
     if (changed) {
 	erts_cpu_topology_t *cpudata;
@@ -1769,7 +1769,7 @@ erts_update_cpu_info(void)
 	write_schedulers_bind_change(cpudata, cpudata_size);
 	destroy_tmp_cpu_topology_copy(cpudata);
     }
-    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rwunlock(&cpuinfo_rwmtx);
     if (changed)
 	erts_sched_notify_check_cpu_bind();
     return changed;
@@ -1786,7 +1786,7 @@ reader_groups_callback(int suspending,
 		       void *unused)
 {
     if (reader_groups && esdp->no <= max_main_threads)
-	erts_smp_rwmtx_set_reader_group(suspending ? 0 : group+1);
+	erts_rwmtx_set_reader_group(suspending ? 0 : group+1);
 }
 
 static Eterm get_cpu_groups_map(Process *c_p,
@@ -1815,9 +1815,9 @@ Eterm
 erts_get_reader_groups_map(Process *c_p)
 {
     Eterm res;
-    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    erts_rwmtx_rlock(&cpuinfo_rwmtx);
     res = get_cpu_groups_map(c_p, reader_groups_map, 1);
-    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    erts_rwmtx_runlock(&cpuinfo_rwmtx);
     return res;
 }
 
@@ -2197,7 +2197,7 @@ add_cpu_groups(int groups,
     erts_cpu_groups_callback_list_t *cgcl;
     erts_cpu_groups_map_t *cgm;
 
-    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
 
     if (use_groups > max_main_threads)
 	use_groups = max_main_threads;
@@ -2244,7 +2244,7 @@ cpu_groups_lookup(erts_cpu_groups_map_t *map,
 {
     int start, logical, ix;
 
-    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx)
+    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx)
 		       || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
 
     if (esdp->cpu_id < 0)
@@ -2272,7 +2272,7 @@ static void
 update_cpu_groups_maps(void)
 {
     erts_cpu_groups_map_t *cgm;
-    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
 
     for (cgm = cpu_groups_maps; cgm; cgm = cgm->next)
 	make_cpu_groups_map(cgm, 0);
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 2e5d401ca4..a21b9b9c0c 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -44,7 +44,7 @@
 #include "erl_binary.h"
 
 
-erts_smp_atomic_t erts_ets_misc_mem_size;
+erts_atomic_t erts_ets_misc_mem_size;
 
 /*
 ** Utility macros
@@ -189,7 +189,7 @@ static void delete_sched_table(Process *c_p, DbTable *tb);
 
 static void table_dec_refc(DbTable *tb, erts_aint_t min_val)
 {
-    if (erts_smp_refc_dectest(&tb->common.refc, min_val) == 0)
+    if (erts_refc_dectest(&tb->common.refc, min_val) == 0)
 	schedule_free_dbtable(tb);
 }
 
@@ -203,21 +203,21 @@ static ERTS_INLINE void
 make_btid(DbTable *tb)
 {
     Binary *btid = erts_create_magic_indirection(db_table_tid_destructor);
-    erts_smp_atomic_t *tbref = erts_smp_binary_to_magic_indirection(btid);
-    erts_smp_atomic_init_nob(tbref, (erts_aint_t) tb);
+    erts_atomic_t *tbref = erts_binary_to_magic_indirection(btid);
+    erts_atomic_init_nob(tbref, (erts_aint_t) tb);
     tb->common.btid = btid;
     /*
      * Table and magic indirection refer eachother,
      * and table is refered once by being alive...
      */
-    erts_smp_refc_init(&tb->common.refc, 2);
+    erts_refc_init(&tb->common.refc, 2);
     erts_refc_inc(&btid->intern.refc, 1);
 }
 
 static ERTS_INLINE DbTable* btid2tab(Binary* btid)
 {
-    erts_smp_atomic_t *tbref = erts_smp_binary_to_magic_indirection(btid);
-    return (DbTable *) erts_smp_atomic_read_nob(tbref);
+    erts_atomic_t *tbref = erts_binary_to_magic_indirection(btid);
+    return (DbTable *) erts_atomic_read_nob(tbref);
 }
 
 static DbTable *
@@ -225,7 +225,7 @@ tid2tab(Eterm tid)
 {
     DbTable *tb;
     Binary *btid;
-    erts_smp_atomic_t *tbref;
+    erts_atomic_t *tbref;
     if (!is_internal_magic_ref(tid))
         return NULL;
 
@@ -233,8 +233,8 @@ tid2tab(Eterm tid)
     if (ERTS_MAGIC_BIN_DESTRUCTOR(btid) != db_table_tid_destructor)
         return NULL;
 
-    tbref = erts_smp_binary_to_magic_indirection(btid);
-    tb = (DbTable *) erts_smp_atomic_read_nob(tbref);
+    tbref = erts_binary_to_magic_indirection(btid);
+    tb = (DbTable *) erts_atomic_read_nob(tbref);
 
     ASSERT(!tb || tb->common.btid == btid);
 
@@ -244,11 +244,11 @@ tid2tab(Eterm tid)
 static ERTS_INLINE int
 is_table_alive(DbTable *tb)
 {
-    erts_smp_atomic_t *tbref;
+    erts_atomic_t *tbref;
     DbTable *rtb;
 
-    tbref = erts_smp_binary_to_magic_indirection(tb->common.btid);
-    rtb = (DbTable *) erts_smp_atomic_read_nob(tbref);
+    tbref = erts_binary_to_magic_indirection(tb->common.btid);
+    rtb = (DbTable *) erts_atomic_read_nob(tbref);
 
     ASSERT(!rtb || rtb == tb);
 
@@ -267,8 +267,8 @@ tid_clear(Process *c_p, DbTable *tb)
 {
     DbTable *rtb;
     Binary *btid = tb->common.btid;
-    erts_smp_atomic_t *tbref = erts_smp_binary_to_magic_indirection(btid);
-    rtb = (DbTable *) erts_smp_atomic_xchg_nob(tbref, (erts_aint_t) NULL);
+    erts_atomic_t *tbref = erts_binary_to_magic_indirection(btid);
+    rtb = (DbTable *) erts_atomic_xchg_nob(tbref, (erts_aint_t) NULL);
     ASSERT(!rtb || tb == rtb);
     if (rtb) {
         table_dec_refc(tb, 1);
@@ -289,7 +289,7 @@ make_tid(Process *c_p, DbTable *tb)
 */
 #  define META_NAME_TAB_LOCK_CNT 16
 union {
-    erts_smp_rwmtx_t lck;
+    erts_rwmtx_t lck;
     byte _cache_line_alignment[64];
 }meta_name_tab_rwlocks[META_NAME_TAB_LOCK_CNT];
 static struct meta_name_tab_entry {
@@ -307,7 +307,7 @@ static unsigned meta_name_tab_mask;
 
 static ERTS_INLINE
 struct meta_name_tab_entry* meta_name_tab_bucket(Eterm name, 
-						 erts_smp_rwmtx_t** lockp)
+						 erts_rwmtx_t** lockp)
 {
     unsigned bix = atom_val(name) & meta_name_tab_mask;
     struct meta_name_tab_entry* bucket = &meta_name_tab[bix];
@@ -376,14 +376,14 @@ free_dbtable(void *vtb)
 {
     DbTable *tb = (DbTable *) vtb;
 #ifdef HARDDEBUG
-	if (erts_smp_atomic_read_nob(&tb->common.memory_size) != sizeof(DbTable)) {
+	if (erts_atomic_read_nob(&tb->common.memory_size) != sizeof(DbTable)) {
 	    erts_fprintf(stderr, "ets: free_dbtable memory remain=%ld fix=%x\n",
-			 erts_smp_atomic_read_nob(&tb->common.memory_size)-sizeof(DbTable),
+			 erts_atomic_read_nob(&tb->common.memory_size)-sizeof(DbTable),
 			 tb->common.fixations);
 	}
 #endif
-	erts_smp_rwmtx_destroy(&tb->common.rwlock);
-	erts_smp_mtx_destroy(&tb->common.fixlock);
+	erts_rwmtx_destroy(&tb->common.rwlock);
+	erts_mtx_destroy(&tb->common.fixlock);
 	ASSERT(is_immed(tb->common.heir_data));
 
         if (tb->common.btid)
@@ -403,8 +403,8 @@ static void schedule_free_dbtable(DbTable* tb)
      *  	     Caller is *not* allowed to access the specialized part
      *  	     (hash or tree) of *tb after this function has returned.
      */
-    ASSERT(erts_smp_refc_read(&tb->common.refc, 0) == 0);
-    ASSERT(erts_smp_refc_read(&tb->common.fix_count, 0) == 0);
+    ASSERT(erts_refc_read(&tb->common.refc, 0) == 0);
+    ASSERT(erts_refc_read(&tb->common.fix_count, 0) == 0);
     erts_schedule_thr_prgr_later_cleanup_op(free_dbtable,
 					    (void *) tb,
 					    &tb->release.data,
@@ -419,7 +419,7 @@ save_sched_table(Process *c_p, DbTable *tb)
 
     ASSERT(esdp);
     esdp->ets_tables.count++;
-    erts_smp_refc_inc(&tb->common.refc, 1);
+    erts_refc_inc(&tb->common.refc, 1);
 
     first = esdp->ets_tables.clist;
     if (!first) {
@@ -509,11 +509,11 @@ save_owned_table(Process *c_p, DbTable *tb)
 {
     DbTable *first;
 
-    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
 
     first = (DbTable*) erts_psd_get(c_p, ERTS_PSD_ETS_OWNED_TABLES);
 
-    erts_smp_refc_inc(&tb->common.refc, 1);
+    erts_refc_inc(&tb->common.refc, 1);
 
     if (!first) {
         tb->common.owned.next = tb->common.owned.prev = tb;
@@ -525,13 +525,13 @@ save_owned_table(Process *c_p, DbTable *tb)
         tb->common.owned.prev->common.owned.next = tb;
         first->common.owned.prev = tb;
     }
-    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
 }
 
 static ERTS_INLINE void
 delete_owned_table(Process *p, DbTable *tb)
 {
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
     if (tb->common.owned.next == tb) {
         DbTable* old;
         ASSERT(tb->common.owned.prev == tb);
@@ -554,21 +554,21 @@ delete_owned_table(Process *p, DbTable *tb)
         if (tb == first)
             erts_psd_set(p, ERTS_PSD_ETS_OWNED_TABLES, tb->common.owned.next);
     }
-    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 
     table_dec_refc(tb, 1);
 }
 
 static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock)
 {
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER;
     if (use_frequent_read_lock)
-	rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+	rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ;
     if (erts_ets_rwmtx_spin_count >= 0)
 	rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count;
-    erts_smp_rwmtx_init_opt(&tb->common.rwlock, &rwmtx_opt, "db_tab",
+    erts_rwmtx_init_opt(&tb->common.rwlock, &rwmtx_opt, "db_tab",
         tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
-    erts_smp_mtx_init(&tb->common.fixlock, "db_tab_fix",
+    erts_mtx_init(&tb->common.fixlock, "db_tab_fix",
         tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
     tb->common.is_thread_safe = !(tb->common.status & DB_FINE_LOCKED);
 }
@@ -577,10 +577,10 @@ static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind)
 {
     if (tb->common.type & DB_FINE_LOCKED) {
 	if (kind == LCK_WRITE) {	   
-	    erts_smp_rwmtx_rwlock(&tb->common.rwlock);
+	    erts_rwmtx_rwlock(&tb->common.rwlock);
 	    tb->common.is_thread_safe = 1;
 	} else {	
-	    erts_smp_rwmtx_rlock(&tb->common.rwlock);
+	    erts_rwmtx_rlock(&tb->common.rwlock);
 	    ASSERT(!tb->common.is_thread_safe);
 	}
     }
@@ -589,10 +589,10 @@ static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind)
 	switch (kind) {
 	case LCK_WRITE:
 	case LCK_WRITE_REC:
-	    erts_smp_rwmtx_rwlock(&tb->common.rwlock);
+	    erts_rwmtx_rwlock(&tb->common.rwlock);
 	    break;
 	default:
-	    erts_smp_rwmtx_rlock(&tb->common.rwlock);
+	    erts_rwmtx_rlock(&tb->common.rwlock);
 	}
 	ASSERT(tb->common.is_thread_safe);
     }
@@ -609,11 +609,11 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
 	if (kind == LCK_WRITE) {
 	    ASSERT(tb->common.is_thread_safe);
 	    tb->common.is_thread_safe = 0;
-	    erts_smp_rwmtx_rwunlock(&tb->common.rwlock);
+	    erts_rwmtx_rwunlock(&tb->common.rwlock);
 	}
 	else {
 	    ASSERT(!tb->common.is_thread_safe);
-	    erts_smp_rwmtx_runlock(&tb->common.rwlock);
+	    erts_rwmtx_runlock(&tb->common.rwlock);
 	}
     }
     else {
@@ -621,10 +621,10 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
 	switch (kind) {
 	case LCK_WRITE:
 	case LCK_WRITE_REC:
-	    erts_smp_rwmtx_rwunlock(&tb->common.rwlock);
+	    erts_rwmtx_rwunlock(&tb->common.rwlock);
 	    break;
 	default:
-	    erts_smp_rwmtx_runlock(&tb->common.rwlock);
+	    erts_rwmtx_runlock(&tb->common.rwlock);
 	}
     }
 }
@@ -637,7 +637,7 @@ DbTable* db_get_table_aux(Process *p,
 			  int meta_already_locked)
 {
     DbTable *tb;
-    erts_smp_rwmtx_t *mtl = NULL;
+    erts_rwmtx_t *mtl = NULL;
 
     /*
      * IMPORTANT: Only scheduler threads are allowed
@@ -649,9 +649,9 @@ DbTable* db_get_table_aux(Process *p,
     if (is_atom(id)) {
 	struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl);
 	if (!meta_already_locked)
-	    erts_smp_rwmtx_rlock(mtl);
+	    erts_rwmtx_rlock(mtl);
 	else{
-	    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(mtl)
+	    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(mtl)
 			       || erts_lc_rwmtx_is_rwlocked(mtl));
 	    mtl = NULL;
 	}
@@ -685,7 +685,7 @@ DbTable* db_get_table_aux(Process *p,
 	}
     }
     if (mtl)
-	erts_smp_rwmtx_runlock(mtl);
+	erts_rwmtx_runlock(mtl);
     return tb;
 }
 
@@ -701,12 +701,12 @@ DbTable* db_get_table(Process *p,
 static int insert_named_tab(Eterm name_atom, DbTable* tb, int have_lock)
 {
     int ret = 0;
-    erts_smp_rwmtx_t* rwlock;
+    erts_rwmtx_t* rwlock;
     struct meta_name_tab_entry* new_entry;
     struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom,
 							      &rwlock);
     if (!have_lock)
-	erts_smp_rwmtx_rwlock(rwlock);
+	erts_rwmtx_rwlock(rwlock);
 
     if (bucket->pu.tb == NULL) { /* empty */
 	new_entry = bucket;
@@ -754,25 +754,25 @@ static int insert_named_tab(Eterm name_atom, DbTable* tb, int have_lock)
 
 done:
     if (!have_lock)
-	erts_smp_rwmtx_rwunlock(rwlock);
+	erts_rwmtx_rwunlock(rwlock);
     return ret;
 }
 
 static int remove_named_tab(DbTable *tb, int have_lock)
 {
     int ret = 0;
-    erts_smp_rwmtx_t* rwlock;
+    erts_rwmtx_t* rwlock;
     Eterm name_atom = tb->common.the_name;
     struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom,
 							      &rwlock);
     ASSERT(is_table_named(tb));
-    if (!have_lock && erts_smp_rwmtx_tryrwlock(rwlock) == EBUSY) {
+    if (!have_lock && erts_rwmtx_tryrwlock(rwlock) == EBUSY) {
 	db_unlock(tb, LCK_WRITE);
-	erts_smp_rwmtx_rwlock(rwlock);
+	erts_rwmtx_rwlock(rwlock);
 	db_lock(tb, LCK_WRITE);
     }
 
-    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(rwlock));
+    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(rwlock));
 
     if (bucket->pu.tb == NULL) {
 	goto done;
@@ -825,7 +825,7 @@ static int remove_named_tab(DbTable *tb, int have_lock)
 
 done:
     if (!have_lock)
-	erts_smp_rwmtx_rwunlock(rwlock);
+	erts_rwmtx_rwunlock(rwlock);
     return ret;
 }
 
@@ -834,11 +834,11 @@ done:
 */
 static ERTS_INLINE void local_fix_table(DbTable* tb)
 {
-    erts_smp_refc_inc(&tb->common.fix_count, 1);
+    erts_refc_inc(&tb->common.fix_count, 1);
 }	    
 static ERTS_INLINE void local_unfix_table(DbTable* tb)
 {	
-    if (erts_smp_refc_dectest(&tb->common.fix_count, 0) == 0) {
+    if (erts_refc_dectest(&tb->common.fix_count, 0) == 0) {
 	ASSERT(IS_HASH_TABLE(tb->common.status));
 	db_unfix_table_hash(&(tb->hash));
     }
@@ -1479,7 +1479,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
     DbTable* tb;
     Eterm ret;
     Eterm old_name;
-    erts_smp_rwmtx_t *lck1, *lck2;
+    erts_rwmtx_t *lck1, *lck2;
 
 #ifdef HARDDEBUG
     erts_fprintf(stderr,
@@ -1502,7 +1502,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
 	if (lck1 == lck2)
 	    lck2 = NULL;
 	else if (lck1 > lck2) {
-	    erts_smp_rwmtx_t *tmp = lck1;
+	    erts_rwmtx_t *tmp = lck1;
 	    lck1 = lck2;
 	    lck2 = tmp;
 	}
@@ -1520,9 +1520,9 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
         }
     }
 
-    erts_smp_rwmtx_rwlock(lck1);
+    erts_rwmtx_rwlock(lck1);
     if (lck2)
-	erts_smp_rwmtx_rwlock(lck2);
+	erts_rwmtx_rwlock(lck2);
 
     tb = db_get_table_aux(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE, 1);
     if (!tb)
@@ -1542,16 +1542,16 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
     tb->common.the_name = BIF_ARG_2;
 
     db_unlock(tb, LCK_WRITE);
-    erts_smp_rwmtx_rwunlock(lck1);
+    erts_rwmtx_rwunlock(lck1);
     if (lck2)
-	erts_smp_rwmtx_rwunlock(lck2);
+	erts_rwmtx_rwunlock(lck2);
     BIF_RET(ret);
  badarg:
     if (tb)
 	db_unlock(tb, LCK_WRITE);
-    erts_smp_rwmtx_rwunlock(lck1);
+    erts_rwmtx_rwunlock(lck1);
     if (lck2)
-	erts_smp_rwmtx_rwunlock(lck2);
+	erts_rwmtx_rwunlock(lck2);
     BIF_ERROR(BIF_P, BADARG);    
 }
 
@@ -1691,11 +1691,11 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     {
         DbTable init_tb;
 
-	erts_smp_atomic_init_nob(&init_tb.common.memory_size, 0);
+	erts_atomic_init_nob(&init_tb.common.memory_size, 0);
 	tb = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE,
 				      &init_tb, sizeof(DbTable));
-	erts_smp_atomic_init_nob(&tb->common.memory_size,
-				 erts_smp_atomic_read_nob(&init_tb.common.memory_size));
+	erts_atomic_init_nob(&tb->common.memory_size,
+				 erts_atomic_read_nob(&init_tb.common.memory_size));
     }
 
     tb->common.meth = meth;
@@ -1703,13 +1703,13 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     tb->common.status = status;    
     tb->common.type = status & ERTS_ETS_TABLE_TYPES;
     /* Note, 'type' is *read only* from now on... */
-    erts_smp_refc_init(&tb->common.fix_count, 0);
+    erts_refc_init(&tb->common.fix_count, 0);
     db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ));
     tb->common.keypos = keypos;
     tb->common.owner = BIF_P->common.id;
     set_heir(BIF_P, tb, heir, heir_data);
 
-    erts_smp_atomic_init_nob(&tb->common.nitems, 0);
+    erts_atomic_init_nob(&tb->common.nitems, 0);
 
     tb->common.fixing_procs = NULL;
     tb->common.compress = is_compressed;
@@ -1892,7 +1892,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
          * Process 'rp' might be exiting, but our table lock prevents it
          * from terminating as it cannot complete erts_db_process_exiting().
          */
-        ASSERT(!(ERTS_PSFLG_FREE & erts_smp_atomic32_read_nob(&rp->state)));
+        ASSERT(!(ERTS_PSFLG_FREE & erts_atomic32_read_nob(&rp->state)));
 
         delete_owned_table(rp, tb);
         BIF_P->flags |= F_USING_DB;
@@ -1967,12 +1967,12 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
     db_unlock(tb,LCK_WRITE);
     send_ets_transfer_message(BIF_P, to_proc, &to_locks,
                               tb, BIF_ARG_3);
-    erts_smp_proc_unlock(to_proc, to_locks);
+    erts_proc_unlock(to_proc, to_locks);
     UnUseTmpHeap(5,BIF_P);
     BIF_RET(am_true);
 
 badarg:
-    if (to_proc != NULL && to_proc != BIF_P) erts_smp_proc_unlock(to_proc, to_locks);
+    if (to_proc != NULL && to_proc != BIF_P) erts_proc_unlock(to_proc, to_locks);
     if (tb != NULL) db_unlock(tb, LCK_WRITE);
     BIF_ERROR(BIF_P, BADARG);
 }
@@ -2196,7 +2196,7 @@ BIF_RETTYPE ets_select_delete_2(BIF_ALIST_2)
 	if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) {
 	    BIF_ERROR(BIF_P, BADARG);
 	}
-	nitems = erts_smp_atomic_read_nob(&tb->common.nitems);
+	nitems = erts_atomic_read_nob(&tb->common.nitems);
 	tb->common.meth->db_delete_all_objects(BIF_P, tb);
 	db_unlock(tb, LCK_WRITE);
 	BIF_RET(erts_make_integer(nitems,BIF_P));
@@ -2247,7 +2247,7 @@ BIF_RETTYPE ets_select_delete_2(BIF_ALIST_2)
  */
 
 struct ErtsEtsAllReq_ {
-    erts_smp_atomic32_t refc;
+    erts_atomic32_t refc;
     Process *proc;
     ErtsOIRefStorage ref;
     ErtsEtsAllReqList list[1]; /* one per scheduler */
@@ -2380,7 +2380,7 @@ ets_all_reply(ErtsSchedulerData *esdp, ErtsEtsAllReq **reqpp,
 
     erts_proc_dec_refc(reqp->proc);
 
-    if (erts_smp_atomic32_dec_read_nob(&reqp->refc) == 0)
+    if (erts_atomic32_dec_read_nob(&reqp->refc) == 0)
         erts_free(ERTS_ALC_T_ETS_ALL_REQ, reqp);
 
     *reqpp = NULL;
@@ -2468,7 +2468,7 @@ BIF_RETTYPE ets_internal_request_all_0(BIF_ALIST_0)
     Eterm ref = erts_make_ref(BIF_P);
     ErtsEtsAllReq *req = erts_alloc(ERTS_ALC_T_ETS_ALL_REQ,
                                     ERTS_ETS_ALL_REQ_SIZE);
-    erts_smp_atomic32_init_nob(&req->refc,
+    erts_atomic32_init_nob(&req->refc,
 			       (erts_aint32_t) erts_no_schedulers);
     erts_oiref_storage_save(&req->ref, ref);
     req->proc = BIF_P;
@@ -3162,7 +3162,7 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1)
 	if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL
 	    || tb->common.owner != owner) {
 	    if (BIF_P != rp)
-		erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
+		erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
 	    if (is_atom(BIF_ARG_1) || is_small(BIF_ARG_1)) {
 		BIF_RET(am_undefined);
 	    }
@@ -3176,7 +3176,7 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1)
     db_unlock(tb, LCK_READ);
 
     /*if (rp != NULL && rp != BIF_P)
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);*/
+	erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);*/
 
     hp = HAlloc(BIF_P, 5*sizeof(fields)/sizeof(Eterm));
     res = NIL;
@@ -3296,9 +3296,9 @@ void init_db(ErtsDbSpinCount db_spin_count)
     size_t size;
 
     int max_spin_count = (1 << 15) - 1; /* internal limit */
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
-    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
-    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+    erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_RWMTX_LONG_LIVED;
 
     switch (db_spin_count) {
     case ERTS_DB_SPNCNT_NONE:
@@ -3338,12 +3338,12 @@ void init_db(ErtsDbSpinCount db_spin_count)
 	rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count;
 
     for (i=0; i<META_NAME_TAB_LOCK_CNT; i++) {
-        erts_smp_rwmtx_init_opt(&meta_name_tab_rwlocks[i].lck, &rwmtx_opt,
+        erts_rwmtx_init_opt(&meta_name_tab_rwlocks[i].lck, &rwmtx_opt,
             "meta_name_tab", make_small(i),
             ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DB);
     }
 
-    erts_smp_atomic_init_nob(&erts_ets_misc_mem_size, 0);
+    erts_atomic_init_nob(&erts_ets_misc_mem_size, 0);
     db_initialize_util();
 
     if (user_requested_db_max_tabs < DB_DEF_MAX_TABS)
@@ -3444,14 +3444,14 @@ retry:
     
 	if (tb->common.owner != p->common.id) {
 	    if (to_proc != NULL ) {
-		erts_smp_proc_unlock(to_proc, to_locks);
+		erts_proc_unlock(to_proc, to_locks);
 	    }
 	    db_unlock(tb,LCK_WRITE);
 	    return !0; /* ok, someone already gave my table away */
 	}
 	if (tb->common.heir != to_pid) {  /* someone changed the heir */ 
 	    if (to_proc != NULL ) {
-		erts_smp_proc_unlock(to_proc, to_locks);
+		erts_proc_unlock(to_proc, to_locks);
 	    }
 	    if (to_pid == p->common.id || to_pid == am_none) {
 		return 0; /* no real heir, table still mine */
@@ -3464,7 +3464,7 @@ retry:
     }
     if (to_proc->common.u.alive.started_interval
 	!= tb->common.heir_started_interval) {
-	erts_smp_proc_unlock(to_proc, to_locks);
+	erts_proc_unlock(to_proc, to_locks);
 	return 0; /* heir dead and pid reused, table still mine */
     }
     
@@ -3481,7 +3481,7 @@ retry:
 	heir_data = tpv[1];
     }
     send_ets_transfer_message(p, to_proc, &to_locks, tb, heir_data);
-    erts_smp_proc_unlock(to_proc, to_locks);
+    erts_proc_unlock(to_proc, to_locks);
     return !0;
 }
 
@@ -3532,17 +3532,17 @@ static SWord proc_cleanup_fixed_table(Process* p, DbFixation* fix)
 	db_lock(tb, LCK_WRITE_REC);
 	if (!(tb->common.status & DB_DELETE)) {
 	    erts_aint_t diff;
-	    erts_smp_mtx_lock(&tb->common.fixlock);
+	    erts_mtx_lock(&tb->common.fixlock);
 
 	    ASSERT(fixing_procs_rbt_lookup(tb->common.fixing_procs, p));
 
 	    diff = -((erts_aint_t) fix->counter);
-	    erts_smp_refc_add(&tb->common.fix_count,diff,0);
+	    erts_refc_add(&tb->common.fix_count,diff,0);
 	    fix->counter = 0;
 
 	    fixing_procs_rbt_delete(&tb->common.fixing_procs, fix);
 
-	    erts_smp_mtx_unlock(&tb->common.fixlock);
+	    erts_mtx_unlock(&tb->common.fixlock);
 	    if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)) {
 		work += db_unfix_table_hash(&(tb->hash));
 	    }
@@ -3605,9 +3605,9 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 	switch (state->op) {
         case GET_OWNED_TABLE: {
             DbTable* tb;
-            erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
+            erts_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
             tb = (DbTable*) erts_psd_get(c_p, ERTS_PSD_ETS_OWNED_TABLES);
-            erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
+            erts_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
 
             if (!tb) {
                 /* Done with owned tables; now fixations */
@@ -3698,8 +3698,8 @@ static void fix_table_locked(Process* p, DbTable* tb)
 {
     DbFixation *fix;
 
-    erts_smp_mtx_lock(&tb->common.fixlock);
-    erts_smp_refc_inc(&tb->common.fix_count,1);
+    erts_mtx_lock(&tb->common.fixlock);
+    erts_refc_inc(&tb->common.fix_count,1);
     fix = tb->common.fixing_procs;
     if (fix == NULL) {
 	tb->common.time.monotonic
@@ -3712,7 +3712,7 @@ static void fix_table_locked(Process* p, DbTable* tb)
 	    ASSERT(fixed_tabs_find(NULL, fix));
 	    ++(fix->counter);
 
-	    erts_smp_mtx_unlock(&tb->common.fixlock);
+	    erts_mtx_unlock(&tb->common.fixlock);
 	    return;
 	}
     }
@@ -3725,7 +3725,7 @@ static void fix_table_locked(Process* p, DbTable* tb)
     fix->counter = 1;
     fixing_procs_rbt_insert(&tb->common.fixing_procs, fix);
 
-    erts_smp_mtx_unlock(&tb->common.fixlock);
+    erts_mtx_unlock(&tb->common.fixlock);
     p->flags |= F_USING_DB;
 
     fixed_tabs_insert(p, fix);
@@ -3738,16 +3738,16 @@ static void unfix_table_locked(Process* p,  DbTable* tb,
 {
     DbFixation* fix;
 
-    erts_smp_mtx_lock(&tb->common.fixlock);
+    erts_mtx_lock(&tb->common.fixlock);
     fix = fixing_procs_rbt_lookup(tb->common.fixing_procs, p);
 
     if (fix) {
-	erts_smp_refc_dec(&tb->common.fix_count,0);
+	erts_refc_dec(&tb->common.fix_count,0);
 	--(fix->counter);
 	ASSERT(fix->counter >= 0);
 	if (fix->counter == 0) {
 	    fixing_procs_rbt_delete(&tb->common.fixing_procs, fix);
-	    erts_smp_mtx_unlock(&tb->common.fixlock);
+	    erts_mtx_unlock(&tb->common.fixlock);
 	    fixed_tabs_delete(p, fix);
 
 	    erts_refc_dec(&fix->tabs.btid->intern.refc, 1);
@@ -3758,15 +3758,15 @@ static void unfix_table_locked(Process* p,  DbTable* tb,
 	    goto unlocked;
 	}
     }
-    erts_smp_mtx_unlock(&tb->common.fixlock);
+    erts_mtx_unlock(&tb->common.fixlock);
 unlocked:
 
     if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)
-	&& erts_smp_atomic_read_nob(&tb->hash.fixdel) != (erts_aint_t)NULL) {
+	&& erts_atomic_read_nob(&tb->hash.fixdel) != (erts_aint_t)NULL) {
 	if (*kind_p == LCK_READ && tb->common.is_thread_safe) {
 	    /* Must have write lock while purging pseudo-deleted (OTP-8166) */
-	    erts_smp_rwmtx_runlock(&tb->common.rwlock);
-	    erts_smp_rwmtx_rwlock(&tb->common.rwlock);
+	    erts_rwmtx_runlock(&tb->common.rwlock);
+	    erts_rwmtx_rwlock(&tb->common.rwlock);
 	    *kind_p = LCK_WRITE;
 	    if (tb->common.status & DB_DELETE) return;
 	}
@@ -3794,7 +3794,7 @@ static void free_fixations_op(DbFixation* fix, void* vctx)
     ASSERT(ctx->tb->common.status & DB_DELETE);
 
     diff = -((erts_aint_t) fix->counter);
-    erts_smp_refc_add(&ctx->tb->common.fix_count, diff, 0);
+    erts_refc_add(&ctx->tb->common.fix_count, diff, 0);
 
     if (fix->procs.p != ctx->p) { /* Fixated by other process */
         fix->counter = 0;
@@ -3839,7 +3839,7 @@ static SWord free_fixations_locked(Process* p, DbTable *tb)
 {
     struct free_fixations_ctx ctx;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rwlocked(&tb->common.rwlock));
+    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock));
 
     ctx.p = p;
     ctx.tb = tb;
@@ -3982,7 +3982,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
     int use_monotonic;
 
     if (What == am_size) {
-	ret = make_small(erts_smp_atomic_read_nob(&tb->common.nitems));
+	ret = make_small(erts_atomic_read_nob(&tb->common.nitems));
     } else if (What == am_type) {
 	if (tb->common.status & DB_SET)  {
 	    ret = am_set;
@@ -3995,7 +3995,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	    ret = am_bag;
 	}
     } else if (What == am_memory) {
-	Uint words = (Uint) ((erts_smp_atomic_read_nob(&tb->common.memory_size)
+	Uint words = (Uint) ((erts_atomic_read_nob(&tb->common.memory_size)
 			      + sizeof(Uint)
 			      - 1)
 			     / sizeof(Uint));
@@ -4041,7 +4041,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 		= ERTS_IS_ATOM_STR("safe_fixed_monotonic_time",
 				   What))
 	       || ERTS_IS_ATOM_STR("safe_fixed", What)) {
-	erts_smp_mtx_lock(&tb->common.fixlock);
+	erts_mtx_lock(&tb->common.fixlock);
 	if (IS_FIXED(tb)) {
 	    Uint need;
 	    Eterm *hp;
@@ -4083,7 +4083,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	} else {
 	    ret = am_false;
 	}
-	erts_smp_mtx_unlock(&tb->common.fixlock);
+	erts_mtx_unlock(&tb->common.fixlock);
     } else if (What == am_atom_put("stats",5)) {
 	if (IS_HASH_TABLE(tb->common.status)) {
 	    FloatDef f;
@@ -4107,7 +4107,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	    std_dev_exp = make_float(hp);
 	    PUT_DOUBLE(f, hp);
 	    hp += FLOAT_SIZE_OBJECT;
-	    ret = TUPLE7(hp, make_small(erts_smp_atomic_read_nob(&tb->hash.nactive)),
+	    ret = TUPLE7(hp, make_small(erts_atomic_read_nob(&tb->hash.nactive)),
 			 avg, std_dev_real, std_dev_exp,
 			 make_small(stats.min_chain_len),
 			 make_small(stats.max_chain_len),
@@ -4139,9 +4139,9 @@ static void print_table(fmtfn_t to, void *to_arg, int show,  DbTable* tb)
 
     tb->common.meth->db_print(to, to_arg, show, tb);
 
-    erts_print(to, to_arg, "Objects: %d\n", (int)erts_smp_atomic_read_nob(&tb->common.nitems));
+    erts_print(to, to_arg, "Objects: %d\n", (int)erts_atomic_read_nob(&tb->common.nitems));
     erts_print(to, to_arg, "Words: %bpu\n",
-	       (Uint) ((erts_smp_atomic_read_nob(&tb->common.memory_size)
+	       (Uint) ((erts_atomic_read_nob(&tb->common.memory_size)
 			+ sizeof(Uint)
 			- 1)
 		       / sizeof(Uint)));
@@ -4181,9 +4181,9 @@ void db_info(fmtfn_t to, void *to_arg, int show)    /* Called by break handler *
 Uint
 erts_get_ets_misc_mem_size(void)
 {
-    ERTS_SMP_MEMORY_BARRIER;
+    ERTS_THR_MEMORY_BARRIER;
     /* Memory not allocated in ets_alloc */
-    return (Uint) erts_smp_atomic_read_nob(&erts_ets_misc_mem_size);
+    return (Uint) erts_atomic_read_nob(&erts_ets_misc_mem_size);
 }
 
 /* SMP Note: May only be used when system is locked */
@@ -4192,7 +4192,7 @@ erts_db_foreach_table(void (*func)(DbTable *, void *), void *arg)
 {
     int ix;
 
-    ASSERT(erts_smp_thr_progress_is_blocking());
+    ASSERT(erts_thr_progress_is_blocking());
 
     for (ix = 0; ix < erts_no_schedulers; ix++) {
         ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix);
@@ -4298,4 +4298,4 @@ void erts_lcnt_update_db_locks(int enable) {
         &lcnt_update_db_locks_per_sched, (void*)(UWord)enable);
 }
 
-#endif /* ERTS_ENABLE_LOCK_COUNT */
\ No newline at end of file
+#endif /* ERTS_ENABLE_LOCK_COUNT */
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index d83126b3a2..318e90cb28 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -124,7 +124,7 @@ extern Export ets_select_delete_continue_exp;
 extern Export ets_select_count_continue_exp;
 extern Export ets_select_replace_continue_exp;
 extern Export ets_select_continue_exp;
-extern erts_smp_atomic_t erts_ets_misc_mem_size;
+extern erts_atomic_t erts_ets_misc_mem_size;
 
 Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt);
 Uint erts_db_get_max_tabs(void);
@@ -151,11 +151,11 @@ do {									\
     erts_aint_t sz__ = (((erts_aint_t) (ALLOC_SZ))			\
 			- ((erts_aint_t) (FREE_SZ)));			\
     ASSERT((TAB));							\
-    erts_smp_atomic_add_nob(&(TAB)->common.memory_size, sz__);		\
+    erts_atomic_add_nob(&(TAB)->common.memory_size, sz__);		\
 } while (0)
 
 #define ERTS_ETS_MISC_MEM_ADD(SZ) \
-  erts_smp_atomic_add_nob(&erts_ets_misc_mem_size, (SZ));
+  erts_atomic_add_nob(&erts_ets_misc_mem_size, (SZ));
 
 ERTS_GLB_INLINE void *erts_db_alloc(ErtsAlcType_t type,
 				    DbTable *tab,
@@ -292,7 +292,7 @@ erts_db_free(ErtsAlcType_t type, DbTable *tab, void *ptr, Uint size)
     ERTS_DB_ALC_MEM_UPDATE_(tab, size, 0);
 
     ASSERT(((void *) tab) != ptr
-	   || erts_smp_atomic_read_nob(&tab->common.memory_size) == 0);
+	   || erts_atomic_read_nob(&tab->common.memory_size) == 0);
 
     erts_free(type, ptr);
 }
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 2945afe275..25072ede97 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -112,15 +112,15 @@
 #  define DB_USING_FINE_LOCKING(TB) (((TB))->common.type & DB_FINE_LOCKED)
 
 #ifdef ETHR_ORDERED_READ_DEPEND
-#define SEGTAB(tb) ((struct segment**) erts_smp_atomic_read_nob(&(tb)->segtab))
+#define SEGTAB(tb) ((struct segment**) erts_atomic_read_nob(&(tb)->segtab))
 #else
 #define SEGTAB(tb)							\
     (DB_USING_FINE_LOCKING(tb)						\
-     ? ((struct segment**) erts_smp_atomic_read_ddrb(&(tb)->segtab))	\
-     : ((struct segment**) erts_smp_atomic_read_nob(&(tb)->segtab)))
+     ? ((struct segment**) erts_atomic_read_ddrb(&(tb)->segtab))	\
+     : ((struct segment**) erts_atomic_read_nob(&(tb)->segtab)))
 #endif
-#define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive))
-#define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems))
+#define NACTIVE(tb) ((int)erts_atomic_read_nob(&(tb)->nactive))
+#define NITEMS(tb) ((int)erts_atomic_read_nob(&(tb)->common.nitems))
 
 #define SLOT_IX_TO_SEG_IX(i) (((i)+(EXT_SEGSZ-FIRST_SEGSZ)) >> EXT_SEGSZ_EXP)
 
@@ -138,12 +138,12 @@
 static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval)
 {
     Uint mask = (DB_USING_FINE_LOCKING(tb)
-		 ? erts_smp_atomic_read_acqb(&tb->szm)
-		 : erts_smp_atomic_read_nob(&tb->szm));
+		 ? erts_atomic_read_acqb(&tb->szm)
+		 : erts_atomic_read_nob(&tb->szm));
     Uint ix = hval & mask; 
-    if (ix >= erts_smp_atomic_read_nob(&tb->nactive)) {
+    if (ix >= erts_atomic_read_nob(&tb->nactive)) {
 	ix &= mask>>1;
-	ASSERT(ix < erts_smp_atomic_read_nob(&tb->nactive));
+	ASSERT(ix < erts_atomic_read_nob(&tb->nactive));
     }
     return ix;
 }
@@ -162,7 +162,7 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix,
 							 sizeof(FixedDeletion));
     ERTS_ETS_MISC_MEM_ADD(sizeof(FixedDeletion));
     fixd->slot = ix;
-    was_next = erts_smp_atomic_read_acqb(&tb->fixdel);
+    was_next = erts_atomic_read_acqb(&tb->fixdel);
     do { /* Lockless atomic insertion in linked list: */
         if (NFIXED(tb) <= fixated_by_me) {
             erts_db_free(ERTS_ALC_T_DB_FIX_DEL, (DbTable*)tb,
@@ -171,7 +171,7 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix,
         }
         exp_next = was_next;
 	fixd->next = (FixedDeletion*) exp_next;
-	was_next = erts_smp_atomic_cmpxchg_mb(&tb->fixdel,
+	was_next = erts_atomic_cmpxchg_mb(&tb->fixdel,
                                               (erts_aint_t) fixd,
                                               exp_next);
     }while (was_next != exp_next);
@@ -192,50 +192,50 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix,
 #  define GET_LOCK_MAYBE(tb,hval) ((tb)->common.is_thread_safe ? NULL : GET_LOCK(tb,hval))
 
 /* Fine grained read lock */
-static ERTS_INLINE erts_smp_rwmtx_t* RLOCK_HASH(DbTableHash* tb, HashValue hval)
+static ERTS_INLINE erts_rwmtx_t* RLOCK_HASH(DbTableHash* tb, HashValue hval)
 {
     if (tb->common.is_thread_safe) {
 	return NULL;
     } else {
-	erts_smp_rwmtx_t* lck = GET_LOCK(tb,hval);
+	erts_rwmtx_t* lck = GET_LOCK(tb,hval);
 	ASSERT(tb->common.type & DB_FINE_LOCKED);
-	erts_smp_rwmtx_rlock(lck);
+	erts_rwmtx_rlock(lck);
 	return lck;
     }
 }
 /* Fine grained write lock */
-static ERTS_INLINE erts_smp_rwmtx_t* WLOCK_HASH(DbTableHash* tb, HashValue hval)
+static ERTS_INLINE erts_rwmtx_t* WLOCK_HASH(DbTableHash* tb, HashValue hval)
 {
     if (tb->common.is_thread_safe) {
 	return NULL;
     } else {
-	erts_smp_rwmtx_t* lck = GET_LOCK(tb,hval);
+	erts_rwmtx_t* lck = GET_LOCK(tb,hval);
 	ASSERT(tb->common.type & DB_FINE_LOCKED);
-	erts_smp_rwmtx_rwlock(lck);
+	erts_rwmtx_rwlock(lck);
 	return lck;
     }
 }
 
-static ERTS_INLINE void RUNLOCK_HASH(erts_smp_rwmtx_t* lck)
+static ERTS_INLINE void RUNLOCK_HASH(erts_rwmtx_t* lck)
 {
     if (lck != NULL) {
-	erts_smp_rwmtx_runlock(lck);
+	erts_rwmtx_runlock(lck);
     }
 }
 
-static ERTS_INLINE void WUNLOCK_HASH(erts_smp_rwmtx_t* lck)
+static ERTS_INLINE void WUNLOCK_HASH(erts_rwmtx_t* lck)
 {
     if (lck != NULL) {
-	erts_smp_rwmtx_rwunlock(lck);
+	erts_rwmtx_rwunlock(lck);
     }
 }
 
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
 #  define IFN_EXCL(tb,cmd) (((tb)->common.is_thread_safe) || (cmd))
-#  define IS_HASH_RLOCKED(tb,hval) IFN_EXCL(tb,erts_smp_lc_rwmtx_is_rlocked(GET_LOCK(tb,hval)))
-#  define IS_HASH_WLOCKED(tb,lck) IFN_EXCL(tb,erts_smp_lc_rwmtx_is_rwlocked(lck))
-#  define IS_TAB_WLOCKED(tb) erts_smp_lc_rwmtx_is_rwlocked(&(tb)->common.rwlock)
+#  define IS_HASH_RLOCKED(tb,hval) IFN_EXCL(tb,erts_lc_rwmtx_is_rlocked(GET_LOCK(tb,hval)))
+#  define IS_HASH_WLOCKED(tb,lck) IFN_EXCL(tb,erts_lc_rwmtx_is_rwlocked(lck))
+#  define IS_TAB_WLOCKED(tb) erts_lc_rwmtx_is_rwlocked(&(tb)->common.rwlock)
 #else
 #  define IS_HASH_RLOCKED(tb,hval) (1)
 #  define IS_HASH_WLOCKED(tb,hval) (1)
@@ -248,7 +248,7 @@ static ERTS_INLINE void WUNLOCK_HASH(erts_smp_rwmtx_t* lck)
 ** Slot READ locks updated accordingly, unlocked if EOT.
 */
 static ERTS_INLINE Sint next_slot(DbTableHash* tb, Uint ix,
-				  erts_smp_rwmtx_t** lck_ptr)
+				  erts_rwmtx_t** lck_ptr)
 {
     ix += DB_HASH_LOCK_CNT;
     if (ix < NACTIVE(tb)) return ix;
@@ -259,7 +259,7 @@ static ERTS_INLINE Sint next_slot(DbTableHash* tb, Uint ix,
 }
 /* Same as next_slot but with WRITE locking */
 static ERTS_INLINE Sint next_slot_w(DbTableHash* tb, Uint ix,
-				    erts_smp_rwmtx_t** lck_ptr)
+				    erts_rwmtx_t** lck_ptr)
 {
     ix += DB_HASH_LOCK_CNT;
     if (ix < NACTIVE(tb)) return ix;
@@ -329,9 +329,9 @@ static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb,
 				   struct segment** segtab)
 {
     if (DB_USING_FINE_LOCKING(tb))
-	erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t) segtab);
+	erts_atomic_set_wb(&tb->segtab, (erts_aint_t) segtab);
     else
-	erts_smp_atomic_set_nob(&tb->segtab, (erts_aint_t) segtab);
+	erts_atomic_set_nob(&tb->segtab, (erts_aint_t) segtab);
 }
 
 /* Used by select_replace on analyze_pattern */
@@ -343,7 +343,7 @@ typedef int (*extra_match_validator_t)(int keypos, Eterm match, Eterm guard, Ete
 static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix);
 static void alloc_seg(DbTableHash *tb);
 static int free_seg(DbTableHash *tb, int free_records);
-static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_smp_rwmtx_t** lck_ptr,
+static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr,
 			HashDbTerm *list);
 static HashDbTerm* search_list(DbTableHash* tb, Eterm key, 
 			       HashValue hval, HashDbTerm *list);
@@ -541,7 +541,7 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel)
 {
     /*int tries = 0;*/
     DEBUG_WAIT();
-    if (erts_smp_atomic_cmpxchg_relb(&tb->fixdel,
+    if (erts_atomic_cmpxchg_relb(&tb->fixdel,
 				     (erts_aint_t) fixdel,
 				     (erts_aint_t) NULL) != (erts_aint_t) NULL) {
 	/* Oboy, must join lists */    
@@ -550,13 +550,13 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel)
 	erts_aint_t exp_tail;
 
 	while (last->next != NULL) last = last->next;
-	was_tail = erts_smp_atomic_read_acqb(&tb->fixdel);
+	was_tail = erts_atomic_read_acqb(&tb->fixdel);
 	do { /* Lockless atomic list insertion */
 	    exp_tail = was_tail;
 	    last->next = (FixedDeletion*) exp_tail;
 	    /*++tries;*/
 	    DEBUG_WAIT();
-	    was_tail = erts_smp_atomic_cmpxchg_relb(&tb->fixdel,
+	    was_tail = erts_atomic_cmpxchg_relb(&tb->fixdel,
 						    (erts_aint_t) fixdel,
 						    exp_tail);
 	}while (was_tail != exp_tail);
@@ -572,18 +572,18 @@ SWord db_unfix_table_hash(DbTableHash *tb)
     FixedDeletion* fixdel;
     SWord work = 0;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rwlocked(&tb->common.rwlock)
-		       || (erts_smp_lc_rwmtx_is_rlocked(&tb->common.rwlock)
+    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock)
+		       || (erts_lc_rwmtx_is_rlocked(&tb->common.rwlock)
 			   && !tb->common.is_thread_safe));
 restart:
-    fixdel = (FixedDeletion*) erts_smp_atomic_xchg_mb(&tb->fixdel,
+    fixdel = (FixedDeletion*) erts_atomic_xchg_mb(&tb->fixdel,
                                                       (erts_aint_t) NULL);
     while (fixdel != NULL) {
 	FixedDeletion *fx = fixdel;
 	int ix = fx->slot;
 	HashDbTerm **bp;
 	HashDbTerm *b;
-	erts_smp_rwmtx_t* lck = WLOCK_HASH(tb,ix);
+	erts_rwmtx_t* lck = WLOCK_HASH(tb,ix);
 
 	if (IS_FIXED(tb)) { /* interrupted by fixer */
 	    WUNLOCK_HASH(lck);
@@ -629,10 +629,10 @@ int db_create_hash(Process *p, DbTable *tbl)
 {
     DbTableHash *tb = &tbl->hash;
 
-    erts_smp_atomic_init_nob(&tb->szm, FIRST_SEGSZ_MASK);
-    erts_smp_atomic_init_nob(&tb->nactive, FIRST_SEGSZ);
-    erts_smp_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL);
-    erts_smp_atomic_init_nob(&tb->segtab, (erts_aint_t)NULL);
+    erts_atomic_init_nob(&tb->szm, FIRST_SEGSZ_MASK);
+    erts_atomic_init_nob(&tb->nactive, FIRST_SEGSZ);
+    erts_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL);
+    erts_atomic_init_nob(&tb->segtab, (erts_aint_t)NULL);
     SET_SEGTAB(tb, tb->first_segtab);
     tb->nsegs = NSEG_1;
     tb->nslots = FIRST_SEGSZ;
@@ -641,25 +641,25 @@ int db_create_hash(Process *p, DbTable *tbl)
                                                           SIZEOF_SEGMENT(FIRST_SEGSZ));
     sys_memset(tb->first_segtab[0], 0, SIZEOF_SEGMENT(FIRST_SEGSZ));
 
-    erts_smp_atomic_init_nob(&tb->is_resizing, 0);
+    erts_atomic_init_nob(&tb->is_resizing, 0);
     if (tb->common.type & DB_FINE_LOCKED) {
-	erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+	erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER;
 	int i;
 	if (tb->common.type & DB_FREQ_READ)
-	    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+	    rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ;
 	if (erts_ets_rwmtx_spin_count >= 0)
 	    rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count;
 	tb->locks = (DbTableHashFineLocks*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, /* Other type maybe? */ 
 							      (DbTable *) tb,
 							      sizeof(DbTableHashFineLocks));	    	    
 	for (i=0; i<DB_HASH_LOCK_CNT; ++i) {
-            erts_smp_rwmtx_init_opt(&tb->locks->lck_vec[i].lck, &rwmtx_opt,
+            erts_rwmtx_init_opt(&tb->locks->lck_vec[i].lck, &rwmtx_opt,
                 "db_hash_slot", tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
 	}
 	/* This important property is needed to guarantee the two buckets
     	 * involved in a grow/shrink operation it protected by the same lock:
 	 */
-	ASSERT(erts_smp_atomic_read_nob(&tb->nactive) % DB_HASH_LOCK_CNT == 0);
+	ASSERT(erts_atomic_read_nob(&tb->nactive) % DB_HASH_LOCK_CNT == 0);
     }
     else { /* coarse locking */
 	tb->locks = NULL;
@@ -672,7 +672,7 @@ static int db_first_hash(Process *p, DbTable *tbl, Eterm *ret)
 {
     DbTableHash *tb = &tbl->hash;
     Uint ix = 0;
-    erts_smp_rwmtx_t* lck = RLOCK_HASH(tb,ix);
+    erts_rwmtx_t* lck = RLOCK_HASH(tb,ix);
     HashDbTerm* list;
 
     for (;;) {
@@ -705,7 +705,7 @@ static int db_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
     HashValue hval;
     Uint ix;
     HashDbTerm* b;
-    erts_smp_rwmtx_t* lck;
+    erts_rwmtx_t* lck;
 
     hval = MAKE_HASH(key);
     lck = RLOCK_HASH(tb,hval);
@@ -752,7 +752,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail)
     HashDbTerm** bp;
     HashDbTerm* b;
     HashDbTerm* q;
-    erts_smp_rwmtx_t* lck;
+    erts_rwmtx_t* lck;
     int nitems;
     int ret = DB_ERROR_NONE;
 
@@ -778,7 +778,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail)
     if (tb->common.status & DB_SET) {
 	HashDbTerm* bnext = b->next;
 	if (b->hvalue == INVALID_HASH) {
-	    erts_smp_atomic_inc_nob(&tb->common.nitems);
+	    erts_atomic_inc_nob(&tb->common.nitems);
 	}
 	else if (key_clash_fail) {
 	    ret = DB_ERROR_BADKEY;
@@ -806,7 +806,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail)
 	do {
 	    if (db_eq(&tb->common,obj,&q->dbterm)) {
 		if (q->hvalue == INVALID_HASH) {
-		    erts_smp_atomic_inc_nob(&tb->common.nitems);
+		    erts_atomic_inc_nob(&tb->common.nitems);
 		    q->hvalue = hval;
 		    if (q != b) { /* must move to preserve key insertion order */
 			*qp = q->next;
@@ -827,7 +827,7 @@ Lnew:
     q->hvalue = hval;
     q->next = b;
     *bp = q;
-    nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems);
+    nitems = erts_atomic_inc_read_nob(&tb->common.nitems);
     WUNLOCK_HASH(lck);
     {
 	int nactive = NACTIVE(tb);       
@@ -871,7 +871,7 @@ int db_get_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
     HashValue hval;
     int ix;
     HashDbTerm* b;
-    erts_smp_rwmtx_t* lck;
+    erts_rwmtx_t* lck;
 
     hval = MAKE_HASH(key);
     lck = RLOCK_HASH(tb,hval);
@@ -897,7 +897,7 @@ static int db_member_hash(DbTable *tbl, Eterm key, Eterm *ret)
     HashValue hval;
     int ix;
     HashDbTerm* b1;
-    erts_smp_rwmtx_t* lck;
+    erts_rwmtx_t* lck;
 
     hval = MAKE_HASH(key);
     ix = hash_to_ix(tb, hval);
@@ -926,7 +926,7 @@ static int db_get_element_hash(Process *p, DbTable *tbl,
     HashValue hval;
     int ix;
     HashDbTerm* b1;
-    erts_smp_rwmtx_t* lck;
+    erts_rwmtx_t* lck;
     int retval;
     
     hval = MAKE_HASH(key);
@@ -991,7 +991,7 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret)
     int ix;
     HashDbTerm** bp;
     HashDbTerm* b;
-    erts_smp_rwmtx_t* lck;
+    erts_rwmtx_t* lck;
     int nitems_diff = 0;
 
     hval = MAKE_HASH(key);
@@ -1023,7 +1023,7 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret)
     }
     WUNLOCK_HASH(lck);
     if (nitems_diff) {
-	erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff);
+	erts_atomic_add_nob(&tb->common.nitems, nitems_diff);
 	try_shrink(tb);
     }
     *ret = am_true;
@@ -1040,7 +1040,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret)
     int ix;
     HashDbTerm** bp;
     HashDbTerm* b;
-    erts_smp_rwmtx_t* lck;
+    erts_rwmtx_t* lck;
     int nitems_diff = 0;
     int nkeys = 0;
     Eterm key;
@@ -1081,7 +1081,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret)
     }
     WUNLOCK_HASH(lck);
     if (nitems_diff) {
-	erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff);
+	erts_atomic_add_nob(&tb->common.nitems, nitems_diff);
 	try_shrink(tb);
     }
     *ret = am_true;
@@ -1092,7 +1092,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret)
 static int db_slot_hash(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret)
 {
     DbTableHash *tb = &tbl->hash;
-    erts_smp_rwmtx_t* lck;
+    erts_rwmtx_t* lck;
     Sint slot;
     int retval;
     int nactive;
@@ -1207,13 +1207,13 @@ static int match_traverse(Process* p, DbTableHash* tb,
     unsigned current_list_pos = 0; /* Prefound buckets list index */
     Eterm match_res;
     Sint got = 0;                  /* Matched terms counter */
-    erts_smp_rwmtx_t* lck;         /* Slot lock */
+    erts_rwmtx_t* lck;         /* Slot lock */
     int ret_value;
-    erts_smp_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue)
+    erts_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue)
         = (lock_for_write ? WLOCK_HASH : RLOCK_HASH);
-    void (*unlock_hash_function)(erts_smp_rwmtx_t*)
+    void (*unlock_hash_function)(erts_rwmtx_t*)
         = (lock_for_write ? WUNLOCK_HASH : RUNLOCK_HASH);
-    Sint (*next_slot_function)(DbTableHash*, Uint, erts_smp_rwmtx_t**)
+    Sint (*next_slot_function)(DbTableHash*, Uint, erts_rwmtx_t**)
         = (lock_for_write ? next_slot_w : next_slot);
 
     if ((ret_value = analyze_pattern(tb, pattern, extra_match_validator, &mpi))
@@ -1357,13 +1357,13 @@ static int match_traverse_continue(Process* p, DbTableHash* tb,
                                        */
     HashDbTerm* saved_current; /* Helper to avoid double skip on match */
     Eterm match_res;
-    erts_smp_rwmtx_t* lck;
+    erts_rwmtx_t* lck;
     int ret_value;
-    erts_smp_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue)
+    erts_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue)
         = (lock_for_write ? WLOCK_HASH : RLOCK_HASH);
-    void (*unlock_hash_function)(erts_smp_rwmtx_t*)
+    void (*unlock_hash_function)(erts_rwmtx_t*)
         = (lock_for_write ? WUNLOCK_HASH : RUNLOCK_HASH);
-    Sint (*next_slot_function)(DbTableHash* tb, Uint ix, erts_smp_rwmtx_t** lck_ptr)
+    Sint (*next_slot_function)(DbTableHash* tb, Uint ix, erts_rwmtx_t** lck_ptr)
         = (lock_for_write ? next_slot_w : next_slot);
 
     if (got < 0) {
@@ -1970,7 +1970,7 @@ static int mtraversal_select_delete_on_match_res(void* context_ptr, Sint slot_ix
         *current_ptr = (*current_ptr)->next; // replace pointer to term using next
         free_term(sd_context_ptr->tb, del);
     }
-    erts_smp_atomic_dec_nob(&sd_context_ptr->tb->common.nitems);
+    erts_atomic_dec_nob(&sd_context_ptr->tb->common.nitems);
 
     return 1;
 }
@@ -2209,7 +2209,7 @@ static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
     DbTableHash *tb = &tbl->hash;
     HashDbTerm **bp, *b;
     HashValue hval = MAKE_HASH(key);
-    erts_smp_rwmtx_t *lck = WLOCK_HASH(tb, hval);
+    erts_rwmtx_t *lck = WLOCK_HASH(tb, hval);
     int ix = hash_to_ix(tb, hval);
     int nitems_diff = 0;
 
@@ -2238,7 +2238,7 @@ static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
     }
     WUNLOCK_HASH(lck);
     if (nitems_diff) {
-        erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff);
+        erts_atomic_add_nob(&tb->common.nitems, nitems_diff);
         try_shrink(tb);
     }
     return DB_ERROR_NONE;
@@ -2260,7 +2260,7 @@ int db_mark_all_deleted_hash(DbTable *tbl)
     HashDbTerm* list;
     int i;
 
-    ERTS_SMP_LC_ASSERT(IS_TAB_WLOCKED(tb));
+    ERTS_LC_ASSERT(IS_TAB_WLOCKED(tb));
 
     for (i = 0; i < NACTIVE(tb); i++) {
 	if ((list = BUCKET(tb,i)) != NULL) {
@@ -2271,7 +2271,7 @@ int db_mark_all_deleted_hash(DbTable *tbl)
 	    }while(list != NULL);
 	}
     }
-    erts_smp_atomic_set_nob(&tb->common.nitems, 0);    
+    erts_atomic_set_nob(&tb->common.nitems, 0);
     return DB_ERROR_NONE;
 }
 
@@ -2345,8 +2345,8 @@ static int db_free_table_hash(DbTable *tbl)
 static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds)
 {
     DbTableHash *tb = &tbl->hash;
-    FixedDeletion* fixdel = (FixedDeletion*) erts_smp_atomic_read_acqb(&tb->fixdel);
-    ERTS_SMP_LC_ASSERT(IS_TAB_WLOCKED(tb) || (tb->common.status & DB_DELETE));
+    FixedDeletion* fixdel = (FixedDeletion*) erts_atomic_read_acqb(&tb->fixdel);
+    ERTS_LC_ASSERT(IS_TAB_WLOCKED(tb) || (tb->common.status & DB_DELETE));
 
     while (fixdel != NULL) {
 	FixedDeletion *fx = fixdel;
@@ -2358,11 +2358,11 @@ static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds)
 		     sizeof(FixedDeletion));
 	ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion));
 	if (--reds < 0) {
-	    erts_smp_atomic_set_relb(&tb->fixdel, (erts_aint_t)fixdel);
+	    erts_atomic_set_relb(&tb->fixdel, (erts_aint_t)fixdel);
 	    return reds;		/* Not done */
 	}
     }
-    erts_smp_atomic_set_relb(&tb->fixdel, (erts_aint_t)NULL);
+    erts_atomic_set_relb(&tb->fixdel, (erts_aint_t)NULL);
 
     while(tb->nslots != 0) {
 	reds -= EXT_SEGSZ/64 + free_seg(tb, 1);
@@ -2383,7 +2383,7 @@ static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds)
 		     (void*)tb->locks, sizeof(DbTableHashFineLocks));
 	tb->locks = NULL;
     }
-    ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size) == sizeof(DbTable));
+    ASSERT(erts_atomic_read_nob(&tb->common.memory_size) == sizeof(DbTable));
     return reds;			/* Done */
 }
 
@@ -2482,7 +2482,7 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern,
 		if (!db_has_variable(key)) {   /* Bound key */
 		    int ix, search_slot;
 		    HashDbTerm** bp;
-		    erts_smp_rwmtx_t* lck;
+		    erts_rwmtx_t* lck;
 		    hval = MAKE_HASH(key);
 		    lck = RLOCK_HASH(tb,hval);
 		    ix = hash_to_ix(tb, hval);
@@ -2729,7 +2729,7 @@ static void grow(DbTableHash* tb, int nitems)
     HashDbTerm** pnext;
     HashDbTerm** to_pnext;
     HashDbTerm* p;
-    erts_smp_rwmtx_t* lck;
+    erts_rwmtx_t* lck;
     int nactive;
     int from_ix, to_ix;
     int szm;
@@ -2751,7 +2751,7 @@ static void grow(DbTableHash* tb, int nitems)
         }
         ASSERT(nactive < tb->nslots);
 
-        szm = erts_smp_atomic_read_nob(&tb->szm);
+        szm = erts_atomic_read_nob(&tb->szm);
         if (nactive <= szm) {
             from_ix = nactive & (szm >> 1);
         } else {
@@ -2762,7 +2762,7 @@ static void grow(DbTableHash* tb, int nitems)
         to_ix = nactive;
 
         lck = WLOCK_HASH(tb, from_ix);
-        ERTS_SMP_ASSERT(lck == GET_LOCK_MAYBE(tb,to_ix));
+        ERTS_ASSERT(lck == GET_LOCK_MAYBE(tb,to_ix));
         /* Now a final double check (with the from_ix lock held)
          * that we did not get raced by a table fixer.
          */
@@ -2770,12 +2770,12 @@ static void grow(DbTableHash* tb, int nitems)
             WUNLOCK_HASH(lck);
             goto abort;
         }
-        erts_smp_atomic_set_nob(&tb->nactive, ++nactive);
+        erts_atomic_set_nob(&tb->nactive, ++nactive);
         if (from_ix == 0) {
             if (DB_USING_FINE_LOCKING(tb))
-                erts_smp_atomic_set_relb(&tb->szm, szm);
+                erts_atomic_set_relb(&tb->szm, szm);
             else
-                erts_smp_atomic_set_nob(&tb->szm, szm);
+                erts_atomic_set_nob(&tb->szm, szm);
         }
         done_resizing(tb);
 
@@ -2823,7 +2823,7 @@ static void shrink(DbTableHash* tb, int nitems)
     HashDbTerm** src_bp;
     HashDbTerm** dst_bp;
     HashDbTerm** bp;
-    erts_smp_rwmtx_t* lck;
+    erts_rwmtx_t* lck;
     int src_ix, dst_ix, low_szm;
     int nactive;
     int loop_limit = 5;
@@ -2836,13 +2836,13 @@ static void shrink(DbTableHash* tb, int nitems)
             goto abort; /* already done (race) */
         }
         src_ix = nactive - 1;
-        low_szm = erts_smp_atomic_read_nob(&tb->szm) >> 1;
+        low_szm = erts_atomic_read_nob(&tb->szm) >> 1;
         dst_ix = src_ix & low_szm;
 
         ASSERT(dst_ix < src_ix);
         ASSERT(nactive > FIRST_SEGSZ);
         lck = WLOCK_HASH(tb, dst_ix);
-        ERTS_SMP_ASSERT(lck == GET_LOCK_MAYBE(tb,src_ix));
+        ERTS_ASSERT(lck == GET_LOCK_MAYBE(tb,src_ix));
         /* Double check for racing table fixers */
         if (IS_FIXED(tb)) {
             WUNLOCK_HASH(lck);
@@ -2871,9 +2871,9 @@ static void shrink(DbTableHash* tb, int nitems)
         *src_bp = NULL;
 
         nactive = src_ix;
-        erts_smp_atomic_set_nob(&tb->nactive, nactive);
+        erts_atomic_set_nob(&tb->nactive, nactive);
         if (dst_ix == 0) {
-            erts_smp_atomic_set_relb(&tb->szm, low_szm);
+            erts_atomic_set_relb(&tb->szm, low_szm);
         }
         WUNLOCK_HASH(lck);
 
@@ -2908,12 +2908,12 @@ static HashDbTerm* search_list(DbTableHash* tb, Eterm key,
 /* It return the next live object in a table, NULL if no more */
 /* In-bucket: RLOCKED */
 /* Out-bucket: RLOCKED unless NULL */
-static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_smp_rwmtx_t** lck_ptr,
+static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr,
 			HashDbTerm *list)
 {
     int i;
 
-    ERTS_SMP_LC_ASSERT(IS_HASH_RLOCKED(tb,*iptr));
+    ERTS_LC_ASSERT(IS_HASH_RLOCKED(tb,*iptr));
 
     for (list = list->next; list != NULL; list = list->next) {
 	if (list->hvalue != INVALID_HASH)
@@ -2943,7 +2943,7 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj,
     DbTableHash *tb = &tbl->hash;
     HashValue hval;
     HashDbTerm **bp, *b;
-    erts_smp_rwmtx_t* lck;
+    erts_rwmtx_t* lck;
     int flags = 0;
 
     ASSERT(tb->common.status & DB_SET);
@@ -2999,7 +2999,7 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj,
             q->next = next;
             q->hvalue = hval;
             *bp = b = q;
-            erts_smp_atomic_inc_nob(&tb->common.nitems);
+            erts_atomic_inc_nob(&tb->common.nitems);
         }
 
         HRelease(p, hend, htop);
@@ -3025,10 +3025,10 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle)
     DbTableHash *tb = &tbl->hash;
     HashDbTerm **bp = (HashDbTerm **) handle->bp;
     HashDbTerm *b = *bp;
-    erts_smp_rwmtx_t* lck = (erts_smp_rwmtx_t*) handle->lck;
+    erts_rwmtx_t* lck = (erts_rwmtx_t*) handle->lck;
     HashDbTerm* free_me = NULL;
 
-    ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(tb, lck));  /* locked by db_lookup_dbterm_hash */
+    ERTS_LC_ASSERT(IS_HASH_WLOCKED(tb, lck));  /* locked by db_lookup_dbterm_hash */
 
     ASSERT((&b->dbterm == handle->dbterm) == !(tb->common.compress && handle->flags & DB_MUST_RESIZE));
 
@@ -3042,7 +3042,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle)
         }
 
         WUNLOCK_HASH(lck);
-        erts_smp_atomic_dec_nob(&tb->common.nitems);
+        erts_atomic_dec_nob(&tb->common.nitems);
         try_shrink(tb);
     } else {
         if (handle->flags & DB_MUST_RESIZE) {
@@ -3051,7 +3051,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle)
         }
         if (handle->flags & DB_INC_TRY_GROW) {
             int nactive;
-            int nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems);
+            int nitems = erts_atomic_inc_read_nob(&tb->common.nitems);
             WUNLOCK_HASH(lck);
             nactive = NACTIVE(tb);
 
@@ -3079,7 +3079,7 @@ static int db_delete_all_objects_hash(Process* p, DbTable* tbl)
     } else {
 	db_free_table_hash(tbl);
 	db_create_hash(p, tbl);
-	erts_smp_atomic_set_nob(&tbl->hash.common.nitems, 0);
+	erts_atomic_set_nob(&tbl->hash.common.nitems, 0);
     }
     return 0;
 }
@@ -3109,7 +3109,7 @@ void db_foreach_offheap_hash(DbTable *tbl,
 void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats)
 {
     HashDbTerm* b;
-    erts_smp_rwmtx_t* lck;
+    erts_rwmtx_t* lck;
     int sum = 0;
     int sq_sum = 0;
     int kept_items = 0;
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index 319d563859..7d27609825 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -42,8 +42,8 @@ typedef struct hash_db_term {
 
 typedef struct db_table_hash_fine_locks {
     union {
-	erts_smp_rwmtx_t lck;
-	byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_smp_rwmtx_t))];
+	erts_rwmtx_t lck;
+	byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_rwmtx_t))];
     }lck_vec[DB_HASH_LOCK_CNT];
 } DbTableHashFineLocks;
 
@@ -51,10 +51,10 @@ typedef struct db_table_hash {
     DbTableCommon common;
 
     /* SMP: szm and nactive are write-protected by is_resizing or table write lock */
-    erts_smp_atomic_t szm;     /* current size mask. */
-    erts_smp_atomic_t nactive; /* Number of "active" slots */
+    erts_atomic_t szm;     /* current size mask. */
+    erts_atomic_t nactive; /* Number of "active" slots */
 
-    erts_smp_atomic_t segtab;  /* The segment table (struct segment**) */
+    erts_atomic_t segtab;  /* The segment table (struct segment**) */
     struct segment* first_segtab[1];
 
     /* SMP: nslots and nsegs are protected by is_resizing or table write lock */
@@ -62,8 +62,8 @@ typedef struct db_table_hash {
     int nsegs;        /* Size of segment table */
 
     /* List of slots where elements have been deleted while table was fixed */
-    erts_smp_atomic_t fixdel;  /* (FixedDeletion*) */	
-    erts_smp_atomic_t is_resizing; /* grow/shrink in progress */
+    erts_atomic_t fixdel;  /* (FixedDeletion*) */
+    erts_atomic_t is_resizing; /* grow/shrink in progress */
     DbTableHashFineLocks* locks;
 } DbTableHash;
 
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index d7deadacf0..038f6602bf 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -50,7 +50,7 @@
 #include "erl_db_tree.h"
 
 #define GETKEY_WITH_POS(Keypos, Tplp) (*((Tplp) + Keypos))
-#define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems))
+#define NITEMS(tb) ((int)erts_atomic_read_nob(&(tb)->common.nitems))
 
 /*
 ** A stack of this size is enough for an AVL tree with more than
@@ -94,7 +94,7 @@
 */
 static DbTreeStack* get_static_stack(DbTableTree* tb)
 {
-    if (!erts_smp_atomic_xchg_acqb(&tb->is_stack_busy, 1)) {
+    if (!erts_atomic_xchg_acqb(&tb->is_stack_busy, 1)) {
 	return &tb->static_stack;
     }
     return NULL;
@@ -106,7 +106,7 @@ static DbTreeStack* get_static_stack(DbTableTree* tb)
 static DbTreeStack* get_any_stack(DbTableTree* tb)
 {
     DbTreeStack* stack;
-    if (!erts_smp_atomic_xchg_acqb(&tb->is_stack_busy, 1)) {
+    if (!erts_atomic_xchg_acqb(&tb->is_stack_busy, 1)) {
 	return &tb->static_stack;
     }
     stack = erts_db_alloc(ERTS_ALC_T_DB_STK, (DbTable *) tb,
@@ -120,8 +120,8 @@ static DbTreeStack* get_any_stack(DbTableTree* tb)
 static void release_stack(DbTableTree* tb, DbTreeStack* stack)
 {
     if (stack == &tb->static_stack) {
-	ASSERT(erts_smp_atomic_read_nob(&tb->is_stack_busy) == 1);
-	erts_smp_atomic_set_relb(&tb->is_stack_busy, 0);
+	ASSERT(erts_atomic_read_nob(&tb->is_stack_busy) == 1);
+	erts_atomic_set_relb(&tb->is_stack_busy, 0);
     }
     else {
 	erts_db_free(ERTS_ALC_T_DB_STK, (DbTable *) tb,
@@ -517,7 +517,7 @@ int db_create_tree(Process *p, DbTable *tbl)
 					   sizeof(TreeDbTerm *) * STACK_NEED);
     tb->static_stack.pos = 0;
     tb->static_stack.slot = 0;
-    erts_smp_atomic_init_nob(&tb->is_stack_busy, 0);
+    erts_atomic_init_nob(&tb->is_stack_busy, 0);
     tb->deletion = 0;
     return DB_ERROR_NONE;
 }
@@ -646,8 +646,8 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail)
     for (;;)
 	if (!*this) { /* Found our place */
 	    state = 1;
-	    if (erts_smp_atomic_inc_read_nob(&tb->common.nitems) >= TREE_MAX_ELEMENTS) {
-		erts_smp_atomic_dec_nob(&tb->common.nitems);
+	    if (erts_atomic_inc_read_nob(&tb->common.nitems) >= TREE_MAX_ELEMENTS) {
+		erts_atomic_dec_nob(&tb->common.nitems);
 		return DB_ERROR_SYSRES;
 	    }
 	    *this = new_dbterm(tb, obj);
@@ -1608,7 +1608,7 @@ static int db_select_delete_continue_tree(Process *p,
     sc.max = 1000;
     sc.keypos = tb->common.keypos;
 
-    ASSERT(!erts_smp_atomic_read_nob(&tb->is_stack_busy));
+    ASSERT(!erts_atomic_read_nob(&tb->is_stack_busy));
     traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc);
 
     BUMP_REDS(p, 1000 - sc.max);
@@ -2020,7 +2020,7 @@ static SWord db_free_table_continue_tree(DbTable *tbl, SWord reds)
 		     (DbTable *) tb,
 		     (void *) tb->static_stack.array,
 		     sizeof(TreeDbTerm *) * STACK_NEED);
-	ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size)
+	ASSERT(erts_atomic_read_nob(&tb->common.memory_size)
 	       == sizeof(DbTable));
     }
     return reds;
@@ -2030,7 +2030,7 @@ static int db_delete_all_objects_tree(Process* p, DbTable* tbl)
 {
     db_free_table_tree(tbl);
     db_create_tree(p, tbl);
-    erts_smp_atomic_set_nob(&tbl->tree.common.nitems, 0);
+    erts_atomic_set_nob(&tbl->tree.common.nitems, 0);
     return 0;
 }
 
@@ -2110,7 +2110,7 @@ static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key) {
 		tstack[tpos++] = this;
 		state = delsub(this);
 	    }
-	    erts_smp_atomic_dec_nob(&tb->common.nitems);
+	    erts_atomic_dec_nob(&tb->common.nitems);
 	    break;
 	}
     }
@@ -2177,7 +2177,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb,
 		tstack[tpos++] = this;
 		state = delsub(this);
 	    }
-	    erts_smp_atomic_dec_nob(&tb->common.nitems);
+	    erts_atomic_dec_nob(&tb->common.nitems);
 	    break;
 	}
     }
diff --git a/erts/emulator/beam/erl_db_tree.h b/erts/emulator/beam/erl_db_tree.h
index 72749ead1e..dc1b93d410 100644
--- a/erts/emulator/beam/erl_db_tree.h
+++ b/erts/emulator/beam/erl_db_tree.h
@@ -41,7 +41,7 @@ typedef struct db_table_tree {
     /* Tree-specific fields */
     TreeDbTerm *root;         /* The tree root */
     Uint deletion;		/* Being deleted */
-    erts_smp_atomic_t is_stack_busy;
+    erts_atomic_t is_stack_busy;
     DbTreeStack static_stack;
 } DbTableTree;
 
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 7310694e63..e017b9552b 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -170,7 +170,7 @@ static Eterm
 set_match_trace(Process *tracee_p, Eterm fail_term, ErtsTracer tracer,
 		Uint d_flags, Uint e_flags) {
 
-    ERTS_SMP_LC_ASSERT(
+    ERTS_LC_ASSERT(
         ERTS_PROC_LOCKS_ALL == erts_proc_lc_my_proc_locks(tracee_p)
         || erts_thr_progress_is_blocking());
 
@@ -361,7 +361,7 @@ typedef struct {
 } ErtsMatchPseudoProcess;
 
 
-static erts_smp_tsd_key_t match_pseudo_process_key;
+static erts_tsd_key_t match_pseudo_process_key;
 
 static ERTS_INLINE void
 cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap)
@@ -415,21 +415,21 @@ get_match_pseudo_process(Process *c_p, Uint heap_size)
     esdp = c_p ? c_p->scheduler_data : erts_get_scheduler_data();
 
     mpsp = esdp ? esdp->match_pseudo_process :
-        (ErtsMatchPseudoProcess*) erts_smp_tsd_get(match_pseudo_process_key);
+        (ErtsMatchPseudoProcess*) erts_tsd_get(match_pseudo_process_key);
 
     if (mpsp) {
-        ASSERT(mpsp == erts_smp_tsd_get(match_pseudo_process_key));
+        ASSERT(mpsp == erts_tsd_get(match_pseudo_process_key));
         ASSERT(mpsp->process.scheduler_data == esdp);
 	cleanup_match_pseudo_process(mpsp, 0);
     }
     else {
-	ASSERT(erts_smp_tsd_get(match_pseudo_process_key) == NULL);
+	ASSERT(erts_tsd_get(match_pseudo_process_key) == NULL);
 	mpsp = create_match_pseudo_process();
         if (esdp) {
             esdp->match_pseudo_process = (void *) mpsp;
         }
         mpsp->process.scheduler_data = esdp;
-	erts_smp_tsd_set(match_pseudo_process_key, (void *) mpsp);
+	erts_tsd_set(match_pseudo_process_key, (void *) mpsp);
     }
     if (heap_size > ERTS_DEFAULT_MS_HEAP_SIZE*sizeof(Eterm)) {
 	mpsp->u.heap = (Eterm*) erts_alloc(ERTS_ALC_T_DB_MS_RUN_HEAP, heap_size);
@@ -444,11 +444,11 @@ static void
 destroy_match_pseudo_process(void)
 {
     ErtsMatchPseudoProcess *mpsp;
-    mpsp = (ErtsMatchPseudoProcess *)erts_smp_tsd_get(match_pseudo_process_key);
+    mpsp = (ErtsMatchPseudoProcess *)erts_tsd_get(match_pseudo_process_key);
     if (mpsp) {
 	cleanup_match_pseudo_process(mpsp, 0);
 	erts_free(ERTS_ALC_T_DB_MS_PSDO_PROC, (void *) mpsp);
-	erts_smp_tsd_set(match_pseudo_process_key, (void *) NULL);
+	erts_tsd_set(match_pseudo_process_key, (void *) NULL);
     }
 }
 
@@ -456,9 +456,9 @@ static
 void
 match_pseudo_process_init(void)
 {
-    erts_smp_tsd_key_create(&match_pseudo_process_key,
+    erts_tsd_key_create(&match_pseudo_process_key,
 			    "erts_match_pseudo_process_key");
-    erts_smp_install_exit_handler(destroy_match_pseudo_process);
+    erts_thr_install_exit_handler(destroy_match_pseudo_process);
 }
 
 void
@@ -469,7 +469,7 @@ erts_match_set_release_result(Process* c_p)
 
 /* The trace control word. */
 
-static erts_smp_atomic32_t trace_control_word;
+static erts_atomic32_t trace_control_word;
 
 /* This needs to be here, before the bif table... */
 
@@ -908,7 +908,7 @@ static void db_free_tmp_uncompressed(DbTerm* obj);
 */
 BIF_RETTYPE db_get_trace_control_word(Process *p)
 {
-    Uint32 tcw = (Uint32) erts_smp_atomic32_read_acqb(&trace_control_word);
+    Uint32 tcw = (Uint32) erts_atomic32_read_acqb(&trace_control_word);
     BIF_RET(erts_make_integer((Uint) tcw, p));
 }
 
@@ -926,7 +926,7 @@ BIF_RETTYPE db_set_trace_control_word(Process *p, Eterm new)
     if (val != ((Uint32)val))
 	BIF_ERROR(p, BADARG);
 
-    old_tcw = (Uint32) erts_smp_atomic32_xchg_relb(&trace_control_word,
+    old_tcw = (Uint32) erts_atomic32_xchg_relb(&trace_control_word,
 						   (erts_aint32_t) val);
     BIF_RET(erts_make_integer((Uint) old_tcw, p));
 }
@@ -1451,7 +1451,7 @@ void db_initialize_util(void){
 	  sizeof(DMCGuardBif), 
 	  (int (*)(const void *, const void *)) &cmp_guard_bif);
     match_pseudo_process_init();
-    erts_smp_atomic32_init_nob(&trace_control_word, 0);
+    erts_atomic32_init_nob(&trace_control_word, 0);
 }
 
 
@@ -2513,9 +2513,9 @@ restart:
         case matchEnableTrace:
             ASSERT(c_p == self);
 	    if ( (n = erts_trace_flag2bit(esp[-1]))) {
-                erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+                erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		set_tracee_flags(c_p, ERTS_TRACER(c_p), 0, n);
-                erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+                erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		esp[-1] = am_true;
 	    } else {
 		esp[-1] = FAIL_TERM;
@@ -2530,9 +2530,9 @@ restart:
 		    /* Always take over the tracer of the current process */
 		    set_tracee_flags(tmpp, ERTS_TRACER(c_p), 0, n);
                     if (tmpp == c_p)
-                        erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL_MINOR);
+                        erts_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL_MINOR);
                     else
-                        erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL);
+                        erts_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL);
                     esp[-1] = am_true;
 		}
 	    }
@@ -2540,9 +2540,9 @@ restart:
         case matchDisableTrace:
             ASSERT(c_p == self);
 	    if ( (n = erts_trace_flag2bit(esp[-1]))) {
-                erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+                erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		set_tracee_flags(c_p, ERTS_TRACER(c_p), n, 0);
-                erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+                erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		esp[-1] = am_true;
 	    } else {
 		esp[-1] = FAIL_TERM;
@@ -2557,9 +2557,9 @@ restart:
 		    /* Always take over the tracer of the current process */
 		    set_tracee_flags(tmpp, ERTS_TRACER(c_p), n, 0);
                     if (tmpp == c_p)
-                        erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL_MINOR);
+                        erts_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL_MINOR);
                     else
-                        erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL);
+                        erts_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL);
                     esp[-1] = am_true;
 		}
 	    }
@@ -2583,14 +2583,14 @@ restart:
 	    if (in_flags & ERTS_PAM_IGNORE_TRACE_SILENT)
 	      break;
 	    if (*esp == am_true) {
-		erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+		erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		ERTS_TRACE_FLAGS(c_p) |= F_TRACE_SILENT;
-		erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+		erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 	    }
 	    else if (*esp == am_false) {
-		erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+		erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		ERTS_TRACE_FLAGS(c_p) &= ~F_TRACE_SILENT;
-		erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+		erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 	    }
 	    break;
         case matchTrace2:
@@ -2619,10 +2619,10 @@ restart:
                     ERTS_TRACER_CLEAR(&tracer);
 		    break;
 		}
-		erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+		erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		(--esp)[-1] = set_match_trace(c_p, FAIL_TERM, tracer,
 					      d_flags, e_flags);
-		erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+		erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
                 ERTS_TRACER_CLEAR(&tracer);
 	    }
 	    break;
@@ -2652,13 +2652,13 @@ restart:
 		if (tmpp == c_p) {
 		    (--esp)[-1] = set_match_trace(c_p, FAIL_TERM, tracer,
 						  d_flags, e_flags);
-		    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+		    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		} else {
-		    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+		    erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
 		    (--esp)[-1] = set_match_trace(tmpp, FAIL_TERM, tracer,
 						  d_flags, e_flags);
-		    erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL);
-		    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+		    erts_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL);
+		    erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 		}
                 ERTS_TRACER_CLEAR(&tracer);
 	    }
@@ -3262,7 +3262,7 @@ void db_cleanup_offheap_comp(DbTerm* obj)
 	    break;
 	case FUN_SUBTAG:
 	    ASSERT(u.pb != &tmp);
-	    if (erts_smp_refc_dectest(&u.fun->fe->refc, 0) == 0) {
+	    if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) {
 		erts_erase_fun_entry(u.fun->fe);
 	    }
 	    break;
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index fb9db3b010..1c99b661e4 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -237,12 +237,12 @@ typedef struct {
  */
 
 typedef struct db_table_common {
-    erts_smp_refc_t refc;     /* reference count of table struct */
-    erts_smp_refc_t fix_count;/* fixation counter */
+    erts_refc_t refc;     /* reference count of table struct */
+    erts_refc_t fix_count;/* fixation counter */
     DbTableList all;
     DbTableList owned;
-    erts_smp_rwmtx_t rwlock;  /* rw lock on table */
-    erts_smp_mtx_t fixlock;   /* Protects fixing_procs and time */
+    erts_rwmtx_t rwlock;  /* rw lock on table */
+    erts_mtx_t fixlock;   /* Protects fixing_procs and time */
     int is_thread_safe;       /* No fine locking inside table needed */
     Uint32 type;              /* table type, *read only* after creation */
     Eterm owner;              /* Pid of the creator */
@@ -252,8 +252,8 @@ typedef struct db_table_common {
     Eterm the_name;           /* an atom */
     Binary *btid;
     DbTableMethod* meth;      /* table methods */
-    erts_smp_atomic_t nitems; /* Total number of items in table */
-    erts_smp_atomic_t memory_size;/* Total memory size. NOTE: in bytes! */
+    erts_atomic_t nitems; /* Total number of items in table */
+    erts_atomic_t memory_size;/* Total memory size. NOTE: in bytes! */
     struct {                  /* Last fixation time */
 	ErtsMonotonicTime monotonic;
 	ErtsMonotonicTime offset;
@@ -286,7 +286,7 @@ typedef struct db_table_common {
 				  (DB_BAG | DB_SET | DB_DUPLICATE_BAG)))
 #define IS_TREE_TABLE(Status) (!!((Status) & \
 				  DB_ORDERED_SET))
-#define NFIXED(T) (erts_smp_refc_read(&(T)->common.fix_count,0))
+#define NFIXED(T) (erts_refc_read(&(T)->common.fix_count,0))
 #define IS_FIXED(T) (NFIXED(T) != 0) 
 
 /*
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 7b19724814..ff9f2c0d22 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -141,7 +141,7 @@ void erl_drv_thr_init(void)
 /*
  * These functions implement the driver thread interface in erl_driver.h.
  * NOTE: Only use this interface from drivers. From within the emulator use
- * either the erl_threads.h, the erl_smp.h or the ethread.h interface.
+ * either the erl_threads.h or the ethread.h interface.
  */
 
 ErlDrvMutex *
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index bfecb7a85f..9c866250bb 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -30,17 +30,16 @@
 
 static Hash erts_fun_table;
 
-#include "erl_smp.h"
 #ifdef HIPE
 # include "hipe_mode_switch.h"
 #endif
 
-static erts_smp_rwmtx_t erts_fun_table_lock;
+static erts_rwmtx_t erts_fun_table_lock;
 
-#define erts_fun_read_lock()	erts_smp_rwmtx_rlock(&erts_fun_table_lock)
-#define erts_fun_read_unlock()	erts_smp_rwmtx_runlock(&erts_fun_table_lock)
-#define erts_fun_write_lock()	erts_smp_rwmtx_rwlock(&erts_fun_table_lock)
-#define erts_fun_write_unlock()	erts_smp_rwmtx_rwunlock(&erts_fun_table_lock)
+#define erts_fun_read_lock()	erts_rwmtx_rlock(&erts_fun_table_lock)
+#define erts_fun_read_unlock()	erts_rwmtx_runlock(&erts_fun_table_lock)
+#define erts_fun_write_lock()	erts_rwmtx_rwlock(&erts_fun_table_lock)
+#define erts_fun_write_unlock()	erts_rwmtx_rwunlock(&erts_fun_table_lock)
 
 static HashValue fun_hash(ErlFunEntry* obj);
 static int fun_cmp(ErlFunEntry* obj1, ErlFunEntry* obj2);
@@ -59,11 +58,11 @@ void
 erts_init_fun_table(void)
 {
     HashFunctions f;
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
-    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
-    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+    erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_RWMTX_LONG_LIVED;
 
-    erts_smp_rwmtx_init_opt(&erts_fun_table_lock, &rwmtx_opt, "fun_tab", NIL,
+    erts_rwmtx_init_opt(&erts_fun_table_lock, &rwmtx_opt, "fun_tab", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
     f.hash = (H_FUN) fun_hash;
@@ -114,9 +113,9 @@ erts_put_fun_entry(Eterm mod, int uniq, int index)
     fe = (ErlFunEntry *) hash_put(&erts_fun_table, (void*) &template);
     sys_memset(fe->uniq, 0, sizeof(fe->uniq));
     fe->index = 0;
-    refc = erts_smp_refc_inctest(&fe->refc, 0);
+    refc = erts_refc_inctest(&fe->refc, 0);
     if (refc < 2) /* New or pending delete */
-	erts_smp_refc_inc(&fe->refc, 1);
+	erts_refc_inc(&fe->refc, 1);
     erts_fun_write_unlock();
     return fe;
 }
@@ -138,9 +137,9 @@ erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index,
     sys_memcpy(fe->uniq, uniq, sizeof(fe->uniq));
     fe->index = index;
     fe->arity = arity;
-    refc = erts_smp_refc_inctest(&fe->refc, 0);
+    refc = erts_refc_inctest(&fe->refc, 0);
     if (refc < 2) /* New or pending delete */
-	erts_smp_refc_inc(&fe->refc, 1);
+	erts_refc_inc(&fe->refc, 1);
     erts_fun_write_unlock();
     return fe;
 }
@@ -165,9 +164,9 @@ erts_get_fun_entry(Eterm mod, int uniq, int index)
     erts_fun_read_lock();
     ret = (ErlFunEntry *) hash_get(&erts_fun_table, (void*) &template);
     if (ret) {
-	erts_aint_t refc = erts_smp_refc_inctest(&ret->refc, 1);
+	erts_aint_t refc = erts_refc_inctest(&ret->refc, 1);
 	if (refc < 2) /* Pending delete */
-	    erts_smp_refc_inc(&ret->refc, 1);
+	    erts_refc_inc(&ret->refc, 1);
     }
     erts_fun_read_unlock();
     return ret;
@@ -187,7 +186,7 @@ erts_erase_fun_entry(ErlFunEntry* fe)
      * We have to check refc again since someone might have looked up
      * the fun entry and incremented refc after last check.
      */
-    if (erts_smp_refc_dectest(&fe->refc, -1) <= 0)
+    if (erts_refc_dectest(&fe->refc, -1) <= 0)
     {
 	if (fe->address != unloaded_fun)
 	    erts_exit(ERTS_ERROR_EXIT,
@@ -219,7 +218,7 @@ erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end)
 
 	    if (start <= addr && addr < end) {
 		fe->pend_purge_address = addr;
-		ERTS_SMP_WRITE_MEMORY_BARRIER;
+		ERTS_THR_WRITE_MEMORY_BARRIER;
 		fe->address = unloaded_fun;
 #ifdef HIPE
                 fe->pend_purge_native_address = fe->native_address;
@@ -273,10 +272,10 @@ erts_fun_purge_complete(ErlFunEntry **funs, Uint no)
 #ifdef HIPE
         fe->pend_purge_native_address = NULL;
 #endif
-	if (erts_smp_refc_dectest(&fe->refc, 0) == 0)
+	if (erts_refc_dectest(&fe->refc, 0) == 0)
 	    erts_erase_fun_entry(fe);
     }
-    ERTS_SMP_WRITE_MEMORY_BARRIER;
+    ERTS_THR_WRITE_MEMORY_BARRIER;
 }
 
 void
@@ -305,7 +304,7 @@ erts_dump_fun_entries(fmtfn_t to, void *to_arg)
 #ifdef HIPE
 	    erts_print(to, to_arg, "Native_address: %p\n", fe->native_address);
 #endif
-	    erts_print(to, to_arg, "Refc: %ld\n", erts_smp_refc_read(&fe->refc, 1));
+	    erts_print(to, to_arg, "Refc: %ld\n", erts_refc_read(&fe->refc, 1));
 	    b = b->next;
 	}
     }
@@ -336,7 +335,7 @@ fun_alloc(ErlFunEntry* template)
     obj->old_uniq = template->old_uniq;
     obj->old_index = template->old_index;
     obj->module = template->module;
-    erts_smp_refc_init(&obj->refc, -1);
+    erts_refc_init(&obj->refc, -1);
     obj->address = unloaded_fun;
     obj->pend_purge_address = NULL;
 #ifdef HIPE
diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h
index 289d0d0b28..fb2901d866 100644
--- a/erts/emulator/beam/erl_fun.h
+++ b/erts/emulator/beam/erl_fun.h
@@ -21,7 +21,7 @@
 #ifndef __ERLFUNTABLE_H__
 #define __ERLFUNTABLE_H__
 
-#include "erl_smp.h"
+#include "erl_threads.h"
 
 /*
  * Fun entry.
@@ -42,7 +42,7 @@ typedef struct erl_fun_entry {
 
     Uint arity;			/* The arity of the fun. */
     Eterm module;		/* Tagged atom for module. */
-    erts_smp_refc_t refc;		/* Reference count: One for code + one for each
+    erts_refc_t refc;		/* Reference count: One for code + one for each
 				   fun object in each process. */
     BeamInstr *pend_purge_address; /* address stored during a pending purge */
 #ifdef HIPE
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 14eeeaf70a..bc3bcdc9ad 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -180,7 +180,7 @@ typedef struct {
     Eterm ref;
     Eterm ref_heap[ERTS_REF_THING_SIZE];
     Uint req_sched;
-    erts_smp_atomic32_t refc;
+    erts_atomic32_t refc;
 } ErtsGCInfoReq;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -274,7 +274,7 @@ erts_init_gc(void)
     }
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-    erts_smp_mtx_init(&dirty_gc.mtx, "dirty_gc_info", NIL,
+    erts_mtx_init(&dirty_gc.mtx, "dirty_gc_info", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
     init_gc_info(&dirty_gc.info);
 #endif
@@ -672,7 +672,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
 
     ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) >= esdp->virtual_reds);
 
-    state = erts_smp_atomic32_read_nob(&p->state);
+    state = erts_atomic32_read_nob(&p->state);
 
     if ((p->flags & (F_DISABLE_GC|F_DELAY_GC)) || state & ERTS_PSFLG_EXITING) {
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -698,7 +698,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
 
     ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_GC);
 
-    erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
+    erts_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
     if (erts_system_monitor_long_gc != 0)
 	start_time = erts_get_monotonic_time(esdp);
 
@@ -779,17 +779,17 @@ do_major_collection:
         ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL;
         int res;
 
-        erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+        erts_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
         erts_send_exit_signal(p, p->common.id, p, &locks,
                               am_kill, NIL, NULL, 0);
-        erts_smp_proc_unlock(p, locks & ERTS_PROC_LOCKS_ALL_MINOR);
+        erts_proc_unlock(p, locks & ERTS_PROC_LOCKS_ALL_MINOR);
 
 #ifdef ERTS_DIRTY_SCHEDULERS
     delay_gc_after_start:
 #endif
         /* erts_send_exit_signal looks for ERTS_PSFLG_GC, so
            we have to remove it after the signal is sent */
-        erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
+        erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
 
         /* We have to make sure that we have space for need on the heap */
         res = delay_garbage_collection(p, live_hf_end, need, fcalls);
@@ -797,7 +797,7 @@ do_major_collection:
         return res;
     }
 
-    erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
+    erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
 
     if (IS_TRACED_FL(p, F_TRACE_GC)) {
         trace_gc(p, gc_trace_end_tag, reclaimed_now, THE_NON_VALUE);
@@ -924,7 +924,7 @@ garbage_collect_hibernate(Process* p, int check_long_gc)
     /*
      * Preliminaries.
      */
-    erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
+    erts_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
     ErtsGcQuickSanityCheck(p);
     ASSERT(p->stop == p->hend);	/* Stack must be empty. */
 
@@ -1015,7 +1015,7 @@ garbage_collect_hibernate(Process* p, int check_long_gc)
 
     p->flags |= F_HIBERNATED;
 
-    erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
+    erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
 
     reds = gc_cost(actual_size, actual_size);
     return reds;
@@ -1137,7 +1137,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
     /*
      * Set GC state.
      */
-    erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
+    erts_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
 
     /*
      * Just did a major collection (which has discarded the old heap),
@@ -1284,7 +1284,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
     /*
      * Restore status.
      */
-    erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
+    erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
 
     reds += (Sint64) gc_cost((p->htop - p->heap) + byte_lit_size/sizeof(Uint), 0);
 
@@ -2914,7 +2914,7 @@ sweep_off_heap(Process *p, int fullsweep)
 	    case FUN_SUBTAG:
 		{
 		    ErlFunEntry* fe = ((ErlFunThing*)ptr)->fe;
-		    if (erts_smp_refc_dectest(&fe->refc, 0) == 0) {
+		    if (erts_refc_dectest(&fe->refc, 0) == 0) {
 			erts_erase_fun_entry(fe);
 		    }
 		    break;
@@ -3274,11 +3274,11 @@ reply_gc_info(void *vgcirp)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
  
     if (rp_locks)
-	erts_smp_proc_unlock(rp, rp_locks);
+	erts_proc_unlock(rp, rp_locks);
 
     erts_proc_dec_refc(rp);
 
-    if (erts_smp_atomic32_dec_read_nob(&gcirp->refc) == 0)
+    if (erts_atomic32_dec_read_nob(&gcirp->refc) == 0)
 	gcireq_free(vgcirp);
 }
 
@@ -3330,7 +3330,7 @@ erts_gc_info_request(Process *c_p)
     gcirp->proc = c_p;
     gcirp->ref = STORE_NC(&hp, NULL, ref);
     gcirp->req_sched = esdp->no;
-    erts_smp_atomic32_init_nob(&gcirp->refc,
+    erts_atomic32_init_nob(&gcirp->refc,
 			       (erts_aint32_t) erts_no_schedulers);
 
     erts_proc_add_refc(c_p, (Sint) erts_no_schedulers);
@@ -3626,12 +3626,12 @@ erts_check_off_heap2(Process *p, Eterm *htop)
 	    refc = erts_refc_read(&u.pb->val->intern.refc, 1);
 	    break;
 	case FUN_SUBTAG:
-	    refc = erts_smp_refc_read(&u.fun->fe->refc, 1);
+	    refc = erts_refc_read(&u.fun->fe->refc, 1);
 	    break;
 	case EXTERNAL_PID_SUBTAG:
 	case EXTERNAL_PORT_SUBTAG:
 	case EXTERNAL_REF_SUBTAG:
-	    refc = erts_smp_refc_read(&u.ext->node->refc, 1);
+	    refc = erts_refc_read(&u.ext->node->refc, 1);
 	    break;
 	case REF_SUBTAG:
 	    ASSERT(is_magic_ref_thing(u.hdr));
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 80f3aa04ab..f8cbe6f49a 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -152,7 +152,7 @@ typedef struct {
 
 typedef struct {
     Uint32 roflgs;
-    erts_smp_atomic32_t refc;
+    erts_atomic32_t refc;
     union {
 	void *arg;
 	erts_atomic_t next;
@@ -193,7 +193,7 @@ struct ErtsBifTimer_ {
         ErtsTWTimer twt;
     } type;
     struct {
-        erts_smp_atomic32_t state;
+        erts_atomic32_t state;
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
         ErtsMagicBinary *mbin;
         ErtsHLTimerList proc_list;
@@ -776,13 +776,13 @@ get_time_left(ErtsSchedulerData *esdp, ErtsMonotonicTime timeout_pos)
 static ERTS_INLINE int
 proc_timeout_common(Process *proc, void *tmr)
 {
-    if (tmr == (void *) erts_smp_atomic_cmpxchg_mb(&proc->common.timer,
+    if (tmr == (void *) erts_atomic_cmpxchg_mb(&proc->common.timer,
 						   ERTS_PTMR_TIMEDOUT,
 						   (erts_aint_t) tmr)) {
 	erts_aint32_t state;
-	erts_smp_proc_lock(proc, ERTS_PROC_LOCKS_MSG_RECEIVE);
-	state = erts_smp_atomic32_read_acqb(&proc->state);
-	erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	erts_proc_lock(proc, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	state = erts_atomic32_read_acqb(&proc->state);
+	erts_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	if (!(state & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_EXITING)))
 	    erts_schedule_process(proc, state, 0);
 	return 1;
@@ -793,7 +793,7 @@ proc_timeout_common(Process *proc, void *tmr)
 static ERTS_INLINE int
 port_timeout_common(Port *port, void *tmr)
 {
-    if (tmr == (void *) erts_smp_atomic_cmpxchg_mb(&port->common.timer,
+    if (tmr == (void *) erts_atomic_cmpxchg_mb(&port->common.timer,
 						   ERTS_PTMR_TIMEDOUT,
 						   (erts_aint_t) tmr)) {
 	erts_port_task_schedule(port->common.id,
@@ -806,24 +806,24 @@ port_timeout_common(Port *port, void *tmr)
 
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
 
-static erts_smp_atomic_t *
+static erts_atomic_t *
 mbin_to_btmref__(ErtsMagicBinary *mbin)
 {
-    return erts_smp_binary_to_magic_indirection((Binary *) mbin);
+    return erts_binary_to_magic_indirection((Binary *) mbin);
 }
 
 static ERTS_INLINE void
 magic_binary_init(ErtsMagicBinary *mbin, ErtsBifTimer *tmr)
 {
-    erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin);
-    erts_smp_atomic_init_nob(aptr, (erts_aint_t) tmr);
+    erts_atomic_t *aptr = mbin_to_btmref__(mbin);
+    erts_atomic_init_nob(aptr, (erts_aint_t) tmr);
 }
 
 static ERTS_INLINE ErtsBifTimer *
 magic_binary_to_btm(ErtsMagicBinary *mbin)
 {
-    erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin);
-    ErtsBifTimer *tmr = (ErtsBifTimer *) erts_smp_atomic_read_nob(aptr);
+    erts_atomic_t *aptr = mbin_to_btmref__(mbin);
+    ErtsBifTimer *tmr = (ErtsBifTimer *) erts_atomic_read_nob(aptr);
     ERTS_HLT_ASSERT(!tmr || tmr->btm.mbin == mbin);
     return tmr;
 }
@@ -869,7 +869,7 @@ init_btm_specifics(ErtsSchedulerData *esdp,
     btm_rbt_insert(&esdp->timer_service->btm_tree, tmr);
 #endif
 
-    erts_smp_atomic32_init_nob(&tmr->btm.state, ERTS_TMR_STATE_ACTIVE);
+    erts_atomic32_init_nob(&tmr->btm.state, ERTS_TMR_STATE_ACTIVE);
     return refc; /* refc from magic binary... */
 }
 
@@ -902,10 +902,10 @@ timer_pre_dec_refc(ErtsTimer *tmr)
 {
 #ifdef ERTS_HLT_DEBUG
     erts_aint_t refc;
-    refc = erts_smp_atomic32_dec_read_nob(&tmr->head.refc);
+    refc = erts_atomic32_dec_read_nob(&tmr->head.refc);
     ERTS_HLT_ASSERT(refc > 0);
 #else
-    erts_smp_atomic32_dec_nob(&tmr->head.refc);
+    erts_atomic32_dec_nob(&tmr->head.refc);
 #endif
 }
 
@@ -954,7 +954,7 @@ schedule_tw_timer_destroy(ErtsTWTimer *tmr)
 static ERTS_INLINE void
 tw_timer_dec_refc(ErtsTWTimer *tmr)
 {
-    if (erts_smp_atomic32_dec_read_relb(&tmr->head.refc) == 0) {
+    if (erts_atomic32_dec_read_relb(&tmr->head.refc) == 0) {
         ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
 	schedule_tw_timer_destroy(tmr);
     }
@@ -1099,7 +1099,7 @@ create_tw_timer(ErtsSchedulerData *esdp,
 	return NULL;
     }
 
-    erts_smp_atomic32_init_nob(&tmr->head.refc, refc);
+    erts_atomic32_init_nob(&tmr->head.refc, refc);
 
     erts_twheel_set_timer(esdp->timer_wheel,
 			  &tmr->u.tw_tmr,
@@ -1132,7 +1132,7 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs)
      * at once...
      */
 
-    ERTS_HLT_ASSERT(erts_smp_atomic32_read_nob(&tmr->head.refc) == 0);
+    ERTS_HLT_ASSERT(erts_atomic32_read_nob(&tmr->head.refc) == 0);
 
     if (roflgs & ERTS_TMR_ROFLG_REG_NAME) {
 	ERTS_HLT_ASSERT(is_atom(tmr->head.receiver.name));
@@ -1164,7 +1164,7 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs)
 static ERTS_INLINE void
 hl_timer_dec_refc(ErtsHLTimer *tmr, Uint32 roflgs)
 {
-    if (erts_smp_atomic32_dec_read_relb(&tmr->head.refc) == 0) {
+    if (erts_atomic32_dec_read_relb(&tmr->head.refc) == 0) {
         ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
 	schedule_hl_timer_destroy(tmr, roflgs);
     }
@@ -1202,14 +1202,14 @@ bif_timer_ref_destructor(Binary *unused)
 static ERTS_INLINE void
 btm_clear_magic_binary(ErtsBifTimer *tmr)
 {
-    erts_smp_atomic_t *aptr = mbin_to_btmref__(tmr->btm.mbin);
+    erts_atomic_t *aptr = mbin_to_btmref__(tmr->btm.mbin);
     Uint32 roflgs = tmr->type.head.roflgs;
 #ifdef ERTS_HLT_DEBUG
-    erts_aint_t tval = erts_smp_atomic_xchg_nob(aptr,
+    erts_aint_t tval = erts_atomic_xchg_nob(aptr,
                                                 (erts_aint_t) NULL);
     ERTS_HLT_ASSERT(tval == (erts_aint_t) tmr);
 #else
-    erts_smp_atomic_set_nob(aptr, (erts_aint_t) NULL);
+    erts_atomic_set_nob(aptr, (erts_aint_t) NULL);
 #endif
     if (roflgs & ERTS_TMR_ROFLG_HLT)
         hl_timer_dec_refc(&tmr->type.hlt, roflgs);
@@ -1229,7 +1229,7 @@ bif_timer_timeout(ErtsHLTimerService *srv,
     ERTS_HLT_ASSERT(tmr->type.head.roflgs == roflgs);    
     ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR);
 
-    state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state,
+    state = erts_atomic32_cmpxchg_acqb(&tmr->btm.state,
 					   ERTS_TMR_STATE_TIMED_OUT,
 					   ERTS_TMR_STATE_ACTIVE);
 
@@ -1263,7 +1263,7 @@ bif_timer_timeout(ErtsHLTimerService *srv,
                 tmr->btm.bp = NULL;
                 erts_queue_message(proc, 0, mp, tmr->btm.message,
                                    am_clock_service);
-                erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM);
+                erts_proc_lock(proc, ERTS_PROC_LOCK_BTM);
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
                 if (tmr->btm.proc_list.next) {
                     proc_btm_list_delete(&proc->bif_timers, tmr);
@@ -1276,7 +1276,7 @@ bif_timer_timeout(ErtsHLTimerService *srv,
                     dec_refc = 1;
                 }
 #endif
-                erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
+                erts_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
                 if (dec_refc)
                     timer_pre_dec_refc((ErtsTimer *) tmr);
             }
@@ -1414,7 +1414,7 @@ create_hl_timer(ErtsSchedulerData *esdp,
     }
 
     tmr->head.roflgs = roflgs;
-    erts_smp_atomic32_init_nob(&tmr->head.refc, refc);
+    erts_atomic32_init_nob(&tmr->head.refc, refc);
 
     if (!srv->next_timeout
 	|| tmr->timeout < srv->next_timeout->timeout) {
@@ -1971,7 +1971,7 @@ setup_bif_timer(Process *c_p, int twheel, ErtsMonotonicTime timeout_pos,
 #else
 	    proc_btm_rbt_insert(&proc->bif_timers, tmr);
 #endif
-	    erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
+	    erts_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
             tmr->type.head.receiver.proc = proc;
 	}
     }
@@ -1992,7 +1992,7 @@ cancel_bif_timer(ErtsBifTimer *tmr)
     Uint32 roflgs;
     int res;
 
-    state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state,
+    state = erts_atomic32_cmpxchg_acqb(&tmr->btm.state,
 					   ERTS_TMR_STATE_CANCELED,
 					   ERTS_TMR_STATE_ACTIVE);
     if (state != ERTS_TMR_STATE_ACTIVE)
@@ -2014,7 +2014,7 @@ cancel_bif_timer(ErtsBifTimer *tmr)
         proc = tmr->type.head.receiver.proc;
 	ERTS_HLT_ASSERT(!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME));
 
-	erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM);
+	erts_proc_lock(proc, ERTS_PROC_LOCK_BTM);
 	/*
 	 * If process is exiting, let it clean up
 	 * the btm tree by itself (it may be in
@@ -2033,7 +2033,7 @@ cancel_bif_timer(ErtsBifTimer *tmr)
 	    res = 1;
 	}
 #endif
-	erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
+	erts_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
     }
 
     return res;
@@ -2056,7 +2056,7 @@ access_btm(ErtsBifTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel)
                : erts_tweel_read_timeout(&tmr->type.twt.u.tw_tmr));
 
     if (!cancel) {
-        erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->btm.state);
+        erts_aint32_t state = erts_atomic32_read_acqb(&tmr->btm.state);
         if (state == ERTS_TMR_STATE_ACTIVE)
             return get_time_left(esdp, timeout);
         return -1;
@@ -2150,7 +2150,7 @@ send_async_info(Process *proc, ErtsProcLocks initial_locks,
 
     locks &= ~initial_locks;
     if (locks)
-        erts_smp_proc_unlock(proc, locks);
+        erts_proc_unlock(proc, locks);
 
     return am_ok;
 }
@@ -2236,7 +2236,7 @@ send_sync_info(Process *proc, ErtsProcLocks initial_locks,
 
     locks &= ~initial_locks;
     if (locks)
-        erts_smp_proc_unlock(proc, locks);
+        erts_proc_unlock(proc, locks);
 
     return am_ok;
 }
@@ -2350,9 +2350,9 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
      * Check if the timer is aimed at current
      * process...
      */
-    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_BTM);
+    erts_proc_lock(c_p, ERTS_PROC_LOCK_BTM);
     tmr = proc_btm_rbt_lookup(c_p->bif_timers, trefn);
-    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_BTM);
+    erts_proc_unlock(c_p, ERTS_PROC_LOCK_BTM);
     if (!tmr)
 	return 0;
 
@@ -2393,7 +2393,7 @@ no_timer_result(Process *c_p, Eterm tref, int cancel, int async, int info)
     erts_queue_message(c_p, locks, mp, msg, am_clock_service);
     locks &= ~ERTS_PROC_LOCK_MAIN;
     if (locks)
-        erts_smp_proc_unlock(c_p, locks);
+        erts_proc_unlock(c_p, locks);
     return am_ok;
 }
 
@@ -2469,7 +2469,7 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info)
 	    req->rrefn[1] = rrefn[1];
 	    req->rrefn[2] = rrefn[2];
 
-	    erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	    erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 
 	    if (ERTS_PROC_PENDING_EXIT(c_p))
 		ERTS_VBUMP_ALL_REDS(c_p);
@@ -2487,10 +2487,10 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info)
 		 *       otherwise, next receive will *not* work
 		 *       as expected!
 		 */
-		ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
+		ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
 		c_p->msg.save = c_p->msg.last;
 	    }
-	    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 
 	    ERTS_BIF_PREP_TRAP1(ret, erts_await_result, c_p, rref);
 	}
@@ -2580,7 +2580,7 @@ exit_cancel_bif_timer(ErtsBifTimer *tmr, void *vesdp)
     erts_aint_t state;
     int is_hlt;
 
-    state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state,
+    state = erts_atomic32_cmpxchg_acqb(&tmr->btm.state,
 					   ERTS_TMR_STATE_CANCELED,
 					   ERTS_TMR_STATE_ACTIVE);
 
@@ -2966,7 +2966,7 @@ set_proc_timer_common(Process *c_p, ErtsSchedulerData *esdp, Sint64 tmo,
                                     ERTS_TMR_PROC, (void *) c_p,
                                     c_p->common.id, THE_NON_VALUE,
                                     NULL, NULL, NULL);
-	erts_smp_atomic_set_relb(&c_p->common.timer, (erts_aint_t) tmr);
+	erts_atomic_set_relb(&c_p->common.timer, (erts_aint_t) tmr);
     }
 }
 
@@ -2977,7 +2977,7 @@ erts_set_proc_timer_term(Process *c_p, Eterm etmo)
     ErtsMonotonicTime tmo, timeout_pos;
     int short_time, tres;
 
-    ERTS_HLT_ASSERT(erts_smp_atomic_read_nob(&c_p->common.timer)
+    ERTS_HLT_ASSERT(erts_atomic_read_nob(&c_p->common.timer)
 		    == ERTS_PTMR_NONE);
 
     tres = parse_timeout_pos(esdp, etmo, &tmo, 0,
@@ -2997,7 +2997,7 @@ erts_set_proc_timer_uword(Process *c_p, UWord tmo)
 {
     ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
 
-    ERTS_HLT_ASSERT(erts_smp_atomic_read_nob(&c_p->common.timer)
+    ERTS_HLT_ASSERT(erts_atomic_read_nob(&c_p->common.timer)
 		    == ERTS_PTMR_NONE);
 
 #ifndef ARCH_32
@@ -3020,13 +3020,13 @@ void
 erts_cancel_proc_timer(Process *c_p)
 {
     erts_aint_t tval;
-    tval = erts_smp_atomic_xchg_acqb(&c_p->common.timer,
+    tval = erts_atomic_xchg_acqb(&c_p->common.timer,
 				     ERTS_PTMR_NONE);
     c_p->flags &= ~(F_INSLPQUEUE|F_TIMO);
     if (tval == ERTS_PTMR_NONE)
 	return;
     if (tval == ERTS_PTMR_TIMEDOUT) {
-	erts_smp_atomic_set_nob(&c_p->common.timer, ERTS_PTMR_NONE);
+	erts_atomic_set_nob(&c_p->common.timer, ERTS_PTMR_NONE);
 	return;
     }
     continue_cancel_ptimer(erts_proc_sched_data(c_p),
@@ -3041,7 +3041,7 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo)
     ErtsMonotonicTime timeout_pos;
     ErtsCreateTimerFunc create_timer;
 
-    if (erts_smp_atomic_read_nob(&c_prt->common.timer) != ERTS_PTMR_NONE)
+    if (erts_atomic_read_nob(&c_prt->common.timer) != ERTS_PTMR_NONE)
 	erts_cancel_port_timer(c_prt);
 
     check_canceled_queue(esdp, esdp->timer_service);
@@ -3054,14 +3054,14 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo)
     tmr = (void *) create_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT,
                                 (void *) c_prt, c_prt->common.id,
                                 THE_NON_VALUE, NULL, NULL, NULL);
-    erts_smp_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr);
+    erts_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr);
 }
 
 void
 erts_cancel_port_timer(Port *c_prt)
 {
     erts_aint_t tval;
-    tval = erts_smp_atomic_xchg_acqb(&c_prt->common.timer,
+    tval = erts_atomic_xchg_acqb(&c_prt->common.timer,
 				     ERTS_PTMR_NONE);
     if (tval == ERTS_PTMR_NONE)
 	return;
@@ -3069,7 +3069,7 @@ erts_cancel_port_timer(Port *c_prt)
 	while (!erts_port_task_is_scheduled(&c_prt->timeout_task))
 	    erts_thr_yield();
 	erts_port_task_abort(&c_prt->timeout_task);
-	erts_smp_atomic_set_nob(&c_prt->common.timer, ERTS_PTMR_NONE);
+	erts_atomic_set_nob(&c_prt->common.timer, ERTS_PTMR_NONE);
 	return;
     }
     continue_cancel_ptimer(erts_get_scheduler_data(),
@@ -3083,7 +3083,7 @@ erts_read_port_timer(Port *c_prt)
     erts_aint_t itmr;
     ErtsMonotonicTime timeout_pos;
 
-    itmr = erts_smp_atomic_read_acqb(&c_prt->common.timer);
+    itmr = erts_atomic_read_acqb(&c_prt->common.timer);
     if (itmr == ERTS_PTMR_NONE)
 	return (Sint64) -1;
     if (itmr == ERTS_PTMR_TIMEDOUT)
@@ -3220,7 +3220,7 @@ debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd)
     if (!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_BIF_TMR))
         return;
 #endif
-    if (erts_smp_atomic32_read_nob(&tmr->btm.state) == ERTS_TMR_STATE_ACTIVE) {
+    if (erts_atomic32_read_nob(&tmr->btm.state) == ERTS_TMR_STATE_ACTIVE) {
 	ErtsBTMForeachDebug *btmfd = (ErtsBTMForeachDebug *) vbtmfd;
         Eterm id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
                     ? tmr->type.head.receiver.name
@@ -3258,7 +3258,7 @@ erts_debug_bif_timer_foreach(void (*func)(Eterm,
     btmfd.func = func;
     btmfd.arg = arg;
 
-    if (!erts_smp_thr_progress_is_blocking())
+    if (!erts_thr_progress_is_blocking())
 	ERTS_INTERNAL_ERROR("Not blocking thread progress");
 
     for (six = 0; six < erts_no_schedulers; six++) {
@@ -3349,7 +3349,7 @@ erts_debug_callback_timer_foreach(void (*tclbk)(void *),
     dfct.func = func;
     dfct.arg = arg;
 
-    if (!erts_smp_thr_progress_is_blocking())
+    if (!erts_thr_progress_is_blocking())
 	ERTS_INTERNAL_ERROR("Not blocking thread progress");
 
     for (six = 0; six < erts_no_schedulers; six++) {
diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h
index 852bf88174..e6f5e8b67d 100644
--- a/erts/emulator/beam/erl_hl_timer.h
+++ b/erts/emulator/beam/erl_hl_timer.h
@@ -36,16 +36,16 @@ typedef struct ErtsHLTimerService_ ErtsHLTimerService;
 #define ERTS_PTMR_TIMEDOUT (ERTS_PTMR_NONE + ((erts_aint_t) 1))
 
 #define ERTS_PTMR_INIT(P) \
-    erts_smp_atomic_init_nob(&(P)->common.timer, ERTS_PTMR_NONE)
+    erts_atomic_init_nob(&(P)->common.timer, ERTS_PTMR_NONE)
 #define ERTS_PTMR_IS_SET(P) \
-    (ERTS_PTMR_NONE != erts_smp_atomic_read_nob(&(P)->common.timer))
+    (ERTS_PTMR_NONE != erts_atomic_read_nob(&(P)->common.timer))
 #define ERTS_PTMR_IS_TIMED_OUT(P) \
-    (ERTS_PTMR_TIMEDOUT == erts_smp_atomic_read_nob(&(P)->common.timer))
+    (ERTS_PTMR_TIMEDOUT == erts_atomic_read_nob(&(P)->common.timer))
 
 #define ERTS_PTMR_CLEAR(P)					\
     do {							\
 	ASSERT(ERTS_PTMR_IS_TIMED_OUT((P)));			\
-	erts_smp_atomic_set_nob(&(P)->common.timer,		\
+	erts_atomic_set_nob(&(P)->common.timer,		\
 				ERTS_PTMR_NONE);		\
     } while (0)
 
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index fe7f09e5fa..34affaa015 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -148,7 +148,7 @@ static void erl_init(int ncpu,
 
 static erts_atomic_t exiting;
 
-erts_smp_atomic32_t erts_writing_erl_crash_dump;
+erts_atomic32_t erts_writing_erl_crash_dump;
 erts_tsd_key_t erts_is_crash_dumping_key;
 int erts_initialized = 0;
 
@@ -170,7 +170,7 @@ int erts_backtrace_depth;	/* How many functions to show in a backtrace
 				 * in error codes.
 				 */
 
-erts_smp_atomic32_t erts_max_gen_gcs;
+erts_atomic32_t erts_max_gen_gcs;
 
 Eterm erts_error_logger_warnings; /* What to map warning logs to, am_error, 
 				     am_info or am_warning, am_error is 
@@ -395,7 +395,7 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char**
      */
 
     erts_init_empty_process(&parent);
-    erts_smp_proc_lock(&parent, ERTS_PROC_LOCK_MAIN);
+    erts_proc_lock(&parent, ERTS_PROC_LOCK_MAIN);
     hp = HAlloc(&parent, argc*2 + 4);
     args = NIL;
     for (i = argc-1; i >= 0; i--) {
@@ -410,7 +410,7 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char**
 
     so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC;
     res = erl_create_process(&parent, start_mod, am_start, args, &so);
-    erts_smp_proc_unlock(&parent, ERTS_PROC_LOCK_MAIN);
+    erts_proc_unlock(&parent, ERTS_PROC_LOCK_MAIN);
     erts_cleanup_empty_process(&parent);
     return res;
 }
@@ -435,7 +435,7 @@ erl_system_process_otp(Eterm parent_pid, char* modname, int off_heap_msgq)
     if (off_heap_msgq)
         so.flags |= SPO_OFF_HEAP_MSGQ;
     res = erl_create_process(parent, start_mod, am_start, NIL, &so);
-    erts_smp_proc_unlock(parent, ERTS_PROC_LOCK_MAIN);
+    erts_proc_unlock(parent, ERTS_PROC_LOCK_MAIN);
     return res;
 }
 
@@ -774,10 +774,10 @@ early_init(int *argc, char **argv) /*
     erts_atomic_init_nob(&exiting, 0);
     erts_thr_progress_pre_init();
 
-    erts_smp_atomic32_init_nob(&erts_writing_erl_crash_dump, 0L);
+    erts_atomic32_init_nob(&erts_writing_erl_crash_dump, 0L);
     erts_tsd_key_create(&erts_is_crash_dumping_key,"erts_is_crash_dumping_key");
 
-    erts_smp_atomic32_init_nob(&erts_max_gen_gcs,
+    erts_atomic32_init_nob(&erts_max_gen_gcs,
 			       (erts_aint32_t) ((Uint16) -1));
 
     erts_pre_init_process();
@@ -1222,7 +1222,7 @@ erl_start(int argc, char **argv)
     envbufsz = sizeof(envbuf);
     if (erts_sys_getenv_raw("ERL_FULLSWEEP_AFTER", envbuf, &envbufsz) == 0) {
 	Uint16 max_gen_gcs = atoi(envbuf);
-	erts_smp_atomic32_set_nob(&erts_max_gen_gcs,
+	erts_atomic32_set_nob(&erts_max_gen_gcs,
 				  (erts_aint32_t) max_gen_gcs);
     }
 
diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h
index ca5a72b183..cad9bf74c7 100644
--- a/erts/emulator/beam/erl_lock_check.h
+++ b/erts/emulator/beam/erl_lock_check.h
@@ -116,9 +116,7 @@ int erts_lc_is_emu_thr(void);
 
 #define ERTS_LC_ASSERT(A) \
     ((void) (((A) || ERTS_SOMEONE_IS_CRASH_DUMPING) ? 1 : erts_lc_assert_failed(__FILE__, __LINE__, #A)))
-#define ERTS_SMP_LC_ASSERT(A) ERTS_LC_ASSERT(A)
 #else /* #ifdef ERTS_ENABLE_LOCK_CHECK */
-#define ERTS_SMP_LC_ASSERT(A) ((void) 1)
 #define ERTS_LC_ASSERT(A) ((void) 1)
 #endif /* #ifdef ERTS_ENABLE_LOCK_CHECK */
 
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index d4e9ff7d18..3418a7f4df 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -170,7 +170,7 @@ erts_cleanup_offheap(ErlOffHeap *offheap)
             erts_bin_release(u.pb->val);
 	    break;
 	case FUN_SUBTAG:
-	    if (erts_smp_refc_dectest(&u.fun->fe->refc, 0) == 0) {
+	    if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) {
 		erts_erase_fun_entry(u.fun->fe);		    
 	    }
 	    break;
@@ -267,7 +267,7 @@ erts_queue_dist_message(Process *rcvr,
 #endif
     erts_aint_t state;
 
-    ERTS_SMP_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
+    ERTS_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
 
     mp = erts_alloc_message(0, NULL);
     mp->data.dist_ext = dist_ext;
@@ -282,22 +282,22 @@ erts_queue_dist_message(Process *rcvr,
 	ERL_MESSAGE_TOKEN(mp) = token;
 
     if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) {
-	if (erts_smp_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
+	if (erts_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
 	    ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ;
             ErtsProcLocks unlocks =
                 rcvr_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
 	    if (unlocks) {
-		erts_smp_proc_unlock(rcvr, unlocks);
+		erts_proc_unlock(rcvr, unlocks);
 		need_locks |= unlocks;
 	    }
-	    erts_smp_proc_lock(rcvr, need_locks);
+	    erts_proc_lock(rcvr, need_locks);
 	}
     }
 
-    state = erts_smp_atomic32_read_acqb(&rcvr->state);
+    state = erts_atomic32_read_acqb(&rcvr->state);
     if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
 	if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
-	    erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
+	    erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
 	/* Drop message if receiver is exiting or has a pending exit ... */
 	erts_cleanup_messages(mp);
     }
@@ -308,7 +308,7 @@ erts_queue_dist_message(Process *rcvr,
 
 	/* Ahh... need to decode it in order to trace it... */
 	if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
-	    erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
+	    erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
 	if (!erts_decode_dist_message(rcvr, rcvr_locks, mp, 0))
 	    erts_free_message(mp);
 	else {
@@ -357,7 +357,7 @@ erts_queue_dist_message(Process *rcvr,
 	LINK_MESSAGE(rcvr, mp, &mp->next, 1);
 
 	if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
-	    erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
+	    erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
 
 	erts_proc_notify_new_message(rcvr,
 				     rcvr_locks
@@ -386,39 +386,39 @@ queue_messages(Process* receiver,
            is_tuple(ERL_MESSAGE_TOKEN(first)));
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(receiver) < ERTS_PROC_LOCK_MSGQ ||
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(receiver) < ERTS_PROC_LOCK_MSGQ ||
                        receiver_locks == erts_proc_lc_my_proc_locks(receiver));
 #endif
 
     if (!(receiver_locks & ERTS_PROC_LOCK_MSGQ)) {
-	if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
+	if (erts_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
             ErtsProcLocks need_locks;
 
 	    if (receiver_state)
 		state = *receiver_state;
 	    else
-		state = erts_smp_atomic32_read_nob(&receiver->state);
+		state = erts_atomic32_read_nob(&receiver->state);
 	    if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
 		goto exiting;
 
             need_locks = receiver_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
 	    if (need_locks) {
-		erts_smp_proc_unlock(receiver, need_locks);
+		erts_proc_unlock(receiver, need_locks);
 	    }
             need_locks |= ERTS_PROC_LOCK_MSGQ;
-	    erts_smp_proc_lock(receiver, need_locks);
+	    erts_proc_lock(receiver, need_locks);
 	}
 	locked_msgq = 1;
     }
 
 
-    state = erts_smp_atomic32_read_nob(&receiver->state);
+    state = erts_atomic32_read_nob(&receiver->state);
 
     if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
     exiting:
 	/* Drop message if receiver is exiting or has a pending exit... */
 	if (locked_msgq)
-	    erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
+	    erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
 	erts_cleanup_messages(first);
 	return 0;
     }
@@ -434,7 +434,7 @@ queue_messages(Process* receiver,
 	 * the root set when garbage collecting.
 	 */
 	res += receiver->msg_inq.len;
-	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver);
+	ERTS_MSGQ_MV_INQ2PRIVQ(receiver);
         LINK_MESSAGE_PRIVQ(receiver, first, last, len);
     }
     else
@@ -475,7 +475,7 @@ queue_messages(Process* receiver,
 
     }
     if (locked_msgq) {
-	erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
+	erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
     }
 
     erts_proc_notify_new_message(receiver, receiver_locks);
@@ -599,7 +599,7 @@ erts_try_alloc_message_on_heap(Process *pp,
 	     */
 	    if (locked_main) {
 		*plp &= ~ERTS_PROC_LOCK_MAIN;
-		erts_smp_proc_unlock(pp, ERTS_PROC_LOCK_MAIN);
+		erts_proc_unlock(pp, ERTS_PROC_LOCK_MAIN);
 	    }
 	    goto in_message_fragment;
 	}
@@ -611,9 +611,9 @@ erts_try_alloc_message_on_heap(Process *pp,
 	mp->data.attached = NULL;
 	*on_heap_p = !0;
     }
-    else if (pp && erts_smp_proc_trylock(pp, ERTS_PROC_LOCK_MAIN) == 0) {
+    else if (pp && erts_proc_trylock(pp, ERTS_PROC_LOCK_MAIN) == 0) {
 	locked_main = 1;
-	*psp = erts_smp_atomic32_read_nob(&pp->state);
+	*psp = erts_atomic32_read_nob(&pp->state);
 	*plp |= ERTS_PROC_LOCK_MAIN;
 	goto try_on_heap;
     }
@@ -685,7 +685,7 @@ erts_send_message(Process* sender,
     }
 #endif
 
-    receiver_state = erts_smp_atomic32_read_nob(&receiver->state);
+    receiver_state = erts_atomic32_read_nob(&receiver->state);
 
     if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) {
         Eterm* hp;
@@ -934,7 +934,7 @@ erts_move_messages_off_heap(Process *c_p)
 
     reds += c_p->msg.len / 10;
 
-    ASSERT(erts_smp_atomic32_read_nob(&c_p->state)
+    ASSERT(erts_atomic32_read_nob(&c_p->state)
 	   & ERTS_PSFLG_OFF_HEAP_MSGQ);
     ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG);
 
@@ -999,9 +999,9 @@ erts_complete_off_heap_message_queue_change(Process *c_p)
 {
     int reds = 1;
 
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
     ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG);
-    ASSERT(erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ);
+    ASSERT(erts_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ);
 
     /*
      * This job was first initiated when the process changed to off heap
@@ -1013,13 +1013,13 @@ erts_complete_off_heap_message_queue_change(Process *c_p)
      */
 
     if (!(c_p->flags & F_OFF_HEAP_MSGQ))
-	erts_smp_atomic32_read_band_nob(&c_p->state,
+	erts_atomic32_read_band_nob(&c_p->state,
 					~ERTS_PSFLG_OFF_HEAP_MSGQ);
     else {
 	reds += 2;
-	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
-	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
-	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
+	erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
+	ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
+	erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
 	reds += erts_move_messages_off_heap(c_p);
     }
     c_p->flags &= ~F_OFF_HEAP_MSGQ_CHNG;
@@ -1056,16 +1056,16 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state)
 
 #ifdef DEBUG
     if (c_p->flags & F_OFF_HEAP_MSGQ) {
-	ASSERT(erts_smp_atomic32_read_nob(&c_p->state)
+	ASSERT(erts_atomic32_read_nob(&c_p->state)
 	       & ERTS_PSFLG_OFF_HEAP_MSGQ);
     }
     else {
 	if (c_p->flags & F_OFF_HEAP_MSGQ_CHNG) {
-	    ASSERT(erts_smp_atomic32_read_nob(&c_p->state)
+	    ASSERT(erts_atomic32_read_nob(&c_p->state)
 		   & ERTS_PSFLG_OFF_HEAP_MSGQ);
 	}
 	else {
-	    ASSERT(!(erts_smp_atomic32_read_nob(&c_p->state)
+	    ASSERT(!(erts_atomic32_read_nob(&c_p->state)
 		     & ERTS_PSFLG_OFF_HEAP_MSGQ));
 	}
     }
@@ -1082,7 +1082,7 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state)
 	case am_on_heap:
 	    c_p->flags |= F_ON_HEAP_MSGQ;
 	    c_p->flags &= ~F_OFF_HEAP_MSGQ;
-	    erts_smp_atomic32_read_bor_nob(&c_p->state,
+	    erts_atomic32_read_bor_nob(&c_p->state,
 					   ERTS_PSFLG_ON_HEAP_MSGQ);
 	    /*
 	     * We are not allowed to clear ERTS_PSFLG_OFF_HEAP_MSGQ
@@ -1091,7 +1091,7 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state)
 	     */
 	    if (!(c_p->flags & F_OFF_HEAP_MSGQ_CHNG)) {
 		/* Safe to clear ERTS_PSFLG_OFF_HEAP_MSGQ... */
-		erts_smp_atomic32_read_band_nob(&c_p->state,
+		erts_atomic32_read_band_nob(&c_p->state,
 						~ERTS_PSFLG_OFF_HEAP_MSGQ);
 	    }
 	    break;
@@ -1109,7 +1109,7 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state)
 	    break;
 	case am_off_heap:
 	    c_p->flags &= ~F_ON_HEAP_MSGQ;
-	    erts_smp_atomic32_read_band_nob(&c_p->state,
+	    erts_atomic32_read_band_nob(&c_p->state,
 					    ~ERTS_PSFLG_ON_HEAP_MSGQ);
 	    goto change_to_off_heap;
 	default:
@@ -1144,7 +1144,7 @@ change_to_off_heap:
 	 * change has completed, GC does not need to inspect
 	 * the message queue at all.
 	 */
-	erts_smp_atomic32_read_bor_nob(&c_p->state,
+	erts_atomic32_read_bor_nob(&c_p->state,
 				       ERTS_PSFLG_OFF_HEAP_MSGQ);
 	c_p->flags |= F_OFF_HEAP_MSGQ_CHNG;
 	cohmq = erts_alloc(ERTS_ALC_T_MSGQ_CHNG,
@@ -1425,7 +1425,7 @@ erts_factory_message_create(ErtsHeapFactory* factory,
     int on_heap;
     erts_aint32_t state;
 
-    state = proc ? erts_smp_atomic32_read_nob(&proc->state) : 0;
+    state = proc ? erts_atomic32_read_nob(&proc->state) : 0;
 
     if (state & ERTS_PSFLG_OFF_HEAP_MSGQ) {
 	msgp = erts_alloc_message(sz, &hp);
@@ -1440,7 +1440,7 @@ erts_factory_message_create(ErtsHeapFactory* factory,
     }
 
     if (on_heap) {
-	ERTS_SMP_ASSERT(*proc_locksp & ERTS_PROC_LOCK_MAIN);
+	ERTS_ASSERT(*proc_locksp & ERTS_PROC_LOCK_MAIN);
 	ASSERT(ohp == &proc->off_heap);
 	factory->mode = FACTORY_HALLOC;
 	factory->p = proc;
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 1f95ffaa5a..9c8cf84e43 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -216,7 +216,7 @@ typedef struct erl_trace_message_queue__ {
 #define LINK_MESSAGE(p, first_msg, last_msg, len)       \
     LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg_inq)
 
-#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p)                   \
+#define ERTS_MSGQ_MV_INQ2PRIVQ(p)                   \
     do {                                                \
         if (p->msg_inq.first) {                         \
             *p->msg.last = p->msg_inq.first;            \
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
index 3994800ba7..67c552b364 100644
--- a/erts/emulator/beam/erl_monitors.c
+++ b/erts/emulator/beam/erl_monitors.c
@@ -54,7 +54,7 @@
 #define DIR_RIGHT 1
 #define DIR_END 2 
 
-static erts_smp_atomic_t tot_link_lh_size;
+static erts_atomic_t tot_link_lh_size;
 
 /* Implements the sort order in monitor trees, which is different from 
    the ordinary term order.
@@ -123,7 +123,7 @@ do {								\
 	    (*((Hp)++)) = boxed_val((From))[i__];		\
 	if (is_external((To))) {				\
 	    external_thing_ptr((To))->next = NULL;		\
-	    erts_smp_refc_inc(&(external_thing_ptr((To))->node->refc), 2);\
+	    erts_refc_inc(&(external_thing_ptr((To))->node->refc), 2);\
 	}							\
     }								\
 } while (0)
@@ -145,7 +145,7 @@ static ErtsMonitor *create_monitor(Uint type, Eterm ref, UWord entity, Eterm nam
      } else {
 	 n = (ErtsMonitor *) erts_alloc(ERTS_ALC_T_MONITOR_LH,
 					mon_size*sizeof(Uint));
-	 erts_smp_atomic_add_nob(&tot_link_lh_size, mon_size*sizeof(Uint));
+	 erts_atomic_add_nob(&tot_link_lh_size, mon_size*sizeof(Uint));
      } 
      hp = n->heap;
 
@@ -179,7 +179,7 @@ static ErtsLink *create_link(Uint type, Eterm pid)
      } else {
 	 n = (ErtsLink *) erts_alloc(ERTS_ALC_T_NLINK_LH,
 				     lnk_size*sizeof(Uint));
-	 erts_smp_atomic_add_nob(&tot_link_lh_size, lnk_size*sizeof(Uint));
+	 erts_atomic_add_nob(&tot_link_lh_size, lnk_size*sizeof(Uint));
      } 
      hp = n->heap;
 
@@ -214,13 +214,13 @@ static ErtsSuspendMonitor *create_suspend_monitor(Eterm pid)
 void
 erts_init_monitors(void)
 {
-    erts_smp_atomic_init_nob(&tot_link_lh_size, 0);
+    erts_atomic_init_nob(&tot_link_lh_size, 0);
 }
 
 Uint
 erts_tot_link_lh_size(void)
 {
-    return (Uint) erts_smp_atomic_read_nob(&tot_link_lh_size);
+    return (Uint) erts_atomic_read_nob(&tot_link_lh_size);
 }
 
 void erts_destroy_monitor(ErtsMonitor *mon)
@@ -245,7 +245,7 @@ void erts_destroy_monitor(ErtsMonitor *mon)
 	erts_free(ERTS_ALC_T_MONITOR_SH, (void *) mon);
     } else {
 	erts_free(ERTS_ALC_T_MONITOR_LH, (void *) mon);
-	erts_smp_atomic_add_nob(&tot_link_lh_size, -1*mon_size*sizeof(Uint));
+	erts_atomic_add_nob(&tot_link_lh_size, -1*mon_size*sizeof(Uint));
     }
 }
     
@@ -267,7 +267,7 @@ void erts_destroy_link(ErtsLink *lnk)
 	erts_free(ERTS_ALC_T_NLINK_SH, (void *) lnk);
     } else {
 	erts_free(ERTS_ALC_T_NLINK_LH, (void *) lnk);
-	erts_smp_atomic_add_nob(&tot_link_lh_size, -1*lnk_size*sizeof(Uint));
+	erts_atomic_add_nob(&tot_link_lh_size, -1*lnk_size*sizeof(Uint));
     }
 }
 
@@ -985,13 +985,13 @@ Eterm erts_debug_dump_monitors_1(BIF_ALIST_1)
     DistEntry *dep;
     rp = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, pid, ERTS_PROC_LOCK_LINK);
     if (!rp) {
-	ERTS_SMP_ASSERT_IS_NOT_EXITING(p);
+	ERTS_ASSERT_IS_NOT_EXITING(p);
 	if (is_atom(pid) && is_node_name_atom(pid) &&
 	    (dep = erts_find_dist_entry(pid)) != NULL) {
 	    erts_printf("Dumping dist monitors-------------------\n");
-	    erts_smp_de_links_lock(dep);
+	    erts_de_links_lock(dep);
 	    erts_dump_monitors(dep->monitors,0);
-	    erts_smp_de_links_unlock(dep);
+	    erts_de_links_unlock(dep);
 	    erts_printf("Monitors dumped-------------------------\n");
 	    erts_deref_dist_entry(dep);
 	    BIF_RET(am_true);
@@ -1002,7 +1002,7 @@ Eterm erts_debug_dump_monitors_1(BIF_ALIST_1)
 	erts_printf("Dumping pid monitors--------------------\n");
 	erts_dump_monitors(ERTS_P_MONITORS(rp),0);
 	erts_printf("Monitors dumped-------------------------\n");
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 	BIF_RET(am_true);
     }
 }
@@ -1030,13 +1030,13 @@ Eterm erts_debug_dump_links_1(BIF_ALIST_1)
     } else {
 	rp = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, pid, ERTS_PROC_LOCK_LINK);
 	if (!rp) {
-	    ERTS_SMP_ASSERT_IS_NOT_EXITING(p);
+	    ERTS_ASSERT_IS_NOT_EXITING(p);
 	    if (is_atom(pid) && is_node_name_atom(pid) &&
 		(dep = erts_find_dist_entry(pid)) != NULL) {
 		erts_printf("Dumping dist links----------------------\n");
-		erts_smp_de_links_lock(dep);
+		erts_de_links_lock(dep);
 		erts_dump_links(dep->nlinks,0);
-		erts_smp_de_links_unlock(dep);
+		erts_de_links_unlock(dep);
 		erts_printf("Links dumped----------------------------\n");
 		erts_deref_dist_entry(dep);
 		BIF_RET(am_true);
@@ -1048,7 +1048,7 @@ Eterm erts_debug_dump_links_1(BIF_ALIST_1)
 	    erts_printf("Dumping pid links-----------------------\n");
 	    erts_dump_links(ERTS_P_LINKS(rp), 0);
 	    erts_printf("Links dumped----------------------------\n");
-	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+	    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 	    BIF_RET(am_true);
 	}
     }
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index 957358bcf5..d659842b7e 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -202,7 +202,7 @@ typedef struct {
     Eterm ref;
     Eterm ref_heap[ERTS_REF_THING_SIZE];
     Uint req_sched;
-    erts_smp_atomic32_t refc;
+    erts_atomic32_t refc;
 } ErtsMSAccReq;
 
 static ErtsMsAcc* get_msacc(void) {
@@ -253,7 +253,7 @@ static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) {
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
 
     if (rp_locks)
-	erts_smp_proc_unlock(rp, rp_locks);
+	erts_proc_unlock(rp, rp_locks);
 
 }
 
@@ -289,7 +289,7 @@ reply_msacc(void *vmsaccrp)
 
     erts_proc_dec_refc(msaccrp->proc);
 
-    if (erts_smp_atomic32_dec_read_nob(&msaccrp->refc) == 0)
+    if (erts_atomic32_dec_read_nob(&msaccrp->refc) == 0)
       erts_free(ERTS_ALC_T_MSACC, vmsaccrp);
 }
 
@@ -359,7 +359,7 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads)
     *threads = erts_no_schedulers;
     *threads += 1; /* aux thread */
 
-    erts_smp_atomic32_init_nob(&msaccrp->refc,(erts_aint32_t)*threads);
+    erts_atomic32_init_nob(&msaccrp->refc,(erts_aint32_t)*threads);
 
     erts_proc_add_refc(c_p, *threads);
 
diff --git a/erts/emulator/beam/erl_nfunc_sched.c b/erts/emulator/beam/erl_nfunc_sched.c
index 1bebc1eda4..f97e86bf95 100644
--- a/erts/emulator/beam/erl_nfunc_sched.c
+++ b/erts/emulator/beam/erl_nfunc_sched.c
@@ -113,7 +113,7 @@ erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc,
     NifExport* nep;
     int i;
 
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
 		       & ERTS_PROC_LOCK_MAIN);
 
     if (dirty_shadow_proc) {
diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h
index 55a3a6dbf6..69008084df 100644
--- a/erts/emulator/beam/erl_nfunc_sched.h
+++ b/erts/emulator/beam/erl_nfunc_sched.h
@@ -144,9 +144,9 @@ ERTS_GLB_INLINE void
 erts_nif_export_restore(Process *c_p, NifExport *ep, Eterm result)
 {
     ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
-    ERTS_SMP_LC_ASSERT(!(c_p->static_flags
+    ERTS_LC_ASSERT(!(c_p->static_flags
 			 & ERTS_STC_FLG_SHADOW_PROC));
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
 		       & ERTS_PROC_LOCK_MAIN);
 
     c_p->current = ep->current;
@@ -235,7 +235,7 @@ erts_flush_dirty_shadow_proc(Process *sproc)
     Process *c_p = sproc->next;
 
     ASSERT(sproc->common.id == c_p->common.id);
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
 		       & ERTS_PROC_LOCK_MAIN);
 
     ASSERT(c_p->stop == sproc->stop);
@@ -283,7 +283,7 @@ erts_cache_dirty_shadow_proc(Process *sproc)
     Process *c_p = sproc->next;
     ASSERT(c_p);
     ASSERT(sproc->common.id == c_p->common.id);
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
 		       & ERTS_PROC_LOCK_MAIN);
 
     sproc->htop = c_p->htop;
@@ -311,7 +311,7 @@ erts_make_dirty_shadow_proc(ErtsSchedulerData *esdp, Process *c_p)
     sproc = esdp->dirty_shadow_process;
     ASSERT(sproc);
     ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
-    ASSERT(erts_smp_atomic32_read_nob(&sproc->state)
+    ASSERT(erts_atomic32_read_nob(&sproc->state)
 	   == (ERTS_PSFLG_ACTIVE
 	       | ERTS_PSFLG_DIRTY_RUNNING
 	       | ERTS_PSFLG_PROXY));
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index b586a4636a..af25971130 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -138,7 +138,7 @@ execution_state(ErlNifEnv *env, Process **c_pp, int *schedp)
 	    Process *c_p = env->proc;
 
 	    if (!(c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC)) {
-		ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
+		ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
 				   & ERTS_PROC_LOCK_MAIN);
 	    }
 	    else {
@@ -220,7 +220,7 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif,
 	ASSERT(esdp);
 
 	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-	    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
+	    erts_aint32_t state = erts_atomic32_read_nob(&p->state);
 
 	    ASSERT(p->scheduler_data == esdp);
 	    ASSERT((state & (ERTS_PSFLG_RUNNING
@@ -287,7 +287,7 @@ schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp,
     else
 	dirty_shadow_proc = env->proc;
 
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
 
     ep = erts_nif_export_schedule(c_p, dirty_shadow_proc,
 				  c_p->current,
@@ -320,7 +320,7 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *
     ErlNifEnv env;
     ERL_NIF_TERM result;
 #ifdef DEBUG
-    erts_aint32_t state = erts_smp_atomic32_read_nob(&c_p->state);
+    erts_aint32_t state = erts_atomic32_read_nob(&c_p->state);
 
     ASSERT(nep == ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p));
 
@@ -343,14 +343,14 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *
 
     ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p)));
 
-    erts_smp_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC
+    erts_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC
 						   | ERTS_PSFLG_DIRTY_IO_PROC));
 
-    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+    erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
 
     result = (*dirty_nif)(&env, codemfa->arity, argv); /* Call dirty NIF */
 
-    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+    erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 
     ASSERT(env.proc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
     ASSERT(env.proc->next == c_p);
@@ -587,7 +587,7 @@ int erts_flush_trace_messages(Process *c_p, ErtsProcLocks c_p_locks)
     ErlTraceMessageQueue *msgq, **last_msgq;
     int reds = 0;
 
-    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_TRACE);
+    erts_proc_lock(c_p, ERTS_PROC_LOCK_TRACE);
 
     msgq = c_p->trace_msg_q;
 
@@ -606,7 +606,7 @@ int erts_flush_trace_messages(Process *c_p, ErtsProcLocks c_p_locks)
         msgq->first = NULL;
         msgq->last = &msgq->first;
         msgq->len = 0;
-        erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_TRACE);
+        erts_proc_unlock(c_p, ERTS_PROC_LOCK_TRACE);
 
         ASSERT(len != 0);
 
@@ -619,13 +619,13 @@ int erts_flush_trace_messages(Process *c_p, ErtsProcLocks c_p_locks)
             if (rp->common.id == c_p->common.id)
                 rp_locks &= ~c_p_locks;
             if (rp_locks)
-                erts_smp_proc_unlock(rp, rp_locks);
+                erts_proc_unlock(rp, rp_locks);
             reds += len;
         } else {
             erts_cleanup_messages(first);
         }
         reds += 1;
-        erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_TRACE);
+        erts_proc_lock(c_p, ERTS_PROC_LOCK_TRACE);
         msgq = msgq->next;
     } while (msgq);
 
@@ -642,7 +642,7 @@ int erts_flush_trace_messages(Process *c_p, ErtsProcLocks c_p_locks)
     }
 
 error:
-    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_TRACE);
+    erts_proc_unlock(c_p, ERTS_PROC_LOCK_TRACE);
 
     return reds;
 }
@@ -675,7 +675,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 		return 0;
 
 	    if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
-		erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+		erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 	    }
 	}
 
@@ -684,7 +684,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 			       ERTS_P2P_FLG_INC_REFC);
 	if (!rp) {
 	    if (c_p && (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC))
-		erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+		erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
 	    return 0;
 	}
     }
@@ -719,7 +719,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 	    full_cache_env(env);
 	}
 	else {
-	    erts_aint_t state = erts_smp_atomic32_read_nob(&rp->state);
+	    erts_aint_t state = erts_atomic32_read_nob(&rp->state);
 	    if (state & ERTS_PSFLG_OFF_HEAP_MSGQ) {
 		mp = erts_alloc_message(sz, &hp);
 		ohp = sz == 0 ? NULL : &mp->hfrag.off_heap;
@@ -755,7 +755,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
         Process *t_p = env->tracee;
 
 
-        erts_smp_proc_lock(t_p, ERTS_PROC_LOCK_TRACE);
+        erts_proc_lock(t_p, ERTS_PROC_LOCK_TRACE);
 
         msgq = t_p->trace_msg_q;
 
@@ -772,7 +772,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 #endif
         if (ERTS_FORCE_ENIF_SEND_DELAY() || msgq ||
             rp_locks & ERTS_PROC_LOCK_MSGQ ||
-            erts_smp_proc_trylock(rp, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
+            erts_proc_trylock(rp, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
 
             if (!msgq) {
                 msgq = erts_alloc(ERTS_ALC_T_TRACE_MSG_QUEUE,
@@ -786,18 +786,18 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
                 msgq->next = t_p->trace_msg_q;
                 t_p->trace_msg_q = msgq;
 
-                erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE);
+                erts_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE);
 
 		erts_schedule_flush_trace_messages(t_p, 0);
             } else {
                 msgq->len++;
                 *msgq->last = mp;
                 msgq->last = &mp->next;
-                erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE);
+                erts_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE);
             }
             goto done;
         } else {
-            erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE);
+            erts_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE);
             rp_locks &= ~ERTS_PROC_LOCK_TRACE;
             rp_locks |= ERTS_PROC_LOCK_MSGQ;
         }
@@ -810,9 +810,9 @@ done:
     if (c_p == rp)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
     if (rp_locks & ~lc_locks)
-	erts_smp_proc_unlock(rp, rp_locks & ~lc_locks);
+	erts_proc_unlock(rp, rp_locks & ~lc_locks);
     if (c_p && (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC))
-	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+	erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
     if (scheduler <= 0)
 	erts_proc_dec_refc(rp);
 
@@ -880,14 +880,14 @@ static Eterm call_whereis(ErlNifEnv *env, Eterm name)
             return 0;
 
         if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
-            erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+            erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
             unlock = 1;
         }
     }
     res = erts_whereis_name_to_id(c_p, name);
 
     if (unlock)
-        erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+        erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
 
     return res;
 }
@@ -2057,7 +2057,7 @@ ErlNifResourceType* open_resource_type(ErlNifEnv* env,
     ErlNifResourceFlags op = flags;
     Eterm module_am, name_am;
 
-    ASSERT(erts_smp_thr_progress_is_blocking());
+    ASSERT(erts_thr_progress_is_blocking());
     module_am = make_atom(env->mod_nif->mod->module);
     name_am = enif_make_atom(env, name_str);
 
@@ -2203,7 +2203,7 @@ static void destroy_one_monitor(ErtsMonitor* mon, void* context)
         is_exiting = 1;
     }
     if (rp) {
-        erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK);
+        erts_proc_lock(rp, ERTS_PROC_LOCK_LINK);
         if (ERTS_PROC_IS_EXITING(rp)) {
             is_exiting = 1;
         } else {
@@ -2211,7 +2211,7 @@ static void destroy_one_monitor(ErtsMonitor* mon, void* context)
             ASSERT(rmon);
             is_exiting = 0;
         }
-        erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+        erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
         if (ctx->scheduler <= 0)
             erts_proc_dec_refc(rp);
     }
@@ -2251,7 +2251,7 @@ static int nif_resource_dtor(Binary* bin)
         ErtsResourceMonitors* rm = resource->monitors;
 
         ASSERT(type->down);
-        erts_smp_mtx_lock(&rm->lock);
+        erts_mtx_lock(&rm->lock);
         ASSERT(erts_refc_read(&bin->intern.refc, 0) == 0);
         if (rm->root) {
             ASSERT(!rm->is_dying);
@@ -2273,11 +2273,11 @@ static int nif_resource_dtor(Binary* bin)
              */
             ASSERT(!rm->is_dying);
             rm->is_dying = 1;
-            erts_smp_mtx_unlock(&rm->lock);
+            erts_mtx_unlock(&rm->lock);
             return 0;
         }
-        erts_smp_mtx_unlock(&rm->lock);
-        erts_smp_mtx_destroy(&rm->lock);
+        erts_mtx_unlock(&rm->lock);
+        erts_mtx_destroy(&rm->lock);
     }
 
     if (type->dtor != NULL) {
@@ -2318,12 +2318,12 @@ void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref)
     ASSERT(rmp);
     ASSERT(resource->type->down);
 
-    erts_smp_mtx_lock(&rmp->lock);
+    erts_mtx_lock(&rmp->lock);
     rmon = erts_remove_monitor(&rmp->root, ref);
     if (!rmon) {
         int free_me = (--rmp->pending_failed_fire == 0) && rmp->is_dying;
         ASSERT(rmp->pending_failed_fire >= 0);
-        erts_smp_mtx_unlock(&rmp->lock);
+        erts_mtx_unlock(&rmp->lock);
 
         if (free_me) {
             ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) == 0);
@@ -2339,10 +2339,10 @@ void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref)
          * we avoid calling 'down' and just silently remove the monitor.
          * This can happen even for non smp as destructor calls may be scheduled.
          */
-        erts_smp_mtx_unlock(&rmp->lock);
+        erts_mtx_unlock(&rmp->lock);
     }
     else {
-        erts_smp_mtx_unlock(&rmp->lock);
+        erts_mtx_unlock(&rmp->lock);
 
         ASSERT(rmon->u.pid == pid);
         erts_ref_to_driver_monitor(ref, &nif_monitor);
@@ -2387,7 +2387,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t data_sz)
     erts_refc_inc(&resource->type->refc, 2);
     if (type->down) {
         resource->monitors = (ErtsResourceMonitors*) (resource->data + monitors_offs);
-        erts_smp_mtx_init(&resource->monitors->lock, "resource_monitors", NIL,
+        erts_mtx_init(&resource->monitors->lock, "resource_monitors", NIL,
             ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
         resource->monitors->root = NULL;
         resource->monitors->pending_failed_fire = 0;
@@ -2656,7 +2656,7 @@ schedule_dirty_nif(ErlNifEnv* env, int flags, NativeFunPtr fp,
 
     execution_state(env, &proc, NULL);
 
-    (void) erts_smp_atomic32_read_bset_nob(&proc->state,
+    (void) erts_atomic32_read_bset_nob(&proc->state,
 					   (ERTS_PSFLG_DIRTY_CPU_PROC
 					    | ERTS_PSFLG_DIRTY_IO_PROC),
 					   (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND
@@ -2694,7 +2694,7 @@ static_schedule_dirty_nif(ErlNifEnv* env, erts_aint32_t dirty_psflg,
     ASSERT(is_atom(mod) && is_atom(func));
     ASSERT(fp);
 
-    (void) erts_smp_atomic32_read_bset_nob(&proc->state,
+    (void) erts_atomic32_read_bset_nob(&proc->state,
 					   (ERTS_PSFLG_DIRTY_CPU_PROC
 					    | ERTS_PSFLG_DIRTY_IO_PROC),
 					   dirty_psflg);
@@ -2788,7 +2788,7 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags,
     if (scheduler <= 0) {
 	if (scheduler == 0)
 	    enif_make_badarg(env);
-	erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
+	erts_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
     }
 
     if (flags == 0)
@@ -2805,7 +2805,7 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags,
 	result = enif_make_badarg(env);
 
     if (scheduler < 0)
-	erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
+	erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
 
     return result;
 }
@@ -3157,9 +3157,9 @@ int enif_monitor_process(ErlNifEnv* env, void* obj, const ErlNifPid* target_pid,
 
     ref = erts_make_ref_in_buffer(tmp);
 
-    erts_smp_mtx_lock(&rsrc->monitors->lock);
-    erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK);
-    if (ERTS_PSFLG_FREE & erts_smp_atomic32_read_nob(&rp->state)) {
+    erts_mtx_lock(&rsrc->monitors->lock);
+    erts_proc_lock(rp, ERTS_PROC_LOCK_LINK);
+    if (ERTS_PSFLG_FREE & erts_atomic32_read_nob(&rp->state)) {
         retval = 1;
     }
     else {
@@ -3167,8 +3167,8 @@ int enif_monitor_process(ErlNifEnv* env, void* obj, const ErlNifPid* target_pid,
         erts_add_monitor(&ERTS_P_MONITORS(rp), MON_NIF_TARGET, ref, (UWord)rsrc, NIL);
         retval = 0;
     }
-    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-    erts_smp_mtx_unlock(&rsrc->monitors->lock);
+    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+    erts_mtx_unlock(&rsrc->monitors->lock);
 
     if (scheduler <= 0)
         erts_proc_dec_refc(rp);
@@ -3199,11 +3199,11 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit
 
     ref = erts_driver_monitor_to_ref(ref_heap, monitor);
 
-    erts_smp_mtx_lock(&rsrc->monitors->lock);
+    erts_mtx_lock(&rsrc->monitors->lock);
     mon = erts_remove_monitor(&rsrc->monitors->root, ref);
 
     if (mon == NULL) {
-        erts_smp_mtx_unlock(&rsrc->monitors->lock);
+        erts_mtx_unlock(&rsrc->monitors->lock);
         return 1;
     }
 
@@ -3219,7 +3219,7 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit
         is_exiting = 1;
     }
     else {
-        erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK);
+        erts_proc_lock(rp, ERTS_PROC_LOCK_LINK);
         if (ERTS_PROC_IS_EXITING(rp)) {
             is_exiting = 1;
         } else {
@@ -3227,7 +3227,7 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit
             ASSERT(rmon);
             is_exiting = 0;
         }
-        erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+        erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 
         if (scheduler <= 0)
             erts_proc_dec_refc(rp);
@@ -3235,7 +3235,7 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit
     if (is_exiting) {
         rsrc->monitors->pending_failed_fire++;
     }
-    erts_smp_mtx_unlock(&rsrc->monitors->lock);
+    erts_mtx_unlock(&rsrc->monitors->lock);
 
     if (rmon) {
         ASSERT(rmon->type == MON_NIF_TARGET);
@@ -3461,8 +3461,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     }
 
     /* Block system (is this the right place to do it?) */
-    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-    erts_smp_thr_progress_block();
+    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_thr_progress_block();
 
     /* Find calling module */
     ASSERT(BIF_P->current != NULL);
@@ -3673,8 +3673,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	erts_sys_ddll_free_error(&errdesc);
     }
 
-    erts_smp_thr_progress_unblock();
-    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_thr_progress_unblock();
+    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     erts_release_code_write_permission();
     erts_free(ERTS_ALC_T_TMP, lib_name);
 
@@ -3687,7 +3687,7 @@ erts_unload_nif(struct erl_module_nif* lib)
 {
     ErlNifResourceType* rt;
     ErlNifResourceType* next;
-    ASSERT(erts_smp_thr_progress_is_blocking());
+    ASSERT(erts_thr_progress_is_blocking());
     ASSERT(lib != NULL);
     ASSERT(lib->mod != NULL);
 
@@ -3759,8 +3759,8 @@ Eterm erts_nif_call_function(Process *p, Process *tracee,
             break;
     ASSERT(i < mod->entry.num_of_funcs);
     if (p)
-        ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN
-                           || erts_smp_thr_progress_is_blocking());
+        ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN
+                           || erts_thr_progress_is_blocking());
 #endif
     if (p) {
         /* This is almost a normal nif call like in beam_emu,
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index c12f231a3b..f8e9fec27a 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -32,8 +32,8 @@
 
 Hash erts_dist_table;
 Hash erts_node_table;
-erts_smp_rwmtx_t erts_dist_table_rwmtx;
-erts_smp_rwmtx_t erts_node_table_rwmtx;
+erts_rwmtx_t erts_dist_table_rwmtx;
+erts_rwmtx_t erts_node_table_rwmtx;
 
 DistEntry *erts_hidden_dist_entries;
 DistEntry *erts_visible_dist_entries;
@@ -87,8 +87,8 @@ dist_table_alloc(void *dep_tmpl)
 {
     Eterm sysname;
     DistEntry *dep;
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
-    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+    erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ;
 
     sysname = ((DistEntry *) dep_tmpl)->sysname;
     dep = (DistEntry *) erts_alloc(ERTS_ALC_T_DIST_ENTRY, sizeof(DistEntry));
@@ -96,8 +96,8 @@ dist_table_alloc(void *dep_tmpl)
     dist_entries++;
 
     dep->prev				= NULL;
-    erts_smp_refc_init(&dep->refc, -1);
-    erts_smp_rwmtx_init_opt(&dep->rwmtx, &rwmtx_opt, "dist_entry", sysname,
+    erts_refc_init(&dep->refc, -1);
+    erts_rwmtx_init_opt(&dep->rwmtx, &rwmtx_opt, "dist_entry", sysname,
         ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
     dep->sysname			= sysname;
     dep->cid				= NIL;
@@ -106,13 +106,13 @@ dist_table_alloc(void *dep_tmpl)
     dep->flags				= 0;
     dep->version			= 0;
 
-    erts_smp_mtx_init(&dep->lnk_mtx, "dist_entry_links", sysname,
+    erts_mtx_init(&dep->lnk_mtx, "dist_entry_links", sysname,
         ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
     dep->node_links			= NULL;
     dep->nlinks				= NULL;
     dep->monitors			= NULL;
 
-    erts_smp_mtx_init(&dep->qlock, "dist_entry_out_queue", sysname,
+    erts_mtx_init(&dep->qlock, "dist_entry_out_queue", sysname,
         ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
     dep->qflgs				= 0;
     dep->qsize				= 0;
@@ -123,7 +123,7 @@ dist_table_alloc(void *dep_tmpl)
     dep->finalized_out_queue.first	= NULL;
     dep->finalized_out_queue.last	= NULL;
 
-    erts_smp_atomic_init_nob(&dep->dist_cmd_scheduled, 0);
+    erts_atomic_init_nob(&dep->dist_cmd_scheduled, 0);
     erts_port_task_handle_init(&dep->dist_cmd);
     dep->send				= NULL;
     dep->cache				= NULL;
@@ -174,9 +174,9 @@ dist_table_free(void *vdep)
     erts_no_of_not_connected_dist_entries--;
 
     ASSERT(!dep->cache);
-    erts_smp_rwmtx_destroy(&dep->rwmtx);
-    erts_smp_mtx_destroy(&dep->lnk_mtx);
-    erts_smp_mtx_destroy(&dep->qlock);
+    erts_rwmtx_destroy(&dep->rwmtx);
+    erts_mtx_destroy(&dep->lnk_mtx);
+    erts_mtx_destroy(&dep->qlock);
 
 #ifdef DEBUG
     sys_memset(vdep, 0x77, sizeof(DistEntry));
@@ -193,10 +193,10 @@ erts_dist_table_info(fmtfn_t to, void *to_arg)
 {
     int lock = !ERTS_IS_CRASH_DUMPING;
     if (lock)
-	erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
+	erts_rwmtx_rlock(&erts_dist_table_rwmtx);
     hash_info(to, to_arg, &erts_dist_table);
     if (lock)
-	erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
+	erts_rwmtx_runlock(&erts_dist_table_rwmtx);
 }
 
 DistEntry *
@@ -209,7 +209,7 @@ erts_channel_no_to_dist_entry(Uint cno)
  * to the node name is used as channel no.
  */
     if(cno == ERST_INTERNAL_CHANNEL_NO) {
-	erts_smp_refc_inc(&erts_this_dist_entry->refc, 2);
+	erts_refc_inc(&erts_this_dist_entry->refc, 2);
 	return erts_this_dist_entry;
     }
 
@@ -232,23 +232,23 @@ erts_sysname_to_connected_dist_entry(Eterm sysname)
     de.sysname = sysname;
   
     if(erts_this_dist_entry->sysname == sysname) {
-	erts_smp_refc_inc(&erts_this_dist_entry->refc, 2);
+	erts_refc_inc(&erts_this_dist_entry->refc, 2);
 	return erts_this_dist_entry;
     }
 
-    erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
+    erts_rwmtx_rlock(&erts_dist_table_rwmtx);
     res_dep = (DistEntry *) hash_get(&erts_dist_table, (void *) &de);
     if (res_dep) {
-	erts_aint_t refc = erts_smp_refc_inctest(&res_dep->refc, 1);
+	erts_aint_t refc = erts_refc_inctest(&res_dep->refc, 1);
 	if (refc < 2) /* Pending delete */
-	    erts_smp_refc_inc(&res_dep->refc, 1);
+	    erts_refc_inc(&res_dep->refc, 1);
     }
-    erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
+    erts_rwmtx_runlock(&erts_dist_table_rwmtx);
     if (res_dep) {
 	int deref;
-	erts_smp_rwmtx_rlock(&res_dep->rwmtx);
+	erts_rwmtx_rlock(&res_dep->rwmtx);
 	deref = is_nil(res_dep->cid);
-	erts_smp_rwmtx_runlock(&res_dep->rwmtx);
+	erts_rwmtx_runlock(&res_dep->rwmtx);
 	if (deref) {
 	    erts_deref_dist_entry(res_dep);
 	    res_dep = NULL;
@@ -266,12 +266,12 @@ DistEntry *erts_find_or_insert_dist_entry(Eterm sysname)
     if (res)
 	return res;
     de.sysname = sysname;
-    erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx);
+    erts_rwmtx_rwlock(&erts_dist_table_rwmtx);
     res = hash_put(&erts_dist_table, (void *) &de);
-    refc = erts_smp_refc_inctest(&res->refc, 0);
+    refc = erts_refc_inctest(&res->refc, 0);
     if (refc < 2) /* New or pending delete */
-	erts_smp_refc_inc(&res->refc, 1);
-    erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
+	erts_refc_inc(&res->refc, 1);
+    erts_rwmtx_rwunlock(&erts_dist_table_rwmtx);
     return res;
 }
 
@@ -280,14 +280,14 @@ DistEntry *erts_find_dist_entry(Eterm sysname)
     DistEntry *res;
     DistEntry de;
     de.sysname = sysname;
-    erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
+    erts_rwmtx_rlock(&erts_dist_table_rwmtx);
     res = hash_get(&erts_dist_table, (void *) &de);
     if (res) {
-	erts_aint_t refc = erts_smp_refc_inctest(&res->refc, 1);
+	erts_aint_t refc = erts_refc_inctest(&res->refc, 1);
 	if (refc < 2) /* Pending delete */
-	    erts_smp_refc_inc(&res->refc, 1);
+	    erts_refc_inc(&res->refc, 1);
     }
-    erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
+    erts_rwmtx_runlock(&erts_dist_table_rwmtx);
     return res;
 }
 
@@ -296,7 +296,7 @@ static void try_delete_dist_entry(void *vdep)
     DistEntry *dep = (DistEntry *) vdep;
     erts_aint_t refc;
 
-    erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx);
+    erts_rwmtx_rwlock(&erts_dist_table_rwmtx);
     /*
      * Another thread might have looked up this dist entry after
      * we decided to delete it (refc became zero). If so, the other
@@ -312,10 +312,10 @@ static void try_delete_dist_entry(void *vdep)
      *
      * If refc > 0, the entry is in use. Keep the entry.
      */
-    refc = erts_smp_refc_dectest(&dep->refc, -1);
+    refc = erts_refc_dectest(&dep->refc, -1);
     if (refc == -1)
 	(void) hash_erase(&erts_dist_table, (void *) dep);
-    erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
+    erts_rwmtx_rwunlock(&erts_dist_table_rwmtx);
 
     if (refc == 0)
 	erts_schedule_delete_dist_entry(dep);
@@ -346,7 +346,7 @@ erts_dist_table_size(void)
     int lock = !ERTS_IS_CRASH_DUMPING;
 
     if (lock)
-	erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
+	erts_rwmtx_rlock(&erts_dist_table_rwmtx);
 #ifdef DEBUG
     hash_get_info(&hi, &erts_dist_table);
     ASSERT(dist_entries == hi.objs);
@@ -373,15 +373,15 @@ erts_dist_table_size(void)
 	   + dist_entries*sizeof(DistEntry)
 	   + erts_dist_cache_size());
     if (lock)
-	erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
+	erts_rwmtx_runlock(&erts_dist_table_rwmtx);
     return res;
 }
 
 void
 erts_set_dist_entry_not_connected(DistEntry *dep)
 {
-    ERTS_SMP_LC_ASSERT(erts_lc_is_de_rwlocked(dep));
-    erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx);
+    ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep));
+    erts_rwmtx_rwlock(&erts_dist_table_rwmtx);
 
     ASSERT(dep != erts_this_dist_entry);
     ASSERT(is_internal_port(dep->cid));
@@ -428,14 +428,14 @@ erts_set_dist_entry_not_connected(DistEntry *dep)
     }
     erts_not_connected_dist_entries = dep;
     erts_no_of_not_connected_dist_entries++;
-    erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
+    erts_rwmtx_rwunlock(&erts_dist_table_rwmtx);
 }
 
 void
 erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags)
 {
-    ERTS_SMP_LC_ASSERT(erts_lc_is_de_rwlocked(dep));
-    erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx);
+    ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep));
+    erts_rwmtx_rwlock(&erts_dist_table_rwmtx);
 
     ASSERT(dep != erts_this_dist_entry);
     ASSERT(is_nil(dep->cid));
@@ -481,7 +481,7 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags)
 	erts_hidden_dist_entries = dep;
 	erts_no_of_hidden_dist_entries++;
     }
-    erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
+    erts_rwmtx_rwunlock(&erts_dist_table_rwmtx);
 }
 
 /* -- Node table --------------------------------------------------------- */
@@ -519,7 +519,7 @@ node_table_alloc(void *venp_tmpl)
 
     node_entries++;
 
-    erts_smp_refc_init(&enp->refc, -1);
+    erts_refc_init(&enp->refc, -1);
     enp->creation = ((ErlNode *) venp_tmpl)->creation;
     enp->sysname = ((ErlNode *) venp_tmpl)->sysname;
     enp->dist_entry = erts_find_or_insert_dist_entry(((ErlNode *) venp_tmpl)->sysname);
@@ -532,7 +532,7 @@ node_table_free(void *venp)
 {
     ErlNode *enp = (ErlNode *) venp;
 
-    ERTS_SMP_LC_ASSERT(enp != erts_this_node || erts_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(enp != erts_this_node || erts_thr_progress_is_blocking());
 
     erts_deref_dist_entry(enp->dist_entry);
 #ifdef DEBUG
@@ -553,14 +553,14 @@ erts_node_table_size(void)
 #endif
     int lock = !ERTS_IS_CRASH_DUMPING;
     if (lock)
-	erts_smp_rwmtx_rlock(&erts_node_table_rwmtx);
+	erts_rwmtx_rlock(&erts_node_table_rwmtx);
 #ifdef DEBUG
     hash_get_info(&hi, &erts_node_table);
     ASSERT(node_entries == hi.objs);
 #endif
     res = hash_table_sz(&erts_node_table) + node_entries*sizeof(ErlNode);
     if (lock)
-	erts_smp_rwmtx_runlock(&erts_node_table_rwmtx);
+	erts_rwmtx_runlock(&erts_node_table_rwmtx);
     return res;
 }
 
@@ -569,10 +569,10 @@ erts_node_table_info(fmtfn_t to, void *to_arg)
 {
     int lock = !ERTS_IS_CRASH_DUMPING;
     if (lock)
-	erts_smp_rwmtx_rlock(&erts_node_table_rwmtx);
+	erts_rwmtx_rlock(&erts_node_table_rwmtx);
     hash_info(to, to_arg, &erts_node_table);
     if (lock)
-	erts_smp_rwmtx_runlock(&erts_node_table_rwmtx);
+	erts_rwmtx_runlock(&erts_node_table_rwmtx);
 }
 
 
@@ -583,26 +583,26 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint32 creation)
     ne.sysname = sysname;
     ne.creation = creation;
 
-    erts_smp_rwmtx_rlock(&erts_node_table_rwmtx);
+    erts_rwmtx_rlock(&erts_node_table_rwmtx);
     res = hash_get(&erts_node_table, (void *) &ne);
     if (res && res != erts_this_node) {
-	erts_aint_t refc = erts_smp_refc_inctest(&res->refc, 0);
+	erts_aint_t refc = erts_refc_inctest(&res->refc, 0);
 	if (refc < 2) /* New or pending delete */
-	    erts_smp_refc_inc(&res->refc, 1);
+	    erts_refc_inc(&res->refc, 1);
     }
-    erts_smp_rwmtx_runlock(&erts_node_table_rwmtx);
+    erts_rwmtx_runlock(&erts_node_table_rwmtx);
     if (res)
 	return res;
 
-    erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx);
+    erts_rwmtx_rwlock(&erts_node_table_rwmtx);
     res = hash_put(&erts_node_table, (void *) &ne);
     ASSERT(res);
     if (res != erts_this_node) {
-	erts_aint_t refc = erts_smp_refc_inctest(&res->refc, 0);
+	erts_aint_t refc = erts_refc_inctest(&res->refc, 0);
 	if (refc < 2) /* New or pending delete */
-	    erts_smp_refc_inc(&res->refc, 1);
+	    erts_refc_inc(&res->refc, 1);
     }
-    erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx);
+    erts_rwmtx_rwunlock(&erts_node_table_rwmtx);
     return res;
 }
 
@@ -611,7 +611,7 @@ static void try_delete_node(void *venp)
     ErlNode *enp = (ErlNode *) venp;
     erts_aint_t refc;
 
-    erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx);
+    erts_rwmtx_rwlock(&erts_node_table_rwmtx);
     /*
      * Another thread might have looked up this node after we
      * decided to delete it (refc became zero). If so, the other
@@ -627,10 +627,10 @@ static void try_delete_node(void *venp)
      *
      * If refc > 0, the entry is in use. Keep the entry.
      */
-    refc = erts_smp_refc_dectest(&enp->refc, -1);
+    refc = erts_refc_dectest(&enp->refc, -1);
     if (refc == -1)
 	(void) hash_erase(&erts_node_table, (void *) enp);
-    erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx);
+    erts_rwmtx_rwunlock(&erts_node_table_rwmtx);
 
     if (refc == 0)
 	erts_schedule_delete_node(enp);
@@ -673,7 +673,7 @@ static void print_node(void *venp, void *vpndp)
 	erts_print(pndp->to, pndp->to_arg, " %d", enp->creation);
 #ifdef DEBUG
 	erts_print(pndp->to, pndp->to_arg, " (refc=%ld)",
-		   erts_smp_refc_read(&enp->refc, 0));
+		   erts_refc_read(&enp->refc, 0));
 #endif
 	pndp->no_sysname++;
     }
@@ -696,13 +696,13 @@ void erts_print_node_info(fmtfn_t to,
     pnd.no_total = 0;
 
     if (lock)
-	erts_smp_rwmtx_rlock(&erts_node_table_rwmtx);
+	erts_rwmtx_rlock(&erts_node_table_rwmtx);
     hash_foreach(&erts_node_table, print_node, (void *) &pnd);
     if (pnd.no_sysname != 0) {
 	erts_print(to, to_arg, "\n");
     }
     if (lock)
-	erts_smp_rwmtx_runlock(&erts_node_table_rwmtx);
+	erts_rwmtx_runlock(&erts_node_table_rwmtx);
 
     if(no_sysname)
 	*no_sysname = pnd.no_sysname;
@@ -715,20 +715,20 @@ void erts_print_node_info(fmtfn_t to,
 void
 erts_set_this_node(Eterm sysname, Uint creation)
 {
-    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_blocking());
-    ASSERT(erts_smp_refc_read(&erts_this_dist_entry->refc, 2));
+    ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
+    ASSERT(erts_refc_read(&erts_this_dist_entry->refc, 2));
 
-    if (erts_smp_refc_dectest(&erts_this_node->refc, 0) == 0)
+    if (erts_refc_dectest(&erts_this_node->refc, 0) == 0)
         try_delete_node(erts_this_node);
 
-    if (erts_smp_refc_dectest(&erts_this_dist_entry->refc, 0) == 0)
+    if (erts_refc_dectest(&erts_this_dist_entry->refc, 0) == 0)
         try_delete_dist_entry(erts_this_dist_entry);
 
     erts_this_node = NULL; /* to make sure refc is bumped for this node */
     erts_this_node = erts_find_or_insert_node(sysname, creation);
     erts_this_dist_entry = erts_this_node->dist_entry;
 
-    erts_smp_refc_inc(&erts_this_dist_entry->refc, 2);
+    erts_refc_inc(&erts_this_dist_entry->refc, 2);
 
     erts_this_node_sysname = erts_this_node_sysname_BUFFER;
     erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER),
@@ -747,7 +747,7 @@ erts_delayed_node_table_gc(void)
 
 void erts_init_node_tables(int dd_sec)
 {
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER;
     HashFunctions f;
     ErlNode node_tmpl;
 
@@ -758,12 +758,12 @@ void erts_init_node_tables(int dd_sec)
 
     orig_node_tab_delete_delay = node_tab_delete_delay;
 
-    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
-    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+    rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_RWMTX_LONG_LIVED;
 
-    erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table", NIL,
+    erts_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
-    erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table", NIL,
+    erts_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
 
     f.hash       = (H_FUN)		dist_table_hash;
@@ -792,13 +792,13 @@ void erts_init_node_tables(int dd_sec)
     node_tmpl.creation = 0;
     erts_this_node = hash_put(&erts_node_table, &node_tmpl);
      /* +1 for erts_this_node */
-    erts_smp_refc_init(&erts_this_node->refc, 1);
+    erts_refc_init(&erts_this_node->refc, 1);
 
     ASSERT(erts_this_node->dist_entry != NULL);
     erts_this_dist_entry = erts_this_node->dist_entry;
     /* +1 for erts_this_dist_entry */
     /* +1 for erts_this_node->dist_entry */
-    erts_smp_refc_init(&erts_this_dist_entry->refc, 2);
+    erts_refc_init(&erts_this_dist_entry->refc, 2);
 
 
     erts_this_node_sysname = erts_this_node_sysname_BUFFER;
@@ -811,11 +811,11 @@ void erts_init_node_tables(int dd_sec)
 #ifdef ERTS_ENABLE_LOCK_CHECK
 int erts_lc_is_de_rwlocked(DistEntry *dep)
 {
-    return erts_smp_lc_rwmtx_is_rwlocked(&dep->rwmtx);
+    return erts_lc_rwmtx_is_rwlocked(&dep->rwmtx);
 }
 int erts_lc_is_de_rlocked(DistEntry *dep)
 {
-    return erts_smp_lc_rwmtx_is_rlocked(&dep->rwmtx);
+    return erts_lc_rwmtx_is_rlocked(&dep->rwmtx);
 }
 #endif
 
@@ -839,10 +839,10 @@ static void erts_lcnt_enable_dist_lock_count(void *dep_raw, void *enable) {
 }
 
 void erts_lcnt_update_distribution_locks(int enable) {
-    erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
+    erts_rwmtx_rlock(&erts_dist_table_rwmtx);
     hash_foreach(&erts_dist_table, erts_lcnt_enable_dist_lock_count,
         (void*)(UWord)enable);
-    erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
+    erts_rwmtx_runlock(&erts_dist_table_rwmtx);
 }
 #endif
 
@@ -944,8 +944,8 @@ erts_get_node_and_dist_references(struct process *proc)
     Uint *endp;
 #endif
 
-    erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
-    erts_smp_thr_progress_block();
+    erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
+    erts_thr_progress_block();
     /* No need to lock any thing since we are alone... */
 
     if (references_atoms_need_init) {
@@ -987,8 +987,8 @@ erts_get_node_and_dist_references(struct process *proc)
 
     delete_reference_table();
 
-    erts_smp_thr_progress_unblock();
-    erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
+    erts_thr_progress_unblock();
+    erts_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
     return res;
 }
 
@@ -1658,7 +1658,7 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp)
 
 	tup = MK_2TUP(referred_nodes[i].node->sysname,
 		      MK_UINT(referred_nodes[i].node->creation));
-	tup = MK_3TUP(tup, MK_UINT(erts_smp_refc_read(&referred_nodes[i].node->refc, 0)), nril);
+	tup = MK_3TUP(tup, MK_UINT(erts_refc_read(&referred_nodes[i].node->refc, 0)), nril);
 	nl = MK_CONS(tup, nl);
     }
 
@@ -1719,7 +1719,7 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp)
 
 	/* DistList = [{Dist, Refc, ReferenceIdList}] */
 	tup = MK_3TUP(referred_dists[i].dist->sysname,
-		      MK_UINT(erts_smp_refc_read(&referred_dists[i].dist->refc, 0)),
+		      MK_UINT(erts_refc_read(&referred_dists[i].dist->refc, 0)),
 		      dril);
 	dl = MK_CONS(tup, dl);
     }
@@ -1778,12 +1778,12 @@ delete_reference_table(void)
 void
 erts_debug_test_node_tab_delayed_delete(Sint64 millisecs)
 {
-    erts_smp_thr_progress_block();
+    erts_thr_progress_block();
 
     if (millisecs < 0)
 	node_tab_delete_delay = orig_node_tab_delete_delay;
     else
 	node_tab_delete_delay = millisecs;
 
-    erts_smp_thr_progress_unblock();
+    erts_thr_progress_unblock();
 }
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index b036a55609..7974b25444 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -44,7 +44,6 @@
 #include "erl_alloc.h"
 #include "erl_process.h"
 #include "erl_monitors.h"
-#include "erl_smp.h"
 #define ERTS_PORT_TASK_ONLY_BASIC_TYPES__
 #include "erl_port_task.h"
 #undef ERTS_PORT_TASK_ONLY_BASIC_TYPES__
@@ -107,9 +106,9 @@ typedef struct dist_entry_ {
     HashBucket hash_bucket;     /* Hash bucket */
     struct dist_entry_ *next;	/* Next entry in dist_table (not sorted) */
     struct dist_entry_ *prev;	/* Previous entry in dist_table (not sorted) */
-    erts_smp_refc_t refc;		/* Reference count */
+    erts_refc_t refc;		/* Reference count */
 
-    erts_smp_rwmtx_t rwmtx;     /* Protects all fields below until lck_mtx. */
+    erts_rwmtx_t rwmtx;     /* Protects all fields below until lck_mtx. */
     Eterm sysname;		/* name@host atom for efficiency */
     Uint32 creation;		/* creation of connected node */
     Eterm cid;			/* connection handler (pid or port), NIL == free */
@@ -120,7 +119,7 @@ typedef struct dist_entry_ {
     unsigned long version;	/* Protocol version */
 
 
-    erts_smp_mtx_t lnk_mtx;     /* Protects node_links, nlinks, and
+    erts_mtx_t lnk_mtx;     /* Protects node_links, nlinks, and
 				   monitors. */
     ErtsLink *node_links;       /* In a dist entry, node links are kept 
 				   in a separate tree, while they are 
@@ -132,14 +131,14 @@ typedef struct dist_entry_ {
     ErtsLink *nlinks;           /* Link tree with subtrees */
     ErtsMonitor *monitors;      /* Monitor tree */
 
-    erts_smp_mtx_t qlock;       /* Protects qflgs and out_queue */
+    erts_mtx_t qlock;       /* Protects qflgs and out_queue */
     Uint32 qflgs;
     Sint qsize;
     ErtsDistOutputQueue out_queue;
     struct ErtsProcList_ *suspended;
 
     ErtsDistOutputQueue finalized_out_queue;
-    erts_smp_atomic_t dist_cmd_scheduled;
+    erts_atomic_t dist_cmd_scheduled;
     ErtsPortTaskHandle dist_cmd;
 
     Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf);
@@ -149,7 +148,7 @@ typedef struct dist_entry_ {
 
 typedef struct erl_node_ {
   HashBucket hash_bucket;	/* Hash bucket */
-  erts_smp_refc_t refc;		/* Reference count */
+  erts_refc_t refc;		/* Reference count */
   Eterm	sysname;		/* name@host atom for efficiency */
   Uint32 creation;		/* Creation */
   DistEntry *dist_entry;	/* Corresponding dist entry */
@@ -158,8 +157,8 @@ typedef struct erl_node_ {
 
 extern Hash erts_dist_table;
 extern Hash erts_node_table;
-extern erts_smp_rwmtx_t erts_dist_table_rwmtx;
-extern erts_smp_rwmtx_t erts_node_table_rwmtx;
+extern erts_rwmtx_t erts_dist_table_rwmtx;
+extern erts_rwmtx_t erts_node_table_rwmtx;
 
 extern DistEntry *erts_hidden_dist_entries;
 extern DistEntry *erts_visible_dist_entries;
@@ -201,12 +200,12 @@ void erts_lcnt_update_distribution_locks(int enable);
 
 ERTS_GLB_INLINE void erts_deref_dist_entry(DistEntry *dep);
 ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np);
-ERTS_GLB_INLINE void erts_smp_de_rlock(DistEntry *dep);
-ERTS_GLB_INLINE void erts_smp_de_runlock(DistEntry *dep);
-ERTS_GLB_INLINE void erts_smp_de_rwlock(DistEntry *dep);
-ERTS_GLB_INLINE void erts_smp_de_rwunlock(DistEntry *dep);
-ERTS_GLB_INLINE void erts_smp_de_links_lock(DistEntry *dep);
-ERTS_GLB_INLINE void erts_smp_de_links_unlock(DistEntry *dep);
+ERTS_GLB_INLINE void erts_de_rlock(DistEntry *dep);
+ERTS_GLB_INLINE void erts_de_runlock(DistEntry *dep);
+ERTS_GLB_INLINE void erts_de_rwlock(DistEntry *dep);
+ERTS_GLB_INLINE void erts_de_rwunlock(DistEntry *dep);
+ERTS_GLB_INLINE void erts_de_links_lock(DistEntry *dep);
+ERTS_GLB_INLINE void erts_de_links_unlock(DistEntry *dep);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
@@ -214,7 +213,7 @@ ERTS_GLB_INLINE void
 erts_deref_dist_entry(DistEntry *dep)
 {
     ASSERT(dep);
-    if (erts_smp_refc_dectest(&dep->refc, 0) == 0)
+    if (erts_refc_dectest(&dep->refc, 0) == 0)
 	erts_schedule_delete_dist_entry(dep);
 }
 
@@ -222,44 +221,44 @@ ERTS_GLB_INLINE void
 erts_deref_node_entry(ErlNode *np)
 {
     ASSERT(np);
-    if (erts_smp_refc_dectest(&np->refc, 0) == 0)
+    if (erts_refc_dectest(&np->refc, 0) == 0)
 	erts_schedule_delete_node(np);
 }
 
 ERTS_GLB_INLINE void
-erts_smp_de_rlock(DistEntry *dep)
+erts_de_rlock(DistEntry *dep)
 {
-    erts_smp_rwmtx_rlock(&dep->rwmtx);
+    erts_rwmtx_rlock(&dep->rwmtx);
 }
 
 ERTS_GLB_INLINE void
-erts_smp_de_runlock(DistEntry *dep)
+erts_de_runlock(DistEntry *dep)
 {
-    erts_smp_rwmtx_runlock(&dep->rwmtx);
+    erts_rwmtx_runlock(&dep->rwmtx);
 }
 
 ERTS_GLB_INLINE void
-erts_smp_de_rwlock(DistEntry *dep)
+erts_de_rwlock(DistEntry *dep)
 {
-    erts_smp_rwmtx_rwlock(&dep->rwmtx);
+    erts_rwmtx_rwlock(&dep->rwmtx);
 }
 
 ERTS_GLB_INLINE void
-erts_smp_de_rwunlock(DistEntry *dep)
+erts_de_rwunlock(DistEntry *dep)
 {
-    erts_smp_rwmtx_rwunlock(&dep->rwmtx);
+    erts_rwmtx_rwunlock(&dep->rwmtx);
 }
 
 ERTS_GLB_INLINE void
-erts_smp_de_links_lock(DistEntry *dep)
+erts_de_links_lock(DistEntry *dep)
 {
-    erts_smp_mtx_lock(&dep->lnk_mtx);
+    erts_mtx_lock(&dep->lnk_mtx);
 }
 
 ERTS_GLB_INLINE void
-erts_smp_de_links_unlock(DistEntry *dep)
+erts_de_links_unlock(DistEntry *dep)
 {
-    erts_smp_mtx_unlock(&dep->lnk_mtx);
+    erts_mtx_unlock(&dep->lnk_mtx);
 }
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index 8adf56e9fe..98e9b9ccaf 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -158,10 +158,10 @@ struct _erl_drv_port {
     ErtsPortTaskHandle timeout_task;
     erts_mtx_t *lock;
     ErtsXPortsList *xports;
-    erts_smp_atomic_t run_queue;
+    erts_atomic_t run_queue;
     erts_atomic_t connected;	/* A connected process */
     Eterm caller;		/* Current caller. */
-    erts_smp_atomic_t data;	/* Data associated with port. */
+    erts_atomic_t data;	/* Data associated with port. */
     Uint bytes_in;		/* Number of bytes read */
     Uint bytes_out;		/* Number of bytes written */
 
@@ -178,7 +178,7 @@ struct _erl_drv_port {
     int control_flags;		 /* Flags for port_control()  */
     ErlDrvPDL port_data_lock;
 
-    erts_smp_atomic_t psd;	 /* Port specific data */
+    erts_atomic_t psd;	 /* Port specific data */
     int reds; /* Only used while executing driver callbacks */
 
     struct {
@@ -215,15 +215,15 @@ ERTS_GLB_INLINE ErtsRunQueue *
 erts_port_runq(Port *prt)
 {
     ErtsRunQueue *rq1, *rq2;
-    rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
+    rq1 = (ErtsRunQueue *) erts_atomic_read_nob(&prt->run_queue);
     if (!rq1)
 	return NULL;
     while (1) {
-	erts_smp_runq_lock(rq1);
-	rq2 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
+	erts_runq_lock(rq1);
+	rq2 = (ErtsRunQueue *) erts_atomic_read_nob(&prt->run_queue);
 	if (rq1 == rq2)
 	    return rq1;
-	erts_smp_runq_unlock(rq1);
+	erts_runq_unlock(rq1);
 	rq1 = rq2;
 	if (!rq1)
 	    return NULL;
@@ -241,10 +241,10 @@ ERTS_GLB_INLINE void *erts_prtsd_set(Port *p, int ix, void *new);
 ERTS_GLB_INLINE void *
 erts_prtsd_get(Port *prt, int ix)
 {
-    ErtsPrtSD *psd = (ErtsPrtSD *) erts_smp_atomic_read_nob(&prt->psd);
+    ErtsPrtSD *psd = (ErtsPrtSD *) erts_atomic_read_nob(&prt->psd);
     if (!psd)
 	return NULL;
-    ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
+    ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
     return psd->data[ix];
 }
 
@@ -255,7 +255,7 @@ erts_prtsd_set(Port *prt, int ix, void *data)
     void *old;
     int i;
 
-    psd = (ErtsPrtSD *) erts_smp_atomic_read_nob(&prt->psd);
+    psd = (ErtsPrtSD *) erts_atomic_read_nob(&prt->psd);
 
     if (psd) {
 #ifdef ETHR_ORDERED_READ_DEPEND
@@ -274,7 +274,7 @@ erts_prtsd_set(Port *prt, int ix, void *data)
     new_psd = erts_alloc(ERTS_ALC_T_PRTSD, sizeof(ErtsPrtSD));
     for (i = 0; i < ERTS_PRTSD_SIZE; i++)
 	new_psd->data[i] = NULL;
-    psd = (ErtsPrtSD *) erts_smp_atomic_cmpxchg_mb(&prt->psd,
+    psd = (ErtsPrtSD *) erts_atomic_cmpxchg_mb(&prt->psd,
 						   (erts_aint_t) new_psd,
 						   (erts_aint_t) NULL);
     if (psd)
@@ -370,9 +370,9 @@ ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt);
 ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc);
 ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt);
 
-ERTS_GLB_INLINE int erts_smp_port_trylock(Port *prt);
-ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt);
-ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt);
+ERTS_GLB_INLINE int erts_port_trylock(Port *prt);
+ERTS_GLB_INLINE void erts_port_lock(Port *prt);
+ERTS_GLB_INLINE void erts_port_unlock(Port *prt);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
@@ -401,26 +401,26 @@ ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt)
 }
 
 ERTS_GLB_INLINE int
-erts_smp_port_trylock(Port *prt)
+erts_port_trylock(Port *prt)
 {
     /* *Need* to be a managed thread */
-    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
+    ERTS_LC_ASSERT(erts_thr_progress_is_managed_thread());
     return erts_mtx_trylock(prt->lock);
 }
 
 ERTS_GLB_INLINE void
-erts_smp_port_lock(Port *prt)
+erts_port_lock(Port *prt)
 {
     /* *Need* to be a managed thread */
-    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
+    ERTS_LC_ASSERT(erts_thr_progress_is_managed_thread());
     erts_mtx_lock(prt->lock);
 }
 
 ERTS_GLB_INLINE void
-erts_smp_port_unlock(Port *prt)
+erts_port_unlock(Port *prt)
 {
     /* *Need* to be a managed thread */
-    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
+    ERTS_LC_ASSERT(erts_thr_progress_is_managed_thread());
     erts_mtx_unlock(prt->lock);
 }
 
@@ -488,7 +488,7 @@ erts_port_lookup_raw(Eterm id)
 {
     Port *prt;
 
-    ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying());
+    ERTS_LC_ASSERT(erts_thr_progress_lc_is_delaying());
 
     if (is_not_internal_port(id))
 	return NULL;
@@ -517,7 +517,7 @@ erts_id2port(Eterm id)
     Port *prt;
 
     /* Only allowed to be called from managed threads */
-    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
+    ERTS_LC_ASSERT(erts_thr_progress_is_managed_thread());
 
     if (is_not_internal_port(id))
 	return NULL;
@@ -528,10 +528,10 @@ erts_id2port(Eterm id)
     if (!prt || prt->common.id != id)
 	return NULL;
 
-    erts_smp_port_lock(prt);
+    erts_port_lock(prt);
     state = erts_atomic32_read_nob(&prt->state);
     if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP) {
-	erts_smp_port_unlock(prt);
+	erts_port_unlock(prt);
 	return NULL;
     }
 
@@ -549,7 +549,7 @@ erts_id2port_sflgs(Eterm id,
     Port *prt;
 
     /* Only allowed to be called from managed threads */
-    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
+    ERTS_LC_ASSERT(erts_thr_progress_is_managed_thread());
 
     if (is_not_internal_port(id))
 	return NULL;
@@ -561,16 +561,16 @@ erts_id2port_sflgs(Eterm id,
 	return NULL;
 
     if (no_proc_locks)
-	erts_smp_port_lock(prt);
-    else if (erts_smp_port_trylock(prt) == EBUSY) {
+	erts_port_lock(prt);
+    else if (erts_port_trylock(prt) == EBUSY) {
 	/* Unlock process locks, and acquire locks in lock order... */
-	erts_smp_proc_unlock(c_p, c_p_locks);
-	erts_smp_port_lock(prt);
-	erts_smp_proc_lock(c_p, c_p_locks);
+	erts_proc_unlock(c_p, c_p_locks);
+	erts_port_lock(prt);
+	erts_proc_lock(c_p, c_p_locks);
     }
     state = erts_atomic32_read_nob(&prt->state);
     if (state & invalid_sflgs) {
-	erts_smp_port_unlock(prt);
+	erts_port_unlock(prt);
 	return NULL;
     }
 
@@ -581,8 +581,8 @@ ERTS_GLB_INLINE void
 erts_port_release(Port *prt)
 {
     /* Only allowed to be called from managed threads */
-    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
-    erts_smp_port_unlock(prt);
+    ERTS_LC_ASSERT(erts_thr_progress_is_managed_thread());
+    erts_port_unlock(prt);
 }
 
 /*
@@ -689,7 +689,7 @@ erts_thr_drvport2port(ErlDrvPort drvport, int lock_pdl)
 #ifdef ERTS_ENABLE_LOCK_CHECK
     if (!ERTS_IS_CRASH_DUMPING) {
 	if (erts_lc_is_emu_thr()) {
-	    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+	    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 	    ERTS_LC_ASSERT(!prt->port_data_lock
 			   || erts_lc_mtx_is_locked(&prt->port_data_lock->mtx));
 	}
@@ -718,7 +718,7 @@ erts_drvport2port_state(ErlDrvPort drvport, erts_aint32_t *statep)
 //    ERTS_LC_ASSERT(erts_lc_is_emu_thr());
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return ERTS_INVALID_ERL_DRV_PORT;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)
 		       || ERTS_IS_CRASH_DUMPING);
     /* 
      * This state check is only needed since a driver callback
@@ -775,19 +775,19 @@ erts_port_driver_callback_epilogue(Port *prt, erts_aint32_t *statep)
     int reds = 0;
     erts_aint32_t state;
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
     state = erts_atomic32_read_nob(&prt->state);
     if ((state & ERTS_PORT_SFLG_CLOSING) && erts_is_port_ioq_empty(prt)) {
 	reds += ERTS_PORT_REDS_TERMINATE;
 	erts_terminate_port(prt);
 	state = erts_atomic32_read_nob(&prt->state);
-	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+	ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
     }
 
     if (prt->xports) {
 	reds += erts_port_handle_xports(prt);
-	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+	ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 	ASSERT(!prt->xports);
     }
 
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index b13d43af91..1420fb9c06 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -83,14 +83,14 @@ static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_q
 #define LTTNG_DRIVER(TRACEPOINT, PP) do {} while(0)
 #endif
 
-#define ERTS_SMP_LC_VERIFY_RQ(RQ, PP)					\
+#define ERTS_LC_VERIFY_RQ(RQ, PP)					\
     do {								\
-	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));		\
-	ERTS_SMP_LC_ASSERT((RQ) == ((ErtsRunQueue *)			\
-				    erts_smp_atomic_read_nob(&(PP)->run_queue))); \
+	ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq));		\
+	ERTS_LC_ASSERT((RQ) == ((ErtsRunQueue *)			\
+				    erts_atomic_read_nob(&(PP)->run_queue))); \
     } while (0)
 
-erts_smp_atomic_t erts_port_task_outstanding_io_tasks;
+erts_atomic_t erts_port_task_outstanding_io_tasks;
 
 #define ERTS_PT_STATE_SCHEDULED		0
 #define ERTS_PT_STATE_ABORTED		1
@@ -108,7 +108,7 @@ typedef union {
 } ErtsPortTaskTypeData;
 
 struct ErtsPortTask_ {
-    erts_smp_atomic32_t state;
+    erts_atomic32_t state;
     ErtsPortTaskType type;
     union {
 	struct {
@@ -191,7 +191,7 @@ p2p_sig_data_init(ErtsPortTask *ptp)
 
     ptp->type = ERTS_PORT_TASK_PROC_SIG;
     ptp->u.alive.flags = ERTS_PT_FLG_SIG_DEP;
-    erts_smp_atomic32_init_nob(&ptp->state, ERTS_PT_STATE_SCHEDULED);
+    erts_atomic32_init_nob(&ptp->state, ERTS_PT_STATE_SCHEDULED);
 
     ASSERT(ptp == p2p_sig_data_to_task(&ptp->u.alive.td.psig.data));
 
@@ -282,7 +282,7 @@ popped_from_busy_queue(Port *pp, ErtsPortTask *ptp, int last)
 #ifdef DEBUG
 	    erts_aint32_t flags =
 #endif
-		erts_smp_atomic32_read_band_nob(
+		erts_atomic32_read_band_nob(
 		    &pp->sched.flags,
 		    ~ERTS_PTS_FLG_HAVE_BUSY_TASKS);
 	    ASSERT(flags & ERTS_PTS_FLG_HAVE_BUSY_TASKS);
@@ -329,7 +329,7 @@ busy_wait_move_to_busy_queue(Port *pp, ErtsPortTask *ptp)
 #ifdef DEBUG
 	flags = 
 #endif
-	    erts_smp_atomic32_read_bor_nob(&pp->sched.flags,
+	    erts_atomic32_read_bor_nob(&pp->sched.flags,
 					   ERTS_PTS_FLG_HAVE_BUSY_TASKS);
 	ASSERT(!(flags & ERTS_PTS_FLG_HAVE_BUSY_TASKS));
 
@@ -469,7 +469,7 @@ no_sig_dep_move_from_busyq(Port *pp)
 	int bix;
 	erts_aint32_t flags =
 #endif
-	    erts_smp_atomic32_read_band_nob(
+	    erts_atomic32_read_band_nob(
 		&pp->sched.flags,
 		~ERTS_PTS_FLG_HAVE_BUSY_TASKS);
 	ASSERT(flags & ERTS_PTS_FLG_HAVE_BUSY_TASKS);
@@ -502,11 +502,11 @@ chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_queue)
     if (!first) {
 	ASSERT(!tabp);
 	ASSERT(!pp->sched.taskq.local.busy.last);
-	ASSERT(!(erts_smp_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_HAVE_BUSY_TASKS));
+	ASSERT(!(erts_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_HAVE_BUSY_TASKS));
 	return;
     }
 
-    ASSERT(erts_smp_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_HAVE_BUSY_TASKS);
+    ASSERT(erts_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_HAVE_BUSY_TASKS);
     ASSERT(tabp);
 
     tot_count = 0;
@@ -562,13 +562,13 @@ chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_queue)
 static ERTS_INLINE void
 reset_port_task_handle(ErtsPortTaskHandle *pthp)
 {
-    erts_smp_atomic_set_relb(pthp, (erts_aint_t) NULL);
+    erts_atomic_set_relb(pthp, (erts_aint_t) NULL);
 }
 
 static ERTS_INLINE ErtsPortTask *
 handle2task(ErtsPortTaskHandle *pthp)
 {
-    return (ErtsPortTask *) erts_smp_atomic_read_acqb(pthp);
+    return (ErtsPortTask *) erts_atomic_read_acqb(pthp);
 }
 
 static ERTS_INLINE void
@@ -595,7 +595,7 @@ set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp)
 {
     ptp->u.alive.handle = pthp;
     if (pthp) {
-	erts_smp_atomic_set_relb(pthp, (erts_aint_t) ptp);
+	erts_atomic_set_relb(pthp, (erts_aint_t) ptp);
 	ASSERT(ptp == handle2task(ptp->u.alive.handle));
     }
 }
@@ -609,7 +609,7 @@ set_tmp_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp)
 	 * IMPORTANT! Task either need to be aborted, or task handle
 	 * need to be detached before thread progress has been made.
 	 */
-	erts_smp_atomic_set_relb(pthp, (erts_aint_t) ptp);
+	erts_atomic_set_relb(pthp, (erts_aint_t) ptp);
     }
 }
 
@@ -627,20 +627,20 @@ check_unset_busy_port_q(Port *pp,
     int resume_procs = 0;
 
     ASSERT(bpq);
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(pp));
 
     erts_port_task_sched_lock(&pp->sched);
-    qsize = (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->size);
-    low = (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->low);
+    qsize = (ErlDrvSizeT) erts_atomic_read_nob(&bpq->size);
+    low = (ErlDrvSizeT) erts_atomic_read_nob(&bpq->low);
     if (qsize < low) {
 	erts_aint32_t mask = ~(ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q
 			       | ERTS_PTS_FLG_BUSY_PORT_Q);
-	flags = erts_smp_atomic32_read_band_relb(&pp->sched.flags, mask);
+	flags = erts_atomic32_read_band_relb(&pp->sched.flags, mask);
 	if ((flags & ERTS_PTS_FLGS_BUSY) == ERTS_PTS_FLG_BUSY_PORT_Q)
 	    resume_procs = 1;
     }
     else if (flags & ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q) {
-	flags = erts_smp_atomic32_read_band_relb(&pp->sched.flags,
+	flags = erts_atomic32_read_band_relb(&pp->sched.flags,
 						 ~ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q);
 	flags &= ~ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q;
     }
@@ -665,16 +665,16 @@ aborted_proc2port_data(Port *pp, ErlDrvSizeT size)
 
     bpq = pp->sched.taskq.bpq;
 
-    qsz = (ErlDrvSizeT) erts_smp_atomic_add_read_acqb(&bpq->size,
+    qsz = (ErlDrvSizeT) erts_atomic_add_read_acqb(&bpq->size,
 						      (erts_aint_t) -size);
     ASSERT(qsz + size > qsz);
-    flags = erts_smp_atomic32_read_nob(&pp->sched.flags);
+    flags = erts_atomic32_read_nob(&pp->sched.flags);
     ASSERT(pp->sched.taskq.bpq);
     if ((flags & (ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q
 		  | ERTS_PTS_FLG_BUSY_PORT_Q)) != ERTS_PTS_FLG_BUSY_PORT_Q)
 	return;
-    if (qsz < (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->low))
-	erts_smp_atomic32_read_bor_nob(&pp->sched.flags,
+    if (qsz < (ErlDrvSizeT) erts_atomic_read_nob(&bpq->low))
+	erts_atomic32_read_bor_nob(&pp->sched.flags,
 				       ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q);
 }
 
@@ -692,13 +692,13 @@ dequeued_proc2port_data(Port *pp, ErlDrvSizeT size)
 
     bpq = pp->sched.taskq.bpq;
 
-    qsz = (ErlDrvSizeT) erts_smp_atomic_add_read_acqb(&bpq->size,
+    qsz = (ErlDrvSizeT) erts_atomic_add_read_acqb(&bpq->size,
 						      (erts_aint_t) -size);
     ASSERT(qsz + size > qsz);
-    flags = erts_smp_atomic32_read_nob(&pp->sched.flags);
+    flags = erts_atomic32_read_nob(&pp->sched.flags);
     if (!(flags & ERTS_PTS_FLG_BUSY_PORT_Q))
 	return;
-    if (qsz < (ErlDrvSizeT) erts_smp_atomic_read_acqb(&bpq->low))
+    if (qsz < (ErlDrvSizeT) erts_atomic_read_acqb(&bpq->low))
 	check_unset_busy_port_q(pp, flags, bpq);
 }
 
@@ -711,19 +711,19 @@ enqueue_proc2port_data(Port *pp,
     if (sigdp && bpq) {
 	ErlDrvSizeT size = erts_proc2port_sig_command_data_size(sigdp);
 	if (size) {
-	    erts_aint_t asize = erts_smp_atomic_add_read_acqb(&bpq->size,
+	    erts_aint_t asize = erts_atomic_add_read_acqb(&bpq->size,
 							      (erts_aint_t) size);
 	    ErlDrvSizeT qsz = (ErlDrvSizeT) asize;
 
 	    ASSERT(qsz - size < qsz);
 
 	    if (!(flags & ERTS_PTS_FLG_BUSY_PORT_Q) && qsz > bpq->high) {
-		flags = erts_smp_atomic32_read_bor_acqb(&pp->sched.flags,
+		flags = erts_atomic32_read_bor_acqb(&pp->sched.flags,
 							ERTS_PTS_FLG_BUSY_PORT_Q);
 		flags |= ERTS_PTS_FLG_BUSY_PORT_Q;
-		qsz = (ErlDrvSizeT) erts_smp_atomic_read_acqb(&bpq->size);
-		if (qsz < (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->low)) {
-		    flags = (erts_smp_atomic32_read_bor_relb(
+		qsz = (ErlDrvSizeT) erts_atomic_read_acqb(&bpq->size);
+		if (qsz < (ErlDrvSizeT) erts_atomic_read_nob(&bpq->low)) {
+		    flags = (erts_atomic32_read_bor_relb(
 				 &pp->sched.flags,
 				 ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q));
 		    flags |= ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q;
@@ -771,18 +771,18 @@ erl_drv_busy_msgq_limits(ErlDrvPort dport, ErlDrvSizeT *lowp, ErlDrvSizeT *highp
 	erts_aint32_t flags;
 	pp->sched.taskq.bpq = NULL;
 	flags = ~(ERTS_PTS_FLG_BUSY_PORT_Q|ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q);
-	flags = erts_smp_atomic32_read_band_acqb(&pp->sched.flags, flags);
+	flags = erts_atomic32_read_band_acqb(&pp->sched.flags, flags);
 	if ((flags & ERTS_PTS_FLGS_BUSY) == ERTS_PTS_FLG_BUSY_PORT_Q)
 	    resume_procs = 1;
     }
     else {
 
 	if (!low)
-	    low = (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->low);
+	    low = (ErlDrvSizeT) erts_atomic_read_nob(&bpq->low);
 	else {
 	    if (bpq->high < low)
 		bpq->high = low;
-	    erts_smp_atomic_set_relb(&bpq->low, (erts_aint_t) low);
+	    erts_atomic_set_relb(&bpq->low, (erts_aint_t) low);
 	    written = 1;
 	}
     
@@ -791,19 +791,19 @@ erl_drv_busy_msgq_limits(ErlDrvPort dport, ErlDrvSizeT *lowp, ErlDrvSizeT *highp
 	else {
 	    if (low > high) {
 		low = high;
-		erts_smp_atomic_set_relb(&bpq->low, (erts_aint_t) low);
+		erts_atomic_set_relb(&bpq->low, (erts_aint_t) low);
 	    }
 	    bpq->high = high;
 	    written = 1;
 	}
 
 	if (written) {
-	    ErlDrvSizeT size = (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->size);
+	    ErlDrvSizeT size = (ErlDrvSizeT) erts_atomic_read_nob(&bpq->size);
 	    if (size > high)
-		erts_smp_atomic32_read_bor_relb(&pp->sched.flags,
+		erts_atomic32_read_bor_relb(&pp->sched.flags,
 						ERTS_PTS_FLG_BUSY_PORT_Q);
 	    else if (size < low)
-		erts_smp_atomic32_read_bor_relb(&pp->sched.flags,
+		erts_atomic32_read_bor_relb(&pp->sched.flags,
 						ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q);
 	}
     }
@@ -877,7 +877,7 @@ get_free_nosuspend_handles(Port *pp)
 {
     ErtsPortTaskHandleList *nshp, *last_nshp = NULL;
 
-    ERTS_SMP_LC_ASSERT(erts_port_task_sched_lock_is_locked(&pp->sched));
+    ERTS_LC_ASSERT(erts_port_task_sched_lock_is_locked(&pp->sched));
 
     nshp = pp->sched.taskq.local.busy.nosuspend;
 
@@ -893,7 +893,7 @@ get_free_nosuspend_handles(Port *pp)
 	pp->sched.taskq.local.busy.nosuspend = last_nshp->u.next;
 	last_nshp->u.next = NULL;
 	if (!pp->sched.taskq.local.busy.nosuspend)
-	    erts_smp_atomic32_read_band_nob(&pp->sched.flags,
+	    erts_atomic32_read_band_nob(&pp->sched.flags,
 					    ~ERTS_PTS_FLG_HAVE_NS_TASKS);
     }
     return nshp;
@@ -916,7 +916,7 @@ free_nosuspend_handles(ErtsPortTaskHandleList *free_nshp)
 static ERTS_INLINE void
 enqueue_port(ErtsRunQueue *runq, Port *pp)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq));
     pp->sched.next = NULL;
     if (runq->ports.end) {
 	ASSERT(runq->ports.start);
@@ -930,7 +930,7 @@ enqueue_port(ErtsRunQueue *runq, Port *pp)
     runq->ports.end = pp;
     ASSERT(runq->ports.start && runq->ports.end);
 
-    erts_smp_inc_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL);
+    erts_inc_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL);
 
     if (ERTS_RUNQ_FLGS_GET_NOB(runq) & ERTS_RUNQ_FLG_HALTING)
 	erts_non_empty_runq(runq);
@@ -940,7 +940,7 @@ static ERTS_INLINE Port *
 pop_port(ErtsRunQueue *runq)
 {
     Port *pp = runq->ports.start;
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq));
     if (!pp) {
 	ASSERT(!runq->ports.end);
     }
@@ -950,7 +950,7 @@ pop_port(ErtsRunQueue *runq)
 	    ASSERT(runq->ports.end == pp);
 	    runq->ports.end = NULL;
 	}
-	erts_smp_dec_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL);
+	erts_dec_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL);
     }
 
     ASSERT(runq->ports.start || !runq->ports.end);
@@ -977,7 +977,7 @@ enqueue_task(Port *pp,
     if (ns_pthlp)
 	fail_flags |= ERTS_PTS_FLG_BUSY_PORT;
     erts_port_task_sched_lock(&pp->sched);
-    flags = erts_smp_atomic32_read_nob(&pp->sched.flags);
+    flags = erts_atomic32_read_nob(&pp->sched.flags);
     if (flags & fail_flags)
 	res = 0;
     else {
@@ -1008,7 +1008,7 @@ enqueue_task(Port *pp,
 static ERTS_INLINE void
 prepare_exec(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
 {
-    erts_aint32_t act = erts_smp_atomic32_read_nob(&pp->sched.flags);
+    erts_aint32_t act = erts_atomic32_read_nob(&pp->sched.flags);
 
     if (!pp->sched.taskq.local.busy.first || (act & ERTS_PTS_FLG_BUSY_PORT)) {
 	*execqp = pp->sched.taskq.local.first;
@@ -1029,7 +1029,7 @@ prepare_exec(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
 	new &= ~ERTS_PTS_FLG_IN_RUNQ;
 	new |= ERTS_PTS_FLG_EXEC;
 
-	act = erts_smp_atomic32_cmpxchg_nob(&pp->sched.flags, new, exp);
+	act = erts_atomic32_cmpxchg_nob(&pp->sched.flags, new, exp);
 
 	ASSERT(act & ERTS_PTS_FLG_IN_RUNQ);
 
@@ -1056,7 +1056,7 @@ finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q)
 
     *execq = NULL;
 
-    act = erts_smp_atomic32_read_nob(&pp->sched.flags);
+    act = erts_atomic32_read_nob(&pp->sched.flags);
     if (act & ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q)
 	act = check_unset_busy_port_q(pp, act, pp->sched.taskq.bpq);
 
@@ -1073,7 +1073,7 @@ finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q)
 	if (act & ERTS_PTS_FLG_HAVE_TASKS)
 	    new |= ERTS_PTS_FLG_IN_RUNQ;
 
-	act = erts_smp_atomic32_cmpxchg_relb(&pp->sched.flags, new, exp);
+	act = erts_atomic32_cmpxchg_relb(&pp->sched.flags, new, exp);
 
 	ERTS_LC_ASSERT(!(act & ERTS_PTS_FLG_IN_RUNQ));
 	ERTS_LC_ASSERT(!(act & ERTS_PTS_FLG_EXEC_IMM));
@@ -1099,7 +1099,7 @@ finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q)
 static ERTS_INLINE erts_aint32_t
 select_queue_for_exec(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
 {
-    erts_aint32_t flags = erts_smp_atomic32_read_nob(&pp->sched.flags);
+    erts_aint32_t flags = erts_atomic32_read_nob(&pp->sched.flags);
 
     if (flags & ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q)
 	flags = check_unset_busy_port_q(pp, flags, pp->sched.taskq.bpq);
@@ -1209,7 +1209,7 @@ fetch_in_queue(Port *pp, ErtsPortTask **execqp)
     if (ptp)
 	*execqp = ptp->u.alive.next;
     else
-	erts_smp_atomic32_read_band_nob(&pp->sched.flags,
+	erts_atomic32_read_band_nob(&pp->sched.flags,
 					~ERTS_PTS_FLG_HAVE_TASKS);
 
     
@@ -1272,7 +1272,7 @@ erl_drv_consume_timeslice(ErlDrvPort dprt, int percent)
 void
 erts_port_task_tmp_handle_detach(ErtsPortTaskHandle *pthp)
 {
-    ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying());
+    ERTS_LC_ASSERT(erts_thr_progress_lc_is_delaying());
     reset_port_task_handle(pthp);
 }
 
@@ -1295,14 +1295,14 @@ erts_port_task_abort(ErtsPortTaskHandle *pthp)
 
 #ifdef DEBUG
 	ErtsPortTaskHandle *saved_pthp = ptp->u.alive.handle;
-	ERTS_SMP_READ_MEMORY_BARRIER;
-	old_state = erts_smp_atomic32_read_nob(&ptp->state);
+	ERTS_THR_READ_MEMORY_BARRIER;
+	old_state = erts_atomic32_read_nob(&ptp->state);
 	if (old_state == ERTS_PT_STATE_SCHEDULED) {
 	    ASSERT(!saved_pthp || saved_pthp == pthp);
 	}
 #endif
 
-	old_state = erts_smp_atomic32_cmpxchg_nob(&ptp->state,
+	old_state = erts_atomic32_cmpxchg_nob(&ptp->state,
 						  ERTS_PT_STATE_ABORTED,
 						  ERTS_PT_STATE_SCHEDULED);
 	if (old_state != ERTS_PT_STATE_SCHEDULED)
@@ -1315,9 +1315,9 @@ erts_port_task_abort(ErtsPortTaskHandle *pthp)
 	    case ERTS_PORT_TASK_INPUT:
 	    case ERTS_PORT_TASK_OUTPUT:
 	    case ERTS_PORT_TASK_EVENT:
-		ASSERT(erts_smp_atomic_read_nob(
+		ASSERT(erts_atomic_read_nob(
 			   &erts_port_task_outstanding_io_tasks) > 0);
-		erts_smp_atomic_dec_relb(&erts_port_task_outstanding_io_tasks);
+		erts_atomic_dec_relb(&erts_port_task_outstanding_io_tasks);
 		break;
 	    default:
 		break;
@@ -1339,7 +1339,7 @@ erts_port_task_abort_nosuspend_tasks(Port *pp)
     ErtsThrPrgrDelayHandle dhndl = ERTS_THR_PRGR_DHANDLE_INVALID;
 
     erts_port_task_sched_lock(&pp->sched);
-    erts_smp_atomic32_read_band_nob(&pp->sched.flags,
+    erts_atomic32_read_band_nob(&pp->sched.flags,
 				    ~ERTS_PTS_FLG_HAVE_NS_TASKS);
     abort_list = pp->sched.taskq.local.busy.nosuspend;
     pp->sched.taskq.local.busy.nosuspend = NULL;
@@ -1373,14 +1373,14 @@ erts_port_task_abort_nosuspend_tasks(Port *pp)
 
 #ifdef DEBUG
 	saved_pthp = ptp->u.alive.handle;
-	ERTS_SMP_READ_MEMORY_BARRIER;
-	old_state = erts_smp_atomic32_read_nob(&ptp->state);
+	ERTS_THR_READ_MEMORY_BARRIER;
+	old_state = erts_atomic32_read_nob(&ptp->state);
 	if (old_state == ERTS_PT_STATE_SCHEDULED) {
 	    ASSERT(saved_pthp == pthp);
 	}
 #endif
 
-	old_state = erts_smp_atomic32_cmpxchg_nob(&ptp->state,
+	old_state = erts_atomic32_cmpxchg_nob(&ptp->state,
 						  ERTS_PT_STATE_ABORTED,
 						  ERTS_PT_STATE_SCHEDULED);
 	if (old_state != ERTS_PT_STATE_SCHEDULED) {
@@ -1447,7 +1447,7 @@ erts_port_task_schedule(Eterm id,
 	ptp->type = type;
 	ptp->u.alive.flags = 0;
 
-	erts_smp_atomic32_init_nob(&ptp->state, ERTS_PT_STATE_SCHEDULED);
+	erts_atomic32_init_nob(&ptp->state, ERTS_PT_STATE_SCHEDULED);
 
 	set_handle(ptp, pthp);
     }
@@ -1459,7 +1459,7 @@ erts_port_task_schedule(Eterm id,
 	va_start(argp, type);
 	ptp->u.alive.td.io.event = va_arg(argp, ErlDrvEvent);
 	va_end(argp);
-	erts_smp_atomic_inc_relb(&erts_port_task_outstanding_io_tasks);
+	erts_atomic_inc_relb(&erts_port_task_outstanding_io_tasks);
 	break;
     }
     case ERTS_PORT_TASK_EVENT: {
@@ -1468,7 +1468,7 @@ erts_port_task_schedule(Eterm id,
 	ptp->u.alive.td.io.event = va_arg(argp, ErlDrvEvent);
 	ptp->u.alive.td.io.event_data = va_arg(argp, ErlDrvEventData);
 	va_end(argp);
-	erts_smp_atomic_inc_relb(&erts_port_task_outstanding_io_tasks);
+	erts_atomic_inc_relb(&erts_port_task_outstanding_io_tasks);
 	break;
     }
     case ERTS_PORT_TASK_PROC_SIG: {
@@ -1520,7 +1520,7 @@ erts_port_task_schedule(Eterm id,
 	if (!(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC)))
 	    new |= ERTS_PTS_FLG_IN_RUNQ;
 
-	act = erts_smp_atomic32_cmpxchg_relb(&pp->sched.flags, new, exp);
+	act = erts_atomic32_cmpxchg_relb(&pp->sched.flags, new, exp);
 
 	if (exp == act) {
 	    if (!(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC)))
@@ -1546,12 +1546,12 @@ erts_port_task_schedule(Eterm id,
 	ERTS_INTERNAL_ERROR("Missing run-queue");
 
     xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL);
-    ERTS_SMP_LC_ASSERT(runq != xrunq);
-    ERTS_SMP_LC_VERIFY_RQ(runq, pp);
+    ERTS_LC_ASSERT(runq != xrunq);
+    ERTS_LC_VERIFY_RQ(runq, pp);
     if (xrunq) {
 	/* Emigrate port ... */
-	erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq);
-	erts_smp_runq_unlock(runq);
+	erts_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq);
+	erts_runq_unlock(runq);
 	runq = erts_port_runq(pp);
 	if (!runq)
 	    ERTS_INTERNAL_ERROR("Missing run-queue");
@@ -1559,9 +1559,9 @@ erts_port_task_schedule(Eterm id,
 
     enqueue_port(runq, pp);
 
-    erts_smp_runq_unlock(runq);
+    erts_runq_unlock(runq);
 
-    erts_smp_notify_inc_runq(runq);
+    erts_notify_inc_runq(runq);
 
 done:
 
@@ -1611,14 +1611,14 @@ erts_port_task_free_port(Port *pp)
     erts_aint32_t flags;
     ErtsRunQueue *runq;
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(pp));
     ASSERT(!(erts_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD));
 
     runq = erts_port_runq(pp);
     if (!runq)
 	ERTS_INTERNAL_ERROR("Missing run-queue");
     erts_port_task_sched_lock(&pp->sched);
-    flags = erts_smp_atomic32_read_bor_relb(&pp->sched.flags,
+    flags = erts_atomic32_read_bor_relb(&pp->sched.flags,
 					    ERTS_PTS_FLG_EXIT);
     erts_port_task_sched_unlock(&pp->sched);
     erts_atomic32_read_bset_relb(&pp->state,
@@ -1628,7 +1628,7 @@ erts_port_task_free_port(Port *pp)
 				  | ERTS_PORT_SFLG_FREE),
 				 ERTS_PORT_SFLG_FREE);
 
-    erts_smp_runq_unlock(runq);
+    erts_runq_unlock(runq);
 
     if (!(flags & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC)))
 	begin_port_cleanup(pp, NULL, NULL);
@@ -1658,7 +1658,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     ErtsSchedulerData *esdp = runq->scheduler;
     ERTS_MSACC_PUSH_STATE_M();
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq));
 
     pp = pop_port(runq);
     if (!pp) {
@@ -1666,9 +1666,9 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	goto done;
     }
 
-    ERTS_SMP_LC_VERIFY_RQ(runq, pp);
+    ERTS_LC_VERIFY_RQ(runq, pp);
 
-    erts_smp_runq_unlock(runq);
+    erts_runq_unlock(runq);
 
     *curr_port_pp = pp;
     
@@ -1676,19 +1676,19 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	Uint old = ERTS_PORT_SCHED_ID(pp, esdp->no);
 	int migrated = old && old != esdp->no;
 
-	erts_smp_spin_lock(&erts_sched_stat.lock);
+	erts_spin_lock(&erts_sched_stat.lock);
 	erts_sched_stat.prio[ERTS_PORT_PRIO_LEVEL].total_executed++;
 	erts_sched_stat.prio[ERTS_PORT_PRIO_LEVEL].executed++;
 	if (migrated) {
 	    erts_sched_stat.prio[ERTS_PORT_PRIO_LEVEL].total_migrated++;
 	    erts_sched_stat.prio[ERTS_PORT_PRIO_LEVEL].migrated++;
 	}
-	erts_smp_spin_unlock(&erts_sched_stat.lock);
+	erts_spin_unlock(&erts_sched_stat.lock);
     }
 
     prepare_exec(pp, &execq, &processing_busy_q);
 
-    erts_smp_port_lock(pp);
+    erts_port_lock(pp);
 
     /* trace port scheduling, in */
     if (IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) {
@@ -1710,7 +1710,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	if (!ptp)
 	    break;
 
-	task_state = erts_smp_atomic32_cmpxchg_nob(&ptp->state,
+	task_state = erts_atomic32_cmpxchg_nob(&ptp->state,
 						   ERTS_PT_STATE_EXECUTING,
 						   ERTS_PT_STATE_SCHEDULED);
 	if (task_state != ERTS_PT_STATE_SCHEDULED) {
@@ -1722,8 +1722,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    start_time = erts_timestamp_millis();
 	}
 
-	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp));
-	ERTS_SMP_CHK_NO_PROC_LOCKS;
+	ERTS_LC_ASSERT(erts_lc_is_port_locked(pp));
+	ERTS_CHK_NO_PROC_LOCKS;
 	ASSERT(pp->drv_ptr);
 
 	switch (ptp->type) {
@@ -1842,13 +1842,13 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 
 
     if (io_tasks_executed) {
-	ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
+	ASSERT(erts_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
 	       >= io_tasks_executed);
-	erts_smp_atomic_add_relb(&erts_port_task_outstanding_io_tasks,
+	erts_atomic_add_relb(&erts_port_task_outstanding_io_tasks,
 				 -1*io_tasks_executed);
     }
 
-    ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue));
+    ASSERT(runq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue));
 
     active = finalize_exec(pp, &execq, processing_busy_q);
 
@@ -1858,7 +1858,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 
     *curr_port_pp = NULL;
 
-    erts_smp_runq_lock(runq);
+    erts_runq_lock(runq);
  
     if (active) {
 	ErtsRunQueue *xrunq;
@@ -1866,34 +1866,34 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	ASSERT(!(erts_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD));
 
 	xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL);
-	ERTS_SMP_LC_ASSERT(runq != xrunq);
-	ERTS_SMP_LC_VERIFY_RQ(runq, pp);
+	ERTS_LC_ASSERT(runq != xrunq);
+	ERTS_LC_VERIFY_RQ(runq, pp);
 	if (!xrunq) {
 	    enqueue_port(runq, pp);
 	    /* No need to notify ourselves about inc in runq. */
 	}
 	else {
 	    /* Emigrate port... */
-	    erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq);
-	    erts_smp_runq_unlock(runq);
+	    erts_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq);
+	    erts_runq_unlock(runq);
 
 	    xrunq = erts_port_runq(pp);
 	    ASSERT(xrunq);
 	    enqueue_port(xrunq, pp);
-	    erts_smp_runq_unlock(xrunq);
-	    erts_smp_notify_inc_runq(xrunq);
+	    erts_runq_unlock(xrunq);
+	    erts_notify_inc_runq(xrunq);
 
-	    erts_smp_runq_lock(runq);
+	    erts_runq_lock(runq);
 	}
     }
 
  done:
-    res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
+    res = (erts_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
 	   != (erts_aint_t) 0);
 
     runq->scheduler->reductions += reds;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq));
     ERTS_PORT_REDUCTIONS_EXECUTED(esdp, runq, reds);
 
     return res;
@@ -1924,7 +1924,7 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
     ErtsPortTaskHandleList *free_nshp = NULL;
     ErtsProcList *plp;
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(pp));
 
     /*
      * Abort remaining tasks...
@@ -1997,11 +1997,11 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
 	    qs[i] = ptp->u.alive.next;
 
 	    /* Normal case here is aborted tasks... */
-	    state = erts_smp_atomic32_read_nob(&ptp->state);
+	    state = erts_atomic32_read_nob(&ptp->state);
 	    if (state == ERTS_PT_STATE_ABORTED)
 		goto aborted_port_task;
 
-	    state = erts_smp_atomic32_cmpxchg_nob(&ptp->state,
+	    state = erts_atomic32_cmpxchg_nob(&ptp->state,
 						  ERTS_PT_STATE_EXECUTING,
 						  ERTS_PT_STATE_SCHEDULED);
 	    if (state != ERTS_PT_STATE_SCHEDULED) {
@@ -2065,7 +2065,7 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
 	}
     }
 
-    erts_smp_atomic32_read_band_nob(&pp->sched.flags,
+    erts_atomic32_read_band_nob(&pp->sched.flags,
 				    ~(ERTS_PTS_FLG_HAVE_BUSY_TASKS
 				      |ERTS_PTS_FLG_HAVE_TASKS
 				      |ERTS_PTS_FLGS_BUSY));
@@ -2122,9 +2122,9 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
 void
 erts_enqueue_port(ErtsRunQueue *rq, Port *pp)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
-    ASSERT(rq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue));
-    ASSERT(erts_smp_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_IN_RUNQ);
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
+    ASSERT(rq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue));
+    ASSERT(erts_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_IN_RUNQ);
     enqueue_port(rq, pp);
 }
 
@@ -2132,11 +2132,11 @@ Port *
 erts_dequeue_port(ErtsRunQueue *rq)
 {
     Port *pp;
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
     pp = pop_port(rq);
     ASSERT(!pp
-	   || rq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue));
-    ASSERT(!pp || (erts_smp_atomic32_read_nob(&pp->sched.flags)
+	   || rq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue));
+    ASSERT(!pp || (erts_atomic32_read_nob(&pp->sched.flags)
 		   & ERTS_PTS_FLG_IN_RUNQ));
     return pp;
 }
@@ -2148,7 +2148,7 @@ erts_dequeue_port(ErtsRunQueue *rq)
 void
 erts_port_task_init(void)
 {
-    erts_smp_atomic_init_nob(&erts_port_task_outstanding_io_tasks,
+    erts_atomic_init_nob(&erts_port_task_outstanding_io_tasks,
 			     (erts_aint_t) 0);
     init_port_task_alloc();
     init_busy_caller_table_alloc();
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index 1a06041d8e..561f4ca936 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -27,11 +27,11 @@
 #ifndef ERTS_PORT_TASK_H_BASIC_TYPES__
 #define ERTS_PORT_TASK_H_BASIC_TYPES__
 #include "erl_sys_driver.h"
-#include "erl_smp.h"
+#include "erl_threads.h"
 #define ERL_PORT_GET_PORT_TYPE_ONLY__
 #include "erl_port.h"
 #undef ERL_PORT_GET_PORT_TYPE_ONLY__
-typedef erts_smp_atomic_t ErtsPortTaskHandle;
+typedef erts_atomic_t ErtsPortTaskHandle;
 #endif
 
 #ifndef ERTS_PORT_TASK_ONLY_BASIC_TYPES__
@@ -64,7 +64,7 @@ typedef enum {
 
 #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
 /* NOTE: Do not access any of the exported variables directly */
-extern erts_smp_atomic_t erts_port_task_outstanding_io_tasks;
+extern erts_atomic_t erts_port_task_outstanding_io_tasks;
 #endif
 
 #define ERTS_PTS_FLG_IN_RUNQ			(((erts_aint32_t) 1) <<  0)
@@ -98,8 +98,8 @@ extern erts_smp_atomic_t erts_port_task_outstanding_io_tasks;
 
 typedef struct {
     ErlDrvSizeT high;
-    erts_smp_atomic_t low;
-    erts_smp_atomic_t size;
+    erts_atomic_t low;
+    erts_atomic_t size;
 }  ErtsPortTaskBusyPortQ;
 
 typedef struct ErtsPortTask_ ErtsPortTask;
@@ -124,7 +124,7 @@ typedef struct {
 	} in;
 	ErtsPortTaskBusyPortQ *bpq;
     } taskq;
-    erts_smp_atomic32_t flags;
+    erts_atomic32_t flags;
     erts_mtx_t mtx;
 } ErtsPortTaskSched;
 
@@ -149,13 +149,13 @@ ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void);
 ERTS_GLB_INLINE void
 erts_port_task_handle_init(ErtsPortTaskHandle *pthp)
 {
-    erts_smp_atomic_init_nob(pthp, (erts_aint_t) NULL);
+    erts_atomic_init_nob(pthp, (erts_aint_t) NULL);
 }
 
 ERTS_GLB_INLINE int
 erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp)
 {
-    return ((void *) erts_smp_atomic_read_acqb(pthp)) != NULL;
+    return ((void *) erts_atomic_read_acqb(pthp)) != NULL;
 }
 
 ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp,
@@ -163,9 +163,9 @@ ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp,
 {
     if (bpq) {
 	erts_aint_t low = (erts_aint_t) ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_LOW;
-	erts_smp_atomic_init_nob(&bpq->low, low);
+	erts_atomic_init_nob(&bpq->low, low);
  	bpq->high = (ErlDrvSizeT) ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_HIGH;
-	erts_smp_atomic_init_nob(&bpq->size, (erts_aint_t) 0);
+	erts_atomic_init_nob(&bpq->size, (erts_aint_t) 0);
     }
     ptsp->taskq.bpq = bpq;
 }
@@ -182,7 +182,7 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id)
     ptsp->taskq.local.first = NULL;
     ptsp->taskq.in.first = NULL;
     ptsp->taskq.in.last = NULL;
-    erts_smp_atomic32_init_nob(&ptsp->flags, 0);
+    erts_atomic32_init_nob(&ptsp->flags, 0);
     erts_mtx_init(&ptsp->mtx, lock_str, instr_id, ERTS_LOCK_FLAGS_CATEGORY_IO);
 }
 
@@ -218,7 +218,7 @@ erts_port_task_fini_sched(ErtsPortTaskSched *ptsp)
 ERTS_GLB_INLINE void
 erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp)
 {
-    erts_smp_atomic32_read_bor_nob(&ptsp->flags, ERTS_PTS_FLG_EXITING);
+    erts_atomic32_read_bor_nob(&ptsp->flags, ERTS_PTS_FLG_EXITING);
 }
 
 #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
@@ -226,7 +226,7 @@ erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp)
 ERTS_GLB_INLINE int
 erts_port_task_have_outstanding_io_tasks(void)
 {
-    return (erts_smp_atomic_read_acqb(&erts_port_task_outstanding_io_tasks)
+    return (erts_atomic_read_acqb(&erts_port_task_outstanding_io_tasks)
 	    != 0);
 }
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 8a218d9d69..88b2bda59c 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -135,8 +135,8 @@ runq_got_work_to_execute(ErtsRunQueue *rq)
 
 #undef RUNQ_READ_RQ
 #undef RUNQ_SET_RQ
-#define RUNQ_READ_RQ(X) ((ErtsRunQueue *) erts_smp_atomic_read_nob((X)))
-#define RUNQ_SET_RQ(X, RQ) erts_smp_atomic_set_nob((X), (erts_aint_t) (RQ))
+#define RUNQ_READ_RQ(X) ((ErtsRunQueue *) erts_atomic_read_nob((X)))
+#define RUNQ_SET_RQ(X, RQ) erts_atomic_set_nob((X), (erts_aint_t) (RQ))
 
 #ifdef DEBUG
 #  if defined(ARCH_64)
@@ -230,11 +230,11 @@ typedef struct {
 } ErtsSchedTypeCounters;
 
 static struct {
-    erts_smp_mtx_t mtx;
+    erts_mtx_t mtx;
     ErtsSchedTypeCounters online;
     ErtsSchedTypeCounters curr_online;
     ErtsSchedTypeCounters active;
-    erts_smp_atomic32_t changing;
+    erts_atomic32_t changing;
     ErtsProcList *chngq;
     Eterm changer;
     ErtsMultiSchedulingBlock nmsb; /* Normal multi Scheduling Block */
@@ -359,11 +359,11 @@ schdlr_sspnd_set_nscheds(ErtsSchedTypeCounters *valp,
 }
 
 static struct {
-    erts_smp_mtx_t update_mtx;
-    erts_smp_atomic32_t no_runqs;
+    erts_mtx_t update_mtx;
+    erts_atomic32_t no_runqs;
     int last_active_runqs;
     int forced_check_balance;
-    erts_smp_atomic32_t checking_balance;
+    erts_atomic32_t checking_balance;
     int halftime;
     int full_reds_history_index;
     struct {
@@ -386,10 +386,10 @@ erts_sched_stat_t erts_sched_stat;
 
 static erts_tsd_key_t ERTS_WRITE_UNLIKELY(sched_data_key);
 
-static erts_smp_atomic32_t function_calls;
+static erts_atomic32_t function_calls;
 
-static erts_smp_atomic32_t doing_sys_schedule;
-static erts_smp_atomic32_t no_empty_run_queues;
+static erts_atomic32_t doing_sys_schedule;
+static erts_atomic32_t no_empty_run_queues;
 long erts_runq_supervision_interval = 0;
 static ethr_event runq_supervision_event;
 static erts_tid_t runq_supervisor_tid;
@@ -402,11 +402,11 @@ Uint ERTS_WRITE_UNLIKELY(erts_no_run_queues);
 
 struct {
     union {
-        erts_smp_atomic32_t active;
+        erts_atomic32_t active;
         char align__[ERTS_CACHE_LINE_SIZE];
     } cpu;
     union {
-        erts_smp_atomic32_t active;
+        erts_atomic32_t active;
         char align__[ERTS_CACHE_LINE_SIZE];
     } io;
 } dirty_count erts_align_attribute(ERTS_CACHE_LINE_SIZE);
@@ -418,7 +418,7 @@ dirty_active(ErtsSchedulerData *esdp, erts_aint32_t add)
 {
 #ifdef ERTS_DIRTY_SCHEDULERS
     erts_aint32_t val;
-    erts_smp_atomic32_t *ap;
+    erts_atomic32_t *ap;
     switch (esdp->type) {
     case ERTS_SCHED_DIRTY_CPU:
         ap = &dirty_count.cpu.active;
@@ -436,11 +436,11 @@ dirty_active(ErtsSchedulerData *esdp, erts_aint32_t add)
      * All updates done under run-queue lock, so
      * no inc or dec needed...
      */
-    ERTS_SMP_ASSERT(erts_smp_lc_runq_is_locked(esdp->run_queue));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(esdp->run_queue));
 
-    val = erts_smp_atomic32_read_nob(ap);
+    val = erts_atomic32_read_nob(ap);
     val += add;
-    erts_smp_atomic32_set_nob(ap, val);
+    erts_atomic32_set_nob(ap, val);
 #endif
 }
 
@@ -552,9 +552,9 @@ do {									\
     int ix__;								\
     for (ix__ = 0; ix__ < erts_no_run_queues; ix__++) {			\
 	RQVAR = ERTS_RUNQ_IX(ix__);					\
-	erts_smp_runq_lock(RQVAR);					\
+	erts_runq_lock(RQVAR);					\
 	{ DO; }								\
-	erts_smp_runq_unlock(RQVAR);					\
+	erts_runq_unlock(RQVAR);					\
     }									\
 } while (0)
 
@@ -564,12 +564,12 @@ do {									\
     int ix__;								\
     int online__ = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,	\
 						  ERTS_SCHED_NORMAL);	\
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&schdlr_sspnd.mtx));	\
+    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&schdlr_sspnd.mtx));	\
     for (ix__ = 0; ix__ < online__; ix__++) {				\
 	RQVAR = ERTS_RUNQ_IX(ix__);					\
-	erts_smp_runq_lock(RQVAR);					\
+	erts_runq_lock(RQVAR);					\
 	{ DO; }								\
-	erts_smp_runq_unlock(RQVAR);					\
+	erts_runq_unlock(RQVAR);					\
     }									\
 } while (0)
 
@@ -580,12 +580,12 @@ do {									\
     int ix__;								\
     for (ix__ = 0; ix__ < nrqs; ix__++) {                               \
 	RQVAR = ERTS_RUNQ_IX(ix__);					\
-	erts_smp_runq_lock(RQVAR);					\
+	erts_runq_lock(RQVAR);					\
 	{ DO; }								\
     }									\
     { DOX; }								\
     for (ix__ = 0; ix__ < nrqs; ix__++)                                 \
-	erts_smp_runq_unlock(ERTS_RUNQ_IX(ix__));			\
+	erts_runq_unlock(ERTS_RUNQ_IX(ix__));			\
 } while (0)
 
 #define ERTS_ATOMIC_FOREACH_RUNQ(RQVAR, DO)                             \
@@ -662,9 +662,9 @@ static void wake_scheduler(ErtsRunQueue *rq);
 
 #if defined(ERTS_ENABLE_LOCK_CHECK)
 int
-erts_smp_lc_runq_is_locked(ErtsRunQueue *runq)
+erts_lc_runq_is_locked(ErtsRunQueue *runq)
 {
-    return erts_smp_lc_mtx_is_locked(&runq->mtx);
+    return erts_lc_mtx_is_locked(&runq->mtx);
 }
 #endif
 
@@ -672,13 +672,13 @@ erts_smp_lc_runq_is_locked(ErtsRunQueue *runq)
 static ERTS_INLINE Uint64
 ensure_later_proc_interval(Uint64 interval)
 {
-    return erts_smp_ensure_later_interval_nob(erts_ptab_interval(&erts_proc), interval);
+    return erts_ensure_later_interval_nob(erts_ptab_interval(&erts_proc), interval);
 }
 
 Uint64
 erts_get_proc_interval(void)
 {
-    return erts_smp_current_interval_nob(erts_ptab_interval(&erts_proc));
+    return erts_current_interval_nob(erts_ptab_interval(&erts_proc));
 }
 
 Uint64
@@ -690,7 +690,7 @@ erts_ensure_later_proc_interval(Uint64 interval)
 Uint64
 erts_step_proc_interval(void)
 {
-    return erts_smp_step_interval_nob(erts_ptab_interval(&erts_proc));
+    return erts_step_interval_nob(erts_ptab_interval(&erts_proc));
 }
 
 void
@@ -815,7 +815,7 @@ erts_late_init_process(void)
 {
     int ix;
 
-    erts_smp_spinlock_init(&erts_sched_stat.lock, "sched_stat", NIL,
+    erts_spinlock_init(&erts_sched_stat.lock, "sched_stat", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
 
     for (ix = 0; ix < ERTS_NO_PRIO_LEVELS; ix++) {
@@ -1015,14 +1015,14 @@ erts_get_sched_util(ErtsRunQueue *rq, int initially_locked, int short_interval)
 	if (!locked) {
 	    if (++try >= ERTS_GET_AVG_MAX_UNLOCKED_TRY) {
 		/* Writer will eventually block on runq-lock */
-		erts_smp_runq_lock(rq);
+		erts_runq_lock(rq);
 		locked = 1;
 	    }
 	}
     }
 
     if (!initially_locked && locked)
-	erts_smp_runq_unlock(rq);
+	erts_runq_unlock(rq);
 
     now = sched_wall_time_ts();
     worktime = calc_sched_worktime(is_working, now, last, interval, old_worktime);
@@ -1219,7 +1219,7 @@ typedef struct {
     Eterm ref;
     Eterm ref_heap[ERTS_REF_THING_SIZE];
     Uint req_sched;
-    erts_smp_atomic32_t refc;
+    erts_atomic32_t refc;
 #ifdef ERTS_DIRTY_SCHEDULERS
     int want_dirty_cpu;
     int want_dirty_io;
@@ -1231,7 +1231,7 @@ typedef struct {
     Eterm ref;
     Eterm ref_heap[ERTS_REF_THING_SIZE];
     Uint req_sched;
-    erts_smp_atomic32_t refc;
+    erts_atomic32_t refc;
 } ErtsSystemCheckReq;
 
 
@@ -1413,11 +1413,11 @@ reply_sched_wall_time(void *vswtrp)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
  
     if (rp_locks)
-	erts_smp_proc_unlock(rp, rp_locks);
+	erts_proc_unlock(rp, rp_locks);
 
     erts_proc_dec_refc(rp);
 
-    if (erts_smp_atomic32_dec_read_nob(&swtrp->refc) == 0)
+    if (erts_atomic32_dec_read_nob(&swtrp->refc) == 0)
 	swtreq_free(vswtrp);
 }
 
@@ -1448,7 +1448,7 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable,
     swtrp->want_dirty_cpu = want_dirty_cpu;
     swtrp->want_dirty_io = want_dirty_io;
 #endif
-    erts_smp_atomic32_init_nob(&swtrp->refc,
+    erts_atomic32_init_nob(&swtrp->refc,
 			       (erts_aint32_t) erts_no_schedulers);
 
     erts_proc_add_refc(c_p, (Sint32) erts_no_schedulers);
@@ -1491,11 +1491,11 @@ reply_system_check(void *vscrp)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
 
     if (rp_locks)
-	erts_smp_proc_unlock(rp, rp_locks);
+	erts_proc_unlock(rp, rp_locks);
 
     erts_proc_dec_refc(rp);
 
-    if (erts_smp_atomic32_dec_read_nob(&scrp->refc) == 0)
+    if (erts_atomic32_dec_read_nob(&scrp->refc) == 0)
 	screq_free(vscrp);
 }
 
@@ -1513,7 +1513,7 @@ Eterm erts_system_check_request(Process *c_p) {
     scrp->proc = c_p;
     scrp->ref = STORE_NC(&hp, NULL, ref);
     scrp->req_sched = esdp->no;
-    erts_smp_atomic32_init_nob(&scrp->refc, (erts_aint32_t) erts_no_schedulers);
+    erts_atomic32_init_nob(&scrp->refc, (erts_aint32_t) erts_no_schedulers);
 
     erts_proc_add_refc(c_p, (Sint) erts_no_schedulers);
 
@@ -1576,7 +1576,7 @@ erts_psd_set_init(Process *p, int ix, void *data)
     for (i = 0; i < ERTS_PSD_SIZE; i++)
 	new_psd->data[i] = NULL;
 
-    psd = (ErtsPSD *) erts_smp_atomic_cmpxchg_mb(&p->psd,
+    psd = (ErtsPSD *) erts_atomic_cmpxchg_mb(&p->psd,
 						 (erts_aint_t) new_psd,
 						 (erts_aint_t) NULL);
     if (psd)
@@ -2472,7 +2472,7 @@ notify_reap_ports_relb(void)
     }
 }
 
-erts_smp_atomic32_t erts_halt_progress;
+erts_atomic32_t erts_halt_progress;
 int erts_halt_code;
 
 static ERTS_INLINE erts_aint32_t
@@ -2481,9 +2481,9 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_REAP_PORTS);
     ERTS_RUNQ_FLGS_SET(awdp->esdp->run_queue, ERTS_RUNQ_FLG_HALTING);
 
-    if (erts_smp_atomic32_dec_read_acqb(&erts_halt_progress) == 0) {
+    if (erts_atomic32_dec_read_acqb(&erts_halt_progress) == 0) {
 	int i, max = erts_ptab_max(&erts_port);
-	erts_smp_atomic32_set_nob(&erts_halt_progress, 1);
+	erts_atomic32_set_nob(&erts_halt_progress, 1);
 	for (i = 0; i < max; i++) {
 	    erts_aint32_t state;
 	    Port *prt = erts_pix2port(i);
@@ -2496,21 +2496,21 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
 
 	    /* We need to set the halt flag - get the port lock */
 
-	    erts_smp_port_lock(prt);
+	    erts_port_lock(prt);
 
 	    state = erts_atomic32_read_nob(&prt->state);
 	    if (!(state & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
 			   | ERTS_PORT_SFLG_HALT))) {
 		state = erts_atomic32_read_bor_relb(&prt->state,
 						    ERTS_PORT_SFLG_HALT);
-		erts_smp_atomic32_inc_nob(&erts_halt_progress);
+		erts_atomic32_inc_nob(&erts_halt_progress);
 		if (!(state & (ERTS_PORT_SFLG_EXITING|ERTS_PORT_SFLG_CLOSING)))
 		    erts_deliver_port_exit(prt, prt->common.id, am_killed, 0, 1);
 	    }
 
 	    erts_port_release(prt);
 	}
-	if (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0) {
+	if (erts_atomic32_dec_read_nob(&erts_halt_progress) == 0) {
 	    erts_flush_async_exit(erts_halt_code, "");
 	}
     }
@@ -2581,10 +2581,10 @@ handle_pending_exiters(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin
     rq = awdp->esdp->run_queue;
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_PENDING_EXITERS);
 
-    erts_smp_runq_lock(rq);
+    erts_runq_lock(rq);
     pnd_xtrs = rq->procs.pending_exiters;
     rq->procs.pending_exiters = NULL;
-    erts_smp_runq_unlock(rq);
+    erts_runq_unlock(rq);
 
     if (erts_proclist_fetch(&pnd_xtrs, NULL))
 	do_handle_pending_exiters(pnd_xtrs);
@@ -2866,7 +2866,7 @@ erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable)
 static ERTS_INLINE void
 sched_waiting_sys(Uint no, ErtsRunQueue *rq)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
     ASSERT(rq->waiting >= 0);
     (void) ERTS_RUNQ_FLGS_SET(rq, (ERTS_RUNQ_FLG_OUT_OF_WORK
 				   | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK));
@@ -2880,7 +2880,7 @@ sched_waiting_sys(Uint no, ErtsRunQueue *rq)
 static ERTS_INLINE void
 sched_active_sys(Uint no, ErtsRunQueue *rq)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
 
     ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
 
@@ -2905,13 +2905,13 @@ erts_active_schedulers(void)
 static ERTS_INLINE void
 clear_sys_scheduling(void)
 {
-    erts_smp_atomic32_set_mb(&doing_sys_schedule, 0);
+    erts_atomic32_set_mb(&doing_sys_schedule, 0);
 }
 
 static ERTS_INLINE int
 try_set_sys_scheduling(void)
 {
-    return 0 == erts_smp_atomic32_cmpxchg_acqb(&doing_sys_schedule, 1, 0);
+    return 0 == erts_atomic32_cmpxchg_acqb(&doing_sys_schedule, 1, 0);
 }
 
 
@@ -2936,7 +2936,7 @@ prepare_for_sys_schedule(int non_blocking)
 static ERTS_INLINE void
 sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
 
     ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
 
@@ -2947,7 +2947,7 @@ sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq)
 static ERTS_INLINE void
 sched_waiting(Uint no, ErtsRunQueue *rq)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
     (void) ERTS_RUNQ_FLGS_SET(rq, (ERTS_RUNQ_FLG_OUT_OF_WORK
 				   | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK));
     if (rq->waiting < 0)
@@ -2962,7 +2962,7 @@ sched_waiting(Uint no, ErtsRunQueue *rq)
 static ERTS_INLINE void
 sched_active(Uint no, ErtsRunQueue *rq)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
     if (rq->waiting < 0)
 	rq->waiting++;
     else
@@ -2976,7 +2976,7 @@ empty_runq_aux(ErtsRunQueue *rq, Uint32 old_flags)
 {
     if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && old_flags & ERTS_RUNQ_FLG_NONEMPTY) {
 #ifdef DEBUG
-	erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues);
+	erts_aint32_t empty = erts_atomic32_read_nob(&no_empty_run_queues);
 	/*
 	 * For a short period of time no_empty_run_queues may have
 	 * been increased twice for a specific run queue.
@@ -2984,9 +2984,9 @@ empty_runq_aux(ErtsRunQueue *rq, Uint32 old_flags)
 	ASSERT(0 <= empty && empty < 2*erts_no_run_queues);
 #endif
 	if (!erts_runq_supervision_interval)
-	    erts_smp_atomic32_inc_relb(&no_empty_run_queues);
+	    erts_atomic32_inc_relb(&no_empty_run_queues);
 	else {
-	    erts_smp_atomic32_inc_mb(&no_empty_run_queues);
+	    erts_atomic32_inc_mb(&no_empty_run_queues);
 	    if (erts_atomic_read_nob(&runq_supervisor_sleeping))
 		ethr_event_set(&runq_supervision_event);
 	}
@@ -3016,7 +3016,7 @@ non_empty_runq(ErtsRunQueue *rq)
     Uint32 old_flags = ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_NONEMPTY);
     if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && (!(old_flags & ERTS_RUNQ_FLG_NONEMPTY))) {
 #ifdef DEBUG
-	erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues);
+	erts_aint32_t empty = erts_atomic32_read_nob(&no_empty_run_queues);
 	/*
 	 * For a short period of time no_empty_run_queues may have
 	 * been increased twice for a specific run queue.
@@ -3024,10 +3024,10 @@ non_empty_runq(ErtsRunQueue *rq)
 	ASSERT(0 < empty && empty <= 2*erts_no_run_queues);
 #endif
 	if (!erts_runq_supervision_interval)
-	    erts_smp_atomic32_dec_relb(&no_empty_run_queues);
+	    erts_atomic32_dec_relb(&no_empty_run_queues);
 	else {
 	    erts_aint32_t no;
-	    no = erts_smp_atomic32_dec_read_mb(&no_empty_run_queues);
+	    no = erts_atomic32_dec_read_mb(&no_empty_run_queues);
 	    if (no > 0 && erts_atomic_read_nob(&runq_supervisor_sleeping))
 		ethr_event_set(&runq_supervision_event);
 	}
@@ -3056,7 +3056,7 @@ sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi)
     do {
         nflgs = (xflgs & ERTS_SSI_FLG_MSB_EXEC);
         nflgs |= ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING;
-	oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
+	oflgs = erts_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
 	if (oflgs == xflgs)
 	    return nflgs;
 	xflgs = oflgs;
@@ -3073,7 +3073,7 @@ sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi)
     erts_aint32_t xflgs = ERTS_SSI_FLG_WAITING;
 
     do {
-	oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
+	oflgs = erts_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
 	if (oflgs == xflgs)
 	    return nflgs;
 	xflgs = oflgs;
@@ -3090,7 +3090,7 @@ sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount)
     erts_aint32_t flgs;
 
     do {
-	flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+	flgs = erts_atomic32_read_acqb(&ssi->flags);
 	if ((flgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING))
 	    != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) {
 	    break;
@@ -3119,7 +3119,7 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type)
     }
 
     while (1) {
-	oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
+	oflgs = erts_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
 	if (oflgs == xflgs)
 	    return nflgs;
 	if ((oflgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING))
@@ -3146,7 +3146,7 @@ static void
 thr_prgr_prep_wait(void *vssi)
 {
     ErtsSchedulerSleepInfo *ssi = (ErtsSchedulerSleepInfo *) vssi;
-    erts_smp_atomic32_read_bor_acqb(&ssi->flags,
+    erts_atomic32_read_bor_acqb(&ssi->flags,
 				    ERTS_SSI_FLG_SLEEPING);
 }
 
@@ -3161,7 +3161,7 @@ thr_prgr_wait(void *vssi)
     while (1) {
 	erts_aint32_t aflgs, nflgs;
 	nflgs = xflgs | ERTS_SSI_FLG_TSE_SLEEPING;
-	aflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
+	aflgs = erts_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
 	if (aflgs == xflgs) {
 	    erts_tse_wait(ssi->event);
 	    break;
@@ -3176,7 +3176,7 @@ static void
 thr_prgr_fin_wait(void *vssi)
 {
     ErtsSchedulerSleepInfo *ssi = (ErtsSchedulerSleepInfo *) vssi;
-    erts_smp_atomic32_read_band_nob(&ssi->flags,
+    erts_atomic32_read_band_nob(&ssi->flags,
 				    ~(ERTS_SSI_FLG_SLEEPING
 				      | ERTS_SSI_FLG_TSE_SLEEPING));
 }
@@ -3269,18 +3269,18 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
     erts_aint32_t flgs;
     ERTS_MSACC_PUSH_STATE_M();
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
 
 #ifdef ERTS_DIRTY_SCHEDULERS
     if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
-	erts_smp_spin_lock(&rq->sleepers.lock);
+	erts_spin_lock(&rq->sleepers.lock);
 #endif
     flgs = sched_prep_spin_wait(ssi);
     if (flgs & ERTS_SSI_FLG_SUSPENDED) {
 	/* Go suspend instead... */
 #ifdef ERTS_DIRTY_SCHEDULERS
 	if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
-	    erts_smp_spin_unlock(&rq->sleepers.lock);
+	    erts_spin_unlock(&rq->sleepers.lock);
 #endif
 	return;
     }
@@ -3292,7 +3292,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	if (rq->sleepers.list)
 	    rq->sleepers.list->prev = ssi;
 	rq->sleepers.list = ssi;
-	erts_smp_spin_unlock(&rq->sleepers.lock);
+	erts_spin_unlock(&rq->sleepers.lock);
         dirty_active(esdp, -1);
     }
 #endif
@@ -3306,7 +3306,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	sched_waiting(esdp->no, rq);
 
-	erts_smp_runq_unlock(rq);
+	erts_runq_unlock(rq);
 
 	spincount = sched_busy_wait.tse;
 
@@ -3334,7 +3334,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	    if (aux_work) {
 		if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-		    flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+		    flgs = erts_atomic32_read_acqb(&ssi->flags);
 		    current_time = erts_get_monotonic_time(esdp);
 		    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
 			if (!thr_prgr_active) {
@@ -3422,7 +3422,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	}
 
 	if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC))
-	    erts_smp_atomic32_read_band_nob(&ssi->flags,
+	    erts_atomic32_read_band_nob(&ssi->flags,
                                             (ERTS_SSI_FLG_SUSPENDED
                                              | ERTS_SSI_FLG_MSB_EXEC));
 
@@ -3433,21 +3433,21 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    sched_wall_time_change(esdp, 1);
         }
 
-	erts_smp_runq_lock(rq);
+	erts_runq_lock(rq);
 	sched_active(esdp->no, rq);
 
     }
     else
     {
 
-	erts_smp_atomic32_set_relb(&function_calls, 0);
+	erts_atomic32_set_relb(&function_calls, 0);
 	*fcalls = 0;
 
 	ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
 
 	sched_waiting_sys(esdp->no, rq);
 
-	erts_smp_runq_unlock(rq);
+	erts_runq_unlock(rq);
 
 	ASSERT(working);
 	sched_wall_time_change(esdp, working = 0);
@@ -3492,7 +3492,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		    erts_thr_progress_leader_update(esdp);
 	    }
 
-	    flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+	    flgs = erts_atomic32_read_acqb(&ssi->flags);
 	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
 		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
 		goto sys_woken;
@@ -3515,7 +3515,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    }
 	}
 
-	erts_smp_runq_lock(rq);
+	erts_runq_lock(rq);
 
 	/*
 	 * If we got new I/O tasks we aren't allowed to
@@ -3534,13 +3534,13 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		 * do tse wait instead...
 		 */
 		sched_change_waiting_sys_to_waiting(esdp->no, rq);
-		erts_smp_runq_unlock(rq);
+		erts_runq_unlock(rq);
 		spincount = 0;
 		goto tse_wait;
 	    }
 	}
 	if (aux_work) {
-	    erts_smp_runq_unlock(rq);
+	    erts_runq_unlock(rq);
 	    goto sys_poll_aux_work;
 	}
 	flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING);
@@ -3549,7 +3549,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
 		goto sys_locked_woken;
 	    }
-	    erts_smp_runq_unlock(rq);
+	    erts_runq_unlock(rq);
 	    flgs = sched_prep_cont_spin_wait(ssi);
 	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
 		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
@@ -3562,7 +3562,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING);
 	ASSERT(flgs & ERTS_SSI_FLG_WAITING);
 
-	erts_smp_runq_unlock(rq);
+	erts_runq_unlock(rq);
 
 	if (working)
 	    sched_wall_time_change(esdp, working = 0);
@@ -3592,16 +3592,16 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
     sys_woken:
 	if (!thr_prgr_active)
 	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
-	erts_smp_runq_lock(rq);
+	erts_runq_lock(rq);
     sys_locked_woken:
 	if (!thr_prgr_active) {
-	    erts_smp_runq_unlock(rq);
+	    erts_runq_unlock(rq);
 	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
-	    erts_smp_runq_lock(rq);
+	    erts_runq_lock(rq);
 	}
 	clear_sys_scheduling();
 	if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC))
-	    erts_smp_atomic32_read_band_nob(&ssi->flags,
+	    erts_atomic32_read_band_nob(&ssi->flags,
                                             (ERTS_SSI_FLG_SUSPENDED
                                              | ERTS_SSI_FLG_MSB_EXEC));
 	if (!working)
@@ -3612,7 +3612,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
     if (ERTS_SCHEDULER_IS_DIRTY(esdp))
         dirty_active(esdp, 1);
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
 }
 
 
@@ -3624,7 +3624,7 @@ ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi)
     erts_aint32_t nflgs = 0;
     erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING;
     while (1) {
-	oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, nflgs, xflgs);
+	oflgs = erts_atomic32_cmpxchg_relb(&ssi->flags, nflgs, xflgs);
 	if (oflgs == xflgs)
 	    return oflgs;
 	nflgs = oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC);
@@ -3644,7 +3644,7 @@ static void
 dcpu_sched_ix_suspend_wake(Uint ix)
 {
     ErtsSchedulerSleepInfo* ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-    erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+    erts_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
     ssi_wake(ssi);
 }
 
@@ -3652,7 +3652,7 @@ static void
 dio_sched_ix_suspend_wake(Uint ix)
 {
     ErtsSchedulerSleepInfo* ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
-    erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+    erts_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
     ssi_wake(ssi);
 }
 
@@ -3683,7 +3683,7 @@ wake_scheduler(ErtsRunQueue *rq)
      * so all code *should* handle this without having
      * the lock on the run queue.
      */
-    ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq)
+    ERTS_LC_ASSERT(!erts_lc_runq_is_locked(rq)
 		       || ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
 
     ssi_wake(rq->scheduler->ssi);
@@ -3699,10 +3699,10 @@ wake_dirty_schedulers(ErtsRunQueue *rq, int one)
     ASSERT(ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
 
     sl = &rq->sleepers;
-    erts_smp_spin_lock(&sl->lock);
+    erts_spin_lock(&sl->lock);
     ssi = sl->list;
     if (!ssi) {
-	erts_smp_spin_unlock(&sl->lock);
+	erts_spin_unlock(&sl->lock);
 	if (one)
 	    wake_scheduler(rq);
     } else if (one) {
@@ -3716,14 +3716,14 @@ wake_dirty_schedulers(ErtsRunQueue *rq, int one)
 	if (ssi->next)
 	    ssi->next->prev = ssi->prev;
 
-	erts_smp_spin_unlock(&sl->lock);
+	erts_spin_unlock(&sl->lock);
 
 	ERTS_THR_MEMORY_BARRIER;
 	flgs = ssi_flags_set_wake(ssi);
 	erts_sched_finish_poke(ssi, flgs);
     } else {
 	sl->list = NULL;
-	erts_smp_spin_unlock(&sl->lock);
+	erts_spin_unlock(&sl->lock);
 
 	ERTS_THR_MEMORY_BARRIER;
 	do {
@@ -3754,13 +3754,13 @@ init_no_runqs(int active, int used)
 {
     erts_aint32_t no_runqs = (erts_aint32_t) (active & ERTS_NO_RUNQS_MASK);
     no_runqs |= (erts_aint32_t) ((used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT);
-    erts_smp_atomic32_init_nob(&balance_info.no_runqs, no_runqs);
+    erts_atomic32_init_nob(&balance_info.no_runqs, no_runqs);
 }
 
 static ERTS_INLINE void
 get_no_runqs(int *active, int *used)
 {
-    erts_aint32_t no_runqs = erts_smp_atomic32_read_nob(&balance_info.no_runqs);
+    erts_aint32_t no_runqs = erts_atomic32_read_nob(&balance_info.no_runqs);
     if (active)
 	*active = (int) (no_runqs & ERTS_NO_RUNQS_MASK);
     if (used)
@@ -3770,12 +3770,12 @@ get_no_runqs(int *active, int *used)
 static ERTS_INLINE void
 set_no_used_runqs(int used)
 {
-    erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs);
+    erts_aint32_t exp = erts_atomic32_read_nob(&balance_info.no_runqs);
     while (1) {
 	erts_aint32_t act, new;
 	new = (used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT;
 	new |= exp & ERTS_NO_RUNQS_MASK;
-	act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp);
+	act = erts_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp);
 	if (act == exp)
 	    break;
 	exp = act;
@@ -3785,14 +3785,14 @@ set_no_used_runqs(int used)
 static ERTS_INLINE void
 set_no_active_runqs(int active)
 {
-    erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs);
+    erts_aint32_t exp = erts_atomic32_read_nob(&balance_info.no_runqs);
     while (1) {
 	erts_aint32_t act, new;
 	if ((exp & ERTS_NO_RUNQS_MASK) == active)
 	    break;
 	new = exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT);
 	new |= active & ERTS_NO_RUNQS_MASK;
-	act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp);
+	act = erts_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp);
 	if (act == exp)
 	    break;
 	exp = act;
@@ -3802,14 +3802,14 @@ set_no_active_runqs(int active)
 static ERTS_INLINE int
 try_inc_no_active_runqs(int active)
 {
-    erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs);
+    erts_aint32_t exp = erts_atomic32_read_nob(&balance_info.no_runqs);
     if (((exp >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK) < active)
 	return 0;
     if ((exp & ERTS_NO_RUNQS_MASK) + 1 == active) {
 	erts_aint32_t new, act;
 	new = exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT);
 	new |= active & ERTS_NO_RUNQS_MASK;
-	act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp);
+	act = erts_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp);
 	if (act == exp)
 	    return 1;
     }
@@ -3886,7 +3886,7 @@ smp_notify_inc_runq(ErtsRunQueue *runq)
 }
 
 void
-erts_smp_notify_inc_runq(ErtsRunQueue *runq)
+erts_notify_inc_runq(ErtsRunQueue *runq)
 {
     smp_notify_inc_runq(runq);
 }
@@ -3908,9 +3908,9 @@ enqueue_process(ErtsRunQueue *runq, int prio, Process *p)
 {
     ErtsRunPrioQueue *rpq;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq));
 
-    erts_smp_inc_runq_len(runq, &runq->procs.prio_info[prio], prio);
+    erts_inc_runq_len(runq, &runq->procs.prio_info[prio], prio);
 
     if (prio == PRIORITY_LOW) {
 	p->schedule_count = RESCHEDULE_LOW;
@@ -3938,7 +3938,7 @@ unqueue_process(ErtsRunQueue *runq,
 		Process *prev_proc,
 		Process *proc)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq));
 
     if (prev_proc)
 	prev_proc->next = proc->next;
@@ -3950,7 +3950,7 @@ unqueue_process(ErtsRunQueue *runq,
     if (!rpq->first)
 	rpq->last = NULL;
 
-    erts_smp_dec_runq_len(runq, rqi, prio);
+    erts_dec_runq_len(runq, rqi, prio);
 }
 
 
@@ -3963,7 +3963,7 @@ dequeue_process(ErtsRunQueue *runq, int prio_q, erts_aint32_t *statep)
     ErtsRunQueueInfo *rqi;
     Process *p;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq));
 
     ASSERT(PRIORITY_NORMAL == prio_q
 	   || PRIORITY_HIGH == prio_q
@@ -3974,9 +3974,9 @@ dequeue_process(ErtsRunQueue *runq, int prio_q, erts_aint32_t *statep)
     if (!p)
 	return NULL;
 
-    ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
+    ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
 
-    state = erts_smp_atomic32_read_nob(&p->state);
+    state = erts_atomic32_read_nob(&p->state);
     if (statep)
 	*statep = state;
 
@@ -4009,7 +4009,7 @@ check_requeue_process(ErtsRunQueue *rq, int prio_q)
 static ERTS_INLINE void
 free_proxy_proc(Process *proxy)
 {
-    ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY);
+    ASSERT(erts_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY);
     erts_free(ERTS_ALC_T_PROC, proxy);
 }
 
@@ -4065,7 +4065,7 @@ static void
 immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp)
 {
     Uint32 iflags, iflag;
-    erts_smp_runq_unlock(c_rq);
+    erts_runq_unlock(c_rq);
 
     ASSERT(erts_thr_progress_is_managed_thread());
 
@@ -4106,13 +4106,13 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp)
 
 	rq = check_immigration_need(c_rq, mp, prio);
 	if (rq) {
-	    erts_smp_runq_lock(rq);
+	    erts_runq_lock(rq);
 	    if (prio == ERTS_PORT_PRIO_LEVEL) {
 		Port *prt;
 		prt = erts_dequeue_port(rq);
 		if (prt)
 		    RUNQ_SET_RQ(&prt->run_queue, c_rq);
-		erts_smp_runq_unlock(rq);
+		erts_runq_unlock(rq);
 		if (prt) {
 		    /* port might terminate while we have no lock... */
 		    rq = erts_port_runq(prt);
@@ -4124,7 +4124,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp)
 			erts_enqueue_port(c_rq, prt);
 			if (!iflag)
 			    return; /* done */
-			erts_smp_runq_unlock(c_rq);
+			erts_runq_unlock(c_rq);
 		    }
 		}
 	    }
@@ -4138,38 +4138,38 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp)
 
 		while (proc) {
 		    erts_aint32_t state;
-		    state = erts_smp_atomic32_read_acqb(&proc->state);
+		    state = erts_atomic32_read_acqb(&proc->state);
 		    if (!(ERTS_PSFLG_BOUND & state)
 			&& (prio == (int) ERTS_PSFLGS_GET_PRQ_PRIO(state))) {
 			ErtsRunQueueInfo *rqi = &rq->procs.prio_info[prio];
 			unqueue_process(rq, rpq, rqi, prio, prev_proc, proc);
-			erts_smp_runq_unlock(rq);
+			erts_runq_unlock(rq);
 			RUNQ_SET_RQ(&proc->run_queue, c_rq);
 			rq_locked = 0;
 
-			erts_smp_runq_lock(c_rq);
+			erts_runq_lock(c_rq);
 			enqueue_process(c_rq, prio, proc);
 			if (!iflag)
 			    return; /* done */
-			erts_smp_runq_unlock(c_rq);
+			erts_runq_unlock(c_rq);
 			break;
 		    }
 		    prev_proc = proc;
 		    proc = proc->next;
 		}
 		if (rq_locked)
-		    erts_smp_runq_unlock(rq);
+		    erts_runq_unlock(rq);
 	    }
 	}
     }
 
-    erts_smp_runq_lock(c_rq);
+    erts_runq_lock(c_rq);
 }
 
 static ERTS_INLINE void
 suspend_run_queue(ErtsRunQueue *rq)
 {
-    erts_smp_atomic32_read_bor_nob(&rq->scheduler->ssi->flags,
+    erts_atomic32_read_bor_nob(&rq->scheduler->ssi->flags,
 				   ERTS_SSI_FLG_SUSPENDED);
     (void) ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_SUSPENDED);
 
@@ -4186,7 +4186,7 @@ resume_run_queue(ErtsRunQueue *rq)
 
     ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
 
-    erts_smp_runq_lock(rq);
+    erts_runq_lock(rq);
 
     oflgs = ERTS_RUNQ_FLGS_READ_BSET(rq,
 				     (ERTS_RUNQ_FLG_OUT_OF_WORK
@@ -4201,19 +4201,19 @@ resume_run_queue(ErtsRunQueue *rq)
 
 	rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
 	for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) {
-	    len = erts_smp_atomic32_read_dirty(&rq->procs.prio_info[pix].len);
+	    len = erts_atomic32_read_dirty(&rq->procs.prio_info[pix].len);
 	    rq->procs.prio_info[pix].max_len = len;
 	    rq->procs.prio_info[pix].reds = 0;
 	}
-	len = erts_smp_atomic32_read_dirty(&rq->ports.info.len);
+	len = erts_atomic32_read_dirty(&rq->ports.info.len);
 	rq->ports.info.max_len = len;
 	rq->ports.info.reds = 0;
-	len = erts_smp_atomic32_read_dirty(&rq->len);
+	len = erts_atomic32_read_dirty(&rq->len);
 	rq->max_len = len;
 
     }
 
-    erts_smp_runq_unlock(rq);
+    erts_runq_unlock(rq);
 
     nrml_sched_ix_resume_wake(rq->ix);
 }
@@ -4228,11 +4228,11 @@ schedule_bound_processes(ErtsRunQueue *rq,
 			 ErtsStuckBoundProcesses *sbpp)
 {
     Process *proc, *next;
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
 
     proc = sbpp->first;
     while (proc) {
-	erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state);
+	erts_aint32_t state = erts_atomic32_read_acqb(&proc->state);
 	next = proc->next;
 	enqueue_process(rq, (int) ERTS_PSFLGS_GET_PRQ_PRIO(state), proc);
 	proc = next;
@@ -4259,7 +4259,7 @@ clear_proc_dirty_queue_bit(Process *p, ErtsRunQueue *rq, int prio_bit)
 #else
 	(void)
 #endif
-	erts_smp_atomic32_read_band_mb(&p->dirty_state, ~qb);
+	erts_atomic32_read_band_mb(&p->dirty_state, ~qb);
     ASSERT(old & qb);
 }
 
@@ -4275,7 +4275,7 @@ evacuate_run_queue(ErtsRunQueue *rq,
     ErtsMigrationPaths *mps;
     ErtsMigrationPath *mp;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
 
     (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
 
@@ -4298,9 +4298,9 @@ evacuate_run_queue(ErtsRunQueue *rq,
 	rq->misc.start = NULL;
 	rq->misc.end = NULL;
         ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP);
-	erts_smp_runq_unlock(rq);
+	erts_runq_unlock(rq);
 
-	erts_smp_runq_lock(to_rq);
+	erts_runq_lock(to_rq);
 	if (to_rq->misc.end)
 	    to_rq->misc.end->next = start;
 	else
@@ -4310,9 +4310,9 @@ evacuate_run_queue(ErtsRunQueue *rq,
 
 	non_empty_runq(to_rq);
 
-	erts_smp_runq_unlock(to_rq);
+	erts_runq_unlock(to_rq);
 	smp_notify_inc_runq(to_rq);
-	erts_smp_runq_lock(to_rq);
+	erts_runq_lock(to_rq);
     }
 
     if (rq->ports.start) {
@@ -4328,7 +4328,7 @@ evacuate_run_queue(ErtsRunQueue *rq,
 	    ErtsRunQueue *prt_rq;
 	    prt = erts_dequeue_port(rq);
 	    RUNQ_SET_RQ(&prt->run_queue, to_rq);
-	    erts_smp_runq_unlock(rq);
+	    erts_runq_unlock(rq);
 	    /*
 	     * The port might terminate while
 	     * we have no lock on it...
@@ -4340,9 +4340,9 @@ evacuate_run_queue(ErtsRunQueue *rq,
 			     "%s:%d:%s() internal error\n",
 			     __FILE__, __LINE__, __func__);
 		erts_enqueue_port(to_rq, prt);
-		erts_smp_runq_unlock(to_rq);
+		erts_runq_unlock(to_rq);
 	    }
-	    erts_smp_runq_lock(rq);
+	    erts_runq_lock(rq);
 	    prt = rq->ports.start;
 	}
 	smp_notify_inc_runq(to_rq);
@@ -4379,7 +4379,7 @@ evacuate_run_queue(ErtsRunQueue *rq,
 		    free_proxy_proc(proc);
 		    goto handle_next_proc;
 		}
-		real_state =  erts_smp_atomic32_read_acqb(&real_proc->state);
+		real_state =  erts_atomic32_read_acqb(&real_proc->state);
 	    }
 
 	    max_qbit = (state >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET);
@@ -4405,7 +4405,7 @@ evacuate_run_queue(ErtsRunQueue *rq,
 #else
 			(void)
 #endif
-			erts_smp_atomic32_read_band_mb(&proc->state,
+			erts_atomic32_read_band_mb(&proc->state,
 						       ~clr_bits);
 			ASSERT((old & clr_bits) == clr_bits);
 
@@ -4425,17 +4425,17 @@ evacuate_run_queue(ErtsRunQueue *rq,
 	    }
 	    else {
 		int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state);
-		erts_smp_runq_unlock(rq);
+		erts_runq_unlock(rq);
 
                 to_rq = mp->prio[prio].runq;
 		RUNQ_SET_RQ(&proc->run_queue, to_rq);
 
-		erts_smp_runq_lock(to_rq);
+		erts_runq_lock(to_rq);
 		enqueue_process(to_rq, prio, proc);
-		erts_smp_runq_unlock(to_rq);
+		erts_runq_unlock(to_rq);
 		notify = 1;
 
-		erts_smp_runq_lock(rq);
+		erts_runq_lock(rq);
 	    }
 
 	handle_next_proc:
@@ -4454,13 +4454,13 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq,
     ErtsRunPrioQueue *rpq;
 
     if (*rq_lockedp) {
-	erts_smp_runq_unlock(rq);
+	erts_runq_unlock(rq);
 	*rq_lockedp = 0;
     }
 
-    ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq));
+    ERTS_LC_ASSERT(!erts_lc_runq_is_locked(rq));
 
-    erts_smp_runq_lock(vrq);
+    erts_runq_lock(vrq);
 
     if (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_HALTING)
 	goto no_procs;
@@ -4496,16 +4496,16 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq,
 	proc = rpq->first;
 
 	while (proc) {
-	    erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state);
+	    erts_aint32_t state = erts_atomic32_read_acqb(&proc->state);
 	    if (!(ERTS_PSFLG_BOUND & state)) {
 		/* Steal process */
 		int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state);
 		ErtsRunQueueInfo *rqi = &vrq->procs.prio_info[prio];
 		unqueue_process(vrq, rpq, rqi, prio, prev_proc, proc);
-		erts_smp_runq_unlock(vrq);
+		erts_runq_unlock(vrq);
 		RUNQ_SET_RQ(&proc->run_queue, rq);
 
-		erts_smp_runq_lock(rq);
+		erts_runq_lock(rq);
 		*rq_lockedp = 1;
 		enqueue_process(rq, prio, proc);
 		return !0;
@@ -4519,7 +4519,7 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq,
 
 no_procs:
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(vrq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(vrq));
 
     /*
      * Check for a runnable port to steal...
@@ -4529,7 +4529,7 @@ no_procs:
 	ErtsRunQueue *prt_rq;
 	Port *prt = erts_dequeue_port(vrq);
 	RUNQ_SET_RQ(&prt->run_queue, rq);
-	erts_smp_runq_unlock(vrq);
+	erts_runq_unlock(vrq);
 
 	/*
 	 * The port might terminate while
@@ -4550,7 +4550,7 @@ no_procs:
 	}
     }
 
-    erts_smp_runq_unlock(vrq);
+    erts_runq_unlock(vrq);
 
     return 0;
 }
@@ -4582,7 +4582,7 @@ try_steal_task(ErtsRunQueue *rq)
     res = 0;
     rq_locked = 1;
 
-    ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, rq_locked);
+    ERTS_LC_CHK_RUNQ_LOCK(rq, rq_locked);
 
     get_no_runqs(&active_rqs, &blnc_rqs);
 
@@ -4595,7 +4595,7 @@ try_steal_task(ErtsRunQueue *rq)
 	if (active_rqs < blnc_rqs) {
 	    int no = blnc_rqs - active_rqs;
 	    int stop_ix = vix = active_rqs + rq->ix % no;
-	    while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) {
+	    while (erts_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) {
 		res = check_possible_steal_victim(rq, &rq_locked, vix);
 		if (res)
 		    goto done;
@@ -4610,7 +4610,7 @@ try_steal_task(ErtsRunQueue *rq)
 	vix = rq->ix;
 
 	/* ... then try to steal a job from another active queue... */
-	while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) {
+	while (erts_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) {
 	    vix++;
 	    if (vix >= active_rqs)
 		vix = 0;
@@ -4627,7 +4627,7 @@ try_steal_task(ErtsRunQueue *rq)
  done:
 
     if (!rq_locked)
-	erts_smp_runq_lock(rq);
+	erts_runq_lock(rq);
 
     if (res)
         return res;
@@ -4753,7 +4753,7 @@ alloc_mpaths(void)
 {
     void *block;
     ErtsMigrationPaths *res;
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&balance_info.update_mtx));
+    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&balance_info.update_mtx));
 
     res = mpaths.freelist;
     if (res) {
@@ -4776,7 +4776,7 @@ retire_mpaths(ErtsMigrationPaths *mps)
 {
     ErtsThrPrgrVal current;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&balance_info.update_mtx));
+    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&balance_info.update_mtx));
 
     current = erts_thr_progress_current();
 
@@ -4822,7 +4822,7 @@ check_balance(ErtsRunQueue *c_rq)
     int sched_util_balancing;
 #endif
 
-    if (erts_smp_atomic32_xchg_nob(&balance_info.checking_balance, 1)) {
+    if (erts_atomic32_xchg_nob(&balance_info.checking_balance, 1)) {
 	c_rq->check_balance_reds = INT_MAX;
 	return;
     }
@@ -4830,15 +4830,15 @@ check_balance(ErtsRunQueue *c_rq)
     get_no_runqs(NULL, &blnc_no_rqs);
     if (blnc_no_rqs == 1) {
 	c_rq->check_balance_reds = INT_MAX;
-	erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0);
+	erts_atomic32_set_nob(&balance_info.checking_balance, 0);
 	return;
     }
 
-    erts_smp_runq_unlock(c_rq);
+    erts_runq_unlock(c_rq);
 
     if (balance_info.halftime) {	
 	balance_info.halftime = 0;
-	erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0);
+	erts_atomic32_set_nob(&balance_info.checking_balance, 0);
 	ERTS_FOREACH_RUNQ(rq,
 	{
 	    if (rq->waiting)
@@ -4848,7 +4848,7 @@ check_balance(ErtsRunQueue *c_rq)
 	    rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
 	});
 
-	erts_smp_runq_lock(c_rq);
+	erts_runq_lock(c_rq);
 	return;
     }
 
@@ -4861,7 +4861,7 @@ check_balance(ErtsRunQueue *c_rq)
      * is manipulated. Such updates of the migration information
      * might clash with balancing.
      */
-    erts_smp_mtx_lock(&balance_info.update_mtx);
+    erts_mtx_lock(&balance_info.update_mtx);
 
     forced = balance_info.forced_check_balance;
     balance_info.forced_check_balance = 0;
@@ -4869,10 +4869,10 @@ check_balance(ErtsRunQueue *c_rq)
     get_no_runqs(&current_active, &blnc_no_rqs);
 
     if (blnc_no_rqs == 1) {
-	erts_smp_mtx_unlock(&balance_info.update_mtx);
-	erts_smp_runq_lock(c_rq);
+	erts_mtx_unlock(&balance_info.update_mtx);
+	erts_runq_lock(c_rq);
 	c_rq->check_balance_reds = INT_MAX;
-	erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0);
+	erts_atomic32_set_nob(&balance_info.checking_balance, 0);
 	return;
     }
 
@@ -4888,7 +4888,7 @@ check_balance(ErtsRunQueue *c_rq)
     /* Read balance information for all run queues */
     for (qix = 0; qix < blnc_no_rqs; qix++) {
 	ErtsRunQueue *rq = ERTS_RUNQ_IX(qix);
-	erts_smp_runq_lock(rq);
+	erts_runq_lock(rq);
 
 	run_queue_info[qix].flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
 	for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) {
@@ -4916,7 +4916,7 @@ check_balance(ErtsRunQueue *c_rq)
 	    run_queue_info[qix].sched_util = erts_get_sched_util(rq, 1, 0);
 #endif
 
-	erts_smp_runq_unlock(rq);
+	erts_runq_unlock(rq);
     }
 
     full_scheds = 0;
@@ -5355,7 +5355,7 @@ erts_fprintf(stderr, "--------------------------------\n");
 	Uint32 flags = run_queue_info[qix].flags;
 	ErtsRunQueue *rq = ERTS_RUNQ_IX(qix);
 
-	erts_smp_runq_lock(rq);
+	erts_runq_lock(rq);
 	ASSERT(!(flags & ERTS_RUNQ_FLG_OUT_OF_WORK));
 	if (rq->waiting)
 	    flags |= ERTS_RUNQ_FLG_OUT_OF_WORK;
@@ -5370,27 +5370,27 @@ erts_fprintf(stderr, "--------------------------------\n");
 	rq->out_of_work_count = 0;
 	(void) ERTS_RUNQ_FLGS_READ_BSET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags);
 
-	rq->max_len = erts_smp_atomic32_read_dirty(&rq->len);
+	rq->max_len = erts_atomic32_read_dirty(&rq->len);
 	for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) {
 	    ErtsRunQueueInfo *rqi;
 	    rqi = (pix == ERTS_PORT_PRIO_LEVEL
 		   ? &rq->ports.info
 		   : &rq->procs.prio_info[pix]);
-	    erts_smp_reset_max_len(rq, rqi);
+	    erts_reset_max_len(rq, rqi);
 	    rqi->reds = 0;
 	}
 
 	rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
-	erts_smp_runq_unlock(rq);
+	erts_runq_unlock(rq);
     }
 
-    erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0);
+    erts_atomic32_set_nob(&balance_info.checking_balance, 0);
 
     balance_info.n++;
     retire_mpaths(old_mpaths);
-    erts_smp_mtx_unlock(&balance_info.update_mtx);
+    erts_mtx_unlock(&balance_info.update_mtx);
 
-    erts_smp_runq_lock(c_rq);
+    erts_runq_lock(c_rq);
 }
 
 static void
@@ -5398,7 +5398,7 @@ change_no_used_runqs(int used)
 {
     ErtsMigrationPaths *new_mpaths, *old_mpaths;
     int qix;
-    erts_smp_mtx_lock(&balance_info.update_mtx);
+    erts_mtx_lock(&balance_info.update_mtx);
     set_no_used_runqs(used);
 
     old_mpaths = erts_get_migration_paths_managed();
@@ -5445,11 +5445,11 @@ change_no_used_runqs(int used)
     /* Make sure that we balance soon... */
     balance_info.forced_check_balance = 1;
 
-    erts_smp_mtx_unlock(&balance_info.update_mtx);
+    erts_mtx_unlock(&balance_info.update_mtx);
 
-    erts_smp_runq_lock(ERTS_RUNQ_IX(0));
+    erts_runq_lock(ERTS_RUNQ_IX(0));
     ERTS_RUNQ_IX(0)->check_balance_reds = 0;
-    erts_smp_runq_unlock(ERTS_RUNQ_IX(0));
+    erts_runq_unlock(ERTS_RUNQ_IX(0));
 }
 
 
@@ -5458,9 +5458,9 @@ Uint
 erts_debug_nbalance(void)
 {
     Uint n;
-    erts_smp_mtx_lock(&balance_info.update_mtx);
+    erts_mtx_lock(&balance_info.update_mtx);
     n = balance_info.n;
-    erts_smp_mtx_unlock(&balance_info.update_mtx);
+    erts_mtx_unlock(&balance_info.update_mtx);
     return n;
 }
 
@@ -5522,7 +5522,7 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags)
 {
     int wo_reds = rq->wakeup_other_reds;
     if (wo_reds) {
-	int left_len = erts_smp_atomic32_read_dirty(&rq->len) - 1;
+	int left_len = erts_atomic32_read_dirty(&rq->len) - 1;
 	if (left_len < 1) {
 	    int wo_reduce = wo_reds << wakeup_other.dec_shift;
 	    wo_reduce &= wakeup_other.dec_mask;
@@ -5543,7 +5543,7 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags)
 #endif
 		{
 		    int empty_rqs =
-			erts_smp_atomic32_read_acqb(&no_empty_run_queues);
+			erts_atomic32_read_acqb(&no_empty_run_queues);
 		    if (flags & ERTS_RUNQ_FLG_PROTECTED)
 			(void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
 		    if (empty_rqs != 0)
@@ -5595,7 +5595,7 @@ wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags)
 {
     int wo_reds = rq->wakeup_other_reds;
     if (wo_reds) {
-	erts_aint32_t len = erts_smp_atomic32_read_dirty(&rq->len);
+	erts_aint32_t len = erts_atomic32_read_dirty(&rq->len);
 	if (len < 2) {
 	    rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC_LEGACY*wo_reds;
 	    if (rq->wakeup_other < 0)
@@ -5606,7 +5606,7 @@ wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags)
 	else {
 	    if (flags & ERTS_RUNQ_FLG_PROTECTED)
 		(void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
-	    if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) {
+	    if (erts_atomic32_read_acqb(&no_empty_run_queues) != 0) {
 		wake_scheduler_on_empty_runq(rq);
 		rq->wakeup_other = 0;
 	    }
@@ -5657,7 +5657,7 @@ static int
 no_runqs_to_supervise(void)
 {
     int used;
-    erts_aint32_t nerq = erts_smp_atomic32_read_acqb(&no_empty_run_queues);
+    erts_aint32_t nerq = erts_atomic32_read_acqb(&no_empty_run_queues);
     if (nerq <= 0)
 	return 0;
     get_no_runqs(NULL, &used);
@@ -5690,10 +5690,10 @@ runq_supervisor(void *unused)
 	for (ix = 0; ix < no_rqs; ix++) {
 	    ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
 	    if (ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY) {
-		erts_smp_runq_lock(rq);
-		if (erts_smp_atomic32_read_dirty(&rq->len) != 0)
+		erts_runq_lock(rq);
+		if (erts_atomic32_read_dirty(&rq->len) != 0)
 		    wake_scheduler_on_empty_runq(rq); /* forced wakeup... */
-		erts_smp_runq_unlock(rq);
+		erts_runq_unlock(rq);
 	    }
 	}
     }
@@ -5915,7 +5915,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     esdp->dirty_shadow_process = shadow_proc;
     if (shadow_proc) {
 	erts_init_empty_process(shadow_proc);
-	erts_smp_atomic32_init_nob(&shadow_proc->state,
+	erts_atomic32_init_nob(&shadow_proc->state,
 				   (ERTS_PSFLG_ACTIVE
 				    | ERTS_PSFLG_DIRTY_RUNNING
 				    | ERTS_PSFLG_PROXY));
@@ -5998,7 +5998,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     size_runqs = sizeof(ErtsAlignedRunQueue) * tot_rqs;
     erts_aligned_run_queues =
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, size_runqs);
-    erts_smp_atomic32_init_nob(&no_empty_run_queues, 0);
+    erts_atomic32_init_nob(&no_empty_run_queues, 0);
 
     erts_no_run_queues = n;
 
@@ -6012,13 +6012,13 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	 * id if the esdp->no <-> ix+1 mapping change.
 	 */
 
-	erts_smp_mtx_init(&rq->mtx, "run_queue", make_small(ix + 1),
+	erts_mtx_init(&rq->mtx, "run_queue", make_small(ix + 1),
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
-	erts_smp_cnd_init(&rq->cnd);
+	erts_cnd_init(&rq->cnd);
 
 #ifdef ERTS_DIRTY_SCHEDULERS
         if (ERTS_RUNQ_IX_IS_DIRTY(ix)) {
-            erts_smp_spinlock_init(&rq->sleepers.lock, "dirty_run_queue_sleep_list",
+            erts_spinlock_init(&rq->sleepers.lock, "dirty_run_queue_sleep_list",
                 make_small(ix + 1),
                 ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
         }
@@ -6036,7 +6036,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	}
 	rq->out_of_work_count = 0;
 	rq->max_len = 0;
-	erts_smp_atomic32_set_nob(&rq->len, 0);
+	erts_atomic32_set_nob(&rq->len, 0);
 	rq->wakeup_other = 0;
 	rq->wakeup_other_reds = 0;
 
@@ -6045,7 +6045,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	rq->procs.reductions = 0;
 
 	for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) {
-	    erts_smp_atomic32_init_nob(&rq->procs.prio_info[pix].len, 0);
+	    erts_atomic32_init_nob(&rq->procs.prio_info[pix].len, 0);
 	    rq->procs.prio_info[pix].max_len = 0;
 	    rq->procs.prio_info[pix].reds = 0;
 	    if (pix < ERTS_NO_PROC_PRIO_LEVELS - 1) {
@@ -6057,7 +6057,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	rq->misc.start = NULL;
 	rq->misc.end = NULL;
 
-	erts_smp_atomic32_init_nob(&rq->ports.info.len, 0);
+	erts_atomic32_init_nob(&rq->ports.info.len, 0);
 	rq->ports.info.max_len = 0;
 	rq->ports.info.reds = 0;
 	rq->ports.start = NULL;
@@ -6102,7 +6102,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	ssi->next = NULL;
 	ssi->prev = NULL;
 #endif
-	erts_smp_atomic32_init_nob(&ssi->flags, 0);
+	erts_atomic32_init_nob(&ssi->flags, 0);
 	ssi->event = NULL; /* initialized in sched_thread_func */
 	erts_atomic32_init_nob(&ssi->aux_work, 0);
     }
@@ -6116,7 +6116,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	    no_dirty_cpu_schedulers*sizeof(ErtsAlignedSchedulerSleepInfo));
     for (ix = 0; ix < no_dirty_cpu_schedulers; ix++) {
 	ErtsSchedulerSleepInfo *ssi = &aligned_dirty_cpu_sched_sleep_info[ix].ssi;
-	erts_smp_atomic32_init_nob(&ssi->flags, 0);
+	erts_atomic32_init_nob(&ssi->flags, 0);
 	ssi->event = NULL; /* initialized in sched_dirty_cpu_thread_func */
 	erts_atomic32_init_nob(&ssi->aux_work, 0);
     }
@@ -6126,7 +6126,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	    no_dirty_io_schedulers*sizeof(ErtsAlignedSchedulerSleepInfo));
     for (ix = 0; ix < no_dirty_io_schedulers; ix++) {
 	ErtsSchedulerSleepInfo *ssi = &aligned_dirty_io_sched_sleep_info[ix].ssi;
-	erts_smp_atomic32_init_nob(&ssi->flags, 0);
+	erts_atomic32_init_nob(&ssi->flags, 0);
 	ssi->event = NULL; /* initialized in sched_dirty_io_thread_func */
 	erts_atomic32_init_nob(&ssi->aux_work, 0);
     }
@@ -6197,12 +6197,12 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 
     init_no_runqs(no_schedulers_online, no_schedulers_online);
     balance_info.last_active_runqs = no_schedulers;
-    erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update", NIL,
+    erts_mtx_init(&balance_info.update_mtx, "migration_info_update", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
     balance_info.forced_check_balance = 0;
     balance_info.halftime = 1;
     balance_info.full_reds_history_index = 0;
-    erts_smp_atomic32_init_nob(&balance_info.checking_balance, 0);
+    erts_atomic32_init_nob(&balance_info.checking_balance, 0);
     balance_info.prev_rise.active_runqs = 0;
     balance_info.prev_rise.max_len = 0;
     balance_info.prev_rise.reds = 0;
@@ -6250,7 +6250,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	set_schdlr_sspnd_change_flags |= ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN;
 	for (ix = no_dirty_cpu_schedulers_online; ix < no_dirty_cpu_schedulers; ix++) {
 	    ErtsSchedulerData* esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
-	    erts_smp_atomic32_read_bor_nob(&esdp->ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+	    erts_atomic32_read_bor_nob(&esdp->ssi->flags, ERTS_SSI_FLG_SUSPENDED);
 	}
     }
 
@@ -6264,29 +6264,29 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 			     ERTS_SCHED_DIRTY_IO,
 			     no_dirty_io_schedulers);
 
-    erts_smp_atomic32_init_nob(&dirty_count.cpu.active,
+    erts_atomic32_init_nob(&dirty_count.cpu.active,
                                (erts_aint32_t) no_dirty_cpu_schedulers);
-    erts_smp_atomic32_init_nob(&dirty_count.io.active,
+    erts_atomic32_init_nob(&dirty_count.io.active,
                                (erts_aint32_t) no_dirty_io_schedulers);
 
 #endif
 
     if (set_schdlr_sspnd_change_flags)
-	erts_smp_atomic32_set_nob(&schdlr_sspnd.changing,
+	erts_atomic32_set_nob(&schdlr_sspnd.changing,
 				  set_schdlr_sspnd_change_flags);
 
-    erts_smp_atomic32_init_nob(&doing_sys_schedule, 0);
+    erts_atomic32_init_nob(&doing_sys_schedule, 0);
 
     init_misc_aux_work();
 
 
-    erts_smp_atomic32_init_nob(&function_calls, 0);
+    erts_atomic32_init_nob(&function_calls, 0);
 
     /* init port tasks */
     erts_port_task_init();
 
 
-    erts_smp_atomic32_init_relb(&erts_halt_progress, -1);
+    erts_atomic32_init_relb(&erts_halt_progress, -1);
     erts_halt_code = 0;
 
 
@@ -6325,8 +6325,8 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
 
     if (prev_proxy) {
 	proxy = prev_proxy;
-	ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY);
-	erts_smp_atomic32_set_nob(&proxy->state, state);
+	ASSERT(erts_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY);
+	erts_atomic32_set_nob(&proxy->state, state);
 	RUNQ_SET_RQ(&proc->run_queue, rq);
     }
     else {
@@ -6339,9 +6339,9 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
 		ui32[i] = (Uint32) 0xdeadbeef;
 	}
 #endif
-	erts_smp_atomic32_init_nob(&proxy->state, state);
-	erts_smp_atomic_init_nob(&proxy->run_queue,
-				 erts_smp_atomic_read_nob(&proc->run_queue));
+	erts_atomic32_init_nob(&proxy->state, state);
+	erts_atomic_init_nob(&proxy->run_queue,
+				 erts_atomic_read_nob(&proc->run_queue));
     }
 
     proxy->common.id = proc->common.id;
@@ -6385,7 +6385,7 @@ check_dirty_enqueue_in_prio_queue(Process *c_p,
     if ((*newp) & ERTS_PSFLG_ACTIVE_SYS)
 	return ERTS_ENQUEUE_NORMAL_QUEUE;
 
-    dact = erts_smp_atomic32_read_mb(&c_p->dirty_state);
+    dact = erts_atomic32_read_mb(&c_p->dirty_state);
     if (actual & (ERTS_PSFLG_DIRTY_ACTIVE_SYS
 		  | ERTS_PSFLG_DIRTY_CPU_PROC)) {
 	max_qbit = ((dact >> ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET)
@@ -6430,7 +6430,7 @@ fin_dirty_enq_s_change(Process *p,
     erts_aint32_t qbit = 1 << enq_prio;
     qbit <<= qmask_offset;
 
-    if (qbit & erts_smp_atomic32_read_bor_mb(&p->dirty_state, qbit)) {
+    if (qbit & erts_atomic32_read_bor_mb(&p->dirty_state, qbit)) {
 	/* Already enqueue by someone else... */
 	if (pstruct_reserved) {
 	    /* We reserved process struct for enqueue; clear it... */
@@ -6439,7 +6439,7 @@ fin_dirty_enq_s_change(Process *p,
 #else
 		(void)
 #endif
-		erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_IN_RUNQ);
+		erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_IN_RUNQ);
 	    ASSERT(old & ERTS_PSFLG_IN_RUNQ);
 	}
 	return 0;
@@ -6592,7 +6592,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
              */
             ASSERT(!(p->flags & (F_DIRTY_CLA | F_DIRTY_GC_HIBERNATE)));
 
-            state = erts_smp_atomic32_read_band_nob(&p->state,
+            state = erts_atomic32_read_band_nob(&p->state,
                                                     ~ERTS_PSFLG_DIRTY_ACTIVE_SYS);
             state &= ~ERTS_PSFLG_DIRTY_ACTIVE_SYS;
         }
@@ -6613,7 +6613,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
 	    || (a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) {
 	    enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a);
 	}
-	a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
+	a = erts_atomic32_cmpxchg_mb(&p->state, n, e);
 	if (a == e)
 	    break;
     }
@@ -6625,7 +6625,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
 	if (erts_system_profile_flags.runnable_procs) {
 
 	    /* Status lock prevents out of order "runnable proc" trace msgs */
-	    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
+	    ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
 
 	    if (!(a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS))
 		&& (!(a & ERTS_PSFLG_ACTIVE) || (a & ERTS_PSFLG_SUSPENDED))) {
@@ -6637,7 +6637,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
 	if (proxy)
 	    free_proxy_proc(proxy);
 
-	erts_smp_runq_lock(c_rq);
+	erts_runq_lock(c_rq);
 
         /* Decrement refc if scheduled out from dirty scheduler... */
 	return !is_normal_sched;
@@ -6659,7 +6659,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
 
 	ASSERT(runq);
 
-	erts_smp_runq_lock(runq);
+	erts_runq_lock(runq);
 
         if (is_normal_sched && sched_p == p && ERTS_RUNQ_IX_IS_DIRTY(runq->ix))
             erts_proc_inc_refc(p); /* Needs to be done before enqueue_process() */
@@ -6670,11 +6670,11 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
 	if (runq == c_rq)
 	    return 0;
 
-	erts_smp_runq_unlock(runq);
+	erts_runq_unlock(runq);
 
 	smp_notify_inc_runq(runq);
 
-	erts_smp_runq_lock(c_rq);
+	erts_runq_lock(c_rq);
 
         /*
          * Decrement refc if process is scheduled out by a
@@ -6722,12 +6722,12 @@ add2runq(int enqueue, erts_aint32_t prio,
 	    sched_p = make_proxy_proc(pxy, proc, prio);
 	}
 
-	erts_smp_runq_lock(runq);
+	erts_runq_lock(runq);
 
 	/* Enqueue the process */
 	enqueue_process(runq, (int) prio, sched_p);
 
-	erts_smp_runq_unlock(runq);
+	erts_runq_unlock(runq);
 	smp_notify_inc_runq(runq);
     }
 }
@@ -6752,7 +6752,7 @@ change_proc_schedule_state(Process *p,
     unsigned int lock_status = (prof_runnable_procs
 				&& !(locks & ERTS_PROC_LOCK_STATUS));
 
-    ERTS_SMP_LC_ASSERT(locks == erts_proc_lc_my_proc_locks(p));
+    ERTS_LC_ASSERT(locks == erts_proc_lc_my_proc_locks(p));
 
     ASSERT(!(a & ERTS_PSFLG_PROXY));
     ASSERT((clear_state_flags & (ERTS_PSFLG_RUNNING
@@ -6767,7 +6767,7 @@ change_proc_schedule_state(Process *p,
 			       | ERTS_PSFLG_ACTIVE_SYS)) == 0);
 
     if (lock_status)
-	erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+	erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
 
     while (1) {
 	erts_aint32_t e;
@@ -6805,7 +6805,7 @@ change_proc_schedule_state(Process *p,
 	    enqueue = check_enqueue_in_prio_queue(p, enq_prio_p, &n, a);
 	}
 
-	a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
+	a = erts_atomic32_cmpxchg_mb(&p->state, n, e);
 	if (a == e)
 	    break;
 	if (enqueue == ERTS_ENQUEUE_NOT && n == a)
@@ -6830,7 +6830,7 @@ change_proc_schedule_state(Process *p,
 	}
 
 	if (lock_status)
-	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+	    erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
     }
 
 
@@ -6874,7 +6874,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
 
     res = 1; /* prepare for success */
     st->next = st->prev = st; /* Prep for empty prio queue */
-    state = erts_smp_atomic32_read_nob(&p->state);
+    state = erts_atomic32_read_nob(&p->state);
     prof_runnable_procs = erts_system_profile_flags.runnable_procs;
     locked = 0;
     free_stqs = NULL;
@@ -6894,9 +6894,9 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
 
     if (!locked) {
 	locked = 1;
-	erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+	erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
 
-	state = erts_smp_atomic32_read_nob(&p->state);
+	state = erts_atomic32_read_nob(&p->state);
 	if (state & fail_state) {
 	    *fail_state_p = (state & fail_state);
 	    free_stqs = stqs;
@@ -6940,7 +6940,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
 	    n = e = a;
 	    n &= ~ERTS_PSFLGS_ACT_PRIO_MASK;
 	    n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET);
-	    a = erts_smp_atomic32_cmpxchg_nob(&p->state, n, e);    
+	    a = erts_atomic32_cmpxchg_nob(&p->state, n, e);
 	} while (a != e);
 	state = n;
     }
@@ -6950,10 +6950,10 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
     enq_prio = -1;
 
     /* Status lock prevents out of order "runnable proc" trace msgs */
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
 
     if (!prof_runnable_procs) {
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 	locked = 0;
     }
 
@@ -6973,7 +6973,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
 		   | ERTS_PSFLG_DIRTY_RUNNING
 		   | ERTS_PSFLG_DIRTY_RUNNING_SYS)))
 	    enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a);
-	a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
+	a = erts_atomic32_cmpxchg_mb(&p->state, n, e);
 	if (a == e)
 	    break;
 	if (a == n && enqueue == ERTS_ENQUEUE_NOT)
@@ -6992,7 +6992,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
 	    profile_runnable_proc(p, am_active);
 	}
 
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 	locked = 0;
     }
 
@@ -7001,12 +7001,12 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
 cleanup:
 
     if (locked)
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 
     if (free_stqs)
 	proc_sys_task_queues_free(free_stqs);
 
-    ERTS_SMP_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)));
+    ERTS_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)));
 
     return res;
 }
@@ -7016,15 +7016,15 @@ suspend_process(Process *c_p, Process *p)
 {
     erts_aint32_t state;
     int suspended = 0;
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
 
-    state = erts_smp_atomic32_read_acqb(&p->state);
+    state = erts_atomic32_read_acqb(&p->state);
 
     if ((state & ERTS_PSFLG_SUSPENDED))
 	suspended = -1;
     else {
 	if (c_p == p) {
-	    state = erts_smp_atomic32_read_bor_relb(&p->state,
+	    state = erts_atomic32_read_bor_relb(&p->state,
 						    ERTS_PSFLG_SUSPENDED);
 	    ASSERT(state & (ERTS_PSFLG_RUNNING
 			    | ERTS_PSFLG_RUNNING_SYS
@@ -7040,7 +7040,7 @@ suspend_process(Process *c_p, Process *p)
 
 		n = e = state;
 		n |= ERTS_PSFLG_SUSPENDED;
-		state = erts_smp_atomic32_cmpxchg_relb(&p->state, n, e);
+		state = erts_atomic32_cmpxchg_relb(&p->state, n, e);
 		if (state == e) {
 		    suspended = 1;
 		    break;
@@ -7085,14 +7085,14 @@ resume_process(Process *p, ErtsProcLocks locks)
     erts_aint32_t state, enq_prio = -1;
     int enqueue;
 
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
 
     ASSERT(p->rcount > 0);
 
     if (--p->rcount > 0)  /* multiple suspend */
 	return;
 
-    state = erts_smp_atomic32_read_nob(&p->state);
+    state = erts_atomic32_read_nob(&p->state);
     enqueue = change_proc_schedule_state(p,
 					 ERTS_PSFLG_SUSPENDED,
 					 0,
@@ -7112,7 +7112,7 @@ sched_resume_wake__(ErtsSchedulerSleepInfo *ssi)
 			   | ERTS_SSI_FLG_SUSPENDED);
     erts_aint32_t oflgs;
     do {
-	oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, 0, xflgs);
+	oflgs = erts_atomic32_cmpxchg_relb(&ssi->flags, 0, xflgs);
 	if (oflgs == xflgs) {
 	    erts_sched_finish_poke(ssi, oflgs);
 	    break;
@@ -7153,7 +7153,7 @@ sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct)
     erts_aint32_t xflgs = xpct;
 
     do {
-	oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
+	oflgs = erts_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
 	if (oflgs == xflgs)
 	    return nflgs;
 	xflgs = oflgs;
@@ -7170,7 +7170,7 @@ sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount)
     erts_aint32_t flgs;
 
     do {
-	flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+	flgs = erts_atomic32_read_acqb(&ssi->flags);
 	if ((flgs & (ERTS_SSI_FLG_SLEEPING
 		     | ERTS_SSI_FLG_WAITING
 		     | ERTS_SSI_FLG_SUSPENDED))
@@ -7203,7 +7203,7 @@ sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi)
     erts_tse_reset(ssi->event);
 
     while (1) {
-	oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
+	oflgs = erts_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
 	if (oflgs == xflgs)
 	    return nflgs;
 	if ((oflgs & (ERTS_SSI_FLG_SLEEPING
@@ -7221,7 +7221,7 @@ sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi)
 static void
 init_scheduler_suspend(void)
 {
-    erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd", NIL,
+    erts_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
     schdlr_sspnd.online.normal = 1;
     schdlr_sspnd.curr_online.normal = 1;
@@ -7235,7 +7235,7 @@ init_scheduler_suspend(void)
     schdlr_sspnd.active.dirty_io = 0;
     schdlr_sspnd.last_msb_dirty_type = ERTS_SCHED_DIRTY_IO;
 #endif
-    erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0);
+    erts_atomic32_init_nob(&schdlr_sspnd.changing, 0);
     schdlr_sspnd.chngq = NULL;
     schdlr_sspnd.changer = am_false;
     schdlr_sspnd.nmsb.ongoing = 0;
@@ -7266,7 +7266,7 @@ schdlr_sspnd_resume_proc(ErtsSchedType sched_type, Eterm pid)
                            : 0));
     if (p) {
 	resume_process(p, ERTS_PROC_LOCK_STATUS);
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
         if (sched_type != ERTS_SCHED_NORMAL)
             erts_proc_dec_refc(p);
     }
@@ -7465,7 +7465,7 @@ msb_scheduler_type_switch(ErtsSchedType sched_type,
             calls = ERTS_MODIFIED_TIMING_INPUT_REDS + 1;
         else
             calls = INPUT_REDUCTIONS + 1;
-        erts_smp_atomic32_set_nob(&function_calls, calls);
+        erts_atomic32_set_nob(&function_calls, calls);
 
         if ((nrml_prio == ERTS_MSB_NONE_PRIO_BIT)
             & ((dcpu_prio != ERTS_MSB_NONE_PRIO_BIT)
@@ -7500,7 +7500,7 @@ msb_scheduler_type_switch(ErtsSchedType sched_type,
 #else
     (void)
 #endif
-        erts_smp_atomic32_read_bset_mb(&esdp->ssi->flags,
+        erts_atomic32_read_bset_mb(&esdp->ssi->flags,
                                        (ERTS_SSI_FLG_SUSPENDED
                                         | ERTS_SSI_FLG_MSB_EXEC),
                                        ERTS_SSI_FLG_SUSPENDED);
@@ -7527,7 +7527,7 @@ msb_scheduler_type_switch(ErtsSchedType sched_type,
 #else
     (void)
 #endif
-        erts_smp_atomic32_read_bset_mb(&exec_rq->scheduler->ssi->flags,
+        erts_atomic32_read_bset_mb(&exec_rq->scheduler->ssi->flags,
                                        (ERTS_SSI_FLG_SUSPENDED
                                         | ERTS_SSI_FLG_MSB_EXEC),
                                        ERTS_SSI_FLG_MSB_EXEC);
@@ -7595,7 +7595,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
         return;
     }
 
-    if (erts_smp_atomic32_read_nob(&ssi->flags) & ERTS_SSI_FLG_MSB_EXEC) {
+    if (erts_atomic32_read_nob(&ssi->flags) & ERTS_SSI_FLG_MSB_EXEC) {
         ASSERT(no == 1);
         if (!msb_scheduler_type_switch(sched_type, esdp, no))
             return;
@@ -7606,14 +7606,14 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
     if (sched_type != ERTS_SCHED_NORMAL) {
         dirty_active(esdp, -1);
-	erts_smp_runq_unlock(esdp->run_queue);
+	erts_runq_unlock(esdp->run_queue);
         dirty_sched_wall_time_change(esdp, 0);
     }
     else {
         if (no != 1)
             evacuate_run_queue(esdp->run_queue, &sbp);
 
-	erts_smp_runq_unlock(esdp->run_queue);
+	erts_runq_unlock(esdp->run_queue);
 
 	erts_sched_check_cpu_bind_prep_suspend(esdp);
 
@@ -7621,14 +7621,14 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 	    profile_scheduler(make_small(esdp->no), am_inactive);
     }
 
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+    erts_mtx_lock(&schdlr_sspnd.mtx);
 
     flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED);
     if (flgs & ERTS_SSI_FLG_SUSPENDED) {
 
 	schdlr_sspnd_dec_nscheds(&schdlr_sspnd.active, sched_type);
 
-	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+	changing = erts_atomic32_read_nob(&schdlr_sspnd.changing);
 
 	while (1) {
 
@@ -7660,7 +7660,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 		    if (clr_flg) {
 			ErtsProcList *plp, *end_plp;
-			changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+			changing = erts_atomic32_read_band_nob(&schdlr_sspnd.changing,
 								   ~clr_flg);
 			changing &= ~clr_flg;
 			(void) erts_proclist_fetch(&msb[i]->chngq, &end_plp);
@@ -7706,7 +7706,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 			== schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online,
 						    sched_type))) {
 		    ErtsProcList *plp;
-		    changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+		    changing = erts_atomic32_read_band_nob(&schdlr_sspnd.changing,
 							       ~online_flag);
 		    changing &= ~online_flag;
 		    if (sched_type == ERTS_SCHED_NORMAL) {
@@ -7728,11 +7728,11 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 	    }
 
 	    if (curr_online) {
-		flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+		flgs = erts_atomic32_read_acqb(&ssi->flags);
 		if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
 		    break;
 	    }
-	    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+	    erts_mtx_unlock(&schdlr_sspnd.mtx);
 
 	    schdlr_sspnd_resume_procs(sched_type, &resume);
 
@@ -7760,9 +7760,9 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 			if (aux_work && erts_thr_progress_update(esdp))
 			    erts_thr_progress_leader_update(esdp);
 			if (evacuate) {
-			    erts_smp_runq_lock(esdp->run_queue);
+			    erts_runq_lock(esdp->run_queue);
 			    evacuate_run_queue(esdp->run_queue, &sbp);
-			    erts_smp_runq_unlock(esdp->run_queue);
+			    erts_runq_unlock(esdp->run_queue);
 			}
 		    }
 
@@ -7860,23 +7860,23 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 						       | ERTS_SSI_FLG_SUSPENDED));
 		if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
 		    break;
-		changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+		changing = erts_atomic32_read_nob(&schdlr_sspnd.changing);
 		if (changing)
 		    break;
 	    }
 
-	    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-	    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+	    erts_mtx_lock(&schdlr_sspnd.mtx);
+	    changing = erts_atomic32_read_nob(&schdlr_sspnd.changing);
 	}
 
 	schdlr_sspnd_inc_nscheds(&schdlr_sspnd.active, sched_type);
-	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+	changing = erts_atomic32_read_nob(&schdlr_sspnd.changing);
         if (changing) {
             if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
                 && !schdlr_sspnd.msb.ongoing
                 && schdlr_sspnd_eq_nscheds(&schdlr_sspnd.online,
                                            &schdlr_sspnd.active)) {
-                erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+                erts_atomic32_read_band_nob(&schdlr_sspnd.changing,
                                                 ~ERTS_SCHDLR_SSPND_CHNG_MSB);
             }
             if ((changing & ERTS_SCHDLR_SSPND_CHNG_NMSB)
@@ -7885,14 +7885,14 @@ suspend_scheduler(ErtsSchedulerData *esdp)
                                              ERTS_SCHED_NORMAL)
                     == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
                                                 ERTS_SCHED_NORMAL))) {
-                erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+                erts_atomic32_read_band_nob(&schdlr_sspnd.changing,
                                                 ~ERTS_SCHDLR_SSPND_CHNG_NMSB);
             }
         }
 	ASSERT(no <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, sched_type));
     }
 
-    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+    erts_mtx_unlock(&schdlr_sspnd.mtx);
 
     schdlr_sspnd_resume_procs(sched_type, &resume);
 
@@ -7911,7 +7911,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
         }
     }
 
-    erts_smp_runq_lock(esdp->run_queue);
+    erts_runq_lock(esdp->run_queue);
     non_empty_runq(esdp->run_queue);
 
     if (sched_type != ERTS_SCHED_NORMAL)
@@ -7935,7 +7935,7 @@ erts_schedulers_state(Uint *total,
 {
     if (active || online || dirty_cpu_online
 	|| dirty_cpu_active || dirty_io_active) {
-	erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+	erts_mtx_lock(&schdlr_sspnd.mtx);
 	if (active)
 	    *active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
 					       ERTS_SCHED_NORMAL);
@@ -7951,7 +7951,7 @@ erts_schedulers_state(Uint *total,
 	if (dirty_io_active)
 	    *dirty_io_active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
 							ERTS_SCHED_DIRTY_IO);
-	erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+	erts_mtx_unlock(&schdlr_sspnd.mtx);
     }
 
     if (total)
@@ -7967,7 +7967,7 @@ abort_sched_onln_chng_waitq(Process *p)
 {
     Eterm resume = NIL;
 
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+    erts_mtx_lock(&schdlr_sspnd.mtx);
 
 #ifdef DEBUG
     {
@@ -8017,7 +8017,7 @@ abort_sched_onln_chng_waitq(Process *p)
 	}
     }
 
-    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+    erts_mtx_unlock(&schdlr_sspnd.mtx);
 
     if (is_internal_pid(resume))
 	schdlr_sspnd_resume_proc(ERTS_SCHED_NORMAL, resume);
@@ -8060,13 +8060,13 @@ erts_set_schedulers_online(Process *p,
 	 * race...
 	 */
 	if (!(plocks & ERTS_PROC_LOCK_STATUS))
-	    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+	    erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
 	suspend_process(p, p);
 	if (!(plocks & ERTS_PROC_LOCK_STATUS))
-	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+	    erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
     }
 
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+    erts_mtx_lock(&schdlr_sspnd.mtx);
 
     change_flags = 0;
     have_unlocked_plocks = 0;
@@ -8076,7 +8076,7 @@ erts_set_schedulers_online(Process *p,
     if (!dirty_only)
 #endif
     {
-	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+	changing = erts_atomic32_read_nob(&schdlr_sspnd.changing);
 	if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) {
 	enqueue_wait:
 	    p->flags |= F_SCHDLR_ONLN_WAITQ;
@@ -8168,7 +8168,7 @@ erts_set_schedulers_online(Process *p,
 	increase = (no > online);
     }
 
-    erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, change_flags);
+    erts_atomic32_read_bor_nob(&schdlr_sspnd.changing, change_flags);
 
     res = ERTS_SCHDLR_SSPND_DONE;
     if (increase) {
@@ -8196,7 +8196,7 @@ erts_set_schedulers_online(Process *p,
 	    else {
 		if (plocks) {
 		    have_unlocked_plocks = 1;
-		    erts_smp_proc_unlock(p, plocks);
+		    erts_proc_unlock(p, plocks);
 		}
 		change_no_used_runqs(no);
 
@@ -8236,7 +8236,7 @@ erts_set_schedulers_online(Process *p,
 	    else {
 		if (plocks) {
 		    have_unlocked_plocks = 1;
-		    erts_smp_proc_unlock(p, plocks);
+		    erts_proc_unlock(p, plocks);
 		}
 
 		change_no_used_runqs(no);
@@ -8265,17 +8265,17 @@ done:
 	   <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
 				       ERTS_SCHED_NORMAL));
 
-    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+    erts_mtx_unlock(&schdlr_sspnd.mtx);
 
     if (have_unlocked_plocks)
-	erts_smp_proc_lock(p, plocks);
+	erts_proc_lock(p, plocks);
 
     if (resume_proc) {
 	if (!(plocks & ERTS_PROC_LOCK_STATUS))
-	    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+	    erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
 	resume_process(p, plocks|ERTS_PROC_LOCK_STATUS);
 	if (!(plocks & ERTS_PROC_LOCK_STATUS))
-	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+	    erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
     }
 
     return res;
@@ -8313,13 +8313,13 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
     else {
 	resume_proc = 1;
 	if (!(plocks & ERTS_PROC_LOCK_STATUS))
-	    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+	    erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
 	suspend_process(p, p);
 	if (!(plocks & ERTS_PROC_LOCK_STATUS))
-	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+	    erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
     }
 
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+    erts_mtx_lock(&schdlr_sspnd.mtx);
     if (on) { /* ------ BLOCK ------ */
 	if (msbp->chngq) {
 	    ASSERT(msbp->ongoing);
@@ -8349,12 +8349,12 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 	    p->flags |= have_blckd_flg;
 	    if (plocks) {
 		have_unlocked_plocks = 1;
-		erts_smp_proc_unlock(p, plocks);
+		erts_proc_unlock(p, plocks);
 	    }
 	    ASSERT(!msbp->ongoing);
 	    msbp->ongoing = 1;
 
-            erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing,
+            erts_atomic32_read_bor_nob(&schdlr_sspnd.changing,
                                            chng_flg);
             change_no_used_runqs(1);
             for (ix = 1; ix < erts_no_run_queues; ix++)
@@ -8368,7 +8368,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 #ifdef ERTS_DIRTY_SCHEDULERS
             if (!normal) {
                 ERTS_RUNQ_FLGS_SET_NOB(ERTS_RUNQ_IX(0), ERTS_RUNQ_FLG_MSB_EXEC);
-                erts_smp_atomic32_read_bor_nob(&ERTS_RUNQ_IX(0)->scheduler->ssi->flags,
+                erts_atomic32_read_bor_nob(&ERTS_RUNQ_IX(0)->scheduler->ssi->flags,
                                                ERTS_SSI_FLG_MSB_EXEC);
                 for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++)
                     dcpu_sched_ix_suspend_wake(ix);
@@ -8379,7 +8379,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 
         wait_until_msb:
 
-            ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing));
+            ASSERT(chng_flg & erts_atomic32_read_nob(&schdlr_sspnd.changing));
 
             plp = proclist_create(p);
             erts_proclist_store_last(&msbp->chngq, plp);
@@ -8420,14 +8420,14 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 	}
 	if (!msbp->blckrs && !msbp->chngq) {
 	    int online;
-	    erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing,
+	    erts_atomic32_read_bor_nob(&schdlr_sspnd.changing,
 					   chng_flg);
 	    p->flags &= ~have_blckd_flg;
 	    msbp->ongoing = 0;
             if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) {
                 if (plocks) {
 		    have_unlocked_plocks = 1;
-		    erts_smp_proc_unlock(p, plocks);
+		    erts_proc_unlock(p, plocks);
 		}
 
                 online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
@@ -8464,17 +8464,17 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 	    res = ERTS_SCHDLR_SSPND_DONE;
     }
 
-    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+    erts_mtx_unlock(&schdlr_sspnd.mtx);
 
     if (have_unlocked_plocks)
-	erts_smp_proc_lock(p, plocks);
+	erts_proc_lock(p, plocks);
 
     if (resume_proc) {
 	if (!(plocks & ERTS_PROC_LOCK_STATUS))
-	    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+	    erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
 	resume_process(p, plocks|ERTS_PROC_LOCK_STATUS);
 	if (!(plocks & ERTS_PROC_LOCK_STATUS))
-	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+	    erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
     }
 
     return res;
@@ -8484,14 +8484,14 @@ int
 erts_is_multi_scheduling_blocked(void)
 {
     int res;
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+    erts_mtx_lock(&schdlr_sspnd.mtx);
     if (schdlr_sspnd.msb.blckrs)
 	res = 1;
     else if (schdlr_sspnd.nmsb.blckrs)
 	res = -1;
     else
 	res = 0;
-    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+    erts_mtx_unlock(&schdlr_sspnd.mtx);
     return res;
 }
 
@@ -8503,7 +8503,7 @@ erts_multi_scheduling_blockers(Process *p, int normal)
 
     msbp = normal ? &schdlr_sspnd.nmsb : &schdlr_sspnd.msb;
 
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+    erts_mtx_lock(&schdlr_sspnd.mtx);
     if (!erts_proclist_is_empty(msbp->blckrs)) {
 	Eterm *hp, *hp_end;
 	ErtsProcList *plp1, *plp2;
@@ -8531,7 +8531,7 @@ erts_multi_scheduling_blockers(Process *p, int normal)
 	}
 	HRelease(p, hp_end, hp);
     }
-    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+    erts_mtx_unlock(&schdlr_sspnd.mtx);
     return res;
 }
 
@@ -8835,7 +8835,7 @@ handle_pending_suspend(Process *p, ErtsProcLocks p_locks)
     ErtsPendingSuspend *psp;
     int is_alive = !ERTS_PROC_IS_EXITING(p);
 
-    ERTS_SMP_LC_ASSERT(p_locks & ERTS_PROC_LOCK_STATUS);
+    ERTS_LC_ASSERT(p_locks & ERTS_PROC_LOCK_STATUS);
 
     /*
      * New pending suspenders might appear while we are processing
@@ -8861,15 +8861,15 @@ cancel_suspend_of_suspendee(Process *p, ErtsProcLocks p_locks)
     if (is_not_nil(p->suspendee)) {
 	Process *rp;
 	if (!(p_locks & ERTS_PROC_LOCK_STATUS))
-	    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+	    erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
 	rp = erts_pid2proc(p, p_locks|ERTS_PROC_LOCK_STATUS,
 			   p->suspendee, ERTS_PROC_LOCK_STATUS);
 	if (rp) {
 	    erts_resume(rp, ERTS_PROC_LOCK_STATUS);
-	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+	    erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
 	}
 	if (!(p_locks & ERTS_PROC_LOCK_STATUS))
-	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+	    erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 	p->suspendee = NIL;
     }
 }
@@ -8882,7 +8882,7 @@ handle_pend_sync_suspend(Process *suspendee,
 {
     Process *suspender;
 
-    ERTS_SMP_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS);
+    ERTS_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS);
 
     suspender = erts_pid2proc(suspendee,
 			      suspendee_locks,
@@ -8898,7 +8898,7 @@ handle_pend_sync_suspend(Process *suspendee,
 	   resume suspender */
 	ASSERT(suspendee != suspender);
 	resume_process(suspender, ERTS_PROC_LOCK_STATUS);
-	erts_smp_proc_unlock(suspender, ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(suspender, ERTS_PROC_LOCK_STATUS);
     }
 }
 
@@ -8909,10 +8909,10 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
     Process *rp;
     int unlock_c_p_status;
 
-    ERTS_SMP_LC_ASSERT(c_p_locks == erts_proc_lc_my_proc_locks(c_p));
+    ERTS_LC_ASSERT(c_p_locks == erts_proc_lc_my_proc_locks(c_p));
 
-    ERTS_SMP_LC_ASSERT(c_p_locks & ERTS_PROC_LOCK_MAIN);
-    ERTS_SMP_LC_ASSERT(pid_locks & (ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS));
+    ERTS_LC_ASSERT(c_p_locks & ERTS_PROC_LOCK_MAIN);
+    ERTS_LC_ASSERT(pid_locks & (ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS));
 
     if (c_p->common.id == pid)
 	return erts_pid2proc(c_p, c_p_locks, pid, pid_locks);
@@ -8921,7 +8921,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 	unlock_c_p_status = 0;
     else {
 	unlock_c_p_status = 1;
-	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
+	erts_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
     }
 
     if (c_p->suspendee == pid) {
@@ -8960,15 +8960,15 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 	    /* Other process running */
 
 	    ASSERT((ERTS_PSFLG_RUNNING | ERTS_PSFLG_DIRTY_RUNNING)
-		   & erts_smp_atomic32_read_nob(&rp->state));
+		   & erts_atomic32_read_nob(&rp->state));
 
 #ifdef ERTS_DIRTY_SCHEDULERS
 	    if (!suspend
-		&& (erts_smp_atomic32_read_nob(&rp->state)
+		&& (erts_atomic32_read_nob(&rp->state)
 		    & ERTS_PSFLG_DIRTY_RUNNING)) {
 		ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS;
-		if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) {
-		    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+		if (need_locks && erts_proc_trylock(rp, need_locks) == EBUSY) {
+		    erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
 		    rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS,
 				       pid, pid_locks|ERTS_PROC_LOCK_STATUS);
 		}
@@ -8994,19 +8994,19 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 		c_p->flags |= F_P2PNR_RESCHED;
 	    }
 	    /* Yield (caller is assumed to yield immediately in bif). */
-	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+	    erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
 	    rp = ERTS_PROC_LOCK_BUSY;
 	}
 	else {
 	    ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS;
-	    if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) {
+	    if (need_locks && erts_proc_trylock(rp, need_locks) == EBUSY) {
 		if ((ERTS_PSFLG_RUNNING_SYS|ERTS_PSFLG_DIRTY_RUNNING_SYS)
-		    & erts_smp_atomic32_read_nob(&rp->state)) {
+		    & erts_atomic32_read_nob(&rp->state)) {
 		    /* Executing system task... */
 		    resume_process(rp, ERTS_PROC_LOCK_STATUS);
 		    goto running;
 		}
-		erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+		erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
 		/*
 		 * If we are unlucky, the process just got selected for
 		 * execution of a system task. In this case we may be
@@ -9030,7 +9030,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 #ifdef DEBUG
 	    {
 		erts_aint32_t state;
-		state = erts_smp_atomic32_read_nob(&rp->state);
+		state = erts_atomic32_read_nob(&rp->state);
 		ASSERT((state & ERTS_PSFLG_PENDING_EXIT)
 		       || !(state & ERTS_PSFLG_RUNNING));
 	    }
@@ -9044,9 +9044,9 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
  done:
 	
     if (rp && rp != ERTS_PROC_LOCK_BUSY && !(pid_locks & ERTS_PROC_LOCK_STATUS))
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
     if (unlock_c_p_status)
-	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
     return rp;
 }
 
@@ -9092,7 +9092,7 @@ do_bif_suspend_process(Process *c_p,
 {
     ASSERT(suspendee);
     ASSERT(!ERTS_PROC_IS_EXITING(suspendee));
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS
 		       & erts_proc_lc_my_proc_locks(suspendee));
     if (smon) {
 	if (!smon->active) {
@@ -9115,7 +9115,7 @@ handle_pend_bif_sync_suspend(Process *suspendee,
 {
     Process *suspender;
 
-    ERTS_SMP_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS);
+    ERTS_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS);
 
     suspender = erts_pid2proc(suspendee,
 			      suspendee_locks,
@@ -9144,7 +9144,7 @@ handle_pend_bif_sync_suspend(Process *suspendee,
 	   resume suspender */
 	ASSERT(suspender != suspendee);
 	resume_process(suspender, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
-	erts_smp_proc_unlock(suspender,
+	erts_proc_unlock(suspender,
 			     ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
     }
 }
@@ -9158,7 +9158,7 @@ handle_pend_bif_async_suspend(Process *suspendee,
 
     Process *suspender;
 
-    ERTS_SMP_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS);
+    ERTS_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS);
 
     suspender = erts_pid2proc(suspendee,
 			      suspendee_locks,
@@ -9182,7 +9182,7 @@ handle_pend_bif_async_suspend(Process *suspendee,
 	    do_bif_suspend_process(suspendee, smon, suspendee);
 	    ASSERT(!smon || res != 0);
 	}
-	erts_smp_proc_unlock(suspender, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(suspender, ERTS_PROC_LOCK_LINK);
     }
 }
 
@@ -9234,7 +9234,7 @@ suspend_process_2(BIF_ALIST_2)
 				    ? (ErtsProcLocks) 0
 				    : ERTS_PROC_LOCK_STATUS);
 
-    erts_smp_proc_lock(BIF_P, xlocks);
+    erts_proc_lock(BIF_P, xlocks);
 
     suspendee = erts_pid2proc(BIF_P,
 			      ERTS_PROC_LOCK_MAIN|xlocks,
@@ -9251,9 +9251,9 @@ suspend_process_2(BIF_ALIST_2)
     if (asynchronous) {
 	/* --- Asynchronous suspend begin ---------------------------------- */
 
-	ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_LINK
+	ERTS_LC_ASSERT(ERTS_PROC_LOCK_LINK
 			   & erts_proc_lc_my_proc_locks(BIF_P));
-	ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS
+	ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS
 			   == erts_proc_lc_my_proc_locks(suspendee));
 
 	if (smon->active) {
@@ -9293,10 +9293,10 @@ suspend_process_2(BIF_ALIST_2)
     else /* if (!asynchronous) */ {
 	/* --- Synchronous suspend begin ----------------------------------- */
 
-	ERTS_SMP_LC_ASSERT(((ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS)
+	ERTS_LC_ASSERT(((ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS)
 			    & erts_proc_lc_my_proc_locks(BIF_P))
 			   == (ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS));
-	ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS
+	ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS
 			   == erts_proc_lc_my_proc_locks(suspendee));
 
 	if (BIF_P->suspendee == BIF_ARG_1) {
@@ -9364,7 +9364,7 @@ suspend_process_2(BIF_ALIST_2)
 
 #ifdef DEBUG
     {
-	erts_aint32_t state = erts_smp_atomic32_read_acqb(&suspendee->state);
+	erts_aint32_t state = erts_atomic32_read_acqb(&suspendee->state);
 	ASSERT((state & ERTS_PSFLG_SUSPENDED)
 	       || (asynchronous && smon->pending));
 	ASSERT((state & ERTS_PSFLG_SUSPENDED)
@@ -9372,8 +9372,8 @@ suspend_process_2(BIF_ALIST_2)
     }
 #endif
 
-    erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
-    erts_smp_proc_unlock(BIF_P, xlocks);
+    erts_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
+    erts_proc_unlock(BIF_P, xlocks);
     BIF_RET(res);
 
  system_limit:
@@ -9394,9 +9394,9 @@ suspend_process_2(BIF_ALIST_2)
 
  do_return:
     if (suspendee)
-	erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
     if (xlocks)
-	erts_smp_proc_unlock(BIF_P, xlocks);
+	erts_proc_unlock(BIF_P, xlocks);
     return res;
 
 }
@@ -9416,7 +9416,7 @@ resume_process_1(BIF_ALIST_1)
     if (BIF_P->common.id == BIF_ARG_1)
 	BIF_ERROR(BIF_P, BADARG);
 
-    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
+    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
     smon = erts_lookup_suspend_monitor(BIF_P->suspend_monitors, BIF_ARG_1);
 
     if (!smon) {
@@ -9462,17 +9462,17 @@ resume_process_1(BIF_ALIST_1)
 	    goto no_suspendee;
 
 	ASSERT(ERTS_PSFLG_SUSPENDED
-	       & erts_smp_atomic32_read_nob(&suspendee->state));
+	       & erts_atomic32_read_nob(&suspendee->state));
 	ASSERT(BIF_P != suspendee);
 	resume_process(suspendee, ERTS_PROC_LOCK_STATUS);
 
-	erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
     }
 
     if (!smon->active && !smon->pending)
 	erts_delete_suspend_monitor(&BIF_P->suspend_monitors, BIF_ARG_1);
 
-    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
+    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
 
     BIF_RET(am_true);
 
@@ -9481,7 +9481,7 @@ resume_process_1(BIF_ALIST_1)
     erts_delete_suspend_monitor(&BIF_P->suspend_monitors, BIF_ARG_1);
 
  error:
-    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
+    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
     BIF_ERROR(BIF_P, BADARG);
 }
 
@@ -9494,7 +9494,7 @@ erts_internal_is_process_executing_dirty_1(BIF_ALIST_1)
     else {
 	Process *rp = erts_proc_lookup(BIF_ARG_1);
 	if (rp) {
-	    erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state);
+	    erts_aint32_t state = erts_atomic32_read_nob(&rp->state);
 	    if (state & (ERTS_PSFLG_DIRTY_RUNNING
 			 |ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
 		BIF_RET(am_true);
@@ -9511,9 +9511,9 @@ run_queues_len_aux(ErtsRunQueue *rq, Uint *tot_len, Uint *qlen, int *ip, int inc
     Sint rq_len;
 
     if (locked)
-        rq_len = (Sint) erts_smp_atomic32_read_dirty(&rq->len);
+        rq_len = (Sint) erts_atomic32_read_dirty(&rq->len);
     else
-        rq_len = (Sint) erts_smp_atomic32_read_nob(&rq->len);
+        rq_len = (Sint) erts_atomic32_read_nob(&rq->len);
     ASSERT(rq_len >= 0);
 
     if (incl_active_sched) {
@@ -9521,12 +9521,12 @@ run_queues_len_aux(ErtsRunQueue *rq, Uint *tot_len, Uint *qlen, int *ip, int inc
         if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) {
             erts_aint32_t dcnt;
             if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(rq)) {
-                dcnt = erts_smp_atomic32_read_nob(&dirty_count.cpu.active);
+                dcnt = erts_atomic32_read_nob(&dirty_count.cpu.active);
                 ASSERT(0 <= dcnt && dcnt <= erts_no_dirty_cpu_schedulers);
             }
             else {
                 ASSERT(ERTS_RUNQ_IS_DIRTY_IO_RUNQ(rq));
-                dcnt = erts_smp_atomic32_read_nob(&dirty_count.io.active);
+                dcnt = erts_atomic32_read_nob(&dirty_count.io.active);
                 ASSERT(0 <= dcnt && dcnt <= erts_no_dirty_io_schedulers);
             }
             rq_len += (Sint) dcnt;
@@ -9610,7 +9610,7 @@ erts_process_status(Process *rp, Eterm rpid)
     Process *p = rp ? rp : erts_proc_lookup_raw(rpid);
 
     if (p) {
-	erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
+	erts_aint32_t state = erts_atomic32_read_acqb(&p->state);
 	res = erts_process_state2status(state);
     }
     else {
@@ -9619,14 +9619,14 @@ erts_process_status(Process *rp, Eterm rpid)
 
 	for (i = 0; i < erts_no_schedulers; i++) {
 	    esdp = ERTS_SCHEDULER_IX(i);
-	    erts_smp_runq_lock(esdp->run_queue);
+	    erts_runq_lock(esdp->run_queue);
 	    if (esdp->free_process
 		&& esdp->free_process->common.id == rpid) {
 		res = am_free;
-		erts_smp_runq_unlock(esdp->run_queue);
+		erts_runq_unlock(esdp->run_queue);
 		break;
 	    }
-	    erts_smp_runq_unlock(esdp->run_queue);
+	    erts_runq_unlock(esdp->run_queue);
 	}
     }
     return res;
@@ -9644,9 +9644,9 @@ erts_suspend(Process* c_p, ErtsProcLocks c_p_locks, Port *busy_port)
     int suspend;
 
     ASSERT(c_p == erts_get_current_process());
-    ERTS_SMP_LC_ASSERT(c_p_locks == erts_proc_lc_my_proc_locks(c_p));
+    ERTS_LC_ASSERT(c_p_locks == erts_proc_lc_my_proc_locks(c_p));
     if (!(c_p_locks & ERTS_PROC_LOCK_STATUS))
-	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
+	erts_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
 
     if (busy_port)
 	suspend = erts_save_suspend_process_on_port(busy_port, c_p);
@@ -9662,7 +9662,7 @@ erts_suspend(Process* c_p, ErtsProcLocks c_p_locks, Port *busy_port)
     }
 
     if (!(c_p_locks & ERTS_PROC_LOCK_STATUS))
-	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
 
     if (suspend && busy_port && erts_system_monitor_flags.busy_port)
 	monitor_generic(c_p, am_busy_port, busy_port->common.id);
@@ -9671,12 +9671,12 @@ erts_suspend(Process* c_p, ErtsProcLocks c_p_locks, Port *busy_port)
 void
 erts_resume(Process* process, ErtsProcLocks process_locks)
 {
-    ERTS_SMP_LC_ASSERT(process_locks == erts_proc_lc_my_proc_locks(process));
+    ERTS_LC_ASSERT(process_locks == erts_proc_lc_my_proc_locks(process));
     if (!(process_locks & ERTS_PROC_LOCK_STATUS))
-	erts_smp_proc_lock(process, ERTS_PROC_LOCK_STATUS);
+	erts_proc_lock(process, ERTS_PROC_LOCK_STATUS);
     resume_process(process, process_locks|ERTS_PROC_LOCK_STATUS);
     if (!(process_locks & ERTS_PROC_LOCK_STATUS))
-	erts_smp_proc_unlock(process, ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(process, ERTS_PROC_LOCK_STATUS);
 }
 
 int
@@ -9696,7 +9696,7 @@ erts_resume_processes(ErtsProcList *list)
 		resume_process(proc, ERTS_PROC_LOCK_STATUS);
 		nresumed++;
 	    }
-	    erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_STATUS);
+	    erts_proc_unlock(proc, ERTS_PROC_LOCK_STATUS);
 	}
 	fplp = plp;
 	plp = plp->next;
@@ -9708,7 +9708,7 @@ erts_resume_processes(ErtsProcList *list)
 Eterm
 erts_get_process_priority(Process *p)
 {
-    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
+    erts_aint32_t state = erts_atomic32_read_nob(&p->state);
     switch (ERTS_PSFLGS_GET_USR_PRIO(state)) {
     case PRIORITY_MAX:		return am_max;
     case PRIORITY_HIGH:		return am_high;
@@ -9731,7 +9731,7 @@ erts_set_process_priority(Process *p, Eterm value)
     default:		return THE_NON_VALUE;				break;
     }
 
-    a = erts_smp_atomic32_read_nob(&p->state);
+    a = erts_atomic32_read_nob(&p->state);
     if (nprio == ERTS_PSFLGS_GET_USR_PRIO(a))
 	oprio = nprio;
     else {
@@ -9739,7 +9739,7 @@ erts_set_process_priority(Process *p, Eterm value)
 	erts_aint32_t e, n, aprio;
 
 	if (a & ERTS_PSFLG_ACTIVE_SYS) {
-	    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+	    erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
 	    slocked = 1;
 	}
 
@@ -9753,7 +9753,7 @@ erts_set_process_priority(Process *p, Eterm value)
 		int max_qbit;
 
 		if (!slocked) {
-		    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+		    erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
 		    slocked = 1;
 		}
 
@@ -9794,11 +9794,11 @@ erts_set_process_priority(Process *p, Eterm value)
 	    n |= ((nprio << ERTS_PSFLGS_USR_PRIO_OFFSET)
 		  | (aprio << ERTS_PSFLGS_ACT_PRIO_OFFSET));
 
-	    a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
+	    a = erts_atomic32_cmpxchg_mb(&p->state, n, e);
 	} while (a != e);
 
         if (slocked)
-            erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+            erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 
     }
 
@@ -9883,7 +9883,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	input_reductions = INPUT_REDUCTIONS;
     }
 
-    ERTS_SMP_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())
+    ERTS_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())
 		       || !erts_thr_progress_is_blocking());
 
     /*
@@ -9905,9 +9905,9 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 #endif
 	rq = erts_get_runq_current(esdp);
 	ASSERT(esdp);
-	fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls);
+	fcalls = (int) erts_atomic32_read_acqb(&function_calls);
 	actual_reds = reds = 0;
-	erts_smp_runq_lock(rq);
+	erts_runq_lock(rq);
     } else {
 #ifdef ERTS_DIRTY_SCHEDULERS
 	is_normal_sched = !esdp;
@@ -9927,7 +9927,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
     sched_out_proc:
 
-	ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
+	ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
 	reds = actual_reds = calls - esdp->virtual_reds;
 
@@ -9936,14 +9936,14 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST;
 	esdp->virtual_reds = 0;
 
-	fcalls = (int) erts_smp_atomic32_add_read_acqb(&function_calls, reds);
+	fcalls = (int) erts_atomic32_add_read_acqb(&function_calls, reds);
 	ASSERT(esdp && esdp == erts_get_scheduler_data());
 
 	rq = erts_get_runq_current(esdp);
 
 	p->reds += actual_reds;
 
-	state = erts_smp_atomic32_read_nob(&p->state);
+	state = erts_atomic32_read_nob(&p->state);
 
 	if (IS_TRACED(p)) {
 	    if (IS_TRACED_FL(p, F_TRACE_CALLS) && !(state & ERTS_PSFLG_FREE))
@@ -9962,16 +9962,16 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    }
 	}
 
-	erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+	erts_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
 
         if (p->trace_msg_q) {
-	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+	    erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
             erts_schedule_flush_trace_messages(p, 1);
-	    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+	    erts_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
 	}
 
         /* have to re-read state after taking lock */
-        state = erts_smp_atomic32_read_nob(&p->state);
+        state = erts_atomic32_read_nob(&p->state);
 
 	if (is_normal_sched && (state & ERTS_PSFLG_PENDING_EXIT))
 	    erts_handle_pending_exit(p, (ERTS_PROC_LOCK_MAIN
@@ -10001,7 +10001,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
             if (is_normal_sched)
                 p->scheduler_data = NULL;
 
-            erts_smp_proc_unlock(p, (ERTS_PROC_LOCK_MAIN
+            erts_proc_unlock(p, (ERTS_PROC_LOCK_MAIN
                                      | ERTS_PROC_LOCK_STATUS
                                      | ERTS_PROC_LOCK_TRACE));
 
@@ -10024,21 +10024,21 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	ASSERT(!esdp->free_process);
 	ASSERT(!esdp->current_process);
 
-	ERTS_SMP_CHK_NO_PROC_LOCKS;
+	ERTS_CHK_NO_PROC_LOCKS;
 
 	if (is_normal_sched) {
 	    if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS)
 		(void) erts_get_monotonic_time(esdp);
 
 	    if (esdp->last_monotonic_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
-		erts_smp_runq_unlock(rq);
+		erts_runq_unlock(rq);
 		erts_bump_timers(esdp->timer_wheel, esdp->last_monotonic_time);
-		erts_smp_runq_lock(rq);
+		erts_runq_lock(rq);
 	    }
 	}
     }
 
-    ERTS_SMP_LC_ASSERT(!is_normal_sched || !erts_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(!is_normal_sched || !erts_thr_progress_is_blocking());
 
  check_activities_to_run: {
 	erts_aint32_t psflg_running, psflg_running_sys;
@@ -10049,7 +10049,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    if (rq->check_balance_reds <= 0)
 		check_balance(rq);
 
-	    ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
+	    ERTS_LC_ASSERT(!erts_thr_progress_is_blocking());
 
 	    mps = erts_get_migration_paths_managed();
 	    mp = &mps->mpath[rq->ix];
@@ -10058,14 +10058,14 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		immigrate(rq, mp);
 	}
 
-	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+	ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
     continue_check_activities_to_run:
 	flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
     continue_check_activities_to_run_known_flags:
 	ASSERT(!is_normal_sched || (flags & ERTS_RUNQ_FLG_NONEMPTY));
 
 	if (!is_normal_sched) {
-	    if (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
+	    if (erts_atomic32_read_acqb(&esdp->ssi->flags)
 		& (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) {
 		suspend_scheduler(esdp);
 	    }
@@ -10095,17 +10095,17 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    leader_update = erts_thr_progress_update(esdp);
 	    aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work);
 	    if (aux_work | leader_update) {
-		erts_smp_runq_unlock(rq);
+		erts_runq_unlock(rq);
 		if (leader_update)
 		    erts_thr_progress_leader_update(esdp);
 		if (aux_work)
 		    handle_aux_work(&esdp->aux_work_data, aux_work, 0);
-		erts_smp_runq_lock(rq);
+		erts_runq_lock(rq);
 	    }
 
-	    ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
+	    ERTS_LC_ASSERT(!erts_thr_progress_is_blocking());
 	}
-	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+	ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
 
 
 	flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
@@ -10117,7 +10117,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	}
 	else if (!runq_got_work_to_execute_flags(flags)) {
 	    /* Prepare for scheduler wait */
-	    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+	    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
 
 	    rq->wakeup_other = 0;
 	    rq->wakeup_other_reds = 0;
@@ -10131,7 +10131,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
                 ASSERT(!runq_got_work_to_execute(rq));
 		if (!is_normal_sched) {
                     /* Dirty scheduler */
-                    if (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
+                    if (erts_atomic32_read_acqb(&esdp->ssi->flags)
                         & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) {
                         /* Go suspend... */
                         goto continue_check_activities_to_run_known_flags;
@@ -10187,13 +10187,13 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	    ERTS_MSACC_PUSH_STATE_CACHED_M();
 
-	    erts_smp_atomic32_set_relb(&function_calls, 0);
+	    erts_atomic32_set_relb(&function_calls, 0);
 	    fcalls = 0;
 
 #if 0 /* Not needed since we wont wait in sys schedule */
 	    erts_sys_schedule_interrupt(0);
 #endif
-	    erts_smp_runq_unlock(rq);
+	    erts_runq_unlock(rq);
 
             ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO);
             LTTNG2(scheduler_poll, esdp->no, 1);
@@ -10205,7 +10205,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
 		erts_bump_timers(esdp->timer_wheel, current_time);
 
-	    erts_smp_runq_lock(rq);
+	    erts_runq_lock(rq);
 	    clear_sys_scheduling();
 	    goto continue_check_activities_to_run;
 	}
@@ -10311,7 +10311,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		    proxy_p = NULL;
 		    goto pick_next_process;
 		}
-		state = erts_smp_atomic32_read_nob(&p->state);
+		state = erts_atomic32_read_nob(&p->state);
 	    }
 
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -10367,7 +10367,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		    else
 			new |= psflg_running;
 		}
-		state = erts_smp_atomic32_cmpxchg_relb(&p->state, new, exp);
+		state = erts_atomic32_cmpxchg_relb(&p->state, new, exp);
 		if (state == exp) {
 		    if (!run_process) {
 			if (proxy_p) {
@@ -10394,7 +10394,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    calls = 0;
 	    reds = context_reds;
 
-	    erts_smp_runq_unlock(rq);
+	    erts_runq_unlock(rq);
 
 	}
 
@@ -10404,11 +10404,11 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	if (flags & ERTS_RUNQ_FLG_PROTECTED)
 	    (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
 
-	ERTS_SMP_CHK_NO_PROC_LOCKS;
+	ERTS_CHK_NO_PROC_LOCKS;
 
-	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
+	erts_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
 
-	state = erts_smp_atomic32_read_nob(&p->state);
+	state = erts_atomic32_read_nob(&p->state);
 
 	if (erts_sched_stat.enabled) {
 	    int prio;
@@ -10419,17 +10419,17 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	    prio = (int) ERTS_PSFLGS_GET_USR_PRIO(state);
 
-	    erts_smp_spin_lock(&erts_sched_stat.lock);
+	    erts_spin_lock(&erts_sched_stat.lock);
 	    erts_sched_stat.prio[prio].total_executed++;
 	    erts_sched_stat.prio[prio].executed++;
 	    if (migrated) {
 		erts_sched_stat.prio[prio].total_migrated++;
 		erts_sched_stat.prio[prio].migrated++;
 	    }
-	    erts_smp_spin_unlock(&erts_sched_stat.lock);
+	    erts_spin_unlock(&erts_sched_stat.lock);
 	}
 
-	state = erts_smp_atomic32_read_nob(&p->state);
+	state = erts_atomic32_read_nob(&p->state);
 
 #ifndef ERTS_DIRTY_SCHEDULERS
 	ASSERT(!p->scheduler_data);
@@ -10440,7 +10440,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		& (!(state & ERTS_PSFLG_ACTIVE_SYS))) {
 		/* Migrate to dirty scheduler... */
 	    sunlock_sched_out_proc:
-		erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+		erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 		goto sched_out_proc;
 	    }
 	    ASSERT(!p->scheduler_data);
@@ -10475,11 +10475,11 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	if (state & ERTS_PSFLG_PENDING_EXIT) {
 	    erts_handle_pending_exit(p,
 				     ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
-	    state = erts_smp_atomic32_read_nob(&p->state);
+	    state = erts_atomic32_read_nob(&p->state);
 	}
 
 
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 
         /* Clear tracer if it has been removed */
 	if (IS_TRACED(p) && erts_is_tracer_proc_enabled(
@@ -10537,7 +10537,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
                     n &= ~psflg_running_sys;
                     n |= psflg_running;
 
-                    state = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
+                    state = erts_atomic32_cmpxchg_mb(&p->state, n, e);
                     if (state == e) {
                         state = n;
                         break;
@@ -10571,12 +10571,12 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	p->fcalls = reds;
 	    
-	ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
+	ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
 	/* Never run a suspended process */
 #ifdef DEBUG
         {
-            erts_aint32_t dstate = erts_smp_atomic32_read_nob(&p->state);
+            erts_aint32_t dstate = erts_atomic32_read_nob(&p->state);
             ASSERT(!(ERTS_PSFLG_SUSPENDED & dstate)
                    || (ERTS_PSFLG_DIRTY_RUNNING_SYS & dstate));
         }
@@ -10657,7 +10657,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st,
 	    rp_locks &= ~ERTS_PROC_LOCK_MAIN;
 
 	if (rp_locks)
-	    erts_smp_proc_unlock(rp, rp_locks);
+	    erts_proc_unlock(rp, rp_locks);
 
 #ifdef ERTS_DIRTY_SCHEDULERS
 	if (!normal_sched)
@@ -10681,7 +10681,7 @@ fetch_sys_task(Process *c_p, erts_aint32_t state, int *qmaskp, int *priop)
 
     *priop = -1; /* Shut up annoying erroneous warning */
 
-    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
 
     if (!c_p->sys_task_qs) {
 	qmask = 0;
@@ -10801,13 +10801,13 @@ fetch_sys_task(Process *c_p, erts_aint32_t state, int *qmaskp, int *priop)
 
 	    if (a == n)
 		break;
-	    a = erts_smp_atomic32_cmpxchg_nob(&c_p->state, n, e);
+	    a = erts_atomic32_cmpxchg_nob(&c_p->state, n, e);
 	} while (a != e);
     }
 
 done:
 
-    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
 
     if (unused_qs)
 	proc_sys_task_queues_free(unused_qs);
@@ -10831,7 +10831,7 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
     int qmask = 0;
 
     ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p)));
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
 
     do {
 	ErtsProcSysTaskType type;
@@ -10952,7 +10952,7 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	if (st)
 	    reds += notify_sys_task_executed(c_p, st, st_res, 1);
 
-	state = erts_smp_atomic32_read_acqb(&c_p->state);
+	state = erts_atomic32_read_acqb(&c_p->state);
     } while (qmask && reds > 0);
 
     *statep = state;
@@ -10973,7 +10973,7 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
                              * are dirty tasks.
                              */
 
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
 
     do {
 	ErtsProcSysTask *st;
@@ -11016,7 +11016,7 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
 
 	reds += notify_sys_task_executed(c_p, st, st_res, 1);
 
-	state = erts_smp_atomic32_read_acqb(&c_p->state);
+	state = erts_atomic32_read_acqb(&c_p->state);
     } while (qmask && reds < max_reds);
 
     return reds;
@@ -11051,19 +11051,19 @@ erts_execute_dirty_system_task(Process *c_p)
     }
 
     if (c_p->flags & F_DIRTY_GC_HIBERNATE) {
-	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
-	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
+	erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
+	ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
 	if (c_p->msg.len)
             c_p->flags &= ~F_DIRTY_GC_HIBERNATE; /* operation aborted... */
         else {
-	    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
+	    erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
 	    c_p->fvalue = NIL;
 	    erts_garbage_collect_hibernate(c_p);
 	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
+	    erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
 	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	}
-	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
     }
 
     if (c_p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) {
@@ -11107,7 +11107,7 @@ erts_execute_dirty_system_task(Process *c_p)
 
     }
 
-    erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_DIRTY_ACTIVE_SYS);
+    erts_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_DIRTY_ACTIVE_SYS);
 }
 
 static BIF_RETTYPE
@@ -11160,7 +11160,7 @@ dispatch_system_task(Process *c_p, erts_aint_t fail_state,
     erts_queue_message(rp, rp_locks, mp, msg, st->requester);
 
     if (rp_locks)
-	erts_smp_proc_unlock(rp, rp_locks);
+	erts_proc_unlock(rp, rp_locks);
 
     return ret;
 }
@@ -11366,7 +11366,7 @@ erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type, void* arg)
 	st->req_id_sz = 0;
         st->arg[0] = (Eterm)arg;
 	ERTS_INIT_OFF_HEAP(&st->off_heap);
-	state = erts_smp_atomic32_read_nob(&rp->state);
+	state = erts_atomic32_read_nob(&rp->state);
 
 	fail_state = ERTS_PSFLG_EXITING;
 
@@ -11420,7 +11420,7 @@ erts_schedule_flush_trace_messages(Process *proc, int force_on_proc)
     erts_aint32_t state;
 
     if (!force_on_proc) {
-	state = erts_smp_atomic32_read_nob(&proc->state);
+	state = erts_atomic32_read_nob(&proc->state);
 	if (state & (ERTS_PSFLG_DIRTY_RUNNING
 		     | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
 	    goto sched_flush_dirty;
@@ -11436,7 +11436,7 @@ erts_schedule_flush_trace_messages(Process *proc, int force_on_proc)
 
 #ifdef ERTS_DIRTY_SCHEDULERS
     if (!force_on_proc) {
-	state = erts_smp_atomic32_read_mb(&proc->state);
+	state = erts_atomic32_read_mb(&proc->state);
 	if (state & (ERTS_PSFLG_DIRTY_RUNNING
 		     | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
 	    void *vargp;
@@ -11473,7 +11473,7 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio)
     erts_aint32_t state;
     ErtsProcSysTaskQs *qs;
 
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
 
     qs = ERTS_PROC_GET_DELAYED_GC_TASK_QS(c_p);
     if (!qs) {
@@ -11503,7 +11503,7 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio)
 	}
     }
 
-    state = erts_smp_atomic32_read_nob(&c_p->state);
+    state = erts_atomic32_read_nob(&c_p->state);
     ASSERT((ERTS_PSFLG_RUNNING
 	    | ERTS_PSFLG_RUNNING_SYS
 	    | ERTS_PSFLG_DIRTY_RUNNING
@@ -11519,7 +11519,7 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio)
 	    n &= ~ERTS_PSFLGS_ACT_PRIO_MASK;
 	    n |= prio << ERTS_PSFLGS_ACT_PRIO_OFFSET;
 	}
-	state = erts_smp_atomic32_cmpxchg_relb(&c_p->state, n, e);
+	state = erts_atomic32_cmpxchg_relb(&c_p->state, n, e);
 	if (state == e)
 	    break;
     }
@@ -11540,8 +11540,8 @@ erts_set_gc_state(Process *c_p, int enable)
     ErtsProcSysTaskQs *dgc_tsk_qs;
     ASSERT(c_p == erts_get_current_process());
     ASSERT((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)
-	   & erts_smp_atomic32_read_nob(&c_p->state));
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
+	   & erts_atomic32_read_nob(&c_p->state));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
 
     if (!enable) {
 	c_p->flags |= F_DISABLE_GC;
@@ -11556,7 +11556,7 @@ erts_set_gc_state(Process *c_p, int enable)
 
     /* Move delayed gc tasks into sys tasks queues. */
 
-    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
 
     if (!c_p->sys_task_qs) {
 	c_p->sys_task_qs = dgc_tsk_qs;
@@ -11629,7 +11629,7 @@ erts_set_gc_state(Process *c_p, int enable)
 	erts_aint32_t aprio, state =
 #endif
 
-	    erts_smp_atomic32_read_bset_nob(&c_p->state,
+	    erts_atomic32_read_bset_nob(&c_p->state,
 					    (ERTS_PSFLG_DELAYED_SYS
 					     | ERTS_PSFLG_ACTIVE_SYS),
 					    ERTS_PSFLG_ACTIVE_SYS);
@@ -11643,7 +11643,7 @@ erts_set_gc_state(Process *c_p, int enable)
     }
 #endif
 
-    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
 
     (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, NULL);
 
@@ -11659,24 +11659,24 @@ erts_sched_stat_modify(int what)
     int ix;
     switch (what) {
     case ERTS_SCHED_STAT_MODIFY_ENABLE:
-	erts_smp_thr_progress_block();
+	erts_thr_progress_block();
 	erts_sched_stat.enabled = 1;
-	erts_smp_thr_progress_unblock();
+	erts_thr_progress_unblock();
 	break;
     case ERTS_SCHED_STAT_MODIFY_DISABLE:
-	erts_smp_thr_progress_block();
+	erts_thr_progress_block();
 	erts_sched_stat.enabled = 0;
-	erts_smp_thr_progress_unblock();
+	erts_thr_progress_unblock();
 	break;
     case ERTS_SCHED_STAT_MODIFY_CLEAR:
-	erts_smp_spin_lock(&erts_sched_stat.lock);
+	erts_spin_lock(&erts_sched_stat.lock);
 	for (ix = 0; ix < ERTS_NO_PRIO_LEVELS; ix++) {
 	    erts_sched_stat.prio[ix].total_executed = 0;
 	    erts_sched_stat.prio[ix].executed = 0;
 	    erts_sched_stat.prio[ix].total_migrated = 0;
 	    erts_sched_stat.prio[ix].migrated = 0;
 	}
-	erts_smp_spin_unlock(&erts_sched_stat.lock);
+	erts_spin_unlock(&erts_sched_stat.lock);
 	break;
     }
 }
@@ -11690,7 +11690,7 @@ erts_sched_stat_term(Process *p, int total)
     Uint executed[ERTS_NO_PRIO_LEVELS];
     Uint migrated[ERTS_NO_PRIO_LEVELS];
 
-    erts_smp_spin_lock(&erts_sched_stat.lock);
+    erts_spin_lock(&erts_sched_stat.lock);
     if (total) {
 	int i;
 	for (i = 0; i < ERTS_NO_PRIO_LEVELS; i++) {
@@ -11709,7 +11709,7 @@ erts_sched_stat_term(Process *p, int total)
 	    erts_sched_stat.prio[i].migrated = 0;
 	}
     }
-    erts_smp_spin_unlock(&erts_sched_stat.lock);
+    erts_spin_unlock(&erts_sched_stat.lock);
 
     sz = 0;
     (void) erts_bld_atom_2uint_3tup_list(NULL, &sz, ERTS_NO_PRIO_LEVELS,
@@ -11739,7 +11739,7 @@ erts_schedule_misc_op(void (*func)(void *), void *arg)
 	    rq = erq;
     }
 
-    erts_smp_runq_lock(rq);
+    erts_runq_lock(rq);
 
     molp->next = NULL;
     molp->func = func;
@@ -11754,7 +11754,7 @@ erts_schedule_misc_op(void (*func)(void *), void *arg)
 
     ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP);
 
-    erts_smp_runq_unlock(rq);
+    erts_runq_unlock(rq);
 
     smp_notify_inc_runq(rq);
 }
@@ -11787,7 +11787,7 @@ exec_misc_ops(ErtsRunQueue *rq)
     if (!rq->misc.start)
         ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP);
 
-    erts_smp_runq_unlock(rq);
+    erts_runq_unlock(rq);
 
     while (molp) {
 	tmp_molp = molp;
@@ -11796,7 +11796,7 @@ exec_misc_ops(ErtsRunQueue *rq)
 	misc_op_list_free(tmp_molp);
     }
 
-    erts_smp_runq_lock(rq);
+    erts_runq_lock(rq);
 }
 
 Uint
@@ -11827,12 +11827,12 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp)
 {
     Uint reds = erts_current_reductions(c_p, c_p);
     int ix;
-    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+    erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
     /*
      * Wait for other schedulers to schedule out their processes
      * and update 'reductions'.
      */
-    erts_smp_thr_progress_block();
+    erts_thr_progress_block();
     for (reds = 0, ix = 0; ix < erts_no_run_queues; ix++)
 	reds += ERTS_RUNQ_IX(ix)->procs.reductions;
     if (redsp)
@@ -11840,8 +11840,8 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp)
     if (diffp)
 	*diffp = reds - last_exact_reductions;
     last_exact_reductions = reds;
-    erts_smp_thr_progress_unblock();
-    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+    erts_thr_progress_unblock();
+    erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 }
 
 static void delete_process(Process* p);
@@ -11850,7 +11850,7 @@ void
 erts_free_proc(Process *p)
 {
     erts_proc_lock_fin(p);
-    ASSERT(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_FREE);
+    ASSERT(erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_FREE);
     ASSERT(0 == erts_proc_read_refc(p));
     if (p->flags & F_DELAYED_DEL_PROC)
 	delete_process(p);
@@ -11870,10 +11870,10 @@ static void early_init_process_struct(void *varg, Eterm data)
 
     proc->common.id = make_internal_pid(data);
 #ifdef ERTS_DIRTY_SCHEDULERS
-    erts_smp_atomic32_init_nob(&proc->dirty_state, 0);
+    erts_atomic32_init_nob(&proc->dirty_state, 0);
     proc->dirty_sys_tasks = NULL;
 #endif
-    erts_smp_atomic32_init_relb(&proc->state, arg->state);
+    erts_atomic32_init_relb(&proc->state, arg->state);
 
     RUNQ_SET_RQ(&proc->run_queue, arg->run_queue);
 
@@ -11950,7 +11950,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     INITIALIZE_LITERAL_PURGE_AREA(litarea);
 #endif
 
-    erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR);
+    erts_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR);
 
     /*
      * Check for errors.
@@ -11998,9 +11998,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	goto error;
     }
 
-    ASSERT((erts_smp_atomic32_read_nob(&p->state)
+    ASSERT((erts_atomic32_read_nob(&p->state)
 	    & ERTS_PSFLG_ON_HEAP_MSGQ)
-	   || (erts_smp_atomic32_read_nob(&p->state)
+	   || (erts_atomic32_read_nob(&p->state)
 	       & ERTS_PSFLG_OFF_HEAP_MSGQ));
 
 #ifdef SHCOPY_SPAWN
@@ -12026,7 +12026,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	p->min_vheap_size = BIN_VH_MIN_SIZE;
         MAX_HEAP_SIZE_SET(p, H_MAX_SIZE);
         MAX_HEAP_SIZE_FLAGS_SET(p, H_MAX_FLAGS);
-	p->max_gen_gcs    = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
+	p->max_gen_gcs    = (Uint16) erts_atomic32_read_nob(&erts_max_gen_gcs);
     }
     p->schedule_count = 0;
     ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0));
@@ -12128,7 +12128,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->mbuf = NULL;
     p->msg_frag = NULL;
     p->mbuf_sz = 0;
-    erts_smp_atomic_init_nob(&p->psd, (erts_aint_t) NULL);
+    erts_atomic_init_nob(&p->psd, (erts_aint_t) NULL);
     p->dictionary = NULL;
     p->seq_trace_lastcnt = 0;
     p->seq_trace_clock = 0;
@@ -12179,8 +12179,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
         }
         if (ARE_TRACE_FLAGS_ON(parent, F_TRACE_PROCS)) {
             locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
-            erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
-            erts_smp_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+            erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+            erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
             trace_proc_spawn(parent, am_spawn, p->common.id, mod, func, args);
             if (so->flags & SPO_LINK)
                 trace_proc(parent, locks, parent, am_link, p->common.id);
@@ -12192,8 +12192,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
               == (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) {
             /* This happens when parent was not traced, but child is */
             locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
-            erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
-            erts_smp_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+            erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+            erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
         }
         trace_proc_spawn(p, am_spawned, parent->common.id, mod, func, args);
         if (so->flags & SPO_LINK)
@@ -12232,7 +12232,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	so->mref = mref;
     }
 
-    erts_smp_proc_unlock(p, locks);
+    erts_proc_unlock(p, locks);
 
     res = p->common.id;
 
@@ -12240,7 +12240,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
      * Schedule process for execution.
      */
 
-    erts_smp_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR);
+    erts_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR);
 
     schedule_process(p, state, 0);
 
@@ -12260,7 +12260,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 
  error:
 
-    erts_smp_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR);
+    erts_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR);
 
     return res;
 }
@@ -12311,7 +12311,7 @@ void erts_init_empty_process(Process *p)
     p->mbuf = NULL;
     p->msg_frag = NULL;
     p->mbuf_sz = 0;
-    erts_smp_atomic_init_nob(&p->psd, (erts_aint_t) NULL);
+    erts_atomic_init_nob(&p->psd, (erts_aint_t) NULL);
     ERTS_P_MONITORS(p) = NULL;
     ERTS_P_LINKS(p) = NULL;         /* List of links */
     p->nodes_monitors = NULL;
@@ -12363,10 +12363,10 @@ void erts_init_empty_process(Process *p)
 #endif
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-    erts_smp_atomic32_init_nob(&p->dirty_state, 0);
+    erts_atomic32_init_nob(&p->dirty_state, 0);
     p->dirty_sys_tasks = NULL;
 #endif
-    erts_smp_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL);
+    erts_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL);
 
     p->scheduler_data = NULL;
     p->msg_inq.first = NULL;
@@ -12377,7 +12377,7 @@ void erts_init_empty_process(Process *p)
     p->pending_exit.reason = THE_NON_VALUE;
     p->pending_exit.bp = NULL;
     erts_proc_lock_init(p);
-    erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
+    erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
     RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0));
 
 #if !defined(NO_FPE_SIGNALS) || defined(HIPE)
@@ -12487,10 +12487,10 @@ delete_process(Process* p)
 
     /* Cleanup psd */
 
-    psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd);
+    psd = (ErtsPSD *) erts_atomic_read_nob(&p->psd);
 
     if (psd) {
-	erts_smp_atomic_set_nob(&p->psd, (erts_aint_t) NULL); /* Reduction counting depends on this... */
+	erts_atomic_set_nob(&p->psd, (erts_aint_t) NULL); /* Reduction counting depends on this... */
 	erts_free(ERTS_ALC_T_PSD, psd);
     }
 
@@ -12548,7 +12548,7 @@ set_proc_exiting(Process *p,
 {
     erts_aint32_t state = in_state, enq_prio = -1;
     int enqueue;
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCKS_ALL);
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCKS_ALL);
 
     enqueue = change_proc_schedule_state(p,
 					 (ERTS_PSFLG_SUSPENDED
@@ -12583,9 +12583,9 @@ set_proc_self_exiting(Process *c_p)
 #endif
     erts_aint32_t state, enq_prio = -1;
 
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCKS_ALL);
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCKS_ALL);
 
-    state = erts_smp_atomic32_read_nob(&c_p->state);
+    state = erts_atomic32_read_nob(&c_p->state);
     ASSERT(state & (ERTS_PSFLG_RUNNING
 		    |ERTS_PSFLG_RUNNING_SYS
 		    | ERTS_PSFLG_DIRTY_RUNNING
@@ -12611,31 +12611,31 @@ erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks)
 {
     ErtsProcLocks xlocks;
     ASSERT(is_value(c_p->pending_exit.reason));
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == locks);
-    ERTS_SMP_LC_ASSERT(locks & ERTS_PROC_LOCK_MAIN);
-    ERTS_SMP_LC_ASSERT(!((ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE)
-			 & erts_smp_atomic32_read_nob(&c_p->state)));
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == locks);
+    ERTS_LC_ASSERT(locks & ERTS_PROC_LOCK_MAIN);
+    ERTS_LC_ASSERT(!((ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE)
+			 & erts_atomic32_read_nob(&c_p->state)));
 
     /* Ensure that all locks on c_p are locked before proceeding... */
     if (locks == ERTS_PROC_LOCKS_ALL)
 	xlocks = 0;
     else {
 	xlocks = ~locks & ERTS_PROC_LOCKS_ALL;
-	if (erts_smp_proc_trylock(c_p, xlocks) == EBUSY) {
-	    erts_smp_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN);
-	    erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+	if (erts_proc_trylock(c_p, xlocks) == EBUSY) {
+	    erts_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN);
+	    erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 	}
     }
 
     set_proc_exiting(c_p,
-		     erts_smp_atomic32_read_acqb(&c_p->state),
+		     erts_atomic32_read_acqb(&c_p->state),
 		     c_p->pending_exit.reason,
 		     c_p->pending_exit.bp);
     c_p->pending_exit.reason = THE_NON_VALUE;
     c_p->pending_exit.bp = NULL;
 
     if (xlocks)
-	erts_smp_proc_unlock(c_p, xlocks);
+	erts_proc_unlock(c_p, xlocks);
 }
 
 static void save_pending_exiter(Process *p, ErtsProcList *plp);
@@ -12656,9 +12656,9 @@ do_handle_pending_exiters(ErtsProcList *pnd_xtrs)
 	     * pending exit will soon be detected and handled by the
 	     * scheduler running the process (at schedule in/out).
 	     */
-	    if (erts_smp_proc_trylock(p, ERTS_PROC_LOCKS_ALL) != EBUSY) {
+	    if (erts_proc_trylock(p, ERTS_PROC_LOCKS_ALL) != EBUSY) {
 		if (erts_proclist_same(plp, p)) {
-		    state = erts_smp_atomic32_read_acqb(&p->state);
+		    state = erts_atomic32_read_acqb(&p->state);
 		    if (!(state & (ERTS_PSFLG_RUNNING
 				   | ERTS_PSFLG_RUNNING_SYS
 				   | ERTS_PSFLG_EXITING))) {
@@ -12666,12 +12666,12 @@ do_handle_pending_exiters(ErtsProcList *pnd_xtrs)
 			erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL);
 		    }
 		}
-		erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
+		erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
 	    }
 	    else {
-		erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+		erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
 		if (erts_proclist_same(plp, p)) {
-		    state = erts_smp_atomic32_read_acqb(&p->state);
+		    state = erts_atomic32_read_acqb(&p->state);
 		    if (!(state & (ERTS_PSFLG_RUNNING
 				   | ERTS_PSFLG_RUNNING_SYS
 				   | ERTS_PSFLG_EXITING))) {
@@ -12683,7 +12683,7 @@ do_handle_pending_exiters(ErtsProcList *pnd_xtrs)
 			plp = NULL;
 		    }
 		}
-		erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+		erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 	    }
 	}
 	if (plp)
@@ -12698,7 +12698,7 @@ save_pending_exiter(Process *p, ErtsProcList *plp)
     ErtsSchedulerSleepInfo *ssi;
     ErtsRunQueue *rq;
 
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
 
     rq = RUNQ_READ_RQ(&p->run_queue);
     ASSERT(rq && !ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
@@ -12706,7 +12706,7 @@ save_pending_exiter(Process *p, ErtsProcList *plp)
     if (!plp)
 	plp = proclist_create(p);
 
-    erts_smp_runq_lock(rq);
+    erts_runq_lock(rq);
 
     erts_proclist_store_last(&rq->procs.pending_exiters, plp);
 
@@ -12714,7 +12714,7 @@ save_pending_exiter(Process *p, ErtsProcList *plp)
 
     ssi = rq->scheduler->ssi;
 
-    erts_smp_runq_unlock(rq);
+    erts_runq_unlock(rq);
 
     set_aux_work_flags_wakeup_nob(ssi, ERTS_SSI_AUX_WORK_PENDING_EXITERS);
 }
@@ -12854,11 +12854,11 @@ send_exit_signal(Process *c_p,		/* current process if and only
 		 Uint32 flags		/* flags */
     )		
 {
-    erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state);
+    erts_aint32_t state = erts_atomic32_read_nob(&rp->state);
     Eterm rsn = reason == am_kill ? am_killed : reason;
 
-    ERTS_SMP_LC_ASSERT(*rp_locks == erts_proc_lc_my_proc_locks(rp));
-    ERTS_SMP_LC_ASSERT((*rp_locks & ERTS_PROC_LOCKS_XSIG_SEND)
+    ERTS_LC_ASSERT(*rp_locks == erts_proc_lc_my_proc_locks(rp));
+    ERTS_LC_ASSERT((*rp_locks & ERTS_PROC_LOCKS_XSIG_SEND)
 		       == ERTS_PROC_LOCKS_XSIG_SEND);
 
     ASSERT(reason != THE_NON_VALUE);
@@ -12879,7 +12879,7 @@ send_exit_signal(Process *c_p,		/* current process if and only
     if ((state & ERTS_PSFLG_TRAP_EXIT)
 	&& (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) {
         /* have to release the status lock in order to send the exit message */
-        erts_smp_proc_unlock(rp, *rp_locks & ERTS_PROC_LOCKS_XSIG_SEND);
+        erts_proc_unlock(rp, *rp_locks & ERTS_PROC_LOCKS_XSIG_SEND);
         *rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND;
         if (have_seqtrace(token) && token_update)
 	    seq_trace_update_send(token_update);
@@ -12899,10 +12899,10 @@ send_exit_signal(Process *c_p,		/* current process if and only
 		if (*rp_locks != ERTS_PROC_LOCKS_ALL) {
 		    ErtsProcLocks need_locks = (~(*rp_locks)
 						& ERTS_PROC_LOCKS_ALL);
-		    if (erts_smp_proc_trylock(c_p, need_locks) == EBUSY) {
-			erts_smp_proc_unlock(c_p,
+		    if (erts_proc_trylock(c_p, need_locks) == EBUSY) {
+			erts_proc_unlock(c_p,
 					     *rp_locks & ~ERTS_PROC_LOCK_MAIN);
-			erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+			erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 		    }
 		    *rp_locks = ERTS_PROC_LOCKS_ALL;
 		}
@@ -12916,7 +12916,7 @@ send_exit_signal(Process *c_p,		/* current process if and only
 		ErlHeapFragment *bp = NULL;
 		Eterm rsn_cpy;
 		if (need_locks
-		    && erts_smp_proc_trylock(rp, need_locks) == EBUSY) {
+		    && erts_proc_trylock(rp, need_locks) == EBUSY) {
 		    /* ... but we havn't got all locks on it ... */
 		    save_pending_exiter(rp, NULL); 
 		    /*
@@ -12928,7 +12928,7 @@ send_exit_signal(Process *c_p,		/* current process if and only
                 /* ...and we have all locks on it... */
                 *rp_locks = ERTS_PROC_LOCKS_ALL;
 
-		state = erts_smp_atomic32_read_nob(&rp->state);
+		state = erts_atomic32_read_nob(&rp->state);
 
 		if (is_immed(rsn))
 		    rsn_cpy = rsn;
@@ -12984,11 +12984,11 @@ send_exit_signal(Process *c_p,		/* current process if and only
                  * queue...
 		 */
 #ifndef ERTS_DIRTY_SCHEDULERS
-                (void) erts_smp_atomic32_read_bor_relb(&rp->state,
+                (void) erts_atomic32_read_bor_relb(&rp->state,
                                                        ERTS_PSFLG_PENDING_EXIT);
 #else
                 {
-                    erts_aint32_t a = erts_smp_atomic32_read_nob(&rp->state);
+                    erts_aint32_t a = erts_atomic32_read_nob(&rp->state);
                     while (1) {
                         erts_aint32_t n, e;
                         int dwork;
@@ -12996,7 +12996,7 @@ send_exit_signal(Process *c_p,		/* current process if and only
                         n |= ERTS_PSFLG_PENDING_EXIT;
                         dwork = !!(n & ERTS_PSFLGS_DIRTY_WORK);
                         n &= ~ERTS_PSFLGS_DIRTY_WORK;
-                        a = erts_smp_atomic32_cmpxchg_mb(&rp->state, n, e);
+                        a = erts_atomic32_cmpxchg_mb(&rp->state, n, e);
                         if (a == e) {
                             if (dwork)
                                 erts_schedule_process(rp, n, *rp_locks);
@@ -13063,9 +13063,9 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 	    ASSERT(is_node_name_atom(mon->u.pid));
 	    dep = erts_sysname_to_connected_dist_entry(mon->u.pid);
 	    if (dep) {
-		erts_smp_de_links_lock(dep);
+		erts_de_links_lock(dep);
 		rmon = erts_remove_monitor(&(dep->monitors), mon->ref);
-		erts_smp_de_links_unlock(dep);
+		erts_de_links_unlock(dep);
 		if (rmon) {
 		    ErtsDSigData dsd;
 		    int code = erts_dsig_prepare(&dsd, dep, NULL,
@@ -13091,7 +13091,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		    goto done;
 		}
 		rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref);
-		erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+		erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 		if (rmon == NULL) {
 		    goto done;
 		}
@@ -13110,9 +13110,9 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		dep = external_pid_dist_entry(mon->u.pid);
 		ASSERT(dep != NULL);
 		if (dep) {
-		    erts_smp_de_links_lock(dep);
+		    erts_de_links_lock(dep);
 		    rmon = erts_remove_monitor(&(dep->monitors), mon->ref);
-		    erts_smp_de_links_unlock(dep);
+		    erts_de_links_unlock(dep);
 		    if (rmon) {
 			ErtsDSigData dsd;
 			int code = erts_dsig_prepare(&dsd, dep, NULL,
@@ -13163,15 +13163,15 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 	    }
 	    UnUseTmpHeapNoproc(3);
 	    /* else: demonitor while we exited, i.e. do nothing... */
-	    erts_smp_proc_unlock(rp, rp_locks);
+	    erts_proc_unlock(rp, rp_locks);
 	} else { /* external by pid or name */
 	    ASSERT(is_external_pid(mon->u.pid));    
 	    dep = external_pid_dist_entry(mon->u.pid);
 	    ASSERT(dep != NULL);
 	    if (dep) {
-		erts_smp_de_links_lock(dep);
+		erts_de_links_lock(dep);
 		rmon = erts_remove_monitor(&(dep->monitors), mon->ref);
-		erts_smp_de_links_unlock(dep);
+		erts_de_links_unlock(dep);
 		if (rmon) {
 		    ErtsDSigData dsd;
 		    int code = erts_dsig_prepare(&dsd, dep, NULL,
@@ -13279,7 +13279,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
 			/* We didn't exit the process and it is traced */
 			if (IS_TRACED_FL(rp, F_TRACE_PROCS)) {
                             if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) {
-                                erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND);
+                                erts_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND);
                                 rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND;
                             }
 			    trace_proc(NULL, 0, rp, am_getting_unlinked, p->common.id);
@@ -13287,7 +13287,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
 		    }
 		}
 		ASSERT(rp != p);
-		erts_smp_proc_unlock(rp, rp_locks);
+		erts_proc_unlock(rp, rp_locks);
 	    }
 	}
 	else if (is_external_pid(item)) {
@@ -13297,14 +13297,14 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
 		int code;
 		ErtsDistLinkData dld;
 		erts_remove_dist_link(&dld, p->common.id, item, dep);
-		erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+		erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
 		code = erts_dsig_prepare(&dsd, dep, p, ERTS_DSP_NO_LOCK, 0);
 		if (code == ERTS_DSIG_PREP_CONNECTED) {
 		    code = erts_dsig_send_exit_tt(&dsd, p->common.id, item,
 						  reason, SEQ_TRACE_TOKEN(p));
 		    ASSERT(code == ERTS_DSIG_SEND_OK);
 		}
-		erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
+		erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
 		erts_destroy_dist_link(&dld);
 	    }
 	}
@@ -13315,9 +13315,9 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
 	if(dep) {
 	    /* dist entries have node links in a separate structure to 
 	       avoid confusion */
-	    erts_smp_de_links_lock(dep);
+	    erts_de_links_lock(dep);
 	    rlnk = erts_remove_link(&(dep->node_links), p->common.id);
-	    erts_smp_de_links_unlock(dep);
+	    erts_de_links_unlock(dep);
 	    if (rlnk)
 		erts_destroy_link(rlnk);
 	    erts_deref_dist_entry(dep);
@@ -13340,7 +13340,7 @@ resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p)
 	ASSERT(suspendee != vc_p);
 	if (smon->active)
 	    resume_process(suspendee, ERTS_PROC_LOCK_STATUS);
-	erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
     }
     erts_destroy_suspend_monitor(smon);
 }
@@ -13369,12 +13369,12 @@ erts_do_exit_process(Process* p, Eterm reason)
 	erts_exit(ERTS_DUMP_EXIT, "System process %T terminated: %T\n",
                  p->common.id, reason);
 
-    ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
+    ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
     /* By locking all locks (main lock is already locked) when going
        to exiting state (ERTS_PSFLG_EXITING), it is enough to take any lock when
        looking up a process (erts_pid2proc()) to prevent the looked up
        process from exiting until the lock has been released. */
-    erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+    erts_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
 
     if (ERTS_PSFLG_PENDING_EXIT & set_proc_self_exiting(p)) {
 	/* Process exited before pending exit was received... */
@@ -13387,7 +13387,7 @@ erts_do_exit_process(Process* p, Eterm reason)
 
     cancel_suspend_of_suspendee(p, ERTS_PROC_LOCKS_ALL); 
 
-    ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
+    ERTS_MSGQ_MV_INQ2PRIVQ(p);
 
     if (IS_TRACED(p)) {
 	if (IS_TRACED_FL(p, F_TRACE_CALLS))
@@ -13406,7 +13406,7 @@ erts_do_exit_process(Process* p, Eterm reason)
 	ASSERT(erts_proc_read_refc(p) > 0);
     }
 
-    erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+    erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
 
     if (IS_TRACED_FL(p,F_TRACE_PROCS))
         trace_proc(p, ERTS_PROC_LOCK_MAIN, p, am_exit, reason);
@@ -13436,7 +13436,7 @@ erts_continue_exit_process(Process *p)
     int yield_allowed = 1;
 #endif
 
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p));
 
     ASSERT(ERTS_PROC_IS_EXITING(p));
 
@@ -13501,7 +13501,7 @@ erts_continue_exit_process(Process *p)
     }
 
     erts_set_gc_state(p, 1);
-    state = erts_smp_atomic32_read_acqb(&p->state);
+    state = erts_atomic32_read_acqb(&p->state);
     if (state & ERTS_PSFLG_ACTIVE_SYS
 #ifdef ERTS_DIRTY_SCHEDULERS
         || p->dirty_sys_tasks
@@ -13512,13 +13512,13 @@ erts_continue_exit_process(Process *p)
     }
 
 #ifdef DEBUG
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
     ASSERT(p->sys_task_qs == NULL);
     ASSERT(ERTS_PROC_GET_DELAYED_GC_TASK_QS(p) == NULL);
 #ifdef ERTS_DIRTY_SCHEDULERS
     ASSERT(p->dirty_sys_tasks == NULL);
 #endif
-    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+    erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 #endif
 
     if (p->flags & F_USING_DDLL) {
@@ -13551,7 +13551,7 @@ erts_continue_exit_process(Process *p)
     if (IS_TRACED_FL(p, F_TRACE_SCHED_EXIT))
         trace_sched(p, curr_locks, am_out_exited);
 
-    erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+    erts_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
     curr_locks = ERTS_PROC_LOCKS_ALL;
 
     /*
@@ -13574,7 +13574,7 @@ erts_continue_exit_process(Process *p)
 	ErtsRunQueue *rq;
 	rq = erts_get_runq_current(erts_proc_sched_data(p));
 
-	erts_smp_runq_lock(rq);
+	erts_runq_lock(rq);
 
 	ASSERT(p->scheduler_data);
 	ASSERT(p->scheduler_data->current_process == p);
@@ -13586,7 +13586,7 @@ erts_continue_exit_process(Process *p)
 	/* Time of death! */
 	erts_ptab_delete_element(&erts_proc, &p->common);
 
-	erts_smp_runq_unlock(rq);
+	erts_runq_unlock(rq);
     }
 
     /*
@@ -13598,7 +13598,7 @@ erts_continue_exit_process(Process *p)
 
     {
 	/* Inactivate and notify free */
-	erts_aint32_t n, e, a = erts_smp_atomic32_read_nob(&p->state);
+	erts_aint32_t n, e, a = erts_atomic32_read_nob(&p->state);
 	int refc_inced = 0;
 	while (1) {
 	    n = e = a;
@@ -13609,7 +13609,7 @@ erts_continue_exit_process(Process *p)
 		erts_proc_inc_refc(p);
 		refc_inced = 1;
 	    }
-	    a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
+	    a = erts_atomic32_cmpxchg_mb(&p->state, n, e);
 	    if (a == e)
 		break;
 	}
@@ -13632,7 +13632,7 @@ erts_continue_exit_process(Process *p)
     
     dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL;
 
-    erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
+    erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
 
     if (dep) {
 	erts_do_net_exits(dep, reason);
@@ -13674,8 +13674,8 @@ erts_continue_exit_process(Process *p)
     if (!delay_del_proc)
 	delete_process(p);
 
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
-    ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
+    erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+    ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
     return;
 
@@ -13685,20 +13685,20 @@ erts_continue_exit_process(Process *p)
     ASSERT(yield_allowed);
 #endif
 
-    ERTS_SMP_LC_ASSERT(curr_locks == erts_proc_lc_my_proc_locks(p));
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & curr_locks);
+    ERTS_LC_ASSERT(curr_locks == erts_proc_lc_my_proc_locks(p));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & curr_locks);
 
     p->i = (BeamInstr *) beam_continue_exit;
 
     if (!(curr_locks & ERTS_PROC_LOCK_STATUS)) {
-	erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+	erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
 	curr_locks |= ERTS_PROC_LOCK_STATUS;
     }
 
     if (curr_locks != ERTS_PROC_LOCK_MAIN)
-	erts_smp_proc_unlock(p, ~ERTS_PROC_LOCK_MAIN & curr_locks);
+	erts_proc_unlock(p, ~ERTS_PROC_LOCK_MAIN & curr_locks);
 
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p));
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p));
 
     BUMP_ALL_REDS(p);
 }
@@ -13734,7 +13734,7 @@ erts_program_counter_info(fmtfn_t to, void *to_arg, Process *p)
     erts_print(to, to_arg, "CP: %p (", p->cp);
     print_function_from_pc(to, to_arg, p->cp);
     erts_print(to, to_arg, ")\n");
-    state = erts_smp_atomic32_read_acqb(&p->state);
+    state = erts_atomic32_read_acqb(&p->state);
     if (!(state & (ERTS_PSFLG_RUNNING
 		   | ERTS_PSFLG_RUNNING_SYS
 		   | ERTS_PSFLG_GC))) {
@@ -13815,7 +13815,7 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
 
     erts_print(to, to_arg, "=scheduler:%u\n", esdp->no);
 
-    flg = erts_smp_atomic32_read_dirty(&esdp->ssi->flags);
+    flg = erts_atomic32_read_dirty(&esdp->ssi->flags);
     erts_print(to, to_arg, "Scheduler Sleep Info Flags: ");
     for (i = 0; i < ERTS_SSI_FLGS_MAX && flg; i++) {
         erts_aint32_t chk = (1 << i);
@@ -13884,12 +13884,12 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
             break;
         }
         erts_print(to, to_arg, "Length: %d\n",
-                   erts_smp_atomic32_read_dirty(&esdp->run_queue->procs.prio_info[i].len));
+                   erts_atomic32_read_dirty(&esdp->run_queue->procs.prio_info[i].len));
     }
     erts_print(to, to_arg, "Run Queue Port Length: %d\n",
-               erts_smp_atomic32_read_dirty(&esdp->run_queue->ports.info.len));
+               erts_atomic32_read_dirty(&esdp->run_queue->ports.info.len));
 
-    flg = erts_smp_atomic32_read_dirty(&esdp->run_queue->flags);
+    flg = erts_atomic32_read_dirty(&esdp->run_queue->flags);
     erts_print(to, to_arg, "Run Queue Flags: ");
     for (i = 0; i < ERTS_RUNQ_FLG_MAX && flg; i++) {
         erts_aint32_t chk = (1 << i);
@@ -13961,7 +13961,7 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
     p = esdp->current_process;
     erts_print(to, to_arg, "Current Process: ");
     if (esdp->current_process && !(ERTS_TRACE_FLAGS(p) & F_SENSITIVE)) {
-	flg = erts_smp_atomic32_read_dirty(&p->state);
+	flg = erts_atomic32_read_dirty(&p->state);
 	erts_print(to, to_arg, "%T\n", p->common.id);
 
 	erts_print(to, to_arg, "Current Process State: ");
@@ -14011,7 +14011,7 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
  */
 void erts_halt(int code)
 {
-    if (-1 == erts_smp_atomic32_cmpxchg_acqb(&erts_halt_progress,
+    if (-1 == erts_atomic32_cmpxchg_acqb(&erts_halt_progress,
 					     erts_no_schedulers,
 					     -1)) {
 #ifdef ERTS_DIRTY_SCHEDULERS
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 513397ef3f..7ca37882c2 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -47,7 +47,6 @@ typedef struct process Process;
 #include "erl_port.h"
 #undef ERL_PORT_GET_PORT_TYPE_ONLY__
 #include "erl_vm.h"
-#include "erl_smp.h"
 #include "erl_message.h"
 #include "erl_process_dict.h"
 #include "erl_node_container_utils.h"
@@ -222,31 +221,31 @@ extern int erts_dio_sched_thread_suggested_stack_size;
   ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO)))
 
 #define ERTS_RUNQ_FLGS_INIT(RQ, INIT)					\
-    erts_smp_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT))
+    erts_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT))
 #define ERTS_RUNQ_FLGS_SET(RQ, FLGS)					\
-    ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags,		\
+    ((Uint32) erts_atomic32_read_bor_relb(&(RQ)->flags,		\
 					      (erts_aint32_t) (FLGS)))
 #define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS)				\
-    ((Uint32) erts_smp_atomic32_read_bor_nob(&(RQ)->flags,		\
+    ((Uint32) erts_atomic32_read_bor_nob(&(RQ)->flags,		\
 					     (erts_aint32_t) (FLGS)))
 #define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS)				\
-    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags,		\
+    ((Uint32) erts_atomic32_read_bset_relb(&(RQ)->flags,		\
 					       (erts_aint32_t) (MSK),	\
 					       (erts_aint32_t) (FLGS)))
 #define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS)					\
-    ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags,		\
+    ((Uint32) erts_atomic32_read_band_relb(&(RQ)->flags,		\
 					       (erts_aint32_t) ~(FLGS)))
 #define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS)					\
-    ((Uint32) erts_smp_atomic32_read_band_nob(&(RQ)->flags,		\
+    ((Uint32) erts_atomic32_read_band_nob(&(RQ)->flags,		\
 					      (erts_aint32_t) ~(FLGS)))
 #define ERTS_RUNQ_FLGS_GET(RQ)						\
-    ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags))
+    ((Uint32) erts_atomic32_read_acqb(&(RQ)->flags))
 #define ERTS_RUNQ_FLGS_GET_NOB(RQ)					\
-    ((Uint32) erts_smp_atomic32_read_nob(&(RQ)->flags))
+    ((Uint32) erts_atomic32_read_nob(&(RQ)->flags))
 #define ERTS_RUNQ_FLGS_GET_MB(RQ)					\
-    ((Uint32) erts_smp_atomic32_read_mb(&(RQ)->flags))
+    ((Uint32) erts_atomic32_read_mb(&(RQ)->flags))
 #define ERTS_RUNQ_FLGS_READ_BSET(RQ, MSK, FLGS)		  		\
-    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, 		\
+    ((Uint32) erts_atomic32_read_bset_relb(&(RQ)->flags, 		\
 					       (erts_aint32_t) (MSK),	\
 					       (erts_aint32_t) (FLGS)))
 
@@ -365,7 +364,7 @@ typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
 typedef struct {
-    erts_smp_spinlock_t lock;
+    erts_spinlock_t lock;
     ErtsSchedulerSleepInfo *list;
 } ErtsSchedulerSleepList;
 #endif
@@ -373,7 +372,7 @@ typedef struct {
 struct ErtsSchedulerSleepInfo_ {
     ErtsSchedulerSleepInfo *next;
     ErtsSchedulerSleepInfo *prev;
-    erts_smp_atomic32_t flags;
+    erts_atomic32_t flags;
     erts_tse_t *event;
     erts_atomic32_t aux_work;
 };
@@ -418,7 +417,7 @@ typedef struct ErtsSchedulerData_ ErtsSchedulerData;
 typedef struct ErtsRunQueue_ ErtsRunQueue;
 
 typedef struct {
-    erts_smp_atomic32_t len;
+    erts_atomic32_t len;
     erts_aint32_t max_len;
     int reds;
 } ErtsRunQueueInfo;
@@ -475,8 +474,8 @@ struct ErtsMigrationPaths_ {
 struct ErtsRunQueue_ {
     int ix;
 
-    erts_smp_mtx_t mtx;
-    erts_smp_cnd_t cnd;
+    erts_mtx_t mtx;
+    erts_cnd_t cnd;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
     ErtsSchedulerSleepList sleepers;
@@ -485,13 +484,13 @@ struct ErtsRunQueue_ {
     ErtsSchedulerData *scheduler;
     int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
     int woken;
-    erts_smp_atomic32_t flags;
+    erts_atomic32_t flags;
     int check_balance_reds;
     int full_reds_history_sum;
     int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE];
     int out_of_work_count;
     erts_aint32_t max_len;
-    erts_smp_atomic32_t len;
+    erts_atomic32_t len;
     int wakeup_other;
     int wakeup_other_reds;
 
@@ -510,7 +509,7 @@ struct ErtsRunQueue_ {
     struct {
 	ErtsMiscOpList *start;
 	ErtsMiscOpList *end;
-	erts_smp_atomic_t evac_runq;
+	erts_atomic_t evac_runq;
     } misc;
 
     struct {
@@ -695,7 +694,7 @@ extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
 
 
 #if defined(ERTS_ENABLE_LOCK_CHECK)
-int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
+int erts_lc_runq_is_locked(ErtsRunQueue *);
 #endif
 
 #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
@@ -709,20 +708,20 @@ void erts_non_empty_runq(ErtsRunQueue *rq);
  * other threads peek at values without run queue lock.
  */
 
-ERTS_GLB_INLINE void erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
-ERTS_GLB_INLINE void erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
-ERTS_GLB_INLINE void erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi);
+ERTS_GLB_INLINE void erts_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
+ERTS_GLB_INLINE void erts_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
+ERTS_GLB_INLINE void erts_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_INLINE void
-erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
+erts_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
 {
     erts_aint32_t len;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
 
-    len = erts_smp_atomic32_read_dirty(&rq->len);
+    len = erts_atomic32_read_dirty(&rq->len);
 
     if (len == 0)
 	erts_non_empty_runq(rq);
@@ -730,63 +729,63 @@ erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
     if (rq->max_len < len)
 	rq->max_len = len;
     ASSERT(len > 0);
-    erts_smp_atomic32_set_nob(&rq->len, len);
+    erts_atomic32_set_nob(&rq->len, len);
 
-    len = erts_smp_atomic32_read_dirty(&rqi->len);
+    len = erts_atomic32_read_dirty(&rqi->len);
     ASSERT(len >= 0);
     if (len == 0) {
-	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
+	ASSERT((erts_atomic32_read_nob(&rq->flags)
 		& ((erts_aint32_t) (1 << prio))) == 0);
-	erts_smp_atomic32_read_bor_nob(&rq->flags,
+	erts_atomic32_read_bor_nob(&rq->flags,
 				       (erts_aint32_t) (1 << prio));
     }
     len++;
     if (rqi->max_len < len)
 	rqi->max_len = len;
 
-    erts_smp_atomic32_set_relb(&rqi->len, len);
+    erts_atomic32_set_relb(&rqi->len, len);
 }
 
 ERTS_GLB_INLINE void
-erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
+erts_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
 {
     erts_aint32_t len;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
 
-    len = erts_smp_atomic32_read_dirty(&rq->len);
+    len = erts_atomic32_read_dirty(&rq->len);
     len--;
     ASSERT(len >= 0);
-    erts_smp_atomic32_set_nob(&rq->len, len);
+    erts_atomic32_set_nob(&rq->len, len);
 
-    len = erts_smp_atomic32_read_dirty(&rqi->len);
+    len = erts_atomic32_read_dirty(&rqi->len);
     len--;
     ASSERT(len >= 0);
     if (len == 0) {
-	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
+	ASSERT((erts_atomic32_read_nob(&rq->flags)
 		& ((erts_aint32_t) (1 << prio))));
-	erts_smp_atomic32_read_band_nob(&rq->flags,
+	erts_atomic32_read_band_nob(&rq->flags,
 					~((erts_aint32_t) (1 << prio)));
     }
-    erts_smp_atomic32_set_relb(&rqi->len, len);
+    erts_atomic32_set_relb(&rqi->len, len);
 
 }
 
 ERTS_GLB_INLINE void
-erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
+erts_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
 {
     erts_aint32_t len;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
 
-    len = erts_smp_atomic32_read_dirty(&rqi->len);
+    len = erts_atomic32_read_dirty(&rqi->len);
     ASSERT(rqi->max_len >= len);
     rqi->max_len = len;
 }
 
 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-#define RUNQ_READ_LEN(X) erts_smp_atomic32_read_nob((X))
+#define RUNQ_READ_LEN(X) erts_atomic32_read_nob((X))
 
 #endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
 
@@ -859,7 +858,7 @@ extern ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];
 #define ERTS_SCHED_STAT_MODIFY_CLEAR		3
 
 typedef struct {
-    erts_smp_spinlock_t lock;
+    erts_spinlock_t lock;
     int enabled;
     struct {
 	Eterm name;
@@ -1043,7 +1042,7 @@ struct process {
     ErlHeapFragment* live_hf_end;
     ErtsMessage *msg_frag;	/* Pointer to message fragment list */
     Uint mbuf_sz;		/* Total size of heap fragments and message fragments */
-    erts_smp_atomic_t psd;		/* Rarely used process specific data */
+    erts_atomic_t psd;		/* Rarely used process specific data */
 
     Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
     Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
@@ -1054,9 +1053,9 @@ struct process {
     ErtsProcSysTask *dirty_sys_tasks;
 #endif
 
-    erts_smp_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */
+    erts_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */
 #ifdef ERTS_DIRTY_SCHEDULERS
-    erts_smp_atomic32_t dirty_state; /* Process dirty state flags (see ERTS_PDSFLG_*) */
+    erts_atomic32_t dirty_state; /* Process dirty state flags (see ERTS_PDSFLG_*) */
 #endif
 
     ErlMessageInQueue msg_inq;
@@ -1066,7 +1065,7 @@ struct process {
     ErtsSchedulerData *scheduler_data;
     Eterm suspendee;
     ErtsPendingSuspend *pending_suspenders;
-    erts_smp_atomic_t run_queue;
+    erts_atomic_t run_queue;
 #ifdef HIPE
     struct hipe_process_state_smp hipe_smp;
 #endif
@@ -1332,7 +1331,7 @@ Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
 Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
 #endif
 
-extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
+extern erts_rwmtx_t erts_cpu_bind_rwmtx;
 /* If any of the erts_system_monitor_* variables are set (enabled),
 ** erts_system_monitor must be != NIL, to allow testing on just
 ** the erts_system_monitor_* variables.
@@ -1855,7 +1854,7 @@ int erts_send_exit_signal(Process *,
 			  Uint32);
 void erts_handle_pending_exit(Process *, ErtsProcLocks);
 #define ERTS_PROC_PENDING_EXIT(P) \
-    (ERTS_PSFLG_PENDING_EXIT & erts_smp_atomic32_read_acqb(&(P)->state))
+    (ERTS_PSFLG_PENDING_EXIT & erts_atomic32_read_acqb(&(P)->state))
 
 void erts_deep_process_dump(fmtfn_t, void *);
 
@@ -1897,7 +1896,7 @@ ERTS_GLB_INLINE void
 erts_proc_notify_new_message(Process *p, ErtsProcLocks locks)
 {
     /* No barrier needed, due to msg lock */
-    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
+    erts_aint32_t state = erts_atomic32_read_nob(&p->state);
     if (!(state & ERTS_PSFLG_ACTIVE))
 	erts_schedule_process(p, state, locks);
 }
@@ -1907,7 +1906,7 @@ erts_schedule_dirty_sys_execution(Process *c_p)
 {
     erts_aint32_t a, n, e;
 
-    a = erts_smp_atomic32_read_nob(&c_p->state);
+    a = erts_atomic32_read_nob(&c_p->state);
 
     /*
      * Only a currently executing process schedules
@@ -1923,7 +1922,7 @@ erts_schedule_dirty_sys_execution(Process *c_p)
                   | ERTS_PSFLG_PENDING_EXIT))) {
         e = a;
         n = a | ERTS_PSFLG_DIRTY_ACTIVE_SYS;
-        a = erts_smp_atomic32_cmpxchg_mb(&c_p->state, n, e);
+        a = erts_atomic32_cmpxchg_mb(&c_p->state, n, e);
         if (a == e)
             break; /* dirty-active-sys set */
     }
@@ -1937,15 +1936,15 @@ erts_schedule_dirty_sys_execution(Process *c_p)
 #include "erl_process_lock.h"
 #undef ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__
 
-#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)				\
+#define ERTS_LC_CHK_RUNQ_LOCK(RQ, L)				\
 do {									\
     if ((L))								\
-	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked((RQ)));		\
+	ERTS_LC_ASSERT(erts_lc_runq_is_locked((RQ)));		\
     else								\
-	ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked((RQ)));		\
+	ERTS_LC_ASSERT(!erts_lc_runq_is_locked((RQ)));		\
 } while (0)
 #else
-#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)
+#define ERTS_LC_CHK_RUNQ_LOCK(RQ, L)
 #endif
 
 void *erts_psd_set_init(Process *p, int ix, void *data);
@@ -1964,19 +1963,19 @@ erts_psd_get(Process *p, int ix)
 #if defined(ERTS_ENABLE_LOCK_CHECK)
     ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
     if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks)
-	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
+	ERTS_LC_ASSERT(locks || erts_thr_progress_is_blocking());
     else {
 	locks &= erts_psd_required_locks[ix].get_locks;
-	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks
+	ERTS_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks
 			   || erts_thr_progress_is_blocking());
     }
 #endif
 
-    psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd);
+    psd = (ErtsPSD *) erts_atomic_read_nob(&p->psd);
     ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
     if (!psd)
 	return NULL;
-    ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
+    ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
     return psd->data[ix];
 }
 
@@ -1986,18 +1985,18 @@ erts_psd_set(Process *p, int ix, void *data)
     ErtsPSD *psd;
 #if defined(ERTS_ENABLE_LOCK_CHECK)
     ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
-    erts_aint32_t state = state = erts_smp_atomic32_read_nob(&p->state);
+    erts_aint32_t state = state = erts_atomic32_read_nob(&p->state);
     if (!(state & ERTS_PSFLG_FREE)) {
 	if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks)
-	    ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
+	    ERTS_LC_ASSERT(locks || erts_thr_progress_is_blocking());
 	else {
 	    locks &= erts_psd_required_locks[ix].set_locks;
-	    ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks
+	    ERTS_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks
 			       || erts_thr_progress_is_blocking());
 	}
     }
 #endif
-    psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd);
+    psd = (ErtsPSD *) erts_atomic_read_nob(&p->psd);
     ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
     if (psd) {
 	void *old;
@@ -2196,13 +2195,13 @@ ERTS_GLB_INLINE Eterm erts_get_current_pid(void);
 ERTS_GLB_INLINE Uint erts_get_scheduler_id(void);
 ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p);
 ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp);
-ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq);
-ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq);
-ERTS_GLB_INLINE void erts_smp_runq_unlock(ErtsRunQueue *rq);
-ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
-ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
-ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
-ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
+ERTS_GLB_INLINE void erts_runq_lock(ErtsRunQueue *rq);
+ERTS_GLB_INLINE int erts_runq_trylock(ErtsRunQueue *rq);
+ERTS_GLB_INLINE void erts_runq_unlock(ErtsRunQueue *rq);
+ERTS_GLB_INLINE void erts_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
+ERTS_GLB_INLINE void erts_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
+ERTS_GLB_INLINE void erts_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
+ERTS_GLB_INLINE void erts_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
 
 ERTS_GLB_INLINE ErtsMessage *erts_alloc_message_heap_state(Process *pp,
 							   erts_aint32_t *psp,
@@ -2295,70 +2294,70 @@ erts_get_runq_current(ErtsSchedulerData *esdp)
 }
 
 ERTS_GLB_INLINE void
-erts_smp_runq_lock(ErtsRunQueue *rq)
+erts_runq_lock(ErtsRunQueue *rq)
 {
-    erts_smp_mtx_lock(&rq->mtx);
+    erts_mtx_lock(&rq->mtx);
 }
 
 ERTS_GLB_INLINE int
-erts_smp_runq_trylock(ErtsRunQueue *rq)
+erts_runq_trylock(ErtsRunQueue *rq)
 {
-    return erts_smp_mtx_trylock(&rq->mtx);
+    return erts_mtx_trylock(&rq->mtx);
 }
 
 ERTS_GLB_INLINE void
-erts_smp_runq_unlock(ErtsRunQueue *rq)
+erts_runq_unlock(ErtsRunQueue *rq)
 {
-    erts_smp_mtx_unlock(&rq->mtx);
+    erts_mtx_unlock(&rq->mtx);
 }
 
 ERTS_GLB_INLINE void
-erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
+erts_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx));
+    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&rq->mtx));
     if (xrq != rq) {
-	if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) {
+	if (erts_mtx_trylock(&xrq->mtx) == EBUSY) {
 	    if (rq < xrq)
-		erts_smp_mtx_lock(&xrq->mtx);
+		erts_mtx_lock(&xrq->mtx);
 	    else {
-		erts_smp_mtx_unlock(&rq->mtx);
-		erts_smp_mtx_lock(&xrq->mtx);
-		erts_smp_mtx_lock(&rq->mtx);
+		erts_mtx_unlock(&rq->mtx);
+		erts_mtx_lock(&xrq->mtx);
+		erts_mtx_lock(&rq->mtx);
 	    }
 	}
     }
 }
 
 ERTS_GLB_INLINE void
-erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
+erts_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
 {
     if (xrq != rq)
-	erts_smp_mtx_unlock(&xrq->mtx);
+	erts_mtx_unlock(&xrq->mtx);
 }
 
 ERTS_GLB_INLINE void
-erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
+erts_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
 {
     ASSERT(rq1 && rq2);
     if (rq1 == rq2)
-	erts_smp_mtx_lock(&rq1->mtx);
+	erts_mtx_lock(&rq1->mtx);
     else if (rq1 < rq2) {
-	erts_smp_mtx_lock(&rq1->mtx);
-	erts_smp_mtx_lock(&rq2->mtx);	
+	erts_mtx_lock(&rq1->mtx);
+	erts_mtx_lock(&rq2->mtx);
     }
     else {
-	erts_smp_mtx_lock(&rq2->mtx);
-	erts_smp_mtx_lock(&rq1->mtx);	
+	erts_mtx_lock(&rq2->mtx);
+	erts_mtx_lock(&rq1->mtx);
     }
 }
 
 ERTS_GLB_INLINE void
-erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
+erts_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
 {
     ASSERT(rq1 && rq2);
-    erts_smp_mtx_unlock(&rq1->mtx);
+    erts_mtx_unlock(&rq1->mtx);
     if (rq1 != rq2)
-	erts_smp_mtx_unlock(&rq2->mtx);
+	erts_mtx_unlock(&rq2->mtx);
 }
 
 ERTS_GLB_INLINE ErtsMessage *
@@ -2390,7 +2389,7 @@ erts_alloc_message_heap(Process *pp,
 			Eterm **hpp,
 			ErlOffHeap **ohpp)
 {
-    erts_aint32_t state = pp ? erts_smp_atomic32_read_nob(&pp->state) : 0;
+    erts_aint32_t state = pp ? erts_atomic32_read_nob(&pp->state) : 0;
     return erts_alloc_message_heap_state(pp, &state, plp, sz, hpp, ohpp);
 }
 
@@ -2404,7 +2403,7 @@ erts_shrink_message_heap(ErtsMessage **msgpp, Process *pp,
 	*msgpp = erts_shrink_message(*msgpp, used_hp - start_hp,
 				     brefs, brefs_size);
     else if (!(*msgpp)->data.attached) {
-	ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN
+	ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN
 			   & erts_proc_lc_my_proc_locks(pp));
 	HRelease(pp, end_hp, used_hp);
     }
@@ -2478,15 +2477,15 @@ Process *erts_pid2proc_nropt(Process *c_p,
 extern int erts_disable_proc_not_running_opt;
 
 #ifdef DEBUG
-#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) \
+#define ERTS_ASSERT_IS_NOT_EXITING(P) \
     do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0)
 #else
-#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)
+#define ERTS_ASSERT_IS_NOT_EXITING(P)
 #endif
 
 
 #define ERTS_PROC_IS_EXITING(P) \
-    (ERTS_PSFLG_EXITING & erts_smp_atomic32_read_acqb(&(P)->state))
+    (ERTS_PSFLG_EXITING & erts_atomic32_read_acqb(&(P)->state))
 
 
 /* Minimum NUMBER of processes for a small system to start */
@@ -2496,7 +2495,7 @@ extern int erts_disable_proc_not_running_opt;
 #define ERTS_MIN_PROCESSES		ERTS_NO_OF_PIX_LOCKS
 #endif
 
-void erts_smp_notify_inc_runq(ErtsRunQueue *runq);
+void erts_notify_inc_runq(ErtsRunQueue *runq);
 
 void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
 ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);
@@ -2508,9 +2507,9 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
 {
     erts_aint32_t flags;
     ERTS_THR_MEMORY_BARRIER;
-    flags = erts_smp_atomic32_read_nob(&ssi->flags);
+    flags = erts_atomic32_read_nob(&ssi->flags);
     if (flags & ERTS_SSI_FLG_SLEEPING) {
-	flags = erts_smp_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP);
+	flags = erts_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP);
 	erts_sched_finish_poke(ssi, flags);
     }
 }
@@ -2527,5 +2526,5 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
 
 
 void erts_halt(int code);
-extern erts_smp_atomic32_t erts_halt_progress;
+extern erts_atomic32_t erts_halt_progress;
 extern int erts_halt_code;
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index b826e6c5d3..5a2c262ff1 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -69,7 +69,7 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg)
     for (i = 0; i < max; i++) {
 	Process *p = erts_pix2proc(i);
 	if (p && p->i != ENULL) {
-	    erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
+	    erts_aint32_t state = erts_atomic32_read_acqb(&p->state);
 	    if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_GC)))
 		dump_process_info(to, to_arg, p);
        }
@@ -85,7 +85,7 @@ Uint erts_process_memory(Process *p, int incl_msg_inq) {
   size += sizeof(Process);
 
   if (incl_msg_inq)
-      ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
+      ERTS_MSGQ_MV_INQ2PRIVQ(p);
 
   erts_doforall_links(ERTS_P_LINKS(p), &erts_one_link_size, &size);
   erts_doforall_monitors(ERTS_P_MONITORS(p), &erts_one_mon_size, &size);
@@ -106,7 +106,7 @@ Uint erts_process_memory(Process *p, int incl_msg_inq) {
     size += p->arity * sizeof(p->arg_reg[0]);
   }
 
-  if (erts_smp_atomic_read_nob(&p->psd) != (erts_aint_t) NULL)
+  if (erts_atomic_read_nob(&p->psd) != (erts_aint_t) NULL)
     size += sizeof(ErtsPSD);
 
   scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p);
@@ -126,7 +126,7 @@ dump_process_info(fmtfn_t to, void *to_arg, Process *p)
     ErtsMessage* mp;
     int yreg = -1;
 
-    ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
+    ERTS_MSGQ_MV_INQ2PRIVQ(p);
 
     if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0 && p->msg.first) {
 	erts_print(to, to_arg, "=proc_messages:%T\n", p->common.id);
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index 0894c5233a..9d535646df 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -56,9 +56,9 @@
  *     Note that wait flags may be read without the pix lock, but
  *   it is important that wait flags only are modified when the pix
  *   lock is held.
- *     This implementation assumes that erts_smp_atomic_or_retold()
+ *     This implementation assumes that erts_atomic_or_retold()
  *   provides necessary memorybarriers for a lock operation, and that
- *   erts_smp_atomic_and_retold() provides necessary memorybarriers
+ *   erts_atomic_and_retold() provides necessary memorybarriers
  *   for an unlock operation.
  */
 
@@ -463,7 +463,7 @@ wait_for_locks(Process *p,
 }
 
 /*
- * erts_proc_lock_failed() is called when erts_smp_proc_lock()
+ * erts_proc_lock_failed() is called when erts_proc_lock()
  * wasn't able to lock all locks. We may need to transfer locks
  * to waiters and wait for our turn on locks.
  *
@@ -542,7 +542,7 @@ erts_proc_lock_failed(Process *p,
 }
 
 /*
- * erts_proc_unlock_failed() is called when erts_smp_proc_unlock()
+ * erts_proc_unlock_failed() is called when erts_proc_unlock()
  * wasn't able to unlock all locks. We may need to transfer locks
  * to waiters.
  */
@@ -708,7 +708,7 @@ proc_safelock(int is_managed,
 		refc1 = 1;
 		erts_proc_inc_refc(p1);
 	    }
-	    erts_smp_proc_unlock(p1, unlock_locks);
+	    erts_proc_unlock(p1, unlock_locks);
 	}
 	unlock_locks = unlock_mask & have_locks2;
 	if (unlock_locks) {
@@ -718,7 +718,7 @@ proc_safelock(int is_managed,
 		refc2 = 1;
 		erts_proc_inc_refc(p2);
 	    }
-	    erts_smp_proc_unlock(p2, unlock_locks);
+	    erts_proc_unlock(p2, unlock_locks);
 	}
     }
 
@@ -749,7 +749,7 @@ proc_safelock(int is_managed,
 	    if (need_locks2 & lock)
 		lock_no--;
 	    locks = need_locks1 & lock_mask;
-	    erts_smp_proc_lock(p1, locks);
+	    erts_proc_lock(p1, locks);
 	    have_locks1 |= locks;
 	    need_locks1 &= ~locks;
 	}
@@ -760,7 +760,7 @@ proc_safelock(int is_managed,
 		lock = (1 << ++lock_no);
 	    }
 	    locks = need_locks2 & lock_mask;
-	    erts_smp_proc_lock(p2, locks);
+	    erts_proc_lock(p2, locks);
 	    have_locks2 |= locks;
 	    need_locks2 &= ~locks;
 	}
@@ -897,7 +897,7 @@ erts_pid2proc_opt(Process *c_p,
 #endif /* ERTS_PROC_LOCK_OWN_IMPL */
 	    {
 		/* Try a quick trylock to grab all the locks we need. */
-		busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks);
+		busy = (int) erts_proc_raw_trylock__(proc, need_locks);
 
 #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_CHECK)
 		erts_proc_lc_trylock(proc, need_locks, !busy, __FILE__,__LINE__);
@@ -975,7 +975,7 @@ erts_pid2proc_opt(Process *c_p,
 	    : (proc
 	       != (Process *) erts_ptab_pix2intptr_nob(&erts_proc, pix)))) {
 
-	erts_smp_proc_unlock(proc, need_locks);
+	erts_proc_unlock(proc, need_locks);
 
 	if (flags & ERTS_P2P_FLG_INC_REFC)
 	    dec_refc_proc = proc;
@@ -1037,7 +1037,7 @@ erts_proc_lock_init(Process *p)
 #if ERTS_PROC_LOCK_OWN_IMPL
     /* We always start with all locks locked */
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
-    erts_smp_atomic32_init_nob(&p->lock.flags,
+    erts_atomic32_init_nob(&p->lock.flags,
 			       (erts_aint32_t) ERTS_PROC_LOCKS_ALL);
 #else
     p->lock.flags = ERTS_PROC_LOCKS_ALL;
@@ -1088,7 +1088,7 @@ erts_proc_lock_init(Process *p)
 #endif
 #ifdef ERTS_PROC_LOCK_DEBUG
     for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++)
-	erts_smp_atomic32_init_nob(&p->lock.locked[i], (erts_aint32_t) 1);
+	erts_atomic32_init_nob(&p->lock.locked[i], (erts_aint32_t) 1);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_proc_lock_init(p);
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 062dbbe2a6..9d5691d3c4 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -36,7 +36,7 @@
 #include "erl_lock_count.h"
 #endif
 
-#include "erl_smp.h"
+#include "erl_threads.h"
 
 #if defined(VALGRIND) || defined(ETHR_DISABLE_NATIVE_IMPLS)
 #  define ERTS_PROC_LOCK_OWN_IMPL 0
@@ -73,7 +73,7 @@ typedef erts_aint32_t ErtsProcLocks;
 typedef struct erts_proc_lock_t_ {
 #if ERTS_PROC_LOCK_OWN_IMPL
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
-    erts_smp_atomic32_t flags;
+    erts_atomic32_t flags;
 #else
     ErtsProcLocks flags;
 #endif
@@ -103,7 +103,7 @@ typedef struct erts_proc_lock_t_ {
 #  error "no implementation"
 #endif
 #ifdef ERTS_PROC_LOCK_DEBUG
-    erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1];
+    erts_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1];
 #endif
 } erts_proc_lock_t;
 
@@ -243,8 +243,8 @@ typedef struct erts_proc_lock_t_ {
 /* Lock counter implemetation */
 
 #ifdef ERTS_ENABLE_LOCK_POSITION
-#define erts_smp_proc_lock__(P,I,L) erts_smp_proc_lock_x__(P,I,L,__FILE__,__LINE__)
-#define erts_smp_proc_lock(P,L) erts_smp_proc_lock_x(P,L,__FILE__,__LINE__)
+#define erts_proc_lock__(P,I,L) erts_proc_lock_x__(P,I,L,__FILE__,__LINE__)
+#define erts_proc_lock(P,L) erts_proc_lock_x(P,L,__FILE__,__LINE__)
 #endif
 
 #if defined (ERTS_ENABLE_LOCK_COUNT)
@@ -422,9 +422,9 @@ void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res
 /* --- Process lock checking ----------------------------------------------- */
 
 #if defined(ERTS_ENABLE_LOCK_CHECK)
-#define ERTS_SMP_CHK_NO_PROC_LOCKS \
+#define ERTS_CHK_NO_PROC_LOCKS \
   erts_proc_lc_chk_no_proc_locks(__FILE__, __LINE__)
-#define ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(P) \
+#define ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(P) \
   erts_proc_lc_chk_only_proc_main((P))
 void erts_proc_lc_lock(Process *p, ErtsProcLocks locks,
 		       char *file, unsigned int line);
@@ -443,8 +443,8 @@ void erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks,
 			       char* file, unsigned int line);
 void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks);
 #else
-#define ERTS_SMP_CHK_NO_PROC_LOCKS
-#define ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(P)
+#define ERTS_CHK_NO_PROC_LOCKS
+#define ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(P)
 #endif
 
 #endif /* #ifndef ERTS_PROC_LOCK_LOCK_CHECK__ */
@@ -471,21 +471,21 @@ typedef struct {
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
 
 #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) \
-  ((ErtsProcLocks) erts_smp_atomic32_read_band_nob(&(L)->flags, \
+  ((ErtsProcLocks) erts_atomic32_read_band_nob(&(L)->flags, \
 						   (erts_aint32_t) (MSK)))
 #define ERTS_PROC_LOCK_FLGS_BOR_ACQB_(L, MSK) \
-  ((ErtsProcLocks) erts_smp_atomic32_read_bor_acqb(&(L)->flags, \
+  ((ErtsProcLocks) erts_atomic32_read_bor_acqb(&(L)->flags, \
 						   (erts_aint32_t) (MSK)))
 #define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \
-  ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_acqb(&(L)->flags, \
+  ((ErtsProcLocks) erts_atomic32_cmpxchg_acqb(&(L)->flags, \
 						  (erts_aint32_t) (NEW), \
 						  (erts_aint32_t) (EXPECTED)))
 #define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \
-  ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_relb(&(L)->flags, \
+  ((ErtsProcLocks) erts_atomic32_cmpxchg_relb(&(L)->flags, \
 						  (erts_aint32_t) (NEW), \
 						  (erts_aint32_t) (EXPECTED)))
 #define ERTS_PROC_LOCK_FLGS_READ_(L) \
-  ((ErtsProcLocks) erts_smp_atomic32_read_nob(&(L)->flags))
+  ((ErtsProcLocks) erts_atomic32_read_nob(&(L)->flags))
 
 #else /* no opt atomic ops */
 
@@ -556,22 +556,22 @@ ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *);
 ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *);
 ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *);
 
-ERTS_GLB_INLINE ErtsProcLocks erts_smp_proc_raw_trylock__(Process *p,
+ERTS_GLB_INLINE ErtsProcLocks erts_proc_raw_trylock__(Process *p,
 							  ErtsProcLocks locks);
 #ifdef ERTS_ENABLE_LOCK_POSITION
-ERTS_GLB_INLINE void erts_smp_proc_lock_x__(Process *,
+ERTS_GLB_INLINE void erts_proc_lock_x__(Process *,
 					    erts_pix_lock_t *,
 					    ErtsProcLocks,
 					    char *file, unsigned int line);
 #else
-ERTS_GLB_INLINE void erts_smp_proc_lock__(Process *,
+ERTS_GLB_INLINE void erts_proc_lock__(Process *,
 					  erts_pix_lock_t *,
 					  ErtsProcLocks);
 #endif
-ERTS_GLB_INLINE void erts_smp_proc_unlock__(Process *,
+ERTS_GLB_INLINE void erts_proc_unlock__(Process *,
 					    erts_pix_lock_t *,
 					    ErtsProcLocks);
-ERTS_GLB_INLINE int erts_smp_proc_trylock__(Process *,
+ERTS_GLB_INLINE int erts_proc_trylock__(Process *,
 					    erts_pix_lock_t *,
 					    ErtsProcLocks);
 
@@ -599,7 +599,7 @@ ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck)
 }
 
 /*
- * Helper function for erts_smp_proc_lock__ and erts_smp_proc_trylock__.
+ * Helper function for erts_proc_lock__ and erts_proc_trylock__.
  *
  * Attempts to grab all of 'locks' simultaneously.
  *
@@ -612,7 +612,7 @@ ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck)
  * Does not release the pix lock.
  */
 ERTS_GLB_INLINE ErtsProcLocks
-erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks)
+erts_proc_raw_trylock__(Process *p, ErtsProcLocks locks)
 {
 #if ERTS_PROC_LOCK_OWN_IMPL
     ErtsProcLocks expct_lflgs = 0;
@@ -681,12 +681,12 @@ busy_main:
 
 ERTS_GLB_INLINE void
 #ifdef ERTS_ENABLE_LOCK_POSITION
-erts_smp_proc_lock_x__(Process *p,
+erts_proc_lock_x__(Process *p,
 		     erts_pix_lock_t *pix_lck,
 		     ErtsProcLocks locks,
 		     char *file, unsigned int line)
 #else
-erts_smp_proc_lock__(Process *p,
+erts_proc_lock__(Process *p,
 		     erts_pix_lock_t *pix_lck,
 		     ErtsProcLocks locks)
 #endif
@@ -708,7 +708,7 @@ erts_smp_proc_lock__(Process *p,
     erts_proc_lc_lock(p, locks, file, line);
 #endif
 
-    old_lflgs = erts_smp_proc_raw_trylock__(p, locks);
+    old_lflgs = erts_proc_raw_trylock__(p, locks);
 
     if (old_lflgs != 0) {
 	/*
@@ -760,7 +760,7 @@ erts_smp_proc_lock__(Process *p,
 }
 
 ERTS_GLB_INLINE void
-erts_smp_proc_unlock__(Process *p,
+erts_proc_unlock__(Process *p,
 		       erts_pix_lock_t *pix_lck,
 		       ErtsProcLocks locks)
 {
@@ -853,7 +853,7 @@ erts_smp_proc_unlock__(Process *p,
 }
 
 ERTS_GLB_INLINE int
-erts_smp_proc_trylock__(Process *p,
+erts_proc_trylock__(Process *p,
 			erts_pix_lock_t *pix_lck,
 			ErtsProcLocks locks)
 {
@@ -874,7 +874,7 @@ erts_smp_proc_trylock__(Process *p,
 	erts_pix_lock(pix_lck);
 #endif
 
-	if (erts_smp_proc_raw_trylock__(p, locks) != 0) {
+	if (erts_proc_raw_trylock__(p, locks) != 0) {
 	    /* Didn't get all locks... */
 	    res = EBUSY;
 
@@ -911,7 +911,7 @@ erts_smp_proc_trylock__(Process *p,
     return res;
 
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
-    if (erts_smp_proc_raw_trylock__(p, locks) != 0)
+    if (erts_proc_raw_trylock__(p, locks) != 0)
 	return EBUSY;
     else {
 #ifdef ERTS_PROC_LOCK_DEBUG
@@ -932,11 +932,11 @@ erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked)
 	if (locks & lock) {
 	    erts_aint32_t lock_count;
 	    if (locked) {
-		lock_count = erts_smp_atomic32_inc_read_nob(&p->lock.locked[i]);
+		lock_count = erts_atomic32_inc_read_nob(&p->lock.locked[i]);
 		ERTS_LC_ASSERT(lock_count == 1);
 	    }
 	    else {
-		lock_count = erts_smp_atomic32_dec_read_nob(&p->lock.locked[i]);
+		lock_count = erts_atomic32_dec_read_nob(&p->lock.locked[i]);
 		ERTS_LC_ASSERT(lock_count == 0);
 	    }
 	}
@@ -948,12 +948,12 @@ erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked)
 
 
 #ifdef ERTS_ENABLE_LOCK_POSITION
-ERTS_GLB_INLINE void erts_smp_proc_lock_x(Process *, ErtsProcLocks, char *file, unsigned int line);
+ERTS_GLB_INLINE void erts_proc_lock_x(Process *, ErtsProcLocks, char *file, unsigned int line);
 #else
-ERTS_GLB_INLINE void erts_smp_proc_lock(Process *, ErtsProcLocks);
+ERTS_GLB_INLINE void erts_proc_lock(Process *, ErtsProcLocks);
 #endif
-ERTS_GLB_INLINE void erts_smp_proc_unlock(Process *, ErtsProcLocks);
-ERTS_GLB_INLINE int erts_smp_proc_trylock(Process *, ErtsProcLocks);
+ERTS_GLB_INLINE void erts_proc_unlock(Process *, ErtsProcLocks);
+ERTS_GLB_INLINE int erts_proc_trylock(Process *, ErtsProcLocks);
 
 ERTS_GLB_INLINE void erts_proc_inc_refc(Process *);
 ERTS_GLB_INLINE void erts_proc_dec_refc(Process *);
@@ -964,13 +964,13 @@ ERTS_GLB_INLINE Sint erts_proc_read_refc(Process *);
 
 ERTS_GLB_INLINE void
 #ifdef ERTS_ENABLE_LOCK_POSITION
-erts_smp_proc_lock_x(Process *p, ErtsProcLocks locks, char *file, unsigned int line)
+erts_proc_lock_x(Process *p, ErtsProcLocks locks, char *file, unsigned int line)
 #else
-erts_smp_proc_lock(Process *p, ErtsProcLocks locks)
+erts_proc_lock(Process *p, ErtsProcLocks locks)
 #endif 
 {
 #if defined(ERTS_ENABLE_LOCK_POSITION)
-    erts_smp_proc_lock_x__(p,
+    erts_proc_lock_x__(p,
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
 			 NULL,
 #else
@@ -978,7 +978,7 @@ erts_smp_proc_lock(Process *p, ErtsProcLocks locks)
 #endif /*ERTS_PROC_LOCK_ATOMIC_IMPL*/
 			 locks, file, line);
 #else
-    erts_smp_proc_lock__(p,
+    erts_proc_lock__(p,
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
 			 NULL,
 #else
@@ -989,9 +989,9 @@ erts_smp_proc_lock(Process *p, ErtsProcLocks locks)
 }
 
 ERTS_GLB_INLINE void
-erts_smp_proc_unlock(Process *p, ErtsProcLocks locks)
+erts_proc_unlock(Process *p, ErtsProcLocks locks)
 {
-    erts_smp_proc_unlock__(p,
+    erts_proc_unlock__(p,
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
 			   NULL,
 #else
@@ -1001,9 +1001,9 @@ erts_smp_proc_unlock(Process *p, ErtsProcLocks locks)
 }
 
 ERTS_GLB_INLINE int
-erts_smp_proc_trylock(Process *p, ErtsProcLocks locks)
+erts_proc_trylock(Process *p, ErtsProcLocks locks)
 {
-    return erts_smp_proc_trylock__(p,
+    return erts_proc_trylock__(p,
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
 				   NULL,
 #else
@@ -1014,14 +1014,14 @@ erts_smp_proc_trylock(Process *p, ErtsProcLocks locks)
 
 ERTS_GLB_INLINE void erts_proc_inc_refc(Process *p)
 {
-    ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
+    ASSERT(!(erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
     erts_ptab_atmc_inc_refc(&p->common);
 }
 
 ERTS_GLB_INLINE void erts_proc_dec_refc(Process *p)
 {
     Sint referred;
-    ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
+    ASSERT(!(erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
     referred = erts_ptab_atmc_dec_test_refc(&p->common);
     if (!referred) {
 	ASSERT(ERTS_PROC_IS_EXITING(p));
@@ -1032,7 +1032,7 @@ ERTS_GLB_INLINE void erts_proc_dec_refc(Process *p)
 ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc)
 {
     Sint referred;
-    ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
+    ASSERT(!(erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
     referred = erts_ptab_atmc_add_test_refc(&p->common, add_refc);
     if (!referred) {
 	ASSERT(ERTS_PROC_IS_EXITING(p));
@@ -1042,7 +1042,7 @@ ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc)
 
 ERTS_GLB_INLINE Sint erts_proc_read_refc(Process *p)
 {
-    ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
+    ASSERT(!(erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
     return erts_ptab_atmc_read_refc(&p->common);
 }
 
@@ -1103,7 +1103,7 @@ ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid)
 {
     Process *proc;
 
-    ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying());
+    ERTS_LC_ASSERT(erts_thr_progress_lc_is_delaying());
 
     if (is_not_internal_pid(pid))
 	return NULL;
diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index b3bcb3af3f..38c095fb4a 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -284,31 +284,31 @@ struct ErtsPTabListBifData_ {
 static ERTS_INLINE void
 last_data_init_nob(ErtsPTab *ptab, Uint64 val)
 {
-    erts_smp_atomic64_init_nob(&ptab->vola.tile.last_data, (erts_aint64_t) val);
+    erts_atomic64_init_nob(&ptab->vola.tile.last_data, (erts_aint64_t) val);
 }
 
 static ERTS_INLINE void
 last_data_set_relb(ErtsPTab *ptab, Uint64 val)
 {
-    erts_smp_atomic64_set_relb(&ptab->vola.tile.last_data, (erts_aint64_t) val);
+    erts_atomic64_set_relb(&ptab->vola.tile.last_data, (erts_aint64_t) val);
 }
 
 static ERTS_INLINE Uint64
 last_data_read_nob(ErtsPTab *ptab)
 {
-    return (Uint64) erts_smp_atomic64_read_nob(&ptab->vola.tile.last_data);
+    return (Uint64) erts_atomic64_read_nob(&ptab->vola.tile.last_data);
 }
 
 static ERTS_INLINE Uint64
 last_data_read_acqb(ErtsPTab *ptab)
 {
-    return (Uint64) erts_smp_atomic64_read_acqb(&ptab->vola.tile.last_data);
+    return (Uint64) erts_atomic64_read_acqb(&ptab->vola.tile.last_data);
 }
 
 static ERTS_INLINE Uint64
 last_data_cmpxchg_relb(ErtsPTab *ptab, Uint64 new, Uint64 exp)
 {
-    return (Uint64) erts_smp_atomic64_cmpxchg_relb(&ptab->vola.tile.last_data,
+    return (Uint64) erts_atomic64_cmpxchg_relb(&ptab->vola.tile.last_data,
 						   (erts_aint64_t) new,
 						   (erts_aint64_t) exp);
 }
@@ -346,9 +346,9 @@ ix_to_free_id_data_ix(ErtsPTab *ptab, Uint32 ix)
 UWord
 erts_ptab_mem_size(ErtsPTab *ptab)
 {
-    UWord size = ptab->r.o.max*sizeof(erts_smp_atomic_t);
+    UWord size = ptab->r.o.max*sizeof(erts_atomic_t);
     if (ptab->r.o.free_id_data)
-	size += ptab->r.o.max*sizeof(erts_smp_atomic32_t);
+	size += ptab->r.o.max*sizeof(erts_atomic32_t);
     return size;
 }
 
@@ -367,14 +367,14 @@ erts_ptab_init_table(ErtsPTab *ptab,
     size_t tab_sz, alloc_sz;
     Uint32 bits, cl, cli, ix, ix_per_cache_line, tab_cache_lines; 
     char *tab_end;
-    erts_smp_atomic_t *tab_entry;
-    erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
-    rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
-    rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+    erts_atomic_t *tab_entry;
+    erts_rwmtx_opt_t rwmtx_opts = ERTS_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opts.type = ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
+    rwmtx_opts.lived = ERTS_RWMTX_LONG_LIVED;
 
-    erts_smp_rwmtx_init_opt(&ptab->list.data.rwmtx, &rwmtx_opts, name, NIL,
+    erts_rwmtx_init_opt(&ptab->list.data.rwmtx, &rwmtx_opts, name, NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
-    erts_smp_atomic32_init_nob(&ptab->vola.tile.count, 0);
+    erts_atomic32_init_nob(&ptab->vola.tile.count, 0);
     last_data_init_nob(ptab, ~((Uint64) 0));
 
     /* A size that is a power of 2 is to prefer performance wise */
@@ -388,20 +388,20 @@ erts_ptab_init_table(ErtsPTab *ptab,
     ptab->r.o.element_size = element_size;
     ptab->r.o.max = size;
 
-    tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic_t));
+    tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_atomic_t));
     alloc_sz = tab_sz;
     if (!legacy)
-	alloc_sz += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic32_t));
+	alloc_sz += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_atomic32_t));
     ptab->r.o.tab = erts_alloc_permanent_cache_aligned(atype, alloc_sz);
     tab_end = ((char *) ptab->r.o.tab) + tab_sz;
     tab_entry = ptab->r.o.tab;
     while (tab_end > ((char *) tab_entry)) {
-	erts_smp_atomic_init_nob(tab_entry, ERTS_AINT_NULL);
+	erts_atomic_init_nob(tab_entry, ERTS_AINT_NULL);
 	tab_entry++;
     }
 
     tab_cache_lines = tab_sz/ERTS_CACHE_LINE_SIZE;
-    ix_per_cache_line = (ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic_t));
+    ix_per_cache_line = (ERTS_CACHE_LINE_SIZE/sizeof(erts_atomic_t));
     ASSERT((ptab->r.o.max & (ptab->r.o.max - 1)) == 0); /* power of 2 */
     ASSERT((ix_per_cache_line & (ix_per_cache_line - 1)) == 0); /* power of 2 */
     ASSERT((tab_cache_lines & (tab_cache_lines - 1)) == 0); /* power of 2 */
@@ -429,11 +429,11 @@ erts_ptab_init_table(ErtsPTab *ptab,
     }
     else {
 
-	tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic32_t));
-	ptab->r.o.free_id_data = (erts_smp_atomic32_t *) tab_end;
+	tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_atomic32_t));
+	ptab->r.o.free_id_data = (erts_atomic32_t *) tab_end;
 
 	tab_cache_lines = tab_sz/ERTS_CACHE_LINE_SIZE;
-	ix_per_cache_line = (ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic32_t));
+	ix_per_cache_line = (ERTS_CACHE_LINE_SIZE/sizeof(erts_atomic32_t));
 
 	ptab->r.o.dix_cl_mask = tab_cache_lines-1;
 	ptab->r.o.dix_cl_shift = erts_fit_in_bits_int32(ix_per_cache_line-1);
@@ -448,19 +448,19 @@ erts_ptab_init_table(ErtsPTab *ptab,
 	ix = 0;
 	for (cl = 0; cl < tab_cache_lines; cl++) {
 	    for (cli = 0; cli < ix_per_cache_line; cli++) {
-		erts_smp_atomic32_init_nob(&ptab->r.o.free_id_data[ix],
+		erts_atomic32_init_nob(&ptab->r.o.free_id_data[ix],
 					   cli*tab_cache_lines+cl);
-		ASSERT(erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]) != ptab->r.o.invalid_data);
+		ASSERT(erts_atomic32_read_nob(&ptab->r.o.free_id_data[ix]) != ptab->r.o.invalid_data);
 		ix++;
 	    }
 	}
 
-	erts_smp_atomic32_init_nob(&ptab->vola.tile.aid_ix, -1);
-	erts_smp_atomic32_init_nob(&ptab->vola.tile.fid_ix, -1);
+	erts_atomic32_init_nob(&ptab->vola.tile.aid_ix, -1);
+	erts_atomic32_init_nob(&ptab->vola.tile.fid_ix, -1);
 
     }
 
-    erts_smp_interval_init(&ptab->list.data.interval);
+    erts_interval_init(&ptab->list.data.interval);
     ptab->list.data.deleted.start = NULL;
     ptab->list.data.deleted.end = NULL;
     ptab->list.data.chunks = (((ptab->r.o.max - 1)
@@ -480,9 +480,9 @@ erts_ptab_init_table(ErtsPTab *ptab,
 	 * have ERTS_PTAB_MAX_SIZE-1 valid elements in the table while
 	 * still having a table size of the power of 2.
 	 */
-	erts_smp_atomic32_inc_nob(&ptab->vola.tile.count);
+	erts_atomic32_inc_nob(&ptab->vola.tile.count);
 	pix = erts_ptab_data2pix(ptab, ptab->r.o.invalid_data);
-	erts_smp_atomic_set_relb(&ptab->r.o.tab[pix],
+	erts_atomic_set_relb(&ptab->r.o.tab[pix],
 				 (erts_aint_t) ptab->r.o.invalid_element);
     }
 
@@ -506,12 +506,12 @@ erts_ptab_new_element(ErtsPTab *ptab,
 
     erts_ptab_rlock(ptab);
 
-    count = erts_smp_atomic32_inc_read_acqb(&ptab->vola.tile.count);
+    count = erts_atomic32_inc_read_acqb(&ptab->vola.tile.count);
     if (count > ptab->r.o.max) {
 	while (1) {
 	    erts_aint32_t act_count;
 
-	    act_count = erts_smp_atomic32_cmpxchg_relb(&ptab->vola.tile.count,
+	    act_count = erts_atomic32_cmpxchg_relb(&ptab->vola.tile.count,
 						       count-1,
 						       count);
 	    if (act_count == count) {
@@ -525,14 +525,14 @@ erts_ptab_new_element(ErtsPTab *ptab,
     }
 
     ptab_el->u.alive.started_interval
-	= erts_smp_current_interval_nob(erts_ptab_interval(ptab));
+	= erts_current_interval_nob(erts_ptab_interval(ptab));
 
     if (ptab->r.o.free_id_data) {
 	do {
-	    ix = (Uint32) erts_smp_atomic32_inc_read_acqb(&ptab->vola.tile.aid_ix);
+	    ix = (Uint32) erts_atomic32_inc_read_acqb(&ptab->vola.tile.aid_ix);
 	    ix = ix_to_free_id_data_ix(ptab, ix);
 
-	    data = erts_smp_atomic32_xchg_nob(&ptab->r.o.free_id_data[ix],
+	    data = erts_atomic32_xchg_nob(&ptab->r.o.free_id_data[ix],
 					      (erts_aint32_t)ptab->r.o.invalid_data);
 	}while ((Eterm)data == ptab->r.o.invalid_data);
 
@@ -546,10 +546,10 @@ erts_ptab_new_element(ErtsPTab *ptab,
 	pix = erts_ptab_data2pix(ptab, (Eterm) data);
 
 #ifdef DEBUG
-	ASSERT(ERTS_AINT_NULL == erts_smp_atomic_xchg_relb(&ptab->r.o.tab[pix],
+	ASSERT(ERTS_AINT_NULL == erts_atomic_xchg_relb(&ptab->r.o.tab[pix],
 							   (erts_aint_t) ptab_el));
 #else
-	erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el);
+	erts_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el);
 #endif
 
 	erts_ptab_runlock(ptab);
@@ -563,7 +563,7 @@ erts_ptab_new_element(ErtsPTab *ptab,
     restart:
 
 	ptab_el->u.alive.started_interval
-	    = erts_smp_current_interval_nob(erts_ptab_interval(ptab));
+	    = erts_current_interval_nob(erts_ptab_interval(ptab));
 
 	ld = last_data_read_acqb(ptab);
 
@@ -571,10 +571,10 @@ erts_ptab_new_element(ErtsPTab *ptab,
 	while (1) {
 	    ld++;
 	    pix = erts_ptab_data2pix(ptab, ERTS_PTAB_LastData2EtermData(ld));
-	    if (erts_smp_atomic_read_nob(&ptab->r.o.tab[pix])
+	    if (erts_atomic_read_nob(&ptab->r.o.tab[pix])
 		== ERTS_AINT_NULL) {
 		erts_aint_t val;
-		val = erts_smp_atomic_cmpxchg_relb(&ptab->r.o.tab[pix],
+		val = erts_atomic_cmpxchg_relb(&ptab->r.o.tab[pix],
 						   invalid,	
 						   ERTS_AINT_NULL);
 
@@ -621,10 +621,10 @@ erts_ptab_new_element(ErtsPTab *ptab,
 
 	/* Move into slot reserved */
 #ifdef DEBUG
-	ASSERT(invalid == erts_smp_atomic_xchg_relb(&ptab->r.o.tab[pix],
+	ASSERT(invalid == erts_atomic_xchg_relb(&ptab->r.o.tab[pix],
 						(erts_aint_t) ptab_el));
 #else
-	erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el);
+	erts_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el);
 #endif
 
 	if (rlocked)
@@ -644,7 +644,7 @@ save_deleted_element(ErtsPTab *ptab, ErtsPTabElementCommon *ptab_el)
 					       sizeof(ErtsPTabDeletedElement));
     ERTS_PTAB_LIST_ASSERT(ptab->list.data.deleted.start
 			  && ptab->list.data.deleted.end);
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_ptab_is_rwlocked(ptab));
+    ERTS_LC_ASSERT(erts_lc_ptab_is_rwlocked(ptab));
 
     ERTS_PTAB_LIST_DBG_CHK_DEL_LIST(ptab);
 
@@ -654,7 +654,7 @@ save_deleted_element(ErtsPTab *ptab, ErtsPTabElementCommon *ptab_el)
     ptdep->u.element.id = ptab_el->id;
     ptdep->u.element.inserted = ptab_el->u.alive.started_interval;
     ptdep->u.element.deleted =
-	erts_smp_current_interval_nob(erts_ptab_interval(ptab));
+	erts_current_interval_nob(erts_ptab_interval(ptab));
 
     ptab->list.data.deleted.end->next = ptdep;
     ptab->list.data.deleted.end = ptdep;
@@ -678,7 +678,7 @@ erts_ptab_delete_element(ErtsPTab *ptab,
     pix = erts_ptab_id2pix(ptab, ptab_el->id);
 
     /* *Need* to be an managed thread */
-    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
+    ERTS_LC_ASSERT(erts_thr_progress_is_managed_thread());
 
     erts_ptab_rlock(ptab);
     maybe_save = ptab->list.data.deleted.end != NULL;
@@ -687,7 +687,7 @@ erts_ptab_delete_element(ErtsPTab *ptab,
 	erts_ptab_rwlock(ptab);
     }
 
-    erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], ERTS_AINT_NULL);
+    erts_atomic_set_relb(&ptab->r.o.tab[pix], ERTS_AINT_NULL);
 
     if (ptab->r.o.free_id_data) {
 	Uint32 prev_data;
@@ -703,17 +703,17 @@ erts_ptab_delete_element(ErtsPTab *ptab,
 	ASSERT(pix == erts_ptab_data2pix(ptab, data));
 
 	do { 
-	    ix = (Uint32) erts_smp_atomic32_inc_read_relb(&ptab->vola.tile.fid_ix);
+	    ix = (Uint32) erts_atomic32_inc_read_relb(&ptab->vola.tile.fid_ix);
 	    ix = ix_to_free_id_data_ix(ptab, ix);
     
-	    prev_data = erts_smp_atomic32_cmpxchg_nob(&ptab->r.o.free_id_data[ix],
+	    prev_data = erts_atomic32_cmpxchg_nob(&ptab->r.o.free_id_data[ix],
 						      data,
 						      ptab->r.o.invalid_data);
 	}while ((Eterm)prev_data != ptab->r.o.invalid_data);
     }
 
-    ASSERT(erts_smp_atomic32_read_nob(&ptab->vola.tile.count) > 0);
-    erts_smp_atomic32_dec_relb(&ptab->vola.tile.count);
+    ASSERT(erts_atomic32_read_nob(&ptab->vola.tile.count) > 0);
+    erts_atomic32_dec_relb(&ptab->vola.tile.count);
 
     if (!maybe_save)
 	erts_ptab_runlock(ptab);
@@ -927,7 +927,7 @@ ptab_list_bif_engine(Process *c_p, Eterm *res_accp, Binary *mbp)
 				 sizeof(ErtsPTabDeletedElement));
 		ptlbdp->bif_invocation->ix = -1;
 		ptlbdp->bif_invocation->u.bif_invocation.interval
-		    = erts_smp_step_interval_nob(erts_ptab_interval(ptab));
+		    = erts_step_interval_nob(erts_ptab_interval(ptab));
 		ERTS_PTAB_LIST_DBG_CHK_DEL_LIST(ptab);
 
 		ptlbdp->bif_invocation->next = NULL;
@@ -968,12 +968,12 @@ ptab_list_bif_engine(Process *c_p, Eterm *res_accp, Binary *mbp)
 		locked = 1;
 	    }
 
-	    ERTS_SMP_LC_ASSERT(erts_smp_lc_ptab_is_rwlocked(ptab));
+	    ERTS_LC_ASSERT(erts_lc_ptab_is_rwlocked(ptab));
 	    ERTS_PTAB_LIST_DBG_TRACE(p->common.id, insp_table);
 
 	    if (cix != 0)
 		ptlbdp->chunk[cix].interval
-		    = erts_smp_step_interval_nob(erts_ptab_interval(ptab));
+		    = erts_step_interval_nob(erts_ptab_interval(ptab));
 	    else if (ptlbdp->bif_invocation)
 		ptlbdp->chunk[0].interval = *invocation_interval_p;
 	    /* else: interval is irrelevant */
@@ -1331,18 +1331,18 @@ static void assert_ptab_consistency(ErtsPTab *ptab)
 	int null_slots = 0;
 	
 	for (ix=0; ix < ptab->r.o.max; ix++) {
-	    if (erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]) != ptab->r.o.invalid_data) {
+	    if (erts_atomic32_read_nob(&ptab->r.o.free_id_data[ix]) != ptab->r.o.invalid_data) {
 		++free_pids;
-		data = erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]);
+		data = erts_atomic32_read_nob(&ptab->r.o.free_id_data[ix]);
 		pix = erts_ptab_data2pix(ptab, (Eterm) data);
 		ASSERT(erts_ptab_pix2intptr_nob(ptab, pix) == ERTS_AINT_NULL);
 	    }
-	    if (erts_smp_atomic_read_nob(&ptab->r.o.tab[ix]) == ERTS_AINT_NULL) {
+	    if (erts_atomic_read_nob(&ptab->r.o.tab[ix]) == ERTS_AINT_NULL) {
 		++null_slots;
 	    }
 	}	
 	ASSERT(free_pids == null_slots);
-	ASSERT(free_pids == ptab->r.o.max - erts_smp_atomic32_read_nob(&ptab->vola.tile.count));
+	ASSERT(free_pids == ptab->r.o.max - erts_atomic32_read_nob(&ptab->vola.tile.count));
     }
 #endif
 }
@@ -1366,7 +1366,7 @@ erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next)
 	    Uint32 i, max_ix, num, stop_id_ix;
 	    max_ix = ptab->r.o.max - 1;
 	    num = next;
-	    id_ix = (Uint32) erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix);
+	    id_ix = (Uint32) erts_atomic32_read_nob(&ptab->vola.tile.aid_ix);
 
 	    for (i=0; i <= max_ix; ++i) {
 		Uint32 pix;
@@ -1380,26 +1380,26 @@ erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next)
 		if (ERTS_AINT_NULL == erts_ptab_pix2intptr_nob(ptab, pix)) {
 		    ++id_ix;
 		    dix = ix_to_free_id_data_ix(ptab, id_ix);
-		    erts_smp_atomic32_set_nob(&ptab->r.o.free_id_data[dix], num);
+		    erts_atomic32_set_nob(&ptab->r.o.free_id_data[dix], num);
 		    ASSERT(pix == erts_ptab_data2pix(ptab, num));
 		}
 	    }
-	    erts_smp_atomic32_set_nob(&ptab->vola.tile.fid_ix, id_ix);
+	    erts_atomic32_set_nob(&ptab->vola.tile.fid_ix, id_ix);
 
 	    /* Write invalid_data in rest of free_id_data[]: */
-	    stop_id_ix = (1 + erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix)) & max_ix;
+	    stop_id_ix = (1 + erts_atomic32_read_nob(&ptab->vola.tile.aid_ix)) & max_ix;
 	    while (1) {
 		id_ix = (id_ix+1) & max_ix;
 		if (id_ix == stop_id_ix)
 		    break;
 		dix = ix_to_free_id_data_ix(ptab, id_ix);
-		erts_smp_atomic32_set_nob(&ptab->r.o.free_id_data[dix],
+		erts_atomic32_set_nob(&ptab->r.o.free_id_data[dix],
 					  ptab->r.o.invalid_data);
 	    }
 	}
-	id_ix = (Uint32) erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix) + 1;
+	id_ix = (Uint32) erts_atomic32_read_nob(&ptab->vola.tile.aid_ix) + 1;
 	dix = ix_to_free_id_data_ix(ptab, id_ix);
-	res = (Sint) erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[dix]);
+	res = (Sint) erts_atomic32_read_nob(&ptab->r.o.free_id_data[dix]);
     }
     else {
 	/* Deprecated legacy algorithm... */
@@ -1616,11 +1616,11 @@ debug_ptab_list_verify_all_pids(ErtsPTabListBifData *ptlbdp)
 static void
 debug_ptab_list_check_del_list(ErtsPTab *ptab)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_ptab_is_rwlocked(ptab));
+    ERTS_LC_ASSERT(erts_lc_ptab_is_rwlocked(ptab));
     if (!ptab->list.data.deleted.start)
 	ERTS_PTAB_LIST_ASSERT(!ptab->list.data.deleted.end);
     else {
-	Uint64 curr_interval = erts_smp_current_interval_nob(erts_ptab_interval(ptab));
+	Uint64 curr_interval = erts_current_interval_nob(erts_ptab_interval(ptab));
 	Uint64 *prev_x_interval_p = NULL;
 	ErtsPTabDeletedElement *ptdep;
 
diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h
index 285d325cc4..4858cc8ab8 100644
--- a/erts/emulator/beam/erl_ptab.h
+++ b/erts/emulator/beam/erl_ptab.h
@@ -60,7 +60,7 @@ typedef struct {
     } refc;
     ErtsTracer tracer;
     Uint trace_flags;
-    erts_smp_atomic_t timer;
+    erts_atomic_t timer;
     union {
 	/* --- While being alive --- */
 	struct {
@@ -78,7 +78,7 @@ typedef struct {
 typedef struct ErtsPTabDeletedElement_ ErtsPTabDeletedElement;
 
 typedef struct {
-    erts_smp_rwmtx_t rwmtx;
+    erts_rwmtx_t rwmtx;
     erts_interval_t interval;
     struct {
 	ErtsPTabDeletedElement *start;
@@ -88,15 +88,15 @@ typedef struct {
 } ErtsPTabListData;
 
 typedef struct {
-    erts_smp_atomic64_t last_data;
-    erts_smp_atomic32_t count;
-    erts_smp_atomic32_t aid_ix;
-    erts_smp_atomic32_t fid_ix;
+    erts_atomic64_t last_data;
+    erts_atomic32_t count;
+    erts_atomic32_t aid_ix;
+    erts_atomic32_t fid_ix;
 } ErtsPTabVolatileData;
 
 typedef struct {
-    erts_smp_atomic_t *tab;
-    erts_smp_atomic32_t *free_id_data;
+    erts_atomic_t *tab;
+    erts_atomic32_t *free_id_data;
     Uint32 max;
     Uint32 pix_mask;
     Uint32 pix_cl_mask;
@@ -223,8 +223,8 @@ ERTS_GLB_INLINE void erts_ptab_runlock(ErtsPTab *ptab);
 ERTS_GLB_INLINE void erts_ptab_rwlock(ErtsPTab *ptab);
 ERTS_GLB_INLINE int erts_ptab_tryrwlock(ErtsPTab *ptab);
 ERTS_GLB_INLINE void erts_ptab_rwunlock(ErtsPTab *ptab);
-ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rlocked(ErtsPTab *ptab);
-ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rwlocked(ErtsPTab *ptab);
+ERTS_GLB_INLINE int erts_lc_ptab_is_rlocked(ErtsPTab *ptab);
+ERTS_GLB_INLINE int erts_lc_ptab_is_rwlocked(ErtsPTab *ptab);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
@@ -245,7 +245,7 @@ ERTS_GLB_INLINE int
 erts_ptab_count(ErtsPTab *ptab)
 {
     int max = ptab->r.o.max;
-    erts_aint32_t res = erts_smp_atomic32_read_nob(&ptab->vola.tile.count);
+    erts_aint32_t res = erts_atomic32_read_nob(&ptab->vola.tile.count);
     if (max == ERTS_PTAB_MAX_SIZE) {
 	max--;
 	res--;
@@ -352,25 +352,25 @@ erts_ptab_id2data(ErtsPTab *ptab, Eterm id)
 ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_nob(ErtsPTab *ptab, int ix)
 {
     ASSERT(0 <= ix && ix < ptab->r.o.max);
-    return erts_smp_atomic_read_nob(&ptab->r.o.tab[ix]);
+    return erts_atomic_read_nob(&ptab->r.o.tab[ix]);
 }
 
 ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_ddrb(ErtsPTab *ptab, int ix)
 {
     ASSERT(0 <= ix && ix < ptab->r.o.max);
-    return erts_smp_atomic_read_ddrb(&ptab->r.o.tab[ix]);
+    return erts_atomic_read_ddrb(&ptab->r.o.tab[ix]);
 }
 
 ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_rb(ErtsPTab *ptab, int ix)
 {
     ASSERT(0 <= ix && ix < ptab->r.o.max);
-    return erts_smp_atomic_read_rb(&ptab->r.o.tab[ix]);
+    return erts_atomic_read_rb(&ptab->r.o.tab[ix]);
 }
 
 ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_acqb(ErtsPTab *ptab, int ix)
 {
     ASSERT(0 <= ix && ix < ptab->r.o.max);
-    return erts_smp_atomic_read_acqb(&ptab->r.o.tab[ix]);
+    return erts_atomic_read_acqb(&ptab->r.o.tab[ix]);
 }
 
 ERTS_GLB_INLINE void erts_ptab_atmc_inc_refc(ErtsPTabElementCommon *ptab_el)
@@ -386,7 +386,7 @@ ERTS_GLB_INLINE void erts_ptab_atmc_inc_refc(ErtsPTabElementCommon *ptab_el)
 ERTS_GLB_INLINE Sint erts_ptab_atmc_dec_test_refc(ErtsPTabElementCommon *ptab_el)
 {
     erts_aint_t refc = erts_atomic_dec_read_relb(&ptab_el->refc.atmc);
-    ERTS_SMP_LC_ASSERT(refc >= 0);
+    ERTS_LC_ASSERT(refc >= 0);
     if (refc == 0)
 	ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
     return (Sint) refc;
@@ -397,7 +397,7 @@ ERTS_GLB_INLINE Sint erts_ptab_atmc_add_test_refc(ErtsPTabElementCommon *ptab_el
 {
     erts_aint_t refc = erts_atomic_add_read_mb(&ptab_el->refc.atmc,
 					       (erts_aint_t) add_refc);
-    ERTS_SMP_LC_ASSERT(refc >= 0);
+    ERTS_LC_ASSERT(refc >= 0);
     return (Sint) refc;
 }
 
@@ -415,7 +415,7 @@ ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el)
 ERTS_GLB_INLINE Sint erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el)
 {
     Sint refc = --ptab_el->refc.sint;
-    ERTS_SMP_LC_ASSERT(refc >= 0);
+    ERTS_LC_ASSERT(refc >= 0);
     return refc;
 }
 
@@ -423,7 +423,7 @@ ERTS_GLB_INLINE Sint erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el,
 					     Sint add_refc)
 {
     ptab_el->refc.sint += add_refc;
-    ERTS_SMP_LC_ASSERT(ptab_el->refc.sint >= 0);
+    ERTS_LC_ASSERT(ptab_el->refc.sint >= 0);
     return (Sint) ptab_el->refc.sint;
 }
 
@@ -434,42 +434,42 @@ ERTS_GLB_INLINE Sint erts_ptab_read_refc(ErtsPTabElementCommon *ptab_el)
 
 ERTS_GLB_INLINE void erts_ptab_rlock(ErtsPTab *ptab)
 {
-    erts_smp_rwmtx_rlock(&ptab->list.data.rwmtx);
+    erts_rwmtx_rlock(&ptab->list.data.rwmtx);
 }
 
 ERTS_GLB_INLINE int erts_ptab_tryrlock(ErtsPTab *ptab)
 {
-    return erts_smp_rwmtx_tryrlock(&ptab->list.data.rwmtx);
+    return erts_rwmtx_tryrlock(&ptab->list.data.rwmtx);
 }
 
 ERTS_GLB_INLINE void erts_ptab_runlock(ErtsPTab *ptab)
 {
-    erts_smp_rwmtx_runlock(&ptab->list.data.rwmtx);
+    erts_rwmtx_runlock(&ptab->list.data.rwmtx);
 }
 
 ERTS_GLB_INLINE void erts_ptab_rwlock(ErtsPTab *ptab)
 {
-    erts_smp_rwmtx_rwlock(&ptab->list.data.rwmtx);
+    erts_rwmtx_rwlock(&ptab->list.data.rwmtx);
 }
 
 ERTS_GLB_INLINE int erts_ptab_tryrwlock(ErtsPTab *ptab)
 {
-    return erts_smp_rwmtx_tryrwlock(&ptab->list.data.rwmtx);
+    return erts_rwmtx_tryrwlock(&ptab->list.data.rwmtx);
 }
 
 ERTS_GLB_INLINE void erts_ptab_rwunlock(ErtsPTab *ptab)
 {
-    erts_smp_rwmtx_rwunlock(&ptab->list.data.rwmtx);
+    erts_rwmtx_rwunlock(&ptab->list.data.rwmtx);
 }
 
-ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rlocked(ErtsPTab *ptab)
+ERTS_GLB_INLINE int erts_lc_ptab_is_rlocked(ErtsPTab *ptab)
 {
-    return erts_smp_lc_rwmtx_is_rlocked(&ptab->list.data.rwmtx);
+    return erts_lc_rwmtx_is_rlocked(&ptab->list.data.rwmtx);
 }
 
-ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rwlocked(ErtsPTab *ptab)
+ERTS_GLB_INLINE int erts_lc_ptab_is_rwlocked(ErtsPTab *ptab)
 {
-    return erts_smp_lc_rwmtx_is_rwlocked(&ptab->list.data.rwmtx);
+    return erts_lc_rwmtx_is_rwlocked(&ptab->list.data.rwmtx);
 }
 
 #endif
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
deleted file mode 100644
index dabf8702c8..0000000000
--- a/erts/emulator/beam/erl_smp.h
+++ /dev/null
@@ -1,1063 +0,0 @@
-/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 2005-2017. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * %CopyrightEnd%
- */
-/*
- * SMP interface to ethread library.
- * This is essentially "sed s/erts_/erts_smp_/g < erl_threads.h > erl_smp.h",
- * plus changes to NOP operations when ERTS_SMP is disabled.
- * Author: Mikael Pettersson
- */
-#ifndef ERL_SMP_H
-#define ERL_SMP_H
-#include "erl_threads.h"
-
-#ifdef ERTS_ENABLE_LOCK_POSITION
-#define erts_smp_mtx_lock(L) erts_smp_mtx_lock_x(L, __FILE__, __LINE__)
-#define erts_smp_mtx_trylock(L) erts_smp_mtx_trylock_x(L, __FILE__, __LINE__)
-#define erts_smp_spin_lock(L) erts_smp_spin_lock_x(L, __FILE__, __LINE__)
-#define erts_smp_rwmtx_tryrlock(L) erts_smp_rwmtx_tryrlock_x(L, __FILE__, __LINE__)
-#define erts_smp_rwmtx_rlock(L) erts_smp_rwmtx_rlock_x(L, __FILE__, __LINE__)
-#define erts_smp_rwmtx_tryrwlock(L) erts_smp_rwmtx_tryrwlock_x(L, __FILE__, __LINE__)
-#define erts_smp_rwmtx_rwlock(L) erts_smp_rwmtx_rwlock_x(L, __FILE__, __LINE__)
-#define erts_smp_read_lock(L) erts_smp_read_lock_x(L, __FILE__, __LINE__)
-#define erts_smp_write_lock(L) erts_smp_write_lock_x(L, __FILE__, __LINE__)
-#endif
-
-
-#define ERTS_SMP_THR_OPTS_DEFAULT_INITER ERTS_THR_OPTS_DEFAULT_INITER
-typedef erts_thr_opts_t erts_smp_thr_opts_t;
-typedef erts_thr_init_data_t erts_smp_thr_init_data_t;
-typedef erts_tid_t erts_smp_tid_t;
-typedef erts_mtx_t erts_smp_mtx_t;
-typedef erts_cnd_t erts_smp_cnd_t;
-#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER ERTS_RWMTX_OPT_DEFAULT_INITER
-#define ERTS_SMP_RWMTX_TYPE_NORMAL ERTS_RWMTX_TYPE_NORMAL
-#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ ERTS_RWMTX_TYPE_FREQUENT_READ
-#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
-  ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ
-#define ERTS_SMP_RWMTX_LONG_LIVED ERTS_RWMTX_LONG_LIVED
-#define ERTS_SMP_RWMTX_SHORT_LIVED ERTS_RWMTX_SHORT_LIVED
-#define ERTS_SMP_RWMTX_UNKNOWN_LIVED ERTS_RWMTX_UNKNOWN_LIVED
-typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t;
-typedef erts_rwmtx_t erts_smp_rwmtx_t;
-typedef erts_tsd_key_t erts_smp_tsd_key_t;
-#define erts_smp_dw_atomic_t erts_dw_atomic_t
-#define erts_smp_atomic_t erts_atomic_t
-#define erts_smp_atomic32_t erts_atomic32_t
-#define erts_smp_atomic64_t erts_atomic64_t
-typedef erts_spinlock_t erts_smp_spinlock_t;
-typedef erts_rwlock_t erts_smp_rwlock_t;
-void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */
-
-#define ERTS_SMP_MEMORY_BARRIER ERTS_THR_MEMORY_BARRIER
-#define ERTS_SMP_WRITE_MEMORY_BARRIER ERTS_THR_WRITE_MEMORY_BARRIER
-#define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER
-#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
-
-
-ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id);
-ERTS_GLB_INLINE void erts_smp_thr_create(erts_smp_tid_t *tid,
-					 void * (*func)(void *),
-					 void *arg,
-					 erts_smp_thr_opts_t *opts);
-ERTS_GLB_INLINE void erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res);
-ERTS_GLB_INLINE void erts_smp_thr_detach(erts_smp_tid_t tid);
-ERTS_GLB_INLINE void erts_smp_thr_exit(void *res);
-ERTS_GLB_INLINE void erts_smp_install_exit_handler(void (*exit_handler)(void));
-ERTS_GLB_INLINE erts_smp_tid_t erts_smp_thr_self(void);
-ERTS_GLB_INLINE int erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y);
-#ifdef ERTS_HAVE_REC_MTX_INIT
-#define ERTS_SMP_HAVE_REC_MTX_INIT 1
-ERTS_GLB_INLINE void erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx);
-#endif
-ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx,
-                                       char *name,
-                                       Eterm extra,
-                                       erts_lock_flags_t flags);
-ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx,
-                                               char *name,
-                                               Eterm extra,
-                                               erts_lock_flags_t flags);
-ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx);
-#ifdef ERTS_ENABLE_LOCK_POSITION
-ERTS_GLB_INLINE int erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
-ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
-#else
-ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx);
-ERTS_GLB_INLINE void erts_smp_mtx_lock(erts_smp_mtx_t *mtx);
-#endif
-ERTS_GLB_INLINE void erts_smp_mtx_unlock(erts_smp_mtx_t *mtx);
-ERTS_GLB_INLINE int erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx);
-ERTS_GLB_INLINE void erts_smp_cnd_init(erts_smp_cnd_t *cnd);
-ERTS_GLB_INLINE void erts_smp_cnd_destroy(erts_smp_cnd_t *cnd);
-ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd,
-					   erts_smp_mtx_t *mtx);
-ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd);
-ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd);
-ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no);
-ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
-                                             erts_smp_rwmtx_opt_t *opt,
-                                             char *name,
-                                             Eterm extra,
-                                             erts_lock_flags_t flags);
-ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx,
-                                         char *name,
-                                         Eterm extra,
-                                         erts_lock_flags_t flags);
-ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx);
-#ifdef ERTS_ENABLE_LOCK_POSITION
-ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
-ERTS_GLB_INLINE void erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
-ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
-ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
-#else
-ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx);
-ERTS_GLB_INLINE void erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx);
-ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx);
-ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx);
-#endif
-ERTS_GLB_INLINE void erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx);
-ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx);
-ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx);
-ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx);
-ERTS_GLB_INLINE void erts_smp_spinlock_init(erts_smp_spinlock_t *lock,
-                                            char *name,
-                                            Eterm extra,
-                                            erts_lock_flags_t flags);
-ERTS_GLB_INLINE void erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock);
-ERTS_GLB_INLINE void erts_smp_spin_unlock(erts_smp_spinlock_t *lock);
-#ifdef ERTS_ENABLE_LOCK_POSITION
-ERTS_GLB_INLINE void erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line);
-#else
-ERTS_GLB_INLINE void erts_smp_spin_lock(erts_smp_spinlock_t *lock);
-#endif
-ERTS_GLB_INLINE int erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock);
-ERTS_GLB_INLINE void erts_smp_rwlock_init(erts_smp_rwlock_t *lock,
-                                          char *name,
-                                          Eterm extra,
-                                          erts_lock_flags_t flags);
-ERTS_GLB_INLINE void erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock);
-ERTS_GLB_INLINE void erts_smp_read_unlock(erts_smp_rwlock_t *lock);
-#ifdef ERTS_ENABLE_LOCK_POSITION
-ERTS_GLB_INLINE void erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
-ERTS_GLB_INLINE void erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
-#else
-ERTS_GLB_INLINE void erts_smp_read_lock(erts_smp_rwlock_t *lock);
-ERTS_GLB_INLINE void erts_smp_write_lock(erts_smp_rwlock_t *lock);
-#endif
-ERTS_GLB_INLINE void erts_smp_write_unlock(erts_smp_rwlock_t *lock);
-ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock);
-ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock);
-ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp,
-					     char *keyname);
-ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key);
-ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value);
-ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key);
-
-#ifdef ERTS_THR_HAVE_SIG_FUNCS
-#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1
-ERTS_GLB_INLINE void erts_smp_thr_sigmask(int how,
-					  const sigset_t *set,
-					  sigset_t *oset);
-ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
-#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
-
-/*
- * See "Documentation of atomics and memory barriers" at the top
- * of erl_threads.h for info on atomics.
- */
-
-
-/* Double word size atomics */
-
-#define erts_smp_dw_atomic_init_nob erts_dw_atomic_init_nob
-#define erts_smp_dw_atomic_set_nob erts_dw_atomic_set_nob
-#define erts_smp_dw_atomic_read_nob erts_dw_atomic_read_nob
-#define erts_smp_dw_atomic_cmpxchg_nob erts_dw_atomic_cmpxchg_nob
-
-#define erts_smp_dw_atomic_init_mb erts_dw_atomic_init_mb
-#define erts_smp_dw_atomic_set_mb erts_dw_atomic_set_mb
-#define erts_smp_dw_atomic_read_mb erts_dw_atomic_read_mb
-#define erts_smp_dw_atomic_cmpxchg_mb erts_dw_atomic_cmpxchg_mb
-
-#define erts_smp_dw_atomic_init_acqb erts_dw_atomic_init_acqb
-#define erts_smp_dw_atomic_set_acqb erts_dw_atomic_set_acqb
-#define erts_smp_dw_atomic_read_acqb erts_dw_atomic_read_acqb
-#define erts_smp_dw_atomic_cmpxchg_acqb erts_dw_atomic_cmpxchg_acqb
-
-#define erts_smp_dw_atomic_init_relb erts_dw_atomic_init_relb
-#define erts_smp_dw_atomic_set_relb erts_dw_atomic_set_relb
-#define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb
-#define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb
-
-#define erts_smp_dw_atomic_init_ddrb erts_dw_atomic_init_ddrb
-#define erts_smp_dw_atomic_set_ddrb erts_dw_atomic_set_ddrb
-#define erts_smp_dw_atomic_read_ddrb erts_dw_atomic_read_ddrb
-#define erts_smp_dw_atomic_cmpxchg_ddrb erts_dw_atomic_cmpxchg_ddrb
-
-#define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb
-#define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb
-#define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb
-#define erts_smp_dw_atomic_cmpxchg_rb erts_dw_atomic_cmpxchg_rb
-
-#define erts_smp_dw_atomic_init_wb erts_dw_atomic_init_wb
-#define erts_smp_dw_atomic_set_wb erts_dw_atomic_set_wb
-#define erts_smp_dw_atomic_read_wb erts_dw_atomic_read_wb
-#define erts_smp_dw_atomic_cmpxchg_wb erts_dw_atomic_cmpxchg_wb
-
-#define erts_smp_dw_atomic_set_dirty erts_dw_atomic_set_dirty
-#define erts_smp_dw_atomic_read_dirty erts_dw_atomic_read_dirty
-
-/* Word size atomics */
-
-#define erts_smp_atomic_init_nob erts_atomic_init_nob
-#define erts_smp_atomic_set_nob erts_atomic_set_nob
-#define erts_smp_atomic_read_nob erts_atomic_read_nob
-#define erts_smp_atomic_inc_read_nob erts_atomic_inc_read_nob
-#define erts_smp_atomic_dec_read_nob erts_atomic_dec_read_nob
-#define erts_smp_atomic_inc_nob erts_atomic_inc_nob
-#define erts_smp_atomic_dec_nob erts_atomic_dec_nob
-#define erts_smp_atomic_add_read_nob erts_atomic_add_read_nob
-#define erts_smp_atomic_add_nob erts_atomic_add_nob
-#define erts_smp_atomic_read_bor_nob erts_atomic_read_bor_nob
-#define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob
-#define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob
-#define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob
-#define erts_smp_atomic_read_bset_nob erts_atomic_read_bset_nob
-
-#define erts_smp_atomic_init_mb erts_atomic_init_mb
-#define erts_smp_atomic_set_mb erts_atomic_set_mb
-#define erts_smp_atomic_read_mb erts_atomic_read_mb
-#define erts_smp_atomic_inc_read_mb erts_atomic_inc_read_mb
-#define erts_smp_atomic_dec_read_mb erts_atomic_dec_read_mb
-#define erts_smp_atomic_inc_mb erts_atomic_inc_mb
-#define erts_smp_atomic_dec_mb erts_atomic_dec_mb
-#define erts_smp_atomic_add_read_mb erts_atomic_add_read_mb
-#define erts_smp_atomic_add_mb erts_atomic_add_mb
-#define erts_smp_atomic_read_bor_mb erts_atomic_read_bor_mb
-#define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb
-#define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb
-#define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb
-#define erts_smp_atomic_read_bset_mb erts_atomic_read_bset_mb
-
-#define erts_smp_atomic_init_acqb erts_atomic_init_acqb
-#define erts_smp_atomic_set_acqb erts_atomic_set_acqb
-#define erts_smp_atomic_read_acqb erts_atomic_read_acqb
-#define erts_smp_atomic_inc_read_acqb erts_atomic_inc_read_acqb
-#define erts_smp_atomic_dec_read_acqb erts_atomic_dec_read_acqb
-#define erts_smp_atomic_inc_acqb erts_atomic_inc_acqb
-#define erts_smp_atomic_dec_acqb erts_atomic_dec_acqb
-#define erts_smp_atomic_add_read_acqb erts_atomic_add_read_acqb
-#define erts_smp_atomic_add_acqb erts_atomic_add_acqb
-#define erts_smp_atomic_read_bor_acqb erts_atomic_read_bor_acqb
-#define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb
-#define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb
-#define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb
-#define erts_smp_atomic_read_bset_acqb erts_atomic_read_bset_acqb
-
-#define erts_smp_atomic_init_relb erts_atomic_init_relb
-#define erts_smp_atomic_set_relb erts_atomic_set_relb
-#define erts_smp_atomic_read_relb erts_atomic_read_relb
-#define erts_smp_atomic_inc_read_relb erts_atomic_inc_read_relb
-#define erts_smp_atomic_dec_read_relb erts_atomic_dec_read_relb
-#define erts_smp_atomic_inc_relb erts_atomic_inc_relb
-#define erts_smp_atomic_dec_relb erts_atomic_dec_relb
-#define erts_smp_atomic_add_read_relb erts_atomic_add_read_relb
-#define erts_smp_atomic_add_relb erts_atomic_add_relb
-#define erts_smp_atomic_read_bor_relb erts_atomic_read_bor_relb
-#define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb
-#define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb
-#define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb
-#define erts_smp_atomic_read_bset_relb erts_atomic_read_bset_relb
-
-#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb
-#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb
-#define erts_smp_atomic_read_ddrb erts_atomic_read_ddrb
-#define erts_smp_atomic_inc_read_ddrb erts_atomic_inc_read_ddrb
-#define erts_smp_atomic_dec_read_ddrb erts_atomic_dec_read_ddrb
-#define erts_smp_atomic_inc_ddrb erts_atomic_inc_ddrb
-#define erts_smp_atomic_dec_ddrb erts_atomic_dec_ddrb
-#define erts_smp_atomic_add_read_ddrb erts_atomic_add_read_ddrb
-#define erts_smp_atomic_add_ddrb erts_atomic_add_ddrb
-#define erts_smp_atomic_read_bor_ddrb erts_atomic_read_bor_ddrb
-#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb
-#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb
-#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb
-#define erts_smp_atomic_read_bset_ddrb erts_atomic_read_bset_ddrb
-
-#define erts_smp_atomic_init_rb erts_atomic_init_rb
-#define erts_smp_atomic_set_rb erts_atomic_set_rb
-#define erts_smp_atomic_read_rb erts_atomic_read_rb
-#define erts_smp_atomic_inc_read_rb erts_atomic_inc_read_rb
-#define erts_smp_atomic_dec_read_rb erts_atomic_dec_read_rb
-#define erts_smp_atomic_inc_rb erts_atomic_inc_rb
-#define erts_smp_atomic_dec_rb erts_atomic_dec_rb
-#define erts_smp_atomic_add_read_rb erts_atomic_add_read_rb
-#define erts_smp_atomic_add_rb erts_atomic_add_rb
-#define erts_smp_atomic_read_bor_rb erts_atomic_read_bor_rb
-#define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb
-#define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb
-#define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb
-#define erts_smp_atomic_read_bset_rb erts_atomic_read_bset_rb
-
-#define erts_smp_atomic_init_wb erts_atomic_init_wb
-#define erts_smp_atomic_set_wb erts_atomic_set_wb
-#define erts_smp_atomic_read_wb erts_atomic_read_wb
-#define erts_smp_atomic_inc_read_wb erts_atomic_inc_read_wb
-#define erts_smp_atomic_dec_read_wb erts_atomic_dec_read_wb
-#define erts_smp_atomic_inc_wb erts_atomic_inc_wb
-#define erts_smp_atomic_dec_wb erts_atomic_dec_wb
-#define erts_smp_atomic_add_read_wb erts_atomic_add_read_wb
-#define erts_smp_atomic_add_wb erts_atomic_add_wb
-#define erts_smp_atomic_read_bor_wb erts_atomic_read_bor_wb
-#define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb
-#define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb
-#define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb
-#define erts_smp_atomic_read_bset_wb erts_atomic_read_bset_wb
-
-#define erts_smp_atomic_set_dirty erts_atomic_set_dirty
-#define erts_smp_atomic_read_dirty erts_atomic_read_dirty
-
-/* 32-bit atomics */
-
-#define erts_smp_atomic32_init_nob erts_atomic32_init_nob
-#define erts_smp_atomic32_set_nob erts_atomic32_set_nob
-#define erts_smp_atomic32_read_nob erts_atomic32_read_nob
-#define erts_smp_atomic32_inc_read_nob erts_atomic32_inc_read_nob
-#define erts_smp_atomic32_dec_read_nob erts_atomic32_dec_read_nob
-#define erts_smp_atomic32_inc_nob erts_atomic32_inc_nob
-#define erts_smp_atomic32_dec_nob erts_atomic32_dec_nob
-#define erts_smp_atomic32_add_read_nob erts_atomic32_add_read_nob
-#define erts_smp_atomic32_add_nob erts_atomic32_add_nob
-#define erts_smp_atomic32_read_bor_nob erts_atomic32_read_bor_nob
-#define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob
-#define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob
-#define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob
-#define erts_smp_atomic32_read_bset_nob erts_atomic32_read_bset_nob
-
-#define erts_smp_atomic32_init_mb erts_atomic32_init_mb
-#define erts_smp_atomic32_set_mb erts_atomic32_set_mb
-#define erts_smp_atomic32_read_mb erts_atomic32_read_mb
-#define erts_smp_atomic32_inc_read_mb erts_atomic32_inc_read_mb
-#define erts_smp_atomic32_dec_read_mb erts_atomic32_dec_read_mb
-#define erts_smp_atomic32_inc_mb erts_atomic32_inc_mb
-#define erts_smp_atomic32_dec_mb erts_atomic32_dec_mb
-#define erts_smp_atomic32_add_read_mb erts_atomic32_add_read_mb
-#define erts_smp_atomic32_add_mb erts_atomic32_add_mb
-#define erts_smp_atomic32_read_bor_mb erts_atomic32_read_bor_mb
-#define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb
-#define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb
-#define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb
-#define erts_smp_atomic32_read_bset_mb erts_atomic32_read_bset_mb
-
-#define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb
-#define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb
-#define erts_smp_atomic32_read_acqb erts_atomic32_read_acqb
-#define erts_smp_atomic32_inc_read_acqb erts_atomic32_inc_read_acqb
-#define erts_smp_atomic32_dec_read_acqb erts_atomic32_dec_read_acqb
-#define erts_smp_atomic32_inc_acqb erts_atomic32_inc_acqb
-#define erts_smp_atomic32_dec_acqb erts_atomic32_dec_acqb
-#define erts_smp_atomic32_add_read_acqb erts_atomic32_add_read_acqb
-#define erts_smp_atomic32_add_acqb erts_atomic32_add_acqb
-#define erts_smp_atomic32_read_bor_acqb erts_atomic32_read_bor_acqb
-#define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb
-#define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb
-#define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb
-#define erts_smp_atomic32_read_bset_acqb erts_atomic32_read_bset_acqb
-
-#define erts_smp_atomic32_init_relb erts_atomic32_init_relb
-#define erts_smp_atomic32_set_relb erts_atomic32_set_relb
-#define erts_smp_atomic32_read_relb erts_atomic32_read_relb
-#define erts_smp_atomic32_inc_read_relb erts_atomic32_inc_read_relb
-#define erts_smp_atomic32_dec_read_relb erts_atomic32_dec_read_relb
-#define erts_smp_atomic32_inc_relb erts_atomic32_inc_relb
-#define erts_smp_atomic32_dec_relb erts_atomic32_dec_relb
-#define erts_smp_atomic32_add_read_relb erts_atomic32_add_read_relb
-#define erts_smp_atomic32_add_relb erts_atomic32_add_relb
-#define erts_smp_atomic32_read_bor_relb erts_atomic32_read_bor_relb
-#define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb
-#define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb
-#define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb
-#define erts_smp_atomic32_read_bset_relb erts_atomic32_read_bset_relb
-
-#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb
-#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb
-#define erts_smp_atomic32_read_ddrb erts_atomic32_read_ddrb
-#define erts_smp_atomic32_inc_read_ddrb erts_atomic32_inc_read_ddrb
-#define erts_smp_atomic32_dec_read_ddrb erts_atomic32_dec_read_ddrb
-#define erts_smp_atomic32_inc_ddrb erts_atomic32_inc_ddrb
-#define erts_smp_atomic32_dec_ddrb erts_atomic32_dec_ddrb
-#define erts_smp_atomic32_add_read_ddrb erts_atomic32_add_read_ddrb
-#define erts_smp_atomic32_add_ddrb erts_atomic32_add_ddrb
-#define erts_smp_atomic32_read_bor_ddrb erts_atomic32_read_bor_ddrb
-#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb
-#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb
-#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb
-#define erts_smp_atomic32_read_bset_ddrb erts_atomic32_read_bset_ddrb
-
-#define erts_smp_atomic32_init_rb erts_atomic32_init_rb
-#define erts_smp_atomic32_set_rb erts_atomic32_set_rb
-#define erts_smp_atomic32_read_rb erts_atomic32_read_rb
-#define erts_smp_atomic32_inc_read_rb erts_atomic32_inc_read_rb
-#define erts_smp_atomic32_dec_read_rb erts_atomic32_dec_read_rb
-#define erts_smp_atomic32_inc_rb erts_atomic32_inc_rb
-#define erts_smp_atomic32_dec_rb erts_atomic32_dec_rb
-#define erts_smp_atomic32_add_read_rb erts_atomic32_add_read_rb
-#define erts_smp_atomic32_add_rb erts_atomic32_add_rb
-#define erts_smp_atomic32_read_bor_rb erts_atomic32_read_bor_rb
-#define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb
-#define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb
-#define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb
-#define erts_smp_atomic32_read_bset_rb erts_atomic32_read_bset_rb
-
-#define erts_smp_atomic32_init_wb erts_atomic32_init_wb
-#define erts_smp_atomic32_set_wb erts_atomic32_set_wb
-#define erts_smp_atomic32_read_wb erts_atomic32_read_wb
-#define erts_smp_atomic32_inc_read_wb erts_atomic32_inc_read_wb
-#define erts_smp_atomic32_dec_read_wb erts_atomic32_dec_read_wb
-#define erts_smp_atomic32_inc_wb erts_atomic32_inc_wb
-#define erts_smp_atomic32_dec_wb erts_atomic32_dec_wb
-#define erts_smp_atomic32_add_read_wb erts_atomic32_add_read_wb
-#define erts_smp_atomic32_add_wb erts_atomic32_add_wb
-#define erts_smp_atomic32_read_bor_wb erts_atomic32_read_bor_wb
-#define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb
-#define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb
-#define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb
-#define erts_smp_atomic32_read_bset_wb erts_atomic32_read_bset_wb
-
-#define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty
-#define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty
-
-/* 64-bit atomics */
-
-#define erts_smp_atomic64_init_nob erts_atomic64_init_nob
-#define erts_smp_atomic64_set_nob erts_atomic64_set_nob
-#define erts_smp_atomic64_read_nob erts_atomic64_read_nob
-#define erts_smp_atomic64_inc_read_nob erts_atomic64_inc_read_nob
-#define erts_smp_atomic64_dec_read_nob erts_atomic64_dec_read_nob
-#define erts_smp_atomic64_inc_nob erts_atomic64_inc_nob
-#define erts_smp_atomic64_dec_nob erts_atomic64_dec_nob
-#define erts_smp_atomic64_add_read_nob erts_atomic64_add_read_nob
-#define erts_smp_atomic64_add_nob erts_atomic64_add_nob
-#define erts_smp_atomic64_read_bor_nob erts_atomic64_read_bor_nob
-#define erts_smp_atomic64_read_band_nob erts_atomic64_read_band_nob
-#define erts_smp_atomic64_xchg_nob erts_atomic64_xchg_nob
-#define erts_smp_atomic64_cmpxchg_nob erts_atomic64_cmpxchg_nob
-#define erts_smp_atomic64_read_bset_nob erts_atomic64_read_bset_nob
-
-#define erts_smp_atomic64_init_mb erts_atomic64_init_mb
-#define erts_smp_atomic64_set_mb erts_atomic64_set_mb
-#define erts_smp_atomic64_read_mb erts_atomic64_read_mb
-#define erts_smp_atomic64_inc_read_mb erts_atomic64_inc_read_mb
-#define erts_smp_atomic64_dec_read_mb erts_atomic64_dec_read_mb
-#define erts_smp_atomic64_inc_mb erts_atomic64_inc_mb
-#define erts_smp_atomic64_dec_mb erts_atomic64_dec_mb
-#define erts_smp_atomic64_add_read_mb erts_atomic64_add_read_mb
-#define erts_smp_atomic64_add_mb erts_atomic64_add_mb
-#define erts_smp_atomic64_read_bor_mb erts_atomic64_read_bor_mb
-#define erts_smp_atomic64_read_band_mb erts_atomic64_read_band_mb
-#define erts_smp_atomic64_xchg_mb erts_atomic64_xchg_mb
-#define erts_smp_atomic64_cmpxchg_mb erts_atomic64_cmpxchg_mb
-#define erts_smp_atomic64_read_bset_mb erts_atomic64_read_bset_mb
-
-#define erts_smp_atomic64_init_acqb erts_atomic64_init_acqb
-#define erts_smp_atomic64_set_acqb erts_atomic64_set_acqb
-#define erts_smp_atomic64_read_acqb erts_atomic64_read_acqb
-#define erts_smp_atomic64_inc_read_acqb erts_atomic64_inc_read_acqb
-#define erts_smp_atomic64_dec_read_acqb erts_atomic64_dec_read_acqb
-#define erts_smp_atomic64_inc_acqb erts_atomic64_inc_acqb
-#define erts_smp_atomic64_dec_acqb erts_atomic64_dec_acqb
-#define erts_smp_atomic64_add_read_acqb erts_atomic64_add_read_acqb
-#define erts_smp_atomic64_add_acqb erts_atomic64_add_acqb
-#define erts_smp_atomic64_read_bor_acqb erts_atomic64_read_bor_acqb
-#define erts_smp_atomic64_read_band_acqb erts_atomic64_read_band_acqb
-#define erts_smp_atomic64_xchg_acqb erts_atomic64_xchg_acqb
-#define erts_smp_atomic64_cmpxchg_acqb erts_atomic64_cmpxchg_acqb
-#define erts_smp_atomic64_read_bset_acqb erts_atomic64_read_bset_acqb
-
-#define erts_smp_atomic64_init_relb erts_atomic64_init_relb
-#define erts_smp_atomic64_set_relb erts_atomic64_set_relb
-#define erts_smp_atomic64_read_relb erts_atomic64_read_relb
-#define erts_smp_atomic64_inc_read_relb erts_atomic64_inc_read_relb
-#define erts_smp_atomic64_dec_read_relb erts_atomic64_dec_read_relb
-#define erts_smp_atomic64_inc_relb erts_atomic64_inc_relb
-#define erts_smp_atomic64_dec_relb erts_atomic64_dec_relb
-#define erts_smp_atomic64_add_read_relb erts_atomic64_add_read_relb
-#define erts_smp_atomic64_add_relb erts_atomic64_add_relb
-#define erts_smp_atomic64_read_bor_relb erts_atomic64_read_bor_relb
-#define erts_smp_atomic64_read_band_relb erts_atomic64_read_band_relb
-#define erts_smp_atomic64_xchg_relb erts_atomic64_xchg_relb
-#define erts_smp_atomic64_cmpxchg_relb erts_atomic64_cmpxchg_relb
-#define erts_smp_atomic64_read_bset_relb erts_atomic64_read_bset_relb
-
-#define erts_smp_atomic64_init_ddrb erts_atomic64_init_ddrb
-#define erts_smp_atomic64_set_ddrb erts_atomic64_set_ddrb
-#define erts_smp_atomic64_read_ddrb erts_atomic64_read_ddrb
-#define erts_smp_atomic64_inc_read_ddrb erts_atomic64_inc_read_ddrb
-#define erts_smp_atomic64_dec_read_ddrb erts_atomic64_dec_read_ddrb
-#define erts_smp_atomic64_inc_ddrb erts_atomic64_inc_ddrb
-#define erts_smp_atomic64_dec_ddrb erts_atomic64_dec_ddrb
-#define erts_smp_atomic64_add_read_ddrb erts_atomic64_add_read_ddrb
-#define erts_smp_atomic64_add_ddrb erts_atomic64_add_ddrb
-#define erts_smp_atomic64_read_bor_ddrb erts_atomic64_read_bor_ddrb
-#define erts_smp_atomic64_read_band_ddrb erts_atomic64_read_band_ddrb
-#define erts_smp_atomic64_xchg_ddrb erts_atomic64_xchg_ddrb
-#define erts_smp_atomic64_cmpxchg_ddrb erts_atomic64_cmpxchg_ddrb
-#define erts_smp_atomic64_read_bset_ddrb erts_atomic64_read_bset_ddrb
-
-#define erts_smp_atomic64_init_rb erts_atomic64_init_rb
-#define erts_smp_atomic64_set_rb erts_atomic64_set_rb
-#define erts_smp_atomic64_read_rb erts_atomic64_read_rb
-#define erts_smp_atomic64_inc_read_rb erts_atomic64_inc_read_rb
-#define erts_smp_atomic64_dec_read_rb erts_atomic64_dec_read_rb
-#define erts_smp_atomic64_inc_rb erts_atomic64_inc_rb
-#define erts_smp_atomic64_dec_rb erts_atomic64_dec_rb
-#define erts_smp_atomic64_add_read_rb erts_atomic64_add_read_rb
-#define erts_smp_atomic64_add_rb erts_atomic64_add_rb
-#define erts_smp_atomic64_read_bor_rb erts_atomic64_read_bor_rb
-#define erts_smp_atomic64_read_band_rb erts_atomic64_read_band_rb
-#define erts_smp_atomic64_xchg_rb erts_atomic64_xchg_rb
-#define erts_smp_atomic64_cmpxchg_rb erts_atomic64_cmpxchg_rb
-#define erts_smp_atomic64_read_bset_rb erts_atomic64_read_bset_rb
-
-#define erts_smp_atomic64_init_wb erts_atomic64_init_wb
-#define erts_smp_atomic64_set_wb erts_atomic64_set_wb
-#define erts_smp_atomic64_read_wb erts_atomic64_read_wb
-#define erts_smp_atomic64_inc_read_wb erts_atomic64_inc_read_wb
-#define erts_smp_atomic64_dec_read_wb erts_atomic64_dec_read_wb
-#define erts_smp_atomic64_inc_wb erts_atomic64_inc_wb
-#define erts_smp_atomic64_dec_wb erts_atomic64_dec_wb
-#define erts_smp_atomic64_add_read_wb erts_atomic64_add_read_wb
-#define erts_smp_atomic64_add_wb erts_atomic64_add_wb
-#define erts_smp_atomic64_read_bor_wb erts_atomic64_read_bor_wb
-#define erts_smp_atomic64_read_band_wb erts_atomic64_read_band_wb
-#define erts_smp_atomic64_xchg_wb erts_atomic64_xchg_wb
-#define erts_smp_atomic64_cmpxchg_wb erts_atomic64_cmpxchg_wb
-#define erts_smp_atomic64_read_bset_wb erts_atomic64_read_bset_wb
-
-#define erts_smp_atomic64_set_dirty erts_atomic64_set_dirty
-#define erts_smp_atomic64_read_dirty erts_atomic64_read_dirty
-
-
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-
-ERTS_GLB_INLINE void
-erts_smp_thr_init(erts_smp_thr_init_data_t *id)
-{
-    erts_thr_init(id);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_thr_create(erts_smp_tid_t *tid, void * (*func)(void *), void *arg,
-		    erts_smp_thr_opts_t *opts)
-{
-    erts_thr_create(tid, func, arg, opts);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res)
-{
-    erts_thr_join(tid, thr_res);
-}
-
-
-ERTS_GLB_INLINE void
-erts_smp_thr_detach(erts_smp_tid_t tid)
-{
-    erts_thr_detach(tid);
-}
-
-
-ERTS_GLB_INLINE void
-erts_smp_thr_exit(void *res)
-{
-    erts_thr_exit(res);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_install_exit_handler(void (*exit_handler)(void))
-{
-    erts_thr_install_exit_handler(exit_handler);
-}
-
-ERTS_GLB_INLINE erts_smp_tid_t
-erts_smp_thr_self(void)
-{
-    return erts_thr_self();
-}
-
-
-ERTS_GLB_INLINE int
-erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y)
-{
-    return erts_equal_tids(x, y);
-}
-
-
-#ifdef ERTS_HAVE_REC_MTX_INIT
-ERTS_GLB_INLINE void
-erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx)
-{
-    erts_rec_mtx_init(mtx);
-}
-#endif
-
-ERTS_GLB_INLINE void
-erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags)
-{
-    erts_mtx_init(mtx, name, extra, flags);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags)
-{
-    erts_mtx_init_locked(mtx, name, extra, flags);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_mtx_destroy(erts_smp_mtx_t *mtx)
-{
-    erts_mtx_destroy(mtx);
-}
-
-ERTS_GLB_INLINE int
-#ifdef ERTS_ENABLE_LOCK_POSITION
-erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
-#else
-erts_smp_mtx_trylock(erts_smp_mtx_t *mtx)
-#endif
-{
-#if defined(ERTS_ENABLE_LOCK_POSITION)
-    return erts_mtx_trylock_x(mtx,file,line);
-#else
-    return erts_mtx_trylock(mtx);
-#endif
-
-}
-
-
-ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_POSITION
-erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
-#else
-erts_smp_mtx_lock(erts_smp_mtx_t *mtx)
-#endif
-{
-#if defined(ERTS_ENABLE_LOCK_POSITION)
-    erts_mtx_lock_x(mtx, file, line);
-#else
-    erts_mtx_lock(mtx);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_mtx_unlock(erts_smp_mtx_t *mtx)
-{
-    erts_mtx_unlock(mtx);
-}
-
-ERTS_GLB_INLINE int
-erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx)
-{
-#if defined(ERTS_ENABLE_LOCK_CHECK)
-    return erts_lc_mtx_is_locked(mtx);
-#else
-    return 0;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_cnd_init(erts_smp_cnd_t *cnd)
-{
-    erts_cnd_init(cnd);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_cnd_destroy(erts_smp_cnd_t *cnd)
-{
-    erts_cnd_destroy(cnd);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx)
-{
-    erts_cnd_wait(cnd, mtx);
-}
-
-/*
- * IMPORTANT note about erts_smp_cnd_signal() and erts_smp_cnd_broadcast()
- *
- * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
- * even though the associated mutex/mutexes isn't/aren't locked by the
- * caller. Our implementation do not allow that in order to avoid a
- * performance penalty. That is, all associated mutexes *need* to be
- * locked by the caller of erts_smp_cnd_signal()/erts_smp_cnd_broadcast()!
- */
-
-ERTS_GLB_INLINE void
-erts_smp_cnd_signal(erts_smp_cnd_t *cnd)
-{
-    erts_cnd_signal(cnd);
-}
-
-
-ERTS_GLB_INLINE void
-erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd)
-{
-    erts_cnd_broadcast(cnd);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_rwmtx_set_reader_group(int no)
-{
-    erts_rwmtx_set_reader_group(no);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx,
-                    char *name,
-                    Eterm extra,
-                    erts_lock_flags_t flags)
-{
-    erts_smp_rwmtx_init_opt(rwmtx, NULL, name, extra, flags);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
-                        erts_smp_rwmtx_opt_t *opt,
-                        char *name,
-                        Eterm extra,
-                        erts_lock_flags_t flags)
-{
-    erts_rwmtx_init_opt(rwmtx, opt, name, extra, flags);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx)
-{
-    erts_rwmtx_destroy(rwmtx);
-}
-
-ERTS_GLB_INLINE int
-#ifdef ERTS_ENABLE_LOCK_POSITION
-erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
-#else
-erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx)
-#endif
-{
-#if defined(ERTS_ENABLE_LOCK_POSITION)
-    return erts_rwmtx_tryrlock_x(rwmtx, file, line);
-#else
-    return erts_rwmtx_tryrlock(rwmtx);
-#endif
-}
-
-ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_POSITION
-erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
-#else
-erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx)
-#endif
-{
-#if defined(ERTS_ENABLE_LOCK_POSITION)
-    erts_rwmtx_rlock_x(rwmtx, file, line);
-#else
-    erts_rwmtx_rlock(rwmtx);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx)
-{
-    erts_rwmtx_runlock(rwmtx);
-}
-
-
-ERTS_GLB_INLINE int
-#ifdef ERTS_ENABLE_LOCK_POSITION
-erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
-#else
-erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx)
-#endif
-{
-#if defined(ERTS_ENABLE_LOCK_POSITION)
-    return erts_rwmtx_tryrwlock_x(rwmtx, file, line);
-#else
-    return erts_rwmtx_tryrwlock(rwmtx);
-#endif
-}
-
-ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_POSITION
-erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
-#else
-erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx)
-#endif
-{
-#if defined(ERTS_ENABLE_LOCK_POSITION)
-    erts_rwmtx_rwlock_x(rwmtx, file, line);
-#else
-    erts_rwmtx_rwlock(rwmtx);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx)
-{
-    erts_rwmtx_rwunlock(rwmtx);
-}
-
-#if 0 /* The following rwmtx function names are
-	 reserved for potential future use. */ 
-
-/* Try upgrade from r-locked state to rw-locked state */
-ERTS_GLB_INLINE int
-erts_smp_rwmtx_trywlock(erts_smp_rwmtx_t *rwmtx)
-{
-    return 0;
-}
-
-/* Upgrade from r-locked state to rw-locked state */
-ERTS_GLB_INLINE void
-erts_smp_rwmtx_wlock(erts_smp_rwmtx_t *rwmtx)
-{
-
-}
-
-/* Downgrade from rw-locked state to r-locked state */
-ERTS_GLB_INLINE void
-erts_smp_rwmtx_wunlock(erts_smp_rwmtx_t *rwmtx)
-{
-
-}
-
-#endif
-
-ERTS_GLB_INLINE int
-erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx)
-{
-#if defined(ERTS_ENABLE_LOCK_CHECK)
-    return erts_lc_rwmtx_is_rlocked(mtx);
-#else
-    return 0;
-#endif
-}
-
-ERTS_GLB_INLINE int
-erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx)
-{
-#if defined(ERTS_ENABLE_LOCK_CHECK)
-    return erts_lc_rwmtx_is_rwlocked(mtx);
-#else
-    return 0;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_spinlock_init(erts_smp_spinlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags)
-{
-    erts_spinlock_init(lock, name, extra, flags);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock)
-{
-    erts_spinlock_destroy(lock);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_spin_unlock(erts_smp_spinlock_t *lock)
-{
-    erts_spin_unlock(lock);
-}
-
-ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_POSITION
-erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line)
-#else
-erts_smp_spin_lock(erts_smp_spinlock_t *lock)
-#endif
-{
-#if defined(ERTS_ENABLE_LOCK_POSITION)
-    erts_spin_lock_x(lock, file, line);
-#else
-    erts_spin_lock(lock);
-#endif
-}
-
-ERTS_GLB_INLINE int
-erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock)
-{
-#if defined(ERTS_ENABLE_LOCK_CHECK)
-    return erts_lc_spinlock_is_locked(lock);
-#else
-    return 0;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_rwlock_init(erts_smp_rwlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags)
-{
-    erts_rwlock_init(lock, name, extra, flags);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock)
-{
-    erts_rwlock_destroy(lock);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_read_unlock(erts_smp_rwlock_t *lock)
-{
-    erts_read_unlock(lock);
-}
-
-ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_POSITION
-erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
-#else
-erts_smp_read_lock(erts_smp_rwlock_t *lock)
-#endif 
-{
-#if defined(ERTS_ENABLE_LOCK_POSITION)
-    erts_read_lock_x(lock, file, line);
-#else
-    erts_read_lock(lock);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_write_unlock(erts_smp_rwlock_t *lock)
-{
-    erts_write_unlock(lock);
-}
-
-ERTS_GLB_INLINE void
-#ifdef ERTS_ENABLE_LOCK_POSITION
-erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
-#else
-erts_smp_write_lock(erts_smp_rwlock_t *lock)
-#endif 
-{
-#if defined(ERTS_ENABLE_LOCK_POSITION)
-    erts_write_lock_x(lock, file, line);
-#else
-    erts_write_lock(lock);
-#endif
-}
-
-ERTS_GLB_INLINE int
-erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock)
-{
-#if defined(ERTS_ENABLE_LOCK_CHECK)
-    return erts_lc_rwlock_is_rlocked(lock);
-#else
-    return 0;
-#endif
-}
-
-ERTS_GLB_INLINE int
-erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock)
-{
-#if defined(ERTS_ENABLE_LOCK_CHECK)
-    return erts_lc_rwlock_is_rwlocked(lock);
-#else
-    return 0;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp, char* keyname)
-{
-    erts_tsd_key_create(keyp,keyname);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_tsd_key_delete(erts_smp_tsd_key_t key)
-{
-    erts_tsd_key_delete(key);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value)
-{
-    erts_tsd_set(key, value);
-}
-
-ERTS_GLB_INLINE void *
-erts_smp_tsd_get(erts_smp_tsd_key_t key)
-{
-    return erts_tsd_get(key);
-}
-
-#ifdef ERTS_THR_HAVE_SIG_FUNCS
-#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1
-
-ERTS_GLB_INLINE void
-erts_smp_thr_sigmask(int how, const sigset_t *set, sigset_t *oset)
-{
-    erts_thr_sigmask(how, set, oset);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_thr_sigwait(const sigset_t *set, int *sig)
-{
-    erts_thr_sigwait(set, sig);
-}
-
-#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
-
-#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
-
-#endif /* ERL_SMP_H */
-
-#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS
-
-/* Deprecated functions to replace */
-
-#undef erts_smp_atomic_init
-#undef erts_smp_atomic_set
-#undef erts_smp_atomic_read
-#undef erts_smp_atomic_inctest
-#undef erts_smp_atomic_dectest
-#undef erts_smp_atomic_inc
-#undef erts_smp_atomic_dec
-#undef erts_smp_atomic_addtest
-#undef erts_smp_atomic_add
-#undef erts_smp_atomic_xchg
-#undef erts_smp_atomic_cmpxchg
-#undef erts_smp_atomic_bor
-#undef erts_smp_atomic_band
-
-#undef erts_smp_atomic32_init
-#undef erts_smp_atomic32_set
-#undef erts_smp_atomic32_read
-#undef erts_smp_atomic32_inctest
-#undef erts_smp_atomic32_dectest
-#undef erts_smp_atomic32_inc
-#undef erts_smp_atomic32_dec
-#undef erts_smp_atomic32_addtest
-#undef erts_smp_atomic32_add
-#undef erts_smp_atomic32_xchg
-#undef erts_smp_atomic32_cmpxchg
-#undef erts_smp_atomic32_bor
-#undef erts_smp_atomic32_band
-
-#endif
diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h
index 80ede05d73..fa936b5707 100644
--- a/erts/emulator/beam/erl_thr_progress.h
+++ b/erts/emulator/beam/erl_thr_progress.h
@@ -33,11 +33,6 @@
 
 #include "sys.h"
 
-
-#define erts_smp_thr_progress_block erts_thr_progress_block
-#define erts_smp_thr_progress_unblock erts_thr_progress_unblock
-#define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking
-
 void erts_thr_progress_block(void);
 void erts_thr_progress_unblock(void);
 int erts_thr_progress_is_blocking(void);
diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c
index 66566edec5..548c2768e5 100644
--- a/erts/emulator/beam/erl_thr_queue.c
+++ b/erts/emulator/beam/erl_thr_queue.c
@@ -391,7 +391,7 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify)
 		ilast = (erts_aint_t) enqueue_marker(q, NULL);
 
 	    if (q->head.unref_end == (ErtsThrQElement_t *) ilast)
-		ERTS_SMP_MEMORY_BARRIER;
+		ERTS_THR_MEMORY_BARRIER;
 	    else {
 		q->head.next.unref_end = (ErtsThrQElement_t *) ilast;
 		q->head.next.thr_progress = erts_thr_progress_later(NULL);
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 22a359554e..52af77e303 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -45,11 +45,6 @@
  *      Data dependency read barrier. Orders *only* loads
  *      according to data dependency across the barrier.
  *
- * If thread support has been disabled, these barriers will become no-ops.
- *
- * If the prefix ERTS_THR_ is replaced with ERTS_SMP_, the barriers will
- * be enabled only in the SMP enabled runtime system.
- *
  * --- Atomic operations ---
  *
  * Atomics operations exist for 32-bit, word size, and double word size
@@ -86,20 +81,6 @@
  *          barrier. Load in atomic operation is ordered
  *          before the barrier.
  *
- * If thread support has been disabled, these functions are mapped to
- * functions that performs the same operation, but aren't atomic
- * and don't imply any memory barriers.
- *
- * If the atomic operations are prefixed with erts_smp_ instead of only
- * erts_ the atomic operations will only be atomic in the SMP enabled
- * runtime system, and will be mapped to non-atomic operations without
- * memory barriers in the runtime system without SMP support. Atomic
- * operations with erts_smp_ prefix should use the atomic types
- * erts_smp_atomic32_t, erts_smp_atomic_t, and erts_smp_dw_atomic_t
- * instead of erts_atomic32_t, erts_atomic_t, and erts_dw_atomic_t. The
- * integer data types erts_aint32_t, erts_aint_t, and erts_dw_atomic_t
- * are the same.
- *
  * --- 32-bit atomic operations ---
  *
  * The following 32-bit atomic operations exist. <B> should be
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index f6bb52dde1..c06b464458 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -36,8 +36,8 @@
 #include "erl_driver.h"
 #include "erl_nif.h"
  
-static erts_smp_mtx_t erts_timeofday_mtx;
-static erts_smp_mtx_t erts_get_time_mtx;
+static erts_mtx_t erts_timeofday_mtx;
+static erts_mtx_t erts_get_time_mtx;
 
 static SysTimes t_start; /* Used in elapsed_time_both */
 static ErtsMonotonicTime prev_wall_clock_elapsed; /* Used in wall_clock_elapsed_time_both */
@@ -140,7 +140,7 @@ typedef struct {
 struct time_sup_infrequently_changed__ {
 #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
     struct {
-	erts_smp_rwmtx_t rwmtx;
+	erts_rwmtx_t rwmtx;
 	ErtsTWheelTimer timer;
 	ErtsMonotonicCorrectionData cdata;
     } parmon;
@@ -148,9 +148,9 @@ struct time_sup_infrequently_changed__ {
 #endif
     ErtsSystemTime sinit;
     ErtsMonotonicTime not_corrected_moffset;
-    erts_smp_atomic64_t offset;
+    erts_atomic64_t offset;
     ErtsMonotonicTime shadow_offset;
-    erts_smp_atomic32_t preliminary_offset;
+    erts_atomic32_t preliminary_offset;
 };
 
 struct time_sup_frequently_changed__ {
@@ -188,19 +188,19 @@ erts_get_approx_time(void)
 static ERTS_INLINE void
 init_time_offset(ErtsMonotonicTime offset)
 {
-    erts_smp_atomic64_init_nob(&time_sup.inf.c.offset, (erts_aint64_t) offset);
+    erts_atomic64_init_nob(&time_sup.inf.c.offset, (erts_aint64_t) offset);
 }
 
 static ERTS_INLINE void
 set_time_offset(ErtsMonotonicTime offset)
 {
-    erts_smp_atomic64_set_relb(&time_sup.inf.c.offset, (erts_aint64_t) offset);
+    erts_atomic64_set_relb(&time_sup.inf.c.offset, (erts_aint64_t) offset);
 }
 
 static ERTS_INLINE ErtsMonotonicTime
 get_time_offset(void)
 {
-    return (ErtsMonotonicTime) erts_smp_atomic64_read_acqb(&time_sup.inf.c.offset);
+    return (ErtsMonotonicTime) erts_atomic64_read_acqb(&time_sup.inf.c.offset);
 }
 
 static ERTS_INLINE void
@@ -281,7 +281,7 @@ read_corrected_time(int os_drift_corrected)
     ErtsMonotonicTime os_mtime;
     ErtsMonotonicCorrectionInstance ci;
 
-    erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx);
+    erts_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx);
 
     os_mtime = erts_os_monotonic_time();
 
@@ -294,7 +294,7 @@ read_corrected_time(int os_drift_corrected)
 	ci = time_sup.inf.c.parmon.cdata.insts.prev;
     }
 
-    erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
+    erts_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
 
     return calc_corrected_erl_mtime(os_mtime, &ci, NULL,
 				    os_drift_corrected);
@@ -372,13 +372,13 @@ check_time_correction(void *vesdp)
     int os_drift_corrected = time_sup.r.o.os_corrected_monotonic_time;
     int set_new_correction = 0, begin_short_intervals = 0;
 
-    erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx);
+    erts_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx);
 
     erts_os_times(&os_mtime, &os_stime);
 
     ci = time_sup.inf.c.parmon.cdata.insts.curr;
 
-    erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
+    erts_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
 
     if (os_mtime < ci.os_mtime)
 	erts_exit(ERTS_ABORT_EXIT,
@@ -393,7 +393,7 @@ check_time_correction(void *vesdp)
 
     if (time_sup.inf.c.shadow_offset) {
 	ERTS_TIME_ASSERT(time_sup.r.o.warp_mode == ERTS_SINGLE_TIME_WARP_MODE);
-	if (erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset))
+	if (erts_atomic32_read_nob(&time_sup.inf.c.preliminary_offset))
 	    sdiff += time_sup.inf.c.shadow_offset;
 	else
 	    time_sup.inf.c.shadow_offset = 0;
@@ -416,7 +416,7 @@ check_time_correction(void *vesdp)
 	}
     }
     else if ((time_sup.r.o.warp_mode == ERTS_SINGLE_TIME_WARP_MODE
-	      && erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset))
+	      && erts_atomic32_read_nob(&time_sup.inf.c.preliminary_offset))
 	     && (sdiff < -2*time_sup.r.o.adj.small_diff
 		 || 2*time_sup.r.o.adj.small_diff < sdiff)) {
 	/*
@@ -641,7 +641,7 @@ check_time_correction(void *vesdp)
 #endif
 
     if (set_new_correction) {
-	erts_smp_rwmtx_rwlock(&time_sup.inf.c.parmon.rwmtx);
+	erts_rwmtx_rwlock(&time_sup.inf.c.parmon.rwmtx);
 
 	os_mtime = erts_os_monotonic_time();
 
@@ -669,7 +669,7 @@ check_time_correction(void *vesdp)
 	time_sup.inf.c.parmon.cdata.insts.curr.os_mtime = os_mtime;
 	time_sup.inf.c.parmon.cdata.insts.curr.correction = new_correction;
 
-	erts_smp_rwmtx_rwunlock(&time_sup.inf.c.parmon.rwmtx);
+	erts_rwmtx_rwunlock(&time_sup.inf.c.parmon.rwmtx);
     }
 
     if (!esdp)
@@ -787,13 +787,13 @@ finalize_corrected_time_offset(ErtsSystemTime *stimep)
     ErtsMonotonicCorrectionInstance ci;
     int os_drift_corrected = time_sup.r.o.os_corrected_monotonic_time;
 
-    erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx);
+    erts_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx);
 
     erts_os_times(&os_mtime, stimep);
 
     ci = time_sup.inf.c.parmon.cdata.insts.curr;
 
-    erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
+    erts_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx);
 
     if (os_mtime < ci.os_mtime)
 	erts_exit(ERTS_ABORT_EXIT,
@@ -846,7 +846,7 @@ static ErtsMonotonicTime get_not_corrected_time(void)
 {
     ErtsMonotonicTime stime, mtime;
 
-    erts_smp_mtx_lock(&erts_get_time_mtx);
+    erts_mtx_lock(&erts_get_time_mtx);
 
     stime = erts_os_system_time();
 
@@ -872,7 +872,7 @@ static ErtsMonotonicTime get_not_corrected_time(void)
 
     ASSERT(stime == mtime + time_sup.inf.c.not_corrected_moffset);
 
-    erts_smp_mtx_unlock(&erts_get_time_mtx);
+    erts_mtx_unlock(&erts_get_time_mtx);
 
     return mtime;
 }
@@ -954,18 +954,18 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 
     ASSERT(ERTS_MONOTONIC_TIME_MIN < ERTS_MONOTONIC_TIME_MAX);
 
-    erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday", NIL,
+    erts_mtx_init(&erts_timeofday_mtx, "timeofday", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
-    erts_smp_mtx_init(&erts_get_time_mtx, "get_time", NIL,
+    erts_mtx_init(&erts_get_time_mtx, "get_time", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
     time_sup.r.o.correction = time_correction;
     time_sup.r.o.warp_mode = time_warp_mode;
  
     if (time_warp_mode == ERTS_SINGLE_TIME_WARP_MODE)
-	erts_smp_atomic32_init_nob(&time_sup.inf.c.preliminary_offset, 1);
+	erts_atomic32_init_nob(&time_sup.inf.c.preliminary_offset, 1);
     else
-	erts_smp_atomic32_init_nob(&time_sup.inf.c.preliminary_offset, 0);
+	erts_atomic32_init_nob(&time_sup.inf.c.preliminary_offset, 0);
     time_sup.inf.c.shadow_offset = 0;
 
 #if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
@@ -1109,7 +1109,7 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 
     if (time_sup.r.o.correction) {
 	ErtsMonotonicCorrectionData *cdatap;
-	erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+	erts_rwmtx_opt_t rwmtx_opts = ERTS_RWMTX_OPT_DEFAULT_INITER;
 	ErtsMonotonicTime offset;
 	erts_os_times(&time_sup.inf.c.minit,
 		      &time_sup.inf.c.sinit);
@@ -1119,10 +1119,10 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 	offset -= ERTS_MONOTONIC_BEGIN;
 	init_time_offset(offset);
 
-	rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
-	rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+	rwmtx_opts.type = ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
+	rwmtx_opts.lived = ERTS_RWMTX_LONG_LIVED;
 
-        erts_smp_rwmtx_init_opt(&time_sup.inf.c.parmon.rwmtx, &rwmtx_opts,
+        erts_rwmtx_init_opt(&time_sup.inf.c.parmon.rwmtx, &rwmtx_opts,
             "get_corrected_time", NIL,
             ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
@@ -1200,7 +1200,7 @@ ErtsTimeOffsetState erts_time_offset_state(void)
     case ERTS_NO_TIME_WARP_MODE:
 	return ERTS_TIME_OFFSET_FINAL;
     case ERTS_SINGLE_TIME_WARP_MODE:
-	if (erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset))
+	if (erts_atomic32_read_nob(&time_sup.inf.c.preliminary_offset))
 	    return ERTS_TIME_OFFSET_PRELIMINARY;
 	return ERTS_TIME_OFFSET_FINAL;
     case ERTS_MULTI_TIME_WARP_MODE:
@@ -1233,9 +1233,9 @@ erts_finalize_time_offset(void)
     case ERTS_SINGLE_TIME_WARP_MODE: {
 	ErtsTimeOffsetState res = ERTS_TIME_OFFSET_FINAL;
 
-	erts_smp_mtx_lock(&erts_get_time_mtx);
+	erts_mtx_lock(&erts_get_time_mtx);
 
-	if (erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) {
+	if (erts_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) {
 	    ErtsMonotonicTime mtime, new_offset;
 
 #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
@@ -1272,11 +1272,11 @@ erts_finalize_time_offset(void)
 	    set_time_offset(new_offset);
 	    schedule_send_time_offset_changed_notifications(new_offset);
 
-	    erts_smp_atomic32_set_nob(&time_sup.inf.c.preliminary_offset, 0);
+	    erts_atomic32_set_nob(&time_sup.inf.c.preliminary_offset, 0);
 	    res = ERTS_TIME_OFFSET_PRELIMINARY;
 	}
 
-	erts_smp_mtx_unlock(&erts_get_time_mtx);
+	erts_mtx_unlock(&erts_get_time_mtx);
 
 	return res;
     }
@@ -1306,13 +1306,13 @@ elapsed_time_both(ErtsMonotonicTime *ms_user, ErtsMonotonicTime *ms_sys,
 	*ms_sys = total_sys;
 
     if (ms_user_diff || ms_sys_diff) {
-        erts_smp_mtx_lock(&erts_timeofday_mtx);
+        erts_mtx_lock(&erts_timeofday_mtx);
     
         prev_total_user = (ErtsMonotonicTime) ((t_start.tms_utime * 1000) / SYS_CLK_TCK);
         prev_total_sys = (ErtsMonotonicTime) ((t_start.tms_stime * 1000) / SYS_CLK_TCK);
         t_start = now;
     
-        erts_smp_mtx_unlock(&erts_timeofday_mtx);
+        erts_mtx_unlock(&erts_timeofday_mtx);
 
         if (ms_user_diff != NULL)
             *ms_user_diff = total_user - prev_total_user;
@@ -1338,12 +1338,12 @@ wall_clock_elapsed_time_both(ErtsMonotonicTime *ms_total, ErtsMonotonicTime *ms_
     *ms_total = elapsed;
 
     if (ms_diff) {
-        erts_smp_mtx_lock(&erts_timeofday_mtx);
+        erts_mtx_lock(&erts_timeofday_mtx);
 
         *ms_diff = elapsed - prev_wall_clock_elapsed;
         prev_wall_clock_elapsed = elapsed;
 
-        erts_smp_mtx_unlock(&erts_timeofday_mtx);
+        erts_mtx_unlock(&erts_timeofday_mtx);
     }
 }
 
@@ -1729,7 +1729,7 @@ get_now(Uint* megasec, Uint* sec, Uint* microsec)
     update_last_mtime(NULL, mtime);
     now = ERTS_MONOTONIC_TO_USEC(mtime + time_offset);
 
-    erts_smp_mtx_lock(&erts_timeofday_mtx);
+    erts_mtx_lock(&erts_timeofday_mtx);
 
     /* Make sure now time is later than last time */
     if (now <= previous_now)
@@ -1737,7 +1737,7 @@ get_now(Uint* megasec, Uint* sec, Uint* microsec)
 
     previous_now = now;
     
-    erts_smp_mtx_unlock(&erts_timeofday_mtx);
+    erts_mtx_unlock(&erts_timeofday_mtx);
 
     now_megasec = now / ERTS_MONOTONIC_TIME_TERA;
     now_sec = now / ERTS_MONOTONIC_TIME_MEGA;
@@ -1824,10 +1824,10 @@ void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) {
 void
 erts_monitor_time_offset(Eterm id, Eterm ref)
 {
-    erts_smp_mtx_lock(&erts_get_time_mtx);
+    erts_mtx_lock(&erts_get_time_mtx);
     erts_add_monitor(&time_offset_monitors, MON_TIME_OFFSET, ref, id, NIL);
     no_time_offset_monitors++;
-    erts_smp_mtx_unlock(&erts_get_time_mtx);
+    erts_mtx_unlock(&erts_get_time_mtx);
 }
 
 int
@@ -1836,7 +1836,7 @@ erts_demonitor_time_offset(Eterm ref)
     int res;
     ErtsMonitor *mon;
     ASSERT(is_internal_ref(ref));
-    erts_smp_mtx_lock(&erts_get_time_mtx);
+    erts_mtx_lock(&erts_get_time_mtx);
     if (is_internal_ordinary_ref(ref))
         mon = erts_remove_monitor(&time_offset_monitors, ref);
     else
@@ -1848,7 +1848,7 @@ erts_demonitor_time_offset(Eterm ref)
 	no_time_offset_monitors--;
 	res = 1;
     }
-    erts_smp_mtx_unlock(&erts_get_time_mtx);
+    erts_mtx_unlock(&erts_get_time_mtx);
     if (res)
 	erts_destroy_monitor(mon);
     return res;
@@ -1906,7 +1906,7 @@ send_time_offset_changed_notifications(void *new_offsetp)
 #endif
     new_offset -= ERTS_MONOTONIC_OFFSET_NATIVE;
 
-    erts_smp_mtx_lock(&erts_get_time_mtx);
+    erts_mtx_lock(&erts_get_time_mtx);
 
     no_monitors = no_time_offset_monitors;
     if (no_monitors) {
@@ -1931,7 +1931,7 @@ send_time_offset_changed_notifications(void *new_offsetp)
 	ASSERT(cntxt.ix == no_monitors);
     }
 
-    erts_smp_mtx_unlock(&erts_get_time_mtx);
+    erts_mtx_unlock(&erts_get_time_mtx);
 
     if (no_monitors) {
 	Eterm *hp, *patch_refp, new_offset_term, message_template;
@@ -1964,7 +1964,7 @@ send_time_offset_changed_notifications(void *new_offsetp)
 	    if (rp) {
 		Eterm ref = to_mon_info[mix].ref;
 		ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK;
-		erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK);
+		erts_proc_lock(rp, ERTS_PROC_LOCK_LINK);
 		if (erts_lookup_monitor(ERTS_P_MONITORS(rp), ref)) {
 		    ErtsMessage *mp;
 		    ErlOffHeap *ohp;
@@ -1977,7 +1977,7 @@ send_time_offset_changed_notifications(void *new_offsetp)
 		    message = copy_struct(message_template, hsz, &hp, ohp);
 		    erts_queue_message(rp, rp_locks, mp, message, am_clock_service);
 		}
-		erts_smp_proc_unlock(rp, rp_locks);
+		erts_proc_unlock(rp, rp_locks);
 	    }
 	}
 
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 069cc27d1b..a07e3642f6 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -77,8 +77,8 @@ static Eterm system_profile;
 int erts_cpu_timestamp;
 #endif
 
-static erts_smp_mtx_t smq_mtx;
-static erts_smp_rwmtx_t sys_trace_rwmtx;
+static erts_mtx_t smq_mtx;
+static erts_rwmtx_t sys_trace_rwmtx;
 
 enum ErtsSysMsgType {
     SYS_MSG_TYPE_UNDEFINED,
@@ -322,11 +322,11 @@ static void tracer_free_fun(void*);
 typedef struct ErtsTracerNif_ ErtsTracerNif;
 
 void erts_init_trace(void) {
-    erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
-    rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
-    rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+    erts_rwmtx_opt_t rwmtx_opts = ERTS_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opts.type = ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
+    rwmtx_opts.lived = ERTS_RWMTX_LONG_LIVED;
 
-    erts_smp_rwmtx_init_opt(&sys_trace_rwmtx, &rwmtx_opts, "sys_tracers", NIL,
+    erts_rwmtx_init_opt(&sys_trace_rwmtx, &rwmtx_opts, "sys_tracers", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
 
 #ifdef HAVE_ERTS_NOW_CPU
@@ -400,14 +400,14 @@ static Uint active_sched;
 void
 erts_system_profile_setup_active_schedulers(void)
 {
-    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
     active_sched = erts_active_schedulers();
 }
 
 static void
 exiting_reset(Eterm exiting)
 {
-    erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
+    erts_rwmtx_rwlock(&sys_trace_rwmtx);
     if (exiting == system_monitor) {
 	system_monitor = NIL;
 	/* Let the trace message dispatcher clear flags, etc */
@@ -416,19 +416,19 @@ exiting_reset(Eterm exiting)
 	system_profile = NIL;
 	/* Let the trace message dispatcher clear flags, etc */
     }
-    erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
+    erts_rwmtx_rwunlock(&sys_trace_rwmtx);
 }
 
 void
 erts_trace_check_exiting(Eterm exiting)
 {
     int reset = 0;
-    erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
+    erts_rwmtx_rlock(&sys_trace_rwmtx);
     if (exiting == system_monitor)
 	reset = 1;
     else if (exiting == system_profile)
 	reset = 1;
-    erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
+    erts_rwmtx_runlock(&sys_trace_rwmtx);
     if (reset)
 	exiting_reset(exiting);
 }
@@ -448,7 +448,7 @@ erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, ErtsTracer new
         }
     }
 
-    erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
+    erts_rwmtx_rwlock(&sys_trace_rwmtx);
     old = system_seq_tracer;
     system_seq_tracer = erts_tracer_nil;
     erts_tracer_update(&system_seq_tracer, new);
@@ -456,7 +456,7 @@ erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, ErtsTracer new
 #ifdef DEBUG_PRINTOUTS
     erts_fprintf(stderr, "set seq tracer new=%T old=%T\n", new, old);
 #endif
-    erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
+    erts_rwmtx_rwunlock(&sys_trace_rwmtx);
     return old;
 }
 
@@ -464,12 +464,12 @@ ErtsTracer
 erts_get_system_seq_tracer(void)
 {
     ErtsTracer st;
-    erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
+    erts_rwmtx_rlock(&sys_trace_rwmtx);
     st = system_seq_tracer;
 #ifdef DEBUG_PRINTOUTS
     erts_fprintf(stderr, "get seq tracer %T\n", st);
 #endif
-    erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
+    erts_rwmtx_runlock(&sys_trace_rwmtx);
 
     if (st != erts_tracer_nil &&
         call_enabled_tracer(st, NULL, TRACE_FUN_ENABLED,
@@ -502,8 +502,8 @@ get_default_tracing(Uint *flagsp, ErtsTracer *tracerp,
             ErtsTracer curr_default_tracer = *default_tracer;
             if (tracerp) {
                 /* we only have a rlock, so we have to unlock and then rwlock */
-                erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
-                erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
+                erts_rwmtx_runlock(&sys_trace_rwmtx);
+                erts_rwmtx_rwlock(&sys_trace_rwmtx);
             }
             /* check if someone else changed default tracer
                while we got the write lock, if so we don't do
@@ -513,8 +513,8 @@ get_default_tracing(Uint *flagsp, ErtsTracer *tracerp,
                 ERTS_TRACER_CLEAR(default_tracer);
             }
             if (tracerp) {
-                erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
-                erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
+                erts_rwmtx_rwunlock(&sys_trace_rwmtx);
+                erts_rwmtx_rlock(&sys_trace_rwmtx);
             }
         }
         }
@@ -547,81 +547,81 @@ void
 erts_change_default_proc_tracing(int setflags, Uint flagsp,
                                  const ErtsTracer tracer)
 {
-    erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
+    erts_rwmtx_rwlock(&sys_trace_rwmtx);
     erts_change_default_tracing(
         setflags, flagsp, tracer,
         &default_proc_trace_flags,
         &default_proc_tracer);
-    erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
+    erts_rwmtx_rwunlock(&sys_trace_rwmtx);
 }
 
 void
 erts_change_default_port_tracing(int setflags, Uint flagsp,
                                  const ErtsTracer tracer)
 {
-    erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
+    erts_rwmtx_rwlock(&sys_trace_rwmtx);
     erts_change_default_tracing(
         setflags, flagsp, tracer,
         &default_port_trace_flags,
         &default_port_tracer);
-    erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
+    erts_rwmtx_rwunlock(&sys_trace_rwmtx);
 }
 
 void
 erts_get_default_proc_tracing(Uint *flagsp, ErtsTracer *tracerp)
 {
-    erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
+    erts_rwmtx_rlock(&sys_trace_rwmtx);
     *tracerp = erts_tracer_nil; /* initialize */
     get_default_tracing(
         flagsp, tracerp,
         &default_proc_trace_flags,
         &default_proc_tracer);
-    erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
+    erts_rwmtx_runlock(&sys_trace_rwmtx);
 }
 
 void
 erts_get_default_port_tracing(Uint *flagsp, ErtsTracer *tracerp)
 {
-    erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
+    erts_rwmtx_rlock(&sys_trace_rwmtx);
     *tracerp = erts_tracer_nil; /* initialize */
     get_default_tracing(
         flagsp, tracerp,
         &default_port_trace_flags,
         &default_port_tracer);
-    erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
+    erts_rwmtx_runlock(&sys_trace_rwmtx);
 }
 
 void
 erts_set_system_monitor(Eterm monitor)
 {
-    erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
+    erts_rwmtx_rwlock(&sys_trace_rwmtx);
     system_monitor = monitor;
-    erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
+    erts_rwmtx_rwunlock(&sys_trace_rwmtx);
 }
 
 Eterm
 erts_get_system_monitor(void)
 {
     Eterm monitor;
-    erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
+    erts_rwmtx_rlock(&sys_trace_rwmtx);
     monitor = system_monitor;
-    erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
+    erts_rwmtx_runlock(&sys_trace_rwmtx);
     return monitor;
 }
 
 /* Performance monitoring */
 void erts_set_system_profile(Eterm profile) {
-    erts_smp_rwmtx_rwlock(&sys_trace_rwmtx);
+    erts_rwmtx_rwlock(&sys_trace_rwmtx);
     system_profile = profile;
-    erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx);
+    erts_rwmtx_rwunlock(&sys_trace_rwmtx);
 }
 
 Eterm
 erts_get_system_profile(void) {
     Eterm profile;
-    erts_smp_rwmtx_rlock(&sys_trace_rwmtx);
+    erts_rwmtx_rlock(&sys_trace_rwmtx);
     profile = system_profile;
-    erts_smp_rwmtx_runlock(&sys_trace_rwmtx);
+    erts_rwmtx_runlock(&sys_trace_rwmtx);
     return profile;
 }
 
@@ -1093,7 +1093,7 @@ erts_call_trace(Process* p, ErtsCodeInfo *info, Binary *match_spec,
     Eterm transformed_args[MAX_ARG];
     ErtsTracer pre_ms_tracer = erts_tracer_nil;
 
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN);
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN);
 
     ASSERT(tracer);
     if (ERTS_TRACER_COMPARE(*tracer, erts_tracer_true)) {
@@ -1618,7 +1618,7 @@ profile_scheduler(Eterm scheduler_id, Eterm state) {
     bp = new_message_buffer(hsz);
     hp = bp->mem;
 
-    erts_smp_mtx_lock(&smq_mtx);
+    erts_mtx_lock(&smq_mtx);
 
     switch (state) {
     case am_active:
@@ -1641,7 +1641,7 @@ profile_scheduler(Eterm scheduler_id, Eterm state) {
     hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL);
 
     enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
-    erts_smp_mtx_unlock(&smq_mtx);
+    erts_mtx_unlock(&smq_mtx);
 
 }
 
@@ -1650,7 +1650,7 @@ profile_scheduler(Eterm scheduler_id, Eterm state) {
 void
 trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
     ErtsTracerNif *tnif = NULL;
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
     if (is_tracer_enabled(NULL, 0, &p->common, &tnif, TRACE_FUN_E_PORTS, am_open))
         send_to_tracer_nif(NULL, &p->common, p->common.id, tnif, TRACE_FUN_T_PORTS,
                 am_open, calling_pid, drv_name, am_true);
@@ -1667,9 +1667,9 @@ void
 trace_port(Port *t_p, Eterm what, Eterm data) {
 
     ErtsTracerNif *tnif = NULL;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(t_p)
 		       || erts_thr_progress_is_blocking());
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
     if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_PORTS, what))
         send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_PORTS,
                            what, data, THE_NON_VALUE, am_true);
@@ -1712,9 +1712,9 @@ void
 trace_port_receive(Port *t_p, Eterm caller, Eterm what, ...)
 {
     ErtsTracerNif *tnif = NULL;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(t_p)
 		       || erts_thr_progress_is_blocking());
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
     if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_RECEIVE, am_receive)) {
         /* We can use a stack heap here, as the nif is called in the
            context of a port */
@@ -1829,9 +1829,9 @@ trace_port_send(Port *t_p, Eterm receiver, Eterm msg, int exists)
 {
     ErtsTracerNif *tnif = NULL;
     Eterm op = exists ? am_send : am_send_to_non_existing_process;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(t_p)
 		       || erts_thr_progress_is_blocking());
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
     if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SEND, op))
         send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_SEND,
                            op, msg, receiver, am_true);
@@ -1840,9 +1840,9 @@ trace_port_send(Port *t_p, Eterm receiver, Eterm msg, int exists)
 void trace_port_send_binary(Port *t_p, Eterm to, Eterm what, char *bin, Sint sz)
 {
     ErtsTracerNif *tnif = NULL;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(t_p)
 		       || erts_thr_progress_is_blocking());
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
     if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SEND, am_send)) {
         Eterm msg;
         Binary* bptr = NULL;
@@ -1888,9 +1888,9 @@ trace_sched_ports(Port *p, Eterm what) {
 void
 trace_sched_ports_where(Port *t_p, Eterm what, Eterm where) {
     ErtsTracerNif *tnif = NULL;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(t_p)
 		       || erts_thr_progress_is_blocking());
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
     if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SCHED_PORT, what))
         send_to_tracer_nif(NULL, &t_p->common, t_p->common.id,
                            tnif, TRACE_FUN_T_SCHED_PORT,
@@ -1912,7 +1912,7 @@ profile_runnable_port(Port *p, Eterm status) {
     bp = new_message_buffer(hsz);
     hp = bp->mem;
 
-    erts_smp_mtx_lock(&smq_mtx);
+    erts_mtx_lock(&smq_mtx);
 
     msg = TUPLE5(hp, am_profile, p->common.id, status, count,
 		 NIL /* Will be overwritten by timestamp */);
@@ -1922,7 +1922,7 @@ profile_runnable_port(Port *p, Eterm status) {
     hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL);
 
     enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, p->common.id, NIL, msg, bp);
-    erts_smp_mtx_unlock(&smq_mtx);
+    erts_mtx_unlock(&smq_mtx);
 }
 
 /* Process profiling */
@@ -1966,7 +1966,7 @@ profile_runnable_proc(Process *p, Eterm status){
 
     erts_thr_progress_unmanaged_continue(dhndl);
 	
-    erts_smp_mtx_lock(&smq_mtx);
+    erts_mtx_lock(&smq_mtx);
 
     msg = TUPLE5(hp, am_profile, p->common.id, status, where,
 		 NIL /* Will be overwritten by timestamp */);
@@ -1976,7 +1976,7 @@ profile_runnable_proc(Process *p, Eterm status){
     hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL);
 
     enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, p->common.id, NIL, msg, bp);
-    erts_smp_mtx_unlock(&smq_mtx);
+    erts_mtx_unlock(&smq_mtx);
 }
 /* End system_profile tracing */
 
@@ -2027,7 +2027,7 @@ enqueue_sys_msg_unlocked(enum ErtsSysMsgType type,
 	sys_message_queue = smqp;
     }
     sys_message_queue_end = smqp;
-    erts_smp_cnd_signal(&smq_cnd);
+    erts_cnd_signal(&smq_cnd);
 }
 
 static void
@@ -2037,9 +2037,9 @@ enqueue_sys_msg(enum ErtsSysMsgType type,
 		Eterm msg,
 		ErlHeapFragment *bp)
 {
-    erts_smp_mtx_lock(&smq_mtx);
+    erts_mtx_lock(&smq_mtx);
     enqueue_sys_msg_unlocked(type, from, to, msg, bp);
-    erts_smp_mtx_unlock(&smq_mtx);
+    erts_mtx_unlock(&smq_mtx);
 }
 
 void
@@ -2091,10 +2091,10 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver)
 	    && !erts_system_monitor_flags.busy_port
 	    && !erts_system_monitor_flags.busy_dist_port)
 	    break; /* Everything is disabled */
-	erts_smp_thr_progress_block();
+	erts_thr_progress_block();
 	if (system_monitor == receiver || receiver == NIL)
 	    erts_system_monitor_clear(NULL);
-	erts_smp_thr_progress_unblock();
+	erts_thr_progress_unblock();
 	break;
 	 case SYS_MSG_TYPE_SYSPROF:
 	if (receiver == NIL
@@ -2104,11 +2104,11 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver)
 	    && !erts_system_profile_flags.scheduler)
 		 break;
 	/* Block system to clear flags */
-	erts_smp_thr_progress_block();
+	erts_thr_progress_block();
 	if (system_profile == receiver || receiver == NIL) { 
 		erts_system_profile_clear(NULL);
 	}
-	erts_smp_thr_progress_unblock();
+	erts_thr_progress_unblock();
 	break;
     case SYS_MSG_TYPE_ERRLGR: {
 	char *no_elgger = "(no error logger present)";
@@ -2153,38 +2153,38 @@ static void
 sys_msg_dispatcher_wakeup(void *vwait_p)
 {
     int *wait_p = (int *) vwait_p;
-    erts_smp_mtx_lock(&smq_mtx);
+    erts_mtx_lock(&smq_mtx);
     *wait_p = 0;
-    erts_smp_cnd_signal(&smq_cnd);
-    erts_smp_mtx_unlock(&smq_mtx);
+    erts_cnd_signal(&smq_cnd);
+    erts_mtx_unlock(&smq_mtx);
 }
 
 static void
 sys_msg_dispatcher_prep_wait(void *vwait_p)
 {
     int *wait_p = (int *) vwait_p;
-    erts_smp_mtx_lock(&smq_mtx);
+    erts_mtx_lock(&smq_mtx);
     *wait_p = 1;
-    erts_smp_mtx_unlock(&smq_mtx);
+    erts_mtx_unlock(&smq_mtx);
 }
 
 static void
 sys_msg_dispatcher_fin_wait(void *vwait_p)
 {
     int *wait_p = (int *) vwait_p;
-    erts_smp_mtx_lock(&smq_mtx);
+    erts_mtx_lock(&smq_mtx);
     *wait_p = 0;
-    erts_smp_mtx_unlock(&smq_mtx);
+    erts_mtx_unlock(&smq_mtx);
 }
 
 static void
 sys_msg_dispatcher_wait(void *vwait_p)
 {
     int *wait_p = (int *) vwait_p;
-    erts_smp_mtx_lock(&smq_mtx);
+    erts_mtx_lock(&smq_mtx);
     while (*wait_p)
-	erts_smp_cnd_wait(&smq_cnd, &smq_mtx);
-    erts_smp_mtx_unlock(&smq_mtx);
+	erts_cnd_wait(&smq_cnd, &smq_mtx);
+    erts_mtx_unlock(&smq_mtx);
 }
 
 static void *
@@ -2210,9 +2210,9 @@ sys_msg_dispatcher_func(void *unused)
 	int end_wait = 0;
 	ErtsSysMsgQ *smqp;
 
-	ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
+	ERTS_LC_ASSERT(!erts_thr_progress_is_blocking());
 
-	erts_smp_mtx_lock(&smq_mtx);
+	erts_mtx_lock(&smq_mtx);
 
 	/* Free previously used queue ... */
 	while (local_sys_message_queue) {
@@ -2223,21 +2223,21 @@ sys_msg_dispatcher_func(void *unused)
 
 	/* Fetch current trace message queue ... */
 	if (!sys_message_queue) {
-	    erts_smp_mtx_unlock(&smq_mtx);
+	    erts_mtx_unlock(&smq_mtx);
 	    end_wait = 1;
 	    erts_thr_progress_active(NULL, 0);
 	    erts_thr_progress_prepare_wait(NULL);
-	    erts_smp_mtx_lock(&smq_mtx);
+	    erts_mtx_lock(&smq_mtx);
 	}
 
 	while (!sys_message_queue)
-	    erts_smp_cnd_wait(&smq_cnd, &smq_mtx);
+	    erts_cnd_wait(&smq_cnd, &smq_mtx);
 
 	local_sys_message_queue = sys_message_queue;
 	sys_message_queue = NULL;
 	sys_message_queue_end = NULL;
 
-	erts_smp_mtx_unlock(&smq_mtx);
+	erts_mtx_unlock(&smq_mtx);
 
 	if (end_wait) {
 	    erts_thr_progress_finalize_wait(NULL);
@@ -2311,7 +2311,7 @@ sys_msg_dispatcher_func(void *unused)
 #ifdef DEBUG_PRINTOUTS
 		    erts_fprintf(stderr, "delivered\n");
 #endif
-		    erts_smp_proc_unlock(proc, proc_locks);
+		    erts_proc_unlock(proc, proc_locks);
 		}
 	    }
 	    else if (receiver == am_error_logger) {
@@ -2349,7 +2349,7 @@ sys_msg_dispatcher_func(void *unused)
 		sys_msg_disp_failure(smqp, receiver);
 	    drop_sys_msg:
 		if (proc)
-		    erts_smp_proc_unlock(proc, proc_locks);
+		    erts_proc_unlock(proc, proc_locks);
 		if (smqp->bp)
 		    free_message_buffer(smqp->bp);
 #ifdef DEBUG_PRINTOUTS
@@ -2369,7 +2369,7 @@ erts_foreach_sys_msg_in_q(void (*func)(Eterm,
 				       ErlHeapFragment *))
 {
     ErtsSysMsgQ *sm;
-    erts_smp_mtx_lock(&smq_mtx);
+    erts_mtx_lock(&smq_mtx);
     for (sm = sys_message_queue; sm; sm = sm->next) {
 	Eterm to;
 	switch (sm->type) {
@@ -2388,23 +2388,23 @@ erts_foreach_sys_msg_in_q(void (*func)(Eterm,
 	}
 	(*func)(sm->from, to, sm->msg, sm->bp);
     }
-    erts_smp_mtx_unlock(&smq_mtx);
+    erts_mtx_unlock(&smq_mtx);
 }
 
 
 static void
 init_sys_msg_dispatcher(void)
 {
-    erts_smp_thr_opts_t thr_opts = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER;
     thr_opts.detached = 1;
     thr_opts.name = "sys_msg_dispatcher";
     init_smq_element_alloc();
     sys_message_queue = NULL;
     sys_message_queue_end = NULL;
-    erts_smp_cnd_init(&smq_cnd);
-    erts_smp_mtx_init(&smq_mtx, "sys_msg_q", NIL,
+    erts_cnd_init(&smq_cnd);
+    erts_mtx_init(&smq_mtx, "sys_msg_q", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
-    erts_smp_thr_create(&sys_msg_dispatcher_tid,
+    erts_thr_create(&sys_msg_dispatcher_tid,
 			sys_msg_dispatcher_func,
 			NULL,
 			&thr_opts);
@@ -2505,7 +2505,7 @@ static void init_tracer_template(ErtsTracerNif *tnif) {
 }
 
 static Hash *tracer_hash = NULL;
-static erts_smp_rwmtx_t tracer_mtx;
+static erts_rwmtx_t tracer_mtx;
 
 static ErtsTracerNif *
 load_tracer_nif(const ErtsTracer tracer)
@@ -2545,9 +2545,9 @@ load_tracer_nif(const ErtsTracer tracer)
         return NULL;
     }
 
-    erts_smp_rwmtx_rwlock(&tracer_mtx);
+    erts_rwmtx_rwlock(&tracer_mtx);
     tnif = hash_put(tracer_hash, &tnif_tmpl);
-    erts_smp_rwmtx_rwunlock(&tracer_mtx);
+    erts_rwmtx_rwunlock(&tracer_mtx);
 
     return tnif;
 }
@@ -2558,14 +2558,14 @@ lookup_tracer_nif(const ErtsTracer tracer)
     ErtsTracerNif tnif_tmpl;
     ErtsTracerNif *tnif;
     tnif_tmpl.module = ERTS_TRACER_MODULE(tracer);
-    erts_smp_rwmtx_rlock(&tracer_mtx);
+    erts_rwmtx_rlock(&tracer_mtx);
     if ((tnif = hash_get(tracer_hash, &tnif_tmpl)) == NULL) {
-        erts_smp_rwmtx_runlock(&tracer_mtx);
+        erts_rwmtx_runlock(&tracer_mtx);
         tnif = load_tracer_nif(tracer);
         ASSERT(!tnif || tnif->nif_mod);
         return tnif;
     }
-    erts_smp_rwmtx_runlock(&tracer_mtx);
+    erts_rwmtx_runlock(&tracer_mtx);
     ASSERT(tnif->nif_mod);
     return tnif;
 }
@@ -2710,10 +2710,10 @@ send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p,
     }
     if (is_internal_pid(t_p->id)) {
         /* We have to have at least one lock */
-        ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((Process*)t_p) & ERTS_PROC_LOCKS_ALL);
+        ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks((Process*)t_p) & ERTS_PROC_LOCKS_ALL);
     } else {
         ASSERT(is_internal_port(t_p->id));
-        ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked((Port*)t_p));
+        ERTS_LC_ASSERT(erts_lc_is_port_locked((Port*)t_p));
     }
 #endif
 
@@ -2758,15 +2758,15 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks,
 
 #if defined(ERTS_ENABLE_LOCK_CHECK)
     if (c_p)
-        ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == c_p_locks
+        ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == c_p_locks
                            || erts_thr_progress_is_blocking());
     if (is_internal_pid(t_p->id)) {
         /* We have to have at least one lock */
-        ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((Process*)t_p) & ERTS_PROC_LOCKS_ALL
+        ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks((Process*)t_p) & ERTS_PROC_LOCKS_ALL
                            || erts_thr_progress_is_blocking());
     } else {
         ASSERT(is_internal_port(t_p->id));
-        ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked((Port*)t_p)
+        ERTS_LC_ASSERT(erts_lc_is_port_locked((Port*)t_p)
                            || erts_thr_progress_is_blocking());
     }
 #endif
@@ -2788,12 +2788,12 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks,
     if (is_internal_port(t_p->id) || (c_p && c_p->common.id == t_p->id)) {
         ErtsProcLocks c_p_xlocks = 0;
         if (is_internal_pid(t_p->id)) {
-            ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN);
+            ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN);
             if (c_p_locks != ERTS_PROC_LOCKS_ALL) {
                 c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL;
-                if (erts_smp_proc_trylock(c_p, c_p_xlocks) == EBUSY) {
-                    erts_smp_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN);
-                    erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+                if (erts_proc_trylock(c_p, c_p_xlocks) == EBUSY) {
+                    erts_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN);
+                    erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
                 }
             }
         }
@@ -2801,7 +2801,7 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks,
         t_p->trace_flags &= ~TRACEE_FLAGS;
 
         if (c_p_xlocks)
-            erts_smp_proc_unlock(c_p, c_p_xlocks);
+            erts_proc_unlock(c_p, c_p_xlocks);
     }
 
     return 0;
@@ -2959,11 +2959,11 @@ erts_tracer_update(ErtsTracer *tracer, const ErtsTracer new_tracer)
 
 static void init_tracer_nif()
 {
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
-    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
-    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+    erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_RWMTX_LONG_LIVED;
 
-    erts_smp_rwmtx_init_opt(&tracer_mtx, &rwmtx_opt, "tracer_mtx", NIL,
+    erts_rwmtx_init_opt(&tracer_mtx, &rwmtx_opt, "tracer_mtx", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
 
     erts_tracer_nif_clear();
@@ -2973,7 +2973,7 @@ static void init_tracer_nif()
 int erts_tracer_nif_clear()
 {
 
-    erts_smp_rwmtx_rlock(&tracer_mtx);
+    erts_rwmtx_rlock(&tracer_mtx);
     if (!tracer_hash || tracer_hash->nobjs) {
 
         HashFunctions hf;
@@ -2985,19 +2985,19 @@ int erts_tracer_nif_clear()
         hf.meta_free = (HMFREE_FUN) erts_free;
         hf.meta_print = (HMPRINT_FUN) erts_print;
 
-        erts_smp_rwmtx_runlock(&tracer_mtx);
-        erts_smp_rwmtx_rwlock(&tracer_mtx);
+        erts_rwmtx_runlock(&tracer_mtx);
+        erts_rwmtx_rwlock(&tracer_mtx);
 
         if (tracer_hash)
             hash_delete(tracer_hash);
 
         tracer_hash = hash_new(ERTS_ALC_T_TRACER_NIF, "tracer_hash", 10, hf);
 
-        erts_smp_rwmtx_rwunlock(&tracer_mtx);
+        erts_rwmtx_rwunlock(&tracer_mtx);
         return 1;
     }
 
-    erts_smp_rwmtx_runlock(&tracer_mtx);
+    erts_rwmtx_runlock(&tracer_mtx);
     return 0;
 }
 
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 7bbd93713d..dbf7ebd2a1 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -148,7 +148,7 @@ erts_bif_trace_epilogue(Process *p, Eterm result, int applying,
 			ErtsTracer meta_tracer);
 
 void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp);
-#define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP)				\
+#define ERTS_CHK_PEND_TRACE_MSGS(ESDP)				\
 do {									\
     if ((ESDP)->pending_trace_msgs)					\
 	erts_send_pending_trace_msgs((ESDP));				\
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index b1ba47df7b..44d8c85867 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -22,15 +22,11 @@
 #define ERL_UTILS_H__
 
 #include "sys.h"
-#include "erl_smp.h"
 #include "erl_printf.h"
 
 struct process;
 
 typedef struct {
-#ifdef DEBUG
-    int smp_api;
-#endif
     union {
 	Uint64 not_atomic;
 	erts_atomic64_t atomic;
@@ -38,62 +34,25 @@ typedef struct {
 } erts_interval_t;
 
 void erts_interval_init(erts_interval_t *);
-void erts_smp_interval_init(erts_interval_t *);
 Uint64 erts_step_interval_nob(erts_interval_t *);
 Uint64 erts_step_interval_relb(erts_interval_t *);
-Uint64 erts_smp_step_interval_nob(erts_interval_t *);
-Uint64 erts_smp_step_interval_relb(erts_interval_t *);
 Uint64 erts_ensure_later_interval_nob(erts_interval_t *, Uint64);
 Uint64 erts_ensure_later_interval_acqb(erts_interval_t *, Uint64);
-Uint64 erts_smp_ensure_later_interval_nob(erts_interval_t *, Uint64);
-Uint64 erts_smp_ensure_later_interval_acqb(erts_interval_t *, Uint64);
-ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *);
-ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *);
 ERTS_GLB_INLINE Uint64 erts_current_interval_nob(erts_interval_t *);
 ERTS_GLB_INLINE Uint64 erts_current_interval_acqb(erts_interval_t *);
-ERTS_GLB_INLINE Uint64 erts_smp_current_interval_nob(erts_interval_t *);
-ERTS_GLB_INLINE Uint64 erts_smp_current_interval_acqb(erts_interval_t *);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-ERTS_GLB_INLINE Uint64
-erts_current_interval_nob__(erts_interval_t *icp)
-{
-    return (Uint64) erts_atomic64_read_nob(&icp->counter.atomic);
-}
-
-ERTS_GLB_INLINE Uint64
-erts_current_interval_acqb__(erts_interval_t *icp)
-{
-    return (Uint64) erts_atomic64_read_acqb(&icp->counter.atomic);
-}
-
 ERTS_GLB_INLINE Uint64
 erts_current_interval_nob(erts_interval_t *icp)
 {
-    ASSERT(!icp->smp_api);
-    return erts_current_interval_nob__(icp);
+    return (Uint64) erts_atomic64_read_nob(&icp->counter.atomic);
 }
 
 ERTS_GLB_INLINE Uint64
 erts_current_interval_acqb(erts_interval_t *icp)
 {
-    ASSERT(!icp->smp_api);
-    return erts_current_interval_acqb__(icp);
-}
-
-ERTS_GLB_INLINE Uint64
-erts_smp_current_interval_nob(erts_interval_t *icp)
-{
-    ASSERT(icp->smp_api);
-    return erts_current_interval_nob__(icp);
-}
-
-ERTS_GLB_INLINE Uint64
-erts_smp_current_interval_acqb(erts_interval_t *icp)
-{
-    ASSERT(icp->smp_api);
-    return erts_current_interval_acqb__(icp);
+    return (Uint64) erts_atomic64_read_acqb(&icp->counter.atomic);
 }
 
 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 0b8d78c469..42082f9c3e 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -55,7 +55,7 @@
 #define CP_SIZE 1
 
 #define ErtsHAllocLockCheck(P) \
-  ERTS_SMP_LC_ASSERT(erts_dbg_check_halloc_lock((P)))
+  ERTS_LC_ASSERT(erts_dbg_check_halloc_lock((P)))
 
 
 #ifdef DEBUG
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 2524ad40cb..c81503f722 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -41,14 +41,12 @@
 
 static IndexTable export_tables[ERTS_NUM_CODE_IX];  /* Active not locked */
 
-static erts_smp_atomic_t total_entries_bytes;
-
-#include "erl_smp.h"
+static erts_atomic_t total_entries_bytes;
 
 /* This lock protects the staging export table from concurrent access
  * AND it protects the staging table from becoming active.
  */
-erts_smp_mtx_t export_staging_lock;
+erts_mtx_t export_staging_lock;
 
 extern BeamInstr* em_call_error_handler;
 extern BeamInstr* em_call_traced_function;
@@ -125,7 +123,7 @@ export_alloc(struct export_entry* tmpl_e)
 	Export* obj;
 
 	blob = (struct export_blob*) erts_alloc(ERTS_ALC_T_EXPORT, sizeof(*blob));
-	erts_smp_atomic_add_nob(&total_entries_bytes, sizeof(*blob));
+	erts_atomic_add_nob(&total_entries_bytes, sizeof(*blob));
 	obj = &blob->exp;
 	obj->info.op =  0;
 	obj->info.u.gen_bp = NULL;
@@ -169,7 +167,7 @@ export_free(struct export_entry* obj)
     }
     DBG_TRACE_MFA_P(&blob->exp.info.mfa, "export blob deallocation at %p", &blob->exp);
     erts_free(ERTS_ALC_T_EXPORT, blob);
-    erts_smp_atomic_add_nob(&total_entries_bytes, -sizeof(*blob));
+    erts_atomic_add_nob(&total_entries_bytes, -sizeof(*blob));
 }
 
 void
@@ -178,9 +176,9 @@ init_export_table(void)
     HashFunctions f;
     int i;
 
-    erts_smp_mtx_init(&export_staging_lock, "export_tab", NIL,
+    erts_mtx_init(&export_staging_lock, "export_tab", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
-    erts_smp_atomic_init_nob(&total_entries_bytes, 0);
+    erts_atomic_init_nob(&total_entries_bytes, 0);
 
     f.hash = (H_FUN) export_hash;
     f.cmp  = (HCMP_FUN) export_cmp;
@@ -369,7 +367,7 @@ int export_table_sz(void)
 }
 int export_entries_sz(void)
 {
-    return erts_smp_atomic_read_nob(&total_entries_bytes);
+    return erts_atomic_read_nob(&total_entries_bytes);
 }
 Export *export_get(Export *e)
 {
diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h
index 7c812b306c..be6cce07bf 100644
--- a/erts/emulator/beam/export.h
+++ b/erts/emulator/beam/export.h
@@ -66,9 +66,9 @@ Export *export_get(Export*);
 void export_start_staging(void);
 void export_end_staging(int commit);
 
-extern erts_smp_mtx_t export_staging_lock;
-#define export_staging_lock()	erts_smp_mtx_lock(&export_staging_lock)
-#define export_staging_unlock()	erts_smp_mtx_unlock(&export_staging_lock)
+extern erts_mtx_t export_staging_lock;
+#define export_staging_lock()	erts_mtx_lock(&export_staging_lock)
+#define export_staging_unlock()	erts_mtx_unlock(&export_staging_lock)
 
 #include "beam_load.h" /* For em_* extern declarations */ 
 #define ExportIsBuiltIn(EntryPtr) 			\
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 1560844521..0874be7250 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -616,7 +616,7 @@ erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize)
     sys_memcpy((void *) ep, (void *) edep, dist_ext_sz);
     ep += dist_ext_sz;
     if (new_edep->dep)
-	erts_smp_refc_inc(&new_edep->dep->refc, 1);
+	erts_refc_inc(&new_edep->dep->refc, 1);
     new_edep->extp = ep;
     new_edep->ext_endp = ep + ext_sz;
     new_edep->heap_size = -1;
@@ -669,12 +669,12 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
     edep->flags = 0;
     edep->dep = dep;
     if (dep) {
-	erts_smp_de_rlock(dep);
+	erts_de_rlock(dep);
 	if (dep->flags & DFLAG_DIST_HDR_ATOM_CACHE)
 	    edep->flags |= ERTS_DIST_EXT_DFLAG_HDR;
 	    
 	edep->flags |= (dep->connection_id & ERTS_DIST_EXT_CON_ID_MASK);
-	erts_smp_de_runlock(dep);
+	erts_de_runlock(dep);
     }
 
     if (ep[1] != DIST_HEADER) {
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 97054b2ee4..7cb94ba3d2 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -85,7 +85,7 @@ struct enif_resource_type_t
 
 typedef struct
 {
-    erts_smp_mtx_t lock;
+    erts_mtx_t lock;
     ErtsMonitor* root;
     int pending_failed_fire;
     int is_dying;
@@ -183,9 +183,9 @@ typedef struct {
     void         *handle;             /* Handle for DLL or SO (for dyn. drivers). */
     DE_ProcEntry *procs;              /* List of pids that have loaded this driver,
 				         or that wait for it to change state */
-    erts_smp_refc_t  refc;                /* Number of ports/processes having
+    erts_refc_t  refc;                /* Number of ports/processes having
 					 references to the driver */
-    erts_smp_atomic32_t port_count;   /* Number of ports using the driver */
+    erts_atomic32_t port_count;   /* Number of ports using the driver */
     Uint         flags;               /* ERL_DE_FL_KILL_PORTS */
     int          status;              /* ERL_DE_xxx */
     char         *full_path;          /* Full path of the driver */
@@ -209,7 +209,7 @@ struct erts_driver_t_ {
     } version;
     int flags;
     DE_Handle *handle;
-    erts_smp_mtx_t *lock;
+    erts_mtx_t *lock;
     ErlDrvEntry *entry;
     ErlDrvData (*start)(ErlDrvPort port, char *command, SysDriverOpts* opts);
     void (*stop)(ErlDrvData drv_data);
@@ -236,7 +236,7 @@ struct erts_driver_t_ {
 };
 
 extern erts_driver_t *driver_list;
-extern erts_smp_rwmtx_t erts_driver_list_lock;
+extern erts_rwmtx_t erts_driver_list_lock;
 
 extern void erts_ddll_init(void);
 extern void erts_ddll_lock_driver(DE_Handle *dh, char *name);
@@ -297,7 +297,7 @@ extern Eterm node_cookie;
 extern Uint display_items;	/* no of items to display in traces etc */
 
 extern int erts_backtrace_depth;
-extern erts_smp_atomic32_t erts_max_gen_gcs;
+extern erts_atomic32_t erts_max_gen_gcs;
 
 extern int bif_reductions;      /* reductions + fcalls (when doing call_bif) */
 extern int stackdump_on_exit;
@@ -907,9 +907,9 @@ typedef struct ErtsLiteralArea_ {
 #define ERTS_LITERAL_AREA_ALLOC_SIZE(N) \
     (sizeof(ErtsLiteralArea) + sizeof(Eterm)*((N) - 1))
 
-extern erts_smp_atomic_t erts_copy_literal_area__;
+extern erts_atomic_t erts_copy_literal_area__;
 #define ERTS_COPY_LITERAL_AREA()					\
-    ((ErtsLiteralArea *) erts_smp_atomic_read_nob(&erts_copy_literal_area__))
+    ((ErtsLiteralArea *) erts_atomic_read_nob(&erts_copy_literal_area__))
 extern Process *erts_literal_area_collector;
 #ifdef ERTS_DIRTY_SCHEDULERS
 extern Process *erts_dirty_process_code_checker;
diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c
index a1f6f54543..93d1111904 100644
--- a/erts/emulator/beam/index.c
+++ b/erts/emulator/beam/index.c
@@ -98,7 +98,7 @@ index_put_entry(IndexTable* t, void* tmpl)
      * Do a write barrier here to allow readers to do lock free iteration.
      * erts_index_num_entries() does matching read barrier.
      */
-    ERTS_SMP_WRITE_MEMORY_BARRIER;
+    ERTS_THR_WRITE_MEMORY_BARRIER;
     t->entries++;
 
     return p;
diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h
index 6c07571df6..30bc6a1121 100644
--- a/erts/emulator/beam/index.h
+++ b/erts/emulator/beam/index.h
@@ -88,7 +88,7 @@ ERTS_GLB_INLINE int erts_index_num_entries(IndexTable* t)
      * on tables where entries are never erased.
      * index_put_entry() does matching write barrier.
      */
-    ERTS_SMP_READ_MEMORY_BARRIER;
+    ERTS_THR_READ_MEMORY_BARRIER;
     return ret;
 }
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index cc7c717e6d..c8925e159e 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -62,10 +62,10 @@ extern ErlDrvEntry forker_driver_entry;
 extern ErlDrvEntry *driver_tab[]; /* table of static drivers, only used during initialization */
 
 erts_driver_t *driver_list; /* List of all drivers, static and dynamic. */
-erts_smp_rwmtx_t erts_driver_list_lock; /* Mutex for driver list */
-static erts_smp_tsd_key_t driver_list_lock_status_key; /*stop recursive locks when calling 
+erts_rwmtx_t erts_driver_list_lock; /* Mutex for driver list */
+static erts_tsd_key_t driver_list_lock_status_key; /*stop recursive locks when calling
 							 driver init */
-static erts_smp_tsd_key_t driver_list_last_error_key;  /* Save last DDLL error on a 
+static erts_tsd_key_t driver_list_last_error_key;  /* Save last DDLL error on a
 							  per thread basis (for BC interfaces) */
 
 ErtsPTab erts_port erts_align_attribute(ERTS_CACHE_LINE_SIZE); /* The port table */
@@ -115,7 +115,7 @@ static ERTS_INLINE int
 is_port_ioq_empty(Port *pp)
 {
     int res;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(pp));
     if (!pp->port_data_lock)
 	res = (pp->ioq.size == 0);
     else {
@@ -137,7 +137,7 @@ Uint
 erts_port_ioq_size(Port *pp)
 {
     int res;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(pp));
     if (!pp->port_data_lock)
 	res = pp->ioq.size;
     else {
@@ -205,7 +205,7 @@ dtrace_drvport_str(ErlDrvPort drvport, char *port_buf)
 static ERTS_INLINE void
 kill_port(Port *pp)
 {
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(pp));
     ERTS_TRACER_CLEAR(&ERTS_TRACER(pp));
     erts_ptab_delete_element(&erts_port, &pp->common); /* Time of death */
     erts_port_task_free_port(pp);
@@ -219,8 +219,8 @@ erts_lc_is_port_locked(Port *prt)
 {
     if (!prt)
 	return 0;
-    ERTS_SMP_LC_ASSERT(prt->lock);
-    return erts_smp_lc_mtx_is_locked(prt->lock);
+    ERTS_LC_ASSERT(prt->lock);
+    return erts_lc_mtx_is_locked(prt->lock);
 }
 #endif
 
@@ -350,7 +350,7 @@ static Port *create_port(char *name,
         runq = erts_get_runq_current(NULL);
     else
         runq = ERTS_RUNQ_IX(0);
-    erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq);
+    erts_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq);
 
     prt->xports = NULL;
     
@@ -373,7 +373,7 @@ static Port *create_port(char *name,
     prt->common.u.alive.reg = NULL;
     ERTS_PTMR_INIT(prt);
     erts_port_task_handle_init(&prt->timeout_task);
-    erts_smp_atomic_init_nob(&prt->psd, (erts_aint_t) NULL);
+    erts_atomic_init_nob(&prt->psd, (erts_aint_t) NULL);
     prt->async_open_port = NULL;
     prt->drv_data = (SWord) 0;
     prt->os_pid = -1;
@@ -414,7 +414,7 @@ static Port *create_port(char *name,
 
     initq(prt);
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
     if (erts_port_schedule_all_ops)
 	x_pts_flgs |= ERTS_PTS_FLG_FORCE_SCHED;
@@ -423,7 +423,7 @@ static Port *create_port(char *name,
 	x_pts_flgs |= ERTS_PTS_FLG_PARALLELISM;
 
     if (x_pts_flgs)
-	erts_smp_atomic32_read_bor_nob(&prt->sched.flags, x_pts_flgs);
+	erts_atomic32_read_bor_nob(&prt->sched.flags, x_pts_flgs);
 
     erts_atomic32_set_relb(&prt->state, state);
     return prt;
@@ -521,7 +521,7 @@ erts_save_suspend_process_on_port(Port *prt, Process *process)
     int saved;
     erts_aint32_t flags;
     erts_port_task_sched_lock(&prt->sched);
-    flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
+    flags = erts_atomic32_read_nob(&prt->sched.flags);
     saved = (flags & ERTS_PTS_FLGS_BUSY) && !(flags & ERTS_PTS_FLG_EXIT);
     if (saved)
 	erts_proclist_store_last(&prt->suspended, erts_proclist_create(process));
@@ -565,16 +565,16 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
     erts_mtx_t *driver_lock = NULL;
     int cprt_flgs = 0;
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
 
-    erts_smp_rwmtx_rlock(&erts_driver_list_lock);
+    erts_rwmtx_rlock(&erts_driver_list_lock);
     if (!driver) {
 	for (driver = driver_list; driver; driver = driver->next) {
 	    if (sys_strcmp(driver->name, name) == 0)
 		break;
 	}
 	if (!driver) { 
-	    erts_smp_rwmtx_runlock(&erts_driver_list_lock);
+	    erts_rwmtx_runlock(&erts_driver_list_lock);
 	    ERTS_OPEN_DRIVER_RET(NULL, -3, BADARG);
 	}
     }
@@ -619,7 +619,7 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
     }
 
     if (driver == NULL || (driver != &spawn_driver && opts->exit_status)) {
-	erts_smp_rwmtx_runlock(&erts_driver_list_lock);
+	erts_rwmtx_runlock(&erts_driver_list_lock);
 	ERTS_OPEN_DRIVER_RET(NULL, -3, BADARG);
     }
 
@@ -629,7 +629,7 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 	erts_ddll_increment_port_count(driver->handle);
 	erts_ddll_reference_driver(driver->handle);
     }
-    erts_smp_rwmtx_runlock(&erts_driver_list_lock);
+    erts_rwmtx_runlock(&erts_driver_list_lock);
 
     /*
      * We'll set up the port before calling the start function,
@@ -642,9 +642,9 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
     port = create_port(name, driver, driver_lock, cprt_flgs, pid, &port_errno);
     if (!port) {
 	if (driver->handle) {
-	    erts_smp_rwmtx_rlock(&erts_driver_list_lock);
+	    erts_rwmtx_rlock(&erts_driver_list_lock);
 	    erts_ddll_decrement_port_count(driver->handle);
-	    erts_smp_rwmtx_runlock(&erts_driver_list_lock);
+	    erts_rwmtx_runlock(&erts_driver_list_lock);
 	    erts_ddll_dereference_driver(driver->handle);
 	}
 	if (port_errno)
@@ -729,9 +729,9 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 	    port->linebuf = NULL;
 	}
 	if (driver->handle != NULL) {
-	    erts_smp_rwmtx_rlock(&erts_driver_list_lock);
+	    erts_rwmtx_rlock(&erts_driver_list_lock);
 	    erts_ddll_decrement_port_count(driver->handle);
-	    erts_smp_rwmtx_runlock(&erts_driver_list_lock);
+	    erts_rwmtx_runlock(&erts_driver_list_lock);
 	}
 	kill_port(port);
 	erts_port_release(port);
@@ -770,7 +770,7 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
     Process *rp;
     erts_mtx_t *driver_lock = NULL;
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
 
     /* Need to be called from a scheduler thread */
     if (!erts_get_scheduler_id())
@@ -784,12 +784,12 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
     if (!rp)
 	return ERTS_INVALID_ERL_DRV_PORT;
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(creator_port));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(creator_port));
 
     driver = creator_port->drv_ptr;
-    erts_smp_rwmtx_rlock(&erts_driver_list_lock);
+    erts_rwmtx_rlock(&erts_driver_list_lock);
     if (!erts_ddll_driver_ok(driver->handle)) {
-	erts_smp_rwmtx_runlock(&erts_driver_list_lock);
+	erts_rwmtx_runlock(&erts_driver_list_lock);
 	return ERTS_INVALID_ERL_DRV_PORT;
     }
 
@@ -800,31 +800,31 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
 
     driver_lock = driver->lock;
 
-    erts_smp_rwmtx_runlock(&erts_driver_list_lock);
+    erts_rwmtx_runlock(&erts_driver_list_lock);
 
     /* Inherit parallelism flag from parent */
     if (ERTS_PTS_FLG_PARALLELISM &
-	erts_smp_atomic32_read_nob(&creator_port->sched.flags))
+	erts_atomic32_read_nob(&creator_port->sched.flags))
 	cprt_flgs |= ERTS_CREATE_PORT_FLAG_PARALLELISM;
     port = create_port(name, driver, driver_lock, cprt_flgs, pid, NULL);
     if (!port) {
 	if (driver->handle) {
-	    erts_smp_rwmtx_rlock(&erts_driver_list_lock);
+	    erts_rwmtx_rlock(&erts_driver_list_lock);
 	    erts_ddll_decrement_port_count(driver->handle);
-	    erts_smp_rwmtx_runlock(&erts_driver_list_lock);
+	    erts_rwmtx_runlock(&erts_driver_list_lock);
 	    erts_ddll_dereference_driver(driver->handle);
 	}
 	return ERTS_INVALID_ERL_DRV_PORT;
     }
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(port));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(port));
 
-    erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK);
+    erts_proc_lock(rp, ERTS_PROC_LOCK_LINK);
     if (ERTS_PROC_IS_EXITING(rp)) {
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 	if (driver->handle) {
-	    erts_smp_rwmtx_rlock(&erts_driver_list_lock);
+	    erts_rwmtx_rlock(&erts_driver_list_lock);
 	    erts_ddll_decrement_port_count(driver->handle);
-	    erts_smp_rwmtx_runlock(&erts_driver_list_lock);
+	    erts_rwmtx_runlock(&erts_driver_list_lock);
 	}
 	kill_port(port);
 	erts_port_release(port);
@@ -833,7 +833,7 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
 
     erts_add_link(&ERTS_P_LINKS(port), LINK_PID, pid);
     erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, port->common.id);
-    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 
     if (!driver_lock) {
 	ErtsXPortsList *xplp = xports_list_alloc();
@@ -1226,12 +1226,12 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp)
 	invalid_sched_flags |= ERTS_PTS_FLG_PARALLELISM;
 
     if (sp->pre_chk_sched_flags) {
-	sp->sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
+	sp->sched_flags = erts_atomic32_read_nob(&prt->sched.flags);
 	if (sp->sched_flags & invalid_sched_flags)
 	    return ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS;
     }
 
-    if (erts_smp_port_trylock(prt) == EBUSY)
+    if (erts_port_trylock(prt) == EBUSY)
 	return ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK;
 
     invalid_state = sp->state;
@@ -1245,7 +1245,7 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp)
     if (prof_runnable_ports)
 	erts_port_task_sched_lock(&prt->sched);
 
-    act = erts_smp_atomic32_read_nob(&prt->sched.flags);
+    act = erts_atomic32_read_nob(&prt->sched.flags);
 
     do {
 	erts_aint32_t new;
@@ -1257,7 +1257,7 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp)
 	}
 	exp = act;
 	new = act | ERTS_PTS_FLG_EXEC_IMM;
-	act = erts_smp_atomic32_cmpxchg_mb(&prt->sched.flags, new, exp);
+	act = erts_atomic32_cmpxchg_mb(&prt->sched.flags, new, exp);
     } while (act != exp);
     
     sp->sched_flags = act;
@@ -1279,14 +1279,14 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp)
 	    profile_runnable_proc(c_p, am_inactive);
 
 	reds_left_in = ERTS_BIF_REDS_LEFT(c_p);
-	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+	erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
     }
 
     ASSERT(0 <= reds_left_in && reds_left_in <= CONTEXT_REDS);
     sp->reds_left_in = reds_left_in;
     prt->reds = CONTEXT_REDS - reds_left_in;
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
 
     if (prof_runnable_ports | IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) {
 	if (prof_runnable_ports && !(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC)))
@@ -1324,9 +1324,9 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp)
     if (prof_runnable_ports)
 	erts_port_task_sched_lock(&prt->sched);
 
-    act = erts_smp_atomic32_read_band_mb(&prt->sched.flags,
+    act = erts_atomic32_read_band_mb(&prt->sched.flags,
 					 ~ERTS_PTS_FLG_EXEC_IMM);
-    ERTS_SMP_LC_ASSERT(act & ERTS_PTS_FLG_EXEC_IMM);
+    ERTS_LC_ASSERT(act & ERTS_PTS_FLG_EXEC_IMM);
 
     if (prof_runnable_ports | IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) {
 	if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS))
@@ -1341,7 +1341,7 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp)
     erts_port_release(prt);
 
     if (c_p) {
-	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+	erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 
 	if (reds != (CONTEXT_REDS - sp->reds_left_in)) {
 	    int bump_reds = reds - (CONTEXT_REDS - sp->reds_left_in);
@@ -1452,7 +1452,7 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg, Port* prt)
 				  prt);
 
 	if (rp_locks)
-	    erts_smp_proc_unlock(rp, rp_locks);
+	    erts_proc_unlock(rp, rp_locks);
     }
 }
 
@@ -1470,7 +1470,7 @@ erts_schedule_proc2port_signal(Process *c_p,
     int sched_res;
     if (!refp) {
 	if (c_p)
-	    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+	    erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
     }
     else {
 	ASSERT(c_p);
@@ -1491,20 +1491,20 @@ erts_schedule_proc2port_signal(Process *c_p,
 	 *       otherwise, next receive will *not* work
 	 *       as expected!
 	 */
-	erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 
 	if (ERTS_PROC_PENDING_EXIT(c_p)) {
 	    /* need to exit caller instead */
-	    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	    KILL_CATCHES(c_p);
 	    c_p->freason = EXC_EXIT;
 	    return ERTS_PORT_OP_CALLER_EXIT;
 	}
 
-	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
+	ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
 	c_p->msg.save = c_p->msg.last;
 
-        erts_smp_proc_unlock(c_p, (ERTS_PROC_LOCKS_MSG_RECEIVE
+        erts_proc_unlock(c_p, (ERTS_PROC_LOCKS_MSG_RECEIVE
                                    | ERTS_PROC_LOCK_MAIN));
     }
 
@@ -1520,7 +1520,7 @@ erts_schedule_proc2port_signal(Process *c_p,
 					task_flags);
 
     if (c_p)
-	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+	erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 
     if (sched_res != 0) {
 	if (refp) {
@@ -1531,9 +1531,9 @@ erts_schedule_proc2port_signal(Process *c_p,
 	     * containing the reference created above...
 	     */
 	    ASSERT(c_p);
-	    erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	    erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	    JOIN_MESSAGE(c_p);
-	    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	    *refp = NIL;
 	}
 	return ERTS_PORT_OP_DROPPED;
@@ -1566,14 +1566,14 @@ send_badsig(Port *prt) {
     ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
     Process* rp;
     Eterm connected = ERTS_PORT_GET_CONNECTED(prt);
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
     ERTS_LC_ASSERT(erts_get_scheduler_id());
 
     ASSERT(is_internal_pid(connected));
 
     rp = erts_proc_lookup_raw(connected);
     if (rp) {
-	erts_smp_proc_lock(rp, rp_locks);
+	erts_proc_lock(rp, rp_locks);
 	if (!ERTS_PROC_IS_EXITING(rp))
 	    (void) erts_send_exit_signal(NULL,
 					 prt->common.id,
@@ -1584,7 +1584,7 @@ send_badsig(Port *prt) {
 					 NULL,
 					 0);
 	if (rp_locks)
-	    erts_smp_proc_unlock(rp, rp_locks);
+	    erts_proc_unlock(rp, rp_locks);
     } /* exit sent */
 } /* send_badsig */
 
@@ -1707,7 +1707,7 @@ call_driver_outputv(int bang_op,
 	ErlDrvSizeT size = evp->size;
 	ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT);
 
-	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)	
+	ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)
 			   || ERTS_IS_CRASH_DUMPING);
 
 
@@ -1765,7 +1765,7 @@ port_sig_outputv(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *s
     case ERTS_PROC2PORT_SIG_EXEC:
 	/* Execution of a scheduled outputv() call */
 
-	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+	ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
 	if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP)
 	    reply = am_badarg;
@@ -1821,7 +1821,7 @@ call_driver_output(int bang_op,
     else {
 	ErtsSchedulerData *esdp = erts_get_scheduler_data();
         ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT);
-	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)	
+	ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)
 			   || ERTS_IS_CRASH_DUMPING);
 
 #ifdef USE_VM_PROBES
@@ -1872,7 +1872,7 @@ port_sig_output(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *si
     case ERTS_PROC2PORT_SIG_EXEC:
 	/* Execution of a scheduled output() call */
 
-	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+	ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
 	if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP)
 	    reply = am_badarg;
@@ -2116,7 +2116,7 @@ erts_port_output(Process *c_p,
      * Assumes caller have checked that port is valid...
      */
 
-    sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
+    sched_flags = erts_atomic32_read_nob(&prt->sched.flags);
     if (sched_flags & (busy_flgs|ERTS_PTS_FLG_EXIT))
 	return ((sched_flags & ERTS_PTS_FLG_EXIT)
 		? ERTS_PORT_OP_DROPPED
@@ -2495,7 +2495,7 @@ erts_port_output(Process *c_p,
     }
 
     if (!(flags & ERTS_PORT_SIG_FLG_FORCE)) {
-	sched_flags = erts_smp_atomic32_read_acqb(&prt->sched.flags);
+	sched_flags = erts_atomic32_read_acqb(&prt->sched.flags);
 	if (!(sched_flags & ERTS_PTS_FLG_BUSY_PORT)) {
 	    if (async_nosuspend)
 		erts_port_task_tmp_handle_detach(ns_pthp);
@@ -2720,9 +2720,9 @@ set_port_connected(int bang_op,
 	Process *rp = erts_proc_lookup_raw(connect);
 	if (!rp)
 	    return ERTS_PORT_OP_DROPPED;
-	erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK);
+	erts_proc_lock(rp, ERTS_PROC_LOCK_LINK);
 	if (ERTS_PROC_IS_EXITING(rp)) {
-	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+	    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 	    return ERTS_PORT_OP_DROPPED;
 	}
 
@@ -2734,7 +2734,7 @@ set_port_connected(int bang_op,
 
 	ERTS_PORT_SET_CONNECTED(prt, connect);
 
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 
         if (IS_TRACED_FL(prt, F_TRACE_PORTS))
             trace_port(prt, am_getting_linked, connect);
@@ -2927,7 +2927,7 @@ port_link_failure(Eterm port_id, Eterm linker)
 		    trace_proc(NULL, 0, rp, am_getting_unlinked, port_id);
 	    }
 	    if (rp_locks)
-		erts_smp_proc_unlock(rp, rp_locks);
+		erts_proc_unlock(rp, rp_locks);
 	}
     }
 }
@@ -3013,7 +3013,7 @@ port_monitor_failure(Eterm port_id, Eterm origin, Eterm ref_DOWN)
      * caller has never seen it yet. */
     erts_queue_monitor_message(origin_p, &p_locks, ref_DOWN,
                                am_port, port_id, am_noproc);
-    erts_smp_proc_unlock(origin_p, p_locks);
+    erts_proc_unlock(origin_p, p_locks);
 }
 
 /* Origin wants to monitor port Prt. State contains possible error, which has
@@ -3041,7 +3041,7 @@ port_monitor(Port *prt, erts_aint32_t state, Eterm origin,
         erts_add_monitor(&ERTS_P_MONITORS(prt), MON_TARGET, ref,
                          origin, name_or_nil);
 
-        erts_smp_proc_unlock(origin_p, p_locks);
+        erts_proc_unlock(origin_p, p_locks);
     } else {
 failure:
         port_monitor_failure(prt->common.id, origin, ref);
@@ -3132,7 +3132,7 @@ port_demonitor_failure(Eterm port_id, Eterm origin, Eterm ref)
         erts_destroy_monitor(mon1);
     }
 
-    erts_smp_proc_unlock(origin_p, rp_locks);
+    erts_proc_unlock(origin_p, rp_locks);
 }
 
 /* Origin wants to demonitor port Prt. State contains possible error, which has
@@ -3162,7 +3162,7 @@ port_demonitor(Port *port, erts_aint32_t state, Eterm origin, Eterm ref)
             }
         }
         if (origin_p) { /* when origin is dying, it won't be found */
-            erts_smp_proc_unlock(origin_p, p_locks);
+            erts_proc_unlock(origin_p, p_locks);
         }
     } else {
         port_demonitor_failure(port->common.id, origin, ref);
@@ -3249,10 +3249,10 @@ init_ack_send_reply(Port *port, Eterm resp)
 
     if (!is_internal_port(resp)) {
         Process *rp = erts_proc_lookup_raw(port->async_open_port->to);
-        erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK);
+        erts_proc_lock(rp, ERTS_PROC_LOCK_LINK);
         erts_remove_link(&ERTS_P_LINKS(port), port->async_open_port->to);
         erts_remove_link(&ERTS_P_LINKS(rp), port->common.id);
-        erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+        erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
     }
     port_sched_op_reply(port->async_open_port->to,
                         port->async_open_port->ref,
@@ -3316,9 +3316,9 @@ void erts_init_io(int port_tab_size,
 {
     ErlDrvEntry** dp;
     UWord common_element_size;
-    erts_smp_rwmtx_opt_t drv_list_rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
-    drv_list_rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
-    drv_list_rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+    erts_rwmtx_opt_t drv_list_rwmtx_opts = ERTS_RWMTX_OPT_DEFAULT_INITER;
+    drv_list_rwmtx_opts.type = ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
+    drv_list_rwmtx_opts.lived = ERTS_RWMTX_LONG_LIVED;
 
     erts_atomic64_init_nob(&bytes_in, 0);
     erts_atomic64_init_nob(&bytes_out, 0);
@@ -3343,12 +3343,12 @@ void erts_init_io(int port_tab_size,
     else if (port_tab_size < ERTS_MIN_PORTS)
 	port_tab_size = ERTS_MIN_PORTS;
 
-    erts_smp_rwmtx_init_opt(&erts_driver_list_lock, &drv_list_rwmtx_opts, "driver_list", NIL,
+    erts_rwmtx_init_opt(&erts_driver_list_lock, &drv_list_rwmtx_opts, "driver_list", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
     driver_list = NULL;
-    erts_smp_tsd_key_create(&driver_list_lock_status_key,
+    erts_tsd_key_create(&driver_list_lock_status_key,
 			    "erts_driver_list_lock_status_key");
-    erts_smp_tsd_key_create(&driver_list_last_error_key,
+    erts_tsd_key_create(&driver_list_last_error_key,
 			    "erts_driver_list_last_error_key");
 
     erts_ptab_init_table(&erts_port,
@@ -3363,8 +3363,8 @@ void erts_init_io(int port_tab_size,
 
     sys_init_io();
 
-    erts_smp_tsd_set(driver_list_lock_status_key, (void *) 1);
-    erts_smp_rwmtx_rwlock(&erts_driver_list_lock);
+    erts_tsd_set(driver_list_lock_status_key, (void *) 1);
+    erts_rwmtx_rwlock(&erts_driver_list_lock);
 
     init_driver(&fd_driver, &fd_driver_entry, NULL);
     init_driver(&vanilla_driver, &vanilla_driver_entry, NULL);
@@ -3376,8 +3376,8 @@ void erts_init_io(int port_tab_size,
     for (dp = driver_tab; *dp != NULL; dp++)
 	erts_add_driver_entry(*dp, NULL, 1);
 
-    erts_smp_tsd_set(driver_list_lock_status_key, NULL);
-    erts_smp_rwmtx_rwunlock(&erts_driver_list_lock);
+    erts_tsd_set(driver_list_lock_status_key, NULL);
+    erts_rwmtx_rwunlock(&erts_driver_list_lock);
 }
 
 #if defined(ERTS_ENABLE_LOCK_COUNT)
@@ -3649,7 +3649,7 @@ deliver_result(Port *prt, Eterm sender, Eterm pid, Eterm res)
     ErtsProcLocks rp_locks = 0;
     int scheduler = erts_get_scheduler_id() != 0;
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
 
     ASSERT(!prt || prt->common.id == sender);
     ERTS_LC_ASSERT(!prt || erts_lc_is_port_locked(prt));
@@ -3680,7 +3680,7 @@ deliver_result(Port *prt, Eterm sender, Eterm pid, Eterm res)
 	erts_queue_message(rp, rp_locks, mp, tuple, sender);
 
 	if (rp_locks)
-	    erts_smp_proc_unlock(rp, rp_locks);
+	    erts_proc_unlock(rp, rp_locks);
 	if (!scheduler)
 	    erts_proc_dec_refc(rp);
 
@@ -3711,8 +3711,8 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
     int scheduler = erts_get_scheduler_id() != 0;
     int trace_send = IS_TRACED_FL(prt, F_TRACE_SEND);
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));    
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_CHK_NO_PROC_LOCKS;
 
     need = 3 + 3 + 2*hlen;
 
@@ -3779,7 +3779,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
     ERL_MESSAGE_TOKEN(mp) = am_undefined;
     erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id);
     if (rp_locks)
-	erts_smp_proc_unlock(rp, rp_locks);
+	erts_proc_unlock(rp, rp_locks);
     if (!scheduler)
 	erts_proc_dec_refc(rp);
 }
@@ -3814,7 +3814,7 @@ static void flush_linebuf_messages(Port *prt, erts_aint32_t state)
     LineBufContext lc;
     int ret;
 
-    ERTS_SMP_LC_ASSERT(!prt || erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(!prt || erts_lc_is_port_locked(prt));
 
     if (!prt)
 	return;
@@ -3858,8 +3858,8 @@ deliver_vec_message(Port* prt,			/* Port */
     erts_aint32_t state;
     int trace_send = IS_TRACED_FL(prt, F_TRACE_SEND);
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_CHK_NO_PROC_LOCKS;
 
     /*
      * Check arguments for validity.
@@ -3950,7 +3950,7 @@ deliver_vec_message(Port* prt,			/* Port */
 
     ERL_MESSAGE_TOKEN(mp) = am_undefined;
     erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id);
-    erts_smp_proc_unlock(rp, rp_locks);
+    erts_proc_unlock(rp, rp_locks);
     if (!scheduler)
 	erts_proc_dec_refc(rp);
 }
@@ -3985,8 +3985,8 @@ static void flush_port(Port *p)
 {
     int fpe_was_unmasked;
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
+    ERTS_CHK_NO_PROC_LOCKS;
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(p));
 
     if (p->drv_ptr->flush != NULL) {
         ERTS_MSACC_PUSH_STATE_M();
@@ -4038,8 +4038,8 @@ terminate_port(Port *prt)
     erts_aint32_t state;
     ErtsPrtSD *psd;
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_CHK_NO_PROC_LOCKS;
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
     ASSERT(!ERTS_P_LINKS(prt));
     ASSERT(!ERTS_P_MONITORS(prt));
@@ -4091,9 +4091,9 @@ terminate_port(Port *prt)
         trace_port_send(prt, connected_id, am_closed, 1);
 
     if(drv->handle != NULL) {
-	erts_smp_rwmtx_rlock(&erts_driver_list_lock);
+	erts_rwmtx_rlock(&erts_driver_list_lock);
 	erts_ddll_decrement_port_count(drv->handle); 
-	erts_smp_rwmtx_runlock(&erts_driver_list_lock);
+	erts_rwmtx_runlock(&erts_driver_list_lock);
     }
     stopq(prt);        /* clear queue memory */
     if(prt->linebuf != NULL){
@@ -4103,7 +4103,7 @@ terminate_port(Port *prt)
 
     erts_cleanup_port_data(prt);
 
-    psd = (ErtsPrtSD *) erts_smp_atomic_read_nob(&prt->psd);
+    psd = (ErtsPrtSD *) erts_atomic_read_nob(&prt->psd);
     if (psd)
 	erts_free(ERTS_ALC_T_PRTSD, psd);
 
@@ -4116,7 +4116,7 @@ terminate_port(Port *prt)
      * port has been removed from the port table (in kill_port()).
      */
     if ((state & ERTS_PORT_SFLG_HALT)
-	&& (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0)) {
+	&& (erts_atomic32_dec_read_nob(&erts_halt_progress) == 0)) {
 	erts_port_release(prt); /* We will exit and never return */
 	erts_flush_async_exit(erts_halt_code, "");
     }
@@ -4144,7 +4144,7 @@ static void sweep_one_monitor(ErtsMonitor *mon, void *vpsc)
             goto done;
         }
         rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref);
-        erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+        erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
         if (rmon == NULL) {
             goto done;
         }
@@ -4218,7 +4218,7 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc)
 						 0);
 		if (xres >= 0) {
                     if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) {
-                        erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND);
+                        erts_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND);
                         rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND;
                     }
 		    /* We didn't exit the process and it is traced */
@@ -4228,7 +4228,7 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc)
 		erts_destroy_link(rlnk);
 	    }
 
-	    erts_smp_proc_unlock(rp, rp_locks);
+	    erts_proc_unlock(rp, rp_locks);
 	}
     }
     erts_destroy_link(lnk);
@@ -4263,7 +4263,7 @@ port_fire_one_monitor(ErtsMonitor *mon, void *ctx0)
         UnUseTmpHeapNoproc(3);
 
         rmon = erts_remove_monitor(&ERTS_P_MONITORS(origin), mon->ref);
-        erts_smp_proc_unlock(origin, origin_locks);
+        erts_proc_unlock(origin, origin_locks);
 
         if (rmon) {
             erts_destroy_monitor(rmon);
@@ -4289,8 +4289,8 @@ erts_deliver_port_exit(Port *prt, Eterm from, Eterm reason, int send_closed,
    Eterm modified_reason;
    erts_aint32_t state, set_state_flags;
 
-   ERTS_SMP_CHK_NO_PROC_LOCKS;
-   ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+   ERTS_CHK_NO_PROC_LOCKS;
+   ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
    modified_reason = (reason == am_kill) ? am_killed : reason;
 
@@ -4651,7 +4651,7 @@ port_sig_control(Port *prt,
 				      prt);
 
 	    if (rp_locks)
-		erts_smp_proc_unlock(rp, rp_locks);
+		erts_proc_unlock(rp, rp_locks);
 	    goto done;
 	}
     }
@@ -4704,7 +4704,7 @@ erts_port_control(Process* c_p,
     int copy;
     ErtsProc2PortSigData *sigdp;
 
-    sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
+    sched_flags = erts_atomic32_read_nob(&prt->sched.flags);
     if (sched_flags & ERTS_PTS_FLG_EXIT)
 	return ERTS_PORT_OP_BADARG;
 
@@ -5016,11 +5016,11 @@ port_sig_call(Port *prt,
 					      prt);
 
 		    if (rp_locks)
-			erts_smp_proc_unlock(rp, rp_locks);
+			erts_proc_unlock(rp, rp_locks);
 		    goto done;
 		}
 		if (rp_locks)
-		    erts_smp_proc_unlock(rp, rp_locks);
+		    erts_proc_unlock(rp, rp_locks);
 	    }
 	}
     }
@@ -5054,7 +5054,7 @@ erts_port_call(Process* c_p,
     erts_aint32_t sched_flags;
     ErtsProc2PortSigData *sigdp;
 
-    sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
+    sched_flags = erts_atomic32_read_nob(&prt->sched.flags);
     if (sched_flags & ERTS_PTS_FLG_EXIT) {
 	return ERTS_PORT_OP_BADARG;
     }
@@ -5272,7 +5272,7 @@ port_sig_info(Port *prt,
 				      prt);
 	}
 	if (rp_locks)
-	    erts_smp_proc_unlock(rp, rp_locks);
+	    erts_proc_unlock(rp, rp_locks);
     }
     return ERTS_PORT_REDS_INFO;
 }
@@ -5341,7 +5341,7 @@ typedef struct {
     Uint sched_id;
     Eterm pid;
     Uint32 refn[ERTS_REF_NUMBERS];
-    erts_smp_atomic32_t refc;
+    erts_atomic32_t refc;
 } ErtsIOBytesReq;
 
 static void
@@ -5391,10 +5391,10 @@ reply_io_bytes(void *vreq)
 	if (req->sched_id == sched_id)
 	    rp_locks &= ~ERTS_PROC_LOCK_MAIN;
 	if (rp_locks)
-	    erts_smp_proc_unlock(rp, rp_locks);
+	    erts_proc_unlock(rp, rp_locks);
     }
 
-    if (erts_smp_atomic32_dec_read_nob(&req->refc) == 0)
+    if (erts_atomic32_dec_read_nob(&req->refc) == 0)
 	erts_free(ERTS_ALC_T_IOB_REQ, req);
 }
 
@@ -5417,7 +5417,7 @@ erts_request_io_bytes(Process *c_p)
     req->refn[0] = refn[0];
     req->refn[1] = refn[1];
     req->refn[2] = refn[2];
-    erts_smp_atomic32_init_nob(&req->refc,
+    erts_atomic32_init_nob(&req->refc,
 			       (erts_aint32_t) erts_no_schedulers);
 
     if (erts_no_schedulers > 1)
@@ -5509,14 +5509,14 @@ set_busy_port(ErlDrvPort dprt, int on)
     DTRACE_CHARBUF(port_str, 16);
 #endif
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
 
     prt = erts_drvport2port(dprt);
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return;
 
     if (on) {
-	flags = erts_smp_atomic32_read_bor_acqb(&prt->sched.flags,
+	flags = erts_atomic32_read_bor_acqb(&prt->sched.flags,
 						ERTS_PTS_FLG_BUSY_PORT);
 	if (flags & ERTS_PTS_FLG_BUSY_PORT)
 	    return; /* Already busy */
@@ -5532,7 +5532,7 @@ set_busy_port(ErlDrvPort dprt, int on)
         }
 #endif
     } else {
-	flags = erts_smp_atomic32_read_band_acqb(&prt->sched.flags,
+	flags = erts_atomic32_read_band_acqb(&prt->sched.flags,
 						 ~ERTS_PTS_FLG_BUSY_PORT);
 	if (!(flags & ERTS_PTS_FLG_BUSY_PORT))
 	    return; /* Already non-busy */
@@ -5626,7 +5626,7 @@ int get_port_flags(ErlDrvPort ix)
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return 0;
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
     flags = 0;
     if (state & ERTS_PORT_SFLG_BINARY_IO)
@@ -5642,8 +5642,8 @@ void erts_raw_port_command(Port* p, byte* buf, Uint len)
     int fpe_was_unmasked;
     ERTS_MSACC_PUSH_STATE_M();
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
+    ERTS_CHK_NO_PROC_LOCKS;
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(p));
 
     if (len > (Uint) INT_MAX)
 	erts_exit(ERTS_ABORT_EXIT,
@@ -5672,10 +5672,10 @@ int async_ready(Port *p, void* data)
 {
     int need_free = 1;
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
 
     if (p) {
-	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
+	ERTS_LC_ASSERT(erts_lc_is_port_locked(p));
 	if (p->drv_ptr->ready_async != NULL) {
 	    ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT);
 #ifdef USE_VM_PROBES
@@ -5860,8 +5860,8 @@ void driver_report_exit(ErlDrvPort ix, int status)
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
        return;
 
-   ERTS_SMP_CHK_NO_PROC_LOCKS;
-   ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+   ERTS_CHK_NO_PROC_LOCKS;
+   ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
    pid = ERTS_PORT_GET_CONNECTED(prt);
    ASSERT(is_internal_pid(pid));
@@ -5884,7 +5884,7 @@ void driver_report_exit(ErlDrvPort ix, int status)
    ERL_MESSAGE_TOKEN(mp) = am_undefined;
    erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id);
 
-   erts_smp_proc_unlock(rp, rp_locks);
+   erts_proc_unlock(rp, rp_locks);
    if (!scheduler)
        erts_proc_dec_refc(rp);
 }
@@ -6524,7 +6524,7 @@ driver_deliver_term(Port *prt, Eterm to, ErlDrvTermData* data, int len)
     }
     if (rp) {
 	if (rp_locks)
-	    erts_smp_proc_unlock(rp, rp_locks);
+	    erts_proc_unlock(rp, rp_locks);
 	if (!scheduler)
 	    erts_proc_dec_refc(rp);
     }
@@ -6565,12 +6565,12 @@ deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p,
 done:
 
     if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) {
-	ERTS_SMP_LC_ASSERT(!prt || !erts_lc_is_port_locked(prt));
+	ERTS_LC_ASSERT(!prt || !erts_lc_is_port_locked(prt));
 	erts_thr_progress_unmanaged_continue(dhndl);
 	ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
     } else
     if (res == 1) {
-	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+	ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
         *trace_prt = prt;
     }
     return res;
@@ -6598,13 +6598,13 @@ driver_output_term(ErlDrvPort drvport, ErlDrvTermData* data, int len)
     erts_aint32_t state;
     Port* prt;
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
     /* NOTE! It *not* safe to access 'drvport' from unmanaged threads. */
     prt = erts_drvport2port_state(drvport, &state);
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return -1; /* invalid (dead) */
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_CHK_NO_PROC_LOCKS;
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
     if (state & ERTS_PORT_SFLG_CLOSING)
 	return 0;
 
@@ -6641,14 +6641,14 @@ driver_send_term(ErlDrvPort drvport,
      * internal data representation for ErlDrvPort.
      */
     Port* prt = NULL;
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
     if (erts_thr_progress_is_managed_thread())
     {
 	erts_aint32_t state;
 	prt = erts_drvport2port_state(drvport, &state);
 	if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	    return -1; /* invalid (dead) */
-	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+	ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 	if (state & ERTS_PORT_SFLG_CLOSING)
 	    return 0;
     }
@@ -6668,11 +6668,11 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
     Port* prt = erts_drvport2port_state(ix, &state);
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
 
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return -1;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
     if (state & ERTS_PORT_SFLG_CLOSING)
 	return 0;
 
@@ -6707,12 +6707,12 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
     Port* prt = erts_drvport2port_state(ix, &state);
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
 
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return -1;
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
     if (state & ERTS_PORT_SFLG_CLOSING)
 	return 0;
     
@@ -6746,7 +6746,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
 
 int driver_output(ErlDrvPort ix, char* buf, ErlDrvSizeT len)
 {
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
     return driver_output2(ix, NULL, 0, buf, len);
 }
 
@@ -6762,7 +6762,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
     erts_aint32_t state;
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
 
     ASSERT(vec->size >= skip);
     if (vec->size <= skip)
@@ -6773,7 +6773,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return -1;
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
     if (state & ERTS_PORT_SFLG_CLOSING)
 	return 0;
@@ -6995,7 +6995,7 @@ static void driver_monitor_lock_pdl(Port *p) {
     /* Now we either have the port lock or the port_data_lock */
     ERTS_LC_ASSERT(!p->port_data_lock
 		   || erts_lc_mtx_is_locked(&(p->port_data_lock->mtx)));
-    ERTS_SMP_LC_ASSERT(p->port_data_lock
+    ERTS_LC_ASSERT(p->port_data_lock
 		       || erts_lc_is_port_locked(p));
 }
 
@@ -7003,7 +7003,7 @@ static void driver_monitor_unlock_pdl(Port *p) {
     /* We should either have the port lock or the port_data_lock */
     ERTS_LC_ASSERT(!p->port_data_lock
 		   || erts_lc_mtx_is_locked(&(p->port_data_lock->mtx)));
-    ERTS_SMP_LC_ASSERT(p->port_data_lock
+    ERTS_LC_ASSERT(p->port_data_lock
 		       || erts_lc_is_port_locked(p));
     if (p->port_data_lock) {
 	driver_pdl_unlock(p->port_data_lock);
@@ -7468,7 +7468,7 @@ int driver_set_timer(ErlDrvPort ix, unsigned long t)
 {
     Port* prt = erts_drvport2port(ix);
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
 
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return -1;
@@ -7485,7 +7485,7 @@ int driver_cancel_timer(ErlDrvPort ix)
     Port* prt = erts_drvport2port(ix);
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return -1;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
     erts_cancel_port_timer(prt);
     return 0;
 }
@@ -7496,11 +7496,11 @@ driver_read_timer(ErlDrvPort ix, unsigned long* t)
     Port* prt = erts_drvport2port(ix);
     Sint64 left;
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
 
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return -1;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
     left = erts_read_port_timer(prt);
     if (left < 0)
@@ -7515,7 +7515,7 @@ int
 driver_get_now(ErlDrvNowData *now_data)
 {
     Uint mega,secs,micro;
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
 
     if (now_data == NULL) {
 	return -1;
@@ -7589,7 +7589,7 @@ static int do_driver_monitor_process(Port *prt,
     erts_add_monitor(&ERTS_P_MONITORS(prt), MON_ORIGIN, ref, rp->common.id, NIL);
     erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, ref, prt->common.id, NIL);
 
-    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
     erts_ref_to_driver_monitor(ref,monitor);
     return 0;
 }
@@ -7613,7 +7613,7 @@ int driver_monitor_process(ErlDrvPort drvport,
 
     /* Now (in SMP) we should have either the port lock (if we have a scheduler) or the port data lock
        (if we're a driver thread) */
-    ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock));
+    ERTS_LC_ASSERT((sched != NULL || prt->port_data_lock));
     ret = do_driver_monitor_process(prt,process,monitor);
     DRV_MONITOR_UNLOCK_PDL(prt);
     return ret;
@@ -7648,7 +7648,7 @@ static int do_driver_demonitor_process(Port *prt, const ErlDrvMonitor *monitor)
     if (rp) {
 	ErtsMonitor *rmon;
 	rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref);
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 	if (rmon != NULL) {
 	    erts_destroy_monitor(rmon);
 	}
@@ -7671,7 +7671,7 @@ int driver_demonitor_process(ErlDrvPort drvport,
 
     /* Now we should have either the port lock (if we have a scheduler) or the port data lock
        (if we're a driver thread) */
-    ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock));
+    ERTS_LC_ASSERT((sched != NULL || prt->port_data_lock));
     ret = do_driver_demonitor_process(prt,monitor);
     DRV_MONITOR_UNLOCK_PDL(prt);
     return ret;
@@ -7712,7 +7712,7 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport,
 
     /* Now we should have either the port lock (if we have a scheduler) or the port data lock
        (if we're a driver thread) */
-    ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock));
+    ERTS_LC_ASSERT((sched != NULL || prt->port_data_lock));
     ret = do_driver_get_monitored_process(prt,monitor);
     DRV_MONITOR_UNLOCK_PDL(prt);
     return ret;
@@ -7733,7 +7733,7 @@ void erts_fire_port_monitor(Port *prt, Eterm ref)
     int fpe_was_unmasked;
     ERTS_MSACC_PUSH_STATE_M();
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
     ASSERT(prt->drv_ptr != NULL);    
     DRV_MONITOR_LOCK_PDL(prt);
     if (erts_lookup_monitor(ERTS_P_MONITORS(prt), ref) == NULL) {
@@ -7780,11 +7780,11 @@ driver_failure_term(ErlDrvPort ix, Eterm term, int eof)
     erts_aint32_t state;
     Port* prt = erts_drvport2port_state(ix, &state);
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
 
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return -1;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
     if (prt->async_open_port)
         init_ack_send_reply(prt, prt->common.id);
@@ -7819,7 +7819,7 @@ int driver_exit(ErlDrvPort ix, int err)
     ErtsLink *lnk, *rlnk = NULL;
     Eterm connected;
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
   
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
         return -1;
@@ -7833,7 +7833,7 @@ int driver_exit(ErlDrvPort ix, int err)
     lnk = erts_remove_link(&ERTS_P_LINKS(prt), connected);
 
     if (rp)
-	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 
     if (rlnk != NULL) {
 	erts_destroy_link(rlnk);
@@ -7887,7 +7887,7 @@ ErlDrvTermData driver_mk_atom(char* string)
 			     sys_strlen(string),
 			     ERTS_ATOM_ENC_LATIN1,
 			     1);
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
     return (ErlDrvTermData) am;
 }
 
@@ -7896,27 +7896,27 @@ ErlDrvTermData driver_mk_port(ErlDrvPort ix)
     Port* prt = erts_drvport2port(ix);
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return (ErlDrvTermData) NIL;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
     return (ErlDrvTermData) prt->common.id;
 }
 
 ErlDrvTermData driver_connected(ErlDrvPort ix)
 {
     Port* prt = erts_drvport2port(ix);
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return NIL;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
     return ERTS_PORT_GET_CONNECTED(prt);
 }
 
 ErlDrvTermData driver_caller(ErlDrvPort ix)
 {
     Port* prt = erts_drvport2port(ix);
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return NIL;
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
     return prt->caller;
 }
 
@@ -7925,20 +7925,20 @@ int driver_lock_driver(ErlDrvPort ix)
     Port* prt = erts_drvport2port(ix);
     DE_Handle* dh;
 
-    ERTS_SMP_CHK_NO_PROC_LOCKS;
+    ERTS_CHK_NO_PROC_LOCKS;
 
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return -1;
 
-    erts_smp_rwmtx_rwlock(&erts_driver_list_lock);
+    erts_rwmtx_rwlock(&erts_driver_list_lock);
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
     if ((dh = (DE_Handle*)prt->drv_ptr->handle ) == NULL) {
-	erts_smp_rwmtx_rwunlock(&erts_driver_list_lock);
+	erts_rwmtx_rwunlock(&erts_driver_list_lock);
 	return -1;
     }
     erts_ddll_lock_driver(dh, prt->drv_ptr->name);
-    erts_smp_rwmtx_rwunlock(&erts_driver_list_lock);
+    erts_rwmtx_rwunlock(&erts_driver_list_lock);
     return 0;
 }
 
@@ -7946,9 +7946,9 @@ int driver_lock_driver(ErlDrvPort ix)
 static int maybe_lock_driver_list(void) 
 {
     void *rec_lock;
-    rec_lock = erts_smp_tsd_get(driver_list_lock_status_key);
+    rec_lock = erts_tsd_get(driver_list_lock_status_key);
     if (rec_lock == 0) {
-	erts_smp_rwmtx_rwlock(&erts_driver_list_lock);
+	erts_rwmtx_rwlock(&erts_driver_list_lock);
 	return 1;
     }
     return 0;
@@ -7956,7 +7956,7 @@ static int maybe_lock_driver_list(void)
 static void maybe_unlock_driver_list(int doit)
 {
     if (doit) {
-	erts_smp_rwmtx_rwunlock(&erts_driver_list_lock);
+	erts_rwmtx_rwunlock(&erts_driver_list_lock);
     }
 }
 /* 
@@ -7979,7 +7979,7 @@ void *driver_dl_open(char * path)
 {
     void *ptr;
     int res;
-    int *last_error_p = erts_smp_tsd_get(driver_list_last_error_key);
+    int *last_error_p = erts_tsd_get(driver_list_last_error_key);
     int locked = maybe_lock_driver_list();
     if ((res = erts_sys_ddll_open(path, &ptr, NULL)) == 0) {
 	maybe_unlock_driver_list(locked);
@@ -7987,7 +7987,7 @@ void *driver_dl_open(char * path)
     } else {
 	if (!last_error_p) {
 	    last_error_p = erts_alloc(ERTS_ALC_T_DDLL_ERRCODES, sizeof(int));
-	    erts_smp_tsd_set(driver_list_last_error_key,last_error_p);
+	    erts_tsd_set(driver_list_last_error_key,last_error_p);
 	}
 	*last_error_p = res;
 	maybe_unlock_driver_list(locked);
@@ -7999,7 +7999,7 @@ void *driver_dl_sym(void * handle, char *func_name)
 {
     void *ptr;
     int res;
-    int *last_error_p = erts_smp_tsd_get(driver_list_lock_status_key);
+    int *last_error_p = erts_tsd_get(driver_list_lock_status_key);
     int locked = maybe_lock_driver_list();
     if ((res = erts_sys_ddll_sym(handle, func_name, &ptr)) == 0) {
 	maybe_unlock_driver_list(locked);
@@ -8007,7 +8007,7 @@ void *driver_dl_sym(void * handle, char *func_name)
     } else {
 	if (!last_error_p) {
 	    last_error_p = erts_alloc(ERTS_ALC_T_DDLL_ERRCODES, sizeof(int));
-	    erts_smp_tsd_set(driver_list_lock_status_key,last_error_p);
+	    erts_tsd_set(driver_list_lock_status_key,last_error_p);
 	}
 	*last_error_p = res;
 	maybe_unlock_driver_list(locked);
@@ -8027,7 +8027,7 @@ int driver_dl_close(void *handle)
 char *driver_dl_error(void) 
 {
     char *res;
-    int *last_error_p = erts_smp_tsd_get(driver_list_lock_status_key);
+    int *last_error_p = erts_tsd_get(driver_list_lock_status_key);
     int locked = maybe_lock_driver_list();
     res = erts_ddll_error((last_error_p != NULL) ? (*last_error_p) : ERL_DE_ERROR_UNSPECIFIED); 
     maybe_unlock_driver_list(locked);
@@ -8230,7 +8230,7 @@ void
 erts_destroy_driver(erts_driver_t *drv)
 {
     if (drv->lock) {
-	erts_smp_mtx_destroy(drv->lock);
+	erts_mtx_destroy(drv->lock);
 	erts_free(ERTS_ALC_T_DRIVER_LOCK, drv->lock);
     }
     erts_free(ERTS_ALC_T_DRIVER, drv);
@@ -8243,7 +8243,7 @@ erts_destroy_driver(erts_driver_t *drv)
 
 void add_driver_entry(ErlDrvEntry *drv){
     void *rec_lock;
-    rec_lock = erts_smp_tsd_get(driver_list_lock_status_key);
+    rec_lock = erts_tsd_get(driver_list_lock_status_key);
     /* 
      * Ignore result of erts_add_driver_entry, the init is not
      * allowed to fail when drivers are added by drivers.
@@ -8257,7 +8257,7 @@ int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle, int driver_list_lo
     int res;
 
     if (!driver_list_locked) {
-	erts_smp_rwmtx_rwlock(&erts_driver_list_lock);
+	erts_rwmtx_rwlock(&erts_driver_list_lock);
     }
 
     dp->next = driver_list;
@@ -8268,7 +8268,7 @@ int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle, int driver_list_lo
     driver_list = dp;
 
     if (!driver_list_locked) {
-	erts_smp_tsd_set(driver_list_lock_status_key, (void *) 1);
+	erts_tsd_set(driver_list_lock_status_key, (void *) 1);
     }
 
     res = init_driver(dp, de, handle);
@@ -8285,8 +8285,8 @@ int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle, int driver_list_lo
     }
 	
     if (!driver_list_locked) {
-	erts_smp_tsd_set(driver_list_lock_status_key, NULL);
-	erts_smp_rwmtx_rwunlock(&erts_driver_list_lock);
+	erts_tsd_set(driver_list_lock_status_key, NULL);
+	erts_rwmtx_rwunlock(&erts_driver_list_lock);
     }
     return res;
 }
@@ -8297,9 +8297,9 @@ int remove_driver_entry(ErlDrvEntry *drv)
     erts_driver_t *dp;
     void *rec_lock;
     
-    rec_lock = erts_smp_tsd_get(driver_list_lock_status_key);
+    rec_lock = erts_tsd_get(driver_list_lock_status_key);
     if (rec_lock == NULL) {
-	erts_smp_rwmtx_rwlock(&erts_driver_list_lock);
+	erts_rwmtx_rwlock(&erts_driver_list_lock);
     }
     dp = driver_list;
     while (dp && dp->entry != drv)
@@ -8307,7 +8307,7 @@ int remove_driver_entry(ErlDrvEntry *drv)
     if (dp) {
 	if (dp->handle) {
 	    if (rec_lock == NULL) {
-		erts_smp_rwmtx_rwunlock(&erts_driver_list_lock);
+		erts_rwmtx_rwunlock(&erts_driver_list_lock);
 	    }
 	    return -1;
 	}
@@ -8321,12 +8321,12 @@ int remove_driver_entry(ErlDrvEntry *drv)
 	}
 	erts_destroy_driver(dp);
 	if (rec_lock == NULL) {
-	    erts_smp_rwmtx_rwunlock(&erts_driver_list_lock);
+	    erts_rwmtx_rwunlock(&erts_driver_list_lock);
 	}
 	return 1;
     }
     if (rec_lock == NULL) {
-	erts_smp_rwmtx_rwunlock(&erts_driver_list_lock);
+	erts_rwmtx_rwunlock(&erts_driver_list_lock);
     }
     return 0;
 }
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index 7987cb2eb5..baeec115ea 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -39,9 +39,9 @@
 
 static IndexTable module_tables[ERTS_NUM_CODE_IX];
 
-erts_smp_rwmtx_t the_old_code_rwlocks[ERTS_NUM_CODE_IX];
+erts_rwmtx_t the_old_code_rwlocks[ERTS_NUM_CODE_IX];
 
-static erts_smp_atomic_t tot_module_bytes;
+static erts_atomic_t tot_module_bytes;
 
 /* SMP note: Active module table lookup and current module instance can be
  *           read without any locks. Old module instances are protected by
@@ -49,8 +49,6 @@ static erts_smp_atomic_t tot_module_bytes;
  *           Staging table is protected by the "code_ix lock". 
  */
 
-#include "erl_smp.h"
-
 void module_info(fmtfn_t to, void *to_arg)
 {
     index_info(to, to_arg, &module_tables[erts_active_code_ix()]);
@@ -84,7 +82,7 @@ void erts_module_instance_init(struct erl_module_instance* modi)
 static Module* module_alloc(Module* tmpl)
 {
     Module* obj = (Module*) erts_alloc(ERTS_ALC_T_MODULE, sizeof(Module));
-    erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module));
+    erts_atomic_add_nob(&tot_module_bytes, sizeof(Module));
 
     obj->module = tmpl->module;
     obj->slot.index = -1;
@@ -98,7 +96,7 @@ static Module* module_alloc(Module* tmpl)
 static void module_free(Module* mod)
 {
     erts_free(ERTS_ALC_T_MODULE, mod);
-    erts_smp_atomic_add_nob(&tot_module_bytes, -sizeof(Module));
+    erts_atomic_add_nob(&tot_module_bytes, -sizeof(Module));
 }
 
 void init_module_table(void)
@@ -120,10 +118,10 @@ void init_module_table(void)
     }
 
     for (i=0; i<ERTS_NUM_CODE_IX; i++) {
-        erts_smp_rwmtx_init(&the_old_code_rwlocks[i], "old_code", make_small(i),
+        erts_rwmtx_init(&the_old_code_rwlocks[i], "old_code", make_small(i),
             ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
     }
-    erts_smp_atomic_init_nob(&tot_module_bytes, 0);
+    erts_atomic_init_nob(&tot_module_bytes, 0);
 }
 
 
@@ -159,14 +157,14 @@ static Module* put_module(Eterm mod, IndexTable* mod_tab)
     oldsz = index_table_sz(mod_tab);
     res = (Module*) index_put_entry(mod_tab, (void*) &e);
     newsz = index_table_sz(mod_tab);
-    erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz));
+    erts_atomic_add_nob(&tot_module_bytes, (newsz - oldsz));
     return res;
 }
 
 Module*
 erts_put_module(Eterm mod)
 {
-    ERTS_SMP_LC_ASSERT(erts_initialized == 0
+    ERTS_LC_ASSERT(erts_initialized == 0
 		       || erts_has_code_write_permission());
 
     return put_module(mod, &module_tables[erts_staging_code_ix()]);
@@ -184,7 +182,7 @@ int module_code_size(ErtsCodeIndex code_ix)
 
 int module_table_sz(void)
 {
-    return erts_smp_atomic_read_nob(&tot_module_bytes);
+    return erts_atomic_read_nob(&tot_module_bytes);
 }
 
 #ifdef DEBUG
@@ -233,7 +231,7 @@ void module_start_staging(void)
         copy_module(dst_mod, src_mod);
     }
     newsz = index_table_sz(dst);
-    erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz));
+    erts_atomic_add_nob(&tot_module_bytes, (newsz - oldsz));
 
     entries_at_start_staging = dst->entries;
     IF_DEBUG(dbg_load_code_ix = erts_staging_code_ix());
@@ -251,7 +249,7 @@ void module_end_staging(int commit)
 	oldsz = index_table_sz(tab);
 	index_erase_latest_from(tab, entries_at_start_staging);
 	newsz = index_table_sz(tab);
-	erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz));
+	erts_atomic_add_nob(&tot_module_bytes, (newsz - oldsz));
     }
 
     IF_DEBUG(dbg_load_code_ix = -1);
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index 9d258d5dbf..9a81e6035b 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -72,29 +72,29 @@ int erts_is_old_code_rlocked(ErtsCodeIndex);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-extern erts_smp_rwmtx_t the_old_code_rwlocks[ERTS_NUM_CODE_IX];
+extern erts_rwmtx_t the_old_code_rwlocks[ERTS_NUM_CODE_IX];
 
 ERTS_GLB_INLINE void erts_rwlock_old_code(ErtsCodeIndex code_ix)
 {
-    erts_smp_rwmtx_rwlock(&the_old_code_rwlocks[code_ix]);
+    erts_rwmtx_rwlock(&the_old_code_rwlocks[code_ix]);
 }
 ERTS_GLB_INLINE void erts_rwunlock_old_code(ErtsCodeIndex code_ix)
 {
-    erts_smp_rwmtx_rwunlock(&the_old_code_rwlocks[code_ix]);
+    erts_rwmtx_rwunlock(&the_old_code_rwlocks[code_ix]);
 }
 ERTS_GLB_INLINE void erts_rlock_old_code(ErtsCodeIndex code_ix)
 {
-    erts_smp_rwmtx_rlock(&the_old_code_rwlocks[code_ix]);
+    erts_rwmtx_rlock(&the_old_code_rwlocks[code_ix]);
 }
 ERTS_GLB_INLINE void erts_runlock_old_code(ErtsCodeIndex code_ix)
 {
-    erts_smp_rwmtx_runlock(&the_old_code_rwlocks[code_ix]);
+    erts_rwmtx_runlock(&the_old_code_rwlocks[code_ix]);
 }
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
 ERTS_GLB_INLINE int erts_is_old_code_rlocked(ErtsCodeIndex code_ix)
 {
-    return erts_smp_lc_rwmtx_is_rlocked(&the_old_code_rwlocks[code_ix]);
+    return erts_lc_rwmtx_is_rlocked(&the_old_code_rwlocks[code_ix]);
 }
 #endif
 
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index aec8777e8b..92a0854ad3 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -38,14 +38,14 @@ static Hash process_reg;
 
 #define REG_HASH(term) ((HashValue) atom_val(term))
 
-static erts_smp_rwmtx_t regtab_rwmtx;
+static erts_rwmtx_t regtab_rwmtx;
 
-#define reg_try_read_lock()		erts_smp_rwmtx_tryrlock(&regtab_rwmtx)
-#define reg_try_write_lock()		erts_smp_rwmtx_tryrwlock(&regtab_rwmtx)
-#define reg_read_lock()			erts_smp_rwmtx_rlock(&regtab_rwmtx)
-#define reg_write_lock()		erts_smp_rwmtx_rwlock(&regtab_rwmtx)
-#define reg_read_unlock()		erts_smp_rwmtx_runlock(&regtab_rwmtx)
-#define reg_write_unlock()		erts_smp_rwmtx_rwunlock(&regtab_rwmtx)
+#define reg_try_read_lock()		erts_rwmtx_tryrlock(&regtab_rwmtx)
+#define reg_try_write_lock()		erts_rwmtx_tryrwlock(&regtab_rwmtx)
+#define reg_read_lock()			erts_rwmtx_rlock(&regtab_rwmtx)
+#define reg_write_lock()		erts_rwmtx_rwlock(&regtab_rwmtx)
+#define reg_read_unlock()		erts_rwmtx_runlock(&regtab_rwmtx)
+#define reg_write_unlock()		erts_rwmtx_rwunlock(&regtab_rwmtx)
 
 static ERTS_INLINE void
 reg_safe_read_lock(Process *c_p, ErtsProcLocks *c_p_locks)
@@ -63,7 +63,7 @@ reg_safe_read_lock(Process *c_p, ErtsProcLocks *c_p_locks)
 	}
 
 	/* Release process locks in order to avoid deadlock */
-	erts_smp_proc_unlock(c_p, *c_p_locks);
+	erts_proc_unlock(c_p, *c_p_locks);
 	*c_p_locks = 0;
     }
 
@@ -86,7 +86,7 @@ reg_safe_write_lock(Process *c_p, ErtsProcLocks *c_p_locks)
 	}
 
 	/* Release process locks in order to avoid deadlock */
-	erts_smp_proc_unlock(c_p, *c_p_locks);
+	erts_proc_unlock(c_p, *c_p_locks);
 	*c_p_locks = 0;
     }
 
@@ -139,11 +139,11 @@ static void reg_free(RegProc *obj)
 void init_register_table(void)
 {
     HashFunctions f;
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
-    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
-    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+    erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_RWMTX_LONG_LIVED;
 
-    erts_smp_rwmtx_init_opt(&regtab_rwmtx, &rwmtx_opt, "reg_tab", NIL,
+    erts_rwmtx_init_opt(&regtab_rwmtx, &rwmtx_opt, "reg_tab", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
     f.hash = (H_FUN) reg_hash;
@@ -173,7 +173,7 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id)
     Process *proc = NULL;
     Port *port = NULL;
     RegProc r, *rp;
-    ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p);
+    ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p);
 
     if (is_not_atom(name) || name == am_undefined)
 	return res;
@@ -183,7 +183,7 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id)
     else {
 	if (is_not_internal_pid(id) && is_not_internal_port(id))
 	    return res;
-	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+	erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
 	if (is_internal_port(id)) {
 	    port = erts_id2port(id);
 	    if (!port)
@@ -196,7 +196,7 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id)
 	reg_safe_write_lock(proc, &proc_locks);
 
 	if (proc && !proc_locks)
-	    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+	    erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
     }
 
     if (is_internal_pid(id)) {
@@ -211,7 +211,7 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id)
     }
     else {
 	ASSERT(!INVALID_PORT(port, id));
-	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(port));
+	ERTS_LC_ASSERT(erts_lc_is_port_locked(port));
 	r.pt = port;
 	if (r.pt->common.u.alive.reg)
 	    goto done;
@@ -246,8 +246,8 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id)
 	erts_port_release(port);
     if (c_p != proc) {
 	if (proc)
-	    erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
-	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+	    erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
+	erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
     }
     return res;
 }
@@ -271,12 +271,12 @@ erts_whereis_name_to_id(Process *c_p, Eterm name)
     ErtsProcLocks c_p_locks = 0;
     if (c_p) {
         c_p_locks = ERTS_PROC_LOCK_MAIN;
-        ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p);
+        ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p);
     }
     reg_safe_read_lock(c_p, &c_p_locks);
 
     if (c_p && !c_p_locks)
-        erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+        erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 
     hval = REG_HASH(name);
     ix = hval % process_reg.size;
@@ -378,7 +378,7 @@ erts_whereis_name(Process *c_p,
 		    *proc = rp->p;
 		else {
 		    if (need_locks)
-			erts_smp_proc_unlock(rp->p, need_locks);
+			erts_proc_unlock(rp->p, need_locks);
 		    *proc = NULL;
 		}
 	    }
@@ -402,11 +402,11 @@ erts_whereis_name(Process *c_p,
                         pending_port = NULL;
                     }
 		    
-                    if (erts_smp_port_trylock(rp->pt) == EBUSY) {
+                    if (erts_port_trylock(rp->pt) == EBUSY) {
                         Eterm id = rp->pt->common.id; /* id read only... */
                         /* Unlock all locks, acquire port lock, and restart... */
                         if (current_c_p_locks) {
-                            erts_smp_proc_unlock(c_p, current_c_p_locks);
+                            erts_proc_unlock(c_p, current_c_p_locks);
                             current_c_p_locks = 0;
                         }
                         reg_read_unlock();
@@ -414,14 +414,14 @@ erts_whereis_name(Process *c_p,
                         goto restart;
                     }
                 }
-                ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(rp->pt));
+                ERTS_LC_ASSERT(erts_lc_is_port_locked(rp->pt));
             }
 	    *port = rp->pt;
 	}
     }
 
     if (c_p && !current_c_p_locks)
-	erts_smp_proc_lock(c_p, c_p_locks);
+	erts_proc_lock(c_p, c_p_locks);
     if (pending_port)
 	erts_port_release(pending_port);
 
@@ -477,7 +477,7 @@ int erts_unregister_name(Process *c_p,
 	/* Unregister current process name */
 	ASSERT(c_p);
 	if (current_c_p_locks != c_p_locks) {
-	    erts_smp_proc_lock(c_p, c_p_locks);
+	    erts_proc_lock(c_p, c_p_locks);
 	    current_c_p_locks = c_p_locks;
 	}
 	if (c_p->common.u.alive.reg) {
@@ -498,11 +498,11 @@ int erts_unregister_name(Process *c_p,
 		    port = NULL;
 		}
 
-		if (erts_smp_port_trylock(rp->pt) == EBUSY) {
+		if (erts_port_trylock(rp->pt) == EBUSY) {
 		    Eterm id = rp->pt->common.id; /* id read only... */
 		    /* Unlock all locks, acquire port lock, and restart... */
 		    if (current_c_p_locks) {
-			erts_smp_proc_unlock(c_p, current_c_p_locks);
+			erts_proc_unlock(c_p, current_c_p_locks);
 			current_c_p_locks = 0;
 		    }
 		    reg_write_unlock();
@@ -513,13 +513,13 @@ int erts_unregister_name(Process *c_p,
 	    }
 
 	    ASSERT(rp->pt == port);
-	    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(port));
+	    ERTS_LC_ASSERT(erts_lc_is_port_locked(port));
 
 	    rp->pt->common.u.alive.reg = NULL;
 
 	    if (IS_TRACED_FL(port, F_TRACE_PORTS)) {
                 if (current_c_p_locks) {
-                    erts_smp_proc_unlock(c_p, current_c_p_locks);
+                    erts_proc_unlock(c_p, current_c_p_locks);
                     current_c_p_locks = 0;
                 }
 		trace_port(port, am_unregister, r.name);
@@ -540,7 +540,7 @@ int erts_unregister_name(Process *c_p,
                            rp->p, am_unregister, r.name);
 	    }
 	    if (rp->p != c_p) {
-		erts_smp_proc_unlock(rp->p, ERTS_PROC_LOCK_MAIN);
+		erts_proc_unlock(rp->p, ERTS_PROC_LOCK_MAIN);
 	    }
 	}
 	hash_erase(&process_reg, (void*) &r);
@@ -555,11 +555,11 @@ int erts_unregister_name(Process *c_p,
 	    erts_port_release(port);
 	}
 	if (c_prt) {
-	    erts_smp_port_lock(c_prt);
+	    erts_port_lock(c_prt);
 	}
     }
     if (c_p && !current_c_p_locks) {
-	erts_smp_proc_lock(c_p, c_p_locks);
+	erts_proc_lock(c_p, c_p_locks);
     }
     return res;
 }
@@ -603,10 +603,10 @@ BIF_RETTYPE registered_0(BIF_ALIST_0)
     HashBucket **bucket;
     ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN;
 
-    ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(BIF_P);
+    ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(BIF_P);
     reg_safe_read_lock(BIF_P, &proc_locks);
     if (!proc_locks)
-	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
     bucket = process_reg.bucket;
 
diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c
index 527c9efeca..ac9ebd4714 100644
--- a/erts/emulator/beam/safe_hash.c
+++ b/erts/emulator/beam/safe_hash.c
@@ -62,7 +62,7 @@ static ERTS_INLINE int align_up_pow2(int val)
 */
 static void rehash(SafeHash* h, int grow_limit)
 {
-    if (erts_smp_atomic_xchg_acqb(&h->is_rehashing, 1) != 0) {        
+    if (erts_atomic_xchg_acqb(&h->is_rehashing, 1) != 0) {
 	return; /* already in progress */
     }
     if (h->grow_limit == grow_limit) {
@@ -77,7 +77,7 @@ static void rehash(SafeHash* h, int grow_limit)
 	sys_memzero(new_tab, bytes);
 
 	for (i=0; i<SAFE_HASH_LOCK_CNT; i++) { /* stop all traffic */
-	    erts_smp_mtx_lock(&h->lock_vec[i].mtx);
+	    erts_mtx_lock(&h->lock_vec[i].mtx);
 	}
 
 	h->tab = new_tab;
@@ -95,12 +95,12 @@ static void rehash(SafeHash* h, int grow_limit)
 	}
 
 	for (i=0; i<SAFE_HASH_LOCK_CNT; i++) {
-	    erts_smp_mtx_unlock(&h->lock_vec[i].mtx);
+	    erts_mtx_unlock(&h->lock_vec[i].mtx);
 	}
 	erts_free(h->type, (void *) old_tab);
     }
     /*else already done */
-    erts_smp_atomic_set_relb(&h->is_rehashing, 0);
+    erts_atomic_set_relb(&h->is_rehashing, 0);
 }
 
 
@@ -115,7 +115,7 @@ void safe_hash_get_info(SafeHashInfo *hi, SafeHash *h)
     int objects = 0;
 
     for (lock_ix=0; lock_ix<SAFE_HASH_LOCK_CNT; lock_ix++) {
-	erts_smp_mtx_lock(&h->lock_vec[lock_ix].mtx); 
+	erts_mtx_lock(&h->lock_vec[lock_ix].mtx);
 	size = h->size_mask + 1;
 	for (i = lock_ix; i < size; i += SAFE_HASH_LOCK_CNT) {
 	    int depth = 0;
@@ -128,7 +128,7 @@ void safe_hash_get_info(SafeHashInfo *hi, SafeHash *h)
 	    if (depth > max_depth)
 		max_depth = depth;
 	}
-	erts_smp_mtx_unlock(&h->lock_vec[lock_ix].mtx);	
+	erts_mtx_unlock(&h->lock_vec[lock_ix].mtx);
     }
 
     hi->name  = h->name;
@@ -145,9 +145,9 @@ int safe_hash_table_sz(SafeHash *h)
   int i, size;
   for(i=0; h->name[i]; i++);
   i++;
-  erts_smp_mtx_lock(&h->lock_vec[0].mtx); /* any lock will do to read size */
+  erts_mtx_lock(&h->lock_vec[0].mtx); /* any lock will do to read size */
   size = h->size_mask + 1;
-  erts_smp_mtx_unlock(&h->lock_vec[0].mtx);
+  erts_mtx_unlock(&h->lock_vec[0].mtx);
   return sizeof(SafeHash) + size*sizeof(SafeHashBucket*) + i;
 }
 
@@ -168,10 +168,10 @@ SafeHash* safe_hash_init(ErtsAlcType_t type, SafeHash* h, char* name, erts_lock_
     h->name = name;
     h->fun = fun;
     set_size(h,size);
-    erts_smp_atomic_init_nob(&h->is_rehashing, 0);
-    erts_smp_atomic_init_nob(&h->nitems, 0);
+    erts_atomic_init_nob(&h->is_rehashing, 0);
+    erts_atomic_init_nob(&h->nitems, 0);
     for (i=0; i<SAFE_HASH_LOCK_CNT; i++) {
-        erts_smp_mtx_init(&h->lock_vec[i].mtx, "safe_hash", NIL,
+        erts_mtx_init(&h->lock_vec[i].mtx, "safe_hash", NIL,
             flags);
     }
     return h;
@@ -185,8 +185,8 @@ void* safe_hash_get(SafeHash* h, void* tmpl)
 {
     SafeHashValue hval = h->fun.hash(tmpl);
     SafeHashBucket* b;
-    erts_smp_mtx_t* lock = &h->lock_vec[hval % SAFE_HASH_LOCK_CNT].mtx;
-    erts_smp_mtx_lock(lock);
+    erts_mtx_t* lock = &h->lock_vec[hval % SAFE_HASH_LOCK_CNT].mtx;
+    erts_mtx_lock(lock);
     b = h->tab[hval & h->size_mask];
 	
     while(b != NULL) {
@@ -194,7 +194,7 @@ void* safe_hash_get(SafeHash* h, void* tmpl)
 	    break;
 	b = b->next;
     }
-    erts_smp_mtx_unlock(lock);
+    erts_mtx_unlock(lock);
     return (void*) b;
 }
 
@@ -207,13 +207,13 @@ void* safe_hash_put(SafeHash* h, void* tmpl)
     SafeHashValue hval = h->fun.hash(tmpl);
     SafeHashBucket* b;
     SafeHashBucket** head;
-    erts_smp_mtx_t* lock = &h->lock_vec[hval % SAFE_HASH_LOCK_CNT].mtx;
-    erts_smp_mtx_lock(lock);
+    erts_mtx_t* lock = &h->lock_vec[hval % SAFE_HASH_LOCK_CNT].mtx;
+    erts_mtx_lock(lock);
     head = &h->tab[hval & h->size_mask];
     b = *head;
     while(b != NULL) {
 	if ((b->hvalue == hval) && (h->fun.cmp(tmpl, (void*)b) == 0)) {
-	    erts_smp_mtx_unlock(lock);
+	    erts_mtx_unlock(lock);
 	    return b;
 	}
 	b = b->next;
@@ -224,8 +224,8 @@ void* safe_hash_put(SafeHash* h, void* tmpl)
     b->next = *head;
     *head = b;
     grow_limit = h->grow_limit;
-    erts_smp_mtx_unlock(lock);
-    if (erts_smp_atomic_inc_read_nob(&h->nitems) > grow_limit) {
+    erts_mtx_unlock(lock);
+    if (erts_atomic_inc_read_nob(&h->nitems) > grow_limit) {
 	rehash(h, grow_limit);
     }
     return (void*) b;
@@ -240,22 +240,22 @@ void* safe_hash_erase(SafeHash* h, void* tmpl)
     SafeHashValue hval = h->fun.hash(tmpl);
     SafeHashBucket* b;
     SafeHashBucket** prevp;
-    erts_smp_mtx_t* lock = &h->lock_vec[hval % SAFE_HASH_LOCK_CNT].mtx;
-    erts_smp_mtx_lock(lock);
+    erts_mtx_t* lock = &h->lock_vec[hval % SAFE_HASH_LOCK_CNT].mtx;
+    erts_mtx_lock(lock);
     prevp = &h->tab[hval & h->size_mask];
     b = *prevp;
     while(b != NULL) {
 	if ((b->hvalue == hval) && (h->fun.cmp(tmpl, (void*)b) == 0)) {
 	    *prevp = b->next;
-	    erts_smp_mtx_unlock(lock);
-	    erts_smp_atomic_dec_nob(&h->nitems);
+	    erts_mtx_unlock(lock);
+	    erts_atomic_dec_nob(&h->nitems);
 	    h->fun.free((void*)b);
 	    return tmpl;
 	}
 	prevp = &b->next;
 	b = b->next;
     }
-    erts_smp_mtx_unlock(lock);
+    erts_mtx_unlock(lock);
     return NULL;
 }
 
@@ -280,7 +280,7 @@ void erts_lcnt_enable_hash_lock_count(SafeHash *h, erts_lock_flags_t flags, int
     int i;
 
     for(i = 0; i < SAFE_HASH_LOCK_CNT; i++) {
-        erts_smp_mtx_t *lock = &h->lock_vec[i].mtx;
+        erts_mtx_t *lock = &h->lock_vec[i].mtx;
 
         if(enable) {
             erts_lcnt_install_new_lock_info(&lock->lcnt, "safe_hash", NIL,
diff --git a/erts/emulator/beam/safe_hash.h b/erts/emulator/beam/safe_hash.h
index dde48a6de8..259c58cff9 100644
--- a/erts/emulator/beam/safe_hash.h
+++ b/erts/emulator/beam/safe_hash.h
@@ -73,11 +73,11 @@ typedef struct
     int size_mask;	      /* (RW) Number of slots - 1 */
     SafeHashBucket** tab;     /* (RW) Vector of bucket pointers (objects) */
     int grow_limit;           /* (RW) Threshold for growing table */
-    erts_smp_atomic_t nitems;       /* (A) Number of items in table */
-    erts_smp_atomic_t is_rehashing; /* (A) Table rehashing in progress */
+    erts_atomic_t nitems;       /* (A) Number of items in table */
+    erts_atomic_t is_rehashing; /* (A) Table rehashing in progress */
 
     union {
-	erts_smp_mtx_t mtx;
+	erts_mtx_t mtx;
 	byte __cache_line__[64];
     }lock_vec[SAFE_HASH_LOCK_CNT];
 
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 848f104871..615f44364b 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -218,8 +218,6 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f
 #  define ASSERT(e) ((void) 1)
 #endif
 
-#  define ERTS_SMP_ASSERT(e) ASSERT(e)
-
 /* ERTS_UNDEF can be used to silence false warnings about
  * "variable may be used uninitialized" while keeping the variable
  * marked as undefined by valgrind.
@@ -463,23 +461,23 @@ typedef union {
 
 #include "erl_lock_check.h"
 
-/* needed by erl_smp.h */
+/* needed by erl_threads.h */
 int erts_send_warning_to_logger_str_nogl(char *);
 
-#include "erl_smp.h"
+#include "erl_threads.h"
 
 #ifdef ERTS_WANT_BREAK_HANDLING
-extern erts_smp_atomic32_t erts_break_requested;
+extern erts_atomic32_t erts_break_requested;
 #    define ERTS_BREAK_REQUESTED \
-  ((int) erts_smp_atomic32_read_nob(&erts_break_requested))
+  ((int) erts_atomic32_read_nob(&erts_break_requested))
 void erts_do_break_handling(void);
 #endif
 
 
-extern erts_smp_atomic32_t erts_writing_erl_crash_dump;
+extern erts_atomic32_t erts_writing_erl_crash_dump;
 extern erts_tsd_key_t erts_is_crash_dumping_key;
 #define ERTS_SOMEONE_IS_CRASH_DUMPING \
-  ((int) erts_smp_atomic32_read_mb(&erts_writing_erl_crash_dump))
+  ((int) erts_atomic32_read_mb(&erts_writing_erl_crash_dump))
 #define ERTS_IS_CRASH_DUMPING \
   ((int) (SWord) erts_tsd_get(erts_is_crash_dumping_key))
 
@@ -620,7 +618,7 @@ int erts_send_info_to_logger_nogl(erts_dsprintf_buf_t *);
 int erts_send_warning_to_logger_nogl(erts_dsprintf_buf_t *);
 int erts_send_error_to_logger_nogl(erts_dsprintf_buf_t *);
 int erts_send_info_to_logger_str_nogl(char *);
-/* needed by erl_smp.h (declared above)
+/* needed by erl_threads.h (declared above)
    int erts_send_warning_to_logger_str_nogl(char *); */
 int erts_send_error_to_logger_str_nogl(char *);
 
@@ -1002,140 +1000,6 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-typedef erts_smp_atomic_t erts_smp_refc_t;
-
-ERTS_GLB_INLINE void erts_smp_refc_init(erts_smp_refc_t *refcp, erts_aint_t val);
-ERTS_GLB_INLINE void erts_smp_refc_inc(erts_smp_refc_t *refcp, erts_aint_t min_val);
-ERTS_GLB_INLINE erts_aint_t erts_smp_refc_inc_unless(erts_smp_refc_t *refcp,
-                                                     erts_aint_t unless_val,
-                                                     erts_aint_t min_val);
-ERTS_GLB_INLINE erts_aint_t erts_smp_refc_inctest(erts_smp_refc_t *refcp,
-                                                  erts_aint_t min_val);
-ERTS_GLB_INLINE void erts_smp_refc_dec(erts_smp_refc_t *refcp, erts_aint_t min_val);
-ERTS_GLB_INLINE erts_aint_t erts_smp_refc_dectest(erts_smp_refc_t *refcp,
-                                                  erts_aint_t min_val);
-ERTS_GLB_INLINE void erts_smp_refc_add(erts_smp_refc_t *refcp, erts_aint_t diff,
-                                       erts_aint_t min_val);
-ERTS_GLB_INLINE erts_aint_t erts_smp_refc_read(erts_smp_refc_t *refcp,
-                                               erts_aint_t min_val);
-
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-
-ERTS_GLB_INLINE void
-erts_smp_refc_init(erts_smp_refc_t *refcp, erts_aint_t val)
-{
-    erts_smp_atomic_init_nob((erts_smp_atomic_t *) refcp, val);
-}
-
-ERTS_GLB_INLINE void
-erts_smp_refc_inc(erts_smp_refc_t *refcp, erts_aint_t min_val)
-{
-#ifdef ERTS_REFC_DEBUG
-    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
-    if (val < min_val)
-	erts_exit(ERTS_ABORT_EXIT,
-		 "erts_smp_refc_inc(): Bad refc found (refc=%ld < %ld)!\n",
-		 val, min_val);
-#else
-    erts_smp_atomic_inc_nob((erts_smp_atomic_t *) refcp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_refc_inc_unless(erts_smp_refc_t *refcp,
-                         erts_aint_t unless_val,
-                         erts_aint_t min_val)
-{
-    erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp);
-    while (1) {
-        erts_aint_t exp, new;
-#ifdef ERTS_REFC_DEBUG
-        if (val < 0)
-            erts_exit(ERTS_ABORT_EXIT,
-                      "erts_smp_refc_inc_unless(): Bad refc found (refc=%ld < %ld)!\n",
-                      val, min_val);
-#endif
-        if (val == unless_val)
-            return val;
-        new = val + 1;
-        exp = val;
-        val = erts_smp_atomic_cmpxchg_nob((erts_smp_atomic_t *) refcp, new, exp);
-        if (val == exp)
-            return new;
-    }
-}
-
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_refc_inctest(erts_smp_refc_t *refcp, erts_aint_t min_val)
-{
-    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
-#ifdef ERTS_REFC_DEBUG
-    if (val < min_val)
-	erts_exit(ERTS_ABORT_EXIT,
-		 "erts_smp_refc_inctest(): Bad refc found (refc=%ld < %ld)!\n",
-		 val, min_val);
-#endif
-    return val;
-}
-
-ERTS_GLB_INLINE void
-erts_smp_refc_dec(erts_smp_refc_t *refcp, erts_aint_t min_val)
-{
-#ifdef ERTS_REFC_DEBUG
-    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
-    if (val < min_val)
-	erts_exit(ERTS_ABORT_EXIT,
-		 "erts_smp_refc_dec(): Bad refc found (refc=%ld < %ld)!\n",
-		 val, min_val);
-#else
-    erts_smp_atomic_dec_nob((erts_smp_atomic_t *) refcp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_refc_dectest(erts_smp_refc_t *refcp, erts_aint_t min_val)
-{
-    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
-#ifdef ERTS_REFC_DEBUG
-    if (val < min_val)
-	erts_exit(ERTS_ABORT_EXIT,
-		 "erts_smp_refc_dectest(): Bad refc found (refc=%ld < %ld)!\n",
-		 val, min_val);
-#endif
-    return val;
-}
-
-ERTS_GLB_INLINE void
-erts_smp_refc_add(erts_smp_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val)
-{
-#ifdef ERTS_REFC_DEBUG
-    erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff);
-    if (val < min_val)
-	erts_exit(ERTS_ABORT_EXIT,
-		 "erts_smp_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n",
-		 diff, val, min_val);
-#else
-    erts_smp_atomic_add_nob((erts_smp_atomic_t *) refcp, diff);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_refc_read(erts_smp_refc_t *refcp, erts_aint_t min_val)
-{
-    erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp);
-#ifdef ERTS_REFC_DEBUG
-    if (val < min_val)
-	erts_exit(ERTS_ABORT_EXIT,
-		 "erts_smp_refc_read(): Bad refc found (refc=%ld < %ld)!\n",
-		 val, min_val);
-#endif
-    return val;
-}
-
-#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
-
-
 #ifdef ERTS_ENABLE_KERNEL_POLL
 extern int erts_use_kernel_poll;
 #endif
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 3713b756e8..dcb1468d60 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -43,7 +43,6 @@
 #include "erl_printf.h"
 #include "erl_threads.h"
 #include "erl_lock_count.h"
-#include "erl_smp.h"
 #include "erl_time.h"
 #include "erl_thr_progress.h"
 #include "erl_thr_queue.h"
@@ -3527,7 +3526,7 @@ store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns)
     if (is_external_header(*from_hp)) {
 	ExternalThing *etp = (ExternalThing *) from_hp;
 	ASSERT(is_external(ns));
-	erts_smp_refc_inc(&etp->node->refc, 2);
+	erts_refc_inc(&etp->node->refc, 2);
     }
     else if (is_ordinary_ref_thing(from_hp))
 	return make_internal_ref(to_hp);
@@ -4628,18 +4627,6 @@ void
 erts_interval_init(erts_interval_t *icp)
 {
     erts_atomic64_init_nob(&icp->counter.atomic, 0);
-#ifdef DEBUG
-    icp->smp_api = 0;
-#endif
-}
-
-void
-erts_smp_interval_init(erts_interval_t *icp)
-{
-    erts_interval_init(icp);
-#ifdef DEBUG
-    icp->smp_api = 1;
-#endif
 }
 
 static ERTS_INLINE Uint64
@@ -4679,56 +4666,24 @@ ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
 Uint64
 erts_step_interval_nob(erts_interval_t *icp)
 {
-    ASSERT(!icp->smp_api);
     return step_interval_nob(icp);
 }
 
 Uint64
 erts_step_interval_relb(erts_interval_t *icp)
 {
-    ASSERT(!icp->smp_api);
-    return step_interval_relb(icp);
-}
-
-Uint64
-erts_smp_step_interval_nob(erts_interval_t *icp)
-{
-    ASSERT(icp->smp_api);
-    return step_interval_nob(icp);
-}
-
-Uint64
-erts_smp_step_interval_relb(erts_interval_t *icp)
-{
-    ASSERT(icp->smp_api);
     return step_interval_relb(icp);
 }
 
 Uint64
 erts_ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic)
 {
-    ASSERT(!icp->smp_api);
     return ensure_later_interval_nob(icp, ic);
 }
 
 Uint64
 erts_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
 {
-    ASSERT(!icp->smp_api);
-    return ensure_later_interval_acqb(icp, ic);
-}
-
-Uint64
-erts_smp_ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic)
-{
-    ASSERT(icp->smp_api);
-    return ensure_later_interval_nob(icp, ic);
-}
-
-Uint64
-erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
-{
-    ASSERT(icp->smp_api);
     return ensure_later_interval_acqb(icp, ic);
 }
 
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 94bc563fda..05663648e9 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1000,7 +1000,7 @@ BIF_RETTYPE hipe_bifs_set_native_address_in_fe_2(BIF_ALIST_2)
 	BIF_ERROR(BIF_P, BADARG);
 
     fe->native_address = native_address;
-    if (erts_smp_refc_dectest(&fe->refc, 0) == 0)
+    if (erts_refc_dectest(&fe->refc, 0) == 0)
 	erts_erase_fun_entry(fe);
     BIF_RET(am_true);
 }
@@ -1048,7 +1048,7 @@ static struct {
      * they create a new stub for the mfa, which forces locking.
      * XXX: Redesign apply et al to avoid those updates.
      */
-    erts_smp_rwmtx_t lock;
+    erts_rwmtx_t lock;
 } hipe_mfa_info_table;
 
 Hash mod2mfa_tab; /* map from module atom to list of hipe_mfa_info */
@@ -1129,28 +1129,28 @@ struct hipe_ref {
 
 static inline void hipe_mfa_info_table_init_lock(void)
 {
-    erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock", NIL,
+    erts_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 }
 
 static inline void hipe_mfa_info_table_rlock(void)
 {
-    erts_smp_rwmtx_rlock(&hipe_mfa_info_table.lock);
+    erts_rwmtx_rlock(&hipe_mfa_info_table.lock);
 }
 
 static inline void hipe_mfa_info_table_runlock(void)
 {
-    erts_smp_rwmtx_runlock(&hipe_mfa_info_table.lock);
+    erts_rwmtx_runlock(&hipe_mfa_info_table.lock);
 }
 
 static inline void hipe_mfa_info_table_rwlock(void)
 {
-    erts_smp_rwmtx_rwlock(&hipe_mfa_info_table.lock);
+    erts_rwmtx_rwlock(&hipe_mfa_info_table.lock);
 }
 
 static inline void hipe_mfa_info_table_rwunlock(void)
 {
-    erts_smp_rwmtx_rwunlock(&hipe_mfa_info_table.lock);
+    erts_rwmtx_rwunlock(&hipe_mfa_info_table.lock);
 }
 
 static ERTS_INLINE
@@ -1636,7 +1636,7 @@ void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module,
 {
     struct hipe_ref* ref = first_ref;
 
-    ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(is_blocking == erts_thr_progress_is_blocking());
 
     while (ref) {
         struct hipe_ref* free_ref = ref;
@@ -1682,9 +1682,9 @@ void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module,
 {
     struct hipe_sdesc* sdesc = first_sdesc;
 
-    ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(is_blocking == erts_thr_progress_is_blocking());
 
-    ERTS_SMP_LC_ASSERT(is_blocking); /*XXX Fix safe sdesc destruction */
+    ERTS_LC_ASSERT(is_blocking); /*XXX Fix safe sdesc destruction */
 
     while (sdesc) {
         struct hipe_sdesc* free_sdesc = sdesc;
@@ -1702,7 +1702,7 @@ void hipe_purge_module(Module* modp, int is_blocking)
 {
     ASSERT(modp);
 
-    ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(is_blocking == erts_thr_progress_is_blocking());
 
     DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module");
 
@@ -1711,7 +1711,7 @@ void hipe_purge_module(Module* modp, int is_blocking)
          * Remove all hipe_ref's (external calls) from the old module instance
          */
         if (modp->old.hipe_code->first_hipe_ref) {
-            ERTS_SMP_LC_ASSERT(is_blocking);
+            ERTS_LC_ASSERT(is_blocking);
 
             hipe_purge_refs(modp->old.hipe_code->first_hipe_ref,
                             make_atom(modp->module), is_blocking);
@@ -1722,7 +1722,7 @@ void hipe_purge_module(Module* modp, int is_blocking)
          * Remove all hipe_sdesc's for the old module instance
          */
         if (modp->old.hipe_code->first_hipe_sdesc) {
-            ERTS_SMP_LC_ASSERT(is_blocking);
+            ERTS_LC_ASSERT(is_blocking);
 
             hipe_purge_sdescs(modp->old.hipe_code->first_hipe_sdesc,
                              make_atom(modp->module), is_blocking);
@@ -1773,7 +1773,7 @@ void hipe_redirect_to_module(Module* modp)
     struct hipe_mfa_info *p;
     struct hipe_ref_head* refh;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
 
     for (p = mod2mfa_get(modp); p; p = p->next_in_mod) {
         if (p->new_address) {
diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c
index 9cac733db5..9ebbb22846 100644
--- a/erts/emulator/hipe/hipe_bif2.c
+++ b/erts/emulator/hipe/hipe_bif2.c
@@ -45,7 +45,7 @@ static void proc_unlock(Process* c_p, Process* rp)
 	locks &= ~ERTS_PROC_LOCK_MAIN;
     }
     if (rp && locks) {
-	erts_smp_proc_unlock(rp, locks);
+	erts_proc_unlock(rp, locks);
     }
 }
 
@@ -157,10 +157,10 @@ BIF_RETTYPE hipe_bifs_modeswitch_debug_off_0(BIF_ALIST_0)
 
 BIF_RETTYPE hipe_debug_bif_wrapper(NBIF_ALIST_1);
 
-#    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
+#    define ERTS_REQ_PROC_MAIN_LOCK(P) \
        if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\
 					  __FILE__, __LINE__)
-#    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
+#    define ERTS_UNREQ_PROC_MAIN_LOCK(P) \
         if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
 
 BIF_RETTYPE hipe_debug_bif_wrapper(NBIF_ALIST_1)
@@ -168,9 +168,9 @@ BIF_RETTYPE hipe_debug_bif_wrapper(NBIF_ALIST_1)
     typedef BIF_RETTYPE nBif(NBIF_ALIST_1);
     nBif* fp = (nBif*) (BIF_P->hipe.bif_callee);
     BIF_RETTYPE res;
-    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(BIF_P);
+    ERTS_UNREQ_PROC_MAIN_LOCK(BIF_P);
     res = (*fp)(NBIF_CALL_ARGS);
-    ERTS_SMP_REQ_PROC_MAIN_LOCK(BIF_P);
+    ERTS_REQ_PROC_MAIN_LOCK(BIF_P);
     return res;
 }
 
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index fd90c46fa8..b7f81fc4a6 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -37,14 +37,14 @@
 #include "hipe_bif0.h"	/* hipe_mfa_info_table_init() */
 
 #if defined(ERTS_ENABLE_LOCK_CHECK)
-#    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
+#    define ERTS_REQ_PROC_MAIN_LOCK(P) \
         if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,	\
 					   __FILE__, __LINE__)
-#    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
+#    define ERTS_UNREQ_PROC_MAIN_LOCK(P) \
         if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
 #else
-#    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)
-#    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P)
+#    define ERTS_REQ_PROC_MAIN_LOCK(P)
+#    define ERTS_UNREQ_PROC_MAIN_LOCK(P)
 #endif
 
 
@@ -394,7 +394,7 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
 	      goto do_schedule;
 	  }
 
-	  if (!(erts_smp_atomic32_read_acqb(&p->state) & ERTS_PSFLG_ACTIVE)) {
+	  if (!(erts_atomic32_read_acqb(&p->state) & ERTS_PSFLG_ACTIVE)) {
 	      for (i = 0; i < p->arity; ++i)
 		  p->arg_reg[i] = reg[i]; 	      
 	      goto do_schedule;
@@ -495,12 +495,12 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
 	  if (p->hipe_smp.have_receive_locks)
 	      p->hipe_smp.have_receive_locks = 0;
 	  else
-	      erts_smp_proc_lock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	      erts_proc_lock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	  p->i = hipe_beam_pc_resume;
 	  p->arity = 0;
-	  erts_smp_atomic32_read_band_relb(&p->state,
+	  erts_atomic32_read_band_relb(&p->state,
 					   ~ERTS_PSFLG_ACTIVE);
-	  erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	  erts_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
       do_schedule:
 	  {
 	      struct saved_calls *scb;
@@ -511,16 +511,16 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
 
               /* The process may have died while it was executing,
                  if so we return out from native code to the interpreter */
-              if (erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_EXITING)
+              if (erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_EXITING)
                   p->i = beam_exit;
 #ifdef DEBUG
 	      ASSERT(p->debug_reds_in == reds_in);
 #endif
 	      p->flags &= ~F_HIPE_MODE;
 
-	      ERTS_SMP_UNREQ_PROC_MAIN_LOCK(p);
+	      ERTS_UNREQ_PROC_MAIN_LOCK(p);
 	      p = erts_schedule(NULL, p, reds_in - p->fcalls);
-	      ERTS_SMP_REQ_PROC_MAIN_LOCK(p);
+	      ERTS_REQ_PROC_MAIN_LOCK(p);
 	      ASSERT(!(p->flags & F_HIPE_MODE));
 	      p->hipe_smp.have_receive_locks = 0;
 	      reg = p->scheduler_data->x_reg_array;
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index f6ecf841ba..23f64a6991 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -145,7 +145,7 @@ BIF_RETTYPE nbif_impl_hipe_set_timeout(NBIF_ALIST_1)
 	if (tres != 0) { /* Wrong time */
 	    if (p->hipe_smp.have_receive_locks) {
 		p->hipe_smp.have_receive_locks = 0;
-		erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+		erts_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	    }
 	    BIF_ERROR(p, EXC_TIMEOUT_VALUE);
 	}
@@ -526,16 +526,16 @@ Eterm hipe_check_get_msg(Process *c_p)
     msgp = PEEK_MESSAGE(c_p);
 
     if (!msgp) {
-	erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	/* Make sure messages wont pass exit signals... */
 	if (ERTS_PROC_PENDING_EXIT(c_p)) {
-	    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	    return THE_NON_VALUE; /* Will be rescheduled for exit */
 	}
-	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
+	ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
 	msgp = PEEK_MESSAGE(c_p);
 	if (msgp)
-	    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	else {
 	    /* XXX: BEAM doesn't need this */
 	    c_p->hipe_smp.have_receive_locks = 1;
@@ -579,7 +579,7 @@ void hipe_clear_timeout(Process *c_p)
        cases. HiPE doesn't, so we must check dynamically. */
     if (c_p->hipe_smp.have_receive_locks) {
 	c_p->hipe_smp.have_receive_locks = 0;
-	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+	erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
     }
     if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) {
 	trace_receive(c_p, am_clock_service, am_timeout, NULL);
@@ -590,6 +590,6 @@ void hipe_clear_timeout(Process *c_p)
 
 void hipe_atomic_inc(int *counter)
 {
-    erts_smp_atomic_inc_nob((erts_smp_atomic_t*)counter);
+    erts_atomic_inc_nob((erts_atomic_t*)counter);
 }
 
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 3e029303a7..834b77eb58 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -97,16 +97,16 @@ typedef char EventStateFlags;
 static struct pollset_info
 {
     ErtsPollSet ps;
-    erts_smp_atomic_t in_poll_wait;        /* set while doing poll */
+    erts_atomic_t in_poll_wait;        /* set while doing poll */
     struct {
 	int six; /* start index */
 	int eix; /* end index */
-	erts_smp_atomic32_t no;
+	erts_atomic32_t no;
 	int size;
 	ErtsSysFdType *array;
     } active_fd;
     struct removed_fd* removed_list;       /* list of deselected fd's*/
-    erts_smp_spinlock_t removed_list_lock;
+    erts_spinlock_t removed_list_lock;
 }pollset;
 #define NUM_OF_POLLSETS 1
 
@@ -150,11 +150,11 @@ static int max_fds = -1;
 #endif
 #define DRV_EV_STATE_LOCK_CNT 16
 static union {
-    erts_smp_mtx_t lck;
+    erts_mtx_t lck;
     byte _cache_line_alignment[64];
 }drv_ev_state_locks[DRV_EV_STATE_LOCK_CNT];
 
-static ERTS_INLINE erts_smp_mtx_t* fd_mtx(ErtsSysFdType fd)
+static ERTS_INLINE erts_mtx_t* fd_mtx(ErtsSysFdType fd)
 {
     int hash = (int)fd;
 # ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
@@ -165,15 +165,15 @@ static ERTS_INLINE erts_smp_mtx_t* fd_mtx(ErtsSysFdType fd)
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
 
-static erts_smp_atomic_t drv_ev_state_len;
+static erts_atomic_t drv_ev_state_len;
 static ErtsDrvEventState *drv_ev_state;
-static erts_smp_mtx_t drv_ev_state_grow_lock; /* prevent lock-hogging of racing growers */
+static erts_mtx_t drv_ev_state_grow_lock; /* prevent lock-hogging of racing growers */
 
 #else
 static SafeHash drv_ev_state_tab;
 static int num_state_prealloc;
 static ErtsDrvEventState *state_prealloc_first;
-erts_smp_spinlock_t state_prealloc_lock;
+erts_spinlock_t state_prealloc_lock;
 
 static ERTS_INLINE ErtsDrvEventState *hash_get_drv_ev_state(ErtsSysFdType fd)
 {
@@ -244,7 +244,7 @@ static ERTS_INLINE void
 init_iotask(ErtsIoTask *io_task)
 {
     erts_port_task_handle_init(&io_task->task);
-    erts_smp_atomic_init_nob(&io_task->executed_time, ~((erts_aint_t) 0));
+    erts_atomic_init_nob(&io_task->executed_time, ~((erts_aint_t) 0));
 }
 
 static ERTS_INLINE int
@@ -252,7 +252,7 @@ is_iotask_active(ErtsIoTask *io_task, erts_aint_t current_cio_time)
 {    
     if (erts_port_task_is_scheduled(&io_task->task))
 	return 1;
-    if (erts_smp_atomic_read_nob(&io_task->executed_time) == current_cio_time)
+    if (erts_atomic_read_nob(&io_task->executed_time) == current_cio_time)
 	return 1;
     return 0;
 }
@@ -325,8 +325,8 @@ static ERTS_INLINE void
 remember_removed(ErtsDrvEventState *state, struct pollset_info* psi)
 {
     struct removed_fd *fdlp;
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(fd_mtx(state->fd)));
-    if (erts_smp_atomic_read_nob(&psi->in_poll_wait)) {
+    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd)));
+    if (erts_atomic_read_nob(&psi->in_poll_wait)) {
 	state->remove_cnt++;
 	ASSERT(state->remove_cnt > 0);
 	fdlp = removed_fd_alloc();
@@ -336,10 +336,10 @@ remember_removed(ErtsDrvEventState *state, struct pollset_info* psi)
     #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
 	fdlp->state = state;
     #endif
-	erts_smp_spin_lock(&psi->removed_list_lock);
+	erts_spin_lock(&psi->removed_list_lock);
 	fdlp->next = psi->removed_list;
 	psi->removed_list = fdlp;
-	erts_smp_spin_unlock(&psi->removed_list_lock);
+	erts_spin_unlock(&psi->removed_list_lock);
     }
 }
 
@@ -363,29 +363,29 @@ forget_removed(struct pollset_info* psi)
 
     /* Fast track: if (atomic_ptr(removed_list)==NULL) return; */
 
-    erts_smp_spin_lock(&psi->removed_list_lock);
+    erts_spin_lock(&psi->removed_list_lock);
     fdlp = psi->removed_list;
     psi->removed_list = NULL;
-    erts_smp_spin_unlock(&psi->removed_list_lock);
+    erts_spin_unlock(&psi->removed_list_lock);
 
     while (fdlp) {
         ErtsResource* resource = NULL;
 	erts_driver_t* drv_ptr = NULL;
-	erts_smp_mtx_t* mtx;
+	erts_mtx_t* mtx;
 	ErtsSysFdType fd;
 	ErtsDrvEventState *state;
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
 	fd = fdlp->fd;
 	mtx = fd_mtx(fd);
-	erts_smp_mtx_lock(mtx);
+	erts_mtx_lock(mtx);
 	state = &drv_ev_state[(int) fd];
 #else
 	state = fdlp->state;
 	fd = state->fd;
 	ASSERT(fd == fdlp->fd);
 	mtx = fd_mtx(fd);
-	erts_smp_mtx_lock(mtx);
+	erts_mtx_lock(mtx);
 #endif
 	ASSERT(state->remove_cnt > 0);
 	if (--state->remove_cnt == 0) {
@@ -420,7 +420,7 @@ forget_removed(struct pollset_info* psi)
 		ASSERT(0);
 	    }
 	}
-	erts_smp_mtx_unlock(mtx);
+	erts_mtx_unlock(mtx);
 	if (drv_ptr) {
 	    int was_unmasked = erts_block_fpe();
 	    DTRACE1(driver_stop_select, drv_ptr->name);
@@ -450,15 +450,15 @@ grow_drv_ev_state(int min_ix)
     int old_len;
     int new_len;
 
-    erts_smp_mtx_lock(&drv_ev_state_grow_lock);
-    old_len = erts_smp_atomic_read_nob(&drv_ev_state_len);
+    erts_mtx_lock(&drv_ev_state_grow_lock);
+    old_len = erts_atomic_read_nob(&drv_ev_state_len);
     if (min_ix >= old_len) {
         new_len = erts_poll_new_table_len(old_len, min_ix + 1);
         if (new_len > max_fds)
             new_len = max_fds;
 
 	for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { /* lock all fd's */
-	    erts_smp_mtx_lock(&drv_ev_state_locks[i].lck);
+	    erts_mtx_lock(&drv_ev_state_locks[i].lck);
 	}
 	drv_ev_state = (drv_ev_state
 			? erts_realloc(ERTS_ALC_T_DRV_EV_STATE,
@@ -479,14 +479,14 @@ grow_drv_ev_state(int min_ix)
 	    drv_ev_state[i].type = ERTS_EV_TYPE_NONE;
 	    drv_ev_state[i].flags = 0;
 	}
-	erts_smp_atomic_set_nob(&drv_ev_state_len, new_len);
+	erts_atomic_set_nob(&drv_ev_state_len, new_len);
 	for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) {
-	    erts_smp_mtx_unlock(&drv_ev_state_locks[i].lck);
+	    erts_mtx_unlock(&drv_ev_state_locks[i].lck);
 	}
     }
     /*else already grown by racing thread */
 
-    erts_smp_mtx_unlock(&drv_ev_state_grow_lock);
+    erts_mtx_unlock(&drv_ev_state_grow_lock);
 }
 #endif /* ERTS_SYS_CONTINOUS_FD_NUMBERS */
 
@@ -544,7 +544,7 @@ deselect(ErtsDrvEventState *state, int mode)
 {
     int do_wake = 0;
     ErtsPollEvents rm_events;
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(fd_mtx(state->fd)));
+    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd)));
     ASSERT(state->events);
 
     abort_tasks(state, mode);
@@ -616,9 +616,9 @@ check_fd_cleanup(ErtsDrvEventState *state,
 {
     erts_aint_t current_cio_time;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(fd_mtx(state->fd)));
+    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd)));
 
-    current_cio_time = erts_smp_atomic_read_acqb(&erts_check_io_time);
+    current_cio_time = erts_atomic_read_acqb(&erts_check_io_time);
     *free_select = NULL;
     if (state->driver.select
 	&& (state->type != ERTS_EV_TYPE_DRV_SEL)
@@ -671,7 +671,7 @@ check_cleanup_active_fd(ErtsSysFdType fd,
 {
     ErtsDrvEventState *state;
     int active = 0;
-    erts_smp_mtx_t *mtx = fd_mtx(fd);
+    erts_mtx_t *mtx = fd_mtx(fd);
     void *free_select = NULL;
     void *free_nif = NULL;
 #if ERTS_CIO_HAVE_DRV_EVENT
@@ -681,7 +681,7 @@ check_cleanup_active_fd(ErtsSysFdType fd,
     ErtsPollEvents evon = 0, evoff = 0;
 #endif
 
-    erts_smp_mtx_lock(mtx);
+    erts_mtx_lock(mtx);
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     state = &drv_ev_state[(int) fd];
@@ -802,7 +802,7 @@ check_cleanup_active_fd(ErtsSysFdType fd,
 
     }
 
-    erts_smp_mtx_unlock(mtx);
+    erts_mtx_unlock(mtx);
 
     if (free_select)
 	free_drv_select_data(free_select);
@@ -836,7 +836,7 @@ check_cleanup_active_fds(erts_aint_t current_cio_time, int may_sleep)
 {
     int six = pollset.active_fd.six;
     int eix = pollset.active_fd.eix;
-    erts_aint32_t no = erts_smp_atomic32_read_dirty(&pollset.active_fd.no);
+    erts_aint32_t no = erts_atomic32_read_dirty(&pollset.active_fd.no);
     int size = pollset.active_fd.size;
     int ix = six;
 #if ERTS_CIO_DEFER_ACTIVE_EVENTS
@@ -891,7 +891,7 @@ check_cleanup_active_fds(erts_aint_t current_cio_time, int may_sleep)
 
     pollset.active_fd.six = six;
     pollset.active_fd.eix = eix;
-    erts_smp_atomic32_set_relb(&pollset.active_fd.no, no);
+    erts_atomic32_set_relb(&pollset.active_fd.no, no);
 }
 
 static void grow_active_fds(void)
@@ -920,8 +920,8 @@ add_active_fd(ErtsSysFdType fd)
     
     pollset.active_fd.array[eix] = fd;
 
-    erts_smp_atomic32_set_relb(&pollset.active_fd.no,
-			       (erts_smp_atomic32_read_dirty(&pollset.active_fd.no)
+    erts_atomic32_set_relb(&pollset.active_fd.no,
+			       (erts_atomic32_read_dirty(&pollset.active_fd.no)
 				+ 1));
 
     eix++;
@@ -961,10 +961,10 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return -1;
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) {
+    if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state_len)) {
 	if (fd < 0) {
 	    return -1;    
 	}
@@ -976,7 +976,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
     }
 #endif
 
-    erts_smp_mtx_lock(fd_mtx(fd));
+    erts_mtx_lock(fd_mtx(fd));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     state = &drv_ev_state[(int) fd];
@@ -1157,7 +1157,7 @@ done:
                      &free_nif);
 
 done_unknown:
-    erts_smp_mtx_unlock(fd_mtx(fd));
+    erts_mtx_unlock(fd_mtx(fd));
     if (stop_select_fn) {
 	int was_unmasked = erts_block_fpe();
 	DTRACE1(driver_stop_select, name);
@@ -1206,7 +1206,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
     ASSERT(!(resource->monitors && resource->monitors->is_dying));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) {
+    if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state_len)) {
 	if (fd < 0) {
 	    return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT;
 	}
@@ -1218,7 +1218,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
     }
 #endif
 
-    erts_smp_mtx_lock(fd_mtx(fd));
+    erts_mtx_lock(fd_mtx(fd));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     state = &drv_ev_state[(int) fd];
@@ -1412,7 +1412,7 @@ done:
                      &free_nif);
 
 done_unknown:
-    erts_smp_mtx_unlock(fd_mtx(fd));
+    erts_mtx_unlock(fd_mtx(fd));
     if (call_stop) {
         erts_resource_stop(resource, (ErlNifEvent)fd, 1);
         if (call_stop == CALL_STOP_AND_RELEASE) {
@@ -1458,10 +1458,10 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix,
     if (prt == ERTS_INVALID_ERL_DRV_PORT)
 	return -1;
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) {
+    if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state_len)) {
 	if (fd < 0)
 	    return -1;    
 	if (fd >= max_fds) {
@@ -1472,7 +1472,7 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix,
     }
 #endif
 
-    erts_smp_mtx_lock(fd_mtx(fd));
+    erts_mtx_lock(fd_mtx(fd));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     state = &drv_ev_state[(int) fd];
@@ -1565,7 +1565,7 @@ done:
 		     &free_select,
                      &free_nif);
 
-    erts_smp_mtx_unlock(fd_mtx(fd));
+    erts_mtx_unlock(fd_mtx(fd));
 
     if (free_select)
 	free_drv_select_data(free_select);
@@ -1995,7 +1995,7 @@ iready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time)
 				 ERTS_PORT_TASK_INPUT,
 				 current_cio_time)) {
 	ErtsIoTask *iotask = &state->driver.select->iniotask;
-	erts_smp_atomic_set_nob(&iotask->executed_time, current_cio_time);
+	erts_atomic_set_nob(&iotask->executed_time, current_cio_time);
 	if (erts_port_task_schedule(id,
 				    &iotask->task,
 				    ERTS_PORT_TASK_INPUT,
@@ -2013,7 +2013,7 @@ oready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time)
 				 ERTS_PORT_TASK_OUTPUT,
 				 current_cio_time)) {
 	ErtsIoTask *iotask = &state->driver.select->outiotask;
-	erts_smp_atomic_set_nob(&iotask->executed_time, current_cio_time);
+	erts_atomic_set_nob(&iotask->executed_time, current_cio_time);
 	if (erts_port_task_schedule(id,
 				    &iotask->task,
 				    ERTS_PORT_TASK_OUTPUT,
@@ -2069,7 +2069,7 @@ send_event_tuple(struct erts_nif_select_event* e, ErtsResource* resource,
     erts_queue_message(rp, rp_locks, mp, tuple, am_system);
 
     if (rp_locks)
-        erts_smp_proc_unlock(rp, rp_locks);
+        erts_proc_unlock(rp, rp_locks);
 }
 
 
@@ -2082,7 +2082,7 @@ eready(Eterm id, ErtsDrvEventState *state, ErlDrvEventData event_data,
 				 ERTS_PORT_TASK_EVENT,
 				 current_cio_time)) {
 	ErtsIoTask *iotask = &state->driver.event->iotask;
-	erts_smp_atomic_set_nob(&iotask->executed_time, current_cio_time);
+	erts_atomic_set_nob(&iotask->executed_time, current_cio_time);
 	if (erts_port_task_schedule(id,
 				    &iotask->task,
 				    ERTS_PORT_TASK_EVENT,
@@ -2157,9 +2157,9 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
      * erts_check_io_time, since only one thread can
      * check io at a time.
      */
-    current_cio_time = erts_smp_atomic_read_dirty(&erts_check_io_time);
+    current_cio_time = erts_atomic_read_dirty(&erts_check_io_time);
     current_cio_time++;
-    erts_smp_atomic_set_relb(&erts_check_io_time, current_cio_time);
+    erts_atomic_set_relb(&erts_check_io_time, current_cio_time);
 
     check_cleanup_active_fds(current_cio_time,
                              timeout_time != ERTS_POLL_NO_TIMEOUT);
@@ -2168,11 +2168,11 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
     erts_lc_check_exact(NULL, 0); /* No locks should be locked */
 #endif
 
-    pollres_len = erts_smp_atomic32_read_dirty(&pollset.active_fd.no) + ERTS_CHECK_IO_POLL_RES_LEN;
+    pollres_len = erts_atomic32_read_dirty(&pollset.active_fd.no) + ERTS_CHECK_IO_POLL_RES_LEN;
 
     pollres = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollResFd)*pollres_len);
 
-    erts_smp_atomic_set_nob(&pollset.in_poll_wait, 1);
+    erts_atomic_set_nob(&pollset.in_poll_wait, 1);
 
     poll_ret = ERTS_CIO_POLL_WAIT(pollset.ps, pollres, &pollres_len, timeout_time);
 
@@ -2194,7 +2194,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 
 
     if (poll_ret != 0) {
-	erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0);
+	erts_atomic_set_nob(&pollset.in_poll_wait, 0);
 	forget_removed(&pollset);
 	erts_free(ERTS_ALC_T_TMP, pollres);
 	if (poll_ret == EAGAIN) {
@@ -2220,7 +2220,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 	ErtsSysFdType fd = (ErtsSysFdType) pollres[i].fd;
 	ErtsDrvEventState *state;
 
-	erts_smp_mtx_lock(fd_mtx(fd));
+	erts_mtx_lock(fd_mtx(fd));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
 	state = &drv_ev_state[ (int) fd];
@@ -2327,7 +2327,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
                 add_active_fd(state->fd);
             }
 
-            erts_smp_mtx_unlock(fd_mtx(fd));
+            erts_mtx_unlock(fd_mtx(fd));
             if (is_not_nil(in.pid)) {
                 send_event_tuple(&in, resource, am_ready_input);
             }
@@ -2374,11 +2374,11 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 	}
 
 	next_pollres:;
-	erts_smp_mtx_unlock(fd_mtx(fd));
+	erts_mtx_unlock(fd_mtx(fd));
         next_pollres_unlocked:;
     }
 
-    erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0);
+    erts_atomic_set_nob(&pollset.in_poll_wait, 0);
     erts_free(ERTS_ALC_T_TMP, pollres);
     forget_removed(&pollset);
 }
@@ -2474,16 +2474,16 @@ static int drv_ev_state_cmp(void *des1, void *des2)
 static void *drv_ev_state_alloc(void *des_tmpl)
 {
     ErtsDrvEventState *evstate;
-    erts_smp_spin_lock(&state_prealloc_lock);
+    erts_spin_lock(&state_prealloc_lock);
     if (state_prealloc_first == NULL) {
-	erts_smp_spin_unlock(&state_prealloc_lock);
+	erts_spin_unlock(&state_prealloc_lock);
 	evstate = (ErtsDrvEventState *) 
 	    erts_alloc(ERTS_ALC_T_DRV_EV_STATE, sizeof(ErtsDrvEventState));
     } else {
 	evstate = state_prealloc_first;
 	state_prealloc_first = (ErtsDrvEventState *) evstate->hb.next;
 	--num_state_prealloc;
-	erts_smp_spin_unlock(&state_prealloc_lock);
+	erts_spin_unlock(&state_prealloc_lock);
     }
     /* XXX: Already valid data if prealloced, could ignore template! */
     *evstate = *((ErtsDrvEventState *) des_tmpl);
@@ -2493,11 +2493,11 @@ static void *drv_ev_state_alloc(void *des_tmpl)
 
 static void drv_ev_state_free(void *des)
 {
-    erts_smp_spin_lock(&state_prealloc_lock);
+    erts_spin_lock(&state_prealloc_lock);
     ((ErtsDrvEventState *) des)->hb.next = &state_prealloc_first->hb;
     state_prealloc_first = (ErtsDrvEventState *) des;
     ++num_state_prealloc;
-    erts_smp_spin_unlock(&state_prealloc_lock);
+    erts_spin_unlock(&state_prealloc_lock);
 }
 #endif
 
@@ -2508,15 +2508,15 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
                                ERL_NIF_SELECT_STOP_SCHEDULED |
                                ERL_NIF_SELECT_INVALID_EVENT |
                                ERL_NIF_SELECT_FAILED)) == 0);
-    erts_smp_atomic_init_nob(&erts_check_io_time, 0);
-    erts_smp_atomic_init_nob(&pollset.in_poll_wait, 0);
+    erts_atomic_init_nob(&erts_check_io_time, 0);
+    erts_atomic_init_nob(&pollset.in_poll_wait, 0);
 
     ERTS_CIO_POLL_INIT();
     pollset.ps = ERTS_CIO_NEW_POLLSET();
 
     pollset.active_fd.six = 0;
     pollset.active_fd.eix = 0;
-    erts_smp_atomic32_init_nob(&pollset.active_fd.no, 0);
+    erts_atomic32_init_nob(&pollset.active_fd.no, 0);
     pollset.active_fd.size = ERTS_ACTIVE_FD_INC;
     pollset.active_fd.array = erts_alloc(ERTS_ALC_T_ACTIVE_FD_ARR,
 					 sizeof(ErtsSysFdType)*ERTS_ACTIVE_FD_INC);
@@ -2531,20 +2531,20 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
 
     init_removed_fd_alloc();
     pollset.removed_list = NULL;
-    erts_smp_spinlock_init(&pollset.removed_list_lock, "pollset_rm_list", NIL,
+    erts_spinlock_init(&pollset.removed_list_lock, "pollset_rm_list", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
     {
         int i;
         for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) {
-            erts_smp_mtx_init(&drv_ev_state_locks[i].lck, "drv_ev_state", make_small(i),
+            erts_mtx_init(&drv_ev_state_locks[i].lck, "drv_ev_state", make_small(i),
                 ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
         }
     }
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     max_fds = ERTS_CIO_POLL_MAX_FDS();
-    erts_smp_atomic_init_nob(&drv_ev_state_len, 0);
+    erts_atomic_init_nob(&drv_ev_state_len, 0);
     drv_ev_state = NULL;
-    erts_smp_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow", NIL,
+    erts_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
 #else
     {
@@ -2555,7 +2555,7 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
 	hf.free = &drv_ev_state_free;
 	num_state_prealloc = 0;
 	state_prealloc_first = NULL;
-	erts_smp_spinlock_init(&state_prealloc_lock,"state_prealloc", NIL,
+	erts_spinlock_init(&state_prealloc_lock,"state_prealloc", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
 
 	safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state_tab, "drv_ev_state_tab",
@@ -2582,7 +2582,7 @@ ERTS_CIO_EXPORT(erts_check_io_size)(void)
     ERTS_CIO_POLL_INFO(pollset.ps, &pi);
     res = pi.memory_size;
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    res += sizeof(ErtsDrvEventState) * erts_smp_atomic_read_nob(&drv_ev_state_len);
+    res += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state_len);
 #else
     res += safe_hash_table_sz(&drv_ev_state_tab);
     {
@@ -2590,9 +2590,9 @@ ERTS_CIO_EXPORT(erts_check_io_size)(void)
 	safe_hash_get_info(&hi, &drv_ev_state_tab);
 	res += hi.objs * sizeof(ErtsDrvEventState);
     }
-    erts_smp_spin_lock(&state_prealloc_lock);
+    erts_spin_lock(&state_prealloc_lock);
     res += num_state_prealloc * sizeof(ErtsDrvEventState);
-    erts_smp_spin_unlock(&state_prealloc_lock);
+    erts_spin_unlock(&state_prealloc_lock);
 #endif
     return res;
 }
@@ -2605,8 +2605,8 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc)
     Uint sz, *szp, *hp, **hpp, memory_size;
     Sint i;
     ErtsPollInfo pi;
-    erts_aint_t cio_time = erts_smp_atomic_read_acqb(&erts_check_io_time);
-    int active_fds = (int) erts_smp_atomic32_read_acqb(&pollset.active_fd.no);
+    erts_aint_t cio_time = erts_atomic_read_acqb(&erts_check_io_time);
+    int active_fds = (int) erts_atomic32_read_acqb(&pollset.active_fd.no);
 
     while (1) {
 	erts_aint_t post_cio_time;
@@ -2614,8 +2614,8 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc)
 
 	ERTS_CIO_POLL_INFO(pollset.ps, &pi);
 
-	post_cio_time = erts_smp_atomic_read_mb(&erts_check_io_time);
-	post_active_fds = (int) erts_smp_atomic32_read_acqb(&pollset.active_fd.no);
+	post_cio_time = erts_atomic_read_mb(&erts_check_io_time);
+	post_active_fds = (int) erts_atomic32_read_acqb(&pollset.active_fd.no);
 	if (cio_time == post_cio_time && active_fds == post_active_fds)
 	    break;
 	cio_time = post_cio_time;
@@ -2624,7 +2624,7 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc)
 
     memory_size = pi.memory_size;
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    memory_size += sizeof(ErtsDrvEventState) * erts_smp_atomic_read_nob(&drv_ev_state_len);
+    memory_size += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state_len);
 #else
     memory_size += safe_hash_table_sz(&drv_ev_state_tab);
     {
@@ -2632,9 +2632,9 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc)
 	safe_hash_get_info(&hi, &drv_ev_state_tab);
 	memory_size += hi.objs * sizeof(ErtsDrvEventState);
     }
-    erts_smp_spin_lock(&state_prealloc_lock);
+    erts_spin_lock(&state_prealloc_lock);
     memory_size += num_state_prealloc * sizeof(ErtsDrvEventState);
-    erts_smp_spin_unlock(&state_prealloc_lock);
+    erts_spin_unlock(&state_prealloc_lock);
 #endif
 
     hpp = NULL;
@@ -3046,7 +3046,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
     erts_lc_check_exact(NULL, 0); /* No locks should be locked */
 #endif
 
-    erts_smp_thr_progress_block(); /* stop the world to avoid messy locking */
+    erts_thr_progress_block(); /* stop the world to avoid messy locking */
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     counters.epep = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollEvents)*max_fds);
@@ -3059,7 +3059,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
     counters.no_driver_event_structs = 0;
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    len = erts_smp_atomic_read_nob(&drv_ev_state_len);
+    len = erts_atomic_read_nob(&drv_ev_state_len);
     for (fd = 0; fd < len; fd++) {
 	doit_erts_check_io_debug((void *) &drv_ev_state[fd], (void *) &counters);
     }
@@ -3071,7 +3071,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
     safe_hash_for_each(&drv_ev_state_tab, &doit_erts_check_io_debug, (void *) &counters);
 #endif
 
-    erts_smp_thr_progress_unblock();
+    erts_thr_progress_unblock();
 
     ciodip->no_used_fds = counters.used_fds;
     ciodip->no_driver_select_structs = counters.no_driver_select_structs;
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index d74eb764d2..777942a473 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -76,11 +76,11 @@ void erts_lcnt_update_cio_locks(int enable);
 
 #endif
 
-extern erts_smp_atomic_t erts_check_io_time;
+extern erts_atomic_t erts_check_io_time;
 
 typedef struct {
     ErtsPortTaskHandle task;
-    erts_smp_atomic_t executed_time;
+    erts_atomic_t executed_time;
 } ErtsIoTask;
 
 ERTS_GLB_INLINE void erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp);
@@ -91,8 +91,8 @@ ERTS_GLB_INLINE void
 erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp)
 {
     ErtsIoTask *itp = (ErtsIoTask *) (((char *) pthp) - offsetof(ErtsIoTask, task));
-    erts_aint_t ci_time = erts_smp_atomic_read_acqb(&erts_check_io_time);
-    erts_smp_atomic_set_relb(&itp->executed_time, ci_time);
+    erts_aint_t ci_time = erts_atomic_read_acqb(&erts_check_io_time);
+    erts_atomic_set_relb(&itp->executed_time, ci_time);
 }
 
 #endif
diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index 214ed01c82..a9c6e72c5f 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -24,7 +24,6 @@
 #define ERTS_WANT_MEM_MAPPERS
 #include "sys.h"
 #include "erl_process.h"
-#include "erl_smp.h"
 #include "atom.h"
 #include "erl_mmap.h"
 #include <stddef.h>
@@ -62,11 +61,11 @@
     (((UWord) (PTR)) - ((UWord) mm->sa.bot)			\
      < ((UWord) mm->sua.top) - ((UWord) mm->sa.bot))
 #define ERTS_MMAP_IN_SUPERALIGNED_AREA(PTR)				\
-    (ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&mm->mtx)),	\
+    (ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mm->mtx)),	\
      (((UWord) (PTR)) - ((UWord) mm->sa.bot)			\
       < ((UWord) mm->sa.top) - ((UWord) mm->sa.bot)))
 #define ERTS_MMAP_IN_SUPERUNALIGNED_AREA(PTR)				\
-    (ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&mm->mtx)),	\
+    (ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mm->mtx)),	\
      (((UWord) (PTR)) - ((UWord) mm->sua.bot)			\
       < ((UWord) mm->sua.top) - ((UWord) mm->sua.bot)))
 
@@ -199,10 +198,10 @@ static ErtsMMapOp mmap_ops[ERTS_MMAP_OP_RINGBUF_SZ];
 
 #define ERTS_MMAP_OP_LCK(RES, IN_SZ, OUT_SZ)			\
     do {							\
-	erts_smp_mtx_lock(&mm->mtx);			\
+	erts_mtx_lock(&mm->mtx);			\
 	ERTS_MMAP_OP_START((IN_SZ));				\
 	ERTS_MMAP_OP_END((RES), (OUT_SZ));			\
-	erts_smp_mtx_unlock(&mm->mtx);			\
+	erts_mtx_unlock(&mm->mtx);			\
     } while (0)
 
 #define ERTS_MUNMAP_OP(PTR, SZ)					\
@@ -221,9 +220,9 @@ static ErtsMMapOp mmap_ops[ERTS_MMAP_OP_RINGBUF_SZ];
 
 #define ERTS_MUNMAP_OP_LCK(PTR, SZ)				\
     do {							\
-	erts_smp_mtx_lock(&mm->mtx);			\
+	erts_mtx_lock(&mm->mtx);			\
 	ERTS_MUNMAP_OP((PTR), (SZ));				\
-	erts_smp_mtx_unlock(&mm->mtx);			\
+	erts_mtx_unlock(&mm->mtx);			\
     } while (0)
 
 #define ERTS_MREMAP_OP_START(OLD_PTR, OLD_SZ, IN_SZ)		\
@@ -249,10 +248,10 @@ static ErtsMMapOp mmap_ops[ERTS_MMAP_OP_RINGBUF_SZ];
 
 #define ERTS_MREMAP_OP_LCK(RES, OLD_PTR, OLD_SZ, IN_SZ, OUT_SZ)	\
     do {							\
-	erts_smp_mtx_lock(&mm->mtx);			\
+	erts_mtx_lock(&mm->mtx);			\
 	ERTS_MREMAP_OP_START((OLD_PTR), (OLD_SZ), (IN_SZ));	\
 	ERTS_MREMAP_OP_END((RES), (OUT_SZ));			\
-	erts_smp_mtx_unlock(&mm->mtx);			\
+	erts_mtx_unlock(&mm->mtx);			\
     } while (0)
 
 #define ERTS_MMAP_OP_ABORT()					\
@@ -321,7 +320,7 @@ struct ErtsMemMapper_ {
 #if HAVE_MMAP && (!defined(MAP_ANON) && !defined(MAP_ANONYMOUS))
     int mmap_fd;
 #endif
-    erts_smp_mtx_t mtx;
+    erts_mtx_t mtx;
     struct {
 	char *free_list;
 	char *unused_start;
@@ -1536,7 +1535,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep)
 	ErtsFreeSegDesc *desc;
 	Uint32 superaligned = (ERTS_MMAPFLG_SUPERALIGNED & flags);
 
-	erts_smp_mtx_lock(&mm->mtx);
+	erts_mtx_lock(&mm->mtx);
 
 	ERTS_MMAP_OP_START(*sizep);
 
@@ -1660,7 +1659,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep)
 	}
 
 	ERTS_MMAP_OP_ABORT();
-	erts_smp_mtx_unlock(&mm->mtx);
+	erts_mtx_unlock(&mm->mtx);
     }
 
 #if ERTS_HAVE_OS_MMAP
@@ -1724,13 +1723,13 @@ supercarrier_success:
 #endif
 
     ERTS_MMAP_OP_END(seg, asize);
-    erts_smp_mtx_unlock(&mm->mtx);
+    erts_mtx_unlock(&mm->mtx);
 
     *sizep = asize;
     return (void *) seg;
 
 supercarrier_reserve_failure:
-    erts_smp_mtx_unlock(&mm->mtx);
+    erts_mtx_unlock(&mm->mtx);
     *sizep = 0;
     return NULL;
 }
@@ -1760,7 +1759,7 @@ erts_munmap(ErtsMemMapper* mm, Uint32 flags, void *ptr, UWord size)
 	start = (char *) ptr;
 	end = start + size;
 
-	erts_smp_mtx_lock(&mm->mtx);
+	erts_mtx_lock(&mm->mtx);
 
 	ERTS_MUNMAP_OP(ptr, size);
 
@@ -1829,7 +1828,7 @@ erts_munmap(ErtsMemMapper* mm, Uint32 flags, void *ptr, UWord size)
 	    if (unres_sz)
 		mm->unreserve_physical(((char *) ptr) + ad_sz, unres_sz);
 
-            erts_smp_mtx_unlock(&mm->mtx);
+            erts_mtx_unlock(&mm->mtx);
 	}
     }
 }
@@ -1948,12 +1947,12 @@ erts_mremap(ErtsMemMapper* mm,
 		 ? ERTS_SUPERALIGNED_CEILING(*sizep)
 		 : ERTS_PAGEALIGNED_CEILING(*sizep));
 
-	erts_smp_mtx_lock(&mm->mtx);
+	erts_mtx_lock(&mm->mtx);
 
 	if (ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)
 	    ? (!superaligned && lookup_free_seg(&mm->sua.map, asize))
 	    : (superaligned && lookup_free_seg(&mm->sa.map, asize))) {
-	    erts_smp_mtx_unlock(&mm->mtx);
+	    erts_mtx_unlock(&mm->mtx);
 	    /*
 	     * Segment currently in wrong area (due to a previous memory
 	     * shortage), move it to the right area.
@@ -2068,7 +2067,7 @@ erts_mremap(ErtsMemMapper* mm,
 	}
 
 	ERTS_MMAP_OP_ABORT();
-	erts_smp_mtx_unlock(&mm->mtx);
+	erts_mtx_unlock(&mm->mtx);
 
 	/* Failed to resize... */
     }
@@ -2090,14 +2089,14 @@ supercarrier_resize_success:
 #endif
 
     ERTS_MREMAP_OP_END(new_ptr, asize);
-    erts_smp_mtx_unlock(&mm->mtx);
+    erts_mtx_unlock(&mm->mtx);
 
     *sizep = asize;
     return new_ptr;
 
 supercarrier_reserve_failure:
     ERTS_MREMAP_OP_END(NULL, old_size);
-    erts_smp_mtx_unlock(&mm->mtx);
+    erts_mtx_unlock(&mm->mtx);
     *sizep = old_size;
     return NULL;
     
@@ -2212,7 +2211,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable)
 	erts_exit(1, "erts_mmap: Failed to open /dev/zero\n");
 #endif
 
-    erts_smp_mtx_init(&mm->mtx, "erts_mmap", NIL,
+    erts_mtx_init(&mm->mtx, "erts_mmap", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
     if (is_first_call) {
         erts_mtx_init(&am.init_mutex, "mmap_init_atoms", NIL,
@@ -2407,7 +2406,7 @@ Eterm erts_mmap_info(ErtsMemMapper* mm,
     Eterm res = THE_NON_VALUE;
 
     if (!hpp) {
-        erts_smp_mtx_lock(&mm->mtx);
+        erts_mtx_lock(&mm->mtx);
         emis->sizes[0] = mm->size.supercarrier.total;
         emis->sizes[1] = mm->sa.top  - mm->sa.bot;
         emis->sizes[2] = mm->sua.top - mm->sua.bot;
@@ -2423,7 +2422,7 @@ Eterm erts_mmap_info(ErtsMemMapper* mm,
         emis->segs[5] = mm->sua.map.nseg;
 
         emis->os_used = mm->size.os.used;
-        erts_smp_mtx_unlock(&mm->mtx);
+        erts_mtx_unlock(&mm->mtx);
     }
 
     list[lix] = erts_mmap_info_options(mm, "option ", print_to_p, print_to_arg,
@@ -2543,14 +2542,14 @@ Eterm erts_mmap_debug_info(Process* p)
         Eterm *hp, *hp_end;
         Uint may_need;
 
-        erts_smp_mtx_lock(&mm->mtx);
+        erts_mtx_lock(&mm->mtx);
         values[0] = (UWord)mm->sa.bot;
         values[1] = (UWord)mm->sa.top;
         values[2] = (UWord)mm->sua.bot;
         values[3] = (UWord)mm->sua.top;
         sa_list = build_free_seg_list(p, &mm->sa.map);
         sua_list = build_free_seg_list(p, &mm->sua.map);
-        erts_smp_mtx_unlock(&mm->mtx);
+        erts_mtx_unlock(&mm->mtx);
 
         may_need = 4*(2+3+2) + 2*(2+3);
         hp = HAlloc(p, may_need);
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index d5b2fa87e4..bf6de9b13a 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -339,11 +339,11 @@ mseg_recreate(ErtsMsegAllctr_t *ma, Uint flags, void *old_seg, UWord old_size, U
 do {									\
     if ((MA)->is_thread_safe)						\
 	ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&(MA)->mtx)		\
-		       || erts_smp_thr_progress_is_blocking()		\
+		       || erts_thr_progress_is_blocking()		\
 		       || ERTS_IS_CRASH_DUMPING);			\
     else								\
 	ERTS_LC_ASSERT((MA)->ix == (int) erts_get_scheduler_id()	\
-		       || erts_smp_thr_progress_is_blocking()		\
+		       || erts_thr_progress_is_blocking()		\
 		       || ERTS_IS_CRASH_DUMPING);			\
 } while (0)
 #else
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index 4171770f02..7d26839b0f 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -150,9 +150,9 @@ int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds,
 #define ERTS_POLL_COALESCE_KP_RES (ERTS_POLL_USE_KQUEUE || ERTS_POLL_USE_EPOLL)
 
 #define ERTS_POLLSET_LOCK(PS) \
-  erts_smp_mtx_lock(&(PS)->mtx)
+  erts_mtx_lock(&(PS)->mtx)
 #define ERTS_POLLSET_UNLOCK(PS) \
-  erts_smp_mtx_unlock(&(PS)->mtx)
+  erts_mtx_unlock(&(PS)->mtx)
 
 #define ERTS_POLLSET_SET_POLLED_CHK(PS) \
   ((int) erts_atomic32_xchg_nob(&(PS)->polled, (erts_aint32_t) 1))
@@ -163,11 +163,11 @@ int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds,
 
 
 #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) \
-  erts_smp_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 1)
+  erts_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 1)
 #define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) \
-  erts_smp_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 0)
+  erts_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 0)
 #define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) \
-  ((int) erts_smp_atomic32_read_nob(&(PS)->have_update_requests))
+  ((int) erts_atomic32_read_nob(&(PS)->have_update_requests))
 
 #if ERTS_POLL_USE_FALLBACK
 #  if ERTS_POLL_USE_POLL
@@ -232,7 +232,7 @@ struct ErtsPollSet_ {
     ErtsPollSet next;
     int internal_fd_limit;
     ErtsFdStatus *fds_status;
-    erts_smp_atomic_t no_of_user_fds;
+    erts_atomic_t no_of_user_fds;
     int fds_status_len;
 #if ERTS_POLL_USE_KERNEL_POLL
     int kp_fd;
@@ -263,9 +263,9 @@ struct ErtsPollSet_ {
 #endif
     ErtsPollSetUpdateRequestsBlock update_requests;
     ErtsPollSetUpdateRequestsBlock *curr_upd_req_block;
-    erts_smp_atomic32_t have_update_requests;
+    erts_atomic32_t have_update_requests;
     erts_atomic32_t polled;
-    erts_smp_mtx_t mtx;
+    erts_mtx_t mtx;
     int wake_fds[2];
 #if ERTS_POLL_USE_TIMERFD
     int timer_fd;
@@ -276,9 +276,9 @@ struct ErtsPollSet_ {
     erts_atomic32_t wakeup_state;
     erts_atomic64_t timeout_time;
 #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
-    erts_smp_atomic_t no_avoided_wakeups;
-    erts_smp_atomic_t no_avoided_interrupts;
-    erts_smp_atomic_t no_interrupt_timed;
+    erts_atomic_t no_avoided_wakeups;
+    erts_atomic_t no_avoided_interrupts;
+    erts_atomic_t no_interrupt_timed;
 #endif
 };
 
@@ -288,7 +288,7 @@ static void fatal_error_async_signal_safe(char *error_str);
 
 static int max_fds = -1;
 static ErtsPollSet pollsets;
-static erts_smp_mtx_t pollsets_lock;
+static erts_mtx_t pollsets_lock;
 
 #if ERTS_POLL_USE_POLL
 
@@ -955,7 +955,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
 		    ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK;
 		    ASSERT(ps->fds_status[fd].used_events);
 		    ps->fds_status[fd].used_events = 0;
-		    erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
+		    erts_atomic_dec_nob(&ps->no_of_user_fds);
 		    update_fallback_pollset(ps, fd);
 		    ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK);
 		    break;
@@ -1005,11 +1005,11 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp)
     events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events);
     if (!events) {
 	buf[buf_len].events = POLLREMOVE;
-	erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
+	erts_atomic_dec_nob(&ps->no_of_user_fds);
     }
     else if (!ps->fds_status[fd].used_events) {
 	buf[buf_len].events = events;
-	erts_smp_atomic_inc_nob(&ps->no_of_user_fds);
+	erts_atomic_inc_nob(&ps->no_of_user_fds);
     }
     else {
 	if ((ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST)
@@ -1099,12 +1099,12 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp)
 	}
 	if (used_events) {
 	    if (!events) {
-		erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
+		erts_atomic_dec_nob(&ps->no_of_user_fds);
 	    }
 	}
 	else {
 	    if (events)
-		erts_smp_atomic_inc_nob(&ps->no_of_user_fds);
+		erts_atomic_inc_nob(&ps->no_of_user_fds);
 	}
 	ASSERT((events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0);
 	ASSERT((used_events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0);
@@ -1178,7 +1178,7 @@ update_pollset(ErtsPollSet ps, int fd)
 	    epe.data.fd = epe_templ.data.fd;
 	    res = epoll_ctl(ps->kp_fd, EPOLL_CTL_DEL, fd, &epe);
 	} while (res != 0 && errno == EINTR);
-	erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
+	erts_atomic_dec_nob(&ps->no_of_user_fds);
 	ps->fds_status[fd].used_events = 0;
     }
 
@@ -1186,11 +1186,11 @@ update_pollset(ErtsPollSet ps, int fd)
 	/* A note on EPOLL_CTL_DEL: linux kernel versions before 2.6.9
 	   need a non-NULL event pointer even though it is ignored... */
 	op = EPOLL_CTL_DEL;
-	erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
+	erts_atomic_dec_nob(&ps->no_of_user_fds);
     }
     else if (!ps->fds_status[fd].used_events) {
 	op = EPOLL_CTL_ADD;
-	erts_smp_atomic_inc_nob(&ps->no_of_user_fds);
+	erts_atomic_inc_nob(&ps->no_of_user_fds);
     }
     else {
 	op = EPOLL_CTL_MOD;
@@ -1240,7 +1240,7 @@ update_pollset(ErtsPollSet ps, int fd)
 	/* Fall through ... */
 	case EPOLL_CTL_ADD: {
 	    ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK;
-	    erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
+	    erts_atomic_dec_nob(&ps->no_of_user_fds);
 #if ERTS_POLL_USE_CONCURRENT_UPDATE
 	    if (!*update_fallback) {
 		*update_fallback = 1;
@@ -1328,7 +1328,7 @@ static int update_pollset(ErtsPollSet ps, int fd)
 #if ERTS_POLL_USE_FALLBACK
 	ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK);
 #endif
-	erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
+	erts_atomic_dec_nob(&ps->no_of_user_fds);
 	last_pix = --ps->no_poll_fds;
 	if (pix != last_pix) {
 	/* Move last pix to this pix */
@@ -1355,7 +1355,7 @@ static int update_pollset(ErtsPollSet ps, int fd)
 	    ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)
 		    || fd == ps->kp_fd);
 #endif
-	    erts_smp_atomic_inc_nob(&ps->no_of_user_fds);
+	    erts_atomic_inc_nob(&ps->no_of_user_fds);
 	    ps->fds_status[fd].pix = pix = ps->no_poll_fds++;
 	    if (pix >= ps->poll_fds_len)
 		grow_poll_fds(ps, pix);
@@ -1407,7 +1407,7 @@ static int update_pollset(ErtsPollSet ps, int fd)
 
 	if (!ps->fds_status[fd].used_events) {
 	    ASSERT(events);
-	    erts_smp_atomic_inc_nob(&ps->no_of_user_fds);
+	    erts_atomic_inc_nob(&ps->no_of_user_fds);
 #if ERTS_POLL_USE_FALLBACK
 	    ps->no_select_fds++;
 	    ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INFLBCK;
@@ -1415,7 +1415,7 @@ static int update_pollset(ErtsPollSet ps, int fd)
 	}
 	else if (!events) {
 	    ASSERT(ps->fds_status[fd].used_events);
-	    erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
+	    erts_atomic_dec_nob(&ps->no_of_user_fds);
 	    ps->fds_status[fd].events = events;
 #if ERTS_POLL_USE_FALLBACK
 	    ps->no_select_fds--;
@@ -2139,7 +2139,7 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res)
 {
     int res;
     ERTS_MSACC_PUSH_STATE_M();
-    if (erts_smp_atomic_read_nob(&ps->no_of_user_fds) == 0
+    if (erts_atomic_read_nob(&ps->no_of_user_fds) == 0
 	&& timeout_time == ERTS_POLL_NO_TIMEOUT) {
 	/* Nothing to poll and zero timeout; done... */
 	return 0;
@@ -2197,7 +2197,7 @@ check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res)
 	     * the maximum number of file descriptors in the poll set.
 	     */
 	    struct dvpoll poll_res;
-	    int nfds = (int) erts_smp_atomic_read_nob(&ps->no_of_user_fds);
+	    int nfds = (int) erts_atomic_read_nob(&ps->no_of_user_fds);
 	    nfds++; /* Wakeup pipe */
 	    timeout = (int) get_timeout(ps, 1000, timeout_time);
 	    poll_res.dp_nfds = nfds < max_res ? nfds : max_res;
@@ -2416,10 +2416,10 @@ ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps,
 #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
 	else {
 	    if (ERTS_POLLSET_IS_POLLED(ps))
-		erts_smp_atomic_inc_nob(&ps->no_avoided_wakeups);
-	    erts_smp_atomic_inc_nob(&ps->no_avoided_interrupts);
+		erts_atomic_inc_nob(&ps->no_avoided_wakeups);
+	    erts_atomic_inc_nob(&ps->no_avoided_interrupts);
 	}
-	erts_smp_atomic_inc_nob(&ps->no_interrupt_timed);
+	erts_atomic_inc_nob(&ps->no_interrupt_timed);
 #endif
     }
 }
@@ -2436,7 +2436,7 @@ ERTS_POLL_EXPORT(erts_poll_max_fds)(void)
 void
 ERTS_POLL_EXPORT(erts_poll_init)(void)
 {
-    erts_smp_mtx_init(&pollsets_lock, "pollsets_lock", NIL,
+    erts_mtx_init(&pollsets_lock, "pollsets_lock", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
     pollsets = NULL;
 
@@ -2476,7 +2476,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
     ps->internal_fd_limit = 0;
     ps->fds_status = NULL;
     ps->fds_status_len = 0;
-    erts_smp_atomic_init_nob(&ps->no_of_user_fds, 0);
+    erts_atomic_init_nob(&ps->no_of_user_fds, 0);
 #if ERTS_POLL_USE_KERNEL_POLL
     ps->kp_fd = -1;
 #if ERTS_POLL_USE_EPOLL
@@ -2538,9 +2538,9 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
     ps->update_requests.next = NULL;
     ps->update_requests.len = 0;
     ps->curr_upd_req_block = &ps->update_requests;
-    erts_smp_atomic32_init_nob(&ps->have_update_requests, 0);
+    erts_atomic32_init_nob(&ps->have_update_requests, 0);
     erts_atomic32_init_nob(&ps->polled, 0);
-    erts_smp_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO);
+    erts_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO);
     erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0);
     create_wakeup_pipe(ps);
 #if ERTS_POLL_USE_TIMERFD
@@ -2565,20 +2565,20 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
 #endif
     init_timeout_time(ps);
 #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
-    erts_smp_atomic_init_nob(&ps->no_avoided_wakeups, 0);
-    erts_smp_atomic_init_nob(&ps->no_avoided_interrupts, 0);
-    erts_smp_atomic_init_nob(&ps->no_interrupt_timed, 0);
+    erts_atomic_init_nob(&ps->no_avoided_wakeups, 0);
+    erts_atomic_init_nob(&ps->no_avoided_interrupts, 0);
+    erts_atomic_init_nob(&ps->no_interrupt_timed, 0);
 #endif
     handle_update_requests(ps);
 #if ERTS_POLL_USE_FALLBACK
     ps->fallback_used = 0;
 #endif
-    erts_smp_atomic_set_nob(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */
+    erts_atomic_set_nob(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */
 
-    erts_smp_mtx_lock(&pollsets_lock);
+    erts_mtx_lock(&pollsets_lock);
     ps->next = pollsets;
     pollsets = ps;
-    erts_smp_mtx_unlock(&pollsets_lock);
+    erts_mtx_unlock(&pollsets_lock);
 
     return ps;
 }
@@ -2623,7 +2623,7 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps)
 	    free_update_requests_block(ps, free_urqbp);
 	}
     }
-    erts_smp_mtx_destroy(&ps->mtx);
+    erts_mtx_destroy(&ps->mtx);
     if (ps->wake_fds[0] >= 0)
 	close(ps->wake_fds[0]);
     if (ps->wake_fds[1] >= 0)
@@ -2633,7 +2633,7 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps)
         close(ps->timer_fd);
 #endif
 
-    erts_smp_mtx_lock(&pollsets_lock);
+    erts_mtx_lock(&pollsets_lock);
     if (ps == pollsets)
 	pollsets = pollsets->next;
     else {
@@ -2643,7 +2643,7 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps)
 	ASSERT(ps == prev_ps->next);
 	prev_ps->next = ps->next;
     }
-    erts_smp_mtx_unlock(&pollsets_lock);
+    erts_mtx_unlock(&pollsets_lock);
 
     erts_free(ERTS_ALC_T_POLLSET, (void *) ps);
 }
@@ -2728,7 +2728,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
 
     pip->memory_size = size;
 
-    pip->poll_set_size = (int) erts_smp_atomic_read_nob(&ps->no_of_user_fds);
+    pip->poll_set_size = (int) erts_atomic_read_nob(&ps->no_of_user_fds);
     pip->poll_set_size++; /* Wakeup pipe */
 #if ERTS_POLL_USE_TIMERFD
     pip->poll_set_size++; /* timerfd */
@@ -2779,9 +2779,9 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
     pip->max_fds = max_fds;
 
 #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
-    pip->no_avoided_wakeups = erts_smp_atomic_read_nob(&ps->no_avoided_wakeups);
-    pip->no_avoided_interrupts = erts_smp_atomic_read_nob(&ps->no_avoided_interrupts);
-    pip->no_interrupt_timed = erts_smp_atomic_read_nob(&ps->no_interrupt_timed);
+    pip->no_avoided_wakeups = erts_atomic_read_nob(&ps->no_avoided_wakeups);
+    pip->no_avoided_interrupts = erts_atomic_read_nob(&ps->no_avoided_interrupts);
+    pip->no_interrupt_timed = erts_atomic_read_nob(&ps->no_interrupt_timed);
 #endif
 
     ERTS_POLLSET_UNLOCK(ps);
@@ -2980,12 +2980,12 @@ static void erts_lcnt_enable_pollset_lock_count(ErtsPollSet pollset, int enable)
 void ERTS_POLL_EXPORT(erts_lcnt_update_pollset_locks)(int enable) {
     ErtsPollSet iterator;
 
-    erts_smp_mtx_lock(&pollsets_lock);
+    erts_mtx_lock(&pollsets_lock);
 
     for(iterator = pollsets; iterator != NULL; iterator = iterator->next) {
         erts_lcnt_enable_pollset_lock_count(iterator, enable);
     }
 
-    erts_smp_mtx_unlock(&pollsets_lock);
+    erts_mtx_unlock(&pollsets_lock);
 }
 #endif
diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c
index 79f87eb3a9..09237c81ce 100644
--- a/erts/emulator/sys/common/erl_sys_common_misc.c
+++ b/erts/emulator/sys/common/erl_sys_common_misc.c
@@ -51,7 +51,7 @@
  * (often) exist two versions of erl_check_io (kernel-poll and
  * non-kernel-poll), and we dont want two versions of this variable.
  */
-erts_smp_atomic_t erts_check_io_time;
+erts_atomic_t erts_check_io_time;
 
 /* Written once and only once */
 
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 83edca1c2f..237614b0fb 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -63,7 +63,7 @@
 #include "erl_mseg.h"
 
 extern char **environ;
-erts_smp_rwmtx_t environ_rwmtx;
+erts_rwmtx_t environ_rwmtx;
 
 #define MAX_VSIZE 16		/* Max number of entries allowed in an I/O
 				 * vector sock_sendv().
@@ -92,11 +92,11 @@ extern void erts_sys_init_float(void);
 static int debug_log = 0;
 #endif
 
-static erts_smp_atomic32_t have_prepared_crash_dump;
+static erts_atomic32_t have_prepared_crash_dump;
 #define ERTS_PREPARED_CRASH_DUMP \
-  ((int) erts_smp_atomic32_xchg_nob(&have_prepared_crash_dump, 1))
+  ((int) erts_atomic32_xchg_nob(&have_prepared_crash_dump, 1))
 
-erts_smp_atomic_t sys_misc_mem_sz;
+erts_atomic_t sys_misc_mem_sz;
 
 static void smp_sig_notify(int signum);
 static int sig_notify_fds[2] = {-1, -1};
@@ -118,11 +118,11 @@ static int max_files = -1;
 /* 
  * a few variables used by the break handler 
  */
-erts_smp_atomic32_t erts_break_requested;
+erts_atomic32_t erts_break_requested;
 #define ERTS_SET_BREAK_REQUESTED \
-  erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1)
+  erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1)
 #define ERTS_UNSET_BREAK_REQUESTED \
-  erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0)
+  erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0)
 
 
 /* set early so the break handler has access to initial mode */
@@ -262,7 +262,7 @@ Uint
 erts_sys_misc_mem_sz(void)
 {
     Uint res = ERTS_CHK_IO_SZ();
-    res += erts_smp_atomic_read_mb(&sys_misc_mem_sz);
+    res += erts_atomic_read_mb(&sys_misc_mem_sz);
     return res;
 }
 
@@ -399,11 +399,11 @@ erts_sys_pre_init(void)
     erts_init_sys_time_sup();
 
 
-    erts_smp_atomic32_init_nob(&erts_break_requested, 0);
-    erts_smp_atomic32_init_nob(&have_prepared_crash_dump, 0);
+    erts_atomic32_init_nob(&erts_break_requested, 0);
+    erts_atomic32_init_nob(&have_prepared_crash_dump, 0);
 
 
-    erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0);
+    erts_atomic_init_nob(&sys_misc_mem_sz, 0);
 
     {
       /*
@@ -605,7 +605,7 @@ static void signal_notify_requested(Eterm type) {
         erts_queue_message(p, locks, msgp, msg, am_system);
 
         if (locks)
-            erts_smp_proc_unlock(p, locks);
+            erts_proc_unlock(p, locks);
         erts_proc_dec_refc(p);
     }
 }
@@ -869,7 +869,7 @@ void os_version(int *pMajor, int *pMinor, int *pBuild) {
 
 void init_getenv_state(GETENV_STATE *state)
 {
-   erts_smp_rwmtx_rlock(&environ_rwmtx);
+   erts_rwmtx_rlock(&environ_rwmtx);
    *state = NULL;
 }
 
@@ -878,7 +878,7 @@ char *getenv_string(GETENV_STATE *state0)
    char **state = (char **) *state0;
    char *cp;
 
-   ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&environ_rwmtx));
+   ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&environ_rwmtx));
 
    if (state == NULL)
       state = environ;
@@ -892,7 +892,7 @@ char *getenv_string(GETENV_STATE *state0)
 void fini_getenv_state(GETENV_STATE *state)
 {
    *state = NULL;
-   erts_smp_rwmtx_runlock(&environ_rwmtx);
+   erts_rwmtx_runlock(&environ_rwmtx);
 }
 
 void erts_do_break_handling(void)
@@ -905,7 +905,7 @@ void erts_do_break_handling(void)
      * therefore, make sure that all threads but this one are blocked before
      * proceeding!
      */
-    erts_smp_thr_progress_block();
+    erts_thr_progress_block();
 
     /* during break we revert to initial settings */
     /* this is done differently for oldshell */
@@ -933,7 +933,7 @@ void erts_do_break_handling(void)
       tcsetattr(0,TCSANOW,&temp_mode);
     }
 
-    erts_smp_thr_progress_unblock();
+    erts_thr_progress_unblock();
 }
 
 
@@ -963,14 +963,14 @@ erts_sys_putenv(char *key, char *value)
     env = erts_alloc(ERTS_ALC_T_TMP, need);
 #else
     env = erts_alloc(ERTS_ALC_T_PUTENV_STR, need);
-    erts_smp_atomic_add_nob(&sys_misc_mem_sz, need);
+    erts_atomic_add_nob(&sys_misc_mem_sz, need);
 #endif
     strcpy(env,key);
     strcat(env,"=");
     strcat(env,value);
-    erts_smp_rwmtx_rwlock(&environ_rwmtx);
+    erts_rwmtx_rwlock(&environ_rwmtx);
     res = putenv(env);
-    erts_smp_rwmtx_rwunlock(&environ_rwmtx);
+    erts_rwmtx_rwunlock(&environ_rwmtx);
 #ifdef HAVE_COPYING_PUTENV
     erts_free(ERTS_ALC_T_TMP, env);
 #endif
@@ -1017,9 +1017,9 @@ int
 erts_sys_getenv(char *key, char *value, size_t *size)
 {
     int res;
-    erts_smp_rwmtx_rlock(&environ_rwmtx);
+    erts_rwmtx_rlock(&environ_rwmtx);
     res = erts_sys_getenv__(key, value, size);
-    erts_smp_rwmtx_runlock(&environ_rwmtx);
+    erts_rwmtx_runlock(&environ_rwmtx);
     return res;
 }
 
@@ -1027,9 +1027,9 @@ int
 erts_sys_unsetenv(char *key)
 {
     int res;
-    erts_smp_rwmtx_rwlock(&environ_rwmtx);
+    erts_rwmtx_rwlock(&environ_rwmtx);
     res = unsetenv(key);
-    erts_smp_rwmtx_rwunlock(&environ_rwmtx);
+    erts_rwmtx_rwunlock(&environ_rwmtx);
     return res;
 }
 
@@ -1200,12 +1200,12 @@ void
 erl_sys_schedule(int runnable)
 {
     ERTS_CHK_IO(!runnable);
-    ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(!erts_thr_progress_is_blocking());
 }
 
 
 
-static erts_smp_tid_t sig_dispatcher_tid;
+static erts_tid_t sig_dispatcher_tid;
 
 static void
 smp_sig_notify(int signum)
@@ -1279,7 +1279,7 @@ signal_dispatcher_thread_func(void *unused)
                 }
                 signal_notify_requested(signal);
         }
-        ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
+        ERTS_LC_ASSERT(!erts_thr_progress_is_blocking());
     }
     return NULL;
 }
@@ -1287,7 +1287,7 @@ signal_dispatcher_thread_func(void *unused)
 static void
 init_smp_sig_notify(void)
 {
-    erts_smp_thr_opts_t thr_opts = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER;
     thr_opts.detached = 1;
     thr_opts.name = "sys_sig_dispatcher";
 
@@ -1299,7 +1299,7 @@ init_smp_sig_notify(void)
     }
 
     /* Start signal handler thread */
-    erts_smp_thr_create(&sig_dispatcher_tid,
+    erts_thr_create(&sig_dispatcher_tid,
 			signal_dispatcher_thread_func,
 			NULL,
 			&thr_opts);
@@ -1425,7 +1425,7 @@ erl_sys_args(int* argc, char** argv)
 {
     int i, j;
 
-    erts_smp_rwmtx_init(&environ_rwmtx, "environ", NIL,
+    erts_rwmtx_init(&environ_rwmtx, "environ", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
     i = 1;
diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c
index c451a0a674..7c9a532fed 100644
--- a/erts/emulator/sys/unix/sys_drivers.c
+++ b/erts/emulator/sys/unix/sys_drivers.c
@@ -56,9 +56,9 @@
 #include "erl_threads.h"
 
 extern char **environ;
-extern erts_smp_rwmtx_t environ_rwmtx;
+extern erts_rwmtx_t environ_rwmtx;
 
-extern erts_smp_atomic_t sys_misc_mem_sz;
+extern erts_atomic_t sys_misc_mem_sz;
 
 static Eterm forker_port;
 
@@ -343,7 +343,7 @@ static int set_blocking_data(ErtsSysDriverData *dd) {
 
     dd->blocking = erts_alloc(ERTS_ALC_T_SYS_BLOCKING, sizeof(ErtsSysBlocking));
 
-    erts_smp_atomic_add_nob(&sys_misc_mem_sz, sizeof(ErtsSysBlocking));
+    erts_atomic_add_nob(&sys_misc_mem_sz, sizeof(ErtsSysBlocking));
 
     dd->blocking->pdl = driver_pdl_create(dd->port_num);
     dd->blocking->res = 0;
@@ -386,7 +386,7 @@ create_driver_data(ErlDrvPort port_num,
         size += sizeof(ErtsSysFdData);
 
     data = erts_alloc(ERTS_ALC_T_DRV_TAB,size);
-    erts_smp_atomic_add_nob(&sys_misc_mem_sz, size);
+    erts_atomic_add_nob(&sys_misc_mem_sz, size);
 
     driver_data = (ErtsSysDriverData*)data;
     data += sizeof(*driver_data);
@@ -452,7 +452,7 @@ static char **build_unix_environment(char *block)
     char **cpp;
     char** old_env;
     
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&environ_rwmtx));
+    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&environ_rwmtx));
 
     cp = block;
     len = 0;
@@ -600,12 +600,12 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name,
 	len = CMD_LINE_PREFIX_STR_SZ + len + 1;
     }
 
-    erts_smp_rwmtx_rlock(&environ_rwmtx);
+    erts_rwmtx_rlock(&environ_rwmtx);
 
     if (opts->envir == NULL) {
 	new_environ = environ;
     } else if ((new_environ = build_unix_environment(opts->envir)) == NULL) {
-	erts_smp_rwmtx_runlock(&environ_rwmtx);
+	erts_rwmtx_runlock(&environ_rwmtx);
         close_pipes(ifd, ofd);
 	erts_free(ERTS_ALC_T_TMP, (void *) cmd_line);
 	errno = ENOMEM;
@@ -621,7 +621,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name,
         erts_free(ERTS_ALC_T_TMP, (void *) cmd_line);
         if (new_environ != environ)
             erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ);
-        erts_smp_rwmtx_runlock(&environ_rwmtx);
+        erts_rwmtx_runlock(&environ_rwmtx);
         errno = err;
         return ERL_DRV_ERROR_ERRNO;
     }
@@ -661,7 +661,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name,
 
         if (!io_vector) {
             close_pipes(ifd, ofd);
-            erts_smp_rwmtx_runlock(&environ_rwmtx);
+            erts_rwmtx_runlock(&environ_rwmtx);
             erts_free(ERTS_ALC_T_TMP, (void *) cmd_line);
             if (new_environ != environ)
                 erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ);
@@ -746,7 +746,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name,
                 erts_free(ERTS_ALC_T_TMP, io_vector);
                 if (new_environ != environ)
                     erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ);
-                erts_smp_rwmtx_runlock(&environ_rwmtx);
+                erts_rwmtx_runlock(&environ_rwmtx);
                 erts_free(ERTS_ALC_T_TMP, (void *) cmd_line);
                 errno = err;
                 return ERL_DRV_ERROR_ERRNO;
@@ -775,7 +775,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name,
     if (new_environ != environ)
 	erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ);
 
-    erts_smp_rwmtx_runlock(&environ_rwmtx);
+    erts_rwmtx_runlock(&environ_rwmtx);
 
     dd = create_driver_data(port_num, ifd[0], ofd[1], opts->packet_bytes,
                              DO_WRITE | DO_READ, opts->exit_status,
@@ -1048,8 +1048,8 @@ static void clear_fd_data(ErtsSysFdData *fdd)
 {
     if (fdd->sz > 0) {
 	erts_free(ERTS_ALC_T_FD_ENTRY_BUF, (void *) fdd->buf);
-	ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= fdd->sz);
-	erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*fdd->sz);
+	ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= fdd->sz);
+	erts_atomic_add_nob(&sys_misc_mem_sz, -1*fdd->sz);
     }
     fdd->buf = NULL;
     fdd->sz = 0;
@@ -1088,7 +1088,7 @@ static void fd_stop(ErlDrvData ev)  /* Does not close the fds */
     }
 
      erts_free(ERTS_ALC_T_DRV_TAB, dd);
-     erts_smp_atomic_add_nob(&sys_misc_mem_sz, -sz);
+     erts_atomic_add_nob(&sys_misc_mem_sz, -sz);
 }
 
 static void fd_flush(ErlDrvData ev)
@@ -1384,7 +1384,7 @@ static void ready_input(ErlDrvData e, ErlDrvEvent ready_fd)
 
             if (dd->ifd->fd < 0) {
                 driver_select(port_num, abs(dd->ifd->fd), ERL_DRV_READ|ERL_DRV_USE, 0);
-                erts_smp_atomic_add_nob(&sys_misc_mem_sz, -sizeof(ErtsSysFdData));
+                erts_atomic_add_nob(&sys_misc_mem_sz, -sizeof(ErtsSysFdData));
                 dd->ifd = NULL;
             }
 
@@ -1490,7 +1490,7 @@ static void ready_input(ErlDrvData e, ErlDrvEvent ready_fd)
 			port_inp_failure(dd, -1);
 		    }
 		    else {
-			erts_smp_atomic_add_nob(&sys_misc_mem_sz, h);
+			erts_atomic_add_nob(&sys_misc_mem_sz, h);
 			sys_memcpy(buf, cpos, bytes_left);
 			dd->ifd->buf = buf;
 			dd->ifd->sz = h;
@@ -1525,7 +1525,7 @@ static void ready_output(ErlDrvData e, ErlDrvEvent ready_fd)
                should close the output fd as soon as the command has
                been sent. */
             driver_select(ix, ready_fd, ERL_DRV_WRITE|ERL_DRV_USE, 0);
-            erts_smp_atomic_add_nob(&sys_misc_mem_sz, -sizeof(ErtsSysFdData));
+            erts_atomic_add_nob(&sys_misc_mem_sz, -sizeof(ErtsSysFdData));
             dd->ofd = NULL;
         }
         if (dd->terminating)
diff --git a/erts/emulator/sys/unix/sys_time.c b/erts/emulator/sys/unix/sys_time.c
index 60afb3b672..ef05380d17 100644
--- a/erts/emulator/sys/unix/sys_time.c
+++ b/erts/emulator/sys/unix/sys_time.c
@@ -160,7 +160,7 @@ struct sys_time_internal_state_read_mostly__ {
 
 #ifdef ERTS_SYS_TIME_INTERNAL_STATE_WRITE_FREQ__
 struct sys_time_internal_state_write_freq__ {
-    erts_smp_mtx_t mtx;
+    erts_mtx_t mtx;
 #if defined(__linux__) && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)
     ErtsMonotonicTime last_delivered;
 #endif
@@ -304,7 +304,7 @@ sys_init_time(ErtsSysInitTimeResult *init_resp)
 	    erts_sys_time_data__.r.o.os_times =
 		clock_gettime_times_verified;
 #endif
-            erts_smp_mtx_init(&internal_state.w.f.mtx, "os_monotonic_time", NIL,
+            erts_mtx_init(&internal_state.w.f.mtx, "os_monotonic_time", NIL,
                 ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
 	    internal_state.w.f.last_delivered
 		= clock_gettime_monotonic();
@@ -525,12 +525,12 @@ static ErtsMonotonicTime clock_gettime_monotonic_verified(void)
     mtime = (ErtsMonotonicTime) posix_clock_gettime(MONOTONIC_CLOCK_ID,
 						    MONOTONIC_CLOCK_ID_STR);
 
-    erts_smp_mtx_lock(&internal_state.w.f.mtx);
+    erts_mtx_lock(&internal_state.w.f.mtx);
     if (mtime < internal_state.w.f.last_delivered)
 	mtime = internal_state.w.f.last_delivered;
     else
 	internal_state.w.f.last_delivered = mtime;
-    erts_smp_mtx_unlock(&internal_state.w.f.mtx);
+    erts_mtx_unlock(&internal_state.w.f.mtx);
 
     return mtime;
 }
@@ -547,12 +547,12 @@ static void clock_gettime_times_verified(ErtsMonotonicTime *mtimep,
 			      WALL_CLOCK_ID_STR,
 			      stimep);
 
-    erts_smp_mtx_lock(&internal_state.w.f.mtx);
+    erts_mtx_lock(&internal_state.w.f.mtx);
     if (*mtimep < internal_state.w.f.last_delivered)
 	*mtimep = internal_state.w.f.last_delivered;
     else
 	internal_state.w.f.last_delivered = *mtimep;
-    erts_smp_mtx_unlock(&internal_state.w.f.mtx);
+    erts_mtx_unlock(&internal_state.w.f.mtx);
 }
 
 #endif /* defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME) */
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index 41ff186c44..0bd43bb4fb 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -286,26 +286,26 @@ struct ErtsPollSet_ {
     CRITICAL_SECTION standby_crit;     /* CS to guard the counter */
     HANDLE standby_wait_event;         /* Event signalled when counte == 0 */
     erts_atomic32_t wakeup_state;
-    erts_smp_mtx_t mtx;
+    erts_mtx_t mtx;
     erts_atomic64_t timeout_time;
 };
 
 
 #define ERTS_POLLSET_LOCK(PS) \
-  erts_smp_mtx_lock(&(PS)->mtx)
+  erts_mtx_lock(&(PS)->mtx)
 #define ERTS_POLLSET_UNLOCK(PS) \
-  erts_smp_mtx_unlock(&(PS)->mtx)
+  erts_mtx_unlock(&(PS)->mtx)
 
 
 /*
  * Communication with sys_interrupt
  */
 
-extern erts_smp_atomic32_t erts_break_requested;
+extern erts_atomic32_t erts_break_requested;
 #define ERTS_SET_BREAK_REQUESTED \
-  erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1)
+  erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1)
 #define ERTS_UNSET_BREAK_REQUESTED \
-  erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0)
+  erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0)
 
 static erts_mtx_t break_waiter_lock;
 static HANDLE break_happened_event;
@@ -1340,7 +1340,7 @@ ErtsPollSet erts_poll_create_pollset(void)
     ps->restore_events = 0;
 
     erts_atomic32_init_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
-    erts_smp_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO);
+    erts_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO);
     init_timeout_time(ps);
 
     HARDTRACEF(("Out erts_poll_create_pollset"));
@@ -1370,7 +1370,7 @@ void erts_poll_destroy_pollset(ErtsPollSet ps)
     CloseHandle(ps->event_io_ready);
     CloseHandle(ps->standby_wait_event);
     ERTS_POLLSET_UNLOCK(ps);
-    erts_smp_mtx_destroy(&ps->mtx);
+    erts_mtx_destroy(&ps->mtx);
     SEL_FREE(ERTS_ALC_T_POLLSET, (void *) ps);
     HARDTRACEF(("Out erts_poll_destroy_pollset"));
 }
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index 4c06535e4e..b23dbecbac 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -80,9 +80,9 @@ static int application_type(const wchar_t* originalName, wchar_t fullPath[MAX_PA
 
 HANDLE erts_service_event;
 
-static erts_smp_tsd_key_t win32_errstr_key;
+static erts_tsd_key_t win32_errstr_key;
 
-static erts_smp_atomic_t pipe_creation_counter;
+static erts_atomic_t pipe_creation_counter;
 
 /* Results from application_type(_w) is one of */
 #define APPL_NONE 0
@@ -141,7 +141,7 @@ static BOOL (WINAPI *fpCancelIoEx)(HANDLE,LPOVERLAPPED);
    - call erl_start() to parse arguments and do other init
 */
 
-static erts_smp_atomic_t sys_misc_mem_sz;
+static erts_atomic_t sys_misc_mem_sz;
 
 HMODULE beam_module = NULL;
 
@@ -192,7 +192,7 @@ Uint
 erts_sys_misc_mem_sz(void)
 {
     Uint res = (Uint) erts_check_io_size();
-    res += (Uint) erts_smp_atomic_read_mb(&sys_misc_mem_sz);
+    res += (Uint) erts_atomic_read_mb(&sys_misc_mem_sz);
     return res;
 }
 
@@ -659,7 +659,7 @@ new_driver_data(ErlDrvPort port_num, int packet_bytes, int wait_objs_required, i
     dp->inbuf = DRV_BUF_ALLOC(dp->inBufSize);
     if (dp->inbuf == NULL)
 	goto buf_alloc_error;
-    erts_smp_atomic_add_nob(&sys_misc_mem_sz, dp->inBufSize);
+    erts_atomic_add_nob(&sys_misc_mem_sz, dp->inBufSize);
     dp->outBufSize = 0;
     dp->outbuf = NULL;
     dp->port_num = port_num;
@@ -729,8 +729,8 @@ release_driver_data(DriverData* dp)
     }
 
     if (dp->inbuf != NULL) {
-	ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize);
-	erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->inBufSize);
+	ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize);
+	erts_atomic_add_nob(&sys_misc_mem_sz, -1*dp->inBufSize);
 	DRV_BUF_FREE(dp->inbuf);
 	dp->inBufSize = 0;
 	dp->inbuf = NULL;
@@ -738,8 +738,8 @@ release_driver_data(DriverData* dp)
     ASSERT(dp->inBufSize == 0);
 
     if (dp->outbuf != NULL) {
-	ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize);
-	erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize);
+	ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize);
+	erts_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize);
 	DRV_BUF_FREE(dp->outbuf);
 	dp->outBufSize = 0;
 	dp->outbuf = NULL;
@@ -1737,7 +1737,7 @@ static int create_pipe(HANDLE *phRead, HANDLE *phWrite, BOOL inheritRead, BOOL o
      * Otherwise, create named pipes.
      */
 
-    calls = (UWord) erts_smp_atomic_inc_read_nob(&pipe_creation_counter);
+    calls = (UWord) erts_atomic_inc_read_nob(&pipe_creation_counter);
     erts_snprintf(pipe_name, sizeof(pipe_name),
 		  "\\\\.\\pipe\\erlang44_%d_%bpu", getpid(), calls);
 
@@ -2422,7 +2422,7 @@ output(ErlDrvData drv_data, char* buf, ErlDrvSizeT len)
     }
 
     dp->outBufSize = pb+len;
-    erts_smp_atomic_add_nob(&sys_misc_mem_sz, dp->outBufSize);
+    erts_atomic_add_nob(&sys_misc_mem_sz, dp->outBufSize);
 
     /*
      * Store header bytes (if any).
@@ -2451,8 +2451,8 @@ output(ErlDrvData drv_data, char* buf, ErlDrvSizeT len)
     } else {
 	dp->out.ov.Offset += pb+len; /* For vanilla driver. */
 	/* XXX OffsetHigh should be changed too. */
-	ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize);
-	erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize);
+	ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize);
+	erts_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize);
 	DRV_BUF_FREE(dp->outbuf);
 	dp->outBufSize = 0;
 	dp->outbuf = NULL;
@@ -2563,8 +2563,8 @@ ready_input(ErlDrvData drv_data, ErlDrvEvent ready_event)
 			    error = ERROR_NOT_ENOUGH_MEMORY;
 			    break; /* Break out of loop into error handler. */
 			}
-			ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize);
-			erts_smp_atomic_add_nob(&sys_misc_mem_sz,
+			ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize);
+			erts_atomic_add_nob(&sys_misc_mem_sz,
 						dp->totalNeeded - dp->inBufSize);
 			dp->inBufSize = dp->totalNeeded;
 			dp->inbuf = new_buf;
@@ -2663,8 +2663,8 @@ ready_output(ErlDrvData drv_data, ErlDrvEvent ready_event)
 	   write... */
 	return;
     }
-    ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize);
-    erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize);
+    ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize);
+    erts_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize);
     DRV_BUF_FREE(dp->outbuf);
     dp->outBufSize = 0;
     dp->outbuf = NULL;
@@ -2812,7 +2812,7 @@ Preload* sys_preloaded(void)
 			   (num_preloaded+1)*sizeof(Preload));
     res_name = erts_alloc(ERTS_ALC_T_PRELOADED,
 			  (num_preloaded+1)*sizeof(unsigned));
-    erts_smp_atomic_add_nob(&sys_misc_mem_sz,
+    erts_atomic_add_nob(&sys_misc_mem_sz,
 			    (num_preloaded+1)*sizeof(Preload)
 			    + (num_preloaded+1)*sizeof(unsigned));
     for (i = 0; i < num_preloaded; i++) {
@@ -2825,7 +2825,7 @@ Preload* sys_preloaded(void)
 	n = GETWORD(data);
 	data += 2;
 	preloaded[i].name = erts_alloc(ERTS_ALC_T_PRELOADED, n+1);
-	erts_smp_atomic_add_nob(&sys_misc_mem_sz, n+1);
+	erts_atomic_add_nob(&sys_misc_mem_sz, n+1);
 	sys_memcpy(preloaded[i].name, data, n);
 	preloaded[i].name[n] = '\0';
 	data += n;
@@ -2907,7 +2907,7 @@ sys_get_key(int fd)
 
 char* win32_errorstr(int error)
 {
-  LPTSTR lpBufPtr = erts_smp_tsd_get(win32_errstr_key);
+  LPTSTR lpBufPtr = erts_tsd_get(win32_errstr_key);
   if (lpBufPtr) {
       LocalFree(lpBufPtr);
   }
@@ -2921,7 +2921,7 @@ char* win32_errorstr(int error)
 		0,
 		NULL);
   SetLastError(error);
-  erts_smp_tsd_set(win32_errstr_key,lpBufPtr);
+  erts_tsd_set(win32_errstr_key,lpBufPtr);
   return lpBufPtr;
 }
 
@@ -3170,7 +3170,7 @@ erts_sys_pre_init(void)
 
     erts_init_sys_time_sup();
 
-    erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0);
+    erts_atomic_init_nob(&sys_misc_mem_sz, 0);
 }
 
 void noinherit_std_handle(DWORD type)
@@ -3190,9 +3190,9 @@ void erl_sys_init(void)
     noinherit_std_handle(STD_INPUT_HANDLE);
     noinherit_std_handle(STD_ERROR_HANDLE);
 
-    erts_smp_tsd_key_create(&win32_errstr_key,"win32_errstr_key");
+    erts_tsd_key_create(&win32_errstr_key,"win32_errstr_key");
     InitializeCriticalSection(&htbc_lock);
-    erts_smp_atomic_init_nob(&pipe_creation_counter,0);
+    erts_atomic_init_nob(&pipe_creation_counter,0);
     /*
      * Test if we have named pipes or not.
      */
@@ -3269,6 +3269,6 @@ void
 erl_sys_schedule(int runnable)
 {
     erts_check_io(!runnable);
-    ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
+    ERTS_LC_ASSERT(!erts_thr_progress_is_blocking());
 }
 
diff --git a/erts/emulator/sys/win32/sys_env.c b/erts/emulator/sys/win32/sys_env.c
index 8fcee1cbb6..5792816267 100644
--- a/erts/emulator/sys/win32/sys_env.c
+++ b/erts/emulator/sys/win32/sys_env.c
@@ -32,12 +32,12 @@ static WCHAR **env_to_arg(WCHAR *env);
 static WCHAR **find_arg(WCHAR **arg, WCHAR *str);
 static int compare(const void *a, const void *b);
 
-static erts_smp_rwmtx_t environ_rwmtx;
+static erts_rwmtx_t environ_rwmtx;
 
 void
 erts_sys_env_init(void)
 {
-    erts_smp_rwmtx_init(&environ_rwmtx, "environ", NIL,
+    erts_rwmtx_init(&environ_rwmtx, "environ", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 }
 
@@ -45,10 +45,10 @@ int
 erts_sys_putenv_raw(char *key, char *value)
 {
     int res;
-    erts_smp_rwmtx_rwlock(&environ_rwmtx);
+    erts_rwmtx_rwlock(&environ_rwmtx);
     res = (SetEnvironmentVariable((LPCTSTR) key,
 				  (LPCTSTR) value) ? 0 : 1);
-    erts_smp_rwmtx_rwunlock(&environ_rwmtx);
+    erts_rwmtx_rwunlock(&environ_rwmtx);
     return res;
 }
 
@@ -58,10 +58,10 @@ erts_sys_putenv(char *key, char *value)
     int res;
     WCHAR *wkey = (WCHAR *) key;
     WCHAR *wvalue = (WCHAR *) value;
-    erts_smp_rwmtx_rwlock(&environ_rwmtx);
+    erts_rwmtx_rwlock(&environ_rwmtx);
     res = (SetEnvironmentVariableW(wkey,
 				   wvalue) ? 0 : 1);
-    erts_smp_rwmtx_rwunlock(&environ_rwmtx);
+    erts_rwmtx_rwunlock(&environ_rwmtx);
     return res;
 }
 
@@ -76,12 +76,12 @@ erts_sys_getenv(char *key, char *value, size_t *size)
     DWORD wsize = *size / (sizeof(WCHAR) / sizeof(char));
 
     SetLastError(0);
-    erts_smp_rwmtx_rlock(&environ_rwmtx);
+    erts_rwmtx_rlock(&environ_rwmtx);
     new_size = GetEnvironmentVariableW(wkey,
 				       wvalue,
 				      (DWORD) wsize);
     res = !new_size && GetLastError() == ERROR_ENVVAR_NOT_FOUND ? -1 : 0;
-    erts_smp_rwmtx_runlock(&environ_rwmtx);
+    erts_rwmtx_runlock(&environ_rwmtx);
     if (res < 0)
 	return res;
     res = new_size > wsize ? 1 : 0;
@@ -111,22 +111,22 @@ int
 erts_sys_getenv_raw(char *key, char *value, size_t *size)
 {
     int res;
-    erts_smp_rwmtx_rlock(&environ_rwmtx);
+    erts_rwmtx_rlock(&environ_rwmtx);
     res = erts_sys_getenv__(key, value, size);
-    erts_smp_rwmtx_runlock(&environ_rwmtx);
+    erts_rwmtx_runlock(&environ_rwmtx);
     return res;
 }
 
 void init_getenv_state(GETENV_STATE *state)
 {
-    erts_smp_rwmtx_rlock(&environ_rwmtx);
+    erts_rwmtx_rlock(&environ_rwmtx);
     state->environment_strings = GetEnvironmentStringsW();
     state->next_string = state->environment_strings;
 }
 
 char *getenv_string(GETENV_STATE *state)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&environ_rwmtx));
+    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&environ_rwmtx));
     if (state->next_string[0] == L'\0') {
 	return NULL;
     } else {
@@ -140,7 +140,7 @@ void fini_getenv_state(GETENV_STATE *state)
 {
     FreeEnvironmentStringsW(state->environment_strings);
     state->environment_strings = state->next_string = NULL;
-    erts_smp_rwmtx_runlock(&environ_rwmtx);
+    erts_rwmtx_runlock(&environ_rwmtx);
 }
 
 int erts_sys_unsetenv(char *key)
@@ -149,7 +149,7 @@ int erts_sys_unsetenv(char *key)
     WCHAR *wkey = (WCHAR *) key;
 
     SetLastError(0);
-    erts_smp_rwmtx_rlock(&environ_rwmtx);
+    erts_rwmtx_rlock(&environ_rwmtx);
     GetEnvironmentVariableW(wkey,
                             NULL,
                             0);
@@ -157,7 +157,7 @@ int erts_sys_unsetenv(char *key)
         res = (SetEnvironmentVariableW(wkey,
                                        NULL) ? 0 : 1);
     }
-    erts_smp_rwmtx_runlock(&environ_rwmtx);
+    erts_rwmtx_runlock(&environ_rwmtx);
     return res;
 }
 
@@ -171,12 +171,12 @@ win_build_environment(char* new_env)
 
 	tmp_new = (WCHAR *) new_env;
 	
-	erts_smp_rwmtx_rlock(&environ_rwmtx);
+	erts_rwmtx_rlock(&environ_rwmtx);
 	tmp = GetEnvironmentStringsW();
 	merged = merge_environment(tmp, tmp_new);
 
 	FreeEnvironmentStringsW(tmp);
-	erts_smp_rwmtx_runlock(&environ_rwmtx);
+	erts_rwmtx_runlock(&environ_rwmtx);
 	return (char *) merged;
     }
 }
diff --git a/erts/emulator/sys/win32/sys_interrupt.c b/erts/emulator/sys/win32/sys_interrupt.c
index 4632ab52b1..02aa50500f 100644
--- a/erts/emulator/sys/win32/sys_interrupt.c
+++ b/erts/emulator/sys/win32/sys_interrupt.c
@@ -35,11 +35,11 @@
 #  define WIN_SYS_INLINE __forceinline
 #endif
 
-erts_smp_atomic32_t erts_break_requested;
+erts_atomic32_t erts_break_requested;
 #define ERTS_SET_BREAK_REQUESTED \
-  erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1)
+  erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1)
 #define ERTS_UNSET_BREAK_REQUESTED \
-  erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0)
+  erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0)
 
 extern int nohup;
 HANDLE erts_sys_break_event = NULL;
@@ -51,14 +51,14 @@ void erts_do_break_handling(void)
      * therefore, make sure that all threads but this one are blocked before
      * proceeding!
      */
-    erts_smp_thr_progress_block();
+    erts_thr_progress_block();
     /* call the break handling function, reset the flag */
     do_break();
 
     ResetEvent(erts_sys_break_event);
     ERTS_UNSET_BREAK_REQUESTED;
 
-    erts_smp_thr_progress_unblock();
+    erts_thr_progress_unblock();
 }
 
 
diff --git a/erts/emulator/sys/win32/sys_time.c b/erts/emulator/sys/win32/sys_time.c
index 359010e9f1..25c2ad385c 100644
--- a/erts/emulator/sys/win32/sys_time.c
+++ b/erts/emulator/sys/win32/sys_time.c
@@ -95,7 +95,7 @@ struct sys_time_internal_state_read_mostly__ {
 };
 
 struct sys_time_internal_state_write_freq__ {
-    erts_smp_mtx_t mtime_mtx;
+    erts_mtx_t mtime_mtx;
     ULONGLONG wrap;
     ULONGLONG last_tick_count;
 };
@@ -294,7 +294,7 @@ sys_init_time(ErtsSysInitTimeResult *init_resp)
     module = GetModuleHandle(kernel_dll_name);
     if (!module) {
     get_tick_count:
-        erts_smp_mtx_init(&internal_state.w.f.mtime_mtx, "os_monotonic_time", NIL,
+        erts_mtx_init(&internal_state.w.f.mtime_mtx, "os_monotonic_time", NIL,
             ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 	internal_state.w.f.wrap = 0;
 	internal_state.w.f.last_tick_count = 0;
-- 
cgit v1.2.3


From 0322232e3603ae098177e7fe5fcf81f2ed58ea00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 26 Jul 2017 16:09:55 +0200
Subject: Check for overflow when appending binaries, and error out with
 system_limit

This fixes the following bug:

A = <<0:((1 bsl 32)-8)>>,
B = <<2, 3>>.
B =:= <<A/binary,1,B/binary>>. %% Evaluated to true...
---
 erts/emulator/beam/erl_bits.c             | 31 ++++++++++++++++++++++++++++---
 erts/emulator/test/bs_construct_SUITE.erl | 28 +++++++++++++++++++++++++++-
 2 files changed, 55 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 01734c55d7..c9e09501ff 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1324,7 +1324,14 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term,
 	    goto badarg;
 	}
     }
+
+    if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) {
+        c_p->freason = SYSTEM_LIMIT;
+        return THE_NON_VALUE;
+    }
+
     used_size_in_bits = erts_bin_offset + build_size_in_bits;
+
     sb->is_writable = 0;	/* Make sure that no one else can write. */
     pb->size = NBYTES(used_size_in_bits);
     pb->flags |= PB_ACTIVE_WRITER;
@@ -1398,9 +1405,21 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term,
 		goto badarg;
 	    }
 	}
-	used_size_in_bits = erts_bin_offset + build_size_in_bits;
-	used_size_in_bytes = NBYTES(used_size_in_bits);
-	bin_size = 2*used_size_in_bytes;
+
+        if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) {
+            c_p->freason = SYSTEM_LIMIT;
+            return THE_NON_VALUE;
+        }
+
+        used_size_in_bits = erts_bin_offset + build_size_in_bits;
+        used_size_in_bytes = NBYTES(used_size_in_bits);
+
+        if(used_size_in_bits < (ERTS_UINT_MAX / 2)) {
+            bin_size = 2 * used_size_in_bytes;
+        } else {
+            bin_size = NBYTES(ERTS_UINT_MAX);
+        }
+
 	bin_size = (bin_size < 256) ? 256 : bin_size;
 
 	/*
@@ -1491,6 +1510,12 @@ erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit)
      * Calculate new size in bytes.
      */
     erts_bin_offset = 8*sb->size + sb->bitsize;
+
+    if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) {
+        p->freason = SYSTEM_LIMIT;
+        return THE_NON_VALUE;
+    }
+
     pos_in_bits_after_build = erts_bin_offset + build_size_in_bits;
     pb->size = (pos_in_bits_after_build+7) >> 3;
     pb->flags |= PB_ACTIVE_WRITER;
diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl
index cadb30e1a4..06f3bc2ff8 100644
--- a/erts/emulator/test/bs_construct_SUITE.erl
+++ b/erts/emulator/test/bs_construct_SUITE.erl
@@ -911,5 +911,31 @@ append_unit_8(Bin) ->
 append_unit_16(Bin) ->
     <<Bin/binary-unit:16,0:1>>.
 
-    
+bs_add_overflow(Config) ->
+    case erlang:system_info(wordsize) of
+        8 ->
+            {skip, "64-bit architecture"};
+        4 ->
+            {'EXIT', {system_limit, _}} = (catch bs_add_overflow_signed()),
+            {'EXIT', {system_limit, _}} = (catch bs_add_overflow_unsigned()),
+            ok
+    end.
+
+bs_add_overflow_signed() ->
+    %% Produce a large result of bs_add that, if cast to signed int, would
+    %% overflow into a negative number that fits a smallnum.
+    Large = <<0:((1 bsl 30)-1)>>,
+    <<Large/bits, Large/bits, Large/bits, Large/bits,
+      Large/bits, Large/bits, Large/bits, Large/bits,
+      Large/bits>>.
+
+bs_add_overflow_unsigned() ->
+    %% Produce a large result of bs_add that goes beyond the limit of an
+    %% unsigned word. This used to succeed but produced an incorrect result
+    %% where B =:= C!
+    A = <<0:((1 bsl 32)-8)>>,
+    B = <<2, 3>>,
+    C = <<A/binary,1,B/binary>>,
+    true = byte_size(B) < byte_size(C).
+
 id(I) -> I.
-- 
cgit v1.2.3


From e04e011cc0335f1ccd964c5197c3122f3ee8259e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 18 May 2017 05:11:40 +0200
Subject: Simplify specifying implementation of instructions

Eliminate the need to write pre-processor macros for each instruction.

Instead allow the implementation of instruction to be written in
C directly in the .tab files. Rewrite all existing macros in this
way and remove the %macro directive.
---
 erts/emulator/Makefile.in        |   2 +-
 erts/emulator/beam/beam_emu.c    | 674 +--------------------------------------
 erts/emulator/beam/bs_instrs.tab | 179 +++++++++++
 erts/emulator/beam/instrs.tab    | 594 ++++++++++++++++++++++++++++++++++
 erts/emulator/beam/ops.tab       | 140 +-------
 erts/emulator/utils/beam_makeops | 407 ++++++++++++++---------
 6 files changed, 1055 insertions(+), 941 deletions(-)
 create mode 100644 erts/emulator/beam/bs_instrs.tab
 create mode 100644 erts/emulator/beam/instrs.tab

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 521fc46b47..015a3a42ed 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -31,7 +31,7 @@ USE_VM_PROBES=@USE_VM_PROBES@
 LIBS = @LIBS@
 Z_LIB=@Z_LIB@
 NO_INLINE_FUNCTIONS=false
-OPCODE_TABLES = $(ERL_TOP)/lib/compiler/src/genop.tab beam/ops.tab
+OPCODE_TABLES = $(ERL_TOP)/lib/compiler/src/genop.tab beam/ops.tab beam/instrs.tab beam/bs_instrs.tab
 
 DEBUG_CFLAGS = @DEBUG_CFLAGS@
 CONFIGURE_CFLAGS = @CFLAGS@
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 25e16764ab..a5a462944b 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -296,79 +296,16 @@ void** beam_ops;
      PROCESS_MAIN_CHK_LOCKS((P));					\
      ERTS_UNREQ_PROC_MAIN_LOCK((P))
 
-#define db(N) (N)
 #define tb(N) (N)
 #define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N)))
 #define yb(N) (*(Eterm *) (((unsigned char *)E) + (N)))
-#define fb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N)))
+#define lb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N)))
 #define Qb(N) (N)
 #define Ib(N) (N)
 #define x(N) reg[N]
 #define y(N) E[N]
 #define r(N) x(N)
 
-/*
- * Makes sure that there are StackNeed + HeapNeed + 1 words available
- * on the combined heap/stack segment, then allocates StackNeed + 1
- * words on the stack and saves CP.
- *
- * M is number of live registers to preserve during garbage collection
- */
-
-#define AH(StackNeed, HeapNeed, M) \
-  do { \
-     int needed; \
-     needed = (StackNeed) + 1; \
-     if (E - HTOP < (needed + (HeapNeed))) { \
-           SWAPOUT; \
-           PROCESS_MAIN_CHK_LOCKS(c_p); \
-           FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), \
-						 reg, (M), FCALLS);	\
-           ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
-           PROCESS_MAIN_CHK_LOCKS(c_p); \
-           SWAPIN; \
-     } \
-     E -= needed; \
-     SAVE_CP(E); \
-  } while (0)
-
-#define Allocate(Ns, Live) AH(Ns, 0, Live)
-
-#define AllocateZero(Ns, Live)             \
- do { Eterm* ptr;                          \
-      int i = (Ns);                        \
-      AH(i, 0, Live);                      \
-      for (ptr = E + i; ptr > E; ptr--) {  \
-	 make_blank(*ptr);                 \
-     }                                     \
-  } while (0)
-
-#define AllocateHeap(Ns, Nh, Live) AH(Ns, Nh, Live)
-
-#define AllocateHeapZero(Ns, Nh, Live)     \
- do { Eterm* ptr;                          \
-      int i = (Ns);                        \
-      AH(i, Nh, Live);                     \
-      for (ptr = E + i; ptr > E; ptr--) {  \
-	 make_blank(*ptr);                 \
-     }                                     \
-  } while (0)
-
-#define AllocateInit(Ns, Live, Y) \
-  do { AH(Ns, 0, Live); make_blank(Y); } while (0)
-
-/*
- * Like the AH macro, but allocates no additional heap space.
- */
-
-#define A(StackNeed, M) AH(StackNeed, 0, M)
-
-#define D(N)             \
-     RESTORE_CP(E);      \
-     E += (N) + 1;
-
-
-
 #define TestBinVHeap(VNh, Nh, Live)                             		\
   do {                                                          		\
     unsigned need = (Nh);                                       		\
@@ -426,32 +363,6 @@ void** beam_ops;
     HEAP_SPACE_VERIFIED(need);                                      \
   } while (0)
 
-#define TestHeapPutList(Need, Reg)		\
-  do {						\
-     TestHeap((Need), 1);			\
-     PutList(Reg, r(0), r(0));                  \
-     CHECK_TERM(r(0));				\
-  } while (0)
-
-#define Init(N) make_blank(yb(N))
-
-#define Init2(Y1, Y2) do { make_blank(Y1); make_blank(Y2); } while (0)
-#define Init3(Y1, Y2, Y3) \
-   do { make_blank(Y1); make_blank(Y2); make_blank(Y3); } while (0)
-
-#define MakeFun(FunP, NumFree)					\
-  do {								\
-     HEAVY_SWAPOUT;						\
-     r(0) = new_fun(c_p, reg, (ErlFunEntry *) FunP, NumFree);	\
-     HEAVY_SWAPIN;						\
-  } while (0)
-
-#define PutTuple(Dst, Arity)			\
- do {						\
-   Dst = make_tuple(HTOP);			\
-   pt_arity = (Arity);				\
- } while (0)
-
 /*
  * Check that we haven't used the reductions and jump to function pointed to by
  * the I register.  If we are out of reductions, do a context switch.
@@ -518,9 +429,6 @@ void** beam_ops;
 #  define Dispatchfun() DispatchMacroFun()
 #endif
 
-#define Self(R) R = c_p->common.id
-#define Node(R) R = erts_this_node->sysname
-
 #define Arg(N)       I[(N)+1]
 #define Next(N)                \
     I += (N) + 1;              \
@@ -556,90 +464,6 @@ void** beam_ops;
      GetR((N)+1, Dst2);            \
    } while (0)
 
-#define PutList(H, T, Dst)                      \
-  do {                                          \
-   HTOP[0] = (H); HTOP[1] = (T);                \
-   Dst = make_list(HTOP);                       \
-   HTOP += 2;                                   \
-  } while (0)
-
-#define Swap(R1, R2)				\
-  do {						\
-    Eterm V = R1;				\
-    R1 = R2;					\
-    R2 = V;					\
-  } while (0)
-
-#define SwapTemp(R1, R2, Tmp)			\
-  do {						\
-    Eterm V = R1;				\
-    R1 = R2;					\
-    R2 = Tmp = V;				\
-  } while (0)
-
-#define Move(Src, Dst) Dst = (Src)
-
-#define Move2Par(S1, D1, S2, D2)		\
-  do {						\
-      Eterm V1, V2;				\
-      V1 = (S1); V2 = (S2); D1 = V1; D2 = V2;	\
-  } while (0)
-
-#define MoveShift(Src, SD, D)			\
-  do {						\
-    Eterm V;					\
-    V = Src; D = SD; SD = V;			\
-  } while (0)
-
-#define MoveDup(Src, D1, D2)			\
-  do {						\
-    D1 = D2 = (Src);				\
-  } while (0)
-
-#define Move3(S1, D1, S2, D2, S3, D3) D1 = (S1); D2 = (S2); D3 = (S3)
-
-#define MoveWindow3(S1, S2, S3, D)		\
-  do {						\
-      Eterm xt0, xt1, xt2;			\
-      Eterm *y = &D;				\
-      xt0  = S1;				\
-      xt1  = S2;				\
-      xt2  = S3;				\
-      y[0] = xt0;				\
-      y[1] = xt1;				\
-      y[2] = xt2;				\
- } while (0)
-
-#define MoveWindow4(S1, S2, S3, S4, D)		\
-  do {						\
-      Eterm xt0, xt1, xt2, xt3;			\
-      Eterm *y = &D;				\
-      xt0  = S1;				\
-      xt1  = S2;				\
-      xt2  = S3;				\
-      xt3  = S4;				\
-      y[0] = xt0;				\
-      y[1] = xt1;				\
-      y[2] = xt2;				\
-      y[3] = xt3;				\
- } while (0)
-
-#define MoveWindow5(S1, S2, S3, S4, S5, D)	\
-  do {						\
-      Eterm xt0, xt1, xt2, xt3, xt4;		\
-      Eterm *y = &D;				\
-      xt0  = S1;				\
-      xt1  = S2;				\
-      xt2  = S3;				\
-      xt3  = S4;				\
-      xt4  = S5;				\
-      y[0] = xt0;				\
-      y[1] = xt1;				\
-      y[2] = xt2;				\
-      y[3] = xt3;				\
-      y[4] = xt4;				\
- } while (0)
-
 #define DispatchReturn                          \
 do {                                            \
     if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
@@ -653,225 +477,14 @@ do {                                            \
     }						\
 } while (0)
 
-#define MoveReturn(Src)				\
-    x(0) = (Src);				\
-    I = c_p->cp;				\
-    ASSERT(VALID_INSTR(*c_p->cp));		\
-    c_p->cp = 0;				\
-    CHECK_TERM(r(0));				\
-    DispatchReturn
-
-#define DeallocateReturn(Deallocate)       \
-  do {                                     \
-    int words_to_pop = (Deallocate);       \
-    SET_I((BeamInstr *) cp_val(*E));       \
-    E = ADD_BYTE_OFFSET(E, words_to_pop);  \
-    CHECK_TERM(r(0));                      \
-    DispatchReturn;                        \
-  } while (0)
-
-#define MoveDeallocateReturn(Src, Deallocate)	\
-    x(0) = (Src);				\
-    DeallocateReturn(Deallocate)
-
-#define MoveCall(Src, CallDest, Size)		\
-    x(0) = (Src);				\
-    SET_CP(c_p, I+Size+1);			\
-    SET_I((BeamInstr *) CallDest);		\
-    Dispatch();
-
-#define MoveCallLast(Src, CallDest, Deallocate)	\
-    x(0) = (Src);				\
-    RESTORE_CP(E);				\
-    E = ADD_BYTE_OFFSET(E, (Deallocate));	\
-    SET_I((BeamInstr *) CallDest);		\
-    Dispatch();
-
-#define MoveCallOnly(Src, CallDest)		\
-    x(0) = (Src);				\
-    SET_I((BeamInstr *) CallDest);		\
-    Dispatch();
-
-#define MoveJump(Src)				\
-     r(0) = (Src);				\
-     SET_I((BeamInstr *) Arg(0));		\
-     Goto(*I);
-
-#define GetList(Src, H, T)			\
-  do {						\
-    Eterm* tmp_ptr = list_val(Src);		\
-    Eterm hd, tl;				\
-    hd = CAR(tmp_ptr);				\
-    tl = CDR(tmp_ptr);				\
-    H = hd; T = tl;				\
-  } while (0)
-
-#define GetTupleElement(Src, Element, Dest)		\
-  do {							\
-    Eterm* src;						\
-    src = ADD_BYTE_OFFSET(tuple_val(Src), (Element));	\
-    (Dest) = *src;					\
-  } while (0)
-
-#define GetTupleElement2(Src, Element, Dest)		\
-  do {							\
-    Eterm* src;						\
-    Eterm* dst;						\
-    Eterm E1, E2;					\
-    src = ADD_BYTE_OFFSET(tuple_val(Src), (Element));	\
-    dst = &(Dest);					\
-    E1 = src[0];					\
-    E2 = src[1];					\
-    dst[0] = E1;					\
-    dst[1] = E2;					\
-  } while (0)
-
-#define GetTupleElement2Y(Src, Element, D1, D2)		\
-  do {							\
-    Eterm* src;						\
-    Eterm E1, E2;					\
-    src = ADD_BYTE_OFFSET(tuple_val(Src), (Element));	\
-    E1 = src[0];					\
-    E2 = src[1];					\
-    D1 = E1;						\
-    D2 = E2;						\
-  } while (0)
-
-#define GetTupleElement3(Src, Element, Dest)		\
-  do {							\
-    Eterm* src;						\
-    Eterm* dst;						\
-    Eterm E1, E2, E3;					\
-    src = ADD_BYTE_OFFSET(tuple_val(Src), (Element));	\
-    dst = &(Dest);					\
-    E1 = src[0];					\
-    E2 = src[1];					\
-    E3 = src[2];					\
-    dst[0] = E1;					\
-    dst[1] = E2;					\
-    dst[2] = E3;					\
-  } while (0)
-
-#define EqualImmed(X, Y, Action) if (X != Y) { Action; }
-#define NotEqualImmed(X, Y, Action) if (X == Y) { Action; }
-#define EqualExact(X, Y, Action) if (!EQ(X,Y)) { Action; }
-#define NotEqualExact(X, Y, Action) if (EQ(X,Y)) { Action; }
-#define Equal(X, Y, Action) CMP_EQ_ACTION(X,Y,Action)
-#define NotEqual(X, Y, Action) CMP_NE_ACTION(X,Y,Action)
-#define IsLessThan(X, Y, Action) CMP_LT_ACTION(X,Y,Action)
-#define IsGreaterEqual(X, Y, Action) CMP_GE_ACTION(X,Y,Action)
-
-#define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; }
-
-#define IsInteger(Src, Fail) if (is_not_integer(Src)) { Fail; }
-
-#define IsNumber(X, Fail) if (is_not_integer(X) && is_not_float(X)) { Fail; }
-
-#define IsAtom(Src, Fail) if (is_not_atom(Src)) { Fail; }
-
-#define IsIntegerAllocate(Src, Need, Alive, Fail)  \
-    if (is_not_integer(Src)) { Fail; }             \
-    A(Need, Alive)
-
-#define IsNil(Src, Fail) if (is_not_nil(Src)) { Fail; }
-
-#define IsList(Src, Fail) if (is_not_list(Src) && is_not_nil(Src)) { Fail; }
-
-#define IsNonemptyList(Src, Fail) if (is_not_list(Src)) { Fail; }
-
-#define IsNonemptyListAllocate(Src, Need, Alive, Fail)  \
-    if (is_not_list(Src)) { Fail; }                     \
-    A(Need, Alive)
-
-#define IsNonemptyListTestHeap(Need, Alive, Fail)  \
-    if (is_not_list(x(0))) { Fail; }			\
-    TestHeap(Need, Alive)
-
-#define IsNonemptyListGetList(Src, H, T, Fail)	\
-    if (is_not_list(Src)) {			\
-         Fail;					\
-    } else {					\
-       Eterm* tmp_ptr = list_val(Src);		\
-       Eterm hd, tl;				\
-       hd = CAR(tmp_ptr);			\
-       tl = CDR(tmp_ptr);			\
-       H = hd; T = tl;				\
-    }
-
-#define IsTuple(X, Action) if (is_not_tuple(X)) Action
-
-#define IsArity(Pointer, Arity, Fail)		\
-    if (*tuple_val(Pointer) != (Arity)) {	\
-        Fail;					\
-    }
-
-#define IsMap(Src, Fail) if (!is_map(Src)) { Fail; }
-
-#define GetMapElement(Src, Key, Dst, Fail)	\
-  do {						\
-     Eterm _res = get_map_element(Src, Key);	\
-     if (is_non_value(_res)) {			\
-        Fail;					\
-     }						\
-     Dst = _res;				\
-  } while (0)
-
-#define GetMapElementHash(Src, Key, Hx, Dst, Fail)	\
-  do {							\
-     Eterm _res = get_map_element_hash(Src, Key, Hx);	\
-     if (is_non_value(_res)) {				\
-        Fail;						\
-     }							\
-     Dst = _res;					\
-  } while (0)
-
-#define IsFunction(X, Action)			\
-  do {						\
-     if ( !(is_any_fun(X)) ) {			\
-          Action;				\
-     }						\
-  } while (0)
-
-#define IsFunction2(F, A, Action)			\
-  do {							\
-     if (erl_is_function(c_p, F, A) != am_true ) {	\
-          Action;					\
-     }							\
-  } while (0)
-
 #ifdef DEBUG
-#define IsTupleOfArity(Src, Arityval, Fail)			\
-  do {								\
-    if (!(is_tuple(Src) && *tuple_val(Src) == Arityval)) {	\
-        Fail;							\
-    }								\
-  } while (0)
+/* Better static type testing by the C compiler */
+#  define BEAM_IS_TUPLE(Src) is_tuple(Src)
 #else
-#define IsTupleOfArity(Src, Arityval, Fail)			\
-  do {								\
-    if (!(is_boxed(Src) && *tuple_val(Src) == Arityval)) {	\
-        Fail;							\
-    }								\
-  } while (0)
+/* Better performance */
+# define BEAM_IS_TUPLE(Src) is_boxed(Src)
 #endif
 
-#define IsTaggedTuple(Src,Arityval,Tag,Fail) \
-  do {                                       \
-    if (!(is_tuple(Src) &&                   \
-         (tuple_val(Src))[0] == Arityval &&  \
-         (tuple_val(Src))[1] == Tag)) {      \
-        Fail;                                \
-    }                                        \
-  } while (0)
-
-#define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; }
-
-#define IsBinary(Src, Fail) \
- if (is_not_binary(Src) || binary_bitsize(Src) != 0) { Fail; }
-
-#define IsBitstring(Src, Fail) \
-  if (is_not_binary(Src)) { Fail; }
-
 #if defined(ARCH_64)
 #define BsSafeMul(A, B, Fail, Target)		\
    do { Uint64 _res = (A) * (B);		\
@@ -916,145 +529,6 @@ do {                                            \
       Target = _uint_size * Unit;				\
    } while (0)
 
-#define BsGetFloat2(Ms, Live, Sz, Flags, Dst, Fail)                     \
- do {									\
-   ErlBinMatchBuffer *_mb;						\
-   Eterm _result; Sint _size;						\
-   if (!is_small(Sz) || (_size = unsigned_val(Sz)) > 64) { Fail; }	\
-   _size *= ((Flags) >> 3);						\
-   TestHeap(FLOAT_SIZE_OBJECT, Live);					\
-   _mb = ms_matchbuffer(Ms);						\
-   LIGHT_SWAPOUT;							\
-   _result = erts_bs_get_float_2(c_p, _size, (Flags), _mb);		\
-   LIGHT_SWAPIN;							\
-   HEAP_SPACE_VERIFIED(0);                                              \
-   if (is_non_value(_result)) { Fail; }					\
-   else { Dst = _result; }                                              \
- } while (0)
-
-#define BsGetBinaryImm_2(Ms, Live, Sz, Flags, Dst, Fail)	\
-  do {								\
-    ErlBinMatchBuffer *_mb;					\
-    Eterm _result;						\
-    TestHeap(heap_bin_size(ERL_ONHEAP_BIN_LIMIT), Live);	\
-    _mb = ms_matchbuffer(Ms);					\
-    LIGHT_SWAPOUT;						\
-    _result = erts_bs_get_binary_2(c_p, (Sz), (Flags), _mb);	\
-    LIGHT_SWAPIN;						\
-    HEAP_SPACE_VERIFIED(0);                                     \
-    if (is_non_value(_result)) { Fail; }			\
-    else { Dst = _result; }                                     \
-  } while (0)
-
-#define BsGetBinary_2(Ms, Live, Sz, Flags, Dst, Fail)           \
-  do {								\
-    ErlBinMatchBuffer *_mb;					\
-    Eterm _result; Uint _size;					\
-    BsGetFieldSize(Sz, ((Flags) >> 3), Fail, _size);		\
-    TestHeap(ERL_SUB_BIN_SIZE, Live);				\
-    _mb = ms_matchbuffer(Ms);					\
-    LIGHT_SWAPOUT;						\
-    _result = erts_bs_get_binary_2(c_p, _size, (Flags), _mb);	\
-    LIGHT_SWAPIN;						\
-    HEAP_SPACE_VERIFIED(0);                                     \
-    if (is_non_value(_result)) { Fail; }			\
-    else { Dst = _result; }                                     \
-  } while (0)
-
-#define BsGetBinaryAll_2(Ms, Live, Unit, Dst, Fail)	\
-  do {                                                  \
-    ErlBinMatchBuffer *_mb;                             \
-    Eterm _result;                                      \
-    TestHeap(ERL_SUB_BIN_SIZE, Live);                   \
-    _mb = ms_matchbuffer(Ms);                           \
-    if (((_mb->size - _mb->offset) % Unit) == 0) {      \
-      LIGHT_SWAPOUT;                                    \
-      _result = erts_bs_get_binary_all_2(c_p, _mb);     \
-      LIGHT_SWAPIN;                                     \
-      HEAP_SPACE_VERIFIED(0);                           \
-      ASSERT(is_value(_result));                        \
-      Dst = _result;					\
-    } else {                                            \
-	HEAP_SPACE_VERIFIED(0);                         \
-	Fail; }                                         \
- } while (0)
-
-#define BsSkipBits2(Ms, Bits, Unit, Fail)			\
- do {								\
-   ErlBinMatchBuffer *_mb;					\
-   size_t new_offset;						\
-   Uint _size;							\
-    _mb = ms_matchbuffer(Ms);					\
-   BsGetFieldSize(Bits, Unit, Fail, _size);			\
-   new_offset = _mb->offset + _size;				\
-   if (new_offset <= _mb->size) { _mb->offset = new_offset; }	\
-   else { Fail; }						\
- } while (0)
-
-#define BsSkipBitsAll2(Ms, Unit, Fail)		\
- do {						\
-    ErlBinMatchBuffer *_mb;			\
-   _mb = ms_matchbuffer(Ms);			\
-   if (((_mb->size - _mb->offset) % Unit) == 0) {_mb->offset = _mb->size; } \
-   else { Fail; }					\
- } while (0)
-
-#define BsSkipBitsImm2(Ms, Bits, Fail)				\
- do {								\
-   ErlBinMatchBuffer *_mb;					\
-   size_t new_offset;						\
-   _mb = ms_matchbuffer(Ms);					\
-   new_offset = _mb->offset + (Bits);				\
-   if (new_offset <= _mb->size) { _mb->offset = new_offset; }	\
-   else { Fail; }						\
- } while (0)
-
-#define NewBsPutIntegerImm(Sz, Flags, Src)					\
- do {									\
-    if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3((Src), (Sz), (Flags)))) { goto badarg; }	\
- } while (0)
-
-#define NewBsPutInteger(Sz, Flags, Src)						\
- do {										\
-    Sint _size;									\
-    BsGetUncheckedFieldSize(Sz, ((Flags) >> 3), goto badarg, _size);		\
-    if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3((Src), _size, (Flags))))	\
-       { goto badarg; }								\
- } while (0)
-
-#define NewBsPutFloatImm(Sz, Flags, Src)					\
- do {									\
-    if (!erts_new_bs_put_float(c_p, (Src), (Sz), (Flags))) { goto badarg; }	\
- } while (0)
-
-#define NewBsPutFloat(Sz, Flags, Src)						\
- do {										\
-    Sint _size;									\
-    BsGetUncheckedFieldSize(Sz, ((Flags) >> 3), goto badarg, _size);		\
-    if (!erts_new_bs_put_float(c_p, (Src), _size, (Flags))) { goto badarg; }	\
- } while (0)
-
-#define NewBsPutBinary(Sz, Flags, Src)							\
- do {											\
-    Sint _size;										\
-    BsGetUncheckedFieldSize(Sz, ((Flags) >> 3), goto badarg, _size);			\
-    if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2((Src), _size))) { goto badarg; }	\
- } while (0)
-
-#define NewBsPutBinaryImm(Sz, Src)				        \
- do {							        \
-    if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2((Src), (Sz)))) { goto badarg; }	\
- } while (0)
-
-#define NewBsPutBinaryAll(Src, Unit)							\
- do {											\
-    if (!erts_new_bs_put_binary_all(ERL_BITS_ARGS_2((Src), (Unit)))) { goto badarg; }	\
- } while (0)
-
-
-#define IsPort(Src, Fail) if (is_not_port(Src)) { Fail; }
-#define IsPid(Src, Fail) if (is_not_pid(Src)) { Fail; }
-#define IsRef(Src, Fail) if (is_not_ref(Src)) { Fail; }
 
 /*
  * process_main() is already huge, so we want to avoid inlining
@@ -1603,40 +1077,6 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 	Next(3);
     }
 
- OpCase(i_move_call_only_fc): {
-     r(0) = Arg(1);
- }
- /* FALL THROUGH */
- OpCase(i_call_only_f): {
-     SET_I((BeamInstr *) Arg(0));
-     DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
-     Dispatch();
- }
-
- OpCase(i_move_call_last_fPc): {
-     r(0) = Arg(2);
- }
- /* FALL THROUGH */
- OpCase(i_call_last_fP): {
-     RESTORE_CP(E);
-     E = ADD_BYTE_OFFSET(E, Arg(1));
-     SET_I((BeamInstr *) Arg(0));
-     DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
-     Dispatch();
- }
-
- OpCase(i_move_call_cf): {
-     r(0) = Arg(0);
-     I++;
- }
- /* FALL THROUGH */
- OpCase(i_call_f): {
-     SET_CP(c_p, I+2);
-     SET_I((BeamInstr *) Arg(0));
-     DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
-     Dispatch();
- }
-
  OpCase(i_move_call_ext_last_ePc): {
      r(0) = Arg(2);
  }
@@ -1671,37 +1111,6 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
     DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0));
     Dispatchx();
 
- OpCase(init_y): {
-     BeamInstr *next;
-
-     PreFetch(1, next);
-     make_blank(yb(Arg(0)));
-     NextPF(1, next);
- }
-
- OpCase(i_trim_I): {
-     BeamInstr *next;
-     Uint words;
-     Uint cp;
-
-     words = Arg(0);
-     cp = E[0];
-     PreFetch(1, next);
-     E += words;
-     E[0] = cp;
-     NextPF(1, next);
- }
-
- OpCase(move_x1_c): {
-	x(1) = Arg(0);
-	Next(1);
-    }
-
- OpCase(move_x2_c): {
-	x(2) = Arg(0);
-	Next(1);
-    }
-
  OpCase(return): {
     SET_I(c_p->cp);
     DTRACE_RETURN_FROM_PC(c_p);
@@ -3457,21 +2866,6 @@ do {						\
      goto do_schedule1;
  }
 
- OpCase(set_tuple_element_sdP): {
-     Eterm element;
-     Eterm tuple;
-     BeamInstr *next;
-     Eterm* p;
-     
-     PreFetch(3, next);
-     GetArg1(0, element);
-     tuple = REG_TARGET(Arg(1));
-     ASSERT(is_tuple(tuple));
-     p = (Eterm *) ((unsigned char *) tuple_val(tuple) + Arg(2));
-     *p = element;
-     NextPF(3, next);
- }
-
  OpCase(normal_exit): {
      SWAPOUT;
      c_p->freason = EXC_NORMAL;
@@ -3707,26 +3101,6 @@ do {						\
 	}
     }
 
- OpCase(i_get_sd):
-    {
-	Eterm arg;
-	Eterm result;
-
-	GetArg1(0, arg);
-	result = erts_pd_hash_get(c_p, arg);
-	StoreBifResult(1, result);
-    }
-
- OpCase(i_get_hash_cId):
-    {
-	Eterm arg;
-	Eterm result;
-
-	GetArg1(0, arg);
-	result = erts_pd_hash_get_with_hx(c_p, Arg(1), arg);
-	StoreBifResult(2, result);
-    }
-
     {
 	Eterm case_end_val;
 
@@ -4729,20 +4103,6 @@ do {						\
 
 #include "beam_cold.h"
 
-
- /*
-  * This instruction is probably never used (because it is combined with a
-  * a return). However, a future compiler might for some reason emit a
-  * deallocate not followed by a return, and that should work.
-  */
- OpCase(deallocate_I): {
-     BeamInstr *next;
-
-     PreFetch(1, next);
-     D(Arg(0));
-     NextPF(1, next);
- }
-
     /*
      * Trace and debugging support.
      */
@@ -4847,9 +4207,9 @@ do {						\
      targ1 = REG_TARGET(Arg(0));
      PreFetch(2, next);
      if (is_small(targ1)) {
-	 fb(fr) = (double) signed_val(targ1);
+	 lb(fr) = (double) signed_val(targ1);
      } else if (is_big(targ1)) {
-	 if (big_to_double(targ1, &fb(fr)) < 0) {
+	 if (big_to_double(targ1, &lb(fr)) < 0) {
 	     goto fbadarith;
 	 }
      } else if (is_float(targ1)) {
@@ -4893,8 +4253,8 @@ do {						\
 
      PreFetch(3, next);
      ERTS_NO_FPE_CHECK_INIT(c_p);
-     fb(Arg(2)) = fb(Arg(0)) + fb(Arg(1));
-     ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith);
+     lb(Arg(2)) = lb(Arg(0)) + lb(Arg(1));
+     ERTS_NO_FPE_ERROR(c_p, lb(Arg(2)), goto fbadarith);
      NextPF(3, next);
  }
  OpCase(i_fsub_lll): {
@@ -4902,8 +4262,8 @@ do {						\
 
      PreFetch(3, next);
      ERTS_NO_FPE_CHECK_INIT(c_p);
-     fb(Arg(2)) = fb(Arg(0)) - fb(Arg(1));
-     ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith);
+     lb(Arg(2)) = lb(Arg(0)) - lb(Arg(1));
+     ERTS_NO_FPE_ERROR(c_p, lb(Arg(2)), goto fbadarith);
      NextPF(3, next);
  }
  OpCase(i_fmul_lll): {
@@ -4911,8 +4271,8 @@ do {						\
 
      PreFetch(3, next);
      ERTS_NO_FPE_CHECK_INIT(c_p);
-     fb(Arg(2)) = fb(Arg(0)) * fb(Arg(1));
-     ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith);
+     lb(Arg(2)) = lb(Arg(0)) * lb(Arg(1));
+     ERTS_NO_FPE_ERROR(c_p, lb(Arg(2)), goto fbadarith);
      NextPF(3, next);
  }
  OpCase(i_fdiv_lll): {
@@ -4920,8 +4280,8 @@ do {						\
 
      PreFetch(3, next);
      ERTS_NO_FPE_CHECK_INIT(c_p);
-     fb(Arg(2)) = fb(Arg(0)) / fb(Arg(1));
-     ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith);
+     lb(Arg(2)) = lb(Arg(0)) / lb(Arg(1));
+     ERTS_NO_FPE_ERROR(c_p, lb(Arg(2)), goto fbadarith);
      NextPF(3, next);
  }
  OpCase(i_fnegate_ll): {
@@ -4929,8 +4289,8 @@ do {						\
 
      PreFetch(2, next);
      ERTS_NO_FPE_CHECK_INIT(c_p);
-     fb(Arg(1)) = -fb(Arg(0));
-     ERTS_NO_FPE_ERROR(c_p, fb(Arg(1)), goto fbadarith);
+     lb(Arg(1)) = -lb(Arg(0));
+     ERTS_NO_FPE_ERROR(c_p, lb(Arg(1)), goto fbadarith);
      NextPF(2, next);
 
  fbadarith:
diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab
new file mode 100644
index 0000000000..420fd3074c
--- /dev/null
+++ b/erts/emulator/beam/bs_instrs.tab
@@ -0,0 +1,179 @@
+// -*- c -*-
+//
+// %CopyrightBegin%
+//
+// Copyright Ericsson AB 2017. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// %CopyrightEnd%
+//
+
+i_bs_get_binary_all2(Fail, Ms, Live, Unit, Dst) {
+    ErlBinMatchBuffer *_mb;
+    Eterm _result;
+    TestHeap(ERL_SUB_BIN_SIZE, $Live);
+    _mb = ms_matchbuffer($Ms);
+    if (((_mb->size - _mb->offset) % $Unit) == 0) {
+      LIGHT_SWAPOUT;
+      _result = erts_bs_get_binary_all_2(c_p, _mb);
+      LIGHT_SWAPIN;
+      HEAP_SPACE_VERIFIED(0);
+      ASSERT(is_value(_result));
+      $Dst = _result;
+    } else {
+	HEAP_SPACE_VERIFIED(0);
+	$FAIL($Fail);
+    }
+}
+
+i_bs_get_binary2(Fail, Ms, Live, Sz, Flags, Dst) {
+    ErlBinMatchBuffer *_mb;
+    Eterm _result;
+    Uint _size;
+    BsGetFieldSize($Sz, (($Flags) >> 3), $FAIL($Fail), _size);
+    TestHeap(ERL_SUB_BIN_SIZE, $Live);
+    _mb = ms_matchbuffer($Ms);
+    LIGHT_SWAPOUT;
+    _result = erts_bs_get_binary_2(c_p, _size, $Flags, _mb);
+    LIGHT_SWAPIN;
+    HEAP_SPACE_VERIFIED(0);
+    if (is_non_value(_result)) {
+        $FAIL($Fail);
+    } else {
+        $Dst = _result;
+    }
+}
+
+i_bs_get_binary_imm2(Fail, Ms, Live, Sz, Flags, Dst) {
+    ErlBinMatchBuffer *_mb;
+    Eterm _result;
+    TestHeap(heap_bin_size(ERL_ONHEAP_BIN_LIMIT), $Live);
+    _mb = ms_matchbuffer($Ms);
+    LIGHT_SWAPOUT;
+    _result = erts_bs_get_binary_2(c_p, $Sz, $Flags, _mb);
+    LIGHT_SWAPIN;
+    HEAP_SPACE_VERIFIED(0);
+    if (is_non_value(_result)) {
+        $FAIL($Fail);
+    } else {
+        $Dst = _result;
+    }
+}
+
+i_bs_get_float2(Fail, Ms, Live, Sz, Flags, Dst) {
+    ErlBinMatchBuffer *_mb;
+    Eterm _result;
+    Sint _size;
+
+    if (!is_small($Sz) || (_size = unsigned_val($Sz)) > 64) {
+        $FAIL($Fail);
+    }
+    _size *= (($Flags) >> 3);
+    TestHeap(FLOAT_SIZE_OBJECT, $Live);
+    _mb = ms_matchbuffer($Ms);
+    LIGHT_SWAPOUT;
+    _result = erts_bs_get_float_2(c_p, _size, ($Flags), _mb);
+    LIGHT_SWAPIN;
+    HEAP_SPACE_VERIFIED(0);
+    if (is_non_value(_result)) {
+        $FAIL($Fail);
+    } else {
+        $Dst = _result;
+    }
+}
+
+i_bs_skip_bits2(Fail, Ms, Bits, Unit) {
+    ErlBinMatchBuffer *_mb;
+    size_t new_offset;
+    Uint _size;
+
+    _mb = ms_matchbuffer($Ms);
+    BsGetFieldSize($Bits, $Unit, $FAIL($Fail), _size);
+    new_offset = _mb->offset + _size;
+    if (new_offset <= _mb->size) {
+        _mb->offset = new_offset;
+    } else {
+        $FAIL($Fail);
+    }
+}
+
+i_bs_skip_bits_all2(Fail, Ms, Unit) {
+    ErlBinMatchBuffer *_mb;
+    _mb = ms_matchbuffer($Ms);
+    if (((_mb->size - _mb->offset) % $Unit) == 0) {
+        _mb->offset = _mb->size;
+    } else {
+        $FAIL($Fail);
+    }
+}
+
+i_bs_skip_bits_imm2(Fail, Ms, Bits) {
+    ErlBinMatchBuffer *_mb;
+    size_t new_offset;
+    _mb = ms_matchbuffer($Ms);
+    new_offset = _mb->offset + ($Bits);
+    if (new_offset <= _mb->size) {
+        _mb->offset = new_offset;
+    } else {
+        $FAIL($Fail);
+    }
+}
+
+i_new_bs_put_binary(Fail, Sz, Flags, Src) {
+    Sint _size;
+    BsGetUncheckedFieldSize($Sz, (($Flags) >> 3), goto badarg, _size);
+    if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2(($Src), _size))) {
+        goto badarg;
+    }
+}
+
+i_new_bs_put_binary_all(Fail, Src, Unit) {
+    if (!erts_new_bs_put_binary_all(ERL_BITS_ARGS_2(($Src), ($Unit)))) {
+        goto badarg;
+    }
+}
+
+i_new_bs_put_binary_imm(Fail, Sz, Src) {
+    if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2(($Src), ($Sz)))) {
+        goto badarg;
+    }
+}
+
+i_new_bs_put_float(Fail, Sz, Flags, Src) {
+    Sint _size;
+    BsGetUncheckedFieldSize($Sz, (($Flags) >> 3), goto badarg, _size);
+    if (!erts_new_bs_put_float(c_p, ($Src), _size, ($Flags))) {
+        goto badarg;
+    }
+}
+
+i_new_bs_put_float_imm(Fail, Sz, Flags, Src) {
+    if (!erts_new_bs_put_float(c_p, ($Src), ($Sz), ($Flags))) {
+        goto badarg;
+    }
+}
+
+i_new_bs_put_integer(Fail, Sz, Flags, Src) {
+   Sint _size;
+   BsGetUncheckedFieldSize($Sz, (($Flags) >> 3), goto badarg, _size);
+   if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), _size, ($Flags)))) {
+       goto badarg;
+   }
+}
+
+i_new_bs_put_integer_imm(Fail, Sz, Flags, Src) {
+    if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), ($Sz), ($Flags)))) {
+        goto badarg;
+    }
+}
diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
new file mode 100644
index 0000000000..ca0cb2f63d
--- /dev/null
+++ b/erts/emulator/beam/instrs.tab
@@ -0,0 +1,594 @@
+// -*- c -*-
+//
+// %CopyrightBegin%
+//
+// Copyright Ericsson AB 2017. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// %CopyrightEnd%
+//
+
+// Macros only used to generate instructions.
+
+FAIL(Fail) {
+    //| -no_prefetch
+    SET_I((BeamInstr *) $Fail);
+    Goto(*I);
+}
+
+JUMP(Fail) {
+    //| -no_next
+    SET_I((BeamInstr *) $Fail);
+    Goto(*I);
+}
+
+GC_TEST(Ns, Nh, Live) {
+    unsigned need = $Nh + $Ns;
+    if (E - HTOP < need) {
+       SWAPOUT;
+       PROCESS_MAIN_CHK_LOCKS(c_p);
+       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, $Live, FCALLS);
+       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+       PROCESS_MAIN_CHK_LOCKS(c_p);
+       SWAPIN;
+    }
+    HEAP_SPACE_VERIFIED($Nh);
+}
+
+// Make sure that there are NeedStack + NeedHeap + 1 words available
+// on the combined heap/stack segment, then allocates NeedHeap + 1
+// words on the stack and saves CP.
+AH(NeedStack, NeedHeap, Live) {
+    unsigned needed = $NeedStack + 1;
+    $GC_TEST(needed, $NeedHeap, $Live);
+    E -= needed;
+    SAVE_CP(E);
+}
+
+// Start of instruction listings
+
+// Call instructions
+
+DO_CALL(CallDest, NextInstr) {
+    //| -no_next
+    SET_CP(c_p, $NextInstr);
+    SET_I((BeamInstr *) $CallDest);
+    DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
+    Dispatch();
+}
+
+i_call(CallDest) {
+    $DO_CALL($CallDest, $NEXT_INSTRUCTION);
+}
+
+move_call(Src, CallDest) {
+    x(0) = $Src;
+    $DO_CALL($CallDest, $NEXT_INSTRUCTION);
+}
+
+i_call_last(CallDest, Deallocate) {
+    //| -no_next
+    RESTORE_CP(E);
+    E = ADD_BYTE_OFFSET(E, ($Deallocate));
+    SET_I((BeamInstr *) $CallDest);
+    DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
+    Dispatch();
+}
+
+move_call_last(Src, CallDest, Deallocate) {
+    x(0) = $Src;
+    $i_call_last($CallDest, $Deallocate);
+}
+
+i_call_only(CallDest) {
+    //| -no_next
+    SET_I((BeamInstr *) $CallDest);
+    DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
+    Dispatch();
+}
+
+move_call_only(Src, CallDest) {
+    x(0) = $Src;
+    $i_call_only($CallDest);
+}
+
+// Other instructions
+
+allocate(NeedStack, Live) {
+    $AH($NeedStack, 0, $Live);
+}
+
+allocate_heap(NeedStack, NeedHeap, Live) {
+    $AH($NeedStack, $NeedHeap, $Live);
+}
+
+allocate_init(NeedStack, Live, Y) {
+    $AH($NeedStack, 0, $Live);
+    make_blank($Y);
+}
+
+allocate_zero(NeedStack, Live) {
+    Eterm* ptr;
+    int i = $NeedStack;
+    $AH(i, 0, $Live);
+    for (ptr = E + i; ptr > E; ptr--) {
+	 make_blank(*ptr);
+    }
+}
+
+allocate_heap_zero(NeedStack, NeedHeap, Live) {
+    Eterm* ptr;
+    int i = $NeedStack;
+    $AH(i, $NeedHeap, $Live);
+    for (ptr = E + i; ptr > E; ptr--) {
+        make_blank(*ptr);
+    }
+}
+
+// This instruction is probably never used (because it is combined with a
+// a return). However, a future compiler might for some reason emit a
+// deallocate not followed by a return, and that should work.
+
+deallocate(Deallocate) {
+    //| -no_prefetch
+    RESTORE_CP(E);
+    E = ADD_BYTE_OFFSET(E, $Deallocate);
+}
+
+deallocate_return(Deallocate) {
+    //| -no_next
+    int words_to_pop = $Deallocate;
+    SET_I((BeamInstr *) cp_val(*E));
+    E = ADD_BYTE_OFFSET(E, words_to_pop);
+    CHECK_TERM(x(0));
+    DispatchReturn;
+}
+
+get_list(Src, Hd, Tl) {
+    Eterm* tmp_ptr = list_val($Src);
+    Eterm hd, tl;
+    hd = CAR(tmp_ptr);
+    tl = CDR(tmp_ptr);
+    $Hd = hd;
+    $Tl = tl;
+}
+
+i_get(Src, Dst) {
+    $Dst = erts_pd_hash_get(c_p, $Src);
+}
+
+i_get_hash(Src, Hash, Dst) {
+    $Dst = erts_pd_hash_get_with_hx(c_p, $Hash, $Src);
+}
+
+i_get_tuple_element(Src, Element, Dst) {
+    Eterm* src = ADD_BYTE_OFFSET(tuple_val($Src), $Element);
+    $Dst = *src;
+}
+
+i_get_tuple_element2(Src, Element, Dst) {
+    Eterm* src;
+    Eterm* dst;
+    Eterm E1, E2;
+    src = ADD_BYTE_OFFSET(tuple_val($Src), $Element);
+    dst = &($Dst);
+    E1 = src[0];
+    E2 = src[1];
+    dst[0] = E1;
+    dst[1] = E2;
+}
+
+i_get_tuple_element2y(Src, Element, D1, D2) {
+    Eterm* src;
+    Eterm E1, E2;
+    src = ADD_BYTE_OFFSET(tuple_val($Src), $Element);
+    E1 = src[0];
+    E2 = src[1];
+    $D1 = E1;
+    $D2 = E2;
+}
+
+i_get_tuple_element3(Src, Element, Dst) {
+    Eterm* src;
+    Eterm* dst;
+    Eterm E1, E2, E3;
+    src = ADD_BYTE_OFFSET(tuple_val($Src), $Element);
+    dst = &($Dst);
+    E1 = src[0];
+    E2 = src[1];
+    E3 = src[2];
+    dst[0] = E1;
+    dst[1] = E2;
+    dst[2] = E3;
+}
+
+init(Y) {
+    make_blank($Y);
+}
+
+init2(Y1, Y2) {
+    make_blank($Y1);
+    make_blank($Y2);
+}
+
+init3(Y1, Y2, Y3) {
+    make_blank($Y1);
+    make_blank($Y2);
+    make_blank($Y3);
+}
+
+i_make_fun(FunP, NumFree) {
+    HEAVY_SWAPOUT;
+    x(0) = new_fun(c_p, reg, (ErlFunEntry *) $FunP, $NumFree);
+    HEAVY_SWAPIN;
+}
+
+i_trim(Words) {
+    Uint cp = E[0];
+    E += $Words;
+    E[0] = cp;
+}
+
+move(Src, Dst) {
+    $Dst = $Src;
+}
+
+move3(S1, D1, S2, D2, S3, D3) {
+    $D1 = $S1;
+    $D2 = $S2;
+    $D3 = $S3;
+}
+
+move_deallocate_return(Src, Deallocate) {
+    //| -no_next
+    x(0) = $Src;
+    $deallocate_return($Deallocate);
+}
+
+move_dup(Src, D1, D2) {
+    $D1 = $D2 = $Src;
+}
+
+move2_par(S1, D1, S2, D2) {
+    Eterm V1, V2;
+    V1 = $S1;
+    V2 = $S2;
+    $D1 = V1;
+    $D2 = V2;
+}
+
+move_shift(Src, SD, D) {
+    Eterm V;
+    V = $Src;
+    $D = $SD;
+    $SD = V;
+}
+
+move_window3(S1, S2, S3, D) {
+    Eterm xt0, xt1, xt2;
+    Eterm* y = &$D;
+    xt0  = $S1;
+    xt1  = $S2;
+    xt2  = $S3;
+    y[0] = xt0;
+    y[1] = xt1;
+    y[2] = xt2;
+}
+
+move_window4(S1, S2, S3, S4, D) {
+    Eterm xt0, xt1, xt2, xt3;
+    Eterm* y = &$D;
+    xt0  = $S1;
+    xt1  = $S2;
+    xt2  = $S3;
+    xt3  = $S4;
+    y[0] = xt0;
+    y[1] = xt1;
+    y[2] = xt2;
+    y[3] = xt3;
+}
+
+move_window5(S1, S2, S3, S4, S5, D) {
+    Eterm xt0, xt1, xt2, xt3, xt4;
+    Eterm *y = &$D;
+    xt0  = $S1;
+    xt1  = $S2;
+    xt2  = $S3;
+    xt3  = $S4;
+    xt4  = $S5;
+    y[0] = xt0;
+    y[1] = xt1;
+    y[2] = xt2;
+    y[3] = xt3;
+    y[4] = xt4;
+}
+
+move_return(Src) {
+    //| -no_next
+    x(0) = $Src;
+    SET_I(c_p->cp);
+    c_p->cp = 0;
+    DispatchReturn;
+}
+
+move_x1(Src) {
+    x(1) = $Src;
+}
+
+move_x2(Src) {
+    x(2) = $Src;
+}
+
+node(Dst) {
+    $Dst = erts_this_node->sysname;
+}
+
+put_list(Hd, Tl, Dst) {
+    HTOP[0] = $Hd;
+    HTOP[1] = $Tl;
+    $Dst = make_list(HTOP);
+    HTOP += 2;
+}
+
+i_put_tuple(Dst, Arity) {
+    //| -no_next
+    $Dst = make_tuple(HTOP);
+    pt_arity = $Arity;
+    I = $NEXT_INSTRUCTION;
+    goto do_put_tuple;
+}
+
+self(Dst) {
+    $Dst = c_p->common.id;
+}
+
+set_tuple_element(Element, Tuple, Offset) {
+    Eterm* p;
+
+    ASSERT(is_tuple($Tuple));
+    p = (Eterm *) ((unsigned char *) tuple_val($Tuple) + $Offset);
+    *p = $Element;
+}
+
+swap(R1, R2) {
+    Eterm V = $R1;
+    $R1 = $R2;
+    $R2 = V;
+}
+
+swap_temp(R1, R2, Tmp) {
+    Eterm V = $R1;
+    $R1 = $R2;
+    $R2 = $Tmp = V;
+}
+
+test_heap(Nh, Live) {
+    $GC_TEST(0, $Nh, $Live);
+}
+
+test_heap_1_put_list(Nh, Reg) {
+    $test_heap($Nh, 1);
+    $put_list($Reg, x(0), x(0));
+}
+
+is_integer_allocate(Fail, Src, NeedStack, Live) {
+    //| -no_prefetch
+    $is_integer($Fail, $Src);
+    $AH($NeedStack, 0, $Live);
+}
+
+is_nonempty_list(Fail, Src) {
+    //| -no_prefetch
+    if (is_not_list($Src)) {
+        $FAIL($Fail);
+    }
+}
+
+is_nonempty_list_test_heap(Fail, Need, Live) {
+    //| -no_prefetch
+    $is_nonempty_list($Fail, x(0));
+    $test_heap($Need, $Live);
+}
+
+is_nonempty_list_allocate(Fail, Src, Need, Live) {
+    //| -no_prefetch
+    $is_nonempty_list($Fail, $Src);
+    $AH($Need, 0, $Live);
+}
+
+is_nonempty_list_get_list(Fail, Src, Hd, Tl) {
+    //| -no_prefetch
+    $is_nonempty_list($Fail, $Src);
+    $get_list($Src, $Hd, $Tl);
+}
+
+move_jump(Fail, Src) {
+    x(0) = $Src;
+    $JUMP($Fail);
+}
+
+//
+// Test instructions.
+//
+
+is_atom(Fail, Src) {
+    if (is_not_atom($Src)) {
+        $FAIL($Fail);
+    }
+}
+
+is_boolean(Fail, Src) {
+    if (($Src) != am_true && ($Src) != am_false) {
+        $FAIL($Fail);
+    }
+}
+
+is_binary(Fail, Src) {
+    if (is_not_binary($Src) || binary_bitsize($Src) != 0) {
+        $FAIL($Fail);
+    }
+}
+
+is_bitstring(Fail, Src) {
+  if (is_not_binary($Src)) {
+        $FAIL($Fail);
+    }
+}
+
+is_float(Fail, Src) {
+    if (is_not_float($Src)) {
+        $FAIL($Fail);
+    }
+}
+
+is_function(Fail, Src) {
+    if ( !(is_any_fun($Src)) ) {
+        $FAIL($Fail);
+    }
+}
+
+is_function2(Fail, Fun, Arity) {
+    if (erl_is_function(c_p, $Fun, $Arity) != am_true ) {
+        $FAIL($Fail);
+    }
+}
+
+is_integer(Fail, Src) {
+    if (is_not_integer($Src)) {
+        $FAIL($Fail);
+    }
+}
+
+is_list(Fail, Src) {
+    if (is_not_list($Src) && is_not_nil($Src)) {
+        $FAIL($Fail);
+    }
+}
+
+is_map(Fail, Src) {
+    if (is_not_map($Src)) {
+        $FAIL($Fail);
+    }
+}
+
+is_nil(Fail, Src) {
+    if (is_not_nil($Src)) {
+        $FAIL($Fail);
+    }
+}
+
+is_number(Fail, Src) {
+    if (is_not_integer($Src) && is_not_float($Src)) {
+        $FAIL($Fail);
+    }
+}
+
+is_pid(Fail, Src) {
+    if (is_not_pid($Src)) {
+        $FAIL($Fail);
+    }
+}
+
+is_port(Fail, Src) {
+    if (is_not_port($Src)) {
+        $FAIL($Fail);
+    }
+}
+
+is_reference(Fail, Src) {
+    if (is_not_ref($Src)) {
+        $FAIL($Fail);
+    }
+}
+
+is_tagged_tuple(Fail, Src, Arityval, Tag) {
+    if (!(BEAM_IS_TUPLE($Src) &&
+          (tuple_val($Src))[0] == $Arityval &&
+          (tuple_val($Src))[1] == $Tag)) {
+        $FAIL($Fail);
+    }
+}
+
+is_tuple(Fail, Src) {
+    if (is_not_tuple($Src)) {
+        $FAIL($Fail);
+    }
+}
+
+is_tuple_of_arity(Fail, Src, Arityval) {
+    if (!(BEAM_IS_TUPLE($Src) && *tuple_val($Src) == $Arityval)) {
+        $FAIL($Fail);
+    }
+}
+
+test_arity(Fail, Pointer, Arity) {
+    if (*tuple_val($Pointer) != $Arity) {
+        $FAIL($Fail);
+    }
+}
+i_is_eq_exact_immed(Fail, X, Y) {
+    if ($X != $Y) {
+        $FAIL($Fail);
+    }
+}
+
+i_is_ne_exact_immed(Fail, X, Y) {
+    if ($X == $Y) {
+        $FAIL($Fail);
+    }
+}
+
+is_eq_exact(Fail, X, Y) {
+    if (!EQ($X, $Y)) {
+        $FAIL($Fail);
+    }
+}
+
+is_ne_exact(Fail, X, Y) {
+    if (EQ($X, $Y)) {
+        $FAIL($Fail);
+    }
+}
+
+is_eq(Fail, X, Y) {
+    CMP_EQ_ACTION($X, $Y, $FAIL($Fail));
+}
+
+is_ne(Fail, X, Y) {
+    CMP_NE_ACTION($X, $Y, $FAIL($Fail));
+}
+
+is_lt(Fail, X, Y) {
+    CMP_LT_ACTION($X, $Y, $FAIL($Fail));
+}
+
+is_ge(Fail, X, Y) {
+    CMP_GE_ACTION($X, $Y, $FAIL($Fail));
+}
+
+i_get_map_element(Fail, Src, Key, Dst) {
+    Eterm res = get_map_element($Src, $Key);
+    if (is_non_value(res)) {
+        $FAIL($Fail);
+    }
+    $Dst = res;
+}
+
+i_get_map_element_hash(Fail, Src, Key, Hx, Dst) {
+     Eterm res = get_map_element_hash($Src, $Key, $Hx);
+     if (is_non_value(res)) {
+        $FAIL($Fail);
+     }
+     $Dst = res;
+}
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index ed856b760b..8c9034518b 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -96,16 +96,13 @@ line Loc | func_info M F A => func_info M F A | line Loc
 
 line I
 
-
-%macro: allocate Allocate -pack
-%macro: allocate_zero AllocateZero -pack
-%macro: allocate_heap AllocateHeap -pack
-%macro: allocate_heap_zero AllocateHeapZero -pack
-%macro: test_heap TestHeap -pack
-
 allocate t t
 allocate_heap t I t
-deallocate I
+
+%cold
+deallocate Q
+%hot
+
 init y
 allocate_zero t t
 allocate_heap_zero t I t
@@ -122,8 +119,6 @@ init2 y y
 init3 y y y
 init Y1 | init Y2 | init Y3 => init3 Y1 Y2 Y3
 init Y1 | init Y2 => init2 Y1 Y2
-%macro: init2 Init2 -pack
-%macro: init3 Init3 -pack
 
 # Selecting values
 
@@ -174,7 +169,6 @@ i_jump_on_val_zero xy f I
 
 i_jump_on_val xy f I I
 
-%macro: get_list GetList -pack
 get_list xy xy xy
 
 # The following get_list instructions using x(0) are frequently used.
@@ -202,23 +196,17 @@ set_tuple_element s d P
 
 # Get tuple element
 
-%macro: i_get_tuple_element GetTupleElement -pack
 i_get_tuple_element xy P x
 
 %cold
 i_get_tuple_element xy P y
 %hot
 
-%macro: i_get_tuple_element2 GetTupleElement2 -pack
 i_get_tuple_element2 x P x
-
-%macro: i_get_tuple_element2y GetTupleElement2Y -pack
 i_get_tuple_element2y x P y y
 
-%macro: i_get_tuple_element3 GetTupleElement3 -pack
 i_get_tuple_element3 x P x
 
-%macro: is_number IsNumber -fail_action
 %cold
 is_number f x
 is_number f y
@@ -252,7 +240,6 @@ system_limit j
 
 move C=cxy x==0 | jump Lbl => move_jump Lbl C
 
-%macro: move_jump MoveJump -nonext
 move_jump f ncxy
 
 # Movement to and from the stack is common
@@ -276,10 +263,6 @@ move_window X1=x X2=x X3=x X4=x Y1=y Y4=y | move X5=x Y5=y | succ(Y4,Y5) => \
 move_window X1=x X2=x X3=x Y1=y Y3=y => move_window3 X1 X2 X3 Y1
 move_window X1=x X2=x X3=x X4=x Y1=y Y4=y => move_window4 X1 X2 X3 X4 Y1
 
-%macro: move_window3 MoveWindow3 -pack
-%macro: move_window4 MoveWindow4 -pack
-%macro: move_window5 MoveWindow5 -pack
-
 move_window3 x x x y
 move_window4 x x x x y
 move_window5 x x x x x y
@@ -304,10 +287,8 @@ swap_temp R1 R2 Tmp | line Loc | call_ext_only Live Addr | \
 swap_temp R1 R2 Tmp | line Loc | call_ext_last Live Addr D | \
   is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_last Live Addr D
 
-%macro: swap_temp SwapTemp -pack
 swap_temp x xy x
 
-%macro: swap Swap -pack
 swap x xy
 
 move Src=x D1=x | move Src=x D2=x => move_dup Src D1 D2
@@ -351,17 +332,13 @@ move C=aiq X=x==2 => move_x2 C
 move_x1 c
 move_x2 c
 
-%macro: move_shift MoveShift -pack
 move_shift x x x
 move_shift y x x
 move_shift x y x
 move_shift x x y
 
-%macro: move_dup MoveDup -pack
 move_dup xy x xy
 
-%macro: move2_par Move2Par -pack
-
 move2_par x y x y
 move2_par y x y x
 move2_par x x x x
@@ -373,7 +350,6 @@ move2_par y x x y
 move2_par x x y x
 move2_par y x x x
 
-%macro: move3 Move3 -pack
 move3 x y x y x y
 move3 y x y x y x
 move3 x x x x x x
@@ -383,7 +359,6 @@ move3 x x x x x x
 move S=n D=y => init D
 move S=c D=y => move S x | move x D
 
-%macro:move Move -pack
 move x x
 move x y
 move y x
@@ -440,22 +415,18 @@ is_eq_exact Lbl R=xy C=q => i_is_eq_exact_literal Lbl R C
 is_ne_exact Lbl R=xy C=ian => i_is_ne_exact_immed Lbl R C
 is_ne_exact Lbl R=xy C=q => i_is_ne_exact_literal Lbl R C
 
-%macro: i_is_eq_exact_immed EqualImmed -fail_action
-
 i_is_eq_exact_immed f rxy c
+
 i_is_eq_exact_literal f xy c
 
-%macro: i_is_ne_exact_immed NotEqualImmed -fail_action
 i_is_ne_exact_immed f xy c
 
 i_is_ne_exact_literal f xy c
 
 is_eq_exact Lbl Y=y X=x => is_eq_exact Lbl X Y
-%macro: is_eq_exact EqualExact -fail_action -pack
 is_eq_exact f x xy
 is_eq_exact f s s
 
-%macro: is_lt IsLessThan -fail_action
 is_lt f x x
 is_lt f x c
 is_lt f c x
@@ -463,7 +434,6 @@ is_lt f c x
 is_lt f s s
 %hot
 
-%macro: is_ge IsGreaterEqual -fail_action
 is_ge f x x
 is_ge f x c
 is_ge f c x
@@ -471,13 +441,10 @@ is_ge f c x
 is_ge f s s
 %hot
 
-%macro: is_ne_exact NotEqualExact -fail_action
 is_ne_exact f s s
 
-%macro: is_eq Equal -fail_action
 is_eq f s s
 
-%macro: is_ne NotEqual -fail_action
 is_ne f s s
 
 #
@@ -495,7 +462,6 @@ i_put_tuple Dst Arity Puts=* | put S => \
 
 i_put_tuple/2
 
-%macro:i_put_tuple PutTuple -pack -goto:do_put_tuple
 i_put_tuple xy I
 
 #
@@ -505,8 +471,6 @@ i_put_tuple xy I
 #
 put_list Const=c n Dst => move Const x | put_list x n Dst
 
-%macro:put_list PutList -pack
-
 put_list x n x
 put_list y n x
 put_list x x x
@@ -564,22 +528,18 @@ return_trace
 
 move S x==0 | return => move_return S
 
-%macro: move_return MoveReturn -nonext
 move_return xcn
 
 move S x==0 | deallocate D | return => move_deallocate_return S D
 
-%macro: move_deallocate_return MoveDeallocateReturn -pack -nonext
 move_deallocate_return xycn Q
 
 deallocate D | return => deallocate_return D
 
-%macro: deallocate_return DeallocateReturn -nonext
 deallocate_return Q
 
 test_heap Need u==1 | put_list Y=y x==0 x==0 => test_heap_1_put_list Need Y
 
-%macro: test_heap_1_put_list TestHeapPutList -pack
 test_heap_1_put_list I y
 
 #
@@ -590,8 +550,6 @@ is_tagged_tuple Fail Literal=q Arity Atom => \
     move Literal x | is_tagged_tuple Fail x Arity Atom
 is_tagged_tuple Fail=f c Arity Atom  => jump Fail
 
-%macro:is_tagged_tuple IsTaggedTuple -fail_action
-
 is_tagged_tuple f rxy A a
 
 # Test tuple & arity (head)
@@ -600,17 +558,13 @@ is_tuple Fail Literal=q => move Literal x | is_tuple Fail x
 is_tuple Fail=f c => jump Fail
 is_tuple Fail=f S=xy | test_arity Fail=f S=xy Arity => is_tuple_of_arity Fail S Arity
 
-%macro:is_tuple_of_arity IsTupleOfArity -fail_action
-
 is_tuple_of_arity f rxy A
 
-%macro: is_tuple IsTuple -fail_action
 is_tuple f rxy
 
 test_arity Fail Literal=q Arity => move Literal x | test_arity Fail x Arity
 test_arity Fail=f c Arity => jump Fail
 
-%macro: test_arity IsArity -fail_action
 test_arity f xy A
 
 get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \
@@ -632,16 +586,13 @@ is_integer Fail Literal=q => move Literal x | is_integer Fail x
 
 is_integer Fail=f S=x | allocate Need Regs => is_integer_allocate Fail S Need Regs
 
-%macro: is_integer_allocate IsIntegerAllocate -fail_action
 is_integer_allocate f x I I
 
-%macro: is_integer IsInteger -fail_action
 is_integer f xy
 
 is_list Fail=f n =>
 is_list Fail Literal=q => move Literal x | is_list Fail x
 is_list Fail=f c => jump Fail
-%macro: is_list IsList -fail_action
 is_list f x
 %cold
 is_list f y
@@ -649,24 +600,16 @@ is_list f y
 
 is_nonempty_list Fail=f S=x | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs
 
-%macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -pack
-is_nonempty_list_allocate f rx I t
-
-is_nonempty_list F=f x==0 | test_heap I1 I2 => is_non_empty_list_test_heap F I1 I2
-
-%macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -pack
-is_non_empty_list_test_heap f I t
+is_nonempty_list F=f x==0 | test_heap I1 I2 => is_nonempty_list_test_heap F I1 I2
 
 is_nonempty_list Fail=f S=x | get_list S D1=x D2=x => \
   is_nonempty_list_get_list Fail S D1 D2
 
-%macro: is_nonempty_list_get_list IsNonemptyListGetList -fail_action -pack
+is_nonempty_list_allocate f rx I t
+is_nonempty_list_test_heap f I t
 is_nonempty_list_get_list f rx x x
-
-%macro: is_nonempty_list IsNonemptyList -fail_action
 is_nonempty_list f xy
 
-%macro: is_atom IsAtom -fail_action
 is_atom f x
 %cold
 is_atom f y
@@ -674,7 +617,6 @@ is_atom f y
 is_atom Fail=f a =>
 is_atom Fail=f niq => jump Fail
 
-%macro: is_float IsFloat -fail_action
 is_float f x
 %cold
 is_float f y
@@ -685,12 +627,10 @@ is_float Fail Literal=q => move Literal x | is_float Fail x
 is_nil Fail=f n =>
 is_nil Fail=f qia => jump Fail
 
-%macro: is_nil IsNil -fail_action
 is_nil f xy
 
 is_binary Fail Literal=q => move Literal x | is_binary Fail x
 is_binary Fail=f c => jump Fail
-%macro: is_binary IsBinary -fail_action
 is_binary f x
 %cold
 is_binary f y
@@ -701,28 +641,24 @@ is_bitstr Fail Term => is_bitstring Fail Term
 
 is_bitstring Fail Literal=q => move Literal x | is_bitstring Fail x
 is_bitstring Fail=f c => jump Fail
-%macro: is_bitstring IsBitstring -fail_action
 is_bitstring f x
 %cold
 is_bitstring f y
 %hot
 
 is_reference Fail=f cq => jump Fail
-%macro: is_reference IsRef -fail_action
 is_reference f x
 %cold
 is_reference f y
 %hot
 
 is_pid Fail=f cq => jump Fail
-%macro: is_pid IsPid -fail_action
 is_pid f x
 %cold
 is_pid f y
 %hot
 
 is_port Fail=f cq => jump Fail
-%macro: is_port IsPort -fail_action
 is_port f x
 %cold
 is_port f y
@@ -733,7 +669,6 @@ is_boolean Fail=f a==am_false =>
 is_boolean Fail=f ac => jump Fail
 
 %cold
-%macro: is_boolean IsBoolean -fail_action
 is_boolean f xy
 %hot
 
@@ -741,13 +676,11 @@ is_function2 Fail=f acq Arity => jump Fail
 is_function2 Fail=f Fun a => jump Fail
 
 is_function2 f s s
-%macro: is_function2 IsFunction2 -fail_action
 
 # Allocating & initializing.
 allocate Need Regs | init Y => allocate_init Need Regs Y
 init Y1 | init Y2 => init2 Y1 Y2
 
-%macro: allocate_init AllocateInit -pack
 allocate_init t I y
 
 #################################################################
@@ -1027,10 +960,8 @@ bif2 Fail Bif S1 S2 Dst => i_bif2 Fail Bif S1 S2 Dst
 i_get_hash c I d
 i_get s d
 
-%macro: self Self
 self xy
 
-%macro: node Node
 node x
 %cold
 node y
@@ -1050,35 +981,20 @@ i_bif2_body b s s d
 # Internal calls.
 #
 
-move S=c x==0 | call Ar P=f => i_move_call S P
-move S=s x==0 | call Ar P=f => move_call S P
-
-i_move_call c f
+move S=cxy x==0 | call Ar P=f => move_call S P
 
-%macro:move_call MoveCall -arg_f -size -nonext
 move_call/2
+move_call cxy f
 
-move_call xy f
-
-move S=c x==0 | call_last Ar P=f D => i_move_call_last P D S
 move S x==0 | call_last Ar P=f D => move_call_last S P D
 
-i_move_call_last f P c
-
-%macro:move_call_last MoveCallLast -arg_f -nonext -pack
-
 move_call_last/3
-move_call_last xy f Q
-
-move S=c x==0 | call_only Ar P=f => i_move_call_only P S
-move S=x x==0 | call_only Ar P=f => move_call_only S P
+move_call_last cxy f Q
 
-i_move_call_only f c
+move S=cx x==0 | call_only Ar P=f => move_call_only S P
 
-%macro:move_call_only MoveCallOnly -arg_f -nonext
 move_call_only/2
-
-move_call_only x f
+move_call_only cx f
 
 call Ar Func        => i_call Func
 call_last Ar Func D => i_call_last Func D
@@ -1106,12 +1022,10 @@ i_call_fun_last I P
 
 make_fun2 OldIndex=u => gen_make_fun2(OldIndex)
 
-%macro: i_make_fun MakeFun -pack
 %cold
 i_make_fun I t
 %hot
 
-%macro: is_function IsFunction -fail_action
 is_function f xy
 is_function Fail=f c => jump Fail
 
@@ -1152,10 +1066,6 @@ i_bs_get_integer_32 x f I x
 bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \
 			gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
 
-%macro: i_bs_get_binary_imm2 BsGetBinaryImm_2 -fail_action
-%macro: i_bs_get_binary2 BsGetBinary_2 -fail_action
-%macro: i_bs_get_binary_all2 BsGetBinaryAll_2 -fail_action
-
 i_bs_get_binary_imm2 f x I I I x
 i_bs_get_binary2 f x I s I x
 i_bs_get_binary_all2 f x I I x
@@ -1167,7 +1077,6 @@ bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \
 
 bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail
 
-%macro: i_bs_get_float2 BsGetFloat2 -fail_action
 i_bs_get_float2 f x I s I x
 
 # Miscellanous
@@ -1175,14 +1084,9 @@ i_bs_get_float2 f x I s I x
 bs_skip_bits2 Fail=f Ms=x Sz=sq Unit=u Flags=u => \
 			gen_skip_bits2(Fail, Ms, Sz, Unit, Flags)
 
-%macro: i_bs_skip_bits_imm2 BsSkipBitsImm2 -fail_action
 i_bs_skip_bits_imm2 f x I
-
-%macro: i_bs_skip_bits2 BsSkipBits2 -fail_action
 i_bs_skip_bits2 f x xy I
-
-%macro: i_bs_skip_bits_all2 BsSkipBitsAll2 -fail_action
-i_bs_skip_bits_all2 f x I 
+i_bs_skip_bits_all2 f x I
 
 bs_test_tail2 Fail=f Ms=x Bits=u==0 => bs_test_zero_tail2 Fail Ms
 bs_test_tail2 Fail=f Ms=x Bits=u => bs_test_tail_imm2 Fail Ms Bits
@@ -1294,9 +1198,6 @@ i_bs_private_append j I s s x
 bs_put_integer Fail=j Sz=sq Unit=u Flags=u Src=s => \
 			gen_put_integer(Fail, Sz, Unit, Flags, Src)
 
-%macro: i_new_bs_put_integer NewBsPutInteger
-%macro: i_new_bs_put_integer_imm NewBsPutIntegerImm
-
 i_new_bs_put_integer j s I s
 i_new_bs_put_integer_imm j I I s
 
@@ -1331,9 +1232,6 @@ bs_put_float Fail Sz=q Unit Flags Val => badarg Fail
 bs_put_float Fail=j Sz=s Unit=u Flags=u Src=s => \
 			gen_put_float(Fail, Sz, Unit, Flags, Src)
 
-%macro: i_new_bs_put_float NewBsPutFloat
-%macro: i_new_bs_put_float_imm NewBsPutFloatImm
-
 i_new_bs_put_float j s I s
 i_new_bs_put_float_imm j I I s
 
@@ -1344,13 +1242,8 @@ i_new_bs_put_float_imm j I I s
 bs_put_binary Fail=j Sz=s Unit=u Flags=u Src=s => \
 			gen_put_binary(Fail, Sz, Unit, Flags, Src)
 
-%macro: i_new_bs_put_binary NewBsPutBinary
 i_new_bs_put_binary j s I s
-
-%macro: i_new_bs_put_binary_imm NewBsPutBinaryImm
 i_new_bs_put_binary_imm j I s
-
-%macro: i_new_bs_put_binary_all NewBsPutBinaryAll
 i_new_bs_put_binary_all j s I
 
 #
@@ -1436,7 +1329,6 @@ update_map_exact j s d I I
 is_map Fail Lit=q | literal_is_map(Lit) =>
 is_map Fail cq => jump Fail
 
-%macro: is_map IsMap -fail_action
 is_map f xy
 
 ## Transform has_map_fields #{ K1 := _, K2 := _ } to has_map_elements
@@ -1456,10 +1348,8 @@ i_get_map_elements f s I
 i_get_map_element Fail Src=xy Key=y Dst => \
     move Key x | i_get_map_element Fail Src x Dst
 
-%macro: i_get_map_element_hash GetMapElementHash -fail_action
 i_get_map_element_hash f xy c I xy
 
-%macro: i_get_map_element GetMapElement -fail_action
 i_get_map_element f xy x xy
 
 #
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 0a30553f71..8041a96bcb 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -91,8 +91,9 @@ my @op_to_name;
 
 my @obsolete;
 
-my %macro;
-my %macro_flags;
+# Instructions implemented in C.
+my %c_code;                     # C code block, location, arguments.
+my %c_code_used;                # Used or not.
 
 my %hot_code;
 my %cold_code;
@@ -259,8 +260,25 @@ if ($wordsize == 64) {
 # Parse the input files.
 #
 
+my $in_c_code = '';
+my $c_code_block;
+my $c_code_loc;
+my @c_args;
+
 while (<>) {
     my($op_num);
+    if ($in_c_code) {
+        if (/^\}/) {
+            $c_code_block =~ s/^    //mg;
+            chomp $c_code_block;
+            $c_code{$in_c_code} =
+                [$c_code_block,$c_code_loc,@c_args];
+            $in_c_code = '';
+        } else {
+            $c_code_block .= $_;
+        }
+        next;
+    }
     chomp;
     if (s/\\$//) {
 	$_ .= <>;
@@ -268,6 +286,7 @@ while (<>) {
     }
     next if /^\s*$/;
     next if /^\#/;
+    next if m@^//@;
 
     #
     # Handle %if.
@@ -325,23 +344,6 @@ while (<>) {
 	$hot = 0;
 	next;
     }
-    
-    #
-    # Handle macro definitions.
-    #
-    if (/^\%macro:(.*)/) {
-	my($op, $macro, @flags) = split(' ', $1);
-	defined($macro) and $macro =~ /^-/ and
-	    error("A macro must not start with a hyphen");
-	foreach (@flags) {
-	    /^-/ or error("Flags for macros should start with a hyphen");
-	}
-	error("Macro for '$op' is already defined")
-	    if defined $macro{$op};
-	$macro{$op} = $macro;
-	$macro_flags{$op} = join('', @flags);
-	next;
-    }
 
     #
     # Handle transformations.
@@ -351,6 +353,22 @@ while (<>) {
 	next;
     }
 
+    #
+    # Handle C code blocks.
+    #
+    if (/^(\w+)\(([^\)]*)\)\s*{/) {
+        my $name = $1;
+        $in_c_code = $name;
+        $c_code_block = '';
+        @c_args = parse_c_args($2);
+        $c_code_loc = "$ARGV($.)";
+        if (defined $c_code{$name}) {
+            my $where = $c_code{$name}->[1];
+            error("$name: already defined at $where");
+        }
+        next;
+    }
+
     #
     # Parse off the number of the operation.
     #
@@ -448,6 +466,18 @@ $num_file_opcodes = @gen_opname;
 
 &$target();
 
+#
+# Ensure that all C code implementations have been used.
+#
+{
+    my(@unused) = grep(!$c_code_used{$_}, keys %c_code);
+    foreach my $unused (@unused) {
+        my(undef,$where) = @{$c_code{$unused}};
+        warn "$where: $unused is unused\n";
+    }
+    die "\n" if @unused;
+}
+
 #
 # Produce output needed by the emulator/loader.
 #
@@ -893,6 +923,18 @@ sub save_specific_ops {
     }
 }
 
+sub parse_c_args {
+    local($_) = @_;
+    my @res;
+
+    while (s/^(\w[\w\d]*)\s*//) {
+        push @res, $1;
+        s/^,\s*// or last;
+    }
+    $_ eq '' or error("garbage in argument list: $_");
+    @res;
+}
+
 sub error {
     my(@message) = @_;
     my($where) = $. ? "$ARGV($.): " : "";
@@ -940,51 +982,20 @@ sub comment {
 
 sub basic_generator {
     my($name, $hot, @args) = @_;
-    my($size) = 0;
-    my($macro) = '';
-    my($flags) = '';
-    my(@f);
-    my(@f_types);
-    my($fail_type);
-    my($prefix) = '';
-    my($tmp_arg_num) = 1;
-    my($pack_spec) = '';
-    my($var_decls) = '';
-    my($i);
-    my($no_prefetch) = 0;
-
-    # The following argument types should be included as macro arguments.
-    my(%incl_arg) = ('c' => 1,
-		     'i' => 1,
-		     'a' => 1,
-		     'A' => 1,
-		     'N' => 1,
-		     'U' => 1,
-		     'I' => 1,
-		     't' => 1,
-		     'P' => 1,
-		     'Q' => 1,
-		     );
-
-    # Pick up the macro to use and its flags (if any).
-
-    $macro = $macro{$name} if defined $macro{$name};
-    $flags = $macro_flags{$name} if defined $macro_flags{$name};
+    my $size = 0;
+    my $flags = '';
+    my @f;
+    my $prefix = '';
+    my $tmp_arg_num = 1;
+    my $pack_spec = '';
+    my $var_decls = '';
 
     #
-    # Add any arguments to be included as macro arguments (for instance,
-    # 'p' is usually not an argument, except for calls).
+    # Pack arguments for hot code with an implementation.
     #
 
-    while ($flags =~ /-arg_(\w)/g) {
-	$incl_arg{$1} = 1;
-    };
-
-    #
-    # Pack arguments if requested.
-    #
-
-    if ($flags =~ /-pack/ && $hot) {
+    my $c_code = $c_code{$name};
+    if ($hot and defined $c_code) {
         ($prefix, $pack_spec, @args) = do_pack(@args);
     }
 
@@ -997,40 +1008,41 @@ sub basic_generator {
 	my($this_size) = $arg_size{$_};
       SWITCH:
 	{
-	    /^pack:(\d):(.*)/ and do { push(@f, $2);
-				       push(@f_types, 'packed');
-				       $this_size = $1;
-				       last SWITCH;
-				   };
-	    /r/    and do { push(@f, "r(0)"); push(@f_types, $_); last SWITCH };
-	    /[xy]/ and do { push(@f, "$_" . "b(Arg($size))");
-			     push(@f_types, $_);
-			     last SWITCH;
-			};
-	    /n/    and do { push(@f, "NIL"); push(@f_types, $_); last SWITCH };
-	    /s/    and do { my($tmp) = "targ$tmp_arg_num";
-			    $var_decls .= "Eterm $tmp; ";
-			    $tmp_arg_num++;
-			    push(@f, $tmp);
-			    push(@f_types, $_);
-			    $prefix .= "GetR($size, $tmp);\n";
-			    last SWITCH; };
-	    /d/    and do { $var_decls .= "Eterm dst; Eterm* dst_ptr; ";
-			    push(@f, "*dst_ptr");
-			    push(@f_types, $_);
-			    $prefix .= "dst = Arg($size);\n";
-			    $prefix .= "dst_ptr = REG_TARGET_PTR(dst);\n";
-			    last SWITCH;
-			};
-	    defined($incl_arg{$_})
-		and do { push(@f, "Arg($size)");
-			 push(@f_types, $_);
-			 last SWITCH;
-		     };
-
-	    /[fp]/ and do { $fail_type = $_; last SWITCH };
-
-	    /[eLIFEbASjPowlq]/ and do { last SWITCH; };
+	    /^pack:(\d):(.*)/ and do {
+                push(@f, $2);
+                $this_size = $1;
+                last SWITCH;
+            };
+	    /r/ and do {
+                push(@f, "r(0)");
+                last SWITCH;
+            };
+	    /[lxy]/ and do {
+                push(@f, $_ . "b(Arg($size))");
+                last SWITCH;
+            };
+	    /n/ and do {
+                push(@f, "NIL");
+                last SWITCH;
+            };
+	    /s/ and do {
+                my($tmp) = "targ$tmp_arg_num";
+                $var_decls .= "Eterm $tmp;\n";
+                $tmp_arg_num++;
+                push(@f, $tmp);
+                $prefix .= "GetR($size, $tmp);\n";
+                last SWITCH;
+            };
+	    /d/ and do {
+                $var_decls .= "Eterm dst = Arg($size);\n" .
+                    "Eterm* dst_ptr = REG_TARGET_PTR(dst);\n";
+                push(@f, "*dst_ptr");
+                last SWITCH;
+            };
+            defined $arg_size{$_} and do {
+                push(@f, "Arg($size)");
+                last SWITCH;
+            };
 
 	    die "$name: The generator can't handle $_, at";
 	}
@@ -1038,77 +1050,156 @@ sub basic_generator {
     }
 
     #
-    # Add a fail action macro if requested.
+    # If the implementation is in beam_emu.c, there is nothing
+    # more to do.
     #
+    unless (defined $c_code) {
+        return ($size+1, undef, '');
+    }
 
-    $flags =~ /-fail_action/ and do {
-	$no_prefetch = 1;
-	if (!defined $fail_type) {
-	    my($i);
-	    for ($i = 0; $i < @f_types; $i++) {
-		local($_) = $f_types[$i];
-		/[rxycians]/ and do { push(@f, "Badmatch($f[$i])"); next };
-	    }
-	} elsif ($fail_type eq 'f') {
-	    push(@f, "ClauseFail()");
-	} else {
-	    my($i);
-	    for ($i = 0; $i < @f_types; $i++) {
-		local($_) = $f_types[$i];
-		/[rxycians]/ and do { push(@f, "Badmatch($f[$i])"); next };
-	    }
-	}
-    };
+    #
+    # Generate main body of the implementation.
+    #
+    my $macro_code;
+    if (defined $c_code) {
+        my($c_code,$where,@c_args) = @{$c_code};
+        my %bindings;
+        $c_code_used{$name} = 1;
+
+        if (@f != @c_args) {
+            error("$where: defining '$name' with ", scalar(@c_args),
+                  " arguments instead of expected ", scalar(@f), " arguments");
+        }
+
+        for (my $i = 0; $i < @f; $i++) {
+            my $var = $c_args[$i];
+            $bindings{$var} = $f[$i];
+        }
+        $bindings{'NEXT_INSTRUCTION'} = "I+" . ($size + 1);
+        $c_code = eval { expand_all($c_code, \%bindings) };
+        unless (defined $c_code) {
+            warn $@;
+            error("... from the body of $name at $where");
+        }
+        my(@comments) = $c_code =~ m@//[|]\s*(.*)@g;
+        $flags = "@comments";
+        $macro_code = "$prefix$c_code";
+    }
 
     #
-    # Add a size argument if requested.
+    # Generate code for transferring to the next instruction.
     #
+    my $dispatch_next;
+    my $offset = $size + 1;
+
+    if ($flags =~ /-no_next/) {
+        $dispatch_next = "";
+    } elsif ($flags =~ /-no_prefetch/) {
+        $dispatch_next = "\nI += $offset;\n" .
+            "ASSERT(VALID_INSTR(*I));\n" .
+            "Goto(*I);";
+    } else {
+        $var_decls .= "BeamInstr* _nextpf = " .
+            "(BeamInstr *) I[$offset];\n";
+        $dispatch_next = "\nI += $offset;\n" .
+            "ASSERT(VALID_INSTR(_nextpf));\n" .
+            "Goto(_nextpf);";
+    }
 
-    $flags =~ /-size/ and do {
-	push(@f, $size);
-    };
-
-    # Generate the macro if requested.
-    my($code);
-    if (defined $macro{$name}) {
-	my($macro_code) = "$prefix$macro(" . join(', ', @f) . ");";
-	$var_decls .= "BeamInstr tmp_packed1;"
-	    if $macro_code =~ /tmp_packed1/;
-	$var_decls .= "BeamInstr tmp_packed2;"
-	    if $macro_code =~ /tmp_packed2/;
-	if ($flags =~ /-nonext/) {
-	    $code = join("\n",
-			 "{ $var_decls",
-			 $macro_code,
-			 "}");
-	} elsif ($flags =~ /-goto:(\S*)/) {
-	    my $goto = $1;
-	    $code = join("\n",
-			 "{ $var_decls",
-			 $macro_code,
-			 "I += $size + 1;",
-			 "goto $goto;",
-			 "}");
-	} elsif ($no_prefetch) {
-	    $code = join("\n",
-			 "{ $var_decls",
-			 $macro_code,
-			 "Next($size);",
-			 "}", "");
-	} else {
-	    $code = join("\n",
-			 "{ $var_decls",
-			 "BeamInstr* next;",
-			 "PreFetch($size, next);",
-			 "$macro_code",
-			 "NextPF($size, next);",
-			 "}", "");
-	}
+    #
+    # Assemble the complete code for the instruction.
+    #
+    my $code = join("\n",
+                    "{",
+                    "$var_decls$macro_code$dispatch_next",
+                    "}", "");
+    ($size+1, $code, $pack_spec);
+}
+
+sub expand_all {
+    my($code,$bindings_ref) = @_;
+    my %bindings = %{$bindings_ref};
+
+    # Expand all $Var occurrences.
+    $code =~ s/[\$](\w[\w\d]*)(?!\()/defined $bindings{$1} ? $bindings{$1} : "\$$1"/ge;
+
+    # Find calls to macros, $name(...), and expand them.
+    my $res = "";
+    while ($code =~ /[\$](\w[\w\d]*)\(/) {
+        my $macro_name = $1;
+        my $keep = substr($code, 0, $-[0]);
+        my $after = substr($code, $+[0]);
+
+        # Keep the special, pre-defined bindings.
+        my %new_bindings;
+        foreach my $key (qw(NEXT_INSTRUCTION)) {
+            $new_bindings{$key} = $bindings{$key};
+        }
+
+        my $body;
+        ($body,$code) = expand_macro($macro_name, $after, \%new_bindings);
+        $res .= "$keep$body";
+    }
+
+    $res . $code;
+}
+
+sub expand_macro {
+    my($name,$rest,$bindings_ref) = @_;
+
+    my $c_code = $c_code{$name};
+    defined $c_code or
+        error("calling undefined macro '$name'...");
+    $c_code_used{$name} = 1;
+    my ($body,$where,@vars) = @{$c_code};
+
+    # Separate the arguments into @args;
+    my @args;
+    my $level = 1;
+    my %inc = ('(' => 1, ')' => -1,
+               '[' => 1, ']' => -1,
+               '{' => 1, '}' => -1);
+    my $arg = undef;
+    while ($rest =~ /([,\(\[\{\}\]\)]|([^,\(\[\{\}\]\)]*))/g) {
+        my $token = $1;
+        my $inc = $inc{$token} || 0;
+        $level += $inc;
+        if ($level == 0) {
+            $rest = substr($rest, pos($rest));
+            push @args, $arg if defined $arg;
+            last;
+        }
+        if ($token eq ',') {
+            if ($level == 1) {
+                push @args, $arg;
+                $arg = "";
+            }
+            next;
+        }
+        $arg .= $token;
+    }
+
+    # Trim leading whitespace from each argument.
+    foreach my $arg (@args) {
+        $arg =~ s/^\s*//;
     }
 
-    # Return the size and code for the macro (if any).
-    $size++;
-    ($size, $code, $pack_spec);
+    # Now combine bindings from the parameter names and arguments.
+    my %bindings = %{$bindings_ref};
+    if (@vars != @args) {
+        error("calling $name with ", scalar(@args),
+              " arguments instead of expected ", scalar(@vars), " arguments...");
+    }
+    for (my $i = 0; $i < @vars; $i++) {
+        $bindings{$vars[$i]} = $args[$i];
+    }
+
+    $body = eval { expand_all($body, \%bindings) };
+    unless (defined $body) {
+        warn $@;
+        die "... from the body of $name at $where\n";
+    }
+    ("do {\n$body\n} while (0)",$rest);
 }
 
 sub do_pack {
@@ -1201,7 +1292,7 @@ sub do_pack {
 	    $did_some_packing = 1;
 
 	    if ($ap == 0) {
-		$pack_prefix .= "tmp_packed$tmpnum = Arg($size);\n";
+		$pack_prefix .= "Eterm tmp_packed$tmpnum = Arg($size);\n";
 		$up .= "p";
 		$down = "P$down";
 		$this_size = 1;
-- 
cgit v1.2.3


From e201a3d8ff9c8b1dfb978a8cf86a729834024c1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 23 May 2017 18:11:51 +0200
Subject: Introduce micro instructions

beam_makeops will place all micro instructions in a block and generate
goto instructions from one micro instruction to the next. It will also
add adjustments of 'I' if necessary (if the micro instructions have
different length).
---
 erts/emulator/utils/beam_makeops | 305 +++++++++++++++++++++++++++++++++------
 1 file changed, 263 insertions(+), 42 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 8041a96bcb..6b202176d0 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -91,10 +91,14 @@ my @op_to_name;
 
 my @obsolete;
 
-# Instructions implemented in C.
+# Instructions and micro instructions implemented in C.
 my %c_code;                     # C code block, location, arguments.
 my %c_code_used;                # Used or not.
 
+# Definitions for instructions combined from micro instructions.
+my %combined_instrs;
+my %combined_code;           # Combined micro instructions.
+
 my %hot_code;
 my %cold_code;
 
@@ -103,6 +107,7 @@ my %unnumbered;
 
 my %is_transformed;
 
+
 #
 # Pre-processor.
 #
@@ -265,15 +270,21 @@ my $c_code_block;
 my $c_code_loc;
 my @c_args;
 
+sub save_c_code {
+    my($name,$block,$loc,@args) = @_;
+
+}
+
 while (<>) {
     my($op_num);
     if ($in_c_code) {
         if (/^\}/) {
-            $c_code_block =~ s/^    //mg;
-            chomp $c_code_block;
-            $c_code{$in_c_code} =
-                [$c_code_block,$c_code_loc,@c_args];
+            my $name = $in_c_code;
+            my $block = $c_code_block;
             $in_c_code = '';
+            $block =~ s/^    //mg;
+            chomp $block;
+            $c_code{$name} = [$block,$c_code_loc,@c_args];
         } else {
             $c_code_block .= $_;
         }
@@ -356,7 +367,7 @@ while (<>) {
     #
     # Handle C code blocks.
     #
-    if (/^(\w+)\(([^\)]*)\)\s*{/) {
+    if (/^(\w[\w.]*)\(([^\)]*)\)\s*{/) {
         my $name = $1;
         $in_c_code = $name;
         $c_code_block = '';
@@ -369,6 +380,15 @@ while (<>) {
         next;
     }
 
+    #
+    # Handle definition of instructions in terms of
+    # micro instructions.
+    #
+    if (/^(\w+)\s*:=\s*([\w.]+)\s*;\s*$/) {
+        $combined_instrs{$1} = ["$ARGV($.)","beam_instrs.h",$2];
+        next;
+    }
+
     #
     # Parse off the number of the operation.
     #
@@ -515,6 +535,11 @@ sub emulator_output {
     print "\n};\n";
     print "\n";
 
+    #
+    # Combine micro instruction into instruction blocks.
+    #
+    combine_micro_instructions();
+
     #
     # Generate code for specific ops.
     #
@@ -555,7 +580,8 @@ sub emulator_output {
 	    # Call a generator to calculate size and generate macros
 	    # for the emulator.
 	    #
-	    my($size, $code, $pack) = basic_generator($name, $hot, @args);
+	    my($size, $code, $pack) =
+                basic_generator($name, $hot, '', 0, undef, @args);
 
 	    #
 	    # Save the generated $code for later.
@@ -787,6 +813,12 @@ sub emulator_output {
     comment('C');
     print_code(\%cold_code);
 
+    foreach my $key (keys %combined_code) {
+        my $name = "$outdir/$key";
+        open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
+        comment('C');
+        print @{$combined_code{$key}};
+    }
 }
 
 sub init_item {
@@ -975,13 +1007,193 @@ sub comment {
     print "\n";
 }
 
+#
+# Combine micro instruction into instruction blocks.
+#
+sub combine_micro_instructions {
+    my %groups;
+    my %group_file;
+
+    # Sanity check, normalize micro instructions.
+    foreach my $instr (keys %combined_instrs) {
+        my $ref = $combined_instrs{$instr};
+        my($def_loc,$outfile,$def) = @$ref;
+        my($group,@subs) = split /[.]/, $def;
+        my $arity = 0;
+        @subs = map { "$group.$_" } @subs;
+        foreach my $s (@subs) {
+            my $code = $c_code{$s};
+            defined $code or
+                error("$def_loc: no definition of $s");
+            $c_code_used{$s} = 1;
+            my(undef,undef,@c_args) = @{$code};
+            $arity += scalar(@c_args);
+        }
+        push @{$groups{$group}}, [$instr,$arity,@subs];
+        $group_file{$group} = $outfile;
+    }
+
+    # Now generate code for each group.
+    foreach my $group (sort keys %groups) {
+        my $code = combine_instruction_group($group, @{$groups{$group}});
+        my $outfile = $group_file{$group};
+        push @{$combined_code{$outfile}}, $code;
+    }
+}
+
+sub combine_instruction_group {
+    my($group,@in_instrs) = @_;
+    my $gcode = '';             # Code for the entire group.
+
+    # Get code for the head of the group (if any).
+    my $head_name = "$group.head";
+    $c_code_used{$head_name} = 1;
+    my $head_code_ref = $c_code{$head_name};
+    if (defined $head_code_ref) {
+        my($head_code,$where,@c_args) = @{$head_code_ref};
+        @c_args and error("$where: no arguments allowed for " .
+                          "head function '$head_name()'");
+        $gcode = $head_code . "\n";
+    }
+
+    # Variables.
+    my %offsets;
+    my @instrs;
+    my %num_references;
+    my $group_size = 0;
+
+    # Do basic error checking. Associate operands of instructions
+    # with the correct micro instructions. Calculate offsets for micro
+    # instructions.
+    foreach my $ref_instr (@in_instrs) {
+        my($specific,$arity,@subs) = @$ref_instr;
+        my $specific_key = "$specific/$arity";
+        my $specific_op_ref = $specific_op{$specific_key};
+        error("no $specific_key instruction")
+            unless defined $specific_op_ref;
+        foreach my $specific_op (@$specific_op_ref) {
+            my($name, $hot, @args) = @{$specific_op};
+            my $offset = 0;
+            my @rest = @args;
+            my @new_subs;
+            my $opcase = $specific;
+            $opcase .= "_" . join '', @args if @args;
+            foreach my $s (@subs) {
+                my $code = $c_code{$s};
+                my(undef,undef,@c_args) = @{$code};
+                my @first;
+                foreach (0..$#c_args) {
+                    push @first, shift @rest;
+                }
+                my($size,undef) = basic_generator($s, 0, '', 0, undef, @first);
+                $offsets{$s} = $offset
+                    unless defined $offsets{$s} and $offsets{$s} >= $offset;
+                $offset += $size - 1;
+                my $label = micro_label($s);
+                $num_references{$label} = 0;
+                push @new_subs, [$opcase,$label,$s,$size-1,@first];
+                $opcase = '';
+            }
+            $group_size = $offset if $group_size < $offset;
+            push @instrs, [$specific_key,@new_subs];
+        }
+    }
+
+    # Link the sub instructions for each instructions to each
+    # other.
+    my @all_instrs;
+    foreach my $instr (@instrs) {
+        my($specific_key,@subs) = @{$instr};
+        for (my $i = 0; $i < @subs; $i++) {
+            my($opcase,$label,$s,$size,@args) = @{$subs[$i]};
+            my $next = '';
+            (undef,$next) = @{$subs[$i+1]} if $i < $#subs;
+            $num_references{$next}++ if $next;
+            my $instr_info = "$opcase:$label:$next:$s:$size:@args";
+            push @all_instrs, [$label,$offsets{$s},$instr_info];
+        }
+    }
+
+    my %order_to_instrs;
+    my %label_to_offset;
+    my %order_to_offset;
+    foreach my $instr (@all_instrs) {
+        my($label,$offset,$instr_info) = @$instr;
+        my $sort_key = sprintf("%02d.%02d", $offset, $num_references{$label});
+        push @{$order_to_instrs{$sort_key}}, $instr_info;
+        $label_to_offset{$label} = $offset;
+        $order_to_offset{$sort_key} = $offset;
+    }
+
+    my(@slots) = sort {$a <=> $b} keys %order_to_instrs;
+
+    # Now generate the code for the entire group.
+    my $offset = 0;
+    for(my $i = 0; $i < @slots; $i++) {
+        my $key = $slots[$i];
+
+        # Sort micro-instructions with OpCase before other micro-instructions.
+        my(@instrs) = @{$order_to_instrs{$key}};
+        my $order_func = sub {
+            my $a_key = ($a =~ /^:/) ? "1$a" : "0$a";
+            my $b_key = ($b =~ /^:/) ? "1$b" : "0$b";
+            $a_key cmp $b_key;
+        };
+        @instrs = sort $order_func @instrs;
+
+        my %seen;
+        foreach my $instr (@instrs) {
+            my($opcase,$label,$next,$s,$size,$args) = split ":", $instr;
+            my(@first) = split " ", $args;
+
+            my $seen_key = "$label:$next:" . scalar(@first);
+            next if $opcase eq '' and $seen{$seen_key};
+            $seen{$seen_key} = 1;
+
+            if ($opcase ne '') {
+                $gcode .= "OpCase($opcase):\n";
+            }
+            if ($num_references{$label}) {
+                $gcode .= "$label:\n";
+            }
+
+            my $flags = '';
+            my $transfer_to_next = '';
+            my $dec = 0;
+
+            unless ($i == $#slots) {
+                $flags = "-no_next";
+                my $next_offset = $label_to_offset{$next};
+                $dec = $next_offset - ($offset + $size);
+                $transfer_to_next = "I -= $dec;\n" if $dec;
+                $transfer_to_next .= "goto $next;\n\n";
+            }
+
+            my(undef,$gen_code) =
+                basic_generator($s, 0, $flags, $offset,
+                                $group_size-$offset-$dec, @first);
+            $gcode .= $gen_code . $transfer_to_next;
+        }
+        $offset = $order_to_offset{$slots[$i+1]} if $i < $#slots;
+    }
+
+    "{\n$gcode\n}\n\n";
+}
+
+sub micro_label {
+    my $label = shift;
+    $label =~ s/[.]/__/g;
+    $label;
+}
+
+
 #
 # Basic implementation of instruction in emulator loop
 # (assuming no packing).
 #
 
 sub basic_generator {
-    my($name, $hot, @args) = @_;
+    my($name,$hot,$extra_comments,$offset,$group_size,@args) = @_;
     my $size = 0;
     my $flags = '';
     my @f;
@@ -994,8 +1206,8 @@ sub basic_generator {
     # Pack arguments for hot code with an implementation.
     #
 
-    my $c_code = $c_code{$name};
-    if ($hot and defined $c_code) {
+    my $c_code_ref = $c_code{$name};
+    if ($hot and defined $c_code_ref) {
         ($prefix, $pack_spec, @args) = do_pack(@args);
     }
 
@@ -1004,6 +1216,8 @@ sub basic_generator {
     # the macro.
     #
 
+    my $need_block = 0;
+    my $arg_offset = $offset;
     foreach (@args) {
 	my($this_size) = $arg_size{$_};
       SWITCH:
@@ -1018,7 +1232,7 @@ sub basic_generator {
                 last SWITCH;
             };
 	    /[lxy]/ and do {
-                push(@f, $_ . "b(Arg($size))");
+                push(@f, $_ . "b(Arg($arg_offset))");
                 last SWITCH;
             };
 	    /n/ and do {
@@ -1030,78 +1244,79 @@ sub basic_generator {
                 $var_decls .= "Eterm $tmp;\n";
                 $tmp_arg_num++;
                 push(@f, $tmp);
-                $prefix .= "GetR($size, $tmp);\n";
+                $prefix .= "GetR($arg_offset, $tmp);\n";
+                $need_block = 1;
                 last SWITCH;
             };
 	    /d/ and do {
-                $var_decls .= "Eterm dst = Arg($size);\n" .
+                $var_decls .= "Eterm dst = Arg($arg_offset);\n" .
                     "Eterm* dst_ptr = REG_TARGET_PTR(dst);\n";
                 push(@f, "*dst_ptr");
                 last SWITCH;
             };
             defined $arg_size{$_} and do {
-                push(@f, "Arg($size)");
+                push(@f, "Arg($arg_offset)");
                 last SWITCH;
             };
 
 	    die "$name: The generator can't handle $_, at";
 	}
 	$size += $this_size;
+        $arg_offset += $this_size;
     }
 
     #
     # If the implementation is in beam_emu.c, there is nothing
     # more to do.
     #
-    unless (defined $c_code) {
+    unless (defined $c_code_ref) {
         return ($size+1, undef, '');
     }
 
+    $group_size = $size unless defined $group_size;
+
     #
     # Generate main body of the implementation.
     #
-    my $macro_code;
-    if (defined $c_code) {
-        my($c_code,$where,@c_args) = @{$c_code};
-        my %bindings;
-        $c_code_used{$name} = 1;
+    my($c_code,$where,@c_args) = @{$c_code_ref};
+    my %bindings;
+    $c_code_used{$name} = 1;
 
-        if (@f != @c_args) {
-            error("$where: defining '$name' with ", scalar(@c_args),
-                  " arguments instead of expected ", scalar(@f), " arguments");
-        }
+    if (@f != @c_args) {
+        error("$where: defining '$name' with ", scalar(@c_args),
+              " arguments instead of expected ", scalar(@f), " arguments");
+    }
 
-        for (my $i = 0; $i < @f; $i++) {
-            my $var = $c_args[$i];
-            $bindings{$var} = $f[$i];
-        }
-        $bindings{'NEXT_INSTRUCTION'} = "I+" . ($size + 1);
-        $c_code = eval { expand_all($c_code, \%bindings) };
-        unless (defined $c_code) {
-            warn $@;
-            error("... from the body of $name at $where");
-        }
-        my(@comments) = $c_code =~ m@//[|]\s*(.*)@g;
-        $flags = "@comments";
-        $macro_code = "$prefix$c_code";
+    for (my $i = 0; $i < @f; $i++) {
+        my $var = $c_args[$i];
+        $bindings{$var} = $f[$i];
     }
+    $bindings{'NEXT_INSTRUCTION'} = "I+" . ($group_size+$offset+1);
+    $c_code = eval { expand_all($c_code, \%bindings) };
+    unless (defined $c_code) {
+        warn $@;
+        error("... from the body of $name at $where");
+    }
+    my(@comments) = $c_code =~ m@//[|]\s*(.*)@g;
+    $c_code =~ s@//[|]\s*(.*)\n?@@g;
+    $flags = "@comments $extra_comments";
 
     #
     # Generate code for transferring to the next instruction.
     #
     my $dispatch_next;
-    my $offset = $size + 1;
+    my $instr_offset = $group_size + $offset + 1;
 
     if ($flags =~ /-no_next/) {
         $dispatch_next = "";
     } elsif ($flags =~ /-no_prefetch/) {
-        $dispatch_next = "\nI += $offset;\n" .
+        $dispatch_next = "\nI += $instr_offset;\n" .
             "ASSERT(VALID_INSTR(*I));\n" .
             "Goto(*I);";
     } else {
         $var_decls .= "BeamInstr* _nextpf = " .
-            "(BeamInstr *) I[$offset];\n";
-        $dispatch_next = "\nI += $offset;\n" .
+            "(BeamInstr *) I[$instr_offset];\n";
+        $dispatch_next = "\nI += $instr_offset;\n" .
             "ASSERT(VALID_INSTR(_nextpf));\n" .
             "Goto(_nextpf);";
     }
@@ -1109,9 +1324,15 @@ sub basic_generator {
     #
     # Assemble the complete code for the instruction.
     #
+    my $body = "$c_code$dispatch_next";
+    if ($need_block) {
+        $body = "$prefix\{\n$body\n}";
+    } else {
+        $body = "$prefix$body";
+    }
     my $code = join("\n",
                     "{",
-                    "$var_decls$macro_code$dispatch_next",
+                    "$var_decls$body",
                     "}", "");
     ($size+1, $code, $pack_spec);
 }
-- 
cgit v1.2.3


From 426af3250c3f1bdb121413b8b054b63c1bc60fa6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 27 Jun 2017 09:21:39 +0200
Subject: beam_makeops: Define ARCH_32 and ARCH_64

---
 erts/emulator/utils/beam_makeops | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 6b202176d0..fca596e4e5 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -246,6 +246,14 @@ while (@ARGV && $ARGV[0] =~ /^-(.*)/) {
     die "$0: Bad option: -$_\n";
 }
 
+if ($wordsize == 32) {
+    $defs{'ARCH_32'} = 1;
+    $defs{'ARCH_64'} = 0;
+} elsif ($wordsize == 64) {
+    $defs{'ARCH_32'} = 0;
+    $defs{'ARCH_64'} = 1;
+}
+
 #
 # Initialize number of arguments per packed word.
 #
-- 
cgit v1.2.3


From e1be82aba88fd01926bcfc88757ae3257eadaf16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 6 Jul 2017 16:00:50 +0200
Subject: beam_makeops: Pretty-print the generated code

---
 erts/emulator/utils/beam_makeops | 35 +++++++++++++++++++++++++++++------
 1 file changed, 29 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index fca596e4e5..da69b13e87 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -825,7 +825,7 @@ sub emulator_output {
         my $name = "$outdir/$key";
         open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
         comment('C');
-        print @{$combined_code{$key}};
+        print_indented_code(@{$combined_code{$key}});
     }
 }
 
@@ -865,18 +865,41 @@ sub print_code {
 	    $code .= "OpCase($label):\n";
 	    $sort_key = $label;
 	}
-	foreach (split("\n", $key)) {
-	    $code .= "    $_\n";
-	}
-	$code .= "\n";
+        $code .= "$key\n";
 	$sorted{$sort_key} = $code;
     }
 
     foreach (sort keys %sorted) {
-	print $sorted{$_};
+	print_indented_code($sorted{$_});
+    }
+}
+
+sub print_indented_code {
+    my(@code) = @_;
+
+    foreach my $chunk (@code) {
+        my $indent = 0;
+        foreach (split "\n", $chunk) {
+            s/^\s*//;
+            if (/\}/) {
+                $indent -= 2;
+            }
+            if ($_ eq '') {
+                print "\n";
+            } elsif (/^#/) {
+                print $_, "\n";
+            } else {
+                print ' ' x $indent, $_, "\n";
+            }
+            if (/\{/) {
+                $indent += 2;
+            }
+        }
+        print "\n";
     }
 }
 
+
 #
 # Produce output needed by the compiler back-end (assembler).
 #
-- 
cgit v1.2.3


From 81a6adab693a75f89bc87911ac23a21308673d2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sat, 20 May 2017 17:59:03 +0200
Subject: Break out most instructions from beam_emu.c

---
 erts/emulator/Makefile.in            |   18 +-
 erts/emulator/beam/arith_instrs.tab  |  393 ++++
 erts/emulator/beam/beam_emu.c        | 4002 ++--------------------------------
 erts/emulator/beam/beam_load.c       |   51 +-
 erts/emulator/beam/bif_instrs.tab    |  539 +++++
 erts/emulator/beam/bs_instrs.tab     |  888 +++++++-
 erts/emulator/beam/float_instrs.tab  |   88 +
 erts/emulator/beam/instrs.tab        |  509 ++++-
 erts/emulator/beam/macros.tab        |  139 ++
 erts/emulator/beam/map_instrs.tab    |  162 ++
 erts/emulator/beam/msg_instrs.tab    |  382 ++++
 erts/emulator/beam/ops.tab           |   86 +-
 erts/emulator/beam/select_instrs.tab |  196 ++
 erts/emulator/beam/trace_instrs.tab  |  155 ++
 erts/emulator/hipe/hipe_instrs.tab   |  141 ++
 15 files changed, 3781 insertions(+), 3968 deletions(-)
 create mode 100644 erts/emulator/beam/arith_instrs.tab
 create mode 100644 erts/emulator/beam/bif_instrs.tab
 create mode 100644 erts/emulator/beam/float_instrs.tab
 create mode 100644 erts/emulator/beam/macros.tab
 create mode 100644 erts/emulator/beam/map_instrs.tab
 create mode 100644 erts/emulator/beam/msg_instrs.tab
 create mode 100644 erts/emulator/beam/select_instrs.tab
 create mode 100644 erts/emulator/beam/trace_instrs.tab
 create mode 100644 erts/emulator/hipe/hipe_instrs.tab

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 015a3a42ed..c6511a4b49 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -28,10 +28,22 @@ HIPE_ENABLED=@HIPE_ENABLED@
 DTRACE_ENABLED=@DTRACE_ENABLED@
 DTRACE_ENABLED_2STEP=@DTRACE_ENABLED_2STEP@
 USE_VM_PROBES=@USE_VM_PROBES@
+FPE=@FPE@
 LIBS = @LIBS@
 Z_LIB=@Z_LIB@
 NO_INLINE_FUNCTIONS=false
-OPCODE_TABLES = $(ERL_TOP)/lib/compiler/src/genop.tab beam/ops.tab beam/instrs.tab beam/bs_instrs.tab
+OPCODE_TABLES = $(ERL_TOP)/lib/compiler/src/genop.tab \
+beam/ops.tab \
+beam/macros.tab \
+beam/instrs.tab \
+beam/arith_instrs.tab \
+beam/bif_instrs.tab \
+beam/bs_instrs.tab \
+beam/float_instrs.tab \
+beam/map_instrs.tab \
+beam/msg_instrs.tab \
+beam/select_instrs.tab \
+beam/trace_instrs.tab
 
 DEBUG_CFLAGS = @DEBUG_CFLAGS@
 CONFIGURE_CFLAGS = @CFLAGS@
@@ -529,11 +541,12 @@ DTRACE_HEADERS =
 endif
 
 ifdef HIPE_ENABLED
-OPCODE_TABLES += hipe/hipe_ops.tab
+OPCODE_TABLES += hipe/hipe_ops.tab hipe/hipe_instrs.tab
 endif
 
 $(TTF_DIR)/beam_cold.h \
 $(TTF_DIR)/beam_hot.h \
+$(TTF_DIR)/beam_instrs.h \
 $(TTF_DIR)/beam_opcodes.c \
 $(TTF_DIR)/beam_opcodes.h \
 $(TTF_DIR)/beam_pred_funcs.h \
@@ -544,6 +557,7 @@ $(TTF_DIR)/OPCODES-GENERATED: $(OPCODE_TABLES) utils/beam_makeops
 		-wordsize @EXTERNAL_WORD_SIZE@ \
 		-outdir $(TTF_DIR) \
 		-DUSE_VM_PROBES=$(if $(USE_VM_PROBES),1,0) \
+		-DNO_FPE_SIGNALS=$(if $filter(unreliable,$(FPE)),1,0) \
 		-emulator $(OPCODE_TABLES) && echo $? >$(TTF_DIR)/OPCODES-GENERATED
 GENERATE += $(TTF_DIR)/OPCODES-GENERATED
 
diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab
new file mode 100644
index 0000000000..91fe21e161
--- /dev/null
+++ b/erts/emulator/beam/arith_instrs.tab
@@ -0,0 +1,393 @@
+// -*- c -*-
+//
+// %CopyrightBegin%
+//
+// Copyright Ericsson AB 2017. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// %CopyrightEnd%
+//
+
+OUTLINED_ARITH_2(Fail, Live, Name, BIF, Op1, Op2, Dst) {
+    Eterm result;
+    Uint live = $Live;
+    HEAVY_SWAPOUT;
+    reg[live] = $Op1;
+    reg[live+1] = $Op2;
+    result = erts_gc_$Name (c_p, reg, live);
+    HEAVY_SWAPIN;
+    ERTS_HOLE_CHECK(c_p);
+    if (is_value(result)) {
+        $REFRESH_GEN_DEST();
+        $Dst = result;
+        $NEXT0();
+    }
+    $BIF_ERROR_ARITY_2($Fail, $BIF, reg[live], reg[live+1]);
+}
+
+
+i_plus := plus.fetch.execute;
+
+plus.head() {
+    Eterm PlusOp1, PlusOp2;
+}
+
+plus.fetch(Op1, Op2) {
+    PlusOp1 = $Op1;
+    PlusOp2 = $Op2;
+}
+
+plus.execute(Fail, Live, Dst) {
+    if (is_both_small(PlusOp1, PlusOp2)) {
+        Sint i = signed_val(PlusOp1) + signed_val(PlusOp2);
+        ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
+        if (MY_IS_SSMALL(i)) {
+            $Dst = make_small(i);
+            $NEXT0();
+        }
+    }
+    $OUTLINED_ARITH_2($Fail, $Live, mixed_plus, BIF_splus_2, PlusOp1, PlusOp2, $Dst);
+}
+
+i_minus := minus.fetch.execute;
+
+minus.head() {
+    Eterm MinusOp1, MinusOp2;
+}
+
+minus.fetch(Op1, Op2) {
+    MinusOp1 = $Op1;
+    MinusOp2 = $Op2;
+}
+
+minus.execute(Fail, Live, Dst) {
+    if (is_both_small(MinusOp1, MinusOp2)) {
+        Sint i = signed_val(MinusOp1) - signed_val(MinusOp2);
+        ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
+        if (MY_IS_SSMALL(i)) {
+            $Dst = make_small(i);
+            $NEXT0();
+        }
+    }
+    $OUTLINED_ARITH_2($Fail, $Live, mixed_minus, BIF_sminus_2, MinusOp1, MinusOp2, $Dst);
+}
+
+i_increment := increment.fetch.execute;
+
+increment.head() {
+    Eterm increment_reg_val;
+    Eterm increment_val;
+    Uint live;
+    Eterm result;
+}
+
+increment.fetch(Src) {
+    increment_reg_val = $Src;
+}
+
+increment.execute(IncrementVal, Live, Dst) {
+    increment_val = $IncrementVal;
+    if (is_small(increment_reg_val)) {
+        Sint i = signed_val(increment_reg_val) + increment_val;
+        ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
+        if (MY_IS_SSMALL(i)) {
+            $Dst = make_small(i);
+            $NEXT0();
+        }
+    }
+    live = $Live;
+    HEAVY_SWAPOUT;
+    reg[live] = increment_reg_val;
+    reg[live+1] = make_small(increment_val);
+    result = erts_gc_mixed_plus(c_p, reg, live);
+    HEAVY_SWAPIN;
+    ERTS_HOLE_CHECK(c_p);
+    if (is_value(result)) {
+        $REFRESH_GEN_DEST();
+        $Dst = result;
+        $NEXT0();
+    }
+    ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue));
+    goto find_func_info;
+}
+
+i_times(Fail, Live, Op1, Op2, Dst) {
+    Eterm op1 = $Op1;
+    Eterm op2 = $Op2;
+    $OUTLINED_ARITH_2($Fail, $Live, mixed_times, BIF_stimes_2, op1, op2, $Dst);
+}
+
+i_m_div(Fail, Live, Op1, Op2, Dst) {
+    Eterm op1 = $Op1;
+    Eterm op2 = $Op2;
+    $OUTLINED_ARITH_2($Fail, $Live, mixed_div, BIF_div_2, op1, op2, $Dst);
+}
+
+i_int_div(Fail, Live, Op1, Op2, Dst) {
+    Eterm op1 = $Op1;
+    Eterm op2 = $Op2;
+    if (op2 == SMALL_ZERO) {
+        c_p->freason = BADARITH;
+        $BIF_ERROR_ARITY_2($Fail, BIF_intdiv_2, op1, op2);
+    } else if (is_both_small(op1, op2)) {
+        Sint ires = signed_val(op1) / signed_val(op2);
+        if (MY_IS_SSMALL(ires)) {
+            $Dst = make_small(ires);
+            $NEXT0();
+        }
+    }
+    $OUTLINED_ARITH_2($Fail, $Live, int_div, BIF_intdiv_2, op1, op2, $Dst);
+}
+
+i_rem := rem.fetch.execute;
+
+rem.head() {
+     Eterm RemOp1, RemOp2;
+}
+
+rem.fetch(Src1, Src2) {
+    RemOp1 = $Src1;
+    RemOp2 = $Src2;
+}
+
+rem.execute(Fail, Live, Dst) {
+     if (RemOp2 == SMALL_ZERO) {
+         c_p->freason = BADARITH;
+         $BIF_ERROR_ARITY_2($Fail, BIF_rem_2, RemOp1, RemOp2);
+     } else if (is_both_small(RemOp1, RemOp2)) {
+         $Dst = make_small(signed_val(RemOp1) % signed_val(RemOp2));
+         $NEXT0();
+     } else {
+         $OUTLINED_ARITH_2($Fail, $Live, int_rem, BIF_rem_2, RemOp1, RemOp2, $Dst);
+     }
+}
+
+i_band := band.fetch.execute;
+
+band.head() {
+    Eterm BandOp1, BandOp2;
+}
+
+band.fetch(Src1, Src2) {
+    BandOp1 = $Src1;
+    BandOp2 = $Src2;
+}
+
+band.execute(Fail, Live, Dst) {
+    if (is_both_small(BandOp1, BandOp2)) {
+        /*
+         * No need to untag -- TAG & TAG == TAG.
+         */
+        $Dst = BandOp1 & BandOp2;
+        $NEXT0();
+    }
+    $OUTLINED_ARITH_2($Fail, $Live, band, BIF_band_2, BandOp1, BandOp2, $Dst);
+}
+
+i_bor(Fail, Live, Src1, Src2, Dst) {
+    if (is_both_small($Src1, $Src2)) {
+        /*
+         * No need to untag -- TAG | TAG == TAG.
+         */
+        $Dst = $Src1 | $Src2;
+        $NEXT0();
+    }
+    $OUTLINED_ARITH_2($Fail, $Live, bor, BIF_bor_2, $Src1, $Src2, $Dst);
+}
+
+i_bxor(Fail, Live, Src1, Src2, Dst) {
+    if (is_both_small($Src1, $Src2)) {
+        /*
+         * TAG ^ TAG == 0.
+         *
+         * Therefore, we perform the XOR operation on the tagged values,
+         * and OR in the tag bits.
+         */
+        $Dst = ($Src1 ^ $Src2) | make_small(0);
+        $NEXT0();
+    }
+    $OUTLINED_ARITH_2($Fail, $Live, bxor, BIF_bxor_2, $Src1, $Src2, $Dst);
+}
+
+i_bsl := shift.setup_bsl.execute;
+i_bsr := shift.setup_bsr.execute;
+
+shift.head() {
+    Eterm Op1, Op2;
+    Sint shift_left_count;
+    Sint ires;
+    Eterm* bigp;
+    Eterm tmp_big[2];
+    Uint BIF;
+}
+
+shift.setup_bsr(Src1, Src2) {
+    Op1 = $Src1;
+    Op2 = $Src2;
+    BIF = BIF_bsr_2;
+    if (is_small(Op2)) {
+        shift_left_count = -signed_val(Op2);
+    } else if (is_big(Op2)) {
+        /*
+         * N bsr NegativeBigNum == N bsl MAX_SMALL
+         * N bsr PositiveBigNum == N bsl MIN_SMALL
+         */
+        shift_left_count = make_small(bignum_header_is_neg(*big_val(Op2)) ?
+                                      MAX_SMALL : MIN_SMALL);
+    } else {
+        shift_left_count = 0;
+    }
+}
+
+shift.setup_bsl(Src1, Src2) {
+    Op1 = $Src1;
+    Op2 = $Src2;
+    BIF = BIF_bsl_2;
+    if (is_small(Op2)) {
+        shift_left_count = signed_val(Op2);
+    } else if (is_big(Op2)) {
+        if (bignum_header_is_neg(*big_val(Op2))) {
+            /*
+             * N bsl NegativeBigNum is either 0 or -1, depending on
+             * the sign of N. Since we don't believe this case
+             * is common, do the calculation with the minimum
+             * amount of code.
+             */
+            shift_left_count = MIN_SMALL;
+        } else if (is_integer(Op1)) {
+            /*
+             * N bsl PositiveBigNum is too large to represent.
+             */
+            shift_left_count = MAX_SMALL;
+        }
+    } else {
+        shift_left_count = 0;
+    }
+}
+
+shift.execute(Fail, Live, Dst) {
+    if (is_small(Op1)) {
+        ires = signed_val(Op1);
+        if (shift_left_count == 0 || ires == 0) {
+            if (is_not_integer(Op2)) {
+                c_p->freason = BADARITH;
+                $BIF_ERROR_ARITY_2($Fail, BIF, Op1, Op2);
+            }
+            if (ires == 0) {
+                $Dst = Op1;
+                $NEXT0();
+            }
+        } else if (shift_left_count < 0)  { /* Right shift */
+            shift_left_count = -shift_left_count;
+            if (shift_left_count >= SMALL_BITS-1) {
+                $Dst = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO;
+            } else {
+                $Dst = make_small(ires >> shift_left_count);
+            }
+            $NEXT0();
+        } else if (shift_left_count < SMALL_BITS-1) { /* Left shift */
+            if ((ires > 0 &&
+                 ((~(Uint)0 << ((SMALL_BITS-1)-shift_left_count)) &
+                  ires) == 0) ||
+                ((~(Uint)0 << ((SMALL_BITS-1)-shift_left_count)) &
+                 ~ires) == 0) {
+                $Dst = make_small(ires << shift_left_count);
+                $NEXT0();
+            }
+        }
+        ires = 1;               /* big_size(small_to_big(Op1)) */
+        goto big_shift;
+    } else if (is_big(Op1)) {
+        if (shift_left_count == 0) {
+            if (is_not_integer(Op2)) {
+                c_p->freason = BADARITH;
+                $BIF_ERROR_ARITY_2($Fail, BIF, Op1, Op2);
+            }
+            $Dst = Op1;
+            $NEXT0();
+        }
+        ires = big_size(Op1);
+
+    big_shift:
+        if (shift_left_count > 0) {	/* Left shift. */
+            ires += (shift_left_count / D_EXP);
+        } else {	/* Right shift. */
+            if (ires <= (-shift_left_count / D_EXP)) {
+                ires = 3;       /* ??? */
+            } else {
+                ires -= (-shift_left_count / D_EXP);
+            }
+        }
+        {
+            ires = BIG_NEED_SIZE(ires+1);
+
+            /*
+             * Slightly conservative check the size to avoid
+             * allocating huge amounts of memory for bignums that
+             * clearly would overflow the arity in the header
+             * word.
+             */
+            if (ires-8 > BIG_ARITY_MAX) {
+                $SYSTEM_LIMIT($Fail);
+            }
+            $GC_TEST_PRESERVE(ires+1, $Live, Op1);
+            if (is_small(Op1)) {
+                Op1 = small_to_big(signed_val(Op1), tmp_big);
+            }
+            bigp = HTOP;
+            Op1 = big_lshift(Op1, shift_left_count, bigp);
+            if (is_big(Op1)) {
+                HTOP += bignum_header_arity(*HTOP) + 1;
+            }
+            HEAP_SPACE_VERIFIED(0);
+            if (is_nil(Op1)) {
+                /*
+                 * This result must have been only slighty larger
+                 * than allowed since it wasn't caught by the
+                 * previous test.
+                 */
+                $SYSTEM_LIMIT($Fail);
+            }
+            ERTS_HOLE_CHECK(c_p);
+            $REFRESH_GEN_DEST();
+            $Dst = Op1;
+            $NEXT0();
+        }
+    }
+
+    /*
+     * One or more non-integer arguments.
+     */
+    c_p->freason = BADARITH;
+    $BIF_ERROR_ARITY_2($Fail, BIF, Op1, Op2);
+}
+
+i_int_bnot(Fail, Src, Live, Dst) {
+    Eterm bnot_val = $Src;
+    if (is_small(bnot_val)) {
+        bnot_val = make_small(~signed_val(bnot_val));
+    } else {
+        Uint live = $Live;
+        HEAVY_SWAPOUT;
+        reg[live] = bnot_val;
+        bnot_val = erts_gc_bnot(c_p, reg, live);
+        HEAVY_SWAPIN;
+        ERTS_HOLE_CHECK(c_p);
+        if (is_nil(bnot_val)) {
+            $BIF_ERROR_ARITY_1($Fail, BIF_bnot_1, reg[live]);
+        }
+        $REFRESH_GEN_DEST();
+    }
+    $Dst = bnot_val;
+}
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index a5a462944b..ff3c725bbc 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -62,17 +62,17 @@
 #endif
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
-#    define PROCESS_MAIN_CHK_LOCKS(P)					\
-do {									\
-    if ((P))								\
-	erts_proc_lc_chk_only_proc_main((P));				\
-    ERTS_LC_ASSERT(!erts_thr_progress_is_blocking());		\
+#    define PROCESS_MAIN_CHK_LOCKS(P)                   \
+do {                                                    \
+    if ((P))                                            \
+	erts_proc_lc_chk_only_proc_main((P));           \
+    ERTS_LC_ASSERT(!erts_thr_progress_is_blocking());   \
 } while (0)
 #    define ERTS_REQ_PROC_MAIN_LOCK(P)				\
-do {									\
-    if ((P))								\
-	erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,		\
-				  __FILE__, __LINE__);			\
+do {                                                            \
+    if ((P))                                                    \
+	erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,     \
+				  __FILE__, __LINE__);          \
 } while (0)
 #    define ERTS_UNREQ_PROC_MAIN_LOCK(P)				\
 do {									\
@@ -156,63 +156,8 @@ do {                                     \
 #define REG_TARGET_PTR(Target) (((Target) & 1) ? &yb(Target-1) : &xb(Target))
 #define REG_TARGET(Target) (*REG_TARGET_PTR(Target))
 
-/*
- * Store a result into a register given a destination descriptor.
- */
-
-#define StoreResult(Result, DestDesc)		\
-  do {						\
-    Eterm stb_reg;				\
-    stb_reg = (DestDesc);			\
-    CHECK_TERM(Result);				\
-    REG_TARGET(stb_reg) = (Result);		\
-  } while (0)
-
-/*
- * Store a result into a register and execute the next instruction.
- * Dst points to the word with a destination descriptor, which MUST
- * be just before the next instruction.
- */
- 
-#define StoreBifResult(Dst, Result)		\
-  do {						\
-    BeamInstr* stb_next;			\
-    Eterm stb_reg;				\
-    stb_reg = Arg(Dst);				\
-    I += (Dst) + 2;				\
-    stb_next = (BeamInstr *) *I;		\
-    CHECK_TERM(Result);				\
-    REG_TARGET(stb_reg) = (Result);		\
-    Goto(stb_next);				\
-  } while (0)
-
-#define ClauseFail() goto jump_f
-
-#define SAVE_CP(X)				\
-   do {						\
-      *(X) = make_cp(c_p->cp);			\
-      c_p->cp = 0;				\
-   } while(0)
-
-#define RESTORE_CP(X)		SET_CP(c_p, (BeamInstr *) cp_val(*(X)))
-
 #define ISCATCHEND(instr) ((Eterm *) *(instr) == OpCode(catch_end_y))
 
-#define BIF_ERROR_ARITY_1(Op1, BIF)				\
-  if (Arg(0) != 0) goto jump_f;					\
-  reg[0] = Op1;							\
-  SWAPOUT;							\
-  I = handle_error(c_p, I, reg, &bif_export[BIF]->info.mfa);	\
-  goto post_error_handling
-
-#define BIF_ERROR_ARITY_2(Op1, Op2, BIF)			\
-  if (Arg(0) != 0) goto jump_f;					\
-  reg[0] = Op1;							\
-  reg[1] = Op2;							\
-  SWAPOUT;							\
-  I = handle_error(c_p, I, reg, &bif_export[BIF]->info.mfa);	\
-  goto post_error_handling
-
 /*
  * Special Beam instructions.
  */
@@ -306,63 +251,6 @@ void** beam_ops;
 #define y(N) E[N]
 #define r(N) x(N)
 
-#define TestBinVHeap(VNh, Nh, Live)                             		\
-  do {                                                          		\
-    unsigned need = (Nh);                                       		\
-    if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\
-       SWAPOUT;                                                 		\
-       PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
-       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS);	\
-       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);					\
-       PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
-       SWAPIN;                                                  		\
-    }                                                           		\
-    HEAP_SPACE_VERIFIED(need);                                                  \
-  } while (0)
-
-
-
-/*
- * Check if Nh words of heap are available; if not, do a garbage collection.
- * Live is number of active argument registers to be preserved.
- */
-
-#define TestHeap(Nh, Live)                                      \
-  do {                                                          \
-    unsigned need = (Nh);                                       \
-    if (E - HTOP < need) {                                      \
-       SWAPOUT;                                                 \
-       PROCESS_MAIN_CHK_LOCKS(c_p);                             \
-       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \
-       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);			\
-       PROCESS_MAIN_CHK_LOCKS(c_p);                             \
-       SWAPIN;                                                  \
-    }                                                           \
-    HEAP_SPACE_VERIFIED(need);                             \
-  } while (0)
-
-/*
- * Check if Nh words of heap are available; if not, do a garbage collection.
- * Live is number of active argument registers to be preserved.
- * Takes special care to preserve Extra if a garbage collection occurs.
- */
-
-#define TestHeapPreserve(Nh, Live, Extra)				\
-  do {									\
-    unsigned need = (Nh);						\
-    if (E - HTOP < need) {						\
-       SWAPOUT;								\
-       reg[Live] = Extra;						\
-       PROCESS_MAIN_CHK_LOCKS(c_p);					\
-       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1, FCALLS); \
-       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);				\
-       PROCESS_MAIN_CHK_LOCKS(c_p);					\
-       Extra = reg[Live];						\
-       SWAPIN;								\
-    }									\
-    HEAP_SPACE_VERIFIED(need);                                      \
-  } while (0)
-
 /*
  * Check that we haven't used the reductions and jump to function pointed to by
  * the I register.  If we are out of reductions, do a context switch.
@@ -435,12 +323,6 @@ void** beam_ops;
     ASSERT(VALID_INSTR(*I));   \
     Goto(*I)
 
-#define PreFetch(N, Dst) do { Dst = (BeamInstr *) *(I + N + 1); } while (0)
-#define NextPF(N, Dst)         \
-    I += N + 1;                \
-    ASSERT(VALID_INSTR(Dst));  \
-    Goto(Dst)
-
 #define GetR(pos, tr)				\
    do {						\
      tr = Arg(pos);				\
@@ -456,13 +338,20 @@ void** beam_ops;
      CHECK_TERM(tr);				\
    } while (0)
 
-#define GetArg1(N, Dst) GetR((N), Dst)
-
-#define GetArg2(N, Dst1, Dst2)     \
-   do {                            \
-     GetR(N, Dst1);                \
-     GetR((N)+1, Dst2);            \
-   } while (0)
+#define PUT_TERM_REG(term, desc)		\
+do {						\
+    switch (loader_tag(desc)) {			\
+    case LOADER_X_REG:				\
+	x(loader_x_reg_index(desc)) = (term);	\
+	break;					\
+    case LOADER_Y_REG:				\
+	y(loader_y_reg_index(desc)) = (term);	\
+	break;					\
+    default:					\
+	ASSERT(0);				\
+	break;					\
+    }						\
+} while(0)
 
 #define DispatchReturn                          \
 do {                                            \
@@ -485,51 +374,6 @@ do {                                            \
 # define BEAM_IS_TUPLE(Src) is_boxed(Src)
 #endif
 
-#if defined(ARCH_64)
-#define BsSafeMul(A, B, Fail, Target)		\
-   do { Uint64 _res = (A) * (B);		\
-      if (_res / B != A) { Fail; }		\
-      Target = _res;				\
-   } while (0)
-#else
-#define BsSafeMul(A, B, Fail, Target)			\
-   do { Uint64 _res = (Uint64)(A) * (Uint64)(B);	\
-      if ((_res >> (8*sizeof(Uint))) != 0) { Fail; }	\
-      Target = _res;					\
-   } while (0)
-#endif
-
-#define BsGetFieldSize(Bits, Unit, Fail, Target)	\
-   do {							\
-      Sint _signed_size; Uint _uint_size;		\
-      Uint temp_bits;					\
-      if (is_small(Bits)) {				\
-        _signed_size = signed_val(Bits);		\
-         if (_signed_size < 0) { Fail; }		\
-         _uint_size = (Uint) _signed_size;		\
-      } else {						\
-         if (!term_to_Uint(Bits, &temp_bits)) { Fail; }	\
-         _uint_size = temp_bits;			\
-      }							\
-      BsSafeMul(_uint_size, Unit, Fail, Target);	\
-   } while (0)
-
-#define BsGetUncheckedFieldSize(Bits, Unit, Fail, Target)	\
-   do {								\
-      Sint _signed_size; Uint _uint_size;			\
-      Uint temp_bits;						\
-      if (is_small(Bits)) {					\
-        _signed_size = signed_val(Bits);			\
-         if (_signed_size < 0) { Fail; }			\
-         _uint_size = (Uint) _signed_size;			\
-      } else {							\
-         if (!term_to_Uint(Bits, &temp_bits)) { Fail; }		\
-         _uint_size = (Uint) temp_bits;				\
-      }								\
-      Target = _uint_size * Unit;				\
-   } while (0)
-
-
 /*
  * process_main() is already huge, so we want to avoid inlining
  * into it. Especially functions that are seldom used.
@@ -718,6 +562,13 @@ init_emulator(void)
 #define ERTS_DBG_CHK_REDS(P, FC)
 #endif
 
+#ifdef NO_FPE_SIGNALS
+#  define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT
+#  define ERTS_NO_FPE_ERROR ERTS_FP_ERROR
+#else
+#  define ERTS_NO_FPE_CHECK_INIT(p)
+#  define ERTS_NO_FPE_ERROR(p, a, b)
+#endif
 
 /*
  * process_main() is called twice:
@@ -781,8 +632,6 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 #endif
 #endif
 
-    Eterm pt_arity;		/* Used by do_put_tuple */
-
     Uint64 start_time = 0;          /* Monitor long schedule */
     BeamInstr* start_time_i = NULL;
 
@@ -846,3643 +695,251 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 
     ERTS_MSACC_UPDATE_CACHE_X();
 
-    if (erts_system_monitor_long_schedule != 0) {
-	start_time = erts_timestamp_millis();
-	start_time_i = c_p->i;
-    }
-
-    ERL_BITS_RELOAD_STATEP(c_p);
-    {
-	int reds;
-	Eterm* argp;
-	BeamInstr *next;
-	int i;
-
-	argp = c_p->arg_reg;
-	for (i = c_p->arity - 1; i >= 0; i--) {
-	    reg[i] = argp[i];
-	    CHECK_TERM(reg[i]);
-	}
-
-	/*
-	 * We put the original reduction count in the process structure, to reduce
-	 * the code size (referencing a field in a struct through a pointer stored
-	 * in a register gives smaller code than referencing a global variable).
-	 */
-
-	SET_I(c_p->i);
-
-	REDS_IN(c_p) = reds = c_p->fcalls;
-#ifdef DEBUG
-	c_p->debug_reds_in = reds;
-#endif
-
-	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
-	    neg_o_reds = -CONTEXT_REDS;
-	    FCALLS = neg_o_reds + reds;
-	} else {
-	    neg_o_reds = 0;
-	    FCALLS = reds;
-	}
-
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-
-	next = (BeamInstr *) *I;
-	SWAPIN;
-	ASSERT(VALID_INSTR(next));
-
-#ifdef USE_VM_PROBES
-        if (DTRACE_ENABLED(process_scheduled)) {
-            DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE);
-            DTRACE_CHARBUF(fun_buf, DTRACE_TERM_BUF_SIZE);
-            dtrace_proc_str(c_p, process_buf);
-
-            if (ERTS_PROC_IS_EXITING(c_p)) {
-                strcpy(fun_buf, "<exiting>");
-            } else {
-                ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i);
-                if (cmfa) {
-                    dtrace_fun_decode(c_p, cmfa,
-                                      NULL, fun_buf);
-                } else {
-                    erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)),
-                                  "<unknown/%p>", next);
-                }
-            }
-
-            DTRACE2(process_scheduled, process_buf, fun_buf);
-        }
-#endif
-	Goto(next);
-    }
-
-#if defined(DEBUG) || defined(NO_JUMP_TABLE)
- emulator_loop:
-#endif
-
-#ifdef NO_JUMP_TABLE
-    switch (Go) {
-#endif
-#include "beam_hot.h"
-
-	{
-	    Eterm increment_reg_val;
-	    Eterm increment_val;
-	    Uint live;
-	    Eterm result;
-
-	OpCase(i_increment_rIId):
-	    increment_reg_val = x(0);
-	    I--;
-	    goto do_increment;
-
-	OpCase(i_increment_xIId):
-	    increment_reg_val = xb(Arg(0));
-	    goto do_increment;
-
-	OpCase(i_increment_yIId):
-	    increment_reg_val = yb(Arg(0));
-	    goto do_increment;
-
-	do_increment:
-	    increment_val = Arg(1);
-	    if (is_small(increment_reg_val)) {
-		Sint i = signed_val(increment_reg_val) + increment_val;
-		ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
-		if (MY_IS_SSMALL(i)) {
-		    result = make_small(i);
-		    StoreBifResult(3, result);
-		}
-	    }
-
-	    live = Arg(2);
-	    HEAVY_SWAPOUT;
-	    reg[live] = increment_reg_val;
-	    reg[live+1] = make_small(increment_val);
-	    result = erts_gc_mixed_plus(c_p, reg, live);
-	    HEAVY_SWAPIN;
-	    ERTS_HOLE_CHECK(c_p);
-	    if (is_value(result)) {
-		StoreBifResult(3, result);
-	    }
-	    ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue));
-	    goto find_func_info;
-	}
-
-#define DO_OUTLINED_ARITH_2(name, Op1, Op2, BIF)\
- do {						\
-     Eterm result;				\
-     Uint live = Arg(1);			\
-						\
-     HEAVY_SWAPOUT;				\
-     reg[live] = Op1;				\
-     reg[live+1] = Op2;				\
-     result = erts_gc_##name(c_p, reg, live);	\
-     HEAVY_SWAPIN;				\
-     ERTS_HOLE_CHECK(c_p);			\
-     if (is_value(result)) {			\
-	 StoreBifResult(4, result);		\
-     }						\
-     BIF_ERROR_ARITY_2(reg[live], reg[live+1], BIF);\
- } while (0)
-
- {
-     Eterm PlusOp1, PlusOp2;
-     Eterm result;
-
- OpCase(i_plus_jIxxd):
-     PlusOp1 = xb(Arg(2));
-     PlusOp2 = xb(Arg(3));
-     goto do_plus;
-
- OpCase(i_plus_jIxyd):
-     PlusOp1 = xb(Arg(2));
-     PlusOp2 = yb(Arg(3));
-     goto do_plus;
-
- OpCase(i_plus_jIssd):
-     GetArg2(2, PlusOp1, PlusOp2);
-     goto do_plus;
-
- do_plus:
-     if (is_both_small(PlusOp1, PlusOp2)) {
-	 Sint i = signed_val(PlusOp1) + signed_val(PlusOp2);
-	 ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
-	 if (MY_IS_SSMALL(i)) {
-	     result = make_small(i);
-             StoreBifResult(4, result);
-	 }
-     }
-     DO_OUTLINED_ARITH_2(mixed_plus, PlusOp1, PlusOp2, BIF_splus_2);
- }
-
- {
-     Eterm MinusOp1, MinusOp2;
-     Eterm result;
-
- OpCase(i_minus_jIxxd):
-     MinusOp1 = xb(Arg(2));
-     MinusOp2 = xb(Arg(3));
-     goto do_minus;
-
- OpCase(i_minus_jIssd):
-     GetArg2(2, MinusOp1, MinusOp2);
-     goto do_minus;
-
- do_minus:
-     if (is_both_small(MinusOp1, MinusOp2)) {
-	 Sint i = signed_val(MinusOp1) - signed_val(MinusOp2);
-	 ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
-	 if (MY_IS_SSMALL(i)) {
-	     result = make_small(i);
-             StoreBifResult(4, result);
-	 }
-     }
-     DO_OUTLINED_ARITH_2(mixed_minus, MinusOp1, MinusOp2, BIF_sminus_2);
- }
-
-    {
-	Eterm is_eq_exact_lit_val;
-
-    OpCase(i_is_eq_exact_literal_fxc):
-	is_eq_exact_lit_val = xb(Arg(1));
-	goto do_is_eq_exact_literal;
-
-    OpCase(i_is_eq_exact_literal_fyc):
-	is_eq_exact_lit_val = yb(Arg(1));
-	goto do_is_eq_exact_literal;
-
-    do_is_eq_exact_literal:
-	if (!eq(Arg(2), is_eq_exact_lit_val)) {
-	    ClauseFail();
-	}
-	Next(3);
-    }
-
-    {
-	Eterm is_ne_exact_lit_val;
-
-    OpCase(i_is_ne_exact_literal_fxc):
-	is_ne_exact_lit_val = xb(Arg(1));
-	goto do_is_ne_exact_literal;
-
-    OpCase(i_is_ne_exact_literal_fyc):
-	is_ne_exact_lit_val = yb(Arg(1));
-	goto do_is_ne_exact_literal;
-
-    do_is_ne_exact_literal:
-	if (eq(Arg(2), is_ne_exact_lit_val)) {
-	    ClauseFail();
-	}
-	Next(3);
-    }
-
- OpCase(i_move_call_ext_last_ePc): {
-     r(0) = Arg(2);
- }
- /* FALL THROUGH */
- OpCase(i_call_ext_last_eP):
-    RESTORE_CP(E);
-    E = ADD_BYTE_OFFSET(E, Arg(1));
-
-    /*
-     * Note: The pointer to the export entry is never NULL; if the module
-     * is not loaded, it points to code which will invoke the error handler
-     * (see lb_call_error_handler below).
-     */
-    DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0));
-    Dispatchx();
-
- OpCase(i_move_call_ext_ce): {
-     r(0) = Arg(0);
-     I++;
- }
- /* FALL THROUGH */
- OpCase(i_call_ext_e):
-    SET_CP(c_p, I+2);
-    DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0));
-    Dispatchx();
-
- OpCase(i_move_call_ext_only_ec): {
-     r(0) = Arg(1);
- }
- /* FALL THROUGH */
- OpCase(i_call_ext_only_e):
-    DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0));
-    Dispatchx();
-
- OpCase(return): {
-    SET_I(c_p->cp);
-    DTRACE_RETURN_FROM_PC(c_p);
-    /*
-     * We must clear the CP to make sure that a stale value do not
-     * create a false module dependcy preventing code upgrading.
-     * It also means that we can use the CP in stack backtraces.
-     */
-    c_p->cp = 0;
-    CHECK_TERM(r(0));
-    HEAP_SPACE_VERIFIED(0);
-    DispatchReturn;
- }
-
-    /*
-     * Send is almost a standard call-BIF with two arguments, except for:
-     *    1) It cannot be traced.
-     *	  2) There is no pointer to the send_2 function stored in
-     *       the instruction.
-     */
-
- OpCase(send): {
-     BeamInstr *next;
-     Eterm result;
-
-     if (!(FCALLS > 0 || FCALLS > neg_o_reds)) {
-         /* If we have run out of reductions, we do a context
-            switch before calling the bif */
-         c_p->arity = 2;
-         c_p->current = NULL;
-         goto context_switch3;
-     }
-
-     PRE_BIF_SWAPOUT(c_p);
-     c_p->fcalls = FCALLS - 1;
-     result = erl_send(c_p, r(0), x(1));
-     PreFetch(0, next);
-     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-     ERTS_REQ_PROC_MAIN_LOCK(c_p);
-     PROCESS_MAIN_CHK_LOCKS(c_p);
-     HTOP = HEAP_TOP(c_p);
-     FCALLS = c_p->fcalls;
-     if (is_value(result)) {
-	 r(0) = result;
-	 CHECK_TERM(r(0));
-	 NextPF(0, next);
-     } else if (c_p->freason == TRAP) {
-	 SET_CP(c_p, I+1);
-	 SET_I(c_p->i);
-	 SWAPIN;
-	 Dispatch();
-     }
-     goto find_func_info;
- }
-
-    {
-	Eterm element_index;
-	Eterm element_tuple;
-
-    OpCase(i_element_jxsd):
-	element_tuple = xb(Arg(1));
-	goto do_element;
-
-    OpCase(i_element_jysd):
-	element_tuple = yb(Arg(1));
-	goto do_element;
-
-    do_element:
-	GetArg1(2, element_index);
-	if (is_small(element_index) && is_tuple(element_tuple)) {
-	    Eterm* tp = tuple_val(element_tuple);
-
-	    if ((signed_val(element_index) >= 1) &&
-		(signed_val(element_index) <= arityval(*tp))) {
-		Eterm result = tp[signed_val(element_index)];
-		StoreBifResult(3, result);
-	    }
-	}
-	c_p->freason = BADARG;
-	BIF_ERROR_ARITY_2(element_index, element_tuple, BIF_element_2);
-    }
-
- OpCase(badarg_j):
- badarg:
-    c_p->freason = BADARG;
-    goto lb_Cl_error;
-
-    {
-	Eterm fast_element_tuple;
-
-    OpCase(i_fast_element_jxId):
-     fast_element_tuple = xb(Arg(1));
-     goto do_fast_element;
-
-    OpCase(i_fast_element_jyId):
-     fast_element_tuple = yb(Arg(1));
-     goto do_fast_element;
-
-    do_fast_element:
-	if (is_tuple(fast_element_tuple)) {
-	    Eterm* tp = tuple_val(fast_element_tuple);
-	    Eterm pos = Arg(2);	/* Untagged integer >= 1 */
-	    if (pos <= arityval(*tp)) {
-		Eterm result = tp[pos];
-		StoreBifResult(3, result);
-	    }
-	}
-	c_p->freason = BADARG;
-	BIF_ERROR_ARITY_2(make_small(Arg(2)), fast_element_tuple, BIF_element_2);
- }
-
- OpCase(catch_yf):
-     c_p->catches++;
-     yb(Arg(0)) = Arg(1);
-     Next(2);
-
- OpCase(catch_end_y): {
-     c_p->catches--;
-     make_blank(yb(Arg(0)));
-     if (is_non_value(r(0))) {
-         c_p->fvalue = NIL;
-	 if (x(1) == am_throw) {
-	     r(0) = x(2);
-	 } else {
-	     if (x(1) == am_error) {
-	         SWAPOUT;
-		 x(2) = add_stacktrace(c_p, x(2), x(3));
-		 SWAPIN;
-	     }
-	     /* only x(2) is included in the rootset here */
-	     if (E - HTOP < 3) {
-		 SWAPOUT;
-		 PROCESS_MAIN_CHK_LOCKS(c_p);
-		 FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1, FCALLS);
-		 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-		 PROCESS_MAIN_CHK_LOCKS(c_p);
-		 SWAPIN;
-	     }
-	     r(0) = TUPLE2(HTOP, am_EXIT, x(2));
-	     HTOP += 3;
-	 }
-     }
-     CHECK_TERM(r(0));
-     Next(1);
- }
-
- OpCase(try_end_y): {
-     c_p->catches--;
-     make_blank(yb(Arg(0)));
-     if (is_non_value(r(0))) {
-         c_p->fvalue = NIL;
-	 r(0) = x(1);
-	 x(1) = x(2);
-	 x(2) = x(3);
-     }
-     Next(1);
- }
-
- /*
-  * Skeleton for receive statement:
-  *
-  *             recv_mark L1                     Optional
-  *             call make_ref/monitor            Optional
-  *             ...
-  *             recv_set L1                      Optional
-  *      L1:          <-------------------+
-  *                   <-----------+       |
-  *     	     	       	  |   	  |
-  *             loop_rec L2 ------+---+   |
-  *             ...               |   |   |
-  *             remove_message 	  |   |	  |
-  *             jump L3           |   |   |
-  *		...	          |   |   |
-  *		loop_rec_end L1 --+   |   |
-  *      L2:          <---------------+   |
-  *	   	wait L1  -----------------+      or wait_timeout
-  *		timeout
-  *
-  *	 L3:    Code after receive...
-  *
-  *
-  */
-
- OpCase(recv_mark_f): {
-     /*
-      * Save the current position in message buffer and the
-      * the label for the loop_rec/2 instruction for the
-      * the receive statement.
-      */
-     c_p->msg.mark = (BeamInstr *) Arg(0);
-     c_p->msg.saved_last = c_p->msg.last;
-     Next(1);
- }
-
- OpCase(i_recv_set): {
-     /*
-      * If the mark is valid (points to the loop_rec/2
-      * instruction that follows), we know that the saved
-      * position points to the first message that could
-      * possibly be matched out.
-      *
-      * If the mark is invalid, we do nothing, meaning that
-      * we will look through all messages in the message queue.
-      */
-     if (c_p->msg.mark == (BeamInstr *) (I+1)) {
-	 c_p->msg.save = c_p->msg.saved_last;
-     }
-     I++;
-     /* Fall through to the loop_rec/2 instruction */
- }
-
-    /*
-     * Pick up the next message and place it in x(0).
-     * If no message, jump to a wait or wait_timeout instruction.
-     */
- OpCase(i_loop_rec_f):
- {
-     BeamInstr *next;
-     ErtsMessage* msgp;
-
-     /*
-      * We need to disable GC while matching messages
-      * in the queue. This since messages with data outside
-      * the heap will be corrupted by a GC.
-      */
-     ASSERT(!(c_p->flags & F_DELAY_GC));
-     c_p->flags |= F_DELAY_GC;
-
- loop_rec__:
-
-     PROCESS_MAIN_CHK_LOCKS(c_p);
-
-     msgp = PEEK_MESSAGE(c_p);
-
-     if (!msgp) {
-	 erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-	 /* Make sure messages wont pass exit signals... */
-	 if (ERTS_PROC_PENDING_EXIT(c_p)) {
-	     erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-	     SWAPOUT;
-	     c_p->flags &= ~F_DELAY_GC;
-             c_p->arity = 0;
-	     goto do_schedule; /* Will be rescheduled for exit */
-	 }
-	 ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
-	 msgp = PEEK_MESSAGE(c_p);
-	 if (msgp)
-	     erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-	 else
-	 {
-	     c_p->flags &= ~F_DELAY_GC;
-	     SET_I((BeamInstr *) Arg(0));
-	     Goto(*I);		/* Jump to a wait or wait_timeout instruction */
-	 }
-     }
-     if (is_non_value(ERL_MESSAGE_TERM(msgp))) {
-	 SWAPOUT; /* erts_decode_dist_message() may write to heap... */
-	 if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) {
-	     /*
-	      * A corrupt distribution message that we weren't able to decode;
-	      * remove it...
-	      */
-	     /* No swapin should be needed */
-	     ASSERT(HTOP == c_p->htop && E == c_p->stop);
-	     /* TODO: Add DTrace probe for this bad message situation? */
-	     UNLINK_MESSAGE(c_p, msgp);
-	     msgp->next = NULL;
-	     erts_cleanup_messages(msgp);
-	     goto loop_rec__;
-	 }
-	 SWAPIN;
-     }
-     PreFetch(1, next);
-     r(0) = ERL_MESSAGE_TERM(msgp);
-     NextPF(1, next);
- }
-
- /*
-  * Remove a (matched) message from the message queue.
-  */
- OpCase(remove_message): {
-     BeamInstr *next;
-     ErtsMessage* msgp;
-     PROCESS_MAIN_CHK_LOCKS(c_p);
-
-     ERTS_CHK_MBUF_SZ(c_p);
-
-     PreFetch(0, next);
-     msgp = PEEK_MESSAGE(c_p);
-
-     if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
-	 save_calls(c_p, &exp_receive);
-     }
-     if (ERL_MESSAGE_TOKEN(msgp) == NIL) {
-#ifdef USE_VM_PROBES
-	 if (DT_UTAG(c_p) != NIL) {
-	     if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) {
-		 SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag;
-	     } else {
-		 DT_UTAG(c_p) = NIL;
-		 SEQ_TRACE_TOKEN(c_p) = NIL;
-	     }
-	 } else {
-#endif
-	     SEQ_TRACE_TOKEN(c_p) = NIL;
-#ifdef USE_VM_PROBES
-	 }
-	 DT_UTAG_FLAGS(c_p) &= ~DT_UTAG_SPREADING;
-#endif
-     } else if (ERL_MESSAGE_TOKEN(msgp) != am_undefined) {
-	 Eterm msg;
-	 SEQ_TRACE_TOKEN(c_p) = ERL_MESSAGE_TOKEN(msgp);
-#ifdef USE_VM_PROBES
-	 if (ERL_MESSAGE_TOKEN(msgp) == am_have_dt_utag) {
-	     if (DT_UTAG(c_p) == NIL) {
-		 DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp);
-	     }
-	     DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING;
-	 } else {
-#endif
-	     ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p)));
-	     ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5);
-	     ASSERT(is_small(SEQ_TRACE_TOKEN_SERIAL(c_p)));
-	     ASSERT(is_small(SEQ_TRACE_TOKEN_LASTCNT(c_p)));
-	     ASSERT(is_small(SEQ_TRACE_TOKEN_FLAGS(c_p)));
-	     ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p)));
-	     c_p->seq_trace_lastcnt = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p));
-	     if (c_p->seq_trace_clock < unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p))) {
-		 c_p->seq_trace_clock = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p));
-	     }
-	     msg = ERL_MESSAGE_TERM(msgp);
-	     seq_trace_output(SEQ_TRACE_TOKEN(c_p), msg, SEQ_TRACE_RECEIVE, 
-			      c_p->common.id, c_p);
-#ifdef USE_VM_PROBES
-	 }
-#endif
-     }
-#ifdef USE_VM_PROBES
-     if (DTRACE_ENABLED(message_receive)) {
-         Eterm token2 = NIL;
-         DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
-         Sint tok_label = 0;
-         Sint tok_lastcnt = 0;
-         Sint tok_serial = 0;
-
-         dtrace_proc_str(c_p, receiver_name);
-         token2 = SEQ_TRACE_TOKEN(c_p);
-         if (have_seqtrace(token2)) {
-             tok_label = signed_val(SEQ_TRACE_T_LABEL(token2));
-             tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2));
-             tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2));
-         }
-         DTRACE6(message_receive,
-                 receiver_name, size_object(ERL_MESSAGE_TERM(msgp)),
-                 c_p->msg.len - 1, tok_label, tok_lastcnt, tok_serial);
-     }
-#endif
-     UNLINK_MESSAGE(c_p, msgp);
-     JOIN_MESSAGE(c_p);
-     CANCEL_TIMER(c_p);
-
-     erts_save_message_in_proc(c_p, msgp);
-     c_p->flags &= ~F_DELAY_GC;
-
-     if (ERTS_IS_GC_DESIRED_INTERNAL(c_p, HTOP, E)) {
-	 /*
-	  * We want to GC soon but we leave a few
-	  * reductions giving the message some time
-	  * to turn into garbage.
-	  */
-	 ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS);
-     }
-
-     ERTS_DBG_CHK_REDS(c_p, FCALLS);
-     ERTS_CHK_MBUF_SZ(c_p);
-
-     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-     PROCESS_MAIN_CHK_LOCKS(c_p);
-     NextPF(0, next);
- }
-
-    /*
-     * Advance the save pointer to the next message (the current
-     * message didn't match), then jump to the loop_rec instruction.
-     */
- OpCase(loop_rec_end_f): {
-
-     ASSERT(c_p->flags & F_DELAY_GC);
-
-     SET_I((BeamInstr *) Arg(0));
-     SAVE_MESSAGE(c_p);
-     if (FCALLS > 0 || FCALLS > neg_o_reds) {
-	 FCALLS--;
-	 goto loop_rec__;
-     }
-
-     c_p->flags &= ~F_DELAY_GC;
-     c_p->i = I;
-     SWAPOUT;
-     c_p->arity = 0;
-     c_p->current = NULL;
-     goto do_schedule;
- }
-    /*
-     * Prepare to wait for a message or a timeout, whichever occurs first.
-     *
-     * Note: In order to keep the compatibility between 32 and 64 bits
-     * emulators, only timeout values that can be represented in 32 bits
-     * (unsigned) or less are allowed.
-     */
-
-
- OpCase(i_wait_timeout_fs): {
-     erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-
-     /* Fall through */
- }
- OpCase(i_wait_timeout_locked_fs): {
-     Eterm timeout_value;
-
-     /*
-      * If we have already set the timer, we must NOT set it again.  Therefore,
-      * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag.
-      */
-     if (c_p->flags & (F_INSLPQUEUE | F_TIMO)) {
-	 goto wait2;
-     }
-     GetArg1(1, timeout_value);
-     if (timeout_value != make_small(0)) {
-
-	 if (timeout_value == am_infinity)
-	     c_p->flags |= F_TIMO;
-	 else {
-	     int tres = erts_set_proc_timer_term(c_p, timeout_value);
-	     if (tres == 0) {
-		 /*
-		  * The timer routiner will set c_p->i to the value in
-		  * c_p->def_arg_reg[0].  Note that it is safe to use this
-		  * location because there are no living x registers in
-		  * a receive statement.
-		  */
-		 BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
-		 *pi = I+3;
-	     }
-	     else { /* Wrong time */
-	     OpCase(i_wait_error_locked): {
-		     erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-		     /* Fall through */
-		 }
-	     OpCase(i_wait_error): {
-		     c_p->freason = EXC_TIMEOUT_VALUE;
-		     goto find_func_info;
-		 }
-	     }
-	 }
-
-	 /*
-	  * Prepare to wait indefinitely for a new message to arrive
-	  * (or the time set above if falling through from above).
-	  *
-	  * When a new message arrives, control will be transferred
-	  * the loop_rec instruction (at label L1).  In case of
-	  * of timeout, control will be transferred to the timeout
-	  * instruction following the wait_timeout instruction.
-	  */
-
-	 OpCase(wait_locked_f):
-	 OpCase(wait_f):
-
-	 wait2: {
-	     c_p->i = (BeamInstr *) Arg(0); /* L1 */
-	     SWAPOUT;
-	     c_p->arity = 0;
-
-	     if (!ERTS_PTMR_IS_TIMED_OUT(c_p))
-		 erts_atomic32_read_band_relb(&c_p->state,
-						  ~ERTS_PSFLG_ACTIVE);
-	     ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	     erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-	     c_p->current = NULL;
-	     goto do_schedule;
-	 }
-	 OpCase(wait_unlocked_f): {
-	     erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-	     goto wait2;
-	 }
-     }
-     erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-     Next(2);
- }
-
- OpCase(i_wait_timeout_fI): {
-     erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
- }
-
- OpCase(i_wait_timeout_locked_fI):
- {
-     /*
-      * If we have already set the timer, we must NOT set it again.  Therefore,
-      * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag.
-      */
-     if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) {
-	 BeamInstr** p = (BeamInstr **) c_p->def_arg_reg;
-	 *p = I+3;
-	 erts_set_proc_timer_uword(c_p, Arg(1));
-     }
-     goto wait2;
- }
-
-    /*
-     * A timeout has occurred.  Reset the save pointer so that the next
-     * receive statement will examine the first message first.
-     */
- OpCase(timeout_locked): {
-     erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
- }
-
- OpCase(timeout): {
-     BeamInstr *next;
-
-     PreFetch(0, next);
-     if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) {
-	 trace_receive(c_p, am_clock_service, am_timeout, NULL);
-     }
-     if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
-	 save_calls(c_p, &exp_timeout);
-     }
-     c_p->flags &= ~F_TIMO;
-     JOIN_MESSAGE(c_p);
-     NextPF(0, next);
- }
-
- {
-     Eterm select_val2;
-
- OpCase(i_select_tuple_arity2_yfAAff):
-     select_val2 = yb(Arg(0));
-     goto do_select_tuple_arity2;
-
- OpCase(i_select_tuple_arity2_xfAAff):
-     select_val2 = xb(Arg(0));
-     goto do_select_tuple_arity2;
-
- do_select_tuple_arity2:
-     if (is_not_tuple(select_val2)) {
-	 goto select_val2_fail;
-     }
-     select_val2 = *tuple_val(select_val2);
-     goto do_select_val2;
-
- OpCase(i_select_val2_yfccff):
-     select_val2 = yb(Arg(0));
-     goto do_select_val2;
-
- OpCase(i_select_val2_xfccff):
-     select_val2 = xb(Arg(0));
-     goto do_select_val2;
-
- do_select_val2:
-     if (select_val2 == Arg(2)) {
-	 I += 3;
-     } else if (select_val2 == Arg(3)) {
-	 I += 4;
-     }
-
- select_val2_fail:
-     SET_I((BeamInstr *) Arg(1));
-     Goto(*I);
- }
-
- {
-     Eterm select_val;
-
- OpCase(i_select_tuple_arity_xfI):
-     select_val = xb(Arg(0));
-     goto do_select_tuple_arity;
-
- OpCase(i_select_tuple_arity_yfI):
-     select_val = yb(Arg(0));
-     goto do_select_tuple_arity;
-
- do_select_tuple_arity:
-     if (is_tuple(select_val)) {
-	 select_val = *tuple_val(select_val);
-	 goto do_linear_search;
-     }
-     SET_I((BeamInstr *) Arg(1));
-     Goto(*I);
-
- OpCase(i_select_val_lins_xfI):
-     select_val = xb(Arg(0));
-     goto do_linear_search;
-
- OpCase(i_select_val_lins_yfI):
-     select_val = yb(Arg(0));
-     goto do_linear_search;
-
- do_linear_search: {
-     BeamInstr *vs = &Arg(3);
-     int ix = 0;
-
-     for(;;) {
-	 if (vs[ix+0] >= select_val) { ix += 0; break; }
-	 if (vs[ix+1] >= select_val) { ix += 1; break; }
-	 ix += 2;
-     }
-
-     if (vs[ix] == select_val) {
-	 I += ix + Arg(2) + 2;
-     }
-
-     SET_I((BeamInstr *) Arg(1));
-     Goto(*I);
-   }
-
- OpCase(i_select_val_bins_xfI):
-     select_val = xb(Arg(0));
-     goto do_binary_search;
-
- OpCase(i_select_val_bins_yfI):
-     select_val = yb(Arg(0));
-     goto do_binary_search;
-     
- do_binary_search:
- {
-     struct Pairs {
-	 BeamInstr val;
-	 BeamInstr* addr;
-     };
-     struct Pairs* low;
-     struct Pairs* high;
-     struct Pairs* mid;
-     int bdiff; /* int not long because the arrays aren't that large */
-
-     low = (struct Pairs *) &Arg(3);
-     high = low + Arg(2);
-
-     /* The pointer subtraction (high-low) below must produce
-      * a signed result, because high could be < low. That
-      * requires the compiler to insert quite a bit of code.
-      *
-      * However, high will be > low so the result will be
-      * positive. We can use that knowledge to optimise the
-      * entire sequence, from the initial comparison to the
-      * computation of mid.
-      *
-      * -- Mikael Pettersson, Acumem AB
-      *
-      * Original loop control code:
-      *
-      * while (low < high) {
-      *    mid = low + (high-low) / 2;
-      *
-      */
-     while ((bdiff = (int)((char*)high - (char*)low)) > 0) {
-	 unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Pairs)-1);
-
-	 mid = (struct Pairs*)((char*)low + boffset);
-	 if (select_val < mid->val) {
-	     high = mid;
-	 } else if (select_val > mid->val) {
-	     low = mid + 1;
-	 } else {
-	     SET_I(mid->addr);
-	     Goto(*I);
-	 }
-     }
-     SET_I((BeamInstr *) Arg(1));
-     Goto(*I);
- }
- }
-
- {
-     Eterm jump_on_val_zero_index;
-     
- OpCase(i_jump_on_val_zero_yfI):
-     jump_on_val_zero_index = yb(Arg(0));
-     goto do_jump_on_val_zero_index;
-
- OpCase(i_jump_on_val_zero_xfI):
-     jump_on_val_zero_index = xb(Arg(0));
-     goto do_jump_on_val_zero_index;
-
- do_jump_on_val_zero_index:
-     if (is_small(jump_on_val_zero_index)) {
-	 jump_on_val_zero_index = signed_val(jump_on_val_zero_index);
-	 if (jump_on_val_zero_index < Arg(2)) {
-	     SET_I((BeamInstr *) (&Arg(3))[jump_on_val_zero_index]);
-	     Goto(*I);
-	 }
-     }
-     SET_I((BeamInstr *) Arg(1));
-     Goto(*I);
- }
-
- {
-     Eterm jump_on_val_index;
-
- 
- OpCase(i_jump_on_val_yfII):
-     jump_on_val_index = yb(Arg(0));
-     goto do_jump_on_val_index;
-
- OpCase(i_jump_on_val_xfII):
-     jump_on_val_index = xb(Arg(0));
-     goto do_jump_on_val_index;
-
- do_jump_on_val_index:
-     if (is_small(jump_on_val_index)) {
-	 jump_on_val_index = (Uint) (signed_val(jump_on_val_index) - Arg(3));
-	 if (jump_on_val_index < Arg(2)) {
-	     SET_I((BeamInstr *) (&Arg(4))[jump_on_val_index]);
-	     Goto(*I);
-	 }
-     }
-     SET_I((BeamInstr *) Arg(1));
-     Goto(*I);
- }
-
- do_put_tuple: {
-     Eterm* hp = HTOP;
-
-     *hp++ = make_arityval(pt_arity);
-
-     do {
-	 Eterm term = *I++;
-	 switch (loader_tag(term)) {
-	 case LOADER_X_REG:
-	     *hp++ = x(loader_x_reg_index(term));
-	     break;
-	 case LOADER_Y_REG:
-	     *hp++ = y(loader_y_reg_index(term));
-	     break;
-	 default:
-	     *hp++ = term;
-	     break;
-	 }
-     } while (--pt_arity != 0);
-     HTOP = hp;
-     Goto(*I);
- }
-
- OpCase(new_map_dII): {
-     Eterm res;
-
-     HEAVY_SWAPOUT;
-     res = new_map(c_p, reg, I-1);
-     HEAVY_SWAPIN;
-     StoreResult(res, Arg(0));
-     Next(3+Arg(2));
- }
-
- OpCase(i_new_small_map_lit_dIq): {
-     Eterm res;
-     Uint n;
-
-     HEAVY_SWAPOUT;
-     res = new_small_map_lit(c_p, reg, &n, I-1);
-     HEAVY_SWAPIN;
-     StoreResult(res, Arg(0));
-     Next(3+n);
- }
-
-#define PUT_TERM_REG(term, desc)		\
-do {						\
-    switch (loader_tag(desc)) {			\
-    case LOADER_X_REG:				\
-	x(loader_x_reg_index(desc)) = (term);	\
-	break;					\
-    case LOADER_Y_REG:				\
-	y(loader_y_reg_index(desc)) = (term);	\
-	break;					\
-    default:					\
-	ASSERT(0);				\
-	break;					\
-    }						\
-} while(0)
-
- OpCase(i_get_map_elements_fsI): {
-    Eterm map;
-    BeamInstr *fs;
-    Uint sz, n;
-
-    GetArg1(1, map);
-
-    /* this instruction assumes Arg1 is a map,
-     * i.e. that it follows a test is_map if needed.
-     */
-
-    n  = (Uint)Arg(2) / 3;
-    fs = &Arg(3); /* pattern fields and target registers */
-
-    if (is_flatmap(map)) {
-	flatmap_t *mp;
-	Eterm *ks;
-	Eterm *vs;
-
-	mp = (flatmap_t *)flatmap_val(map);
-	sz = flatmap_get_size(mp);
-
-	if (sz == 0) {
-	    ClauseFail();
-	}
-
-	ks = flatmap_get_keys(mp);
-	vs = flatmap_get_values(mp);
-
-	while(sz) {
-	    if (EQ((Eterm) fs[0], *ks)) {
-		PUT_TERM_REG(*vs, fs[1]);
-		n--;
-		fs += 3;
-		/* no more values to fetch, we are done */
-		if (n == 0) {
-		    I = fs;
-		    Next(-1);
-		}
-	    }
-	    ks++, sz--, vs++;
-	}
-
-	ClauseFail();
-    } else {
-	const Eterm *v;
-	Uint32 hx;
-	ASSERT(is_hashmap(map));
-	while(n--) {
-	    hx = fs[2];
-	    ASSERT(hx == hashmap_make_hash((Eterm)fs[0]));
-	    if ((v = erts_hashmap_get(hx, (Eterm)fs[0], map)) == NULL) {
-		ClauseFail();
-	    }
-	    PUT_TERM_REG(*v, fs[1]);
-	    fs += 3;
-	}
-	I = fs;
-	Next(-1);
-    }
- }
-#undef PUT_TERM_REG
-
- OpCase(update_map_assoc_jsdII): {
-     Eterm res;
-     Eterm map;
-
-     GetArg1(1, map);
-     HEAVY_SWAPOUT;
-     res = update_map_assoc(c_p, reg, map, I);
-     HEAVY_SWAPIN;
-     if (is_value(res)) {
-	 StoreResult(res, Arg(2));
-	 Next(5+Arg(4));
-     } else {
-	 /*
-	  * This can only happen if the code was compiled
-	  * with the compiler in OTP 17.
-	  */
-	 c_p->freason = BADMAP;
-	 c_p->fvalue = map;
-	 goto lb_Cl_error;
-     }
- }
-
- OpCase(update_map_exact_jsdII): {
-     Eterm res;
-     Eterm map;
-
-     GetArg1(1, map);
-     HEAVY_SWAPOUT;
-     res = update_map_exact(c_p, reg, map, I);
-     HEAVY_SWAPIN;
-     if (is_value(res)) {
-	 StoreResult(res, Arg(2));
-	 Next(5+Arg(4));
-     } else {
-	 goto lb_Cl_error;
-     }
- }
-
-
-    /*
-     * All guards with zero arguments have special instructions:
-     * 	self/0
-     * 	node/0
-     *
-     * All other guard BIFs take one or two arguments.
-     */
-
-    /*
-     * Guard BIF in head.  On failure, ignore the error and jump
-     * to the code for the next clause.  We don't support tracing
-     * of guard BIFs.
-     */
-
- OpCase(bif1_fbsd):
-    {
-	ErtsBifFunc bf;
-	Eterm tmp_reg[1];
-	Eterm result;
-
-	GetArg1(2, tmp_reg[0]);
-	bf = (BifFunction) Arg(1);
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-	c_p->fcalls = FCALLS;
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	ERTS_CHK_MBUF_SZ(c_p);
-	result = (*bf)(c_p, tmp_reg, I);
-	ERTS_CHK_MBUF_SZ(c_p);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
-	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ERTS_HOLE_CHECK(c_p);
-	FCALLS = c_p->fcalls;
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-	if (is_value(result)) {
-	    StoreBifResult(3, result);
-	}
-	SET_I((BeamInstr *) Arg(0));
-	Goto(*I);
-    }
-
-    /*
-     * Guard BIF in body.  It can fail like any BIF.  No trace support.
-     */
-
- OpCase(bif1_body_bsd):
-    {
-	ErtsBifFunc bf;
-
-	Eterm tmp_reg[1];
-	Eterm result;
-
-	GetArg1(1, tmp_reg[0]);
-	bf = (ErtsBifFunc) Arg(0);
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-	c_p->fcalls = FCALLS;
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	ERTS_CHK_MBUF_SZ(c_p);
-	result = (*bf)(c_p, tmp_reg, I);
-	ERTS_CHK_MBUF_SZ(c_p);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
-	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ERTS_HOLE_CHECK(c_p);
-	FCALLS = c_p->fcalls;
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-	if (is_value(result)) {
-	    StoreBifResult(2, result);
-	}
-	reg[0] = tmp_reg[0];
-	SWAPOUT;
-	I = handle_error(c_p, I, reg, ubif2mfa((void *) bf));
-	goto post_error_handling;
-    }
-
- OpCase(i_gc_bif1_jIsId):
-    {
-	typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
-	GcBifFunction bf;
-	Eterm result;
-	Uint live = (Uint) Arg(3);
-
-	GetArg1(2, x(live));
-	bf = (GcBifFunction) Arg(1);
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-	c_p->fcalls = FCALLS;
-	SWAPOUT;
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
-	ERTS_CHK_MBUF_SZ(c_p);
-	result = (*bf)(c_p, reg, live);
-	ERTS_CHK_MBUF_SZ(c_p);
-	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	ERTS_REQ_PROC_MAIN_LOCK(c_p);
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	SWAPIN;
-	ERTS_HOLE_CHECK(c_p);
-	FCALLS = c_p->fcalls;
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-	if (is_value(result)) {
-	    StoreBifResult(4, result);
-	}
-	if (Arg(0) != 0) {
-	    SET_I((BeamInstr *) Arg(0));
-	    Goto(*I);
-	}
-	x(0) = x(live);
-	I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf));
-	goto post_error_handling;
-    }
-
- OpCase(i_gc_bif2_jIIssd): /* Note, one less parameter than the i_gc_bif1
-			    and i_gc_bif3 */
-    {
-	typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
-	GcBifFunction bf;
-	Eterm result;
-	Uint live = (Uint) Arg(2);
-
-	GetArg2(3, x(live), x(live+1));
-	/*
-	 * XXX This calling convention does not make sense. 'live'
-	 * should point out the first argument, not the second
-	 * (i.e. 'live' should not be incremented below).
-	 */
-	live++;
-	bf = (GcBifFunction) Arg(1);
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-	c_p->fcalls = FCALLS;
-	SWAPOUT;
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
-	ERTS_CHK_MBUF_SZ(c_p);
-	result = (*bf)(c_p, reg, live);
-	ERTS_CHK_MBUF_SZ(c_p);
-	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	ERTS_REQ_PROC_MAIN_LOCK(c_p);
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	SWAPIN;
-	ERTS_HOLE_CHECK(c_p);
-	FCALLS = c_p->fcalls;
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-	if (is_value(result)) {
-	    StoreBifResult(5, result);
-	}
-	if (Arg(0) != 0) {
-	    SET_I((BeamInstr *) Arg(0));
-	    Goto(*I);
-	}
-	live--;
-	x(0) = x(live);
-	x(1) = x(live+1);
-	I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf));
-	goto post_error_handling;
-    }
-
- OpCase(i_gc_bif3_jIIssd):
-    {
-	typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
-	GcBifFunction bf;
-	Eterm result;
-	Uint live = (Uint) Arg(2);
-
-	x(live) = x(SCRATCH_X_REG);
-	GetArg2(3, x(live+1), x(live+2));
-	/*
-	 * XXX This calling convention does not make sense. 'live'
-	 * should point out the first argument, not the third
-	 * (i.e. 'live' should not be incremented below).
-	 */
-	live += 2;
-	bf = (GcBifFunction) Arg(1);
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-	c_p->fcalls = FCALLS;
-	SWAPOUT;
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
-	ERTS_CHK_MBUF_SZ(c_p);
-	result = (*bf)(c_p, reg, live);
-	ERTS_CHK_MBUF_SZ(c_p);
-	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	ERTS_REQ_PROC_MAIN_LOCK(c_p);
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	SWAPIN;
-	ERTS_HOLE_CHECK(c_p);
-	FCALLS = c_p->fcalls;
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-	if (is_value(result)) {
-	    StoreBifResult(5, result);
-	}
-	if (Arg(0) != 0) {
-	    SET_I((BeamInstr *) Arg(0));
-	    Goto(*I);
-	}
-	live -= 2;
-	x(0) = x(live);
-	x(1) = x(live+1);
-	x(2) = x(live+2);
-	I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf));
-	goto post_error_handling;
-    }
-
- /*
-  * Guards bifs and, or, xor in guards.
-  */
- OpCase(i_bif2_fbssd):
-    {
-	Eterm tmp_reg[2];
-	ErtsBifFunc bf;
-	Eterm result;
-
-	GetArg2(2, tmp_reg[0], tmp_reg[1]);
-	bf = (ErtsBifFunc) Arg(1);
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-	c_p->fcalls = FCALLS;
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	ERTS_CHK_MBUF_SZ(c_p);
-	result = (*bf)(c_p, tmp_reg, I);
-	ERTS_CHK_MBUF_SZ(c_p);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
-	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ERTS_HOLE_CHECK(c_p);
-	FCALLS = c_p->fcalls;
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-	if (is_value(result)) {
-	    StoreBifResult(4, result);
-	}
-	SET_I((BeamInstr *) Arg(0));
-	Goto(*I);
-    }
-
- /*
-  * Guards bifs and, or, xor, relational operators in body.
-  */
- OpCase(i_bif2_body_bssd):
-    {
-	Eterm tmp_reg[2];
-	ErtsBifFunc bf;
-	Eterm result;
-
-	GetArg2(1, tmp_reg[0], tmp_reg[1]);
-	bf = (ErtsBifFunc) Arg(0);
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	ERTS_CHK_MBUF_SZ(c_p);
-	result = (*bf)(c_p, tmp_reg, I);
-	ERTS_CHK_MBUF_SZ(c_p);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
-	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	ERTS_HOLE_CHECK(c_p);
-	if (is_value(result)) {
-	    ASSERT(!is_CP(result));
-	    StoreBifResult(3, result);
-	}
-	reg[0] = tmp_reg[0];
-	reg[1] = tmp_reg[1];
-	SWAPOUT;
-	I = handle_error(c_p, I, reg, ubif2mfa((void *) bf));
-	goto post_error_handling;
-    }
-
-    /*
-     * The most general BIF call.  The BIF may build any amount of data
-     * on the heap.  The result is always returned in r(0).
-     */
- OpCase(call_bif_e):
-    {
-	ErtsBifFunc bf;
-	Eterm result;
-	BeamInstr *next;
-	ErlHeapFragment *live_hf_end;
-        Export *export = (Export*)Arg(0);
-
-
-        if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) {
-            /* If we have run out of reductions, we do a context
-               switch before calling the bif */
-            c_p->arity = GET_BIF_ARITY(export);
-            c_p->current = &export->info.mfa;
-            goto context_switch3;
-        }
-
-        ERTS_MSACC_SET_BIF_STATE_CACHED_X(
-            GET_BIF_MODULE(export), GET_BIF_ADDRESS(export));
-
-	bf = GET_BIF_ADDRESS(export);
-
-	PRE_BIF_SWAPOUT(c_p);
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-	c_p->fcalls = FCALLS - 1;
-	if (FCALLS <= 0) {
-	   save_calls(c_p, export);
-	}
-	PreFetch(1, next);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	live_hf_end = c_p->mbuf;
-	ERTS_CHK_MBUF_SZ(c_p);
-	result = (*bf)(c_p, reg, I);
-	ERTS_CHK_MBUF_SZ(c_p);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
-	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	ERTS_HOLE_CHECK(c_p);
-	ERTS_REQ_PROC_MAIN_LOCK(c_p);
-	if (ERTS_IS_GC_DESIRED(c_p)) {
-	    Uint arity = GET_BIF_ARITY(export);
-	    result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity);
-	    E = c_p->stop;
-	}
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	HTOP = HEAP_TOP(c_p);
-	FCALLS = c_p->fcalls;
-	ERTS_DBG_CHK_REDS(c_p, FCALLS);
-        /* We have to update the cache if we are enabled in order
-           to make sure no book keeping is done after we disabled
-           msacc. We don't always do this as it is quite expensive. */
-        if (ERTS_MSACC_IS_ENABLED_CACHED_X())
-            ERTS_MSACC_UPDATE_CACHE_X();
-	ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
-	if (is_value(result)) {
-	    r(0) = result;
-	    CHECK_TERM(r(0));
-	    NextPF(1, next);
-	} else if (c_p->freason == TRAP) {
-	    SET_CP(c_p, I+2);
-	    SET_I(c_p->i);
-	    SWAPIN;
-	    Dispatch();
-	}
-
-	/*
-	 * Error handling.  SWAPOUT is not needed because it was done above.
-	 */
-	ASSERT(c_p->stop == E);
-	I = handle_error(c_p, I, reg, &export->info.mfa);
-	goto post_error_handling;
-    }
-
- /*
-  * Arithmetic operations.
-  */
-
- OpCase(i_times_jIssd):
- {
-     Eterm Op1, Op2;
-     GetArg2(2, Op1, Op2);
-     DO_OUTLINED_ARITH_2(mixed_times, Op1, Op2, BIF_stimes_2);
- }
-
- OpCase(i_m_div_jIssd):
- {
-     Eterm Op1, Op2;
-     GetArg2(2, Op1, Op2);
-     DO_OUTLINED_ARITH_2(mixed_div, Op1, Op2, BIF_div_2);
- }
-
- OpCase(i_int_div_jIssd):
- {
-     Eterm Op1, Op2;
-
-     GetArg2(2, Op1, Op2);
-     if (Op2 == SMALL_ZERO) {
-	 c_p->freason = BADARITH;
-	 BIF_ERROR_ARITY_2(Op1, Op2, BIF_intdiv_2);
-     } else if (is_both_small(Op1, Op2)) {
-	 Sint ires = signed_val(Op1) / signed_val(Op2);
-	 if (MY_IS_SSMALL(ires)) {
-	     Eterm result = make_small(ires);
-	     StoreBifResult(4, result);
-	 }
-     }
-     DO_OUTLINED_ARITH_2(int_div, Op1, Op2, BIF_intdiv_2);
- }
-
- {
-     Eterm RemOp1, RemOp2;
-
- OpCase(i_rem_jIxxd):
-     RemOp1 = xb(Arg(2));
-     RemOp2 = xb(Arg(3));
-     goto do_rem;
-
- OpCase(i_rem_jIssd):
-     GetArg2(2, RemOp1, RemOp2);
-     goto do_rem;
-
- do_rem:
-     if (RemOp2 == SMALL_ZERO) {
-	 c_p->freason = BADARITH;
-	 BIF_ERROR_ARITY_2(RemOp1, RemOp2, BIF_rem_2);
-     } else if (is_both_small(RemOp1, RemOp2)) {
-         Eterm result = make_small(signed_val(RemOp1) % signed_val(RemOp2));
-         StoreBifResult(4, result);
-     } else {
-	 DO_OUTLINED_ARITH_2(int_rem, RemOp1, RemOp2, BIF_rem_2);
-     }
- }
-
- {
-     Eterm BandOp1, BandOp2;
-
- OpCase(i_band_jIxcd):
-     BandOp1 = xb(Arg(2));
-     BandOp2 = Arg(3);
-     goto do_band;
-
- OpCase(i_band_jIssd):
-     GetArg2(2, BandOp1, BandOp2);
-     goto do_band;
-
- do_band:
-     if (is_both_small(BandOp1, BandOp2)) {
-         /*
-          * No need to untag -- TAG & TAG == TAG.
-          */
-         Eterm result = BandOp1 & BandOp2;
-         StoreBifResult(4, result);
-     }
-     DO_OUTLINED_ARITH_2(band, BandOp1, BandOp2, BIF_band_2);
- }
-
- /*
-  * An error occurred in an arithmetic operation or test that could
-  * appear either in a head or in a body.
-  * In a head, execution should continue at failure address in Arg(0).
-  * In a body, Arg(0) == 0 and an exception should be raised.
-  */
- lb_Cl_error: {
-     if (Arg(0) != 0) {
-	 OpCase(jump_f): {
-	 jump_f:
-	     SET_I((BeamInstr *) Arg(0));
-	     Goto(*I);
-	 }
-     }
-     ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue));
-     goto find_func_info;
- }
-
- OpCase(i_bor_jIssd):
- {
-     Eterm Op1, Op2;
-
-     GetArg2(2, Op1, Op2);
-     if (is_both_small(Op1, Op2)) {
-	 /*
-	  * No need to untag -- TAG | TAG == TAG.
-	  */
-	 Eterm result = Op1 | Op2;
-	 StoreBifResult(4, result);
-     }
-     DO_OUTLINED_ARITH_2(bor, Op1, Op2, BIF_bor_2);
- }
-
- OpCase(i_bxor_jIssd):
- {
-     Eterm Op1, Op2;
-
-     GetArg2(2, Op1, Op2);
-     if (is_both_small(Op1, Op2)) {
-	 /*
-          * TAG ^ TAG == 0.
-          *
-          * Therefore, we perform the XOR operation on the tagged values,
-          * and OR in the tag bits.
-	  */
-	 Eterm result = (Op1 ^ Op2) | make_small(0);
-	 StoreBifResult(4, result);
-     }
-     DO_OUTLINED_ARITH_2(bxor, Op1, Op2, BIF_bxor_2);
- }
-
- {
-     Eterm Op1, Op2;
-     Sint i;
-     Sint ires;
-     Eterm* bigp;
-     Eterm tmp_big[2];
-
-     OpCase(i_bsr_jIssd):
-         GetArg2(2, Op1, Op2);
-	 if (is_small(Op2)) {
-	     i = -signed_val(Op2);
-	     if (is_small(Op1)) {
-		 goto small_shift;
-	     } else if (is_big(Op1)) {
-		 if (i == 0) {
-		     StoreBifResult(4, Op1);
-		 }
-		 ires = big_size(Op1);
-		 goto big_shift;
-	     }
-	 } else if (is_big(Op2)) {
-	     /*
-	      * N bsr NegativeBigNum == N bsl MAX_SMALL
-	      * N bsr PositiveBigNum == N bsl MIN_SMALL
-	      */
-	     Op2 = make_small(bignum_header_is_neg(*big_val(Op2)) ?
-				   MAX_SMALL : MIN_SMALL);
-	     goto do_bsl;
-	 }
-	 c_p->freason = BADARITH;
-	 BIF_ERROR_ARITY_2(Op1, Op2, BIF_bsr_2);
-     
-     OpCase(i_bsl_jIssd):
-         GetArg2(2, Op1, Op2);
- do_bsl:
-	 if (is_small(Op2)) {
-	     i = signed_val(Op2);
-
-	     if (is_small(Op1)) {
-	     small_shift:
-		 ires = signed_val(Op1);
-	     
-		 if (i == 0 || ires == 0) {
-		     StoreBifResult(4, Op1);
-		 } else if (i < 0)  { /* Right shift */
-		     i = -i;
-		     if (i >= SMALL_BITS-1) {
-			 Op1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO;
-		     } else {
-			 Op1 = make_small(ires >> i);
-		     }
-		     StoreBifResult(4, Op1);
-		 } else if (i < SMALL_BITS-1) { /* Left shift */
-		     if ((ires > 0 && ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ires) == 0) ||
-			 ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ~ires) == 0) {
-			 Op1 = make_small(ires << i);
-			 StoreBifResult(4, Op1);
-		     }
-		 }
-		 ires = 1; /* big_size(small_to_big(Op1)) */
-
-	     big_shift:
-		 if (i > 0) {	/* Left shift. */
-		     ires += (i / D_EXP);
-		 } else {	/* Right shift. */
-		     if (ires <= (-i / D_EXP))
-			 ires = 3; /* ??? */
-		     else
-			 ires -= (-i / D_EXP);
-		 }
-		 {
-		     ires = BIG_NEED_SIZE(ires+1);
-		     /*
-		      * Slightly conservative check the size to avoid
-		      * allocating huge amounts of memory for bignums that 
-		      * clearly would overflow the arity in the header
-		      * word.
-		      */
-		     if (ires-8 > BIG_ARITY_MAX) {
-			 c_p->freason = SYSTEM_LIMIT;
-			 goto lb_Cl_error;
-		     }
-		     TestHeapPreserve(ires+1, Arg(1), Op1);
-		     if (is_small(Op1)) {
-			 Op1 = small_to_big(signed_val(Op1), tmp_big);
-		     }
-		     bigp = HTOP;
-		     Op1 = big_lshift(Op1, i, bigp);
-		     if (is_big(Op1)) {
-			 HTOP += bignum_header_arity(*HTOP) + 1;
-		     }
-		     HEAP_SPACE_VERIFIED(0);
-		     if (is_nil(Op1)) {
-			 /*
-			  * This result must have been only slight larger
-			  * than allowed since it wasn't caught by the
-			  * previous test.
-			  */
-			 c_p->freason = SYSTEM_LIMIT;
-			 goto lb_Cl_error;
-		     }
-		     ERTS_HOLE_CHECK(c_p);
-		     StoreBifResult(4, Op1);
-		 }
-	     } else if (is_big(Op1)) {
-		 if (i == 0) {
-		     StoreBifResult(4, Op1);
-		 }
-		 ires = big_size(Op1);
-		 goto big_shift;
-	     }
-	 } else if (is_big(Op2)) {
-	     if (bignum_header_is_neg(*big_val(Op2))) {
-		 /*
-		  * N bsl NegativeBigNum is either 0 or -1, depending on
-		  * the sign of N. Since we don't believe this case
-		  * is common, do the calculation with the minimum
-		  * amount of code.
-		  */
-		 Op2 = make_small(MIN_SMALL);
-		 goto do_bsl;
-	     } else if (is_small(Op1) || is_big(Op1)) {
-		 /*
-		  * N bsl PositiveBigNum is too large to represent.
-		  */
-		 c_p->freason = SYSTEM_LIMIT;
-		 goto lb_Cl_error;
-	     }
-	     /* Fall through if the left argument is not an integer. */
-	 }
-	 c_p->freason = BADARITH;
-	 BIF_ERROR_ARITY_2(Op1, Op2, BIF_bsl_2);
- }
-
- OpCase(i_int_bnot_jsId):
- {
-     Eterm bnot_val;
-
-     GetArg1(1, bnot_val);
-     if (is_small(bnot_val)) {
-	 bnot_val = make_small(~signed_val(bnot_val));
-     } else {
-	 Uint live = Arg(2);
-	 HEAVY_SWAPOUT;
-	 reg[live] = bnot_val;
-	 bnot_val = erts_gc_bnot(c_p, reg, live);
-	 HEAVY_SWAPIN;
-	 ERTS_HOLE_CHECK(c_p);
-	 if (is_nil(bnot_val)) {
-	     BIF_ERROR_ARITY_1(reg[live], BIF_bnot_1);
-	 }
-     }
-     StoreBifResult(3, bnot_val);
- }
-
- OpCase(i_apply): {
-     BeamInstr *next;
-     HEAVY_SWAPOUT;
-     next = apply(c_p, r(0), x(1), x(2), reg, NULL, 0);
-     HEAVY_SWAPIN;
-     if (next != NULL) {
-	 SET_CP(c_p, I+1);
-	 SET_I(next);
-	 Dispatch();
-     }
-     I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa);
-     goto post_error_handling;
- }
-
- OpCase(i_apply_last_P): {
-     BeamInstr *next;
-     HEAVY_SWAPOUT;
-     next = apply(c_p, r(0), x(1), x(2), reg, I, Arg(0));
-     HEAVY_SWAPIN;
-     if (next != NULL) {
-	 SET_CP(c_p, (BeamInstr *) E[0]);
-	 E = ADD_BYTE_OFFSET(E, Arg(0));
-	 SET_I(next);
-	 Dispatch();
-     }
-     I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa);
-     goto post_error_handling;
- }
-
- OpCase(i_apply_only): {
-     BeamInstr *next;
-     HEAVY_SWAPOUT;
-     next = apply(c_p, r(0), x(1), x(2), reg, I, 0);
-     HEAVY_SWAPIN;
-     if (next != NULL) {
-	 SET_I(next);
-	 Dispatch();
-     }
-     I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa);
-     goto post_error_handling;
- }
-
- OpCase(apply_I): {
-     BeamInstr *next;
-
-     HEAVY_SWAPOUT;
-     next = fixed_apply(c_p, reg, Arg(0), NULL, 0);
-     HEAVY_SWAPIN;
-     if (next != NULL) {
-	 SET_CP(c_p, I+2);
-	 SET_I(next);
-	 Dispatch();
-     }
-     I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa);
-     goto post_error_handling;
- }
-
- OpCase(apply_last_IP): {
-     BeamInstr *next;
-
-     HEAVY_SWAPOUT;
-     next = fixed_apply(c_p, reg, Arg(0), I, Arg(1));
-     HEAVY_SWAPIN;
-     if (next != NULL) {
-	 SET_CP(c_p, (BeamInstr *) E[0]);
-	 E = ADD_BYTE_OFFSET(E, Arg(1));
-	 SET_I(next);
-	 Dispatch();
-     }
-     I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa);
-     goto post_error_handling;
- }
-
- OpCase(i_apply_fun): {
-     BeamInstr *next;
-
-     HEAVY_SWAPOUT;
-     next = apply_fun(c_p, r(0), x(1), reg);
-     HEAVY_SWAPIN;
-     if (next != NULL) {
-	 SET_CP(c_p, I+1);
-	 SET_I(next);
-	 Dispatchfun();
-     }
-     goto find_func_info;
- }
-
- OpCase(i_apply_fun_last_P): {
-     BeamInstr *next;
-
-     HEAVY_SWAPOUT;
-     next = apply_fun(c_p, r(0), x(1), reg);
-     HEAVY_SWAPIN;
-     if (next != NULL) {
-	 SET_CP(c_p, (BeamInstr *) E[0]);
-	 E = ADD_BYTE_OFFSET(E, Arg(0));
-	 SET_I(next);
-	 Dispatchfun();
-     }
-     goto find_func_info;
- }
-
- OpCase(i_apply_fun_only): {
-     BeamInstr *next;
-
-     HEAVY_SWAPOUT;
-     next = apply_fun(c_p, r(0), x(1), reg);
-     HEAVY_SWAPIN;
-     if (next != NULL) {
-	 SET_I(next);
-	 Dispatchfun();
-     }
-     goto find_func_info;
- }
-
- OpCase(i_call_fun_I): {
-     BeamInstr *next;
-
-     HEAVY_SWAPOUT;
-     next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
-     HEAVY_SWAPIN;
-     if (next != NULL) {
-	 SET_CP(c_p, I+2);
-	 SET_I(next);
-	 Dispatchfun();
-     }
-     goto find_func_info;
- }
-
- OpCase(i_call_fun_last_IP): {
-     BeamInstr *next;
-
-     HEAVY_SWAPOUT;
-     next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
-     HEAVY_SWAPIN;
-     if (next != NULL) {
-	SET_CP(c_p, (BeamInstr *) E[0]);
-	E = ADD_BYTE_OFFSET(E, Arg(1));
-	SET_I(next);
-	Dispatchfun();
-     }
-     goto find_func_info;
- }
-
-#ifdef DEBUG
-    /*
-     * Set a breakpoint here to get control just after a call instruction.
-     * I points to the first instruction in the called function.
-     *
-     * In gdb, use 'call dis(I-5, 1)' to show the name of the function.
-     */
- do_dispatch:
-     DispatchMacro();
-
- do_dispatchx:
-     DispatchMacrox();
-
- do_dispatchfun:
-     DispatchMacroFun();
-
-#endif
-
-    /*
-     * Jumped to from the Dispatch() macro when the reductions are used up.
-     *
-     * Since the I register points just beyond the FuncBegin instruction, we
-     * can get the module, function, and arity for the function being
-     * called from I[-3], I[-2], and I[-1] respectively.
-     */
- context_switch_fun:
-    /* Add one for the environment of the fun */
-    c_p->arity = erts_code_to_codemfa(I)->arity + 1;
-    goto context_switch2;
-
- context_switch:
-    c_p->arity = erts_code_to_codemfa(I)->arity;
-
- context_switch2: 		/* Entry for fun calls. */
-    c_p->current = erts_code_to_codemfa(I);
-
- context_switch3:
-
- {
-     Eterm* argp;
-     int i;
-
-     if (erts_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_EXITING) {
-         c_p->i = beam_exit;
-         c_p->arity = 0;
-         c_p->current = NULL;
-         goto do_schedule;
-     }
-
-     /*
-      * Make sure that there is enough room for the argument registers to be saved.
-      */
-     if (c_p->arity > c_p->max_arg_reg) {
-	 /*
-	  * Yes, this is an expensive operation, but you only pay it the first
-	  * time you call a function with more than 6 arguments which is
-	  * scheduled out.  This is better than paying for 26 words of wasted
-	  * space for most processes which never call functions with more than
-	  * 6 arguments.
-	  */
-	 Uint size = c_p->arity * sizeof(c_p->arg_reg[0]);
-	 if (c_p->arg_reg != c_p->def_arg_reg) {
-	     c_p->arg_reg = (Eterm *) erts_realloc(ERTS_ALC_T_ARG_REG,
-						   (void *) c_p->arg_reg,
-						   size);
-	 } else {
-	     c_p->arg_reg = (Eterm *) erts_alloc(ERTS_ALC_T_ARG_REG, size);
-	 }
-	 c_p->max_arg_reg = c_p->arity;
-     }
-
-     /*
-      * Since REDS_IN(c_p) is stored in the save area (c_p->arg_reg) we must read it
-      * now before saving registers.
-      *
-      * The '+ 1' compensates for the last increment which was not done
-      * (beacuse the code for the Dispatch() macro becomes shorter that way).
-      */
-
-     ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
-    if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
-	reds_used = REDS_IN(c_p) - FCALLS;
-    else
-	reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
-    ASSERT(reds_used >= 0);
-
-     /*
-      * Save the argument registers and everything else.
-      */
-
-     argp = c_p->arg_reg;
-     for (i = c_p->arity - 1; i >= 0; i--) {
-	 argp[i] = reg[i];
-     }
-     SWAPOUT;
-     c_p->i = I;
-     goto do_schedule1;
- }
-
- OpCase(normal_exit): {
-     SWAPOUT;
-     c_p->freason = EXC_NORMAL;
-     c_p->arity = 0;		/* In case this process will never be garbed again. */
-     ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
-     erts_do_exit_process(c_p, am_normal);
-     ERTS_REQ_PROC_MAIN_LOCK(c_p);
-     goto do_schedule;
- }
-
- OpCase(continue_exit): {
-     ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
-     erts_continue_exit_process(c_p);
-     ERTS_REQ_PROC_MAIN_LOCK(c_p);
-     goto do_schedule;
- }
-
- OpCase(i_raise): {
-     Eterm raise_trace = x(2);
-     Eterm raise_value = x(1);
-     struct StackTrace *s;
-
-     c_p->fvalue = raise_value;
-     c_p->ftrace = raise_trace;
-     s = get_trace_from_exc(raise_trace);
-     if (s == NULL) {
-	 c_p->freason = EXC_ERROR;
-     } else {
-	 c_p->freason = PRIMARY_EXCEPTION(s->freason);
-     }
-     goto find_func_info;
- }
-
-    {
-	Eterm badmatch_val;
-
-    OpCase(badmatch_x):
-	badmatch_val = xb(Arg(0));
-	c_p->fvalue = badmatch_val;
-	c_p->freason = BADMATCH;
-    }
- /* Fall through here */
-
- find_func_info: {
-     SWAPOUT;
-     I = handle_error(c_p, I, reg, NULL);
-     goto post_error_handling;
- }
-
- OpCase(call_error_handler):
-    /*
-     * At this point, I points to the code[3] in the export entry for
-     * a function which is not loaded.
-     *
-     * code[0]: Module
-     * code[1]: Function
-     * code[2]: Arity
-     * code[3]: &&call_error_handler
-     * code[4]: Not used
-     */
-    HEAVY_SWAPOUT;
-    I = call_error_handler(c_p, erts_code_to_codemfa(I),
-                           reg, am_undefined_function);
-    HEAVY_SWAPIN;
-    if (I) {
-	Goto(*I);
-    }
-
- /* Fall through */
- OpCase(error_action_code): {
-    handle_error:
-     SWAPOUT;
-     I = handle_error(c_p, NULL, reg, NULL);
- post_error_handling:
-     if (I == 0) {
-	 goto do_schedule;
-     } else {
-	 ASSERT(!is_value(r(0)));
-	 SWAPIN;
-	 Goto(*I);
-     }
- }
-
-    {
-	Eterm nif_bif_result;
-	Eterm bif_nif_arity;
-
-    OpCase(call_nif):
-	{
-	    /*
-	     * call_nif is always first instruction in function:
-	     *
-	     * I[-3]: Module
-	     * I[-2]: Function
-	     * I[-1]: Arity
-	     * I[0]: &&call_nif
-	     * I[1]: Function pointer to NIF function
-	     * I[2]: Pointer to erl_module_nif
-	     * I[3]: Function pointer to dirty NIF
-             *
-             * This layout is determined by the NifExport struct
-	     */
-	    BifFunction vbf;
-	    ErlHeapFragment *live_hf_end;
-            ErtsCodeMFA *codemfa;
-
-	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
-
-	    codemfa = erts_code_to_codemfa(I);
-
-            c_p->current = codemfa; /* current and vbf set to please handle_error */
-
-	    DTRACE_NIF_ENTRY(c_p, codemfa);
-
-	    HEAVY_SWAPOUT;
-
-	    PROCESS_MAIN_CHK_LOCKS(c_p);
-	    bif_nif_arity = codemfa->arity;
-	    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
-
-	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	    {
-		typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
-		NifF* fp = vbf = (NifF*) I[1];
-		struct enif_environment_t env;
-		ASSERT(c_p->scheduler_data);
-		live_hf_end = c_p->mbuf;
-		ERTS_CHK_MBUF_SZ(c_p);
-		erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
-		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
-		if (env.exception_thrown)
-		    nif_bif_result = THE_NON_VALUE;
-		erts_post_nif(&env);
-		ERTS_CHK_MBUF_SZ(c_p);
-
-		PROCESS_MAIN_CHK_LOCKS(c_p);
-		ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-		ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
-		ASSERT(!env.exiting);
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	    }
-
-	    DTRACE_NIF_RETURN(c_p, codemfa);
-	    goto apply_bif_or_nif_epilogue;
-	 
-	OpCase(apply_bif):
-	    /*
-	     * At this point, I points to the code[0] in the export entry for
-	     * the BIF:
-	     *
-	     * code[-3]: Module
-	     * code[-2]: Function
-	     * code[-1]: Arity
-	     * code[0]: &&apply_bif
-	     * code[1]: Function pointer to BIF function
-	     */
-
-            if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
-                /* If we have run out of reductions, we do a context
-                   switch before calling the bif */
-                goto context_switch;
-            }
-
-	    codemfa = erts_code_to_codemfa(I);
-
-            ERTS_MSACC_SET_BIF_STATE_CACHED_X(codemfa->module, (BifFunction)Arg(0));
-
-
-            /* In case we apply process_info/1,2 or load_nif/1 */
-	    c_p->current = codemfa;
-	    c_p->i = I;		/* In case we apply check_process_code/2. */
-	    c_p->arity = 0;		/* To allow garbage collection on ourselves
-					 * (check_process_code/2).
-					 */
-	    DTRACE_BIF_ENTRY(c_p, codemfa);
-
-	    SWAPOUT;
-	    ERTS_DBG_CHK_REDS(c_p, FCALLS - 1);
-	    c_p->fcalls = FCALLS - 1;
-	    vbf = (BifFunction) Arg(0);
-	    PROCESS_MAIN_CHK_LOCKS(c_p);
-	    bif_nif_arity = codemfa->arity;
-            ASSERT(bif_nif_arity <= 4);
-	    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
-	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	    {
-		ErtsBifFunc bf = vbf;
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-		live_hf_end = c_p->mbuf;
-		ERTS_CHK_MBUF_SZ(c_p);
-		nif_bif_result = (*bf)(c_p, reg, I);
-		ERTS_CHK_MBUF_SZ(c_p);
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
-		       is_non_value(nif_bif_result));
-		ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-		PROCESS_MAIN_CHK_LOCKS(c_p);
-	    }
-            /* We have to update the cache if we are enabled in order
-               to make sure no book keeping is done after we disabled
-               msacc. We don't always do this as it is quite expensive. */
-            if (ERTS_MSACC_IS_ENABLED_CACHED_X())
-                ERTS_MSACC_UPDATE_CACHE_X();
-	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
-	    DTRACE_BIF_RETURN(c_p, codemfa);
-
-	apply_bif_or_nif_epilogue:
-	    ERTS_REQ_PROC_MAIN_LOCK(c_p);
-	    ERTS_HOLE_CHECK(c_p);
-	    if (ERTS_IS_GC_DESIRED(c_p)) {
-		nif_bif_result = erts_gc_after_bif_call_lhf(c_p, live_hf_end,
-							    nif_bif_result,
-							    reg, bif_nif_arity);
-	    }
-	    SWAPIN;  /* There might have been a garbage collection. */
-	    FCALLS = c_p->fcalls;
-	    ERTS_DBG_CHK_REDS(c_p, FCALLS);
-	    if (is_value(nif_bif_result)) {
-		r(0) = nif_bif_result;
-		CHECK_TERM(r(0));
-		SET_I(c_p->cp);
-		c_p->cp = 0;
-		Goto(*I);
-	    } else if (c_p->freason == TRAP) {
-		SET_I(c_p->i);
-		if (c_p->flags & F_HIBERNATE_SCHED) {
-		    c_p->flags &= ~F_HIBERNATE_SCHED;
-		    goto do_schedule;
-		}
-		Dispatch();
-	    }
-	    I = handle_error(c_p, c_p->cp, reg, c_p->current);
-	    goto post_error_handling;
-	}
-    }
-
-    {
-	Eterm case_end_val;
-
-    OpCase(case_end_x):
-	case_end_val = xb(Arg(0));
-	c_p->fvalue = case_end_val;
-	c_p->freason = EXC_CASE_CLAUSE;
-	goto find_func_info;
-    }
-
- OpCase(if_end):
-    c_p->freason = EXC_IF_CLAUSE;
-    goto find_func_info;
-
- OpCase(i_func_info_IaaI): {
-     ErtsCodeInfo *ci = (ErtsCodeInfo*)I;
-     c_p->freason = EXC_FUNCTION_CLAUSE;
-     c_p->current = &ci->mfa;
-     goto handle_error;
- }
-
- OpCase(try_case_end_s):
-    {
-	Eterm try_case_end_val;
-	GetArg1(0, try_case_end_val);
-	c_p->fvalue = try_case_end_val;
-	c_p->freason = EXC_TRY_CLAUSE;
-	goto find_func_info;
-    }
-
- /*
-  * Construction of binaries using new instructions.
-  */
- {
-     Eterm new_binary;
-     Eterm num_bits_term;
-     Uint num_bits;
-     Uint alloc;
-     Uint num_bytes;
-
-     OpCase(i_bs_init_bits_heap_IIIx): {
-	 num_bits = Arg(0);
-	 alloc = Arg(1);
-	 I++;
-	 goto do_bs_init_bits_known;
-     }
-     
-     OpCase(i_bs_init_bits_IIx): {
-	 num_bits = Arg(0);
-	 alloc = 0;
-	 goto do_bs_init_bits_known;
-     }
-
-     OpCase(i_bs_init_bits_fail_heap_sIjIx): {
-	 GetArg1(0, num_bits_term);
-	 alloc = Arg(1);
-	 I += 2;
-	 goto do_bs_init_bits;
-     }
-
-     OpCase(i_bs_init_bits_fail_yjIx): {
-	 num_bits_term = yb(Arg(0));
-	 I++;
-	 alloc = 0;
-	 goto do_bs_init_bits;
-     }
-     OpCase(i_bs_init_bits_fail_xjIx): {
-	 num_bits_term = xb(Arg(0));
-	 I++;
-	 alloc = 0;
-     /* FALL THROUGH */
-     }
-
-     /* num_bits_term = Term for number of bits to build (small/big)
-      * alloc = Number of words to allocate on heap
-      * Operands: Fail Live Dst
-      */
-
- do_bs_init_bits:
-     if (is_small(num_bits_term)) {
-	 Sint size = signed_val(num_bits_term);
-	 if (size < 0) {
-	     goto badarg;
-	 }
-	 num_bits = (Uint) size;
-     } else {
-	 Uint bits;
-
-	 if (!term_to_Uint(num_bits_term, &bits)) {
-	     c_p->freason = bits;
-	     goto lb_Cl_error;
-
-	 }
-	 num_bits = (Eterm) bits;
-     }
-
-     /* num_bits = Number of bits to build
-      * alloc = Number of extra words to allocate on heap
-      * Operands: NotUsed Live Dst
-      */
- do_bs_init_bits_known:
-     num_bytes = ((Uint64)num_bits+(Uint64)7) >> 3;
-     if (num_bits & 7) {
-	 alloc += ERL_SUB_BIN_SIZE;
-     }
-     if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) {
-	 alloc += heap_bin_size(num_bytes);
-     } else {
-	 alloc += PROC_BIN_SIZE;
-     }
-     TestHeap(alloc, Arg(1));
-
-     /* num_bits = Number of bits to build
-      * num_bytes = Number of bytes to allocate in the binary
-      * alloc = Total number of words to allocate on heap
-      * Operands: NotUsed NotUsed Dst
-      */
-     if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) {
-	 ErlHeapBin* hb;
-
-	 erts_bin_offset = 0;
-	 erts_writable_bin = 0;
-	 hb = (ErlHeapBin *) HTOP;
-	 HTOP += heap_bin_size(num_bytes);
-	 hb->thing_word = header_heap_bin(num_bytes);
-	 hb->size = num_bytes;
-	 erts_current_bin = (byte *) hb->data;
-	 new_binary = make_binary(hb);
-
-     do_bits_sub_bin:
-	 if (num_bits & 7) {
-	     ErlSubBin* sb;
-
-	     sb = (ErlSubBin *) HTOP;
-	     HTOP += ERL_SUB_BIN_SIZE;
-	     sb->thing_word = HEADER_SUB_BIN;
-	     sb->size = num_bytes - 1;
-	     sb->bitsize = num_bits & 7;
-	     sb->offs = 0;
-	     sb->bitoffs = 0;
-	     sb->is_writable = 0;
-	     sb->orig = new_binary;
-	     new_binary = make_binary(sb);
-	 }
-	 HEAP_SPACE_VERIFIED(0);
-         xb(Arg(2)) = new_binary;
-         Next(3);
-     } else {
-	 Binary* bptr;
-	 ProcBin* pb;
-
-	 erts_bin_offset = 0;
-	 erts_writable_bin = 0;
-
-	 /*
-	  * Allocate the binary struct itself.
-	  */
-	 bptr = erts_bin_nrml_alloc(num_bytes);
-	 erts_current_bin = (byte *) bptr->orig_bytes;
-
-	 /*
-	  * Now allocate the ProcBin on the heap.
-	  */
-	 pb = (ProcBin *) HTOP;
-	 HTOP += PROC_BIN_SIZE;
-	 pb->thing_word = HEADER_PROC_BIN;
-	 pb->size = num_bytes;
-	 pb->next = MSO(c_p).first;
-	 MSO(c_p).first = (struct erl_off_heap_header*) pb;
-	 pb->val = bptr;
-	 pb->bytes = (byte*) bptr->orig_bytes;
-	 pb->flags = 0;
-	 OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm));
-	 new_binary = make_binary(pb);
-	 goto do_bits_sub_bin;
-     }
- }
-
- {
-     Eterm BsOp1, BsOp2;
-
-     OpCase(i_bs_init_fail_heap_sIjIx): {
-	 GetArg1(0, BsOp1);
-	 BsOp2 = Arg(1);
-	 I += 2;
-	 goto do_bs_init;
-     }
-
-     OpCase(i_bs_init_fail_yjIx): {
-	 BsOp1 = yb(Arg(0));
-	 BsOp2 = 0;
-	 I++;
-	 goto do_bs_init;
-     }
-
-     OpCase(i_bs_init_fail_xjIx): {
-	 BsOp1 = xb(Arg(0));
-	 BsOp2 = 0;
-	 I++;
-     }
-	 /* FALL THROUGH */
-     do_bs_init:
-         if (is_small(BsOp1)) {
-	     Sint size = signed_val(BsOp1);
-	     if (size < 0) {
-		 goto badarg;
-	     }
-	     BsOp1 = (Eterm) size;
-	 } else {
-	     Uint bytes;
-
-	     if (!term_to_Uint(BsOp1, &bytes)) {
-		 c_p->freason = bytes;
-		 goto lb_Cl_error;
-	     }
-	     if ((bytes >> (8*sizeof(Uint)-3)) != 0) {
-		 goto system_limit;
-	     }
-	     BsOp1 = (Eterm) bytes;
-	 }
-	 if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) {
-	     goto do_heap_bin_alloc;
-	 } else {
-	     goto do_proc_bin_alloc;
-	 }
-
-
-     OpCase(i_bs_init_heap_IIIx): {
-	 BsOp1 = Arg(0);
-	 BsOp2 = Arg(1);
-	 I++;
-	 goto do_proc_bin_alloc;
-     }
-
-     OpCase(i_bs_init_IIx): {
-	 BsOp1 = Arg(0);
-	 BsOp2 = 0;
-     }
-     /* FALL THROUGH */
-     do_proc_bin_alloc: {
-	 Binary* bptr;
-	 ProcBin* pb;
-
-	 erts_bin_offset = 0;
-	 erts_writable_bin = 0;
-	 TestBinVHeap(BsOp1 / sizeof(Eterm),
-		      BsOp2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1));
-
-	 /*
-	  * Allocate the binary struct itself.
-	  */
-	 bptr = erts_bin_nrml_alloc(BsOp1);
-	 erts_current_bin = (byte *) bptr->orig_bytes;
-
-	 /*
-	  * Now allocate the ProcBin on the heap.
-	  */
-	 pb = (ProcBin *) HTOP;
-	 HTOP += PROC_BIN_SIZE;
-	 pb->thing_word = HEADER_PROC_BIN;
-	 pb->size = BsOp1;
-	 pb->next = MSO(c_p).first;
-	 MSO(c_p).first = (struct erl_off_heap_header*) pb;
-	 pb->val = bptr;
-	 pb->bytes = (byte*) bptr->orig_bytes;
-	 pb->flags = 0;
-	 
-	 OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm));
-
-	 xb(Arg(2)) = make_binary(pb);
-         Next(3);
-     }
-
-     OpCase(i_bs_init_heap_bin_heap_IIIx): {
-	 BsOp1 = Arg(0);
-	 BsOp2 = Arg(1);
-	 I++;
-	 goto do_heap_bin_alloc;
-     }
-
-     OpCase(i_bs_init_heap_bin_IIx): {
-	 BsOp1 = Arg(0);
-	 BsOp2 = 0;
-     }
-     /* Fall through */
-     do_heap_bin_alloc:
-	 {
-	     ErlHeapBin* hb;
-	     Uint bin_need;
-
-	     bin_need = heap_bin_size(BsOp1);
-	     erts_bin_offset = 0;
-	     erts_writable_bin = 0;
-	     TestHeap(bin_need+BsOp2+ERL_SUB_BIN_SIZE, Arg(1));
-	     hb = (ErlHeapBin *) HTOP;
-	     HTOP += bin_need;
-	     hb->thing_word = header_heap_bin(BsOp1);
-	     hb->size = BsOp1;
-	     erts_current_bin = (byte *) hb->data;
-	     BsOp1 = make_binary(hb);
-             xb(Arg(2)) = BsOp1;
-             Next(3);
-	 }
- }
-
- OpCase(bs_add_jssIx): {
-     Eterm Op1, Op2;
-     Uint Unit = Arg(3);
-
-     GetArg2(1, Op1, Op2);
-     if (is_both_small(Op1, Op2)) {
-	 Sint Arg1 = signed_val(Op1);
-	 Sint Arg2 = signed_val(Op2);
-
-	 if (Arg1 >= 0 && Arg2 >= 0) {
-	     BsSafeMul(Arg2, Unit, goto system_limit, Op1);
-	     Op1 += Arg1;
-
-	 store_bs_add_result:
-	     if (Op1 <= MAX_SMALL) {
-		 Op1 = make_small(Op1);
-	     } else {
-		 /*
-		  * May generate a heap fragment, but in this
-		  * particular case it is OK, since the value will be
-		  * stored into an x register (the GC will scan x
-		  * registers for references to heap fragments) and
-		  * there is no risk that value can be stored into a
-		  * location that is not scanned for heap-fragment
-		  * references (such as the heap).
-		  */
-		 SWAPOUT;
-		 Op1 = erts_make_integer(Op1, c_p);
-		 HTOP = HEAP_TOP(c_p);
-	     }
-             xb(Arg(4)) = Op1;
-             Next(5);
-	 }
-	 goto badarg;
-     } else {
-	 Uint a;
-	 Uint b;
-	 Uint c;
-
-	 /*
-	  * Now we know that one of the arguments is
-	  * not a small. We must convert both arguments
-	  * to Uints and check for errors at the same time.
-	  *
-	  * Error checking is tricky.
-	  *
-	  * If one of the arguments is not numeric or
-	  * not positive, the error reason is BADARG.
-	  *
-	  * Otherwise if both arguments are numeric,
-	  * but at least one argument does not fit in
-	  * an Uint, the reason is SYSTEM_LIMIT.
-	  */
-
-	 if (!term_to_Uint(Op1, &a)) {
-	     if (a == BADARG) {
-		 goto badarg;
-	     }
-	     if (!term_to_Uint(Op2, &b)) {
-		 c_p->freason = b;
-		 goto lb_Cl_error;
-	     }
-	     goto system_limit;
-	 } else if (!term_to_Uint(Op2, &b)) {
-	     c_p->freason = b;
-	     goto lb_Cl_error;
-	 }
-
-	 /*
-	  * The arguments are now correct and stored in a and b.
-	  */
-	 
-	 BsSafeMul(b, Unit, goto system_limit, c);
-	 Op1 = a + c;
-	 if (Op1 < a) {
-	     /*
-	      * If the result is less than one of the
-	      * arguments, there must have been an overflow.
-	      */
-	     goto system_limit;
-	 }
-	 goto store_bs_add_result;
-     }
-     /* No fallthrough */
-     ASSERT(0);
- }
-
- OpCase(bs_put_string_II):
-    {
-	BeamInstr *next;
-	PreFetch(2, next);
-	erts_new_bs_put_string(ERL_BITS_ARGS_2((byte *) Arg(1), Arg(0)));
-	NextPF(2, next);
-    }
-
- /*
-  * x(SCRATCH_X_REG);
-  * Operands: Fail ExtraHeap Live Unit Size Dst
-  */
-
- OpCase(i_bs_append_jIIIsx): {
-     Uint live = Arg(2);
-     Uint res;
-     Eterm Size;
-
-     GetArg1(4, Size);
-     HEAVY_SWAPOUT;
-     reg[live] = x(SCRATCH_X_REG);
-     res = erts_bs_append(c_p, reg, live, Size, Arg(1), Arg(3));
-     HEAVY_SWAPIN;
-     if (is_non_value(res)) {
-	 /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */
-	 goto lb_Cl_error;
-     }
-     xb(Arg(5)) = res;
-     Next(6);
- }
-
- /*
-  * Operands: Fail Size Src Unit Dst
-  */
- OpCase(i_bs_private_append_jIssx): {
-     Eterm res;
-     Eterm Size, Src;
-
-     GetArg2(2, Size, Src);
-     res = erts_bs_private_append(c_p, Src, Size, Arg(1));
-     if (is_non_value(res)) {
-	 /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */
-	 goto lb_Cl_error;
-     }
-     xb(Arg(4)) = res;
-     Next(5);
- }
-
- OpCase(bs_init_writable): {
-     HEAVY_SWAPOUT;
-     r(0) = erts_bs_init_writable(c_p, r(0));
-     HEAVY_SWAPIN;
-     Next(0);
- }
-
- /*
-  * Calculate the number of bytes needed to encode the source
-  * operarand to UTF-8. If the source operand is invalid (e.g. wrong
-  * type or range) we return a nonsense integer result (0 or 4). We
-  * can get away with that because we KNOW that bs_put_utf8 will do
-  * full error checking.
-  */
- OpCase(i_bs_utf8_size_sx): {
-     Eterm arg;
-     Eterm result;
-
-     GetArg1(0, arg);
-     if (arg < make_small(0x80UL)) {
-	 result = make_small(1);
-     } else if (arg < make_small(0x800UL)) {
-	 result = make_small(2);
-     } else if (arg < make_small(0x10000UL)) {
-	 result = make_small(3);
-     } else {
-	 result = make_small(4);
-     }
-     xb(Arg(1)) = result;
-     Next(2);
- }
-
- OpCase(i_bs_put_utf8_js): {
-     Eterm arg;
-
-     GetArg1(1, arg);
-     if (!erts_bs_put_utf8(ERL_BITS_ARGS_1(arg))) {
-	 goto badarg;
-     }
-     Next(2);
- }
-
- /*
-  * Calculate the number of bytes needed to encode the source
-  * operarand to UTF-8. If the source operand is invalid (e.g. wrong
-  * type or range) we return a nonsense integer result (2 or 4). We
-  * can get away with that because we KNOW that bs_put_utf16 will do
-  * full error checking.
-  */
-
- OpCase(i_bs_utf16_size_sx): {
-     Eterm arg;
-     Eterm result = make_small(2);
-
-     GetArg1(0, arg);
-     if (arg >= make_small(0x10000UL)) {
-	 result = make_small(4);
-     }
-     xb(Arg(1)) = result;
-     Next(2);
- }
-
- OpCase(bs_put_utf16_jIs): {
-     Eterm arg;
-
-     GetArg1(2, arg);
-     if (!erts_bs_put_utf16(ERL_BITS_ARGS_2(arg, Arg(1)))) {
-	 goto badarg;
-     }
-     Next(3);
- }
-
- /*
-  * Only used for validating a value about to be stored in a binary.
-  */
- OpCase(i_bs_validate_unicode_js): {
-     Eterm val;
-
-     GetArg1(1, val);
-
-     /*
-      * There is no need to untag the integer, but it IS necessary
-      * to make sure it is small (if the term is a bignum, it could
-      * slip through the test, and there is no further test that
-      * would catch it, since bit syntax construction silently masks
-      * too big numbers).
-      */
-     if (is_not_small(val) || val > make_small(0x10FFFFUL) ||
-	 (make_small(0xD800UL) <= val && val <= make_small(0xDFFFUL))) {
-	 goto badarg;
-     }
-     Next(2);
- }
-
- /*
-  * Only used for validating a value matched out. 
-  */
- OpCase(i_bs_validate_unicode_retract_jss): {
-	Eterm i; 		/* Integer to validate */
-
-	/*
-	 * There is no need to untag the integer, but it IS necessary
-	 * to make sure it is small (a bignum pointer could fall in
-	 * the valid range).
-	 */
-
-	GetArg1(1, i);
-	if (is_not_small(i) || i > make_small(0x10FFFFUL) ||
-	    (make_small(0xD800UL) <= i && i <= make_small(0xDFFFUL))) {
-	    Eterm ms;		/* Match context */
-	    ErlBinMatchBuffer* mb;
-
-	    GetArg1(2, ms);
-	    mb = ms_matchbuffer(ms);
-	    mb->offset -= 32;
-	    goto badarg;
-	}
-	Next(3);
-    }
-
- /*
-  * Matching of binaries.
-  */
-
- {
-     Eterm header;
-     BeamInstr *next;
-     Uint slots;
-     Eterm context;
-
-     do_start_match:
-	 slots = Arg(2);
-	 if (!is_boxed(context)) {
-	     ClauseFail();
-	 }
-	 PreFetch(4, next);
-	 header = *boxed_val(context);
-	 if (header_is_bin_matchstate(header)) {
-	     ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(context);
-	     Uint actual_slots = HEADER_NUM_SLOTS(header);
-	     ms->save_offset[0] = ms->mb.offset;
-	     if (actual_slots < slots) {
-		 ErlBinMatchState* dst;
-		 Uint live = Arg(1);
-		 Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);
-
-		 TestHeapPreserve(wordsneeded, live, context);
-		 ms = (ErlBinMatchState *) boxed_val(context);
-		 dst = (ErlBinMatchState *) HTOP;
-		 *dst = *ms;
-		 *HTOP = HEADER_BIN_MATCHSTATE(slots);
-		 HTOP += wordsneeded;
-		 HEAP_SPACE_VERIFIED(0);
-                 xb(Arg(3)) = make_matchstate(dst);
-	     }
-	 } else if (is_binary_header(header)) {
-	     Eterm result;
-	     Uint live = Arg(1);
-	     Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);
-	     TestHeapPreserve(wordsneeded, live, context);
-	     HEAP_TOP(c_p) = HTOP;
-#ifdef DEBUG
-	     c_p->stop = E;	/* Needed for checking in HeapOnlyAlloc(). */
-#endif
-	     result = erts_bs_start_match_2(c_p, context, slots);
-	     HTOP = HEAP_TOP(c_p);
-	     HEAP_SPACE_VERIFIED(0);
-	     if (is_non_value(result)) {
-		 ClauseFail();
-	     } else {
-                 xb(Arg(3)) = result;
-	     }
-	 } else {
-	     ClauseFail();
-	 }
-	 NextPF(4, next);
-
-     OpCase(i_bs_start_match2_xfIIx): {
-	 context = xb(Arg(0));
-	 I++;
-	 goto do_start_match;
-     }
-     OpCase(i_bs_start_match2_yfIIx): {
-	 context = yb(Arg(0));
-	 I++;
-	 goto do_start_match;
-     }
- }
-
- OpCase(bs_test_zero_tail2_fx): {
-     BeamInstr *next;
-     ErlBinMatchBuffer *_mb;
-     
-     PreFetch(2, next);
-     _mb = (ErlBinMatchBuffer*) ms_matchbuffer(xb(Arg(1)));
-     if (_mb->size != _mb->offset) {
-	 ClauseFail();
-     }
-     NextPF(2, next);
- }
-
- OpCase(bs_test_tail_imm2_fxI): {
-     BeamInstr *next;
-     ErlBinMatchBuffer *_mb;
-     PreFetch(3, next);
-     _mb = ms_matchbuffer(xb(Arg(1)));
-     if (_mb->size - _mb->offset != Arg(2)) {
-	 ClauseFail();
-     }
-     NextPF(3, next);
- }
-
- OpCase(bs_test_unit_fxI): {
-     BeamInstr *next;
-     ErlBinMatchBuffer *_mb;
-     PreFetch(3, next);
-     _mb = ms_matchbuffer(xb(Arg(1)));
-     if ((_mb->size - _mb->offset) % Arg(2)) {
-	 ClauseFail();
-     }
-     NextPF(3, next);
- }
-
- OpCase(bs_test_unit8_fx): {
-     BeamInstr *next;
-     ErlBinMatchBuffer *_mb;
-     PreFetch(2, next);
-     _mb = ms_matchbuffer(xb(Arg(1)));
-     if ((_mb->size - _mb->offset) & 7) {
-	 ClauseFail();
-     }
-     NextPF(2, next);
- }
-
- {
-     Eterm bs_get_integer8_context;
-
- OpCase(i_bs_get_integer_8_xfx): {
-	 ErlBinMatchBuffer *_mb;
-	 Eterm _result;
-	 bs_get_integer8_context = xb(Arg(0));
-	 I++;
-	 _mb = ms_matchbuffer(bs_get_integer8_context);
-	 if (_mb->size - _mb->offset < 8) {
-	     ClauseFail();
-	 }
-	 if (BIT_OFFSET(_mb->offset) != 0) {
-	     _result = erts_bs_get_integer_2(c_p, 8, 0, _mb);
-	 } else {
-	     _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]);
-	     _mb->offset += 8;
-	 }
-         xb(Arg(1)) = _result;
-         Next(2);
-     }
- }
-
- {
-     Eterm bs_get_integer_16_context;
-
- OpCase(i_bs_get_integer_16_xfx):
-     bs_get_integer_16_context = xb(Arg(0));
-     I++;
-
-     {
-	 ErlBinMatchBuffer *_mb;
-	 Eterm _result;
-	 _mb = ms_matchbuffer(bs_get_integer_16_context);
-	 if (_mb->size - _mb->offset < 16) {
-	     ClauseFail(); 
-	 }
-	 if (BIT_OFFSET(_mb->offset) != 0) {
-	     _result = erts_bs_get_integer_2(c_p, 16, 0, _mb);
-	 } else {
-	     _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset)));
-	     _mb->offset += 16;
-	 }
-         xb(Arg(1)) = _result;
-         Next(2);
-     }
- }
-
- {
-     Eterm bs_get_integer_32_context;
-
- OpCase(i_bs_get_integer_32_xfIx):
-     bs_get_integer_32_context = xb(Arg(0));
-     I++;
-
-     {
-	 ErlBinMatchBuffer *_mb;
-	 Uint32 _integer;
-	 Eterm _result;
-	 _mb = ms_matchbuffer(bs_get_integer_32_context);
-	 if (_mb->size - _mb->offset < 32) { ClauseFail(); }
-	 if (BIT_OFFSET(_mb->offset) != 0) {
-	     _integer = erts_bs_get_unaligned_uint32(_mb);
-	 } else {
-	     _integer = get_int32(_mb->base + _mb->offset/8);
-	 }
-	 _mb->offset += 32;
-#if !defined(ARCH_64)
-	 if (IS_USMALL(0, _integer)) {
-#endif
-	     _result = make_small(_integer);
-#if !defined(ARCH_64)
-	 } else {
-	     TestHeap(BIG_UINT_HEAP_SIZE, Arg(1));
-	     _result = uint_to_big((Uint) _integer, HTOP);
-	     HTOP += BIG_UINT_HEAP_SIZE;
-	     HEAP_SPACE_VERIFIED(0);
-	 }
-#endif
-         xb(Arg(2)) = _result;
-         Next(3);
-     }
- }
-
- {
-     Eterm Ms, Sz;
-
-     /* Operands: x(Reg) Size Live Fail Flags Dst */
- OpCase(i_bs_get_integer_imm_xIIfIx): {
-	 Uint wordsneeded;
-	 Ms = xb(Arg(0));
-	 Sz = Arg(1);
-	 wordsneeded = 1+WSIZE(NBYTES(Sz));
-	 TestHeapPreserve(wordsneeded, Arg(2), Ms);
-	 I += 3;
-	 /* Operands: Fail Flags Dst */
-	 goto do_bs_get_integer_imm;
-     }
-
-     /* Operands: x(Reg) Size Fail Flags Dst */
- OpCase(i_bs_get_integer_small_imm_xIfIx): {
-	 Ms = xb(Arg(0));
-	 Sz = Arg(1);
-	 I += 2;
-	 /* Operands: Fail Flags Dst */
-	 goto do_bs_get_integer_imm;
-     }
- 
-     /*
-      * Ms = match context
-      * Sz = size of field
-      * Operands: Fail Flags Dst
-      */
- do_bs_get_integer_imm: {
-	 ErlBinMatchBuffer* mb;
-	 Eterm result;
-
-	 mb = ms_matchbuffer(Ms);
-	 LIGHT_SWAPOUT;
-	 result = erts_bs_get_integer_2(c_p, Sz, Arg(1), mb);
-	 LIGHT_SWAPIN;
-	 HEAP_SPACE_VERIFIED(0);
-	 if (is_non_value(result)) {
-	     ClauseFail();
-	 }
-         xb(Arg(2)) = result;
-         Next(3);
-     }
- }
-
- /*
-  * Operands: Fail Live FlagsAndUnit Ms Sz Dst
-  */
- OpCase(i_bs_get_integer_fIIssx): {
-     Uint flags;
-     Uint size;
-     Eterm Ms;
-     Eterm Sz;
-     ErlBinMatchBuffer* mb;
-     Eterm result;
-
-     flags = Arg(2);
-     GetArg2(3, Ms, Sz);
-     BsGetFieldSize(Sz, (flags >> 3), ClauseFail(), size);
-     if (size >= SMALL_BITS) {
-	 Uint wordsneeded;
-	 /* Check bits size before potential gc.
-	  * We do not want a gc and then realize we don't need
-	  * the allocated space (i.e. if the op fails).
-	  *
-	  * Remember to re-acquire the matchbuffer after gc.
-	  */
-
-	 mb = ms_matchbuffer(Ms);
-	 if (mb->size - mb->offset < size) {
-	     ClauseFail();
-	 }
-	 wordsneeded = 1+WSIZE(NBYTES((Uint) size));
-	 TestHeapPreserve(wordsneeded, Arg(1), Ms);
-     }
-     mb = ms_matchbuffer(Ms);
-     LIGHT_SWAPOUT;
-     result = erts_bs_get_integer_2(c_p, size, flags, mb);
-     LIGHT_SWAPIN;
-     HEAP_SPACE_VERIFIED(0);
-     if (is_non_value(result)) {
-	 ClauseFail();
-     }
-     xb(Arg(5)) = result;
-     Next(6);
- }
-
- {
-     Eterm get_utf8_context;
-
-     /* Operands: MatchContext Fail Dst */
- OpCase(i_bs_get_utf8_xfx): {
-	 get_utf8_context = xb(Arg(0));
-	 I++;
-     }
-
-     /*
-      * get_utf8_context = match_context
-      * Operands: Fail Dst
-      */
-
-     {
-	 Eterm result = erts_bs_get_utf8(ms_matchbuffer(get_utf8_context));
-	 if (is_non_value(result)) {
-	     ClauseFail();
-	 }
-         xb(Arg(1)) = result;
-         Next(2);
-     }
- }
-
- {
-     Eterm get_utf16_context;
-
-     /* Operands: MatchContext Fail Flags Dst */
-     OpCase(i_bs_get_utf16_xfIx): {
-	 get_utf16_context = xb(Arg(0));
-	 I++;
-     }
-
-     /*
-      * get_utf16_context = match_context
-      * Operands: Fail Flags Dst
-      */
-     {
-	 Eterm result = erts_bs_get_utf16(ms_matchbuffer(get_utf16_context),
-					  Arg(1));
-	 if (is_non_value(result)) {
-	     ClauseFail();
-	 }
-         xb(Arg(2)) = result;
-         Next(3);
-     }
- }
-
- {
-     Eterm context_to_binary_context;
-     ErlBinMatchBuffer* mb;
-     ErlSubBin* sb;
-     Uint size;
-     Uint offs;
-     Uint orig;
-     Uint hole_size;
-
-     OpCase(bs_context_to_binary_x):
-	 context_to_binary_context = xb(Arg(0));
-	 I--;
-
-     if (is_boxed(context_to_binary_context) &&
-	 header_is_bin_matchstate(*boxed_val(context_to_binary_context))) {
-	 ErlBinMatchState* ms;
-	 ms = (ErlBinMatchState *) boxed_val(context_to_binary_context);
-	 mb = &ms->mb;
-	 offs = ms->save_offset[0];
-	 size = mb->size - offs;
-	 goto do_bs_get_binary_all_reuse_common;
-     }
-     Next(2);
-
-     OpCase(i_bs_get_binary_all_reuse_xfI): {
-	 context_to_binary_context = xb(Arg(0));
-	 I++;
-     }
-
-     mb = ms_matchbuffer(context_to_binary_context);
-     size = mb->size - mb->offset;
-     if (size % Arg(1) != 0) {
-	 ClauseFail();
-     }
-     offs = mb->offset;
-
- do_bs_get_binary_all_reuse_common:
-     orig = mb->orig;
-     sb = (ErlSubBin *) boxed_val(context_to_binary_context);
-     hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE;
-     sb->thing_word = HEADER_SUB_BIN;
-     sb->size = BYTE_OFFSET(size);
-     sb->bitsize = BIT_OFFSET(size);
-     sb->offs = BYTE_OFFSET(offs);
-     sb->bitoffs = BIT_OFFSET(offs);
-     sb->is_writable = 0;
-     sb->orig = orig;
-     if (hole_size) {
-	 sb[1].thing_word = make_pos_bignum_header(hole_size-1);
-     }
-     Next(2);
- }
-
- {
-     Eterm match_string_context;
-
-     OpCase(i_bs_match_string_xfII): {
-	 match_string_context = xb(Arg(0));
-	 I++;
-     }
-
-     {
-	 BeamInstr *next;
-	 byte* bytes;
-	 Uint bits;
-	 ErlBinMatchBuffer* mb;
-	 Uint offs;
-
-	 PreFetch(3, next);
-	 bits = Arg(1);
-	 bytes = (byte *) Arg(2);
-	 mb = ms_matchbuffer(match_string_context);
-	 if (mb->size - mb->offset < bits) {
-	     ClauseFail();
-	 }
-	 offs = mb->offset & 7;
-	 if (offs == 0 && (bits & 7) == 0) {
-	     if (sys_memcmp(bytes, mb->base+(mb->offset>>3), bits>>3)) {
-		 ClauseFail();
-	     }
-	 } else if (erts_cmp_bits(bytes, 0, mb->base+(mb->offset>>3), mb->offset & 7, bits)) {
-	     ClauseFail();
-	 }
-	 mb->offset += bits;
-	 NextPF(3, next);
-     }
- }
-
- OpCase(i_bs_save2_xI): {
-     BeamInstr *next;
-     ErlBinMatchState *_ms;
-     PreFetch(2, next);
-     _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0)));
-     _ms->save_offset[Arg(1)] = _ms->mb.offset;
-     NextPF(2, next);
- }
-
- OpCase(i_bs_restore2_xI): {
-     BeamInstr *next;
-     ErlBinMatchState *_ms;
-     PreFetch(2, next);
-     _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0)));
-     _ms->mb.offset = _ms->save_offset[Arg(1)];
-     NextPF(2, next);
- }
-
-#include "beam_cold.h"
+    if (erts_system_monitor_long_schedule != 0) {
+	start_time = erts_timestamp_millis();
+	start_time_i = c_p->i;
+    }
 
-    /*
-     * Trace and debugging support.
-     */
+    ERL_BITS_RELOAD_STATEP(c_p);
+    {
+	int reds;
+	Eterm* argp;
+	BeamInstr *next;
+	int i;
 
- OpCase(return_trace): {
-     ErtsCodeMFA* mfa = (ErtsCodeMFA *)(E[0]);
-     
-     SWAPOUT;		/* Needed for shared heap */
-     ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
-     erts_trace_return(c_p, mfa, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */);
-     ERTS_REQ_PROC_MAIN_LOCK(c_p);
-     SWAPIN;
-     c_p->cp = NULL;
-     SET_I((BeamInstr *) cp_val(E[2]));
-     E += 3;
-     Goto(*I);
- }
+	argp = c_p->arg_reg;
+	for (i = c_p->arity - 1; i >= 0; i--) {
+	    reg[i] = argp[i];
+	    CHECK_TERM(reg[i]);
+	}
 
- OpCase(i_generic_breakpoint): {
-     BeamInstr real_I;
-     HEAVY_SWAPOUT;
-     real_I = erts_generic_breakpoint(c_p, erts_code_to_codeinfo(I), reg);
-     HEAVY_SWAPIN;
-     ASSERT(VALID_INSTR(real_I));
-     Goto(real_I);
- }
+	/*
+	 * We put the original reduction count in the process structure, to reduce
+	 * the code size (referencing a field in a struct through a pointer stored
+	 * in a register gives smaller code than referencing a global variable).
+	 */
 
- OpCase(i_return_time_trace): {
-     BeamInstr *pc = (BeamInstr *) (UWord) E[0];
-     SWAPOUT;
-     erts_trace_time_return(c_p, erts_code_to_codeinfo(pc));
-     SWAPIN;
-     c_p->cp = NULL;
-     SET_I((BeamInstr *) cp_val(E[1]));
-     E += 2;
-     Goto(*I);
- }
+	SET_I(c_p->i);
 
- OpCase(i_return_to_trace): {
-     if (IS_TRACED_FL(c_p, F_TRACE_RETURN_TO)) {
-	 Uint *cpp = (Uint*) E;
-	 for(;;) {
-	     ASSERT(is_CP(*cpp));
-	     if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) {
-		 do ++cpp; while(is_not_CP(*cpp));
-		 cpp += 2;
-	     } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) {
-		 do ++cpp; while(is_not_CP(*cpp));
-	     } else break;
-	 }
-	 SWAPOUT;		/* Needed for shared heap */
-	 ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
-	 erts_trace_return_to(c_p, cp_val(*cpp));
-	 ERTS_REQ_PROC_MAIN_LOCK(c_p);
-	 SWAPIN;
-     }
-     c_p->cp = NULL;
-     SET_I((BeamInstr *) cp_val(E[0]));
-     E += 1;
-     Goto(*I);
- }
+	REDS_IN(c_p) = reds = c_p->fcalls;
+#ifdef DEBUG
+	c_p->debug_reds_in = reds;
+#endif
 
- /*
-  * New floating point instructions.
-  */
+	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
+	    neg_o_reds = -CONTEXT_REDS;
+	    FCALLS = neg_o_reds + reds;
+	} else {
+	    neg_o_reds = 0;
+	    FCALLS = reds;
+	}
 
- OpCase(fmove_ql): {
-     Eterm fr = Arg(1);
-     BeamInstr *next;
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 
-     PreFetch(2, next);
-     GET_DOUBLE(Arg(0), *(FloatDef*)ADD_BYTE_OFFSET(freg, fr));
-     NextPF(2, next);
- }
+	next = (BeamInstr *) *I;
+	SWAPIN;
+	ASSERT(VALID_INSTR(next));
 
- OpCase(fmove_dl): {
-     Eterm targ1;
-     Eterm fr = Arg(1);
-     BeamInstr *next;
+#ifdef USE_VM_PROBES
+        if (DTRACE_ENABLED(process_scheduled)) {
+            DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE);
+            DTRACE_CHARBUF(fun_buf, DTRACE_TERM_BUF_SIZE);
+            dtrace_proc_str(c_p, process_buf);
 
-     PreFetch(2, next);
-     targ1 = REG_TARGET(Arg(0));
-     /* Arg(0) == HEADER_FLONUM */
-     GET_DOUBLE(targ1, *(FloatDef*)ADD_BYTE_OFFSET(freg, fr));
-     NextPF(2, next);
- }
+            if (ERTS_PROC_IS_EXITING(c_p)) {
+                strcpy(fun_buf, "<exiting>");
+            } else {
+                ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i);
+                if (cmfa) {
+                    dtrace_fun_decode(c_p, cmfa,
+                                      NULL, fun_buf);
+                } else {
+                    erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)),
+                                  "<unknown/%p>", next);
+                }
+            }
 
- OpCase(fmove_ld): {
-     Eterm fr = Arg(0);
-     Eterm dest = make_float(HTOP);
+            DTRACE2(process_scheduled, process_buf, fun_buf);
+        }
+#endif
+	Goto(next);
+    }
 
-     PUT_DOUBLE(*(FloatDef*)ADD_BYTE_OFFSET(freg, fr), HTOP);
-     HTOP += FLOAT_SIZE_OBJECT;
-     StoreBifResult(1, dest);
- }
+#if defined(DEBUG) || defined(NO_JUMP_TABLE)
+ emulator_loop:
+#endif
 
- OpCase(fconv_dl): {
-     Eterm targ1;
-     Eterm fr = Arg(1);
-     BeamInstr *next;
-
-     targ1 = REG_TARGET(Arg(0));
-     PreFetch(2, next);
-     if (is_small(targ1)) {
-	 lb(fr) = (double) signed_val(targ1);
-     } else if (is_big(targ1)) {
-	 if (big_to_double(targ1, &lb(fr)) < 0) {
-	     goto fbadarith;
-	 }
-     } else if (is_float(targ1)) {
-	 GET_DOUBLE(targ1, *(FloatDef*)ADD_BYTE_OFFSET(freg, fr));
-     } else {
-	 goto fbadarith;
-     }
-     NextPF(2, next);
- }
+#ifdef NO_JUMP_TABLE
+    switch (Go) {
+#endif
 
-#ifdef NO_FPE_SIGNALS
-     OpCase(fclearerror):
-     OpCase(i_fcheckerror):
-	 erts_exit(ERTS_ERROR_EXIT, "fclearerror/i_fcheckerror without fpe signals (beam_emu)");
-#  define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT
-#  define ERTS_NO_FPE_ERROR ERTS_FP_ERROR
-#else
-#  define ERTS_NO_FPE_CHECK_INIT(p)
-#  define ERTS_NO_FPE_ERROR(p, a, b)
+#include "beam_hot.h"
+#include "beam_instrs.h"
 
-     OpCase(fclearerror): {
-	 BeamInstr *next;
+#ifdef DEBUG
+    /*
+     * Set a breakpoint here to get control just after a call instruction.
+     * I points to the first instruction in the called function.
+     *
+     * In gdb, use 'call dis(I-5, 1)' to show the name of the function.
+     */
+ do_dispatch:
+     DispatchMacro();
 
-	 PreFetch(0, next);
-	 ERTS_FP_CHECK_INIT(c_p);
-	 NextPF(0, next);
-     }
+ do_dispatchx:
+     DispatchMacrox();
 
-     OpCase(i_fcheckerror): {
-	 BeamInstr *next;
+ do_dispatchfun:
+     DispatchMacroFun();
 
-	 PreFetch(0, next);
-	 ERTS_FP_ERROR(c_p, freg[0].fd, goto fbadarith);
-	 NextPF(0, next);
-     }
 #endif
 
+    /*
+     * Jumped to from the Dispatch() macro when the reductions are used up.
+     *
+     * Since the I register points just beyond the FuncBegin instruction, we
+     * can get the module, function, and arity for the function being
+     * called from I[-3], I[-2], and I[-1] respectively.
+     */
+ context_switch_fun:
+    /* Add one for the environment of the fun */
+    c_p->arity = erts_code_to_codemfa(I)->arity + 1;
+    goto context_switch2;
 
- OpCase(i_fadd_lll): {
-     BeamInstr *next;
+ context_switch:
+    c_p->arity = erts_code_to_codemfa(I)->arity;
 
-     PreFetch(3, next);
-     ERTS_NO_FPE_CHECK_INIT(c_p);
-     lb(Arg(2)) = lb(Arg(0)) + lb(Arg(1));
-     ERTS_NO_FPE_ERROR(c_p, lb(Arg(2)), goto fbadarith);
-     NextPF(3, next);
- }
- OpCase(i_fsub_lll): {
-     BeamInstr *next;
-
-     PreFetch(3, next);
-     ERTS_NO_FPE_CHECK_INIT(c_p);
-     lb(Arg(2)) = lb(Arg(0)) - lb(Arg(1));
-     ERTS_NO_FPE_ERROR(c_p, lb(Arg(2)), goto fbadarith);
-     NextPF(3, next);
- }
- OpCase(i_fmul_lll): {
-     BeamInstr *next;
-
-     PreFetch(3, next);
-     ERTS_NO_FPE_CHECK_INIT(c_p);
-     lb(Arg(2)) = lb(Arg(0)) * lb(Arg(1));
-     ERTS_NO_FPE_ERROR(c_p, lb(Arg(2)), goto fbadarith);
-     NextPF(3, next);
- }
- OpCase(i_fdiv_lll): {
-     BeamInstr *next;
-
-     PreFetch(3, next);
-     ERTS_NO_FPE_CHECK_INIT(c_p);
-     lb(Arg(2)) = lb(Arg(0)) / lb(Arg(1));
-     ERTS_NO_FPE_ERROR(c_p, lb(Arg(2)), goto fbadarith);
-     NextPF(3, next);
- }
- OpCase(i_fnegate_ll): {
-     BeamInstr *next;
-
-     PreFetch(2, next);
-     ERTS_NO_FPE_CHECK_INIT(c_p);
-     lb(Arg(1)) = -lb(Arg(0));
-     ERTS_NO_FPE_ERROR(c_p, lb(Arg(1)), goto fbadarith);
-     NextPF(2, next);
-
- fbadarith:
-     c_p->freason = BADARITH;
-     goto find_func_info;
- }
+ context_switch2: 		/* Entry for fun calls. */
+    c_p->current = erts_code_to_codemfa(I);
+
+ context_switch3:
 
-#ifdef HIPE
  {
-#define HIPE_MODE_SWITCH(Cmd)			\
-     SWAPOUT;					\
-     ERTS_DBG_CHK_REDS(c_p, FCALLS);		\
-     c_p->fcalls = FCALLS;			\
-     c_p->def_arg_reg[4] = -neg_o_reds;		\
-     c_p = hipe_mode_switch(c_p, Cmd, reg);     \
-     goto L_post_hipe_mode_switch
-
-     OpCase(hipe_trap_call): {
-	 /*
-	  * I[-5]: &&lb_i_func_info_IaaI
-	  * I[-4]: Native code callee (inserted by HiPE)
-	  * I[-3]: Module (tagged atom)
-	  * I[-2]: Function (tagged atom)
-	  * I[-1]: Arity (untagged integer)
-	  * I[ 0]: &&lb_hipe_trap_call
-	  * ... remainder of original BEAM code
-	  */
-         ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
-	 ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
-	 c_p->hipe.u.ncallee = ci->u.ncallee;
-	 ++hipe_trap_count;
-	 HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8));
-     }
-     OpCase(hipe_trap_call_closure): {
-       ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
-       ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
-       c_p->hipe.u.ncallee = ci->u.ncallee;
-       ++hipe_trap_count;
-       HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8));
-     }
-     OpCase(hipe_trap_return): {
-	 HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RETURN);
-     }
-     OpCase(hipe_trap_throw): {
-	 HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_THROW);
-     }
-     OpCase(hipe_trap_resume): {
-	 HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RESUME);
+     Eterm* argp;
+     int i;
+
+     if (erts_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_EXITING) {
+         c_p->i = beam_exit;
+         c_p->arity = 0;
+         c_p->current = NULL;
+         goto do_schedule;
      }
-#undef HIPE_MODE_SWITCH
 
- L_post_hipe_mode_switch:
-#ifdef DEBUG
-     pid = c_p->common.id; /* may have switched process... */
-#endif
-     reg = erts_proc_sched_data(c_p)->x_reg_array;
-     freg = erts_proc_sched_data(c_p)->f_reg_array;
-     ERL_BITS_RELOAD_STATEP(c_p);
-     /* XXX: this abuse of def_arg_reg[] is horrid! */
-     neg_o_reds = -c_p->def_arg_reg[4];
-     FCALLS = c_p->fcalls;
-     SWAPIN;
-     ERTS_DBG_CHK_REDS(c_p, FCALLS);
-     switch( c_p->def_arg_reg[3] ) {
-       case HIPE_MODE_SWITCH_RES_RETURN:
-	 ASSERT(is_value(reg[0]));
-	 SET_I(c_p->cp);
-	 c_p->cp = 0;
-	 Goto(*I);
-       case HIPE_MODE_SWITCH_RES_CALL_EXPORTED:
-	 c_p->i = c_p->hipe.u.callee_exp->addressv[erts_active_code_ix()];
-	 /*fall through*/
-       case HIPE_MODE_SWITCH_RES_CALL_BEAM:
-	 SET_I(c_p->i);
-	 Dispatch();
-       case HIPE_MODE_SWITCH_RES_CALL_CLOSURE:
-	 /* This can be used to call any function value, but currently it's
-	    only used to call closures referring to unloaded modules. */
-	 {
-	     BeamInstr *next;
-
-	     next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE);
-	     HEAVY_SWAPIN;
-	     if (next != NULL) {
-		 SET_I(next);
-		 Dispatchfun();
-	     }
-	     goto find_func_info;
+     /*
+      * Make sure that there is enough room for the argument registers to be saved.
+      */
+     if (c_p->arity > c_p->max_arg_reg) {
+	 /*
+	  * Yes, this is an expensive operation, but you only pay it the first
+	  * time you call a function with more than 6 arguments which is
+	  * scheduled out.  This is better than paying for 26 words of wasted
+	  * space for most processes which never call functions with more than
+	  * 6 arguments.
+	  */
+	 Uint size = c_p->arity * sizeof(c_p->arg_reg[0]);
+	 if (c_p->arg_reg != c_p->def_arg_reg) {
+	     c_p->arg_reg = (Eterm *) erts_realloc(ERTS_ALC_T_ARG_REG,
+						   (void *) c_p->arg_reg,
+						   size);
+	 } else {
+	     c_p->arg_reg = (Eterm *) erts_alloc(ERTS_ALC_T_ARG_REG, size);
 	 }
-       case HIPE_MODE_SWITCH_RES_THROW:
-	 c_p->cp = NULL;
-	 I = handle_error(c_p, I, reg, NULL);
-	 goto post_error_handling;
-       default:
-	 erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]);
+	 c_p->max_arg_reg = c_p->arity;
      }
- }
- OpCase(hipe_call_count): {
+
      /*
-      * I[-5]: &&lb_i_func_info_IaaI
-      * I[-4]: pointer to struct hipe_call_count (inserted by HiPE)
-      * I[-3]: Module (tagged atom)
-      * I[-2]: Function (tagged atom)
-      * I[-1]: Arity (untagged integer)
-      * I[ 0]: &&lb_hipe_call_count
-      * ... remainder of original BEAM code
+      * Since REDS_IN(c_p) is stored in the save area (c_p->arg_reg) we must read it
+      * now before saving registers.
+      *
+      * The '+ 1' compensates for the last increment which was not done
+      * (beacuse the code for the Dispatch() macro becomes shorter that way).
       */
-     ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
-     struct hipe_call_count *hcc = ci->u.hcc;
-     ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
-     ASSERT(hcc != NULL);
-     ASSERT(VALID_INSTR(hcc->opcode));
-     ++(hcc->count);
-     Goto(hcc->opcode);
- }
-#endif /* HIPE */
 
- OpCase(i_yield):
- {
-     /* This is safe as long as REDS_IN(c_p) is never stored 
-      * in c_p->arg_reg[0]. It is currently stored in c_p->def_arg_reg[5],
-      * which may be c_p->arg_reg[5], which is close, but no banana.
+     ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
+    if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+	reds_used = REDS_IN(c_p) - FCALLS;
+    else
+	reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
+    ASSERT(reds_used >= 0);
+
+     /*
+      * Save the argument registers and everything else.
       */
-     c_p->arg_reg[0] = am_true;
-     c_p->arity = 1; /* One living register (the 'true' return value) */
+
+     argp = c_p->arg_reg;
+     for (i = c_p->arity - 1; i >= 0; i--) {
+	 argp[i] = reg[i];
+     }
      SWAPOUT;
-     c_p->i = I + 1; /* Next instruction */
-     c_p->current = NULL;
-     goto do_schedule;
+     c_p->i = I;
+     goto do_schedule1;
  }
 
- OpCase(i_hibernate): {
-     HEAVY_SWAPOUT;
-     if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) {
-	 FCALLS = c_p->fcalls;
-	 c_p->flags &= ~F_HIBERNATE_SCHED;
-	 goto do_schedule;
-     } else {
-	 HEAVY_SWAPIN;
-	 I = handle_error(c_p, I, reg, &bif_export[BIF_hibernate_3]->info.mfa);
-	 goto post_error_handling;
-     }
+ OpCase(normal_exit): {
+     SWAPOUT;
+     c_p->freason = EXC_NORMAL;
+     c_p->arity = 0; /* In case this process will ever be garbed again. */
+     ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
+     erts_do_exit_process(c_p, am_normal);
+     ERTS_REQ_PROC_MAIN_LOCK(c_p);
+     goto do_schedule;
  }
 
- /* This is optimised as an instruction because
-    it has to be very very fast */
- OpCase(i_perf_counter): {
-    BeamInstr* next;
-    ErtsSysPerfCounter ts;
-    PreFetch(0, next);
+ OpCase(continue_exit): {
+     ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
+     erts_continue_exit_process(c_p);
+     ERTS_REQ_PROC_MAIN_LOCK(c_p);
+     goto do_schedule;
+ }
 
-    ts = erts_sys_perf_counter();
+ find_func_info: {
+     SWAPOUT;
+     I = handle_error(c_p, I, reg, NULL);
+     goto post_error_handling;
+ }
 
-    if (IS_SSMALL(ts)) {
-        r(0) = make_small((Sint)ts);
-    } else {
-        TestHeap(ERTS_SINT64_HEAP_SIZE(ts),0);
-        r(0) = make_big(HTOP);
-#if defined(ARCH_32)
-        if (ts >= (((Uint64) 1) << 32)) {
-            *HTOP = make_pos_bignum_header(2);
-            BIG_DIGIT(HTOP, 0) = (Uint) (ts & ((Uint) 0xffffffff));
-            BIG_DIGIT(HTOP, 1) = (Uint) ((ts >> 32) & ((Uint) 0xffffffff));
-            HTOP += 3;
-        }
-        else
-#endif
-        {
-                *HTOP = make_pos_bignum_header(1);
-                BIG_DIGIT(HTOP, 0) = (Uint) ts;
-                HTOP += 2;
-        }
+ OpCase(call_error_handler):
+    /*
+     * At this point, I points to the code[3] in the export entry for
+     * a function which is not loaded.
+     *
+     * code[0]: Module
+     * code[1]: Function
+     * code[2]: Arity
+     * code[3]: &&call_error_handler
+     * code[4]: Not used
+     */
+    HEAVY_SWAPOUT;
+    I = call_error_handler(c_p, erts_code_to_codemfa(I),
+                           reg, am_undefined_function);
+    HEAVY_SWAPIN;
+    if (I) {
+	Goto(*I);
     }
-    NextPF(0, next);
- }
 
- OpCase(i_debug_breakpoint): {
-     HEAVY_SWAPOUT;
-     I = call_error_handler(c_p, erts_code_to_codemfa(I), reg, am_breakpoint);
-     HEAVY_SWAPIN;
-     if (I) {
+ /* Fall through */
+ OpCase(error_action_code): {
+    handle_error:
+     SWAPOUT;
+     I = handle_error(c_p, NULL, reg, NULL);
+ post_error_handling:
+     if (I == 0) {
+	 goto do_schedule;
+     } else {
+	 ASSERT(!is_value(r(0)));
+	 SWAPIN;
 	 Goto(*I);
      }
-     goto handle_error;
  }
 
+ OpCase(i_func_info_IaaI): {
+     ErtsCodeInfo *ci = (ErtsCodeInfo*)I;
+     c_p->freason = EXC_FUNCTION_CLAUSE;
+     c_p->current = &ci->mfa;
+     goto handle_error;
+ }
 
- OpCase(system_limit_j):
- system_limit:
-    c_p->freason = SYSTEM_LIMIT;
-    goto lb_Cl_error;
-
+#include "beam_cold.h"
 
 #ifdef ERTS_OPCODE_COUNTER_SUPPORT
     DEFINE_COUNTING_LABELS;
@@ -5872,7 +2329,6 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
     int arity;
     Eterm tmp;
 
-
     if (is_not_atom(module) || is_not_atom(function)) {
 	/*
 	 * No need to test args here -- done below.
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 5429a61d7b..d38e71f489 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -83,8 +83,9 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int);
 
 typedef struct {
     Uint value;			/* Value of label (NULL if not known yet). */
-    Sint patches;		/* Index (into code buffer) to first location
-				 * which must be patched with the value of this label.
+    Sint patches;		/* Index (into code buffer) to first
+				 * location which must be patched with
+				 * the value of this label.
 				 */
     Uint looprec_targeted;	/* Non-zero if this label is the target of a loop_rec
 				 * instruction.
@@ -2871,15 +2872,15 @@ gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index,
     if (Index.type == TAG_i && Index.val > 0 &&
 	(Tuple.type == TAG_x || Tuple.type == TAG_y)) {
 	op->op = genop_i_fast_element_4;
-	op->a[0] = Fail;
-	op->a[1] = Tuple;
+	op->a[0] = Tuple;
+	op->a[1] = Fail;
 	op->a[2].type = TAG_u;
 	op->a[2].val = Index.val;
 	op->a[3] = Dst;
     } else {
 	op->op = genop_i_element_4;
-	op->a[0] = Fail;
-	op->a[1] = Tuple;
+	op->a[0] = Tuple;
+	op->a[1] = Fail;
 	op->a[2] = Index;
 	op->a[3] = Dst;
     }
@@ -2959,13 +2960,14 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
 	    op->a[0] = Ms;
 	    op->a[1] = Fail;
 	    op->a[2] = Dst;
+#ifdef ARCH_64
 	} else if (bits == 32 && (Flags.val & BSF_LITTLE) == 0) {
-	    op->op = genop_i_bs_get_integer_32_4;
-	    op->arity = 4;
+	    op->op = genop_i_bs_get_integer_32_3;
+	    op->arity = 3;
 	    op->a[0] = Ms;
 	    op->a[1] = Fail;
-	    op->a[2] = Live;
-	    op->a[3] = Dst;
+	    op->a[2] = Dst;
+#endif
 	} else {
 	generic:
 	    if (bits < SMALL_BITS) {
@@ -3099,16 +3101,6 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
     return op;
 }
 
-/*
- * Predicate to test whether a heap binary should be generated.
- */
-
-static int
-should_gen_heap_bin(LoaderState* stp, GenOpArg Src)
-{
-    return Src.val <= ERL_ONHEAP_BIN_LIMIT;
-}
-
 /*
  * Predicate to test whether a binary construction is too big.
  */
@@ -3381,13 +3373,6 @@ negation_is_small(LoaderState* stp, GenOpArg Int)
            IS_SSMALL(-((Sint)Int.val));
 }
 
-
-static int
-smp(LoaderState* stp)
-{
-    return 1;
-}
-
 /*
  * Mark this label.
  */
@@ -3421,11 +3406,11 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
     Sint timeout;
 
     NEW_GENOP(stp, op);
-    op->op = genop_i_wait_timeout_2;
+    op->op = genop_wait_timeout_unlocked_2;
     op->next = NULL;
     op->arity = 2;
-    op->a[0] = Fail;
-    op->a[1].type = TAG_u;
+    op->a[0].type = TAG_u;
+    op->a[1] = Fail;
     
     if (Time.type == TAG_i && (timeout = Time.val) >= 0 &&
 #if defined(ARCH_64)
@@ -3434,7 +3419,7 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
 	1
 #endif
 	) {
-	op->a[1].val = timeout;
+	op->a[0].val = timeout;
 #if !defined(ARCH_64)
     } else if (Time.type == TAG_q) {
 	Eterm big;
@@ -3448,7 +3433,7 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
 	} else {
 	    Uint u;
 	    (void) term_to_Uint(big, &u);
-	    op->a[1].val = (BeamInstr) u;
+	    op->a[0].val = (BeamInstr) u;
 	}
 #endif
     } else {
@@ -3468,7 +3453,7 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
     Sint timeout;
 
     NEW_GENOP(stp, op);
-    op->op = genop_i_wait_timeout_locked_2;
+    op->op = genop_wait_timeout_locked_2;
     op->next = NULL;
     op->arity = 2;
     op->a[0] = Fail;
diff --git a/erts/emulator/beam/bif_instrs.tab b/erts/emulator/beam/bif_instrs.tab
new file mode 100644
index 0000000000..5aa0523e06
--- /dev/null
+++ b/erts/emulator/beam/bif_instrs.tab
@@ -0,0 +1,539 @@
+// -*- c -*-
+//
+// %CopyrightBegin%
+//
+// Copyright Ericsson AB 2017. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// %CopyrightEnd%
+//
+
+// ================================================================
+// All guards with zero arguments have special instructions,
+// for example:
+//
+// 	self/0
+// 	node/0
+//
+// All other guard BIFs take one or two arguments.
+// ================================================================
+
+CALL_GUARD_BIF(BF, TmpReg, Dst) {
+    Eterm result;
+
+    ERTS_DBG_CHK_REDS(c_p, FCALLS);
+    c_p->fcalls = FCALLS;
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+    ERTS_CHK_MBUF_SZ(c_p);
+    result = (*$BF)(c_p, $TmpReg, I);
+    ERTS_CHK_MBUF_SZ(c_p);
+    ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    ERTS_HOLE_CHECK(c_p);
+    FCALLS = c_p->fcalls;
+    ERTS_DBG_CHK_REDS(c_p, FCALLS);
+    if (is_value(result)) {
+        $Dst = result;
+        $NEXT0();
+    }
+}
+
+// Guard BIF in head.  On failure, ignore the error and jump
+// to the code for the next clause.  We don't support tracing
+// of guard BIFs.
+
+bif1(Fail, Bif, Src, Dst) {
+    ErtsBifFunc bf;
+    Eterm tmp_reg[1];
+
+    tmp_reg[0] = $Src;
+    bf = (BifFunction) $Bif;
+    $CALL_GUARD_BIF(bf, tmp_reg, $Dst);
+
+    $FAIL($Fail);
+}
+
+//
+// Guard BIF in body.  It can fail like any BIF.  No trace support.
+//
+
+bif1_body(Bif, Src, Dst) {
+    ErtsBifFunc bf;
+    Eterm tmp_reg[1];
+
+    tmp_reg[0] = $Src;
+    bf = (BifFunction) $Bif;
+    $CALL_GUARD_BIF(bf, tmp_reg, $Dst);
+
+    reg[0] = tmp_reg[0];
+    SWAPOUT;
+    I = handle_error(c_p, I, reg, ubif2mfa((void *) bf));
+    goto post_error_handling;
+}
+
+//
+// Guard bif in guard with two arguments ('and'/2, 'or'/2, 'xor'/2).
+//
+
+i_bif2(Fail, Bif, Src1, Src2, Dst) {
+    Eterm tmp_reg[2];
+    ErtsBifFunc bf;
+
+    tmp_reg[0] = $Src1;
+    tmp_reg[1] = $Src2;
+    bf = (ErtsBifFunc) $Bif;
+    $CALL_GUARD_BIF(bf, tmp_reg, $Dst);
+    $FAIL($Fail);
+}
+
+//
+// Guard bif in body with two arguments ('and'/2, 'or'/2, 'xor'/2).
+//
+
+i_bif2_body(Bif, Src1, Src2, Dst) {
+    Eterm tmp_reg[2];
+    ErtsBifFunc bf;
+
+    tmp_reg[0] = $Src1;
+    tmp_reg[1] = $Src2;
+    bf = (ErtsBifFunc) $Bif;
+    $CALL_GUARD_BIF(bf, tmp_reg, $Dst);
+    reg[0] = tmp_reg[0];
+    reg[1] = tmp_reg[1];
+    SWAPOUT;
+    I = handle_error(c_p, I, reg, ubif2mfa((void *) bf));
+    goto post_error_handling;
+}
+
+//
+// Garbage-collecting BIF with one argument in either guard or body.
+//
+
+i_gc_bif1(Fail, Bif, Src, Live, Dst) {
+    typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
+    GcBifFunction bf;
+    Eterm result;
+    Uint live = (Uint) $Live;
+
+    x(live) = $Src;
+    bf = (GcBifFunction) $Bif;
+    ERTS_DBG_CHK_REDS(c_p, FCALLS);
+    c_p->fcalls = FCALLS;
+    SWAPOUT;
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
+    ERTS_CHK_MBUF_SZ(c_p);
+    result = (*bf)(c_p, reg, live);
+    ERTS_CHK_MBUF_SZ(c_p);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    SWAPIN;
+    ERTS_HOLE_CHECK(c_p);
+    FCALLS = c_p->fcalls;
+    ERTS_DBG_CHK_REDS(c_p, FCALLS);
+    if (is_value(result)) {
+        $REFRESH_GEN_DEST();
+        $Dst = result;
+        $NEXT0();
+    }
+    if ($Fail != 0) {          /* Handle error in guard. */
+        $NEXT($Fail);
+    }
+
+    /* Handle error in body. */
+    x(0) = x(live);
+    I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf));
+    goto post_error_handling;
+}
+
+//
+// Garbage-collecting BIF with two arguments in either guard or body.
+//
+
+i_gc_bif2(Fail, Bif, Live, Src1, Src2, Dst) {
+    typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
+    GcBifFunction bf;
+    Eterm result;
+    Uint live = (Uint) $Live;
+
+    /*
+     * XXX This calling convention does not make sense. 'live'
+     * should point out the first argument, not the second
+     * (i.e. 'live' should not be incremented below).
+     */
+    x(live) = $Src1;
+    x(live+1) = $Src2;
+    live++;
+
+    bf = (GcBifFunction) $Bif;
+    ERTS_DBG_CHK_REDS(c_p, FCALLS);
+    c_p->fcalls = FCALLS;
+    SWAPOUT;
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
+    ERTS_CHK_MBUF_SZ(c_p);
+    result = (*bf)(c_p, reg, live);
+    ERTS_CHK_MBUF_SZ(c_p);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    SWAPIN;
+    ERTS_HOLE_CHECK(c_p);
+    FCALLS = c_p->fcalls;
+    ERTS_DBG_CHK_REDS(c_p, FCALLS);
+    if (is_value(result)) {
+        $REFRESH_GEN_DEST();
+        $Dst = result;
+        $NEXT0();
+    }
+
+    if ($Fail != 0) {          /* Handle error in guard. */
+        $NEXT($Fail);
+    }
+
+    /* Handle error in body. */
+    live--;
+    x(0) = x(live);
+    x(1) = x(live+1);
+    I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf));
+    goto post_error_handling;
+}
+
+//
+// Garbage-collecting BIF with three arguments in either guard or body.
+//
+
+i_gc_bif3(Fail, Bif, Live, Src2, Src3, Dst) {
+    typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
+    GcBifFunction bf;
+    Eterm result;
+    Uint live = (Uint) $Live;
+
+    /*
+     * XXX This calling convention does not make sense. 'live'
+     * should point out the first argument, not the third
+     * (i.e. 'live' should not be incremented below).
+     */
+    x(live) = x(SCRATCH_X_REG);
+    x(live+1) = $Src2;
+    x(live+2) = $Src3;
+    live += 2;
+
+    bf = (GcBifFunction) $Bif;
+    ERTS_DBG_CHK_REDS(c_p, FCALLS);
+    c_p->fcalls = FCALLS;
+    SWAPOUT;
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
+    ERTS_CHK_MBUF_SZ(c_p);
+    result = (*bf)(c_p, reg, live);
+    ERTS_CHK_MBUF_SZ(c_p);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    SWAPIN;
+    ERTS_HOLE_CHECK(c_p);
+    FCALLS = c_p->fcalls;
+    ERTS_DBG_CHK_REDS(c_p, FCALLS);
+    if (is_value(result)) {
+        $REFRESH_GEN_DEST();
+        $Dst = result;
+        $NEXT0();
+    }
+
+    /* Handle error in guard. */
+    if ($Fail != 0) {
+        $NEXT($Fail);
+    }
+
+    /* Handle error in body. */
+    live -= 2;
+    x(0) = x(live);
+    x(1) = x(live+1);
+    x(2) = x(live+2);
+    I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf));
+    goto post_error_handling;
+}
+
+//
+// The most general BIF call.  The BIF may build any amount of data
+// on the heap.  The result is always returned in r(0).
+//
+call_bif(Exp) {
+    ErtsBifFunc bf;
+    Eterm result;
+    ErlHeapFragment *live_hf_end;
+    Export *export = (Export*) $Exp;
+
+    if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) {
+        /* If we have run out of reductions, we do a context
+           switch before calling the bif */
+        c_p->arity = GET_BIF_ARITY(export);
+        c_p->current = &export->info.mfa;
+        goto context_switch3;
+    }
+
+    ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_BIF_MODULE(export),
+                                      GET_BIF_ADDRESS(export));
+
+    bf = GET_BIF_ADDRESS(export);
+
+    PRE_BIF_SWAPOUT(c_p);
+    ERTS_DBG_CHK_REDS(c_p, FCALLS);
+    c_p->fcalls = FCALLS - 1;
+    if (FCALLS <= 0) {
+        save_calls(c_p, export);
+    }
+    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+    live_hf_end = c_p->mbuf;
+    ERTS_CHK_MBUF_SZ(c_p);
+    result = (*bf)(c_p, reg, I);
+    ERTS_CHK_MBUF_SZ(c_p);
+    ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+    ERTS_HOLE_CHECK(c_p);
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
+    if (ERTS_IS_GC_DESIRED(c_p)) {
+        Uint arity = GET_BIF_ARITY(export);
+        result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result,
+                                            reg, arity);
+        E = c_p->stop;
+    }
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    HTOP = HEAP_TOP(c_p);
+    FCALLS = c_p->fcalls;
+    ERTS_DBG_CHK_REDS(c_p, FCALLS);
+    /* We have to update the cache if we are enabled in order
+       to make sure no book keeping is done after we disabled
+       msacc. We don't always do this as it is quite expensive. */
+    if (ERTS_MSACC_IS_ENABLED_CACHED_X()) {
+        ERTS_MSACC_UPDATE_CACHE_X();
+    }
+    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
+    if (is_value(result)) {
+        r(0) = result;
+        CHECK_TERM(r(0));
+        $NEXT0();
+    } else if (c_p->freason == TRAP) {
+        SET_CP(c_p, I+2);
+        SET_I(c_p->i);
+        SWAPIN;
+        Dispatch();
+    }
+
+    /*
+     * Error handling.  SWAPOUT is not needed because it was done above.
+     */
+    ASSERT(c_p->stop == E);
+    I = handle_error(c_p, I, reg, &export->info.mfa);
+    goto post_error_handling;
+}
+
+//
+// Send is almost a standard call-BIF with two arguments, except for:
+//   1. It cannot be traced.
+//   2. There is no pointer to the send_2 function stored in
+//      the instruction.
+//
+
+send() {
+    Eterm result;
+
+    if (!(FCALLS > 0 || FCALLS > neg_o_reds)) {
+        /* If we have run out of reductions, we do a context
+           switch before calling the bif */
+        c_p->arity = 2;
+        c_p->current = NULL;
+        goto context_switch3;
+    }
+
+    PRE_BIF_SWAPOUT(c_p);
+    c_p->fcalls = FCALLS - 1;
+    result = erl_send(c_p, r(0), x(1));
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    HTOP = HEAP_TOP(c_p);
+    FCALLS = c_p->fcalls;
+    if (is_value(result)) {
+        r(0) = result;
+        CHECK_TERM(r(0));
+    } else if (c_p->freason == TRAP) {
+        SET_CP(c_p, I+1);
+        SET_I(c_p->i);
+        SWAPIN;
+        Dispatch();
+    } else {
+        goto find_func_info;
+    }
+}
+
+call_nif := nif_bif.call_nif.epilogue;
+apply_bif := nif_bif.apply_bif.epilogue;
+
+nif_bif.head() {
+    Eterm nif_bif_result;
+    Eterm bif_nif_arity;
+    BifFunction vbf;
+    ErlHeapFragment *live_hf_end;
+    ErtsCodeMFA *codemfa;
+}
+
+nif_bif.call_nif() {
+    /*
+     * call_nif is always first instruction in function:
+     *
+     * I[-3]: Module
+     * I[-2]: Function
+     * I[-1]: Arity
+     * I[0]: &&call_nif
+     * I[1]: Function pointer to NIF function
+     * I[2]: Pointer to erl_module_nif
+     * I[3]: Function pointer to dirty NIF
+     *
+     * This layout is determined by the NifExport struct
+     */
+
+    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
+
+    codemfa = erts_code_to_codemfa(I);
+
+    c_p->current = codemfa; /* current and vbf set to please handle_error */
+
+    DTRACE_NIF_ENTRY(c_p, codemfa);
+
+    HEAVY_SWAPOUT;
+
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    bif_nif_arity = codemfa->arity;
+    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
+
+    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+    {
+        typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
+        NifF* fp = vbf = (NifF*) I[1];
+        struct enif_environment_t env;
+        ASSERT(c_p->scheduler_data);
+        live_hf_end = c_p->mbuf;
+        ERTS_CHK_MBUF_SZ(c_p);
+        erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
+        nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
+        if (env.exception_thrown)
+            nif_bif_result = THE_NON_VALUE;
+        erts_post_nif(&env);
+        ERTS_CHK_MBUF_SZ(c_p);
+
+        PROCESS_MAIN_CHK_LOCKS(c_p);
+        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+        ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
+        ASSERT(!env.exiting);
+        ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+    }
+
+    DTRACE_NIF_RETURN(c_p, codemfa);
+}
+
+nif_bif.apply_bif() {
+    /*
+     * At this point, I points to the code[0] in the export entry for
+     * the BIF:
+     *
+     * code[-3]: Module
+     * code[-2]: Function
+     * code[-1]: Arity
+     * code[0]: &&apply_bif
+     * code[1]: Function pointer to BIF function
+     */
+
+    if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
+        /* If we have run out of reductions, we do a context
+           switch before calling the bif */
+        goto context_switch;
+    }
+
+    codemfa = erts_code_to_codemfa(I);
+
+    ERTS_MSACC_SET_BIF_STATE_CACHED_X(codemfa->module, (BifFunction)Arg(0));
+
+
+    /* In case we apply process_info/1,2 or load_nif/1 */
+    c_p->current = codemfa;
+    c_p->i = I;		/* In case we apply check_process_code/2. */
+    c_p->arity = 0;		/* To allow garbage collection on ourselves
+                                 * (check_process_code/2).
+                                 */
+    DTRACE_BIF_ENTRY(c_p, codemfa);
+
+    SWAPOUT;
+    ERTS_DBG_CHK_REDS(c_p, FCALLS - 1);
+    c_p->fcalls = FCALLS - 1;
+    vbf = (BifFunction) Arg(0);
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    bif_nif_arity = codemfa->arity;
+    ASSERT(bif_nif_arity <= 4);
+    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+    {
+        ErtsBifFunc bf = vbf;
+        ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+        live_hf_end = c_p->mbuf;
+        ERTS_CHK_MBUF_SZ(c_p);
+        nif_bif_result = (*bf)(c_p, reg, I);
+        ERTS_CHK_MBUF_SZ(c_p);
+        ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
+               is_non_value(nif_bif_result));
+        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+        PROCESS_MAIN_CHK_LOCKS(c_p);
+    }
+    /* We have to update the cache if we are enabled in order
+       to make sure no book keeping is done after we disabled
+       msacc. We don't always do this as it is quite expensive. */
+    if (ERTS_MSACC_IS_ENABLED_CACHED_X())
+        ERTS_MSACC_UPDATE_CACHE_X();
+    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
+    DTRACE_BIF_RETURN(c_p, codemfa);
+}
+
+nif_bif.epilogue() {
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
+    ERTS_HOLE_CHECK(c_p);
+    if (ERTS_IS_GC_DESIRED(c_p)) {
+        nif_bif_result = erts_gc_after_bif_call_lhf(c_p, live_hf_end,
+                                                    nif_bif_result,
+                                                    reg, bif_nif_arity);
+    }
+    SWAPIN;  /* There might have been a garbage collection. */
+    FCALLS = c_p->fcalls;
+    ERTS_DBG_CHK_REDS(c_p, FCALLS);
+    if (is_value(nif_bif_result)) {
+        r(0) = nif_bif_result;
+        CHECK_TERM(r(0));
+        SET_I(c_p->cp);
+        c_p->cp = 0;
+        Goto(*I);
+    } else if (c_p->freason == TRAP) {
+        SET_I(c_p->i);
+        if (c_p->flags & F_HIBERNATE_SCHED) {
+            c_p->flags &= ~F_HIBERNATE_SCHED;
+            goto do_schedule;
+        }
+        Dispatch();
+    }
+    I = handle_error(c_p, c_p->cp, reg, c_p->current);
+    goto post_error_handling;
+}
diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab
index 420fd3074c..a4d4afe7d4 100644
--- a/erts/emulator/beam/bs_instrs.tab
+++ b/erts/emulator/beam/bs_instrs.tab
@@ -19,18 +19,90 @@
 // %CopyrightEnd%
 //
 
+%if ARCH_64
+BS_SAFE_MUL(A, B, Fail, Dst) {
+    Uint64 res = ($A) * ($B);
+    if (res / $B != $A) {
+        $Fail;
+    }
+    $Dst = res;
+}
+%else
+BS_SAFE_MUL(A, B, Fail, Dst) {
+    Uint64 res = (Uint64)($A) * (Uint64)($B);
+    if ((res >> (8*sizeof(Uint))) != 0) {
+        $Fail;
+    }
+    $Dst = res;
+}
+%endif
+
+BS_GET_FIELD_SIZE(Bits, Unit, Fail, Dst) {
+    Sint signed_size;
+    Uint uint_size;
+    Uint temp_bits;
+
+    if (is_small($Bits)) {
+        signed_size = signed_val($Bits);
+        if (signed_size < 0) {
+            $Fail;
+        }
+        uint_size = (Uint) signed_size;
+    } else {
+        if (!term_to_Uint($Bits, &temp_bits)) {
+            $Fail;
+        }
+        uint_size = temp_bits;
+    }
+    $BS_SAFE_MUL(uint_size, $Unit, $Fail, $Dst);
+}
+
+BS_GET_UNCHECKED_FIELD_SIZE(Bits, Unit, Fail, Dst) {
+    Sint signed_size;
+    Uint uint_size;
+    Uint temp_bits;
+
+    if (is_small($Bits)) {
+        signed_size = signed_val($Bits);
+        if (signed_size < 0) {
+            $Fail;
+        }
+        uint_size = (Uint) signed_size;
+    } else {
+        if (!term_to_Uint($Bits, &temp_bits)) {
+            $Fail;
+        }
+        uint_size = temp_bits;
+    }
+    $Dst = uint_size * $Unit;
+}
+
+TEST_BIN_VHEAP(VNh, Nh, Live) {
+    Uint need = $Nh;
+    if (E - HTOP < need || MSO(c_p).overhead + $VNh >= BIN_VHEAP_SZ(c_p)) {
+        SWAPOUT;
+        PROCESS_MAIN_CHK_LOCKS(c_p);
+        FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, $Live, FCALLS);
+        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+        PROCESS_MAIN_CHK_LOCKS(c_p);
+        SWAPIN;
+    }
+    HEAP_SPACE_VERIFIED(need);
+}
+
 i_bs_get_binary_all2(Fail, Ms, Live, Unit, Dst) {
     ErlBinMatchBuffer *_mb;
     Eterm _result;
-    TestHeap(ERL_SUB_BIN_SIZE, $Live);
+
+    $GC_TEST(0, ERL_SUB_BIN_SIZE, $Live);
     _mb = ms_matchbuffer($Ms);
     if (((_mb->size - _mb->offset) % $Unit) == 0) {
-      LIGHT_SWAPOUT;
-      _result = erts_bs_get_binary_all_2(c_p, _mb);
-      LIGHT_SWAPIN;
-      HEAP_SPACE_VERIFIED(0);
-      ASSERT(is_value(_result));
-      $Dst = _result;
+        LIGHT_SWAPOUT;
+        _result = erts_bs_get_binary_all_2(c_p, _mb);
+        LIGHT_SWAPIN;
+        HEAP_SPACE_VERIFIED(0);
+        ASSERT(is_value(_result));
+        $Dst = _result;
     } else {
 	HEAP_SPACE_VERIFIED(0);
 	$FAIL($Fail);
@@ -41,8 +113,8 @@ i_bs_get_binary2(Fail, Ms, Live, Sz, Flags, Dst) {
     ErlBinMatchBuffer *_mb;
     Eterm _result;
     Uint _size;
-    BsGetFieldSize($Sz, (($Flags) >> 3), $FAIL($Fail), _size);
-    TestHeap(ERL_SUB_BIN_SIZE, $Live);
+    $BS_GET_FIELD_SIZE($Sz, (($Flags) >> 3), $FAIL($Fail), _size);
+    $GC_TEST(0, ERL_SUB_BIN_SIZE, $Live);
     _mb = ms_matchbuffer($Ms);
     LIGHT_SWAPOUT;
     _result = erts_bs_get_binary_2(c_p, _size, $Flags, _mb);
@@ -58,7 +130,7 @@ i_bs_get_binary2(Fail, Ms, Live, Sz, Flags, Dst) {
 i_bs_get_binary_imm2(Fail, Ms, Live, Sz, Flags, Dst) {
     ErlBinMatchBuffer *_mb;
     Eterm _result;
-    TestHeap(heap_bin_size(ERL_ONHEAP_BIN_LIMIT), $Live);
+    $GC_TEST(0, heap_bin_size(ERL_ONHEAP_BIN_LIMIT), $Live);
     _mb = ms_matchbuffer($Ms);
     LIGHT_SWAPOUT;
     _result = erts_bs_get_binary_2(c_p, $Sz, $Flags, _mb);
@@ -80,7 +152,7 @@ i_bs_get_float2(Fail, Ms, Live, Sz, Flags, Dst) {
         $FAIL($Fail);
     }
     _size *= (($Flags) >> 3);
-    TestHeap(FLOAT_SIZE_OBJECT, $Live);
+    $GC_TEST(0, FLOAT_SIZE_OBJECT, $Live);
     _mb = ms_matchbuffer($Ms);
     LIGHT_SWAPOUT;
     _result = erts_bs_get_float_2(c_p, _size, ($Flags), _mb);
@@ -99,7 +171,7 @@ i_bs_skip_bits2(Fail, Ms, Bits, Unit) {
     Uint _size;
 
     _mb = ms_matchbuffer($Ms);
-    BsGetFieldSize($Bits, $Unit, $FAIL($Fail), _size);
+    $BS_GET_FIELD_SIZE($Bits, $Unit, $FAIL($Fail), _size);
     new_offset = _mb->offset + _size;
     if (new_offset <= _mb->size) {
         _mb->offset = new_offset;
@@ -132,48 +204,820 @@ i_bs_skip_bits_imm2(Fail, Ms, Bits) {
 
 i_new_bs_put_binary(Fail, Sz, Flags, Src) {
     Sint _size;
-    BsGetUncheckedFieldSize($Sz, (($Flags) >> 3), goto badarg, _size);
+    $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size);
     if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2(($Src), _size))) {
-        goto badarg;
+        $BADARG($Fail);
     }
 }
 
 i_new_bs_put_binary_all(Fail, Src, Unit) {
     if (!erts_new_bs_put_binary_all(ERL_BITS_ARGS_2(($Src), ($Unit)))) {
-        goto badarg;
+        $BADARG($Fail);
     }
 }
 
 i_new_bs_put_binary_imm(Fail, Sz, Src) {
     if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2(($Src), ($Sz)))) {
-        goto badarg;
+        $BADARG($Fail);
     }
 }
 
 i_new_bs_put_float(Fail, Sz, Flags, Src) {
     Sint _size;
-    BsGetUncheckedFieldSize($Sz, (($Flags) >> 3), goto badarg, _size);
+    $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size);
     if (!erts_new_bs_put_float(c_p, ($Src), _size, ($Flags))) {
-        goto badarg;
+        $BADARG($Fail);
     }
 }
 
 i_new_bs_put_float_imm(Fail, Sz, Flags, Src) {
     if (!erts_new_bs_put_float(c_p, ($Src), ($Sz), ($Flags))) {
-        goto badarg;
+        $BADARG($Fail);
     }
 }
 
 i_new_bs_put_integer(Fail, Sz, Flags, Src) {
    Sint _size;
-   BsGetUncheckedFieldSize($Sz, (($Flags) >> 3), goto badarg, _size);
+   $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size);
    if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), _size, ($Flags)))) {
-       goto badarg;
+       $BADARG($Fail);
    }
 }
 
 i_new_bs_put_integer_imm(Fail, Sz, Flags, Src) {
     if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), ($Sz), ($Flags)))) {
-        goto badarg;
+        $BADARG($Fail);
+    }
+}
+
+#
+# i_bs_init*
+#
+
+i_bs_init_fail_heap := bs_init.fail_heap.verify.execute;
+i_bs_init_fail := bs_init.fail.verify.execute;
+i_bs_init := bs_init.plain.execute;
+i_bs_init_heap := bs_init.heap.execute;
+
+bs_init.head() {
+    Eterm BsOp1;
+    Eterm BsOp2;
+}
+
+bs_init.fail_heap(Size, HeapAlloc) {
+    BsOp1 = $Size;
+    BsOp2 = $HeapAlloc;
+}
+
+bs_init.fail(Size) {
+    BsOp1 = $Size;
+    BsOp2 = 0;
+}
+
+bs_init.plain(Size) {
+    BsOp1 = $Size;
+    BsOp2 = 0;
+}
+
+bs_init.heap(Size, HeapAlloc) {
+    BsOp1 = $Size;
+    BsOp2 = $HeapAlloc;
+}
+
+bs_init.verify(Fail) {
+    if (is_small(BsOp1)) {
+        Sint size = signed_val(BsOp1);
+        if (size < 0) {
+            $BADARG($Fail);
+        }
+        BsOp1 = (Eterm) size;
+    } else {
+        Uint bytes;
+
+        if (!term_to_Uint(BsOp1, &bytes)) {
+            c_p->freason = bytes;
+            $FAIL_HEAD_OR_BODY($Fail);
+        }
+        if ((bytes >> (8*sizeof(Uint)-3)) != 0) {
+            $SYSTEM_LIMIT($Fail);
+        }
+        BsOp1 = (Eterm) bytes;
+    }
+}
+
+bs_init.execute(Live, Dst) {
+    if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) {
+        ErlHeapBin* hb;
+        Uint bin_need;
+
+        bin_need = heap_bin_size(BsOp1);
+        erts_bin_offset = 0;
+        erts_writable_bin = 0;
+        $GC_TEST(0, bin_need+BsOp2+ERL_SUB_BIN_SIZE, $Live);
+        hb = (ErlHeapBin *) HTOP;
+        HTOP += bin_need;
+        hb->thing_word = header_heap_bin(BsOp1);
+        hb->size = BsOp1;
+        erts_current_bin = (byte *) hb->data;
+        $Dst = make_binary(hb);
+    } else {
+        Binary* bptr;
+        ProcBin* pb;
+
+        erts_bin_offset = 0;
+        erts_writable_bin = 0;
+        $TEST_BIN_VHEAP(BsOp1 / sizeof(Eterm),
+                        BsOp2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, $Live);
+
+        /*
+         * Allocate the binary struct itself.
+         */
+        bptr = erts_bin_nrml_alloc(BsOp1);
+        erts_current_bin = (byte *) bptr->orig_bytes;
+
+        /*
+         * Now allocate the ProcBin on the heap.
+         */
+        pb = (ProcBin *) HTOP;
+        HTOP += PROC_BIN_SIZE;
+        pb->thing_word = HEADER_PROC_BIN;
+        pb->size = BsOp1;
+        pb->next = MSO(c_p).first;
+        MSO(c_p).first = (struct erl_off_heap_header*) pb;
+        pb->val = bptr;
+        pb->bytes = (byte*) bptr->orig_bytes;
+        pb->flags = 0;
+
+        OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm));
+
+        $Dst = make_binary(pb);
+    }
+}
+
+#
+# i_bs_init_bits*
+#
+
+i_bs_init_bits           := bs_init_bits.plain.execute;
+i_bs_init_bits_heap      := bs_init_bits.heap.execute;
+i_bs_init_bits_fail      := bs_init_bits.fail.verify.execute;
+i_bs_init_bits_fail_heap := bs_init_bits.fail_heap.verify.execute;
+
+bs_init_bits.head() {
+    Eterm new_binary;
+    Eterm num_bits_term;
+    Uint num_bits;
+    Uint alloc;
+    Uint num_bytes;
+}
+
+bs_init_bits.plain(NumBits) {
+    num_bits = $NumBits;
+    alloc = 0;
+}
+
+bs_init_bits.heap(NumBits, Alloc) {
+    num_bits = $NumBits;
+    alloc = $Alloc;
+}
+
+bs_init_bits.fail(NumBitsTerm) {
+    num_bits_term = $NumBitsTerm;
+    alloc = 0;
+}
+
+bs_init_bits.fail_heap(NumBitsTerm, Alloc) {
+    num_bits_term = $NumBitsTerm;
+    alloc = $Alloc;
+}
+
+bs_init_bits.verify(Fail) {
+    if (is_small(num_bits_term)) {
+        Sint size = signed_val(num_bits_term);
+        if (size < 0) {
+            $BADARG($Fail);
+        }
+        num_bits = (Uint) size;
+    } else {
+        Uint bits;
+
+        if (!term_to_Uint(num_bits_term, &bits)) {
+            c_p->freason = bits;
+            $FAIL_HEAD_OR_BODY($Fail);
+        }
+        num_bits = (Uint) bits;
+    }
+}
+
+bs_init_bits.execute(Live, Dst) {
+     num_bytes = ((Uint64)num_bits+(Uint64)7) >> 3;
+     if (num_bits & 7) {
+	 alloc += ERL_SUB_BIN_SIZE;
+     }
+     if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) {
+	 alloc += heap_bin_size(num_bytes);
+     } else {
+	 alloc += PROC_BIN_SIZE;
+     }
+     $test_heap(alloc, $Live);
+
+     /* num_bits = Number of bits to build
+      * num_bytes = Number of bytes to allocate in the binary
+      * alloc = Total number of words to allocate on heap
+      * Operands: NotUsed NotUsed Dst
+      */
+     if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) {
+	 ErlHeapBin* hb;
+
+	 erts_bin_offset = 0;
+	 erts_writable_bin = 0;
+	 hb = (ErlHeapBin *) HTOP;
+	 HTOP += heap_bin_size(num_bytes);
+	 hb->thing_word = header_heap_bin(num_bytes);
+	 hb->size = num_bytes;
+	 erts_current_bin = (byte *) hb->data;
+	 new_binary = make_binary(hb);
+
+     do_bits_sub_bin:
+	 if (num_bits & 7) {
+	     ErlSubBin* sb;
+
+	     sb = (ErlSubBin *) HTOP;
+	     HTOP += ERL_SUB_BIN_SIZE;
+	     sb->thing_word = HEADER_SUB_BIN;
+	     sb->size = num_bytes - 1;
+	     sb->bitsize = num_bits & 7;
+	     sb->offs = 0;
+	     sb->bitoffs = 0;
+	     sb->is_writable = 0;
+	     sb->orig = new_binary;
+	     new_binary = make_binary(sb);
+	 }
+	 HEAP_SPACE_VERIFIED(0);
+         $Dst = new_binary;
+     } else {
+	 Binary* bptr;
+	 ProcBin* pb;
+
+	 erts_bin_offset = 0;
+	 erts_writable_bin = 0;
+
+	 /*
+	  * Allocate the binary struct itself.
+	  */
+	 bptr = erts_bin_nrml_alloc(num_bytes);
+	 erts_current_bin = (byte *) bptr->orig_bytes;
+
+	 /*
+	  * Now allocate the ProcBin on the heap.
+	  */
+	 pb = (ProcBin *) HTOP;
+	 HTOP += PROC_BIN_SIZE;
+	 pb->thing_word = HEADER_PROC_BIN;
+	 pb->size = num_bytes;
+	 pb->next = MSO(c_p).first;
+	 MSO(c_p).first = (struct erl_off_heap_header*) pb;
+	 pb->val = bptr;
+	 pb->bytes = (byte*) bptr->orig_bytes;
+	 pb->flags = 0;
+	 OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm));
+	 new_binary = make_binary(pb);
+	 goto do_bits_sub_bin;
+     }
+}
+
+bs_add(Fail, Src1, Src2, Unit, Dst) {
+    Eterm Op1 = $Src1;
+    Eterm Op2 = $Src2;
+    Uint unit = $Unit;
+
+    if (is_both_small(Op1, Op2)) {
+        Sint Arg1 = signed_val(Op1);
+        Sint Arg2 = signed_val(Op2);
+
+        if (Arg1 >= 0 && Arg2 >= 0) {
+            $BS_SAFE_MUL(Arg2, unit, $SYSTEM_LIMIT($Fail), Op1);
+            Op1 += Arg1;
+
+        store_bs_add_result:
+            if (Op1 <= MAX_SMALL) {
+                Op1 = make_small(Op1);
+            } else {
+                /*
+                 * May generate a heap fragment, but in this
+                 * particular case it is OK, since the value will be
+                 * stored into an x register (the GC will scan x
+                 * registers for references to heap fragments) and
+                 * there is no risk that value can be stored into a
+                 * location that is not scanned for heap-fragment
+                 * references (such as the heap).
+                 */
+                SWAPOUT;
+                Op1 = erts_make_integer(Op1, c_p);
+                HTOP = HEAP_TOP(c_p);
+            }
+            $Dst = Op1;
+            $NEXT0();
+        }
+        $BADARG($Fail);
+    } else {
+        Uint a;
+        Uint b;
+        Uint c;
+
+        /*
+         * Now we know that one of the arguments is
+         * not a small. We must convert both arguments
+         * to Uints and check for errors at the same time.
+         *
+         * Error checking is tricky.
+         *
+         * If one of the arguments is not numeric or
+         * not positive, the error reason is BADARG.
+         *
+         * Otherwise if both arguments are numeric,
+         * but at least one argument does not fit in
+         * an Uint, the reason is SYSTEM_LIMIT.
+         */
+
+        if (!term_to_Uint(Op1, &a)) {
+            if (a == BADARG) {
+                $BADARG($Fail);
+            }
+            if (!term_to_Uint(Op2, &b)) {
+                c_p->freason = b;
+                $FAIL_HEAD_OR_BODY($Fail);
+            }
+            $SYSTEM_LIMIT($Fail);
+        } else if (!term_to_Uint(Op2, &b)) {
+            c_p->freason = b;
+            $FAIL_HEAD_OR_BODY($Fail);
+        }
+
+        /*
+         * The arguments are now correct and stored in a and b.
+         */
+
+        $BS_SAFE_MUL(b, unit, $SYSTEM_LIMIT($Fail), c);
+        Op1 = a + c;
+        if (Op1 < a) {
+            /*
+             * If the result is less than one of the
+             * arguments, there must have been an overflow.
+             */
+            $SYSTEM_LIMIT($Fail);
+        }
+        goto store_bs_add_result;
+    }
+    /* No fallthrough */
+    ASSERT(0);
+}
+
+bs_put_string(Len, Ptr) {
+    erts_new_bs_put_string(ERL_BITS_ARGS_2((byte *) $Ptr, $Len));
+}
+
+i_bs_append(Fail, ExtraHeap, Live, Unit, Size, Dst) {
+    Uint live = $Live;
+    Uint res;
+
+    HEAVY_SWAPOUT;
+    reg[live] = x(SCRATCH_X_REG);
+    res = erts_bs_append(c_p, reg, live, $Size, $ExtraHeap, $Unit);
+    HEAVY_SWAPIN;
+    if (is_non_value(res)) {
+        /* c_p->freason is already set (to BADARG or SYSTEM_LIMIT). */
+        $FAIL_HEAD_OR_BODY($Fail);
+    }
+    $Dst = res;
+}
+
+i_bs_private_append(Fail, Unit, Size, Src, Dst) {
+    Eterm res;
+
+    res = erts_bs_private_append(c_p, $Src, $Size, $Unit);
+    if (is_non_value(res)) {
+        /* c_p->freason is already set (to BADARG or SYSTEM_LIMIT). */
+        $FAIL_HEAD_OR_BODY($Fail);
+    }
+    $Dst = res;
+}
+
+bs_init_writable() {
+    HEAVY_SWAPOUT;
+    r(0) = erts_bs_init_writable(c_p, r(0));
+    HEAVY_SWAPIN;
+}
+
+i_bs_utf8_size(Src, Dst) {
+    Eterm arg = $Src;
+    Eterm result;
+
+    /*
+     * Calculate the number of bytes needed to encode the source
+     * operand to UTF-8. If the source operand is invalid (e.g. wrong
+     * type or range) we return a nonsense integer result (0 or 4). We
+     * can get away with that because we KNOW that bs_put_utf8 will do
+     * full error checking.
+     */
+
+    if (arg < make_small(0x80UL)) {
+        result = make_small(1);
+    } else if (arg < make_small(0x800UL)) {
+        result = make_small(2);
+    } else if (arg < make_small(0x10000UL)) {
+        result = make_small(3);
+    } else {
+        result = make_small(4);
+    }
+    $Dst = result;
+}
+
+i_bs_put_utf8(Fail, Src) {
+    if (!erts_bs_put_utf8(ERL_BITS_ARGS_1($Src))) {
+        $BADARG($Fail);
+    }
+}
+
+i_bs_utf16_size(Src, Dst) {
+    Eterm arg = $Src;
+    Eterm result = make_small(2);
+
+    /*
+     * Calculate the number of bytes needed to encode the source
+     * operarand to UTF-16. If the source operand is invalid (e.g. wrong
+     * type or range) we return a nonsense integer result (2 or 4). We
+     * can get away with that because we KNOW that bs_put_utf16 will do
+     * full error checking.
+     */
+
+    if (arg >= make_small(0x10000UL)) {
+        result = make_small(4);
+    }
+    $Dst = result;
+}
+
+bs_put_utf16(Fail, Flags, Src) {
+    if (!erts_bs_put_utf16(ERL_BITS_ARGS_2($Src, $Flags))) {
+        $BADARG($Fail);
+    }
+}
+
+// Validate a value about to be stored in a binary.
+i_bs_validate_unicode(Fail, Src) {
+    Eterm val = $Src;
+
+    /*
+     * There is no need to untag the integer, but it IS necessary
+     * to make sure it is small (if the term is a bignum, it could
+     * slip through the test, and there is no further test that
+     * would catch it, since bit syntax construction silently masks
+     * too big numbers).
+     */
+    if (is_not_small(val) || val > make_small(0x10FFFFUL) ||
+        (make_small(0xD800UL) <= val && val <= make_small(0xDFFFUL))) {
+        $BADARG($Fail);
+    }
+}
+
+// Validate a value that has been matched out.
+i_bs_validate_unicode_retract(Fail, Src, Ms) {
+    /*
+     * There is no need to untag the integer, but it IS necessary
+     * to make sure it is small (a bignum pointer could fall in
+     * the valid range).
+     */
+
+    Eterm i = $Src;
+    if (is_not_small(i) || i > make_small(0x10FFFFUL) ||
+        (make_small(0xD800UL) <= i && i <= make_small(0xDFFFUL))) {
+        Eterm ms = $Ms;		/* Match context */
+        ErlBinMatchBuffer* mb;
+
+        /* Invalid value. Retract the position in the binary. */
+        mb = ms_matchbuffer(ms);
+        mb->offset -= 32;
+        $BADARG($Fail);
+    }
+}
+
+
+//
+// Matching of binaries.
+//
+
+i_bs_start_match2 := bs_start_match.fetch.execute;
+
+bs_start_match.head() {
+    Uint slots;
+    Uint live;
+    Eterm header;
+    Eterm context;
+}
+
+bs_start_match.fetch(Src) {
+    context = $Src;
+}
+
+bs_start_match.execute(Fail, Live, Slots, Dst) {
+    if (!is_boxed(context)) {
+        $FAIL($Fail);
+    }
+    header = *boxed_val(context);
+    slots = $Slots;
+    live = $Live;
+    if (header_is_bin_matchstate(header)) {
+        ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(context);
+        Uint actual_slots = HEADER_NUM_SLOTS(header);
+        ms->save_offset[0] = ms->mb.offset;
+        if (actual_slots < slots) {
+            ErlBinMatchState* dst;
+            Uint live = $Live;
+            Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);
+
+            $GC_TEST_PRESERVE(wordsneeded, live, context);
+            ms = (ErlBinMatchState *) boxed_val(context);
+            dst = (ErlBinMatchState *) HTOP;
+            *dst = *ms;
+            *HTOP = HEADER_BIN_MATCHSTATE(slots);
+            HTOP += wordsneeded;
+            HEAP_SPACE_VERIFIED(0);
+            $Dst = make_matchstate(dst);
+        }
+    } else if (is_binary_header(header)) {
+        Eterm result;
+        Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);
+        $GC_TEST_PRESERVE(wordsneeded, live, context);
+        HEAP_TOP(c_p) = HTOP;
+#ifdef DEBUG
+        c_p->stop = E;	/* Needed for checking in HeapOnlyAlloc(). */
+#endif
+        result = erts_bs_start_match_2(c_p, context, slots);
+        HTOP = HEAP_TOP(c_p);
+        HEAP_SPACE_VERIFIED(0);
+        if (is_non_value(result)) {
+            $FAIL($Fail);
+        }
+        $Dst = result;
+    } else {
+        $FAIL($Fail);
+    }
+}
+
+bs_test_zero_tail2(Fail, Ctx) {
+    ErlBinMatchBuffer *_mb;
+    _mb = (ErlBinMatchBuffer*) ms_matchbuffer($Ctx);
+    if (_mb->size != _mb->offset) {
+        $FAIL($Fail);
+    }
+}
+
+bs_test_tail_imm2(Fail, Ctx, Offset) {
+    ErlBinMatchBuffer *_mb;
+    _mb = ms_matchbuffer($Ctx);
+    if (_mb->size - _mb->offset != $Offset) {
+        $FAIL($Fail);
+    }
+}
+
+bs_test_unit(Fail, Ctx, Unit) {
+    ErlBinMatchBuffer *_mb;
+    _mb = ms_matchbuffer($Ctx);
+    if ((_mb->size - _mb->offset) % $Unit) {
+        $FAIL($Fail);
+    }
+}
+
+bs_test_unit8(Fail, Ctx) {
+    ErlBinMatchBuffer *_mb;
+    _mb = ms_matchbuffer($Ctx);
+    if ((_mb->size - _mb->offset) & 7) {
+        $FAIL($Fail);
+    }
+}
+
+i_bs_get_integer_8(Ctx, Fail, Dst) {
+    Eterm _result;
+    ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx);
+
+    if (_mb->size - _mb->offset < 8) {
+        $FAIL($Fail);
+    }
+    if (BIT_OFFSET(_mb->offset) != 0) {
+        _result = erts_bs_get_integer_2(c_p, 8, 0, _mb);
+    } else {
+        _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]);
+        _mb->offset += 8;
+    }
+    $Dst = _result;
+}
+
+i_bs_get_integer_16(Ctx, Fail, Dst) {
+    Eterm _result;
+    ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx);
+
+    if (_mb->size - _mb->offset < 16) {
+        $FAIL($Fail);
+    }
+    if (BIT_OFFSET(_mb->offset) != 0) {
+        _result = erts_bs_get_integer_2(c_p, 16, 0, _mb);
+    } else {
+        _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset)));
+        _mb->offset += 16;
+    }
+    $Dst = _result;
+}
+
+%if ARCH_64
+i_bs_get_integer_32(Ctx, Fail, Dst) {
+    Uint32 _integer;
+    ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx);
+
+    if (_mb->size - _mb->offset < 32) {
+        $FAIL($Fail);
+    }
+    if (BIT_OFFSET(_mb->offset) != 0) {
+        _integer = erts_bs_get_unaligned_uint32(_mb);
+    } else {
+        _integer = get_int32(_mb->base + _mb->offset/8);
     }
+    _mb->offset += 32;
+    $Dst = make_small(_integer);
+}
+%endif
+
+i_bs_get_integer_imm := bs_get_integer.fetch.execute;
+i_bs_get_integer_small_imm := bs_get_integer.fetch_small.execute;
+
+bs_get_integer.head() {
+    Eterm Ms, Sz;
+}
+
+bs_get_integer.fetch(Ctx, Size, Live) {
+    Uint wordsneeded;
+    Ms = $Ctx;
+    Sz = $Size;
+    wordsneeded = 1+WSIZE(NBYTES(Sz));
+    $GC_TEST_PRESERVE(wordsneeded, $Live, Ms);
+}
+
+bs_get_integer.fetch_small(Ctx, Size) {
+    Ms = $Ctx;
+    Sz = $Size;
+}
+
+bs_get_integer.execute(Fail, Flags, Dst) {
+    ErlBinMatchBuffer* mb;
+    Eterm result;
+
+    mb = ms_matchbuffer(Ms);
+    LIGHT_SWAPOUT;
+    result = erts_bs_get_integer_2(c_p, Sz, $Flags, mb);
+    LIGHT_SWAPIN;
+    HEAP_SPACE_VERIFIED(0);
+    if (is_non_value(result)) {
+        $FAIL($Fail);
+    }
+    $Dst = result;
+}
+
+i_bs_get_integer(Fail, Live, FlagsAndUnit, Ms, Sz, Dst) {
+    Uint flags;
+    Uint size;
+    Eterm ms;
+    ErlBinMatchBuffer* mb;
+    Eterm result;
+
+    flags = $FlagsAndUnit;
+    ms = $Ms;
+    $BS_GET_FIELD_SIZE($Sz, (flags >> 3), $FAIL($Fail), size);
+    if (size >= SMALL_BITS) {
+        Uint wordsneeded;
+        /* Check bits size before potential gc.
+         * We do not want a gc and then realize we don't need
+         * the allocated space (i.e. if the op fails).
+         *
+         * Remember to re-acquire the matchbuffer after gc.
+         */
+
+        mb = ms_matchbuffer(ms);
+        if (mb->size - mb->offset < size) {
+            $FAIL($Fail);
+        }
+        wordsneeded = 1+WSIZE(NBYTES((Uint) size));
+        $GC_TEST_PRESERVE(wordsneeded, $Live, ms);
+    }
+    mb = ms_matchbuffer(ms);
+    LIGHT_SWAPOUT;
+    result = erts_bs_get_integer_2(c_p, size, flags, mb);
+    LIGHT_SWAPIN;
+    HEAP_SPACE_VERIFIED(0);
+    if (is_non_value(result)) {
+        $FAIL($Fail);
+    }
+    $Dst = result;
+}
+
+i_bs_get_utf8(Ctx, Fail, Dst) {
+    ErlBinMatchBuffer* mb = ms_matchbuffer($Ctx);
+    Eterm result = erts_bs_get_utf8(mb);
+
+    if (is_non_value(result)) {
+        $FAIL($Fail);
+    }
+    $Dst = result;
+}
+
+i_bs_get_utf16(Ctx, Fail, Flags, Dst) {
+    ErlBinMatchBuffer* mb = ms_matchbuffer($Ctx);
+    Eterm result = erts_bs_get_utf16(mb, $Flags);
+
+    if (is_non_value(result)) {
+        $FAIL($Fail);
+    }
+    $Dst = result;
+}
+
+bs_context_to_binary := ctx_to_bin.fetch.execute;
+i_bs_get_binary_all_reuse := ctx_to_bin.fetch_bin.execute;
+
+ctx_to_bin.head() {
+     Eterm context;
+     ErlBinMatchBuffer* mb;
+     ErlSubBin* sb;
+     Uint size;
+     Uint offs;
+     Uint orig;
+     Uint hole_size;
+}
+
+ctx_to_bin.fetch(Src) {
+    context = $Src;
+    if (is_boxed(context) &&
+        header_is_bin_matchstate(*boxed_val(context))) {
+        ErlBinMatchState* ms;
+        ms = (ErlBinMatchState *) boxed_val(context);
+        mb = &ms->mb;
+        offs = ms->save_offset[0];
+        size = mb->size - offs;
+    } else {
+        $NEXT0();
+    }
+}
+
+ctx_to_bin.fetch_bin(Src, Fail, Unit) {
+    context = $Src;
+    mb = ms_matchbuffer(context);
+    size = mb->size - mb->offset;
+    if (size % $Unit != 0) {
+        $FAIL($Fail);
+    }
+    offs = mb->offset;
+}
+
+ctx_to_bin.execute() {
+    orig = mb->orig;
+    sb = (ErlSubBin *) boxed_val(context);
+    hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE;
+    sb->thing_word = HEADER_SUB_BIN;
+    sb->size = BYTE_OFFSET(size);
+    sb->bitsize = BIT_OFFSET(size);
+    sb->offs = BYTE_OFFSET(offs);
+    sb->bitoffs = BIT_OFFSET(offs);
+    sb->is_writable = 0;
+    sb->orig = orig;
+    if (hole_size) {
+        sb[1].thing_word = make_pos_bignum_header(hole_size-1);
+    }
+}
+
+i_bs_match_string(Ctx, Fail, Bits, Ptr) {
+    byte* bytes = (byte *) $Ptr;
+    Uint bits = $Bits;
+    ErlBinMatchBuffer* mb;
+    Uint offs;
+
+    mb = ms_matchbuffer($Ctx);
+    if (mb->size - mb->offset < bits) {
+        $FAIL($Fail);
+    }
+    offs = mb->offset & 7;
+    if (offs == 0 && (bits & 7) == 0) {
+        if (sys_memcmp(bytes, mb->base+(mb->offset>>3), bits>>3)) {
+            $FAIL($Fail);
+        }
+    } else if (erts_cmp_bits(bytes, 0, mb->base+(mb->offset>>3), mb->offset & 7, bits)) {
+        $FAIL($Fail);
+    }
+    mb->offset += bits;
+}
+
+i_bs_save2(Src, Slot) {
+    ErlBinMatchState* _ms = (ErlBinMatchState*) boxed_val((Eterm) $Src);
+    _ms->save_offset[$Slot] = _ms->mb.offset;
+}
+
+i_bs_restore2(Src, Slot) {
+    ErlBinMatchState* _ms = (ErlBinMatchState*) boxed_val((Eterm) $Src);
+    _ms->mb.offset = _ms->save_offset[$Slot];
 }
diff --git a/erts/emulator/beam/float_instrs.tab b/erts/emulator/beam/float_instrs.tab
new file mode 100644
index 0000000000..3d4db77892
--- /dev/null
+++ b/erts/emulator/beam/float_instrs.tab
@@ -0,0 +1,88 @@
+// -*- c -*-
+//
+// %CopyrightBegin%
+//
+// Copyright Ericsson AB 2017. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// %CopyrightEnd%
+//
+
+LOAD_DOUBLE(Src, Dst) {
+    GET_DOUBLE($Src, *(FloatDef *) &$Dst);
+}
+
+fload(Reg, Dst) {
+    $LOAD_DOUBLE($Reg, $Dst);
+}
+
+fstore(Float, Dst) {
+    PUT_DOUBLE(*((FloatDef *) &$Float), HTOP);
+    $Dst = make_float(HTOP);
+    HTOP += FLOAT_SIZE_OBJECT;
+}
+
+fconv(Src, Dst) {
+    Eterm src = $Src;
+
+    if (is_small(src)) {
+        $Dst = (double) signed_val(src);
+    } else if (is_big(src)) {
+        if (big_to_double(src, &$Dst) < 0) {
+            $BADARITH0();
+        }
+    } else if (is_float(src)) {
+        $LOAD_DOUBLE(src, $Dst);
+    } else {
+        $BADARITH0();
+    }
+}
+
+FLOAT_OP(Src1, OP, Src2, Dst) {
+    ERTS_NO_FPE_CHECK_INIT(c_p);
+    $Dst = $Src1 $OP $Src2;
+    ERTS_NO_FPE_ERROR(c_p, $Dst, $BADARITH0());
+}
+
+i_fadd(Src1, Src2, Dst) {
+    $FLOAT_OP($Src1, +, $Src2, $Dst);
+}
+
+i_fsub(Src1, Src2, Dst) {
+    $FLOAT_OP($Src1, -, $Src2, $Dst);
+}
+
+i_fmul(Src1, Src2, Dst) {
+    $FLOAT_OP($Src1, *, $Src2, $Dst);
+}
+
+i_fdiv(Src1, Src2, Dst) {
+    $FLOAT_OP($Src1, /, $Src2, $Dst);
+}
+
+i_fnegate(Src, Dst) {
+    ERTS_NO_FPE_CHECK_INIT(c_p);
+    $Dst = -$Src;
+    ERTS_NO_FPE_ERROR(c_p, $Dst, $BADARITH0());
+}
+
+%unless NO_FPE_SIGNALS
+fclearerror() {
+    ERTS_FP_CHECK_INIT(c_p);
+}
+
+i_fcheckerror() {
+    ERTS_FP_ERROR(c_p, freg[0].fd, $BADARITH0());
+}
+%endif
diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index ca0cb2f63d..d45da62d03 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -19,71 +19,86 @@
 // %CopyrightEnd%
 //
 
-// Macros only used to generate instructions.
+// Stack manipulation instructions
 
-FAIL(Fail) {
-    //| -no_prefetch
-    SET_I((BeamInstr *) $Fail);
-    Goto(*I);
+allocate(NeedStack, Live) {
+    $AH($NeedStack, 0, $Live);
 }
 
-JUMP(Fail) {
-    //| -no_next
-    SET_I((BeamInstr *) $Fail);
-    Goto(*I);
+allocate_heap(NeedStack, NeedHeap, Live) {
+    $AH($NeedStack, $NeedHeap, $Live);
 }
 
-GC_TEST(Ns, Nh, Live) {
-    unsigned need = $Nh + $Ns;
-    if (E - HTOP < need) {
-       SWAPOUT;
-       PROCESS_MAIN_CHK_LOCKS(c_p);
-       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, $Live, FCALLS);
-       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-       PROCESS_MAIN_CHK_LOCKS(c_p);
-       SWAPIN;
+allocate_init(NeedStack, Live, Y) {
+    $AH($NeedStack, 0, $Live);
+    make_blank($Y);
+}
+
+allocate_zero(NeedStack, Live) {
+    Eterm* ptr;
+    int i = $NeedStack;
+    $AH(i, 0, $Live);
+    for (ptr = E + i; ptr > E; ptr--) {
+	 make_blank(*ptr);
     }
-    HEAP_SPACE_VERIFIED($Nh);
 }
 
-// Make sure that there are NeedStack + NeedHeap + 1 words available
-// on the combined heap/stack segment, then allocates NeedHeap + 1
-// words on the stack and saves CP.
-AH(NeedStack, NeedHeap, Live) {
-    unsigned needed = $NeedStack + 1;
-    $GC_TEST(needed, $NeedHeap, $Live);
-    E -= needed;
-    SAVE_CP(E);
+allocate_heap_zero(NeedStack, NeedHeap, Live) {
+    Eterm* ptr;
+    int i = $NeedStack;
+    $AH(i, $NeedHeap, $Live);
+    for (ptr = E + i; ptr > E; ptr--) {
+        make_blank(*ptr);
+    }
 }
 
-// Start of instruction listings
+// This instruction is probably never used (because it is combined with a
+// a return). However, a future compiler might for some reason emit a
+// deallocate not followed by a return, and that should work.
+
+deallocate(Deallocate) {
+    //| -no_prefetch
+    SET_CP(c_p, (BeamInstr *) cp_val(*E));
+    E = ADD_BYTE_OFFSET(E, $Deallocate);
+}
+
+deallocate_return(Deallocate) {
+    //| -no_next
+    int words_to_pop = $Deallocate;
+    SET_I((BeamInstr *) cp_val(*E));
+    E = ADD_BYTE_OFFSET(E, words_to_pop);
+    CHECK_TERM(x(0));
+    DispatchReturn;
+}
+
+move_deallocate_return(Src, Deallocate) {
+    x(0) = $Src;
+    $deallocate_return($Deallocate);
+}
 
 // Call instructions
 
-DO_CALL(CallDest, NextInstr) {
+DISPATCH(CallDest) {
     //| -no_next
-    SET_CP(c_p, $NextInstr);
     SET_I((BeamInstr *) $CallDest);
     DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
     Dispatch();
 }
 
 i_call(CallDest) {
-    $DO_CALL($CallDest, $NEXT_INSTRUCTION);
+    SET_CP(c_p, $NEXT_INSTRUCTION);
+    $DISPATCH($CallDest);
 }
 
 move_call(Src, CallDest) {
     x(0) = $Src;
-    $DO_CALL($CallDest, $NEXT_INSTRUCTION);
+    SET_CP(c_p, $NEXT_INSTRUCTION);
+    $DISPATCH($CallDest);
 }
 
 i_call_last(CallDest, Deallocate) {
-    //| -no_next
-    RESTORE_CP(E);
-    E = ADD_BYTE_OFFSET(E, ($Deallocate));
-    SET_I((BeamInstr *) $CallDest);
-    DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
-    Dispatch();
+    $deallocate($Deallocate);
+    $DISPATCH($CallDest);
 }
 
 move_call_last(Src, CallDest, Deallocate) {
@@ -92,66 +107,202 @@ move_call_last(Src, CallDest, Deallocate) {
 }
 
 i_call_only(CallDest) {
-    //| -no_next
-    SET_I((BeamInstr *) $CallDest);
-    DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
-    Dispatch();
+    $DISPATCH($CallDest);
 }
 
-move_call_only(Src, CallDest) {
+i_move_call_only(CallDest, Src) {
     x(0) = $Src;
     $i_call_only($CallDest);
 }
 
-// Other instructions
+move_call_only(Src, CallDest) {
+    $i_move_call_only($CallDest, $Src);
+}
 
-allocate(NeedStack, Live) {
-    $AH($NeedStack, 0, $Live);
+DISPATCHX(Dest) {
+    //| -no_next
+    DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, $Dest);
+    // Dispatchx assumes the Export* is in Arg(0)
+    I = (&$Dest) - 1;
+    Dispatchx();
 }
 
-allocate_heap(NeedStack, NeedHeap, Live) {
-    $AH($NeedStack, $NeedHeap, $Live);
+i_call_ext(Dest) {
+    SET_CP(c_p, $NEXT_INSTRUCTION);
+    $DISPATCHX($Dest);
 }
 
-allocate_init(NeedStack, Live, Y) {
-    $AH($NeedStack, 0, $Live);
-    make_blank($Y);
+i_move_call_ext(Src, Dest) {
+    x(0) = $Src;
+    $i_call_ext($Dest);
 }
 
-allocate_zero(NeedStack, Live) {
-    Eterm* ptr;
-    int i = $NeedStack;
-    $AH(i, 0, $Live);
-    for (ptr = E + i; ptr > E; ptr--) {
-	 make_blank(*ptr);
+i_call_ext_only(Dest) {
+    $DISPATCHX($Dest);
+}
+
+i_move_call_ext_only(Dest, Src) {
+    x(0) = $Src;
+    $i_call_ext_only($Dest);
+}
+
+i_call_ext_last(Dest, Deallocate) {
+    $deallocate($Deallocate);
+    $DISPATCHX($Dest);
+}
+
+i_move_call_ext_last(Dest, StackOffset, Src) {
+    x(0) = $Src;
+    $i_call_ext_last($Dest, $StackOffset);
+}
+
+APPLY(I, Deallocate) {
+    //| -no_next
+    HEAVY_SWAPOUT;
+    next = apply(c_p, r(0), x(1), x(2), reg, $I, $Deallocate);
+    HEAVY_SWAPIN;
+}
+
+HANDLE_APPLY_ERROR() {
+    I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa);
+    goto post_error_handling;
+}
+
+i_apply() {
+    BeamInstr *next;
+    $APPLY(NULL, 0);
+    if (next != NULL) {
+        $i_call(next);
     }
+    $HANDLE_APPLY_ERROR();
 }
 
-allocate_heap_zero(NeedStack, NeedHeap, Live) {
-    Eterm* ptr;
-    int i = $NeedStack;
-    $AH(i, $NeedHeap, $Live);
-    for (ptr = E + i; ptr > E; ptr--) {
-        make_blank(*ptr);
+i_apply_last(Deallocate) {
+    BeamInstr *next;
+    $APPLY(I, $Deallocate);
+    if (next != NULL) {
+        $i_call_last(next, $Deallocate);
     }
+    $HANDLE_APPLY_ERROR();
 }
 
-// This instruction is probably never used (because it is combined with a
-// a return). However, a future compiler might for some reason emit a
-// deallocate not followed by a return, and that should work.
+i_apply_only() {
+    BeamInstr *next;
+    $APPLY(I, 0);
+    if (next != NULL) {
+        $i_call_only(next);
+    }
+    $HANDLE_APPLY_ERROR();
+}
 
-deallocate(Deallocate) {
-    //| -no_prefetch
-    RESTORE_CP(E);
-    E = ADD_BYTE_OFFSET(E, $Deallocate);
+FIXED_APPLY(Arity, I, Deallocate) {
+    //| -no_next
+    HEAVY_SWAPOUT;
+    next = fixed_apply(c_p, reg, $Arity, $I, $Deallocate);
+    HEAVY_SWAPIN;
 }
 
-deallocate_return(Deallocate) {
+apply(Arity) {
+    BeamInstr *next;
+    $FIXED_APPLY($Arity, NULL, 0);
+    if (next != NULL) {
+        $i_call(next);
+    }
+    $HANDLE_APPLY_ERROR();
+}
+
+apply_last(Arity, Deallocate) {
+    BeamInstr *next;
+    $FIXED_APPLY($Arity, I, $Deallocate);
+    if (next != NULL) {
+        $i_call_last(next, $Deallocate);
+    }
+    $HANDLE_APPLY_ERROR();
+}
+
+APPLY_FUN() {
+    HEAVY_SWAPOUT;
+    next = apply_fun(c_p, r(0), x(1), reg);
+    HEAVY_SWAPIN;
+}
+
+HANDLE_APPLY_FUN_ERROR() {
+     goto find_func_info;
+}
+
+DISPATCH_FUN(I) {
+    SET_I($I);
+    Dispatchfun();
+}
+
+i_apply_fun() {
+    BeamInstr *next;
+    $APPLY_FUN();
+    if (next != NULL) {
+        SET_CP(c_p, $NEXT_INSTRUCTION);
+        $DISPATCH_FUN(next);
+    }
+    $HANDLE_APPLY_FUN_ERROR();
+}
+
+i_apply_fun_last(Deallocate) {
+    BeamInstr *next;
+    $APPLY_FUN();
+    if (next != NULL) {
+        $deallocate($Deallocate);
+        $DISPATCH_FUN(next);
+    }
+    $HANDLE_APPLY_FUN_ERROR();
+}
+
+i_apply_fun_only() {
+    BeamInstr *next;
+    $APPLY_FUN();
+    if (next != NULL) {
+        $DISPATCH_FUN(next);
+    }
+    $HANDLE_APPLY_FUN_ERROR();
+}
+
+CALL_FUN(Fun) {
     //| -no_next
-    int words_to_pop = $Deallocate;
-    SET_I((BeamInstr *) cp_val(*E));
-    E = ADD_BYTE_OFFSET(E, words_to_pop);
-    CHECK_TERM(x(0));
+    HEAVY_SWAPOUT;
+    next = call_fun(c_p, $Fun, reg, THE_NON_VALUE);
+    HEAVY_SWAPIN;
+}
+
+i_call_fun(Fun) {
+    BeamInstr *next;
+    $CALL_FUN($Fun);
+    if (next != NULL) {
+        SET_CP(c_p, $NEXT_INSTRUCTION);
+        $DISPATCH_FUN(next);
+    }
+    $HANDLE_APPLY_FUN_ERROR();
+}
+
+i_call_fun_last(Fun, Deallocate) {
+    BeamInstr *next;
+    $CALL_FUN($Fun);
+    if (next != NULL) {
+        $deallocate($Deallocate);
+        $DISPATCH_FUN(next);
+    }
+    $HANDLE_APPLY_FUN_ERROR();
+}
+
+return() {
+    SET_I(c_p->cp);
+    DTRACE_RETURN_FROM_PC(c_p);
+
+    /*
+     * We must clear the CP to make sure that a stale value do not
+     * create a false module dependcy preventing code upgrading.
+     * It also means that we can use the CP in stack backtraces.
+     */
+    c_p->cp = 0;
+    CHECK_TERM(r(0));
+    HEAP_SPACE_VERIFIED(0);
     DispatchReturn;
 }
 
@@ -213,6 +364,56 @@ i_get_tuple_element3(Src, Element, Dst) {
     dst[2] = E3;
 }
 
+i_element := element_group.fetch.execute;
+
+
+element_group.head() {
+    Eterm element_index;
+    Eterm element_tuple;
+}
+
+element_group.fetch(Src) {
+    element_tuple = $Src;
+}
+
+element_group.execute(Fail, Index, Dst) {
+    element_index = $Index;
+    if (is_small(element_index) && is_tuple(element_tuple)) {
+        Eterm* tp = tuple_val(element_tuple);
+
+        if ((signed_val(element_index) >= 1) &&
+            (signed_val(element_index) <= arityval(*tp))) {
+            $Dst = tp[signed_val(element_index)];
+            $NEXT0();
+        }
+    }
+    c_p->freason = BADARG;
+    $BIF_ERROR_ARITY_2($Fail, BIF_element_2, element_index, element_tuple);
+}
+
+i_fast_element := fast_element_group.fetch.execute;
+
+fast_element_group.head() {
+    Eterm fast_element_tuple;
+}
+
+fast_element_group.fetch(Src) {
+    fast_element_tuple = $Src;
+}
+
+fast_element_group.execute(Fail, Index, Dst) {
+    if (is_tuple(fast_element_tuple)) {
+        Eterm* tp = tuple_val(fast_element_tuple);
+        Eterm pos = $Index;	/* Untagged integer >= 1 */
+        if (pos <= arityval(*tp)) {
+            $Dst = tp[pos];
+            $NEXT0();
+	}
+    }
+    c_p->freason = BADARG;
+    $BIF_ERROR_ARITY_2($Fail, BIF_element_2, make_small($Index), fast_element_tuple);
+}
+
 init(Y) {
     make_blank($Y);
 }
@@ -250,12 +451,6 @@ move3(S1, D1, S2, D2, S3, D3) {
     $D3 = $S3;
 }
 
-move_deallocate_return(Src, Deallocate) {
-    //| -no_next
-    x(0) = $Src;
-    $deallocate_return($Deallocate);
-}
-
 move_dup(Src, D1, D2) {
     $D1 = $D2 = $Src;
 }
@@ -341,12 +536,35 @@ put_list(Hd, Tl, Dst) {
     HTOP += 2;
 }
 
-i_put_tuple(Dst, Arity) {
-    //| -no_next
+i_put_tuple := i_put_tuple.make.fill;
+
+i_put_tuple.make(Dst) {
     $Dst = make_tuple(HTOP);
-    pt_arity = $Arity;
+}
+
+i_put_tuple.fill(Arity) {
+    Eterm* hp = HTOP;
+    Eterm arity = $Arity;
+
+    //| -no_next
+    *hp++ = make_arityval(arity);
     I = $NEXT_INSTRUCTION;
-    goto do_put_tuple;
+    do {
+        Eterm term = *I++;
+        switch (loader_tag(term)) {
+        case LOADER_X_REG:
+            *hp++ = x(loader_x_reg_index(term));
+            break;
+        case LOADER_Y_REG:
+            *hp++ = y(loader_y_reg_index(term));
+            break;
+        default:
+            *hp++ = term;
+            break;
+        }
+    } while (--arity != 0);
+    HTOP = hp;
+    Goto(*I);
 }
 
 self(Dst) {
@@ -413,9 +631,13 @@ is_nonempty_list_get_list(Fail, Src, Hd, Tl) {
     $get_list($Src, $Hd, $Tl);
 }
 
+jump(Fail) {
+    $JUMP($Fail);
+}
+
 move_jump(Fail, Src) {
     x(0) = $Src;
-    $JUMP($Fail);
+    $jump($Fail);
 }
 
 //
@@ -537,6 +759,7 @@ test_arity(Fail, Pointer, Arity) {
         $FAIL($Fail);
     }
 }
+
 i_is_eq_exact_immed(Fail, X, Y) {
     if ($X != $Y) {
         $FAIL($Fail);
@@ -555,12 +778,24 @@ is_eq_exact(Fail, X, Y) {
     }
 }
 
+i_is_eq_exact_literal(Fail, Src, Literal) {
+    if (!eq($Src, $Literal)) {
+        $FAIL($Fail);
+    }
+}
+
 is_ne_exact(Fail, X, Y) {
     if (EQ($X, $Y)) {
         $FAIL($Fail);
     }
 }
 
+i_is_ne_exact_literal(Fail, Src, Literal) {
+    if (eq($Src, $Literal)) {
+        $FAIL($Fail);
+    }
+}
+
 is_eq(Fail, X, Y) {
     CMP_EQ_ACTION($X, $Y, $FAIL($Fail));
 }
@@ -577,18 +812,98 @@ is_ge(Fail, X, Y) {
     CMP_GE_ACTION($X, $Y, $FAIL($Fail));
 }
 
-i_get_map_element(Fail, Src, Key, Dst) {
-    Eterm res = get_map_element($Src, $Key);
-    if (is_non_value(res)) {
-        $FAIL($Fail);
+badarg(Fail) {
+    $BADARG($Fail);
+}
+
+badmatch(Src) {
+    c_p->fvalue = $Src;
+    c_p->freason = BADMATCH;
+    goto find_func_info;
+}
+
+case_end(Src) {
+    c_p->fvalue = $Src;
+    c_p->freason = EXC_CASE_CLAUSE;
+    goto find_func_info;
+}
+
+if_end() {
+    c_p->freason = EXC_IF_CLAUSE;
+    goto find_func_info;
+    //| -no_next;
+}
+
+system_limit(Fail) {
+    $SYSTEM_LIMIT($Fail);
+    //| -no_next;
+}
+
+catch(Y, Fail) {
+    c_p->catches++;
+    $Y = $Fail;
+}
+
+catch_end(Y) {
+    c_p->catches--;
+    make_blank($Y);
+    if (is_non_value(r(0))) {
+        c_p->fvalue = NIL;
+        if (x(1) == am_throw) {
+            r(0) = x(2);
+        } else {
+            if (x(1) == am_error) {
+                SWAPOUT;
+                x(2) = add_stacktrace(c_p, x(2), x(3));
+                SWAPIN;
+            }
+            /* only x(2) is included in the rootset here */
+            if (E - HTOP < 3) {
+                SWAPOUT;
+                PROCESS_MAIN_CHK_LOCKS(c_p);
+                FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1, FCALLS);
+                ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+                PROCESS_MAIN_CHK_LOCKS(c_p);
+                SWAPIN;
+            }
+            r(0) = TUPLE2(HTOP, am_EXIT, x(2));
+            HTOP += 3;
+        }
     }
-    $Dst = res;
+    CHECK_TERM(r(0));
 }
 
-i_get_map_element_hash(Fail, Src, Key, Hx, Dst) {
-     Eterm res = get_map_element_hash($Src, $Key, $Hx);
-     if (is_non_value(res)) {
-        $FAIL($Fail);
-     }
-     $Dst = res;
+try_end(Y) {
+    c_p->catches--;
+    make_blank($Y);
+    if (is_non_value(r(0))) {
+        c_p->fvalue = NIL;
+        r(0) = x(1);
+        x(1) = x(2);
+        x(2) = x(3);
+    }
 }
+
+try_case_end(Src) {
+    c_p->fvalue = $Src;
+    c_p->freason = EXC_TRY_CLAUSE;
+    goto find_func_info;
+    //| -no_next;
+}
+
+i_raise() {
+    Eterm raise_trace = x(2);
+    Eterm raise_value = x(1);
+    struct StackTrace *s;
+
+    c_p->fvalue = raise_value;
+    c_p->ftrace = raise_trace;
+    s = get_trace_from_exc(raise_trace);
+    if (s == NULL) {
+        c_p->freason = EXC_ERROR;
+    } else {
+        c_p->freason = PRIMARY_EXCEPTION(s->freason);
+    }
+    goto find_func_info;
+}
+
diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab
new file mode 100644
index 0000000000..57015fac31
--- /dev/null
+++ b/erts/emulator/beam/macros.tab
@@ -0,0 +1,139 @@
+// -*- c -*-
+//
+// %CopyrightBegin%
+//
+// Copyright Ericsson AB 2017. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// %CopyrightEnd%
+//
+
+//
+// Use if there is a garbage collection before storing to a
+// general destination (either X or Y register).
+//
+
+REFRESH_GEN_DEST() {
+    dst_ptr = REG_TARGET_PTR(dst);
+}
+
+FAIL(Fail) {
+    //| -no_prefetch
+    SET_I((BeamInstr *) $Fail);
+    Goto(*I);
+}
+
+JUMP(Fail) {
+    //| -no_next
+    SET_I((BeamInstr *) $Fail);
+    Goto(*I);
+}
+
+GC_TEST(Ns, Nh, Live) {
+    Uint need = $Nh + $Ns;
+    if (E - HTOP < need) {
+       SWAPOUT;
+       PROCESS_MAIN_CHK_LOCKS(c_p);
+       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, $Live, FCALLS);
+       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+       PROCESS_MAIN_CHK_LOCKS(c_p);
+       SWAPIN;
+    }
+    HEAP_SPACE_VERIFIED($Nh);
+}
+
+GC_TEST_PRESERVE(NeedHeap, Live, PreserveTerm) {
+    Uint need = $NeedHeap;
+    if (E - HTOP < need) {
+       SWAPOUT;
+       reg[$Live] = $PreserveTerm;
+       PROCESS_MAIN_CHK_LOCKS(c_p);
+       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, $Live+1, FCALLS);
+       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+       PROCESS_MAIN_CHK_LOCKS(c_p);
+       $PreserveTerm = reg[$Live];
+       SWAPIN;
+    }
+    HEAP_SPACE_VERIFIED($Nh);
+}
+
+
+// Make sure that there are NeedStack + NeedHeap + 1 words available
+// on the combined heap/stack segment, then allocates NeedHeap + 1
+// words on the stack and saves CP.
+AH(NeedStack, NeedHeap, Live) {
+    unsigned needed = $NeedStack + 1;
+    $GC_TEST(needed, $NeedHeap, $Live);
+    E -= needed;
+    *E = make_cp(c_p->cp);
+    c_p->cp = 0;
+}
+
+NEXT0() {
+    //| -no_next
+    SET_I((BeamInstr *) $NEXT_INSTRUCTION);
+    Goto(*I);
+}
+
+NEXT(Addr) {
+    //| -no_next
+    SET_I((BeamInstr *) $Addr);
+    Goto(*I);
+}
+
+FAIL_HEAD_OR_BODY(Fail) {
+    //| -no_prefetch
+    if ($Fail) {
+        $FAIL($Fail);
+    }
+    goto find_func_info;
+}
+
+BADARG(Fail) {
+    c_p->freason = BADARG;
+    $FAIL_HEAD_OR_BODY($Fail);
+}
+
+BADARITH0() {
+    c_p->freason = BADARITH;
+    goto find_func_info;
+}
+
+SYSTEM_LIMIT(Fail) {
+    c_p->freason = SYSTEM_LIMIT;
+    $FAIL_HEAD_OR_BODY($Fail);
+}
+
+BIF_ERROR_ARITY_1(Fail, BIF, Op1) {
+    //| -no_prefetch
+    if ($Fail) {
+        $FAIL($Fail);
+    }
+    reg[0] = $Op1;
+    SWAPOUT;
+    I = handle_error(c_p, I, reg, &bif_export[$BIF]->info.mfa);
+    goto post_error_handling;
+}
+
+BIF_ERROR_ARITY_2(Fail, BIF, Op1, Op2) {
+    //| -no_prefetch
+    if ($Fail) {
+        $FAIL($Fail);
+    }
+    reg[0] = $Op1;
+    reg[1] = $Op2;
+    SWAPOUT;
+    I = handle_error(c_p, I, reg, &bif_export[$BIF]->info.mfa);
+    goto post_error_handling;
+}
diff --git a/erts/emulator/beam/map_instrs.tab b/erts/emulator/beam/map_instrs.tab
new file mode 100644
index 0000000000..7f9346d029
--- /dev/null
+++ b/erts/emulator/beam/map_instrs.tab
@@ -0,0 +1,162 @@
+// -*- c -*-
+//
+// %CopyrightBegin%
+//
+// Copyright Ericsson AB 2017. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// %CopyrightEnd%
+//
+
+BADMAP(Fail, Map) {
+    c_p->freason = BADMAP;
+    c_p->fvalue = $Map;
+    $FAIL_HEAD_OR_BODY($Fail);
+}
+
+new_map(Dst, Live, N) {
+    Eterm res;
+
+    HEAVY_SWAPOUT;
+    res = new_map(c_p, reg, I-1);
+    HEAVY_SWAPIN;
+    $REFRESH_GEN_DEST();
+    $Dst = res;
+    $NEXT($NEXT_INSTRUCTION+$N);
+}
+
+i_new_small_map_lit(Dst, Live, Literal) {
+    Eterm res;
+    Uint n;
+
+    HEAVY_SWAPOUT;
+    res = new_small_map_lit(c_p, reg, &n, I-1);
+    HEAVY_SWAPIN;
+    $REFRESH_GEN_DEST();
+    $Dst = res;
+    $NEXT($NEXT_INSTRUCTION+n);
+}
+
+i_get_map_element(Fail, Src, Key, Dst) {
+    Eterm res = get_map_element($Src, $Key);
+    if (is_non_value(res)) {
+        $FAIL($Fail);
+    }
+    $Dst = res;
+}
+
+i_get_map_element_hash(Fail, Src, Key, Hx, Dst) {
+     Eterm res = get_map_element_hash($Src, $Key, $Hx);
+     if (is_non_value(res)) {
+        $FAIL($Fail);
+     }
+     $Dst = res;
+}
+
+i_get_map_elements(Fail, Src, N) {
+    Eterm map;
+    BeamInstr *fs;
+    Uint sz, n;
+
+    map = $Src;
+
+    /* This instruction assumes Arg1 is a map,
+     * i.e. that it follows a test is_map if needed.
+     */
+
+    n  = (Uint)$N / 3;
+    fs = $NEXT_INSTRUCTION;
+
+    if (is_flatmap(map)) {
+	flatmap_t *mp;
+	Eterm *ks;
+	Eterm *vs;
+
+	mp = (flatmap_t *)flatmap_val(map);
+	sz = flatmap_get_size(mp);
+
+	if (sz == 0) {
+            $FAIL($Fail);
+	}
+
+	ks = flatmap_get_keys(mp);
+	vs = flatmap_get_values(mp);
+
+	while(sz) {
+	    if (EQ((Eterm) fs[0], *ks)) {
+		PUT_TERM_REG(*vs, fs[1]);
+		n--;
+		fs += 3;
+		/* no more values to fetch, we are done */
+		if (n == 0) {
+                    $NEXT(fs);
+		}
+	    }
+	    ks++, sz--, vs++;
+	}
+        $FAIL($Fail);
+    } else {
+	const Eterm *v;
+	Uint32 hx;
+	ASSERT(is_hashmap(map));
+	while(n--) {
+	    hx = fs[2];
+	    ASSERT(hx == hashmap_make_hash((Eterm)fs[0]));
+	    if ((v = erts_hashmap_get(hx, (Eterm)fs[0], map)) == NULL) {
+		$FAIL($Fail);
+	    }
+	    PUT_TERM_REG(*v, fs[1]);
+	    fs += 3;
+	}
+        $NEXT(fs);
+    }
+}
+
+update_map_assoc(Fail, Src, Dst, Live, N) {
+    Eterm res;
+    Eterm map;
+
+    map = $Src;
+    HEAVY_SWAPOUT;
+    res = update_map_assoc(c_p, reg, map, I);
+    HEAVY_SWAPIN;
+    if (is_value(res)) {
+        $REFRESH_GEN_DEST();
+        $Dst = res;
+        $NEXT($NEXT_INSTRUCTION+$N);
+    } else {
+        /*
+         * This can only happen if the code was compiled
+         * with the compiler in OTP 17.
+         */
+        $BADMAP($Fail, map);
+    }
+}
+
+update_map_exact(Fail, Src, Dst, Live, N) {
+    Eterm res;
+    Eterm map;
+
+    map = $Src;
+    HEAVY_SWAPOUT;
+    res = update_map_exact(c_p, reg, map, I);
+    HEAVY_SWAPIN;
+    if (is_value(res)) {
+        $REFRESH_GEN_DEST();
+        $Dst = res;
+        $NEXT($NEXT_INSTRUCTION+$N);
+    } else {
+        $FAIL_HEAD_OR_BODY($Fail);
+    }
+}
diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab
new file mode 100644
index 0000000000..509143268b
--- /dev/null
+++ b/erts/emulator/beam/msg_instrs.tab
@@ -0,0 +1,382 @@
+// -*- c -*-
+//
+// %CopyrightBegin%
+//
+// Copyright Ericsson AB 2017. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// %CopyrightEnd%
+//
+
+// /*
+//  * Skeleton for receive statement:
+//  *
+//  *             recv_mark L1                     Optional
+//  *             call make_ref/monitor            Optional
+//  *             ...
+//  *             recv_set L1                      Optional
+//  *      L1:          <-------------------+
+//  *                   <-----------+       |
+//  *                               |       |
+//  *             loop_rec L2 ------+---+   |
+//  *             ...               |   |   |
+//  *             remove_message    |   |   |
+//  *             jump L3           |   |   |
+//  *		...                 |   |   |
+//  *		loop_rec_end L1   --+   |   |
+//  *      L2:          <---------------+   |
+//  *	   	wait L1  -------------------+      or wait_timeout
+//  *		timeout
+//  *
+//  *	 L3:    Code after receive...
+//  *
+//  */
+
+recv_mark(Dest) {
+    /*
+     * Save the current position in message buffer and the
+     * the label for the loop_rec/2 instruction for the
+     * the receive statement.
+     */
+    c_p->msg.mark = (BeamInstr *) $Dest;
+    c_p->msg.saved_last = c_p->msg.last;
+}
+
+i_recv_set() {
+    /*
+     * If the mark is valid (points to the loop_rec/2
+     * instruction that follows), we know that the saved
+     * position points to the first message that could
+     * possibly be matched out.
+     *
+     * If the mark is invalid, we do nothing, meaning that
+     * we will look through all messages in the message queue.
+     */
+    if (c_p->msg.mark == (BeamInstr *) ($NEXT_INSTRUCTION)) {
+        c_p->msg.save = c_p->msg.saved_last;
+    }
+    /* Fall through to the loop_rec/2 instruction */
+}
+
+i_loop_rec(Dest) {
+    //| -no_prefetch
+
+    /*
+     * Pick up the next message and place it in x(0).
+     * If no message, jump to a wait or wait_timeout instruction.
+     */
+
+    ErtsMessage* msgp;
+
+    /*
+     * We need to disable GC while matching messages
+     * in the queue. This since messages with data outside
+     * the heap will be corrupted by a GC.
+     */
+    ASSERT(!(c_p->flags & F_DELAY_GC));
+    c_p->flags |= F_DELAY_GC;
+
+loop_rec__:
+
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+
+    msgp = PEEK_MESSAGE(c_p);
+
+    if (!msgp) {
+        erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+        /* Make sure messages wont pass exit signals... */
+        if (ERTS_PROC_PENDING_EXIT(c_p)) {
+            erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+            SWAPOUT;
+            c_p->flags &= ~F_DELAY_GC;
+            c_p->arity = 0;
+            goto do_schedule; /* Will be rescheduled for exit */
+        }
+        ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
+        msgp = PEEK_MESSAGE(c_p);
+        if (msgp) {
+            erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+        } else {
+            c_p->flags &= ~F_DELAY_GC;
+            SET_I((BeamInstr *) $Dest);
+            Goto(*I);		/* Jump to a wait or wait_timeout instruction */
+        }
+    }
+    if (is_non_value(ERL_MESSAGE_TERM(msgp))) {
+        SWAPOUT; /* erts_decode_dist_message() may write to heap... */
+        if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) {
+            /*
+             * A corrupt distribution message that we weren't able to decode;
+             * remove it...
+             */
+            /* No swapin should be needed */
+            ASSERT(HTOP == c_p->htop && E == c_p->stop);
+            /* TODO: Add DTrace probe for this bad message situation? */
+            UNLINK_MESSAGE(c_p, msgp);
+            msgp->next = NULL;
+            erts_cleanup_messages(msgp);
+            goto loop_rec__;
+        }
+        SWAPIN;
+    }
+    r(0) = ERL_MESSAGE_TERM(msgp);
+}
+
+remove_message() {
+    //| -no_prefetch
+
+    /*
+     * Remove a (matched) message from the message queue.
+     */
+
+    ErtsMessage* msgp;
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+
+    ERTS_CHK_MBUF_SZ(c_p);
+
+    msgp = PEEK_MESSAGE(c_p);
+
+    if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
+        save_calls(c_p, &exp_receive);
+    }
+    if (ERL_MESSAGE_TOKEN(msgp) == NIL) {
+#ifdef USE_VM_PROBES
+        if (DT_UTAG(c_p) != NIL) {
+            if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) {
+                SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag;
+            } else {
+                DT_UTAG(c_p) = NIL;
+                SEQ_TRACE_TOKEN(c_p) = NIL;
+            }
+        } else {
+#endif
+            SEQ_TRACE_TOKEN(c_p) = NIL;
+#ifdef USE_VM_PROBES
+        }
+        DT_UTAG_FLAGS(c_p) &= ~DT_UTAG_SPREADING;
+#endif
+    } else if (ERL_MESSAGE_TOKEN(msgp) != am_undefined) {
+        Eterm msg;
+        SEQ_TRACE_TOKEN(c_p) = ERL_MESSAGE_TOKEN(msgp);
+#ifdef USE_VM_PROBES
+        if (ERL_MESSAGE_TOKEN(msgp) == am_have_dt_utag) {
+            if (DT_UTAG(c_p) == NIL) {
+                DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp);
+            }
+            DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING;
+        } else {
+#endif
+            ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p)));
+            ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5);
+            ASSERT(is_small(SEQ_TRACE_TOKEN_SERIAL(c_p)));
+            ASSERT(is_small(SEQ_TRACE_TOKEN_LASTCNT(c_p)));
+            ASSERT(is_small(SEQ_TRACE_TOKEN_FLAGS(c_p)));
+            ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p)));
+            c_p->seq_trace_lastcnt = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p));
+            if (c_p->seq_trace_clock < unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p))) {
+                c_p->seq_trace_clock = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p));
+            }
+            msg = ERL_MESSAGE_TERM(msgp);
+            seq_trace_output(SEQ_TRACE_TOKEN(c_p), msg, SEQ_TRACE_RECEIVE, 
+                             c_p->common.id, c_p);
+#ifdef USE_VM_PROBES
+        }
+#endif
+    }
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(message_receive)) {
+        Eterm token2 = NIL;
+        DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
+        Sint tok_label = 0;
+        Sint tok_lastcnt = 0;
+        Sint tok_serial = 0;
+
+        dtrace_proc_str(c_p, receiver_name);
+        token2 = SEQ_TRACE_TOKEN(c_p);
+        if (have_seqtrace(token2)) {
+            tok_label = signed_val(SEQ_TRACE_T_LABEL(token2));
+            tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2));
+            tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2));
+        }
+        DTRACE6(message_receive,
+                receiver_name, size_object(ERL_MESSAGE_TERM(msgp)),
+                c_p->msg.len - 1, tok_label, tok_lastcnt, tok_serial);
+    }
+#endif
+    UNLINK_MESSAGE(c_p, msgp);
+    JOIN_MESSAGE(c_p);
+    CANCEL_TIMER(c_p);
+
+    erts_save_message_in_proc(c_p, msgp);
+    c_p->flags &= ~F_DELAY_GC;
+
+    if (ERTS_IS_GC_DESIRED_INTERNAL(c_p, HTOP, E)) {
+        /*
+         * We want to GC soon but we leave a few
+         * reductions giving the message some time
+         * to turn into garbage.
+         */
+        ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS);
+    }
+
+    ERTS_DBG_CHK_REDS(c_p, FCALLS);
+    ERTS_CHK_MBUF_SZ(c_p);
+
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+}
+
+loop_rec_end(Dest) {
+    //| -no_next
+    /*
+     * Advance the save pointer to the next message (the current
+     * message didn't match), then jump to the loop_rec instruction.
+     */
+
+    ASSERT(c_p->flags & F_DELAY_GC);
+
+    SET_I((BeamInstr *) $Dest);
+    SAVE_MESSAGE(c_p);
+    if (FCALLS > 0 || FCALLS > neg_o_reds) {
+        FCALLS--;
+        goto loop_rec__;
+    }
+
+    c_p->flags &= ~F_DELAY_GC;
+    c_p->i = I;
+    SWAPOUT;
+    c_p->arity = 0;
+    c_p->current = NULL;
+    goto do_schedule;
+}
+
+timeout_locked() {
+    /*
+     * A timeout has occurred.  Reset the save pointer so that the next
+     * receive statement will examine the first message first.
+     */
+
+    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+    $timeout();
+}
+
+timeout() {
+    if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) {
+        trace_receive(c_p, am_clock_service, am_timeout, NULL);
+    }
+    if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
+        save_calls(c_p, &exp_timeout);
+    }
+    c_p->flags &= ~F_TIMO;
+    JOIN_MESSAGE(c_p);
+}
+
+TIMEOUT_VALUE() {
+    c_p->freason = EXC_TIMEOUT_VALUE;
+    goto find_func_info;
+}
+
+i_wait_error_locked() {
+    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+    $TIMEOUT_VALUE();
+}
+
+i_wait_error() {
+    $TIMEOUT_VALUE();
+}
+
+wait_timeout_unlocked_int := wait.lock.int.execute;
+wait_timeout_locked_int := wait.int.execute;
+
+wait_timeout_unlocked := wait.lock.src.execute;
+wait_timeout_locked := wait.src.execute;
+
+wait_unlocked := wait.lock.execute;
+wait_locked := wait.unlocked.execute;
+
+wait.lock() {
+    erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+}
+
+wait.unlocked() {
+}
+
+wait.int(Int) {
+    /*
+     * If we have already set the timer, we must NOT set it again.  Therefore,
+     * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag.
+     */
+    if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) {
+        BeamInstr** pi = (BeamInstr **) c_p->def_arg_reg;
+        *pi = $NEXT_INSTRUCTION;
+        erts_set_proc_timer_uword(c_p, $Int);
+    }
+}
+
+wait.src(Src) {
+    /*
+     * If we have already set the timer, we must NOT set it again.  Therefore,
+     * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag.
+     */
+    if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) {
+        Eterm timeout_value = $Src;
+        if (timeout_value == make_small(0)) {
+            erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+            $NEXT0();
+        } else if (timeout_value == am_infinity) {
+            c_p->flags |= F_TIMO;
+        } else {
+            int tres = erts_set_proc_timer_term(c_p, timeout_value);
+            if (tres == 0) {
+                /*
+                 * The timer routiner will set c_p->i to the value in
+                 * c_p->def_arg_reg[0].  Note that it is safe to use this
+                 * location because there are no living x registers in
+                 * a receive statement.
+                 */
+                BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
+                *pi = $NEXT_INSTRUCTION;
+            } else { /* Wrong time */
+                erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+                c_p->freason = EXC_TIMEOUT_VALUE;
+                goto find_func_info;
+            }
+        }
+    }
+}
+
+//
+// Prepare to wait indefinitely for a new message to arrive
+// (or the time set above if falling through from above).
+// When a new message arrives, control will be transferred
+// the loop_rec instruction (at label L1).  In case of
+// of timeout, control will be transferred to the timeout
+// instruction following the wait_timeout instruction.
+//
+
+wait.execute(JumpTarget) {
+    c_p->i = (BeamInstr *) $JumpTarget; /* L1 */
+    SWAPOUT;
+    c_p->arity = 0;
+
+    if (!ERTS_PTMR_IS_TIMED_OUT(c_p)) {
+        erts_atomic32_read_band_relb(&c_p->state,
+                                         ~ERTS_PSFLG_ACTIVE);
+    }
+    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+    c_p->current = NULL;
+    goto do_schedule;
+    //| -no_next
+}
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 8c9034518b..fdc4506351 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -378,13 +378,15 @@ move r y
 
 loop_rec Fail x==0 | smp_mark_target_label(Fail) => i_loop_rec Fail
 
-label L | wait_timeout Fail Src | smp_already_locked(L) => label L | i_wait_timeout_locked Fail Src
-wait_timeout Fail Src => i_wait_timeout Fail Src
-i_wait_timeout Fail Src=aiq => gen_literal_timeout(Fail, Src)
-i_wait_timeout_locked Fail Src=aiq => gen_literal_timeout_locked(Fail, Src)
+label L | wait_timeout Fail Src | smp_already_locked(L) => \
+    label L | wait_timeout_locked Src Fail
+wait_timeout Fail Src => wait_timeout_unlocked Src Fail
+
+wait_timeout_unlocked Fail Src=aiq => gen_literal_timeout(Fail, Src)
+wait_timeout_locked Fail Src=aiq => gen_literal_timeout_locked(Fail, Src)
 
 label L | wait Fail | smp_already_locked(L) => label L | wait_locked Fail
-wait Fail | smp() => wait_unlocked Fail
+wait Fail => wait_unlocked Fail
 
 label L | timeout | smp_already_locked(L) => label L | timeout_locked
 
@@ -393,13 +395,14 @@ timeout
 timeout_locked
 i_loop_rec f
 loop_rec_end f
-wait f
 wait_locked f
 wait_unlocked f
-i_wait_timeout f I
-i_wait_timeout f s
-i_wait_timeout_locked f I
-i_wait_timeout_locked f s
+
+wait_timeout_unlocked_int I f
+wait_timeout_unlocked s f
+wait_timeout_locked_int I f
+wait_timeout_locked s f
+
 i_wait_error
 i_wait_error_locked
 
@@ -967,10 +970,9 @@ node x
 node y
 %hot
 
-i_fast_element j x I d
-i_fast_element j y I d
+i_fast_element xy j I d
 
-i_element j xy s d
+i_element xy j s d
 
 bif1 f b s d
 bif1_body b s d
@@ -1060,7 +1062,10 @@ i_bs_get_integer_imm x I I f I x
 i_bs_get_integer f I I s s x
 i_bs_get_integer_8 x f x
 i_bs_get_integer_16 x f x
-i_bs_get_integer_32 x f I x
+
+%if ARCH_64
+i_bs_get_integer_32 x f x
+%endif
 
 # Fetching binaries from binaries.
 bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \
@@ -1134,12 +1139,8 @@ i_bs_validate_unicode_retract j s s
 
 bs_init2 Fail Sz Words Regs Flags Dst | binary_too_big(Sz) => system_limit Fail
 
-bs_init2 Fail Sz=u Words=u==0 Regs Flags Dst | should_gen_heap_bin(Sz) => \
-	i_bs_init_heap_bin Sz Regs Dst
 bs_init2 Fail Sz=u Words=u==0 Regs Flags Dst => i_bs_init Sz Regs Dst
 
-bs_init2 Fail Sz=u Words Regs Flags Dst | should_gen_heap_bin(Sz) => \
-	i_bs_init_heap_bin_heap Sz Words Regs Dst
 bs_init2 Fail Sz=u Words Regs Flags Dst => \
    i_bs_init_heap Sz Words Regs Dst
 
@@ -1153,10 +1154,8 @@ i_bs_init_fail xy j I x
 i_bs_init_fail_heap s I j I x
 
 i_bs_init I I x
-i_bs_init_heap_bin I I x
 
 i_bs_init_heap I I I x
-i_bs_init_heap_bin_heap I I I x
 
 
 bs_init_bits Fail Sz=o Words Regs Flags Dst => system_limit Fail
@@ -1254,8 +1253,6 @@ i_new_bs_put_binary_all j s I
 
 bs_put_string I I
 
-%hot
-
 #
 # New floating point instructions (R8).
 #
@@ -1268,9 +1265,11 @@ fnegate p FR1 FR2 => i_fnegate FR1 FR2
 
 fconv Arg=iqan Dst=l => move Arg x | fconv x Dst
 
-fmove q l
-fmove d l
-fmove l d
+fmove Arg=l Dst=d => fstore Arg Dst
+fmove Arg=dq Dst=l => fload Arg Dst
+
+fstore l d
+fload dq l
 
 fconv d l
 
@@ -1282,10 +1281,15 @@ i_fnegate l l
 
 fclearerror | no_fpe_signals() =>
 fcheckerror p | no_fpe_signals() =>
+
+%unless NO_FPE_SIGNALS
 fcheckerror p => i_fcheckerror
 
 i_fcheckerror
 fclearerror
+%endif
+
+%hot
 
 #
 # New apply instructions in R10B.
@@ -1385,9 +1389,9 @@ gen_minus p Live Reg=d Int=i Dst | negation_is_small(Int) => \
 # GCing arithmetic instructions.
 #
 
-gen_plus Fail Live S1 S2 Dst => i_plus Fail Live S1 S2 Dst
+gen_plus Fail Live S1 S2 Dst => i_plus S1 S2 Fail Live Dst
 
-gen_minus Fail Live S1 S2 Dst => i_minus Fail Live S1 S2 Dst
+gen_minus Fail Live S1 S2 Dst => i_minus S1 S2 Fail Live Dst
 
 gc_bif2 Fail Live u$bif:erlang:stimes/2 S1 S2 Dst => \
   i_times Fail Live S1 S2 Dst
@@ -1398,15 +1402,15 @@ gc_bif2 Fail Live u$bif:erlang:intdiv/2 S1 S2 Dst => \
   i_int_div Fail Live S1 S2 Dst
 
 gc_bif2 Fail Live u$bif:erlang:rem/2 S1 S2 Dst => \
-  i_rem Fail Live S1 S2 Dst
+  i_rem S1 S2 Fail Live Dst
 
 gc_bif2 Fail Live u$bif:erlang:bsl/2 S1 S2 Dst => \
-  i_bsl Fail Live S1 S2 Dst
+  i_bsl S1 S2 Fail Live Dst
 gc_bif2 Fail Live u$bif:erlang:bsr/2 S1 S2 Dst => \
-  i_bsr Fail Live S1 S2 Dst
+  i_bsr S1 S2 Fail Live Dst
 
 gc_bif2 Fail Live u$bif:erlang:band/2 S1 S2 Dst => \
-  i_band Fail Live S1 S2 Dst
+  i_band S1 S2 Fail Live Dst
 
 gc_bif2 Fail Live u$bif:erlang:bor/2 S1 S2 Dst => \
   i_bor Fail Live S1 S2 Dst
@@ -1418,25 +1422,25 @@ gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst
 
 i_increment rxy I I d
 
-i_plus j I x xy d
-i_plus j I s s d
+i_plus x xy j I d
+i_plus s s  j I d
 
-i_minus j I x x d
-i_minus j I s s d
+i_minus x x j I d
+i_minus s s j I d
 
 i_times j I s s d
 
 i_m_div j I s s d
 i_int_div j I s s d
 
-i_rem j I x x d
-i_rem j I s s d
+i_rem x x j I d
+i_rem s s j I d
 
-i_bsl j I s s d
-i_bsr j I s s d
+i_bsl s s j I d
+i_bsr s s j I d
 
-i_band j I x c d
-i_band j I s s d
+i_band x c j I d
+i_band s s j I d
 
 i_bor j I s s d
 i_bxor j I s s d
diff --git a/erts/emulator/beam/select_instrs.tab b/erts/emulator/beam/select_instrs.tab
new file mode 100644
index 0000000000..e85ed2c304
--- /dev/null
+++ b/erts/emulator/beam/select_instrs.tab
@@ -0,0 +1,196 @@
+// -*- c -*-
+//
+// %CopyrightBegin%
+//
+// Copyright Ericsson AB 2017. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// %CopyrightEnd%
+//
+
+i_select_val_bins := select_val_bins.fetch.select;
+
+select_val_bins.head() {
+    Eterm select_val;
+}
+
+select_val_bins.fetch(Src) {
+    select_val = $Src;
+}
+
+select_val_bins.select(Fail, NumElements) {
+    struct Pairs {
+        BeamInstr val;
+        BeamInstr* addr;
+    };
+    struct Pairs* low;
+    struct Pairs* high;
+    struct Pairs* mid;
+    int bdiff; /* int not long because the arrays aren't that large */
+
+    low = (struct Pairs *) (&$NumElements + 1);
+    high = low + $NumElements;
+
+    /* The pointer subtraction (high-low) below must produce
+     * a signed result, because high could be < low. That
+     * requires the compiler to insert quite a bit of code.
+     *
+     * However, high will be > low so the result will be
+     * positive. We can use that knowledge to optimise the
+     * entire sequence, from the initial comparison to the
+     * computation of mid.
+     *
+     * -- Mikael Pettersson, Acumem AB
+     *
+     * Original loop control code:
+     *
+     * while (low < high) {
+     *    mid = low + (high-low) / 2;
+     *
+     */
+    while ((bdiff = (int)((char*)high - (char*)low)) > 0) {
+        unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Pairs)-1);
+
+        mid = (struct Pairs*)((char*)low + boffset);
+        if (select_val < mid->val) {
+            high = mid;
+        } else if (select_val > mid->val) {
+            low = mid + 1;
+        } else {
+            $NEXT(mid->addr);
+        }
+    }
+    $NEXT($Fail);
+}
+
+i_select_tuple_arity2 := select_val2.src.ta_fail.execute;
+i_select_val2 := select_val2.src.fail.execute;
+
+select_val2.head() {
+    Eterm select_val2;
+    BeamInstr* select_fail;
+}
+
+select_val2.src(Src) {
+    select_val2 = $Src;
+}
+
+select_val2.ta_fail(Fail) {
+    select_fail = &$Fail;
+    if (is_not_tuple(select_val2)) {
+        $FAIL(*select_fail);
+    }
+    select_val2 = *tuple_val(select_val2);
+}
+
+select_val2.fail(Fail) {
+    select_fail = &$Fail;
+}
+
+select_val2.execute(T1, T2, D1, D2) {
+    if (select_val2 == $T1) {
+        $JUMP($D1);
+    } else if (select_val2 == $T2) {
+        $JUMP($D2);
+    } else {
+        $FAIL(*select_fail);
+    }
+}
+
+i_select_tuple_arity := select_val_lin.fetch.ta_fail.execute;
+i_select_val_lins := select_val_lin.fetch.fail.execute;
+
+select_val_lin.head() {
+    Eterm select_val;
+    BeamInstr* select_fail;
+}
+
+select_val_lin.fetch(Src) {
+    select_val = $Src;
+}
+
+select_val_lin.ta_fail(Fail) {
+    select_fail = &$Fail;
+    if (is_tuple(select_val)) {
+        select_val = *tuple_val(select_val);
+    } else {
+        $JUMP(*select_fail);
+    }
+}
+
+select_val_lin.fail(Fail) {
+    select_fail = &$Fail;
+}
+
+select_val_lin.execute(N) {
+    BeamInstr* vs = $NEXT_INSTRUCTION;
+    int ix = 0;
+
+    for (;;) {
+        if (vs[ix+0] >= select_val) {
+            ix += 0;
+            break;
+        }
+        if (vs[ix+1] >= select_val) {
+            ix += 1;
+            break;
+        }
+        ix += 2;
+    }
+
+    if (vs[ix] == select_val) {
+        I = $NEXT_INSTRUCTION + $N + ix;
+        $JUMP(*I);
+    } else {
+        $JUMP(*select_fail);
+    }
+}
+
+JUMP_ON_VAL(Fail, Index, N, Base) {
+    if (is_small($Index)) {
+        $Index = (Uint) (signed_val($Index) - $Base);
+        if ($Index < $N) {
+            $JUMP((($NEXT_INSTRUCTION)[$Index]));
+        }
+    }
+    $FAIL($Fail);
+}
+
+i_jump_on_val_zero := jump_on_val_zero.fetch.execute;
+
+jump_on_val_zero.head() {
+    Eterm index;
+}
+
+jump_on_val_zero.fetch(Src) {
+    index = $Src;
+}
+
+jump_on_val_zero.execute(Fail, N) {
+    $JUMP_ON_VAL($Fail, index, $N, 0);
+}
+
+i_jump_on_val := jump_on_val.fetch.execute;
+
+jump_on_val.head() {
+    Eterm index;
+}
+
+jump_on_val.fetch(Src) {
+    index = $Src;
+}
+
+jump_on_val.execute(Fail, N, Base) {
+    $JUMP_ON_VAL($Fail, index, $N, $Base);
+}
diff --git a/erts/emulator/beam/trace_instrs.tab b/erts/emulator/beam/trace_instrs.tab
new file mode 100644
index 0000000000..dfd1d16d58
--- /dev/null
+++ b/erts/emulator/beam/trace_instrs.tab
@@ -0,0 +1,155 @@
+// -*- c -*-
+//
+// %CopyrightBegin%
+//
+// Copyright Ericsson AB 2017. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// %CopyrightEnd%
+//
+
+return_trace() {
+    ErtsCodeMFA* mfa = (ErtsCodeMFA *)(E[0]);
+
+    SWAPOUT;		/* Needed for shared heap */
+    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
+    erts_trace_return(c_p, mfa, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */);
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
+    SWAPIN;
+    c_p->cp = NULL;
+    SET_I((BeamInstr *) cp_val(E[2]));
+    E += 3;
+    Goto(*I);
+    //| -no_next
+}
+
+i_generic_breakpoint() {
+    BeamInstr real_I;
+    HEAVY_SWAPOUT;
+    real_I = erts_generic_breakpoint(c_p, erts_code_to_codeinfo(I), reg);
+    HEAVY_SWAPIN;
+    ASSERT(VALID_INSTR(real_I));
+    Goto(real_I);
+    //| -no_next
+}
+
+i_return_time_trace() {
+    BeamInstr *pc = (BeamInstr *) (UWord) E[0];
+    SWAPOUT;
+    erts_trace_time_return(c_p, erts_code_to_codeinfo(pc));
+    SWAPIN;
+    c_p->cp = NULL;
+    SET_I((BeamInstr *) cp_val(E[1]));
+    E += 2;
+    Goto(*I);
+    //| -no_next
+}
+
+i_return_to_trace() {
+    if (IS_TRACED_FL(c_p, F_TRACE_RETURN_TO)) {
+        Uint *cpp = (Uint*) E;
+        for(;;) {
+            ASSERT(is_CP(*cpp));
+            if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) {
+                do
+                    ++cpp;
+                while (is_not_CP(*cpp));
+                cpp += 2;
+            } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) {
+                do
+                    ++cpp;
+                while (is_not_CP(*cpp));
+            } else {
+                break;
+            }
+        }
+        SWAPOUT;		/* Needed for shared heap */
+        ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
+        erts_trace_return_to(c_p, cp_val(*cpp));
+        ERTS_REQ_PROC_MAIN_LOCK(c_p);
+        SWAPIN;
+    }
+    c_p->cp = NULL;
+    SET_I((BeamInstr *) cp_val(E[0]));
+    E += 1;
+    Goto(*I);
+    //| -no_next
+}
+
+i_yield() {
+    /* This is safe as long as REDS_IN(c_p) is never stored
+     * in c_p->arg_reg[0]. It is currently stored in c_p->def_arg_reg[5].
+     */
+    c_p->arg_reg[0] = am_true;
+    c_p->arity = 1; /* One living register (the 'true' return value) */
+    SWAPOUT;
+    c_p->i = $NEXT_INSTRUCTION;
+    c_p->current = NULL;
+    goto do_schedule;
+    //| -no_next
+}
+
+i_hibernate() {
+    HEAVY_SWAPOUT;
+    if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) {
+        FCALLS = c_p->fcalls;
+        c_p->flags &= ~F_HIBERNATE_SCHED;
+        goto do_schedule;
+    } else {
+        HEAVY_SWAPIN;
+        I = handle_error(c_p, I, reg, &bif_export[BIF_hibernate_3]->info.mfa);
+        goto post_error_handling;
+    }
+    //| -no_next
+}
+
+// This is optimised as an instruction because
+// it has to be very very fast.
+
+i_perf_counter() {
+    ErtsSysPerfCounter ts;
+
+    ts = erts_sys_perf_counter();
+    if (IS_SSMALL(ts)) {
+        r(0) = make_small((Sint)ts);
+    } else {
+        $GC_TEST(0, ERTS_SINT64_HEAP_SIZE(ts), 0);
+        r(0) = make_big(HTOP);
+#if defined(ARCH_32)
+        if (ts >= (((Uint64) 1) << 32)) {
+            *HTOP = make_pos_bignum_header(2);
+            BIG_DIGIT(HTOP, 0) = (Uint) (ts & ((Uint) 0xffffffff));
+            BIG_DIGIT(HTOP, 1) = (Uint) ((ts >> 32) & ((Uint) 0xffffffff));
+            HTOP += 3;
+        }
+        else
+#endif
+            {
+                *HTOP = make_pos_bignum_header(1);
+                BIG_DIGIT(HTOP, 0) = (Uint) ts;
+                HTOP += 2;
+            }
+    }
+}
+
+i_debug_breakpoint() {
+    HEAVY_SWAPOUT;
+    I = call_error_handler(c_p, erts_code_to_codemfa(I), reg, am_breakpoint);
+    HEAVY_SWAPIN;
+    if (I) {
+        Goto(*I);
+    }
+    goto handle_error;
+    //| -no_next
+}
diff --git a/erts/emulator/hipe/hipe_instrs.tab b/erts/emulator/hipe/hipe_instrs.tab
new file mode 100644
index 0000000000..bcce196a1d
--- /dev/null
+++ b/erts/emulator/hipe/hipe_instrs.tab
@@ -0,0 +1,141 @@
+// -*- c -*-
+//
+// %CopyrightBegin%
+//
+// Copyright Ericsson AB 2017. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// %CopyrightEnd%
+//
+
+
+HIPE_MODE_SWITCH(Cmd) {
+    SWAPOUT;
+    ERTS_DBG_CHK_REDS(c_p, FCALLS);
+    c_p->fcalls = FCALLS;
+    c_p->def_arg_reg[4] = -neg_o_reds;
+    c_p = hipe_mode_switch(c_p, $Cmd, reg);
+}
+
+hipe_trap_call := hipe_trap.call.post;
+hipe_trap_call_closure := hipe_trap.call_closure.post;
+hipe_trap_return := hipe_trap.return.post;
+hipe_trap_throw := hipe_trap.throw.post;
+hipe_trap_resume := hipe_trap.resume.post;
+
+hipe_trap.call() {
+    /*
+     * I[-5]: &&lb_i_func_info_IaaI
+     * I[-4]: Native code callee (inserted by HiPE)
+     * I[-3]: Module (tagged atom)
+     * I[-2]: Function (tagged atom)
+     * I[-1]: Arity (untagged integer)
+     * I[ 0]: &&lb_hipe_trap_call
+     * ... remainder of original BEAM code
+     */
+    ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
+    ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
+    c_p->hipe.u.ncallee = ci->u.ncallee;
+    ++hipe_trap_count;
+    $HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8));
+}
+
+hipe_trap.call_closure() {
+    ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
+    ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
+    c_p->hipe.u.ncallee = ci->u.ncallee;
+    ++hipe_trap_count;
+    $HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8));
+}
+
+hipe_trap.return() {
+    $HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RETURN);
+}
+
+hipe_trap.throw() {
+    $HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_THROW);
+}
+
+hipe_trap.resume() {
+    $HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RESUME);
+}
+
+hipe_trap.post() {
+#ifdef DEBUG
+    pid = c_p->common.id; /* may have switched process... */
+#endif
+    reg = erts_proc_sched_data(c_p)->x_reg_array;
+    freg = erts_proc_sched_data(c_p)->f_reg_array;
+    ERL_BITS_RELOAD_STATEP(c_p);
+    /* XXX: this abuse of def_arg_reg[] is horrid! */
+    neg_o_reds = -c_p->def_arg_reg[4];
+    FCALLS = c_p->fcalls;
+    SWAPIN;
+    ERTS_DBG_CHK_REDS(c_p, FCALLS);
+    switch( c_p->def_arg_reg[3] ) {
+    case HIPE_MODE_SWITCH_RES_RETURN:
+        ASSERT(is_value(reg[0]));
+        SET_I(c_p->cp);
+        c_p->cp = 0;
+        Goto(*I);
+    case HIPE_MODE_SWITCH_RES_CALL_EXPORTED:
+        c_p->i = c_p->hipe.u.callee_exp->addressv[erts_active_code_ix()];
+        /*fall through*/
+    case HIPE_MODE_SWITCH_RES_CALL_BEAM:
+        SET_I(c_p->i);
+        Dispatch();
+    case HIPE_MODE_SWITCH_RES_CALL_CLOSURE:
+        /* This can be used to call any function value, but currently
+           it's only used to call closures referring to unloaded
+           modules. */
+        {
+            BeamInstr *next;
+
+            next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE);
+            HEAVY_SWAPIN;
+            if (next != NULL) {
+                SET_I(next);
+                Dispatchfun();
+            }
+            goto find_func_info;
+        }
+    case HIPE_MODE_SWITCH_RES_THROW:
+        c_p->cp = NULL;
+        I = handle_error(c_p, I, reg, NULL);
+        goto post_error_handling;
+    default:
+        erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]);
+    }
+    //| -no_next;
+}
+
+hipe_call_count() {
+    /*
+     * I[-5]: &&lb_i_func_info_IaaI
+     * I[-4]: pointer to struct hipe_call_count (inserted by HiPE)
+     * I[-3]: Module (tagged atom)
+     * I[-2]: Function (tagged atom)
+     * I[-1]: Arity (untagged integer)
+     * I[ 0]: &&lb_hipe_call_count
+     * ... remainder of original BEAM code
+     */
+    ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
+    struct hipe_call_count *hcc = ci->u.hcc;
+    ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
+    ASSERT(hcc != NULL);
+    ASSERT(VALID_INSTR(hcc->opcode));
+    ++(hcc->count);
+    Goto(hcc->opcode);
+    //| -no_next;
+}
-- 
cgit v1.2.3


From 8c0569d2b5c26545cb7abd9063a1abda7ebd13e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 10 Aug 2017 11:46:48 +0200
Subject: Make '0 bsl BigNumber' consistently succeed

'0 bsl 134217728' would fail with a system limit exception on a 32-bit
BEAM machine, but not on a 64-bit BEAM machine. Smaller values
on the right would always work.

Make erlang:bsl(0, BigNumber) always return 0 to make for consistency.
(The previous commit accidentally did that change for
'0 bsl BigNumber'.)
---
 erts/emulator/beam/erl_arith.c   | 6 +++++-
 erts/emulator/test/big_SUITE.erl | 7 +++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c
index 861532f241..f2a3e411ec 100644
--- a/erts/emulator/beam/erl_arith.c
+++ b/erts/emulator/beam/erl_arith.c
@@ -276,8 +276,12 @@ shift(Process* p, Eterm arg1, Eterm arg2, int right)
 		goto do_bsl;
 	    } else if (is_small(arg1) || is_big(arg1)) {
 		/*
-		 * N bsl PositiveBigNum is too large to represent.
+		 * N bsl PositiveBigNum is too large to represent,
+                 * unless N is 0.
 		 */
+                if (arg1 == make_small(0)) {
+                    BIF_RET(arg1);
+                }
 		BIF_ERROR(p, SYSTEM_LIMIT);
 	    }
 	     /* Fall through if the left argument is not an integer. */
diff --git a/erts/emulator/test/big_SUITE.erl b/erts/emulator/test/big_SUITE.erl
index c308760211..5939d024ae 100644
--- a/erts/emulator/test/big_SUITE.erl
+++ b/erts/emulator/test/big_SUITE.erl
@@ -339,6 +339,13 @@ system_limit(Config) when is_list(Config) ->
     {'EXIT',{system_limit,_}} = (catch apply(erlang, id('bsl'), [Maxbig,2])),
     {'EXIT',{system_limit,_}} = (catch id(1) bsl (1 bsl 45)),
     {'EXIT',{system_limit,_}} = (catch id(1) bsl (1 bsl 69)),
+
+    %% There should be no system_limit exception when shifting a zero.
+    0 = id(0) bsl (1 bsl 128),
+    0 = id(0) bsr -(1 bsl 128),
+    Erlang = id(erlang),
+    0 = Erlang:'bsl'(id(0), 1 bsl 128),
+    0 = Erlang:'bsr'(id(0), -(1 bsl 128)),
     ok.
 
 maxbig() ->
-- 
cgit v1.2.3


From c95813cdea2425a27a439f20b290864a0a3339f2 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 3 Aug 2017 15:18:48 +0200
Subject: erts: Remove unused prototypes after non-smp removal

---
 erts/emulator/beam/erl_threads.h | 78 ----------------------------------------
 1 file changed, 78 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index ffb1f72200..e306df818d 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -385,12 +385,6 @@ __decl_noreturn void  __noreturn erts_thr_fatal_error(int, char *);
 #  define ERTS_HAVE_REC_MTX_INIT	ETHR_HAVE_ETHR_REC_MUTEX_INIT
 #endif
 
-
-#define erts_no_dw_atomic_t erts_dw_aint_t
-#define erts_no_atomic_t erts_aint_t
-#define erts_no_atomic32_t erts_aint32_t
-#define erts_no_atomic64_t erts_aint64_t
-
 #define ERTS_AINT_NULL ((erts_aint_t) NULL)
 
 #define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
@@ -461,78 +455,6 @@ ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx);
 ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx);
 
-ERTS_GLB_INLINE void erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val);
-ERTS_GLB_INLINE void erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val);
-ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var,
-					      erts_no_dw_atomic_t *val,
-					      erts_no_dw_atomic_t *old_val);
-ERTS_GLB_INLINE void erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i);
-ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read(erts_no_atomic_t *var);
-ERTS_GLB_INLINE erts_aint_t erts_no_atomic_inc_read(erts_no_atomic_t *incp);
-ERTS_GLB_INLINE erts_aint_t erts_no_atomic_dec_read(erts_no_atomic_t *decp);
-ERTS_GLB_INLINE void erts_no_atomic_inc(erts_no_atomic_t *incp);
-ERTS_GLB_INLINE void erts_no_atomic_dec(erts_no_atomic_t *decp);
-ERTS_GLB_INLINE erts_aint_t erts_no_atomic_add_read(erts_no_atomic_t *addp,
-						    erts_aint_t i);
-ERTS_GLB_INLINE void erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i);
-ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bor(erts_no_atomic_t *var,
-						    erts_aint_t mask);
-ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_band(erts_no_atomic_t *var,
-						     erts_aint_t mask);
-ERTS_GLB_INLINE erts_aint_t erts_no_atomic_xchg(erts_no_atomic_t *xchgp,
-						erts_aint_t new);
-ERTS_GLB_INLINE erts_aint_t erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
-						   erts_aint_t new,
-						   erts_aint_t expected);
-ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bset(erts_no_atomic_t *var,
-						     erts_aint_t mask,
-						     erts_aint_t set);
-ERTS_GLB_INLINE void erts_no_atomic32_set(erts_no_atomic32_t *var,
-					  erts_aint32_t i);
-ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read(erts_no_atomic32_t *var);
-ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_inc_read(erts_no_atomic32_t *incp);
-ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_dec_read(erts_no_atomic32_t *decp);
-ERTS_GLB_INLINE void erts_no_atomic32_inc(erts_no_atomic32_t *incp);
-ERTS_GLB_INLINE void erts_no_atomic32_dec(erts_no_atomic32_t *decp);
-ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_add_read(erts_no_atomic32_t *addp,
-							erts_aint32_t i);
-ERTS_GLB_INLINE void erts_no_atomic32_add(erts_no_atomic32_t *addp,
-					  erts_aint32_t i);
-ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bor(erts_no_atomic32_t *var,
-							erts_aint32_t mask);
-ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_band(erts_no_atomic32_t *var,
-							 erts_aint32_t mask);
-ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp,
-						    erts_aint32_t new);
-ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
-						       erts_aint32_t new,
-						       erts_aint32_t expected);
-ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bset(erts_no_atomic32_t *var,
-							 erts_aint32_t mask,
-							 erts_aint32_t set);
-ERTS_GLB_INLINE void erts_no_atomic64_set(erts_no_atomic64_t *var,
-					  erts_aint64_t i);
-ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read(erts_no_atomic64_t *var);
-ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_inc_read(erts_no_atomic64_t *incp);
-ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_dec_read(erts_no_atomic64_t *decp);
-ERTS_GLB_INLINE void erts_no_atomic64_inc(erts_no_atomic64_t *incp);
-ERTS_GLB_INLINE void erts_no_atomic64_dec(erts_no_atomic64_t *decp);
-ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_add_read(erts_no_atomic64_t *addp,
-							erts_aint64_t i);
-ERTS_GLB_INLINE void erts_no_atomic64_add(erts_no_atomic64_t *addp,
-					  erts_aint64_t i);
-ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bor(erts_no_atomic64_t *var,
-							erts_aint64_t mask);
-ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_band(erts_no_atomic64_t *var,
-							 erts_aint64_t mask);
-ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp,
-						    erts_aint64_t new);
-ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp,
-						       erts_aint64_t new,
-						       erts_aint64_t expected);
-ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
-							 erts_aint64_t mask,
-							 erts_aint64_t set);
 ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock,
                                         char *name,
                                         Eterm extra,
-- 
cgit v1.2.3


From 9689ec29c328be096a33a4ac261f426c626425e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A1niel=20Szoboszlay?= <daniel.szoboszlay@klarna.com>
Date: Tue, 15 Aug 2017 13:29:54 +0200
Subject: Execute the dropped_commands test case in port_SUITE

Also, fix cleaning up the OS environment setting modified by this test
case to prevent problems with later test cases using the echo_drv too.
---
 erts/emulator/test/port_SUITE.erl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index ab0b1a82bd..c1e004c212 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -159,7 +159,7 @@ suite() ->
 all() ->
     [otp_6224, {group, stream}, basic_ping, slow_writes,
      bad_packet, bad_port_messages, {group, options},
-     {group, multiple_packets}, parallell, dying_port,
+     {group, multiple_packets}, parallell, dying_port, dropped_commands,
      port_program_with_path, open_input_file_port,
      open_output_file_port, name1, env, huge_env, bad_env, cd,
      cd_relative, pipe_limit_env, bad_args,
@@ -569,6 +569,7 @@ dropped_commands(Config, Outputv, Cmd) ->
     [dropped_commands_test(Cmd) || _ <- lists:seq(1, 100)],
     timer:sleep(100),
     erl_ddll:unload_driver("echo_drv"),
+    os:unsetenv("ECHO_DRV_USE_OUTPUTV"),
     ok.
 
 dropped_commands_test(Cmd) ->
-- 
cgit v1.2.3


From 17bb6bfa8d435300ee2205f1e0c20b0c6b50591a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 15 Aug 2017 13:34:45 +0200
Subject: Slightly optimize updating of maps

The instruction put_map_assoc/5 (used for updating a map) has a
failure operand, but it can't actually fail provided that its "map"
argument is a map.

The following code:

    M#{key=>value}.

will be compiled to:

    {test,is_map,{f,3},[{x,0}]}.
    {line,[...]}.
    {put_map_assoc,{f,0},{x,0},{x,0},1,{list,[{atom,key},{atom,value}]}}.
    return.
  {label,3}.
    %% Code that produces a 'badmap' exception follows.

Because of the is_map instruction, {x,0} always contains a map when
the put_map_assoc instruction is executed. Therefore we can remove
the failure operand. That will save one word, and also eliminate
two tests at run-time.

The only problem is that the compiler in OTP 17 did not emit a
is_map instruction before the put_map_assoc instruction. Therefore,
we must add an instruction that tests for a map if the code was
compiled with the OTP 17 compiler.

Unfortunately, there is no safe and relatively easy way to known that
the OTP 17 compiler was used, so we will check whether a compiler
before OTP 20 was used. OTP 20 introduced a new chunk type for atoms,
which is trivial to check.
---
 erts/emulator/beam/beam_debug.c                  |   2 +-
 erts/emulator/beam/beam_emu.c                    |  15 +++----
 erts/emulator/beam/beam_load.c                   |  14 +++++++
 erts/emulator/beam/macros.tab                    |   5 +++
 erts/emulator/beam/map_instrs.tab                |  27 +++++--------
 erts/emulator/beam/ops.tab                       |  49 ++++++++++++++++++-----
 erts/emulator/test/map_SUITE_data/badmap_17.beam | Bin 592 -> 1192 bytes
 erts/emulator/test/map_SUITE_data/badmap_17.erl  |  36 ++++++++++++++++-
 8 files changed, 109 insertions(+), 39 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index fa912e52e9..afe87288ce 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -697,7 +697,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
     case op_i_put_tuple_xI:
     case op_i_put_tuple_yI:
     case op_new_map_dII:
-    case op_update_map_assoc_jsdII:
+    case op_update_map_assoc_sdII:
     case op_update_map_exact_jsdII:
 	{
 	    int n = unpacked[-1];
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index ff3c725bbc..e5935f5f02 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2849,19 +2849,14 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     Eterm new_key;
     Eterm* kp;
 
-    new_p = &Arg(5);
-    num_updates = Arg(4) / 2;
+    new_p = &Arg(4);
+    num_updates = Arg(3) / 2;
 
     if (is_not_flatmap(map)) {
 	Uint32 hx;
 	Eterm val;
 
-	/* apparently the compiler does not emit is_map instructions,
-	 * bad compiler */
-
-	if (is_not_hashmap(map))
-	    return THE_NON_VALUE;
-
+	ASSERT(is_hashmap(map));
 	res = map;
 	E = p->stop;
 	while(num_updates--) {
@@ -2885,7 +2880,7 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
      */
 
     if (num_old == 0) {
-	return new_map(p, reg, I+1);
+	return new_map(p, reg, I);
     }
 
     /*
@@ -2895,7 +2890,7 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 
     need = 2*(num_old+num_updates) + 1 + MAP_HEADER_FLATMAP_SZ;
     if (HeapWordsLeft(p) < need) {
-	Uint live = Arg(3);
+	Uint live = Arg(2);
 	reg[live] = map;
 	erts_garbage_collect(p, need, reg, live+1);
 	map      = reg[live];
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index d38e71f489..dcd312f54f 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -306,6 +306,7 @@ typedef struct LoaderState {
     int on_load;		/* Index in the code for the on_load function
 				 * (or 0 if there is no on_load function)
 				 */
+    int otp_20_or_higher;       /* Compiled with OTP 20 or higher */
 
     /*
      * Atom table.
@@ -739,6 +740,13 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader,
 	}
     }
 
+    /*
+     * Find out whether the code was compiled with OTP 20
+     * or higher.
+     */
+
+    stp->otp_20_or_higher = stp->chunks[UTF8_ATOM_CHUNK].size > 0;
+
     /*
      * Load the code chunk.
      */
@@ -2731,6 +2739,12 @@ load_code(LoaderState* stp)
 
 #define never(St) 0
 
+static int
+compiled_with_otp_20_or_higher(LoaderState* stp)
+{
+    return stp->otp_20_or_higher;
+}
+
 /*
  * Predicate that tests whether a jump table can be used.
  */
diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab
index 57015fac31..41dc761e90 100644
--- a/erts/emulator/beam/macros.tab
+++ b/erts/emulator/beam/macros.tab
@@ -92,6 +92,11 @@ NEXT(Addr) {
     Goto(*I);
 }
 
+FAIL_BODY() {
+    //| -no_prefetch
+    goto find_func_info;
+}
+
 FAIL_HEAD_OR_BODY(Fail) {
     //| -no_prefetch
     if ($Fail) {
diff --git a/erts/emulator/beam/map_instrs.tab b/erts/emulator/beam/map_instrs.tab
index 7f9346d029..30c3d7743f 100644
--- a/erts/emulator/beam/map_instrs.tab
+++ b/erts/emulator/beam/map_instrs.tab
@@ -19,10 +19,12 @@
 // %CopyrightEnd%
 //
 
-BADMAP(Fail, Map) {
-    c_p->freason = BADMAP;
-    c_p->fvalue = $Map;
-    $FAIL_HEAD_OR_BODY($Fail);
+ensure_map(Map) {
+    if (is_not_map($Map)) {
+        c_p->freason = BADMAP;
+        c_p->fvalue = $Map;
+        $FAIL_BODY();
+    }
 }
 
 new_map(Dst, Live, N) {
@@ -123,7 +125,7 @@ i_get_map_elements(Fail, Src, N) {
     }
 }
 
-update_map_assoc(Fail, Src, Dst, Live, N) {
+update_map_assoc(Src, Dst, Live, N) {
     Eterm res;
     Eterm map;
 
@@ -131,17 +133,10 @@ update_map_assoc(Fail, Src, Dst, Live, N) {
     HEAVY_SWAPOUT;
     res = update_map_assoc(c_p, reg, map, I);
     HEAVY_SWAPIN;
-    if (is_value(res)) {
-        $REFRESH_GEN_DEST();
-        $Dst = res;
-        $NEXT($NEXT_INSTRUCTION+$N);
-    } else {
-        /*
-         * This can only happen if the code was compiled
-         * with the compiler in OTP 17.
-         */
-        $BADMAP($Fail, map);
-    }
+    ASSERT(is_value(res));
+    $REFRESH_GEN_DEST();
+    $Dst = res;
+    $NEXT($NEXT_INSTRUCTION+$N);
 }
 
 update_map_exact(Fail, Src, Dst, Live, N) {
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index fdc4506351..92e67bb470 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1299,23 +1299,52 @@ apply I
 apply_last I P
 
 #
-# Map instructions in R17.
+# Handle compatibility with OTP 17 here.
 #
 
-sorted_put_map_assoc/5
-put_map_assoc F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \
-  sorted_put_map_assoc F Map Dst Live Size Rest
+i_put_map_assoc/4
+
+# We KNOW that in OTP 20 (actually OTP 18 and higher), a put_map_assoc instruction
+# is always preceded by an is_map test. That means that put_map_assoc can never
+# fail and does not need any failure label.
+
+put_map_assoc Fail Map Dst Live Size Rest=* | compiled_with_otp_20_or_higher() => \
+	      i_put_map_assoc Map Dst Live Size Rest
+
+# Translate the put_map_assoc instruction if the module was compiled by a compiler
+# before 20. This is only necessary if the OTP 17 compiler was used, but we
+# have no safe and relatively easy way to know whether OTP 18/19 was used.
+
+put_map_assoc Fail=p Map Dst Live Size Rest=* => \
+	      ensure_map Map | i_put_map_assoc Map Dst Live Size Rest
+put_map_assoc Fail=f Map Dst Live Size Rest=* => \
+	      is_map Fail Map | i_put_map_assoc Map Dst Live Size Rest
+
+ensure_map Lit=q | literal_is_map(Lit) =>
+ensure_map Src=cqy => move Src x | ensure_map x
+
+%cold
+ensure_map x
+%hot
+
+#
+# Map instructions. First introduced in R17.
+#
+
+sorted_put_map_assoc/4
+i_put_map_assoc Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \
+  sorted_put_map_assoc Map Dst Live Size Rest
 
 sorted_put_map_exact/5
 put_map_exact F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \
   sorted_put_map_exact F Map Dst Live Size Rest
 
-sorted_put_map_assoc j Map Dst Live Size Rest=* | is_empty_map(Map) => \
+sorted_put_map_assoc Map Dst Live Size Rest=* | is_empty_map(Map) => \
    new_map Dst Live Size Rest
-sorted_put_map_assoc F Src=s Dst Live Size Rest=* => \
-	       update_map_assoc F Src Dst Live Size Rest
-sorted_put_map_assoc F Src Dst Live Size Rest=* => \
-	       move Src x | update_map_assoc F x Dst Live Size Rest
+sorted_put_map_assoc Src=s Dst Live Size Rest=* => \
+	       update_map_assoc Src Dst Live Size Rest
+sorted_put_map_assoc Src Dst Live Size Rest=* => \
+	       move Src x | update_map_assoc x Dst Live Size Rest
 
 sorted_put_map_exact F Src=s Dst Live Size Rest=* => \
 	      update_map_exact F Src Dst Live Size Rest
@@ -1327,7 +1356,7 @@ new_map Dst Live Size Rest=* | is_small_map_literal_keys(Size, Rest) => \
 
 new_map d I I
 i_new_small_map_lit d I q
-update_map_assoc j s d I I
+update_map_assoc s d I I
 update_map_exact j s d I I
 
 is_map Fail Lit=q | literal_is_map(Lit) =>
diff --git a/erts/emulator/test/map_SUITE_data/badmap_17.beam b/erts/emulator/test/map_SUITE_data/badmap_17.beam
index 277fc34b94..6f79bb8c2c 100644
Binary files a/erts/emulator/test/map_SUITE_data/badmap_17.beam and b/erts/emulator/test/map_SUITE_data/badmap_17.beam differ
diff --git a/erts/emulator/test/map_SUITE_data/badmap_17.erl b/erts/emulator/test/map_SUITE_data/badmap_17.erl
index 0ec65e0e33..887fc2e5e3 100644
--- a/erts/emulator/test/map_SUITE_data/badmap_17.erl
+++ b/erts/emulator/test/map_SUITE_data/badmap_17.erl
@@ -1,7 +1,7 @@
 -module(badmap_17).
 -export([update/1]).
 
-%% Compile this source file with OTP 17.
+%% Compile this source file with OTP 17.0.
 
 update(Map) ->
     try
@@ -17,10 +17,42 @@ update(Map) ->
     catch
 	error:{badmap,Map} ->
 	    ok
-    end.
+    end,
+    try
+	update_3(Map),
+	error(update_did_not_fail)
+    catch
+	error:{badmap,Map} ->
+	    ok
+    end,
+    ok = update_4(Map),
+    ok = update_5(Map),
+    ok.
 
 update_1(M) ->
     M#{a=>42}.
 
 update_2(M) ->
     M#{a:=42}.
+
+update_3(M) ->
+    id(M),
+    M#{a=>42}.
+
+update_4(M) when M#{a=>b} =:= M ->
+    did_not_fail;
+update_4(_) ->
+    ok.
+
+update_5(M) ->
+    id(M),
+    case id(true) of
+        true when M#{a=>b} =:= M ->
+            did_not_fail;
+        true ->
+            ok
+    end.
+
+id(I) ->
+    I.
+
-- 
cgit v1.2.3


From 5f56b49c752a16ee28244981ca9b197ffd5fa691 Mon Sep 17 00:00:00 2001
From: Dan Gudmundsson <dgud@erlang.org>
Date: Fri, 2 Jun 2017 14:25:02 +0200
Subject: stdlib: Improve edlin handling of unicode chars

Let edlin handle grapheme clusters instead of codepoints to
improve the handling multi-codepoints characters.

The ttsl driver (and protocol) still expects all lengths as
codepoints.

Previously it was expected that each codepoint used (at least) one
terminal column for each codepoint, and a hack was made for wide
characters (multicolumn) by patching in TAGGED characters to occupy
the extra space so that codepoint index was equal column index.

This didn't work at all for combining codepoints that do not occupy any
more space than the previous character.

Improved this handling by calculating column positions in move_cursor.
This is based on wcwidth() and is not perfect, wcwidth() is wrong for
some codepoints and wcwidth() can not know with Hangul graphemes for
example.  But it works better than before without making a major change
in the protocol.
---
 erts/emulator/drivers/unix/ttsl_drv.c | 93 +++++++++++++++++++----------------
 1 file changed, 50 insertions(+), 43 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/unix/ttsl_drv.c b/erts/emulator/drivers/unix/ttsl_drv.c
index e425b99f16..bce097d944 100644
--- a/erts/emulator/drivers/unix/ttsl_drv.c
+++ b/erts/emulator/drivers/unix/ttsl_drv.c
@@ -108,16 +108,15 @@ static int lbuf_size = BUFSIZ;
 static Uint32 *lbuf;		/* The current line buffer */
 static int llen;		/* The current line length */
 static int lpos;                /* The current "cursor position" in the line buffer */
-
+                                /* NOTE: not the same as column position a char may not take a"
+                                 * column to display or it might take many columns
+                                 */
 /* 
  * Tags used in line buffer to show that these bytes represent special characters,
  * Max unicode is 0x0010ffff, so we have lots of place for meta tags... 
  */
 #define CONTROL_TAG 0x10000000U /* Control character, value in first position */
 #define ESCAPED_TAG 0x01000000U /* Escaped character, value in first position */
-#ifdef HAVE_WCWIDTH
-#define WIDE_TAG    0x02000000U /* Wide character, value in first position    */
-#endif
 #define TAG_MASK    0xFF000000U
 
 #define MAXSIZE (1 << 16)
@@ -156,6 +155,8 @@ static int insert_buf(byte*,int);
 static int write_buf(Uint32 *,int);
 static int outc(int c);
 static int move_cursor(int,int);
+static int cp_pos_to_col(int cp_pos);
+
 
 /* Termcap functions. */
 static int start_termcap(void);
@@ -991,24 +992,26 @@ static int del_chars(int n)
 {
     int i, l, r;
     int pos;
+    int gcs; /* deleted grapheme characters */
 
     update_cols();
 
     /* Step forward or backwards over n logical characters. */
     pos = step_over_chars(n);
-
+    DEBUGLOG(("del_chars: %d from %d %d %d\n", n, lpos, pos, llen));
     if (pos > lpos) {
 	l = pos - lpos;		/* Buffer characters to delete */
 	r = llen - lpos - l;	/* Characters after deleted */
+        gcs = cp_pos_to_col(pos) - cp_pos_to_col(lpos);
 	/* Fix up buffer and buffer pointers. */
 	if (r > 0)
 	    memmove(lbuf + lpos, lbuf + pos, r * sizeof(Uint32));
 	llen -= l;
 	/* Write out characters after, blank the tail and jump back to lpos. */
 	write_buf(lbuf + lpos, r);
-	for (i = l ; i > 0; --i)
+	for (i = gcs ; i > 0; --i)
 	  outc(' ');
-	if (COL(llen+l) == 0 && xn)
+	if (xn && COL(cp_pos_to_col(llen)+gcs) == 0)
 	{
 	   outc(' ');
 	   move_left(1);
@@ -1018,7 +1021,7 @@ static int del_chars(int n)
     else if (pos < lpos) {
 	l = lpos - pos;		/* Buffer characters */
 	r = llen - lpos;	/* Characters after deleted */
-	move_cursor(lpos, lpos-l);	/* Move back */
+	gcs = -move_cursor(lpos, lpos-l);	/* Move back */
 	/* Fix up buffer and buffer pointers. */
 	if (r > 0)
 	    memmove(lbuf + pos, lbuf + lpos, r * sizeof(Uint32));
@@ -1026,14 +1029,14 @@ static int del_chars(int n)
 	llen -= l;
 	/* Write out characters after, blank the tail and jump back to lpos. */
 	write_buf(lbuf + lpos, r);
-	for (i = l ; i > 0; --i)
-	  outc(' ');
-	if (COL(llen+l) == 0 && xn)
+	for (i = gcs ; i > 0; --i)
+          outc(' ');
+        if (xn && COL(cp_pos_to_col(llen)+gcs) == 0)
 	{
-	   outc(' ');
-	   move_left(1);
+          outc(' ');
+          move_left(1);
 	}
-	move_cursor(llen + l, lpos);
+        move_cursor(llen + l, lpos);
     }
     return TRUE;
 }
@@ -1047,22 +1050,12 @@ static int step_over_chars(int n)
     end = lbuf + llen;
     c = lbuf + lpos;
     for ( ; n > 0 && c < end; --n) {
-#ifdef HAVE_WCWIDTH
-	while (*c & WIDE_TAG) {
-	    c++;
-	}
-#endif
 	c++;
 	while (c < end && (*c & TAG_MASK) && ((*c & ~TAG_MASK) == 0))
 	    c++;
     }
     for ( ; n < 0 && c > beg; n++) {
 	--c;
-#ifdef HAVE_WCWIDTH
-	while (c > beg + 1 && (c[-1] & WIDE_TAG)) {
-	    --c;
-	}
-#endif
 	while (c > beg && (*c & TAG_MASK) && ((*c & ~TAG_MASK) == 0))
 	    --c;
     }
@@ -1088,15 +1081,6 @@ static int insert_buf(byte *s, int n)
 	    ++pos;
 	}
 	if ((utf8_mode && (ch >= 128 || isprint(ch))) || (ch <= 255 && isprint(ch))) {
-#ifdef HAVE_WCWIDTH
-	    int width;
-	    if ((width = wcwidth(ch)) > 1) {
-		while (--width) {
-		    DEBUGLOG(("insert_buf: Wide(UTF-8):%d,%d",width,ch));
-		    lbuf[lpos++] = (WIDE_TAG | ((Uint32) ch));
-		}
-	    }
-#endif
 	    DEBUGLOG(("insert_buf: Printable(UTF-8):%d",ch));
 	    lbuf[lpos++] = (Uint32) ch;
 	} else if (ch >= 128) { /* not utf8 mode */
@@ -1204,10 +1188,6 @@ static int write_buf(Uint32 *s, int n)
 	    if (octbuff != octtmp) {
 		driver_free(octbuff);
 	    }
-#ifdef HAVE_WCWIDTH
-	} else if (*s & WIDE_TAG) {
-	    --n; s++;
-#endif
 	} else {
 	    DEBUGLOG(("write_buf: Very unexpected character %d",(int) *s));
 	    ++n;
@@ -1216,7 +1196,7 @@ static int write_buf(Uint32 *s, int n)
     }
     /* Check landed in first column of new line and have 'xn' bug. */
     n = s - lbuf;
-    if (COL(n) == 0 && xn && n != 0) {
+    if (xn && n != 0 && COL(cp_pos_to_col(n)) == 0) {
 	if (n >= llen) {
 	    outc(' ');
 	} else if (lastput == 0) { /* A multibyte UTF8 character */
@@ -1246,14 +1226,19 @@ static int outc(int c)
     return 1;
 }
 
-static int move_cursor(int from, int to)
+static int move_cursor(int from_pos, int to_pos)
 {
+    int from_col, to_col;
     int dc, dl;
-
     update_cols();
 
-    dc = COL(to) - COL(from);
-    dl = LINE(to) - LINE(from);
+    from_col = cp_pos_to_col(from_pos);
+    to_col = cp_pos_to_col(to_pos);
+
+    dc = COL(to_col) - COL(from_col);
+    dl = LINE(to_col) - LINE(from_col);
+    DEBUGLOG(("move_cursor: from %d %d to %d %d => %d %d\n",
+              from_pos, from_col, to_pos, to_col, dl, dc));
     if (dl > 0)
       move_down(dl);
     else if (dl < 0)
@@ -1262,7 +1247,29 @@ static int move_cursor(int from, int to)
       move_right(dc);
     else if (dc < 0)
       move_left(-dc);
-    return TRUE;
+    return to_col-from_col;
+}
+
+static int cp_pos_to_col(int cp_pos)
+{
+#ifdef HAVE_WCWIDTH
+    int i;
+    int col = 0;
+
+    for (i = 0; i < cp_pos; i++) {
+        int w = wcwidth(lbuf[i]);
+        if (w > 0) {
+            col += w;
+        }
+    }
+    return col;
+#else
+    /*
+     * We dont' have any character width information. Assume that
+     * code points are one column wide.
+     */
+    return cp_pos;
+#endif
 }
 
 static int start_termcap(void)
-- 
cgit v1.2.3


From 6a179d28022b46c287a3e075561f97aa85208e31 Mon Sep 17 00:00:00 2001
From: Glauber Campinho <glauber.campinho@soundcloud.com>
Date: Thu, 17 Aug 2017 02:23:22 +0200
Subject: Fix ANSI support in the console

The ANSI support doesn't work properly with edlin, the issue can be noticed when you try to use the history of the shell and the prompt prefix has ANSI (https://github.com/elixir-lang/elixir/issues/6448). The problem is that when a `\e` character appears, it handles it like a new line, dropping the buffer before it.

The solution is to always add the `\e` to the buffer like a regular character and handle it when writing the buffer instead.
---
 erts/emulator/drivers/unix/ttsl_drv.c | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/unix/ttsl_drv.c b/erts/emulator/drivers/unix/ttsl_drv.c
index bce097d944..f3c1aa1c4a 100644
--- a/erts/emulator/drivers/unix/ttsl_drv.c
+++ b/erts/emulator/drivers/unix/ttsl_drv.c
@@ -1094,15 +1094,13 @@ static int insert_buf(byte *s, int n)
 		lbuf[lpos++] = (CONTROL_TAG | ((Uint32) ch));
 		ch = 0;
 	    } while (lpos % 8);
-	} else if (ch == '\e' || ch == '\n' || ch == '\r') {
+	} else if (ch == '\e') {
+	    lbuf[lpos++] = (CONTROL_TAG | ((Uint32) ch));
+	} else if (ch == '\n' || ch == '\r') {
 	    write_buf(lbuf + buffpos, lpos - buffpos);
-            if (ch == '\e') {
-                outc('\e');
-            } else {
                 outc('\r');
                 if (ch == '\n')
                     outc('\n');
-            }
 	    if (llen > lpos) {
 		memcpy(lbuf, lbuf + lpos, llen - lpos);
 	    }
@@ -1150,14 +1148,17 @@ static int write_buf(Uint32 *s, int n)
 	    }
 	    --n;
 	    ++s;
-	}
-	else if (*s == (CONTROL_TAG | ((Uint32) '\t'))) {
+	} else if (*s == (CONTROL_TAG | ((Uint32) '\t'))) {
 	    outc(lastput = ' ');
 	    --n; s++;
 	    while (n > 0 && *s == CONTROL_TAG) {
 		outc(lastput = ' ');
 		--n; s++;
 	    }
+	} else if (*s == (CONTROL_TAG | ((Uint32) '\e'))) {
+	    outc('\e');
+	    --n;
+	    ++s;
 	} else if (*s & CONTROL_TAG) {
 	    outc('^');
 	    outc(lastput = ((byte) ((*s == 0177) ? '?' : *s | 0x40)));
-- 
cgit v1.2.3


From 65d137fd73696604817a866a72a868a65498b500 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 22 Aug 2017 07:14:38 +0200
Subject: Add missing -no_next for badarg instruction

---
 erts/emulator/beam/instrs.tab | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index d45da62d03..1af01e53bd 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -814,6 +814,7 @@ is_ge(Fail, X, Y) {
 
 badarg(Fail) {
     $BADARG($Fail);
+    //| -no_next;
 }
 
 badmatch(Src) {
-- 
cgit v1.2.3


From 31f43558f0b6b1652b4d3be96fe83a616e070a0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 22 Aug 2017 07:15:55 +0200
Subject: beam_makeops: Remove unused subroutine save_c_code

---
 erts/emulator/utils/beam_makeops | 5 -----
 1 file changed, 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index da69b13e87..f9e22444f3 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -278,11 +278,6 @@ my $c_code_block;
 my $c_code_loc;
 my @c_args;
 
-sub save_c_code {
-    my($name,$block,$loc,@args) = @_;
-
-}
-
 while (<>) {
     my($op_num);
     if ($in_c_code) {
-- 
cgit v1.2.3


From 83d5a3cfb5e55e6877ddecf3d33dccf0d3197520 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 16 Aug 2017 07:03:13 +0200
Subject: beam_emu: Remove unused macros

---
 erts/emulator/beam/beam_emu.c | 7 -------
 1 file changed, 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index e5935f5f02..8ae5c23e0d 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -154,9 +154,6 @@ do {                                     \
  */
 
 #define REG_TARGET_PTR(Target) (((Target) & 1) ? &yb(Target-1) : &xb(Target))
-#define REG_TARGET(Target) (*REG_TARGET_PTR(Target))
-
-#define ISCATCHEND(instr) ((Eterm *) *(instr) == OpCode(catch_end_y))
 
 /*
  * Special Beam instructions.
@@ -318,10 +315,6 @@ void** beam_ops;
 #endif
 
 #define Arg(N)       I[(N)+1]
-#define Next(N)                \
-    I += (N) + 1;              \
-    ASSERT(VALID_INSTR(*I));   \
-    Goto(*I)
 
 #define GetR(pos, tr)				\
    do {						\
-- 
cgit v1.2.3


From a9ea42890b6ce8fa35aeba5c6de3a45f97802b23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 14 Aug 2017 15:39:41 +0200
Subject: arith_instrs.tab: Eliminate warning for uninitialized value

Make it clear to GCC that shift_left_count is initialized.
---
 erts/emulator/beam/arith_instrs.tab | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab
index 91fe21e161..b8fbd80dad 100644
--- a/erts/emulator/beam/arith_instrs.tab
+++ b/erts/emulator/beam/arith_instrs.tab
@@ -236,6 +236,7 @@ shift.setup_bsr(Src1, Src2) {
     Op1 = $Src1;
     Op2 = $Src2;
     BIF = BIF_bsr_2;
+    shift_left_count = 0;
     if (is_small(Op2)) {
         shift_left_count = -signed_val(Op2);
     } else if (is_big(Op2)) {
@@ -245,8 +246,6 @@ shift.setup_bsr(Src1, Src2) {
          */
         shift_left_count = make_small(bignum_header_is_neg(*big_val(Op2)) ?
                                       MAX_SMALL : MIN_SMALL);
-    } else {
-        shift_left_count = 0;
     }
 }
 
@@ -254,6 +253,7 @@ shift.setup_bsl(Src1, Src2) {
     Op1 = $Src1;
     Op2 = $Src2;
     BIF = BIF_bsl_2;
+    shift_left_count = 0;
     if (is_small(Op2)) {
         shift_left_count = signed_val(Op2);
     } else if (is_big(Op2)) {
@@ -271,8 +271,6 @@ shift.setup_bsl(Src1, Src2) {
              */
             shift_left_count = MAX_SMALL;
         }
-    } else {
-        shift_left_count = 0;
     }
 }
 
-- 
cgit v1.2.3


From acad85c9831452e5865f2bd3fdd31455b493f091 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 16 Aug 2017 11:41:40 +0200
Subject: Improve performance for bsl/bsr

Eliminate the variable used for holding which BIF (bsl or bsr).
It seems to improve performance slightly.
---
 erts/emulator/beam/arith_instrs.tab | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab
index b8fbd80dad..a5c84d41d6 100644
--- a/erts/emulator/beam/arith_instrs.tab
+++ b/erts/emulator/beam/arith_instrs.tab
@@ -229,13 +229,11 @@ shift.head() {
     Sint ires;
     Eterm* bigp;
     Eterm tmp_big[2];
-    Uint BIF;
 }
 
 shift.setup_bsr(Src1, Src2) {
     Op1 = $Src1;
     Op2 = $Src2;
-    BIF = BIF_bsr_2;
     shift_left_count = 0;
     if (is_small(Op2)) {
         shift_left_count = -signed_val(Op2);
@@ -252,7 +250,6 @@ shift.setup_bsr(Src1, Src2) {
 shift.setup_bsl(Src1, Src2) {
     Op1 = $Src1;
     Op2 = $Src2;
-    BIF = BIF_bsl_2;
     shift_left_count = 0;
     if (is_small(Op2)) {
         shift_left_count = signed_val(Op2);
@@ -279,8 +276,7 @@ shift.execute(Fail, Live, Dst) {
         ires = signed_val(Op1);
         if (shift_left_count == 0 || ires == 0) {
             if (is_not_integer(Op2)) {
-                c_p->freason = BADARITH;
-                $BIF_ERROR_ARITY_2($Fail, BIF, Op1, Op2);
+                goto shift_error;
             }
             if (ires == 0) {
                 $Dst = Op1;
@@ -309,8 +305,7 @@ shift.execute(Fail, Live, Dst) {
     } else if (is_big(Op1)) {
         if (shift_left_count == 0) {
             if (is_not_integer(Op2)) {
-                c_p->freason = BADARITH;
-                $BIF_ERROR_ARITY_2($Fail, BIF, Op1, Op2);
+                goto shift_error;
             }
             $Dst = Op1;
             $NEXT0();
@@ -367,8 +362,22 @@ shift.execute(Fail, Live, Dst) {
     /*
      * One or more non-integer arguments.
      */
+ shift_error:
     c_p->freason = BADARITH;
-    $BIF_ERROR_ARITY_2($Fail, BIF, Op1, Op2);
+    if ($Fail) {
+        $FAIL($Fail);
+    } else {
+        reg[0] = Op1;
+        reg[1] = Op2;
+        SWAPOUT;
+        if (I[0] == (BeamInstr) OpCode(i_bsl_ssjId)) {
+            I = handle_error(c_p, I, reg, &bif_export[BIF_bsl_2]->info.mfa);
+        } else {
+            ASSERT(I[0] == (BeamInstr) OpCode(i_bsr_ssjId));
+            I = handle_error(c_p, I, reg, &bif_export[BIF_bsr_2]->info.mfa);
+        }
+        goto post_error_handling;
+    }
 }
 
 i_int_bnot(Fail, Src, Live, Dst) {
-- 
cgit v1.2.3


From 961f96e9c7c4ad0b64bc6c88700c6eedd30162b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 18 Aug 2017 15:50:30 +0200
Subject: beam_makeops: Prevent truncation when packing 'I' values

BEAM_WIDE_MASK covered the 16 right-most bits, instead of the 32
right-most bits. This bug will bite us when we'll do more packing in
the future.

This bug has been harmless in the past. It has been used in
test_heap and allocate instructions for the number of heap words
needed. It would be theoretically possible to construct a program
that would need 65536 or more heap words, but it is hard to imagine
a practical use for such a program. (The program would have to build
a tuple or list with at least one variable and the rest of the elements
being literals.)
---
 erts/emulator/utils/beam_makeops | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index f9e22444f3..c3ff7bf31f 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -705,7 +705,7 @@ sub emulator_output {
 	print "#if !defined(ARCH_64)\n";
 	print qq[  #error "64-bit architecture assumed, but ARCH_64 not defined"\n];
 	print "#endif\n";
-	print "#define BEAM_WIDE_MASK 0xFFFFUL\n";
+	print "#define BEAM_WIDE_MASK 0xFFFFFFFFUL\n";
 	print "#define BEAM_LOOSE_MASK 0xFFFFUL\n";
 	print "#define BEAM_TIGHT_MASK 0xFFFFUL\n";
 	print "#define BEAM_WIDE_SHIFT 32\n";
-- 
cgit v1.2.3


From f840659e84d16641ca4d821a5928fc8bd2a5c654 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 16 Aug 2017 16:09:05 +0200
Subject: beam_makeops: Add an additional sanity check

If a type has a size in %arg_size, it should also have
a defined pattern in %bit_type.
---
 erts/emulator/utils/beam_makeops | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index c3ff7bf31f..9775a01b89 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -228,6 +228,12 @@ $match_engine_ops{'TOP_fail'} = 1;
 	sanity("tag '$tag': primitive tags must be named with lowercase letters")
 	    unless $tag =~ /^[a-z]$/;
     }
+
+    foreach my $tag (keys %arg_size) {
+        defined $type_bit{$tag} or
+            sanity("the tag '$tag' has a size in %arg_size, " .
+                   "but has no defined bit pattern");
+    }
 }
 
 #
-- 
cgit v1.2.3


From 231ccd91c27dd51fe0e1da3c4ca0484bfe88a018 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 16 Aug 2017 16:15:43 +0200
Subject: beam_makeops: Remove the unused aliases 'N' and 'U'

I don't remember what they were used for, but they are certainly
no longer used.
---
 erts/emulator/utils/beam_makeops | 2 --
 1 file changed, 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 9775a01b89..bbc487dd61 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -200,8 +200,6 @@ sub define_type_bit {
     define_type_bit('A', $type_bit{'u'});
     define_type_bit('L', $type_bit{'u'});
     define_type_bit('b', $type_bit{'u'});
-    define_type_bit('N', $type_bit{'u'});
-    define_type_bit('U', $type_bit{'u'});
     define_type_bit('e', $type_bit{'u'});
     define_type_bit('P', $type_bit{'u'});
     define_type_bit('Q', $type_bit{'u'});
-- 
cgit v1.2.3


From 7f7905f653170daf8a185636329701486fec4ad8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 21 Aug 2017 06:34:20 +0200
Subject: Use the wait_timeout_{un}locked_int instructions

The transformations were incorrect.
---
 erts/emulator/beam/beam_load.c | 14 +++++++-------
 erts/emulator/beam/ops.tab     |  5 +++--
 2 files changed, 10 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index dcd312f54f..2ab91926ad 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -3420,7 +3420,7 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
     Sint timeout;
 
     NEW_GENOP(stp, op);
-    op->op = genop_wait_timeout_unlocked_2;
+    op->op = genop_wait_timeout_unlocked_int_2;
     op->next = NULL;
     op->arity = 2;
     op->a[0].type = TAG_u;
@@ -3467,12 +3467,12 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
     Sint timeout;
 
     NEW_GENOP(stp, op);
-    op->op = genop_wait_timeout_locked_2;
+    op->op = genop_wait_timeout_locked_int_2;
     op->next = NULL;
     op->arity = 2;
-    op->a[0] = Fail;
-    op->a[1].type = TAG_u;
-    
+    op->a[0].type = TAG_u;
+    op->a[1] = Fail;
+
     if (Time.type == TAG_i && (timeout = Time.val) >= 0 &&
 #if defined(ARCH_64)
 	(timeout >> 32) == 0
@@ -3480,7 +3480,7 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
 	1
 #endif
 	) {
-	op->a[1].val = timeout;
+	op->a[0].val = timeout;
 #if !defined(ARCH_64)
     } else if (Time.type == TAG_q) {
 	Eterm big;
@@ -3494,7 +3494,7 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
 	} else {
 	    Uint u;
 	    (void) term_to_Uint(big, &u);
-	    op->a[1].val = (BeamInstr) u;
+	    op->a[0].val = (BeamInstr) u;
 	}
 #endif
     } else {
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 92e67bb470..955eecd115 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -382,8 +382,8 @@ label L | wait_timeout Fail Src | smp_already_locked(L) => \
     label L | wait_timeout_locked Src Fail
 wait_timeout Fail Src => wait_timeout_unlocked Src Fail
 
-wait_timeout_unlocked Fail Src=aiq => gen_literal_timeout(Fail, Src)
-wait_timeout_locked Fail Src=aiq => gen_literal_timeout_locked(Fail, Src)
+wait_timeout_unlocked Src=aiq Fail => gen_literal_timeout(Fail, Src)
+wait_timeout_locked Src=aiq Fail => gen_literal_timeout_locked(Fail, Src)
 
 label L | wait Fail | smp_already_locked(L) => label L | wait_locked Fail
 wait Fail => wait_unlocked Fail
@@ -398,6 +398,7 @@ loop_rec_end f
 wait_locked f
 wait_unlocked f
 
+# Note that a timeout value must fit in 32 bits.
 wait_timeout_unlocked_int I f
 wait_timeout_unlocked s f
 wait_timeout_locked_int I f
-- 
cgit v1.2.3


From cbeeed095739223a425649f6085b6959ad905c83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 17 Aug 2017 06:49:22 +0200
Subject: beam_makeops: Introduce the new type 'W' (machine word)

As a preparation for potentially improving packing in the future,
we will need to make sure that packable types have a defined maximum
size.

The packer algorithm assumes that two 'I' operands can be packed
into one 64-bit word, but there are instructions that use an 'I'
operand to store a pointer. It only works because those instructions
are not packed for other reasons.

Introduce the 'W' type and use it for operands that don't fit in
32 bits.
---
 erts/emulator/beam/beam_debug.c    | 15 +++++++-------
 erts/emulator/beam/beam_load.c     | 33 +++++++++++++++++++++++-------
 erts/emulator/beam/ops.tab         | 42 ++++++++++++++++++++------------------
 erts/emulator/test/tuple_SUITE.erl |  7 +++++++
 erts/emulator/utils/beam_makeops   |  8 +++++---
 5 files changed, 68 insertions(+), 37 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index afe87288ce..15b18f0667 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -522,12 +522,13 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	    }
 	    ap++;
 	    break;
-	case 'I':		/* Untagged integer. */
-	case 't':
+	case 't':               /* Untagged integers */
+	case 'I':
+        case 'W':
 	    switch (op) {
-	    case op_i_gc_bif1_jIsId:
-	    case op_i_gc_bif2_jIIssd:
-	    case op_i_gc_bif3_jIIssd:
+	    case op_i_gc_bif1_jWsId:
+	    case op_i_gc_bif2_jWIssd:
+	    case op_i_gc_bif3_jWIssd:
 		{
 		    const ErtsGcBif* p;
 		    BifFunction gcf = (BifFunction) *ap;
@@ -672,8 +673,8 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
             }
         }
         break;
-    case op_i_jump_on_val_xfII:
-    case op_i_jump_on_val_yfII:
+    case op_i_jump_on_val_xfIW:
+    case op_i_jump_on_val_yfIW:
 	{
 	    int n;
 	    for (n = ap[-2]; n > 0; n--) {
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 2ab91926ad..98701db558 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2388,11 +2388,29 @@ load_code(LoaderState* stp)
 		    break;
 		}
 		break;
-	    case 'I':	/* Untagged integer (or pointer). */
-		VerifyTag(stp, tag, TAG_u);
-		code[ci++] = tmp_op->a[arg].val;
-		break;
-	    case 't':	/* Small untagged integer -- can be packed. */
+	    case 't':	/* Small untagged integer (16 bits) -- can be packed. */
+	    case 'I':	/* Untagged integer (32 bits) -- can be packed.  */
+	    case 'W':	/* Untagged integer or pointer (machine word). */
+#ifdef DEBUG
+                switch (*sign) {
+                case 't':
+                    if (tmp_op->a[arg].val >> 16 != 0) {
+                        load_printf(__LINE__, stp, "value %lu of type 't' does not fit in 16 bits",
+                                    tmp_op->a[arg].val);
+                        ASSERT(0);
+                    }
+                    break;
+#ifdef ARCH_64
+                case 'I':
+                    if (tmp_op->a[arg].val >> 32 != 0) {
+                        load_printf(__LINE__, stp, "value %lu of type 'I' does not fit in 32 bits",
+                                    tmp_op->a[arg].val);
+                        ASSERT(0);
+                    }
+                    break;
+#endif
+                }
+#endif
 		VerifyTag(stp, tag, TAG_u);
 		code[ci++] = tmp_op->a[arg].val;
 		break;
@@ -2627,8 +2645,8 @@ load_code(LoaderState* stp)
 	    /* Remember offset for the on_load function. */
 	    stp->on_load = ci;
 	    break;
-	case op_bs_put_string_II:
-	case op_i_bs_match_string_xfII:
+	case op_bs_put_string_WW:
+	case op_i_bs_match_string_xfWW:
 	    new_string_patch(stp, ci-1);
 	    break;
 
@@ -2884,6 +2902,7 @@ gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index,
     op->next = NULL;
 
     if (Index.type == TAG_i && Index.val > 0 &&
+        Index.val <= ERTS_MAX_TUPLE_SIZE &&
 	(Tuple.type == TAG_x || Tuple.type == TAG_y)) {
 	op->op = genop_i_fast_element_4;
 	op->a[0] = Tuple;
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 955eecd115..09fad05ffd 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -167,7 +167,7 @@ i_select_tuple_arity2 xy f A A f f
 
 i_jump_on_val_zero xy f I
 
-i_jump_on_val xy f I I
+i_jump_on_val xy f I W
 
 get_list xy xy xy
 
@@ -971,6 +971,8 @@ node x
 node y
 %hot
 
+# Note: 'I' is sufficient because this instruction will only be used
+# if the arity fits in 24 bits.
 i_fast_element xy j I d
 
 i_element xy j s d
@@ -1026,7 +1028,7 @@ i_call_fun_last I P
 make_fun2 OldIndex=u => gen_make_fun2(OldIndex)
 
 %cold
-i_make_fun I t
+i_make_fun W t
 %hot
 
 is_function f xy
@@ -1052,14 +1054,14 @@ i_bs_restore2 x I
 # Matching integers
 bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val
 
-i_bs_match_string x f I I
+i_bs_match_string x f W W
 
 # Fetching integers from binaries.
 bs_get_integer2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \
 			gen_get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
 
-i_bs_get_integer_small_imm x I f I x
-i_bs_get_integer_imm x I I f I x
+i_bs_get_integer_small_imm x W f t x
+i_bs_get_integer_imm x W t f t x
 i_bs_get_integer f I I s s x
 i_bs_get_integer_8 x f x
 i_bs_get_integer_16 x f x
@@ -1072,7 +1074,7 @@ i_bs_get_integer_32 x f x
 bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \
 			gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
 
-i_bs_get_binary_imm2 f x I I I x
+i_bs_get_binary_imm2 f x t W t x
 i_bs_get_binary2 f x I s I x
 i_bs_get_binary_all2 f x I I x
 i_bs_get_binary_all_reuse x f I
@@ -1090,14 +1092,14 @@ i_bs_get_float2 f x I s I x
 bs_skip_bits2 Fail=f Ms=x Sz=sq Unit=u Flags=u => \
 			gen_skip_bits2(Fail, Ms, Sz, Unit, Flags)
 
-i_bs_skip_bits_imm2 f x I
+i_bs_skip_bits_imm2 f x W
 i_bs_skip_bits2 f x xy I
 i_bs_skip_bits_all2 f x I
 
 bs_test_tail2 Fail=f Ms=x Bits=u==0 => bs_test_zero_tail2 Fail Ms
 bs_test_tail2 Fail=f Ms=x Bits=u => bs_test_tail_imm2 Fail Ms Bits
 bs_test_zero_tail2 f x
-bs_test_tail_imm2 f x I
+bs_test_tail_imm2 f x W
 
 bs_test_unit F Ms Unit=u==8 => bs_test_unit8 F Ms
 bs_test_unit f x I
@@ -1154,9 +1156,9 @@ i_bs_init_fail xy j I x
 
 i_bs_init_fail_heap s I j I x
 
-i_bs_init I I x
+i_bs_init W t x
 
-i_bs_init_heap I I I x
+i_bs_init_heap W I t x
 
 
 bs_init_bits Fail Sz=o Words Regs Flags Dst => system_limit Fail
@@ -1173,8 +1175,8 @@ i_bs_init_bits_fail xy j I x
 
 i_bs_init_bits_fail_heap s I j I x
 
-i_bs_init_bits I I x
-i_bs_init_bits_heap I I I x
+i_bs_init_bits W t x
+i_bs_init_bits_heap W I t x
 
 bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D
 
@@ -1199,7 +1201,7 @@ bs_put_integer Fail=j Sz=sq Unit=u Flags=u Src=s => \
 			gen_put_integer(Fail, Sz, Unit, Flags, Src)
 
 i_new_bs_put_integer j s I s
-i_new_bs_put_integer_imm j I I s
+i_new_bs_put_integer_imm j W t s
 
 #
 # Utf8/utf16/utf32 support. (R12B-5)
@@ -1233,7 +1235,7 @@ bs_put_float Fail=j Sz=s Unit=u Flags=u Src=s => \
 			gen_put_float(Fail, Sz, Unit, Flags, Src)
 
 i_new_bs_put_float j s I s
-i_new_bs_put_float_imm j I I s
+i_new_bs_put_float_imm j W I s
 
 #
 # Storing binaries into binaries.
@@ -1243,7 +1245,7 @@ bs_put_binary Fail=j Sz=s Unit=u Flags=u Src=s => \
 			gen_put_binary(Fail, Sz, Unit, Flags, Src)
 
 i_new_bs_put_binary j s I s
-i_new_bs_put_binary_imm j I s
+i_new_bs_put_binary_imm j W s
 i_new_bs_put_binary_all j s I
 
 #
@@ -1252,7 +1254,7 @@ i_new_bs_put_binary_all j s I
 # Don't change the instruction format unless you change the loader too.
 #
 
-bs_put_string I I
+bs_put_string W W
 
 #
 # New floating point instructions (R8).
@@ -1450,7 +1452,7 @@ gc_bif2 Fail Live u$bif:erlang:bxor/2 S1 S2 Dst => \
 
 gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst
 
-i_increment rxy I I d
+i_increment rxy W t d
 
 i_plus x xy j I d
 i_plus s s  j I d
@@ -1499,9 +1501,9 @@ gc_bif2 Fail I Bif S1 S2 Dst => \
 gc_bif3 Fail I Bif S1 S2 S3 Dst => \
 	gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst)
 
-i_gc_bif1 j I s I d
+i_gc_bif1 j W s I d
 
-i_gc_bif2 j I I s s d
+i_gc_bif2 j W I s s d
 
 ii_gc_bif3/7
 
@@ -1510,7 +1512,7 @@ ii_gc_bif3/7
 ii_gc_bif3 Fail Bif Live S1 S2 S3 Dst => \
   move S1 x | i_gc_bif3 Fail Bif Live S2 S3 Dst
 
-i_gc_bif3 j I I s s d
+i_gc_bif3 j W I s s d
 
 #
 # The following instruction is specially handled in beam_load.c
diff --git a/erts/emulator/test/tuple_SUITE.erl b/erts/emulator/test/tuple_SUITE.erl
index 79b681b4d1..baf41180e0 100644
--- a/erts/emulator/test/tuple_SUITE.erl
+++ b/erts/emulator/test/tuple_SUITE.erl
@@ -134,6 +134,13 @@ t_element(Config) when is_list(Config) ->
     {'EXIT', {badarg, _}} = (catch element(1, id(42))),
     {'EXIT', {badarg, _}} = (catch element(id(1.5), id({a,b}))),
 
+    %% Make sure that the loader does not reject the module when
+    %% huge literal index values are used.
+    {'EXIT', {badarg, _}} = (catch element((1 bsl 24)-1, id({a,b,c}))),
+    {'EXIT', {badarg, _}} = (catch element(1 bsl 24, id({a,b,c}))),
+    {'EXIT', {badarg, _}} = (catch element(1 bsl 32, id({a,b,c}))),
+    {'EXIT', {badarg, _}} = (catch element(1 bsl 64, id({a,b,c}))),
+
     ok.
 
 get_elements([Element|Rest], Tuple, Pos) ->
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index bbc487dd61..4108138b51 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -149,8 +149,9 @@ my %arg_size = ('r' => 0,	# x(0) - x register zero
 		'j' => 1,	# either 'f' or 'p'
 		'e' => 1,	# pointer to export entry
 		'L' => 0,	# label
-		'I' => 1,	# untagged integer
-		't' => 1,	# untagged integer -- can be packed
+		't' => 1,	# untagged integer (12 bits) -- can be packed
+		'I' => 1,	# untagged integer (32 bits) -- can be packed
+                'W' => 1,       # untagged integer/pointer (one word)
 		'b' => 1,	# pointer to bif
 		'A' => 1,	# arity value
 		'P' => 1,	# byte offset into tuple or stack
@@ -195,8 +196,9 @@ sub define_type_bit {
     define_type_bit('j', $type_bit{'f'} | $type_bit{'p'});
 
     # Aliases (for matching purposes).
-    define_type_bit('I', $type_bit{'u'});
     define_type_bit('t', $type_bit{'u'});
+    define_type_bit('I', $type_bit{'u'});
+    define_type_bit('W', $type_bit{'u'});
     define_type_bit('A', $type_bit{'u'});
     define_type_bit('L', $type_bit{'u'});
     define_type_bit('b', $type_bit{'u'});
-- 
cgit v1.2.3


From 234ef0eb9803d4c4cfb82848e3d3d14d89d1ddf5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 18 Aug 2017 04:06:52 +0200
Subject: Make map update instruction functions indepedent of instruction
 format

Having the helper functions for map update knowing all the details
of operands for the instruction will make it difficult to make
improvements such as better packing.
---
 erts/emulator/beam/beam_debug.c   |  8 +++---
 erts/emulator/beam/beam_emu.c     | 59 +++++++++++++++------------------------
 erts/emulator/beam/map_instrs.tab | 20 +++++++------
 erts/emulator/beam/ops.tab        |  8 +++---
 4 files changed, 42 insertions(+), 53 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 15b18f0667..06b9db2877 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -697,9 +697,9 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	break;
     case op_i_put_tuple_xI:
     case op_i_put_tuple_yI:
-    case op_new_map_dII:
-    case op_update_map_assoc_sdII:
-    case op_update_map_exact_jsdII:
+    case op_new_map_dtI:
+    case op_update_map_assoc_sdtI:
+    case op_update_map_exact_jsdtI:
 	{
 	    int n = unpacked[-1];
 
@@ -719,7 +719,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	    }
 	}
 	break;
-    case op_i_new_small_map_lit_dIq:
+    case op_i_new_small_map_lit_dtq:
         {
             Eterm *tp = tuple_val(unpacked[-1]);
             int n = arityval(*tp);
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8ae5c23e0d..dc242229e5 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -399,12 +399,13 @@ static BeamInstr* apply_fun(Process* p, Eterm fun,
 			    Eterm args, Eterm* reg) NOINLINE;
 static Eterm new_fun(Process* p, Eterm* reg,
 		     ErlFunEntry* fe, int num_free) NOINLINE;
-static Eterm new_map(Process* p, Eterm* reg, BeamInstr* I) NOINLINE;
-static Eterm new_small_map_lit(Process* p, Eterm* reg, Uint* n_exp, BeamInstr* I) NOINLINE;
-static Eterm update_map_assoc(Process* p, Eterm* reg,
-			      Eterm map, BeamInstr* I) NOINLINE;
-static Eterm update_map_exact(Process* p, Eterm* reg,
-			      Eterm map, BeamInstr* I) NOINLINE;
+static Eterm new_map(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* ptr) NOINLINE;
+static Eterm new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal,
+                               Uint live, BeamInstr* ptr) NOINLINE;
+static Eterm update_map_assoc(Process* p, Eterm* reg, Uint live,
+                              Uint n, BeamInstr* new_p) NOINLINE;
+static Eterm update_map_exact(Process* p, Eterm* reg, Uint live,
+                              Uint n, Eterm* new_p) NOINLINE;
 static Eterm get_map_element(Eterm map, Eterm key);
 static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx);
 
@@ -2727,24 +2728,20 @@ do {						\
 
 
 static Eterm
-new_map(Process* p, Eterm* reg, BeamInstr* I)
+new_map(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* ptr)
 {
-    Uint n = Arg(3);
     Uint i;
     Uint need = n + 1 /* hdr */ + 1 /*size*/ + 1 /* ptr */ + 1 /* arity */;
     Eterm keys;
     Eterm *mhp,*thp;
     Eterm *E;
-    BeamInstr *ptr;
     flatmap_t *mp;
     ErtsHeapFactory factory;
 
-    ptr = &Arg(4);
-
     if (n > 2*MAP_SMALL_MAP_LIMIT) {
         Eterm res;
 	if (HeapWordsLeft(p) < n) {
-	    erts_garbage_collect(p, n, reg, Arg(2));
+	    erts_garbage_collect(p, n, reg, live);
 	}
 
 	mhp = p->htop;
@@ -2765,7 +2762,7 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
     }
 
     if (HeapWordsLeft(p) < need) {
-	erts_garbage_collect(p, need, reg, Arg(2));
+	erts_garbage_collect(p, need, reg, live);
     }
 
     thp    = p->htop;
@@ -2788,24 +2785,20 @@ new_map(Process* p, Eterm* reg, BeamInstr* I)
 }
 
 static Eterm
-new_small_map_lit(Process* p, Eterm* reg, Uint* n_exp, BeamInstr* I)
+new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal, Uint live, BeamInstr* ptr)
 {
-    Eterm* keys = tuple_val(Arg(3));
+    Eterm* keys = tuple_val(keys_literal);
     Uint n = arityval(*keys);
     Uint need = n + 1 /* hdr */ + 1 /*size*/ + 1 /* ptr */ + 1 /* arity */;
     Uint i;
-    BeamInstr *ptr;
     flatmap_t *mp;
     Eterm *mhp;
     Eterm *E;
 
-    *n_exp = n;
-    ptr = &Arg(4);
-
     ASSERT(n <= MAP_SMALL_MAP_LIMIT);
 
     if (HeapWordsLeft(p) < need) {
-        erts_garbage_collect(p, need, reg, Arg(2));
+        erts_garbage_collect(p, need, reg, live);
     }
 
     mhp = p->htop;
@@ -2814,7 +2807,7 @@ new_small_map_lit(Process* p, Eterm* reg, Uint* n_exp, BeamInstr* I)
     mp = (flatmap_t *)mhp; mhp += MAP_HEADER_FLATMAP_SZ;
     mp->thing_word = MAP_HEADER_FLATMAP;
     mp->size = n;
-    mp->keys = Arg(3);
+    mp->keys = keys_literal;
 
     for (i = 0; i < n; i++) {
         GET_TERM(*ptr++, *mhp++);
@@ -2826,9 +2819,8 @@ new_small_map_lit(Process* p, Eterm* reg, Uint* n_exp, BeamInstr* I)
 }
 
 static Eterm
-update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
+update_map_assoc(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* new_p)
 {
-    Uint n;
     Uint num_old;
     Uint num_updates;
     Uint need;
@@ -2838,12 +2830,12 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     Eterm* E;
     Eterm* old_keys;
     Eterm* old_vals;
-    BeamInstr* new_p;
     Eterm new_key;
     Eterm* kp;
+    Eterm map;
 
-    new_p = &Arg(4);
-    num_updates = Arg(3) / 2;
+    num_updates = n / 2;
+    map = reg[live];
 
     if (is_not_flatmap(map)) {
 	Uint32 hx;
@@ -2873,7 +2865,7 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
      */
 
     if (num_old == 0) {
-	return new_map(p, reg, I);
+	return new_map(p, reg, live, n, new_p);
     }
 
     /*
@@ -2883,8 +2875,6 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 
     need = 2*(num_old+num_updates) + 1 + MAP_HEADER_FLATMAP_SZ;
     if (HeapWordsLeft(p) < need) {
-	Uint live = Arg(2);
-	reg[live] = map;
 	erts_garbage_collect(p, need, reg, live+1);
 	map      = reg[live];
 	old_mp   = (flatmap_t *)flatmap_val(map);
@@ -3031,9 +3021,8 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
  */
 
 static Eterm
-update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
+update_map_exact(Process* p, Eterm* reg, Uint live, Uint n, Eterm* new_p)
 {
-    Uint n;
     Uint i;
     Uint num_old;
     Uint need;
@@ -3043,12 +3032,12 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
     Eterm* E;
     Eterm* old_keys;
     Eterm* old_vals;
-    BeamInstr* new_p;
     Eterm new_key;
+    Eterm map;
 
-    new_p = &Arg(5);
-    n = Arg(4) / 2;		/* Number of values to be updated */
+    n /= 2;		/* Number of values to be updated */
     ASSERT(n > 0);
+    map = reg[live];
 
     if (is_not_flatmap(map)) {
 	Uint32 hx;
@@ -3102,8 +3091,6 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 
     need = num_old + MAP_HEADER_FLATMAP_SZ;
     if (HeapWordsLeft(p) < need) {
-	Uint live = Arg(3);
-	reg[live] = map;
 	erts_garbage_collect(p, need, reg, live+1);
 	map      = reg[live];
 	old_mp   = (flatmap_t *)flatmap_val(map);
diff --git a/erts/emulator/beam/map_instrs.tab b/erts/emulator/beam/map_instrs.tab
index 30c3d7743f..bbb2f49b66 100644
--- a/erts/emulator/beam/map_instrs.tab
+++ b/erts/emulator/beam/map_instrs.tab
@@ -31,22 +31,24 @@ new_map(Dst, Live, N) {
     Eterm res;
 
     HEAVY_SWAPOUT;
-    res = new_map(c_p, reg, I-1);
+    res = new_map(c_p, reg, $Live, $N, $NEXT_INSTRUCTION);
     HEAVY_SWAPIN;
     $REFRESH_GEN_DEST();
     $Dst = res;
     $NEXT($NEXT_INSTRUCTION+$N);
 }
 
-i_new_small_map_lit(Dst, Live, Literal) {
+i_new_small_map_lit(Dst, Live, Keys) {
     Eterm res;
     Uint n;
+    Eterm keys = $Keys;
 
     HEAVY_SWAPOUT;
-    res = new_small_map_lit(c_p, reg, &n, I-1);
+    res = new_small_map_lit(c_p, reg, keys, $Live, $NEXT_INSTRUCTION);
     HEAVY_SWAPIN;
     $REFRESH_GEN_DEST();
     $Dst = res;
+    n = arityval(*tuple_val(keys));
     $NEXT($NEXT_INSTRUCTION+n);
 }
 
@@ -127,11 +129,11 @@ i_get_map_elements(Fail, Src, N) {
 
 update_map_assoc(Src, Dst, Live, N) {
     Eterm res;
-    Eterm map;
+    Uint live = $Live;
 
-    map = $Src;
+    reg[live] = $Src;
     HEAVY_SWAPOUT;
-    res = update_map_assoc(c_p, reg, map, I);
+    res = update_map_assoc(c_p, reg, live, $N, $NEXT_INSTRUCTION);
     HEAVY_SWAPIN;
     ASSERT(is_value(res));
     $REFRESH_GEN_DEST();
@@ -141,11 +143,11 @@ update_map_assoc(Src, Dst, Live, N) {
 
 update_map_exact(Fail, Src, Dst, Live, N) {
     Eterm res;
-    Eterm map;
+    Uint live = $Live;
 
-    map = $Src;
+    reg[live] = $Src;
     HEAVY_SWAPOUT;
-    res = update_map_exact(c_p, reg, map, I);
+    res = update_map_exact(c_p, reg, live, $N, $NEXT_INSTRUCTION);
     HEAVY_SWAPIN;
     if (is_value(res)) {
         $REFRESH_GEN_DEST();
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 09fad05ffd..deee1a4de9 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1357,10 +1357,10 @@ sorted_put_map_exact F Src Dst Live Size Rest=* => \
 new_map Dst Live Size Rest=* | is_small_map_literal_keys(Size, Rest) => \
    gen_new_small_map_lit(Dst, Live, Size, Rest)
 
-new_map d I I
-i_new_small_map_lit d I q
-update_map_assoc s d I I
-update_map_exact j s d I I
+new_map d t I
+i_new_small_map_lit d t q
+update_map_assoc s d t I
+update_map_exact j s d t I
 
 is_map Fail Lit=q | literal_is_map(Lit) =>
 is_map Fail cq => jump Fail
-- 
cgit v1.2.3


From 8f2646de9a74d82d8527782bdb58584c60e99742 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 17 Aug 2017 11:59:26 +0200
Subject: beam_makeops: Rewrite the packer, fixing several bugs

The packer had several bugs and limitations. For instance, on
a 32-bit Erlang virtual machine it would gladly pack three
't' values into one word even though it would be not safe.

The rewritten version will be more careful how much it packs
into each word. It will also be able to do packing for more
instructions.
---
 erts/emulator/beam/beam_emu.c    |   1 +
 erts/emulator/utils/beam_makeops | 259 +++++++++++++++++++++++++--------------
 2 files changed, 166 insertions(+), 94 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index dc242229e5..badfe62aa9 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -238,6 +238,7 @@ void** beam_ops;
      PROCESS_MAIN_CHK_LOCKS((P));					\
      ERTS_UNREQ_PROC_MAIN_LOCK((P))
 
+#define db(N) (N)
 #define tb(N) (N)
 #define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N)))
 #define yb(N) (*(Eterm *) (((unsigned char *)E) + (N)))
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 4108138b51..2a957d2c9d 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -54,11 +54,6 @@ $pack_mask[4]  = ['BEAM_LOOSE_MASK', # Only for 64 bit wordsize
 		  'BEAM_LOOSE_MASK',
 		  $WHOLE_WORD];
 
-# Mapping from packagable arguments to number of packed arguments per
-# word.  Initialized after the wordsize is known.
-
-my @args_per_word;
-
 # There are two types of instructions: generic and specific.
 # The generic instructions are those generated by the Beam compiler.
 # Corresponding to each generic instruction, there is generally a
@@ -264,15 +259,8 @@ if ($wordsize == 32) {
 # Initialize number of arguments per packed word.
 #
 
-$args_per_word[2] = 2;
-$args_per_word[3] = 3;
-$args_per_word[4] = 2;
-$args_per_word[5] = 3;
-$args_per_word[6] = 3;
-
 if ($wordsize == 64) {
     $pack_mask[3] = ['BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK', $WHOLE_WORD];
-    $args_per_word[4] = 4;
 }
 
 #
@@ -1240,7 +1228,7 @@ sub basic_generator {
 
     my $c_code_ref = $c_code{$name};
     if ($hot and defined $c_code_ref) {
-        ($prefix, $pack_spec, @args) = do_pack(@args);
+        ($var_decls, $pack_spec, @args) = do_pack(@args);
     }
 
     #
@@ -1254,7 +1242,14 @@ sub basic_generator {
 	my($this_size) = $arg_size{$_};
       SWITCH:
 	{
-	    /^pack:(\d):(.*)/ and do {
+	    /^packed:d:(\d):(.*)/ and do {
+                $var_decls .= "Eterm dst = $2;\n" .
+                    "Eterm* dst_ptr = REG_TARGET_PTR(dst);\n";
+                push(@f, "*dst_ptr");
+                $this_size = $1;
+                last SWITCH;
+            };
+	    /^packed:[a-zA-z]:(\d):(.*)/ and do {
                 push(@f, $2);
                 $this_size = $1;
                 last SWITCH;
@@ -1458,29 +1453,28 @@ sub expand_macro {
 sub do_pack {
     my(@args) = @_;
     my($packable_args) = 0;
-    my @is_packable;		# Packability (boolean) for each argument.
-    my $wide_packing = 0;
-    my(@orig_args) = @args;
+    my @bits_needed;            # Bits needed for each argument.
 
     #
-    # Count the number of packable arguments.  If we encounter any 's' or 'd'
-    # arguments, packing is not possible.
+    # Define the minimum number of bits needed for the packable argument types.
+    #
+    my %bits_needed = ('x' => 10,
+                       'y' => 10,
+                       'Q' => 10,
+                       'l' => 10,
+                       'd' => 16,
+                       't' => 16);
+    if ($wordsize == 64) {
+        $bits_needed{'I'} = 32;
+    }
+
+    #
+    # Count the number of packable arguments.
     #
-    my $packable_types = "xytQ";
     foreach my $arg (@args) {
-	if ($arg =~ /^[$packable_types]/) {
+        if (defined $bits_needed{$arg}) {
 	    $packable_args++;
-	    push @is_packable, 1;
-	} elsif ($arg =~ /^I/ and $wordsize == 64 and $packable_args < 2) {
-	    $wide_packing = 1;
-	    push @is_packable, 1;
-	    if (++$packable_args == 2) {
-		# We can only pack two arguments. Turn off packing
-		# for the rest of the arguments.
-		$packable_types = "\xFF";
-	    }
-	} elsif ($arg =~ /^[sd]/) {
-	    return ('', '', @args);
+            push @bits_needed, $bits_needed{$arg};
 	} elsif ($arg =~ /^[scq]/ and $packable_args > 0) {
 	    # When packing, this operand will be picked up from the
 	    # code array, put onto the packing stack, and later put
@@ -1494,87 +1488,164 @@ sub do_pack {
 	    # just turn off packing.
 	    return ('', '', @args);
 	} else {
-	    push @is_packable, 0;
+	    push @bits_needed, 0;
 	}
     }
 
     #
-    # Get out of here if too few or too many arguments.
+    # Nothing to pack unless there are at least 2 packable arguments.
     #
     return ('', '', @args) if $packable_args < 2;
 
-    my($size) = 0;
-    my($pack_prefix) = '';
-    my($down) = '';		# Pack commands (towards instruction
+    #
+    # Determine how many arguments we should pack into each word.
+    #
+    my @args_per_word;
+    my @need_wide_mask;
+    my $bits = 0;
+    my $word = 0;
+    $args_per_word[0] = 0;
+    $need_wide_mask[0] = 0;
+    for (my $i = 0; $i < @args; $i++) {
+        if ($bits_needed[$i]) {
+            my $needed = $bits_needed[$i];
+
+            my $next_word = sub {
+                $word++;
+                $args_per_word[$word] = 0;
+                $need_wide_mask[$word] = 0;
+                $bits = 0;
+            };
+
+            if ($bits+$needed > $wordsize) { # Does not fit.
+                $next_word->();
+            }
+            if ($args_per_word[$word] == 4) { # Can't handle more than 4 args.
+                $next_word->();
+            }
+            if ($needed == 32 and $args_per_word[$word] > 1) {
+                # Must only pack two arguments in this word, and there
+                # are already at least two arguments here.
+                $next_word->();
+            }
+            $args_per_word[$word]++;
+            $bits += $needed;
+            if ($needed == 32) {
+                $need_wide_mask[$word]++;
+            }
+            if ($need_wide_mask[$word] and $bits > 32) {
+                # Can only pack two things in a word where one
+                # item is 32 bits. Force the next item into
+                # the next word.
+                $bits = $wordsize;
+            }
+        }
+    }
+
+    #
+    # Try to balance packing between words.
+    #
+    if ($args_per_word[$#args_per_word] == 1) {
+        if ($args_per_word[$#args_per_word-1] < 3) {
+            pop @args_per_word;
+        } else {
+            $args_per_word[$#args_per_word-1]--;
+            $args_per_word[$#args_per_word]++;
+        }
+    } elsif (@args_per_word == 2 and
+             $args_per_word[0] == 4 and
+             $args_per_word[1] == 2) {
+        $args_per_word[0] = 3;
+        $args_per_word[1] = 3;
+    } elsif (@args_per_word == 2 and
+             $args_per_word[0] == 3 and
+             $args_per_word[1] == 1) {
+        $args_per_word[0] = 2;
+        $args_per_word[1] = 2;
+    }
+
+    my $size = 0;
+    my $pack_prefix = '';
+    my $down = '';		# Pack commands (towards instruction
 				# beginning).
-    my($up) = '';		# Pack commands (storing back while
+    my $up = '';		# Pack commands (storing back while
 				# moving forward).
+    my $did_some_packing = 0;   # Nothing packed yet.
 
-    my $args_per_word = $args_per_word[$packable_args];
-    my @shift;
-    my @mask;
-    my @instr;
+    # Skip an unpackable argument.
+    my $skip_unpackable = sub {
+        my($arg) = @_;
 
-    if ($wide_packing) {
-	@shift = ('0', 'BEAM_WIDE_SHIFT');
-	@mask = ('BEAM_WIDE_MASK', $WHOLE_WORD);
-	@instr = ('w', 'i');
-    } else {
-	@shift = @{$pack_shift[$args_per_word]};
-	@mask = @{$pack_mask[$args_per_word]};
-	@instr = @{$pack_instr[$args_per_word]};
-    }
+        if ($arg_size{$arg} and $did_some_packing) {
+            # Save the argument on the pack engine's stack.
+            $down = "g${down}";
+            $up = "${up}p";
+        } else {
+            # The argument has either zero size (e.g. r(0)),
+            # or is to the left of the first packed argument
+            # and will never be accessed. No need to do
+            # anything.
+        }
+    };
 
     #
     # Now generate the packing instructions.  One complication is that
     # the packing engine works from right-to-left, but we must generate
     # the instructions from left-to-right because we must calculate
     # instruction sizes from left-to-right.
-    #
-    # XXX Packing 3 't's in one word won't work.  Sorry.
 
-    my $did_some_packing = 0;	# Nothing packed yet.
-    my($ap) = 0;		# Argument number within word.
-    my($tmpnum) = 1;		# Number of temporary variable.
-    my($expr) = '';
-    for (my $i = 0; $i < @args; $i++) {
-	my($reg) = $args[$i];
-	my($this_size) = $arg_size{$reg};
-	if ($is_packable[$i]) {
-	    $this_size = 0;
-	    $did_some_packing = 1;
-
-	    if ($ap == 0) {
-		$pack_prefix .= "Eterm tmp_packed$tmpnum = Arg($size);\n";
-		$up .= "p";
-		$down = "P$down";
-		$this_size = 1;
-	    }
+    my $arg_num = 0;
+    for (my $word = 0; $word < @args_per_word; $word++) {
+        my $ap = 0;              # Argument number within word.
+        my $packed_var = "tmp_packed" . ($word+1);
+        my $args_per_word = $args_per_word[$word];
+        my @shift;
+        my @mask;
+        my @instr;
+
+        if ($need_wide_mask[$word]) {
+            @shift = ('0', 'BEAM_WIDE_SHIFT');
+            @mask = ('BEAM_WIDE_MASK', $WHOLE_WORD);
+            @instr = ('w', 'i');
+        } else {
+            @shift = @{$pack_shift[$args_per_word]};
+            @mask = @{$pack_mask[$args_per_word]};
+            @instr = @{$pack_instr[$args_per_word]};
+        }
 
-	    $down = "$instr[$ap]$down";
-	    my($unpack) = make_unpack($tmpnum, $shift[$ap], $mask[$ap]);
-	    $args[$i] = "pack:$this_size:$reg" . "b($unpack)";
+        while ($ap < $args_per_word) {
+            my $reg = $args[$arg_num];
+            my $this_size = $arg_size{$reg};
+            if ($bits_needed[$arg_num]) {
+                $this_size = 0;
+                $did_some_packing = 1;
+
+                if ($ap == 0) {
+                    $pack_prefix .= "Eterm $packed_var = Arg($size);\n";
+                    $up .= "p";
+                    $down = "P$down";
+                    $this_size = 1;
+                }
 
-	    if (++$ap == $args_per_word) {
-		$ap = 0;
-		$tmpnum++;
-	    }
-	} elsif ($arg_size{$reg} && $did_some_packing) {
-	    #
-	    # This is an argument that can't be packed.  Normally, we must
-	    # save it on the pack engine's stack, unless:
-	    #
-	    # 1. The argument has zero size (e.g. r(0)).  Such arguments
-	    #    will not be loaded.  They disappear.
-	    # 2. If the argument is on the left of the first packed argument,
-	    #    the packing engine will never access it (because the engine
-	    #    operates from right-to-left).
-	    #
+                $down = "$instr[$ap]$down";
+                my $unpack = make_unpack($packed_var, $shift[$ap], $mask[$ap]);
+                $args[$arg_num] = "packed:$reg:$this_size:$reg" . "b($unpack)";
 
-	    $down = "g${down}";
-	    $up = "${up}p";
-	}
-	$size += $this_size;
+                $ap++;
+            } else {
+                $skip_unpackable->($reg);
+            }
+            $size += $this_size;
+            $arg_num++;
+        }
+    }
+
+    #
+    # Skip any unpackable arguments at the end.
+    #
+    while ($arg_num < @args) {
+        $skip_unpackable->($args[$arg_num]);
+        $arg_num++;
     }
 
     my $pack_spec = $down . $up;
@@ -1582,9 +1653,9 @@ sub do_pack {
 }
 
 sub make_unpack {
-    my($tmpnum, $shift, $mask) = @_;
+    my($packed_var, $shift, $mask) = @_;
 
-    my($e) = "tmp_packed$tmpnum";
+    my $e = $packed_var;
     $e = "($e>>$shift)" if $shift;
     $e .= "&$mask" unless $mask eq $WHOLE_WORD;
     $e;
-- 
cgit v1.2.3


From 0848894878ef90b1c6d81e4a6e2741508d0b99f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 18 Aug 2017 08:27:51 +0200
Subject: Pack instructions using 'q', 'c', and 's'

Update the pack engine to safely push literal operands to the pack
stack and to safely pop them back to another code address. That
will allow packing of more instructions.
---
 erts/emulator/beam/beam_load.c   | 39 +++++++++++++++++++++++++++++++--------
 erts/emulator/utils/beam_makeops | 12 ------------
 2 files changed, 31 insertions(+), 20 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 98701db558..982f998ae3 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2495,16 +2495,32 @@ load_code(LoaderState* stp)
 	 * The packing engine.
 	 */
 	if (opc[stp->specific_op].pack[0]) {
-	    char* prog;		/* Program for packing engine. */
-	    BeamInstr stack[8];	/* Stack. */
-	    BeamInstr* sp = stack;	/* Points to next free position. */
-	    BeamInstr packed = 0;	/* Accumulator for packed operations. */
+	    char* prog;            /* Program for packing engine. */
+	    struct pack_stack {
+                BeamInstr instr;
+                LiteralPatch* patch;
+            } stack[8];            /* Stack. */
+	    struct pack_stack* sp = stack; /* Points to next free position. */
+	    BeamInstr packed = 0; /* Accumulator for packed operations. */
 
 	    for (prog = opc[stp->specific_op].pack; *prog; prog++) {
 		switch (*prog) {
 		case 'g':	/* Get instruction; push on stack. */
-		    *sp++ = code[--ci];
-		    break;
+                    {
+                        LiteralPatch* lp;
+
+                        ci--;
+                        sp->instr = code[ci];
+                        sp->patch = 0;
+                        for (lp = stp->literal_patches; lp && lp->pos > ci-MAX_OPARGS; lp = lp->next) {
+                            if (lp->pos == ci) {
+                                sp->patch = lp;
+                                break;
+                            }
+                        }
+                        sp++;
+                    }
+                    break;
 		case 'i':	/* Initialize packing accumulator. */
 		    packed = code[--ci];
 		    break;
@@ -2520,10 +2536,17 @@ load_code(LoaderState* stp)
 		    break;
 #endif
 		case 'p':	/* Put instruction (from stack). */
-		    code[ci++] = *--sp;
+                    --sp;
+                    code[ci] = sp->instr;
+                    if (sp->patch) {
+                        sp->patch->pos = ci;
+                    }
+                    ci++;
 		    break;
 		case 'P':	/* Put packed operands. */
-		    *sp++ = packed;
+                    sp->instr = packed;
+                    sp->patch = 0;
+                    sp++;
 		    packed = 0;
 		    break;
 		default:
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 2a957d2c9d..5f8395a289 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -1475,18 +1475,6 @@ sub do_pack {
         if (defined $bits_needed{$arg}) {
 	    $packable_args++;
             push @bits_needed, $bits_needed{$arg};
-	} elsif ($arg =~ /^[scq]/ and $packable_args > 0) {
-	    # When packing, this operand will be picked up from the
-	    # code array, put onto the packing stack, and later put
-	    # back into a different location in the code. The problem
-	    # is that if this operand is a literal, the original
-	    # location in the code would have been remembered in a
-	    # literal patch.  For packing to work, we would have to
-	    # adjust the position in the literal patch. For the
-	    # moment, adding additional instructions to the packing
-	    # engine to handle this does not seem worth it, so we will
-	    # just turn off packing.
-	    return ('', '', @args);
 	} else {
 	    push @bits_needed, 0;
 	}
-- 
cgit v1.2.3


From 5725b59f0c14a3bdc665f8543a30a8a5985a3d3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 18 Aug 2017 09:03:05 +0200
Subject: Pack cold instructions too

Cold instructions used to be cooler (less frequently executed),
so it did not seem worthwhile to pack their operands. Now bit
syntax instructions are included among the cold instructions,
and they are frequently used.
---
 erts/emulator/utils/beam_makeops | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 5f8395a289..fcbd3de4c1 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -578,7 +578,7 @@ sub emulator_output {
 	    # for the emulator.
 	    #
 	    my($size, $code, $pack) =
-                basic_generator($name, $hot, '', 0, undef, @args);
+                basic_generator($name, 1, '', 0, undef, @args);
 
 	    #
 	    # Save the generated $code for later.
-- 
cgit v1.2.3


From 8fc304e1e0c4a36dcb5abbe7a51de63ddc2cb285 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 21 Aug 2017 07:18:49 +0200
Subject: Introduce more packable types

The 'I' type can be replaced with 't' if we know that the value
will fit in 16 bits. The 'P' type can be replaced with 'Q' if
it is used for deallocating a stack frame.
---
 erts/emulator/beam/arith_instrs.tab |  4 +-
 erts/emulator/beam/beam_debug.c     |  6 +--
 erts/emulator/beam/ops.tab          | 82 ++++++++++++++++++-------------------
 3 files changed, 46 insertions(+), 46 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab
index a5c84d41d6..224945287b 100644
--- a/erts/emulator/beam/arith_instrs.tab
+++ b/erts/emulator/beam/arith_instrs.tab
@@ -370,10 +370,10 @@ shift.execute(Fail, Live, Dst) {
         reg[0] = Op1;
         reg[1] = Op2;
         SWAPOUT;
-        if (I[0] == (BeamInstr) OpCode(i_bsl_ssjId)) {
+        if (I[0] == (BeamInstr) OpCode(i_bsl_ssjtd)) {
             I = handle_error(c_p, I, reg, &bif_export[BIF_bsl_2]->info.mfa);
         } else {
-            ASSERT(I[0] == (BeamInstr) OpCode(i_bsr_ssjId));
+            ASSERT(I[0] == (BeamInstr) OpCode(i_bsr_ssjtd));
             I = handle_error(c_p, I, reg, &bif_export[BIF_bsr_2]->info.mfa);
         }
         goto post_error_handling;
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 06b9db2877..49414ae8fc 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -526,9 +526,9 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	case 'I':
         case 'W':
 	    switch (op) {
-	    case op_i_gc_bif1_jWsId:
-	    case op_i_gc_bif2_jWIssd:
-	    case op_i_gc_bif3_jWIssd:
+	    case op_i_gc_bif1_jWstd:
+	    case op_i_gc_bif2_jWtssd:
+	    case op_i_gc_bif3_jWtssd:
 		{
 		    const ErtsGcBif* p;
 		    BifFunction gcf = (BifFunction) *ap;
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index deee1a4de9..4f79177c89 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -108,7 +108,7 @@ allocate_zero t t
 allocate_heap_zero t I t
 
 trim N Remaining => i_trim N
-i_trim I
+i_trim t
 
 test_heap I t
 
@@ -590,7 +590,7 @@ is_integer Fail Literal=q => move Literal x | is_integer Fail x
 
 is_integer Fail=f S=x | allocate Need Regs => is_integer_allocate Fail S Need Regs
 
-is_integer_allocate f x I I
+is_integer_allocate f x t t
 
 is_integer f xy
 
@@ -609,7 +609,7 @@ is_nonempty_list F=f x==0 | test_heap I1 I2 => is_nonempty_list_test_heap F I1 I
 is_nonempty_list Fail=f S=x | get_list S D1=x D2=x => \
   is_nonempty_list_get_list Fail S D1 D2
 
-is_nonempty_list_allocate f rx I t
+is_nonempty_list_allocate f rx t t
 is_nonempty_list_test_heap f I t
 is_nonempty_list_get_list f rx x x
 is_nonempty_list f xy
@@ -685,7 +685,7 @@ is_function2 f s s
 allocate Need Regs | init Y => allocate_init Need Regs Y
 init Y1 | init Y2 => init2 Y1 Y2
 
-allocate_init t I y
+allocate_init t t y
 
 #################################################################
 # External function and bif calls.
@@ -1006,15 +1006,15 @@ call_last Ar Func D => i_call_last Func D
 call_only Ar Func   => i_call_only Func
 
 i_call f
-i_call_last f P
+i_call_last f Q
 i_call_only f
 
 i_call_ext e
-i_call_ext_last e P
+i_call_ext_last e Q
 i_call_ext_only e
 
 i_move_call_ext c e
-i_move_call_ext_last e P c
+i_move_call_ext_last e Q c
 i_move_call_ext_only e c
 
 # Fun calls.
@@ -1022,8 +1022,8 @@ i_move_call_ext_only e c
 call_fun Arity | deallocate D | return => i_call_fun_last Arity D
 call_fun Arity => i_call_fun Arity
 
-i_call_fun I
-i_call_fun_last I P
+i_call_fun t
+i_call_fun_last t Q
 
 make_fun2 OldIndex=u => gen_make_fun2(OldIndex)
 
@@ -1152,9 +1152,9 @@ bs_init2 Fail Sz Words=u==0 Regs Flags Dst => \
 bs_init2 Fail Sz Words Regs Flags Dst => \
   i_bs_init_fail_heap Sz Words Fail Regs Dst
 
-i_bs_init_fail xy j I x
+i_bs_init_fail xy j t x
 
-i_bs_init_fail_heap s I j I x
+i_bs_init_fail_heap s I j t x
 
 i_bs_init W t x
 
@@ -1171,16 +1171,16 @@ bs_init_bits Fail Sz Words=u==0 Regs Flags Dst => \
 bs_init_bits Fail Sz Words Regs Flags Dst => \
   i_bs_init_bits_fail_heap Sz Words Fail Regs Dst
 
-i_bs_init_bits_fail xy j I x
+i_bs_init_bits_fail xy j t x
 
-i_bs_init_bits_fail_heap s I j I x
+i_bs_init_bits_fail_heap s I j t x
 
 i_bs_init_bits W t x
 i_bs_init_bits_heap W I t x
 
 bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D
 
-bs_add j s s I x
+bs_add j s s t x
 
 bs_append Fail Size Extra Live Unit Bin Flags Dst => \
   move Bin x | i_bs_append Fail Extra Live Unit Size Dst
@@ -1190,8 +1190,8 @@ bs_private_append Fail Size Unit Bin Flags Dst => \
 
 bs_init_writable
 
-i_bs_append j I I I s x
-i_bs_private_append j I s s x
+i_bs_append j I t t s x
+i_bs_private_append j t s s x
 
 #
 # Storing integers into binaries.
@@ -1200,7 +1200,7 @@ i_bs_private_append j I s s x
 bs_put_integer Fail=j Sz=sq Unit=u Flags=u Src=s => \
 			gen_put_integer(Fail, Sz, Unit, Flags, Src)
 
-i_new_bs_put_integer j s I s
+i_new_bs_put_integer j s t s
 i_new_bs_put_integer_imm j W t s
 
 #
@@ -1219,7 +1219,7 @@ bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src
 
 i_bs_put_utf8 j s
 
-bs_put_utf16 j I s
+bs_put_utf16 j t s
 
 bs_put_utf32 Fail=j Flags=u Src=s => \
    i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src
@@ -1234,8 +1234,8 @@ bs_put_float Fail Sz=q Unit Flags Val => badarg Fail
 bs_put_float Fail=j Sz=s Unit=u Flags=u Src=s => \
 			gen_put_float(Fail, Sz, Unit, Flags, Src)
 
-i_new_bs_put_float j s I s
-i_new_bs_put_float_imm j W I s
+i_new_bs_put_float j s t s
+i_new_bs_put_float_imm j W t s
 
 #
 # Storing binaries into binaries.
@@ -1244,9 +1244,9 @@ i_new_bs_put_float_imm j W I s
 bs_put_binary Fail=j Sz=s Unit=u Flags=u Src=s => \
 			gen_put_binary(Fail, Sz, Unit, Flags, Src)
 
-i_new_bs_put_binary j s I s
+i_new_bs_put_binary j s t s
 i_new_bs_put_binary_imm j W s
-i_new_bs_put_binary_all j s I
+i_new_bs_put_binary_all j s t
 
 #
 # Warning: The i_bs_put_string and i_new_bs_put_string instructions
@@ -1298,8 +1298,8 @@ fclearerror
 # New apply instructions in R10B.
 #
 
-apply I
-apply_last I P
+apply t
+apply_last t Q
 
 #
 # Handle compatibility with OTP 17 here.
@@ -1454,30 +1454,30 @@ gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst
 
 i_increment rxy W t d
 
-i_plus x xy j I d
-i_plus s s  j I d
+i_plus x xy j t d
+i_plus s s  j t d
 
-i_minus x x j I d
-i_minus s s j I d
+i_minus x x j t d
+i_minus s s j t d
 
-i_times j I s s d
+i_times j t s s d
 
-i_m_div j I s s d
-i_int_div j I s s d
+i_m_div j t s s d
+i_int_div j t s s d
 
-i_rem x x j I d
-i_rem s s j I d
+i_rem x x j t d
+i_rem s s j t d
 
-i_bsl s s j I d
-i_bsr s s j I d
+i_bsl s s j t d
+i_bsr s s j t d
 
-i_band x c j I d
-i_band s s j I d
+i_band x c j t d
+i_band s s j t d
 
 i_bor j I s s d
 i_bxor j I s s d
 
-i_int_bnot j s I d
+i_int_bnot j s t d
 
 #
 # Old guard BIFs that creates heap fragments are no longer allowed.
@@ -1501,9 +1501,9 @@ gc_bif2 Fail I Bif S1 S2 Dst => \
 gc_bif3 Fail I Bif S1 S2 S3 Dst => \
 	gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst)
 
-i_gc_bif1 j W s I d
+i_gc_bif1 j W s t d
 
-i_gc_bif2 j W I s s d
+i_gc_bif2 j W t s s d
 
 ii_gc_bif3/7
 
@@ -1512,7 +1512,7 @@ ii_gc_bif3/7
 ii_gc_bif3 Fail Bif Live S1 S2 S3 Dst => \
   move S1 x | i_gc_bif3 Fail Bif Live S2 S3 Dst
 
-i_gc_bif3 j W I s s d
+i_gc_bif3 j W t s s d
 
 #
 # The following instruction is specially handled in beam_load.c
-- 
cgit v1.2.3


From 7c9ea890cd2530bffeafa82abeeee876fd24a6ba Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 23 Aug 2017 12:29:50 +0200
Subject: erts: Allow any unicode string as crash dump slogan

i.e the first argument to erlang:halt
---
 erts/emulator/beam/bif.c         | 2 +-
 erts/emulator/test/bif_SUITE.erl | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 8bf7e3926e..4c92b9ba2b 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4020,7 +4020,7 @@ BIF_RETTYPE halt_2(BIF_ALIST_2)
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	erts_exit(ERTS_ABORT_EXIT, "");
     }
-    else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) {
+    else if (is_list(BIF_ARG_1) || BIF_ARG_1 == NIL) {
 #       define HALT_MSG_SIZE 200
         static byte halt_msg[4*HALT_MSG_SIZE+1];
         Sint written;
diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl
index 339c827602..4908ac410c 100644
--- a/erts/emulator/test/bif_SUITE.erl
+++ b/erts/emulator/test/bif_SUITE.erl
@@ -691,6 +691,9 @@ erlang_halt(Config) when is_list(Config) ->
     {badrpc,nodedown} = rpc:call(N3, erlang, halt, [0,[]]),
     {ok,N4} = slave:start(H, halt_node4),
     {badrpc,nodedown} = rpc:call(N4, erlang, halt, [lists:duplicate(300,$x)]),
+    %% Test unicode slogan
+    {ok,N4} = slave:start(H, halt_node4),
+    {badrpc,nodedown} = rpc:call(N4, erlang, halt, [[339,338,254,230,198,295,167,223,32,12507,12531,12480]]),
 
     % This test triggers a segfault when dumping a crash dump
     % to make sure that we can handle it properly.
-- 
cgit v1.2.3


From 5bf73db9fd77bce195b7e78a8754730263068fb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 21 Aug 2017 18:03:04 +0200
Subject: Add the 'S' type for a register source

The type 'd' could be used both for destination registers and
source register.

Restrict the 'd' type to only be used for destinations, and
introduce the new 'S' type to be used when a source must be
a register.
---
 erts/emulator/beam/beam_emu.c    | 3 ++-
 erts/emulator/beam/beam_load.c   | 3 ++-
 erts/emulator/beam/ops.tab       | 6 +++---
 erts/emulator/utils/beam_makeops | 5 ++++-
 4 files changed, 11 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index badfe62aa9..482f99cb78 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -153,7 +153,7 @@ do {                                     \
  * Register target (X or Y register).
  */
 
-#define REG_TARGET_PTR(Target) (((Target) & 1) ? &yb(Target-1) : &xb(Target))
+#define REG_TARGET_PTR(Target) (((Target) & 1) ? &yb((Target)-1) : &xb(Target))
 
 /*
  * Special Beam instructions.
@@ -242,6 +242,7 @@ void** beam_ops;
 #define tb(N) (N)
 #define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N)))
 #define yb(N) (*(Eterm *) (((unsigned char *)E) + (N)))
+#define Sb(N) (*REG_TARGET_PTR(N))
 #define lb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N)))
 #define Qb(N) (N)
 #define Ib(N) (N)
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 982f998ae3..9ff32e30f3 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2374,7 +2374,8 @@ load_code(LoaderState* stp)
 		    break;
 		}
 		break;
-	    case 'd':	/* Destination (x(0), x(N), y(N) */
+	    case 'd':	/* Destination (x(N), y(N) */
+            case 'S':   /* Source (x(N), y(N)) */
 		switch (tag) {
 		case TAG_x:
 		    code[ci++] = tmp_op->a[arg].val * sizeof(Eterm);
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 4f79177c89..b6e995fdbe 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -192,7 +192,7 @@ try_case_end s
 
 # Destructive set tuple element
 
-set_tuple_element s d P
+set_tuple_element s S P
 
 # Get tuple element
 
@@ -1272,9 +1272,9 @@ fmove Arg=l Dst=d => fstore Arg Dst
 fmove Arg=dq Dst=l => fload Arg Dst
 
 fstore l d
-fload dq l
+fload Sq l
 
-fconv d l
+fconv S l
 
 i_fadd l l l
 i_fsub l l l
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index fcbd3de4c1..ee8b12681f 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -139,6 +139,7 @@ my %arg_size = ('r' => 0,	# x(0) - x register zero
 		'n' => 0,	# NIL (implicit)
 		'c' => 1,	# tagged constant (integer, atom, nil)
 		's' => 1,	# tagged source; any of the above
+                'S' => 1,       # tagged source register (x or y)
 		'd' => 1,	# tagged destination register (r, x, y)
 		'f' => 1,	# failure label
 		'j' => 1,	# either 'f' or 'p'
@@ -188,6 +189,7 @@ sub define_type_bit {
     define_type_bit('s', $type_bit{'d'} | $type_bit{'i'} |
 		    $type_bit{'a'} | $type_bit{'n'} |
 		    $type_bit{'q'});
+    define_type_bit('S', $type_bit{'d'});
     define_type_bit('j', $type_bit{'f'} | $type_bit{'p'});
 
     # Aliases (for matching purposes).
@@ -1258,7 +1260,7 @@ sub basic_generator {
                 push(@f, "r(0)");
                 last SWITCH;
             };
-	    /[lxy]/ and do {
+	    /[lxyS]/ and do {
                 push(@f, $_ . "b(Arg($arg_offset))");
                 last SWITCH;
             };
@@ -1462,6 +1464,7 @@ sub do_pack {
                        'y' => 10,
                        'Q' => 10,
                        'l' => 10,
+                       'S' => 16,
                        'd' => 16,
                        't' => 16);
     if ($wordsize == 64) {
-- 
cgit v1.2.3


From eb8ffa6a071c3fe532389ba1907fc89aa9ea15eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 23 Aug 2017 12:46:42 +0200
Subject: Eliminate the beam_instrs.h file

The beam_instrs.h file serves no useful purpose. Put the
instructions in beam_hot.h instead.
---
 erts/emulator/Makefile.in        |  1 -
 erts/emulator/beam/beam_emu.c    |  1 -
 erts/emulator/utils/beam_makeops | 59 ++++++++++++++--------------------------
 3 files changed, 21 insertions(+), 40 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index c6511a4b49..bc7eb72221 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -546,7 +546,6 @@ endif
 
 $(TTF_DIR)/beam_cold.h \
 $(TTF_DIR)/beam_hot.h \
-$(TTF_DIR)/beam_instrs.h \
 $(TTF_DIR)/beam_opcodes.c \
 $(TTF_DIR)/beam_opcodes.h \
 $(TTF_DIR)/beam_pred_funcs.h \
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 482f99cb78..2c37dc42b3 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -770,7 +770,6 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 #endif
 
 #include "beam_hot.h"
-#include "beam_instrs.h"
 
 #ifdef DEBUG
     /*
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index ee8b12681f..c12073d943 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -92,10 +92,9 @@ my %c_code_used;                # Used or not.
 
 # Definitions for instructions combined from micro instructions.
 my %combined_instrs;
-my %combined_code;           # Combined micro instructions.
 
-my %hot_code;
-my %cold_code;
+my @generated_code;             # Generated code.
+my %sort_order;
 
 my @unnumbered_generic;
 my %unnumbered;
@@ -384,7 +383,7 @@ while (<>) {
     # micro instructions.
     #
     if (/^(\w+)\s*:=\s*([\w.]+)\s*;\s*$/) {
-        $combined_instrs{$1} = ["$ARGV($.)","beam_instrs.h",$2];
+        $combined_instrs{$1} = ["$ARGV($.)",$2];
         next;
     }
 
@@ -586,11 +585,8 @@ sub emulator_output {
 	    # Save the generated $code for later.
 	    #
 	    if (defined $code) {
-		if ($hot) {
-		    push(@{$hot_code{$code}}, $instr);
-		} else {
-		    push(@{$cold_code{$code}}, $instr);
-		}
+                $code = "OpCase($instr):\n$code";
+                push @generated_code, [$hot,$code,($instr)];
 	    }
 
 	    #
@@ -805,19 +801,12 @@ sub emulator_output {
     $name = "$outdir/beam_hot.h";
     open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
     comment('C');
-    print_code(\%hot_code);
+    print_code(1);
 
     $name = "$outdir/beam_cold.h";
     open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
     comment('C');
-    print_code(\%cold_code);
-
-    foreach my $key (keys %combined_code) {
-        my $name = "$outdir/$key";
-        open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
-        comment('C');
-        print_indented_code(@{$combined_code{$key}});
-    }
+    print_code(0);
 }
 
 sub init_item {
@@ -845,19 +834,14 @@ sub q {
 }
 
 sub print_code {
-    my($ref) = @_;
-    my(%sorted);
-    my($key, $label);		# Loop variables.
-
-    foreach $key (keys %$ref) {
-	my($sort_key);
-	my($code) = '';
-	foreach $label (@{$ref->{$key}}) {
-	    $code .= "OpCase($label):\n";
-	    $sort_key = $label;
-	}
-        $code .= "$key\n";
-	$sorted{$sort_key} = $code;
+    my($include_hot) = @_;
+    my %sorted;
+
+    foreach my $ref (@generated_code) {
+        my($hot,$code,@labels) = @$ref;
+        next unless $hot == $include_hot;
+        my($sort_key) = @labels; # Use the first label as sort key.
+        $sorted{$sort_key} = $code;
     }
 
     foreach (sort keys %sorted) {
@@ -1034,12 +1018,11 @@ sub comment {
 #
 sub combine_micro_instructions {
     my %groups;
-    my %group_file;
 
     # Sanity check, normalize micro instructions.
     foreach my $instr (keys %combined_instrs) {
         my $ref = $combined_instrs{$instr};
-        my($def_loc,$outfile,$def) = @$ref;
+        my($def_loc,$def) = @$ref;
         my($group,@subs) = split /[.]/, $def;
         my $arity = 0;
         @subs = map { "$group.$_" } @subs;
@@ -1052,14 +1035,12 @@ sub combine_micro_instructions {
             $arity += scalar(@c_args);
         }
         push @{$groups{$group}}, [$instr,$arity,@subs];
-        $group_file{$group} = $outfile;
     }
 
     # Now generate code for each group.
     foreach my $group (sort keys %groups) {
-        my $code = combine_instruction_group($group, @{$groups{$group}});
-        my $outfile = $group_file{$group};
-        push @{$combined_code{$outfile}}, $code;
+        my($code,@labels) = combine_instruction_group($group, @{$groups{$group}});
+        push @generated_code, [1,$code,@labels];
     }
 }
 
@@ -1151,6 +1132,7 @@ sub combine_instruction_group {
 
     # Now generate the code for the entire group.
     my $offset = 0;
+    my @opcase_labels;
     for(my $i = 0; $i < @slots; $i++) {
         my $key = $slots[$i];
 
@@ -1174,6 +1156,7 @@ sub combine_instruction_group {
 
             if ($opcase ne '') {
                 $gcode .= "OpCase($opcase):\n";
+                push @opcase_labels, $opcase;
             }
             if ($num_references{$label}) {
                 $gcode .= "$label:\n";
@@ -1199,7 +1182,7 @@ sub combine_instruction_group {
         $offset = $order_to_offset{$slots[$i+1]} if $i < $#slots;
     }
 
-    "{\n$gcode\n}\n\n";
+    ("{\n$gcode\n}\n\n",@opcase_labels);
 }
 
 sub micro_label {
-- 
cgit v1.2.3


From e337151a9bc869367032af334ebd479d92dfbdee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 23 Aug 2017 12:56:36 +0200
Subject: beam_makeops: Stop using the Arg() macro

Generated code uses 'I' explicitly in other places, so it
can as well use 'I' when accessing the operands for instructions.
---
 erts/emulator/utils/beam_makeops | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index c12073d943..6c54ab3421 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -1244,7 +1244,7 @@ sub basic_generator {
                 last SWITCH;
             };
 	    /[lxyS]/ and do {
-                push(@f, $_ . "b(Arg($arg_offset))");
+                push(@f, $_ . "b(" . arg_offset($arg_offset) . ")");
                 last SWITCH;
             };
 	    /n/ and do {
@@ -1261,13 +1261,13 @@ sub basic_generator {
                 last SWITCH;
             };
 	    /d/ and do {
-                $var_decls .= "Eterm dst = Arg($arg_offset);\n" .
+                $var_decls .= "Eterm dst = " . arg_offset($arg_offset) . ";\n" .
                     "Eterm* dst_ptr = REG_TARGET_PTR(dst);\n";
                 push(@f, "*dst_ptr");
                 last SWITCH;
             };
             defined $arg_size{$_} and do {
-                push(@f, "Arg($arg_offset)");
+                push @f, arg_offset($arg_offset);
                 last SWITCH;
             };
 
@@ -1349,6 +1349,11 @@ sub basic_generator {
     ($size+1, $code, $pack_spec);
 }
 
+sub arg_offset {
+    my $offset = shift;
+    "I[" . ($offset+1) . "]";
+}
+
 sub expand_all {
     my($code,$bindings_ref) = @_;
     my %bindings = %{$bindings_ref};
@@ -1595,7 +1600,8 @@ sub do_pack {
                 $did_some_packing = 1;
 
                 if ($ap == 0) {
-                    $pack_prefix .= "Eterm $packed_var = Arg($size);\n";
+                    $pack_prefix .= "Eterm $packed_var = " .
+                        arg_offset($size) . ";\n";
                     $up .= "p";
                     $down = "P$down";
                     $this_size = 1;
-- 
cgit v1.2.3


From 203b66236c5c2c9a2ce81b5998f01e24706984ec Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 23 Aug 2017 13:20:27 +0200
Subject: erts: Fix incorrect merge of dirty_trace_message_flush fix to master

---
 erts/emulator/beam/erl_process.c | 2 --
 1 file changed, 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 0ff9484daf..875b681f1c 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -14107,9 +14107,7 @@ erts_continue_exit_process(Process *p)
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
-#ifdef ERTS_SMP
     erts_flush_trace_messages(p, ERTS_PROC_LOCK_MAIN);
-#endif
 
     ERTS_TRACER_CLEAR(&ERTS_TRACER(p));
 
-- 
cgit v1.2.3


From 319eefea3b2097b543751c63cebbb4a92011dbbf Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 21 Aug 2017 15:25:00 +0200
Subject: Unicode support for erlang:display_string/1

---
 erts/emulator/beam/bif.c         | 15 +++++++------
 erts/emulator/beam/global.h      |  1 +
 erts/emulator/beam/utils.c       | 46 ++++++++++++++++++++++++++++++++++++++++
 erts/emulator/test/bif_SUITE.erl | 26 +++++++++++++++++++++--
 4 files changed, 80 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 4c92b9ba2b..22805b5c02 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3947,15 +3947,18 @@ BIF_RETTYPE display_string_1(BIF_ALIST_1)
 {
     Process* p = BIF_P;
     Eterm string = BIF_ARG_1;
-    Sint len = is_string(string);
-    char *str;
+    Sint len = erts_unicode_list_to_buf_len(string);
+    Sint written;
+    byte *str;
+    int res;
 
-    if (len <= 0) {
+    if (len < 0) {
 	BIF_ERROR(p, BADARG);
     }
-    str = (char *) erts_alloc(ERTS_ALC_T_TMP, sizeof(char)*(len + 1));
-    if (intlist_to_buf(string, str, len) != len)
-	erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__);
+    str = (byte *) erts_alloc(ERTS_ALC_T_TMP, sizeof(char)*(len + 1));
+    res = erts_unicode_list_to_buf(string, str, len, &written);
+    if (res != 0 || written != len)
+	erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error (%d)\n", __FILE__, __LINE__, res);
     str[len] = '\0';
     erts_fprintf(stderr, "%s", str);
     erts_free(ERTS_ALC_T_TMP, (void *) str);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index fcb88712e9..64e543586a 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1288,6 +1288,7 @@ int erts_utf8_to_latin1(byte* dest, const byte* source, int slen);
 void bin_write(fmtfn_t, void*, byte*, size_t);
 Sint intlist_to_buf(Eterm, char*, Sint); /* most callers pass plain char*'s */
 int erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len, Sint* written);
+Sint erts_unicode_list_to_buf_len(Eterm list);
 
 struct Sint_buf {
 #if defined(ARCH_64)
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index ba7596cde4..16523659d9 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3704,6 +3704,52 @@ erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len, Sint* written)
     return res;
 }
 
+Sint
+erts_unicode_list_to_buf_len(Eterm list)
+{
+    Eterm* listptr;
+    Sint sz = 0;
+
+    if (is_nil(list)) {
+	return 0;
+    }
+    if (is_not_list(list)) {
+	return -1;
+    }
+    listptr = list_val(list);
+
+    while (1) {
+	Sint val;
+
+	if (is_not_small(CAR(listptr))) {
+	    return -1;
+	}
+	val = signed_val(CAR(listptr));
+	if (0 <= val && val < 0x80) {
+	    sz++;
+	} else if (val < 0x800) {
+	    sz += 2;
+	} else if (val < 0x10000UL) {
+	    if (0xD800 <= val && val <= 0xDFFF) {
+		return -1;
+	    }
+	    sz += 3;
+	} else if (val < 0x110000) {
+	    sz += 4;
+	} else {
+	    return -1;
+	}
+	list = CDR(listptr);
+	if (is_nil(list)) {
+	    return sz;
+	}
+	if (is_not_list(list)) {
+	    return -1;
+	}
+	listptr = list_val(list);
+    }
+}
+
 /*
 ** Convert an integer to a byte list
 ** return pointer to converted stuff (need not to be at start of buf!)
diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl
index 4908ac410c..2320870a0e 100644
--- a/erts/emulator/test/bif_SUITE.erl
+++ b/erts/emulator/test/bif_SUITE.erl
@@ -24,7 +24,7 @@
 -include_lib("kernel/include/file.hrl").
 
 -export([all/0, suite/0,
-	 display/1, display_huge/0,
+	 display/1, display_huge/0, display_string/1,
 	 erl_bif_types/1,guard_bifs_in_erl_bif_types/1,
 	 shadow_comments/1,list_to_utf8_atom/1,
 	 specs/1,improper_bif_stubs/1,auto_imports/1,
@@ -43,7 +43,7 @@ all() ->
     [erl_bif_types, guard_bifs_in_erl_bif_types, shadow_comments,
      specs, improper_bif_stubs, auto_imports,
      t_list_to_existing_atom, os_env, otp_7526,
-     display, list_to_utf8_atom,
+     display, display_string, list_to_utf8_atom,
      atom_to_binary, binary_to_atom, binary_to_existing_atom,
      erl_crash_dump_bytes, min_max, erlang_halt, is_builtin,
      error_stacktrace, error_stacktrace_during_call_trace].
@@ -68,6 +68,28 @@ deeep(N,Acc) ->
 deeep(N) ->
     deeep(N,[hello]).
 
+display_string(Config) when is_list(Config) ->
+    true = erlang:display_string("hej"),
+    true = erlang:display_string(""),
+    true = erlang:display_string("hopp"),
+    true = erlang:display_string("\n"),
+    true = erlang:display_string(lists:seq(1100,1200)),
+    {error,badarg} = try
+                         erlang:display_string(atom),
+                         ok
+                     catch
+                         T0:E0 ->
+                             {T0, E0}
+                     end,
+    {error,badarg} = try
+                         erlang:display_string(make_ref()),
+                         ok
+                     catch
+                         T1:E1 ->
+                             {T1, E1}
+                     end,
+    ok.
+
 erl_bif_types(Config) when is_list(Config) ->
     ensure_erl_bif_types_compiled(),
 
-- 
cgit v1.2.3


From 7b64965d7a22d2250d3c6582a6d1737ca325a8dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 23 Aug 2017 07:12:52 +0200
Subject: arith_instrs.tab: Clean up bsl/bsr

Eliminate the all-purpose variable 'ires'. Replace it with several
variables used that are used for their specific purpose. Narrow
the scope of the variables.

Do this to improve readability. It is not expected that it should
any impact on performance.
---
 erts/emulator/beam/arith_instrs.tab | 49 ++++++++++++++++++-------------------
 1 file changed, 24 insertions(+), 25 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab
index 224945287b..67cd7c6a2a 100644
--- a/erts/emulator/beam/arith_instrs.tab
+++ b/erts/emulator/beam/arith_instrs.tab
@@ -226,9 +226,6 @@ i_bsr := shift.setup_bsr.execute;
 shift.head() {
     Eterm Op1, Op2;
     Sint shift_left_count;
-    Sint ires;
-    Eterm* bigp;
-    Eterm tmp_big[2];
 }
 
 shift.setup_bsr(Src1, Src2) {
@@ -272,35 +269,37 @@ shift.setup_bsl(Src1, Src2) {
 }
 
 shift.execute(Fail, Live, Dst) {
+    Uint big_words_needed;
+
     if (is_small(Op1)) {
-        ires = signed_val(Op1);
-        if (shift_left_count == 0 || ires == 0) {
+        Sint int_res = signed_val(Op1);
+        if (shift_left_count == 0 || int_res == 0) {
             if (is_not_integer(Op2)) {
                 goto shift_error;
             }
-            if (ires == 0) {
+            if (int_res == 0) {
                 $Dst = Op1;
                 $NEXT0();
             }
         } else if (shift_left_count < 0)  { /* Right shift */
+            Eterm bsr_res;
             shift_left_count = -shift_left_count;
             if (shift_left_count >= SMALL_BITS-1) {
-                $Dst = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO;
+                bsr_res = (int_res < 0) ? SMALL_MINUS_ONE : SMALL_ZERO;
             } else {
-                $Dst = make_small(ires >> shift_left_count);
+                bsr_res = make_small(int_res >> shift_left_count);
             }
+            $Dst = bsr_res;
             $NEXT0();
         } else if (shift_left_count < SMALL_BITS-1) { /* Left shift */
-            if ((ires > 0 &&
-                 ((~(Uint)0 << ((SMALL_BITS-1)-shift_left_count)) &
-                  ires) == 0) ||
-                ((~(Uint)0 << ((SMALL_BITS-1)-shift_left_count)) &
-                 ~ires) == 0) {
-                $Dst = make_small(ires << shift_left_count);
+            if ((int_res > 0 &&
+                 ((~(Uint)0 << ((SMALL_BITS-1)-shift_left_count)) & int_res) == 0) ||
+                ((~(Uint)0 << ((SMALL_BITS-1)-shift_left_count)) & ~int_res) == 0) {
+                $Dst = make_small(int_res << shift_left_count);
                 $NEXT0();
             }
         }
-        ires = 1;               /* big_size(small_to_big(Op1)) */
+        big_words_needed = 1;   /* big_size(small_to_big(Op1)) */
         goto big_shift;
     } else if (is_big(Op1)) {
         if (shift_left_count == 0) {
@@ -310,20 +309,21 @@ shift.execute(Fail, Live, Dst) {
             $Dst = Op1;
             $NEXT0();
         }
-        ires = big_size(Op1);
+        big_words_needed = big_size(Op1);
 
     big_shift:
         if (shift_left_count > 0) {	/* Left shift. */
-            ires += (shift_left_count / D_EXP);
+            big_words_needed += (shift_left_count / D_EXP);
         } else {	/* Right shift. */
-            if (ires <= (-shift_left_count / D_EXP)) {
-                ires = 3;       /* ??? */
+            if (big_words_needed <= (-shift_left_count / D_EXP)) {
+                big_words_needed = 3;       /* ??? */
             } else {
-                ires -= (-shift_left_count / D_EXP);
+                big_words_needed -= (-shift_left_count / D_EXP);
             }
         }
         {
-            ires = BIG_NEED_SIZE(ires+1);
+            Eterm tmp_big[2];
+            Sint big_need_size = BIG_NEED_SIZE(big_words_needed+1);
 
             /*
              * Slightly conservative check the size to avoid
@@ -331,15 +331,14 @@ shift.execute(Fail, Live, Dst) {
              * clearly would overflow the arity in the header
              * word.
              */
-            if (ires-8 > BIG_ARITY_MAX) {
+            if (big_need_size-8 > BIG_ARITY_MAX) {
                 $SYSTEM_LIMIT($Fail);
             }
-            $GC_TEST_PRESERVE(ires+1, $Live, Op1);
+            $GC_TEST_PRESERVE(big_need_size+1, $Live, Op1);
             if (is_small(Op1)) {
                 Op1 = small_to_big(signed_val(Op1), tmp_big);
             }
-            bigp = HTOP;
-            Op1 = big_lshift(Op1, shift_left_count, bigp);
+            Op1 = big_lshift(Op1, shift_left_count, HTOP);
             if (is_big(Op1)) {
                 HTOP += bignum_header_arity(*HTOP) + 1;
             }
-- 
cgit v1.2.3


From 076167bae6cc57d188cdf59fd94c82f3a36ee7ee Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 23 Aug 2017 16:46:12 +0200
Subject: Fix scheduler id field in timers

---
 erts/emulator/beam/erl_hl_timer.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 51a0d68247..fce1cdc473 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -98,16 +98,16 @@ typedef enum {
 #  define ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore
 #endif
 
-/* Bit 0 to 9 contains scheduler id (see mask below) */
-#define ERTS_TMR_ROFLG_HLT		(((Uint32) 1) << 10)
-#define ERTS_TMR_ROFLG_BIF_TMR		(((Uint32) 1) << 11)
-#define ERTS_TMR_ROFLG_PRE_ALC		(((Uint32) 1) << 12)
-#define ERTS_TMR_ROFLG_REG_NAME		(((Uint32) 1) << 13)
-#define ERTS_TMR_ROFLG_PROC		(((Uint32) 1) << 14)
-#define ERTS_TMR_ROFLG_PORT		(((Uint32) 1) << 15)
-#define ERTS_TMR_ROFLG_CALLBACK		(((Uint32) 1) << 16)
+/* Bit 0 to 10 contains scheduler id (see mask below) */
+#define ERTS_TMR_ROFLG_HLT		(((Uint32) 1) << 11)
+#define ERTS_TMR_ROFLG_BIF_TMR		(((Uint32) 1) << 12)
+#define ERTS_TMR_ROFLG_PRE_ALC		(((Uint32) 1) << 13)
+#define ERTS_TMR_ROFLG_REG_NAME		(((Uint32) 1) << 14)
+#define ERTS_TMR_ROFLG_PROC		(((Uint32) 1) << 15)
+#define ERTS_TMR_ROFLG_PORT		(((Uint32) 1) << 16)
+#define ERTS_TMR_ROFLG_CALLBACK		(((Uint32) 1) << 17)
 #ifdef ERTS_BTM_ACCESSOR_SUPPORT
-#define ERTS_TMR_ROFLG_ABIF_TMR		(((Uint32) 1) << 17)
+#define ERTS_TMR_ROFLG_ABIF_TMR		(((Uint32) 1) << 18)
 #endif
 
 #define ERTS_TMR_ROFLG_SID_MASK	\
-- 
cgit v1.2.3


From a4a05420aef591bbec457b1762b7aa54666c3ad5 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 23 Aug 2017 17:04:33 +0200
Subject: Fix BIF timer race between timeout and auto cleanup

---
 erts/emulator/beam/erl_hl_timer.c | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 99995be464..078c2b87c3 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -1273,14 +1273,15 @@ bif_timer_timeout(ErtsHLTimerService *srv,
             ERTS_HLT_ASSERT(proc);
         }
         if (proc) {
+            int dec_refc = 0;
+            ErtsMessage *mp = erts_alloc_message(0, NULL);
+            mp->data.heap_frag = tmr->btm.bp;
+            tmr->btm.bp = NULL;
+            erts_queue_message(proc, 0, mp, tmr->btm.message,
+                               am_clock_service);
+            erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM);
+            /* If the process is exiting do not disturb the cleanup... */
             if (!ERTS_PROC_IS_EXITING(proc)) {
-                int dec_refc = 0;
-                ErtsMessage *mp = erts_alloc_message(0, NULL);
-                mp->data.heap_frag = tmr->btm.bp;
-                tmr->btm.bp = NULL;
-                erts_queue_message(proc, 0, mp, tmr->btm.message,
-                                   am_clock_service);
-                erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM);
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
                 if (tmr->btm.proc_list.next) {
                     proc_btm_list_delete(&proc->bif_timers, tmr);
@@ -1293,10 +1294,10 @@ bif_timer_timeout(ErtsHLTimerService *srv,
                     dec_refc = 1;
                 }
 #endif
-                erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
-                if (dec_refc)
-                    timer_pre_dec_refc((ErtsTimer *) tmr);
             }
+            erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM);
+            if (dec_refc)
+                timer_pre_dec_refc((ErtsTimer *) tmr);
         }
         if (tmr->btm.bp)
             free_message_buffer(tmr->btm.bp);
-- 
cgit v1.2.3


From c49cdbe34e979dabcda56a05e4a104a9693fc56c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 28 Aug 2017 13:06:20 +0200
Subject: Remove backward compatibility support for float literals

Starting from R16B, floating point constants are stored
in the literal pool.

The backward compatibility support for the old representation
of floats has been kept long enough. Also, we might want to
ensure that all literals are unique in the future, and we certainly
don't want to update this code to ensure uniqueness.
---
 erts/emulator/beam/beam_load.c | 26 +++-----------------------
 1 file changed, 3 insertions(+), 23 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 9ff32e30f3..cc3219989e 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2034,30 +2034,10 @@ load_code(LoaderState* stp)
 		    case 0:
 			/* Floating point number.
 			 * Not generated by the compiler in R16B and later.
+                         * (The literal pool is used instead.)
 			 */
-			{
-			    Eterm* hp;
-#if !defined(ARCH_64)
-			    Uint high, low;
-# endif
-			    last_op->a[arg].val = new_literal(stp, &hp,
-							      FLOAT_SIZE_OBJECT);
-			    hp[0] = HEADER_FLONUM;
-			    last_op->a[arg].type = TAG_q;
-#if defined(ARCH_64)
-			    GetInt(stp, 8, hp[1]);
-# else
-			    GetInt(stp, 4, high);
-			    GetInt(stp, 4, low);
-			    if (must_swap_floats) {
-				Uint t = high;
-				high = low;
-				low = t;
-			    }
-			    hp[1] = high;
-			    hp[2] = low;
-# endif
-			}
+                        LoadError0(stp, "please re-compile this module with an "
+                                   ERLANG_OTP_RELEASE " compiler");
 			break;
 		    case 1:	/* List. */
 			if (arg+1 != arity) {
-- 
cgit v1.2.3


From 78fad16ef7c5477239bc0b51125fabfe6567039d Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 30 Jun 2017 16:32:42 +0200
Subject: Support for distribution controller processes

---
 erts/emulator/beam/atom.names        |   2 +
 erts/emulator/beam/bif.tab           |   6 +
 erts/emulator/beam/dist.c            | 872 ++++++++++++++++++++++++++++-------
 erts/emulator/beam/erl_bif_info.c    |   4 +-
 erts/emulator/beam/erl_node_tables.c |  26 +-
 erts/emulator/beam/erl_node_tables.h |  16 +-
 erts/emulator/beam/erl_process.c     |  16 +-
 erts/emulator/beam/erl_process.h     |  15 +-
 erts/emulator/beam/external.c        |  56 ++-
 erts/emulator/beam/external.h        |   7 +-
 erts/emulator/beam/io.c              |   2 +
 11 files changed, 817 insertions(+), 205 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index a44d23b181..fc55b687d4 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -217,6 +217,8 @@ atom discard
 atom display_items
 atom dist
 atom dist_cmd
+atom dist_ctrl_put_data
+atom dist_data
 atom Div='/'
 atom div
 atom dlink
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index a8bbf5f8c1..28ca449a23 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -154,6 +154,12 @@ bif erlang:spawn_opt/1
 bif erlang:setnode/2
 bif erlang:setnode/3
 bif erlang:dist_exit/3
+bif erlang:dist_get_stat/1
+bif erlang:dist_ctrl_input_handler/2
+bif erlang:dist_ctrl_put_data/2
+bif erlang:dist_ctrl_get_data/1
+bif erlang:dist_ctrl_get_data_notification/1
+
 
 # Static native functions in erts_internal
 bif erts_internal:port_info/1
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index d9feec4722..5d7501f234 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -121,7 +121,7 @@ Export* dexit_trap = NULL;
 Export* dmonitor_p_trap = NULL;
 
 /* local variables */
-
+static Export *dist_ctrl_put_data_trap;
 
 /* forward declarations */
 
@@ -156,9 +156,7 @@ create_cache(DistEntry *dep)
     int i;
     ErtsAtomCache *cp;
 
-    ERTS_SMP_LC_ASSERT(
-	is_internal_port(dep->cid)
-	&& erts_lc_is_port_locked(erts_port_lookup_raw(dep->cid)));
+    ERTS_SMP_LC_ASSERT(is_nil(dep->cid));
     ASSERT(!dep->cache);
 
     dep->cache = cp = (ErtsAtomCache*) erts_alloc(ERTS_ALC_T_DCACHE,
@@ -450,7 +448,35 @@ inc_no_nodes(void)
 #endif
     erts_smp_atomic_inc_mb(&no_nodes);
 }
-	
+
+static void
+kill_dist_ctrl_proc(void *vpid)
+{
+    Eterm pid = (Eterm) vpid;
+    ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
+    Process *rp = erts_pid2proc(NULL, 0, pid, rp_locks);
+    if (rp) {
+        erts_send_exit_signal(NULL, rp->common.id, rp, &rp_locks,
+                              am_kill, NIL, NULL, 0);
+        if (rp_locks)
+            erts_smp_proc_unlock(rp, rp_locks);
+    }
+}
+
+static void
+schedule_kill_dist_ctrl_proc(Eterm pid)
+{
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    int sched_id = 1;
+    if (!esdp || ERTS_SCHEDULER_IS_DIRTY(esdp))
+        sched_id = 1;
+    else
+        sched_id = (int) esdp->no;
+    erts_schedule_misc_aux_work(sched_id,
+                                kill_dist_ctrl_proc,
+                                (void *) (UWord) pid);
+}
+
 /*
  * proc is currently running or exiting process.
  */
@@ -460,58 +486,62 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 
     if (dep == erts_this_dist_entry) {  /* Net kernel has died (clean up!!) */
 	DistEntry *tdep;
-	int no_dist_port = 0;
+	int no_dist_ctrl = 0;
 	Eterm nd_reason = (reason == am_no_network
 			   ? am_no_network
 			   : am_net_kernel_terminated);
 	erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
 
 	for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next)
-	    no_dist_port++;
+	    no_dist_ctrl++;
 	for (tdep = erts_visible_dist_entries; tdep; tdep = tdep->next)
-	    no_dist_port++;
+	    no_dist_ctrl++;
 
 	/* KILL all port controllers */
-	if (no_dist_port == 0)
+	if (no_dist_ctrl == 0)
 	    erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
 	else {
 	    Eterm def_buf[128];
 	    int i = 0;
-	    Eterm *dist_port;
+	    Eterm *dist_ctrl;
 
-	    if (no_dist_port <= sizeof(def_buf)/sizeof(def_buf[0]))
-		dist_port = &def_buf[0];
+	    if (no_dist_ctrl <= sizeof(def_buf)/sizeof(def_buf[0]))
+		dist_ctrl = &def_buf[0];
 	    else
-		dist_port = erts_alloc(ERTS_ALC_T_TMP,
-				       sizeof(Eterm)*no_dist_port);
+		dist_ctrl = erts_alloc(ERTS_ALC_T_TMP,
+				       sizeof(Eterm)*no_dist_ctrl);
 	    for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next) {
-		ASSERT(is_internal_port(tdep->cid));
-		dist_port[i++] = tdep->cid;
+		ASSERT(is_internal_port(tdep->cid) || is_internal_pid(tdep->cid));
+		dist_ctrl[i++] = tdep->cid;
 	    }
 	    for (tdep = erts_visible_dist_entries; tdep; tdep = tdep->next) {
-		ASSERT(is_internal_port(tdep->cid));
-		dist_port[i++] = tdep->cid;
+		ASSERT(is_internal_port(tdep->cid) || is_internal_pid(tdep->cid));
+		dist_ctrl[i++] = tdep->cid;
 	    }
 	    erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
 
-	    for (i = 0; i < no_dist_port; i++) {
-		Port *prt = erts_port_lookup(dist_port[i],
-					     ERTS_PORT_SFLGS_INVALID_LOOKUP);
-		if (!prt)
-		    continue;
-		ASSERT(erts_atomic32_read_nob(&prt->state)
-		       & ERTS_PORT_SFLG_DISTRIBUTION);
-
-		erts_port_exit(NULL, ERTS_PORT_SIG_FLG_FORCE_SCHED,
-			       prt, dist_port[i], nd_reason, NULL);
+	    for (i = 0; i < no_dist_ctrl; i++) {
+                if (is_internal_pid(dist_ctrl[i]))
+                    schedule_kill_dist_ctrl_proc(dist_ctrl[i]);
+                else {
+                    Port *prt = erts_port_lookup(dist_ctrl[i],
+                                                 ERTS_PORT_SFLGS_INVALID_LOOKUP);
+                    if (prt) {
+                        ASSERT(erts_atomic32_read_nob(&prt->state)
+                               & ERTS_PORT_SFLG_DISTRIBUTION);
+
+                        erts_port_exit(NULL, ERTS_PORT_SIG_FLG_FORCE_SCHED,
+                                       prt, dist_ctrl[i], nd_reason, NULL);
+                    }
+                }
 	    }
 
-	    if (dist_port != &def_buf[0])
-		erts_free(ERTS_ALC_T_TMP, dist_port);
+	    if (dist_ctrl != &def_buf[0])
+		erts_free(ERTS_ALC_T_TMP, dist_ctrl);
 	}
 
 	/*
-	 * When last dist port exits, node will be taken
+	 * When last dist ctrl exits, node will be taken
 	 * from alive to not alive.
 	 */
 	ASSERT(is_nil(nodedown.reason) && !nodedown.bp);
@@ -528,7 +558,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 					  &nodedown.bp->off_heap);
 	}
     }
-    else { /* Call from distribution port */
+    else { /* Call from distribution controller (port/process) */
 	NetExitsContext nec = {dep};
 	ErtsLink *nlinks;
 	ErtsLink *node_links;
@@ -538,11 +568,12 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 	erts_smp_atomic_set_mb(&dep->dist_cmd_scheduled, 1);
 	erts_smp_de_rwlock(dep);
 
-	ERTS_SMP_LC_ASSERT(is_internal_port(dep->cid)
-			   && erts_lc_is_port_locked(erts_port_lookup_raw(dep->cid)));
+        if (is_internal_port(dep->cid)) {
+            ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(erts_port_lookup_raw(dep->cid)));
 
-	if (erts_port_task_is_scheduled(&dep->dist_cmd))
-	    erts_port_task_abort(&dep->dist_cmd);
+            if (erts_port_task_is_scheduled(&dep->dist_cmd))
+                erts_port_task_abort(&dep->dist_cmd);
+        }
 
 	if (dep->status & ERTS_DE_SFLG_EXITING) {
 #ifdef DEBUG
@@ -618,6 +649,9 @@ void init_dist(void)
     dgroup_leader_trap = trap_function(am_dgroup_leader,2);
     dexit_trap = trap_function(am_dexit, 2);
     dmonitor_p_trap = trap_function(am_dmonitor_p, 2);
+    dist_ctrl_put_data_trap = erts_export_put(am_erts_internal,
+                                              am_dist_ctrl_put_data,
+                                              2);
 }
 
 #define ErtsDistOutputBuf2Binary(OB) \
@@ -660,6 +694,8 @@ static void clear_dist_entry(DistEntry *dep)
     ErtsDistOutputBuf *obuf;
 
     erts_smp_de_rwlock(dep);
+    erts_smp_atomic_set_nob(&dep->input_handler,
+                            (erts_aint_t) NIL);
     cache = dep->cache;
     dep->cache = NULL;
 
@@ -673,6 +709,9 @@ static void clear_dist_entry(DistEntry *dep)
 
     erts_smp_mtx_lock(&dep->qlock);
 
+    erts_smp_atomic64_set_nob(&dep->in, 0);
+    erts_smp_atomic64_set_nob(&dep->out, 0);
+
     if (!dep->out_queue.last)
 	obuf = dep->finalized_out_queue.first;
     else {
@@ -680,8 +719,15 @@ static void clear_dist_entry(DistEntry *dep)
 	obuf = dep->out_queue.first;
     }
 
+    if (dep->tmp_out_queue.first) {
+        dep->tmp_out_queue.last->next = obuf;
+        obuf = dep->tmp_out_queue.first;
+    }
+
     dep->out_queue.first = NULL;
     dep->out_queue.last = NULL;
+    dep->tmp_out_queue.first = NULL;
+    dep->tmp_out_queue.last = NULL;
     dep->finalized_out_queue.first = NULL;
     dep->finalized_out_queue.last = NULL;
     dep->status = 0;
@@ -1148,6 +1194,7 @@ int erts_net_message(Port *prt,
     ErtsLink *lnk;
     Uint tuple_arity;
     int res;
+    Uint32 connection_id;
 #ifdef ERTS_DIST_MSG_DBG
     ErlDrvSizeT orig_len = len;
 #endif
@@ -1156,14 +1203,17 @@ int erts_net_message(Port *prt,
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
-    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+    ERTS_SMP_LC_ASSERT(!prt || erts_lc_is_port_locked(prt));
 
     if (!erts_is_alive) {
 	UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
 	return 0;
     }
+
+
     if (hlen != 0)
 	goto data_error;
+
     if (len == 0) {  /* HANDLE TICK !!! */
 	UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
 	return 0;
@@ -1187,25 +1237,31 @@ int erts_net_message(Port *prt,
 	goto data_error;
     }
 
-    res = erts_prepare_dist_ext(&ede, t, len, dep, dep->cache);
+    res = erts_prepare_dist_ext(&ede, t, len, dep, dep->cache, &connection_id);
 
-    if (res >= 0)
-	res = ctl_len = erts_decode_dist_ext_size(&ede);
-    else {
+    switch (res) {
+    case ERTS_PREP_DIST_EXT_CLOSED:
+        return 0; /* Connection not alive; ignore signal... */
+    case ERTS_PREP_DIST_EXT_FAILED:
 #ifdef ERTS_DIST_MSG_DBG
 	erts_fprintf(stderr, "DIST MSG DEBUG: erts_prepare_dist_ext() failed:\n");
 	bw(buf, orig_len);
 #endif
-	ctl_len = 0;
-    }
-
-    if (res < 0) {
+        goto data_error;
+    case ERTS_PREP_DIST_EXT_SUCCESS:
+	ctl_len = erts_decode_dist_ext_size(&ede);
+        if (ctl_len < 0) {
 #ifdef ERTS_DIST_MSG_DBG
-	erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext_size(CTL) failed:\n");
-	bw(buf, orig_len);
+            erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext_size(CTL) failed:\n");
+            bw(buf, orig_len);
 #endif
-	PURIFY_MSG("data error");
-	goto data_error;
+            PURIFY_MSG("data error");
+            goto data_error;
+        }
+        break;
+    default:
+        ERTS_INTERNAL_ERROR("Unexpected result from erts_prepare_dist_ext()");
+        break;
     }
 
     if (ctl_len > DIST_CTL_DEFAULT_SIZE) {
@@ -1726,7 +1782,7 @@ decode_error:
     }
 data_error:
     UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
-    erts_deliver_port_exit(prt, dep->cid, am_killed, 0, 1);
+    erts_kill_dist_connection(dep, connection_id);
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     return -1;
 }
@@ -1747,6 +1803,31 @@ static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy)
     return ret;
 }
 
+static ERTS_INLINE void
+notify_dist_data(Process *c_p, Eterm pid)
+{
+    Process *rp;
+    ErtsProcLocks rp_locks;
+
+    ASSERT(erts_get_scheduler_data()
+           && !ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
+    ASSERT(is_internal_pid(pid));
+
+    if (c_p && c_p->common.id == pid) {
+        rp = c_p;
+        rp_locks = ERTS_PROC_LOCK_MAIN;
+    }
+    else {
+        rp = erts_proc_lookup(pid);
+        rp_locks = 0;
+    }
+
+    if (rp) {
+        ErtsMessage *mp = erts_alloc_message(0, NULL);
+        erts_queue_message(rp, rp_locks, mp, am_dist_data, am_system);
+    }
+}
+
 int
 erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 {
@@ -1865,14 +1946,28 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
                 Sint qsize;
                 erts_aint32_t qflgs;
 		ErtsProcList *plp = NULL;
+                Eterm notify_proc = NIL;
+                Sint obsz = size_obuf(ctx->obuf);
+
 		erts_smp_mtx_lock(&dep->qlock);
-		qsize = erts_smp_atomic_add_read_nob(&dep->qsize,
-                                                     (erts_aint_t) size_obuf(ctx->obuf));
+		qsize = erts_smp_atomic_add_read_nob(&dep->qsize, (erts_aint_t) obsz);
+                ASSERT(qsize >= obsz);
                 qflgs = erts_smp_atomic32_read_nob(&dep->qflgs);
 		if (!(qflgs & ERTS_DE_QFLG_BUSY) && qsize >= erts_dist_buf_busy_limit) {
 		    erts_smp_atomic32_read_bor_relb(&dep->qflgs, ERTS_DE_QFLG_BUSY);
                     qflgs |= ERTS_DE_QFLG_BUSY;
                 }
+                if (qsize == obsz && (qflgs & ERTS_DE_QFLG_REQ_INFO)) {
+                    /* Previously empty queue and info requested... */
+                    qflgs = erts_smp_atomic32_read_band_mb(&dep->qflgs,
+                                                           ~ERTS_DE_QFLG_REQ_INFO);
+                    if (qflgs & ERTS_DE_QFLG_REQ_INFO) {
+                        notify_proc = dep->cid;
+                        ASSERT(is_internal_pid(notify_proc));
+                    }
+                    /* else: requester will send itself the message... */
+                    qflgs &= ~ERTS_DE_QFLG_REQ_INFO;
+                }
 		if (!ctx->force_busy && (qflgs & ERTS_DE_QFLG_BUSY)) {
 		    erts_smp_mtx_unlock(&dep->qlock);
 
@@ -1916,8 +2011,11 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 		}
 
 		erts_smp_mtx_unlock(&dep->qlock);
-		erts_schedule_dist_command(NULL, dep);
+                if (is_internal_port(dep->cid))
+                    erts_schedule_dist_command(NULL, dep);
 		erts_smp_de_runlock(dep);
+                if (is_internal_pid(notify_proc))
+                    notify_dist_data(ctx->c_p, notify_proc);
 
 		if (resume) {
 		    erts_resume(ctx->c_p, ERTS_PROC_LOCK_MAIN);
@@ -1971,16 +2069,20 @@ static Uint
 dist_port_command(Port *prt, ErtsDistOutputBuf *obuf)
 {
     int fpe_was_unmasked;
-    Uint size = obuf->ext_endp - obuf->extp;
+    ErlDrvSizeT size;
+    char *bufp;
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 
-    if (size > (Uint) INT_MAX)
-	erts_exit(ERTS_DUMP_EXIT,
-		 "Absurdly large distribution output data buffer "
-		 "(%beu bytes) passed.\n",
-		 size);
+    if (!obuf) {
+        size = 0;
+        bufp = NULL;
+    }
+    else {
+        size = obuf->ext_endp - obuf->extp;
+        bufp = (char*) obuf->extp;
+    }
 
 #ifdef USE_VM_PROBES
     if (DTRACE_ENABLED(dist_output)) {
@@ -1995,11 +2097,10 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf)
                 remote_str, size);
     }
 #endif
+
     prt->caller = NIL;
     fpe_was_unmasked = erts_block_fpe();
-    (*prt->drv_ptr->output)((ErlDrvData) prt->drv_data,
-			    (char*) obuf->extp,
-			    (int) size);
+    (*prt->drv_ptr->output)((ErlDrvData) prt->drv_data, bufp, size);
     erts_unblock_fpe(fpe_was_unmasked);
     return size;
 }
@@ -2008,7 +2109,7 @@ static Uint
 dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf)
 {
     int fpe_was_unmasked;
-    Uint size = obuf->ext_endp - obuf->extp;
+    ErlDrvSizeT size;
     SysIOVec iov[2];
     ErlDrvBinary* bv[2];
     ErlIOVec eiov;
@@ -2016,25 +2117,33 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf)
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 
-    if (size > (Uint) INT_MAX)
-	erts_exit(ERTS_DUMP_EXIT,
-		 "Absurdly large distribution output data buffer "
-		 "(%beu bytes) passed.\n",
-		 size);
-
     iov[0].iov_base = NULL;
     iov[0].iov_len = 0;
     bv[0] = NULL;
 
-    iov[1].iov_base = obuf->extp;
-    iov[1].iov_len = size;
-    bv[1] = Binary2ErlDrvBinary(ErtsDistOutputBuf2Binary(obuf));
+    if (!obuf) {
+        size = 0;
+        eiov.vsize = 1;
+    }
+    else {
+        size = obuf->ext_endp - obuf->extp;
+        eiov.vsize = 2;
+
+        iov[1].iov_base = obuf->extp;
+        iov[1].iov_len = size;
+        bv[1] = Binary2ErlDrvBinary(ErtsDistOutputBuf2Binary(obuf));
+    }
 
-    eiov.vsize = 2;
     eiov.size = size;
     eiov.iov = iov;
     eiov.binv = bv;
 
+    if (size > (Uint) INT_MAX)
+	erts_exit(ERTS_DUMP_EXIT,
+		 "Absurdly large distribution output data buffer "
+		 "(%beu bytes) passed.\n",
+		 size);
+
     ASSERT(prt->drv_ptr->outputv);
 
 #ifdef USE_VM_PROBES
@@ -2138,20 +2247,20 @@ erts_dist_command(Port *prt, int reds_limit)
     if (!(sched_flags & ERTS_PTS_FLG_BUSY_PORT) && foq.first) {
 	int preempt = 0;
 	do {
-	    Uint size;
-	    ErtsDistOutputBuf *fob;
-
-	    size = (*send)(prt, foq.first);
-	    esdp->io.out += (Uint64) size;
+            Uint size;
+            ErtsDistOutputBuf *fob;
+            size = (*send)(prt, foq.first);
+            erts_smp_atomic64_inc_nob(&dep->out);
+            esdp->io.out += (Uint64) size;
 #ifdef ERTS_RAW_DIST_MSG_DBG
-	    erts_fprintf(stderr, ">> ");
-	    bw(foq.first->extp, size);
+            erts_fprintf(stderr, ">> ");
+            bw(foq.first->extp, size);
 #endif
-	    reds += ERTS_PORT_REDS_DIST_CMD_DATA(size);
-	    fob = foq.first;
-	    obufsize += size_obuf(fob);
-	    foq.first = foq.first->next;
-	    free_dist_obuf(fob);
+            reds += ERTS_PORT_REDS_DIST_CMD_DATA(size);
+            fob = foq.first;
+            obufsize += size_obuf(fob);
+            foq.first = foq.first->next;
+            free_dist_obuf(fob);
 	    sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
 	    preempt = reds > reds_limit || (sched_flags & ERTS_PTS_FLG_EXIT);
 	    if (sched_flags & ERTS_PTS_FLG_BUSY_PORT)
@@ -2215,29 +2324,30 @@ erts_dist_command(Port *prt, int reds_limit)
         int de_busy;
 	int preempt = 0;
 	while (oq.first && !preempt) {
-	    ErtsDistOutputBuf *fob;
-	    Uint size;
-	    oq.first->extp
-		= erts_encode_ext_dist_header_finalize(oq.first->extp,
-						       dep->cache,
-						       flags);
-	    reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE;
-	    if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
-		*--oq.first->extp = PASS_THROUGH; /* Old node; 'pass through'
-						     needed */
-	    ASSERT(&oq.first->data[0] <= oq.first->extp
-		   && oq.first->extp < oq.first->ext_endp);
-	    size = (*send)(prt, oq.first);
-	    esdp->io.out += (Uint64) size;
+            ErtsDistOutputBuf *fob;
+            Uint size;
+            oq.first->extp
+                = erts_encode_ext_dist_header_finalize(oq.first->extp,
+                                                       dep->cache,
+                                                       flags);
+            reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE;
+            if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
+                *--oq.first->extp = PASS_THROUGH; /* Old node; 'pass through'
+                                                     needed */
+            ASSERT(&oq.first->data[0] <= oq.first->extp
+                   && oq.first->extp < oq.first->ext_endp);
+            size = (*send)(prt, oq.first);
+            erts_smp_atomic64_inc_nob(&dep->out);
+            esdp->io.out += (Uint64) size;
 #ifdef ERTS_RAW_DIST_MSG_DBG
-	    erts_fprintf(stderr, ">> ");
-	    bw(oq.first->extp, size);
+            erts_fprintf(stderr, ">> ");
+            bw(oq.first->extp, size);
 #endif
-	    reds += ERTS_PORT_REDS_DIST_CMD_DATA(size);
-	    fob = oq.first;
-	    obufsize += size_obuf(fob);
-	    oq.first = oq.first->next;
-	    free_dist_obuf(fob);
+            reds += ERTS_PORT_REDS_DIST_CMD_DATA(size);
+            fob = oq.first;
+            obufsize += size_obuf(fob);
+            oq.first = oq.first->next;
+            free_dist_obuf(fob);
 	    sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
 	    preempt = reds > reds_limit || (sched_flags & ERTS_PTS_FLG_EXIT);
 	    if ((sched_flags & ERTS_PTS_FLG_BUSY_PORT) && oq.first && !preempt)
@@ -2379,6 +2489,374 @@ erts_dist_command(Port *prt, int reds_limit)
     goto done;
 }
 
+#if 0
+
+int
+dist_data_finalize(Process *c_p, int reds_limit)
+{
+    int reds = 5;
+    DistEntry *dep = ;
+    ErtsDistOutputQueue oq, foq;
+    ErtsDistOutputBuf *ob;
+    int preempt;
+
+
+    erts_smp_mtx_lock(&dep->qlock);
+    flags = dep->flags;
+    oq.first = dep->out_queue.first;
+    oq.last = dep->out_queue.last;
+    dep->out_queue.first = NULL;
+    dep->out_queue.last = NULL;
+    erts_smp_mtx_unlock(&dep->qlock);
+
+    if (!oq.first) {
+        ASSERT(!oq.last);
+        oq.first = dep->tmp_out_queue.first;
+        oq.last = dep->tmp_out_queue.last;
+    }
+    else {
+        ErtsDistOutputBuf *f, *l;
+        ASSERT(oq.last);
+        if (dep->tmp_out_queue.last) {
+            dep->tmp_out_queue.last->next = oq.first;
+            oq.first = dep->tmp_out_queue.first;
+        }
+    }
+
+    if (!oq.first) {
+        /* Nothing to do... */
+        ASSERT(!oq.last);
+        return reds;
+    }
+
+    foq.first = dep->finalized_out_queue.first;
+    foq.last = dep->finalized_out_queue.last;
+
+    preempt = 0;
+    ob = oq.first;
+    ASSERT(ob);
+
+    do {
+        ob->extp = erts_encode_ext_dist_header_finalize(ob->extp,
+                                                        dep->cache,
+                                                        flags);
+        if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
+            *--ob->extp = PASS_THROUGH; /* Old node; 'pass through'
+                                           needed */
+        ASSERT(&ob->data[0] <= ob->extp && ob->extp < ob->ext_endp);
+        reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE;
+        preempt = reds > reds_limit;
+        if (preempt)
+            break;
+        ob = ob->next;
+    } while (ob);
+    /*
+     * At least one buffer was finalized; if we got preempted,
+     * ob points to the last buffer that we finalized.
+     */
+    if (foq.last)
+        foq.last->next = oq.first;
+    else
+        foq.first = oq.first;
+    if (!preempt) {
+        /* All buffers finalized */
+        foq.last = oq.last;
+        oq.first = oq.last = NULL;
+    }
+    else {
+        /* Not all buffers finalized; split oq. */
+        foq.last = ob;
+        oq.first = ob->next;
+        if (oq.first)
+            ob->next = NULL;
+        else
+            oq.last = NULL;
+    }
+
+    dep->finalized_out_queue.first = foq.first;
+    dep->finalized_out_queue.last = foq.last;
+    dep->tmp_out_queue.first = oq.first;
+    dep->tmp_out_queue.last = oq.last;
+
+    return reds;
+}
+
+#endif
+
+BIF_RETTYPE
+dist_ctrl_get_data_notification_1(BIF_ALIST_1)
+{
+    DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P);
+    erts_aint32_t qflgs;
+    erts_aint_t qsize;
+    Eterm receiver = NIL;
+
+    if (!dep)
+        BIF_ERROR(BIF_P, EXC_NOTSUP);
+
+    if (dep->sysname != BIF_ARG_1)
+        BIF_ERROR(BIF_P, BADARG);
+
+    /*
+     * Caller is the only one that can consume from this queue
+     * and the only one that can set the req-info flag...
+     */
+
+    erts_smp_de_rlock(dep);
+
+    ASSERT(dep->cid == BIF_P->common.id);
+
+    qflgs = erts_smp_atomic32_read_acqb(&dep->qflgs);
+
+    if (!(qflgs & ERTS_DE_QFLG_REQ_INFO)) {
+        qsize = erts_smp_atomic_read_acqb(&dep->qsize);
+        ASSERT(qsize >= 0);
+        if (qsize > 0)
+            receiver = BIF_P->common.id; /* Notify ourselves... */
+        else { /* Empty queue; set req-info flag... */
+            qflgs = erts_smp_atomic32_read_bor_mb(&dep->qflgs,
+                                                  ERTS_DE_QFLG_REQ_INFO);
+            qsize = erts_smp_atomic_read_acqb(&dep->qsize);
+            ASSERT(qsize >= 0);
+            if (qsize > 0) {
+                qflgs = erts_smp_atomic32_read_band_mb(&dep->qflgs,
+                                                       ~ERTS_DE_QFLG_REQ_INFO);
+                if (qflgs & ERTS_DE_QFLG_REQ_INFO)
+                    receiver = BIF_P->common.id; /* Notify ourselves... */
+                /* else: someone else will notify us... */
+            }
+            /* else: still empty queue... */
+        }
+    }
+    /* else: Already requested... */
+
+    erts_smp_de_runlock(dep);
+
+    if (is_internal_pid(receiver))
+        notify_dist_data(BIF_P, receiver);
+
+    BIF_RET(am_ok);
+}
+
+BIF_RETTYPE
+dist_ctrl_put_data_2(BIF_ALIST_2)
+{
+    DistEntry *dep;
+    ErlDrvSizeT size;
+    Eterm input_handler;
+
+    if (is_binary(BIF_ARG_2))
+        size = binary_size(BIF_ARG_2);
+    else if (is_nil(BIF_ARG_2))
+        size = 0;
+    else if (is_list(BIF_ARG_2))
+        BIF_TRAP2(dist_ctrl_put_data_trap,
+                  BIF_P, BIF_ARG_1, BIF_ARG_2);
+    else
+        BIF_ERROR(BIF_P, BADARG);
+
+    dep = erts_find_dist_entry(BIF_ARG_1);
+    if (!dep)
+        BIF_ERROR(BIF_P, BADARG);
+
+    input_handler = (Eterm) erts_smp_atomic_read_nob(&dep->input_handler);
+
+    if (input_handler != BIF_P->common.id)
+        BIF_ERROR(BIF_P, EXC_NOTSUP);
+
+    erts_smp_atomic64_inc_nob(&dep->in);
+
+    if (size != 0) {
+        byte *data, *temp_alloc = NULL;
+
+        data = (byte *) erts_get_aligned_binary_bytes(BIF_ARG_2, &temp_alloc);
+        if (!data)
+            BIF_ERROR(BIF_P, BADARG);
+
+        erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+
+        (void) erts_net_message(NULL, dep, NULL, 0, data, size);
+        /*
+         * We ignore any decode failures. On fatal failures the
+         * connection will be taken down by killing the
+         * distribution channel controller...
+         */
+
+        erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+
+        BUMP_REDS(BIF_P, 5);
+
+        erts_free_aligned_binary_bytes(temp_alloc);
+
+    }
+
+    BIF_RET(am_ok);
+}
+
+BIF_RETTYPE
+dist_get_stat_1(BIF_ALIST_1)
+{
+    Sint64 read, write, pend;
+    Eterm res, *hp, **hpp;
+    Uint sz, *szp;
+    DistEntry *dep = erts_find_dist_entry(BIF_ARG_1);
+
+    if (!dep)
+        BIF_ERROR(BIF_P, BADARG);
+
+    ASSERT(dep->sysname == BIF_ARG_1);
+
+    erts_smp_de_rlock(dep);
+
+    read = (Sint64) erts_smp_atomic64_read_nob(&dep->in);
+    write = (Sint64) erts_smp_atomic64_read_nob(&dep->out);
+    pend = (Sint64) erts_smp_atomic_read_nob(&dep->qsize);
+
+    erts_smp_de_runlock(dep);
+
+    erts_deref_dist_entry(dep);
+
+    sz = 0;
+    szp = &sz;
+    hpp = NULL;
+
+    while (1) {
+        res = erts_bld_tuple(hpp, szp, 4,
+                             am_ok,
+                             erts_bld_sint64(hpp, szp, read),
+                             erts_bld_sint64(hpp, szp, write),
+                             pend ? am_true : am_false);
+        if (hpp)
+            break;
+        hp = HAlloc(BIF_P, sz);
+        hpp = &hp;
+        szp = NULL;
+    }
+
+    BIF_RET(res);
+}
+
+BIF_RETTYPE
+dist_ctrl_input_handler_2(BIF_ALIST_2)
+{
+    DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P);
+
+    if (!dep)
+        BIF_ERROR(BIF_P, EXC_NOTSUP);
+
+    if (dep->sysname != BIF_ARG_1)
+        BIF_ERROR(BIF_P, BADARG);
+
+    if (is_not_internal_pid(BIF_ARG_2))
+        BIF_ERROR(BIF_P, BADARG);
+
+    erts_smp_atomic_set_nob(&dep->input_handler,
+                            (erts_aint_t) BIF_ARG_2);
+
+    BIF_RET(am_ok);
+}
+
+BIF_RETTYPE
+dist_ctrl_get_data_1(BIF_ALIST_1)
+{
+    DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P);
+    int reds = 1;
+    ErtsDistOutputBuf *obuf;
+    Eterm *hp;
+    ProcBin *pb;
+    erts_aint_t qsize;
+
+    if (!dep)
+        BIF_ERROR(BIF_P, EXC_NOTSUP);
+
+    if (dep->sysname != BIF_ARG_1)
+        BIF_ERROR(BIF_P, BADARG);
+
+    erts_smp_de_rlock(dep);
+
+    if (dep->status & ERTS_DE_SFLG_EXITING)
+        goto return_none;
+
+    ASSERT(dep->cid == BIF_P->common.id);
+
+#if 0
+    if (dep->finalized_out_queue.first) {
+        obuf = dep->finalized_out_queue.first;
+        dep->finalized_out_queue.first = obuf->next;
+        if (!obuf->next)
+            dep->finalized_out_queue.last = NULL;
+    }
+    else
+#endif
+    {
+        if (!dep->tmp_out_queue.first) {
+            ASSERT(!dep->tmp_out_queue.last);
+            qsize = erts_smp_atomic_read_acqb(&dep->qsize);
+            if (qsize > 0) {
+                erts_smp_mtx_lock(&dep->qlock);
+                dep->tmp_out_queue.first = dep->out_queue.first;
+                dep->tmp_out_queue.last = dep->out_queue.last;
+                dep->out_queue.first = NULL;
+                dep->out_queue.last = NULL;
+                erts_smp_mtx_unlock(&dep->qlock);
+            }
+        }
+
+        if (!dep->tmp_out_queue.first) {
+            ASSERT(!dep->tmp_out_queue.last);
+        return_none:
+            erts_smp_de_runlock(dep);
+            BIF_RET(am_none);
+        }
+        else {
+            obuf = dep->tmp_out_queue.first;
+            dep->tmp_out_queue.first = obuf->next;
+            if (!obuf->next)
+                dep->tmp_out_queue.last = NULL;
+        }
+
+        obuf->extp = erts_encode_ext_dist_header_finalize(obuf->extp,
+                                                          dep->cache,
+                                                          dep->flags);
+        reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE;
+        if (!(dep->flags & DFLAG_DIST_HDR_ATOM_CACHE))
+            *--obuf->extp = PASS_THROUGH; /* 'pass through' needed */
+        ASSERT(&obuf->data[0] <= obuf->extp
+               && obuf->extp < obuf->ext_endp);
+    }
+
+    erts_smp_atomic64_inc_nob(&dep->out);
+
+    erts_smp_de_runlock(dep);
+
+    hp =  HAlloc(BIF_P, PROC_BIN_SIZE);
+    pb = (ProcBin *) (char *) hp;
+    pb->thing_word = HEADER_PROC_BIN;
+    pb->size = obuf->ext_endp - obuf->extp;
+    pb->next = MSO(BIF_P).first;
+    MSO(BIF_P).first = (struct erl_off_heap_header*) pb;
+    pb->val = ErtsDistOutputBuf2Binary(obuf);
+    pb->bytes = (byte*) obuf->extp;
+    pb->flags = 0;
+
+    qsize = erts_smp_atomic_add_read_nob(&dep->qsize, -size_obuf(obuf));
+    ASSERT(qsize >= 0);
+
+    if (qsize < erts_dist_buf_busy_limit/2
+        && (erts_smp_atomic32_read_acqb(&dep->qflgs) & ERTS_DE_QFLG_BUSY)) {
+        ErtsProcList *resume_procs = NULL;
+        erts_smp_mtx_lock(&dep->qlock);
+        resume_procs = get_suspended_on_de(dep, ERTS_DE_QFLG_BUSY);
+        erts_smp_mtx_unlock(&dep->qlock);
+        if (resume_procs) {
+            int resumed = erts_resume_processes(resume_procs);
+            reds += resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED;
+        }
+    }
+
+    BIF_RET2(make_binary(pb), reds);
+}
+
 void
 erts_dist_port_not_busy(Port *prt)
 {
@@ -2402,8 +2880,7 @@ void
 erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id)
 {
     erts_smp_de_rwlock(dep);
-    if (is_internal_port(dep->cid)
-	&& connection_id == dep->connection_id
+    if (connection_id == dep->connection_id
 	&& !(dep->status & ERTS_DE_SFLG_EXITING)) {
 
 	dep->status |= ERTS_DE_SFLG_EXITING;
@@ -2413,7 +2890,10 @@ erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id)
 	erts_smp_atomic32_read_bor_nob(&dep->qflgs, ERTS_DE_QFLG_EXIT);
 	erts_smp_mtx_unlock(&dep->qlock);
 
-	erts_schedule_dist_command(NULL, dep);
+        if (is_internal_port(dep->cid))
+            erts_schedule_dist_command(NULL, dep);
+        else if (is_internal_pid(dep->cid))
+            schedule_kill_dist_ctrl_proc(dep->cid);
     }
     erts_smp_de_rwunlock(dep);
 }
@@ -2652,17 +3132,23 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2)
 	goto error;
     }
 
-    net_kernel = erts_whereis_process(BIF_P, ERTS_PROC_LOCK_MAIN,
-				      am_net_kernel, ERTS_PROC_LOCK_MAIN, 0);
-    if (!net_kernel)
+    net_kernel = erts_whereis_process(BIF_P,
+                                      ERTS_PROC_LOCK_MAIN,
+				      am_net_kernel,
+                                      ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS,
+                                      0);
+    if (!net_kernel || ERTS_PROC_GET_DIST_ENTRY(net_kernel))
 	goto error;
 
     /* By setting F_DISTRIBUTION on net_kernel,
-     * do_net_exist will be called when net_kernel is terminated !! */
+     * erts_do_net_exits will be called when net_kernel is terminated !! */
     net_kernel->flags |= F_DISTRIBUTION;
 
-    if (net_kernel != BIF_P)
-	erts_smp_proc_unlock(net_kernel, ERTS_PROC_LOCK_MAIN);
+    erts_smp_proc_unlock(net_kernel,
+                         (ERTS_PROC_LOCK_STATUS
+                          | ((net_kernel != BIF_P)
+                             ? ERTS_PROC_LOCK_MAIN
+                             : 0)));
 
 #ifdef DEBUG
     erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
@@ -2679,6 +3165,14 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2)
     erts_smp_thr_progress_unblock();
     erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
+    /*
+     * Note erts_this_dist_entry is changed by erts_set_this_node(),
+     * so we *need* to use the new one after erts_set_this_node()
+     * is called.
+     */
+    erts_smp_refc_inc(&erts_this_dist_entry->refc, 1);
+    ERTS_PROC_SET_DIST_ENTRY(net_kernel, erts_this_dist_entry);
+
     BIF_RET(am_true);
 
  error:
@@ -2708,18 +3202,21 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     Eterm ic, oc;
     Eterm *tp;
     DistEntry *dep = NULL;
+    ErtsProcLocks proc_unlock = 0;
+    Process *proc;
     Port *pp = NULL;
 
     /* Prepare for success */
-    ERTS_BIF_PREP_RET(ret, am_true);
+    ERTS_BIF_PREP_RET(ret, BIF_ARG_1);
 
     /*
      * Check and pick out arguments
      */
 
     if (!is_node_name_atom(BIF_ARG_1) ||
-	is_not_internal_port(BIF_ARG_2) ||
-	(erts_this_node->sysname == am_Noname)) {
+	!(is_internal_port(BIF_ARG_2)
+          || is_internal_pid(BIF_ARG_2))
+	|| (erts_this_node->sysname == am_Noname)) {
 	goto badarg;
     }
 
@@ -2763,74 +3260,116 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     else if (!dep)
 	goto system_limit; /* Should never happen!!! */
 
-    pp = erts_id2port_sflgs(BIF_ARG_2,
-			    BIF_P,
-			    ERTS_PROC_LOCK_MAIN,
-			    ERTS_PORT_SFLGS_INVALID_LOOKUP);
-    erts_smp_de_rwlock(dep);
+    if (is_internal_pid(BIF_ARG_2)) {
+        if (BIF_P->common.id == BIF_ARG_2) {
+            proc_unlock = 0;
+            proc = BIF_P;
+        }
+        else {
+            proc_unlock = ERTS_PROC_LOCK_MAIN;
+            proc = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN,
+                                             BIF_ARG_2, proc_unlock);
+        }
+        erts_smp_de_rwlock(dep);
 
-    if (!pp || (erts_atomic32_read_nob(&pp->state)
-		& ERTS_PORT_SFLG_EXITING))
-	goto badarg;
+        if (!proc)
+            goto badarg;
+        else if (proc == ERTS_PROC_LOCK_BUSY) {
+            proc_unlock = 0;
+            goto yield;
+        }
 
-    if ((pp->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY) == 0)
-	goto badarg;
+        erts_smp_proc_lock(proc, ERTS_PROC_LOCK_STATUS);
+        proc_unlock |= ERTS_PROC_LOCK_STATUS;
 
-    if (dep->cid == BIF_ARG_2 && pp->dist_entry == dep)
-	goto done; /* Already set */
+        if (ERTS_PROC_GET_DIST_ENTRY(proc)) {
+            if (dep == ERTS_PROC_GET_DIST_ENTRY(proc)
+                && (proc->flags & F_DISTRIBUTION)
+                && dep->cid == BIF_ARG_2)
+                goto done;
+            goto badarg;
+        }
 
-    if (dep->status & ERTS_DE_SFLG_EXITING) {
-	/* Suspend on dist entry waiting for the exit to finish */
-	ErtsProcList *plp = erts_proclist_create(BIF_P);
-	plp->next = NULL;
-	erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
-	erts_smp_mtx_lock(&dep->qlock);
-	erts_proclist_store_last(&dep->suspended, plp);
-	erts_smp_mtx_unlock(&dep->qlock);
-	goto yield;
-    }
+        if (is_not_nil(dep->cid))
+            goto badarg;
 
-    ASSERT(!(dep->status & ERTS_DE_SFLG_EXITING));
+        proc->flags |= F_DISTRIBUTION;
+        ERTS_PROC_SET_DIST_ENTRY(proc, dep);
 
-    if (pp->dist_entry || is_not_nil(dep->cid))
-	goto badarg;
+        proc_unlock &= ~ERTS_PROC_LOCK_STATUS;
+        erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_STATUS);
 
-    erts_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION);
+        dep->send = NULL; /* Only for distr ports... */
 
-    /*
-     * Dist-ports do not use the "busy port message queue" functionality, but
-     * instead use "busy dist entry" functionality.
-     */
-    {
-	ErlDrvSizeT disable = ERL_DRV_BUSY_MSGQ_DISABLED;
-	erl_drv_busy_msgq_limits(ERTS_Port2ErlDrvPort(pp), &disable, NULL);
     }
+    else {
 
-    pp->dist_entry = dep;
+        pp = erts_id2port_sflgs(BIF_ARG_2,
+                                BIF_P,
+                                ERTS_PROC_LOCK_MAIN,
+                                ERTS_PORT_SFLGS_INVALID_LOOKUP);
+        erts_smp_de_rwlock(dep);
+
+        if (!pp || (erts_atomic32_read_nob(&pp->state)
+                    & ERTS_PORT_SFLG_EXITING))
+            goto badarg;
+
+        if ((pp->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY) == 0)
+            goto badarg;
+
+        if (dep->cid == BIF_ARG_2 && pp->dist_entry == dep)
+            goto done; /* Already set */
+
+        if (dep->status & ERTS_DE_SFLG_EXITING) {
+            /* Suspend on dist entry waiting for the exit to finish */
+            ErtsProcList *plp = erts_proclist_create(BIF_P);
+            plp->next = NULL;
+            erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
+            erts_smp_mtx_lock(&dep->qlock);
+            erts_proclist_store_last(&dep->suspended, plp);
+            erts_smp_mtx_unlock(&dep->qlock);
+            goto yield;
+        }
 
-    dep->version = version;
-    dep->creation = 0;
+        ASSERT(!(dep->status & ERTS_DE_SFLG_EXITING));
 
-    ASSERT(pp->drv_ptr->outputv || pp->drv_ptr->output);
+        if (pp->dist_entry || is_not_nil(dep->cid))
+            goto badarg;
 
-#if 1
-    dep->send = (pp->drv_ptr->outputv
-		 ? dist_port_commandv
-		 : dist_port_command);
-#else
-    dep->send = dist_port_command;
-#endif
-    ASSERT(dep->send);
+        erts_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION);
+
+        pp->dist_entry = dep;
+
+        ASSERT(pp->drv_ptr->outputv || pp->drv_ptr->output);
+
+        dep->send = (pp->drv_ptr->outputv
+                     ? dist_port_commandv
+                     : dist_port_command);
+        ASSERT(dep->send);
+
+        /*
+         * Dist-ports do not use the "busy port message queue" functionality, but
+         * instead use "busy dist entry" functionality.
+         */
+        {
+            ErlDrvSizeT disable = ERL_DRV_BUSY_MSGQ_DISABLED;
+            erl_drv_busy_msgq_limits(ERTS_Port2ErlDrvPort(pp), &disable, NULL);
+        }
+
+    }
+
+    dep->version = version;
+    dep->creation = 0;
 
 #ifdef DEBUG
     ASSERT(erts_smp_atomic_read_nob(&dep->qsize) == 0);
 #endif
 
-    erts_set_dist_entry_connected(dep, BIF_ARG_2, flags);
-
     if (flags & DFLAG_DIST_HDR_ATOM_CACHE)
 	create_cache(dep);
 
+    erts_set_dist_entry_connected(dep, BIF_ARG_2, flags);
+
     erts_smp_de_rwunlock(dep);
     dep = NULL; /* inc of refc transferred to port (dist_entry field) */
 
@@ -2851,6 +3390,9 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     if (pp)
 	erts_port_release(pp);
 
+    if (proc_unlock)
+        erts_smp_proc_unlock(proc, proc_unlock);
+
     return ret;
 
  yield:
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 96f9b284b3..8846866eb7 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3938,12 +3938,12 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 			       DFLAG_BIT_BINARIES);
 		BIF_RET(erts_term_to_binary(BIF_P, tp[2], 0, dflags));
 	    }
-	    else if (ERTS_IS_ATOM_STR("dist_port", tp[1])) {
+	    else if (ERTS_IS_ATOM_STR("dist_ctrl", tp[1])) {
 		Eterm res = am_undefined;
 		DistEntry *dep = erts_sysname_to_connected_dist_entry(tp[2]);
 		if (dep) {
 		    erts_smp_de_rlock(dep);
-		    if (is_internal_port(dep->cid))
+		    if (is_internal_port(dep->cid) || is_internal_pid(dep->cid))
 			res = dep->cid;
 		    erts_smp_de_runlock(dep);
 		    erts_deref_dist_entry(dep);
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index c1796a8894..4ef1d54906 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -102,6 +102,7 @@ dist_table_alloc(void *dep_tmpl)
     erts_smp_rwmtx_init_opt_x(&dep->rwmtx, &rwmtx_opt, "dist_entry", chnl_nr);
     dep->sysname			= sysname;
     dep->cid				= NIL;
+    erts_smp_atomic_init_nob(&dep->input_handler, (erts_aint_t) NIL);
     dep->connection_id			= 0;
     dep->status				= 0;
     dep->flags				= 0;
@@ -115,10 +116,14 @@ dist_table_alloc(void *dep_tmpl)
     erts_smp_mtx_init_x(&dep->qlock, "dist_entry_out_queue", chnl_nr);
     erts_smp_atomic32_init_nob(&dep->qflgs, 0);
     erts_smp_atomic_init_nob(&dep->qsize, 0);
+    erts_smp_atomic64_init_nob(&dep->in, 0);
+    erts_smp_atomic64_init_nob(&dep->out, 0);
     dep->out_queue.first		= NULL;
     dep->out_queue.last			= NULL;
     dep->suspended			= NULL;
 
+    dep->tmp_out_queue.first    	= NULL;
+    dep->tmp_out_queue.last     	= NULL;
     dep->finalized_out_queue.first	= NULL;
     dep->finalized_out_queue.last	= NULL;
 
@@ -383,7 +388,7 @@ erts_set_dist_entry_not_connected(DistEntry *dep)
     erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx);
 
     ASSERT(dep != erts_this_dist_entry);
-    ASSERT(is_internal_port(dep->cid));
+    ASSERT(is_internal_port(dep->cid) || is_internal_pid(dep->cid));
 
     if(dep->flags & DFLAG_PUBLISHED) {
 	if(dep->prev) {
@@ -438,7 +443,7 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags)
 
     ASSERT(dep != erts_this_dist_entry);
     ASSERT(is_nil(dep->cid));
-    ASSERT(is_internal_port(cid));
+    ASSERT(is_internal_port(cid) || is_internal_pid(cid));
 
     if(dep->prev) {
 	ASSERT(is_in_de_list(dep, erts_not_connected_dist_entries));
@@ -458,10 +463,19 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags)
     dep->status |= ERTS_DE_SFLG_CONNECTED;
     dep->flags = flags;
     dep->cid = cid;
+    erts_smp_atomic_set_nob(&dep->input_handler,
+                            (erts_aint_t) cid);
+
     dep->connection_id++;
     dep->connection_id &= ERTS_DIST_EXT_CON_ID_MASK;
     dep->prev = NULL;
 
+    erts_smp_atomic64_set_nob(&dep->in, 0);
+    erts_smp_atomic64_set_nob(&dep->out, 0);
+    erts_smp_atomic32_set_nob(&dep->qflgs,
+                              (is_internal_port(cid)
+                               ? ERTS_DE_QFLG_PORT_CTRL
+                               : ERTS_DE_QFLG_PROC_CTRL));
     if(flags & DFLAG_PUBLISHED) {
 	dep->next = erts_visible_dist_entries;
 	if(erts_visible_dist_entries) {
@@ -1397,6 +1411,14 @@ setup_reference_table(void)
 		insert_links(ERTS_P_LINKS(proc), proc->common.id);
 	    if (ERTS_P_MONITORS(proc))
 		insert_monitors(ERTS_P_MONITORS(proc), proc->common.id);
+            {
+                DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc);
+                if (dep)
+                    insert_dist_entry(dep,
+                                      CTRL_REF,
+                                      proc->common.id,
+                                      0);
+            }
 	}
     }
     
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 04acfe41b1..5cff5e14c2 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -61,11 +61,17 @@
 #define ERTS_DE_SFLGS_ALL			(ERTS_DE_SFLG_CONNECTED \
 						 | ERTS_DE_SFLG_EXITING)
 
-#define ERTS_DE_QFLG_BUSY			(((Uint32) 1) <<  0)
-#define ERTS_DE_QFLG_EXIT			(((Uint32) 1) <<  1)
+#define ERTS_DE_QFLG_BUSY			(((erts_aint32_t) 1) <<  0)
+#define ERTS_DE_QFLG_EXIT			(((erts_aint32_t) 1) <<  1)
+#define ERTS_DE_QFLG_REQ_INFO			(((erts_aint32_t) 1) <<  2)
+#define ERTS_DE_QFLG_PORT_CTRL                  (((erts_aint32_t) 1) <<  3)
+#define ERTS_DE_QFLG_PROC_CTRL                  (((erts_aint32_t) 1) <<  4)
 
 #define ERTS_DE_QFLGS_ALL			(ERTS_DE_QFLG_BUSY \
-						 | ERTS_DE_QFLG_EXIT)
+						 | ERTS_DE_QFLG_EXIT \
+                                                 | ERTS_DE_QFLG_REQ_INFO \
+                                                 | ERTS_DE_QFLG_PORT_CTRL \
+                                                 | ERTS_DE_QFLG_PROC_CTRL)
 
 #if defined(ARCH_64)
 #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713f713f713UL)
@@ -112,6 +118,7 @@ typedef struct dist_entry_ {
     erts_smp_rwmtx_t rwmtx;     /* Protects all fields below until lck_mtx. */
     Eterm sysname;		/* name@host atom for efficiency */
     Uint32 creation;		/* creation of connected node */
+    erts_smp_atomic_t input_handler; /* Input handler */
     Eterm cid;			/* connection handler (pid or port), NIL == free */
     Uint32 connection_id;	/* Connection id incremented on connect */
     Uint32 status;		/* Slot status, like exiting reserved etc */
@@ -135,9 +142,12 @@ typedef struct dist_entry_ {
     erts_smp_mtx_t qlock;       /* Protects qflgs and out_queue */
     erts_smp_atomic32_t qflgs;
     erts_smp_atomic_t qsize;
+    erts_smp_atomic64_t in;
+    erts_smp_atomic64_t out;
     ErtsDistOutputQueue out_queue;
     struct ErtsProcList_ *suspended;
 
+    ErtsDistOutputQueue tmp_out_queue;
     ErtsDistOutputQueue finalized_out_queue;
     erts_smp_atomic_t dist_cmd_scheduled;
     ErtsPortTaskHandle dist_cmd;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index fc2b34e70f..2780c111af 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -796,6 +796,11 @@ erts_pre_init_process(void)
         = ERTS_PSD_ETS_FIXED_TABLES_GET_LOCKS;
     erts_psd_required_locks[ERTS_PSD_ETS_FIXED_TABLES].set_locks
         = ERTS_PSD_ETS_FIXED_TABLES_SET_LOCKS;
+
+    erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks
+        = ERTS_PSD_DIST_ENTRY_GET_LOCKS;
+    erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks
+        = ERTS_PSD_DIST_ENTRY_SET_LOCKS;
 #endif
 }
 
@@ -13857,7 +13862,7 @@ erts_continue_exit_process(Process *p)
     ErtsMonitor *mon;
     ErtsProcLocks curr_locks = ERTS_PROC_LOCK_MAIN;
     Eterm reason = p->fvalue;
-    DistEntry *dep;
+    DistEntry *dep = NULL;
     erts_aint32_t state;
     int delay_del_proc = 0;
 
@@ -14064,13 +14069,16 @@ erts_continue_exit_process(Process *p)
 	if (refc_inced && !(n & ERTS_PSFLG_IN_RUNQ))
 	    erts_proc_dec_refc(p);
     }
-    
-    dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL;
+
+    dep = ((p->flags & F_DISTRIBUTION)
+           ? ERTS_PROC_SET_DIST_ENTRY(p, NULL)
+           : NULL);
 
     erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
 
     if (dep) {
-	erts_do_net_exits(dep, reason);
+        erts_do_net_exits(dep, (reason == am_kill) ? am_killed : reason);
+        erts_deref_dist_entry(dep);
     }
 
     /*
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 9d7ba27c50..9e2ac15f13 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -835,14 +835,15 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
 #define ERTS_PSD_NIF_TRAP_EXPORT		5
 #define ERTS_PSD_ETS_OWNED_TABLES               6
 #define ERTS_PSD_ETS_FIXED_TABLES               7
-#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF	8
+#define ERTS_PSD_DIST_ENTRY	                8
+#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF	9 /* keep last... */
 
-#define ERTS_PSD_SIZE				9
+#define ERTS_PSD_SIZE				10
 
 #if !defined(HIPE)
 #  undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF
 #  undef ERTS_PSD_SIZE
-#  define ERTS_PSD_SIZE 8
+#  define ERTS_PSD_SIZE 9
 #endif
 
 typedef struct {
@@ -876,6 +877,9 @@ typedef struct {
 #define ERTS_PSD_ETS_FIXED_TABLES_GET_LOCKS ERTS_PROC_LOCK_MAIN
 #define ERTS_PSD_ETS_FIXED_TABLES_SET_LOCKS ERTS_PROC_LOCK_MAIN
 
+#define ERTS_PSD_DIST_ENTRY_GET_LOCKS ERTS_PROC_LOCK_MAIN
+#define ERTS_PSD_DIST_ENTRY_SET_LOCKS ERTS_PROC_LOCK_MAIN
+
 typedef struct {
     ErtsProcLocks get_locks;
     ErtsProcLocks set_locks;
@@ -2103,6 +2107,11 @@ erts_psd_set(Process *p, int ix, void *data)
 #define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, NTE) \
     erts_psd_set((P), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))
 
+#define ERTS_PROC_GET_DIST_ENTRY(P) \
+    ((DistEntry *) erts_psd_get((P), ERTS_PSD_DIST_ENTRY))
+#define ERTS_PROC_SET_DIST_ENTRY(P, DE) \
+    ((DistEntry *) erts_psd_set((P), ERTS_PSD_DIST_ENTRY, (void *) (DE)))
+
 #ifdef HIPE
 #define ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(P) \
   ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF))
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 1560844521..5bce4c3c92 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -629,7 +629,8 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
 		      byte *ext,
 		      Uint size,
 		      DistEntry *dep,
-		      ErtsAtomCache *cache)
+		      ErtsAtomCache *cache,
+                      Uint32 *connection_id)
 {
 #undef ERTS_EXT_FAIL
 #undef ERTS_EXT_HDR_FAIL
@@ -650,33 +651,36 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
     if (size < 2)
 	ERTS_EXT_FAIL;
 
+    if (!dep)
+        ERTS_INTERNAL_ERROR("Invalid use");
+
     if (ep[0] != VERSION_MAGIC) {
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	if (dep)
-	    erts_dsprintf(dsbufp,
-			  "** Got message from incompatible erlang on "
-			  "channel %d\n",
-			  dist_entry_channel_no(dep));
-	else
-	    erts_dsprintf(dsbufp,
-			  "** Attempt to convert old incompatible "
-			  "binary %d\n",
-			  *ep);
+        erts_dsprintf(dsbufp,
+                      "** Got message from incompatible erlang on "
+                      "channel %d\n",
+                      dist_entry_channel_no(dep));
 	erts_send_error_to_logger_nogl(dsbufp);
 	ERTS_EXT_FAIL;
     }
 
     edep->flags = 0;
     edep->dep = dep;
-    if (dep) {
-	erts_smp_de_rlock(dep);
-	if (dep->flags & DFLAG_DIST_HDR_ATOM_CACHE)
-	    edep->flags |= ERTS_DIST_EXT_DFLAG_HDR;
-	    
-	edep->flags |= (dep->connection_id & ERTS_DIST_EXT_CON_ID_MASK);
-	erts_smp_de_runlock(dep);
+
+    erts_smp_de_rlock(dep);
+
+    if ((dep->status & (ERTS_DE_SFLG_EXITING|ERTS_DE_SFLG_CONNECTED))
+        != ERTS_DE_SFLG_CONNECTED) {
+        erts_smp_de_runlock(dep);
+        return ERTS_PREP_DIST_EXT_CLOSED;
     }
 
+    if (dep->flags & DFLAG_DIST_HDR_ATOM_CACHE)
+        edep->flags |= ERTS_DIST_EXT_DFLAG_HDR;
+
+    *connection_id = dep->connection_id;
+    edep->flags |= (dep->connection_id & ERTS_DIST_EXT_CON_ID_MASK);
+
     if (ep[1] != DIST_HEADER) {
 	if (edep->flags & ERTS_DIST_EXT_DFLAG_HDR)
 	    ERTS_EXT_HDR_FAIL;
@@ -835,14 +839,15 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
 	ERTS_EXT_FAIL;
 #endif
 
-    return 0;
+    erts_smp_de_runlock(dep);
+
+    return ERTS_PREP_DIST_EXT_SUCCESS;
 
 #undef CHKSIZE
 #undef ERTS_EXT_FAIL
 #undef ERTS_EXT_HDR_FAIL
 
- bad_hdr:
-    if (dep) {
+ bad_hdr: {
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
 	erts_dsprintf(dsbufp,
 		      "%T got a corrupted distribution header from %T "
@@ -855,10 +860,11 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
 	erts_dsprintf(dsbufp, ">>");
 	erts_send_warning_to_logger_nogl(dsbufp);
     }
- fail:
-    if (dep)
-	erts_kill_dist_connection(dep, dep->connection_id);
-    return -1;
+ fail: {
+	erts_smp_de_runlock(dep);
+	erts_kill_dist_connection(dep, *connection_id);
+    }
+    return ERTS_PREP_DIST_EXT_FAILED;
 }
 
 static void
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index f00426cc16..3c61d013da 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -185,8 +185,13 @@ ERTS_GLB_INLINE void *erts_dist_ext_trailer(ErtsDistExternal *);
 ErtsDistExternal *erts_make_dist_ext_copy(ErtsDistExternal *, Uint);
 void *erts_dist_ext_trailer(ErtsDistExternal *);
 void erts_destroy_dist_ext_copy(ErtsDistExternal *);
+
+#define ERTS_PREP_DIST_EXT_FAILED       (-1)
+#define ERTS_PREP_DIST_EXT_SUCCESS      (0)
+#define ERTS_PREP_DIST_EXT_CLOSED       (1)
+
 int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint,
-			  DistEntry *, ErtsAtomCache *);
+			  DistEntry *, ErtsAtomCache *, Uint32 *);
 Sint erts_decode_dist_ext_size(ErtsDistExternal *);
 Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *);
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 75545df80a..8c59d65332 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -6727,6 +6727,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
     else
 	erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len));
     if (state & ERTS_PORT_SFLG_DISTRIBUTION) {
+        erts_smp_atomic64_inc_nob(&prt->dist_entry->in);
 	return erts_net_message(prt,
 				prt->dist_entry,
 				(byte*) hbuf, hlen,
@@ -6767,6 +6768,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
     else
 	erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len));
     if (state & ERTS_PORT_SFLG_DISTRIBUTION) {
+        erts_smp_atomic64_inc_nob(&prt->dist_entry->in);
 	if (len == 0)
 	    return erts_net_message(prt,
 				    prt->dist_entry,
-- 
cgit v1.2.3


From ca4b42a159c5cc937967c2d04818afa6b8022e0b Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 30 Jun 2017 19:10:31 +0200
Subject: Support for running test suites with gen_tcp_dist

---
 erts/emulator/test/distribution_SUITE.erl | 98 ++++++++++++++++---------------
 erts/emulator/test/erl_link_SUITE.erl     | 85 ++++++++++++++-------------
 2 files changed, 96 insertions(+), 87 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index b4ec99f902..28be4bfe37 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -418,18 +418,20 @@ make_busy(Node, Time) when is_integer(Time) ->
     Own = 500,
     freeze_node(Node, Time+Own),
     Data = make_busy_data(),
+    DCtrl = dctrl(Node),
     %% first make port busy
     Pid = spawn_link(fun () ->
                              forever(fun () ->
-                                             dport_reg_send(Node,
-                                                            '__noone__',
-                                                            Data)
+                                             dctrl_dop_reg_send(Node,
+                                                                '__noone__',
+                                                                Data)
                                      end)
                      end),
     receive after Own -> ok end,
     until(fun () ->
-                  case process_info(Pid, status) of
-                      {status, suspended} -> true;
+                  case {DCtrl, process_info(Pid, status)} of
+                      {DPrt, {status, suspended}} when is_port(DPrt) -> true;
+                      {DPid, {status, waiting}} when is_pid(DPid) -> true;
                       _ -> false
                   end
           end),
@@ -1703,37 +1705,38 @@ bad_dist_ext_check_msgs([M|Ms]) ->
             bad_dist_ext_check_msgs(Ms)
     end.
 
+ensure_dctrl(Node) ->
+    case dctrl(Node) of
+        undefined ->
+            pong = net_adm:ping(Node),
+            dctrl(Node);
+        DCtrl ->
+            DCtrl
+    end.
 
-dport_reg_send(Node, Name, Msg) ->
-    DPrt = case dport(Node) of
-               undefined ->
-                   pong = net_adm:ping(Node),
-                   dport(Node);
-               Prt ->
-                   Prt
-           end,
-    port_command(DPrt, [dmsg_hdr(),
-                        dmsg_ext({?DOP_REG_SEND,
-                                  self(),
-                                  ?COOKIE,
-                                  Name}),
-                        dmsg_ext(Msg)]).
-
-
-dport_send(To, Msg) ->
+dctrl_send(DPrt, Data) when is_port(DPrt) ->
+    port_command(DPrt, Data);
+dctrl_send(DPid, Data) when is_pid(DPid) ->
+    Ref = make_ref(),
+    DPid ! {send, self(), Ref, Data},
+    receive {Ref, Res} -> Res end.
+
+dctrl_dop_reg_send(Node, Name, Msg) ->
+    dctrl_send(ensure_dctrl(Node),
+               [dmsg_hdr(),
+                dmsg_ext({?DOP_REG_SEND,
+                          self(),
+                          ?COOKIE,
+                          Name}),
+                dmsg_ext(Msg)]).
+
+dctrl_dop_send(To, Msg) ->
     Node = node(To),
-    DPrt = case dport(Node) of
-               undefined ->
-                   pong = net_adm:ping(Node),
-                   dport(Node);
-               Prt ->
-                   Prt
-           end,
-    port_command(DPrt, [dmsg_hdr(),
-                        dmsg_ext({?DOP_SEND,
-                                  ?COOKIE,
-                                  To}),
-                        dmsg_ext(Msg)]).
+    dctrl_send(ensure_dctrl(Node),
+               [dmsg_hdr(),
+                dmsg_ext({?DOP_SEND, ?COOKIE, To}),
+                dmsg_ext(Msg)]).
+
 send_bad_structure(Offender,Victim,Bad,WhereToPutSelf) ->
     send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,[]).
 send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) ->
@@ -1743,7 +1746,7 @@ send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) ->
           fun () ->
                   Node = node(Victim),
                   pong = net_adm:ping(Node),
-                  DPrt = dport(Node),
+                  DCtrl = dctrl(Node),
                   Bad1 = case WhereToPutSelf of
                              0 ->
                                  Bad;
@@ -1756,7 +1759,7 @@ send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) ->
                       [] -> [];
                       _Other -> [dmsg_ext(PayLoad)]
                   end,
-                  port_command(DPrt, DData),
+                  dctrl_send(DCtrl, DData),
                   Parent ! {DData,Done}
           end),
     receive
@@ -1784,11 +1787,11 @@ send_bad_msgs(BadNode, To, Repeat) when is_atom(BadNode),
                fun () ->
                        Node = node(To),
                        pong = net_adm:ping(Node),
-                       DPrt = dport(Node),
+                       DCtrl = dctrl(Node),
                        DData = [dmsg_hdr(),
                                 dmsg_ext({?DOP_SEND, ?COOKIE, To}),
                                 dmsg_bad_atom_cache_ref()],
-                       repeat(fun () -> port_command(DPrt, DData) end, Repeat),
+		       repeat(fun () -> dctrl_send(DCtrl, DData) end, Repeat),
                        Parent ! Done
                end),
     receive Done -> ok end.
@@ -1810,11 +1813,12 @@ send_bad_ctl(BadNode, ToNode) when is_atom(BadNode), is_atom(ToNode) ->
                                        replace}),
                        CtlBeginSize = size(Ctl) - size(Replace),
                        <<CtlBegin:CtlBeginSize/binary, Replace/binary>> = Ctl,
-                       port_command(dport(ToNode),
-                                    [dmsg_fake_hdr2(),
-                                     CtlBegin,
-                                     dmsg_bad_atom_cache_ref(),
-                                     dmsg_ext({a, message})]),
+                       DCtrl = dctrl(ToNode),
+                       Data = [dmsg_fake_hdr2(),
+                               CtlBegin,
+                               dmsg_bad_atom_cache_ref(),
+                               dmsg_ext({a, message})],
+                       dctrl_send(DCtrl, Data),
                        Parent ! Done
                end),
     receive Done -> ok end.
@@ -1827,17 +1831,17 @@ send_bad_dhdr(BadNode, ToNode) when is_atom(BadNode), is_atom(ToNode) ->
     spawn_link(BadNode,
                fun () ->
                        pong = net_adm:ping(ToNode),
-                       port_command(dport(ToNode), dmsg_bad_hdr()),
+                       dctrl_send(dctrl(ToNode), dmsg_bad_hdr()),
                        Parent ! Done
                end),
     receive Done -> ok end.
 
-dport(Node) when is_atom(Node) ->
+dctrl(Node) when is_atom(Node) ->
     case catch erts_debug:get_internal_state(available_internal_state) of
         true -> true;
         _ -> erts_debug:set_internal_state(available_internal_state, true)
     end,
-    erts_debug:get_internal_state({dist_port, Node}).
+    erts_debug:get_internal_state({dist_ctrl, Node}).
 
 dmsg_hdr() ->
     [131, % Version Magic
@@ -1979,7 +1983,7 @@ freeze_node(Node, MS) ->
                fun () ->
                        erts_debug:set_internal_state(available_internal_state,
                                                      true),
-                       dport_send(Freezer, DoingIt),
+                       dctrl_dop_send(Freezer, DoingIt),
                        receive after Own -> ok end,
                        erts_debug:set_internal_state(block, MS+Own)
                end),
diff --git a/erts/emulator/test/erl_link_SUITE.erl b/erts/emulator/test/erl_link_SUITE.erl
index 5622cce980..d8c5b663e3 100644
--- a/erts/emulator/test/erl_link_SUITE.erl
+++ b/erts/emulator/test/erl_link_SUITE.erl
@@ -533,7 +533,7 @@ freeze_node(Node, MS) ->
                fun () ->
                        erts_debug:set_internal_state(available_internal_state,
                                                      true),
-                       dport_send(Freezer, DoingIt),
+                       dctrl_dop_send(Freezer, DoingIt),
                        receive after Own -> ok end,
                        erts_debug:set_internal_state(block, MS+Own)
                end),
@@ -544,20 +544,22 @@ make_busy(Node, Time) when is_integer(Time) ->
     Own = 500,
     freeze_node(Node, Time+Own), 
     Data = busy_data(),
+    DCtrl = dctrl(Node),
     %% first make port busy
     Pid = spawn_link(fun () ->
                              forever(fun () ->
-                                             dport_reg_send(Node,
-                                                            '__noone__',
-                                                            Data)
+                                             dctrl_dop_reg_send(Node,
+                                                                '__noone__',
+                                                                Data)
                                      end)
                      end),
     receive after Own -> ok end,
     wait_until(fun () ->
-                       case process_info(Pid, status) of
-                           {status, suspended} -> true;
-                           _ -> false
-                       end
+                  case {DCtrl, process_info(Pid, status)} of
+                      {DPrt, {status, suspended}} when is_port(DPrt) -> true;
+                      {DPid, {status, waiting}} when is_pid(DPid) -> true;
+                      _ -> false
+                  end
                end),
     %% then dist entry
     make_busy(Node, [nosuspend], Data),
@@ -1048,42 +1050,45 @@ stop_node(Node) ->
 -define(DOP_DEMONITOR_P,	20).
 -define(DOP_MONITOR_P_EXIT,	21).
 
-dport_send(To, Msg) ->
-    Node = node(To),
-    DPrt = case dport(Node) of
-               undefined ->
-                   pong = net_adm:ping(Node),
-                   dport(Node);
-               Prt ->
-                   Prt
-           end,
-    port_command(DPrt, [dmsg_hdr(),
-                        dmsg_ext({?DOP_SEND,
-                                  ?COOKIE,
-                                  To}),
-                        dmsg_ext(Msg)]).
-
-dport_reg_send(Node, Name, Msg) ->
-    DPrt = case dport(Node) of
-               undefined ->
-                   pong = net_adm:ping(Node),
-                   dport(Node);
-               Prt ->
-                   Prt
-           end,
-    port_command(DPrt, [dmsg_hdr(),
-                        dmsg_ext({?DOP_REG_SEND,
-                                  self(),
-                                  ?COOKIE,
-                                  Name}),
-                        dmsg_ext(Msg)]).
-
-dport(Node) when is_atom(Node) ->
+ensure_dctrl(Node) ->
+    case dctrl(Node) of
+        undefined ->
+            pong = net_adm:ping(Node),
+            dctrl(Node);
+        DCtrl ->
+            DCtrl
+    end.
+
+dctrl_send(DPrt, Data) when is_port(DPrt) ->
+    port_command(DPrt, Data);
+dctrl_send(DPid, Data) when is_pid(DPid) ->
+    Ref = make_ref(),
+    DPid ! {send, self(), Ref, Data},
+    receive {Ref, Res} -> Res end.
+
+dctrl_dop_send(To, Msg) ->
+    dctrl_send(ensure_dctrl(node(To)),
+               [dmsg_hdr(),
+                dmsg_ext({?DOP_SEND,
+                          ?COOKIE,
+                          To}),
+                dmsg_ext(Msg)]).
+
+dctrl_dop_reg_send(Node, Name, Msg) ->
+    dctrl_send(ensure_dctrl(Node),
+               [dmsg_hdr(),
+                dmsg_ext({?DOP_REG_SEND,
+                          self(),
+                          ?COOKIE,
+                          Name}),
+                dmsg_ext(Msg)]).
+
+dctrl(Node) when is_atom(Node) ->
     case catch erts_debug:get_internal_state(available_internal_state) of
         true -> true;
         _ -> erts_debug:set_internal_state(available_internal_state, true)
     end,
-    erts_debug:get_internal_state({dist_port, Node}).
+    erts_debug:get_internal_state({dist_ctrl, Node}).
 
 dmsg_hdr() ->
     [131, % Version Magic
-- 
cgit v1.2.3


From 4dcb2ae7810a507b701a30072b2f514cab7ebbdb Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 10 Jul 2017 18:01:42 +0200
Subject: Optimize dist entry management

---
 erts/emulator/beam/bif.c                    |  41 +---
 erts/emulator/beam/dist.c                   |  32 +--
 erts/emulator/beam/erl_bif_info.c           |   5 -
 erts/emulator/beam/erl_monitors.c           |   2 -
 erts/emulator/beam/erl_node_tables.c        | 316 ++++++++++++++++++++--------
 erts/emulator/beam/erl_node_tables.h        |  20 +-
 erts/emulator/beam/erl_process.c            |  23 +-
 erts/emulator/beam/erl_process.h            |   5 +
 erts/emulator/beam/external.c               |   2 +-
 erts/emulator/test/node_container_SUITE.erl |   1 +
 10 files changed, 284 insertions(+), 163 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 40dd4129d2..b1f915b133 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -475,7 +475,6 @@ BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip)
    ErtsMonitor  *mon = NULL;  /* The monitor entry to delete */
    Eterm        to = NIL;     /* Monitor link traget */
    DistEntry    *dep = NULL;  /* Target's distribution entry */
-   int          deref_de = 0;
    BIF_RETTYPE  res = am_false;
    int          unlock_link = 1;
 
@@ -505,8 +504,6 @@ BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip)
            ASSERT(is_node_name_atom(to));
            dep = erts_sysname_to_connected_dist_entry(to);
            ASSERT(dep != erts_this_dist_entry);
-           if (dep)
-               deref_de = 1;
        } else if (is_port(to)) {
            if (port_dist_entry(to) != erts_this_dist_entry) {
                goto badarg;
@@ -524,11 +521,6 @@ BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip)
            unlock_link = 0;
        }
        else { /* Local monitor */
-           if (deref_de) {
-               deref_de = 0;
-               erts_deref_dist_entry(dep);
-           }
-           dep = NULL;
            demonitor_local_process(c_p, ref, to, &res);
        }
        break;
@@ -543,11 +535,6 @@ done:
    if (unlock_link)
        erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_LINK);
 
-   if (deref_de) {
-       ASSERT(dep);
-       erts_deref_dist_entry(dep);
-   }
-
    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
    BIF_RET(res);
 }
@@ -882,7 +869,6 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2)
     Eterm target = BIF_ARG_2;
     BIF_RETTYPE ret;
     DistEntry  *dep = NULL; 
-    int deref_de = 0;
 
     /* Only process monitors are implemented */
     switch (BIF_ARG_1) {
@@ -942,21 +928,14 @@ local_port:
 	}
 	dep = erts_sysname_to_connected_dist_entry(remote_node);
 	if (dep == erts_this_dist_entry) {
-	    deref_de = 1;
             ret = local_name_monitor(BIF_P, BIF_ARG_1, name);
 	} else {
-	    if (dep)
-		deref_de = 1;
 	    ret = remote_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2, dep, name, 1);
 	}
     } else {
 badarg:
 	ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
     }
-    if (deref_de) {
-	deref_de = 0;
-	erts_deref_dist_entry(dep);
-    }
 
     return ret;
 }
@@ -2257,7 +2236,6 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx)
 
 	if (dep == erts_this_dist_entry) {
 	    Eterm id;
-	    erts_deref_dist_entry(dep);
 	    if (IS_TRACED_FL(p, F_TRACE_SEND))
 		trace_send(p, to, msg);
 	    if (ERTS_PROC_GET_SAVED_CALLS_BUF(p))
@@ -2280,11 +2258,9 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx)
 	}
 
 	ret = remote_send(p, dep, tp[1], to, msg, ctx);
-	if (ret != SEND_YIELD_CONTINUE) {
-	    if (dep) {
-		erts_deref_dist_entry(dep);
-	    }
-	} else {
+	if (ret == SEND_YIELD_CONTINUE) {
+            if (dep)
+                erts_ref_dist_entry(dep);
 	    ctx->dep_to_deref = dep;
 	}
 	return ret;
@@ -4220,7 +4196,6 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1)
 	goto bad;
 
     if(dep == erts_this_dist_entry) {
-	erts_deref_dist_entry(dep);
 	BIF_RET(make_internal_pid(make_pid_data(c, b)));
     }
     else {
@@ -4240,13 +4215,10 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1)
       etp->data.ui[0] = make_pid_data(c, b);
 
       MSO(BIF_P).first = (struct erl_off_heap_header*) etp;
-      erts_deref_dist_entry(dep);
       BIF_RET(make_external_pid(etp));
     }
 
  bad:
-    if (dep)
-	erts_deref_dist_entry(dep);
     if (buf)
 	erts_free(ERTS_ALC_T_TMP, (void *) buf);
     BIF_ERROR(BIF_P, BADARG);
@@ -4291,7 +4263,6 @@ BIF_RETTYPE list_to_port_1(BIF_ALIST_1)
 	goto bad;
 
     if(dep == erts_this_dist_entry) {
-	erts_deref_dist_entry(dep);
 	BIF_RET(make_internal_port(p));
     }
     else {
@@ -4311,13 +4282,10 @@ BIF_RETTYPE list_to_port_1(BIF_ALIST_1)
       etp->data.ui[0] = p;
 
       MSO(BIF_P).first = (struct erl_off_heap_header*) etp;
-      erts_deref_dist_entry(dep);
       BIF_RET(make_external_port(etp));
     }
 
  bad:
-    if (dep)
-	erts_deref_dist_entry(dep);
     BIF_ERROR(BIF_P, BADARG);
 }
 
@@ -4437,12 +4405,9 @@ BIF_RETTYPE list_to_ref_1(BIF_ALIST_1)
       res = make_external_ref(etp);
     }
 
-    erts_deref_dist_entry(dep);
     BIF_RET(res);
 
  bad:
-    if (dep)
-	erts_deref_dist_entry(dep);
     BIF_ERROR(BIF_P, BADARG);
 }
 
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 5d7501f234..d5c39485d0 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -2200,9 +2200,6 @@ erts_dist_command(Port *prt, int reds_limit)
 
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 
-    erts_smp_refc_inc(&dep->refc, 1); /* Otherwise dist_entry might be
-				     removed if port command fails */
-
     erts_smp_atomic_set_mb(&dep->dist_cmd_scheduled, 0);
 
     erts_smp_de_rlock(dep);
@@ -2213,7 +2210,6 @@ erts_dist_command(Port *prt, int reds_limit)
 
     if (status & ERTS_DE_SFLG_EXITING) {
 	erts_deliver_port_exit(prt, prt->common.id, am_killed, 0, 1);
-	erts_deref_dist_entry(dep);
 	return reds + ERTS_PORT_REDS_DIST_CMD_EXIT;
     }
 
@@ -2426,8 +2422,6 @@ erts_dist_command(Port *prt, int reds_limit)
     if (reds > INT_MAX/2)
 	reds = INT_MAX/2;
 
-    erts_deref_dist_entry(dep);
-
     return reds;
 
  preempted:
@@ -2594,7 +2588,7 @@ dist_ctrl_get_data_notification_1(BIF_ALIST_1)
     if (!dep)
         BIF_ERROR(BIF_P, EXC_NOTSUP);
 
-    if (dep->sysname != BIF_ARG_1)
+    if (erts_dhandle_to_dist_entry(BIF_ARG_1) != dep)
         BIF_ERROR(BIF_P, BADARG);
 
     /*
@@ -2655,7 +2649,7 @@ dist_ctrl_put_data_2(BIF_ALIST_2)
     else
         BIF_ERROR(BIF_P, BADARG);
 
-    dep = erts_find_dist_entry(BIF_ARG_1);
+    dep = erts_dhandle_to_dist_entry(BIF_ARG_1);
     if (!dep)
         BIF_ERROR(BIF_P, BADARG);
 
@@ -2699,13 +2693,11 @@ dist_get_stat_1(BIF_ALIST_1)
     Sint64 read, write, pend;
     Eterm res, *hp, **hpp;
     Uint sz, *szp;
-    DistEntry *dep = erts_find_dist_entry(BIF_ARG_1);
+    DistEntry *dep = erts_dhandle_to_dist_entry(BIF_ARG_1);
 
     if (!dep)
         BIF_ERROR(BIF_P, BADARG);
 
-    ASSERT(dep->sysname == BIF_ARG_1);
-
     erts_smp_de_rlock(dep);
 
     read = (Sint64) erts_smp_atomic64_read_nob(&dep->in);
@@ -2714,8 +2706,6 @@ dist_get_stat_1(BIF_ALIST_1)
 
     erts_smp_de_runlock(dep);
 
-    erts_deref_dist_entry(dep);
-
     sz = 0;
     szp = &sz;
     hpp = NULL;
@@ -2744,7 +2734,7 @@ dist_ctrl_input_handler_2(BIF_ALIST_2)
     if (!dep)
         BIF_ERROR(BIF_P, EXC_NOTSUP);
 
-    if (dep->sysname != BIF_ARG_1)
+    if (erts_dhandle_to_dist_entry(BIF_ARG_1) != dep)
         BIF_ERROR(BIF_P, BADARG);
 
     if (is_not_internal_pid(BIF_ARG_2))
@@ -2769,7 +2759,7 @@ dist_ctrl_get_data_1(BIF_ALIST_1)
     if (!dep)
         BIF_ERROR(BIF_P, EXC_NOTSUP);
 
-    if (dep->sysname != BIF_ARG_1)
+    if (erts_dhandle_to_dist_entry(BIF_ARG_1) != dep)
         BIF_ERROR(BIF_P, BADARG);
 
     erts_smp_de_rlock(dep);
@@ -3010,9 +3000,6 @@ info_dist_entry(fmtfn_t to, void *arg, DistEntry *dep, int visible, int connecte
   }
 
   erts_print(to, arg, "Name: %T", dep->sysname);
-#ifdef DEBUG
-  erts_print(to, arg, " (refc=%d)", erts_smp_refc_read(&dep->refc, 0));
-#endif
   erts_print(to, arg, "\n");
   if (!connected && is_nil(dep->cid)) {
     if (dep->nlinks) {
@@ -3170,7 +3157,7 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2)
      * so we *need* to use the new one after erts_set_this_node()
      * is called.
      */
-    erts_smp_refc_inc(&erts_this_dist_entry->refc, 1);
+    erts_ref_dist_entry(erts_this_dist_entry);
     ERTS_PROC_SET_DIST_ENTRY(net_kernel, erts_this_dist_entry);
 
     BIF_RET(am_true);
@@ -3206,9 +3193,6 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     Process *proc;
     Port *pp = NULL;
 
-    /* Prepare for success */
-    ERTS_BIF_PREP_RET(ret, BIF_ARG_1);
-
     /*
      * Check and pick out arguments
      */
@@ -3371,6 +3355,9 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     erts_set_dist_entry_connected(dep, BIF_ARG_2, flags);
 
     erts_smp_de_rwunlock(dep);
+
+    ERTS_BIF_PREP_RET(ret, erts_make_dhandle(BIF_P, dep));
+
     dep = NULL; /* inc of refc transferred to port (dist_entry field) */
 
     inc_no_nodes();
@@ -3699,7 +3686,6 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options)
     erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 
  done:
-    erts_deref_dist_entry(dep);
     BIF_RET(am_true);
 }
 
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 8846866eb7..f07c84a8df 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3830,7 +3830,6 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 			subres = make_link_list(BIF_P, dep->nlinks, NIL);
 			subres = make_link_list(BIF_P, dep->node_links, subres);
 			erts_smp_de_links_unlock(dep);
-			erts_deref_dist_entry(dep);
 			BIF_RET(subres);
 		    } else {
 			BIF_RET(am_undefined);
@@ -3861,7 +3860,6 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 			erts_smp_de_links_lock(dep);
 			ml = make_monitor_list(BIF_P, dep->monitors);
 			erts_smp_de_links_unlock(dep);
-			erts_deref_dist_entry(dep);
 			BIF_RET(ml);
 		    } else {
 			BIF_RET(am_undefined);
@@ -3876,7 +3874,6 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 		else {
 		    Uint cno = dist_entry_channel_no(dep);
 		    res = make_small(cno);
-		    erts_deref_dist_entry(dep);
 		}
 		BIF_RET(res);
 	    }
@@ -3946,7 +3943,6 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 		    if (is_internal_port(dep->cid) || is_internal_pid(dep->cid))
 			res = dep->cid;
 		    erts_smp_de_runlock(dep);
-		    erts_deref_dist_entry(dep);
 		}
 		BIF_RET(res);
 	    }
@@ -4359,7 +4355,6 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 		con_id = dep->connection_id;
 		erts_smp_de_runlock(dep);
 		erts_kill_dist_connection(dep, con_id);
-		erts_deref_dist_entry(dep);
 		BIF_RET(am_true);
 	    }
 	}
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
index 3994800ba7..366e226a9e 100644
--- a/erts/emulator/beam/erl_monitors.c
+++ b/erts/emulator/beam/erl_monitors.c
@@ -993,7 +993,6 @@ Eterm erts_debug_dump_monitors_1(BIF_ALIST_1)
 	    erts_dump_monitors(dep->monitors,0);
 	    erts_smp_de_links_unlock(dep);
 	    erts_printf("Monitors dumped-------------------------\n");
-	    erts_deref_dist_entry(dep);
 	    BIF_RET(am_true);
 	} else {
 	    BIF_ERROR(p,BADARG);
@@ -1038,7 +1037,6 @@ Eterm erts_debug_dump_links_1(BIF_ALIST_1)
 		erts_dump_links(dep->nlinks,0);
 		erts_smp_de_links_unlock(dep);
 		erts_printf("Links dumped----------------------------\n");
-		erts_deref_dist_entry(dep);
 		BIF_RET(am_true);
 	    } else {
 		BIF_ERROR(p,BADARG);
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 4ef1d54906..dd1b6b2cc8 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -29,6 +29,8 @@
 #include "error.h"
 #include "erl_thr_progress.h"
 #include "dtrace-wrapper.h"
+#include "erl_binary.h"
+#include "erl_bif_unique.h"
 
 Hash erts_dist_table;
 Hash erts_node_table;
@@ -57,6 +59,58 @@ static ErtsMonotonicTime node_tab_delete_delay;
 
 /* -- The distribution table ---------------------------------------------- */
 
+#define ErtsBin2DistEntry(B) \
+    ((DistEntry *) ERTS_MAGIC_BIN_DATA((B)))
+#define ErtsDistEntry2Bin(DEP) \
+    ((Binary *) ERTS_MAGIC_BIN_FROM_DATA((DEP)))
+
+static ERTS_INLINE erts_aint_t
+de_refc_read(DistEntry *dep, erts_aint_t min)
+{
+    return erts_refc_read(&ErtsDistEntry2Bin(dep)->intern.refc, min);
+}
+
+static ERTS_INLINE erts_aint_t
+de_refc_inc_read(DistEntry *dep, erts_aint_t min)
+{
+    return erts_refc_inctest(&ErtsDistEntry2Bin(dep)->intern.refc, min);
+}
+
+static ERTS_INLINE void
+de_refc_inc(DistEntry *dep, erts_aint_t min)
+{
+    erts_refc_inc(&ErtsDistEntry2Bin(dep)->intern.refc, min);
+}
+
+static ERTS_INLINE void
+de_refc_dec(DistEntry *dep, erts_aint_t min)
+{
+#ifdef DEBUG
+    (void) erts_refc_read(&ErtsDistEntry2Bin(dep)->intern.refc, min+1);
+#endif
+    erts_bin_release(ErtsDistEntry2Bin(dep));
+}
+
+static ERTS_INLINE erts_aint_t
+de_refc_dec_read(DistEntry *dep, erts_aint_t min)
+{
+    return erts_refc_dectest(&ErtsDistEntry2Bin(dep)->intern.refc, min);
+}
+
+void
+erts_ref_dist_entry(DistEntry *dep)
+{
+    ASSERT(dep);
+    de_refc_inc(dep, 1);
+}
+
+void
+erts_deref_dist_entry(DistEntry *dep)
+{
+    ASSERT(dep);
+    de_refc_dec(dep, 0);
+}
+
 #ifdef DEBUG
 static int
 is_in_de_list(DistEntry *dep, DistEntry *dep_list)
@@ -85,20 +139,36 @@ dist_table_cmp(void *dep1, void *dep2)
 static void*
 dist_table_alloc(void *dep_tmpl)
 {
+#ifdef DEBUG
+    erts_aint_t refc;
+#endif
     Eterm chnl_nr;
     Eterm sysname;
+    Binary *bin;
     DistEntry *dep;
     erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
 
     sysname = ((DistEntry *) dep_tmpl)->sysname;
     chnl_nr = make_small((Uint) atom_val(sysname));
-    dep = (DistEntry *) erts_alloc(ERTS_ALC_T_DIST_ENTRY, sizeof(DistEntry));
+
+    bin = erts_create_magic_binary_x(sizeof(DistEntry),
+                                     erts_dist_entry_destructor,
+                                     ERTS_ALC_T_DIST_ENTRY,
+                                     0);
+    dep = ErtsBin2DistEntry(bin);
 
     dist_entries++;
 
+#ifdef DEBUG
+    refc =
+#else
+    (void)
+#endif
+        de_refc_dec_read(dep, -1);
+    ASSERT(refc == -1);
+
     dep->prev				= NULL;
-    erts_smp_refc_init(&dep->refc, -1);
     erts_smp_rwmtx_init_opt_x(&dep->rwmtx, &rwmtx_opt, "dist_entry", chnl_nr);
     dep->sysname			= sysname;
     dep->cid				= NIL;
@@ -185,7 +255,7 @@ dist_table_free(void *vdep)
 #ifdef DEBUG
     sys_memset(vdep, 0x77, sizeof(DistEntry));
 #endif
-    erts_free(ERTS_ALC_T_DIST_ENTRY, (void *) dep);
+    erts_bin_free(ErtsDistEntry2Bin(dep));
 
     ASSERT(dist_entries > 0);
     dist_entries--;
@@ -203,19 +273,52 @@ erts_dist_table_info(fmtfn_t to, void *to_arg)
 	erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
 }
 
+static ERTS_INLINE DistEntry *find_dist_entry(Eterm sysname,
+                                              int inc_refc,
+                                              int connected_only)
+{
+    DistEntry *res;
+    DistEntry de;
+    de.sysname = sysname;
+    erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
+    res = hash_get(&erts_dist_table, (void *) &de);
+    if (res) {
+        if (connected_only && is_nil(res->cid))
+            res = NULL;
+        else {
+            int pend_delete;
+            erts_aint_t refc;
+            if (inc_refc) {
+                refc = de_refc_inc_read(res, 1);
+                pend_delete = refc < 2;
+            }
+            else {
+                refc = de_refc_read(res, 0);
+                pend_delete = refc < 1;
+            }
+            if (pend_delete) /* Pending delete */
+                de_refc_inc(res, 1);
+        }
+    }
+    erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
+    return res;
+}
+
 DistEntry *
 erts_channel_no_to_dist_entry(Uint cno)
 {
+    /*
+     * Does NOT increase reference count!
+     */
+
 /*
  * For this node (and previous incarnations of this node),
  * ERST_INTERNAL_CHANNEL_NO (will always be 0 I guess) is used as
  * channel no. For other nodes, the atom index of the atom corresponding
  * to the node name is used as channel no.
  */
-    if(cno == ERST_INTERNAL_CHANNEL_NO) {
-	erts_smp_refc_inc(&erts_this_dist_entry->refc, 2);
+    if (cno == ERST_INTERNAL_CHANNEL_NO)
 	return erts_this_dist_entry;
-    }
 
     if((cno > MAX_ATOM_INDEX)
        || (cno >= atom_table_size())
@@ -224,80 +327,97 @@ erts_channel_no_to_dist_entry(Uint cno)
         
     /* cno is a valid atom index; find corresponding dist entry (if there
        is one) */
-    return erts_find_dist_entry(make_atom(cno));
+    return find_dist_entry(make_atom(cno), 0, 0);
 }
 
-
 DistEntry *
 erts_sysname_to_connected_dist_entry(Eterm sysname)
 {
-    DistEntry de;
-    DistEntry *res_dep;
-    de.sysname = sysname;
-  
-    if(erts_this_dist_entry->sysname == sysname) {
-	erts_smp_refc_inc(&erts_this_dist_entry->refc, 2);
+    /*
+     * Does NOT increase reference count!
+     */
+    if(erts_this_dist_entry->sysname == sysname)
 	return erts_this_dist_entry;
-    }
-
-    erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
-    res_dep = (DistEntry *) hash_get(&erts_dist_table, (void *) &de);
-    if (res_dep) {
-	erts_aint_t refc = erts_smp_refc_inctest(&res_dep->refc, 1);
-	if (refc < 2) /* Pending delete */
-	    erts_smp_refc_inc(&res_dep->refc, 1);
-    }
-    erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
-    if (res_dep) {
-	int deref;
-	erts_smp_rwmtx_rlock(&res_dep->rwmtx);
-	deref = is_nil(res_dep->cid);
-	erts_smp_rwmtx_runlock(&res_dep->rwmtx);
-	if (deref) {
-	    erts_deref_dist_entry(res_dep);
-	    res_dep = NULL;
-	}
-    }
-    return res_dep;
+    return find_dist_entry(sysname, 0, 1);
 }
 
 DistEntry *erts_find_or_insert_dist_entry(Eterm sysname)
 {
+    /*
+     * This function DOES increase reference count!
+     */
     DistEntry *res;
     DistEntry de;
     erts_aint_t refc;
-    res = erts_find_dist_entry(sysname);
+    res = find_dist_entry(sysname, 1, 0);
     if (res)
 	return res;
     de.sysname = sysname;
     erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx);
     res = hash_put(&erts_dist_table, (void *) &de);
-    refc = erts_smp_refc_inctest(&res->refc, 0);
+    refc = de_refc_inc_read(res, 0);
     if (refc < 2) /* New or pending delete */
-	erts_smp_refc_inc(&res->refc, 1);
+        de_refc_inc(res, 1);
     erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
     return res;
 }
 
 DistEntry *erts_find_dist_entry(Eterm sysname)
 {
-    DistEntry *res;
-    DistEntry de;
-    de.sysname = sysname;
-    erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
-    res = hash_get(&erts_dist_table, (void *) &de);
-    if (res) {
-	erts_aint_t refc = erts_smp_refc_inctest(&res->refc, 1);
-	if (refc < 2) /* Pending delete */
-	    erts_smp_refc_inc(&res->refc, 1);
-    }
-    erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx);
-    return res;
+    /*
+     * Does NOT increase reference count!
+     */
+    return find_dist_entry(sysname, 0, 0);
 }
 
-static void try_delete_dist_entry(void *vdep)
+DistEntry *
+erts_dhandle_to_dist_entry(Eterm dhandle)
 {
-    DistEntry *dep = (DistEntry *) vdep;
+    Binary *bin;
+    if (!is_internal_magic_ref(dhandle))
+        return NULL;
+    bin = erts_magic_ref2bin(dhandle);
+    if (ERTS_MAGIC_BIN_DESTRUCTOR(bin) != erts_dist_entry_destructor)
+        return NULL;
+    return ErtsBin2DistEntry(bin);
+}
+
+Eterm
+erts_make_dhandle(Process *c_p, DistEntry *dep)
+{
+    Binary *bin;
+    Eterm *hp;
+
+    bin = ErtsDistEntry2Bin(dep);
+    ASSERT(bin);
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == erts_dist_entry_destructor);
+    hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE);
+    return erts_mk_magic_ref(&hp, &c_p->off_heap, bin);
+}
+
+static void try_delete_dist_entry(void *vbin);
+
+static void
+prepare_try_delete_dist_entry(void *vbin)
+{
+    Binary *bin = (Binary *) vbin;
+    DistEntry *dep = ErtsBin2DistEntry(bin);
+    Uint size;
+    erts_aint_t refc;
+
+    refc = de_refc_read(dep, 0);
+    if (refc > 0)
+        return;
+
+    size = ERTS_MAGIC_BIN_SIZE(sizeof(DistEntry));
+    erts_schedule_thr_prgr_later_cleanup_op(try_delete_dist_entry,
+                                            vbin, &dep->later_op, size);
+}
+
+static void try_delete_dist_entry(void *vbin)
+{
+    Binary *bin = (Binary *) vbin;
+    DistEntry *dep = ErtsBin2DistEntry(bin);
     erts_aint_t refc;
 
     erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx);
@@ -316,26 +436,39 @@ static void try_delete_dist_entry(void *vdep)
      *
      * If refc > 0, the entry is in use. Keep the entry.
      */
-    refc = erts_smp_refc_dectest(&dep->refc, -1);
+    refc = de_refc_dec_read(dep, -1);
     if (refc == -1)
 	(void) hash_erase(&erts_dist_table, (void *) dep);
     erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
 
-    if (refc == 0)
-	erts_schedule_delete_dist_entry(dep);
+    if (refc == 0) {
+        if (node_tab_delete_delay == 0)
+            prepare_try_delete_dist_entry(vbin);
+        else if (node_tab_delete_delay > 0)
+            erts_start_timer_callback(node_tab_delete_delay,
+                                      prepare_try_delete_dist_entry,
+                                      vbin);
+    }
 }
 
-void erts_schedule_delete_dist_entry(DistEntry *dep)
+int erts_dist_entry_destructor(Binary *bin)
 {
-    ASSERT(dep != erts_this_dist_entry);
-    if (dep != erts_this_dist_entry) {
-	if (node_tab_delete_delay == 0)
-	    try_delete_dist_entry((void *) dep);
-	else if (node_tab_delete_delay > 0)
-	    erts_start_timer_callback(node_tab_delete_delay,
-				      try_delete_dist_entry,
-				      (void *) dep);
-    }
+    DistEntry *dep = ErtsBin2DistEntry(bin);
+    erts_aint_t refc;
+
+    refc = de_refc_read(dep, -1);
+
+    if (refc == -1)
+        return 1; /* Allow deallocation of structure... */
+
+    if (node_tab_delete_delay == 0)
+        prepare_try_delete_dist_entry((void *) bin);
+    else if (node_tab_delete_delay > 0)
+        erts_start_timer_callback(node_tab_delete_delay,
+                                  prepare_try_delete_dist_entry,
+                                  (void *) bin);
+
+    return 0;
 }
 
 Uint
@@ -729,19 +862,18 @@ void
 erts_set_this_node(Eterm sysname, Uint creation)
 {
     ERTS_SMP_LC_ASSERT(erts_thr_progress_is_blocking());
-    ASSERT(erts_smp_refc_read(&erts_this_dist_entry->refc, 2));
+    ASSERT(2 <= de_refc_read(erts_this_dist_entry, 2));
 
     if (erts_smp_refc_dectest(&erts_this_node->refc, 0) == 0)
         try_delete_node(erts_this_node);
 
-    if (erts_smp_refc_dectest(&erts_this_dist_entry->refc, 0) == 0)
-        try_delete_dist_entry(erts_this_dist_entry);
+    erts_deref_dist_entry(erts_this_dist_entry);
 
     erts_this_node = NULL; /* to make sure refc is bumped for this node */
     erts_this_node = erts_find_or_insert_node(sysname, creation);
     erts_this_dist_entry = erts_this_node->dist_entry;
 
-    erts_smp_refc_inc(&erts_this_dist_entry->refc, 2);
+    erts_ref_dist_entry(erts_this_dist_entry);
 
     erts_this_node_sysname = erts_this_node_sysname_BUFFER;
     erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER),
@@ -808,9 +940,9 @@ void erts_init_node_tables(int dd_sec)
     ASSERT(erts_this_node->dist_entry != NULL);
     erts_this_dist_entry = erts_this_node->dist_entry;
     /* +1 for erts_this_dist_entry */
-    /* +1 for erts_this_node->dist_entry */
-    erts_smp_refc_init(&erts_this_dist_entry->refc, 2);
+    erts_ref_dist_entry(erts_this_dist_entry);
 
+    ASSERT(2 == de_refc_read(erts_this_dist_entry, 2));
 
     erts_this_node_sysname = erts_this_node_sysname_BUFFER;
     erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER),
@@ -862,6 +994,7 @@ static Eterm AM_node_references;
 static Eterm AM_system;
 static Eterm AM_timer;
 static Eterm AM_delayed_delete_timer;
+static Eterm AM_thread_progress_delete_timer;
 
 static void setup_reference_table(void);
 static Eterm reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp);
@@ -951,6 +1084,7 @@ erts_get_node_and_dist_references(struct process *proc)
 	INIT_AM(timer);
 	INIT_AM(system);
 	INIT_AM(delayed_delete_timer);
+        INIT_AM(thread_progress_delete_timer);
 	references_atoms_need_init = 0;
     }
 
@@ -1134,6 +1268,10 @@ insert_offheap2(ErlOffHeap *oh, void *arg)
     insert_offheap(oh, a->type, a->id);
 }
 
+#define ErtsIsDistEntryBinary(Bin)                                      \
+    (((Bin)->intern.flags & BIN_FLAG_MAGIC)                             \
+     && ERTS_MAGIC_BIN_DESTRUCTOR((Bin)) == erts_dist_entry_destructor)
+
 static void
 insert_offheap(ErlOffHeap *oh, int type, Eterm id)
 {
@@ -1144,7 +1282,10 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id)
     for (u.hdr = oh->first; u.hdr; u.hdr = u.hdr->next) {
 	switch (thing_subtag(u.hdr->thing_word)) {
 	case REF_SUBTAG:
-	    if(IsMatchProgBinary(u.mref->mb)) {
+            if (ErtsIsDistEntryBinary(u.mref->mb))
+                insert_dist_entry(ErtsBin2DistEntry(u.mref->mb),
+                                  type, id, 0);
+	    else if(IsMatchProgBinary(u.mref->mb)) {
 		InsertedBin *ib;
 		int insert_bin = 1;
 		for (ib = inserted_bins; ib; ib = ib->next)
@@ -1289,26 +1430,34 @@ insert_delayed_delete_node(void *state,
 			   ErtsMonotonicTime timeout_pos,
 			   void *vnp)
 {
-    DeclareTmpHeapNoproc(heap,3);
-    UseTmpHeapNoproc(3);
+    Eterm heap[3];
     insert_node((ErlNode *) vnp,
 		SYSTEM_REF,
 		TUPLE2(&heap[0], AM_system, AM_delayed_delete_timer));
-    UnUseTmpHeapNoproc(3);
+}
+
+static void
+insert_thr_prgr_delete_dist_entry(void *arg, ErtsThrPrgrVal thr_prgr, void *vbin)
+{
+    DistEntry *dep = ErtsBin2DistEntry(vbin);
+    Eterm heap[3];
+    insert_dist_entry(dep,
+		      SYSTEM_REF,
+		      TUPLE2(&heap[0], AM_system, AM_thread_progress_delete_timer),
+		      0);
 }
 
 static void
 insert_delayed_delete_dist_entry(void *state,
 				 ErtsMonotonicTime timeout_pos,
-				 void *vdep)
+				 void *vbin)
 {
-    DeclareTmpHeapNoproc(heap,3);
-    UseTmpHeapNoproc(3);
-    insert_dist_entry((DistEntry *) vdep,
+    DistEntry *dep = ErtsBin2DistEntry(vbin);
+    Eterm heap[3];
+    insert_dist_entry(dep,
 		      SYSTEM_REF,
 		      TUPLE2(&heap[0], AM_system, AM_delayed_delete_timer),
 		      0);
-    UnUseTmpHeapNoproc(3);
 }
 
 static void
@@ -1342,9 +1491,12 @@ setup_reference_table(void)
     erts_debug_callback_timer_foreach(try_delete_node,
 				      insert_delayed_delete_node,
 				      NULL);
-    erts_debug_callback_timer_foreach(try_delete_dist_entry,
+    erts_debug_callback_timer_foreach(prepare_try_delete_dist_entry,
 				      insert_delayed_delete_dist_entry,
 				      NULL);
+    erts_debug_later_op_foreach(try_delete_dist_entry,
+                                insert_thr_prgr_delete_dist_entry,
+                                NULL);
 
     UseTmpHeapNoproc(3);
     insert_node(erts_this_node,
@@ -1719,7 +1871,7 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp)
 
 	/* DistList = [{Dist, Refc, ReferenceIdList}] */
 	tup = MK_3TUP(referred_dists[i].dist->sysname,
-		      MK_UINT(erts_smp_refc_read(&referred_dists[i].dist->refc, 0)),
+		      MK_UINT(de_refc_read(referred_dists[i].dist, 0)),
 		      dril);
 	dl = MK_CONS(tup, dl);
     }
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 5cff5e14c2..521811da21 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -48,6 +48,9 @@
 #define ERTS_PORT_TASK_ONLY_BASIC_TYPES__
 #include "erl_port_task.h"
 #undef ERTS_PORT_TASK_ONLY_BASIC_TYPES__
+#define ERTS_BINARY_TYPES_ONLY__
+#include "erl_binary.h"
+#undef ERTS_BINARY_TYPES_ONLY__
 
 #define ERTS_NODE_TAB_DELAY_GC_DEFAULT (60)
 #define ERTS_NODE_TAB_DELAY_GC_MAX (100*1000*1000)
@@ -113,7 +116,6 @@ typedef struct dist_entry_ {
     HashBucket hash_bucket;     /* Hash bucket */
     struct dist_entry_ *next;	/* Next entry in dist_table (not sorted) */
     struct dist_entry_ *prev;	/* Previous entry in dist_table (not sorted) */
-    erts_smp_refc_t refc;		/* Reference count */
 
     erts_smp_rwmtx_t rwmtx;     /* Protects all fields below until lck_mtx. */
     Eterm sysname;		/* name@host atom for efficiency */
@@ -155,6 +157,8 @@ typedef struct dist_entry_ {
     Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf);
 
     struct cache* cache;	/* The atom cache */
+
+    ErtsThrPrgrLaterOp later_op;
 } DistEntry;
 
 typedef struct erl_node_ {
@@ -204,8 +208,12 @@ Eterm erts_get_node_and_dist_references(struct process *);
 int erts_lc_is_de_rwlocked(DistEntry *);
 int erts_lc_is_de_rlocked(DistEntry *);
 #endif
+int erts_dist_entry_destructor(Binary *bin);
+DistEntry *erts_dhandle_to_dist_entry(Eterm dhandle);
+Eterm erts_make_dhandle(Process *c_p, DistEntry *dep);
+void erts_ref_dist_entry(DistEntry *dep);
+void erts_deref_dist_entry(DistEntry *dep);
 
-ERTS_GLB_INLINE void erts_deref_dist_entry(DistEntry *dep);
 ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np);
 ERTS_GLB_INLINE void erts_smp_de_rlock(DistEntry *dep);
 ERTS_GLB_INLINE void erts_smp_de_runlock(DistEntry *dep);
@@ -216,14 +224,6 @@ ERTS_GLB_INLINE void erts_smp_de_links_unlock(DistEntry *dep);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-ERTS_GLB_INLINE void
-erts_deref_dist_entry(DistEntry *dep)
-{
-    ASSERT(dep);
-    if (erts_smp_refc_dectest(&dep->refc, 0) == 0)
-	erts_schedule_delete_dist_entry(dep);
-}
-
 ERTS_GLB_INLINE void
 erts_deref_node_entry(ErlNode *np)
 {
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 2780c111af..52631d4f0d 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -13508,7 +13508,6 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		    }
 		    erts_destroy_monitor(rmon);
 		}
-		erts_deref_dist_entry(dep);
 	    }
 	} else {
             ASSERT(is_pid(mon->u.pid) || is_port(mon->u.pid));
@@ -13748,7 +13747,6 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
 	    erts_smp_de_links_unlock(dep);
 	    if (rlnk)
 		erts_destroy_link(rlnk);
-	    erts_deref_dist_entry(dep);
 	}
 	break;
 	
@@ -14492,3 +14490,24 @@ erts_dbg_check_halloc_lock(Process *p)
     return 0;
 }
 #endif
+
+void
+erts_debug_later_op_foreach(void (*callback)(void*),
+                            void (*func)(void *, ErtsThrPrgrVal, void *),
+                            void *arg)
+{
+    int six;
+    if (!erts_smp_thr_progress_is_blocking())
+	ERTS_INTERNAL_ERROR("Not blocking thread progress");
+
+    for (six = 0; six < erts_no_schedulers; six++) {
+        ErtsSchedulerData *esdp = &erts_aligned_scheduler_data[six].esd;
+	ErtsThrPrgrLaterOp *lop = esdp->aux_work_data.later_op.first;
+
+        while (lop) {
+            if (lop->func == callback)
+                func(arg, lop->later, lop->data);
+            lop = lop->next;
+        }
+    }
+}
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 9e2ac15f13..96bb97a686 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -725,6 +725,11 @@ extern ErtsSchedulerData *erts_scheduler_data;
 int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
 #endif
 
+void
+erts_debug_later_op_foreach(void (*callback)(void*),
+                            void (*func)(void *, ErtsThrPrgrVal, void *),
+                            void *arg);
+
 #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
 
 #ifdef ERTS_SMP
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 5bce4c3c92..88a0e4bff2 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -616,7 +616,7 @@ erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize)
     sys_memcpy((void *) ep, (void *) edep, dist_ext_sz);
     ep += dist_ext_sz;
     if (new_edep->dep)
-	erts_smp_refc_inc(&new_edep->dep->refc, 1);
+        erts_ref_dist_entry(new_edep->dep);
     new_edep->extp = ep;
     new_edep->ext_endp = ep + ext_sz;
     new_edep->heap_size = -1;
diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl
index 8e9e3cb05a..be90f929df 100644
--- a/erts/emulator/test/node_container_SUITE.erl
+++ b/erts/emulator/test/node_container_SUITE.erl
@@ -405,6 +405,7 @@ node_table_gc(Config) when is_list(Config) ->
     PreKnown = nodes(known),
     io:format("PreKnown = ~p~n", [PreKnown]),
     make_node_garbage(0, 200000, 1000, []),
+    receive after 1000 -> ok end, %% Wait for thread progress...
     PostKnown = nodes(known),
     PostAreas = erlang:system_info(allocated_areas),
     io:format("PostKnown = ~p~n", [PostKnown]),
-- 
cgit v1.2.3


From ffd59fbd9ac262b7aba4b86e7da4992a3e668e24 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 1 Aug 2017 18:34:58 +0200
Subject: Introduce sender in distributed signals and dflag configuration

---
 erts/emulator/beam/bif.c  |  1 +
 erts/emulator/beam/dist.c | 72 ++++++++++++++++++++++++++++++++++-------------
 erts/emulator/beam/dist.h |  5 ++++
 3 files changed, 59 insertions(+), 19 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index b1f915b133..53b48ba6e1 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2035,6 +2035,7 @@ static Sint remote_send(Process *p, DistEntry *dep,
 
     ASSERT(is_atom(to) || is_external_pid(to));
 
+    ctx->dep = dep;
     code = erts_dsig_prepare(&ctx->dsd, dep, p, ERTS_DSP_NO_LOCK, !ctx->suspend);
     switch (code) {
     case ERTS_DSIG_PREP_NOT_ALIVE:
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index d5c39485d0..a50072fe98 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -953,11 +953,30 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx)
     }
 #endif
 
-    if (token != NIL)
-	ctl = TUPLE4(&ctx->ctl_heap[0],
-		     make_small(DOP_SEND_TT), am_Empty, remote, token);
-    else
-	ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_SEND), am_Empty, remote);
+    if (token != NIL) {
+        Eterm el1, el2;
+        if (ctx->dep->flags & DFLAG_SEND_SENDER) {
+            el1 = make_small(DOP_SEND_SENDER_TT);
+            el2 = sender->common.id;
+        }
+        else {
+            el1 = make_small(DOP_SEND_TT);
+            el2 = am_Empty;
+        }
+	ctl = TUPLE4(&ctx->ctl_heap[0], el1, el2, remote, token);
+    }
+    else {
+        Eterm el1, el2;
+        if (ctx->dep->flags & DFLAG_SEND_SENDER) {
+            el1 = make_small(DOP_SEND_SENDER);
+            el2 = sender->common.id;
+        }
+        else {
+            el1 = make_small(DOP_SEND);
+            el2 = am_Empty;
+        }
+	ctl = TUPLE3(&ctx->ctl_heap[0], el1, el2, remote);
+    }
     DTRACE6(message_send, sender_name, receiver_name,
             msize, tok_label, tok_lastcnt, tok_serial);
     DTRACE7(message_send_remote, sender_name, node_name, receiver_name,
@@ -1292,6 +1311,7 @@ int erts_net_message(Port *prt,
     }
 
     token_size = 0;
+    token = NIL;
 
     switch (type = unsigned_val(tuple[1])) {
     case DOP_LINK:
@@ -1521,38 +1541,52 @@ int erts_net_message(Port *prt,
 	}
 	break;
 
+    case DOP_SEND_SENDER_TT: {
+        Uint xsize;
     case DOP_SEND_TT:
+
 	if (tuple_arity != 4) {
 	    goto invalid_message;
 	}
-	
-	token_size = size_object(tuple[4]);
-	/* Fall through ... */
+
+	token = tuple[4];
+	token_size = size_object(token);
+        xsize = ERTS_HEAP_FRAG_SIZE(token_size);
+        goto send_common;
+
+    case DOP_SEND_SENDER:
     case DOP_SEND:
+
+        token = NIL;
+        xsize = 0;
+	if (tuple_arity != 3)
+	    goto invalid_message;
+
+    send_common:
+
 	/*
-	 * There is intentionally no testing of the cookie (it is always '')
-	 * from R9B and onwards.
+         * If DOP_SEND_SENDER or DOP_SEND_SENDER_TT element 2 contains
+         * the sender pid (i.e. DFLAG_SEND_SENDER is set); otherwise,
+         * the atom '' (empty cookie).
 	 */
+        ASSERT((type == DOP_SEND_SENDER || type == DOP_SEND_SENDER_TT)
+               ? (is_pid(tuple[2]) && (dep->flags & DFLAG_SEND_SENDER))
+               : tuple[2] == am_Empty);
+
 #ifdef ERTS_DIST_MSG_DBG
 	dist_msg_dbg(&ede, "MSG", buf, orig_len);
 #endif
-	if (type != DOP_SEND_TT && tuple_arity != 3) {
-	    goto invalid_message;
-	}
 	to = tuple[3];
 	if (is_not_pid(to)) {
 	    goto invalid_message;
 	}
 	rp = erts_proc_lookup(to);
 	if (rp) {
-	    Uint xsize = type == DOP_SEND ? 0 : ERTS_HEAP_FRAG_SIZE(token_size);
 	    ErtsProcLocks locks = 0;
 	    ErtsDistExternal *ede_copy;
 
 	    ede_copy = erts_make_dist_ext_copy(&ede, xsize);
-	    if (type == DOP_SEND) {
-		token = NIL;
-	    } else {
+	    if (is_not_nil(token)) {
 		ErlHeapFragment *heap_frag;
 		ErlOffHeap *ohp;
 		ASSERT(xsize);
@@ -1560,15 +1594,15 @@ int erts_net_message(Port *prt,
 		ERTS_INIT_HEAP_FRAG(heap_frag, token_size, token_size);
 		hp = heap_frag->mem;
 		ohp = &heap_frag->off_heap;
-		token = tuple[4];
 		token = copy_struct(token, token_size, &hp, ohp);
 	    }
 
-	    erts_queue_dist_message(rp, locks, ede_copy, token, tuple[2]);
+	    erts_queue_dist_message(rp, locks, ede_copy, token, am_Empty);
 	    if (locks)
 		erts_smp_proc_unlock(rp, locks);
 	}
 	break;
+    }
 
     case DOP_MONITOR_P_EXIT: {
 	/* We are monitoring a process on the remote node which dies, we get
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index dcd5846ca5..06c13a9f2b 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -44,6 +44,7 @@
 #define DFLAG_UTF8_ATOMS          0x10000
 #define DFLAG_MAP_TAG             0x20000
 #define DFLAG_BIG_CREATION        0x40000
+#define DFLAG_SEND_SENDER         0x80000
 
 /* All flags that should be enabled when term_to_binary/1 is used. */
 #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES	\
@@ -74,6 +75,9 @@
 #define DOP_DEMONITOR_P		20
 #define DOP_MONITOR_P_EXIT	21
 
+#define DOP_SEND_SENDER         22
+#define DOP_SEND_SENDER_TT      23
+
 /* distribution trap functions */
 extern Export* dsend2_trap;
 extern Export* dsend3_trap;
@@ -346,6 +350,7 @@ typedef struct {
     Eterm ctl_heap[6];
     ErtsDSigData dsd;
     DistEntry* dep_to_deref;
+    DistEntry *dep;
     struct erts_dsig_send_context dss;
 
     Eterm return_term;
-- 
cgit v1.2.3


From 11ee8c813ac824015a85d51fb455b31bab6f648c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 28 Aug 2017 18:40:43 +0200
Subject: erts: Fix harmless use of uninitialised value

Conditional jump or move depends on uninitialised value(s):

erts_msgq_update_internal_pointers:487 (-> 0x5A7AF9) [erl_message.h]
erts_msgq_replace_msg_ref:496 (-> 0x5A7B4C) [erl_message.h]
move_msgq_to_heap:2487 (-> 0x5AED4F) [erl_gc.c]
minor_collection:1380 (-> 0x5AB3F4) [erl_gc.c]
---
 erts/emulator/beam/erl_process.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 63c838a91d..6ad686dab2 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -12512,6 +12512,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->msg.first = NULL;
     p->msg.last = &p->msg.first;
     p->msg.save = &p->msg.first;
+    p->msg.saved_last = &p->msg.first;
     p->msg.len = 0;
 #ifdef ERTS_SMP
     p->msg_inq.first = NULL;
-- 
cgit v1.2.3


From 8b8f013dbe159819136e71f0349620ff8fc45f74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 29 Aug 2017 06:56:42 +0200
Subject: De-duplicate bignums in the literal pool

Duplicate literals in the literal pool is a waste of memory.

Also, having unique literals can simplify some loader optimizations,
because we can known that if two literal indices are distinct, the
values must be distcint too.
---
 erts/emulator/beam/beam_load.c | 71 ++++++++++++++++++++++++++++--------------
 1 file changed, 48 insertions(+), 23 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 9ff32e30f3..6cd1287ee0 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -5372,12 +5372,15 @@ get_tag_and_value(LoaderState* stp, Uint len_code,
 {
     Uint count;
     Sint val;
-    byte default_buf[128];
-    byte* bigbuf = default_buf;
+    byte default_byte_buf[128];
+    byte* byte_buf = default_byte_buf;
+    Eterm default_big_buf[128/sizeof(Eterm)];
+    Eterm* big_buf = default_big_buf;
+    Eterm tmp_big;
     byte* s;
     int i;
     int neg = 0;
-    Uint arity;
+    Uint words_needed;
     Eterm* hp;
 
     /*
@@ -5454,8 +5457,11 @@ get_tag_and_value(LoaderState* stp, Uint len_code,
 	    *result = val;
 	    return TAG_i;
 	} else {
-	    *result = new_literal(stp, &hp, BIG_UINT_HEAP_SIZE);
-	    (void) small_to_big(val, hp);
+            tmp_big = small_to_big(val, big_buf);
+            if (!find_literal(stp, tmp_big, result)) {
+                *result = new_literal(stp, &hp, BIG_UINT_HEAP_SIZE);
+                sys_memcpy(hp, big_buf, BIG_UINT_HEAP_SIZE*sizeof(Eterm));
+            }
 	    return TAG_q;
 	}
     }
@@ -5465,8 +5471,8 @@ get_tag_and_value(LoaderState* stp, Uint len_code,
      * (including margin).
      */
 
-    if (count+8 > sizeof(default_buf)) {
-	bigbuf = erts_alloc(ERTS_ALC_T_LOADER_TMP, count+8);
+    if (count+8 > sizeof(default_byte_buf)) {
+	byte_buf = erts_alloc(ERTS_ALC_T_LOADER_TMP, count+8);
     }
 
     /*
@@ -5475,20 +5481,20 @@ get_tag_and_value(LoaderState* stp, Uint len_code,
 
     GetString(stp, s, count);
     for (i = 0; i < count; i++) {
-	bigbuf[count-i-1] = *s++;
+	byte_buf[count-i-1] = *s++;
     }
 
     /*
      * Check if the number is negative, and negate it if so.
      */
 
-    if ((bigbuf[count-1] & 0x80) != 0) {
+    if ((byte_buf[count-1] & 0x80) != 0) {
 	unsigned carry = 1;
 
 	neg = 1;
 	for (i = 0; i < count; i++) {
-	    bigbuf[i] = ~bigbuf[i] + carry;
-	    carry = (bigbuf[i] == 0 && carry == 1);
+	    byte_buf[i] = ~byte_buf[i] + carry;
+	    carry = (byte_buf[i] == 0 && carry == 1);
 	}
 	ASSERT(carry == 0);
     }
@@ -5497,33 +5503,52 @@ get_tag_and_value(LoaderState* stp, Uint len_code,
      * Align to word boundary.
      */
 
-    if (bigbuf[count-1] == 0) {
+    if (byte_buf[count-1] == 0) {
 	count--;
     }
-    if (bigbuf[count-1] == 0) {
+    if (byte_buf[count-1] == 0) {
 	LoadError0(stp, "bignum not normalized");
     }
     while (count % sizeof(Eterm) != 0) {
-	bigbuf[count++] = 0;
+	byte_buf[count++] = 0;
     }
 
     /*
-     * Allocate heap space for the bignum and copy it.
+     * Convert to a bignum.
      */
 
-    arity = count/sizeof(Eterm);
-    *result = new_literal(stp, &hp, arity+1);
-    if (is_nil(bytes_to_big(bigbuf, count, neg, hp)))
-	goto load_error;
+    words_needed = count/sizeof(Eterm) + 1;
+    if (words_needed*sizeof(Eterm) > sizeof(default_big_buf)) {
+        big_buf = erts_alloc(ERTS_ALC_T_LOADER_TMP, words_needed*sizeof(Eterm));
+    }
+    tmp_big = bytes_to_big(byte_buf, count, neg, big_buf);
+    if (is_nil(tmp_big)) {
+        goto load_error;
+    }
 
-    if (bigbuf != default_buf) {
-	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) bigbuf);
+    /*
+     * Create a literal if there is no previous literal with the same value.
+     */
+
+    if (!find_literal(stp, tmp_big, result)) {
+        *result = new_literal(stp, &hp, words_needed);
+        sys_memcpy(hp, big_buf, words_needed*sizeof(Eterm));
+    }
+
+    if (byte_buf != default_byte_buf) {
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) byte_buf);
+    }
+    if (big_buf != default_big_buf) {
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) big_buf);
     }
     return TAG_q;
 
  load_error:
-    if (bigbuf != default_buf) {
-	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) bigbuf);
+    if (byte_buf != default_byte_buf) {
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) byte_buf);
+    }
+    if (big_buf != default_big_buf) {
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) big_buf);
     }
     return -1;
 }
-- 
cgit v1.2.3


From 18144987906393df88a85d1cc5a7a2b9773f6dda Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 29 Aug 2017 10:17:15 +0200
Subject: erts: Fix port_SUITE:dropped_commands tc

We start the port in the started process in order
to get the messages in the correct place and also
so that unloading the port driver does not kill the
test process.
---
 erts/emulator/test/port_SUITE.erl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index ab0b1a82bd..df28ab0e7b 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -572,9 +572,10 @@ dropped_commands(Config, Outputv, Cmd) ->
     ok.
 
 dropped_commands_test(Cmd) ->
-    Port = erlang:open_port({spawn_driver, "echo_drv"}, [{parallelism, true}]),
     spawn_monitor(
       fun() ->
+              Port = erlang:open_port({spawn_driver, "echo_drv"},
+                                      [{parallelism, true}]),
               [spawn_link(fun() -> spin(Port, Cmd) end) || _ <- lists:seq(1,8)],
               timer:sleep(5),
               port_close(Port),
-- 
cgit v1.2.3


From cdcbf85b4f00f8dc2f61e895055943fc7209027c Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 29 Aug 2017 18:28:23 +0200
Subject: Upgrade to PCRE 8.41 from PCRE 8.40

---
 erts/emulator/pcre/local_config.h     |   2 +-
 erts/emulator/pcre/pcre-8.40.tar.bz2  | Bin 1560119 -> 0 bytes
 erts/emulator/pcre/pcre-8.41.tar.bz2  | Bin 0 -> 1561874 bytes
 erts/emulator/pcre/pcre.h             |   4 +-
 erts/emulator/pcre/pcre_compile.c     |  16 +-
 erts/emulator/pcre/pcre_dfa_exec.c    |   4 +-
 erts/emulator/pcre/pcre_exec.c        |   2 +-
 erts/emulator/pcre/pcre_internal.h    |  11 +-
 erts/emulator/pcre/pcre_jit_compile.c | 958 +++++++++++++++++-----------------
 erts/emulator/pcre/pcre_tables.c      |   4 +-
 erts/emulator/pcre/pcre_ucd.c         |  14 +
 11 files changed, 535 insertions(+), 480 deletions(-)
 delete mode 100644 erts/emulator/pcre/pcre-8.40.tar.bz2
 create mode 100644 erts/emulator/pcre/pcre-8.41.tar.bz2

(limited to 'erts/emulator')

diff --git a/erts/emulator/pcre/local_config.h b/erts/emulator/pcre/local_config.h
index e90f4dcada..c6af423d72 100644
--- a/erts/emulator/pcre/local_config.h
+++ b/erts/emulator/pcre/local_config.h
@@ -86,4 +86,4 @@
 #define SUPPORT_UTF 
 
 /* Version number of package */
-#define VERSION "8.40"
+#define VERSION "8.41"
diff --git a/erts/emulator/pcre/pcre-8.40.tar.bz2 b/erts/emulator/pcre/pcre-8.40.tar.bz2
deleted file mode 100644
index 6147917f4e..0000000000
Binary files a/erts/emulator/pcre/pcre-8.40.tar.bz2 and /dev/null differ
diff --git a/erts/emulator/pcre/pcre-8.41.tar.bz2 b/erts/emulator/pcre/pcre-8.41.tar.bz2
new file mode 100644
index 0000000000..1798432dc9
Binary files /dev/null and b/erts/emulator/pcre/pcre-8.41.tar.bz2 differ
diff --git a/erts/emulator/pcre/pcre.h b/erts/emulator/pcre/pcre.h
index 9cbd9c0293..ab8f40cfc1 100644
--- a/erts/emulator/pcre/pcre.h
+++ b/erts/emulator/pcre/pcre.h
@@ -43,9 +43,9 @@ POSSIBILITY OF SUCH DAMAGE.
 /* The current PCRE version information. */
 
 #define PCRE_MAJOR          8
-#define PCRE_MINOR          40
+#define PCRE_MINOR          41
 #define PCRE_PRERELEASE     
-#define PCRE_DATE           2017-01-11
+#define PCRE_DATE           2017-07-05
 
 /* When an application links to a PCRE DLL in Windows, the symbols that are
 imported have to be identified as such. When building PCRE, the appropriate
diff --git a/erts/emulator/pcre/pcre_compile.c b/erts/emulator/pcre/pcre_compile.c
index 6e841c9cf8..e79284ab79 100644
--- a/erts/emulator/pcre/pcre_compile.c
+++ b/erts/emulator/pcre/pcre_compile.c
@@ -5740,6 +5740,21 @@ for (;; ptr++)
       ptr = p - 1;    /* Character before the next significant one. */
       }
 
+    /* We also need to skip over (?# comments, which are not dependent on
+    extended mode. */
+
+    if (ptr[1] == CHAR_LEFT_PARENTHESIS && ptr[2] == CHAR_QUESTION_MARK &&
+        ptr[3] == CHAR_NUMBER_SIGN)
+      {
+      ptr += 4;
+      while (*ptr != CHAR_NULL && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++;
+      if (*ptr == CHAR_NULL)
+        {
+        *errorcodeptr = ERR18;
+        goto FAILED;
+        }
+      }
+
     /* If the next character is '+', we have a possessive quantifier. This
     implies greediness, whatever the setting of the PCRE_UNGREEDY option.
     If the next character is '?' this is a minimizing repeat, by default,
@@ -8211,7 +8226,6 @@ for (;; ptr++)
 
       if (mclength == 1 || req_caseopt == 0)
         {
-        firstchar = mcbuffer[0] | req_caseopt;
         firstchar = mcbuffer[0];
         firstcharflags = req_caseopt;
 
diff --git a/erts/emulator/pcre/pcre_dfa_exec.c b/erts/emulator/pcre/pcre_dfa_exec.c
index 529f40685b..c859d67fc7 100644
--- a/erts/emulator/pcre/pcre_dfa_exec.c
+++ b/erts/emulator/pcre/pcre_dfa_exec.c
@@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language (but see
 below for why this module is different).
 
                        Written by Philip Hazel
-           Copyright (c) 1997-2014 University of Cambridge
+           Copyright (c) 1997-2017 University of Cambridge
 
 -----------------------------------------------------------------------------
 Redistribution and use in source and binary forms, with or without
@@ -2626,7 +2626,7 @@ for (;;)
           if (isinclass)
             {
             int max = (int)GET2(ecode, 1 + IMM2_SIZE);
-            if (*ecode == OP_CRPOSRANGE)
+            if (*ecode == OP_CRPOSRANGE && count >= (int)GET2(ecode, 1))
               {
               active_count--;           /* Remove non-match possibility */
               next_active_state--;
diff --git a/erts/emulator/pcre/pcre_exec.c b/erts/emulator/pcre/pcre_exec.c
index 0f682d3daf..6708ba92a6 100644
--- a/erts/emulator/pcre/pcre_exec.c
+++ b/erts/emulator/pcre/pcre_exec.c
@@ -755,7 +755,7 @@ if (ecode == NULL)
     return match((PCRE_PUCHAR)&rdepth, NULL, NULL, 0, NULL, NULL, 1);
   else
     {
-    int len = (char *)&rdepth - (char *)eptr;
+    int len = (int)((char *)&rdepth - (char *)eptr);
     return (len > 0)? -len : len;
     }
   }
diff --git a/erts/emulator/pcre/pcre_internal.h b/erts/emulator/pcre/pcre_internal.h
index cc4f171438..c84dcb5a38 100644
--- a/erts/emulator/pcre/pcre_internal.h
+++ b/erts/emulator/pcre/pcre_internal.h
@@ -2791,6 +2791,9 @@ extern const pcre_uint8  PRIV(ucd_stage1)[];
 extern const pcre_uint16 PRIV(ucd_stage2)[];
 extern const pcre_uint32 PRIV(ucp_gentype)[];
 extern const pcre_uint32 PRIV(ucp_gbtable)[];
+#ifdef COMPILE_PCRE32
+extern const ucd_record  PRIV(dummy_ucd_record)[];
+#endif
 #ifdef SUPPORT_JIT
 extern const int         PRIV(ucp_typerange)[];
 #endif
@@ -2799,10 +2802,16 @@ extern const int         PRIV(ucp_typerange)[];
 /* UCD access macros */
 
 #define UCD_BLOCK_SIZE 128
-#define GET_UCD(ch) (PRIV(ucd_records) + \
+#define REAL_GET_UCD(ch) (PRIV(ucd_records) + \
         PRIV(ucd_stage2)[PRIV(ucd_stage1)[(int)(ch) / UCD_BLOCK_SIZE] * \
         UCD_BLOCK_SIZE + (int)(ch) % UCD_BLOCK_SIZE])
 
+#ifdef COMPILE_PCRE32
+#define GET_UCD(ch) ((ch > 0x10ffff)? PRIV(dummy_ucd_record) : REAL_GET_UCD(ch))
+#else
+#define GET_UCD(ch) REAL_GET_UCD(ch)
+#endif
+
 #define UCD_CHARTYPE(ch)    GET_UCD(ch)->chartype
 #define UCD_SCRIPT(ch)      GET_UCD(ch)->script
 #define UCD_CATEGORY(ch)    PRIV(ucp_gentype)[UCD_CHARTYPE(ch)]
diff --git a/erts/emulator/pcre/pcre_jit_compile.c b/erts/emulator/pcre/pcre_jit_compile.c
index 89400498f0..932ca2c389 100644
--- a/erts/emulator/pcre/pcre_jit_compile.c
+++ b/erts/emulator/pcre/pcre_jit_compile.c
@@ -487,7 +487,7 @@ typedef struct compare_context {
 #undef CMP
 
 /* Used for accessing the elements of the stack. */
-#define STACK(i)      ((-(i) - 1) * (int)sizeof(sljit_sw))
+#define STACK(i)      ((i) * (int)sizeof(sljit_sw))
 
 #define TMP1          SLJIT_R0
 #define TMP2          SLJIT_R2
@@ -552,13 +552,15 @@ the start pointers when the end of the capturing group has not yet reached. */
   sljit_emit_cmp(compiler, (type), (src1), (src1w), (src2), (src2w))
 #define CMPTO(type, src1, src1w, src2, src2w, label) \
   sljit_set_label(sljit_emit_cmp(compiler, (type), (src1), (src1w), (src2), (src2w)), (label))
-#define OP_FLAGS(op, dst, dstw, src, srcw, type) \
-  sljit_emit_op_flags(compiler, (op), (dst), (dstw), (src), (srcw), (type))
+#define OP_FLAGS(op, dst, dstw, type) \
+  sljit_emit_op_flags(compiler, (op), (dst), (dstw), (type))
 #define GET_LOCAL_BASE(dst, dstw, offset) \
   sljit_get_local_base(compiler, (dst), (dstw), (offset))
 
 #define READ_CHAR_MAX 0x7fffffff
 
+#define INVALID_UTF_CHAR 888
+
 static pcre_uchar *bracketend(pcre_uchar *cc)
 {
 SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND));
@@ -784,7 +786,7 @@ switch(*cc)
 
   default:
   /* All opcodes are supported now! */
-  SLJIT_ASSERT_STOP();
+  SLJIT_UNREACHABLE();
   return NULL;
   }
 }
@@ -1660,9 +1662,9 @@ while (cc < ccend)
       {
       OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0));
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -OVECTOR(0));
-      stackpos += (int)sizeof(sljit_sw);
+      stackpos -= (int)sizeof(sljit_sw);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
-      stackpos += (int)sizeof(sljit_sw);
+      stackpos -= (int)sizeof(sljit_sw);
       setsom_found = TRUE;
       }
     cc += 1;
@@ -1676,9 +1678,9 @@ while (cc < ccend)
       {
       OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->mark_ptr);
-      stackpos += (int)sizeof(sljit_sw);
+      stackpos -= (int)sizeof(sljit_sw);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
-      stackpos += (int)sizeof(sljit_sw);
+      stackpos -= (int)sizeof(sljit_sw);
       setmark_found = TRUE;
       }
     cc += 1 + 2 + cc[1];
@@ -1689,27 +1691,27 @@ while (cc < ccend)
       {
       OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0));
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -OVECTOR(0));
-      stackpos += (int)sizeof(sljit_sw);
+      stackpos -= (int)sizeof(sljit_sw);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
-      stackpos += (int)sizeof(sljit_sw);
+      stackpos -= (int)sizeof(sljit_sw);
       setsom_found = TRUE;
       }
     if (common->mark_ptr != 0 && !setmark_found)
       {
       OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->mark_ptr);
-      stackpos += (int)sizeof(sljit_sw);
+      stackpos -= (int)sizeof(sljit_sw);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
-      stackpos += (int)sizeof(sljit_sw);
+      stackpos -= (int)sizeof(sljit_sw);
       setmark_found = TRUE;
       }
     if (common->capture_last_ptr != 0 && !capture_last_found)
       {
       OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->capture_last_ptr);
-      stackpos += (int)sizeof(sljit_sw);
+      stackpos -= (int)sizeof(sljit_sw);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
-      stackpos += (int)sizeof(sljit_sw);
+      stackpos -= (int)sizeof(sljit_sw);
       capture_last_found = TRUE;
       }
     cc += 1 + LINK_SIZE;
@@ -1723,20 +1725,20 @@ while (cc < ccend)
       {
       OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->capture_last_ptr);
-      stackpos += (int)sizeof(sljit_sw);
+      stackpos -= (int)sizeof(sljit_sw);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
-      stackpos += (int)sizeof(sljit_sw);
+      stackpos -= (int)sizeof(sljit_sw);
       capture_last_found = TRUE;
       }
     offset = (GET2(cc, 1 + LINK_SIZE)) << 1;
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, OVECTOR(offset));
-    stackpos += (int)sizeof(sljit_sw);
+    stackpos -= (int)sizeof(sljit_sw);
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset));
     OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1));
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0);
-    stackpos += (int)sizeof(sljit_sw);
+    stackpos -= (int)sizeof(sljit_sw);
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP2, 0);
-    stackpos += (int)sizeof(sljit_sw);
+    stackpos -= (int)sizeof(sljit_sw);
 
     cc += 1 + LINK_SIZE + IMM2_SIZE;
     break;
@@ -1887,18 +1889,17 @@ BOOL tmp1empty = TRUE;
 BOOL tmp2empty = TRUE;
 pcre_uchar *alternative;
 enum {
-  start,
   loop,
   end
 } status;
 
-status = save ? start : loop;
-stackptr = STACK(stackptr - 2);
+status = loop;
+stackptr = STACK(stackptr);
 stacktop = STACK(stacktop - 1);
 
 if (!save)
   {
-  stackptr += (needs_control_head ? 2 : 1) * sizeof(sljit_sw);
+  stacktop -= (needs_control_head ? 2 : 1) * sizeof(sljit_sw);
   if (stackptr < stacktop)
     {
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), stackptr);
@@ -1914,196 +1915,186 @@ if (!save)
   /* The tmp1next must be TRUE in either way. */
   }
 
+SLJIT_ASSERT(common->recursive_head_ptr != 0);
+
 do
   {
   count = 0;
-  switch(status)
+  if (cc >= ccend)
     {
-    case start:
-    SLJIT_ASSERT(save && common->recursive_head_ptr != 0);
+    if (!save)
+      break;
+
     count = 1;
     srcw[0] = common->recursive_head_ptr;
     if (needs_control_head)
       {
       SLJIT_ASSERT(common->control_head_ptr != 0);
       count = 2;
-      srcw[1] = common->control_head_ptr;
+      srcw[0] = common->control_head_ptr;
+      srcw[1] = common->recursive_head_ptr;
+      }
+    status = end;
+    }
+  else switch(*cc)
+    {
+    case OP_KET:
+    if (PRIVATE_DATA(cc) != 0)
+      {
+      count = 1;
+      srcw[0] = PRIVATE_DATA(cc);
+      SLJIT_ASSERT(PRIVATE_DATA(cc + 1) != 0);
+      cc += PRIVATE_DATA(cc + 1);
       }
-    status = loop;
+    cc += 1 + LINK_SIZE;
+    break;
+
+    case OP_ASSERT:
+    case OP_ASSERT_NOT:
+    case OP_ASSERTBACK:
+    case OP_ASSERTBACK_NOT:
+    case OP_ONCE:
+    case OP_ONCE_NC:
+    case OP_BRAPOS:
+    case OP_SBRA:
+    case OP_SBRAPOS:
+    case OP_SCOND:
+    count = 1;
+    srcw[0] = PRIVATE_DATA(cc);
+    SLJIT_ASSERT(srcw[0] != 0);
+    cc += 1 + LINK_SIZE;
     break;
 
-    case loop:
-    if (cc >= ccend)
+    case OP_CBRA:
+    case OP_SCBRA:
+    if (common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0)
       {
-      status = end;
-      break;
+      count = 1;
+      srcw[0] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE));
       }
+    cc += 1 + LINK_SIZE + IMM2_SIZE;
+    break;
 
-    switch(*cc)
-      {
-      case OP_KET:
-      if (PRIVATE_DATA(cc) != 0)
-        {
-        count = 1;
-        srcw[0] = PRIVATE_DATA(cc);
-        SLJIT_ASSERT(PRIVATE_DATA(cc + 1) != 0);
-        cc += PRIVATE_DATA(cc + 1);
-        }
-      cc += 1 + LINK_SIZE;
-      break;
+    case OP_CBRAPOS:
+    case OP_SCBRAPOS:
+    count = 2;
+    srcw[0] = PRIVATE_DATA(cc);
+    srcw[1] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE));
+    SLJIT_ASSERT(srcw[0] != 0 && srcw[1] != 0);
+    cc += 1 + LINK_SIZE + IMM2_SIZE;
+    break;
 
-      case OP_ASSERT:
-      case OP_ASSERT_NOT:
-      case OP_ASSERTBACK:
-      case OP_ASSERTBACK_NOT:
-      case OP_ONCE:
-      case OP_ONCE_NC:
-      case OP_BRAPOS:
-      case OP_SBRA:
-      case OP_SBRAPOS:
-      case OP_SCOND:
+    case OP_COND:
+    /* Might be a hidden SCOND. */
+    alternative = cc + GET(cc, 1);
+    if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN)
+      {
       count = 1;
       srcw[0] = PRIVATE_DATA(cc);
       SLJIT_ASSERT(srcw[0] != 0);
-      cc += 1 + LINK_SIZE;
-      break;
-
-      case OP_CBRA:
-      case OP_SCBRA:
-      if (common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0)
-        {
-        count = 1;
-        srcw[0] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE));
-        }
-      cc += 1 + LINK_SIZE + IMM2_SIZE;
-      break;
+      }
+    cc += 1 + LINK_SIZE;
+    break;
 
-      case OP_CBRAPOS:
-      case OP_SCBRAPOS:
-      count = 2;
+    CASE_ITERATOR_PRIVATE_DATA_1
+    if (PRIVATE_DATA(cc))
+      {
+      count = 1;
       srcw[0] = PRIVATE_DATA(cc);
-      srcw[1] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE));
-      SLJIT_ASSERT(srcw[0] != 0 && srcw[1] != 0);
-      cc += 1 + LINK_SIZE + IMM2_SIZE;
-      break;
-
-      case OP_COND:
-      /* Might be a hidden SCOND. */
-      alternative = cc + GET(cc, 1);
-      if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN)
-        {
-        count = 1;
-        srcw[0] = PRIVATE_DATA(cc);
-        SLJIT_ASSERT(srcw[0] != 0);
-        }
-      cc += 1 + LINK_SIZE;
-      break;
-
-      CASE_ITERATOR_PRIVATE_DATA_1
-      if (PRIVATE_DATA(cc))
-        {
-        count = 1;
-        srcw[0] = PRIVATE_DATA(cc);
-        }
-      cc += 2;
+      }
+    cc += 2;
 #ifdef SUPPORT_UTF
-      if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
+    if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
 #endif
-      break;
+    break;
 
-      CASE_ITERATOR_PRIVATE_DATA_2A
-      if (PRIVATE_DATA(cc))
-        {
-        count = 2;
-        srcw[0] = PRIVATE_DATA(cc);
-        srcw[1] = PRIVATE_DATA(cc) + sizeof(sljit_sw);
-        }
-      cc += 2;
+    CASE_ITERATOR_PRIVATE_DATA_2A
+    if (PRIVATE_DATA(cc))
+      {
+      count = 2;
+      srcw[0] = PRIVATE_DATA(cc);
+      srcw[1] = PRIVATE_DATA(cc) + sizeof(sljit_sw);
+      }
+    cc += 2;
 #ifdef SUPPORT_UTF
-      if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
+    if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
 #endif
-      break;
+    break;
 
-      CASE_ITERATOR_PRIVATE_DATA_2B
-      if (PRIVATE_DATA(cc))
-        {
-        count = 2;
-        srcw[0] = PRIVATE_DATA(cc);
-        srcw[1] = PRIVATE_DATA(cc) + sizeof(sljit_sw);
-        }
-      cc += 2 + IMM2_SIZE;
+    CASE_ITERATOR_PRIVATE_DATA_2B
+    if (PRIVATE_DATA(cc))
+      {
+      count = 2;
+      srcw[0] = PRIVATE_DATA(cc);
+      srcw[1] = PRIVATE_DATA(cc) + sizeof(sljit_sw);
+      }
+    cc += 2 + IMM2_SIZE;
 #ifdef SUPPORT_UTF
-      if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
+    if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
 #endif
-      break;
+    break;
 
-      CASE_ITERATOR_TYPE_PRIVATE_DATA_1
-      if (PRIVATE_DATA(cc))
+    CASE_ITERATOR_TYPE_PRIVATE_DATA_1
+    if (PRIVATE_DATA(cc))
+      {
+      count = 1;
+      srcw[0] = PRIVATE_DATA(cc);
+      }
+    cc += 1;
+    break;
+
+    CASE_ITERATOR_TYPE_PRIVATE_DATA_2A
+    if (PRIVATE_DATA(cc))
+      {
+      count = 2;
+      srcw[0] = PRIVATE_DATA(cc);
+      srcw[1] = srcw[0] + sizeof(sljit_sw);
+      }
+    cc += 1;
+    break;
+
+    CASE_ITERATOR_TYPE_PRIVATE_DATA_2B
+    if (PRIVATE_DATA(cc))
+      {
+      count = 2;
+      srcw[0] = PRIVATE_DATA(cc);
+      srcw[1] = srcw[0] + sizeof(sljit_sw);
+      }
+    cc += 1 + IMM2_SIZE;
+    break;
+
+    case OP_CLASS:
+    case OP_NCLASS:
+#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
+    case OP_XCLASS:
+    size = (*cc == OP_XCLASS) ? GET(cc, 1) : 1 + 32 / (int)sizeof(pcre_uchar);
+#else
+    size = 1 + 32 / (int)sizeof(pcre_uchar);
+#endif
+    if (PRIVATE_DATA(cc))
+      switch(get_class_iterator_size(cc + size))
         {
+        case 1:
         count = 1;
         srcw[0] = PRIVATE_DATA(cc);
-        }
-      cc += 1;
-      break;
+        break;
 
-      CASE_ITERATOR_TYPE_PRIVATE_DATA_2A
-      if (PRIVATE_DATA(cc))
-        {
+        case 2:
         count = 2;
         srcw[0] = PRIVATE_DATA(cc);
         srcw[1] = srcw[0] + sizeof(sljit_sw);
-        }
-      cc += 1;
-      break;
+        break;
 
-      CASE_ITERATOR_TYPE_PRIVATE_DATA_2B
-      if (PRIVATE_DATA(cc))
-        {
-        count = 2;
-        srcw[0] = PRIVATE_DATA(cc);
-        srcw[1] = srcw[0] + sizeof(sljit_sw);
+        default:
+        SLJIT_UNREACHABLE();
+        break;
         }
-      cc += 1 + IMM2_SIZE;
-      break;
-
-      case OP_CLASS:
-      case OP_NCLASS:
-#if defined SUPPORT_UTF || !defined COMPILE_PCRE8
-      case OP_XCLASS:
-      size = (*cc == OP_XCLASS) ? GET(cc, 1) : 1 + 32 / (int)sizeof(pcre_uchar);
-#else
-      size = 1 + 32 / (int)sizeof(pcre_uchar);
-#endif
-      if (PRIVATE_DATA(cc))
-        switch(get_class_iterator_size(cc + size))
-          {
-          case 1:
-          count = 1;
-          srcw[0] = PRIVATE_DATA(cc);
-          break;
-
-          case 2:
-          count = 2;
-          srcw[0] = PRIVATE_DATA(cc);
-          srcw[1] = srcw[0] + sizeof(sljit_sw);
-          break;
-
-          default:
-          SLJIT_ASSERT_STOP();
-          break;
-          }
-      cc += size;
-      break;
-
-      default:
-      cc = next_opcode(common, cc);
-      SLJIT_ASSERT(cc != NULL);
-      break;
-      }
+    cc += size;
     break;
 
-    case end:
-    SLJIT_ASSERT_STOP();
+    default:
+    cc = next_opcode(common, cc);
+    SLJIT_ASSERT(cc != NULL);
     break;
     }
 
@@ -2312,7 +2303,7 @@ static SLJIT_INLINE void count_match(compiler_common *common)
 {
 DEFINE_COMPILER;
 
-OP2(SLJIT_SUB | SLJIT_SET_E, COUNT_MATCH, 0, COUNT_MATCH, 0, SLJIT_IMM, 1);
+OP2(SLJIT_SUB | SLJIT_SET_Z, COUNT_MATCH, 0, COUNT_MATCH, 0, SLJIT_IMM, 1);
 add_jump(compiler, &common->calllimit, JUMP(SLJIT_ZERO));
 }
 
@@ -2322,7 +2313,7 @@ static SLJIT_INLINE void allocate_stack(compiler_common *common, int size)
 DEFINE_COMPILER;
 
 SLJIT_ASSERT(size > 0);
-OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw));
+OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw));
 #ifdef DESTROY_REGISTERS
 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 12345);
 OP1(SLJIT_MOV, TMP3, 0, TMP1, 0);
@@ -2330,7 +2321,7 @@ OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0);
 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, TMP1, 0);
 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP1, 0);
 #endif
-add_stub(common, CMP(SLJIT_GREATER, STACK_TOP, 0, STACK_LIMIT, 0));
+add_stub(common, CMP(SLJIT_LESS, STACK_TOP, 0, STACK_LIMIT, 0));
 }
 
 static SLJIT_INLINE void free_stack(compiler_common *common, int size)
@@ -2338,7 +2329,7 @@ static SLJIT_INLINE void free_stack(compiler_common *common, int size)
 DEFINE_COMPILER;
 
 SLJIT_ASSERT(size > 0);
-OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw));
+OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw));
 }
 
 static sljit_uw * allocate_read_only_data(compiler_common *common, sljit_uw size)
@@ -2396,7 +2387,7 @@ else
   OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, length - 1);
   loop = LABEL();
   OP1(SLJIT_MOVU, SLJIT_MEM1(SLJIT_R1), sizeof(sljit_sw), SLJIT_R0, 0);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, 1);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, 1);
   JUMPTO(SLJIT_NOT_ZERO, loop);
   }
 }
@@ -2434,7 +2425,7 @@ else
   OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_IMM, length - 2);
   loop = LABEL();
   OP1(SLJIT_MOVU, SLJIT_MEM1(TMP2), sizeof(sljit_sw), TMP1, 0);
-  OP2(SLJIT_SUB | SLJIT_SET_E, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 1);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 1);
   JUMPTO(SLJIT_NOT_ZERO, loop);
   }
 
@@ -2452,22 +2443,22 @@ static sljit_sw SLJIT_CALL do_search_mark(sljit_sw *current, const pcre_uchar *s
 {
 while (current != NULL)
   {
-  switch (current[-2])
+  switch (current[1])
     {
     case type_then_trap:
     break;
 
     case type_mark:
-    if (STRCMP_UC_UC(skip_arg, (pcre_uchar *)current[-3]) == 0)
-      return current[-4];
+    if (STRCMP_UC_UC(skip_arg, (pcre_uchar *)current[2]) == 0)
+      return current[3];
     break;
 
     default:
-    SLJIT_ASSERT_STOP();
+    SLJIT_UNREACHABLE();
     break;
     }
-  SLJIT_ASSERT(current > (sljit_sw*)current[-1]);
-  current = (sljit_sw*)current[-1];
+  SLJIT_ASSERT(current[0] == 0 || current < (sljit_sw*)current[0]);
+  current = (sljit_sw*)current[0];
   }
 return -1;
 }
@@ -2501,7 +2492,7 @@ OP2(SLJIT_ADD, SLJIT_S0, 0, SLJIT_S0, 0, SLJIT_IMM, sizeof(sljit_sw));
 OP2(SLJIT_ASHR, SLJIT_S1, 0, SLJIT_S1, 0, SLJIT_IMM, UCHAR_SHIFT);
 #endif
 OP1(SLJIT_MOVU_S32, SLJIT_MEM1(SLJIT_R2), sizeof(int), SLJIT_S1, 0);
-OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1);
+OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1);
 JUMPTO(SLJIT_NOT_ZERO, loop);
 JUMPHERE(early_quit);
 
@@ -3106,8 +3097,8 @@ if (common->utf)
   OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
   /* Skip low surrogate if necessary. */
   OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xdc00);
-  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xdc00);
+  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL);
   OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
   OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
   return;
@@ -3126,6 +3117,7 @@ struct sljit_jump *jump;
 if (nltype == NLTYPE_ANY)
   {
   add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL));
+  sljit_set_current_flags(compiler, SLJIT_SET_Z);
   add_jump(compiler, backtracks, JUMP(jumpifmatch ? SLJIT_NOT_ZERO : SLJIT_ZERO));
   }
 else if (nltype == NLTYPE_ANYCRLF)
@@ -3167,7 +3159,7 @@ OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
 
 /* Searching for the first zero. */
-OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800);
+OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800);
 jump = JUMP(SLJIT_NOT_ZERO);
 /* Two byte sequence. */
 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
@@ -3181,7 +3173,7 @@ OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6);
 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
 
-OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x10000);
+OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x10000);
 jump = JUMP(SLJIT_NOT_ZERO);
 /* Three byte sequence. */
 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));
@@ -3215,15 +3207,15 @@ OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f);
 OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);
 
 /* Searching for the first zero. */
-OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800);
+OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800);
 jump = JUMP(SLJIT_NOT_ZERO);
 /* Two byte sequence. */
 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 
 JUMPHERE(jump);
-OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x400);
-OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_NOT_ZERO);
+OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x400);
+OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_NOT_ZERO);
 /* This code runs only in 8 bit mode. No need to shift the value. */
 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
 OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
@@ -3246,7 +3238,7 @@ struct sljit_jump *compare;
 
 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
 
-OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x20);
+OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x20);
 jump = JUMP(SLJIT_NOT_ZERO);
 /* Two byte sequence. */
 OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
@@ -3287,10 +3279,30 @@ static void do_getucd(compiler_common *common)
 /* Search the UCD record for the character comes in TMP1.
 Returns chartype in TMP1 and UCD offset in TMP2. */
 DEFINE_COMPILER;
+#ifdef COMPILE_PCRE32
+struct sljit_jump *jump;
+#endif
+
+#if defined SLJIT_DEBUG && SLJIT_DEBUG
+/* dummy_ucd_record */
+const ucd_record *record = GET_UCD(INVALID_UTF_CHAR);
+SLJIT_ASSERT(record->script == ucp_Common && record->chartype == ucp_Cn && record->gbprop == ucp_gbOther);
+SLJIT_ASSERT(record->caseset == 0 && record->other_case == 0);
+#endif
 
 SLJIT_ASSERT(UCD_BLOCK_SIZE == 128 && sizeof(ucd_record) == 8);
 
 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
+
+#ifdef COMPILE_PCRE32
+if (!common->utf)
+  {
+  jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x10ffff + 1);
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR);
+  JUMPHERE(jump);
+  }
+#endif
+
 OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT);
 OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_stage1));
 OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK);
@@ -3365,8 +3377,8 @@ if (newlinecheck)
   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
   end = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
   OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, common->newline & 0xff);
-  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, common->newline & 0xff);
+  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL);
 #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32
   OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT);
 #endif
@@ -3403,8 +3415,8 @@ if (common->utf)
   {
   singlechar = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800);
   OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800);
-  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800);
+  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL);
   OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
   JUMPHERE(singlechar);
@@ -3853,7 +3865,7 @@ while (TRUE)
   }
 }
 
-#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
+#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) && !(defined SUPPORT_VALGRIND)
 
 static sljit_s32 character_to_int32(pcre_uchar chr)
 {
@@ -4019,6 +4031,7 @@ instruction[0] = 0x0f;
 instruction[1] = 0xbc;
 instruction[2] = 0xc0 | (tmp1_ind << 3) | tmp1_ind;
 sljit_emit_op_custom(compiler, instruction, 3);
+sljit_set_current_flags(compiler, SLJIT_SET_Z);
 
 nomatch = JUMP(SLJIT_ZERO);
 
@@ -4119,6 +4132,7 @@ instruction[0] = 0x0f;
 instruction[1] = 0xbc;
 instruction[2] = 0xc0 | (tmp1_ind << 3) | tmp1_ind;
 sljit_emit_op_custom(compiler, instruction, 3);
+sljit_set_current_flags(compiler, SLJIT_SET_Z);
 
 JUMPTO(SLJIT_ZERO, start);
 
@@ -4155,18 +4169,8 @@ if (has_match_end)
   OP1(SLJIT_MOV, TMP3, 0, STR_END, 0);
 
   OP2(SLJIT_ADD, STR_END, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, SLJIT_IMM, IN_UCHARS(offset + 1));
-#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
-  if (sljit_x86_is_cmov_available())
-    {
-    OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_END, 0, TMP3, 0);
-    sljit_x86_emit_cmov(compiler, SLJIT_GREATER, STR_END, TMP3, 0);
-    }
-#endif
-    {
-    quit = CMP(SLJIT_LESS_EQUAL, STR_END, 0, TMP3, 0);
-    OP1(SLJIT_MOV, STR_END, 0, TMP3, 0);
-    JUMPHERE(quit);
-    }
+  OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, STR_END, 0, TMP3, 0);
+  sljit_emit_cmov(compiler, SLJIT_GREATER, STR_END, TMP3, 0);
   }
 
 #if defined SUPPORT_UTF && !defined COMPILE_PCRE32
@@ -4174,11 +4178,11 @@ if (common->utf && offset > 0)
   utf_start = LABEL();
 #endif
 
-#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
+#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) && !(defined SUPPORT_VALGRIND)
 
 /* SSE2 accelerated first character search. */
 
-if (sljit_x86_is_sse2_available())
+if (sljit_has_cpu_feature(SLJIT_HAS_SSE2))
   {
   fast_forward_first_char2_sse2(common, char1, char2);
 
@@ -4213,16 +4217,16 @@ if (sljit_x86_is_sse2_available())
     if (offset > 0)
       OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset));
     }
-  else if (sljit_x86_is_cmov_available())
-    {
-    OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_PTR, 0, STR_END, 0);
-    sljit_x86_emit_cmov(compiler, SLJIT_GREATER_EQUAL, STR_PTR, has_match_end ? SLJIT_MEM1(SLJIT_SP) : STR_END, has_match_end ? common->match_end_ptr : 0);
-    }
   else
     {
-    quit = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0);
-    OP1(SLJIT_MOV, STR_PTR, 0, has_match_end ? SLJIT_MEM1(SLJIT_SP) : STR_END, has_match_end ? common->match_end_ptr : 0);
-    JUMPHERE(quit);
+    OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, STR_PTR, 0, STR_END, 0);
+    if (has_match_end)
+      {
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr);
+      sljit_emit_cmov(compiler, SLJIT_GREATER_EQUAL, STR_PTR, TMP1, 0);
+      }
+    else
+      sljit_emit_cmov(compiler, SLJIT_GREATER_EQUAL, STR_PTR, STR_END, 0);
     }
 
   if (has_match_end)
@@ -4249,10 +4253,10 @@ else
     }
   else
     {
-    OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char1);
-    OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
-    OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char2);
-    OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+    OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char1);
+    OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL);
+    OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char2);
+    OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL);
     found = JUMP(SLJIT_NOT_ZERO);
     }
   }
@@ -4571,8 +4575,8 @@ if (common->nltype == NLTYPE_FIXED && common->newline > 255)
   firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0);
 
   OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(2));
-  OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_PTR, 0, TMP1, 0);
-  OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_GREATER_EQUAL);
+  OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, STR_PTR, 0, TMP1, 0);
+  OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER_EQUAL);
 #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32
   OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCHAR_SHIFT);
 #endif
@@ -4616,8 +4620,8 @@ if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF)
   JUMPHERE(foundcr);
   notfoundnl = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
   OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL);
-  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL);
+  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL);
 #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32
   OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT);
 #endif
@@ -4670,7 +4674,7 @@ if (!check_class_ranges(common, start_bits, (start_bits[31] & 0x80) != 0, TRUE,
   OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
   OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)start_bits);
   OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
-  OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
+  OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
   found = JUMP(SLJIT_NOT_ZERO);
   }
 
@@ -4692,8 +4696,8 @@ if (common->utf)
   {
   CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800, start);
   OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800);
-  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800);
+  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL);
   OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
   }
@@ -4780,31 +4784,31 @@ struct sljit_jump *jump;
 struct sljit_label *mainloop;
 
 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
-OP1(SLJIT_MOV, TMP1, 0, STACK_TOP, 0);
-GET_LOCAL_BASE(TMP3, 0, 0);
+OP1(SLJIT_MOV, TMP3, 0, STACK_TOP, 0);
+GET_LOCAL_BASE(TMP1, 0, 0);
 
 /* Drop frames until we reach STACK_TOP. */
 mainloop = LABEL();
-OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), 0);
-OP2(SLJIT_SUB | SLJIT_SET_S, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0);
-jump = JUMP(SLJIT_SIG_LESS_EQUAL);
-
-OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP3, 0);
-OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(TMP1), sizeof(sljit_sw));
-OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), sizeof(sljit_sw), SLJIT_MEM1(TMP1), 2 * sizeof(sljit_sw));
-OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 3 * sizeof(sljit_sw));
+OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), -sizeof(sljit_sw));
+jump = CMP(SLJIT_SIG_LESS_EQUAL, TMP2, 0, SLJIT_IMM, 0);
+
+OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0);
+OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(STACK_TOP), -2 * sizeof(sljit_sw));
+OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), sizeof(sljit_sw), SLJIT_MEM1(STACK_TOP), -3 * sizeof(sljit_sw));
+OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 3 * sizeof(sljit_sw));
 JUMPTO(SLJIT_JUMP, mainloop);
 
 JUMPHERE(jump);
-jump = JUMP(SLJIT_SIG_LESS);
-/* End of dropping frames. */
+jump = CMP(SLJIT_NOT_ZERO /* SIG_LESS */, TMP2, 0, SLJIT_IMM, 0);
+/* End of reverting values. */
+OP1(SLJIT_MOV, STACK_TOP, 0, TMP3, 0);
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 
 JUMPHERE(jump);
 OP1(SLJIT_NEG, TMP2, 0, TMP2, 0);
-OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP3, 0);
-OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(TMP1), sizeof(sljit_sw));
-OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_sw));
+OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0);
+OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(STACK_TOP), -2 * sizeof(sljit_sw));
+OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 2 * sizeof(sljit_sw));
 JUMPTO(SLJIT_JUMP, mainloop);
 }
 
@@ -4837,11 +4841,11 @@ if (common->use_ucp)
   jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE);
   add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
   OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll);
-  OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
-  OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL);
+  OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
+  OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL);
   OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll);
-  OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd);
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
+  OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL);
   JUMPHERE(jump);
   OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP2, 0);
   }
@@ -4881,11 +4885,11 @@ if (common->use_ucp)
   jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE);
   add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL));
   OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll);
-  OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
-  OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL);
+  OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
+  OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL);
   OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll);
-  OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd);
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
+  OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL);
   JUMPHERE(jump);
   }
 else
@@ -4913,7 +4917,7 @@ else
   }
 set_jumps(skipread_list, LABEL());
 
-OP2(SLJIT_XOR | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1);
+OP2(SLJIT_XOR | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1);
 sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0);
 }
 
@@ -5064,7 +5068,7 @@ switch(length)
   return TRUE;
 
   default:
-  SLJIT_ASSERT_STOP();
+  SLJIT_UNREACHABLE();
   return FALSE;
   }
 }
@@ -5077,22 +5081,22 @@ DEFINE_COMPILER;
 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
 
 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a);
-OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a);
-OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL);
-OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a);
+OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a);
+OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL);
+OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a);
 #if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32
 #ifdef COMPILE_PCRE8
 if (common->utf)
   {
 #endif
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
   OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a);
 #ifdef COMPILE_PCRE8
   }
 #endif
 #endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */
-OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL);
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 }
 
@@ -5103,34 +5107,34 @@ DEFINE_COMPILER;
 
 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
 
-OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x09);
-OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
-OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x20);
-OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
-OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xa0);
+OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x09);
+OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL);
+OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x20);
+OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
+OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xa0);
 #if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32
 #ifdef COMPILE_PCRE8
 if (common->utf)
   {
 #endif
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x1680);
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e);
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x1680);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
   OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x2000);
-  OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x200A - 0x2000);
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x202f - 0x2000);
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x205f - 0x2000);
-  OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x3000 - 0x2000);
+  OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x200A - 0x2000);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x202f - 0x2000);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x205f - 0x2000);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x3000 - 0x2000);
 #ifdef COMPILE_PCRE8
   }
 #endif
 #endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */
-OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL);
 
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 }
@@ -5143,22 +5147,22 @@ DEFINE_COMPILER;
 sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
 
 OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a);
-OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a);
-OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL);
-OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a);
+OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a);
+OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL);
+OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a);
 #if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32
 #ifdef COMPILE_PCRE8
 if (common->utf)
   {
 #endif
-  OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+  OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
   OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a);
 #ifdef COMPILE_PCRE8
   }
 #endif
 #endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */
-OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL);
 
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 }
@@ -5183,7 +5187,7 @@ label = LABEL();
 OP1(MOVU_UCHAR, CHAR1, 0, SLJIT_MEM1(TMP1), IN_UCHARS(1));
 OP1(MOVU_UCHAR, CHAR2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
 jump = CMP(SLJIT_NOT_EQUAL, CHAR1, 0, CHAR2, 0);
-OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1));
+OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1));
 JUMPTO(SLJIT_NOT_ZERO, label);
 
 JUMPHERE(jump);
@@ -5227,7 +5231,7 @@ OP1(SLJIT_MOV_U8, CHAR2, 0, SLJIT_MEM2(LCC_TABLE, CHAR2), 0);
 JUMPHERE(jump);
 #endif
 jump = CMP(SLJIT_NOT_EQUAL, CHAR1, 0, CHAR2, 0);
-OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1));
+OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1));
 JUMPTO(SLJIT_NOT_ZERO, label);
 
 JUMPHERE(jump);
@@ -5394,7 +5398,7 @@ do
 #endif
 
       default:
-      SLJIT_ASSERT_STOP();
+      SLJIT_UNREACHABLE();
       break;
       }
     context->ucharptr = 0;
@@ -5568,7 +5572,7 @@ while (*cc != XCL_END)
       break;
 
       default:
-      SLJIT_ASSERT_STOP();
+      SLJIT_UNREACHABLE();
       break;
       }
     cc += 2;
@@ -5592,7 +5596,7 @@ if ((cc[-1] & XCL_HASPROP) == 0)
       OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
       OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc);
       OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
-      OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
+      OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
       add_jump(compiler, &found, JUMP(SLJIT_NOT_ZERO));
       }
 
@@ -5625,7 +5629,7 @@ else if ((cc[-1] & XCL_MAP) != 0)
     OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
     OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc);
     OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
-    OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
+    OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
     add_jump(compiler, list, JUMP(SLJIT_NOT_ZERO));
 
 #ifdef COMPILE_PCRE8
@@ -5644,6 +5648,15 @@ if (needstype || needsscript)
   if (needschar && !charsaved)
     OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0);
 
+#ifdef COMPILE_PCRE32
+  if (!common->utf)
+    {
+    jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x10ffff + 1);
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR);
+    JUMPHERE(jump);
+    }
+#endif
+
   OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT);
   OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_stage1));
   OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK);
@@ -5735,14 +5748,14 @@ while (*cc != XCL_END)
 
     if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE))
       {
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
-      OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, numberofcmps == 0 ? SLJIT_UNUSED : TMP2, 0, SLJIT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
+      OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
       numberofcmps++;
       }
     else if (numberofcmps > 0)
       {
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL);
       jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
       numberofcmps = 0;
       }
@@ -5761,14 +5774,14 @@ while (*cc != XCL_END)
 
     if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE))
       {
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
-      OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, numberofcmps == 0 ? SLJIT_UNUSED : TMP2, 0, SLJIT_LESS_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
+      OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL);
       numberofcmps++;
       }
     else if (numberofcmps > 0)
       {
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL);
       jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
       numberofcmps = 0;
       }
@@ -5793,12 +5806,12 @@ while (*cc != XCL_END)
       break;
 
       case PT_LAMP:
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - typeoffset);
-      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ll - typeoffset);
-      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lt - typeoffset);
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - typeoffset);
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ll - typeoffset);
+      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lt - typeoffset);
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL);
       jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
       break;
 
@@ -5820,33 +5833,33 @@ while (*cc != XCL_END)
       case PT_SPACE:
       case PT_PXSPACE:
       SET_CHAR_OFFSET(9);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd - 0x9);
-      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd - 0x9);
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL);
 
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x9);
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x9);
+      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
 
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x9);
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x9);
+      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
 
       SET_TYPE_OFFSET(ucp_Zl);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Zl);
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Zl);
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL);
       jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
       break;
 
       case PT_WORD:
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_UNDERSCORE - charoffset));
-      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_UNDERSCORE - charoffset));
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL);
       /* Fall through. */
 
       case PT_ALNUM:
       SET_TYPE_OFFSET(ucp_Ll);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
-      OP_FLAGS((*cc == PT_ALNUM) ? SLJIT_MOV : SLJIT_OR, TMP2, 0, (*cc == PT_ALNUM) ? SLJIT_UNUSED : TMP2, 0, SLJIT_LESS_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
+      OP_FLAGS((*cc == PT_ALNUM) ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL);
       SET_TYPE_OFFSET(ucp_Nd);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_No - ucp_Nd);
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_No - ucp_Nd);
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL);
       jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
       break;
 
@@ -5868,8 +5881,8 @@ while (*cc != XCL_END)
           OP2(SLJIT_ADD, TMP2, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)charoffset);
           OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]);
           }
-        OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[1]);
-        OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
+        OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[1]);
+        OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL);
         other_cases += 2;
         }
       else if (is_powerof2(other_cases[2] ^ other_cases[1]))
@@ -5881,63 +5894,63 @@ while (*cc != XCL_END)
           OP2(SLJIT_ADD, TMP2, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)charoffset);
           OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]);
           }
-        OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[2]);
-        OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
+        OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[2]);
+        OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL);
 
-        OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(other_cases[0] - charoffset));
-        OP_FLAGS(SLJIT_OR | ((other_cases[3] == NOTACHAR) ? SLJIT_SET_E : 0), TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+        OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(other_cases[0] - charoffset));
+        OP_FLAGS(SLJIT_OR | ((other_cases[3] == NOTACHAR) ? SLJIT_SET_Z : 0), TMP2, 0, SLJIT_EQUAL);
 
         other_cases += 3;
         }
       else
         {
-        OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset));
-        OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
+        OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset));
+        OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL);
         }
 
       while (*other_cases != NOTACHAR)
         {
-        OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset));
-        OP_FLAGS(SLJIT_OR | ((*other_cases == NOTACHAR) ? SLJIT_SET_E : 0), TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+        OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset));
+        OP_FLAGS(SLJIT_OR | ((*other_cases == NOTACHAR) ? SLJIT_SET_Z : 0), TMP2, 0, SLJIT_EQUAL);
         }
       jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
       break;
 
       case PT_UCNC:
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_DOLLAR_SIGN - charoffset));
-      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_COMMERCIAL_AT - charoffset));
-      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_GRAVE_ACCENT - charoffset));
-      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_DOLLAR_SIGN - charoffset));
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_COMMERCIAL_AT - charoffset));
+      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_GRAVE_ACCENT - charoffset));
+      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
 
       SET_CHAR_OFFSET(0xa0);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(0xd7ff - charoffset));
-      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(0xd7ff - charoffset));
+      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL);
       SET_CHAR_OFFSET(0);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xe000 - 0);
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_GREATER_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xe000 - 0);
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_GREATER_EQUAL);
       jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
       break;
 
       case PT_PXGRAPH:
       /* C and Z groups are the farthest two groups. */
       SET_TYPE_OFFSET(ucp_Ll);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll);
-      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_GREATER);
+      OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll);
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER);
 
       jump = CMP(SLJIT_NOT_EQUAL, typereg, 0, SLJIT_IMM, ucp_Cf - ucp_Ll);
 
       /* In case of ucp_Cf, we overwrite the result. */
       SET_CHAR_OFFSET(0x2066);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066);
-      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066);
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL);
 
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066);
-      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066);
+      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
 
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x2066);
-      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x2066);
+      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
 
       JUMPHERE(jump);
       jump = CMP(SLJIT_ZERO ^ invertcmp, TMP2, 0, SLJIT_IMM, 0);
@@ -5946,21 +5959,21 @@ while (*cc != XCL_END)
       case PT_PXPRINT:
       /* C and Z groups are the farthest two groups. */
       SET_TYPE_OFFSET(ucp_Ll);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll);
-      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_GREATER);
+      OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll);
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER);
 
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Ll);
-      OP_FLAGS(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_NOT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Ll);
+      OP_FLAGS(SLJIT_AND, TMP2, 0, SLJIT_NOT_EQUAL);
 
       jump = CMP(SLJIT_NOT_EQUAL, typereg, 0, SLJIT_IMM, ucp_Cf - ucp_Ll);
 
       /* In case of ucp_Cf, we overwrite the result. */
       SET_CHAR_OFFSET(0x2066);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066);
-      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066);
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL);
 
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066);
-      OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066);
+      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
 
       JUMPHERE(jump);
       jump = CMP(SLJIT_ZERO ^ invertcmp, TMP2, 0, SLJIT_IMM, 0);
@@ -5968,21 +5981,21 @@ while (*cc != XCL_END)
 
       case PT_PXPUNCT:
       SET_TYPE_OFFSET(ucp_Sc);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Sc);
-      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Sc);
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL);
 
       SET_CHAR_OFFSET(0);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x7f);
-      OP_FLAGS(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x7f);
+      OP_FLAGS(SLJIT_AND, TMP2, 0, SLJIT_LESS_EQUAL);
 
       SET_TYPE_OFFSET(ucp_Pc);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ps - ucp_Pc);
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ps - ucp_Pc);
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL);
       jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
       break;
 
       default:
-      SLJIT_ASSERT_STOP();
+      SLJIT_UNREACHABLE();
       break;
       }
     cc += 2;
@@ -6028,6 +6041,7 @@ switch(type)
   case OP_NOT_WORD_BOUNDARY:
   case OP_WORD_BOUNDARY:
   add_jump(compiler, &common->wordboundary, JUMP(SLJIT_FAST_CALL));
+  sljit_set_current_flags(compiler, SLJIT_SET_Z);
   add_jump(compiler, backtracks, JUMP(type == OP_NOT_WORD_BOUNDARY ? SLJIT_NOT_ZERO : SLJIT_ZERO));
   return cc;
 
@@ -6043,10 +6057,10 @@ switch(type)
     else
       {
       jump[1] = CMP(SLJIT_EQUAL, TMP2, 0, STR_END, 0);
-      OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0);
-      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS);
-      OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff);
-      OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_NOT_EQUAL);
+      OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0);
+      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff);
+      OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_NOT_EQUAL);
       add_jump(compiler, backtracks, JUMP(SLJIT_NOT_EQUAL));
       check_partial(common, TRUE);
       add_jump(compiler, backtracks, JUMP(SLJIT_JUMP));
@@ -6068,9 +6082,9 @@ switch(type)
     OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
     jump[1] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
     OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));
-    OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0);
+    OP2(SLJIT_SUB | SLJIT_SET_Z | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0);
     jump[2] = JUMP(SLJIT_GREATER);
-    add_jump(compiler, backtracks, JUMP(SLJIT_LESS));
+    add_jump(compiler, backtracks, JUMP(SLJIT_NOT_EQUAL) /* LESS */);
     /* Equal. */
     OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
     jump[3] = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL);
@@ -6089,6 +6103,7 @@ switch(type)
       read_char_range(common, common->nlmin, common->nlmax, TRUE);
       add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, STR_END, 0));
       add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL));
+      sljit_set_current_flags(compiler, SLJIT_SET_Z);
       add_jump(compiler, backtracks, JUMP(SLJIT_ZERO));
       OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1);
       }
@@ -6204,7 +6219,7 @@ switch(type)
     label = LABEL();
     add_jump(compiler, backtracks, CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP3, 0));
     skip_char_back(common);
-    OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
+    OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
     JUMPTO(SLJIT_NOT_ZERO, label);
     }
   else
@@ -6217,7 +6232,7 @@ switch(type)
   check_start_used_ptr(common);
   return cc + LINK_SIZE;
   }
-SLJIT_ASSERT_STOP();
+SLJIT_UNREACHABLE();
 return cc;
 }
 
@@ -6250,7 +6265,7 @@ switch(type)
 #endif
     read_char8_type(common, type == OP_NOT_DIGIT);
     /* Flip the starting bit in the negative case. */
-  OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_digit);
+  OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_digit);
   add_jump(compiler, backtracks, JUMP(type == OP_DIGIT ? SLJIT_ZERO : SLJIT_NOT_ZERO));
   return cc;
 
@@ -6264,7 +6279,7 @@ switch(type)
   else
 #endif
     read_char8_type(common, type == OP_NOT_WHITESPACE);
-  OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_space);
+  OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_space);
   add_jump(compiler, backtracks, JUMP(type == OP_WHITESPACE ? SLJIT_ZERO : SLJIT_NOT_ZERO));
   return cc;
 
@@ -6278,7 +6293,7 @@ switch(type)
   else
 #endif
     read_char8_type(common, type == OP_NOT_WORDCHAR);
-  OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_word);
+  OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_word);
   add_jump(compiler, backtracks, JUMP(type == OP_WORDCHAR ? SLJIT_ZERO : SLJIT_NOT_ZERO));
   return cc;
 
@@ -6320,8 +6335,8 @@ switch(type)
 #elif defined COMPILE_PCRE16
     jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800);
     OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00);
-    OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800);
-    OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL);
+    OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800);
+    OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL);
     OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
     OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
 #endif
@@ -6383,6 +6398,7 @@ switch(type)
     detect_partial_match(common, backtracks);
   read_char_range(common, 0x9, 0x3000, type == OP_NOT_HSPACE);
   add_jump(compiler, &common->hspace, JUMP(SLJIT_FAST_CALL));
+  sljit_set_current_flags(compiler, SLJIT_SET_Z);
   add_jump(compiler, backtracks, JUMP(type == OP_NOT_HSPACE ? SLJIT_NOT_ZERO : SLJIT_ZERO));
   return cc;
 
@@ -6392,6 +6408,7 @@ switch(type)
     detect_partial_match(common, backtracks);
   read_char_range(common, 0xa, 0x2029, type == OP_NOT_VSPACE);
   add_jump(compiler, &common->vspace, JUMP(SLJIT_FAST_CALL));
+  sljit_set_current_flags(compiler, SLJIT_SET_Z);
   add_jump(compiler, backtracks, JUMP(type == OP_NOT_VSPACE ? SLJIT_NOT_ZERO : SLJIT_ZERO));
   return cc;
 
@@ -6418,7 +6435,7 @@ switch(type)
   OP1(SLJIT_MOV_U32, TMP1, 0, SLJIT_MEM1(STACK_TOP), (sljit_sw)PRIV(ucp_gbtable));
   OP1(SLJIT_MOV, STACK_TOP, 0, TMP2, 0);
   OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
-  OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
+  OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
   JUMPTO(SLJIT_NOT_ZERO, label);
 
   OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0);
@@ -6587,7 +6604,7 @@ switch(type)
   OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
   OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc);
   OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
-  OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
+  OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0);
   add_jump(compiler, backtracks, JUMP(SLJIT_ZERO));
 
 #if defined SUPPORT_UTF || !defined COMPILE_PCRE8
@@ -6604,7 +6621,7 @@ switch(type)
   return cc + GET(cc, 0) - 1;
 #endif
   }
-SLJIT_ASSERT_STOP();
+SLJIT_UNREACHABLE();
 return cc;
 }
 
@@ -6790,9 +6807,9 @@ else
 #endif /* SUPPORT_UTF && SUPPORT_UCP */
   {
   if (ref)
-    OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP1, 0);
+    OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP1, 0);
   else
-    OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw), TMP1, 0);
+    OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw), TMP1, 0);
 
   if (withchecks)
     jump = JUMP(SLJIT_ZERO);
@@ -6883,7 +6900,7 @@ switch(type)
   cc += 1 + IMM2_SIZE + 1 + 2 * IMM2_SIZE;
   break;
   default:
-  SLJIT_ASSERT_STOP();
+  SLJIT_UNREACHABLE();
   break;
   }
 
@@ -6897,7 +6914,7 @@ if (!minimize)
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0);
     /* Temporary release of STR_PTR. */
-    OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
+    OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
     /* Handles both invalid and empty cases. Since the minimum repeat,
     is zero the invalid case is basically the same as an empty case. */
     if (ref)
@@ -6910,7 +6927,7 @@ if (!minimize)
       zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw));
       }
     /* Restore if not zero length. */
-    OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
+    OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
     }
   else
     {
@@ -7157,7 +7174,7 @@ return (*PUBL(callout))(callout_block);
     (((int)sizeof(PUBL(callout_block)) + 7) & ~7)
 
 #define CALLOUT_ARG_OFFSET(arg) \
-    (-CALLOUT_ARG_SIZE + SLJIT_OFFSETOF(PUBL(callout_block), arg))
+    SLJIT_OFFSETOF(PUBL(callout_block), arg)
 
 static SLJIT_INLINE pcre_uchar *compile_callout_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent)
 {
@@ -7187,7 +7204,8 @@ OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(mark), (common->mark_pt
 
 /* Needed to save important temporary registers. */
 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0);
-OP2(SLJIT_SUB, SLJIT_R1, 0, STACK_TOP, 0, SLJIT_IMM, CALLOUT_ARG_SIZE);
+/* SLJIT_R0 = arguments */
+OP1(SLJIT_MOV, SLJIT_R1, 0, STACK_TOP, 0);
 GET_LOCAL_BASE(SLJIT_R2, 0, OVECTOR_START);
 sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_callout));
 OP1(SLJIT_MOV_S32, SLJIT_RETURN_REG, 0, SLJIT_RETURN_REG, 0);
@@ -7195,12 +7213,12 @@ OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
 free_stack(common, CALLOUT_ARG_SIZE / sizeof(sljit_sw));
 
 /* Check return value. */
-OP2(SLJIT_SUB | SLJIT_SET_S, SLJIT_UNUSED, 0, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0);
+OP2(SLJIT_SUB | SLJIT_SET_Z | SLJIT_SET_SIG_GREATER, SLJIT_UNUSED, 0, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0);
 add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_SIG_GREATER));
 if (common->forced_quit_label == NULL)
-  add_jump(compiler, &common->forced_quit, JUMP(SLJIT_SIG_LESS));
+  add_jump(compiler, &common->forced_quit, JUMP(SLJIT_NOT_EQUAL) /* SIG_LESS */);
 else
-  JUMPTO(SLJIT_SIG_LESS, common->forced_quit_label);
+  JUMPTO(SLJIT_NOT_EQUAL /* SIG_LESS */, common->forced_quit_label);
 return cc + 2 + 2 * LINK_SIZE;
 }
 
@@ -7321,7 +7339,7 @@ else
   allocate_stack(common, framesize + extrasize);
 
   OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
-  OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw));
+  OP2(SLJIT_ADD, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw));
   OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP2, 0);
   if (needs_control_head)
     OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr);
@@ -7392,22 +7410,22 @@ while (1)
       free_stack(common, extrasize);
 
     if (needs_control_head)
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), 0);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(-1));
     }
   else
     {
     if ((opcode != OP_ASSERT_NOT && opcode != OP_ASSERTBACK_NOT) || conditional)
       {
       /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */
-      OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw));
+      OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw));
       if (needs_control_head)
-        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), 0);
+        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(-1));
       }
     else
       {
       OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
       if (needs_control_head)
-        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), (framesize + 1) * sizeof(sljit_sw));
+        OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(-framesize - 2));
       add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
       }
     }
@@ -7418,25 +7436,25 @@ while (1)
     if (conditional)
       {
       if (extrasize > 0)
-        OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), needs_control_head ? sizeof(sljit_sw) : 0);
+        OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), needs_control_head ? STACK(-2) : STACK(-1));
       }
     else if (bra == OP_BRAZERO)
       {
       if (framesize < 0)
-        OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (extrasize - 1) * sizeof(sljit_sw));
+        OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-extrasize));
       else
         {
-        OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw));
-        OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (framesize + extrasize - 1) * sizeof(sljit_sw));
+        OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(-framesize - 1));
+        OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-framesize - extrasize));
         OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0);
         }
-      OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
+      OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
       }
     else if (framesize >= 0)
       {
       /* For OP_BRA and OP_BRAMINZERO. */
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw));
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-framesize - 1));
       }
     }
   add_jump(compiler, found, JUMP(SLJIT_JUMP));
@@ -7480,12 +7498,12 @@ if (common->positive_assert_quit != NULL)
   set_jumps(common->positive_assert_quit, LABEL());
   SLJIT_ASSERT(framesize != no_stack);
   if (framesize < 0)
-    OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, extrasize * sizeof(sljit_sw));
+    OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, extrasize * sizeof(sljit_sw));
   else
     {
     OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
     add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
-    OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw));
+    OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw));
     }
   JUMPHERE(jump);
   }
@@ -7534,18 +7552,18 @@ if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK)
     {
     /* We know that STR_PTR was stored on the top of the stack. */
     if (extrasize > 0)
-      OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (extrasize - 1) * sizeof(sljit_sw));
+      OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-extrasize));
 
     /* Keep the STR_PTR on the top of the stack. */
     if (bra == OP_BRAZERO)
       {
-      OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
+      OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
       if (extrasize == 2)
         OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
       }
     else if (bra == OP_BRAMINZERO)
       {
-      OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
+      OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
       }
     }
@@ -7554,13 +7572,13 @@ if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK)
     if (bra == OP_BRA)
       {
       /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */
-      OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw));
-      OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (extrasize - 2) * sizeof(sljit_sw));
+      OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw));
+      OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-extrasize + 1));
       }
     else
       {
       /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */
-      OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 2) * sizeof(sljit_sw));
+      OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 2) * sizeof(sljit_sw));
       if (extrasize == 2)
         {
         OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
@@ -7588,7 +7606,7 @@ if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK)
       {
       OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
       add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw));
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-framesize - 1));
       }
     set_jumps(backtrack->common.topbacktracks, LABEL());
     }
@@ -7675,23 +7693,23 @@ if (framesize < 0)
     }
 
   if (needs_control_head)
-    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), (ket != OP_KET || has_alternatives) ? sizeof(sljit_sw) : 0);
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), (ket != OP_KET || has_alternatives) ? STACK(-2) : STACK(-1));
 
   /* TMP2 which is set here used by OP_KETRMAX below. */
   if (ket == OP_KETRMAX)
-    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), 0);
+    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(-1));
   else if (ket == OP_KETRMIN)
     {
     /* Move the STR_PTR to the private_data_ptr. */
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-1));
     }
   }
 else
   {
   stacksize = (ket != OP_KET || has_alternatives) ? 2 : 1;
-  OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + stacksize) * sizeof(sljit_sw));
+  OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + stacksize) * sizeof(sljit_sw));
   if (needs_control_head)
-    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), 0);
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(-1));
 
   if (ket == OP_KETRMAX)
     {
@@ -7927,7 +7945,7 @@ if (bra == OP_BRAMINZERO)
         {
         /* Except when the whole stack frame must be saved. */
         OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
-        braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (BACKTRACK_AS(bracket_backtrack)->u.framesize + 1) * sizeof(sljit_sw));
+        braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), STACK(-BACKTRACK_AS(bracket_backtrack)->u.framesize - 2));
         }
       JUMPHERE(skip);
       }
@@ -8000,7 +8018,7 @@ if (opcode == OP_ONCE)
         OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0);
       if (BACKTRACK_AS(bracket_backtrack)->u.framesize == no_frame)
-        OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, needs_control_head ? (2 * sizeof(sljit_sw)) : sizeof(sljit_sw));
+        OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, needs_control_head ? (2 * sizeof(sljit_sw)) : sizeof(sljit_sw));
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize + 1), TMP2, 0);
       }
     else if (ket == OP_KETRMAX || has_alternatives)
@@ -8018,7 +8036,7 @@ if (opcode == OP_ONCE)
       OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0);
 
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
-    OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw));
+    OP2(SLJIT_ADD, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw));
 
     stacksize = needs_control_head ? 1 : 0;
     if (ket != OP_KET || has_alternatives)
@@ -8090,13 +8108,13 @@ if (opcode == OP_COND || opcode == OP_SCOND)
     slot = common->name_table + GET2(matchingpath, 1) * common->name_entry_size;
     OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0);
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1));
-    OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(slot, 0) << 1), TMP1, 0);
+    OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(slot, 0) << 1), TMP1, 0);
     slot += common->name_entry_size;
     i--;
     while (i-- > 0)
       {
       OP2(SLJIT_SUB, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(slot, 0) << 1), TMP1, 0);
-      OP2(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, STR_PTR, 0);
+      OP2(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, TMP2, 0, STR_PTR, 0);
       slot += common->name_entry_size;
       }
     OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0);
@@ -8111,7 +8129,7 @@ if (opcode == OP_COND || opcode == OP_SCOND)
 
     if (*matchingpath == OP_FAIL)
       stacksize = 0;
-    if (*matchingpath == OP_RREF)
+    else if (*matchingpath == OP_RREF)
       {
       stacksize = GET2(matchingpath, 1);
       if (common->currententry == NULL)
@@ -8244,7 +8262,7 @@ if (ket == OP_KETRMAX)
     {
     if (has_alternatives)
       BACKTRACK_AS(bracket_backtrack)->alternative_matchingpath = LABEL();
-    OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1);
+    OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1);
     JUMPTO(SLJIT_NOT_ZERO, rmax_label);
     /* Drop STR_PTR for greedy plus quantifier. */
     if (opcode != OP_ONCE)
@@ -8274,7 +8292,7 @@ if (ket == OP_KETRMAX)
 if (repeat_type == OP_EXACT)
   {
   count_match(common);
-  OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1);
   JUMPTO(SLJIT_NOT_ZERO, rmax_label);
   }
 else if (repeat_type == OP_UPTO)
@@ -8374,7 +8392,7 @@ switch(opcode)
   break;
 
   default:
-  SLJIT_ASSERT_STOP();
+  SLJIT_UNREACHABLE();
   break;
   }
 
@@ -8452,7 +8470,7 @@ else
   OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
   if (needs_control_head)
     OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr);
-  OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, -STACK(stacksize - 1));
+  OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw));
 
   stack = 0;
   if (!zero)
@@ -8524,7 +8542,7 @@ while (*cc != OP_KETRPOS)
     {
     if (offset != 0)
       {
-      OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, stacksize * sizeof(sljit_sw));
+      OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, stacksize * sizeof(sljit_sw));
       OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), cbraprivptr);
       OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0);
       OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), cbraprivptr, STR_PTR, 0);
@@ -8535,10 +8553,10 @@ while (*cc != OP_KETRPOS)
     else
       {
       OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
-      OP2(SLJIT_ADD, STACK_TOP, 0, TMP2, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw));
+      OP2(SLJIT_SUB, STACK_TOP, 0, TMP2, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw));
       if (opcode == OP_SBRAPOS)
-        OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_sw));
-      OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_sw), STR_PTR, 0);
+        OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), STACK(-framesize - 2));
+      OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), STACK(-framesize - 2), STR_PTR, 0);
       }
 
     /* Even if the match is empty, we need to reset the control head. */
@@ -8584,7 +8602,7 @@ while (*cc != OP_KETRPOS)
     else
       {
       OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
-      OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_sw));
+      OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP2), STACK(-framesize - 2));
       }
     }
 
@@ -8601,7 +8619,7 @@ if (!zero)
   if (framesize < 0)
     add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0));
   else /* TMP2 is set to [private_data_ptr] above. */
-    add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(TMP2), (stacksize - 1) * sizeof(sljit_sw), SLJIT_IMM, 0));
+    add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(TMP2), STACK(-stacksize), SLJIT_IMM, 0));
   }
 
 /* None of them matched. */
@@ -8824,7 +8842,7 @@ if (exact > 1)
     OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, exact);
     label = LABEL();
     compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, FALSE);
-    OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+    OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
     JUMPTO(SLJIT_NOT_ZERO, label);
     }
   else
@@ -8832,7 +8850,7 @@ if (exact > 1)
     OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, exact);
     label = LABEL();
     compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, TRUE);
-    OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+    OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
     JUMPTO(SLJIT_NOT_ZERO, label);
     }
   }
@@ -8862,7 +8880,7 @@ switch(opcode)
     if (opcode == OP_UPTO)
       {
       OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0);
-      OP2(SLJIT_SUB | SLJIT_SET_E, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
+      OP2(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
       jump = JUMP(SLJIT_ZERO);
       OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0, TMP1, 0);
       }
@@ -8924,7 +8942,7 @@ switch(opcode)
       label = LABEL();
       if (opcode == OP_UPTO)
         {
-        OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+        OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
         add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_ZERO));
         }
       compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, FALSE);
@@ -8944,7 +8962,7 @@ switch(opcode)
       OP1(SLJIT_MOV, base, offset1, STR_PTR, 0);
       if (opcode == OP_UPTO)
         {
-        OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+        OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
         add_jump(compiler, &no_match, JUMP(SLJIT_ZERO));
         }
 
@@ -8971,7 +8989,7 @@ switch(opcode)
 
       if (opcode == OP_UPTO)
         {
-        OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+        OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
         JUMPTO(SLJIT_NOT_ZERO, label);
         }
       else
@@ -9000,7 +9018,7 @@ switch(opcode)
 
       if (opcode == OP_UPTO)
         {
-        OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+        OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
         JUMPTO(SLJIT_NOT_ZERO, label);
         }
       else
@@ -9026,7 +9044,7 @@ switch(opcode)
       compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE);
       if (opcode == OP_UPTO)
         {
-        OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+        OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
         JUMPTO(SLJIT_NOT_ZERO, label);
         OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
         }
@@ -9113,7 +9131,7 @@ switch(opcode)
     label = LABEL();
     compile_char1_matchingpath(common, type, cc, &no_match, TRUE);
     OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, STR_PTR, 0);
-    OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+    OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
     JUMPTO(SLJIT_NOT_ZERO, label);
     set_jumps(no_match, LABEL());
     OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1);
@@ -9124,7 +9142,7 @@ switch(opcode)
   label = LABEL();
   detect_partial_match(common, &no_match);
   compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE);
-  OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1);
   JUMPTO(SLJIT_NOT_ZERO, label);
   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
   set_jumps(no_char1_match, LABEL());
@@ -9142,7 +9160,7 @@ switch(opcode)
   break;
 
   default:
-  SLJIT_ASSERT_STOP();
+  SLJIT_UNREACHABLE();
   break;
   }
 
@@ -9264,7 +9282,7 @@ size = 3 + (size < 0 ? 0 : size);
 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr);
 allocate_stack(common, size);
 if (size > 3)
-  OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0, SLJIT_IMM, (size - 3) * sizeof(sljit_sw));
+  OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0, SLJIT_IMM, (size - 3) * sizeof(sljit_sw));
 else
   OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0);
 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(size - 1), SLJIT_IMM, BACKTRACK_AS(then_trap_backtrack)->start);
@@ -9569,7 +9587,7 @@ while (cc < ccend)
     break;
 
     default:
-    SLJIT_ASSERT_STOP();
+    SLJIT_UNREACHABLE();
     return;
     }
   if (cc == NULL)
@@ -9677,7 +9695,7 @@ switch(opcode)
   case OP_MINUPTO:
   OP1(SLJIT_MOV, TMP1, 0, base, offset1);
   OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
-  OP2(SLJIT_SUB | SLJIT_SET_E, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
   add_jump(compiler, &jumplist, JUMP(SLJIT_ZERO));
 
   OP1(SLJIT_MOV, base, offset1, TMP1, 0);
@@ -9723,7 +9741,7 @@ switch(opcode)
   break;
 
   default:
-  SLJIT_ASSERT_STOP();
+  SLJIT_UNREACHABLE();
   break;
   }
 
@@ -9831,7 +9849,7 @@ if (*cc == OP_ASSERT || *cc == OP_ASSERTBACK)
   {
   OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(assert_backtrack)->private_data_ptr);
   add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(assert_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(assert_backtrack)->framesize * sizeof(sljit_sw));
+  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(assert_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-CURRENT_AS(assert_backtrack)->framesize - 1));
 
   set_jumps(current->topbacktracks, LABEL());
   }
@@ -9841,7 +9859,7 @@ else
 if (bra == OP_BRAZERO)
   {
   /* We know there is enough place on the stack. */
-  OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
+  OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
   OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
   JUMPTO(SLJIT_JUMP, CURRENT_AS(assert_backtrack)->matchingpath);
   JUMPHERE(brajump);
@@ -9954,7 +9972,7 @@ else if (ket == OP_KETRMIN)
       else
         {
         OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr);
-        CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (CURRENT_AS(bracket_backtrack)->u.framesize + 1) * sizeof(sljit_sw), CURRENT_AS(bracket_backtrack)->recursive_matchingpath);
+        CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), STACK(-CURRENT_AS(bracket_backtrack)->u.framesize - 2), CURRENT_AS(bracket_backtrack)->recursive_matchingpath);
         }
       /* Drop STR_PTR for non-greedy plus quantifier. */
       if (opcode != OP_ONCE)
@@ -10060,7 +10078,7 @@ if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND))
       {
       OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr);
       add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_sw));
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-assert->framesize - 1));
       }
     cond = JUMP(SLJIT_JUMP);
     set_jumps(CURRENT_AS(bracket_backtrack)->u.assert->condfailed, LABEL());
@@ -10201,7 +10219,7 @@ if (has_alternatives)
       {
       OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr);
       add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
-      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_sw));
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-assert->framesize - 1));
       }
     JUMPHERE(cond);
     }
@@ -10256,7 +10274,7 @@ else if (opcode == OP_ONCE)
   JUMPHERE(once);
   /* Restore previous private_data_ptr */
   if (CURRENT_AS(bracket_backtrack)->u.framesize >= 0)
-    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracket_backtrack)->u.framesize * sizeof(sljit_sw));
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-CURRENT_AS(bracket_backtrack)->u.framesize - 1));
   else if (ket == OP_KETRMIN)
     {
     OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
@@ -10346,7 +10364,7 @@ if (current->topbacktracks)
   free_stack(common, CURRENT_AS(bracketpos_backtrack)->stacksize);
   JUMPHERE(jump);
   }
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(bracketpos_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracketpos_backtrack)->framesize * sizeof(sljit_sw));
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(bracketpos_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-CURRENT_AS(bracketpos_backtrack)->framesize - 1));
 }
 
 static SLJIT_INLINE void compile_braminzero_backtrackingpath(compiler_common *common, struct backtrack_common *current)
@@ -10392,10 +10410,10 @@ if (opcode == OP_THEN || opcode == OP_THEN_ARG)
     jump = JUMP(SLJIT_JUMP);
 
     loop = LABEL();
-    OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), -(int)sizeof(sljit_sw));
+    OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
     JUMPHERE(jump);
-    CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), -(int)(2 * sizeof(sljit_sw)), TMP1, 0, loop);
-    CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), -(int)(3 * sizeof(sljit_sw)), TMP2, 0, loop);
+    CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0, loop);
+    CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(2), TMP2, 0, loop);
     add_jump(compiler, &common->then_trap->quit, JUMP(SLJIT_JUMP));
     return;
     }
@@ -10645,7 +10663,7 @@ while (current)
     break;
 
     default:
-    SLJIT_ASSERT_STOP();
+    SLJIT_UNREACHABLE();
     break;
     }
   current = current->prev;
@@ -10684,7 +10702,7 @@ sljit_emit_fast_enter(compiler, TMP2, 0);
 count_match(common);
 allocate_stack(common, private_data_size + framesize + alternativesize);
 OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(private_data_size + framesize + alternativesize - 1), TMP2, 0);
-copy_private_data(common, ccbegin, ccend, TRUE, private_data_size + framesize + alternativesize, framesize + alternativesize, needs_control_head);
+copy_private_data(common, ccbegin, ccend, TRUE, framesize + alternativesize, private_data_size + framesize + alternativesize, needs_control_head);
 if (needs_control_head)
   OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0);
 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, STACK_TOP, 0);
@@ -10737,9 +10755,9 @@ if (common->quit != NULL)
   OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr);
   if (needs_frame)
     {
-    OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw));
-    add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
     OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw));
+    add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
+    OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw));
     }
   OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 0);
   common->quit = NULL;
@@ -10750,32 +10768,32 @@ set_jumps(common->accept, LABEL());
 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr);
 if (needs_frame)
   {
-  OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw));
-  add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
   OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw));
+  add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL));
+  OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw));
   }
 OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 1);
 
 JUMPHERE(jump);
 if (common->quit != NULL)
   set_jumps(common->quit, LABEL());
-copy_private_data(common, ccbegin, ccend, FALSE, private_data_size + framesize + alternativesize, framesize + alternativesize, needs_control_head);
+copy_private_data(common, ccbegin, ccend, FALSE, framesize + alternativesize, private_data_size + framesize + alternativesize, needs_control_head);
 free_stack(common, private_data_size + framesize + alternativesize);
 if (needs_control_head)
   {
-  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), 2 * sizeof(sljit_sw));
-  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), sizeof(sljit_sw));
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(-3));
+  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(-2));
   OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, TMP1, 0);
   OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
   OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, TMP2, 0);
   }
 else
   {
-  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), sizeof(sljit_sw));
+  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(-2));
   OP1(SLJIT_MOV, TMP1, 0, TMP3, 0);
   OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, TMP2, 0);
   }
-sljit_emit_fast_return(compiler, SLJIT_MEM1(STACK_TOP), 0);
+sljit_emit_fast_return(compiler, SLJIT_MEM1(STACK_TOP), STACK(-1));
 }
 
 #undef COMPILE_BACKTRACKINGPATH
@@ -11237,7 +11255,7 @@ OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP2, 0);
 OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack));
 OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top), STACK_TOP, 0);
-OP2(SLJIT_ADD, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit), SLJIT_IMM, STACK_GROWTH_RATE);
+OP2(SLJIT_SUB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit), SLJIT_IMM, STACK_GROWTH_RATE);
 
 sljit_emit_ijump(compiler, SLJIT_CALL2, SLJIT_IMM, SLJIT_FUNC_OFFSET(sljit_stack_resize));
 jump = CMP(SLJIT_NOT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0);
@@ -11391,10 +11409,10 @@ union {
 sljit_u8 local_space[MACHINE_STACK_SIZE];
 struct sljit_stack local_stack;
 
-local_stack.top = (sljit_sw)&local_space;
-local_stack.base = local_stack.top;
-local_stack.limit = local_stack.base + MACHINE_STACK_SIZE;
-local_stack.max_limit = local_stack.limit;
+local_stack.max_limit = local_space;
+local_stack.limit = local_space;
+local_stack.base = local_space + MACHINE_STACK_SIZE;
+local_stack.top = local_space + MACHINE_STACK_SIZE;
 arguments->stack = &local_stack;
 convert_executable_func.executable_func = executable_func;
 return convert_executable_func.call_executable_func(arguments);
diff --git a/erts/emulator/pcre/pcre_tables.c b/erts/emulator/pcre/pcre_tables.c
index 2f6302e2e1..08e31f1460 100644
--- a/erts/emulator/pcre/pcre_tables.c
+++ b/erts/emulator/pcre/pcre_tables.c
@@ -6,7 +6,7 @@
 and semantics are as close as possible to those of the Perl 5 language.
 
                        Written by Philip Hazel
-           Copyright (c) 1997-2012 University of Cambridge
+           Copyright (c) 1997-2017 University of Cambridge
 
 -----------------------------------------------------------------------------
 Redistribution and use in source and binary forms, with or without
@@ -162,7 +162,7 @@ const pcre_uint32 PRIV(ucp_gbtable[]) = {
 
    (1<<ucp_gbExtend)|(1<<ucp_gbSpacingMark),                /*  5 SpacingMark */
    (1<<ucp_gbExtend)|(1<<ucp_gbSpacingMark)|(1<<ucp_gbL)|   /*  6 L */
-     (1<<ucp_gbL)|(1<<ucp_gbV)|(1<<ucp_gbLV)|(1<<ucp_gbLVT),
+     (1<<ucp_gbV)|(1<<ucp_gbLV)|(1<<ucp_gbLVT),
 
    (1<<ucp_gbExtend)|(1<<ucp_gbSpacingMark)|(1<<ucp_gbV)|   /*  7 V */
      (1<<ucp_gbT),
diff --git a/erts/emulator/pcre/pcre_ucd.c b/erts/emulator/pcre/pcre_ucd.c
index 9b700c0785..2dd4b05751 100644
--- a/erts/emulator/pcre/pcre_ucd.c
+++ b/erts/emulator/pcre/pcre_ucd.c
@@ -38,6 +38,20 @@ const pcre_uint16 PRIV(ucd_stage2)[] = {0};
 const pcre_uint32 PRIV(ucd_caseless_sets)[] = {0};
 #else
 
+/* If the 32-bit library is run in non-32-bit mode, character values
+greater than 0x10ffff may be encountered. For these we set up a
+special record. */
+
+#ifdef COMPILE_PCRE32
+const ucd_record PRIV(dummy_ucd_record)[] = {{
+  ucp_Common,    /* script */
+  ucp_Cn,        /* type unassigned */
+  ucp_gbOther,   /* grapheme break property */
+  0,             /* case set */
+  0,             /* other case */
+  }};
+#endif
+
 /* When recompiling tables with a new Unicode version, please check the
 types in this structure definition from pcre_internal.h (the actual
 field names will be different):
-- 
cgit v1.2.3


From 7c5fcd3f2701cbb614930682ac52ff75b9c26e6c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 28 Aug 2017 20:19:43 +0200
Subject: erts: Fix memory leak when sending to terminating port

Error: Leak_DefinitelyLost

erts_alloc:230 (-> 0x52E54D) [erl_alloc.h]
port_task_alloc:154 (-> 0x52F3CA) [erl_port_task.c]
erts_port_task_alloc_p2p_sig_data:212 (-> 0x52F5D3) [erl_port_task.c]
erts_port_output:2147 (-> 0x4F6057) [io.c]
erts_port_command:4126 (-> 0x4FA10E) [io.c]
do_send:2200 (-> 0x4E4C64) [bif.c]
erl_send:2494 (-> 0x4E5E09) [bif.c]
process_main:1730 (-> 0x43ADA5) [beam_emu.c]
---
 erts/emulator/beam/erl_port_task.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 1ab1e47254..4d7a86398a 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -1474,10 +1474,10 @@ erts_port_task_schedule(Eterm id,
     }
 #endif
 
-    if (!pp)
-	goto fail;
-
     if (type != ERTS_PORT_TASK_PROC_SIG) {
+        if (!pp)
+            goto fail;
+
 	ptp = port_task_alloc();
 
 	ptp->type = type;
@@ -1515,6 +1515,9 @@ erts_port_task_schedule(Eterm id,
 	ptp->u.alive.td.psig.callback = va_arg(argp, ErtsProc2PortSigCallback);
  	ptp->u.alive.flags |= va_arg(argp, int);
 	va_end(argp);
+        if (!pp)
+            goto fail;
+
 	if (!(ptp->u.alive.flags & ERTS_PT_FLG_NOSUSPEND))
 	    set_tmp_handle(ptp, pthp);
 	else {
-- 
cgit v1.2.3


From d83975178956dbaed84c315ba249a571ce796cb3 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 29 Aug 2017 16:16:33 +0200
Subject: Revert "remove unused purify functions"

This reverts commit d8c8e0c66d6faf5402682f3a8568362eedebdfee.
---
 erts/emulator/test/process_SUITE.erl | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index 6ded7ff1c9..bcf459e4fb 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -152,7 +152,11 @@ spawn_with_binaries(Config) when is_list(Config) ->
     TwoMeg = lists:duplicate(1024, L),
     Fun = fun() -> spawn(?MODULE, binary_owner, [list_to_binary(TwoMeg)]),
 			 receive after 1 -> ok end end,
-    test_server:do_times(150, Fun),
+    Iter = case test_server:purify_is_running() of
+		     true -> 10;
+		     false -> 150
+		 end,
+    test_server:do_times(Iter, Fun),
     ok.
 
 binary_owner(Bin) when is_binary(Bin) ->
-- 
cgit v1.2.3


From 214d71f577e8e21ea2e27da60b1e3bd0494a6627 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 29 Aug 2017 16:42:14 +0200
Subject: [ct] Cleanup and rename purify related functions as valgrind

---
 erts/emulator/test/process_SUITE.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index bcf459e4fb..a9f20f9928 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -152,7 +152,7 @@ spawn_with_binaries(Config) when is_list(Config) ->
     TwoMeg = lists:duplicate(1024, L),
     Fun = fun() -> spawn(?MODULE, binary_owner, [list_to_binary(TwoMeg)]),
 			 receive after 1 -> ok end end,
-    Iter = case test_server:purify_is_running() of
+    Iter = case test_server:is_valgrind() of
 		     true -> 10;
 		     false -> 150
 		 end,
-- 
cgit v1.2.3


From 97dc5e7f396129222419811c173edc7fa767b0f8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 30 Aug 2017 19:53:37 +0200
Subject: erts: Fix crash in binary_to_atom/term for invalid utf8

such as a sub-binary, of a correct utf8 string,
that ends in the middle of a character.
---
 erts/emulator/beam/atom.c           | 2 +-
 erts/emulator/test/bif_SUITE.erl    | 3 +++
 erts/emulator/test/binary_SUITE.erl | 3 +++
 3 files changed, 7 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index 84d2d5e3ed..f95582d1ab 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -137,7 +137,7 @@ atom_hash(Atom* obj)
     while(len--) {
 	v = *p++;
 	/* latin1 clutch for r16 */
-	if ((v & 0xFE) == 0xC2 && (*p & 0xC0) == 0x80) {
+	if (len && (v & 0xFE) == 0xC2 && (*p & 0xC0) == 0x80) {
 	    v = (v << 6) | (*p & 0x3F);
 	    p++; len--;
 	}
diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl
index 02c6de8cb1..dd1949d041 100644
--- a/erts/emulator/test/bif_SUITE.erl
+++ b/erts/emulator/test/bif_SUITE.erl
@@ -482,6 +482,9 @@ binary_to_atom(Config) when is_list(Config) ->
     ?line ?BADARG(binary_to_atom(id(<<255>>), utf8)),
     ?line ?BADARG(binary_to_atom(id(<<255,0>>), utf8)),
     ?line ?BADARG(binary_to_atom(id(<<16#C0,16#80>>), utf8)), %Overlong 0.
+    <<B:1/binary, _/binary>> = id(<<194, 163>>), %Truncated character ERL-474
+    ?BADARG(binary_to_atom(B, utf8)),
+
     ?line [?BADARG(binary_to_atom(<<C/utf8>>, utf8)) ||
 	      C <- lists:seq(256, 16#D7FF)],
     ?line [?BADARG(binary_to_atom(<<C/utf8>>, utf8)) ||
diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index fe0a745db8..fc3be38519 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -599,6 +599,9 @@ bad_binary_to_term(Config) when is_list(Config) ->
 
     %% Bad float.
     ?line bad_bin_to_term(<<131,70,-1:64>>),
+
+    %% Truncated UTF8 character (ERL-474)
+    bad_bin_to_term(<<131,119,1,194,163>>),
     ok.
 
 bad_bin_to_term(BadBin) ->
-- 
cgit v1.2.3


From 1b8cbb336698ab0a1992a35e2c08e2d25c6a66d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 31 Aug 2017 14:56:33 +0200
Subject: Introduce '%warm' and beam_warm.h

The bit syntax instructions are mixed among other instructions
in beam_hot.h and beam_cold.h.

Introduce a new hotness level called '%warm' with is associated
file beam_warm.h. Mark all bit syntax instructions as '%warm'.
---
 erts/emulator/Makefile.in        |  1 +
 erts/emulator/beam/beam_emu.c    |  2 ++
 erts/emulator/beam/ops.tab       |  4 ++--
 erts/emulator/utils/beam_makeops | 36 +++++++++++++++++++++++++-----------
 4 files changed, 30 insertions(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index bc7eb72221..cdc14a0c8b 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -545,6 +545,7 @@ OPCODE_TABLES += hipe/hipe_ops.tab hipe/hipe_instrs.tab
 endif
 
 $(TTF_DIR)/beam_cold.h \
+$(TTF_DIR)/beam_warm.h \
 $(TTF_DIR)/beam_hot.h \
 $(TTF_DIR)/beam_opcodes.c \
 $(TTF_DIR)/beam_opcodes.h \
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 2c37dc42b3..48569a90a9 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -870,6 +870,8 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
      goto do_schedule1;
  }
 
+#include "beam_warm.h"
+
  OpCase(normal_exit): {
      SWAPOUT;
      c_p->freason = EXC_NORMAL;
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index b6e995fdbe..d199c9e9f2 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1040,7 +1040,7 @@ func_info M F A => i_func_info u M F A
 # New bit syntax matching (R11B).
 # ================================================================
 
-%cold
+%warm
 bs_start_match2 Fail=f ica X Y D => jump Fail
 bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D
 i_bs_start_match2 xy f I I x
@@ -1138,7 +1138,7 @@ i_bs_validate_unicode_retract j s s
 #
 # Constructing binaries
 #
-%cold
+%warm
 
 bs_init2 Fail Sz Words Regs Flags Dst | binary_too_big(Sz) => system_limit Fail
 
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 6c54ab3421..bcc92472d4 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -20,13 +20,16 @@
 #
 use strict;
 use vars qw($BEAM_FORMAT_NUMBER);
+use constant COLD => 0;
+use constant WARM => 1;
+use constant HOT => 2;
 
 $BEAM_FORMAT_NUMBER = undef;
 
 my $target = \&emulator_output;
 my $outdir = ".";		# Directory for output files.
 my $verbose = 0;
-my $hot = 1;
+my $hotness = 1;
 my $num_file_opcodes = 0;
 my $wordsize = 32;
 my %defs;			# Defines (from command line).
@@ -344,13 +347,16 @@ while (<>) {
     }
 
     #
-    # Handle %hot/%cold.
+    # Handle %hot, %warm, and %cold.
     # 
     if (/^\%hot/) {
-	$hot = 1;
+	$hotness = HOT;
 	next;
+    } elsif (/^\%warm/) {
+	$hotness = WARM;
+        next;
     } elsif (/^\%cold/) {
-	$hot = 0;
+	$hotness = COLD;
 	next;
     }
 
@@ -438,7 +444,7 @@ while (<>) {
     if (defined $gen_opnum{$name,$arity} and $obsolete[$gen_opnum{$name,$arity}]) {
 	error("specific instructions may not be specified for obsolete instructions");
     }
-    save_specific_ops($name, $arity, $hot, @args);
+    save_specific_ops($name, $arity, $hotness, @args);
     if (defined $op_num) {
 	error("specific instructions must not be numbered");
     } elsif (!defined($gen_arity{$name}) && !defined($unnumbered{$name,$arity})) {
@@ -801,12 +807,17 @@ sub emulator_output {
     $name = "$outdir/beam_hot.h";
     open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
     comment('C');
-    print_code(1);
+    print_code(HOT);
+
+    $name = "$outdir/beam_warm.h";
+    open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
+    comment('C');
+    print_code(WARM);
 
     $name = "$outdir/beam_cold.h";
     open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
     comment('C');
-    print_code(0);
+    print_code(COLD);
 }
 
 sub init_item {
@@ -1039,14 +1050,16 @@ sub combine_micro_instructions {
 
     # Now generate code for each group.
     foreach my $group (sort keys %groups) {
-        my($code,@labels) = combine_instruction_group($group, @{$groups{$group}});
-        push @generated_code, [1,$code,@labels];
+        my($hotness,$code,@labels) =
+            combine_instruction_group($group, @{$groups{$group}});
+        push @generated_code, [$hotness,$code,@labels];
     }
 }
 
 sub combine_instruction_group {
     my($group,@in_instrs) = @_;
     my $gcode = '';             # Code for the entire group.
+    my $group_hotness = COLD;
 
     # Get code for the head of the group (if any).
     my $head_name = "$group.head";
@@ -1075,7 +1088,8 @@ sub combine_instruction_group {
         error("no $specific_key instruction")
             unless defined $specific_op_ref;
         foreach my $specific_op (@$specific_op_ref) {
-            my($name, $hot, @args) = @{$specific_op};
+            my($name, $hotness, @args) = @{$specific_op};
+            $group_hotness = $hotness unless $group_hotness >= $hotness;
             my $offset = 0;
             my @rest = @args;
             my @new_subs;
@@ -1182,7 +1196,7 @@ sub combine_instruction_group {
         $offset = $order_to_offset{$slots[$i+1]} if $i < $#slots;
     }
 
-    ("{\n$gcode\n}\n\n",@opcase_labels);
+    ($group_hotness,"{\n$gcode\n}\n\n",@opcase_labels);
 }
 
 sub micro_label {
-- 
cgit v1.2.3


From 3280aa8f8398b2b6a1bb78aedc27fc7e73653288 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 31 Aug 2017 12:29:25 +0200
Subject: ops.tab: Mark infrequently used instructions as %cold

Instructions that used to be implemented in beam_emu.c
were not marked as cold as it would make no difference.
---
 erts/emulator/beam/ops.tab      | 19 +++++++++++++++++++
 erts/emulator/hipe/hipe_ops.tab |  3 +++
 2 files changed, 22 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index d199c9e9f2..d848c1cceb 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -59,6 +59,7 @@ put_tuple u==0 d => too_old_compiler
 # All the other instructions.
 #
 
+%cold
 label L
 i_func_info I a a I
 int_code_end
@@ -68,6 +69,7 @@ i_debug_breakpoint
 i_return_time_trace
 i_return_to_trace
 i_yield
+%hot
 
 return
 
@@ -217,6 +219,11 @@ is_number Fail Literal=q => move Literal x | is_number Fail x
 
 jump f
 
+#
+# Expection rasing instructions. Infrequently executed.
+#
+
+%cold
 case_end NotInX=cy => move NotInX x | case_end x
 badmatch NotInX=cy => move NotInX x | badmatch x
 
@@ -238,6 +245,12 @@ i_raise
 badarg j
 system_limit j
 
+%hot
+
+#
+# Move instructions.
+#
+
 move C=cxy x==0 | jump Lbl => move_jump Lbl C
 
 move_jump f ncxy
@@ -404,8 +417,10 @@ wait_timeout_unlocked s f
 wait_timeout_locked_int I f
 wait_timeout_locked s f
 
+%cold
 i_wait_error
 i_wait_error_locked
+%hot
 
 send
 
@@ -515,6 +530,7 @@ put_list s s d
 # Some more only used by the emulator
 #
 
+%cold
 normal_exit
 continue_exit
 apply_bif
@@ -522,6 +538,7 @@ call_nif
 call_error_handler
 error_action_code
 return_trace
+%hot
 
 #
 # Instruction transformations & folded instructions.
@@ -939,9 +956,11 @@ i_apply_fun
 i_apply_fun_last P
 i_apply_fun_only
 
+%cold
 i_hibernate
 
 i_perf_counter
+%hot
 
 call_bif e
 
diff --git a/erts/emulator/hipe/hipe_ops.tab b/erts/emulator/hipe/hipe_ops.tab
index 96e4c0da91..19a3820a6a 100644
--- a/erts/emulator/hipe/hipe_ops.tab
+++ b/erts/emulator/hipe/hipe_ops.tab
@@ -23,4 +23,7 @@ hipe_trap_call_closure
 hipe_trap_return
 hipe_trap_throw
 hipe_trap_resume
+
+%cold
 hipe_call_count
+%hot
-- 
cgit v1.2.3


From 53e841b4e09f2c715fe459a1e4a84e4de5c161fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 25 Aug 2017 10:15:25 +0200
Subject: Add annotations for likely/unlikely

In a correct Erlang programs, we can expect that:

* A GC test instruction (such as test_heap) is more likely
  not to do the GC.
* A BIF is more likely to succeed than to fail.
* A BIF is more likely to fail in a guard than in a body.
* An apply or fun call is likely to succeed.

Annotate conditions accordingly.
---
 erts/emulator/beam/bif_instrs.tab | 20 ++++++++++----------
 erts/emulator/beam/instrs.tab     | 24 ++++++++++++------------
 erts/emulator/beam/macros.tab     | 16 +++++++++++-----
 3 files changed, 33 insertions(+), 27 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif_instrs.tab b/erts/emulator/beam/bif_instrs.tab
index 5aa0523e06..3c95113907 100644
--- a/erts/emulator/beam/bif_instrs.tab
+++ b/erts/emulator/beam/bif_instrs.tab
@@ -45,7 +45,7 @@ CALL_GUARD_BIF(BF, TmpReg, Dst) {
     ERTS_HOLE_CHECK(c_p);
     FCALLS = c_p->fcalls;
     ERTS_DBG_CHK_REDS(c_p, FCALLS);
-    if (is_value(result)) {
+    if (ERTS_LIKELY(is_value(result))) {
         $Dst = result;
         $NEXT0();
     }
@@ -145,12 +145,12 @@ i_gc_bif1(Fail, Bif, Src, Live, Dst) {
     ERTS_HOLE_CHECK(c_p);
     FCALLS = c_p->fcalls;
     ERTS_DBG_CHK_REDS(c_p, FCALLS);
-    if (is_value(result)) {
+    if (ERTS_LIKELY(is_value(result))) {
         $REFRESH_GEN_DEST();
         $Dst = result;
         $NEXT0();
     }
-    if ($Fail != 0) {          /* Handle error in guard. */
+    if (ERTS_LIKELY($Fail != 0)) { /* Handle error in guard. */
         $NEXT($Fail);
     }
 
@@ -195,13 +195,13 @@ i_gc_bif2(Fail, Bif, Live, Src1, Src2, Dst) {
     ERTS_HOLE_CHECK(c_p);
     FCALLS = c_p->fcalls;
     ERTS_DBG_CHK_REDS(c_p, FCALLS);
-    if (is_value(result)) {
+    if (ERTS_LIKELY(is_value(result))) {
         $REFRESH_GEN_DEST();
         $Dst = result;
         $NEXT0();
     }
 
-    if ($Fail != 0) {          /* Handle error in guard. */
+    if (ERTS_LIKELY($Fail != 0)) { /* Handle error in guard. */
         $NEXT($Fail);
     }
 
@@ -249,14 +249,14 @@ i_gc_bif3(Fail, Bif, Live, Src2, Src3, Dst) {
     ERTS_HOLE_CHECK(c_p);
     FCALLS = c_p->fcalls;
     ERTS_DBG_CHK_REDS(c_p, FCALLS);
-    if (is_value(result)) {
+    if (ERTS_LIKELY(is_value(result))) {
         $REFRESH_GEN_DEST();
         $Dst = result;
         $NEXT0();
     }
 
     /* Handle error in guard. */
-    if ($Fail != 0) {
+    if (ERTS_LIKELY($Fail != 0)) {
         $NEXT($Fail);
     }
 
@@ -325,7 +325,7 @@ call_bif(Exp) {
         ERTS_MSACC_UPDATE_CACHE_X();
     }
     ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
-    if (is_value(result)) {
+    if (ERTS_LIKELY(is_value(result))) {
         r(0) = result;
         CHECK_TERM(r(0));
         $NEXT0();
@@ -370,7 +370,7 @@ send() {
     PROCESS_MAIN_CHK_LOCKS(c_p);
     HTOP = HEAP_TOP(c_p);
     FCALLS = c_p->fcalls;
-    if (is_value(result)) {
+    if (ERTS_LIKELY(is_value(result))) {
         r(0) = result;
         CHECK_TERM(r(0));
     } else if (c_p->freason == TRAP) {
@@ -520,7 +520,7 @@ nif_bif.epilogue() {
     SWAPIN;  /* There might have been a garbage collection. */
     FCALLS = c_p->fcalls;
     ERTS_DBG_CHK_REDS(c_p, FCALLS);
-    if (is_value(nif_bif_result)) {
+    if (ERTS_LIKELY(is_value(nif_bif_result))) {
         r(0) = nif_bif_result;
         CHECK_TERM(r(0));
         SET_I(c_p->cp);
diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index 1af01e53bd..58432cbbb4 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -171,7 +171,7 @@ HANDLE_APPLY_ERROR() {
 i_apply() {
     BeamInstr *next;
     $APPLY(NULL, 0);
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         $i_call(next);
     }
     $HANDLE_APPLY_ERROR();
@@ -180,7 +180,7 @@ i_apply() {
 i_apply_last(Deallocate) {
     BeamInstr *next;
     $APPLY(I, $Deallocate);
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         $i_call_last(next, $Deallocate);
     }
     $HANDLE_APPLY_ERROR();
@@ -189,7 +189,7 @@ i_apply_last(Deallocate) {
 i_apply_only() {
     BeamInstr *next;
     $APPLY(I, 0);
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         $i_call_only(next);
     }
     $HANDLE_APPLY_ERROR();
@@ -205,7 +205,7 @@ FIXED_APPLY(Arity, I, Deallocate) {
 apply(Arity) {
     BeamInstr *next;
     $FIXED_APPLY($Arity, NULL, 0);
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         $i_call(next);
     }
     $HANDLE_APPLY_ERROR();
@@ -214,7 +214,7 @@ apply(Arity) {
 apply_last(Arity, Deallocate) {
     BeamInstr *next;
     $FIXED_APPLY($Arity, I, $Deallocate);
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         $i_call_last(next, $Deallocate);
     }
     $HANDLE_APPLY_ERROR();
@@ -238,7 +238,7 @@ DISPATCH_FUN(I) {
 i_apply_fun() {
     BeamInstr *next;
     $APPLY_FUN();
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         SET_CP(c_p, $NEXT_INSTRUCTION);
         $DISPATCH_FUN(next);
     }
@@ -248,7 +248,7 @@ i_apply_fun() {
 i_apply_fun_last(Deallocate) {
     BeamInstr *next;
     $APPLY_FUN();
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         $deallocate($Deallocate);
         $DISPATCH_FUN(next);
     }
@@ -258,7 +258,7 @@ i_apply_fun_last(Deallocate) {
 i_apply_fun_only() {
     BeamInstr *next;
     $APPLY_FUN();
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         $DISPATCH_FUN(next);
     }
     $HANDLE_APPLY_FUN_ERROR();
@@ -274,7 +274,7 @@ CALL_FUN(Fun) {
 i_call_fun(Fun) {
     BeamInstr *next;
     $CALL_FUN($Fun);
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         SET_CP(c_p, $NEXT_INSTRUCTION);
         $DISPATCH_FUN(next);
     }
@@ -284,7 +284,7 @@ i_call_fun(Fun) {
 i_call_fun_last(Fun, Deallocate) {
     BeamInstr *next;
     $CALL_FUN($Fun);
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         $deallocate($Deallocate);
         $DISPATCH_FUN(next);
     }
@@ -378,7 +378,7 @@ element_group.fetch(Src) {
 
 element_group.execute(Fail, Index, Dst) {
     element_index = $Index;
-    if (is_small(element_index) && is_tuple(element_tuple)) {
+    if (ERTS_LIKELY(is_small(element_index) && is_tuple(element_tuple))) {
         Eterm* tp = tuple_val(element_tuple);
 
         if ((signed_val(element_index) >= 1) &&
@@ -402,7 +402,7 @@ fast_element_group.fetch(Src) {
 }
 
 fast_element_group.execute(Fail, Index, Dst) {
-    if (is_tuple(fast_element_tuple)) {
+    if (ERTS_LIKELY(is_tuple(fast_element_tuple))) {
         Eterm* tp = tuple_val(fast_element_tuple);
         Eterm pos = $Index;	/* Untagged integer >= 1 */
         if (pos <= arityval(*tp)) {
diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab
index 41dc761e90..bac96be7d3 100644
--- a/erts/emulator/beam/macros.tab
+++ b/erts/emulator/beam/macros.tab
@@ -42,7 +42,7 @@ JUMP(Fail) {
 
 GC_TEST(Ns, Nh, Live) {
     Uint need = $Nh + $Ns;
-    if (E - HTOP < need) {
+    if (ERTS_UNLIKELY(E - HTOP < need)) {
        SWAPOUT;
        PROCESS_MAIN_CHK_LOCKS(c_p);
        FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, $Live, FCALLS);
@@ -55,7 +55,7 @@ GC_TEST(Ns, Nh, Live) {
 
 GC_TEST_PRESERVE(NeedHeap, Live, PreserveTerm) {
     Uint need = $NeedHeap;
-    if (E - HTOP < need) {
+    if (ERTS_UNLIKELY(E - HTOP < need)) {
        SWAPOUT;
        reg[$Live] = $PreserveTerm;
        PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -99,7 +99,13 @@ FAIL_BODY() {
 
 FAIL_HEAD_OR_BODY(Fail) {
     //| -no_prefetch
-    if ($Fail) {
+
+    /*
+     * In a correctly working program, we expect failures in
+     * guards to be more likely than failures in bodies.
+     */
+
+    if (ERTS_LIKELY($Fail)) {
         $FAIL($Fail);
     }
     goto find_func_info;
@@ -122,7 +128,7 @@ SYSTEM_LIMIT(Fail) {
 
 BIF_ERROR_ARITY_1(Fail, BIF, Op1) {
     //| -no_prefetch
-    if ($Fail) {
+    if (ERTS_LIKELY($Fail)) {
         $FAIL($Fail);
     }
     reg[0] = $Op1;
@@ -133,7 +139,7 @@ BIF_ERROR_ARITY_1(Fail, BIF, Op1) {
 
 BIF_ERROR_ARITY_2(Fail, BIF, Op1, Op2) {
     //| -no_prefetch
-    if ($Fail) {
+    if (ERTS_LIKELY($Fail)) {
         $FAIL($Fail);
     }
     reg[0] = $Op1;
-- 
cgit v1.2.3


From 0ce65931f6f4c5dc1c55e390379c81de05822f91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 31 Aug 2017 11:50:29 +0200
Subject: Annotate arithmetic instructions with likely/unlikely

We expect that:

* An arithmetic instruction is more likely to succeed
  than to fail.

* An arithmetic instruction is more likely to have small
  operands than bignum operands.
---
 erts/emulator/beam/arith_instrs.tab | 62 ++++++++++++++++++-------------------
 1 file changed, 31 insertions(+), 31 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab
index 67cd7c6a2a..7c9cd47e28 100644
--- a/erts/emulator/beam/arith_instrs.tab
+++ b/erts/emulator/beam/arith_instrs.tab
@@ -28,7 +28,7 @@ OUTLINED_ARITH_2(Fail, Live, Name, BIF, Op1, Op2, Dst) {
     result = erts_gc_$Name (c_p, reg, live);
     HEAVY_SWAPIN;
     ERTS_HOLE_CHECK(c_p);
-    if (is_value(result)) {
+    if (ERTS_LIKELY(is_value(result))) {
         $REFRESH_GEN_DEST();
         $Dst = result;
         $NEXT0();
@@ -49,10 +49,10 @@ plus.fetch(Op1, Op2) {
 }
 
 plus.execute(Fail, Live, Dst) {
-    if (is_both_small(PlusOp1, PlusOp2)) {
+    if (ERTS_LIKELY(is_both_small(PlusOp1, PlusOp2))) {
         Sint i = signed_val(PlusOp1) + signed_val(PlusOp2);
         ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
-        if (MY_IS_SSMALL(i)) {
+        if (ERTS_LIKELY(MY_IS_SSMALL(i))) {
             $Dst = make_small(i);
             $NEXT0();
         }
@@ -72,10 +72,10 @@ minus.fetch(Op1, Op2) {
 }
 
 minus.execute(Fail, Live, Dst) {
-    if (is_both_small(MinusOp1, MinusOp2)) {
+    if (ERTS_LIKELY(is_both_small(MinusOp1, MinusOp2))) {
         Sint i = signed_val(MinusOp1) - signed_val(MinusOp2);
         ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
-        if (MY_IS_SSMALL(i)) {
+        if (ERTS_LIKELY(MY_IS_SSMALL(i))) {
             $Dst = make_small(i);
             $NEXT0();
         }
@@ -98,10 +98,10 @@ increment.fetch(Src) {
 
 increment.execute(IncrementVal, Live, Dst) {
     increment_val = $IncrementVal;
-    if (is_small(increment_reg_val)) {
+    if (ERTS_LIKELY(is_small(increment_reg_val))) {
         Sint i = signed_val(increment_reg_val) + increment_val;
         ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
-        if (MY_IS_SSMALL(i)) {
+        if (ERTS_LIKELY(MY_IS_SSMALL(i))) {
             $Dst = make_small(i);
             $NEXT0();
         }
@@ -113,7 +113,7 @@ increment.execute(IncrementVal, Live, Dst) {
     result = erts_gc_mixed_plus(c_p, reg, live);
     HEAVY_SWAPIN;
     ERTS_HOLE_CHECK(c_p);
-    if (is_value(result)) {
+    if (ERTS_LIKELY(is_value(result))) {
         $REFRESH_GEN_DEST();
         $Dst = result;
         $NEXT0();
@@ -137,12 +137,12 @@ i_m_div(Fail, Live, Op1, Op2, Dst) {
 i_int_div(Fail, Live, Op1, Op2, Dst) {
     Eterm op1 = $Op1;
     Eterm op2 = $Op2;
-    if (op2 == SMALL_ZERO) {
+    if (ERTS_UNLIKELY(op2 == SMALL_ZERO)) {
         c_p->freason = BADARITH;
         $BIF_ERROR_ARITY_2($Fail, BIF_intdiv_2, op1, op2);
-    } else if (is_both_small(op1, op2)) {
+    } else if (ERTS_LIKELY(is_both_small(op1, op2))) {
         Sint ires = signed_val(op1) / signed_val(op2);
-        if (MY_IS_SSMALL(ires)) {
+        if (ERTS_LIKELY(MY_IS_SSMALL(ires))) {
             $Dst = make_small(ires);
             $NEXT0();
         }
@@ -162,15 +162,15 @@ rem.fetch(Src1, Src2) {
 }
 
 rem.execute(Fail, Live, Dst) {
-     if (RemOp2 == SMALL_ZERO) {
-         c_p->freason = BADARITH;
-         $BIF_ERROR_ARITY_2($Fail, BIF_rem_2, RemOp1, RemOp2);
-     } else if (is_both_small(RemOp1, RemOp2)) {
-         $Dst = make_small(signed_val(RemOp1) % signed_val(RemOp2));
-         $NEXT0();
-     } else {
-         $OUTLINED_ARITH_2($Fail, $Live, int_rem, BIF_rem_2, RemOp1, RemOp2, $Dst);
-     }
+    if (ERTS_UNLIKELY(RemOp2 == SMALL_ZERO)) {
+        c_p->freason = BADARITH;
+        $BIF_ERROR_ARITY_2($Fail, BIF_rem_2, RemOp1, RemOp2);
+    } else if (ERTS_LIKELY(is_both_small(RemOp1, RemOp2))) {
+        $Dst = make_small(signed_val(RemOp1) % signed_val(RemOp2));
+        $NEXT0();
+    } else {
+        $OUTLINED_ARITH_2($Fail, $Live, int_rem, BIF_rem_2, RemOp1, RemOp2, $Dst);
+    }
 }
 
 i_band := band.fetch.execute;
@@ -185,7 +185,7 @@ band.fetch(Src1, Src2) {
 }
 
 band.execute(Fail, Live, Dst) {
-    if (is_both_small(BandOp1, BandOp2)) {
+    if (ERTS_LIKELY(is_both_small(BandOp1, BandOp2))) {
         /*
          * No need to untag -- TAG & TAG == TAG.
          */
@@ -196,7 +196,7 @@ band.execute(Fail, Live, Dst) {
 }
 
 i_bor(Fail, Live, Src1, Src2, Dst) {
-    if (is_both_small($Src1, $Src2)) {
+    if (ERTS_LIKELY(is_both_small($Src1, $Src2))) {
         /*
          * No need to untag -- TAG | TAG == TAG.
          */
@@ -207,7 +207,7 @@ i_bor(Fail, Live, Src1, Src2, Dst) {
 }
 
 i_bxor(Fail, Live, Src1, Src2, Dst) {
-    if (is_both_small($Src1, $Src2)) {
+    if (ERTS_LIKELY(is_both_small($Src1, $Src2))) {
         /*
          * TAG ^ TAG == 0.
          *
@@ -232,7 +232,7 @@ shift.setup_bsr(Src1, Src2) {
     Op1 = $Src1;
     Op2 = $Src2;
     shift_left_count = 0;
-    if (is_small(Op2)) {
+    if (ERTS_LIKELY(is_small(Op2))) {
         shift_left_count = -signed_val(Op2);
     } else if (is_big(Op2)) {
         /*
@@ -248,7 +248,7 @@ shift.setup_bsl(Src1, Src2) {
     Op1 = $Src1;
     Op2 = $Src2;
     shift_left_count = 0;
-    if (is_small(Op2)) {
+    if (ERTS_LIKELY(is_small(Op2))) {
         shift_left_count = signed_val(Op2);
     } else if (is_big(Op2)) {
         if (bignum_header_is_neg(*big_val(Op2))) {
@@ -271,10 +271,10 @@ shift.setup_bsl(Src1, Src2) {
 shift.execute(Fail, Live, Dst) {
     Uint big_words_needed;
 
-    if (is_small(Op1)) {
+    if (ERTS_LIKELY(is_small(Op1))) {
         Sint int_res = signed_val(Op1);
-        if (shift_left_count == 0 || int_res == 0) {
-            if (is_not_integer(Op2)) {
+        if (ERTS_UNLIKELY(shift_left_count == 0 || int_res == 0)) {
+            if (ERTS_UNLIKELY(is_not_integer(Op2))) {
                 goto shift_error;
             }
             if (int_res == 0) {
@@ -343,7 +343,7 @@ shift.execute(Fail, Live, Dst) {
                 HTOP += bignum_header_arity(*HTOP) + 1;
             }
             HEAP_SPACE_VERIFIED(0);
-            if (is_nil(Op1)) {
+            if (ERTS_UNLIKELY(is_nil(Op1))) {
                 /*
                  * This result must have been only slighty larger
                  * than allowed since it wasn't caught by the
@@ -381,7 +381,7 @@ shift.execute(Fail, Live, Dst) {
 
 i_int_bnot(Fail, Src, Live, Dst) {
     Eterm bnot_val = $Src;
-    if (is_small(bnot_val)) {
+    if (ERTS_LIKELY(is_small(bnot_val))) {
         bnot_val = make_small(~signed_val(bnot_val));
     } else {
         Uint live = $Live;
@@ -390,7 +390,7 @@ i_int_bnot(Fail, Src, Live, Dst) {
         bnot_val = erts_gc_bnot(c_p, reg, live);
         HEAVY_SWAPIN;
         ERTS_HOLE_CHECK(c_p);
-        if (is_nil(bnot_val)) {
+        if (ERTS_UNLIKELY(is_nil(bnot_val))) {
             $BIF_ERROR_ARITY_1($Fail, BIF_bnot_1, reg[live]);
         }
         $REFRESH_GEN_DEST();
-- 
cgit v1.2.3


From 2d21799e64f79a5bb8d81727a521093472e89fb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 24 Aug 2017 16:00:34 +0200
Subject: Assign machine registers for X86-64

Keep frequently used variables in machine registers.
---
 erts/emulator/beam/beam_emu.c | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 48569a90a9..367da88cb6 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -439,6 +439,12 @@ init_emulator(void)
 #  define REG_stop asm("%l3")
 #  define REG_I asm("%l4")
 #  define REG_fcalls asm("%l5")
+#elif defined(__GNUC__) && defined(__amd64__) && !defined(DEBUG)
+#  define REG_xregs asm("%r12")
+#  define REG_htop
+#  define REG_stop asm("%r13")
+#  define REG_I asm("%rbx")
+#  define REG_fcalls asm("%r14")
 #else
 #  define REG_xregs
 #  define REG_htop
-- 
cgit v1.2.3


From 50da607331bc2de990828c3c74bbf4ee7efa27f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 25 Aug 2017 14:40:50 +0200
Subject: Eliminate three arguments for the apply() helper

We don't need to pass x(0), x(1), and x(2) because they
can already be found in the register array.
---
 erts/emulator/beam/beam_emu.c | 17 +++++++++--------
 erts/emulator/beam/instrs.tab |  2 +-
 2 files changed, 10 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 367da88cb6..8692b99c57 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -392,9 +392,8 @@ static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa,
 				     Eterm* reg, Eterm func) NOINLINE;
 static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity,
 			      BeamInstr *I, Uint offs) NOINLINE;
-static BeamInstr* apply(Process* p, Eterm module, Eterm function,
-			Eterm args, Eterm* reg,
-			BeamInstr *I, Uint offs) NOINLINE;
+static BeamInstr* apply(Process* p, Eterm* reg,
+                        BeamInstr *I, Uint offs) NOINLINE;
 static BeamInstr* call_fun(Process* p, int arity,
 			   Eterm* reg, Eterm args) NOINLINE;
 static BeamInstr* apply_fun(Process* p, Eterm fun,
@@ -2182,13 +2181,14 @@ apply_bif_error_adjustment(Process *p, Export *ep,
 }
 
 static BeamInstr*
-apply(
-Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg,
-BeamInstr *I, Uint stack_offset)
+apply(Process* p, Eterm* reg, BeamInstr *I, Uint stack_offset)
 {
     int arity;
     Export* ep;
     Eterm tmp;
+    Eterm module = reg[0];
+    Eterm function = reg[1];
+    Eterm args = reg[2];
 
     /*
      * Check the arguments which should be of the form apply(Module,
@@ -2305,8 +2305,9 @@ fixed_apply(Process* p, Eterm* reg, Uint arity,
     if (is_not_atom(module)) goto error;
 
     /* Handle apply of apply/3... */
-    if (module == am_erlang && function == am_apply && arity == 3)
-	return apply(p, reg[0], reg[1], reg[2], reg, I, stack_offset);
+    if (module == am_erlang && function == am_apply && arity == 3) {
+	return apply(p, reg, I, stack_offset);
+    }
     
     /*
      * Get the index into the export table, or failing that the export
diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index 58432cbbb4..19219d34bd 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -159,7 +159,7 @@ i_move_call_ext_last(Dest, StackOffset, Src) {
 APPLY(I, Deallocate) {
     //| -no_next
     HEAVY_SWAPOUT;
-    next = apply(c_p, r(0), x(1), x(2), reg, $I, $Deallocate);
+    next = apply(c_p, reg, $I, $Deallocate);
     HEAVY_SWAPIN;
 }
 
-- 
cgit v1.2.3


From e53b75431e40cec0c26032df1867301e76688009 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 25 Aug 2017 14:40:50 +0200
Subject: Eliminate three arguments for erts_hibernate()

We don't need to pass x(0), x(1), and x(2) because they
can already be found in the register array.
---
 erts/emulator/beam/beam_emu.c       | 5 ++++-
 erts/emulator/beam/bif.c            | 6 +++++-
 erts/emulator/beam/global.h         | 2 +-
 erts/emulator/beam/trace_instrs.tab | 2 +-
 4 files changed, 11 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8692b99c57..7ef0772703 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2328,10 +2328,13 @@ fixed_apply(Process* p, Eterm* reg, Uint arity,
 }
 
 int
-erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg)
+erts_hibernate(Process* c_p, Eterm* reg)
 {
     int arity;
     Eterm tmp;
+    Eterm module = reg[0];
+    Eterm function = reg[1];
+    Eterm args = reg[2];
 
     if (is_not_atom(module) || is_not_atom(function)) {
 	/*
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index cbbc2e88a1..ded6c6f1a4 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1250,7 +1250,11 @@ BIF_RETTYPE hibernate_3(BIF_ALIST_3)
      */
     Eterm reg[3];
 
-    if (erts_hibernate(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, reg)) {
+    reg[0] = BIF_ARG_1;
+    reg[1] = BIF_ARG_2;
+    reg[2] = BIF_ARG_3;
+
+    if (erts_hibernate(BIF_P, reg)) {
         /*
          * If hibernate succeeded, TRAP. The process will be wait in a
          * hibernated state if its state is inactive (!ERTS_PSFLG_ACTIVE);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 27a6202bb7..c72ae32f2c 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1386,7 +1386,7 @@ Uint erts_current_reductions(Process* current, Process *p);
 
 int erts_print_system_version(fmtfn_t to, void *arg, Process *c_p);
 
-int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg);
+int erts_hibernate(Process* c_p, Eterm* reg);
 
 ERTS_GLB_FORCE_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr);
 
diff --git a/erts/emulator/beam/trace_instrs.tab b/erts/emulator/beam/trace_instrs.tab
index dfd1d16d58..c71f2ef003 100644
--- a/erts/emulator/beam/trace_instrs.tab
+++ b/erts/emulator/beam/trace_instrs.tab
@@ -102,7 +102,7 @@ i_yield() {
 
 i_hibernate() {
     HEAVY_SWAPOUT;
-    if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) {
+    if (erts_hibernate(c_p, reg)) {
         FCALLS = c_p->fcalls;
         c_p->flags &= ~F_HIBERNATE_SCHED;
         goto do_schedule;
-- 
cgit v1.2.3


From c3b8fa5742b5fbcd262552fa1d6cad0706fa6a94 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 31 Aug 2017 12:48:34 +0200
Subject: Add missing -no_next directives

---
 erts/emulator/beam/instrs.tab     | 1 +
 erts/emulator/beam/msg_instrs.tab | 1 +
 2 files changed, 2 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index 19219d34bd..9c3e615cea 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -906,5 +906,6 @@ i_raise() {
         c_p->freason = PRIMARY_EXCEPTION(s->freason);
     }
     goto find_func_info;
+    //| -no_next
 }
 
diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab
index 509143268b..3b91d8d61f 100644
--- a/erts/emulator/beam/msg_instrs.tab
+++ b/erts/emulator/beam/msg_instrs.tab
@@ -285,6 +285,7 @@ timeout() {
 TIMEOUT_VALUE() {
     c_p->freason = EXC_TIMEOUT_VALUE;
     goto find_func_info;
+    //| -no_next
 }
 
 i_wait_error_locked() {
-- 
cgit v1.2.3


From 84deb2692839ff876581f9ba8d74c5eefd6c3010 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 31 Aug 2017 12:49:07 +0200
Subject: Optimize dispatch of loop_rec from recv_set

We KNOW that recv_set instruction is immediately followed
by a loop_rec instruction.
---
 erts/emulator/beam/msg_instrs.tab | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab
index 3b91d8d61f..093d48c64c 100644
--- a/erts/emulator/beam/msg_instrs.tab
+++ b/erts/emulator/beam/msg_instrs.tab
@@ -66,7 +66,9 @@ i_recv_set() {
     if (c_p->msg.mark == (BeamInstr *) ($NEXT_INSTRUCTION)) {
         c_p->msg.save = c_p->msg.saved_last;
     }
-    /* Fall through to the loop_rec/2 instruction */
+    SET_I($NEXT_INSTRUCTION);
+    goto loop_rec_top__;
+    //| -no_next
 }
 
 i_loop_rec(Dest) {
@@ -79,6 +81,10 @@ i_loop_rec(Dest) {
 
     ErtsMessage* msgp;
 
+    /* Entry point from recv_set */
+ loop_rec_top__:
+    ;
+
     /*
      * We need to disable GC while matching messages
      * in the queue. This since messages with data outside
@@ -87,7 +93,8 @@ i_loop_rec(Dest) {
     ASSERT(!(c_p->flags & F_DELAY_GC));
     c_p->flags |= F_DELAY_GC;
 
-loop_rec__:
+    /* Entry point from loop_rec_end */
+ loop_rec__:
 
     PROCESS_MAIN_CHK_LOCKS(c_p);
 
-- 
cgit v1.2.3


From f388cb3d4c3c442220b75c9d1626601791fea4bc Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 31 Aug 2017 16:43:48 +0200
Subject: erts: Suppress false memory leak for dlerror

---
 erts/emulator/valgrind/suppress.patched.3.6.0 | 7 +++++++
 erts/emulator/valgrind/suppress.standard      | 8 ++++++++
 2 files changed, 15 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/valgrind/suppress.patched.3.6.0 b/erts/emulator/valgrind/suppress.patched.3.6.0
index fcde4a0123..29f2d3d62d 100644
--- a/erts/emulator/valgrind/suppress.patched.3.6.0
+++ b/erts/emulator/valgrind/suppress.patched.3.6.0
@@ -374,3 +374,10 @@ fun:erts_debug_set_internal_state_2
 fun:process_main
 }
 
+{
+Thread specific dlerror buffer. Either bug in libc or valgrind.
+Memcheck:Leak
+...
+fun:_dlerror_run
+...
+}
diff --git a/erts/emulator/valgrind/suppress.standard b/erts/emulator/valgrind/suppress.standard
index bb07c92fc1..99a3ee4048 100644
--- a/erts/emulator/valgrind/suppress.standard
+++ b/erts/emulator/valgrind/suppress.standard
@@ -342,3 +342,11 @@ fun:erts_debug_set_internal_state_2
 fun:process_main
 }
 
+{
+Thread specific dlerror buffer. Either bug in libc or valgrind.
+Memcheck:Leak
+...
+fun:_dlerror_run
+...
+}
+
-- 
cgit v1.2.3


From 1e57f7471d60d60d9c837051d4625a35d779c520 Mon Sep 17 00:00:00 2001
From: Glauber Campinho <glauber.campinho@soundcloud.com>
Date: Tue, 29 Aug 2017 01:22:54 +0200
Subject: Make cp_pos_to_col function aware of the ANSI escape codes

This fixes the issue with the function `move_cursor` described in #1536 and that causes the bug described in https://github.com/elixir-lang/elixir/issues/6504.

The function `cp_pos_to_col` maps the current position in the buffer to the correct column on the screen, but it didn't handle ANSI escape codes. Since the ANSI escape codes aren't visible, the `cp_pos_to_col` now skips them when executing the calculations using `ansi_escape_width`. This new function only considers color escape codes, but also handles invalid codes.
---
 erts/emulator/drivers/unix/ttsl_drv.c | 68 +++++++++++++++++++++++++++--------
 1 file changed, 53 insertions(+), 15 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/unix/ttsl_drv.c b/erts/emulator/drivers/unix/ttsl_drv.c
index f3c1aa1c4a..3d007173c4 100644
--- a/erts/emulator/drivers/unix/ttsl_drv.c
+++ b/erts/emulator/drivers/unix/ttsl_drv.c
@@ -1095,6 +1095,7 @@ static int insert_buf(byte *s, int n)
 		ch = 0;
 	    } while (lpos % 8);
 	} else if (ch == '\e') {
+	    DEBUGLOG(("insert_buf: ANSI Escape: \\e"));
 	    lbuf[lpos++] = (CONTROL_TAG | ((Uint32) ch));
 	} else if (ch == '\n' || ch == '\r') {
 	    write_buf(lbuf + buffpos, lpos - buffpos);
@@ -1156,7 +1157,7 @@ static int write_buf(Uint32 *s, int n)
 		--n; s++;
 	    }
 	} else if (*s == (CONTROL_TAG | ((Uint32) '\e'))) {
-	    outc('\e');
+	    outc(lastput = '\e');
 	    --n;
 	    ++s;
 	} else if (*s & CONTROL_TAG) {
@@ -1251,26 +1252,63 @@ static int move_cursor(int from_pos, int to_pos)
     return to_col-from_col;
 }
 
-static int cp_pos_to_col(int cp_pos)
+/*
+ * Returns the length of an ANSI escape code in a buffer, this function only consider
+ * color escape sequences like `\e[33m` or `\e[21;33m`. If a sequence has no valid
+ * terminator, the length is equal the number of characters between `\e` and the first
+ * invalid character, inclusive.
+ */
+
+static int ansi_escape_width(Uint32 *s, int max_length)
 {
-#ifdef HAVE_WCWIDTH
     int i;
-    int col = 0;
-
-    for (i = 0; i < cp_pos; i++) {
-        int w = wcwidth(lbuf[i]);
-        if (w > 0) {
-            col += w;
-        }
+    
+    if (*s != (CONTROL_TAG | ((Uint32) '\e'))) {
+       return 0;
+    } else if (max_length <= 1) {
+       return 1;
+    } else if (s[1] != '[') {
+       return 2;
     }
-    return col;
-#else
+    
+    for (i = 2; i < max_length && (s[i] == ';' || (s[i] >= '0' && s[i] <= '9')); i++);
+
+    return i + 1;
+}
+
+static int cp_pos_to_col(int cp_pos)
+{
     /*
-     * We dont' have any character width information. Assume that
-     * code points are one column wide.
+     * If we don't have any character width information. Assume that
+     * code points are one column wide
      */
-    return cp_pos;
+    int w = 1;
+    int col = 0;
+    int i = 0;
+    int j;
+
+    if (cp_pos > llen) {
+        col += cp_pos - llen;
+	cp_pos = llen;
+    }
+
+    while (i < cp_pos) {
+       j = ansi_escape_width(lbuf + i, llen - i);
+
+       if (j > 0) {
+           i += j;
+       } else {
+#ifdef HAVE_WCWIDTH
+           w = wcwidth(lbuf[i]);
 #endif
+           if (w > 0) {
+              col += w;
+           }
+           i++;
+       }
+    }
+
+    return col;    
 }
 
 static int start_termcap(void)
-- 
cgit v1.2.3


From cc39ad3445291c15a7f8460f892c115b4125be9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 1 Sep 2017 12:30:58 +0200
Subject: Eliminate unecessary instruction macro i_move_call_only()

---
 erts/emulator/beam/instrs.tab | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index 9c3e615cea..e9d1f6a305 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -110,15 +110,11 @@ i_call_only(CallDest) {
     $DISPATCH($CallDest);
 }
 
-i_move_call_only(CallDest, Src) {
+move_call_only(Src, CallDest) {
     x(0) = $Src;
     $i_call_only($CallDest);
 }
 
-move_call_only(Src, CallDest) {
-    $i_move_call_only($CallDest, $Src);
-}
-
 DISPATCHX(Dest) {
     //| -no_next
     DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, $Dest);
-- 
cgit v1.2.3


From 86298264166f9d1b5e2e4c13bcd4db7c9d8d0ca1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 1 Sep 2017 13:12:51 +0200
Subject: Try to avoid generating unecessary do/while wrappers

Make the generated code easier to read.
---
 erts/emulator/utils/beam_makeops | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index bcc92472d4..38754e67a1 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -1451,7 +1451,32 @@ sub expand_macro {
         warn $@;
         die "... from the body of $name at $where\n";
     }
-    ("do {\n$body\n} while (0)",$rest);
+    $body = "do {\n$body\n} while (0)"
+        if needs_do_wrapper($body);
+    ($body,$rest);
+}
+
+# Conservative heuristic to determine whether a do { ... } while(0)
+# wrapper is needed.
+sub needs_do_wrapper {
+    local $_ = shift;
+
+    s@^//[|][^\n]*\n@@;
+    s@^\s*@@s;
+    s@^/[*].*[*]/\s*@@s;
+    return 1 if /^(Eterm|Uint|Sint|int|unsigned)/; # Definitely needed.
+    return 0 if /^do/;
+    return 0 if /^SET_I/;
+    return 0 if /^SET_CP/;
+    return 0 if /^ERTS_NO_FPE_CHECK_INIT/;
+    return 0 if /^ASSERT/;
+    return 0 if /^DTRACE/;
+    return 0 if /^[A-Za-z_]*\s*=/;
+    return 0 if /^c_p->/;
+    return 0 if /^[A-Z_]*SWAPOUT/;
+    return 0 if /^if\s*[(]/;
+    return 0 if /^goto\b/;
+    return 1;                   # Not sure, say that it is needed.
 }
 
 sub do_pack {
-- 
cgit v1.2.3


From 3b1703009a4c484c02809fadf14163a121d09198 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 1 Sep 2017 15:05:20 +0200
Subject: beam_emu.c: Mark initialization code as unlikely

---
 erts/emulator/beam/beam_emu.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 7ef0772703..3dce16aedd 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -649,7 +649,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
      * Note: c_p->arity must be set to reflect the number of useful terms in
      * c_p->arg_reg before calling the scheduler.
      */
-    if (!init_done) {
+    if (ERTS_UNLIKELY(!init_done)) {
        /* This should only be reached during the init phase when only the main
         * process is running. I.e. there is no race for init_done.
         */
-- 
cgit v1.2.3


From 2f659a35a16610778d36ae3e01744843bee9889f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 1 Sep 2017 15:29:05 +0200
Subject: Annotate try_case_end as cold

try_case_end is an excepting-generating instruction that is
infrequently executed.
---
 erts/emulator/beam/ops.tab | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index d848c1cceb..d64f6f2cfc 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -190,7 +190,9 @@ try Y F => catch Y F
 try_case Y => try_end Y
 try_end y
 
+%cold
 try_case_end s
+%hot
 
 # Destructive set tuple element
 
-- 
cgit v1.2.3


From b30e726197fd3a94079c955879e5db745eda5ecd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 4 Sep 2017 13:04:17 +0200
Subject: Don't allow macros to assign to hard-coded variables

It's bad style. Pass the name of the variable as an
extra argument to the macro.
---
 erts/emulator/beam/instrs.tab | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index e9d1f6a305..b79b960fd7 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -152,10 +152,10 @@ i_move_call_ext_last(Dest, StackOffset, Src) {
     $i_call_ext_last($Dest, $StackOffset);
 }
 
-APPLY(I, Deallocate) {
+APPLY(I, Deallocate, Next) {
     //| -no_next
     HEAVY_SWAPOUT;
-    next = apply(c_p, reg, $I, $Deallocate);
+    $Next = apply(c_p, reg, $I, $Deallocate);
     HEAVY_SWAPIN;
 }
 
@@ -166,7 +166,7 @@ HANDLE_APPLY_ERROR() {
 
 i_apply() {
     BeamInstr *next;
-    $APPLY(NULL, 0);
+    $APPLY(NULL, 0, next);
     if (ERTS_LIKELY(next != NULL)) {
         $i_call(next);
     }
@@ -175,7 +175,7 @@ i_apply() {
 
 i_apply_last(Deallocate) {
     BeamInstr *next;
-    $APPLY(I, $Deallocate);
+    $APPLY(I, $Deallocate, next);
     if (ERTS_LIKELY(next != NULL)) {
         $i_call_last(next, $Deallocate);
     }
@@ -184,23 +184,23 @@ i_apply_last(Deallocate) {
 
 i_apply_only() {
     BeamInstr *next;
-    $APPLY(I, 0);
+    $APPLY(I, 0, next);
     if (ERTS_LIKELY(next != NULL)) {
         $i_call_only(next);
     }
     $HANDLE_APPLY_ERROR();
 }
 
-FIXED_APPLY(Arity, I, Deallocate) {
+FIXED_APPLY(Arity, I, Deallocate, Next) {
     //| -no_next
     HEAVY_SWAPOUT;
-    next = fixed_apply(c_p, reg, $Arity, $I, $Deallocate);
+    $Next = fixed_apply(c_p, reg, $Arity, $I, $Deallocate);
     HEAVY_SWAPIN;
 }
 
 apply(Arity) {
     BeamInstr *next;
-    $FIXED_APPLY($Arity, NULL, 0);
+    $FIXED_APPLY($Arity, NULL, 0, next);
     if (ERTS_LIKELY(next != NULL)) {
         $i_call(next);
     }
@@ -209,16 +209,16 @@ apply(Arity) {
 
 apply_last(Arity, Deallocate) {
     BeamInstr *next;
-    $FIXED_APPLY($Arity, I, $Deallocate);
+    $FIXED_APPLY($Arity, I, $Deallocate, next);
     if (ERTS_LIKELY(next != NULL)) {
         $i_call_last(next, $Deallocate);
     }
     $HANDLE_APPLY_ERROR();
 }
 
-APPLY_FUN() {
+APPLY_FUN(Next) {
     HEAVY_SWAPOUT;
-    next = apply_fun(c_p, r(0), x(1), reg);
+    $Next = apply_fun(c_p, r(0), x(1), reg);
     HEAVY_SWAPIN;
 }
 
@@ -233,7 +233,7 @@ DISPATCH_FUN(I) {
 
 i_apply_fun() {
     BeamInstr *next;
-    $APPLY_FUN();
+    $APPLY_FUN(next);
     if (ERTS_LIKELY(next != NULL)) {
         SET_CP(c_p, $NEXT_INSTRUCTION);
         $DISPATCH_FUN(next);
@@ -243,7 +243,7 @@ i_apply_fun() {
 
 i_apply_fun_last(Deallocate) {
     BeamInstr *next;
-    $APPLY_FUN();
+    $APPLY_FUN(next);
     if (ERTS_LIKELY(next != NULL)) {
         $deallocate($Deallocate);
         $DISPATCH_FUN(next);
@@ -253,23 +253,23 @@ i_apply_fun_last(Deallocate) {
 
 i_apply_fun_only() {
     BeamInstr *next;
-    $APPLY_FUN();
+    $APPLY_FUN(next);
     if (ERTS_LIKELY(next != NULL)) {
         $DISPATCH_FUN(next);
     }
     $HANDLE_APPLY_FUN_ERROR();
 }
 
-CALL_FUN(Fun) {
+CALL_FUN(Fun, Next) {
     //| -no_next
     HEAVY_SWAPOUT;
-    next = call_fun(c_p, $Fun, reg, THE_NON_VALUE);
+    $Next = call_fun(c_p, $Fun, reg, THE_NON_VALUE);
     HEAVY_SWAPIN;
 }
 
 i_call_fun(Fun) {
     BeamInstr *next;
-    $CALL_FUN($Fun);
+    $CALL_FUN($Fun, next);
     if (ERTS_LIKELY(next != NULL)) {
         SET_CP(c_p, $NEXT_INSTRUCTION);
         $DISPATCH_FUN(next);
@@ -279,7 +279,7 @@ i_call_fun(Fun) {
 
 i_call_fun_last(Fun, Deallocate) {
     BeamInstr *next;
-    $CALL_FUN($Fun);
+    $CALL_FUN($Fun, next);
     if (ERTS_LIKELY(next != NULL)) {
         $deallocate($Deallocate);
         $DISPATCH_FUN(next);
-- 
cgit v1.2.3


From 5a45d2917cd14bf5eab690453e8a191549de4c2c Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 1 Sep 2017 17:17:34 +0200
Subject: Bug fixes of statistics(wall_clock) and statistics(runtime)

---
 erts/emulator/beam/erl_bif_info.c       |   5 +-
 erts/emulator/beam/erl_lock_check.c     |   2 +-
 erts/emulator/beam/erl_time.h           |   7 ++
 erts/emulator/beam/erl_time_sup.c       | 145 ++++++++++++++++++++++----------
 erts/emulator/beam/sys.h                |   4 -
 erts/emulator/sys/unix/erl_unix_sys.h   |   4 +
 erts/emulator/test/statistics_SUITE.erl |  34 +++++++-
 7 files changed, 148 insertions(+), 53 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 96f9b284b3..62302ca519 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -48,6 +48,7 @@
 #include "erl_map.h"
 #define ERTS_PTAB_WANT_DEBUG_FUNCS__
 #include "erl_ptab.h"
+#include "erl_time.h"
 #ifdef HIPE
 #include "hipe_arch.h"
 #endif
@@ -3547,7 +3548,7 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
 	ErtsMonotonicTime u1, u2;
 	Eterm b1, b2;
         Uint hsz;
-	elapsed_time_both(&u1, NULL, &u2, NULL);
+	erts_runtime_elapsed_both(&u1, NULL, &u2, NULL);
         hsz = 3; /* 2-tuple */
         (void) erts_bld_monotonic_time(NULL, &hsz, u1);
         (void) erts_bld_monotonic_time(NULL, &hsz, u2);
@@ -3563,7 +3564,7 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
 	ErtsMonotonicTime w1, w2;
 	Eterm b1, b2;
         Uint hsz;
-	wall_clock_elapsed_time_both(&w1, &w2);
+	erts_wall_clock_elapsed_both(&w1, &w2);
         hsz = 3; /* 2-tuple */
         (void) erts_bld_monotonic_time(NULL, &hsz, w1);
         (void) erts_bld_monotonic_time(NULL, &hsz, w2);
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index f270d8baef..378f86f097 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -169,9 +169,9 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"xports_list_pre_alloc_lock",		"address"		},
     {	"inet_buffer_stack_lock",		NULL			},
     {	"system_block",				NULL			},
-    {	"timeofday",				NULL			},
     {	"get_time",				NULL			},
     {	"get_corrected_time",			NULL			},
+    {	"runtime",				NULL			},
     {	"breakpoints",				NULL			},
     {	"pollsets_lock",			NULL			},
     {	"pix_lock",				"address"		},
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index ccc5526664..27164d50a0 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -130,6 +130,13 @@ Eterm erts_get_monotonic_end_time(struct process *c_p);
 Eterm erts_monotonic_time_source(struct process*c_p);
 Eterm erts_system_time_source(struct process*c_p);
 
+void erts_runtime_elapsed_both(ErtsMonotonicTime *ms_user,
+                               ErtsMonotonicTime *ms_sys, 
+                               ErtsMonotonicTime *ms_user_diff,
+                               ErtsMonotonicTime *ms_sys_diff);
+void erts_wall_clock_elapsed_both(ErtsMonotonicTime *total,
+                                  ErtsMonotonicTime *diff);
+
 #ifdef SYS_CLOCK_RESOLUTION
 #define ERTS_CLKTCK_RESOLUTION ((ErtsMonotonicTime) (SYS_CLOCK_RESOLUTION*1000))
 #else
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 0421adb409..c0fd417113 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -36,12 +36,29 @@
 #include "erl_driver.h"
 #include "erl_nif.h"
  
-static erts_smp_mtx_t erts_timeofday_mtx;
 static erts_smp_mtx_t erts_get_time_mtx;
 
-static SysTimes t_start; /* Used in elapsed_time_both */
-static ErtsMonotonicTime prev_wall_clock_elapsed; /* Used in wall_clock_elapsed_time_both */
-static ErtsMonotonicTime previous_now; /* Used in get_now */
+ /* used by erts_runtime_elapsed_both */
+typedef struct {
+    erts_smp_mtx_t mtx;
+    ErtsMonotonicTime user;
+    ErtsMonotonicTime sys;
+} ErtsRunTimePrevData;
+
+static union {
+    ErtsRunTimePrevData data;
+    char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunTimePrevData))];
+} runtime_prev erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+
+static union {
+    erts_smp_atomic64_t time;
+    char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_smp_atomic64_t))];
+} wall_clock_prev erts_align_attribute(ERTS_CACHE_LINE_SIZE);
+
+static union {
+    erts_smp_atomic64_t time;
+    char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_smp_atomic64_t))];
+} now_prev erts_align_attribute(ERTS_CACHE_LINE_SIZE);
 
 static ErtsMonitor *time_offset_monitors = NULL;
 static Uint no_time_offset_monitors = 0;
@@ -954,8 +971,10 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 
     ASSERT(ERTS_MONOTONIC_TIME_MIN < ERTS_MONOTONIC_TIME_MAX);
 
-    erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday");
     erts_smp_mtx_init(&erts_get_time_mtx, "get_time");
+    erts_smp_mtx_init(&runtime_prev.data.mtx, "runtime");
+    runtime_prev.data.user = 0;
+    runtime_prev.data.sys = 0;
 
     time_sup.r.o.correction = time_correction;
     time_sup.r.o.warp_mode = time_warp_mode;
@@ -1154,9 +1173,13 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode)
 	time_sup.f.c.last_not_corrected_time = 0;
     }
 
-    prev_wall_clock_elapsed = 0;
+    erts_smp_atomic64_init_nob(&wall_clock_prev.time,
+                               (erts_aint64_t) 0);
+
+    erts_smp_atomic64_init_nob(
+        &now_prev.time,
+        (erts_aint64_t) ERTS_MONOTONIC_TO_USEC(get_time_offset()));
 
-    previous_now = ERTS_MONOTONIC_TO_USEC(get_time_offset());
 
 #ifdef DEBUG
     time_sup_initialized = 1;
@@ -1286,36 +1309,65 @@ erts_finalize_time_offset(void)
 /* info functions */
 
 void 
-elapsed_time_both(ErtsMonotonicTime *ms_user, ErtsMonotonicTime *ms_sys, 
-		  ErtsMonotonicTime *ms_user_diff, ErtsMonotonicTime *ms_sys_diff)
+erts_runtime_elapsed_both(ErtsMonotonicTime *ms_user, ErtsMonotonicTime *ms_sys, 
+                          ErtsMonotonicTime *ms_user_diff, ErtsMonotonicTime *ms_sys_diff)
 {
-    ErtsMonotonicTime prev_total_user, prev_total_sys;
-    ErtsMonotonicTime total_user, total_sys;
+    ErtsMonotonicTime prev_user, prev_sys, user, sys;
+
+#ifdef HAVE_GETRUSAGE
+
+    struct rusage now;
+
+    if (getrusage(RUSAGE_SELF, &now) != 0) {
+        erts_exit(ERTS_ABORT_EXIT, "getrusage(RUSAGE_SELF, _) failed: %d\n", errno);
+	return;
+    }
+
+    user = (ErtsMonotonicTime) now.ru_utime.tv_sec;
+    user *= (ErtsMonotonicTime) 1000000;
+    user += (ErtsMonotonicTime) now.ru_utime.tv_usec;
+    user /= (ErtsMonotonicTime) 1000;
+
+    sys = (ErtsMonotonicTime) now.ru_stime.tv_sec;
+    sys *= (ErtsMonotonicTime) 1000000;
+    sys += (ErtsMonotonicTime) now.ru_stime.tv_usec;
+    sys /= (ErtsMonotonicTime) 1000;
+
+#else
+
     SysTimes now;
 
     sys_times(&now);
-    total_user = (ErtsMonotonicTime) ((now.tms_utime * 1000) / SYS_CLK_TCK);
-    total_sys = (ErtsMonotonicTime) ((now.tms_stime * 1000) / SYS_CLK_TCK);
+    user = (ErtsMonotonicTime) now.tms_utime;
+    user *= (ErtsMonotonicTime) 1000;
+    user /= (ErtsMonotonicTime) SYS_CLK_TCK;
 
-    if (ms_user != NULL)
-	*ms_user = total_user;
-    if (ms_sys != NULL)
-	*ms_sys = total_sys;
+    sys = (ErtsMonotonicTime) now.tms_stime;
+    sys *= (ErtsMonotonicTime) 1000;
+    sys /= (ErtsMonotonicTime) SYS_CLK_TCK;
+
+#endif
+
+    if (ms_user)
+	*ms_user = user;
+    if (ms_sys)
+	*ms_sys = sys;
 
     if (ms_user_diff || ms_sys_diff) {
-        erts_smp_mtx_lock(&erts_timeofday_mtx);
-    
-        prev_total_user = (ErtsMonotonicTime) ((t_start.tms_utime * 1000) / SYS_CLK_TCK);
-        prev_total_sys = (ErtsMonotonicTime) ((t_start.tms_stime * 1000) / SYS_CLK_TCK);
-        t_start = now;
-    
-        erts_smp_mtx_unlock(&erts_timeofday_mtx);
+ 
+        erts_smp_mtx_lock(&runtime_prev.data.mtx);
+
+        prev_user = runtime_prev.data.user;
+        prev_sys = runtime_prev.data.sys;
+        runtime_prev.data.user = user;
+        runtime_prev.data.sys = sys;
 
-        if (ms_user_diff != NULL)
-            *ms_user_diff = total_user - prev_total_user;
-	  
-        if (ms_sys_diff != NULL)
-            *ms_sys_diff = total_sys - prev_total_sys;
+        erts_smp_mtx_unlock(&runtime_prev.data.mtx);
+
+        if (ms_user_diff)
+            *ms_user_diff = user - prev_user;
+        if (ms_sys_diff)
+            *ms_sys_diff = sys - prev_sys;
     }
 }
 
@@ -1323,7 +1375,7 @@ elapsed_time_both(ErtsMonotonicTime *ms_user, ErtsMonotonicTime *ms_sys,
 /* wall clock routines */
 
 void 
-wall_clock_elapsed_time_both(ErtsMonotonicTime *ms_total, ErtsMonotonicTime *ms_diff)
+erts_wall_clock_elapsed_both(ErtsMonotonicTime *ms_total, ErtsMonotonicTime *ms_diff)
 {
     ErtsMonotonicTime now, elapsed;
 
@@ -1331,16 +1383,18 @@ wall_clock_elapsed_time_both(ErtsMonotonicTime *ms_total, ErtsMonotonicTime *ms_
     update_last_mtime(NULL, now);
 
     elapsed = ERTS_MONOTONIC_TO_MSEC(now);
+    elapsed -= ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_BEGIN);
 
     *ms_total = elapsed;
 
     if (ms_diff) {
-        erts_smp_mtx_lock(&erts_timeofday_mtx);
+        ErtsMonotonicTime prev;
 
-        *ms_diff = elapsed - prev_wall_clock_elapsed;
-        prev_wall_clock_elapsed = elapsed;
+        prev = ((ErtsMonotonicTime)
+                erts_smp_atomic64_xchg_mb(&wall_clock_prev.time,
+                                          (erts_aint64_t) elapsed));
 
-        erts_smp_mtx_unlock(&erts_timeofday_mtx);
+        *ms_diff = elapsed - prev;
     }
 }
 
@@ -1719,22 +1773,27 @@ univ_to_local(Sint *year, Sint *month, Sint *day,
 void
 get_now(Uint* megasec, Uint* sec, Uint* microsec)
 {
-    ErtsMonotonicTime now_megasec, now_sec, now, mtime, time_offset;
+    ErtsMonotonicTime now_megasec, now_sec, now, prev, mtime, time_offset;
     
     mtime = time_sup.r.o.get_time();
     time_offset = get_time_offset();
     update_last_mtime(NULL, mtime);
     now = ERTS_MONOTONIC_TO_USEC(mtime + time_offset);
 
-    erts_smp_mtx_lock(&erts_timeofday_mtx);
-
     /* Make sure now time is later than last time */
-    if (now <= previous_now)
-	now = previous_now + 1;
-
-    previous_now = now;
-    
-    erts_smp_mtx_unlock(&erts_timeofday_mtx);
+    prev = erts_smp_atomic64_read_nob(&now_prev.time);
+    while (1) {
+        ErtsMonotonicTime act;
+        if (now <= prev)
+            now = prev + 1;
+        act = ((ErtsMonotonicTime)
+               erts_smp_atomic64_cmpxchg_mb(&now_prev.time,
+                                            (erts_aint64_t) now,
+                                            (erts_aint64_t) prev));
+        if (act == prev)
+            break;
+        prev = act;
+    }
 
     now_megasec = now / ERTS_MONOTONIC_TIME_TERA;
     now_sec = now / ERTS_MONOTONIC_TIME_MEGA;
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index b6c77794d2..6662685f54 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -784,10 +784,6 @@ Preload* sys_preloaded(void);
 unsigned char* sys_preload_begin(Preload*);
 void sys_preload_end(Preload*);
 int sys_get_key(int);
-void elapsed_time_both(ErtsMonotonicTime *ms_user, ErtsMonotonicTime *ms_sys, 
-		       ErtsMonotonicTime *ms_user_diff, ErtsMonotonicTime *ms_sys_diff);
-void wall_clock_elapsed_time_both(ErtsMonotonicTime *ms_total, 
-				  ErtsMonotonicTime *ms_diff);
 void get_time(int *hour, int *minute, int *second);
 void get_date(int *year, int *month, int *day);
 void get_localtime(int *year, int *month, int *day, 
diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index 22059d21d5..b83837a7d2 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -86,6 +86,10 @@
 
 #include <sys/times.h>
 
+#ifdef HAVE_SYS_RESOURCE_H
+#  include <sys/resource.h>
+#endif
+
 #ifdef HAVE_IEEEFP_H
 #include <ieeefp.h>
 #endif
diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl
index 7690557fda..40cc940a94 100644
--- a/erts/emulator/test/statistics_SUITE.erl
+++ b/erts/emulator/test/statistics_SUITE.erl
@@ -23,8 +23,10 @@
 %% Tests the statistics/1 bif.
 
 -export([all/0, suite/0, groups/0,
+         wall_clock_sanity/1,
 	 wall_clock_zero_diff/1, wall_clock_update/1,
-	 runtime_zero_diff/1,
+         runtime_sanity/1,
+         runtime_zero_diff/1,
 	 runtime_update/1, runtime_diff/1,
 	 run_queue_one/1,
 	 scheduler_wall_time/1,
@@ -54,11 +56,23 @@ all() ->
 
 groups() -> 
     [{wall_clock, [],
-      [wall_clock_zero_diff, wall_clock_update]},
+      [wall_clock_sanity, wall_clock_zero_diff, wall_clock_update]},
      {runtime, [],
-      [runtime_zero_diff, runtime_update, runtime_diff]},
+      [runtime_sanity, runtime_zero_diff, runtime_update, runtime_diff]},
      {run_queue, [], [run_queue_one]}].
 
+wall_clock_sanity(Config) when is_list(Config) ->
+    erlang:yield(),
+    {WallClock, _} = statistics(wall_clock),
+    MT = erlang:monotonic_time(),
+    Time = erlang:convert_time_unit(MT - erlang:system_info(start_time),
+                                    native, millisecond),
+    io:format("Time=~p WallClock=~p~n",
+              [Time, WallClock]),
+    true = WallClock =< Time,
+    true = Time - 100 =< WallClock,
+    ok.
+
 %%% Testing statistics(wall_clock).
 
 %% Tests that the 'Wall clock since last call' element of the result
@@ -102,6 +116,20 @@ wall_clock_update1(0) ->
 
 %%% Test statistics(runtime).
 
+runtime_sanity(Config) when is_list(Config) ->
+    case erlang:system_info(logical_processors_available) of
+        unknown ->
+            {skipped, "Don't know available logical processors"};
+        LP when is_integer(LP) ->
+            erlang:yield(),
+            {RunTime, _} = statistics(runtime),
+            MT = erlang:monotonic_time(),
+            Time = erlang:convert_time_unit(MT - erlang:system_info(start_time),
+                                            native, millisecond),
+            io:format("Time=~p RunTime=~p~n",
+                      [Time, RunTime]),
+            true = RunTime =< Time*LP
+    end.
 
 %% Tests that the difference between the times returned from two consectuitive
 %% calls to statistics(runtime) is zero.
-- 
cgit v1.2.3


From b8870376d42b3f629d2dac394925e02f0e6e48ca Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 4 Sep 2017 17:49:38 +0200
Subject: Fix setnode/3 and erts_net_message()

---
 erts/emulator/beam/dist.c | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index a50072fe98..0e4361308f 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1230,8 +1230,7 @@ int erts_net_message(Port *prt,
     }
 
 
-    if (hlen != 0)
-	goto data_error;
+    ASSERT(hlen == 0);
 
     if (len == 0) {  /* HANDLE TICK !!! */
 	UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
@@ -1251,11 +1250,6 @@ int erts_net_message(Port *prt,
 	len--;
     }
 
-    if (len == 0) {
-	PURIFY_MSG("data error");
-	goto data_error;
-    }
-
     res = erts_prepare_dist_ext(&ede, t, len, dep, dep->cache, &connection_id);
 
     switch (res) {
@@ -3303,8 +3297,10 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
         if (ERTS_PROC_GET_DIST_ENTRY(proc)) {
             if (dep == ERTS_PROC_GET_DIST_ENTRY(proc)
                 && (proc->flags & F_DISTRIBUTION)
-                && dep->cid == BIF_ARG_2)
+                && dep->cid == BIF_ARG_2) {
+                ERTS_BIF_PREP_RET(ret, erts_make_dhandle(BIF_P, dep));
                 goto done;
+            }
             goto badarg;
         }
 
@@ -3335,8 +3331,10 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
         if ((pp->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY) == 0)
             goto badarg;
 
-        if (dep->cid == BIF_ARG_2 && pp->dist_entry == dep)
+        if (dep->cid == BIF_ARG_2 && pp->dist_entry == dep) {
+            ERTS_BIF_PREP_RET(ret, erts_make_dhandle(BIF_P, dep));
             goto done; /* Already set */
+        }
 
         if (dep->status & ERTS_DE_SFLG_EXITING) {
             /* Suspend on dist entry waiting for the exit to finish */
-- 
cgit v1.2.3


From 98bd5f66a03b945916baa2a91c4aba3c837676d1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 4 Sep 2017 20:53:00 +0200
Subject: erts: Fix faulty ASSERT of table fixation counter

Cannot read fix->counter safely without table lock.
---
 erts/emulator/beam/erl_db.c      | 11 +----------
 erts/emulator/beam/erl_db_util.h |  3 +++
 2 files changed, 4 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 17e0f2aeec..e84171649c 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -3607,14 +3607,8 @@ static SWord proc_cleanup_fixed_table(Process* p, DbFixation* fix)
 	    ASSERT(sizeof(DbFixation) == ERTS_ALC_DBG_BLK_SZ(fix));
 	    ERTS_DB_ALC_MEM_UPDATE_(tb, sizeof(DbFixation), 0);
 	}
-	else {
-	    ASSERT(fix->counter == 0);
-	}
 	db_unlock(tb, LCK_WRITE_REC);
     }
-    else {
-	ASSERT(fix->counter == 0);
-    }
 
     erts_bin_release(fix->tabs.btid);
     erts_free(ERTS_ALC_T_DB_FIXATION, fix);
@@ -3856,11 +3850,8 @@ static void free_fixations_op(DbFixation* fix, void* vctx)
 {
     struct free_fixations_ctx* ctx = (struct free_fixations_ctx*) vctx;
     erts_aint_t diff;
-#ifdef DEBUG
-    DbTable* dbg_tb = btid2tab(fix->tabs.btid);
-#endif
 
-    ASSERT(!dbg_tb || dbg_tb == ctx->tb);
+    ASSERT(!btid2tab(fix->tabs.btid));
     ASSERT(fix->counter > 0);
     ASSERT(ctx->tb->common.status & DB_DELETE);
 
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 19055c6110..7ce104a84c 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -220,6 +220,9 @@ typedef struct db_fixation {
         Process* p;
     } procs;
 
+    /* Number of fixations on table from procs.p
+     * Protected by table write lock or read lock + fixlock
+     */
     Uint counter;
 } DbFixation;
 
-- 
cgit v1.2.3


From 314702f86c9199957e40edfb73bcdbddb422f9d7 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 24 Feb 2017 19:38:37 +0100
Subject: erts: Add nif ioq

---
 erts/emulator/Makefile.in                     |   3 +-
 erts/emulator/beam/erl_driver.h               |  19 -
 erts/emulator/beam/erl_drv_nif.h              |  25 +-
 erts/emulator/beam/erl_io_queue.c             | 738 ++++++++++++++++++++++
 erts/emulator/beam/erl_io_queue.h             | 201 ++++++
 erts/emulator/beam/erl_nif.c                  | 365 +++++++++++
 erts/emulator/beam/erl_nif.h                  |  23 +
 erts/emulator/beam/erl_nif_api_funcs.h        |  25 +
 erts/emulator/beam/erl_port.h                 |  22 +-
 erts/emulator/beam/io.c                       | 844 ++++----------------------
 erts/emulator/beam/utils.c                    |   1 +
 erts/emulator/test/nif_SUITE.erl              | 184 +++++-
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 241 +++++++-
 13 files changed, 1919 insertions(+), 772 deletions(-)
 create mode 100644 erts/emulator/beam/erl_io_queue.c
 create mode 100644 erts/emulator/beam/erl_io_queue.h

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 1ae905276d..cc1fa571df 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -815,7 +815,8 @@ RUN_OBJS = \
 	$(OBJDIR)/erl_bif_binary.o      $(OBJDIR)/erl_ao_firstfit_alloc.o \
 	$(OBJDIR)/erl_thr_queue.o	$(OBJDIR)/erl_sched_spec_pre_alloc.o \
 	$(OBJDIR)/erl_ptab.o		$(OBJDIR)/erl_map.o \
-	$(OBJDIR)/erl_msacc.o		$(OBJDIR)/erl_lock_flags.o
+	$(OBJDIR)/erl_msacc.o		$(OBJDIR)/erl_lock_flags.o \
+	$(OBJDIR)/erl_io_queue.o
 
 LTTNG_OBJS = $(OBJDIR)/erlang_lttng.o
 NIF_OBJS = $(OBJDIR)/erl_tracer_nif.o
diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 0e8ebf0c98..5ad616fec3 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -40,7 +40,6 @@
 #include "erl_drv_nif.h"
 
 #include <stdlib.h>
-#include <sys/types.h>	/* ssize_t */
 
 #if defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)
 #ifndef STATIC_ERLANG_DRIVER
@@ -48,24 +47,6 @@
 #define ERL_DRIVER_TYPES_ONLY
 #define WIN32_DYNAMIC_ERL_DRIVER
 #endif
-/*
- * This structure can be cast to a WSABUF structure.
- */
-typedef struct _SysIOVec {
-    unsigned long iov_len;
-    char* iov_base;
-} SysIOVec;
-#else  /* Unix */
-#  ifdef HAVE_SYS_UIO_H
-#    include <sys/types.h>
-#    include <sys/uio.h>
-typedef struct iovec SysIOVec;
-#  else
-typedef struct {
-    char* iov_base;
-    size_t iov_len;
-} SysIOVec;
-#  endif
 #endif
 
 #ifndef EXTERN
diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h
index f88138063e..31b4817fb1 100644
--- a/erts/emulator/beam/erl_drv_nif.h
+++ b/erts/emulator/beam/erl_drv_nif.h
@@ -144,8 +144,25 @@ typedef signed int ErlNapiSInt;
 #define ERTS_NAPI_USEC__	2
 #define ERTS_NAPI_NSEC__	3
 
-#endif  /* __ERL_DRV_NIF_H__ */
-
-
-
+#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
+/*
+ * This structure can be cast to a WSABUF structure.
+ */
+typedef struct _SysIOVec {
+    unsigned long iov_len;
+    char* iov_base;
+} SysIOVec;
+#else  /* Unix */
+#  include <sys/types.h>
+#  ifdef HAVE_SYS_UIO_H
+#    include <sys/uio.h>
+typedef struct iovec SysIOVec;
+#  else
+typedef struct {
+    char* iov_base;
+    size_t iov_len;
+} SysIOVec;
+#  endif
+#endif
 
+#endif  /* __ERL_DRV_NIF_H__ */
diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c
new file mode 100644
index 0000000000..d072376d1c
--- /dev/null
+++ b/erts/emulator/beam/erl_io_queue.c
@@ -0,0 +1,738 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2017. All Rights Reserved.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "sys.h"
+#include "global.h"
+#include "erl_bits.h"
+#include "erl_io_queue.h"
+
+static void free_binary(ErtsIOQBinary *b, int driver);
+static ErtsIOQBinary *alloc_binary(Uint size, char *source, void **iov_base, int driver);
+
+void erts_ioq_init(ErtsIOQueue *q, ErtsAlcType_t alct, int driver)
+{
+
+    ERTS_CT_ASSERT(offsetof(ErlNifIOVec,flags) == sizeof(ErtsIOVecCommon));
+    ERTS_CT_ASSERT(sizeof(ErlIOVec) == sizeof(ErtsIOVecCommon));
+    ERTS_CT_ASSERT(sizeof(size_t) == sizeof(ErlDrvSizeT));
+    ERTS_CT_ASSERT(sizeof(size_t) == sizeof(Uint));
+
+    q->alct = alct;
+    q->driver = driver;
+    q->size = 0;
+    q->v_head = q->v_tail = q->v_start = q->v_small;
+    q->v_end = q->v_small + ERTS_SMALL_IO_QUEUE;
+    q->b_head = q->b_tail = q->b_start = q->b_small;
+    q->b_end = q->b_small + ERTS_SMALL_IO_QUEUE;
+}
+
+void erts_ioq_clear(ErtsIOQueue *q)
+{
+    ErtsIOQBinary** binp = q->b_head;
+    int driver = q->driver;
+
+    if (q->v_start != q->v_small)
+	erts_free(q->alct, (void *) q->v_start);
+
+    while(binp < q->b_tail) {
+	if (*binp != NULL)
+            free_binary(*binp, driver);
+	binp++;
+    }
+    if (q->b_start != q->b_small)
+	erts_free(q->alct, (void *) q->b_start);
+    q->v_start = q->v_end = q->v_head = q->v_tail = NULL;
+    q->b_start = q->b_end = q->b_head = q->b_tail = NULL;
+    q->size = 0;
+}
+
+static void free_binary(ErtsIOQBinary *b, int driver)
+{
+    if (driver)
+        driver_free_binary(&b->driver);
+    else if (erts_refc_dectest(&b->nif.intern.refc, 0) == 0)
+        erts_bin_free(&b->nif);
+}
+
+static ErtsIOQBinary *alloc_binary(Uint size, char *source, void **iov_base, int driver)
+{
+    if (driver) {
+        ErlDrvBinary *bin = driver_alloc_binary(size);
+        if (!bin) return NULL;
+        sys_memcpy(bin->orig_bytes, source, size);
+        *iov_base = bin->orig_bytes;
+        return (ErtsIOQBinary *)bin;
+    } else {
+        /* This clause can be triggered in enif_ioq_enq_binary is used */
+        Binary *bin = erts_bin_nrml_alloc(size);
+        if (!bin) return NULL;
+        erts_refc_init(&bin->intern.refc, 1);
+        sys_memcpy(bin->orig_bytes, source, size);
+        *iov_base = bin->orig_bytes;
+        return (ErtsIOQBinary *)bin;
+    }
+}
+
+Uint erts_ioq_size(ErtsIOQueue *q)
+{
+    return q->size;
+}
+
+/* expand queue to hold n elements in tail or head */
+static int expandq(ErtsIOQueue* q, int n, int tail)
+/* tail: 0 if make room in head, make room in tail otherwise */
+{
+    int h_sz;  /* room before header */
+    int t_sz;  /* room after tail */
+    int q_sz;  /* occupied */
+    int nvsz;
+    SysIOVec* niov;
+    ErtsIOQBinary** nbinv;
+
+    h_sz = q->v_head - q->v_start;
+    t_sz = q->v_end -  q->v_tail;
+    q_sz = q->v_tail - q->v_head;
+
+    if (tail && (n <= t_sz)) /* do we need to expand tail? */
+	return 0;
+    else if (!tail && (n <= h_sz))  /* do we need to expand head? */
+	return 0;
+    else if (n > (h_sz + t_sz)) { /* need to allocate */
+	/* we may get little extra but it ok */
+	nvsz = (q->v_end - q->v_start) + n;
+
+	niov = erts_alloc_fnf(q->alct, nvsz * sizeof(SysIOVec));
+	if (!niov)
+	    return -1;
+	nbinv = erts_alloc_fnf(q->alct, nvsz * sizeof(ErtsIOQBinary**));
+	if (!nbinv) {
+	    erts_free(q->alct, (void *) niov);
+	    return -1;
+	}
+	if (tail) {
+	    sys_memcpy(niov, q->v_head, q_sz*sizeof(SysIOVec));
+	    if (q->v_start != q->v_small)
+		erts_free(q->alct, (void *) q->v_start);
+	    q->v_start = niov;
+	    q->v_end = niov + nvsz;
+	    q->v_head = q->v_start;
+	    q->v_tail = q->v_head + q_sz;
+
+	    sys_memcpy(nbinv, q->b_head, q_sz*sizeof(ErtsIOQBinary*));
+	    if (q->b_start != q->b_small)
+		erts_free(q->alct, (void *) q->b_start);
+	    q->b_start = nbinv;
+	    q->b_end = nbinv + nvsz;
+	    q->b_head = q->b_start;
+	    q->b_tail = q->b_head + q_sz;
+	}
+	else {
+	    sys_memcpy(niov+nvsz-q_sz, q->v_head, q_sz*sizeof(SysIOVec));
+	    if (q->v_start != q->v_small)
+		erts_free(q->alct, (void *) q->v_start);
+	    q->v_start = niov;
+	    q->v_end = niov + nvsz;
+	    q->v_tail = q->v_end;
+	    q->v_head = q->v_tail - q_sz;
+
+	    sys_memcpy(nbinv+nvsz-q_sz, q->b_head, q_sz*sizeof(ErtsIOQBinary*));
+	    if (q->b_start != q->b_small)
+		erts_free(q->alct, (void *) q->b_start);
+	    q->b_start = nbinv;
+	    q->b_end = nbinv + nvsz;
+	    q->b_tail = q->b_end;
+	    q->b_head = q->b_tail - q_sz;
+	}
+    }
+    else if (tail) {  /* move to beginning to make room in tail */
+	sys_memmove(q->v_start, q->v_head, q_sz*sizeof(SysIOVec));
+	q->v_head = q->v_start;
+	q->v_tail = q->v_head + q_sz;
+	sys_memmove(q->b_start, q->b_head, q_sz*sizeof(ErtsIOQBinary*));
+	q->b_head = q->b_start;
+	q->b_tail = q->b_head + q_sz;
+    }
+    else {   /* move to end to make room */
+	sys_memmove(q->v_end-q_sz, q->v_head, q_sz*sizeof(SysIOVec));
+	q->v_tail = q->v_end;
+	q->v_head = q->v_tail-q_sz;
+	sys_memmove(q->b_end-q_sz, q->b_head, q_sz*sizeof(ErtsIOQBinary*));
+	q->b_tail = q->b_end;
+	q->b_head = q->b_tail-q_sz;
+    }
+
+    return 0;
+}
+
+static
+int skip(ErtsIOVec* vec, Uint skipbytes,
+         SysIOVec **iovp, ErtsIOQBinary ***binvp,
+         Uint *lenp)
+{
+    int n;
+    Uint len;
+    SysIOVec* iov;
+    ErtsIOQBinary** binv;
+
+    if (vec->common.size <= skipbytes)
+	return -1;
+
+    iov = vec->common.iov;
+    binv = vec->common.binv;
+    n = vec->common.vsize;
+    /* we use do here to strip iov_len=0 from beginning */
+    do {
+	len = iov->iov_len;
+	if (len <= skipbytes) {
+	    skipbytes -= len;
+	    iov++;
+	    binv++;
+	    n--;
+	}
+	else {
+	    iov->iov_base = ((char *)(iov->iov_base)) + skipbytes;
+	    iov->iov_len -= skipbytes;
+	    skipbytes = 0;
+	}
+    } while(skipbytes > 0);
+
+    *binvp = binv;
+    *iovp = iov;
+    *lenp = len;
+
+    return n;
+}
+
+/* Put elements from vec at q tail */
+int erts_ioq_enqv(ErtsIOQueue *q, ErtsIOVec *eiov, Uint skipbytes)
+{
+    int n;
+    Uint len;
+    Uint size = eiov->common.size - skipbytes;
+    SysIOVec *iov;
+    ErtsIOQBinary** binv;
+    ErtsIOQBinary*  b;
+
+    if (q == NULL)
+	return -1;
+
+    ASSERT(eiov->common.size >= skipbytes);
+    if (eiov->common.size <= skipbytes)
+	return 0;
+
+    n = skip(eiov, skipbytes, &iov, &binv, &len);
+
+    if (n < 0)
+        return n;
+
+    if (q->v_tail + n >= q->v_end)
+	if (expandq(q, n, 1))
+            return -1;
+
+    /* Queue and reference all binaries (remove zero length items) */
+    while(n--) {
+	if ((len = iov->iov_len) > 0) {
+	    if ((b = *binv) == NULL) { /* special case create binary ! */
+		b = alloc_binary(len, iov->iov_base, (void**)&q->v_tail->iov_base,
+                                 q->driver);
+                if (!b) return -1;
+		*q->b_tail++ = b;
+		q->v_tail->iov_len = len;
+		q->v_tail++;
+	    }
+	    else {
+                if (q->driver)
+                    driver_binary_inc_refc(&b->driver);
+                else
+                    erts_refc_inc(&b->nif.intern.refc, 1);
+		*q->b_tail++ = b;
+		*q->v_tail++ = *iov;
+	    }
+	}
+	iov++;
+	binv++;
+    }
+    q->size += size;      /* update total size in queue */
+    return 0;
+}
+
+/* Put elements from vec at q head */
+int erts_ioq_pushqv(ErtsIOQueue *q, ErtsIOVec* vec, Uint skipbytes)
+{
+    int n;
+    Uint len;
+    Uint size = vec->common.size - skipbytes;
+    SysIOVec* iov;
+    ErtsIOQBinary** binv;
+    ErtsIOQBinary* b;
+
+    if (q == NULL)
+	return -1;
+
+    ASSERT(vec->common.size >= skipbytes);
+    if (vec->common.size <= skipbytes)
+	return 0;
+
+    n = skip(vec, skipbytes, &iov, &binv, &len);
+
+    if (n < 0)
+        return n;
+
+    if (q->v_head - n < q->v_start)
+	if (expandq(q, n, 0))
+            return -1;
+
+    /* Queue and reference all binaries (remove zero length items) */
+    iov += (n-1);  /* move to end */
+    binv += (n-1); /* move to end */
+    while(n--) {
+	if ((len = iov->iov_len) > 0) {
+	    if ((b = *binv) == NULL) { /* special case create binary ! */
+                if (q->driver) {
+                    ErlDrvBinary *bin = driver_alloc_binary(len);
+                    if (!bin) return -1;
+                    sys_memcpy(bin->orig_bytes, iov->iov_base, len);
+                    b = (ErtsIOQBinary *)bin;
+                    q->v_head->iov_base = bin->orig_bytes;
+                }
+		*--q->b_head = b;
+		q->v_head--;
+		q->v_head->iov_len = len;
+	    }
+	    else {
+                if (q->driver)
+                    driver_binary_inc_refc(&b->driver);
+                else
+                    erts_refc_inc(&b->nif.intern.refc, 1);
+		*--q->b_head = b;
+		*--q->v_head = *iov;
+	    }
+	}
+	iov--;
+	binv--;
+    }
+    q->size += size;      /* update total size in queue */
+    return 0;
+}
+
+
+/*
+** Remove size bytes from queue head
+** Return number of bytes that remain in queue
+*/
+int erts_ioq_deq(ErtsIOQueue *q, Uint size)
+{
+    Uint len;
+
+    if ((q == NULL) || (q->size < size))
+	return -1;
+    q->size -= size;
+    while (size > 0) {
+	ASSERT(q->v_head != q->v_tail);
+
+	len = q->v_head->iov_len;
+	if (len <= size) {
+	    size -= len;
+            free_binary(*q->b_head, q->driver);
+	    *q->b_head++ = NULL;
+	    q->v_head++;
+	}
+	else {
+	    q->v_head->iov_base = ((char *)(q->v_head->iov_base)) + size;
+	    q->v_head->iov_len -= size;
+	    size = 0;
+	}
+    }
+
+    /* restart pointers (optimised for enq) */
+    if (q->v_head == q->v_tail) {
+	q->v_head = q->v_tail = q->v_start;
+	q->b_head = q->b_tail = q->b_start;
+    }
+    return 0;
+}
+
+
+Uint erts_ioq_peekqv(ErtsIOQueue *q, ErtsIOVec *ev) {
+    ASSERT(ev);
+
+    if (! q) {
+	return (Uint) -1;
+    } else {
+	if ((ev->common.vsize = q->v_tail - q->v_head) == 0) {
+	    ev->common.size = 0;
+	    ev->common.iov = NULL;
+	    ev->common.binv = NULL;
+	} else {
+	    ev->common.size = q->size;
+	    ev->common.iov = q->v_head;
+	    ev->common.binv = q->b_head;
+	}
+	return q->size;
+    }
+}
+
+SysIOVec* erts_ioq_peekq(ErtsIOQueue *q, int* vlenp)  /* length of io-vector */
+{
+
+    if (q == NULL) {
+	*vlenp = -1;
+	return NULL;
+    }
+    if ((*vlenp = (q->v_tail - q->v_head)) == 0)
+	return NULL;
+    return q->v_head;
+}
+
+/* Fills a possibly deep list of chars and binaries into vec
+** Small characters are first stored in the buffer buf of length ln
+** binaries found are copied and linked into msoh
+** Return  vector length on succsess,
+**        -1 on overflow
+**        -2 on type error
+*/
+
+static ERTS_INLINE void
+io_list_to_vec_set_vec(SysIOVec **iov, ErtsIOQBinary ***binv,
+                       ErtsIOQBinary *bin, byte *ptr, Uint len,
+                       int *vlen)
+{
+    while (len > MAX_SYSIOVEC_IOVLEN) {
+        (*iov)->iov_base = ptr;
+        (*iov)->iov_len = MAX_SYSIOVEC_IOVLEN;
+        ptr += MAX_SYSIOVEC_IOVLEN;
+        len -= MAX_SYSIOVEC_IOVLEN;
+        (*iov)++;
+        (*vlen)++;
+        *(*binv)++ = bin;
+    }
+    (*iov)->iov_base = ptr;
+    (*iov)->iov_len = len;
+    *(*binv)++ = bin;
+    (*iov)++;
+    (*vlen)++;
+}
+
+int
+erts_ioq_iolist_to_vec(Eterm obj,	  /* io-list */
+                       SysIOVec* iov,	  /* io vector */
+                       ErtsIOQBinary** binv,       /* binary reference vector */
+                       ErtsIOQBinary* cbin,        /* binary to store characters */
+                       Uint bin_limit,  /* small binaries limit */
+                       int driver)
+{
+    DECLARE_ESTACK(s);
+    Eterm* objp;
+    byte *buf  = NULL;
+    Uint len = 0;
+    Uint csize  = 0;
+    int vlen   = 0;
+    byte* cptr;
+
+    if (cbin) {
+        if (driver) {
+            buf = (byte*)cbin->driver.orig_bytes;
+            len = cbin->driver.orig_size;
+        } else {
+            buf = (byte*)cbin->nif.orig_bytes;
+            len = cbin->nif.orig_size;
+        }
+    }
+    cptr = buf;
+
+    goto L_jump_start;  /* avoid push */
+
+    while (!ESTACK_ISEMPTY(s)) {
+	obj = ESTACK_POP(s);
+    L_jump_start:
+	if (is_list(obj)) {
+	L_iter_list:
+	    objp = list_val(obj);
+	    obj = CAR(objp);
+	    if (is_byte(obj)) {
+		if (len == 0)
+		    goto L_overflow;
+		*buf++ = unsigned_val(obj);
+		csize++;
+		len--;
+	    } else if (is_binary(obj)) {
+		ESTACK_PUSH(s, CDR(objp));
+		goto handle_binary;
+	    } else if (is_list(obj)) {
+		ESTACK_PUSH(s, CDR(objp));
+		goto L_iter_list;    /* on head */
+	    } else if (!is_nil(obj)) {
+		goto L_type_error;
+	    }
+	    obj = CDR(objp);
+	    if (is_list(obj))
+		goto L_iter_list; /* on tail */
+	    else if (is_binary(obj)) {
+		goto handle_binary;
+	    } else if (!is_nil(obj)) {
+		goto L_type_error;
+	    }
+	} else if (is_binary(obj)) {
+	    Eterm real_bin;
+	    Uint offset;
+	    Eterm* bptr;
+	    Uint size;
+	    int bitoffs;
+	    int bitsize;
+
+	handle_binary:
+	    size = binary_size(obj);
+	    ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize);
+	    ASSERT(bitsize == 0);
+	    bptr = binary_val(real_bin);
+	    if (*bptr == HEADER_PROC_BIN) {
+		ProcBin* pb = (ProcBin *) bptr;
+		if (bitoffs != 0) {
+		    if (len < size) {
+			goto L_overflow;
+		    }
+		    erts_copy_bits(pb->bytes+offset, bitoffs, 1,
+				   (byte *) buf, 0, 1, size*8);
+		    csize += size;
+		    buf += size;
+		    len -= size;
+		} else if (bin_limit && size < bin_limit) {
+		    if (len < size) {
+			goto L_overflow;
+		    }
+		    sys_memcpy(buf, pb->bytes+offset, size);
+		    csize += size;
+		    buf += size;
+		    len -= size;
+		} else {
+                    ErtsIOQBinary *qbin;
+		    if (csize != 0) {
+                        io_list_to_vec_set_vec(&iov, &binv, cbin,
+                                               cptr, csize, &vlen);
+			cptr = buf;
+			csize = 0;
+		    }
+		    if (pb->flags) {
+			erts_emasculate_writable_binary(pb);
+		    }
+                    if (driver)
+                        qbin = (ErtsIOQBinary*)Binary2ErlDrvBinary(pb->val);
+                    else
+                        qbin = (ErtsIOQBinary*)pb->val;
+
+                    io_list_to_vec_set_vec(
+                        &iov, &binv, qbin,
+                        pb->bytes+offset, size, &vlen);
+		}
+	    } else {
+		ErlHeapBin* hb = (ErlHeapBin *) bptr;
+		if (len < size) {
+		    goto L_overflow;
+		}
+		copy_binary_to_buffer(buf, 0,
+				      ((byte *) hb->data)+offset, bitoffs,
+				      8*size);
+		csize += size;
+		buf += size;
+		len -= size;
+	    }
+	} else if (!is_nil(obj)) {
+	    goto L_type_error;
+	}
+    }
+
+    if (csize != 0) {
+        io_list_to_vec_set_vec(&iov, &binv, cbin, cptr, csize, &vlen);
+    }
+
+    DESTROY_ESTACK(s);
+    return vlen;
+
+ L_type_error:
+    DESTROY_ESTACK(s);
+    return -2;
+
+ L_overflow:
+    DESTROY_ESTACK(s);
+    return -1;
+}
+
+static ERTS_INLINE int
+io_list_vec_count(Eterm obj, Uint *v_size,
+                  Uint *c_size, Uint *b_size, Uint *in_clist,
+                  Uint *p_v_size, Uint *p_c_size, Uint *p_in_clist,
+                  Uint blimit)
+{
+    Uint size = binary_size(obj);
+    Eterm real;
+    ERTS_DECLARE_DUMMY(Uint offset);
+    int bitoffs;
+    int bitsize;
+    ERTS_GET_REAL_BIN(obj, real, offset, bitoffs, bitsize);
+    if (bitsize != 0) return 1;
+    if (thing_subtag(*binary_val(real)) == REFC_BINARY_SUBTAG &&
+	bitoffs == 0) {
+	*b_size += size;
+        if (*b_size < size) return 2;
+	*in_clist = 0;
+        ++*v_size;
+        /* If iov_len is smaller then Uint we split the binary into*/
+        /* multiple smaller (2GB) elements in the iolist.*/
+	*v_size += size / MAX_SYSIOVEC_IOVLEN;
+        if (size >= blimit) {
+            *p_in_clist = 0;
+            ++*p_v_size;
+        } else {
+            *p_c_size += size;
+            if (!*p_in_clist) {
+                *p_in_clist = 1;
+                ++*p_v_size;
+            }
+        }
+    } else {
+	*c_size += size;
+        if (*c_size < size) return 2;
+	if (!*in_clist) {
+	    *in_clist = 1;
+	    ++*v_size;
+	}
+	*p_c_size += size;
+	if (!*p_in_clist) {
+	    *p_in_clist = 1;
+	    ++*p_v_size;
+	}
+    }
+    return 0;
+}
+
+#define IO_LIST_VEC_COUNT(obj)                                          \
+    do {                                                                \
+        switch (io_list_vec_count(obj, &v_size, &c_size,                \
+                                  &b_size, &in_clist,                   \
+                                  &p_v_size, &p_c_size, &p_in_clist,    \
+                                  blimit)) {                            \
+        case 1: goto L_type_error;                                      \
+        case 2: goto L_overflow_error;                                  \
+        default: break;                                                 \
+        }                                                               \
+    } while(0)
+
+/* 
+ * Returns 0 if successful and a non-zero value otherwise.
+ *
+ * Return values through pointers:
+ *    *vsize      - SysIOVec size needed for a writev
+ *    *csize      - Number of bytes not in binary (in the common binary)
+ *    *pvsize     - SysIOVec size needed if packing small binaries
+ *    *pcsize     - Number of bytes in the common binary if packing
+ *    *total_size - Total size of iolist in bytes
+ */
+int
+erts_ioq_iolist_vec_len(Eterm obj, int* vsize, Uint* csize,
+                        Uint* pvsize, Uint* pcsize,
+                        Uint* total_size, Uint blimit)
+{
+    DECLARE_ESTACK(s);
+    Eterm* objp;
+    Uint v_size = 0;
+    Uint c_size = 0;
+    Uint b_size = 0;
+    Uint in_clist = 0;
+    Uint p_v_size = 0;
+    Uint p_c_size = 0;
+    Uint p_in_clist = 0;
+    Uint total;
+
+    goto L_jump_start;  /* avoid a push */
+
+    while (!ESTACK_ISEMPTY(s)) {
+	obj = ESTACK_POP(s);
+    L_jump_start:
+	if (is_list(obj)) {
+	L_iter_list:
+	    objp = list_val(obj);
+	    obj = CAR(objp);
+
+	    if (is_byte(obj)) {
+		c_size++;
+		if (c_size == 0) {
+		    goto L_overflow_error;
+		}
+		if (!in_clist) {
+		    in_clist = 1;
+		    v_size++;
+		}
+		p_c_size++;
+		if (!p_in_clist) {
+		    p_in_clist = 1;
+		    p_v_size++;
+		}
+	    }
+	    else if (is_binary(obj)) {
+                IO_LIST_VEC_COUNT(obj);
+	    }
+	    else if (is_list(obj)) {
+		ESTACK_PUSH(s, CDR(objp));
+		goto L_iter_list;   /* on head */
+	    }
+	    else if (!is_nil(obj)) {
+		goto L_type_error;
+	    }
+
+	    obj = CDR(objp);
+	    if (is_list(obj))
+		goto L_iter_list;   /* on tail */
+	    else if (is_binary(obj)) {  /* binary tail is OK */
+		IO_LIST_VEC_COUNT(obj);
+	    }
+	    else if (!is_nil(obj)) {
+		goto L_type_error;
+	    }
+	}
+	else if (is_binary(obj)) {
+	    IO_LIST_VEC_COUNT(obj);
+	}
+	else if (!is_nil(obj)) {
+	    goto L_type_error;
+	}
+    }
+
+    total = c_size + b_size;
+    if (total < c_size) {
+	goto L_overflow_error;
+    }
+    *total_size = total;
+
+    DESTROY_ESTACK(s);
+    *vsize = v_size;
+    *csize = c_size;
+    *pvsize = p_v_size;
+    *pcsize = p_c_size;
+    return 0;
+
+ L_type_error:
+ L_overflow_error:
+    DESTROY_ESTACK(s);
+    return 1;
+}
diff --git a/erts/emulator/beam/erl_io_queue.h b/erts/emulator/beam/erl_io_queue.h
new file mode 100644
index 0000000000..51abe99510
--- /dev/null
+++ b/erts/emulator/beam/erl_io_queue.h
@@ -0,0 +1,201 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2017. All Rights Reserved.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: A queue used for storing binary data that should be
+ *              passed to writev or similar functions. Used by both
+ *              the nif and driver api.
+ *
+ * Author:      Lukas Larsson
+ */
+
+#ifndef ERL_IO_QUEUE_H__TYPES__
+#define ERL_IO_QUEUE_H__TYPES__
+
+#define ERTS_BINARY_TYPES_ONLY__
+#include "erl_binary.h"
+#undef ERTS_BINARY_TYPES_ONLY__
+#include "erl_nif.h"
+
+#ifdef DEBUG
+#define MAX_SYSIOVEC_IOVLEN (1ull << (32 - 1))
+#else
+#define MAX_SYSIOVEC_IOVLEN (1ull << (sizeof(((SysIOVec*)0)->iov_len) * 8 - 1))
+#endif
+
+#define ERTS_SMALL_IO_QUEUE 5
+
+typedef union {
+    ErlDrvBinary driver;
+    Binary nif;
+} ErtsIOQBinary;
+
+typedef struct {
+    int vsize;         /* length of vectors */
+    Uint size;         /* total size in bytes */
+    SysIOVec* iov;
+    ErtsIOQBinary** binv;
+} ErtsIOVecCommon;
+
+typedef union {
+    ErtsIOVecCommon common;
+    ErlIOVec driver;
+    ErlNifIOVec nif;
+} ErtsIOVec;
+
+/* head/tail represent the data in the queue
+ * start/end represent the edges of the allocated queue
+ * small is used when the number of iovec elements is < SMALL_IO_QUEUE
+ */
+typedef struct erts_io_queue {
+    ErtsAlcType_t alct;
+    int driver;
+    Uint size;       /* total size in bytes */
+
+    SysIOVec* v_start;
+    SysIOVec* v_end;
+    SysIOVec* v_head;
+    SysIOVec* v_tail;
+    SysIOVec  v_small[ERTS_SMALL_IO_QUEUE];
+
+    ErtsIOQBinary **b_start;
+    ErtsIOQBinary **b_end;
+    ErtsIOQBinary **b_head;
+    ErtsIOQBinary **b_tail;
+    ErtsIOQBinary  *b_small[ERTS_SMALL_IO_QUEUE];
+
+} ErtsIOQueue;
+
+#endif /* ERL_IO_QUEUE_H__TYPES__ */
+
+#if !defined(ERL_IO_QUEUE_H) && !defined(ERTS_IO_QUEUE_TYPES_ONLY__)
+#define ERL_IO_QUEUE_H
+
+#include "erl_binary.h"
+#include "erl_bits.h"
+
+void erts_ioq_init(ErtsIOQueue *q, ErtsAlcType_t alct, int driver);
+void erts_ioq_clear(ErtsIOQueue *q);
+Uint erts_ioq_size(ErtsIOQueue *q);
+int erts_ioq_enqv(ErtsIOQueue *q, ErtsIOVec *vec, Uint skip);
+int erts_ioq_pushqv(ErtsIOQueue *q, ErtsIOVec *vec, Uint skip);
+int erts_ioq_deq(ErtsIOQueue *q, Uint Uint);
+Uint erts_ioq_peekqv(ErtsIOQueue *q, ErtsIOVec *ev);
+SysIOVec *erts_ioq_peekq(ErtsIOQueue *q, int *vlenp);
+Uint erts_ioq_sizeq(ErtsIOQueue *q);
+
+int erts_ioq_iolist_vec_len(Eterm obj, int* vsize, Uint* csize,
+                            Uint* pvsize, Uint* pcsize,
+                            Uint* total_size, Uint blimit);
+int erts_ioq_iolist_to_vec(Eterm obj, SysIOVec* iov,
+                           ErtsIOQBinary** binv, ErtsIOQBinary* cbin,
+                           Uint bin_limit, int driver_binary);
+
+ERTS_GLB_INLINE
+int erts_ioq_iodata_vec_len(Eterm obj, int* vsize, Uint* csize,
+                            Uint* pvsize, Uint* pcsize,
+                            Uint* total_size, Uint blimit);
+ERTS_GLB_INLINE
+int erts_ioq_iodata_to_vec(Eterm obj, SysIOVec* iov,
+                           ErtsIOQBinary** binv, ErtsIOQBinary* cbin,
+                           Uint bin_limit, int driver_binary);
+
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE
+int erts_ioq_iodata_vec_len(Eterm obj, int* vsize, Uint* csize,
+                            Uint* pvsize, Uint* pcsize,
+                            Uint* total_size, Uint blimit) {
+  if (is_binary(obj)) {
+    /* We optimize for when we get a procbin without a bit-offset
+     * that fits in one iov slot
+     */
+    Eterm real_bin;
+    byte bitoffs;
+    byte bitsize;
+    ERTS_DECLARE_DUMMY(Uint offset);
+    Uint size = binary_size(obj);
+    ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize);
+    if (size < MAX_SYSIOVEC_IOVLEN && bitoffs == 0 && bitsize == 0) {
+      *vsize = 1;
+      *pvsize = 1;
+      if (thing_subtag(*binary_val(real_bin)) == REFC_BINARY_SUBTAG) {
+          *csize = 0;
+          *pcsize = 0;
+      } else {
+          *csize = size;
+          *pcsize = size;
+      }
+      *total_size = size;
+      return 0;
+    }
+  }
+
+  return erts_ioq_iolist_vec_len(obj, vsize, csize,
+                                 pvsize, pcsize, total_size, blimit);
+}
+
+ERTS_GLB_INLINE
+int erts_ioq_iodata_to_vec(Eterm obj,
+                           SysIOVec *iov,
+                           ErtsIOQBinary **binv,
+                           ErtsIOQBinary  *cbin,
+                           Uint bin_limit,
+                           int driver)
+{
+    if (is_binary(obj)) {
+        Eterm real_bin;
+        byte bitoffs;
+        byte bitsize;
+        Uint offset;
+        Uint size = binary_size(obj);
+        ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize);
+        if (size < MAX_SYSIOVEC_IOVLEN && bitoffs == 0 && bitsize == 0) {
+            Eterm *bptr = binary_val(real_bin);
+            if (thing_subtag(*bptr) == REFC_BINARY_SUBTAG) {
+                ProcBin *pb = (ProcBin *)bptr;
+                if (pb->flags)
+                    erts_emasculate_writable_binary(pb);
+                iov[0].iov_base = pb->bytes+offset;
+                iov[0].iov_len = size;
+                if (driver)
+                    binv[0] = (ErtsIOQBinary*)Binary2ErlDrvBinary(pb->val);
+                else
+                    binv[0] = (ErtsIOQBinary*)pb->val;
+                return 1;
+            } else {
+                ErlHeapBin* hb = (ErlHeapBin *)bptr;
+                byte *buf = driver ? (byte*)cbin->driver.orig_bytes :
+                    (byte*)cbin->nif.orig_bytes;
+		copy_binary_to_buffer(buf, 0, ((byte *) hb->data)+offset, 0, 8*size);
+                iov[0].iov_base = buf;
+                iov[0].iov_len = size;
+                binv[0] = cbin;
+                return 1;
+            }
+        }
+    }
+    return erts_ioq_iolist_to_vec(obj, iov, binv, cbin, bin_limit, driver);
+}
+
+#endif
+
+#endif /* ERL_IO_QUEUE_H */
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index d3c5af3a83..0ebb2b6db0 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -56,6 +56,7 @@
 #include "erl_process.h"
 #include "erl_bif_unique.h"
 #include "erl_utils.h"
+#include "erl_io_queue.h"
 #undef ERTS_WANT_NFUNC_SCHED_INTERNALS__
 #define ERTS_WANT_NFUNC_SCHED_INTERNALS__
 #include "erl_nfunc_sched.h"
@@ -66,6 +67,13 @@
 #include <limits.h>
 #include <stddef.h> /* offsetof */
 
+#ifndef MAX
+#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
+#endif
+
+#ifndef MIN
+#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
+#endif
 
 /* Information about a loaded nif library.
  * Each successful call to erlang:load_nif will allocate an instance of
@@ -3344,6 +3352,363 @@ int enif_compare_monitors(const ErlNifMonitor *monitor1,
                       ERTS_REF_THING_SIZE*sizeof(Eterm));
 }
 
+ErlNifIOQueue *enif_ioq_create(ErlNifIOQueueOpts opts)
+{
+    ErlNifIOQueue *q;
+
+    if (opts != ERL_NIF_IOQ_NORMAL)
+        return NULL;
+
+    q = enif_alloc(sizeof(ErlNifIOQueue));
+    if (!q) return NULL;
+    erts_ioq_init(q, ERTS_ALC_T_NIF, 0);
+
+    return q;
+}
+
+void enif_ioq_destroy(ErlNifIOQueue *q)
+{
+    erts_ioq_clear(q);
+    enif_free(q);
+}
+
+/* If the iovec was preallocated (Stack or otherwise) it needs to be marked as
+ * such to perform a proper free. */
+#define ERL_NIF_IOVEC_FLAGS_PREALLOC (1 << 0)
+
+void enif_free_iovec(ErlNifIOVec *iov)
+{
+    int i;
+    /* Decrement the refc of all the binaries */
+    for (i = 0; i < iov->iovcnt; i++) {
+        Binary *bptr = ((Binary**)iov->ref_bins)[i];
+        /* bptr can be null if enq_binary was used */
+        if (bptr && erts_refc_dectest(&bptr->intern.refc, 0) == 0) {
+            erts_bin_free(bptr);
+        }
+    }
+
+    if (!(iov->flags & ERL_NIF_IOVEC_FLAGS_PREALLOC)) {
+        enif_free(iov);
+    }
+}
+
+typedef struct {
+    UWord sublist_length;
+    Eterm sublist_start;
+    Eterm sublist_end;
+
+    UWord offheap_size;
+    UWord onheap_size;
+
+    UWord iovec_len;
+} iovec_slice_t;
+
+static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *result) {
+    Eterm lookahead;
+
+    result->sublist_start = list;
+    result->sublist_length = 0;
+    result->offheap_size = 0;
+    result->onheap_size = 0;
+    result->iovec_len = 0;
+
+    lookahead = result->sublist_start;
+
+    while (is_list(lookahead)) {
+        Eterm *binary_header, binary;
+        Eterm *cell;
+        UWord size;
+
+        cell = list_val(lookahead);
+        binary = CAR(cell);
+
+        if (!is_binary(binary)) {
+            return 0;
+        }
+
+        size = binary_size(binary);
+        binary_header = binary_val(binary);
+
+        /* If we're a sub-binary we'll need to check our underlying binary to
+         * determine whether we're on-heap or not. */
+        if(thing_subtag(*binary_header) == SUB_BINARY_SUBTAG) {
+            ErlSubBin *sb = (ErlSubBin*)binary_header;
+
+            /* Reject bitstrings */
+            if((sb->bitoffs + sb->bitsize) > 0) {
+                return 0;
+            }
+
+            ASSERT(size <= binary_size(sb->orig));
+            binary_header = binary_val(sb->orig);
+        }
+
+        if(thing_subtag(*binary_header) == HEAP_BINARY_SUBTAG) {
+            ASSERT(size <= ERL_ONHEAP_BIN_LIMIT);
+
+            result->iovec_len += 1;
+            result->onheap_size += size;
+        } else {
+            ASSERT(thing_subtag(*binary_header) == REFC_BINARY_SUBTAG);
+
+            result->iovec_len += 1 + size / MAX_SYSIOVEC_IOVLEN;
+            result->offheap_size += size;
+        }
+
+        result->sublist_length += 1;
+        lookahead = CDR(cell);
+
+        if(result->sublist_length >= max_length) {
+            break;
+        }
+    }
+
+    if (!is_nil(lookahead) && !is_list(lookahead)) {
+        return 0;
+    }
+
+    result->sublist_end = lookahead;
+
+    return 1;
+}
+
+static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) {
+    Eterm *parent_header;
+    Eterm parent_binary;
+
+    int bit_offset, bit_size;
+    Uint byte_offset;
+
+    ASSERT(is_binary(binary));
+
+    ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset, bit_offset, bit_size);
+
+    parent_header = binary_val(parent_binary);
+
+    result->size = binary_size(binary);
+    result->bin_term = binary;
+
+    if (thing_subtag(*parent_header) == REFC_BINARY_SUBTAG) {
+        ProcBin *pb = (ProcBin*)parent_header;
+
+        ASSERT(pb->val != NULL);
+        ASSERT(byte_offset < pb->size);
+        ASSERT(&pb->bytes[byte_offset] >= (byte*)(pb->val)->orig_bytes);
+
+        result->data = (unsigned char*)&pb->bytes[byte_offset];
+        result->ref_bin = (void*)pb->val;
+    } else {
+        ErlHeapBin *hb = (ErlHeapBin*)parent_header;
+
+        ASSERT(thing_subtag(*parent_header) == HEAP_BINARY_SUBTAG);
+
+        result->data = &((unsigned char*)&hb->data)[byte_offset];
+        result->ref_bin = NULL;
+    }
+}
+
+static int fill_iovec_with_slice(ErlNifEnv *env,
+                                 iovec_slice_t *slice,
+                                 ErlNifIOVec *iovec) {
+    UWord onheap_offset, iovec_idx;
+    ErlNifBinary onheap_data;
+    Eterm sublist_iterator;
+
+    /* Set up a common refc binary for all on-heap binaries. */
+    if (slice->onheap_size > 0) {
+        if (!enif_alloc_binary(slice->onheap_size, &onheap_data)) {
+            return 0;
+        }
+    }
+
+    sublist_iterator = slice->sublist_start;
+    onheap_offset = 0;
+    iovec_idx = 0;
+
+    while (sublist_iterator != slice->sublist_end) {
+        ErlNifBinary raw_data;
+        Eterm *cell;
+
+        cell = list_val(sublist_iterator);
+        inspect_raw_binary_data(CAR(cell), &raw_data);
+
+        /* If this isn't a refc binary, copy its contents to the onheap buffer
+         * and reference that instead. */
+        if (raw_data.ref_bin == NULL) {
+            ASSERT(onheap_offset < onheap_data.size);
+            ASSERT(slice->onheap_size > 0);
+
+            sys_memcpy(&onheap_data.data[onheap_offset],
+                       raw_data.data, raw_data.size);
+
+            raw_data.data = &onheap_data.data[onheap_offset];
+            raw_data.ref_bin = onheap_data.ref_bin;
+        }
+
+        ASSERT(raw_data.ref_bin != NULL);
+
+        while (raw_data.size > 0) {
+            UWord chunk_len = MIN(raw_data.size, MAX_SYSIOVEC_IOVLEN);
+
+            ASSERT(iovec_idx < iovec->iovcnt);
+
+            iovec->iov[iovec_idx].iov_base = raw_data.data;
+            iovec->iov[iovec_idx].iov_len = chunk_len;
+
+            iovec->ref_bins[iovec_idx] = raw_data.ref_bin;
+
+            raw_data.data += chunk_len;
+            raw_data.size -= chunk_len;
+
+            iovec_idx += 1;
+        }
+
+        sublist_iterator = CDR(cell);
+    }
+
+    ASSERT(iovec_idx == iovec->iovcnt);
+
+    if (env == NULL) {
+        int i;
+        for (i = 0; i < iovec->iovcnt; i++) {
+            Binary *refc_binary = (Binary*)(iovec->ref_bins[i]);
+            erts_refc_inc(&refc_binary->intern.refc, 1);
+        }
+
+        if (slice->onheap_size > 0) {
+            /* Transfer ownership to the iovec; we've taken references to it in
+             * the above loop. */
+            enif_release_binary(&onheap_data);
+        }
+    } else {
+        if (slice->onheap_size > 0) {
+            /* Attach the binary to our environment and let the GC take care of
+             * it after returning. */
+            enif_make_binary(env, &onheap_data);
+        }
+    }
+
+    return 1;
+}
+
+static int create_iovec_from_slice(ErlNifEnv *env,
+                                   iovec_slice_t *slice,
+                                   ErlNifIOVec **result) {
+    ErlNifIOVec *iovec = *result;
+
+    if (iovec && slice->iovec_len < ERL_NIF_IOVEC_SIZE) {
+        iovec->iov = iovec->small_iov;
+        iovec->ref_bins = iovec->small_ref_bin;
+        iovec->flags = ERL_NIF_IOVEC_FLAGS_PREALLOC;
+    } else {
+        UWord iov_offset, binv_offset, alloc_size;
+        char *alloc_base;
+
+        iov_offset = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErlNifIOVec));
+        binv_offset = iov_offset;
+        binv_offset += ERTS_ALC_DATA_ALIGN_SIZE(slice->iovec_len * sizeof(SysIOVec));
+        alloc_size = binv_offset;
+        alloc_size += slice->iovec_len * sizeof(Binary*);
+
+        /* If we have an environment we'll attach the allocated data to it. The
+         * GC will take care of releasing it later on. */
+        if (env != NULL) {
+            ErlNifBinary gc_bin;
+
+            if (!enif_alloc_binary(alloc_size, &gc_bin)) {
+                return 0;
+            }
+
+            alloc_base = (char*)gc_bin.data;
+            enif_make_binary(env, &gc_bin);
+        } else {
+            alloc_base = enif_alloc(alloc_size);
+        }
+
+        iovec = (ErlNifIOVec*)alloc_base;
+        iovec->iov = (SysIOVec*)(alloc_base + iov_offset);
+        iovec->ref_bins = (void**)(alloc_base + binv_offset);
+        iovec->flags = 0;
+    }
+
+    iovec->size = slice->offheap_size + slice->onheap_size;
+    iovec->iovcnt = slice->iovec_len;
+
+    if(!fill_iovec_with_slice(env, slice, iovec)) {
+        if (env == NULL && !(iovec->flags & ERL_NIF_IOVEC_FLAGS_PREALLOC)) {
+            enif_free(iovec);
+        }
+
+        return 0;
+    }
+
+    *result = iovec;
+
+    return 1;
+}
+
+int enif_inspect_iovec(ErlNifEnv *env, size_t max_elements,
+                       ERL_NIF_TERM list, ERL_NIF_TERM *tail,
+                       ErlNifIOVec **iov) {
+    iovec_slice_t slice;
+
+    if(!examine_iovec_term(list, max_elements, &slice)) {
+        return 0;
+    } else if(!create_iovec_from_slice(env, &slice, iov)) {
+        return 0;
+    }
+
+    (*tail) = slice.sublist_end;
+
+    return 1;
+}
+
+/* */
+int enif_ioq_enqv(ErlNifIOQueue *q, ErlNifIOVec *iov, size_t skip)
+{
+    if(skip <= iov->size) {
+        return !erts_ioq_enqv(q, (ErtsIOVec*)iov, skip);
+    }
+
+    return 0;
+}
+
+int enif_ioq_enq_binary(ErlNifIOQueue *q, ErlNifBinary *bin, size_t skip)
+{
+    ErlNifIOVec vec = {1, bin->size, NULL, NULL, ERL_NIF_IOVEC_FLAGS_PREALLOC };
+    Binary *ref_bin = (Binary*)bin->ref_bin;
+    int res;
+    vec.iov = vec.small_iov;
+    vec.ref_bins = vec.small_ref_bin;
+    vec.iov[0].iov_base = bin->data;
+    vec.iov[0].iov_len = bin->size;
+    ((Binary**)(vec.ref_bins))[0] = ref_bin;
+
+    res = enif_ioq_enqv(q, &vec, skip);
+    enif_release_binary(bin);
+    return res;
+}
+
+size_t enif_ioq_size(ErlNifIOQueue *q)
+{
+    return erts_ioq_size(q);
+}
+
+int enif_ioq_deq(ErlNifIOQueue *q, size_t elems, size_t *size)
+{
+    if (erts_ioq_deq(q, elems) == -1)
+        return 0;
+    if (size)
+        *size = erts_ioq_size(q);
+    return 1;
+}
+
+SysIOVec *enif_ioq_peek(ErlNifIOQueue *q, int *iovlen)
+{
+    return erts_ioq_peekq(q, iovlen);
+}
+
 /***************************************************************************
  **                              load_nif/2                               **
  ***************************************************************************/
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index b0d5c39798..d195721054 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -50,6 +50,7 @@
 ** 2.9: 18.2 enif_getenv
 ** 2.10: Time API
 ** 2.11: 19.0 enif_snprintf 
+** 2.12: 20.0 add enif_queue
 */
 #define ERL_NIF_MAJOR_VERSION 2
 #define ERL_NIF_MINOR_VERSION 12
@@ -241,6 +242,28 @@ typedef enum {
     ERL_NIF_PHASH2 = 2
 } ErlNifHash;
 
+#define ERL_NIF_IOVEC_SIZE 16
+
+typedef struct erl_nif_io_vec {
+    int iovcnt;  /* length of vectors */
+    size_t size; /* total size in bytes */
+    SysIOVec *iov;
+
+    /* internals (avert your eyes) */
+    void **ref_bins; /* Binary[] */
+    int flags;
+
+    /* Used when stack allocating the io vec */
+    SysIOVec small_iov[ERL_NIF_IOVEC_SIZE];
+    void *small_ref_bin[ERL_NIF_IOVEC_SIZE];
+} ErlNifIOVec;
+
+typedef struct erts_io_queue ErlNifIOQueue;
+
+typedef enum {
+    ERL_NIF_IOQ_NORMAL = 1
+} ErlNifIOQueueOpts;
+
 /*
  * Return values from enif_thread_type(). Negative values
  * reserved for specific types of non-scheduler threads.
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 94c04cd126..9e573307d8 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -184,6 +184,21 @@ ERL_NIF_API_FUNC_DECL(ErlNifUInt64,enif_hash,(ErlNifHash type, ERL_NIF_TERM term
 ERL_NIF_API_FUNC_DECL(int, enif_whereis_pid, (ErlNifEnv *env, ERL_NIF_TERM name, ErlNifPid *pid));
 ERL_NIF_API_FUNC_DECL(int, enif_whereis_port, (ErlNifEnv *env, ERL_NIF_TERM name, ErlNifPort *port));
 
+ERL_NIF_API_FUNC_DECL(ErlNifIOQueue *,enif_ioq_create,(ErlNifIOQueueOpts opts));
+ERL_NIF_API_FUNC_DECL(void,enif_ioq_destroy,(ErlNifIOQueue *q));
+
+ERL_NIF_API_FUNC_DECL(int,enif_ioq_enq_binary,(ErlNifIOQueue *q, ErlNifBinary *bin, size_t skip));
+ERL_NIF_API_FUNC_DECL(int,enif_ioq_enqv,(ErlNifIOQueue *q, ErlNifIOVec *iov, size_t skip));
+
+ERL_NIF_API_FUNC_DECL(size_t,enif_ioq_size,(ErlNifIOQueue *q));
+ERL_NIF_API_FUNC_DECL(int,enif_ioq_deq,(ErlNifIOQueue *q, size_t count, size_t *size));
+
+ERL_NIF_API_FUNC_DECL(SysIOVec*,enif_ioq_peek,(ErlNifIOQueue *q, int *iovlen));
+
+ERL_NIF_API_FUNC_DECL(int,enif_inspect_iovec,(ErlNifEnv *env, size_t max_length, ERL_NIF_TERM iovec_term, ERL_NIF_TERM *tail, ErlNifIOVec **iovec));
+ERL_NIF_API_FUNC_DECL(void,enif_free_iovec,(ErlNifIOVec *iov));
+
+
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
 */
@@ -348,6 +363,16 @@ ERL_NIF_API_FUNC_DECL(int, enif_whereis_port, (ErlNifEnv *env, ERL_NIF_TERM name
 #  define enif_hash ERL_NIF_API_FUNC_MACRO(enif_hash)
 #  define enif_whereis_pid ERL_NIF_API_FUNC_MACRO(enif_whereis_pid)
 #  define enif_whereis_port ERL_NIF_API_FUNC_MACRO(enif_whereis_port)
+#  define enif_ioq_create ERL_NIF_API_FUNC_MACRO(enif_ioq_create)
+#  define enif_ioq_destroy ERL_NIF_API_FUNC_MACRO(enif_ioq_destroy)
+#  define enif_ioq_enq ERL_NIF_API_FUNC_MACRO(enif_ioq_enq)
+#  define enif_ioq_enq_binary ERL_NIF_API_FUNC_MACRO(enif_ioq_enq_binary)
+#  define enif_ioq_enqv ERL_NIF_API_FUNC_MACRO(enif_ioq_enqv)
+#  define enif_ioq_size ERL_NIF_API_FUNC_MACRO(enif_ioq_size)
+#  define enif_ioq_deq ERL_NIF_API_FUNC_MACRO(enif_ioq_deq)
+#  define enif_ioq_peek ERL_NIF_API_FUNC_MACRO(enif_ioq_peek)
+#  define enif_inspect_iovec ERL_NIF_API_FUNC_MACRO(enif_inspect_iovec)
+#  define enif_free_iovec ERL_NIF_API_FUNC_MACRO(enif_free_iovec)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index 6a3213ec52..b64de624dd 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -31,6 +31,9 @@ typedef struct ErtsProc2PortSigData_ ErtsProc2PortSigData;
 #include "erl_ptab.h"
 #include "erl_thr_progress.h"
 #include "erl_trace.h"
+#define ERTS_IO_QUEUE_TYPES_ONLY__
+#include "erl_io_queue.h"
+#undef ERTS_IO_QUEUE_TYPES_ONLY__
 
 #ifndef __WIN32__
 #define ERTS_DEFAULT_MAX_PORTS (1 << 16)
@@ -75,23 +78,8 @@ typedef struct erts_driver_t_ erts_driver_t;
 #define ERTS_Port2ErlDrvPort(PH) ((ErlDrvPort) (PH))
 #endif
 
-#define SMALL_IO_QUEUE 5   /* Number of fixed elements */
+typedef ErtsIOQueue ErlPortIOQueue;
 
-typedef struct {
-    ErlDrvSizeT size;       /* total size in bytes */
-
-    SysIOVec* v_start;
-    SysIOVec* v_end;
-    SysIOVec* v_head;
-    SysIOVec* v_tail;
-    SysIOVec  v_small[SMALL_IO_QUEUE];
-
-    ErlDrvBinary** b_start;
-    ErlDrvBinary** b_end;
-    ErlDrvBinary** b_head;
-    ErlDrvBinary** b_tail;
-    ErlDrvBinary*  b_small[SMALL_IO_QUEUE];
-} ErlIOQueue;
 
 typedef struct line_buf {  /* Buffer used in line oriented I/O */
     ErlDrvSizeT bufsiz;      /* Size of character buffer */
@@ -172,7 +160,7 @@ struct _erl_drv_port {
     Uint bytes_in;		/* Number of bytes read */
     Uint bytes_out;		/* Number of bytes written */
 
-    ErlIOQueue ioq;              /* driver accessible i/o queue */
+    ErlPortIOQueue ioq;          /* driver accessible i/o queue */
     DistEntry *dist_entry;       /* Dist entry used in DISTRIBUTION */
     char *name;		         /* String used in the open */
     erts_driver_t* drv_ptr;
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index b609f6de39..3a5ddde5f4 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -52,6 +52,7 @@
 #include "erl_bif_unique.h"
 #include "erl_hl_timer.h"
 #include "erl_time.h"
+#include "erl_io_queue.h"
 
 extern ErlDrvEntry fd_driver_entry;
 extern ErlDrvEntry vanilla_driver_entry;
@@ -108,7 +109,7 @@ static void driver_monitor_unlock_pdl(Port *p);
 #define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT)
 #define SMALL_WRITE_VEC  16
 
-static ERTS_INLINE ErlIOQueue*
+static ERTS_INLINE ErlPortIOQueue*
 drvport2ioq(ErlDrvPort drvport)
 {
     Port *prt = erts_thr_drvport2port(drvport, 0);
@@ -123,11 +124,11 @@ is_port_ioq_empty(Port *pp)
     int res;
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp));
     if (!pp->port_data_lock)
-	res = (pp->ioq.size == 0);
+	res = (erts_ioq_size(&pp->ioq) == 0);
     else {
 	ErlDrvPDL pdl = pp->port_data_lock;
 	erts_mtx_lock(&pdl->mtx);
-	res = (pp->ioq.size == 0);
+	res = (erts_ioq_size(&pp->ioq) == 0);
 	erts_mtx_unlock(&pdl->mtx);
     }
     return res;
@@ -142,14 +143,14 @@ erts_is_port_ioq_empty(Port *pp)
 Uint
 erts_port_ioq_size(Port *pp)
 {
-    int res;
+    ErlDrvSizeT res;
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp));
     if (!pp->port_data_lock)
-	res = pp->ioq.size;
+	res = erts_ioq_size(&pp->ioq);
     else {
 	ErlDrvPDL pdl = pp->port_data_lock;
 	erts_mtx_lock(&pdl->mtx);
-	res = pp->ioq.size;
+	res = erts_ioq_size(&pp->ioq);
 	erts_mtx_unlock(&pdl->mtx);
     }
     return (Uint) res;
@@ -508,41 +509,17 @@ erts_port_free(Port *prt)
 */
 static void initq(Port* prt)
 {
-    ErlIOQueue* q = &prt->ioq;
-
     ERTS_LC_ASSERT(!prt->port_data_lock);
-
-    q->size = 0;
-    q->v_head = q->v_tail = q->v_start = q->v_small;
-    q->v_end = q->v_small + SMALL_IO_QUEUE;
-    q->b_head = q->b_tail = q->b_start = q->b_small;
-    q->b_end = q->b_small + SMALL_IO_QUEUE;
+    erts_ioq_init(&prt->ioq, ERTS_ALC_T_IOQ, 1);
 }
 
 static void stopq(Port* prt)
 {
-    ErlIOQueue* q;
-    ErlDrvBinary** binp;
 
     if (prt->port_data_lock)
 	driver_pdl_lock(prt->port_data_lock);
 
-    q = &prt->ioq;
-    binp = q->b_head;
-
-    if (q->v_start != q->v_small)
-	erts_free(ERTS_ALC_T_IOQ, (void *) q->v_start);
-
-    while(binp < q->b_tail) {
-	if (*binp != NULL)
-	    driver_free_binary(*binp);
-	binp++;
-    }
-    if (q->b_start != q->b_small)
-	erts_free(ERTS_ALC_T_IOQ, (void *) q->b_start);
-    q->v_start = q->v_end = q->v_head = q->v_tail = NULL;
-    q->b_start = q->b_end = q->b_head = q->b_tail = NULL;
-    q->size = 0;
+    erts_ioq_clear(&prt->ioq);
 
     if (prt->port_data_lock) {
 	driver_pdl_unlock(prt->port_data_lock);
@@ -923,311 +900,6 @@ int erts_port_handle_xports(Port *prt)
 }
 #endif
 
-/* Fills a possibly deep list of chars and binaries into vec
-** Small characters are first stored in the buffer buf of length ln
-** binaries found are copied and linked into msoh
-** Return  vector length on succsess,
-**        -1 on overflow
-**        -2 on type error
-*/
-
-#ifdef DEBUG
-#define MAX_SYSIOVEC_IOVLEN (1ull << (32 - 1))
-#else
-#define MAX_SYSIOVEC_IOVLEN (1ull << (sizeof(((SysIOVec*)0)->iov_len) * 8 - 1))
-#endif
-
-static ERTS_INLINE void
-io_list_to_vec_set_vec(SysIOVec **iov, ErlDrvBinary ***binv,
-                        ErlDrvBinary *bin, byte *ptr, Uint len,
-                        int *vlen)
-{
-    while (len > MAX_SYSIOVEC_IOVLEN) {
-        (*iov)->iov_base = ptr;
-        (*iov)->iov_len = MAX_SYSIOVEC_IOVLEN;
-        ptr += MAX_SYSIOVEC_IOVLEN;
-        len -= MAX_SYSIOVEC_IOVLEN;
-        (*iov)++;
-        (*vlen)++;
-        *(*binv)++ = bin;
-    }
-    (*iov)->iov_base = ptr;
-    (*iov)->iov_len = len;
-    *(*binv)++ = bin;
-    (*iov)++;
-    (*vlen)++;
-}
-
-static int
-io_list_to_vec(Eterm obj,	/* io-list */
-	       SysIOVec* iov,	/* io vector */
-	       ErlDrvBinary** binv, /* binary reference vector */
-	       ErlDrvBinary* cbin, /* binary to store characters */
-	       ErlDrvSizeT bin_limit)	/* small binaries limit */
-{
-    DECLARE_ESTACK(s);
-    Eterm* objp;
-    byte *buf  = (byte*)cbin->orig_bytes;
-    Uint len = cbin->orig_size;
-    Uint csize  = 0;
-    int vlen   = 0;
-    byte* cptr = buf;
-
-    goto L_jump_start;  /* avoid push */
-
-    while (!ESTACK_ISEMPTY(s)) {
-	obj = ESTACK_POP(s);
-    L_jump_start:
-	if (is_list(obj)) {
-	L_iter_list:
-	    objp = list_val(obj);
-	    obj = CAR(objp);
-	    if (is_byte(obj)) {
-		if (len == 0)
-		    goto L_overflow;
-		*buf++ = unsigned_val(obj);
-		csize++;
-		len--;
-	    } else if (is_binary(obj)) {
-		ESTACK_PUSH(s, CDR(objp));
-		goto handle_binary;
-	    } else if (is_list(obj)) {
-		ESTACK_PUSH(s, CDR(objp));
-		goto L_iter_list;    /* on head */
-	    } else if (!is_nil(obj)) {
-		goto L_type_error;
-	    }	    
-	    obj = CDR(objp);
-	    if (is_list(obj))
-		goto L_iter_list; /* on tail */
-	    else if (is_binary(obj)) {
-		goto handle_binary;
-	    } else if (!is_nil(obj)) {
-		goto L_type_error;
-	    }
-	} else if (is_binary(obj)) {
-	    Eterm real_bin;
-	    Uint offset;
-	    Eterm* bptr;
-	    ErlDrvSizeT size;
-	    int bitoffs;
-	    int bitsize;
-
-	handle_binary:
-	    size = binary_size(obj);
-	    ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize);
-	    ASSERT(bitsize == 0);
-	    bptr = binary_val(real_bin);
-	    if (*bptr == HEADER_PROC_BIN) {
-		ProcBin* pb = (ProcBin *) bptr;
-		if (bitoffs != 0) {
-		    if (len < size) {
-			goto L_overflow;
-		    }
-		    erts_copy_bits(pb->bytes+offset, bitoffs, 1,
-				   (byte *) buf, 0, 1, size*8);
-		    csize += size;
-		    buf += size;
-		    len -= size;
-		} else if (bin_limit && size < bin_limit) {
-		    if (len < size) {
-			goto L_overflow;
-		    }
-		    sys_memcpy(buf, pb->bytes+offset, size);
-		    csize += size;
-		    buf += size;
-		    len -= size;
-		} else {
-		    if (csize != 0) {
-                        io_list_to_vec_set_vec(&iov, &binv, cbin,
-                                               cptr, csize, &vlen);
-			cptr = buf;
-			csize = 0;
-		    }
-		    if (pb->flags) {
-			erts_emasculate_writable_binary(pb);
-		    }
-                    io_list_to_vec_set_vec(
-                        &iov, &binv, Binary2ErlDrvBinary(pb->val),
-                        pb->bytes+offset, size, &vlen);
-		}
-	    } else {
-		ErlHeapBin* hb = (ErlHeapBin *) bptr;
-		if (len < size) {
-		    goto L_overflow;
-		}
-		copy_binary_to_buffer(buf, 0,
-				      ((byte *) hb->data)+offset, bitoffs,
-				      8*size);
-		csize += size;
-		buf += size;
-		len -= size;
-	    }
-	} else if (!is_nil(obj)) {
-	    goto L_type_error;
-	}
-    }
-
-    if (csize != 0) {
-        io_list_to_vec_set_vec(&iov, &binv, cbin, cptr, csize, &vlen);
-    }
-
-    DESTROY_ESTACK(s);
-    return vlen;
-
- L_type_error:
-    DESTROY_ESTACK(s);
-    return -2;
-
- L_overflow:
-    DESTROY_ESTACK(s);
-    return -1;
-}
-
-#define IO_LIST_VEC_COUNT(obj)						\
-do {									\
-    Uint _size = binary_size(obj);					\
-    Eterm _real;							\
-    ERTS_DECLARE_DUMMY(Uint _offset);					\
-    int _bitoffs;							\
-    int _bitsize;							\
-    ERTS_GET_REAL_BIN(obj, _real, _offset, _bitoffs, _bitsize);		\
-    if (_bitsize != 0) goto L_type_error;				\
-    if (thing_subtag(*binary_val(_real)) == REFC_BINARY_SUBTAG &&	\
-	_bitoffs == 0) {						\
-	b_size += _size;                                                \
-        if (b_size < _size) goto L_overflow_error;			\
-	in_clist = 0;							\
-        v_size++;                                                       \
-        /* If iov_len is smaller then Uint we split the binary into*/   \
-        /* multiple smaller (2GB) elements in the iolist.*/             \
-	v_size += _size / MAX_SYSIOVEC_IOVLEN;                          \
-        if (_size >= ERL_SMALL_IO_BIN_LIMIT) {				\
-            p_in_clist = 0;						\
-            p_v_size++;							\
-        } else {							\
-            p_c_size += _size;						\
-            if (!p_in_clist) {						\
-                p_in_clist = 1;						\
-                p_v_size++;						\
-            }								\
-        }								\
-    } else {								\
-	c_size += _size;						\
-        if (c_size < _size) goto L_overflow_error;			\
-	if (!in_clist) {						\
-	    in_clist = 1;						\
-	    v_size++;							\
-	}								\
-	p_c_size += _size;						\
-	if (!p_in_clist) {						\
-	    p_in_clist = 1;						\
-	    p_v_size++;							\
-	}								\
-    }									\
-} while (0)
-
-
-/* 
- * Returns 0 if successful and a non-zero value otherwise.
- *
- * Return values through pointers:
- *    *vsize      - SysIOVec size needed for a writev
- *    *csize      - Number of bytes not in binary (in the common binary)
- *    *pvsize     - SysIOVec size needed if packing small binaries
- *    *pcsize     - Number of bytes in the common binary if packing
- *    *total_size - Total size of iolist in bytes
- */
-
-static int 
-io_list_vec_len(Eterm obj, int* vsize, Uint* csize,
-		Uint* pvsize, Uint* pcsize,
-		ErlDrvSizeT* total_size)
-{
-    DECLARE_ESTACK(s);
-    Eterm* objp;
-    Uint v_size = 0;
-    Uint c_size = 0;
-    Uint b_size = 0;
-    Uint in_clist = 0;
-    Uint p_v_size = 0;
-    Uint p_c_size = 0;
-    Uint p_in_clist = 0;
-    Uint total;
-
-    goto L_jump_start;  /* avoid a push */
-
-    while (!ESTACK_ISEMPTY(s)) {
-	obj = ESTACK_POP(s);
-    L_jump_start:
-	if (is_list(obj)) {
-	L_iter_list:
-	    objp = list_val(obj);
-	    obj = CAR(objp);
-
-	    if (is_byte(obj)) {
-		c_size++;
-		if (c_size == 0) {
-		    goto L_overflow_error;
-		}
-		if (!in_clist) {
-		    in_clist = 1;
-		    v_size++;
-		}
-		p_c_size++;
-		if (!p_in_clist) {
-		    p_in_clist = 1;
-		    p_v_size++;
-		}
-	    }
-	    else if (is_binary(obj)) {
-		IO_LIST_VEC_COUNT(obj);
-	    }
-	    else if (is_list(obj)) {
-		ESTACK_PUSH(s, CDR(objp));
-		goto L_iter_list;   /* on head */
-	    }
-	    else if (!is_nil(obj)) {
-		goto L_type_error;
-	    }
-
-	    obj = CDR(objp);
-	    if (is_list(obj))
-		goto L_iter_list;   /* on tail */
-	    else if (is_binary(obj)) {  /* binary tail is OK */
-		IO_LIST_VEC_COUNT(obj);
-	    }
-	    else if (!is_nil(obj)) {
-		goto L_type_error;
-	    }
-	}
-	else if (is_binary(obj)) {
-	    IO_LIST_VEC_COUNT(obj);
-	}
-	else if (!is_nil(obj)) {
-	    goto L_type_error;
-	}
-    }
-
-    total = c_size + b_size;
-    if (total < c_size) {
-	goto L_overflow_error;
-    }
-    *total_size = (ErlDrvSizeT) total;
-
-    DESTROY_ESTACK(s);
-    *vsize = v_size;
-    *csize = c_size;
-    *pvsize = p_v_size;
-    *pcsize = p_c_size;
-    return 0;
-
- L_type_error:
- L_overflow_error:
-    DESTROY_ESTACK(s);
-    return 1;
-}
-
 typedef enum {
     ERTS_TRY_IMM_DRV_CALL_OK,
     ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK,
@@ -1797,8 +1469,7 @@ cleanup_scheduled_outputv(ErlIOVec *ev, ErlDrvBinary *cbinp)
     int i;
     /* Need to free all binaries */
     for (i = 1; i < ev->vsize; i++)
-	if (ev->binv[i])
-	    driver_free_binary(ev->binv[i]);
+        driver_free_binary(ev->binv[i]);
     if (cbinp)
 	driver_free_binary(cbinp);
 }
@@ -1966,15 +1637,14 @@ erts_port_output_async(Port *prt, Eterm from, Eterm list)
     size_t size;
     int task_flags;
     ErtsProc2PortSigCallback port_sig_callback;
-    ErlDrvBinary *cbin = NULL;
-    ErlIOVec *evp = NULL;
+    ErtsIOQBinary *cbin = NULL;
+    ErtsIOVec *evp = NULL;
     char *buf = NULL;
     ErtsPortTaskHandle *ns_pthp;
 
     if (drv->outputv) {
-        ErlIOVec ev;
 	SysIOVec* ivp;
-	ErlDrvBinary**  bvp;
+	ErtsIOQBinary**  bvp;
         int vsize;
 	Uint csize;
 	Uint pvsize;
@@ -1984,91 +1654,63 @@ erts_port_output_async(Port *prt, Eterm from, Eterm list)
         char *ptr;
         int i;
 
-        Eterm* bptr = NULL;
-        Uint offset;
-
-        if (is_binary(list)) {
-            /* We optimize for when we get a procbin without offset */
-            Eterm real_bin;
-            int bitoffs;
-            int bitsize;
-            ERTS_GET_REAL_BIN(list, real_bin, offset, bitoffs, bitsize);
-            bptr = binary_val(real_bin);
-            if (*bptr == HEADER_PROC_BIN && bitoffs == 0) {
-                size = binary_size(list);
-                vsize = 1;
-            } else
-                bptr = NULL;
-        }
-
-        if (!bptr) {
-            if (io_list_vec_len(list, &vsize, &csize, &pvsize, &pcsize, &size))
-                goto bad_value;
+        if (erts_ioq_iodata_vec_len(list, &vsize, &csize, &pvsize, &pcsize,
+                                    &size, ERL_SMALL_IO_BIN_LIMIT))
+            goto bad_value;
 
-            /* To pack or not to pack (small binaries) ...? */
-            if (vsize >= SMALL_WRITE_VEC) {
-                /* Do pack */
-                vsize = pvsize + 1;
-                csize = pcsize;
-                blimit = ERL_SMALL_IO_BIN_LIMIT;
-            }
-            cbin = driver_alloc_binary(csize);
+        /* To pack or not to pack (small binaries) ...? */
+        if (vsize >= SMALL_WRITE_VEC) {
+            /* Do pack */
+            vsize = pvsize + 1;
+            csize = pcsize;
+            blimit = ERL_SMALL_IO_BIN_LIMIT;
+        }
+        if (csize) {
+            cbin = (ErtsIOQBinary *)driver_alloc_binary(csize);
             if (!cbin)
                 erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, ERTS_SIZEOF_Binary(csize));
         }
 
-
         iov_offset = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErlIOVec));
 	binv_offset = iov_offset;
         binv_offset += ERTS_ALC_DATA_ALIGN_SIZE((vsize+1)*sizeof(SysIOVec));
         alloc_size = binv_offset;
-	alloc_size += (vsize+1)*sizeof(ErlDrvBinary *);
+	alloc_size += (vsize+1)*sizeof(ErtsIOQBinary *);
 
         sigdp = erts_port_task_alloc_p2p_sig_data_extra(alloc_size, (void**)&ptr);
 
-        evp = (ErlIOVec *) ptr;
-        ivp = evp->iov = (SysIOVec *) (ptr + iov_offset);
-        bvp = evp->binv = (ErlDrvBinary **) (ptr + binv_offset);
+        evp = (ErtsIOVec *) ptr;
+        ivp = evp->driver.iov = (SysIOVec *) (ptr + iov_offset);
+        bvp = evp->common.binv = (ErtsIOQBinary **) (ptr + binv_offset);
 
         ivp[0].iov_base = NULL;
 	ivp[0].iov_len = 0;
 	bvp[0] = NULL;
 
-        if (bptr) {
-            ProcBin* pb = (ProcBin *) bptr;
-
-            ivp[1].iov_base = pb->bytes+offset;
-            ivp[1].iov_len = size;
-            bvp[1] = Binary2ErlDrvBinary(pb->val);
-
-            evp->vsize = 1;
-        } else {
-
-            evp->vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit);
-            if (evp->vsize < 0) {
-                if (evp != &ev)
-                    erts_free(ERTS_ALC_T_DRV_CMD_DATA, evp);
-                driver_free_binary(cbin);
-                goto bad_value;
-            }
+        evp->driver.vsize = erts_ioq_iodata_to_vec(list, ivp+1, bvp+1, cbin,
+                                                   blimit, 1);
+        if (evp->driver.vsize < 0) {
+            erts_free(ERTS_ALC_T_DRV_CMD_DATA, evp);
+            driver_free_binary(&cbin->driver);
+            goto bad_value;
         }
 #if 0
 	/* This assertion may say something useful, but it can
         be falsified during the emulator test suites. */
 	ASSERT(evp->vsize == vsize);
 #endif
-	evp->vsize++;
-	evp->size = size;  /* total size */
+	evp->driver.vsize++;
+	evp->driver.size = size;  /* total size */
 
         /* Need to increase refc on all binaries */
-        for (i = 1; i < evp->vsize; i++)
+        for (i = 1; i < evp->driver.vsize; i++)
             if (bvp[i])
-                driver_binary_inc_refc(bvp[i]);
+                driver_binary_inc_refc(&bvp[i]->driver);
 
 	sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUTV;
 	sigdp->u.outputv.from = from;
-	sigdp->u.outputv.evp = evp;
-	sigdp->u.outputv.cbinp = cbin;
+	sigdp->u.outputv.evp = &evp->driver;
+	sigdp->u.outputv.cbinp = &cbin->driver;
 	port_sig_callback = port_sig_outputv;
     } else {
         ErlDrvSizeT ERTS_DECLARE_DUMMY(r);
@@ -2139,8 +1781,8 @@ erts_port_output(Process *c_p,
     erts_aint32_t sched_flags, busy_flgs, invalid_flags;
     int task_flags;
     ErtsProc2PortSigCallback port_sig_callback;
-    ErlDrvBinary *cbin = NULL;
-    ErlIOVec *evp = NULL;
+    ErtsIOQBinary *cbin = NULL;
+    ErtsIOVec *evp = NULL;
     char *buf = NULL;
     int force_immediate_call = (flags & ERTS_PORT_SIG_FLG_FORCE_IMM_CALL);
     int async_nosuspend;
@@ -2186,11 +1828,11 @@ erts_port_output(Process *c_p,
     }
 #endif
     if (drv->outputv) {
-	ErlIOVec ev;
+	ErtsIOVec ev;
 	SysIOVec iv[SMALL_WRITE_VEC];
-	ErlDrvBinary* bv[SMALL_WRITE_VEC];
+	ErtsIOQBinary* bv[SMALL_WRITE_VEC];
 	SysIOVec* ivp;
-	ErlDrvBinary**  bvp;
+	ErtsIOQBinary**  bvp;
 	int vsize;
 	Uint csize;
 	Uint pvsize;
@@ -2198,18 +1840,19 @@ erts_port_output(Process *c_p,
 	Uint blimit;
 	size_t iov_offset, binv_offset, alloc_size;
 
-	if (io_list_vec_len(list, &vsize, &csize, &pvsize, &pcsize, &size))
+	if (erts_ioq_iodata_vec_len(list, &vsize, &csize, &pvsize, &pcsize,
+                                    &size, ERL_SMALL_IO_BIN_LIMIT))
 	    goto bad_value;
 
 	iov_offset = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErlIOVec));
 	binv_offset = iov_offset;
 	binv_offset += ERTS_ALC_DATA_ALIGN_SIZE((vsize+1)*sizeof(SysIOVec));
 	alloc_size = binv_offset;
-	alloc_size += (vsize+1)*sizeof(ErlDrvBinary *);
+	alloc_size += (vsize+1)*sizeof(ErtsIOQBinary *);
 
 	if (try_call && vsize < SMALL_WRITE_VEC) {
-	    ivp = ev.iov = iv;
-	    bvp = ev.binv = bv;
+	    ivp = ev.common.iov = iv;
+	    bvp = ev.common.binv = bv;
 	    evp = &ev;
 	}
 	else {
@@ -2220,9 +1863,9 @@ erts_port_output(Process *c_p,
                 sigdp = erts_port_task_alloc_p2p_sig_data_extra(
                     alloc_size, (void**)&ptr);
             }
-	    evp = (ErlIOVec *) ptr;
-	    ivp = evp->iov = (SysIOVec *) (ptr + iov_offset);
-	    bvp = evp->binv = (ErlDrvBinary **) (ptr + binv_offset);
+	    evp = (ErtsIOVec *) ptr;
+	    ivp = evp->driver.iov = (SysIOVec *) (ptr + iov_offset);
+	    bvp = evp->common.binv = (ErtsIOQBinary **) (ptr + binv_offset);
 	}
 
 	/* To pack or not to pack (small binaries) ...? */
@@ -2238,23 +1881,26 @@ erts_port_output(Process *c_p,
 	}
 	/* Use vsize and csize from now on */
 
-	cbin = driver_alloc_binary(csize);
-	if (!cbin)
-	    erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, ERTS_SIZEOF_Binary(csize));
+        if (csize) {
+            cbin = (ErtsIOQBinary *)driver_alloc_binary(csize);
+            if (!cbin)
+                erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, ERTS_SIZEOF_Binary(csize));
+        }
 
 	/* Element 0 is for driver usage to add header block */
 	ivp[0].iov_base = NULL;
 	ivp[0].iov_len = 0;
 	bvp[0] = NULL;
-	evp->vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit);
-	if (evp->vsize < 0) {
+	evp->driver.vsize = erts_ioq_iodata_to_vec(list, ivp+1, bvp+1,
+                                                   cbin, blimit, 1);
+	if (evp->driver.vsize < 0) {
             if (evp != &ev) {
                 if (try_call)
                     erts_free(ERTS_ALC_T_TMP, evp);
                 else
                     erts_port_task_free_p2p_sig_data(sigdp);
             }
-	    driver_free_binary(cbin);
+            driver_free_binary(&cbin->driver);
 	    goto bad_value;
 	}
 #if 0
@@ -2262,19 +1908,19 @@ erts_port_output(Process *c_p,
 	   be falsified during the emulator test suites. */
 	ASSERT(evp->vsize == vsize);
 #endif
-	evp->vsize++;
-	evp->size = size;  /* total size */
+	evp->driver.vsize++;
+	evp->driver.size = size;  /* total size */
 
 	if (!try_call) {
 	    int i;
 	    /* Need to increase refc on all binaries */
-	    for (i = 1; i < evp->vsize; i++)
-		if (bvp[i])
-		    driver_binary_inc_refc(bvp[i]);
+	    for (i = 1; i < evp->driver.vsize; i++)
+                if (bvp[i])
+                    driver_binary_inc_refc(&bvp[i]->driver);
 	}
 	else {
 	    int i;
-	    ErlIOVec *new_evp;
+	    ErtsIOVec *new_evp;
 	    ErtsTryImmDrvCallResult try_call_res;
 	    ErtsTryImmDrvCallState try_call_state
 		= ERTS_INIT_TRY_IMM_DRV_CALL_STATE(
@@ -2297,14 +1943,14 @@ erts_port_output(Process *c_p,
 				    from,
 				    prt,
 				    drv,
-				    evp);
+				    &evp->driver);
 		if (force_immediate_call)
 		    finalize_force_imm_drv_call(&try_call_state);
 		else
 		    finalize_imm_drv_call(&try_call_state);
 		/* Fall through... */
 	    case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
-		driver_free_binary(cbin);
+		driver_free_binary(&cbin->driver);
 		if (evp != &ev) {
                     ASSERT(!sigdp);
 		    erts_free(ERTS_ALC_T_TMP, evp);
@@ -2318,7 +1964,7 @@ erts_port_output(Process *c_p,
 		sched_flags = try_call_state.sched_flags;
 		if (async_nosuspend
 		    && (sched_flags & (busy_flgs|ERTS_PTS_FLG_EXIT))) {
-		    driver_free_binary(cbin);
+		    driver_free_binary(&cbin->driver);
                     if (evp != &ev) {
                         ASSERT(!sigdp);
 			erts_free(ERTS_ALC_T_TMP, evp);
@@ -2333,9 +1979,9 @@ erts_port_output(Process *c_p,
 	    }
 
 	    /* Need to increase refc on all binaries */
-	    for (i = 1; i < evp->vsize; i++)
+	    for (i = 1; i < evp->driver.vsize; i++)
 		if (bvp[i])
-		    driver_binary_inc_refc(bvp[i]);
+		    driver_binary_inc_refc(&bvp[i]->driver);
 
             /* The port task and iovec is allocated in the
                same structure as an optimization. This
@@ -2348,18 +1994,18 @@ erts_port_output(Process *c_p,
 	    if (evp != &ev) {
                 /* Copy from TMP alloc to port task */
 		sys_memcpy((void *) new_evp, (void *) evp, alloc_size);
-		new_evp->iov = (SysIOVec *) (((char *) new_evp)
-					     + iov_offset);
-		bvp = new_evp->binv = (ErlDrvBinary **) (((char *) new_evp)
-							 + binv_offset);
+		new_evp->driver.iov = (SysIOVec *) (((char *) new_evp)
+                                                    + iov_offset);
+		bvp = new_evp->common.binv = (ErtsIOQBinary **) (((char *) new_evp)
+                                                                 + binv_offset);
 
 #ifdef DEBUG
-		ASSERT(new_evp->vsize == evp->vsize);
-		ASSERT(new_evp->size == evp->size);
-		for (i = 0; i < evp->vsize; i++) {
-		    ASSERT(new_evp->iov[i].iov_len == evp->iov[i].iov_len);
-		    ASSERT(new_evp->iov[i].iov_base == evp->iov[i].iov_base);
-		    ASSERT(new_evp->binv[i] == evp->binv[i]);
+		ASSERT(new_evp->driver.vsize == evp->driver.vsize);
+		ASSERT(new_evp->driver.size == evp->driver.size);
+		for (i = 0; i < evp->driver.vsize; i++) {
+		    ASSERT(new_evp->driver.iov[i].iov_len == evp->driver.iov[i].iov_len);
+		    ASSERT(new_evp->driver.iov[i].iov_base == evp->driver.iov[i].iov_base);
+		    ASSERT(new_evp->driver.binv[i] == evp->driver.binv[i]);
 		}
 #endif
 
@@ -2368,24 +2014,24 @@ erts_port_output(Process *c_p,
 	    else { /* from stack allocated structure; offsets may differ */
 
 		sys_memcpy((void *) new_evp, (void *) evp, sizeof(ErlIOVec));
-		new_evp->iov = (SysIOVec *) (((char *) new_evp)
-					     + iov_offset);
-		sys_memcpy((void *) new_evp->iov,
-			   (void *) evp->iov,
-			   evp->vsize * sizeof(SysIOVec));
-		new_evp->binv = (ErlDrvBinary **) (((char *) new_evp)
-						   + binv_offset);
-		sys_memcpy((void *) new_evp->binv,
-			   (void *) evp->binv,
-			   evp->vsize * sizeof(ErlDrvBinary *));
+		new_evp->driver.iov = (SysIOVec *) (((char *) new_evp)
+                                                    + iov_offset);
+		sys_memcpy((void *) new_evp->driver.iov,
+			   (void *) evp->driver.iov,
+			   evp->driver.vsize * sizeof(SysIOVec));
+		new_evp->common.binv = (ErtsIOQBinary **) (((char *) new_evp)
+                                                           + binv_offset);
+		sys_memcpy((void *) new_evp->common.binv,
+			   (void *) evp->common.binv,
+			   evp->driver.vsize * sizeof(ErtsIOQBinary *));
 
 #ifdef DEBUG
-		ASSERT(new_evp->vsize == evp->vsize);
-		ASSERT(new_evp->size == evp->size);
-		for (i = 0; i < evp->vsize; i++) {
-		    ASSERT(new_evp->iov[i].iov_len == evp->iov[i].iov_len);
-		    ASSERT(new_evp->iov[i].iov_base == evp->iov[i].iov_base);
-		    ASSERT(new_evp->binv[i] == evp->binv[i]);
+		ASSERT(new_evp->driver.vsize == evp->driver.vsize);
+		ASSERT(new_evp->driver.size == evp->driver.size);
+		for (i = 0; i < evp->driver.vsize; i++) {
+		    ASSERT(new_evp->driver.iov[i].iov_len == evp->driver.iov[i].iov_len);
+		    ASSERT(new_evp->driver.iov[i].iov_base == evp->driver.iov[i].iov_base);
+		    ASSERT(new_evp->driver.binv[i] == evp->driver.binv[i]);
 		}
 #endif
 
@@ -2396,8 +2042,8 @@ erts_port_output(Process *c_p,
 
 	sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUTV;
 	sigdp->u.outputv.from = from;
-	sigdp->u.outputv.evp = evp;
-	sigdp->u.outputv.cbinp = cbin;
+	sigdp->u.outputv.evp = &evp->driver;
+	sigdp->u.outputv.cbinp = &cbin->driver;
 	port_sig_callback = port_sig_outputv;
     }
     else {
@@ -7154,307 +6800,51 @@ driver_pdl_dec_refc(ErlDrvPDL pdl)
     return refc;
 }
 
-/* expand queue to hold n elements in tail or head */
-static int expandq(ErlIOQueue* q, int n, int tail)
-/* tail: 0 if make room in head, make room in tail otherwise */
-{
-    int h_sz;  /* room before header */
-    int t_sz;  /* room after tail */
-    int q_sz;  /* occupied */
-    int nvsz;
-    SysIOVec* niov;
-    ErlDrvBinary** nbinv;
-
-    h_sz = q->v_head - q->v_start;
-    t_sz = q->v_end -  q->v_tail;
-    q_sz = q->v_tail - q->v_head;
-
-    if (tail && (n <= t_sz)) /* do we need to expand tail? */
-	return 0;
-    else if (!tail && (n <= h_sz))  /* do we need to expand head? */
-	return 0;
-    else if (n > (h_sz + t_sz)) { /* need to allocate */
-	/* we may get little extra but it ok */
-	nvsz = (q->v_end - q->v_start) + n; 
-
-	niov = erts_alloc_fnf(ERTS_ALC_T_IOQ, nvsz * sizeof(SysIOVec));
-	if (!niov)
-	    return -1;
-	nbinv = erts_alloc_fnf(ERTS_ALC_T_IOQ, nvsz * sizeof(ErlDrvBinary**));
-	if (!nbinv) {
-	    erts_free(ERTS_ALC_T_IOQ, (void *) niov);
-	    return -1;
-	}
-	if (tail) {
-	    sys_memcpy(niov, q->v_head, q_sz*sizeof(SysIOVec));
-	    if (q->v_start != q->v_small)
-		erts_free(ERTS_ALC_T_IOQ, (void *) q->v_start);
-	    q->v_start = niov;
-	    q->v_end = niov + nvsz;
-	    q->v_head = q->v_start;
-	    q->v_tail = q->v_head + q_sz;
-
-	    sys_memcpy(nbinv, q->b_head, q_sz*sizeof(ErlDrvBinary*));
-	    if (q->b_start != q->b_small)
-		erts_free(ERTS_ALC_T_IOQ, (void *) q->b_start);
-	    q->b_start = nbinv;
-	    q->b_end = nbinv + nvsz;
-	    q->b_head = q->b_start;
-	    q->b_tail = q->b_head + q_sz;	
-	}
-	else {
-	    sys_memcpy(niov+nvsz-q_sz, q->v_head, q_sz*sizeof(SysIOVec));
-	    if (q->v_start != q->v_small)
-		erts_free(ERTS_ALC_T_IOQ, (void *) q->v_start);
-	    q->v_start = niov;
-	    q->v_end = niov + nvsz;
-	    q->v_tail = q->v_end;
-	    q->v_head = q->v_tail - q_sz;
-	    
-	    sys_memcpy(nbinv+nvsz-q_sz, q->b_head, q_sz*sizeof(ErlDrvBinary*));
-	    if (q->b_start != q->b_small)
-		erts_free(ERTS_ALC_T_IOQ, (void *) q->b_start);
-	    q->b_start = nbinv;
-	    q->b_end = nbinv + nvsz;
-	    q->b_tail = q->b_end;
-	    q->b_head = q->b_tail - q_sz;
-	}
-    }
-    else if (tail) {  /* move to beginning to make room in tail */
-	sys_memmove(q->v_start, q->v_head, q_sz*sizeof(SysIOVec));
-	q->v_head = q->v_start;
-	q->v_tail = q->v_head + q_sz;
-	sys_memmove(q->b_start, q->b_head, q_sz*sizeof(ErlDrvBinary*));
-	q->b_head = q->b_start;
-	q->b_tail = q->b_head + q_sz;
-    }
-    else {   /* move to end to make room */
-	sys_memmove(q->v_end-q_sz, q->v_head, q_sz*sizeof(SysIOVec));
-	q->v_tail = q->v_end;
-	q->v_head = q->v_tail-q_sz;
-	sys_memmove(q->b_end-q_sz, q->b_head, q_sz*sizeof(ErlDrvBinary*));
-	q->b_tail = q->b_end;
-	q->b_head = q->b_tail-q_sz;
-    }
-
-    return 0;
-}
-
-
-
 /* Put elements from vec at q tail */
 int driver_enqv(ErlDrvPort ix, ErlIOVec* vec, ErlDrvSizeT skip)
 {
-    int n;
-    size_t len;
-    ErlDrvSizeT size;
-    SysIOVec* iov;
-    ErlDrvBinary** binv;
-    ErlDrvBinary*  b;
-    ErlIOQueue* q = drvport2ioq(ix);
-
-    if (q == NULL)
-	return -1;
-
-    ASSERT(vec->size >= skip);       /* debug only */
-    if (vec->size <= skip)
-	return 0;
-    size = vec->size - skip;
-
-    iov = vec->iov;
-    binv = vec->binv;
-    n = vec->vsize;
-
-    /* we use do here to strip iov_len=0 from beginning */
-    do {
-	len = iov->iov_len;
-	if (len <= skip) {
-	    skip -= len;
-	    iov++;
-	    binv++;
-	    n--;
-	}
-	else {
-	    iov->iov_base = ((char *)(iov->iov_base)) + skip;
-	    iov->iov_len -= skip;
-	    skip = 0;
-	}
-    } while(skip > 0);
-
-    if (q->v_tail + n >= q->v_end)
-	expandq(q, n, 1);
-
-    /* Queue and reference all binaries (remove zero length items) */
-    while(n--) {
-	if ((len = iov->iov_len) > 0) {
-	    if ((b = *binv) == NULL) { /* speical case create binary ! */
-		b = driver_alloc_binary(len);
-		sys_memcpy(b->orig_bytes, iov->iov_base, len);
-		*q->b_tail++ = b;
-		q->v_tail->iov_len = len;
-		q->v_tail->iov_base = b->orig_bytes;
-		q->v_tail++;
-	    }
-	    else {
-		driver_binary_inc_refc(b);
-		*q->b_tail++ = b;
-		*q->v_tail++ = *iov;
-	    }
-	}
-	iov++;
-	binv++;
-    }
-    q->size += size;      /* update total size in queue */
-    return 0;
+    ASSERT(vec->size >= skip);
+    return erts_ioq_enqv(drvport2ioq(ix), (ErtsIOVec*)vec, skip);
 }
 
 /* Put elements from vec at q head */
 int driver_pushqv(ErlDrvPort ix, ErlIOVec* vec, ErlDrvSizeT skip)
 {
-    int n;
-    size_t len;
-    ErlDrvSizeT size;
-    SysIOVec* iov;
-    ErlDrvBinary** binv;
-    ErlDrvBinary* b;
-    ErlIOQueue* q = drvport2ioq(ix);
-
-    if (q == NULL)
-	return -1;
-
-    if (vec->size <= skip)
-	return 0;
-    size = vec->size - skip;
-
-    iov = vec->iov;
-    binv = vec->binv;
-    n = vec->vsize;
-
-    /* we use do here to strip iov_len=0 from beginning */
-    do {
-	len = iov->iov_len;
-	if (len <= skip) {
-	    skip -= len;
-	    iov++;
-	    binv++;
-	    n--;
-	}
-	else {
-	    iov->iov_base = ((char *)(iov->iov_base)) + skip;
-	    iov->iov_len -= skip;
-	    skip = 0;
-	}
-    } while(skip > 0);
-
-    if (q->v_head - n < q->v_start)
-	expandq(q, n, 0);
-
-    /* Queue and reference all binaries (remove zero length items) */
-    iov += (n-1);  /* move to end */
-    binv += (n-1); /* move to end */
-    while(n--) {
-	if ((len = iov->iov_len) > 0) {
-	    if ((b = *binv) == NULL) { /* speical case create binary ! */
-		b = driver_alloc_binary(len);
-		sys_memcpy(b->orig_bytes, iov->iov_base, len);
-		*--q->b_head = b;
-		q->v_head--;
-		q->v_head->iov_len = len;
-		q->v_head->iov_base = b->orig_bytes;
-	    }
-	    else {
-		driver_binary_inc_refc(b);
-		*--q->b_head = b;
-		*--q->v_head = *iov;
-	    }
-	}
-	iov--;
-	binv--;
-    }
-    q->size += size;      /* update total size in queue */
-    return 0;
+    ASSERT(vec->size >= skip);
+    return erts_ioq_pushqv(drvport2ioq(ix), (ErtsIOVec*)vec, skip);
 }
 
-
 /*
 ** Remove size bytes from queue head
 ** Return number of bytes that remain in queue
 */
 ErlDrvSizeT driver_deq(ErlDrvPort ix, ErlDrvSizeT size)
 {
-    ErlIOQueue* q = drvport2ioq(ix);
-    ErlDrvSizeT len;
-
-    if ((q == NULL) || (q->size < size))
-	return -1;
-    q->size -= size;
-    while (size > 0) {
-	ASSERT(q->v_head != q->v_tail);
-
-	len = q->v_head->iov_len;
-	if (len <= size) {
-	    size -= len;
-	    driver_free_binary(*q->b_head);
-	    *q->b_head++ = NULL;
-	    q->v_head++;
-	}
-	else {
-	    q->v_head->iov_base = ((char *)(q->v_head->iov_base)) + size;
-	    q->v_head->iov_len -= size;
-	    size = 0;
-	}
-    }
-
-    /* restart pointers (optimised for enq) */
-    if (q->v_head == q->v_tail) {
-	q->v_head = q->v_tail = q->v_start;
-	q->b_head = q->b_tail = q->b_start;
-    }
-    return q->size;
+    ErlPortIOQueue *q = drvport2ioq(ix);
+    if (erts_ioq_deq(q, size) == -1)
+        return -1;
+    return erts_ioq_size(q);
 }
 
 
-ErlDrvSizeT driver_peekqv(ErlDrvPort ix, ErlIOVec *ev) {
-    ErlIOQueue *q = drvport2ioq(ix);
-    ASSERT(ev);
-
-    if (! q) {
-	return (ErlDrvSizeT) -1;
-    } else {
-	if ((ev->vsize = q->v_tail - q->v_head) == 0) {
-	    ev->size = 0;
-	    ev->iov = NULL;
-	    ev->binv = NULL;
-	} else {
-	    ev->size = q->size;
-	    ev->iov = q->v_head;
-	    ev->binv = q->b_head;
-	}
-	return q->size;
-    }
+ErlDrvSizeT driver_peekqv(ErlDrvPort ix, ErlIOVec *ev)
+{
+    return erts_ioq_peekqv(drvport2ioq(ix), (ErtsIOVec*)ev);
 }
 
 SysIOVec* driver_peekq(ErlDrvPort ix, int* vlenp)  /* length of io-vector */
 {
-    ErlIOQueue* q = drvport2ioq(ix);
-
-    if (q == NULL) {
-	*vlenp = -1;
-	return NULL;
-    }
-    if ((*vlenp = (q->v_tail - q->v_head)) == 0)
-	return NULL;
-    return q->v_head;
+    return erts_ioq_peekq(drvport2ioq(ix), vlenp);
 }
 
 
 ErlDrvSizeT driver_sizeq(ErlDrvPort ix)
 {
-    ErlIOQueue* q = drvport2ioq(ix);
+    ErlPortIOQueue *q = drvport2ioq(ix);
 
     if (q == NULL)
-	return (size_t) -1;
-    return q->size;
+	return (ErlDrvSizeT) -1;
+    return erts_ioq_size(q);
 }
 
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 0fb25c2082..a3305da47d 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -52,6 +52,7 @@
 #include "erl_ptab.h"
 #include "erl_check_io.h"
 #include "erl_bif_unique.h"
+#include "erl_io_queue.h"
 #define ERTS_WANT_TIMER_WHEEL_API
 #include "erl_time.h"
 #ifdef HIPE
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 0337274178..4811244b98 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -61,7 +61,8 @@
          nif_internal_hash_salted/1,
          nif_phash2/1,
          nif_whereis/1, nif_whereis_parallel/1,
-         nif_whereis_threaded/1, nif_whereis_proxy/1
+         nif_whereis_threaded/1, nif_whereis_proxy/1,
+         nif_ioq/1
 	]).
 
 -export([many_args_100/100]).
@@ -99,7 +100,8 @@ all() ->
      nif_internal_hash,
      nif_internal_hash_salted,
      nif_phash2,
-     nif_whereis, nif_whereis_parallel, nif_whereis_threaded].
+     nif_whereis, nif_whereis_parallel, nif_whereis_threaded,
+     nif_ioq].
 
 groups() ->
     [{G, [], api_repeaters()} || G <- api_groups()]
@@ -2957,6 +2959,180 @@ nif_whereis_proxy(Ref) ->
         {Ref, quit} ->
             ok
     end.
+nif_ioq(Config) ->
+    ensure_lib_loaded(Config),
+
+    Script =
+        [{create, a},
+
+         %% Test enq of erlang term binary
+         {enqb,   a},
+         {enqb,   a, 3},
+
+         %% Test enq of non-erlang term binary
+         {enqbraw,a},
+         {enqbraw,a, 5},
+         {peek,   a},
+         {deq,    a, 42},
+
+         %% Test enqv
+         {enqv,   a, 2, 100},
+         {deq,    a, all},
+
+         %% This skips all elements but one in the iolist
+         {enqv,   a, 5, iolist_size(nif_ioq_payload(5)) - 1},
+         {peek,   a},
+
+         %% Test to enqueue a bunch of refc binaries
+         {enqv,   a, [nif_ioq_payload(refcbin) || _ <- lists:seq(1,20)], 0},
+
+         %% Enq stuff to destroy with data in queue
+         {enqv,   a, 2, 100},
+         {destroy,a},
+
+         %% Test destroy of new queue
+         {create, a},
+         {destroy,a}
+        ],
+
+    nif_ioq_run(Script),
+
+    %% Test that only enif_inspect_as_vec works
+    Payload = nif_ioq_payload(5),
+    PayloadBin = iolist_to_binary(Payload),
+
+    [begin
+         PayloadBin = iolist_to_binary(ioq_nif(inspect,Payload,Stack,Env)),
+         <<>>       = iolist_to_binary(ioq_nif(inspect,[],Stack,Env))
+     end || Stack <- [no_stack, use_stack], Env <- [use_env, no_env]],
+
+    %% Test error cases
+
+    Q = ioq_nif(create),
+
+    {'EXIT', {badarg, _}} = (catch ioq_nif(deq, Q, 1)),
+    {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, 1, 1234)),
+
+    {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [atom_in_list], 0)),
+    {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [make_ref()], 0)),
+    {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [256], 0)),
+    {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [-1], 0)),
+    {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [#{}], 0)),
+    {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [1 bsl 64], 0)),
+    {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [{tuple}], 0)),
+
+    {'EXIT', {badarg, _}} = (catch ioq_nif(inspect,  [atom_in_list], use_stack)),
+    {'EXIT', {badarg, _}} = (catch ioq_nif(inspect,  [make_ref()], no_stack)),
+    {'EXIT', {badarg, _}} = (catch ioq_nif(inspect,  [256], use_stack)),
+    {'EXIT', {badarg, _}} = (catch ioq_nif(inspect,  [-1], no_stack)),
+    {'EXIT', {badarg, _}} = (catch ioq_nif(inspect,  [#{}], use_stack)),
+    {'EXIT', {badarg, _}} = (catch ioq_nif(inspect,  [1 bsl 64], no_stack)),
+    {'EXIT', {badarg, _}} = (catch ioq_nif(inspect,  [{tuple}], use_stack)),
+    {'EXIT', {badarg, _}} = (catch ioq_nif(inspect,  <<"binary">>, use_stack)),
+
+    ioq_nif(destroy, Q),
+
+    %% Test that the example in the docs works
+    ExampleQ = ioq_nif(create),
+    true = ioq_nif(example, ExampleQ, nif_ioq_payload(5)),
+    ioq_nif(destroy, ExampleQ),
+
+    ok.
+
+
+nif_ioq_run(Script) ->
+    nif_ioq_run(Script, #{}).
+
+nif_ioq_run([{Action, Name}|T], State)
+  when Action =:= enqb; Action =:= enqbraw ->
+    nif_ioq_run([{Action, Name, heapbin}|T], State);
+nif_ioq_run([{Action, Name, Skip}|T], State)
+  when Action =:= enqb, is_integer(Skip);
+       Action =:= enqbraw, is_integer(Skip) ->
+    nif_ioq_run([{Action, Name, heapbin, Skip}|T], State);
+nif_ioq_run([{Action, Name, N}|T], State)
+  when Action =:= enqv; Action =:= enqb; Action =:= enqbraw ->
+    nif_ioq_run([{Action, Name, N, 0}|T], State);
+nif_ioq_run([{Action, Name, N, Skip}|T], State)
+  when Action =:= enqv; Action =:= enqb; Action =:= enqbraw ->
+
+    #{ q := IOQ, b := B } = Q = maps:get(Name, State),
+    true = ioq_nif(size, IOQ) == iolist_size(B),
+
+    %% Sanitize the log output a bit so that it doesn't become too large.
+    H = {Action, Name, try iolist_size(N) of Sz -> Sz catch _:_ -> N end, Skip},
+    ct:log("~p", [H]),
+
+    Data = nif_ioq_payload(N),
+    ioq_nif(Action, IOQ, Data, Skip),
+
+    <<_:Skip/binary, SkippedData/binary>> = iolist_to_binary(Data),
+
+    true = ioq_nif(size, IOQ) == (iolist_size([B|SkippedData])),
+
+    nif_ioq_run(T, State#{ Name := Q#{ b := [B|SkippedData]}});
+nif_ioq_run([{peek, Name} = H|T], State) ->
+    #{ q := IOQ, b := B } = maps:get(Name, State),
+    true = ioq_nif(size, IOQ) == iolist_size(B),
+
+    ct:log("~p", [H]),
+
+    Data = ioq_nif(peek, IOQ, ioq_nif(size, IOQ)),
+
+    true = iolist_to_binary(B) == iolist_to_binary(Data),
+    nif_ioq_run(T, State);
+nif_ioq_run([{deq, Name, all}|T], State) ->
+    #{ q := IOQ, b := B } = maps:get(Name, State),
+    Size = ioq_nif(size, IOQ),
+    true = Size == iolist_size(B),
+    nif_ioq_run([{deq, Name, Size}|T], State);
+nif_ioq_run([{deq, Name, N} = H|T], State) ->
+    #{ q := IOQ, b := B } = Q = maps:get(Name, State),
+    true = ioq_nif(size, IOQ) == iolist_size(B),
+
+    ct:log("~p", [H]),
+
+    <<_:N/binary,Remain/binary>> = iolist_to_binary(B),
+    NewQ = Q#{ b := Remain },
+
+    Sz = ioq_nif(deq, IOQ, N),
+
+    true = Sz == iolist_size(Remain),
+    true = ioq_nif(size, IOQ) == iolist_size(Remain),
+
+    nif_ioq_run(T, State#{ Name := NewQ });
+nif_ioq_run([{create, Name} = H|T], State) ->
+    ct:log("~p", [H]),
+    nif_ioq_run(T, State#{ Name => #{ q => ioq_nif(create), b => [] } });
+nif_ioq_run([{destroy, Name} = H|T], State) ->
+    #{ q := IOQ, b := B } = maps:get(Name, State),
+    true = ioq_nif(size, IOQ) == iolist_size(B),
+
+    ct:log("~p", [H]),
+
+    ioq_nif(destroy, IOQ),
+
+    nif_ioq_run(T, maps:remove(Name, State));
+nif_ioq_run([], State) ->
+    State.
+
+nif_ioq_payload(N) when is_integer(N) ->
+    Tail = if N > 3 -> nif_ioq_payload(N-3); true -> [] end,
+    Head = element(1, lists:split(N,[nif_ioq_payload(subbin),
+                                     nif_ioq_payload(heapbin),
+                                     nif_ioq_payload(refcbin) | Tail])),
+    erlang:iolist_to_iovec(Head);
+nif_ioq_payload(subbin) ->
+    Bin = nif_ioq_payload(refcbin),
+    Sz = size(Bin) - 1,
+    <<_:8,SubBin:Sz/binary,_/bits>> = Bin,
+    SubBin;
+nif_ioq_payload(heapbin) ->
+    <<"a literal heap binary">>;
+nif_ioq_payload(refcbin) ->
+    iolist_to_binary([lists:seq(1,255) || _ <- lists:seq(1,255)]);
+nif_ioq_payload(Else) ->
+    Else.
 
 %% The NIFs:
 lib_version() -> undefined.
@@ -3032,6 +3208,10 @@ monitor_process_nif(_,_,_,_) -> ?nif_stub.
 demonitor_process_nif(_,_) -> ?nif_stub.
 compare_monitors_nif(_,_) -> ?nif_stub.
 monitor_frenzy_nif(_,_,_,_) -> ?nif_stub.
+ioq_nif(_) -> ?nif_stub.
+ioq_nif(_,_) -> ?nif_stub.
+ioq_nif(_,_,_) -> ?nif_stub.
+ioq_nif(_,_,_,_) -> ?nif_stub.
 
 %% whereis
 whereis_send(_Type,_Name,_Msg) -> ?nif_stub.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 307d1c390f..b47d013bd2 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -186,6 +186,12 @@ static ErlNifResourceTypeInit frenzy_rt_init = {
 
 static ErlNifResourceType* whereis_resource_type;
 static void whereis_thread_resource_dtor(ErlNifEnv* env, void* obj);
+static ErlNifResourceType* ioq_resource_type;
+
+static void ioq_resource_dtor(ErlNifEnv* env, void* obj);
+struct ioq_resource {
+    ErlNifIOQueue *q;
+};
 
 static int get_pointer(ErlNifEnv* env, ERL_NIF_TERM term, void** pp)
 {
@@ -243,6 +249,10 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
     whereis_resource_type = enif_open_resource_type(env, NULL, "nif_SUITE.whereis",
                             whereis_thread_resource_dtor, ERL_NIF_RT_CREATE, NULL);
 
+    ioq_resource_type = enif_open_resource_type(env,NULL,"ioq",
+                                                ioq_resource_dtor,
+                                                ERL_NIF_RT_CREATE, NULL);
+
     atom_false = enif_make_atom(env,"false");
     atom_true = enif_make_atom(env,"true");
     atom_self = enif_make_atom(env,"self");
@@ -2430,7 +2440,6 @@ static ERL_NIF_TERM format_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg
     return enif_make_binary(env,&obin);
 }
 
-
 static int get_fd(ErlNifEnv* env, ERL_NIF_TERM term, struct fd_resource** rsrc)
 {
     if (!enif_get_resource(env, term, fd_resource_type, (void**)rsrc)) {
@@ -3158,7 +3167,231 @@ static void frenzy_resource_down(ErlNifEnv* env, void* obj, ErlNifPid* pid,
     abort();
 }
 
+/*********** testing ioq ************/
+
+static void ioq_resource_dtor(ErlNifEnv* env, void* obj) {
+
+}
+
+#ifndef __WIN32__
+static int writeiovec(ErlNifEnv *env, ERL_NIF_TERM term, ERL_NIF_TERM *tail, ErlNifIOQueue *q, int fd) {
+    ErlNifIOVec vec, *iovec = &vec;
+    SysIOVec *sysiovec;
+    int saved_errno;
+    int iovcnt, n;
+
+    if (!enif_inspect_iovec(env, 64, term, tail, &iovec))
+        return -2;
+
+    if (enif_ioq_size(q) > 0) {
+        /* If the I/O queue contains data we enqueue the iovec and then
+           peek the data to write out of the queue. */
+        if (!enif_ioq_enqv(q, iovec, 0))
+            return -3;
+
+        sysiovec = enif_ioq_peek(q, &iovcnt);
+    } else {
+        /* If the I/O queue is empty we skip the trip through it. */
+        iovcnt = iovec->iovcnt;
+        sysiovec = iovec->iov;
+    }
+
+    /* Attempt to write the data */
+    n = writev(fd, sysiovec, iovcnt);
+    saved_errno = errno;
+
+    if (enif_ioq_size(q) == 0) {
+        /* If the I/O queue was initially empty we enqueue any
+           remaining data into the queue for writing later. */
+        if (n >= 0 && !enif_ioq_enqv(q, iovec, n))
+            return -3;
+    } else {
+        /* Dequeue any data that was written from the queue. */
+        if (n > 0 && !enif_ioq_deq(q, n, NULL))
+            return -4;
+    }
+
+    /* return n, which is either number of bytes written or -1 if
+       some error happened */
+    errno = saved_errno;
+    return n;
+}
+#endif
+
+static ERL_NIF_TERM ioq(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    struct ioq_resource *ioq;
+    ERL_NIF_TERM ret;
+    if (enif_is_identical(argv[0], enif_make_atom(env, "create"))) {
+        ErlNifIOQueue *q = enif_ioq_create(ERL_NIF_IOQ_NORMAL);
+        ioq = (struct ioq_resource *)enif_alloc_resource(ioq_resource_type,
+                                                         sizeof(*ioq));
+        ioq->q = q;
+        ret = enif_make_resource(env, ioq);
+        enif_release_resource(ioq);
+        return ret;
+    } else if (enif_is_identical(argv[0], enif_make_atom(env, "inspect"))) {
+        ErlNifIOVec vec, *iovec = NULL;
+        int i, iovcnt;
+        ERL_NIF_TERM *elems, tail, list;
+        ErlNifEnv *myenv = NULL;
+
+        if (enif_is_identical(argv[2], enif_make_atom(env, "use_stack")))
+            iovec = &vec;
+        if (enif_is_identical(argv[3], enif_make_atom(env, "use_env")))
+            myenv = env;
+        if (!enif_inspect_iovec(myenv, ~(size_t)0, argv[1], &tail, &iovec))
+            return enif_make_badarg(env);
+
+        iovcnt = iovec->iovcnt;
+        elems = enif_alloc(sizeof(ERL_NIF_TERM) * iovcnt);
+
+        for (i = 0; i < iovcnt; i++) {
+            ErlNifBinary bin;
+            if (!enif_alloc_binary(iovec->iov[i].iov_len, &bin)) {
+                enif_free_iovec(iovec);
+                enif_free(elems);
+                return enif_make_badarg(env);
+            }
+            memcpy(bin.data, iovec->iov[i].iov_base, iovec->iov[i].iov_len);
+            elems[i] = enif_make_binary(env, &bin);
+        }
+
+        if (!myenv)
+            enif_free_iovec(iovec);
+
+	list = enif_make_list_from_array(env, elems, iovcnt);
+	enif_free(elems);
+	return list;
+    } else {
+        unsigned skip;
+        if (!enif_get_resource(env, argv[1], ioq_resource_type, (void**)&ioq)
+            || !ioq->q)
+            return enif_make_badarg(env);
+
+        if (enif_is_identical(argv[0], enif_make_atom(env, "example"))) {
+#ifndef __WIN32__
+            int fd[2], res = 0, cnt = 0, queue_cnt;
+            ERL_NIF_TERM tail;
+            char buff[255];
+            pipe(fd);
+            fcntl(fd[0], F_SETFL, fcntl(fd[0], F_GETFL) | O_NONBLOCK);
+            fcntl(fd[1], F_SETFL, fcntl(fd[1], F_GETFL) | O_NONBLOCK);
+
+            /* Write until the pipe buffer is full, which should result in data
+             * being queued up. */
+            for (res = 0; res >= 0; ) {
+                cnt += res;
+                res = writeiovec(env, argv[2], &tail, ioq->q, fd[1]);
+            }
+
+            /* Flush the queue while reading from the other end of the pipe. */
+            tail = enif_make_list(env, 0);
+            while (enif_ioq_size(ioq->q) > 0) {
+                res = writeiovec(env, tail, &tail, ioq->q, fd[1]);
 
+                if (res < 0 && errno != EAGAIN) {
+                    break;
+                } else if (res > 0) {
+                    cnt += res;
+                }
+
+                for (res = 0; res >= 0; ) {
+                    cnt -= res;
+                    res = read(fd[0], buff, sizeof(buff));
+                }
+            }
+
+            close(fd[0]);
+            close(fd[1]);
+
+            /* Check that we read as much as we wrote */
+            if (cnt == 0 && enif_ioq_size(ioq->q) == 0)
+                return enif_make_atom(env, "true");
+
+            return enif_make_int(env, cnt);
+#else
+            return enif_make_atom(env, "true");
+#endif
+        }
+        if (enif_is_identical(argv[0], enif_make_atom(env, "destroy"))) {
+            enif_ioq_destroy(ioq->q);
+            ioq->q = NULL;
+            return enif_make_atom(env, "false");
+        } else if (enif_is_identical(argv[0], enif_make_atom(env, "enqv"))) {
+            ErlNifIOVec vec, *iovec = &vec;
+            ERL_NIF_TERM tail;
+
+            if (!enif_get_uint(env, argv[3], &skip))
+                return enif_make_badarg(env);
+            if (!enif_inspect_iovec(env, ~0ul, argv[2], &tail, &iovec))
+                return enif_make_badarg(env);
+            if (!enif_ioq_enqv(ioq->q, iovec, skip))
+                return enif_make_badarg(env);
+
+            return enif_make_atom(env, "true");
+        } else if (enif_is_identical(argv[0], enif_make_atom(env, "enqb"))) {
+            ErlNifBinary bin;
+            if (!enif_get_uint(env, argv[3], &skip) ||
+                !enif_inspect_binary(env, argv[2], &bin))
+                return enif_make_badarg(env);
+
+            if (!enif_ioq_enq_binary(ioq->q, &bin, skip))
+                return enif_make_badarg(env);
+
+            return enif_make_atom(env, "true");
+        } else if (enif_is_identical(argv[0], enif_make_atom(env, "enqbraw"))) {
+            ErlNifBinary bin;
+            ErlNifBinary localbin;
+	    int i;
+            if (!enif_get_uint(env, argv[3], &skip) ||
+                !enif_inspect_binary(env, argv[2], &bin) ||
+                !enif_alloc_binary(bin.size, &localbin))
+                return enif_make_badarg(env);
+
+            memcpy(localbin.data, bin.data, bin.size);
+            i = enif_ioq_enq_binary(ioq->q, &localbin, skip);
+	    if (!i)
+		return enif_make_badarg(env);
+	    else
+		return enif_make_atom(env, "true");
+        } else if (enif_is_identical(argv[0], enif_make_atom(env, "peek"))) {
+            int iovlen, num, i, off = 0;
+            SysIOVec *iov = enif_ioq_peek(ioq->q, &iovlen);
+            ErlNifBinary bin;
+
+            if (!enif_get_int(env, argv[2], &num) || !enif_alloc_binary(num, &bin))
+                return enif_make_badarg(env);
+
+            for (i = 0; i < iovlen && num > 0; i++) {
+                int to_copy = num < iov[i].iov_len ? num : iov[i].iov_len;
+                memcpy(bin.data + off, iov[i].iov_base, to_copy);
+                num -= to_copy;
+                off += to_copy;
+            }
+
+            return enif_make_binary(env, &bin);
+        } else if (enif_is_identical(argv[0], enif_make_atom(env, "deq"))) {
+            int num;
+            size_t sz;
+            ErlNifUInt64 sz64;
+            if (!enif_get_int(env, argv[2], &num))
+                return enif_make_badarg(env);
+
+            if (!enif_ioq_deq(ioq->q, num, &sz))
+                return enif_make_badarg(env);
+
+            sz64 = sz;
+
+            return enif_make_uint64(env, sz64);
+        } else if (enif_is_identical(argv[0], enif_make_atom(env, "size"))) {
+            ErlNifUInt64 size = enif_ioq_size(ioq->q);
+            return enif_make_uint64(env, size);
+        }
+    }
+
+    return enif_make_badarg(env);
+}
 
 static ErlNifFunc nif_funcs[] =
 {
@@ -3255,7 +3488,11 @@ static ErlNifFunc nif_funcs[] =
     {"whereis_send", 3, whereis_send},
     {"whereis_term", 2, whereis_term},
     {"whereis_thd_lookup", 2, whereis_thd_lookup},
-    {"whereis_thd_result", 1, whereis_thd_result}
+    {"whereis_thd_result", 1, whereis_thd_result},
+    {"ioq_nif", 1, ioq},
+    {"ioq_nif", 2, ioq},
+    {"ioq_nif", 3, ioq},
+    {"ioq_nif", 4, ioq}
 };
 
 ERL_NIF_INIT(nif_SUITE,nif_funcs,load,NULL,upgrade,unload)
-- 
cgit v1.2.3


From eed25a02ba2416c48587699542aaecdd09609718 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 5 Sep 2017 13:28:52 +0200
Subject: erts: Add erlang:iolist_to_iovec

OTP-14520
---
 erts/emulator/beam/bif.tab         |   6 +
 erts/emulator/beam/erl_io_queue.c  | 500 +++++++++++++++++++++++++++++++++++++
 erts/emulator/test/Makefile        |   1 +
 erts/emulator/test/iovec_SUITE.erl | 168 +++++++++++++
 4 files changed, 675 insertions(+)
 create mode 100644 erts/emulator/test/iovec_SUITE.erl

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 962b00ae7b..10ca0b5066 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -679,3 +679,9 @@ bif math:ceil/1
 bif math:fmod/2
 bif os:set_signal/2
 bif erts_internal:maps_to_list/2
+
+#
+# New in 20.1
+#
+
+bif erlang:iolist_to_iovec/1
diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c
index d072376d1c..301b55b315 100644
--- a/erts/emulator/beam/erl_io_queue.c
+++ b/erts/emulator/beam/erl_io_queue.c
@@ -24,9 +24,24 @@
 
 #include "sys.h"
 #include "global.h"
+
+#define ERL_WANT_HIPE_BIF_WRAPPER__
+#include "bif.h"
+#undef ERL_WANT_HIPE_BIF_WRAPPER__
+
 #include "erl_bits.h"
 #include "erl_io_queue.h"
 
+#ifndef MAX
+#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
+#endif
+
+#ifndef MIN
+#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
+#endif
+
+#define IOL2V_SMALL_BIN_LIMIT (ERL_ONHEAP_BIN_LIMIT * 4)
+
 static void free_binary(ErtsIOQBinary *b, int driver);
 static ErtsIOQBinary *alloc_binary(Uint size, char *source, void **iov_base, int driver);
 
@@ -736,3 +751,488 @@ erts_ioq_iolist_vec_len(Eterm obj, int* vsize, Uint* csize,
     DESTROY_ESTACK(s);
     return 1;
 }
+
+typedef struct {
+    Eterm result_head;
+    Eterm result_tail;
+    Eterm input_list;
+
+    UWord acc_size;
+    Binary *acc;
+
+    /* We yield after copying this many bytes into the accumulator (Minus
+     * eating a few on consing etc). Large binaries will only count to the
+     * extent their split (if any) resulted in a copy op. */
+    UWord bytereds_available;
+    UWord bytereds_spent;
+
+    Process *process;
+    ErtsEStack estack;
+
+    Eterm magic_reference;
+} iol2v_state_t;
+
+static int iol2v_state_destructor(Binary *data) {
+    iol2v_state_t *state = ERTS_MAGIC_BIN_UNALIGNED_DATA(data);
+
+    DESTROY_SAVED_ESTACK(&state->estack);
+
+    if (state->acc != NULL) {
+        erts_bin_free(state->acc);
+    }
+
+    return 1;
+}
+
+static void iol2v_init(iol2v_state_t *state, Process *process, Eterm input) {
+    state->process = process;
+
+    state->result_head = NIL;
+    state->result_tail = NIL;
+    state->input_list = input;
+
+    state->magic_reference = NIL;
+    state->acc_size = 0;
+    state->acc = NULL;
+
+    CLEAR_SAVED_ESTACK(&state->estack);
+}
+
+static Eterm iol2v_make_sub_bin(iol2v_state_t *state, Eterm bin_term,
+        UWord offset, UWord size) {
+    Uint byte_offset, bit_offset, bit_size;
+    ErlSubBin *sb;
+    Eterm orig_pb_term;
+
+    sb = (ErlSubBin*)HAlloc(state->process, ERL_SUB_BIN_SIZE);
+
+    ERTS_GET_REAL_BIN(bin_term, orig_pb_term,
+        byte_offset, bit_offset, bit_size);
+
+    (void)bit_offset;
+    (void)bit_size;
+
+    sb->thing_word = HEADER_SUB_BIN;
+    sb->bitsize = 0;
+    sb->bitoffs = 0;
+    sb->orig = orig_pb_term;
+    sb->is_writable = 0;
+
+    sb->offs = byte_offset + offset;
+    sb->size = size;
+
+    return make_binary(sb);
+}
+
+static Eterm iol2v_promote_acc(iol2v_state_t *state) {
+    ProcBin *pb;
+
+    state->acc = erts_bin_realloc(state->acc, state->acc_size);
+
+    pb = (ProcBin*)HAlloc(state->process, PROC_BIN_SIZE);
+    pb->thing_word = HEADER_PROC_BIN;
+    pb->size = state->acc_size;
+    pb->val = state->acc;
+    pb->bytes = (byte*)(state->acc)->orig_bytes;
+    pb->flags = 0;
+    pb->next = MSO(state->process).first;
+    OH_OVERHEAD(&(MSO(state->process)), pb->size / sizeof(Eterm));
+    MSO(state->process).first = (struct erl_off_heap_header*)pb;
+
+    state->acc_size = 0;
+    state->acc = NULL;
+
+    return make_binary(pb);
+}
+
+/* Destructively enqueues a term to the result list, saving us the hassle of
+ * having to reverse it later. This is safe since GC is disabled and we never
+ * leak the unfinished term to the outside. */
+static void iol2v_enqueue_result(iol2v_state_t *state, Eterm term) {
+    Eterm prev_tail;
+    Eterm *hp;
+
+    prev_tail = state->result_tail;
+
+    hp = HAlloc(state->process, 2);
+    state->result_tail = CONS(hp, term, NIL);
+
+    if(prev_tail != NIL) {
+        Eterm *prev_cell = list_val(prev_tail);
+        CDR(prev_cell) = state->result_tail;
+    } else {
+        state->result_head = state->result_tail;
+    }
+
+    state->bytereds_spent += 1;
+}
+
+#ifndef DEBUG
+    #define ACC_REALLOCATION_LIMIT (IOL2V_SMALL_BIN_LIMIT * 32)
+#else
+    #define ACC_REALLOCATION_LIMIT (IOL2V_SMALL_BIN_LIMIT * 4)
+#endif
+
+static void iol2v_expand_acc(iol2v_state_t *state, UWord extra) {
+    UWord required_bytes, acc_alloc_size;
+
+    ERTS_CT_ASSERT(ERTS_UWORD_MAX > ACC_REALLOCATION_LIMIT / 2);
+    ASSERT(extra >= 1);
+
+    acc_alloc_size = state->acc != NULL ? (state->acc)->orig_size : 0;
+    required_bytes = state->acc_size + extra;
+
+    if (state->acc == NULL) {
+        UWord new_size = MAX(required_bytes, IOL2V_SMALL_BIN_LIMIT);
+
+        state->acc = erts_bin_nrml_alloc(new_size);
+    } else if (required_bytes > acc_alloc_size) {
+        Binary *prev_acc;
+        UWord new_size;
+
+        if (acc_alloc_size >= ACC_REALLOCATION_LIMIT) {
+            /* We skip reallocating once we hit a certain point; it often
+             * results in extra copying and we're very likely to overallocate
+             * on anything other than absurdly long byte/heapbin sequences. */
+            iol2v_enqueue_result(state, iol2v_promote_acc(state));
+            iol2v_expand_acc(state, extra);
+            return;
+        }
+
+        new_size = MAX(required_bytes, acc_alloc_size * 2);
+        prev_acc = state->acc;
+
+        state->acc = erts_bin_realloc(prev_acc, new_size);
+
+        if (prev_acc != state->acc) {
+            state->bytereds_spent += state->acc_size;
+        }
+    }
+
+    state->bytereds_spent += extra;
+}
+
+static int iol2v_append_byte_seq(iol2v_state_t *state, Eterm seq_start, Eterm *seq_end) {
+    Eterm lookahead, iterator;
+    Uint observed_bits;
+    SWord seq_length;
+    char *acc_data;
+
+    lookahead = seq_start;
+    seq_length = 0;
+
+    ASSERT(state->bytereds_available > state->bytereds_spent);
+
+    while (is_list(lookahead)) {
+        Eterm *cell = list_val(lookahead);
+
+        if (!is_small(CAR(cell))) {
+            break;
+        }
+
+        if (seq_length * 2 >= (state->bytereds_available - state->bytereds_spent)) {
+            break;
+        }
+
+        lookahead = CDR(cell);
+        seq_length += 1;
+    }
+
+    ASSERT(seq_length >= 1);
+
+    iol2v_expand_acc(state, seq_length);
+
+    /* Bump a few extra reductions to account for list traversal. */
+    state->bytereds_spent += seq_length;
+
+    acc_data = &(state->acc)->orig_bytes[state->acc_size];
+    state->acc_size += seq_length;
+
+    iterator = seq_start;
+    observed_bits = 0;
+
+    while (iterator != lookahead) {
+        Eterm *cell;
+        Uint byte;
+
+        cell = list_val(iterator);
+        iterator = CDR(cell);
+
+        byte = unsigned_val(CAR(cell));
+        observed_bits |= byte;
+
+        ASSERT(acc_data < &(state->acc)->orig_bytes[state->acc_size]);
+        *(acc_data++) = byte;
+    }
+
+    if (observed_bits > UCHAR_MAX) {
+        return 0;
+    }
+
+    ASSERT(acc_data == &(state->acc)->orig_bytes[state->acc_size]);
+    *seq_end = iterator;
+
+    return 1;
+}
+
+static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) {
+    int is_acc_small, is_bin_small;
+    UWord combined_size;
+    UWord binary_size;
+
+    Uint byte_offset, bit_offset, bit_size;
+    Eterm *parent_header;
+    Eterm parent_binary;
+    byte *binary_data;
+
+    ASSERT(state->bytereds_available > state->bytereds_spent);
+
+    ERTS_GET_REAL_BIN(bin_term, parent_binary, byte_offset, bit_offset, bit_size);
+    parent_header = binary_val(parent_binary);
+    binary_size = binary_size(bin_term);
+
+    if (bit_offset != 0 || bit_size != 0) {
+        return 0;
+    } else if (binary_size == 0) {
+        state->bytereds_spent += 1;
+        return 1;
+    }
+
+    is_acc_small = state->acc_size < IOL2V_SMALL_BIN_LIMIT;
+    is_bin_small = binary_size < IOL2V_SMALL_BIN_LIMIT;
+    combined_size = binary_size + state->acc_size;
+
+    if (thing_subtag(*parent_header) == REFC_BINARY_SUBTAG) {
+        ProcBin *pb = (ProcBin*)parent_header;
+
+        if (pb->flags) {
+            erts_emasculate_writable_binary(pb);
+        }
+
+        binary_data = pb->bytes;
+    } else {
+        ErlHeapBin *hb = (ErlHeapBin*)parent_header;
+
+        ASSERT(thing_subtag(*parent_header) == HEAP_BINARY_SUBTAG);
+        ASSERT(is_bin_small);
+
+        binary_data = &((unsigned char*)&hb->data)[byte_offset];
+    }
+
+    if (!is_bin_small && (state->acc_size == 0 || !is_acc_small)) {
+        /* Avoid combining if we encounter an acceptably large binary while the
+         * accumulator is either empty or large enough to be returned on its
+         * own. */
+        if (state->acc_size != 0) {
+            iol2v_enqueue_result(state, iol2v_promote_acc(state));
+        }
+
+        iol2v_enqueue_result(state, bin_term);
+    } else if (is_bin_small || combined_size < (IOL2V_SMALL_BIN_LIMIT * 2)) {
+        /* If the candidate is small or we can't split the combination in two,
+         * then just copy it into the accumulator. */
+        iol2v_expand_acc(state, binary_size);
+
+        sys_memcpy(&(state->acc)->orig_bytes[state->acc_size],
+            binary_data, binary_size);
+
+        state->acc_size += binary_size;
+    } else {
+        /* Otherwise, append enough data for the accumulator to be valid, and
+         * then return the rest as a sub-binary. */
+        UWord spill = IOL2V_SMALL_BIN_LIMIT - state->acc_size;
+        Eterm binary_tail;
+
+        iol2v_expand_acc(state, spill);
+
+        sys_memcpy(&(state->acc)->orig_bytes[state->acc_size],
+            binary_data, spill);
+
+        state->acc_size += spill;
+
+        binary_tail = iol2v_make_sub_bin(state, bin_term, spill,
+            binary_size - spill);
+
+        iol2v_enqueue_result(state, iol2v_promote_acc(state));
+        iol2v_enqueue_result(state, binary_tail);
+    }
+
+    return 1;
+}
+
+static BIF_RETTYPE iol2v_yield(iol2v_state_t *state) {
+    if (is_nil(state->magic_reference)) {
+        iol2v_state_t *boxed_state;
+        Binary *magic_binary;
+        Eterm *hp;
+
+        magic_binary = erts_create_magic_binary_x(sizeof(*state),
+            &iol2v_state_destructor, ERTS_ALC_T_BINARY, 1);
+
+        boxed_state = ERTS_MAGIC_BIN_UNALIGNED_DATA(magic_binary);
+        sys_memcpy(boxed_state, state, sizeof(*state));
+
+        hp = HAlloc(boxed_state->process, ERTS_MAGIC_REF_THING_SIZE);
+        boxed_state->magic_reference =
+            erts_mk_magic_ref(&hp, &MSO(boxed_state->process), magic_binary);
+
+        state = boxed_state;
+    }
+
+    ERTS_BIF_YIELD1(bif_export[BIF_iolist_to_iovec_1],
+        state->process, state->magic_reference);
+}
+
+static BIF_RETTYPE iol2v_continue(iol2v_state_t *state) {
+    Eterm iterator;
+
+    DECLARE_ESTACK(s);
+    ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
+
+    state->bytereds_available =
+        ERTS_BIF_REDS_LEFT(state->process) * IOL2V_SMALL_BIN_LIMIT;
+    state->bytereds_spent = 0;
+
+    if (state->estack.start) {
+        ESTACK_RESTORE(s, &state->estack);
+    }
+
+    iterator = state->input_list;
+
+    for(;;) {
+        if (state->bytereds_spent >= state->bytereds_available) {
+            ESTACK_SAVE(s, &state->estack);
+            state->input_list = iterator;
+
+            return iol2v_yield(state);
+        }
+
+        while (is_list(iterator)) {
+            Eterm *cell;
+            Eterm head;
+
+            cell = list_val(iterator);
+            head = CAR(cell);
+
+            if (is_binary(head)) {
+                if (!iol2v_append_binary(state, head)) {
+                    goto l_badarg;
+                }
+
+                iterator = CDR(cell);
+            } else if (is_small(head)) {
+                Eterm seq_end;
+
+                if (!iol2v_append_byte_seq(state, iterator, &seq_end)) {
+                    goto l_badarg;
+                }
+
+                iterator = seq_end;
+            } else if (is_list(head) || is_nil(head)) {
+                Eterm tail = CDR(cell);
+
+                if (!is_nil(tail)) {
+                    ESTACK_PUSH(s, tail);
+                }
+
+                state->bytereds_spent += 1;
+                iterator = head;
+            } else {
+                goto l_badarg;
+            }
+
+            if (state->bytereds_spent >= state->bytereds_available) {
+                ESTACK_SAVE(s, &state->estack);
+                state->input_list = iterator;
+
+                return iol2v_yield(state);
+            }
+        }
+
+        if (is_binary(iterator)) {
+            if (!iol2v_append_binary(state, iterator)) {
+                goto l_badarg;
+            }
+        } else if (!is_nil(iterator)) {
+            goto l_badarg;
+        }
+
+        if(ESTACK_ISEMPTY(s)) {
+            break;
+        }
+
+        iterator = ESTACK_POP(s);
+    }
+
+    if (state->acc_size != 0) {
+        iol2v_enqueue_result(state, iol2v_promote_acc(state));
+    }
+
+    BUMP_REDS(state->process, state->bytereds_spent / IOL2V_SMALL_BIN_LIMIT);
+
+    CLEAR_SAVED_ESTACK(&state->estack);
+    DESTROY_ESTACK(s);
+
+    BIF_RET(state->result_head);
+
+l_badarg:
+    CLEAR_SAVED_ESTACK(&state->estack);
+    DESTROY_ESTACK(s);
+
+    if (state->acc != NULL) {
+        erts_bin_free(state->acc);
+        state->acc = NULL;
+    }
+
+    BIF_ERROR(state->process, BADARG);
+}
+
+HIPE_WRAPPER_BIF_DISABLE_GC(iolist_to_iovec, 1)
+
+BIF_RETTYPE iolist_to_iovec_1(BIF_ALIST_1) {
+    BIF_RETTYPE result;
+
+    if (is_nil(BIF_ARG_1)) {
+        BIF_RET(NIL);
+    } else if (is_binary(BIF_ARG_1)) {
+        if (binary_size(BIF_ARG_1) != 0) {
+            Eterm *hp = HAlloc(BIF_P, 2);
+
+            BIF_RET(CONS(hp, BIF_ARG_1, NIL));
+        } else {
+            BIF_RET(NIL);
+        }
+    } else if (is_internal_magic_ref(BIF_ARG_1)) {
+        iol2v_state_t *state;
+        Binary *magic;
+
+        magic = erts_magic_ref2bin(BIF_ARG_1);
+
+        if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != &iol2v_state_destructor) {
+            ASSERT(!(BIF_P->flags & F_DISABLE_GC));
+            BIF_ERROR(BIF_P, BADARG);
+        }
+
+        ASSERT(BIF_P->flags & F_DISABLE_GC);
+
+        state = ERTS_MAGIC_BIN_UNALIGNED_DATA(magic);
+        result = iol2v_continue(state);
+    } else if (!is_list(BIF_ARG_1)) {
+        ASSERT(!(BIF_P->flags & F_DISABLE_GC));
+        BIF_ERROR(BIF_P, BADARG);
+    } else {
+        iol2v_state_t state;
+
+        iol2v_init(&state, BIF_P, BIF_ARG_1);
+
+        erts_set_gc_state(BIF_P, 0);
+
+        result = iol2v_continue(&state);
+    }
+
+    if (result != THE_NON_VALUE || BIF_P->freason != TRAP) {
+        erts_set_gc_state(BIF_P, 1);
+    }
+
+    BIF_RET(result);
+}
diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index 370fcb0f3a..b17170c8b8 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -71,6 +71,7 @@ MODULES= \
 	hash_SUITE \
 	hibernate_SUITE \
 	hipe_SUITE \
+	iovec_SUITE \
 	list_bif_SUITE \
 	lttng_SUITE \
 	lcnt_SUITE \
diff --git a/erts/emulator/test/iovec_SUITE.erl b/erts/emulator/test/iovec_SUITE.erl
new file mode 100644
index 0000000000..a5f605bfff
--- /dev/null
+++ b/erts/emulator/test/iovec_SUITE.erl
@@ -0,0 +1,168 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%% 
+%% %CopyrightEnd%
+%%
+
+-module(iovec_SUITE).
+
+-export([all/0, suite/0]).
+
+-export([integer_lists/1, binary_lists/1, empty_lists/1, empty_binary_lists/1,
+         mixed_lists/1, improper_lists/1, illegal_lists/1, cons_bomb/1,
+         iolist_to_iovec_idempotence/1, iolist_to_iovec_correctness/1]).
+
+-include_lib("common_test/include/ct.hrl").
+
+suite() ->
+    [{ct_hooks,[ts_install_cth]},
+     {timetrap, {minutes, 2}}].
+
+all() ->
+    [integer_lists, binary_lists, empty_lists, empty_binary_lists, mixed_lists,
+     illegal_lists, improper_lists, cons_bomb, iolist_to_iovec_idempotence,
+     iolist_to_iovec_correctness].
+
+integer_lists(Config) when is_list(Config) ->
+    Variations = gen_variations([I || I <- lists:seq(1, 255)]),
+
+    equivalence_test(fun erlang:iolist_to_iovec/1, Variations),
+
+    ok.
+
+binary_lists(Config) when is_list(Config) ->
+    Variations = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]),
+    equivalence_test(fun erlang:iolist_to_iovec/1, Variations),
+    ok.
+
+empty_lists(Config) when is_list(Config) ->
+    Variations = gen_variations([[] || _ <- lists:seq(1, 256)]),
+    equivalence_test(fun erlang:iolist_to_iovec/1, Variations),
+    [] = erlang:iolist_to_iovec([]),
+    ok.
+
+empty_binary_lists(Config) when is_list(Config) ->
+    Variations = gen_variations([<<>> || _ <- lists:seq(1, 8192)]),
+    equivalence_test(fun erlang:iolist_to_iovec/1, Variations),
+    [] = erlang:iolist_to_iovec(Variations),
+    ok.
+
+mixed_lists(Config) when is_list(Config) ->
+    Variations = gen_variations([<<>>, lists:seq(1, 40), <<12, 45, 78>>]),
+    equivalence_test(fun erlang:iolist_to_iovec/1, Variations),
+    ok.
+
+illegal_lists(Config) when is_list(Config) ->
+    BitStrs = gen_variations(["gurka", <<1:1>>, "gaffel"]),
+    BadInts = gen_variations(["gurka", 890, "gaffel"]),
+    Atoms = gen_variations([gurka, "gaffel"]),
+    BadTails = [["test" | 0], ["gurka", gaffel]],
+
+    Variations =
+        BitStrs ++ BadInts ++ Atoms ++ BadTails,
+
+    illegality_test(fun erlang:iolist_to_iovec/1, Variations),
+
+    ok.
+
+improper_lists(Config) when is_list(Config) ->
+    Variations = [
+        [[[[1 | <<2>>] | <<3>>] | <<4>>] | <<5>>],
+        [[<<"test">>, 3] | <<"improper tail">>],
+        [1, 2, 3 | <<"improper tail">>]
+    ],
+    equivalence_test(fun erlang:iolist_to_iovec/1, Variations),
+    ok.
+
+cons_bomb(Config) when is_list(Config) ->
+    IntBase = gen_variations([I || I <- lists:seq(1, 255)]),
+    BinBase = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]),
+    MixBase = gen_variations([<<12, 45, 78>>, lists:seq(1, 255)]),
+
+    Rounds =
+        case system_mem_size() of
+            Mem when Mem >= (16 bsl 30) -> 32;
+            Mem when Mem >= (3 bsl 30) -> 28;
+            _ -> 20
+        end,
+
+    Variations = gen_variations([IntBase, BinBase, MixBase], Rounds),
+    equivalence_test(fun erlang:iolist_to_iovec/1, Variations),
+    ok.
+
+iolist_to_iovec_idempotence(Config) when is_list(Config) ->
+    IntVariations = gen_variations([I || I <- lists:seq(1, 255)]),
+    BinVariations = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]),
+    MixVariations = gen_variations([<<12, 45, 78>>, lists:seq(1, 255)]),
+
+    Variations = [IntVariations, BinVariations, MixVariations],
+    Optimized = erlang:iolist_to_iovec(Variations),
+
+    true = Optimized =:= erlang:iolist_to_iovec(Optimized),
+    ok.
+
+iolist_to_iovec_correctness(Config) when is_list(Config) ->
+    IntVariations = gen_variations([I || I <- lists:seq(1, 255)]),
+    BinVariations = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]),
+    MixVariations = gen_variations([<<12, 45, 78>>, lists:seq(1, 255)]),
+
+    Variations = [IntVariations, BinVariations, MixVariations],
+    Optimized = erlang:iolist_to_iovec(Variations),
+
+    true = is_iolist_equal(Optimized, Variations),
+    ok.
+
+illegality_test(Fun, Variations) ->
+    [{'EXIT',{badarg, _}} = (catch Fun(Variation)) || Variation <- Variations].
+
+equivalence_test(Fun, [Head | _] = Variations) ->
+    Comparand = Fun(Head),
+    [is_iolist_equal(Comparand, Fun(Variation)) || Variation <- Variations],
+    ok.
+
+is_iolist_equal(A, B) ->
+    iolist_to_binary(A) =:= iolist_to_binary(B).
+
+%% Generates a bunch of lists whose contents will be equal to Base repeated a
+%% few times. The lists only differ by their structure, so their reduction to
+%% a simpler format should yield the same result.
+gen_variations(Base) ->
+    gen_variations(Base, 16).
+gen_variations(Base, N) ->
+    [gen_flat_list(Base, N),
+     gen_nested_list(Base, N),
+     gen_nasty_list(Base, N)].
+
+gen_flat_list(Base, N) ->
+    lists:flatten(gen_nested_list(Base, N)).
+
+gen_nested_list(Base, N) ->
+    [Base || _ <- lists:seq(1, N)].
+
+gen_nasty_list(Base, N) ->
+    gen_nasty_list_1(gen_nested_list(Base, N), []).
+gen_nasty_list_1([], Result) ->
+    Result;
+gen_nasty_list_1([Head | Base], Result) when is_list(Head) ->
+    gen_nasty_list_1(Base, [[Result], [gen_nasty_list_1(Head, [])]]);
+gen_nasty_list_1([Head | Base], Result) ->
+    gen_nasty_list_1(Base, [[Result], [Head]]).
+
+system_mem_size() ->
+    application:ensure_all_started(os_mon),
+    {Tot,_Used,_}  = memsup:get_memory_data(),
+    Tot.
-- 
cgit v1.2.3


From 948ee9b34f34a55cb4b70b7077a849c7dc7a5f18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Fri, 21 Jul 2017 14:20:16 +0200
Subject: Replace the zlib driver with a NIF

All operations will now yield appropriately, allowing them to be used
freely in concurrent applications. This commit also deprecates the
functions listed below, although they won't raise deprecation
warnings until OTP 21:

zlib:adler32
zlib:crc32
zlib:inflateChunk
zlib:getBufSize
zlib:setBufSize

The behavior of throwing an error when a dictionary is required for
decompression has also been deprecated.
---
 erts/emulator/Makefile.in               |   6 +-
 erts/emulator/drivers/common/zlib_drv.c | 792 -------------------------
 erts/emulator/nifs/common/zlib_nif.c    | 998 ++++++++++++++++++++++++++++++++
 3 files changed, 1001 insertions(+), 795 deletions(-)
 delete mode 100644 erts/emulator/drivers/common/zlib_drv.c
 create mode 100644 erts/emulator/nifs/common/zlib_nif.c

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index cc1fa571df..1916b97a89 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -819,14 +819,15 @@ RUN_OBJS = \
 	$(OBJDIR)/erl_io_queue.o
 
 LTTNG_OBJS = $(OBJDIR)/erlang_lttng.o
-NIF_OBJS = $(OBJDIR)/erl_tracer_nif.o
+NIF_OBJS = \
+	   $(OBJDIR)/erl_tracer_nif.o \
+	   $(OBJDIR)/zlib_nif.o
 
 ifeq ($(TARGET),win32)
 DRV_OBJS = \
 	$(OBJDIR)/registry_drv.o \
 	$(OBJDIR)/efile_drv.o \
 	$(OBJDIR)/inet_drv.o \
-	$(OBJDIR)/zlib_drv.o \
 	$(OBJDIR)/ram_file_drv.o \
 	$(OBJDIR)/ttsl_drv.o
 OS_OBJS = \
@@ -856,7 +857,6 @@ OS_OBJS = \
 DRV_OBJS = \
 	$(OBJDIR)/efile_drv.o \
 	$(OBJDIR)/inet_drv.o \
-	$(OBJDIR)/zlib_drv.o \
 	$(OBJDIR)/ram_file_drv.o \
 	$(OBJDIR)/ttsl_drv.o
 endif
diff --git a/erts/emulator/drivers/common/zlib_drv.c b/erts/emulator/drivers/common/zlib_drv.c
deleted file mode 100644
index e342e414b5..0000000000
--- a/erts/emulator/drivers/common/zlib_drv.c
+++ /dev/null
@@ -1,792 +0,0 @@
-/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 2003-2017. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * %CopyrightEnd%
- */
-
-/*
- * ZLib interface for erlang
- *
- */
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-#include <stdio.h>
-#include <zlib.h>
-#include <errno.h>
-#include <string.h>
-
-#include "erl_driver.h"
-
-
-#define DEFLATE_INIT    1
-#define DEFLATE_INIT2   2
-#define DEFLATE_SETDICT 3
-#define DEFLATE_RESET   4
-#define DEFLATE_END     5
-#define DEFLATE_PARAMS  6
-#define DEFLATE         7
-
-#define INFLATE_INIT    8
-#define INFLATE_INIT2   9
-#define INFLATE_SETDICT 10
-#define INFLATE_GETDICT 11
-#define INFLATE_SYNC    12
-#define INFLATE_RESET   13
-#define INFLATE_END     14
-#define INFLATE         15
-
-#define CRC32_0         16
-#define CRC32_1         17
-#define CRC32_2         18
-
-#define SET_BUFSZ       19
-#define GET_BUFSZ       20
-#define GET_QSIZE       21
-
-#define ADLER32_1       22
-#define ADLER32_2       23
-
-#define CRC32_COMBINE   24
-#define ADLER32_COMBINE 25
-
-#define INFLATE_CHUNK   26
-
-
-#define DEFAULT_BUFSZ   4000
-
-/* According to zlib documentation, it can never exceed this */
-#define INFL_DICT_SZ    32768
-
-/* This flag is used in the same places, where zlib return codes
- * (Z_OK, Z_STREAM_END, Z_NEED_DICT) are. So, we need to set it to
- * relatively large value to avoid possible value clashes in future.
- * */
-#define INFLATE_HAS_MORE 100
-
-static int zlib_init(void);
-static ErlDrvData zlib_start(ErlDrvPort port, char* buf);
-static void zlib_stop(ErlDrvData e);
-static void zlib_flush(ErlDrvData e);
-static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *buf,
-			     ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen);
-static void zlib_outputv(ErlDrvData drv_data, ErlIOVec *ev);
-
-ErlDrvEntry zlib_driver_entry = {
-    zlib_init,
-    zlib_start,
-    zlib_stop,
-    NULL,                           /* output */
-    NULL,                           /* ready_input */
-    NULL,                           /* ready_output */
-    "zlib_drv",
-    NULL,                           /* finish */
-    NULL,                           /* handle */
-    zlib_ctl,
-    NULL,                           /* timeout */
-    zlib_outputv,
-    NULL,                           /* read_async */
-    zlib_flush,
-    NULL,                           /* call */
-    NULL,                           /* event */
-    ERL_DRV_EXTENDED_MARKER,
-    ERL_DRV_EXTENDED_MAJOR_VERSION,
-    ERL_DRV_EXTENDED_MINOR_VERSION,
-    ERL_DRV_FLAG_USE_PORT_LOCKING,
-    NULL,                           /* handle2 */
-    NULL,                           /* process_exit */
-};
-
-typedef enum {
-    ST_NONE    = 0,
-    ST_DEFLATE = 1,
-    ST_INFLATE = 2
-} ZLibState;
-
-
-typedef struct {
-    z_stream s;
-    ZLibState state;
-    ErlDrvBinary* bin;
-    int binsz;
-    int binsz_need;
-    uLong crc;
-    int inflate_eos_seen;
-    int want_crc;       /* 1 if crc is calculated on clear text */
-    ErlDrvPort port;    /* the associcated port */
-} ZLibData;
-
-static int zlib_inflate(ZLibData* d, int flush);
-static int zlib_deflate(ZLibData* d, int flush);
-
-#if defined(__WIN32__)
-static int i32(char* buf)
-#else
-static __inline__ int i32(char* buf)
-#endif
-{
-    return (int) (
-		  (((int)((unsigned char*)buf)[0]) << 24) |
-		  (((int)((unsigned char*)buf)[1]) << 16) |
-		  (((int)((unsigned char*)buf)[2]) << 8) |
-		  (((int)((unsigned char*)buf)[3]) << 0));
-}
-
-static char* zlib_reason(int code, int* err)
-{
-    switch(code) {
-    case Z_OK:
-	*err = 0;
-	return "ok";
-    case Z_STREAM_END:
-	*err = 0;
-	return "stream_end"; 
-    case Z_ERRNO:
-	*err = 1;
-	return erl_errno_id(errno);
-    case Z_STREAM_ERROR:
-	*err = 1;
-	return "stream_error";
-    case Z_DATA_ERROR:
-	*err = 1;
-	return "data_error";
-    case Z_MEM_ERROR:
-	*err = 1;
-	return "mem_error";
-    case Z_BUF_ERROR:
-	*err = 1;
-	return "buf_error";
-    case Z_VERSION_ERROR:
-	*err = 1;
-	return "version_error";
-    default:
-	*err = 1;
-	return "unknown_error";
-    }
-}
-
-
-static ErlDrvSSizeT zlib_return(int code, char** rbuf, ErlDrvSizeT rlen)
-{
-    int msg_code = 0; /* 0=ok, 1=error */
-    char* dst = *rbuf;
-    char* src;
-    ErlDrvSizeT len = 0;
-
-    src = zlib_reason(code, &msg_code);
-    *dst++ = msg_code;
-    rlen--;
-    len = 1;
-
-    while((rlen > 0) && *src) {
-	*dst++ = *src++;
-	rlen--;
-	len++;
-    }
-    return len;
-}
-
-static ErlDrvSSizeT zlib_value2(int msg_code, int value,
-				char** rbuf, ErlDrvSizeT rlen)
-{
-    char* dst = *rbuf;
-
-    if (rlen  < 5) {
-	return -1;
-    }
-    *dst++ = msg_code;
-    *dst++ = (value >> 24) & 0xff;
-    *dst++ = (value >> 16) & 0xff;
-    *dst++ = (value >> 8) & 0xff;
-    *dst++ = value & 0xff;
-    return 5;
-}
-
-static ErlDrvSSizeT zlib_value(int value, char** rbuf, ErlDrvSizeT rlen)
-{
-    return zlib_value2(2, value, rbuf, rlen);
-}
-
-static int zlib_output_init(ZLibData* d)
-{
-    if (d->bin != NULL)
-	driver_free_binary(d->bin);
-    if ((d->bin = driver_alloc_binary(d->binsz_need)) == NULL)
-	return -1;
-    d->binsz = d->binsz_need;
-    d->s.next_out = (unsigned char*)d->bin->orig_bytes;
-    d->s.avail_out = d->binsz;
-    return 0;
-}
-
-/*
- * Send compressed or uncompressed data
- * and restart output procesing
- */
-static int zlib_output(ZLibData* d)
-{
-    if (d->bin != NULL) {
-	int len = d->binsz - d->s.avail_out;
-	if (len > 0) {
-	    if (driver_output_binary(d->port, NULL, 0, d->bin, 0, len) < 0) 
-		return -1;
-	}
-	driver_free_binary(d->bin);
-	d->bin = NULL;
-	d->binsz = 0;
-    }
-    return zlib_output_init(d);
-}
-
-static int zlib_inflate_get_dictionary(ZLibData* d)
-{
-#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY
-    ErlDrvBinary* dbin = driver_alloc_binary(INFL_DICT_SZ);
-    uInt dlen = 0;
-    int res = inflateGetDictionary(&d->s, (unsigned char*)dbin->orig_bytes, &dlen);
-    if ((res == Z_OK) && (driver_output_binary(d->port, NULL, 0, dbin, 0, dlen) < 0)) {
-        res = Z_ERRNO;
-    }
-    driver_free_binary(dbin);
-    return res;
-#else
-    abort(); /* never called, just to silence 'unresolved symbol'
-                for non-optimizing compiler */
-#endif
-}
-
-static int zlib_inflate(ZLibData* d, int flush)
-{
-    int res = Z_OK;
-
-    if ((d->bin == NULL) && (zlib_output_init(d) < 0)) {
-	errno = ENOMEM;
-	return Z_ERRNO;
-    }
-
-    while ((driver_sizeq(d->port) > 0) && (res != Z_STREAM_END)) {
-	int vlen;
-	SysIOVec* iov = driver_peekq(d->port, &vlen);
-	int len;
-	int possibly_more_output = 0;
-
-	d->s.next_in = iov[0].iov_base;
-	d->s.avail_in = iov[0].iov_len;
-	while((possibly_more_output || (d->s.avail_in > 0)) && (res != Z_STREAM_END)) {
-	    res = inflate(&d->s, Z_NO_FLUSH);
-	    if (res == Z_NEED_DICT) {
-		/* Essential to eat the header bytes that zlib has looked at */
-		len = iov[0].iov_len - d->s.avail_in;
-		driver_deq(d->port, len);
-		return res;
-	    }
-	    if (res == Z_BUF_ERROR) {
-		/* Was possible more output, but actually not */
-		res = Z_OK;
-	    }
-	    else if (res < 0) {
-		return res;
-	    }
-	    if (d->s.avail_out != 0) {
-		possibly_more_output = 0;
-	    } else {
-		if (d->want_crc)
-		    d->crc = crc32(d->crc, (unsigned char*)d->bin->orig_bytes,
-				   d->binsz - d->s.avail_out);
-		zlib_output(d);
-		possibly_more_output = 1;
-	    }
-	}
-	len = iov[0].iov_len - d->s.avail_in;
-	driver_deq(d->port, len);
-    }
-
-    if (d->want_crc) {
-       d->crc = crc32(d->crc, (unsigned char*) d->bin->orig_bytes,
-		      d->binsz - d->s.avail_out);
-    }
-    zlib_output(d);
-    if (res == Z_STREAM_END) {       
-       d->inflate_eos_seen = 1;
-    }
-    return res;
-}
-
-static int zlib_inflate_chunk(ZLibData* d)
-{
-    int res = Z_OK;
-
-    if ((d->bin == NULL) && (zlib_output_init(d) < 0)) {
-	errno = ENOMEM;
-	return Z_ERRNO;
-    }
-
-    while ((driver_sizeq(d->port) > 0) && (d->s.avail_out > 0) &&
-           (res != Z_STREAM_END)) {
-	int vlen;
-	SysIOVec* iov = driver_peekq(d->port, &vlen);
-	int len;
-
-	d->s.next_in = iov[0].iov_base;
-	d->s.avail_in = iov[0].iov_len;
-	while((d->s.avail_in > 0) && (d->s.avail_out > 0) && (res != Z_STREAM_END)) {
-	    res = inflate(&d->s, Z_NO_FLUSH);
-	    if (res == Z_NEED_DICT) {
-		/* Essential to eat the header bytes that zlib has looked at */
-		len = iov[0].iov_len - d->s.avail_in;
-		driver_deq(d->port, len);
-		return res;
-	    }
-	    if (res == Z_BUF_ERROR) {
-		/* Was possible more output, but actually not */
-		res = Z_OK;
-	    }
-	    else if (res < 0) {
-		return res;
-	    }
-	}
-	len = iov[0].iov_len - d->s.avail_in;
-	driver_deq(d->port, len);
-    }
-
-    /* We are here because all input was consumed or EOS reached or output
-     * buffer is full */
-    if (d->want_crc) {
-       d->crc = crc32(d->crc, (unsigned char*) d->bin->orig_bytes,
-		      d->binsz - d->s.avail_out);
-    }
-    zlib_output(d);
-    if ((res == Z_OK) && (d->s.avail_in > 0))
-       res = INFLATE_HAS_MORE;
-    else if (res == Z_STREAM_END) {
-       d->inflate_eos_seen = 1;
-    }
-    return res;
-}
-
-static int zlib_deflate(ZLibData* d, int flush)
-{
-    int res = Z_OK;
-
-    if ((d->bin == NULL) && (zlib_output_init(d) < 0)) {
-	errno = ENOMEM;
-	return Z_ERRNO;
-    }
-
-    while ((driver_sizeq(d->port) > 0) && (res != Z_STREAM_END)) {
-	int vlen;
-	SysIOVec* iov = driver_peekq(d->port, &vlen);
-	int len;
-
-	d->s.next_in = iov[0].iov_base;
-	d->s.avail_in = iov[0].iov_len;
-
-	while((d->s.avail_in > 0) && (res != Z_STREAM_END)) {
-	    if ((res = deflate(&d->s, Z_NO_FLUSH)) < 0) {
-		return res;
-	    }
-	    if (d->s.avail_out == 0) {
-		zlib_output(d);
-	    }
-	}
-	len = iov[0].iov_len - d->s.avail_in;
-	if (d->want_crc) {
-	    d->crc = crc32(d->crc, iov[0].iov_base, len);
-	}
-	driver_deq(d->port, len);
-    }
-
-    if (flush != Z_NO_FLUSH) {
-	if ((res = deflate(&d->s, flush)) < 0) {
-	    return res;
-	}
-	if (flush == Z_FINISH) {
-	    while (d->s.avail_out < d->binsz) {
-		zlib_output(d);
-		if (res == Z_STREAM_END) {
-		    break;
-		}
-		if ((res = deflate(&d->s, flush)) < 0) {
-		    return res;
-		}
-	    }
-	} else {
-	    while (d->s.avail_out == 0) {
-	       zlib_output(d);
-	       if ((res = deflate(&d->s, flush)) < 0) {
-		  return res;
-	       }
-	    }
-	    if (d->s.avail_out < d->binsz) {
-	       zlib_output(d);
-	    }
-	}
-    }
-    return res;
-}
-
-
-
-static void* zlib_alloc(void* data, unsigned int items, unsigned int size)
-{
-    return (void*) driver_alloc(items*size);
-}
-
-static void zlib_free(void* data, void* addr)
-{
-    driver_free(addr);
-}
-
-#if defined(__APPLE__) && defined(__MACH__) && defined(HAVE_ZLIB_INFLATEGETDICTIONARY)
-
-/* Work around broken build system with runtime version test */
-static int have_inflateGetDictionary;
-
-static int zlib_init()
-{
-    unsigned int v[4] = {0, 0, 0, 0};
-    unsigned hexver;
-
-    sscanf(zlibVersion(), "%u.%u.%u.%u", &v[0], &v[1], &v[2], &v[3]);
-
-    hexver = (v[0] << (8*3)) | (v[1] << (8*2)) | (v[2] << (8)) | v[3];
-
-    have_inflateGetDictionary = (hexver >= 0x1020701); /* 1.2.7.1 */
-
-    return 0;
-}
-#else /* trust configure got it right */
-#  ifdef HAVE_ZLIB_INFLATEGETDICTIONARY
-#    define have_inflateGetDictionary 1
-#  else
-#    define have_inflateGetDictionary 0
-#  endif
-static int zlib_init()
-{
-    return 0;
-}
-#endif
-
-static ErlDrvData zlib_start(ErlDrvPort port, char* buf)
-{
-    ZLibData* d;
-
-    if ((d = (ZLibData*) driver_alloc(sizeof(ZLibData))) == NULL)
-        return ERL_DRV_ERROR_GENERAL;
-
-    memset(&d->s, 0, sizeof(z_stream));
-
-    d->s.zalloc = zlib_alloc;
-    d->s.zfree  = zlib_free;
-    d->s.opaque = d;
-    d->s.data_type = Z_BINARY;
-
-    d->port      = port;
-    d->state     = ST_NONE;
-    d->bin       = NULL;
-    d->binsz     = 0;
-    d->binsz_need = DEFAULT_BUFSZ;
-    d->crc       = crc32(0L, Z_NULL, 0);
-    d->inflate_eos_seen = 0;
-    d->want_crc  = 0;
-    return (ErlDrvData)d;
-}
-
-
-static void zlib_stop(ErlDrvData e)
-{
-    ZLibData* d = (ZLibData*)e;
-
-    if (d->state == ST_DEFLATE)
-	deflateEnd(&d->s);
-    else if (d->state == ST_INFLATE)
-	inflateEnd(&d->s);
-
-    if (d->bin != NULL)
-	driver_free_binary(d->bin);
-
-    driver_free(d);
-}
-
-static void zlib_flush(ErlDrvData drv_data)
-{
-    ZLibData* d = (ZLibData*) drv_data;
-
-    driver_deq(d->port, driver_sizeq(d->port));
-}
-
-static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *buf,
-			     ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen)
-{
-    ZLibData* d = (ZLibData*)drv_data;
-    int res;
-
-    switch(command) {
-    case DEFLATE_INIT:
-	if (len != 4) goto badarg;
-	if (d->state != ST_NONE) goto badarg;
-	res = deflateInit(&d->s, i32(buf));
-	if (res == Z_OK) {
-	    d->state = ST_DEFLATE;
-	    d->want_crc = 0;
-	    d->crc = crc32(0L, Z_NULL, 0);
-	}
-	return zlib_return(res, rbuf, rlen);
-
-    case DEFLATE_INIT2: {
-	int wbits;
-
-	if (len != 20) goto badarg;
-	if (d->state != ST_NONE) goto badarg;
-	wbits = i32(buf+8);
-	res = deflateInit2(&d->s, i32(buf), i32(buf+4), wbits, 
-			   i32(buf+12), i32(buf+16));
-	if (res == Z_OK) {
-	    d->state = ST_DEFLATE;
-	    d->want_crc = (wbits < 0);
-	    d->crc = crc32(0L, Z_NULL, 0);
-	}
-	return zlib_return(res, rbuf, rlen);
-    }
-	
-    case DEFLATE_SETDICT:
-	if (d->state != ST_DEFLATE) goto badarg;
-	res = deflateSetDictionary(&d->s, (unsigned char*)buf, len);
-	if (res == Z_OK) {
-	    return zlib_value(d->s.adler, rbuf, rlen);
-	} else {
-	    return zlib_return(res, rbuf, rlen);
-	}
-
-    case DEFLATE_RESET:
-	if (len != 0) goto badarg;
-	if (d->state != ST_DEFLATE) goto badarg;
-	driver_deq(d->port, driver_sizeq(d->port));
-	res = deflateReset(&d->s);
-	return zlib_return(res, rbuf, rlen);	
-	
-    case DEFLATE_END:
-	if (len != 0) goto badarg;
-	if (d->state != ST_DEFLATE) goto badarg;
-	driver_deq(d->port, driver_sizeq(d->port));
-	res = deflateEnd(&d->s);
-	d->state = ST_NONE;
-	return zlib_return(res, rbuf, rlen);
-
-    case DEFLATE_PARAMS:
-	if (len != 8) goto badarg;
-	if (d->state != ST_DEFLATE) goto badarg;
-	res = deflateParams(&d->s, i32(buf), i32(buf+4));
-	return zlib_return(res, rbuf, rlen);
-
-    case DEFLATE:
-	if (d->state != ST_DEFLATE) goto badarg;
-	if (len != 4) goto badarg;
-	res = zlib_deflate(d, i32(buf));
-	return zlib_return(res, rbuf, rlen);
-
-    case INFLATE_INIT:
-	if (len != 0) goto badarg;
-	if (d->state != ST_NONE) goto badarg;
-	res = inflateInit(&d->s);
-	if (res == Z_OK) {
-	    d->state = ST_INFLATE;
-	    d->inflate_eos_seen = 0;
-	    d->want_crc = 0;
-	    d->crc = crc32(0L, Z_NULL, 0);
-	}
-	return zlib_return(res, rbuf, rlen);	
-	
-    case INFLATE_INIT2: {
-	int wbits;
-
-	if (len != 4) goto badarg;	
-	if (d->state != ST_NONE) goto badarg;
-	wbits = i32(buf);
-	res = inflateInit2(&d->s, wbits);
-	if (res == Z_OK) {
-	    d->state = ST_INFLATE;
-	    d->inflate_eos_seen = 0;
-	    d->want_crc = (wbits < 0);
-	    d->crc = crc32(0L, Z_NULL, 0);
-	}
-	return zlib_return(res, rbuf, rlen);
-    }
-	
-    case INFLATE_SETDICT:
-	if (d->state != ST_INFLATE) goto badarg;
-	res = inflateSetDictionary(&d->s, (unsigned char*)buf, len);
-	return zlib_return(res, rbuf, rlen);
-
-    case INFLATE_GETDICT:
-        if (have_inflateGetDictionary) {
-            if (d->state != ST_INFLATE) goto badarg;
-            res = zlib_inflate_get_dictionary(d);
-        } else {
-            errno = ENOTSUP;
-            res = Z_ERRNO;
-        }
-        return zlib_return(res, rbuf, rlen);
-
-    case INFLATE_SYNC:
-	if (d->state != ST_INFLATE) goto badarg;
-	if (len != 0) goto badarg;
-	if (driver_sizeq(d->port) == 0) {
-	    res = Z_BUF_ERROR;
-	} else {
-	    int vlen;
-	    SysIOVec* iov = driver_peekq(d->port, &vlen);
-
-	    d->s.next_in = iov[0].iov_base;
-	    d->s.avail_in = iov[0].iov_len;
-	    res = inflateSync(&d->s);
-	}
-	return zlib_return(res, rbuf, rlen);
-
-    case INFLATE_RESET:
-	if (d->state != ST_INFLATE) goto badarg;
-	if (len != 0) goto badarg;
-	driver_deq(d->port, driver_sizeq(d->port));
-	res = inflateReset(&d->s);
-	d->inflate_eos_seen = 0;
-	return zlib_return(res, rbuf, rlen);
-
-    case INFLATE_END:
-	if (d->state != ST_INFLATE) goto badarg;
-	if (len != 0) goto badarg;
-	driver_deq(d->port, driver_sizeq(d->port));
-	res = inflateEnd(&d->s);
-	if (res == Z_OK && d->inflate_eos_seen == 0) {
-	    res = Z_DATA_ERROR;
-	}
-	d->state = ST_NONE;
-	return zlib_return(res, rbuf, rlen);
-
-    case INFLATE:
-	if (d->state != ST_INFLATE) goto badarg;
-	if (len != 4) goto badarg;
-	res = zlib_inflate(d, i32(buf));
-	if (res == Z_NEED_DICT) {
-	    return zlib_value2(3, d->s.adler, rbuf, rlen);
-	} else {
-	    return zlib_return(res, rbuf, rlen);
-	}
-
-    case INFLATE_CHUNK:
-	if (d->state != ST_INFLATE) goto badarg;
-	if (len != 0) goto badarg;
-	res = zlib_inflate_chunk(d);
-	if (res == INFLATE_HAS_MORE) {
-	    return zlib_value2(4, 0, rbuf, rlen);
-	} else if (res == Z_NEED_DICT) {
-	    return zlib_value2(3, d->s.adler, rbuf, rlen);
-	} else {
-	    return zlib_return(res, rbuf, rlen);
-	}
-
-    case GET_QSIZE:
-	return zlib_value(driver_sizeq(d->port), rbuf, rlen);
-
-    case GET_BUFSZ:
-	return zlib_value(d->binsz_need, rbuf, rlen);
-
-    case SET_BUFSZ: {
-	int need;
-	if (len != 4) goto badarg;
-	need = i32(buf);
-	if ((need < 16) || (need > 0x00ffffff))
-	    goto badarg;
-	if (d->binsz_need != need) {
-	    d->binsz_need = need;
-	    if (d->bin != NULL) {
-		if (d->s.avail_out == d->binsz) {
-		    driver_free_binary(d->bin);
-		    d->bin = NULL;
-		    d->binsz = 0;
-		}
-		else
-		    zlib_output(d);
-	    }
-	}
-	return zlib_return(Z_OK, rbuf, rlen);
-    }
-
-    case CRC32_0:
-	return zlib_value(d->crc, rbuf, rlen);
-
-    case CRC32_1: {
-	uLong crc = crc32(0L, Z_NULL, 0);
-	crc = crc32(crc, (unsigned char*) buf, len);
-	return zlib_value(crc, rbuf, rlen);
-    }
-	
-    case CRC32_2: {
-	uLong crc;
-	if (len < 4) goto badarg;
-	crc = (unsigned int) i32(buf);
-	crc = crc32(crc, (unsigned char*) buf+4, len-4);
-	return zlib_value(crc, rbuf, rlen);
-    }
-
-    case ADLER32_1: {
-	uLong adler = adler32(0L, Z_NULL, 0);
-	adler = adler32(adler, (unsigned char*) buf, len);
-	return zlib_value(adler, rbuf, rlen);
-    }
-	
-    case ADLER32_2: {
-       uLong adler;
-       if (len < 4) goto badarg;
-       adler = (unsigned int) i32(buf);
-       adler = adler32(adler, (unsigned char*) buf+4, len-4);
-       return zlib_value(adler, rbuf, rlen);
-    }
-
-    case CRC32_COMBINE: {
-       uLong crc, crc1, crc2, len2;
-       if (len != 12) goto badarg;
-       crc1 = (unsigned int) i32(buf);
-       crc2 = (unsigned int) i32(buf+4);
-       len2 = (unsigned int) i32(buf+8);
-       crc = crc32_combine(crc1, crc2, len2);
-       return zlib_value(crc, rbuf, rlen);
-    }
-
-    case ADLER32_COMBINE: {
-       uLong adler, adler1, adler2, len2;
-       if (len != 12) goto badarg;
-       adler1 = (unsigned int) i32(buf);
-       adler2 = (unsigned int) i32(buf+4);
-       len2   = (unsigned int) i32(buf+8);
-       adler  = adler32_combine(adler1, adler2, len2);
-       return zlib_value(adler, rbuf, rlen);
-    }       
-    }
-
- badarg:
-    errno = EINVAL;
-    return zlib_return(Z_ERRNO, rbuf, rlen);
-}
-
-
-
-static void zlib_outputv(ErlDrvData drv_data, ErlIOVec *ev)
-{
-    ZLibData* d = (ZLibData*) drv_data;
-
-    driver_enqv(d->port, ev, 0);
-}
diff --git a/erts/emulator/nifs/common/zlib_nif.c b/erts/emulator/nifs/common/zlib_nif.c
new file mode 100644
index 0000000000..cffb3f3f63
--- /dev/null
+++ b/erts/emulator/nifs/common/zlib_nif.c
@@ -0,0 +1,998 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson 2017. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#define STATIC_ERLANG_NIF 1
+
+#include <stdio.h>
+#include <zlib.h>
+
+#include "erl_nif.h"
+#include "config.h"
+#include "sys.h"
+
+#ifdef VALGRIND
+#  include <valgrind/memcheck.h>
+#endif
+
+#define INFL_DICT_SZ    (32768)
+
+#ifndef MAX
+#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
+#endif
+
+#ifndef MIN
+#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
+#endif
+
+/* NIF interface declarations */
+static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info);
+static int upgrade(ErlNifEnv *env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info);
+static void unload(ErlNifEnv *env, void* priv_data);
+
+static ErlNifResourceType *rtype_zlib;
+
+static ERL_NIF_TERM am_ok;
+static ERL_NIF_TERM am_error;
+static ERL_NIF_TERM am_continue;
+static ERL_NIF_TERM am_finished;
+static ERL_NIF_TERM am_empty;
+static ERL_NIF_TERM am_not_supported;
+static ERL_NIF_TERM am_need_dictionary;
+static ERL_NIF_TERM am_stream_end;
+static ERL_NIF_TERM am_stream_error;
+static ERL_NIF_TERM am_data_error;
+static ERL_NIF_TERM am_mem_error;
+static ERL_NIF_TERM am_buf_error;
+static ERL_NIF_TERM am_version_error;
+static ERL_NIF_TERM am_unknown_error;
+
+typedef enum {
+    ST_NONE    = 0,
+    ST_DEFLATE = 1,
+    ST_INFLATE = 2,
+    ST_CLOSED = 3
+} zlib_state;
+
+typedef struct {
+    z_stream s;
+    zlib_state state;
+
+    /* These refer to the plaintext CRC, and are only needed for zlib:crc32/1
+     * which is deprecated. */
+    uLong input_crc;
+    uLong output_crc;
+    int want_input_crc;
+    int want_output_crc;
+
+    int is_raw_stream;
+
+    int eos_seen;
+
+    /* DEPRECATED */
+    int inflateChunk_buffer_size;
+
+    ErlNifPid controlling_process;
+
+    ErlNifIOQueue *input_queue;
+
+    ErlNifEnv *stash_env;
+    ERL_NIF_TERM stash_term;
+} zlib_data_t;
+
+/* The NIFs: */
+
+static ERL_NIF_TERM zlib_open(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_close(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_deflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_deflateInit2(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_deflateSetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_deflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_deflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_deflateParams(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_deflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+static ERL_NIF_TERM zlib_inflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_inflateInit2(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_inflateSetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_inflateGetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_inflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_inflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_inflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+static ERL_NIF_TERM zlib_crc32(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+static ERL_NIF_TERM zlib_clearStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_setStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_getStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+static ERL_NIF_TERM zlib_getBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_setBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+static ERL_NIF_TERM zlib_enqueue_input(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+static ErlNifFunc nif_funcs[] = {
+    /* deflate */
+    {"deflateInit_nif", 2, zlib_deflateInit},
+    {"deflateInit_nif", 6, zlib_deflateInit2},
+    {"deflateSetDictionary_nif", 2, zlib_deflateSetDictionary},
+    {"deflateReset_nif", 1, zlib_deflateReset},
+    {"deflateEnd_nif", 1, zlib_deflateEnd},
+    {"deflateParams_nif", 3, zlib_deflateParams},
+    {"deflate_nif", 4, zlib_deflate},
+
+    /* inflate */
+    {"inflateInit_nif", 1, zlib_inflateInit},
+    {"inflateInit_nif", 2, zlib_inflateInit2},
+    {"inflateSetDictionary_nif", 2, zlib_inflateSetDictionary},
+    {"inflateGetDictionary_nif", 1, zlib_inflateGetDictionary},
+    {"inflateReset_nif", 1, zlib_inflateReset},
+    {"inflateEnd_nif", 1, zlib_inflateEnd},
+    {"inflate_nif", 4, zlib_inflate},
+
+    /* running checksum */
+    {"crc32_nif", 1, zlib_crc32},
+
+    /* open & close */
+    {"close_nif", 1, zlib_close},
+    {"open_nif", 0, zlib_open},
+
+    /* The stash keeps a single term alive across calls, and is used in
+     * exception_on_need_dict/1 to retain the old error behavior, and for
+     * saving data flushed through deflateParams/3. */
+    {"getStash_nif", 1, zlib_getStash},
+    {"clearStash_nif", 1, zlib_clearStash},
+    {"setStash_nif", 2, zlib_setStash},
+
+    /* DEPRECATED: buffer size for inflateChunk */
+    {"getBufSize_nif", 1, zlib_getBufSize},
+    {"setBufSize_nif", 2, zlib_setBufSize},
+
+    {"enqueue_nif", 2, zlib_enqueue_input},
+};
+
+ERL_NIF_INIT(zlib, nif_funcs, load, NULL, upgrade, unload)
+
+static void gc_zlib(ErlNifEnv *env, void* data);
+
+static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info)
+{
+    am_ok = enif_make_atom(env, "ok");
+    am_error = enif_make_atom(env, "error");
+    am_empty = enif_make_atom(env, "empty");
+    am_continue = enif_make_atom(env, "continue");
+    am_finished = enif_make_atom(env, "finished");
+    am_need_dictionary = enif_make_atom(env, "need_dictionary");
+    am_not_supported = enif_make_atom(env, "not_supported");
+    am_stream_end = enif_make_atom(env, "stream_end");
+    am_stream_error = enif_make_atom(env, "stream_error");
+    am_data_error = enif_make_atom(env, "data_error");
+    am_mem_error = enif_make_atom(env, "mem_error");
+    am_buf_error = enif_make_atom(env, "buf_error");
+    am_version_error = enif_make_atom(env, "version_error");
+    am_unknown_error = enif_make_atom(env, "unknown_error");
+
+    rtype_zlib = enif_open_resource_type(env, NULL,
+            "gc_zlib", gc_zlib, ERL_NIF_RT_CREATE, NULL);
+    *priv_data = NULL;
+
+    return 0;
+}
+
+static void unload(ErlNifEnv *env, void* priv_data)
+{
+
+}
+
+static int upgrade(ErlNifEnv *env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
+{
+    if(*old_priv_data != NULL) {
+        return -1; /* Don't know how to do that */
+    }
+    if(*priv_data != NULL) {
+        return -1; /* Don't know how to do that */
+    }
+    if(load(env, priv_data, load_info)) {
+        return -1;
+    }
+    return 0;
+}
+
+static void* zlib_alloc(void* data, unsigned int items, unsigned int size)
+{
+    return (void*) enif_alloc(items * size);
+}
+
+static void zlib_free(void* data, void* addr)
+{
+    enif_free(addr);
+}
+
+static ERL_NIF_TERM zlib_return(ErlNifEnv *env, int code) {
+    ERL_NIF_TERM reason;
+    switch(code) {
+    case Z_OK:
+        reason = am_ok;
+        break;
+    case Z_STREAM_END:
+        reason = am_stream_end;
+        break;
+    case Z_ERRNO:
+        reason = enif_make_int(env, errno);
+        //enif_make_tuple2(env, enif_make_int(env, err), reason);
+        break;
+    case Z_STREAM_ERROR:
+        //reason = am_stream_error;
+        reason = enif_raise_exception(env, am_stream_error);
+        break;
+    case Z_DATA_ERROR:
+        //reason = am_data_error;
+        reason = enif_raise_exception(env, am_data_error);
+        break;
+    case Z_MEM_ERROR:
+        reason = am_mem_error;
+        break;
+    case Z_BUF_ERROR:
+        reason = am_buf_error;
+        break;
+    case Z_VERSION_ERROR:
+        reason = am_version_error;
+        break;
+    default:
+        reason = am_unknown_error;
+        break;
+    }
+    return reason;
+}
+
+static void gc_zlib(ErlNifEnv *env, void* data) {
+    zlib_data_t *d = (zlib_data_t*)data;
+
+    if(d->state == ST_DEFLATE) {
+        deflateEnd(&d->s);
+    } else if(d->state == ST_INFLATE) {
+        inflateEnd(&d->s);
+    }
+
+    if(d->state != ST_CLOSED) {
+        enif_ioq_destroy(d->input_queue);
+
+        if(d->stash_env != NULL) {
+            enif_free_env(d->stash_env);
+        }
+
+        d->state = ST_CLOSED;
+    }
+}
+
+static void zlib_reset_input(zlib_data_t *d) {
+    enif_ioq_destroy(d->input_queue);
+    d->input_queue = enif_ioq_create(ERL_NIF_IOQ_NORMAL);
+
+    if(d->stash_env != NULL) {
+        enif_free_env(d->stash_env);
+        d->stash_env = NULL;
+        d->stash_term = NIL;
+    }
+}
+
+static int get_zlib_data(ErlNifEnv* env, ERL_NIF_TERM opaque, zlib_data_t **d) {
+    ErlNifPid current_process;
+
+    if(!enif_get_resource(env, opaque, rtype_zlib, (void **)d)) {
+        return 0;
+    }
+
+    enif_self(env, &current_process);
+
+    return enif_is_identical(enif_make_pid(env, &current_process),
+        enif_make_pid(env, &(*d)->controlling_process));
+}
+
+static int zlib_flush_queue(int (*codec)(z_stream*, int), ErlNifEnv *env,
+        zlib_data_t *d, size_t input_limit, ErlNifBinary *output_buffer, int flush,
+        size_t *bytes_produced, size_t *bytes_consumed, size_t *bytes_remaining) {
+
+    int vec_len, vec_idx;
+    SysIOVec *input_vec;
+    int res;
+
+    input_vec = enif_ioq_peek(d->input_queue, &vec_len);
+    vec_idx = 0;
+    res = Z_OK;
+
+    *bytes_produced = 0;
+    *bytes_consumed = 0;
+
+    d->s.avail_out = output_buffer->size;
+    d->s.next_out = output_buffer->data;
+
+    while(res == Z_OK && vec_idx < vec_len && *bytes_consumed < input_limit) {
+        size_t timeslice_percent, block_consumed, block_size;
+
+        block_size = MIN(input_vec[vec_idx].iov_len, input_limit);
+
+        d->s.next_in = input_vec[vec_idx].iov_base;
+        d->s.avail_in = block_size;
+
+        res = codec(&d->s, Z_NO_FLUSH);
+
+        ASSERT(d->s.avail_in == 0 || d->s.avail_out == 0 || res != Z_OK);
+
+        block_consumed = block_size - d->s.avail_in;
+        *bytes_consumed += block_consumed;
+
+        if(d->want_input_crc) {
+            d->input_crc =
+                crc32(d->input_crc, input_vec[vec_idx].iov_base, block_consumed);
+        }
+
+        timeslice_percent = (100 * block_consumed) / input_limit;
+        if(enif_consume_timeslice(env, MAX(1, timeslice_percent))) {
+            break;
+        }
+
+        vec_idx++;
+    }
+
+    if(!enif_ioq_deq(d->input_queue, *bytes_consumed, bytes_remaining)) {
+        *bytes_remaining = 0;
+        res = Z_BUF_ERROR;
+    }
+
+    if(res == Z_OK && flush != Z_NO_FLUSH && (*bytes_remaining == 0)) {
+        d->s.next_in = NULL;
+        d->s.avail_in = 0;
+
+        res = codec(&d->s, flush);
+    }
+
+    *bytes_produced = output_buffer->size - d->s.avail_out;
+
+    return res;
+}
+
+static ERL_NIF_TERM zlib_codec(int (*codec)(z_stream*, int),
+                               ErlNifEnv *env, zlib_data_t *d,
+                               int input_chunk_size,
+                               int output_chunk_size,
+                               int flush) {
+
+    size_t bytes_produced, bytes_consumed, bytes_remaining;
+    ErlNifBinary output_buffer;
+    int res;
+
+    if(!enif_alloc_binary(output_chunk_size, &output_buffer)) {
+        return zlib_return(env, Z_MEM_ERROR);
+    }
+
+    res = zlib_flush_queue(codec, env, d, input_chunk_size, &output_buffer,
+        flush, &bytes_produced, &bytes_consumed, &bytes_remaining);
+
+    if(res < 0 && res != Z_BUF_ERROR) {
+        enif_release_binary(&output_buffer);
+        return zlib_return(env, res);
+    }
+
+    if(res == Z_STREAM_END) {
+        d->eos_seen = 1;
+    }
+
+    if(d->want_output_crc) {
+        d->output_crc =
+            crc32(d->output_crc, output_buffer.data, bytes_produced);
+    }
+
+    if(bytes_consumed == 0 && bytes_produced == 0 && bytes_remaining != 0) {
+        /* Die if we've made zero progress; this should not happen on
+         * well-formed input. */
+
+        enif_release_binary(&output_buffer);
+        return zlib_return(env, Z_DATA_ERROR);
+    } else {
+        ERL_NIF_TERM flushed_output;
+
+        if(bytes_produced > 0) {
+            if(bytes_produced < output_buffer.size) {
+                enif_realloc_binary(&output_buffer, bytes_produced);
+            }
+
+            flushed_output =
+                enif_make_list1(env, enif_make_binary(env, &output_buffer));
+        } else {
+            enif_release_binary(&output_buffer);
+            flushed_output = enif_make_list(env, 0);
+        }
+
+        if(bytes_remaining == 0 && bytes_produced < output_chunk_size) {
+            return enif_make_tuple2(env, am_finished, flushed_output);
+        } else if(res != Z_NEED_DICT) {
+            return enif_make_tuple2(env, am_continue, flushed_output);
+        }
+
+        return enif_make_tuple3(env, am_need_dictionary,
+            enif_make_int(env, d->s.adler), flushed_output);
+    }
+}
+
+/* zlib nifs */
+
+static ERL_NIF_TERM zlib_getStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+
+    if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->stash_env == NULL) {
+        return am_empty;
+    }
+
+    return enif_make_tuple2(env, am_ok, enif_make_copy(env, d->stash_term));
+}
+
+static ERL_NIF_TERM zlib_clearStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+
+    if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->stash_env == NULL) {
+        return enif_make_badarg(env);
+    }
+
+    enif_free_env(d->stash_env);
+    d->stash_env = NULL;
+    d->stash_term = NIL;
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM zlib_setStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+
+    if(argc != 2 || !get_zlib_data(env, argv[0], &d)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->stash_env != NULL) {
+        return enif_make_badarg(env);
+    }
+
+    d->stash_env = enif_alloc_env();
+    d->stash_term = enif_make_copy(d->stash_env, argv[1]);
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM zlib_open(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+    ERL_NIF_TERM result;
+
+    d = (zlib_data_t *) enif_alloc_resource(rtype_zlib, sizeof(zlib_data_t));
+
+    memset(&d->s, 0, sizeof(z_stream));
+
+    enif_self(env, &d->controlling_process);
+
+    d->input_queue = enif_ioq_create(ERL_NIF_IOQ_NORMAL);
+
+    d->s.zalloc = zlib_alloc;
+    d->s.zfree  = zlib_free;
+    d->s.opaque = d;
+    d->s.data_type = Z_BINARY;
+
+    d->state = ST_NONE;
+    d->eos_seen = 0;
+
+    d->want_output_crc = 0;
+    d->want_input_crc = 0;
+    d->is_raw_stream = 0;
+
+    d->output_crc = crc32(0L, Z_NULL, 0);
+    d->input_crc = crc32(0L, Z_NULL, 0);
+
+    d->stash_env = NULL;
+    d->stash_term = NIL;
+
+    d->inflateChunk_buffer_size = 4000;
+
+    result = enif_make_resource(env, d);
+    enif_release_resource(d);
+
+    return result;
+}
+
+static ERL_NIF_TERM zlib_close(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+
+    /* strictly speaking not needed since the gc will handle this */
+    if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state == ST_CLOSED) {
+        return enif_make_badarg(env);
+    }
+
+    gc_zlib(env, d);
+
+    return am_ok;
+}
+
+/* deflate */
+
+static ERL_NIF_TERM zlib_deflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+    int level, res;
+
+    if(argc != 2 || !get_zlib_data(env, argv[0], &d) ||
+                    !enif_get_int(env, argv[1], &level)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state != ST_NONE) {
+        return enif_make_badarg(env);
+    }
+
+    res = deflateInit(&d->s, level);
+
+    if(res == Z_OK) {
+        d->state = ST_DEFLATE;
+        d->eos_seen = 0;
+
+        /* FIXME: crc32/1 is documented as returning "the current calculated
+         * checksum," but failed to mention that the old implementation only
+         * calculated it when WindowBits < 0 (See zlib_deflateInit2).
+         *
+         * We could fix this behavior by setting d->want_input_crc to 1 here,
+         * but we've decided to retain this quirk since the performance hit is
+         * quite significant. */
+        d->want_output_crc = 0;
+        d->want_input_crc = 0;
+
+        d->output_crc = crc32(0L, Z_NULL, 0);
+        d->input_crc = crc32(0L, Z_NULL, 0);
+    }
+
+    return zlib_return(env, res);
+}
+
+static ERL_NIF_TERM zlib_deflateInit2(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+    int level, method, windowBits, memLevel, strategy, res;
+
+    if(argc != 6 || !get_zlib_data(env, argv[0], &d) 
+                 || !enif_get_int(env, argv[1], &level)
+                 || !enif_get_int(env, argv[2], &method)
+                 || !enif_get_int(env, argv[3], &windowBits)
+                 || !enif_get_int(env, argv[4], &memLevel)
+                 || !enif_get_int(env, argv[5], &strategy)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state != ST_NONE) {
+        return enif_make_badarg(env);
+    }
+
+    res = deflateInit2(&d->s, level, method, windowBits, memLevel, strategy);
+
+    if(res == Z_OK) {
+        d->state = ST_DEFLATE;
+        d->eos_seen = 0;
+
+        d->is_raw_stream = (windowBits < 0);
+
+        d->want_output_crc = 0;
+        d->want_input_crc = d->is_raw_stream;
+
+        d->output_crc = crc32(0L, Z_NULL, 0);
+        d->input_crc = crc32(0L, Z_NULL, 0);
+    }
+
+    return zlib_return(env, res);
+}
+
+static ERL_NIF_TERM zlib_deflateSetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+    ErlNifBinary bin;
+    int res;
+
+    if(argc != 2 || !get_zlib_data(env, argv[0], &d)
+                 || !enif_inspect_iolist_as_binary(env, argv[1], &bin)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state != ST_DEFLATE) {
+        return enif_make_badarg(env);
+    }
+
+    if((res = deflateSetDictionary(&d->s, bin.data, bin.size)) == Z_OK) {
+        uLong checksum = d->s.adler;
+
+        /* d->s.adler is not updated in raw deflate mode, so we'll calculate it
+         * ourselves in case the user wants to rely on that behavior. */
+        if(d->is_raw_stream) {
+            checksum = adler32(0, bin.data, bin.size);
+        }
+
+        return enif_make_int(env, checksum);
+    }
+
+    return zlib_return(env, res);
+}
+
+static ERL_NIF_TERM zlib_deflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+    int res;
+
+    if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state != ST_DEFLATE) {
+        return enif_make_badarg(env);
+    }
+
+    res = deflateReset(&d->s);
+
+    d->input_crc = crc32(0L, Z_NULL, 0);
+    d->eos_seen = 0;
+
+    zlib_reset_input(d);
+
+    return zlib_return(env, res);
+}
+
+static ERL_NIF_TERM zlib_deflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+    int res;
+
+    if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state != ST_DEFLATE) {
+        return enif_make_badarg(env);
+    }
+
+    res = deflateEnd(&d->s);
+
+    if(res == Z_OK && enif_ioq_size(d->input_queue) > 0) {
+        res = Z_DATA_ERROR;
+    }
+
+    zlib_reset_input(d);
+    d->state = ST_NONE;
+
+    return zlib_return(env, res);
+}
+
+static ERL_NIF_TERM zlib_deflateParams(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+    int res, level, strategy;
+
+    if(argc != 3 || !get_zlib_data(env, argv[0], &d)
+                 || !enif_get_int(env, argv[1], &level)
+                 || !enif_get_int(env, argv[2], &strategy)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state != ST_DEFLATE) {
+        return enif_make_badarg(env);
+    }
+
+    /* deflateParams will flush everything currently in the stream, corrupting
+     * the heap unless it's empty. We therefore pretend to have a full output
+     * buffer, forcing a Z_BUF_ERROR if there's anything left to be flushed. */
+    d->s.avail_out = 0;
+    res = deflateParams(&d->s, level, strategy);
+
+    return zlib_return(env, res);
+}
+
+static ERL_NIF_TERM zlib_deflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+
+    int input_chunk_size, output_chunk_size, flush;
+
+    if(argc != 4 || !get_zlib_data(env, argv[0], &d)
+                 || !enif_get_int(env, argv[1], &input_chunk_size)
+                 || !enif_get_int(env, argv[2], &output_chunk_size)
+                 || !enif_get_int(env, argv[3], &flush)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state != ST_DEFLATE) {
+        return enif_make_badarg(env);
+    }
+
+    return zlib_codec(&deflate, env, d, input_chunk_size, output_chunk_size, flush);
+}
+
+/* inflate */
+
+static ERL_NIF_TERM zlib_inflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+    int res;
+
+    if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state != ST_NONE) {
+        return enif_make_badarg(env);
+    }
+
+    res = inflateInit(&d->s);
+
+    if(res == Z_OK) {
+        d->state = ST_INFLATE;
+        d->eos_seen = 0;
+
+        d->want_output_crc = 0;
+        d->want_input_crc = 0;
+        d->is_raw_stream = 0;
+
+        d->output_crc = crc32(0L, Z_NULL, 0);
+        d->input_crc = crc32(0L, Z_NULL, 0);
+    }
+
+    return zlib_return(env, res);
+}
+
+static ERL_NIF_TERM zlib_inflateInit2(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+    int windowBits, res;
+
+    if(argc != 2 || !get_zlib_data(env, argv[0], &d)
+                 || !enif_get_int(env, argv[1], &windowBits)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state != ST_NONE) {
+        return enif_make_badarg(env);
+    }
+
+    res = inflateInit2(&d->s, windowBits);
+
+    if(res == Z_OK) {
+        d->state = ST_INFLATE;
+        d->eos_seen = 0;
+
+        d->is_raw_stream = (windowBits < 0);
+
+        d->want_output_crc = d->is_raw_stream;
+        d->want_input_crc = 0;
+
+        d->output_crc = crc32(0L, Z_NULL, 0);
+        d->input_crc = crc32(0L, Z_NULL, 0);
+    }
+
+    return zlib_return(env, res);
+}
+
+static ERL_NIF_TERM zlib_inflateSetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+    ErlNifBinary bin;
+    int res;
+
+    if(argc != 2 || !get_zlib_data(env, argv[0], &d)
+                 || !enif_inspect_iolist_as_binary(env, argv[1], &bin)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state != ST_INFLATE) {
+        return enif_make_badarg(env);
+    }
+
+    res = inflateSetDictionary(&d->s, bin.data, bin.size);
+
+    return zlib_return(env, res);
+}
+
+#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY
+/* Work around broken build system with runtime version test */
+static int zlib_supports_inflateGetDictionary(void) {
+    static int supportsGetDictionary = -1;
+
+#if defined(__APPLE__) && defined(__MACH__)
+    if(supportsGetDictionary < 0) {
+        unsigned int v[4] = {0, 0, 0, 0};
+        unsigned hexver;
+
+        sscanf(zlibVersion(), "%u.%u.%u.%u", &v[0], &v[1], &v[2], &v[3]);
+
+        hexver = (v[0] << (8*3)) | (v[1] << (8*2)) | (v[2] << (8)) | v[3];
+        supportsGetDictionary = (hexver >= 0x1020701); /* 1.2.7.1 */
+    }
+#endif
+
+    return supportsGetDictionary;
+}
+
+static ERL_NIF_TERM zlib_inflateGetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+    ErlNifBinary obin;
+    uInt len;
+    int res;
+
+    if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state != ST_INFLATE) {
+        return enif_make_badarg(env);
+    }
+
+    if(!zlib_supports_inflateGetDictionary()) {
+        return enif_make_badarg(env);
+    }
+
+    enif_alloc_binary(INFL_DICT_SZ, &obin);
+
+    len = 0;
+    if((res = inflateGetDictionary(&d->s, obin.data, &len)) < 0) {
+        enif_release_binary(&obin);
+        return zlib_return(env, res);
+    }
+
+    enif_realloc_binary(&obin, (size_t)len);
+    return enif_make_binary(env, &obin);
+}
+
+#else /* !HAVE_ZLIB_INFLATEGETDICTIONARY */
+
+static ERL_NIF_TERM zlib_inflateGetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    return enif_make_badarg(env);
+}
+
+#endif
+
+static ERL_NIF_TERM zlib_inflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+    int res;
+
+    if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state != ST_INFLATE) {
+        return enif_make_badarg(env);
+    }
+
+    res = inflateReset(&d->s);
+
+    d->output_crc = crc32(0L, Z_NULL, 0);
+    d->eos_seen = 0;
+
+    zlib_reset_input(d);
+
+    return zlib_return(env, res);
+}
+
+static ERL_NIF_TERM zlib_inflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+    int res;
+
+    if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state != ST_INFLATE) {
+        return enif_make_badarg(env);
+    }
+
+    res = inflateEnd(&d->s);
+
+    if(res == Z_OK && (!d->eos_seen || enif_ioq_size(d->input_queue) > 0)) {
+        res = Z_DATA_ERROR;
+    }
+
+    zlib_reset_input(d);
+    d->state = ST_NONE;
+
+    return zlib_return(env, res);
+}
+
+static ERL_NIF_TERM zlib_inflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+
+    int input_chunk_size, output_chunk_size, flush;
+
+    if(argc != 4 || !get_zlib_data(env, argv[0], &d)
+                 || !enif_get_int(env, argv[1], &input_chunk_size)
+                 || !enif_get_int(env, argv[2], &output_chunk_size)
+                 || !enif_get_int(env, argv[3], &flush)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state != ST_INFLATE) {
+        return enif_make_badarg(env);
+    }
+
+    return zlib_codec(&inflate, env, d, input_chunk_size, output_chunk_size, flush);
+}
+
+static ERL_NIF_TERM zlib_crc32(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+
+    if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state == ST_DEFLATE) {
+        return enif_make_ulong(env, d->input_crc);
+    } else if(d->state == ST_INFLATE) {
+        return enif_make_ulong(env, d->output_crc);
+    }
+
+    return enif_make_badarg(env);
+}
+
+static ERL_NIF_TERM zlib_getBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+
+    if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
+        return enif_make_badarg(env);
+    }
+
+    return enif_make_int(env, d->inflateChunk_buffer_size);
+}
+
+static ERL_NIF_TERM zlib_setBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+
+    if(argc != 2 || !get_zlib_data(env, argv[0], &d)
+                 || !enif_get_int(env, argv[1], &d->inflateChunk_buffer_size)) {
+        return enif_make_badarg(env);
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM zlib_enqueue_input(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+
+    ErlNifIOVec prealloc, *iovec = &prealloc;
+    ERL_NIF_TERM tail;
+
+    if(argc != 2 || !get_zlib_data(env, argv[0], &d)) {
+        return enif_make_badarg(env);
+    }
+
+    if(d->state != ST_DEFLATE && d->state != ST_INFLATE) {
+        return enif_make_badarg(env);
+    }
+
+    if(!enif_inspect_iovec(env, 256, argv[1], &tail, &iovec)) {
+        return enif_make_badarg(env);
+    }
+
+    if(!enif_ioq_enqv(d->input_queue, iovec, 0)) {
+        return enif_make_badarg(env);
+    }
+
+    if(!enif_is_empty_list(env, tail)) {
+        return enif_make_tuple2(env, am_continue, tail);
+    }
+
+    return am_ok;
+}
-- 
cgit v1.2.3


From 8e8a7f16acb479fc2737ff73e5e0b5569a7ab71f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 10 Aug 2017 14:25:34 +0200
Subject: Improve zlib error messages and update test suite to fit

OTP-14527
---
 erts/emulator/nifs/common/zlib_nif.c | 255 +++++++++++++++++++----------------
 1 file changed, 138 insertions(+), 117 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/nifs/common/zlib_nif.c b/erts/emulator/nifs/common/zlib_nif.c
index cffb3f3f63..b7f3adaffe 100644
--- a/erts/emulator/nifs/common/zlib_nif.c
+++ b/erts/emulator/nifs/common/zlib_nif.c
@@ -48,13 +48,22 @@ static void unload(ErlNifEnv *env, void* priv_data);
 
 static ErlNifResourceType *rtype_zlib;
 
+static ERL_NIF_TERM am_not_on_controlling_process;
+
+static ERL_NIF_TERM am_not_initialized;
+static ERL_NIF_TERM am_already_initialized;
+
 static ERL_NIF_TERM am_ok;
 static ERL_NIF_TERM am_error;
+
 static ERL_NIF_TERM am_continue;
 static ERL_NIF_TERM am_finished;
-static ERL_NIF_TERM am_empty;
+
 static ERL_NIF_TERM am_not_supported;
 static ERL_NIF_TERM am_need_dictionary;
+
+static ERL_NIF_TERM am_empty;
+
 static ERL_NIF_TERM am_stream_end;
 static ERL_NIF_TERM am_stream_error;
 static ERL_NIF_TERM am_data_error;
@@ -173,13 +182,23 @@ static void gc_zlib(ErlNifEnv *env, void* data);
 
 static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info)
 {
+    am_not_on_controlling_process =
+        enif_make_atom(env, "not_on_controlling_process");
+
+    am_not_initialized = enif_make_atom(env, "not_initialized");
+    am_already_initialized = enif_make_atom(env, "already_initialized");
+
     am_ok = enif_make_atom(env, "ok");
     am_error = enif_make_atom(env, "error");
-    am_empty = enif_make_atom(env, "empty");
+
     am_continue = enif_make_atom(env, "continue");
     am_finished = enif_make_atom(env, "finished");
-    am_need_dictionary = enif_make_atom(env, "need_dictionary");
+
     am_not_supported = enif_make_atom(env, "not_supported");
+    am_need_dictionary = enif_make_atom(env, "need_dictionary");
+
+    am_empty = enif_make_atom(env, "empty");
+
     am_stream_end = enif_make_atom(env, "stream_end");
     am_stream_error = enif_make_atom(env, "stream_error");
     am_data_error = enif_make_atom(env, "data_error");
@@ -235,14 +254,11 @@ static ERL_NIF_TERM zlib_return(ErlNifEnv *env, int code) {
         break;
     case Z_ERRNO:
         reason = enif_make_int(env, errno);
-        //enif_make_tuple2(env, enif_make_int(env, err), reason);
         break;
     case Z_STREAM_ERROR:
-        //reason = am_stream_error;
         reason = enif_raise_exception(env, am_stream_error);
         break;
     case Z_DATA_ERROR:
-        //reason = am_data_error;
         reason = enif_raise_exception(env, am_data_error);
         break;
     case Z_MEM_ERROR:
@@ -281,6 +297,19 @@ static void gc_zlib(ErlNifEnv *env, void* data) {
     }
 }
 
+static int get_zlib_data(ErlNifEnv *env, ERL_NIF_TERM opaque, zlib_data_t **d) {
+    return enif_get_resource(env, opaque, rtype_zlib, (void **)d);
+}
+
+static int zlib_process_check(ErlNifEnv *env, zlib_data_t *d) {
+    ErlNifPid current_process;
+
+    enif_self(env, &current_process);
+
+    return enif_is_identical(enif_make_pid(env, &current_process),
+        enif_make_pid(env, &d->controlling_process));
+}
+
 static void zlib_reset_input(zlib_data_t *d) {
     enif_ioq_destroy(d->input_queue);
     d->input_queue = enif_ioq_create(ERL_NIF_IOQ_NORMAL);
@@ -292,19 +321,6 @@ static void zlib_reset_input(zlib_data_t *d) {
     }
 }
 
-static int get_zlib_data(ErlNifEnv* env, ERL_NIF_TERM opaque, zlib_data_t **d) {
-    ErlNifPid current_process;
-
-    if(!enif_get_resource(env, opaque, rtype_zlib, (void **)d)) {
-        return 0;
-    }
-
-    enif_self(env, &current_process);
-
-    return enif_is_identical(enif_make_pid(env, &current_process),
-        enif_make_pid(env, &(*d)->controlling_process));
-}
-
 static int zlib_flush_queue(int (*codec)(z_stream*, int), ErlNifEnv *env,
         zlib_data_t *d, size_t input_limit, ErlNifBinary *output_buffer, int flush,
         size_t *bytes_produced, size_t *bytes_consumed, size_t *bytes_remaining) {
@@ -438,6 +454,8 @@ static ERL_NIF_TERM zlib_getStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM a
 
     if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
         return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
     }
 
     if(d->stash_env == NULL) {
@@ -452,10 +470,10 @@ static ERL_NIF_TERM zlib_clearStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM
 
     if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->stash_env == NULL) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->stash_env == NULL) {
+        return enif_raise_exception(env, am_error);
     }
 
     enif_free_env(d->stash_env);
@@ -470,10 +488,10 @@ static ERL_NIF_TERM zlib_setStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM a
 
     if(argc != 2 || !get_zlib_data(env, argv[0], &d)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->stash_env != NULL) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->stash_env != NULL) {
+        return enif_raise_exception(env, am_error);
     }
 
     d->stash_env = enif_alloc_env();
@@ -526,10 +544,10 @@ static ERL_NIF_TERM zlib_close(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv
     /* strictly speaking not needed since the gc will handle this */
     if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->state == ST_CLOSED) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->state == ST_CLOSED) {
+        return enif_raise_exception(env, am_not_initialized);
     }
 
     gc_zlib(env, d);
@@ -546,10 +564,10 @@ static ERL_NIF_TERM zlib_deflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TER
     if(argc != 2 || !get_zlib_data(env, argv[0], &d) ||
                     !enif_get_int(env, argv[1], &level)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->state != ST_NONE) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->state != ST_NONE) {
+        return enif_raise_exception(env, am_already_initialized);
     }
 
     res = deflateInit(&d->s, level);
@@ -586,10 +604,10 @@ static ERL_NIF_TERM zlib_deflateInit2(ErlNifEnv *env, int argc, const ERL_NIF_TE
                  || !enif_get_int(env, argv[4], &memLevel)
                  || !enif_get_int(env, argv[5], &strategy)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->state != ST_NONE) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->state != ST_NONE) {
+        return enif_raise_exception(env, am_already_initialized);
     }
 
     res = deflateInit2(&d->s, level, method, windowBits, memLevel, strategy);
@@ -618,10 +636,10 @@ static ERL_NIF_TERM zlib_deflateSetDictionary(ErlNifEnv *env, int argc, const ER
     if(argc != 2 || !get_zlib_data(env, argv[0], &d)
                  || !enif_inspect_iolist_as_binary(env, argv[1], &bin)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->state != ST_DEFLATE) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->state != ST_DEFLATE) {
+        return enif_raise_exception(env, am_not_initialized);
     }
 
     if((res = deflateSetDictionary(&d->s, bin.data, bin.size)) == Z_OK) {
@@ -645,10 +663,10 @@ static ERL_NIF_TERM zlib_deflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TE
 
     if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->state != ST_DEFLATE) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->state != ST_DEFLATE) {
+        return enif_raise_exception(env, am_not_initialized);
     }
 
     res = deflateReset(&d->s);
@@ -667,10 +685,10 @@ static ERL_NIF_TERM zlib_deflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM
 
     if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->state != ST_DEFLATE) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->state != ST_DEFLATE) {
+        return enif_raise_exception(env, am_not_initialized);
     }
 
     res = deflateEnd(&d->s);
@@ -693,10 +711,10 @@ static ERL_NIF_TERM zlib_deflateParams(ErlNifEnv *env, int argc, const ERL_NIF_T
                  || !enif_get_int(env, argv[1], &level)
                  || !enif_get_int(env, argv[2], &strategy)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->state != ST_DEFLATE) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->state != ST_DEFLATE) {
+        return enif_raise_exception(env, am_not_initialized);
     }
 
     /* deflateParams will flush everything currently in the stream, corrupting
@@ -718,10 +736,10 @@ static ERL_NIF_TERM zlib_deflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar
                  || !enif_get_int(env, argv[2], &output_chunk_size)
                  || !enif_get_int(env, argv[3], &flush)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->state != ST_DEFLATE) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->state != ST_DEFLATE) {
+        return enif_raise_exception(env, am_not_initialized);
     }
 
     return zlib_codec(&deflate, env, d, input_chunk_size, output_chunk_size, flush);
@@ -735,10 +753,10 @@ static ERL_NIF_TERM zlib_inflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TER
 
     if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->state != ST_NONE) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->state != ST_NONE) {
+        return enif_raise_exception(env, am_already_initialized);
     }
 
     res = inflateInit(&d->s);
@@ -765,10 +783,10 @@ static ERL_NIF_TERM zlib_inflateInit2(ErlNifEnv *env, int argc, const ERL_NIF_TE
     if(argc != 2 || !get_zlib_data(env, argv[0], &d)
                  || !enif_get_int(env, argv[1], &windowBits)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->state != ST_NONE) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->state != ST_NONE) {
+        return enif_raise_exception(env, am_already_initialized);
     }
 
     res = inflateInit2(&d->s, windowBits);
@@ -797,10 +815,10 @@ static ERL_NIF_TERM zlib_inflateSetDictionary(ErlNifEnv *env, int argc, const ER
     if(argc != 2 || !get_zlib_data(env, argv[0], &d)
                  || !enif_inspect_iolist_as_binary(env, argv[1], &bin)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->state != ST_INFLATE) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->state != ST_INFLATE) {
+        return enif_raise_exception(env, am_not_initialized);
     }
 
     res = inflateSetDictionary(&d->s, bin.data, bin.size);
@@ -827,55 +845,51 @@ static int zlib_supports_inflateGetDictionary(void) {
 
     return supportsGetDictionary;
 }
+#endif
 
 static ERL_NIF_TERM zlib_inflateGetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
     zlib_data_t *d;
-    ErlNifBinary obin;
-    uInt len;
-    int res;
 
     if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
         return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->state != ST_INFLATE) {
+        return enif_raise_exception(env, am_not_initialized);
     }
 
-    if(d->state != ST_INFLATE) {
-        return enif_make_badarg(env);
-    }
+#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY
+    if(zlib_supports_inflateGetDictionary()) {
+        ErlNifBinary obin;
+        uInt len;
+        int res;
 
-    if(!zlib_supports_inflateGetDictionary()) {
-        return enif_make_badarg(env);
-    }
+        enif_alloc_binary(INFL_DICT_SZ, &obin);
+        len = 0;
 
-    enif_alloc_binary(INFL_DICT_SZ, &obin);
+        if((res = inflateGetDictionary(&d->s, obin.data, &len)) < 0) {
+            enif_release_binary(&obin);
+            return zlib_return(env, res);
+        }
 
-    len = 0;
-    if((res = inflateGetDictionary(&d->s, obin.data, &len)) < 0) {
-        enif_release_binary(&obin);
-        return zlib_return(env, res);
+        enif_realloc_binary(&obin, (size_t)len);
+        return enif_make_binary(env, &obin);
     }
+#endif
 
-    enif_realloc_binary(&obin, (size_t)len);
-    return enif_make_binary(env, &obin);
-}
-
-#else /* !HAVE_ZLIB_INFLATEGETDICTIONARY */
-
-static ERL_NIF_TERM zlib_inflateGetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
-    return enif_make_badarg(env);
+    return enif_raise_exception(env, am_not_supported);
 }
 
-#endif
-
 static ERL_NIF_TERM zlib_inflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
     zlib_data_t *d;
     int res;
 
     if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->state != ST_INFLATE) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->state != ST_INFLATE) {
+        return enif_raise_exception(env, am_not_initialized);
     }
 
     res = inflateReset(&d->s);
@@ -894,10 +908,10 @@ static ERL_NIF_TERM zlib_inflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM
 
     if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->state != ST_INFLATE) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->state != ST_INFLATE) {
+        return enif_raise_exception(env, am_not_initialized);
     }
 
     res = inflateEnd(&d->s);
@@ -922,10 +936,10 @@ static ERL_NIF_TERM zlib_inflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar
                  || !enif_get_int(env, argv[2], &output_chunk_size)
                  || !enif_get_int(env, argv[3], &flush)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->state != ST_INFLATE) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->state != ST_INFLATE) {
+        return enif_raise_exception(env, am_not_initialized);
     }
 
     return zlib_codec(&inflate, env, d, input_chunk_size, output_chunk_size, flush);
@@ -936,6 +950,8 @@ static ERL_NIF_TERM zlib_crc32(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv
 
     if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
         return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
     }
 
     if(d->state == ST_DEFLATE) {
@@ -944,7 +960,7 @@ static ERL_NIF_TERM zlib_crc32(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv
         return enif_make_ulong(env, d->output_crc);
     }
 
-    return enif_make_badarg(env);
+    return enif_raise_exception(env, am_not_initialized);
 }
 
 static ERL_NIF_TERM zlib_getBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
@@ -952,6 +968,8 @@ static ERL_NIF_TERM zlib_getBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM
 
     if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
         return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
     }
 
     return enif_make_int(env, d->inflateChunk_buffer_size);
@@ -960,8 +978,13 @@ static ERL_NIF_TERM zlib_getBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM
 static ERL_NIF_TERM zlib_setBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
     zlib_data_t *d;
 
-    if(argc != 2 || !get_zlib_data(env, argv[0], &d)
-                 || !enif_get_int(env, argv[1], &d->inflateChunk_buffer_size)) {
+    if(argc != 2 || !get_zlib_data(env, argv[0], &d)) {
+        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    }
+
+    if(!enif_get_int(env, argv[1], &d->inflateChunk_buffer_size)) {
         return enif_make_badarg(env);
     }
 
@@ -976,17 +999,15 @@ static ERL_NIF_TERM zlib_enqueue_input(ErlNifEnv *env, int argc, const ERL_NIF_T
 
     if(argc != 2 || !get_zlib_data(env, argv[0], &d)) {
         return enif_make_badarg(env);
-    }
-
-    if(d->state != ST_DEFLATE && d->state != ST_INFLATE) {
-        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    } else if(d->state != ST_DEFLATE && d->state != ST_INFLATE) {
+        return enif_raise_exception(env, am_not_initialized);
     }
 
     if(!enif_inspect_iovec(env, 256, argv[1], &tail, &iovec)) {
         return enif_make_badarg(env);
-    }
-
-    if(!enif_ioq_enqv(d->input_queue, iovec, 0)) {
+    } else if(!enif_ioq_enqv(d->input_queue, iovec, 0)) {
         return enif_make_badarg(env);
     }
 
-- 
cgit v1.2.3


From 964354ba3b44b0e3a8b8341e63bc43ce24de2c02 Mon Sep 17 00:00:00 2001
From: Andrew Bennett <potatosaladx@gmail.com>
Date: Mon, 29 May 2017 11:14:51 -0600
Subject: stdlib: Improved BIF for binary matches and split.

---
 erts/emulator/beam/erl_alloc.types  |    1 +
 erts/emulator/beam/erl_bif_binary.c | 1108 +++++++++++++++++++----------------
 2 files changed, 602 insertions(+), 507 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 8142ea8893..11884299e2 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -258,6 +258,7 @@ type    MREF_ENT	STANDARD	SYSTEM		magic_ref_entry
 type	MREF_TAB_BKTS	STANDARD	SYSTEM		magic_ref_table_buckets
 type	MREF_TAB	LONG_LIVED	SYSTEM		magic_ref_table
 type	MINDIRECTION	FIXED_SIZE	SYSTEM		magic_indirection
+type	BINARY_FIND	SHORT_LIVED	PROCESSES	binary_find
 
 type	THR_Q_EL	STANDARD   	SYSTEM		thr_q_element
 type	THR_Q_EL_SL	FIXED_SIZE	SYSTEM		sl_thr_q_element
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 756c7dce05..fe31b44b1e 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -171,6 +171,16 @@ static void *my_alloc(MyAllocator *my, Uint size)
 
 #define ALPHABET_SIZE 256
 
+typedef struct _findall_data {
+    Uint pos;
+    Uint len;
+#ifdef HARDDEBUG
+    Uint id;
+#endif
+    Eterm epos;
+    Eterm elen;
+} FindallData;
+
 typedef struct _ac_node {
 #ifdef HARDDEBUG
     Uint32 id;                        /* To identify h pointer targets when
@@ -208,6 +218,103 @@ typedef struct _bm_data {
     Sint badshift[ALPHABET_SIZE];
 } BMData;
 
+typedef struct _ac_find_all_state {
+    ACNode *q;
+    Uint pos;
+    Uint len;
+    Uint m;
+    Uint allocated;
+    FindallData *out;
+} ACFindAllState;
+
+typedef struct _ac_find_first_state {
+    ACNode *q;
+    Uint pos;
+    Uint len;
+    ACNode *candidate;
+    Uint candidate_start;
+} ACFindFirstState;
+
+typedef struct _bm_find_all_state {
+    Sint pos;
+    Sint len;
+    Uint m;
+    Uint allocated;
+    FindallData *out;
+} BMFindAllState;
+
+typedef struct _bm_find_first_state {
+    Sint pos;
+    Sint len;
+} BMFindFirstState;
+
+typedef enum _bf_return {
+    BF_RESTART = -3,
+    BF_NOT_FOUND,
+    BF_BADARG,
+    BF_OK
+} BFReturn;
+
+typedef struct _binary_find_all_context {
+    ErtsHeapFactory factory;
+    Eterm term;
+    Sint head;
+    Sint tail;
+    Uint end_pos;
+    Uint size;
+    FindallData *data;
+    union {
+	ACFindAllState ac;
+	BMFindAllState bm;
+    } d;
+} BinaryFindAllContext;
+
+typedef struct _binary_find_first_context {
+    Uint pos;
+    Uint len;
+    union {
+	ACFindFirstState ac;
+	BMFindFirstState bm;
+    } d;
+} BinaryFindFirstContext;
+
+typedef struct _binary_find_context BinaryFindContext;
+
+typedef struct _binary_find_search {
+    void (*init) (BinaryFindContext *);
+    BFReturn (*find) (BinaryFindContext *, byte *);
+    void (*done) (BinaryFindContext *);
+} BinaryFindSearch;
+
+typedef Eterm (*BinaryFindResult)(Process *, Eterm, BinaryFindContext **);
+
+typedef enum _binary_find_state {
+    BFSearch,
+    BFResult,
+    BFDone
+} BinaryFindState;
+
+struct _binary_find_context {
+    Eterm pat_type;
+    Eterm pat_term;
+    Binary *pat_bin;
+    Uint flags;
+    Uint hsstart;
+    Uint hsend;
+    int loop_factor;
+    int exported;
+    Uint reds;
+    BinaryFindState state;
+    Eterm trap_term;
+    BinaryFindSearch *search;
+    BinaryFindResult not_found;
+    BinaryFindResult found;
+    union {
+	BinaryFindAllContext fa;
+	BinaryFindFirstContext ff;
+    } u;
+};
+
 #ifdef HARDDEBUG
 static void dump_bm_data(BMData *bm);
 static void dump_ac_trie(ACTrie *act);
@@ -421,32 +528,25 @@ static void ac_compute_failure_functions(ACTrie *act, ACNode **qbuff)
  * Basic AC finds the first end before the first start...
  *
  */
-typedef struct {
-    ACNode *q;
-    Uint pos;
-    Uint len;
-    ACNode *candidate;
-    Uint candidate_start;
-} ACFindFirstState;
-
-
-static void ac_init_find_first_match(ACFindFirstState *state, ACTrie *act, Sint startpos, Uint len)
+static void ac_init_find_first_match(BinaryFindContext *ctx)
 {
+    ACFindFirstState *state = &(ctx->u.ff.d.ac);
+    ACTrie *act = ERTS_MAGIC_BIN_DATA(ctx->pat_bin);
     state->q = act->root;
-    state->pos = startpos;
-    state->len = len;
+    state->pos = ctx->hsstart;
+    state->len = ctx->hsend;
     state->candidate = NULL;
     state->candidate_start = 0;
 }
-#define AC_OK 0
-#define AC_NOT_FOUND -1
-#define AC_RESTART -2
 
 #define AC_LOOP_FACTOR 10
 
-static int ac_find_first_match(ACFindFirstState *state, byte *haystack,
-				Uint *mpos, Uint *mlen, Uint *reductions)
+static BFReturn ac_find_first_match(BinaryFindContext *ctx, byte *haystack)
 {
+    ACFindFirstState *state = &(ctx->u.ff.d.ac);
+    Uint *mpos = &(ctx->u.ff.pos);
+    Uint *mlen = &(ctx->u.ff.len);
+    Uint *reductions = &(ctx->reds);
     ACNode *q = state->q;
     Uint i = state->pos;
     ACNode *candidate = state->candidate, *r;
@@ -462,7 +562,7 @@ static int ac_find_first_match(ACFindFirstState *state, byte *haystack,
 	    state->len = len;
 	    state->candidate = candidate;
 	    state->candidate_start = candidate_start;
-	    return AC_RESTART;
+	    return BF_RESTART;
 	}
 
 	while (q->g[haystack[i]] == NULL && q->h != q) {
@@ -492,68 +592,33 @@ static int ac_find_first_match(ACFindFirstState *state, byte *haystack,
     }
     *reductions = reds;
     if (!candidate) {
-	return AC_NOT_FOUND;
+	return BF_NOT_FOUND;
     }
 #ifdef HARDDEBUG
     dump_ac_node(candidate,0,'?');
 #endif
     *mpos = candidate_start;
     *mlen = candidate->d;
-    return AC_OK;
+    return BF_OK;
 }
 
-typedef struct _findall_data {
-    Uint pos;
-    Uint len;
-#ifdef HARDDEBUG
-    Uint id;
-#endif
-    Eterm epos;
-    Eterm elen;
-} FindallData;
-
-typedef struct {
-    ACNode *q;
-    Uint pos;
-    Uint len;
-    Uint m;
-    Uint allocated;
-    FindallData *out;
-} ACFindAllState;
-
-static void ac_init_find_all(ACFindAllState *state, ACTrie *act, Sint startpos, Uint len)
+static void ac_init_find_all(BinaryFindContext *ctx)
 {
+    ACFindAllState *state = &(ctx->u.fa.d.ac);
+    ACTrie *act = ERTS_MAGIC_BIN_DATA(ctx->pat_bin);
     state->q = act->root;
-    state->pos = startpos;
-    state->len = len;
+    state->pos = ctx->hsstart;
+    state->len = ctx->hsend;
     state->m = 0;
     state->allocated = 0;
     state->out = NULL;
 }
 
-static void ac_restore_find_all(ACFindAllState *state,
-                                const ACFindAllState *src)
-{
-    memcpy(state, src, sizeof(ACFindAllState));
-    if (state->allocated > 0) {
-	state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * (state->allocated));
-	memcpy(state->out, src+1, sizeof(FindallData)*state->m);
-    } else {
-	state->out = NULL;
-    }
-}
-
-static void ac_serialize_find_all(const ACFindAllState *state,
-                                  ACFindAllState *dst)
-{
-    memcpy(dst, state, sizeof(ACFindAllState));
-    memcpy(dst+1, state->out, sizeof(FindallData)*state->m);
-}
-
-static void ac_clean_find_all(ACFindAllState *state)
+static void ac_clean_find_all(BinaryFindContext *ctx)
 {
+    ACFindAllState *state = &(ctx->u.fa.d.ac);
     if (state->out != NULL) {
-	erts_free(ERTS_ALC_T_TMP, state->out);
+	erts_free(ERTS_ALC_T_BINARY_FIND, state->out);
     }
 #ifdef HARDDEBUG
     state->out = NULL;
@@ -565,9 +630,10 @@ static void ac_clean_find_all(ACFindAllState *state)
  * Differs to the find_first function in that it stores all matches and the values
  * arte returned only in the state.
  */
-static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack,
-				       Uint *reductions)
+static BFReturn ac_find_all_non_overlapping(BinaryFindContext *ctx, byte *haystack)
 {
+    ACFindAllState *state = &(ctx->u.fa.d.ac);
+    Uint *reductions = &(ctx->reds);
     ACNode *q = state->q;
     Uint i = state->pos;
     Uint rstart;
@@ -578,7 +644,6 @@ static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack,
     FindallData *out = state->out;
     register Uint reds = *reductions;
 
-
     while (i < len) {
 	if (--reds == 0) {
 	    state->q = q;
@@ -587,7 +652,7 @@ static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack,
 	    state->m = m;
 	    state->allocated = allocated;
 	    state->out = out;
-	    return AC_RESTART;
+	    return BF_RESTART;
 	}
 	while (q->g[haystack[i]] == NULL && q->h != q) {
 	    q = q->h;
@@ -625,11 +690,11 @@ static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack,
 		    if (m >= allocated) {
 			if (!allocated) {
 			    allocated = 10;
-			    out = erts_alloc(ERTS_ALC_T_TMP,
+			    out = erts_alloc(ERTS_ALC_T_BINARY_FIND,
 					     sizeof(FindallData) * allocated);
 			} else {
 			    allocated *= 2;
-			    out = erts_realloc(ERTS_ALC_T_TMP, out,
+			    out = erts_realloc(ERTS_ALC_T_BINARY_FIND, out,
 					       sizeof(FindallData) *
 					       allocated);
 			}
@@ -656,7 +721,7 @@ static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack,
     *reductions = reds;
     state->m = m;
     state->out = out;
-    return (m == 0) ? AC_NOT_FOUND : AC_OK;
+    return (m == 0) ? BF_NOT_FOUND : BF_OK;
 }
 
 /*
@@ -743,27 +808,22 @@ static void compute_goodshifts(BMData *bmd)
     erts_free(ERTS_ALC_T_TMP, suffixes);
 }
 
-typedef struct {
-    Sint pos;
-    Sint len;
-} BMFindFirstState;
-
-#define BM_OK 0 /* used only for find_all */
-#define BM_NOT_FOUND -1
-#define BM_RESTART -2
 #define BM_LOOP_FACTOR 10 /* Should we have a higher value? */
 
-static void bm_init_find_first_match(BMFindFirstState *state, Sint startpos,
-				     Uint len)
+static void bm_init_find_first_match(BinaryFindContext *ctx)
 {
-    state->pos = startpos;
-    state->len = (Sint) len;
+    BMFindFirstState *state = &(ctx->u.ff.d.bm);
+    state->pos = ctx->hsstart;
+    state->len = ctx->hsend;
 }
 
-
-static Sint bm_find_first_match(BMFindFirstState *state, BMData *bmd,
-				byte *haystack, Uint *reductions)
+static BFReturn bm_find_first_match(BinaryFindContext *ctx, byte *haystack)
 {
+    BMFindFirstState *state = &(ctx->u.ff.d.bm);
+    BMData *bmd = ERTS_MAGIC_BIN_DATA(ctx->pat_bin);
+    Uint *mpos = &(ctx->u.ff.pos);
+    Uint *mlen = &(ctx->u.ff.len);
+    Uint *reductions = &(ctx->reds);
     Sint blen = bmd->len;
     Sint len = state->len;
     Sint *gs = bmd->goodshift;
@@ -776,61 +836,37 @@ static Sint bm_find_first_match(BMFindFirstState *state, BMData *bmd,
     while (j <= len - blen) {
 	if (--reds == 0) {
 	    state->pos = j;
-	    return BM_RESTART;
+	    return BF_RESTART;
 	}
 	for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i)
 	    ;
 	if (i < 0) { /* found */
 	    *reductions = reds;
-	    return j;
+	    *mpos = (Uint) j;
+	    *mlen = (Uint) blen;
+	    return BF_OK;
 	}
 	j += MAX(gs[i],bs[haystack[i+j]] - blen + 1 + i);
     }
     *reductions = reds;
-    return BM_NOT_FOUND;
+    return BF_NOT_FOUND;
 }
 
-typedef struct {
-    Sint pos;
-    Sint len;
-    Uint m;
-    Uint allocated;
-    FindallData *out;
-} BMFindAllState;
-
-static void bm_init_find_all(BMFindAllState *state, Sint startpos, Uint len)
+static void bm_init_find_all(BinaryFindContext *ctx)
 {
-    state->pos = startpos;
-    state->len = (Sint) len;
+    BMFindAllState *state = &(ctx->u.fa.d.bm);
+    state->pos = ctx->hsstart;
+    state->len = ctx->hsend;
     state->m = 0;
     state->allocated = 0;
     state->out = NULL;
 }
 
-static void bm_restore_find_all(BMFindAllState *state,
-                                const BMFindAllState *src)
-{
-    memcpy(state, src, sizeof(BMFindAllState));
-    if (state->allocated > 0) {
-	state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) *
-				(state->allocated));
-	memcpy(state->out, src+1, sizeof(FindallData)*state->m);
-    } else {
-	state->out = NULL;
-    }
-}
-
-static void bm_serialize_find_all(const BMFindAllState *state,
-                                  BMFindAllState *dst)
-{
-    memcpy(dst, state, sizeof(BMFindAllState));
-    memcpy(dst+1, state->out, sizeof(FindallData)*state->m);
-}
-
-static void bm_clean_find_all(BMFindAllState *state)
+static void bm_clean_find_all(BinaryFindContext *ctx)
 {
+    BMFindAllState *state = &(ctx->u.fa.d.bm);
     if (state->out != NULL) {
-	erts_free(ERTS_ALC_T_TMP, state->out);
+	erts_free(ERTS_ALC_T_BINARY_FIND, state->out);
     }
 #ifdef HARDDEBUG
     state->out = NULL;
@@ -842,10 +878,11 @@ static void bm_clean_find_all(BMFindAllState *state)
  * Differs to the find_first function in that it stores all matches and the
  * values are returned only in the state.
  */
-static Sint bm_find_all_non_overlapping(BMFindAllState *state,
-					BMData *bmd, byte *haystack,
-					Uint *reductions)
+static BFReturn bm_find_all_non_overlapping(BinaryFindContext *ctx, byte *haystack)
 {
+    BMFindAllState *state = &(ctx->u.fa.d.bm);
+    BMData *bmd = ERTS_MAGIC_BIN_DATA(ctx->pat_bin);
+    Uint *reductions = &(ctx->reds);
     Sint blen = bmd->len;
     Sint len = state->len;
     Sint *gs = bmd->goodshift;
@@ -864,7 +901,7 @@ static Sint bm_find_all_non_overlapping(BMFindAllState *state,
 	    state->m = m;
 	    state->allocated = allocated;
 	    state->out = out;
-	    return BM_RESTART;
+	    return BF_RESTART;
 	}
 	for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i)
 	    ;
@@ -872,10 +909,11 @@ static Sint bm_find_all_non_overlapping(BMFindAllState *state,
 	    if (m >= allocated) {
 		if (!allocated) {
 		    allocated = 10;
-		    out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * allocated);
+		    out = erts_alloc(ERTS_ALC_T_BINARY_FIND,
+				     sizeof(FindallData) * allocated);
 		} else {
 		    allocated *= 2;
-		    out = erts_realloc(ERTS_ALC_T_TMP, out,
+		    out = erts_realloc(ERTS_ALC_T_BINARY_FIND, out,
 				       sizeof(FindallData) * allocated);
 		}
 	    }
@@ -890,7 +928,7 @@ static Sint bm_find_all_non_overlapping(BMFindAllState *state,
     state->m = m;
     state->out = out;
     *reductions = reds;
-    return (m == 0) ? BM_NOT_FOUND : BM_OK;
+    return (m == 0) ? BF_NOT_FOUND : BF_OK;
 }
 
 /*
@@ -1016,51 +1054,160 @@ BIF_RETTYPE binary_compile_pattern_1(BIF_ALIST_1)
     BIF_RET(ret);
 }
 
-#define DO_BIN_MATCH_OK 0
-#define DO_BIN_MATCH_BADARG -1
-#define DO_BIN_MATCH_RESTART -2
+#define BF_FLAG_GLOBAL		0x01
+#define BF_FLAG_SPLIT_TRIM	0x02
+#define BF_FLAG_SPLIT_TRIM_ALL	0x04
 
-#define BINARY_FIND_ALL		0x01
-#define BINARY_SPLIT_TRIM	0x02
-#define BINARY_SPLIT_TRIM_ALL	0x04
+static void bf_context_init(BinaryFindContext *ctx, BinaryFindResult not_found,
+			    BinaryFindResult single, BinaryFindResult global,
+			    Binary *pat_bin);
+static BinaryFindContext *bf_context_export(Process *p, BinaryFindContext *src);
+static int bf_context_destructor(Binary *ctx_bin);
+#ifdef HARDDEBUG
+static void bf_context_dump(BinaryFindContext *ctx);
+#endif
 
-typedef struct BinaryFindState {
-    Eterm type;
-    Uint  flags;
-    Uint  hsstart;
-    Uint  hsend;
-    Eterm (*not_found_result) (Process *, Eterm, struct BinaryFindState *);
-    Eterm (*single_result)    (Process *, Eterm, struct BinaryFindState *, Sint, Sint);
-    Eterm (*global_result)    (Process *, Eterm, struct BinaryFindState *, FindallData *, Uint);
-} BinaryFindState;
+static BinaryFindSearch bf_search_ac_global = {
+    ac_init_find_all,
+    ac_find_all_non_overlapping,
+    ac_clean_find_all
+};
+
+static BinaryFindSearch bf_search_ac_single = {
+    ac_init_find_first_match,
+    ac_find_first_match,
+    NULL
+};
+
+static BinaryFindSearch bf_search_bm_global = {
+    bm_init_find_all,
+    bm_find_all_non_overlapping,
+    bm_clean_find_all
+};
+
+static BinaryFindSearch bf_search_bm_single = {
+    bm_init_find_first_match,
+    bm_find_first_match,
+    NULL
+};
+
+static void bf_context_init(BinaryFindContext *ctx, BinaryFindResult not_found,
+			    BinaryFindResult single, BinaryFindResult global,
+			    Binary *pat_bin)
+{
+    ctx->exported = 0;
+    ctx->state = BFSearch;
+    ctx->not_found = not_found;
+    if (ctx->flags & BF_FLAG_GLOBAL) {
+	ctx->found = global;
+	if (ctx->pat_type == am_bm) {
+	    ctx->search = &bf_search_bm_global;
+	    ctx->loop_factor = BM_LOOP_FACTOR;
+	} else if (ctx->pat_type == am_ac) {
+	    ctx->search = &bf_search_ac_global;
+	    ctx->loop_factor = AC_LOOP_FACTOR;
+	}
+    } else {
+	ctx->found = single;
+	if (ctx->pat_type == am_bm) {
+	    ctx->search = &bf_search_bm_single;
+	    ctx->loop_factor = BM_LOOP_FACTOR;
+	} else if (ctx->pat_type == am_ac) {
+	    ctx->search = &bf_search_ac_single;
+	    ctx->loop_factor = AC_LOOP_FACTOR;
+	}
+    }
+    ctx->trap_term = THE_NON_VALUE;
+    ctx->pat_bin = pat_bin;
+    ctx->search->init(ctx);
+}
 
-typedef struct BinaryFindState_bignum {
-    Eterm           bignum_hdr;
-    BinaryFindState bfs;
-    union {
-	BMFindFirstState bmffs;
-	BMFindAllState   bmfas;
-	ACFindFirstState acffs;
-	ACFindAllState   acfas;
-    } data;
-} BinaryFindState_bignum;
-
-#define SIZEOF_BINARY_FIND_STATE(S) \
-	  (sizeof(BinaryFindState)+sizeof(S))
-
-#define SIZEOF_BINARY_FIND_ALL_STATE(S) \
-	  (sizeof(BinaryFindState)+sizeof(S)+(sizeof(FindallData)*(S).m))
-
-static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs);
-static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindState *bfs,
-				    Sint pos, Sint len);
-static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindState *bfs,
-				    FindallData *fad, Uint fad_sz);
-static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs);
-static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState *bfs,
-				    Sint pos, Sint len);
-static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState *bfs,
-				    FindallData *fad, Uint fad_sz);
+static BinaryFindContext *bf_context_export(Process *p, BinaryFindContext *src)
+{
+    Binary *ctx_bin;
+    BinaryFindContext *ctx;
+    Eterm *hp;
+
+    ASSERT(src->exported == 0);
+    ctx_bin = erts_create_magic_binary(sizeof(BinaryFindContext),
+				       bf_context_destructor);
+    ctx = ERTS_MAGIC_BIN_DATA(ctx_bin);
+    sys_memcpy(ctx, src, sizeof(BinaryFindContext));
+    if (ctx->pat_bin != NULL && ctx->pat_term == THE_NON_VALUE) {
+	hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE * 2);
+	ctx->pat_term = erts_mk_magic_ref(&hp, &MSO(p), ctx->pat_bin);
+    } else {
+	hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
+    }
+    ctx->trap_term = erts_mk_magic_ref(&hp, &MSO(p), ctx_bin);
+    ctx->exported = 1;
+    return ctx;
+}
+
+static int bf_context_destructor(Binary *ctx_bin)
+{
+    BinaryFindContext *ctx;
+
+    ctx = ERTS_MAGIC_BIN_DATA(ctx_bin);
+    if (ctx->state != BFDone) {
+	if (ctx->search->done != NULL) {
+	    ctx->search->done(ctx);
+	}
+	ctx->state = BFDone;
+    }
+    return 1;
+}
+
+#ifdef HARDDEBUG
+static void bf_context_dump(BinaryFindContext *ctx)
+{
+    if (ctx->pat_type == am_bm) {
+	BMData *bm;
+	bm = ERTS_MAGIC_BIN_DATA(ctx->pat_bin);
+	dump_bm_data(bm);
+    } else {
+	ACTrie *act;
+	act = ERTS_MAGIC_BIN_DATA(ctx->pat_bin);
+	dump_ac_trie(act);
+    }
+}
+#endif
+
+static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindContext **ctxp);
+static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindContext **ctxp);
+static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindContext **ctxp);
+static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindContext **ctxp);
+static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindContext **ctxp);
+static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindContext **ctxp);
+
+static BFReturn maybe_binary_match_compile(BinaryFindContext *ctx, Eterm arg, Binary **pat_bin)
+{
+    Eterm *tp;
+    ctx->pat_term = THE_NON_VALUE;
+    if (is_tuple(arg)) {
+	tp = tuple_val(arg);
+	if (arityval(*tp) != 2 || is_not_atom(tp[1])) {
+	    return BF_BADARG;
+	}
+	if (((tp[1] != am_bm) && (tp[1] != am_ac)) ||
+	    !is_internal_magic_ref(tp[2])) {
+	    return BF_BADARG;
+	}
+	*pat_bin = erts_magic_ref2bin(tp[2]);
+	if ((tp[1] == am_bm &&
+	     ERTS_MAGIC_BIN_DESTRUCTOR(*pat_bin) != cleanup_my_data_bm) ||
+	    (tp[1] == am_ac &&
+	     ERTS_MAGIC_BIN_DESTRUCTOR(*pat_bin) != cleanup_my_data_ac)) {
+	    *pat_bin = NULL;
+	    return BF_BADARG;
+	}
+	ctx->pat_type = tp[1];
+	ctx->pat_term = tp[2];
+    } else if (do_binary_match_compile(arg, &(ctx->pat_type), pat_bin) != 0) {
+	return BF_BADARG;
+    }
+    return BF_OK;
+}
 
 static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp)
 {
@@ -1141,17 +1288,17 @@ static int parse_split_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp, Uin
 	    Uint orig_size;
 	    if (is_atom(t)) {
 		if (t == am_global) {
-		    *optp |= BINARY_FIND_ALL;
+		    *optp |= BF_FLAG_GLOBAL;
 		    l = CDR(list_val(l));
 		    continue;
 		}
 		if (t == am_trim) {
-		    *optp |= BINARY_SPLIT_TRIM;
+		    *optp |= BF_FLAG_SPLIT_TRIM;
 		    l = CDR(list_val(l));
 		    continue;
 		}
 		if (t == am_trim_all) {
-		    *optp |= BINARY_SPLIT_TRIM_ALL;
+		    *optp |= BF_FLAG_SPLIT_TRIM_ALL;
 		    l = CDR(list_val(l));
 		    continue;
 		}
@@ -1204,266 +1351,160 @@ static int parse_split_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp, Uin
     }
 }
 
-static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binary *bin,
-			  Eterm state_term, Eterm *res_term)
+static BFReturn do_binary_find(Process *p, Eterm subject, BinaryFindContext **ctxp,
+			       Binary *pat_bin, Binary *ctx_bin, Eterm *res_term)
 {
-    byte *bytes;
-    Uint bitoffs, bitsize;
-    byte *temp_alloc = NULL;
-    BinaryFindState_bignum *state_ptr = NULL;
+    BinaryFindContext *ctx;
+    int is_first_call;
+    Uint initial_reds;
+    BFReturn runres;
 
-    ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize);
-    if (bitsize != 0) {
-	goto badarg;
-    }
-    if (bitoffs != 0) {
-	bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc);
-    }
-    if (state_term != NIL) {
-	state_ptr = (BinaryFindState_bignum *)(big_val(state_term));
-	bfs = &(state_ptr->bfs);
+    if (ctx_bin == NULL) {
+	is_first_call = 1;
+	ctx = *ctxp;
+    } else {
+	is_first_call = 0;
+	ctx = ERTS_MAGIC_BIN_DATA(ctx_bin);
+	ctx->pat_bin = pat_bin;
+	*ctxp = ctx;
     }
 
-    if (bfs->flags & BINARY_FIND_ALL) {
-	if (bfs->type == am_bm) {
-	    BMData *bm;
-	    Sint pos;
-	    BMFindAllState state;
-	    Uint reds = get_reds(p, BM_LOOP_FACTOR);
-	    Uint save_reds = reds;
+    initial_reds = ctx->reds = get_reds(p, ctx->loop_factor);
 
-	    bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin);
-#ifdef HARDDEBUG
-	    dump_bm_data(bm);
-#endif
-	    if (state_term == NIL) {
-		bm_init_find_all(&state, bfs->hsstart, bfs->hsend);
-	    } else {
-		bm_restore_find_all(&state, &(state_ptr->data.bmfas));
-	    }
+    switch (ctx->state) {
+    case BFSearch: {
+	byte *bytes;
+	Uint bitoffs, bitsize;
+	byte *temp_alloc = NULL;
 
-	    pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds);
-	    if (pos == BM_NOT_FOUND) {
-		*res_term = bfs->not_found_result(p, subject, bfs);
-	    } else if (pos == BM_RESTART) {
-		int x =
-		    (SIZEOF_BINARY_FIND_ALL_STATE(state) / sizeof(Eterm)) +
-		    !!(SIZEOF_BINARY_FIND_ALL_STATE(state) % sizeof(Eterm));
-#ifdef HARDDEBUG
-		erts_printf("Trap bm!\n");
-#endif
-		state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1);
-		state_ptr->bignum_hdr = make_pos_bignum_header(x);
-		memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState));
-		bm_serialize_find_all(&state, &state_ptr->data.bmfas);
-		*res_term = make_big(&state_ptr->bignum_hdr);
-		erts_free_aligned_binary_bytes(temp_alloc);
-		bm_clean_find_all(&state);
-		return DO_BIN_MATCH_RESTART;
-	    } else {
-		*res_term = bfs->global_result(p, subject, bfs, state.out, state.m);
-	    }
-	    erts_free_aligned_binary_bytes(temp_alloc);
-	    bm_clean_find_all(&state);
-	    BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
-	    return DO_BIN_MATCH_OK;
-	} else if (bfs->type == am_ac) {
-	    ACTrie *act;
-	    int acr;
-	    ACFindAllState state;
-	    Uint reds = get_reds(p, AC_LOOP_FACTOR);
-	    Uint save_reds = reds;
-
-	    act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin);
+	ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize);
+	if (bitsize != 0) {
+	    goto badarg;
+	}
+	if (bitoffs != 0) {
+	    bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc);
+	}
 #ifdef HARDDEBUG
-	    dump_ac_trie(act);
+	bf_context_dump(ctx);
 #endif
-	    if (state_term == NIL) {
-		ac_init_find_all(&state, act, bfs->hsstart, bfs->hsend);
-	    } else {
-		ac_restore_find_all(&state, &(state_ptr->data.acfas));
-	    }
-	    acr = ac_find_all_non_overlapping(&state, bytes, &reds);
-	    if (acr == AC_NOT_FOUND) {
-		*res_term = bfs->not_found_result(p, subject, bfs);
-	    } else if (acr == AC_RESTART) {
-		int x =
-		    (SIZEOF_BINARY_FIND_ALL_STATE(state) / sizeof(Eterm)) +
-		    !!(SIZEOF_BINARY_FIND_ALL_STATE(state) % sizeof(Eterm));
+	runres = ctx->search->find(ctx, bytes);
+	if (runres == BF_NOT_FOUND) {
+	    *res_term = ctx->not_found(p, subject, &ctx);
+	    *ctxp = ctx;
+	} else if (runres == BF_RESTART) {
 #ifdef HARDDEBUG
+	    if (ctx->pat_type == am_ac) {
 		erts_printf("Trap ac!\n");
-#endif
-                state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1);
-		state_ptr->bignum_hdr = make_pos_bignum_header(x);
-		memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState));
-		ac_serialize_find_all(&state, &state_ptr->data.acfas);
-		*res_term = make_big(&state_ptr->bignum_hdr);
-		erts_free_aligned_binary_bytes(temp_alloc);
-		ac_clean_find_all(&state);
-		return DO_BIN_MATCH_RESTART;
-	    } else {
-		*res_term = bfs->global_result(p, subject, bfs, state.out, state.m);
-	    }
-	    erts_free_aligned_binary_bytes(temp_alloc);
-	    ac_clean_find_all(&state);
-	    BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
-	    return DO_BIN_MATCH_OK;
-	}
-    } else {
-	if (bfs->type == am_bm) {
-	    BMData *bm;
-	    Sint pos;
-	    BMFindFirstState state;
-	    Uint reds = get_reds(p, BM_LOOP_FACTOR);
-	    Uint save_reds = reds;
-
-	    bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin);
-#ifdef HARDDEBUG
-	    dump_bm_data(bm);
-#endif
-	    if (state_term == NIL) {
-		bm_init_find_first_match(&state, bfs->hsstart, bfs->hsend);
 	    } else {
-		memcpy(&state, &state_ptr->data.bmffs, sizeof(BMFindFirstState));
-	    }
-
-#ifdef HARDDEBUG
-	    erts_printf("(bm) state->pos = %ld, state->len = %lu\n",state.pos,
-			state.len);
-#endif
-	    pos = bm_find_first_match(&state, bm, bytes, &reds);
-	    if (pos == BM_NOT_FOUND) {
-		*res_term = bfs->not_found_result(p, subject, bfs);
-	    } else if (pos == BM_RESTART) {
-		int x =
-		    (SIZEOF_BINARY_FIND_STATE(state) / sizeof(Eterm)) +
-		    !!(SIZEOF_BINARY_FIND_STATE(state) % sizeof(Eterm));
-#ifdef HARDDEBUG
 		erts_printf("Trap bm!\n");
+	    }
 #endif
-                state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1);
-                state_ptr->bignum_hdr = make_pos_bignum_header(x);
-		memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState));
-		memcpy(&state_ptr->data.acffs, &state, sizeof(BMFindFirstState));
-		*res_term = make_big(&state_ptr->bignum_hdr);
-		erts_free_aligned_binary_bytes(temp_alloc);
-		return DO_BIN_MATCH_RESTART;
-	    } else {
-		*res_term = bfs->single_result(p, subject, bfs, pos, bm->len);
+	    if (is_first_call) {
+		ctx = bf_context_export(p, ctx);
+		*ctxp = ctx;
+		erts_set_gc_state(p, 0);
 	    }
 	    erts_free_aligned_binary_bytes(temp_alloc);
-	    BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
-	    return DO_BIN_MATCH_OK;
-	} else if (bfs->type == am_ac) {
-	    ACTrie *act;
-	    Uint pos, rlen;
-	    int acr;
-	    ACFindFirstState state;
-	    Uint reds = get_reds(p, AC_LOOP_FACTOR);
-	    Uint save_reds = reds;
-
-	    act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin);
-#ifdef HARDDEBUG
-	    dump_ac_trie(act);
-#endif
-	    if (state_term == NIL) {
-		ac_init_find_first_match(&state, act, bfs->hsstart, bfs->hsend);
-	    } else {
-		memcpy(&state, &state_ptr->data.acffs, sizeof(ACFindFirstState));
+	    *res_term = THE_NON_VALUE;
+	    BUMP_ALL_REDS(p);
+	    return BF_RESTART;
+	} else {
+	    *res_term = ctx->found(p, subject, &ctx);
+	    *ctxp = ctx;
+	}
+	erts_free_aligned_binary_bytes(temp_alloc);
+	if (*res_term == THE_NON_VALUE) {
+	    if (is_first_call) {
+		erts_set_gc_state(p, 0);
 	    }
-	    acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds);
-	    if (acr == AC_NOT_FOUND) {
-		*res_term = bfs->not_found_result(p, subject, bfs);
-	    } else if (acr == AC_RESTART) {
-		int x =
-		    (SIZEOF_BINARY_FIND_STATE(state) / sizeof(Eterm)) +
-		    !!(SIZEOF_BINARY_FIND_STATE(state) % sizeof(Eterm));
-#ifdef HARDDEBUG
-		erts_printf("Trap ac!\n");
-#endif
-		state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1);
-		state_ptr->bignum_hdr = make_pos_bignum_header(x);
-		memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState));
-		memcpy(&state_ptr->data.acffs, &state, sizeof(ACFindFirstState));
-		*res_term = make_big(&state_ptr->bignum_hdr);
-		erts_free_aligned_binary_bytes(temp_alloc);
-		return DO_BIN_MATCH_RESTART;
-	    } else {
-		*res_term = bfs->single_result(p, subject, bfs, pos, rlen);
+	    BUMP_ALL_REDS(p);
+	    return BF_RESTART;
+	}
+	if (ctx->search->done != NULL) {
+	    ctx->search->done(ctx);
+	}
+	ctx->state = BFDone;
+	if (!is_first_call) {
+	    erts_set_gc_state(p, 1);
+	}
+	BUMP_REDS(p, (initial_reds - ctx->reds) / ctx->loop_factor);
+	return BF_OK;
+    }
+    case BFResult: {
+	*res_term = ctx->found(p, subject, &ctx);
+	*ctxp = ctx;
+	if (*res_term == THE_NON_VALUE) {
+	    if (is_first_call) {
+		erts_set_gc_state(p, 0);
 	    }
-	    erts_free_aligned_binary_bytes(temp_alloc);
-	    BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
-	    return DO_BIN_MATCH_OK;
+	    BUMP_ALL_REDS(p);
+	    return BF_RESTART;
+	}
+	if (ctx->search->done != NULL) {
+	    ctx->search->done(ctx);
 	}
+	ctx->state = BFDone;
+	if (!is_first_call) {
+	    erts_set_gc_state(p, 1);
+	}
+	BUMP_REDS(p, (initial_reds - ctx->reds) / ctx->loop_factor);
+	return BF_OK;
     }
- badarg:
-    return DO_BIN_MATCH_BADARG;
+    default:
+	ASSERT(!"Unknown state in do_binary_find");
+    }
+
+badarg:
+    if (!is_first_call) {
+	if (ctx->search->done != NULL) {
+	    ctx->search->done(ctx);
+	}
+	ctx->state = BFDone;
+	erts_set_gc_state(p, 1);
+    }
+    return BF_BADARG;
 }
 
 static BIF_RETTYPE
 binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Uint flags)
 {
-    BinaryFindState bfs;
-    Eterm *tp;
-    Binary *bin;
-    Eterm bin_term = NIL;
+    BinaryFindContext c_buff;
+    BinaryFindContext *ctx = &c_buff;
+    Binary *pat_bin;
     int runres;
     Eterm result;
 
-    if (is_not_binary(arg1)) {
+    if (is_not_binary(arg1) || binary_bitsize(arg1) != 0) {
 	goto badarg;
     }
-    bfs.flags = flags;
-    if (parse_match_opts_list(arg3, arg1, &(bfs.hsstart), &(bfs.hsend))) {
+    ctx->flags = flags;
+    if (parse_match_opts_list(arg3, arg1, &(ctx->hsstart), &(ctx->hsend))) {
 	goto badarg;
     }
-    if (bfs.hsend == 0) {
-	BIF_RET(do_match_not_found_result(p, arg1, &bfs));
+    if (ctx->hsend == 0) {
+	result = do_match_not_found_result(p, arg1, &ctx);
+	BIF_RET(result);
     }
-    if (is_tuple(arg2)) {
-	tp = tuple_val(arg2);
-	if (arityval(*tp) != 2 || is_not_atom(tp[1])) {
-	    goto badarg;
-	}
-	if (((tp[1] != am_bm) && (tp[1] != am_ac)) ||
-	    !is_internal_magic_ref(tp[2])) {
-	    goto badarg;
-	}
-	bfs.type = tp[1];
-	bin = erts_magic_ref2bin(tp[2]);
-	if (bfs.type == am_bm &&
-	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) {
-	    goto badarg;
-	}
-	if (bfs.type == am_ac &&
-	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) {
-	    goto badarg;
-	}
-	bin_term = tp[2];
-    } else if (do_binary_match_compile(arg2, &(bfs.type), &bin)) {
+    if (maybe_binary_match_compile(ctx, arg2, &pat_bin) != BF_OK) {
 	goto badarg;
     }
-    bfs.not_found_result = &do_match_not_found_result;
-    bfs.single_result = &do_match_single_result;
-    bfs.global_result = &do_match_global_result;
-    runres = do_binary_find(p, arg1, &bfs, bin, NIL, &result);
-    if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) {
-	Eterm *hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
-	bin_term = erts_mk_magic_ref(&hp, &MSO(p), bin);
-    } else if (bin_term == NIL) {
-	erts_bin_free(bin);
+    bf_context_init(ctx, do_match_not_found_result, do_match_single_result,
+		    do_match_global_result, pat_bin);
+    runres = do_binary_find(p, arg1, &ctx, pat_bin, NULL, &result);
+    if (runres == BF_OK && ctx->pat_term == THE_NON_VALUE) {
+	erts_bin_free(pat_bin);
     }
     switch (runres) {
-    case DO_BIN_MATCH_OK:
+    case BF_OK:
 	BIF_RET(result);
-    case DO_BIN_MATCH_RESTART:
-	BUMP_ALL_REDS(p);
-	BIF_TRAP3(&binary_find_trap_export, p, arg1, result, bin_term);
+    case BF_RESTART:
+	ASSERT(result == THE_NON_VALUE && ctx->trap_term != result && ctx->pat_term != result);
+	BIF_TRAP3(&binary_find_trap_export, p, arg1, ctx->trap_term, ctx->pat_term);
     default:
 	goto badarg;
     }
- badarg:
-    BIF_ERROR(p,BADARG);
+badarg:
+    BIF_ERROR(p, BADARG);
 }
 
 BIF_RETTYPE binary_match_2(BIF_ALIST_2)
@@ -1478,76 +1519,52 @@ BIF_RETTYPE binary_match_3(BIF_ALIST_3)
 
 BIF_RETTYPE binary_matches_2(BIF_ALIST_2)
 {
-    return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE, BINARY_FIND_ALL);
+    return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE, BF_FLAG_GLOBAL);
 }
 
 BIF_RETTYPE binary_matches_3(BIF_ALIST_3)
 {
-    return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BINARY_FIND_ALL);
+    return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BF_FLAG_GLOBAL);
 }
 
 static BIF_RETTYPE
 binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 {
-    BinaryFindState bfs;
-    Eterm *tp;
-    Binary *bin;
-    Eterm bin_term = NIL;
+    BinaryFindContext c_buff;
+    BinaryFindContext *ctx = &c_buff;
+    Binary *pat_bin;
     int runres;
     Eterm result;
 
-    if (is_not_binary(arg1)) {
+    if (is_not_binary(arg1) || binary_bitsize(arg1) != 0) {
 	goto badarg;
     }
-    if (parse_split_opts_list(arg3, arg1, &(bfs.hsstart), &(bfs.hsend), &(bfs.flags))) {
+    if (parse_split_opts_list(arg3, arg1, &(ctx->hsstart), &(ctx->hsend), &(ctx->flags))) {
 	goto badarg;
     }
-    if (bfs.hsend == 0) {
-	result = do_split_not_found_result(p, arg1, &bfs);
+    if (ctx->hsend == 0) {
+	result = do_split_not_found_result(p, arg1, &ctx);
 	BIF_RET(result);
     }
-    if (is_tuple(arg2)) {
-	tp = tuple_val(arg2);
-	if (arityval(*tp) != 2 || is_not_atom(tp[1])) {
-	    goto badarg;
-	}
-	if (((tp[1] != am_bm) && (tp[1] != am_ac)) ||
-	    !is_internal_magic_ref(tp[2])) {
-	    goto badarg;
-	}
-	bfs.type = tp[1];
-	bin = erts_magic_ref2bin(tp[2]);
-	if (bfs.type == am_bm &&
-	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) {
-	    goto badarg;
-	}
-	if (bfs.type == am_ac &&
-	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) {
-	    goto badarg;
-	}
-	bin_term = tp[2];
-    } else if (do_binary_match_compile(arg2, &(bfs.type), &bin)) {
+    if (maybe_binary_match_compile(ctx, arg2, &pat_bin) != BF_OK) {
 	goto badarg;
     }
-    bfs.not_found_result = &do_split_not_found_result;
-    bfs.single_result = &do_split_single_result;
-    bfs.global_result = &do_split_global_result;
-    runres = do_binary_find(p, arg1, &bfs, bin, NIL, &result);
-    if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) {
-	Eterm *hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
-	bin_term = erts_mk_magic_ref(&hp, &MSO(p), bin);
-    } else if (bin_term == NIL) {
-	erts_bin_free(bin);
-    }
-    switch(runres) {
-    case DO_BIN_MATCH_OK:
+    bf_context_init(ctx, do_split_not_found_result, do_split_single_result,
+		    do_split_global_result, pat_bin);
+    runres = do_binary_find(p, arg1, &ctx, pat_bin, NULL, &result);
+    if (runres == BF_OK && ctx->pat_term == THE_NON_VALUE) {
+	erts_bin_free(pat_bin);
+    }
+    switch (runres) {
+    case BF_OK:
 	BIF_RET(result);
-    case DO_BIN_MATCH_RESTART:
-	BIF_TRAP3(&binary_find_trap_export, p, arg1, result, bin_term);
+    case BF_RESTART:
+	ASSERT(result == THE_NON_VALUE && ctx->trap_term != result && ctx->pat_term != result);
+	BIF_TRAP3(&binary_find_trap_export, p, arg1, ctx->trap_term, ctx->pat_term);
     default:
 	goto badarg;
     }
- badarg:
+badarg:
     BIF_ERROR(p, BADARG);
 }
 
@@ -1561,72 +1578,117 @@ BIF_RETTYPE binary_split_3(BIF_ALIST_3)
     return binary_split(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
 }
 
-static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs)
+static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindContext **ctxp)
 {
-    if (bfs->flags & BINARY_FIND_ALL) {
+    if ((*ctxp)->flags & BF_FLAG_GLOBAL) {
 	return NIL;
     } else {
 	return am_nomatch;
     }
 }
 
-static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindState *bfs,
-				    Sint pos, Sint len)
+static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindContext **ctxp)
 {
+    BinaryFindContext *ctx = (*ctxp);
+    BinaryFindFirstContext *ff = &(ctx->u.ff);
     Eterm erlen;
     Eterm *hp;
     Eterm ret;
 
-    erlen = erts_make_integer((Uint)(len), p);
-    ret = erts_make_integer(pos, p);
+    erlen = erts_make_integer((Uint)(ff->len), p);
+    ret = erts_make_integer(ff->pos, p);
     hp = HAlloc(p, 3);
     ret = TUPLE2(hp, ret, erlen);
 
     return ret;
 }
 
-static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindState *bfs,
-				    FindallData *fad, Uint fad_sz)
+static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindContext **ctxp)
 {
-    Sint i;
+    BinaryFindContext *ctx = (*ctxp);
+    BinaryFindAllContext *fa = &(ctx->u.fa);
+    FindallData *fad;
     Eterm tpl;
-    Eterm *hp;
-    Eterm ret;
+    Sint i;
+    register Uint reds = ctx->reds;
 
-    for (i = 0; i < fad_sz; ++i) {
-	fad[i].epos = erts_make_integer(fad[i].pos, p);
-	fad[i].elen = erts_make_integer(fad[i].len, p);
+    if (ctx->state == BFSearch) {
+	if (ctx->pat_type == am_ac) {
+	    fa->data = fa->d.ac.out;
+	    fa->size = fa->d.ac.m;
+	} else {
+	    fa->data = fa->d.bm.out;
+	    fa->size = fa->d.bm.m;
+	}
+	fa->tail = fa->size - 1;
+	fa->head = 0;
+	fa->end_pos = 0;
+	fa->term = NIL;
+	if (ctx->exported == 0 && ((fa->size * 2) >= reds)) {
+	    ctx = bf_context_export(p, ctx);
+	    *ctxp = ctx;
+	    fa = &(ctx->u.fa);
+	}
+	erts_factory_proc_prealloc_init(&(fa->factory), p, fa->size * (3 + 2));
+	ctx->state = BFResult;
+    }
+
+    fad = fa->data;
+
+    if (fa->end_pos == 0) {
+	for (i = fa->head; i < fa->size; ++i) {
+	    if (--reds == 0) {
+		ASSERT(ctx->exported == 1);
+		fa->head = i;
+		ctx->reds = reds;
+		return THE_NON_VALUE;
+	    }
+	    fad[i].epos = erts_make_integer(fad[i].pos, p);
+	    fad[i].elen = erts_make_integer(fad[i].len, p);
+	}
+	fa->end_pos = 1;
+	fa->head = fa->tail;
     }
-    hp = HAlloc(p, fad_sz * (3 + 2));
-    ret = NIL;
-    for (i = fad_sz - 1; i >= 0; --i) {
-	tpl = TUPLE2(hp, fad[i].epos, fad[i].elen);
-	hp += 3;
-	ret = CONS(hp, tpl, ret);
-	hp += 2;
+
+    for (i = fa->head; i >= 0; --i) {
+	if (--reds == 0) {
+	    ASSERT(ctx->exported == 1);
+	    fa->head = i;
+	    ctx->reds = reds;
+	    return THE_NON_VALUE;
+	}
+	tpl = TUPLE2(fa->factory.hp, fad[i].epos, fad[i].elen);
+	fa->factory.hp += 3;
+	fa->term = CONS(fa->factory.hp, tpl, fa->term);
+	fa->factory.hp += 2;
     }
+    ctx->reds = reds;
+    erts_factory_close(&(fa->factory));
 
-    return ret;
+    return fa->term;
 }
 
-static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs)
+static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindContext **ctxp)
 {
+    BinaryFindContext *ctx = (*ctxp);
     Eterm *hp;
     Eterm ret;
 
-    if (bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)
+    if (ctx->flags & (BF_FLAG_SPLIT_TRIM | BF_FLAG_SPLIT_TRIM_ALL)
         && binary_size(subject) == 0) {
-        return NIL;
+	return NIL;
     }
     hp = HAlloc(p, 2);
     ret = CONS(hp, subject, NIL);
-
     return ret;
 }
 
-static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState *bfs,
-				    Sint pos, Sint len)
+static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindContext **ctxp)
 {
+    BinaryFindContext *ctx = (*ctxp);
+    BinaryFindFirstContext *ff = &(ctx->u.ff);
+    Sint pos;
+    Sint len;
     size_t orig_size;
     Eterm orig;
     Uint offset;
@@ -1637,9 +1699,12 @@ static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState *
     Eterm *hp;
     Eterm ret;
 
+    pos = ff->pos;
+    len = ff->len;
+
     orig_size = binary_size(subject);
 
-    if ((bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)) &&
+    if ((ctx->flags & (BF_FLAG_SPLIT_TRIM | BF_FLAG_SPLIT_TRIM_ALL)) &&
 	(orig_size - pos - len) == 0) {
 	if (pos == 0) {
 	    ret = NIL;
@@ -1660,7 +1725,7 @@ static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState *
 	    hp += 2;
 	}
     } else {
-	if ((bfs->flags & BINARY_SPLIT_TRIM_ALL) && (pos == 0)) {
+	if ((ctx->flags & BF_FLAG_SPLIT_TRIM_ALL) && (pos == 0)) {
 	    hp = HAlloc(p, 1 * (ERL_SUB_BIN_SIZE + 2));
 	    ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
 	    sb1 = NULL;
@@ -1698,39 +1763,60 @@ static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState *
     return ret;
 }
 
-static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState *bfs,
-				    FindallData *fad, Uint fad_sz)
+static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindContext **ctxp)
 {
-    size_t orig_size;
+    BinaryFindContext *ctx = (*ctxp);
+    BinaryFindAllContext *fa = &(ctx->u.fa);
+    FindallData *fad;
     Eterm orig;
+    size_t orig_size;
     Uint offset;
     Uint bit_offset;
     Uint bit_size;
     ErlSubBin *sb;
+    Uint do_trim;
     Sint i;
-    Sint tail;
-    Uint list_size;
-    Uint end_pos;
-    Uint do_trim = bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL);
-    Eterm *hp;
-    Eterm *hendp;
-    Eterm ret;
+    register Uint reds = ctx->reds;
 
-    tail = fad_sz - 1;
-    list_size = fad_sz + 1;
-    orig_size = binary_size(subject);
-    end_pos = (Uint)(orig_size);
+    if (ctx->state == BFSearch) {
+	if (ctx->pat_type == am_ac) {
+	    fa->data = fa->d.ac.out;
+	    fa->size = fa->d.ac.m;
+	} else {
+	    fa->data = fa->d.bm.out;
+	    fa->size = fa->d.bm.m;
+	}
+	fa->tail = fa->size - 1;
+	fa->head = fa->tail;
+	orig_size = binary_size(subject);
+	fa->end_pos = (Uint)(orig_size);
+	fa->term = NIL;
+	if (ctx->exported == 0 && ((fa->head + 1) >= reds)) {
+	    ctx = bf_context_export(p, ctx);
+	    *ctxp = ctx;
+	    fa = &(ctx->u.fa);
+	}
+	erts_factory_proc_prealloc_init(&(fa->factory), p, (fa->size + 1) * (ERL_SUB_BIN_SIZE + 2));
+	ctx->state = BFResult;
+    }
 
-    hp = HAlloc(p, list_size * (ERL_SUB_BIN_SIZE + 2));
-    hendp = hp + list_size * (ERL_SUB_BIN_SIZE + 2);
     ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size);
     ASSERT(bit_size == 0);
+    fad = fa->data;
+    do_trim = ctx->flags & (BF_FLAG_SPLIT_TRIM | BF_FLAG_SPLIT_TRIM_ALL);
 
-    ret = NIL;
-
-    for (i = tail; i >= 0; --i) {
-	sb = (ErlSubBin *)(hp);
-	sb->size = end_pos - (fad[i].pos + fad[i].len);
+    for (i = fa->head; i >= 0; --i) {
+	if (--reds == 0) {
+	    ASSERT(ctx->exported == 1);
+	    fa->head = i;
+	    ctx->reds = reds;
+	    if (!do_trim && (ctx->flags & BF_FLAG_SPLIT_TRIM)) {
+		ctx->flags &= ~BF_FLAG_SPLIT_TRIM;
+	    }
+	    return THE_NON_VALUE;
+	}
+	sb = (ErlSubBin *)(fa->factory.hp);
+	sb->size = fa->end_pos - (fad[i].pos + fad[i].len);
 	if (!(sb->size == 0 && do_trim)) {
 	    sb->thing_word = HEADER_SUB_BIN;
 	    sb->offs = offset + fad[i].pos + fad[i].len;
@@ -1738,15 +1824,18 @@ static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState *
 	    sb->bitoffs = bit_offset;
 	    sb->bitsize = 0;
 	    sb->is_writable = 0;
-	    hp += ERL_SUB_BIN_SIZE;
-	    ret = CONS(hp, make_binary(sb), ret);
-	    hp += 2;
-	    do_trim &= ~BINARY_SPLIT_TRIM;
+	    fa->factory.hp += ERL_SUB_BIN_SIZE;
+	    fa->term = CONS(fa->factory.hp, make_binary(sb), fa->term);
+	    fa->factory.hp += 2;
+	    do_trim &= ~BF_FLAG_SPLIT_TRIM;
 	}
-	end_pos = fad[i].pos;
+	fa->end_pos = fad[i].pos;
     }
 
-    sb = (ErlSubBin *)(hp);
+    fa->head = i;
+    ctx->reds = reds;
+
+    sb = (ErlSubBin *)(fa->factory.hp);
     sb->size = fad[0].pos;
     if (!(sb->size == 0 && do_trim)) {
 	sb->thing_word = HEADER_SUB_BIN;
@@ -1755,26 +1844,31 @@ static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState *
 	sb->bitoffs = bit_offset;
 	sb->bitsize = 0;
 	sb->is_writable = 0;
-	hp += ERL_SUB_BIN_SIZE;
-	ret = CONS(hp, make_binary(sb), ret);
-	hp += 2;
+	fa->factory.hp += ERL_SUB_BIN_SIZE;
+	fa->term = CONS(fa->factory.hp, make_binary(sb), fa->term);
+	fa->factory.hp += 2;
     }
-    HRelease(p, hendp, hp);
-    return ret;
+    erts_factory_close(&(fa->factory));
+
+    return fa->term;
 }
 
 static BIF_RETTYPE binary_find_trap(BIF_ALIST_3)
 {
     int runres;
     Eterm result;
-    Binary *bin = erts_magic_ref2bin(BIF_ARG_3);
-
-    runres = do_binary_find(BIF_P, BIF_ARG_1, NULL, bin, BIF_ARG_2, &result);
-    if (runres == DO_BIN_MATCH_OK) {
+    Binary *ctx_bin = erts_magic_ref2bin(BIF_ARG_2);
+    Binary *pat_bin = erts_magic_ref2bin(BIF_ARG_3);
+    BinaryFindContext *ctx = NULL;
+
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == bf_context_destructor);
+    runres = do_binary_find(BIF_P, BIF_ARG_1, &ctx, pat_bin, ctx_bin, &result);
+    if (runres == BF_OK) {
+	ASSERT(result != THE_NON_VALUE);
 	BIF_RET(result);
     } else {
-	BUMP_ALL_REDS(BIF_P);
-	BIF_TRAP3(&binary_find_trap_export, BIF_P, BIF_ARG_1, result, BIF_ARG_3);
+	ASSERT(result == THE_NON_VALUE && ctx->trap_term != result && ctx->pat_term != result);
+	BIF_TRAP3(&binary_find_trap_export, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
     }
 }
 
-- 
cgit v1.2.3


From 1e67f9542a8d1be6c1d31048db19b7abe6f47372 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 7 Sep 2017 08:18:11 +0200
Subject: Replace ad-hoc MIN/MAX macros with common ones

Besides being noisy, they were already defined by a global Unix-
specific header, causing the Windows build to fail if one forgot to
define them.
---
 erts/emulator/beam/beam_emu.c        | 4 ----
 erts/emulator/beam/erl_alloc_util.h  | 4 ----
 erts/emulator/beam/erl_bif_binary.c  | 7 -------
 erts/emulator/beam/erl_bits.c        | 9 ---------
 erts/emulator/beam/erl_db_hash.c     | 3 ---
 erts/emulator/beam/erl_db_tree.c     | 3 ---
 erts/emulator/beam/erl_io_queue.c    | 8 --------
 erts/emulator/beam/erl_nif.c         | 8 --------
 erts/emulator/beam/sys.h             | 8 ++++++++
 erts/emulator/nifs/common/zlib_nif.c | 8 --------
 10 files changed, 8 insertions(+), 54 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index bc83699951..b4e6c35579 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -113,10 +113,6 @@ do {                                     \
 #  define CHECK_ARGS(T)
 #endif
 
-#ifndef MAX
-#define MAX(x, y) (((x) > (y)) ? (x) : (y))
-#endif
-
 #define GET_BIF_MODULE(p)  (p->info.mfa.module)
 #define GET_BIF_FUNCTION(p)  (p->info.mfa.function)
 #define GET_BIF_ARITY(p)  (p->info.mfa.arity)
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 73c467aa0a..aa13cda9fb 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -238,10 +238,6 @@ void erts_lcnt_update_allocator_locks(int enable);
 #  endif
 #endif
 
-#undef MIN
-#undef MAX
-#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
-#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
 #define FLOOR(X, I) (((X)/(I))*(I))
 #define CEILING(X, I)  ((((X) - 1)/(I) + 1)*(I))
 
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 756c7dce05..dcffde5777 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -229,13 +229,6 @@ static void dump_ac_node(ACNode *node, int indent, int ch);
  MYALIGN(sizeof(ACTrie)))      /* Structure */
 
 
-#ifndef MAX
-#define MAX(A,B) (((A) > (B)) ? (A) : (B))
-#endif
-
-#ifndef MIN
-#define MIN(A,B) (((A) > (B)) ? (B) : (A))
-#endif
 /*
  * Callback for the magic binary
  */
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index b4e611f01b..51d23a8965 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -32,15 +32,6 @@
 #include "erl_bits.h"
 #include "erl_binary.h"
 
-#ifdef MAX
-#undef MAX
-#endif
-#define MAX(x,y) (((x)>(y))?(x):(y))
-#ifdef MIN
-#undef MIN
-#endif
-#define MIN(x,y) (((x)<(y))?(x):(y))
-
 #if defined(WORDS_BIGENDIAN)
 # define BIT_ENDIAN_MACHINE 0
 #else
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index ae9322dfd3..cf928a9035 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -288,9 +288,6 @@ static ERTS_INLINE Sint next_slot_w(DbTableHash* tb, Uint ix,
 #endif
 }
 
-#ifndef MIN
-#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
-#endif
 
 /*
  * Some special binary flags
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index d7deadacf0..7c80e92e50 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -85,9 +85,6 @@
 
 #define EMPTY_NODE(Dtt) (TOP_NODE(Dtt) == NULL)
 
-#ifndef MIN
-#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
-#endif
 
 /* Obtain table static stack if available. NULL if not.
 ** Must be released with release_stack()
diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c
index 301b55b315..a01b676d39 100644
--- a/erts/emulator/beam/erl_io_queue.c
+++ b/erts/emulator/beam/erl_io_queue.c
@@ -32,14 +32,6 @@
 #include "erl_bits.h"
 #include "erl_io_queue.h"
 
-#ifndef MAX
-#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
-#endif
-
-#ifndef MIN
-#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
-#endif
-
 #define IOL2V_SMALL_BIN_LIMIT (ERL_ONHEAP_BIN_LIMIT * 4)
 
 static void free_binary(ErtsIOQBinary *b, int driver);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index a3016ca912..ac4ecd77e5 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -67,14 +67,6 @@
 #include <limits.h>
 #include <stddef.h> /* offsetof */
 
-#ifndef MAX
-#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
-#endif
-
-#ifndef MIN
-#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
-#endif
-
 /* Information about a loaded nif library.
  * Each successful call to erlang:load_nif will allocate an instance of
  * erl_module_nif. Two calls opening the same library will thus have the same
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 6662685f54..704d567337 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1268,6 +1268,14 @@ void erl_bin_write(unsigned char *, int, int);
 #  define DEBUGF(x)
 #endif
 
+#ifndef MAX
+#define MAX(A, B) ((A) > (B) ? (A) : (B))
+#endif
+
+#ifndef MIN
+#define MIN(A, B) ((A) < (B) ? (A) : (B))
+#endif
+
 #ifdef __WIN32__
 #ifdef ARCH_64
 #define ERTS_ALLOC_ALIGN_BYTES 16
diff --git a/erts/emulator/nifs/common/zlib_nif.c b/erts/emulator/nifs/common/zlib_nif.c
index b7f3adaffe..a1a65e1946 100644
--- a/erts/emulator/nifs/common/zlib_nif.c
+++ b/erts/emulator/nifs/common/zlib_nif.c
@@ -33,14 +33,6 @@
 
 #define INFL_DICT_SZ    (32768)
 
-#ifndef MAX
-#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
-#endif
-
-#ifndef MIN
-#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
-#endif
-
 /* NIF interface declarations */
 static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info);
 static int upgrade(ErlNifEnv *env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info);
-- 
cgit v1.2.3


From f9a323d10a9f5da305cbdec632d6bdf7ad4770b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Sat, 26 Aug 2017 15:15:01 +0200
Subject: Optimise equality comparisons

* In both loader and compiler, make sure constants are always the second
  operand - many passes of the compiler assume that's always the case.
* In loader rewrite is_eq_exact with same arguments to skip the instruction
  and with different constants move one to an x register to maintain
  the properly outlined above.
* The same (but in reverse) is done with the is_ne_exact, where we rewrite
  to an unconditional jump or add a move to an x register.
* All of the above allow to replace is_eq_exact_fss with is_eq_exact_fyy and
  is_ne_exact_fss with is_ne_exact_fSS as those are the only possibilities left.
---
 erts/emulator/beam/ops.tab | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index b6e995fdbe..ba0aa005c9 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -413,9 +413,18 @@ send
 # Optimized comparisons with one immediate/literal operand.
 #
 
-is_eq_exact Lbl R=xy C=ian => i_is_eq_exact_immed Lbl R C
+is_eq_exact Lbl S S =>
+is_eq_exact Lbl C1=c C2=c => move C1 x | is_eq_exact Lbl x C2
+is_eq_exact Lbl C=c R=xy => is_eq_exact Lbl R C
+
+is_eq_exact Lbl R=xy n => is_nil Lbl R
+is_eq_exact Lbl R=xy C=ia => i_is_eq_exact_immed Lbl R C
 is_eq_exact Lbl R=xy C=q => i_is_eq_exact_literal Lbl R C
 
+is_ne_exact Lbl S S => jump Lbl
+is_ne_exact Lbl C1=c C2=c => move C1 x | is_ne_exact Lbl x C2
+is_ne_exact Lbl C=c R=xy => is_ne_exact Lbl R C
+
 is_ne_exact Lbl R=xy C=ian => i_is_ne_exact_immed Lbl R C
 is_ne_exact Lbl R=xy C=q => i_is_ne_exact_literal Lbl R C
 
@@ -429,7 +438,9 @@ i_is_ne_exact_literal f xy c
 
 is_eq_exact Lbl Y=y X=x => is_eq_exact Lbl X Y
 is_eq_exact f x xy
-is_eq_exact f s s
+is_eq_exact f y y
+
+is_ne_exact f S S
 
 is_lt f x x
 is_lt f x c
@@ -445,8 +456,6 @@ is_ge f c x
 is_ge f s s
 %hot
 
-is_ne_exact f s s
-
 is_eq f s s
 
 is_ne f s s
-- 
cgit v1.2.3


From a9812e6307fe335d077f96d3a6342cbd4894ed0b Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 24 Aug 2017 16:20:36 +0200
Subject: Add support for building a pgo beam_emu

---
 erts/emulator/Makefile.in           | 102 ++++++++++++++++++++++++++++--------
 erts/emulator/test/estone_SUITE.erl |  31 ++++++++---
 2 files changed, 104 insertions(+), 29 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index bc7eb72221..2b203c9ee6 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -63,6 +63,28 @@ ARFLAGS=rc
 OMIT_OMIT_FP=no
 TYPE_LIBS=
 
+PROFILE_COMPILER=@PROFILE_COMPILER@
+PROFILE_MARKER=
+ifeq ($(PROFILE),generate)
+PROFILE_MARKER=_pg
+else
+ifeq ($(PROFILE),use)
+PROFILE_MARKER=_pu
+endif
+endif
+
+ifeq ($(PROFILE_COMPILER), gcc)
+PROFILE_CORRECTION=@PROFILE_CORRECTION@
+PROFILE_GENERATE=-fprofile-generate
+PROFILE_USE=-fprofile-use $(PROFILE_CORRECTION)
+PROFILE_USE_DEPS=$(OBJDIR)/%_pu.gcda
+endif
+ifeq ($(PROFILE_COMPILER), clang)
+PROFILE_GENERATE=-fprofile-instr-generate
+PROFILE_USE=-fprofile-instr-use=$(OBJDIR)/default.profdata
+PROFILE_USE_DEPS=$(OBJDIR)/default.profdata
+endif
+
 DIRTY_SCHEDULER_SUPPORT=@DIRTY_SCHEDULER_SUPPORT@
 DIRTY_SCHEDULER_TEST=@DIRTY_SCHEDULER_TEST@
 
@@ -418,9 +440,20 @@ ifeq ($(TARGET), win32)
 EMULATOR_EXECUTABLE = beam$(TF_MARKER).dll
 else
 EMULATOR_EXECUTABLE = beam$(TF_MARKER)
+PROFILE_EXECUTABLE = beam.prof$(TF_MARKER)
 endif
 CS_EXECUTABLE = erl_child_setup$(TYPEMARKER)
 
+ifeq ($(PROFILE), generate)
+EMULATOR_EXECUTABLE = $(PROFILE_EXECUTABLE)
+ifeq ($(PROFILE_COMPILER), gcc)
+PROFILE_LDFLAGS = -fprofile-generate
+endif
+ifeq ($(PROFILE_COMPILER), clang)
+PROFILE_LDFLAGS = -fprofile-instr-generate
+endif
+endif
+
 # ----------------------------------------------------------------------
 
 ifeq ($(ERLANG_OSTYPE), unix)
@@ -687,16 +720,33 @@ $(OBJDIR)/beams.$(RES_EXT): $(TARGET)/beams.rc
 
 endif
 
-ifneq ($(filter tile-%,$(TARGET)),)
-$(OBJDIR)/beam_emu.o: beam/beam_emu.c
-	$(V_CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) \
-         $(INCLUDES) -c $< -o $@
-else
 # Usually the same as the default rule, but certain platforms (e.g. win32) mix
 # different compilers
 $(OBJDIR)/beam_emu.o: beam/beam_emu.c
 	$(V_EMU_CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
-endif
+
+$(OBJDIR)/%_pg.o: beam/%.c
+	$(V_CC) $(PROFILE_GENERATE) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
+$(OBJDIR)/%_pu.o: beam/%.c $(PROFILE_USE_DEPS)
+	$(V_CC) $(PROFILE_USE) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
+
+$(OBJDIR)/PROFILE: $(BINDIR)/$(PROFILE_EXECUTABLE)
+	$(V_at)echo " PROFILE ${PROFILE_EXECUTABLE}"
+	$(V_at)rm -f $(OBJDIR)/erl*.profraw
+	$(V_at)set -e; LLVM_PROFILE_FILE="$(OBJDIR)/erlc-%m.profraw" \
+          ERL_FLAGS="-emu_type prof${TYPEMARKER} +S 1" $(ERLC) -DPGO \
+	  -o $(OBJDIR) test/estone_SUITE.erl > $(OBJDIR)/PROFILE_LOG
+	$(V_at)set -e; LLVM_PROFILE_FILE="$(OBJDIR)/erl-%m.profraw" \
+	  ERL_FLAGS="-emu_type prof${TYPEMARKER} +S 1" $(ERL) -pa $(OBJDIR) \
+	  -noshell -s estone_SUITE pgo -s init stop >> $(OBJDIR)/PROFILE_LOG
+	$(V_at)touch $@
+
+$(OBJDIR)/%_pu.gcda: $(OBJDIR)/PROFILE
+	$(V_at)mv $(OBJDIR)/$*_pg.gcda $@
+	$(V_at)touch $@
+
+$(OBJDIR)/default.profdata: $(OBJDIR)/PROFILE
+	$(V_LLVM_PROFDATA) merge -output $@ $(OBJDIR)/*.profraw
 
 $(OBJDIR)/%.o: beam/%.c
 	$(V_CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
@@ -758,15 +808,23 @@ $(ERL_TOP)/lib/%.beam:
 
 INIT_OBJS = $(OBJDIR)/erl_main.o	$(PRELOAD_OBJ)
 
+# -fprofile-correction is needed in order to use PGO on erl_process
+# as multiple threads execute in that file.
+ifeq ($(PROFILE_CORRECTION),)
+PROFILE_OBJS = $(OBJDIR)/beam_emu.o
+RUN_OBJS = $(OBJDIR)/erl_process.o
+else
+PROFILE_OBJS = $(OBJDIR)/beam_emu.o	$(OBJDIR)/erl_process.o
+endif
+
 EMU_OBJS = \
-	$(OBJDIR)/beam_emu.o		$(OBJDIR)/beam_opcodes.o \
+	$(OBJDIR)/beam_opcodes.o \
 	$(OBJDIR)/beam_load.o		$(OBJDIR)/beam_bif_load.o \
 	$(OBJDIR)/beam_debug.o		$(OBJDIR)/beam_bp.o \
-	$(OBJDIR)/beam_catches.o \
-	$(OBJDIR)/code_ix.o \
+	$(OBJDIR)/beam_catches.o 	$(OBJDIR)/code_ix.o \
 	$(OBJDIR)/beam_ranges.o
 
-RUN_OBJS = \
+RUN_OBJS += \
 	$(OBJDIR)/erl_alloc.o		$(OBJDIR)/erl_mtrace.o \
 	$(OBJDIR)/erl_alloc_util.o	$(OBJDIR)/erl_goodfit_alloc.o \
 	$(OBJDIR)/erl_bestfit_alloc.o	$(OBJDIR)/erl_afit_alloc.o \
@@ -782,7 +840,7 @@ RUN_OBJS = \
 	$(OBJDIR)/utils.o		$(OBJDIR)/bif.o \
 	$(OBJDIR)/io.o 			$(OBJDIR)/erl_printf_term.o\
 	$(OBJDIR)/erl_debug.o		$(OBJDIR)/erl_md5.o \
-	$(OBJDIR)/erl_message.o		$(OBJDIR)/erl_process.o \
+	$(OBJDIR)/erl_message.o		 \
 	$(OBJDIR)/erl_process_dict.o	$(OBJDIR)/erl_process_lock.o \
 	$(OBJDIR)/erl_port_task.o	$(OBJDIR)/erl_arith.o \
 	$(OBJDIR)/time.o		$(OBJDIR)/erl_time_sup.o  \
@@ -923,21 +981,23 @@ ifdef HIPE_ENABLED
 EXTRA_BASE_OBJS += $(HIPE_OBJS)
 endif
 
-BASE_OBJS = $(EMU_OBJS) $(RUN_OBJS) $(OS_OBJS) $(EXTRA_BASE_OBJS) $(LTTNG_OBJS)
+BASE_OBJS = $(EMU_OBJS) $(RUN_OBJS) $(OS_OBJS) $(EXTRA_BASE_OBJS) \
+	$(LTTNG_OBJS) $(DRV_OBJS) $(NIF_OBJS)
 
-before_DTrace_OBJS =	$(BASE_OBJS) $(DRV_OBJS) $(NIF_OBJS)
+PROF_OBJS = $(patsubst %.o,%$(PROFILE_MARKER).o,$(PROFILE_OBJS)) $(BASE_OBJS)
+
+OBJS = $(PROF_OBJS)
 
-DTRACE_OBJS =
 ifdef DTRACE_ENABLED_2STEP
-DTRACE_OBJS = $(OBJDIR)/erlang_dtrace.o
-$(OBJDIR)/erlang_dtrace.o: $(before_DTrace_OBJS) $(TARGET)/erlang_dtrace.h
+# The $(PROFILE_MARKER) is placed in the object file name in order to
+# make sure we re-compile with the new object files for the profiled emulator
+OBJS += $(OBJDIR)/erlang$(PROFILE_MARKER)_dtrace.o
+$(OBJDIR)/erlang$(PROFILE_MARKER)_dtrace.o: $(PROF_OBJS) $(TARGET)/erlang_dtrace.h
 	dtrace -G -C -Ibeam \
 	  -s beam/erlang_dtrace.d \
-	  -o $@ $(before_DTrace_OBJS)
+	  -o $@ $(PROF_OBJS)
 endif
 
-OBJS = $(before_DTrace_OBJS) $(DTRACE_OBJS)
-
 $(INIT_OBJS): $(TTF_DIR)/GENERATED
 $(OBJS): $(TTF_DIR)/GENERATED
 
@@ -1029,8 +1089,8 @@ $(BINDIR)/$(EMULATOR_EXECUTABLE): $(INIT_OBJS) $(OBJS) $(DEPLIBS)
 
 else
 $(BINDIR)/$(EMULATOR_EXECUTABLE): $(INIT_OBJS) $(OBJS) $(DEPLIBS)
-	$(ld_verbose)$(PURIFY) $(LD) -o $(BINDIR)/$(EMULATOR_EXECUTABLE) \
-	$(HIPEBEAMLDFLAGS) $(LDFLAGS) $(DEXPORT) $(INIT_OBJS) $(OBJS) \
+	$(ld_verbose)$(PURIFY) $(LD) -o $@ \
+	$(HIPEBEAMLDFLAGS) $(PROFILE_LDFLAGS) $(LDFLAGS) $(DEXPORT) $(INIT_OBJS) $(OBJS) \
 	$(STATIC_NIF_LIBS) $(STATIC_DRIVER_LIBS) $(LIBS)
 
 endif
diff --git a/erts/emulator/test/estone_SUITE.erl b/erts/emulator/test/estone_SUITE.erl
index 8b336b366d..c4899967ca 100644
--- a/erts/emulator/test/estone_SUITE.erl
+++ b/erts/emulator/test/estone_SUITE.erl
@@ -20,7 +20,7 @@
 -module(estone_SUITE).
 %% Test functions
 -export([all/0, suite/0, groups/0,
-	 estone/1, estone_bench/1]).
+	 estone/1, estone_bench/1, pgo/0]).
 
 %% Internal exports for EStone tests
 -export([lists/1,
@@ -44,9 +44,9 @@
 	 links/1,lproc/1,
 	 run_micro/3,p1/1,ppp/3,macro/2,micros/0]).
 
-
--include_lib("common_test/include/ct.hrl").
+-ifndef(PGO).
 -include_lib("common_test/include/ct_event.hrl").
+-endif.
 
 %% EStone defines
 -define(TOTAL, (3000 * 1000 * 100)).   %% 300 secs
@@ -85,13 +85,28 @@ estone(Config) when is_list(Config) ->
 estone_bench(Config) ->
     DataDir = proplists:get_value(data_dir,Config),
     L = ?MODULE:macro(?MODULE:micros(),DataDir),
-    [ct_event:notify(
-       #event{name = benchmark_data, 
-	      data = [{name,proplists:get_value(title,Mark)},
-		      {value,proplists:get_value(estones,Mark)}]})
-     || Mark <- L],
+    {Total, Stones} = sum_micros(L, 0, 0),
+    notify([[{title,"ESTONES"}, {estones, Stones}] | L]),
     L.
 
+-ifndef(PGO).
+notify(Marks) ->
+    [ct_event:notify(
+       #event{name = benchmark_data,
+	      data = [{name,proplists:get_value(title, Mark)},
+		      {value,proplists:get_value(estones, Mark)}]})
+     || Mark <- Marks].
+-else.
+notify(_) ->
+    ok.
+-endif.
+
+%% The benchmarks to run in order to guide PGO (profile guided optimisation)
+pgo() ->
+    %% We run all benchmarks except the port_io as we don't want to
+    %% have to build a custom port.
+    Micros = ?MODULE:micros() -- [micro(port_io)],
+    ?MODULE:macro(Micros,[]).
 
 %%
 %% Calculate CPU speed
-- 
cgit v1.2.3


From 3a7bddaae1b1d3c499f0c03f65a6986ad3bdeefe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sat, 9 Sep 2017 05:35:26 +0200
Subject: Check the right side of a transformation better

The right side of a transformation must be either a single call
to a transformation function OR a list of instructions. Mixing
them like this is not supported:

  some_instruction A B C => gen_something(A) | other B C

Unfortunately, beam_makeops would silenty ignore anything after the
function call, basically handling it in the same way as:

  some_instruction A B C => gen_something(A)

Add a sanity check to reject such mixed right-hand sides.
---
 erts/emulator/utils/beam_makeops | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 38754e67a1..e55d3eadb5 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -1720,8 +1720,11 @@ sub parse_transformation {
     #
 
     my @to;
-    if ($to =~ /^(\w+)\((.*?)\)/) {
-	my($name, $arglist) = ($1, $2);
+    if ($to =~ /^(\w+)\((.*?)\)(.*)/) {
+	my($name, $arglist, $garbage) = ($1, $2, $3);
+        if ($garbage =~ /\S/) {
+            error("garbage after call to '$name()'");
+        }
 	@to = (compile_transform_function($name, split(/\s*,\s*/, $arglist)));
     } else {
 	@to = split(/\s*\|\s*/, $to);
-- 
cgit v1.2.3


From 87b57377864d3161a79c65e32844d7539d1a9264 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 11 Sep 2017 06:47:22 +0200
Subject: beam_debug: Add support for printing 'S' operands

The 'S' operand type was added in 5bf73db9fd77.
---
 erts/emulator/beam/beam_debug.c | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 49414ae8fc..21e295c63a 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -489,6 +489,14 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	case 'n':		/* Nil */
 	    erts_print(to, to_arg, "[]");
 	    break;
+        case 'S':               /* Register */
+            {
+                Uint reg_type = (*ap & 1) ? 'y' : 'x';
+                Uint n = ap[0] / sizeof(Eterm);
+                erts_print(to, to_arg, "%c(%d)", reg_type, n);
+		ap++;
+                break;
+            }
 	case 's':		/* Any source (tagged constant or register) */
 	    tag = loader_tag(*ap);
 	    if (tag == LOADER_X_REG) {
-- 
cgit v1.2.3


From 769ff22c750d939fdc9cb45fae1e44817ec04307 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 8 Sep 2017 10:08:45 +0200
Subject: erts: Remove possibility to disable dirty schedulers

---
 erts/emulator/Makefile.in            |  12 +-
 erts/emulator/beam/beam_bif_load.c   |  10 --
 erts/emulator/beam/beam_bp.c         |  12 --
 erts/emulator/beam/beam_bp.h         |   2 -
 erts/emulator/beam/beam_debug.c      |  12 --
 erts/emulator/beam/beam_emu.c        |   2 -
 erts/emulator/beam/beam_load.h       |   4 -
 erts/emulator/beam/bif.c             |  13 ---
 erts/emulator/beam/bif.h             |   2 -
 erts/emulator/beam/erl_alloc.c       |   2 -
 erts/emulator/beam/erl_bif_info.c    |  21 ----
 erts/emulator/beam/erl_bif_trace.c   |   4 -
 erts/emulator/beam/erl_bif_unique.c  |   4 -
 erts/emulator/beam/erl_gc.c          |  26 -----
 erts/emulator/beam/erl_init.c        |  30 -----
 erts/emulator/beam/erl_lock_check.c  |   2 -
 erts/emulator/beam/erl_nfunc_sched.h |   4 -
 erts/emulator/beam/erl_nif.c         |  21 ----
 erts/emulator/beam/erl_process.c     | 215 -----------------------------------
 erts/emulator/beam/erl_process.h     |  44 -------
 erts/emulator/beam/global.h          |   4 -
 erts/emulator/beam/io.c              |   4 -
 erts/emulator/utils/make_tables      |  33 +++---
 23 files changed, 14 insertions(+), 469 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 1b29065486..02a9a9a93b 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -85,7 +85,6 @@ PROFILE_USE=-fprofile-instr-use=$(OBJDIR)/default.profdata
 PROFILE_USE_DEPS=$(OBJDIR)/default.profdata
 endif
 
-DIRTY_SCHEDULER_SUPPORT=@DIRTY_SCHEDULER_SUPPORT@
 DIRTY_SCHEDULER_TEST=@DIRTY_SCHEDULER_TEST@
 
 ifeq ($(TYPE),debug)
@@ -216,10 +215,6 @@ override FLAVOR=smp
 
 FLAVOR_MARKER=.smp
 ENABLE_ALLOC_TYPE_VARS += nofrag
-ifeq ($(DIRTY_SCHEDULER_SUPPORT),yes)
-THR_DEFS += -DERTS_DIRTY_SCHEDULERS
-DS_SUPPORT=yes
-
 ifeq ($(DIRTY_SCHEDULER_TEST),yes)
 DS_TEST=yes
 THR_DEFS += -DERTS_DIRTY_SCHEDULERS_TEST
@@ -227,11 +222,6 @@ else # DIRTY_SCHEDULER_TEST
 DS_TEST=no
 endif # DIRTY_SCHEDULER_TEST
 
-else # DIRTY_SCHEDULER_SUPPORT
-DS_SUPPORT=no
-DS_TEST=no
-endif # DIRTY_SCHEDULER_SUPPORT
-
 TF_MARKER=$(TYPEMARKER)$(FLAVOR_MARKER)
 
 ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no)
@@ -627,7 +617,7 @@ $(HIPE_NBIF_FILES) \
 	: $(TTF_DIR)/TABLES-GENERATED
 $(TTF_DIR)/TABLES-GENERATED: $(ATOMS) $(DIRTY_BIFS) $(BIFS) utils/make_tables
 	$(gen_verbose)LANG=C $(PERL) utils/make_tables -src $(TTF_DIR) -include $(TTF_DIR)\
-		-ds $(DS_SUPPORT) -dst $(DS_TEST) -hipe $(HIPE) $(ATOMS) $(DIRTY_BIFS) $(BIFS) && echo $? >$(TTF_DIR)/TABLES-GENERATED
+		-dst $(DS_TEST) -hipe $(HIPE) $(ATOMS) $(DIRTY_BIFS) $(BIFS) && echo $? >$(TTF_DIR)/TABLES-GENERATED
 GENERATE += $(TTF_DIR)/TABLES-GENERATED
 
 $(TTF_DIR)/erl_alloc_types.h: beam/erl_alloc.types utils/make_alloc_types
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index b78f617560..7f8dc42aca 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -65,9 +65,7 @@ static struct {
 
 Process *erts_code_purger = NULL;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 Process *erts_dirty_process_code_checker;
-#endif
 erts_atomic_t erts_copy_literal_area__;
 #define ERTS_SET_COPY_LITERAL_AREA(LA)			\
     erts_atomic_set_nob(&erts_copy_literal_area__,	\
@@ -604,9 +602,6 @@ badarg:
 
 BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2)
 {
-#if !defined(ERTS_DIRTY_SCHEDULERS)
-    BIF_ERROR(BIF_P, EXC_NOTSUP);
-#else
     Process *rp;
     int reds = 0;
     Eterm res;
@@ -636,7 +631,6 @@ BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2)
     ASSERT(is_value(res));
 
     BIF_RET2(res, reds);
-#endif
 }
 
 BIF_RETTYPE delete_module_1(BIF_ALIST_1)
@@ -1053,10 +1047,8 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed
 
 return_ok:
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p)))
 	c_p->flags &= ~F_DIRTY_CLA;
-#endif
 
     return am_ok;
 
@@ -1071,10 +1063,8 @@ literal_gc:
     *redsp += erts_garbage_collect_literals(c_p, (Eterm *) literals, lit_bsize,
 					    oh, fcalls);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (c_p->flags & F_DIRTY_CLA)
 	return THE_NON_VALUE;
-#endif
 
     return am_ok;
 }
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 79a75f6698..49ec59c989 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -75,9 +75,7 @@ extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */
 
 erts_atomic32_t erts_active_bp_index;
 erts_atomic32_t erts_staging_bp_index;
-#ifdef ERTS_DIRTY_SCHEDULERS
 erts_mtx_t erts_dirty_bp_ix_mtx;
-#endif
 
 /*
  * Inlined helpers
@@ -94,22 +92,18 @@ acquire_bp_sched_ix(Process *c_p)
 {
     ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
     ASSERT(esdp);
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
 	erts_mtx_lock(&erts_dirty_bp_ix_mtx);
         return (Uint32) erts_no_schedulers;
     }
-#endif
     return (Uint32) esdp->no - 1;
 }
 
 static ERTS_INLINE void
 release_bp_sched_ix(Uint32 ix)
 {
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (ix == (Uint32) erts_no_schedulers)
         erts_mtx_unlock(&erts_dirty_bp_ix_mtx);
-#endif
 }
 
 
@@ -164,10 +158,8 @@ void
 erts_bp_init(void) {
     erts_atomic32_init_nob(&erts_active_bp_index, 0);
     erts_atomic32_init_nob(&erts_staging_bp_index, 1);
-#ifdef ERTS_DIRTY_SCHEDULERS
     erts_mtx_init(&erts_dirty_bp_ix_mtx, "dirty_break_point_index", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
-#endif
 }
 
 
@@ -1585,11 +1577,7 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags,
 	ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0);
 	bdt = Alloc(sizeof(BpDataTime));
 	erts_refc_init(&bdt->refc, 1);
-#ifdef ERTS_DIRTY_SCHEDULERS
 	bdt->n = erts_no_schedulers + 1;
-#else
-	bdt->n = erts_no_schedulers;
-#endif
 	bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n));
 	for (i = 0; i < bdt->n; i++) {
 	    bp_hash_init(&(bdt->hash[i]), 32);
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index 1e1f6a7534..a64765822b 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -79,9 +79,7 @@ typedef struct generic_bp {
 #define ERTS_BP_CALL_TIME_SCHEDULE_OUT     (1)
 #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2)
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 extern erts_mtx_t erts_dirty_bp_ix_mtx;
-#endif
 
 enum erts_break_op{
     ERTS_BREAK_NOP   =  0, /* Must be false */
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 21e295c63a..f8c7f9a0fe 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -804,10 +804,8 @@ static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif)
  * test suite.
  */
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 static int ms_wait(Process *c_p, Eterm etimeout, int busy);
 static int dirty_send_message(Process *c_p, Eterm to, Eterm tag);
-#endif
 static BIF_RETTYPE dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I);
 
 /*
@@ -836,7 +834,6 @@ erts_debug_dirty_io_2(BIF_ALIST_2)
 BIF_RETTYPE
 erts_debug_dirty_3(BIF_ALIST_3)
 {
-#ifdef ERTS_DIRTY_SCHEDULERS
     Eterm argv[2];
     switch (BIF_ARG_1) {
     case am_normal:
@@ -866,9 +863,6 @@ erts_debug_dirty_3(BIF_ALIST_3)
     default:
 	BIF_ERROR(BIF_P, EXC_BADARG);
     }
-#else
-	BIF_ERROR(BIF_P, EXC_UNDEF);
-#endif
 }
 
 
@@ -876,7 +870,6 @@ static BIF_RETTYPE
 dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I)
 {
     BIF_RETTYPE ret;
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (am_scheduler == arg1) {
 	ErtsSchedulerData *esdp;
 	if (arg2 != am_type) 
@@ -1062,13 +1055,9 @@ dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I)
     badarg:
 	ERTS_BIF_PREP_ERROR(ret, c_p, BADARG);
     }
-#else
-    ERTS_BIF_PREP_ERROR(ret, c_p, EXC_UNDEF);
-#endif
     return ret;
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 static int
 dirty_send_message(Process *c_p, Eterm to, Eterm tag)
@@ -1155,7 +1144,6 @@ ms_wait(Process *c_p, Eterm etimeout, int busy)
     return 1;
 }
 
-#endif /* ERTS_DIRTY_SCHEDULERS */
 
 #  define ERTS_STACK_LIMIT ((char *) ethr_get_stacklimit())
 
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 55990362ff..e086b3cf7b 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1034,7 +1034,6 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
  */
 void erts_dirty_process_main(ErtsSchedulerData *esdp)
 {
-#ifdef ERTS_DIRTY_SCHEDULERS
     Process* c_p = NULL;
     ErtsMonotonicTime start_time;
 #ifdef DEBUG
@@ -1273,7 +1272,6 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 	I = c_p->i;
 	goto context_switch;
     }
-#endif /* ERTS_DIRTY_SCHEDULERS */
 }
 
 static ErtsCodeMFA *
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index c088bdb751..c4a90d3f3a 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -111,11 +111,7 @@ typedef struct beam_code_header {
 
 }BeamCodeHeader;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 #  define BEAM_NIF_MIN_FUNC_SZ 4
-#else
-#  define BEAM_NIF_MIN_FUNC_SZ 3
-#endif
 
 void erts_release_literal_area(struct ErtsLiteralArea_* literal_area);
 int erts_is_module_native(BeamCodeHeader* code);
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index ded6c6f1a4..80f391e91e 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4701,7 +4701,6 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 		      ref,
 		      old ? am_true : am_false);
 	}
-#if defined(ERTS_DIRTY_SCHEDULERS)
     } else if (BIF_ARG_1 == am_dirty_cpu_schedulers_online) {
 	Sint old_no;
 	if (!is_small(BIF_ARG_2))
@@ -4727,7 +4726,6 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	    BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
 	    break;
 	}
-#endif
     } else if (BIF_ARG_1 == am_time_offset
 	       && ERTS_IS_ATOM_STR("finalize", BIF_ARG_2)) {
 	ErtsTimeOffsetState res;
@@ -5104,7 +5102,6 @@ schedule(Process *c_p, Process *dirty_shadow_proc,
 				    argc, argv);
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 static BIF_RETTYPE dirty_bif_result(BIF_ALIST_1)
 {
@@ -5147,7 +5144,6 @@ static BIF_RETTYPE dirty_bif_exception(BIF_ALIST_2)
     BIF_ERROR(BIF_P, freason);
 }
 
-#endif /* ERTS_DIRTY_SCHEDULERS */
 
 extern BeamInstr* em_call_bif_e;
 static BIF_RETTYPE call_bif(Process *c_p, Eterm *reg, BeamInstr *I);
@@ -5165,7 +5161,6 @@ erts_schedule_bif(Process *proc,
     Process *c_p, *dirty_shadow_proc;
     ErtsCodeMFA *mfa;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
 	dirty_shadow_proc = proc;
 	c_p = proc->next;
@@ -5173,7 +5168,6 @@ erts_schedule_bif(Process *proc,
 	erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
     }
     else
-#endif
     {
 	dirty_shadow_proc = NULL;
 	c_p = proc;
@@ -5189,7 +5183,6 @@ erts_schedule_bif(Process *proc,
 	 * ibif - indirect bif
 	 */
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 	erts_aint32_t set, mask;
 	mask = (ERTS_PSFLG_DIRTY_CPU_PROC
 		| ERTS_PSFLG_DIRTY_IO_PROC);
@@ -5213,10 +5206,6 @@ erts_schedule_bif(Process *proc,
 	}
 
 	(void) erts_atomic32_read_bset_nob(&c_p->state, mask, set);
-#else
-	dbif = call_bif;
-	ibif = bif;
-#endif
 
 	if (i == NULL) {
 	    ERTS_INTERNAL_ERROR("Missing instruction pointer");
@@ -5292,7 +5281,6 @@ call_bif(Process *c_p, Eterm *reg, BeamInstr *I)
     return ret;
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 int
 erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg)
@@ -5386,7 +5374,6 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *
     return exiting;
 }
 
-#endif /* ERTS_DIRTY_SCHEDULERS */
 
 
 #ifdef HARDDEBUG
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 9b0870dee2..a2bc883dbe 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -498,10 +498,8 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
 					 Eterm args[],
 					 int nargs);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 int erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p,
 			BeamInstr *I, Eterm *reg);
-#endif
 
 BIF_RETTYPE
 erts_schedule_bif(Process *proc,
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index aee54ad0a8..845cef24c7 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1841,9 +1841,7 @@ erts_alloc_register_scheduler(void *vesdp)
     int ix = (int) esdp->no;
     int aix;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
-#endif
     for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
 	ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
 	esdp->alloc_data.deallctr[aix] = NULL;
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 93f5f9ec11..36939d6acc 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -89,9 +89,7 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
 				     " [64-bit]"
 #endif
 				     " [smp:%beu:%beu]"
-#if defined(ERTS_DIRTY_SCHEDULERS)
 				     " [ds:%beu:%beu:%beu]"
-#endif
 #if defined(ERTS_DIRTY_SCHEDULERS_TEST)
 				     " [dirty-schedulers-TEST]"
 #endif
@@ -371,9 +369,7 @@ erts_print_system_version(fmtfn_t to, void *arg, Process *c_p)
     return erts_print(to, arg, erts_system_version,
 		      rc_str
 		      , total, online
-#ifdef ERTS_DIRTY_SCHEDULERS
 		      , dirty_cpu, dirty_cpu_onln, dirty_io
-#endif
 		      , erts_async_max_threads
 #ifdef ERTS_ENABLE_KERNEL_POLL
 		      , erts_use_kernel_poll ? "true" : "false"
@@ -2131,11 +2127,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	ASSERT(erts_compat_rel > 0);
 	BIF_RET(make_small(erts_compat_rel));
     } else if (BIF_ARG_1 == am_multi_scheduling) {
-#ifndef ERTS_DIRTY_SCHEDULERS
-	if (erts_no_schedulers == 1)
-	    BIF_RET(am_disabled);
-	else
-#endif
 	{
 	    int msb = erts_is_multi_scheduling_blocked();
 	    BIF_RET(!msb
@@ -2674,27 +2665,15 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(make_small(active));
     } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers", BIF_ARG_1)) {
 	Uint dirty_cpu;
-#ifdef ERTS_DIRTY_SCHEDULERS
 	erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, NULL, NULL);
-#else
-        dirty_cpu = 0;
-#endif
 	BIF_RET(make_small(dirty_cpu));
     } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers_online", BIF_ARG_1)) {
 	Uint dirty_cpu_onln;
-#ifdef ERTS_DIRTY_SCHEDULERS
 	erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, NULL, NULL);
-#else
-        dirty_cpu_onln = 0;
-#endif
 	BIF_RET(make_small(dirty_cpu_onln));
     } else if (ERTS_IS_ATOM_STR("dirty_io_schedulers", BIF_ARG_1)) {
 	Uint dirty_io;
-#ifdef ERTS_DIRTY_SCHEDULERS
 	erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, NULL, &dirty_io, NULL);
-#else
-        dirty_io = 0;
-#endif
 	BIF_RET(make_small(dirty_io));
     } else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) {
 	res = make_small(erts_no_run_queues);
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index b02f966558..ffc1c3261f 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1043,16 +1043,12 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 	erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
 	erts_thr_progress_block();
     }
-#ifdef ERTS_DIRTY_SCHEDULERS
     erts_mtx_lock(&erts_dirty_bp_ix_mtx);
-#endif
 
 
     r = function_is_traced(p, mfa, &ms, &ms_meta, &meta, &count, &call_time);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     erts_mtx_unlock(&erts_dirty_bp_ix_mtx);
-#endif
     if ( (key == am_call_time) || (key == am_all)) {
 	erts_thr_progress_unblock();
 	erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c
index aa79503819..19d46537f9 100644
--- a/erts/emulator/beam/erl_bif_unique.c
+++ b/erts/emulator/beam/erl_bif_unique.c
@@ -77,10 +77,8 @@ init_reference(void)
     ref_init_value += (Uint64) tv.tv_usec;
 #ifdef DEBUG
     max_thr_id = (Uint32) erts_no_schedulers;
-#ifdef ERTS_DIRTY_SCHEDULERS
     max_thr_id += (Uint32) erts_no_dirty_cpu_schedulers;
     max_thr_id += (Uint32) erts_no_dirty_io_schedulers;
-#endif
 #endif
     erts_atomic64_init_nob(&global_reference.count,
 			   (erts_aint64_t) ref_init_value);
@@ -439,10 +437,8 @@ init_unique_integer(void)
 {
     int bits;
     unique_data.r.o.val0_max = (Uint64) erts_no_schedulers;
-#ifdef ERTS_DIRTY_SCHEDULERS
     unique_data.r.o.val0_max += (Uint64) erts_no_dirty_cpu_schedulers;
     unique_data.r.o.val0_max += (Uint64) erts_no_dirty_io_schedulers;
-#endif
     bits = erts_fit_in_bits_int64(unique_data.r.o.val0_max);
     unique_data.r.o.left_shift = bits;
     unique_data.r.o.right_shift = 64 - bits;
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index bc3bcdc9ad..8344c164fa 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -183,12 +183,10 @@ typedef struct {
     erts_atomic32_t refc;
 } ErtsGCInfoReq;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 static struct {
     erts_mtx_t mtx;
     ErtsGCInfo info;
 } dirty_gc;
-#endif
 
 static ERTS_INLINE int
 gc_cost(Uint gc_moved_live_words, Uint resize_moved_words)
@@ -273,11 +271,9 @@ erts_init_gc(void)
       init_gc_info(&esdp->gc_info);
     }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     erts_mtx_init(&dirty_gc.mtx, "dirty_gc_info", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
     init_gc_info(&dirty_gc.info);
-#endif
 
     init_gcireq_alloc();
 }
@@ -481,12 +477,10 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int
     }
 
     if (need == 0) {
-#ifdef ERTS_DIRTY_SCHEDULERS
         if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) {
             ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p)));
             goto force_reschedule;
         }
-#endif
 	return 1;
     }
     /*
@@ -541,9 +535,7 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int
     p->heap_hfrag = hfrag;
 #endif
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 force_reschedule:
-#endif
 
     /* Make sure that we do a proper GC as soon as possible... */
     p->flags |= F_FORCE_GC;
@@ -616,7 +608,6 @@ young_gen_usage(Process *p)
 	}							\
     } while (0)
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 static ERTS_INLINE void
 check_for_possibly_long_gc(Process *p, Uint ygen_usage)
@@ -640,7 +631,6 @@ check_for_possibly_long_gc(Process *p, Uint ygen_usage)
     }
 }
 
-#endif
 
 /*
  * Garbage collect a process.
@@ -675,21 +665,17 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     state = erts_atomic32_read_nob(&p->state);
 
     if ((p->flags & (F_DISABLE_GC|F_DELAY_GC)) || state & ERTS_PSFLG_EXITING) {
-#ifdef ERTS_DIRTY_SCHEDULERS
     delay_gc_before_start:
-#endif
 	return delay_garbage_collection(p, live_hf_end, need, fcalls);
     }
 
     ygen_usage = max_young_gen_usage ? max_young_gen_usage : young_gen_usage(p);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
 	check_for_possibly_long_gc(p, ygen_usage);
 	if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC))
 	    goto delay_gc_before_start;
     }
-#endif
 
     if (p->abandoned_heap)
 	live_hf_end = ERTS_INVALID_HFRAG_PTR;
@@ -731,14 +717,12 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
             if (IS_TRACED_FL(p, F_TRACE_GC)) {
                 trace_gc(p, am_gc_minor_end, reclaimed_now, THE_NON_VALUE);
             }
-#ifdef ERTS_DIRTY_SCHEDULERS
 	    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
 		p->flags |= F_NEED_FULLSWEEP;
 		check_for_possibly_long_gc(p, ygen_usage);
 		if (p->flags & F_DIRTY_MAJOR_GC)
 		    goto delay_gc_after_start;
 	    }
-#endif
             goto do_major_collection;
         }
         if (ERTS_SCHEDULER_IS_DIRTY(esdp))
@@ -784,9 +768,7 @@ do_major_collection:
                               am_kill, NIL, NULL, 0);
         erts_proc_unlock(p, locks & ERTS_PROC_LOCKS_ALL_MINOR);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     delay_gc_after_start:
-#endif
         /* erts_send_exit_signal looks for ERTS_PSFLG_GC, so
            we have to remove it after the signal is sent */
         erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
@@ -821,7 +803,6 @@ do_major_collection:
 	    monitor_large_heap(p);
     }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
 	erts_mtx_lock(&dirty_gc.mtx);
 	dirty_gc.info.garbage_cols++;
@@ -829,7 +810,6 @@ do_major_collection:
 	erts_mtx_unlock(&dirty_gc.mtx);
     }
     else
-#endif
     {
 	esdp->gc_info.garbage_cols++;
 	esdp->gc_info.reclaimed += reclaimed_now;
@@ -907,7 +887,6 @@ garbage_collect_hibernate(Process* p, int check_long_gc)
     if (p->flags & F_DISABLE_GC)
 	ERTS_INTERNAL_ERROR("GC disabled");
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p)))
         p->flags &= ~(F_DIRTY_GC_HIBERNATE|F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC);
     else if (check_long_gc) {
@@ -920,7 +899,6 @@ garbage_collect_hibernate(Process* p, int check_long_gc)
 	}
 	p->flags = flags;
     }
-#endif
     /*
      * Preliminaries.
      */
@@ -1110,7 +1088,6 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
 
     p->flags |= F_NEED_FULLSWEEP;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p)))
 	p->flags &= ~F_DIRTY_CLA;
     else {
@@ -1126,7 +1103,6 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
 	    return 10;
 	}
     }
-#endif
 
     reds = (Sint64) garbage_collect(p, ERTS_INVALID_HFRAG_PTR, 0,
 				    p->arg_reg, p->arity, fcalls,
@@ -3230,7 +3206,6 @@ reply_gc_info(void *vgcirp)
 
     reclaimed = esdp->gc_info.reclaimed;
     garbage_cols = esdp->gc_info.garbage_cols;
-#ifdef ERTS_DIRTY_SCHEDULERS
     /*
      * Add dirty schedulers info on requesting
      * schedulers info
@@ -3241,7 +3216,6 @@ reply_gc_info(void *vgcirp)
 	garbage_cols += dirty_gc.info.garbage_cols;
 	erts_mtx_unlock(&dirty_gc.mtx);
     }
-#endif
 
     sz = 0;
     hpp = NULL;
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 34affaa015..8c14c86219 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -180,11 +180,9 @@ int erts_compat_rel;
 
 static int no_schedulers;
 static int no_schedulers_online;
-#ifdef ERTS_DIRTY_SCHEDULERS
 static int no_dirty_cpu_schedulers;
 static int no_dirty_cpu_schedulers_online;
 static int no_dirty_io_schedulers;
-#endif
 
 #ifdef DEBUG
 Uint32 verbose;             /* See erl_debug.h for information about verbose */
@@ -313,11 +311,9 @@ erl_init(int ncpu,
     erts_init_process(ncpu, proc_tab_sz, legacy_proc_tab);
     erts_init_scheduling(no_schedulers,
 			 no_schedulers_online
-#ifdef ERTS_DIRTY_SCHEDULERS
 			 , no_dirty_cpu_schedulers,
 			 no_dirty_cpu_schedulers_online,
 			 no_dirty_io_schedulers
-#endif
 			 );
     erts_late_init_time_sup();
     erts_init_cpu_topology(); /* Must be after init_scheduling */
@@ -609,7 +605,6 @@ void erts_usage(void)
 		 ERTS_SCHED_THREAD_MIN_STACK_SIZE,
 		 ERTS_SCHED_THREAD_MAX_STACK_SIZE,
                  ERTS_DEFAULT_SCHED_STACK_SIZE);
-#ifdef ERTS_DIRTY_SCHEDULERS
     erts_fprintf(stderr, "-sssdcpu size  suggested stack size in kilo words for dirty CPU scheduler\n");
     erts_fprintf(stderr, "               threads, valid range is [%d-%d] (default %d)\n",
 		 ERTS_SCHED_THREAD_MIN_STACK_SIZE,
@@ -620,7 +615,6 @@ void erts_usage(void)
 		 ERTS_SCHED_THREAD_MIN_STACK_SIZE,
 		 ERTS_SCHED_THREAD_MAX_STACK_SIZE,
                  ERTS_DEFAULT_DIO_SCHED_STACK_SIZE);
-#endif
     erts_fprintf(stderr, "-spp Bool      set port parallelism scheduling hint\n");
     erts_fprintf(stderr, "-S n1:n2       set number of schedulers (n1), and number of\n");
     erts_fprintf(stderr, "               schedulers online (n2), maximum for both\n");
@@ -629,7 +623,6 @@ void erts_usage(void)
     erts_fprintf(stderr, "-SP p1:p2      specify schedulers (p1) and schedulers online (p2)\n");
     erts_fprintf(stderr, "               as percentages of logical processors configured and logical\n");
     erts_fprintf(stderr, "               processors available, respectively\n");
-#ifdef ERTS_DIRTY_SCHEDULERS
     erts_fprintf(stderr, "-SDcpu n1:n2   set number of dirty CPU schedulers (n1), and number of\n");
     erts_fprintf(stderr, "               dirty CPU schedulers online (n2), valid range for both\n");
     erts_fprintf(stderr, "               numbers is [1-%d], and n2 must be less than or equal to n1\n",
@@ -639,7 +632,6 @@ void erts_usage(void)
     erts_fprintf(stderr, "               and logical processors available, respectively\n");
     erts_fprintf(stderr, "-SDio n        set number of dirty I/O schedulers, valid range is [0-%d]\n",
 		 ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS);
-#endif
     erts_fprintf(stderr, "-t size        set the maximum number of atoms the emulator can handle\n");
     erts_fprintf(stderr, "               valid range is [%d-%d]\n",
 		 MIN_ATOM_TABLE_SIZE, MAX_ATOM_TABLE_SIZE);
@@ -725,13 +717,11 @@ early_init(int *argc, char **argv) /*
     int schdlrs_percentage = 100;
     int schdlrs_onln_percentage = 100;
     int max_main_threads;
-#ifdef ERTS_DIRTY_SCHEDULERS
     int dirty_cpu_scheds;
     int dirty_cpu_scheds_online;
     int dirty_cpu_scheds_pctg = 100;
     int dirty_cpu_scheds_onln_pctg = 100;
     int dirty_io_scheds;
-#endif
     int max_reader_groups;
     int reader_groups;
     char envbuf[21]; /* enough for any 64-bit integer */
@@ -794,11 +784,9 @@ early_init(int *argc, char **argv) /*
     schdlrs = no_schedulers;
     schdlrs_onln = no_schedulers_online;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     dirty_cpu_scheds = no_schedulers;
     dirty_cpu_scheds_online = no_schedulers_online;
     dirty_io_scheds = 10;
-#endif
 
     envbufsz = sizeof(envbuf);
 
@@ -891,7 +879,6 @@ early_init(int *argc, char **argv) /*
                                 ("using %d:%d scheduler percentages\n",
                                  schdlrs_percentage, schdlrs_onln_percentage));
                     }
-#ifdef ERTS_DIRTY_SCHEDULERS
 		    else if (argv[i][2] == 'D') {
 			char *arg;
 			char *type = argv[i]+3;
@@ -1003,7 +990,6 @@ early_init(int *argc, char **argv) /*
 			    break;
 			}
 		    }
-#endif
 		    else {
 			int tot, onln;
 			char *arg = get_arg(argv[i]+2, argv[i+1], &i);
@@ -1085,7 +1071,6 @@ early_init(int *argc, char **argv) /*
 		erts_usage();
 	    }
 	}
-#ifdef ERTS_DIRTY_SCHEDULERS
 	/* apply any dirty scheduler precentages */
 	if (dirty_cpu_scheds_pctg != 100 || dirty_cpu_scheds_onln_pctg != 100) {
 	    dirty_cpu_scheds = dirty_cpu_scheds * dirty_cpu_scheds_pctg / 100;
@@ -1099,7 +1084,6 @@ early_init(int *argc, char **argv) /*
 	    dirty_cpu_scheds_online = schdlrs_onln;
 	if (dirty_cpu_scheds_online < 1)
 	    dirty_cpu_scheds_online = 1;
-#endif
     }
 
 
@@ -1107,11 +1091,9 @@ early_init(int *argc, char **argv) /*
     no_schedulers_online = schdlrs_onln;
 
     erts_no_schedulers = (Uint) no_schedulers;
-#ifdef ERTS_DIRTY_SCHEDULERS
     erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers = dirty_cpu_scheds;
     no_dirty_cpu_schedulers_online = dirty_cpu_scheds_online;
     erts_no_dirty_io_schedulers = no_dirty_io_schedulers = dirty_io_scheds;
-#endif
     erts_early_init_scheduling(no_schedulers);
 
     alloc_opts.ncpu = ncpu;
@@ -1132,13 +1114,9 @@ early_init(int *argc, char **argv) /*
      */
     erts_thr_progress_init(no_schedulers,
 			   no_schedulers+2,
-#ifndef ERTS_DIRTY_SCHEDULERS
-			   erts_async_max_threads
-#else
 			   erts_async_max_threads +
 			   erts_no_dirty_cpu_schedulers +
 			   erts_no_dirty_io_schedulers
-#endif
 			   );
     erts_thr_q_init();
     erts_init_utils();
@@ -1237,10 +1215,8 @@ erl_start(int argc, char **argv)
      * a lot of stack.
      */
     erts_sched_thread_suggested_stack_size = ERTS_DEFAULT_SCHED_STACK_SIZE;
-#ifdef ERTS_DIRTY_SCHEDULERS
     erts_dcpu_sched_thread_suggested_stack_size = ERTS_DEFAULT_DCPU_SCHED_STACK_SIZE;
     erts_dio_sched_thread_suggested_stack_size = ERTS_DEFAULT_DIO_SCHED_STACK_SIZE;
-#endif
 
 #ifdef DEBUG
     verbose = DEBUG_DEFAULT;
@@ -1855,7 +1831,6 @@ erl_start(int argc, char **argv)
 		VERBOSE(DEBUG_SYSTEM,
 			("scheduler wakeup threshold: %s\n", arg));
 	    }
-#ifdef ERTS_DIRTY_SCHEDULERS
 	    else if (has_prefix("ssdcpu", sub_param)) {
 		/* suggested stack size (Kilo Words) for dirty CPU scheduler threads */
 		arg = get_arg(sub_param+6, argv[i+1], &i);
@@ -1890,7 +1865,6 @@ erl_start(int argc, char **argv)
 			("suggested dirty IO scheduler thread stack size %d kilo words\n",
 			 erts_dio_sched_thread_suggested_stack_size));
 	    }
-#endif
 	    else if (has_prefix("ss", sub_param)) {
 		/* suggested stack size (Kilo Words) for scheduler threads */
 		arg = get_arg(sub_param+2, argv[i+1], &i);
@@ -2205,12 +2179,10 @@ erl_start(int argc, char **argv)
 
     if (erts_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE)
         erts_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE;
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (erts_dcpu_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE)
         erts_dcpu_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE;
     if (erts_dio_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE)
         erts_dio_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE;
-#endif
 
     erl_init(ncpu,
 	     proc_tab_sz,
@@ -2255,7 +2227,6 @@ erl_start(int argc, char **argv)
 	       && erts_literal_area_collector->common.id == pid);
 	erts_proc_inc_refc(erts_literal_area_collector);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 	pid = erl_system_process_otp(otp_ring0_pid, "erts_dirty_process_code_checker", !0);
 	erts_dirty_process_code_checker
 	    = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc,
@@ -2263,7 +2234,6 @@ erl_start(int argc, char **argv)
 	ASSERT(erts_dirty_process_code_checker
 	       && erts_dirty_process_code_checker->common.id == pid);
 	erts_proc_inc_refc(erts_dirty_process_code_checker);
-#endif
 
     }
 
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 47c0fd2869..f81c90818f 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -119,11 +119,9 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"schdlr_sspnd",				NULL			},
     {	"migration_info_update",		NULL			},
     {	"run_queue",				"address"		},
-#ifdef ERTS_DIRTY_SCHEDULERS
     {   "dirty_run_queue_sleep_list",		"address"		},
     {	"dirty_gc_info",			NULL			},
     {	"dirty_break_point_index",		NULL			},
-#endif
     {	"process_table",			NULL			},
     {	"cpu_info",				NULL			},
     {	"pollset",				"address"		},
diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h
index 69008084df..9be0e6f7c7 100644
--- a/erts/emulator/beam/erl_nfunc_sched.h
+++ b/erts/emulator/beam/erl_nfunc_sched.h
@@ -193,7 +193,6 @@ erts_nif_export_check_save_trace(Process *c_p, Eterm result,
 ERTS_GLB_INLINE Process *
 erts_proc_shadow2real(Process *c_p)
 {
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
 	Process *real_c_p = c_p->next;
 	ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
@@ -201,7 +200,6 @@ erts_proc_shadow2real(Process *c_p)
 	return real_c_p;
     }
     ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
-#endif
     return c_p;
 }
 
@@ -217,7 +215,6 @@ erts_proc_shadow2real(Process *c_p)
 	    || BeamOp(op_call_nif) == (BeamInstr *) (*(I))),		\
      ((NifExport *) (((char *) (I)) - offsetof(NifExport, exp.beam[0]))))
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 #include "erl_message.h"
 #include <stddef.h>
@@ -326,7 +323,6 @@ erts_make_dirty_shadow_proc(ErtsSchedulerData *esdp, Process *c_p)
 
 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-#endif /* ERTS_DIRTY_SCHEDULERS */
 
 #endif /* defined(ERTS_WANT_NFUNC_SCHED_INTERNALS__) && !defined(ERTS_NFUNC_SCHED_INTERNALS__) */
 
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index d3271a4560..43441e0228 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -304,7 +304,6 @@ schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp,
     return (ERL_NIF_TERM) THE_NON_VALUE;
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 static ERL_NIF_TERM dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
 static ERL_NIF_TERM dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
@@ -394,24 +393,19 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *
     return exiting;
 }
 
-#endif
 
 static void full_flush_env(ErlNifEnv* env)
 {
     flush_env(env);
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)
 	/* Dirty nif call using shadow process struct */
 	erts_flush_dirty_shadow_proc(env->proc);
-#endif
 }
 
 static void full_cache_env(ErlNifEnv* env)
 {    
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)
 	erts_cache_dirty_shadow_proc(env->proc);
-#endif
     cache_env(env);
 }
 
@@ -2591,7 +2585,6 @@ nif_export_restore(Process *c_p, NifExport *ep, Eterm res)
 }
 
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 /*
  * Finalize a dirty NIF call. This function is scheduled to cause the VM to
@@ -2719,7 +2712,6 @@ static_schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
     return static_schedule_dirty_nif(env, ERTS_PSFLG_DIRTY_CPU_PROC, argc, argv);
 }
 
-#endif /* ERTS_DIRTY_SCHEDULERS */
 
 /*
  * NIF execution wrapper used by enif_schedule_nif() for regular NIFs. It
@@ -2800,11 +2792,7 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags,
 	result = schedule(env, execute_nif, fp, proc->current->module,
 			  fun_name_atom, argc, argv);
     else if (!(flags & ~(ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND))) {
-#ifdef ERTS_DIRTY_SCHEDULERS
 	result = schedule_dirty_nif(env, flags, fp, fun_name_atom, argc, argv);
-#else
-        result = enif_raise_exception(env, am_notsup);
-#endif
     }
     else
 	result = enif_make_badarg(env);
@@ -2826,12 +2814,10 @@ enif_thread_type(void)
     switch (esdp->type) {
     case ERTS_SCHED_NORMAL:
 	return ERL_NIF_THR_NORMAL_SCHEDULER;
-#ifdef ERTS_DIRTY_SCHEDULERS
     case ERTS_SCHED_DIRTY_CPU:
 	return ERL_NIF_THR_DIRTY_CPU_SCHEDULER;
     case ERTS_SCHED_DIRTY_IO:
         return ERL_NIF_THR_DIRTY_IO_SCHEDULER;
-#endif
     default:
         ERTS_INTERNAL_ERROR("Invalid scheduler type");
 	return -1;
@@ -3929,14 +3915,9 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 		 * dirty scheduler support, treat a non-zero flags field as
 		 * a load error.
 		 */
-#ifdef ERTS_DIRTY_SCHEDULERS
 		if (f->flags != ERL_NIF_DIRTY_JOB_IO_BOUND && f->flags != ERL_NIF_DIRTY_JOB_CPU_BOUND)
 		    ret = load_nif_error(BIF_P, bad_lib, "Illegal flags field value %d for NIF %T:%s/%u",
 					 f->flags, mod_atom, f->name, f->arity);
-#else
-		ret = load_nif_error(BIF_P, bad_lib, "NIF %T:%s/%u requires a runtime with dirty scheduler support.",
-				     mod_atom, f->name, f->arity);
-#endif
 	    }
 	    else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0])
                      < BEAM_NIF_MIN_FUNC_SZ)
@@ -4009,7 +3990,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 		       (BeamInstr) BeamOp(op_i_generic_breakpoint));
 		g->orig_instr = (BeamInstr) BeamOp(op_call_nif);
 	    }
-#ifdef ERTS_DIRTY_SCHEDULERS
 	    if (f->flags) {
 		code_ptr[3] = (BeamInstr) f->fptr;
 		code_ptr[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ?
@@ -4017,7 +3997,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 		    (BeamInstr) static_schedule_dirty_cpu_nif;
 	    }
 	    else
-#endif
 		code_ptr[1] = (BeamInstr) f->fptr;
 	    code_ptr[2] = (BeamInstr) lib;
 	}
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index ab5030e5b9..8c59116fb6 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -192,10 +192,8 @@ static UWord thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_O
 ErtsPTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE);
 
 int erts_sched_thread_suggested_stack_size = -1;
-#ifdef ERTS_DIRTY_SCHEDULERS
 int erts_dcpu_sched_thread_suggested_stack_size = -1;
 int erts_dio_sched_thread_suggested_stack_size = -1;
-#endif
 #ifdef ERTS_ENABLE_LOCK_CHECK
 ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];
 #endif
@@ -223,10 +221,8 @@ typedef struct {
 
 typedef struct {
     Uint32 normal;
-#ifdef ERTS_DIRTY_SCHEDULERS
     Uint32 dirty_cpu;
     Uint32 dirty_io;
-#endif
 } ErtsSchedTypeCounters;
 
 static struct {
@@ -239,9 +235,7 @@ static struct {
     Eterm changer;
     ErtsMultiSchedulingBlock nmsb; /* Normal multi Scheduling Block */
     ErtsMultiSchedulingBlock msb; /* Multi Scheduling Block */
-#ifdef ERTS_DIRTY_SCHEDULERS
     ErtsSchedType last_msb_dirty_type;
-#endif
 } schdlr_sspnd;
 
 static void init_scheduler_suspend(void);
@@ -250,10 +244,8 @@ static ERTS_INLINE Uint32
 schdlr_sspnd_eq_nscheds(ErtsSchedTypeCounters *val1p, ErtsSchedTypeCounters *val2p)
 {
     int res = val1p->normal == val2p->normal;
-#ifdef ERTS_DIRTY_SCHEDULERS
     res &= val1p->dirty_cpu == val2p->dirty_cpu;
     res &= val1p->dirty_io == val2p->dirty_io;
-#endif
     return res;
 }
 
@@ -264,16 +256,10 @@ schdlr_sspnd_get_nscheds(ErtsSchedTypeCounters *valp,
     switch (type) {
     case ERTS_SCHED_NORMAL:
         return valp->normal;
-#ifdef ERTS_DIRTY_SCHEDULERS
     case ERTS_SCHED_DIRTY_CPU:
         return valp->dirty_cpu;
     case ERTS_SCHED_DIRTY_IO:
         return valp->dirty_io;
-#else
-    case ERTS_SCHED_DIRTY_CPU:
-    case ERTS_SCHED_DIRTY_IO:
-        return 0;
-#endif
     default:
 	ERTS_INTERNAL_ERROR("Invalid scheduler type");
 	return 0;
@@ -285,10 +271,8 @@ static ERTS_INLINE Uint32
 schdlr_sspnd_get_nscheds_tot(ErtsSchedTypeCounters *valp)
 {
     Uint32 res = valp->normal;
-#ifdef ERTS_DIRTY_SCHEDULERS
     res += valp->dirty_cpu;
     res += valp->dirty_io;
-#endif
     return res;
 }
 #endif
@@ -303,14 +287,12 @@ schdlr_sspnd_dec_nscheds(ErtsSchedTypeCounters *valp,
     case ERTS_SCHED_NORMAL:
         valp->normal--;
 	break;
-#ifdef ERTS_DIRTY_SCHEDULERS
     case ERTS_SCHED_DIRTY_CPU:
         valp->dirty_cpu--;
 	break;
     case ERTS_SCHED_DIRTY_IO:
         valp->dirty_io--;
 	break;
-#endif
     default:
 	ERTS_INTERNAL_ERROR("Invalid scheduler type");
     }
@@ -324,14 +306,12 @@ schdlr_sspnd_inc_nscheds(ErtsSchedTypeCounters *valp,
     case ERTS_SCHED_NORMAL:
         valp->normal++;
 	break;
-#ifdef ERTS_DIRTY_SCHEDULERS
     case ERTS_SCHED_DIRTY_CPU:
         valp->dirty_cpu++;
 	break;
     case ERTS_SCHED_DIRTY_IO:
         valp->dirty_io++;
 	break;
-#endif
     default:
 	ERTS_INTERNAL_ERROR("Invalid scheduler type");
     }
@@ -345,14 +325,12 @@ schdlr_sspnd_set_nscheds(ErtsSchedTypeCounters *valp,
     case ERTS_SCHED_NORMAL:
         valp->normal = no;
 	break;
-#ifdef ERTS_DIRTY_SCHEDULERS
     case ERTS_SCHED_DIRTY_CPU:
         valp->dirty_cpu = no;
 	break;
     case ERTS_SCHED_DIRTY_IO:
         valp->dirty_io = no;
 	break;
-#endif
     default:
 	ERTS_INTERNAL_ERROR("Invalid scheduler type");
     }
@@ -398,7 +376,6 @@ static erts_atomic_t runq_supervisor_sleeping;
 ErtsAlignedRunQueue * ERTS_WRITE_UNLIKELY(erts_aligned_run_queues);
 Uint ERTS_WRITE_UNLIKELY(erts_no_run_queues);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 struct {
     union {
@@ -411,12 +388,10 @@ struct {
     } io;
 } dirty_count erts_align_attribute(ERTS_CACHE_LINE_SIZE);
 
-#endif
 
 static ERTS_INLINE void
 dirty_active(ErtsSchedulerData *esdp, erts_aint32_t add)
 {
-#ifdef ERTS_DIRTY_SCHEDULERS
     erts_aint32_t val;
     erts_atomic32_t *ap;
     switch (esdp->type) {
@@ -441,18 +416,15 @@ dirty_active(ErtsSchedulerData *esdp, erts_aint32_t add)
     val = erts_atomic32_read_nob(ap);
     val += add;
     erts_atomic32_set_nob(ap, val);
-#endif
 }
 
 ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_scheduler_data);
-#ifdef ERTS_DIRTY_SCHEDULERS
 ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_cpu_scheduler_data);
 ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_io_scheduler_data);
 typedef union {
     Process dsp;
     char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(Process))];
 } ErtsAlignedDirtyShadowProcess;
-#endif
 
 typedef union {
     ErtsSchedulerSleepInfo ssi;
@@ -460,10 +432,8 @@ typedef union {
 } ErtsAlignedSchedulerSleepInfo;
 
 static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info;
-#ifdef ERTS_DIRTY_SCHEDULERS
 static ErtsAlignedSchedulerSleepInfo *aligned_dirty_cpu_sched_sleep_info;
 static ErtsAlignedSchedulerSleepInfo *aligned_dirty_io_sched_sleep_info;
-#endif
 
 static Uint last_reductions;
 static Uint last_exact_reductions;
@@ -535,7 +505,6 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist,
     (ASSERT(-1 <= ((int) (IX))					        \
 		 && ((int) (IX)) < ((int) erts_no_schedulers)),		\
      &aligned_sched_sleep_info[(IX)].ssi)
-#ifdef ERTS_DIRTY_SCHEDULERS
 #define ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(IX)				\
     (ASSERT(0 <= ((int) (IX))					        \
 	    && ((int) (IX)) < ((int) erts_no_dirty_cpu_schedulers)),	\
@@ -544,7 +513,6 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist,
     (ASSERT(0 <= ((int) (IX))					        \
 	    && ((int) (IX)) < ((int) erts_no_dirty_io_schedulers)),	\
      &aligned_dirty_io_sched_sleep_info[(IX)].ssi)
-#endif
 
 #define ERTS_FOREACH_RUNQ(RQVAR, DO)					\
 do {									\
@@ -862,7 +830,6 @@ erts_late_init_process(void)
 static void
 init_sched_wall_time(ErtsSchedulerData *esdp, Uint64 time_stamp)
 {
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (esdp->type != ERTS_SCHED_NORMAL) {
         erts_atomic32_init_nob(&esdp->sched_wall_time.u.mod, 0);
         esdp->sched_wall_time.enabled = 1;
@@ -871,7 +838,6 @@ init_sched_wall_time(ErtsSchedulerData *esdp, Uint64 time_stamp)
         esdp->sched_wall_time.working.start = ERTS_SCHED_WTIME_IDLE;
     }
     else
-#endif
     {
         esdp->sched_wall_time.u.need = erts_sched_balance_util;
         esdp->sched_wall_time.enabled = 0;
@@ -1069,7 +1035,6 @@ init_runq_sched_util(ErtsRunQueueSchedUtil *rqsu, int enabled)
 
 #endif /* ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT */
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 typedef struct {
     Uint64 working;
@@ -1122,7 +1087,6 @@ read_dirty_sched_wall_time(ErtsSchedulerData *esdp, ErtsDirtySchedWallTime *info
         info->working = info->total;
 }
 
-#endif
 
 
 static void
@@ -1225,10 +1189,8 @@ typedef struct {
     Eterm ref_heap[ERTS_REF_THING_SIZE];
     Uint req_sched;
     erts_atomic32_t refc;
-#ifdef ERTS_DIRTY_SCHEDULERS
     int want_dirty_cpu;
     int want_dirty_io;
-#endif
 } ErtsSchedWallTimeReq;
 
 typedef struct {
@@ -1299,7 +1261,6 @@ reply_sched_wall_time(void *vswtrp)
     hpp = NULL;
     szp = &sz;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (esdp->sched_wall_time.enabled
         && swtrp->req_sched == esdp->no
         && (swtrp->want_dirty_cpu || swtrp->want_dirty_io)) {
@@ -1381,7 +1342,6 @@ reply_sched_wall_time(void *vswtrp)
         erts_free(ERTS_ALC_T_TMP, dswt);
     }
     else
-#endif
     {
         /* Reply with info about this scheduler only... */
 
@@ -1449,10 +1409,8 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable,
     swtrp->proc = c_p;
     swtrp->ref = STORE_NC(&hp, NULL, ref);
     swtrp->req_sched = esdp->no;
-#ifdef ERTS_DIRTY_SCHEDULERS
     swtrp->want_dirty_cpu = want_dirty_cpu;
     swtrp->want_dirty_io = want_dirty_io;
-#endif
     erts_atomic32_init_nob(&swtrp->refc,
 			       (erts_aint32_t) erts_no_schedulers);
 
@@ -3276,21 +3234,16 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
     ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
 	erts_spin_lock(&rq->sleepers.lock);
-#endif
     flgs = sched_prep_spin_wait(ssi);
     if (flgs & ERTS_SSI_FLG_SUSPENDED) {
 	/* Go suspend instead... */
-#ifdef ERTS_DIRTY_SCHEDULERS
 	if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
 	    erts_spin_unlock(&rq->sleepers.lock);
-#endif
 	return;
     }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) {
 	ssi->prev = NULL;
 	ssi->next = rq->sleepers.list;
@@ -3300,7 +3253,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	erts_spin_unlock(&rq->sleepers.lock);
         dirty_active(esdp, -1);
     }
-#endif
 
     /*
      * If all schedulers are waiting, one of them *should*
@@ -3643,7 +3595,6 @@ ssi_wake(ErtsSchedulerSleepInfo *ssi)
     erts_sched_finish_poke(ssi, ssi_flags_set_wake(ssi));
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 static void
 dcpu_sched_ix_suspend_wake(Uint ix)
@@ -3675,7 +3626,6 @@ dio_sched_ix_wake(Uint ix)
 }
 #endif
 
-#endif
 
 static void
 wake_scheduler(ErtsRunQueue *rq)
@@ -3694,7 +3644,6 @@ wake_scheduler(ErtsRunQueue *rq)
     ssi_wake(rq->scheduler->ssi);
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 static void
 wake_dirty_schedulers(ErtsRunQueue *rq, int one)
 {
@@ -3745,7 +3694,6 @@ wake_dirty_scheduler(ErtsRunQueue *rq)
     wake_dirty_schedulers(rq, 1);
 }
 
-#endif
 
 #define ERTS_NO_USED_RUNQS_SHIFT 16
 #define ERTS_NO_RUNQS_MASK 0xffffU
@@ -3881,11 +3829,9 @@ static ERTS_INLINE void
 smp_notify_inc_runq(ErtsRunQueue *runq)
 {
     if (runq) {
-#ifdef ERTS_DIRTY_SCHEDULERS
 	if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix))
 	    wake_dirty_scheduler(runq);
 	else
-#endif
 	    wake_scheduler(runq);
     }
 }
@@ -4244,7 +4190,6 @@ schedule_bound_processes(ErtsRunQueue *rq,
     }
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 static ERTS_INLINE void
 clear_proc_dirty_queue_bit(Process *p, ErtsRunQueue *rq, int prio_bit)
@@ -4268,7 +4213,6 @@ clear_proc_dirty_queue_bit(Process *p, ErtsRunQueue *rq, int prio_bit)
     ASSERT(old & qb);
 }
 
-#endif /* ERTS_DIRTY_SCHEDULERS */
 
 
 static void
@@ -5539,13 +5483,11 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags)
 	    rq->wakeup_other += (left_len*wo_reds
 				 + ERTS_WAKEUP_OTHER_FIXED_INC);
 	    if (rq->wakeup_other > wakeup_other.limit) {
-#ifdef ERTS_DIRTY_SCHEDULERS
 		if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) {
 		    if (rq->waiting) {
 			wake_dirty_scheduler(rq);
 		    }
 		} else
-#endif
 		{
 		    int empty_rqs =
 			erts_atomic32_read_acqb(&no_empty_run_queues);
@@ -5822,7 +5764,6 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
         case ERTS_SCHED_NORMAL:
             id = (int) esdp->no;
             break;
-#ifdef ERTS_DIRTY_SCHEDULERS
         case ERTS_SCHED_DIRTY_CPU:
             id = (int) erts_no_schedulers;
             id += (int) esdp->dirty_no;
@@ -5832,7 +5773,6 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
             id += (int) erts_no_dirty_cpu_schedulers;
             id += (int) esdp->dirty_no;
             break;
-#endif
         default:
             ERTS_INTERNAL_ERROR("Invalid scheduler type");
             break;
@@ -5892,7 +5832,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     esdp->f_reg_array =
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER,
 					   MAX_REG * sizeof(FloatDef));
-#ifdef ERTS_DIRTY_SCHEDULERS
     esdp->run_queue = runq;
     if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) {
 	esdp->no = 0;
@@ -5926,12 +5865,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 				    | ERTS_PSFLG_PROXY));
 	shadow_proc->static_flags = ERTS_STC_FLG_SHADOW_PROC;
     }
-#else
-    runq->scheduler = esdp;
-    esdp->run_queue = runq;
-    esdp->no = (Uint) num;
-    esdp->type = ERTS_SCHED_NORMAL;
-#endif
 
     esdp->ssi = ssi;
     esdp->current_process = NULL;
@@ -5964,10 +5897,8 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 
 void
 erts_init_scheduling(int no_schedulers, int no_schedulers_online
-#ifdef ERTS_DIRTY_SCHEDULERS
 		     , int no_dirty_cpu_schedulers, int no_dirty_cpu_schedulers_online,
 		     int no_dirty_io_schedulers
-#endif
 		     )
 {
     int ix, n, no_ssi, tot_rqs;
@@ -5989,12 +5920,10 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     ASSERT(no_schedulers_online <= no_schedulers);
     ASSERT(no_schedulers_online >= 1);
     ASSERT(no_schedulers >= 1);
-#ifdef ERTS_DIRTY_SCHEDULERS
     ASSERT(no_dirty_cpu_schedulers <= no_schedulers);
     ASSERT(no_dirty_cpu_schedulers >= 1);
     ASSERT(no_dirty_cpu_schedulers_online <= no_schedulers_online);
     ASSERT(no_dirty_cpu_schedulers_online >= 1);
-#endif
 
     /* Create and initialize run queues */
 
@@ -6021,14 +5950,12 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
 	erts_cnd_init(&rq->cnd);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
         if (ERTS_RUNQ_IX_IS_DIRTY(ix)) {
             erts_spinlock_init(&rq->sleepers.lock, "dirty_run_queue_sleep_list",
                 make_small(ix + 1),
                 ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER);
         }
 	rq->sleepers.list = NULL;
-#endif
 
 	rq->waiting = 0;
 	rq->woken = 0;
@@ -6088,12 +6015,10 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     n = (int) no_schedulers;
     erts_no_schedulers = n;
     erts_no_total_schedulers = n;
-#ifdef ERTS_DIRTY_SCHEDULERS
     erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers;
     erts_no_total_schedulers += no_dirty_cpu_schedulers;
     erts_no_dirty_io_schedulers = no_dirty_io_schedulers;
     erts_no_total_schedulers += no_dirty_io_schedulers;
-#endif
 
     /* Create and initialize scheduler sleep info */
     no_ssi = n+1;
@@ -6114,7 +6039,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 
     aligned_sched_sleep_info++;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     aligned_dirty_cpu_sched_sleep_info =
 	erts_alloc_permanent_cache_aligned(
 	    ERTS_ALC_T_SCHDLR_SLP_INFO,
@@ -6135,7 +6059,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	ssi->event = NULL; /* initialized in sched_dirty_io_thread_func */
 	erts_atomic32_init_nob(&ssi->aux_work, 0);
     }
-#endif
 
     /* Create and initialize scheduler specific data */
 
@@ -6155,7 +6078,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 			    NULL, 0);
     }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     {
         Uint64 ts = sched_wall_time_ts();
 	int dirty_scheds = no_dirty_cpu_schedulers + no_dirty_io_schedulers;
@@ -6186,7 +6108,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 				&adsp[adspix++].dsp, ts);
 	}
     }
-#endif
 
     init_misc_aux_work();
     init_swtreq_alloc();
@@ -6238,7 +6159,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	    suspend_run_queue(ERTS_RUNQ_IX(ix));
     }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
     schdlr_sspnd_set_nscheds(&schdlr_sspnd.online,
 			     ERTS_SCHED_DIRTY_CPU,
@@ -6274,7 +6194,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     erts_atomic32_init_nob(&dirty_count.io.active,
                                (erts_aint32_t) no_dirty_io_schedulers);
 
-#endif
 
     if (set_schdlr_sspnd_change_flags)
 	erts_atomic32_set_nob(&schdlr_sspnd.changing,
@@ -6359,7 +6278,6 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
 #define ERTS_ENQUEUE_DIRTY_CPU_QUEUE 2
 #define ERTS_ENQUEUE_DIRTY_IO_QUEUE 3
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 static int
 check_dirty_enqueue_in_prio_queue(Process *c_p,
@@ -6453,7 +6371,6 @@ fin_dirty_enq_s_change(Process *p,
     return !0;
 }
 
-#endif /* ERTS_DIRTY_SCHEDULERS */
 
 static ERTS_INLINE int
 check_enqueue_in_prio_queue(Process *c_p,
@@ -6468,14 +6385,12 @@ check_enqueue_in_prio_queue(Process *c_p,
 
     *prq_prio_p = aprio;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (actual & ERTS_PSFLGS_DIRTY_WORK) {
 	int res = check_dirty_enqueue_in_prio_queue(c_p, newp, actual,
 						    aprio, qbit);
 	if (res != ERTS_ENQUEUE_NORMAL_QUEUE)
 	    return res;
     }
-#endif
 
     max_qbit = (actual >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET) & ERTS_PSFLGS_QMASK;
     max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS;
@@ -6518,7 +6433,6 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st
 
 	return NULL;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
     case ERTS_ENQUEUE_DIRTY_CPU_QUEUE:
     case -ERTS_ENQUEUE_DIRTY_CPU_QUEUE:
@@ -6539,7 +6453,6 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st
 
 	return NULL;
 
-#endif
 
     default: {
 	ErtsRunQueue* runq;
@@ -6583,7 +6496,6 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
 	running_flgs = ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS;
     else {
 	running_flgs = ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS;
-#ifdef ERTS_DIRTY_SCHEDULERS
         if (state & ERTS_PSFLG_DIRTY_ACTIVE_SYS
             && (p->flags & (F_DELAY_GC|F_DISABLE_GC))) {
             /*
@@ -6601,7 +6513,6 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
                                                     ~ERTS_PSFLG_DIRTY_ACTIVE_SYS);
             state &= ~ERTS_PSFLG_DIRTY_ACTIVE_SYS;
         }
-#endif
     }
 
     a = state;
@@ -6796,11 +6707,9 @@ change_proc_schedule_state(Process *p,
 		  | ERTS_PSFLG_DIRTY_RUNNING_SYS
 		  | ERTS_PSFLG_IN_RUNQ
 		  | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE
-#ifdef ERTS_DIRTY_SCHEDULERS
 	    || (n & (ERTS_PSFLG_RUNNING
 		     | ERTS_PSFLG_RUNNING_SYS
 		     | ERTS_PSFLG_EXITING)) == ERTS_PSFLG_EXITING
-#endif
 	    ) {
 	    /*
 	     * Active and seemingly need to be enqueued, but
@@ -7132,7 +7041,6 @@ nrml_sched_ix_resume_wake(Uint ix)
     sched_resume_wake__(ERTS_SCHED_SLEEP_INFO_IX(ix));
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 static void
 dcpu_sched_ix_resume_wake(Uint ix)
@@ -7146,7 +7054,6 @@ dio_sched_ix_resume_wake(Uint ix)
     sched_resume_wake__(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix));
 }
 
-#endif
 
 static erts_aint32_t
 sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct)
@@ -7231,7 +7138,6 @@ init_scheduler_suspend(void)
     schdlr_sspnd.online.normal = 1;
     schdlr_sspnd.curr_online.normal = 1;
     schdlr_sspnd.active.normal = 1;
-#ifdef ERTS_DIRTY_SCHEDULERS
     schdlr_sspnd.online.dirty_cpu = 0;
     schdlr_sspnd.curr_online.dirty_cpu = 0;
     schdlr_sspnd.active.dirty_cpu = 0;
@@ -7239,7 +7145,6 @@ init_scheduler_suspend(void)
     schdlr_sspnd.curr_online.dirty_io = 0;
     schdlr_sspnd.active.dirty_io = 0;
     schdlr_sspnd.last_msb_dirty_type = ERTS_SCHED_DIRTY_IO;
-#endif
     erts_atomic32_init_nob(&schdlr_sspnd.changing, 0);
     schdlr_sspnd.chngq = NULL;
     schdlr_sspnd.changer = am_false;
@@ -7300,7 +7205,6 @@ schdlr_sspnd_resume_procs(ErtsSchedType sched_type,
     }
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 static ERTS_INLINE int
 have_dirty_work(void)
@@ -7544,7 +7448,6 @@ msb_scheduler_type_switch(ErtsSchedType sched_type,
 
 }
 
-#endif
 
 static void
 suspend_scheduler(ErtsSchedulerData *esdp)
@@ -7572,14 +7475,6 @@ suspend_scheduler(ErtsSchedulerData *esdp)
      */
 
 
-#if !defined(ERTS_DIRTY_SCHEDULERS)
-
-    sched_type = ERTS_SCHED_NORMAL;
-    online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN;
-    no = esdp->no;
-    ASSERT(no != 1);
-
-#else
 
     sched_type = esdp->type;
     switch (sched_type) {
@@ -7607,7 +7502,6 @@ suspend_scheduler(ErtsSchedulerData *esdp)
         /* Suspend and let scheduler 1 of another type execute... */
     }
 
-#endif
 
     if (sched_type != ERTS_SCHED_NORMAL) {
         dirty_active(esdp, -1);
@@ -8039,11 +7933,7 @@ erts_set_schedulers_online(Process *p,
     erts_aint32_t changing = 0, change_flags;
     int online, increase;
     ErtsProcList *plp;
-#ifdef ERTS_DIRTY_SCHEDULERS
     int dirty_no, change_dirty, dirty_online;
-#else
-    ASSERT(!dirty_only);
-#endif
 
     if (new_no < 1)
 	return ERTS_SCHDLR_SSPND_EINVAL;
@@ -8052,11 +7942,9 @@ erts_set_schedulers_online(Process *p,
     else if (erts_no_schedulers < new_no)
 	return ERTS_SCHDLR_SSPND_EINVAL;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (dirty_only)
 	resume_proc = 0;
     else
-#endif
     {
 	resume_proc = 1;
 	/*
@@ -8077,9 +7965,7 @@ erts_set_schedulers_online(Process *p,
     have_unlocked_plocks = 0;
     no = (int) new_no;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (!dirty_only)
-#endif
     {
 	changing = erts_atomic32_read_nob(&schdlr_sspnd.changing);
 	if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) {
@@ -8107,12 +7993,6 @@ erts_set_schedulers_online(Process *p,
 
     *old_no = online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
 						ERTS_SCHED_NORMAL);
-#ifndef ERTS_DIRTY_SCHEDULERS
-    if (no == online) {
-	res = ERTS_SCHDLR_SSPND_DONE;
-	goto done;
-    }
-#else /* ERTS_DIRTY_SCHEDULERS */
     dirty_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
 					    ERTS_SCHED_DIRTY_CPU);
     if (dirty_only)
@@ -8164,7 +8044,6 @@ erts_set_schedulers_online(Process *p,
     if (dirty_only)
 	increase = (dirty_no > dirty_online);
     else
-#endif /* ERTS_DIRTY_SCHEDULERS */
     {
 	change_flags |= ERTS_SCHDLR_SSPND_CHNG_ONLN;
 	schdlr_sspnd_set_nscheds(&schdlr_sspnd.online,
@@ -8178,7 +8057,6 @@ erts_set_schedulers_online(Process *p,
     res = ERTS_SCHDLR_SSPND_DONE;
     if (increase) {
 	int ix;
-#ifdef ERTS_DIRTY_SCHEDULERS
 	if (change_dirty) {
 	    ErtsSchedulerSleepInfo* ssi;
 	    if (schdlr_sspnd.msb.ongoing) {
@@ -8192,7 +8070,6 @@ erts_set_schedulers_online(Process *p,
 	    }
 	}
 	if (!dirty_only)
-#endif
 	{
 	    if (schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) {
 		for (ix = online; ix < no; ix++)
@@ -8214,7 +8091,6 @@ erts_set_schedulers_online(Process *p,
 	}
     }
     else /* if decrease */ {
-#ifdef ERTS_DIRTY_SCHEDULERS
 	if (change_dirty) {
 	    if (schdlr_sspnd.msb.ongoing) {
 		for (ix = dirty_no; ix < dirty_online; ix++)
@@ -8232,7 +8108,6 @@ erts_set_schedulers_online(Process *p,
 	    }
 	}
 	if (!dirty_only)
-#endif
 	{
 	    if (schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) {
 		for (ix = no; ix < online; ix++)
@@ -8370,7 +8245,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
                 wake_scheduler(rq);
             }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
             if (!normal) {
                 ERTS_RUNQ_FLGS_SET_NOB(ERTS_RUNQ_IX(0), ERTS_RUNQ_FLG_MSB_EXEC);
                 erts_atomic32_read_bor_nob(&ERTS_RUNQ_IX(0)->scheduler->ssi->flags,
@@ -8380,7 +8254,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
                 for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++)
                     dio_sched_ix_suspend_wake(ix);
             }
-#endif
 
         wait_until_msb:
 
@@ -8446,7 +8319,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 		for (ix = online; ix < erts_no_run_queues; ix++)
 		    suspend_run_queue(ERTS_RUNQ_IX(ix));
 	    }
-#ifdef ERTS_DIRTY_SCHEDULERS
             if (!schdlr_sspnd.msb.ongoing) {
                 /* Get rid of msb-exec flag in run-queue of scheduler 1 */
                 resume_run_queue(ERTS_RUNQ_IX(0));
@@ -8457,7 +8329,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 		for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++)
 		    dio_sched_ix_resume_wake(ix);
 	    }
-#endif
 	}
 
     unblock_res:
@@ -8609,7 +8480,6 @@ sched_thread_func(void *vesdp)
     return NULL;
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 static void*
 sched_dirty_cpu_thread_func(void *vesdp)
 {
@@ -8701,7 +8571,6 @@ sched_dirty_io_thread_func(void *vesdp)
 	     no);
     return NULL;
 }
-#endif
 
 static ethr_tid aux_tid;
 
@@ -8757,7 +8626,6 @@ erts_start_schedulers(void)
     }
     erts_no_schedulers = actual;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     {
 	int ix;
 	for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
@@ -8777,7 +8645,6 @@ erts_start_schedulers(void)
 		erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d\n", ix);
 	}
     }
-#endif
 
     ERTS_THR_MEMORY_BARRIER;
 
@@ -8967,7 +8834,6 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 	    ASSERT((ERTS_PSFLG_RUNNING | ERTS_PSFLG_DIRTY_RUNNING)
 		   & erts_atomic32_read_nob(&rp->state));
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 	    if (!suspend
 		&& (erts_atomic32_read_nob(&rp->state)
 		    & ERTS_PSFLG_DIRTY_RUNNING)) {
@@ -8979,7 +8845,6 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 		}
 		goto done;
 	    }
-#endif
 
 	running:
 
@@ -9495,7 +9360,6 @@ erts_internal_is_process_executing_dirty_1(BIF_ALIST_1)
 {
     if (is_not_internal_pid(BIF_ARG_1))
 	BIF_ERROR(BIF_P, BADARG);
-#ifdef ERTS_DIRTY_SCHEDULERS
     else {
 	Process *rp = erts_proc_lookup(BIF_ARG_1);
 	if (rp) {
@@ -9506,7 +9370,6 @@ erts_internal_is_process_executing_dirty_1(BIF_ALIST_1)
 	    }
 	}
     }
-#endif
     BIF_RET(am_false);
 }
 
@@ -9522,7 +9385,6 @@ run_queues_len_aux(ErtsRunQueue *rq, Uint *tot_len, Uint *qlen, int *ip, int inc
     ASSERT(rq_len >= 0);
 
     if (incl_active_sched) {
-#ifdef ERTS_DIRTY_SCHEDULERS
         if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) {
             erts_aint32_t dcnt;
             if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(rq)) {
@@ -9537,7 +9399,6 @@ run_queues_len_aux(ErtsRunQueue *rq, Uint *tot_len, Uint *qlen, int *ip, int inc
             rq_len += (Sint) dcnt;
         }
         else
-#endif
         {
             if (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)
                 rq_len++;
@@ -9556,12 +9417,10 @@ erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched,
     Uint len = 0;
     int no_rqs = erts_no_run_queues;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (incl_dirty_io)
         no_rqs += ERTS_NUM_DIRTY_RUNQS;
     else
         no_rqs += ERTS_NUM_DIRTY_CPU_RUNQS;
-#endif
 
     if (atomic_queues_read) {
         ERTS_ATOMIC_FOREACH_RUNQ_X(rq, no_rqs,
@@ -9895,7 +9754,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
      * Clean up after the process being scheduled out.
      */
     if (!p) {	/* NULL in the very first schedule() call */
-#ifdef ERTS_DIRTY_SCHEDULERS
 	is_normal_sched = !esdp;
 	if (is_normal_sched) {
 	    esdp = erts_get_scheduler_data();
@@ -9904,17 +9762,12 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	else {
 	    ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp));
 	}
-#else
-	esdp = erts_get_scheduler_data();
-	is_normal_sched = 1;
-#endif
 	rq = erts_get_runq_current(esdp);
 	ASSERT(esdp);
 	fcalls = (int) erts_atomic32_read_acqb(&function_calls);
 	actual_reds = reds = 0;
 	erts_runq_lock(rq);
     } else {
-#ifdef ERTS_DIRTY_SCHEDULERS
 	is_normal_sched = !esdp;
 	if (is_normal_sched) {
 	    esdp = p->scheduler_data;
@@ -9923,10 +9776,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	else {
 	    ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp));
 	}
-#else
-	esdp = p->scheduler_data;
-	is_normal_sched = 1;
-#endif
 	ASSERT(esdp->current_process == p
 	       || esdp->free_process == p);
 
@@ -10152,7 +10001,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
                      */
                     flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
                     if ((flags & ERTS_RUNQ_FLG_SUSPENDED)
-#ifdef ERTS_DIRTY_SCHEDULERS
                         /* If multi scheduling block and we have
                          * dirty work, suspend and let dirty
                          * scheduler handle work... */
@@ -10160,7 +10008,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
                                        | ERTS_RUNQ_FLG_MSB_EXEC))
                               == ERTS_RUNQ_FLG_MSB_EXEC))
                             && have_dirty_work())
-#endif
                         ) {
                         non_empty_runq(rq);
                         flags |= ERTS_RUNQ_FLG_NONEMPTY;
@@ -10319,10 +10166,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		state = erts_atomic32_read_nob(&p->state);
 	    }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 	    if (!is_normal_sched)
 		clear_proc_dirty_queue_bit(p, rq, qbit);
-#endif
 
 	    while (1) {
 		erts_aint32_t exp, new;
@@ -10340,7 +10185,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 					    | ERTS_PSFLG_DIRTY_RUNNING
 					    | ERTS_PSFLG_DIRTY_RUNNING_SYS
 					    | ERTS_PSFLG_FREE)))
-#ifdef ERTS_DIRTY_SCHEDULERS
 				| (((state & (ERTS_PSFLG_RUNNING
 
 					      | ERTS_PSFLG_FREE
@@ -10349,7 +10193,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 					      | ERTS_PSFLG_EXITING))
 				    == ERTS_PSFLG_EXITING)
 				   & (!!is_normal_sched))
-#endif
 				   )
 			       & ((state & (ERTS_PSFLG_SUSPENDED
 					    | ERTS_PSFLG_EXITING
@@ -10358,11 +10201,9 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 					    | ERTS_PSFLG_ACTIVE_SYS
 					    | ERTS_PSFLG_DIRTY_ACTIVE_SYS))
 				  != ERTS_PSFLG_SUSPENDED)
-#ifdef ERTS_DIRTY_SCHEDULERS
                                & (!(state & (ERTS_PSFLG_EXITING
                                              | ERTS_PSFLG_PENDING_EXIT))
                                   | (!!is_normal_sched))
-#endif
                     );
 
 		if (run_process) {
@@ -10436,10 +10277,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	state = erts_atomic32_read_nob(&p->state);
 
-#ifndef ERTS_DIRTY_SCHEDULERS
-	ASSERT(!p->scheduler_data);
-	p->scheduler_data = esdp;
-#else /* ERTS_DIRTY_SCHEDULERS */
 	if (is_normal_sched) {
 	    if ((!!(state & ERTS_PSFLGS_DIRTY_WORK))
 		& (!(state & ERTS_PSFLG_ACTIVE_SYS))) {
@@ -10475,7 +10312,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		   : (rq == ERTS_DIRTY_IO_RUNQ
 		      && (state & ERTS_PSFLG_DIRTY_IO_PROC)));	    
 	}
-#endif
 
 	if (state & ERTS_PSFLG_PENDING_EXIT) {
 	    erts_handle_pending_exit(p,
@@ -10520,10 +10356,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
                     reds -= cost;
                     if (reds <= 0)
                         goto sched_out_proc;
-#ifdef ERTS_DIRTY_SCHEDULERS
                     if (state & ERTS_PSFLGS_DIRTY_WORK)
                         goto sched_out_proc;
-#endif
                 }
 
                 ASSERT(state & psflg_running_sys);
@@ -10561,10 +10395,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
                     reds -= cost;
                     if (reds <= 0)
                         goto sched_out_proc;
-#ifdef ERTS_DIRTY_SCHEDULERS
                     if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC))
                         goto sched_out_proc;
-#endif
                 }
             }
         }
@@ -10608,13 +10440,11 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st,
 			 Eterm st_result, int normal_sched)
 {
     Process *rp;
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (!normal_sched)
 	rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
 			       st->requester, 0,
 			       ERTS_P2P_FLG_INC_REFC);
     else
-#endif
 	rp = erts_proc_lookup(st->requester);
     if (rp) {
 	ErtsProcLocks rp_locks;
@@ -10664,10 +10494,8 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st,
 	if (rp_locks)
 	    erts_proc_unlock(rp, rp_locks);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 	if (!normal_sched)
 	    erts_proc_dec_refc(rp);
-#endif
     }
 
     erts_cleanup_offheap(&st->off_heap);
@@ -10823,9 +10651,7 @@ done:
 }
 
 static void save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio);
-#ifdef ERTS_DIRTY_SCHEDULERS
 static void save_dirty_task(Process *c_p, ErtsProcSysTask *st);
-#endif
 
 static int
 execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
@@ -10873,13 +10699,11 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
                         FLAGS(c_p) |= F_NEED_FULLSWEEP;
                     }
 		    reds -= scheduler_gc_proc(c_p, reds);
-#ifdef ERTS_DIRTY_SCHEDULERS
 		    if (c_p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) {
 			save_dirty_task(c_p, st);
 			st = NULL;
 			break;
 		    }
-#endif
 		    if (type == ERTS_PSTT_GC_MAJOR)
 			minor_gc = major_gc = 1;
 		    else
@@ -10921,13 +10745,11 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 						 fcalls, do_gc);
             reds -= cla_reds;
 	    if (is_non_value(st_res)) {
-#ifdef ERTS_DIRTY_SCHEDULERS
 		if (c_p->flags & F_DIRTY_CLA) {
 		    save_dirty_task(c_p, st);
 		    st = NULL;
 		    break;
 		}
-#endif
 		/* Needed gc, but gc was disabled */
 		save_gc_task(c_p, st, st_prio);
 		st = NULL;
@@ -10985,13 +10807,11 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
 	Eterm st_res;
 	int st_prio;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 	if (c_p->dirty_sys_tasks) {
 	    st = c_p->dirty_sys_tasks;
 	    c_p->dirty_sys_tasks = st->next;
 	}
 	else
-#endif
 	{
 	    st = fetch_sys_task(c_p, state, &qmask, &st_prio);
 	    if (!st)
@@ -11027,7 +10847,6 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
     return reds;
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 void
 erts_execute_dirty_system_task(Process *c_p)
@@ -11170,7 +10989,6 @@ dispatch_system_task(Process *c_p, erts_aint_t fail_state,
     return ret;
 }
 
-#endif
 
 static BIF_RETTYPE
 request_system_task(Process *c_p, Eterm requester, Eterm target,
@@ -11275,7 +11093,6 @@ request_system_task(Process *c_p, Eterm requester, Eterm target,
 	st->type = ERTS_PSTT_CPC;
 	if (!rp)
 	    goto noproc;
-#ifdef ERTS_DIRTY_SCHEDULERS
 	/*
 	 * If the process should start executing dirty
 	 * code it is important that this task is
@@ -11283,7 +11100,6 @@ request_system_task(Process *c_p, Eterm requester, Eterm target,
 	 */
 	fail_state |= (ERTS_PSFLG_DIRTY_RUNNING
 		       | ERTS_PSFLG_DIRTY_RUNNING_SYS);
-#endif
 	break;
 
     case am_copy_literals:
@@ -11305,14 +11121,12 @@ request_system_task(Process *c_p, Eterm requester, Eterm target,
 	noproc:
 	    failure = noproc_res;
 	}
-#ifdef ERTS_DIRTY_SCHEDULERS
 	else if (fail_state & (ERTS_PSFLG_DIRTY_RUNNING
 			       | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
 	    ret = dispatch_system_task(c_p, fail_state, st,
 				       target, priority, operation);
 	    goto cleanup_return;
 	}
-#endif
 	else {
 	    ERTS_INTERNAL_ERROR("Unknown failure schedule_process_sys_task()");
 	    failure = am_internal_error;
@@ -11328,9 +11142,7 @@ badarg:
 
     ERTS_BIF_PREP_ERROR(ret, c_p, BADARG);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 cleanup_return:
-#endif
 
     if (st) {
 	erts_cleanup_offheap(&st->off_heap);
@@ -11394,7 +11206,6 @@ erts_schedule_ets_free_fixation(Eterm pid, DbFixation* fix)
     erts_schedule_generic_sys_task(pid, ERTS_PSTT_ETS_FREE_FIXATION, fix);
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 static void
 flush_dirty_trace_messages(void *vpid)
@@ -11415,7 +11226,6 @@ flush_dirty_trace_messages(void *vpid)
     }
 }
 
-#endif /* ERTS_DIRTY_SCHEDULERS */
 
 void
 erts_schedule_flush_trace_messages(Process *proc, int force_on_proc)
@@ -11423,7 +11233,6 @@ erts_schedule_flush_trace_messages(Process *proc, int force_on_proc)
     ErtsThrPrgrDelayHandle dhndl;
     Eterm pid = proc->common.id;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     erts_aint32_t state;
 
     if (!force_on_proc) {
@@ -11433,7 +11242,6 @@ erts_schedule_flush_trace_messages(Process *proc, int force_on_proc)
 	    goto sched_flush_dirty;
 	}
     }
-#endif
 
     dhndl = erts_thr_progress_unmanaged_delay();
 
@@ -11441,7 +11249,6 @@ erts_schedule_flush_trace_messages(Process *proc, int force_on_proc)
 
     erts_thr_progress_unmanaged_continue(dhndl);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (!force_on_proc) {
 	state = erts_atomic32_read_mb(&proc->state);
 	if (state & (ERTS_PSFLG_DIRTY_RUNNING
@@ -11471,7 +11278,6 @@ erts_schedule_flush_trace_messages(Process *proc, int force_on_proc)
 	    erts_schedule_misc_aux_work(1, flush_dirty_trace_messages, vargp);
 	}
     }
-#endif
 }
 
 static void
@@ -11532,14 +11338,12 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio)
     }
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 static void
 save_dirty_task(Process *c_p, ErtsProcSysTask *st)
 {
     st->next = c_p->dirty_sys_tasks;
     c_p->dirty_sys_tasks = st;
 }
-#endif
 
 int
 erts_set_gc_state(Process *c_p, int enable)
@@ -11876,10 +11680,8 @@ static void early_init_process_struct(void *varg, Eterm data)
     Process *proc = arg->proc;
 
     proc->common.id = make_internal_pid(data);
-#ifdef ERTS_DIRTY_SCHEDULERS
     erts_atomic32_init_nob(&proc->dirty_state, 0);
     proc->dirty_sys_tasks = NULL;
-#endif
     erts_atomic32_init_relb(&proc->state, arg->state);
 
     RUNQ_SET_RQ(&proc->run_queue, arg->run_queue);
@@ -12369,10 +12171,8 @@ void erts_init_empty_process(Process *p)
     p->last_old_htop = NULL;
 #endif
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     erts_atomic32_init_nob(&p->dirty_state, 0);
     p->dirty_sys_tasks = NULL;
-#endif
     erts_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL);
 
     p->scheduler_data = NULL;
@@ -12943,14 +12743,12 @@ send_exit_signal(Process *c_p,		/* current process if and only
 		    Eterm *hp;
 		    ErlOffHeap *ohp;
 		    Uint rsn_sz = size_object(rsn);
-#ifdef ERTS_DIRTY_SCHEDULERS
 		    if (state & ERTS_PSFLG_DIRTY_RUNNING) {
 			bp = new_message_buffer(rsn_sz);
 			ohp = &bp->off_heap;
 			hp = &bp->mem[0];
 		    }
 		    else
-#endif
 		    {
 			hp = HAlloc(rp, rsn_sz);
 			ohp = &rp->off_heap;
@@ -12990,10 +12788,6 @@ send_exit_signal(Process *c_p,		/* current process if and only
                  * has been scheduled, we may need to add it to a normal run
                  * queue...
 		 */
-#ifndef ERTS_DIRTY_SCHEDULERS
-                (void) erts_atomic32_read_bor_relb(&rp->state,
-                                                       ERTS_PSFLG_PENDING_EXIT);
-#else
                 {
                     erts_aint32_t a = erts_atomic32_read_nob(&rp->state);
                     while (1) {
@@ -13011,7 +12805,6 @@ send_exit_signal(Process *c_p,		/* current process if and only
                         }
                     }
                 }
-#endif
 	    }
 	}
 	/* else:
@@ -13508,9 +13301,7 @@ erts_continue_exit_process(Process *p)
     erts_set_gc_state(p, 1);
     state = erts_atomic32_read_acqb(&p->state);
     if (state & ERTS_PSFLG_ACTIVE_SYS
-#ifdef ERTS_DIRTY_SCHEDULERS
         || p->dirty_sys_tasks
-#endif
         ) {
 	if (cleanup_sys_tasks(p, state, CONTEXT_REDS) >= CONTEXT_REDS/2)
 	    goto yield;
@@ -13520,9 +13311,7 @@ erts_continue_exit_process(Process *p)
     erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
     ASSERT(p->sys_task_qs == NULL);
     ASSERT(ERTS_PROC_GET_DELAYED_GC_TASK_QS(p) == NULL);
-#ifdef ERTS_DIRTY_SCHEDULERS
     ASSERT(p->dirty_sys_tasks == NULL);
-#endif
     erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 #endif
 
@@ -13619,7 +13408,6 @@ erts_continue_exit_process(Process *p)
 		break;
 	}
 
-#ifdef ERTS_DIRTY_SCHEDULERS
         if (a & (ERTS_PSFLG_DIRTY_RUNNING
                  | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
 	    p->flags |= F_DELAYED_DEL_PROC;
@@ -13629,7 +13417,6 @@ erts_continue_exit_process(Process *p)
              * when done with the process...
 	     */
 	}
-#endif
 
 	if (refc_inced && !(n & ERTS_PSFLG_IN_RUNQ))
 	    erts_proc_dec_refc(p);
@@ -14022,10 +13809,8 @@ void erts_halt(int code)
     if (-1 == erts_atomic32_cmpxchg_acqb(&erts_halt_progress,
 					     erts_no_schedulers,
 					     -1)) {
-#ifdef ERTS_DIRTY_SCHEDULERS
         ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_CPU_RUNQ, ERTS_RUNQ_FLG_HALTING);
         ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_IO_RUNQ, ERTS_RUNQ_FLG_HALTING);
-#endif
 	erts_halt_code = code;
 	notify_reap_ports_relb();
     }
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 150f22556c..5afe0acb7b 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -110,16 +110,12 @@ extern int erts_sched_compact_load;
 extern int erts_sched_balance_util;
 extern Uint erts_no_schedulers;
 extern Uint erts_no_total_schedulers;
-#ifdef ERTS_DIRTY_SCHEDULERS
 extern Uint erts_no_dirty_cpu_schedulers;
 extern Uint erts_no_dirty_io_schedulers;
-#endif
 extern Uint erts_no_run_queues;
 extern int erts_sched_thread_suggested_stack_size;
-#ifdef ERTS_DIRTY_SCHEDULERS
 extern int erts_dcpu_sched_thread_suggested_stack_size;
 extern int erts_dio_sched_thread_suggested_stack_size;
-#endif
 #define ERTS_SCHED_THREAD_MIN_STACK_SIZE 20	/* Kilo words */
 #define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */
 
@@ -362,12 +358,10 @@ typedef enum {
 
 typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 typedef struct {
     erts_spinlock_t lock;
     ErtsSchedulerSleepInfo *list;
 } ErtsSchedulerSleepList;
-#endif
 
 struct ErtsSchedulerSleepInfo_ {
     ErtsSchedulerSleepInfo *next;
@@ -477,9 +471,7 @@ struct ErtsRunQueue_ {
     erts_mtx_t mtx;
     erts_cnd_t cnd;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
     ErtsSchedulerSleepList sleepers;
-#endif
 
     ErtsSchedulerData *scheduler;
     int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
@@ -616,13 +608,11 @@ typedef struct {
     (&(ESDP)->aux_work_data.yield.NAME)
 void erts_notify_new_aux_yield_work(ErtsSchedulerData *esdp);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 typedef enum {
     ERTS_DIRTY_CPU_SCHEDULER,
     ERTS_DIRTY_IO_SCHEDULER
 } ErtsDirtySchedulerType;
 
-#endif
 
 struct ErtsSchedulerData_ {
     /*
@@ -645,10 +635,8 @@ struct ErtsSchedulerData_ {
     Process *current_process;
     ErtsSchedType type;
     Uint no;			/* Scheduler number for normal schedulers */
-#ifdef ERTS_DIRTY_SCHEDULERS
     Uint dirty_no;  /* Scheduler number for dirty schedulers */
     Process *dirty_shadow_process;
-#endif
     Port *current_port;
     ErtsRunQueue *run_queue;
     int virtual_reds;
@@ -687,10 +675,8 @@ typedef union {
 } ErtsAlignedSchedulerData;
 
 extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
-#ifdef ERTS_DIRTY_SCHEDULERS
 extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data;
 extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
-#endif
 
 
 #if defined(ERTS_ENABLE_LOCK_CHECK)
@@ -1058,14 +1044,10 @@ struct process {
     Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */
 
     ErtsProcSysTaskQs *sys_task_qs;
-#ifdef ERTS_DIRTY_SCHEDULERS
     ErtsProcSysTask *dirty_sys_tasks;
-#endif
 
     erts_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */
-#ifdef ERTS_DIRTY_SCHEDULERS
     erts_atomic32_t dirty_state; /* Process dirty state flags (see ERTS_PDSFLG_*) */
-#endif
 
     ErlMessageInQueue msg_inq;
     ErlTraceMessageQueue *trace_msg_q;
@@ -1222,7 +1204,6 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \
     (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 
 /*
  * Flags in the dirty_state field.
@@ -1249,7 +1230,6 @@ void erts_check_for_holes(Process* p);
 					 | ERTS_PDSFLG_IN_CPU_PRQ_HIGH	\
 					 | ERTS_PDSFLG_IN_CPU_PRQ_NORMAL\
 					 | ERTS_PDSFLG_IN_CPU_PRQ_LOW)
-#endif
 
 
 /*
@@ -1514,20 +1494,14 @@ extern int erts_system_profile_ts_type;
 	}						\
     } while (0)
 
-#if defined(ERTS_DIRTY_SCHEDULERS)
 #define ERTS_NUM_DIRTY_CPU_RUNQS 1
 #define ERTS_NUM_DIRTY_IO_RUNQS 1
-#else
-#define ERTS_NUM_DIRTY_CPU_RUNQS 0
-#define ERTS_NUM_DIRTY_IO_RUNQS 0
-#endif
 
 #define ERTS_NUM_DIRTY_RUNQS (ERTS_NUM_DIRTY_CPU_RUNQS+ERTS_NUM_DIRTY_IO_RUNQS)
 
 #define ERTS_RUNQ_IX(IX)						\
   (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues+ERTS_NUM_DIRTY_RUNQS), \
    &erts_aligned_run_queues[(IX)].runq)
-#ifdef ERTS_DIRTY_SCHEDULERS
 #define ERTS_RUNQ_IX_IS_DIRTY(IX)					\
   (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues+ERTS_NUM_DIRTY_RUNQS), \
    (erts_no_run_queues <= (IX)))
@@ -1538,13 +1512,9 @@ extern int erts_system_profile_ts_type;
 #define ERTS_DIRTY_IO_RUNQ  (&erts_aligned_run_queues[erts_no_run_queues+1].runq)
 #define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ) == ERTS_DIRTY_CPU_RUNQ)
 #define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ) == ERTS_DIRTY_IO_RUNQ)
-#else
-#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
-#endif
 #define ERTS_SCHEDULER_IX(IX)						\
   (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers),			\
    &erts_aligned_scheduler_data[(IX)].esd)
-#ifdef ERTS_DIRTY_SCHEDULERS
 #define ERTS_DIRTY_CPU_SCHEDULER_IX(IX)					\
   (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_cpu_schedulers),		\
    &erts_aligned_dirty_cpu_scheduler_data[(IX)].esd)
@@ -1557,24 +1527,14 @@ extern int erts_system_profile_ts_type;
     ((ESDP)->type == ERTS_SCHED_DIRTY_CPU)
 #define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP)				\
     ((ESDP)->type == ERTS_SCHED_DIRTY_IO)
-#else /* !ERTS_DIRTY_SCHEDULERS */
-#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
-#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
-#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
-#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
-#endif
 
 void erts_pre_init_process(void);
 void erts_late_init_process(void);
 void erts_early_init_scheduling(int);
 void erts_init_scheduling(int, int
-#ifdef ERTS_DIRTY_SCHEDULERS
 			  , int, int, int
-#endif
 			  );
-#ifdef ERTS_DIRTY_SCHEDULERS
 void erts_execute_dirty_system_task(Process *c_p);
-#endif
 int erts_set_gc_state(Process *c_p, int enable);
 Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable,
                                    int dirty_cpu, int want_dirty_io);
@@ -2241,7 +2201,6 @@ ErtsSchedulerData *erts_proc_sched_data(Process *c_p)
     ErtsSchedulerData *esdp;
     ASSERT(c_p);
     esdp = c_p->scheduler_data;
-#  if defined(ERTS_DIRTY_SCHEDULERS)
     if (esdp) {
 	ASSERT(esdp == erts_get_scheduler_data());
 	ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
@@ -2251,7 +2210,6 @@ ErtsSchedulerData *erts_proc_sched_data(Process *c_p)
 	ASSERT(esdp);
 	ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp));
     }
-#  endif
     ASSERT(esdp);
     return esdp;
 }
@@ -2283,11 +2241,9 @@ ERTS_GLB_INLINE
 Uint erts_get_scheduler_id(void)
 {
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
-#ifdef ERTS_DIRTY_SCHEDULERS
     if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp))
 	return 0;
     else
-#endif
 	return esdp ? esdp->no : (Uint) 0;
 }
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index c72ae32f2c..2b0ad0b98a 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -128,10 +128,8 @@ extern Eterm erts_nif_call_function(Process *p, Process *tracee,
                                     struct enif_func_t *,
                                     int argc, Eterm *argv);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
 int erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p,
 			BeamInstr *I, Eterm *reg);
-#endif /* ERTS_DIRTY_SCHEDULERS */
 
 
 /* Driver handle (wrapper for old plain handle) */
@@ -911,9 +909,7 @@ extern erts_atomic_t erts_copy_literal_area__;
 #define ERTS_COPY_LITERAL_AREA()					\
     ((ErtsLiteralArea *) erts_atomic_read_nob(&erts_copy_literal_area__))
 extern Process *erts_literal_area_collector;
-#ifdef ERTS_DIRTY_SCHEDULERS
 extern Process *erts_dirty_process_code_checker;
-#endif
 
 extern Process *erts_code_purger;
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 3f44d6e242..bc1b9b6ef4 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -7484,11 +7484,7 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size)
      */
     if (si_size >= ERL_DRV_SYS_INFO_SIZE(dirty_scheduler_support)) {
 	sip->dirty_scheduler_support =
-#ifdef ERTS_DIRTY_SCHEDULERS
 	    1
-#else
-	    0
-#endif
 	    ;
     }
 
diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables
index 47e1528958..094a35ae4b 100755
--- a/erts/emulator/utils/make_tables
+++ b/erts/emulator/utils/make_tables
@@ -59,7 +59,6 @@ my %dirty_bif_tab;
 
 my @bif;
 my @bif_info;
-my $dirty_schedulers = 'no';
 my $dirty_schedulers_test = 'no';
 my $hipe = 'no';
 
@@ -73,10 +72,6 @@ while (@ARGV && $ARGV[0] =~ /^-(\w+)/) {
 	$include = shift;
 	die "No directory for -include argument specified"
 	    unless defined $include;
-    } elsif($opt eq '-ds') {
-	$dirty_schedulers = shift;
-	die "No -ds argument specified"
-	    unless defined $dirty_schedulers;
     } elsif($opt eq '-dst') {
 	$dirty_schedulers_test = shift;
 	die "No -dst argument specified"
@@ -140,21 +135,19 @@ while (<>) {
 	push(@bif_info, [$type, $sched_type, $alias3, $alias]);
     } elsif ($type eq 'dirty-cpu' or $type eq 'dirty-io'
 	     or $type eq 'dirty-cpu-test' or $type eq 'dirty-io-test') {
-	if ($dirty_schedulers eq 'yes') {
-	    my($bif,$other) = (@args);
-	    $bif =~ m@^([a-z_.'0-9]+):(.*)/(\d)$@ or error("invalid BIF");
-	    my($mod,$name,$arity) = ($1,$2,$3);
-	    my $mfa = "$mod:$name/$arity";
-	    if (($type eq 'dirty-cpu')
-		or (($dirty_schedulers_test eq 'yes')
-		    and ($type eq 'dirty-cpu-test'))) {
-		$dirty_bif_tab{$mfa} = 'dirty_cpu';
-	    } elsif (($type eq 'dirty-io')
-		or (($dirty_schedulers_test eq 'yes')
-		    and ($type eq 'dirty-io-test'))) {
-		$dirty_bif_tab{$mfa} = 'dirty_io';
-	    }
-	}
+        my($bif,$other) = (@args);
+        $bif =~ m@^([a-z_.'0-9]+):(.*)/(\d)$@ or error("invalid BIF");
+        my($mod,$name,$arity) = ($1,$2,$3);
+        my $mfa = "$mod:$name/$arity";
+        if (($type eq 'dirty-cpu')
+            or (($dirty_schedulers_test eq 'yes')
+                and ($type eq 'dirty-cpu-test'))) {
+            $dirty_bif_tab{$mfa} = 'dirty_cpu';
+        } elsif (($type eq 'dirty-io')
+                 or (($dirty_schedulers_test eq 'yes')
+                     and ($type eq 'dirty-io-test'))) {
+            $dirty_bif_tab{$mfa} = 'dirty_io';
+        }
     } else {
 	error("invalid line");
     }
-- 
cgit v1.2.3


From 7f8e78dee2ff485de7eddce1bfab28d0823031ae Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 11 Jul 2017 18:24:59 +0200
Subject: erts: Allow read in ttsl driver to return EAGAIN

---
 erts/emulator/drivers/unix/ttsl_drv.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/unix/ttsl_drv.c b/erts/emulator/drivers/unix/ttsl_drv.c
index e425b99f16..3d1e58abe3 100644
--- a/erts/emulator/drivers/unix/ttsl_drv.c
+++ b/erts/emulator/drivers/unix/ttsl_drv.c
@@ -891,8 +891,8 @@ static void ttysl_from_tty(ErlDrvData ttysl_data, ErlDrvEvent fd)
 		tpos = 0;
 	    }
 	}
-    } else {
-        DEBUGLOG(("ttysl_from_tty: driver failure in read(%d,..) = %d\n", (int)(SWord)fd, i)); 
+    } else if (errno != EAGAIN && errno != EWOULDBLOCK) {
+        DEBUGLOG(("ttysl_from_tty: driver failure in read(%d,..) = %d (errno = %d)\n", (int)(SWord)fd, i, errno));
 	driver_failure(ttysl_port, -1);
     }
 }
-- 
cgit v1.2.3


From 69721d73cb00e31e6b27ccdd363991cb1965b869 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 21 Aug 2017 14:57:03 +0200
Subject: Make estone work with older releases

---
 erts/emulator/test/estone_SUITE.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/estone_SUITE.erl b/erts/emulator/test/estone_SUITE.erl
index 8b336b366d..01eb1852f6 100644
--- a/erts/emulator/test/estone_SUITE.erl
+++ b/erts/emulator/test/estone_SUITE.erl
@@ -364,7 +364,7 @@ monotonic_time() ->
     try erlang:monotonic_time() catch error:undef -> erlang:now() end.
 
 subtr(Before, After) when is_integer(Before), is_integer(After) ->
-    erlang:convert_time_unit(After-Before, native, microsecond);
+    erlang:convert_time_unit(After-Before, native, 1000000);
 subtr({_,_,_}=Before, {_,_,_}=After) ->
     timer:now_diff(After, Before).
 
-- 
cgit v1.2.3


From 6eae9b508dc172ca207df6131bfa36e2aaf2333f Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 4 Sep 2017 17:20:07 +0200
Subject: Fix unused-functions warnings

---
 erts/emulator/beam/erl_process.c | 6 ++++++
 erts/emulator/beam/erl_process.h | 1 +
 erts/emulator/sys/unix/sys.c     | 4 ++++
 3 files changed, 11 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 63c838a91d..22d77ccbdc 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1597,6 +1597,12 @@ erts_proclist_create(Process *p)
     return proclist_create(p);
 }
 
+ErtsProcList *
+erts_proclist_copy(ErtsProcList *plp)
+{
+    return proclist_copy(plp);
+}
+
 void
 erts_proclist_destroy(ErtsProcList *plp)
 {
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 9d7ba27c50..639818c20c 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1612,6 +1612,7 @@ Uint64 erts_ensure_later_proc_interval(Uint64);
 Uint64 erts_step_proc_interval(void);
 
 ErtsProcList *erts_proclist_create(Process *);
+ErtsProcList *erts_proclist_copy(ErtsProcList *);
 void erts_proclist_destroy(ErtsProcList *);
 
 ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *);
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 50d8a35217..c1fa660cf9 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -789,6 +789,8 @@ signalterm_to_signum(Eterm signal)
     }
 }
 
+#ifdef ERTS_SMP
+
 static ERTS_INLINE Eterm
 signum_to_signalterm(int signum)
 {
@@ -810,6 +812,8 @@ signum_to_signalterm(int signum)
     }
 }
 
+#endif
+
 #ifndef ERTS_SMP
 static ERTS_INLINE Uint
 signum_to_signalstate(int signum)
-- 
cgit v1.2.3


From 9b085b7f6d7dddcb83924c4fa50e18e8aec3c74d Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 4 Sep 2017 13:21:02 +0200
Subject: Fix some clang warnings

---
 erts/emulator/beam/external.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 1560844521..c0a3838d42 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1923,7 +1923,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		}
 
 		result_bin = erts_bin_nrml_alloc(size);
-		result_bin->orig_bytes[0] = VERSION_MAGIC;
+		result_bin->orig_bytes[0] = (byte)VERSION_MAGIC;
 		/* Next state immediately, no need to export context */
 		context->state = TTBEncode;
 		context->s.ec.flags = flags;
@@ -1981,7 +1981,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		context->s.cc.result_bin = result_bin;
 
 		result_bin = erts_bin_nrml_alloc(real_size);
-		result_bin->orig_bytes[0] = VERSION_MAGIC;
+		result_bin->orig_bytes[0] = (byte) VERSION_MAGIC;
 
 		context->s.cc.destination_bin = result_bin;
 		context->s.cc.dest_len = 0;
-- 
cgit v1.2.3


From 9674613dd9d1c5ac026c9bff05ea19df2886a2c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 6 Sep 2017 05:07:41 +0200
Subject: Avoid using $Src more than once

The C compiler will probably optimize this, but just to be sure...
---
 erts/emulator/beam/instrs.tab | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index b79b960fd7..690bd3a848 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -731,9 +731,10 @@ is_reference(Fail, Src) {
 }
 
 is_tagged_tuple(Fail, Src, Arityval, Tag) {
-    if (!(BEAM_IS_TUPLE($Src) &&
-          (tuple_val($Src))[0] == $Arityval &&
-          (tuple_val($Src))[1] == $Tag)) {
+    Eterm term = $Src;
+    if (!(BEAM_IS_TUPLE(term) &&
+          (tuple_val(term))[0] == $Arityval &&
+          (tuple_val(term))[1] == $Tag)) {
         $FAIL($Fail);
     }
 }
@@ -745,7 +746,8 @@ is_tuple(Fail, Src) {
 }
 
 is_tuple_of_arity(Fail, Src, Arityval) {
-    if (!(BEAM_IS_TUPLE($Src) && *tuple_val($Src) == $Arityval)) {
+    Eterm term = $Src;
+    if (!(BEAM_IS_TUPLE(term) && *tuple_val(term) == $Arityval)) {
         $FAIL($Fail);
     }
 }
-- 
cgit v1.2.3


From bffbd4fb504e9551fc7feb9177ef0a2394c00cae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 4 Sep 2017 15:25:58 +0200
Subject: Introduce a new trace_jump/1 instruction for tracing

As a preparation for introducing relative jumps, introduce
"trace_jump W" that can be used for tracing. This instruction
will continue to have an absolute address for the jump target.
(Note: This instruction is never created during loading; it
is only created in stubs when tracing is active.)
---
 erts/emulator/beam/erl_bif_trace.c  |  8 ++++----
 erts/emulator/beam/ops.tab          |  1 +
 erts/emulator/beam/trace_instrs.tab | 13 +++++++++++++
 3 files changed, 18 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index b02f966558..2859cdb0f1 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1367,7 +1367,7 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
 #ifdef DEBUG
 		ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI);
 #endif
-                ep->beam[0] = (BeamInstr) BeamOp(op_jump_f);
+                ep->beam[0] = (BeamInstr) BeamOp(op_trace_jump_W);
 		ep->beam[1] = (BeamInstr) ep->addressv[code_ix];
 	    }
 	    erts_set_call_trace_bif(ci, match_prog_set, 0);
@@ -1383,7 +1383,7 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
 	     */
 	    erts_clear_call_trace_bif(ci, 0);
 	    if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
-		ep->beam[0] = (BeamInstr) BeamOp(op_jump_f);
+		ep->beam[0] = (BeamInstr) BeamOp(op_trace_jump_W);
 	    }
 	}
     }
@@ -1675,7 +1675,7 @@ uninstall_exp_breakpoints(BpFunctions* f)
 	if (ep->addressv[code_ix] != ep->beam) {
 	    continue;
 	}
-	ASSERT(ep->beam[0] == (BeamInstr) BeamOp(op_jump_f));
+	ASSERT(ep->beam[0] == (BeamInstr) BeamOp(op_trace_jump_W));
 	ep->addressv[code_ix] = (BeamInstr *) ep->beam[1];
     }
 }
@@ -1694,7 +1694,7 @@ clean_export_entries(BpFunctions* f)
 	if (ep->addressv[code_ix] == ep->beam) {
 	    continue;
 	}
-	if (ep->beam[0] == (BeamInstr) BeamOp(op_jump_f)) {
+	if (ep->beam[0] == (BeamInstr) BeamOp(op_trace_jump_W)) {
 	    ep->beam[0] = (BeamInstr) 0;
 	    ep->beam[1] = (BeamInstr) 0;
 	}
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index d64f6f2cfc..4152e0ced5 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -69,6 +69,7 @@ i_debug_breakpoint
 i_return_time_trace
 i_return_to_trace
 i_yield
+trace_jump W
 %hot
 
 return
diff --git a/erts/emulator/beam/trace_instrs.tab b/erts/emulator/beam/trace_instrs.tab
index c71f2ef003..c28bc8ebcb 100644
--- a/erts/emulator/beam/trace_instrs.tab
+++ b/erts/emulator/beam/trace_instrs.tab
@@ -153,3 +153,16 @@ i_debug_breakpoint() {
     goto handle_error;
     //| -no_next
 }
+
+
+
+//
+// Special jump instruction used for tracing. Takes an absolute
+// failure address.
+//
+
+trace_jump(Fail) {
+    //| -no_next
+    SET_I((BeamInstr *) $Fail);
+    Goto(*I);
+}
-- 
cgit v1.2.3


From 0c99036e21464c643dabcf285c9e122c57958192 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 8 Sep 2017 12:52:31 +0200
Subject: Use the correct name of the parameter

---
 erts/emulator/beam/macros.tab | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab
index bac96be7d3..35bbc92fea 100644
--- a/erts/emulator/beam/macros.tab
+++ b/erts/emulator/beam/macros.tab
@@ -65,7 +65,7 @@ GC_TEST_PRESERVE(NeedHeap, Live, PreserveTerm) {
        $PreserveTerm = reg[$Live];
        SWAPIN;
     }
-    HEAP_SPACE_VERIFIED($Nh);
+    HEAP_SPACE_VERIFIED($NeedHeap);
 }
 
 
-- 
cgit v1.2.3


From 08fa79f049a9038fbcf5a40cedf86372f100080b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sun, 10 Sep 2017 09:52:20 +0200
Subject: Change operand type from 's' to 'S' for a few instructions

'S' is slightly more efficient. Using 'S' may also enable more
packing.
---
 erts/emulator/beam/ops.tab | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index d64f6f2cfc..14d2d39e0e 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -698,7 +698,7 @@ is_boolean f xy
 is_function2 Fail=f acq Arity => jump Fail
 is_function2 Fail=f Fun a => jump Fail
 
-is_function2 f s s
+is_function2 f S s
 
 # Allocating & initializing.
 allocate Need Regs | init Y => allocate_init Need Regs Y
@@ -1153,7 +1153,7 @@ bs_skip_utf32 Fail=f Ms=x Live=u Flags=u => \
 	bs_get_integer2 Fail Ms Live i=32 u=1 Flags x | \
 	i_bs_validate_unicode_retract Fail x Ms
 
-i_bs_validate_unicode_retract j s s
+i_bs_validate_unicode_retract j s S
 %hot
 
 #
@@ -1212,7 +1212,7 @@ bs_private_append Fail Size Unit Bin Flags Dst => \
 bs_init_writable
 
 i_bs_append j I t t s x
-i_bs_private_append j t s s x
+i_bs_private_append j t s S x
 
 #
 # Storing integers into binaries.
@@ -1498,7 +1498,9 @@ i_band s s j t d
 i_bor j I s s d
 i_bxor j I s s d
 
-i_int_bnot j s t d
+i_int_bnot Fail Src=c Live Dst => move Src x | i_int_bnot Fail x Live Dst
+
+i_int_bnot j S t d
 
 #
 # Old guard BIFs that creates heap fragments are no longer allowed.
-- 
cgit v1.2.3


From e54eef1aa71dc8edd75720fbd1be13266ad928ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 11 Sep 2017 07:09:40 +0200
Subject: Optimize operand type for match context in i_bs_get_integer

The match context is always in an X register. Change the 's' operand
to 'x'.

While we are it, also change the operands for Live and FlagsAndUnit
to 't' (they will both fit comfortably in 16 bits).
---
 erts/emulator/beam/ops.tab | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 14d2d39e0e..7b4462feda 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1083,7 +1083,7 @@ bs_get_integer2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \
 
 i_bs_get_integer_small_imm x W f t x
 i_bs_get_integer_imm x W t f t x
-i_bs_get_integer f I I s s x
+i_bs_get_integer f t t x s x
 i_bs_get_integer_8 x f x
 i_bs_get_integer_16 x f x
 
-- 
cgit v1.2.3


From 7471c87214aec6f28a9fba2bf0356b45ee717d33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 11 Sep 2017 14:44:45 +0200
Subject: Use 't' instead of 'I' bit syntax operands

The number of live registers, unit and flags, and the number
of slots all fit comortably in 16 bits. This change will give
more opportunities for packing.
---
 erts/emulator/beam/ops.tab | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 7b4462feda..32db287d89 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1064,13 +1064,13 @@ func_info M F A => i_func_info u M F A
 %warm
 bs_start_match2 Fail=f ica X Y D => jump Fail
 bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D
-i_bs_start_match2 xy f I I x
+i_bs_start_match2 xy f t t x
 
 bs_save2 Reg Index => gen_bs_save(Reg, Index)
-i_bs_save2 x I
+i_bs_save2 x t
 
 bs_restore2 Reg Index => gen_bs_restore(Reg, Index)
-i_bs_restore2 x I
+i_bs_restore2 x t
 
 # Matching integers
 bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val
@@ -1096,9 +1096,9 @@ bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \
 			gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
 
 i_bs_get_binary_imm2 f x t W t x
-i_bs_get_binary2 f x I s I x
-i_bs_get_binary_all2 f x I I x
-i_bs_get_binary_all_reuse x f I
+i_bs_get_binary2 f x t s t x
+i_bs_get_binary_all2 f x t t x
+i_bs_get_binary_all_reuse x f t
 
 # Fetching float from binaries.
 bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \
@@ -1106,7 +1106,7 @@ bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \
 
 bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail
 
-i_bs_get_float2 f x I s I x
+i_bs_get_float2 f x t s t x
 
 # Miscellanous
 
@@ -1114,8 +1114,8 @@ bs_skip_bits2 Fail=f Ms=x Sz=sq Unit=u Flags=u => \
 			gen_skip_bits2(Fail, Ms, Sz, Unit, Flags)
 
 i_bs_skip_bits_imm2 f x W
-i_bs_skip_bits2 f x xy I
-i_bs_skip_bits_all2 f x I
+i_bs_skip_bits2 f x xy t
+i_bs_skip_bits_all2 f x t
 
 bs_test_tail2 Fail=f Ms=x Bits=u==0 => bs_test_zero_tail2 Fail Ms
 bs_test_tail2 Fail=f Ms=x Bits=u => bs_test_tail_imm2 Fail Ms Bits
@@ -1123,7 +1123,7 @@ bs_test_zero_tail2 f x
 bs_test_tail_imm2 f x W
 
 bs_test_unit F Ms Unit=u==8 => bs_test_unit8 F Ms
-bs_test_unit f x I
+bs_test_unit f x t
 bs_test_unit8 f x
 
 # An y register operand for bs_context_to_binary is rare,
@@ -1144,7 +1144,7 @@ bs_skip_utf8 Fail=f Ms=x u u => i_bs_get_utf8 Ms Fail x
 bs_get_utf16 Fail=f Ms=x u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst
 bs_skip_utf16 Fail=f Ms=x u Flags=u => i_bs_get_utf16 Ms Fail Flags x
 
-i_bs_get_utf16 x f I x
+i_bs_get_utf16 x f t x
 
 bs_get_utf32 Fail=f Ms=x Live=u Flags=u Dst=d => \
 	bs_get_integer2 Fail Ms Live i=32 u=1 Flags Dst | \
-- 
cgit v1.2.3


From bcfa4564df41ad5806a4848dcb3be31fbfea6a9e Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 7 Sep 2017 17:26:25 +0200
Subject: erts: Fix leaking of fds in iovec_SUITE

---
 erts/emulator/test/iovec_SUITE.erl | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/iovec_SUITE.erl b/erts/emulator/test/iovec_SUITE.erl
index a5f605bfff..28df36d293 100644
--- a/erts/emulator/test/iovec_SUITE.erl
+++ b/erts/emulator/test/iovec_SUITE.erl
@@ -20,7 +20,7 @@
 
 -module(iovec_SUITE).
 
--export([all/0, suite/0]).
+-export([all/0, suite/0, init_per_suite/1, end_per_suite/1]).
 
 -export([integer_lists/1, binary_lists/1, empty_lists/1, empty_binary_lists/1,
          mixed_lists/1, improper_lists/1, illegal_lists/1, cons_bomb/1,
@@ -37,6 +37,13 @@ all() ->
      illegal_lists, improper_lists, cons_bomb, iolist_to_iovec_idempotence,
      iolist_to_iovec_correctness].
 
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(Config) ->
+    application:stop(os_mon),
+    Config.
+
 integer_lists(Config) when is_list(Config) ->
     Variations = gen_variations([I || I <- lists:seq(1, 255)]),
 
-- 
cgit v1.2.3


From a58daf5c38b4921c82008c811cd5dd178deef49f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 8 Sep 2017 14:07:41 +0200
Subject: Add built-in macros $ARG_POSITION() and $IS_PACKED()

---
 erts/emulator/utils/beam_makeops | 55 ++++++++++++++++++++++++++++++++--------
 1 file changed, 44 insertions(+), 11 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index e55d3eadb5..8e34e07035 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -267,6 +267,16 @@ if ($wordsize == 64) {
     $pack_mask[3] = ['BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK', $WHOLE_WORD];
 }
 
+#
+# Add placeholders for built-in macros.
+#
+
+$c_code{'IS_PACKED'} = ['$Expr',"built-in macro",('Expr')];
+$c_code{'ARG_POSITION'} = ['$Expr',"built-in macro",('Expr')];
+foreach my $name (keys %c_code) {
+    $c_code_used{$name} = 1;
+}
+
 #
 # Parse the input files.
 #
@@ -1382,14 +1392,8 @@ sub expand_all {
         my $keep = substr($code, 0, $-[0]);
         my $after = substr($code, $+[0]);
 
-        # Keep the special, pre-defined bindings.
-        my %new_bindings;
-        foreach my $key (qw(NEXT_INSTRUCTION)) {
-            $new_bindings{$key} = $bindings{$key};
-        }
-
         my $body;
-        ($body,$code) = expand_macro($macro_name, $after, \%new_bindings);
+        ($body,$code) = expand_macro($macro_name, $after, \%bindings);
         $res .= "$keep$body";
     }
 
@@ -1436,21 +1440,49 @@ sub expand_macro {
         $arg =~ s/^\s*//;
     }
 
-    # Now combine bindings from the parameter names and arguments.
-    my %bindings = %{$bindings_ref};
+    # Make sure that the number of arguments are correct.
     if (@vars != @args) {
         error("calling $name with ", scalar(@args),
               " arguments instead of expected ", scalar(@vars), " arguments...");
     }
+
+    # Now combine bindings from the parameter names and arguments.
+    my %bindings = %{$bindings_ref};
+    my %new_bindings;
+
+    # Keep the special, pre-defined bindings.
+    foreach my $key (qw(NEXT_INSTRUCTION)) {
+        $new_bindings{$key} = $bindings{$key};
+    }
+
     for (my $i = 0; $i < @vars; $i++) {
-        $bindings{$vars[$i]} = $args[$i];
+        my $arg = $args[$i];
+        $arg = eval { expand_all($arg, \%bindings) };
+        unless (defined $arg) {
+            warn $@;
+            die "... from the body of $name at $where\n";
+        }
+        $new_bindings{$vars[$i]} = $arg;
     }
 
-    $body = eval { expand_all($body, \%bindings) };
+    $body = eval { expand_all($body, \%new_bindings) };
     unless (defined $body) {
         warn $@;
         die "... from the body of $name at $where\n";
     }
+
+    # Handle built-in macros.
+    if ($name eq 'ARG_POSITION') {
+        if ($body =~ /^I\[(\d+)\]$/) {
+            $body = $1;
+        } else {
+            $body = 0;
+        }
+    } elsif ($name eq 'IS_PACKED') {
+        $body = ($body =~ /^I\[\d+\]$/) ? 0 : 1;
+    }
+
+    # Wrap body if needed and return resul.t
     $body = "do {\n$body\n} while (0)"
         if needs_do_wrapper($body);
     ($body,$rest);
@@ -1476,6 +1508,7 @@ sub needs_do_wrapper {
     return 0 if /^[A-Z_]*SWAPOUT/;
     return 0 if /^if\s*[(]/;
     return 0 if /^goto\b/;
+    return 0 if /^\d+/;
     return 1;                   # Not sure, say that it is needed.
 }
 
-- 
cgit v1.2.3


From 8650d6d811bb99490e1a75cbbab684a673a950ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sat, 2 Sep 2017 14:35:39 +0200
Subject: Refactor instructions to support relative jumps

Introduce new macros that can be used for relative jumps and
use them consistently.

Test that everything works by using a non-zero constant JUMP_OFFSET.
The loader subtracts JUMP_OFFSET from loaded labels, and all
instructions that use 'f' operands add it back.
---
 erts/emulator/beam/beam_load.c       |  8 ++++++--
 erts/emulator/beam/beam_load.h       |  2 ++
 erts/emulator/beam/bif_instrs.tab    | 14 +++++++-------
 erts/emulator/beam/instrs.tab        | 32 ++++++++++++++++++++++----------
 erts/emulator/beam/macros.tab        | 19 +++++++++++++++++--
 erts/emulator/beam/msg_instrs.tab    | 10 +++++-----
 erts/emulator/beam/select_instrs.tab |  8 ++++----
 erts/emulator/beam/trace_instrs.tab  |  2 +-
 8 files changed, 64 insertions(+), 31 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 7d3a19ff86..9b8f33f576 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4840,7 +4840,11 @@ freeze_code(LoaderState* stp)
 	    ASSERT(this_patch < stp->ci);
 	    next_patch = codev[this_patch];
 	    ASSERT(next_patch < stp->ci);
-	    codev[this_patch] = (BeamInstr) (codev + value);
+            if (this_patch < stp->num_functions) {
+                codev[this_patch] = (BeamInstr) (codev + value);
+            } else {
+                codev[this_patch] = (BeamInstr) (codev + value - JUMP_OFFSET);
+            }
 	    this_patch = next_patch;
 	}
     }
@@ -4885,7 +4889,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
     while (index != 0) {
 	BeamInstr next = codev[index];
 	codev[index] = BeamOpCode(op_catch_yf);
-	catches = beam_catches_cons((BeamInstr *)codev[index+2], catches);
+	catches = beam_catches_cons((BeamInstr *)codev[index+2]+JUMP_OFFSET, catches);
 	codev[index+2] = make_catch(catches);
 	index = next;
     }
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index c088bdb751..0b6543638f 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -21,6 +21,8 @@
 #ifndef _BEAM_LOAD_H
 #  define _BEAM_LOAD_H
 
+#define JUMP_OFFSET 0x5
+
 #include "beam_opcodes.h"
 #include "erl_process.h"
 
diff --git a/erts/emulator/beam/bif_instrs.tab b/erts/emulator/beam/bif_instrs.tab
index 3c95113907..0932b8b985 100644
--- a/erts/emulator/beam/bif_instrs.tab
+++ b/erts/emulator/beam/bif_instrs.tab
@@ -151,7 +151,7 @@ i_gc_bif1(Fail, Bif, Src, Live, Dst) {
         $NEXT0();
     }
     if (ERTS_LIKELY($Fail != 0)) { /* Handle error in guard. */
-        $NEXT($Fail);
+        $JUMP($Fail);
     }
 
     /* Handle error in body. */
@@ -202,7 +202,7 @@ i_gc_bif2(Fail, Bif, Live, Src1, Src2, Dst) {
     }
 
     if (ERTS_LIKELY($Fail != 0)) { /* Handle error in guard. */
-        $NEXT($Fail);
+        $JUMP($Fail);
     }
 
     /* Handle error in body. */
@@ -257,7 +257,7 @@ i_gc_bif3(Fail, Bif, Live, Src2, Src3, Dst) {
 
     /* Handle error in guard. */
     if (ERTS_LIKELY($Fail != 0)) {
-        $NEXT($Fail);
+        $JUMP($Fail);
     }
 
     /* Handle error in body. */
@@ -473,10 +473,10 @@ nif_bif.apply_bif() {
 
     /* In case we apply process_info/1,2 or load_nif/1 */
     c_p->current = codemfa;
-    c_p->i = I;		/* In case we apply check_process_code/2. */
-    c_p->arity = 0;		/* To allow garbage collection on ourselves
-                                 * (check_process_code/2).
-                                 */
+    $SET_CP_I_ABS(I);     /* In case we apply check_process_code/2. */
+    c_p->arity = 0;       /* To allow garbage collection on ourselves
+                           * (check_process_code/2).
+                           */
     DTRACE_BIF_ENTRY(c_p, codemfa);
 
     SWAPOUT;
diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index 690bd3a848..7ea9dee299 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -78,7 +78,14 @@ move_deallocate_return(Src, Deallocate) {
 
 // Call instructions
 
-DISPATCH(CallDest) {
+DISPATCH_REL(CallDest) {
+    //| -no_next
+    $SET_I_REL($CallDest);
+    DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
+    Dispatch();
+}
+
+DISPATCH_ABS(CallDest) {
     //| -no_next
     SET_I((BeamInstr *) $CallDest);
     DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
@@ -87,18 +94,18 @@ DISPATCH(CallDest) {
 
 i_call(CallDest) {
     SET_CP(c_p, $NEXT_INSTRUCTION);
-    $DISPATCH($CallDest);
+    $DISPATCH_REL($CallDest);
 }
 
 move_call(Src, CallDest) {
     x(0) = $Src;
     SET_CP(c_p, $NEXT_INSTRUCTION);
-    $DISPATCH($CallDest);
+    $DISPATCH_REL($CallDest);
 }
 
 i_call_last(CallDest, Deallocate) {
     $deallocate($Deallocate);
-    $DISPATCH($CallDest);
+    $DISPATCH_REL($CallDest);
 }
 
 move_call_last(Src, CallDest, Deallocate) {
@@ -107,7 +114,7 @@ move_call_last(Src, CallDest, Deallocate) {
 }
 
 i_call_only(CallDest) {
-    $DISPATCH($CallDest);
+    $DISPATCH_REL($CallDest);
 }
 
 move_call_only(Src, CallDest) {
@@ -168,7 +175,8 @@ i_apply() {
     BeamInstr *next;
     $APPLY(NULL, 0, next);
     if (ERTS_LIKELY(next != NULL)) {
-        $i_call(next);
+        SET_CP(c_p, $NEXT_INSTRUCTION);
+        $DISPATCH_ABS(next);
     }
     $HANDLE_APPLY_ERROR();
 }
@@ -177,7 +185,8 @@ i_apply_last(Deallocate) {
     BeamInstr *next;
     $APPLY(I, $Deallocate, next);
     if (ERTS_LIKELY(next != NULL)) {
-        $i_call_last(next, $Deallocate);
+        $deallocate($Deallocate);
+        $DISPATCH_ABS(next);
     }
     $HANDLE_APPLY_ERROR();
 }
@@ -186,7 +195,7 @@ i_apply_only() {
     BeamInstr *next;
     $APPLY(I, 0, next);
     if (ERTS_LIKELY(next != NULL)) {
-        $i_call_only(next);
+        $DISPATCH_ABS(next);
     }
     $HANDLE_APPLY_ERROR();
 }
@@ -202,7 +211,8 @@ apply(Arity) {
     BeamInstr *next;
     $FIXED_APPLY($Arity, NULL, 0, next);
     if (ERTS_LIKELY(next != NULL)) {
-        $i_call(next);
+        SET_CP(c_p, $NEXT_INSTRUCTION);
+        $DISPATCH_ABS(next);
     }
     $HANDLE_APPLY_ERROR();
 }
@@ -211,7 +221,8 @@ apply_last(Arity, Deallocate) {
     BeamInstr *next;
     $FIXED_APPLY($Arity, I, $Deallocate, next);
     if (ERTS_LIKELY(next != NULL)) {
-        $i_call_last(next, $Deallocate);
+        $deallocate($Deallocate);
+        $DISPATCH_ABS(next);
     }
     $HANDLE_APPLY_ERROR();
 }
@@ -560,6 +571,7 @@ i_put_tuple.fill(Arity) {
         }
     } while (--arity != 0);
     HTOP = hp;
+    ASSERT(VALID_INSTR(* (Eterm *)I));
     Goto(*I);
 }
 
diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab
index bac96be7d3..959f7b28df 100644
--- a/erts/emulator/beam/macros.tab
+++ b/erts/emulator/beam/macros.tab
@@ -28,15 +28,30 @@ REFRESH_GEN_DEST() {
     dst_ptr = REG_TARGET_PTR(dst);
 }
 
+SET_I_REL(Offset) {
+    ASSERT(VALID_INSTR(* (Eterm *)((BeamInstr *) ($Offset) + JUMP_OFFSET)));
+    I = (BeamInstr *) $Offset + JUMP_OFFSET;
+}
+
+SET_CP_I_ABS(Target) {
+    c_p->i = $Target;
+    ASSERT(VALID_INSTR(* (Eterm *)c_p->i));
+}
+
+SET_REL_I(Dst, Offset) {
+    $Dst = (BeamInstr *) ($Offset) + JUMP_OFFSET;
+    ASSERT(VALID_INSTR(*$Dst));
+}
+
 FAIL(Fail) {
     //| -no_prefetch
-    SET_I((BeamInstr *) $Fail);
+    $SET_I_REL($Fail);
     Goto(*I);
 }
 
 JUMP(Fail) {
     //| -no_next
-    SET_I((BeamInstr *) $Fail);
+    $SET_I_REL($Fail);
     Goto(*I);
 }
 
diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab
index 093d48c64c..8055a8616f 100644
--- a/erts/emulator/beam/msg_instrs.tab
+++ b/erts/emulator/beam/msg_instrs.tab
@@ -49,7 +49,7 @@ recv_mark(Dest) {
      * the label for the loop_rec/2 instruction for the
      * the receive statement.
      */
-    c_p->msg.mark = (BeamInstr *) $Dest;
+    $SET_REL_I(c_p->msg.mark, $Dest);
     c_p->msg.saved_last = c_p->msg.last;
 }
 
@@ -116,7 +116,7 @@ i_loop_rec(Dest) {
             erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
         } else {
             c_p->flags &= ~F_DELAY_GC;
-            SET_I((BeamInstr *) $Dest);
+            $SET_I_REL($Dest);
             Goto(*I);		/* Jump to a wait or wait_timeout instruction */
         }
     }
@@ -253,7 +253,7 @@ loop_rec_end(Dest) {
 
     ASSERT(c_p->flags & F_DELAY_GC);
 
-    SET_I((BeamInstr *) $Dest);
+    $SET_I_REL($Dest);
     SAVE_MESSAGE(c_p);
     if (FCALLS > 0 || FCALLS > neg_o_reds) {
         FCALLS--;
@@ -261,7 +261,7 @@ loop_rec_end(Dest) {
     }
 
     c_p->flags &= ~F_DELAY_GC;
-    c_p->i = I;
+    $SET_CP_I_ABS(I);
     SWAPOUT;
     c_p->arity = 0;
     c_p->current = NULL;
@@ -374,7 +374,7 @@ wait.src(Src) {
 //
 
 wait.execute(JumpTarget) {
-    c_p->i = (BeamInstr *) $JumpTarget; /* L1 */
+    $SET_REL_I(c_p->i, $JumpTarget); /* L1 */
     SWAPOUT;
     c_p->arity = 0;
 
diff --git a/erts/emulator/beam/select_instrs.tab b/erts/emulator/beam/select_instrs.tab
index e85ed2c304..88049bbcf0 100644
--- a/erts/emulator/beam/select_instrs.tab
+++ b/erts/emulator/beam/select_instrs.tab
@@ -68,10 +68,10 @@ select_val_bins.select(Fail, NumElements) {
         } else if (select_val > mid->val) {
             low = mid + 1;
         } else {
-            $NEXT(mid->addr);
+            $JUMP(mid->addr);
         }
     }
-    $NEXT($Fail);
+    $JUMP($Fail);
 }
 
 i_select_tuple_arity2 := select_val2.src.ta_fail.execute;
@@ -150,8 +150,8 @@ select_val_lin.execute(N) {
     }
 
     if (vs[ix] == select_val) {
-        I = $NEXT_INSTRUCTION + $N + ix;
-        $JUMP(*I);
+        Eterm offset = *($NEXT_INSTRUCTION + $N + ix);
+        $JUMP(offset);
     } else {
         $JUMP(*select_fail);
     }
diff --git a/erts/emulator/beam/trace_instrs.tab b/erts/emulator/beam/trace_instrs.tab
index c28bc8ebcb..b10442c5e7 100644
--- a/erts/emulator/beam/trace_instrs.tab
+++ b/erts/emulator/beam/trace_instrs.tab
@@ -94,7 +94,7 @@ i_yield() {
     c_p->arg_reg[0] = am_true;
     c_p->arity = 1; /* One living register (the 'true' return value) */
     SWAPOUT;
-    c_p->i = $NEXT_INSTRUCTION;
+    $SET_CP_I_ABS($NEXT_INSTRUCTION);
     c_p->current = NULL;
     goto do_schedule;
     //| -no_next
-- 
cgit v1.2.3


From c128db95440cc4caf34cbad5e91e1e26bb5fa6a3 Mon Sep 17 00:00:00 2001
From: Glauber Campinho <glauber.campinho@soundcloud.com>
Date: Fri, 1 Sep 2017 15:12:48 +0200
Subject: Fix del_chars not considering wide chars and update buffer length
 before calling write_buf

After deleting the chars the function `del_chars` was considering the code points to move the cursor back and not the graphemes
---
 erts/emulator/drivers/unix/ttsl_drv.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/unix/ttsl_drv.c b/erts/emulator/drivers/unix/ttsl_drv.c
index 3d007173c4..2a508b02eb 100644
--- a/erts/emulator/drivers/unix/ttsl_drv.c
+++ b/erts/emulator/drivers/unix/ttsl_drv.c
@@ -936,10 +936,10 @@ static int put_chars(byte *s, int l)
     int n;
 
     n = insert_buf(s, l);
+    if (lpos > llen)
+        llen = lpos;
     if (n > 0)
       write_buf(lbuf + lpos - n, n);
-    if (lpos > llen)
-      llen = lpos;
     return TRUE;
 }
 
@@ -1016,7 +1016,7 @@ static int del_chars(int n)
 	   outc(' ');
 	   move_left(1);
 	}
-	move_cursor(llen + l, lpos);
+	move_cursor(llen + gcs, lpos);
     }
     else if (pos < lpos) {
 	l = lpos - pos;		/* Buffer characters */
@@ -1036,7 +1036,7 @@ static int del_chars(int n)
           outc(' ');
           move_left(1);
 	}
-        move_cursor(llen + l, lpos);
+        move_cursor(llen + gcs, lpos);
     }
     return TRUE;
 }
@@ -1289,7 +1289,7 @@ static int cp_pos_to_col(int cp_pos)
 
     if (cp_pos > llen) {
         col += cp_pos - llen;
-	cp_pos = llen;
+        cp_pos = llen;
     }
 
     while (i < cp_pos) {
-- 
cgit v1.2.3


From e90609f5e3e08cfcb88baa01189889fad302bbf7 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 13 Sep 2017 15:40:17 +0200
Subject: erts: Fix 'on_load' tracing bug for modules with -on_load

Symptom: VM crash when erlang:trace_pattern(on_load, ..) is set and
module with -on_load is loaded.

Problem: Tracing and -on_load clash in their use of Export.beam[1]

Solution: Do not do call set_default_trace_pattern in finish_loading_1
for modules with -on_load. finish_after_on_load_2 will do that anyway.
---
 erts/emulator/beam/beam_bif_load.c | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index b664532c1c..8fc8613c66 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -242,7 +242,7 @@ struct m {
     Binary* code;
     Eterm module;
     Module* modp;
-    Uint exception;
+    Eterm exception;
 };
 
 static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int);
@@ -263,7 +263,7 @@ exception_list(Process* p, Eterm tag, struct m* mp, Sint exceptions)
     Eterm res = NIL;
 
     while (exceptions > 0) {
-	if (mp->exception) {
+	if (is_value(mp->exception)) {
 	    res = CONS(hp, mp->module, res);
 	    hp += 2;
 	    exceptions--;
@@ -366,9 +366,9 @@ finish_loading_1(BIF_ALIST_1)
 
     exceptions = 0;
     for (i = 0; i < n; i++) {
-	p[i].exception = 0;
+	p[i].exception = THE_NON_VALUE;
 	if (p[i].modp->seen) {
-	    p[i].exception = 1;
+	    p[i].exception = am_duplicated;
 	    exceptions++;
 	}
 	p[i].modp->seen = 1;
@@ -401,9 +401,9 @@ finish_loading_1(BIF_ALIST_1)
 
     exceptions = 0;
     for (i = 0; i < n; i++) {
-	p[i].exception = 0;
+	p[i].exception = THE_NON_VALUE;
 	if (p[i].modp->curr.code_hdr && p[i].modp->old.code_hdr) {
-	    p[i].exception = 1;
+	    p[i].exception = am_not_purged;
 	    exceptions++;
 	}
     }
@@ -424,7 +424,7 @@ finish_loading_1(BIF_ALIST_1)
 	    retval = erts_finish_loading(p[i].code, BIF_P, 0, &mod);
 	    ASSERT(retval == NIL || retval == am_on_load);
 	    if (retval == am_on_load) {
-		p[i].exception = 1;
+		p[i].exception = am_on_load;
 		exceptions++;
 	    }
 	}
@@ -456,8 +456,9 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
 	    erts_commit_staging_code_ix();
 	    if (loaded) {
 		int i;
-		for (i=0; i < nloaded; i++) {		
-		    set_default_trace_pattern(loaded[i].module);
+		for (i=0; i < nloaded; i++) {
+		    if (loaded[i].exception != am_on_load)
+			set_default_trace_pattern(loaded[i].module);
 		}
 	    }
 	}
-- 
cgit v1.2.3


From ef43e32b450df2fdf1c731e9be03693fb0b69e31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 4 Sep 2017 15:39:58 +0200
Subject: Remove JUMP_OFFSET

It has served its purpose.
---
 erts/emulator/beam/beam_load.c | 8 ++------
 erts/emulator/beam/beam_load.h | 2 --
 erts/emulator/beam/macros.tab  | 6 +++---
 3 files changed, 5 insertions(+), 11 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 9b8f33f576..7d3a19ff86 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4840,11 +4840,7 @@ freeze_code(LoaderState* stp)
 	    ASSERT(this_patch < stp->ci);
 	    next_patch = codev[this_patch];
 	    ASSERT(next_patch < stp->ci);
-            if (this_patch < stp->num_functions) {
-                codev[this_patch] = (BeamInstr) (codev + value);
-            } else {
-                codev[this_patch] = (BeamInstr) (codev + value - JUMP_OFFSET);
-            }
+	    codev[this_patch] = (BeamInstr) (codev + value);
 	    this_patch = next_patch;
 	}
     }
@@ -4889,7 +4885,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
     while (index != 0) {
 	BeamInstr next = codev[index];
 	codev[index] = BeamOpCode(op_catch_yf);
-	catches = beam_catches_cons((BeamInstr *)codev[index+2]+JUMP_OFFSET, catches);
+	catches = beam_catches_cons((BeamInstr *)codev[index+2], catches);
 	codev[index+2] = make_catch(catches);
 	index = next;
     }
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 0b6543638f..c088bdb751 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -21,8 +21,6 @@
 #ifndef _BEAM_LOAD_H
 #  define _BEAM_LOAD_H
 
-#define JUMP_OFFSET 0x5
-
 #include "beam_opcodes.h"
 #include "erl_process.h"
 
diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab
index 959f7b28df..35460da1f4 100644
--- a/erts/emulator/beam/macros.tab
+++ b/erts/emulator/beam/macros.tab
@@ -29,8 +29,8 @@ REFRESH_GEN_DEST() {
 }
 
 SET_I_REL(Offset) {
-    ASSERT(VALID_INSTR(* (Eterm *)((BeamInstr *) ($Offset) + JUMP_OFFSET)));
-    I = (BeamInstr *) $Offset + JUMP_OFFSET;
+    ASSERT(VALID_INSTR(* (Eterm *)((BeamInstr *) ($Offset))));
+    I = (BeamInstr *) $Offset;
 }
 
 SET_CP_I_ABS(Target) {
@@ -39,7 +39,7 @@ SET_CP_I_ABS(Target) {
 }
 
 SET_REL_I(Dst, Offset) {
-    $Dst = (BeamInstr *) ($Offset) + JUMP_OFFSET;
+    $Dst = (BeamInstr *) ($Offset);
     ASSERT(VALID_INSTR(*$Dst));
 }
 
-- 
cgit v1.2.3


From 76b4ff5c5db9610ecd09b11d3816f25b1e6b58f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 5 Sep 2017 08:30:30 +0200
Subject: Add information about offset to common group start position

---
 erts/emulator/beam/erl_vm.h      | 1 +
 erts/emulator/utils/beam_makeops | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index f2d0af64df..076767c7cd 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -159,6 +159,7 @@ typedef struct op_entry {
    Uint32 mask[3];		/* Signature mask. */
    unsigned involves_r;		/* Needs special attention when matching. */
    int sz;			/* Number of loaded words. */
+   int adjust;                  /* Adjustment for start of instruction. */
    char* pack;			/* Instructions for packing engine. */
    char* sign;			/* Signature string. */
 } OpEntry;
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index e55d3eadb5..ccbd0dbaf4 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -80,6 +80,7 @@ my %gen_opnum;
 my %num_specific;
 my %gen_to_spec;
 my %specific_op;
+my %group_size;                 # Group size for specific operators.
 
 my %gen_arity;
 my @gen_arity;
@@ -623,7 +624,11 @@ sub emulator_output {
 		$sep = ",";
 	    }
 	    $init .= "}";
-	    init_item($print_name, $init, $involves_r, $size, $pack, $sign);
+            my $adj = 0;
+            if (defined $group_size{$print_name}) {
+                $adj = $size - $group_size{$print_name};
+            }
+	    init_item($print_name, $init, $involves_r, $size, $adj, $pack, $sign);
 	    $op_to_name[$spec_opnum] = $instr;
 	    $spec_opnum++;
 	}
@@ -1171,6 +1176,7 @@ sub combine_instruction_group {
             if ($opcase ne '') {
                 $gcode .= "OpCase($opcase):\n";
                 push @opcase_labels, $opcase;
+                $group_size{$opcase} = $group_size + 1;
             }
             if ($num_references{$label}) {
                 $gcode .= "$label:\n";
-- 
cgit v1.2.3


From df1f7395a206d1fd4e70b7380fccdbd53540db40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 5 Sep 2017 08:33:02 +0200
Subject: Use relative failure labels

Relative failure in itself is not an optimization, but we plan to
pack failure labels in the future to save memory.
---
 erts/emulator/beam/beam_debug.c      |  45 +++++++++-----
 erts/emulator/beam/beam_load.c       | 116 ++++++++++++++++++++++++-----------
 erts/emulator/beam/macros.tab        |   8 +--
 erts/emulator/beam/select_instrs.tab |   4 +-
 4 files changed, 115 insertions(+), 58 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 21e295c63a..0ff4ed39b4 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -53,6 +53,7 @@ void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg);
 
 static int print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr);
 static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif);
+static BeamInstr* f_to_addr(BeamInstr* base, int op, BeamInstr* ap);
 
 BIF_RETTYPE
 erts_debug_same_2(BIF_ALIST_2)
@@ -558,9 +559,10 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	    break;
 	case 'f':		/* Destination label */
 	    {
-		ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap);
-		if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) {
-		    erts_print(to, to_arg, "f(" HEXF ")", *ap);
+                BeamInstr* target = f_to_addr(addr, op, ap);
+		ErtsCodeMFA* cmfa = find_function_from_pc(target);
+		if (!cmfa || erts_codemfa_to_code(cmfa) != target) {
+		    erts_print(to, to_arg, "f(" HEXF ")", target);
 		} else {
 		    erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module,
                                cmfa->function, cmfa->arity);
@@ -570,18 +572,18 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	    break;
 	case 'p':		/* Pointer (to label) */
 	    {
-		ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap);
-		if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) {
-		    erts_print(to, to_arg, "p(" HEXF ")", *ap);
-		} else {
-		    erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module,
-                               cmfa->function, cmfa->arity);
-		}
+                BeamInstr* target = f_to_addr(addr, op, ap);
+                erts_print(to, to_arg, "p(" HEXF ")", target);
 		ap++;
 	    }
 	    break;
 	case 'j':		/* Pointer (to label) */
-	    erts_print(to, to_arg, "j(" HEXF ")", *ap);
+            if (*ap == 0) {
+                erts_print(to, to_arg, "j(0)");
+            } else {
+                BeamInstr* target = f_to_addr(addr, op, ap);
+                erts_print(to, to_arg, "j(" HEXF ")", target);
+            }
 	    ap++;
 	    break;
 	case 'e':		/* Export entry */
@@ -638,7 +640,8 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	    }
 	    ix = n;
 	    while (ix--) {
-		erts_print(to, to_arg, "f(" HEXF ") ", (Eterm) ap[0]);
+                BeamInstr* target = f_to_addr(addr, op, ap);
+		erts_print(to, to_arg, "f(" HEXF ") ", target);
 		ap++;
 		size++;
 	    }
@@ -650,7 +653,8 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	    int n = ap[-1];
 
 	    while (n > 0) {
-		erts_print(to, to_arg, "%T f(" HEXF ") ", (Eterm) ap[0], ap[1]);
+                BeamInstr* target = f_to_addr(addr, op, ap+1);
+		erts_print(to, to_arg, "%T f(" HEXF ") ", (Eterm) ap[0], target);
 		ap += 2;
 		size += 2;
 		n--;
@@ -675,7 +679,8 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
             size++;
             ix = n;
             while (ix--) {
-                erts_print(to, to_arg, "f(" HEXF ") ", ap[0]);
+                BeamInstr* target = f_to_addr(addr, op, ap);
+                erts_print(to, to_arg, "f(" HEXF ") ", target);
                 ap++;
                 size++;
             }
@@ -686,7 +691,8 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	{
 	    int n;
 	    for (n = ap[-2]; n > 0; n--) {
-		erts_print(to, to_arg, "f(" HEXF ") ", ap[0]);
+                BeamInstr* target = f_to_addr(addr, op, ap);
+		erts_print(to, to_arg, "f(" HEXF ") ", target);
 		ap++;
 		size++;
 	    }
@@ -697,7 +703,8 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	{
 	    int n;
 	    for (n = ap[-1]; n > 0; n--) {
-		erts_print(to, to_arg, "f(" HEXF ") ", ap[0]);
+                BeamInstr* target = f_to_addr(addr, op, ap);
+		erts_print(to, to_arg, "f(" HEXF ") ", target);
 		ap++;
 		size++;
 	    }
@@ -796,6 +803,12 @@ static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif)
     }
 }
 
+static BeamInstr* f_to_addr(BeamInstr* base, int op, BeamInstr* ap)
+{
+    return base - 1 + opc[op].adjust + *ap;
+}
+
+
 /*
  * Dirty BIF testing.
  *
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 7d3a19ff86..2888158c7b 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -81,15 +81,26 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int);
 #define TE_FAIL (-1)
 #define TE_SHORT_WINDOW (-2)
 
+/*
+ * Type for a reference to a label that must be patched.
+ */
+
 typedef struct {
-    Uint value;			/* Value of label (NULL if not known yet). */
-    Sint patches;		/* Index (into code buffer) to first
-				 * location which must be patched with
-				 * the value of this label.
-				 */
+    Uint pos;                   /* Position of label reference to patch. */
+} LabelPatch;
+
+/*
+ * Type for a label.
+ */
+
+typedef struct {
+    Uint value;			/* Value of label (0 if not known yet). */
     Uint looprec_targeted;	/* Non-zero if this label is the target of a loop_rec
 				 * instruction.
 				 */
+    LabelPatch* patches;       /* Array of label patches. */
+    Uint num_patches;           /* Number of patches in array. */
+    Uint num_allocated;         /* Number of allocated patches. */
 } Label;
 
 /*
@@ -507,6 +518,7 @@ static int read_lambda_table(LoaderState* stp);
 static int read_literal_table(LoaderState* stp);
 static int read_line_table(LoaderState* stp);
 static int read_code_header(LoaderState* stp);
+static void init_label(Label* lp);
 static int load_code(LoaderState* stp);
 static GenOp* gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index,
 			  GenOpArg Tuple, GenOpArg Dst);
@@ -1051,6 +1063,10 @@ loader_state_dtor(Binary* magic)
         stp->codev = 0;
     }
     if (stp->labels != 0) {
+        Uint num;
+        for (num = 0; num < stp->num_labels; num++) {
+            erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels[num].patches);
+        }
 	erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels);
 	stp->labels = 0;
     }
@@ -1534,7 +1550,7 @@ read_export_table(LoaderState* stp)
 	 * any other functions that walk through all local functions.
 	 */
 
-	if (stp->labels[n].patches >= 0) {
+	if (stp->labels[n].num_patches > 0) {
 	    LoadError3(stp, "there are local calls to the stub for "
 		       "the BIF %T:%T/%d",
 		       stp->module, func, arity);
@@ -1880,9 +1896,7 @@ read_code_header(LoaderState* stp)
     stp->labels = (Label *) erts_alloc(ERTS_ALC_T_PREPARED_CODE,
 				       stp->num_labels * sizeof(Label));
     for (i = 0; i < stp->num_labels; i++) {
-	stp->labels[i].value = 0;
-	stp->labels[i].patches = -1;
-	stp->labels[i].looprec_targeted = 0;
+        init_label(&stp->labels[i]);
     }
 
     stp->catches = 0;
@@ -1911,12 +1925,40 @@ read_code_header(LoaderState* stp)
     
 #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm))))
 
+static void init_label(Label* lp)
+{
+    lp->value = 0;
+    lp->looprec_targeted = 0;
+    lp->num_patches = 0;
+    lp->num_allocated = 4;
+    lp->patches = erts_alloc(ERTS_ALC_T_PREPARED_CODE,
+                             lp->num_allocated * sizeof(LabelPatch));
+}
+
+static void
+register_label_patch(LoaderState* stp, Uint label, Uint ci, Uint offset)
+{
+    Label* lp;
+
+    ASSERT(label < stp->num_labels);
+    lp = &stp->labels[label];
+    if (lp->num_allocated <= lp->num_patches) {
+        lp->num_allocated *= 2;
+        lp->patches = erts_realloc(ERTS_ALC_T_PREPARED_CODE,
+                                   (void *) lp->patches,
+                                   lp->num_allocated * sizeof(LabelPatch));
+    }
+    lp->patches[lp->num_patches++].pos = ci;
+    stp->codev[ci] = offset;
+}
+
 static int
 load_code(LoaderState* stp)
 {
     int i;
-    int ci;
-    int last_func_start = 0;	/* Needed by nif loading and line instructions */
+    Uint ci;
+    Uint last_instr_start;       /* Needed for relative jumps */
+    Uint last_func_start = 0;	/* Needed by nif loading and line instructions */
     char* sign;
     int arg;			/* Number of current argument. */
     int num_specific;		/* Number of specific ops for current. */
@@ -2272,6 +2314,7 @@ load_code(LoaderState* stp)
 
 	    stp->specific_op = specific;
 	    CodeNeed(opc[stp->specific_op].sz+16); /* Extra margin for packing */
+            last_instr_start = ci + opc[stp->specific_op].adjust;
 	    code[ci++] = BeamOpCode(stp->specific_op);
 	}
 	
@@ -2401,16 +2444,14 @@ load_code(LoaderState* stp)
 		break;
 	    case 'f':		/* Destination label */
 		VerifyTag(stp, tag_to_letter[tag], *sign);
-		code[ci] = stp->labels[tmp_op->a[arg].val].patches;
-		stp->labels[tmp_op->a[arg].val].patches = ci;
+                register_label_patch(stp, tmp_op->a[arg].val, ci, -last_instr_start);
 		ci++;
 		break;
 	    case 'j':		/* 'f' or 'p' */
 		if (tag == TAG_p) {
 		    code[ci] = 0;
 		} else if (tag == TAG_f) {
-		    code[ci] = stp->labels[tmp_op->a[arg].val].patches;
-		    stp->labels[tmp_op->a[arg].val].patches = ci;
+                    register_label_patch(stp, tmp_op->a[arg].val, ci, -last_instr_start);
 		} else {
 		    LoadError3(stp, "bad tag %d; expected %d or %d",
 			       tag, TAG_f, TAG_p);
@@ -2430,7 +2471,6 @@ load_code(LoaderState* stp)
 		    LoadError1(stp, "label %d defined more than once", last_label);
 		}
 		stp->labels[last_label].value = ci;
-		ASSERT(stp->labels[last_label].patches < ci);
 		break;
 	    case 'e':		/* Export entry */
 		VerifyTag(stp, tag, TAG_u);
@@ -2555,8 +2595,7 @@ load_code(LoaderState* stp)
 		break;
 	    case TAG_f:
 		CodeNeed(1);
-		code[ci] = stp->labels[tmp_op->a[arg].val].patches;
-		stp->labels[tmp_op->a[arg].val].patches = ci;
+                register_label_patch(stp, tmp_op->a[arg].val, ci, -last_instr_start);
 		ci++;
 		break;
 	    case TAG_x:
@@ -2628,17 +2667,16 @@ load_code(LoaderState* stp)
                    the size of the ops.tab i_func_info instruction is not
                    the same as FUNC_INFO_SZ */
 		ASSERT(stp->labels[last_label].value == ci - FUNC_INFO_SZ);
-		stp->hdr->functions[function_number] = (ErtsCodeInfo*) stp->labels[last_label].patches;
 		offset = function_number;
-		stp->labels[last_label].patches = offset;
+                register_label_patch(stp, last_label, offset, 0);
 		function_number++;
 		if (stp->arity > MAX_ARG) {
 		    LoadError1(stp, "too many arguments: %d", stp->arity);
 		}
 #ifdef DEBUG
-		ASSERT(stp->labels[0].patches < 0); /* Should not be referenced. */
+		ASSERT(stp->labels[0].num_patches == 0); /* Should not be referenced. */
 		for (i = 1; i < stp->num_labels; i++) {
-		    ASSERT(stp->labels[i].patches < ci);
+                    ASSERT(stp->labels[i].num_patches <= stp->labels[i].num_allocated);
 		}
 #endif
 	    }
@@ -4827,21 +4865,25 @@ freeze_code(LoaderState* stp)
      */
 
     for (i = 0; i < stp->num_labels; i++) {
-	Sint this_patch;
-	Sint next_patch;
+	Uint patch;
 	Uint value = stp->labels[i].value;
-	
-	if (value == 0 && stp->labels[i].patches >= 0) {
+
+	if (value == 0 && stp->labels[i].num_patches != 0) {
 	    LoadError1(stp, "label %d not resolved", i);
 	}
 	ASSERT(value < stp->ci);
-	this_patch = stp->labels[i].patches;
-	while (this_patch >= 0) {
-	    ASSERT(this_patch < stp->ci);
-	    next_patch = codev[this_patch];
-	    ASSERT(next_patch < stp->ci);
-	    codev[this_patch] = (BeamInstr) (codev + value);
-	    this_patch = next_patch;
+        for (patch = 0; patch < stp->labels[i].num_patches; patch++) {
+            Uint pos = stp->labels[i].patches[patch].pos;
+	    ASSERT(pos < stp->ci);
+            if (pos < stp->num_functions) {
+                /*
+                 * This is the array of pointers to the beginning of
+                 * each function. The pointers must remain absolute.
+                 */
+                codev[pos] = (BeamInstr) (codev + value);
+            } else {
+                codev[pos] += value;
+            }
 	}
     }
     CHKBLK(ERTS_ALC_T_CODE,code_hdr);
@@ -4884,8 +4926,11 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
     catches = BEAM_CATCHES_NIL;
     while (index != 0) {
 	BeamInstr next = codev[index];
+        BeamInstr* abs_addr;
 	codev[index] = BeamOpCode(op_catch_yf);
-	catches = beam_catches_cons((BeamInstr *)codev[index+2], catches);
+        /* We must make the address of the label absolute again. */
+        abs_addr = (BeamInstr *)codev + index + codev[index+2];
+	catches = beam_catches_cons(abs_addr, catches);
 	codev[index+2] = make_catch(catches);
 	index = next;
     }
@@ -5573,8 +5618,7 @@ new_label(LoaderState* stp)
     stp->labels = (Label *) erts_realloc(ERTS_ALC_T_PREPARED_CODE,
 					 (void *) stp->labels,
 					 stp->num_labels * sizeof(Label));
-    stp->labels[num].value = 0;
-    stp->labels[num].patches = -1;
+    init_label(&stp->labels[num]);
     return num;
 }
 
diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab
index 35460da1f4..debeafa651 100644
--- a/erts/emulator/beam/macros.tab
+++ b/erts/emulator/beam/macros.tab
@@ -29,17 +29,17 @@ REFRESH_GEN_DEST() {
 }
 
 SET_I_REL(Offset) {
-    ASSERT(VALID_INSTR(* (Eterm *)((BeamInstr *) ($Offset))));
-    I = (BeamInstr *) $Offset;
+    ASSERT(VALID_INSTR(*(I + ($Offset))));
+    I += $Offset;
 }
 
 SET_CP_I_ABS(Target) {
     c_p->i = $Target;
-    ASSERT(VALID_INSTR(* (Eterm *)c_p->i));
+    ASSERT(VALID_INSTR(*c_p->i));
 }
 
 SET_REL_I(Dst, Offset) {
-    $Dst = (BeamInstr *) ($Offset);
+    $Dst = I + ($Offset);
     ASSERT(VALID_INSTR(*$Dst));
 }
 
diff --git a/erts/emulator/beam/select_instrs.tab b/erts/emulator/beam/select_instrs.tab
index 88049bbcf0..da6b7dbe62 100644
--- a/erts/emulator/beam/select_instrs.tab
+++ b/erts/emulator/beam/select_instrs.tab
@@ -32,7 +32,7 @@ select_val_bins.fetch(Src) {
 select_val_bins.select(Fail, NumElements) {
     struct Pairs {
         BeamInstr val;
-        BeamInstr* addr;
+        Eterm offset;
     };
     struct Pairs* low;
     struct Pairs* high;
@@ -68,7 +68,7 @@ select_val_bins.select(Fail, NumElements) {
         } else if (select_val > mid->val) {
             low = mid + 1;
         } else {
-            $JUMP(mid->addr);
+            $JUMP(mid->offset);
         }
     }
     $JUMP($Fail);
-- 
cgit v1.2.3


From be8fb5a57d1a30c203e79ae2baf9e541226020d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 8 Sep 2017 05:18:33 +0200
Subject: Make sure that mask literals are 64 bits

Use the "ull" suffix for the mask literals instead of "ul"
to ensure that the literals are 64 bits also on Windows.
---
 erts/emulator/utils/beam_makeops | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index ccbd0dbaf4..6248583b1c 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -708,9 +708,9 @@ sub emulator_output {
 	print "#if !defined(ARCH_64)\n";
 	print qq[  #error "64-bit architecture assumed, but ARCH_64 not defined"\n];
 	print "#endif\n";
-	print "#define BEAM_WIDE_MASK 0xFFFFFFFFUL\n";
-	print "#define BEAM_LOOSE_MASK 0xFFFFUL\n";
-	print "#define BEAM_TIGHT_MASK 0xFFFFUL\n";
+	print "#define BEAM_WIDE_MASK 0xFFFFFFFFull\n";
+	print "#define BEAM_LOOSE_MASK 0xFFFFull\n";
+	print "#define BEAM_TIGHT_MASK 0xFFFFull\n";
 	print "#define BEAM_WIDE_SHIFT 32\n";
 	print "#define BEAM_LOOSE_SHIFT 16\n";
 	print "#define BEAM_TIGHT_SHIFT 16\n";
-- 
cgit v1.2.3


From 41b832fd133dd2bfebe858c42ae51db8f0ac0584 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 6 Sep 2017 04:26:47 +0200
Subject: Implement packing of 'f' and 'j'

---
 erts/emulator/beam/beam_debug.c  |   4 +-
 erts/emulator/beam/beam_emu.c    |   2 +
 erts/emulator/beam/beam_load.c   | 142 ++++++++++++++++++++++++++++++++++-----
 erts/emulator/utils/beam_makeops |  31 ++++++++-
 4 files changed, 160 insertions(+), 19 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 0ff4ed39b4..39e96126bf 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -425,7 +425,9 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	while (start_prog < prog) {
 	    prog--;
 	    switch (*prog) {
+	    case 'f':
 	    case 'g':
+	    case 'q':
 		*ap++ = *--sp;
 		break;
 	    case 'i':		/* Initialize packing accumulator. */
@@ -805,7 +807,7 @@ static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif)
 
 static BeamInstr* f_to_addr(BeamInstr* base, int op, BeamInstr* ap)
 {
-    return base - 1 + opc[op].adjust + *ap;
+    return base - 1 + opc[op].adjust + (Sint32) *ap;
 }
 
 
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 55990362ff..1baf3ff7ab 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -235,6 +235,8 @@ void** beam_ops;
      ERTS_UNREQ_PROC_MAIN_LOCK((P))
 
 #define db(N) (N)
+#define fb(N) ((Sint)(Sint32)(N))
+#define jb(N) ((Sint)(Sint32)(N))
 #define tb(N) (N)
 #define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N)))
 #define yb(N) (*(Eterm *) (((unsigned char *)E) + (N)))
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 2888158c7b..2acf3a2c1c 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -87,6 +87,8 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int);
 
 typedef struct {
     Uint pos;                   /* Position of label reference to patch. */
+    Uint offset;                /* Offset from patch location.  */
+    int packed;                 /* 0 (not packed), 1 (lsw), 2 (msw) */
 } LabelPatch;
 
 /*
@@ -237,7 +239,7 @@ typedef struct {
 
 typedef struct literal_patch LiteralPatch;
 struct literal_patch {
-    int pos;			/* Position in code */
+    Uint pos;			/* Position in code */
     LiteralPatch* next;
 };
 
@@ -1948,8 +1950,11 @@ register_label_patch(LoaderState* stp, Uint label, Uint ci, Uint offset)
                                    (void *) lp->patches,
                                    lp->num_allocated * sizeof(LabelPatch));
     }
-    lp->patches[lp->num_patches++].pos = ci;
-    stp->codev[ci] = offset;
+    lp->patches[lp->num_patches].pos = ci;
+    lp->patches[lp->num_patches].offset = offset;
+    lp->patches[lp->num_patches].packed = 0;
+    lp->num_patches++;
+    stp->codev[ci] = label;
 }
 
 static int
@@ -2519,23 +2524,58 @@ load_code(LoaderState* stp)
 	    char* prog;            /* Program for packing engine. */
 	    struct pack_stack {
                 BeamInstr instr;
-                LiteralPatch* patch;
+                Uint* patch_pos;
             } stack[8];            /* Stack. */
 	    struct pack_stack* sp = stack; /* Points to next free position. */
 	    BeamInstr packed = 0; /* Accumulator for packed operations. */
+            LabelPatch* packed_label = 0;
 
 	    for (prog = opc[stp->specific_op].pack; *prog; prog++) {
 		switch (*prog) {
-		case 'g':	/* Get instruction; push on stack. */
+		case 'g':	/* Get operand and push on stack. */
+                    ci--;
+                    sp->instr = code[ci];
+                    sp->patch_pos = 0;
+                    sp++;
+                    break;
+		case 'f':	/* Get possible 'f' operand and push on stack. */
+                    {
+                        Uint w = code[--ci];
+                        sp->instr = w;
+                        sp->patch_pos = 0;
+
+                        if (w != 0) {
+                            LabelPatch* lbl_p;
+                            int num_patches;
+                            int patch;
+
+                            ASSERT(w < stp->num_labels);
+                            lbl_p = stp->labels[w].patches;
+                            num_patches = stp->labels[w].num_patches;
+                            for (patch = num_patches - 1; patch >= 0; patch--) {
+                                if (lbl_p[patch].pos == ci) {
+                                    sp->patch_pos = &lbl_p[patch].pos;
+                                    break;
+                                }
+                            }
+                            ASSERT(sp->patch_pos);
+                        }
+                        sp++;
+                    }
+                    break;
+		case 'q':	/* Get possible 'q' operand and push on stack. */
                     {
                         LiteralPatch* lp;
 
                         ci--;
                         sp->instr = code[ci];
-                        sp->patch = 0;
-                        for (lp = stp->literal_patches; lp && lp->pos > ci-MAX_OPARGS; lp = lp->next) {
+                        sp->patch_pos = 0;
+
+                        for (lp = stp->literal_patches;
+                             lp && lp->pos > ci-MAX_OPARGS;
+                             lp = lp->next) {
                             if (lp->pos == ci) {
-                                sp->patch = lp;
+                                sp->patch_pos = &lp->pos;
                                 break;
                             }
                         }
@@ -2547,28 +2587,68 @@ load_code(LoaderState* stp)
 		    break;
 		case '0':	/* Tight shift */
 		    packed = (packed << BEAM_TIGHT_SHIFT) | code[--ci];
+                    if (packed_label) {
+                        packed_label->packed++;
+                    }
 		    break;
 		case '6':	/* Shift 16 steps */
 		    packed = (packed << BEAM_LOOSE_SHIFT) | code[--ci];
+                    if (packed_label) {
+                        packed_label->packed++;
+                    }
 		    break;
 #ifdef ARCH_64
 		case 'w':	/* Shift 32 steps */
-		    packed = (packed << BEAM_WIDE_SHIFT) | code[--ci];
-		    break;
+                    {
+                        Uint w = code[--ci];
+
+                        if (packed_label) {
+                            packed_label->packed++;
+                        }
+
+                        /*
+                         * 'w' can handle both labels ('f' and 'j'), as well
+                         * as 'I'. Test whether this is a label.
+                         */
+
+                        if (w < stp->num_labels) {
+                            /*
+                             * Probably a label. Look for patch pointing to this
+                             * position.
+                             */
+                            LabelPatch* lp = stp->labels[w].patches;
+                            int num_patches = stp->labels[w].num_patches;
+                            int patch;
+                            for (patch = num_patches - 1; patch >= 0; patch--) {
+                                if (lp[patch].pos == ci) {
+                                    lp[patch].packed = 1;
+                                    packed_label = &lp[patch];
+                                    break;
+                                }
+                            }
+                        }
+                        packed = (packed << BEAM_WIDE_SHIFT) |
+                            (code[ci] & BEAM_WIDE_MASK);
+                    }
+                    break;
 #endif
 		case 'p':	/* Put instruction (from stack). */
                     --sp;
                     code[ci] = sp->instr;
-                    if (sp->patch) {
-                        sp->patch->pos = ci;
+                    if (sp->patch_pos) {
+                        *sp->patch_pos = ci;
                     }
                     ci++;
 		    break;
 		case 'P':	/* Put packed operands. */
                     sp->instr = packed;
-                    sp->patch = 0;
+                    sp->patch_pos = 0;
                     sp++;
 		    packed = 0;
+                    if (packed_label) {
+                        packed_label->pos = ci;
+                        packed_label = 0;
+                    }
 		    break;
 		default:
 		    ASSERT(0);
@@ -4873,7 +4953,8 @@ freeze_code(LoaderState* stp)
 	}
 	ASSERT(value < stp->ci);
         for (patch = 0; patch < stp->labels[i].num_patches; patch++) {
-            Uint pos = stp->labels[i].patches[patch].pos;
+            LabelPatch* lp = &stp->labels[i].patches[patch];
+            Uint pos = lp->pos;
 	    ASSERT(pos < stp->ci);
             if (pos < stp->num_functions) {
                 /*
@@ -4882,7 +4963,38 @@ freeze_code(LoaderState* stp)
                  */
                 codev[pos] = (BeamInstr) (codev + value);
             } else {
-                codev[pos] += value;
+#ifdef DEBUG
+                Uint w;
+#endif
+                Sint32 rel = lp->offset + value;
+                switch (lp->packed) {
+                case 0:         /* Not packed */
+                    ASSERT(codev[pos] == i);
+                    codev[pos] = rel;
+                    break;
+#ifdef BEAM_WIDE_MASK
+                case 1:         /* Least significant word. */
+#ifdef DEBUG
+                    w = codev[pos] & BEAM_WIDE_MASK;
+                    /* Correct label in least significant word? */
+                    ASSERT(w == i);
+#endif
+                    codev[pos] = (codev[pos] & ~BEAM_WIDE_MASK) |
+                        (rel & BEAM_WIDE_MASK);
+                    break;
+                case 2:         /* Most significant word */
+#ifdef DEBUG
+                    w = (codev[pos] >> BEAM_WIDE_SHIFT) & BEAM_WIDE_MASK;
+                    /* Correct label in most significant word? */
+                    ASSERT(w == i);
+#endif
+                    codev[pos] = ((Uint)rel << BEAM_WIDE_SHIFT) |
+                        (codev[pos] & BEAM_WIDE_MASK);
+                    break;
+#endif
+                default:
+                    ASSERT(0);
+                }
             }
 	}
     }
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 6248583b1c..d156bd83c0 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -1232,7 +1232,7 @@ sub basic_generator {
     #
 
     my $c_code_ref = $c_code{$name};
-    if ($hot and defined $c_code_ref) {
+    if ($hot and defined $c_code_ref and $name ne 'catch') {
         ($var_decls, $pack_spec, @args) = do_pack(@args);
     }
 
@@ -1516,6 +1516,23 @@ sub do_pack {
 	}
     }
 
+    #
+    # Try to pack 'f' and 'j', but not at expense at worse packing
+    # for other operands. For example, given the arguments "f x x", we
+    # want the 'x' operands to be packed, not 'f' and 'x' packed and
+    # the final 'x' not packed.
+    #
+
+    if ($wordsize == 64 and $packable_args == 1) {
+        for (my $i = 0; $i < @args; $i++) {
+            if ($args[$i] =~ /^[fj]$/) {
+                $bits_needed[$i] = 32;
+                $packable_args++;
+                last;
+            }
+        }
+    }
+
     #
     # Nothing to pack unless there are at least 2 packable arguments.
     #
@@ -1602,7 +1619,15 @@ sub do_pack {
 
         if ($arg_size{$arg} and $did_some_packing) {
             # Save the argument on the pack engine's stack.
-            $down = "g${down}";
+            my $push = 'g';
+            if ($type_bit{$arg} & $type_bit{'q'}) {
+                # The operand may be a literal.
+                $push = 'q';
+            } elsif ($type_bit{$arg} & $type_bit{'f'}) {
+                # The operand may be a failure label.
+                $push = 'f';
+            }
+            $down = "$push${down}";
             $up = "${up}p";
         } else {
             # The argument has either zero size (e.g. r(0)),
@@ -1630,7 +1655,7 @@ sub do_pack {
         if ($need_wide_mask[$word]) {
             @shift = ('0', 'BEAM_WIDE_SHIFT');
             @mask = ('BEAM_WIDE_MASK', $WHOLE_WORD);
-            @instr = ('w', 'i');
+            @instr = ('w', 'w');
         } else {
             @shift = @{$pack_shift[$args_per_word]};
             @mask = @{$pack_mask[$args_per_word]};
-- 
cgit v1.2.3


From cfe29c3386477b76a1daf4d631b1178dc7359a40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 7 Sep 2017 11:54:22 +0200
Subject: Pack sequences of trailing 'f' operands

Pack sequences of trailing 'f' operands for instructions
such at jump_on_val or i_select_val_lins.
---
 erts/emulator/beam/beam_debug.c      | 44 ++++++++++++++---------
 erts/emulator/beam/beam_load.c       | 68 ++++++++++++++++++++++++++++++++++++
 erts/emulator/beam/select_instrs.tab |  6 ++--
 3 files changed, 100 insertions(+), 18 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 39e96126bf..9ba65e2464 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -54,6 +54,7 @@ void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg);
 static int print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr);
 static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif);
 static BeamInstr* f_to_addr(BeamInstr* base, int op, BeamInstr* ap);
+static BeamInstr* f_to_addr_packed(BeamInstr* base, int op, Sint32* ap);
 
 BIF_RETTYPE
 erts_debug_same_2(BIF_ALIST_2)
@@ -634,6 +635,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	{
 	    int n = ap[-1];
 	    int ix = n;
+            Sint32* jump_tab = (Sint32 *)(ap + n);
 
 	    while (ix--) {
 		erts_print(to, to_arg, "%T ", (Eterm) ap[0]);
@@ -642,11 +644,11 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	    }
 	    ix = n;
 	    while (ix--) {
-                BeamInstr* target = f_to_addr(addr, op, ap);
+                BeamInstr* target = f_to_addr_packed(addr, op, jump_tab);
 		erts_print(to, to_arg, "f(" HEXF ") ", target);
-		ap++;
-		size++;
+                jump_tab++;
 	    }
+            size += (n+1) / 2;
 	}
 	break;
     case op_i_select_val_bins_xfI:
@@ -668,6 +670,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
         {
             int n = ap[-1];
             int ix = n - 1; /* without sentinel */
+            Sint32* jump_tab = (Sint32 *)(ap + n);
 
             while (ix--) {
                 Uint arity = arityval(ap[0]);
@@ -681,34 +684,38 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
             size++;
             ix = n;
             while (ix--) {
-                BeamInstr* target = f_to_addr(addr, op, ap);
+                BeamInstr* target = f_to_addr_packed(addr, op, jump_tab);
                 erts_print(to, to_arg, "f(" HEXF ") ", target);
-                ap++;
-                size++;
+                jump_tab++;
             }
+            size += (n+1) / 2;
         }
         break;
     case op_i_jump_on_val_xfIW:
     case op_i_jump_on_val_yfIW:
 	{
-	    int n;
-	    for (n = ap[-2]; n > 0; n--) {
-                BeamInstr* target = f_to_addr(addr, op, ap);
+	    int n = ap[-2];
+            Sint32* jump_tab = (Sint32 *) ap;
+
+            size += (n+1) / 2;
+            while (n-- > 0) {
+                BeamInstr* target = f_to_addr_packed(addr, op, jump_tab);
 		erts_print(to, to_arg, "f(" HEXF ") ", target);
-		ap++;
-		size++;
+                jump_tab++;
 	    }
 	}
 	break;
     case op_i_jump_on_val_zero_xfI:
     case op_i_jump_on_val_zero_yfI:
 	{
-	    int n;
-	    for (n = ap[-1]; n > 0; n--) {
-                BeamInstr* target = f_to_addr(addr, op, ap);
+	    int n = ap[-1];
+            Sint32* jump_tab = (Sint32 *) ap;
+
+            size += (n+1) / 2;
+            while (n-- > 0) {
+                BeamInstr* target = f_to_addr_packed(addr, op, jump_tab);
 		erts_print(to, to_arg, "f(" HEXF ") ", target);
-		ap++;
-		size++;
+                jump_tab++;
 	    }
 	}
 	break;
@@ -810,6 +817,11 @@ static BeamInstr* f_to_addr(BeamInstr* base, int op, BeamInstr* ap)
     return base - 1 + opc[op].adjust + (Sint32) *ap;
 }
 
+static BeamInstr* f_to_addr_packed(BeamInstr* base, int op, Sint32* ap)
+{
+    return base - 1 + opc[op].adjust + *ap;
+}
+
 
 /*
  * Dirty BIF testing.
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 2acf3a2c1c..9bb5866cb3 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -1976,6 +1976,9 @@ load_code(LoaderState* stp)
     GenOp** last_op_next = NULL;
     int arity;
     int retval = 1;
+#if defined(BEAM_WIDE_SHIFT)
+    int num_trailing_f;     /* Number of extra 'f' arguments in a list */
+#endif
 
     /*
      * The size of the loaded func_info instruction is needed
@@ -2661,7 +2664,17 @@ load_code(LoaderState* stp)
 	 * Load any list arguments using the primitive tags.
 	 */
 
+#if defined(BEAM_WIDE_SHIFT)
+        num_trailing_f = 0;
+#endif
 	for ( ; arg < tmp_op->arity; arg++) {
+#if defined(BEAM_WIDE_SHIFT)
+	    if (tmp_op->a[arg].type == TAG_f) {
+                num_trailing_f++;
+            } else {
+                num_trailing_f = 0;
+            }
+#endif
 	    switch (tmp_op->a[arg].type) {
 	    case TAG_i:
 		CodeNeed(1);
@@ -2701,6 +2714,61 @@ load_code(LoaderState* stp)
 	    }
 	}
 
+        /*
+         * If all the extra arguments were 'f' operands,
+         * and the wordsize is 64 bits, pack two 'f' operands
+         * into each word.
+         */
+
+#if defined(BEAM_WIDE_SHIFT)
+        if (num_trailing_f >= 1) {
+            Uint src_index = ci - num_trailing_f;
+            Uint src_limit = ci;
+            Uint dst_limit = src_index + (num_trailing_f+1)/2;
+
+            ci = src_index;
+            while (ci < dst_limit) {
+                Uint w[2];
+                BeamInstr packed = 0;
+                int wi;
+
+                w[0] = code[src_index];
+                if (src_index+1 < src_limit) {
+                    w[1] = code[src_index+1];
+                } else {
+                    w[1] = 0;
+                }
+                for (wi = 0; wi < 2; wi++) {
+                    Uint lbl = w[wi];
+                    LabelPatch* lp = stp->labels[lbl].patches;
+                    int num_patches = stp->labels[lbl].num_patches;
+
+#if defined(WORDS_BIGENDIAN)
+                    packed <<= BEAM_WIDE_SHIFT;
+                    packed |= lbl & BEAM_WIDE_MASK;
+#else
+                    packed >>= BEAM_WIDE_SHIFT;
+                    packed |= lbl << BEAM_WIDE_SHIFT;
+#endif
+                    while (num_patches-- > 0) {
+                        if (lp->pos == src_index + wi) {
+                            lp->pos = ci;
+#if defined(WORDS_BIGENDIAN)
+                            lp->packed = 2 - wi;
+#else
+                            lp->packed = wi + 1;
+#endif
+                            break;
+                        }
+                        lp++;
+                    }
+                }
+                code[ci++] = packed;
+                src_index += 2;
+            }
+        }
+#endif
+
 	/*
 	 * Handle a few special cases.
 	 */
diff --git a/erts/emulator/beam/select_instrs.tab b/erts/emulator/beam/select_instrs.tab
index da6b7dbe62..6b59f02925 100644
--- a/erts/emulator/beam/select_instrs.tab
+++ b/erts/emulator/beam/select_instrs.tab
@@ -150,7 +150,8 @@ select_val_lin.execute(N) {
     }
 
     if (vs[ix] == select_val) {
-        Eterm offset = *($NEXT_INSTRUCTION + $N + ix);
+        Sint32* jump_tab = (Sint32 *) ($NEXT_INSTRUCTION + $N);
+        Eterm offset = jump_tab[ix];
         $JUMP(offset);
     } else {
         $JUMP(*select_fail);
@@ -161,7 +162,8 @@ JUMP_ON_VAL(Fail, Index, N, Base) {
     if (is_small($Index)) {
         $Index = (Uint) (signed_val($Index) - $Base);
         if ($Index < $N) {
-            $JUMP((($NEXT_INSTRUCTION)[$Index]));
+            Sint32* jump_tab = (Sint32 *) ($NEXT_INSTRUCTION);
+            $JUMP(jump_tab[$Index]);
         }
     }
     $FAIL($Fail);
-- 
cgit v1.2.3


From a8e391f3baf2cfbcfa57e46d4cfe95c089921100 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 7 Sep 2017 17:51:43 +0200
Subject: Rewrite select_val_bins so that its labels can be packed

---
 erts/emulator/beam/beam_debug.c      | 16 ++---------
 erts/emulator/beam/beam_load.c       | 52 +++++++++---------------------------
 erts/emulator/beam/select_instrs.tab | 19 ++++++-------
 3 files changed, 24 insertions(+), 63 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 9ba65e2464..df152f1605 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -632,6 +632,8 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
     switch (op) {
     case op_i_select_val_lins_xfI:
     case op_i_select_val_lins_yfI:
+    case op_i_select_val_bins_xfI:
+    case op_i_select_val_bins_yfI:
 	{
 	    int n = ap[-1];
 	    int ix = n;
@@ -651,20 +653,6 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
             size += (n+1) / 2;
 	}
 	break;
-    case op_i_select_val_bins_xfI:
-    case op_i_select_val_bins_yfI:
-	{
-	    int n = ap[-1];
-
-	    while (n > 0) {
-                BeamInstr* target = f_to_addr(addr, op, ap+1);
-		erts_print(to, to_arg, "%T f(" HEXF ") ", (Eterm) ap[0], target);
-		ap += 2;
-		size += 2;
-		n--;
-	    }
-	}
-	break;
     case op_i_select_tuple_arity_xfI:
     case op_i_select_tuple_arity_yfI:
         {
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 9bb5866cb3..944ff7bc4c 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4039,7 +4039,6 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     int i, j, align = 0;
 
     if (size == 2) {
-
 	/*
 	 * Use a special-cased instruction if there are only two values.
 	 */
@@ -4056,47 +4055,19 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 	op->a[5] = Rest[3];
 
 	return op;
-
-    } else if (size > 10) {
-
-	/* binary search instruction */
-
-	NEW_GENOP(stp, op);
-	op->next = NULL;
-	op->op = genop_i_select_val_bins_3;
-	GENOP_ARITY(op, arity);
-	op->a[0] = S;
-	op->a[1] = Fail;
-	op->a[2].type = TAG_u;
-	op->a[2].val = size;
-	for (i = 3; i < arity; i++) {
-	    op->a[i] = Rest[i-3];
-	}
-
-	/*
-	 * Sort the values to make them useful for a binary search.
-	 */
-
-	qsort(op->a+3, size, 2*sizeof(GenOpArg),
-		(int (*)(const void *, const void *)) genopargcompare);
-#ifdef DEBUG
-	for (i = 3; i < arity-2; i += 2) {
-	    ASSERT(op->a[i].val < op->a[i+2].val);
-	}
-#endif
-	return op;
     }
 
-    /* linear search instruction */
-
-    align = 1;
+    if (size <= 10) {
+        /* Use linear search. Reserve place for a sentinel. */
+        align = 1;
+    }
 
     arity += 2*align;
     size  += align;
 
     NEW_GENOP(stp, op);
     op->next = NULL;
-    op->op = genop_i_select_val_lins_3;
+    op->op = (align == 0) ? genop_i_select_val_bins_3 : genop_i_select_val_lins_3;
     GENOP_ARITY(op, arity);
     op->a[0] = S;
     op->a[1] = Fail;
@@ -4110,7 +4081,7 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     }
 
     /*
-     * Sort the values to make them useful for a sentinel search
+     * Sort the values to make them useful for a binary or sentinel search.
      */
 
     qsort(tmp, size - align, 2*sizeof(GenOpArg),
@@ -4125,11 +4096,12 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 
     erts_free(ERTS_ALC_T_LOADER_TMP, (void *) tmp);
 
-    /* add sentinel */
-
-    op->a[j].type = TAG_u;
-    op->a[j].val  = ~((BeamInstr)0);
-    op->a[j+size] = Fail;
+    if (align) {
+        /* Add sentinel for linear search. */
+        op->a[j].type = TAG_u;
+        op->a[j].val  = ~((BeamInstr)0);
+        op->a[j+size] = Fail;
+    }
 
 #ifdef DEBUG
     for (i = 0; i < size - 1; i++) {
diff --git a/erts/emulator/beam/select_instrs.tab b/erts/emulator/beam/select_instrs.tab
index 6b59f02925..4a8d9cce96 100644
--- a/erts/emulator/beam/select_instrs.tab
+++ b/erts/emulator/beam/select_instrs.tab
@@ -30,16 +30,15 @@ select_val_bins.fetch(Src) {
 }
 
 select_val_bins.select(Fail, NumElements) {
-    struct Pairs {
+    struct Singleton {
         BeamInstr val;
-        Eterm offset;
     };
-    struct Pairs* low;
-    struct Pairs* high;
-    struct Pairs* mid;
+    struct Singleton* low;
+    struct Singleton* high;
+    struct Singleton* mid;
     int bdiff; /* int not long because the arrays aren't that large */
 
-    low = (struct Pairs *) (&$NumElements + 1);
+    low = (struct Singleton *) ($NEXT_INSTRUCTION);
     high = low + $NumElements;
 
     /* The pointer subtraction (high-low) below must produce
@@ -60,15 +59,17 @@ select_val_bins.select(Fail, NumElements) {
      *
      */
     while ((bdiff = (int)((char*)high - (char*)low)) > 0) {
-        unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Pairs)-1);
+        unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Singleton)-1);
 
-        mid = (struct Pairs*)((char*)low + boffset);
+        mid = (struct Singleton*)((char*)low + boffset);
         if (select_val < mid->val) {
             high = mid;
         } else if (select_val > mid->val) {
             low = mid + 1;
         } else {
-            $JUMP(mid->offset);
+            Sint32* jump_tab = (Sint32 *) ($NEXT_INSTRUCTION + $NumElements);
+            Sint32 offset = jump_tab[mid - (struct Singleton *)($NEXT_INSTRUCTION)];
+            $JUMP(offset);
         }
     }
     $JUMP($Fail);
-- 
cgit v1.2.3


From 6f45ff73583aa7d0352b8a16df78872f47defd35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sat, 9 Sep 2017 12:01:14 +0200
Subject: Optimize i_select_tuple_arity2 and is_select_lins

Don't save a pointer to the default failure label. That could
relieve register pressure. Instead, if we'll need to signal an
error in i_select_tuple_arity2, delay to the execution phase.
That should be a clear win because i_select_tuple_arity2 very
rarely fails because of the term being selected is not a tuple.
---
 erts/emulator/beam/select_instrs.tab | 43 ++++++++++++++----------------------
 1 file changed, 16 insertions(+), 27 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/select_instrs.tab b/erts/emulator/beam/select_instrs.tab
index 4a8d9cce96..f5ce5d5a32 100644
--- a/erts/emulator/beam/select_instrs.tab
+++ b/erts/emulator/beam/select_instrs.tab
@@ -75,66 +75,55 @@ select_val_bins.select(Fail, NumElements) {
     $JUMP($Fail);
 }
 
-i_select_tuple_arity2 := select_val2.src.ta_fail.execute;
-i_select_val2 := select_val2.src.fail.execute;
+i_select_tuple_arity2 := select_val2.src.get_arity.execute;
+i_select_val2 := select_val2.src.execute;
 
 select_val2.head() {
     Eterm select_val2;
-    BeamInstr* select_fail;
 }
 
 select_val2.src(Src) {
     select_val2 = $Src;
 }
 
-select_val2.ta_fail(Fail) {
-    select_fail = &$Fail;
-    if (is_not_tuple(select_val2)) {
-        $FAIL(*select_fail);
+select_val2.get_arity() {
+    if (ERTS_LIKELY(is_tuple(select_val2))) {
+        select_val2 = *tuple_val(select_val2);
+    } else {
+        select_val2 = NIL;
     }
-    select_val2 = *tuple_val(select_val2);
-}
-
-select_val2.fail(Fail) {
-    select_fail = &$Fail;
 }
 
-select_val2.execute(T1, T2, D1, D2) {
+select_val2.execute(Fail, T1, T2, D1, D2) {
     if (select_val2 == $T1) {
         $JUMP($D1);
     } else if (select_val2 == $T2) {
         $JUMP($D2);
     } else {
-        $FAIL(*select_fail);
+        $FAIL($Fail);
     }
 }
 
-i_select_tuple_arity := select_val_lin.fetch.ta_fail.execute;
-i_select_val_lins := select_val_lin.fetch.fail.execute;
+i_select_tuple_arity := select_val_lin.fetch.get_arity.execute;
+i_select_val_lins := select_val_lin.fetch.execute;
 
 select_val_lin.head() {
     Eterm select_val;
-    BeamInstr* select_fail;
 }
 
 select_val_lin.fetch(Src) {
     select_val = $Src;
 }
 
-select_val_lin.ta_fail(Fail) {
-    select_fail = &$Fail;
-    if (is_tuple(select_val)) {
+select_val_lin.get_arity() {
+    if (ERTS_LIKELY(is_tuple(select_val))) {
         select_val = *tuple_val(select_val);
     } else {
-        $JUMP(*select_fail);
+        select_val = NIL;
     }
 }
 
-select_val_lin.fail(Fail) {
-    select_fail = &$Fail;
-}
-
-select_val_lin.execute(N) {
+select_val_lin.execute(Fail, N) {
     BeamInstr* vs = $NEXT_INSTRUCTION;
     int ix = 0;
 
@@ -155,7 +144,7 @@ select_val_lin.execute(N) {
         Eterm offset = jump_tab[ix];
         $JUMP(offset);
     } else {
-        $JUMP(*select_fail);
+        $JUMP($Fail);
     }
 }
 
-- 
cgit v1.2.3


From e8ee9f4cba07c7aa05685207c54ae1d773bf1814 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sat, 9 Sep 2017 12:30:35 +0200
Subject: Pack failure labels in i_select_val2 and i_select_tuple_arity2

---
 erts/emulator/beam/beam_debug.c      | 16 ++++++++++++++++
 erts/emulator/beam/beam_load.c       |  4 ++--
 erts/emulator/beam/ops.tab           |  4 ++--
 erts/emulator/beam/select_instrs.tab |  8 +++++---
 4 files changed, 25 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index df152f1605..afde45ba71 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -679,6 +679,22 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
             size += (n+1) / 2;
         }
         break;
+    case op_i_select_val2_xfcc:
+    case op_i_select_val2_yfcc:
+    case op_i_select_tuple_arity2_xfAA:
+    case op_i_select_tuple_arity2_yfAA:
+        {
+            Sint32* jump_tab = (Sint32 *) ap;
+            BeamInstr* target;
+            int i;
+
+            for (i = 0; i < 2; i++) {
+                target = f_to_addr_packed(addr, op, jump_tab++);
+                erts_print(to, to_arg, "f(" HEXF ") ", target);
+            }
+            size += 1;
+        }
+        break;
     case op_i_jump_on_val_xfIW:
     case op_i_jump_on_val_yfIW:
 	{
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 944ff7bc4c..3f9dc2c1aa 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -3749,7 +3749,7 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     if (size == 2) {
 	NEW_GENOP(stp, op);
 	op->next = NULL;
-	op->op = genop_i_select_tuple_arity2_6;
+	op->op = genop_i_select_tuple_arity2_4;
 	GENOP_ARITY(op, arity - 1);
 	op->a[0] = S;
 	op->a[1] = Fail;
@@ -4045,7 +4045,7 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 
 	NEW_GENOP(stp, op);
 	op->next = NULL;
-	op->op = genop_i_select_val2_6;
+	op->op = genop_i_select_val2_4;
 	GENOP_ARITY(op, arity - 1);
 	op->a[0] = S;
 	op->a[1] = Fail;
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 4152e0ced5..52a13d4e18 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -162,11 +162,11 @@ i_select_val_bins xy f I
 
 i_select_val_lins xy f I
 
-i_select_val2 xy f c c f f
+i_select_val2 xy f c c
 
 i_select_tuple_arity xy f I
 
-i_select_tuple_arity2 xy f A A f f
+i_select_tuple_arity2 xy f A A
 
 i_jump_on_val_zero xy f I
 
diff --git a/erts/emulator/beam/select_instrs.tab b/erts/emulator/beam/select_instrs.tab
index f5ce5d5a32..2951949d38 100644
--- a/erts/emulator/beam/select_instrs.tab
+++ b/erts/emulator/beam/select_instrs.tab
@@ -94,11 +94,13 @@ select_val2.get_arity() {
     }
 }
 
-select_val2.execute(Fail, T1, T2, D1, D2) {
+select_val2.execute(Fail, T1, T2) {
+    Sint32* jump_tab = (Sint32 *) ($NEXT_INSTRUCTION);
+
     if (select_val2 == $T1) {
-        $JUMP($D1);
+        $JUMP(jump_tab[0]);
     } else if (select_val2 == $T2) {
-        $JUMP($D2);
+        $JUMP(jump_tab[1]);
     } else {
         $FAIL($Fail);
     }
-- 
cgit v1.2.3


From 4bf642e54bca72d77b64af4409a13188dd5c7378 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 13 Sep 2017 12:56:06 +0200
Subject: Correct disassembly of select instructions

Make sure to use the 'unpacked[-1]' when accessing the unpacked
arguments.
---
 erts/emulator/beam/beam_debug.c | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index afde45ba71..7017656476 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -629,13 +629,20 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 
     unpacked = ap;
     ap = addr + size;
+
+    /*
+     * In the code below, never use ap[-1], ap[-2], ...
+     * (will not work if the arguments have been packed).
+     *
+     * Instead use unpacked[-1], unpacked[-2], ...
+     */
     switch (op) {
     case op_i_select_val_lins_xfI:
     case op_i_select_val_lins_yfI:
     case op_i_select_val_bins_xfI:
     case op_i_select_val_bins_yfI:
 	{
-	    int n = ap[-1];
+	    int n = unpacked[-1];
 	    int ix = n;
             Sint32* jump_tab = (Sint32 *)(ap + n);
 
@@ -656,7 +663,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
     case op_i_select_tuple_arity_xfI:
     case op_i_select_tuple_arity_yfI:
         {
-            int n = ap[-1];
+            int n = unpacked[-1];
             int ix = n - 1; /* without sentinel */
             Sint32* jump_tab = (Sint32 *)(ap + n);
 
@@ -698,7 +705,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
     case op_i_jump_on_val_xfIW:
     case op_i_jump_on_val_yfIW:
 	{
-	    int n = ap[-2];
+	    int n = unpacked[-2];
             Sint32* jump_tab = (Sint32 *) ap;
 
             size += (n+1) / 2;
@@ -712,7 +719,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
     case op_i_jump_on_val_zero_xfI:
     case op_i_jump_on_val_zero_yfI:
 	{
-	    int n = ap[-1];
+	    int n = unpacked[-1];
             Sint32* jump_tab = (Sint32 *) ap;
 
             size += (n+1) / 2;
-- 
cgit v1.2.3


From 4bbaf12b7442abb8fa6dc0798f15e46c2a629e5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 12 Sep 2017 02:40:50 +0200
Subject: beam_makeops: Refactor code generation

The refactoring will simplify packing of combined instructions.
---
 erts/emulator/utils/beam_makeops | 171 ++++++++++++++++++++++++++-------------
 1 file changed, 114 insertions(+), 57 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index bb31db7eb5..ac2b140c2a 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -80,7 +80,10 @@ my %gen_opnum;
 my %num_specific;
 my %gen_to_spec;
 my %specific_op;
-my %group_size;                 # Group size for specific operators.
+
+# Information about each specific operator. Key is the print name (e.g. get_list_xxy).
+# Value is a hash.
+my %spec_op_info;
 
 my %gen_arity;
 my @gen_arity;
@@ -522,6 +525,37 @@ sub emulator_output {
     my $name;
     my $key;			# Loop variable.
 
+    #
+    # Generate code and meta information for all instructions.
+    #
+    foreach $key (keys %specific_op) {
+	foreach (@{$specific_op{$key}}) {
+	    my($name, $hotness, @args) = @$_;
+	    my $sign = join('', @args);
+	    my $print_name = print_name($name, @args);
+
+	    my($size, $code, $pack_spec) = cg_basic($name, @args);
+	    if (defined $code) {
+                $code = "OpCase($print_name):\n$code";
+                push @generated_code, [$hotness,$code,($print_name)];
+	    }
+
+            # Note: Some of the information below will be modified
+            # for combined instructions.
+            my %info = ('size' => $size,
+                        'pack_spec' => $pack_spec,
+                        'adj' => 0,
+                        'args' => \@args);
+            $spec_op_info{$print_name} = \%info;
+	}
+    }
+
+    #
+    # Combine micro instruction into instruction blocks and generate
+    # code for them.
+    #
+    combine_micro_instructions();
+
     #
     # Information about opcodes (beam_opcodes.c).
     #
@@ -550,15 +584,10 @@ sub emulator_output {
     print "\n};\n";
     print "\n";
 
-    #
-    # Combine micro instruction into instruction blocks.
-    #
-    combine_micro_instructions();
-
     #
     # Generate code for specific ops.
     #
-    my($spec_opnum) = 0;
+    my $spec_opnum = 0;
     print "const OpEntry opc[] = {\n";
     foreach $key (sort keys %specific_op) {
 	$gen_to_spec{$key} = $spec_opnum;
@@ -576,35 +605,21 @@ sub emulator_output {
 
 	    # The primitive types should sort before other types.
 
-	    my($sort_key) = $sign;
+	    my $sort_key = $sign;
 	    eval "\$sort_key =~ tr/$genop_types/./";
 	    $sort_key .= ":$sign";
-	    $items{$sort_key} = [$name, $hot, $sign, @args];
+            my $print_name = print_name($name, @args);
+	    $items{$sort_key} = $print_name;
 	}
 
 	#
 	# Now call the generator for the sorted result.
 	#
-	foreach (sort keys %items) {
-	    my($name, $hot, $sign, @args) = @{$items{$_}};
+	foreach my $sort_key (sort keys %items) {
+            my $print_name = $items{$sort_key};
+            my $info = $spec_op_info{$print_name};
+            my(@args) = @{$info->{'args'}};
 	    my $arity = @args;
-	    my($instr) = "${name}_$sign";
-	    $instr =~ s/_$//;
-
-	    #
-	    # Call a generator to calculate size and generate macros
-	    # for the emulator.
-	    #
-	    my($size, $code, $pack) =
-                basic_generator($name, 1, '', 0, undef, @args);
-
-	    #
-	    # Save the generated $code for later.
-	    #
-	    if (defined $code) {
-                $code = "OpCase($instr):\n$code";
-                push @generated_code, [$hot,$code,($instr)];
-	    }
 
 	    #
 	    # Calculate the bit mask which should be used to match this
@@ -626,7 +641,6 @@ sub emulator_output {
 	    }
 
 	    printf "/* %3d */  ", $spec_opnum;
-	    my $print_name = $sign ne '' ? "${name}_$sign" : $name;
 	    my $init = "{";
 	    my $sep = "";
 	    foreach (@bits) {
@@ -634,12 +648,12 @@ sub emulator_output {
 		$sep = ",";
 	    }
 	    $init .= "}";
-            my $adj = 0;
-            if (defined $group_size{$print_name}) {
-                $adj = $size - $group_size{$print_name};
-            }
-	    init_item($print_name, $init, $involves_r, $size, $adj, $pack, $sign);
-	    $op_to_name[$spec_opnum] = $instr;
+            my $adj = $info->{'adj'};
+            my $size = $info->{'size'};
+            my $pack_spec = $info->{'pack_spec'};
+            my $sign = join '', @args;
+	    init_item($print_name, $init, $involves_r, $size, $adj, $pack_spec, $sign);
+	    $op_to_name[$spec_opnum] = $print_name;
 	    $spec_opnum++;
 	}
     }
@@ -835,6 +849,12 @@ sub emulator_output {
     print_code(COLD);
 }
 
+sub print_name {
+    my($name,@args) = @_;
+    my $sign = join '', @args;
+    $sign ne '' ? "${name}_$sign" : $name;
+}
+
 sub init_item {
     my($sep) = "";
 
@@ -1108,8 +1128,7 @@ sub combine_instruction_group {
             my $offset = 0;
             my @rest = @args;
             my @new_subs;
-            my $opcase = $specific;
-            $opcase .= "_" . join '', @args if @args;
+            my $opcase = print_name($specific, @args);
             foreach my $s (@subs) {
                 my $code = $c_code{$s};
                 my(undef,undef,@c_args) = @{$code};
@@ -1117,7 +1136,7 @@ sub combine_instruction_group {
                 foreach (0..$#c_args) {
                     push @first, shift @rest;
                 }
-                my($size,undef) = basic_generator($s, 0, '', 0, undef, @first);
+                my $size = cg_combined_size($s, 0, @first);
                 $offsets{$s} = $offset
                     unless defined $offsets{$s} and $offsets{$s} >= $offset;
                 $offset += $size - 1;
@@ -1186,7 +1205,6 @@ sub combine_instruction_group {
             if ($opcase ne '') {
                 $gcode .= "OpCase($opcase):\n";
                 push @opcase_labels, $opcase;
-                $group_size{$opcase} = $group_size + 1;
             }
             if ($num_references{$label}) {
                 $gcode .= "$label:\n";
@@ -1204,14 +1222,19 @@ sub combine_instruction_group {
                 $transfer_to_next .= "goto $next;\n\n";
             }
 
-            my(undef,$gen_code) =
-                basic_generator($s, 0, $flags, $offset,
-                                $group_size-$offset-$dec, @first);
+            my($gen_code) =
+                cg_combined_code($s, 0, $flags, $offset,
+                                 $group_size-$offset-$dec, @first);
             $gcode .= $gen_code . $transfer_to_next;
         }
         $offset = $order_to_offset{$slots[$i+1]} if $i < $#slots;
     }
 
+    foreach my $print_name (@opcase_labels) {
+        my $info = $spec_op_info{$print_name};
+        $info->{'adj'} = $info->{'size'} - $group_size - 1;
+    }
+
     ($group_hotness,"{\n$gcode\n}\n\n",@opcase_labels);
 }
 
@@ -1223,12 +1246,42 @@ sub micro_label {
 
 
 #
-# Basic implementation of instruction in emulator loop
-# (assuming no packing).
+# Basic code generation for one instruction.
 #
 
-sub basic_generator {
-    my($name,$hot,$extra_comments,$offset,$group_size,@args) = @_;
+sub cg_basic {
+    my($name,@args) = @_;
+    my($size,$code,$pack_spec) = code_gen($name, 1, '', 0, undef, @args);
+    $pack_spec = build_pack_spec($pack_spec);
+    ($size,$code,$pack_spec);
+}
+
+#
+# Calculate size for a micro instruction.
+#
+
+sub cg_combined_size {
+    my($name,$pack,@args) = @_;
+    my($size) = code_gen($name, $pack, '', 0, undef, @args);
+    $size;
+}
+
+#
+# Generate code for a micro instruction.
+#
+
+sub cg_combined_code {
+    my($size,$code,$pack_spec) = code_gen(@_);
+    if ($pack_spec eq '') {
+        ($code,'','');
+    } else {
+        my($down,$up) = split /:/, $pack_spec;
+        ($code,$down,$up);
+    }
+}
+
+sub code_gen {
+    my($name,$pack,$extra_comments,$offset,$group_size,@args) = @_;
     my $size = 0;
     my $flags = '';
     my @f;
@@ -1242,7 +1295,7 @@ sub basic_generator {
     #
 
     my $c_code_ref = $c_code{$name};
-    if ($hot and defined $c_code_ref and $name ne 'catch') {
+    if ($pack and defined $c_code_ref and $name ne 'catch') {
         ($var_decls, $pack_spec, @args) = do_pack(@args);
     }
 
@@ -1569,7 +1622,7 @@ sub do_pack {
     #
     # Nothing to pack unless there are at least 2 packable arguments.
     #
-    return ('', '', @args) if $packable_args < 2;
+    return ('', ':', @args) if $packable_args < 2;
 
     #
     # Determine how many arguments we should pack into each word.
@@ -1644,13 +1697,12 @@ sub do_pack {
 				# beginning).
     my $up = '';		# Pack commands (storing back while
 				# moving forward).
-    my $did_some_packing = 0;   # Nothing packed yet.
 
     # Skip an unpackable argument.
     my $skip_unpackable = sub {
         my($arg) = @_;
 
-        if ($arg_size{$arg} and $did_some_packing) {
+        if ($arg_size{$arg}) {
             # Save the argument on the pack engine's stack.
             my $push = 'g';
             if ($type_bit{$arg} & $type_bit{'q'}) {
@@ -1662,11 +1714,6 @@ sub do_pack {
             }
             $down = "$push${down}";
             $up = "${up}p";
-        } else {
-            # The argument has either zero size (e.g. r(0)),
-            # or is to the left of the first packed argument
-            # and will never be accessed. No need to do
-            # anything.
         }
     };
 
@@ -1700,7 +1747,6 @@ sub do_pack {
             my $this_size = $arg_size{$reg};
             if ($bits_needed[$arg_num]) {
                 $this_size = 0;
-                $did_some_packing = 1;
 
                 if ($ap == 0) {
                     $pack_prefix .= "Eterm $packed_var = " .
@@ -1731,7 +1777,7 @@ sub do_pack {
         $arg_num++;
     }
 
-    my $pack_spec = $down . $up;
+    my $pack_spec = "$down:$up";
     return ($pack_prefix, $pack_spec, @args);
 }
 
@@ -1744,6 +1790,17 @@ sub make_unpack {
     $e;
 }
 
+sub build_pack_spec {
+    my $pack_spec = shift;
+    return '' if $pack_spec eq '';
+    my($down,$up) = split /:/, $pack_spec;
+    while ($down =~ /[gfq]$/ and $up =~ /^p/) {
+        $down = substr($down, 0, -1);
+        $up = substr($up, 1);
+    }
+    "$down$up";
+}
+
 sub quote {
     local($_) = @_;
     return "'$_'" if $_ eq 'try';
-- 
cgit v1.2.3


From 127b96806b732cb7c39d2e174c77a884de73626b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 12 Sep 2017 03:22:40 +0200
Subject: Pack combined instructions

---
 erts/emulator/utils/beam_makeops | 43 +++++++++++++++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index ac2b140c2a..a9b2c8861c 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -1128,7 +1128,9 @@ sub combine_instruction_group {
             my $offset = 0;
             my @rest = @args;
             my @new_subs;
-            my $opcase = print_name($specific, @args);
+            my $print_name = print_name($specific, @args);
+            my $opcase = $print_name;
+            my $last = $subs[$#subs];
             foreach my $s (@subs) {
                 my $code = $c_code{$s};
                 my(undef,undef,@c_args) = @{$code};
@@ -1136,7 +1138,7 @@ sub combine_instruction_group {
                 foreach (0..$#c_args) {
                     push @first, shift @rest;
                 }
-                my $size = cg_combined_size($s, 0, @first);
+                my $size = cg_combined_size($s, 1, @first);
                 $offsets{$s} = $offset
                     unless defined $offsets{$s} and $offsets{$s} >= $offset;
                 $offset += $size - 1;
@@ -1145,6 +1147,7 @@ sub combine_instruction_group {
                 push @new_subs, [$opcase,$label,$s,$size-1,@first];
                 $opcase = '';
             }
+            $spec_op_info{$print_name}->{'size'} = $offset + 1;
             $group_size = $offset if $group_size < $offset;
             push @instrs, [$specific_key,@new_subs];
         }
@@ -1181,6 +1184,8 @@ sub combine_instruction_group {
     # Now generate the code for the entire group.
     my $offset = 0;
     my @opcase_labels;
+    my %down;
+    my %up;
     for(my $i = 0; $i < @slots; $i++) {
         my $key = $slots[$i];
 
@@ -1201,6 +1206,7 @@ sub combine_instruction_group {
             my $seen_key = "$label:$next:" . scalar(@first);
             next if $opcase eq '' and $seen{$seen_key};
             $seen{$seen_key} = 1;
+            $seen_key .= $opcase;
 
             if ($opcase ne '') {
                 $gcode .= "OpCase($opcase):\n";
@@ -1222,9 +1228,12 @@ sub combine_instruction_group {
                 $transfer_to_next .= "goto $next;\n\n";
             }
 
-            my($gen_code) =
-                cg_combined_code($s, 0, $flags, $offset,
+            my($gen_code,$down,$up) =
+                cg_combined_code($s, 1, $flags, $offset,
                                  $group_size-$offset-$dec, @first);
+            my $spec_label = "$opcase$label";
+            $down{$spec_label} = $down;
+            $up{$spec_label} = $up;
             $gcode .= $gen_code . $transfer_to_next;
         }
         $offset = $order_to_offset{$slots[$i+1]} if $i < $#slots;
@@ -1235,6 +1244,26 @@ sub combine_instruction_group {
         $info->{'adj'} = $info->{'size'} - $group_size - 1;
     }
 
+    #
+    # Assemble pack specifications for all instructions in the group.
+    #
+    foreach my $instr (@instrs) {
+        my(undef,@subs) = @{$instr};
+        my $down = '';
+        my $up = '';
+        for (my $i = 0; $i < @subs; $i++) {
+            my($opcase,$label) = @{$subs[$i]};
+            my $spec_label = "$opcase$label";
+            if (defined $down{$spec_label}) {
+                $down = $down{$spec_label} . $down;
+                $up = $up . $up{$spec_label};
+            }
+        }
+        my $print_name = $subs[0]->[0];
+        my $info = $spec_op_info{$print_name};
+        $info->{'pack_spec'} = build_pack_spec("$down:$up");
+    }
+
     ($group_hotness,"{\n$gcode\n}\n\n",@opcase_labels);
 }
 
@@ -1296,7 +1325,7 @@ sub code_gen {
 
     my $c_code_ref = $c_code{$name};
     if ($pack and defined $c_code_ref and $name ne 'catch') {
-        ($var_decls, $pack_spec, @args) = do_pack(@args);
+        ($var_decls, $pack_spec, @args) = do_pack($offset, @args);
     }
 
     #
@@ -1572,7 +1601,7 @@ sub needs_do_wrapper {
 }
 
 sub do_pack {
-    my(@args) = @_;
+    my($offset,@args) = @_;
     my($packable_args) = 0;
     my @bits_needed;            # Bits needed for each argument.
 
@@ -1750,7 +1779,7 @@ sub do_pack {
 
                 if ($ap == 0) {
                     $pack_prefix .= "Eterm $packed_var = " .
-                        arg_offset($size) . ";\n";
+                        arg_offset($size+$offset) . ";\n";
                     $up .= "p";
                     $down = "P$down";
                     $this_size = 1;
-- 
cgit v1.2.3


From 6c1bb4bcfce103b3ef95e43fa5376b7d611ccc79 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 12 Apr 2017 17:34:46 +0200
Subject: erts: Increase number of check_io-fd-locks

and use correct cache alignment.
---
 erts/emulator/sys/common/erl_check_io.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 834b77eb58..f0854e8730 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -148,10 +148,11 @@ struct removed_fd {
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
 static int max_fds = -1;
 #endif
-#define DRV_EV_STATE_LOCK_CNT 16
+
+#define DRV_EV_STATE_LOCK_CNT 128
 static union {
     erts_mtx_t lck;
-    byte _cache_line_alignment[64];
+    byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_mtx_t))];
 }drv_ev_state_locks[DRV_EV_STATE_LOCK_CNT];
 
 static ERTS_INLINE erts_mtx_t* fd_mtx(ErtsSysFdType fd)
-- 
cgit v1.2.3


From 5c7a869b8d0e4aeec9902a1316d672860790076c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 20 Apr 2017 17:53:31 +0200
Subject: erts: Rename ErtsPollSet_ structS to not confuse gdb

by having different structs with same name.
---
 erts/emulator/sys/common/erl_poll.c | 10 +++++++---
 erts/emulator/sys/common/erl_poll.h |  2 +-
 erts/emulator/sys/win32/erl_poll.c  |  6 +++---
 3 files changed, 11 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index 7d26839b0f..341370ca14 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -228,7 +228,11 @@ typedef struct {
 
 #endif
 
-struct ErtsPollSet_ {
+/*
+ * This struct is not really exported, but it's nice to
+ * get unique names in debugger for kp/nkp
+ */
+struct ERTS_POLL_EXPORT(erts_pollset) {
     ErtsPollSet next;
     int internal_fd_limit;
     ErtsFdStatus *fds_status;
@@ -2472,7 +2476,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
     int kp_fd;
 #endif
     ErtsPollSet ps = erts_alloc(ERTS_ALC_T_POLLSET,
-				sizeof(struct ErtsPollSet_));
+				sizeof(struct ERTS_POLL_EXPORT(erts_pollset)));
     ps->internal_fd_limit = 0;
     ps->fds_status = NULL;
     ps->fds_status_len = 0;
@@ -2660,7 +2664,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
 
     ERTS_POLLSET_LOCK(ps);
 
-    size += sizeof(struct ErtsPollSet_);
+    size += sizeof(struct ERTS_POLL_EXPORT(erts_pollset));
     size += ps->fds_status_len*sizeof(ErtsFdStatus);
 
 #if ERTS_POLL_USE_EPOLL
diff --git a/erts/emulator/sys/common/erl_poll.h b/erts/emulator/sys/common/erl_poll.h
index a57dc51e5b..12dfc66e51 100644
--- a/erts/emulator/sys/common/erl_poll.h
+++ b/erts/emulator/sys/common/erl_poll.h
@@ -195,7 +195,7 @@ typedef Uint32 ErtsPollEvents;
 
 #endif
 
-typedef struct ErtsPollSet_ *ErtsPollSet;
+typedef struct ERTS_POLL_EXPORT(erts_pollset) *ErtsPollSet;
 
 typedef struct {
     ErtsSysFdType fd;
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index 0bd43bb4fb..7a87e81141 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -274,7 +274,7 @@ typedef struct _Waiter {
 /*
  * The structure for a pollset. There can currently be only one...
  */
-struct ErtsPollSet_ {
+struct erts_pollset {
     Waiter** waiter;
     int allocated_waiters;  /* Size ow waiter array */ 
     int num_waiters;	    /* Number of waiter threads. */
@@ -1284,7 +1284,7 @@ void erts_poll_info(ErtsPollSet ps,
     HARDTRACEF(("In erts_poll_info"));
     ERTS_POLLSET_LOCK(ps);
 
-    size += sizeof(struct ErtsPollSet_);
+    size += sizeof(struct erts_pollset);
     size += sizeof(Waiter *) * ps->allocated_waiters;
     for (i = 0; i < ps->num_waiters; ++i) {
 	Waiter *w = ps->waiter[i];
@@ -1326,7 +1326,7 @@ void erts_poll_info(ErtsPollSet ps,
 ErtsPollSet erts_poll_create_pollset(void)
 {
     ErtsPollSet ps = SEL_ALLOC(ERTS_ALC_T_POLLSET,
-			       sizeof(struct ErtsPollSet_));
+			       sizeof(struct erts_pollset));
     HARDTRACEF(("In erts_poll_create_pollset"));
 
     ps->num_waiters = 0;
-- 
cgit v1.2.3


From ba93bbc768928bb6213cd583ea91a108fa9d16ca Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 24 Apr 2017 20:08:53 +0200
Subject: erts: Refactor move check_io interface from sys to check_io

# Conflicts:
#	erts/emulator/beam/erl_process.c
#	erts/emulator/beam/sys.h
#	erts/emulator/sys/common/erl_check_io.c
#	erts/emulator/sys/common/erl_check_io.h
#	erts/emulator/sys/unix/sys.c
---
 erts/emulator/beam/erl_bif_info.c       |   1 +
 erts/emulator/beam/erl_port_task.c      |   1 +
 erts/emulator/beam/erl_process.c        |   9 +-
 erts/emulator/beam/sys.h                |   4 -
 erts/emulator/sys/common/erl_check_io.c | 153 ++++++++++++++++++++++++++++----
 erts/emulator/sys/common/erl_check_io.h |  33 -------
 erts/emulator/sys/unix/sys.c            | 129 ++-------------------------
 erts/emulator/sys/win32/sys.c           |  13 ---
 8 files changed, 150 insertions(+), 193 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 36939d6acc..d296e794d5 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -46,6 +46,7 @@
 #include "erl_thr_progress.h"
 #include "erl_bif_unique.h"
 #include "erl_map.h"
+#include "erl_check_io.h"
 #define ERTS_PTAB_WANT_DEBUG_FUNCS__
 #include "erl_ptab.h"
 #include "erl_time.h"
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 14977dfa17..b9152a7b5e 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -36,6 +36,7 @@
 #include "erl_check_io.h"
 #include "dtrace-wrapper.h"
 #include "lttng-wrapper.h"
+#include "erl_check_io.h"
 #include <stdarg.h>
 
 /*
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 1f696f7ba4..9c92d2a1a4 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -49,6 +49,7 @@
 #define ERTS_WANT_TIMER_WHEEL_API
 #include "erl_time.h"
 #include "erl_nfunc_sched.h"
+#include "erl_check_io.h"
 
 #define ERTS_CHECK_TIME_REDS CONTEXT_REDS
 #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0)
@@ -1563,14 +1564,14 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags)
 {
     switch (flags & ERTS_SSI_FLGS_SLEEP_TYPE) {
     case ERTS_SSI_FLG_POLL_SLEEPING:
-	erts_sys_schedule_interrupt(1);
+	erts_check_io_interrupt(1);
 	break;
     case ERTS_SSI_FLG_POLL_SLEEPING|ERTS_SSI_FLG_TSE_SLEEPING:
 	/*
 	 * Thread progress blocking while poll sleeping; need
 	 * to signal on both...
 	 */
-	erts_sys_schedule_interrupt(1);
+	erts_check_io_interrupt(1);
 	/* fall through */
     case ERTS_SSI_FLG_TSE_SLEEPING:
 	erts_tse_set(ssi->event);
@@ -3084,7 +3085,7 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type)
 	erts_tse_reset(ssi->event);
     else {
 	ASSERT(sleep_type == ERTS_SSI_FLG_POLL_SLEEPING);
-	erts_sys_schedule_interrupt(0);
+	erts_check_io_interrupt(0);
     }
 
     while (1) {
@@ -10049,7 +10050,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    fcalls = 0;
 
 #if 0 /* Not needed since we wont wait in sys schedule */
-	    erts_sys_schedule_interrupt(0);
+	    erts_check_io_interrupt(0);
 #endif
 	    erts_runq_unlock(rq);
 
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 68ef0a23f3..3dd6ef333b 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -575,8 +575,6 @@ __decl_noreturn void __noreturn erts_exit(int n, char*, ...);
     erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \
 	     __FILE__, __LINE__, __func__, What)
 
-Eterm erts_check_io_info(void *p);
-
 UWord erts_sys_get_page_size(void);
 
 /* Size of misc memory allocated from system dependent code */
@@ -739,8 +737,6 @@ extern char *erts_sys_ddll_error(int code);
 /*
  * System interfaces for startup.
  */
-void erts_sys_schedule_interrupt(int set);
-void erts_sys_schedule_interrupt_timed(int, ErtsMonotonicTime);
 void erts_sys_main_thread(void);
 
 extern int erts_sys_prepare_crash_dump(int secs);
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index f0854e8730..bed815c64b 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -110,6 +110,23 @@ static struct pollset_info
 }pollset;
 #define NUM_OF_POLLSETS 1
 
+
+#ifdef ERTS_ENABLE_KERNEL_POLL
+void erts_init_check_io_kp(void);
+void erts_init_check_io_nkp(void);
+int ERTS_CIO_EXPORT(driver_select)(ErlDrvPort, ErlDrvEvent, int, int);
+int ERTS_CIO_EXPORT(enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
+int ERTS_CIO_EXPORT(driver_event)(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
+Uint ERTS_CIO_EXPORT(erts_check_io_size)(void);
+Eterm ERTS_CIO_EXPORT(erts_check_io_info)(void *);
+int ERTS_CIO_EXPORT(erts_check_io_max_files)(void);
+void ERTS_CIO_EXPORT(erts_check_io_interrupt)(int);
+void ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(int, ErtsMonotonicTime);
+void ERTS_CIO_EXPORT(erts_check_io)(int);
+int ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *);
+#endif
+
+
 typedef struct {
 #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
     SafeHashBucket hb;
@@ -2098,7 +2115,6 @@ eready(Eterm id, ErtsDrvEventState *state, ErlDrvEventData event_data,
 
 static void bad_fd_in_pollset(ErtsDrvEventState *, Eterm inport, Eterm outport);
 
-
 void
 ERTS_CIO_EXPORT(erts_check_io_interrupt)(int set)
 {
@@ -2142,12 +2158,6 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 	erts_do_break_handling();
 #endif
 
-#ifdef ERTS_SIGNAL_STATE /* ifndef ERTS_SMP */
-    if (ERTS_SIGNAL_STATE) {
-        erts_handle_signal_state();
-    }
-#endif
-
     /* Figure out timeout value */
     timeout_time = (do_wait
 		    ? erts_check_next_timeout_time(esdp)
@@ -2186,14 +2196,6 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 	erts_do_break_handling();
 #endif
 
-
-#ifdef ERTS_SIGNAL_STATE /* ifndef ERTS_SMP */
-    if (ERTS_SIGNAL_STATE) {
-        erts_handle_signal_state();
-    }
-#endif
-
-
     if (poll_ret != 0) {
 	erts_atomic_set_nob(&pollset.in_poll_wait, 0);
 	forget_removed(&pollset);
@@ -2259,9 +2261,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 		    oready(state->driver.select->outport, state, current_cio_time);
 		}
 		/* Someone might have deselected input since revents
-		   was read (true also on the non-smp emulator since
-		   oready() may have been called); therefore, update
-		   revents... */
+		   was read therefore, update revents... */
 		revents &= state->events;
 		if (revents & ERTS_POLL_EV_IN) {
 		    iready(state->driver.select->inport, state, current_cio_time);
@@ -2502,6 +2502,31 @@ static void drv_ev_state_free(void *des)
 }
 #endif
 
+
+#ifdef ERTS_ENABLE_KERNEL_POLL
+
+struct io_functions {
+    int (*select)(ErlDrvPort, ErlDrvEvent, int, int);
+    int (*enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
+    int (*event)(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
+    void (*check_io_as_interrupt)(void);
+    void (*check_io_interrupt)(int);
+    void (*check_io_interrupt_tmd)(int, ErtsMonotonicTime);
+    void (*check_io)(int);
+    int (*max_files)(void);
+    Uint (*size)(void);
+    Eterm (*info)(void *);
+    int (*check_io_debug)(ErtsCheckIoDebugInfo *);
+};
+
+# ifdef ERTS_KERNEL_POLL_VERSION
+struct io_functions erts_io_funcs = {0};
+# else
+extern struct io_functions erts_io_funcs;
+# endif
+
+#endif /* ERTS_ENABLE_KERNEL_POLL */
+
 void
 ERTS_CIO_EXPORT(erts_init_check_io)(void)
 {
@@ -2512,6 +2537,20 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
     erts_atomic_init_nob(&erts_check_io_time, 0);
     erts_atomic_init_nob(&pollset.in_poll_wait, 0);
 
+#ifdef ERTS_ENABLE_KERNEL_POLL
+    ASSERT(erts_io_funcs.select == NULL);
+    erts_io_funcs.select		= ERTS_CIO_EXPORT(driver_select);
+    erts_io_funcs.enif_select		= ERTS_CIO_EXPORT(enif_select);
+    erts_io_funcs.event		= ERTS_CIO_EXPORT(driver_event);
+    erts_io_funcs.check_io_interrupt	= ERTS_CIO_EXPORT(erts_check_io_interrupt);
+    erts_io_funcs.check_io_interrupt_tmd= ERTS_CIO_EXPORT(erts_check_io_interrupt_timed);
+    erts_io_funcs.check_io		= ERTS_CIO_EXPORT(erts_check_io);
+    erts_io_funcs.max_files		= ERTS_CIO_EXPORT(erts_check_io_max_files);
+    erts_io_funcs.size		= ERTS_CIO_EXPORT(erts_check_io_size);
+    erts_io_funcs.info		= ERTS_CIO_EXPORT(erts_check_io_info);
+    erts_io_funcs.check_io_debug	= ERTS_CIO_EXPORT(erts_check_io_debug);
+#endif
+
     ERTS_CIO_POLL_INIT();
     pollset.ps = ERTS_CIO_NEW_POLLSET();
 
@@ -3097,7 +3136,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
 }
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
-void ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks)(int enable) {
+void erts_lcnt_update_cio_locks(int enable) {
 #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
     erts_lcnt_enable_hash_lock_count(&drv_ev_state_tab, ERTS_LOCK_FLAGS_CATEGORY_IO, enable);
 #else
@@ -3105,3 +3144,79 @@ void ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks)(int enable) {
 #endif
 }
 #endif /* ERTS_ENABLE_LOCK_COUNT */
+
+
+#ifdef ERTS_ENABLE_KERNEL_POLL
+# ifdef ERTS_KERNEL_POLL_VERSION
+
+/*
+ * Compile these only once for kp/nkp
+ */
+
+void erts_init_check_io(void)
+{
+    if (erts_use_kernel_poll)
+        erts_init_check_io_kp();
+    else
+        erts_init_check_io_nkp();
+}
+
+int
+driver_select(ErlDrvPort port, ErlDrvEvent event, int mode, int on)
+{
+    return (*erts_io_funcs.select)(port, event, mode, on);
+}
+
+int
+driver_event(ErlDrvPort port, ErlDrvEvent event, ErlDrvEventData event_data)
+{
+    return (*erts_io_funcs.event)(port, event, event_data);
+}
+
+int enif_select(ErlNifEnv* env, ErlNifEvent event,
+                enum ErlNifSelectFlags flags, void* obj, const ErlNifPid* pid, Eterm ref)
+{
+    return (*erts_io_funcs.enif_select)(env, event, flags, obj, pid, ref);
+}
+
+void erts_check_io(int do_wait)
+{
+    erts_io_funcs.check_io(do_wait);
+}
+
+Uint erts_check_io_size(void)
+{
+    return erts_io_funcs.size();
+}
+
+
+Eterm erts_check_io_info(void *p)
+{
+    return (*erts_io_funcs.info)(p);
+}
+
+int erts_check_io_max_files(void)
+{
+    return erts_io_funcs.max_files();
+}
+
+int
+erts_check_io_debug(ErtsCheckIoDebugInfo *ip)
+{
+    return (*erts_io_funcs.check_io_debug)(ip);
+}
+
+void erts_check_io_interrupt(int set)
+{
+    erts_io_funcs.check_io_interrupt(set);
+}
+
+void erts_check_io_interrupt_timed(int set,
+                                   ErtsMonotonicTime timeout_time)
+{
+    erts_io_funcs.check_io_interrupt_tmd(set, timeout_time);
+}
+
+#endif /* ERTS_KERNEL_POLL_VERSION */
+
+#endif /* !ERTS_ENABLE_KERNEL_POLL */
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index 777942a473..c4ab22aaff 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -30,38 +30,6 @@
 #include "sys.h"
 #include "erl_sys_driver.h"
 
-#ifdef ERTS_ENABLE_KERNEL_POLL
-
-int driver_select_kp(ErlDrvPort, ErlDrvEvent, int, int);
-int driver_select_nkp(ErlDrvPort, ErlDrvEvent, int, int);
-int enif_select_kp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
-int enif_select_nkp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
-int driver_event_kp(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
-int driver_event_nkp(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
-Uint erts_check_io_size_kp(void);
-Uint erts_check_io_size_nkp(void);
-Eterm erts_check_io_info_kp(void *);
-Eterm erts_check_io_info_nkp(void *);
-int erts_check_io_max_files_kp(void);
-int erts_check_io_max_files_nkp(void);
-void erts_check_io_interrupt_kp(int);
-void erts_check_io_interrupt_nkp(int);
-void erts_check_io_interrupt_timed_kp(int, ErtsMonotonicTime);
-void erts_check_io_interrupt_timed_nkp(int, ErtsMonotonicTime);
-void erts_check_io_kp(int);
-void erts_check_io_nkp(int);
-void erts_init_check_io_kp(void);
-void erts_init_check_io_nkp(void);
-int erts_check_io_debug_kp(ErtsCheckIoDebugInfo *);
-int erts_check_io_debug_nkp(ErtsCheckIoDebugInfo *);
-
-#ifdef ERTS_ENABLE_LOCK_COUNT
-void erts_lcnt_update_cio_locks_kp(int enable);
-void erts_lcnt_update_cio_locks_nkp(int enable);
-#endif
-
-#else /* !ERTS_ENABLE_KERNEL_POLL */
-
 Uint erts_check_io_size(void);
 Eterm erts_check_io_info(void *);
 int erts_check_io_max_files(void);
@@ -69,7 +37,6 @@ void erts_check_io_interrupt(int);
 void erts_check_io_interrupt_timed(int, ErtsMonotonicTime);
 void erts_check_io(int);
 void erts_init_check_io(void);
-
 #ifdef ERTS_ENABLE_LOCK_COUNT
 void erts_lcnt_update_cio_locks(int enable);
 #endif
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index d05028cabc..e88d7aa6df 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -131,121 +131,6 @@ static int replace_intr = 0;
 /* assume yes initially, ttsl_init will clear it */
 int using_oldshell = 1;
 
-#ifdef ERTS_ENABLE_KERNEL_POLL
-
-int erts_use_kernel_poll = 0;
-
-struct {
-    int (*select)(ErlDrvPort, ErlDrvEvent, int, int);
-    int (*enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
-    int (*event)(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
-    void (*check_io_as_interrupt)(void);
-    void (*check_io_interrupt)(int);
-    void (*check_io_interrupt_tmd)(int, ErtsMonotonicTime);
-    void (*check_io)(int);
-    Uint (*size)(void);
-    Eterm (*info)(void *);
-    int (*check_io_debug)(ErtsCheckIoDebugInfo *);
-} io_func = {0};
-
-
-int
-driver_select(ErlDrvPort port, ErlDrvEvent event, int mode, int on)
-{
-    return (*io_func.select)(port, event, mode, on);
-}
-
-int
-driver_event(ErlDrvPort port, ErlDrvEvent event, ErlDrvEventData event_data)
-{
-    return (*io_func.event)(port, event, event_data);
-}
-
-int enif_select(ErlNifEnv* env, ErlNifEvent event,
-                enum ErlNifSelectFlags flags, void* obj, const ErlNifPid* pid, Eterm ref)
-{
-    return (*io_func.enif_select)(env, event, flags, obj, pid, ref);
-}
-
-
-Eterm erts_check_io_info(void *p)
-{
-    return (*io_func.info)(p);
-}
-
-int
-erts_check_io_debug(ErtsCheckIoDebugInfo *ip)
-{
-    return (*io_func.check_io_debug)(ip);
-}
-
-
-static void
-init_check_io(void)
-{
-    if (erts_use_kernel_poll) {
-	io_func.select			= driver_select_kp;
-        io_func.enif_select		= enif_select_kp;
-	io_func.event			= driver_event_kp;
-	io_func.check_io_interrupt	= erts_check_io_interrupt_kp;
-	io_func.check_io_interrupt_tmd	= erts_check_io_interrupt_timed_kp;
-	io_func.check_io		= erts_check_io_kp;
-	io_func.size			= erts_check_io_size_kp;
-	io_func.info			= erts_check_io_info_kp;
-	io_func.check_io_debug		= erts_check_io_debug_kp;
-	erts_init_check_io_kp();
-	max_files = erts_check_io_max_files_kp();
-    }
-    else {
-	io_func.select			= driver_select_nkp;
-        io_func.enif_select		= enif_select_nkp;
-	io_func.event			= driver_event_nkp;
-	io_func.check_io_interrupt	= erts_check_io_interrupt_nkp;
-	io_func.check_io_interrupt_tmd	= erts_check_io_interrupt_timed_nkp;
-	io_func.check_io		= erts_check_io_nkp;
-	io_func.size			= erts_check_io_size_nkp;
-	io_func.info			= erts_check_io_info_nkp;
-	io_func.check_io_debug		= erts_check_io_debug_nkp;
-	erts_init_check_io_nkp();
-	max_files = erts_check_io_max_files_nkp();
-    }
-}
-
-#define ERTS_CHK_IO_AS_INTR()	(*io_func.check_io_interrupt)(1)
-#define ERTS_CHK_IO_INTR	(*io_func.check_io_interrupt)
-#define ERTS_CHK_IO_INTR_TMD	(*io_func.check_io_interrupt_tmd)
-#define ERTS_CHK_IO		(*io_func.check_io)
-#define ERTS_CHK_IO_SZ		(*io_func.size)
-
-#else /* !ERTS_ENABLE_KERNEL_POLL */
-
-static void
-init_check_io(void)
-{
-    erts_init_check_io();
-    max_files = erts_check_io_max_files();
-}
-
-#define ERTS_CHK_IO_AS_INTR()	erts_check_io_interrupt(1)
-#define ERTS_CHK_IO_INTR	erts_check_io_interrupt
-#define ERTS_CHK_IO_INTR_TMD	erts_check_io_interrupt_timed
-#define ERTS_CHK_IO		erts_check_io
-#define ERTS_CHK_IO_SZ		erts_check_io_size
-
-#endif
-
-void
-erts_sys_schedule_interrupt(int set)
-{
-    ERTS_CHK_IO_INTR(set);
-}
-
-void
-erts_sys_schedule_interrupt_timed(int set, ErtsMonotonicTime timeout_time)
-{
-    ERTS_CHK_IO_INTR_TMD(set, timeout_time);
-}
-
 UWord
 erts_sys_get_page_size(void)
 {
@@ -261,7 +146,7 @@ erts_sys_get_page_size(void)
 Uint
 erts_sys_misc_mem_sz(void)
 {
-    Uint res = ERTS_CHK_IO_SZ();
+    Uint res = erts_check_io_size();
     res += erts_atomic_read_mb(&sys_misc_mem_sz);
     return res;
 }
@@ -625,7 +510,7 @@ break_requested(void)
       erts_exit(ERTS_INTR_EXIT, "");
 
   ERTS_SET_BREAK_REQUESTED;
-  ERTS_CHK_IO_AS_INTR(); /* Make sure we don't sleep in poll */
+  erts_check_io_sig_interrupt();
 }
 
 static RETSIGTYPE request_break(int signum)
@@ -813,7 +698,7 @@ erts_sys_unix_later_init(void)
 
 int sys_max_files(void)
 {
-   return(max_files);
+   return max_files;
 }
 
 /************************** OS info *******************************/
@@ -1198,7 +1083,7 @@ erl_debug(char* fmt, ...)
 void
 erl_sys_schedule(int runnable)
 {
-    ERTS_CHK_IO(!runnable);
+    erts_check_io(!runnable);
     ERTS_LC_ASSERT(!erts_thr_progress_is_blocking());
 }
 
@@ -1417,6 +1302,8 @@ get_value(char* rest, char** argv, int* ip)
     return rest;
 }
 
+int erts_use_kernel_poll = 0;
+
 #endif /* ERTS_ENABLE_KERNEL_POLL */
 
 void
@@ -1475,7 +1362,9 @@ erl_sys_args(int* argc, char** argv)
     }
 #endif
 
-    init_check_io();
+    erts_init_check_io();
+    max_files = erts_check_io_max_files();
+
 
     init_smp_sig_notify();
     init_smp_sig_suspend();
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index b23dbecbac..de391f078b 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -3248,18 +3248,6 @@ erl_sys_late_init(void)
     /* do nothing */
 }
 
-void
-erts_sys_schedule_interrupt(int set)
-{
-    erts_check_io_interrupt(set);
-}
-
-void
-erts_sys_schedule_interrupt_timed(int set, ErtsMonotonicTime timeout_time)
-{
-    erts_check_io_interrupt_timed(set, timeout_time);
-}
-
 /*
  * Called from schedule() when it runs out of runnable processes,
  * or when Erlang code has performed INPUT_REDUCTIONS reduction
@@ -3271,4 +3259,3 @@ erl_sys_schedule(int runnable)
     erts_check_io(!runnable);
     ERTS_LC_ASSERT(!erts_thr_progress_is_blocking());
 }
-
-- 
cgit v1.2.3


From 7b0c3b7e51bf6b13c95565c99cd80191525295e1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 25 Apr 2017 17:42:37 +0200
Subject: erts: Remove undocumented driver_event

---
 erts/emulator/beam/erl_alloc.c                     |   2 -
 erts/emulator/beam/erl_alloc.types                 |   1 -
 erts/emulator/beam/erl_bif_info.c                  |   6 +-
 erts/emulator/beam/erl_driver.h                    |  15 +-
 erts/emulator/beam/erl_lock_count.c                |   4 +-
 erts/emulator/beam/erl_port_task.c                 |  29 --
 erts/emulator/beam/erl_port_task.h                 |   1 -
 erts/emulator/beam/erl_threads.h                   |   1 -
 erts/emulator/beam/erl_trace.c                     |   1 -
 erts/emulator/beam/erlang_lttng.h                  |  15 -
 erts/emulator/beam/global.h                        |   2 -
 erts/emulator/beam/io.c                            |  28 +-
 erts/emulator/beam/sys.h                           |   1 -
 erts/emulator/sys/common/erl_check_io.c            | 504 +--------------------
 erts/emulator/sys/common/erl_check_io.h            |  18 -
 erts/emulator/sys/unix/sys.c                       |   2 +-
 erts/emulator/sys/win32/erl_win_dyn_driver.h       |   6 +-
 erts/emulator/test/driver_SUITE.erl                |  26 +-
 erts/emulator/test/driver_SUITE_data/chkio_drv.c   | 308 +------------
 .../emulator/test/driver_SUITE_data/ioq_exit_drv.c |  75 +--
 .../test/driver_SUITE_data/missing_callback_drv.c  |  18 -
 erts/emulator/test/lttng_SUITE.erl                 |   3 -
 22 files changed, 58 insertions(+), 1008 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 845cef24c7..1c365f906b 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -644,8 +644,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	= ERTS_MONITOR_SH_SIZE * sizeof(Uint);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)]
 	= ERTS_LINK_SH_SIZE * sizeof(Uint);
-    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_EV_D_STATE)]
-	= sizeof(ErtsDrvEventDataState);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)]
 	= sizeof(ErtsDrvSelectDataState);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_SEL_D_STATE)]
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 11884299e2..3ec1389d47 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -351,7 +351,6 @@ type	SYS_CHECK_REQ	SHORT_LIVED	SYSTEM		system_check_request
 type  	TEMP_TERM 	TEMPORARY	SYSTEM		temp_term
 type	DRV_TAB		LONG_LIVED	SYSTEM		drv_tab
 type	DRV_EV_STATE	LONG_LIVED	SYSTEM		driver_event_state
-type	DRV_EV_D_STATE	FIXED_SIZE	SYSTEM		driver_event_data_state
 type	DRV_SEL_D_STATE	FIXED_SIZE	SYSTEM		driver_select_data_state
 type	NIF_SEL_D_STATE	FIXED_SIZE	SYSTEM		enif_select_data_state
 type	FD_LIST		SHORT_LIVED	SYSTEM		fd_list
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index d296e794d5..3b6e83c242 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3574,15 +3574,13 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    szp = &sz;
 	    hpp = NULL;
 	    while (1) {
-		res = erts_bld_tuple(hpp, szp, 4,
+		res = erts_bld_tuple(hpp, szp, 3,
 				     erts_bld_uint(hpp, szp,
 						   (Uint) no_errors),
 				     erts_bld_uint(hpp, szp,
 						   (Uint) ciodi.no_used_fds),
 				     erts_bld_uint(hpp, szp,
-						   (Uint) ciodi.no_driver_select_structs),
-				     erts_bld_uint(hpp, szp,
-						   (Uint) ciodi.no_driver_event_structs));
+						   (Uint) ciodi.no_driver_select_structs));
 		if (hpp)
 		    break;
 		hp = HAlloc(BIF_P, sz);
diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 5ad616fec3..d5379a40d5 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -148,14 +148,6 @@ typedef struct _erl_drv_event* ErlDrvEvent; /* An event to be selected on. */
 typedef struct _erl_drv_port* ErlDrvPort; /* A port descriptor. */
 typedef struct _erl_drv_port* ErlDrvThreadData; /* Thread data. */
 
-#if !defined(__WIN32__) && !defined(_WIN32) && !defined(_WIN32_) && !defined(USE_SELECT)
-struct erl_drv_event_data {
-    short events;
-    short revents;
-};
-#endif
-typedef struct erl_drv_event_data *ErlDrvEventData; /* Event data */
-
 typedef struct {
     unsigned long megasecs;
     unsigned long secs;
@@ -270,10 +262,7 @@ typedef struct erl_drv_entry {
 			 unsigned int *flags); /* Works mostly like 'control',
 						  a synchronous
 						  call into the driver. */
-    void (*event)(ErlDrvData drv_data, ErlDrvEvent event,
-		  ErlDrvEventData event_data);
-                                /* Called when an event selected by 
-				   driver_event() has occurred */
+    void (*unused_event_callback)(void);
     int extended_marker;	/* ERL_DRV_EXTENDED_MARKER */
     int major_version;		/* ERL_DRV_EXTENDED_MAJOR_VERSION */
     int minor_version;		/* ERL_DRV_EXTENDED_MINOR_VERSION */
@@ -340,8 +329,6 @@ EXTERN void erl_drv_busy_msgq_limits(ErlDrvPort port,
 				     ErlDrvSizeT *high);
 
 EXTERN int driver_select(ErlDrvPort port, ErlDrvEvent event, int mode, int on);
-EXTERN int driver_event(ErlDrvPort port, ErlDrvEvent event, 
-			ErlDrvEventData event_data);
 
 EXTERN int driver_output(ErlDrvPort port, char *buf, ErlDrvSizeT len);
 EXTERN int driver_output2(ErlDrvPort port, char *hbuf, ErlDrvSizeT hlen,
diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index d2e8f47d59..11890e44d1 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -593,16 +593,14 @@ void erts_lcnt_set_category_mask(erts_lock_flags_t mask) {
 #ifdef ERTS_ENABLE_KERNEL_POLL
         if(erts_use_kernel_poll) {
             erts_lcnt_update_pollset_locks_kp(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
-            erts_lcnt_update_cio_locks_kp(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
         } else {
             erts_lcnt_update_pollset_locks_nkp(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
-            erts_lcnt_update_cio_locks_nkp(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
         }
 #else
         erts_lcnt_update_pollset_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
-        erts_lcnt_update_cio_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
 #endif
 
+        erts_lcnt_update_cio_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
         erts_lcnt_update_driver_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
         erts_lcnt_update_port_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
     }
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index b9152a7b5e..bdce811847 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -100,7 +100,6 @@ erts_atomic_t erts_port_task_outstanding_io_tasks;
 typedef union {
     struct { /* I/O tasks */
 	ErlDrvEvent event;
-	ErlDrvEventData event_data;
     } io;
     struct {
 	ErtsProc2PortSigCallback callback;
@@ -1315,7 +1314,6 @@ erts_port_task_abort(ErtsPortTaskHandle *pthp)
 	    switch (ptp->type) {
 	    case ERTS_PORT_TASK_INPUT:
 	    case ERTS_PORT_TASK_OUTPUT:
-	    case ERTS_PORT_TASK_EVENT:
 		ASSERT(erts_atomic_read_nob(
 			   &erts_port_task_outstanding_io_tasks) > 0);
 		erts_atomic_dec_relb(&erts_port_task_outstanding_io_tasks);
@@ -1463,15 +1461,6 @@ erts_port_task_schedule(Eterm id,
 	erts_atomic_inc_relb(&erts_port_task_outstanding_io_tasks);
 	break;
     }
-    case ERTS_PORT_TASK_EVENT: {
-	va_list argp;
-	va_start(argp, type);
-	ptp->u.alive.td.io.event = va_arg(argp, ErlDrvEvent);
-	ptp->u.alive.td.io.event_data = va_arg(argp, ErlDrvEventData);
-	va_end(argp);
-	erts_atomic_inc_relb(&erts_port_task_outstanding_io_tasks);
-	break;
-    }
     case ERTS_PORT_TASK_PROC_SIG: {
 	va_list argp;
 	va_start(argp, type);
@@ -1769,17 +1758,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    reset_executed_io_task_handle(ptp);
 	    io_tasks_executed++;
 	    break;
-	case ERTS_PORT_TASK_EVENT:
-	    reds = ERTS_PORT_REDS_EVENT;
-	    ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0);
-            DTRACE_DRIVER(driver_event, pp);
-            LTTNG_DRIVER(driver_event, pp);
-	    (*pp->drv_ptr->event)((ErlDrvData) pp->drv_data,
-				  ptp->u.alive.td.io.event,
-				  ptp->u.alive.td.io.event_data);
-	    reset_executed_io_task_handle(ptp);
-	    io_tasks_executed++;
-	    break;
 	case ERTS_PORT_TASK_PROC_SIG: {
 	    ErtsProc2PortSigData *sigdp = &ptp->u.alive.td.psig.data;
 	    reset_handle(ptp);
@@ -2032,13 +2010,6 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
 				      DO_WRITE,
 				      1);
 		break;
-	    case ERTS_PORT_TASK_EVENT:
-		erts_stale_drv_select(pp->common.id,
-				      ERTS_Port2ErlDrvPort(pp),
-				      ptp->u.alive.td.io.event,
-				      0,
-				      1);
-		break;
 	    case ERTS_PORT_TASK_DIST_CMD:
 		break;
 	    case ERTS_PORT_TASK_PROC_SIG: {
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index 561f4ca936..ffd42c9bab 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -56,7 +56,6 @@ typedef erts_atomic_t ErtsPortTaskHandle;
 typedef enum {
     ERTS_PORT_TASK_INPUT,
     ERTS_PORT_TASK_OUTPUT,
-    ERTS_PORT_TASK_EVENT,
     ERTS_PORT_TASK_TIMEOUT,
     ERTS_PORT_TASK_DIST_CMD,
     ERTS_PORT_TASK_PROC_SIG
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index e306df818d..aedceb6fc2 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -454,7 +454,6 @@ ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx);
 ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx);
-
 ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock,
                                         char *name,
                                         Eterm extra,
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index a07e3642f6..4b996d8fc2 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1449,7 +1449,6 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time)
     case ERTS_PORT_TASK_TIMEOUT: op = am_timeout; break;
     case ERTS_PORT_TASK_INPUT: op = am_input; break;
     case ERTS_PORT_TASK_OUTPUT: op = am_output; break;
-    case ERTS_PORT_TASK_EVENT: op = am_event; break;
     case ERTS_PORT_TASK_DIST_CMD: op = am_dist_cmd; break;
     default: op = am_undefined; break;
     }
diff --git a/erts/emulator/beam/erlang_lttng.h b/erts/emulator/beam/erlang_lttng.h
index 4e869671f7..feb05f4f4c 100644
--- a/erts/emulator/beam/erlang_lttng.h
+++ b/erts/emulator/beam/erlang_lttng.h
@@ -157,21 +157,6 @@ TRACEPOINT_EVENT(
     )
 )
 
-TRACEPOINT_EVENT(
-    org_erlang_otp,
-    driver_event,
-    TP_ARGS(
-        char*, pid,
-        char*, port,
-        char*, driver
-    ),
-    TP_FIELDS(
-        ctf_string(pid, pid)
-        ctf_string(port, port)
-        ctf_string(driver, driver)
-    )
-)
-
 TRACEPOINT_EVENT(
     org_erlang_otp,
     driver_timeout,
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 2b0ad0b98a..c3f8e92f13 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -222,8 +222,6 @@ struct erts_driver_t_ {
 			 char *buf, ErlDrvSizeT len,
 			 char **rbuf, ErlDrvSizeT rlen, /* Might be NULL */
 			 unsigned int *flags);
-    void (*event)(ErlDrvData drv_data, ErlDrvEvent event,
-		  ErlDrvEventData event_data);
     void (*ready_input)(ErlDrvData drv_data, ErlDrvEvent event); 
     void (*ready_output)(ErlDrvData drv_data, ErlDrvEvent event);  
     void (*timeout)(ErlDrvData drv_data);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index bc1b9b6ef4..85013af3ad 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -5390,24 +5390,17 @@ erts_stale_drv_select(Eterm port,
     switch (mode) {
     case ERL_DRV_READ | ERL_DRV_WRITE:
 	type = "Input/Output";
-	goto deselect;
     case ERL_DRV_WRITE:
 	type = "Output";
-	goto deselect;
     case ERL_DRV_READ:
 	type = "Input";
-    deselect:
-	if (deselect) {
-	    driver_select(drv_port, hndl,
-			  mode | ERL_DRV_USE_NO_CALLBACK,
-			  0);
-	}
-	break;
     default:
-	type = "Event";
-	if (deselect)
-	    driver_event(drv_port, hndl, NULL);
-	break;
+        type = "";
+    }
+    if (deselect) {
+        driver_select(drv_port, hndl,
+                      mode | ERL_DRV_USE_NO_CALLBACK,
+                      0);
     }
 
     dsbufp = erts_create_logger_dsbuf();
@@ -7510,14 +7503,6 @@ no_output_callback(ErlDrvData drv_data, char *buf, ErlDrvSizeT len)
 
 }
 
-static void
-no_event_callback(ErlDrvData drv_data, ErlDrvEvent event, ErlDrvEventData event_data)
-{
-    Port *prt = get_current_port();
-    report_missing_drv_callback(prt, "Event", "event()");
-    driver_event(ERTS_Port2ErlDrvPort(prt), event, NULL);
-}
-
 static void
 no_ready_input_callback(ErlDrvData drv_data, ErlDrvEvent event)
 {
@@ -7585,7 +7570,6 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
     drv->outputv = de->outputv;
     drv->control = de->control;
     drv->call = de->call;
-    drv->event = de->event ? de->event : no_event_callback;
     drv->ready_input = de->ready_input ? de->ready_input : no_ready_input_callback;
     drv->ready_output = de->ready_output ? de->ready_output : no_ready_output_callback;
     drv->timeout = de->timeout ? de->timeout : no_timeout_callback;
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 3dd6ef333b..e2a21c0ecd 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -790,7 +790,6 @@ void fini_getenv_state(GETENV_STATE *);
 typedef struct {
     int no_used_fds;
     int no_driver_select_structs;
-    int no_driver_event_structs;
 } ErtsCheckIoDebugInfo;
 int erts_check_io_debug(ErtsCheckIoDebugInfo *ip);
 
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index bed815c64b..cd695e236f 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -52,10 +52,9 @@
 typedef char EventStateType;
 #define ERTS_EV_TYPE_NONE     ((EventStateType) 0)
 #define ERTS_EV_TYPE_DRV_SEL  ((EventStateType) 1) /* driver_select */
-#define ERTS_EV_TYPE_DRV_EV   ((EventStateType) 2) /* driver_event */
-#define ERTS_EV_TYPE_STOP_USE ((EventStateType) 3) /* pending stop_select */
-#define ERTS_EV_TYPE_NIF      ((EventStateType) 4) /* enif_select */
-#define ERTS_EV_TYPE_STOP_NIF ((EventStateType) 5) /* pending nif stop */
+#define ERTS_EV_TYPE_STOP_USE ((EventStateType) 2) /* pending stop_select */
+#define ERTS_EV_TYPE_NIF      ((EventStateType) 3) /* enif_select */
+#define ERTS_EV_TYPE_STOP_NIF ((EventStateType) 4) /* pending nif stop */
 
 typedef char EventStateFlags;
 #define ERTS_EV_FLAG_USED         ((EventStateFlags) 1)   /* ERL_DRV_USE has been turned on */
@@ -78,9 +77,6 @@ typedef char EventStateFlags;
 #  define ERTS_CIO_EXPORT(FUNC) FUNC
 #endif
 
-#define ERTS_CIO_HAVE_DRV_EVENT \
-  (ERTS_POLL_USE_POLL && !ERTS_POLL_USE_KERNEL_POLL)
-
 #define ERTS_CIO_POLL_CTL	ERTS_POLL_EXPORT(erts_poll_control)
 #define ERTS_CIO_POLL_CTLV	ERTS_POLL_EXPORT(erts_poll_controlv)
 #define ERTS_CIO_POLL_WAIT	ERTS_POLL_EXPORT(erts_poll_wait)
@@ -116,7 +112,6 @@ void erts_init_check_io_kp(void);
 void erts_init_check_io_nkp(void);
 int ERTS_CIO_EXPORT(driver_select)(ErlDrvPort, ErlDrvEvent, int, int);
 int ERTS_CIO_EXPORT(enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
-int ERTS_CIO_EXPORT(driver_event)(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
 Uint ERTS_CIO_EXPORT(erts_check_io_size)(void);
 Eterm ERTS_CIO_EXPORT(erts_check_io_info)(void *);
 int ERTS_CIO_EXPORT(erts_check_io_max_files)(void);
@@ -124,6 +119,9 @@ void ERTS_CIO_EXPORT(erts_check_io_interrupt)(int);
 void ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(int, ErtsMonotonicTime);
 void ERTS_CIO_EXPORT(erts_check_io)(int);
 int ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *);
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks)(int enable);
+#endif
 #endif
 
 
@@ -134,9 +132,6 @@ typedef struct {
     ErtsSysFdType fd;
     struct {
 	ErtsDrvSelectDataState *select;   /* ERTS_EV_TYPE_DRV_SEL */
-#if ERTS_CIO_HAVE_DRV_EVENT
-	ErtsDrvEventDataState *event;     /* ERTS_EV_TYPE_DRV_EV */
-#endif
         ErtsNifSelectDataState *nif;      /* ERTS_EV_TYPE_NIF */
         union {
             erts_driver_t*  drv_ptr;    /* ERTS_EV_TYPE_STOP_USE */
@@ -205,9 +200,6 @@ static ERTS_INLINE ErtsDrvEventState* hash_new_drv_ev_state(ErtsSysFdType fd)
     ErtsDrvEventState tmpl;
     tmpl.fd = fd;
     tmpl.driver.select = NULL;
-#if ERTS_CIO_HAVE_DRV_EVENT
-    tmpl.driver.event = NULL;
-#endif
     tmpl.driver.nif = NULL;
     tmpl.driver.stop.drv_ptr = NULL;
     tmpl.events = 0;
@@ -240,15 +232,6 @@ static void print_nif_select_op(erts_dsprintf_buf_t*, ErtsSysFdType,
 static void drv_select_large_fd_error(ErlDrvPort, ErtsSysFdType, int, int);
 static void nif_select_large_fd_error(ErtsSysFdType, int, ErtsResource*,Eterm ref);
 #endif
-#if ERTS_CIO_HAVE_DRV_EVENT
-static void drv_event_steal(ErlDrvPort ix, ErtsDrvEventState *state,
-			ErlDrvEventData event_data);
-static void print_drv_event_op(erts_dsprintf_buf_t *dsbufp,
-                               ErlDrvPort, ErtsSysFdType, ErlDrvEventData);
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-static void event_large_fd_error(ErlDrvPort, ErtsSysFdType, ErlDrvEventData);
-#endif
-#endif
 static void
 steal_pending_stop_use(erts_dsprintf_buf_t*, ErlDrvPort, ErtsDrvEventState*,
                        int mode, int on);
@@ -313,32 +296,6 @@ free_nif_select_data(ErtsNifSelectDataState *dsp)
     erts_free(ERTS_ALC_T_NIF_SEL_D_STATE, dsp);
 }
 
-#if ERTS_CIO_HAVE_DRV_EVENT
-
-static ERTS_INLINE ErtsDrvEventDataState *
-alloc_drv_event_data(void)
-{
-    ErtsDrvEventDataState *dep = erts_alloc(ERTS_ALC_T_DRV_EV_D_STATE,
-					    sizeof(ErtsDrvEventDataState));
-    dep->port = NIL;
-    dep->data = NULL;
-    dep->removed_events = 0;
-#if ERTS_CIO_DEFER_ACTIVE_EVENTS
-    dep->deferred_events = 0;
-#endif
-    init_iotask(&dep->iotask);
-    return dep;
-}
-
-static ERTS_INLINE void
-free_drv_event_data(ErtsDrvEventDataState *dep)
-{
-    ASSERT(!erts_port_task_is_scheduled(&dep->iotask.task));
-    erts_free(ERTS_ALC_T_DRV_EV_D_STATE, dep);   
-}
-
-#endif /* ERTS_CIO_HAVE_DRV_EVENT */
-
 static ERTS_INLINE void
 remember_removed(ErtsDrvEventState *state, struct pollset_info* psi)
 {
@@ -432,7 +389,6 @@ forget_removed(struct pollset_info* psi)
 #endif
 		break;
 	    case ERTS_EV_TYPE_DRV_SEL:
-	    case ERTS_EV_TYPE_DRV_EV:
 		break;
 	    default:
 		ASSERT(0);
@@ -487,9 +443,6 @@ grow_drv_ev_state(int min_ix)
 	for (i = old_len; i < new_len; i++) {
 	    drv_ev_state[i].fd = (ErtsSysFdType) i;
 	    drv_ev_state[i].driver.select = NULL;
-#if ERTS_CIO_HAVE_DRV_EVENT
-	    drv_ev_state[i].driver.event = NULL;
-#endif
 	    drv_ev_state[i].driver.stop.drv_ptr = NULL;
             drv_ev_state[i].driver.nif = NULL;
 	    drv_ev_state[i].events = 0;
@@ -524,13 +477,6 @@ abort_tasks(ErtsDrvEventState *state, int mode)
     switch (mode) {
     case 0: check_type:
 	switch (state->type) {
-#if ERTS_CIO_HAVE_DRV_EVENT
-	case ERTS_EV_TYPE_DRV_EV:
-	    abort_task(state->driver.event->port,
-		       &state->driver.event->iotask.task,
-		       ERTS_EV_TYPE_DRV_EV);
-	    return;
-#endif
         case ERTS_EV_TYPE_NIF:
 	case ERTS_EV_TYPE_NONE:
 	    return;
@@ -598,13 +544,6 @@ deselect(ErtsDrvEventState *state, int mode)
 	    state->driver.select->inport = NIL;
 	    state->driver.select->outport = NIL;
 	    break;
-#if ERTS_CIO_HAVE_DRV_EVENT
-	case ERTS_EV_TYPE_DRV_EV:
-	    state->driver.event->port = NIL;
-	    state->driver.event->data = NULL;
-	    state->driver.event->removed_events = (ErtsPollEvents) 0;
-	    break;
-#endif
 	case ERTS_EV_TYPE_NONE:
 	    break;
 	default:
@@ -626,9 +565,6 @@ deselect(ErtsDrvEventState *state, int mode)
 
 static ERTS_INLINE void
 check_fd_cleanup(ErtsDrvEventState *state,
-#if ERTS_CIO_HAVE_DRV_EVENT
-		 ErtsDrvEventDataState **free_event,
-#endif
 		 ErtsDrvSelectDataState **free_select,
                  ErtsNifSelectDataState **free_nif)
 {
@@ -653,23 +589,9 @@ check_fd_cleanup(ErtsDrvEventState *state,
         state->driver.nif = NULL;
     }
 
-#if ERTS_CIO_HAVE_DRV_EVENT
-    *free_event = NULL;
-    if (state->driver.event
-	&& (state->type != ERTS_EV_TYPE_DRV_EV)
-	&& !is_iotask_active(&state->driver.event->iotask,  current_cio_time)) {
-	
-	*free_event = state->driver.event;
-	state->driver.event = NULL;
-    }
-#endif
-
 #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
     if (((state->type != ERTS_EV_TYPE_NONE)
 	 | state->remove_cnt
-#if ERTS_CIO_HAVE_DRV_EVENT
-	 | (state->driver.event != NULL)
-#endif
 	 | (state->driver.select != NULL)) == 0) {
 
 	hash_erase_drv_ev_state(state);
@@ -692,9 +614,6 @@ check_cleanup_active_fd(ErtsSysFdType fd,
     erts_mtx_t *mtx = fd_mtx(fd);
     void *free_select = NULL;
     void *free_nif = NULL;
-#if ERTS_CIO_HAVE_DRV_EVENT
-    void *free_event = NULL;
-#endif
 #if ERTS_CIO_DEFER_ACTIVE_EVENTS
     ErtsPollEvents evon = 0, evoff = 0;
 #endif
@@ -784,35 +703,6 @@ check_cleanup_active_fd(ErtsSysFdType fd,
             }
         }
 
-#if ERTS_CIO_HAVE_DRV_EVENT
-	if (state->driver.event) {
-	    if (is_iotask_active(&state->driver.event->iotask, current_cio_time)) {
-#if ERTS_CIO_DEFER_ACTIVE_EVENTS
-		ErtsPollEvents evs = state->events & ~state->driver.event->deferred_events;
-		if (evs) {
-		    evoff |= evs;
-		    state->driver.event->deferred_events |= evs;
-		}
-#endif
-		active = 1;
-	    }
-	    else if (state->type != ERTS_EV_TYPE_DRV_EV) {
-		free_event = state->driver.event;
-		state->driver.event = NULL;
-	    }
-#if ERTS_CIO_DEFER_ACTIVE_EVENTS
-	    else {
-		ErtsPollEvents evs = state->events & state->driver.event->deferred_events;
-		if (evs) {
-		    evon |= evs;
-		    state->driver.event->deferred_events = 0;
-		}
-	    }
-#endif
-
-	}
-#endif
-
 #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
 	if (((state->type != ERTS_EV_TYPE_NONE) | state->remove_cnt | active) == 0)
 	    hash_erase_drv_ev_state(state);
@@ -826,10 +716,6 @@ check_cleanup_active_fd(ErtsSysFdType fd,
 	free_drv_select_data(free_select);
     if (free_nif)
         free_nif_select_data(free_nif);
-#if ERTS_CIO_HAVE_DRV_EVENT
-    if (free_event)
-	free_drv_event_data(free_event);
-#endif
 
 #if ERTS_CIO_DEFER_ACTIVE_EVENTS
     if (evoff) {
@@ -967,9 +853,6 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
     ErtsDrvEventState *state;
     int wake_poller;
     int ret;
-#if ERTS_CIO_HAVE_DRV_EVENT
-    ErtsDrvEventDataState *free_event = NULL;
-#endif
     ErtsDrvSelectDataState *free_select = NULL;
     ErtsNifSelectDataState *free_nif = NULL;
 #ifdef USE_VM_PROBES
@@ -1026,9 +909,6 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
 #endif
 
     switch (state->type) {
-#if ERTS_CIO_HAVE_DRV_EVENT
-    case ERTS_EV_TYPE_DRV_EV:
-#endif
     case ERTS_EV_TYPE_NIF:
         drv_select_steal(ix, state, mode, on);
         break;
@@ -1168,9 +1048,6 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
 done:
 
     check_fd_cleanup(state,
-#if ERTS_CIO_HAVE_DRV_EVENT
-		     &free_event,
-#endif
 		     &free_select,
                      &free_nif);
 
@@ -1188,10 +1065,6 @@ done_unknown:
     if (free_nif)
         free_nif_select_data(free_nif);
 
-#if ERTS_CIO_HAVE_DRV_EVENT
-    if (free_event)
-	free_drv_event_data(free_event);
-#endif
     return ret;
 }
 
@@ -1212,9 +1085,6 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
     int wake_poller;
     int ret;
     enum { NO_STOP=0, CALL_STOP, CALL_STOP_AND_RELEASE } call_stop = NO_STOP;
-#if ERTS_CIO_HAVE_DRV_EVENT
-    ErtsDrvEventDataState *free_event = NULL;
-#endif
     ErtsDrvSelectDataState *free_select = NULL;
     ErtsNifSelectDataState *free_nif = NULL;
 #ifdef USE_VM_PROBES
@@ -1284,9 +1154,6 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
         if (state->driver.stop.resource != resource)
             nif_select_steal(state, ERL_DRV_READ | ERL_DRV_WRITE, resource, ref);
         break;
-#if ERTS_CIO_HAVE_DRV_EVENT
-    case ERTS_EV_TYPE_DRV_EV:
-#endif
     case ERTS_EV_TYPE_DRV_SEL:
         nif_select_steal(state, mode, resource, ref);
         break;
@@ -1423,9 +1290,6 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
 done:
 
     check_fd_cleanup(state,
-#if ERTS_CIO_HAVE_DRV_EVENT
-		     &free_event,
-#endif
 		     &free_select,
                      &free_nif);
 
@@ -1442,162 +1306,9 @@ done_unknown:
     if (free_nif)
         free_nif_select_data(free_nif);
 
-#if ERTS_CIO_HAVE_DRV_EVENT
-    if (free_event)
-	free_drv_event_data(free_event);
-#endif
     return ret;
 }
 
-
-int
-ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix,
-			      ErlDrvEvent e,
-			      ErlDrvEventData event_data)
-{
-#if !ERTS_CIO_HAVE_DRV_EVENT
-    return -1;
-#else
-    ErtsSysFdType fd = (ErtsSysFdType) e;
-    ErtsPollEvents events;
-    ErtsPollEvents add_events;
-    ErtsPollEvents remove_events;
-    Eterm id = erts_drvport2id(ix);
-    ErtsDrvEventState *state;
-    int do_wake = 0;
-    int ret;
-#if ERTS_CIO_HAVE_DRV_EVENT
-    ErtsDrvEventDataState *free_event;
-#endif
-    ErtsDrvSelectDataState *free_select;
-    ErtsNifSelectDataState *free_nif;
-    Port *prt = erts_drvport2port(ix);
-
-    if (prt == ERTS_INVALID_ERL_DRV_PORT)
-	return -1;
-
-    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
-
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state_len)) {
-	if (fd < 0)
-	    return -1;    
-	if (fd >= max_fds) {
-	    event_large_fd_error(ix, fd, event_data);
-	    return -1;
-	}
-	grow_drv_ev_state(fd);
-    }
-#endif
-
-    erts_mtx_lock(fd_mtx(fd));
-
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    state = &drv_ev_state[(int) fd];
-#else
-    /* Could use hash_new directly, but want to keep the normal case fast */
-    state = hash_get_drv_ev_state(fd);
-    if (state == NULL) {
-	state = hash_new_drv_ev_state(fd);
-    }
-#endif
-
-    switch (state->type) {
-    case ERTS_EV_TYPE_DRV_EV:
-	if (state->driver.event->port == id) break;
-	/*fall through*/
-    case ERTS_EV_TYPE_DRV_SEL:
-	drv_event_steal(ix, state, event_data);
-	break;
-    case ERTS_EV_TYPE_STOP_USE: {
-	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	print_drv_event_op(dsbufp, ix, fd, event_data);
-	steal_pending_stop_use(dsbufp, ix, state, 0, 1);
-	break;
-      }
-    }
-
-    ASSERT(state->type == ERTS_EV_TYPE_DRV_EV
-	   || state->type == ERTS_EV_TYPE_NONE);
-
-    events = state->events;
-
-    if (!event_data) {
-	remove_events = events;
-	add_events = 0;
-    }
-    else {
-	remove_events = ~event_data->events & events;
-	add_events = ~events & event_data->events;
-    }
-
-    if (add_events) {
-	events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, add_events, 1, &do_wake);
-	if (events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
-	    ret = -1;
-	    goto done;
-	}
-    }
-    if (remove_events) {
-	events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, remove_events, 0, &do_wake);
-	if (events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
-	    ret = -1;
-	    goto done;
-	}
-    }
-    if (event_data && event_data->events != 0) {
-	if (state->type == ERTS_EV_TYPE_DRV_EV) {
-	    state->driver.event->removed_events &= ~add_events;
-	    state->driver.event->removed_events |= remove_events;
-	}
-	else {
-	    if (!state->driver.event)
-		state->driver.event = alloc_drv_event_data();
-	    state->driver.event->port = id;
-	    state->driver.event->removed_events = (ErtsPollEvents) 0;
-	    state->type = ERTS_EV_TYPE_DRV_EV;
-	}
-	state->driver.event->data = event_data;
-    }
-    else {
-	if (state->type == ERTS_EV_TYPE_DRV_EV) {
-	    abort_tasks(state, 0);
-	    state->driver.event->port = NIL;
-	    state->driver.event->data = NULL;
-	    state->driver.event->removed_events = (ErtsPollEvents) 0;
-	}
-	state->type = ERTS_EV_TYPE_NONE;
-	remember_removed(state, &pollset);
-    }
-    state->events = events;
-    ASSERT(event_data ? events == event_data->events : events == 0); 
-
-    ret = 0;
-
-done:
-
-    check_fd_cleanup(state,
-#if ERTS_CIO_HAVE_DRV_EVENT
-		     &free_event,
-#endif
-		     &free_select,
-                     &free_nif);
-
-    erts_mtx_unlock(fd_mtx(fd));
-
-    if (free_select)
-	free_drv_select_data(free_select);
-    if (free_nif)
-        free_nif_select_data(free_nif);
-#if ERTS_CIO_HAVE_DRV_EVENT
-    if (free_event)
-	free_drv_event_data(free_event);
-#endif
-
-    return ret;
-#endif
-}
-
 static ERTS_INLINE int
 chk_stale(Eterm id, ErtsDrvEventState *state, int mode)
 {
@@ -1629,11 +1340,6 @@ need2steal(ErtsDrvEventState *state, int mode)
         do_steal = 1;
         break;
 
-#if ERTS_CIO_HAVE_DRV_EVENT
-    case ERTS_EV_TYPE_DRV_EV:
-	do_steal |= chk_stale(state->driver.event->port, state, 0);
-	break;
-#endif
     case ERTS_EV_TYPE_STOP_USE:
     case ERTS_EV_TYPE_STOP_NIF:
         ASSERT(0);
@@ -1716,23 +1422,6 @@ steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode)
         erts_dsprintf(dsbufp, "\n");
         break;
     }
-#if ERTS_CIO_HAVE_DRV_EVENT
-    case ERTS_EV_TYPE_DRV_EV: {
-	Eterm eid = state->driver.event->port;
-	if (is_nil(eid)) {
-	    erts_dsprintf(dsbufp, "no one", (int) state->fd);
-	    ASSERT(0);
-	}
-	else {
-	    erts_dsprintf(dsbufp, "event driver ");
-	    print_driver_name(dsbufp, eid);
-	    erts_dsprintf(dsbufp, "%T ", eid);
-	}
-	erts_dsprintf(dsbufp, "\n");
-	deselect(state, 0);
-	break;
-    }
-#endif
     case ERTS_EV_TYPE_STOP_USE:
     case ERTS_EV_TYPE_STOP_NIF: {
 	ASSERT(0);
@@ -1914,55 +1603,6 @@ steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource* resource,
 
 }
 
-
-#if ERTS_CIO_HAVE_DRV_EVENT
-
-static void
-print_drv_event_op(erts_dsprintf_buf_t *dsbufp,
-                   ErlDrvPort ix, ErtsSysFdType fd, ErlDrvEventData event_data)
-{
-    Port *pp = erts_drvport2port(ix);
-    erts_dsprintf(dsbufp, "driver_event(%p, %d, ", ix, (int) fd);
-    if (!event_data)
-	erts_dsprintf(dsbufp, "NULL");
-    else
-	erts_dsprintf(dsbufp, "{0x%x, 0x%x}",
-		      (unsigned int) event_data->events,
-		      (unsigned int) event_data->revents);
-    erts_dsprintf(dsbufp, ") by ");
-    if (pp != ERTS_INVALID_ERL_DRV_PORT)
-	print_driver_name(dsbufp, pp->common.id);
-    erts_dsprintf(dsbufp, "driver %T ", pp != ERTS_INVALID_ERL_DRV_PORT ? pp->common.id : NIL);
-}
-
-static void
-drv_event_steal(ErlDrvPort ix, ErtsDrvEventState *state, ErlDrvEventData event_data)
-{
-    if (need2steal(state, ERL_DRV_READ|ERL_DRV_WRITE)) {
-	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	print_drv_event_op(dsbufp, ix, state->fd, event_data);
-	steal(dsbufp, state, ERL_DRV_READ|ERL_DRV_WRITE);
-	erts_send_error_to_logger_nogl(dsbufp);
-    }
-    else if (state->type == ERTS_EV_TYPE_DRV_SEL) {
-	ASSERT(state->flags & ERTS_EV_FLAG_USED);
-	deselect(state, 0);
-    }
-}
-
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-static void
-event_large_fd_error(ErlDrvPort ix, ErtsSysFdType fd, ErlDrvEventData event_data)
-{
-    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-    print_drv_event_op(dsbufp, ix, fd, event_data);
-    erts_dsprintf(dsbufp, "failed: ");
-    large_fd_error_common(dsbufp, fd);
-    erts_send_error_to_logger_nogl(dsbufp);
-}
-#endif
-#endif
-
 static ERTS_INLINE int
 io_task_schedule_allowed(ErtsDrvEventState *state,
 			 ErtsPortTaskType type,
@@ -1974,30 +1614,13 @@ io_task_schedule_allowed(ErtsDrvEventState *state,
     case ERTS_PORT_TASK_INPUT:
 	if (!state->driver.select)
 	    return 0;
-#if ERTS_CIO_HAVE_DRV_EVENT
-	if (state->driver.event)
-	    return 0;
-#endif
 	io_task = &state->driver.select->iniotask;
 	break;
     case ERTS_PORT_TASK_OUTPUT:
 	if (!state->driver.select)
 	    return 0;
-#if ERTS_CIO_HAVE_DRV_EVENT
-	if (state->driver.event)
-	    return 0;
-#endif
 	io_task = &state->driver.select->outiotask;
 	break;
-#if ERTS_CIO_HAVE_DRV_EVENT
-    case ERTS_PORT_TASK_EVENT:
-	if (!state->driver.event)
-	    return 0;
-	if (state->driver.select)
-	    return 0;
-	io_task = &state->driver.event->iotask;
-	break;
-#endif
     default:
 	ERTS_INTERNAL_ERROR("Invalid I/O-task type");
 	return 0;
@@ -2090,29 +1713,6 @@ send_event_tuple(struct erts_nif_select_event* e, ErtsResource* resource,
         erts_proc_unlock(rp, rp_locks);
 }
 
-
-#if ERTS_CIO_HAVE_DRV_EVENT
-static ERTS_INLINE void
-eready(Eterm id, ErtsDrvEventState *state, ErlDrvEventData event_data,
-       erts_aint_t current_cio_time)
-{
-    if (io_task_schedule_allowed(state,
-				 ERTS_PORT_TASK_EVENT,
-				 current_cio_time)) {
-	ErtsIoTask *iotask = &state->driver.event->iotask;
-	erts_atomic_set_nob(&iotask->executed_time, current_cio_time);
-	if (erts_port_task_schedule(id,
-				    &iotask->task,
-				    ERTS_PORT_TASK_EVENT,
-				    (ErlDrvEvent) state->fd,
-				    event_data) != 0) {
-	    stale_drv_select(id, state, 0);
-	}
-	add_active_fd(state->fd);
-    }
-}
-#endif
-
 static void bad_fd_in_pollset(ErtsDrvEventState *, Eterm inport, Eterm outport);
 
 void
@@ -2338,25 +1938,6 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
             goto next_pollres_unlocked;
         }
 
-#if ERTS_CIO_HAVE_DRV_EVENT
-	case ERTS_EV_TYPE_DRV_EV: { /* Requested via driver_event()... */
-	    ErlDrvEventData event_data;
-	    ErtsPollEvents revents;
-	    ASSERT(state->driver.event);
-	    ASSERT(state->driver.event->data);
-	    event_data = state->driver.event->data;
-	    revents = pollres[i].events;
-	    revents &= ~state->driver.event->removed_events;
-
-	    if (revents) {
-		event_data->events = state->events;
-		event_data->revents = revents;
-		eready(state->driver.event->port, state, event_data, current_cio_time);
-	    }
-	    break;
-	}
-#endif
-
 	case ERTS_EV_TYPE_NONE: /* Deselected ... */
 	    break;
 
@@ -2508,7 +2089,6 @@ static void drv_ev_state_free(void *des)
 struct io_functions {
     int (*select)(ErlDrvPort, ErlDrvEvent, int, int);
     int (*enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
-    int (*event)(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
     void (*check_io_as_interrupt)(void);
     void (*check_io_interrupt)(int);
     void (*check_io_interrupt_tmd)(int, ErtsMonotonicTime);
@@ -2517,6 +2097,9 @@ struct io_functions {
     Uint (*size)(void);
     Eterm (*info)(void *);
     int (*check_io_debug)(ErtsCheckIoDebugInfo *);
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    void (*lcnt_update_cio_locks)(int enable);
+#endif
 };
 
 # ifdef ERTS_KERNEL_POLL_VERSION
@@ -2541,7 +2124,6 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
     ASSERT(erts_io_funcs.select == NULL);
     erts_io_funcs.select		= ERTS_CIO_EXPORT(driver_select);
     erts_io_funcs.enif_select		= ERTS_CIO_EXPORT(enif_select);
-    erts_io_funcs.event		= ERTS_CIO_EXPORT(driver_event);
     erts_io_funcs.check_io_interrupt	= ERTS_CIO_EXPORT(erts_check_io_interrupt);
     erts_io_funcs.check_io_interrupt_tmd= ERTS_CIO_EXPORT(erts_check_io_interrupt_timed);
     erts_io_funcs.check_io		= ERTS_CIO_EXPORT(erts_check_io);
@@ -2549,6 +2131,9 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
     erts_io_funcs.size		= ERTS_CIO_EXPORT(erts_check_io_size);
     erts_io_funcs.info		= ERTS_CIO_EXPORT(erts_check_io_info);
     erts_io_funcs.check_io_debug	= ERTS_CIO_EXPORT(erts_check_io_debug);
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_io_funcs.lcnt_update_cio_locks	= ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks);
+#endif
 #endif
 
     ERTS_CIO_POLL_INIT();
@@ -2783,7 +2368,6 @@ typedef struct {
     int used_fds;
     int num_errors;
     int no_driver_select_structs;
-    int no_driver_event_structs;
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     int internal_fds;
     ErtsPollEvents *epep;
@@ -2808,10 +2392,6 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters)
 
     if (state->driver.select)
 	counters->no_driver_select_structs++;
-#if ERTS_CIO_HAVE_DRV_EVENT
-    if (state->driver.event)
-	counters->no_driver_event_structs++;
-#endif
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     if (state->events || ep_events) {
@@ -2984,43 +2564,6 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters)
             r = state->driver.stop.resource;
             erts_printf(" resource=%p(%T:%T)", r, r->type->module, r->type->name);
         }
-#if ERTS_CIO_HAVE_DRV_EVENT
-	else if (state->type == ERTS_EV_TYPE_DRV_EV) {
-	    Eterm id;
-	    erts_printf("driver_event ");
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-	    if (internal) {
-		erts_printf("internal ");
-		err = 1;		    
-	    }
-	    if (cio_events == ep_events) {
-		erts_printf("ev=0x%b32x", (Uint32) cio_events);
-	    }
-	    else {
-		err = 1;
-		erts_printf("cio_ev=0x%b32x", (Uint32) cio_events);
-		erts_printf(" ep_ev=0x%b32x", (Uint32) ep_events);
-	    }
-#else
-	    erts_printf("ev=0x%b32x", (Uint32) cio_events);
-#endif
-	    id = state->driver.event->port;
-	    if (is_nil(id)) {
-		erts_printf(" port=none name=none drv=none ");
-		err = 1;
-	    }
-	    else {
-		ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT);
-		erts_printf(" port=%T name=%s drv=%s ",
-			    id,
-			    pnp->name ? pnp->name : "unknown",
-			    (pnp->driver_name
-			     ? pnp->driver_name
-			     : "unknown"));
-		erts_free_port_names(pnp);
-	    }
-	}
-#endif
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
 	else if (internal) {
 	    erts_printf("internal ");
@@ -3071,9 +2614,6 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
     ErtsDrvEventState null_des;
 
     null_des.driver.select = NULL;
-#if ERTS_CIO_HAVE_DRV_EVENT
-    null_des.driver.event = NULL;
-#endif
     null_des.driver.stop.drv_ptr = NULL;
     null_des.events = 0;
     null_des.remove_cnt = 0;
@@ -3096,7 +2636,6 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
     counters.used_fds = 0;
     counters.num_errors = 0;
     counters.no_driver_select_structs = 0;
-    counters.no_driver_event_structs = 0;
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     len = erts_atomic_read_nob(&drv_ev_state_len);
@@ -3115,14 +2654,10 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
 
     ciodip->no_used_fds = counters.used_fds;
     ciodip->no_driver_select_structs = counters.no_driver_select_structs;
-    ciodip->no_driver_event_structs = counters.no_driver_event_structs;
 
     erts_printf("\n");
     erts_printf("used fds=%d\n", counters.used_fds);
     erts_printf("Number of driver_select() structures=%d\n", counters.no_driver_select_structs);
-#if ERTS_CIO_HAVE_DRV_EVENT
-    erts_printf("Number of driver_event() structures=%d\n", counters.no_driver_event_structs);
-#endif
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     erts_printf("internal fds=%d\n", counters.internal_fds);
 #endif
@@ -3136,7 +2671,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
 }
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
-void erts_lcnt_update_cio_locks(int enable) {
+void ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks)(int enable) {
 #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
     erts_lcnt_enable_hash_lock_count(&drv_ev_state_tab, ERTS_LOCK_FLAGS_CATEGORY_IO, enable);
 #else
@@ -3167,12 +2702,6 @@ driver_select(ErlDrvPort port, ErlDrvEvent event, int mode, int on)
     return (*erts_io_funcs.select)(port, event, mode, on);
 }
 
-int
-driver_event(ErlDrvPort port, ErlDrvEvent event, ErlDrvEventData event_data)
-{
-    return (*erts_io_funcs.event)(port, event, event_data);
-}
-
 int enif_select(ErlNifEnv* env, ErlNifEvent event,
                 enum ErlNifSelectFlags flags, void* obj, const ErlNifPid* pid, Eterm ref)
 {
@@ -3217,6 +2746,13 @@ void erts_check_io_interrupt_timed(int set,
     erts_io_funcs.check_io_interrupt_tmd(set, timeout_time);
 }
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void erts_lcnt_update_cio_locks(int enable)
+{
+    erts_io_funcs.lcnt_update_cio_locks(enable);
+}
+#endif
+
 #endif /* ERTS_KERNEL_POLL_VERSION */
 
 #endif /* !ERTS_ENABLE_KERNEL_POLL */
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index c4ab22aaff..f4d7983002 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -41,8 +41,6 @@ void erts_init_check_io(void);
 void erts_lcnt_update_cio_locks(int enable);
 #endif
 
-#endif
-
 extern erts_atomic_t erts_check_io_time;
 
 typedef struct {
@@ -85,22 +83,6 @@ erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp)
 #  define ERTS_CIO_DEFER_ACTIVE_EVENTS 0
 #endif
 
-/*
- * ErtsDrvEventDataState is used by driver_event() which is almost never
- * used. We allocate ErtsDrvEventDataState separate since we dont wan't
- * the size of ErtsDrvEventState to increase due to driver_event()
- * information.
- */
-typedef struct {
-    Eterm port;
-    ErlDrvEventData data;
-    ErtsPollEvents removed_events;
-#if ERTS_CIO_DEFER_ACTIVE_EVENTS
-    ErtsPollEvents deferred_events;
-#endif
-    ErtsIoTask iotask;
-} ErtsDrvEventDataState;
-
 typedef struct {
     Eterm inport;
     Eterm outport;
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index e88d7aa6df..09c515291a 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -510,7 +510,7 @@ break_requested(void)
       erts_exit(ERTS_INTR_EXIT, "");
 
   ERTS_SET_BREAK_REQUESTED;
-  erts_check_io_sig_interrupt();
+  erts_check_io_interrupt(1);
 }
 
 static RETSIGTYPE request_break(int signum)
diff --git a/erts/emulator/sys/win32/erl_win_dyn_driver.h b/erts/emulator/sys/win32/erl_win_dyn_driver.h
index 6f28d513c2..0d1a6d4c87 100644
--- a/erts/emulator/sys/win32/erl_win_dyn_driver.h
+++ b/erts/emulator/sys/win32/erl_win_dyn_driver.h
@@ -40,7 +40,6 @@ WDD_TYPEDEF(int, driver_exit, (ErlDrvPort, int));
 WDD_TYPEDEF(int, driver_failure_eof, (ErlDrvPort));
 WDD_TYPEDEF(void, erl_drv_busy_msgq_limits, (ErlDrvPort, ErlDrvSizeT *, ErlDrvSizeT *));
 WDD_TYPEDEF(int, driver_select, (ErlDrvPort, ErlDrvEvent, int, int));
-WDD_TYPEDEF(int, driver_event, (ErlDrvPort, ErlDrvEvent,ErlDrvEventData));
 WDD_TYPEDEF(int, driver_output, (ErlDrvPort, char *, ErlDrvSizeT));
 WDD_TYPEDEF(int, driver_output2, (ErlDrvPort, char *, ErlDrvSizeT ,char *, ErlDrvSizeT));
 WDD_TYPEDEF(int, driver_output_binary, (ErlDrvPort, char *, ErlDrvSizeT, ErlDrvBinary*, ErlDrvSizeT, ErlDrvSizeT));
@@ -162,7 +161,7 @@ typedef struct {
     WDD_FTYPE(driver_failure_eof) *driver_failure_eof;
     WDD_FTYPE(erl_drv_busy_msgq_limits) *erl_drv_busy_msgq_limits;
     WDD_FTYPE(driver_select) *driver_select;
-    WDD_FTYPE(driver_event) *driver_event;
+    void *REMOVED_driver_event;
     WDD_FTYPE(driver_output) *driver_output;
     WDD_FTYPE(driver_output2) *driver_output2;
     WDD_FTYPE(driver_output_binary) *driver_output_binary;
@@ -276,7 +275,6 @@ extern TWinDynDriverCallbacks WinDynDriverCallbacks;
 #define driver_failure_eof (WinDynDriverCallbacks.driver_failure_eof)
 #define erl_drv_busy_msgq_limits (WinDynDriverCallbacks.erl_drv_busy_msgq_limits)
 #define driver_select (WinDynDriverCallbacks.driver_select)
-#define driver_event (WinDynDriverCallbacks.driver_event)
 #define driver_output (WinDynDriverCallbacks.driver_output)
 #define driver_output2 (WinDynDriverCallbacks.driver_output2)
 #define driver_output_binary (WinDynDriverCallbacks.driver_output_binary)
@@ -414,7 +412,7 @@ do {				                        \
 ((W).driver_failure_eof) = driver_failure_eof;		\
 ((W).erl_drv_busy_msgq_limits) = erl_drv_busy_msgq_limits;\
 ((W).driver_select) = driver_select;			\
-((W).driver_event) = driver_event;			\
+((W).REMOVED_driver_event) = NULL;			\
 ((W).driver_output) = driver_output;			\
 ((W).driver_output2) = driver_output2;			\
 ((W).driver_output_binary) = driver_output_binary;	\
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 5fca191679..3dffe0902c 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -45,7 +45,6 @@
          io_ready_exit/1,
          use_fallback_pollset/1,
          bad_fd_in_pollset/1,
-         driver_event/1,
          fd_change/1,
          steal_control/1,
          otp_6602/1,
@@ -58,11 +57,9 @@
          ioq_exit_ready_output/1,
          ioq_exit_timeout/1,
          ioq_exit_ready_async/1,
-         ioq_exit_event/1,
          ioq_exit_ready_input_async/1,
          ioq_exit_ready_output_async/1,
          ioq_exit_timeout_async/1,
-         ioq_exit_event_async/1,
          zero_extended_marker_garb_drv/1,
          invalid_extended_marker_drv/1,
          larger_major_vsn_drv/1,
@@ -139,7 +136,7 @@ suite() ->
 all() -> %% Keep a_test first and z_test last...
     [a_test, outputv_errors, outputv_echo, queue_echo, {group, timer},
      driver_unloaded, io_ready_exit, use_fallback_pollset,
-     bad_fd_in_pollset, driver_event, fd_change,
+     bad_fd_in_pollset, fd_change,
      steal_control, otp_6602, driver_system_info_base_ver,
      driver_system_info_prev_ver,
      driver_system_info_current_ver, driver_monitor,
@@ -163,9 +160,9 @@ groups() ->
        timer_change]},
      {ioq_exit, [],
       [ioq_exit_ready_input, ioq_exit_ready_output,
-       ioq_exit_timeout, ioq_exit_ready_async, ioq_exit_event,
+       ioq_exit_timeout, ioq_exit_ready_async,
        ioq_exit_ready_input_async, ioq_exit_ready_output_async,
-       ioq_exit_timeout_async, ioq_exit_event_async]}].
+       ioq_exit_timeout_async]}].
 
 init_per_suite(Config) ->
     Config.
@@ -778,7 +775,6 @@ io_ready_exit(Config) when is_list(Config) ->
 -define(CHKIO_STOP, 0).
 -define(CHKIO_USE_FALLBACK_POLLSET, 1).
 -define(CHKIO_BAD_FD_IN_POLLSET, 2).
--define(CHKIO_DRIVER_EVENT, 3).
 -define(CHKIO_FD_CHANGE, 4).
 -define(CHKIO_STEAL, 5).
 -define(CHKIO_STEAL_AUX, 6).
@@ -829,11 +825,6 @@ bad_fd_in_pollset(Config) when is_list(Config) ->
                                ?CHKIO_BAD_FD_IN_POLLSET,
                                fun () -> sleep(1000) end)).
 
-driver_event(Config) when is_list(Config) ->
-    chkio_test_fini(chkio_test(chkio_test_init(Config),
-                               ?CHKIO_DRIVER_EVENT,
-                               fun () -> sleep(1000) end)).
-
 fd_change(Config) when is_list(Config) ->
     chkio_test_fini(chkio_test(chkio_test_init(Config),
                                ?CHKIO_FD_CHANGE,
@@ -1336,11 +1327,9 @@ driver_monitor(Config) when is_list(Config) ->
 -define(IOQ_EXIT_READY_OUTPUT, 2).
 -define(IOQ_EXIT_TIMEOUT, 3).
 -define(IOQ_EXIT_READY_ASYNC, 4).
--define(IOQ_EXIT_EVENT, 5).
 -define(IOQ_EXIT_READY_INPUT_ASYNC, 6).
 -define(IOQ_EXIT_READY_OUTPUT_ASYNC, 7).
 -define(IOQ_EXIT_TIMEOUT_ASYNC, 8).
--define(IOQ_EXIT_EVENT_ASYNC, 9).
 
 ioq_exit_test(Config, TestNo) ->
     Drv = ioq_exit_drv,
@@ -1393,9 +1382,6 @@ ioq_exit_timeout(Config) when is_list(Config) ->
 ioq_exit_ready_async(Config) when is_list(Config) ->
     ioq_exit_test(Config, ?IOQ_EXIT_READY_ASYNC).
 
-ioq_exit_event(Config) when is_list(Config) ->
-    ioq_exit_test(Config, ?IOQ_EXIT_EVENT).
-
 ioq_exit_ready_input_async(Config) when is_list(Config) ->
     ioq_exit_test(Config, ?IOQ_EXIT_READY_INPUT_ASYNC).
 
@@ -1405,9 +1391,6 @@ ioq_exit_ready_output_async(Config) when is_list(Config) ->
 ioq_exit_timeout_async(Config) when is_list(Config) ->
     ioq_exit_test(Config, ?IOQ_EXIT_TIMEOUT_ASYNC).
 
-ioq_exit_event_async(Config) when is_list(Config) ->
-    ioq_exit_test(Config, ?IOQ_EXIT_EVENT_ASYNC).
-
 
 vsn_mismatch_test(Config, LoadResult) ->
     Path = proplists:get_value(data_dir, Config),
@@ -2275,7 +2258,7 @@ z_test(Config) when is_list(Config) ->
 
 check_io_debug() ->
     get_stable_check_io_info(),
-    {NoErrorFds, NoUsedFds, NoDrvSelStructs, NoDrvEvStructs} = CheckIoDebug
+    {NoErrorFds, NoUsedFds, NoDrvSelStructs} = CheckIoDebug
     = erts_debug:get_internal_state(check_io_debug),
     HasGetHost = has_gethost(),
     ct:log("check_io_debug: ~p~n"
@@ -2289,7 +2272,6 @@ check_io_debug() ->
             %% one extra used fd that is not selected on
             ok
     end,
-    0 = NoDrvEvStructs,
     ok.
 
 has_gethost() ->
diff --git a/erts/emulator/test/driver_SUITE_data/chkio_drv.c b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
index 8e5e81665c..aed6752033 100644
--- a/erts/emulator/test/driver_SUITE_data/chkio_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
@@ -42,7 +42,6 @@
 #define CHKIO_STOP				0
 #define CHKIO_USE_FALLBACK_POLLSET		1
 #define CHKIO_BAD_FD_IN_POLLSET			2
-#define CHKIO_DRIVER_EVENT			3
 #define CHKIO_FD_CHANGE				4
 #define CHKIO_STEAL				5
 #define CHKIO_STEAL_AUX				6
@@ -66,15 +65,6 @@ typedef struct {
     ChkioFallbackFd pipe_out[CHKIO_FALLBACK_FDS];
 } ChkioFallbackData;
 
-typedef struct {
-    int in_fd;
-    struct erl_drv_event_data in_data;
-    int in_ok;
-    int out_fd;
-    struct erl_drv_event_data out_data;
-    int out_ok;
-} ChkioDriverEvent;
-
 typedef struct {
     int fds[2];
     int same_fd;
@@ -86,14 +76,10 @@ typedef struct {
 
 typedef struct {
     int driver_select_fds[2];
-    int driver_event_fds[2];
-    struct erl_drv_event_data event_data[2];
 } ChkioSteal;
 
 typedef struct {
     int driver_select_fds[2];
-    int driver_event_fds[2];
-    struct erl_drv_event_data event_data[2];
 } ChkioStealAux;
 
 
@@ -141,7 +127,6 @@ static ErlDrvData chkio_drv_start(ErlDrvPort, char *);
 static void chkio_drv_stop(ErlDrvData);
 static void chkio_drv_ready_input(ErlDrvData, ErlDrvEvent);
 static void chkio_drv_ready_output(ErlDrvData, ErlDrvEvent);
-static void chkio_drv_ready_event(ErlDrvData, ErlDrvEvent, ErlDrvEventData);
 static ErlDrvSSizeT chkio_drv_control(ErlDrvData, unsigned int,
 				      char *, ErlDrvSizeT, char **, ErlDrvSizeT);
 static void chkio_drv_timeout(ErlDrvData);
@@ -164,7 +149,7 @@ static ErlDrvEntry chkio_drv_entry = {
     NULL, /* ready_async */
     NULL, /* flush */
     NULL, /* call */
-    chkio_drv_ready_event,
+    NULL, /* unused_event_callback */
 
     ERL_DRV_EXTENDED_MARKER,
     ERL_DRV_EXTENDED_MAJOR_VERSION,
@@ -242,25 +227,6 @@ stop_use_fallback_pollset(ChkioDrvData *cddp)
     cddp->test = CHKIO_STOP;
 }
 
-static void
-stop_driver_event(ChkioDrvData *cddp)
-{
-    if (cddp->test_data) {
-	ChkioDriverEvent *cdep = cddp->test_data;
-	cddp->test_data = NULL;
-
-	if (cdep->in_fd >= 0) {
-	    driver_event(cddp->port, (ErlDrvEvent) (ErlDrvSInt) cdep->in_fd, NULL);
-	    close(cdep->in_fd);
-	}
-	if (cdep->out_fd >= 0) {
-	    driver_event(cddp->port, (ErlDrvEvent) (ErlDrvSInt) cdep->out_fd, NULL);
-	    close(cdep->out_fd);
-	}
-	driver_free(cdep);	    
-    }
-}
-
 static void
 stop_fd_change(ChkioDrvData *cddp)
 {
@@ -305,14 +271,6 @@ stop_steal(ChkioDrvData *cddp)
 			  (ErlDrvEvent) (ErlDrvSInt) csp->driver_select_fds[1],
 			  DO_WRITE,
 			  0);
-	if (csp->driver_event_fds[0] >= 0)
-	    driver_event(cddp->port,
-			 (ErlDrvEvent) (ErlDrvSInt) csp->driver_event_fds[0],
-			 NULL);
-	if (csp->driver_event_fds[1] >= 0)
-	    driver_event(cddp->port,
-			 (ErlDrvEvent) (ErlDrvSInt) csp->driver_event_fds[1],
-			 NULL);
 	driver_free(csp);
     }
 }
@@ -327,10 +285,6 @@ stop_steal_aux(ChkioDrvData *cddp)
 	    close(csap->driver_select_fds[0]);
 	if (csap->driver_select_fds[1] >= 0)
 	    close(csap->driver_select_fds[1]);
-	if (csap->driver_event_fds[0] >= 0)
-	    close(csap->driver_event_fds[0]);
-	if (csap->driver_event_fds[1] >= 0)
-	    close(csap->driver_event_fds[1]);
 	driver_free(csap);
     }
 }
@@ -445,9 +399,6 @@ chkio_drv_stop(ErlDrvData drv_data) {
     case CHKIO_BAD_FD_IN_POLLSET:
 	stop_bad_fd_in_pollset(cddp);
 	break;
-    case CHKIO_DRIVER_EVENT:
-	stop_driver_event(cddp);
-	break;
     case CHKIO_FD_CHANGE:
 	stop_fd_change(cddp);
 	break;
@@ -620,55 +571,6 @@ chkio_drv_ready_input(ErlDrvData drv_data, ErlDrvEvent event)
 #endif
 }
 
-static void
-chkio_drv_ready_event(ErlDrvData drv_data,
-		      ErlDrvEvent event,
-		      ErlDrvEventData event_data)
-{
-#ifdef UNIX
-    ChkioDrvData *cddp = (ChkioDrvData *) drv_data;
-    switch (cddp->test) {
-    case CHKIO_DRIVER_EVENT: {
-#ifdef HAVE_POLL_H
-	ChkioDriverEvent *cdep = cddp->test_data;
-	int fd = (int) (ErlDrvSInt) event;
-	if (fd == cdep->in_fd) {
-	    if (event_data->events == POLLIN
-		&& event_data->revents == POLLIN) {
-		cdep->in_ok++;
-	    }
-	    else {
-		driver_failure_atom(cddp->port, "invalid_input_fd_events");
-	    }
-	    break;
-	}
-	if (fd == cdep->out_fd) {
-	    if (event_data->events == POLLOUT
-		&& event_data->revents == POLLOUT) {
-		cdep->out_ok++;
-	    }
-	    else {
-		driver_failure_atom(cddp->port, "invalid_output_fd_events");
-	    }
-	    break;
-	}
-#endif
-    }
-    case CHKIO_STEAL:
-#ifdef HAVE_POLL_H
-	break;
-#endif
-    case CHKIO_STEAL_AUX:
-#ifdef HAVE_POLL_H
-	break;
-#endif
-    default:
-	driver_failure_atom(cddp->port, "unexpected_ready_event");
-	break;
-    }
-#endif /* UNIX */
-}
-
 static void
 chkio_drv_timeout(ErlDrvData drv_data)
 {
@@ -779,25 +681,6 @@ chkio_drv_control(ErlDrvData drv_data,
 	    res_len = -1;
 	    stop_bad_fd_in_pollset(cddp);
 	    break;
-	case CHKIO_DRIVER_EVENT: {
-	    ChkioDriverEvent *cdep = cddp->test_data;
-	    if (!cdep->in_ok || !cdep->out_ok) {
-		if (!cdep->in_ok)
-		    driver_failure_atom(cddp->port, "got_no_input_events");
-		if (!cdep->out_ok)
-		    driver_failure_atom(cddp->port, "got_no_output_events");
-	    }
-	    else {
-		char *c = driver_alloc(sizeof(char)*2*30);
-		if (!c)
-		    driver_failure_posix(cddp->port, ENOMEM);
-		*rbuf = c;
-		res_len = sprintf(c, "in=%d\nout=%d\n",
-				  cdep->in_ok, cdep->out_ok);
-	    }
-	    stop_driver_event(cddp);
-	    break;
-	}
 	case CHKIO_FD_CHANGE: {
 	    ChkioFdChange *cfcp = cddp->test_data;
 	    if (!cfcp->same_fd)
@@ -937,69 +820,6 @@ chkio_drv_control(ErlDrvData drv_data,
 	res_len = -1;
 	break;
     }
-    case CHKIO_DRIVER_EVENT: {
-#ifndef HAVE_POLL_H
-	res_str = "skip: Need the poll.h header for this test, but it doesn't exist";
-	res_len = -1;
-#else /* HAVE_POLL_H */
-	int in_fd = open("/dev/zero", O_RDONLY);
-	int out_fd = open("/dev/null", O_WRONLY);
-
-	if (in_fd < 0 || out_fd < 0) {
-	    if (in_fd >= 0)
-		close(in_fd);
-	    if (out_fd >= 0)
-		close(out_fd);
-	    driver_failure_posix(cddp->port, errno);
-	}
-	else {
-	    ChkioDriverEvent *cdep = driver_alloc(sizeof(ChkioDriverEvent));
-	    if (!cdep)
-		driver_failure_posix(cddp->port, ENOMEM);
-	    else {
-		int res;
-		cddp->test_data = cdep;
-
-		cdep->in_fd = in_fd;
-		cdep->in_data.events = POLLIN;
-		cdep->in_data.revents = 0;
-		cdep->in_ok = 0;
-
-		res = driver_event(cddp->port,
-				   (ErlDrvEvent) (ErlDrvSInt) in_fd,
-				   &cdep->in_data);
-		if (res < 0) {
-		    res_str = "skip: driver_event() not supported";
-		    res_len = -1;
-		    close(in_fd);
-		    close(out_fd);
-		    cdep->in_fd = -1;
-		    cdep->out_fd = -1;
-		}
-		else {
-		    res_str = "ok";
-		    res_len = -1;
-
-		    cdep->out_fd = out_fd;
-		    cdep->out_data.events = POLLOUT;
-		    cdep->out_data.revents = 0;
-		    cdep->out_ok = 0;
-
-		    res = driver_event(cddp->port,
-				       (ErlDrvEvent) (ErlDrvSInt) out_fd,
-				       &cdep->out_data);
-		    if (res < 0) {
-			close(out_fd);
-			cdep->out_fd = -1;
-			driver_failure_atom(cddp->port, "driver_event_failed");
-		    }
-		}
-
-	    }
-	}
-#endif /* HAVE_POLL_H */
-	break;
-    }
     case CHKIO_FD_CHANGE: {
 	ChkioFdChange *cfcp = driver_alloc(sizeof(ChkioFdChange));
 	if (!cfcp)
@@ -1028,58 +848,19 @@ chkio_drv_control(ErlDrvData drv_data,
 	    res_len = -1;
 	}
 	else {
-	    int driver_event_fds[2];
 	    int driver_select_fds[2];
 	    cddp->test_data = csp; 
 	    memcpy(c, buf, len);
 	    c[len] = '\0';
 	    if (sscanf(c,
-		       "fds:%d:%d:%d:%d",
+		       "fds:%d:%d",
 		       &driver_select_fds[0],
-		       &driver_select_fds[1],
-		       &driver_event_fds[0],
-		       &driver_event_fds[1]) != 4)
-		driver_failure_atom(cddp->port, "bad_input");
+		       &driver_select_fds[1]) != 2)
+                driver_failure_atom(cddp->port, "bad_input");
 	    else {
 		int res = 0;
-		if (driver_event_fds[0] < 0) { /* Have no working driver_event() ... */
-		    csp->driver_select_fds[0] = driver_select_fds[0]; /* In */
-		    csp->driver_select_fds[1] = driver_select_fds[1]; /* Out */
-		    csp->driver_event_fds[0] = -1;
-		    csp->driver_event_fds[1] = -1;
-		}
-		else { /* Have working driver_event() ... */
-#ifndef HAVE_POLL_H
-		    driver_failure_atom(cddp->port, "unexpected_result");
-		    res = -1;
-#else
-		    csp->driver_select_fds[0] = driver_select_fds[0]; /* In */
-		    csp->driver_event_fds[1] = driver_select_fds[1]; /* Out */
-		    csp->driver_event_fds[0] = driver_event_fds[0]; /* In */
-		    csp->driver_select_fds[1] = driver_event_fds[1]; /* Out */
-
-		    /* Steal with driver_event() */
-
-		    csp->event_data[0].events = POLLIN;
-		    csp->event_data[0].revents = 0;
-		    res = driver_event(cddp->port,
-				       (ErlDrvEvent) (ErlDrvSInt) csp->driver_event_fds[0],
-				       &csp->event_data[0]);
-		    if (res < 0)
-			driver_failure_atom(cddp->port,
-					    "driver_event_failed_to_steal");
-		    if (res >= 0) {
-			csp->event_data[1].events = POLLOUT;
-			csp->event_data[1].revents = 0;
-			res = driver_event(cddp->port,
-					   (ErlDrvEvent) (ErlDrvSInt) csp->driver_event_fds[1],
-					   &csp->event_data[1]);
-			if (res < 0)
-			    driver_failure_atom(cddp->port,
-						"driver_event_failed_to_steal");
-		    }
-#endif
-		}
+                csp->driver_select_fds[0] = driver_select_fds[0]; /* In */
+                csp->driver_select_fds[1] = driver_select_fds[1]; /* Out */
 
 		/* Steal with driver_select() */
 		if (res >= 0) {
@@ -1109,37 +890,17 @@ chkio_drv_control(ErlDrvData drv_data,
 	break;
     }
     case CHKIO_STEAL_AUX: {
-	int read_fds[2];
-	int write_fds[2];
-
-	read_fds[0] = open("/dev/zero", O_RDONLY);
-	write_fds[0] = open("/dev/null", O_WRONLY);
+	int read_fd;
+	int write_fd;
 
-#ifdef HAVE_POLL_H
-	read_fds[1] = open("/dev/zero", O_RDONLY);
-	write_fds[1] = open("/dev/null", O_WRONLY);
-#else
-	read_fds[1] = -1;
-	write_fds[1] = -1;
-#endif
+	read_fd = open("/dev/zero", O_RDONLY);
+	write_fd = open("/dev/null", O_WRONLY);
 	
-	if (read_fds[0] < 0
-	    || write_fds[0] < 0
-#ifdef HAVE_POLL_H
-	    || read_fds[1] < 0 
-	    || write_fds[1] < 0
-#endif
-	    ) {
-	    if (read_fds[0] < 0)
-		close(read_fds[0]);
-	    if (write_fds[0] < 0)
-		close(write_fds[0]);
-#ifdef HAVE_POLL_H
-	    if (read_fds[1] < 0)
-		close(read_fds[1]);
-	    if (write_fds[1] < 0)
-		close(write_fds[1]);
-#endif
+	if (read_fd < 0 || write_fd < 0) {
+	    if (read_fd < 0)
+		close(read_fd);
+	    if (write_fd < 0)
+		close(write_fd);
 	    driver_failure_posix(cddp->port, errno);
 	}
 	else {
@@ -1153,11 +914,8 @@ chkio_drv_control(ErlDrvData drv_data,
 		int res;
 		cddp->test_data = csap;
 
-		csap->driver_select_fds[0] = read_fds[0];
-		csap->driver_select_fds[1] = write_fds[0];
-
-		csap->driver_event_fds[0] = read_fds[1];
-		csap->driver_event_fds[1] = write_fds[1];
+		csap->driver_select_fds[0] = read_fd;
+		csap->driver_select_fds[1] = write_fd;
 
 		res = driver_select(cddp->port,
 				    (ErlDrvEvent) (ErlDrvSInt) csap->driver_select_fds[0],
@@ -1173,32 +931,6 @@ chkio_drv_control(ErlDrvData drv_data,
 		    if (res < 0)
 			driver_failure_atom(cddp->port, "driver_select_failed");
 		}
-#ifdef HAVE_POLL_H
-		if (res >= 0) {
-		    csap->event_data[0].events = POLLIN;
-		    csap->event_data[0].revents = 0;
-		    res = driver_event(cddp->port,
-				       (ErlDrvEvent) (ErlDrvSInt) csap->driver_event_fds[0],
-				       &csap->event_data[0]);
-		    if (res < 0) {
-			close(csap->driver_event_fds[0]);
-			csap->driver_event_fds[0] = -1;
-			close(csap->driver_event_fds[1]);
-			csap->driver_event_fds[1] = -1;
-			res = 0;
-		    }
-		    else {
-			csap->event_data[1].events = POLLOUT;
-			csap->event_data[1].revents = 0;
-			res = driver_event(cddp->port,
-					   (ErlDrvEvent) (ErlDrvSInt) csap->driver_event_fds[1],
-					   &csap->event_data[1]);
-			if (res < 0)
-			    driver_failure_atom(cddp->port,
-						"driver_event_failed");
-		    }
-		}
-#endif
 		if (res < 0) {
 		    res_str = "error";
 		    res_len = -1;
@@ -1213,11 +945,9 @@ chkio_drv_control(ErlDrvData drv_data,
 		    else {
 			*rbuf = c;
 			res_len = sprintf(c,
-					  "fds:%d:%d:%d:%d",
+					  "fds:%d:%d",
 					  csap->driver_select_fds[0],
-					  csap->driver_select_fds[1],
-					  csap->driver_event_fds[0],
-					  csap->driver_event_fds[1]);
+					  csap->driver_select_fds[1]);
 		    }
 		}
 	    }
diff --git a/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c b/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c
index d87c2bec93..fa58e9d5ec 100644
--- a/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c
@@ -25,8 +25,7 @@
  *		- ready_input(),
  *		- ready_output(),
  *		- timeout(),
- *		- driver_async() -> read_async(), and
- *		- event()
+ *		- driver_async() -> read_async()
  */
 
 #ifndef UNIX
@@ -65,11 +64,9 @@ typedef enum {
     IOQ_EXIT_READY_OUTPUT = 2,
     IOQ_EXIT_TIMEOUT = 3,
     IOQ_EXIT_READY_ASYNC = 4,
-    IOQ_EXIT_EVENT = 5,
     IOQ_EXIT_READY_INPUT_ASYNC = 6,
     IOQ_EXIT_READY_OUTPUT_ASYNC = 7,
     IOQ_EXIT_TIMEOUT_ASYNC = 8,
-    IOQ_EXIT_EVENT_ASYNC = 9
 } IOQExitTest;
 
 typedef struct {
@@ -80,9 +77,6 @@ typedef struct {
     int outstanding_async_task;
     long async_task;
     ErlDrvPDL pdl;
-#ifdef HAVE_POLL_H
-    struct erl_drv_event_data event_data;
-#endif
 } IOQExitDrvData;
 
 #define EV2FD(EV) ((int) ((long) (EV)))
@@ -97,8 +91,6 @@ static ErlDrvSSizeT control(ErlDrvData, unsigned int,
 static void timeout(ErlDrvData drv_data);
 static void ready_async(ErlDrvData drv_data, ErlDrvThreadData thread_data);
 static void flush(ErlDrvData drv_data);
-static void event(ErlDrvData drv_data, ErlDrvEvent event,
-		  ErlDrvEventData event_data);
 static void async_invoke(void*);
 static void do_driver_async(IOQExitDrvData *);
 
@@ -118,7 +110,7 @@ static ErlDrvEntry ioq_exit_drv_entry = {
     ready_async,
     flush,
     NULL /* call */,
-    event,
+    NULL /* unused_event_callback*/,
     ERL_DRV_EXTENDED_MARKER,
     ERL_DRV_EXTENDED_MAJOR_VERSION,
     ERL_DRV_EXTENDED_MINOR_VERSION,
@@ -149,10 +141,6 @@ start(ErlDrvPort port, char *command)
     ddp->outstanding_async_task = 0;
     ddp->async_task = -1;
     ddp->pdl = driver_pdl_create(port);
-#ifdef HAVE_POLL_H
-    ddp->event_data.events = (short) 0;
-    ddp->event_data.revents = (short) 0;
-#endif
 
     return (ErlDrvData) ddp;
 }
@@ -191,27 +179,6 @@ static ErlDrvSSizeT control(ErlDrvData drv_data,
 	break;
 #else
 	goto done;
-#endif
-    case IOQ_EXIT_EVENT:
-    case IOQ_EXIT_EVENT_ASYNC:
-#ifdef UNIX
-#ifdef HAVE_POLL_H
-	ddp->ofd = open("/dev/null", O_WRONLY);
-	if (ddp->ofd < 0) {
-	    driver_failure_posix(ddp->port, errno);
-	    return 0;
-	}
-	else if (driver_event(ddp->port, FD2EV(ddp->ofd), NULL) != 0) {
-	    res_str = "skip: driver_event() not supported";
-	    goto done;
-	}
-#else
-	res_str = "skip: No poll.h found which is needed for this test";
-	goto done;
-#endif
-	break;
-#else /* UNIX */
-	goto done;
 #endif
     case IOQ_EXIT_TIMEOUT:
     case IOQ_EXIT_TIMEOUT_ASYNC:
@@ -266,13 +233,6 @@ static void stop(ErlDrvData drv_data)
 		close(ddp->ofd);
 	    }
 	    break;
-	case IOQ_EXIT_EVENT:
-	case IOQ_EXIT_EVENT_ASYNC:
-	    if (ddp->ofd >= 0) {
-		driver_event(ddp->port, FD2EV(ddp->ofd), NULL);
-		close(ddp->ofd);
-	    }
-	    break;
 #endif
 	case IOQ_EXIT_TIMEOUT:
 	case IOQ_EXIT_TIMEOUT_ASYNC:
@@ -302,13 +262,6 @@ static void flush(ErlDrvData drv_data)
     case IOQ_EXIT_READY_OUTPUT_ASYNC:
 	driver_select(ddp->port, FD2EV(ddp->ofd), DO_WRITE, 1);
 	break;
-    case IOQ_EXIT_EVENT:
-    case IOQ_EXIT_EVENT_ASYNC:
-#ifdef HAVE_POLL_H
-	ddp->event_data.events |= POLLOUT;
-	driver_event(ddp->port, FD2EV(ddp->ofd), &ddp->event_data);
-#endif
-	break;
 #endif
     case IOQ_EXIT_TIMEOUT:
     case IOQ_EXIT_TIMEOUT_ASYNC:
@@ -395,30 +348,6 @@ static void ready_async(ErlDrvData drv_data, ErlDrvThreadData thread_data)
     }
 }
 
-static void event(ErlDrvData drv_data,
-		  ErlDrvEvent event,
-		  ErlDrvEventData event_data)
-{
-    IOQExitDrvData *ddp = (IOQExitDrvData *) drv_data;
-
-    PRINTF(("event(%p, %d, %p) called\r\n", drv_data, EV2FD(event), event_data));
-
-#if defined(UNIX) && defined(HAVE_POLL_H)
-    if (ddp->ofd == EV2FD(event)) {
-	driver_event(ddp->port, FD2EV(ddp->ofd), NULL);
-	close(ddp->ofd);
-	ddp->ofd = -1;
-	if (ddp->test == IOQ_EXIT_EVENT_ASYNC)
-	    do_driver_async(ddp);
-	else {
-	    driver_pdl_lock(ddp->pdl);
-	    driver_deq(ddp->port, 1);
-	    driver_pdl_unlock(ddp->pdl);
-	}
-    }
-#endif
-}
-
 static void async_invoke(void *arg)
 {
     PRINTF(("async_invoke(%p) called\r\n", arg));
diff --git a/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c b/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c
index e7480d2e00..14838f0377 100644
--- a/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c
@@ -41,10 +41,6 @@
 typedef struct {
     int ofd;
     int ifd;
-    int efd;
-#ifdef HAVE_POLL_H
-    struct erl_drv_event_data edata;
-#endif
 } mcd_data_t;
 
 static ErlDrvData start(ErlDrvPort port, char *command);
@@ -90,7 +86,6 @@ start(ErlDrvPort port, char *command)
 
     mcd->ofd = -1;
     mcd->ifd = -1;
-    mcd->efd = -1;
 
 #ifdef UNIX
 
@@ -105,15 +100,6 @@ start(ErlDrvPort port, char *command)
 	goto error;
     if (driver_select(port, (ErlDrvEvent) (long) mcd->ifd, DO_READ, 1) != 0)
 	goto error;
-
-#ifdef HAVE_POLL_H
-    mcd->efd = open("/dev/null", O_WRONLY);
-    if (mcd->efd < 0)
-	goto error;
-    mcd->edata.events = POLLOUT;
-    mcd->edata.revents = 0;
-    driver_event(port, (ErlDrvEvent) (long) mcd->efd, &mcd->edata);
-#endif
 #endif
 
     driver_set_timer(port, 0);
@@ -135,10 +121,6 @@ stop(ErlDrvData data)
 	    close(mcd->ofd);
 	if (mcd->ifd >= 0)
 	    close(mcd->ifd);
-#ifdef HAVE_POLL_H
-	if (mcd->efd >= 0)
-	    close(mcd->efd);
-#endif
 #endif
 	driver_free(mcd);
     }
diff --git a/erts/emulator/test/lttng_SUITE.erl b/erts/emulator/test/lttng_SUITE.erl
index a012fa1da2..19c3844c40 100644
--- a/erts/emulator/test/lttng_SUITE.erl
+++ b/erts/emulator/test/lttng_SUITE.erl
@@ -81,7 +81,6 @@ end_per_testcase(Case, _Config) ->
 
 %% Not tested yet
 %%   org_erlang_otp:driver_process_exit
-%%   org_erlang_otp:driver_event
 
 %% tracepoints
 %%
@@ -100,7 +99,6 @@ end_per_testcase(Case, _Config) ->
 %%   org_erlang_otp:driver_flush
 %%   org_erlang_otp:driver_stop_select
 %%   org_erlang_otp:driver_timeout
-%%   org_erlang_otp:driver_event
 %%   org_erlang_otp:driver_ready_output
 %%   org_erlang_otp:driver_ready_input
 %%   org_erlang_otp:driver_output
@@ -431,7 +429,6 @@ txt() ->
       "%%   org_erlang_otp:driver_flush\n"
       "%%   org_erlang_otp:driver_stop_select\n"
       "%%   org_erlang_otp:driver_timeout\n"
-      "%%   org_erlang_otp:driver_event\n"
       "%%   org_erlang_otp:driver_ready_output\n"
       "%%   org_erlang_otp:driver_ready_input\n"
       "%%   org_erlang_otp:driver_output\n"
-- 
cgit v1.2.3


From 23f09f3bbeb924b44fbcef882b1f0c24e90626bc Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 25 Apr 2017 17:51:31 +0200
Subject: erts: Add number of enif_select's to check_io_debug

---
 erts/emulator/beam/erl_bif_info.c       | 6 ++++--
 erts/emulator/beam/sys.h                | 1 +
 erts/emulator/sys/common/erl_check_io.c | 7 +++++++
 erts/emulator/test/driver_SUITE.erl     | 5 +++--
 4 files changed, 15 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 3b6e83c242..c782d7a8ce 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3574,13 +3574,15 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    szp = &sz;
 	    hpp = NULL;
 	    while (1) {
-		res = erts_bld_tuple(hpp, szp, 3,
+		res = erts_bld_tuple(hpp, szp, 4,
 				     erts_bld_uint(hpp, szp,
 						   (Uint) no_errors),
 				     erts_bld_uint(hpp, szp,
 						   (Uint) ciodi.no_used_fds),
 				     erts_bld_uint(hpp, szp,
-						   (Uint) ciodi.no_driver_select_structs));
+						   (Uint) ciodi.no_driver_select_structs),
+                                     erts_bld_uint(hpp, szp,
+                                                   (Uint) ciodi.no_enif_select_structs));
 		if (hpp)
 		    break;
 		hp = HAlloc(BIF_P, sz);
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index e2a21c0ecd..0bf2ecfc8b 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -790,6 +790,7 @@ void fini_getenv_state(GETENV_STATE *);
 typedef struct {
     int no_used_fds;
     int no_driver_select_structs;
+    int no_enif_select_structs;
 } ErtsCheckIoDebugInfo;
 int erts_check_io_debug(ErtsCheckIoDebugInfo *ip);
 
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index cd695e236f..15662f9a70 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -2368,6 +2368,7 @@ typedef struct {
     int used_fds;
     int num_errors;
     int no_driver_select_structs;
+    int no_enif_select_structs;
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     int internal_fds;
     ErtsPollEvents *epep;
@@ -2392,6 +2393,8 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters)
 
     if (state->driver.select)
 	counters->no_driver_select_structs++;
+    if (state->driver.nif)
+        counters->no_enif_select_structs++;
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     if (state->events || ep_events) {
@@ -2614,6 +2617,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
     ErtsDrvEventState null_des;
 
     null_des.driver.select = NULL;
+    null_des.driver.nif = NULL;
     null_des.driver.stop.drv_ptr = NULL;
     null_des.events = 0;
     null_des.remove_cnt = 0;
@@ -2636,6 +2640,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
     counters.used_fds = 0;
     counters.num_errors = 0;
     counters.no_driver_select_structs = 0;
+    counters.no_enif_select_structs = 0;
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     len = erts_atomic_read_nob(&drv_ev_state_len);
@@ -2654,10 +2659,12 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
 
     ciodip->no_used_fds = counters.used_fds;
     ciodip->no_driver_select_structs = counters.no_driver_select_structs;
+    ciodip->no_enif_select_structs = counters.no_enif_select_structs;
 
     erts_printf("\n");
     erts_printf("used fds=%d\n", counters.used_fds);
     erts_printf("Number of driver_select() structures=%d\n", counters.no_driver_select_structs);
+    erts_printf("Number of enif_select() structures=%d\n", counters.no_enif_select_structs);
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     erts_printf("internal fds=%d\n", counters.internal_fds);
 #endif
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 3dffe0902c..c6d7f708be 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -2258,8 +2258,8 @@ z_test(Config) when is_list(Config) ->
 
 check_io_debug() ->
     get_stable_check_io_info(),
-    {NoErrorFds, NoUsedFds, NoDrvSelStructs} = CheckIoDebug
-    = erts_debug:get_internal_state(check_io_debug),
+    {NoErrorFds, NoUsedFds, NoDrvSelStructs, NoEnifSelStructs}
+        = CheckIoDebug = erts_debug:get_internal_state(check_io_debug),
     HasGetHost = has_gethost(),
     ct:log("check_io_debug: ~p~n"
            "HasGetHost: ~p",[CheckIoDebug, HasGetHost]),
@@ -2272,6 +2272,7 @@ check_io_debug() ->
             %% one extra used fd that is not selected on
             ok
     end,
+    0 = NoEnifSelStructs,
     ok.
 
 has_gethost() ->
-- 
cgit v1.2.3


From 7734946e210f39628ad954bada7ee81573fddf85 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 27 Apr 2017 18:35:16 +0200
Subject: erts: Replace check_io spinlock with lock-less list insertion

---
 erts/emulator/beam/erl_lock_check.c     |  1 -
 erts/emulator/sys/common/erl_check_io.c | 33 ++++++++++++++++-----------------
 2 files changed, 16 insertions(+), 18 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index f81c90818f..4cdef0200f 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -113,7 +113,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"drv_ev_state_grow",			NULL,   		},
     {	"drv_ev_state",				"address"		},
     {	"safe_hash",				"address"		},
-    {   "pollset_rm_list",                      NULL                    },
     {   "removed_fd_pre_alloc_lock",            "address"               },
     {   "state_prealloc",                       NULL                    },
     {	"schdlr_sspnd",				NULL			},
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 15662f9a70..a5a851d122 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -101,11 +101,10 @@ static struct pollset_info
 	int size;
 	ErtsSysFdType *array;
     } active_fd;
-    struct removed_fd* removed_list;       /* list of deselected fd's*/
-    erts_spinlock_t removed_list_lock;
+    erts_atomic_t removed_list;    /* struct removed_fd* */
 }pollset;
-#define NUM_OF_POLLSETS 1
 
+#define NUM_OF_POLLSETS 1
 
 #ifdef ERTS_ENABLE_KERNEL_POLL
 void erts_init_check_io_kp(void);
@@ -302,6 +301,7 @@ remember_removed(ErtsDrvEventState *state, struct pollset_info* psi)
     struct removed_fd *fdlp;
     ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd)));
     if (erts_atomic_read_nob(&psi->in_poll_wait)) {
+        erts_aint_t was_next, exp_next;
 	state->remove_cnt++;
 	ASSERT(state->remove_cnt > 0);
 	fdlp = removed_fd_alloc();
@@ -311,10 +311,16 @@ remember_removed(ErtsDrvEventState *state, struct pollset_info* psi)
     #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
 	fdlp->state = state;
     #endif
-	erts_spin_lock(&psi->removed_list_lock);
-	fdlp->next = psi->removed_list;
-	psi->removed_list = fdlp;
-	erts_spin_unlock(&psi->removed_list_lock);
+
+         /* Lockless atomic insertion in removed_list */
+        was_next = erts_atomic_read_acqb(&psi->removed_list);
+        do {
+            exp_next = was_next;
+            fdlp->next = (struct removed_fd*) exp_next;
+            was_next = erts_atomic_cmpxchg_mb(&psi->removed_list,
+                                              (erts_aint_t) fdlp,
+                                              exp_next);
+        }while (was_next != exp_next);
     }
 }
 
@@ -336,12 +342,8 @@ forget_removed(struct pollset_info* psi)
     struct removed_fd* fdlp;
     struct removed_fd* tofree;
 
-    /* Fast track: if (atomic_ptr(removed_list)==NULL) return; */
-
-    erts_spin_lock(&psi->removed_list_lock);
-    fdlp = psi->removed_list;
-    psi->removed_list = NULL;
-    erts_spin_unlock(&psi->removed_list_lock);
+    fdlp = (struct removed_fd*) erts_atomic_xchg_mb(&psi->removed_list,
+                                                    (erts_aint_t) NULL);
 
     while (fdlp) {
         ErtsResource* resource = NULL;
@@ -2153,11 +2155,8 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
     }
 #endif
 
-
     init_removed_fd_alloc();
-    pollset.removed_list = NULL;
-    erts_spinlock_init(&pollset.removed_list_lock, "pollset_rm_list", NIL,
-        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
+    erts_atomic_init_nob(&pollset.removed_list,  (erts_aint_t)NULL);
     {
         int i;
         for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) {
-- 
cgit v1.2.3


From 6e2023777af660ef32d48d0a57d9157745975847 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 12 May 2017 16:00:54 +0200
Subject: erts: Refactor check_io to use one static struct

that is shared between _kp and _nkp versions.
Makes it easier to access in debugger.
---
 erts/emulator/sys/common/erl_check_io.c | 198 +++++++++++++++++---------------
 1 file changed, 104 insertions(+), 94 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index a5a851d122..52584e4246 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -156,15 +156,36 @@ struct removed_fd {
 
 };
 
+struct drv_ev_state_shared {
+    /* The layout of this struct must be independent of kp/nkp compilation */
+
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-static int max_fds = -1;
+    int max_fds;
 #endif
 
 #define DRV_EV_STATE_LOCK_CNT 128
-static union {
-    erts_mtx_t lck;
-    byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_mtx_t))];
-}drv_ev_state_locks[DRV_EV_STATE_LOCK_CNT];
+    union {
+        erts_mtx_t lck;
+        byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_mtx_t))];
+    }locks[DRV_EV_STATE_LOCK_CNT];
+
+#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
+    erts_atomic_t len;
+    ErtsDrvEventState *v;
+    erts_mtx_t grow_lock; /* prevent lock-hogging of racing growers */
+#else
+    SafeHash tab;
+    int num_prealloc;
+    ErtsDrvEventState *prealloc_first;
+    erts_spinlock_t prealloc_lock;
+#endif
+};
+
+#ifndef ERTS_KERNEL_POLL_VERSION
+struct drv_ev_state_shared drv_ev_state;
+#else
+extern struct drv_ev_state_shared drv_ev_state;
+#endif
 
 static ERTS_INLINE erts_mtx_t* fd_mtx(ErtsSysFdType fd)
 {
@@ -172,26 +193,16 @@ static ERTS_INLINE erts_mtx_t* fd_mtx(ErtsSysFdType fd)
 # ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
     hash ^= (hash >> 9);
 # endif
-    return &drv_ev_state_locks[hash % DRV_EV_STATE_LOCK_CNT].lck;
+    return &drv_ev_state.locks[hash % DRV_EV_STATE_LOCK_CNT].lck;
 }
 
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-
-static erts_atomic_t drv_ev_state_len;
-static ErtsDrvEventState *drv_ev_state;
-static erts_mtx_t drv_ev_state_grow_lock; /* prevent lock-hogging of racing growers */
-
-#else
-static SafeHash drv_ev_state_tab;
-static int num_state_prealloc;
-static ErtsDrvEventState *state_prealloc_first;
-erts_spinlock_t state_prealloc_lock;
+#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
 
 static ERTS_INLINE ErtsDrvEventState *hash_get_drv_ev_state(ErtsSysFdType fd)
 {
     ErtsDrvEventState tmpl;
     tmpl.fd = fd;
-    return  (ErtsDrvEventState *) safe_hash_get(&drv_ev_state_tab, (void *) &tmpl);
+    return  (ErtsDrvEventState *) safe_hash_get(&drv_ev_state.tab, (void *) &tmpl);
 }
 
 static ERTS_INLINE ErtsDrvEventState* hash_new_drv_ev_state(ErtsSysFdType fd)
@@ -205,13 +216,13 @@ static ERTS_INLINE ErtsDrvEventState* hash_new_drv_ev_state(ErtsSysFdType fd)
     tmpl.remove_cnt = 0;
     tmpl.type = ERTS_EV_TYPE_NONE;
     tmpl.flags = 0;
-    return  (ErtsDrvEventState *) safe_hash_put(&drv_ev_state_tab, (void *) &tmpl);
+    return  (ErtsDrvEventState *) safe_hash_put(&drv_ev_state.tab, (void *) &tmpl);
 }
 
 static ERTS_INLINE void hash_erase_drv_ev_state(ErtsDrvEventState *state)
 {
     ASSERT(state->remove_cnt == 0);
-    safe_hash_erase(&drv_ev_state_tab, (void *) state);
+    safe_hash_erase(&drv_ev_state.tab, (void *) state);
 }
 
 #endif /* !ERTS_SYS_CONTINOUS_FD_NUMBERS */
@@ -356,7 +367,7 @@ forget_removed(struct pollset_info* psi)
 	fd = fdlp->fd;
 	mtx = fd_mtx(fd);
 	erts_mtx_lock(mtx);
-	state = &drv_ev_state[(int) fd];
+	state = &drv_ev_state.v[(int) fd];
 #else
 	state = fdlp->state;
 	fd = state->fd;
@@ -426,40 +437,40 @@ grow_drv_ev_state(int min_ix)
     int old_len;
     int new_len;
 
-    erts_mtx_lock(&drv_ev_state_grow_lock);
-    old_len = erts_atomic_read_nob(&drv_ev_state_len);
+    erts_mtx_lock(&drv_ev_state.grow_lock);
+    old_len = erts_atomic_read_nob(&drv_ev_state.len);
     if (min_ix >= old_len) {
         new_len = erts_poll_new_table_len(old_len, min_ix + 1);
-        if (new_len > max_fds)
-            new_len = max_fds;
+        if (new_len > drv_ev_state.max_fds)
+            new_len = drv_ev_state.max_fds;
 
 	for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { /* lock all fd's */
-	    erts_mtx_lock(&drv_ev_state_locks[i].lck);
+	    erts_mtx_lock(&drv_ev_state.locks[i].lck);
 	}
-	drv_ev_state = (drv_ev_state
+	drv_ev_state.v = (drv_ev_state.v
 			? erts_realloc(ERTS_ALC_T_DRV_EV_STATE,
-				       drv_ev_state,
+				       drv_ev_state.v,
 				       sizeof(ErtsDrvEventState)*new_len)
 			: erts_alloc(ERTS_ALC_T_DRV_EV_STATE,
 				     sizeof(ErtsDrvEventState)*new_len));
 	for (i = old_len; i < new_len; i++) {
-	    drv_ev_state[i].fd = (ErtsSysFdType) i;
-	    drv_ev_state[i].driver.select = NULL;
-	    drv_ev_state[i].driver.stop.drv_ptr = NULL;
-            drv_ev_state[i].driver.nif = NULL;
-	    drv_ev_state[i].events = 0;
-	    drv_ev_state[i].remove_cnt = 0;
-	    drv_ev_state[i].type = ERTS_EV_TYPE_NONE;
-	    drv_ev_state[i].flags = 0;
+	    drv_ev_state.v[i].fd = (ErtsSysFdType) i;
+	    drv_ev_state.v[i].driver.select = NULL;
+	    drv_ev_state.v[i].driver.stop.drv_ptr = NULL;
+            drv_ev_state.v[i].driver.nif = NULL;
+	    drv_ev_state.v[i].events = 0;
+	    drv_ev_state.v[i].remove_cnt = 0;
+	    drv_ev_state.v[i].type = ERTS_EV_TYPE_NONE;
+	    drv_ev_state.v[i].flags = 0;
 	}
-	erts_atomic_set_nob(&drv_ev_state_len, new_len);
+	erts_atomic_set_nob(&drv_ev_state.len, new_len);
 	for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) {
-	    erts_mtx_unlock(&drv_ev_state_locks[i].lck);
+	    erts_mtx_unlock(&drv_ev_state.locks[i].lck);
 	}
     }
     /*else already grown by racing thread */
 
-    erts_mtx_unlock(&drv_ev_state_grow_lock);
+    erts_mtx_unlock(&drv_ev_state.grow_lock);
 }
 #endif /* ERTS_SYS_CONTINOUS_FD_NUMBERS */
 
@@ -623,7 +634,7 @@ check_cleanup_active_fd(ErtsSysFdType fd,
     erts_mtx_lock(mtx);
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    state = &drv_ev_state[(int) fd];
+    state = &drv_ev_state.v[(int) fd];
 #else
     state = hash_get_drv_ev_state(fd); /* may be NULL! */
     if (state)
@@ -867,11 +878,11 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
     ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state_len)) {
+    if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state.len)) {
 	if (fd < 0) {
 	    return -1;    
 	}
-	if (fd >= max_fds) {
+	if (fd >= drv_ev_state.max_fds) {
 	    drv_select_large_fd_error(ix, fd, mode, on);
 	    return -1;
 	}
@@ -882,7 +893,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
     erts_mtx_lock(fd_mtx(fd));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    state = &drv_ev_state[(int) fd];
+    state = &drv_ev_state.v[(int) fd];
 #else
     state = hash_get_drv_ev_state(fd); /* may be NULL! */
 #endif
@@ -1096,11 +1107,11 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
     ASSERT(!(resource->monitors && resource->monitors->is_dying));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state_len)) {
+    if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state.len)) {
 	if (fd < 0) {
 	    return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT;
 	}
-	if (fd >= max_fds) {
+	if (fd >= drv_ev_state.max_fds) {
 	    nif_select_large_fd_error(fd, mode, resource, ref);
             return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT;
 	}
@@ -1111,7 +1122,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
     erts_mtx_lock(fd_mtx(fd));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    state = &drv_ev_state[(int) fd];
+    state = &drv_ev_state.v[(int) fd];
 #else
     state = hash_get_drv_ev_state(fd); /* may be NULL! */
 #endif
@@ -1500,7 +1511,7 @@ large_fd_error_common(erts_dsprintf_buf_t *dsbufp, ErtsSysFdType fd)
 {
     erts_dsprintf(dsbufp,
 		  "fd=%d is larger than the largest allowed fd=%d\n",
-		  (int) fd, max_fds - 1);
+		  (int) fd, drv_ev_state.max_fds - 1);
 }
 
 static void
@@ -1828,7 +1839,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 	erts_mtx_lock(fd_mtx(fd));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-	state = &drv_ev_state[ (int) fd];
+	state = &drv_ev_state.v[ (int) fd];
 #else
 	state = hash_get_drv_ev_state(fd);
 	if (!state) {
@@ -2058,16 +2069,16 @@ static int drv_ev_state_cmp(void *des1, void *des2)
 static void *drv_ev_state_alloc(void *des_tmpl)
 {
     ErtsDrvEventState *evstate;
-    erts_spin_lock(&state_prealloc_lock);
-    if (state_prealloc_first == NULL) {
-	erts_spin_unlock(&state_prealloc_lock);
+    erts_spin_lock(&drv_ev_state.prealloc_lock);
+    if (drv_ev_state.prealloc_first == NULL) {
+	erts_spin_unlock(&drv_ev_state.prealloc_lock);
 	evstate = (ErtsDrvEventState *) 
 	    erts_alloc(ERTS_ALC_T_DRV_EV_STATE, sizeof(ErtsDrvEventState));
     } else {
-	evstate = state_prealloc_first;
-	state_prealloc_first = (ErtsDrvEventState *) evstate->hb.next;
-	--num_state_prealloc;
-	erts_spin_unlock(&state_prealloc_lock);
+	evstate = drv_ev_state.prealloc_first;
+	drv_ev_state.prealloc_first = (ErtsDrvEventState *) evstate->hb.next;
+	--drv_ev_state.num_prealloc;
+	erts_spin_unlock(&drv_ev_state.prealloc_lock);
     }
     /* XXX: Already valid data if prealloced, could ignore template! */
     *evstate = *((ErtsDrvEventState *) des_tmpl);
@@ -2077,11 +2088,11 @@ static void *drv_ev_state_alloc(void *des_tmpl)
 
 static void drv_ev_state_free(void *des)
 {
-    erts_spin_lock(&state_prealloc_lock);
-    ((ErtsDrvEventState *) des)->hb.next = &state_prealloc_first->hb;
-    state_prealloc_first = (ErtsDrvEventState *) des;
-    ++num_state_prealloc;
-    erts_spin_unlock(&state_prealloc_lock);
+    erts_spin_lock(&drv_ev_state.prealloc_lock);
+    ((ErtsDrvEventState *) des)->hb.next = &drv_ev_state.prealloc_first->hb;
+    drv_ev_state.prealloc_first = (ErtsDrvEventState *) des;
+    ++drv_ev_state.num_prealloc;
+    erts_spin_unlock(&drv_ev_state.prealloc_lock);
 }
 #endif
 
@@ -2158,17 +2169,17 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
     init_removed_fd_alloc();
     erts_atomic_init_nob(&pollset.removed_list,  (erts_aint_t)NULL);
     {
-        int i;
-        for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) {
-            erts_mtx_init(&drv_ev_state_locks[i].lck, "drv_ev_state", make_small(i),
+	int i;
+	for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) {
+            erts_mtx_init(&drv_ev_state.locks[i].lck, "drv_ev_state", make_small(i),
                 ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
-        }
+	}
     }
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    max_fds = ERTS_CIO_POLL_MAX_FDS();
-    erts_atomic_init_nob(&drv_ev_state_len, 0);
-    drv_ev_state = NULL;
-    erts_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow", NIL,
+    drv_ev_state.max_fds = ERTS_CIO_POLL_MAX_FDS();
+    erts_atomic_init_nob(&drv_ev_state.len, 0);
+    drv_ev_state.v = NULL;
+    erts_mtx_init(&drv_ev_state.grow_lock, "drv_ev_state_grow", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
 #else
     {
@@ -2177,12 +2188,11 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
 	hf.cmp = &drv_ev_state_cmp;
 	hf.alloc = &drv_ev_state_alloc;
 	hf.free = &drv_ev_state_free;
-	num_state_prealloc = 0;
-	state_prealloc_first = NULL;
-	erts_spinlock_init(&state_prealloc_lock,"state_prealloc", NIL,
-        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
-
-	safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state_tab, "drv_ev_state_tab",
+	drv_ev_state.num_prealloc = 0;
+	drv_ev_state.prealloc_first = NULL;
+	erts_spinlock_init(&drv_ev_state.prealloc_lock, "state_prealloc", NIL,
+                               ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
+	safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state.tab, "drv_ev_state_tab",
             ERTS_LOCK_FLAGS_CATEGORY_IO, DRV_EV_STATE_HTAB_SIZE, hf);
     }
 #endif
@@ -2192,7 +2202,7 @@ int
 ERTS_CIO_EXPORT(erts_check_io_max_files)(void)
 {
 #ifdef  ERTS_SYS_CONTINOUS_FD_NUMBERS
-    return max_fds;
+    return drv_ev_state.max_fds;
 #else
     return ERTS_POLL_EXPORT(erts_poll_max_fds)();
 #endif
@@ -2206,17 +2216,17 @@ ERTS_CIO_EXPORT(erts_check_io_size)(void)
     ERTS_CIO_POLL_INFO(pollset.ps, &pi);
     res = pi.memory_size;
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    res += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state_len);
+    res += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state.len);
 #else
-    res += safe_hash_table_sz(&drv_ev_state_tab);
+    res += safe_hash_table_sz(&drv_ev_state.tab);
     {
 	SafeHashInfo hi;
-	safe_hash_get_info(&hi, &drv_ev_state_tab);
+	safe_hash_get_info(&hi, &drv_ev_state.tab);
 	res += hi.objs * sizeof(ErtsDrvEventState);
     }
-    erts_spin_lock(&state_prealloc_lock);
-    res += num_state_prealloc * sizeof(ErtsDrvEventState);
-    erts_spin_unlock(&state_prealloc_lock);
+    erts_spin_lock(&drv_ev_state.prealloc_lock);
+    res += drv_ev_state.num_prealloc * sizeof(ErtsDrvEventState);
+    erts_spin_unlock(&drv_ev_state.prealloc_lock);
 #endif
     return res;
 }
@@ -2248,17 +2258,17 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc)
 
     memory_size = pi.memory_size;
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    memory_size += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state_len);
+    memory_size += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state.len);
 #else
-    memory_size += safe_hash_table_sz(&drv_ev_state_tab);
+    memory_size += safe_hash_table_sz(&drv_ev_state.tab);
     {
 	SafeHashInfo hi;
-	safe_hash_get_info(&hi, &drv_ev_state_tab);
+	safe_hash_get_info(&hi, &drv_ev_state.tab);
 	memory_size += hi.objs * sizeof(ErtsDrvEventState);
     }
-    erts_spin_lock(&state_prealloc_lock);
-    memory_size += num_state_prealloc * sizeof(ErtsDrvEventState);
-    erts_spin_unlock(&state_prealloc_lock);
+    erts_spin_lock(&drv_ev_state.prealloc_lock);
+    memory_size += drv_ev_state.num_prealloc * sizeof(ErtsDrvEventState);
+    erts_spin_unlock(&drv_ev_state.prealloc_lock);
 #endif
 
     hpp = NULL;
@@ -2632,8 +2642,8 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
     erts_thr_progress_block(); /* stop the world to avoid messy locking */
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    counters.epep = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollEvents)*max_fds);
-    ERTS_POLL_EXPORT(erts_poll_get_selected_events)(pollset.ps, counters.epep, max_fds);
+    counters.epep = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollEvents)*drv_ev_state.max_fds);
+    ERTS_POLL_EXPORT(erts_poll_get_selected_events)(pollset.ps, counters.epep, drv_ev_state.max_fds);
     counters.internal_fds = 0;
 #endif
     counters.used_fds = 0;
@@ -2642,16 +2652,16 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
     counters.no_enif_select_structs = 0;
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    len = erts_atomic_read_nob(&drv_ev_state_len);
+    len = erts_atomic_read_nob(&drv_ev_state.len);
     for (fd = 0; fd < len; fd++) {
-	doit_erts_check_io_debug((void *) &drv_ev_state[fd], (void *) &counters);
+	doit_erts_check_io_debug((void *) &drv_ev_state.v[fd], (void *) &counters);
     }
-    for ( ; fd < max_fds; fd++) {
+    for ( ; fd < drv_ev_state.max_fds; fd++) {
 	null_des.fd = fd;
 	doit_erts_check_io_debug((void *) &null_des, (void *) &counters);
     }
 #else
-    safe_hash_for_each(&drv_ev_state_tab, &doit_erts_check_io_debug, (void *) &counters);
+    safe_hash_for_each(&drv_ev_state.tab, &doit_erts_check_io_debug, (void *) &counters);
 #endif
 
     erts_thr_progress_unblock();
@@ -2679,7 +2689,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
 #ifdef ERTS_ENABLE_LOCK_COUNT
 void ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks)(int enable) {
 #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    erts_lcnt_enable_hash_lock_count(&drv_ev_state_tab, ERTS_LOCK_FLAGS_CATEGORY_IO, enable);
+    erts_lcnt_enable_hash_lock_count(&drv_ev_state.tab, ERTS_LOCK_FLAGS_CATEGORY_IO, enable);
 #else
     (void)enable;
 #endif
-- 
cgit v1.2.3


From 7f89338e2dddb47228aeef590b958f5640e17bfb Mon Sep 17 00:00:00 2001
From: Gregory Ostermayr <gregory.ostermayr@gmail.com>
Date: Mon, 18 Sep 2017 15:02:41 -0400
Subject: fix off by one error in docs

---
 erts/emulator/pcre/README.pcre_update.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/pcre/README.pcre_update.md b/erts/emulator/pcre/README.pcre_update.md
index 8caf575d31..599e3d0d12 100644
--- a/erts/emulator/pcre/README.pcre_update.md
+++ b/erts/emulator/pcre/README.pcre_update.md
@@ -2,7 +2,7 @@
 
 ## The basic changes to the PCRE library
 
-To work with the Erlang VM, PCRE has been changed in two important ways:
+To work with the Erlang VM, PCRE has been changed in three important ways:
 
 1. The main execution machine in pcre\_exec has been modified so that
 matching can be interrupted and restarted. This functionality utilizes
-- 
cgit v1.2.3


From 3aaeab5fb7a259aaec66126d5d50231269e32961 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 20 Sep 2017 14:16:14 +0200
Subject: erts: Improve distribution send operations

by ignoring reply earlier.
---
 erts/emulator/drivers/common/inet_drv.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 7b1f4a0e9c..554c48059f 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -2199,13 +2199,16 @@ static int inet_reply_ok(inet_descriptor* desc)
     ErlDrvTermData caller = desc->caller;
     int i = 0;
     
+    desc->caller = 0;
+    if (is_not_internal_pid(caller))
+        return 0;
+
     i = LOAD_ATOM(spec, i, am_inet_reply);
     i = LOAD_PORT(spec, i, desc->dport);
     i = LOAD_ATOM(spec, i, am_ok);
     i = LOAD_TUPLE(spec, i, 3);
     ASSERT(i == sizeof(spec)/sizeof(*spec));
     
-    desc->caller = 0;
     return erl_drv_send_term(desc->dport, caller, spec, i);    
 }
 
-- 
cgit v1.2.3


From 64dec9661d599b71592879c0361a120b1138dac8 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 21 Sep 2017 17:06:56 +0200
Subject: erts: Print the error reason when threads fail to start

---
 erts/emulator/beam/erl_process.c | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 1f696f7ba4..f3f99f1835 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -8600,11 +8600,13 @@ erts_start_schedulers(void)
 	erts_atomic_init_nob(&runq_supervisor_sleeping, 0);
 	if (0 != ethr_event_init(&runq_supervision_event))
 	    erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision event\n");
-	if (0 != ethr_thr_create(&runq_supervisor_tid,
-				 runq_supervisor,
-				 NULL,
-				 &opts))
-	    erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision thread\n");
+        res = ethr_thr_create(&runq_supervisor_tid,
+                              runq_supervisor,
+                              NULL,
+                              &opts);
+	if (0 != res)
+	    erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision thread, "
+                      "error = %d\n", res);
 
     }
 
@@ -8640,7 +8642,7 @@ erts_start_schedulers(void)
             opts.suggested_stack_size = erts_dcpu_sched_thread_suggested_stack_size;
 	    res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts);
 	    if (res != 0)
-		erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d\n", ix);
+		erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d, error = %d\n", ix, res);
 	}
 	for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
 	    ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
@@ -8648,7 +8650,7 @@ erts_start_schedulers(void)
             opts.suggested_stack_size = erts_dio_sched_thread_suggested_stack_size;
 	    res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts);
 	    if (res != 0)
-		erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d\n", ix);
+		erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d, error = %d\n", ix, res);
 	}
     }
 
@@ -8658,7 +8660,7 @@ erts_start_schedulers(void)
 
     res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts);
     if (res != 0)
-	erts_exit(ERTS_ERROR_EXIT, "Failed to create aux thread\n");
+	erts_exit(ERTS_ERROR_EXIT, "Failed to create aux thread, error = %d\n", res);
 
     if (actual < 1)
 	erts_exit(ERTS_ERROR_EXIT,
-- 
cgit v1.2.3


From b54115e73a56d24618d75b83b89820691503a7da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 22 Sep 2017 07:33:19 +0200
Subject: Add testing of erts_debug:df() to the emulator smoke tests

It is too easy to break the disassembler. Make sure that we notice.
---
 erts/emulator/test/emulator_smoke.spec | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/emulator_smoke.spec b/erts/emulator/test/emulator_smoke.spec
index b2d0de8835..fc98ba6823 100644
--- a/erts/emulator/test/emulator_smoke.spec
+++ b/erts/emulator/test/emulator_smoke.spec
@@ -7,3 +7,4 @@
     [consistency],"Not reliable in October and March"}.
 {cases,'Dir',crypto_SUITE,[t_md5]}.
 {cases,'Dir',float_SUITE,[fpe,cmp_integer]}.
+{cases,'Dir',erts_debug_SUITE,[df]}.
-- 
cgit v1.2.3


From ae559453fed714fecd4284c7a4332bc2c8483c29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 20 Sep 2017 19:55:22 +0200
Subject: Correctly append sub-binaries in iolist_to_iovec

The byte_offset of sub-binaries wasn't taken into account for
ProcBins, subtly ruining the results. The test suite didn't catch
it since it didn't check for sub-binaries in particular, and only
checked for equality between variations -- not whether the output
was equal to the input.
---
 erts/emulator/beam/erl_io_queue.c  |  7 ++++---
 erts/emulator/test/iovec_SUITE.erl | 43 +++++++++++++++++++-------------------
 2 files changed, 26 insertions(+), 24 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c
index a01b676d39..190ba6bbb9 100644
--- a/erts/emulator/beam/erl_io_queue.c
+++ b/erts/emulator/beam/erl_io_queue.c
@@ -973,9 +973,10 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) {
     UWord binary_size;
 
     Uint byte_offset, bit_offset, bit_size;
+    byte *binary_data;
+
     Eterm *parent_header;
     Eterm parent_binary;
-    byte *binary_data;
 
     ASSERT(state->bytereds_available > state->bytereds_spent);
 
@@ -1001,14 +1002,14 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) {
             erts_emasculate_writable_binary(pb);
         }
 
-        binary_data = pb->bytes;
+        binary_data = &((byte*)pb->bytes)[byte_offset];
     } else {
         ErlHeapBin *hb = (ErlHeapBin*)parent_header;
 
         ASSERT(thing_subtag(*parent_header) == HEAP_BINARY_SUBTAG);
         ASSERT(is_bin_small);
 
-        binary_data = &((unsigned char*)&hb->data)[byte_offset];
+        binary_data = &((byte*)&hb->data)[byte_offset];
     }
 
     if (!is_bin_small && (state->acc_size == 0 || !is_acc_small)) {
diff --git a/erts/emulator/test/iovec_SUITE.erl b/erts/emulator/test/iovec_SUITE.erl
index 28df36d293..49dc64b0d2 100644
--- a/erts/emulator/test/iovec_SUITE.erl
+++ b/erts/emulator/test/iovec_SUITE.erl
@@ -24,7 +24,8 @@
 
 -export([integer_lists/1, binary_lists/1, empty_lists/1, empty_binary_lists/1,
          mixed_lists/1, improper_lists/1, illegal_lists/1, cons_bomb/1,
-         iolist_to_iovec_idempotence/1, iolist_to_iovec_correctness/1]).
+         sub_binary_lists/1, iolist_to_iovec_idempotence/1,
+         iolist_to_iovec_correctness/1]).
 
 -include_lib("common_test/include/ct.hrl").
 
@@ -34,8 +35,8 @@ suite() ->
 
 all() ->
     [integer_lists, binary_lists, empty_lists, empty_binary_lists, mixed_lists,
-     illegal_lists, improper_lists, cons_bomb, iolist_to_iovec_idempotence,
-     iolist_to_iovec_correctness].
+     sub_binary_lists, illegal_lists, improper_lists, cons_bomb,
+     iolist_to_iovec_idempotence, iolist_to_iovec_correctness].
 
 init_per_suite(Config) ->
     Config.
@@ -46,15 +47,16 @@ end_per_suite(Config) ->
 
 integer_lists(Config) when is_list(Config) ->
     Variations = gen_variations([I || I <- lists:seq(1, 255)]),
+    equivalence_test(fun erlang:iolist_to_iovec/1, Variations).
 
-    equivalence_test(fun erlang:iolist_to_iovec/1, Variations),
-
-    ok.
+sub_binary_lists(Config) when is_list(Config) ->
+    Parent = <<0:256/unit:8, "gazurka">>,
+    <<0:196/unit:8, Child/binary>> = Parent,
+    equivalence_test(fun erlang:iolist_to_iovec/1, gen_variations(Child)).
 
 binary_lists(Config) when is_list(Config) ->
     Variations = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]),
-    equivalence_test(fun erlang:iolist_to_iovec/1, Variations),
-    ok.
+    equivalence_test(fun erlang:iolist_to_iovec/1, Variations).
 
 empty_lists(Config) when is_list(Config) ->
     Variations = gen_variations([[] || _ <- lists:seq(1, 256)]),
@@ -70,8 +72,7 @@ empty_binary_lists(Config) when is_list(Config) ->
 
 mixed_lists(Config) when is_list(Config) ->
     Variations = gen_variations([<<>>, lists:seq(1, 40), <<12, 45, 78>>]),
-    equivalence_test(fun erlang:iolist_to_iovec/1, Variations),
-    ok.
+    equivalence_test(fun erlang:iolist_to_iovec/1, Variations).
 
 illegal_lists(Config) when is_list(Config) ->
     BitStrs = gen_variations(["gurka", <<1:1>>, "gaffel"]),
@@ -82,18 +83,15 @@ illegal_lists(Config) when is_list(Config) ->
     Variations =
         BitStrs ++ BadInts ++ Atoms ++ BadTails,
 
-    illegality_test(fun erlang:iolist_to_iovec/1, Variations),
-
-    ok.
+    illegality_test(fun erlang:iolist_to_iovec/1, Variations).
 
 improper_lists(Config) when is_list(Config) ->
     Variations = [
         [[[[1 | <<2>>] | <<3>>] | <<4>>] | <<5>>],
-        [[<<"test">>, 3] | <<"improper tail">>],
-        [1, 2, 3 | <<"improper tail">>]
+        [[<<1>>, 2] | <<3, 4, 5>>],
+        [1, 2, 3 | <<4, 5>>]
     ],
-    equivalence_test(fun erlang:iolist_to_iovec/1, Variations),
-    ok.
+    equivalence_test(fun erlang:iolist_to_iovec/1, Variations).
 
 cons_bomb(Config) when is_list(Config) ->
     IntBase = gen_variations([I || I <- lists:seq(1, 255)]),
@@ -108,8 +106,7 @@ cons_bomb(Config) when is_list(Config) ->
         end,
 
     Variations = gen_variations([IntBase, BinBase, MixBase], Rounds),
-    equivalence_test(fun erlang:iolist_to_iovec/1, Variations),
-    ok.
+    equivalence_test(fun erlang:iolist_to_iovec/1, Variations).
 
 iolist_to_iovec_idempotence(Config) when is_list(Config) ->
     IntVariations = gen_variations([I || I <- lists:seq(1, 255)]),
@@ -134,11 +131,15 @@ iolist_to_iovec_correctness(Config) when is_list(Config) ->
     ok.
 
 illegality_test(Fun, Variations) ->
-    [{'EXIT',{badarg, _}} = (catch Fun(Variation)) || Variation <- Variations].
+    [{'EXIT',{badarg, _}} = (catch Fun(Variation)) || Variation <- Variations],
+    ok.
 
 equivalence_test(Fun, [Head | _] = Variations) ->
+    %% Check that each variation is equal to the others, and that the sum of
+    %% them is equal to the input.
     Comparand = Fun(Head),
-    [is_iolist_equal(Comparand, Fun(Variation)) || Variation <- Variations],
+    [true = is_iolist_equal(Comparand, Fun(V)) || V <- Variations],
+    true = is_iolist_equal(Variations, Fun(Variations)),
     ok.
 
 is_iolist_equal(A, B) ->
-- 
cgit v1.2.3


From 8521b555ae3e956fd76d9de14c59f64ac13bc9a2 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 26 Sep 2017 10:46:53 +0200
Subject: erts: Add makefile target to check emu register allocation

---
 erts/emulator/Makefile.in         |  18 +++++-
 erts/emulator/utils/beam_emu_vars | 122 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 139 insertions(+), 1 deletion(-)
 create mode 100755 erts/emulator/utils/beam_emu_vars

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 02a9a9a93b..0cf8fee15d 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -499,7 +499,6 @@ ifeq ($(TARGET),win32)
 RELEASE_INCLUDES += sys/$(ERLANG_OSTYPE)/erl_win_dyn_driver.h
 endif
 
-
 .PHONY: release_spec
 ifdef VOID_EMULATOR
 release_spec:
@@ -711,16 +710,27 @@ $(OBJDIR)/beams.$(RES_EXT): $(TARGET)/beams.rc
 
 endif
 
+# We disable the implicit rule of .S -> .o so that the verbose asm
+# generate is not used for compiling erts. This is only a problem on
+# old solaris make
+%.o : %.S
+
 # Usually the same as the default rule, but certain platforms (e.g. win32) mix
 # different compilers
 $(OBJDIR)/beam_emu.o: beam/beam_emu.c
 	$(V_EMU_CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
 
+$(OBJDIR)/beam_emu.S: beam/beam_emu.c
+	$(V_EMU_CC) -S -fverbose-asm $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
+
 $(OBJDIR)/%_pg.o: beam/%.c
 	$(V_CC) $(PROFILE_GENERATE) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
 $(OBJDIR)/%_pu.o: beam/%.c $(PROFILE_USE_DEPS)
 	$(V_CC) $(PROFILE_USE) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
 
+$(OBJDIR)/%_pu.S: beam/%.c $(PROFILE_USE_DEPS)
+	$(V_CC) -S -fverbose-asm $(PROFILE_USE) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
+
 $(OBJDIR)/PROFILE: $(BINDIR)/$(PROFILE_EXECUTABLE)
 	$(V_at)echo " PROFILE ${PROFILE_EXECUTABLE}"
 	$(V_at)rm -f $(OBJDIR)/erl*.profraw
@@ -732,6 +742,8 @@ $(OBJDIR)/PROFILE: $(BINDIR)/$(PROFILE_EXECUTABLE)
 	  -noshell -s estone_SUITE pgo -s init stop >> $(OBJDIR)/PROFILE_LOG
 	$(V_at)touch $@
 
+.SECONDARY: $(patsubst %.o,%_pu.gcda,$(PROFILE_OBJS))
+
 $(OBJDIR)/%_pu.gcda: $(OBJDIR)/PROFILE
 	$(V_at)mv $(OBJDIR)/$*_pg.gcda $@
 	$(V_at)touch $@
@@ -1167,6 +1179,10 @@ DEP_FLAGS=-MM $(MG_FLAG) $(CFLAGS) $(INCLUDES) -Inifs/common -Idrivers/common -I
 SYS_SRC=$(ALL_SYS_SRC)
 endif
 
+.PHONY: check_emu_registers
+check_emu_registers: $(OBJDIR)/beam_emu$(PROFILE_MARKER).S
+	utils/beam_emu_vars -vars 'c_p E HTOP FCALLS I reg' $^
+
 .PHONY: $(TARGET)/gen_git_version.mk
 $(TARGET)/gen_git_version.mk:
 # We touch beam/erl_bif.info.c if we regenerated the git version to force a
diff --git a/erts/emulator/utils/beam_emu_vars b/erts/emulator/utils/beam_emu_vars
new file mode 100755
index 0000000000..c798a4dada
--- /dev/null
+++ b/erts/emulator/utils/beam_emu_vars
@@ -0,0 +1,122 @@
+#!/usr/bin/perl -w
+use strict;
+
+# Analyse beam_emu.s and try to find out the registers
+# used for the important variables in process_main().
+#
+# Works for .s files from clang or gcc. For gcc, the -fverbose-asm
+# option must be used.
+#
+# Example:
+#
+# $ beam-emu-vars -vars 'c_p E HTOP FCALLS I reg freg' beam_emu.s
+# E: %r13
+# FCALLS: %rcx:98 %rax:88 16(%rsp):50 %rdi:6
+# HTOP: %r10:382 64(%rsp):88 72(%rsp):9 24(%rsp):7 %rcx:6 %r15:6 80(%rsp):3 88(%rsp):2
+# I: %rbx
+# c_p: %rbp
+# freg: 48(%rsp):11 %rcx:8 %rdi:5 %rax:4
+# reg: %r12
+#
+# That means that E, I, c_p, reg seems to be assigned to permanent registers.
+# HTOP seems to be assigned %r10, but it is saved to a scratch location
+# before any function calls. FCALLS and freg seems to be saved in a location on
+# the stack and loaded into a register when used.
+#
+# The exit status will be 0 if all variables are assigned to registers (most of
+# the time), and 1 if one or more variables are assigned to a stack location.
+
+my $vars = 'c_p E FCALLS freg HTOP I reg';
+
+while (@ARGV and $ARGV[0] =~ /^-(.*)/) {
+    $_ = $1;
+    shift;
+    ($vars = shift), next if /^vars/;
+    die "$0: Bad option: -$_\n";
+}
+
+my @vars = split(" ", $vars);
+my %vars;
+@vars{@vars} = @vars;
+
+my $inside;
+my %count;
+
+if (@ARGV != 1) {
+    usage();
+}
+
+while (<>) {
+    if (!$inside && /[.]globl\s*_?process_main/) {
+	$inside = 1;
+    } elsif ($inside && /[.]globl/) {
+	last;
+    }
+    if ($inside) {
+	if (/##DEBUG_VALUE:\s*process_main:([A-Za-z]*)\s*<-\s*(.*)/) {
+	    # clang
+	    my($var,$reg) = ($1,$2);
+	    next if $reg =~ /^[-\d]+$/; # Ignore if number.
+	    $count{$var}->{$reg}++ if $vars{$var};
+	    next;
+	}
+
+	# Parse gcc verbose arguments. Comments are marked with
+	# one '#' (clang marks its comments with two '#').
+	my($src,$dst,$comment) = /movq\s+([^#]+), ([^#]+)#(?!#)\s*(.*)/;
+	next unless $comment;
+	$dst =~ s/\s*$//;
+	my($vsrc,$vdst) = split /,/, $comment, 2;
+	$vdst =~ s/^\s//;
+	update_count(\%count, $vsrc, $src);
+	update_count(\%count, $vdst, $dst);
+	if ($vars{$vdst} and $vsrc eq '%sfp') {
+	    $count{$vdst}->{$src}++;
+	}
+    }
+}
+
+my @first;
+
+OUTER:
+for my $var (sort keys %count) {
+    my $total = 0;
+
+    foreach my $reg (keys %{$count{$var}}) {
+	$total += $count{$var}->{$reg}++;
+    }
+
+    foreach my $reg (keys %{$count{$var}}) {
+	if ($count{$var}->{$reg} > 0.9*$total) {
+	    print "$var: $reg\n";
+	    push @first, $var;
+	    next OUTER;
+	}
+    }
+
+    my @r;
+    foreach my $reg (keys %{$count{$var}}) {
+	push @r, $reg;
+    }
+    @r = sort { $count{$var}->{$b} <=> $count{$var}->{$a} } @r;
+    @r = map { "$_:$count{$var}->{$_}" } @r;
+    push @first, $r[0];
+    print "$var: ", join(' ', @r), "\n";
+}
+
+foreach (@first) {
+    exit 1 if /%rsp/;
+}
+exit 0;
+
+sub update_count {
+    my($count_ref,$var,$reg) = @_;
+    return unless $vars{$var};
+    ${${$count_ref}{$var}}{$reg}++;
+}
+
+sub usage {
+    die qq[usage: beam_emu_vars [ -vars "var1 var2..." ] <filename>.s\n\n] .
+	"The exit status is 0 if all variables are assigned to registers,\n" .
+	"and 1 if one or more variables are allocated to a stack location.\n";
+}
-- 
cgit v1.2.3


From c9515d7a93b57ed43c16fd88eb5892e270935814 Mon Sep 17 00:00:00 2001
From: Dmytro Lytovchenko <dmytro.lytovchenko@gmail.com>
Date: Mon, 31 Jul 2017 17:57:35 +0200
Subject: erts: On zero-size files attempt to read until EOF

---
 erts/emulator/drivers/common/efile_drv.c | 61 ++++++++++++++++++++++++++++++--
 erts/emulator/test/efile_SUITE.erl       | 49 +++++++++++++++++++++++--
 2 files changed, 105 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c
index a1b15a2199..4e1d2f0d7f 100644
--- a/erts/emulator/drivers/common/efile_drv.c
+++ b/erts/emulator/drivers/common/efile_drv.c
@@ -1261,6 +1261,50 @@ static void free_read_line(void *data)
     EF_FREE(d);
 }
 
+void read_file_zero_size(struct t_data* d);
+#define ZERO_FILE_CHUNK (64 * 1024)
+
+/* [ERL-327] Some special files like /proc/... have reported size 0 */
+void read_file_zero_size(struct t_data* d) {
+    size_t total_read_size = 0;
+    size_t allocated_size = ZERO_FILE_CHUNK; /* allocd in invoke_read_file */
+    for (;;) {
+        size_t read_result;
+
+        /* Read until we hit EOF (read less than FILE_SEGMENT_READ) */
+        d->result_ok = efile_read(&d->errInfo,
+                                  EFILE_MODE_READ,
+                                  (int) d->fd,
+                                  (d->c.read_file.binp->orig_bytes +
+                                   total_read_size),
+                                  ZERO_FILE_CHUNK,
+                                  &read_result);
+        if (!d->result_ok) {
+            break;
+        }
+
+        total_read_size += read_result;
+        d->c.read_file.offset += read_result;
+        if (read_result < ZERO_FILE_CHUNK) {
+            break;
+        }
+
+        /* Grow before the next read call */
+        allocated_size = total_read_size + ZERO_FILE_CHUNK;
+        d->c.read_file.binp = driver_realloc_binary(d->c.read_file.binp,
+                                                    allocated_size);
+    }
+
+    /* Finalize the memory usage. Hopefully it was read fully on the first
+     * go, so the binary allocation overhead becomes:
+     * alloc ZERO_FILE_CHUNK (64kb) -> realloc real_size */
+    if (allocated_size != total_read_size) {
+        d->c.read_file.binp = driver_realloc_binary(d->c.read_file.binp,
+                                                    total_read_size);
+    }
+    d->again = 0;
+}
+
 static void invoke_read_file(void *data)
 {
     struct t_data *d = (struct t_data *) data;
@@ -1279,9 +1323,15 @@ static void invoke_read_file(void *data)
 	}
 	d->fd = fd;
 	d->c.read_file.size = (int) size;
-	if (size < 0 || size != d->c.read_file.size ||
-	    ! (d->c.read_file.binp = 
-	       driver_alloc_binary(d->c.read_file.size))) {
+
+        /* For zero sized files allocate a reasonable chunk to attempt reading
+         * anyway. Note: This will eat ZERO_FILE_CHUNK bytes for any 0 file
+         * and free them immediately after (if the file was empty). */
+        ERTS_ASSERT(size >= 0);
+        d->c.read_file.binp = driver_alloc_binary(size != 0 ? (size_t)size
+                                                            : ZERO_FILE_CHUNK);
+
+	if (size < 0 || size != d->c.read_file.size || !d->c.read_file.binp) {
 	    d->result_ok = 0;
 	    d->errInfo.posix_errno = ENOMEM;
 	    goto close;
@@ -1290,6 +1340,11 @@ static void invoke_read_file(void *data)
     }
     /* Invariant: d->c.read_file.size >= d->c.read_file.offset */
     
+    if (d->c.read_file.size == 0) {
+        read_file_zero_size(d);
+        goto close;
+    }
+
     read_size = (size_t) (d->c.read_file.size - d->c.read_file.offset);
     if (! read_size) goto close;
     chop = d->again && read_size >= FILE_SEGMENT_READ*2;
diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl
index f0e1bcf04b..08d5597d78 100644
--- a/erts/emulator/test/efile_SUITE.erl
+++ b/erts/emulator/test/efile_SUITE.erl
@@ -19,16 +19,20 @@
 
 -module(efile_SUITE).
 -export([all/0, suite/0]).
--export([iter_max_files/1, async_dist/1]).
+-export([async_dist/1,
+         iter_max_files/1,
+         proc_zero_sized_files/1
+        ]).
 
 -export([do_iter_max_files/2, do_async_dist/1]).
 
 -include_lib("common_test/include/ct.hrl").
+-include_lib("stdlib/include/assert.hrl").
 
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
 all() -> 
-    [iter_max_files, async_dist].
+    [iter_max_files, async_dist, proc_zero_sized_files].
 
 do_async_dist(Dir) ->
     X = 100,
@@ -162,3 +166,44 @@ open_files(Name) ->
             %		  io:format("Error reason: ~p", [_Reason]),
             []
     end.
+
+%% @doc If /proc filesystem exists (no way to know if it is real proc or just
+%% a /proc directory), let's read some zero sized files 500 times each, while
+%% ensuring that response isn't empty << >>
+proc_zero_sized_files(Config) when is_list(Config) ->
+    {Type, Flavor} = os:type(),
+    %% Some files which exist on Linux but might be missing on other systems
+    Inputs = ["/proc/cpuinfo",
+              "/proc/meminfo",
+              "/proc/partitions",
+              "/proc/swaps",
+              "/proc/version",
+              "/proc/uptime",
+              %% curproc is present on freebsd
+              "/proc/curproc/cmdline"],
+    case filelib:is_dir("/proc") of
+        false -> {skip, "/proc not found"}; % skip the test if no /proc
+        _ when Type =:= unix andalso Flavor =:= sunos ->
+            %% SunOS has a /proc, but no zero sized special files
+            {skip, "sunos does not have any zero sized special files"};
+        true ->
+            %% Take away files which do not exist in proc
+            Inputs1 = lists:filter(fun filelib:is_file/1, Inputs),
+
+            %% Fail if none of mentioned files exist in /proc, did we just get
+            %% a normal /proc directory without any special files?
+            ?assertNotEqual([], Inputs1),
+
+            %% For 6 inputs and 500 attempts each this do run anywhere
+            %% between 500 and 3000 function calls.
+            lists:foreach(
+                fun(Filename) -> do_proc_zero_sized(Filename, 500) end,
+                Inputs1)
+    end.
+
+%% @doc Test one file N times to also trigger possible leaking fds and memory
+do_proc_zero_sized(_Filename, 0) -> ok;
+do_proc_zero_sized(Filename, N) ->
+    Data = file:read_file(Filename),
+    ?assertNotEqual(<<>>, Data),
+    do_proc_zero_sized(Filename, N-1).
-- 
cgit v1.2.3


From eae496a72e270fd7af411714738e99a7fadfd19b Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 6 Sep 2017 17:00:14 +0200
Subject: Don't allow null in filenames

---
 erts/emulator/beam/erl_bif_port.c |  2 +-
 erts/emulator/beam/erl_unicode.c  | 58 +++++++++++++++++++++++++++++++++++----
 erts/emulator/beam/global.h       |  2 +-
 3 files changed, 55 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index ff03151619..1121a450cd 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -1084,7 +1084,7 @@ static byte* convert_environment(Process* p, Eterm env)
 	goto done;
     }
 
-    if ((size = erts_native_filename_need(all,encoding)) < 0) {
+    if ((size = erts_native_filename_need(all, encoding, 1)) < 0) {
 	goto done;
     }
 
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 2d1d1443a7..efd2ca3db2 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -1988,7 +1988,7 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu
 	is_list(name) || 
 	(allow_empty && is_nil(name))) {
 	Sint need;
-	if ((need = erts_native_filename_need(name,encoding)) < 0) {
+	if ((need = erts_native_filename_need(name, encoding, 0)) < 0) {
 	    return NULL;
 	}
 	if (encoding == ERL_FILENAME_WIN_WCHAR) {
@@ -2152,12 +2152,13 @@ Eterm erts_convert_native_to_filename(Process *p, byte *bytes)
 }
 
 
-Sint erts_native_filename_need(Eterm ioterm, int encoding) 
+Sint erts_native_filename_need(Eterm ioterm, int encoding, int allow_null) 
 {
     Eterm *objp;
     Eterm obj;
     DECLARE_ESTACK(stack);
     Sint need = 0;
+    int seen_null = 0;
 
     if (is_atom(ioterm)) {
 	Atom* ap;
@@ -2194,6 +2195,24 @@ Sint erts_native_filename_need(Eterm ioterm, int encoding)
 	default:
 	    need = -1;
 	}
+        if (!allow_null) {
+            /*
+             * Do not allow null in
+             * the middle of filenames
+             */
+            if (need > 0) {
+                byte *name = ap->name;
+                int len = ap->len;
+                for (i = 0; i < len; i++) {
+                    if (name[i] == 0)
+                        seen_null = 1;
+                    else if (seen_null) {
+                        need = -1;
+                        break;
+                    }
+                }
+            }
+        }
 	DESTROY_ESTACK(stack);
 	return need;
     }
@@ -2224,6 +2243,18 @@ L_Again:   /* Restart with sublist, old listend was pushed on stack */
 		if (is_small(obj)) { /* Always small */
 		    for(;;) {
 			Uint x = unsigned_val(obj);
+                        if (!allow_null) {
+                            /*
+                             * Do not allow null in
+                             * the middle of filenames
+                             */
+                            if (x == 0)
+                                seen_null = 1;
+                            else if (seen_null) {
+                                DESTROY_ESTACK(stack);
+                                return ((Sint) -1);
+                            }
+                        }
 			switch (encoding) {
 			case ERL_FILENAME_LATIN1:
 			    if (x > 255) {
@@ -2515,6 +2546,7 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
 	BIF_ERROR(BIF_P,BADARG);
     }
     if (is_binary(BIF_ARG_1)) {
+        int seen_null = 0;
 	byte *temp_alloc = NULL;
 	byte *bytes;
 	byte *err_pos;
@@ -2524,10 +2556,18 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
 	size = binary_size(BIF_ARG_1);
 	bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
 	if (encoding != ERL_FILENAME_WIN_WCHAR) {
+            Uint i;
 	    /*Add 0 termination only*/
 	    bin_term = new_binary(BIF_P, NULL, size+1);
 	    bin_p = binary_bytes(bin_term);
-	    memcpy(bin_p,bytes,size);
+            for (i = 0; i < size; i++) {
+                /* Don't allow null in the middle of filenames... */
+                if (bytes[i] == 0)
+                    seen_null = 1;
+                else if (seen_null)
+                    goto bin_name_error;
+                bin_p[i] = bytes[i];
+            }
 	    bin_p[size]=0;
 	    erts_free_aligned_binary_bytes(temp_alloc);
 	    BIF_RET(bin_term);
@@ -2541,6 +2581,11 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
 	    bin_term = new_binary(BIF_P, 0, (size+1)*2);
 	    bin_p = binary_bytes(bin_term);
 	    while (size--) {
+                /* Don't allow null in the middle of filenames... */
+                if (*bytes == 0)
+                    seen_null = 1;
+                else if (seen_null)
+                    goto bin_name_error;
 		*bin_p++ = *bytes++;
 		*bin_p++ = 0;
 	    }
@@ -2558,11 +2603,14 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
 	bin_p[num_chars*2+1] = 0;
 	erts_free_aligned_binary_bytes(temp_alloc);
 	BIF_RET(bin_term);
+    bin_name_error:
+        erts_free_aligned_binary_bytes(temp_alloc);
+        BIF_ERROR(BIF_P,BADARG);
     } /* binary */   
 	    
 
-    if ((need = erts_native_filename_need(BIF_ARG_1,encoding)) < 0) {
-	BIF_ERROR(BIF_P,BADARG);
+    if ((need = erts_native_filename_need(BIF_ARG_1, encoding, 0)) < 0) {
+        BIF_ERROR(BIF_P,BADARG);
     }
     if (encoding == ERL_FILENAME_WIN_WCHAR) {
 	need += 2;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 182d3aa44e..ddf7f03265 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1253,7 +1253,7 @@ void erts_init_unicode(void);
 Sint erts_unicode_set_loop_limit(Sint limit);
 
 void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) ;
-Sint erts_native_filename_need(Eterm ioterm, int encoding);
+Sint erts_native_filename_need(Eterm ioterm, int encoding, int allow_null);
 void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars);
 int erts_analyze_utf8(byte *source, Uint size, 
 			byte **err_pos, Uint *num_chars, int *left);
-- 
cgit v1.2.3


From 281ae7c2cf7e2dfd48cf50b2f68fd76f7c6ab0e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 25 Sep 2017 07:18:56 +0200
Subject: Eliminate MY_IS_SSMALL()

For a long time, there has been the two macros IS_SSMALL() and
MY_IS_SSMALL() that do exactly the same thing.

There should only be one, and it should be called IS_SSMALL().

However, we must decide which implementation to use. When
MY_IS_SSMALL() was introduced a long time ago, it was the most
efficient. In a modern C compiler, there might not be any
difference.

To find out, I used the following small C program to examine
the code generation:

    #include <stdio.h>

    typedef unsigned int Uint32;
    typedef unsigned long Uint64;
    typedef long Sint;

    #define SWORD_CONSTANT(Const) Const##L
    #define SMALL_BITS	(64-4)
    #define MAX_SMALL	((SWORD_CONSTANT(1) << (SMALL_BITS-1))-1)
    #define MIN_SMALL	(-(SWORD_CONSTANT(1) << (SMALL_BITS-1)))

    #define MY_IS_SSMALL32(x) (((Uint32) ((((x)) >> (SMALL_BITS-1)) + 1)) < 2)
    #define MY_IS_SSMALL64(x) (((Uint64) ((((x)) >> (SMALL_BITS-1)) + 1)) < 2)
    #define MY_IS_SSMALL(x) (sizeof(x) == sizeof(Uint32) ? MY_IS_SSMALL32(x) : MY_IS_SSMALL64(x))

    #define IS_SSMALL(x)      (((x) >= MIN_SMALL) && ((x) <= MAX_SMALL))

    void original(Sint n)
    {
        if (IS_SSMALL(n)) {
            printf("yes\n");
        }
    }

    void enhanced(Sint n)
    {
        if (MY_IS_SSMALL(n)) {
            printf("yes\n");
        }
    }

gcc 7.2 produced the following code for the original() function:

.LC0:
  .string "yes"
original(long):
  movabs rax, 576460752303423488
  add rdi, rax
  movabs rax, 1152921504606846975
  cmp rdi, rax
  jbe .L4
  rep ret
.L4:
  mov edi, OFFSET FLAT:.LC0
  jmp puts

clang 5.0.0 produced the following code which is slightly better:

original(long):
  movabs rax, 576460752303423488
  add rax, rdi
  shr rax, 60
  jne .LBB0_1
  mov edi, .Lstr
  jmp puts # TAILCALL
.LBB0_1:
  ret
.Lstr:
  .asciz "yes"

However, in the context of beam_emu.c, clang could produce
similar to what gcc produced.

gcc 7.2 produced the following code when MY_IS_SSMALL() was used:

.LC0:
  .string "yes"
enhanced(long):
  sar rdi, 59
  add rdi, 1
  cmp rdi, 1
  jbe .L4
  rep ret
.L4:
  mov edi, OFFSET FLAT:.LC0
  jmp puts

clang produced similar code.

This code seems to be the cheapest. There are four instructions, and
there is no loading of huge integer constants.
---
 erts/emulator/beam/arith_instrs.tab | 11 ++++-------
 erts/emulator/beam/big.h            | 15 ++++++++++++++-
 erts/emulator/beam/erl_arith.c      | 14 +++++---------
 erts/emulator/beam/erl_gc.c         |  4 ++--
 erts/emulator/beam/erl_term.h       |  1 -
 erts/emulator/beam/external.c       |  2 +-
 6 files changed, 26 insertions(+), 21 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab
index 7c9cd47e28..3db19df3c4 100644
--- a/erts/emulator/beam/arith_instrs.tab
+++ b/erts/emulator/beam/arith_instrs.tab
@@ -51,8 +51,7 @@ plus.fetch(Op1, Op2) {
 plus.execute(Fail, Live, Dst) {
     if (ERTS_LIKELY(is_both_small(PlusOp1, PlusOp2))) {
         Sint i = signed_val(PlusOp1) + signed_val(PlusOp2);
-        ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
-        if (ERTS_LIKELY(MY_IS_SSMALL(i))) {
+        if (ERTS_LIKELY(IS_SSMALL(i))) {
             $Dst = make_small(i);
             $NEXT0();
         }
@@ -74,8 +73,7 @@ minus.fetch(Op1, Op2) {
 minus.execute(Fail, Live, Dst) {
     if (ERTS_LIKELY(is_both_small(MinusOp1, MinusOp2))) {
         Sint i = signed_val(MinusOp1) - signed_val(MinusOp2);
-        ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
-        if (ERTS_LIKELY(MY_IS_SSMALL(i))) {
+        if (ERTS_LIKELY(IS_SSMALL(i))) {
             $Dst = make_small(i);
             $NEXT0();
         }
@@ -100,8 +98,7 @@ increment.execute(IncrementVal, Live, Dst) {
     increment_val = $IncrementVal;
     if (ERTS_LIKELY(is_small(increment_reg_val))) {
         Sint i = signed_val(increment_reg_val) + increment_val;
-        ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
-        if (ERTS_LIKELY(MY_IS_SSMALL(i))) {
+        if (ERTS_LIKELY(IS_SSMALL(i))) {
             $Dst = make_small(i);
             $NEXT0();
         }
@@ -142,7 +139,7 @@ i_int_div(Fail, Live, Op1, Op2, Dst) {
         $BIF_ERROR_ARITY_2($Fail, BIF_intdiv_2, op1, op2);
     } else if (ERTS_LIKELY(is_both_small(op1, op2))) {
         Sint ires = signed_val(op1) / signed_val(op2);
-        if (ERTS_LIKELY(MY_IS_SSMALL(ires))) {
+        if (ERTS_LIKELY(IS_SSMALL(ires))) {
             $Dst = make_small(ires);
             $NEXT0();
         }
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index 48efce20e7..7556205063 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -70,7 +70,20 @@ typedef Uint  dsize_t;	 /* Vector size type */
 
 /* Check for small */
 #define IS_USMALL(sgn,x)  ((sgn) ? ((x) <= MAX_SMALL+1) : ((x) <= MAX_SMALL))
-#define IS_SSMALL(x)      (((x) >= MIN_SMALL) && ((x) <= MAX_SMALL))
+
+/*
+ * It seems that both clang and gcc will generate sub-optimal code
+ * for the more obvious way to write the range check:
+ *
+ *    #define IS_SSMALL(x)  (((x) >= MIN_SMALL) && ((x) <= MAX_SMALL))
+ *
+ * Note that IS_SSMALL() may be used in the 32-bit emulator with
+ * a Uint64 argument. Therefore, we must test the size of the argument
+ * to ensure that the cast does not discard the high-order 32 bits.
+ */
+#define _IS_SSMALL32(x) (((Uint32) ((((x)) >> (SMALL_BITS-1)) + 1)) < 2)
+#define _IS_SSMALL64(x) (((Uint64) ((((x)) >> (SMALL_BITS-1)) + 1)) < 2)
+#define IS_SSMALL(x) (sizeof(x) == sizeof(Uint32) ? _IS_SSMALL32(x) : _IS_SSMALL64(x))
 
 /* The heap size needed for a bignum */
 #define BIG_NEED_SIZE(x)  ((x) + 1)
diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c
index f2a3e411ec..b6625db0d3 100644
--- a/erts/emulator/beam/erl_arith.c
+++ b/erts/emulator/beam/erl_arith.c
@@ -114,7 +114,7 @@ BIF_RETTYPE intdiv_2(BIF_ALIST_2)
     }
     if (is_both_small(BIF_ARG_1,BIF_ARG_2)){
 	Sint ires = signed_val(BIF_ARG_1) / signed_val(BIF_ARG_2);
-	if (MY_IS_SSMALL(ires))
+	if (IS_SSMALL(ires))
 	    BIF_RET(make_small(ires));
     } 
     BIF_RET(erts_int_div(BIF_P, BIF_ARG_1, BIF_ARG_2));
@@ -340,8 +340,7 @@ erts_mixed_plus(Process* p, Eterm arg1, Eterm arg2)
 		switch ((arg2 & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) {
 		case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE):
 		    ires = signed_val(arg1) + signed_val(arg2);
-		    ASSERT(MY_IS_SSMALL(ires) == IS_SSMALL(ires));
-		    if (MY_IS_SSMALL(ires)) {
+		    if (IS_SSMALL(ires)) {
 			return make_small(ires);
 		    } else {
 			hp = HAlloc(p, 2);
@@ -486,8 +485,7 @@ erts_mixed_minus(Process* p, Eterm arg1, Eterm arg2)
 		switch ((arg2 & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) {
 		case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE):
 		    ires = signed_val(arg1) - signed_val(arg2);
-		    ASSERT(MY_IS_SSMALL(ires) == IS_SSMALL(ires));
-		    if (MY_IS_SSMALL(ires)) {
+		    if (IS_SSMALL(ires)) {
 			return make_small(ires);
 		    } else {
 			hp = HAlloc(p, 2);
@@ -1181,8 +1179,7 @@ erts_gc_mixed_plus(Process* p, Eterm* reg, Uint live)
 		switch ((arg2 & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) {
 		case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE):
 		    ires = signed_val(arg1) + signed_val(arg2);
-		    ASSERT(MY_IS_SSMALL(ires) == IS_SSMALL(ires));
-		    if (MY_IS_SSMALL(ires)) {
+		    if (IS_SSMALL(ires)) {
 			return make_small(ires);
 		    } else {
 			if (ERTS_NEED_GC(p, 2)) {
@@ -1349,8 +1346,7 @@ erts_gc_mixed_minus(Process* p, Eterm* reg, Uint live)
 		switch ((arg2 & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) {
 		case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE):
 		    ires = signed_val(arg1) - signed_val(arg2);
-		    ASSERT(MY_IS_SSMALL(ires) == IS_SSMALL(ires));
-		    if (MY_IS_SSMALL(ires)) {
+		    if (IS_SSMALL(ires)) {
 			return make_small(ires);
 		    } else {
 			if (ERTS_NEED_GC(p, 2)) {
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 8344c164fa..97a1ca915f 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -337,7 +337,7 @@ erts_heap_sizes(Process* p)
 
     for (i = num_heap_sizes-1; i >= 0; i--) {
 	n += 2;
-	if (!MY_IS_SSMALL(heap_sizes[i])) {
+	if (!IS_SSMALL(heap_sizes[i])) {
 	    big += BIG_UINT_HEAP_SIZE;
 	}
     }
@@ -352,7 +352,7 @@ erts_heap_sizes(Process* p)
 	Eterm num;
 	Sint sz = heap_sizes[i];
 
-	if (MY_IS_SSMALL(sz)) {
+	if (IS_SSMALL(sz)) {
 	    num = make_small(sz);
 	} else {
 	    num = uint_to_big(sz, bigp);
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 842802f8d9..6daf043117 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -270,7 +270,6 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm)
 #define is_byte(x)	(((x) & ((~(Uint)0 << (_TAG_IMMED1_SIZE+8)) + _TAG_IMMED1_MASK)) == _TAG_IMMED1_SMALL)
 #define is_valid_bit_size(x) (((Sint)(x)) >= 0 && ((x) & 0x7F) == _TAG_IMMED1_SMALL)
 #define is_not_valid_bit_size(x) (!is_valid_bit_size((x)))
-#define MY_IS_SSMALL(x) (((Uint) ((((x)) >> (SMALL_BITS-1)) + 1)) < 2)
 #define _unchecked_unsigned_val(x)	((x) >> _TAG_IMMED1_SIZE)
 _ET_DECLARE_CHECKED(Uint,unsigned_val,Eterm)
 #define unsigned_val(x)	_ET_APPLY(unsigned_val,(x))
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 60cf09dc07..970158933f 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3115,7 +3115,7 @@ dec_term(ErtsDistExternal *edep,
 #if defined(ARCH_64)
 		*objp = make_small(sn);
 #else
-		if (MY_IS_SSMALL(sn)) {
+		if (IS_SSMALL(sn)) {
 		    *objp = make_small(sn);
 		} else {
 		    *objp = small_to_big(sn, hp);
-- 
cgit v1.2.3


From bf9f00f16b9825c5df235c2df4b325f97fed199a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Fri, 29 Sep 2017 14:51:09 +0200
Subject: Fix minor incompatibilities in inflate behavior

When presented with multiple valid but concatenated streams, the
old driver returned an empty result once the end of the first
stream was reached, and kept doing so even if fed new data. The
new driver/NIF returned a data_error instead.

zlib:inflateInit/3 has been added to control this behavior, but is
not yet ready for public use.
---
 erts/emulator/nifs/common/zlib_nif.c | 127 ++++++++++++++---------------------
 1 file changed, 51 insertions(+), 76 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/nifs/common/zlib_nif.c b/erts/emulator/nifs/common/zlib_nif.c
index a1a65e1946..a9c5b05e47 100644
--- a/erts/emulator/nifs/common/zlib_nif.c
+++ b/erts/emulator/nifs/common/zlib_nif.c
@@ -69,11 +69,27 @@ typedef enum {
     ST_DEFLATE = 1,
     ST_INFLATE = 2,
     ST_CLOSED = 3
-} zlib_state;
+} zlib_state_t;
+
+/* Controls what to do when the user attempts to decompress more data after
+ * Z_STREAM_END has been returned:
+ *
+ * - 'cut' wipes all further input and returns empty results until reset by
+ * the user. This is the default behavior, matching that of the old driver.
+ * - 'reset' resets the state without discarding any input, making it possible
+ * to decompress blindly concatenated streams.
+ * - 'error' crashes with a data error. */
+typedef enum {
+    EOS_BEHAVIOR_ERROR = 0,
+    EOS_BEHAVIOR_RESET = 1,
+    EOS_BEHAVIOR_CUT = 2
+} zlib_eos_behavior_t;
 
 typedef struct {
     z_stream s;
-    zlib_state state;
+    zlib_state_t state;
+
+    zlib_eos_behavior_t eos_behavior;
 
     /* These refer to the plaintext CRC, and are only needed for zlib:crc32/1
      * which is deprecated. */
@@ -102,7 +118,6 @@ typedef struct {
 static ERL_NIF_TERM zlib_open(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 static ERL_NIF_TERM zlib_close(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 static ERL_NIF_TERM zlib_deflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
-static ERL_NIF_TERM zlib_deflateInit2(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 static ERL_NIF_TERM zlib_deflateSetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 static ERL_NIF_TERM zlib_deflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 static ERL_NIF_TERM zlib_deflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
@@ -110,7 +125,6 @@ static ERL_NIF_TERM zlib_deflateParams(ErlNifEnv *env, int argc, const ERL_NIF_T
 static ERL_NIF_TERM zlib_deflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 
 static ERL_NIF_TERM zlib_inflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
-static ERL_NIF_TERM zlib_inflateInit2(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 static ERL_NIF_TERM zlib_inflateSetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 static ERL_NIF_TERM zlib_inflateGetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 static ERL_NIF_TERM zlib_inflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
@@ -130,8 +144,7 @@ static ERL_NIF_TERM zlib_enqueue_input(ErlNifEnv *env, int argc, const ERL_NIF_T
 
 static ErlNifFunc nif_funcs[] = {
     /* deflate */
-    {"deflateInit_nif", 2, zlib_deflateInit},
-    {"deflateInit_nif", 6, zlib_deflateInit2},
+    {"deflateInit_nif", 6, zlib_deflateInit},
     {"deflateSetDictionary_nif", 2, zlib_deflateSetDictionary},
     {"deflateReset_nif", 1, zlib_deflateReset},
     {"deflateEnd_nif", 1, zlib_deflateEnd},
@@ -139,8 +152,7 @@ static ErlNifFunc nif_funcs[] = {
     {"deflate_nif", 4, zlib_deflate},
 
     /* inflate */
-    {"inflateInit_nif", 1, zlib_inflateInit},
-    {"inflateInit_nif", 2, zlib_inflateInit2},
+    {"inflateInit_nif", 3, zlib_inflateInit},
     {"inflateSetDictionary_nif", 2, zlib_inflateSetDictionary},
     {"inflateGetDictionary_nif", 1, zlib_inflateGetDictionary},
     {"inflateReset_nif", 1, zlib_inflateReset},
@@ -509,9 +521,11 @@ static ERL_NIF_TERM zlib_open(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[
     d->s.opaque = d;
     d->s.data_type = Z_BINARY;
 
-    d->state = ST_NONE;
+    d->eos_behavior = EOS_BEHAVIOR_CUT;
     d->eos_seen = 0;
 
+    d->state = ST_NONE;
+
     d->want_output_crc = 0;
     d->want_input_crc = 0;
     d->is_raw_stream = 0;
@@ -550,42 +564,6 @@ static ERL_NIF_TERM zlib_close(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv
 /* deflate */
 
 static ERL_NIF_TERM zlib_deflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
-    zlib_data_t *d;
-    int level, res;
-
-    if(argc != 2 || !get_zlib_data(env, argv[0], &d) ||
-                    !enif_get_int(env, argv[1], &level)) {
-        return enif_make_badarg(env);
-    } else if(!zlib_process_check(env, d)) {
-        return enif_raise_exception(env, am_not_on_controlling_process);
-    } else if(d->state != ST_NONE) {
-        return enif_raise_exception(env, am_already_initialized);
-    }
-
-    res = deflateInit(&d->s, level);
-
-    if(res == Z_OK) {
-        d->state = ST_DEFLATE;
-        d->eos_seen = 0;
-
-        /* FIXME: crc32/1 is documented as returning "the current calculated
-         * checksum," but failed to mention that the old implementation only
-         * calculated it when WindowBits < 0 (See zlib_deflateInit2).
-         *
-         * We could fix this behavior by setting d->want_input_crc to 1 here,
-         * but we've decided to retain this quirk since the performance hit is
-         * quite significant. */
-        d->want_output_crc = 0;
-        d->want_input_crc = 0;
-
-        d->output_crc = crc32(0L, Z_NULL, 0);
-        d->input_crc = crc32(0L, Z_NULL, 0);
-    }
-
-    return zlib_return(env, res);
-}
-
-static ERL_NIF_TERM zlib_deflateInit2(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
     zlib_data_t *d;
     int level, method, windowBits, memLevel, strategy, res;
 
@@ -741,39 +719,12 @@ static ERL_NIF_TERM zlib_deflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar
 
 static ERL_NIF_TERM zlib_inflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
     zlib_data_t *d;
-    int res;
 
-    if(argc != 1 || !get_zlib_data(env, argv[0], &d)) {
-        return enif_make_badarg(env);
-    } else if(!zlib_process_check(env, d)) {
-        return enif_raise_exception(env, am_not_on_controlling_process);
-    } else if(d->state != ST_NONE) {
-        return enif_raise_exception(env, am_already_initialized);
-    }
-
-    res = inflateInit(&d->s);
-
-    if(res == Z_OK) {
-        d->state = ST_INFLATE;
-        d->eos_seen = 0;
-
-        d->want_output_crc = 0;
-        d->want_input_crc = 0;
-        d->is_raw_stream = 0;
-
-        d->output_crc = crc32(0L, Z_NULL, 0);
-        d->input_crc = crc32(0L, Z_NULL, 0);
-    }
-
-    return zlib_return(env, res);
-}
+    int windowBits, eosBehavior, res;
 
-static ERL_NIF_TERM zlib_inflateInit2(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
-    zlib_data_t *d;
-    int windowBits, res;
-
-    if(argc != 2 || !get_zlib_data(env, argv[0], &d)
-                 || !enif_get_int(env, argv[1], &windowBits)) {
+    if(argc != 3 || !get_zlib_data(env, argv[0], &d)
+                 || !enif_get_int(env, argv[1], &windowBits)
+                 || !enif_get_int(env, argv[2], &eosBehavior)) {
         return enif_make_badarg(env);
     } else if(!zlib_process_check(env, d)) {
         return enif_raise_exception(env, am_not_on_controlling_process);
@@ -785,6 +736,8 @@ static ERL_NIF_TERM zlib_inflateInit2(ErlNifEnv *env, int argc, const ERL_NIF_TE
 
     if(res == Z_OK) {
         d->state = ST_INFLATE;
+
+        d->eos_behavior = eosBehavior;
         d->eos_seen = 0;
 
         d->is_raw_stream = (windowBits < 0);
@@ -934,6 +887,28 @@ static ERL_NIF_TERM zlib_inflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar
         return enif_raise_exception(env, am_not_initialized);
     }
 
+    if(d->eos_seen) {
+        int res;
+
+        switch(d->eos_behavior) {
+        case EOS_BEHAVIOR_ERROR:
+            return zlib_return(env, Z_DATA_ERROR);
+        case EOS_BEHAVIOR_RESET:
+            res = inflateReset(&d->s);
+
+            if(res != Z_OK) {
+                return zlib_return(env, res);
+            }
+
+            d->eos_seen = 0;
+            break;
+        case EOS_BEHAVIOR_CUT:
+            zlib_reset_input(d);
+
+            return enif_make_tuple2(env, am_finished, enif_make_list(env, 0));
+        }
+    }
+
     return zlib_codec(&inflate, env, d, input_chunk_size, output_chunk_size, flush);
 }
 
-- 
cgit v1.2.3


From 9a50a5d5fc1c74d91efad74e1cb3e78f38e2b75c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sat, 30 Sep 2017 07:48:34 +0200
Subject: Point out the correct line number in stack traces

Sometimes the line number in a stack trace could be wrong,
for example for this code:

    t() ->
        Res = id(x), %<== Wrong line number.
        Res + 1.

    id(I) -> I.

The line number pointed out in the stack trace would be the
line before the line where the exception occurred.

The reason is the way the increment instruction instruction
is implemented:

    OpCase(i_increment_rWtd):
    {
      increment_reg_val = r(0);
    }
    I -= 1;
    goto increment__execute;

    OpCase(i_increment_xWtd):
    {
      increment_reg_val = xb(I[1]);
    }
    goto increment__execute;

    increment__execute:
    /* Common code for increment */
    .
    .
    .

(The implementation in OTP 20 is similar, but hand-coded directly
in beam_emu.c instead of generated.)

The instruction i_increment_rWtd decrements the instruction pointer (I)
before jumping to the common code. That means that I points *before*
the 'increment' instruction. If there is a 'line' instruction directly
before the 'increment' instruction (as there is in this example), the
instruction pointer will point before that line. Thus the previous line
will be picked up instead.

To eliminate this bug, we must never decrement the instruction pointer.
Instead, we can increment the other (longer) instructions in the
same group of combined instructions:

    OpCase(i_increment_rWtd):
    {
      increment_reg_val = r(0);
    }
    goto increment__execute;

    OpCase(i_increment_xWtd):
    {
      increment_reg_val = xb(I[1]);
    }
    I += 1;
    goto increment__execute;

    increment__execute:
    /* Common code for increment */
    .
    .
    .

Also fix a bug that was only a potential bug when ddaed7774eb0a
introduced relative jumps, but is now a real bug. See the added
comment for SET_I_REL() in macros.tab.
---
 erts/emulator/beam/macros.tab          | 11 ++++++++++-
 erts/emulator/test/exception_SUITE.erl | 21 +++++++++++++++++++++
 erts/emulator/utils/beam_makeops       | 25 ++++++++++++++-----------
 3 files changed, 45 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab
index 6f9b78af6f..0d175a7ec6 100644
--- a/erts/emulator/beam/macros.tab
+++ b/erts/emulator/beam/macros.tab
@@ -28,9 +28,18 @@ REFRESH_GEN_DEST() {
     dst_ptr = REG_TARGET_PTR(dst);
 }
 
+// $Offset is relative to the start of the instruction (not to the
+// location of the failure label reference). Since combined
+// instructions may increment the instruction pointer (e.g. in
+// 'increment') for some of the instructions in the group, we actually
+// use a virtual start position common to all instructions in the
+// group. To calculate the correct virtual position, we will need to
+// add $IP_ADJUSTMENT to the offset. ($IP_ADJUSTMENT will usually be
+// zero, except in a few bit syntax instructions.)
+
 SET_I_REL(Offset) {
     ASSERT(VALID_INSTR(*(I + ($Offset))));
-    I += $Offset;
+    I += $Offset + $IP_ADJUSTMENT;
 }
 
 SET_CP_I_ABS(Target) {
diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl
index e473a10be7..0f27251fcb 100644
--- a/erts/emulator/test/exception_SUITE.erl
+++ b/erts/emulator/test/exception_SUITE.erl
@@ -662,6 +662,15 @@ line_numbers(Config) when is_list(Config) ->
               {?MODULE,line_numbers,1,_}|_]}} =
     (catch applied_bif_2()),
 
+    {'EXIT',{badarith,
+             [{?MODULE,increment1,1,[{file,"increment.erl"},{line,45}]},
+              {?MODULE,line_numbers,1,_}|_]}} =
+        (catch increment1(x)),
+    {'EXIT',{badarith,
+             [{?MODULE,increment2,1,[{file,"increment.erl"},{line,48}]},
+              {?MODULE,line_numbers,1,_}|_]}} =
+        (catch increment2(x)),
+
     ok.
 
 id(I) -> I.
@@ -762,3 +771,15 @@ applied_bif_2() ->				%Line 8
     R = process_info(self(), current_location),	%Line 9
     fail = R,					%Line 10
     ok.						%Line 11
+
+%% The increment instruction used to decrement the instruction
+%% pointer, which would cause the line number in a stack trace to
+%% be the previous line number.
+
+-file("increment.erl", 42).
+increment1(Arg) ->                              %Line 43
+    Res = id(Arg),                              %Line 44
+    Res + 1.                                    %Line 45
+increment2(Arg) ->                              %Line 46
+    _ = id(Arg),                                %Line 47
+    Arg + 1.                                    %Line 48
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index a9b2c8861c..e4c91f1685 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -1111,7 +1111,7 @@ sub combine_instruction_group {
     my %offsets;
     my @instrs;
     my %num_references;
-    my $group_size = 0;
+    my $group_size = 999;
 
     # Do basic error checking. Associate operands of instructions
     # with the correct micro instructions. Calculate offsets for micro
@@ -1140,7 +1140,7 @@ sub combine_instruction_group {
                 }
                 my $size = cg_combined_size($s, 1, @first);
                 $offsets{$s} = $offset
-                    unless defined $offsets{$s} and $offsets{$s} >= $offset;
+                    unless defined $offsets{$s} and $offsets{$s} < $offset;
                 $offset += $size - 1;
                 my $label = micro_label($s);
                 $num_references{$label} = 0;
@@ -1148,7 +1148,7 @@ sub combine_instruction_group {
                 $opcase = '';
             }
             $spec_op_info{$print_name}->{'size'} = $offset + 1;
-            $group_size = $offset if $group_size < $offset;
+            $group_size = $offset if $group_size >= $offset;
             push @instrs, [$specific_key,@new_subs];
         }
     }
@@ -1218,19 +1218,19 @@ sub combine_instruction_group {
 
             my $flags = '';
             my $transfer_to_next = '';
-            my $dec = 0;
+            my $inc = 0;
 
             unless ($i == $#slots) {
                 $flags = "-no_next";
                 my $next_offset = $label_to_offset{$next};
-                $dec = $next_offset - ($offset + $size);
-                $transfer_to_next = "I -= $dec;\n" if $dec;
+                $inc = ($offset + $size) - $next_offset;
+                $transfer_to_next = "I += $inc;\n" if $inc;
                 $transfer_to_next .= "goto $next;\n\n";
             }
 
             my($gen_code,$down,$up) =
                 cg_combined_code($s, 1, $flags, $offset,
-                                 $group_size-$offset-$dec, @first);
+                                 $group_size-$offset, $inc, @first);
             my $spec_label = "$opcase$label";
             $down{$spec_label} = $down;
             $up{$spec_label} = $up;
@@ -1280,7 +1280,7 @@ sub micro_label {
 
 sub cg_basic {
     my($name,@args) = @_;
-    my($size,$code,$pack_spec) = code_gen($name, 1, '', 0, undef, @args);
+    my($size,$code,$pack_spec) = code_gen($name, 1, '', 0, undef, undef, @args);
     $pack_spec = build_pack_spec($pack_spec);
     ($size,$code,$pack_spec);
 }
@@ -1291,7 +1291,7 @@ sub cg_basic {
 
 sub cg_combined_size {
     my($name,$pack,@args) = @_;
-    my($size) = code_gen($name, $pack, '', 0, undef, @args);
+    my($size) = code_gen($name, $pack, '', 0, undef, undef, @args);
     $size;
 }
 
@@ -1300,6 +1300,7 @@ sub cg_combined_size {
 #
 
 sub cg_combined_code {
+    my($name,$pack,$extra_comments,$offset,$comp_size,$inc,@args) = @_;
     my($size,$code,$pack_spec) = code_gen(@_);
     if ($pack_spec eq '') {
         ($code,'','');
@@ -1310,7 +1311,8 @@ sub cg_combined_code {
 }
 
 sub code_gen {
-    my($name,$pack,$extra_comments,$offset,$group_size,@args) = @_;
+    my($name,$pack,$extra_comments,$offset,$comp_size,$inc,@args) = @_;
+    my $group_size = defined $comp_size ? $comp_size + $inc : undef;
     my $size = 0;
     my $flags = '';
     my @f;
@@ -1416,6 +1418,7 @@ sub code_gen {
         $bindings{$var} = $f[$i];
     }
     $bindings{'NEXT_INSTRUCTION'} = "I+" . ($group_size+$offset+1);
+    $bindings{'IP_ADJUSTMENT'} = defined $inc ? $inc : 0;
     $c_code = eval { expand_all($c_code, \%bindings) };
     unless (defined $c_code) {
         warn $@;
@@ -1539,7 +1542,7 @@ sub expand_macro {
     my %new_bindings;
 
     # Keep the special, pre-defined bindings.
-    foreach my $key (qw(NEXT_INSTRUCTION)) {
+    foreach my $key (qw(NEXT_INSTRUCTION IP_ADJUSTMENT)) {
         $new_bindings{$key} = $bindings{$key};
     }
 
-- 
cgit v1.2.3


From 3da0edc053709a8c9e3ea40e430befa027ef0bf0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sat, 23 Sep 2017 07:58:33 +0200
Subject: beam_SUITE: Strengthen test of packed registers

Test more instructions and use register numbers >= 512.
---
 erts/emulator/test/beam_SUITE.erl | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/beam_SUITE.erl b/erts/emulator/test/beam_SUITE.erl
index 6a54fa87e0..bdf8f6c34e 100644
--- a/erts/emulator/test/beam_SUITE.erl
+++ b/erts/emulator/test/beam_SUITE.erl
@@ -113,20 +113,41 @@ packed_registers(Config) when is_list(Config) ->
 		   VarName = list_to_atom("M"++integer_to_list(V)),
 		   merl:var(VarName)
 	       end || V <- Seq],
+    MoreNewVars = [begin
+                       VarName = list_to_atom("MM"++integer_to_list(V)),
+                       merl:var(VarName)
+                   end || V <- Seq],
+    TupleEls = [?Q("id(_@Value@)") || {_,Value} <- S0],
     S = [?Q("_@Var = id(_@Value@)") || {Var,Value} <- S0],
     Code = ?Q(["-module('@Mod@').\n"
 	       "-export([f/0]).\n"
 	       "f() ->\n"
+               "Tuple = id({_@TupleEls}),\n"
+               "{_@MoreNewVars} = Tuple,\n"
 	       "_@S,\n"
 	       "_ = id(0),\n"
 	       "L = [_@Vars],\n"
 	       "_ = id(1),\n"
 	       "[_@NewVars] = L,\n"		%Test get_list/3.
 	       "_ = id(2),\n"
-	       "id([_@Vars,_@NewVars]).\n"
+	       "id([_@Vars,_@NewVars,_@MoreNewVars]).\n"
 	       "id(I) -> I.\n"]),
     merl:compile_and_load(Code),
-    CombinedSeq = Seq ++ Seq,
+
+    %% Optionally print the generated code.
+    PrintCode = false,                          %Change to true to print code.
+
+    case PrintCode of
+        false ->
+            ok;
+        true ->
+            merl:print(Code),
+            erts_debug:df(Mod),
+            {ok,Dis} = file:read_file(atom_to_list(Mod)++".dis"),
+            io:put_chars(Dis)
+    end,
+
+    CombinedSeq = Seq ++ Seq ++ Seq,
     CombinedSeq = Mod:f(),
 
     %% Clean up.
-- 
cgit v1.2.3


From 13956599d609156bd2c054213d8a5ed0a1ae7087 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 18 Sep 2017 14:01:45 +0200
Subject: beam_emu: Make order of macros consistent

The inconsistent order has annoyed me for a long time.

While at it, also remove the unecessary definition of LabelAddr() if
NO_JUMP_TABLE is defined.
---
 erts/emulator/beam/beam_emu.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 81c4417b1e..c2aab32995 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -52,13 +52,12 @@
 #  define CountCase(OpCode) case op_count_##OpCode
 #  define OpCode(OpCode)    ((Uint*)op_##OpCode)
 #  define Goto(Rel) {Go = (int)(UWord)(Rel); goto emulator_loop;}
-#  define LabelAddr(Addr) &&##Addr
 #else
 #  define OpCase(OpCode)    lb_##OpCode
 #  define CountCase(OpCode) lb_count_##OpCode
+#  define OpCode(OpCode)  (&&lb_##OpCode)
 #  define Goto(Rel) goto *((void *)Rel)
 #  define LabelAddr(Label) &&Label
-#  define OpCode(OpCode)  (&&lb_##OpCode)
 #endif
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
-- 
cgit v1.2.3


From e64a26414428c2f9c10cd91991bbc9dd81f0d8ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 18 Sep 2017 14:03:24 +0200
Subject: Refactor macros for accessing Beam instructions

The BeamOp() macro in erl_vm.h is clumsy to use. All users
cast the return value to BeamInstr.

Define new macros that are easier to use. In the future,
we might want to pack an operand into the same word as
the pointer to the instruction, so we will define two macros.

BeamIsOpCode() is used to rewrite code like this:

  if (Instr == (BeamInstr) BeamOp(op_i_func_info_IaaI) {
    ...
  }

to:

  if (BeamIsOpCode(Instr, op_i_func_info_IaaI)) {
    ...
  }

BeamOpCodeAddr(op_apply_bif) is used when we need the address
for an instruction.

Also elimiminate the global variables em_* in beam_emu.c.
They are not really needed. Use the BeamOpCodeAddr() macro
instead.
---
 erts/emulator/beam/beam_bif_load.c    | 23 +++++++++---------
 erts/emulator/beam/beam_bp.c          | 44 +++++++++++++++++------------------
 erts/emulator/beam/beam_debug.c       |  4 ++--
 erts/emulator/beam/beam_emu.c         | 26 +++++++--------------
 erts/emulator/beam/beam_load.c        | 35 +++++++++++-----------------
 erts/emulator/beam/beam_load.h        |  5 ----
 erts/emulator/beam/bif.c              |  9 ++++---
 erts/emulator/beam/code_ix.h          |  4 ++--
 erts/emulator/beam/erl_bif_trace.c    | 20 ++++++++--------
 erts/emulator/beam/erl_nfunc_sched.h  |  4 ++--
 erts/emulator/beam/erl_nif.c          | 11 ++++-----
 erts/emulator/beam/erl_vm.h           | 10 +++++---
 erts/emulator/beam/export.c           |  8 ++++---
 erts/emulator/beam/export.h           |  2 +-
 erts/emulator/hipe/hipe_bif0.c        |  4 +---
 erts/emulator/hipe/hipe_bif1.c        | 23 +++++++++---------
 erts/emulator/hipe/hipe_mode_switch.c | 12 ++++------
 17 files changed, 111 insertions(+), 133 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index e48415ecc4..00822d1415 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -823,11 +823,11 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 		ep->beam[1] = 0;
 	    } else {
 		if (ep->addressv[code_ix] == ep->beam &&
-		    ep->beam[0] == (BeamInstr) em_apply_bif) {
+		    BeamIsOpCode(ep->beam[0], op_apply_bif)) {
 		    continue;
 		}
-		ep->addressv[code_ix] = ep->beam;
-		ep->beam[0] = (BeamInstr) em_call_error_handler;
+                ep->addressv[code_ix] = ep->beam;
+                ep->beam[0] = BeamOpCodeAddr(op_call_error_handler);
 	    }
 	}
 	modp->curr.code_hdr->on_load_function_ptr = NULL;
@@ -849,7 +849,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	    if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) {
 		continue;
 	    }
-	    if (ep->beam[0] == (BeamInstr) em_apply_bif) {
+	    if (BeamIsOpCode(ep->beam[0], op_apply_bif)) {
 		continue;
 	    }
 	    ep->beam[1] = 0;
@@ -1766,22 +1766,23 @@ delete_code(Module* modp)
 	Export *ep = export_list(i, code_ix);
         if (ep != NULL && (ep->info.mfa.module == module)) {
 	    if (ep->addressv[code_ix] == ep->beam) {
-		if (ep->beam[0] == (BeamInstr) em_apply_bif) {
+		if (BeamIsOpCode(ep->beam[0], op_apply_bif)) {
 		    continue;
 		}
-		else if (ep->beam[0] ==
-			 (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
+		else if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) {
 		    ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
 		    ASSERT(modp->curr.num_traced_exports > 0);
 		    DBG_TRACE_MFA_P(&ep->info.mfa,
 				  "export trace cleared, code_ix=%d", code_ix);
 		    erts_clear_export_break(modp, &ep->info);
 		}
-		else ASSERT(ep->beam[0] == (BeamInstr) em_call_error_handler
-			    || !erts_initialized);
-	    }
+		else {
+                    ASSERT(BeamIsOpCode(ep->beam[0], op_call_error_handler) ||
+                           !erts_initialized);
+                }
+            }
 	    ep->addressv[code_ix] = ep->beam;
-	    ep->beam[0] = (BeamInstr) em_call_error_handler;
+	    ep->beam[0] = BeamOpCodeAddr(op_call_error_handler);
 	    ep->beam[1] = 0;
 	    DBG_TRACE_MFA_P(&ep->info.mfa,
 			    "export invalidation, code_ix=%d", code_ix);
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 49ec59c989..c81380c14d 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -203,7 +203,7 @@ erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified)
 	for (fi = 0; fi < num_functions; fi++) {
 
 	    ci = code_hdr->functions[fi];
-	    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+	    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
 	    if (erts_is_function_native(ci)) {
 		continue;
 	    }
@@ -265,11 +265,11 @@ erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified)
 
 	pc = ep->beam;
 	if (ep->addressv[code_ix] == pc) {
-	    if ((*pc == (BeamInstr) em_apply_bif ||
-		 *pc == (BeamInstr) em_call_error_handler)) {
-		continue;
+	    if (BeamIsOpCode(*pc, op_apply_bif) ||
+                BeamIsOpCode(*pc, op_call_error_handler)) {
+                continue;
 	    }
-	    ASSERT(*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint));
+	    ASSERT(BeamIsOpCode(*pc, op_i_generic_breakpoint));
 	} else if (erts_is_function_native(erts_code_to_codeinfo(ep->addressv[code_ix]))) {
 	    continue;
 	}
@@ -366,8 +366,8 @@ consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local)
 	    }
 	    ASSERT(modp->curr.num_breakpoints >= 0);
 	    ASSERT(modp->curr.num_traced_exports >= 0);
-	    ASSERT(*erts_codeinfo_to_code(ci) !=
-                   (BeamInstr) BeamOp(op_i_generic_breakpoint));
+	    ASSERT(! BeamIsOpCode(*erts_codeinfo_to_code(ci),
+                                  op_i_generic_breakpoint));
 	}
 	ci->u.gen_bp = NULL;
 	Free(g);
@@ -415,7 +415,7 @@ erts_install_breakpoints(BpFunctions* f)
 {
     Uint i;
     Uint n = f->matched;
-    BeamInstr br = (BeamInstr) BeamOp(op_i_generic_breakpoint);
+    BeamInstr br = BeamOpCodeAddr(op_i_generic_breakpoint);
 
     for (i = 0; i < n; i++) {
 	ErtsCodeInfo* ci = f->matching[i].ci;
@@ -460,7 +460,7 @@ static void
 uninstall_breakpoint(ErtsCodeInfo *ci)
 {
     BeamInstr *pc = erts_codeinfo_to_code(ci);
-    if (*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
+    if (BeamIsOpCode(*pc, op_i_generic_breakpoint)) {
 	GenericBp* g = ci->u.gen_bp;
 	if (g->data[erts_active_bp_ix()].flags == 0) {
 	    /*
@@ -642,7 +642,7 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg)
     Uint bp_flags;
     ErtsBpIndex ix = erts_active_bp_ix();
 
-    ASSERT(info->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+    ASSERT(BeamIsOpCode(info->op, op_i_func_info_IaaI));
 
     g = info->u.gen_bp;
     bp = &g->data[ix];
@@ -695,9 +695,9 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg)
 	Eterm w;
 	erts_trace_time_call(c_p, info, bp->time);
 	w = (BeamInstr) *c_p->cp;
-	if (! (w == (BeamInstr) BeamOp(op_i_return_time_trace) ||
-	       w == (BeamInstr) BeamOp(op_return_trace) ||
-	       w == (BeamInstr) BeamOp(op_i_return_to_trace)) ) {
+	if (! (BeamIsOpCode(w, op_i_return_time_trace) ||
+	       BeamIsOpCode(w, op_return_trace) ||
+               BeamIsOpCode(w, op_i_return_to_trace)) ) {
 	    Eterm* E = c_p->stop;
 	    ASSERT(c_p->htop <= E && E <= c_p->hend);
 	    if (E - 2 < c_p->htop) {
@@ -717,7 +717,7 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg)
     }
 
     if (bp_flags & ERTS_BPF_DEBUG) {
-	return (BeamInstr) BeamOp(op_i_debug_breakpoint);
+	return BeamOpCodeAddr(op_i_debug_breakpoint);
     } else {
 	return g->orig_instr;
     }
@@ -946,12 +946,12 @@ do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg,
     Eterm* E = c_p->stop;
 
     w = *c_p->cp;
-    if (w == (BeamInstr) BeamOp(op_return_trace)) {
+    if (BeamIsOpCode(w, op_return_trace)) {
 	cpp = &E[2];
-    } else if (w == (BeamInstr) BeamOp(op_i_return_to_trace)) {
+    } else if (BeamIsOpCode(w, op_i_return_to_trace)) {
 	return_to_trace = 1;
 	cpp = &E[0];
-    } else if (w == (BeamInstr) BeamOp(op_i_return_time_trace)) {
+    } else if (BeamIsOpCode(w, op_i_return_time_trace)) {
 	cpp = &E[0];
     } else {
 	cpp = NULL;
@@ -959,12 +959,12 @@ do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg,
     if (cpp) {
 	for (;;) {
 	    BeamInstr w = *cp_val(*cpp);
-	    if (w == (BeamInstr) BeamOp(op_return_trace)) {
+	    if (BeamIsOpCode(w, op_return_trace)) {
 		cpp += 3;
-	    } else if (w == (BeamInstr) BeamOp(op_i_return_to_trace)) {
+	    } else if (BeamIsOpCode(w, op_i_return_to_trace)) {
 		return_to_trace = 1;
 		cpp += 1;
-	    } else if (w == (BeamInstr) BeamOp(op_i_return_time_trace)) {
+            } else if (BeamIsOpCode(w, op_i_return_time_trace)) {
 		cpp += 2;
 	    } else {
 		break;
@@ -1293,7 +1293,7 @@ erts_find_local_func(ErtsCodeMFA *mfa) {
     n = (BeamInstr) code_hdr->num_functions;
     for (i = 0; i < n; ++i) {
 	ci = code_hdr->functions[i];
-	ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == ci->op);
+	ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
 	ASSERT(mfa->module == ci->mfa.module || is_nil(ci->mfa.module));
 	if (mfa->function == ci->mfa.function &&
 	    mfa->arity == ci->mfa.arity) {
@@ -1708,7 +1708,7 @@ check_break(ErtsCodeInfo *ci, Uint break_flags)
 {
     GenericBp* g = ci->u.gen_bp;
 
-    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
     if (erts_is_function_native(ci)) {
 	return 0;
     }
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 7819e9907d..0f332da63f 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -199,7 +199,7 @@ void debug_dump_code(BeamInstr *I, int num)
 	erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr);
 	instr = (BeamInstr) code_ptr[0];
 	for (i = 0; i < NUM_SPECIFIC_OPS; i++) {
-	    if (instr == (BeamInstr) BeamOp(i) && opc[i].name[0] != '\0') {
+	    if (BeamIsOpCode(instr, i) && opc[i].name[0] != '\0') {
 		code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp,
 				     i, opc[i].sz-1, code_ptr+1) + 1;
 		break;
@@ -319,7 +319,7 @@ erts_debug_disassemble_1(BIF_ALIST_1)
     erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr);
     instr = (BeamInstr) code_ptr[0];
     for (i = 0; i < NUM_SPECIFIC_OPS; i++) {
-	if (instr == (BeamInstr) BeamOp(i) && opc[i].name[0] != '\0') {
+	if (BeamIsOpCode(instr, i) && opc[i].name[0] != '\0') {
 	    code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp,
 				 i, opc[i].sz-1, code_ptr+1) + 1;
 	    break;
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index c2aab32995..8d5cc76015 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -158,11 +158,6 @@ BeamInstr beam_apply[2];
 BeamInstr beam_exit[1];
 BeamInstr beam_continue_exit[1];
 
-BeamInstr* em_call_error_handler;
-BeamInstr* em_apply_bif;
-BeamInstr* em_call_nif;
-BeamInstr* em_call_bif_e;
-
 
 /* NOTE These should be the only variables containing trace instructions.
 **      Sometimes tests are form the instruction value, and sometimes
@@ -977,11 +972,6 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 #endif /* ERTS_OPCODE_COUNTER_SUPPORT */
 #endif /* NO_JUMP_TABLE */
      
-     em_call_error_handler = OpCode(call_error_handler);
-     em_apply_bif = OpCode(apply_bif);
-     em_call_nif = OpCode(call_nif);
-     em_call_bif_e = OpCode(call_bif_e);
-
      beam_apply[0]             = (BeamInstr) OpCode(i_apply);
      beam_apply[1]             = (BeamInstr) OpCode(normal_exit);
      beam_exit[0]              = (BeamInstr) OpCode(error_action_code);
@@ -1002,7 +992,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 	 ep->beam[0] = (BeamInstr) OpCode(apply_bif);
 	 ep->beam[1] = (BeamInstr) bif_table[i].f;
 	 /* XXX: set func info for bifs */
-	 ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI);
+	 ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI);
      }
 
      return;
@@ -1249,12 +1239,12 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 	ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
 
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	if (em_apply_bif == (BeamInstr *) *I) {
+	if (BeamIsOpCode(*I, op_apply_bif)) {
 	    exiting = erts_call_dirty_bif(esdp, c_p, I, reg);
 	}
 	else {
-	    ASSERT(em_call_nif == (BeamInstr *) *I);
-	    exiting = erts_call_dirty_nif(esdp, c_p, I, reg);
+	    ASSERT(BeamIsOpCode(*I, op_call_nif));
+            exiting = erts_call_dirty_nif(esdp, c_p, I, reg);
 	}
 
 	ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
@@ -2089,8 +2079,8 @@ apply_bif_error_adjustment(Process *p, Export *ep,
      * and apply_last_IP.
      */
     if (I
-	&& ep->beam[0] == (BeamInstr) em_apply_bif
-	&& (ep == bif_export[BIF_error_1]
+	&& BeamIsOpCode(ep->beam[0], op_apply_bif)
+        && (ep == bif_export[BIF_error_1]
 	    || ep == bif_export[BIF_error_2]
 	    || ep == bif_export[BIF_exit_1]
 	    || ep == bif_export[BIF_throw_1])) {
@@ -3196,8 +3186,8 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity)
     if ((ep = export_get(&e)) == NULL) {
 	return 0;
     }
-    return ep->addressv[erts_active_code_ix()] == ep->beam
-	&& (ep->beam[0] == (BeamInstr) em_apply_bif);
+    return ep->addressv[erts_active_code_ix()] == ep->beam &&
+	BeamIsOpCode(ep->beam[0], op_apply_bif);
 }
 
 
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 3f9dc2c1aa..1fb284a303 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -55,12 +55,6 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int);
 #define DEFINED   1
 #define EXPORTED  2
 
-#ifdef NO_JUMP_TABLE
-#  define BeamOpCode(Op) ((BeamInstr)(Op))
-#else
-#  define BeamOpCode(Op) ((BeamInstr)beam_ops[Op])
-#endif
-
 #if defined(WORDS_BIGENDIAN)
 # define NATIVE_ENDIAN(F)			\
   if ((F).val & BSF_NATIVE) {			\
@@ -849,10 +843,9 @@ erts_finish_loading(Binary* magic, Process* c_p,
 		continue;
 	    }
 	    if (ep->addressv[code_ix] == ep->beam) {
-		if (ep->beam[0] == (BeamInstr) em_apply_bif) {
+		if (BeamIsOpCode(ep->beam[0], op_apply_bif)) {
 		    continue;
-		} else if (ep->beam[0] ==
-			   (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
+		} else if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) {
 		    ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
 		    ASSERT(mod_tab_p->curr.num_traced_exports > 0);
 		    erts_clear_export_break(mod_tab_p, &ep->info);
@@ -1482,7 +1475,7 @@ load_import_table(LoaderState* stp)
 	 * the BIF function.
 	 */
 	if ((e = erts_active_export_entry(mod, func, arity)) != NULL) {
-	    if (e->beam[0] == (BeamInstr) em_apply_bif) {
+	    if (BeamIsOpCode(e->beam[0], op_apply_bif)) {
 		stp->import[i].bf = (BifFunction) e->beam[1];
 		if (func == am_load_nif && mod == am_erlang && arity == 2) {
 		    stp->may_load_nif = 1;
@@ -1576,7 +1569,7 @@ is_bif(Eterm mod, Eterm func, unsigned arity)
     if (e == NULL) {
 	return 0;
     }
-    if (e->beam[0] != (BeamInstr) em_apply_bif) {
+    if (! BeamIsOpCode(e->beam[0], op_apply_bif)) {
 	return 0;
     }
     if (mod == am_erlang && func == am_apply && arity == 3) {
@@ -2323,7 +2316,7 @@ load_code(LoaderState* stp)
 	    stp->specific_op = specific;
 	    CodeNeed(opc[stp->specific_op].sz+16); /* Extra margin for packing */
             last_instr_start = ci + opc[stp->specific_op].adjust;
-	    code[ci++] = BeamOpCode(stp->specific_op);
+	    code[ci++] = BeamOpCodeAddr(stp->specific_op);
 	}
 	
 	/*
@@ -5079,7 +5072,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
     while (index != 0) {
 	BeamInstr next = codev[index];
         BeamInstr* abs_addr;
-	codev[index] = BeamOpCode(op_catch_yf);
+	codev[index] = BeamOpCodeAddr(op_catch_yf);
         /* We must make the address of the label absolute again. */
         abs_addr = (BeamInstr *)codev + index + codev[index+2];
 	catches = beam_catches_cons(abs_addr, catches);
@@ -6033,9 +6026,9 @@ int
 erts_is_function_native(ErtsCodeInfo *ci)
 {
 #ifdef HIPE
-    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
-    return erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call)
-	|| erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure);
+    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
+    return BeamIsOpCode(erts_codeinfo_to_code(ci)[0], op_hipe_trap_call) ||
+	BeamIsOpCode(erts_codeinfo_to_code(ci)[0], op_hipe_trap_call_closure);
 #else
     return 0;
 #endif
@@ -6104,7 +6097,7 @@ exported_from_module(Process* p, /* Process whose heap to use. */
 	    Eterm tuple;
 	    
 	    if (ep->addressv[code_ix] == ep->beam &&
-		ep->beam[0] == (BeamInstr) em_call_error_handler) {
+		BeamIsOpCode(ep->beam[0], op_call_error_handler)) {
 		/* There is a call to the function, but it does not exist. */ 
 		continue;
 	    }
@@ -6375,7 +6368,7 @@ make_stub(ErtsCodeInfo* info, Eterm mod, Eterm func, Uint arity, Uint native, Be
 {
     DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", erts_codeinfo_to_code(info));
     ASSERT(WORDS_PER_FUNCTION == 6);
-    info->op = (BeamInstr) BeamOp(op_i_func_info_IaaI);
+    info->op = BeamOpCodeAddr(op_i_func_info_IaaI);
     info->u.ncallee = (void (*)(void)) native;
     info->mfa.module = mod;
     info->mfa.function = func;
@@ -6480,7 +6473,7 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci)
     for (i = 0, lp = stp->lambdas; i < n; i++, lp++) {
         ErlFunEntry* fe = stp->lambdas[i].fe;
 	if (lp->function == ci->mfa.function && lp->arity == ci->mfa.arity) {
-	    *erts_codeinfo_to_code(ci) = (Eterm) BeamOpCode(op_hipe_trap_call_closure);
+	    *erts_codeinfo_to_code(ci) = BeamOpCodeAddr(op_hipe_trap_call_closure);
             fe->address = erts_codeinfo_to_code(ci);
 	}
     }
@@ -6804,7 +6797,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
 	 * as the body until we know what kind of trap we should put there.
 	 */
 	code_hdr->functions[i] = (ErtsCodeInfo*)fp;
-	op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */
+	op = BeamOpCodeAddr(op_hipe_trap_call); /* Might be changed later. */
 	fp = make_stub((ErtsCodeInfo*)fp, hipe_stp->module, func, arity,
                        (Uint)native_address, op);
     }
@@ -6814,7 +6807,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
      */
 
     code_hdr->functions[i] = (ErtsCodeInfo*)fp;
-    *fp++ = (BeamInstr) BeamOp(op_int_code_end);
+    *fp++ = BeamOpCodeAddr(op_int_code_end);
 
     /*
      * Copy attributes and compilation information.
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index c4a90d3f3a..156c3c45e2 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -37,11 +37,6 @@ typedef struct gen_op_entry {
 
 extern const GenOpEntry gen_opc[];
 
-extern BeamInstr beam_debug_apply[];
-extern BeamInstr* em_call_error_handler;
-extern BeamInstr* em_apply_bif;
-extern BeamInstr* em_call_nif;
-
 struct ErtsLiteralArea_;
 
 /*
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 80f391e91e..ad555bb195 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -5031,7 +5031,7 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a,
     ep->info.mfa.module = m;
     ep->info.mfa.function = f;
     ep->info.mfa.arity = a;
-    ep->beam[0] = (BeamInstr) em_apply_bif;
+    ep->beam[0] = BeamOpCodeAddr(op_apply_bif);
     ep->beam[1] = (BeamInstr) bif;
 }
 
@@ -5096,7 +5096,7 @@ schedule(Process *c_p, Process *dirty_shadow_proc,
 {
     ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
     (void) erts_nif_export_schedule(c_p, dirty_shadow_proc,
-				    mfa, pc, (BeamInstr) em_apply_bif,
+				    mfa, pc, BeamOpCodeAddr(op_apply_bif),
 				    dfunc, ifunc,
 				    module, function,
 				    argc, argv);
@@ -5145,7 +5145,6 @@ static BIF_RETTYPE dirty_bif_exception(BIF_ALIST_2)
 }
 
 
-extern BeamInstr* em_call_bif_e;
 static BIF_RETTYPE call_bif(Process *c_p, Eterm *reg, BeamInstr *I);
 
 BIF_RETTYPE
@@ -5218,13 +5217,13 @@ erts_schedule_bif(Process *proc,
 	    mfa = &exp->info.mfa;
 	}
 #endif
-	else if (em_call_bif_e == (BeamInstr *) *i) {
+	else if (BeamIsOpCode(*i, op_call_bif_e)) {
 	    /* Pointer to bif export in i+1 */
 	    exp = (Export *) i[1];
 	    pc = i;
 	    mfa = &exp->info.mfa;
 	}
-	else if (em_apply_bif == (BeamInstr *) *i) {
+	else if (BeamIsOpCode(*i, op_apply_bif)) {
 	    /* Pointer to bif in i+1, and mfa in i-3 */	    
 	    pc = c_p->cp;
 	    mfa = erts_code_to_codemfa(i);
diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h
index 9e3280cd98..42976d2301 100644
--- a/erts/emulator/beam/code_ix.h
+++ b/erts/emulator/beam/code_ix.h
@@ -176,7 +176,7 @@ int erts_has_code_write_permission(void);
 ERTS_GLB_INLINE
 BeamInstr *erts_codeinfo_to_code(ErtsCodeInfo *ci)
 {
-    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op);
+    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI) || !ci->op);
     ASSERT_MFA(&ci->mfa);
     return (BeamInstr*)(ci + 1);
 }
@@ -185,7 +185,7 @@ ERTS_GLB_INLINE
 ErtsCodeInfo *erts_code_to_codeinfo(BeamInstr *I)
 {
     ErtsCodeInfo *ci = ((ErtsCodeInfo *)(((char *)(I)) - sizeof(ErtsCodeInfo)));
-    ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op);
+    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI) || !ci->op);
     ASSERT_MFA(&ci->mfa);
     return ci;
 }
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 3fe089a00e..22942b40c4 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -966,12 +966,12 @@ static int function_is_traced(Process *p,
     if ((ep = export_get(&e)) != NULL) {
 	pc = ep->beam;
 	if (ep->addressv[erts_active_code_ix()] == pc &&
-	    *pc != (BeamInstr) em_call_error_handler) {
+	    ! BeamIsOpCode(*pc, op_call_error_handler)) {
 
 	    int r = 0;
 
-	    ASSERT(*pc == (BeamInstr) em_apply_bif ||
-		   *pc == (BeamInstr) BeamOp(op_i_generic_breakpoint));
+	    ASSERT(BeamIsOpCode(*pc, op_apply_bif) ||
+		   BeamIsOpCode(*pc, op_i_generic_breakpoint));
 
 	    if (erts_is_trace_break(&ep->info, ms, 0)) {
 		return FUNC_TRACE_GLOBAL_TRACE;
@@ -1361,14 +1361,14 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
 	    if (ep->addressv[code_ix] != pc) {
 		fp[i].mod->curr.num_traced_exports++;
 #ifdef DEBUG
-		ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI);
+		ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI);
 #endif
-                ep->beam[0] = (BeamInstr) BeamOp(op_trace_jump_W);
+                ep->beam[0] = BeamOpCodeAddr(op_trace_jump_W);
 		ep->beam[1] = (BeamInstr) ep->addressv[code_ix];
 	    }
 	    erts_set_call_trace_bif(ci, match_prog_set, 0);
 	    if (ep->addressv[code_ix] != pc) {
-		ep->beam[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint);
+		ep->beam[0] = BeamOpCodeAddr(op_i_generic_breakpoint);
 	    }
 	} else if (!on && flags.breakpoint) {
 	    /* Turn off breakpoint tracing -- nothing to do here. */
@@ -1378,8 +1378,8 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
 	     * before turning on breakpoint tracing.
 	     */
 	    erts_clear_call_trace_bif(ci, 0);
-	    if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
-		ep->beam[0] = (BeamInstr) BeamOp(op_trace_jump_W);
+	    if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) {
+		ep->beam[0] = BeamOpCodeAddr(op_trace_jump_W);
 	    }
 	}
     }
@@ -1671,7 +1671,7 @@ uninstall_exp_breakpoints(BpFunctions* f)
 	if (ep->addressv[code_ix] != ep->beam) {
 	    continue;
 	}
-	ASSERT(ep->beam[0] == (BeamInstr) BeamOp(op_trace_jump_W));
+	ASSERT(BeamIsOpCode(ep->beam[0], op_trace_jump_W));
 	ep->addressv[code_ix] = (BeamInstr *) ep->beam[1];
     }
 }
@@ -1690,7 +1690,7 @@ clean_export_entries(BpFunctions* f)
 	if (ep->addressv[code_ix] == ep->beam) {
 	    continue;
 	}
-	if (ep->beam[0] == (BeamInstr) BeamOp(op_trace_jump_W)) {
+	if (BeamIsOpCode(ep->beam[0], op_trace_jump_W)) {
 	    ep->beam[0] = (BeamInstr) 0;
 	    ep->beam[1] = (BeamInstr) 0;
 	}
diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h
index 9be0e6f7c7..b8a4e4ebc3 100644
--- a/erts/emulator/beam/erl_nfunc_sched.h
+++ b/erts/emulator/beam/erl_nfunc_sched.h
@@ -211,8 +211,8 @@ erts_proc_shadow2real(Process *c_p)
 #define ERTS_NFUNC_SCHED_INTERNALS__
 
 #define ERTS_I_BEAM_OP_TO_NIF_EXPORT(I)					\
-    (ASSERT(BeamOp(op_apply_bif) == (BeamInstr *) (*(I))		\
-	    || BeamOp(op_call_nif) == (BeamInstr *) (*(I))),		\
+    (ASSERT(BeamIsOpCode(*(I), op_apply_bif) ||                         \
+            BeamIsOpCode(*(I), op_call_nif)),                           \
      ((NifExport *) (((char *) (I)) - offsetof(NifExport, exp.beam[0]))))
 
 
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 43441e0228..f7f12efe28 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -292,7 +292,7 @@ schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp,
     ep = erts_nif_export_schedule(c_p, dirty_shadow_proc,
 				  c_p->current,
 				  c_p->cp,
-				  (BeamInstr) em_call_nif,
+				  BeamOpCodeAddr(op_call_nif),
 				  direct_fp, indirect_fp,
 				  mod, func_name,
 				  argc, (const Eterm *) argv);
@@ -3613,7 +3613,7 @@ static ErtsCodeInfo** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsign
     int j;
     for (j = 0; j < n; ++j) {
 	ErtsCodeInfo* ci = mod_code->functions[j];
-	ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+	ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
 	if (f_atom == ci->mfa.function
 	    && arity == ci->mfa.arity) {
 	    return mod_code->functions+j;
@@ -3982,13 +3982,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
             code_ptr = erts_codeinfo_to_code(ci);
 
 	    if (ci->u.gen_bp == NULL) {
-		code_ptr[0] = (BeamInstr) BeamOp(op_call_nif);
+		code_ptr[0] = BeamOpCodeAddr(op_call_nif);
 	    }
 	    else { /* Function traced, patch the original instruction word */
 		GenericBp* g = ci->u.gen_bp;
-		ASSERT(code_ptr[0] ==
-		       (BeamInstr) BeamOp(op_i_generic_breakpoint));
-		g->orig_instr = (BeamInstr) BeamOp(op_call_nif);
+		ASSERT(BeamIsOpCode(code_ptr[0], op_i_generic_breakpoint));
+		g->orig_instr = BeamOpCodeAddr(op_call_nif);
 	    }
 	    if (f->flags) {
 		code_ptr[3] = (BeamInstr) f->fptr;
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 076767c7cd..0e32559f1b 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -202,11 +202,15 @@ extern int erts_pd_initial_size;/* Initial Process dictionary table size */
 
 #include "erl_term.h"
 
-#ifdef NO_JUMP_TABLE 
-#define BeamOp(Op) (Op)
+#if defined(NO_JUMP_TABLE)
+#  define BeamOpsAreInitialized() (1)
+#  define BeamOpCodeAddr(OpCode) ((BeamInstr)(OpCode))
 #else
 extern void** beam_ops;
-#define BeamOp(Op) beam_ops[(Op)]
+#  define BeamOpsAreInitialized() (beam_ops != 0)
+#  define BeamOpCodeAddr(OpCode) ((BeamInstr)beam_ops[(OpCode)])
 #endif
 
+#define BeamIsOpCode(InstrWord, OpCode) ((InstrWord) == BeamOpCodeAddr(OpCode))
+
 #endif	/* __ERL_VM_H__ */
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index c81503f722..946ffeffb8 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -48,7 +48,6 @@ static erts_atomic_t total_entries_bytes;
  */
 erts_mtx_t export_staging_lock;
 
-extern BeamInstr* em_call_error_handler;
 extern BeamInstr* em_call_traced_function;
 
 struct export_entry
@@ -130,7 +129,10 @@ export_alloc(struct export_entry* tmpl_e)
 	obj->info.mfa.module = tmpl->info.mfa.module;
 	obj->info.mfa.function = tmpl->info.mfa.function;
 	obj->info.mfa.arity = tmpl->info.mfa.arity;
-	obj->beam[0] = (BeamInstr) em_call_error_handler;
+        obj->beam[0] = 0;
+        if (BeamOpsAreInitialized()) {
+            obj->beam[0] = BeamOpCodeAddr(op_call_error_handler);
+        }
 	obj->beam[1] = 0;
 
 	for (ix=0; ix<ERTS_NUM_CODE_IX; ix++) {
@@ -267,7 +269,7 @@ erts_find_function(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix)
     ee = hash_get(&export_tables[code_ix].htable, init_template(&templ, m, f, a));
     if (ee == NULL ||
 	(ee->ep->addressv[code_ix] == ee->ep->beam &&
-	 ee->ep->beam[0] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) {
+	 ! BeamIsOpCode(ee->ep->beam[0], op_i_generic_breakpoint))) {
 	return NULL;
     }
     return ee->ep;
diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h
index be6cce07bf..194e514b12 100644
--- a/erts/emulator/beam/export.h
+++ b/erts/emulator/beam/export.h
@@ -73,7 +73,7 @@ extern erts_mtx_t export_staging_lock;
 #include "beam_load.h" /* For em_* extern declarations */ 
 #define ExportIsBuiltIn(EntryPtr) 			\
 (((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->beam) && \
- ((EntryPtr)->beam[0] == (BeamInstr) em_apply_bif))
+ (BeamIsOpCode((EntryPtr)->beam[0], op_apply_bif)))
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 05663648e9..380031bf13 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -53,8 +53,6 @@
 #include "hipe_literals.h"
 #endif
 
-#define BeamOpCode(Op)	((Uint)BeamOp(Op))
-
 
 int term_to_Sint32(Eterm term, Sint *sp)
 {
@@ -615,7 +613,7 @@ static ErtsCodeInfo* hipe_find_emu_address(Eterm mod, Eterm name, unsigned int a
     n = code_hdr->num_functions;
     for (i = 0; i < n; ++i) {
 	ErtsCodeInfo *ci = code_hdr->functions[i];
-	ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
+	ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
 	if (ci->mfa.function == name && ci->mfa.arity == arity)
 	    return ci;
     }
diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c
index 3d3df4fd48..73d07f0ce5 100644
--- a/erts/emulator/hipe/hipe_bif1.c
+++ b/erts/emulator/hipe/hipe_bif1.c
@@ -32,11 +32,10 @@
 #include "big.h"
 #include "error.h"
 #include "beam_load.h"
+#include "erl_vm.h"
 #include "hipe_bif0.h"
 #include "hipe_bif1.h"
 
-#define BeamOpCode(Op)	((Uint)BeamOp(Op))
-
 BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1)
 {
     ErtsCodeInfo *ci;
@@ -46,17 +45,17 @@ BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1)
     ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
     if (!ci)
 	BIF_ERROR(BIF_P, BADARG);
-    ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
+    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
     pc = erts_codeinfo_to_code(ci);
-    if (pc[0] == BeamOpCode(op_hipe_trap_call))
+    if (BeamIsOpCode(pc[0], op_hipe_trap_call))
 	BIF_ERROR(BIF_P, BADARG);
-    if (pc[0] == BeamOpCode(op_hipe_call_count))
+    if (BeamIsOpCode(pc[0], op_hipe_call_count))
 	BIF_RET(NIL);
     hcc = erts_alloc(ERTS_ALC_T_HIPE_SL, sizeof(*hcc));
     hcc->count = 0;
     hcc->opcode = pc[0];
     ci->u.hcc = hcc;
-    pc[0] = BeamOpCode(op_hipe_call_count);
+    pc[0] = BeamOpCodeAddr(op_hipe_call_count);
     BIF_RET(am_true);
 }
 
@@ -70,9 +69,9 @@ BIF_RETTYPE hipe_bifs_call_count_off_1(BIF_ALIST_1)
     ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
     if (!ci)
 	BIF_ERROR(BIF_P, BADARG);
-    ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
+    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
     pc = erts_codeinfo_to_code(ci);
-    if (pc[0] != BeamOpCode(op_hipe_call_count))
+    if (! BeamIsOpCode(pc[0], op_hipe_call_count))
 	BIF_RET(am_false);
     hcc = ci->u.hcc;
     count = hcc->count;
@@ -91,9 +90,9 @@ BIF_RETTYPE hipe_bifs_call_count_get_1(BIF_ALIST_1)
     ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
     if (!ci)
 	BIF_ERROR(BIF_P, BADARG);
-    ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
+    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
     pc = erts_codeinfo_to_code(ci);
-    if (pc[0] != BeamOpCode(op_hipe_call_count))
+    if (! BeamIsOpCode(pc[0], op_hipe_call_count))
 	BIF_RET(am_false);
     hcc = ci->u.hcc;
     BIF_RET(make_small(hcc->count));
@@ -109,9 +108,9 @@ BIF_RETTYPE hipe_bifs_call_count_clear_1(BIF_ALIST_1)
     ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
     if (!ci)
 	BIF_ERROR(BIF_P, BADARG);
-    ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
+    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
     pc = erts_codeinfo_to_code(ci);
-    if (pc[0] != BeamOpCode(op_hipe_call_count))
+    if (! BeamIsOpCode(pc[0], op_hipe_call_count))
 	BIF_RET(am_false);
     hcc = ci->u.hcc;
     count = hcc->count;
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index b7f81fc4a6..8b497c9970 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -155,8 +155,6 @@ void hipe_check_pcb(Process *p, const char *file, unsigned line)
 #include "hipe_arm_glue.h"
 #endif
 
-#define BeamOpCode(Op)		((Uint)BeamOp(Op))
-
 Uint hipe_beam_pc_return[1];	/* needed in hipe_debug.c */
 Uint hipe_beam_pc_throw[1];	/* needed in hipe_debug.c */
 Uint hipe_beam_pc_resume[1];	/* needed by hipe_set_timeout() */
@@ -166,9 +164,9 @@ void hipe_mode_switch_init(void)
 {
     hipe_arch_glue_init();
 
-    hipe_beam_pc_return[0] = BeamOpCode(op_hipe_trap_return);
-    hipe_beam_pc_throw[0] = BeamOpCode(op_hipe_trap_throw);
-    hipe_beam_pc_resume[0] = BeamOpCode(op_hipe_trap_resume);
+    hipe_beam_pc_return[0] = BeamOpCodeAddr(op_hipe_trap_return);
+    hipe_beam_pc_throw[0] = BeamOpCodeAddr(op_hipe_trap_throw);
+    hipe_beam_pc_resume[0] = BeamOpCodeAddr(op_hipe_trap_resume);
 
     hipe_beam_catch_throw =
 	make_catch(beam_catches_cons(hipe_beam_pc_throw, BEAM_CATCHES_NIL));
@@ -182,8 +180,8 @@ void hipe_set_call_trap(ErtsCodeInfo* ci, void *nfun, int is_closure)
     HIPE_ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI));
     bfun[0] =
 	is_closure
-	? BeamOpCode(op_hipe_trap_call_closure)
-	: BeamOpCode(op_hipe_trap_call);
+	? BeamOpCodeAddr(op_hipe_trap_call_closure)
+	: BeamOpCodeAddr(op_hipe_trap_call);
     ci->u.ncallee = (void (*)(void)) nfun;
 }
 
-- 
cgit v1.2.3


From 4e94a3dfc5cade0f8334efa1c785f44eaa55fdac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 19 Sep 2017 07:15:03 +0200
Subject: Eliminate unnecessary and inconsistent casts

Consider the types in the code below:

    BeamInstr* I;
      .
      .
      .
    BeamInstr* next;
    next = (BeamInstr *) *I;
    Goto(next);

This is illogical. If 'I' points to a BeamInstr, then 'next' should
be a BeamInstr, not a pointer to a BeamInstr. The Goto() macros does
not require a pointer, because it will cast its argument to a void*
anyway.

Therefore, this code example can be simplified to:

    BeamInstr* I;
      .
      .
      .
    BeamInstr next;
    next = *I;
    Goto(next);

Similarly, we can remove the casts in the macros when NO_JUMP_TABLE
is defined.
---
 erts/emulator/beam/beam_emu.c    | 58 ++++++++++++++++++++--------------------
 erts/emulator/utils/beam_makeops |  7 +++--
 2 files changed, 32 insertions(+), 33 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8d5cc76015..245091c887 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -50,8 +50,8 @@
 #if defined(NO_JUMP_TABLE)
 #  define OpCase(OpCode)    case op_##OpCode
 #  define CountCase(OpCode) case op_count_##OpCode
-#  define OpCode(OpCode)    ((Uint*)op_##OpCode)
-#  define Goto(Rel) {Go = (int)(UWord)(Rel); goto emulator_loop;}
+#  define OpCode(OpCode)    (op_##OpCode)
+#  define Goto(Rel) {Go = (Rel); goto emulator_loop;}
 #else
 #  define OpCase(OpCode)    lb_##OpCode
 #  define CountCase(OpCode) lb_count_##OpCode
@@ -249,8 +249,8 @@ void** beam_ops;
 
 #define DispatchMacro()				\
   do {						\
-     BeamInstr* dis_next;				\
-     dis_next = (BeamInstr *) *I;			\
+     BeamInstr dis_next;                        \
+     dis_next = *I;                             \
      CHECK_ARGS(I);				\
      if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
         FCALLS--;				\
@@ -258,12 +258,12 @@ void** beam_ops;
      } else {					\
 	goto context_switch;			\
      }						\
- } while (0)
+ } while (0)                                    \
 
 #define DispatchMacroFun()			\
   do {						\
-     BeamInstr* dis_next;				\
-     dis_next = (BeamInstr *) *I;			\
+     BeamInstr dis_next;                        \
+     dis_next = *I;                             \
      CHECK_ARGS(I);				\
      if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
         FCALLS--;				\
@@ -273,23 +273,23 @@ void** beam_ops;
      }						\
  } while (0)
 
-#define DispatchMacrox()					\
-  do {								\
-     if (FCALLS > 0) {						\
-        Eterm* dis_next;					\
-        SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \
-        dis_next = (Eterm *) *I;				\
-        FCALLS--;						\
-        CHECK_ARGS(I);						\
-        Goto(dis_next);						\
-     } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)		\
-		&& FCALLS > neg_o_reds) {			\
-        goto save_calls1;					\
-     } else {							\
-        SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \
-        CHECK_ARGS(I);						\
-	goto context_switch;					\
-     }								\
+#define DispatchMacrox()                                                \
+  do {                                                                  \
+     if (FCALLS > 0) {                                                  \
+        BeamInstr dis_next;                                             \
+        SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]);    \
+        dis_next = *I;                                                  \
+        FCALLS--;                                                       \
+        CHECK_ARGS(I);                                                  \
+        Goto(dis_next);                                                 \
+     } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)                      \
+		&& FCALLS > neg_o_reds) {                               \
+        goto save_calls1;                                               \
+     } else {                                                           \
+        SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]);    \
+        CHECK_ARGS(I);                                                  \
+	goto context_switch;                                            \
+     }                                                                  \
  } while (0)
 
 #ifdef DEBUG
@@ -621,7 +621,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 #ifndef NO_JUMP_TABLE
     static void* opcodes[] = { DEFINE_OPCODES };
 #else
-    int Go;
+    register BeamInstr Go;
 #endif
 #endif
 
@@ -697,7 +697,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
     {
 	int reds;
 	Eterm* argp;
-	BeamInstr *next;
+	BeamInstr next;
 	int i;
 
 	argp = c_p->arg_reg;
@@ -729,7 +729,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 
 	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 
-	next = (BeamInstr *) *I;
+	next = *I;
 	SWAPIN;
 	ASSERT(VALID_INSTR(next));
 
@@ -1006,13 +1006,13 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 
   save_calls1:
     {
-	Eterm* dis_next;
+	BeamInstr dis_next;
 
 	save_calls(c_p, (Export *) Arg(0));
 
 	SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]);
 
-	dis_next = (Eterm *) *I;
+	dis_next = *I;
 	FCALLS--;
 	Goto(dis_next);
     }
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index a9b2c8861c..083e7eb523 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -1438,11 +1438,10 @@ sub code_gen {
             "ASSERT(VALID_INSTR(*I));\n" .
             "Goto(*I);";
     } else {
-        $var_decls .= "BeamInstr* _nextpf = " .
-            "(BeamInstr *) I[$instr_offset];\n";
+        $var_decls .= "BeamInstr next_pf = I[$instr_offset];\n";
         $dispatch_next = "\nI += $instr_offset;\n" .
-            "ASSERT(VALID_INSTR(_nextpf));\n" .
-            "Goto(_nextpf);";
+            "ASSERT(VALID_INSTR(next_pf));\n" .
+            "Goto(next_pf);";
     }
 
     #
-- 
cgit v1.2.3


From 53e6fb04c52703a222884b709e86d1fd8bcf98d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 19 Sep 2017 08:07:48 +0200
Subject: Eliminate the OpCode() macro

Introduce the IsOpCode() macro that can be used to compare
instructions.
---
 erts/emulator/beam/arith_instrs.tab |  4 ++--
 erts/emulator/beam/beam_emu.c       | 24 ++++++++++++------------
 erts/emulator/beam/trace_instrs.tab |  4 ++--
 erts/emulator/hipe/hipe_instrs.tab  |  6 +++---
 4 files changed, 19 insertions(+), 19 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab
index 3db19df3c4..23fdb6d5b5 100644
--- a/erts/emulator/beam/arith_instrs.tab
+++ b/erts/emulator/beam/arith_instrs.tab
@@ -366,10 +366,10 @@ shift.execute(Fail, Live, Dst) {
         reg[0] = Op1;
         reg[1] = Op2;
         SWAPOUT;
-        if (I[0] == (BeamInstr) OpCode(i_bsl_ssjtd)) {
+        if (IsOpCode(I[0], i_bsl_ssjtd)) {
             I = handle_error(c_p, I, reg, &bif_export[BIF_bsl_2]->info.mfa);
         } else {
-            ASSERT(I[0] == (BeamInstr) OpCode(i_bsr_ssjtd));
+            ASSERT(IsOpCode(I[0], i_bsr_ssjtd));
             I = handle_error(c_p, I, reg, &bif_export[BIF_bsr_2]->info.mfa);
         }
         goto post_error_handling;
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 245091c887..009be37d40 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -50,12 +50,12 @@
 #if defined(NO_JUMP_TABLE)
 #  define OpCase(OpCode)    case op_##OpCode
 #  define CountCase(OpCode) case op_count_##OpCode
-#  define OpCode(OpCode)    (op_##OpCode)
+#  define IsOpCode(InstrWord, OpCode)  ((InstrWord) == (BeamInstr)op_##OpCode)
 #  define Goto(Rel) {Go = (Rel); goto emulator_loop;}
 #else
 #  define OpCase(OpCode)    lb_##OpCode
 #  define CountCase(OpCode) lb_count_##OpCode
-#  define OpCode(OpCode)  (&&lb_##OpCode)
+#  define IsOpCode(InstrWord, OpCode)  ((InstrWord) == (BeamInstr)&&lb_##OpCode)
 #  define Goto(Rel) goto *((void *)Rel)
 #  define LabelAddr(Label) &&Label
 #endif
@@ -971,15 +971,15 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
      beam_ops = opcodes;
 #endif /* ERTS_OPCODE_COUNTER_SUPPORT */
 #endif /* NO_JUMP_TABLE */
-     
-     beam_apply[0]             = (BeamInstr) OpCode(i_apply);
-     beam_apply[1]             = (BeamInstr) OpCode(normal_exit);
-     beam_exit[0]              = (BeamInstr) OpCode(error_action_code);
-     beam_continue_exit[0]     = (BeamInstr) OpCode(continue_exit);
-     beam_return_to_trace[0]   = (BeamInstr) OpCode(i_return_to_trace);
-     beam_return_trace[0]      = (BeamInstr) OpCode(return_trace);
-     beam_exception_trace[0]   = (BeamInstr) OpCode(return_trace); /* UGLY */
-     beam_return_time_trace[0] = (BeamInstr) OpCode(i_return_time_trace);
+
+     beam_apply[0]             = BeamOpCodeAddr(op_i_apply);
+     beam_apply[1]             = BeamOpCodeAddr(op_normal_exit);
+     beam_exit[0]              = BeamOpCodeAddr(op_error_action_code);
+     beam_continue_exit[0]     = BeamOpCodeAddr(op_continue_exit);
+     beam_return_to_trace[0]   = BeamOpCodeAddr(op_i_return_to_trace);
+     beam_return_trace[0]      = BeamOpCodeAddr(op_return_trace);
+     beam_exception_trace[0]   = BeamOpCodeAddr(op_return_trace); /* UGLY */
+     beam_return_time_trace[0] = BeamOpCodeAddr(op_i_return_time_trace);
 
      /*
       * Enter all BIFs into the export table.
@@ -989,7 +989,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 			      bif_table[i].name,
 			      bif_table[i].arity);
 	 bif_export[i] = ep;
-	 ep->beam[0] = (BeamInstr) OpCode(apply_bif);
+	 ep->beam[0] = BeamOpCodeAddr(op_apply_bif);
 	 ep->beam[1] = (BeamInstr) bif_table[i].f;
 	 /* XXX: set func info for bifs */
 	 ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI);
diff --git a/erts/emulator/beam/trace_instrs.tab b/erts/emulator/beam/trace_instrs.tab
index b10442c5e7..3eee81c053 100644
--- a/erts/emulator/beam/trace_instrs.tab
+++ b/erts/emulator/beam/trace_instrs.tab
@@ -61,12 +61,12 @@ i_return_to_trace() {
         Uint *cpp = (Uint*) E;
         for(;;) {
             ASSERT(is_CP(*cpp));
-            if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) {
+            if (IsOpCode(*cp_val(*cpp), return_trace)) {
                 do
                     ++cpp;
                 while (is_not_CP(*cpp));
                 cpp += 2;
-            } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) {
+            } else if (IsOpCode(*cp_val(*cpp), i_return_to_trace)) {
                 do
                     ++cpp;
                 while (is_not_CP(*cpp));
diff --git a/erts/emulator/hipe/hipe_instrs.tab b/erts/emulator/hipe/hipe_instrs.tab
index bcce196a1d..a01baebddf 100644
--- a/erts/emulator/hipe/hipe_instrs.tab
+++ b/erts/emulator/hipe/hipe_instrs.tab
@@ -45,7 +45,7 @@ hipe_trap.call() {
      * ... remainder of original BEAM code
      */
     ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
-    ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
+    ASSERT(IsOpCode(ci->op, i_func_info_IaaI));
     c_p->hipe.u.ncallee = ci->u.ncallee;
     ++hipe_trap_count;
     $HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8));
@@ -53,7 +53,7 @@ hipe_trap.call() {
 
 hipe_trap.call_closure() {
     ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
-    ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
+    ASSERT(IsOpCode(ci->op, i_func_info_IaaI));
     c_p->hipe.u.ncallee = ci->u.ncallee;
     ++hipe_trap_count;
     $HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8));
@@ -132,7 +132,7 @@ hipe_call_count() {
      */
     ErtsCodeInfo *ci = erts_code_to_codeinfo(I);
     struct hipe_call_count *hcc = ci->u.hcc;
-    ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI));
+    ASSERT(IsOpCode(ci->op, i_func_info_IaaI));
     ASSERT(hcc != NULL);
     ASSERT(VALID_INSTR(hcc->opcode));
     ++(hcc->count);
-- 
cgit v1.2.3


From 27c7c501b52f27eb252dc6b7633223bad7a732c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 20 Sep 2017 12:50:10 +0200
Subject: Break out most of the initialization from process_main()

process_main() is already too big.
---
 erts/emulator/beam/beam_emu.c | 60 +++++++++++++++++++++++++------------------
 1 file changed, 35 insertions(+), 25 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 009be37d40..b412cf8b79 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -376,6 +376,7 @@ do {                                            \
  * The following functions are called directly by process_main().
  * Don't inline them.
  */
+static void init_emulator_finish(void) NOINLINE;
 static ErtsCodeMFA *ubif2mfa(void* uf) NOINLINE;
 static ErtsCodeMFA *gcbif2mfa(void* gcf) NOINLINE;
 static BeamInstr* handle_error(Process* c_p, BeamInstr* pc,
@@ -957,9 +958,6 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 
  init_emulator:
  {
-     int i;
-     Export* ep;
-
 #ifndef NO_JUMP_TABLE
 #ifdef ERTS_OPCODE_COUNTER_SUPPORT
 #ifdef DEBUG
@@ -972,6 +970,40 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 #endif /* ERTS_OPCODE_COUNTER_SUPPORT */
 #endif /* NO_JUMP_TABLE */
 
+     init_emulator_finish();
+     return;
+ }
+#ifdef NO_JUMP_TABLE
+ default:
+    erts_exit(ERTS_ERROR_EXIT, "unexpected op code %d\n",Go);
+  }
+#endif
+    return;			/* Never executed */
+
+  save_calls1:
+    {
+	BeamInstr dis_next;
+
+	save_calls(c_p, (Export *) Arg(0));
+
+	SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]);
+
+	dis_next = *I;
+	FCALLS--;
+	Goto(dis_next);
+    }
+}
+
+/*
+ * One-time initialization of emulator. Does not need to be
+ * in process_main().
+ */
+static void
+init_emulator_finish(void)
+{
+     int i;
+     Export* ep;
+
      beam_apply[0]             = BeamOpCodeAddr(op_i_apply);
      beam_apply[1]             = BeamOpCodeAddr(op_normal_exit);
      beam_exit[0]              = BeamOpCodeAddr(op_error_action_code);
@@ -994,28 +1026,6 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 	 /* XXX: set func info for bifs */
 	 ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI);
      }
-
-     return;
- }
-#ifdef NO_JUMP_TABLE
- default:
-    erts_exit(ERTS_ERROR_EXIT, "unexpected op code %d\n",Go);
-  }
-#endif
-    return;			/* Never executed */
-
-  save_calls1:
-    {
-	BeamInstr dis_next;
-
-	save_calls(c_p, (Export *) Arg(0));
-
-	SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]);
-
-	dis_next = *I;
-	FCALLS--;
-	Goto(dis_next);
-    }
 }
 
 /*
-- 
cgit v1.2.3


From e10dd6abd48a6f93b10b37673f31075a2264727a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 19 Sep 2017 15:15:39 +0200
Subject: beam_load.c: Generalize the 'P' operator in the packing engine

In the 'P' operator, don't assume that a packed target label ('f'
or 'j') is always the leftmost argument. Instead, transfer the
patch position from the accumulator to the stack.
---
 erts/emulator/beam/beam_load.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 1fb284a303..9835b1c096 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2636,15 +2636,15 @@ load_code(LoaderState* stp)
                     }
                     ci++;
 		    break;
-		case 'P':	/* Put packed operands. */
+		case 'P':	/* Put packed operands (on the stack). */
                     sp->instr = packed;
                     sp->patch_pos = 0;
-                    sp++;
-		    packed = 0;
                     if (packed_label) {
-                        packed_label->pos = ci;
+                        sp->patch_pos = &packed_label->pos;
                         packed_label = 0;
                     }
+                    sp++;
+		    packed = 0;
 		    break;
 		default:
 		    ASSERT(0);
-- 
cgit v1.2.3


From 72f0f6d16d1aa06205aa2efba9a4b17ea1f8d849 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 20 Sep 2017 05:15:22 +0200
Subject: instrs.tab: Add missing -no_next directives

---
 erts/emulator/beam/instrs.tab | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index 7ea9dee299..b85e3eaeb5 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -831,12 +831,14 @@ badmatch(Src) {
     c_p->fvalue = $Src;
     c_p->freason = BADMATCH;
     goto find_func_info;
+    //| -no_next;
 }
 
 case_end(Src) {
     c_p->fvalue = $Src;
     c_p->freason = EXC_CASE_CLAUSE;
     goto find_func_info;
+    //| -no_next;
 }
 
 if_end() {
-- 
cgit v1.2.3


From 9de3ee84b934ea373a4ce7d9dbb5b59a15cc2094 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sat, 23 Sep 2017 06:43:06 +0200
Subject: Add CHECK_ALIGNED() for testing storage destinations

Add the CHECK_ALIGNED() macro that can be used for testing that
the storage destination is word-aligned.
---
 erts/emulator/beam/beam_emu.c | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index b412cf8b79..9a77c63390 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -106,6 +106,8 @@ do {                                     \
 #  define CHECK_ARGS(T)
 #endif
 
+#define CHECK_ALIGNED(Dst) ASSERT((((Uint)&Dst) & (sizeof(Uint)-1)) == 0)
+
 #define GET_BIF_MODULE(p)  (p->info.mfa.module)
 #define GET_BIF_FUNCTION(p)  (p->info.mfa.function)
 #define GET_BIF_ARITY(p)  (p->info.mfa.arity)
-- 
cgit v1.2.3


From b597ba86170ee142790e97ae298cf8b7c2797ce2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sat, 23 Sep 2017 05:32:37 +0200
Subject: Change operand from 'P' to 'Q' for i_apply_last and i_apply_fun_last

All other instructions that increment the stack pointer takes a 'Q'
operand.
---
 erts/emulator/beam/ops.tab | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 87ff92d354..75ff40606b 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -961,11 +961,11 @@ call_ext_last Ar Func D => i_call_ext_last Func D
 call_ext_only Ar Func   => i_call_ext_only Func
 
 i_apply
-i_apply_last P
+i_apply_last Q
 i_apply_only
 
 i_apply_fun
-i_apply_fun_last P
+i_apply_fun_last Q
 i_apply_fun_only
 
 %cold
-- 
cgit v1.2.3


From f589687e9bd172f6f7841043a8de05d260889c8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sun, 24 Sep 2017 07:53:18 +0200
Subject: Move out variables from the head of combined instructions

Move out from the head the variables that are only used in the excute
phase.
---
 erts/emulator/beam/arith_instrs.tab |  8 ++++----
 erts/emulator/beam/bs_instrs.tab    | 28 +++++++++++++---------------
 erts/emulator/beam/instrs.tab       |  3 +--
 3 files changed, 18 insertions(+), 21 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab
index 23fdb6d5b5..b828e86788 100644
--- a/erts/emulator/beam/arith_instrs.tab
+++ b/erts/emulator/beam/arith_instrs.tab
@@ -85,9 +85,6 @@ i_increment := increment.fetch.execute;
 
 increment.head() {
     Eterm increment_reg_val;
-    Eterm increment_val;
-    Uint live;
-    Eterm result;
 }
 
 increment.fetch(Src) {
@@ -95,7 +92,10 @@ increment.fetch(Src) {
 }
 
 increment.execute(IncrementVal, Live, Dst) {
-    increment_val = $IncrementVal;
+    Eterm increment_val = $IncrementVal;
+    Uint live;
+    Eterm result;
+
     if (ERTS_LIKELY(is_small(increment_reg_val))) {
         Sint i = signed_val(increment_reg_val) + increment_val;
         if (ERTS_LIKELY(IS_SSMALL(i))) {
diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab
index a4d4afe7d4..9f03b19731 100644
--- a/erts/emulator/beam/bs_instrs.tab
+++ b/erts/emulator/beam/bs_instrs.tab
@@ -364,11 +364,9 @@ i_bs_init_bits_fail      := bs_init_bits.fail.verify.execute;
 i_bs_init_bits_fail_heap := bs_init_bits.fail_heap.verify.execute;
 
 bs_init_bits.head() {
-    Eterm new_binary;
     Eterm num_bits_term;
     Uint num_bits;
     Uint alloc;
-    Uint num_bytes;
 }
 
 bs_init_bits.plain(NumBits) {
@@ -410,7 +408,9 @@ bs_init_bits.verify(Fail) {
 }
 
 bs_init_bits.execute(Live, Dst) {
-     num_bytes = ((Uint64)num_bits+(Uint64)7) >> 3;
+     Eterm new_binary;
+     Uint num_bytes = ((Uint64)num_bits+(Uint64)7) >> 3;
+
      if (num_bits & 7) {
 	 alloc += ERL_SUB_BIN_SIZE;
      }
@@ -709,9 +709,6 @@ i_bs_validate_unicode_retract(Fail, Src, Ms) {
 i_bs_start_match2 := bs_start_match.fetch.execute;
 
 bs_start_match.head() {
-    Uint slots;
-    Uint live;
-    Eterm header;
     Eterm context;
 }
 
@@ -720,6 +717,9 @@ bs_start_match.fetch(Src) {
 }
 
 bs_start_match.execute(Fail, Live, Slots, Dst) {
+    Uint slots;
+    Uint live;
+    Eterm header;
     if (!is_boxed(context)) {
         $FAIL($Fail);
     }
@@ -942,13 +942,10 @@ bs_context_to_binary := ctx_to_bin.fetch.execute;
 i_bs_get_binary_all_reuse := ctx_to_bin.fetch_bin.execute;
 
 ctx_to_bin.head() {
-     Eterm context;
-     ErlBinMatchBuffer* mb;
-     ErlSubBin* sb;
-     Uint size;
-     Uint offs;
-     Uint orig;
-     Uint hole_size;
+    Eterm context;
+    ErlBinMatchBuffer* mb;
+    Uint size;
+    Uint offs;
 }
 
 ctx_to_bin.fetch(Src) {
@@ -976,8 +973,9 @@ ctx_to_bin.fetch_bin(Src, Fail, Unit) {
 }
 
 ctx_to_bin.execute() {
-    orig = mb->orig;
-    sb = (ErlSubBin *) boxed_val(context);
+    Uint hole_size;
+    Uint orig = mb->orig;
+    ErlSubBin* sb = (ErlSubBin *) boxed_val(context);
     hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE;
     sb->thing_word = HEADER_SUB_BIN;
     sb->size = BYTE_OFFSET(size);
diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index b85e3eaeb5..07cc4bd527 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -375,7 +375,6 @@ i_element := element_group.fetch.execute;
 
 
 element_group.head() {
-    Eterm element_index;
     Eterm element_tuple;
 }
 
@@ -384,7 +383,7 @@ element_group.fetch(Src) {
 }
 
 element_group.execute(Fail, Index, Dst) {
-    element_index = $Index;
+    Eterm element_index = $Index;
     if (ERTS_LIKELY(is_small(element_index) && is_tuple(element_tuple))) {
         Eterm* tp = tuple_val(element_tuple);
 
-- 
cgit v1.2.3


From 300925cc0de95650cf5b13e938ccf702a8eadc9e Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 4 May 2017 17:07:41 +0200
Subject: erts: temp_alloc can no longer be disabled

temp_alloc is used in such a way that if it ever results
in a malloc/free sequence it will slow down the system
alot. So it will no longer be possible to disable it and
it will not be disabled when using +Mea min.

OTP-14651
---
 erts/emulator/beam/erl_alloc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 1c365f906b..88285d8be6 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -385,6 +385,7 @@ set_default_temp_alloc_opts(struct au_init *ip)
     SET_DEFAULT_ALLOC_OPTS(ip);
     ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
     ip->thr_spec		= 1;
+    ip->disable_allowed         = 0;
     ip->carrier_migration_allowed = 0;
     ip->atype			= AFIT;
     ip->init.util.name_prefix	= "temp_";
@@ -1492,8 +1493,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 	&init->ll_alloc,
 	&init->driver_alloc,
 	&init->fix_alloc,
-	&init->sl_alloc,
-	&init->temp_alloc
+	&init->sl_alloc
 	/* test_alloc not affected by +Mea??? or +Mu???  */
     };
     int aui_sz = (int) sizeof(aui)/sizeof(aui[0]);
-- 
cgit v1.2.3


From 48e77453536e49b07ddb6be63ba322ddaa5dac45 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 10 May 2017 18:54:22 +0200
Subject: erts: Some code cleanup for gdb to work better

---
 erts/emulator/beam/erl_process.c | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 9c92d2a1a4..5d33c9f6d6 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -199,7 +199,7 @@ int erts_dio_sched_thread_suggested_stack_size = -1;
 ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];
 #endif
 
-static struct {
+static struct ErtsSchedBusyWait_ {
     int aux_work;
     int tse;
     int sys_schedule;
@@ -214,19 +214,19 @@ static ErtsAuxWorkData *aux_thread_aux_work_data;
 #define ERTS_SCHDLR_SSPND_CHNG_ONLN		(((erts_aint32_t) 1) << 2)
 #define ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN	(((erts_aint32_t) 1) << 3)
 
-typedef struct {
+typedef struct ErtsMultiSchedulingBlock_ {
     int ongoing;
     ErtsProcList *blckrs;
     ErtsProcList *chngq;
 } ErtsMultiSchedulingBlock;
 
-typedef struct {
+typedef struct ErtsSchedTypeCounters_ {
     Uint32 normal;
     Uint32 dirty_cpu;
     Uint32 dirty_io;
 } ErtsSchedTypeCounters;
 
-static struct {
+static struct ErtsSchedSuspend_ {
     erts_mtx_t mtx;
     ErtsSchedTypeCounters online;
     ErtsSchedTypeCounters curr_online;
@@ -1136,13 +1136,11 @@ dirty_sched_wall_time_change(ErtsSchedulerData *esdp, int working)
     mod++;
     erts_atomic32_set_nob(&esdp->sched_wall_time.u.mod, mod);
 
-#if 0
     if (!working) {
-        ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_BUSY_WAIT);
+        ERTS_MSACC_SET_STATE_X(ERTS_MSACC_STATE_BUSY_WAIT);
     } else {
-        ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_OTHER);
+        ERTS_MSACC_SET_STATE_X(ERTS_MSACC_STATE_OTHER);
     }
-#endif
 }
 
 
-- 
cgit v1.2.3


From 9a0970257aaaf9d343f8045548a34abf30dc0c92 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 12 May 2017 18:05:03 +0200
Subject: erts: Add multiple poll sets

---
 erts/emulator/beam/erl_port_task.c             |  37 +-
 erts/emulator/beam/erl_port_task.h             |  22 +-
 erts/emulator/beam/erl_process.c               | 144 +----
 erts/emulator/beam/erl_process.h               |   3 +
 erts/emulator/beam/erl_vm.h                    |   2 +-
 erts/emulator/sys/common/erl_check_io.c        | 734 +++++++++++++++----------
 erts/emulator/sys/common/erl_check_io.h        |  27 +-
 erts/emulator/sys/common/erl_poll.h            |   1 +
 erts/emulator/sys/common/erl_sys_common_misc.c |   8 -
 erts/emulator/sys/unix/sys.c                   |   6 +-
 erts/emulator/sys/unix/sys_drivers.c           |  11 +-
 erts/emulator/test/driver_SUITE.erl            |  56 +-
 erts/emulator/test/z_SUITE.erl                 |   2 +-
 13 files changed, 543 insertions(+), 510 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index bdce811847..141e815d1f 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -91,8 +91,6 @@ static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_q
 				    erts_atomic_read_nob(&(PP)->run_queue))); \
     } while (0)
 
-erts_atomic_t erts_port_task_outstanding_io_tasks;
-
 #define ERTS_PT_STATE_SCHEDULED		0
 #define ERTS_PT_STATE_ABORTED		1
 #define ERTS_PT_STATE_EXECUTING		2
@@ -584,9 +582,12 @@ static ERTS_INLINE void
 reset_executed_io_task_handle(ErtsPortTask *ptp)
 {
     if (ptp->u.alive.handle) {
+	ErtsIoTask *itp = ErtsContainerStruct(ptp->u.alive.handle, ErtsIoTask, task);
+
 	ASSERT(ptp == handle2task(ptp->u.alive.handle));
 	erts_io_notify_port_task_executed(ptp->u.alive.handle);
 	reset_port_task_handle(ptp->u.alive.handle);
+	erts_check_io_interrupt(itp->pollset, 1);
     }
 }
 
@@ -1308,20 +1309,7 @@ erts_port_task_abort(ErtsPortTaskHandle *pthp)
 	if (old_state != ERTS_PT_STATE_SCHEDULED)
 	    res = - 1; /* Task already aborted, executing, or executed */
 	else {
-
 	    reset_port_task_handle(pthp);
-
-	    switch (ptp->type) {
-	    case ERTS_PORT_TASK_INPUT:
-	    case ERTS_PORT_TASK_OUTPUT:
-		ASSERT(erts_atomic_read_nob(
-			   &erts_port_task_outstanding_io_tasks) > 0);
-		erts_atomic_dec_relb(&erts_port_task_outstanding_io_tasks);
-		break;
-	    default:
-		break;
-	    }
-
 	    res = 0;
 	}
     }
@@ -1458,7 +1446,6 @@ erts_port_task_schedule(Eterm id,
 	va_start(argp, type);
 	ptp->u.alive.td.io.event = va_arg(argp, ErlDrvEvent);
 	va_end(argp);
-	erts_atomic_inc_relb(&erts_port_task_outstanding_io_tasks);
 	break;
     }
     case ERTS_PORT_TASK_PROC_SIG: {
@@ -1634,13 +1621,12 @@ erts_port_task_free_port(Port *pp)
  * scheduling of processes. Run-queue lock should be held by caller.
  */
 
-int
+void
 erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 {
     Port *pp;
     ErtsPortTask *execq;
     int processing_busy_q;
-    int res = 0;
     int vreds = 0;
     int reds = 0;
     erts_aint_t io_tasks_executed = 0;
@@ -1655,7 +1641,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 
     pp = pop_port(runq);
     if (!pp) {
-	res = 0;
 	goto done;
     }
 
@@ -1822,14 +1807,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     erts_unblock_fpe(fpe_was_unmasked);
     ERTS_MSACC_POP_STATE_M();
 
-
-    if (io_tasks_executed) {
-	ASSERT(erts_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
-	       >= io_tasks_executed);
-	erts_atomic_add_relb(&erts_port_task_outstanding_io_tasks,
-				 -1*io_tasks_executed);
-    }
-
     ASSERT(runq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue));
 
     active = finalize_exec(pp, &execq, processing_busy_q);
@@ -1870,15 +1847,11 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     }
 
  done:
-    res = (erts_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
-	   != (erts_aint_t) 0);
 
     runq->scheduler->reductions += reds;
 
     ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq));
     ERTS_PORT_REDUCTIONS_EXECUTED(esdp, runq, reds);
-
-    return res;
 }
 
 static void
@@ -2123,8 +2096,6 @@ erts_dequeue_port(ErtsRunQueue *rq)
 void
 erts_port_task_init(void)
 {
-    erts_atomic_init_nob(&erts_port_task_outstanding_io_tasks,
-			     (erts_aint_t) 0);
     init_port_task_alloc();
     init_busy_caller_table_alloc();
 }
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index ffd42c9bab..ce63e4a7c0 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -61,11 +61,6 @@ typedef enum {
     ERTS_PORT_TASK_PROC_SIG
 } ErtsPortTaskType;
 
-#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
-/* NOTE: Do not access any of the exported variables directly */
-extern erts_atomic_t erts_port_task_outstanding_io_tasks;
-#endif
-
 #define ERTS_PTS_FLG_IN_RUNQ			(((erts_aint32_t) 1) <<  0)
 #define ERTS_PTS_FLG_EXEC			(((erts_aint32_t) 1) <<  1)
 #define ERTS_PTS_FLG_HAVE_TASKS			(((erts_aint32_t) 1) <<  2)
@@ -139,10 +134,6 @@ ERTS_GLB_INLINE void erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp);
 ERTS_GLB_INLINE int erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp);
 ERTS_GLB_INLINE void erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp);
 
-#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
-ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void);
-#endif
-
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_INLINE void
@@ -220,21 +211,10 @@ erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp)
     erts_atomic32_read_bor_nob(&ptsp->flags, ERTS_PTS_FLG_EXITING);
 }
 
-#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
-
-ERTS_GLB_INLINE int
-erts_port_task_have_outstanding_io_tasks(void)
-{
-    return (erts_atomic_read_acqb(&erts_port_task_outstanding_io_tasks)
-	    != 0);
-}
-
-#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
-
 #endif
 
 #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
-int erts_port_task_execute(ErtsRunQueue *, Port **);
+void erts_port_task_execute(ErtsRunQueue *, Port **);
 void erts_port_task_init(void);
 #endif
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 5d33c9f6d6..2a34410d5b 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -365,9 +365,6 @@ erts_sched_stat_t erts_sched_stat;
 
 static erts_tsd_key_t ERTS_WRITE_UNLIKELY(sched_data_key);
 
-static erts_atomic32_t function_calls;
-
-static erts_atomic32_t doing_sys_schedule;
 static erts_atomic32_t no_empty_run_queues;
 long erts_runq_supervision_interval = 0;
 static ethr_event runq_supervision_event;
@@ -1558,18 +1555,19 @@ erts_psd_set_init(Process *p, int ix, void *data)
 
 
 void
-erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags)
+erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi,
+                       erts_aint32_t flags)
 {
     switch (flags & ERTS_SSI_FLGS_SLEEP_TYPE) {
     case ERTS_SSI_FLG_POLL_SLEEPING:
-	erts_check_io_interrupt(1);
+	erts_check_io_interrupt(ssi->esdp->pollset, 1);
 	break;
     case ERTS_SSI_FLG_POLL_SLEEPING|ERTS_SSI_FLG_TSE_SLEEPING:
 	/*
 	 * Thread progress blocking while poll sleeping; need
 	 * to signal on both...
 	 */
-	erts_check_io_interrupt(1);
+	erts_check_io_interrupt(ssi->esdp->pollset, 1);
 	/* fall through */
     case ERTS_SSI_FLG_TSE_SLEEPING:
 	erts_tse_set(ssi->event);
@@ -2869,49 +2867,13 @@ erts_active_schedulers(void)
     return as;
 }
 
-
-static ERTS_INLINE void
-clear_sys_scheduling(void)
-{
-    erts_atomic32_set_mb(&doing_sys_schedule, 0);
-}
-
-static ERTS_INLINE int
-try_set_sys_scheduling(void)
-{
-    return 0 == erts_atomic32_cmpxchg_acqb(&doing_sys_schedule, 1, 0);
-}
-
-
 static ERTS_INLINE int
 prepare_for_sys_schedule(int non_blocking)
 {
-    if (non_blocking && erts_eager_check_io) {
-	return try_set_sys_scheduling();
-    }
-    else {
-	while (!erts_port_task_have_outstanding_io_tasks()
-	       && try_set_sys_scheduling()) {
-	    if (!erts_port_task_have_outstanding_io_tasks())
-		return 1;
-	    clear_sys_scheduling();
-	}
-	return 0;
-    }
+    return 1;
 }
 
 
-static ERTS_INLINE void
-sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq)
-{
-    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
-
-    ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
-
-    ASSERT(rq->waiting < 0);
-    rq->waiting *= -1;
-}
-
 static ERTS_INLINE void
 sched_waiting(Uint no, ErtsRunQueue *rq)
 {
@@ -3083,7 +3045,7 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type)
 	erts_tse_reset(ssi->event);
     else {
 	ASSERT(sleep_type == ERTS_SSI_FLG_POLL_SLEEPING);
-	erts_check_io_interrupt(0);
+	erts_check_io_interrupt(ssi->esdp->pollset, 0);
     }
 
     while (1) {
@@ -3272,8 +3234,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	spincount = sched_busy_wait.tse;
 
-    tse_wait:
-
 	if (ERTS_SCHEDULER_IS_DIRTY(esdp))
 	    dirty_sched_wall_time_change(esdp, working = 0);
         else if (thr_prgr_active != working)
@@ -3402,7 +3362,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
     else
     {
 
-	erts_atomic32_set_relb(&function_calls, 0);
+	esdp->function_calls = 0;
 	*fcalls = 0;
 
 	ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
@@ -3428,7 +3388,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	    ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO);
 
-	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
             LTTNG2(scheduler_poll, esdp->no, 1);
 	    erl_sys_schedule(1); /* Might give us something to do */
 
@@ -3459,48 +3418,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
 		goto sys_woken;
 	    }
-
-	    /*
-	     * If we got new I/O tasks we aren't allowed to
-	     * call erl_sys_schedule() until it is handled.
-	     */
-	    if (erts_port_task_have_outstanding_io_tasks()) {
-		clear_sys_scheduling();
-		/*
-		 * Got to check that we still got I/O tasks; otherwise
-		 * we have to continue checking for I/O...
-		 */
-		if (!prepare_for_sys_schedule(0)) {
-		    spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT;
-		    goto tse_wait;
-		}
-	    }
 	}
 
 	erts_runq_lock(rq);
 
-	/*
-	 * If we got new I/O tasks we aren't allowed to
-	 * sleep in erl_sys_schedule().
-	 */
-	if (erts_port_task_have_outstanding_io_tasks()) {
-	    clear_sys_scheduling();
-
-	    /*
-	     * Got to check that we still got I/O tasks; otherwise
-	     * we have to wait in erl_sys_schedule() after all...
-	     */
-	    if (!prepare_for_sys_schedule(0)) {
-		/*
-		 * Not allowed to wait in erl_sys_schedule;
-		 * do tse wait instead...
-		 */
-		sched_change_waiting_sys_to_waiting(esdp->no, rq);
-		erts_runq_unlock(rq);
-		spincount = 0;
-		goto tse_wait;
-	    }
-	}
 	if (aux_work) {
 	    erts_runq_unlock(rq);
 	    goto sys_poll_aux_work;
@@ -3517,7 +3438,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
 		goto sys_woken;
 	    }
-	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
 	    goto sys_poll_aux_work;
 	}
 
@@ -3532,8 +3452,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	if (thr_prgr_active)
 	    erts_thr_progress_active(esdp, thr_prgr_active = 0);
 
-	ASSERT(!erts_port_task_have_outstanding_io_tasks());
-
 	ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO);
         LTTNG2(scheduler_poll, esdp->no, 0);
 
@@ -3561,7 +3479,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
 	    erts_runq_lock(rq);
 	}
-	clear_sys_scheduling();
 	if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC))
 	    erts_atomic32_read_band_nob(&ssi->flags,
                                             (ERTS_SSI_FLG_SUSPENDED
@@ -5840,6 +5757,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     esdp->run_queue = runq;
     if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) {
 	esdp->no = 0;
+        esdp->pollset = NULL;
         if (runq == ERTS_DIRTY_CPU_RUNQ)
             esdp->type = ERTS_SCHED_DIRTY_CPU;
         else {
@@ -5859,6 +5777,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
         esdp->type = ERTS_SCHED_NORMAL;
 	esdp->no = (Uint) num;
 	esdp->dirty_no = 0;
+        esdp->pollset = erts_get_pollset(num);
         runq->scheduler = esdp;
     }
     esdp->dirty_shadow_process = shadow_proc;
@@ -5871,6 +5790,9 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 	shadow_proc->static_flags = ERTS_STC_FLG_SHADOW_PROC;
     }
 
+    esdp->function_calls = 0;
+
+    ssi->esdp = esdp;
     esdp->ssi = ssi;
     esdp->current_process = NULL;
     esdp->current_port = NULL;
@@ -6204,13 +6126,9 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	erts_atomic32_set_nob(&schdlr_sspnd.changing,
 				  set_schdlr_sspnd_change_flags);
 
-    erts_atomic32_init_nob(&doing_sys_schedule, 0);
-
     init_misc_aux_work();
 
 
-    erts_atomic32_init_nob(&function_calls, 0);
-
     /* init port tasks */
     erts_port_task_init();
 
@@ -7379,7 +7297,7 @@ msb_scheduler_type_switch(ErtsSchedType sched_type,
             calls = ERTS_MODIFIED_TIMING_INPUT_REDS + 1;
         else
             calls = INPUT_REDUCTIONS + 1;
-        erts_atomic32_set_nob(&function_calls, calls);
+        esdp->function_calls = calls;
 
         if ((nrml_prio == ERTS_MSB_NONE_PRIO_BIT)
             & ((dcpu_prio != ERTS_MSB_NONE_PRIO_BIT)
@@ -9769,7 +9687,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	}
 	rq = erts_get_runq_current(esdp);
 	ASSERT(esdp);
-	fcalls = (int) erts_atomic32_read_acqb(&function_calls);
+	fcalls = esdp->function_calls;
 	actual_reds = reds = 0;
 	erts_runq_lock(rq);
     } else {
@@ -9795,7 +9713,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST;
 	esdp->virtual_reds = 0;
 
-	fcalls = (int) erts_atomic32_add_read_acqb(&function_calls, reds);
+        esdp->function_calls += reds;
+	fcalls = esdp->function_calls;
 	ASSERT(esdp && esdp == erts_get_scheduler_data());
 
 	rq = erts_get_runq_current(esdp);
@@ -10044,11 +9963,11 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	    ERTS_MSACC_PUSH_STATE_CACHED_M();
 
-	    erts_atomic32_set_relb(&function_calls, 0);
+	    esdp->function_calls = 0;
 	    fcalls = 0;
 
 #if 0 /* Not needed since we wont wait in sys schedule */
-	    erts_check_io_interrupt(0);
+	    erts_check_io_interrupt(esdp->pollset, 0);
 #endif
 	    erts_runq_unlock(rq);
 
@@ -10063,7 +9982,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		erts_bump_timers(esdp->timer_wheel, current_time);
 
 	    erts_runq_lock(rq);
-	    clear_sys_scheduling();
 	    goto continue_check_activities_to_run;
 	}
 
@@ -10079,29 +9997,9 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
         flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
 
 	if (flags & PORT_BIT) {
-	    int have_outstanding_io;
-	    have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port);
-	    if ((!erts_eager_check_io
-		 && have_outstanding_io
-		 && fcalls > 2*input_reductions)
-		|| (flags & ERTS_RUNQ_FLG_HALTING)) {
-		/*
-		 * If we have performed more than 2*INPUT_REDUCTIONS since
-		 * last call to erl_sys_schedule() and we still haven't
-		 * handled all I/O tasks we stop running processes and
-		 * focus completely on ports.
-		 *
-		 * One could argue that this is a strange behavior. The
-		 * reason for doing it this way is that it is similar
-		 * to the behavior before port tasks were introduced.
-		 * We don't want to change the behavior too much, at
-		 * least not at the time of writing. This behavior
-		 * might change in the future.
-		 *
-		 * /rickard
-		 */
-		goto check_activities_to_run;
-	    }
+	    erts_port_task_execute(rq, &esdp->current_port);
+            if (flags & ERTS_RUNQ_FLG_HALTING)
+                goto check_activities_to_run;
 	}
 
 	/*
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index e63da2d9db..b372146846 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -364,6 +364,7 @@ typedef struct {
 } ErtsSchedulerSleepList;
 
 struct ErtsSchedulerSleepInfo_ {
+    struct ErtsSchedulerData_ *esdp;
     ErtsSchedulerSleepInfo *next;
     ErtsSchedulerSleepInfo *prev;
     erts_atomic32_t flags;
@@ -631,6 +632,8 @@ struct ErtsSchedulerData_ {
     void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
     Process *free_process;
     ErtsThrPrgrData thr_progress_data;
+    struct pollset_info* pollset;
+    int function_calls;
     ErtsSchedulerSleepInfo *ssi;
     Process *current_process;
     ErtsSchedType type;
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 076767c7cd..82977e62ea 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -46,7 +46,7 @@
  */
 #define ERTS_X_REGS_ALLOCATED (MAX_REG+3)
 
-#define INPUT_REDUCTIONS (2 * CONTEXT_REDS)
+#define INPUT_REDUCTIONS (CONTEXT_REDS / 4)
 
 #define H_DEFAULT_SIZE  233        /* default (heap + stack) min size */
 #define VH_DEFAULT_SIZE  32768     /* default virtual (bin) heap min size (words) */
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 52584e4246..ff0f3ea121 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -94,6 +94,7 @@ static struct pollset_info
 {
     ErtsPollSet ps;
     erts_atomic_t in_poll_wait;        /* set while doing poll */
+    erts_atomic_t check_io_time;
     struct {
 	int six; /* start index */
 	int eix; /* end index */
@@ -102,7 +103,7 @@ static struct pollset_info
 	ErtsSysFdType *array;
     } active_fd;
     erts_atomic_t removed_list;    /* struct removed_fd* */
-}pollset;
+}*pollsetv;
 
 #define NUM_OF_POLLSETS 1
 
@@ -114,21 +115,39 @@ int ERTS_CIO_EXPORT(enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags
 Uint ERTS_CIO_EXPORT(erts_check_io_size)(void);
 Eterm ERTS_CIO_EXPORT(erts_check_io_info)(void *);
 int ERTS_CIO_EXPORT(erts_check_io_max_files)(void);
-void ERTS_CIO_EXPORT(erts_check_io_interrupt)(int);
-void ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(int, ErtsMonotonicTime);
+void ERTS_CIO_EXPORT(erts_check_io_interrupt)(struct pollset_info*, int);
+void ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(struct pollset_info*, int, ErtsMonotonicTime);
 void ERTS_CIO_EXPORT(erts_check_io)(int);
+struct pollset_info* ERTS_CIO_EXPORT(erts_get_pollset)(int);
 int ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *);
+void ERTS_CIO_EXPORT(erts_io_notify_port_task_executed)(ErtsPortTaskHandle *pthp);
 #ifdef ERTS_ENABLE_LOCK_COUNT
 void ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks)(int enable);
 #endif
 #endif
 
+/* ToDo: Was inline in erl_check_io.h but now need struct pollset_info */
+void
+ERTS_CIO_EXPORT(erts_io_notify_port_task_executed)(ErtsPortTaskHandle *pthp)
+{
+    ErtsIoTask *itp = ErtsContainerStruct(pthp, ErtsIoTask, task);
+    erts_aint_t ci_time = erts_atomic_read_acqb(&itp->pollset->check_io_time);
+    erts_atomic_set_relb(&itp->executed_time, ci_time);
+}
+
+
+struct pollset_info* ERTS_CIO_EXPORT(erts_get_pollset)(int sched_nr)
+{
+    ASSERT(sched_nr > 0 && sched_nr <= erts_no_schedulers);
+    return &pollsetv[sched_nr - 1];
+}
 
 typedef struct {
 #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
     SafeHashBucket hb;
 #endif
     ErtsSysFdType fd;
+    struct pollset_info *pollset;
     struct {
 	ErtsDrvSelectDataState *select;   /* ERTS_EV_TYPE_DRV_SEL */
         ErtsNifSelectDataState *nif;      /* ERTS_EV_TYPE_NIF */
@@ -209,6 +228,7 @@ static ERTS_INLINE ErtsDrvEventState* hash_new_drv_ev_state(ErtsSysFdType fd)
 {
     ErtsDrvEventState tmpl;
     tmpl.fd = fd;
+    tmpl.pollset = NULL;
     tmpl.driver.select = NULL;
     tmpl.driver.nif = NULL;
     tmpl.driver.stop.drv_ptr = NULL;
@@ -252,10 +272,11 @@ steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource*,
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(removed_fd, struct removed_fd, 64, ERTS_ALC_T_FD_LIST)
 
 static ERTS_INLINE void
-init_iotask(ErtsIoTask *io_task)
+init_iotask(ErtsIoTask *io_task, struct pollset_info* psi)
 {
     erts_port_task_handle_init(&io_task->task);
     erts_atomic_init_nob(&io_task->executed_time, ~((erts_aint_t) 0));
+    io_task->pollset = psi;
 }
 
 static ERTS_INLINE int
@@ -269,14 +290,14 @@ is_iotask_active(ErtsIoTask *io_task, erts_aint_t current_cio_time)
 }
 
 static ERTS_INLINE ErtsDrvSelectDataState *
-alloc_drv_select_data(void)
+alloc_drv_select_data(struct pollset_info* psi)
 {
     ErtsDrvSelectDataState *dsp = erts_alloc(ERTS_ALC_T_DRV_SEL_D_STATE,
 					     sizeof(ErtsDrvSelectDataState));
     dsp->inport = NIL;
     dsp->outport = NIL;
-    init_iotask(&dsp->iniotask);
-    init_iotask(&dsp->outiotask);
+    init_iotask(&dsp->iniotask, psi);
+    init_iotask(&dsp->outiotask, psi);
     return dsp;
 }
 
@@ -307,11 +328,11 @@ free_nif_select_data(ErtsNifSelectDataState *dsp)
 }
 
 static ERTS_INLINE void
-remember_removed(ErtsDrvEventState *state, struct pollset_info* psi)
+remember_removed(ErtsDrvEventState *state)
 {
     struct removed_fd *fdlp;
     ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd)));
-    if (erts_atomic_read_nob(&psi->in_poll_wait)) {
+    if (erts_atomic_read_nob(&state->pollset->in_poll_wait)) {
         erts_aint_t was_next, exp_next;
 	state->remove_cnt++;
 	ASSERT(state->remove_cnt > 0);
@@ -324,11 +345,11 @@ remember_removed(ErtsDrvEventState *state, struct pollset_info* psi)
     #endif
 
          /* Lockless atomic insertion in removed_list */
-        was_next = erts_atomic_read_acqb(&psi->removed_list);
+        was_next = erts_atomic_read_acqb(&state->pollset->removed_list);
         do {
             exp_next = was_next;
             fdlp->next = (struct removed_fd*) exp_next;
-            was_next = erts_atomic_cmpxchg_mb(&psi->removed_list,
+            was_next = erts_atomic_cmpxchg_mb(&state->pollset->removed_list,
                                               (erts_aint_t) fdlp,
                                               exp_next);
         }while (was_next != exp_next);
@@ -384,7 +405,7 @@ forget_removed(struct pollset_info* psi)
                 state->driver.stop.resource = NULL;
                 ASSERT(resource);
 		state->type = ERTS_EV_TYPE_NONE;
-		state->flags &= ~ERTS_EV_FLAG_USED;
+		state->flags = 0;
                 goto case_ERTS_EV_TYPE_NONE;
 
 	    case ERTS_EV_TYPE_STOP_USE:
@@ -392,11 +413,12 @@ forget_removed(struct pollset_info* psi)
 		drv_ptr = state->driver.stop.drv_ptr;
 		ASSERT(drv_ptr);
 		state->type = ERTS_EV_TYPE_NONE;
-		state->flags &= ~ERTS_EV_FLAG_USED;
+		state->flags = 0;
 		state->driver.stop.drv_ptr = NULL;
 		/* Fall through */
             case ERTS_EV_TYPE_NONE:
             case_ERTS_EV_TYPE_NONE:
+                state->pollset = NULL;
 #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
 		hash_erase_drv_ev_state(state);
 #endif
@@ -455,6 +477,7 @@ grow_drv_ev_state(int min_ix)
 				     sizeof(ErtsDrvEventState)*new_len));
 	for (i = old_len; i < new_len; i++) {
 	    drv_ev_state.v[i].fd = (ErtsSysFdType) i;
+            drv_ev_state.v[i].pollset = NULL;
 	    drv_ev_state.v[i].driver.select = NULL;
 	    drv_ev_state.v[i].driver.stop.drv_ptr = NULL;
             drv_ev_state.v[i].driver.nif = NULL;
@@ -541,7 +564,9 @@ deselect(ErtsDrvEventState *state, int mode)
 	}
     }
 
-    state->events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, rm_events, 0, &do_wake);
+    ERTS_CIO_POLL_CTL(state->pollset->ps, state->fd, rm_events,
+                      0, &do_wake);
+    state->events &= ~rm_events;
 
     if (!(state->events)) {
 	switch (state->type) {
@@ -565,8 +590,8 @@ deselect(ErtsDrvEventState *state, int mode)
 	}
 	    
 	state->type = ERTS_EV_TYPE_NONE;
-	state->flags &= ~ERTS_EV_FLAG_USED;
-	remember_removed(state, &pollset);
+	state->flags = 0;
+	remember_removed(state);
     }
 }
 
@@ -585,7 +610,7 @@ check_fd_cleanup(ErtsDrvEventState *state,
 
     ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd)));
 
-    current_cio_time = erts_atomic_read_acqb(&erts_check_io_time);
+    current_cio_time = erts_atomic_read_acqb(&state->pollset->check_io_time);
     *free_select = NULL;
     if (state->driver.select
 	&& (state->type != ERTS_EV_TYPE_DRV_SEL)
@@ -602,19 +627,27 @@ check_fd_cleanup(ErtsDrvEventState *state,
         state->driver.nif = NULL;
     }
 
-#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
     if (((state->type != ERTS_EV_TYPE_NONE)
 	 | state->remove_cnt
+         | (state->driver.nif != NULL)
 	 | (state->driver.select != NULL)) == 0) {
 
+        state->pollset = NULL;
+#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
 	hash_erase_drv_ev_state(state);
-
-    }
 #endif
+    }
 }
 
+#ifdef __WIN32__
+# define MUST_DEFER(MAY_SLEEP) 1
+#else
+# define MUST_DEFER(MAY_SLEEP) (MAY_SLEEP)
+#endif
+
 static ERTS_INLINE int
-check_cleanup_active_fd(ErtsSysFdType fd,
+check_cleanup_active_fd(struct pollset_info* psi,
+                        ErtsSysFdType fd,
 #if ERTS_CIO_DEFER_ACTIVE_EVENTS
 			ErtsPollControlEntry *pce,
 			int *pce_ix,
@@ -637,14 +670,15 @@ check_cleanup_active_fd(ErtsSysFdType fd,
     state = &drv_ev_state.v[(int) fd];
 #else
     state = hash_get_drv_ev_state(fd); /* may be NULL! */
-    if (state)
 #endif
+    if (state && state->pollset == psi)
     {
 	if (state->driver.select) {
 #if ERTS_CIO_DEFER_ACTIVE_EVENTS
 	    if (is_iotask_active(&state->driver.select->iniotask, current_cio_time)) {
 		active = 1;
-		if ((state->events & ERTS_POLL_EV_IN)
+		if (MUST_DEFER(may_sleep)
+                    && (state->events & ERTS_POLL_EV_IN)
 		    && !(state->flags & ERTS_EV_FLAG_DEFER_IN_EV)) {
 		    evoff |= ERTS_POLL_EV_IN;
 		    state->flags |= ERTS_EV_FLAG_DEFER_IN_EV;
@@ -657,15 +691,17 @@ check_cleanup_active_fd(ErtsSysFdType fd,
 	    }
 	    if (is_iotask_active(&state->driver.select->outiotask, current_cio_time)) {
 		active = 1;
-		if ((state->events & ERTS_POLL_EV_OUT)
+		if (MUST_DEFER(may_sleep)
+                    && (state->events & ERTS_POLL_EV_OUT)
 		    && !(state->flags & ERTS_EV_FLAG_DEFER_OUT_EV)) {
 		    evoff |= ERTS_POLL_EV_OUT;
 		    state->flags |= ERTS_EV_FLAG_DEFER_OUT_EV;
 		}
 	    }
 	    else if (state->flags & ERTS_EV_FLAG_DEFER_OUT_EV) {
-		if (state->events & ERTS_POLL_EV_OUT)
+		if (state->events & ERTS_POLL_EV_OUT) {
 		    evon |= ERTS_POLL_EV_OUT;
+                }
 		state->flags &= ~ERTS_EV_FLAG_DEFER_OUT_EV;
 	    }
 	    if (active)
@@ -681,6 +717,12 @@ check_cleanup_active_fd(ErtsSysFdType fd,
 		free_select = state->driver.select;
 		state->driver.select = NULL;
 	    }
+#if ERTS_CIO_DEFER_ACTIVE_EVENTS
+             if (evon) {
+                 int do_wake = 0;
+                 ERTS_CIO_POLL_CTL(psi->ps, state->fd, evon, 1, &do_wake);
+             }
+#endif
 	}
 
         if (state->driver.nif) {
@@ -705,7 +747,7 @@ check_cleanup_active_fd(ErtsSysFdType fd,
             }
             if (rm_events) {
                 int do_wake = 0;
-                state->events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd,
+                state->events = ERTS_CIO_POLL_CTL(state->pollset->ps, state->fd,
                                                   rm_events, 0, &do_wake);
             }
             if (state->events)
@@ -720,7 +762,6 @@ check_cleanup_active_fd(ErtsSysFdType fd,
 	if (((state->type != ERTS_EV_TYPE_NONE) | state->remove_cnt | active) == 0)
 	    hash_erase_drv_ev_state(state);
 #endif
-
     }
 
     erts_mtx_unlock(mtx);
@@ -737,28 +778,23 @@ check_cleanup_active_fd(ErtsSysFdType fd,
 	pcep->events = evoff;
 	pcep->on = 0;
     }
-    if (evon) {
-	ErtsPollControlEntry *pcep = &pce[(*pce_ix)++];
-	pcep->fd = fd;
-	pcep->events = evon;
-	pcep->on = 1;
-    }
 #endif
 
     return active;
 }
 
 static void
-check_cleanup_active_fds(erts_aint_t current_cio_time, int may_sleep)
+check_cleanup_active_fds(struct pollset_info* psi,
+                         erts_aint_t current_cio_time, int may_sleep)
 {
-    int six = pollset.active_fd.six;
-    int eix = pollset.active_fd.eix;
-    erts_aint32_t no = erts_atomic32_read_dirty(&pollset.active_fd.no);
-    int size = pollset.active_fd.size;
+    int six = psi->active_fd.six;
+    int eix = psi->active_fd.eix;
+    erts_aint32_t no = erts_atomic32_read_dirty(&psi->active_fd.no);
+    const int size = psi->active_fd.size;
     int ix = six;
 #if ERTS_CIO_DEFER_ACTIVE_EVENTS
-    /* every fd might add two entries */
-    Uint pce_sz = 2*sizeof(ErtsPollControlEntry)*no;
+    /* every fd might add one entry */
+    Uint pce_sz = sizeof(ErtsPollControlEntry)*no;
     ErtsPollControlEntry *pctrl_entries = (pce_sz
 					   ? erts_alloc(ERTS_ALC_T_TMP, pce_sz)
 					   : NULL);
@@ -766,12 +802,12 @@ check_cleanup_active_fds(erts_aint_t current_cio_time, int may_sleep)
 #endif
 
     while (ix != eix) {
-	ErtsSysFdType fd = pollset.active_fd.array[ix];
+	ErtsSysFdType fd = psi->active_fd.array[ix];
 	int nix = ix + 1;
 	if (nix >= size)
 	    nix = 0;
 	ASSERT(fd != ERTS_SYS_FD_INVALID);
-	if (!check_cleanup_active_fd(fd,
+	if (!check_cleanup_active_fd(psi, fd,
 #if ERTS_CIO_DEFER_ACTIVE_EVENTS
 				     pctrl_entries,
 				     &pctrl_ix,
@@ -781,14 +817,14 @@ check_cleanup_active_fds(erts_aint_t current_cio_time, int may_sleep)
 	    no--;
 	    if (ix == six) {
 #ifdef DEBUG
-		pollset.active_fd.array[ix] = ERTS_SYS_FD_INVALID;
+		psi->active_fd.array[ix] = ERTS_SYS_FD_INVALID;
 #endif
 		six = nix;
 	    }
 	    else {
-		pollset.active_fd.array[ix] = pollset.active_fd.array[six];
+		psi->active_fd.array[ix] = psi->active_fd.array[six];
 #ifdef DEBUG
-		pollset.active_fd.array[six] = ERTS_SYS_FD_INVALID;
+		psi->active_fd.array[six] = ERTS_SYS_FD_INVALID;
 #endif
 		six++;
 		if (six >= size)
@@ -801,53 +837,53 @@ check_cleanup_active_fds(erts_aint_t current_cio_time, int may_sleep)
 #if ERTS_CIO_DEFER_ACTIVE_EVENTS
     ASSERT(pctrl_ix <= pce_sz/sizeof(ErtsPollControlEntry));
     if (pctrl_ix)
-	ERTS_CIO_POLL_CTLV(pollset.ps, pctrl_entries, pctrl_ix);
+	ERTS_CIO_POLL_CTLV(psi->ps, pctrl_entries, pctrl_ix);
     if (pctrl_entries)
 	erts_free(ERTS_ALC_T_TMP, pctrl_entries);
 #endif
 
-    pollset.active_fd.six = six;
-    pollset.active_fd.eix = eix;
-    erts_atomic32_set_relb(&pollset.active_fd.no, no);
+    psi->active_fd.six = six;
+    psi->active_fd.eix = eix;
+    erts_atomic32_set_relb(&psi->active_fd.no, no);
 }
 
-static void grow_active_fds(void)
+static void grow_active_fds(struct pollset_info *psi)
 {
-    ASSERT(pollset.active_fd.six == pollset.active_fd.eix);
-    pollset.active_fd.six = 0;
-    pollset.active_fd.eix = pollset.active_fd.size;
-    pollset.active_fd.size += ERTS_ACTIVE_FD_INC;
-    pollset.active_fd.array = erts_realloc(ERTS_ALC_T_ACTIVE_FD_ARR,
-                                           pollset.active_fd.array,
-                                           pollset.active_fd.size*sizeof(ErtsSysFdType));
+    ASSERT(psi->active_fd.six == psi->active_fd.eix);
+    psi->active_fd.six = 0;
+    psi->active_fd.eix = psi->active_fd.size;
+    psi->active_fd.size += ERTS_ACTIVE_FD_INC;
+    psi->active_fd.array = erts_realloc(ERTS_ALC_T_ACTIVE_FD_ARR,
+                                           psi->active_fd.array,
+                                           psi->active_fd.size*sizeof(ErtsSysFdType));
 #ifdef DEBUG
     {
         int i;
-        for (i = pollset.active_fd.eix + 1; i < pollset.active_fd.size; i++)
-            pollset.active_fd.array[i] = ERTS_SYS_FD_INVALID;
+        for (i = psi->active_fd.eix + 1; i < psi->active_fd.size; i++)
+            psi->active_fd.array[i] = ERTS_SYS_FD_INVALID;
     }
 #endif
 }
 
 static ERTS_INLINE void
-add_active_fd(ErtsSysFdType fd)
+add_active_fd(struct pollset_info *psi, ErtsSysFdType fd)
 {
-    int eix = pollset.active_fd.eix;
-    int size = pollset.active_fd.size;
+    int eix = psi->active_fd.eix;
+    const int size = psi->active_fd.size;
     
-    pollset.active_fd.array[eix] = fd;
+    psi->active_fd.array[eix] = fd;
 
-    erts_atomic32_set_relb(&pollset.active_fd.no,
-			       (erts_atomic32_read_dirty(&pollset.active_fd.no)
+    erts_atomic32_set_relb(&psi->active_fd.no,
+			       (erts_atomic32_read_dirty(&psi->active_fd.no)
 				+ 1));
 
     eix++;
     if (eix >= size)
 	eix = 0;
-    pollset.active_fd.eix = eix;
+    psi->active_fd.eix = eix;
 
-    if (pollset.active_fd.six == eix) {
-        grow_active_fds();
+    if (psi->active_fd.six == eix) {
+        grow_active_fds(psi);
     }
 }
 
@@ -862,9 +898,9 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
     Eterm id = erts_drvport2id(ix);
     ErtsSysFdType fd = (ErtsSysFdType) e;
     ErtsPollEvents ctl_events = (ErtsPollEvents) 0;
-    ErtsPollEvents new_events, old_events;
+    ErtsPollEvents old_events;
     ErtsDrvEventState *state;
-    int wake_poller;
+    int wake_poller = 0;
     int ret;
     ErtsDrvSelectDataState *free_select = NULL;
     ErtsNifSelectDataState *free_nif = NULL;
@@ -898,22 +934,25 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
     state = hash_get_drv_ev_state(fd); /* may be NULL! */
 #endif
 
-    if (!on && (mode&ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) {
-	if (IS_FD_UNKNOWN(state)) {
-	    /* fast track to stop_select callback */
-	    stop_select_fn = prt->drv_ptr->stop_select;
-#ifdef USE_VM_PROBES
-	    strncpy(name, prt->drv_ptr->name,
-		    sizeof(DTRACE_CHARBUF_NAME(name))-1);
-	    name[sizeof(name)-1] = '\0';
-#endif
-	    ret = 0;
-	    goto done_unknown;
-	}
-	mode |= (ERL_DRV_READ | ERL_DRV_WRITE);
-	wake_poller = 1; /* to eject fd from pollset (if needed) */
+    if (!on) {
+        if (IS_FD_UNKNOWN(state)) {
+            if ((mode&ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) {
+                /* fast track to stop_select callback */
+                stop_select_fn = prt->drv_ptr->stop_select;
+        #ifdef USE_VM_PROBES
+                strncpy(name, prt->drv_ptr->name,
+                        sizeof(DTRACE_CHARBUF_NAME(name))-1);
+                name[sizeof(name)-1] = '\0';
+        #endif
+            }
+            ret = 0;
+            goto done_unknown;
+        }
+        else if ((mode&ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) {
+            mode |= (ERL_DRV_READ | ERL_DRV_WRITE);
+            wake_poller = 1; /* to eject fd from pollset (if needed) */
+        }
     }
-    else wake_poller = 0;
 
 #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
     if (state == NULL) {
@@ -951,7 +990,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
 	    if (owner != id && is_not_nil(owner))
 		drv_select_steal(ix, state, mode, on);
 	}
-	ctl_events |= ERTS_POLL_EV_IN;
+        ctl_events = ERTS_POLL_EV_IN;
     }
     if (mode & ERL_DRV_WRITE) {
 	if (state->type == ERTS_EV_TYPE_DRV_SEL) {
@@ -962,44 +1001,61 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
 	ctl_events |= ERTS_POLL_EV_OUT;
     }	
 
+
     ASSERT((state->type == ERTS_EV_TYPE_DRV_SEL) ||
 	   (state->type == ERTS_EV_TYPE_NONE && !state->events));
 
-    if (!on && !(state->flags & ERTS_EV_FLAG_USED) 
-	&& state->events && !(state->events & ~ctl_events)) {	
-	/* Old driver removing all events. At least wake poller.
-	   It will not make close() 100% safe but it will prevent
-	   actions delayed by poll timeout. */
-	wake_poller = 1;
-    }
+    old_events = state->events;
 
-    new_events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, ctl_events, on, &wake_poller);
+    if (on) {
+        ctl_events &= ~old_events;
+        state->events |= ctl_events;
+    }
+    else {
+        ctl_events &= old_events;
+        state->events &= ~ctl_events;
 
-    if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
-	if (state->type == ERTS_EV_TYPE_DRV_SEL && !state->events) {
-	    state->type = ERTS_EV_TYPE_NONE;
-	    state->flags &= ~ERTS_EV_FLAG_USED;
-	    state->driver.select->inport = NIL;
-	    state->driver.select->outport = NIL;
-	}
-	ret = -1;
-	goto done;
+        if (!(state->flags & ERTS_EV_FLAG_USED)
+            && old_events && !state->events) {
+            /*
+             * Old driver removing all events. At least wake poller.
+             * It will not make close() 100% safe but it will prevent
+             * actions delayed by poll timeout.
+             */
+            wake_poller = 1;
+        }
     }
 
-    old_events = state->events;
+    if (ctl_events) {
+        ErtsPollEvents new_events;
 
-    ASSERT(on
-	   ? (new_events == (state->events | ctl_events))
-	   : (new_events == (state->events & ~ctl_events)));
+        if (!state->pollset) {
+            ErtsSchedulerData* esdp = erts_get_scheduler_data();
+            ASSERT(esdp);
+            state->pollset = esdp->pollset;
+        }
 
-    ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL
-	   || state->type == ERTS_EV_TYPE_NONE);
+        new_events = ERTS_CIO_POLL_CTL(state->pollset->ps, state->fd, ctl_events, on, &wake_poller);
 
-    state->events = new_events;
-    if (ctl_events) {
-	if (on) {
+        if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
+            if (state->type == ERTS_EV_TYPE_DRV_SEL && !old_events) {
+                state->type = ERTS_EV_TYPE_NONE;
+                state->flags = 0;
+                state->driver.select->inport = NIL;
+                state->driver.select->outport = NIL;
+            }
+            ret = -1;
+            goto done;
+        }
+
+        ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL
+               || state->type == ERTS_EV_TYPE_NONE);
+    }
+
+    if (on) {
+        if (ctl_events) {
 	    if (!state->driver.select)
-		state->driver.select = alloc_drv_select_data();
+		state->driver.select = alloc_drv_select_data(state->pollset);
 	    if (state->type == ERTS_EV_TYPE_NONE)
 		state->type = ERTS_EV_TYPE_DRV_SEL;
 	    ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL);
@@ -1010,8 +1066,9 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
 	    if (mode & ERL_DRV_USE) {
 		state->flags |= ERTS_EV_FLAG_USED;
 	    }
-	}
-	else { /* off */
+        }
+    }
+    else { /* off */
 	    if (state->type == ERTS_EV_TYPE_DRV_SEL) {
 		if (ctl_events & ERTS_POLL_EV_IN) {
 		    abort_tasks(state, ERL_DRV_READ);
@@ -1021,20 +1078,20 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
 		    abort_tasks(state, ERL_DRV_WRITE);
 		    state->driver.select->outport = NIL;
 		}
-		if (new_events == 0) {
+		if (state->events == 0) {
 		    if (old_events != 0) {
-			remember_removed(state, &pollset);
+			remember_removed(state);
 		    }		    
 		    if ((mode & ERL_DRV_USE) || !(state->flags & ERTS_EV_FLAG_USED)) {
 			state->type = ERTS_EV_TYPE_NONE;
-			state->flags &= ~ERTS_EV_FLAG_USED;
+			state->flags = 0;
 		    }
 		    /*else keep it, as fd will probably be selected upon again */
 		}
 	    }
 	    if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) {
 		erts_driver_t* drv_ptr = prt->drv_ptr;
-		ASSERT(new_events==0);
+		ASSERT(state->events==0);
 		if (state->remove_cnt == 0 || !wake_poller) {
 		    /* Safe to close fd now as it is not in pollset
 		       or there was no need to eject fd (kernel poll) */
@@ -1053,7 +1110,6 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
 		    }
 		}
 	    }
-	}
     }
    
     ret = 0;
@@ -1093,7 +1149,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
     ErtsResource* resource = DATA_TO_RESOURCE(obj);
     ErtsSysFdType fd = (ErtsSysFdType) e;
     ErtsPollEvents ctl_events = (ErtsPollEvents) 0;
-    ErtsPollEvents new_events, old_events;
+    ErtsPollEvents old_events;
     ErtsDrvEventState *state;
     int wake_poller;
     int ret;
@@ -1192,32 +1248,45 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
     ASSERT((state->type == ERTS_EV_TYPE_NIF) ||
 	   (state->type == ERTS_EV_TYPE_NONE && !state->events));
 
-    new_events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, ctl_events, on, &wake_poller);
+    old_events = state->events;
 
-    if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
-	if (state->type == ERTS_EV_TYPE_NIF && !state->events) {
-	    state->type = ERTS_EV_TYPE_NONE;
-	    state->flags &= ~ERTS_EV_FLAG_USED;
-	    state->driver.nif->in.pid = NIL;
-	    state->driver.nif->out.pid = NIL;
-            state->driver.nif->in.ddeselect_cnt = 0;
-            state->driver.nif->out.ddeselect_cnt = 0;
-            state->driver.stop.resource = NULL;
-	}
-	ret = INT_MIN | ERL_NIF_SELECT_FAILED;
-	goto done;
+    if (on) {
+        ctl_events &= ~old_events;
+        state->events |= ctl_events;
+    }
+    else {
+        ctl_events &= old_events;
+        state->events &= ~ctl_events;
     }
 
-    old_events = state->events;
+    if (ctl_events) {
+        ErtsPollEvents new_events;
+
+        if (!state->pollset) {
+            state->pollset = erts_get_scheduler_data()->pollset;
+        }
 
-    ASSERT(on
-	   ? (new_events == (state->events | ctl_events))
-	   : (new_events == (state->events & ~ctl_events)));
+        new_events = ERTS_CIO_POLL_CTL(state->pollset->ps, state->fd, ctl_events, on, &wake_poller);
+
+        if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
+            if (state->type == ERTS_EV_TYPE_NIF && !old_events) {
+                state->type = ERTS_EV_TYPE_NONE;
+                state->flags = 0;
+                state->driver.nif->in.pid = NIL;
+                state->driver.nif->out.pid = NIL;
+                state->driver.nif->in.ddeselect_cnt = 0;
+                state->driver.nif->out.ddeselect_cnt = 0;
+                state->driver.stop.resource = NULL;
+            }
+            ret = INT_MIN | ERL_NIF_SELECT_FAILED;
+            goto done;
+        }
+        ASSERT(new_events == state->events);
+    }
 
     ASSERT(state->type == ERTS_EV_TYPE_NIF
 	   || state->type == ERTS_EV_TYPE_NONE);
 
-    state->events = new_events;
     if (on) {
         const Eterm recipient = pid ? pid->pid : env->proc->common.id;
         Uint32* refn;
@@ -1230,7 +1299,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
         }
         ASSERT(state->type == ERTS_EV_TYPE_NIF);
         ASSERT(state->driver.stop.resource == resource);
-        if (ctl_events & ERTS_POLL_EV_IN) {
+        if (mode & ERL_DRV_READ) {
             state->driver.nif->in.pid = recipient;
             if (is_immed(ref)) {
                 state->driver.nif->in.immed = ref;
@@ -1244,7 +1313,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
             }
             state->driver.nif->in.ddeselect_cnt = 0;
         }
-        if (ctl_events & ERTS_POLL_EV_OUT) {
+        if (mode & ERL_DRV_WRITE) {
             state->driver.nif->out.pid = recipient;
             if (is_immed(ref)) {
                 state->driver.nif->out.immed = ref;
@@ -1267,10 +1336,10 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
             state->driver.nif->in.ddeselect_cnt = 0;
             state->driver.nif->out.ddeselect_cnt = 0;
             if (old_events != 0) {
-                remember_removed(state, &pollset);
+                remember_removed(state);
             }
         }
-        ASSERT(new_events==0);
+        ASSERT(state->events==0);
         if (state->remove_cnt == 0 || !wake_poller) {
             /*
              * Safe to close fd now as it is not in pollset
@@ -1567,7 +1636,7 @@ steal_pending_stop_use(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix,
             erts_ddll_dereference_driver(state->driver.stop.drv_ptr->handle);
         }
         state->type = ERTS_EV_TYPE_NONE;
-        state->flags &= ~ERTS_EV_FLAG_USED;
+        state->flags = 0;
         state->driver.stop.drv_ptr = NULL;
     }
     else {
@@ -1606,7 +1675,7 @@ steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource* resource,
 
         enif_release_resource(state->driver.stop.resource);
         state->type = ERTS_EV_TYPE_NONE;
-        state->flags &= ~ERTS_EV_FLAG_USED;
+        state->flags = 0;
         state->driver.stop.resource = NULL;
     }
     else {
@@ -1656,7 +1725,7 @@ iready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time)
 				    (ErlDrvEvent) state->fd) != 0) {
 	    stale_drv_select(id, state, ERL_DRV_READ);
 	}
-	add_active_fd(state->fd);
+	add_active_fd(state->pollset, state->fd);
     }
 }
 
@@ -1674,7 +1743,7 @@ oready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time)
 				    (ErlDrvEvent) state->fd) != 0) {
 	    stale_drv_select(id, state, ERL_DRV_WRITE);
 	}
-	add_active_fd(state->fd);
+	add_active_fd(state->pollset, state->fd);
     }
 }
 
@@ -1729,19 +1798,20 @@ send_event_tuple(struct erts_nif_select_event* e, ErtsResource* resource,
 static void bad_fd_in_pollset(ErtsDrvEventState *, Eterm inport, Eterm outport);
 
 void
-ERTS_CIO_EXPORT(erts_check_io_interrupt)(int set)
+ERTS_CIO_EXPORT(erts_check_io_interrupt)(struct pollset_info *psi, int set)
 {
-    ERTS_CIO_POLL_INTR(pollset.ps, set);
+    ERTS_CIO_POLL_INTR(psi->ps, set);
 }
 
 void
-ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(int set,
+ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(struct pollset_info *psi,
+                                               int set,
 					       ErtsMonotonicTime timeout_time)
 {
-    ERTS_CIO_POLL_INTR_TMD(pollset.ps, set, timeout_time);
+    ERTS_CIO_POLL_INTR_TMD(psi->ps, set, timeout_time);
 }
 
-#if !ERTS_CIO_DEFER_ACTIVE_EVENTS
+#ifndef __WIN32__
 /*
  * Number of ignored events, for a lingering fd added by enif_select(),
  * until we deselect fd-event from pollset.
@@ -1761,8 +1831,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
     int poll_ret, i;
     erts_aint_t current_cio_time;
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
-
-    ASSERT(esdp);
+    struct pollset_info *psi = esdp->pollset;
 
  restart:
 
@@ -1781,24 +1850,25 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
      * erts_check_io_time, since only one thread can
      * check io at a time.
      */
-    current_cio_time = erts_atomic_read_dirty(&erts_check_io_time);
+    current_cio_time = erts_atomic_read_dirty(&psi->check_io_time);
     current_cio_time++;
-    erts_atomic_set_relb(&erts_check_io_time, current_cio_time);
+    erts_atomic_set_relb(&psi->check_io_time, current_cio_time);
 
-    check_cleanup_active_fds(current_cio_time,
+    check_cleanup_active_fds(psi,
+                             current_cio_time,
                              timeout_time != ERTS_POLL_NO_TIMEOUT);
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_check_exact(NULL, 0); /* No locks should be locked */
 #endif
 
-    pollres_len = erts_atomic32_read_dirty(&pollset.active_fd.no) + ERTS_CHECK_IO_POLL_RES_LEN;
+    pollres_len = erts_atomic32_read_dirty(&psi->active_fd.no) + ERTS_CHECK_IO_POLL_RES_LEN;
 
     pollres = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollResFd)*pollres_len);
 
-    erts_atomic_set_nob(&pollset.in_poll_wait, 1);
+    erts_atomic_set_nob(&psi->in_poll_wait, 1);
 
-    poll_ret = ERTS_CIO_POLL_WAIT(pollset.ps, pollres, &pollres_len, timeout_time);
+    poll_ret = ERTS_CIO_POLL_WAIT(psi->ps, pollres, &pollres_len, timeout_time);
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_check_exact(NULL, 0); /* No locks should be locked */
@@ -1810,8 +1880,8 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 #endif
 
     if (poll_ret != 0) {
-	erts_atomic_set_nob(&pollset.in_poll_wait, 0);
-	forget_removed(&pollset);
+	erts_atomic_set_nob(&psi->in_poll_wait, 0);
+	forget_removed(psi);
 	erts_free(ERTS_ALC_T_TMP, pollres);
 	if (poll_ret == EAGAIN) {
 	    goto restart;
@@ -1838,6 +1908,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 
 	erts_mtx_lock(fd_mtx(fd));
 
+
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
 	state = &drv_ev_state.v[ (int) fd];
 #else
@@ -1848,7 +1919,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 #endif
 
 	/* Skip this fd if it was removed from pollset */
-	if (is_removed(state)) {
+	if (is_removed(state) || state->pollset != psi) {
 	    goto next_pollres;
 	}
 
@@ -1884,7 +1955,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 		bad_fd_in_pollset(state,
                                   state->driver.select->inport,
                                   state->driver.select->outport);
-		add_active_fd(state->fd);
+		add_active_fd(psi, state->fd);
 	    }
 	    break;
 	}
@@ -1915,7 +1986,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
                         resource = state->driver.stop.resource;
                         state->driver.nif->out.ddeselect_cnt = ERTS_NIF_DELAYED_DESELECT;
                         state->driver.nif->out.pid = NIL;
-                        add_active_fd(state->fd);
+                        add_active_fd(psi, state->fd);
                     }
                     else {
                         ASSERT(state->driver.nif->out.ddeselect_cnt >= 2);
@@ -1928,7 +1999,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
                         resource = state->driver.stop.resource;
                         state->driver.nif->in.ddeselect_cnt = ERTS_NIF_DELAYED_DESELECT;
                         state->driver.nif->in.pid = NIL;
-                        add_active_fd(state->fd);
+                        add_active_fd(psi, state->fd);
                     }
                     else {
                         ASSERT(state->driver.nif->in.ddeselect_cnt >= 2);
@@ -1938,7 +2009,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
             }
             else if (revents & ERTS_POLL_EV_NVAL) {
                 bad_fd_in_pollset(state, NIL, NIL);
-                add_active_fd(state->fd);
+                add_active_fd(psi, state->fd);
             }
 
             erts_mtx_unlock(fd_mtx(fd));
@@ -1963,7 +2034,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 			  (int) state->type);
 	    ASSERT(0);
 	    deselect(state, 0);
-	    add_active_fd(state->fd);
+	    add_active_fd(psi, state->fd);
 	    break;
 	}
 	}
@@ -1973,9 +2044,9 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
         next_pollres_unlocked:;
     }
 
-    erts_atomic_set_nob(&pollset.in_poll_wait, 0);
+    erts_atomic_set_nob(&psi->in_poll_wait, 0);
     erts_free(ERTS_ALC_T_TMP, pollres);
-    forget_removed(&pollset);
+    forget_removed(psi);
 }
 
 static void
@@ -2096,16 +2167,17 @@ static void drv_ev_state_free(void *des)
 }
 #endif
 
-
 #ifdef ERTS_ENABLE_KERNEL_POLL
 
 struct io_functions {
     int (*select)(ErlDrvPort, ErlDrvEvent, int, int);
     int (*enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
-    void (*check_io_as_interrupt)(void);
-    void (*check_io_interrupt)(int);
-    void (*check_io_interrupt_tmd)(int, ErtsMonotonicTime);
+    void (*check_io_as_interrupt)(struct pollset_info*);
+    void (*check_io_interrupt)(struct pollset_info*, int);
+    void (*check_io_interrupt_tmd)(struct pollset_info*, int, ErtsMonotonicTime);
     void (*check_io)(int);
+    struct pollset_info* (*get_pollset)(int sched_nr);
+    void (*notify_port_task_executed)(ErtsPortTaskHandle *pthp);
     int (*max_files)(void);
     Uint (*size)(void);
     Eterm (*info)(void *);
@@ -2126,12 +2198,11 @@ extern struct io_functions erts_io_funcs;
 void
 ERTS_CIO_EXPORT(erts_init_check_io)(void)
 {
+    int j;
     ERTS_CT_ASSERT((INT_MIN & (ERL_NIF_SELECT_STOP_CALLED |
                                ERL_NIF_SELECT_STOP_SCHEDULED |
                                ERL_NIF_SELECT_INVALID_EVENT |
                                ERL_NIF_SELECT_FAILED)) == 0);
-    erts_atomic_init_nob(&erts_check_io_time, 0);
-    erts_atomic_init_nob(&pollset.in_poll_wait, 0);
 
 #ifdef ERTS_ENABLE_KERNEL_POLL
     ASSERT(erts_io_funcs.select == NULL);
@@ -2140,6 +2211,8 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
     erts_io_funcs.check_io_interrupt	= ERTS_CIO_EXPORT(erts_check_io_interrupt);
     erts_io_funcs.check_io_interrupt_tmd= ERTS_CIO_EXPORT(erts_check_io_interrupt_timed);
     erts_io_funcs.check_io		= ERTS_CIO_EXPORT(erts_check_io);
+    erts_io_funcs.get_pollset		= ERTS_CIO_EXPORT(erts_get_pollset);
+    erts_io_funcs.notify_port_task_executed = ERTS_CIO_EXPORT(erts_io_notify_port_task_executed);
     erts_io_funcs.max_files		= ERTS_CIO_EXPORT(erts_check_io_max_files);
     erts_io_funcs.size		= ERTS_CIO_EXPORT(erts_check_io_size);
     erts_io_funcs.info		= ERTS_CIO_EXPORT(erts_check_io_info);
@@ -2149,25 +2222,34 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
 #endif
 #endif
 
+    init_removed_fd_alloc();
+
     ERTS_CIO_POLL_INIT();
-    pollset.ps = ERTS_CIO_NEW_POLLSET();
-
-    pollset.active_fd.six = 0;
-    pollset.active_fd.eix = 0;
-    erts_atomic32_init_nob(&pollset.active_fd.no, 0);
-    pollset.active_fd.size = ERTS_ACTIVE_FD_INC;
-    pollset.active_fd.array = erts_alloc(ERTS_ALC_T_ACTIVE_FD_ARR,
-					 sizeof(ErtsSysFdType)*ERTS_ACTIVE_FD_INC);
+    pollsetv = erts_alloc(ERTS_ALC_T_POLLSET,
+                               sizeof(struct pollset_info)*erts_no_schedulers);
+    for (j=0; j < erts_no_schedulers; j++) {
+        struct pollset_info* psi = &pollsetv[j];
+
+        erts_atomic_init_nob(&psi->check_io_time, 0);
+        erts_atomic_init_nob(&psi->in_poll_wait, 0);
+        psi->ps = ERTS_CIO_NEW_POLLSET();
+        psi->active_fd.six = 0;
+        psi->active_fd.eix = 0;
+        erts_atomic32_init_nob(&psi->active_fd.no, 0);
+        psi->active_fd.size = ERTS_ACTIVE_FD_INC;
+        psi->active_fd.array = erts_alloc(ERTS_ALC_T_ACTIVE_FD_ARR,
+                                          sizeof(ErtsSysFdType)*ERTS_ACTIVE_FD_INC);
 #ifdef DEBUG
-    {
-	int i;
-	for (i = 0; i < ERTS_ACTIVE_FD_INC; i++)
-	    pollset.active_fd.array[i] = ERTS_SYS_FD_INVALID;
-    }
+        {
+            int i;
+            for (i = 0; i < ERTS_ACTIVE_FD_INC; i++)
+                psi->active_fd.array[i] = ERTS_SYS_FD_INVALID;
+        }
 #endif
 
-    init_removed_fd_alloc();
-    erts_atomic_init_nob(&pollset.removed_list,  (erts_aint_t)NULL);
+        erts_atomic_init_nob(&psi->removed_list,  (erts_aint_t)NULL);
+    }
+
     {
 	int i;
 	for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) {
@@ -2211,10 +2293,14 @@ ERTS_CIO_EXPORT(erts_check_io_max_files)(void)
 Uint
 ERTS_CIO_EXPORT(erts_check_io_size)(void)
 {
-    Uint res;
+    Uint res = 0;
     ErtsPollInfo pi;
-    ERTS_CIO_POLL_INFO(pollset.ps, &pi);
-    res = pi.memory_size;
+    int i;
+
+    for (i = 0; i < erts_no_schedulers; i++) {
+        ERTS_CIO_POLL_INFO(pollsetv[i].ps, &pi);
+        res += pi.memory_size;
+    }
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     res += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state.len);
 #else
@@ -2235,105 +2321,122 @@ Eterm
 ERTS_CIO_EXPORT(erts_check_io_info)(void *proc)
 {
     Process *p = (Process *) proc;
-    Eterm tags[16], values[16], res;
-    Uint sz, *szp, *hp, **hpp, memory_size;
-    Sint i;
-    ErtsPollInfo pi;
-    erts_aint_t cio_time = erts_atomic_read_acqb(&erts_check_io_time);
-    int active_fds = (int) erts_atomic32_read_acqb(&pollset.active_fd.no);
-
-    while (1) {
-	erts_aint_t post_cio_time;
-	int post_active_fds;
-
-	ERTS_CIO_POLL_INFO(pollset.ps, &pi);
-
-	post_cio_time = erts_atomic_read_mb(&erts_check_io_time);
-	post_active_fds = (int) erts_atomic32_read_acqb(&pollset.active_fd.no);
-	if (cio_time == post_cio_time && active_fds == post_active_fds)
-	    break;
-	cio_time = post_cio_time;
-	active_fds = post_active_fds;
-    }
+    Eterm tags[16], values[16], res, list = NIL;
+    Uint sz, *szp, *hp, **hpp;
+    ErtsPollInfo *piv;
+    Sint i, j;
+
+    piv = erts_alloc(ERTS_ALC_T_TMP,
+                     sizeof(ErtsPollInfo) * erts_no_schedulers);
+
+    for (j = 0; j < erts_no_schedulers; j++) {
+        struct pollset_info *psi = &pollsetv[j];
+        erts_aint_t cio_time = erts_atomic_read_acqb(&psi->check_io_time);
+
+        piv[j].active_fds = (int) erts_atomic32_read_acqb(&psi->active_fd.no);
+        while (1) {
+            erts_aint_t post_cio_time;
+            int post_active_fds;
+
+            ERTS_CIO_POLL_INFO(psi->ps, &piv[j]);
+
+            post_cio_time = erts_atomic_read_mb(&psi->check_io_time);
+            post_active_fds = (int) erts_atomic32_read_acqb(&psi->active_fd.no);
+            if (cio_time == post_cio_time && piv[j].active_fds == post_active_fds)
+                break;
+            cio_time = post_cio_time;
+            piv[j].active_fds = post_active_fds;
+        }
 
-    memory_size = pi.memory_size;
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    memory_size += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state.len);
-#else
-    memory_size += safe_hash_table_sz(&drv_ev_state.tab);
-    {
-	SafeHashInfo hi;
-	safe_hash_get_info(&hi, &drv_ev_state.tab);
-	memory_size += hi.objs * sizeof(ErtsDrvEventState);
+    #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
+        piv[j].memory_size += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state.len);
+    #else
+        piv[j].memory_size += safe_hash_table_sz(&drv_ev_state.tab);
+        {
+            SafeHashInfo hi;
+            safe_hash_get_info(&hi, &drv_ev_state.tab);
+            piv[j].memory_size += hi.objs * sizeof(ErtsDrvEventState);
+        }
+        erts_spin_lock(&drv_ev_state.prealloc_lock);
+        piv[j].memory_size += drv_ev_state.num_prealloc * sizeof(ErtsDrvEventState);
+        erts_spin_unlock(&drv_ev_state.prealloc_lock);
+    #endif
     }
-    erts_spin_lock(&drv_ev_state.prealloc_lock);
-    memory_size += drv_ev_state.num_prealloc * sizeof(ErtsDrvEventState);
-    erts_spin_unlock(&drv_ev_state.prealloc_lock);
-#endif
 
     hpp = NULL;
     szp = &sz;
     sz = 0;
 
  bld_it:
-    i = 0;
 
-    tags[i] = erts_bld_atom(hpp, szp, "name");
-    values[i++] = erts_bld_atom(hpp, szp, "erts_poll");
+    for (j = erts_no_schedulers-1; j >= 0; j--) {
+        i = 0;
 
-    tags[i] = erts_bld_atom(hpp, szp, "primary");
-    values[i++] = erts_bld_atom(hpp, szp, pi.primary);
+        tags[i] = erts_bld_atom(hpp, szp, "name");
+        values[i++] = erts_bld_atom(hpp, szp, "erts_poll");
 
-    tags[i] = erts_bld_atom(hpp, szp, "fallback");
-    values[i++] = erts_bld_atom(hpp, szp, pi.fallback ? pi.fallback : "false");
+        tags[i] = erts_bld_atom(hpp, szp, "primary");
+        values[i++] = erts_bld_atom(hpp, szp, piv[j].primary);
 
-    tags[i] = erts_bld_atom(hpp, szp, "kernel_poll");
-    values[i++] = erts_bld_atom(hpp, szp,
-				pi.kernel_poll ? pi.kernel_poll : "false");
+        tags[i] = erts_bld_atom(hpp, szp, "fallback");
+        values[i++] = erts_bld_atom(hpp, szp, piv[j].fallback ? piv[j].fallback : "false");
 
-    tags[i] = erts_bld_atom(hpp, szp, "memory_size");
-    values[i++] = erts_bld_uint(hpp, szp, memory_size);
+        tags[i] = erts_bld_atom(hpp, szp, "kernel_poll");
+        values[i++] = erts_bld_atom(hpp, szp,
+                                    piv[j].kernel_poll ? piv[j].kernel_poll : "false");
 
-    tags[i] = erts_bld_atom(hpp, szp, "total_poll_set_size");
-    values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.poll_set_size);
+        tags[i] = erts_bld_atom(hpp, szp, "memory_size");
+        values[i++] = erts_bld_uint(hpp, szp, piv[j].memory_size);
 
-    if (pi.fallback) {
-	tags[i] = erts_bld_atom(hpp, szp, "fallback_poll_set_size");
-	values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.fallback_poll_set_size);
-    }
+        tags[i] = erts_bld_atom(hpp, szp, "total_poll_set_size");
+        values[i++] = erts_bld_uint(hpp, szp, (Uint) piv[j].poll_set_size);
+
+        if (piv[j].fallback) {
+            tags[i] = erts_bld_atom(hpp, szp, "fallback_poll_set_size");
+            values[i++] = erts_bld_uint(hpp, szp, (Uint) piv[j].fallback_poll_set_size);
+        }
 
-    tags[i] = erts_bld_atom(hpp, szp, "lazy_updates");
-    values[i++] = pi.lazy_updates ? am_true : am_false;
+        tags[i] = erts_bld_atom(hpp, szp, "lazy_updates");
+        values[i++] = piv[j].lazy_updates ? am_true : am_false;
 
-    if (pi.lazy_updates) {
-	tags[i] = erts_bld_atom(hpp, szp, "pending_updates");
-	values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.pending_updates);
-    }
+        if (piv[j].lazy_updates) {
+            tags[i] = erts_bld_atom(hpp, szp, "pending_updates");
+            values[i++] = erts_bld_uint(hpp, szp, (Uint) piv[j].pending_updates);
+        }
 
-    tags[i] = erts_bld_atom(hpp, szp, "batch_updates");
-    values[i++] = pi.batch_updates ? am_true : am_false;
+        tags[i] = erts_bld_atom(hpp, szp, "batch_updates");
+        values[i++] = piv[j].batch_updates ? am_true : am_false;
 
-    tags[i] = erts_bld_atom(hpp, szp, "concurrent_updates");
-    values[i++] = pi.concurrent_updates ? am_true : am_false;
+        tags[i] = erts_bld_atom(hpp, szp, "concurrent_updates");
+        values[i++] = piv[j].concurrent_updates ? am_true : am_false;
 
-    tags[i] = erts_bld_atom(hpp, szp, "max_fds");
-    values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.max_fds);
+        tags[i] = erts_bld_atom(hpp, szp, "max_fds");
+        values[i++] = erts_bld_uint(hpp, szp, (Uint) piv[j].max_fds);
 
-    tags[i] = erts_bld_atom(hpp, szp, "active_fds");
-    values[i++] = erts_bld_uint(hpp, szp, (Uint) active_fds);
+        tags[i] = erts_bld_atom(hpp, szp, "active_fds");
+        values[i++] = erts_bld_uint(hpp, szp, (Uint) piv[j].active_fds);
 
-#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
-    tags[i] = erts_bld_atom(hpp, szp, "no_avoided_wakeups");
-    values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.no_avoided_wakeups);
+    #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
+        tags[i] = erts_bld_atom(hpp, szp, "no_avoided_wakeups");
+        values[i++] = erts_bld_uint(hpp, szp, (Uint) piv[j].no_avoided_wakeups);
 
-    tags[i] = erts_bld_atom(hpp, szp, "no_avoided_interrupts");
-    values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.no_avoided_interrupts);
+        tags[i] = erts_bld_atom(hpp, szp, "no_avoided_interrupts");
+        values[i++] = erts_bld_uint(hpp, szp, (Uint) piv[j].no_avoided_interrupts);
 
-    tags[i] = erts_bld_atom(hpp, szp, "no_interrupt_timed");
-    values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.no_interrupt_timed);
-#endif
+        tags[i] = erts_bld_atom(hpp, szp, "no_interrupt_timed");
+        values[i++] = erts_bld_uint(hpp, szp, (Uint) piv[j].no_interrupt_timed);
+    #endif
 
-    res = erts_bld_2tup_list(hpp, szp, i, tags, values);
+        res = erts_bld_2tup_list(hpp, szp, i, tags, values);
+
+        if (!hpp) {
+            *szp += 2;
+        }
+        else {
+            list = CONS(*hpp, res, list);
+            *hpp += 2;
+        }
+    }
 
     if (!hpp) {
 	hp = HAlloc(p, sz);
@@ -2342,7 +2445,9 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc)
 	goto bld_it;
     }
 
-    return res;
+    erts_free(ERTS_ALC_T_TMP, piv);
+
+    return list;
 }
 
 static ERTS_INLINE ErtsPollEvents
@@ -2373,6 +2478,23 @@ print_events(ErtsPollEvents ev)
     return ev;
 }
 
+static ERTS_INLINE void
+print_flags(EventStateFlags f)
+{
+    const char* delim = "";
+    if(f & ERTS_EV_FLAG_USED) {
+	erts_printf("%s","USED");
+	delim = "|";
+    }
+    if(f & ERTS_EV_FLAG_DEFER_IN_EV) {
+	erts_printf("%s%s", delim, "DRIN");
+        delim = "|";
+    }
+    if(f & ERTS_EV_FLAG_DEFER_OUT_EV) {
+        erts_printf("%s%s", delim, "DROUT");
+    }
+}
+
 typedef struct {
     int used_fds;
     int num_errors;
@@ -2419,7 +2541,8 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters)
 	counters->used_fds++;
 #endif
 	
-	erts_printf("fd=%d ", (int) fd);
+	erts_printf("pollset=%d fd=%d ",
+                    (int)(state->pollset - pollsetv), (int) fd);
 	
 #if defined(HAVE_FSTAT) && !defined(NO_FSTAT_ON_SYS_FD_TYPE)
 	if (fstat((int) fd, &stat_buf) < 0)
@@ -2497,7 +2620,15 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters)
 		    err = 1;
 	    }
 	    else {
-		err = 1;
+                ErtsPollEvents ev = cio_events;
+#if ERTS_CIO_DEFER_ACTIVE_EVENTS
+                if (state->flags & ERTS_EV_FLAG_DEFER_IN_EV)
+                    ev &= ~ERTS_POLL_EV_IN;
+                if (state->flags & ERTS_EV_FLAG_DEFER_OUT_EV)
+                    ev &= ~ERTS_POLL_EV_OUT;
+#endif
+                if (ev != ep_events)
+                    err = 1;
 		erts_printf("cio_ev=");
 		print_events(cio_events);
 		erts_printf(" ep_ev=");
@@ -2607,6 +2738,8 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters)
 #endif
 	}
 	
+        erts_printf(" flags="); print_flags(state->flags);
+
 	if (err) {
 	    counters->num_errors++;
 	    erts_printf(" ERROR");
@@ -2619,18 +2752,23 @@ int
 ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
 {
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    int fd, len;
+    int fd, len, i;
 #endif
-    IterDebugCounters counters;
+    IterDebugCounters counters = {0};
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     ErtsDrvEventState null_des;
 
+    null_des.pollset = NULL;
     null_des.driver.select = NULL;
     null_des.driver.nif = NULL;
     null_des.driver.stop.drv_ptr = NULL;
     null_des.events = 0;
     null_des.remove_cnt = 0;
     null_des.type = ERTS_EV_TYPE_NONE;
+    null_des.flags = 0;
+
+    counters.epep = erts_alloc(ERTS_ALC_T_TMP,
+                               sizeof(ErtsPollEvents)*drv_ev_state.max_fds);
 #endif
 
     erts_printf("--- fds in pollset --------------------------------------\n");
@@ -2641,29 +2779,26 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
 
     erts_thr_progress_block(); /* stop the world to avoid messy locking */
 
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    counters.epep = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollEvents)*drv_ev_state.max_fds);
-    ERTS_POLL_EXPORT(erts_poll_get_selected_events)(pollset.ps, counters.epep, drv_ev_state.max_fds);
-    counters.internal_fds = 0;
-#endif
-    counters.used_fds = 0;
-    counters.num_errors = 0;
-    counters.no_driver_select_structs = 0;
-    counters.no_enif_select_structs = 0;
-
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     len = erts_atomic_read_nob(&drv_ev_state.len);
-    for (fd = 0; fd < len; fd++) {
-	doit_erts_check_io_debug((void *) &drv_ev_state.v[fd], (void *) &counters);
+
+    for (i = 0; i < erts_no_schedulers; i++) {
+        ERTS_POLL_EXPORT(erts_poll_get_selected_events)(pollsetv[i].ps,
+                                                        counters.epep,
+                                                        drv_ev_state.max_fds);
+        for (fd = 0; fd < len; fd++) {
+            if (drv_ev_state.v[fd].pollset == &pollsetv[i])
+                doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters);
+        }
     }
-    for ( ; fd < drv_ev_state.max_fds; fd++) {
-	null_des.fd = fd;
-	doit_erts_check_io_debug((void *) &null_des, (void *) &counters);
+    for (fd = len ; fd < drv_ev_state.max_fds; fd++) {
+        null_des.fd = fd;
+        doit_erts_check_io_debug(&null_des, &counters);
     }
 #else
-    safe_hash_for_each(&drv_ev_state.tab, &doit_erts_check_io_debug, (void *) &counters);
+    safe_hash_for_each(&drv_ev_state.tab, &doit_erts_check_io_debug,
+                       &counters);
 #endif
-
     erts_thr_progress_unblock();
 
     ciodip->no_used_fds = counters.used_fds;
@@ -2696,7 +2831,6 @@ void ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks)(int enable) {
 }
 #endif /* ERTS_ENABLE_LOCK_COUNT */
 
-
 #ifdef ERTS_ENABLE_KERNEL_POLL
 # ifdef ERTS_KERNEL_POLL_VERSION
 
@@ -2724,6 +2858,16 @@ int enif_select(ErlNifEnv* env, ErlNifEvent event,
     return (*erts_io_funcs.enif_select)(env, event, flags, obj, pid, ref);
 }
 
+struct pollset_info* erts_get_pollset(int sched_nr)
+{
+    return (*erts_io_funcs.get_pollset)(sched_nr);
+}
+
+void erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp)
+{
+    erts_io_funcs.notify_port_task_executed(pthp);
+}
+
 void erts_check_io(int do_wait)
 {
     erts_io_funcs.check_io(do_wait);
@@ -2751,15 +2895,15 @@ erts_check_io_debug(ErtsCheckIoDebugInfo *ip)
     return (*erts_io_funcs.check_io_debug)(ip);
 }
 
-void erts_check_io_interrupt(int set)
+void erts_check_io_interrupt(struct pollset_info* psi, int set)
 {
-    erts_io_funcs.check_io_interrupt(set);
+    erts_io_funcs.check_io_interrupt(psi, set);
 }
 
-void erts_check_io_interrupt_timed(int set,
+void erts_check_io_interrupt_timed(struct pollset_info* psi, int set,
                                    ErtsMonotonicTime timeout_time)
 {
-    erts_io_funcs.check_io_interrupt_tmd(set, timeout_time);
+    erts_io_funcs.check_io_interrupt_tmd(psi, set, timeout_time);
 }
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index f4d7983002..ab53d91756 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -30,38 +30,29 @@
 #include "sys.h"
 #include "erl_sys_driver.h"
 
+struct pollset_info;
+
 Uint erts_check_io_size(void);
 Eterm erts_check_io_info(void *);
+void erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp);
+void erts_check_io_async_sig_interrupt(struct pollset_info *psi);
 int erts_check_io_max_files(void);
-void erts_check_io_interrupt(int);
-void erts_check_io_interrupt_timed(int, ErtsMonotonicTime);
 void erts_check_io(int);
 void erts_init_check_io(void);
 #ifdef ERTS_ENABLE_LOCK_COUNT
 void erts_lcnt_update_cio_locks(int enable);
 #endif
 
-extern erts_atomic_t erts_check_io_time;
+void erts_check_io_interrupt(struct pollset_info*, int);
+void erts_check_io_interrupt_timed(struct pollset_info*, int, ErtsMonotonicTime);
+struct pollset_info* erts_get_pollset(int sched_num);
 
 typedef struct {
     ErtsPortTaskHandle task;
     erts_atomic_t executed_time;
+    struct pollset_info *pollset;
 } ErtsIoTask;
 
-ERTS_GLB_INLINE void erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp);
-
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-
-ERTS_GLB_INLINE void
-erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp)
-{
-    ErtsIoTask *itp = (ErtsIoTask *) (((char *) pthp) - offsetof(ErtsIoTask, task));
-    erts_aint_t ci_time = erts_atomic_read_acqb(&erts_check_io_time);
-    erts_atomic_set_relb(&itp->executed_time, ci_time);
-}
-
-#endif
-
 #endif /*  ERL_CHECK_IO_H__ */
 
 #if !defined(ERL_CHECK_IO_C__) && !defined(ERTS_ALLOC_C__)
@@ -80,7 +71,7 @@ erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp)
  */
 #  define ERTS_CIO_DEFER_ACTIVE_EVENTS 1
 #else
-#  define ERTS_CIO_DEFER_ACTIVE_EVENTS 0
+#  define ERTS_CIO_DEFER_ACTIVE_EVENTS 1
 #endif
 
 typedef struct {
diff --git a/erts/emulator/sys/common/erl_poll.h b/erts/emulator/sys/common/erl_poll.h
index 12dfc66e51..6c961205fe 100644
--- a/erts/emulator/sys/common/erl_poll.h
+++ b/erts/emulator/sys/common/erl_poll.h
@@ -225,6 +225,7 @@ typedef struct {
     long no_avoided_interrupts;
     long no_interrupt_timed;
 #endif
+    int active_fds;
 } ErtsPollInfo;
 
 void		ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet,
diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c
index 09237c81ce..420138ff0a 100644
--- a/erts/emulator/sys/common/erl_sys_common_misc.c
+++ b/erts/emulator/sys/common/erl_sys_common_misc.c
@@ -45,14 +45,6 @@
 #endif
 #endif
 
-/*
- * erts_check_io_time is used by the erl_check_io implementation. The
- * global erts_check_io_time variable is declared here since there
- * (often) exist two versions of erl_check_io (kernel-poll and
- * non-kernel-poll), and we dont want two versions of this variable.
- */
-erts_atomic_t erts_check_io_time;
-
 /* Written once and only once */
 
 static int filename_encoding = ERL_FILENAME_UNKNOWN;
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 09c515291a..acd7920e86 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -499,6 +499,7 @@ static void signal_notify_requested(Eterm type) {
 static ERTS_INLINE void
 break_requested(void)
 {
+    int i;
   /*
    * just set a flag - checked for and handled by
    * scheduler threads erts_check_io() (not signal handler).
@@ -510,7 +511,10 @@ break_requested(void)
       erts_exit(ERTS_INTR_EXIT, "");
 
   ERTS_SET_BREAK_REQUESTED;
-  erts_check_io_interrupt(1);
+  for (i=0; i < erts_no_schedulers; i++) {
+       /* Make sure we don't sleep in poll */
+      erts_check_io_interrupt(ERTS_SCHEDULER_IX(i)->pollset, 1);
+  }
 }
 
 static RETSIGTYPE request_break(int signum)
diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c
index 7c9a532fed..0228e1af54 100644
--- a/erts/emulator/sys/unix/sys_drivers.c
+++ b/erts/emulator/sys/unix/sys_drivers.c
@@ -1723,8 +1723,6 @@ static ErlDrvData forker_start(ErlDrvPort port_num, char* name,
 
     SET_NONBLOCKING(forker_fd);
 
-    driver_select(port_num, forker_fd, ERL_DRV_READ|ERL_DRV_USE, 1);
-
     return (ErlDrvData)port_num;
 }
 
@@ -1821,10 +1819,19 @@ static void forker_ready_output(ErlDrvData e, ErlDrvEvent fd)
 static ErlDrvSSizeT forker_control(ErlDrvData e, unsigned int cmd, char *buf,
                                    ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen)
 {
+    static int first_call = 1;
     ErtsSysForkerProto *proto = (ErtsSysForkerProto *)buf;
     ErlDrvPort port_num = (ErlDrvPort)e;
     int res;
 
+    if (first_call) {
+        /*
+         * Do driver_select here when schedulers and their pollsets have started.
+         */
+        driver_select(port_num, forker_fd, ERL_DRV_READ|ERL_DRV_USE, 1);
+        first_call = 0;
+    }
+
     driver_enq(port_num, buf, len);
     if (driver_sizeq(port_num) > sizeof(*proto)) {
         return 0;
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index c6d7f708be..332d11345b 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -83,6 +83,8 @@
 
 -export([bin_prefix/2]).
 
+-export([get_check_io_total/1]).   % for z_SUITE.erl
+
 -include_lib("common_test/include/ct.hrl").
 
 
@@ -783,7 +785,7 @@ io_ready_exit(Config) when is_list(Config) ->
 
 use_fallback_pollset(Config) when is_list(Config) ->
     FlbkFun = fun () ->
-                      ChkIoDuring = erlang:system_info(check_io),
+                      ChkIoDuring = get_check_io_total(erlang:system_info(check_io)),
                       case lists:keysearch(fallback_poll_set_size,
                                            1,
                                            ChkIoDuring) of
@@ -939,7 +941,7 @@ chkio_test({erts_poll_info, Before},
         "ok" ->
             chk_chkio_port(Port),
             Fun(),
-            During = erlang:system_info(check_io),
+            During = get_check_io_total(erlang:system_info(check_io)),
             erlang:display(During),
             0 = element(1, erts_debug:get_internal_state(check_io_debug)),
             io:format("During test: ~p~n", [During]),
@@ -992,14 +994,14 @@ verify_chkio_state(Before, After) ->
     ok.
 
 get_stable_check_io_info() ->
-    ChkIo = erlang:system_info(check_io),
-    PendUpdNo = case lists:keysearch(pending_updates, 1, ChkIo) of
-                    {value, {pending_updates, PendNo}} ->
-                        PendNo;
+    ChkIo = get_check_io_total(erlang:system_info(check_io)),
+    PendUpdNo = case lists:keyfind(pending_updates, 1, ChkIo) of
+                    {pending_updates, Value} ->
+                        Value;
                     false ->
                         0
                 end,
-    {value, {active_fds, ActFds}} = lists:keysearch(active_fds, 1, ChkIo),
+    {active_fds, ActFds} = lists:keyfind(active_fds, 1, ChkIo),
     case {PendUpdNo, ActFds} of
         {0, 0} ->
             ChkIo;
@@ -1008,6 +1010,46 @@ get_stable_check_io_info() ->
             get_stable_check_io_info()
     end.
 
+%% Merge return from erlang:system_info(check_io)
+%% as if it was one big pollset.
+get_check_io_total(ChkIo) ->
+    lists:foldl(fun(Pollset, Acc) ->
+                        lists:zipwith(fun(A, B) ->
+                                              add_pollset_infos(A,B)
+                                      end,
+                                      Pollset, Acc)
+                end,
+                hd(ChkIo),
+                tl(ChkIo)).
+
+add_pollset_infos({Tag, A}=TA , {Tag, B}=TB) ->
+    case tag_type(Tag) of
+        sum ->
+            {Tag, A + B};
+        const ->
+            case A of
+                B -> TA;
+                _ ->
+                    ct:fail("Unexpected diff in pollsets ~p != ~p",
+                            [TA,TB])
+            end
+    end.
+
+tag_type(name) -> const;
+tag_type(primary) -> const;
+tag_type(fallback) -> const;
+tag_type(kernel_poll) -> const;
+tag_type(memory_size) -> sum;
+tag_type(total_poll_set_size) -> sum;
+tag_type(fallback_poll_set_size) -> sum;
+tag_type(lazy_updates) -> const;
+tag_type(pending_updates) -> sum;
+tag_type(batch_updates) -> const;
+tag_type(concurrent_updates) -> const;
+tag_type(max_fds) -> const;
+tag_type(active_fds) -> sum.
+
+
 %% Missed port lock when stealing control of fd from a
 %% driver that didn't use the same lock. The lock checker
 %% used to trigger on this and dump core.
diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl
index feea7432a9..ac3df8bfbf 100644
--- a/erts/emulator/test/z_SUITE.erl
+++ b/erts/emulator/test/z_SUITE.erl
@@ -330,7 +330,7 @@ display_check_io(ChkIo) ->
     ok.
 
 get_check_io_info() ->
-    ChkIo = erlang:system_info(check_io),
+    ChkIo = driver_SUITE:get_check_io_total(erlang:system_info(check_io)),
     PendUpdNo = case lists:keysearch(pending_updates, 1, ChkIo) of
 		    {value, {pending_updates, PendNo}} ->
 			PendNo;
-- 
cgit v1.2.3


From ae02bac9a23bd2d9d8502e0a35f1db5cd6394da8 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 28 Apr 2017 09:41:55 +0200
Subject: erts: Update suspend of scheduler to handle multiple pollsets

---
 erts/emulator/beam/erl_process.c | 191 +++++++++++++++++++--------------------
 1 file changed, 95 insertions(+), 96 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 2a34410d5b..624b907fb6 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -7024,18 +7024,24 @@ sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount)
 }
 
 static erts_aint32_t
-sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi)
+sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi,
+                              erts_aint32_t sleep_type)
 {
     erts_aint32_t oflgs;
-    erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING
-			   | ERTS_SSI_FLG_TSE_SLEEPING
-			   | ERTS_SSI_FLG_WAITING
-			   | ERTS_SSI_FLG_SUSPENDED);
+    erts_aint32_t nflgs = ((ERTS_SSI_FLG_SLEEPING
+                            | ERTS_SSI_FLG_WAITING
+                            | ERTS_SSI_FLG_SUSPENDED)
+                           | sleep_type);
     erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING
 			   | ERTS_SSI_FLG_WAITING
 			   | ERTS_SSI_FLG_SUSPENDED);
 
-    erts_tse_reset(ssi->event);
+    if (sleep_type == ERTS_SSI_FLG_TSE_SLEEPING)
+	erts_tse_reset(ssi->event);
+    else {
+	ASSERT(sleep_type == ERTS_SSI_FLG_POLL_SLEEPING);
+	erts_check_io_interrupt(ssi->esdp->pollset, 0);
+    }
 
     while (1) {
 	oflgs = erts_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
@@ -7371,6 +7377,61 @@ msb_scheduler_type_switch(ErtsSchedType sched_type,
 
 }
 
+static ERTS_INLINE void
+suspend_dirty_scheduler_sleep(ErtsSchedulerData *esdp)
+{
+    ErtsSchedulerSleepInfo *ssi = esdp->ssi;
+    erts_aint32_t flgs = sched_spin_suspended(ssi,
+                                              ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT);
+    if (flgs == (ERTS_SSI_FLG_SLEEPING
+                 | ERTS_SSI_FLG_WAITING
+                 | ERTS_SSI_FLG_SUSPENDED)) {
+        flgs = sched_set_suspended_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
+        if (flgs == (ERTS_SSI_FLG_SLEEPING
+                     | ERTS_SSI_FLG_TSE_SLEEPING
+                     | ERTS_SSI_FLG_WAITING
+                     | ERTS_SSI_FLG_SUSPENDED)) {
+            int res;
+
+            do {
+                res = erts_tse_wait(ssi->event);
+            } while (res == EINTR);
+        }
+    }
+}
+
+static ERTS_INLINE void
+suspend_normal_scheduler_sleep(ErtsSchedulerData *esdp)
+{
+    ErtsSchedulerSleepInfo *ssi = esdp->ssi;
+    erts_aint32_t flgs;
+    int spincount;
+
+    spincount = sched_busy_wait.sys_schedule;
+
+    while (spincount-- > 0) {
+
+        erl_sys_schedule(1);
+
+        flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+        if (flgs != (ERTS_SSI_FLG_SLEEPING
+                     | ERTS_SSI_FLG_WAITING
+                     | ERTS_SSI_FLG_SUSPENDED)) {
+            return;
+        }
+    }
+
+    flgs = sched_set_suspended_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING);
+    if (flgs != (ERTS_SSI_FLG_SLEEPING
+                 | ERTS_SSI_FLG_POLL_SLEEPING
+                 | ERTS_SSI_FLG_WAITING
+                 | ERTS_SSI_FLG_SUSPENDED)) {
+        return;
+    }
+
+    /* Sleep on pollset... */
+    erl_sys_schedule(0);
+}
 
 static void
 suspend_scheduler(ErtsSchedulerData *esdp)
@@ -7559,14 +7620,15 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 	    schdlr_sspnd_resume_procs(sched_type, &resume);
 
 	    while (1) {
-		ErtsMonotonicTime current_time;
-		erts_aint32_t flgs;
-
 		if (sched_type != ERTS_SCHED_NORMAL)
-		    aux_work = 0;
-		else {
+                    suspend_dirty_scheduler_sleep(esdp);
+		else
+                {
+                    ErtsMonotonicTime current_time, timeout_time;
                     int evacuate = no == 1 ? 0 : !ERTS_EMPTY_RUNQ(esdp->run_queue);
 
+		    ASSERT(sched_type == ERTS_SCHED_NORMAL);
+
 		    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 
 		    if (aux_work|evacuate) {
@@ -7588,95 +7650,32 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 			}
 		    }
 
-		}
 
-		if (aux_work) {
-		    ASSERT(sched_type == ERTS_SCHED_NORMAL);
-		    current_time = erts_get_monotonic_time(esdp);
-		    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
-			if (!thr_prgr_active) {
-			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
-			    sched_wall_time_change(esdp, 1);
-			}
-			erts_bump_timers(esdp->timer_wheel, current_time);
-		    }
-		}
-		else {
-		    ErtsMonotonicTime timeout_time;
-		    int do_timeout;
+                    if (aux_work)
+                        timeout_time = erts_next_timeout_time(esdp->next_tmo_ref);
+                    else
+                        timeout_time = erts_check_next_timeout_time(esdp);
 
-		    if (sched_type == ERTS_SCHED_NORMAL) {
-			timeout_time = erts_check_next_timeout_time(esdp);
-			current_time = erts_get_monotonic_time(esdp);
-			do_timeout = (current_time >= timeout_time);
-		    }
-		    else {
-			timeout_time = ERTS_MONOTONIC_TIME_MAX;
-			current_time = 0;
-			do_timeout = 0;
-		    }
+                    current_time = erts_get_monotonic_time(esdp);
 
-		    if (do_timeout) {
-			ASSERT(sched_type == ERTS_SCHED_NORMAL);
-			if (!thr_prgr_active) {
-			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
-			    sched_wall_time_change(esdp, 1);
-			}
-		    }
-		    else {
-			if (sched_type == ERTS_SCHED_NORMAL) {
-			    if (thr_prgr_active) {
-				erts_thr_progress_active(esdp, thr_prgr_active = 0);
-				sched_wall_time_change(esdp, 0);
-			    }
-			    erts_thr_progress_prepare_wait(esdp);
-			}
-			flgs = sched_spin_suspended(ssi,
-						    ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT);
-			if (flgs == (ERTS_SSI_FLG_SLEEPING
-				     | ERTS_SSI_FLG_WAITING
-				     | ERTS_SSI_FLG_SUSPENDED)) {
-			    flgs = sched_set_suspended_sleeptype(ssi);
-			    if (flgs == (ERTS_SSI_FLG_SLEEPING
-					 | ERTS_SSI_FLG_TSE_SLEEPING
-					 | ERTS_SSI_FLG_WAITING
-					 | ERTS_SSI_FLG_SUSPENDED)) {
-				int res;
-
-				if (sched_type == ERTS_SCHED_NORMAL)
-				    current_time = erts_get_monotonic_time(esdp);
-				else
-				    current_time = 0;
-
-				do {
-				    Sint64 timeout;
-				    if (current_time >= timeout_time)
-					break;
-				    if (sched_type != ERTS_SCHED_NORMAL)
-					timeout = -1;
-				    else
-					timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
-									 - current_time
-									 - 1) + 1;
-				    res = erts_tse_twait(ssi->event, timeout);
-
-				    if (sched_type == ERTS_SCHED_NORMAL)
-					current_time = erts_get_monotonic_time(esdp);
-				    else
-					current_time = 0;
-
-				} while (res == EINTR);
-			    }
-			}
-			if (sched_type == ERTS_SCHED_NORMAL)
-			    erts_thr_progress_finalize_wait(esdp);
-		    }
+                    if (!aux_work && current_time < timeout_time) {
+                        /* go to sleep... */
+                        if (thr_prgr_active) {
+                            erts_thr_progress_active(esdp, thr_prgr_active = 0);
+                            sched_wall_time_change(esdp, 0);
+                        }
+                        suspend_normal_scheduler_sleep(esdp);
+                        current_time = erts_get_monotonic_time(esdp);;
+                    }
 
-		    if (current_time >= timeout_time) {
-			ASSERT(sched_type == ERTS_SCHED_NORMAL);
-			erts_bump_timers(esdp->timer_wheel, current_time);
-		    }
-		}
+                    if (current_time >= timeout_time) {
+                        if (!thr_prgr_active) {
+                            erts_thr_progress_active(esdp, thr_prgr_active = 1);
+                            sched_wall_time_change(esdp, 1);
+                        }
+                        erts_bump_timers(esdp->timer_wheel, current_time);
+                    }
+                }
 
 		flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING
 						       | ERTS_SSI_FLG_SUSPENDED));
-- 
cgit v1.2.3


From af8380c319c01cfbf6262c3356462823b9e88c1c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 28 Jun 2017 20:00:09 +0200
Subject: erts: Add ERTS_THR_PREF_QUICK_ALLOC_IMPL

usable from any (managed?) thread.
---
 erts/emulator/beam/erl_alloc.h                | 80 ++++++++++++++++++++++++++-
 erts/emulator/beam/erl_sched_spec_pre_alloc.c | 30 +++++++---
 erts/emulator/beam/erl_sched_spec_pre_alloc.h | 13 ++++-
 3 files changed, 111 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index c661d0b226..117f96a4ad 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -402,6 +402,32 @@ NAME##_free(TYPE *p)							\
 	erts_free(ALCT, (void *) p);					\
 }
 
+#define ERTS_THR_PREF_AUX(NAME, TYPE, PASZ)				\
+ERTS_THR_PREF_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ)
+
+#define ERTS_THR_PREF_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT)	        \
+ERTS_THR_PREF_AUX(NAME, TYPE, PASZ)					\
+static void								\
+init_##NAME##_alloc(int nthreads)					\
+{									\
+    init_##NAME##_pre_alloc(nthreads);			                \
+}									\
+static ERTS_INLINE TYPE *						\
+NAME##_alloc(void)							\
+{									\
+    TYPE *res = NAME##_pre_alloc();					\
+    if (!res)								\
+	res = erts_alloc(ALCT, sizeof(TYPE));				\
+    return res;								\
+}									\
+static ERTS_INLINE void							\
+NAME##_free(TYPE *p)							\
+{									\
+    if (!NAME##_pre_free(p))						\
+	erts_free(ALCT, (void *) p);					\
+}
+
+
 #ifdef DEBUG
 #define ERTS_PRE_ALLOC_SIZE(SZ) ((SZ) < 1000 ? (SZ)/10 + 10 : 100)
 #define ERTS_PRE_ALLOC_CLOBBER(P, T) memset((void *) (P), 0xfd, sizeof(T))
@@ -482,7 +508,8 @@ init_##NAME##_alloc(void)						\
 {									\
     sspa_data_##NAME##__ =						\
 	erts_sspa_create(sizeof(union erts_sspa_##NAME##__),		\
-			 ERTS_PRE_ALLOC_SIZE((PASZ)));			\
+			 ERTS_PRE_ALLOC_SIZE((PASZ)), 			\
+                         0, NULL);                                      \
 }									\
 									\
 static TYPE *								\
@@ -504,6 +531,57 @@ NAME##_free(TYPE *p)							\
 			  (char *) p);					\
 }
 
+
+#define ERTS_THR_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ)		        \
+union erts_sspa_##NAME##__ {						\
+    erts_sspa_blk_t next;						\
+    TYPE type;								\
+};									\
+									\
+static erts_sspa_data_t *sspa_data_##NAME##__;				\
+									\
+static void								\
+init_##NAME##_alloc(int nthreads)					\
+{									\
+    sspa_data_##NAME##__ =						\
+	erts_sspa_create(sizeof(union erts_sspa_##NAME##__),		\
+			 ERTS_PRE_ALLOC_SIZE((PASZ)),			\
+                         nthreads,                                      \
+                         #NAME);                                        \
+}									\
+                                                                        \
+void								        \
+erts_##NAME##_alloc_init_thread(void)				        \
+{									\
+    int id = erts_atomic_inc_read_nob(&sspa_data_##NAME##__->id_generator);\
+    if (id > sspa_data_##NAME##__->nthreads) {                          \
+        erts_exit(ERTS_ABORT_EXIT,                                      \
+                  "%s:%d:%s(): Too many threads for '" #NAME "'\n",     \
+                  __FILE__, __LINE__, __func__);                        \
+    }                                                                   \
+    erts_tsd_set(sspa_data_##NAME##__->tsd_key, (void*)(SWord)id);      \
+}									\
+									\
+static TYPE *								\
+NAME##_alloc(void)							\
+{									\
+    int id = (int)(SWord)erts_tsd_get(sspa_data_##NAME##__->tsd_key);   \
+    if (id == 0)                                                        \
+        return NULL;                                                    \
+    return (TYPE *) erts_sspa_alloc(sspa_data_##NAME##__,		\
+                                    id-1);		                \
+}									\
+									\
+static int								\
+NAME##_free(TYPE *p)							\
+{									\
+    int id = (int)(SWord)erts_tsd_get(sspa_data_##NAME##__->tsd_key);   \
+    return erts_sspa_free(sspa_data_##NAME##__,				\
+			  id - 1,		                        \
+			  (char *) p);					\
+}
+
+
 #ifdef DEBUG
 #define ERTS_ALC_DBG_BLK_SZ(PTR) (*(((UWord *) (PTR)) - 2))
 #endif /* #ifdef DEBUG */
diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
index 6cb7ccab8d..ab204303d7 100644
--- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
@@ -37,7 +37,7 @@
 #include "erl_thr_progress.h"
 
 erts_sspa_data_t *
-erts_sspa_create(size_t blk_sz, int pa_size)
+erts_sspa_create(size_t blk_sz, int pa_size, int nthreads, const char* name)
 {
     erts_sspa_data_t *data;
     size_t tot_size;
@@ -48,22 +48,30 @@ erts_sspa_create(size_t blk_sz, int pa_size)
     int no_blocks = pa_size;
     int no_blocks_per_chunk;
 
-    if (erts_no_schedulers == 1)
+    if (!name) { /* schedulers only variant */
+        ASSERT(!nthreads);
+        nthreads = erts_no_schedulers;
+    }
+    else {
+        ASSERT(nthreads > 0);
+    }
+
+    if (nthreads == 1)
 	no_blocks_per_chunk = no_blocks;
     else {
 	int extra = (no_blocks - 1)/4 + 1;
 	if (extra == 0)
 	    extra = 1;
 	no_blocks_per_chunk = no_blocks;
-	no_blocks_per_chunk += extra*erts_no_schedulers;
-	no_blocks_per_chunk /= erts_no_schedulers;
+	no_blocks_per_chunk += extra * nthreads;
+	no_blocks_per_chunk /= nthreads;
     }
-    no_blocks = no_blocks_per_chunk * erts_no_schedulers;
+    no_blocks = no_blocks_per_chunk * nthreads;
     chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_chunk_header_t));
     chunk_mem_size += blk_sz * no_blocks_per_chunk;
     chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size);
     tot_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t));
-    tot_size += chunk_mem_size*erts_no_schedulers;
+    tot_size += chunk_mem_size * nthreads;
 
     p = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_PRE_ALLOC_DATA, tot_size);
     data = (erts_sspa_data_t *) p;
@@ -72,10 +80,16 @@ erts_sspa_create(size_t blk_sz, int pa_size)
 
     data->chunks_mem_size = chunk_mem_size;
     data->start = chunk_start;
-    data->end = chunk_start + chunk_mem_size*erts_no_schedulers;
+    data->end = chunk_start + chunk_mem_size * nthreads;
+    data->nthreads = nthreads;
+
+    if (name) { /* thread variant */
+        erts_tsd_key_create(&data->tsd_key, (char*)name);
+        erts_atomic_init_nob(&data->id_generator, 0);
+    }
 
     /* Initialize all chunks */
-    for (cix = 0; cix < erts_no_schedulers; cix++) {
+    for (cix = 0; cix < nthreads; cix++) {
 	erts_sspa_chunk_t *chnk = erts_sspa_cix2chunk(data, cix);
 	erts_sspa_chunk_header_t *chdr = &chnk->aligned.header;
 	erts_sspa_blk_t *blk;
diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.h b/erts/emulator/beam/erl_sched_spec_pre_alloc.h
index 1307e65962..d232db0e69 100644
--- a/erts/emulator/beam/erl_sched_spec_pre_alloc.h
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.h
@@ -59,6 +59,11 @@ typedef struct {
     char *start;
     char *end;
     int chunks_mem_size;
+    int nthreads;
+
+    /* Used only by thread variant: */
+    erts_tsd_key_t tsd_key;
+    erts_atomic_t id_generator;
 } erts_sspa_data_t;
 
 typedef union erts_sspa_blk_t_ erts_sspa_blk_t;
@@ -140,7 +145,9 @@ check_local_list(erts_sspa_chunk_header_t *chdr)
 #endif
 
 erts_sspa_data_t *erts_sspa_create(size_t blk_sz,
-				   int pa_size);
+				   int pa_size,
+                                   int nthreads,
+                                   const char* name);
 void erts_sspa_remote_free(erts_sspa_chunk_header_t *chdr,
 			   erts_sspa_blk_t *blk,
 			   int cinit);
@@ -158,7 +165,7 @@ ERTS_GLB_INLINE int erts_sspa_free(erts_sspa_data_t *data, int cix, char *blk);
 ERTS_GLB_INLINE erts_sspa_chunk_t *
 erts_sspa_cix2chunk(erts_sspa_data_t *data, int cix)
 {
-    ASSERT(0 <= cix && cix < erts_no_schedulers);
+    ASSERT(0 <= cix && cix < data->nthreads);
     return (erts_sspa_chunk_t *) (data->start + cix*data->chunks_mem_size);
 }
 
@@ -171,7 +178,7 @@ erts_sspa_ptr2cix(erts_sspa_data_t *data, void *ptr)
 	return -1;
     diff = ((char *) ptr) - data->start;
     cix = (int) diff / data->chunks_mem_size;
-    ASSERT(0 <= cix && cix < erts_no_schedulers);
+    ASSERT(0 <= cix && cix < data->nthreads);
     return cix;
 }
 
-- 
cgit v1.2.3


From 5e1a70e5eb0bfc39add1acdb60d5c49021edebcd Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 28 Jun 2017 20:02:30 +0200
Subject: erts: Optimize port_task quick allocator

for non scheduler threads by using ERTS_THR_PREF_QUICK_ALLOC_IMPL.
---
 erts/emulator/beam/erl_port_task.c | 11 ++++++-----
 erts/emulator/beam/erl_port_task.h |  3 +++
 erts/emulator/beam/erl_process.c   |  5 ++---
 3 files changed, 11 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 141e815d1f..86df0f4cb8 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -147,10 +147,10 @@ static void begin_port_cleanup(Port *pp,
 			       ErtsPortTask **execq,
 			       int *processing_busy_q_p);
 
-ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(port_task,
-				 ErtsPortTask,
-				 1000,
-				 ERTS_ALC_T_PORT_TASK)
+ERTS_THR_PREF_QUICK_ALLOC_IMPL(port_task,
+                               ErtsPortTask,
+                               1000,
+                               ERTS_ALC_T_PORT_TASK)
 
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(busy_caller_table,
 				 ErtsPortTaskBusyCallerTable,
@@ -2096,6 +2096,7 @@ erts_dequeue_port(ErtsRunQueue *rq)
 void
 erts_port_task_init(void)
 {
-    init_port_task_alloc();
+    init_port_task_alloc(erts_no_schedulers + erts_no_poll_threads
+                         + 1); /* aux_thread */
     init_busy_caller_table_alloc();
 }
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index ce63e4a7c0..ae78a7d8a3 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -218,6 +218,9 @@ void erts_port_task_execute(ErtsRunQueue *, Port **);
 void erts_port_task_init(void);
 #endif
 
+/* generated for 'port_task' quick allocator */
+void erts_port_task_pre_alloc_init_thread(void);
+
 void erts_port_task_tmp_handle_detach(ErtsPortTaskHandle *);
 int erts_port_task_abort(ErtsPortTaskHandle *);
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 624b907fb6..66dcd85af6 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -3129,6 +3129,7 @@ aux_thread(void *unused)
     }
 #endif
 
+    erts_port_task_pre_alloc_init_thread();
     ssi->event = erts_tse_fetch();
 
     erts_msacc_init_thread("aux", 1, 1);
@@ -3185,9 +3186,6 @@ aux_thread(void *unused)
     return NULL;
 }
 
-static void suspend_scheduler(ErtsSchedulerData *esdp);
-
-
 static void
 scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 {
@@ -8341,6 +8339,7 @@ sched_thread_func(void *vesdp)
     Uint no = esdp->no;
     erts_tse_t *tse;
 
+    erts_port_task_pre_alloc_init_thread();
     erts_sched_init_time_sup(esdp);
 
     if (no == 1)
-- 
cgit v1.2.3


From ce7b76ac099689a9f814e1901dbf77f50ffad42c Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 30 Jun 2017 08:14:00 +0200
Subject: erts: Fix msacc unmanaged state counter

OTP-14652
---
 erts/emulator/beam/erl_msacc.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h
index 8349a7e297..2d4637f800 100644
--- a/erts/emulator/beam/erl_msacc.h
+++ b/erts/emulator/beam/erl_msacc.h
@@ -318,8 +318,8 @@ ERTS_GLB_INLINE
 void erts_msacc_set_state_um__(ErtsMsAcc *msacc, Uint new_state, int increment) {
     if (ERTS_UNLIKELY(msacc->unmanaged)) {
         erts_mtx_lock(&msacc->mtx);
-        msacc->state = new_state;
         if (ERTS_LIKELY(!msacc->perf_counter)) {
+            msacc->state = new_state;
             erts_mtx_unlock(&msacc->mtx);
             return;
         }
-- 
cgit v1.2.3


From 1f9003a3dd9bbf9e3dcd84b8fdecef04a3b5e9c7 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 7 Jul 2017 16:26:23 +0200
Subject: erts: Fix smp_select testcase to use ERL_DRV_USE

This is needed with the new poll-thread implementation
as now closed fd's in the pollset will be triggered much
faster than before.
---
 erts/emulator/test/driver_SUITE_data/chkio_drv.c | 41 ++++++++++++++++--------
 1 file changed, 28 insertions(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/driver_SUITE_data/chkio_drv.c b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
index aed6752033..5167036602 100644
--- a/erts/emulator/test/driver_SUITE_data/chkio_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
@@ -308,10 +308,13 @@ static void free_smp_select(ChkioSmpSelect* pip, ErlDrvPort port)
 	abort();
     }
     case Selected:
-	driver_select(port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd, DO_READ, 0);
-	/*fall through*/ 
     case Opened:
-	close(pip->read_fd);
+        TRACEF(("%T: Close pipe [%d->%d]\n", driver_mk_port(port), pip->write_fd,
+                pip->read_fd));
+        if (pip->wasSelected)
+            driver_select(port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd, DO_READ|ERL_DRV_USE, 0);
+        else
+            close(pip->read_fd);
 	close(pip->write_fd);
 	pip->state = Closed;
 	break;
@@ -987,7 +990,7 @@ chkio_drv_control(ErlDrvData drv_data,
 		}
 		TRACEF(("%T: Created pipe [%d->%d]\n", cddp->id, fds[1], fds[0]));
 		pip->read_fd = fds[0];
-		pip->write_fd = fds[1];	       
+		pip->write_fd = fds[1];
 		pip->state = Opened;
 		pip->wasSelected = 0;
 		pip->next_write = pip->next_read = rand_r(&pip->rand_state) % 1024;
@@ -997,7 +1000,8 @@ chkio_drv_control(ErlDrvData drv_data,
 	    }/*fall through*/
 	    case Opened: {
 		if (op & 1) {
-		    TRACEF(("%T: Write %d to opened pipe [%d->%d]\n", cddp->id, pip->next_write, pip->write_fd, pip->read_fd));
+		    TRACEF(("%T: Write %d to opened pipe [%d->%d]\n", cddp->id,
+                            pip->next_write, pip->write_fd, pip->read_fd));
 		    if (write(pip->write_fd, &pip->next_write, sizeof(int)) != sizeof(int)) {
 			fprintf(stderr, "Failed to write to pipe fd=%d, errno=%d\n", pip->write_fd, errno);
 			abort();
@@ -1006,8 +1010,12 @@ chkio_drv_control(ErlDrvData drv_data,
 		}
 		op >>= 1;
 		if (pip->wasSelected && (op & 1)) {
-		    TRACEF(("%T: Close pipe [%d->%d]\n", cddp->id, pip->write_fd, pip->read_fd));
-		    if (close(pip->read_fd) || close(pip->write_fd)) {
+		    TRACEF(("%T: Close pipe [%d->%d]\n", cddp->id, pip->write_fd,
+                            pip->read_fd));
+                    drv_use_singleton.fd_stop_select = -2; /* disable stop_select asserts */
+		    if (driver_select(cddp->port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd,
+                                      DO_READ|ERL_DRV_USE, 0)
+                        || close(pip->write_fd)) {
 			fprintf(stderr, "Failed to close pipe, errno=%d\n", errno);
 			abort();
 		    }
@@ -1015,8 +1023,10 @@ chkio_drv_control(ErlDrvData drv_data,
 		    break;
 		}
 		else {
-		    TRACEF(("%T: Select on pipe [%d->%d]\n", cddp->id, pip->write_fd, pip->read_fd));
-		    if (driver_select(cddp->port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd, DO_READ, 1)) {
+		    TRACEF(("%T: Select on pipe [%d->%d]\n", cddp->id,
+                            pip->write_fd, pip->read_fd));
+		    if (driver_select(cddp->port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd,
+                                      DO_READ|ERL_DRV_USE, 1)) {
 			fprintf(stderr, "driver_select failed for fd=%d\n", pip->read_fd);
 			abort();
 		    }
@@ -1024,13 +1034,13 @@ chkio_drv_control(ErlDrvData drv_data,
 		    pip->wasSelected = 1;
 		    op >>= 1;
 		    if (pip->next_write != pip->next_read) { /* pipe not empty */
-			if (op & 1) {  
+			if (op & 1) {
 			    pip->state = Waiting; /* Wait for reader */
 			    break;
 			}
 			op >>= 1;
 		    }
-		}  
+		}
 	    }/*fall through*/
 	    case Selected:
 		if (op & 1) {
@@ -1059,7 +1069,7 @@ chkio_drv_control(ErlDrvData drv_data,
 			fprintf(stderr, "Failed to write to pipe fd=%d, errno=%d\n", pip->write_fd, errno);
 			abort();
 		    }
-		    pip->next_write++;		    
+		    pip->next_write++;
 		}
 		break;
 	    case Waiting:
@@ -1313,7 +1323,12 @@ static void chkio_drv_stop_select(ErlDrvEvent e, void* null)
     if (!(drv_use_singleton.fd_stop_select < 0)) {
 	assert_print("fd_stop_select<0", __LINE__); abort(); 
     }
-    drv_use_singleton.fd_stop_select = (int)(long)e;
+    /* fd_stop_select counting is disabled if this is set to -2 */
+    if (drv_use_singleton.fd_stop_select == -2) {
+        TRACEF(("closing %d\n", (int)(long)e));
+        close((int)(long)e);
+    } else
+        drv_use_singleton.fd_stop_select = (int)(long)e;
     /* Can't call chkio_drv_use directly here. That could even be recursive.
      * Next timeout will detect it instead.
      */
-- 
cgit v1.2.3


From 988f5f5e8061ce2e135a314ca782788eda478a06 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 30 May 2017 16:35:18 +0200
Subject: erts: Move all I/O polling to a seperate thread

---
 erts/emulator/Makefile.in                        |   16 +-
 erts/emulator/beam/erl_alloc.types               |    1 -
 erts/emulator/beam/erl_bif_info.c                |    8 +-
 erts/emulator/beam/erl_init.c                    |   58 +-
 erts/emulator/beam/erl_lock_count.c              |   20 -
 erts/emulator/beam/erl_port_task.c               |    8 +-
 erts/emulator/beam/erl_process.c                 |  700 +++---
 erts/emulator/beam/erl_process.h                 |    8 +-
 erts/emulator/beam/erl_vm.h                      |    2 -
 erts/emulator/beam/global.h                      |    1 -
 erts/emulator/beam/sys.h                         |    4 -
 erts/emulator/sys/common/erl_check_io.c          | 2304 +++++++++----------
 erts/emulator/sys/common/erl_check_io.h          |   92 +-
 erts/emulator/sys/common/erl_poll.c              | 2682 ++++++++--------------
 erts/emulator/sys/common/erl_poll.h              |  281 ++-
 erts/emulator/sys/common/erl_poll_api.h          |  122 +
 erts/emulator/sys/unix/sys.c                     |  109 +-
 erts/emulator/sys/win32/erl_poll.c               |  259 +--
 erts/emulator/sys/win32/sys.c                    |   15 +-
 erts/emulator/test/driver_SUITE.erl              |   49 +-
 erts/emulator/test/driver_SUITE_data/chkio_drv.c |    3 +
 21 files changed, 2802 insertions(+), 3940 deletions(-)
 create mode 100644 erts/emulator/sys/common/erl_poll_api.h

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 02a9a9a93b..915cad4e18 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -251,7 +251,7 @@ endif
 HIPEBEAMLDFLAGS=$($(ARCH)BEAMLDFLAGS)
 endif
 
-ERTS_ENABLE_KERNEL_POLL=@ERTS_ENABLE_KERNEL_POLL@
+ERTS_BUILD_FALLBACK_POLL=@ERTS_BUILD_FALLBACK_POLL@
 
 #
 #
@@ -928,17 +928,15 @@ $(STATIC_NIF_LIBS) $(STATIC_DRIVER_LIBS):
 	echo "=== Leaving lib after making static libs"
 endif
 
-ifeq ($(ERTS_ENABLE_KERNEL_POLL),yes)
+ifeq ($(ERTS_BUILD_FALLBACK_POLL),yes)
 OS_OBJS += 	$(OBJDIR)/erl_poll.kp.o \
-		$(OBJDIR)/erl_check_io.kp.o \
-		$(OBJDIR)/erl_poll.nkp.o \
-		$(OBJDIR)/erl_check_io.nkp.o
+		$(OBJDIR)/erl_poll.nkp.o
 else
-OS_OBJS +=	$(OBJDIR)/erl_poll.o \
-		$(OBJDIR)/erl_check_io.o
+OS_OBJS +=	$(OBJDIR)/erl_poll.o
 endif
 
-OS_OBJS +=	$(OBJDIR)/erl_mseg.o \
+OS_OBJS +=	$(OBJDIR)/erl_check_io.o \
+		$(OBJDIR)/erl_mseg.o \
 		$(OBJDIR)/erl_mmap.o \
 		$(OBJDIR)/erl_$(ERLANG_OSTYPE)_sys_ddll.o \
 		$(OBJDIR)/erl_mtrace_sys_wrap.o \
@@ -1112,7 +1110,7 @@ else
 SED_PREFIX=
 endif
 
-ifeq ($(ERTS_ENABLE_KERNEL_POLL),yes)
+ifeq ($(ERTS_BUILD_FALLBACK_POLL),yes)
 SED_SUFFIX=;$(SED_REPL_POLL);$(SED_REPL_CHK_IO)
 else
 SED_SUFFIX=
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 3ec1389d47..bf5487d31e 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -354,7 +354,6 @@ type	DRV_EV_STATE	LONG_LIVED	SYSTEM		driver_event_state
 type	DRV_SEL_D_STATE	FIXED_SIZE	SYSTEM		driver_select_data_state
 type	NIF_SEL_D_STATE	FIXED_SIZE	SYSTEM		enif_select_data_state
 type	FD_LIST		SHORT_LIVED	SYSTEM		fd_list
-type	ACTIVE_FD_ARR	SHORT_LIVED	SYSTEM		active_fd_array
 type	POLLSET		LONG_LIVED	SYSTEM		pollset
 type	POLLSET_UPDREQ	SHORT_LIVED	SYSTEM		pollset_update_req
 type	POLL_FDS	LONG_LIVED	SYSTEM		poll_fds
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index c782d7a8ce..d0c2accec6 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -98,9 +98,6 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
 #ifdef HIPE
 				     " [hipe]"
 #endif	
-#ifdef ERTS_ENABLE_KERNEL_POLL
-				     " [kernel-poll:%s]"
-#endif	
 #ifdef ET_DEBUG
 #if ET_DEBUG
 				     " [type-assertions]"
@@ -372,9 +369,6 @@ erts_print_system_version(fmtfn_t to, void *arg, Process *c_p)
 		      , total, online
 		      , dirty_cpu, dirty_cpu_onln, dirty_io
 		      , erts_async_max_threads
-#ifdef ERTS_ENABLE_KERNEL_POLL
-		      , erts_use_kernel_poll ? "true" : "false"
-#endif
 	);
 }
 
@@ -2696,7 +2690,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(make_small(CONTEXT_REDS));
     } else if (ERTS_IS_ATOM_STR("kernel_poll", BIF_ARG_1)) {
 #ifdef ERTS_ENABLE_KERNEL_POLL
-	BIF_RET(erts_use_kernel_poll ? am_true : am_false);
+	BIF_RET(am_true);
 #else
 	BIF_RET(am_false);
 #endif    
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 8c14c86219..b4d84f6b16 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -49,6 +49,7 @@
 #include "erl_bif_unique.h"
 #define ERTS_WANT_TIMER_WHEEL_API
 #include "erl_time.h"
+#include "erl_check_io.h"
 
 #ifdef HIPE
 #include "hipe_mode_switch.h"	/* for hipe_mode_switch_init() */
@@ -76,8 +77,10 @@ const char etp_otp_release[] = ERLANG_OTP_RELEASE;
 const char etp_compile_date[] = ERLANG_COMPILE_DATE;
 const char etp_arch[] = ERLANG_ARCHITECTURE;
 #ifdef ERTS_ENABLE_KERNEL_POLL
+const int erts_use_kernel_poll = 1;
 const int etp_kernel_poll_support = 1;
 #else
+const int erts_use_kernel_poll = 0;
 const int etp_kernel_poll_support = 0;
 #endif
 #if defined(ARCH_64)
@@ -203,16 +206,16 @@ int erts_no_line_info = 0;	/* -L: Don't load line information */
  */
 
 ErtsModifiedTimings erts_modified_timings[] = {
-    /* 0 */	{make_small(0), CONTEXT_REDS, INPUT_REDUCTIONS},
-    /* 1 */	{make_small(0), (3*CONTEXT_REDS)/4, 2*INPUT_REDUCTIONS},
-    /* 2 */	{make_small(0), CONTEXT_REDS/2, INPUT_REDUCTIONS/2},
-    /* 3 */	{make_small(0), (7*CONTEXT_REDS)/8, 3*INPUT_REDUCTIONS},
-    /* 4 */	{make_small(0), CONTEXT_REDS/3, 3*INPUT_REDUCTIONS},
-    /* 5 */	{make_small(0), (10*CONTEXT_REDS)/11, INPUT_REDUCTIONS/2},
-    /* 6 */	{make_small(1), CONTEXT_REDS/4, 2*INPUT_REDUCTIONS},
-    /* 7 */	{make_small(1), (5*CONTEXT_REDS)/7, INPUT_REDUCTIONS/3},
-    /* 8 */	{make_small(10), CONTEXT_REDS/5, 3*INPUT_REDUCTIONS},
-    /* 9 */	{make_small(10), (6*CONTEXT_REDS)/7, INPUT_REDUCTIONS/4}
+    /* 0 */	{make_small(0), CONTEXT_REDS},
+    /* 1 */	{make_small(0), (3*CONTEXT_REDS)/4},
+    /* 2 */	{make_small(0), CONTEXT_REDS/2},
+    /* 3 */	{make_small(0), (7*CONTEXT_REDS)/8},
+    /* 4 */	{make_small(0), CONTEXT_REDS/3},
+    /* 5 */	{make_small(0), (10*CONTEXT_REDS)/11},
+    /* 6 */	{make_small(1), CONTEXT_REDS/4},
+    /* 7 */	{make_small(1), (5*CONTEXT_REDS)/7},
+    /* 8 */	{make_small(10), CONTEXT_REDS/5},
+    /* 9 */	{make_small(10), (6*CONTEXT_REDS)/7}
 };
 
 #define ERTS_MODIFIED_TIMING_LEVELS \
@@ -310,8 +313,9 @@ erl_init(int ncpu,
     erts_init_sys_common_misc();
     erts_init_process(ncpu, proc_tab_sz, legacy_proc_tab);
     erts_init_scheduling(no_schedulers,
-			 no_schedulers_online
-			 , no_dirty_cpu_schedulers,
+			 no_schedulers_online,
+                         erts_no_poll_threads,
+			 no_dirty_cpu_schedulers,
 			 no_dirty_cpu_schedulers_online,
 			 no_dirty_io_schedulers
 			 );
@@ -558,9 +562,19 @@ void erts_usage(void)
     erts_fprintf(stderr, "-hmqd  val     set default message queue data flag for processes,\n");
     erts_fprintf(stderr, "               valid values are: off_heap | on_heap\n");
 
+    erts_fprintf(stderr, "-IOp number    set number of pollsets to be used to poll for I/O,\n");
+    erts_fprintf(stderr, "               This value has to be equal or smaller than the\n");
+    erts_fprintf(stderr, "               number of poll threads. If the current platform\n");
+    erts_fprintf(stderr, "               does not support concurrent update of pollsets\n");
+    erts_fprintf(stderr, "               this value is ignored.\n");
+    erts_fprintf(stderr, "-IOt number    set number of threads to be used to poll for I/O\n");
+    erts_fprintf(stderr, "-IOPp number   set number of pollsets as a percentage of the\n");
+    erts_fprintf(stderr, "               number of poll threads.");
+    erts_fprintf(stderr, "-IOPt number   set number of threads to be used to poll for I/O\n");
+    erts_fprintf(stderr, "               as a percentage of the number of schedulers.");
+
     /*    erts_fprintf(stderr, "-i module  set the boot module (default init)\n"); */
 
-    erts_fprintf(stderr, "-K boolean     enable or disable kernel poll\n");
     erts_fprintf(stderr, "-n[s|a|d]      Control behavior of signals to ports\n");
     erts_fprintf(stderr, "               Note that this flag is deprecated!\n");
     erts_fprintf(stderr, "-M<X> <Y>      memory allocator switches,\n");
@@ -727,7 +741,6 @@ early_init(int *argc, char **argv) /*
     char envbuf[21]; /* enough for any 64-bit integer */
     size_t envbufsz;
 
-
     erts_save_emu_args(*argc, argv);
 
     erts_sched_compact_load = 1;
@@ -839,6 +852,7 @@ early_init(int *argc, char **argv) /*
 		    }
 		    break;
 		}
+
 		case 'S' :
 		    if (argv[i][2] == 'P') {
 			int ptot, ponln;
@@ -1100,6 +1114,9 @@ early_init(int *argc, char **argv) /*
     erts_alloc_init(argc, argv, &alloc_opts); /* Handles (and removes)
 						 -M flags. */
     /* Require allocators */
+
+    erts_init_check_io(argc, argv);
+
     /*
      * Thread progress management:
      *
@@ -1107,13 +1124,14 @@ early_init(int *argc, char **argv) /*
      * ** Scheduler threads (see erl_process.c)
      * ** Aux thread (see erl_process.c)
      * ** Sys message dispatcher thread (see erl_trace.c)
+     * ** IO Poll threads (see erl_check_io.c)
      *
      * * Unmanaged threads that need to register:
      * ** Async threads (see erl_async.c)
      * ** Dirty scheduler threads
      */
     erts_thr_progress_init(no_schedulers,
-			   no_schedulers+2,
+			   no_schedulers+2+erts_no_poll_threads,
 			   erts_async_max_threads +
 			   erts_no_dirty_cpu_schedulers +
 			   erts_no_dirty_io_schedulers
@@ -1561,16 +1579,6 @@ erl_start(int argc, char **argv)
 	    have_break_handler = 0;
 	  break;
 
-	case 'K':
-	    /* If kernel poll support is present,
-	       erl_sys_args() will remove the K parameter
-	       and value */
-	    get_arg(argv[i]+2, argv[i+1], &i);
-	    erts_fprintf(stderr,
-		       "kernel-poll not supported; \"K\" parameter ignored\n",
-		       arg);
-	    break;
-
 	case 'n':
 	    arg = get_arg(argv[i]+2, argv[i+1], &i);
 	    switch (arg[0]) {
diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index 11890e44d1..1ae6076b12 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -554,16 +554,6 @@ erts_lock_flags_t erts_lcnt_get_category_mask() {
     return lcnt_category_mask__;
 }
 
-#ifdef ERTS_ENABLE_KERNEL_POLL
-/* erl_poll/erl_check_io only exports one of these variants at a time, and we
- * may need to use either one depending on emulator startup flags. */
-void erts_lcnt_update_pollset_locks_nkp(int);
-void erts_lcnt_update_pollset_locks_kp(int);
-
-void erts_lcnt_update_cio_locks_nkp(int);
-void erts_lcnt_update_cio_locks_kp(int);
-#endif
-
 void erts_lcnt_set_category_mask(erts_lock_flags_t mask) {
     erts_lock_flags_t changed_categories;
 
@@ -590,16 +580,6 @@ void erts_lcnt_set_category_mask(erts_lock_flags_t mask) {
     }
 
     if(changed_categories & ERTS_LOCK_FLAGS_CATEGORY_IO) {
-#ifdef ERTS_ENABLE_KERNEL_POLL
-        if(erts_use_kernel_poll) {
-            erts_lcnt_update_pollset_locks_kp(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
-        } else {
-            erts_lcnt_update_pollset_locks_nkp(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
-        }
-#else
-        erts_lcnt_update_pollset_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
-#endif
-
         erts_lcnt_update_cio_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
         erts_lcnt_update_driver_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
         erts_lcnt_update_port_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO);
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 86df0f4cb8..a588477320 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -582,12 +582,10 @@ static ERTS_INLINE void
 reset_executed_io_task_handle(ErtsPortTask *ptp)
 {
     if (ptp->u.alive.handle) {
-	ErtsIoTask *itp = ErtsContainerStruct(ptp->u.alive.handle, ErtsIoTask, task);
-
 	ASSERT(ptp == handle2task(ptp->u.alive.handle));
-	erts_io_notify_port_task_executed(ptp->u.alive.handle);
-	reset_port_task_handle(ptp->u.alive.handle);
-	erts_check_io_interrupt(itp->pollset, 1);
+        /* The port task handle is reset inside task_executed */
+	erts_io_notify_port_task_executed(ptp->type, ptp->u.alive.handle,
+                                          reset_port_task_handle);
     }
 }
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 66dcd85af6..ffb67aefaf 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -24,6 +24,8 @@
 #  include "config.h"
 #endif
 
+#define ERTS_WANT_BREAK_HANDLING
+
 #include <stddef.h> /* offsetof() */
 #include "sys.h"
 #include "erl_vm.h"
@@ -50,6 +52,7 @@
 #include "erl_time.h"
 #include "erl_nfunc_sched.h"
 #include "erl_check_io.h"
+#include "erl_poll.h"
 
 #define ERTS_CHECK_TIME_REDS CONTEXT_REDS
 #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0)
@@ -208,6 +211,7 @@ static struct ErtsSchedBusyWait_ {
 int erts_disable_proc_not_running_opt;
 
 static ErtsAuxWorkData *aux_thread_aux_work_data;
+static ErtsAuxWorkData *poll_thread_aux_work_data;
 
 #define ERTS_SCHDLR_SSPND_CHNG_NMSB		(((erts_aint32_t) 1) << 0)
 #define ERTS_SCHDLR_SSPND_CHNG_MSB		(((erts_aint32_t) 1) << 1)
@@ -432,6 +436,7 @@ typedef union {
 static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info;
 static ErtsAlignedSchedulerSleepInfo *aligned_dirty_cpu_sched_sleep_info;
 static ErtsAlignedSchedulerSleepInfo *aligned_dirty_io_sched_sleep_info;
+static ErtsAlignedSchedulerSleepInfo *aligned_poll_thread_sleep_info;
 
 static Uint last_reductions;
 static Uint last_exact_reductions;
@@ -499,9 +504,13 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist,
 				 200,
 				 ERTS_ALC_T_PROC_LIST)
 
+#define ERTS_POLL_THREAD_SLEEP_INFO_IX(IX)                              \
+        (ASSERT(0 <= ((int) (IX))                                       \
+                && ((int) (IX)) < ((int) erts_no_poll_threads)),        \
+         &aligned_poll_thread_sleep_info[(IX)].ssi)
 #define ERTS_SCHED_SLEEP_INFO_IX(IX)					\
-    (ASSERT(-1 <= ((int) (IX))					        \
-		 && ((int) (IX)) < ((int) erts_no_schedulers)),		\
+    (ASSERT(((int)-1) <= ((int) (IX))                 \
+            && ((int) (IX)) < ((int) erts_no_schedulers)),		\
      &aligned_sched_sleep_info[(IX)].ssi)
 #define ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(IX)				\
     (ASSERT(0 <= ((int) (IX))					        \
@@ -1560,14 +1569,14 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi,
 {
     switch (flags & ERTS_SSI_FLGS_SLEEP_TYPE) {
     case ERTS_SSI_FLG_POLL_SLEEPING:
-	erts_check_io_interrupt(ssi->esdp->pollset, 1);
+	erts_check_io_interrupt(ssi->psi, 1);
 	break;
     case ERTS_SSI_FLG_POLL_SLEEPING|ERTS_SSI_FLG_TSE_SLEEPING:
 	/*
 	 * Thread progress blocking while poll sleeping; need
 	 * to signal on both...
 	 */
-	erts_check_io_interrupt(ssi->esdp->pollset, 1);
+	erts_check_io_interrupt(ssi->psi, 1);
 	/* fall through */
     case ERTS_SSI_FLG_TSE_SLEEPING:
 	erts_tse_set(ssi->event);
@@ -2827,36 +2836,6 @@ erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable)
     return old;
 }
 
-
-
-static ERTS_INLINE void
-sched_waiting_sys(Uint no, ErtsRunQueue *rq)
-{
-    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
-    ASSERT(rq->waiting >= 0);
-    (void) ERTS_RUNQ_FLGS_SET(rq, (ERTS_RUNQ_FLG_OUT_OF_WORK
-				   | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK));
-    rq->waiting++;
-    rq->waiting *= -1;
-    rq->woken = 0;
-    if (erts_system_profile_flags.scheduler)
-	profile_scheduler(make_small(no), am_inactive);
-}
-
-static ERTS_INLINE void
-sched_active_sys(Uint no, ErtsRunQueue *rq)
-{
-    ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
-
-    ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
-
-    ASSERT(rq->waiting < 0);
-    rq->waiting *= -1;
-    rq->waiting--;
-    if (erts_system_profile_flags.scheduler)
-	profile_scheduler(make_small(no), am_active);
-}
-
 Uint
 erts_active_schedulers(void)
 {
@@ -2867,13 +2846,6 @@ erts_active_schedulers(void)
     return as;
 }
 
-static ERTS_INLINE int
-prepare_for_sys_schedule(int non_blocking)
-{
-    return 1;
-}
-
-
 static ERTS_INLINE void
 sched_waiting(Uint no, ErtsRunQueue *rq)
 {
@@ -3045,7 +3017,7 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type)
 	erts_tse_reset(ssi->event);
     else {
 	ASSERT(sleep_type == ERTS_SSI_FLG_POLL_SLEEPING);
-	erts_check_io_interrupt(ssi->esdp->pollset, 0);
+        erts_check_io_interrupt(ssi->psi, 0);
     }
 
     while (1) {
@@ -3113,6 +3085,12 @@ thr_prgr_fin_wait(void *vssi)
 
 static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp);
 
+void
+erts_aux_thread_poke()
+{
+    erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(-1));
+}
+
 static void *
 aux_thread(void *unused)
 {
@@ -3121,6 +3099,7 @@ aux_thread(void *unused)
     erts_aint32_t aux_work;
     ErtsThrPrgrCallbacks callbacks;
     int thr_prgr_active = 1;
+    ERTS_MSACC_DECLARE_CACHE();
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
@@ -3144,9 +3123,14 @@ aux_thread(void *unused)
     init_aux_work_data(awdp, NULL, NULL);
     awdp->ssi = ssi;
 
+#if ERTS_POLL_USE_FALLBACK
+    ssi->psi = erts_create_pollset_thread(-1);
+#endif
 
     sched_prep_spin_wait(ssi);
 
+    ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER);
+
     while (1) {
 	erts_aint32_t flgs;
 
@@ -3155,30 +3139,128 @@ aux_thread(void *unused)
 	    if (!thr_prgr_active)
 		erts_thr_progress_active(NULL, thr_prgr_active = 1);
 	    aux_work = handle_aux_work(awdp, aux_work, 1);
+            ERTS_MSACC_UPDATE_CACHE();
 	    if (aux_work && erts_thr_progress_update(NULL))
 		erts_thr_progress_leader_update(NULL);
 	}
 
 	if (!aux_work) {
+
+#ifdef ERTS_BREAK_REQUESTED
+            if (ERTS_BREAK_REQUESTED)
+                erts_do_break_handling();
+#endif
+
 	    if (thr_prgr_active)
 		erts_thr_progress_active(NULL, thr_prgr_active = 0);
-	    erts_thr_progress_prepare_wait(NULL);
+
+#if ERTS_POLL_USE_FALLBACK
 
 	    flgs = sched_spin_wait(ssi, 0);
 
 	    if (flgs & ERTS_SSI_FLG_SLEEPING) {
 		ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+		flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING);
+		if (flgs & ERTS_SSI_FLG_SLEEPING) {
+		    ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING);
+		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+                    erts_check_io(ssi->psi);
+		}
+            }
+#else
+            erts_thr_progress_prepare_wait(NULL);
+
+	    flgs = sched_spin_wait(ssi, 0);
+
+	    if (flgs & ERTS_SSI_FLG_SLEEPING) {
 		flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
 		if (flgs & ERTS_SSI_FLG_SLEEPING) {
-		    int res;
+                    int res;
 		    ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
 		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-		    do {
-			res = erts_tse_wait(ssi->event);
-		    } while (res == EINTR);
+                    ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP);
+                    do {
+                        res = erts_tse_wait(ssi->event);
+                    } while (res == EINTR);
+                    ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER);
+		}
+            }
+            erts_thr_progress_finalize_wait(NULL);
+#endif
+	}
+
+	flgs = sched_prep_spin_wait(ssi);
+    }
+    return NULL;
+}
+
+static void *
+poll_thread(void *arg)
+{
+    int id = (int)(UWord)arg;
+    ErtsAuxWorkData *awdp = poll_thread_aux_work_data+id;
+    ErtsSchedulerSleepInfo *ssi = ERTS_POLL_THREAD_SLEEP_INFO_IX(id);
+    erts_aint32_t aux_work;
+    ErtsThrPrgrCallbacks callbacks;
+    int thr_prgr_active = 1;
+    struct erts_poll_thread *psi = erts_create_pollset_thread(id);
+    ERTS_MSACC_DECLARE_CACHE();
+
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    {
+	char buf[] = "poll_thread";
+	erts_lc_set_thread_name(buf);
+    }
+#endif
+
+    erts_port_task_pre_alloc_init_thread();
+    ssi->event = erts_tse_fetch();
+
+    erts_msacc_init_thread("poll", id, 0);
+
+    callbacks.arg = (void *) ssi;
+    callbacks.wakeup = thr_prgr_wakeup;
+    callbacks.prepare_wait = thr_prgr_prep_wait;
+    callbacks.wait = thr_prgr_wait;
+    callbacks.finalize_wait = thr_prgr_fin_wait;
+
+    erts_thr_progress_register_managed_thread(NULL, &callbacks, 0);
+    init_aux_work_data(awdp, NULL, NULL);
+    awdp->ssi = ssi;
+    ssi->psi = psi;
+
+    sched_prep_spin_wait(ssi);
+
+    ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER);
+
+    while (1) {
+	erts_aint32_t flgs;
+
+	aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
+	if (aux_work) {
+	    if (!thr_prgr_active)
+		erts_thr_progress_active(NULL, thr_prgr_active = 1);
+	    aux_work = handle_aux_work(awdp, aux_work, 1);
+            ERTS_MSACC_UPDATE_CACHE();
+	    if (aux_work && erts_thr_progress_update(NULL))
+		erts_thr_progress_leader_update(NULL);
+	}
+
+	if (!aux_work) {
+	    if (thr_prgr_active)
+		erts_thr_progress_active(NULL, thr_prgr_active = 0);
+
+	    flgs = sched_spin_wait(ssi, 0);
+
+	    if (flgs & ERTS_SSI_FLG_SLEEPING) {
+		ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+		flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING);
+		if (flgs & ERTS_SSI_FLG_SLEEPING) {
+		    ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING);
+		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+                    erts_check_io(psi);
 		}
 	    }
-	    erts_thr_progress_finalize_wait(NULL);
 	}
 
 	flgs = sched_prep_spin_wait(ssi);
@@ -3219,273 +3301,136 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
         dirty_active(esdp, -1);
     }
 
-    /*
-     * If all schedulers are waiting, one of them *should*
-     * be waiting in erl_sys_schedule()
-     */
-
-    if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(0)) {
+    sched_waiting(esdp->no, rq);
 
-	sched_waiting(esdp->no, rq);
-
-	erts_runq_unlock(rq);
-
-	spincount = sched_busy_wait.tse;
-
-	if (ERTS_SCHEDULER_IS_DIRTY(esdp))
-	    dirty_sched_wall_time_change(esdp, working = 0);
-        else if (thr_prgr_active != working)
-	    sched_wall_time_change(esdp, working = thr_prgr_active);
-
-	while (1) {
-	    ErtsMonotonicTime current_time = 0;
+    erts_runq_unlock(rq);
 
-	    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
-	    if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-		if (!thr_prgr_active) {
-		    erts_thr_progress_active(esdp, thr_prgr_active = 1);
-		    sched_wall_time_change(esdp, 1);
-		}
-		aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1);
-                ERTS_MSACC_UPDATE_CACHE();
-		if (aux_work && erts_thr_progress_update(esdp))
-		    erts_thr_progress_leader_update(esdp);
-	    }
+    spincount = sched_busy_wait.tse;
 
-	    if (aux_work) {
-		if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-		    flgs = erts_atomic32_read_acqb(&ssi->flags);
-		    current_time = erts_get_monotonic_time(esdp);
-		    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
-			if (!thr_prgr_active) {
-			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
-			    sched_wall_time_change(esdp, 1);
-			}
-			erts_bump_timers(esdp->timer_wheel, current_time);
-		    }
-		}
-	    }
-	    else {
-		ErtsMonotonicTime timeout_time;
-		int do_timeout = 0;
-		if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-		    timeout_time = erts_check_next_timeout_time(esdp);
-		    current_time = erts_get_monotonic_time(esdp);
-		    do_timeout = (current_time >= timeout_time);
-		} else {
-		    current_time = 0;
-		    timeout_time = ERTS_MONOTONIC_TIME_MAX;
-		}
-		if (do_timeout) {
-		    if (!thr_prgr_active) {
-			erts_thr_progress_active(esdp, thr_prgr_active = 1);
-			sched_wall_time_change(esdp, 1);
-		    }
-		}
-		else {
-		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-			if (thr_prgr_active) {
-			    erts_thr_progress_active(esdp, thr_prgr_active = 0);
-			    sched_wall_time_change(esdp, 0);
-			}
-			erts_thr_progress_prepare_wait(esdp);
-		    }
+    if (ERTS_SCHEDULER_IS_DIRTY(esdp))
+        dirty_sched_wall_time_change(esdp, working = 0);
+    else if (thr_prgr_active != working)
+        sched_wall_time_change(esdp, working = thr_prgr_active);
 
-		    flgs = sched_spin_wait(ssi, spincount);
-		    if (flgs & ERTS_SSI_FLG_SLEEPING) {
-			ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-			flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
-			if (flgs & ERTS_SSI_FLG_SLEEPING) {
-			    int res;
-			    ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
-			    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-			    current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
-				erts_get_monotonic_time(esdp);
-			    do {
-				Sint64 timeout;
-				if (current_time >= timeout_time)
-				    break;
-				if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-				    timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
-								     - current_time
-								     - 1) + 1;
-				} else
-				    timeout = -1;
-                                ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
-				res = erts_tse_twait(ssi->event, timeout);
-                                ERTS_MSACC_POP_STATE_M();
-				current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
-				    erts_get_monotonic_time(esdp);
-			    } while (res == EINTR);
-			}
-		    }
-		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-			erts_thr_progress_finalize_wait(esdp);
-		}
-		if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time)
-		    erts_bump_timers(esdp->timer_wheel, current_time);
-	    }
+    while (1) {
+        ErtsMonotonicTime current_time = 0;
 
-	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
-		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
-		break;
-	    }
+        aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
+        if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+            if (!thr_prgr_active) {
+                erts_thr_progress_active(esdp, thr_prgr_active = 1);
+                sched_wall_time_change(esdp, 1);
+            }
+            aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1);
+            ERTS_MSACC_UPDATE_CACHE();
+            if (aux_work && erts_thr_progress_update(esdp))
+                erts_thr_progress_leader_update(esdp);
+        }
 
-	    flgs = sched_prep_cont_spin_wait(ssi);
-	    spincount = sched_busy_wait.aux_work;
+        if (aux_work) {
+            if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+                flgs = erts_atomic32_read_acqb(&ssi->flags);
+                current_time = erts_get_monotonic_time(esdp);
+                if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
+                    if (!thr_prgr_active) {
+                        erts_thr_progress_active(esdp, thr_prgr_active = 1);
+                        sched_wall_time_change(esdp, 1);
+                    }
+                    erts_bump_timers(esdp->timer_wheel, current_time);
+                }
+            }
+        }
+        else {
+            ErtsMonotonicTime timeout_time;
+            int do_timeout = 0;
+            if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+                timeout_time = erts_check_next_timeout_time(esdp);
+                current_time = erts_get_monotonic_time(esdp);
+                do_timeout = (current_time >= timeout_time);
+            } else {
+                current_time = 0;
+                timeout_time = ERTS_MONOTONIC_TIME_MAX;
+            }
+            if (do_timeout) {
+                if (!thr_prgr_active) {
+                    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+                    sched_wall_time_change(esdp, 1);
+                }
+            }
+            else {
+                if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+                    if (thr_prgr_active) {
+                        erts_thr_progress_active(esdp, thr_prgr_active = 0);
+                        sched_wall_time_change(esdp, 0);
+                    }
+                    erts_thr_progress_prepare_wait(esdp);
+                }
 
-	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
-		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
-		break;
-	    }
+                flgs = sched_spin_wait(ssi, spincount);
+                if (flgs & ERTS_SSI_FLG_SLEEPING) {
+                    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+                    flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
+                    if (flgs & ERTS_SSI_FLG_SLEEPING) {
+                        int res;
+                        ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
+                        ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+                        current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
+                            erts_get_monotonic_time(esdp);
+                        do {
+                            Sint64 timeout;
+                            if (current_time >= timeout_time)
+                                break;
+                            if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+                                timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
+                                                                 - current_time
+                                                                 - 1) + 1;
+                            } else
+                                timeout = -1;
+                            ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
+                            res = erts_tse_twait(ssi->event, timeout);
+                            ERTS_MSACC_POP_STATE_M();
+                            current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
+                                erts_get_monotonic_time(esdp);
+                        } while (res == EINTR);
+                    }
+                }
+                if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+                    erts_thr_progress_finalize_wait(esdp);
+            }
+            if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time)
+                erts_bump_timers(esdp->timer_wheel, current_time);
+        }
 
-	}
+        if (!(flgs & ERTS_SSI_FLG_WAITING)) {
+            ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
+            break;
+        }
 
-	if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC))
-	    erts_atomic32_read_band_nob(&ssi->flags,
-                                            (ERTS_SSI_FLG_SUSPENDED
-                                             | ERTS_SSI_FLG_MSB_EXEC));
+        flgs = sched_prep_cont_spin_wait(ssi);
+        spincount = sched_busy_wait.aux_work;
 
-	if (ERTS_SCHEDULER_IS_DIRTY(esdp))
-	    dirty_sched_wall_time_change(esdp, working = 1);
-        else if (!thr_prgr_active) {
-	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
-	    sched_wall_time_change(esdp, 1);
+        if (!(flgs & ERTS_SSI_FLG_WAITING)) {
+            ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
+            break;
         }
 
-	erts_runq_lock(rq);
-	sched_active(esdp->no, rq);
-
     }
-    else
-    {
-
-	esdp->function_calls = 0;
-	*fcalls = 0;
-
-	ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
-
-	sched_waiting_sys(esdp->no, rq);
-
-	erts_runq_unlock(rq);
-
-	ASSERT(working);
-	sched_wall_time_change(esdp, working = 0);
 
-	spincount = sched_busy_wait.sys_schedule;
-	if (spincount == 0)
-	    goto sys_aux_work;
+    if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC))
+        erts_atomic32_read_band_nob(&ssi->flags,
+                                        (ERTS_SSI_FLG_SUSPENDED
+                                         | ERTS_SSI_FLG_MSB_EXEC));
 
-	while (spincount-- > 0) {
-	    ErtsMonotonicTime current_time;
-
-	sys_poll_aux_work:
-
-	    if (working)
-		sched_wall_time_change(esdp, working = 0);
-
-	    ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO);
-
-            LTTNG2(scheduler_poll, esdp->no, 1);
-	    erl_sys_schedule(1); /* Might give us something to do */
-
-	    ERTS_MSACC_POP_STATE_M();
-
-	    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-		current_time = erts_get_monotonic_time(esdp);
-		if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
-		    erts_bump_timers(esdp->timer_wheel, current_time);
-	    }
-
-	sys_aux_work:
-
-	    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
-	    if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-		if (!working)
-		    sched_wall_time_change(esdp, working = 1);
-		if (!thr_prgr_active)
-		    erts_thr_progress_active(esdp, thr_prgr_active = 1);
-		aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1);
-                ERTS_MSACC_UPDATE_CACHE();
-		if (aux_work && erts_thr_progress_update(esdp))
-		    erts_thr_progress_leader_update(esdp);
-	    }
-
-	    flgs = erts_atomic32_read_acqb(&ssi->flags);
-	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
-		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
-		goto sys_woken;
-	    }
-	}
-
-	erts_runq_lock(rq);
-
-	if (aux_work) {
-	    erts_runq_unlock(rq);
-	    goto sys_poll_aux_work;
-	}
-	flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING);
-	if (!(flgs & ERTS_SSI_FLG_SLEEPING)) {
-	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
-		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
-		goto sys_locked_woken;
-	    }
-	    erts_runq_unlock(rq);
-	    flgs = sched_prep_cont_spin_wait(ssi);
-	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
-		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
-		goto sys_woken;
-	    }
-	    goto sys_poll_aux_work;
-	}
-
-	ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING);
-	ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-
-	erts_runq_unlock(rq);
-
-	if (working)
-	    sched_wall_time_change(esdp, working = 0);
-
-	if (thr_prgr_active)
-	    erts_thr_progress_active(esdp, thr_prgr_active = 0);
-
-	ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO);
-        LTTNG2(scheduler_poll, esdp->no, 0);
-
-	erl_sys_schedule(0);
-
-	ERTS_MSACC_POP_STATE_M();
-
-	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-	    ErtsMonotonicTime current_time = erts_get_monotonic_time(esdp);
-	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
-		erts_bump_timers(esdp->timer_wheel, current_time);
-	}
-
-	flgs = sched_prep_cont_spin_wait(ssi);
-	if (flgs & ERTS_SSI_FLG_WAITING)
-	    goto sys_aux_work;
-
-    sys_woken:
-	if (!thr_prgr_active)
-	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
-	erts_runq_lock(rq);
-    sys_locked_woken:
-	if (!thr_prgr_active) {
-	    erts_runq_unlock(rq);
-	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
-	    erts_runq_lock(rq);
-	}
-	if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC))
-	    erts_atomic32_read_band_nob(&ssi->flags,
-                                            (ERTS_SSI_FLG_SUSPENDED
-                                             | ERTS_SSI_FLG_MSB_EXEC));
-	if (!working)
-	    sched_wall_time_change(esdp, working = 1);
-	sched_active_sys(esdp->no, rq);
+    if (ERTS_SCHEDULER_IS_DIRTY(esdp))
+        dirty_sched_wall_time_change(esdp, working = 1);
+    else if (!thr_prgr_active) {
+        erts_thr_progress_active(esdp, thr_prgr_active = 1);
+        sched_wall_time_change(esdp, 1);
     }
 
+    erts_runq_lock(rq);
+    sched_active(esdp->no, rq);
+
     if (ERTS_SCHEDULER_IS_DIRTY(esdp))
         dirty_active(esdp, 1);
 
@@ -5755,7 +5700,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     esdp->run_queue = runq;
     if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) {
 	esdp->no = 0;
-        esdp->pollset = NULL;
         if (runq == ERTS_DIRTY_CPU_RUNQ)
             esdp->type = ERTS_SCHED_DIRTY_CPU;
         else {
@@ -5775,7 +5719,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
         esdp->type = ERTS_SCHED_NORMAL;
 	esdp->no = (Uint) num;
 	esdp->dirty_no = 0;
-        esdp->pollset = erts_get_pollset(num);
         runq->scheduler = esdp;
     }
     esdp->dirty_shadow_process = shadow_proc;
@@ -5788,8 +5731,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 	shadow_proc->static_flags = ERTS_STC_FLG_SHADOW_PROC;
     }
 
-    esdp->function_calls = 0;
-
     ssi->esdp = esdp;
     esdp->ssi = ssi;
     esdp->current_process = NULL;
@@ -5821,8 +5762,8 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 }
 
 void
-erts_init_scheduling(int no_schedulers, int no_schedulers_online
-		     , int no_dirty_cpu_schedulers, int no_dirty_cpu_schedulers_online,
+erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_threads,
+		     int no_dirty_cpu_schedulers, int no_dirty_cpu_schedulers_online,
 		     int no_dirty_io_schedulers
 		     )
 {
@@ -5849,6 +5790,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     ASSERT(no_dirty_cpu_schedulers >= 1);
     ASSERT(no_dirty_cpu_schedulers_online <= no_schedulers_online);
     ASSERT(no_dirty_cpu_schedulers_online >= 1);
+    ASSERT(erts_no_poll_threads == no_poll_threads);
 
     /* Create and initialize run queues */
 
@@ -5946,7 +5888,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     erts_no_total_schedulers += no_dirty_io_schedulers;
 
     /* Create and initialize scheduler sleep info */
-    no_ssi = n+1;
+    no_ssi = n + 1 /* aux thread */;
     aligned_sched_sleep_info =
 	erts_alloc_permanent_cache_aligned(
 	    ERTS_ALC_T_SCHDLR_SLP_INFO,
@@ -5957,12 +5899,13 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	ssi->next = NULL;
 	ssi->prev = NULL;
 #endif
+        ssi->esdp = NULL;
 	erts_atomic32_init_nob(&ssi->flags, 0);
 	ssi->event = NULL; /* initialized in sched_thread_func */
 	erts_atomic32_init_nob(&ssi->aux_work, 0);
     }
 
-    aligned_sched_sleep_info++;
+    aligned_sched_sleep_info += 1 /* aux thread */;
 
     aligned_dirty_cpu_sched_sleep_info =
 	erts_alloc_permanent_cache_aligned(
@@ -5985,6 +5928,18 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	erts_atomic32_init_nob(&ssi->aux_work, 0);
     }
 
+    aligned_poll_thread_sleep_info =
+        erts_alloc_permanent_cache_aligned(
+	    ERTS_ALC_T_SCHDLR_SLP_INFO,
+	    no_poll_threads*sizeof(ErtsAlignedSchedulerSleepInfo));
+    for (ix = 0; ix < no_poll_threads; ix++) {
+        ErtsSchedulerSleepInfo *ssi = &aligned_poll_thread_sleep_info[ix].ssi;
+        ssi->esdp = NULL;
+	erts_atomic32_init_nob(&ssi->flags, 0);
+	ssi->event = NULL; /* initialized in poll_thread */
+	erts_atomic32_init_nob(&ssi->aux_work, 0);
+    }
+
     /* Create and initialize scheduler specific data */
 
     daww_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE((sizeof(ErtsDelayedAuxWorkWakeupJob)
@@ -6041,11 +5996,14 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */
     debug_wait_completed_flags = 0;
 
-
     aux_thread_aux_work_data =
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
 					   sizeof(ErtsAuxWorkData));
 
+    poll_thread_aux_work_data =
+	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
+					   no_poll_threads * sizeof(ErtsAuxWorkData));
+
     init_no_runqs(no_schedulers_online, no_schedulers_online);
     balance_info.last_active_runqs = no_schedulers;
     erts_mtx_init(&balance_info.update_mtx, "migration_info_update", NIL,
@@ -7034,12 +6992,8 @@ sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi,
 			   | ERTS_SSI_FLG_WAITING
 			   | ERTS_SSI_FLG_SUSPENDED);
 
-    if (sleep_type == ERTS_SSI_FLG_TSE_SLEEPING)
-	erts_tse_reset(ssi->event);
-    else {
-	ASSERT(sleep_type == ERTS_SSI_FLG_POLL_SLEEPING);
-	erts_check_io_interrupt(ssi->esdp->pollset, 0);
-    }
+    ASSERT(sleep_type == ERTS_SSI_FLG_TSE_SLEEPING);
+    erts_tse_reset(ssi->event);
 
     while (1) {
 	oflgs = erts_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
@@ -7291,17 +7245,6 @@ msb_scheduler_type_switch(ErtsSchedType sched_type,
     if (exec_type != ERTS_SCHED_NORMAL)
         schdlr_sspnd.last_msb_dirty_type = exec_type;
     else {
-        erts_aint32_t calls;
-        /*
-         * Going back to normal scheduler after
-         * dirty execution; make sure it will check
-         * for I/O...
-         */
-        if (ERTS_USE_MODIFIED_TIMING())
-            calls = ERTS_MODIFIED_TIMING_INPUT_REDS + 1;
-        else
-            calls = INPUT_REDUCTIONS + 1;
-        esdp->function_calls = calls;
 
         if ((nrml_prio == ERTS_MSB_NONE_PRIO_BIT)
             & ((dcpu_prio != ERTS_MSB_NONE_PRIO_BIT)
@@ -7376,7 +7319,7 @@ msb_scheduler_type_switch(ErtsSchedType sched_type,
 }
 
 static ERTS_INLINE void
-suspend_dirty_scheduler_sleep(ErtsSchedulerData *esdp)
+suspend_normal_scheduler_sleep(ErtsSchedulerData *esdp)
 {
     ErtsSchedulerSleepInfo *ssi = esdp->ssi;
     erts_aint32_t flgs = sched_spin_suspended(ssi,
@@ -7399,36 +7342,9 @@ suspend_dirty_scheduler_sleep(ErtsSchedulerData *esdp)
 }
 
 static ERTS_INLINE void
-suspend_normal_scheduler_sleep(ErtsSchedulerData *esdp)
+suspend_dirty_scheduler_sleep(ErtsSchedulerData *esdp)
 {
-    ErtsSchedulerSleepInfo *ssi = esdp->ssi;
-    erts_aint32_t flgs;
-    int spincount;
-
-    spincount = sched_busy_wait.sys_schedule;
-
-    while (spincount-- > 0) {
-
-        erl_sys_schedule(1);
-
-        flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
-        if (flgs != (ERTS_SSI_FLG_SLEEPING
-                     | ERTS_SSI_FLG_WAITING
-                     | ERTS_SSI_FLG_SUSPENDED)) {
-            return;
-        }
-    }
-
-    flgs = sched_set_suspended_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING);
-    if (flgs != (ERTS_SSI_FLG_SLEEPING
-                 | ERTS_SSI_FLG_POLL_SLEEPING
-                 | ERTS_SSI_FLG_WAITING
-                 | ERTS_SSI_FLG_SUSPENDED)) {
-        return;
-    }
-
-    /* Sleep on pollset... */
-    erl_sys_schedule(0);
+    suspend_normal_scheduler_sleep(esdp);
 }
 
 static void
@@ -7662,8 +7578,10 @@ suspend_scheduler(ErtsSchedulerData *esdp)
                             erts_thr_progress_active(esdp, thr_prgr_active = 0);
                             sched_wall_time_change(esdp, 0);
                         }
+                        erts_thr_progress_prepare_wait(NULL);
                         suspend_normal_scheduler_sleep(esdp);
-                        current_time = erts_get_monotonic_time(esdp);;
+                        erts_thr_progress_finalize_wait(NULL);
+                        current_time = erts_get_monotonic_time(esdp);
                     }
 
                     if (current_time >= timeout_time) {
@@ -8493,17 +8411,17 @@ sched_dirty_io_thread_func(void *vesdp)
     return NULL;
 }
 
-static ethr_tid aux_tid;
-
 void
 erts_start_schedulers(void)
 {
+    ethr_tid tid;
     int res = 0;
     Uint actual;
     Uint wanted = erts_no_schedulers;
     Uint wanted_no_schedulers = erts_no_schedulers;
     char name[16];
     ethr_thr_opts opts = ETHR_THR_OPTS_DEFAULT_INITER;
+    int ix;
 
     opts.detached = 1;
 
@@ -8548,7 +8466,6 @@ erts_start_schedulers(void)
     erts_no_schedulers = actual;
 
     {
-	int ix;
 	for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
 	    ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
 	    erts_snprintf(opts.name, 16, "%d_dirty_cpu_scheduler", ix + 1);
@@ -8571,10 +8488,18 @@ erts_start_schedulers(void)
 
     erts_snprintf(opts.name, 16, "aux");
 
-    res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts);
+    res = ethr_thr_create(&tid, aux_thread, NULL, &opts);
     if (res != 0)
 	erts_exit(ERTS_ERROR_EXIT, "Failed to create aux thread\n");
 
+    for (ix = 0; ix < erts_no_poll_threads; ix++) {
+        erts_snprintf(opts.name, 16, "%d_poller", ix);
+
+        res = ethr_thr_create(&tid, poll_thread, (void*)(UWord)ix, &opts);
+        if (res != 0)
+            erts_exit(ERTS_ERROR_EXIT, "Failed to create poll thread\n");
+    }
+
     if (actual < 1)
 	erts_exit(ERTS_ERROR_EXIT,
 		 "Failed to create any scheduler-threads: %s (%d)\n",
@@ -9641,7 +9566,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
     ErtsRunQueue *rq;
     int context_reds;
     int fcalls;
-    int input_reductions;
     int actual_reds;
     int reds;
     Uint32 flags;
@@ -9661,11 +9585,9 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
     if (ERTS_USE_MODIFIED_TIMING()) {
 	context_reds = ERTS_MODIFIED_TIMING_CONTEXT_REDS;
-	input_reductions = ERTS_MODIFIED_TIMING_INPUT_REDS;
     }
     else {
 	context_reds = CONTEXT_REDS;
-	input_reductions = INPUT_REDUCTIONS;
     }
 
     ERTS_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())
@@ -9685,7 +9607,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	}
 	rq = erts_get_runq_current(esdp);
 	ASSERT(esdp);
-	fcalls = esdp->function_calls;
 	actual_reds = reds = 0;
 	erts_runq_lock(rq);
     } else {
@@ -9711,8 +9632,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST;
 	esdp->virtual_reds = 0;
 
-        esdp->function_calls += reds;
-	fcalls = esdp->function_calls;
 	ASSERT(esdp && esdp == erts_get_scheduler_data());
 
 	rq = erts_get_runq_current(esdp);
@@ -9802,16 +9721,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	ERTS_CHK_NO_PROC_LOCKS;
 
-	if (is_normal_sched) {
-	    if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS)
-		(void) erts_get_monotonic_time(esdp);
-
-	    if (esdp->last_monotonic_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
-		erts_runq_unlock(rq);
-		erts_bump_timers(esdp->timer_wheel, esdp->last_monotonic_time);
-		erts_runq_lock(rq);
-	    }
-	}
     }
 
     ERTS_LC_ASSERT(!is_normal_sched || !erts_thr_progress_is_blocking());
@@ -9822,6 +9731,16 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	ErtsMigrationPath *mp;
 
 	if (is_normal_sched) {
+
+	    if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS)
+		(void) erts_get_monotonic_time(esdp);
+
+	    if (esdp->last_monotonic_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
+		erts_runq_unlock(rq);
+		erts_bump_timers(esdp->timer_wheel, esdp->last_monotonic_time);
+		erts_runq_lock(rq);
+	    }
+
 	    if (rq->check_balance_reds <= 0)
 		check_balance(rq);
 
@@ -9951,37 +9870,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	    goto check_activities_to_run;
 	}
-	else if (is_normal_sched
-		 && (fcalls > input_reductions
-		     && prepare_for_sys_schedule(!0))) {
-	    ErtsMonotonicTime current_time;
-	    /*
-	     * Schedule system-level activities.
-	     */
-
-	    ERTS_MSACC_PUSH_STATE_CACHED_M();
-
-	    esdp->function_calls = 0;
-	    fcalls = 0;
-
-#if 0 /* Not needed since we wont wait in sys schedule */
-	    erts_check_io_interrupt(esdp->pollset, 0);
-#endif
-	    erts_runq_unlock(rq);
-
-            ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO);
-            LTTNG2(scheduler_poll, esdp->no, 1);
-
-	    erl_sys_schedule(1);
-	    ERTS_MSACC_POP_STATE_M();
-
-	    current_time = erts_get_monotonic_time(esdp);
-	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
-		erts_bump_timers(esdp->timer_wheel, current_time);
-
-	    erts_runq_lock(rq);
-	    goto continue_check_activities_to_run;
-	}
 
         if (flags & ERTS_RUNQ_FLG_MISC_OP)
 	    exec_misc_ops(rq);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index b372146846..b4c06adc95 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -369,6 +369,7 @@ struct ErtsSchedulerSleepInfo_ {
     ErtsSchedulerSleepInfo *prev;
     erts_atomic32_t flags;
     erts_tse_t *event;
+    struct erts_poll_thread *psi;
     erts_atomic32_t aux_work;
 };
 
@@ -632,8 +633,6 @@ struct ErtsSchedulerData_ {
     void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
     Process *free_process;
     ErtsThrPrgrData thr_progress_data;
-    struct pollset_info* pollset;
-    int function_calls;
     ErtsSchedulerSleepInfo *ssi;
     Process *current_process;
     ErtsSchedType type;
@@ -1534,9 +1533,7 @@ extern int erts_system_profile_ts_type;
 void erts_pre_init_process(void);
 void erts_late_init_process(void);
 void erts_early_init_scheduling(int);
-void erts_init_scheduling(int, int
-			  , int, int, int
-			  );
+void erts_init_scheduling(int, int, int, int, int, int);
 void erts_execute_dirty_system_task(Process *c_p);
 int erts_set_gc_state(Process *c_p, int enable);
 Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable,
@@ -2473,6 +2470,7 @@ void erts_notify_inc_runq(ErtsRunQueue *runq);
 
 void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
 ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);
+void erts_aux_thread_poke(void);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 82977e62ea..295130f60c 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -46,8 +46,6 @@
  */
 #define ERTS_X_REGS_ALLOCATED (MAX_REG+3)
 
-#define INPUT_REDUCTIONS (CONTEXT_REDS / 4)
-
 #define H_DEFAULT_SIZE  233        /* default (heap + stack) min size */
 #define VH_DEFAULT_SIZE  32768     /* default virtual (bin) heap min size (words) */
 #define H_DEFAULT_MAX_SIZE 0       /* default max heap size is off */
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index c3f8e92f13..09aeba00fa 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1101,7 +1101,6 @@ void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth);
 typedef struct {
     Eterm delay_time;
     int context_reds;
-    int input_reds;
 } ErtsModifiedTimings;
 
 extern Export *erts_delay_trap;
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 0bf2ecfc8b..9106091ca6 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -992,10 +992,6 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-#ifdef ERTS_ENABLE_KERNEL_POLL
-extern int erts_use_kernel_poll;
-#endif
-
 #define sys_memcpy(s1,s2,n)  memcpy(s1,s2,n)
 #define sys_memmove(s1,s2,n) memmove(s1,s2,n)
 #define sys_memcmp(s1,s2,n)  memcmp(s1,s2,n)
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index ff0f3ea121..3131b96536 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -29,7 +29,6 @@
 #endif
 
 #define ERL_CHECK_IO_C__
-#define ERTS_WANT_BREAK_HANDLING
 #ifndef WANT_NONBLOCKING
 #  define WANT_NONBLOCKING
 #endif
@@ -44,151 +43,108 @@
 #define ERTS_WANT_TIMER_WHEEL_API
 #include "erl_time.h"
 
+#if 0
+#define DEBUG_PRINT(FMT, ...) erts_printf(FMT "\r\n", ##__VA_ARGS__)
+#define DEBUG_PRINT_FD(FMT, STATE, ...)                                 \
+    DEBUG_PRINT("%d: " FMT " (ev=%s, ac=%s, flg=%d)",                   \
+                (STATE) ? (STATE)->fd : (ErtsSysFdType)-1, ##__VA_ARGS__, \
+                ev2str((STATE) ? (STATE)->events : ERTS_POLL_EV_NONE),  \
+                ev2str((STATE) ? (STATE)->active_events : ERTS_POLL_EV_NONE), \
+                (STATE) ? (STATE)->flags : ERTS_EV_FLAG_CLEAR)
+#define DEBUG_PRINT_MODE
+#else
+#define DEBUG_PRINT(...)
+#endif
+
+#ifndef DEBUG_PRINT_FD
+#define DEBUG_PRINT_FD(...)
+#endif
+
 #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
 #  include "safe_hash.h"
 #  define DRV_EV_STATE_HTAB_SIZE 1024
 #endif
 
-typedef char EventStateType;
-#define ERTS_EV_TYPE_NONE     ((EventStateType) 0)
-#define ERTS_EV_TYPE_DRV_SEL  ((EventStateType) 1) /* driver_select */
-#define ERTS_EV_TYPE_STOP_USE ((EventStateType) 2) /* pending stop_select */
-#define ERTS_EV_TYPE_NIF      ((EventStateType) 3) /* enif_select */
-#define ERTS_EV_TYPE_STOP_NIF ((EventStateType) 4) /* pending nif stop */
-
-typedef char EventStateFlags;
-#define ERTS_EV_FLAG_USED         ((EventStateFlags) 1)   /* ERL_DRV_USE has been turned on */
-#define ERTS_EV_FLAG_DEFER_IN_EV  ((EventStateFlags) 2)
-#define ERTS_EV_FLAG_DEFER_OUT_EV ((EventStateFlags) 4)
+typedef enum {
+    ERTS_EV_TYPE_NONE     = 0,
+    ERTS_EV_TYPE_DRV_SEL  = 1, /* driver_select */
+    ERTS_EV_TYPE_STOP_USE = 2, /* pending stop_select */
+    ERTS_EV_TYPE_NIF      = 3, /* enif_select */
+    ERTS_EV_TYPE_STOP_NIF = 4  /* pending nif stop */
+} EventStateType;
 
-#ifdef DEBUG
-#  define ERTS_ACTIVE_FD_INC 2
+typedef enum {
+    ERTS_EV_FLAG_CLEAR         = 0,
+    ERTS_EV_FLAG_USED          = 1,   /* ERL_DRV_USE has been turned on */
+#ifdef ERTS_ENABLE_KERNEL_POLL
+    ERTS_EV_FLAG_FALLBACK      = 2,   /* Set when kernel poll rejected fd
+                                         and it was put in the nkp version */
 #else
-#  define ERTS_ACTIVE_FD_INC 128
+    ERTS_EV_FLAG_FALLBACK      = ERTS_EV_FLAG_CLEAR,
 #endif
 
-#define ERTS_CHECK_IO_POLL_RES_LEN 512
-
-#if defined(ERTS_KERNEL_POLL_VERSION)
-#  define ERTS_CIO_EXPORT(FUNC) FUNC ## _kp
-#elif defined(ERTS_NO_KERNEL_POLL_VERSION)
-#  define ERTS_CIO_EXPORT(FUNC) FUNC ## _nkp
-#else
-#  define ERTS_CIO_EXPORT(FUNC) FUNC
-#endif
+    /* Combinations */
+    ERTS_EV_FLAG_USED_FALLBACK = ERTS_EV_FLAG_USED | ERTS_EV_FLAG_FALLBACK
+} EventStateFlags;
 
-#define ERTS_CIO_POLL_CTL	ERTS_POLL_EXPORT(erts_poll_control)
-#define ERTS_CIO_POLL_CTLV	ERTS_POLL_EXPORT(erts_poll_controlv)
-#define ERTS_CIO_POLL_WAIT	ERTS_POLL_EXPORT(erts_poll_wait)
-#define ERTS_CIO_POLL_INTR 	ERTS_POLL_EXPORT(erts_poll_interrupt)
-#define ERTS_CIO_POLL_INTR_TMD	ERTS_POLL_EXPORT(erts_poll_interrupt_timed)
-#define ERTS_CIO_NEW_POLLSET 	ERTS_POLL_EXPORT(erts_poll_create_pollset)
-#define ERTS_CIO_FREE_POLLSET	ERTS_POLL_EXPORT(erts_poll_destroy_pollset)
-#define ERTS_CIO_POLL_MAX_FDS	ERTS_POLL_EXPORT(erts_poll_max_fds)
-#define ERTS_CIO_POLL_INIT	ERTS_POLL_EXPORT(erts_poll_init)
-#define ERTS_CIO_POLL_INFO	ERTS_POLL_EXPORT(erts_poll_info)
+#define flag2str(flags)                                                 \
+    ((flags) == ERTS_EV_FLAG_CLEAR ? "CLEAR" :                          \
+     ((flags) == ERTS_EV_FLAG_USED ? "USED" :                           \
+      ((flags) == ERTS_EV_FLAG_FALLBACK ? "FLBK" :                      \
+       ((flags) == ERTS_EV_FLAG_USED_FALLBACK ? "USED|FLBK" : "ERROR"))))
 
-#define GET_FD(fd) fd
+/* How many events that can be handled at once by one erts_poll_wait call */
+#define ERTS_CHECK_IO_POLL_RES_LEN 512
 
-static struct pollset_info
+/* Each I/O Poll Thread has one ErtsPollThread each. The ps field
+   can point to either a private ErtsPollSet or a shared one.
+   At the moment only kqueue and epoll pollsets can be
+   shared across threads.
+*/
+typedef struct erts_poll_thread
 {
-    ErtsPollSet ps;
-    erts_atomic_t in_poll_wait;        /* set while doing poll */
-    erts_atomic_t check_io_time;
-    struct {
-	int six; /* start index */
-	int eix; /* end index */
-	erts_atomic32_t no;
-	int size;
-	ErtsSysFdType *array;
-    } active_fd;
-    erts_atomic_t removed_list;    /* struct removed_fd* */
-}*pollsetv;
-
-#define NUM_OF_POLLSETS 1
+    ErtsPollSet *ps;
+    ErtsPollResFd *pollres;
+    int pollres_len;
+} ErtsPollThread;
 
-#ifdef ERTS_ENABLE_KERNEL_POLL
-void erts_init_check_io_kp(void);
-void erts_init_check_io_nkp(void);
-int ERTS_CIO_EXPORT(driver_select)(ErlDrvPort, ErlDrvEvent, int, int);
-int ERTS_CIO_EXPORT(enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
-Uint ERTS_CIO_EXPORT(erts_check_io_size)(void);
-Eterm ERTS_CIO_EXPORT(erts_check_io_info)(void *);
-int ERTS_CIO_EXPORT(erts_check_io_max_files)(void);
-void ERTS_CIO_EXPORT(erts_check_io_interrupt)(struct pollset_info*, int);
-void ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(struct pollset_info*, int, ErtsMonotonicTime);
-void ERTS_CIO_EXPORT(erts_check_io)(int);
-struct pollset_info* ERTS_CIO_EXPORT(erts_get_pollset)(int);
-int ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *);
-void ERTS_CIO_EXPORT(erts_io_notify_port_task_executed)(ErtsPortTaskHandle *pthp);
-#ifdef ERTS_ENABLE_LOCK_COUNT
-void ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks)(int enable);
-#endif
+/* pollsetv contains pointers to the ErtsPollSets that are in use.
+ * Which pollset to use is determined by hashing the fd.
+ */
+static ErtsPollSet **pollsetv;
+#if ERTS_POLL_USE_FALLBACK
+static ErtsPollSet *flbk_pollset;
 #endif
-
-/* ToDo: Was inline in erl_check_io.h but now need struct pollset_info */
-void
-ERTS_CIO_EXPORT(erts_io_notify_port_task_executed)(ErtsPortTaskHandle *pthp)
-{
-    ErtsIoTask *itp = ErtsContainerStruct(pthp, ErtsIoTask, task);
-    erts_aint_t ci_time = erts_atomic_read_acqb(&itp->pollset->check_io_time);
-    erts_atomic_set_relb(&itp->executed_time, ci_time);
-}
-
-
-struct pollset_info* ERTS_CIO_EXPORT(erts_get_pollset)(int sched_nr)
-{
-    ASSERT(sched_nr > 0 && sched_nr <= erts_no_schedulers);
-    return &pollsetv[sched_nr - 1];
-}
+static ErtsPollThread *psiv;
 
 typedef struct {
 #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
     SafeHashBucket hb;
 #endif
     ErtsSysFdType fd;
-    struct pollset_info *pollset;
     struct {
 	ErtsDrvSelectDataState *select;   /* ERTS_EV_TYPE_DRV_SEL */
         ErtsNifSelectDataState *nif;      /* ERTS_EV_TYPE_NIF */
         union {
             erts_driver_t*  drv_ptr;    /* ERTS_EV_TYPE_STOP_USE */
             ErtsResource* resource;   /* ERTS_EV_TYPE_STOP_NIF */
-        }stop;
+        } stop;
     } driver;
-    ErtsPollEvents events;
-    unsigned short remove_cnt; /* number of removed_fd's referring to this fd */
+    ErtsPollEvents events; /* The events that have been selected upon */
+    ErtsPollEvents active_events; /* The events currently active in the pollset */
     EventStateType type;
     EventStateFlags flags;
 } ErtsDrvEventState;
 
-struct removed_fd {
-    struct removed_fd *next;
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    ErtsSysFdType fd;
-#else
-    ErtsDrvEventState* state;
-    #ifdef DEBUG
-    ErtsSysFdType fd;
-    #endif
-#endif
-
-};
-
 struct drv_ev_state_shared {
-    /* The layout of this struct must be independent of kp/nkp compilation */
-
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    int max_fds;
-#endif
 
-#define DRV_EV_STATE_LOCK_CNT 128
     union {
         erts_mtx_t lck;
         byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_mtx_t))];
-    }locks[DRV_EV_STATE_LOCK_CNT];
+    } locks[ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT];
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
+    int max_fds;
     erts_atomic_t len;
     ErtsDrvEventState *v;
     erts_mtx_t grow_lock; /* prevent lock-hogging of racing growers */
@@ -200,51 +156,121 @@ struct drv_ev_state_shared {
 #endif
 };
 
-#ifndef ERTS_KERNEL_POLL_VERSION
+int ERTS_WRITE_UNLIKELY(erts_no_pollsets) = 1;
+int ERTS_WRITE_UNLIKELY(erts_no_poll_threads) = 1;
 struct drv_ev_state_shared drv_ev_state;
-#else
-extern struct drv_ev_state_shared drv_ev_state;
-#endif
 
-static ERTS_INLINE erts_mtx_t* fd_mtx(ErtsSysFdType fd)
-{
+static ERTS_INLINE int fd_hash(ErtsSysFdType fd) {
     int hash = (int)fd;
 # ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
     hash ^= (hash >> 9);
 # endif
-    return &drv_ev_state.locks[hash % DRV_EV_STATE_LOCK_CNT].lck;
+    return hash;
 }
 
-#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
+static ERTS_INLINE erts_mtx_t* fd_mtx(ErtsSysFdType fd)
+{
+    return &drv_ev_state.locks[fd_hash(fd) % ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT].lck;
+}
+
+#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
 
-static ERTS_INLINE ErtsDrvEventState *hash_get_drv_ev_state(ErtsSysFdType fd)
+static ERTS_INLINE ErtsDrvEventState *get_drv_ev_state(ErtsSysFdType fd)
+{
+    return &drv_ev_state.v[(int) fd];
+}
+
+#define new_drv_ev_state(State, fd) (State)
+#define erase_drv_ev_state(State)
+
+static ERTS_INLINE int grow_drv_ev_state(ErtsSysFdType fd) {
+    int i;
+    int old_len;
+    int new_len;
+
+    if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state.len)) {
+
+	if (fd < 0 || fd >= drv_ev_state.max_fds)
+            return 0;
+
+        erts_mtx_lock(&drv_ev_state.grow_lock);
+        old_len = erts_atomic_read_nob(&drv_ev_state.len);
+        if (fd >= old_len) {
+            new_len = erts_poll_new_table_len(old_len, fd + 1);
+            if (new_len > drv_ev_state.max_fds)
+                new_len = drv_ev_state.max_fds;
+
+            for (i=0; i<ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT; i++) { /* lock all fd's */
+                erts_mtx_lock(&drv_ev_state.locks[i].lck);
+            }
+            drv_ev_state.v = (drv_ev_state.v
+                              ? erts_realloc(ERTS_ALC_T_DRV_EV_STATE,
+                                             drv_ev_state.v,
+                                             sizeof(ErtsDrvEventState)*new_len)
+                              : erts_alloc(ERTS_ALC_T_DRV_EV_STATE,
+                                           sizeof(ErtsDrvEventState)*new_len));
+            ERTS_CT_ASSERT(ERTS_EV_TYPE_NONE == 0);
+            sys_memzero(drv_ev_state.v+old_len,
+                        sizeof(ErtsDrvEventState) * (new_len - old_len));
+            for (i = old_len; i < new_len; i++) {
+                drv_ev_state.v[i].fd = (ErtsSysFdType) i;
+            }
+            erts_atomic_set_nob(&drv_ev_state.len, new_len);
+            for (i=0; i<ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT; i++) {
+                erts_mtx_unlock(&drv_ev_state.locks[i].lck);
+            }
+        }
+        /*else already grown by racing thread */
+
+        erts_mtx_unlock(&drv_ev_state.grow_lock);
+    }
+    return 1;
+}
+
+static int drv_ev_state_len(void)
+{
+    return erts_atomic_read_nob(&drv_ev_state.len);
+}
+
+#else /* !ERTS_SYS_CONTINOUS_FD_NUMBERS */
+
+static ERTS_INLINE ErtsDrvEventState *get_drv_ev_state(ErtsSysFdType fd)
 {
     ErtsDrvEventState tmpl;
     tmpl.fd = fd;
     return  (ErtsDrvEventState *) safe_hash_get(&drv_ev_state.tab, (void *) &tmpl);
 }
 
-static ERTS_INLINE ErtsDrvEventState* hash_new_drv_ev_state(ErtsSysFdType fd)
+static ERTS_INLINE ErtsDrvEventState* new_drv_ev_state(ErtsDrvEventState *state,
+                                                       ErtsSysFdType fd)
 {
     ErtsDrvEventState tmpl;
+
+    if (state)
+        return state;
+
     tmpl.fd = fd;
-    tmpl.pollset = NULL;
     tmpl.driver.select = NULL;
     tmpl.driver.nif = NULL;
     tmpl.driver.stop.drv_ptr = NULL;
     tmpl.events = 0;
-    tmpl.remove_cnt = 0;
+    tmpl.active_events = 0;
     tmpl.type = ERTS_EV_TYPE_NONE;
     tmpl.flags = 0;
+
     return  (ErtsDrvEventState *) safe_hash_put(&drv_ev_state.tab, (void *) &tmpl);
 }
 
-static ERTS_INLINE void hash_erase_drv_ev_state(ErtsDrvEventState *state)
+static ERTS_INLINE void erase_drv_ev_state(ErtsDrvEventState *state)
 {
-    ASSERT(state->remove_cnt == 0);
     safe_hash_erase(&drv_ev_state.tab, (void *) state);
 }
 
+static int drv_ev_state_len(void)
+{
+    return erts_atomic_read_nob(&drv_ev_state.tab.nitems);
+}
+
 #endif /* !ERTS_SYS_CONTINOUS_FD_NUMBERS */
 
 static void stale_drv_select(Eterm id, ErtsDrvEventState *state, int mode);
@@ -268,36 +294,39 @@ steal_pending_stop_use(erts_dsprintf_buf_t*, ErlDrvPort, ErtsDrvEventState*,
 static void
 steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource*,
                        ErtsDrvEventState *state, int mode, int on);
-
-ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(removed_fd, struct removed_fd, 64, ERTS_ALC_T_FD_LIST)
+static ERTS_INLINE void
+check_fd_cleanup(ErtsDrvEventState *state,
+		 ErtsDrvSelectDataState **free_select,
+                 ErtsNifSelectDataState **free_nif);
+#ifdef DEBUG_PRINT_MODE
+static char *drvmode2str(int mode);
+static char *nifmode2str(enum ErlNifSelectFlags mode);
+#endif
 
 static ERTS_INLINE void
-init_iotask(ErtsIoTask *io_task, struct pollset_info* psi)
+init_iotask(ErtsIoTask *io_task, ErtsSysFdType fd)
 {
     erts_port_task_handle_init(&io_task->task);
-    erts_atomic_init_nob(&io_task->executed_time, ~((erts_aint_t) 0));
-    io_task->pollset = psi;
+    io_task->fd = fd;
 }
 
 static ERTS_INLINE int
-is_iotask_active(ErtsIoTask *io_task, erts_aint_t current_cio_time)
-{    
+is_iotask_active(ErtsIoTask *io_task)
+{
     if (erts_port_task_is_scheduled(&io_task->task))
 	return 1;
-    if (erts_atomic_read_nob(&io_task->executed_time) == current_cio_time)
-	return 1;
     return 0;
 }
 
 static ERTS_INLINE ErtsDrvSelectDataState *
-alloc_drv_select_data(struct pollset_info* psi)
+alloc_drv_select_data(ErtsSysFdType fd)
 {
     ErtsDrvSelectDataState *dsp = erts_alloc(ERTS_ALC_T_DRV_SEL_D_STATE,
 					     sizeof(ErtsDrvSelectDataState));
     dsp->inport = NIL;
     dsp->outport = NIL;
-    init_iotask(&dsp->iniotask, psi);
-    init_iotask(&dsp->outiotask, psi);
+    init_iotask(&dsp->iniotask, fd);
+    init_iotask(&dsp->outiotask, fd);
     return dsp;
 }
 
@@ -308,8 +337,6 @@ alloc_nif_select_data(void)
 					     sizeof(ErtsNifSelectDataState));
     dsp->in.pid = NIL;
     dsp->out.pid = NIL;
-    dsp->in.ddeselect_cnt = 0;
-    dsp->out.ddeselect_cnt = 0;
     return dsp;
 }
 
@@ -327,176 +354,126 @@ free_nif_select_data(ErtsNifSelectDataState *dsp)
     erts_free(ERTS_ALC_T_NIF_SEL_D_STATE, dsp);
 }
 
-static ERTS_INLINE void
-remember_removed(ErtsDrvEventState *state)
+static ERTS_INLINE int
+get_pollset_id(ErtsSysFdType fd)
 {
-    struct removed_fd *fdlp;
-    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd)));
-    if (erts_atomic_read_nob(&state->pollset->in_poll_wait)) {
-        erts_aint_t was_next, exp_next;
-	state->remove_cnt++;
-	ASSERT(state->remove_cnt > 0);
-	fdlp = removed_fd_alloc();
-    #if defined(ERTS_SYS_CONTINOUS_FD_NUMBERS) || defined(DEBUG)
-    	fdlp->fd = state->fd;
-    #endif
-    #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
-	fdlp->state = state;
-    #endif
-
-         /* Lockless atomic insertion in removed_list */
-        was_next = erts_atomic_read_acqb(&state->pollset->removed_list);
-        do {
-            exp_next = was_next;
-            fdlp->next = (struct removed_fd*) exp_next;
-            was_next = erts_atomic_cmpxchg_mb(&state->pollset->removed_list,
-                                              (erts_aint_t) fdlp,
-                                              exp_next);
-        }while (was_next != exp_next);
-    }
+    return fd_hash(fd) % erts_no_pollsets;
 }
 
+static ERTS_INLINE ErtsPollSet *
+get_pollset(ErtsSysFdType fd)
+{
+    return pollsetv[get_pollset_id(fd)];
+}
 
-static ERTS_INLINE int
-is_removed(ErtsDrvEventState *state)
+#if ERTS_POLL_USE_FALLBACK
+static ERTS_INLINE ErtsPollSet *
+get_fallback(void)
 {
-    /* Note that there is a possible race here, where an fd is removed
-       (increasing remove_cnt) and then added again just before erts_poll_wait
-       is called by erts_check_io. Any polled event on the re-added fd will then
-       be falsely ignored. But that does not matter, as the event will trigger
-       again next time erl_check_io is called. */
-    return state->remove_cnt > 0;
+    return flbk_pollset;
 }
+#endif
 
-static void
-forget_removed(struct pollset_info* psi)
+/*
+ * Place a fd within a pollset. This will automatically use
+ * the fallback ps if needed.
+ */
+static ERTS_INLINE ErtsPollEvents
+erts_io_control_wakeup(ErtsDrvEventState *state, ErtsPollOp op,
+                       ErtsPollEvents pe, int *wake_poller)
 {
-    struct removed_fd* fdlp;
-    struct removed_fd* tofree;
+    ErtsSysFdType fd = state->fd;
+    ErtsPollEvents res = 0;
+    EventStateFlags flags = state->flags;
 
-    fdlp = (struct removed_fd*) erts_atomic_xchg_mb(&psi->removed_list,
-                                                    (erts_aint_t) NULL);
+    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd)));
 
-    while (fdlp) {
-        ErtsResource* resource = NULL;
-	erts_driver_t* drv_ptr = NULL;
-	erts_mtx_t* mtx;
-	ErtsSysFdType fd;
-	ErtsDrvEventState *state;
+    if (!(flags & ERTS_EV_FLAG_FALLBACK)) {
+        res = erts_poll_control(get_pollset(fd), fd, op, pe, wake_poller);
 
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-	fd = fdlp->fd;
-	mtx = fd_mtx(fd);
-	erts_mtx_lock(mtx);
-	state = &drv_ev_state.v[(int) fd];
-#else
-	state = fdlp->state;
-	fd = state->fd;
-	ASSERT(fd == fdlp->fd);
-	mtx = fd_mtx(fd);
-	erts_mtx_lock(mtx);
-#endif
-	ASSERT(state->remove_cnt > 0);
-	if (--state->remove_cnt == 0) {
-	    switch (state->type) {
-	    case ERTS_EV_TYPE_STOP_NIF:
-		/* Now we can call stop */
-		resource = state->driver.stop.resource;
-                state->driver.stop.resource = NULL;
-                ASSERT(resource);
-		state->type = ERTS_EV_TYPE_NONE;
-		state->flags = 0;
-                goto case_ERTS_EV_TYPE_NONE;
-
-	    case ERTS_EV_TYPE_STOP_USE:
-		/* Now we can call stop_select */
-		drv_ptr = state->driver.stop.drv_ptr;
-		ASSERT(drv_ptr);
-		state->type = ERTS_EV_TYPE_NONE;
-		state->flags = 0;
-		state->driver.stop.drv_ptr = NULL;
-		/* Fall through */
-            case ERTS_EV_TYPE_NONE:
-            case_ERTS_EV_TYPE_NONE:
-                state->pollset = NULL;
-#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
-		hash_erase_drv_ev_state(state);
-#endif
-		break;
-	    case ERTS_EV_TYPE_DRV_SEL:
-		break;
-	    default:
-		ASSERT(0);
-	    }
-	}
-	erts_mtx_unlock(mtx);
-	if (drv_ptr) {
-	    int was_unmasked = erts_block_fpe();
-	    DTRACE1(driver_stop_select, drv_ptr->name);
-	    LTTNG1(driver_stop_select, drv_ptr->name);
-	    (*drv_ptr->stop_select) ((ErlDrvEvent) fd, NULL);
-	    erts_unblock_fpe(was_unmasked);
-	    if (drv_ptr->handle) {
-		erts_ddll_dereference_driver(drv_ptr->handle);
-	    }
-	}
-        if (resource) {
-            erts_resource_stop(resource, (ErlNifEvent)fd, 0);
-            enif_release_resource(resource->data);
+#if ERTS_POLL_USE_FALLBACK
+        if (op == ERTS_POLL_OP_ADD && res == ERTS_POLL_EV_NVAL) {
+            /* When an add fails with NVAL, the poll/kevent operation could not
+               put that fd in the pollset, so we instead put it into a fallback pollset */
+            state->flags |= ERTS_EV_FLAG_FALLBACK;
+            res = erts_poll_control_flbk(get_fallback(), fd, op, pe, wake_poller);
         }
-
-	tofree = fdlp;
-	fdlp = fdlp->next;
-	removed_fd_free(tofree);
+    } else {
+        ASSERT(op != ERTS_POLL_OP_ADD);
+        res = erts_poll_control_flbk(get_fallback(), fd, op, pe, wake_poller);
+#endif
     }
+
+    return res;
 }
 
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-static void
-grow_drv_ev_state(int min_ix)
+static ERTS_INLINE ErtsPollEvents
+erts_io_control(ErtsDrvEventState *state, ErtsPollOp op, ErtsPollEvents pe)
 {
-    int i;
-    int old_len;
-    int new_len;
+    int wake_poller = 0;
+    return erts_io_control_wakeup(state, op, pe, &wake_poller);
+}
 
-    erts_mtx_lock(&drv_ev_state.grow_lock);
-    old_len = erts_atomic_read_nob(&drv_ev_state.len);
-    if (min_ix >= old_len) {
-        new_len = erts_poll_new_table_len(old_len, min_ix + 1);
-        if (new_len > drv_ev_state.max_fds)
-            new_len = drv_ev_state.max_fds;
-
-	for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { /* lock all fd's */
-	    erts_mtx_lock(&drv_ev_state.locks[i].lck);
-	}
-	drv_ev_state.v = (drv_ev_state.v
-			? erts_realloc(ERTS_ALC_T_DRV_EV_STATE,
-				       drv_ev_state.v,
-				       sizeof(ErtsDrvEventState)*new_len)
-			: erts_alloc(ERTS_ALC_T_DRV_EV_STATE,
-				     sizeof(ErtsDrvEventState)*new_len));
-	for (i = old_len; i < new_len; i++) {
-	    drv_ev_state.v[i].fd = (ErtsSysFdType) i;
-            drv_ev_state.v[i].pollset = NULL;
-	    drv_ev_state.v[i].driver.select = NULL;
-	    drv_ev_state.v[i].driver.stop.drv_ptr = NULL;
-            drv_ev_state.v[i].driver.nif = NULL;
-	    drv_ev_state.v[i].events = 0;
-	    drv_ev_state.v[i].remove_cnt = 0;
-	    drv_ev_state.v[i].type = ERTS_EV_TYPE_NONE;
-	    drv_ev_state.v[i].flags = 0;
-	}
-	erts_atomic_set_nob(&drv_ev_state.len, new_len);
-	for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) {
-	    erts_mtx_unlock(&drv_ev_state.locks[i].lck);
-	}
+/* ToDo: Was inline in erl_check_io.h but now need struct erts_poll_thread */
+void
+erts_io_notify_port_task_executed(ErtsPortTaskType type,
+                                  ErtsPortTaskHandle *pthp,
+                                  void (*reset_handle)(ErtsPortTaskHandle *))
+{
+    ErtsIoTask *itp = ErtsContainerStruct(pthp, ErtsIoTask, task);
+    ErtsSysFdType fd = itp->fd;
+    erts_mtx_t *mtx = fd_mtx(fd);
+    int active_events;
+    ErtsDrvEventState *state;
+    ErtsDrvSelectDataState *free_select = NULL;
+    ErtsNifSelectDataState *free_nif = NULL;
+
+    erts_mtx_lock(mtx);
+    state = get_drv_ev_state(fd);
+
+    active_events = state->active_events;
+
+    switch (type) {
+    case ERTS_PORT_TASK_INPUT:
+
+        DEBUG_PRINT_FD("executed ready_input", state);
+
+        ASSERT(!(state->active_events & ERTS_POLL_EV_IN));
+        if (state->events & ERTS_POLL_EV_IN)
+            active_events |= ERTS_POLL_EV_IN;
+        break;
+    case ERTS_PORT_TASK_OUTPUT:
+
+        DEBUG_PRINT_FD("executed ready_output", state);
+
+        ASSERT(!(state->active_events & ERTS_POLL_EV_OUT));
+        if (state->events & ERTS_POLL_EV_OUT)
+            active_events |= ERTS_POLL_EV_OUT;
+        break;
+    default:
+        erts_exit(ERTS_ABORT_EXIT, "Invalid IO port task type");
+        break;
     }
-    /*else already grown by racing thread */
 
-    erts_mtx_unlock(&drv_ev_state.grow_lock);
-}
-#endif /* ERTS_SYS_CONTINOUS_FD_NUMBERS */
+    reset_handle(pthp);
+
+    if (active_events) {
+        /* This is not needed if active_events has not changed */
+        if (state->active_events != active_events) {
+            state->active_events = active_events;
+            erts_io_control(state, ERTS_POLL_OP_MOD, active_events);
+        }
+    } else {
+        check_fd_cleanup(state, &free_select, &free_nif);
+    }
 
+    erts_mtx_unlock(mtx);
+
+    if (free_select)
+        free_drv_select_data(free_select);
+    if (free_nif)
+        free_nif_select_data(free_nif);
+}
 
 static ERTS_INLINE void
 abort_task(Eterm id, ErtsPortTaskHandle *pthp, EventStateType type)
@@ -542,16 +519,14 @@ abort_tasks(ErtsDrvEventState *state, int mode)
 static void
 deselect(ErtsDrvEventState *state, int mode)
 {
-    int do_wake = 0;
     ErtsPollEvents rm_events;
     ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd)));
-    ASSERT(state->events);
 
     abort_tasks(state, mode);
 
-    if (!mode)
+    if (!mode) {
 	rm_events = state->events;
-    else {
+    } else {
 	rm_events = 0;
 	ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL);
 	if (mode & ERL_DRV_READ) {
@@ -564,18 +539,16 @@ deselect(ErtsDrvEventState *state, int mode)
 	}
     }
 
-    ERTS_CIO_POLL_CTL(state->pollset->ps, state->fd, rm_events,
-                      0, &do_wake);
     state->events &= ~rm_events;
+    state->active_events &= ~rm_events;
 
     if (!(state->events)) {
+        erts_io_control(state, ERTS_POLL_OP_DEL, 0);
 	switch (state->type) {
         case ERTS_EV_TYPE_NIF:
             state->driver.nif->in.pid = NIL;
             state->driver.nif->out.pid = NIL;
-            state->driver.nif->in.ddeselect_cnt = 0;
-            state->driver.nif->out.ddeselect_cnt = 0;
-            enif_release_resource(state->driver.stop.resource);
+            enif_release_resource(state->driver.stop.resource->data);
             state->driver.stop.resource = NULL;
             break;
 	case ERTS_EV_TYPE_DRV_SEL:
@@ -588,15 +561,15 @@ deselect(ErtsDrvEventState *state, int mode)
 	    ASSERT(0);
 	    break;
 	}
-	    
 	state->type = ERTS_EV_TYPE_NONE;
 	state->flags = 0;
-	remember_removed(state);
+    } else {
+        erts_io_control(state, ERTS_POLL_OP_MOD, state->active_events);
     }
 }
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-#  define IS_FD_UNKNOWN(state) ((state)->type == ERTS_EV_TYPE_NONE && (state)->remove_cnt == 0)
+#  define IS_FD_UNKNOWN(state) ((state)->type == ERTS_EV_TYPE_NONE)
 #else
 #  define IS_FD_UNKNOWN(state) ((state) == NULL)
 #endif
@@ -606,17 +579,13 @@ check_fd_cleanup(ErtsDrvEventState *state,
 		 ErtsDrvSelectDataState **free_select,
                  ErtsNifSelectDataState **free_nif)
 {
-    erts_aint_t current_cio_time;
-
     ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd)));
-
-    current_cio_time = erts_atomic_read_acqb(&state->pollset->check_io_time);
     *free_select = NULL;
     if (state->driver.select
 	&& (state->type != ERTS_EV_TYPE_DRV_SEL)
-	&& !is_iotask_active(&state->driver.select->iniotask, current_cio_time)
-	&& !is_iotask_active(&state->driver.select->outiotask, current_cio_time)) {
-	
+	&& !is_iotask_active(&state->driver.select->iniotask)
+	&& !is_iotask_active(&state->driver.select->outiotask)) {
+
 	*free_select = state->driver.select;
 	state->driver.select = NULL;
     }
@@ -628,14 +597,10 @@ check_fd_cleanup(ErtsDrvEventState *state,
     }
 
     if (((state->type != ERTS_EV_TYPE_NONE)
-	 | state->remove_cnt
          | (state->driver.nif != NULL)
 	 | (state->driver.select != NULL)) == 0) {
 
-        state->pollset = NULL;
-#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
-	hash_erase_drv_ev_state(state);
-#endif
+	erase_drv_ev_state(state);
     }
 }
 
@@ -645,253 +610,8 @@ check_fd_cleanup(ErtsDrvEventState *state,
 # define MUST_DEFER(MAY_SLEEP) (MAY_SLEEP)
 #endif
 
-static ERTS_INLINE int
-check_cleanup_active_fd(struct pollset_info* psi,
-                        ErtsSysFdType fd,
-#if ERTS_CIO_DEFER_ACTIVE_EVENTS
-			ErtsPollControlEntry *pce,
-			int *pce_ix,
-#endif
-			erts_aint_t current_cio_time,
-                        int may_sleep)
-{
-    ErtsDrvEventState *state;
-    int active = 0;
-    erts_mtx_t *mtx = fd_mtx(fd);
-    void *free_select = NULL;
-    void *free_nif = NULL;
-#if ERTS_CIO_DEFER_ACTIVE_EVENTS
-    ErtsPollEvents evon = 0, evoff = 0;
-#endif
-
-    erts_mtx_lock(mtx);
-
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    state = &drv_ev_state.v[(int) fd];
-#else
-    state = hash_get_drv_ev_state(fd); /* may be NULL! */
-#endif
-    if (state && state->pollset == psi)
-    {
-	if (state->driver.select) {
-#if ERTS_CIO_DEFER_ACTIVE_EVENTS
-	    if (is_iotask_active(&state->driver.select->iniotask, current_cio_time)) {
-		active = 1;
-		if (MUST_DEFER(may_sleep)
-                    && (state->events & ERTS_POLL_EV_IN)
-		    && !(state->flags & ERTS_EV_FLAG_DEFER_IN_EV)) {
-		    evoff |= ERTS_POLL_EV_IN;
-		    state->flags |= ERTS_EV_FLAG_DEFER_IN_EV;
-		}
-	    }
-	    else if (state->flags & ERTS_EV_FLAG_DEFER_IN_EV) {
-		if (state->events & ERTS_POLL_EV_IN)
-		    evon |= ERTS_POLL_EV_IN;
-		state->flags &= ~ERTS_EV_FLAG_DEFER_IN_EV;
-	    }
-	    if (is_iotask_active(&state->driver.select->outiotask, current_cio_time)) {
-		active = 1;
-		if (MUST_DEFER(may_sleep)
-                    && (state->events & ERTS_POLL_EV_OUT)
-		    && !(state->flags & ERTS_EV_FLAG_DEFER_OUT_EV)) {
-		    evoff |= ERTS_POLL_EV_OUT;
-		    state->flags |= ERTS_EV_FLAG_DEFER_OUT_EV;
-		}
-	    }
-	    else if (state->flags & ERTS_EV_FLAG_DEFER_OUT_EV) {
-		if (state->events & ERTS_POLL_EV_OUT) {
-		    evon |= ERTS_POLL_EV_OUT;
-                }
-		state->flags &= ~ERTS_EV_FLAG_DEFER_OUT_EV;
-	    }
-	    if (active)
-		(void) 0;
-	    else
-#else
-	    if (is_iotask_active(&state->driver.select->iniotask, current_cio_time)
-		|| is_iotask_active(&state->driver.select->outiotask, current_cio_time))
-		active = 1;
-	    else
-#endif
-		if (state->type != ERTS_EV_TYPE_DRV_SEL) {
-		free_select = state->driver.select;
-		state->driver.select = NULL;
-	    }
-#if ERTS_CIO_DEFER_ACTIVE_EVENTS
-             if (evon) {
-                 int do_wake = 0;
-                 ERTS_CIO_POLL_CTL(psi->ps, state->fd, evon, 1, &do_wake);
-             }
-#endif
-	}
-
-        if (state->driver.nif) {
-            ErtsPollEvents rm_events = 0;
-            if (state->driver.nif->in.ddeselect_cnt) {
-                ASSERT(state->type == ERTS_EV_TYPE_NIF);
-                ASSERT(state->events & ERTS_POLL_EV_IN);
-                ASSERT(is_nil(state->driver.nif->in.pid));
-                if (may_sleep || state->driver.nif->in.ddeselect_cnt == 1) {
-                    rm_events = ERTS_POLL_EV_IN;
-                    state->driver.nif->in.ddeselect_cnt = 0;
-                }
-            }
-            if (state->driver.nif->out.ddeselect_cnt) {
-                ASSERT(state->type == ERTS_EV_TYPE_NIF);
-                ASSERT(state->events & ERTS_POLL_EV_OUT);
-                ASSERT(is_nil(state->driver.nif->out.pid));
-                if (may_sleep || state->driver.nif->out.ddeselect_cnt == 1) {
-                    rm_events |= ERTS_POLL_EV_OUT;
-                    state->driver.nif->out.ddeselect_cnt = 0;
-                }
-            }
-            if (rm_events) {
-                int do_wake = 0;
-                state->events = ERTS_CIO_POLL_CTL(state->pollset->ps, state->fd,
-                                                  rm_events, 0, &do_wake);
-            }
-            if (state->events)
-                active = 1;
-            else if (state->type != ERTS_EV_TYPE_NIF) {
-                free_nif = state->driver.nif;
-                state->driver.nif = NULL;
-            }
-        }
-
-#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
-	if (((state->type != ERTS_EV_TYPE_NONE) | state->remove_cnt | active) == 0)
-	    hash_erase_drv_ev_state(state);
-#endif
-    }
-
-    erts_mtx_unlock(mtx);
-
-    if (free_select)
-	free_drv_select_data(free_select);
-    if (free_nif)
-        free_nif_select_data(free_nif);
-
-#if ERTS_CIO_DEFER_ACTIVE_EVENTS
-    if (evoff) {
-	ErtsPollControlEntry *pcep = &pce[(*pce_ix)++];
-	pcep->fd = fd;
-	pcep->events = evoff;
-	pcep->on = 0;
-    }
-#endif
-
-    return active;
-}
-
-static void
-check_cleanup_active_fds(struct pollset_info* psi,
-                         erts_aint_t current_cio_time, int may_sleep)
-{
-    int six = psi->active_fd.six;
-    int eix = psi->active_fd.eix;
-    erts_aint32_t no = erts_atomic32_read_dirty(&psi->active_fd.no);
-    const int size = psi->active_fd.size;
-    int ix = six;
-#if ERTS_CIO_DEFER_ACTIVE_EVENTS
-    /* every fd might add one entry */
-    Uint pce_sz = sizeof(ErtsPollControlEntry)*no;
-    ErtsPollControlEntry *pctrl_entries = (pce_sz
-					   ? erts_alloc(ERTS_ALC_T_TMP, pce_sz)
-					   : NULL);
-    int pctrl_ix = 0;
-#endif
-
-    while (ix != eix) {
-	ErtsSysFdType fd = psi->active_fd.array[ix];
-	int nix = ix + 1;
-	if (nix >= size)
-	    nix = 0;
-	ASSERT(fd != ERTS_SYS_FD_INVALID);
-	if (!check_cleanup_active_fd(psi, fd,
-#if ERTS_CIO_DEFER_ACTIVE_EVENTS
-				     pctrl_entries,
-				     &pctrl_ix,
-#endif
-				     current_cio_time,
-                                     may_sleep)) {
-	    no--;
-	    if (ix == six) {
-#ifdef DEBUG
-		psi->active_fd.array[ix] = ERTS_SYS_FD_INVALID;
-#endif
-		six = nix;
-	    }
-	    else {
-		psi->active_fd.array[ix] = psi->active_fd.array[six];
-#ifdef DEBUG
-		psi->active_fd.array[six] = ERTS_SYS_FD_INVALID;
-#endif
-		six++;
-		if (six >= size)
-		    six = 0;
-	    }
-	}
-	ix = nix;
-    }
-
-#if ERTS_CIO_DEFER_ACTIVE_EVENTS
-    ASSERT(pctrl_ix <= pce_sz/sizeof(ErtsPollControlEntry));
-    if (pctrl_ix)
-	ERTS_CIO_POLL_CTLV(psi->ps, pctrl_entries, pctrl_ix);
-    if (pctrl_entries)
-	erts_free(ERTS_ALC_T_TMP, pctrl_entries);
-#endif
-
-    psi->active_fd.six = six;
-    psi->active_fd.eix = eix;
-    erts_atomic32_set_relb(&psi->active_fd.no, no);
-}
-
-static void grow_active_fds(struct pollset_info *psi)
-{
-    ASSERT(psi->active_fd.six == psi->active_fd.eix);
-    psi->active_fd.six = 0;
-    psi->active_fd.eix = psi->active_fd.size;
-    psi->active_fd.size += ERTS_ACTIVE_FD_INC;
-    psi->active_fd.array = erts_realloc(ERTS_ALC_T_ACTIVE_FD_ARR,
-                                           psi->active_fd.array,
-                                           psi->active_fd.size*sizeof(ErtsSysFdType));
-#ifdef DEBUG
-    {
-        int i;
-        for (i = psi->active_fd.eix + 1; i < psi->active_fd.size; i++)
-            psi->active_fd.array[i] = ERTS_SYS_FD_INVALID;
-    }
-#endif
-}
-
-static ERTS_INLINE void
-add_active_fd(struct pollset_info *psi, ErtsSysFdType fd)
-{
-    int eix = psi->active_fd.eix;
-    const int size = psi->active_fd.size;
-    
-    psi->active_fd.array[eix] = fd;
-
-    erts_atomic32_set_relb(&psi->active_fd.no,
-			       (erts_atomic32_read_dirty(&psi->active_fd.no)
-				+ 1));
-
-    eix++;
-    if (eix >= size)
-	eix = 0;
-    psi->active_fd.eix = eix;
-
-    if (psi->active_fd.six == eix) {
-        grow_active_fds(psi);
-    }
-}
-
 int
-ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
-			       ErlDrvEvent e,
-			       int mode,
-			       int on)
+driver_select(ErlDrvPort ix, ErlDrvEvent e, int mode, int on)
 {
     void (*stop_select_fn)(ErlDrvEvent, void*) = NULL;
     Port *prt = erts_drvport2port(ix);
@@ -899,6 +619,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
     ErtsSysFdType fd = (ErtsSysFdType) e;
     ErtsPollEvents ctl_events = (ErtsPollEvents) 0;
     ErtsPollEvents old_events;
+    ErtsPollOp ctl_op = ERTS_POLL_OP_MOD;
     ErtsDrvEventState *state;
     int wake_poller = 0;
     int ret;
@@ -907,32 +628,29 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
 #ifdef USE_VM_PROBES
     DTRACE_CHARBUF(name, 64);
 #endif
+    ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_CHECK_IO);
 
-    if (prt == ERTS_INVALID_ERL_DRV_PORT)
+    if (prt == ERTS_INVALID_ERL_DRV_PORT) {
+        ERTS_MSACC_POP_STATE();
 	return -1;
+    }
 
     ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state.len)) {
-	if (fd < 0) {
-	    return -1;    
-	}
-	if (fd >= drv_ev_state.max_fds) {
-	    drv_select_large_fd_error(ix, fd, mode, on);
-	    return -1;
-	}
-	grow_drv_ev_state(fd);
+    if (!grow_drv_ev_state(fd)) {
+        if (fd > 0) drv_select_large_fd_error(ix, fd, mode, on);
+        ERTS_MSACC_POP_STATE();
+        return -1;
     }
 #endif
 
     erts_mtx_lock(fd_mtx(fd));
 
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    state = &drv_ev_state.v[(int) fd];
-#else
-    state = hash_get_drv_ev_state(fd); /* may be NULL! */
-#endif
+    state = get_drv_ev_state(fd); /* may be NULL! */
+
+    DEBUG_PRINT_FD("driver_select(%T, %p, %s, %d)",
+                   state, id, fd, drvmode2str(mode), on);
 
     if (!on) {
         if (IS_FD_UNKNOWN(state)) {
@@ -950,15 +668,10 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
         }
         else if ((mode&ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) {
             mode |= (ERL_DRV_READ | ERL_DRV_WRITE);
-            wake_poller = 1; /* to eject fd from pollset (if needed) */
         }
     }
 
-#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    if (state == NULL) {
-	state = hash_new_drv_ev_state(fd);
-    }
-#endif
+    state = new_drv_ev_state(state, fd);
 
     switch (state->type) {
     case ERTS_EV_TYPE_NIF:
@@ -982,7 +695,9 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
         ASSERT(state->type == ERTS_EV_TYPE_NONE);
         break;
 
-    }}
+    }
+    default: break;
+    }
 
     if (mode & ERL_DRV_READ) {
 	if (state->type == ERTS_EV_TYPE_DRV_SEL) {
@@ -999,7 +714,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
 		drv_select_steal(ix, state, mode, on);
 	}
 	ctl_events |= ERTS_POLL_EV_OUT;
-    }	
+    }
 
 
     ASSERT((state->type == ERTS_EV_TYPE_DRV_SEL) ||
@@ -1010,32 +725,31 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
     if (on) {
         ctl_events &= ~old_events;
         state->events |= ctl_events;
+        if (ctl_events & ERTS_POLL_EV_IN && (!state->driver.select || !is_iotask_active(&state->driver.select->iniotask)))
+            state->active_events |= ERTS_POLL_EV_IN;
+        if (ctl_events & ERTS_POLL_EV_OUT && (!state->driver.select || !is_iotask_active(&state->driver.select->outiotask)))
+            state->active_events |= ERTS_POLL_EV_OUT;
+        if (old_events == 0 && !(state->flags & ERTS_EV_FLAG_USED)) {
+            ctl_op = ERTS_POLL_OP_ADD;
+        }
     }
     else {
         ctl_events &= old_events;
         state->events &= ~ctl_events;
+        state->active_events &= ~ctl_events;
 
-        if (!(state->flags & ERTS_EV_FLAG_USED)
-            && old_events && !state->events) {
-            /*
-             * Old driver removing all events. At least wake poller.
-             * It will not make close() 100% safe but it will prevent
-             * actions delayed by poll timeout.
-             */
-            wake_poller = 1;
+        if (!state->events) {
+            if (!(state->flags & ERTS_EV_FLAG_USED) || mode & ERL_DRV_USE)
+                ctl_op = ERTS_POLL_OP_DEL;
         }
     }
 
-    if (ctl_events) {
+    if (ctl_events || ctl_op == ERTS_POLL_OP_DEL) {
         ErtsPollEvents new_events;
 
-        if (!state->pollset) {
-            ErtsSchedulerData* esdp = erts_get_scheduler_data();
-            ASSERT(esdp);
-            state->pollset = esdp->pollset;
-        }
-
-        new_events = ERTS_CIO_POLL_CTL(state->pollset->ps, state->fd, ctl_events, on, &wake_poller);
+        new_events = erts_io_control_wakeup(state, ctl_op,
+                                            state->active_events,
+                                            &wake_poller);
 
         if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
             if (state->type == ERTS_EV_TYPE_DRV_SEL && !old_events) {
@@ -1048,14 +762,13 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
             goto done;
         }
 
-        ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL
-               || state->type == ERTS_EV_TYPE_NONE);
+        ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL || state->type == ERTS_EV_TYPE_NONE);
     }
 
     if (on) {
         if (ctl_events) {
 	    if (!state->driver.select)
-		state->driver.select = alloc_drv_select_data(state->pollset);
+		state->driver.select = alloc_drv_select_data(state->fd);
 	    if (state->type == ERTS_EV_TYPE_NONE)
 		state->type = ERTS_EV_TYPE_DRV_SEL;
 	    ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL);
@@ -1069,47 +782,44 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
         }
     }
     else { /* off */
-	    if (state->type == ERTS_EV_TYPE_DRV_SEL) {
-		if (ctl_events & ERTS_POLL_EV_IN) {
-		    abort_tasks(state, ERL_DRV_READ);
-		    state->driver.select->inport = NIL;
-		}
-		if (ctl_events & ERTS_POLL_EV_OUT) {
-		    abort_tasks(state, ERL_DRV_WRITE);
-		    state->driver.select->outport = NIL;
-		}
-		if (state->events == 0) {
-		    if (old_events != 0) {
-			remember_removed(state);
-		    }		    
-		    if ((mode & ERL_DRV_USE) || !(state->flags & ERTS_EV_FLAG_USED)) {
-			state->type = ERTS_EV_TYPE_NONE;
-			state->flags = 0;
-		    }
-		    /*else keep it, as fd will probably be selected upon again */
-		}
-	    }
-	    if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) {
-		erts_driver_t* drv_ptr = prt->drv_ptr;
-		ASSERT(state->events==0);
-		if (state->remove_cnt == 0 || !wake_poller) {
-		    /* Safe to close fd now as it is not in pollset
-		       or there was no need to eject fd (kernel poll) */
-		    stop_select_fn = drv_ptr->stop_select;
+        if (state->type == ERTS_EV_TYPE_DRV_SEL) {
+            if (ctl_events & ERTS_POLL_EV_IN) {
+                abort_tasks(state, ERL_DRV_READ);
+                state->driver.select->inport = NIL;
+            }
+            if (ctl_events & ERTS_POLL_EV_OUT) {
+                abort_tasks(state, ERL_DRV_WRITE);
+                state->driver.select->outport = NIL;
+            }
+            if (state->events == 0) {
+                if ((mode & ERL_DRV_USE) || !(state->flags & ERTS_EV_FLAG_USED)) {
+                    state->type = ERTS_EV_TYPE_NONE;
+                    state->flags = 0;
+                }
+                /*else keep it, as fd will probably be selected upon again */
+            }
+        }
+        if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) {
+            erts_driver_t* drv_ptr = prt->drv_ptr;
+            ASSERT(state->events==0);
+            if (!wake_poller) {
+                /* Safe to close fd now as it is not in pollset
+                   or there was no need to eject fd (kernel poll) */
+                stop_select_fn = drv_ptr->stop_select;
 #ifdef USE_VM_PROBES
-		    strncpy(name, prt->drv_ptr->name, sizeof(name)-1);
-		    name[sizeof(name)-1] = '\0';
+                strncpy(name, prt->drv_ptr->name, sizeof(name)-1);
+                name[sizeof(name)-1] = '\0';
 #endif
-		}
-		else {
-		    /* Not safe to close fd, postpone stop_select callback. */
-		    state->type = ERTS_EV_TYPE_STOP_USE;
-		    state->driver.stop.drv_ptr = drv_ptr;
-		    if (drv_ptr->handle) {
-			erts_ddll_reference_referenced_driver(drv_ptr->handle);
-		    }
-		}
-	    }
+            }
+            else {
+                /* Not safe to close fd, postpone stop_select callback. */
+                state->type = ERTS_EV_TYPE_STOP_USE;
+                state->driver.stop.drv_ptr = drv_ptr;
+                if (drv_ptr->handle) {
+                    erts_ddll_reference_referenced_driver(drv_ptr->handle);
+                }
+            }
+        }
     }
    
     ret = 0;
@@ -1134,54 +844,47 @@ done_unknown:
     if (free_nif)
         free_nif_select_data(free_nif);
 
+    ERTS_MSACC_POP_STATE();
+
     return ret;
 }
 
 int
-ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
-                             ErlNifEvent e,
-                             enum ErlNifSelectFlags mode,
-                             void* obj,
-                             const ErlNifPid* pid,
-                             Eterm ref)
+enif_select(ErlNifEnv* env,
+            ErlNifEvent e,
+            enum ErlNifSelectFlags mode,
+            void* obj,
+            const ErlNifPid* pid,
+            Eterm ref)
 {
     int on;
     ErtsResource* resource = DATA_TO_RESOURCE(obj);
     ErtsSysFdType fd = (ErtsSysFdType) e;
     ErtsPollEvents ctl_events = (ErtsPollEvents) 0;
     ErtsPollEvents old_events;
+    ErtsPollOp ctl_op = ERTS_POLL_OP_MOD;
     ErtsDrvEventState *state;
-    int wake_poller;
-    int ret;
+    int ret, wake_poller = 0;
     enum { NO_STOP=0, CALL_STOP, CALL_STOP_AND_RELEASE } call_stop = NO_STOP;
     ErtsDrvSelectDataState *free_select = NULL;
     ErtsNifSelectDataState *free_nif = NULL;
-#ifdef USE_VM_PROBES
-    DTRACE_CHARBUF(name, 64);
-#endif
 
     ASSERT(!(resource->monitors && resource->monitors->is_dying));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state.len)) {
-	if (fd < 0) {
-	    return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT;
-	}
-	if (fd >= drv_ev_state.max_fds) {
-	    nif_select_large_fd_error(fd, mode, resource, ref);
-            return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT;
-	}
-	grow_drv_ev_state(fd);
+    if (!grow_drv_ev_state(fd)) {
+        if (fd > 0) nif_select_large_fd_error(fd, mode, resource, ref);
+        return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT;
     }
 #endif
 
     erts_mtx_lock(fd_mtx(fd));
 
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    state = &drv_ev_state.v[(int) fd];
-#else
-    state = hash_get_drv_ev_state(fd); /* may be NULL! */
-#endif
+    state = get_drv_ev_state(fd); /* may be NULL! */
+
+    DEBUG_PRINT_FD("enif_select(%T, %d, %s, %p, %T, %T)",
+                   state, env->proc->common.id, fd, nifmode2str(mode), resource,
+                   pid ? pid->pid : THE_NON_VALUE, ref);
 
     if (mode & ERL_NIF_SELECT_STOP) {
         ASSERT(resource->type->stop);
@@ -1193,13 +896,12 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
         }
         on = 0;
         mode = ERL_DRV_READ | ERL_DRV_WRITE | ERL_DRV_USE;
-        wake_poller = 1; /* to eject fd from pollset (if needed) */
         ctl_events = ERTS_POLL_EV_IN | ERTS_POLL_EV_OUT;
+        ctl_op = ERTS_POLL_OP_DEL;
     }
     else {
         on = 1;
         ASSERT(mode);
-        wake_poller = 0;
         if (mode & ERL_DRV_READ) {
             ctl_events |= ERTS_POLL_EV_IN;
         }
@@ -1208,11 +910,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
         }
     }
 
-#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    if (state == NULL) {
-	state = hash_new_drv_ev_state(fd);
-    }
-#endif
+    state = new_drv_ev_state(state,fd);
 
     switch (state->type) {
     case ERTS_EV_TYPE_NIF:
@@ -1243,7 +941,9 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
         }
         ASSERT(state->type == ERTS_EV_TYPE_NONE);
         break;
-    }}
+    }
+    default: break;
+    }
 
     ASSERT((state->type == ERTS_EV_TYPE_NIF) ||
 	   (state->type == ERTS_EV_TYPE_NONE && !state->events));
@@ -1253,20 +953,22 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
     if (on) {
         ctl_events &= ~old_events;
         state->events |= ctl_events;
+        state->active_events |= ctl_events;
+        if (state->type == ERTS_EV_TYPE_NONE)
+            ctl_op = ERTS_POLL_OP_ADD;
     }
     else {
         ctl_events &= old_events;
         state->events &= ~ctl_events;
+        state->active_events &= ~ctl_events;
     }
 
-    if (ctl_events) {
+    if (ctl_events || ctl_op == ERTS_POLL_OP_DEL) {
         ErtsPollEvents new_events;
 
-        if (!state->pollset) {
-            state->pollset = erts_get_scheduler_data()->pollset;
-        }
-
-        new_events = ERTS_CIO_POLL_CTL(state->pollset->ps, state->fd, ctl_events, on, &wake_poller);
+        new_events = erts_io_control_wakeup(state, ctl_op,
+                                            state->active_events,
+                                            &wake_poller);
 
         if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
             if (state->type == ERTS_EV_TYPE_NIF && !old_events) {
@@ -1274,8 +976,6 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
                 state->flags = 0;
                 state->driver.nif->in.pid = NIL;
                 state->driver.nif->out.pid = NIL;
-                state->driver.nif->in.ddeselect_cnt = 0;
-                state->driver.nif->out.ddeselect_cnt = 0;
                 state->driver.stop.resource = NULL;
             }
             ret = INT_MIN | ERL_NIF_SELECT_FAILED;
@@ -1307,11 +1007,9 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
                 ASSERT(is_internal_ref(ref));
                 refn = internal_ref_numbers(ref);
                 state->driver.nif->in.immed = THE_NON_VALUE;
-                state->driver.nif->in.refn[0] = refn[0];
-                state->driver.nif->in.refn[1] = refn[1];
-                state->driver.nif->in.refn[2] = refn[2];
+                sys_memcpy(state->driver.nif->in.refn, refn,
+                           sizeof(state->driver.nif->in.refn));
             }
-            state->driver.nif->in.ddeselect_cnt = 0;
         }
         if (mode & ERL_DRV_WRITE) {
             state->driver.nif->out.pid = recipient;
@@ -1321,11 +1019,9 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
                 ASSERT(is_internal_ref(ref));
                 refn = internal_ref_numbers(ref);
                 state->driver.nif->out.immed = THE_NON_VALUE;
-                state->driver.nif->out.refn[0] = refn[0];
-                state->driver.nif->out.refn[1] = refn[1];
-                state->driver.nif->out.refn[2] = refn[2];
+                sys_memcpy(state->driver.nif->out.refn, refn,
+                           sizeof(state->driver.nif->out.refn));
             }
-            state->driver.nif->out.ddeselect_cnt = 0;
         }
         ret = 0;
     }
@@ -1333,14 +1029,9 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env,
         if (state->type == ERTS_EV_TYPE_NIF) {
             state->driver.nif->in.pid = NIL;
             state->driver.nif->out.pid = NIL;
-            state->driver.nif->in.ddeselect_cnt = 0;
-            state->driver.nif->out.ddeselect_cnt = 0;
-            if (old_events != 0) {
-                remember_removed(state);
-            }
         }
         ASSERT(state->events==0);
-        if (state->remove_cnt == 0 || !wake_poller) {
+        if (!wake_poller) {
             /*
              * Safe to close fd now as it is not in pollset
              * or there was no need to eject fd (kernel poll)
@@ -1455,7 +1146,7 @@ print_driver_name(erts_dsprintf_buf_t *dsbufp, Eterm id)
 static void
 steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode)
 {
-    erts_dsprintf(dsbufp, "stealing control of fd=%d from ", (int) GET_FD(state->fd));
+    erts_dsprintf(dsbufp, "stealing control of fd=%d from ", (int) state->fd);
     switch (state->type) {
     case ERTS_EV_TYPE_DRV_SEL: {
 	int deselect_mode = 0;
@@ -1479,7 +1170,7 @@ steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode)
 	if (deselect_mode)
 	    deselect(state, deselect_mode);
 	else {
-	    erts_dsprintf(dsbufp, "no one", (int) GET_FD(state->fd));
+	    erts_dsprintf(dsbufp, "no one", (int) state->fd);
 	    ASSERT(0);
 	}
 	erts_dsprintf(dsbufp, "\n");
@@ -1510,7 +1201,7 @@ steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode)
 	break;
     }
     default:
-	erts_dsprintf(dsbufp, "no one\n", (int) GET_FD(state->fd));
+	erts_dsprintf(dsbufp, "no one\n", (int) state->fd);
 	ASSERT(0);
     }
 }
@@ -1524,7 +1215,7 @@ print_drv_select_op(erts_dsprintf_buf_t *dsbufp,
 		  "driver_select(%p, %d,%s%s%s%s, %d) "
 		  "by ",
 		  ix,
-		  (int) GET_FD(fd),
+		  (int) fd,
 		  mode & ERL_DRV_READ ? " ERL_DRV_READ" : "",
 		  mode & ERL_DRV_WRITE ? " ERL_DRV_WRITE" : "",
 		  mode & ERL_DRV_USE ? " ERL_DRV_USE" : "",
@@ -1541,7 +1232,7 @@ print_nif_select_op(erts_dsprintf_buf_t *dsbufp,
 {
     erts_dsprintf(dsbufp,
 		  "enif_select(_, %d,%s%s%s, %T:%T, %T) ",
-		  (int) GET_FD(fd),
+		  (int) fd,
 		  mode & ERL_NIF_SELECT_READ ? " READ" : "",
 		  mode & ERL_NIF_SELECT_WRITE ? " WRITE" : "",
 		  mode & ERL_NIF_SELECT_STOP ? " STOP" : "",
@@ -1673,7 +1364,7 @@ steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource* resource,
         erts_dsprintf(dsbufp, "called before stop was called for NIF resource %T:%T\n",
                       rt->module, rt->name);
 
-        enif_release_resource(state->driver.stop.resource);
+        enif_release_resource(state->driver.stop.resource->data);
         state->type = ERTS_EV_TYPE_NONE;
         state->flags = 0;
         state->driver.stop.resource = NULL;
@@ -1687,8 +1378,7 @@ steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource* resource,
 
 static ERTS_INLINE int
 io_task_schedule_allowed(ErtsDrvEventState *state,
-			 ErtsPortTaskType type,
-			 erts_aint_t current_cio_time)
+			 ErtsPortTaskType type)
 {
     ErtsIoTask *io_task;
 
@@ -1708,42 +1398,41 @@ io_task_schedule_allowed(ErtsDrvEventState *state,
 	return 0;
     }
 
-    return !is_iotask_active(io_task, current_cio_time);
+    return !is_iotask_active(io_task);
 }
 
 static ERTS_INLINE void
-iready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time)
+iready(Eterm id, ErtsDrvEventState *state)
 {
     if (io_task_schedule_allowed(state,
-				 ERTS_PORT_TASK_INPUT,
-				 current_cio_time)) {
+				 ERTS_PORT_TASK_INPUT)) {
 	ErtsIoTask *iotask = &state->driver.select->iniotask;
-	erts_atomic_set_nob(&iotask->executed_time, current_cio_time);
 	if (erts_port_task_schedule(id,
 				    &iotask->task,
 				    ERTS_PORT_TASK_INPUT,
 				    (ErlDrvEvent) state->fd) != 0) {
 	    stale_drv_select(id, state, ERL_DRV_READ);
-	}
-	add_active_fd(state->pollset, state->fd);
+	} else {
+            DEBUG_PRINT_FD("schedule ready_input(%T, %d)",
+                           state, id, state->fd);
+        }
     }
 }
 
 static ERTS_INLINE void
-oready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time)
+oready(Eterm id, ErtsDrvEventState *state)
 {
     if (io_task_schedule_allowed(state,
-				 ERTS_PORT_TASK_OUTPUT,
-				 current_cio_time)) {
+				 ERTS_PORT_TASK_OUTPUT)) {
 	ErtsIoTask *iotask = &state->driver.select->outiotask;
-	erts_atomic_set_nob(&iotask->executed_time, current_cio_time);
 	if (erts_port_task_schedule(id,
 				    &iotask->task,
 				    ERTS_PORT_TASK_OUTPUT,
 				    (ErlDrvEvent) state->fd) != 0) {
 	    stale_drv_select(id, state, ERL_DRV_WRITE);
-	}
-	add_active_fd(state->pollset, state->fd);
+	} else {
+            DEBUG_PRINT_FD("schedule ready_output(%T, %d)", state, id, state->fd);
+        }
     }
 }
 
@@ -1798,91 +1487,56 @@ send_event_tuple(struct erts_nif_select_event* e, ErtsResource* resource,
 static void bad_fd_in_pollset(ErtsDrvEventState *, Eterm inport, Eterm outport);
 
 void
-ERTS_CIO_EXPORT(erts_check_io_interrupt)(struct pollset_info *psi, int set)
+erts_check_io_interrupt(ErtsPollThread *psi, int set)
 {
-    ERTS_CIO_POLL_INTR(psi->ps, set);
+    if (psi) {
+#if ERTS_POLL_USE_FALLBACK
+        if (psi->ps == get_fallback()) {
+            erts_poll_interrupt_flbk(psi->ps, set);
+            return;
+        }
+#endif
+        erts_poll_interrupt(psi->ps, set);
+    }
 }
 
-void
-ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(struct pollset_info *psi,
-                                               int set,
-					       ErtsMonotonicTime timeout_time)
-{
-    ERTS_CIO_POLL_INTR_TMD(psi->ps, set, timeout_time);
+ErtsPollThread *
+erts_create_pollset_thread(int id) {
+    return psiv+id;
 }
 
-#ifndef __WIN32__
-/*
- * Number of ignored events, for a lingering fd added by enif_select(),
- * until we deselect fd-event from pollset.
- */
-# define ERTS_NIF_DELAYED_DESELECT 20
-#else
-/* Disable delayed deselect as pollset cannot handle active events */
-# define ERTS_NIF_DELAYED_DESELECT 1
-#endif
-
 void
-ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
+erts_check_io(ErtsPollThread *psi)
 {
-    ErtsPollResFd *pollres;
     int pollres_len;
-    ErtsMonotonicTime timeout_time;
     int poll_ret, i;
-    erts_aint_t current_cio_time;
-    ErtsSchedulerData *esdp = erts_get_scheduler_data();
-    struct pollset_info *psi = esdp->pollset;
+    ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_CHECK_IO);
 
  restart:
 
-#ifdef ERTS_BREAK_REQUESTED
-    if (ERTS_BREAK_REQUESTED)
-	erts_do_break_handling();
-#endif
-
-    /* Figure out timeout value */
-    timeout_time = (do_wait
-		    ? erts_check_next_timeout_time(esdp)
-		    : ERTS_POLL_NO_TIMEOUT /* poll only */);
-
-    /*
-     * No need for an atomic inc op when incrementing
-     * erts_check_io_time, since only one thread can
-     * check io at a time.
-     */
-    current_cio_time = erts_atomic_read_dirty(&psi->check_io_time);
-    current_cio_time++;
-    erts_atomic_set_relb(&psi->check_io_time, current_cio_time);
-
-    check_cleanup_active_fds(psi,
-                             current_cio_time,
-                             timeout_time != ERTS_POLL_NO_TIMEOUT);
-
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_check_exact(NULL, 0); /* No locks should be locked */
 #endif
 
-    pollres_len = erts_atomic32_read_dirty(&psi->active_fd.no) + ERTS_CHECK_IO_POLL_RES_LEN;
+    pollres_len = psi->pollres_len;
 
-    pollres = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollResFd)*pollres_len);
+#if ERTS_POLL_USE_FALLBACK
+    if (psi->ps == get_fallback()) {
 
-    erts_atomic_set_nob(&psi->in_poll_wait, 1);
+        poll_ret = erts_poll_wait_flbk(psi->ps, psi->pollres, &pollres_len);
 
-    poll_ret = ERTS_CIO_POLL_WAIT(psi->ps, pollres, &pollres_len, timeout_time);
+    } else
+#endif
+    {
+        poll_ret = erts_poll_wait(psi->ps, psi->pollres, &pollres_len);
+    }
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_check_exact(NULL, 0); /* No locks should be locked */
 #endif
 
-#ifdef ERTS_BREAK_REQUESTED
-    if (ERTS_BREAK_REQUESTED)
-	erts_do_break_handling();
-#endif
-
     if (poll_ret != 0) {
-	erts_atomic_set_nob(&psi->in_poll_wait, 0);
-	forget_removed(psi);
-	erts_free(ERTS_ALC_T_TMP, pollres);
+
 	if (poll_ret == EAGAIN) {
 	    goto restart;
 	}
@@ -1898,64 +1552,78 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 			  erl_errno_id(poll_ret), poll_ret);
 	    erts_send_error_to_logger_nogl(dsbufp);
 	}
+        ERTS_MSACC_POP_STATE();
 	return;
     }
 
     for (i = 0; i < pollres_len; i++) {
 
-	ErtsSysFdType fd = (ErtsSysFdType) pollres[i].fd;
+        erts_driver_t* drv_ptr = NULL;
+        ErtsResource* resource = NULL;
+        ErtsDrvSelectDataState *free_select = NULL;
+        ErtsNifSelectDataState *free_nif = NULL;
+	ErtsSysFdType fd = (ErtsSysFdType) ERTS_POLL_RES_GET_FD(&psi->pollres[i]);
 	ErtsDrvEventState *state;
+        ErtsPollEvents revents;
 
 	erts_mtx_lock(fd_mtx(fd));
 
+	state = get_drv_ev_state(fd);
 
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-	state = &drv_ev_state.v[ (int) fd];
-#else
-	state = hash_get_drv_ev_state(fd);
 	if (!state) {
-	    goto next_pollres;
+            erts_mtx_unlock(fd_mtx(fd));
+            continue;
 	}
-#endif
 
-	/* Skip this fd if it was removed from pollset */
-	if (is_removed(state) || state->pollset != psi) {
-	    goto next_pollres;
-	}
+        revents = ERTS_POLL_RES_GET_EVTS(&psi->pollres[i]);
+
+        DEBUG_PRINT_FD("triggered %s", state, ev2str(revents));
+
+        if (revents & ERTS_POLL_EV_ERR) {
+            /*
+             * Handle error events by triggering all in/out events
+             * that has been selected on.
+             * We *do not* want to call a callback that corresponds
+             * to an event not selected.
+             */
+            revents = state->active_events;
+            state->active_events = 0;
+        } else {
+
+            /* Disregard any events that are not active at the moment,
+               for instance this could happen if the driver/nif does
+               select/deselect in rapid succession. */
+            revents &= state->active_events | ERTS_POLL_EV_NVAL;
+            state->active_events &= ~revents;
+
+            /* Reactivate the poll op if there are still active events */
+            if (state->active_events) {
+                DEBUG_PRINT_FD("re-enable %s", state, ev2str(state->active_events));
+                erts_io_control(state, ERTS_POLL_OP_MOD, state->active_events);
+            }
+        }
 
 	switch (state->type) {
 	case ERTS_EV_TYPE_DRV_SEL: { /* Requested via driver_select()... */
-	    ErtsPollEvents revents = pollres[i].events;
-
-            if (revents & ERTS_POLL_EV_ERR) {
-                /*
-                 * Handle error events by triggering all in/out events
-                 * that the driver has selected.
-                 * We *do not* want to call a callback that corresponds
-		 * to an event not selected.
-                 */
-                revents = state->events;
-            }
-            else {
-                revents &= (state->events | ERTS_POLL_EV_NVAL);
-            }
 
             if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) {
 		if (revents & ERTS_POLL_EV_OUT) {
-		    oready(state->driver.select->outport, state, current_cio_time);
+		    oready(state->driver.select->outport, state);
 		}
 		/* Someone might have deselected input since revents
-		   was read therefore, update revents... */
-		revents &= state->events;
+		   was read (true also on the non-smp emulator since
+		   oready() may have been called); therefore, update
+		   revents... */
+                revents &= state->events;
 		if (revents & ERTS_POLL_EV_IN) {
-		    iready(state->driver.select->inport, state, current_cio_time);
+		    iready(state->driver.select->inport, state);
 		}
 	    }
 	    else if (revents & ERTS_POLL_EV_NVAL) {
 		bad_fd_in_pollset(state,
                                   state->driver.select->inport,
                                   state->driver.select->outport);
-		add_active_fd(psi, state->fd);
+                check_fd_cleanup(state, &free_select, &free_nif);
 	    }
 	    break;
 	}
@@ -1964,89 +1632,116 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
             struct erts_nif_select_event in = {NIL};
             struct erts_nif_select_event out = {NIL};
             ErtsResource* resource = NULL;
-            ErtsPollEvents revents = pollres[i].events;
-
-            if (revents & ERTS_POLL_EV_ERR) {
-                /*
-                 * Handle error events by triggering all in/out events
-                 * that the NIF has selected.
-                 * We *do not* want to send a message that corresponds
-		 * to an event not selected.
-                 */
-                revents = state->events;
-            }
-            else {
-                revents &= (state->events | ERTS_POLL_EV_NVAL);
-            }
 
             if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) {
                 if (revents & ERTS_POLL_EV_OUT) {
                     if (is_not_nil(state->driver.nif->out.pid)) {
                         out = state->driver.nif->out;
                         resource = state->driver.stop.resource;
-                        state->driver.nif->out.ddeselect_cnt = ERTS_NIF_DELAYED_DESELECT;
                         state->driver.nif->out.pid = NIL;
-                        add_active_fd(psi, state->fd);
-                    }
-                    else {
-                        ASSERT(state->driver.nif->out.ddeselect_cnt >= 2);
-                        state->driver.nif->out.ddeselect_cnt--;
                     }
                 }
                 if (revents & ERTS_POLL_EV_IN) {
                     if (is_not_nil(state->driver.nif->in.pid)) {
                         in = state->driver.nif->in;
                         resource = state->driver.stop.resource;
-                        state->driver.nif->in.ddeselect_cnt = ERTS_NIF_DELAYED_DESELECT;
                         state->driver.nif->in.pid = NIL;
-                        add_active_fd(psi, state->fd);
-                    }
-                    else {
-                        ASSERT(state->driver.nif->in.ddeselect_cnt >= 2);
-                        state->driver.nif->in.ddeselect_cnt--;
                     }
                 }
+                state->events &= ~revents;
             }
             else if (revents & ERTS_POLL_EV_NVAL) {
                 bad_fd_in_pollset(state, NIL, NIL);
-                add_active_fd(psi, state->fd);
+                check_fd_cleanup(state, &free_select, &free_nif);
             }
 
             erts_mtx_unlock(fd_mtx(fd));
+
             if (is_not_nil(in.pid)) {
                 send_event_tuple(&in, resource, am_ready_input);
             }
             if (is_not_nil(out.pid)) {
                 send_event_tuple(&out, resource, am_ready_output);
             }
-            goto next_pollres_unlocked;
+            continue;
         }
 
+        case ERTS_EV_TYPE_STOP_NIF: {
+            resource = state->driver.stop.resource;
+            state->type = ERTS_EV_TYPE_NONE;
+            goto case_ERTS_EV_TYPE_NONE;
+        }
+
+        case ERTS_EV_TYPE_STOP_USE: {
+#if ERTS_POLL_USE_FALLBACK
+            ASSERT(psi->ps == get_fallback());
+#endif
+            drv_ptr = state->driver.stop.drv_ptr;
+            state->type = ERTS_EV_TYPE_NONE;
+            /* fallthrough */
 	case ERTS_EV_TYPE_NONE: /* Deselected ... */
+        case_ERTS_EV_TYPE_NONE:
+            ASSERT(!state->events && !state->active_events && !state->flags);
+            check_fd_cleanup(state, &free_select, &free_nif);
 	    break;
+        }
 
 	default: { /* Error */
 	    erts_dsprintf_buf_t *dsbufp;
 	    dsbufp = erts_create_logger_dsbuf();
 	    erts_dsprintf(dsbufp,
 			  "Invalid event request type for fd in erts_poll()! "
-			  "fd=%d, event request type=%sd\n", (int) state->fd,
+			  "fd=%d, event request type=%d\n", (int) state->fd,
 			  (int) state->type);
 	    ASSERT(0);
 	    deselect(state, 0);
-	    add_active_fd(psi, state->fd);
 	    break;
 	}
 	}
 
-	next_pollres:;
 	erts_mtx_unlock(fd_mtx(fd));
-        next_pollres_unlocked:;
-    }
 
-    erts_atomic_set_nob(&psi->in_poll_wait, 0);
-    erts_free(ERTS_ALC_T_TMP, pollres);
-    forget_removed(psi);
+        if (drv_ptr) {
+            int was_unmasked = erts_block_fpe();
+            DTRACE1(driver_stop_select, drv_ptr->name);
+            LTTNG1(driver_stop_select, drv_ptr->name);
+            (*drv_ptr->stop_select)((ErlDrvEvent) fd, NULL);
+            erts_unblock_fpe(was_unmasked);
+            if (drv_ptr->handle) {
+		erts_ddll_dereference_driver(drv_ptr->handle);
+	    }
+        }
+        if (resource) {
+            erts_resource_stop(resource, (ErlNifEvent)fd, 1);
+            enif_release_resource(resource->data);
+        }
+        if (free_select)
+            free_drv_select_data(free_select);
+        if (free_nif)
+            free_nif_select_data(free_nif);
+    }
+
+    /* The entire pollres array was filled with events,
+     * grow it for the next call. We do this for two reasons:
+     * 1. Pulling out more events in on go will increase throughput
+     * 2. If the polling implementation is not fair, this will make
+     *    sure that we get all fds that we can. i.e. if 12 fds are
+     *    constantly active, but we only have a pollres_len of 10,
+     *    two of the fds may never be triggered depending on what the
+     *    kernel decides to do.
+     **/
+    if (pollres_len == psi->pollres_len) {
+        int ev_state_len = drv_ev_state_len();
+        erts_free(ERTS_ALC_T_POLLSET, psi->pollres);
+        psi->pollres_len *= 2;
+        /* Never grow it larger than the current drv_ev_state.len size */
+        if (psi->pollres_len > ev_state_len)
+            psi->pollres_len = ev_state_len;
+        psi->pollres = erts_alloc(ERTS_ALC_T_POLLSET,
+                                  sizeof(ErtsPollResFd) * psi->pollres_len);
+    }
+
+    ERTS_MSACC_POP_STATE();
 }
 
 static void
@@ -2167,98 +1862,189 @@ static void drv_ev_state_free(void *des)
 }
 #endif
 
-#ifdef ERTS_ENABLE_KERNEL_POLL
+#define ERTS_MAX_NO_OF_POLL_THREADS ERTS_MAX_NO_OF_SCHEDULERS
 
-struct io_functions {
-    int (*select)(ErlDrvPort, ErlDrvEvent, int, int);
-    int (*enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm);
-    void (*check_io_as_interrupt)(struct pollset_info*);
-    void (*check_io_interrupt)(struct pollset_info*, int);
-    void (*check_io_interrupt_tmd)(struct pollset_info*, int, ErtsMonotonicTime);
-    void (*check_io)(int);
-    struct pollset_info* (*get_pollset)(int sched_nr);
-    void (*notify_port_task_executed)(ErtsPortTaskHandle *pthp);
-    int (*max_files)(void);
-    Uint (*size)(void);
-    Eterm (*info)(void *);
-    int (*check_io_debug)(ErtsCheckIoDebugInfo *);
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    void (*lcnt_update_cio_locks)(int enable);
-#endif
-};
+static char *
+get_arg(char* rest, char** argv, int* ip)
+{
+    int i = *ip;
+    if (*rest == '\0') {
+	if (argv[i+1] == NULL) {
+	    erts_fprintf(stderr, "too few arguments\n");
+	    erts_usage();
+	}
+        argv[i++] = NULL;
+        rest = argv[i];
+    }
+    argv[i] = NULL;
+    *ip = i;
+    return rest;
+}
 
-# ifdef ERTS_KERNEL_POLL_VERSION
-struct io_functions erts_io_funcs = {0};
-# else
-extern struct io_functions erts_io_funcs;
-# endif
+static void
+parse_args(int *argc, char **argv, int concurrent_waiters)
+{
+    int i = 0, j;
+    int no_pollsets = 0, no_poll_threads = 0,
+        no_pollsets_percentage = 0,
+        no_poll_threads_percentage = 0;
+    ASSERT(argc && argv);
+    while (i < *argc) {
+	if(argv[i][0] == '-') {
+	    switch (argv[i][1]) {
+            case 'I': {
+                if (strncmp(argv[i]+2, "Ot", 2) == 0) {
+                    char *arg = get_arg(argv[i]+4, argv, &i);
+                    if (sscanf(arg, "%d", &no_poll_threads) != 1 ||
+                        no_poll_threads < 1 ||
+                        ERTS_MAX_NO_OF_POLL_THREADS < no_poll_threads) {
+                        erts_fprintf(stderr,"bad I/O poll threads number: %s\n", arg);
+                        erts_usage();
+                    }
+                } else if (strncmp(argv[i]+2, "Op", 3) == 0) {
+                    char *arg = get_arg(argv[i]+4, argv, &i);
+                    if (sscanf(arg, "%d", &no_pollsets) != 1 ||
+                        no_pollsets < 1) {
+                        erts_fprintf(stderr,"bad I/O pollset number: %s\n", arg);
+                        erts_usage();
+                    }
+                } else if (strncmp(argv[i]+2, "OPt", 4) == 0) {
+                    char *arg = get_arg(argv[i]+5, argv, &i);
+                    if (sscanf(arg, "%d", &no_poll_threads_percentage) != 1 ||
+                        no_poll_threads_percentage < 0 ||
+                        no_poll_threads_percentage > 100) {
+                        erts_fprintf(stderr,"bad I/O poll thread percentage number: %s\n", arg);
+                        erts_usage();
+                    }
+                } else if (strncmp(argv[i]+2, "OPp", 4) == 0) {
+                    char *arg = get_arg(argv[i]+5, argv, &i);
+                    if (sscanf(arg, "%d", &no_pollsets_percentage) != 1 ||
+                        no_pollsets_percentage < 0 ||
+                        no_pollsets_percentage > 100) {
+                        erts_fprintf(stderr,"bad I/O pollset percentage number: %s\n", arg);
+                        erts_usage();
+                    }
+                } else {
+                    break;
+                }
+                break;
+            }
+            case 'K':
+                (void)get_arg(argv[i]+2, argv, &i);
+                break;
+            case '-':
+                goto args_parsed;
+            default:
+                break;
+            }
+        }
+        i++;
+    }
+
+args_parsed:
 
-#endif /* ERTS_ENABLE_KERNEL_POLL */
+    if (!concurrent_waiters) {
+        no_pollsets = no_poll_threads;
+        no_pollsets_percentage = 100;
+    }
+
+    if (no_poll_threads == 0) {
+        if (no_poll_threads_percentage == 0)
+            no_poll_threads = 1; /* This is the default */
+        else {
+            no_poll_threads = erts_no_schedulers * no_poll_threads_percentage / 100;
+            if (no_poll_threads < 1)
+                no_poll_threads = 1;
+        }
+    }
+
+    if (no_pollsets == 0) {
+        if (no_pollsets_percentage == 0)
+            no_pollsets = 1; /* This is the default */
+        else {
+            no_pollsets = no_poll_threads * no_pollsets_percentage / 100;
+            if (no_pollsets < 1)
+                no_pollsets = 1;
+        }
+    }
+
+    if (no_poll_threads < no_pollsets) {
+        erts_fprintf(stderr,
+                     "number of IO poll threads has to be greater or equal to "
+                     "the number of \nIO pollsets. Current values are set to: \n"
+                     "  -IOt %d -IOp %d\n",
+                     no_poll_threads, no_pollsets);
+        erts_usage();
+    }
+
+    /* Handled arguments have been marked with NULL. Slide arguments
+       not handled towards the beginning of argv. */
+    for (i = 0, j = 0; i < *argc; i++) {
+	if (argv[i])
+	    argv[j++] = argv[i];
+    }
+    *argc = j;
+
+    erts_no_pollsets = no_pollsets;
+    erts_no_poll_threads = no_poll_threads;
+}
 
 void
-ERTS_CIO_EXPORT(erts_init_check_io)(void)
+erts_init_check_io(int *argc, char **argv)
 {
-    int j;
+    int j, concurrent_waiters, no_poll_threads;
     ERTS_CT_ASSERT((INT_MIN & (ERL_NIF_SELECT_STOP_CALLED |
                                ERL_NIF_SELECT_STOP_SCHEDULED |
                                ERL_NIF_SELECT_INVALID_EVENT |
                                ERL_NIF_SELECT_FAILED)) == 0);
 
-#ifdef ERTS_ENABLE_KERNEL_POLL
-    ASSERT(erts_io_funcs.select == NULL);
-    erts_io_funcs.select		= ERTS_CIO_EXPORT(driver_select);
-    erts_io_funcs.enif_select		= ERTS_CIO_EXPORT(enif_select);
-    erts_io_funcs.check_io_interrupt	= ERTS_CIO_EXPORT(erts_check_io_interrupt);
-    erts_io_funcs.check_io_interrupt_tmd= ERTS_CIO_EXPORT(erts_check_io_interrupt_timed);
-    erts_io_funcs.check_io		= ERTS_CIO_EXPORT(erts_check_io);
-    erts_io_funcs.get_pollset		= ERTS_CIO_EXPORT(erts_get_pollset);
-    erts_io_funcs.notify_port_task_executed = ERTS_CIO_EXPORT(erts_io_notify_port_task_executed);
-    erts_io_funcs.max_files		= ERTS_CIO_EXPORT(erts_check_io_max_files);
-    erts_io_funcs.size		= ERTS_CIO_EXPORT(erts_check_io_size);
-    erts_io_funcs.info		= ERTS_CIO_EXPORT(erts_check_io_info);
-    erts_io_funcs.check_io_debug	= ERTS_CIO_EXPORT(erts_check_io_debug);
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    erts_io_funcs.lcnt_update_cio_locks	= ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks);
-#endif
-#endif
-
-    init_removed_fd_alloc();
-
-    ERTS_CIO_POLL_INIT();
-    pollsetv = erts_alloc(ERTS_ALC_T_POLLSET,
-                               sizeof(struct pollset_info)*erts_no_schedulers);
-    for (j=0; j < erts_no_schedulers; j++) {
-        struct pollset_info* psi = &pollsetv[j];
-
-        erts_atomic_init_nob(&psi->check_io_time, 0);
-        erts_atomic_init_nob(&psi->in_poll_wait, 0);
-        psi->ps = ERTS_CIO_NEW_POLLSET();
-        psi->active_fd.six = 0;
-        psi->active_fd.eix = 0;
-        erts_atomic32_init_nob(&psi->active_fd.no, 0);
-        psi->active_fd.size = ERTS_ACTIVE_FD_INC;
-        psi->active_fd.array = erts_alloc(ERTS_ALC_T_ACTIVE_FD_ARR,
-                                          sizeof(ErtsSysFdType)*ERTS_ACTIVE_FD_INC);
-#ifdef DEBUG
-        {
-            int i;
-            for (i = 0; i < ERTS_ACTIVE_FD_INC; i++)
-                psi->active_fd.array[i] = ERTS_SYS_FD_INVALID;
-        }
+
+    erts_poll_init(&concurrent_waiters);
+#if ERTS_POLL_USE_FALLBACK
+    erts_poll_init_flbk(NULL);
+#endif
+
+    parse_args(argc, argv, concurrent_waiters);
+
+    /* Create the actual pollsets */
+    pollsetv = erts_alloc(ERTS_ALC_T_POLLSET,sizeof(ErtsPollSet *) * erts_no_pollsets);
+
+    for (j=0; j < erts_no_pollsets; j++)
+        pollsetv[j] = erts_poll_create_pollset(j);
+
+#if ERTS_POLL_USE_FALLBACK
+    flbk_pollset = erts_poll_create_pollset_flbk(-1);
+#endif
+
+    no_poll_threads = erts_no_poll_threads;
+#if ERTS_POLL_USE_FALLBACK
+    no_poll_threads++;
+#endif
+
+    psiv = erts_alloc(ERTS_ALC_T_POLLSET, sizeof(ErtsPollThread) * no_poll_threads);
+
+#if ERTS_POLL_USE_FALLBACK
+    psiv[0].pollres_len = ERTS_CHECK_IO_POLL_RES_LEN;
+    psiv[0].pollres = erts_alloc(ERTS_ALC_T_POLLSET,
+        sizeof(ErtsPollResFd) * ERTS_CHECK_IO_POLL_RES_LEN);
+    psiv[0].ps = get_fallback();
+    psiv++;
 #endif
 
-        erts_atomic_init_nob(&psi->removed_list,  (erts_aint_t)NULL);
+    for (j = 0; j < erts_no_poll_threads; j++) {
+        psiv[j].pollres_len = ERTS_CHECK_IO_POLL_RES_LEN;
+        psiv[j].pollres = erts_alloc(ERTS_ALC_T_POLLSET,
+                                      sizeof(ErtsPollResFd) * ERTS_CHECK_IO_POLL_RES_LEN);
+        psiv[j].ps = pollsetv[j % erts_no_pollsets];
     }
 
-    {
-	int i;
-	for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) {
-            erts_mtx_init(&drv_ev_state.locks[i].lck, "drv_ev_state", make_small(i),
-                ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
-	}
+    for (j=0; j < ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT; j++) {
+        erts_mtx_init(&drv_ev_state.locks[j].lck, "drv_ev_state", make_small(j),
+                          ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
     }
+
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    drv_ev_state.max_fds = ERTS_CIO_POLL_MAX_FDS();
+    drv_ev_state.max_fds = erts_poll_max_fds();
     erts_atomic_init_nob(&drv_ev_state.len, 0);
     drv_ev_state.v = NULL;
     erts_mtx_init(&drv_ev_state.grow_lock, "drv_ev_state_grow", NIL,
@@ -2281,24 +2067,29 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
 }
 
 int
-ERTS_CIO_EXPORT(erts_check_io_max_files)(void)
+erts_check_io_max_files(void)
 {
 #ifdef  ERTS_SYS_CONTINOUS_FD_NUMBERS
     return drv_ev_state.max_fds;
 #else
-    return ERTS_POLL_EXPORT(erts_poll_max_fds)();
+    return erts_poll_max_fds();
 #endif
 }
 
 Uint
-ERTS_CIO_EXPORT(erts_check_io_size)(void)
+erts_check_io_size(void)
 {
     Uint res = 0;
     ErtsPollInfo pi;
     int i;
 
-    for (i = 0; i < erts_no_schedulers; i++) {
-        ERTS_CIO_POLL_INFO(pollsetv[i].ps, &pi);
+#if ERTS_POLL_USE_FALLBACK
+    erts_poll_info(get_fallback(), &pi);
+    res += pi.memory_size;
+#endif
+
+    for (i = 0; i < erts_no_pollsets; i++) {
+        erts_poll_info(pollsetv[i], &pi);
         res += pi.memory_size;
     }
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
@@ -2318,58 +2109,77 @@ ERTS_CIO_EXPORT(erts_check_io_size)(void)
 }
 
 Eterm
-ERTS_CIO_EXPORT(erts_check_io_info)(void *proc)
+erts_check_io_info(void *proc)
 {
     Process *p = (Process *) proc;
     Eterm tags[16], values[16], res, list = NIL;
     Uint sz, *szp, *hp, **hpp;
     ErtsPollInfo *piv;
-    Sint i, j;
-
-    piv = erts_alloc(ERTS_ALC_T_TMP,
-                     sizeof(ErtsPollInfo) * erts_no_schedulers);
+    Sint i, j = 0, len;
+    int no_pollsets = erts_no_pollsets + ERTS_POLL_USE_FALLBACK;
+    ERTS_CT_ASSERT(ERTS_POLL_USE_FALLBACK == 0 || ERTS_POLL_USE_FALLBACK == 1);
 
-    for (j = 0; j < erts_no_schedulers; j++) {
-        struct pollset_info *psi = &pollsetv[j];
-        erts_aint_t cio_time = erts_atomic_read_acqb(&psi->check_io_time);
+    piv = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollInfo) * no_pollsets);
 
-        piv[j].active_fds = (int) erts_atomic32_read_acqb(&psi->active_fd.no);
-        while (1) {
-            erts_aint_t post_cio_time;
-            int post_active_fds;
+#if ERTS_POLL_USE_FALLBACK
+    erts_poll_info_flbk(get_fallback(), &piv[0]);
+    piv[0].poll_threads = 1;
+    piv[0].active_fds = 0;
+    piv++;
+#endif
 
-            ERTS_CIO_POLL_INFO(psi->ps, &piv[j]);
+    for (j = 0; j < erts_no_pollsets; j++) {
+        erts_poll_info(pollsetv[j], &piv[j]);
+        piv[j].active_fds = 0;
+        piv[j].poll_threads = erts_no_poll_threads / erts_no_pollsets;
+        if (erts_no_poll_threads % erts_no_pollsets > j)
+            piv[j].poll_threads++;
+    }
 
-            post_cio_time = erts_atomic_read_mb(&psi->check_io_time);
-            post_active_fds = (int) erts_atomic32_read_acqb(&psi->active_fd.no);
-            if (cio_time == post_cio_time && piv[j].active_fds == post_active_fds)
-                break;
-            cio_time = post_cio_time;
-            piv[j].active_fds = post_active_fds;
+#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
+    i = 0;
+    erts_mtx_lock(&drv_ev_state.grow_lock);
+    len = erts_atomic_read_nob(&drv_ev_state.len);
+    for (i = 0; i < ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT; i++) {
+        erts_mtx_lock(&drv_ev_state.locks[i].lck);
+        for (j = i; j < len; j+=ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT) {
+            ErtsDrvEventState *state = get_drv_ev_state(j);
+            int pollsetid = get_pollset_id(j);
+            ASSERT(fd_mtx(j) == &drv_ev_state.locks[i].lck);
+            if (state->flags & ERTS_EV_FLAG_FALLBACK)
+                pollsetid = -1;
+            if (state->driver.select
+                && (state->type == ERTS_EV_TYPE_DRV_SEL)
+                && (is_iotask_active(&state->driver.select->iniotask)
+                    || is_iotask_active(&state->driver.select->outiotask)))
+                piv[pollsetid].active_fds++;
         }
+        erts_mtx_unlock(&drv_ev_state.locks[i].lck);
+    }
+    erts_mtx_unlock(&drv_ev_state.grow_lock);
 
-    #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-        piv[j].memory_size += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state.len);
-    #else
-        piv[j].memory_size += safe_hash_table_sz(&drv_ev_state.tab);
-        {
-            SafeHashInfo hi;
-            safe_hash_get_info(&hi, &drv_ev_state.tab);
-            piv[j].memory_size += hi.objs * sizeof(ErtsDrvEventState);
-        }
-        erts_spin_lock(&drv_ev_state.prealloc_lock);
-        piv[j].memory_size += drv_ev_state.num_prealloc * sizeof(ErtsDrvEventState);
-        erts_spin_unlock(&drv_ev_state.prealloc_lock);
-    #endif
+    piv[0].memory_size += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state.len);
+#else
+    piv[0].memory_size += safe_hash_table_sz(&drv_ev_state.tab);
+    {
+        SafeHashInfo hi;
+        safe_hash_get_info(&hi, &drv_ev_state.tab);
+        piv[0].memory_size += hi.objs * sizeof(ErtsDrvEventState);
     }
+    erts_spin_lock(&drv_ev_state.prealloc_lock);
+    piv[0].memory_size += drv_ev_state.num_prealloc * sizeof(ErtsDrvEventState);
+    erts_spin_unlock(&drv_ev_state.prealloc_lock);
+#endif
 
     hpp = NULL;
     szp = &sz;
     sz = 0;
 
+    piv -= ERTS_POLL_USE_FALLBACK;
+
  bld_it:
 
-    for (j = erts_no_schedulers-1; j >= 0; j--) {
+    for (j = no_pollsets-1; j >= 0; j--) {
         i = 0;
 
         tags[i] = erts_bld_atom(hpp, szp, "name");
@@ -2378,9 +2188,6 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc)
         tags[i] = erts_bld_atom(hpp, szp, "primary");
         values[i++] = erts_bld_atom(hpp, szp, piv[j].primary);
 
-        tags[i] = erts_bld_atom(hpp, szp, "fallback");
-        values[i++] = erts_bld_atom(hpp, szp, piv[j].fallback ? piv[j].fallback : "false");
-
         tags[i] = erts_bld_atom(hpp, szp, "kernel_poll");
         values[i++] = erts_bld_atom(hpp, szp,
                                     piv[j].kernel_poll ? piv[j].kernel_poll : "false");
@@ -2389,20 +2196,13 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc)
         values[i++] = erts_bld_uint(hpp, szp, piv[j].memory_size);
 
         tags[i] = erts_bld_atom(hpp, szp, "total_poll_set_size");
-        values[i++] = erts_bld_uint(hpp, szp, (Uint) piv[j].poll_set_size);
-
-        if (piv[j].fallback) {
-            tags[i] = erts_bld_atom(hpp, szp, "fallback_poll_set_size");
-            values[i++] = erts_bld_uint(hpp, szp, (Uint) piv[j].fallback_poll_set_size);
-        }
+        values[i++] = erts_bld_uint(hpp, szp, piv[j].poll_set_size);
 
         tags[i] = erts_bld_atom(hpp, szp, "lazy_updates");
         values[i++] = piv[j].lazy_updates ? am_true : am_false;
 
-        if (piv[j].lazy_updates) {
-            tags[i] = erts_bld_atom(hpp, szp, "pending_updates");
-            values[i++] = erts_bld_uint(hpp, szp, (Uint) piv[j].pending_updates);
-        }
+        tags[i] = erts_bld_atom(hpp, szp, "pending_updates");
+        values[i++] = erts_bld_uint(hpp, szp, piv[j].pending_updates);
 
         tags[i] = erts_bld_atom(hpp, szp, "batch_updates");
         values[i++] = piv[j].batch_updates ? am_true : am_false;
@@ -2410,22 +2210,17 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc)
         tags[i] = erts_bld_atom(hpp, szp, "concurrent_updates");
         values[i++] = piv[j].concurrent_updates ? am_true : am_false;
 
+        tags[i] = erts_bld_atom(hpp, szp, "fallback");
+        values[i++] = piv[j].is_fallback ? am_true : am_false;
+
         tags[i] = erts_bld_atom(hpp, szp, "max_fds");
-        values[i++] = erts_bld_uint(hpp, szp, (Uint) piv[j].max_fds);
+        values[i++] = erts_bld_uint(hpp, szp, piv[j].max_fds);
 
         tags[i] = erts_bld_atom(hpp, szp, "active_fds");
-        values[i++] = erts_bld_uint(hpp, szp, (Uint) piv[j].active_fds);
-
-    #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
-        tags[i] = erts_bld_atom(hpp, szp, "no_avoided_wakeups");
-        values[i++] = erts_bld_uint(hpp, szp, (Uint) piv[j].no_avoided_wakeups);
-
-        tags[i] = erts_bld_atom(hpp, szp, "no_avoided_interrupts");
-        values[i++] = erts_bld_uint(hpp, szp, (Uint) piv[j].no_avoided_interrupts);
+        values[i++] = erts_bld_uint(hpp, szp, piv[j].active_fds);
 
-        tags[i] = erts_bld_atom(hpp, szp, "no_interrupt_timed");
-        values[i++] = erts_bld_uint(hpp, szp, (Uint) piv[j].no_interrupt_timed);
-    #endif
+        tags[i] = erts_bld_atom(hpp, szp, "poll_threads");
+        values[i++] = erts_bld_uint(hpp, szp, piv[j].poll_threads);
 
         res = erts_bld_2tup_list(hpp, szp, i, tags, values);
 
@@ -2454,6 +2249,10 @@ static ERTS_INLINE ErtsPollEvents
 print_events(ErtsPollEvents ev)
 {
     int first = 1;
+    if(ev == ERTS_POLL_EV_NONE) {
+        erts_printf("N/A");
+        return 0;
+    }
     if(ev & ERTS_POLL_EV_IN) {
 	ev &= ~ERTS_POLL_EV_IN;
 	erts_printf("%s%s", first ? "" : "|", "IN");
@@ -2486,15 +2285,40 @@ print_flags(EventStateFlags f)
 	erts_printf("%s","USED");
 	delim = "|";
     }
-    if(f & ERTS_EV_FLAG_DEFER_IN_EV) {
-	erts_printf("%s%s", delim, "DRIN");
+    if(f & ERTS_EV_FLAG_FALLBACK) {
+	erts_printf("%s%s", delim, "FLBK");
         delim = "|";
     }
-    if(f & ERTS_EV_FLAG_DEFER_OUT_EV) {
-        erts_printf("%s%s", delim, "DROUT");
+}
+
+#ifdef DEBUG_PRINT_MODE
+
+static ERTS_INLINE char *
+drvmode2str(int mode) {
+    switch (mode) {
+    case ERL_DRV_READ|ERL_DRV_USE: return "READ|USE";
+    case ERL_DRV_WRITE|ERL_DRV_USE: return "WRITE|USE";
+    case ERL_DRV_READ|ERL_DRV_WRITE|ERL_DRV_USE: return "READ|WRITE|USE";
+    case ERL_DRV_USE: return "USE";
+    case ERL_DRV_READ: return "READ";
+    case ERL_DRV_WRITE: return "WRITE";
+    case ERL_DRV_READ|ERL_DRV_WRITE: return "READ|WRITE";
+    default: return "UNKNOWN";
+    }
+}
+
+static ERTS_INLINE char *
+nifmode2str(enum ErlNifSelectFlags mode) {
+    switch (mode) {
+    case ERL_NIF_SELECT_READ: return "READ";
+    case ERL_NIF_SELECT_WRITE: return "WRITE";
+    case ERL_NIF_SELECT_STOP: return "STOP";
+    default: return "UNKNOWN";
     }
 }
 
+#endif
+
 typedef struct {
     int used_fds;
     int num_errors;
@@ -2506,53 +2330,32 @@ typedef struct {
 #endif
 } IterDebugCounters;
 
-static void doit_erts_check_io_debug(void *vstate, void *vcounters)
+static int erts_debug_print_checkio_state(ErtsDrvEventState *state,
+                                          ErtsPollEvents ep_events,
+                                          int internal)
 {
-    ErtsDrvEventState *state = (ErtsDrvEventState *) vstate;
-    IterDebugCounters *counters = (IterDebugCounters *) vcounters;
-    ErtsPollEvents cio_events = state->events;
-    ErtsSysFdType fd = state->fd;
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    int internal = 0;
-    ErtsPollEvents ep_events = counters->epep[(int) fd];
-#endif
-    int err = 0;
-
 #if defined(HAVE_FSTAT) && !defined(NO_FSTAT_ON_SYS_FD_TYPE)
     struct stat stat_buf;
 #endif
 
-    if (state->driver.select)
-	counters->no_driver_select_structs++;
-    if (state->driver.nif)
-        counters->no_enif_select_structs++;
-
+    ErtsSysFdType fd = state->fd;
+    ErtsPollEvents cio_events = state->events;
+    int err = 0;
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    if (state->events || ep_events) {
-	if (ep_events & ERTS_POLL_EV_NVAL) {
-	    ep_events &= ~ERTS_POLL_EV_NVAL;
-	    internal = 1;
-	    counters->internal_fds++;
-	}
-	else
-	    counters->used_fds++;
-#else
-    if (state->events) {
-	counters->used_fds++;
+    ErtsPollEvents aio_events = state->active_events;
 #endif
-	
-	erts_printf("pollset=%d fd=%d ",
-                    (int)(state->pollset - pollsetv), (int) fd);
-	
+    erts_printf("pollset=%d fd=%d ",
+                state->flags & ERTS_EV_FLAG_FALLBACK ? -1 : get_pollset_id(fd), (int) fd);
+
 #if defined(HAVE_FSTAT) && !defined(NO_FSTAT_ON_SYS_FD_TYPE)
-	if (fstat((int) fd, &stat_buf) < 0)
-	    erts_printf("type=unknown ");
-	else {
-	    erts_printf("type=");
+    if (fstat((int) fd, &stat_buf) < 0)
+        erts_printf("type=unknown ");
+    else {
+        erts_printf("type=");
 #ifdef S_ISSOCK
-	    if (S_ISSOCK(stat_buf.st_mode))
-		erts_printf("sock ");
-	    else
+        if (S_ISSOCK(stat_buf.st_mode))
+            erts_printf("sock ");
+        else
 #endif
 #ifdef S_ISFIFO
 	    if (S_ISFIFO(stat_buf.st_mode))
@@ -2560,196 +2363,239 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters)
 	    else
 #endif
 #ifdef S_ISCHR
-	    if (S_ISCHR(stat_buf.st_mode))
-		erts_printf("chr ");
-	    else
+                if (S_ISCHR(stat_buf.st_mode))
+                    erts_printf("chr ");
+                else
 #endif
 #ifdef S_ISDIR
-	    if (S_ISDIR(stat_buf.st_mode))
-		erts_printf("dir ");
-	    else
+                    if (S_ISDIR(stat_buf.st_mode))
+                        erts_printf("dir ");
+                    else
 #endif
 #ifdef S_ISBLK
-	    if (S_ISBLK(stat_buf.st_mode))
-		erts_printf("blk ");
-	    else
+                        if (S_ISBLK(stat_buf.st_mode))
+                            erts_printf("blk ");
+                        else
 #endif
 #ifdef S_ISREG
-	    if (S_ISREG(stat_buf.st_mode))
-		erts_printf("reg ");
-	    else
+                            if (S_ISREG(stat_buf.st_mode))
+                                erts_printf("reg ");
+                            else
 #endif
 #ifdef S_ISLNK
-	    if (S_ISLNK(stat_buf.st_mode))
-		erts_printf("lnk ");
-	    else
+                                if (S_ISLNK(stat_buf.st_mode))
+                                    erts_printf("lnk ");
+                                else
 #endif
 #ifdef S_ISDOOR
-	    if (S_ISDOOR(stat_buf.st_mode))
-		erts_printf("door ");
-	    else
+                                    if (S_ISDOOR(stat_buf.st_mode))
+                                        erts_printf("door ");
+                                    else
 #endif
 #ifdef S_ISWHT
-	    if (S_ISWHT(stat_buf.st_mode))
-		erts_printf("wht ");
-	    else
+                                        if (S_ISWHT(stat_buf.st_mode))
+                                            erts_printf("wht ");
+                                        else
 #endif
 #ifdef S_ISXATTR
-	    if (S_ISXATTR(stat_buf.st_mode))
-		erts_printf("xattr ");
-	    else
+                                            if (S_ISXATTR(stat_buf.st_mode))
+                                                erts_printf("xattr ");
+                                            else
 #endif
-		erts_printf("unknown ");
-	}
+                                                erts_printf("unknown ");
+    }
 #else
-	erts_printf("type=unknown ");
+    erts_printf("type=unknown ");
 #endif
 
-	if (state->type == ERTS_EV_TYPE_DRV_SEL) {
-	    erts_printf("driver_select ");
-	    
-#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-	    if (internal) {
-		erts_printf("internal ");
-		err = 1;		    
-	    }
-	    
-	    if (cio_events == ep_events) {
-		erts_printf("ev=");
-		if (print_events(cio_events) != 0)
-		    err = 1;
-	    }
-	    else {
-                ErtsPollEvents ev = cio_events;
-#if ERTS_CIO_DEFER_ACTIVE_EVENTS
-                if (state->flags & ERTS_EV_FLAG_DEFER_IN_EV)
-                    ev &= ~ERTS_POLL_EV_IN;
-                if (state->flags & ERTS_EV_FLAG_DEFER_OUT_EV)
-                    ev &= ~ERTS_POLL_EV_OUT;
-#endif
-                if (ev != ep_events)
-                    err = 1;
-		erts_printf("cio_ev=");
-		print_events(cio_events);
-		erts_printf(" ep_ev=");
-		print_events(ep_events);
-	    }
-#else
-	    if (print_events(cio_events) != 0)
-		err = 1;
-#endif
-	    erts_printf(" ");
-	    if (cio_events & ERTS_POLL_EV_IN) {
-		Eterm id = state->driver.select->inport;
-		if (is_nil(id)) {
-		    erts_printf("inport=none inname=none indrv=none ");
-		    err = 1;
-		}
-		else {
-		    ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT);
-		    erts_printf(" inport=%T inname=%s indrv=%s ",
-				id,
-				pnp->name ? pnp->name : "unknown",
-				(pnp->driver_name
-				 ? pnp->driver_name
-				 : "unknown"));
-		    erts_free_port_names(pnp);
-		}
-	    }
-	    if (cio_events & ERTS_POLL_EV_OUT) {
-		Eterm id = state->driver.select->outport;
-		if (is_nil(id)) {
-		    erts_printf("outport=none outname=none outdrv=none ");
-		    err = 1;
-		}
-		else {
-		    ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT);
-		    erts_printf(" outport=%T outname=%s outdrv=%s ",
-				id,
-				pnp->name ? pnp->name : "unknown",
-				(pnp->driver_name
-				 ? pnp->driver_name
-				 : "unknown"));
-		    erts_free_port_names(pnp);
-		}
-	    }
-	}
-        else if (state->type == ERTS_EV_TYPE_NIF) {
-            ErtsResource* r;
-            erts_printf("enif_select ");
+    if (state->type == ERTS_EV_TYPE_DRV_SEL) {
+        erts_printf("driver_select ");
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-            if (internal) {
-                erts_printf("internal ");
-                err = 1;
-            }
-
+        if (internal) {
+            erts_printf("internal ");
+            err = 1;
+        }
+        if (aio_events == cio_events) {
             if (cio_events == ep_events) {
                 erts_printf("ev=");
                 if (print_events(cio_events) != 0)
                     err = 1;
             }
             else {
-                err = 1;
+                ErtsPollEvents ev = cio_events;
+                if (ev != ep_events && ep_events != ERTS_POLL_EV_NONE)
+                    err = 1;
                 erts_printf("cio_ev=");
                 print_events(cio_events);
                 erts_printf(" ep_ev=");
                 print_events(ep_events);
             }
+        } else {
+            erts_printf("cio_ev=");
+            print_events(cio_events);
+            erts_printf(" aio_ev=");
+            print_events(aio_events);
+            if ((aio_events != ep_events && ep_events != ERTS_POLL_EV_NONE) ||
+                (aio_events != 0 && ep_events == ERTS_POLL_EV_NONE)) {
+                erts_printf(" ep_ev=");
+                print_events(ep_events);
+                err = 1;
+            }
+        }
 #else
+        if (print_events(cio_events) != 0)
+            err = 1;
+#endif
+        erts_printf(" ");
+        if (cio_events & ERTS_POLL_EV_IN) {
+            Eterm id = state->driver.select->inport;
+            if (is_nil(id)) {
+                erts_printf("inport=none inname=none indrv=none ");
+                err = 1;
+            }
+            else {
+                ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT);
+                erts_printf(" inport=%T inname=%s indrv=%s ",
+                            id,
+                            pnp->name ? pnp->name : "unknown",
+                            (pnp->driver_name
+                             ? pnp->driver_name
+                             : "unknown"));
+                erts_free_port_names(pnp);
+            }
+        }
+        if (cio_events & ERTS_POLL_EV_OUT) {
+            Eterm id = state->driver.select->outport;
+            if (is_nil(id)) {
+                erts_printf("outport=none outname=none outdrv=none ");
+                err = 1;
+            }
+            else {
+                ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT);
+                erts_printf(" outport=%T outname=%s outdrv=%s ",
+                            id,
+                            pnp->name ? pnp->name : "unknown",
+                            (pnp->driver_name
+                             ? pnp->driver_name
+                             : "unknown"));
+                erts_free_port_names(pnp);
+            }
+        }
+    }
+    else if (state->type == ERTS_EV_TYPE_NIF) {
+        ErtsResource* r;
+        erts_printf("enif_select ");
+
+#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
+        if (internal) {
+            erts_printf("internal ");
+            err = 1;
+        }
+
+        if (cio_events == ep_events) {
+            erts_printf("ev=");
             if (print_events(cio_events) != 0)
                 err = 1;
+        }
+        else {
+            err = 1;
+            erts_printf("cio_ev=");
+            print_events(cio_events);
+            erts_printf(" ep_ev=");
+            print_events(ep_events);
+        }
+#else
+        if (print_events(cio_events) != 0)
+            err = 1;
 #endif
-            erts_printf(" inpid=%T dd_cnt=%b32d", state->driver.nif->in.pid,
-                        state->driver.nif->in.ddeselect_cnt);
-            erts_printf(" outpid=%T dd_cnt=%b32d", state->driver.nif->out.pid,
-                        state->driver.nif->out.ddeselect_cnt);
-            r = state->driver.stop.resource;
-            erts_printf(" resource=%p(%T:%T)", r, r->type->module, r->type->name);
+        erts_printf(" inpid=%T", state->driver.nif->in.pid);
+        erts_printf(" outpid=%T", state->driver.nif->out.pid);
+        r = state->driver.stop.resource;
+        erts_printf(" resource=%p(%T:%T)", r, r->type->module, r->type->name);
+    }
+#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
+    else if (internal) {
+        erts_printf("internal ");
+        if (cio_events) {
+            err = 1;
+            erts_printf("cio_ev=");
+            print_events(cio_events);
         }
+        if (ep_events) {
+            erts_printf("ep_ev=");
+            print_events(ep_events);
+        }
+    }
+#endif
+    else {
+        err = 1;
+        erts_printf("control_type=%d ", (int)state->type);
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-	else if (internal) {
-	    erts_printf("internal ");
-	    if (cio_events) {
-		err = 1;
-		erts_printf("cio_ev=");
-		print_events(cio_events);
-	    }
-	    if (ep_events) {
-		erts_printf("ep_ev=");
-		print_events(ep_events);
-	    }
-	}
+        if (cio_events == ep_events) {
+            erts_printf("ev=");
+            print_events(cio_events);
+        }
+        else {
+            erts_printf("cio_ev="); print_events(cio_events);
+            erts_printf(" ep_ev="); print_events(ep_events);
+        }
+#else
+        erts_printf("ev=0x%b32x", (Uint32) cio_events);
 #endif
-	else {
-	    err = 1;
-	    erts_printf("control_type=%d ", (int)state->type);
+    }
+
+    erts_printf(" flags="); print_flags(state->flags);
+    if (err) {
+        erts_printf(" ERROR");
+    }
+    erts_printf("\r\n");
+    return err;
+}
+
+static void doit_erts_check_io_debug(void *vstate, void *vcounters)
+{
+    ErtsDrvEventState *state = (ErtsDrvEventState *) vstate;
+    IterDebugCounters *counters = (IterDebugCounters *) vcounters;
+    int internal = 0;
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-	    if (cio_events == ep_events) {
-                erts_printf("ev=");
-                print_events(cio_events);
-	    }
-	    else {
-		erts_printf("cio_ev="); print_events(cio_events);
-		erts_printf(" ep_ev="); print_events(ep_events);
-	    }
+    ErtsSysFdType fd = state->fd;
+    ErtsPollEvents ep_events = counters->epep[(int) fd];
 #else
-	    erts_printf("ev=0x%b32x", (Uint32) cio_events);
+    ErtsPollEvents ep_events = ERTS_POLL_EV_NONE;
 #endif
-	}
-	
-        erts_printf(" flags="); print_flags(state->flags);
 
-	if (err) {
+    if (state->driver.select) {
+	counters->no_driver_select_structs++;
+        ASSERT(state->events || (ep_events != 0 && ep_events != ERTS_POLL_EV_NONE));
+    }
+    if (state->driver.nif) {
+        counters->no_enif_select_structs++;
+        ASSERT(state->events || (ep_events != 0 && ep_events != ERTS_POLL_EV_NONE));
+    }
+
+#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
+    if (state->events || (ep_events != 0 && ep_events != ERTS_POLL_EV_NONE)) {
+	if (ep_events & ERTS_POLL_EV_NVAL) {
+	    ep_events &= ~ERTS_POLL_EV_NVAL;
+	    internal = 1;
+	    counters->internal_fds++;
+	}
+	else
+	    counters->used_fds++;
+#else
+    if (state->events) {
+	counters->used_fds++;
+#endif
+	if (erts_debug_print_checkio_state(state, ep_events, internal)) {
 	    counters->num_errors++;
-	    erts_printf(" ERROR");
 	}
-	erts_printf("\n");
     }
 }
-    
+
+/* ciodpi can be NULL when called from etp-commands */
 int
-ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
+erts_check_io_debug(ErtsCheckIoDebugInfo *ciodip)
 {
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     int fd, len, i;
@@ -2758,12 +2604,10 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     ErtsDrvEventState null_des;
 
-    null_des.pollset = NULL;
     null_des.driver.select = NULL;
     null_des.driver.nif = NULL;
     null_des.driver.stop.drv_ptr = NULL;
     null_des.events = 0;
-    null_des.remove_cnt = 0;
     null_des.type = ERTS_EV_TYPE_NONE;
     null_des.flags = 0;
 
@@ -2771,24 +2615,36 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
                                sizeof(ErtsPollEvents)*drv_ev_state.max_fds);
 #endif
 
-    erts_printf("--- fds in pollset --------------------------------------\n");
 
 #if defined(ERTS_ENABLE_LOCK_CHECK)
     erts_lc_check_exact(NULL, 0); /* No locks should be locked */
 #endif
 
-    erts_thr_progress_block(); /* stop the world to avoid messy locking */
+    if (ciodip)
+        erts_thr_progress_block(); /* stop the world to avoid messy locking */
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     len = erts_atomic_read_nob(&drv_ev_state.len);
 
-    for (i = 0; i < erts_no_schedulers; i++) {
-        ERTS_POLL_EXPORT(erts_poll_get_selected_events)(pollsetv[i].ps,
-                                                        counters.epep,
-                                                        drv_ev_state.max_fds);
+#if ERTS_POLL_USE_FALLBACK
+    erts_printf("--- fds in flbk pollset ---------------------------------\n");
+    erts_poll_get_selected_events_flbk(get_fallback(), counters.epep,
+                                       drv_ev_state.max_fds);
+    for (fd = 0; fd < len; fd++) {
+        if (drv_ev_state.v[fd].flags & ERTS_EV_FLAG_FALLBACK)
+            doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters);
+    }
+#endif
+    erts_printf("--- fds in pollset --------------------------------------\n");
+
+    for (i = 0; i < erts_no_pollsets; i++) {
+        erts_poll_get_selected_events(pollsetv[i],
+                                      counters.epep,
+                                      drv_ev_state.max_fds);
         for (fd = 0; fd < len; fd++) {
-            if (drv_ev_state.v[fd].pollset == &pollsetv[i])
-                doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters);
+            if (!(drv_ev_state.v[fd].flags & ERTS_EV_FLAG_FALLBACK)
+                && get_pollset_id(fd) == i)
+            doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters);
         }
     }
     for (fd = len ; fd < drv_ev_state.max_fds; fd++) {
@@ -2799,11 +2655,15 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
     safe_hash_for_each(&drv_ev_state.tab, &doit_erts_check_io_debug,
                        &counters);
 #endif
-    erts_thr_progress_unblock();
 
-    ciodip->no_used_fds = counters.used_fds;
-    ciodip->no_driver_select_structs = counters.no_driver_select_structs;
-    ciodip->no_enif_select_structs = counters.no_enif_select_structs;
+    if (ciodip)
+        erts_thr_progress_unblock();
+
+    if (ciodip) {
+        ciodip->no_used_fds = counters.used_fds;
+        ciodip->no_driver_select_structs = counters.no_driver_select_structs;
+        ciodip->no_enif_select_structs = counters.no_enif_select_structs;
+    }
 
     erts_printf("\n");
     erts_printf("used fds=%d\n", counters.used_fds);
@@ -2822,97 +2682,19 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
 }
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
-void ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks)(int enable) {
+void erts_lcnt_update_cio_locks(int enable) {
+    int i;
 #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
     erts_lcnt_enable_hash_lock_count(&drv_ev_state.tab, ERTS_LOCK_FLAGS_CATEGORY_IO, enable);
 #else
     (void)enable;
 #endif
-}
-#endif /* ERTS_ENABLE_LOCK_COUNT */
-
-#ifdef ERTS_ENABLE_KERNEL_POLL
-# ifdef ERTS_KERNEL_POLL_VERSION
-
-/*
- * Compile these only once for kp/nkp
- */
-
-void erts_init_check_io(void)
-{
-    if (erts_use_kernel_poll)
-        erts_init_check_io_kp();
-    else
-        erts_init_check_io_nkp();
-}
-
-int
-driver_select(ErlDrvPort port, ErlDrvEvent event, int mode, int on)
-{
-    return (*erts_io_funcs.select)(port, event, mode, on);
-}
-
-int enif_select(ErlNifEnv* env, ErlNifEvent event,
-                enum ErlNifSelectFlags flags, void* obj, const ErlNifPid* pid, Eterm ref)
-{
-    return (*erts_io_funcs.enif_select)(env, event, flags, obj, pid, ref);
-}
-
-struct pollset_info* erts_get_pollset(int sched_nr)
-{
-    return (*erts_io_funcs.get_pollset)(sched_nr);
-}
 
-void erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp)
-{
-    erts_io_funcs.notify_port_task_executed(pthp);
-}
-
-void erts_check_io(int do_wait)
-{
-    erts_io_funcs.check_io(do_wait);
-}
-
-Uint erts_check_io_size(void)
-{
-    return erts_io_funcs.size();
-}
-
-
-Eterm erts_check_io_info(void *p)
-{
-    return (*erts_io_funcs.info)(p);
-}
-
-int erts_check_io_max_files(void)
-{
-    return erts_io_funcs.max_files();
-}
-
-int
-erts_check_io_debug(ErtsCheckIoDebugInfo *ip)
-{
-    return (*erts_io_funcs.check_io_debug)(ip);
-}
-
-void erts_check_io_interrupt(struct pollset_info* psi, int set)
-{
-    erts_io_funcs.check_io_interrupt(psi, set);
-}
-
-void erts_check_io_interrupt_timed(struct pollset_info* psi, int set,
-                                   ErtsMonotonicTime timeout_time)
-{
-    erts_io_funcs.check_io_interrupt_tmd(psi, set, timeout_time);
-}
-
-#ifdef ERTS_ENABLE_LOCK_COUNT
-void erts_lcnt_update_cio_locks(int enable)
-{
-    erts_io_funcs.lcnt_update_cio_locks(enable);
-}
+#if ERTS_POLL_USE_FALLBACK
+    erts_lcnt_enable_pollset_lock_count_flbk(get_fallback(), enable);
 #endif
 
-#endif /* ERTS_KERNEL_POLL_VERSION */
-
-#endif /* !ERTS_ENABLE_KERNEL_POLL */
+    for (i = 0; i < erts_no_pollsets; i++)
+        erts_lcnt_enable_pollset_lock_count(pollsetv[i], enable);
+}
+#endif /* ERTS_ENABLE_LOCK_COUNT */
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index ab53d91756..443ef1264c 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -18,10 +18,11 @@
  * %CopyrightEnd%
  */
 
-/*
- * Description:	Check I/O
+/**
+ * @description Check I/O, a cross platform IO polling framework for ERTS
  *
- * Author: 	Rickard Green
+ * @author Rickard Green
+ * @author Lukas Larsson
  */
 
 #ifndef ERL_CHECK_IO_H__
@@ -30,35 +31,96 @@
 #include "sys.h"
 #include "erl_sys_driver.h"
 
-struct pollset_info;
+/** @brief a structure that is used by each polling thread */
+struct erts_poll_thread;
 
+/**
+ * Get the memory size of the check io framework
+ */
 Uint erts_check_io_size(void);
-Eterm erts_check_io_info(void *);
-void erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp);
-void erts_check_io_async_sig_interrupt(struct pollset_info *psi);
+/**
+ * Returns an Eterm with information about all the pollsets active at the
+ * moment.
+ *
+ * @param proc the Process* to allocate the result on. It is passed as
+ *  void * because of header include problems.
+ */
+Eterm erts_check_io_info(void *proc);
+/**
+ * Should be called when a port IO task has been executed in order to re-enable
+ * or clear the information about the fd.
+ *
+ * @param type The type of event that has been completed.
+ * @param handle The port task handle of the event.
+ * @param reset A function pointer to be called when the port task handle
+ *  should be reset.
+ */
+void erts_io_notify_port_task_executed(ErtsPortTaskType type,
+                                       ErtsPortTaskHandle *handle,
+                                       void (*reset)(ErtsPortTaskHandle *));
+/**
+ * Returns the maximum number of fds that the check io framework can handle.
+ */
 int erts_check_io_max_files(void);
-void erts_check_io(int);
-void erts_init_check_io(void);
+/**
+ * Called by any thread that should check for new IO events. This function will
+ * not return unless erts_check_io_interrupt(pt, 1) is called by another thread.
+ *
+ * @param pt the poll thread structure to use.
+ */
+void erts_check_io(struct erts_poll_thread *pt);
+/**
+ * Initialize the check io framework. This function will parse the arguments
+ * and delete any entries that it is interested in.
+ *
+ * @param argc the number of arguments
+ * @param argv an array with the arguments
+ */
+void erts_init_check_io(int *argc, char **argv);
+/**
+ * Interrupt the poll thread so that it can execute other code.
+ *
+ * Should be called with set = 0 by the waiting thread before calling
+ * erts_check_io.
+ *
+ * @param pt the poll thread to wake
+ * @param set whether to set or clear the interrupt flag
+ */
+void erts_check_io_interrupt(struct erts_poll_thread *pt, int set);
+/**
+ * Create a new poll thread structure that is associated with the number no.
+ * It is the callers responsibility that no is unique.
+ */
+struct erts_poll_thread* erts_create_pollset_thread(int no);
 #ifdef ERTS_ENABLE_LOCK_COUNT
+/**
+ * Toggle lock counting on all check io locks
+ */
 void erts_lcnt_update_cio_locks(int enable);
 #endif
 
-void erts_check_io_interrupt(struct pollset_info*, int);
-void erts_check_io_interrupt_timed(struct pollset_info*, int, ErtsMonotonicTime);
-struct pollset_info* erts_get_pollset(int sched_num);
-
 typedef struct {
     ErtsPortTaskHandle task;
-    erts_atomic_t executed_time;
-    struct pollset_info *pollset;
+    ErtsSysFdType fd;
 } ErtsIoTask;
 
+
 #endif /*  ERL_CHECK_IO_H__ */
 
 #if !defined(ERL_CHECK_IO_C__) && !defined(ERTS_ALLOC_C__)
 #define ERL_CHECK_IO_INTERNAL__
 #endif
 
+#define ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT 128
+
+/* Controls how many pollsets to allocate. Fd's are hashed into
+   each pollset based on the FD. When doing non-concurrent updates
+   there will be one pollset per thread.
+*/
+extern int erts_no_pollsets;
+extern int erts_no_poll_threads;
+
+
 #ifndef ERL_CHECK_IO_INTERNAL__
 #define ERL_CHECK_IO_INTERNAL__
 #include "erl_poll.h"
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index 341370ca14..c671ea99f4 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -18,9 +18,8 @@
  * %CopyrightEnd%
  */
 
-/*
- * Description:	Poll interface suitable for ERTS with or without
- *              SMP support.
+/**
+ * @description Poll interface suitable for ERTS
  *
  *		The interface is currently implemented using:
  *		- select
@@ -29,12 +28,36 @@
  *              - epoll with poll or select as fallback
  *              - kqueue with poll or select as fallback
  *
- *		Some time in the future it will also be
- *		implemented using Solaris ports.
  *
+ * @author Rickard Green
+ * @author Lukas Larsson
+ *
+ * There are two major different implementations off IO polling in this
+ * file. The concurrent and non-concurrent implementations.
+ * When available epoll/kqueue are used to implement the concurrent
+ * versions. poll, select and dev/poll use non-concurrent updates.
+ *
+ * Concurrent version:
+ *   In the concurrent version erts_poll_control directly modifies
+ *   the kernel pollset without waking the thread that is waiting
+ *   on events. Also the ErtsPollResFd type is directly mapped to
+ *   the native event type, so no extra copying is needed. Note that
+ *   as no locking at all is done, fds can be triggered that have been
+ *   removed from the pollset. The check_io layer has to deal with this.
+ *
+ * Non-concurrent version:
+ *   In the non-concurrent version, the pollset has an internal representation
+ *   of the pollset that is updated by erts_poll_control. When an fd is updated,
+ *   its number is placed in the update request queue and then the waiting thread
+ *   is woken in order to see the change. The internal data in the pollset is
+ *   protected by a mutex that has to be taken by both the modifying and waiting
+ *   thread at different times.
  *
+ *   The non-concurrent pollset cannot have fd's closed in it while a thread is
+ *   waiting on that fd. In order to fix this, when an ERTS_POLL_OP_DEL command
+ *   is issued, the fd is marked as closing and the waiting thread is woken. The
+ *   fd is then returned in the waiting threads results as ERTS_POLL_EV_NONE.
  *
- * Author: 	Rickard Green
  */
 
 #ifdef HAVE_CONFIG_H
@@ -62,6 +85,8 @@
 #  ifdef SYS_SELECT_H
 #    include <sys/select.h>
 #  endif
+#elif defined(_DARWIN_UNLIMITED_SELECT)
+#  undef _DARWIN_UNLIMITED_SELECT
 #endif
 #ifdef NO_SYSCONF
 #  if ERTS_POLL_USE_SELECT
@@ -83,20 +108,35 @@
 #error "Missing implementation of erts_poll()"
 #endif
 
-#if defined(ERTS_KERNEL_POLL_VERSION) && !ERTS_POLL_USE_KERNEL_POLL
-#error "Missing kernel poll implementation of erts_poll()"
-#endif
+#if 0
+#define ERTS_POLL_DEBUG_PRINT 1
+
+#define DEBUG_PRINT(FMT, PS, ...)                                       \
+    do {                                                                \
+        int myerrno = errno;                                            \
+        erts_printf("%d: " FMT "\r\n", (PS)->id, ##__VA_ARGS__);        \
+        errno = myerrno;                                                \
+    } while(0)
+
+/* Define to print info about modifications done to each fd */
+#define DEBUG_PRINT_FD(FMT, PS, FD, ...) DEBUG_PRINT("%d: " FMT, PS, FD, ##__VA_ARGS__)
+/* Define to print entry and exit from erts_poll_wait (can be very spammy) */
+//#define DEBUG_PRINT_WAIT(FMT, PS, ...) DEBUG_PRINT(FMT, PS, ##__VA_ARGS__)
 
-#if defined(ERTS_NO_KERNEL_POLL_VERSION) && ERTS_POLL_USE_KERNEL_POLL
-#error "Kernel poll used when it shouldn't be used"
+#else
+#define ERTS_POLL_DEBUG_PRINT 0
+#define DEBUG_PRINT(...)
 #endif
 
-#if 0
-#define ERTS_POLL_DEBUG_PRINT
+#ifndef DEBUG_PRINT_FD
+#define DEBUG_PRINT_FD(...)
+#endif
+#ifndef DEBUG_PRINT_WAIT
+#define DEBUG_PRINT_WAIT(...)
 #endif
 
 
-#ifdef _DARWIN_UNLIMITED_SELECT
+#if defined(_DARWIN_UNLIMITED_SELECT) && ERTS_POLL_USE_SELECT
 typedef struct {
     size_t sz;
     fd_set* ptr;
@@ -142,44 +182,41 @@ int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds,
 #  define ERTS_SELECT	select
 #endif
 
-#define ERTS_POLL_USE_BATCH_UPDATE_POLLSET (ERTS_POLL_USE_DEVPOLL \
-					    || ERTS_POLL_USE_KQUEUE)
+#define ERTS_POLL_IS_FALLBACK (ERTS_POLL_USE_POLL || ERTS_POLL_USE_SELECT) && ERTS_ENABLE_KERNEL_POLL
 
-#define ERTS_POLL_USE_CONCURRENT_UPDATE ERTS_POLL_USE_EPOLL
+#define ERTS_POLL_USE_CONCURRENT_UPDATE (ERTS_POLL_USE_EPOLL || ERTS_POLL_USE_KQUEUE)
 
-#define ERTS_POLL_COALESCE_KP_RES (ERTS_POLL_USE_KQUEUE || ERTS_POLL_USE_EPOLL)
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
+
+#define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) \
+  erts_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 1)
+#define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) \
+  erts_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 0)
+#define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) \
+  ((int) erts_atomic32_read_nob(&(PS)->have_update_requests))
 
 #define ERTS_POLLSET_LOCK(PS) \
   erts_mtx_lock(&(PS)->mtx)
 #define ERTS_POLLSET_UNLOCK(PS) \
   erts_mtx_unlock(&(PS)->mtx)
 
-#define ERTS_POLLSET_SET_POLLED_CHK(PS) \
-  ((int) erts_atomic32_xchg_nob(&(PS)->polled, (erts_aint32_t) 1))
-#define ERTS_POLLSET_UNSET_POLLED(PS) \
-  erts_atomic32_set_nob(&(PS)->polled, (erts_aint32_t) 0)
-#define ERTS_POLLSET_IS_POLLED(PS) \
-  ((int) erts_atomic32_read_nob(&(PS)->polled))
+#else
 
+#define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS)
+#define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS)
+#define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) 0
 
-#define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) \
-  erts_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 1)
-#define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) \
-  erts_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 0)
-#define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) \
-  ((int) erts_atomic32_read_nob(&(PS)->have_update_requests))
+#define ERTS_POLLSET_LOCK(PS)
+#define ERTS_POLLSET_UNLOCK(PS)
 
-#if ERTS_POLL_USE_FALLBACK
-#  if ERTS_POLL_USE_POLL
-#    define ERTS_POLL_NEED_FALLBACK(PS) ((PS)->no_poll_fds > 1)
-#  elif ERTS_POLL_USE_SELECT
-#    define ERTS_POLL_NEED_FALLBACK(PS) ((PS)->no_select_fds > 1)
-#  endif
 #endif
+
 /*
  * --- Data types ------------------------------------------------------------
  */
 
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
+
 #define ERTS_POLLSET_UPDATE_REQ_BLOCK_SIZE 128
 
 typedef struct ErtsPollSetUpdateRequestsBlock_ ErtsPollSetUpdateRequestsBlock;
@@ -189,43 +226,20 @@ struct ErtsPollSetUpdateRequestsBlock_ {
     int fds[ERTS_POLLSET_UPDATE_REQ_BLOCK_SIZE];
 };
 
-
-
 #  define ERTS_POLL_FD_FLG_INURQ	(((unsigned short) 1) << 0)
-#if ERTS_POLL_USE_FALLBACK
-#  define ERTS_POLL_FD_FLG_INFLBCK	(((unsigned short) 1) << 1)
-#  define ERTS_POLL_FD_FLG_USEFLBCK	(((unsigned short) 1) << 2)
-#endif
-#  define ERTS_POLL_FD_FLG_RST		(((unsigned short) 1) << 3)
+#  define ERTS_POLL_FD_FLG_RST		(((unsigned short) 1) << 1)
+
 typedef struct {
 #if ERTS_POLL_USE_POLL
     int pix;
 #endif
+
     ErtsPollEvents used_events;
     ErtsPollEvents events;
-#if ERTS_POLL_COALESCE_KP_RES
-    unsigned short res_ev_ix;
-#endif
     unsigned short flags;
 
 } ErtsFdStatus;
 
-
-#if ERTS_POLL_COALESCE_KP_RES
-/* res_ev_ix max value */
-#define ERTS_POLL_MAX_RES ((1 << sizeof(unsigned short)*8) - 1)
-#endif
-
-#if ERTS_POLL_USE_KQUEUE
-
-#define ERTS_POLL_KQ_OP_HANDLED			1
-#define ERTS_POLL_KQ_OP_DEL_R			2
-#define ERTS_POLL_KQ_OP_DEL_W			3
-#define ERTS_POLL_KQ_OP_ADD_R			4
-#define ERTS_POLL_KQ_OP_ADD_W			5
-#define ERTS_POLL_KQ_OP_ADD2_R			6
-#define ERTS_POLL_KQ_OP_ADD2_W			7
-
 #endif
 
 /*
@@ -233,196 +247,172 @@ typedef struct {
  * get unique names in debugger for kp/nkp
  */
 struct ERTS_POLL_EXPORT(erts_pollset) {
-    ErtsPollSet next;
+    int id;
     int internal_fd_limit;
-    ErtsFdStatus *fds_status;
     erts_atomic_t no_of_user_fds;
-    int fds_status_len;
+
 #if ERTS_POLL_USE_KERNEL_POLL
     int kp_fd;
-    int res_events_len;
-#if ERTS_POLL_USE_EPOLL
-    struct epoll_event *res_events;
-#elif ERTS_POLL_USE_KQUEUE
-    struct kevent *res_events;
-#elif ERTS_POLL_USE_DEVPOLL
-    struct pollfd *res_events;
-#endif
 #endif /* ERTS_POLL_USE_KERNEL_POLL */
+
 #if ERTS_POLL_USE_POLL
     int next_poll_fds_ix;
     int no_poll_fds;
     int poll_fds_len;
-    struct pollfd*poll_fds;
+    struct pollfd *poll_fds;
 #elif ERTS_POLL_USE_SELECT
     int next_sel_fd;
     int max_fd;
-#if ERTS_POLL_USE_FALLBACK
-    int no_select_fds;
-#endif
     ERTS_fd_set input_fds;
     ERTS_fd_set res_input_fds;
     ERTS_fd_set output_fds;
     ERTS_fd_set res_output_fds;
+#elif ERTS_POLL_USE_DEVPOLL
+    struct pollfd *poll_fds;
+    int poll_fds_ix;
 #endif
+
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
+    ErtsFdStatus *fds_status;
+    int fds_status_len;
     ErtsPollSetUpdateRequestsBlock update_requests;
     ErtsPollSetUpdateRequestsBlock *curr_upd_req_block;
     erts_atomic32_t have_update_requests;
-    erts_atomic32_t polled;
     erts_mtx_t mtx;
     int wake_fds[2];
-#if ERTS_POLL_USE_TIMERFD
-    int timer_fd;
-#endif
-#if ERTS_POLL_USE_FALLBACK
-    int fallback_used;
-#endif
     erts_atomic32_t wakeup_state;
-    erts_atomic64_t timeout_time;
-#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
-    erts_atomic_t no_avoided_wakeups;
-    erts_atomic_t no_avoided_interrupts;
-    erts_atomic_t no_interrupt_timed;
 #endif
 };
 
 void erts_silence_warn_unused_result(long unused);
 static void fatal_error(char *format, ...);
-static void fatal_error_async_signal_safe(char *error_str);
 
 static int max_fds = -1;
-static ErtsPollSet pollsets;
-static erts_mtx_t pollsets_lock;
 
 #if ERTS_POLL_USE_POLL
 
+#if !ERTS_POLL_IS_FALLBACK
+static ERTS_INLINE short ev2pollev(ErtsPollEvents ev)
+{
+    return ERTS_POLL_EV_E2N(ev);
+}
+
+static ERTS_INLINE ErtsPollEvents pollev2ev(short ev)
+{
+    return ERTS_POLL_EV_N2E(ev);
+}
+
+#else /* ERTS_POLL_IS_FALLBACK */
+
 static ERTS_INLINE short
 ev2pollev(ErtsPollEvents ev)
 {
-#if !ERTS_POLL_USE_FALLBACK || ERTS_POLL_USE_KQUEUE
-    return ERTS_POLL_EV_E2N(ev);
-#else /* Note, we only map events we are interested in */
     short res_ev = (short) 0;
     if (ev & ERTS_POLL_EV_IN)
-	res_ev |= ERTS_POLL_EV_NKP_IN;
+       res_ev |= ERTS_POLL_EV_NKP_IN;
     if (ev & ERTS_POLL_EV_OUT)
-	res_ev |= ERTS_POLL_EV_NKP_OUT;
+       res_ev |= ERTS_POLL_EV_NKP_OUT;
     return res_ev;
-#endif
 }
 
 static ERTS_INLINE ErtsPollEvents
 pollev2ev(short ev)
 {
-#if !ERTS_POLL_USE_FALLBACK || ERTS_POLL_USE_KQUEUE
-    return ERTS_POLL_EV_N2E(ev);
-#else /* Note, we only map events we are interested in */
     ErtsPollEvents res_ev = (ErtsPollEvents) 0;
     if (ev & ERTS_POLL_EV_NKP_IN)
-	res_ev |= ERTS_POLL_EV_IN;
+       res_ev |= ERTS_POLL_EV_IN;
     if (ev & ERTS_POLL_EV_NKP_OUT)
-	res_ev |= ERTS_POLL_EV_OUT;
+       res_ev |= ERTS_POLL_EV_OUT;
     if (ev & ERTS_POLL_EV_NKP_ERR)
-	res_ev |= ERTS_POLL_EV_ERR;
+       res_ev |= ERTS_POLL_EV_ERR;
     if (ev & ERTS_POLL_EV_NKP_NVAL)
-	res_ev |= ERTS_POLL_EV_NVAL;
-    return res_ev;
-#endif
+       res_ev |= ERTS_POLL_EV_NVAL;
+   return res_ev;
 }
 
-#endif
+#endif /* !ERTS_POLL_IS_FALLBACK */
+
+#endif /* ERTS_POLL_USE_POLL */
+
 
 #ifdef HARD_DEBUG
 static void check_poll_result(ErtsPollResFd pr[], int len);
-#if ERTS_POLL_USE_DEVPOLL
-static void check_poll_status(ErtsPollSet ps);
-#endif /* ERTS_POLL_USE_DEVPOLL */
 #endif /* HARD_DEBUG */
-#ifdef ERTS_POLL_DEBUG_PRINT
+#if ERTS_POLL_USE_DEVPOLL && defined(DEBUG)
+static void check_poll_status(ErtsPollSet *ps);
+#endif /* ERTS_POLL_USE_DEVPOLL && DEBUG */
 static void print_misc_debug_info(void);
+#if ERTS_POLL_USE_EPOLL
+uint32_t epoll_events(int kp_fd, int fd);
 #endif
 
-static ERTS_INLINE void
-init_timeout_time(ErtsPollSet ps)
-{
-    erts_atomic64_init_nob(&ps->timeout_time,
-			   (erts_aint64_t) ERTS_MONOTONIC_TIME_MAX);
-}
-
-static ERTS_INLINE void
-set_timeout_time(ErtsPollSet ps, ErtsMonotonicTime time)
-{
-    erts_atomic64_set_relb(&ps->timeout_time,
-			   (erts_aint64_t) time);
-}
-
-static ERTS_INLINE ErtsMonotonicTime
-get_timeout_time(ErtsPollSet ps)
-{
-    return (ErtsMonotonicTime) erts_atomic64_read_acqb(&ps->timeout_time);
-}
 
 #define ERTS_POLL_NOT_WOKEN	0
 #define ERTS_POLL_WOKEN		-1
 #define ERTS_POLL_WOKEN_INTR	1
 
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
 static ERTS_INLINE void
-reset_wakeup_state(ErtsPollSet ps)
+reset_wakeup_state(ErtsPollSet *ps)
 {
     erts_atomic32_set_mb(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
 }
+#endif
 
 static ERTS_INLINE int
-is_woken(ErtsPollSet ps)
+is_woken(ErtsPollSet *ps)
 {
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     return erts_atomic32_read_acqb(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN;
+#else
+    return 0;
+#endif
 }
 
 static ERTS_INLINE int
-is_interrupted_reset(ErtsPollSet ps)
+is_interrupted_reset(ErtsPollSet *ps)
 {
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     return (erts_atomic32_xchg_acqb(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN)
 	    == ERTS_POLL_WOKEN_INTR);
+#else
+    return 0;
+#endif
 }
 
 static ERTS_INLINE void
-woke_up(ErtsPollSet ps)
+woke_up(ErtsPollSet *ps)
 {
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     erts_aint32_t wakeup_state = erts_atomic32_read_acqb(&ps->wakeup_state);
     if (wakeup_state == ERTS_POLL_NOT_WOKEN)
 	(void) erts_atomic32_cmpxchg_nob(&ps->wakeup_state,
 					 ERTS_POLL_WOKEN,
 					 ERTS_POLL_NOT_WOKEN);
     ASSERT(erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN);
+#endif
 }
 
 /*
  * --- Wakeup pipe -----------------------------------------------------------
  */
 
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
 
 static ERTS_INLINE void
-wake_poller(ErtsPollSet ps, int interrupted, int async_signal_safe)
+wake_poller(ErtsPollSet *ps, int interrupted)
 {
     int wake;
-    if (async_signal_safe)
-	wake = 1;
-    else {
-	erts_aint32_t wakeup_state;
-	if (!interrupted)
-	    wakeup_state = erts_atomic32_cmpxchg_relb(&ps->wakeup_state,
-						      ERTS_POLL_WOKEN,
-						      ERTS_POLL_NOT_WOKEN);
-	else
-            wakeup_state = erts_atomic32_xchg_relb(&ps->wakeup_state,
-                                                   ERTS_POLL_WOKEN_INTR);
-	wake = wakeup_state == ERTS_POLL_NOT_WOKEN;
-    }
-    /*
-     * NOTE: This function might be called from signal handlers in the
-     *       non-smp case; therefore, it has to be async-signal safe in
-     *       the non-smp case.
-     */
+    erts_aint32_t wakeup_state;
+    if (!interrupted)
+        wakeup_state = erts_atomic32_cmpxchg_relb(&ps->wakeup_state,
+                                                  ERTS_POLL_WOKEN,
+                                                  ERTS_POLL_NOT_WOKEN);
+    else
+        wakeup_state = erts_atomic32_xchg_relb(&ps->wakeup_state,
+                                               ERTS_POLL_WOKEN_INTR);
+    wake = wakeup_state == ERTS_POLL_NOT_WOKEN;
+
     if (wake) {
 	ssize_t res;
 	if (ps->wake_fds[1] < 0)
@@ -432,29 +422,27 @@ wake_poller(ErtsPollSet ps, int interrupted, int async_signal_safe)
 	    res = write(ps->wake_fds[1], "!", 1);
 	} while (res < 0 && errno == EINTR);
 	if (res <= 0 && errno != ERRNO_BLOCK) {
-	    if (async_signal_safe)
-		fatal_error_async_signal_safe(__FILE__
-					      ":XXX:wake_poller(): "
-					      "Failed to write on wakeup pipe\n");
-	    else
-		fatal_error("%s:%d:wake_poller(): "
-			    "Failed to write to wakeup pipe fd=%d: "
-			    "%s (%d)\n",
-			    __FILE__, __LINE__,
-			    ps->wake_fds[1],
-			    erl_errno_id(errno), errno);
+            fatal_error("%s:%d:wake_poller(): "
+                        "Failed to write to wakeup pipe fd=%d: "
+                        "%s (%d)\n",
+                        __FILE__, __LINE__,
+                        ps->wake_fds[1],
+                        erl_errno_id(errno), errno);
 	}
     }
 }
 
 static ERTS_INLINE void
-cleanup_wakeup_pipe(ErtsPollSet ps)
+cleanup_wakeup_pipe(ErtsPollSet *ps)
 {
+    int intr = 0;
     int fd = ps->wake_fds[0];
     int res;
     do {
 	char buf[32];
 	res = read(fd, buf, sizeof(buf));
+	if (res > 0)
+	    intr = 1;
     } while (res > 0 || (res < 0 && errno == EINTR));
     if (res < 0 && errno != ERRNO_BLOCK) {
 	fatal_error("%s:%d:cleanup_wakeup_pipe(): "
@@ -464,10 +452,12 @@ cleanup_wakeup_pipe(ErtsPollSet ps)
 		    fd,
 		    erl_errno_id(errno), errno);
     }
+    if (intr)
+	erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_INTR);
 }
 
 static void
-create_wakeup_pipe(ErtsPollSet ps)
+create_wakeup_pipe(ErtsPollSet *ps)
 {
     int do_wake = 0;
     int wake_fds[2];
@@ -484,20 +474,13 @@ create_wakeup_pipe(ErtsPollSet ps)
     SET_NONBLOCKING(wake_fds[0]);
     SET_NONBLOCKING(wake_fds[1]);
 
-#ifdef ERTS_POLL_DEBUG_PRINT
-    erts_printf("wakeup fds = {%d, %d}\n", wake_fds[0], wake_fds[1]);
-#endif
+    DEBUG_PRINT("wakeup fds = {%d, %d}", ps, wake_fds[0], wake_fds[1]);
 
     ERTS_POLL_EXPORT(erts_poll_control)(ps,
 					wake_fds[0],
+                                        ERTS_POLL_OP_ADD,
 					ERTS_POLL_EV_IN,
-					1, &do_wake);
-#if ERTS_POLL_USE_FALLBACK
-    /* We depend on the wakeup pipe being handled by kernel poll */
-    if (ps->fds_status[wake_fds[0]].flags & ERTS_POLL_FD_FLG_INFLBCK)
-	fatal_error("%s:%d:create_wakeup_pipe(): Internal error\n",
-		    __FILE__, __LINE__);
-#endif
+                                        &do_wake);
     if (ps->internal_fd_limit <= wake_fds[1])
 	ps->internal_fd_limit = wake_fds[1] + 1;
     if (ps->internal_fd_limit <= wake_fds[0])
@@ -507,81 +490,12 @@ create_wakeup_pipe(ErtsPollSet ps)
 }
 
 
-/*
- * --- timer fd -----------------------------------------------------------
- */
-
-#if ERTS_POLL_USE_TIMERFD
-
-/* We use the timerfd when using epoll_wait to get high accuracy
-   timeouts, i.e. we want to sleep with < ms accuracy. */
-
-static void
-create_timerfd(ErtsPollSet ps)
-{
-    int do_wake = 0;
-    int timer_fd;
-    timer_fd = timerfd_create(CLOCK_MONOTONIC,0);
-    ERTS_POLL_EXPORT(erts_poll_control)(ps,
-					timer_fd,
-					ERTS_POLL_EV_IN,
-					1, &do_wake);
-#if ERTS_POLL_USE_FALLBACK
-    /* We depend on the wakeup pipe being handled by kernel poll */
-    if (ps->fds_status[timer_fd].flags & ERTS_POLL_FD_FLG_INFLBCK)
-	fatal_error("%s:%d:create_wakeup_pipe(): Internal error\n",
-		    __FILE__, __LINE__);
-#endif
-    if (ps->internal_fd_limit <= timer_fd)
-	ps->internal_fd_limit = timer_fd + 1;
-    ps->timer_fd = timer_fd;
-}
-
-static ERTS_INLINE void
-timerfd_set(ErtsPollSet ps, struct itimerspec *its)
-{
-#ifdef DEBUG
-    struct itimerspec old_its;
-    int res;
-    res = timerfd_settime(ps->timer_fd, 0, its, &old_its);
-    ASSERT(res == 0);
-    ASSERT(old_its.it_interval.tv_sec == 0 &&
-           old_its.it_interval.tv_nsec == 0 &&
-           old_its.it_value.tv_sec == 0 &&
-           old_its.it_value.tv_nsec == 0);
-
-#else
-    timerfd_settime(ps->timer_fd, 0, its, NULL);
-#endif
-}
-
-static ERTS_INLINE int
-timerfd_clear(ErtsPollSet ps, int res, int max_res) {
-
-    struct itimerspec its;
-    /* we always have to clear the timer */
-    its.it_interval.tv_sec = 0;
-    its.it_interval.tv_nsec = 0;
-    its.it_value.tv_sec = 0;
-    its.it_value.tv_nsec = 0;
-    timerfd_settime(ps->timer_fd, 0, &its, NULL);
-
-    /* only timeout fd triggered */
-    if (res == 1 && ps->res_events[0].data.fd == ps->timer_fd)
-        return 0;
-
-    return res;
-}
-
-#endif /* ERTS_POLL_USE_TIMERFD */
-
-
 /*
  * --- Poll set update requests ----------------------------------------------
  */
 
 static ERTS_INLINE void
-enqueue_update_request(ErtsPollSet ps, int fd)
+enqueue_update_request(ErtsPollSet *ps, int fd)
 {
     ErtsPollSetUpdateRequestsBlock *urqbp;
 
@@ -596,13 +510,11 @@ enqueue_update_request(ErtsPollSet ps, int fd)
     urqbp = ps->curr_upd_req_block;
 
     if (urqbp->len == ERTS_POLLSET_UPDATE_REQ_BLOCK_SIZE) {
-	ASSERT(!urqbp->next);
 	urqbp = erts_alloc(ERTS_ALC_T_POLLSET_UPDREQ,
 			   sizeof(ErtsPollSetUpdateRequestsBlock));
-	ps->curr_upd_req_block->next = urqbp;
-	ps->curr_upd_req_block = urqbp;
-	urqbp->next = NULL;
+	urqbp->next = ps->curr_upd_req_block;
 	urqbp->len = 0;
+	ps->curr_upd_req_block = urqbp;
     }
 
     ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INURQ;
@@ -610,28 +522,29 @@ enqueue_update_request(ErtsPollSet ps, int fd)
 }
 
 static ERTS_INLINE void
-free_update_requests_block(ErtsPollSet ps,
+free_update_requests_block(ErtsPollSet *ps,
 			   ErtsPollSetUpdateRequestsBlock *urqbp)
 {
     if (urqbp != &ps->update_requests)
 	erts_free(ERTS_ALC_T_POLLSET_UPDREQ, (void *) urqbp);
     else {
-	urqbp->next = NULL;
 	urqbp->len = 0;
     }
 }
 
+#endif /* !ERTS_POLL_USE_CONCURRENT_UPDATE */
 
 /*
  * --- Growing poll set structures -------------------------------------------
  */
 
-#ifndef ERTS_KERNEL_POLL_VERSION   /* only one shared implementation */
+#if !ERTS_NO_KERNEL_POLL_VERSION || !ERTS_ENABLE_KERNEL_POLL
+/* only one shared implementation */
 
 #define ERTS_FD_TABLE_MIN_LENGTH	1024
 #define ERTS_FD_TABLE_EXP_THRESHOLD	(2048*1024)
 
-int erts_poll_new_table_len (int old_len, int need_len)
+int erts_poll_new_table_len(int old_len, int need_len)
 {
     int new_len;
 
@@ -641,7 +554,7 @@ int erts_poll_new_table_len (int old_len, int need_len)
     }
     else {
         new_len = old_len;
-        do {            
+        do {
             if (new_len < ERTS_FD_TABLE_EXP_THRESHOLD)
                 new_len *= 2;
             else
@@ -654,30 +567,9 @@ int erts_poll_new_table_len (int old_len, int need_len)
 }
 #endif
 
-#if ERTS_POLL_USE_KERNEL_POLL
-static void
-grow_res_events(ErtsPollSet ps, int new_len)
-{
-    size_t new_size = sizeof(
-#if ERTS_POLL_USE_EPOLL
-	struct epoll_event
-#elif ERTS_POLL_USE_DEVPOLL
-	struct pollfd
-#elif ERTS_POLL_USE_KQUEUE
-	struct kevent
-#endif
-	) * erts_poll_new_table_len(ps->res_events_len, new_len);
-    /* We do not need to save previously stored data */
-    if (ps->res_events)
-	erts_free(ERTS_ALC_T_POLL_RES_EVS, ps->res_events);
-    ps->res_events = erts_alloc(ERTS_ALC_T_POLL_RES_EVS, new_size);
-    ps->res_events_len = new_len;
-}
-#endif /* ERTS_POLL_USE_KERNEL_POLL */
-
 #if ERTS_POLL_USE_POLL
 static void
-grow_poll_fds(ErtsPollSet ps, int min_ix)
+grow_poll_fds(ErtsPollSet *ps, int min_ix)
 {
     int i;
     int new_len = erts_poll_new_table_len(ps->poll_fds_len, min_ix + 1);
@@ -725,8 +617,9 @@ ensure_select_fds(int fd, ERTS_fd_set* in, ERTS_fd_set* out)
 #  define ensure_select_fds(fd, in, out) do {} while(0)
 #endif /* _DARWIN_UNLIMITED_SELECT */
 
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
 static void
-grow_fds_status(ErtsPollSet ps, int min_fd)
+grow_fds_status(ErtsPollSet *ps, int min_fd)
 {
     int i;
     int new_len = erts_poll_new_table_len(ps->fds_status_len, min_fd + 1);
@@ -745,95 +638,290 @@ grow_fds_status(ErtsPollSet ps, int min_fd)
 #endif
 	ps->fds_status[i].used_events = (ErtsPollEvents) 0;
 	ps->fds_status[i].events = (ErtsPollEvents) 0;
-#if ERTS_POLL_COALESCE_KP_RES
-	ps->fds_status[i].res_ev_ix = (unsigned short) ERTS_POLL_MAX_RES;
-#endif
 	ps->fds_status[i].flags = (unsigned short) 0;
     }
     ps->fds_status_len = new_len;
 }
+#endif
 
 /*
  * --- Selecting fd to poll on -----------------------------------------------
  */
 
-#if ERTS_POLL_USE_FALLBACK
-static int update_fallback_pollset(ErtsPollSet ps, int fd);
-#endif
-
-static ERTS_INLINE int
-need_update(ErtsPollSet ps, int fd)
+#if ERTS_POLL_USE_EPOLL
+static int
+update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events)
 {
-#if ERTS_POLL_USE_KERNEL_POLL
-    int reset;
-#endif
+    int res;
+    int epoll_op = EPOLL_CTL_MOD;
+    struct epoll_event epe_templ;
+    struct epoll_event epe;
 
-    ASSERT(fd < ps->fds_status_len);
+    epe_templ.events = ERTS_POLL_EV_E2N(events) | EPOLLONESHOT;
+    epe_templ.data.fd = fd;
 
-#if ERTS_POLL_USE_KERNEL_POLL
-    reset = (int) (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST);
-    if (reset && !ps->fds_status[fd].used_events) {
-	ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST;
-	reset = 0;
-    }
-#else
-    ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST;
+#ifdef VALGRIND
+    /* Silence invalid valgrind warning ... */
+    memset((void *) &epe.data, 0, sizeof(epoll_data_t));
 #endif
 
-    if (ps->fds_status[fd].used_events != ps->fds_status[fd].events)
-	return 1;
+    switch (op) {
+    case ERTS_POLL_OP_DEL:
+	/* A note on EPOLL_CTL_DEL: linux kernel versions before 2.6.9
+	   need a non-NULL event pointer even though it is ignored... */
+        epoll_op = EPOLL_CTL_DEL;
+        epe_templ.events = 0;
+	erts_atomic_dec_nob(&ps->no_of_user_fds);
+        break;
+    case ERTS_POLL_OP_ADD:
+        epoll_op = EPOLL_CTL_ADD;
+	erts_atomic_inc_nob(&ps->no_of_user_fds);
+        break;
+    case ERTS_POLL_OP_MOD:
+        epoll_op = EPOLL_CTL_MOD;
+        break;
+    default:
+        ASSERT(0);
+        break;
+    }
 
-#if ERTS_POLL_USE_KERNEL_POLL
-    return reset;
-#else
-    return 0;
+    do {
+	/* We init 'epe' every time since epoll_ctl() may modify it
+	   (not declared const and not documented as const). */
+	epe.events = epe_templ.events;
+	epe.data.fd = epe_templ.data.fd;
+	res = epoll_ctl(ps->kp_fd, epoll_op, fd, &epe);
+    } while (res != 0 && errno == EINTR);
+
+#if ERTS_POLL_DEBUG_PRINT
+    {
+	int saved_errno = errno;
+	DEBUG_PRINT_FD("%s = epoll_ctl(%d, %s, %d, {0x%x, %d})",
+                       ps, fd,
+                       res == 0 ? "0" : erl_errno_id(errno),
+                       ps->kp_fd,
+                       (epoll_op == EPOLL_CTL_ADD
+                        ? "EPOLL_CTL_ADD"
+                        : (epoll_op == EPOLL_CTL_MOD
+                           ? "EPOLL_CTL_MOD"
+                           : (epoll_op == EPOLL_CTL_DEL
+                              ? "EPOLL_CTL_DEL"
+                              : "UNKNOWN"))),
+                       fd,
+                       epe_templ.events,
+                       fd);
+	errno = saved_errno;
+    }
 #endif
+    if (res != 0) {
+	switch (op) {
+	case ERTS_POLL_OP_MOD:
+	    epe.events = 0;
+	    do {
+		/* We init 'epe' every time since epoll_ctl() may modify it
+		   (not declared const and not documented as const). */
+		epe.events = 0;
+		epe.data.fd = fd;
+		res = epoll_ctl(ps->kp_fd, EPOLL_CTL_DEL, fd, &epe);
+	    } while (res != 0 && errno == EINTR);
+	/* Fall through ... */
+	case ERTS_POLL_OP_ADD: {
+	    erts_atomic_dec_nob(&ps->no_of_user_fds);
+            res = ERTS_POLL_EV_NVAL;
+	    break;
+	}
+	case ERTS_POLL_OP_DEL: {
+	    /*
+	     * Since we use a lazy update approach EPOLL_CTL_DEL will
+	     * frequently fail. This since epoll automatically removes
+	     * a filedescriptor that is closed from the poll set.
+	     */
+	    res = 0;
+	    break;
+	}
+	default:
+	    fatal_error("%s:%d:update_pollset(): Internal error\n",
+			__FILE__, __LINE__);
+	    break;
+	}
+    } else {
+        res = events;
+    }
+    return res;
 }
 
-#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET
+#endif /* ERTS_POLL_USE_EPOLL */
 
 #if ERTS_POLL_USE_KQUEUE
-#define ERTS_POLL_MIN_BATCH_BUF_SIZE 128
+
+/* Some versions of the EV_SET macro used kevp multiple times,
+   so we define out own version that make sure that it is safe
+   to do kevp++ in the argument list. */
+#define ERTS_EV_SET(kevp, a, b, c, f) do {              \
+        struct kevent *kevp_ = kevp;                    \
+        EV_SET(kevp_, a, b, c, 0, 0, f);                \
+    } while(0)
+
+static int
+update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events)
+{
+    int res = 0, len = 0;
+    struct kevent evts[2];
+    struct timespec ts = {0, 0};
+
+#ifdef EV_DISPATCH
+    /* If we have EV_DISPATCH we use it. The kevent descriptions for both
+       read and write are added on OP_ADD and removed on OP_DEL. And then
+       after than only EV_ENABLE|EV_DISPATCH are used.
+
+       It could be possible to not modify the pollset when disabling and/or
+       deleting events, but that may cause the poll threads to be awoken
+       a lot more than they should so we take the cost here instead of
+       in the poll thread.
+
+       Note: We need to have EV_DISPATCH both when the event is enabled and
+       disabled, as otherwise the event may be triggered twice on each re-arm.
+       Not sure if this is intended or not (can't find anything about it in the
+       man page), but it seems to be the way it works...
+    */
+
+    if (op == ERTS_POLL_OP_DEL) {
+        erts_atomic_dec_nob(&ps->no_of_user_fds);
+        /* We could probably skip this delete, do we want to? */
+        ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, EV_DELETE, (void *) 0);
+        ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, EV_DELETE, (void *) 0);
+    } else if (op == ERTS_POLL_OP_ADD) {
+        uint32_t flags;
+        erts_atomic_inc_nob(&ps->no_of_user_fds);
+
+        flags = EV_ADD|EV_DISPATCH;
+        flags |= ((events & ERTS_POLL_EV_IN) ? 0 : EV_DISABLE);
+        ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, flags, (void *) ERTS_POLL_EV_IN);
+
+        flags = EV_ADD|EV_DISPATCH;
+        flags |= ((events & ERTS_POLL_EV_OUT) ? 0 : EV_DISABLE);
+        ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, flags, (void *) ERTS_POLL_EV_OUT);
+    } else {
+        uint32_t flags;
+        ASSERT(op == ERTS_POLL_OP_MOD);
+
+        flags = EV_DISPATCH;
+        flags |= (events & ERTS_POLL_EV_IN) ? EV_ENABLE : EV_DISABLE;
+        ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, flags, (void *) ERTS_POLL_EV_IN);
+
+        flags = EV_DISPATCH;
+        flags |= (events & ERTS_POLL_EV_OUT) ? EV_ENABLE : EV_DISABLE;
+        ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, flags, (void *) ERTS_POLL_EV_OUT);
+    }
 #else
-#define ERTS_POLL_MIN_BATCH_BUF_SIZE 64
-#endif
+    uint32_t flags = EV_ADD|EV_ONESHOT;
 
-typedef struct {
-    int len;
-    int size;
-#if ERTS_POLL_USE_DEVPOLL
-    struct pollfd *buf;
-#elif ERTS_POLL_USE_KQUEUE
-    struct kevent *buf;
-    struct kevent *ebuf;
+    if (op == ERTS_POLL_OP_DEL) {
+        erts_atomic_dec_nob(&ps->no_of_user_fds);
+        /* We don't do anything when a delete is issued. The fds will be removed
+           when they are triggered, or when they are closed. */
+        events = 0;
+    } else if (op == ERTS_POLL_OP_ADD) {
+        erts_atomic_inc_nob(&ps->no_of_user_fds);
+    }
+
+    if (events & ERTS_POLL_EV_IN) {
+        ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, flags, (void *) ERTS_POLL_EV_IN);
+    }
+    if (events & ERTS_POLL_EV_OUT) {
+        ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, flags, (void *) ERTS_POLL_EV_OUT);
+    }
+
+#endif
+    if (len)
+        do {
+            res = kevent(ps->kp_fd, evts, len, NULL, 0, &ts);
+        } while (res < 0 && errno == EINTR);
+#if ERTS_POLL_DEBUG_PRINT
+    {
+        int saved_errno = errno, i;
+        char keventb[255], *keventbp = keventb;
+        if (res < 0)
+            keventbp += sprintf(keventbp,"%s = ",erl_errno_id(saved_errno));
+        else
+            keventbp += sprintf(keventbp,"%d = ",res);
+        keventbp += sprintf(keventbp, "kevent(%d, {",ps->kp_fd);
+        for (i = 0; i < len; i++) {
+            const char *flags = "UNKNOWN";
+            if (evts[i].flags == EV_DELETE) flags = "EV_DELETE";
+            if (evts[i].flags == (EV_ADD|EV_ONESHOT)) flags = "EV_ADD|EV_ONESHOT";
+#ifdef EV_DISPATCH
+            if (evts[i].flags == (EV_ADD|EV_DISPATCH)) flags = "EV_ADD|EV_DISPATCH";
+            if (evts[i].flags == (EV_ADD|EV_DISABLE)) flags = "EV_ADD|EV_DISABLE";
+            if (evts[i].flags == (EV_ENABLE|EV_DISPATCH)) flags = "EV_ENABLE|EV_DISPATCH";
+            if (evts[i].flags == EV_DISABLE) flags = "EV_DISABLE";
+            if (evts[i].flags == (EV_DISABLE|EV_DISPATCH)) flags = "EV_DISABLE|EV_DISABLE";
+#endif
+
+            keventbp += sprintf(keventbp, "%s{%lu, %s, %s}",i > 0 ? ", " : "",
+                                evts[i].ident,
+                                (evts[i].filter == EVFILT_READ
+                                 ? "EVFILT_READ"
+                                 : (evts[i].filter == EVFILT_WRITE
+                                    ? "EVFILT_WRITE"
+                                    : "UNKNOWN")), flags);
+        }
+        keventbp += sprintf(keventbp, "}, %d)", len);
+        DEBUG_PRINT_FD("%s", ps, fd, keventb);
+        errno = saved_errno;
+    }
 #endif
-} ErtsPollBatchBuf;
+    if (res < 0) {
+        if (op != ERTS_POLL_OP_DEL) {
+#ifdef EV_RECEIPT
+            struct kevent receipt_evts[2];
+            len = 0;
+            ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, EV_DELETE|EV_RECEIPT, (void *) 0);
+            ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, EV_DELETE|EV_RECEIPT, (void *) 0);
+            do {
+                res = kevent(ps->kp_fd, evts, len, receipt_evts, 2, &ts);
+            } while (res < 0 && errno == EINTR);
+#else
+            ERTS_EV_SET(&evts[0], fd, EVFILT_WRITE, EV_DELETE, (void *) 0);
+            do {
+                res = kevent(ps->kp_fd, evts, 1, NULL, 0, &ts);
+            } while (res < 0 && errno == EINTR);
+            ERTS_EV_SET(&evts[0], fd, EVFILT_READ, EV_DELETE, (void *) 0);
+            do {
+                res = kevent(ps->kp_fd, evts, 1, NULL, 0, &ts);
+            } while (res < 0 && errno == EINTR);
+#endif
+            if (op == ERTS_POLL_OP_ADD)
+                erts_atomic_dec_nob(&ps->no_of_user_fds);
+            return ERTS_POLL_EV_NVAL;
+        }
+        return 0;
+    }
+    return events;
+}
+
+#endif /* ERTS_POLL_USE_KQUEUE */
 
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
 
 static ERTS_INLINE void
-setup_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
+init_batch_update(ErtsPollSet *ps, int len)
 {
-    bbp->len = 0;
 #if ERTS_POLL_USE_DEVPOLL
-    bbp->size = ps->res_events_len;
-    bbp->buf = ps->res_events;
-#elif ERTS_POLL_USE_KQUEUE
-    bbp->size = ps->res_events_len/2;
-    bbp->buf = ps->res_events;
-    bbp->ebuf = bbp->buf + bbp->size;
+    ASSERT(ps->poll_fds == NULL);
+    ps->poll_fds = erts_alloc(ERTS_ALC_T_TMP, sizeof(struct pollfd) * len);
+    ps->poll_fds_ix = 0;
 #endif
 }
 
-
-#if ERTS_POLL_USE_DEVPOLL
-
-static void
-write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
+static ERTS_INLINE void
+write_batch_update(ErtsPollSet *ps)
 {
+#if ERTS_POLL_USE_DEVPOLL
     ssize_t wres;
-    char *buf = (char *) bbp->buf;
-    size_t buf_size = sizeof(struct pollfd)*bbp->len;
-    
+    char *buf = (char *) ps->poll_fds;
+    size_t buf_size = sizeof(struct pollfd)*ps->poll_fds_ix;
+
     while (1) {
 	wres = write(ps->kp_fd, (void *) buf, buf_size);
 	if (wres < 0) {
@@ -845,9 +933,33 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
 			__FILE__, __LINE__,
 			erl_errno_id(errno), errno);
 	}
-	buf_size -= wres;
-	if (buf_size <= 0)
-	    break;
+#if ERTS_POLL_DEBUG_PRINT
+        {
+            int saved_errno = errno, i;
+            char devpollb[2048], *devpollbp = devpollb;
+            devpollbp += sprintf(devpollbp, "%d = devpoll(%d, {", wres, ps->kp_fd);
+            for (i = 0; i < wres / sizeof(struct pollfd); i++) {
+                if (devpollbp == devpollb)
+                    devpollbp += sprintf(devpollbp, "%d = devpoll(%d, {", wres, ps->kp_fd);
+                devpollbp += sprintf(devpollbp, "%s{fd = %d, events = %s}",
+                                     i > 0 ? ", " : "",
+                                     ps->poll_fds[i].fd,
+                                     ev2str(ps->poll_fds[i].events));
+                if (devpollbp - devpollb > 512) {
+                    devpollbp += sprintf(devpollbp, "}, %d)", ps->poll_fds_ix);
+                    DEBUG_PRINT("%s", ps, devpollb);
+                    devpollbp = devpollb;
+                }
+            }
+            devpollbp += sprintf(devpollbp, "}, %d)", ps->poll_fds_ix);
+            DEBUG_PRINT("%s", ps, devpollb);
+            errno = saved_errno;
+        }
+#endif
+
+	buf_size -= wres;
+	if (buf_size <= 0)
+	    break;
 	buf += wres;
     }
 
@@ -855,483 +967,54 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
 	fatal_error("%s:%d:write_devpoll_buf(): Internal error\n",
 		    __FILE__, __LINE__);
     }
-    bbp->len = 0;
-}
-
-#elif ERTS_POLL_USE_KQUEUE
-
-static void
-write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
-{
-    int res;
-    int len = bbp->len;
-    struct kevent *buf = bbp->buf;
-    struct timespec ts = {0, 0};
-
-    do {
-	res = kevent(ps->kp_fd, buf, len, NULL, 0, &ts);
-    } while (res < 0 && errno == EINTR);
-    if (res < 0) {
-	int i;
-	struct kevent *ebuf = bbp->ebuf;
-	do {
-	    res = kevent(ps->kp_fd, buf, len, ebuf, len, &ts);
-	} while (res < 0 && errno == EINTR);
-	if (res < 0) {
-	    fatal_error("%s:%d: kevent() failed: %s (%d)\n",
-			__FILE__, __LINE__, erl_errno_id(errno), errno);
-	}
-	for (i = 0; i < res; i++) {
-	    if (ebuf[i].flags & EV_ERROR) {
-		short filter;
-		int fd = (int) ebuf[i].ident;
-
-		switch ((int) (long) ebuf[i].udata) {
-
-		    /*
-		     * Since we use a lazy update approach EV_DELETE will
-		     * frequently fail. This since kqueue automatically
-		     * removes a file descriptor that is closed from the
-		     * poll set.
-		     */
-		case ERTS_POLL_KQ_OP_DEL_R:
-		case ERTS_POLL_KQ_OP_DEL_W:
-		case ERTS_POLL_KQ_OP_HANDLED:
-		    break;
-
-		    /*
-		     * According to the kqueue man page EVFILT_READ support
-		     * does not imply EVFILT_WRITE support; therefore,
-		     * if an EV_ADD fail, we may have to remove other
-		     * events on this fd in the kqueue pollset before
-		     * adding fd to the fallback pollset.
-		     */
-		case ERTS_POLL_KQ_OP_ADD_W:
-		    if (ps->fds_status[fd].used_events & ERTS_POLL_EV_IN) {
-			filter = EVFILT_READ;
-			goto rm_add_fb;
-		    }
-		    goto add_fb;
-		case ERTS_POLL_KQ_OP_ADD_R:
-		    if (ps->fds_status[fd].used_events & ERTS_POLL_EV_OUT) {
-			filter = EVFILT_WRITE;
-			goto rm_add_fb;
-		    }
-		    goto add_fb;
-		case ERTS_POLL_KQ_OP_ADD2_W:
-		case ERTS_POLL_KQ_OP_ADD2_R: {
-		    int j;
-		    for (j = i+1; j < res; j++) {
-			if (fd == (int) ebuf[j].ident) {
-			    ebuf[j].udata = (void *) ERTS_POLL_KQ_OP_HANDLED;
-			    if (!(ebuf[j].flags & EV_ERROR)) {
-				switch ((int) (long) ebuf[j].udata) {
-				case ERTS_POLL_KQ_OP_ADD2_W:
-				    filter = EVFILT_WRITE;
-				    goto rm_add_fb;
-				case ERTS_POLL_KQ_OP_ADD2_R:
-				    filter = EVFILT_READ;
-				    goto rm_add_fb;
-				default:
-				    fatal_error("%s:%d:write_batch_buf(): "
-						"Internal error",
-						__FILE__, __LINE__);
-				    break;
-				}
-			    }
-			    goto add_fb;
-			}
-		    }
-		    /* The other add succeded... */
-		    filter = ((((int) (long) ebuf[i].udata)
-			       == ERTS_POLL_KQ_OP_ADD2_W)
-			      ? EVFILT_READ
-			      : EVFILT_WRITE);
-		rm_add_fb:
-		    { 
-			struct kevent kev;
-			struct timespec ts = {0, 0};
-			EV_SET(&kev, fd, filter, EV_DELETE, 0, 0, 0);
-			(void) kevent(ps->kp_fd, &kev, 1, NULL, 0, &ts);
-		    }
-
-		add_fb:
-		    ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK;
-		    ASSERT(ps->fds_status[fd].used_events);
-		    ps->fds_status[fd].used_events = 0;
-		    erts_atomic_dec_nob(&ps->no_of_user_fds);
-		    update_fallback_pollset(ps, fd);
-		    ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK);
-		    break;
-		}
-		default:
-		    fatal_error("%s:%d:write_batch_buf(): Internal error",
-				__FILE__, __LINE__);
-		    break;
-		}
-	    }
-	}
-    }
-    bbp->len = 0;
-}
-
-#endif /* ERTS_POLL_USE_KQUEUE */
-
-static ERTS_INLINE void
-batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp)
-{
-    int buf_len;
-#if ERTS_POLL_USE_DEVPOLL
-    short events;
-    struct pollfd *buf;
-#elif ERTS_POLL_USE_KQUEUE
-    struct kevent *buf;
-#endif
-
-#ifdef ERTS_POLL_DEBUG_PRINT
-    erts_printf("Doing lazy update on fd=%d\n", fd);
-#endif
-
-    if (!need_update(ps, fd))
-	return;
-
-    /* Make sure we have room for at least maximum no of entries
-       per fd */
-    if (bbp->size - bbp->len < 2)
-	write_batch_buf(ps, bbp);
-
-    buf_len = bbp->len;
-    buf = bbp->buf;
-
-    ASSERT(fd < ps->fds_status_len);
-
-#if ERTS_POLL_USE_DEVPOLL
-    events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events);
-    if (!events) {
-	buf[buf_len].events = POLLREMOVE;
-	erts_atomic_dec_nob(&ps->no_of_user_fds);
-    }
-    else if (!ps->fds_status[fd].used_events) {
-	buf[buf_len].events = events;
-	erts_atomic_inc_nob(&ps->no_of_user_fds);
-    }
-    else {
-	if ((ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST)
-	    || (ps->fds_status[fd].used_events & ~events)) {
-	    /* Reset or removed events... */
-	    buf[buf_len].fd = fd;
-	    buf[buf_len].events = POLLREMOVE;
-	    buf[buf_len++].revents = 0;
-	}
-	buf[buf_len].events = events;
-    }
-    buf[buf_len].fd = fd;
-    buf[buf_len++].revents = 0;
-
-#elif ERTS_POLL_USE_KQUEUE
-
-    if (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK) {
-	if (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_USEFLBCK)
-	    update_fallback_pollset(ps, fd);
-	else { /* Remove from fallback and try kqueue */
-	    ErtsPollEvents events = ps->fds_status[fd].events;
-	    ps->fds_status[fd].events = (ErtsPollEvents) 0;
-	    update_fallback_pollset(ps, fd);
-	    ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK));
-	    if (events) {
-		ps->fds_status[fd].events = events;
-		goto try_kqueue;
-	    }
-	}
-    }
-    else {
-	ErtsPollEvents events, used_events;
-	int mod_w, mod_r;
-    try_kqueue:
-	events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events);
-	used_events = ERTS_POLL_EV_E2N(ps->fds_status[fd].used_events);
-	if (!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST)) {
-	    if (!used_events &&
-		(events & ERTS_POLL_EV_IN) && (events & ERTS_POLL_EV_OUT))
-		goto do_add_rw;
-	    mod_r = ((events & ERTS_POLL_EV_IN)
-		     != (used_events & ERTS_POLL_EV_IN));
-	    mod_w = ((events & ERTS_POLL_EV_OUT)
-		     != (used_events & ERTS_POLL_EV_OUT));
-	    goto do_mod;
-	}
-	else { /* Reset */
-	    if ((events & ERTS_POLL_EV_IN) && (events & ERTS_POLL_EV_OUT)) {
-	    do_add_rw:
-		EV_SET(&buf[buf_len], fd, EVFILT_READ, EV_ADD,
-		       0, 0, (void *) ERTS_POLL_KQ_OP_ADD2_R);
-		buf_len++;
-		EV_SET(&buf[buf_len], fd, EVFILT_WRITE, EV_ADD,
-		       0, 0, (void *) ERTS_POLL_KQ_OP_ADD2_W);
-		buf_len++;
-
-	    }
-	    else {
-		mod_r = 1;
-		mod_w = 1;
-	    do_mod:
-		if (mod_r) {
-		    if (events & ERTS_POLL_EV_IN) {
-			EV_SET(&buf[buf_len], fd, EVFILT_READ, EV_ADD,
-			       0, 0, (void *) ERTS_POLL_KQ_OP_ADD_R);
-			buf_len++;
-		    }
-		    else if (used_events & ERTS_POLL_EV_IN) {
-			EV_SET(&buf[buf_len], fd, EVFILT_READ, EV_DELETE,
-			       0, 0, (void *) ERTS_POLL_KQ_OP_DEL_R);
-			buf_len++;
-		    }
-		}
-		if (mod_w) {
-		    if (events & ERTS_POLL_EV_OUT) {
-			EV_SET(&buf[buf_len], fd, EVFILT_WRITE, EV_ADD,
-			       0, 0, (void *) ERTS_POLL_KQ_OP_ADD_W);
-			buf_len++;
-		    }
-		    else if (used_events & ERTS_POLL_EV_OUT) {
-			EV_SET(&buf[buf_len], fd, EVFILT_WRITE, EV_DELETE,
-			       0, 0, (void *) ERTS_POLL_KQ_OP_DEL_W);
-			buf_len++;
-		    }
-		}
-	    }
-	}
-	if (used_events) {
-	    if (!events) {
-		erts_atomic_dec_nob(&ps->no_of_user_fds);
-	    }
-	}
-	else {
-	    if (events)
-		erts_atomic_inc_nob(&ps->no_of_user_fds);
-	}
-	ASSERT((events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0);
-	ASSERT((used_events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0);
-    }	    
-
+    erts_free(ERTS_ALC_T_TMP, ps->poll_fds);
+    ps->poll_fds = NULL;
 #endif
-
-    ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST;
-    ps->fds_status[fd].used_events = ps->fds_status[fd].events;
-
-    bbp->len = buf_len;
 }
 
-#else /* !ERTS_POLL_USE_BATCH_UPDATE_POLLSET */
-
-#if ERTS_POLL_USE_EPOLL
-static int
-#if ERTS_POLL_USE_CONCURRENT_UPDATE
-conc_update_pollset(ErtsPollSet ps, int fd, int *update_fallback)
-#else
-update_pollset(ErtsPollSet ps, int fd)
-#endif
+static ERTS_INLINE int
+need_update(ErtsPollSet *ps, int fd, int *resetp)
 {
-    int res;
-    int op;
-    struct epoll_event epe_templ;
-    struct epoll_event epe;
-
+    int reset;
     ASSERT(fd < ps->fds_status_len);
 
-    if (!need_update(ps, fd))
-	return 0;
-
-#ifdef ERTS_POLL_DEBUG_PRINT
-    erts_printf("Doing update on fd=%d\n", fd);
-#endif
-    if (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK) {
-#if ERTS_POLL_USE_CONCURRENT_UPDATE
-	if (!*update_fallback) {
-	    *update_fallback = 1;
-	    return 0;
-	}
-#endif
-	if (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_USEFLBCK) {
-	    return update_fallback_pollset(ps, fd);
-	}
-	else { /* Remove from fallback and try epoll */
-	    ErtsPollEvents events = ps->fds_status[fd].events;
-	    ps->fds_status[fd].events = (ErtsPollEvents) 0;
-	    res = update_fallback_pollset(ps, fd);
-	    ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK));
-	    if (!events)
-		return res;
-	    ps->fds_status[fd].events = events;
-	}
-    }
-
-    epe_templ.events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events);
-    epe_templ.data.fd = fd;
-
-#ifdef VALGRIND
-    /* Silence invalid valgrind warning ... */
-    memset((void *) &epe.data, 0, sizeof(epoll_data_t));
-#endif
-
-    if (epe_templ.events && ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST) {
-	do {
-	    /* We init 'epe' every time since epoll_ctl() may modify it
-	       (not declared const and not documented as const). */
-	    epe.events = epe_templ.events;
-	    epe.data.fd = epe_templ.data.fd;
-	    res = epoll_ctl(ps->kp_fd, EPOLL_CTL_DEL, fd, &epe);
-	} while (res != 0 && errno == EINTR);
-	erts_atomic_dec_nob(&ps->no_of_user_fds);
-	ps->fds_status[fd].used_events = 0;
-    }
+    reset = (int) (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST);
+    ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST;
 
-    if (!epe_templ.events) {
-	/* A note on EPOLL_CTL_DEL: linux kernel versions before 2.6.9
-	   need a non-NULL event pointer even though it is ignored... */
-	op = EPOLL_CTL_DEL;
-	erts_atomic_dec_nob(&ps->no_of_user_fds);
-    }
-    else if (!ps->fds_status[fd].used_events) {
-	op = EPOLL_CTL_ADD;
-	erts_atomic_inc_nob(&ps->no_of_user_fds);
-    }
-    else {
-	op = EPOLL_CTL_MOD;
-    }
+    *resetp = reset;
 
-    do {
-	/* We init 'epe' every time since epoll_ctl() may modify it
-	   (not declared const and not documented as const). */
-	epe.events = epe_templ.events;
-	epe.data.fd = epe_templ.data.fd;
-	res = epoll_ctl(ps->kp_fd, op, fd, &epe);
-    } while (res != 0 && errno == EINTR);
+    if (reset || ps->fds_status[fd].used_events != ps->fds_status[fd].events)
+	return 1;
 
-#if defined(ERTS_POLL_DEBUG_PRINT) && 1
-    {
-	int saved_errno = errno;
-	erts_printf("%s = epoll_ctl(%d, %s, %d, {Ox%x, %d})\n",
-		     res == 0 ? "0" : erl_errno_id(errno),
-		     ps->kp_fd,
-		     (op == EPOLL_CTL_ADD
-		      ? "EPOLL_CTL_ADD"
-		      : (op == EPOLL_CTL_MOD
-			 ? "EPOLL_CTL_MOD"
-			 : (op == EPOLL_CTL_DEL
-			    ? "EPOLL_CTL_DEL"
-			    : "UNKNOWN"))),
-		     fd,
-		     epe_templ.events,
-		     fd);
-	errno = saved_errno;
-    }
-#endif
-    if (res == 0)
-	ps->fds_status[fd].used_events = ps->fds_status[fd].events;
-    else {
-	switch (op) {
-	case EPOLL_CTL_MOD:
-	    epe.events = 0;
-	    do {
-		/* We init 'epe' every time since epoll_ctl() may modify it
-		   (not declared const and not documented as const). */
-		epe.events = 0;
-		epe.data.fd = fd;
-		res = epoll_ctl(ps->kp_fd, EPOLL_CTL_DEL, fd, &epe);
-	    } while (res != 0 && errno == EINTR);
-	    ps->fds_status[fd].used_events = 0;
-	/* Fall through ... */
-	case EPOLL_CTL_ADD: {
-	    ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK;
-	    erts_atomic_dec_nob(&ps->no_of_user_fds);
-#if ERTS_POLL_USE_CONCURRENT_UPDATE
-	    if (!*update_fallback) {
-		*update_fallback = 1;
-		return 0;
-	    }
-#endif
-	    ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK));
-	    res = update_fallback_pollset(ps, fd);
-	    ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK);
-	    break;
-	}
-	case EPOLL_CTL_DEL: {
-	    /*
-	     * Since we use a lazy update approach EPOLL_CTL_DEL will
-	     * frequently fail. This since epoll automatically removes
-	     * a filedescriptor that is closed from the poll set.
-	     */
-	    ps->fds_status[fd].used_events = 0;
-	    res = 0;
-	    break;
-	}
-	default:
-	    fatal_error("%s:%d:update_pollset(): Internal error\n",
-			__FILE__, __LINE__);
-	    break;
-	}
-    }
-    ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST;
-    return res;
+    return 0;
 }
 
-#if ERTS_POLL_USE_CONCURRENT_UPDATE
-static int
-update_pollset(ErtsPollSet ps, int fd)
+static int update_pollset(ErtsPollSet *ps, ErtsPollResFd pr[], int fd)
 {
-    int update_fallback = 1;
-    return conc_update_pollset(ps, fd, &update_fallback);
-}
-#endif
-
-#endif /* ERTS_POLL_USE_EPOLL */
-
-#endif /* ERTS_POLL_USE_BATCH_UPDATE_POLLSET */
-
-#if ERTS_POLL_USE_POLL || ERTS_POLL_USE_SELECT || ERTS_POLL_USE_FALLBACK
-
-#if ERTS_POLL_USE_FALLBACK
-static int update_fallback_pollset(ErtsPollSet ps, int fd)
-#else
-static int update_pollset(ErtsPollSet ps, int fd)
-#endif
-{
-#ifdef ERTS_POLL_DEBUG_PRINT
-#if ERTS_POLL_USE_FALLBACK
-    erts_printf("Doing fallback update on fd=%d\n", fd);
-#else
-    erts_printf("Doing update on fd=%d\n", fd);
-#endif
-#endif
-
+    int res = 0, reset = 0;
+    ErtsPollEvents events = ps->fds_status[fd].events;
     ASSERT(fd < ps->fds_status_len);
-#if ERTS_POLL_USE_FALLBACK
-    ASSERT(ps->fds_status[fd].used_events
-	   ? (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)
-	   : (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_USEFLBCK));
-#endif
 
-    if (!need_update(ps, fd))
-	return 0;
-
-#if ERTS_POLL_USE_FALLBACK
-    ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST;
-#endif
+    if (!need_update(ps, fd, &reset))
+	return res;
 
 #if ERTS_POLL_USE_POLL	/* --- poll -------------------------------- */
-    if (!ps->fds_status[fd].events) {
+    if (!events) {
 	int pix = ps->fds_status[fd].pix;
 	int last_pix;
+
+        if (reset) {
+            /* When a fd has been reset, we tell the caller of erts_poll_wait
+               this by setting the fd as ERTS_POLL_EV_NONE */
+            ERTS_POLL_RES_SET_FD(&pr[res], fd);
+            ERTS_POLL_RES_SET_EVTS(&pr[res], ERTS_POLL_EV_NONE);
+            DEBUG_PRINT_FD("trig %s (poll)", ps, fd, ev2str(ERTS_POLL_EV_NONE));
+            res++;
+        }
+
 	if (pix < 0) {
-#if ERTS_POLL_USE_FALLBACK
-	    ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK));
-#endif
-	    return -1;
+	    return res;
 	}
-#if ERTS_POLL_USE_FALLBACK
-	ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK);
-#endif
 	erts_atomic_dec_nob(&ps->no_of_user_fds);
 	last_pix = --ps->no_poll_fds;
 	if (pix != last_pix) {
@@ -1348,126 +1031,151 @@ static int update_pollset(ErtsPollSet ps, int fd)
 	/* Clear this fd status */
 	ps->fds_status[fd].pix = -1;
 	ps->fds_status[fd].used_events = (ErtsPollEvents) 0;
-#if ERTS_POLL_USE_FALLBACK
-	ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_INFLBCK;
-#endif
+
     }
     else {
 	int pix = ps->fds_status[fd].pix;
 	if (pix < 0) {
-#if ERTS_POLL_USE_FALLBACK
-	    ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)
-		    || fd == ps->kp_fd);
-#endif
 	    erts_atomic_inc_nob(&ps->no_of_user_fds);
 	    ps->fds_status[fd].pix = pix = ps->no_poll_fds++;
 	    if (pix >= ps->poll_fds_len)
 		grow_poll_fds(ps, pix);
 	    ps->poll_fds[pix].fd = fd;
 	    ps->fds_status[fd].pix = pix;
-#if ERTS_POLL_USE_FALLBACK
-	    ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INFLBCK;
-#endif
 	}
 
-#if ERTS_POLL_USE_FALLBACK
-	ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK);
-#endif
-
 	/* Events to be used in next poll */
-	ps->poll_fds[pix].events = ev2pollev(ps->fds_status[fd].events);
+	ps->poll_fds[pix].events = ev2pollev(events);
 	if (ps->poll_fds[pix].revents) {
 	    /* Remove result events that we should not poll for anymore */
 	    ps->poll_fds[pix].revents
 		&= ev2pollev(~(~ps->fds_status[fd].used_events
-			       & ps->fds_status[fd].events));
+			       & events));
 	}
 	/* Save events to be used in next poll */
-	ps->fds_status[fd].used_events = ps->fds_status[fd].events;
+	ps->fds_status[fd].used_events = events;
     }
-    return 0;
+    return res;
 #elif ERTS_POLL_USE_SELECT	/* --- select ------------------------------ */
-    {
-	ErtsPollEvents events = ps->fds_status[fd].events;
+    if (!events) {
+
+        if (reset) {
+            /* When a fd has been reset, we tell the caller of erts_poll_wait
+               this by setting the fd as ERTS_POLL_EV_NONE */
+            ERTS_POLL_RES_SET_FD(&pr[res], fd);
+            ERTS_POLL_RES_SET_EVTS(&pr[res], ERTS_POLL_EV_NONE);
+            DEBUG_PRINT_FD("trig %s (select)", ps, fd, ev2str(ERTS_POLL_EV_NONE));
+            res++;
+        }
+
+        ERTS_FD_CLR(fd, &ps->input_fds);
+        ERTS_FD_CLR(fd, &ps->output_fds);
+
+        if (ps->fds_status[fd].used_events) {
+            erts_atomic_dec_nob(&ps->no_of_user_fds);
+            ps->fds_status[fd].used_events = (ErtsPollEvents) 0;
+        }
+
+        if (fd == ps->max_fd) {
+            int max = ps->max_fd;
+            for (max = ps->max_fd; max >= 0; max--)
+                if (ps->fds_status[max].used_events)
+                    break;
+            ps->max_fd = max;
+        }
+
+    } else {
+
 	ensure_select_fds(fd, &ps->input_fds, &ps->output_fds);
-	if ((ERTS_POLL_EV_IN & events)
-	    != (ERTS_POLL_EV_IN & ps->fds_status[fd].used_events)) {
-	    if (ERTS_POLL_EV_IN & events) {
-		ERTS_FD_SET(fd, &ps->input_fds);
-	    }
-	    else {
-		ERTS_FD_CLR(fd, &ps->input_fds);
-	    }
-	}
-	if ((ERTS_POLL_EV_OUT & events)
-	    != (ERTS_POLL_EV_OUT & ps->fds_status[fd].used_events)) {
-	    if (ERTS_POLL_EV_OUT & events) {
-		ERTS_FD_SET(fd, &ps->output_fds);
-	    }
-	    else {
-		ERTS_FD_CLR(fd, &ps->output_fds);
-	    }
-	}
 
-	if (!ps->fds_status[fd].used_events) {
-	    ASSERT(events);
-	    erts_atomic_inc_nob(&ps->no_of_user_fds);
-#if ERTS_POLL_USE_FALLBACK
-	    ps->no_select_fds++;
-	    ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INFLBCK;
-#endif
-	}
-	else if (!events) {
-	    ASSERT(ps->fds_status[fd].used_events);
-	    erts_atomic_dec_nob(&ps->no_of_user_fds);
-	    ps->fds_status[fd].events = events;
-#if ERTS_POLL_USE_FALLBACK
-	    ps->no_select_fds--;
-	    ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_INFLBCK;
-#endif
-	}
+        if (!ps->fds_status[fd].used_events)
+            erts_atomic_inc_nob(&ps->no_of_user_fds);
+
+        if (events & ERTS_POLL_EV_IN)
+            ERTS_FD_SET(fd, &ps->input_fds);
+        else
+            ERTS_FD_CLR(fd, &ps->input_fds);
+
+        if (events & ERTS_POLL_EV_OUT)
+            ERTS_FD_SET(fd, &ps->output_fds);
+        else
+            ERTS_FD_CLR(fd, &ps->output_fds);
 
 	ps->fds_status[fd].used_events = events;
 
-	if (events && fd > ps->max_fd)
-	    ps->max_fd = fd;
-	else if (!events && fd == ps->max_fd) {
-	    int max = ps->max_fd;
-	    for (max = ps->max_fd; max >= 0; max--)
-		if (ps->fds_status[max].used_events)
-		    break;
-	    ps->max_fd = max;
-	}
+        if (fd > ps->max_fd)
+            ps->max_fd = fd;
     }
-    return 0;
-#endif
-}
 
-#endif /* ERTS_POLL_USE_POLL || ERTS_POLL_USE_SELECT || ERTS_POLL_USE_FALLBACK */
+    return res;
+#elif ERTS_POLL_USE_DEVPOLL
+
+    if (!events) {
 
+        if (reset) {
+            /* When a fd has been reset, we tell the caller of erts_poll_wait
+               this by setting the fd as ERTS_POLL_EV_NONE */
+            ERTS_POLL_RES_SET_FD(&pr[res], fd);
+            ERTS_POLL_RES_SET_EVTS(&pr[res], ERTS_POLL_EV_NONE);
+            DEBUG_PRINT_FD("trig %s (devpoll)", ps, fd, ev2str(ERTS_POLL_EV_NONE));
+            res++;
+        }
 
-static void
-handle_update_requests(ErtsPollSet ps)
-{
-    ErtsPollSetUpdateRequestsBlock *urqbp = &ps->update_requests;
-#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET
-    ErtsPollBatchBuf bb;
-    setup_batch_buf(ps, &bb);
+        ps->poll_fds[ps->poll_fds_ix].fd = fd;
+        ps->poll_fds[ps->poll_fds_ix].revents = 0;
+        ps->poll_fds[ps->poll_fds_ix++].events = POLLREMOVE;
+
+        if (ps->fds_status[fd].used_events) {
+            erts_atomic_dec_nob(&ps->no_of_user_fds);
+            ps->fds_status[fd].used_events = 0;
+        }
+
+    } else {
+        if (!ps->fds_status[fd].used_events) {
+            erts_atomic_inc_nob(&ps->no_of_user_fds);
+        }
+        ps->poll_fds[ps->poll_fds_ix].fd = fd;
+        ps->poll_fds[ps->poll_fds_ix].revents = 0;
+        ps->poll_fds[ps->poll_fds_ix++].events = ERTS_POLL_EV_E2N(events);
+        ps->fds_status[fd].used_events = ps->fds_status[fd].events;
+    }
+
+    return res;
 #endif
+}
+
+static int
+handle_update_requests(ErtsPollSet *ps, ErtsPollResFd pr[], int no_fds)
+{
+    int res = 0;
+    ErtsPollSetUpdateRequestsBlock *urqbp = ps->curr_upd_req_block;
 
     while (urqbp) {
 	ErtsPollSetUpdateRequestsBlock *free_urqbp = urqbp;
 	int i;
 	int len = urqbp->len;
+
+        init_batch_update(ps, len);
+
 	for (i = 0; i < len; i++) {
 	    int fd = urqbp->fds[i];
 	    ASSERT(fd < ps->fds_status_len);
-	    ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_INURQ;
-#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET
-	    batch_update_pollset(ps, fd, &bb);
-#else
-	    update_pollset(ps, fd);
-#endif
+            ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INURQ);
+
+            /* We have run out of PollResFd slots to put results in,
+               so we yield here and return later for more. */
+            if (res == no_fds && pr != NULL) {
+                memmove(urqbp->fds, urqbp->fds+i, sizeof(int) * (len - i));
+                urqbp->len -= i;
+                ps->curr_upd_req_block = urqbp;
+                write_batch_update(ps);
+                return res;
+            }
+
+            if (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INURQ) {
+                ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_INURQ;
+                res += update_pollset(ps, pr + res, fd);
+            }
 	}
 
 	free_urqbp = urqbp;
@@ -1475,12 +1183,9 @@ handle_update_requests(ErtsPollSet ps)
 
 	free_update_requests_block(ps, free_urqbp);
 
-    }
+        write_batch_update(ps);
 
-#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET
-    if (bb.len)
-	write_batch_buf(ps, &bb);
-#endif
+    }
 
     ps->curr_upd_req_block = &ps->update_requests;
 
@@ -1489,16 +1194,19 @@ handle_update_requests(ErtsPollSet ps)
 #endif
 
     ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(ps);
+    return res;
 }
 
+#endif /* !ERTS_POLL_USE_CONCURRENT_UPDATE */
 
 static ERTS_INLINE ErtsPollEvents
-poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake)
+poll_control(ErtsPollSet *ps, int fd, ErtsPollOp op,
+             ErtsPollEvents events, int *do_wake)
 {
     ErtsPollEvents new_events;
 
     if (fd < ps->internal_fd_limit || fd >= max_fds) {
-	if (fd < 0) {
+	if (fd < 0 || fd >= max_fds) {
 	    new_events = ERTS_POLL_EV_ERR;
 	    goto done;
 	}
@@ -1508,115 +1216,55 @@ poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake
 	    goto done;
 	}
 #endif
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
 	if (fd == ps->wake_fds[0] || fd == ps->wake_fds[1]) {
 	    new_events = ERTS_POLL_EV_NVAL;
 	    goto done;
 	}
-#if ERTS_POLL_USE_TIMERFD
-	if (fd == ps->timer_fd) {
-	    new_events = ERTS_POLL_EV_NVAL;
-	    goto done;
-	}
 #endif
     }
 
+#if ERTS_POLL_USE_CONCURRENT_UPDATE
+
+    new_events = update_pollset(ps, fd, op, events);
+
+#else /* !ERTS_POLL_USE_CONCURRENT_UPDATE */
     if (fd >= ps->fds_status_len)
 	grow_fds_status(ps, fd);
 
     ASSERT(fd < ps->fds_status_len);
 
-    new_events = ps->fds_status[fd].events;
-
-    if (events == 0) {
-	*do_wake = 0;
-	goto done;
-    }
-
-    if (on)
-	new_events |= events;
-    else
-	new_events &= ~events;
-
-    if (new_events == (ErtsPollEvents) 0) {
-	ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_RST;
-#if ERTS_POLL_USE_FALLBACK
-	ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_USEFLBCK;
-#endif
-    }
-
-    ps->fds_status[fd].events = new_events;
-
-    if (new_events == ps->fds_status[fd].used_events
-	&& !(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST)
-	) {
-	*do_wake = 0;
-	goto done;
-    }
-
-
-#if ERTS_POLL_USE_CONCURRENT_UPDATE
-    if (ERTS_POLLSET_IS_POLLED(ps)) {
-	int update_fallback = 0;
-	conc_update_pollset(ps, fd, &update_fallback);
-	if (!update_fallback) {
-	    *do_wake = 0; /* no need to wake kernel poller */
-	    goto done;
-	}
+    if (op == ERTS_POLL_OP_DEL) {
+        ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_RST;
+        ps->fds_status[fd].events = 0;
+        *do_wake = 1;
+    } else if (op == ERTS_POLL_OP_ADD) {
+        ASSERT(ps->fds_status[fd].events == 0);
+        ps->fds_status[fd].events = events;
+        *do_wake = 1;
+    } else {
+        ASSERT(op == ERTS_POLL_OP_MOD);
+        ps->fds_status[fd].events = events;
+        *do_wake = 1;
     }
-#endif
+    new_events = ps->fds_status[fd].events;
 
     enqueue_update_request(ps, fd);
-	
-    /*
-     * If new events have been added, we need to wake up the
-     * polling thread, but if events have been removed we don't.
-     */
-    if ((new_events && (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST))
-	|| (~ps->fds_status[fd].used_events & new_events))
-	*do_wake = 1;
 
+#endif /* !ERTS_POLL_USE_CONCURRENT_UPDATE */
 
  done:
-#ifdef ERTS_POLL_DEBUG_PRINT
-     erts_printf("0x%x = poll_control(ps, %d, 0x%x, %s) do_wake=%d\n",
-		 (int) new_events, fd, (int) events, (on ? "on" : "off"), *do_wake);
-#endif
+    DEBUG_PRINT_FD("%s = %s(%p, %d, %s, %s) do_wake=%d",
+                   ps, fd, ev2str(new_events), __FUNCTION__, ps,
+                   fd, op2str(op), ev2str(events), *do_wake);
     return new_events;
 }
 
-void
-ERTS_POLL_EXPORT(erts_poll_controlv)(ErtsPollSet ps,
-				     ErtsPollControlEntry pcev[],
-				     int len)
-{
-    int i;
-    int do_wake;
-    int final_do_wake = 0;
-
-    ERTS_POLLSET_LOCK(ps);
-
-    for (i = 0; i < len; i++) {
-	do_wake = 0;
-	pcev[i].events = poll_control(ps,
-				      pcev[i].fd,
-				      pcev[i].events,
-				      pcev[i].on,
-				      &do_wake);
-	final_do_wake |= do_wake;
-    }
-
-    ERTS_POLLSET_UNLOCK(ps);
-
-    if (final_do_wake)
-	wake_poller(ps, 0, 0);
-
-}
-
 ErtsPollEvents
-ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet ps,
+ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet *ps,
 				    ErtsSysFdType fd,
+                                    ErtsPollOp op,
 				    ErtsPollEvents events,
-				    int on,
 				    int* do_wake) /* In: Wake up polling thread */
 				                  /* Out: Poller is woken */
 {
@@ -1624,13 +1272,15 @@ ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet ps,
 
     ERTS_POLLSET_LOCK(ps);
 
-    res = poll_control(ps, fd, events, on, do_wake);
+    res = poll_control(ps, fd, op, events, do_wake);
 
     ERTS_POLLSET_UNLOCK(ps);
 
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     if (*do_wake) {
-	wake_poller(ps, 0, 0);
+	wake_poller(ps, 0);
     }
+#endif
 
     return res;
 }
@@ -1642,180 +1292,60 @@ ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet ps,
 #if ERTS_POLL_USE_KERNEL_POLL
 
 static ERTS_INLINE int
-save_kp_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, int chk_fds_res)
+ERTS_POLL_EXPORT(save_result)(ErtsPollSet *ps, ErtsPollResFd pr[], int max_res, int chk_fds_res, int ebadf)
 {
-    int res = 0;
-    int i;
-    int n = chk_fds_res < max_res ? chk_fds_res : max_res;
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE || ERTS_POLL_DEBUG_PRINT
+    int n = chk_fds_res < max_res ? chk_fds_res : max_res, i;
+    int res = n;
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     int wake_fd = ps->wake_fds[0];
-#if ERTS_POLL_USE_TIMERFD
-    int timer_fd = ps->timer_fd;
 #endif
 
     for (i = 0; i < n; i++) {
-
-#if ERTS_POLL_USE_EPOLL		/* --- epoll ------------------------------- */
-
-	if (ps->res_events[i].events) {
-	    int fd = ps->res_events[i].data.fd;
-	    int ix;
-	    ErtsPollEvents revents;
-	    if (fd == wake_fd) {
-		cleanup_wakeup_pipe(ps);
-		continue;
-	    }
-#if ERTS_POLL_USE_TIMERFD
-            if (fd == timer_fd) {
-                continue;
-            }
+        int fd = ERTS_POLL_RES_GET_FD(&pr[i]);
+#ifdef DEBUG_PRINT_MODE
+        ErtsPollEvents evts = ERTS_POLL_RES_GET_EVTS(pr+i);
 #endif
-	    ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK));
-	    /* epoll_wait() can repeat the same fd in result array... */
-	    ix = (int) ps->fds_status[fd].res_ev_ix;
-	    ASSERT(ix >= 0);
-	    if (ix >= res || pr[ix].fd != fd) {
-		ix = res;
-		pr[ix].fd = fd;
-		pr[ix].events = (ErtsPollEvents) 0;
-	    }
-
-	    revents = ERTS_POLL_EV_N2E(ps->res_events[i].events);
-	    pr[ix].events |= revents;
-	    if (revents) {
-		if (res == ix) {
-		    ps->fds_status[fd].res_ev_ix = (unsigned short) ix;
-		    res++;
-		}
-	    }
-	}
-
-#elif ERTS_POLL_USE_KQUEUE	/* --- kqueue ------------------------------ */
-
-	struct kevent *ev;
-	int fd;
-	int ix;
-
-	ev = &ps->res_events[i];
-	fd = (int) ev->ident;
-	ASSERT(fd < ps->fds_status_len);
-	ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK));
-	ix = (int) ps->fds_status[fd].res_ev_ix;
 
-	ASSERT(ix >= 0);
-	if (ix >= res || pr[ix].fd != fd) {
-	    ix = res;
-	    pr[ix].fd = (int) ev->ident;
-	    pr[ix].events = (ErtsPollEvents) 0;
-	}
-
-	if (ev->filter == EVFILT_READ) {
-	    if (fd == wake_fd) {
-		cleanup_wakeup_pipe(ps);
-		continue;
-	    }
-	    pr[ix].events |= ERTS_POLL_EV_IN;
-	}
-	else if (ev->filter == EVFILT_WRITE)
-	    pr[ix].events |= ERTS_POLL_EV_OUT;
-	if (ev->flags & (EV_ERROR|EV_EOF)) {
-	    if ((ev->flags & EV_ERROR) && (((int) ev->data) == EBADF))
-		pr[ix].events |= ERTS_POLL_EV_NVAL;
-	    else
-		pr[ix].events |= ERTS_POLL_EV_ERR;
-	}
-	if (pr[ix].events) {
-	    if (res == ix) {
-		ps->fds_status[fd].res_ev_ix = (unsigned short) ix;
-		res++;
-	    }
-	}
-
-#elif ERTS_POLL_USE_DEVPOLL	/* --- devpoll ----------------------------- */
-
-	if (ps->res_events[i].revents) {
-	    int fd = ps->res_events[i].fd;
-	    ErtsPollEvents revents;
-	    if (fd == wake_fd) {
-		cleanup_wakeup_pipe(ps);
-		continue;
-	    }
-#if ERTS_POLL_USE_TIMERFD
-	    if (fd == timer_fd) {
-		continue;
-	    }
-#endif
-	    revents = ERTS_POLL_EV_N2E(ps->res_events[i].events);
-	    pr[res].fd = fd;
-	    pr[res].events = revents;
-	    res++;
-	}
-
-#endif
-
-    }
-
-    return res;
-}
-
-#endif /* ERTS_POLL_USE_KERNEL_POLL */
-
-#if ERTS_POLL_USE_FALLBACK
-
-static int
-get_kp_results(ErtsPollSet ps, ErtsPollResFd pr[], int max_res)
-{
-    int res;
+        DEBUG_PRINT_FD("trig %s (%s)", ps, fd,
+                       ev2str(evts),
 #if ERTS_POLL_USE_KQUEUE
-    struct timespec ts = {0, 0};
+                       "kqueue"
+#elif ERTS_POLL_USE_EPOLL
+                       "epoll"
+#else
+                       "/dev/poll"
 #endif
+            );
 
-    if (max_res > ps->res_events_len)
-	grow_res_events(ps, max_res);
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
 
-    do {
-#if ERTS_POLL_USE_EPOLL
-	res = epoll_wait(ps->kp_fd, ps->res_events, max_res, 0);
-#elif ERTS_POLL_USE_KQUEUE
-	res = kevent(ps->kp_fd, NULL, 0, ps->res_events, max_res, &ts);
+        if (fd == wake_fd) {
+            cleanup_wakeup_pipe(ps);
+            ERTS_POLL_RES_SET_EVTS(&pr[i], ERTS_POLL_EV_NONE);
+        } else {
+            /* Reset the events to emulate ONESHOT semantics */
+            ps->fds_status[fd].events = 0;
+            enqueue_update_request(ps, fd);
+        }
 #endif
-    } while (res < 0 && errno == EINTR);
-
-    if (res < 0) {
-	fatal_error("%s:%d: %s() failed: %s (%d)\n",
-		    __FILE__, __LINE__,
-#if ERTS_POLL_USE_EPOLL
-		    "epoll_wait",
-#elif ERTS_POLL_USE_KQUEUE
-		    "kevent",
-#endif
-		    erl_errno_id(errno), errno);
     }
 
-    return save_kp_result(ps, pr, max_res, res);
+    return res;
+#else
+    ASSERT(chk_fds_res <= max_res);
+    return chk_fds_res;
+#endif
 }
 
-#endif /* ERTS_POLL_USE_FALLBACK */
-
-
+#else /* !ERTS_POLL_USE_KERNEL_POLL */
 
 static ERTS_INLINE int
-save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res,
-		 int chk_fds_res, int ebadf)
+ERTS_POLL_EXPORT(save_result)(ErtsPollSet *ps, ErtsPollResFd pr[], int max_res, int chk_fds_res, int ebadf)
 {
-#if ERTS_POLL_USE_DEVPOLL
-    return save_kp_result(ps, pr, max_res, chk_fds_res);
-#elif ERTS_POLL_USE_FALLBACK
-    if (!ps->fallback_used)
-	return save_kp_result(ps, pr, max_res, chk_fds_res);
-    else
-#endif /* ERTS_POLL_USE_FALLBACK */
-    {
-
 #if ERTS_POLL_USE_POLL	/* --- poll -------------------------------- */
 	int res = 0;
-#if !ERTS_POLL_USE_FALLBACK
 	int wake_fd = ps->wake_fds[0];
-#endif
 	int i, first_ix, end_ix;
 
 	/*
@@ -1832,23 +1362,30 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res,
 		if (ps->poll_fds[i].revents != (short) 0) {
 		    int fd = ps->poll_fds[i].fd;
 		    ErtsPollEvents revents;
-#if ERTS_POLL_USE_FALLBACK
-		    if (fd == ps->kp_fd) {
-			res += get_kp_results(ps, &pr[res], max_res-res);
-			i++;
-			continue;
-		    }
-#else
 		    if (fd == wake_fd) {
 			cleanup_wakeup_pipe(ps);
 			i++;
 			continue;
 		    }
-#endif
 		    revents = pollev2ev(ps->poll_fds[i].revents);
-		    pr[res].fd = fd;
-		    pr[res].events = revents;
+		    ERTS_POLL_RES_SET_FD(&pr[res], fd);
+		    ERTS_POLL_RES_SET_EVTS(&pr[res], revents);
+
+                    /* If an fd returns as error, we may want to check the
+                       update_requests queue to see if it has been reset
+                       before delivering the result?!?! This should allow
+                       the user to do driver_dselect + close without waiting
+                       for stop_select... */
+
+                    DEBUG_PRINT_FD("trig %s (poll)", ps, ERTS_POLL_RES_GET_FD(&pr[res]),
+                                   ev2str(ERTS_POLL_RES_GET_EVTS(&pr[res])));
+
 		    res++;
+
+                    /* Clear the events for this fd in order to mimic
+                       how epoll ONESHOT works */
+                    ps->fds_status[fd].events = 0;
+                    enqueue_update_request(ps, fd);
 		}
 		i++;
 	    }
@@ -1864,9 +1401,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res,
 
 #elif ERTS_POLL_USE_SELECT	/* --- select ------------------------------ */
 	int res = 0;
-#if !ERTS_POLL_USE_FALLBACK
 	int wake_fd = ps->wake_fds[0];
-#endif
 	int fd, first_fd, end_fd;
 
 	/*
@@ -1879,29 +1414,23 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res,
 	if (!ebadf) {
 	    while (1) {
 		while (fd < end_fd && res < max_res) {
-
-		    pr[res].events = (ErtsPollEvents) 0;
+                    ErtsPollEvents events = 0;
 		    if (ERTS_FD_ISSET(fd, &ps->res_input_fds)) {
-#if ERTS_POLL_USE_FALLBACK
-			if (fd == ps->kp_fd) {
-			    res += get_kp_results(ps, &pr[res], max_res-res);
-			    fd++;
-			    continue;
-			}
-#else
 			if (fd == wake_fd) {
 			    cleanup_wakeup_pipe(ps);
 			    fd++;
 			    continue;
 			}
-#endif
-			pr[res].events |= ERTS_POLL_EV_IN;
+			events |= ERTS_POLL_EV_IN;
 		    }
 		    if (ERTS_FD_ISSET(fd, &ps->res_output_fds))
-			pr[res].events |= ERTS_POLL_EV_OUT;
-		    if (pr[res].events) {
-			pr[res].fd = fd;
+			events |= ERTS_POLL_EV_OUT;
+		    if (events) {
+                        ERTS_POLL_RES_SET_FD(&pr[res], fd);
+                        ERTS_POLL_RES_SET_EVTS(&pr[res], events);
 			res++;
+                        ps->fds_status[fd].events = 0;
+                        enqueue_update_request(ps, fd);
 		    }
 		    fd++;
 		}
@@ -1934,7 +1463,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res,
 			if (ps->fds_status[fd].events & ERTS_POLL_EV_OUT) {
 			    oset = &ps->res_output_fds;
 			    ERTS_FD_ZERO(oset);
-			    ERTS_FD_SET(fd, oset);			
+			    ERTS_FD_SET(fd, oset);
 			}
 			do {
 			    /* Initiate 'tv' each time;
@@ -1943,49 +1472,31 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res,
 			    sres = ERTS_SELECT(ps->max_fd+1, iset, oset, NULL, &tv);
 			} while (sres < 0 && errno == EINTR);
 			if (sres < 0) {
-#if ERTS_POLL_USE_FALLBACK
-			    if (fd == ps->kp_fd) {
-				res += get_kp_results(ps,
-						      &pr[res],
-						      max_res-res);
-				fd++;
-				continue;
-			    }
-#else
 			    if (fd == wake_fd) {
 				cleanup_wakeup_pipe(ps);
 				fd++;
 				continue;
 			    }
-#endif
-			    pr[res].fd = fd;
-			    pr[res].events = ERTS_POLL_EV_NVAL;
+                            ERTS_POLL_RES_SET_FD(&pr[res], fd);
+                            ERTS_POLL_RES_SET_EVTS(&pr[res], ERTS_POLL_EV_NVAL);
 			    res++;
 			}
 			else if (sres > 0) {
-			    pr[res].fd = fd;
+                            ErtsPollEvents events = 0;
+                            ERTS_POLL_RES_SET_FD(&pr[res], fd);
 			    if (iset && ERTS_FD_ISSET(fd, iset)) {
-#if ERTS_POLL_USE_FALLBACK
-				if (fd == ps->kp_fd) {
-				    res += get_kp_results(ps,
-							  &pr[res],
-							  max_res-res);
-				    fd++;
-				    continue;
-				}
-#else
 				if (fd == wake_fd) {
 				    cleanup_wakeup_pipe(ps);
 				    fd++;
 				    continue;
 				}
-#endif
-				pr[res].events |= ERTS_POLL_EV_IN;
+				events |= ERTS_POLL_EV_IN;
 			    }
 			    if (oset && ERTS_FD_ISSET(fd, oset)) {
-				pr[res].events |= ERTS_POLL_EV_OUT;
+				events |= ERTS_POLL_EV_OUT;
 			    }
-			    ASSERT(pr[res].events);
+			    ASSERT(events);
+                            ERTS_POLL_RES_SET_EVTS(&pr[res], events);
 			    res++;
 			}
 		    }
@@ -2000,353 +1511,145 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res,
 	}
 	ps->next_sel_fd = fd;
 	return res;
-#endif
-    }
-}
-
-static ERTS_INLINE ErtsMonotonicTime
-get_timeout(ErtsPollSet ps,
-	    int resolution,
-	    ErtsMonotonicTime timeout_time)
-{
-    ErtsMonotonicTime timeout, save_timeout_time;
-
-    if (timeout_time == ERTS_POLL_NO_TIMEOUT) {
-	save_timeout_time = ERTS_MONOTONIC_TIME_MIN;
-	timeout = 0;
-    }
-    else {
-	ErtsMonotonicTime diff_time, current_time;
-	current_time = erts_get_monotonic_time(NULL);
-	diff_time = timeout_time - current_time;
-	if (diff_time <= 0) {
-	    save_timeout_time = ERTS_MONOTONIC_TIME_MIN;
-	    timeout = 0;
-	}
-	else {
-	    save_timeout_time = current_time;
-	    switch (resolution) {
-	    case 1000:
-		/* Round up to nearest even milli second */
-		timeout = ERTS_MONOTONIC_TO_MSEC(diff_time - 1) + 1;
-		if (timeout > (ErtsMonotonicTime) INT_MAX)
-		    timeout = (ErtsMonotonicTime) INT_MAX;
-		save_timeout_time += ERTS_MSEC_TO_MONOTONIC(timeout);
-		timeout -= ERTS_PREMATURE_TIMEOUT(timeout, 1000);
-		break;
-	    case 1000000:
-		/* Round up to nearest even micro second */
-		timeout = ERTS_MONOTONIC_TO_USEC(diff_time - 1) + 1;
-		save_timeout_time += ERTS_USEC_TO_MONOTONIC(timeout);
-		timeout -= ERTS_PREMATURE_TIMEOUT(timeout, 1000*1000);
-		break;
-	    case 1000000000:
-		/* Round up to nearest even nano second */
-		timeout = ERTS_MONOTONIC_TO_NSEC(diff_time - 1) + 1;
-		save_timeout_time += ERTS_NSEC_TO_MONOTONIC(timeout);
-		timeout -= ERTS_PREMATURE_TIMEOUT(timeout, 1000*1000*1000);
-		break;
-	    default:
-		ERTS_INTERNAL_ERROR("Invalid resolution");
-		timeout = 0;
-		save_timeout_time = 0;
-		break;
-	    }
-	}
-    }
-    set_timeout_time(ps, save_timeout_time);
-    return timeout;
-}
-
-#if ERTS_POLL_USE_SELECT
-
-static ERTS_INLINE int
-get_timeout_timeval(ErtsPollSet ps,
-		    SysTimeval *tvp,
-		    ErtsMonotonicTime timeout_time)
-{
-    ErtsMonotonicTime timeout = get_timeout(ps,
-					    1000*1000,
-					    timeout_time);
-
-    if (!timeout) {
-	tvp->tv_sec = 0;
-	tvp->tv_usec = 0;
-
-	return 0;
-    }
-    else {
-	ErtsMonotonicTime sec = timeout/(1000*1000);
-	tvp->tv_sec = sec;
-	tvp->tv_usec = timeout - sec*(1000*1000);
-
-	ASSERT(tvp->tv_sec >= 0);
-	ASSERT(tvp->tv_usec >= 0);
-	ASSERT(tvp->tv_usec < 1000*1000);
-
-	return !0;
-    }
-
-}
-
-#endif
-
-#if ERTS_POLL_USE_KQUEUE || (ERTS_POLL_USE_POLL && defined(HAVE_PPOLL)) || ERTS_POLL_USE_TIMERFD
-
-static ERTS_INLINE int
-get_timeout_timespec(ErtsPollSet ps,
-		     struct timespec *tsp,
-		     ErtsMonotonicTime timeout_time)
-{
-    ErtsMonotonicTime timeout = get_timeout(ps,
-					    1000*1000*1000,
-					    timeout_time);
-
-    if (!timeout) {
-	tsp->tv_sec = 0;
-	tsp->tv_nsec = 0;
-	return 0;
-    }
-    else {
-	ErtsMonotonicTime sec = timeout/(1000*1000*1000);
-	tsp->tv_sec = sec;
-	tsp->tv_nsec = timeout - sec*(1000*1000*1000);
-
-	ASSERT(tsp->tv_sec >= 0);
-	ASSERT(tsp->tv_nsec >= 0);
-	ASSERT(tsp->tv_nsec < 1000*1000*1000);
-
-	return !0;
-    }
-}
-
-#endif
-
-#if ERTS_POLL_USE_TIMERFD
-
-static ERTS_INLINE int
-get_timeout_itimerspec(ErtsPollSet ps,
-                       struct itimerspec *itsp,
-                       ErtsMonotonicTime timeout_time)
-{
-
-    itsp->it_interval.tv_sec = 0;
-    itsp->it_interval.tv_nsec = 0;
-
-    return get_timeout_timespec(ps, &itsp->it_value, timeout_time);
+#endif /* ERTS_POLL_USE_SELECT */
 }
 
-#endif
+#endif /* !ERTS_POLL_USE_KERNEL_POLL */
 
 static ERTS_INLINE int
-check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res)
+check_fd_events(ErtsPollSet *ps, ErtsPollResFd pr[], int do_wait, int max_res)
 {
     int res;
-    ERTS_MSACC_PUSH_STATE_M();
-    if (erts_atomic_read_nob(&ps->no_of_user_fds) == 0
-	&& timeout_time == ERTS_POLL_NO_TIMEOUT) {
-	/* Nothing to poll and zero timeout; done... */
-	return 0;
-    }
-    else {
-	int timeout;
-#if ERTS_POLL_USE_FALLBACK
-	if (!(ps->fallback_used = ERTS_POLL_NEED_FALLBACK(ps))) {
-
-#if ERTS_POLL_USE_EPOLL		/* --- epoll ------------------------------- */
-	    if (max_res > ps->res_events_len)
-		grow_res_events(ps, max_res);
-#if ERTS_POLL_USE_TIMERFD
-            {
-                struct itimerspec its;
-                timeout = get_timeout_itimerspec(ps, &its, timeout_time);
-                if (timeout) {
-                    erts_thr_progress_prepare_wait(NULL);
-                    ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
-                    timerfd_set(ps, &its);
-                    res = epoll_wait(ps->kp_fd, ps->res_events, max_res, -1);
-                    res = timerfd_clear(ps, res, max_res);
-                } else {
-                    res = epoll_wait(ps->kp_fd, ps->res_events, max_res, 0);
-                }
-            }
-#else /* !ERTS_POLL_USE_TIMERFD */
-	    timeout = (int) get_timeout(ps, 1000, timeout_time);
-            if (timeout) {
-		erts_thr_progress_prepare_wait(NULL);
-                ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
-            }
-	    res = epoll_wait(ps->kp_fd, ps->res_events, max_res, timeout);
-#endif /* !ERTS_POLL_USE_TIMERFD */
-#elif ERTS_POLL_USE_KQUEUE	/* --- kqueue ------------------------------ */
-	    struct timespec ts;
-	    if (max_res > ps->res_events_len)
-		grow_res_events(ps, max_res);
-	    timeout = get_timeout_timespec(ps, &ts, timeout_time);
-            if (timeout) {
-		erts_thr_progress_prepare_wait(NULL);
-		ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
-            }
-	    res = kevent(ps->kp_fd, NULL, 0, ps->res_events, max_res, &ts);
-#endif				/* ----------------------------------------- */
-	}
-	else /* use fallback (i.e. poll() or select()) */
-#endif /* ERTS_POLL_USE_FALLBACK */
-	{
+    int timeout = do_wait ? -1 : 0;
+    DEBUG_PRINT_WAIT("Entering check_fd_events(), do_wait=%d", ps, do_wait);
+    {
+#if ERTS_POLL_USE_EPOLL                /* --- epoll ------------------------------- */
+        res = epoll_wait(ps->kp_fd, pr, max_res, timeout);
+
+#elif ERTS_POLL_USE_KQUEUE     /* --- kqueue ------------------------------ */
+        struct timespec ts = {0, 0};
+        struct timespec *tsp = timeout ? NULL : &ts;
+        res = kevent(ps->kp_fd, NULL, 0, pr, max_res, tsp);
+#elif ERTS_POLL_USE_DEVPOLL	/* --- devpoll ----------------------------- */
+        /*
+         * The ioctl() will fail with EINVAL on Solaris 10 if dp_nfds
+         * is set too high. dp_nfds should not be set greater than
+         * the maximum number of file descriptors in the poll set.
+         */
+        struct dvpoll poll_res;
+        int nfds = (int) erts_atomic_read_nob(&ps->no_of_user_fds) + 1 /* wakeup pipe */;
+        poll_res.dp_nfds = nfds < max_res ? nfds : max_res;
+        poll_res.dp_fds = pr;
+        poll_res.dp_timeout = timeout;
+        res = ioctl(ps->kp_fd, DP_POLL, &poll_res);
 
-#if ERTS_POLL_USE_DEVPOLL	/* --- devpoll ----------------------------- */
-	    /*
-	     * The ioctl() will fail with EINVAL on Solaris 10 if dp_nfds
-	     * is set too high. dp_nfds should not be set greater than
-	     * the maximum number of file descriptors in the poll set.
-	     */
-	    struct dvpoll poll_res;
-	    int nfds = (int) erts_atomic_read_nob(&ps->no_of_user_fds);
-	    nfds++; /* Wakeup pipe */
-	    timeout = (int) get_timeout(ps, 1000, timeout_time);
-	    poll_res.dp_nfds = nfds < max_res ? nfds : max_res;
-	    if (poll_res.dp_nfds > ps->res_events_len)
-		grow_res_events(ps, poll_res.dp_nfds);
-	    poll_res.dp_fds = ps->res_events;
-	    if (timeout) {
-		erts_thr_progress_prepare_wait(NULL);
-                ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
-            }
-	    poll_res.dp_timeout = timeout;
-	    res = ioctl(ps->kp_fd, DP_POLL, &poll_res);
-#elif ERTS_POLL_USE_POLL && defined(HAVE_PPOLL)	/* --- ppoll ---------------- */
-            struct timespec ts;
-	    timeout = get_timeout_timespec(ps, &ts, timeout_time);
-            if (timeout) {
-		erts_thr_progress_prepare_wait(NULL);
-		ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
-            }
-            res = ppoll(ps->poll_fds, ps->no_poll_fds, &ts, NULL);
 #elif ERTS_POLL_USE_POLL        /* --- poll --------------------------------- */
-	    timeout = (int) get_timeout(ps, 1000, timeout_time);
 
-	    if (timeout) {
-		erts_thr_progress_prepare_wait(NULL);
-		ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
-            }
-	    res = poll(ps->poll_fds, ps->no_poll_fds, timeout);
+        res = poll(ps->poll_fds, ps->no_poll_fds, timeout);
+
 #elif ERTS_POLL_USE_SELECT	/* --- select ------------------------------ */
-	    SysTimeval to;
-	    timeout = get_timeout_timeval(ps, &to, timeout_time);
+        SysTimeval tv = {0, 0};
+        SysTimeval *tvp = timeout ? NULL : &tv;
 
-	    ERTS_FD_COPY(&ps->input_fds, &ps->res_input_fds);
-	    ERTS_FD_COPY(&ps->output_fds, &ps->res_output_fds);
+        ERTS_FD_COPY(&ps->input_fds, &ps->res_input_fds);
+        ERTS_FD_COPY(&ps->output_fds, &ps->res_output_fds);
 
-	    if (timeout) {
-		erts_thr_progress_prepare_wait(NULL);
-		ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
-	    }
-	    res = ERTS_SELECT(ps->max_fd + 1,
-			      &ps->res_input_fds,
-			      &ps->res_output_fds,
-			      NULL,
-			      &to);
-	    if (timeout) {
-		erts_thr_progress_finalize_wait(NULL);
-		ERTS_MSACC_POP_STATE_M();
-	    }
-	    if (res < 0
-		&& errno == EBADF
-		&& ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) {
-		/*
-		 * This may have happened because another thread deselected
-		 * a fd in our poll set and then closed it, i.e. the driver
-		 * behaved correctly. We wan't to avoid looking for a bad
-		 * fd, that may even not exist anymore. Therefore, handle
-		 * update requests and try again.
-		 *
-		 * We don't know how much of the timeout is left; therfore,
-		 * we use a zero timeout. If no error occur and no events
-		 * have triggered, we fake an EAGAIN error and let the caller
-		 * restart us.
-		 */
-		to.tv_sec = 0;
-		to.tv_usec = 0;
-		ERTS_POLLSET_LOCK(ps);
-		handle_update_requests(ps);
-		ERTS_POLLSET_UNLOCK(ps);
-		res = ERTS_SELECT(ps->max_fd + 1,
-				  &ps->res_input_fds,
-				  &ps->res_output_fds,
-				  NULL,
-				  &to);
-		if (res == 0) {
-		    errno = EAGAIN;
-		    res = -1;
-		}
-	    }
-	    return res;
+        res = ERTS_SELECT(ps->max_fd + 1,
+                          &ps->res_input_fds,
+                          &ps->res_output_fds,
+                          NULL,
+                          tvp);
 #endif				/* ----------------------------------------- */
-	}
-	if (timeout) {
-	    erts_thr_progress_finalize_wait(NULL);
-	    ERTS_MSACC_POP_STATE_M();
-	}
-	return res;
     }
+    DEBUG_PRINT_WAIT("Leaving check_fd_events(), res=%d", ps, res);
+    return res;
 }
 
 int
-ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
+ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet *ps,
 				 ErtsPollResFd pr[],
-				 int *len,
-				 ErtsMonotonicTime timeout_time)
+				 int *len)
 {
-    ErtsMonotonicTime to;
-    int res, no_fds;
+    int res, no_fds, used_fds = 0;
     int ebadf = 0;
+    int do_wait;
     int ps_locked = 0;
+    ERTS_MSACC_DECLARE_CACHE();
 
     no_fds = *len;
-#ifdef ERTS_POLL_MAX_RES
-    if (no_fds >= ERTS_POLL_MAX_RES)
-	no_fds = ERTS_POLL_MAX_RES;
-#endif
-
     *len = 0;
+    ASSERT(no_fds > 0);
 
-#ifdef ERTS_POLL_DEBUG_PRINT
-    erts_printf("Entering erts_poll_wait(), timeout_time=%bps\n",
-		timeout_time);
-#endif
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
+    if (ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) {
+	ERTS_POLLSET_LOCK(ps);
+	used_fds = handle_update_requests(ps, pr, no_fds);
+	ERTS_POLLSET_UNLOCK(ps);
 
-    if (ERTS_POLLSET_SET_POLLED_CHK(ps)) {
-	res = EINVAL; /* Another thread is in erts_poll_wait()
-			 on this pollset... */
-	goto done;
+        if (used_fds == no_fds) {
+            *len = used_fds;
+            return 0;
+        }
     }
+#endif
 
-    to = (is_woken(ps)
-	  ? ERTS_POLL_NO_TIMEOUT /* Use zero timeout */
-	  : timeout_time);
+    do_wait = !is_woken(ps) && used_fds == 0;
 
-    if (ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) {
-	ERTS_POLLSET_LOCK(ps);
-	handle_update_requests(ps);
-	ERTS_POLLSET_UNLOCK(ps);
+    DEBUG_PRINT_WAIT("Entering %s(), do_wait=%d", ps, __FUNCTION__, do_wait);
+
+    if (do_wait) {
+        erts_thr_progress_prepare_wait(NULL);
+        ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP);
     }
 
     while (1) {
-	res = check_fd_events(ps, to, no_fds);
+	res = check_fd_events(ps, pr + used_fds, do_wait, no_fds - used_fds);
+
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
+        if (res < 0
+            && errno == EBADF
+            && ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) {
+            /*
+             * This may have happened because another thread deselected
+             * a fd in our poll set and then closed it, i.e. the driver
+             * behaved correctly. We wan't to avoid looking for a bad
+             * fd, that may even not exist anymore. Therefore, handle
+             * update requests and try again. This behaviour should only
+             * happen when using SELECT as the polling mechanism.
+             */
+            ERTS_POLLSET_LOCK(ps);
+            used_fds += handle_update_requests(ps, pr + used_fds, no_fds - used_fds);
+            if (used_fds == no_fds) {
+                *len = used_fds;
+                ERTS_POLLSET_UNLOCK(ps);
+                return 0;
+            }
+            res = check_fd_events(ps, pr + used_fds, 0, no_fds - used_fds);
+            /* Keep the lock over the non-blocking poll in order to not
+               get any nasty races happening. */
+            ERTS_POLLSET_UNLOCK(ps);
+            if (res == 0) {
+                errno = EAGAIN;
+                res = -1;
+            }
+        }
+#endif
+
 	if (res != 0)
 	    break;
-	if (to == ERTS_POLL_NO_TIMEOUT)
-	    break;
-	if (erts_get_monotonic_time(NULL) >= timeout_time)
-	    break;
+        if (!do_wait)
+            break;
+    }
+
+    if (do_wait) {
+        erts_thr_progress_finalize_wait(NULL);
+        ERTS_MSACC_UPDATE_CACHE();
+        ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_CHECK_IO);
     }
 
     woke_up(ps);
 
-    if (res == 0) {
-	res = ETIMEDOUT;
-    }
-    else if (res < 0) {
+    if (res < 0) {
 #if ERTS_POLL_USE_SELECT
 	if (errno == EBADF) {
 	    ebadf = 1;
@@ -2363,26 +1666,21 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
 	ps_locked = 1;
 	ERTS_POLLSET_LOCK(ps);
 
-	no_fds = save_poll_result(ps, pr, no_fds, res, ebadf);
+	used_fds += ERTS_POLL_EXPORT(save_result)(ps, pr + used_fds, no_fds - used_fds, res, ebadf);
 
 #ifdef HARD_DEBUG
-	check_poll_result(pr, no_fds);
+	check_poll_result(pr, used_fds);
 #endif
 
-	res = (no_fds == 0 ? (is_interrupted_reset(ps) ? EINTR : EAGAIN) : 0);
-	*len = no_fds;
+	res = (used_fds == 0 ? (is_interrupted_reset(ps) ? EINTR : EAGAIN) : 0);
+	*len = used_fds;
     }
 
     if (ps_locked)
 	ERTS_POLLSET_UNLOCK(ps);
-    ERTS_POLLSET_UNSET_POLLED(ps);
 
- done:
-    set_timeout_time(ps, ERTS_MONOTONIC_TIME_MAX);
-#ifdef ERTS_POLL_DEBUG_PRINT
-    erts_printf("Leaving %s = erts_poll_wait()\n",
-		 res == 0 ? "0" : erl_errno_id(res));
-#endif
+    DEBUG_PRINT_WAIT("Leaving %s = %s(len = %d)", ps,
+                     res == 0 ? "0" : erl_errno_id(res), __FUNCTION__, *len);
 
     return res;
 }
@@ -2392,40 +1690,14 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
  */
 
 void
-ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet ps, int set)
+ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet *ps, int set)
 {
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     if (!set)
 	reset_wakeup_state(ps);
     else
-	wake_poller(ps, 1, 0);
-}
-
-
-/*
- * erts_poll_interrupt_timed():
- *   If 'set' != 0, interrupt thread blocked in erts_poll_wait() if it
- * is not guaranteed that it will timeout before 'msec' milli seconds.
- */
-void
-ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps,
-					    int set,
-					    ErtsMonotonicTime timeout_time)
-{
-    if (!set)
-	reset_wakeup_state(ps);
-    else {
-	ErtsMonotonicTime max_wait_time = get_timeout_time(ps);
-	if (max_wait_time > timeout_time)
-	    wake_poller(ps, 1, 0);
-#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
-	else {
-	    if (ERTS_POLLSET_IS_POLLED(ps))
-		erts_atomic_inc_nob(&ps->no_avoided_wakeups);
-	    erts_atomic_inc_nob(&ps->no_avoided_interrupts);
-	}
-	erts_atomic_inc_nob(&ps->no_interrupt_timed);
+	wake_poller(ps, 1);
 #endif
-    }
 }
 
 int
@@ -2438,14 +1710,19 @@ ERTS_POLL_EXPORT(erts_poll_max_fds)(void)
  */
 
 void
-ERTS_POLL_EXPORT(erts_poll_init)(void)
+ERTS_POLL_EXPORT(erts_poll_init)(int *concurrent_updates)
 {
-    erts_mtx_init(&pollsets_lock, "pollsets_lock", NIL,
-        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
-    pollsets = NULL;
 
     errno = 0;
 
+    if (concurrent_updates) {
+#if ERTS_POLL_USE_CONCURRENT_UPDATE
+        *concurrent_updates = 1;
+#else
+        *concurrent_updates = 0;
+#endif
+    }
+
 #if !defined(NO_SYSCONF)
     max_fds = sysconf(_SC_OPEN_MAX);
 #elif ERTS_POLL_USE_SELECT
@@ -2464,37 +1741,28 @@ ERTS_POLL_EXPORT(erts_poll_init)(void)
 	fatal_error("erts_poll_init(): Failed to get max number of files: %s\n",
 		    erl_errno_id(errno));
 
-#ifdef ERTS_POLL_DEBUG_PRINT
     print_misc_debug_info();
-#endif
 }
 
-ErtsPollSet
-ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
+ErtsPollSet *
+ERTS_POLL_EXPORT(erts_poll_create_pollset)(int id)
 {
 #if ERTS_POLL_USE_KERNEL_POLL
     int kp_fd;
 #endif
-    ErtsPollSet ps = erts_alloc(ERTS_ALC_T_POLLSET,
+    ErtsPollSet *ps = erts_alloc(ERTS_ALC_T_POLLSET,
 				sizeof(struct ERTS_POLL_EXPORT(erts_pollset)));
+    ps->id = id;
     ps->internal_fd_limit = 0;
-    ps->fds_status = NULL;
-    ps->fds_status_len = 0;
     erts_atomic_init_nob(&ps->no_of_user_fds, 0);
 #if ERTS_POLL_USE_KERNEL_POLL
     ps->kp_fd = -1;
 #if ERTS_POLL_USE_EPOLL
     kp_fd = epoll_create(256);
-    ps->res_events_len = 0;
-    ps->res_events = NULL;
 #elif ERTS_POLL_USE_DEVPOLL
     kp_fd = open("/dev/poll", O_RDWR);
-    ps->res_events_len = 0;
-    ps->res_events = NULL;
 #elif ERTS_POLL_USE_KQUEUE
     kp_fd = kqueue();
-    ps->res_events_len = 0;
-    ps->res_events = NULL;
 #endif
     if (kp_fd < 0)
 	fatal_error("erts_poll_create_pollset(): Failed to "
@@ -2508,10 +1776,6 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
 		    ": %s (%d)\n",
 		    erl_errno_id(errno), errno);
 #endif /* ERTS_POLL_USE_KERNEL_POLL */
-#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET
-    /* res_events is also used as write buffer */
-    grow_res_events(ps, ERTS_POLL_MIN_BATCH_BUF_SIZE);
-#endif
 #if ERTS_POLL_USE_POLL
     ps->next_poll_fds_ix = 0;
     ps->no_poll_fds = 0;
@@ -2520,9 +1784,6 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
 #elif ERTS_POLL_USE_SELECT
     ps->next_sel_fd = 0;
     ps->max_fd = -1;
-#if ERTS_POLL_USE_FALLBACK
-    ps->no_select_fds = 0;
-#endif
 #ifdef _DARWIN_UNLIMITED_SELECT
     ps->input_fds.sz = 0;
     ps->input_fds.ptr = NULL;
@@ -2539,140 +1800,48 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
     ERTS_FD_ZERO(&ps->res_output_fds);
 #endif
 #endif
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
+    ps->fds_status = NULL;
+    ps->fds_status_len = 0;
     ps->update_requests.next = NULL;
     ps->update_requests.len = 0;
     ps->curr_upd_req_block = &ps->update_requests;
     erts_atomic32_init_nob(&ps->have_update_requests, 0);
-    erts_atomic32_init_nob(&ps->polled, 0);
     erts_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO);
-    erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0);
-    create_wakeup_pipe(ps);
-#if ERTS_POLL_USE_TIMERFD
-    create_timerfd(ps);
-#endif
-#if ERTS_POLL_USE_FALLBACK
-    if (kp_fd >= ps->fds_status_len)
-	grow_fds_status(ps, kp_fd);
-    /* Force kernel poll fd into fallback (poll/select) set */
-    ps->fds_status[kp_fd].flags
-	|= ERTS_POLL_FD_FLG_INFLBCK|ERTS_POLL_FD_FLG_USEFLBCK;
-    {
-	int do_wake = 0;
-	ERTS_POLL_EXPORT(erts_poll_control)(ps, kp_fd, ERTS_POLL_EV_IN, 1,
-					    &do_wake);
-    }    
 #endif
 #if ERTS_POLL_USE_KERNEL_POLL
     if (ps->internal_fd_limit <= kp_fd)
 	ps->internal_fd_limit = kp_fd + 1;
     ps->kp_fd = kp_fd;
 #endif
-    init_timeout_time(ps);
-#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
-    erts_atomic_init_nob(&ps->no_avoided_wakeups, 0);
-    erts_atomic_init_nob(&ps->no_avoided_interrupts, 0);
-    erts_atomic_init_nob(&ps->no_interrupt_timed, 0);
-#endif
-    handle_update_requests(ps);
-#if ERTS_POLL_USE_FALLBACK
-    ps->fallback_used = 0;
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
+    erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0);
+    create_wakeup_pipe(ps);
+    handle_update_requests(ps, NULL, 0);
+    cleanup_wakeup_pipe(ps);
 #endif
     erts_atomic_set_nob(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */
 
-    erts_mtx_lock(&pollsets_lock);
-    ps->next = pollsets;
-    pollsets = ps;
-    erts_mtx_unlock(&pollsets_lock);
-
     return ps;
 }
 
-void
-ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps)
-{
-
-    if (ps->fds_status)
-	erts_free(ERTS_ALC_T_FD_STATUS, (void *) ps->fds_status);
-
-#if ERTS_POLL_USE_EPOLL
-    if (ps->kp_fd >= 0)
-	close(ps->kp_fd);
-    if (ps->res_events)
-	erts_free(ERTS_ALC_T_POLL_RES_EVS, (void *) ps->res_events);
-#elif ERTS_POLL_USE_DEVPOLL
-    if (ps->kp_fd >= 0)
-	close(ps->kp_fd);
-    if (ps->res_events)
-	erts_free(ERTS_ALC_T_POLL_RES_EVS, (void *) ps->res_events);
-#elif ERTS_POLL_USE_POLL
-    if (ps->poll_fds)
-	erts_free(ERTS_ALC_T_POLL_FDS, (void *) ps->poll_fds);
-#elif ERTS_POLL_USE_SELECT
-#ifdef _DARWIN_UNLIMITED_SELECT
-    if (ps->input_fds.ptr)
-	erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->input_fds.ptr);
-    if (ps->res_input_fds.ptr)
-	erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->res_input_fds.ptr);
-    if (ps->output_fds.ptr)
-	erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->output_fds.ptr);
-    if (ps->res_output_fds.ptr)
-	erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->res_output_fds.ptr);
-#endif
-#endif
-    {
-	ErtsPollSetUpdateRequestsBlock *urqbp = ps->update_requests.next;
-	while (urqbp) {
-	    ErtsPollSetUpdateRequestsBlock *free_urqbp = urqbp;
-	    urqbp = urqbp->next;
-	    free_update_requests_block(ps, free_urqbp);
-	}
-    }
-    erts_mtx_destroy(&ps->mtx);
-    if (ps->wake_fds[0] >= 0)
-	close(ps->wake_fds[0]);
-    if (ps->wake_fds[1] >= 0)
-	close(ps->wake_fds[1]);
-#if ERTS_POLL_USE_TIMERFD
-    if (ps->timer_fd >= 0)
-        close(ps->timer_fd);
-#endif
-
-    erts_mtx_lock(&pollsets_lock);
-    if (ps == pollsets)
-	pollsets = pollsets->next;
-    else {
-	ErtsPollSet prev_ps;
-	for (prev_ps = pollsets; ps != prev_ps->next; prev_ps = prev_ps->next)
-            ;
-	ASSERT(ps == prev_ps->next);
-	prev_ps->next = ps->next;
-    }
-    erts_mtx_unlock(&pollsets_lock);
-
-    erts_free(ERTS_ALC_T_POLLSET, (void *) ps);
-}
-
 /*
  * --- Info ------------------------------------------------------------------
  */
 
 void
-ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
+ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet *ps, ErtsPollInfo *pip)
 {
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     int pending_updates;
+#endif
     Uint size = 0;
 
     ERTS_POLLSET_LOCK(ps);
 
     size += sizeof(struct ERTS_POLL_EXPORT(erts_pollset));
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     size += ps->fds_status_len*sizeof(ErtsFdStatus);
-
-#if ERTS_POLL_USE_EPOLL
-    size += ps->res_events_len*sizeof(struct epoll_event);
-#elif ERTS_POLL_USE_DEVPOLL
-    size += ps->res_events_len*sizeof(struct pollfd);
-#elif ERTS_POLL_USE_KQUEUE
-    size += ps->res_events_len*sizeof(struct kevent);
 #endif
 
 #if ERTS_POLL_USE_POLL
@@ -2684,6 +1853,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
 #endif
 #endif
 
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     {
 	ErtsPollSetUpdateRequestsBlock *urqbp = ps->update_requests.next;
 	pending_updates = ps->update_requests.len;
@@ -2693,8 +1863,9 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
 	    urqbp = urqbp->next;
 	}
     }
+#endif
 
-    pip->primary = 
+    pip->primary =
 #if ERTS_POLL_USE_KQUEUE
 	"kqueue"
 #elif ERTS_POLL_USE_EPOLL
@@ -2708,17 +1879,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
 #endif
 	;
 
-    pip->fallback = 
-#if !ERTS_POLL_USE_FALLBACK
-	NULL
-#elif ERTS_POLL_USE_POLL
-	"poll"
-#elif ERTS_POLL_USE_SELECT
-	"select"
-#endif
-	;
-
-    pip->kernel_poll = 
+    pip->kernel_poll =
 #if !ERTS_POLL_USE_KERNEL_POLL
 	NULL
 #elif ERTS_POLL_USE_KQUEUE
@@ -2733,40 +1894,21 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
     pip->memory_size = size;
 
     pip->poll_set_size = (int) erts_atomic_read_nob(&ps->no_of_user_fds);
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     pip->poll_set_size++; /* Wakeup pipe */
-#if ERTS_POLL_USE_TIMERFD
-    pip->poll_set_size++; /* timerfd */
-#endif
-
-    pip->fallback_poll_set_size =
-#if !ERTS_POLL_USE_FALLBACK
-	0
-#elif ERTS_POLL_USE_POLL
-	ps->no_poll_fds
-#elif ERTS_POLL_USE_SELECT
-	ps->no_select_fds
-#endif
-	;
-
-#if ERTS_POLL_USE_FALLBACK
-    /* If only kp_fd is in fallback poll set we don't use fallback... */
-    if (pip->fallback_poll_set_size == 1)
-	pip->fallback_poll_set_size = 0;
-    else
-	pip->poll_set_size++; /* kp_fd */
 #endif
 
     pip->lazy_updates =
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
 	1
+#else
+	0
+#endif
 	;
 
     pip->pending_updates = 
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
 	pending_updates
-	;
-
-    pip->batch_updates = 
-#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET
-	1
 #else
 	0
 #endif
@@ -2780,13 +1922,15 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
 #endif
 	;
 
-    pip->max_fds = max_fds;
-
-#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
-    pip->no_avoided_wakeups = erts_atomic_read_nob(&ps->no_avoided_wakeups);
-    pip->no_avoided_interrupts = erts_atomic_read_nob(&ps->no_avoided_interrupts);
-    pip->no_interrupt_timed = erts_atomic_read_nob(&ps->no_interrupt_timed);
+    pip->is_fallback =
+#if ERTS_POLL_IS_FALLBACK
+        1
+#else
+        0
 #endif
+        ;
+
+    pip->max_fds = max_fds;
 
     ERTS_POLLSET_UNLOCK(ps);
 
@@ -2822,35 +1966,60 @@ fatal_error(char *format, ...)
     abort();
 }
 
-static void
-fatal_error_async_signal_safe(char *error_str)
+/*
+ * --- Debug -----------------------------------------------------------------
+ */
+
+#if ERTS_POLL_USE_EPOLL
+uint32_t epoll_events(int kp_fd, int fd)
 {
-    if (ERTS_SOMEONE_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) {
-	/* See comment above in fatal_error() */
-	return;
+    /* For epoll we read the information about what is selected upon from the proc fs.*/
+    char fname[30];
+    FILE *f;
+    unsigned int pos, flags, mnt_id;
+    int line = 0;
+    sprintf(fname,"/proc/%d/fdinfo/%d",getpid(), kp_fd);
+    f = fopen(fname,"r");
+    if (!f) {
+        fprintf(stderr,"failed to open file %s, errno = %d\n", fname, errno);
+        ASSERT(0);
+        return 0;
     }
-    if (error_str) {
-	int len = 0;
-	while (error_str[len])
-	    len++;
-	if (len) {
-	    /* async signal safe */
-	    erts_silence_warn_unused_result(write(2, error_str, len));
-	}
+    if (fscanf(f,"pos:\t%x\nflags:\t%x", &pos, &flags) != 2) {
+        fprintf(stderr,"failed to parse file %s, errno = %d\n", fname, errno);
+        ASSERT(0);
+        return 0;
     }
-    abort();
+    if (fscanf(f,"\nmnt_id:\t%x\n", &mnt_id));
+    line += 3;
+    while (!feof(f)) {
+        /* tfd:       10 events: 40000019 data:       180000000a */
+        int ev_fd;
+        uint32_t events;
+        uint64_t data;
+        if (fscanf(f,"tfd:%d events:%x data:%lx\n", &ev_fd, &events, &data) != 3) {
+            fprintf(stderr,"failed to parse file %s on line %d, errno = %d\n", fname,
+                    line,
+                    errno);
+            return 0;
+        }
+        if (fd == ev_fd) {
+            fclose(f);
+            return events;
+        }
+    }
+    fclose(f);
+    return 0;
 }
- 
-/*
- * --- Debug -----------------------------------------------------------------
- */
+#endif
 
 void
-ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet ps,
+ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet *ps,
 						ErtsPollEvents ev[],
 						int len)
 {
     int fd;
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     ERTS_POLLSET_LOCK(ps);
     for (fd = 0; fd < len; fd++) {
 	if (fd >= ps->fds_status_len)
@@ -2859,9 +2028,6 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet ps,
 	    ev[fd] = ps->fds_status[fd].events;
             if (
                 fd == ps->wake_fds[0] || fd == ps->wake_fds[1] ||
-#if ERTS_POLL_USE_TIMERFD
-                fd == ps->timer_fd ||
-#endif
 #if ERTS_POLL_USE_KERNEL_POLL
                 fd == ps->kp_fd ||
 #endif
@@ -2870,7 +2036,50 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet ps,
 	}
     }
     ERTS_POLLSET_UNLOCK(ps);
+#elif ERTS_POLL_USE_EPOLL
 
+    /* For epoll we read the information about what is selected upon from the proc fs.*/
+    char fname[30];
+    FILE *f;
+    unsigned int pos, flags, mnt_id;
+    int line = 0;
+    sprintf(fname,"/proc/%d/fdinfo/%d",getpid(), ps->kp_fd);
+    for (fd = 0; fd < len; fd++)
+        ev[fd] = ERTS_POLL_EV_NONE;
+    f = fopen(fname,"r");
+    if (!f) {
+        fprintf(stderr,"failed to open file %s, errno = %d\n", fname, errno);
+        return;
+    }
+    if (fscanf(f,"pos:\t%x\nflags:\t%x", &pos, &flags) != 2) {
+        fprintf(stderr,"failed to parse file %s, errno = %d\n", fname, errno);
+        ASSERT(0);
+        return;
+    }
+    if (fscanf(f,"\nmnt_id:\t%x\n", &mnt_id));
+    line += 3;
+    while (!feof(f)) {
+        /* tfd:       10 events: 40000019 data:       180000000a */
+        int fd;
+        uint32_t events;
+        uint64_t data;
+        if (fscanf(f,"tfd:%d events:%x data:%lx\n", &fd, &events, &data) != 3) {
+            fprintf(stderr,"failed to parse file %s on line %d, errno = %d\n",
+                    fname, line, errno);
+            ASSERT(0);
+            return;
+        }
+        data &= 0xFFFFFFFF;
+        ASSERT(fd == data);
+        /* Events are the events that are being monitored, which of course include
+           error and hup events, but we are only interested in IN/OUT events */
+        ev[fd] = (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT) & ERTS_POLL_EV_N2E(events);
+        line++;
+    }
+#else
+    for (fd = 0; fd < len; fd++)
+        ev[fd] = ERTS_POLL_EV_NONE;
+#endif
 }
 
 #ifdef HARD_DEBUG
@@ -2890,10 +2099,10 @@ check_poll_result(ErtsPollResFd pr[], int len)
 }
 
 
-#if ERTS_POLL_USE_DEVPOLL
+#if ERTS_POLL_USE_DEVPOLL && defined(DEBUG)
 
 static void
-check_poll_status(ErtsPollSet ps)
+check_poll_status(ErtsPollSet *ps)
 {
     int i;
     for (i = 0; i < ps->fds_status_len; i++) {
@@ -2925,30 +2134,24 @@ check_poll_status(ErtsPollSet ps)
 #endif /* ERTS_POLL_USE_DEVPOLL */
 #endif /* HARD_DEBUG */
 
-#ifdef ERTS_POLL_DEBUG_PRINT
 static void
 print_misc_debug_info(void)
 {
-    erts_printf("erts_poll using: %s lazy_updates:%s batch_updates:%s\n",
+#if ERTS_POLL_DEBUG_PRINT
+    erts_printf("erts_poll using: %s lazy_updates:%s\n",
 #if ERTS_POLL_USE_KQUEUE
 		"kqueue"
 #elif ERTS_POLL_USE_EPOLL
 		"epoll"
 #elif ERTS_POLL_USE_DEVPOLL
 		"/dev/poll"
-#endif
-#if ERTS_POLL_USE_FALLBACK
-		"-"
-#endif
-#if ERTS_POLL_USE_POLL
+#elif ERTS_POLL_USE_POLL
 		"poll"
 #elif ERTS_POLL_USE_SELECT
 		"select"
 #endif
 		,
-		"true"
-		,
-#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
 		"true"
 #else
 		"false"
@@ -2967,29 +2170,20 @@ print_misc_debug_info(void)
 #ifdef FD_SETSIZE
     erts_printf("FD_SETSIZE=%d\n", FD_SETSIZE);
 #endif
-}
-    
 #endif
+}
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
-static void erts_lcnt_enable_pollset_lock_count(ErtsPollSet pollset, int enable) {
+void ERTS_POLL_EXPORT(erts_lcnt_enable_pollset_lock_count)(ErtsPollSet *pollset, int enable)
+{
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     if(enable) {
         erts_lcnt_install_new_lock_info(&pollset->mtx.lcnt, "pollset_rm", NIL,
             ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO);
     } else {
         erts_lcnt_uninstall(&pollset->mtx.lcnt);
     }
-}
-
-void ERTS_POLL_EXPORT(erts_lcnt_update_pollset_locks)(int enable) {
-    ErtsPollSet iterator;
-
-    erts_mtx_lock(&pollsets_lock);
-
-    for(iterator = pollsets; iterator != NULL; iterator = iterator->next) {
-        erts_lcnt_enable_pollset_lock_count(iterator, enable);
-    }
-
-    erts_mtx_unlock(&pollsets_lock);
+#endif
+    return;
 }
 #endif
diff --git a/erts/emulator/sys/common/erl_poll.h b/erts/emulator/sys/common/erl_poll.h
index 6c961205fe..a1e2709a9f 100644
--- a/erts/emulator/sys/common/erl_poll.h
+++ b/erts/emulator/sys/common/erl_poll.h
@@ -18,11 +18,31 @@
  * %CopyrightEnd%
  */
 
-/*
- * Description:	Poll interface suitable for ERTS with or without
- *              SMP support.
+/**
+ * @description: Poll interface suitable for ERTS with SMP support.
+ *
+ * @author: 	Rickard Green
+ * @author: 	Lukas Larsson
+ *
+ * This header file exports macros and functions that are used to
+ * react to I/O polling events from file descriptors or wait-able
+ * objects. The API exported is the following:
  *
- * Author: 	Rickard Green
+ * defines:
+ *   ERTS_POLL_EV_NONE - No events have been set. This is not the same as 0.
+ *   ERTS_POLL_EV_IN   - Represent an IN event
+ *   ERTS_POLL_EV_OUT  - Represent an OUT event
+ *   ERTS_POLL_EV_ERR  - Represent an error event
+ *   ERTS_POLL_EV_NVAL - Represent an invalid event
+ *
+ * macro functions:
+ *   ErtsSysFdType ERTS_POLL_RES_GET_FD(ErtsPollResFd *evt);
+ *   void ERTS_POLL_RES_SET_FD(ErtsPollResFd *evt, ErtsSysFdType fd);
+ *   ErtsPollEvents ERTS_POLL_RES_GET_EVTS(ErtsPollResFd *evt)
+ *   void ERTS_POLL_RES_SET_EVTS(ErtsPollResFd *evt, ErtsPollEvents fd);
+ *
+ * functions:
+ *   See erl_poll_api.h
  */
 
 #ifndef ERL_POLL_H__
@@ -32,33 +52,27 @@
 
 #define ERTS_POLL_NO_TIMEOUT ERTS_MONOTONIC_TIME_MIN
 
-#if 0
-#define ERTS_POLL_COUNT_AVOIDED_WAKEUPS
-#endif
-
 #ifdef ERTS_ENABLE_KERNEL_POLL
-#  if defined(ERTS_KERNEL_POLL_VERSION)
-#    define ERTS_POLL_EXPORT(FUNC) FUNC ## _kp
+#  undef ERTS_ENABLE_KERNEL_POLL
+#  define ERTS_ENABLE_KERNEL_POLL 1
+#  if defined(ERTS_NO_KERNEL_POLL_VERSION)
+#    define ERTS_POLL_EXPORT(FUNC) FUNC ## _flbk
+#    undef ERTS_NO_KERNEL_POLL_VERSION
+#    define ERTS_NO_KERNEL_POLL_VERSION 1
+#    define ERTS_KERNEL_POLL_VERSION 0
 #  else
-#    define ERTS_POLL_EXPORT(FUNC) FUNC ## _nkp
-#    undef ERTS_POLL_DISABLE_KERNEL_POLL
-#    define ERTS_POLL_DISABLE_KERNEL_POLL
+#    undef ERTS_KERNEL_POLL_VERSION
+#    define ERTS_KERNEL_POLL_VERSION 1
+#    define ERTS_NO_KERNEL_POLL_VERSION 0
+#    define ERTS_POLL_EXPORT(FUNC) FUNC
 #  endif
 #else
 #    define ERTS_POLL_EXPORT(FUNC) FUNC
-#    undef ERTS_POLL_DISABLE_KERNEL_POLL
-#    define ERTS_POLL_DISABLE_KERNEL_POLL
-#endif
-
-#ifdef ERTS_POLL_DISABLE_KERNEL_POLL
-#  undef HAVE_SYS_EPOLL_H
-#  undef HAVE_SYS_EVENT_H
-#  undef HAVE_SYS_DEVPOLL_H
+#    define ERTS_ENABLE_KERNEL_POLL 0
+#    define ERTS_NO_KERNEL_POLL_VERSION 1
+#    define ERTS_KERNEL_POLL_VERSION 0
 #endif
 
-#undef ERTS_POLL_USE_KERNEL_POLL
-#define ERTS_POLL_USE_KERNEL_POLL 0
-
 #undef ERTS_POLL_USE_KQUEUE
 #define ERTS_POLL_USE_KQUEUE 0
 #undef ERTS_POLL_USE_EPOLL
@@ -70,68 +84,96 @@
 #undef ERTS_POLL_USE_SELECT
 #define ERTS_POLL_USE_SELECT 0
 
-#if defined(HAVE_SYS_EVENT_H)
-#  undef ERTS_POLL_USE_KQUEUE
-#  define ERTS_POLL_USE_KQUEUE 1
-#  undef ERTS_POLL_USE_KERNEL_POLL
-#  define ERTS_POLL_USE_KERNEL_POLL 1
-#elif defined(HAVE_SYS_EPOLL_H)
-#  undef ERTS_POLL_USE_EPOLL
-#  define ERTS_POLL_USE_EPOLL 1
-#  undef ERTS_POLL_USE_KERNEL_POLL
-#  define ERTS_POLL_USE_KERNEL_POLL 1
-#elif defined(HAVE_SYS_DEVPOLL_H)
-#  undef ERTS_POLL_USE_DEVPOLL
-#  define ERTS_POLL_USE_DEVPOLL 1
-#  undef ERTS_POLL_USE_KERNEL_POLL
-#  define ERTS_POLL_USE_KERNEL_POLL 1
+/* Defines which structure that erts_poll_wait should use to wait with
+   and how events should be represented */
+#define ERTS_POLL_USE_EPOLL_EVS 0
+#define ERTS_POLL_USE_KQUEUE_EVS 0
+#define ERTS_POLL_USE_DEVPOLL_EVS 0
+#define ERTS_POLL_USE_POLL_EVS 0
+#define ERTS_POLL_USE_SELECT_EVS 0
+
+#define ERTS_POLL_USE_KERNEL_POLL ERTS_KERNEL_POLL_VERSION
+
+#if ERTS_ENABLE_KERNEL_POLL
+#  if defined(HAVE_SYS_EVENT_H)
+#    undef ERTS_POLL_USE_KQUEUE_EVS
+#    define ERTS_POLL_USE_KQUEUE_EVS 1
+#    undef ERTS_POLL_USE_KQUEUE
+#    define ERTS_POLL_USE_KQUEUE ERTS_KERNEL_POLL_VERSION
+#  elif defined(HAVE_SYS_EPOLL_H)
+#    undef ERTS_POLL_USE_EPOLL_EVS
+#    define ERTS_POLL_USE_EPOLL_EVS 1
+#    undef ERTS_POLL_USE_EPOLL
+#    define ERTS_POLL_USE_EPOLL ERTS_KERNEL_POLL_VERSION
+#  elif defined(HAVE_SYS_DEVPOLL_H)
+#    undef ERTS_POLL_USE_DEVPOLL_EVS
+#    define ERTS_POLL_USE_DEVPOLL_EVS 1
+#    undef ERTS_POLL_USE_DEVPOLL
+#    define ERTS_POLL_USE_DEVPOLL ERTS_KERNEL_POLL_VERSION
+#  else
+#    error "Missing kernel poll implementation of erts_poll()"
+#  endif
 #endif
 
-#define ERTS_POLL_USE_FALLBACK (ERTS_POLL_USE_KQUEUE || ERTS_POLL_USE_EPOLL)
-
-#if !ERTS_POLL_USE_KERNEL_POLL || ERTS_POLL_USE_FALLBACK
+#if ERTS_NO_KERNEL_POLL_VERSION
 #  if defined(ERTS_USE_POLL)
+#    undef ERTS_POLL_USE_POLL_EVS
+#    define ERTS_POLL_USE_POLL_EVS 1
 #    undef ERTS_POLL_USE_POLL
 #    define ERTS_POLL_USE_POLL 1
 #  elif !defined(__WIN32__)
+#    undef ERTS_POLL_USE_SELECT_EVS
+#    define ERTS_POLL_USE_SELECT_EVS 1
 #    undef ERTS_POLL_USE_SELECT
 #    define ERTS_POLL_USE_SELECT 1
 #  endif
 #endif
 
-#define ERTS_POLL_USE_TIMERFD 0
+#define ERTS_POLL_USE_FALLBACK (ERTS_POLL_USE_KQUEUE || ERTS_POLL_USE_EPOLL)
 
 typedef Uint32 ErtsPollEvents;
-#undef ERTS_POLL_EV_E2N
+
+typedef enum {
+    ERTS_POLL_OP_ADD = 0, /* Add the FD to the pollset */
+    ERTS_POLL_OP_MOD = 1, /* Modify the FD in the pollset */
+    ERTS_POLL_OP_DEL = 2  /* Delete the FD from the pollset */
+} ErtsPollOp;
+
+#define op2str(op) (op == ERTS_POLL_OP_ADD ? "add" :            \
+                    (op == ERTS_POLL_OP_MOD ? "mod" : "del"))
 
 #if defined(__WIN32__)	/* --- win32  --------------------------------------- */
 
-#define ERTS_POLL_EV_IN   1
-#define ERTS_POLL_EV_OUT  2
-#define ERTS_POLL_EV_ERR  4
-#define ERTS_POLL_EV_NVAL 8
+#define ERTS_POLL_EV_IN    1
+#define ERTS_POLL_EV_OUT   2
+#define ERTS_POLL_EV_ERR   4
+#define ERTS_POLL_EV_NVAL  8
 
-#elif ERTS_POLL_USE_EPOLL	/* --- epoll ------------------------------- */
+#define ERTS_POLL_EV_E2N(EV) 		(EV)
+#define ERTS_POLL_EV_N2E(EV) 		(EV)
 
-#include <sys/epoll.h>
+#elif ERTS_POLL_USE_EPOLL_EVS	/* --- epoll ------------------------------- */
 
-#ifdef HAVE_SYS_TIMERFD_H
-#include <sys/timerfd.h>
-#undef ERTS_POLL_USE_TIMERFD
-#define ERTS_POLL_USE_TIMERFD 1
-#endif
+#include <sys/epoll.h>
 
 #define ERTS_POLL_EV_E2N(EV) \
   ((uint32_t) (EV))
 #define ERTS_POLL_EV_N2E(EV) \
-  ((ErtsPollEvents) (EV))
+  ((ErtsPollEvents) (EV) & ~EPOLLONESHOT)
 
 #define ERTS_POLL_EV_IN			ERTS_POLL_EV_N2E(EPOLLIN)
 #define ERTS_POLL_EV_OUT		ERTS_POLL_EV_N2E(EPOLLOUT)
 #define ERTS_POLL_EV_NVAL		ERTS_POLL_EV_N2E(EPOLLET)
 #define ERTS_POLL_EV_ERR		ERTS_POLL_EV_N2E(EPOLLERR|EPOLLHUP)
 
-#elif ERTS_POLL_USE_DEVPOLL	/* --- devpoll ----------------------------- */
+typedef struct epoll_event ErtsPollResFd;
+
+#define ERTS_POLL_RES_GET_FD(evt) ((ErtsSysFdType)((evt)->data.fd))
+#define ERTS_POLL_RES_SET_FD(evt, ident) (evt)->data.fd = ident
+#define ERTS_POLL_RES_GET_EVTS(evt) ERTS_POLL_EV_N2E((evt)->events)
+#define ERTS_POLL_RES_SET_EVTS(evt, evts) (evt)->events = ERTS_POLL_EV_E2N(evts)
+
+#elif ERTS_POLL_USE_DEVPOLL_EVS	/* --- devpoll ----------------------------- */
 
 #include <sys/devpoll.h>
 
@@ -145,12 +187,37 @@ typedef Uint32 ErtsPollEvents;
 #define ERTS_POLL_EV_NVAL		ERTS_POLL_EV_N2E(POLLNVAL)
 #define ERTS_POLL_EV_ERR		ERTS_POLL_EV_N2E(POLLERR|POLLHUP)
 
-#elif ERTS_POLL_USE_KQUEUE	/* --- kqueue ------------------------------ */
+typedef struct pollfd ErtsPollResFd;
+
+#define ERTS_POLL_RES_GET_FD(evt) ((ErtsSysFdType)((evt)->fd))
+#define ERTS_POLL_RES_SET_FD(evt, ident) (evt)->fd = ident
+#define ERTS_POLL_RES_GET_EVTS(evt) ERTS_POLL_EV_N2E((evt)->revents)
+#define ERTS_POLL_RES_SET_EVTS(evt, evts) (evt)->revents = ERTS_POLL_EV_E2N(evts)
+
+#elif ERTS_POLL_USE_KQUEUE_EVS	/* --- kqueue ------------------------------ */
 /* Kqueue use fallback defines (poll() or select()) */
+
+#include <sys/event.h>
+
+#ifdef ERTS_USE_POLL
+#  undef ERTS_POLL_USE_POLL_EVS
+#  define ERTS_POLL_USE_POLL_EVS   1
+#elif !defined(__WIN32__)
+#  undef ERTS_POLL_USE_SELECT_EVS
+#  define ERTS_POLL_USE_SELECT_EVS 1
 #endif
 
-#if ERTS_POLL_USE_POLL	/* --- poll -------------------------------- */
+typedef struct kevent ErtsPollResFd;
+
+#define ERTS_POLL_RES_GET_FD(evt) ((ErtsSysFdType)((evt)->ident))
+#define ERTS_POLL_RES_SET_FD(evt, fd) (evt)->ident = fd
+#define ERTS_POLL_RES_GET_EVTS(evt) ERTS_POLL_EV_N2E((ErtsPollEvents)(evt)->udata)
+#define ERTS_POLL_RES_SET_EVTS(evt, evts) (evt)->udata = (void*)(UWord)(ERTS_POLL_EV_E2N(evts))
+
+#endif
 
+#if ERTS_POLL_USE_POLL_EVS
+				/* --- poll -------------------------------- */
 #include <poll.h>
 
 #define ERTS_POLL_EV_NKP_E2N(EV) \
@@ -169,7 +236,7 @@ typedef Uint32 ErtsPollEvents;
 #define ERTS_POLL_EV_NKP_NVAL		ERTS_POLL_EV_N2E(POLLNVAL)
 #define ERTS_POLL_EV_NKP_ERR		ERTS_POLL_EV_N2E(POLLERR|POLLHUP)
 
-#elif ERTS_POLL_USE_SELECT	/* --- select ------------------------------ */
+#elif ERTS_POLL_USE_SELECT_EVS	/* --- select ------------------------------ */
 
 #define ERTS_POLL_EV_NKP_E2N(EV) (EV)
 #define ERTS_POLL_EV_NKP_N2E(EV) (EV)
@@ -195,71 +262,65 @@ typedef Uint32 ErtsPollEvents;
 
 #endif
 
-typedef struct ERTS_POLL_EXPORT(erts_pollset) *ErtsPollSet;
+#if !ERTS_ENABLE_KERNEL_POLL
 
-typedef struct {
-    ErtsSysFdType fd;
-    ErtsPollEvents events;
-    int on;
-} ErtsPollControlEntry;
-
-typedef struct {
+typedef struct _ErtsPollResFd {
     ErtsSysFdType fd;
     ErtsPollEvents events;
 } ErtsPollResFd;
 
+#define ERTS_POLL_RES_GET_FD(evt) (evt)->fd
+#define ERTS_POLL_RES_SET_FD(evt, ident) (evt)->fd = (ident)
+#define ERTS_POLL_RES_GET_EVTS(evt) ERTS_POLL_EV_N2E((evt)->events)
+#define ERTS_POLL_RES_SET_EVTS(evt, evts) (evt)->events = ERTS_POLL_EV_E2N(evts)
+
+#endif
+
+#define ERTS_POLL_EV_NONE (UINT_MAX & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT|ERTS_POLL_EV_NVAL|ERTS_POLL_EV_ERR))
+
+#define ev2str(ev)                                                     \
+    (((ev) == 0 || (ev) == ERTS_POLL_EV_NONE) ? "NONE" :               \
+     ((ev) == ERTS_POLL_EV_IN ? "IN" :                                 \
+      ((ev) == ERTS_POLL_EV_OUT ? "OUT" :                              \
+       ((ev) == (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT) ? "IN|OUT" :        \
+        ((ev) & ERTS_POLL_EV_ERR ? "ERR" :                             \
+         ((ev) & ERTS_POLL_EV_NVAL ? "NVAL" : "OTHER"))))))
+
+
+typedef struct ERTS_POLL_EXPORT(erts_pollset) ErtsPollSet;
+
 typedef struct {
     char *primary;
-    char *fallback;
     char *kernel_poll;
     Uint memory_size;
-    int poll_set_size;
-    int fallback_poll_set_size;
+    Uint poll_set_size;
     int lazy_updates;
-    int pending_updates;
+    Uint pending_updates;
     int batch_updates;
     int concurrent_updates;
-    int max_fds;
-#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
-    long no_avoided_wakeups;
-    long no_avoided_interrupts;
-    long no_interrupt_timed;
-#endif
-    int active_fds;
+    int is_fallback;
+    Uint max_fds;
+    Uint active_fds;
+    Uint poll_threads;
 } ErtsPollInfo;
 
-void		ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet,
-						      int);
-void		ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet,
-							    int,
-							    ErtsMonotonicTime);
-ErtsPollEvents	ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet,
-						    ErtsSysFdType,
-						    ErtsPollEvents,
-						    int on,
-						    int* wake_poller
-						    );
-void		ERTS_POLL_EXPORT(erts_poll_controlv)(ErtsPollSet,
-						     ErtsPollControlEntry [],
-						     int on);
-int		ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet,
-						 ErtsPollResFd [],
-						 int *,
-						 ErtsMonotonicTime);
-int		ERTS_POLL_EXPORT(erts_poll_max_fds)(void);
-void		ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet,
-						 ErtsPollInfo *);
-ErtsPollSet	ERTS_POLL_EXPORT(erts_poll_create_pollset)(void);
-void		ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet);
-void		ERTS_POLL_EXPORT(erts_poll_init)(void);
-void		ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet,
-								ErtsPollEvents [],
-								int);
+#if defined(ERTS_POLL_USE_FALLBACK) && ERTS_KERNEL_POLL_VERSION
+#  undef ERTS_POLL_EXPORT
+#  define ERTS_POLL_EXPORT(FUNC) FUNC ## _flbk
+#  include "erl_poll_api.h"
+#  undef ERTS_POLL_EXPORT
+#  define ERTS_POLL_EXPORT(FUNC) FUNC
+#elif !defined(ERTS_POLL_USE_FALLBACK)
+#  define ERTS_POLL_USE_FALLBACK 0
+#endif
 
-int erts_poll_new_table_len(int old_len, int need_len);
+#include "erl_poll_api.h"
 
-#ifdef ERTS_ENABLE_LOCK_COUNT
-void ERTS_POLL_EXPORT(erts_lcnt_update_pollset_locks)(int enable);
-#endif
+/**
+ * Get the next size of the array that holds the file descriptors.
+ * This function is used in order for the check io array and the
+ * pollset array to be of the same size.
+ */
+int erts_poll_new_table_len(int old_len, int need_len);
 
 #endif /* #ifndef ERL_POLL_H__ */
diff --git a/erts/emulator/sys/common/erl_poll_api.h b/erts/emulator/sys/common/erl_poll_api.h
new file mode 100644
index 0000000000..04beb37d1c
--- /dev/null
+++ b/erts/emulator/sys/common/erl_poll_api.h
@@ -0,0 +1,122 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2006-2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+/**
+ * @description: Poll interface functions
+ * @author Lukas Larsson
+ *
+ * The functions in the header are used to interact with the poll
+ * implementation. Iff the kernel-poll implementation needs a fallback
+ * pollset, then all functions are exported twice. Once with a _flbk
+ * suffix and once without any suffix. If no fallback is needed, then
+ * only the non-suffix version is exported.
+ */
+
+/**
+ * Initialize the poll implementation. Has to be called before any other function.
+ * @param[out] concurrent_waiters if not NULL, set to 1 if more then one thread
+ * is allowed to wait in the pollsets at the same time.
+ */
+void ERTS_POLL_EXPORT(erts_poll_init)(int *concurrent_waiters);
+/**
+ * @brief Create a new pollset.
+ * @param id The unique debug id of this pollset.
+ */
+ErtsPollSet *ERTS_POLL_EXPORT(erts_poll_create_pollset)(int id);
+
+/**
+ * Modify the contents of a pollset. This function can be called while one
+ * (or possibly more) thread is waiting in the pollset.
+ *
+ * @param ps the pollset to modify
+ * @param fd the file descriptor to modify
+ * @param op the type of operation to do. Normal usage is ADD,MOD...MOD,DEL.
+ * @param evts the events that we are changing interest to. Ignored if op is DEL.
+ * @param[in] wake_poller if set to 1 any thread waiting in the pollset will be woken.
+ *   This parameter is ignored if the pollset supports concurrent waiters.
+ * @param[out] wake_poller set to 1 if the waiting thread was woken.
+ * @return The events set, or ERTS_POLL_EV_NVAL if it was not possible to add the
+ * fd to the pollset.
+ */
+ErtsPollEvents ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet *ps,
+                                                   ErtsSysFdType fd,
+                                                   ErtsPollOp op,
+                                                   ErtsPollEvents evts,
+                                                   int *wake_poller);
+
+/**
+ * Wait for events to be ready in the pollset. If the erts_poll_init call
+ * set concurrent_waiters to 1, then multiple threads are allowed to call
+ * this function at the same time.
+ *
+ * When an event has been triggered on a fd, that event is disabled. To
+ * re-enable it the implementation has to call erts_poll_control again.
+ *
+ * @param ps the pollset to wait for events in
+ * @param res an array of fd results that the ready fds are put in.
+ * @param[in] length the length of the res array
+ * @param[out] length the number of ready events returned in res
+ * @return 0 on success, else the ERRNO of the error that happened.
+ */
+int ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet *ps,
+                                     ErtsPollResFd res[],
+                                     int *length);
+/**
+ * Interrupt the thread waiting in the pollset. This function should be called
+ * with set = 0 before any thread calls erts_poll_wait in order to clear any
+ * interrupts that have happened while the thread was awake.
+ *
+ * This function has no effect on pollsets that support concurrent waiters.
+ *
+ * @param ps the pollset to wake
+ * @param set if 1, interrupt the pollset, if 0 clear the interrupt flag.
+ */
+void ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet *ps, int set);
+
+/* Debug functions */
+
+/**
+ * Get the maximum number of fds supported by the pollset
+ */
+int ERTS_POLL_EXPORT(erts_poll_max_fds)(void);
+/**
+ * Get information about the given pollset
+ */
+void ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet *ps,
+                                      ErtsPollInfo *info);
+/**
+ * Get information about which events are currently selected.
+ *
+ * The unix fd is used to index into the array, so naturally this function does
+ * not work on windows. If the pollset cannot figure out what the selected
+ * events for a given fd is, it is set to ERTS_POLL_EV_NONE.
+ *
+ * @param ps the pollset to get events from
+ * @param evts an array of which events are selected on.
+ */
+void ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet *ps,
+                                                     ErtsPollEvents evts[],
+                                                     int length);
+
+#ifdef ERTS_ENABLE_LOCK_COUNT
+/**
+ * Enable lock counting of any locks within the pollset.
+ */
+void ERTS_POLL_EXPORT(erts_lcnt_enable_pollset_lock_count)(ErtsPollSet *, int enable);
+#endif
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index acd7920e86..6315135151 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -499,22 +499,16 @@ static void signal_notify_requested(Eterm type) {
 static ERTS_INLINE void
 break_requested(void)
 {
-    int i;
   /*
    * just set a flag - checked for and handled by
    * scheduler threads erts_check_io() (not signal handler).
    */
-#ifdef DEBUG			
-  fprintf(stderr,"break!\n");
-#endif
   if (ERTS_BREAK_REQUESTED)
       erts_exit(ERTS_INTR_EXIT, "");
 
   ERTS_SET_BREAK_REQUESTED;
-  for (i=0; i < erts_no_schedulers; i++) {
-       /* Make sure we don't sleep in poll */
-      erts_check_io_interrupt(ERTS_SCHEDULER_IX(i)->pollset, 1);
-  }
+  /* Wake aux thread to get handle break */
+  erts_aux_thread_poke();
 }
 
 static RETSIGTYPE request_break(int signum)
@@ -1079,20 +1073,6 @@ erl_debug(char* fmt, ...)
 
 #endif /* DEBUG */
 
-/*
- * Called from schedule() when it runs out of runnable processes,
- * or when Erlang code has performed INPUT_REDUCTIONS reduction
- * steps. runnable == 0 iff there are no runnable Erlang processes.
- */
-void
-erl_sys_schedule(int runnable)
-{
-    erts_check_io(!runnable);
-    ERTS_LC_ASSERT(!erts_thr_progress_is_blocking());
-}
-
-
-
 static erts_tid_t sig_dispatcher_tid;
 
 static void
@@ -1281,103 +1261,18 @@ erts_sys_main_thread(void)
 }
 
 
-#ifdef ERTS_ENABLE_KERNEL_POLL /* get_value() is currently only used when
-				  kernel-poll is enabled */
-
-/* Get arg marks argument as handled by
-   putting NULL in argv */
-static char *
-get_value(char* rest, char** argv, int* ip)
-{
-    char *param = argv[*ip]+1;
-    argv[*ip] = NULL;
-    if (*rest == '\0') {
-	char *next = argv[*ip + 1];
-	if (next[0] == '-'
-	    && next[1] == '-'
-	    &&  next[2] == '\0') {
-	    erts_fprintf(stderr, "bad \"%s\" value: \n", param);
-	    erts_usage();
-	}
-	(*ip)++;
-	argv[*ip] = NULL;
-	return next;
-    }
-    return rest;
-}
-
-int erts_use_kernel_poll = 0;
-
-#endif /* ERTS_ENABLE_KERNEL_POLL */
-
 void
 erl_sys_args(int* argc, char** argv)
 {
-    int i, j;
 
     erts_rwmtx_init(&environ_rwmtx, "environ", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
-    i = 1;
-
     ASSERT(argc && argv);
 
-    while (i < *argc) {
-	if(argv[i][0] == '-') {
-	    switch (argv[i][1]) {
-#ifdef ERTS_ENABLE_KERNEL_POLL
-	    case 'K': {
-		char *arg = get_value(argv[i] + 2, argv, &i);
-		if (strcmp("true", arg) == 0) {
-		    erts_use_kernel_poll = 1;
-		}
-		else if (strcmp("false", arg) == 0) {
-		    erts_use_kernel_poll = 0;
-		}
-		else {
-		    erts_fprintf(stderr, "bad \"K\" value: %s\n", arg);
-		    erts_usage();
-		}
-		break;
-	    }
-#endif
-	    case '-':
-		goto done_parsing;
-	    default:
-		break;
-	    }
-	}
-	i++;
-    }
-
- done_parsing:
-
-#ifdef ERTS_ENABLE_KERNEL_POLL
-    if (erts_use_kernel_poll) {
-	char no_kp[10];
-	size_t no_kp_sz = sizeof(no_kp);
-	int res = erts_sys_getenv_raw("ERL_NO_KERNEL_POLL", no_kp, &no_kp_sz);
-	if (res > 0
-	    || (res == 0
-		&& sys_strcmp("false", no_kp) != 0
-		&& sys_strcmp("FALSE", no_kp) != 0)) {
-	    erts_use_kernel_poll = 0;
-	}
-    }
-#endif
-
-    erts_init_check_io();
     max_files = erts_check_io_max_files();
 
-
     init_smp_sig_notify();
     init_smp_sig_suspend();
 
-    /* Handled arguments have been marked with NULL. Slide arguments
-       not handled towards the beginning of argv. */
-    for (i = 0, j = 0; i < *argc; i++) {
-	if (argv[i])
-	    argv[j++] = argv[i];
-    }
-    *argc = j;
 }
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index 7a87e81141..24c8c7bbc7 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -34,6 +34,7 @@
  */
 
 /*#define HARDDEBUG */
+/*#define HARDTRACE */
 #ifdef HARDDEBUG
 #ifdef HARDTRACE
 #define HARDTRACEF(X) my_debug_printf##X
@@ -50,7 +51,7 @@ static void my_debug_printf(char *fmt, ...)
     va_start(args, fmt);
     erts_vsnprintf(buffer,1024,fmt,args);
     va_end(args);
-    erts_fprintf(stderr,"%s\r\n",buffer);
+    erts_printf("%s\r\n",buffer);
 }
 #else
 #define HARDTRACEF(X)
@@ -278,16 +279,13 @@ struct erts_pollset {
     Waiter** waiter;
     int allocated_waiters;  /* Size ow waiter array */ 
     int num_waiters;	    /* Number of waiter threads. */
-    int restore_events;        /* Tells us to restore waiters events 
-				  next time around */
     HANDLE event_io_ready;     /* To be used when waiting for io */
     /* These are used to wait for workers to enter standby */
     volatile int standby_wait_counter; /* Number of threads to wait for */
     CRITICAL_SECTION standby_crit;     /* CS to guard the counter */
-    HANDLE standby_wait_event;         /* Event signalled when counte == 0 */
+    HANDLE standby_wait_event;         /* Event signalled when counter == 0 */
     erts_atomic32_t wakeup_state;
     erts_mtx_t mtx;
-    erts_atomic64_t timeout_time;
 };
 
 
@@ -352,39 +350,19 @@ do { \
     wait_standby(PS); \
  } while(0)
 
-static ERTS_INLINE void
-init_timeout_time(ErtsPollSet ps)
-{
-    erts_atomic64_init_nob(&ps->timeout_time,
-			   (erts_aint64_t) ERTS_MONOTONIC_TIME_MAX);
-}
-
-static ERTS_INLINE void
-set_timeout_time(ErtsPollSet ps, ErtsMonotonicTime time)
-{
-    erts_atomic64_set_relb(&ps->timeout_time,
-			   (erts_aint64_t) time);
-}
-
-static ERTS_INLINE ErtsMonotonicTime
-get_timeout_time(ErtsPollSet ps)
-{
-    return (ErtsMonotonicTime) erts_atomic64_read_acqb(&ps->timeout_time);
-}
-
 #define ERTS_POLL_NOT_WOKEN		((erts_aint32_t) 0)
 #define ERTS_POLL_WOKEN_IO_READY	((erts_aint32_t) 1)
 #define ERTS_POLL_WOKEN_INTR		((erts_aint32_t) 2)
 #define ERTS_POLL_WOKEN_TIMEDOUT	((erts_aint32_t) 3)
 
 static ERTS_INLINE int
-is_io_ready(ErtsPollSet ps)
+is_io_ready(ErtsPollSet *ps)
 {
     return erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_WOKEN_IO_READY;
 }
 
 static ERTS_INLINE void
-woke_up(ErtsPollSet ps)
+woke_up(ErtsPollSet *ps)
 {
     if (erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_NOT_WOKEN)
 	erts_atomic32_cmpxchg_nob(&ps->wakeup_state,
@@ -407,7 +385,7 @@ woke_up(ErtsPollSet ps)
 }
 
 static ERTS_INLINE int
-wakeup_cause(ErtsPollSet ps)
+wakeup_cause(ErtsPollSet *ps)
 {
     int res;
     erts_aint32_t wakeup_state = erts_atomic32_read_acqb(&ps->wakeup_state);
@@ -430,46 +408,8 @@ wakeup_cause(ErtsPollSet ps)
     return res;
 }
 
-static ERTS_INLINE DWORD
-poll_wait_timeout(ErtsPollSet ps, ErtsMonotonicTime timeout_time)
-{
-    ErtsMonotonicTime current_time, diff_time, timeout;
-
-    if (timeout_time == ERTS_POLL_NO_TIMEOUT) {
-    no_timeout:
-	set_timeout_time(ps, ERTS_MONOTONIC_TIME_MIN);
-	woke_up(ps);
-	return (DWORD) 0;
-    }
-
-    current_time = erts_get_monotonic_time(NULL);
-    diff_time = timeout_time - current_time;
-    if (diff_time <= 0)
-	goto no_timeout;
-
-    /* Round up to nearest milli second */
-    timeout = (ERTS_MONOTONIC_TO_MSEC(diff_time - 1) + 1);
-    if (timeout > INT_MAX)
-	timeout = INT_MAX; /* Also prevents DWORD overflow */
-
-    set_timeout_time(ps, current_time + ERTS_MSEC_TO_MONOTONIC(timeout));
-
-    ResetEvent(ps->event_io_ready);
-    /*
-     * Since we don't know the internals of ResetEvent() we issue
-     * a memory barrier as a safety precaution ensuring that
-     * the load of wakeup_state wont be reordered with stores made
-     * by ResetEvent().
-     */
-    ERTS_THR_MEMORY_BARRIER;
-    if (erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN)
-	return (DWORD) 0;
-
-    return (DWORD) timeout;
-}
-
 static ERTS_INLINE void
-wake_poller(ErtsPollSet ps, int io_ready)
+wake_poller(ErtsPollSet *ps, int io_ready)
 {
     erts_aint32_t wakeup_state;
     if (io_ready) {
@@ -504,13 +444,13 @@ wake_poller(ErtsPollSet ps, int io_ready)
 }
 
 static ERTS_INLINE void
-reset_io_ready(ErtsPollSet ps)
+reset_io_ready(ErtsPollSet *ps)
 {
     erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
 }
 
 static ERTS_INLINE void
-restore_io_ready(ErtsPollSet ps)
+restore_io_ready(ErtsPollSet *ps)
 {
     erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY);
 }
@@ -520,13 +460,13 @@ restore_io_ready(ErtsPollSet ps)
  * notifying a poller thread about I/O ready.
  */
 static ERTS_INLINE void
-notify_io_ready(ErtsPollSet ps)
+notify_io_ready(ErtsPollSet *ps)
 {
     wake_poller(ps, 1);
 }
 
 static ERTS_INLINE void
-reset_interrupt(ErtsPollSet ps)
+reset_interrupt(ErtsPollSet *ps)
 {
     /* We need to keep io-ready if set */
     erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
@@ -543,12 +483,12 @@ reset_interrupt(ErtsPollSet ps)
 }
 
 static ERTS_INLINE void
-set_interrupt(ErtsPollSet ps)
+set_interrupt(ErtsPollSet *ps)
 {
     wake_poller(ps, 0);
 }
 
-static void setup_standby_wait(ErtsPollSet ps, int num_threads)
+static void setup_standby_wait(ErtsPollSet *ps, int num_threads)
 {
     EnterCriticalSection(&(ps->standby_crit));
     ps->standby_wait_counter = num_threads;
@@ -556,7 +496,7 @@ static void setup_standby_wait(ErtsPollSet ps, int num_threads)
     LeaveCriticalSection(&(ps->standby_crit));
 }
 
-static void signal_standby(ErtsPollSet ps) 
+static void signal_standby(ErtsPollSet *ps)
 {
     EnterCriticalSection(&(ps->standby_crit));
     --(ps->standby_wait_counter);
@@ -570,7 +510,7 @@ static void signal_standby(ErtsPollSet ps)
     LeaveCriticalSection(&(ps->standby_crit));
 }
 
-static void wait_standby(ErtsPollSet ps)
+static void wait_standby(ErtsPollSet *ps)
 {
     WaitForSingleObject(ps->standby_wait_event,INFINITE);
 }
@@ -638,7 +578,7 @@ static void consistency_check(Waiter* w)
 
 #endif
 
-static void new_waiter(ErtsPollSet ps)
+static void new_waiter(ErtsPollSet *ps)
 {
     register Waiter* w;
     DWORD tid;			/* Id for thread. */
@@ -732,7 +672,7 @@ static void *break_waiter(void *param)
 static void *threaded_waiter(void *param)
 {
     register Waiter* w = (Waiter *) param;
-    ErtsPollSet ps = (ErtsPollSet) w->xdata;
+    ErtsPollSet *ps = (ErtsPollSet*) w->xdata;
 #ifdef HARD_POLL_DEBUG2
     HANDLE oold_fired[64];
     int num_oold_fired;
@@ -835,9 +775,9 @@ event_happened:
 	    ASSERT(i >= WAIT_OBJECT_0+1);
 	    i -= WAIT_OBJECT_0;
 	    ASSERT(i >= 1);
-	    w->active_events--;
 	    HARDDEBUGF(("i = %d, a,h,t = %d,%d,%d",i,
 			w->active_events, w->highwater, w->total_events));
+	    w->active_events--;
 #ifdef HARD_POLL_DEBUG2
 	    fired[num_fired++] = w->events[i];
 #endif
@@ -867,7 +807,7 @@ event_happened:
  * The actual adding and removing from pollset utilities 
  */
 
-static int set_driver_select(ErtsPollSet ps, HANDLE event, ErtsPollEvents mode)
+static int set_driver_select(ErtsPollSet *ps, HANDLE event, ErtsPollEvents mode)
 {
     int i;
     int best_waiter = -1;	/* The waiter with lowest number of events. */
@@ -957,13 +897,13 @@ static int set_driver_select(ErtsPollSet ps, HANDLE event, ErtsPollEvents mode)
 #endif
     erts_mtx_unlock(&w->mtx);
     START_WAITER(ps,w);
-    HARDDEBUGF(("add select %d %d %d %d",best_waiter,
+    HARDDEBUGF(("%d: add select %d %d %d %d", event, best_waiter,
 		w->active_events,w->highwater,w->total_events));
     return mode;
 }
 
 
-static int cancel_driver_select(ErtsPollSet ps, HANDLE event)
+static int cancel_driver_select(ErtsPollSet *ps, HANDLE event)
 {
     int i;
 
@@ -1018,7 +958,7 @@ static int cancel_driver_select(ErtsPollSet ps, HANDLE event)
  * Interface functions
  */
 
-void  erts_poll_interrupt(ErtsPollSet ps, int set /* bool */)
+void  erts_poll_interrupt(ErtsPollSet *ps, int set /* bool */)
 {
     HARDTRACEF(("In erts_poll_interrupt(%d)",set));
     if (!set)
@@ -1028,35 +968,23 @@ void  erts_poll_interrupt(ErtsPollSet ps, int set /* bool */)
     HARDTRACEF(("Out erts_poll_interrupt(%d)",set));
 }
 
-void erts_poll_interrupt_timed(ErtsPollSet ps,
-			       int set /* bool */,
-			       ErtsMonotonicTime timeout_time)
-{
-    HARDTRACEF(("In erts_poll_interrupt_timed(%d,%ld)",set,timeout_time));
-    if (!set)
-	reset_interrupt(ps);
-    else if (get_timeout_time(ps) > timeout_time)
-	set_interrupt(ps);
-    HARDTRACEF(("Out erts_poll_interrupt_timed"));
-}
-
 
 /*
  * Windows is special, there is actually only one event type, and
  * the only difference between ERTS_POLL_EV_IN and ERTS_POLL_EV_OUT
  * is which driver callback will eventually be called.
  */
-static ErtsPollEvents do_poll_control(ErtsPollSet ps,
-				      ErtsSysFdType fd,
-				      ErtsPollEvents pe,
-				      int on /* bool */)
+static ErtsPollEvents do_poll_control(ErtsPollSet *ps,
+                                      ErtsSysFdType fd,
+                                      ErtsPollOp op,
+                                      ErtsPollEvents pe)
 {
     HANDLE event = (HANDLE) fd;
     ErtsPollEvents mode;
     ErtsPollEvents result;
     ASSERT(event != INVALID_HANDLE_VALUE);
 
-    if (on) {
+    if (op != ERTS_POLL_OP_DEL) {
 	if (pe & ERTS_POLL_EV_IN || !(pe & ERTS_POLL_EV_OUT )) {
 	    mode = ERTS_POLL_EV_IN;
 	} else {
@@ -1069,51 +997,30 @@ static ErtsPollEvents do_poll_control(ErtsPollSet ps,
     return result;
 }
 
-ErtsPollEvents erts_poll_control(ErtsPollSet ps,
+ErtsPollEvents erts_poll_control(ErtsPollSet *ps,
 				 ErtsSysFdType fd,
+                                 ErtsPollOp op,
 				 ErtsPollEvents pe,
-				 int on,
 				 int* do_wake) /* In: Wake up polling thread */
 				               /* Out: Poller is woken */
 {
     ErtsPollEvents result;
-    HARDTRACEF(("In erts_poll_control(0x%08X, %u, %d)",(unsigned long) fd, (unsigned) pe, on));
+    HARDTRACEF(("In erts_poll_control(0x%08X, %s, %s)",
+                (unsigned long) fd, op2str(op), ev2str(pe)));
     ERTS_POLLSET_LOCK(ps);
-    result=do_poll_control(ps,fd,pe,on);
+    result=do_poll_control(ps, fd, op, pe);
     ERTS_POLLSET_UNLOCK(ps);
     *do_wake = 0; /* Never any need to wake polling threads on windows */
     HARDTRACEF(("Out erts_poll_control -> %u",(unsigned) result));
     return result;
 }
 
-void erts_poll_controlv(ErtsPollSet ps,
-			ErtsPollControlEntry pcev[],
-			int len)
-{
-    int i;
-    int hshur = 0;
-    int do_wake = 0;
-
-    HARDTRACEF(("In erts_poll_controlv(%d)",len));
-    ERTS_POLLSET_LOCK(ps);
-
-    for (i = 0; i < len; i++) {
-	pcev[i].events = do_poll_control(ps,
-					 pcev[i].fd,
-					 pcev[i].events,
-					 pcev[i].on);
-    }
-    ERTS_POLLSET_UNLOCK(ps);
-    HARDTRACEF(("Out erts_poll_controlv"));
-}
-
-int erts_poll_wait(ErtsPollSet ps,
+int erts_poll_wait(ErtsPollSet *ps,
 		   ErtsPollResFd pr[],
-		   int *len,
-		   ErtsMonotonicTime timeout_time)
+		   int *len)
 {
     int no_fds;
-    DWORD timeout;
+    DWORD timeout = INFINITE;
     EventData* ev;
     int res = 0;
     int num = 0;
@@ -1124,42 +1031,6 @@ int erts_poll_wait(ErtsPollSet ps,
     HARDTRACEF(("In erts_poll_wait"));
     ERTS_POLLSET_LOCK(ps);
 
-    if (!is_io_ready(ps) && ps->restore_events) {
-	HARDDEBUGF(("Restore events: %d",ps->num_waiters));
-	ps->restore_events = 0;
-	for (i = 0; i < ps->num_waiters; ++i) {
-	   Waiter* w = ps->waiter[i];
-	   erts_mtx_lock(&w->mtx);
-	   HARDDEBUGF(("Maybe reset %d %d %d %d",i,
-		       w->active_events,w->highwater,w->total_events));
-	   if (w->active_events < w->total_events) {
-	       erts_mtx_unlock(&w->mtx);
-	       STOP_WAITER(ps,w);
-	       HARDDEBUGF(("Need reset %d %d %d %d",i,
-			   w->active_events,w->highwater,w->total_events));
-	       erts_mtx_lock(&w->mtx);	       
-	       /* Need reset, just check that it doesn't have got more to tell */
-	       if (w->highwater != w->active_events) {
-		   HARDDEBUGF(("Oups!"));
-		   /* Oups, got signalled before we took the lock, can't reset */
-		   if(!is_io_ready(ps)) {
-		       erts_exit(ERTS_ERROR_EXIT,"Internal error: "
-				"Inconsistent io structures in erl_poll.\n");
-		   }
-		   START_WAITER(ps,w);
-		   erts_mtx_unlock(&w->mtx);
-		   ps->restore_events = 1; 
-		   continue;
-	       }
-	       w->active_events = w->highwater =  w->total_events;
-	       START_WAITER(ps,w);
-	       erts_mtx_unlock(&w->mtx);
-	   } else {
-	       erts_mtx_unlock(&w->mtx);
-	   }
-	}
-    }
-
     no_fds = *len;
 
 #ifdef ERTS_POLL_MAX_RES
@@ -1167,11 +1038,18 @@ int erts_poll_wait(ErtsPollSet ps,
 	no_fds = ERTS_POLL_MAX_RES;
 #endif
 
-    timeout = poll_wait_timeout(ps, timeout_time);
-
-    /*HARDDEBUGF(("timeout = %ld",(long) timeout));*/
+    ResetEvent(ps->event_io_ready);
+    /*
+     * Since we don't know the internals of ResetEvent() we issue
+     * a memory barrier as a safety precaution ensuring that
+     * the load of wakeup_state wont be reordered with stores made
+     * by ResetEvent().
+     */
+    ERTS_THR_MEMORY_BARRIER;
+    if (erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN)
+       timeout = (DWORD) 0;
 
-    if (timeout > 0 && !erts_atomic32_read_nob(&break_waiter_state)) {
+    if (!erts_atomic32_read_nob(&break_waiter_state)) {
 	HANDLE harr[2] = {ps->event_io_ready, break_happened_event};
 	int num_h = 2;
         ERTS_MSACC_PUSH_STATE_M();
@@ -1215,7 +1093,7 @@ int erts_poll_wait(ErtsPollSet ps,
 
     reset_io_ready(ps);
 
-    n = ps->num_waiters;	
+    n = ps->num_waiters;
 
     for (i = 0; i < n; i++) {
 	Waiter* w = ps->waiter[i];
@@ -1241,11 +1119,10 @@ int erts_poll_wait(ErtsPollSet ps,
 		HARDDEBUGF(("To many FD's to report!"));
 		goto done;
 	    }
-	    HARDDEBUGF(("SET! Restore events"));
-	    ps->restore_events = 1;
 	    HARDDEBUGF(("Report %d,%d",i,j));
-	    pr[num].fd = (ErtsSysFdType) w->events[j];
-	    pr[num].events = w->evdata[j]->mode;
+	    ERTS_POLL_RES_SET_FD(&pr[num], w->events[j]);
+	    ERTS_POLL_RES_SET_EVTS(&pr[num], w->evdata[j]->mode);
+            remove_event_from_set(w, j);
 #ifdef HARD_POLL_DEBUG
 	    poll_debug_reported(w->events[j],w->highwater | (j << 16));
 	    poll_debug_reported(w->events[j],first | (last << 16));
@@ -1253,13 +1130,14 @@ int erts_poll_wait(ErtsPollSet ps,
 	    ++num;
 	}
 
+        w->total_events = w->highwater = w->active_events;
+
 #ifdef DEBUG
 	consistency_check(w);
 #endif
 	erts_mtx_unlock(&w->mtx);
     }
  done:
-    set_timeout_time(ps, ERTS_MONOTONIC_TIME_MAX);
     *len = num;
     ERTS_POLLSET_UNLOCK(ps);
     HARDTRACEF(("Out erts_poll_wait"));
@@ -1274,7 +1152,7 @@ int erts_poll_max_fds(void)
     return res;
 }
 
-void erts_poll_info(ErtsPollSet ps,
+void erts_poll_info(ErtsPollSet *ps,
 		    ErtsPollInfo *pip)
 {
     Uint size = 0;
@@ -1299,16 +1177,12 @@ void erts_poll_info(ErtsPollSet ps,
 
     pip->primary = "WaitForMultipleObjects"; 
 
-    pip->fallback = NULL;
-
     pip->kernel_poll = NULL;
 
     pip->memory_size = size;
 
     pip->poll_set_size = num_events;
 
-    pip->fallback_poll_set_size = 0;
-
     pip->lazy_updates = 0;
 
     pip->pending_updates = 0;
@@ -1316,6 +1190,8 @@ void erts_poll_info(ErtsPollSet ps,
     pip->batch_updates = 0;
 
     pip->concurrent_updates = 0;
+
+    pip->is_fallback = 0;
     ERTS_POLLSET_UNLOCK(ps);
 
     pip->max_fds = erts_poll_max_fds();
@@ -1323,9 +1199,9 @@ void erts_poll_info(ErtsPollSet ps,
 
 }
 
-ErtsPollSet erts_poll_create_pollset(void)
+ErtsPollSet *erts_poll_create_pollset(int no)
 {
-    ErtsPollSet ps = SEL_ALLOC(ERTS_ALC_T_POLLSET,
+    ErtsPollSet *ps = SEL_ALLOC(ERTS_ALC_T_POLLSET,
 			       sizeof(struct erts_pollset));
     HARDTRACEF(("In erts_poll_create_pollset"));
 
@@ -1337,17 +1213,15 @@ ErtsPollSet erts_poll_create_pollset(void)
     ps->standby_wait_counter = 0;
     ps->event_io_ready = CreateManualEvent(FALSE);
     ps->standby_wait_event = CreateManualEvent(FALSE); 
-    ps->restore_events = 0;
 
     erts_atomic32_init_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
     erts_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO);
-    init_timeout_time(ps);
 
     HARDTRACEF(("Out erts_poll_create_pollset"));
     return ps;
 }
 
-void erts_poll_destroy_pollset(ErtsPollSet ps)
+void erts_poll_destroy_pollset(ErtsPollSet *ps)
 {
     int i;
     HARDTRACEF(("In erts_poll_destroy_pollset"));
@@ -1378,14 +1252,16 @@ void erts_poll_destroy_pollset(ErtsPollSet ps)
 /*
  * Actually mostly initializes the friend module sys_interrupt...
  */
-void  erts_poll_init(void)
+void  erts_poll_init(int *concurrent_updates)
 {
-    erts_tid_t thread;
 
 #ifdef HARD_POLL_DEBUG
     poll_debug_init();
 #endif
 
+    if (concurrent_updates)
+        *concurrent_updates = 0;
+
     HARDTRACEF(("In erts_poll_init"));
     erts_sys_break_event = CreateManualEvent(FALSE);
 
@@ -1394,21 +1270,26 @@ void  erts_poll_init(void)
     break_happened_event = CreateManualEvent(FALSE);
     erts_atomic32_init_nob(&break_waiter_state, 0); 
 
+    HARDTRACEF(("Out erts_poll_init"));
+}
+
+void erts_poll_late_init(void)
+{
+    erts_tid_t thread;
     erts_thr_create(&thread, &break_waiter, NULL, NULL);
     ERTS_UNSET_BREAK_REQUESTED;
-    HARDTRACEF(("Out erts_poll_init"));
 }
 
 /*
  * Non windows friendly interface, not used when fd's are not continous
  */
-void  erts_poll_get_selected_events(ErtsPollSet ps,
+void  erts_poll_get_selected_events(ErtsPollSet *ps,
 				    ErtsPollEvents ev[],
 				    int len)
 {
     int i;
     HARDTRACEF(("In erts_poll_get_selected_events"));
     for (i = 0; i < len; ++i)
-	ev[i] = 0;
+	ev[i] = ERTS_POLL_EV_NONE;
     HARDTRACEF(("Out erts_poll_get_selected_events"));
 }
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index de391f078b..0598a12351 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -3235,27 +3235,16 @@ void erl_sys_init(void)
 	SetStdHandle(STD_ERROR_HANDLE, GetStdHandle(STD_OUTPUT_HANDLE));
     }
     erts_sys_init_float();
-    erts_init_check_io();
 
     /* Suppress windows error message popups */
     SetErrorMode(SetErrorMode(0) |
 		 SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX); 
 }
+void erts_poll_late_init(void);
 
 void
 erl_sys_late_init(void)
 {
     /* do nothing */
-}
-
-/*
- * Called from schedule() when it runs out of runnable processes,
- * or when Erlang code has performed INPUT_REDUCTIONS reduction
- * steps. runnable == 0 iff there are no runnable Erlang processes.
- */
-void
-erl_sys_schedule(int runnable)
-{
-    erts_check_io(!runnable);
-    ERTS_LC_ASSERT(!erts_thr_progress_is_blocking());
+    erts_poll_late_init();
 }
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 332d11345b..0e691b0c80 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -136,7 +136,7 @@ suite() ->
      {timetrap, {minutes, 1}}].
 
 all() -> %% Keep a_test first and z_test last...
-    [a_test, outputv_errors, outputv_echo, queue_echo, {group, timer},
+    [a_test, outputv_errors, outputv_echo, queue_echo, %{group, timer},
      driver_unloaded, io_ready_exit, use_fallback_pollset,
      bad_fd_in_pollset, fd_change,
      steal_control, otp_6602, driver_system_info_base_ver,
@@ -785,12 +785,9 @@ io_ready_exit(Config) when is_list(Config) ->
 
 use_fallback_pollset(Config) when is_list(Config) ->
     FlbkFun = fun () ->
-                      ChkIoDuring = get_check_io_total(erlang:system_info(check_io)),
-                      case lists:keysearch(fallback_poll_set_size,
-                                           1,
-                                           ChkIoDuring) of
-                          {value,
-                           {fallback_poll_set_size, N}} when N > 0 ->
+                      {Flbk, _} = get_fallback(erlang:system_info(check_io)),
+                      case lists:keysearch(total_poll_set_size, 1, Flbk) of
+                          {value, {total_poll_set_size, N}} when N > 0 ->
                               ok;
                           Error ->
                               ct:fail({failed_to_use_fallback, Error})
@@ -1013,14 +1010,15 @@ get_stable_check_io_info() ->
 %% Merge return from erlang:system_info(check_io)
 %% as if it was one big pollset.
 get_check_io_total(ChkIo) ->
-    lists:foldl(fun(Pollset, Acc) ->
-                        lists:zipwith(fun(A, B) ->
-                                              add_pollset_infos(A,B)
-                                      end,
-                                      Pollset, Acc)
-                end,
-                hd(ChkIo),
-                tl(ChkIo)).
+    {Fallback, Rest} = get_fallback(ChkIo),
+    add_fallback_infos(Fallback,
+      lists:foldl(fun(Pollset, Acc) ->
+                          lists:zipwith(fun(A, B) ->
+                                                add_pollset_infos(A,B)
+                                        end,
+                                        Pollset, Acc)
+                  end,
+                  hd(Rest), tl(Rest))).
 
 add_pollset_infos({Tag, A}=TA , {Tag, B}=TB) ->
     case tag_type(Tag) of
@@ -1035,13 +1033,32 @@ add_pollset_infos({Tag, A}=TA , {Tag, B}=TB) ->
             end
     end.
 
+get_fallback([MaybeFallback | ChkIo] = AllChkIo) ->
+    case proplists:get_value(fallback, MaybeFallback) of
+        true ->
+            {MaybeFallback, ChkIo};
+        false ->
+            {undefined, AllChkIo}
+    end.
+
+add_fallback_infos(undefined, Acc) ->
+    Acc;
+add_fallback_infos(Flbk, Acc) ->
+    lists:zipwith(fun({Tag, A}=TA, {Tag, B}=TB) ->
+                          case tag_type(Tag) of
+                              sum -> {Tag, A + B};
+                              const when Tag =:= fallback -> TA;
+                              const -> TB
+                          end
+                  end,
+                  Flbk, Acc).
+
 tag_type(name) -> const;
 tag_type(primary) -> const;
 tag_type(fallback) -> const;
 tag_type(kernel_poll) -> const;
 tag_type(memory_size) -> sum;
 tag_type(total_poll_set_size) -> sum;
-tag_type(fallback_poll_set_size) -> sum;
 tag_type(lazy_updates) -> const;
 tag_type(pending_updates) -> sum;
 tag_type(batch_updates) -> const;
diff --git a/erts/emulator/test/driver_SUITE_data/chkio_drv.c b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
index 5167036602..d548c4b1dc 100644
--- a/erts/emulator/test/driver_SUITE_data/chkio_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
@@ -511,6 +511,9 @@ chkio_drv_ready_input(ErlDrvData drv_data, ErlDrvEvent event)
 	    driver_failure_atom(cddp->port, "input_fd_not_found");
 	break;
     }
+    case CHKIO_FD_CHANGE:
+        /* This may be triggered when an fd is closed while being selected on. */
+        break;
     case CHKIO_STEAL:
 	break;
     case CHKIO_STEAL_AUX:
-- 
cgit v1.2.3


From 22cde2bda706c0bd8574f0a1301170c80b5f4340 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 19 Jul 2017 11:47:16 +0200
Subject: erts: Remove eager check io

It is not longer relevant when using the poll thread
---
 erts/emulator/beam/erl_bif_info.c |  2 +-
 erts/emulator/beam/erl_init.c     | 19 +++----------------
 erts/emulator/beam/erl_process.c  |  1 -
 erts/emulator/beam/erl_process.h  |  1 -
 4 files changed, 4 insertions(+), 19 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index d0c2accec6..17c936f041 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2843,7 +2843,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(am_disabled);
     }
     else if (ERTS_IS_ATOM_STR("eager_check_io",BIF_ARG_1)) {
-	BIF_RET(erts_eager_check_io ? am_true : am_false);
+	BIF_RET(am_true);
     }
     else if (ERTS_IS_ATOM_STR("literal_test",BIF_ARG_1)) {
 #ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index b4d84f6b16..6cef9bd0e3 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -1748,22 +1748,9 @@ erl_start(int argc, char **argv)
 		    erts_usage();
 		}
 	    }
-	    else if (has_prefix("ecio", sub_param)) {
-		arg = get_arg(sub_param+4, argv[i+1], &i);
-#ifndef __OSE__
-		if (sys_strcmp("true", arg) == 0)
-		    erts_eager_check_io = 1;
-		else
-#endif
-		if (sys_strcmp("false", arg) == 0)
-		    erts_eager_check_io = 0;
-		else {
-		    erts_fprintf(stderr,
-				 "bad schedule eager check I/O value '%s'\n",
-				 arg);
-		    erts_usage();
-		}
-	    }
+            else if (has_prefix("ecio", sub_param)) {
+                /* ignore argument, eager check io no longer used */
+            }
 	    else if (has_prefix("pp", sub_param)) {
 		arg = get_arg(sub_param+2, argv[i+1], &i);
 		if (sys_strcmp(arg, "true") == 0)
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index ffb67aefaf..938dd1b9ac 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -174,7 +174,6 @@ extern BeamInstr beam_exit[];
 extern BeamInstr beam_continue_exit[];
 
 int ERTS_WRITE_UNLIKELY(erts_default_spo_flags) = SPO_ON_HEAP_MSGQ;
-int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1;
 int ERTS_WRITE_UNLIKELY(erts_sched_compact_load);
 int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0;
 Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index b4c06adc95..66d7848f89 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -105,7 +105,6 @@ struct saved_calls {
 };
 
 extern Export exp_send, exp_receive, exp_timeout;
-extern int erts_eager_check_io;
 extern int erts_sched_compact_load;
 extern int erts_sched_balance_util;
 extern Uint erts_no_schedulers;
-- 
cgit v1.2.3


From e1e22afa5429909108560dc2f20912614cbe8a3c Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 14 Aug 2017 16:44:38 +0200
Subject: erts: get_internal_state(check_io_debug) now prints to error_logger

---
 erts/emulator/beam/safe_hash.c          |   7 +-
 erts/emulator/beam/safe_hash.h          |   2 +-
 erts/emulator/sys/common/erl_check_io.c | 178 ++++++++++++++++----------------
 3 files changed, 95 insertions(+), 92 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c
index ac9ebd4714..73306030ae 100644
--- a/erts/emulator/beam/safe_hash.c
+++ b/erts/emulator/beam/safe_hash.c
@@ -260,16 +260,17 @@ void* safe_hash_erase(SafeHash* h, void* tmpl)
 }
 
 /*
-** Call 'func(obj,func_arg2)' for all objects in table. NOT SAFE!!!
+** Call 'func(obj,func_arg2,func_arg3)' for all objects in table. NOT SAFE!!!
 */
-void safe_hash_for_each(SafeHash* h, void (*func)(void *, void *), void *func_arg2)
+void safe_hash_for_each(SafeHash* h, void (*func)(void *, void *, void *),
+                        void *func_arg2, void *func_arg3)
 {
     int i;
 
     for (i = 0; i <= h->size_mask; i++) {
 	SafeHashBucket* b = h->tab[i];
 	while (b != NULL) {
-	    (*func)((void *) b, func_arg2);
+	    (*func)((void *) b, func_arg2, func_arg3);
 	    b = b->next;
 	}
     }
diff --git a/erts/emulator/beam/safe_hash.h b/erts/emulator/beam/safe_hash.h
index 259c58cff9..af97b4cb4d 100644
--- a/erts/emulator/beam/safe_hash.h
+++ b/erts/emulator/beam/safe_hash.h
@@ -95,7 +95,7 @@ void* safe_hash_get(SafeHash*, void*);
 void* safe_hash_put(SafeHash*, void*);
 void* safe_hash_erase(SafeHash*, void*);
 
-void  safe_hash_for_each(SafeHash*, void (*func)(void *, void *), void *);
+void  safe_hash_for_each(SafeHash*, void (*func)(void *, void *, void *), void *, void *);
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
 void erts_lcnt_enable_hash_lock_count(SafeHash*, erts_lock_flags_t, int);
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 3131b96536..4019b78074 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -2246,47 +2246,47 @@ erts_check_io_info(void *proc)
 }
 
 static ERTS_INLINE ErtsPollEvents
-print_events(ErtsPollEvents ev)
+print_events(erts_dsprintf_buf_t *dsbufp, ErtsPollEvents ev)
 {
     int first = 1;
     if(ev == ERTS_POLL_EV_NONE) {
-        erts_printf("N/A");
+        erts_dsprintf(dsbufp, "N/A");
         return 0;
     }
     if(ev & ERTS_POLL_EV_IN) {
 	ev &= ~ERTS_POLL_EV_IN;
-	erts_printf("%s%s", first ? "" : "|", "IN");
+	erts_dsprintf(dsbufp, "%s%s", first ? "" : "|", "IN");
 	first = 0;
     }
     if(ev & ERTS_POLL_EV_OUT) {
 	ev &= ~ERTS_POLL_EV_OUT;
-	erts_printf("%s%s", first ? "" : "|", "OUT");
+	erts_dsprintf(dsbufp, "%s%s", first ? "" : "|", "OUT");
 	first = 0;
     }
     /* The following should not appear... */
     if(ev & ERTS_POLL_EV_NVAL) {
-	erts_printf("%s%s", first ? "" : "|", "NVAL");
+	erts_dsprintf(dsbufp, "%s%s", first ? "" : "|", "NVAL");
 	first = 0;
     }
     if(ev & ERTS_POLL_EV_ERR) {
-	erts_printf("%s%s", first ? "" : "|", "ERR");
+	erts_dsprintf(dsbufp, "%s%s", first ? "" : "|", "ERR");
 	first = 0;
     }
     if (ev)
-	erts_printf("%s0x%b32x", first ? "" : "|", (Uint32) ev);
+	erts_dsprintf(dsbufp, "%s0x%b32x", first ? "" : "|", (Uint32) ev);
     return ev;
 }
 
 static ERTS_INLINE void
-print_flags(EventStateFlags f)
+print_flags(erts_dsprintf_buf_t *dsbufp, EventStateFlags f)
 {
     const char* delim = "";
     if(f & ERTS_EV_FLAG_USED) {
-	erts_printf("%s","USED");
+	erts_dsprintf(dsbufp, "%s","USED");
 	delim = "|";
     }
     if(f & ERTS_EV_FLAG_FALLBACK) {
-	erts_printf("%s%s", delim, "FLBK");
+	erts_dsprintf(dsbufp, "%s%s", delim, "FLBK");
         delim = "|";
     }
 }
@@ -2330,133 +2330,133 @@ typedef struct {
 #endif
 } IterDebugCounters;
 
-static int erts_debug_print_checkio_state(ErtsDrvEventState *state,
+static int erts_debug_print_checkio_state(erts_dsprintf_buf_t *dsbufp,
+                                          ErtsDrvEventState *state,
                                           ErtsPollEvents ep_events,
                                           int internal)
 {
 #if defined(HAVE_FSTAT) && !defined(NO_FSTAT_ON_SYS_FD_TYPE)
     struct stat stat_buf;
 #endif
-
     ErtsSysFdType fd = state->fd;
     ErtsPollEvents cio_events = state->events;
     int err = 0;
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     ErtsPollEvents aio_events = state->active_events;
 #endif
-    erts_printf("pollset=%d fd=%d ",
+    erts_dsprintf(dsbufp, "pollset=%d fd=%d ",
                 state->flags & ERTS_EV_FLAG_FALLBACK ? -1 : get_pollset_id(fd), (int) fd);
 
 #if defined(HAVE_FSTAT) && !defined(NO_FSTAT_ON_SYS_FD_TYPE)
     if (fstat((int) fd, &stat_buf) < 0)
-        erts_printf("type=unknown ");
+        erts_dsprintf(dsbufp, "type=unknown ");
     else {
-        erts_printf("type=");
+        erts_dsprintf(dsbufp, "type=");
 #ifdef S_ISSOCK
         if (S_ISSOCK(stat_buf.st_mode))
-            erts_printf("sock ");
+            erts_dsprintf(dsbufp, "sock ");
         else
 #endif
 #ifdef S_ISFIFO
 	    if (S_ISFIFO(stat_buf.st_mode))
-		erts_printf("fifo ");
+		erts_dsprintf(dsbufp, "fifo ");
 	    else
 #endif
 #ifdef S_ISCHR
                 if (S_ISCHR(stat_buf.st_mode))
-                    erts_printf("chr ");
+                    erts_dsprintf(dsbufp, "chr ");
                 else
 #endif
 #ifdef S_ISDIR
                     if (S_ISDIR(stat_buf.st_mode))
-                        erts_printf("dir ");
+                        erts_dsprintf(dsbufp, "dir ");
                     else
 #endif
 #ifdef S_ISBLK
                         if (S_ISBLK(stat_buf.st_mode))
-                            erts_printf("blk ");
+                            erts_dsprintf(dsbufp, "blk ");
                         else
 #endif
 #ifdef S_ISREG
                             if (S_ISREG(stat_buf.st_mode))
-                                erts_printf("reg ");
+                                erts_dsprintf(dsbufp, "reg ");
                             else
 #endif
 #ifdef S_ISLNK
                                 if (S_ISLNK(stat_buf.st_mode))
-                                    erts_printf("lnk ");
+                                    erts_dsprintf(dsbufp, "lnk ");
                                 else
 #endif
 #ifdef S_ISDOOR
                                     if (S_ISDOOR(stat_buf.st_mode))
-                                        erts_printf("door ");
+                                        erts_dsprintf(dsbufp, "door ");
                                     else
 #endif
 #ifdef S_ISWHT
                                         if (S_ISWHT(stat_buf.st_mode))
-                                            erts_printf("wht ");
+                                            erts_dsprintf(dsbufp, "wht ");
                                         else
 #endif
 #ifdef S_ISXATTR
                                             if (S_ISXATTR(stat_buf.st_mode))
-                                                erts_printf("xattr ");
+                                                erts_dsprintf(dsbufp, "xattr ");
                                             else
 #endif
-                                                erts_printf("unknown ");
+                                                erts_dsprintf(dsbufp, "unknown ");
     }
 #else
-    erts_printf("type=unknown ");
+    erts_dsprintf(dsbufp, "type=unknown ");
 #endif
 
     if (state->type == ERTS_EV_TYPE_DRV_SEL) {
-        erts_printf("driver_select ");
+        erts_dsprintf(dsbufp, "driver_select ");
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
         if (internal) {
-            erts_printf("internal ");
+            erts_dsprintf(dsbufp, "internal ");
             err = 1;
         }
         if (aio_events == cio_events) {
             if (cio_events == ep_events) {
-                erts_printf("ev=");
-                if (print_events(cio_events) != 0)
+                erts_dsprintf(dsbufp, "ev=");
+                if (print_events(dsbufp, cio_events) != 0)
                     err = 1;
             }
             else {
                 ErtsPollEvents ev = cio_events;
                 if (ev != ep_events && ep_events != ERTS_POLL_EV_NONE)
                     err = 1;
-                erts_printf("cio_ev=");
-                print_events(cio_events);
-                erts_printf(" ep_ev=");
-                print_events(ep_events);
+                erts_dsprintf(dsbufp, "cio_ev=");
+                print_events(dsbufp, cio_events);
+                erts_dsprintf(dsbufp, " ep_ev=");
+                print_events(dsbufp, ep_events);
             }
         } else {
-            erts_printf("cio_ev=");
-            print_events(cio_events);
-            erts_printf(" aio_ev=");
-            print_events(aio_events);
+            erts_dsprintf(dsbufp, "cio_ev=");
+            print_events(dsbufp, cio_events);
+            erts_dsprintf(dsbufp, " aio_ev=");
+            print_events(dsbufp, aio_events);
             if ((aio_events != ep_events && ep_events != ERTS_POLL_EV_NONE) ||
                 (aio_events != 0 && ep_events == ERTS_POLL_EV_NONE)) {
-                erts_printf(" ep_ev=");
-                print_events(ep_events);
+                erts_dsprintf(dsbufp, " ep_ev=");
+                print_events(dsbufp, ep_events);
                 err = 1;
             }
         }
 #else
-        if (print_events(cio_events) != 0)
+        if (print_events(dsbufp, cio_events) != 0)
             err = 1;
 #endif
-        erts_printf(" ");
+        erts_dsprintf(dsbufp, " ");
         if (cio_events & ERTS_POLL_EV_IN) {
             Eterm id = state->driver.select->inport;
             if (is_nil(id)) {
-                erts_printf("inport=none inname=none indrv=none ");
+                erts_dsprintf(dsbufp, "inport=none inname=none indrv=none ");
                 err = 1;
             }
             else {
                 ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT);
-                erts_printf(" inport=%T inname=%s indrv=%s ",
+                erts_dsprintf(dsbufp, " inport=%T inname=%s indrv=%s ",
                             id,
                             pnp->name ? pnp->name : "unknown",
                             (pnp->driver_name
@@ -2468,12 +2468,12 @@ static int erts_debug_print_checkio_state(ErtsDrvEventState *state,
         if (cio_events & ERTS_POLL_EV_OUT) {
             Eterm id = state->driver.select->outport;
             if (is_nil(id)) {
-                erts_printf("outport=none outname=none outdrv=none ");
+                erts_dsprintf(dsbufp, "outport=none outname=none outdrv=none ");
                 err = 1;
             }
             else {
                 ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT);
-                erts_printf(" outport=%T outname=%s outdrv=%s ",
+                erts_dsprintf(dsbufp, " outport=%T outname=%s outdrv=%s ",
                             id,
                             pnp->name ? pnp->name : "unknown",
                             (pnp->driver_name
@@ -2485,75 +2485,76 @@ static int erts_debug_print_checkio_state(ErtsDrvEventState *state,
     }
     else if (state->type == ERTS_EV_TYPE_NIF) {
         ErtsResource* r;
-        erts_printf("enif_select ");
+        erts_dsprintf(dsbufp, "enif_select ");
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
         if (internal) {
-            erts_printf("internal ");
+            erts_dsprintf(dsbufp, "internal ");
             err = 1;
         }
 
         if (cio_events == ep_events) {
-            erts_printf("ev=");
-            if (print_events(cio_events) != 0)
+            erts_dsprintf(dsbufp, "ev=");
+            if (print_events(dsbufp, cio_events) != 0)
                 err = 1;
         }
         else {
             err = 1;
-            erts_printf("cio_ev=");
-            print_events(cio_events);
-            erts_printf(" ep_ev=");
-            print_events(ep_events);
+            erts_dsprintf(dsbufp, "cio_ev=");
+            print_events(dsbufp, cio_events);
+            erts_dsprintf(dsbufp, " ep_ev=");
+            print_events(dsbufp, ep_events);
         }
 #else
-        if (print_events(cio_events) != 0)
+        if (print_events(dsbufp, cio_events) != 0)
             err = 1;
 #endif
-        erts_printf(" inpid=%T", state->driver.nif->in.pid);
-        erts_printf(" outpid=%T", state->driver.nif->out.pid);
+        erts_dsprintf(dsbufp, " inpid=%T", state->driver.nif->in.pid);
+        erts_dsprintf(dsbufp, " outpid=%T", state->driver.nif->out.pid);
         r = state->driver.stop.resource;
-        erts_printf(" resource=%p(%T:%T)", r, r->type->module, r->type->name);
+        erts_dsprintf(dsbufp, " resource=%p(%T:%T)", r, r->type->module, r->type->name);
     }
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     else if (internal) {
-        erts_printf("internal ");
+        erts_dsprintf(dsbufp, "internal ");
         if (cio_events) {
             err = 1;
-            erts_printf("cio_ev=");
-            print_events(cio_events);
+            erts_dsprintf(dsbufp, "cio_ev=");
+            print_events(dsbufp, cio_events);
         }
         if (ep_events) {
-            erts_printf("ep_ev=");
-            print_events(ep_events);
+            erts_dsprintf(dsbufp, "ep_ev=");
+            print_events(dsbufp, ep_events);
         }
     }
 #endif
     else {
         err = 1;
-        erts_printf("control_type=%d ", (int)state->type);
+        erts_dsprintf(dsbufp, "control_type=%d ", (int)state->type);
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
         if (cio_events == ep_events) {
-            erts_printf("ev=");
-            print_events(cio_events);
+            erts_dsprintf(dsbufp, "ev=");
+            print_events(dsbufp, cio_events);
         }
         else {
-            erts_printf("cio_ev="); print_events(cio_events);
-            erts_printf(" ep_ev="); print_events(ep_events);
+            erts_dsprintf(dsbufp, "cio_ev="); print_events(dsbufp, cio_events);
+            erts_dsprintf(dsbufp, " ep_ev="); print_events(dsbufp, ep_events);
         }
 #else
-        erts_printf("ev=0x%b32x", (Uint32) cio_events);
+        erts_dsprintf(dsbufp, "ev=0x%b32x", (Uint32) cio_events);
 #endif
     }
 
-    erts_printf(" flags="); print_flags(state->flags);
+    erts_dsprintf(dsbufp, " flags="); print_flags(dsbufp, state->flags);
     if (err) {
-        erts_printf(" ERROR");
+        erts_dsprintf(dsbufp, " ERROR");
     }
-    erts_printf("\r\n");
+    erts_dsprintf(dsbufp, "\r\n");
     return err;
 }
 
-static void doit_erts_check_io_debug(void *vstate, void *vcounters)
+static void doit_erts_check_io_debug(void *vstate, void *vcounters,
+                                     erts_dsprintf_buf_t *dsbufp)
 {
     ErtsDrvEventState *state = (ErtsDrvEventState *) vstate;
     IterDebugCounters *counters = (IterDebugCounters *) vcounters;
@@ -2587,7 +2588,7 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters)
     if (state->events) {
 	counters->used_fds++;
 #endif
-	if (erts_debug_print_checkio_state(state, ep_events, internal)) {
+	if (erts_debug_print_checkio_state(dsbufp, state, ep_events, internal)) {
 	    counters->num_errors++;
 	}
     }
@@ -2597,6 +2598,7 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters)
 int
 erts_check_io_debug(ErtsCheckIoDebugInfo *ciodip)
 {
+    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     int fd, len, i;
 #endif
@@ -2627,15 +2629,15 @@ erts_check_io_debug(ErtsCheckIoDebugInfo *ciodip)
     len = erts_atomic_read_nob(&drv_ev_state.len);
 
 #if ERTS_POLL_USE_FALLBACK
-    erts_printf("--- fds in flbk pollset ---------------------------------\n");
+    erts_dsprintf(dsbufp, "--- fds in flbk pollset ---------------------------------\n");
     erts_poll_get_selected_events_flbk(get_fallback(), counters.epep,
                                        drv_ev_state.max_fds);
     for (fd = 0; fd < len; fd++) {
         if (drv_ev_state.v[fd].flags & ERTS_EV_FLAG_FALLBACK)
-            doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters);
+            doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters, dsbufp);
     }
 #endif
-    erts_printf("--- fds in pollset --------------------------------------\n");
+    erts_dsprintf(dsbufp, "--- fds in pollset --------------------------------------\n");
 
     for (i = 0; i < erts_no_pollsets; i++) {
         erts_poll_get_selected_events(pollsetv[i],
@@ -2644,16 +2646,16 @@ erts_check_io_debug(ErtsCheckIoDebugInfo *ciodip)
         for (fd = 0; fd < len; fd++) {
             if (!(drv_ev_state.v[fd].flags & ERTS_EV_FLAG_FALLBACK)
                 && get_pollset_id(fd) == i)
-            doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters);
+                doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters, dsbufp);
         }
     }
     for (fd = len ; fd < drv_ev_state.max_fds; fd++) {
         null_des.fd = fd;
-        doit_erts_check_io_debug(&null_des, &counters);
+        doit_erts_check_io_debug(&null_des, &counters, dsbufp);
     }
 #else
     safe_hash_for_each(&drv_ev_state.tab, &doit_erts_check_io_debug,
-                       &counters);
+                       &counters, dsbufp);
 #endif
 
     if (ciodip)
@@ -2665,15 +2667,15 @@ erts_check_io_debug(ErtsCheckIoDebugInfo *ciodip)
         ciodip->no_enif_select_structs = counters.no_enif_select_structs;
     }
 
-    erts_printf("\n");
-    erts_printf("used fds=%d\n", counters.used_fds);
-    erts_printf("Number of driver_select() structures=%d\n", counters.no_driver_select_structs);
-    erts_printf("Number of enif_select() structures=%d\n", counters.no_enif_select_structs);
+    erts_dsprintf(dsbufp, "\n");
+    erts_dsprintf(dsbufp, "used fds=%d\n", counters.used_fds);
+    erts_dsprintf(dsbufp, "Number of driver_select() structures=%d\n", counters.no_driver_select_structs);
+    erts_dsprintf(dsbufp, "Number of enif_select() structures=%d\n", counters.no_enif_select_structs);
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    erts_printf("internal fds=%d\n", counters.internal_fds);
+    erts_dsprintf(dsbufp, "internal fds=%d\n", counters.internal_fds);
 #endif
-    erts_printf("---------------------------------------------------------\n");
-    fflush(stdout);
+    erts_dsprintf(dsbufp, "---------------------------------------------------------\n");
+    erts_send_error_to_logger_nogl(dsbufp);
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     erts_free(ERTS_ALC_T_TMP, (void *) counters.epep);
 #endif
-- 
cgit v1.2.3


From 4ac6ce823f96310259e7c5d1e8b242e548150ace Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 19 Jul 2017 14:34:44 +0200
Subject: erts: Add testcases to test IOp and IOt options

---
 erts/emulator/sys/common/erl_poll.c    |   8 ++
 erts/emulator/test/driver_SUITE.erl    | 173 +++++++++++++++++++++++++--------
 erts/emulator/test/scheduler_SUITE.erl |  99 +++++++++++++++++--
 3 files changed, 230 insertions(+), 50 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index c671ea99f4..fc32dbf370 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -1930,6 +1930,14 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet *ps, ErtsPollInfo *pip)
 #endif
         ;
 
+    pip->batch_updates =
+#if ERTS_POLL_USE_DEVPOLL
+        1
+#else
+        0
+#endif
+        ;
+
     pip->max_fds = max_fds;
 
     ERTS_POLLSET_UNLOCK(ps);
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 0e691b0c80..33d0b708cf 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -43,6 +43,7 @@
          outputv_errors/1,
          driver_unloaded/1,
          io_ready_exit/1,
+         use_fallback_pollset/0,
          use_fallback_pollset/1,
          bad_fd_in_pollset/1,
          fd_change/1,
@@ -118,17 +119,26 @@
 -define(heap_binary_size, 64).
 
 init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) ->
-    case catch erts_debug:get_internal_state(available_internal_state) of
-        true -> ok;
-        _ -> erts_debug:set_internal_state(available_internal_state, true)
-    end,
+    CIOD = rpc(Config,
+               fun() ->
+                       case catch erts_debug:get_internal_state(available_internal_state) of
+                           true -> ok;
+                           _ -> erts_debug:set_internal_state(available_internal_state, true)
+                       end,
+                       erts_debug:get_internal_state(check_io_debug)
+               end),
     erlang:display({init_per_testcase, Case}),
-    0 = element(1, erts_debug:get_internal_state(check_io_debug)),
+    0 = element(1, CIOD),
     [{testcase, Case}|Config].
 
-end_per_testcase(Case, _Config) ->
+end_per_testcase(Case, Config) ->
     erlang:display({end_per_testcase, Case}),
-    0 = element(1, erts_debug:get_internal_state(check_io_debug)),
+    CIOD = rpc(Config,
+               fun() ->
+                       get_stable_check_io_info(),
+                       erts_debug:get_internal_state(check_io_debug)
+               end),
+    0 = element(1, CIOD),
     ok.
 
 suite() ->
@@ -136,10 +146,13 @@ suite() ->
      {timetrap, {minutes, 1}}].
 
 all() -> %% Keep a_test first and z_test last...
-    [a_test, outputv_errors, outputv_echo, queue_echo, %{group, timer},
-     driver_unloaded, io_ready_exit, use_fallback_pollset,
-     bad_fd_in_pollset, fd_change,
-     steal_control, otp_6602, driver_system_info_base_ver,
+    [a_test, outputv_errors, outputv_echo, queue_echo,
+     {group, timer},
+     driver_unloaded, io_ready_exit, otp_6602,
+     {group, polling},
+     {group, poll_thread},
+     {group, poll_set},
+     driver_system_info_base_ver,
      driver_system_info_prev_ver,
      driver_system_info_current_ver, driver_monitor,
      {group, ioq_exit}, zero_extended_marker_garb_drv,
@@ -147,7 +160,6 @@ all() -> %% Keep a_test first and z_test last...
      larger_minor_vsn_drv, smaller_major_vsn_drv,
      smaller_minor_vsn_drv, peek_non_existing_queue,
      otp_6879, caller, many_events, missing_callbacks,
-     smp_select, driver_select_use,
      thread_mseg_alloc_cache_clean,
      otp_9302,
      thr_free_drv,
@@ -160,6 +172,13 @@ groups() ->
     [{timer, [],
       [timer_measure, timer_cancel, timer_delay,
        timer_change]},
+     {poll_thread, [], [{group, polling}]},
+     {poll_set, [], [{group, polling}]},
+     {polling, [],
+      [a_test, use_fallback_pollset,
+       bad_fd_in_pollset, fd_change,
+       steal_control, smp_select,
+       driver_select_use, z_test]},
      {ioq_exit, [],
       [ioq_exit_ready_input, ioq_exit_ready_output,
        ioq_exit_timeout, ioq_exit_ready_async,
@@ -172,10 +191,28 @@ init_per_suite(Config) ->
 end_per_suite(_Config) ->
     catch erts_debug:set_internal_state(available_internal_state, false).
 
+init_per_group(poll_thread, Config) ->
+    [{node_args, "+IOt 2"} | Config];
+init_per_group(poll_set, Config) ->
+    [{node_args, "+IOt 2 +IOp 2"} | Config];
+init_per_group(polling, Config) ->
+    case proplists:get_value(node_args, Config) of
+        undefined ->
+            Config;
+        Args ->
+            {ok, Node} = start_node(polling, Args),
+            [{node, Node} | Config]
+    end;
 init_per_group(_GroupName, Config) ->
     Config.
 
 end_per_group(_GroupName, Config) ->
+    case proplists:get_value(node, Config) of
+        undefined ->
+            ok;
+        Node ->
+            stop_node(Node)
+    end,
     Config.
 
 %% Test sending bad types to port with an outputv-capable driver.
@@ -783,7 +820,13 @@ io_ready_exit(Config) when is_list(Config) ->
 -define(CHKIO_SMP_SELECT, 7).
 -define(CHKIO_DRV_USE, 8).
 
+use_fallback_pollset() ->
+    [{timetrap, {minutes, 2}}].
+
 use_fallback_pollset(Config) when is_list(Config) ->
+    rpc(Config, fun() -> use_fallback_pollset_t(Config) end).
+
+use_fallback_pollset_t(Config) when is_list(Config) ->
     FlbkFun = fun () ->
                       {Flbk, _} = get_fallback(erlang:system_info(check_io)),
                       case lists:keysearch(total_poll_set_size, 1, Flbk) of
@@ -809,6 +852,7 @@ use_fallback_pollset(Config) when is_list(Config) ->
           Skip ->
               {fun () -> ok end, Skip, ok}
       end,
+    io:format("Node = ~p~n",[node()]),
     case chkio_test_fini(chkio_test(Handel,
                                     ?CHKIO_USE_FALLBACK_POLLSET,
                                     fun () ->
@@ -820,22 +864,31 @@ use_fallback_pollset(Config) when is_list(Config) ->
     end.
 
 bad_fd_in_pollset(Config) when is_list(Config) ->
-    chkio_test_fini(chkio_test(chkio_test_init(Config),
-                               ?CHKIO_BAD_FD_IN_POLLSET,
-                               fun () -> sleep(1000) end)).
+    rpc(Config,
+        fun() ->
+                chkio_test_fini(chkio_test(chkio_test_init(Config),
+                                           ?CHKIO_BAD_FD_IN_POLLSET,
+                                           fun () -> sleep(1000) end))
+        end).
 
 fd_change(Config) when is_list(Config) ->
-    chkio_test_fini(chkio_test(chkio_test_init(Config),
-                               ?CHKIO_FD_CHANGE,
-                               fun () -> sleep(1000) end)).
+    rpc(Config,
+        fun() ->
+                chkio_test_fini(chkio_test(chkio_test_init(Config),
+                                           ?CHKIO_FD_CHANGE,
+                                           fun () -> sleep(1000) end))
+        end).
 
 steal_control(Config) when is_list(Config) ->
-    chkio_test_fini(case chkio_test_init(Config) of
-                        {erts_poll_info, _} = Hndl ->
-                            steal_control_test(Hndl);
-                        Skip ->
-                            Skip
-                    end).
+    rpc(Config,
+        fun() ->
+                chkio_test_fini(case chkio_test_init(Config) of
+                                    {erts_poll_info, _} = Hndl ->
+                                        steal_control_test(Hndl);
+                                    Skip ->
+                                        Skip
+                                end)
+        end).
 
 steal_control_test(Hndl = {erts_poll_info, Before}) ->
     Port = open_chkio_port(),
@@ -877,7 +930,7 @@ chkio_test_init(Config) when is_list(Config) ->
     ChkIo = get_stable_check_io_info(),
     case catch lists:keysearch(name, 1, ChkIo) of
         {value, {name, erts_poll}} ->
-            io:format("Before test: ~p~n", [ChkIo]),
+            ct:log("Before test: ~p~n", [ChkIo]),
             Path = proplists:get_value(data_dir, Config),
             erl_ddll:start(),
             ok = load_driver(Path, 'chkio_drv'),
@@ -940,6 +993,7 @@ chkio_test({erts_poll_info, Before},
             Fun(),
             During = get_check_io_total(erlang:system_info(check_io)),
             erlang:display(During),
+
             0 = element(1, erts_debug:get_internal_state(check_io_debug)),
             io:format("During test: ~p~n", [During]),
             chk_chkio_port(Port),
@@ -991,25 +1045,25 @@ verify_chkio_state(Before, After) ->
     ok.
 
 get_stable_check_io_info() ->
+    get_stable_check_io_info(10).
+get_stable_check_io_info(0) ->
+    get_check_io_total(erlang:system_info(check_io));
+get_stable_check_io_info(N) ->
     ChkIo = get_check_io_total(erlang:system_info(check_io)),
-    PendUpdNo = case lists:keyfind(pending_updates, 1, ChkIo) of
-                    {pending_updates, Value} ->
-                        Value;
-                    false ->
-                        0
-                end,
-    {active_fds, ActFds} = lists:keyfind(active_fds, 1, ChkIo),
+    PendUpdNo = proplists:get_value(pending_updates, ChkIo, 0),
+    ActFds = proplists:get_value(active_fds, ChkIo),
     case {PendUpdNo, ActFds} of
         {0, 0} ->
             ChkIo;
         _ ->
-            receive after 10 -> ok end,
-            get_stable_check_io_info()
+            receive after 100 -> ok end,
+            get_stable_check_io_info(N-1)
     end.
 
 %% Merge return from erlang:system_info(check_io)
 %% as if it was one big pollset.
 get_check_io_total(ChkIo) ->
+    ct:log("ChkIo = ~p~n",[ChkIo]),
     {Fallback, Rest} = get_fallback(ChkIo),
     add_fallback_infos(Fallback,
       lists:foldl(fun(Pollset, Acc) ->
@@ -1064,7 +1118,8 @@ tag_type(pending_updates) -> sum;
 tag_type(batch_updates) -> const;
 tag_type(concurrent_updates) -> const;
 tag_type(max_fds) -> const;
-tag_type(active_fds) -> sum.
+tag_type(active_fds) -> sum;
+tag_type(poll_threads) -> sum.
 
 
 %% Missed port lock when stealing control of fd from a
@@ -1683,7 +1738,7 @@ missing_callbacks(Config) when is_list(Config) ->
 smp_select(Config) when is_list(Config) -> 
     case os:type() of
         {win32,_} -> {skipped, "Test not implemented for this OS"};
-        _ -> smp_select0(Config)
+        _ -> rpc(Config, fun() -> smp_select0(Config) end)
     end.
 
 smp_select0(Config) ->
@@ -1739,7 +1794,7 @@ smp_select_wait(Pids, TimeoutMsg) ->
 driver_select_use(Config) when is_list(Config) -> 
     case os:type() of
         {win32,_} -> {skipped, "Test not implemented for this OS"};
-        _ -> driver_select_use0(Config)
+        _ -> rpc(Config, fun() -> driver_select_use0(Config) end)
     end.
 
 driver_select_use0(Config) ->
@@ -2306,10 +2361,10 @@ count_proc_sched(Ps, PNs) ->
     end.
 
 a_test(Config) when is_list(Config) ->
-    check_io_debug().
+    rpc(Config, fun check_io_debug/0).
 
 z_test(Config) when is_list(Config) ->
-    check_io_debug().
+    rpc(Config, fun check_io_debug/0).
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% 		Utilities
@@ -2501,15 +2556,19 @@ sleep(Ms) when is_integer(Ms), Ms >= 0 ->
 
 
 start_node(Config) when is_list(Config) ->
+    start_node(proplists:get_value(testcase, Config));
+start_node(Name) ->
+    start_node(Name, "").
+start_node(NodeName, Args) ->
     Pa = filename:dirname(code:which(?MODULE)),
     Name = list_to_atom(atom_to_list(?MODULE)
                         ++ "-"
-                        ++ atom_to_list(proplists:get_value(testcase, Config))
+                        ++ atom_to_list(NodeName)
                         ++ "-"
                         ++ integer_to_list(erlang:system_time(second))
                         ++ "-"
                         ++ integer_to_list(erlang:unique_integer([positive]))),
-    test_server:start_node(Name, slave, [{args, "-pa "++Pa}]).
+    test_server:start_node(Name, slave, [{args, Args ++ " -pa "++Pa}]).
 
 stop_node(Node) ->
     test_server:stop_node(Node).
@@ -2542,3 +2601,35 @@ driver_alloc_size() ->
                       Sz0+Sz
               end, 0, CS)
     end.
+
+rpc(Config, Fun) ->
+    case proplists:get_value(node, Config) of
+        undefined ->
+            Fun();
+        Node ->
+            Self = self(),
+            Ref = make_ref(),
+            Pid = spawn(Node,
+                        fun() ->
+                                Result
+                                    = try Fun() of
+                                          Res -> Res
+                                      catch E:R ->
+                                              {'EXIT',E,R,erlang:get_stacktrace()}
+                                      end,
+                                Self ! {Ref, Result}
+                        end),
+            MRef = monitor(process, Pid),
+            receive
+                {'DOWN', MRef, _Type, _Object, Info} ->
+                    erlang:error({died, Pid, Info});
+                {Ref, {'EXIT',E,R,ST}} ->
+                    erlang:demonitor(MRef, [flush]),
+                    erlang:raise(E,R,ST);
+                {Ref, Ret} ->
+                    erlang:demonitor(MRef, [flush]),
+                    Ret;
+                Other ->
+                    ct:fail(Other)
+            end
+    end.
diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl
index 12e26671c2..7afb82a1b8 100644
--- a/erts/emulator/test/scheduler_SUITE.erl
+++ b/erts/emulator/test/scheduler_SUITE.erl
@@ -57,6 +57,7 @@
 	 scheduler_suspend_basic/1,
 	 scheduler_suspend/1,
 	 dirty_scheduler_threads/1,
+         poll_threads/1,
 	 reader_groups/1]).
 
 suite() ->
@@ -72,6 +73,7 @@ all() ->
      {group, scheduler_bind}, scheduler_threads,
      scheduler_suspend_basic, scheduler_suspend,
      dirty_scheduler_threads,
+     poll_threads,
      reader_groups].
 
 groups() -> 
@@ -1446,6 +1448,79 @@ sst5_loop(N) ->
     erlang:system_flag(multi_scheduling, unblock_normal),
     sst5_loop(N-1).
 
+poll_threads(Config) when is_list(Config) ->
+    {Conc, PollType, KP} = get_ioconfig(Config),
+    {Sched, SchedOnln, _} = get_sstate(Config, ""),
+
+    [1, 1] = get_ionum(Config,"+IOt 2 +IOp 2"),
+    [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 5"),
+
+    [1, 1] = get_ionum(Config, "+S 2 +IOPt 100 +IOPp 100"),
+
+    if
+        Conc ->
+            [5] = get_ionum(Config,"+IOt 5 +IOp 1"),
+            [3, 2] = get_ionum(Config,"+IOt 5 +IOp 2"),
+            [2, 2, 2, 2, 2] = get_ionum(Config,"+IOt 10 +IOPp 50"),
+
+            [2] = get_ionum(Config, "+S 2 +IOPt 100"),
+            [4] = get_ionum(Config, "+S 4 +IOPt 100"),
+            [4] = get_ionum(Config, "+S 4:2 +IOPt 100"),
+            [4, 4] = get_ionum(Config, "+S 8 +IOPt 100 +IOPp 25"),
+
+            fail = get_ionum(Config, "+IOt 1 +IOp 2"),
+
+            ok;
+        not Conc ->
+            [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 1"),
+            [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 2"),
+            [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 10 +IOPp 50"),
+
+            [1, 1] = get_ionum(Config, "+S 2 +IOPt 100"),
+            [1, 1, 1, 1] = get_ionum(Config, "+S 4 +IOPt 100"),
+            [1, 1, 1, 1] = get_ionum(Config, "+S 4:2 +IOPt 100"),
+            [1, 1, 1, 1, 1, 1, 1, 1] = get_ionum(Config, "+S 8 +IOPt 100 +IOPp 25"),
+
+            [1] = get_ionum(Config, "+IOt 1 +IOp 2"),
+
+            ok
+    end,
+
+    fail = get_ionum(Config, "+IOt 1 +IOPp 101"),
+    fail = get_ionum(Config, "+IOt 0"),
+    fail = get_ionum(Config, "+IOPt 101"),
+
+    ok.
+
+get_ioconfig(Config) ->
+    [PS | _] = get_iostate(Config, ""),
+    {proplists:get_value(concurrent_updates, PS),
+     proplists:get_value(primary, PS),
+     proplists:get_value(kernel_poll, PS)}.
+
+get_ionum(Config, Cmd) ->
+    case get_iostate(Config, Cmd) of
+        fail -> fail;
+        PSs ->
+            lists:reverse(
+              lists:sort(
+                [proplists:get_value(poll_threads, PS) || PS <- PSs]))
+    end.
+
+get_iostate(Config, Cmd)->
+    case start_node(Config, Cmd) of
+        {ok, Node} ->
+            [IOStates] = mcall(Node,[fun () ->
+                                             erlang:system_info(check_io)
+                                     end]),
+            IO = [IOState || IOState <- IOStates,
+                             proplists:get_value(fallback, IOState) == false],
+            stop_node(Node),
+            IO;
+        {error,timeout} ->
+            fail
+    end.
+
 reader_groups(Config) when is_list(Config) ->
     %% White box testing. These results are correct, but other results
     %% could be too...
@@ -1770,18 +1845,24 @@ mcall(Node, Funs) ->
     Parent = self(),
     Refs = lists:map(fun (Fun) ->
                              Ref = make_ref(),
-                             spawn_link(Node,
-                                        fun () ->
-                                                Res = Fun(),
-                                                unlink(Parent),
-                                                Parent ! {Ref, Res}
-                                        end),
-                             Ref
+                             Pid = spawn(Node,
+                                         fun () ->
+                                                 Res = Fun(),
+                                                 unlink(Parent),
+                                                 Parent ! {Ref, Res}
+                                         end),
+                             MRef = erlang:monitor(process, Pid),
+                             {Ref, MRef}
                      end, Funs),
-    lists:map(fun (Ref) ->
+    lists:map(fun ({Ref, MRef}) ->
                       receive
                           {Ref, Res} ->
-                              Res
+                              receive
+                                  {'DOWN',MRef,_,_,_} ->
+                                      Res
+                              end;
+                          {'DOWN',MRef,_,_,Reason} ->
+                              Reason
                       end
               end, Refs).
 
-- 
cgit v1.2.3


From 4989ca90cac9ee7782c470ce14e4843225db5db4 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 31 Aug 2017 12:59:18 +0200
Subject: erts: Fix msacc testcase with new poll-thread

---
 erts/emulator/sys/win32/erl_poll.c      |  6 +++---
 erts/emulator/test/statistics_SUITE.erl | 10 ++++++++++
 2 files changed, 13 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index 24c8c7bbc7..fd4c745c3b 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -1052,15 +1052,15 @@ int erts_poll_wait(ErtsPollSet *ps,
     if (!erts_atomic32_read_nob(&break_waiter_state)) {
 	HANDLE harr[2] = {ps->event_io_ready, break_happened_event};
 	int num_h = 2;
-        ERTS_MSACC_PUSH_STATE_M();
+        ERTS_MSACC_PUSH_STATE();
 
 	HARDDEBUGF(("Start waiting %d [%d]",num_h, (int) timeout));
 	ERTS_POLLSET_UNLOCK(ps);
 	erts_thr_progress_prepare_wait(NULL);
-        ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
+        ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP);
 	WaitForMultipleObjects(num_h, harr, FALSE, timeout);
 	erts_thr_progress_finalize_wait(NULL);
-        ERTS_MSACC_POP_STATE_M();
+        ERTS_MSACC_POP_STATE();
 	ERTS_POLLSET_LOCK(ps);
 	HARDDEBUGF(("Stop waiting %d [%d]",num_h, (int) timeout));
 	woke_up(ps);
diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl
index 7a396d273c..029a6de897 100644
--- a/erts/emulator/test/statistics_SUITE.erl
+++ b/erts/emulator/test/statistics_SUITE.erl
@@ -674,6 +674,16 @@ msacc_test(TmpFile) ->
     ets:insert(Tid, {1, hello}),
     ets:delete(Tid),
 
+    %% Check some IO
+    {ok, L} = gen_tcp:listen(0, [{active, true},{reuseaddr,true}]),
+    {ok, Port} = inet:port(L),
+    Pid = spawn(fun() ->
+                        {ok, S} = gen_tcp:accept(L),
+                        (fun F() -> receive M -> F() end end)()
+                end),
+    {ok, C} = gen_tcp:connect("localhost", Port, []),
+    [begin gen_tcp:send(C,"hello"),timer:sleep(1) end || _ <- lists:seq(1,100)],
+
     %% Collect some garbage
     [erlang:garbage_collect() || _ <- lists:seq(1,100)],
 
-- 
cgit v1.2.3


From e93043fa1a8dc9dd524ea8df54ad157324f091da Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 31 Aug 2017 12:27:04 +0200
Subject: erts: disable kernel-poll on OS X vsn < 16

kqueue is broken on earlier versions of OS X.
---
 erts/emulator/sys/common/erl_poll.c | 78 +++++++++++++++++++++++++++++++------
 erts/emulator/sys/common/erl_poll.h |  6 +--
 2 files changed, 69 insertions(+), 15 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index fc32dbf370..30a595c17a 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -186,6 +186,8 @@ int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds,
 
 #define ERTS_POLL_USE_CONCURRENT_UPDATE (ERTS_POLL_USE_EPOLL || ERTS_POLL_USE_KQUEUE)
 
+#define ERTS_POLL_USE_WAKEUP_PIPE (!ERTS_POLL_USE_CONCURRENT_UPDATE)
+
 #if !ERTS_POLL_USE_CONCURRENT_UPDATE
 
 #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) \
@@ -279,9 +281,12 @@ struct ERTS_POLL_EXPORT(erts_pollset) {
     ErtsPollSetUpdateRequestsBlock *curr_upd_req_block;
     erts_atomic32_t have_update_requests;
     erts_mtx_t mtx;
-    int wake_fds[2];
     erts_atomic32_t wakeup_state;
 #endif
+
+#if ERTS_POLL_USE_WAKEUP_PIPE
+    int wake_fds[2];
+#endif
 };
 
 void erts_silence_warn_unused_result(long unused);
@@ -397,11 +402,12 @@ woke_up(ErtsPollSet *ps)
  * --- Wakeup pipe -----------------------------------------------------------
  */
 
-#if !ERTS_POLL_USE_CONCURRENT_UPDATE
+#if ERTS_POLL_USE_WAKEUP_PIPE
 
 static ERTS_INLINE void
 wake_poller(ErtsPollSet *ps, int interrupted)
 {
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     int wake;
     erts_aint32_t wakeup_state;
     if (!interrupted)
@@ -413,7 +419,9 @@ wake_poller(ErtsPollSet *ps, int interrupted)
                                                ERTS_POLL_WOKEN_INTR);
     wake = wakeup_state == ERTS_POLL_NOT_WOKEN;
 
-    if (wake) {
+    if (wake)
+#endif
+    {
 	ssize_t res;
 	if (ps->wake_fds[1] < 0)
 	    return; /* Not initialized yet */
@@ -452,8 +460,10 @@ cleanup_wakeup_pipe(ErtsPollSet *ps)
 		    fd,
 		    erl_errno_id(errno), errno);
     }
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     if (intr)
 	erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_INTR);
+#endif
 }
 
 static void
@@ -489,11 +499,14 @@ create_wakeup_pipe(ErtsPollSet *ps)
     ps->wake_fds[1] = wake_fds[1];
 }
 
+#endif
 
 /*
  * --- Poll set update requests ----------------------------------------------
  */
 
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
+
 static ERTS_INLINE void
 enqueue_update_request(ErtsPollSet *ps, int fd)
 {
@@ -893,9 +906,9 @@ update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events)
 #endif
             if (op == ERTS_POLL_OP_ADD)
                 erts_atomic_dec_nob(&ps->no_of_user_fds);
-            return ERTS_POLL_EV_NVAL;
-        }
-        return 0;
+            events = ERTS_POLL_EV_NVAL;
+        } else
+            events = 0;
     }
     return events;
 }
@@ -1216,7 +1229,7 @@ poll_control(ErtsPollSet *ps, int fd, ErtsPollOp op,
 	    goto done;
 	}
 #endif
-#if !ERTS_POLL_USE_CONCURRENT_UPDATE
+#if ERTS_POLL_USE_WAKEUP_PIPE
 	if (fd == ps->wake_fds[0] || fd == ps->wake_fds[1]) {
 	    new_events = ERTS_POLL_EV_NVAL;
 	    goto done;
@@ -1294,10 +1307,10 @@ ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet *ps,
 static ERTS_INLINE int
 ERTS_POLL_EXPORT(save_result)(ErtsPollSet *ps, ErtsPollResFd pr[], int max_res, int chk_fds_res, int ebadf)
 {
-#if !ERTS_POLL_USE_CONCURRENT_UPDATE || ERTS_POLL_DEBUG_PRINT
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE || ERTS_POLL_DEBUG_PRINT || ERTS_POLL_USE_WAKEUP_PIPE
     int n = chk_fds_res < max_res ? chk_fds_res : max_res, i;
     int res = n;
-#if !ERTS_POLL_USE_CONCURRENT_UPDATE
+#if ERTS_POLL_USE_WAKEUP_PIPE
     int wake_fd = ps->wake_fds[0];
 #endif
 
@@ -1318,12 +1331,16 @@ ERTS_POLL_EXPORT(save_result)(ErtsPollSet *ps, ErtsPollResFd pr[], int max_res,
 #endif
             );
 
-#if !ERTS_POLL_USE_CONCURRENT_UPDATE
-
+#if ERTS_POLL_USE_WAKEUP_PIPE
         if (fd == wake_fd) {
             cleanup_wakeup_pipe(ps);
             ERTS_POLL_RES_SET_EVTS(&pr[i], ERTS_POLL_EV_NONE);
-        } else {
+            if (n == 1)
+                return 0;
+        }
+#endif
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
+        else {
             /* Reset the events to emulate ONESHOT semantics */
             ps->fds_status[fd].events = 0;
             enqueue_update_request(ps, fd);
@@ -1819,6 +1836,43 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(int id)
     create_wakeup_pipe(ps);
     handle_update_requests(ps, NULL, 0);
     cleanup_wakeup_pipe(ps);
+#endif
+#if ERTS_POLL_USE_KERNEL_POLL && (defined(__DARWIN__) || defined(__APPLE__) && defined(__MACH__))
+    {
+        /*
+         * Using kqueue on OS X is a mess of brokenness...
+         *
+         * On OS X version older than 15.6 (i.e. OS X El Capitan released in July 2015),
+         * a thread waiting in kevent is not woken if an event is inserted into the kqueue
+         * by another thread and the event becomes ready. However if a new call to kevent
+         * is done by the waiting thread, the new event is found.
+         *
+         * So on effected OS X versions we could trigger the wakeup pipe so that
+         * the waiters will be woken and re-issue the kevent. However...
+         *
+         * On OS X version older then 16 (i.e. OS X Sierra released in September 2016),
+         * running the emulator driver_SUITE smp_select testcase consistently causes a
+         * kernel panic. I don't know why or what events that trigger it. But it seems
+         * like updates of the pollset while another thread is sleeping in it Creates
+         * some kind of race that triggers the kernel panic.
+         *
+         * So to deal with this, the erts configure check what OS X version is run
+         * and only enabled kernel poll on OS X 16 or newer. In addition, if someone
+         * attempts to compile Erlang on OS X 16 and then run it on OS X 15, we do the
+         * run-time check below to disallow this.
+         */
+        int major, minor, build;
+        os_version(&major,&minor,&build);
+        if (major < 16) {
+            erts_fprintf(stderr,"BROKEN KQUEUE!\n"
+                         "Erlang has been compiled with kernel-poll support,\n"
+                         "but this OS X version is known to have kernel bugs\n"
+                         "when using kernel-poll. You have two options:\n"
+                         " 1) update to a newer OS X version (OS X Sierra or newer)\n"
+                         " 2) recompile erlang without kernel-poll support\n");
+            erts_exit(1, "");
+        }
+    }
 #endif
     erts_atomic_set_nob(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */
 
diff --git a/erts/emulator/sys/common/erl_poll.h b/erts/emulator/sys/common/erl_poll.h
index a1e2709a9f..e9a667cac1 100644
--- a/erts/emulator/sys/common/erl_poll.h
+++ b/erts/emulator/sys/common/erl_poll.h
@@ -134,9 +134,9 @@
 typedef Uint32 ErtsPollEvents;
 
 typedef enum {
-    ERTS_POLL_OP_ADD = 0, /* Add the FD to the pollset */
-    ERTS_POLL_OP_MOD = 1, /* Modify the FD in the pollset */
-    ERTS_POLL_OP_DEL = 2  /* Delete the FD from the pollset */
+    ERTS_POLL_OP_ADD = 0,          /* Add the FD to the pollset */
+    ERTS_POLL_OP_MOD = 1,          /* Modify the FD in the pollset */
+    ERTS_POLL_OP_DEL = 2           /* Delete the FD from the pollset */
 } ErtsPollOp;
 
 #define op2str(op) (op == ERTS_POLL_OP_ADD ? "add" :            \
-- 
cgit v1.2.3


From 340aeafc960eb2d1785d3211a149079255429396 Mon Sep 17 00:00:00 2001
From: Dmytro Lytovchenko <dmytro.lytovchenko@gmail.com>
Date: Wed, 12 Jul 2017 17:26:14 +0200
Subject: erts: enif_select steal test

---
 erts/emulator/test/nif_SUITE.erl              | 83 +++++++++++++++++++++++++--
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 50 ++++++++++++++++
 2 files changed, 127 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 223bd7d586..bec2291867 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -25,13 +25,14 @@
 %%-define(CHECK(Exp,Got), Exp = Got).
 
 -include_lib("common_test/include/ct.hrl").
+-include_lib("stdlib/include/assert.hrl").
 
 -export([all/0, suite/0, groups/0,
          init_per_group/2, end_per_group/2,
 	 init_per_testcase/2, end_per_testcase/2,
          basic/1, reload_error/1, upgrade/1, heap_frag/1,
          t_on_load/1,
-         select/1,
+         select/1, select_steal/1,
          monitor_process_a/1,
          monitor_process_b/1,
          monitor_process_c/1,
@@ -42,9 +43,9 @@
 	 types/1, many_args/1, binaries/1, get_string/1, get_atom/1,
 	 maps/1,
 	 api_macros/1,
-	 from_array/1, iolist_as_binary/1, resource/1, resource_binary/1, 
+	 from_array/1, iolist_as_binary/1, resource/1, resource_binary/1,
 	 resource_takeover/1,
-	 threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1, 
+	 threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1,
 	 is_checks/1,
 	 get_length/1, make_atom/1, make_string/1, reverse_list_test/1,
 	 otp_9828/1,
@@ -79,7 +80,7 @@ all() ->
     [{group, G} || G <- api_groups()]
         ++
     [reload_error, heap_frag, types, many_args,
-     select,
+     select, select_steal,
      {group, monitor},
      monitor_frenzy,
      hipe,
@@ -144,7 +145,8 @@ init_per_testcase(nif_whereis_threaded, Config) ->
         true -> Config;
         false -> {skip, "No thread support"}
     end;
-init_per_testcase(select, Config) ->
+init_per_testcase(Select, Config) when Select =:= select;
+                                       Select =:= select_steal ->
     case os:type() of
         {win32,_} ->
             {skip, "Test not yet implemented for windows"};
@@ -152,6 +154,9 @@ init_per_testcase(select, Config) ->
             Config
     end;
 init_per_testcase(_Case, Config) ->
+    %% Clear any resource dtor data before test starts in case another tc
+    %% left it in a bad state
+    catch last_resource_dtor_call(),
     Config.
 
 end_per_testcase(t_on_load, _Config) ->
@@ -590,7 +595,71 @@ select_3(_Config) ->
     {_,_,2} = last_resource_dtor_call(),
     ok.
 
-check_stop_ret(?ERL_NIF_SELECT_STOP_CALLED) -> ok;    
+%% @doc The stealing child process for the select_steal test. Duplicates given
+%% W/RFds and runs select on them to steal
+select_steal_child_process(Parent, RFd) ->
+    %% Duplicate the resource with the same FD
+    {R2Fd, _R2Ptr} = dupe_resource_nif(RFd),
+    Ref2 = make_ref(),
+
+    %% Try to select from the child pid (steal from parent)
+    ?assertEqual(0, select_nif(R2Fd, ?ERL_NIF_SELECT_READ, R2Fd, null, Ref2)),
+    ?assertEqual([], flush(0)),
+    ?assertEqual(eagain, read_nif(R2Fd, 1)),
+
+    %% Check that now events arrive to this temporary process
+    Parent ! {self(), stage1}, % signal parent to send the <<"stolen1">>
+
+    %% Receive <<"stolen1">> via enif_select
+    ?assertEqual(0, select_nif(R2Fd, ?ERL_NIF_SELECT_READ, R2Fd, null, Ref2)),
+    ?assertMatch([{select, R2Fd, Ref2, ready_input}], flush()),
+    ?assertEqual(<<"stolen1">>, read_nif(R2Fd, 7)),
+
+    clear_select_nif(R2Fd),
+
+    % do not do this here - stop_selecting(R2Fd, R2Rsrc, Ref2),
+    Parent ! {self(), done}.
+
+%% @doc Similar to select/1 test, make a double ended pipe. Then try to steal
+%% the socket, see what happens.
+select_steal(Config) when is_list(Config) ->
+    ensure_lib_loaded(Config),
+
+    Ref = make_ref(),
+    {{RFd, RPtr}, {WFd, WPtr}} = pipe_nif(),
+
+    %% Bind the socket to current pid in enif_select
+    ?assertEqual(0, select_nif(RFd, ?ERL_NIF_SELECT_READ, RFd, null, Ref)),
+    ?assertEqual([], flush(0)),
+
+    %% Spawn a process and do some stealing
+    Parent = self(),
+    Pid = spawn_link(fun() -> select_steal_child_process(Parent, RFd) end),
+
+    %% Signal from the child to send the first message
+    {Pid, stage1} = receive_any(),
+    ?assertEqual(ok, write_nif(WFd, <<"stolen1">>)),
+
+    ?assertMatch([{Pid, done}], flush(1)),  % synchronize with the child
+
+    %% Try to select from the parent pid (steal back)
+    ?assertEqual(0, select_nif(RFd, ?ERL_NIF_SELECT_READ, RFd, Pid, Ref)),
+
+    %% Ensure that no data is hanging and close.
+    %% Rfd is stolen at this point.
+    check_stop_ret(select_nif(WFd, ?ERL_NIF_SELECT_STOP, WFd, null, Ref)),
+    ?assertMatch([{fd_resource_stop, WPtr, _}], flush()),
+    {1, {WPtr, 1}} = last_fd_stop_call(),
+
+    check_stop_ret(select_nif(RFd, ?ERL_NIF_SELECT_STOP, RFd, null, Ref)),
+    ?assertMatch([{fd_resource_stop, RPtr, _}], flush()),
+    {1, {RPtr, 1}} = last_fd_stop_call(),
+
+    ?assert(is_closed_nif(WFd)),
+
+    ok.
+
+check_stop_ret(?ERL_NIF_SELECT_STOP_CALLED) -> ok;
 check_stop_ret(?ERL_NIF_SELECT_STOP_SCHEDULED) -> ok.
 
 write_full(W, C) ->
@@ -3193,10 +3262,12 @@ binary_to_term_nif(_, _, _) -> ?nif_stub.
 port_command_nif(_, _) -> ?nif_stub.
 format_term_nif(_,_) -> ?nif_stub.
 select_nif(_,_,_,_,_) -> ?nif_stub.
+dupe_resource_nif(_) -> ?nif_stub.
 pipe_nif() -> ?nif_stub.
 write_nif(_,_) -> ?nif_stub.
 read_nif(_,_) -> ?nif_stub.
 is_closed_nif(_) -> ?nif_stub.
+clear_select_nif(_) -> ?nif_stub.
 last_fd_stop_call() -> ?nif_stub.
 alloc_monitor_resource_nif() -> ?nif_stub.
 monitor_process_nif(_,_,_,_) -> ?nif_stub.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index b47d013bd2..79560a38aa 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -2448,6 +2448,13 @@ static int get_fd(ErlNifEnv* env, ERL_NIF_TERM term, struct fd_resource** rsrc)
     return 1;
 }
 
+/* Returns: badarg
+ *    Or an enif_select result, which is a combination of bits:
+ *    ERL_NIF_SELECT_STOP_CALLED = 1
+ *    ERL_NIF_SELECT_STOP_SCHEDULED = 2
+ *    ERL_NIF_SELECT_INVALID_EVENT = 4
+ *    ERL_NIF_SELECT_FAILED = 8
+ */
 static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     struct fd_resource* fdr;
@@ -2479,6 +2486,9 @@ static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
 }
 
 #ifndef __WIN32__
+/*
+ * Create a read-write pipe with two fds (to read and to write)
+ */
 static ERL_NIF_TERM pipe_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     struct fd_resource* read_rsrc;
@@ -2514,6 +2524,30 @@ static ERL_NIF_TERM pipe_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]
                enif_make_tuple2(env, write_fd, make_pointer(env, write_rsrc)));
 }
 
+/*
+ * Create (dupe) of a resource with the same fd, to test stealing
+ */
+static ERL_NIF_TERM dupe_resource_nif(ErlNifEnv* env, int argc,
+                                      const ERL_NIF_TERM argv[]) {
+    struct fd_resource* orig_rsrc;
+
+    if (!get_fd(env, argv[0], &orig_rsrc)) {
+        return enif_make_badarg(env);
+    } else {
+        struct fd_resource* new_rsrc;
+        ERL_NIF_TERM new_fd;
+
+        new_rsrc = enif_alloc_resource(fd_resource_type,
+                                       sizeof(struct fd_resource));
+        new_rsrc->fd = orig_rsrc->fd;
+        new_rsrc->was_selected = 0;
+        new_fd = enif_make_resource(env, new_rsrc);
+        enif_release_resource(new_rsrc);
+
+        return enif_make_tuple2(env, new_fd, make_pointer(env, new_rsrc));
+    }
+}
+
 static ERL_NIF_TERM write_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     struct fd_resource* fdr;
@@ -2589,6 +2623,20 @@ static ERL_NIF_TERM is_closed_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM a
 
     return fdr->fd < 0 ? atom_true : atom_false;
 }
+
+static ERL_NIF_TERM clear_select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    struct fd_resource* fdr = NULL;
+
+    if (!get_fd(env, argv[0], &fdr))
+        return enif_make_badarg(env);
+
+    fdr->fd = -1;
+    fdr->was_selected = 0;
+
+    return atom_ok;
+}
+
 #endif /* !__WIN32__ */
 
 
@@ -3476,8 +3524,10 @@ static ErlNifFunc nif_funcs[] =
 #ifndef __WIN32__
     {"pipe_nif", 0, pipe_nif},
     {"write_nif", 2, write_nif},
+    {"dupe_resource_nif", 1, dupe_resource_nif},
     {"read_nif", 2, read_nif},
     {"is_closed_nif", 1, is_closed_nif},
+    {"clear_select_nif", 1, clear_select_nif},
 #endif
     {"last_fd_stop_call", 0, last_fd_stop_call},
     {"alloc_monitor_resource_nif", 0, alloc_monitor_resource_nif},
-- 
cgit v1.2.3


From 0c6df1f92be6337efe4d7e3d1964063f9acf770f Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 25 Sep 2017 18:25:29 +0200
Subject: erts: Trigger ready events when erts_io_control fails

---
 erts/emulator/sys/common/erl_check_io.c | 62 +++++++++++++++++++++++----------
 1 file changed, 43 insertions(+), 19 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 4019b78074..f93d4c1557 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -298,6 +298,8 @@ static ERTS_INLINE void
 check_fd_cleanup(ErtsDrvEventState *state,
 		 ErtsDrvSelectDataState **free_select,
                  ErtsNifSelectDataState **free_nif);
+static ERTS_INLINE void iready(Eterm id, ErtsDrvEventState *state);
+static ERTS_INLINE void oready(Eterm id, ErtsDrvEventState *state);
 #ifdef DEBUG_PRINT_MODE
 static char *drvmode2str(int mode);
 static char *nifmode2str(enum ErlNifSelectFlags mode);
@@ -460,8 +462,18 @@ erts_io_notify_port_task_executed(ErtsPortTaskType type,
     if (active_events) {
         /* This is not needed if active_events has not changed */
         if (state->active_events != active_events) {
+            ErtsPollEvents new_events;
             state->active_events = active_events;
-            erts_io_control(state, ERTS_POLL_OP_MOD, active_events);
+            new_events = erts_io_control(state, ERTS_POLL_OP_MOD, active_events);
+
+            /* We were unable to re-insert the fd into the pollset, signal the callback. */
+            if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
+                if (active_events & ERTS_POLL_EV_IN)
+                    iready(state->driver.select->inport, state);
+                if (active_events & ERTS_POLL_EV_OUT)
+                    oready(state->driver.select->outport, state);
+                state->active_events = 0;
+            }
         }
     } else {
         check_fd_cleanup(state, &free_select, &free_nif);
@@ -564,7 +576,17 @@ deselect(ErtsDrvEventState *state, int mode)
 	state->type = ERTS_EV_TYPE_NONE;
 	state->flags = 0;
     } else {
-        erts_io_control(state, ERTS_POLL_OP_MOD, state->active_events);
+        ErtsPollEvents new_events =
+            erts_io_control(state, ERTS_POLL_OP_MOD, state->active_events);
+
+        /* We were unable to re-insert the fd into the pollset, signal the callback. */
+        if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
+            if (state->active_events & ERTS_POLL_EV_IN)
+                iready(state->driver.select->inport, state);
+            if (state->active_events & ERTS_POLL_EV_OUT)
+                oready(state->driver.select->outport, state);
+            state->active_events = 0;
+        }
     }
 }
 
@@ -619,6 +641,7 @@ driver_select(ErlDrvPort ix, ErlDrvEvent e, int mode, int on)
     ErtsSysFdType fd = (ErtsSysFdType) e;
     ErtsPollEvents ctl_events = (ErtsPollEvents) 0;
     ErtsPollEvents old_events;
+    ErtsPollEvents new_events;
     ErtsPollOp ctl_op = ERTS_POLL_OP_MOD;
     ErtsDrvEventState *state;
     int wake_poller = 0;
@@ -745,23 +768,11 @@ driver_select(ErlDrvPort ix, ErlDrvEvent e, int mode, int on)
     }
 
     if (ctl_events || ctl_op == ERTS_POLL_OP_DEL) {
-        ErtsPollEvents new_events;
 
         new_events = erts_io_control_wakeup(state, ctl_op,
                                             state->active_events,
                                             &wake_poller);
 
-        if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
-            if (state->type == ERTS_EV_TYPE_DRV_SEL && !old_events) {
-                state->type = ERTS_EV_TYPE_NONE;
-                state->flags = 0;
-                state->driver.select->inport = NIL;
-                state->driver.select->outport = NIL;
-            }
-            ret = -1;
-            goto done;
-        }
-
         ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL || state->type == ERTS_EV_TYPE_NONE);
     }
 
@@ -772,13 +783,18 @@ driver_select(ErlDrvPort ix, ErlDrvEvent e, int mode, int on)
 	    if (state->type == ERTS_EV_TYPE_NONE)
 		state->type = ERTS_EV_TYPE_DRV_SEL;
 	    ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL);
-	    if (ctl_events & ERTS_POLL_EV_IN)
+	    if (ctl_events & ERTS_POLL_EV_IN) {
 		state->driver.select->inport = id;
-	    if (ctl_events & ERTS_POLL_EV_OUT)
+                if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL))
+                    iready(id, state);
+            }
+	    if (ctl_events & ERTS_POLL_EV_OUT) {
 		state->driver.select->outport = id;
-	    if (mode & ERL_DRV_USE) {
+                if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL))
+                    oready(id, state);
+            }
+	    if (mode & ERL_DRV_USE)
 		state->flags |= ERTS_EV_FLAG_USED;
-	    }
         }
     }
     else { /* off */
@@ -1598,8 +1614,16 @@ erts_check_io(ErtsPollThread *psi)
 
             /* Reactivate the poll op if there are still active events */
             if (state->active_events) {
+                ErtsPollEvents new_events;
                 DEBUG_PRINT_FD("re-enable %s", state, ev2str(state->active_events));
-                erts_io_control(state, ERTS_POLL_OP_MOD, state->active_events);
+
+                new_events = erts_io_control(state, ERTS_POLL_OP_MOD, state->active_events);
+
+                /* Unable to re-enable the fd, signal all callbacks */
+                if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
+                    revents |= state->active_events;
+                    state->active_events = 0;
+                }
             }
         }
 
-- 
cgit v1.2.3


From 01e5510805ebd8b8cff18ecafadf13dc3af07f47 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 29 Sep 2017 19:31:32 +0200
Subject: Add distribution_SUITE:bad_dist_ext_size

---
 erts/emulator/test/distribution_SUITE.erl | 67 +++++++++++++++++++++++++++++--
 1 file changed, 63 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index 6994bfef83..3a637b5682 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -56,6 +56,7 @@
          bad_dist_ext_process_info/1,
          bad_dist_ext_control/1,
          bad_dist_ext_connection_id/1,
+         bad_dist_ext_size/1,
 	 start_epmd_false/1, epmd_module/1]).
 
 %% Internal exports.
@@ -92,6 +93,7 @@ groups() ->
       [dist_auto_connect_never, dist_auto_connect_once]},
      {bad_dist_ext, [],
       [bad_dist_ext_receive, bad_dist_ext_process_info,
+       bad_dist_ext_size,
        bad_dist_ext_control, bad_dist_ext_connection_id]}].
 
 %% Tests pinging a node in different ways.
@@ -1683,6 +1685,57 @@ bad_dist_ext_connection_id(Config) when is_list(Config) ->
     stop_node(Offender),
     stop_node(Victim).
 
+%% OTP-14661: Bad message is discovered by erts_msg_attached_data_size
+bad_dist_ext_size(Config) when is_list(Config) ->
+    {ok, Offender} = start_node(bad_dist_ext_process_info_offender),
+    %%Prog = "Prog=/home/uabseri/src/otp_new3/bin/cerl -rr -debug",
+    Prog = [],
+    {ok, Victim} = start_node(bad_dist_ext_process_info_victim, [], Prog),
+    start_node_monitors([Offender,Victim]),
+
+    Parent = self(),
+    P = spawn_link(Victim,
+                   fun () ->
+                           Parent ! {self(), started},
+                           receive check_msgs -> ok end,  %% DID CRASH HERE
+                           bad_dist_ext_check_msgs([one]),
+                           Parent ! {self(), messages_checked}
+                   end),
+
+    receive {P, started} -> ok end,
+    P ! one,
+
+    Suspended = make_ref(),
+    S = spawn(Victim,
+              fun () ->
+                      erlang:suspend_process(P),
+                      Parent ! Suspended,
+                      receive after infinity -> ok end
+              end),
+
+    receive Suspended -> ok end,
+    pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    verify_up(Offender, Victim),
+    send_bad_msgs(Offender, P, 1, dmsg_bad_tag()),
+
+    %% Make sure bad msgs has reached Victim
+    rpc:call(Offender, rpc, call, [Victim, erlang, node, []]),
+
+    verify_still_up(Offender, Victim),
+
+    rpc:call(Victim, erlang, process_info, [P, total_heap_size]),
+
+    verify_down(Offender, connection_closed, Victim, killed),
+
+    P ! check_msgs,
+    exit(S, bang),  % resume Victim
+    receive {P, messages_checked} -> ok end,
+
+    unlink(P),
+    verify_no_down(Offender, Victim),
+    stop_node(Offender),
+    stop_node(Victim).
+
 
 bad_dist_struct_check_msgs([]) ->
     receive
@@ -1786,9 +1839,12 @@ send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) ->
 send_bad_msg(BadNode, To) ->
     send_bad_msgs(BadNode, To, 1).
 
-send_bad_msgs(BadNode, To, Repeat) when is_atom(BadNode),
-                                        is_pid(To),
-                                        is_integer(Repeat) ->
+send_bad_msgs(BadNode, To, Repeat) ->
+    send_bad_msgs(BadNode, To, Repeat, dmsg_bad_atom_cache_ref()).
+
+send_bad_msgs(BadNode, To, Repeat, BadTerm) when is_atom(BadNode),
+                                                 is_pid(To),
+                                                 is_integer(Repeat) ->
     Parent = self(),
     Done = make_ref(),
     spawn_link(BadNode,
@@ -1798,7 +1854,7 @@ send_bad_msgs(BadNode, To, Repeat) when is_atom(BadNode),
                        DPrt = dport(Node),
                        DData = [dmsg_hdr(),
                                 dmsg_ext({?DOP_SEND, ?COOKIE, To}),
-                                dmsg_bad_atom_cache_ref()],
+                                BadTerm],
                        repeat(fun () -> port_command(DPrt, DData) end, Repeat),
                        Parent ! Done
                end),
@@ -1885,6 +1941,9 @@ dmsg_ext(Term) ->
 dmsg_bad_atom_cache_ref() ->
     [$R, 137].
 
+dmsg_bad_tag() ->  %% Will fail early at heap size calculation
+    [$?, 66].
+
 start_epmd_false(Config) when is_list(Config) ->
     %% Start a node with the option -start_epmd false.
     {ok, OtherNode} = start_node(start_epmd_false, "-start_epmd false"),
-- 
cgit v1.2.3


From 0ee0af9d6114054cb38fce2377678682181b6788 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 29 Sep 2017 19:32:18 +0200
Subject: erts: Fix bug when detecting bad dist message

We can't just leave it in queue with dist_ext=NULL.

Two symptoms seen:
1. 'receive' trying to deref dist_ext as NULL.
2. GC think it's a term and put THE_NON_VALUE in root set.
---
 erts/emulator/beam/erl_message.c | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index cdd771ef7d..de9f16d088 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -564,14 +564,11 @@ erts_msg_attached_data_size_aux(ErtsMessage *msg)
 
     sz = erts_decode_dist_ext_size(msg->data.dist_ext);
     if (sz < 0) {
-	/* Bad external; remove it */
-	if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) {
-	    ErlHeapFragment *heap_frag;
-	    heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
-	    erts_cleanup_offheap(&heap_frag->off_heap);
-	}
-	erts_free_dist_ext_copy(msg->data.dist_ext);
-	msg->data.dist_ext = NULL;
+	/* Bad external
+	 * We leave the message intact in this case as it's not worth the trouble
+	 * to make all callers remove it from queue. It will be detected again
+	 * and removed from message queue later anyway.
+	 */
 	return 0;
     }
 
-- 
cgit v1.2.3


From 4ab349bd590403edb42bb8972f881b5921bf257f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 5 Oct 2017 10:50:11 +0200
Subject: macros.tab: Fix assertion in SET_I_REL()

9a50a5d5fc1 changed the update of I, but forgot to update
the preceding assertion.
---
 erts/emulator/beam/macros.tab | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab
index 0d175a7ec6..e0b5f56b53 100644
--- a/erts/emulator/beam/macros.tab
+++ b/erts/emulator/beam/macros.tab
@@ -38,7 +38,7 @@ REFRESH_GEN_DEST() {
 // zero, except in a few bit syntax instructions.)
 
 SET_I_REL(Offset) {
-    ASSERT(VALID_INSTR(*(I + ($Offset))));
+    ASSERT(VALID_INSTR(*(I + ($Offset) + $IP_ADJUSTMENT)));
     I += $Offset + $IP_ADJUSTMENT;
 }
 
-- 
cgit v1.2.3


From 6c8c122c487f1a1417dd2a674c4da7e6dd52739c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 4 Oct 2017 13:22:41 +0200
Subject: ops.tab: Slightly optimize badmatch on a Y register

---
 erts/emulator/beam/ops.tab | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 75ff40606b..b7ef1f0599 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1139,6 +1139,9 @@ bs_test_unit8 f x
 # An y register operand for bs_context_to_binary is rare,
 # but can happen because of inlining.
 
+bs_context_to_binary Y=y | line L | badmatch Y => \
+    move Y x | bs_context_to_binary x | line L | badmatch x
+
 bs_context_to_binary Y=y => move Y x | bs_context_to_binary x
 
 bs_context_to_binary x
-- 
cgit v1.2.3


From 15099d5752d99587b24e00335713af9d82f16c34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 5 Oct 2017 12:37:11 +0200
Subject: beam_disasm: Correct printing of y registers

---
 erts/emulator/beam/beam_debug.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 0f332da63f..d8303a651a 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -744,7 +744,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 		    erts_print(to, to_arg, " x(%d)", loader_x_reg_index(ap[0]));
 		    break;
 		case LOADER_Y_REG:
-		    erts_print(to, to_arg, " x(%d)", loader_y_reg_index(ap[0]));
+		    erts_print(to, to_arg, " y(%d)", loader_y_reg_index(ap[0]) - CP_SIZE);
 		    break;
 		default:
 		    erts_print(to, to_arg, " %T", (Eterm) ap[0]);
@@ -765,7 +765,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
                     erts_print(to, to_arg, " x(%d)", loader_x_reg_index(ap[0]));
                     break;
                 case LOADER_Y_REG:
-		    erts_print(to, to_arg, " x(%d)", loader_y_reg_index(ap[0]));
+		    erts_print(to, to_arg, " y(%d)", loader_y_reg_index(ap[0]) - CP_SIZE);
 		    break;
                 default:
 		    erts_print(to, to_arg, " %T", (Eterm) ap[0]);
@@ -788,7 +788,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 			erts_print(to, to_arg, " x(%d)", loader_x_reg_index(ap[0]));
 			break;
 		    case LOADER_Y_REG:
-			erts_print(to, to_arg, " y(%d)", loader_y_reg_index(ap[0]));
+			erts_print(to, to_arg, " y(%d)", loader_y_reg_index(ap[0]) - CP_SIZE);
 			break;
 		    default:
 			erts_print(to, to_arg, " %T", (Eterm) ap[0]);
-- 
cgit v1.2.3


From e1c9772ffdacae4007209ac5a82758b8e0d3cec4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 3 Oct 2017 10:11:18 +0200
Subject: Move LD flags for hipe from Makefile.in to configure.in

We want the flags to be available for other tests in configure.in.
---
 erts/emulator/Makefile.in | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 915cad4e18..85ca145d9f 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -240,16 +240,7 @@ ARCH=@ARCH@
 ultrasparcCFLAGS=-Wa,-xarch=v8plusa
 ARCHCFLAGS=$($(ARCH)CFLAGS)
 
-ifdef HIPE_ENABLED
-ifeq ($(OPSYS),linux)
-ppcBEAMLDFLAGS=-Wl,-m,elf32ppc
-ppc64BEAMLDFLAGS=-Wl,-m,elf64ppc,-T,hipe/elf64ppc.x
-endif
-ifeq ($(OPSYS),darwin)
-amd64BEAMLDFLAGS=-pagezero_size 0x10000000
-endif
-HIPEBEAMLDFLAGS=$($(ARCH)BEAMLDFLAGS)
-endif
+HIPEBEAMLDFLAGS=@HIPEBEAMLDFLAGS@
 
 ERTS_BUILD_FALLBACK_POLL=@ERTS_BUILD_FALLBACK_POLL@
 
-- 
cgit v1.2.3


From 22d2a00aebf0eef878af95d8b7598adbfca06e7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 19 Sep 2017 06:33:06 +0200
Subject: Use 32-bits pointers to C code

On a 64-bit machine, we only need 32 bits to store a pointer to
the C code that implements a BEAM instruction. Refactor the code
to only use the lower 32 bits of each instruction word, and take
care to preserve the high 32 bits.
---
 erts/emulator/beam/beam_bp.c  | 19 +++++++++++++------
 erts/emulator/beam/beam_emu.c | 30 +++++++++++++++++++++---------
 erts/emulator/beam/erl_vm.h   | 11 ++++++++++-
 3 files changed, 44 insertions(+), 16 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index c81380c14d..871670e8c3 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -419,9 +419,11 @@ erts_install_breakpoints(BpFunctions* f)
 
     for (i = 0; i < n; i++) {
 	ErtsCodeInfo* ci = f->matching[i].ci;
-        BeamInstr *pc = erts_codeinfo_to_code(ci);
 	GenericBp* g = ci->u.gen_bp;
-	if (*pc != br && g) {
+        BeamInstr volatile *pc = erts_codeinfo_to_code(ci);
+        BeamInstr instr = *pc;
+
+	if (!BeamIsOpCode(instr, op_i_generic_breakpoint) && g) {
 	    Module* modp = f->matching[i].mod;
 
 	    /*
@@ -435,11 +437,16 @@ erts_install_breakpoints(BpFunctions* f)
 	    /*
 	     * The following write is not protected by any lock. We
 	     * assume that the hardware guarantees that a write of an
-	     * aligned word-size (or half-word) writes is atomic
-	     * (i.e. that other processes executing this code will not
-	     * see a half pointer).
+	     * aligned word-size writes is atomic (i.e. that other
+	     * processes executing this code will not see a half
+	     * pointer).
+             *
+             * The contents of *pc is marked 'volatile' to ensure that
+             * the compiler will do a single full-word write, and not
+             * try any fancy optimizations to write a half word.
 	     */
-	    *pc = br;
+            instr = BeamSetCodeAddr(instr, br);
+            *pc = instr;
 	    modp->curr.num_breakpoints++;
 	}
     }
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 9a77c63390..6a45087a34 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -50,14 +50,14 @@
 #if defined(NO_JUMP_TABLE)
 #  define OpCase(OpCode)    case op_##OpCode
 #  define CountCase(OpCode) case op_count_##OpCode
-#  define IsOpCode(InstrWord, OpCode)  ((InstrWord) == (BeamInstr)op_##OpCode)
-#  define Goto(Rel) {Go = (Rel); goto emulator_loop;}
+#  define IsOpCode(InstrWord, OpCode)  (BeamCodeAddr(InstrWord) == (BeamInstr)op_##OpCode)
+#  define Goto(Rel)         {Go = BeamCodeAddr(Rel); goto emulator_loop;}
 #else
 #  define OpCase(OpCode)    lb_##OpCode
 #  define CountCase(OpCode) lb_count_##OpCode
-#  define IsOpCode(InstrWord, OpCode)  ((InstrWord) == (BeamInstr)&&lb_##OpCode)
-#  define Goto(Rel) goto *((void *)Rel)
-#  define LabelAddr(Label) &&Label
+#  define IsOpCode(InstrWord, OpCode)  (BeamCodeAddr(InstrWord) == (BeamInstr)&&lb_##OpCode)
+#  define Goto(Rel)         goto *((void *)BeamCodeAddr(Rel))
+#  define LabelAddr(Label)  &&Label
 #endif
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -131,11 +131,11 @@ do {                                     \
 
 /* We don't check the range if an ordinary switch is used */
 #ifdef NO_JUMP_TABLE
-#define VALID_INSTR(IP) ((UWord)(IP) < (NUMBER_OF_OPCODES*2+10))
+#  define VALID_INSTR(IP) (BeamCodeAddr(IP) < (NUMBER_OF_OPCODES*2+10))
 #else
-#define VALID_INSTR(IP) \
-   ((SWord)LabelAddr(emulator_loop) <= (SWord)(IP) && \
-    (SWord)(IP) < (SWord)LabelAddr(end_emulator_loop))
+#  define VALID_INSTR(IP) \
+    ((BeamInstr)LabelAddr(emulator_loop) <= BeamCodeAddr(IP) && \
+     BeamCodeAddr(IP) < (BeamInstr)LabelAddr(end_emulator_loop))
 #endif /* NO_JUMP_TABLE */
 
 #define SET_CP(p, ip)           \
@@ -1006,6 +1006,18 @@ init_emulator_finish(void)
      int i;
      Export* ep;
 
+#if defined(ARCH_64) && defined(CODE_MODEL_SMALL)
+     for (i = 0; i < NUMBER_OF_OPCODES; i++) {
+         BeamInstr instr = BeamOpCodeAddr(i);
+         if (instr >= (1ull << 32)) {
+             erts_exit(ERTS_ERROR_EXIT,
+                       "This run-time was supposed be compiled with all code below 2Gb,\n"
+                       "but the instruction '%s' is located at %016lx.\n",
+                       opc[i].name, instr);
+         }
+     }
+#endif
+
      beam_apply[0]             = BeamOpCodeAddr(op_i_apply);
      beam_apply[1]             = BeamOpCodeAddr(op_normal_exit);
      beam_exit[0]              = BeamOpCodeAddr(op_error_action_code);
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 661538eadd..76980b5871 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -209,6 +209,15 @@ extern void** beam_ops;
 #  define BeamOpCodeAddr(OpCode) ((BeamInstr)beam_ops[(OpCode)])
 #endif
 
-#define BeamIsOpCode(InstrWord, OpCode) ((InstrWord) == BeamOpCodeAddr(OpCode))
+#if defined(ARCH_64) && defined(CODE_MODEL_SMALL)
+#  define BeamCodeAddr(InstrWord) ((BeamInstr)(Uint32)(InstrWord))
+#  define BeamSetCodeAddr(InstrWord, Addr) (((InstrWord) & ~((1ull << 32)-1)) | (Addr))
+#  define BeamExtraData(InstrWord) ((InstrWord) >> 32)
+#else
+#  define BeamCodeAddr(InstrWord) ((BeamInstr)(InstrWord))
+#  define BeamSetCodeAddr(InstrWord, Addr) (Addr)
+#endif
+
+#define BeamIsOpCode(InstrWord, OpCode) (BeamCodeAddr(InstrWord) == BeamOpCodeAddr(OpCode))
 
 #endif	/* __ERL_VM_H__ */
-- 
cgit v1.2.3


From f88bd45a2c76f84a16b004922945579898cc35ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 19 Sep 2017 11:56:45 +0200
Subject: Pack operands into the instruction word

On 64-bit machines where the C code is always at address below 4Gb,
pack one or more operands into the instruction word.
---
 erts/emulator/Makefile.in        |   1 +
 erts/emulator/beam/beam_debug.c  |  38 +++--
 erts/emulator/beam/beam_emu.c    |   7 +-
 erts/emulator/beam/beam_load.c   |  43 +++++-
 erts/emulator/utils/beam_makeops | 304 +++++++++++++++++++++++++++------------
 5 files changed, 283 insertions(+), 110 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 85ca145d9f..54d45dce50 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -569,6 +569,7 @@ $(TTF_DIR)/beam_tr_funcs.h \
 $(TTF_DIR)/OPCODES-GENERATED: $(OPCODE_TABLES) utils/beam_makeops
 	$(gen_verbose)LANG=C $(PERL) utils/beam_makeops \
 		-wordsize @EXTERNAL_WORD_SIZE@ \
+		-code-model @CODE_MODEL@ \
 		-outdir $(TTF_DIR) \
 		-DUSE_VM_PROBES=$(if $(USE_VM_PROBES),1,0) \
 		-DNO_FPE_SIGNALS=$(if $filter(unreliable,$(FPE)),1,0) \
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index d8303a651a..70078c8c59 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -201,7 +201,7 @@ void debug_dump_code(BeamInstr *I, int num)
 	for (i = 0; i < NUM_SPECIFIC_OPS; i++) {
 	    if (BeamIsOpCode(instr, i) && opc[i].name[0] != '\0') {
 		code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp,
-				     i, opc[i].sz-1, code_ptr+1) + 1;
+				     i, opc[i].sz-1, code_ptr) + 1;
 		break;
 	    }
 	}
@@ -321,7 +321,7 @@ erts_debug_disassemble_1(BIF_ALIST_1)
     for (i = 0; i < NUM_SPECIFIC_OPS; i++) {
 	if (BeamIsOpCode(instr, i) && opc[i].name[0] != '\0') {
 	    code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp,
-				 i, opc[i].sz-1, code_ptr+1) + 1;
+				 i, opc[i].sz-1, code_ptr) + 1;
 	    break;
 	}
     }
@@ -405,8 +405,11 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	 * Avoid copying because instructions containing bignum operands
 	 * are bigger than actually declared.
 	 */
-	ap = (BeamInstr *) addr;
+        addr++;
+        ap = addr;
     } else {
+        BeamInstr instr_word = addr++[0];
+
 	/*
 	 * Copy all arguments to a local buffer for the unpacking.
 	 */
@@ -431,23 +434,22 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	    case 'q':
 		*ap++ = *--sp;
 		break;
-	    case 'i':		/* Initialize packing accumulator. */
-		*ap++ = packed;
-		break;
-	    case 's':
-		*ap++ = packed & 0x3ff;
-		packed >>= 10;
+#ifdef ARCH_64
+	    case '1':		/* Tightest shift */
+		*ap++ = (packed & BEAM_TIGHTEST_MASK) << 3;
+		packed >>= BEAM_TIGHTEST_SHIFT;
 		break;
-	    case '0':		/* Tight shift */
+#endif
+	    case '2':		/* Tight shift */
 		*ap++ = packed & BEAM_TIGHT_MASK;
 		packed >>= BEAM_TIGHT_SHIFT;
 		break;
-	    case '6':		/* Shift 16 steps */
+	    case '3':		/* Loose shift */
 		*ap++ = packed & BEAM_LOOSE_MASK;
 		packed >>= BEAM_LOOSE_SHIFT;
 		break;
 #ifdef ARCH_64
-	    case 'w':		/* Shift 32 steps */
+	    case '4':		/* Shift 32 steps */
 		*ap++ = packed & BEAM_WIDE_MASK;
 		packed >>= BEAM_WIDE_SHIFT;
 		break;
@@ -458,8 +460,18 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	    case 'P':
 		packed = *--sp;
 		break;
+#if defined(ARCH_64) && defined(CODE_MODEL_SMALL)
+            case '#':       /* -1 */
+            case '$':       /* -2 */
+            case '%':       /* -3 */
+            case '&':       /* -4 */
+            case '\'':      /* -5 */
+            case '(':       /* -6 */
+                packed = (packed << BEAM_WIDE_SHIFT) | BeamExtraData(instr_word);
+		break;
+#endif
 	    default:
-		ASSERT(0);
+                erts_exit(ERTS_ERROR_EXIT, "beam_debug: invalid packing op: %c\n", *prog);
 	    }
 	}
 	ap = args;
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 6a45087a34..73c4e3532b 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -234,15 +234,18 @@ void** beam_ops;
 #define fb(N) ((Sint)(Sint32)(N))
 #define jb(N) ((Sint)(Sint32)(N))
 #define tb(N) (N)
-#define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N)))
-#define yb(N) (*(Eterm *) (((unsigned char *)E) + (N)))
+#define xb(N) (*ADD_BYTE_OFFSET(reg, N))
+#define yb(N) (*ADD_BYTE_OFFSET(E, N))
 #define Sb(N) (*REG_TARGET_PTR(N))
 #define lb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N)))
 #define Qb(N) (N)
 #define Ib(N) (N)
+
 #define x(N) reg[N]
 #define y(N) E[N]
 #define r(N) x(N)
+#define Q(N) (N*sizeof(Eterm *))
+#define l(N) (freg[N].fd)
 
 /*
  * Check that we haven't used the reductions and jump to function pointed to by
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 9835b1c096..00dd28b26c 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2578,23 +2578,31 @@ load_code(LoaderState* stp)
                         sp++;
                     }
                     break;
-		case 'i':	/* Initialize packing accumulator. */
-		    packed = code[--ci];
+#ifdef ARCH_64
+		case '1':	/* Tightest shift (always 10 bits) */
+                    ci--;
+                    ASSERT((code[ci] & ~0x1FF8ull) == 0); /* Fits in 10 bits */
+                    packed = (packed << BEAM_TIGHTEST_SHIFT);
+                    packed |= code[ci] >> 3;
+                    if (packed_label) {
+                        packed_label->packed++;
+                    }
 		    break;
-		case '0':	/* Tight shift */
+#endif
+		case '2':	/* Tight shift (10 or 16 bits) */
 		    packed = (packed << BEAM_TIGHT_SHIFT) | code[--ci];
                     if (packed_label) {
                         packed_label->packed++;
                     }
 		    break;
-		case '6':	/* Shift 16 steps */
+		case '3':	/* Loose shift (16 bits) */
 		    packed = (packed << BEAM_LOOSE_SHIFT) | code[--ci];
                     if (packed_label) {
                         packed_label->packed++;
                     }
 		    break;
 #ifdef ARCH_64
-		case 'w':	/* Shift 32 steps */
+		case '4':	/* Wide shift (32 bits) */
                     {
                         Uint w = code[--ci];
 
@@ -2646,8 +2654,31 @@ load_code(LoaderState* stp)
                     sp++;
 		    packed = 0;
 		    break;
+#if defined(ARCH_64) && defined(CODE_MODEL_SMALL)
+                case '#':       /* -1 */
+                case '$':       /* -2 */
+                case '%':       /* -3 */
+                case '&':       /* -4 */
+                case '\'':      /* -5 */
+                case '(':       /* -6 */
+                    /* Pack accumulator contents into instruction word. */
+                    {
+                        Sint pos = ci - (*prog - '#' + 1);
+                        /* Are the high 32 bits of the instruction word zero? */
+                        ASSERT((code[pos] & ~((1ull << BEAM_WIDE_SHIFT)-1)) == 0);
+                        code[pos] |= packed << BEAM_WIDE_SHIFT;
+                        if (packed_label) {
+                            ASSERT(packed_label->packed == 1);
+                            packed_label->pos = pos;
+                            packed_label->packed = 2;
+                            packed_label = 0;
+                        }
+                        packed >>= BEAM_WIDE_SHIFT;
+                    }
+		    break;
+#endif
 		default:
-		    ASSERT(0);
+                    erts_exit(ERTS_ERROR_EXIT, "beam_load: invalid packing op: %c\n", *prog);
 		}
 	    }
 	    ASSERT(sp == stack); /* Incorrect program? */
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 64a9a49ac8..388e3d164d 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -24,6 +24,16 @@ use constant COLD => 0;
 use constant WARM => 1;
 use constant HOT => 2;
 
+# Instructions for packing
+use constant PACK_JUMP => 1;
+use constant PACK_IN_INSTR_WORD => 2;
+
+# Packing commands
+use constant PACK_CMD_TIGHTEST => '1';
+use constant PACK_CMD_TIGHT => '2';
+use constant PACK_CMD_LOOSE => '3';
+use constant PACK_CMD_WIDE => '4';
+
 $BEAM_FORMAT_NUMBER = undef;
 
 my $target = \&emulator_output;
@@ -32,30 +42,15 @@ my $verbose = 0;
 my $hotness = 1;
 my $num_file_opcodes = 0;
 my $wordsize = 32;
-my %defs;			# Defines (from command line).
+my $code_pointers_are_short = 0; # Whether code pointers (to C code) are short.
+my $code_model = 'unknown';
+my %defs;                        # Defines (from command line).
 
 # This is shift counts and mask for the packer.
 my $WHOLE_WORD = '';
-my @pack_instr;
-my @pack_shift;
-my @pack_mask;
-
-$pack_instr[2] = ['6', 'i'];
-$pack_instr[3] = ['0', '0', 'i'];
-$pack_instr[4] = ['6', '6', '6', 'i']; # Only for 64 bit wordsize
-
-$pack_shift[2] = ['0', 'BEAM_LOOSE_SHIFT'];
-$pack_shift[3] = ['0', 'BEAM_TIGHT_SHIFT', '(2*BEAM_TIGHT_SHIFT)'];
-$pack_shift[4] = ['0', 'BEAM_LOOSE_SHIFT', # Only for 64 bit wordsize
-		  '(2*BEAM_LOOSE_SHIFT)',
-		  '(3*BEAM_LOOSE_SHIFT)'];
-
-$pack_mask[2]  = ['BEAM_LOOSE_MASK', $WHOLE_WORD];
-$pack_mask[3]  = ['BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK'];
-$pack_mask[4]  = ['BEAM_LOOSE_MASK', # Only for 64 bit wordsize
-		  'BEAM_LOOSE_MASK',
-		  'BEAM_LOOSE_MASK',
-		  $WHOLE_WORD];
+
+my @basic_pack_options = (0);
+my @extended_pack_options = @basic_pack_options;
 
 # There are two types of instructions: generic and specific.
 # The generic instructions are those generated by the Beam compiler.
@@ -250,6 +245,7 @@ while (@ARGV && $ARGV[0] =~ /^-(.*)/) {
     ($target = \&compiler_output), next if /^compiler/;
     ($outdir = shift), next if /^outdir/;
     ($wordsize = shift), next if /^wordsize/;
+    ($code_model = shift), next if /^code-model/;
     ($verbose = 1), next if /^v/;
     ($defs{$1} = $2), next if /^D(\w+)=(\w+)/;
     die "$0: Bad option: -$_\n";
@@ -261,14 +257,21 @@ if ($wordsize == 32) {
 } elsif ($wordsize == 64) {
     $defs{'ARCH_32'} = 0;
     $defs{'ARCH_64'} = 1;
+    $code_pointers_are_short = $code_model eq 'small';
 }
 
 #
-# Initialize number of arguments per packed word.
+# Initialize pack options.
 #
 
 if ($wordsize == 64) {
-    $pack_mask[3] = ['BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK', $WHOLE_WORD];
+    @basic_pack_options = (0,PACK_JUMP);
+    @extended_pack_options = @basic_pack_options;
+    if ($code_pointers_are_short) {
+        foreach (@basic_pack_options) {
+            push @extended_pack_options, $_ | PACK_IN_INSTR_WORD;
+        }
+    }
 }
 
 #
@@ -732,12 +735,19 @@ sub emulator_output {
 	print "#if !defined(ARCH_64)\n";
 	print qq[  #error "64-bit architecture assumed, but ARCH_64 not defined"\n];
 	print "#endif\n";
+        if ($code_pointers_are_short) {
+            print "#if !defined(CODE_MODEL_SMALL)\n";
+            print qq[  #error "small code model assumed, but CODE_MODEL_SMALL not defined"\n];
+            print "#endif\n";
+        }
 	print "#define BEAM_WIDE_MASK 0xFFFFFFFFull\n";
 	print "#define BEAM_LOOSE_MASK 0xFFFFull\n";
 	print "#define BEAM_TIGHT_MASK 0xFFFFull\n";
+	print "#define BEAM_TIGHTEST_MASK 0x3FFull\n";
 	print "#define BEAM_WIDE_SHIFT 32\n";
 	print "#define BEAM_LOOSE_SHIFT 16\n";
 	print "#define BEAM_TIGHT_SHIFT 16\n";
+	print "#define BEAM_TIGHTEST_SHIFT 10\n";
     }
     print "\n";
 
@@ -1138,7 +1148,7 @@ sub combine_instruction_group {
                 foreach (0..$#c_args) {
                     push @first, shift @rest;
                 }
-                my $size = cg_combined_size($s, 1, @first);
+                my $size = cg_combined_size($s, @first);
                 $offsets{$s} = $offset
                     unless defined $offsets{$s} and $offsets{$s} < $offset;
                 $offset += $size - 1;
@@ -1229,7 +1239,7 @@ sub combine_instruction_group {
             }
 
             my($gen_code,$down,$up) =
-                cg_combined_code($s, 1, $flags, $offset,
+                cg_combined_code($s, $flags, $offset,
                                  $group_size-$offset, $inc, @first);
             my $spec_label = "$opcase$label";
             $down{$spec_label} = $down;
@@ -1280,7 +1290,7 @@ sub micro_label {
 
 sub cg_basic {
     my($name,@args) = @_;
-    my($size,$code,$pack_spec) = code_gen($name, 1, '', 0, undef, undef, @args);
+    my($size,$code,$pack_spec) = code_gen($name, \@extended_pack_options, '', 0, undef, undef, @args);
     $pack_spec = build_pack_spec($pack_spec);
     ($size,$code,$pack_spec);
 }
@@ -1290,8 +1300,8 @@ sub cg_basic {
 #
 
 sub cg_combined_size {
-    my($name,$pack,@args) = @_;
-    my($size) = code_gen($name, $pack, '', 0, undef, undef, @args);
+    my($name,@args) = @_;
+    my($size) = code_gen($name, \@basic_pack_options, '', 0, undef, undef, @args);
     $size;
 }
 
@@ -1300,8 +1310,9 @@ sub cg_combined_size {
 #
 
 sub cg_combined_code {
-    my($name,$pack,$extra_comments,$offset,$comp_size,$inc,@args) = @_;
-    my($size,$code,$pack_spec) = code_gen(@_);
+    my($name,$extra,$offset,$comp_size,$inc,@args) = @_;
+    my($size,$code,$pack_spec) =
+        code_gen($name, \@basic_pack_options, $extra, $offset, $comp_size, $inc, @args);
     if ($pack_spec eq '') {
         ($code,'','');
     } else {
@@ -1311,7 +1322,7 @@ sub cg_combined_code {
 }
 
 sub code_gen {
-    my($name,$pack,$extra_comments,$offset,$comp_size,$inc,@args) = @_;
+    my($name,$pack_options,$extra_comments,$offset,$comp_size,$inc,@args) = @_;
     my $group_size = defined $comp_size ? $comp_size + $inc : undef;
     my $size = 0;
     my $flags = '';
@@ -1326,8 +1337,8 @@ sub code_gen {
     #
 
     my $c_code_ref = $c_code{$name};
-    if ($pack and defined $c_code_ref and $name ne 'catch') {
-        ($var_decls, $pack_spec, @args) = do_pack($offset, @args);
+    if (defined $c_code_ref and $name ne 'catch') {
+        ($var_decls, $pack_spec, @args) = do_pack($offset, $pack_options, @args);
     }
 
     #
@@ -1603,9 +1614,25 @@ sub needs_do_wrapper {
 }
 
 sub do_pack {
-    my($offset,@args) = @_;
+    my($offset,$pack_options,@args) = @_;
+    my $ret = ['', ':', @args];
+    my $score = 0;
+
+    foreach my $options (@$pack_options) {
+        my($this_score,$this_result) = do_pack_one($options, $offset, @args);
+        if ($this_score > $score) {
+            $ret = $this_result;
+            $score = $this_score;
+        }
+    }
+    return @$ret;
+}
+
+sub do_pack_one {
+    my($options,$offset,@args) = @_;
     my($packable_args) = 0;
     my @bits_needed;            # Bits needed for each argument.
+    my $pack_in_iw = $options & PACK_IN_INSTR_WORD;
 
     #
     # Define the minimum number of bits needed for the packable argument types.
@@ -1619,6 +1646,10 @@ sub do_pack {
                        't' => 16);
     if ($wordsize == 64) {
         $bits_needed{'I'} = 32;
+        if ($options & PACK_JUMP) {
+            $bits_needed{'f'} = 32;
+            $bits_needed{'j'} = 32;
+        }
     }
 
     #
@@ -1631,51 +1662,46 @@ sub do_pack {
 	} else {
 	    push @bits_needed, 0;
 	}
-    }
-
-    #
-    # Try to pack 'f' and 'j', but not at expense at worse packing
-    # for other operands. For example, given the arguments "f x x", we
-    # want the 'x' operands to be packed, not 'f' and 'x' packed and
-    # the final 'x' not packed.
-    #
-
-    if ($wordsize == 64 and $packable_args == 1) {
-        for (my $i = 0; $i < @args; $i++) {
-            if ($args[$i] =~ /^[fj]$/) {
-                $bits_needed[$i] = 32;
-                $packable_args++;
-                last;
-            }
+        if ($arg =~ /^[fj]$/) {
+            # Only pack the first occurrence of 'f' or 'j'.
+            delete $bits_needed{'f'};
+            delete $bits_needed{'j'};
         }
     }
 
     #
-    # Nothing to pack unless there are at least 2 packable arguments.
+    # Return if there is nothing to pack.
     #
-    return ('', ':', @args) if $packable_args < 2;
+    if ($packable_args == 0) {
+        return (-1);
+    } elsif ($packable_args == 1 and !$pack_in_iw) {
+        return (-1);
+    }
 
     #
     # Determine how many arguments we should pack into each word.
     #
     my @args_per_word;
     my @need_wide_mask;
-    my $bits = 0;
-    my $word = 0;
-    $args_per_word[0] = 0;
-    $need_wide_mask[0] = 0;
+    my $bits;
+    my $this_wordsize;
+    my $word = -1;
+
+    my $next_word = sub {
+        $word++;
+        $args_per_word[$word] = 0;
+        $need_wide_mask[$word] = 0;
+        $bits = 0;
+        $this_wordsize = $wordsize;
+    };
+
+    $next_word->();
+    $this_wordsize = 32 if $pack_in_iw;
     for (my $i = 0; $i < @args; $i++) {
-        if ($bits_needed[$i]) {
             my $needed = $bits_needed[$i];
+            next unless $needed;
 
-            my $next_word = sub {
-                $word++;
-                $args_per_word[$word] = 0;
-                $need_wide_mask[$word] = 0;
-                $bits = 0;
-            };
-
-            if ($bits+$needed > $wordsize) { # Does not fit.
+            if ($bits+$needed > $this_wordsize) { # Does not fit.
                 $next_word->();
             }
             if ($args_per_word[$word] == 4) { # Can't handle more than 4 args.
@@ -1695,15 +1721,16 @@ sub do_pack {
                 # Can only pack two things in a word where one
                 # item is 32 bits. Force the next item into
                 # the next word.
-                $bits = $wordsize;
+                $bits = $this_wordsize;
             }
-        }
     }
 
     #
     # Try to balance packing between words.
     #
-    if ($args_per_word[$#args_per_word] == 1) {
+    if (@args_per_word == 1 and $args_per_word[0] == 1 and $pack_in_iw) {
+        # Don't rebalance.
+    } elsif ($args_per_word[$#args_per_word] == 1) {
         if ($args_per_word[$#args_per_word-1] < 3) {
             pop @args_per_word;
         } else {
@@ -1753,43 +1780,51 @@ sub do_pack {
     # the packing engine works from right-to-left, but we must generate
     # the instructions from left-to-right because we must calculate
     # instruction sizes from left-to-right.
-
     my $arg_num = 0;
     for (my $word = 0; $word < @args_per_word; $word++) {
         my $ap = 0;              # Argument number within word.
         my $packed_var = "tmp_packed" . ($word+1);
         my $args_per_word = $args_per_word[$word];
-        my @shift;
-        my @mask;
-        my @instr;
-
-        if ($need_wide_mask[$word]) {
-            @shift = ('0', 'BEAM_WIDE_SHIFT');
-            @mask = ('BEAM_WIDE_MASK', $WHOLE_WORD);
-            @instr = ('w', 'w');
-        } else {
-            @shift = @{$pack_shift[$args_per_word]};
-            @mask = @{$pack_mask[$args_per_word]};
-            @instr = @{$pack_instr[$args_per_word]};
-        }
+        my $pack_word_size = ($pack_in_iw && $word == 0) ? 32 : $wordsize;
+
+        my($shref,$mref,$iref,$unpack_suffix)  =
+            get_pack_parameters($args_per_word, $pack_word_size,
+                                $need_wide_mask[$word]);
+        my @shift = @$shref;
+        my @mask = @$mref;
+        my @instr = @$iref;
 
         while ($ap < $args_per_word) {
             my $reg = $args[$arg_num];
             my $this_size = $arg_size{$reg};
+
             if ($bits_needed[$arg_num]) {
                 $this_size = 0;
 
                 if ($ap == 0) {
-                    $pack_prefix .= "Eterm $packed_var = " .
-                        arg_offset($size+$offset) . ";\n";
-                    $up .= "p";
-                    $down = "P$down";
-                    $this_size = 1;
+                    my $packed_data;
+                    if ($pack_in_iw and $word == 0) {
+                        $packed_data = "BeamExtraData(I[0])";
+                        if ($args_per_word == 1) {
+                            $packed_var = $packed_data;
+                        } else {
+                            $pack_prefix .= "Eterm $packed_var = $packed_data;\n";
+                        }
+                        my $pack = chr(ord('#') + $size);
+                        $down = "$pack$down";
+                    } else {
+                        $packed_data = arg_offset($size + $offset);
+                        $pack_prefix .= "Eterm $packed_var = $packed_data;\n";
+                        $down = "P$down";
+                        $up .= "p";
+                        $this_size = 1;
+                    }
                 }
 
                 $down = "$instr[$ap]$down";
                 my $unpack = make_unpack($packed_var, $shift[$ap], $mask[$ap]);
-                $args[$arg_num] = "packed:$reg:$this_size:$reg" . "b($unpack)";
+                my $macro = "$reg$unpack_suffix";
+                $args[$arg_num] = "packed:$reg:$this_size:$macro($unpack)";
 
                 $ap++;
             } else {
@@ -1809,7 +1844,98 @@ sub do_pack {
     }
 
     my $pack_spec = "$down:$up";
-    return ($pack_prefix, $pack_spec, @args);
+    my $score = pack_score($options, @args);
+
+    return ($score, [$pack_prefix,$pack_spec,@args]);
+}
+
+sub get_pack_parameters {
+    my($args_per_word,$pack_word_size,$wide_mask) = @_;
+    my(@shift,@mask,@instr);
+    my $unpack_suffix = 'b';
+
+    if ($wide_mask and $args_per_word > 1) {
+        @shift = ('0', 'BEAM_WIDE_SHIFT');
+        @mask = ('BEAM_WIDE_MASK', $WHOLE_WORD);
+        @instr = (PACK_CMD_WIDE) x 2;
+    } elsif ($args_per_word == 1) {
+        @shift = ('0');
+        @mask = ($WHOLE_WORD);
+        @instr = (PACK_CMD_WIDE);
+    } elsif ($args_per_word == 2) {
+        if ($pack_word_size != $wordsize) {
+            # 64-bit word size, pack 32 bits into instruction word.
+            @shift = ('0', 'BEAM_TIGHT_SHIFT');
+            @mask  = ('BEAM_TIGHT_MASK', $WHOLE_WORD);
+            @instr = (PACK_CMD_TIGHT) x 2;
+        } else {
+            # 32/64 bit word size
+            @shift = ('0', 'BEAM_LOOSE_SHIFT');
+            @mask  = ('BEAM_LOOSE_MASK', $WHOLE_WORD);
+            @instr = (PACK_CMD_LOOSE) x 2;
+        }
+    } elsif ($args_per_word == 3) {
+        if ($pack_word_size != $wordsize) {
+            # 64-bit word size, pack 3 register numbers into instruction word.
+            @shift = ('0', 'BEAM_TIGHTEST_SHIFT', '(2*BEAM_TIGHTEST_SHIFT)');
+            @mask = ('BEAM_TIGHTEST_MASK', 'BEAM_TIGHTEST_MASK', $WHOLE_WORD);
+            @instr = (PACK_CMD_TIGHTEST) x 3;
+            $unpack_suffix = '';
+        } else {
+            # 32/64 bit word size.
+            @shift = ('0', 'BEAM_TIGHT_SHIFT', '(2*BEAM_TIGHT_SHIFT)');
+            if ($wordsize == 32) {
+                @mask = ('BEAM_TIGHT_MASK') x 3;
+            } elsif ($wordsize == 64) {
+                @mask = ('BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK', $WHOLE_WORD);
+            }
+            @instr = (PACK_CMD_TIGHT) x 3;
+        }
+    } elsif ($args_per_word == 4) {
+        # 64 bit word size only.
+        @shift = ('0',
+                  'BEAM_LOOSE_SHIFT',
+                  '(2*BEAM_LOOSE_SHIFT)',
+		  '(3*BEAM_LOOSE_SHIFT)');
+        @mask = ('BEAM_LOOSE_MASK', 'BEAM_LOOSE_MASK',
+                 'BEAM_LOOSE_MASK', $WHOLE_WORD);
+        @instr = (PACK_CMD_LOOSE) x 4;
+    }
+
+    unless (@shift) {
+        error("internal error: args_per_word=$args_per_word, " .
+              "pack_word_size=$pack_word_size");
+    }
+
+    (\@shift,\@mask,\@instr,$unpack_suffix);
+}
+
+sub pack_score {
+    my($options,@args) = @_;
+    my $size = 0;
+
+    # Calculate the number of words.
+    foreach (@args) {
+        if (/^packed:[^:]*:(\d+)/) {
+            $size += $1;
+        } else {
+            $size += $arg_size{$_}
+        }
+    }
+
+    # Less numbers of words give a higher score; for the same number of
+    # words, using PACK_JUMP or PACK_IN_INSTR_WORD gives a lower score.
+    my $score = 1 + $max_spec_operands*($max_spec_operands - $size);
+    if ($options == PACK_IN_INSTR_WORD) {
+        $score += 0;
+    } elsif ($options == PACK_JUMP) {
+        $score += 1;
+    } elsif ($options == (PACK_JUMP|PACK_IN_INSTR_WORD)) {
+        $score += 2;
+    } elsif ($options == 0) {
+        $score += 3;
+    }
+    $score;
 }
 
 sub make_unpack {
-- 
cgit v1.2.3


From 985524704ddd62cf85385a323861377608d66c9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 15 Sep 2017 12:29:58 +0200
Subject: Optimize instruction prefetch

---
 erts/emulator/beam/beam_emu.c    | 2 ++
 erts/emulator/utils/beam_makeops | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 73c4e3532b..aa94fbf536 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -52,11 +52,13 @@
 #  define CountCase(OpCode) case op_count_##OpCode
 #  define IsOpCode(InstrWord, OpCode)  (BeamCodeAddr(InstrWord) == (BeamInstr)op_##OpCode)
 #  define Goto(Rel)         {Go = BeamCodeAddr(Rel); goto emulator_loop;}
+#  define GotoPF(Rel)       Goto(Rel)
 #else
 #  define OpCase(OpCode)    lb_##OpCode
 #  define CountCase(OpCode) lb_count_##OpCode
 #  define IsOpCode(InstrWord, OpCode)  (BeamCodeAddr(InstrWord) == (BeamInstr)&&lb_##OpCode)
 #  define Goto(Rel)         goto *((void *)BeamCodeAddr(Rel))
+#  define GotoPF(Rel)       goto *((void *)Rel)
 #  define LabelAddr(Label)  &&Label
 #endif
 
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 388e3d164d..322d60b00e 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -1452,10 +1452,10 @@ sub code_gen {
             "ASSERT(VALID_INSTR(*I));\n" .
             "Goto(*I);";
     } else {
-        $var_decls .= "BeamInstr next_pf = I[$instr_offset];\n";
+        $var_decls .= "BeamInstr next_pf = BeamCodeAddr(I[$instr_offset]);\n";
         $dispatch_next = "\nI += $instr_offset;\n" .
             "ASSERT(VALID_INSTR(next_pf));\n" .
-            "Goto(next_pf);";
+            "GotoPF(next_pf);";
     }
 
     #
-- 
cgit v1.2.3


From f2288e390ab660eca5af747e017894ecd8e5ba58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 26 Sep 2017 07:33:18 +0200
Subject: beam_makeops: Refactor parsing of specific instructions

---
 erts/emulator/utils/beam_makeops | 55 +++++++++++++++++++++-------------------
 1 file changed, 29 insertions(+), 26 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 322d60b00e..c8550df1e2 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -453,15 +453,7 @@ while (<>) {
     # Parse specific instructions (only present in emulator/loader):
     #    Name Arg1 Arg2...
     #
-    my($name, @args) = split;
-    error("too many operands")
-	if @args > $max_spec_operands;
-    syntax_check($name, @args);
-    my $arity = @args;
-    if (defined $gen_opnum{$name,$arity} and $obsolete[$gen_opnum{$name,$arity}]) {
-	error("specific instructions may not be specified for obsolete instructions");
-    }
-    save_specific_ops($name, $arity, $hotness, @args);
+    my($name,$arity) = parse_specific_op($_);
     if (defined $op_num) {
 	error("specific instructions must not be numbered");
     } elsif (!defined($gen_arity{$name}) && !defined($unnumbered{$name,$arity})) {
@@ -981,40 +973,51 @@ sub compiler_output {
 }
 
 #
-# Check an operation for validity.
+# Parse and store a specific operation.
 #
-sub syntax_check {
-    my($name, @args) = @_;
-    my($i);
+sub parse_specific_op {
+    my($name, @args) = split " ", shift;
+    my $arity = @args;
 
+    # Check for various errors.
     error("Bad opcode name '$name'")
 	unless $name =~ /^[a-z][\w\d_]*$/;
-    for ($i = 0; $i < @args; $i++) {
+    error("too many operands")
+	if @args > $max_spec_operands;
+    for (my $i = 0; $i < $arity; $i++) {
         foreach my $type (split(//, $args[$i])) {
             error("Argument " . ($i+1) . ": invalid type '$type'")
                 unless defined $arg_size{$type};
         }
     }
-}
-
-sub save_specific_ops {
-    my($name,$arity,$hot,@args) = @_;
-    my(@res) = ("");
+    if (defined $gen_opnum{$name,$arity} and $obsolete[$gen_opnum{$name,$arity}]) {
+	error("specific instructions may not be specified for obsolete instructions");
+    }
 
+    # Expand operands with multiple types to multiple instructions.
+    # (For example, "get_list xy xy xy" will be expanded to six instructions.)
+    my @res = ([]);
     foreach my $arg (@args) {
-        my @new_res = ();
+        my @old_res = @res;
+        @res = ();
         foreach my $type (split(//, $arg)) {
-            foreach my $args (@res) {
-                push @new_res, "$args$type";
+            foreach my $args_ref (@old_res) {
+                my @args = @$args_ref;
+                push @args, $type;
+                push @res, \@args;
             }
         }
-        @res = @new_res;
     }
+
+    # Store each specific instruction.
     my $key = "$name/$arity";
-    foreach my $args (@res) {
-        @args = split //, $args;
-        push @{$specific_op{$key}}, [$name,$hot,@args];
+    foreach my $args_ref (@res) {
+        @args = @$args_ref;
+        push @{$specific_op{$key}}, [$name,$hotness,@args];
     }
+
+    # Done.
+    ($name,$arity);
 }
 
 sub parse_c_args {
-- 
cgit v1.2.3


From e86262c45eb3ccbd055034239ddcd19472e56b2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 26 Sep 2017 08:15:02 +0200
Subject: Introduce a syntax for marking operands as "optional use"

Introduce a syntax to mark an operand that is not always used when
an instrution is executed. Example of such operands are the fail
label for is_nil or the number of live registers for an
allocate instruction.

Use a question mark to annotate optional use:

  is_nil f? xy
  allocate t t?
---
 erts/emulator/beam/ops.tab       | 262 +++++++++++++++++++--------------------
 erts/emulator/utils/beam_makeops |  13 +-
 2 files changed, 141 insertions(+), 134 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index b7ef1f0599..4a915c7762 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -99,21 +99,21 @@ line Loc | func_info M F A => func_info M F A | line Loc
 
 line I
 
-allocate t t
-allocate_heap t I t
+allocate t t?
+allocate_heap t I t?
 
 %cold
 deallocate Q
 %hot
 
 init y
-allocate_zero t t
-allocate_heap_zero t I t
+allocate_zero t t?
+allocate_heap_zero t I t?
 
 trim N Remaining => i_trim N
 i_trim t
 
-test_heap I t
+test_heap I t?
 
 allocate_heap S u==0 R => allocate S R
 allocate_heap_zero S u==0 R => allocate_zero S R
@@ -158,19 +158,19 @@ is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \
 select_tuple_arity S=d Fail=f Size=u Rest=* => \
   gen_select_tuple_arity(S, Fail, Size, Rest)
 
-i_select_val_bins xy f I
+i_select_val_bins xy f? I
 
-i_select_val_lins xy f I
+i_select_val_lins xy f? I
 
-i_select_val2 xy f c c
+i_select_val2 xy f? c c
 
-i_select_tuple_arity xy f I
+i_select_tuple_arity xy f? I
 
-i_select_tuple_arity2 xy f A A
+i_select_tuple_arity2 xy f? A A
 
-i_jump_on_val_zero xy f I
+i_jump_on_val_zero xy f? I
 
-i_jump_on_val xy f I W
+i_jump_on_val xy f? I W
 
 get_list xy xy xy
 
@@ -213,9 +213,9 @@ i_get_tuple_element2y x P y y
 i_get_tuple_element3 x P x
 
 %cold
-is_number f x
-is_number f y
+is_number f? xy
 %hot
+
 is_number Fail=f i =>
 is_number Fail=f na => jump Fail
 is_number Fail Literal=q => move Literal x | is_number Fail x
@@ -446,37 +446,37 @@ is_ne_exact Lbl C=c R=xy => is_ne_exact Lbl R C
 is_ne_exact Lbl R=xy C=ian => i_is_ne_exact_immed Lbl R C
 is_ne_exact Lbl R=xy C=q => i_is_ne_exact_literal Lbl R C
 
-i_is_eq_exact_immed f rxy c
+i_is_eq_exact_immed f? rxy c
 
-i_is_eq_exact_literal f xy c
+i_is_eq_exact_literal f? xy c
 
-i_is_ne_exact_immed f xy c
+i_is_ne_exact_immed f? xy c
 
-i_is_ne_exact_literal f xy c
+i_is_ne_exact_literal f? xy c
 
 is_eq_exact Lbl Y=y X=x => is_eq_exact Lbl X Y
-is_eq_exact f x xy
-is_eq_exact f y y
+is_eq_exact f? x xy
+is_eq_exact f? y y
 
-is_ne_exact f S S
+is_ne_exact f? S S
 
-is_lt f x x
-is_lt f x c
-is_lt f c x
+is_lt f? x x
+is_lt f? x c
+is_lt f? c x
 %cold
-is_lt f s s
+is_lt f? s s
 %hot
 
-is_ge f x x
-is_ge f x c
-is_ge f c x
+is_ge f? x x
+is_ge f? x c
+is_ge f? c x
 %cold
-is_ge f s s
+is_ge f? s s
 %hot
 
-is_eq f s s
+is_eq f? s s
 
-is_ne f s s
+is_ne f? s s
 
 #
 # Putting things.
@@ -583,7 +583,7 @@ is_tagged_tuple Fail Literal=q Arity Atom => \
     move Literal x | is_tagged_tuple Fail x Arity Atom
 is_tagged_tuple Fail=f c Arity Atom  => jump Fail
 
-is_tagged_tuple f rxy A a
+is_tagged_tuple f? rxy A a
 
 # Test tuple & arity (head)
 
@@ -591,14 +591,14 @@ is_tuple Fail Literal=q => move Literal x | is_tuple Fail x
 is_tuple Fail=f c => jump Fail
 is_tuple Fail=f S=xy | test_arity Fail=f S=xy Arity => is_tuple_of_arity Fail S Arity
 
-is_tuple_of_arity f rxy A
+is_tuple_of_arity f? rxy A
 
-is_tuple f rxy
+is_tuple f? rxy
 
 test_arity Fail Literal=q Arity => move Literal x | test_arity Fail x Arity
 test_arity Fail=f c Arity => jump Fail
 
-test_arity f xy A
+test_arity f? xy A
 
 get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \
    get_tuple_element Reg=x P3 D3=x | \
@@ -619,16 +619,16 @@ is_integer Fail Literal=q => move Literal x | is_integer Fail x
 
 is_integer Fail=f S=x | allocate Need Regs => is_integer_allocate Fail S Need Regs
 
-is_integer_allocate f x t t
+is_integer_allocate f? x t t
 
-is_integer f xy
+is_integer f? xy
 
 is_list Fail=f n =>
 is_list Fail Literal=q => move Literal x | is_list Fail x
 is_list Fail=f c => jump Fail
-is_list f x
+is_list f? x
 %cold
-is_list f y
+is_list f? y
 %hot
 
 is_nonempty_list Fail=f S=x | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs
@@ -638,21 +638,21 @@ is_nonempty_list F=f x==0 | test_heap I1 I2 => is_nonempty_list_test_heap F I1 I
 is_nonempty_list Fail=f S=x | get_list S D1=x D2=x => \
   is_nonempty_list_get_list Fail S D1 D2
 
-is_nonempty_list_allocate f rx t t
-is_nonempty_list_test_heap f I t
-is_nonempty_list_get_list f rx x x
-is_nonempty_list f xy
+is_nonempty_list_allocate f? rx t t
+is_nonempty_list_test_heap f? I t
+is_nonempty_list_get_list f? rx x x
+is_nonempty_list f? xy
 
-is_atom f x
+is_atom f? x
 %cold
-is_atom f y
+is_atom f? y
 %hot
 is_atom Fail=f a =>
 is_atom Fail=f niq => jump Fail
 
-is_float f x
+is_float f? x
 %cold
-is_float f y
+is_float f? y
 %hot
 is_float Fail=f nai => jump Fail
 is_float Fail Literal=q => move Literal x | is_float Fail x
@@ -660,13 +660,13 @@ is_float Fail Literal=q => move Literal x | is_float Fail x
 is_nil Fail=f n =>
 is_nil Fail=f qia => jump Fail
 
-is_nil f xy
+is_nil f? xy
 
 is_binary Fail Literal=q => move Literal x | is_binary Fail x
 is_binary Fail=f c => jump Fail
-is_binary f x
+is_binary f? x
 %cold
-is_binary f y
+is_binary f? y
 %hot
 
 # XXX Deprecated.
@@ -674,27 +674,27 @@ is_bitstr Fail Term => is_bitstring Fail Term
 
 is_bitstring Fail Literal=q => move Literal x | is_bitstring Fail x
 is_bitstring Fail=f c => jump Fail
-is_bitstring f x
+is_bitstring f? x
 %cold
-is_bitstring f y
+is_bitstring f? y
 %hot
 
 is_reference Fail=f cq => jump Fail
-is_reference f x
+is_reference f? x
 %cold
-is_reference f y
+is_reference f? y
 %hot
 
 is_pid Fail=f cq => jump Fail
-is_pid f x
+is_pid f? x
 %cold
-is_pid f y
+is_pid f? y
 %hot
 
 is_port Fail=f cq => jump Fail
-is_port f x
+is_port f? x
 %cold
-is_port f y
+is_port f? y
 %hot
 
 is_boolean Fail=f a==am_true =>
@@ -702,19 +702,19 @@ is_boolean Fail=f a==am_false =>
 is_boolean Fail=f ac => jump Fail
 
 %cold
-is_boolean f xy
+is_boolean f? xy
 %hot
 
 is_function2 Fail=f acq Arity => jump Fail
 is_function2 Fail=f Fun a => jump Fail
 
-is_function2 f S s
+is_function2 f? S s
 
 # Allocating & initializing.
 allocate Need Regs | init Y => allocate_init Need Regs Y
 init Y1 | init Y2 => init2 Y1 Y2
 
-allocate_init t t y
+allocate_init t t? y
 
 #################################################################
 # External function and bif calls.
@@ -1004,13 +1004,13 @@ node y
 
 # Note: 'I' is sufficient because this instruction will only be used
 # if the arity fits in 24 bits.
-i_fast_element xy j I d
+i_fast_element xy j? I d
 
-i_element xy j s d
+i_element xy j? s d
 
-bif1 f b s d
+bif1 f? b s d
 bif1_body b s d
-i_bif2 f b s s d
+i_bif2 f? b s s d
 i_bif2_body b s s d
 
 #
@@ -1062,7 +1062,7 @@ make_fun2 OldIndex=u => gen_make_fun2(OldIndex)
 i_make_fun W t
 %hot
 
-is_function f xy
+is_function f? xy
 is_function Fail=f c => jump Fail
 
 func_info M F A => i_func_info u M F A
@@ -1091,24 +1091,24 @@ i_bs_match_string x f W W
 bs_get_integer2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \
 			gen_get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
 
-i_bs_get_integer_small_imm x W f t x
-i_bs_get_integer_imm x W t f t x
-i_bs_get_integer f t t x s x
-i_bs_get_integer_8 x f x
-i_bs_get_integer_16 x f x
+i_bs_get_integer_small_imm x W f? t x
+i_bs_get_integer_imm x W t f? t x
+i_bs_get_integer f? t t x s x
+i_bs_get_integer_8 x f? x
+i_bs_get_integer_16 x f? x
 
 %if ARCH_64
-i_bs_get_integer_32 x f x
+i_bs_get_integer_32 x f? x
 %endif
 
 # Fetching binaries from binaries.
 bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \
 			gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
 
-i_bs_get_binary_imm2 f x t W t x
-i_bs_get_binary2 f x t s t x
-i_bs_get_binary_all2 f x t t x
-i_bs_get_binary_all_reuse x f t
+i_bs_get_binary_imm2 f? x t W t x
+i_bs_get_binary2 f x t? s t x
+i_bs_get_binary_all2 f? x t t x
+i_bs_get_binary_all_reuse x f? t
 
 # Fetching float from binaries.
 bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \
@@ -1116,25 +1116,25 @@ bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \
 
 bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail
 
-i_bs_get_float2 f x t s t x
+i_bs_get_float2 f? x t s t x
 
 # Miscellanous
 
 bs_skip_bits2 Fail=f Ms=x Sz=sq Unit=u Flags=u => \
 			gen_skip_bits2(Fail, Ms, Sz, Unit, Flags)
 
-i_bs_skip_bits_imm2 f x W
-i_bs_skip_bits2 f x xy t
-i_bs_skip_bits_all2 f x t
+i_bs_skip_bits_imm2 f? x W
+i_bs_skip_bits2 f? x xy t
+i_bs_skip_bits_all2 f? x t
 
 bs_test_tail2 Fail=f Ms=x Bits=u==0 => bs_test_zero_tail2 Fail Ms
 bs_test_tail2 Fail=f Ms=x Bits=u => bs_test_tail_imm2 Fail Ms Bits
-bs_test_zero_tail2 f x
-bs_test_tail_imm2 f x W
+bs_test_zero_tail2 f? x
+bs_test_tail_imm2 f? x W
 
 bs_test_unit F Ms Unit=u==8 => bs_test_unit8 F Ms
-bs_test_unit f x t
-bs_test_unit8 f x
+bs_test_unit f? x t
+bs_test_unit8 f? x
 
 # An y register operand for bs_context_to_binary is rare,
 # but can happen because of inlining.
@@ -1150,14 +1150,14 @@ bs_context_to_binary x
 # Utf8/utf16/utf32 support. (R12B-5)
 #
 bs_get_utf8 Fail=f Ms=x u u Dst=d => i_bs_get_utf8 Ms Fail Dst
-i_bs_get_utf8 x f x
+i_bs_get_utf8 x f? x
 
 bs_skip_utf8 Fail=f Ms=x u u => i_bs_get_utf8 Ms Fail x
 
 bs_get_utf16 Fail=f Ms=x u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst
 bs_skip_utf16 Fail=f Ms=x u Flags=u => i_bs_get_utf16 Ms Fail Flags x
 
-i_bs_get_utf16 x f t x
+i_bs_get_utf16 x f? t x
 
 bs_get_utf32 Fail=f Ms=x Live=u Flags=u Dst=d => \
 	bs_get_integer2 Fail Ms Live i=32 u=1 Flags Dst | \
@@ -1186,13 +1186,13 @@ bs_init2 Fail Sz Words=u==0 Regs Flags Dst => \
 bs_init2 Fail Sz Words Regs Flags Dst => \
   i_bs_init_fail_heap Sz Words Fail Regs Dst
 
-i_bs_init_fail xy j t x
+i_bs_init_fail xy j? t? x
 
-i_bs_init_fail_heap s I j t x
+i_bs_init_fail_heap s I j? t? x
 
-i_bs_init W t x
+i_bs_init W t? x
 
-i_bs_init_heap W I t x
+i_bs_init_heap W I t? x
 
 
 bs_init_bits Fail Sz=o Words Regs Flags Dst => system_limit Fail
@@ -1205,16 +1205,16 @@ bs_init_bits Fail Sz Words=u==0 Regs Flags Dst => \
 bs_init_bits Fail Sz Words Regs Flags Dst => \
   i_bs_init_bits_fail_heap Sz Words Fail Regs Dst
 
-i_bs_init_bits_fail xy j t x
+i_bs_init_bits_fail xy j? t? x
 
-i_bs_init_bits_fail_heap s I j t x
+i_bs_init_bits_fail_heap s I j? t? x
 
-i_bs_init_bits W t x
-i_bs_init_bits_heap W I t x
+i_bs_init_bits W t? x
+i_bs_init_bits_heap W I t? x
 
 bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D
 
-bs_add j s s t x
+bs_add j? s s t? x
 
 bs_append Fail Size Extra Live Unit Bin Flags Dst => \
   move Bin x | i_bs_append Fail Extra Live Unit Size Dst
@@ -1224,8 +1224,8 @@ bs_private_append Fail Size Unit Bin Flags Dst => \
 
 bs_init_writable
 
-i_bs_append j I t t s x
-i_bs_private_append j t s S x
+i_bs_append j? I t? t s x
+i_bs_private_append j? t s S x
 
 #
 # Storing integers into binaries.
@@ -1234,8 +1234,8 @@ i_bs_private_append j t s S x
 bs_put_integer Fail=j Sz=sq Unit=u Flags=u Src=s => \
 			gen_put_integer(Fail, Sz, Unit, Flags, Src)
 
-i_new_bs_put_integer j s t s
-i_new_bs_put_integer_imm j W t s
+i_new_bs_put_integer j? s t s
+i_new_bs_put_integer_imm j? W t s
 
 #
 # Utf8/utf16/utf32 support. (R12B-5)
@@ -1251,14 +1251,14 @@ i_bs_utf16_size s x
 
 bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src
 
-i_bs_put_utf8 j s
+i_bs_put_utf8 j? s
 
-bs_put_utf16 j t s
+bs_put_utf16 j? t s
 
 bs_put_utf32 Fail=j Flags=u Src=s => \
    i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src
 
-i_bs_validate_unicode j s
+i_bs_validate_unicode j? s
 
 #
 # Storing floats into binaries.
@@ -1268,8 +1268,8 @@ bs_put_float Fail Sz=q Unit Flags Val => badarg Fail
 bs_put_float Fail=j Sz=s Unit=u Flags=u Src=s => \
 			gen_put_float(Fail, Sz, Unit, Flags, Src)
 
-i_new_bs_put_float j s t s
-i_new_bs_put_float_imm j W t s
+i_new_bs_put_float j? s t s
+i_new_bs_put_float_imm j? W t s
 
 #
 # Storing binaries into binaries.
@@ -1278,9 +1278,9 @@ i_new_bs_put_float_imm j W t s
 bs_put_binary Fail=j Sz=s Unit=u Flags=u Src=s => \
 			gen_put_binary(Fail, Sz, Unit, Flags, Src)
 
-i_new_bs_put_binary j s t s
-i_new_bs_put_binary_imm j W s
-i_new_bs_put_binary_all j s t
+i_new_bs_put_binary j? s t s
+i_new_bs_put_binary_imm j? W s
+i_new_bs_put_binary_all j? s t
 
 #
 # Warning: The i_bs_put_string and i_new_bs_put_string instructions
@@ -1394,12 +1394,12 @@ new_map Dst Live Size Rest=* | is_small_map_literal_keys(Size, Rest) => \
 new_map d t I
 i_new_small_map_lit d t q
 update_map_assoc s d t I
-update_map_exact j s d t I
+update_map_exact j? s d t I
 
 is_map Fail Lit=q | literal_is_map(Lit) =>
 is_map Fail cq => jump Fail
 
-is_map f xy
+is_map f? xy
 
 ## Transform has_map_fields #{ K1 := _, K2 := _ } to has_map_elements
 
@@ -1413,14 +1413,14 @@ get_map_elements Fail Src=xy Size=u==2 Rest=* => \
 get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \
    gen_get_map_elements(Fail, Src, Size, Rest)
 
-i_get_map_elements f s I
+i_get_map_elements f? s I
 
 i_get_map_element Fail Src=xy Key=y Dst => \
     move Key x | i_get_map_element Fail Src x Dst
 
-i_get_map_element_hash f xy c I xy
+i_get_map_element_hash f? xy c I xy
 
-i_get_map_element f xy x xy
+i_get_map_element f? xy x xy
 
 #
 # Convert the plus operations to a generic plus instruction.
@@ -1488,32 +1488,32 @@ gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst
 
 i_increment rxy W t d
 
-i_plus x xy j t d
-i_plus s s  j t d
+i_plus x xy j? t d
+i_plus s s  j? t d
 
-i_minus x x j t d
-i_minus s s j t d
+i_minus x x j? t d
+i_minus s s j? t d
 
-i_times j t s s d
+i_times j? t s s d
 
-i_m_div j t s s d
-i_int_div j t s s d
+i_m_div j? t s s d
+i_int_div j? t s s d
 
-i_rem x x j t d
-i_rem s s j t d
+i_rem x x j? t d
+i_rem s s j? t d
 
-i_bsl s s j t d
-i_bsr s s j t d
+i_bsl s s j? t d
+i_bsr s s j? t d
 
-i_band x c j t d
-i_band s s j t d
+i_band x c j? t d
+i_band s s j? t d
 
-i_bor j I s s d
-i_bxor j I s s d
+i_bor j? I s s d
+i_bxor j? I s s d
 
 i_int_bnot Fail Src=c Live Dst => move Src x | i_int_bnot Fail x Live Dst
 
-i_int_bnot j S t d
+i_int_bnot j? S t d
 
 #
 # Old guard BIFs that creates heap fragments are no longer allowed.
@@ -1537,9 +1537,9 @@ gc_bif2 Fail I Bif S1 S2 Dst => \
 gc_bif3 Fail I Bif S1 S2 S3 Dst => \
 	gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst)
 
-i_gc_bif1 j W s t d
+i_gc_bif1 j? W s t? d
 
-i_gc_bif2 j W t s s d
+i_gc_bif2 j? W t? s s d
 
 ii_gc_bif3/7
 
@@ -1548,7 +1548,7 @@ ii_gc_bif3/7
 ii_gc_bif3 Fail Bif Live S1 S2 S3 Dst => \
   move S1 x | i_gc_bif3 Fail Bif Live S2 S3 Dst
 
-i_gc_bif3 j W t s s d
+i_gc_bif3 j? W t? s s d
 
 #
 # The following instruction is specially handled in beam_load.c
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index c8550df1e2..d1c51e5dad 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -526,7 +526,6 @@ sub emulator_output {
     foreach $key (keys %specific_op) {
 	foreach (@{$specific_op{$key}}) {
 	    my($name, $hotness, @args) = @$_;
-	    my $sign = join('', @args);
 	    my $print_name = print_name($name, @args);
 
 	    my($size, $code, $pack_spec) = cg_basic($name, @args);
@@ -597,6 +596,7 @@ sub emulator_output {
 	foreach (@{$specific_op{$key}}) {
 	    my($name, $hot, @args) = @{$_};
 	    my($sign) = join('', @args);
+            $sign =~ s/[?]//g;
 
 	    # The primitive types should sort before other types.
 
@@ -614,6 +614,7 @@ sub emulator_output {
             my $print_name = $items{$sort_key};
             my $info = $spec_op_info{$print_name};
             my(@args) = @{$info->{'args'}};
+            @args = map { s/[?]$//; $_ } @args;
 	    my $arity = @args;
 
 	    #
@@ -854,6 +855,7 @@ sub emulator_output {
 sub print_name {
     my($name,@args) = @_;
     my $sign = join '', @args;
+    $sign =~ s/[?]//g;
     $sign ne '' ? "${name}_$sign" : $name;
 }
 
@@ -985,7 +987,9 @@ sub parse_specific_op {
     error("too many operands")
 	if @args > $max_spec_operands;
     for (my $i = 0; $i < $arity; $i++) {
-        foreach my $type (split(//, $args[$i])) {
+        my $arg = $args[$i];
+        $arg =~ s/[?]$//;
+        foreach my $type (split(//, $arg)) {
             error("Argument " . ($i+1) . ": invalid type '$type'")
                 unless defined $arg_size{$type};
         }
@@ -1000,10 +1004,11 @@ sub parse_specific_op {
     foreach my $arg (@args) {
         my @old_res = @res;
         @res = ();
+        my $marker = ($arg =~ s/[?]$//) ? '?' : '';
         foreach my $type (split(//, $arg)) {
             foreach my $args_ref (@old_res) {
                 my @args = @$args_ref;
-                push @args, $type;
+                push @args, "$type$marker";
                 push @res, \@args;
             }
         }
@@ -1351,6 +1356,7 @@ sub code_gen {
 
     my $need_block = 0;
     my $arg_offset = $offset;
+    @args = map { s/[?]$//g; $_ } @args;
     foreach (@args) {
 	my($this_size) = $arg_size{$_};
       SWITCH:
@@ -1618,6 +1624,7 @@ sub needs_do_wrapper {
 
 sub do_pack {
     my($offset,$pack_options,@args) = @_;
+    @args = map { s/[?]$//; $_ } @args;
     my $ret = ['', ':', @args];
     my $score = 0;
 
-- 
cgit v1.2.3


From 43f1817f3367bf3aa346c8b3dae10862d3723515 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 27 Sep 2017 15:11:17 +0200
Subject: beam_makeops: Print the instruction name for fatal packing errors

Having the instruction name available in the functions
that implement packing also simplifies debugging.
---
 erts/emulator/utils/beam_makeops | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index d1c51e5dad..a705ba27b7 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -1346,7 +1346,7 @@ sub code_gen {
 
     my $c_code_ref = $c_code{$name};
     if (defined $c_code_ref and $name ne 'catch') {
-        ($var_decls, $pack_spec, @args) = do_pack($offset, $pack_options, @args);
+        ($var_decls, $pack_spec, @args) = do_pack($name, $offset, $pack_options, @args);
     }
 
     #
@@ -1623,13 +1623,13 @@ sub needs_do_wrapper {
 }
 
 sub do_pack {
-    my($offset,$pack_options,@args) = @_;
+    my($name,$offset,$pack_options,@args) = @_;
     @args = map { s/[?]$//; $_ } @args;
     my $ret = ['', ':', @args];
     my $score = 0;
 
     foreach my $options (@$pack_options) {
-        my($this_score,$this_result) = do_pack_one($options, $offset, @args);
+        my($this_score,$this_result) = do_pack_one($name, $options, $offset, @args);
         if ($this_score > $score) {
             $ret = $this_result;
             $score = $this_score;
@@ -1639,7 +1639,7 @@ sub do_pack {
 }
 
 sub do_pack_one {
-    my($options,$offset,@args) = @_;
+    my($name,$options,$offset,@args) = @_;
     my($packable_args) = 0;
     my @bits_needed;            # Bits needed for each argument.
     my $pack_in_iw = $options & PACK_IN_INSTR_WORD;
@@ -1798,7 +1798,7 @@ sub do_pack_one {
         my $pack_word_size = ($pack_in_iw && $word == 0) ? 32 : $wordsize;
 
         my($shref,$mref,$iref,$unpack_suffix)  =
-            get_pack_parameters($args_per_word, $pack_word_size,
+            get_pack_parameters($name, $args_per_word, $pack_word_size,
                                 $need_wide_mask[$word]);
         my @shift = @$shref;
         my @mask = @$mref;
@@ -1860,7 +1860,7 @@ sub do_pack_one {
 }
 
 sub get_pack_parameters {
-    my($args_per_word,$pack_word_size,$wide_mask) = @_;
+    my($name,$args_per_word,$pack_word_size,$wide_mask) = @_;
     my(@shift,@mask,@instr);
     my $unpack_suffix = 'b';
 
@@ -1913,7 +1913,7 @@ sub get_pack_parameters {
     }
 
     unless (@shift) {
-        error("internal error: args_per_word=$args_per_word, " .
+        error("$name: internal packing error: args_per_word=$args_per_word, " .
               "pack_word_size=$pack_word_size");
     }
 
-- 
cgit v1.2.3


From d7588c53399dfa369ae2af6874e69d252d062054 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 27 Sep 2017 17:26:53 +0200
Subject: Optimize packing for "optional use" operands

Operands that are marked with a "?" are not always used when
an instruction is executed. Enhance the packing algorithm to
place optional use operands into the instruction word even they
are not the first operand (as long as the total instruction
size stays the same).

Here are the instructions that will be packed differently
because of this change:

    allocate_heap t I t?
    allocate_heap_zero t I t?
    test_heap I t?

    i_bs_get_integer_8 x f? x
    i_bs_get_integer_16 x f? x
    i_bs_get_integer_32 x f? x
---
 erts/emulator/utils/beam_makeops | 66 +++++++++++++++++++++++++++++++---------
 1 file changed, 52 insertions(+), 14 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index a705ba27b7..9da62f18ac 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -27,6 +27,7 @@ use constant HOT => 2;
 # Instructions for packing
 use constant PACK_JUMP => 1;
 use constant PACK_IN_INSTR_WORD => 2;
+use constant PACK_OPT_IN_INSTR_WORD => 4;
 
 # Packing commands
 use constant PACK_CMD_TIGHTEST => '1';
@@ -1623,13 +1624,38 @@ sub needs_do_wrapper {
 }
 
 sub do_pack {
-    my($name,$offset,$pack_options,@args) = @_;
-    @args = map { s/[?]$//; $_ } @args;
+    my($name,$offset,$pack_opts_ref,@args) = @_;
+    my @pack_opts = @$pack_opts_ref;
+    my $opt_arg_pos = -1;
+
+    # Look for an optional use operand not as the first argument.
+    if (@args and $args[0] !~ /[?]$/) {
+        for (my $pos = 0; $pos < @args; $pos++) {
+            if ($args[$pos] =~ /[?]$/) {
+                $opt_arg_pos = $pos;
+                last;
+            }
+        }
+    }
+
+    @args = map { s/[?]$//; $_ } @args; # Remove any optional use marker.
+
+    # If there is an optional operand, extend the array of pack options.
+    if ($opt_arg_pos >= 0) {
+        my @new_pack_opts = grep { $_ & PACK_IN_INSTR_WORD } @pack_opts;
+        @new_pack_opts = map {
+            ($_ & ~ PACK_IN_INSTR_WORD) | PACK_OPT_IN_INSTR_WORD;
+        } @new_pack_opts;
+        push @pack_opts, @new_pack_opts;
+    }
+
     my $ret = ['', ':', @args];
     my $score = 0;
 
-    foreach my $options (@$pack_options) {
-        my($this_score,$this_result) = do_pack_one($name, $options, $offset, @args);
+    foreach my $options (@pack_opts) {
+        my $this_opt_arg_pos = ($options & PACK_OPT_IN_INSTR_WORD) ? $opt_arg_pos : -1;
+        my($this_score,$this_result) =
+            do_pack_one($name, $options, $this_opt_arg_pos, $offset, @args);
         if ($this_score > $score) {
             $ret = $this_result;
             $score = $this_score;
@@ -1639,7 +1665,7 @@ sub do_pack {
 }
 
 sub do_pack_one {
-    my($name,$options,$offset,@args) = @_;
+    my($name,$options,$opt_arg_pos,$offset,@args) = @_;
     my($packable_args) = 0;
     my @bits_needed;            # Bits needed for each argument.
     my $pack_in_iw = $options & PACK_IN_INSTR_WORD;
@@ -1684,7 +1710,7 @@ sub do_pack_one {
     #
     if ($packable_args == 0) {
         return (-1);
-    } elsif ($packable_args == 1 and !$pack_in_iw) {
+    } elsif ($packable_args == 1 and $options == 0) {
         return (-1);
     }
 
@@ -1707,9 +1733,11 @@ sub do_pack_one {
 
     $next_word->();
     $this_wordsize = 32 if $pack_in_iw;
-    for (my $i = 0; $i < @args; $i++) {
-            my $needed = $bits_needed[$i];
+    for (my $arg_num = 0; $arg_num < @args; $arg_num++) {
+            my $needed = $bits_needed[$arg_num];
+
             next unless $needed;
+            next if $arg_num == $opt_arg_pos;
 
             if ($bits+$needed > $this_wordsize) { # Does not fit.
                 $next_word->();
@@ -1765,12 +1793,19 @@ sub do_pack_one {
 				# beginning).
     my $up = '';		# Pack commands (storing back while
 				# moving forward).
+    my $arg_num = 0;            # Number of argument.
 
-    # Skip an unpackable argument.
+    # Skip an unpackable argument. Also handle packing of
+    # an single operand into the instruction word.
     my $skip_unpackable = sub {
         my($arg) = @_;
 
-        if ($arg_size{$arg}) {
+        if ($arg_num == $opt_arg_pos) {
+            my $pack = chr(ord('#') + $arg_num);
+            $down = PACK_CMD_WIDE . "$pack$down";
+            my $unpack = "BeamExtraData(I[0])";
+            $args[$arg_num] = "packed:$arg:0:${arg}b($unpack)";
+        } elsif ($arg_size{$arg}) {
             # Save the argument on the pack engine's stack.
             my $push = 'g';
             if ($type_bit{$arg} & $type_bit{'q'}) {
@@ -1790,7 +1825,6 @@ sub do_pack_one {
     # the packing engine works from right-to-left, but we must generate
     # the instructions from left-to-right because we must calculate
     # instruction sizes from left-to-right.
-    my $arg_num = 0;
     for (my $word = 0; $word < @args_per_word; $word++) {
         my $ap = 0;              # Argument number within word.
         my $packed_var = "tmp_packed" . ($word+1);
@@ -1849,7 +1883,9 @@ sub do_pack_one {
     # Skip any unpackable arguments at the end.
     #
     while ($arg_num < @args) {
-        $skip_unpackable->($args[$arg_num]);
+        my $arg = $args[$arg_num];
+        $skip_unpackable->($arg);
+        $size += $arg_size{$arg};
         $arg_num++;
     }
 
@@ -1935,8 +1971,10 @@ sub pack_score {
 
     # Less numbers of words give a higher score; for the same number of
     # words, using PACK_JUMP or PACK_IN_INSTR_WORD gives a lower score.
-    my $score = 1 + $max_spec_operands*($max_spec_operands - $size);
-    if ($options == PACK_IN_INSTR_WORD) {
+    my $score = 1 + 10*($max_spec_operands - $size);
+    if (($options & PACK_OPT_IN_INSTR_WORD) != 0) {
+        $score += 4;
+    } elsif ($options == PACK_IN_INSTR_WORD) {
         $score += 0;
     } elsif ($options == PACK_JUMP) {
         $score += 1;
-- 
cgit v1.2.3


From a71ffb9c1b9de6ccc7b8d2278361474b373802c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 3 Oct 2017 07:20:20 +0200
Subject: beam_makeops: Use named arguments for the code generation functions

The number of arguments has become unwieldy.
---
 erts/emulator/utils/beam_makeops | 43 ++++++++++++++++++++++++++--------------
 1 file changed, 28 insertions(+), 15 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 9da62f18ac..50dd534828 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -529,7 +529,7 @@ sub emulator_output {
 	    my($name, $hotness, @args) = @$_;
 	    my $print_name = print_name($name, @args);
 
-	    my($size, $code, $pack_spec) = cg_basic($name, @args);
+	    my($size, $code, $pack_spec) = cg_basic(name => $name, args => \@args);
 	    if (defined $code) {
                 $code = "OpCase($print_name):\n$code";
                 push @generated_code, [$hotness,$code,($print_name)];
@@ -1157,7 +1157,7 @@ sub combine_instruction_group {
                 foreach (0..$#c_args) {
                     push @first, shift @rest;
                 }
-                my $size = cg_combined_size($s, @first);
+                my $size = cg_combined_size(name => $s, args => \@first);
                 $offsets{$s} = $offset
                     unless defined $offsets{$s} and $offsets{$s} < $offset;
                 $offset += $size - 1;
@@ -1248,8 +1248,12 @@ sub combine_instruction_group {
             }
 
             my($gen_code,$down,$up) =
-                cg_combined_code($s, $flags, $offset,
-                                 $group_size-$offset, $inc, @first);
+                cg_combined_code(name => $s,
+                                 extra_comments => $flags,
+                                 offset => $offset,
+                                 comp_size => $group_size-$offset,
+                                 inc => $inc,
+                                 args =>\@first);
             my $spec_label = "$opcase$label";
             $down{$spec_label} = $down;
             $up{$spec_label} = $up;
@@ -1298,8 +1302,8 @@ sub micro_label {
 #
 
 sub cg_basic {
-    my($name,@args) = @_;
-    my($size,$code,$pack_spec) = code_gen($name, \@extended_pack_options, '', 0, undef, undef, @args);
+    my %params = (@_, pack_options => \@extended_pack_options);
+    my($size,$code,$pack_spec) = code_gen(%params);
     $pack_spec = build_pack_spec($pack_spec);
     ($size,$code,$pack_spec);
 }
@@ -1309,8 +1313,8 @@ sub cg_basic {
 #
 
 sub cg_combined_size {
-    my($name,@args) = @_;
-    my($size) = code_gen($name, \@basic_pack_options, '', 0, undef, undef, @args);
+    my %params = (@_, pack_options => \@basic_pack_options);
+    my($size) = code_gen(%params);
     $size;
 }
 
@@ -1319,9 +1323,9 @@ sub cg_combined_size {
 #
 
 sub cg_combined_code {
-    my($name,$extra,$offset,$comp_size,$inc,@args) = @_;
-    my($size,$code,$pack_spec) =
-        code_gen($name, \@basic_pack_options, $extra, $offset, $comp_size, $inc, @args);
+    my %params = (@_, pack_options => \@basic_pack_options);
+
+    my($size,$code,$pack_spec) = code_gen(%params);
     if ($pack_spec eq '') {
         ($code,'','');
     } else {
@@ -1331,8 +1335,16 @@ sub cg_combined_code {
 }
 
 sub code_gen {
-    my($name,$pack_options,$extra_comments,$offset,$comp_size,$inc,@args) = @_;
-    my $group_size = defined $comp_size ? $comp_size + $inc : undef;
+    my %params = (extra_comments => '',
+                  offset => 0,
+                  inc => 0,
+                  @_);
+    my $name = $params{name};
+    my $extra_comments = $params{extra_comments};
+    my $offset = $params{offset};
+    my $inc = $params{inc};
+    my @args = @{$params{args}};
+
     my $size = 0;
     my $flags = '';
     my @f;
@@ -1347,6 +1359,7 @@ sub code_gen {
 
     my $c_code_ref = $c_code{$name};
     if (defined $c_code_ref and $name ne 'catch') {
+        my $pack_options = $params{pack_options};
         ($var_decls, $pack_spec, @args) = do_pack($name, $offset, $pack_options, @args);
     }
 
@@ -1420,7 +1433,7 @@ sub code_gen {
         return ($size+1, undef, '');
     }
 
-    $group_size = $size unless defined $group_size;
+    my $group_size = ($params{comp_size} || $size) + $inc;
 
     #
     # Generate main body of the implementation.
@@ -1439,7 +1452,7 @@ sub code_gen {
         $bindings{$var} = $f[$i];
     }
     $bindings{'NEXT_INSTRUCTION'} = "I+" . ($group_size+$offset+1);
-    $bindings{'IP_ADJUSTMENT'} = defined $inc ? $inc : 0;
+    $bindings{'IP_ADJUSTMENT'} = $inc;
     $c_code = eval { expand_all($c_code, \%bindings) };
     unless (defined $c_code) {
         warn $@;
-- 
cgit v1.2.3


From c2e746117b24235b8f2c508ef8f90e35ad422bf2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 4 Oct 2017 20:44:47 +0200
Subject: Pack operands for combined instructions into the instruction word

The operands for the first part of a combined instruction (the entry
point following the OpCase() label) can safely be packed into the
instruction word.

This commit will make each of the following instructions one word
shorter:

bs_context_to_binary_x
i_band_xcjtd
i_bs_get_binary_all_reuse_xft
i_bs_get_integer_imm_xWtftx
i_bs_get_integer_small_imm_xWftx
i_bs_init_bits_fail_xjtx
i_bs_init_bits_fail_yjtx
i_bs_init_bits_fail_heap_sIjtx
i_bs_init_bits_heap_WItx
i_bs_init_fail_xjtx
i_bs_init_fail_yjtx
i_bs_init_fail_heap_sIjtx
i_bs_init_heap_WItx
i_bs_start_match2_xfttx
i_bs_start_match2_yfttx
i_element_xjsd
i_element_yjsd
i_fast_element_xjId
i_fast_element_yjId
i_increment_xWtd
i_increment_yWtd
i_jump_on_val_xfIW
i_jump_on_val_yfIW
i_jump_on_val_zero_xfI
i_jump_on_val_zero_yfI
i_minus_xxjtd
i_plus_xxjtd
i_plus_xyjtd
i_put_tuple_xI
i_put_tuple_yI
i_rem_xxjtd
i_select_tuple_arity_xfI
i_select_tuple_arity_yfI
i_select_tuple_arity2_xfAA
i_select_tuple_arity2_yfAA
i_select_val2_xfcc
i_select_val2_yfcc
i_select_val_bins_xfI
i_select_val_bins_yfI
i_select_val_lins_xfI
i_select_val_lins_yfI
---
 erts/emulator/utils/beam_makeops | 41 +++++++++++++++++++++++++++++++---------
 1 file changed, 32 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 50dd534828..d7791d23fa 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -1129,9 +1129,28 @@ sub combine_instruction_group {
     # Variables.
     my %offsets;
     my @instrs;
-    my %num_references;
+    my %num_references;         # Number of references from other sub instructions.
     my $group_size = 999;
 
+    #
+    # Calculate the number of references from other sub instructions.
+    # This number is useful in several ways:
+    #
+    # * If this number is 0, it is only used as the entry point for a
+    #   function, implying that it does not need a label and that operands
+    #   can be packed into the instruction word.
+    #
+    # * We'll use this number in the sort key, as a tie breaker for sub instructions
+    #   at the same instruction offset.
+    #
+    foreach my $ref_instr (@in_instrs) {
+        my(undef,undef,$first_sub,@other_subs) = @$ref_instr;
+        $num_references{$first_sub} += 0; # Make sure it is defined.
+        foreach my $sub (@other_subs) {
+            $num_references{$sub}++;
+        }
+    }
+
     # Do basic error checking. Associate operands of instructions
     # with the correct micro instructions. Calculate offsets for micro
     # instructions.
@@ -1157,12 +1176,13 @@ sub combine_instruction_group {
                 foreach (0..$#c_args) {
                     push @first, shift @rest;
                 }
-                my $size = cg_combined_size(name => $s, args => \@first);
+                my $size = cg_combined_size(name => $s,
+                                            first => $num_references{$s} == 0,
+                                            args => \@first);
                 $offsets{$s} = $offset
                     unless defined $offsets{$s} and $offsets{$s} < $offset;
                 $offset += $size - 1;
                 my $label = micro_label($s);
-                $num_references{$label} = 0;
                 push @new_subs, [$opcase,$label,$s,$size-1,@first];
                 $opcase = '';
             }
@@ -1181,9 +1201,8 @@ sub combine_instruction_group {
             my($opcase,$label,$s,$size,@args) = @{$subs[$i]};
             my $next = '';
             (undef,$next) = @{$subs[$i+1]} if $i < $#subs;
-            $num_references{$next}++ if $next;
             my $instr_info = "$opcase:$label:$next:$s:$size:@args";
-            push @all_instrs, [$label,$offsets{$s},$instr_info];
+            push @all_instrs, [$label,$s,$offsets{$s},$instr_info];
         }
     }
 
@@ -1191,8 +1210,8 @@ sub combine_instruction_group {
     my %label_to_offset;
     my %order_to_offset;
     foreach my $instr (@all_instrs) {
-        my($label,$offset,$instr_info) = @$instr;
-        my $sort_key = sprintf("%02d.%02d", $offset, $num_references{$label});
+        my($label,$s,$offset,$instr_info) = @$instr;
+        my $sort_key = sprintf("%02d.%02d", $offset, $num_references{$s});
         push @{$order_to_instrs{$sort_key}}, $instr_info;
         $label_to_offset{$label} = $offset;
         $order_to_offset{$sort_key} = $offset;
@@ -1231,7 +1250,7 @@ sub combine_instruction_group {
                 $gcode .= "OpCase($opcase):\n";
                 push @opcase_labels, $opcase;
             }
-            if ($num_references{$label}) {
+            if ($num_references{$s}) {
                 $gcode .= "$label:\n";
             }
 
@@ -1249,6 +1268,7 @@ sub combine_instruction_group {
 
             my($gen_code,$down,$up) =
                 cg_combined_code(name => $s,
+                                 first => $num_references{$s} == 0,
                                  extra_comments => $flags,
                                  offset => $offset,
                                  comp_size => $group_size-$offset,
@@ -1314,6 +1334,8 @@ sub cg_basic {
 
 sub cg_combined_size {
     my %params = (@_, pack_options => \@basic_pack_options);
+    $params{pack_options} = \@extended_pack_options
+        if $params{first};
     my($size) = code_gen(%params);
     $size;
 }
@@ -1324,7 +1346,8 @@ sub cg_combined_size {
 
 sub cg_combined_code {
     my %params = (@_, pack_options => \@basic_pack_options);
-
+    $params{pack_options} = \@extended_pack_options
+        if $params{first};
     my($size,$code,$pack_spec) = code_gen(%params);
     if ($pack_spec eq '') {
         ($code,'','');
-- 
cgit v1.2.3


From 01ee7763bcaf9192bedba2833f8b0717df1c4b23 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 6 Oct 2017 18:09:22 +0200
Subject: erts: Fix caller trace for apply bifs

Bifs that are called through the export entry
using i_call_last could have their cp set to
return_trace, just like any other call. So we
have to unwind the trace stack to get the correct
cp. Not doing this creates a lot of issues for
fprof.
---
 erts/emulator/beam/beam_bp.c            | 85 ++++++++++++++++++++-------------
 erts/emulator/test/match_spec_SUITE.erl | 48 ++++++++++++++++++-
 2 files changed, 97 insertions(+), 36 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 950639f7ae..069e7391fd 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -642,6 +642,49 @@ erts_clear_export_break(Module* modp, ErtsCodeInfo *ci)
     ASSERT(ci->u.gen_bp == NULL);
 }
 
+/*
+ * If c_p->cp is a trace return instruction, we set cp
+ * to be the place where we again start to execute code.
+ *
+ * cp is used by match spec {caller} to get the calling
+ * function, and if we don't do this fixup it will be
+ * 'undefined'. This has the odd side effect of {caller}
+ * not really being which function is the caller, but
+ * rather which function we are about to return to.
+ */
+static void fixup_cp_before_trace(Process *c_p, int *return_to_trace)
+{
+    Eterm *cpp, *E = c_p->stop;
+    BeamInstr w = *c_p->cp;
+    if (w == (BeamInstr) BeamOp(op_return_trace)) {
+        cpp = &E[2];
+    } else if (w == (BeamInstr) BeamOp(op_i_return_to_trace)) {
+        *return_to_trace = 1;
+        cpp = &E[0];
+    } else if (w == (BeamInstr) BeamOp(op_i_return_time_trace)) {
+        cpp = &E[0];
+    } else {
+        cpp = NULL;
+    }
+    if (cpp) {
+        for (;;) {
+            BeamInstr w = *cp_val(*cpp);
+            if (w == (BeamInstr) BeamOp(op_return_trace)) {
+                cpp += 3;
+            } else if (w == (BeamInstr) BeamOp(op_i_return_to_trace)) {
+                *return_to_trace = 1;
+                cpp += 1;
+            } else if (w == (BeamInstr) BeamOp(op_i_return_time_trace)) {
+                cpp += 2;
+            } else {
+                break;
+            }
+        }
+        c_p->cp = (BeamInstr *) cp_val(*cpp);
+        ASSERT(is_CP(*cpp));
+    }
+}
+
 BeamInstr
 erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg)
 {
@@ -752,6 +795,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
     GenericBp* g;
     GenericBpData* bp = NULL;
     Uint bp_flags = 0;
+    int return_to_trace = 0;
 
     ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
@@ -767,6 +811,8 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
      */
     if (!applying) {
 	p->cp = I;
+    } else {
+        fixup_cp_before_trace(p, &return_to_trace);
     }
     if (bp_flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE) &&
 	IS_TRACED_FL(p, F_TRACE_CALLS)) {
@@ -945,49 +991,20 @@ static ErtsTracer
 do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg,
 	      int local, Binary* ms, ErtsTracer tracer)
 {
-    Eterm* cpp;
     int return_to_trace = 0;
-    BeamInstr w;
     BeamInstr *cp_save = c_p->cp;
     Uint32 flags;
     Uint need = 0;
     Eterm* E = c_p->stop;
 
-    w = *c_p->cp;
-    if (w == (BeamInstr) BeamOp(op_return_trace)) {
-	cpp = &E[2];
-    } else if (w == (BeamInstr) BeamOp(op_i_return_to_trace)) {
-	return_to_trace = 1;
-	cpp = &E[0];
-    } else if (w == (BeamInstr) BeamOp(op_i_return_time_trace)) {
-	cpp = &E[0];
-    } else {
-	cpp = NULL;
-    }
-    if (cpp) {
-	for (;;) {
-	    BeamInstr w = *cp_val(*cpp);
-	    if (w == (BeamInstr) BeamOp(op_return_trace)) {
-		cpp += 3;
-	    } else if (w == (BeamInstr) BeamOp(op_i_return_to_trace)) {
-		return_to_trace = 1;
-		cpp += 1;
-	    } else if (w == (BeamInstr) BeamOp(op_i_return_time_trace)) {
-		cpp += 2;
-	    } else {
-		break;
-	    }
-	}
-	cp_save = c_p->cp;
-	c_p->cp = (BeamInstr *) cp_val(*cpp);
-	ASSERT(is_CP(*cpp));
-    }
+    fixup_cp_before_trace(c_p, &return_to_trace);
+
     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
     flags = erts_call_trace(c_p, info, ms, reg, local, &tracer);
     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
-    if (cpp) {
-	c_p->cp = cp_save;
-    }
+
+    /* restore cp after potential fixup */
+    c_p->cp = cp_save;
 
     ASSERT(!ERTS_PROC_IS_EXITING(c_p));
     if ((flags & MATCH_SET_RETURN_TO_TRACE) && !return_to_trace) {
diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl
index 92ddc23592..08a7b4560c 100644
--- a/erts/emulator/test/match_spec_SUITE.erl
+++ b/erts/emulator/test/match_spec_SUITE.erl
@@ -21,7 +21,7 @@
 -module(match_spec_SUITE).
 
 -export([all/0, suite/0, not_run/1]).
--export([test_1/1, test_2/1, test_3/1, bad_match_spec_bin/1,
+-export([test_1/1, test_2/1, test_3/1, caller_and_return_to/1, bad_match_spec_bin/1,
 	 trace_control_word/1, silent/1, silent_no_ms/1, silent_test/1,
 	 ms_trace2/1, ms_trace3/1, ms_trace_dead/1, boxed_and_small/1,
 	 destructive_in_test_bif/1, guard_exceptions/1,
@@ -47,7 +47,7 @@ suite() ->
 all() -> 
     case test_server:is_native(match_spec_SUITE) of
 	false ->
-	    [test_1, test_2, test_3, bad_match_spec_bin,
+	    [test_1, test_2, test_3, caller_and_return_to, bad_match_spec_bin,
 	     trace_control_word, silent, silent_no_ms, silent_test, ms_trace2,
 	     ms_trace3, ms_trace_dead, boxed_and_small, destructive_in_test_bif,
 	     guard_exceptions, unary_plus, unary_minus, fpe,
@@ -180,6 +180,50 @@ test_3(Config) when is_list(Config) ->
     collect(P1, [{trace, P1, call, {?MODULE, f2, [a, b]}, [true]}]),
     ok.
 
+%% Test that caller and return to work as they should
+%% There was a bug where caller would be undefined when return_to was set
+%% for r the bif erlang:put().
+caller_and_return_to(Config) ->
+    tr(
+      fun do_put_wrapper/0,
+      fun (Tracee) ->
+              MsgCaller = [{'_',[],[{message,{caller}}]}],
+              1 = erlang:trace(Tracee, true, [call,return_to]),
+              1 = erlang:trace_pattern( {?MODULE,do_put,1}, MsgCaller, [local]),
+              1 = erlang:trace_pattern( {?MODULE,do_the_put,1}, MsgCaller, [local]),
+              1 = erlang:trace_pattern( {erlang,integer_to_list,1}, MsgCaller, [local]),
+              1 = erlang:trace_pattern( {erlang,put,2}, MsgCaller, [local]),
+
+              [{trace,Tracee,call,{?MODULE,do_put,[test]},{?MODULE,do_put_wrapper,0}},
+               {trace,Tracee,call,{?MODULE,do_the_put,[test]},{?MODULE,do_put,1}},
+               {trace,Tracee,call,{erlang,integer_to_list,[1]},{?MODULE,do_the_put,1}},
+               {trace,Tracee,return_to,{?MODULE,do_the_put,1}},
+               {trace,Tracee,call,{erlang,put,[test,"1"]},{?MODULE,do_put,1}},
+               {trace,Tracee,return_to,{?MODULE,do_put,1}},
+
+               %% These last trace messages are a bit strange...
+               %% if call tracing had been enabled for do_put_wrapper
+               %% then caller and return_to would have been {?MODULE,do_put_wrapper,1}
+               %% but since it is not, they are set to do_put instead, but we still
+               %% get the do_put_wrapper return_to message...
+               {trace,Tracee,call,{erlang,integer_to_list,[2]},{?MODULE,do_put,1}},
+               {trace,Tracee,return_to,{?MODULE,do_put,1}},
+               {trace,Tracee,return_to,{?MODULE,do_put_wrapper,0}}
+              ]
+      end),
+    ok.
+
+do_put_wrapper() ->
+    do_put(test),
+    ok.
+
+do_put(Var) ->
+    do_the_put(Var),
+    erlang:integer_to_list(id(2)).
+do_the_put(Var) ->
+    Lst = erlang:integer_to_list(id(1)),
+    erlang:put(Var, Lst).
+
 otp_9422(Config) when is_list(Config) ->
     Laps = 10000,
     Fun1 = fun() -> otp_9422_tracee() end,
-- 
cgit v1.2.3


From 44681c2ff6347d7e77195cdec0bccc95ee245855 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 9 Oct 2017 10:20:15 +0200
Subject: erts: Make a copy of erl_poll.c to help debuggers

Especially lldb seems to get very confused when the
same source file is used by two different object files.
---
 erts/emulator/Makefile.in | 44 +++++++++++++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 17 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 915cad4e18..3069c5cb34 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -739,6 +739,25 @@ $(OBJDIR)/%_pu.gcda: $(OBJDIR)/PROFILE
 $(OBJDIR)/default.profdata: $(OBJDIR)/PROFILE
 	$(V_LLVM_PROFDATA) merge -output $@ $(OBJDIR)/*.profraw
 
+ifeq ($(ERTS_BUILD_FALLBACK_POLL),yes)
+# Have to treat erl_poll differently as the same .c file is used
+# twice for kernel poll builds.
+$(OBJDIR)/erl_poll.o: sys/common/erl_poll.c
+	$(V_CC) -DERTS_KERNEL_POLL_VERSION $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
+
+# Do a copy in order to make debuggers less confused
+$(TTF_DIR)/erl_poll.flbk.c: sys/common/erl_poll.c
+	$(V_at) cp $< $@
+	@touch $@
+
+$(OBJDIR)/erl_poll.flbk.o: $(TTF_DIR)/erl_poll.flbk.c
+	$(V_CC) -DERTS_NO_KERNEL_POLL_VERSION $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
+endif
+
+
+# ----------------------------------------------------------------------
+# General targets
+#
 $(OBJDIR)/%.o: beam/%.c
 	$(V_CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
 
@@ -772,12 +791,6 @@ $(BINDIR)/$(CS_EXECUTABLE): $(TTF_DIR)/GENERATED $(PRELOAD_SRC) $(CS_OBJ) $(ERTS
 	$(ld_verbose)$(CS_PURIFY) $(LD) $(CS_LDFLAGS) -o $(BINDIR)/$(CS_EXECUTABLE) \
                $(CS_CFLAGS) $(COMMON_INCLUDES) $(CS_OBJ) $(CS_LIBS)
 
-$(OBJDIR)/%.kp.o: sys/common/%.c
-	$(V_CC) -DERTS_KERNEL_POLL_VERSION $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
-
-$(OBJDIR)/%.nkp.o: sys/common/%.c
-	$(V_CC) -DERTS_NO_KERNEL_POLL_VERSION $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
-
 ifeq ($(GCC),yes)
 
 $(OBJDIR)/erl_goodfit_alloc.o: beam/erl_goodfit_alloc.c
@@ -928,14 +941,8 @@ $(STATIC_NIF_LIBS) $(STATIC_DRIVER_LIBS):
 	echo "=== Leaving lib after making static libs"
 endif
 
-ifeq ($(ERTS_BUILD_FALLBACK_POLL),yes)
-OS_OBJS += 	$(OBJDIR)/erl_poll.kp.o \
-		$(OBJDIR)/erl_poll.nkp.o
-else
-OS_OBJS +=	$(OBJDIR)/erl_poll.o
-endif
-
-OS_OBJS +=	$(OBJDIR)/erl_check_io.o \
+OS_OBJS +=	$(OBJDIR)/erl_poll.o \
+		$(OBJDIR)/erl_check_io.o \
 		$(OBJDIR)/erl_mseg.o \
 		$(OBJDIR)/erl_mmap.o \
 		$(OBJDIR)/erl_$(ERLANG_OSTYPE)_sys_ddll.o \
@@ -943,6 +950,10 @@ OS_OBJS +=	$(OBJDIR)/erl_check_io.o \
 		$(OBJDIR)/erl_sys_common_misc.o \
 		$(OBJDIR)/erl_os_monotonic_time_extender.o
 
+ifeq ($(ERTS_BUILD_FALLBACK_POLL),yes)
+OS_OBJS += 	$(OBJDIR)/erl_poll.flbk.o
+endif
+
 HIPE_ARCH64_OBJS=$(OBJDIR)/hipe_bif64.o
 
 HIPE_x86_OS_OBJS=$(HIPE_x86_$(OPSYS)_OBJS)
@@ -1099,8 +1110,7 @@ SED_REPL_O=s|^\([^:]*:\)|$$(OBJDIR)/\1|g
 SED_REPL_O_ZLIB=s|^\([^:]*:\)|$$(ZLIB_OBJDIR)/\1|g
 SED_REPL_TTF_DIR=s|$(TTF_DIR)/|$$(TTF_DIR)/|g
 SED_REPL_ERL_TOP=s|\([ 	]\)$(ERL_TOP)/|\1$$(ERL_TOP)/|g;s|^$(ERL_TOP)/|$$(ERL_TOP)/|g
-SED_REPL_POLL=s|$$(OBJDIR)/erl_poll.o|$$(OBJDIR)/erl_poll.kp.o $$(OBJDIR)/erl_poll.nkp.o|g
-SED_REPL_CHK_IO=s|$$(OBJDIR)/erl_check_io.o|$$(OBJDIR)/erl_check_io.kp.o $$(OBJDIR)/erl_check_io.nkp.o|g
+SED_REPL_POLL=s|$$(OBJDIR)/erl_poll.o|$$(OBJDIR)/erl_poll.o $$(OBJDIR)/erl_poll.flbk.o|g
 SED_REPL_TTF_COMP_FLAGS=s|\([^/]\)erl_compile_flags\.h|\1$$(TTF_DIR)/erl_compile_flags\.h|g
 
 ifeq ($(TARGET),win32)
@@ -1111,7 +1121,7 @@ SED_PREFIX=
 endif
 
 ifeq ($(ERTS_BUILD_FALLBACK_POLL),yes)
-SED_SUFFIX=;$(SED_REPL_POLL);$(SED_REPL_CHK_IO)
+SED_SUFFIX=;$(SED_REPL_POLL)
 else
 SED_SUFFIX=
 endif
-- 
cgit v1.2.3


From 4fe3f3b8be4825726fab6d4ea7515adf5b5df1ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 6 Oct 2017 08:52:32 +0200
Subject: Slightly speed up try/catch

The try_end and try_case instructions are implemented the same way
(try_case is translated to try_end by the loader).

We can do better than that. We know that try_case will only be executed
when an exception has been caught. Therefore, we know that x(0) is
the non-value and that x(1) through x(3) need to be shifted down to
x(0) through x(2). There is no need to test x(0) before shifting down.

try_end does not need the register shifting code at all.
---
 erts/emulator/beam/instrs.tab | 18 ++++++++++--------
 erts/emulator/beam/ops.tab    |  3 ++-
 2 files changed, 12 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index 07cc4bd527..20d356a81d 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -857,8 +857,7 @@ catch(Y, Fail) {
 }
 
 catch_end(Y) {
-    c_p->catches--;
-    make_blank($Y);
+    $try_end($Y);
     if (is_non_value(r(0))) {
         c_p->fvalue = NIL;
         if (x(1) == am_throw) {
@@ -888,12 +887,15 @@ catch_end(Y) {
 try_end(Y) {
     c_p->catches--;
     make_blank($Y);
-    if (is_non_value(r(0))) {
-        c_p->fvalue = NIL;
-        r(0) = x(1);
-        x(1) = x(2);
-        x(2) = x(3);
-    }
+}
+
+try_case(Y) {
+    $try_end($Y);
+    ASSERT(is_non_value(r(0)));
+    c_p->fvalue = NIL;
+    r(0) = x(1);
+    x(1) = x(2);
+    x(2) = x(3);
 }
 
 try_case_end(Src) {
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 4a915c7762..7a2c39b3a8 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -188,7 +188,8 @@ catch_end y
 
 # Try/catch.
 try Y F => catch Y F
-try_case Y => try_end Y
+
+try_case y
 try_end y
 
 %cold
-- 
cgit v1.2.3


From 5bc8c741897b9af4cfcdd80f3efcedbe6dc2c04e Mon Sep 17 00:00:00 2001
From: Kostis Sagonas <kostis@it.uu.se>
Date: Tue, 10 Oct 2017 23:33:59 +0200
Subject: Explicitly disable HiPE's range analysis

when/if compiling this file to native code to check its exception
behaiour.  Related to the discussion in #1586.
---
 erts/emulator/test/exception_SUITE.erl | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl
index 0f27251fcb..be9b63d534 100644
--- a/erts/emulator/test/exception_SUITE.erl
+++ b/erts/emulator/test/exception_SUITE.erl
@@ -31,6 +31,10 @@
 -include_lib("common_test/include/ct.hrl").
 -import(lists, [foreach/2]).
 
+%% The range analysis of the HiPE compiler results in a system limit error
+%% during compilation instead of at runtime, so do not perform this analysis.
+-compile([{hipe, [no_icode_range]}]).
+
 suite() ->
     [{ct_hooks,[ts_install_cth]},
      {timetrap, {minutes, 1}}].
-- 
cgit v1.2.3


From 0f2e313475bae2d3870333b0340e8453bbff51a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 5 Oct 2017 13:05:18 +0200
Subject: Eliminate warnings for ignoring the result of an expression

---
 erts/emulator/test/system_info_SUITE.erl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl
index 56522039da..fdf4aab24d 100644
--- a/erts/emulator/test/system_info_SUITE.erl
+++ b/erts/emulator/test/system_info_SUITE.erl
@@ -308,9 +308,9 @@ memory_test(_Config) ->
 
     mem_workers_call(MWs,
 		     fun () ->
-			     list_to_atom("an ugly atom "++integer_to_list(erlang:system_info(scheduler_id))),
-			     list_to_atom("another ugly atom "++integer_to_list(erlang:system_info(scheduler_id))),
-			     list_to_atom("yet another ugly atom "++integer_to_list(erlang:system_info(scheduler_id)))
+			     _ = list_to_atom("an ugly atom "++integer_to_list(erlang:system_info(scheduler_id))),
+			     _ = list_to_atom("another ugly atom "++integer_to_list(erlang:system_info(scheduler_id))),
+			     _ = list_to_atom("yet another ugly atom "++integer_to_list(erlang:system_info(scheduler_id)))
 		     end, []),
     cmp_memory(MWs, "new atoms"),
 
-- 
cgit v1.2.3


From 8ab02314baa4bf6fd1e3769b7222943a7084db28 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 9 Oct 2017 08:59:32 +0200
Subject: Add zlib:set_controlling_process/2

---
 erts/emulator/nifs/common/zlib_nif.c | 64 +++++++++++++++++++++++++++++-------
 1 file changed, 53 insertions(+), 11 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/nifs/common/zlib_nif.c b/erts/emulator/nifs/common/zlib_nif.c
index a9c5b05e47..fa29b4fb71 100644
--- a/erts/emulator/nifs/common/zlib_nif.c
+++ b/erts/emulator/nifs/common/zlib_nif.c
@@ -106,6 +106,7 @@ typedef struct {
     int inflateChunk_buffer_size;
 
     ErlNifPid controlling_process;
+    ErlNifMutex *controller_lock;
 
     ErlNifIOQueue *input_queue;
 
@@ -117,6 +118,8 @@ typedef struct {
 
 static ERL_NIF_TERM zlib_open(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 static ERL_NIF_TERM zlib_close(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM zlib_set_controller(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
 static ERL_NIF_TERM zlib_deflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 static ERL_NIF_TERM zlib_deflateSetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 static ERL_NIF_TERM zlib_deflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
@@ -143,6 +146,11 @@ static ERL_NIF_TERM zlib_setBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM
 static ERL_NIF_TERM zlib_enqueue_input(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 
 static ErlNifFunc nif_funcs[] = {
+    {"close_nif", 1, zlib_close},
+    {"open_nif", 0, zlib_open},
+
+    {"set_controller_nif", 2, zlib_set_controller},
+
     /* deflate */
     {"deflateInit_nif", 6, zlib_deflateInit},
     {"deflateSetDictionary_nif", 2, zlib_deflateSetDictionary},
@@ -162,10 +170,6 @@ static ErlNifFunc nif_funcs[] = {
     /* running checksum */
     {"crc32_nif", 1, zlib_crc32},
 
-    /* open & close */
-    {"close_nif", 1, zlib_close},
-    {"open_nif", 0, zlib_open},
-
     /* The stash keeps a single term alive across calls, and is used in
      * exception_on_need_dict/1 to retain the old error behavior, and for
      * saving data flushed through deflateParams/3. */
@@ -281,9 +285,7 @@ static ERL_NIF_TERM zlib_return(ErlNifEnv *env, int code) {
     return reason;
 }
 
-static void gc_zlib(ErlNifEnv *env, void* data) {
-    zlib_data_t *d = (zlib_data_t*)data;
-
+static void zlib_internal_close(zlib_data_t *d) {
     if(d->state == ST_DEFLATE) {
         deflateEnd(&d->s);
     } else if(d->state == ST_INFLATE) {
@@ -291,8 +293,6 @@ static void gc_zlib(ErlNifEnv *env, void* data) {
     }
 
     if(d->state != ST_CLOSED) {
-        enif_ioq_destroy(d->input_queue);
-
         if(d->stash_env != NULL) {
             enif_free_env(d->stash_env);
         }
@@ -301,17 +301,36 @@ static void gc_zlib(ErlNifEnv *env, void* data) {
     }
 }
 
+static void gc_zlib(ErlNifEnv *env, void* data) {
+    zlib_data_t *d = (zlib_data_t*)data;
+
+    enif_mutex_destroy(d->controller_lock);
+    enif_ioq_destroy(d->input_queue);
+
+    zlib_internal_close(d);
+
+    (void)env;
+}
+
 static int get_zlib_data(ErlNifEnv *env, ERL_NIF_TERM opaque, zlib_data_t **d) {
     return enif_get_resource(env, opaque, rtype_zlib, (void **)d);
 }
 
 static int zlib_process_check(ErlNifEnv *env, zlib_data_t *d) {
+    int is_controlling_process;
     ErlNifPid current_process;
 
     enif_self(env, &current_process);
 
-    return enif_is_identical(enif_make_pid(env, &current_process),
+    enif_mutex_lock(d->controller_lock);
+
+    is_controlling_process = enif_is_identical(
+        enif_make_pid(env, &current_process),
         enif_make_pid(env, &d->controlling_process));
+
+    enif_mutex_unlock(d->controller_lock);
+
+    return is_controlling_process;
 }
 
 static void zlib_reset_input(zlib_data_t *d) {
@@ -516,6 +535,8 @@ static ERL_NIF_TERM zlib_open(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[
 
     d->input_queue = enif_ioq_create(ERL_NIF_IOQ_NORMAL);
 
+    d->controller_lock = enif_mutex_create("zlib_controller_lock");
+
     d->s.zalloc = zlib_alloc;
     d->s.zfree  = zlib_free;
     d->s.opaque = d;
@@ -556,7 +577,28 @@ static ERL_NIF_TERM zlib_close(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv
         return enif_raise_exception(env, am_not_initialized);
     }
 
-    gc_zlib(env, d);
+    zlib_internal_close(d);
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM zlib_set_controller(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    zlib_data_t *d;
+
+    ErlNifPid new_owner;
+
+    if(argc != 2 || !get_zlib_data(env, argv[0], &d)
+                 || !enif_get_local_pid(env, argv[1], &new_owner)) {
+        return enif_make_badarg(env);
+    } else if(!zlib_process_check(env, d)) {
+        return enif_raise_exception(env, am_not_on_controlling_process);
+    }
+
+    enif_mutex_lock(d->controller_lock);
+
+    d->controlling_process = new_owner;
+
+    enif_mutex_unlock(d->controller_lock);
 
     return am_ok;
 }
-- 
cgit v1.2.3


From bb0b43eae854125688f3143e53c8974cafed4ad2 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 6 Sep 2017 17:00:14 +0200
Subject: Don't allow null chars in various strings

Various places that now reject null chars inside strings
- Primitive file operations reject it in filenames.
- Primitive environment variable operations reject it in
  names and values.
- os:cmd() reject it in its input.

Also '=' characters are rejected by primitive environment
variable operations in environment variable names.

Documentation has been updated to document null characters
in these types of data as invalid. Currently these operations
accept null chars at the end of strings, but that will change
in the future.
---
 erts/emulator/beam/erl_alloc.types |   1 +
 erts/emulator/beam/erl_bif_os.c    |  75 +++++++++++-----
 erts/emulator/beam/erl_bif_port.c  | 175 ++++++++++++++++++++++++-------------
 erts/emulator/beam/erl_unicode.c   |  97 ++++++++++++++++++--
 erts/emulator/beam/sys.h           |  48 ++++++++++
 5 files changed, 310 insertions(+), 86 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 50a1d97dd5..f8c089247e 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -285,6 +285,7 @@ type    MREF_ENT	STANDARD	SYSTEM		magic_ref_entry
 type	MREF_TAB_BKTS	STANDARD	SYSTEM		magic_ref_table_buckets
 type	MREF_TAB	LONG_LIVED	SYSTEM		magic_ref_table
 type	MINDIRECTION	FIXED_SIZE	SYSTEM		magic_indirection
+type  	OPEN_PORT_ENV 	TEMPORARY	SYSTEM		open_port_env
 
 +if threads_no_smp
 # Need thread safe allocs, but std_alloc and fix_alloc are not;
diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c
index 5cd172c26f..910325a2f4 100644
--- a/erts/emulator/beam/erl_bif_os.c
+++ b/erts/emulator/beam/erl_bif_os.c
@@ -37,6 +37,8 @@
 #include "dist.h"
 #include "erl_version.h"
 
+static int check_env_name(char *name);
+
 /*
  * Return the pid for the Erlang process in the host OS.
  */
@@ -101,8 +103,10 @@ BIF_RETTYPE os_getenv_1(BIF_ALIST_1)
     key_str = erts_convert_filename_to_native(BIF_ARG_1,buf,STATIC_BUF_SIZE,
 					      ERTS_ALC_T_TMP,1,0,&len);
 
-    if (!key_str) {
-	BIF_ERROR(p, BADARG);
+    if (!check_env_name(key_str)) {
+        if (key_str && key_str != &buf[0])
+            erts_free(ERTS_ALC_T_TMP, key_str);
+        BIF_ERROR(p, BADARG);
     }
 
     if (key_str != &buf[0])
@@ -143,25 +147,20 @@ BIF_RETTYPE os_putenv_2(BIF_ALIST_2)
 {
     char def_buf_key[STATIC_BUF_SIZE];
     char def_buf_value[STATIC_BUF_SIZE];
-    char *key_buf, *value_buf;
+    char *key_buf = NULL, *value_buf = NULL;
 
     key_buf = erts_convert_filename_to_native(BIF_ARG_1,def_buf_key,
 					      STATIC_BUF_SIZE,
 					      ERTS_ALC_T_TMP,0,0,NULL);
-    if (!key_buf) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    if (!check_env_name(key_buf))
+        goto badarg;
+
     value_buf = erts_convert_filename_to_native(BIF_ARG_2,def_buf_value,
 						STATIC_BUF_SIZE,
 						ERTS_ALC_T_TMP,1,0,
 						NULL);
-    if (!value_buf) {
-	if (key_buf != def_buf_key) {
-	    erts_free(ERTS_ALC_T_TMP, key_buf);
-	}
-	BIF_ERROR(BIF_P, BADARG);
-    }
-	    
+    if (!value_buf)
+        goto badarg;
 
     if (erts_sys_putenv(key_buf, value_buf)) {
 	if (key_buf != def_buf_key) {
@@ -179,6 +178,13 @@ BIF_RETTYPE os_putenv_2(BIF_ALIST_2)
 	erts_free(ERTS_ALC_T_TMP, value_buf);
     }
     BIF_RET(am_true);
+
+badarg:
+    if (key_buf && key_buf != def_buf_key)
+	erts_free(ERTS_ALC_T_TMP, key_buf);
+    if (value_buf && value_buf != def_buf_value)
+	erts_free(ERTS_ALC_T_TMP, value_buf);
+    BIF_ERROR(BIF_P, BADARG);
 }
 
 BIF_RETTYPE os_unsetenv_1(BIF_ALIST_1)
@@ -188,20 +194,21 @@ BIF_RETTYPE os_unsetenv_1(BIF_ALIST_1)
 
     key_buf = erts_convert_filename_to_native(BIF_ARG_1,buf,STATIC_BUF_SIZE,
 					      ERTS_ALC_T_TMP,0,0,NULL);
-    if (!key_buf) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    if (!check_env_name(key_buf))
+        goto badarg;
+
+    if (erts_sys_unsetenv(key_buf))
+        goto badarg;
 
-    if (erts_sys_unsetenv(key_buf)) {
-	if (key_buf != buf) {
-	    erts_free(ERTS_ALC_T_TMP, key_buf);
-	}
-	BIF_ERROR(BIF_P, BADARG);
-    }
     if (key_buf != buf) {
 	erts_free(ERTS_ALC_T_TMP, key_buf);
     }
     BIF_RET(am_true);
+
+badarg:
+    if (key_buf && key_buf != buf)
+        erts_free(ERTS_ALC_T_TMP, key_buf);
+    BIF_ERROR(BIF_P, BADARG);    
 }
 
 BIF_RETTYPE os_set_signal_2(BIF_ALIST_2) {
@@ -217,3 +224,27 @@ BIF_RETTYPE os_set_signal_2(BIF_ALIST_2) {
 error:
     BIF_ERROR(BIF_P, BADARG);
 }
+
+static int
+check_env_name(char *raw_name)
+{
+    byte *c = (byte *) raw_name;
+    int encoding;
+
+    if (!c)
+        return 0;
+    
+    encoding = erts_get_native_filename_encoding();
+
+    if (erts_raw_env_char_is_7bit_ascii_char('\0', c, encoding))
+        return 0; /* Do not allow empty name... */
+
+    /* Verify no '=' characters in variable name... */
+    do {
+        if (erts_raw_env_char_is_7bit_ascii_char('=', c, encoding))
+            return 0;
+        c = erts_raw_env_next_char(c, encoding);
+    } while (!erts_raw_env_char_is_7bit_ascii_char('\0', c, encoding));
+
+    return 1; /* Seems ok... */
+}
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index ff03151619..ad8be8053f 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -45,7 +45,7 @@
 #include "dtrace-wrapper.h"
 
 static Port *open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump);
-static byte* convert_environment(Process* p, Eterm env);
+static char* convert_environment(Eterm env);
 static char **convert_args(Eterm);
 static void free_args(char **);
 
@@ -726,11 +726,11 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
 			goto badarg;
 		    }
 		} else if (option == am_env) {
-		    byte* bytes;
-		    if ((bytes = convert_environment(p, *tp)) == NULL) {
+                    if (opts.envir) /* ignore previous env option... */
+                        erts_free(ERTS_ALC_T_OPEN_PORT_ENV, opts.envir);
+                    opts.envir = convert_environment(*tp);
+                    if (!opts.envir)
 			goto badarg;
-		    }
-		    opts.envir = (char *) bytes;
 		} else if (option == am_args) {
 		    char **av;
 		    char **oav = opts.argv;
@@ -964,6 +964,8 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
 	erts_atomic32_read_bor_relb(&port->state, sflgs);
  
  do_return:
+    if (opts.envir)
+        erts_free(ERTS_ALC_T_OPEN_PORT_ENV, opts.envir);
     if (name_buf)
 	erts_free(ERTS_ALC_T_TMP, (void *) name_buf);
     if (opts.argv) {
@@ -1029,74 +1031,129 @@ static void free_args(char **av)
     }
     erts_free(ERTS_ALC_T_TMP, av);
 }
-    
 
-static byte* convert_environment(Process* p, Eterm env)
+#ifdef DEBUG
+#define ERTS_CONV_ENV_BUF_EXTRA 2
+#else
+#define ERTS_CONV_ENV_BUF_EXTRA 1024
+#endif
+
+static char* convert_environment(Eterm env)
 {
-    Eterm all;
-    Eterm* temp_heap;
-    Eterm* hp;
-    Uint heap_size;
-    Sint n;
-    Sint size;
+    /*
+     * Returns environment buffer in memory allocated
+     * as ERTS_ALC_T_OPEN_PORT_ENV. Caller *needs*
+     * to deallocate...
+     */
+
+    Sint size, alloc_size;
     byte* bytes;
     int encoding = erts_get_native_filename_encoding();
 
-    if ((n = erts_list_length(env)) < 0) {
-	return NULL;
-    }
-    heap_size = 2*(5*n+1);
-    temp_heap = hp = (Eterm *) erts_alloc(ERTS_ALC_T_TMP, heap_size*sizeof(Eterm));
-    bytes = NULL;		/* Indicating error */
+    alloc_size = ERTS_CONV_ENV_BUF_EXTRA;
+    bytes = erts_alloc(ERTS_ALC_T_OPEN_PORT_ENV,
+                       alloc_size);
+    size = 0;
 
-    /*
-     * All errors below are handled by jumping to 'done', to ensure that the memory
-     * gets deallocated. Do NOT return directly from this function.
-     */
+    /* ERTS_CONV_ENV_BUF_EXTRA >= for end delimiter... */
+    ERTS_CT_ASSERT(ERTS_CONV_ENV_BUF_EXTRA >= 2);
 
-    all = CONS(hp, make_small(0), NIL);
-    hp += 2;
+    while (is_list(env)) {
+        Sint var_sz, val_sz, need;
+        byte *str, *limit;
+	Eterm tmp, *tp, *consp;
 
-    while(is_list(env)) {
-	Eterm tmp;
-	Eterm* tp;
+        consp = list_val(env);
+	tmp = CAR(consp);
+	if (is_not_tuple_arity(tmp, 2))
+	    goto error;
 
-	tmp = CAR(list_val(env));
-	if (is_not_tuple_arity(tmp, 2)) {
-	    goto done;
-	}
 	tp = tuple_val(tmp);
-	tmp = CONS(hp, make_small(0), NIL);
-	hp += 2;
-	if (tp[2] != am_false) {
-	    tmp = CONS(hp, tp[2], tmp);
-	    hp += 2;
-	}
-	tmp = CONS(hp, make_small('='), tmp);
-	hp += 2;
-	tmp = CONS(hp, tp[1], tmp);
-	hp += 2;
-	all = CONS(hp, tmp, all);
-	hp += 2;
-	env = CDR(list_val(env));
-    }
-    if (is_not_nil(env)) {
-	goto done;
-    }
 
-    if ((size = erts_native_filename_need(all,encoding)) < 0) {
-	goto done;
+        /* Check encoding of env variable... */
+        if (is_not_list(tp[1]))
+            goto error;
+        var_sz = erts_native_filename_need(tp[1], encoding);
+        if (var_sz <= 0)
+            goto error;
+        /* Check encoding of value... */
+        if (tp[2] == am_false || is_nil(tp[2]))
+            val_sz = 0;
+        else if (is_not_list(tp[2]))
+            goto error;
+        else {
+            val_sz = erts_native_filename_need(tp[2], encoding);
+            if (val_sz < 0)
+                goto error;
+        }
+
+        /* Ensure enough memory... */
+        need = size;
+        need += var_sz + val_sz;
+        /* '=' and '\0' */
+        need += 2 * erts_raw_env_7bit_ascii_char_need(encoding);
+        if (need > alloc_size) {
+            alloc_size = (need - alloc_size) + alloc_size;
+            alloc_size += ERTS_CONV_ENV_BUF_EXTRA;
+            bytes = erts_realloc(ERTS_ALC_T_OPEN_PORT_ENV,
+                                 bytes, alloc_size);
+        }
+
+        /* Write environment variable name... */
+        str = bytes + size;
+        erts_native_filename_put(tp[1], encoding, str);
+        /* empty variable name is not allowed... */
+        if (erts_raw_env_char_is_7bit_ascii_char('\0', str, encoding))
+            goto error;
+
+        /*
+         * Drop null characters at the end and verify that we do
+         * not have any '=' characters in the name...
+         */
+        limit = str + var_sz;
+        while (str < limit) {
+            if (erts_raw_env_char_is_7bit_ascii_char('\0', str, encoding))
+                break;
+            if (erts_raw_env_char_is_7bit_ascii_char('=', str, encoding))
+                goto error;
+            str = erts_raw_env_next_char(str, encoding);
+        }
+
+        /* Write the equals sign... */
+        str = erts_raw_env_7bit_ascii_char_put('=', str, encoding);
+
+        /* Write the value... */
+        if (val_sz > 0) {
+            limit = str + val_sz;
+            erts_native_filename_put(tp[2], encoding, str);
+            while (str < limit) {
+                if (erts_raw_env_char_is_7bit_ascii_char('\0', str, encoding))
+                    break;
+                str = erts_raw_env_next_char(str, encoding);
+            }
+        }
+
+        /* Delimit... */
+        str = erts_raw_env_7bit_ascii_char_put('\0', str, encoding);
+
+        size = str - bytes;
+        ASSERT(size <= alloc_size);
+
+        env = CDR(consp);
     }
 
-    /*
-     * Put the result in a binary (no risk for a memory leak that way).
-     */
-    (void) erts_new_heap_binary(p, NULL, size, &bytes);
-    erts_native_filename_put(all,encoding,bytes);
+    /* End delimit... */
+    (void) erts_raw_env_7bit_ascii_char_put('\0', &bytes[size], encoding);
+
+    if (is_nil(env))
+        return (char *) bytes;
+
+error:
+
+    if (bytes)
+        erts_free(ERTS_ALC_T_OPEN_PORT_ENV, bytes);
 
- done:
-    erts_free(ERTS_ALC_T_TMP, temp_heap);
-    return bytes;
+    return (char *) NULL; /* error... */
 }
 
 /* ------------ decode_packet() and friends: */
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 2d1d1443a7..b7a5c45fea 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -1988,7 +1988,7 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu
 	is_list(name) || 
 	(allow_empty && is_nil(name))) {
 	Sint need;
-	if ((need = erts_native_filename_need(name,encoding)) < 0) {
+	if ((need = erts_native_filename_need(name, encoding)) < 0) {
 	    return NULL;
 	}
 	if (encoding == ERL_FILENAME_WIN_WCHAR) {
@@ -2152,12 +2152,13 @@ Eterm erts_convert_native_to_filename(Process *p, byte *bytes)
 }
 
 
-Sint erts_native_filename_need(Eterm ioterm, int encoding) 
+Sint erts_native_filename_need(Eterm ioterm, int encoding)
 {
     Eterm *objp;
     Eterm obj;
     DECLARE_ESTACK(stack);
     Sint need = 0;
+    int seen_null = 0;
 
     if (is_atom(ioterm)) {
 	Atom* ap;
@@ -2194,6 +2195,22 @@ Sint erts_native_filename_need(Eterm ioterm, int encoding)
 	default:
 	    need = -1;
 	}
+        /*
+         * Do not allow null in
+         * the middle of filenames
+         */
+        if (need > 0) {
+            byte *name = ap->name;
+            int len = ap->len;
+            for (i = 0; i < len; i++) {
+                if (name[i] == 0)
+                    seen_null = 1;
+                else if (seen_null) {
+                    need = -1;
+                    break;
+                }
+            }
+        }
 	DESTROY_ESTACK(stack);
 	return need;
     }
@@ -2224,6 +2241,16 @@ L_Again:   /* Restart with sublist, old listend was pushed on stack */
 		if (is_small(obj)) { /* Always small */
 		    for(;;) {
 			Uint x = unsigned_val(obj);
+                        /*
+                         * Do not allow null in
+                         * the middle of filenames
+                         */
+                        if (x == 0)
+                            seen_null = 1;
+                        else if (seen_null) {
+                            DESTROY_ESTACK(stack);
+                            return ((Sint) -1);
+                        }
 			switch (encoding) {
 			case ERL_FILENAME_LATIN1:
 			    if (x > 255) {
@@ -2496,6 +2523,38 @@ void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars)
     }
 }
 
+/*
+ * *** Requirements on Raw Filename Format ***
+ *
+ * These requirements are due to the 'filename' module
+ * in stdlib. This since it is documented that it
+ * should be able to operate on raw filenames as well
+ * as ordinary filenames.
+ *
+ * A raw filename *must* be a byte sequence where:
+ * 1. Codepoints 0-127 (7-bit ascii) *must* be encoded
+ *    as a byte with the corresponding value. That is,
+ *    the most significant bit in the byte encoding the
+ *    codepoint is never set.
+ * 2. Codepoints greater than 127 *must* be encoded
+ *    with the most significant bit set in *every* byte
+ *    encoding it.
+ *
+ * Latin1 and UTF-8 meet these requirements while
+ * UTF-16 and UTF-32 don't.
+ *
+ * On Windows filenames are natively stored as malformed
+ * UTF-16LE (lonely surrogates may appear). A more correct
+ * description than UTF-16 would be an array of 16-bit
+ * words... In order to meet the requirements of the
+ * raw file format we convert the malformed UTF-16LE to
+ * malformed UTF-8 which meet the requirements.
+ *
+ * Note that these requirements are today only OTP
+ * internal (erts-stdlib internal) requirements that
+ * could be changed.
+ */
+
 /*
  * This internal bif converts a filename to whatever format is suitable for the file driver
  * It also adds zero termination so that prim_file needn't bother with the character encoding
@@ -2507,6 +2566,12 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
     Sint need;
     Eterm bin_term;
     byte* bin_p;
+
+    /*
+     * See comment on "Requirements on Raw Filename Format"
+     * above.
+     */
+
     /* Prim file explicitly does not allow atoms, although we could 
        very well cope with it. Instead of letting 'file' handle them,
        it would probably be more efficient to handle them here. Subject to 
@@ -2515,6 +2580,7 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
 	BIF_ERROR(BIF_P,BADARG);
     }
     if (is_binary(BIF_ARG_1)) {
+        int seen_null = 0;
 	byte *temp_alloc = NULL;
 	byte *bytes;
 	byte *err_pos;
@@ -2524,10 +2590,18 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
 	size = binary_size(BIF_ARG_1);
 	bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
 	if (encoding != ERL_FILENAME_WIN_WCHAR) {
+            Uint i;
 	    /*Add 0 termination only*/
 	    bin_term = new_binary(BIF_P, NULL, size+1);
 	    bin_p = binary_bytes(bin_term);
-	    memcpy(bin_p,bytes,size);
+            for (i = 0; i < size; i++) {
+                /* Don't allow null in the middle of filenames... */
+                if (bytes[i] == 0)
+                    seen_null = 1;
+                else if (seen_null)
+                    goto bin_name_error;
+                bin_p[i] = bytes[i];
+            }
 	    bin_p[size]=0;
 	    erts_free_aligned_binary_bytes(temp_alloc);
 	    BIF_RET(bin_term);
@@ -2541,6 +2615,11 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
 	    bin_term = new_binary(BIF_P, 0, (size+1)*2);
 	    bin_p = binary_bytes(bin_term);
 	    while (size--) {
+                /* Don't allow null in the middle of filenames... */
+                if (*bytes == 0)
+                    seen_null = 1;
+                else if (seen_null)
+                    goto bin_name_error;
 		*bin_p++ = *bytes++;
 		*bin_p++ = 0;
 	    }
@@ -2558,11 +2637,14 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
 	bin_p[num_chars*2+1] = 0;
 	erts_free_aligned_binary_bytes(temp_alloc);
 	BIF_RET(bin_term);
+    bin_name_error:
+        erts_free_aligned_binary_bytes(temp_alloc);
+        BIF_ERROR(BIF_P,BADARG);
     } /* binary */   
 	    
 
-    if ((need = erts_native_filename_need(BIF_ARG_1,encoding)) < 0) {
-	BIF_ERROR(BIF_P,BADARG);
+    if ((need = erts_native_filename_need(BIF_ARG_1, encoding)) < 0) {
+        BIF_ERROR(BIF_P,BADARG);
     }
     if (encoding == ERL_FILENAME_WIN_WCHAR) {
 	need += 2;
@@ -2596,6 +2678,11 @@ BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1)
     Eterm ret;
     int mac = 0;
 
+    /*
+     * See comment on "Requirements on Raw Filename Format"
+     * above.
+     */
+
     if (is_not_binary(BIF_ARG_1)) {
 	BIF_ERROR(BIF_P,BADARG);
     }
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 704d567337..ce751acdd3 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1341,4 +1341,52 @@ int erts_get_printable_characters(void);
 
 void erts_init_sys_common_misc(void);
 
+ERTS_GLB_INLINE Sint erts_raw_env_7bit_ascii_char_need(int encoding);
+ERTS_GLB_INLINE byte *erts_raw_env_7bit_ascii_char_put(byte c, byte *p,
+                                                       int encoding);
+ERTS_GLB_INLINE int  erts_raw_env_char_is_7bit_ascii_char(byte c, byte *p,
+                                                          int encoding);
+ERTS_GLB_INLINE byte *erts_raw_env_next_char(byte *p, int encoding);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE Sint
+erts_raw_env_7bit_ascii_char_need(int encoding)
+{
+    return (encoding == ERL_FILENAME_WIN_WCHAR) ? 2 : 1;
+}
+
+ERTS_GLB_INLINE byte *
+erts_raw_env_7bit_ascii_char_put(byte c,
+                                 byte *p,
+                                 int encoding)
+{
+    *(p++) = c;
+    if (encoding == ERL_FILENAME_WIN_WCHAR)
+        *(p++) = 0;
+    return p;
+}
+
+ERTS_GLB_INLINE int
+erts_raw_env_char_is_7bit_ascii_char(byte c,
+                                     byte *p,
+                                     int encoding)
+{
+    if (encoding == ERL_FILENAME_WIN_WCHAR)
+        return (p[0] == c) & (p[1] == 0);
+    else
+        return p[0] == c;
+}
+
+ERTS_GLB_INLINE byte *
+erts_raw_env_next_char(byte *p, int encoding)
+{
+    if (encoding == ERL_FILENAME_WIN_WCHAR)
+        return p + 2;
+    else
+        return p + 1;
+}
+
+#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
 #endif
-- 
cgit v1.2.3


From 5204e619ecc8394f399c6e40f15639dd379a9b91 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 9 Oct 2017 11:52:18 +0200
Subject: erts: Fix lock order when recv tracing trapped exit signal

---
 erts/emulator/beam/erl_process.c   |  6 +++---
 erts/emulator/test/trace_SUITE.erl | 33 +++++++++++++++++++++++++++++++--
 2 files changed, 34 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b72bac00c1..1ce2b5071c 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -13307,9 +13307,9 @@ send_exit_signal(Process *c_p,		/* current process if and only
 
     if ((state & ERTS_PSFLG_TRAP_EXIT)
 	&& (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) {
-        /* have to release the status lock in order to send the exit message */
-        erts_smp_proc_unlock(rp, *rp_locks & ERTS_PROC_LOCKS_XSIG_SEND);
-        *rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND;
+        /* have to release the status and trace lock in order to send the exit message */
+        erts_smp_proc_unlock(rp, *rp_locks & (ERTS_PROC_LOCKS_XSIG_SEND|ERTS_PROC_LOCK_TRACE));
+        *rp_locks &= ~(ERTS_PROC_LOCKS_XSIG_SEND|ERTS_PROC_LOCK_TRACE);
         if (have_seqtrace(token) && token_update)
 	    seq_trace_update_send(token_update);
 	if (is_value(exit_tuple))
diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl
index 72acd33033..a81aa64057 100644
--- a/erts/emulator/test/trace_SUITE.erl
+++ b/erts/emulator/test/trace_SUITE.erl
@@ -38,7 +38,7 @@
 	 system_monitor_long_gc_1/1, system_monitor_long_gc_2/1, 
 	 system_monitor_large_heap_1/1, system_monitor_large_heap_2/1,
 	 system_monitor_long_schedule/1,
-	 bad_flag/1, trace_delivered/1]).
+	 bad_flag/1, trace_delivered/1, trap_exit_self_receive/1]).
 
 -include_lib("common_test/include/ct.hrl").
 
@@ -61,7 +61,8 @@ all() ->
      more_system_monitor_args, system_monitor_long_gc_1,
      system_monitor_long_gc_2, system_monitor_large_heap_1,
      system_monitor_long_schedule,
-     system_monitor_large_heap_2, bad_flag, trace_delivered].
+     system_monitor_large_heap_2, bad_flag, trace_delivered,
+     trap_exit_self_receive].
 
 init_per_testcase(_Case, Config) ->
     [{receiver,spawn(fun receiver/0)}|Config].
@@ -1709,6 +1710,31 @@ trace_delivered(Config) when is_list(Config) ->
               ok
     end.
 
+%% This testcase checks that receive trace works on exit signal messages
+%% when the sender of the exit signal is the process itself.
+trap_exit_self_receive(Config) ->
+    Parent = self(),
+    Proc = spawn_link(fun() -> process(Parent) end),
+
+    1 = erlang:trace(Proc, true, ['receive']),
+    Proc ! {trap_exit_please, true},
+    {trace, Proc, 'receive', {trap_exit_please, true}} = receive_first_trace(),
+
+    %% Make the process call exit(self(), signal)
+    Reason1 = make_ref(),
+    Proc ! {exit_signal_please, Reason1},
+    {trace, Proc, 'receive', {exit_signal_please, Reason1}} = receive_first_trace(),
+    {trace, Proc, 'receive', {'EXIT', Proc, Reason1}} = receive_first_trace(),
+    receive {Proc, {'EXIT', Proc, Reason1}} -> ok end,
+    receive_nothing(),
+
+    unlink(Proc),
+    Reason2 = make_ref(),
+    Proc ! {exit_please, Reason2},
+    {trace, Proc, 'receive', {exit_please, Reason2}} = receive_first_trace(),
+    receive_nothing(),
+    ok.
+
 drop_trace_until_down(Proc, Mon) ->
     drop_trace_until_down(Proc, Mon, false, 0, 0).
 
@@ -1791,6 +1817,9 @@ process(Dest) ->
             process(Dest);
         {exit_please, Reason} ->
             exit(Reason);
+        {exit_signal_please, Reason} ->
+            exit(self(), Reason),
+            process(Dest);
         {trap_exit_please, State} ->
             process_flag(trap_exit, State),
             process(Dest);
-- 
cgit v1.2.3


From a8592b3399f1a6516a9f5c8910f588ec0f30b7c3 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 12 Oct 2017 15:17:21 +0200
Subject: erts: Remove scheduler blocking during finish_after_on_load_2

for normal case.

We still block for default trace and hipe.
---
 erts/emulator/beam/beam_bif_load.c | 53 ++++++++++++++++++++++----------------
 erts/emulator/beam/beam_load.c     |  5 ----
 erts/emulator/beam/module.h        |  2 +-
 3 files changed, 32 insertions(+), 28 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index e48415ecc4..cc1c8b6d6a 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -768,36 +768,45 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
     ErtsCodeIndex code_ix;
     Module* modp;
 
+    if (BIF_ARG_2 != am_false && BIF_ARG_2 != am_true) {
+	BIF_ERROR(BIF_P, BADARG);
+    }
+
     if (!erts_try_seize_code_write_permission(BIF_P)) {
 	ERTS_BIF_YIELD2(bif_export[BIF_finish_after_on_load_2],
 			BIF_P, BIF_ARG_1, BIF_ARG_2);
     }
 
-    /* ToDo: Use code_ix staging instead of thread blocking */
-
-    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-    erts_thr_progress_block();
-
     code_ix = erts_active_code_ix();
     modp = erts_get_module(BIF_ARG_1, code_ix);
 
-    if (!modp || !modp->on_load || !modp->on_load->code_hdr) {
-    error:
-	erts_thr_progress_unblock();
-        erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    if (!modp || !modp->on_load || !modp->on_load->code_hdr
+	|| !modp->on_load->code_hdr->on_load_function_ptr) {
+
 	erts_release_code_write_permission();
 	BIF_ERROR(BIF_P, BADARG);
     }
-    if (modp->on_load->code_hdr->on_load_function_ptr == NULL) {
-	goto error;
-    }
-    if (BIF_ARG_2 != am_false && BIF_ARG_2 != am_true) {
-	goto error;
-    }
 
     if (BIF_ARG_2 == am_true) {
+	struct m mods[1];
+	int is_blocking = 0;
 	int i, num_exps;
 
+	erts_start_staging_code_ix(0);
+	code_ix = erts_staging_code_ix();
+	modp = erts_get_module(BIF_ARG_1, code_ix);
+
+	ASSERT(modp && modp->on_load && modp->on_load->code_hdr
+	       && modp->on_load->code_hdr->on_load_function_ptr);
+
+        if (erts_is_default_trace_enabled()
+	    || IF_HIPE(hipe_need_blocking(modp))) {
+
+            erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+            erts_thr_progress_block();
+            is_blocking = 1;
+	}
+
 	/*
 	 * Make the code with the on_load function current.
 	 */
@@ -831,11 +840,13 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	    }
 	}
 	modp->curr.code_hdr->on_load_function_ptr = NULL;
-	set_default_trace_pattern(BIF_ARG_1);
-      #ifdef HIPE
-        hipe_redirect_to_module(modp);
-      #endif
-    } else if (BIF_ARG_2 == am_false) {
+
+	mods[0].modp = modp;
+	mods[0].module = BIF_ARG_1;
+        mods[0].exception = THE_NON_VALUE;
+	return staging_epilogue(BIF_P, 1, am_true, is_blocking, mods, 1, 0);
+    }
+    else if (BIF_ARG_2 == am_false) {
 	int i, num_exps;
 
 	/*
@@ -855,8 +866,6 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	    ep->beam[1] = 0;
 	}
     }
-    erts_thr_progress_unblock();
-    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     erts_release_code_write_permission();
     BIF_RET(am_true);
 }
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 3f9dc2c1aa..84e31a5913 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -812,11 +812,6 @@ erts_finish_loading(Binary* magic, Process* c_p,
     struct erl_module_instance* inst_p;
     Uint size;
 
-    /*
-     * No other process may run since we will update the export
-     * table which is not protected by any locks.
-     */
-
     ERTS_LC_ASSERT(erts_initialized == 0 || erts_has_code_write_permission() ||
 		       erts_thr_progress_is_blocking());
     /*
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index 9a81e6035b..a3f1ce1705 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -45,7 +45,7 @@ typedef struct erl_module {
     int seen;			/* Used by finish_loading() */
 
     struct erl_module_instance curr;
-    struct erl_module_instance old; /* protected by "old_code" rwlock */
+    struct erl_module_instance old; /* active protected by "old_code" rwlock */
     struct erl_module_instance* on_load;
 } Module; 
 
-- 
cgit v1.2.3


From f28539265d753bbaa473cb488e951bfe47304a01 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 12 Oct 2017 16:28:31 +0200
Subject: Revert "Merge branch 'rickard/null-char-filenames/ERL-370/OTP-14543'
 into maint"

This reverts commit 0717a2194e863f3a78595184ccc5637697f03353, reversing
changes made to 71a40658a0cef8b3e25df3a8e48a72d0563a89bf.
---
 erts/emulator/beam/erl_bif_port.c |  2 +-
 erts/emulator/beam/erl_unicode.c  | 58 ++++-----------------------------------
 erts/emulator/beam/global.h       |  2 +-
 3 files changed, 7 insertions(+), 55 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 1121a450cd..ff03151619 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -1084,7 +1084,7 @@ static byte* convert_environment(Process* p, Eterm env)
 	goto done;
     }
 
-    if ((size = erts_native_filename_need(all, encoding, 1)) < 0) {
+    if ((size = erts_native_filename_need(all,encoding)) < 0) {
 	goto done;
     }
 
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index efd2ca3db2..2d1d1443a7 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -1988,7 +1988,7 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu
 	is_list(name) || 
 	(allow_empty && is_nil(name))) {
 	Sint need;
-	if ((need = erts_native_filename_need(name, encoding, 0)) < 0) {
+	if ((need = erts_native_filename_need(name,encoding)) < 0) {
 	    return NULL;
 	}
 	if (encoding == ERL_FILENAME_WIN_WCHAR) {
@@ -2152,13 +2152,12 @@ Eterm erts_convert_native_to_filename(Process *p, byte *bytes)
 }
 
 
-Sint erts_native_filename_need(Eterm ioterm, int encoding, int allow_null) 
+Sint erts_native_filename_need(Eterm ioterm, int encoding) 
 {
     Eterm *objp;
     Eterm obj;
     DECLARE_ESTACK(stack);
     Sint need = 0;
-    int seen_null = 0;
 
     if (is_atom(ioterm)) {
 	Atom* ap;
@@ -2195,24 +2194,6 @@ Sint erts_native_filename_need(Eterm ioterm, int encoding, int allow_null)
 	default:
 	    need = -1;
 	}
-        if (!allow_null) {
-            /*
-             * Do not allow null in
-             * the middle of filenames
-             */
-            if (need > 0) {
-                byte *name = ap->name;
-                int len = ap->len;
-                for (i = 0; i < len; i++) {
-                    if (name[i] == 0)
-                        seen_null = 1;
-                    else if (seen_null) {
-                        need = -1;
-                        break;
-                    }
-                }
-            }
-        }
 	DESTROY_ESTACK(stack);
 	return need;
     }
@@ -2243,18 +2224,6 @@ L_Again:   /* Restart with sublist, old listend was pushed on stack */
 		if (is_small(obj)) { /* Always small */
 		    for(;;) {
 			Uint x = unsigned_val(obj);
-                        if (!allow_null) {
-                            /*
-                             * Do not allow null in
-                             * the middle of filenames
-                             */
-                            if (x == 0)
-                                seen_null = 1;
-                            else if (seen_null) {
-                                DESTROY_ESTACK(stack);
-                                return ((Sint) -1);
-                            }
-                        }
 			switch (encoding) {
 			case ERL_FILENAME_LATIN1:
 			    if (x > 255) {
@@ -2546,7 +2515,6 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
 	BIF_ERROR(BIF_P,BADARG);
     }
     if (is_binary(BIF_ARG_1)) {
-        int seen_null = 0;
 	byte *temp_alloc = NULL;
 	byte *bytes;
 	byte *err_pos;
@@ -2556,18 +2524,10 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
 	size = binary_size(BIF_ARG_1);
 	bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
 	if (encoding != ERL_FILENAME_WIN_WCHAR) {
-            Uint i;
 	    /*Add 0 termination only*/
 	    bin_term = new_binary(BIF_P, NULL, size+1);
 	    bin_p = binary_bytes(bin_term);
-            for (i = 0; i < size; i++) {
-                /* Don't allow null in the middle of filenames... */
-                if (bytes[i] == 0)
-                    seen_null = 1;
-                else if (seen_null)
-                    goto bin_name_error;
-                bin_p[i] = bytes[i];
-            }
+	    memcpy(bin_p,bytes,size);
 	    bin_p[size]=0;
 	    erts_free_aligned_binary_bytes(temp_alloc);
 	    BIF_RET(bin_term);
@@ -2581,11 +2541,6 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
 	    bin_term = new_binary(BIF_P, 0, (size+1)*2);
 	    bin_p = binary_bytes(bin_term);
 	    while (size--) {
-                /* Don't allow null in the middle of filenames... */
-                if (*bytes == 0)
-                    seen_null = 1;
-                else if (seen_null)
-                    goto bin_name_error;
 		*bin_p++ = *bytes++;
 		*bin_p++ = 0;
 	    }
@@ -2603,14 +2558,11 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
 	bin_p[num_chars*2+1] = 0;
 	erts_free_aligned_binary_bytes(temp_alloc);
 	BIF_RET(bin_term);
-    bin_name_error:
-        erts_free_aligned_binary_bytes(temp_alloc);
-        BIF_ERROR(BIF_P,BADARG);
     } /* binary */   
 	    
 
-    if ((need = erts_native_filename_need(BIF_ARG_1, encoding, 0)) < 0) {
-        BIF_ERROR(BIF_P,BADARG);
+    if ((need = erts_native_filename_need(BIF_ARG_1,encoding)) < 0) {
+	BIF_ERROR(BIF_P,BADARG);
     }
     if (encoding == ERL_FILENAME_WIN_WCHAR) {
 	need += 2;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index ddf7f03265..182d3aa44e 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1253,7 +1253,7 @@ void erts_init_unicode(void);
 Sint erts_unicode_set_loop_limit(Sint limit);
 
 void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) ;
-Sint erts_native_filename_need(Eterm ioterm, int encoding, int allow_null);
+Sint erts_native_filename_need(Eterm ioterm, int encoding);
 void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars);
 int erts_analyze_utf8(byte *source, Uint size, 
 			byte **err_pos, Uint *num_chars, int *left);
-- 
cgit v1.2.3


From 19c5a5f7f15d02395694b32090d1b24c3934c5d8 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 22 May 2017 17:28:11 +0200
Subject: erts: Use SIGRTMIN on linux for sys_suspend

The Linux real-time signal is better used on Linux
for suspending schedulers during shutdown as it doesn't
collide with SIGUSR2 usage of other applications.
---
 erts/emulator/beam/erl_bif_info.c |   6 ++
 erts/emulator/beam/sys.h          |  11 +++-
 erts/emulator/test/Makefile       |   1 +
 erts/emulator/test/dump_SUITE.erl | 125 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 140 insertions(+), 3 deletions(-)
 create mode 100644 erts/emulator/test/dump_SUITE.erl

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 2ff95a3338..162d179982 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3699,6 +3699,12 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 		Eterm *hp = HAlloc(BIF_P, hsz);
 		BIF_RET(uword_to_big(size, hp));
 	    }
+        } else if (ERTS_IS_ATOM_STR("scheduler_dump", BIF_ARG_1)) {
+#if defined(ERTS_HAVE_TRY_CATCH) && defined(ERTS_SYS_SUSPEND_SIGNAL)
+            BIF_RET(am_true);
+#else
+            BIF_RET(am_false);
+#endif
         }
     }
     else if (is_tuple(BIF_ARG_1)) {
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 615f44364b..9dc339053f 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -832,11 +832,16 @@ int erts_sys_unsetenv(char *key);
 char *erts_read_env(char *key);
 void erts_free_read_env(void *value);
 
-#if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX)
+#if defined(ERTS_THR_HAVE_SIG_FUNCS) &&                         \
+    (!defined(ETHR_UNUSABLE_SIGUSRX) || defined(SIGRTMIN))
 extern void sys_thr_resume(erts_tid_t tid);
 extern void sys_thr_suspend(erts_tid_t tid);
-#define ERTS_SYS_SUSPEND_SIGNAL SIGUSR2
-#endif
+#ifdef SIGRTMIN
+#define ERTS_SYS_SUSPEND_SIGNAL (SIGRTMIN+1)
+#else
+#define ERTS_SYS_SUSPEND_SIGNAL (SIGUSR2)
+#endif /* SIGRTMIN */
+#endif /* HAVE_SIG_FUNCS */
 
 /* utils.c */
 
diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index 370fcb0f3a..5898a8ee8f 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -57,6 +57,7 @@ MODULES= \
 	dirty_nif_SUITE \
 	distribution_SUITE \
 	driver_SUITE \
+	dump_SUITE \
 	efile_SUITE \
 	erts_debug_SUITE \
 	estone_SUITE \
diff --git a/erts/emulator/test/dump_SUITE.erl b/erts/emulator/test/dump_SUITE.erl
new file mode 100644
index 0000000000..38fa198ea6
--- /dev/null
+++ b/erts/emulator/test/dump_SUITE.erl
@@ -0,0 +1,125 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2005-2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(dump_SUITE).
+
+-include_lib("common_test/include/ct.hrl").
+
+-export([all/0, suite/0, init_per_testcase/2, end_per_testcase/2]).
+
+-export([signal_abort/1]).
+
+-export([load/0]).
+
+-include_lib("kernel/include/file.hrl").
+
+suite() ->
+    [{ct_hooks,[ts_install_cth]},
+     {timetrap, {minutes, 2}}].
+
+all() ->
+    [signal_abort].
+
+init_per_testcase(signal_abort, Config) ->
+    SO = erlang:system_info(schedulers_online),
+    erts_debug:set_internal_state(available_internal_state, true),
+    Dump = erts_debug:get_internal_state(scheduler_dump),
+    erts_debug:set_internal_state(available_internal_state, false),
+    if SO < 3 ->
+            {skip, "not enough schedulers"};
+       not Dump ->
+            {skip, "the platform does not support scheduler dump"};
+       Dump ->
+            Config
+    end.
+
+end_per_testcase(_, Config) ->
+    Config.
+
+%%%
+%%% The test cases -------------------------------------------------------------
+%%%
+
+%% Test that a snapshot is taken of other schedulers using a signal
+%% when a crash dump is generated.
+signal_abort(Config) ->
+
+    Dump = filename:join(proplists:get_value(priv_dir, Config),"signal_abort.dump"),
+
+    {ok, Node} = start_node(Config),
+
+    _P1 = spawn(Node, ?MODULE, load, []),
+    _P2 = spawn(Node, ?MODULE, load, []),
+    _P3 = spawn(Node, ?MODULE, load, []),
+    _P4 = spawn(Node, ?MODULE, load, []),
+    _P5 = spawn(Node, ?MODULE, load, []),
+    _P6 = spawn(Node, ?MODULE, load, []),
+
+    timer:sleep(500),
+
+    true = rpc:call(Node, os, putenv, ["ERL_CRASH_DUMP",Dump]),
+    rpc:call(Node, erlang, halt, ["dump"]),
+
+    {ok, Bin} = get_dump_when_done(Dump),
+
+    ct:log("~s",[Bin]),
+
+    {match, Matches} = re:run(Bin,"Current Process: <",[global]),
+
+    ct:log("Found ~p",[Matches]),
+
+    true = length(Matches) > 1,
+
+    file:delete(Dump),
+
+    ok.
+
+get_dump_when_done(Dump) ->
+    case file:read_file_info(Dump) of
+        {ok, #file_info{ size = Sz }} ->
+            get_dump_when_done(Dump, Sz);
+        {error, enoent} ->
+            timer:sleep(100),
+            get_dump_when_done(Dump)
+    end.
+
+get_dump_when_done(Dump, Sz) ->
+    timer:sleep(100),
+    case file:read_file_info(Dump) of
+        {ok, #file_info{ size = Sz }} ->
+            file:read_file(Dump);
+        {ok, #file_info{ size = NewSz }} ->
+            get_dump_when_done(Dump, NewSz)
+    end.
+
+load() ->
+    lists:seq(1,10000),
+    load().
+
+start_node(Config) when is_list(Config) ->
+    Pa = filename:dirname(code:which(?MODULE)),
+    Name = list_to_atom(atom_to_list(?MODULE)
+                        ++ "-"
+                        ++ atom_to_list(proplists:get_value(testcase, Config))
+                        ++ "-"
+                        ++ integer_to_list(erlang:system_time(second))
+                        ++ "-"
+                        ++ integer_to_list(erlang:unique_integer([positive]))),
+    test_server:start_node(Name, slave, [{args, "-pa "++Pa}]).
-- 
cgit v1.2.3


From 20c2b1b174b44ed24afe73b3ac604e14d8c9318f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 5 Oct 2017 13:10:42 +0200
Subject: Eliminate use of deprecated functions in string

---
 erts/emulator/test/alloc_SUITE.erl         |  9 +++++----
 erts/emulator/test/bif_SUITE.erl           | 10 ++++++----
 erts/emulator/test/decode_packet_SUITE.erl | 12 ++++++------
 erts/emulator/test/distribution_SUITE.erl  |  6 +++---
 erts/emulator/test/driver_SUITE.erl        |  6 +++---
 erts/emulator/test/num_bif_SUITE.erl       |  2 +-
 erts/emulator/test/port_SUITE.erl          | 12 +++---------
 erts/emulator/test/scheduler_SUITE.erl     |  6 ++----
 8 files changed, 29 insertions(+), 34 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl
index f0871ead7d..5e5cd6f578 100644
--- a/erts/emulator/test/alloc_SUITE.erl
+++ b/erts/emulator/test/alloc_SUITE.erl
@@ -89,10 +89,11 @@ mmsc_flags() ->
 mmsc_flags(Env) ->
     case os:getenv(Env) of
 	false -> false;
-	V -> case string:str(V, "+MMsc") of
-		 0 -> false;
-		 P -> Env ++ "=" ++ string:substr(V, P)
-	     end
+	V ->
+            case string:find(V, "+MMsc") of
+                nomatch -> false;
+                SubStr -> Env ++ "=" ++ SubStr
+            end
     end.
 
 erts_mmap_do(Config, SCO, SCRPM, SCRFSD) ->
diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl
index 04b7f2de15..146c37b118 100644
--- a/erts/emulator/test/bif_SUITE.erl
+++ b/erts/emulator/test/bif_SUITE.erl
@@ -731,10 +731,12 @@ erlang_halt(Config) when is_list(Config) ->
                                  [broken_halt, "Validate correct crash dump"]),
     {ok,_} = wait_until_stable_size(CrashDump,-1),
     {ok, Bin} = file:read_file(CrashDump),
-    case {string:str(binary_to_list(Bin),"\n=end\n"),
-          string:str(binary_to_list(Bin),"\r\n=end\r\n")} of
-        {0,0} -> ct:fail("Could not find end marker in crash dump");
-        _ -> ok
+    case {string:find(Bin, <<"\n=end\n">>),
+          string:find(Bin, <<"\r\n=end\r\n">>)} of
+        {nomatch,nomatch} ->
+            ct:fail("Could not find end marker in crash dump");
+        {_,_} ->
+            ok
     end.
 
 wait_until_stable_size(_File,-10) ->
diff --git a/erts/emulator/test/decode_packet_SUITE.erl b/erts/emulator/test/decode_packet_SUITE.erl
index 54ee4d5567..0ccdbd7ee8 100644
--- a/erts/emulator/test/decode_packet_SUITE.erl
+++ b/erts/emulator/test/decode_packet_SUITE.erl
@@ -239,7 +239,7 @@ packet_size(Config) when is_list(Config) ->
     %% Test OTP-9389, long HTTP header lines.
     Opts = [{packet_size, 128}],
     Pkt = list_to_binary(["GET / HTTP/1.1\r\nHost: localhost\r\nLink: /",
-                          string:chars($Y, 64), "\r\n\r\n"]),
+                          lists:duplicate(64, $Y), "\r\n\r\n"]),
     <<Pkt1:50/binary, Pkt2/binary>> = Pkt,
     {ok, {http_request,'GET',{abs_path,"/"},{1,1}}, Rest1} =
     erlang:decode_packet(http, Pkt1, Opts),
@@ -250,7 +250,7 @@ packet_size(Config) when is_list(Config) ->
     erlang:decode_packet(httph, list_to_binary([Rest2, Pkt2]), Opts),
 
     Pkt3 = list_to_binary(["GET / HTTP/1.1\r\nHost: localhost\r\nLink: /",
-                           string:chars($Y, 129), "\r\n\r\n"]),
+                           lists:duplicate(129, $Y), "\r\n\r\n"]),
     {ok, {http_request,'GET',{abs_path,"/"},{1,1}}, Rest3} =
     erlang:decode_packet(http, Pkt3, Opts),
     {ok, {http_header,_,'Host',_,"localhost"}, Rest4} =
@@ -509,9 +509,9 @@ decode_line(Bin,MaxLen) ->
     end.
 
 find_in_binary(Byte, Bin) ->
-    case string:chr(binary_to_list(Bin),Byte) of
-        0 -> notfound;	    
-        P -> P
+    case string:find(Bin, [Byte]) of
+        nomatch -> notfound;
+        Suffix -> byte_size(Bin) - byte_size(Suffix) + 1
     end.    
 
 ssl(Config) when is_list(Config) ->
@@ -562,7 +562,7 @@ decode_pkt(Type,Bin,Opts) ->
 otp_9389(Config) when is_list(Config) ->
     Opts = [{packet_size, 16384}, {line_length, 3000}],
     Pkt = list_to_binary(["GET / HTTP/1.1\r\nHost: localhost\r\nLink: /",
-                          string:chars($X, 8192),
+                          lists:duplicate(8192, $X),
                           "\r\nContent-Length: 0\r\n\r\n"]),
     <<Pkt1:5000/binary, Pkt2/binary>> = Pkt,
     {ok, {http_request,'GET',{abs_path,"/"},{1,1}}, Rest1} =
diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index 17cd1d1a3b..2d0ae9c83e 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -786,8 +786,8 @@ dist_auto_connect_once(Config) when is_list(Config) ->
     {ok, pong} = do_inet_rpc(Sock2,net_adm,ping,[NN]),
     {ok,[NN2]} = do_inet_rpc(Sock,erlang,nodes,[]),
     {ok,[NN]} = do_inet_rpc(Sock2,erlang,nodes,[]),
-    [_,HostPartPeer] = string:tokens(atom_to_list(NN),"@"),
-    [_,MyHostPart] = string:tokens(atom_to_list(node()),"@"),
+    [_,HostPartPeer] = string:lexemes(atom_to_list(NN),"@"),
+    [_,MyHostPart] = string:lexemes(atom_to_list(node()),"@"),
     % Give net_kernel a chance to change the state of the node to up to.
     receive after 1000 -> ok end,
     case HostPartPeer of
@@ -2094,7 +2094,7 @@ start_relay_node(Node, Args) ->
                                       [{args, Args ++
                                         " -setcookie "++Cookie++" -pa "++Pa++" "++
                                         RunArg}]),
-    [N,H] = string:tokens(atom_to_list(NN),"@"),
+    [N,H] = string:lexemes(atom_to_list(NN),"@"),
     {ok, Sock} = gen_tcp:accept(LSock),
     pang = net_adm:ping(NN),
     {N,H,Sock}.
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 33d0b708cf..475e03087a 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -1192,9 +1192,9 @@ check_driver_system_info_result(Result) ->
     io:format("All names: ~p~n", [?EXPECTED_SYSTEM_INFO_NAMES]),
     io:format("Result: ~p~n", [Result]),
     {[], Ns, DDVSN} = chk_sis(lists:map(fun (Str) ->
-                                                string:tokens(Str, "=")
+                                                string:lexemes(Str, "=")
                                         end,
-                                        string:tokens(Result, " ")),
+                                        string:lexemes(Result, " ")),
                               ?EXPECTED_SYSTEM_INFO_NAMES),
     case {DDVSN,
           drv_vsn_str2tup(erlang:system_info(driver_version))} of
@@ -2438,7 +2438,7 @@ wait_until(Fun) ->
     end.
 
 drv_vsn_str2tup(Str) ->
-    [Major, Minor] = string:tokens(Str, "."),
+    [Major, Minor] = string:lexemes(Str, "."),
     {list_to_integer(Major), list_to_integer(Minor)}.
 
 %% Build port data from a template.
diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl
index 1c76eb8019..17555d63c6 100644
--- a/erts/emulator/test/num_bif_SUITE.erl
+++ b/erts/emulator/test/num_bif_SUITE.erl
@@ -145,7 +145,7 @@ t_float_to_string(Config) when is_list(Config) ->
 				     123456789012345678.0, [{decimals, 237}])),
     {'EXIT', {badarg, _}} = (catch float_to_binary(
 				     123456789012345678.0, [{decimals, 237}])),
-    test_fts("1." ++ string:copies("0", 249) ++ "e+00",
+    test_fts("1." ++ lists:duplicate(249, $0) ++ "e+00",
 	     1.0,  [{scientific, 249}, compact]),
 
     X1 = float_to_list(1.0),
diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index 730a17d7e8..f29528a22f 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -1662,13 +1662,7 @@ spawn_executable(Config) when is_list(Config) ->
     [ExactFile2,"hello world","dlrow olleh"] =
     run_echo_args_2(unicode:characters_to_binary("\""++ExactFile2++"\" "++"\"hello world\" \"dlrow olleh\"")),
 
-    ExeExt =
-    case string:to_lower(lists:last(string:tokens(ExactFile2,"."))) of
-        "exe" ->
-            ".exe";
-        _ ->
-            ""
-    end,
+    ExeExt = filename:extension(ExactFile2),
     Executable2 = "spoky name"++ExeExt,
     file:copy(ExactFile1,filename:join([SpaceDir,Executable2])),
     ExactFile3 = filename:nativename(filename:join([SpaceDir,Executable2])),
@@ -1836,7 +1830,7 @@ collect_data(Port) ->
     end.
 
 parse_echo_args_output(Data) ->
-    [lists:last(string:tokens(S,"|")) || S <- string:tokens(Data,"\r\n")].
+    [lists:last(string:lexemes(S,"|")) || S <- string:lexemes(Data,["\r\n",$\n])].
 
 %% Test that the emulator does not mix up ports when the port table wraps
 mix_up_ports(Config) when is_list(Config) ->
@@ -1962,7 +1956,7 @@ max_ports() ->
     erlang:system_info(port_limit).
 
 port_ix(Port) when is_port(Port) ->
-    ["#Port",_,PortIxStr] = string:tokens(erlang:port_to_list(Port),
+    ["#Port",_,PortIxStr] = string:lexemes(erlang:port_to_list(Port),
                                           "<.>"),
     list_to_integer(PortIxStr).
 
diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl
index 7afb82a1b8..7eebbe8b19 100644
--- a/erts/emulator/test/scheduler_SUITE.erl
+++ b/erts/emulator/test/scheduler_SUITE.erl
@@ -894,11 +894,9 @@ adjust_schedulers_online() ->
 
 read_affinity(Data) ->
     Exp = "pid " ++ os:getpid() ++ "'s current affinity mask",
-    case string:tokens(Data, ":") of
+    case string:lexemes(Data, ":") of
 	[Exp, DirtyAffinityStr] ->
-	    AffinityStr = string:strip(string:strip(DirtyAffinityStr,
-						    both, $ ),
-				       both, $\n),
+	    AffinityStr = string:trim(DirtyAffinityStr),
 	    case catch erlang:list_to_integer(AffinityStr, 16) of
 		Affinity when is_integer(Affinity) ->
 		    Affinity;
-- 
cgit v1.2.3


From 513a322941d208d9dcdc4c39db2966ae4c707fe7 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 19 Sep 2017 14:37:59 +0200
Subject: erts: Implement map iterator using a stack

This version does not work great as the subtrees
created are not proper hash maps. Also it is not
all that performant as the extra allocations to
keep the stack there is expensive.
---
 erts/emulator/beam/bif.tab   |  1 +
 erts/emulator/beam/erl_map.c | 72 ++++++++++++++++++++++++++++++++++++++++++--
 erts/emulator/beam/erl_map.h |  3 +-
 3 files changed, 73 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index f7b4451890..4ee5af14bc 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -691,3 +691,4 @@ bif erts_internal:maps_to_list/2
 #
 
 bif erlang:iolist_to_iovec/1
+bif erts_internal:map_get_index/2
\ No newline at end of file
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index f0c54e05f7..37fcedbc6d 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -90,7 +90,6 @@ static BIF_RETTYPE hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_
                                  struct HashmapMergeContext_*);
 static Export hashmap_merge_trap_export;
 static BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1);
-static Uint hashmap_subtree_size(Eterm node);
 static Eterm hashmap_to_list(Process *p, Eterm map, Sint n);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
@@ -1519,7 +1518,7 @@ trap:  /* Yield */
     return trap_ret;
 }
 
-static Uint hashmap_subtree_size(Eterm node) {
+Uint hashmap_subtree_size(Eterm node) {
     DECLARE_WSTACK(stack);
     Uint size = 0;
 
@@ -3060,6 +3059,75 @@ static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[])
     return res;
 }
 
+static Uint32
+index_to_hash(Sint idx, Uint32 hval)
+{
+    Uint32 new_hval = 1;
+    while (1) {
+        if (hval & 0x1) {
+            idx--;
+            if (!idx) break;
+        }
+        new_hval = new_hval << 1;
+        hval = hval >> 1;
+    }
+    return new_hval;
+}
+
+BIF_RETTYPE erts_internal_map_get_index_2(BIF_ALIST_2) {
+    Sint idx;
+    Uint32 hval, sz;
+    Eterm *ptr, hdr;
+
+    if (!is_hashmap(BIF_ARG_2) || is_not_small(BIF_ARG_1))
+        BIF_ERROR(BIF_P, BADARG);
+
+    idx = signed_val(BIF_ARG_1);
+
+    if (idx < 1 || idx > 17)
+        BIF_ERROR(BIF_P, BADARG);
+
+    ASSERT(is_boxed(BIF_ARG_2));
+
+    ptr = boxed_val(BIF_ARG_2);
+again:
+    hdr = *ptr;
+
+    switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+    case HAMT_SUBTAG_HEAD_ARRAY:
+        ptr++;
+        sz = 16;
+        hval = 0xffff;
+        break;
+    case HAMT_SUBTAG_HEAD_BITMAP:
+        ptr++;
+        if (ptr[0] == -1) {
+            ptr = boxed_val(ptr[1]);
+            goto again;
+        }
+    case HAMT_SUBTAG_NODE_BITMAP:
+        hval = MAP_HEADER_VAL(hdr);
+        sz = hashmap_bitcount(hval);
+        ASSERT(sz < 17);
+        break;
+    default:
+        erts_exit(ERTS_ABORT_EXIT, "bad header");
+    }
+
+    if (idx <= sz) {
+        if (is_list(ptr[idx]))
+            return ptr[idx];
+        else {
+            Eterm *hp = HAlloc(BIF_P, HAMT_HEAD_BITMAP_SZ(1));
+            hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(index_to_hash(idx, hval));
+            hp[1] = -1;
+            hp[2] = ptr[idx];
+            return make_hashmap(hp);
+        }
+    } else {
+        return am_none;
+    }
+}
 
 /* implementation of builtin emulations */
 
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index c3ccf80b85..d910e98398 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -56,7 +56,7 @@ typedef struct flatmap_s {
 /* the head-node is a bitmap or array with an untagged size */
 
 
-#define hashmap_size(x)               (((hashmap_head_t*) hashmap_val(x))->size)
+#define hashmap_size(x)               (((hashmap_head_t*) hashmap_val(x))->size == -1 ? hashmap_subtree_size(x) : ((hashmap_head_t*) hashmap_val(x))->size)
 #define hashmap_make_hash(Key)        make_internal_hash(Key, 0)
 
 #define hashmap_restore_hash(Heap,Lvl,Key) \
@@ -106,6 +106,7 @@ Eterm  erts_hashmap_from_ks_and_vs_extra(ErtsHeapFactory *factory,
                                          Eterm *ks, Eterm *vs, Uint n,
 					 Eterm k, Eterm v);
 
+Uint hashmap_subtree_size(Eterm node);
 const Eterm *erts_maps_get(Eterm key, Eterm map);
 
 const Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
-- 
cgit v1.2.3


From 0149a73d15df1f80cb46752ec3829f48c38dd230 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 21 Sep 2017 09:20:30 +0200
Subject: erts: Implement maps path iterator

---
 erts/emulator/beam/bif.tab       |   8 +-
 erts/emulator/beam/erl_map.c     | 288 +++++++++++++++++++++++++++++++--------
 erts/emulator/beam/erl_map.h     |   4 +-
 erts/emulator/test/map_SUITE.erl |  51 +++++++
 4 files changed, 289 insertions(+), 62 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 4ee5af14bc..f2c8331850 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -689,6 +689,10 @@ bif erts_internal:maps_to_list/2
 #
 # New in 20.1
 #
-
 bif erlang:iolist_to_iovec/1
-bif erts_internal:map_get_index/2
\ No newline at end of file
+
+#
+# New in 21.0
+#
+
+bif erts_internal:map_next/2
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 37fcedbc6d..a5fda7b86d 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -90,6 +90,7 @@ static BIF_RETTYPE hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_
                                  struct HashmapMergeContext_*);
 static Export hashmap_merge_trap_export;
 static BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1);
+static Uint hashmap_subtree_size(Eterm node);
 static Eterm hashmap_to_list(Process *p, Eterm map, Sint n);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
@@ -1518,7 +1519,7 @@ trap:  /* Yield */
     return trap_ret;
 }
 
-Uint hashmap_subtree_size(Eterm node) {
+static Uint hashmap_subtree_size(Eterm node) {
     DECLARE_WSTACK(stack);
     Uint size = 0;
 
@@ -3059,74 +3060,247 @@ static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[])
     return res;
 }
 
-static Uint32
-index_to_hash(Sint idx, Uint32 hval)
-{
-    Uint32 new_hval = 1;
-    while (1) {
-        if (hval & 0x1) {
-            idx--;
-            if (!idx) break;
-        }
-        new_hval = new_hval << 1;
-        hval = hval >> 1;
-    }
-    return new_hval;
-}
 
-BIF_RETTYPE erts_internal_map_get_index_2(BIF_ALIST_2) {
-    Sint idx;
-    Uint32 hval, sz;
-    Eterm *ptr, hdr;
+/*
+ * The iterator work in about the same way for both
+ * flat- and hash-maps. It consists of a Path, Map
+ * term where Path describes where to find the
+ * term to get.
+ *
+ * In flatmap Path is just the index of the element
+ * in the flatmap array.
+ *
+ * In hashmap the Path is a bit pattern that describes
+ * which slot we should traverse in each hashmap node.
+ * Since each hashmap node can only be up to 16 elements
+ * large we use 4 bits per level in the path.
+ *
+ * So a Path with value 0x110 will first get the 0:th
+ * slot in the head node, and then the 1:st slot in the
+ * resulting node and then finally the 1:st slot in the
+ * node beneath. If that slot is not a leaf, then the path
+ * continues down the 0:th slot until it finds a leaf and
+ * returns that leaf.
+ *
+ * Once the leaf has been found, the Path to the next leaf
+ * is built using a stack that was created while traversing
+ * the tree down.
+ *
+ * The index can become a bignum, which complicates the code
+ * a bit. However it should be very rare that this happens
+ * even on a 32bit system as you would need a tree of depth
+ * 7 or more.
+ */
 
-    if (!is_hashmap(BIF_ARG_2) || is_not_small(BIF_ARG_1))
-        BIF_ERROR(BIF_P, BADARG);
+#define PATH_ELEM_SIZE 4
+#define PATH_ELEM_MASK 0xf
+#define PATH_ELEM(PATH) ((PATH) & PATH_ELEM_MASK)
+#define PATH_ELEMS_PER_DIGIT (sizeof(ErtsDigit) * 8 / PATH_ELEM_SIZE)
 
-    idx = signed_val(BIF_ARG_1);
+BIF_RETTYPE erts_internal_map_next_2(BIF_ALIST_2) {
 
-    if (idx < 1 || idx > 17)
+    Eterm *hp = NULL, *lst,
+        iter, path, map, k, v;
+
+    path = BIF_ARG_1;
+    map  = BIF_ARG_2;
+
+    if (!is_map(map))
         BIF_ERROR(BIF_P, BADARG);
 
-    ASSERT(is_boxed(BIF_ARG_2));
+    if (path == am_none)
+        BIF_RET(am_none);
 
-    ptr = boxed_val(BIF_ARG_2);
-again:
-    hdr = *ptr;
+    if (is_flatmap(map)) {
+        flatmap_t *mp = (flatmap_t *)flatmap_val(map);
+        Uint idx, sz = flatmap_get_size(mp);
 
-    switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-    case HAMT_SUBTAG_HEAD_ARRAY:
-        ptr++;
-        sz = 16;
-        hval = 0xffff;
-        break;
-    case HAMT_SUBTAG_HEAD_BITMAP:
-        ptr++;
-        if (ptr[0] == -1) {
-            ptr = boxed_val(ptr[1]);
-            goto again;
+        if (is_not_small(path))
+            BIF_ERROR(BIF_P, BADARG);
+
+        idx = unsigned_val(path);
+
+        if (sz == 0) {
+            if (idx == 0)
+                BIF_RET(am_none);
+            else
+                BIF_ERROR(BIF_P, BADARG);
         }
-    case HAMT_SUBTAG_NODE_BITMAP:
-        hval = MAP_HEADER_VAL(hdr);
-        sz = hashmap_bitcount(hval);
-        ASSERT(sz < 17);
-        break;
-    default:
-        erts_exit(ERTS_ABORT_EXIT, "bad header");
-    }
-
-    if (idx <= sz) {
-        if (is_list(ptr[idx]))
-            return ptr[idx];
-        else {
-            Eterm *hp = HAlloc(BIF_P, HAMT_HEAD_BITMAP_SZ(1));
-            hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(index_to_hash(idx, hval));
-            hp[1] = -1;
-            hp[2] = ptr[idx];
-            return make_hashmap(hp);
+
+        k = flatmap_get_keys(mp)[idx];
+        v = flatmap_get_values(mp)[idx];
+
+        if (idx + 1 < sz) {
+            path = make_small(idx + 1);
+        } else if (idx < sz) {
+            path = am_none;
+        } else {
+            BIF_ERROR(BIF_P, BADARG);
         }
     } else {
-        return am_none;
+        Uint curr_path;
+        Uint path_length = 0;
+        Uint *path_rest = NULL;
+        int i;
+        Eterm node = map;
+        DECLARE_WSTACK(stack);
+
+        ASSERT(is_hashmap(node));
+
+        if (is_small(path)) {
+            curr_path = unsigned_val(path);
+        } else if (is_big(path)) {
+            Eterm *big = big_val(path);
+            if (bignum_header_is_neg(*big))
+                BIF_ERROR(BIF_P, BADARG);
+            path_length = BIG_ARITY(big) - 1;
+            curr_path = BIG_DIGIT(big,  0);
+            path_rest = BIG_V(big) + 1;
+        } else {
+            BIF_ERROR(BIF_P, BADARG);
+        }
+
+        /* First we look for the K and V to return using the
+           path given. While doing so, we push each map node
+           and the index onto the stack so use later when
+           building the next index. */
+        for (i = 1; ; i++) {
+            Eterm *ptr = hashmap_val(node),
+                hdr = *ptr;
+            Uint sz;
+
+            ptr++;
+
+            switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+	    case HAMT_SUBTAG_HEAD_ARRAY:
+		ptr++;
+		sz = 16;
+		break;
+	    case HAMT_SUBTAG_HEAD_BITMAP:
+		ptr++;
+	    case HAMT_SUBTAG_NODE_BITMAP:
+		sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+		ASSERT(sz < 17);
+		break;
+            default:
+		erts_exit(ERTS_ABORT_EXIT, "bad header");
+	    }
+
+            if (PATH_ELEM(curr_path) >= sz)
+                BIF_ERROR(BIF_P, BADARG);
+
+            WSTACK_PUSH2(stack, node, PATH_ELEM(curr_path));
+
+            /* We have found a leaf, return it and calculate the path to next */
+            if (is_list(ptr[PATH_ELEM(curr_path)])) {
+                lst = list_val(ptr[PATH_ELEM(curr_path)]);
+                k = CAR(lst);
+                v = CDR(lst);
+                break;
+            }
+
+            node = ptr[PATH_ELEM(curr_path)];
+
+            curr_path >>= PATH_ELEM_SIZE;
+
+            if (i == PATH_ELEMS_PER_DIGIT) {
+                /* Switch to next bignum word if available,
+                   otherwise just follow 0 path */
+                i = 0;
+                if (path_length) {
+                    curr_path = *path_rest;
+                    path_length--;
+                    path_rest++;
+                } else {
+                    curr_path = 0;
+                }
+            }
+        }
+
+        /* We pop the stack until we come to the next leaf to return */
+        while (1) {
+            Uint idx = (Uint)WSTACK_POP(stack);
+            Eterm node = (Eterm)WSTACK_POP(stack);
+            Eterm *ptr = hashmap_val(node),
+                hdr = *ptr;
+            Uint sz;
+
+            ptr++;
+
+            switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
+	    case HAMT_SUBTAG_HEAD_ARRAY:
+		ptr++;
+		sz = 16;
+		break;
+	    case HAMT_SUBTAG_HEAD_BITMAP:
+		ptr++;
+	    case HAMT_SUBTAG_NODE_BITMAP:
+		sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+		ASSERT(sz < 17);
+		break;
+            default:
+		erts_exit(ERTS_ABORT_EXIT, "bad header");
+	    }
+
+            if (idx + 1 < sz) {
+                curr_path = idx + 1;
+                break;
+            }
+
+            /* We have reached the end of the iteration */
+            if (WSTACK_ISEMPTY(stack)) {
+                path = am_none;
+                break;
+            }
+
+        }
+
+        if (path != am_none) {
+            Uint depth = WSTACK_COUNT(stack) / 2 + 1;
+            /* +1 because we already have the first element in curr_path */
+            Eterm *path_digits = NULL;
+
+            /* If the path cannot fit in a small, we allocate a bignum */
+            if (depth >= PATH_ELEMS_PER_DIGIT) {
+                /* We need multiple ErtsDigit's to represent the path */
+                int big_size = BIG_NEED_FOR_BITS(depth * PATH_ELEM_SIZE);
+                hp = HAlloc(BIF_P, big_size);
+                hp[0] = make_pos_bignum_header(big_size - BIG_NEED_SIZE(0));
+                path_digits = hp + big_size - 1;
+            }
+
+            /* Pop the stack to create the complete path to the next leaf */
+            while(!WSTACK_ISEMPTY(stack)) {
+                Uint idx = (Uint)WSTACK_POP(stack);
+                (void)WSTACK_POP(stack);
+
+                depth--;
+                if (depth % PATH_ELEMS_PER_DIGIT == 0) {
+                    path_digits[0] = curr_path;
+                    path_digits--;
+                    curr_path = 0;
+                }
+
+                curr_path <<= PATH_ELEM_SIZE;
+                curr_path |= idx;
+            }
+
+            if (path_digits) {
+                path_digits[0] = curr_path;
+                path = make_big(hp);
+            } else {
+                /* The Uint could be too large for a small */
+                path = erts_make_integer(curr_path, BIF_P);
+            }
+        }
+
+        DESTROY_WSTACK(stack);
     }
+
+    hp = HAlloc(BIF_P, 4 + 3 /* 3-tuple + 2-tuple */);
+    iter = TUPLE2(hp, path, map);
+    hp += 3;
+    BIF_RET(TUPLE3(hp, k, v, iter));
 }
 
 /* implementation of builtin emulations */
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index d910e98398..718d400e22 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -56,7 +56,7 @@ typedef struct flatmap_s {
 /* the head-node is a bitmap or array with an untagged size */
 
 
-#define hashmap_size(x)               (((hashmap_head_t*) hashmap_val(x))->size == -1 ? hashmap_subtree_size(x) : ((hashmap_head_t*) hashmap_val(x))->size)
+#define hashmap_size(x)               (((hashmap_head_t*) hashmap_val(x))->size)
 #define hashmap_make_hash(Key)        make_internal_hash(Key, 0)
 
 #define hashmap_restore_hash(Heap,Lvl,Key) \
@@ -64,7 +64,6 @@ typedef struct flatmap_s {
 #define hashmap_shift_hash(Heap,Hx,Lvl,Key) \
     (((++(Lvl)) & 7) ? (Hx) >> 4 : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), Key)))
 
-
 /* erl_term.h stuff */
 #define flatmap_get_values(x)        (((Eterm *)(x)) + sizeof(flatmap_t)/sizeof(Eterm))
 #define flatmap_get_keys(x)          (((Eterm *)tuple_val(((flatmap_t *)(x))->keys)) + 1)
@@ -106,7 +105,6 @@ Eterm  erts_hashmap_from_ks_and_vs_extra(ErtsHeapFactory *factory,
                                          Eterm *ks, Eterm *vs, Uint n,
 					 Eterm k, Eterm v);
 
-Uint hashmap_subtree_size(Eterm node);
 const Eterm *erts_maps_get(Eterm key, Eterm map);
 
 const Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl
index 02f3c89318..3cf071b590 100644
--- a/erts/emulator/test/map_SUITE.erl
+++ b/erts/emulator/test/map_SUITE.erl
@@ -53,6 +53,7 @@
          t_bif_map_to_list/1,
          t_bif_map_from_list/1,
          t_bif_erts_internal_maps_to_list/1,
+         t_bif_map_next/1,
 
          %% erlang
          t_erlang_hash/1,
@@ -120,6 +121,7 @@ all() -> [t_build_and_match_literals, t_build_and_match_literals_large,
           t_bif_map_values,
           t_bif_map_to_list, t_bif_map_from_list,
           t_bif_erts_internal_maps_to_list,
+          t_bif_map_next,
 
           %% erlang
           t_erlang_hash, t_map_encode_decode,
@@ -2399,6 +2401,55 @@ t_bif_erts_internal_maps_to_list(Config) when is_list(Config) ->
     {'EXIT', {badarg,_}} = (catch erts_internal:maps_to_list(id(#{1=>2}),id(<<>>))),
     ok.
 
+t_bif_map_next(Config) when is_list(Config) ->
+
+    erts_debug:set_internal_state(available_internal_state, true),
+
+    try
+
+        none = maps:next(maps:iterator(id(#{}))),
+
+        verify_iterator(#{}),
+        verify_iterator(#{a => 1, b => 2, c => 3}),
+
+        %% Use fatmap in order to test iterating in very deep maps
+        FM = fatmap(43),
+        verify_iterator(FM),
+
+        {'EXIT', {{badmap,[{a,b},b]},_}} = (catch maps:iterator(id([{a,b},b]))),
+        {'EXIT', {badarg,_}} = (catch maps:next(id(a))),
+        {'EXIT', {badarg,_}} = (catch maps:next(id([a|FM]))),
+        {'EXIT', {badarg,_}} = (catch maps:next(id([1|#{}]))),
+        {'EXIT', {badarg,_}} = (catch maps:next(id([16#FFFFFFF|FM]))),
+        {'EXIT', {badarg,_}} = (catch maps:next(id([16#F0F0F0F|FM]))),
+        {'EXIT', {badarg,_}} = (catch maps:next(id([16#1234567890ABCDEF|FM]))),
+
+        ok
+    after
+            erts_debug:set_internal_state(available_internal_state, false)
+    end.
+
+verify_iterator(Map) ->
+    KVs = t_fold(fun(K, V, A) -> [{K, V} | A] end, [], Map),
+
+    %% Verify that KVs created by iterating Map is of
+    %% correct size and contains all elements
+    true = length(KVs) == maps:size(Map),
+    [maps:get(K, Map) || {K, _} <- KVs],
+    ok.
+
+
+t_fold(Fun, Init, Map) ->
+    t_fold_1(Fun, Init, maps:iterator(Map)).
+
+t_fold_1(Fun, Acc, Iter) ->
+    case maps:next(Iter) of
+        {K, V, NextIter} ->
+            t_fold_1(Fun, Fun(K,V,Acc), NextIter);
+        none ->
+            Acc
+    end.
+
 t_bif_build_and_check(Config) when is_list(Config) ->
     ok = check_build_and_remove(750,[fun(K) -> [K,K] end,
 				     fun(K) -> [float(K),K] end,
-- 
cgit v1.2.3


From 29fbd3acc663c5e4dcc6ff514dccfa20baeb62bd Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 13 Oct 2017 20:14:46 +0200
Subject: erts: Refactor binary_to_term/1/2

Reduce number of arguments to binary_to_term_int
and use the context struct instead.
---
 erts/emulator/beam/external.c | 62 ++++++++++++++++++++++---------------------
 1 file changed, 32 insertions(+), 30 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index c0a3838d42..21d578c8ed 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -122,8 +122,6 @@ static int encode_size_struct_int(struct TTBSizeContext_*, ErtsAtomCacheMap *acm
 
 static Export binary_to_term_trap_export;
 static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1);
-static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b,
-				      Export *bif, Eterm arg0, Eterm arg1);
 
 void erts_init_external(void) {
     erts_init_trap_export(&term_to_binary_trap_export,
@@ -1214,7 +1212,7 @@ typedef struct B2TContext_t {
     ErtsBinary2TermState b2ts;
     Uint32 flags;
     SWord reds;
-    Eterm trap_bin;
+    Eterm trap_bin;  /* THE_NON_VALUE if not exported */
     Export *bif;
     Eterm arg[2];
     enum B2TState state;
@@ -1410,13 +1408,15 @@ static int b2t_context_destructor(Binary *context_bin)
     return 1;
 }
 
+static BIF_RETTYPE binary_to_term_int(Process*, Eterm bin, B2TContext*);
+
+
 static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1)
 {
     Binary *context_bin = erts_magic_ref2bin(BIF_ARG_1);
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor);
 
-    return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin, NULL,
-			      THE_NON_VALUE, THE_NON_VALUE);
+    return binary_to_term_int(BIF_P, THE_NON_VALUE, ERTS_MAGIC_BIN_DATA(context_bin));
 }
 
 
@@ -1442,6 +1442,8 @@ static B2TContext* b2t_export_context(Process* p, B2TContext* src)
                                                  b2t_context_destructor);
     B2TContext* ctx = ERTS_MAGIC_BIN_DATA(context_b);
     Eterm* hp;
+
+    ASSERT(is_non_value(src->trap_bin));
     sys_memcpy(ctx, src, sizeof(B2TContext));
     if (ctx->state >= B2TDecode && ctx->u.dc.next == &src->u.dc.res) {
         ctx->u.dc.next = &ctx->u.dc.res;
@@ -1451,8 +1453,7 @@ static B2TContext* b2t_export_context(Process* p, B2TContext* src)
     return ctx;
 }
 
-static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b,
-				      Export *bif_init, Eterm arg0, Eterm arg1)
+static BIF_RETTYPE binary_to_term_int(Process* p, Eterm bin, B2TContext *ctx)
 {
     BIF_RETTYPE ret_val;
 #ifdef EXTREME_B2T_TRAPPING
@@ -1460,25 +1461,17 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar
 #else
     SWord initial_reds = (Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION);
 #endif
-    B2TContext c_buff;
-    B2TContext *ctx;
     int is_first_call;
 
-    if (context_b == NULL) {
+    if (is_value(bin)) {
 	/* Setup enough to get started */
         is_first_call = 1;
-        ctx = &c_buff;
 	ctx->state = B2TPrepare;
         ctx->aligned_alloc = NULL;
-        ctx->flags = flags;
-	ctx->bif = bif_init;
-	ctx->arg[0] = arg0;
-	ctx->arg[1] = arg1;
-        IF_DEBUG(ctx->trap_bin = THE_NON_VALUE;)
     } else {
-        is_first_call = 0;
-	ctx = ERTS_MAGIC_BIN_DATA(context_b);
+        ASSERT(is_value(ctx->trap_bin));
         ASSERT(ctx->state != B2TPrepare);
+        is_first_call = 0;
     }
     ctx->reds = initial_reds;
 
@@ -1540,11 +1533,11 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar
             break;
 
         case B2TDecodeInit:
-            if (ctx == &c_buff && ctx->b2ts.extsize > ctx->reds) {
+            if (is_non_value(ctx->trap_bin) && ctx->b2ts.extsize > ctx->reds) {
                 /* dec_term will maybe trap, allocate space for magic bin
                    before result term to make it easy to trim with HRelease.
                  */
-                ctx = b2t_export_context(p, &c_buff);
+                ctx = b2t_export_context(p, ctx);
             }
             ctx->u.dc.ep = ctx->b2ts.extp;
             ctx->u.dc.res = (Eterm) (UWord) NULL;
@@ -1607,11 +1600,10 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar
         }
     }while (ctx->reds > 0 || ctx->state >= B2TDone);
 
-    if (ctx == &c_buff) {
-        ASSERT(ctx->trap_bin == THE_NON_VALUE);
-        ctx = b2t_export_context(p, &c_buff);
+    if (is_non_value(ctx->trap_bin)) {
+        ctx = b2t_export_context(p, ctx);
+        ASSERT(is_value(ctx->trap_bin));
     }
-    ASSERT(ctx->trap_bin != THE_NON_VALUE);
 
     if (is_first_call) {
         erts_set_gc_state(p, 0);
@@ -1628,23 +1620,30 @@ HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_term, 1)
 
 BIF_RETTYPE binary_to_term_1(BIF_ALIST_1)
 {
-    return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL, bif_export[BIF_binary_to_term_1],
-			      BIF_ARG_1, THE_NON_VALUE);
+    B2TContext ctx;
+
+    ctx.flags = 0;
+    ctx.trap_bin = THE_NON_VALUE;
+    ctx.bif = bif_export[BIF_binary_to_term_1];
+    ctx.arg[0] = BIF_ARG_1;
+    ctx.arg[1] = THE_NON_VALUE;
+    return binary_to_term_int(BIF_P, BIF_ARG_1, &ctx);
 }
 
 HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_term, 2)
 
 BIF_RETTYPE binary_to_term_2(BIF_ALIST_2)
 {
+    B2TContext ctx;
     Eterm opts;
     Eterm opt;
-    Uint32 flags = 0;
 
+    ctx.flags = 0;
     opts = BIF_ARG_2;
     while (is_list(opts)) {
         opt = CAR(list_val(opts));
         if (opt == am_safe) {
-            flags |= ERTS_DIST_EXT_BTT_SAFE;
+            ctx.flags |= ERTS_DIST_EXT_BTT_SAFE;
         }
 	else {
             goto error;
@@ -1655,8 +1654,11 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2)
     if (is_not_nil(opts))
         goto error;
 
-    return binary_to_term_int(BIF_P, flags, BIF_ARG_1, NULL, bif_export[BIF_binary_to_term_2],
-			      BIF_ARG_1, BIF_ARG_2);
+    ctx.trap_bin = THE_NON_VALUE;
+    ctx.bif = bif_export[BIF_binary_to_term_2];
+    ctx.arg[0] = BIF_ARG_1;
+    ctx.arg[1] = BIF_ARG_2;
+    return binary_to_term_int(BIF_P, BIF_ARG_1, &ctx);
 
 error:
     BIF_ERROR(BIF_P, BADARG);
-- 
cgit v1.2.3


From f0695a70cdaa4fe0fc0e7c58fb791483af0efa1a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 16 Oct 2017 16:29:52 +0200
Subject: erts: Cleanup binary_SUITE:terms

Remove try-catch which must be for some old bitstring limitation.
---
 erts/emulator/test/binary_SUITE.erl | 32 ++++++++++++++------------------
 1 file changed, 14 insertions(+), 18 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index 61536bacd7..a6526d3a1c 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -425,36 +425,32 @@ bad_term_to_binary(Config) when is_list(Config) ->
 
 terms(Config) when is_list(Config) ->
     TestFun = fun(Term) ->
-		      try
-			  S = io_lib:format("~p", [Term]),
-			  io:put_chars(S)
-		      catch
-			  error:badarg ->
-			      io:put_chars("bit sized binary")
-		      end,
+                      S = io_lib:format("~p", [Term]),
+                      io:put_chars(S),
 		      Bin = term_to_binary(Term),
 		      case erlang:external_size(Bin) of
 			  Sz when is_integer(Sz), size(Bin) =< Sz ->
 			      ok
 		      end,
-              Bin1 = term_to_binary(Term, [{minor_version, 1}]),
-              case erlang:external_size(Bin1, [{minor_version, 1}]) of
-              Sz1 when is_integer(Sz1), size(Bin1) =< Sz1 ->
-                  ok
-              end,
+                      Bin1 = term_to_binary(Term, [{minor_version, 1}]),
+                      case erlang:external_size(Bin1, [{minor_version, 1}]) of
+                          Sz1 when is_integer(Sz1), size(Bin1) =< Sz1 ->
+                              ok
+                      end,
 		      Term = binary_to_term_stress(Bin),
 		      Term = binary_to_term_stress(Bin, [safe]),
-		      Unaligned = make_unaligned_sub_binary(Bin),
-		      Term = binary_to_term_stress(Unaligned),
-		      Term = binary_to_term_stress(Unaligned, []),
-		      Term = binary_to_term_stress(Bin, [safe]),
+		      BinU = make_unaligned_sub_binary(Bin),
+		      Term = binary_to_term_stress(BinU),
+		      Term = binary_to_term_stress(BinU, []),
+		      Term = binary_to_term_stress(BinU, [safe]),
 		      BinC = erlang:term_to_binary(Term, [compressed]),
 		      Term = binary_to_term_stress(BinC),
 		      true = size(BinC) =< size(Bin),
 		      Bin = term_to_binary(Term, [{compressed,0}]),
 		      terms_compression_levels(Term, size(Bin), 1),
-		      UnalignedC = make_unaligned_sub_binary(BinC),
-		      Term = binary_to_term_stress(UnalignedC)
+
+		      BinUC = make_unaligned_sub_binary(BinC),
+		      Term = binary_to_term_stress(BinUC),
 	      end,
     test_terms(TestFun),
     ok.
-- 
cgit v1.2.3


From 890fb3bc90cdab64506cd4a43ca0a04727b5b7ea Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 16 Oct 2017 17:56:46 +0200
Subject: erts: Add 'used' option to binary_to_term/2

---
 erts/emulator/beam/external.c       | 35 +++++++++++++++++++++++++++++
 erts/emulator/test/binary_SUITE.erl | 45 ++++++++++++++++++++++++++++++++++++-
 2 files changed, 79 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 21d578c8ed..44ddd251f1 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1212,6 +1212,7 @@ typedef struct B2TContext_t {
     ErtsBinary2TermState b2ts;
     Uint32 flags;
     SWord reds;
+    Uint used_bytes; /* In: boolean, Out: bytes */
     Eterm trap_bin;  /* THE_NON_VALUE if not exported */
     Export *bif;
     Eterm arg[2];
@@ -1306,6 +1307,11 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size,
 
 	    ctx->u.uc.dbytes = state->extp;
 	    ctx->u.uc.dleft = dest_len;
+            if (ctx->used_bytes) {
+                ASSERT(ctx->used_bytes == 1);
+                 /* to be subtracted by stream.avail_in when done */
+                ctx->used_bytes = data_size;
+            }
 	    ctx->state = B2TUncompressChunk;
             *ctxp = ctx;
         }
@@ -1515,6 +1521,10 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Eterm bin, B2TContext *ctx)
                      && zret == Z_STREAM_END
                      && ctx->u.uc.dleft == 0) {
                 ctx->reds -= chunk;
+                if (ctx->used_bytes) {
+                    ASSERT(ctx->used_bytes > 5 + ctx->u.uc.stream.avail_in);
+                    ctx->used_bytes -= ctx->u.uc.stream.avail_in;
+                }
                 ctx->state = B2TSizeInit;
             }
             else {
@@ -1580,6 +1590,25 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Eterm bin, B2TContext *ctx)
 	    return ret_val;
 
         case B2TDone:
+            if (ctx->used_bytes) {
+                Eterm *hp;
+                Eterm used;
+                if (!ctx->b2ts.exttmp) {
+                    ASSERT(ctx->used_bytes == 1);
+                    ctx->used_bytes = (ctx->u.dc.ep - ctx->b2ts.extp
+                                       +1); /* VERSION_MAGIC */
+                }
+                if (IS_USMALL(0, ctx->used_bytes)) {
+                    hp = erts_produce_heap(&ctx->u.dc.factory, 3, 0);
+                    used = make_small(ctx->used_bytes);
+                }
+                else {
+                    hp = erts_produce_heap(&ctx->u.dc.factory, 3+BIG_UINT_HEAP_SIZE, 0);
+                    used = uint_to_big(ctx->used_bytes, hp);
+                    hp += BIG_UINT_HEAP_SIZE;
+                }
+                ctx->u.dc.res = TUPLE2(hp, ctx->u.dc.res, used);
+            }
             b2t_destroy_context(ctx);
 
             if (ctx->u.dc.factory.hp > ctx->u.dc.factory.hp_end) {
@@ -1623,6 +1652,7 @@ BIF_RETTYPE binary_to_term_1(BIF_ALIST_1)
     B2TContext ctx;
 
     ctx.flags = 0;
+    ctx.used_bytes = 0;
     ctx.trap_bin = THE_NON_VALUE;
     ctx.bif = bif_export[BIF_binary_to_term_1];
     ctx.arg[0] = BIF_ARG_1;
@@ -1639,12 +1669,16 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2)
     Eterm opt;
 
     ctx.flags = 0;
+    ctx.used_bytes = 0;
     opts = BIF_ARG_2;
     while (is_list(opts)) {
         opt = CAR(list_val(opts));
         if (opt == am_safe) {
             ctx.flags |= ERTS_DIST_EXT_BTT_SAFE;
         }
+        else if (opt == am_used) {
+            ctx.used_bytes = 1;
+        }
 	else {
             goto error;
         }
@@ -3996,6 +4030,7 @@ dec_term_atom_common:
     if (ctx) {
         ctx->state = B2TDone;
 	ctx->reds = reds;
+        ctx->u.dc.ep = ep;
     }
 
     return ep;
diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index a6526d3a1c..cbc2d8fae5 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -48,6 +48,7 @@
 	 bad_list_to_binary/1, bad_binary_to_list/1,
 	 t_split_binary/1, bad_split/1,
 	 terms/1, terms_float/1, float_middle_endian/1,
+         b2t_used_big/1,
 	 external_size/1, t_iolist_size/1,
 	 t_hash/1,
 	 bad_size/1,
@@ -72,6 +73,7 @@ all() ->
      t_split_binary, bad_split,
      bad_list_to_binary, bad_binary_to_list, terms,
      terms_float, float_middle_endian, external_size, t_iolist_size,
+     b2t_used_big,
      bad_binary_to_term_2, safe_binary_to_term2,
      bad_binary_to_term, bad_terms, t_hash, bad_size,
      bad_term_to_binary, more_bad_terms, otp_5484, otp_5933,
@@ -439,22 +441,63 @@ terms(Config) when is_list(Config) ->
                       end,
 		      Term = binary_to_term_stress(Bin),
 		      Term = binary_to_term_stress(Bin, [safe]),
+                      Bin_sz = byte_size(Bin),
+		      {Term,Bin_sz} = binary_to_term_stress(Bin, [used]),
+
+                      BinE = <<Bin/binary, 1, 2, 3>>,
+		      {Term,Bin_sz} = binary_to_term_stress(BinE, [used]),
+
 		      BinU = make_unaligned_sub_binary(Bin),
 		      Term = binary_to_term_stress(BinU),
 		      Term = binary_to_term_stress(BinU, []),
 		      Term = binary_to_term_stress(BinU, [safe]),
+		      {Term,Bin_sz} = binary_to_term_stress(BinU, [used]),
+
+                      BinUE = make_unaligned_sub_binary(BinE),
+		      {Term,Bin_sz} = binary_to_term_stress(BinUE, [used]),
+
 		      BinC = erlang:term_to_binary(Term, [compressed]),
+                      BinC_sz = byte_size(BinC),
+		      true = BinC_sz =< size(Bin),
 		      Term = binary_to_term_stress(BinC),
-		      true = size(BinC) =< size(Bin),
+		      {Term, BinC_sz} = binary_to_term_stress(BinC, [used]),
+
 		      Bin = term_to_binary(Term, [{compressed,0}]),
 		      terms_compression_levels(Term, size(Bin), 1),
 
 		      BinUC = make_unaligned_sub_binary(BinC),
 		      Term = binary_to_term_stress(BinUC),
+                      {Term,BinC_sz} = binary_to_term_stress(BinUC, [used]),
+
+                      BinCE = <<BinC/binary, 1, 2, 3>>,
+		      {Term,BinC_sz} = binary_to_term_stress(BinCE, [used]),
+
+		      BinUCE = make_unaligned_sub_binary(BinCE),
+		      Term = binary_to_term_stress(BinUCE),
+                      {Term,BinC_sz} = binary_to_term_stress(BinUCE, [used])
 	      end,
     test_terms(TestFun),
     ok.
 
+%% Test binary_to_term(_, [used]) returning a big Used integer.
+b2t_used_big(_Config) ->
+    case erlang:system_info(wordsize) of
+        8 ->
+            {skipped, "This is not a 32-bit machine"};
+        4 ->
+            %% Use a long utf8 atom for large external format but compact on heap.
+            BigAtom = binary_to_atom(<< <<16#F0908D88:32>> || _ <- lists:seq(1,255) >>,
+                                     utf8),
+            Atoms = (1 bsl 17) + (1 bsl 9),
+            BigAtomList = lists:duplicate(Atoms, BigAtom),
+            BigBin = term_to_binary(BigAtomList),
+            {BigAtomList, Used} = binary_to_term(BigBin, [used]),
+            2 = erts_debug:size(Used),
+            Used = byte_size(BigBin),
+            Used = 1 + 1 + 4 + Atoms*(1+2+4*255) + 1,
+            ok
+    end.
+
 terms_compression_levels(Term, UncompressedSz, Level) when Level < 10 ->
     BinC = erlang:term_to_binary(Term, [{compressed,Level}]),
     Term = binary_to_term_stress(BinC),
-- 
cgit v1.2.3


From 360d26b21c169b0c88d6ca43265cbcfa3ce9ef74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 13 Oct 2017 13:28:25 +0200
Subject: Buffer writing of crash dumps

Writing of crash dumps were done using unbuffered IO. This
is slow since many small writes are done.

Use a FILE* with an allocated buffer to obtain buffered IO.

I wrote a small test program that created 50000 binaries of 200 bytes
each and then created a crash dump. The crash dumping was an order of
magnitude faster with buffered IO than without.
---
 erts/emulator/beam/break.c | 49 +++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 44 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 76a0c5c716..8182e55006 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -58,6 +58,8 @@ static void dump_attributes(fmtfn_t to, void *to_arg, byte* ptr, int size);
 
 extern char* erts_system_version[];
 
+#define WRITE_BUFFER_SIZE (64*1024)
+
 static void
 port_info(fmtfn_t to, void *to_arg)
 {
@@ -677,18 +679,28 @@ bin_check(void)
 static Sint64 crash_dump_limit = ERTS_SINT64_MAX;
 static Sint64 crash_dump_written = 0;
 
-static int crash_dump_limited_writer(void* vfdp, char* buf, size_t len)
+typedef struct LimitedWriterInfo_ {
+    fmtfn_t to;
+    void* to_arg;
+} LimitedWriterInfo;
+
+static int
+crash_dump_limited_writer(void* vfdp, char* buf, size_t len)
 {
     const char stop_msg[] = "\n=abort:CRASH DUMP SIZE LIMIT REACHED\n";
+    LimitedWriterInfo* lwi = (LimitedWriterInfo *) vfdp;
 
     crash_dump_written += len;
     if (crash_dump_written <= crash_dump_limit) {
-        return erts_write_fd(vfdp, buf, len);
+        return lwi->to(lwi->to_arg, buf, len);
     }
 
     len -= (crash_dump_written - crash_dump_limit);
-    erts_write_fd(vfdp, buf, len);
-    erts_write_fd(vfdp, (char*)stop_msg, sizeof(stop_msg)-1);
+    lwi->to(lwi->to_arg, buf, len);
+    lwi->to(lwi->to_arg, (char*)stop_msg, sizeof(stop_msg)-1);
+    if (lwi->to == &erts_write_fp) {
+        fclose((FILE *) lwi->to_arg);
+    }
 
     /* We assume that crash dump was called from erts_exit_vv() */
     erts_exit_epilogue();
@@ -713,6 +725,9 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     int i;
     fmtfn_t to = &erts_write_fd;
     void*   to_arg;
+    FILE* fp = 0;
+    LimitedWriterInfo lwi;
+    static char* write_buffer;  /* 'static' to avoid a leak warning in valgrind */
 
     if (ERTS_SOMEONE_IS_CRASH_DUMPING)
 	return;
@@ -820,7 +835,28 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     fd = open(dumpname,O_WRONLY | O_CREAT | O_TRUNC,0640);
     if (fd < 0)
 	return; /* Can't create the crash dump, skip it */
-    to_arg = (void*)&fd;
+
+    /*
+     * Wrap into a FILE* so that we can use buffered output. Set an
+     * explicit buffer to make sure the first write does not fail because
+     * of a failure to allocate a buffer.
+     */
+    write_buffer = (char *) erts_alloc_fnf(ERTS_ALC_T_TMP, WRITE_BUFFER_SIZE);
+    if (write_buffer && (fp = fdopen(fd, "w")) != NULL) {
+        setvbuf(fp, write_buffer, _IOFBF, WRITE_BUFFER_SIZE);
+        lwi.to = &erts_write_fp;
+        lwi.to_arg = (void*)fp;
+    } else {
+        lwi.to = &erts_write_fd;
+        lwi.to_arg = (void*)&fd;
+    }
+    if (to == &crash_dump_limited_writer) {
+        to_arg = (void *) &lwi;
+    } else {
+        to = lwi.to;
+        to_arg = lwi.to_arg;
+    }
+
     time(&now);
     erts_cbprintf(to, to_arg, "=erl_crash_dump:0.3\n%s", ctime(&now));
 
@@ -932,6 +968,9 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     }
 
     erts_cbprintf(to, to_arg, "=end\n");
+    if (fp) {
+        fclose(fp);
+    }
     close(fd);
     erts_fprintf(stderr,"done\n");
 }
-- 
cgit v1.2.3


From 008b44b75a5abf32b26365334597835a3c6fcfd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 9 Oct 2017 12:35:15 +0200
Subject: Implement dumping of maps in crash dumps

Maps would be dumped as the atom 'undefined', which is
not very informative.
---
 erts/emulator/beam/erl_process_dump.c | 74 ++++++++++++++++++++++++++++++++---
 1 file changed, 68 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index b826e6c5d3..9ae1a36d6f 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -32,6 +32,7 @@
 #include "dist.h"
 #include "beam_catches.h"
 #include "erl_binary.h"
+#include "erl_map.h"
 #define ERTS_WANT_EXTERNAL_TAGS
 #include "external.h"
 
@@ -498,11 +499,77 @@ heap_dump(fmtfn_t to, void *to_arg, Eterm x)
 		    erts_print(to, to_arg, "p<%beu.%beu>\n",
 			       port_channel_no(x), port_number(x));
 		    *ptr = OUR_NIL;
+		} else if (is_map_header(hdr)) {
+                    if (is_flatmap_header(hdr)) {
+                        flatmap_t* fmp = (flatmap_t *) flatmap_val(x);
+                        Eterm* values = ptr + sizeof(flatmap_t) / sizeof(Eterm);
+                        Uint map_size = fmp->size;
+                        int i;
+
+                        erts_print(to, to_arg, "Mf" ETERM_FMT ":", map_size);
+                        dump_element(to, to_arg, fmp->keys);
+                        erts_putc(to, to_arg, ':');
+                        for (i = 0; i < map_size; i++) {
+                            dump_element(to, to_arg, values[i]);
+                            if (is_immed(values[i])) {
+                                values[i] = make_small(0);
+                            }
+                            if (i < map_size-1) {
+                                erts_putc(to, to_arg, ',');
+                            }
+                        }
+                        erts_putc(to, to_arg, '\n');
+                        *ptr = OUR_NIL;
+                        x = fmp->keys;
+                        if (map_size) {
+                            fmp->keys = (Eterm) next;
+                            next = &values[map_size-1];
+                        }
+                        continue;
+                    } else {
+                        Uint i;
+                        Uint sz = 0;
+                        Eterm* nodes = ptr + 1;
+
+                        switch (MAP_HEADER_TYPE(hdr)) {
+                        case MAP_HEADER_TAG_HAMT_HEAD_ARRAY:
+                            nodes++;
+                            sz = 16;
+                            erts_print(to, to_arg, "Mh" ETERM_FMT ":" ETERM_FMT ":",
+                                       hashmap_size(x), sz);
+                            break;
+                        case MAP_HEADER_TAG_HAMT_HEAD_BITMAP:
+                            nodes++;
+                            sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+                            erts_print(to, to_arg, "Mh" ETERM_FMT ":" ETERM_FMT ":",
+                                       hashmap_size(x), sz);
+                            break;
+                        case MAP_HEADER_TAG_HAMT_NODE_BITMAP:
+                            sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+                            erts_print(to, to_arg, "Mn" ETERM_FMT ":", sz);
+                            break;
+                        }
+                        *ptr = OUR_NIL;
+                        for (i = 0; i < sz; i++) {
+                            dump_element(to, to_arg, nodes[i]);
+                            if (is_immed(nodes[i])) {
+                                nodes[i] = make_small(0);
+                            }
+                            if (i < sz-1) {
+                                erts_putc(to, to_arg, ',');
+                            }
+                        }
+                        erts_putc(to, to_arg, '\n');
+                        x = nodes[0];
+                        nodes[0] = (Eterm) next;
+                        next = &nodes[sz-1];
+                        continue;
+                    }
 		} else {
 		    /*
 		     * All other we dump in the external term format.
 		     */
-			dump_externally(to, to_arg, x);
+                    dump_externally(to, to_arg, x);
 		    erts_putc(to, to_arg, '\n');
 		    *ptr = OUR_NIL;
 		}
@@ -564,11 +631,6 @@ dump_externally(fmtfn_t to, void *to_arg, Eterm term)
 	}
     }
 
-    /* Do not handle maps */
-    if (is_map(term)) {
-        term = am_undefined;
-    }
-
     s = p = sbuf;
     erts_encode_ext(term, &p);
     erts_print(to, to_arg, "E%X:", p-s);
-- 
cgit v1.2.3


From ea1ddd8800c82fb8ae7ac2f54f4217a35d6f463d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 11 Oct 2017 14:45:11 +0200
Subject: Dump literals separately to avoid incomplete heap data

When a literal was used from several processes, the literal would
be dumped in only one of the processes. The other processes
that referenced the literals would have incomplete heap data.
---
 erts/emulator/beam/erl_process_dump.c | 207 +++++++++++++++++++++++++++++++++-
 1 file changed, 204 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 9ae1a36d6f..d893853063 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -52,6 +52,9 @@ static void print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x);
 static void heap_dump(fmtfn_t to, void *to_arg, Eterm x);
 static void dump_binaries(fmtfn_t to, void *to_arg, Binary* root);
 static void dump_externally(fmtfn_t to, void *to_arg, Eterm term);
+static void dump_literals(fmtfn_t to, void *to_arg);
+static void dump_module_literals(fmtfn_t to, void *to_arg,
+                                 struct erl_module_instance* mp);
 
 static Binary* all_binaries;
 
@@ -59,7 +62,6 @@ extern BeamInstr beam_apply[];
 extern BeamInstr beam_exit[];
 extern BeamInstr beam_continue_exit[];
 
-
 void
 erts_deep_process_dump(fmtfn_t to, void *to_arg)
 {
@@ -76,6 +78,7 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg)
        }
     }
 
+    dump_literals(to, to_arg);
     dump_binaries(to, to_arg, all_binaries);
 }
 
@@ -374,7 +377,7 @@ heap_dump(fmtfn_t to, void *to_arg, Eterm x)
 	    next = (Eterm *) x;
 	} else if (is_list(x)) {
 	    ptr = list_val(x);
-	    if (ptr[0] != OUR_NIL) {
+	    if (ptr[0] != OUR_NIL && !erts_is_literal(x, ptr)) {
 		erts_print(to, to_arg, PTR_FMT ":l", ptr);
 		dump_element(to, to_arg, ptr[0]);
 		erts_putc(to, to_arg, '|');
@@ -393,7 +396,7 @@ heap_dump(fmtfn_t to, void *to_arg, Eterm x)
 
 	    ptr = boxed_val(x);
 	    hdr = *ptr;
-	    if (hdr != OUR_NIL) {	/* If not visited */
+	    if (hdr != OUR_NIL && !erts_is_literal(x, ptr)) {
 		erts_print(to, to_arg, PTR_FMT ":", ptr);
 	        if (is_arity_value(hdr)) {
 		    Uint i;
@@ -639,6 +642,204 @@ dump_externally(fmtfn_t to, void *to_arg, Eterm term)
     }
 }
 
+static void
+dump_literals(fmtfn_t to, void *to_arg)
+{
+    int i;
+    Module* modp;
+    ErtsCodeIndex code_ix;
+
+    code_ix = erts_active_code_ix();
+    erts_rlock_old_code(code_ix);
+
+    erts_print(to, to_arg, "=literals\n");
+
+    for (i = 0; i < module_code_size(code_ix); i++) {
+	modp = module_code(i, code_ix);
+        if (modp == NULL) {
+            continue;
+        }
+        dump_module_literals(to, to_arg, &modp->curr);
+        dump_module_literals(to, to_arg, &modp->old);
+    }
+    erts_runlock_old_code(code_ix);
+}
+
+static void
+dump_module_literals(fmtfn_t to, void *to_arg, struct erl_module_instance* mp)
+{
+    BeamCodeHeader* code;
+    ErtsLiteralArea* lit_area;
+    Eterm* htop;
+    Eterm* hend;
+
+    if (mp->code_length == 0) {
+        return;
+    }
+    code = mp->code_hdr;
+    lit_area = code->literal_area;
+    htop = lit_area->start;
+    hend = lit_area->end;
+    while (htop < hend) {
+        Eterm w = *htop;
+        Eterm term;
+        Uint size;
+
+        switch (primary_tag(w)) {
+        case TAG_PRIMARY_HEADER:
+            term = make_boxed(htop);
+            erts_print(to, to_arg, PTR_FMT ":", htop);
+            if (is_arity_value(w)) {
+                Uint i;
+                Uint arity = arityval(w);
+
+                erts_print(to, to_arg, "t" ETERM_FMT ":", arity);
+                for (i = 1; i <= arity; i++) {
+                    dump_element(to, to_arg, htop[i]);
+                    if (i < arity) {
+                        erts_putc(to, to_arg, ',');
+                    }
+                }
+                erts_putc(to, to_arg, '\n');
+            } else if (w == HEADER_FLONUM) {
+                FloatDef f;
+                char sbuf[31];
+                int i;
+
+                GET_DOUBLE_DATA((htop+1), f);
+                i = sys_double_to_chars(f.fd, sbuf, sizeof(sbuf));
+                sys_memset(sbuf+i, 0, 31-i);
+                erts_print(to, to_arg, "F%X:%s\n", i, sbuf);
+            } else if (_is_bignum_header(w)) {
+                erts_print(to, to_arg, "B%T\n", term);
+            } else if (is_binary_header(w)) {
+                Uint tag = thing_subtag(w);
+                Uint size = binary_size(term);
+                Uint i;
+
+                if (tag == HEAP_BINARY_SUBTAG) {
+                    byte* p;
+
+                    erts_print(to, to_arg, "Yh%X:", size);
+                    p = binary_bytes(term);
+                    for (i = 0; i < size; i++) {
+                        erts_print(to, to_arg, "%02X", p[i]);
+                    }
+                } else if (tag == REFC_BINARY_SUBTAG) {
+                    ProcBin* pb = (ProcBin *) binary_val(term);
+                    Binary* val = pb->val;
+
+                    if (erts_atomic_xchg_nob(&val->intern.refc, 0) != 0) {
+                        val->intern.flags = (UWord) all_binaries;
+                        all_binaries = val;
+                    }
+                    erts_print(to, to_arg,
+                               "Yc" PTR_FMT ":" PTR_FMT ":" PTR_FMT,
+                               val,
+                               pb->bytes - (byte *)val->orig_bytes,
+                               size);
+                } else if (tag == SUB_BINARY_SUBTAG) {
+                    ErlSubBin* Sb = (ErlSubBin *) binary_val(term);
+                    Eterm* real_bin;
+                    void* val;
+
+                    real_bin = boxed_val(Sb->orig);
+                    if (thing_subtag(*real_bin) == REFC_BINARY_SUBTAG) {
+                        /*
+                         * Unvisited REFC_BINARY: Point directly to
+                         * the binary.
+                         */
+                        ProcBin* pb = (ProcBin *) real_bin;
+                        val = pb->val;
+                    } else {
+                        /*
+                         * Heap binary or visited REFC binary: Point
+                         * to heap binary or ProcBin on the heap.
+                         */
+                        val = real_bin;
+                    }
+                    erts_print(to, to_arg,
+                               "Ys" PTR_FMT ":" PTR_FMT ":" PTR_FMT,
+                               val, Sb->offs, size);
+                }
+                erts_putc(to, to_arg, '\n');
+            } else if (is_map_header(w)) {
+                if (is_flatmap_header(w)) {
+                    flatmap_t* fmp = (flatmap_t *) flatmap_val(term);
+                    Eterm* values = htop + sizeof(flatmap_t) / sizeof(Eterm);
+                    Uint map_size = fmp->size;
+                    int i;
+
+                    erts_print(to, to_arg, "Mf" ETERM_FMT ":", map_size);
+                    dump_element(to, to_arg, fmp->keys);
+                    erts_putc(to, to_arg, ':');
+                    for (i = 0; i < map_size; i++) {
+                        dump_element(to, to_arg, values[i]);
+                        if (i < map_size-1) {
+                            erts_putc(to, to_arg, ',');
+                        }
+                    }
+                    erts_putc(to, to_arg, '\n');
+                } else {
+                    Uint i;
+                    Uint sz = 0;
+                    Eterm* nodes = htop + 1;
+
+                    switch (MAP_HEADER_TYPE(w)) {
+                    case MAP_HEADER_TAG_HAMT_HEAD_ARRAY:
+                        nodes++;
+                        sz = 16;
+                        erts_print(to, to_arg, "Mh" ETERM_FMT ":" ETERM_FMT ":",
+                                   hashmap_size(term), sz);
+                        break;
+                    case MAP_HEADER_TAG_HAMT_HEAD_BITMAP:
+                        nodes++;
+                        sz = hashmap_bitcount(MAP_HEADER_VAL(w));
+                        erts_print(to, to_arg, "Mh" ETERM_FMT ":" ETERM_FMT ":",
+                                   hashmap_size(term), sz);
+                        break;
+                    case MAP_HEADER_TAG_HAMT_NODE_BITMAP:
+                        sz = hashmap_bitcount(MAP_HEADER_VAL(w));
+                        erts_print(to, to_arg, "Mn" ETERM_FMT ":", sz);
+                        break;
+                    }
+                    for (i = 0; i < sz; i++) {
+                        dump_element(to, to_arg, nodes[i]);
+                        if (i < sz-1) {
+                            erts_putc(to, to_arg, ',');
+                        }
+                    }
+                    erts_putc(to, to_arg, '\n');
+                }
+            }
+            size = 1 + header_arity(w);
+            switch (w & _HEADER_SUBTAG_MASK) {
+            case MAP_SUBTAG:
+                if (is_flatmap_header(w)) {
+                    size += 1 + flatmap_get_size(htop);
+                } else {
+                    size += hashmap_bitcount(MAP_HEADER_VAL(w));
+                }
+                break;
+            case SUB_BINARY_SUBTAG:
+                size += 1;
+                break;
+            }
+            break;
+        default:
+            ASSERT(!is_header(htop[1]));
+            erts_print(to, to_arg, PTR_FMT ":l", htop);
+            dump_element(to, to_arg, htop[0]);
+            erts_putc(to, to_arg, '|');
+            dump_element(to, to_arg, htop[1]);
+            erts_putc(to, to_arg, '\n');
+            size = 2;
+            break;
+        }
+        htop += size;
+    }
+}
+
 void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg)
 {
     char *s;
-- 
cgit v1.2.3


From 31ad587b6d75e8697964e2b80709fb3b3d2901d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 12 Oct 2017 12:45:18 +0200
Subject: Don't dump literal areas that are not referenced at all

---
 erts/emulator/beam/beam_ranges.c      |  22 +++++++
 erts/emulator/beam/erl_alloc.types    |   1 +
 erts/emulator/beam/erl_process_dump.c | 114 +++++++++++++++++++++++++++++-----
 erts/emulator/beam/global.h           |   2 +
 erts/emulator/beam/module.c           |   1 -
 5 files changed, 122 insertions(+), 18 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c
index 9b0335e83d..fac4289271 100644
--- a/erts/emulator/beam/beam_ranges.c
+++ b/erts/emulator/beam/beam_ranges.c
@@ -32,6 +32,15 @@ typedef struct {
     erts_smp_atomic_t end; /* (BeamInstr*) Points one word beyond last function in module. */
 } Range;
 
+/*
+ * Used for crash dumping of literals. The size of erts_dump_lit_areas is
+ * always twice the number of active ranges (to allow for literals in both
+ * current and old code).
+ */
+
+ErtsLiteralArea** erts_dump_lit_areas;
+Uint erts_dump_num_lit_areas;
+
 /* Range 'end' needs to be atomic as we purge module
     by setting end=start in active code_ix */
 #define RANGE_END(R) ((BeamInstr*)erts_smp_atomic_read_nob(&(R)->end))
@@ -97,6 +106,11 @@ erts_init_ranges(void)
 	r[i].allocated = 0;
 	erts_smp_atomic_init_nob(&r[i].mid, 0);
     }
+
+    erts_dump_num_lit_areas = 8;
+    erts_dump_lit_areas = (ErtsLiteralArea **)
+        erts_alloc(ERTS_ALC_T_CRASH_DUMP,
+                   erts_dump_num_lit_areas * sizeof(ErtsLiteralArea*));
 }
 
 void
@@ -164,6 +178,14 @@ erts_end_staging_ranges(int commit)
 	erts_smp_atomic_set_nob(&r[dst].mid,
 				(erts_aint_t) (r[dst].modules +
 					       r[dst].n / 2));
+
+        if (r[dst].allocated * 2 > erts_dump_num_lit_areas) {
+            erts_dump_num_lit_areas *= 2;
+            erts_dump_lit_areas = (ErtsLiteralArea **)
+                erts_realloc(ERTS_ALC_T_CRASH_DUMP,
+                             (void *) erts_dump_lit_areas,
+                             erts_dump_num_lit_areas * sizeof(ErtsLiteralArea*));
+        }
     }
 }
 
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 50a1d97dd5..252bf1cc7e 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -285,6 +285,7 @@ type    MREF_ENT	STANDARD	SYSTEM		magic_ref_entry
 type	MREF_TAB_BKTS	STANDARD	SYSTEM		magic_ref_table_buckets
 type	MREF_TAB	LONG_LIVED	SYSTEM		magic_ref_table
 type	MINDIRECTION	FIXED_SIZE	SYSTEM		magic_indirection
+type	CRASH_DUMP	STANDARD	SYSTEM		crash_dump
 
 +if threads_no_smp
 # Need thread safe allocs, but std_alloc and fix_alloc are not;
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index d893853063..9fd024cae8 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -52,9 +52,11 @@ static void print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x);
 static void heap_dump(fmtfn_t to, void *to_arg, Eterm x);
 static void dump_binaries(fmtfn_t to, void *to_arg, Binary* root);
 static void dump_externally(fmtfn_t to, void *to_arg, Eterm term);
+static void mark_literal(Eterm* ptr);
+static void init_literal_areas(void);
 static void dump_literals(fmtfn_t to, void *to_arg);
 static void dump_module_literals(fmtfn_t to, void *to_arg,
-                                 struct erl_module_instance* mp);
+                                 ErtsLiteralArea* lit_area);
 
 static Binary* all_binaries;
 
@@ -68,7 +70,8 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg)
     int i, max = erts_ptab_max(&erts_proc);
 
     all_binaries = NULL;
-    
+    init_literal_areas();
+
     for (i = 0; i < max; i++) {
 	Process *p = erts_pix2proc(i);
 	if (p && p->i != ENULL) {
@@ -377,7 +380,9 @@ heap_dump(fmtfn_t to, void *to_arg, Eterm x)
 	    next = (Eterm *) x;
 	} else if (is_list(x)) {
 	    ptr = list_val(x);
-	    if (ptr[0] != OUR_NIL && !erts_is_literal(x, ptr)) {
+            if (erts_is_literal(x, ptr)) {
+                mark_literal(ptr);
+            } else if (ptr[0] != OUR_NIL) {
 		erts_print(to, to_arg, PTR_FMT ":l", ptr);
 		dump_element(to, to_arg, ptr[0]);
 		erts_putc(to, to_arg, '|');
@@ -396,7 +401,9 @@ heap_dump(fmtfn_t to, void *to_arg, Eterm x)
 
 	    ptr = boxed_val(x);
 	    hdr = *ptr;
-	    if (hdr != OUR_NIL && !erts_is_literal(x, ptr)) {
+            if (erts_is_literal(x, ptr)) {
+                mark_literal(ptr);
+            } else if (hdr != OUR_NIL) {
 		erts_print(to, to_arg, PTR_FMT ":", ptr);
 	        if (is_arity_value(hdr)) {
 		    Uint i;
@@ -642,42 +649,115 @@ dump_externally(fmtfn_t to, void *to_arg, Eterm term)
     }
 }
 
+/*
+ * Handle dumping of literal areas.
+ */
+
+static ErtsLiteralArea** lit_areas;
+static Uint num_lit_areas;
+
+static int compare_areas(const void * a, const void * b)
+{
+    ErtsLiteralArea** a_p = (ErtsLiteralArea **) a;
+    ErtsLiteralArea** b_p = (ErtsLiteralArea **) b;
+
+    if (*a_p < *b_p) {
+        return -1;
+    } else if (*b_p < *a_p) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+
 static void
-dump_literals(fmtfn_t to, void *to_arg)
+init_literal_areas(void)
 {
     int i;
     Module* modp;
     ErtsCodeIndex code_ix;
+    ErtsLiteralArea** area_p;
 
     code_ix = erts_active_code_ix();
     erts_rlock_old_code(code_ix);
 
-    erts_print(to, to_arg, "=literals\n");
-
+    lit_areas = area_p = erts_dump_lit_areas;
+    num_lit_areas = 0;
     for (i = 0; i < module_code_size(code_ix); i++) {
 	modp = module_code(i, code_ix);
         if (modp == NULL) {
             continue;
         }
-        dump_module_literals(to, to_arg, &modp->curr);
-        dump_module_literals(to, to_arg, &modp->old);
+        if (modp->curr.code_length > 0 &&
+            modp->curr.code_hdr->literal_area) {
+            *area_p++ = modp->curr.code_hdr->literal_area;
+        }
+        if (modp->old.code_length > 0 && modp->old.code_hdr->literal_area) {
+            *area_p++ = modp->old.code_hdr->literal_area;
+        }
+    }
+
+    num_lit_areas = area_p - lit_areas;
+    ASSERT(num_lit_areas <= erts_dump_num_lit_areas);
+    for (i = 0; i < num_lit_areas; i++) {
+        lit_areas[i]->off_heap = 0;
+    }
+
+    qsort(lit_areas, num_lit_areas, sizeof(ErtsLiteralArea *),
+          compare_areas);
+
+    erts_runlock_old_code(code_ix);
+}
+
+static int search_areas(const void * a, const void * b) {
+    Eterm* key = (Eterm *) a;
+    ErtsLiteralArea** b_p = (ErtsLiteralArea **) b;
+    if (key < b_p[0]->start) {
+        return -1;
+    } else if (b_p[0]->end <= key) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+static void mark_literal(Eterm* ptr)
+{
+    ErtsLiteralArea** ap;
+
+    ap = bsearch(ptr, lit_areas, num_lit_areas, sizeof(ErtsLiteralArea*),
+                 search_areas);
+    ASSERT(ap);
+    ap[0]->off_heap = (struct erl_off_heap_header *) 1;
+}
+
+
+static void
+dump_literals(fmtfn_t to, void *to_arg)
+{
+    ErtsCodeIndex code_ix;
+    int i;
+
+    code_ix = erts_active_code_ix();
+    erts_rlock_old_code(code_ix);
+
+    erts_print(to, to_arg, "=literals\n");
+    for (i = 0; i < num_lit_areas; i++) {
+        if (lit_areas[i]->off_heap) {
+            dump_module_literals(to, to_arg, lit_areas[i]);
+        }
     }
+
     erts_runlock_old_code(code_ix);
 }
 
 static void
-dump_module_literals(fmtfn_t to, void *to_arg, struct erl_module_instance* mp)
+dump_module_literals(fmtfn_t to, void *to_arg, ErtsLiteralArea* lit_area)
 {
-    BeamCodeHeader* code;
-    ErtsLiteralArea* lit_area;
     Eterm* htop;
     Eterm* hend;
 
-    if (mp->code_length == 0) {
-        return;
-    }
-    code = mp->code_hdr;
-    lit_area = code->literal_area;
     htop = lit_area->start;
     hend = lit_area->end;
     while (htop < hend) {
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 182d3aa44e..440723aea6 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -955,6 +955,8 @@ void erts_update_ranges(BeamInstr* code, Uint size);
 void erts_remove_from_ranges(BeamInstr* code);
 UWord erts_ranges_sz(void);
 void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info);
+ErtsLiteralArea** erts_dump_lit_areas;
+Uint erts_dump_num_lit_areas;
 
 /* break.c */
 void init_break_handler(void);
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index 7987cb2eb5..a386407ec2 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -256,4 +256,3 @@ void module_end_staging(int commit)
 
     IF_DEBUG(dbg_load_code_ix = -1);
 }
-
-- 
cgit v1.2.3


From 7f8e605b242913b6f58f008aaa27a2edd1ab94b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 16 Oct 2017 14:51:32 +0200
Subject: Bump version of crash dumps to 0.4

---
 erts/emulator/beam/break.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 8182e55006..e57be5a595 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -858,7 +858,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     }
 
     time(&now);
-    erts_cbprintf(to, to_arg, "=erl_crash_dump:0.3\n%s", ctime(&now));
+    erts_cbprintf(to, to_arg, "=erl_crash_dump:0.4\n%s", ctime(&now));
 
     if (file != NULL)
        erts_cbprintf(to, to_arg, "The error occurred in file %s, line %d\n", file, line);
-- 
cgit v1.2.3


From 75b586ddbe2e73968e57950e189235c2dafa2a46 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 5 Oct 2017 11:57:20 +0200
Subject: erts: Add lcnt prototype for dist locks update

---
 erts/emulator/beam/erl_node_tables.h | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 3bba673435..ee8277b5ea 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -271,5 +271,6 @@ erts_de_links_unlock(DistEntry *dep)
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 void erts_debug_test_node_tab_delayed_delete(Sint64 millisecs);
+void erts_lcnt_update_distribution_locks(int enable);
 
 #endif
-- 
cgit v1.2.3


From 36b5f99ccdf9bbd6cf72afa24ab5ee562e3ed1b9 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 5 Oct 2017 11:58:14 +0200
Subject: erts: Fail port_SUITE:huge_env if error code > 127

This can happen for instance if erl_child_setup segfaults
before it can do the execv.
---
 erts/emulator/test/port_SUITE.erl | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index 730a17d7e8..e7044653f0 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -1035,6 +1035,9 @@ huge_env(Config) when is_list(Config) ->
     try erlang:open_port({spawn,Cmd},[exit_status, {env, Env}]) of
         P ->
             receive
+                {P, {exit_status,N}} = M when N > 127->
+                    %% If exit status is > 127 something went very wrong
+                    ct:fail("Open port failed got ~p",[M]);
                 {P, {exit_status,N}} = M ->
                     %% We test that the exit status is an integer, this means
                     %% that the child program has started. If we get an atom
-- 
cgit v1.2.3


From 19a1af93458ec7a4a252671603d74971f4dec969 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sat, 21 Oct 2017 07:22:47 +0200
Subject: Optimize instructions for comparing a register with a literal

We can avoid calling eq() from the is_eq_exact_literal/3
and is_ne_exact_literal/3 instructions if the source operand is
an immediate (since a literal is either a boxed or a list, never
an immediate).
---
 erts/emulator/beam/instrs.tab | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index 20d356a81d..c17d1a8f69 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -788,7 +788,8 @@ is_eq_exact(Fail, X, Y) {
 }
 
 i_is_eq_exact_literal(Fail, Src, Literal) {
-    if (!eq($Src, $Literal)) {
+    Eterm src = $Src;
+    if (is_immed(src) || !eq(src, $Literal)) {
         $FAIL($Fail);
     }
 }
@@ -800,7 +801,8 @@ is_ne_exact(Fail, X, Y) {
 }
 
 i_is_ne_exact_literal(Fail, Src, Literal) {
-    if (eq($Src, $Literal)) {
+    Eterm src = $Src;
+    if (!is_immed(src) && eq(src, $Literal)) {
         $FAIL($Fail);
     }
 }
-- 
cgit v1.2.3


From 73094884d8113b7290080c6426957d2bf5180ec8 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 23 Oct 2017 15:55:03 +0200
Subject: erts: Fix so that bind correct schedulers

When the cpu ids and scheduler ids don't match,
the end schedulers could end up not being bound
when they should be.

example:
    > taskset -c 1-3 erl +S 4 +sbts
    > erlang:system_info(scheduler_bindings).
    {1,2,3,unbound}

This fix makes it so that all cores are used to
bind schedulers.
---
 erts/emulator/beam/erl_cpu_topology.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
index f8b2fa744f..dfe49ce382 100644
--- a/erts/emulator/beam/erl_cpu_topology.c
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -608,7 +608,7 @@ write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size)
 
 	cpu_bind_order_sort(cpudata, size, cpu_bind_order, 1);
 
-	for (cpu_ix = 0; cpu_ix < size && cpu_ix < erts_no_schedulers; cpu_ix++)
+	for (cpu_ix = 0; cpu_ix < size && s_ix <= erts_no_schedulers; cpu_ix++)
 	    if (erts_is_cpu_available(cpuinfo, cpudata[cpu_ix].logical))
 		scheduler2cpu_map[s_ix++].bind_id = cpudata[cpu_ix].logical;
     }
-- 
cgit v1.2.3


From 8c8467e3ad26932aaacfb01efb2118d986e5200f Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 25 Oct 2017 10:04:25 +0200
Subject: erts: Only do PGO if gcc supports -fprofile-correction

---
 erts/emulator/Makefile.in | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 2b203c9ee6..47d8ee8811 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -74,9 +74,8 @@ endif
 endif
 
 ifeq ($(PROFILE_COMPILER), gcc)
-PROFILE_CORRECTION=@PROFILE_CORRECTION@
 PROFILE_GENERATE=-fprofile-generate
-PROFILE_USE=-fprofile-use $(PROFILE_CORRECTION)
+PROFILE_USE=-fprofile-use -fprofile-correction
 PROFILE_USE_DEPS=$(OBJDIR)/%_pu.gcda
 endif
 ifeq ($(PROFILE_COMPILER), clang)
@@ -808,14 +807,7 @@ $(ERL_TOP)/lib/%.beam:
 
 INIT_OBJS = $(OBJDIR)/erl_main.o	$(PRELOAD_OBJ)
 
-# -fprofile-correction is needed in order to use PGO on erl_process
-# as multiple threads execute in that file.
-ifeq ($(PROFILE_CORRECTION),)
-PROFILE_OBJS = $(OBJDIR)/beam_emu.o
-RUN_OBJS = $(OBJDIR)/erl_process.o
-else
 PROFILE_OBJS = $(OBJDIR)/beam_emu.o	$(OBJDIR)/erl_process.o
-endif
 
 EMU_OBJS = \
 	$(OBJDIR)/beam_opcodes.o \
-- 
cgit v1.2.3


From 62001839d5cd4187538a750752b3d7a30db4e2c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 25 Oct 2017 13:01:22 +0200
Subject: Stop assuming that all schedulers are managed when updating msacc

This fixes statistics_SUITE:msacc when dirty schedulers are used
during the test.
---
 erts/emulator/beam/erl_msacc.h   | 32 +++++++++++++++++++++++---------
 erts/emulator/beam/erl_process.c | 16 ++++++++--------
 2 files changed, 31 insertions(+), 17 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h
index d64ef8c8b9..3f6273f214 100644
--- a/erts/emulator/beam/erl_msacc.h
+++ b/erts/emulator/beam/erl_msacc.h
@@ -279,18 +279,32 @@ void erts_msacc_init_thread(char *type, int id, int liberty);
 #define ERTS_MSACC_PUSH_STATE_M()                         \
     ERTS_MSACC_DECLARE_CACHE();                           \
     ERTS_MSACC_PUSH_STATE_CACHED_M()
-#define ERTS_MSACC_PUSH_STATE_CACHED_M()                                \
-    __erts_msacc_state = ERTS_MSACC_IS_ENABLED_CACHED() ?    \
-        erts_msacc_get_state_m__(__erts_msacc_cache) : ERTS_MSACC_STATE_OTHER
+#define ERTS_MSACC_PUSH_STATE_CACHED_M() \
+    do { \
+        if (ERTS_MSACC_IS_ENABLED_CACHED()) { \
+            ASSERT(!__erts_msacc_cache->unmanaged); \
+            __erts_msacc_state = erts_msacc_get_state_m__(__erts_msacc_cache); \
+        } else { \
+            __erts_msacc_state = ERTS_MSACC_STATE_OTHER; \
+        } \
+    } while(0)
 #define ERTS_MSACC_SET_STATE_M(state)                   \
     ERTS_MSACC_DECLARE_CACHE();                         \
     ERTS_MSACC_SET_STATE_CACHED_M(state)
-#define ERTS_MSACC_SET_STATE_CACHED_M(state)            \
-    if (ERTS_MSACC_IS_ENABLED_CACHED())      \
-        erts_msacc_set_state_m__(__erts_msacc_cache, state, 1)
-#define ERTS_MSACC_POP_STATE_M()                                  \
-    if (ERTS_MSACC_IS_ENABLED_CACHED())                      \
-        erts_msacc_set_state_m__(__erts_msacc_cache, __erts_msacc_state, 0)
+#define ERTS_MSACC_SET_STATE_CACHED_M(state) \
+    do { \
+        if (ERTS_MSACC_IS_ENABLED_CACHED()) { \
+            ASSERT(!__erts_msacc_cache->unmanaged); \
+            erts_msacc_set_state_m__(__erts_msacc_cache, state, 1); \
+        } \
+    } while(0)
+#define ERTS_MSACC_POP_STATE_M() \
+    do { \
+        if (ERTS_MSACC_IS_ENABLED_CACHED()) { \
+            ASSERT(!__erts_msacc_cache->unmanaged); \
+            erts_msacc_set_state_m__(__erts_msacc_cache, __erts_msacc_state, 0); \
+        } \
+    } while(0)
 #define ERTS_MSACC_PUSH_AND_SET_STATE_M(state)                    \
     ERTS_MSACC_PUSH_STATE_M(); ERTS_MSACC_SET_STATE_CACHED_M(state)
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b72bac00c1..099aa02f10 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -3417,7 +3417,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
     int thr_prgr_active = 1;
     erts_aint32_t flgs;
 #endif
-    ERTS_MSACC_PUSH_STATE_M();
+    ERTS_MSACC_PUSH_STATE();
 #ifdef ERTS_SMP
 
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
@@ -3542,9 +3542,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 								     - 1) + 1;
 				} else
 				    timeout = -1;
-                                ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
+                                ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP);
 				res = erts_tse_twait(ssi->event, timeout);
-                                ERTS_MSACC_POP_STATE_M();
+                                ERTS_MSACC_POP_STATE();
 				current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
 				    erts_get_monotonic_time(esdp);
 			    } while (res == EINTR);
@@ -3743,12 +3743,12 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	ASSERT(!erts_port_task_have_outstanding_io_tasks());
 
-	ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO);
+	ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_CHECK_IO);
         LTTNG2(scheduler_poll, esdp->no, 0);
 
 	erl_sys_schedule(0);
 
-	ERTS_MSACC_POP_STATE_M();
+	ERTS_MSACC_POP_STATE();
 
 	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
 	    ErtsMonotonicTime current_time = erts_get_monotonic_time(esdp);
@@ -10350,7 +10350,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
                                      | ERTS_PROC_LOCK_STATUS
                                      | ERTS_PROC_LOCK_TRACE));
 
-            ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER);
+            ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER);
 
 #ifdef ERTS_SMP
             if (state & ERTS_PSFLG_FREE) {
@@ -10642,7 +10642,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    case 0:			/* No process at all */
 	    default:
 		ASSERT(qmask == 0);
-                ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER);
+                ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER);
 		goto check_activities_to_run;
 	    }
 
@@ -10766,7 +10766,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	}
 
-        ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_EMULATOR);
+        ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_EMULATOR);
 
 #ifdef ERTS_SMP
 
-- 
cgit v1.2.3


From 4eb3df5abc90c9fcbae3b22f42779770b7b08526 Mon Sep 17 00:00:00 2001
From: Jonas Falkevik <jonas.falkevik@mobilearts.com>
Date: Wed, 4 Oct 2017 10:41:11 +0200
Subject: sctp: change connect to only return connect errors

gen_sctp:connect_init/4 could return error even though the connect are in progress.
I.e. connect on socket returned "in progress".
The error can come from packet_inet_output() which is checking the socket for errors.

The issue here is that from erlang you don't think the connect is still ongoing.
But you will receive a sctp_assoc_change message later on.
---
 erts/emulator/drivers/common/inet_drv.c | 83 ++-------------------------------
 1 file changed, 3 insertions(+), 80 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 7b1f4a0e9c..a7ee182010 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -1136,7 +1136,6 @@ static int        packet_inet_init(void);
 static void       packet_inet_stop(ErlDrvData);
 static void       packet_inet_command(ErlDrvData, char*, ErlDrvSizeT);
 static void       packet_inet_drv_input(ErlDrvData data, ErlDrvEvent event);
-static void	  packet_inet_drv_output(ErlDrvData data, ErlDrvEvent event);
 static ErlDrvData udp_inet_start(ErlDrvPort, char* command);
 #ifdef HAVE_SCTP
 static ErlDrvData sctp_inet_start(ErlDrvPort, char* command);
@@ -1161,7 +1160,7 @@ static struct erl_drv_entry udp_inet_driver_entry =
     NULL, 
 #else
     packet_inet_drv_input,
-    packet_inet_drv_output,
+    NULL,
 #endif
     "udp_inet",
     NULL,
@@ -1196,7 +1195,7 @@ static struct erl_drv_entry sctp_inet_driver_entry =
     NULL, 
 #else
     packet_inet_drv_input,
-    packet_inet_drv_output,
+    NULL,
 #endif
     "sctp_inet",
     NULL,
@@ -1262,7 +1261,6 @@ typedef struct {
 
 
 static int packet_inet_input(udp_descriptor* udesc, HANDLE event);
-static int packet_inet_output(udp_descriptor* udesc, HANDLE event);
 #endif
 
 /* convert descriptor pointer to inet_descriptor pointer */
@@ -11297,24 +11295,20 @@ static ErlDrvSSizeT packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf,
 		 (desc->sfamily, &remote, &buf, &len)) != NULL)
 	        return ctl_xerror(xerror, rbuf, rsize);
 	
-	    sock_select(desc, FD_CONNECT, 1);
 	    code = sock_connect(desc->s, &remote.sa, len);
 
 	    if (IS_SOCKET_ERROR(code) && (sock_errno() == EINPROGRESS)) {
 		/* XXX: Unix only -- WinSock would have a different cond! */
-		desc->state = INET_STATE_CONNECTING;
 		if (timeout != INET_INFINITY)
 		    driver_set_timer(desc->port, timeout);
 		enq_async(desc, tbuf, INET_REQ_CONNECT);
+		async_ok(desc);
 	    }
 	    else if (code == 0) { /* OK we are connected */
-		sock_select(desc, FD_CONNECT, 0);
-		desc->state = INET_STATE_CONNECTED;
 		enq_async(desc, tbuf, INET_REQ_CONNECT);
 		async_ok(desc);
 	    }
 	    else {
-		sock_select(desc, FD_CONNECT, 0);
 		return ctl_error(sock_errno(), rbuf, rsize);
 	    }
 	    return ctl_reply(INET_REP_OK, tbuf, 2, rbuf, rsize);
@@ -11829,77 +11823,6 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
     return count;
 }
 
-static void packet_inet_drv_output(ErlDrvData e, ErlDrvEvent event)
-{
-    (void)  packet_inet_output((udp_descriptor*)e, (HANDLE)event);
-}
-
-/* UDP/SCTP socket ready for output:
-**	This is a Back-End for Non-Block SCTP Connect (INET_STATE_CONNECTING)
-*/
-static int packet_inet_output(udp_descriptor* udesc, HANDLE event)
-{
-    inet_descriptor* desc = INETP(udesc);
-    int ret = 0;
-    ErlDrvPort ix = desc->port;
-
-    DEBUGF(("packet_inet_output(%ld) {s=%d\r\n", 
-	    (long)desc->port, desc->s));
-
-    if (desc->state == INET_STATE_CONNECTING) {
-	sock_select(desc, FD_CONNECT, 0);
-
-	driver_cancel_timer(ix);  /* posssibly cancel a timer */
-#ifndef __WIN32__
-	/*
-	 * XXX This is strange.  This *should* work on Windows NT too,
-	 * but doesn't.  An bug in Winsock 2.0 for Windows NT?
-	 *
-	 * See "Unix Netwok Programming", W.R.Stevens, p 412 for a
-	 * discussion about Unix portability and non blocking connect.
-	 */
-
-#ifndef SO_ERROR
-	{
-	    int sz = sizeof(desc->remote);
-	    int code = sock_peer(desc->s,
-				 (struct sockaddr*) &desc->remote, &sz);
-
-	    if (IS_SOCKET_ERROR(code)) {
-		desc->state = INET_STATE_OPEN;  /* restore state */
-		ret =  async_error(desc, sock_errno());
-		goto done;
-	    }
-	}
-#else
-	{
-	    int error = 0;	/* Has to be initiated, we check it */
-	    unsigned int sz = sizeof(error);   /* even if we get -1 */
-	    int code = sock_getopt(desc->s, SOL_SOCKET, SO_ERROR, 
-				   (void *)&error, &sz);
-
-	    if ((code < 0) || error) {
-		desc->state = INET_STATE_OPEN;  /* restore state */
-		ret = async_error(desc, error);
-		goto done;
-	    }
-	}
-#endif /* SO_ERROR */
-#endif /* !__WIN32__ */
-
-	desc->state = INET_STATE_CONNECTED;
-	async_ok(desc);
-    }
-    else {
-	sock_select(desc,FD_CONNECT,0);
-
-	DEBUGF(("packet_inet_output(%ld): bad state: %04x\r\n", 
-		(long)desc->port, desc->state));
-    }
- done:
-    DEBUGF(("packet_inet_output(%ld) }\r\n", (long)desc->port));
-    return ret;
-}
 #endif
 
 /*---------------------------------------------------------------------------*/
-- 
cgit v1.2.3


From 696f47b64c8ad5da71cbffd8fecf0f3bae1e7466 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 26 Oct 2017 12:57:24 +0200
Subject: Correct erlang:is_builtin/3 for apply/2 and yield/0

erlang:is_builtin(erlang, M, F) returns false for apply/2 and
yield/0. The documentation for erlang:is_builtin/3 says that it
returns true for BIFs that are implemented in C. apply/2 and
yield/0 are implemented in C (as BEAM instructions), and
therefore the correct return value is true.

Also see a similar argument that was made for apply/3 in the past:

    http://erlang.org/pipermail/erlang-bugs/2015-October/005101.html

https://bugs.erlang.org/browse/ERL-500
---
 erts/emulator/beam/beam_emu.c    | 17 ++++++++++-------
 erts/emulator/test/bif_SUITE.erl | 18 ++++++++++++------
 2 files changed, 22 insertions(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index aa94fbf536..60d0008d8f 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3199,13 +3199,16 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity)
     Export e;
     Export* ep;
 
-    if (Mod == am_erlang && Name == am_apply && arity == 3) {
-	/*
-	 * Special case. apply/3 is built-in (implemented in C),
-	 * but implemented in a different way than all other
-	 * BIFs.
-	 */
-	return 1;
+    if (Mod == am_erlang) {
+        /*
+         * Special case for built-in functions that are implemented
+         * as instructions as opposed to SNIFs.
+         */
+        if (Name == am_apply && (arity == 2 || arity == 3)) {
+            return 1;
+        } else if (Name == am_yield && arity == 0) {
+            return 1;
+        }
     }
 
     e.info.mfa.module = Mod;
diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl
index 146c37b118..e1b42e5d85 100644
--- a/erts/emulator/test/bif_SUITE.erl
+++ b/erts/emulator/test/bif_SUITE.erl
@@ -781,14 +781,20 @@ is_builtin(_Config) ->
 		       {F,A} <- M:module_info(exports)],
     Exp = ordsets:from_list(Exp0),
 
-    %% erlang:apply/3 is considered to be built-in, but is not
-    %% implemented as other BIFs.
+    %% Built-ins implemented as special instructions.
+    Instructions = [{erlang,apply,2},{erlang,apply,3},{erlang,yield,0}],
 
-    Builtins0 = [{erlang,apply,3}|erlang:system_info(snifs)],
+    Builtins0 = Instructions ++ erlang:system_info(snifs),
     Builtins = ordsets:from_list(Builtins0),
-    NotBuiltin = ordsets:subtract(Exp, Builtins),
-    _ = [true = erlang:is_builtin(M, F, A) || {M,F,A} <- Builtins],
-    _ = [false = erlang:is_builtin(M, F, A) || {M,F,A} <- NotBuiltin],
+
+    Fakes = [{M,F,42} || {M,F,_} <- Instructions],
+    All = ordsets:from_list(Fakes ++ Exp),
+    NotBuiltin = ordsets:subtract(All, Builtins),
+
+    _ = [{true,_} = {erlang:is_builtin(M, F, A),MFA} ||
+            {M,F,A}=MFA <- Builtins],
+    _ = [{false,_} = {erlang:is_builtin(M, F, A),MFA} ||
+            {M,F,A}=MFA <- NotBuiltin],
 
     ok.
 
-- 
cgit v1.2.3


From bb52a55b25e54ea362ab98c6306f1371d6a76104 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 27 Oct 2017 10:29:47 +0200
Subject: erts: Add Garbage Collection internal docs

---
 erts/emulator/internal_doc/GarbageCollection.md    | 187 +++++++++++++++++++++
 erts/emulator/internal_doc/figures/.gitignore      |   1 +
 erts/emulator/internal_doc/figures/Makefile        |  20 +++
 .../internal_doc/figures/gc-heap-scan1.dia         | Bin 0 -> 2671 bytes
 .../internal_doc/figures/gc-heap-scan1.png         | Bin 0 -> 63026 bytes
 .../emulator/internal_doc/figures/gc-heap-stop.dia | Bin 0 -> 2612 bytes
 .../emulator/internal_doc/figures/gc-heap-stop.png | Bin 0 -> 62596 bytes
 .../internal_doc/figures/gc-rootset-scan.dia       | Bin 0 -> 2482 bytes
 .../internal_doc/figures/gc-rootset-scan.png       | Bin 0 -> 59022 bytes
 erts/emulator/internal_doc/figures/gc-start.dia    | Bin 0 -> 1812 bytes
 erts/emulator/internal_doc/figures/gc-start.png    | Bin 0 -> 36619 bytes
 .../internal_doc/figures/gc-watermark-2.dia        | Bin 0 -> 2571 bytes
 .../internal_doc/figures/gc-watermark-2.png        | Bin 0 -> 56645 bytes
 .../emulator/internal_doc/figures/gc-watermark.dia | Bin 0 -> 1953 bytes
 .../emulator/internal_doc/figures/gc-watermark.png | Bin 0 -> 48480 bytes
 15 files changed, 208 insertions(+)
 create mode 100644 erts/emulator/internal_doc/GarbageCollection.md
 create mode 100644 erts/emulator/internal_doc/figures/.gitignore
 create mode 100644 erts/emulator/internal_doc/figures/Makefile
 create mode 100644 erts/emulator/internal_doc/figures/gc-heap-scan1.dia
 create mode 100644 erts/emulator/internal_doc/figures/gc-heap-scan1.png
 create mode 100644 erts/emulator/internal_doc/figures/gc-heap-stop.dia
 create mode 100644 erts/emulator/internal_doc/figures/gc-heap-stop.png
 create mode 100644 erts/emulator/internal_doc/figures/gc-rootset-scan.dia
 create mode 100644 erts/emulator/internal_doc/figures/gc-rootset-scan.png
 create mode 100644 erts/emulator/internal_doc/figures/gc-start.dia
 create mode 100644 erts/emulator/internal_doc/figures/gc-start.png
 create mode 100644 erts/emulator/internal_doc/figures/gc-watermark-2.dia
 create mode 100644 erts/emulator/internal_doc/figures/gc-watermark-2.png
 create mode 100644 erts/emulator/internal_doc/figures/gc-watermark.dia
 create mode 100644 erts/emulator/internal_doc/figures/gc-watermark.png

(limited to 'erts/emulator')

diff --git a/erts/emulator/internal_doc/GarbageCollection.md b/erts/emulator/internal_doc/GarbageCollection.md
new file mode 100644
index 0000000000..1d9e3f4160
--- /dev/null
+++ b/erts/emulator/internal_doc/GarbageCollection.md
@@ -0,0 +1,187 @@
+# Erlang Garbage Collector
+
+Erlang manages dynamic memory with a [tracing garbage collector](https://en.wikipedia.org/wiki/Tracing_garbage_collection). More precisely a per process generational semi-space copying collector using [Cheney's](#cheney) copy collection algorithm together with a global large object space.
+
+## Overview
+
+Each Erlang process has its own stack and heap which are allocated in the same memory block and grow towards each other. When the stack and the heap [meet](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/beam_emu.c#L387), the garbage collector is triggered and memory is reclaimed. If not enough memory was reclaimed, the heap will grow.
+
+### Creating Data
+
+Terms are created on the heap by evaluating expressions. There are two major types of terms: [immediate terms](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_term.h#L88-L97) which require no heap space (small integers, atoms, pids, port ids etc) and cons or [boxed terms](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_term.h#L106-L120) (tuple, big num, binaries etc) that do require heap space. Immediate terms do not need any heap space because they are embedded into the containing structure.
+
+Let's look at an example that returns a tuple with the newly created data.
+
+```erlang
+data(Foo) ->
+   Cons = [42|Foo],
+   Literal = {text, "hello world!"},
+   {tag, Cons, Literal}.
+```
+
+In this example we first create a new cons cell with an integer and a tuple with some text. Then a tuple of size three wrapping the other values with an atom tag is created and returned.
+
+On the heap tuples require a word size for each of its elements as well as for the header. Cons cells always require two words. Adding these things together, we get seven words for the tuples and 26 words for the cons cells. The string `"hello world!"` is a list of cons cells and thus requires 24 words. The atom `tag` and the integer `42` do not require any additional heap memory since it is an *immediate*. Adding all the terms together, the heap space required in this example should be 33 words.
+
+Compiling this code to beam assembly (`erlc -S`) shows exactly what is happening.
+
+```erlang
+    ...
+    {test_heap,6,1}.
+    {put_list,{integer,42},{x,0},{x,1}}.
+    {put_tuple,3,{x,0}}.
+    {put,{atom,tag}}.
+    {put,{x,1}}.
+    {put,{literal,{text,"hello world!"}}}.
+    return.
+```
+
+Looking at the assembler code we can see three things; The heap requirement in this function turns out to be only six words, as seen by the `{test_heap,6,1}` instruction. All the allocations are combined to a single instruction. The bulk of the data `{text, "hello world!"}` is a *literal*. Literals, sometimes referred to as constants, are not allocated in the function since they are a part of the module and allocated at load time.
+
+If there is not enough space available on the heap to satisfy the `test_heap` instructions request for memory, then a garbage collection is initiated. It may happen immediately in the `test_heap` instruction, or it can be delayed until a later time depending on what state the process is in. If the garbage collection is delayed, any memory needed will be allocated in heap fragments. Heap fragments are extra memory blocks that are a part of the young heap, but are not allocated in the contigious area where terms normally reside. See [The young heap](#the-young-heap) for more details.
+
+### The collector
+
+Erlang has a copying semi-space garbage collector. This means that when doing a garbage collection, the terms are copied from one distinct area, called the *from space*, to a new clean area, called the *to space*. The collector starts by [scanning the root-set](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_gc.c#L1980) (stack, registers, etc).
+
+![Garbage collection: initial values](figures/gc-start.png)
+
+It follows all the pointers from the root-set to the heap and copies each term word by word to the *to space*.
+
+After the header word has been copied a [*move marker*](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_gc.h#L45-L46) is destructively placed in it pointing to the term in the *to space*. Any other term that points to the already moved term will [see this move marker](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_gc.c#L1125) and copy the referring pointer instead. For example, if the have the following Erlang code:
+
+```erlang
+foo(Arg) ->
+    T = {test, Arg},
+    {wrapper, T, T, T}.
+```
+
+Only one copy of T exists on the heap and during the garbage collection only the first time T is encountered will it be copied.
+
+![Garbage collection: root set scan](figures/gc-rootset-scan.png)
+
+After [all terms](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_gc.c#L1089) referenced by the root-set have been copied, the collector scans the *to space* and copies all terms that these terms reference. When scanning, the collector steps through each term on the *to space* and any term still referencing the *from space* is copied over to the *to space*. Some terms contain non-term data (the payload of a on heap binary for instance). When encountered by the collector, these values are simply skipped.
+
+![Garbage collection: heap scan](figures/gc-heap-scan1.png)
+
+Every term object we can reach is copied to the *to space* and stored on top off the *scan stop* line, and then the scan stop is moved to the end of the last object.
+
+![Garbage collection: heap scan](figures/gc-heap-stop.png)
+
+When *scan stop* marker [catches up](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_gc.c#L1103) to the *scan start* marker, the garbage collection is done. At this point we can [deallocate](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_gc.c#L1206) the entire *from space* and therefore reclaim the entire young heap.
+
+## Generational Garbage Collection
+
+In addition to the collection algorithm described above, the Erlang garbage collector also provides generational garbage collection. An additional heap, called the old heap, is used where the long lived data is stored. The original heap is called the young heap, or sometimes the allocation heap.
+
+With this in mind we can look at the Erlang's garbage collection again. During the copy stage anything that should be copied to the young *to space* is instead copied to the old *to space* *if* it is [below the *high-watermark*](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_gc.c#L1127).
+
+![Garbage collection: heap scan](figures/gc-watermark.png)
+
+The [*high-watermark*](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_process.h#L1021) is placed where the previous garbage collection (described in [Overview](#overview)) ended and we have introduced a new area called the old heap. When doing the normal garbage collection pass, any term that is located below the high-watermark is copied to the old *to space* instead of the young.
+
+![Garbage collection: heap scan](figures/gc-watermark-2.png)
+
+In the next garbage collection, any pointers to the old heap will be ignored and not scanned. This way the garbage collector does not have to scan the long-lived terms.
+
+Generational garbage collection aims to increase performance at the expense of memory. This is achieved because only the young, smaller, heap is considered in most garbage collections.
+
+The generational [hypothesis](#ungar) predicts that most terms tend to die young, and for an immutable language such as Erlang, young terms die even faster than in other languages. So for most usage patterns the data in the new heap will die very soon after it is allocated. This is good because it limits the amount of data copied to the old heap and also because the garbage collection algorithm used is proportional to the amount of live data on the heap.
+
+One critical issue to note here is that any term on the young heap can reference terms on the old heap but *no* term on the old heap may refer to a term on the young heap. This is due to the nature of the copy algorithm. Anything referenced by an old heap term is not included in the reference tree, root-set and its followers, and hence is not copied. If it was, the data would be lost, fire and brimstone would rise to cover the earth. Fortunately, this comes naturally for Erlang because the terms are immutable and thus there can be no pointers modified on the old heap to point to the young heap.
+
+To reclaim data from the old heap, both young and old heaps are included during the collection and copied to a common *to space*. Both the *from space* of the young and old heap are then deallocated and the procedure will start over from the beginning. This type of garbage collection is called a full sweep and is triggered when the size of the area under the high-watermark is larger than the size of the free area of the old heap. It can also be triggered by doing a manual call to [erlang:garbage_collect()](http://erlang.org/doc/man/erlang.html#garbage_collect-0), or by running into the young garbage collection limit set by [spawn_opt(fun(),[{fullsweep_after, N}])](http://erlang.org/doc/man/erlang.html#spawn_opt-4) where N is the number of young garbage collections to do before forcing a garbage collection of both young and old heap.
+
+## The young heap
+
+The young heap, or the allocation heap, consists of the stack and heap as described in the Overview. However, it also includes any heap fragments that are attached to the heap. All of the heap fragments are considered to be above the high-watermark and part of the young generation. Heap fragments contain terms that either did not fit on the heap, or were created by another process and then attached to the heap. For instance if the bif binary_to_term created a term which does not fit on the current heap without doing a garbage collection, it will create a heap-fragment for the term and then schedule a garbage collection for later. Also if a message is sent to the process, the payload may be placed in a heap-fragment and that fragment is added to young heap when the message is matched in a receive clause.
+
+This procedure differs from how it worked prior to Erlang/OTP 19.0. Before 19.0, only a contiguous memory block where the young heap and stack resided was considered to be part of the young heap. Heap fragments and messages were immediately copied into the young heap before they could be inspected by the Erlang program. The behaviour introduced in 19.0 is superior in many ways - most significantly it reduces the number of necessary copy operations and the root set for garbage collection.
+
+## Sizing the heap
+
+As mentioned in the Overview the size of the heap [grows](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_gc.c#L247) to accommodate more data. Heaps grow in two stages, first a [variation of the Fibonacci sequence](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_gc.c#L199-L208) is used starting at 233 words. Then at about 1 mega words the heap only [grows in 20% increments](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_gc.c#L215-L227).
+
+There are two occasions when the young heap grows:
+
+* if the total size of the heap + message and heap fragments exceeds the current heap size.
+* if after a fullsweep, the total amount of live objects is greater than 75%.
+
+There are two occasions when the young heap is shrunk:
+
+* if after a young collection, the total amount of live objects is less than 25% of the heap and the young heap is "big"
+* if after a fullsweep, the total amount of live objects is less than 25% of the heap.
+
+The old heap is always one step ahead in the heap growth stages than the young heap.
+
+## Literals
+
+When garbage collecting a heap (young or old) all literals are left in place and not copied. To figure out if a term should be copied or not when doing a garbage collection the following pseudo code is used:
+
+```c
+if (erts_is_literal(ptr) || (on_old_heap(ptr) && !fullsweep)) {
+  /* literal or non fullsweep - do not copy */
+} else {
+  copy(ptr);
+}
+```
+
+The [`erts_is_literal`](https://github.com/erlang/otp/blob/OTP-19.0/erts/emulator/beam/global.h#L1452-L1465) check works differently on different architectures and operating systems.
+
+On 64 bit systems that allow mapping of unreserved virtual memory areas (most operating systems except Windows), an area of size 1 GB (by default) is mapped and then all literals are placed within that area. Then all that has to be done to determine if something is a literal or not is [two quick pointer checks](https://github.com/erlang/otp/blob/OTP-19.0/erts/emulator/beam/erl_alloc.h#L322-L324). This system relies on the fact that a memory page that has not been touched yet does not take any actual space. So even if 1 GB of virtual memory is mapped, only the memory which is actually needed for literals is allocated in ram. The size of the literal area is configurable through the +MIscs erts_alloc option.
+
+On 32 bit systems, there is not enough virtual memory space to allocate 1 GB for just literals, so instead small 256 KB sized literal regions are created on demand and a card mark bit-array of the entire 32 bit memory space is then used to determine if a term is a literal or not. Since the total memory space is only 32 bits, the card mark bit-array is only 256 words large. On a 64 bit system the same bit-array would have to be 1 tera words large, so this technique is only viable on 32 bit systems. Doing [lookups in the array](https://github.com/erlang/otp/blob/OTP-19.0/erts/emulator/beam/erl_alloc.h#L316-L319) is a little more expensive then just doing the pointer checks that can be done in 64 bit systems, but not extremely so.
+
+On 64 bit windows, on which erts_alloc cannot do unreserved virtual memory mappings, a [special tag](https://github.com/erlang/otp/blob/OTP-19.0/erts/emulator/beam/erl_term.h#L59) within the Erlang term object is used to determine if something [is a literal or not](https://github.com/erlang/otp/blob/OTP-19.0/erts/emulator/beam/erl_term.h#L248-L252). This is very cheap, however, the tag is only available on 64 bit machines, and it is possible to do a great deal of other nice optimizations with this tag in the future (like for instance a more compact list implementation) so it is not used on operating systems where it is not needed.
+
+This behaviour is different from how it worked prior to Erlang/OTP 19.0. Before 19.0 the literal check was done by checking if the pointer pointed to the young or old heap block. If it did not, then it was considered a literal. This lead to considerable overhead and strange memory usage scenarios, so it was removed in 19.0.
+
+## Binary heap
+
+The binary heap works as a large object space for binary terms that are greater than 64 bytes (from now on called off-heap binaries). The binary heap is [reference counted](https://en.wikipedia.org/wiki/Reference_counting) and a pointer to the off-heap binary is stored on the process heap. To keep track of when to decrement the reference counter of the off-heap binary, a linked list (the MSO - mark and sweep object list) containing funs and externals as well as off-heap binaries is woven through the heap. After a garbage collection is done, the [MSO list is swept](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_gc.c#L2299) and any off-heap binary that does not have a [move marker](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_gc.c#L2325) written into the header words has its reference [decremented and is potentially freed](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_gc.c#L2344-L2367).
+
+All items in the MSO list are ordered by the time they were added to the process heap, so when doing a minor garbage collection, the MSO sweeper only has to sweep until it [encounters an off-heap binary that is on the old heap](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_gc.c#L2369).
+
+### Virtual Binary heap
+
+Each process has a virtual binary heap associated with it that has the size of all the current off-heap binaries that the process has references to. The virtual binary heap also has a limit and grows and shrinks depending on how off-heap binaries are used by the process. The same growth and shrink mechanisms are used for the binary heap and for the term heap, so first a Fibonacci like series and then 20% growth.
+
+The virtual binary heap exists in order to [trigger](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/beam_emu.c#L364) garbage collections earlier when potentially there is a very large amount of off-heap binary data that could be reclaimed. This approach does not catch all problems with binary memory not being released soon enough, but it does catch a lot of them.
+
+## Messages
+
+Messages can become a part of the process heap at different times. This depends on how the process is configured.
+We can configure the behaviour of each process using `process_flag(message_queue_data, off_heap | on_heap)` or we can set a default for all processes at start using the option `+hmqd`.
+
+What do these different configurations do and when should we use them?
+Let's start by going through what happens when one Erlang process sends a message to another.
+The sending process needs to do a couple of things:
+
+1. calculate [how large](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_message.c#L1031) the message to be sent is
+2. [allocate enough space](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_message.c#L1033) to fit the entire message
+3. [copy](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_message.c#L1040) the message payload
+4. allocate a message container with some meta data
+5. [insert](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_message.c#L502) the message container in the receiver process' [message queue](https://github.com/erlang/otp/blob/OTP-18.0/erts/emulator/beam/erl_process.h#L1042)
+
+The process flag `message_queue_data`, of the receiver process, controls the message allocating strategy of the sender process in step 2 and also how the message data is treated by the garbage collector.
+
+The procedure above is different from how it worked prior to 19.0. Before 19.0 there was no configuration option, the behaviour was always very similar to how the `on_heap` option is in 19.0.
+
+### Message allocating strategies
+
+If set to `on_heap`, the sending process will first attempt to allocate the space for the message directly on the young heap block of the receiving process.
+This is not always possible as it requires taking the *main lock* of the receiving process. The main lock is also held when the process is executing. The possibility for a lock conflict is thus likely in an intensely collaborating system.
+If the sending process cannot acquire the main lock, a heap fragment is instead created for the message and the message payload is copied onto that.
+With the `off_heap` option the sender process always creates heap fragments for messages sent to that process.
+
+There are a bunch of different tradeoffs that come into play when trying to figure out which of the strategies you want to use.
+
+Using `off_heap` may seem like a nice way to get a more scalable system as you get very little contention on the main locks, however, allocating a heap fragment is more expensive than allocating on the heap of the receiving process. So if it is very unlikely that contention will occur, it is more efficient to try to allocate the message directly on the receiving process' heap.
+
+Using `on_heap` will force all messages to be part of on the young heap which will increase the amount of data that the garbage collector has to move. So if a garbage collection is triggered while processing a large amount of messages, they will be copied to the young heap. This in turn will lead to that the messages will quickly be promoted to the old heap and thus increase its size. This may be good or bad depending on exactly what the process does. A large old heap means that the young heap will also be larger, which in turn means that less garbage collections will be triggered while processing the message queue. This will temporarly increase the throughput of the process at the cost of more memory usage. However, if after all the messages have been consumed the process enters a state where a lot less messages are being received. Then it may be a long time before the next fullsweep garbage collection happens and the messages that are on the old heap will be there until that happens. So while `on_heap` is potentially faster than the other modes, it uses more memory for a longer time. This mode is the legacy mode which is almost how the message queue was handled before Erlang/OTP 19.0.
+
+Which one of these strategies is best depends a lot on what the process is doing and how it interacts with other processes. So, as always, profile the application and see how it behaves with the different options.
+
+   <a name="cheney">[1]</a>: C. J. Cheney. A nonrecursive list compacting algorithm. Commun. ACM, 13(11):677–678, Nov. 1970.
+
+   <a name="ungar">[2]</a>: D. Ungar. Generation scavenging: A non-disruptive high performance storage reclamation algorithm. SIGSOFT Softw. Eng. Notes, 9(3):157–167, Apr. 1984.
diff --git a/erts/emulator/internal_doc/figures/.gitignore b/erts/emulator/internal_doc/figures/.gitignore
new file mode 100644
index 0000000000..c2813ac866
--- /dev/null
+++ b/erts/emulator/internal_doc/figures/.gitignore
@@ -0,0 +1 @@
+*.eps
\ No newline at end of file
diff --git a/erts/emulator/internal_doc/figures/Makefile b/erts/emulator/internal_doc/figures/Makefile
new file mode 100644
index 0000000000..111cb393fb
--- /dev/null
+++ b/erts/emulator/internal_doc/figures/Makefile
@@ -0,0 +1,20 @@
+# In order to update the figures you have to have both dia
+# and imagemagick installed.
+
+DIAGRAMS=$(wildcard *.dia)
+EPS_DIAGRAMS=$(patsubst %.dia,%.eps,$(DIAGRAMS))
+PNG_DIAGRAMS=$(patsubst %.dia,%.png,$(DIAGRAMS))
+
+diagrams: $(EPS_DIAGRAMS)
+
+png: $(PNG_DIAGRAMS)
+
+update_png: png
+	git add $(PNG_DIAGRAMS)
+	git commit -m "Update internal docs figures"
+
+%.eps: %.dia
+	dia --export=$@ $<
+
+%.png: %.eps
+	convert $< -resize 65% $@
diff --git a/erts/emulator/internal_doc/figures/gc-heap-scan1.dia b/erts/emulator/internal_doc/figures/gc-heap-scan1.dia
new file mode 100644
index 0000000000..b8a32dc092
Binary files /dev/null and b/erts/emulator/internal_doc/figures/gc-heap-scan1.dia differ
diff --git a/erts/emulator/internal_doc/figures/gc-heap-scan1.png b/erts/emulator/internal_doc/figures/gc-heap-scan1.png
new file mode 100644
index 0000000000..9724fc8698
Binary files /dev/null and b/erts/emulator/internal_doc/figures/gc-heap-scan1.png differ
diff --git a/erts/emulator/internal_doc/figures/gc-heap-stop.dia b/erts/emulator/internal_doc/figures/gc-heap-stop.dia
new file mode 100644
index 0000000000..af219958c6
Binary files /dev/null and b/erts/emulator/internal_doc/figures/gc-heap-stop.dia differ
diff --git a/erts/emulator/internal_doc/figures/gc-heap-stop.png b/erts/emulator/internal_doc/figures/gc-heap-stop.png
new file mode 100644
index 0000000000..ec79790d36
Binary files /dev/null and b/erts/emulator/internal_doc/figures/gc-heap-stop.png differ
diff --git a/erts/emulator/internal_doc/figures/gc-rootset-scan.dia b/erts/emulator/internal_doc/figures/gc-rootset-scan.dia
new file mode 100644
index 0000000000..d6147740e5
Binary files /dev/null and b/erts/emulator/internal_doc/figures/gc-rootset-scan.dia differ
diff --git a/erts/emulator/internal_doc/figures/gc-rootset-scan.png b/erts/emulator/internal_doc/figures/gc-rootset-scan.png
new file mode 100644
index 0000000000..06509f83c3
Binary files /dev/null and b/erts/emulator/internal_doc/figures/gc-rootset-scan.png differ
diff --git a/erts/emulator/internal_doc/figures/gc-start.dia b/erts/emulator/internal_doc/figures/gc-start.dia
new file mode 100644
index 0000000000..41832d29b0
Binary files /dev/null and b/erts/emulator/internal_doc/figures/gc-start.dia differ
diff --git a/erts/emulator/internal_doc/figures/gc-start.png b/erts/emulator/internal_doc/figures/gc-start.png
new file mode 100644
index 0000000000..6c29e9adb0
Binary files /dev/null and b/erts/emulator/internal_doc/figures/gc-start.png differ
diff --git a/erts/emulator/internal_doc/figures/gc-watermark-2.dia b/erts/emulator/internal_doc/figures/gc-watermark-2.dia
new file mode 100644
index 0000000000..2ea3be3fb2
Binary files /dev/null and b/erts/emulator/internal_doc/figures/gc-watermark-2.dia differ
diff --git a/erts/emulator/internal_doc/figures/gc-watermark-2.png b/erts/emulator/internal_doc/figures/gc-watermark-2.png
new file mode 100644
index 0000000000..bca110282d
Binary files /dev/null and b/erts/emulator/internal_doc/figures/gc-watermark-2.png differ
diff --git a/erts/emulator/internal_doc/figures/gc-watermark.dia b/erts/emulator/internal_doc/figures/gc-watermark.dia
new file mode 100644
index 0000000000..a5de7565dc
Binary files /dev/null and b/erts/emulator/internal_doc/figures/gc-watermark.dia differ
diff --git a/erts/emulator/internal_doc/figures/gc-watermark.png b/erts/emulator/internal_doc/figures/gc-watermark.png
new file mode 100644
index 0000000000..b835c7694e
Binary files /dev/null and b/erts/emulator/internal_doc/figures/gc-watermark.png differ
-- 
cgit v1.2.3


From c9765df137370b692cd9476ff5a4d00122b2ba93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 25 Oct 2017 10:39:29 +0200
Subject: erl_process_dump: Don't assume that literals can be found

Native code does not register its literals in the code header for the
loaded code. Therefore, a literal created by native code can not be
found by mark_literal(). Ignore literals that can't be found instead
of crashing (the crasdump_viewer will report such literals as
incomplete heap data, but will not crash).
---
 erts/emulator/beam/erl_process_dump.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 9fd024cae8..5641195458 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -728,8 +728,15 @@ static void mark_literal(Eterm* ptr)
 
     ap = bsearch(ptr, lit_areas, num_lit_areas, sizeof(ErtsLiteralArea*),
                  search_areas);
-    ASSERT(ap);
-    ap[0]->off_heap = (struct erl_off_heap_header *) 1;
+
+    /*
+     * If the literal was created by native code, this search will not
+     * find it and ap will be NULL.
+     */
+
+    if (ap) {
+        ap[0]->off_heap = (struct erl_off_heap_header *) 1;
+    }
 }
 
 
-- 
cgit v1.2.3


From 553896346c987d27958adc392940a3a197bbcdc4 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 18 Oct 2017 11:46:27 +0200
Subject: erts: tcp send should return {error,closed}

In the scenario where a gen_tcp:recv/2 detected an
error, the next gen_tcp:send should get a closed
error and not a enotconn error as was the case before.
---
 erts/emulator/drivers/common/inet_drv.c | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 7b1f4a0e9c..767fec3c98 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -9813,6 +9813,12 @@ static int tcp_recv_closed(tcp_descriptor* desc)
 	set_busy_port(desc->inet.port, 0);
 	inet_reply_error_am(INETP(desc), am_closed);
 	DEBUGF(("tcp_recv_closed(%ld): busy reply 'closed'\r\n", port));
+    } else {
+        /* No blocking send op to reply to right now.
+         * If next op is a send, make sure it returns {error,closed}
+         * rather than {error,enotconn}.
+         */
+        desc->tcp_add_flags |= TCP_ADDF_DELAYED_CLOSE_SEND;
     }
     if (!desc->inet.active) {
 	/* We must cancel any timer here ! */
-- 
cgit v1.2.3


From 1b00539d6db8e349c6782bcd38ab006420734106 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 18 Oct 2017 16:01:03 +0200
Subject: erts: Fix a bunch of compiler warnings

---
 erts/emulator/beam/beam_debug.c     | 5 ++++-
 erts/emulator/beam/beam_load.c      | 2 +-
 erts/emulator/beam/bif.c            | 2 +-
 erts/emulator/beam/erl_io_queue.c   | 4 ++--
 erts/emulator/beam/erl_io_queue.h   | 6 +++---
 erts/emulator/beam/erl_term.h       | 2 +-
 erts/emulator/sys/common/erl_poll.c | 6 ++++--
 7 files changed, 16 insertions(+), 11 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 70078c8c59..509aa2a84f 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -408,7 +408,10 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
         addr++;
         ap = addr;
     } else {
-        BeamInstr instr_word = addr++[0];
+#if defined(ARCH_64) && defined(CODE_MODEL_SMALL)
+        BeamInstr instr_word = addr[0];
+#endif
+        addr++;
 
 	/*
 	 * Copy all arguments to a local buffer for the unpacking.
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 00dd28b26c..1468540cc6 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -5027,7 +5027,7 @@ freeze_code(LoaderState* stp)
                  */
                 codev[pos] = (BeamInstr) (codev + value);
             } else {
-#ifdef DEBUG
+#if defined(DEBUG) && defined(BEAM_WIDE_MASK)
                 Uint w;
 #endif
                 Sint32 rel = lp->offset + value;
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index ad555bb195..4b11884f38 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4200,7 +4200,7 @@ BIF_RETTYPE list_to_port_1(BIF_ALIST_1)
 
     cp += 6; /* strlen("#Port<") */
 
-    if (sscanf(cp, "%u.%u>", &n, &p) < 2)
+    if (sscanf(cp, "%u.%u>", (unsigned int*)&n, (unsigned int*)&p) < 2)
         goto bad;
 
     if (p > ERTS_MAX_PORT_NUMBER)
diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c
index 190ba6bbb9..40d69ea6b0 100644
--- a/erts/emulator/beam/erl_io_queue.c
+++ b/erts/emulator/beam/erl_io_queue.c
@@ -658,7 +658,7 @@ io_list_vec_count(Eterm obj, Uint *v_size,
 int
 erts_ioq_iolist_vec_len(Eterm obj, int* vsize, Uint* csize,
                         Uint* pvsize, Uint* pcsize,
-                        Uint* total_size, Uint blimit)
+                        size_t* total_size, Uint blimit)
 {
     DECLARE_ESTACK(s);
     Eterm* objp;
@@ -669,7 +669,7 @@ erts_ioq_iolist_vec_len(Eterm obj, int* vsize, Uint* csize,
     Uint p_v_size = 0;
     Uint p_c_size = 0;
     Uint p_in_clist = 0;
-    Uint total;
+    size_t total;
 
     goto L_jump_start;  /* avoid a push */
 
diff --git a/erts/emulator/beam/erl_io_queue.h b/erts/emulator/beam/erl_io_queue.h
index 51abe99510..7d0fe6751c 100644
--- a/erts/emulator/beam/erl_io_queue.h
+++ b/erts/emulator/beam/erl_io_queue.h
@@ -103,7 +103,7 @@ Uint erts_ioq_sizeq(ErtsIOQueue *q);
 
 int erts_ioq_iolist_vec_len(Eterm obj, int* vsize, Uint* csize,
                             Uint* pvsize, Uint* pcsize,
-                            Uint* total_size, Uint blimit);
+                            size_t* total_size, Uint blimit);
 int erts_ioq_iolist_to_vec(Eterm obj, SysIOVec* iov,
                            ErtsIOQBinary** binv, ErtsIOQBinary* cbin,
                            Uint bin_limit, int driver_binary);
@@ -111,7 +111,7 @@ int erts_ioq_iolist_to_vec(Eterm obj, SysIOVec* iov,
 ERTS_GLB_INLINE
 int erts_ioq_iodata_vec_len(Eterm obj, int* vsize, Uint* csize,
                             Uint* pvsize, Uint* pcsize,
-                            Uint* total_size, Uint blimit);
+                            size_t* total_size, Uint blimit);
 ERTS_GLB_INLINE
 int erts_ioq_iodata_to_vec(Eterm obj, SysIOVec* iov,
                            ErtsIOQBinary** binv, ErtsIOQBinary* cbin,
@@ -123,7 +123,7 @@ int erts_ioq_iodata_to_vec(Eterm obj, SysIOVec* iov,
 ERTS_GLB_INLINE
 int erts_ioq_iodata_vec_len(Eterm obj, int* vsize, Uint* csize,
                             Uint* pvsize, Uint* pcsize,
-                            Uint* total_size, Uint blimit) {
+                            size_t* total_size, Uint blimit) {
   if (is_binary(obj)) {
     /* We optimize for when we get a procbin without a bit-offset
      * that fits in one iov slot
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 6daf043117..5ec6b6b44b 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -862,7 +862,7 @@ do {									\
   ((ErtsMRefThing *) (Hp))->mb = (Binp);				\
   ((ErtsMRefThing *) (Hp))->next = (Ohp)->first;			\
   (Ohp)->first = (struct erl_off_heap_header*) (Hp);			\
-  ASSERT(erts_is_ref_numbers_magic(&(Binp)->refn));			\
+  ASSERT(erts_is_ref_numbers_magic((Binp)->refn));			\
 } while (0)
 
 #endif /* ARCH_32 */
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index 30a595c17a..7aa53e8f36 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -2059,7 +2059,8 @@ uint32_t epoll_events(int kp_fd, int fd)
         int ev_fd;
         uint32_t events;
         uint64_t data;
-        if (fscanf(f,"tfd:%d events:%x data:%lx\n", &ev_fd, &events, &data) != 3) {
+        if (fscanf(f,"tfd:%d events:%x data:%llx\n", &ev_fd, &events,
+                   (unsigned long long*)&data) != 3) {
             fprintf(stderr,"failed to parse file %s on line %d, errno = %d\n", fname,
                     line,
                     errno);
@@ -2125,7 +2126,8 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet *ps,
         int fd;
         uint32_t events;
         uint64_t data;
-        if (fscanf(f,"tfd:%d events:%x data:%lx\n", &fd, &events, &data) != 3) {
+        if (fscanf(f,"tfd:%d events:%x data:%llx\n", &fd, &events,
+                   (unsigned long long*)&data) != 3) {
             fprintf(stderr,"failed to parse file %s on line %d, errno = %d\n",
                     fname, line, errno);
             ASSERT(0);
-- 
cgit v1.2.3


From 6b5efa639ae8723745f860b793e74910c8d6798b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 31 Oct 2017 08:54:41 +0100
Subject: beam_makeops: Strengthen the checking of transformations

Any type that is a lower-case letter is allowed in a transformation,
even though some of them don't make sense, could cause compilation
errors when building the emulator, or crash at runtime. For example,
the type 'z' is emitted by the compiler, but eaten by the loader
before the transform engine can see it. Another example is the 'q'
type, which would crash the emulator if it was used on the right
side of a transformation rule.

In beam_makeops, define explicitly which types that are allowed in
patterns (on the left side) and in construction (on the right side).
Using that information, check the transformation rules thoroughly and
reject operands that don't make sense.

While we are it, also correct some misleading comments.
---
 erts/emulator/utils/beam_makeops | 75 ++++++++++++++++++++++++++++++++--------
 1 file changed, 60 insertions(+), 15 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index d7791d23fa..02227bbdfd 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -131,7 +131,10 @@ my $loader_types = "nprvlqo";
 my $genop_types = $compiler_types . $loader_types;
 
 #
-# Defines the argument types and their loaded size assuming no packing.
+# Define the operand types and their loaded size assuming no packing.
+#
+# Those are the types that can be used in the definition of a specific
+# instruction.
 #
 my %arg_size = ('r' => 0,	# x(0) - x register zero
 		'x' => 1,	# x(N), N > 0 - x register
@@ -154,11 +157,34 @@ my %arg_size = ('r' => 0,	# x(0) - x register zero
 		'A' => 1,	# arity value
 		'P' => 1,	# byte offset into tuple or stack
 		'Q' => 1,	# like 'P', but packable
-		'h' => 1,	# character
+		'h' => 1,	# character (not used)
 		'l' => 1,	# float reg
 		'q' => 1,	# literal term
 	     );
 
+#
+# Define the types that may be used in a transformation rule.
+#
+# %pattern_type defines the types that may be used in a pattern
+# on the left side.
+#
+# %construction_type defines the types that may be used when
+# constructing a new instruction on the right side (a subset of
+# the pattern types that are possible to construct).
+#
+my $pattern_types = "acdfjilnopqsuxy";
+my %pattern_type;
+@pattern_type{split("", $pattern_types)} = (1) x length($pattern_types);
+
+my %construction_type;
+foreach my $type (keys %pattern_type) {
+    $construction_type{$type} = 1
+        if index($genop_types, $type) >= 0;
+}
+foreach my $makes_no_sense ('f', 'j', 'o', 'p', 'q') {
+    delete $construction_type{$makes_no_sense};
+}
+
 #
 # Generate bits.
 #
@@ -194,7 +220,8 @@ sub define_type_bit {
     define_type_bit('S', $type_bit{'d'});
     define_type_bit('j', $type_bit{'f'} | $type_bit{'p'});
 
-    # Aliases (for matching purposes).
+    # Aliases of 'u'.  Those specify how to load the operand and
+    # what kind of packing can be done.
     define_type_bit('t', $type_bit{'u'});
     define_type_bit('I', $type_bit{'u'});
     define_type_bit('W', $type_bit{'u'});
@@ -2156,12 +2183,19 @@ sub tr_parse_op {
     if (/^([a-z*]+)(.*)/) {
 	$type = $1;
 	$_ = $2;
+        error("$type: only a single type is allowed on right side of transformations")
+            if not $src and length($type) > 1;
 	foreach (split('', $type)) {
-	    error("bad type in $op")
-		unless defined $type_bit{$_} or $type eq '*';
-	    $_ eq 'r' and
-		error("$op: 'r' is not allowed in transformations")
-	}
+            next if $src and $type eq '*';
+            error("$op: not a type")
+                unless defined $type_bit{$_};
+            error("$op: the type '$_' is not allowed in transformations")
+                unless defined $pattern_type{$_};
+            if (not $src) {
+                error("$op: type '$_' is not allowed on the right side of transformations")
+                    unless defined $construction_type{$_};
+            }
+        }
     }
 
     # Get an optional condition. (In source.)
@@ -2194,10 +2228,18 @@ sub tr_parse_op {
     }
 
     # Get an optional value. (In destination.)
-    $type_val = $type eq 'x' ? 1023 : 0;
+    if ($type eq 'x') {
+        $type_val = 1023;
+    } elsif ($type eq 'a') {
+        $type_val = 'am_Empty';
+    } else {
+        $type_val = 0;
+    }
     if (/^=(.*)/) {
-	error("value not allowed in source: $op")
+	error("$op: value not allowed in source")
 	    if $src;
+        error("$op: the type 'n' must not be given a value")
+            if $type eq 'n';
 	$type_val = $1;
 	$_ = '';
     }
@@ -2207,13 +2249,16 @@ sub tr_parse_op {
     error("garbage '$_' after operand: $op")
 	unless /^\s*$/;
 
-    # Test that destination has no conditions.
+    # Check the conditions.
 
-    unless ($src) {
-	error("condition not allowed in destination: $op")
+    if ($src) {
+        error("$op: the type '$type' is not allowed to be compared with a literal value")
+            if $cond and not $construction_type{$type};
+    } else {
+	error("$op: condition not allowed in destination")
 	    if $cond;
-	error("variable name and type cannot be combined in destination: $op")
-	    if $var && $type;
+	error("$op: variable name and type cannot be combined in destination")
+	    if $var and $type;
     }
 
     ($var,$type,$type_val,$cond,$cond_val);
-- 
cgit v1.2.3


From 9b1f2439845e5627a00fb816244af06d77c2d70a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 1 Nov 2017 16:53:51 +0100
Subject: Only apply EOS behaviors if there's pending data

---
 erts/emulator/nifs/common/zlib_nif.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/nifs/common/zlib_nif.c b/erts/emulator/nifs/common/zlib_nif.c
index fa29b4fb71..104e52754a 100644
--- a/erts/emulator/nifs/common/zlib_nif.c
+++ b/erts/emulator/nifs/common/zlib_nif.c
@@ -929,7 +929,7 @@ static ERL_NIF_TERM zlib_inflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar
         return enif_raise_exception(env, am_not_initialized);
     }
 
-    if(d->eos_seen) {
+    if(d->eos_seen && enif_ioq_size(d->input_queue) > 0) {
         int res;
 
         switch(d->eos_behavior) {
@@ -943,11 +943,10 @@ static ERL_NIF_TERM zlib_inflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar
             }
 
             d->eos_seen = 0;
+
             break;
         case EOS_BEHAVIOR_CUT:
             zlib_reset_input(d);
-
-            return enif_make_tuple2(env, am_finished, enif_make_list(env, 0));
         }
     }
 
-- 
cgit v1.2.3


From 47502282d75c3206dcfc7e2d31e768ca3d5458d7 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 2 Nov 2017 19:18:31 +0100
Subject: Remove process start time for crash dumps

---
 erts/emulator/beam/break.c        |  3 ---
 erts/emulator/beam/erl_process.c  |  2 --
 erts/emulator/beam/erl_process.h  |  1 -
 erts/emulator/beam/erl_time.h     |  3 ---
 erts/emulator/beam/erl_time_sup.c | 11 -----------
 5 files changed, 20 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index d9ee940662..a46063142e 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -209,7 +209,6 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext)
 void
 print_process_info(fmtfn_t to, void *to_arg, Process *p)
 {
-    time_t approx_started;
     int garbing = 0;
     int running = 0;
     struct saved_calls *scb;
@@ -258,8 +257,6 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p)
     }
 
     erts_print(to, to_arg, "Spawned by: %T\n", p->parent);
-    approx_started = (time_t) p->approx_started;
-    erts_print(to, to_arg, "Started: %s", ctime(&approx_started));
     ERTS_MSGQ_MV_INQ2PRIVQ(p);
     erts_print(to, to_arg, "Message queue length: %d\n", p->msg.len);
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 6654468fb6..0433b4ea17 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11511,7 +11511,6 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state)
 
     ASSERT(internal_pid_serial(p->common.id) <= ERTS_MAX_PID_SERIAL);
     
-    p->approx_started = erts_get_approx_time();
     p->rcount = 0;
     p->heap = NULL;
 
@@ -11947,7 +11946,6 @@ void erts_init_empty_process(Process *p)
     p->def_arg_reg[5] = 0;
 
     p->parent = NIL;
-    p->approx_started = 0;
     p->static_flags = 0;
 
     p->common.u.alive.started_interval = 0;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 66d7848f89..16b3526208 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1019,7 +1019,6 @@ struct process {
      * Information mainly for post-mortem use (erl crash dump).
      */
     Eterm parent;		/* Pid of process that created this process. */
-    erts_approx_time_t approx_started; /* Time when started. */
 
     Uint32 static_flags;        /* Flags that do *not* change */
 
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 27164d50a0..65211e4e6f 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -107,9 +107,6 @@ void erts_p_slpq(void);
 void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec);
 #endif
 
-typedef UWord erts_approx_time_t;
-erts_approx_time_t erts_get_approx_time(void);
-
 int erts_has_time_correction(void);
 int erts_check_time_adj_support(int time_correction,
 				ErtsTimeWarpMode time_warp_mode);
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index f2e0900fec..8cbdf9fa0f 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -191,17 +191,6 @@ static struct {
 
 ErtsTimeSupData erts_time_sup__ erts_align_attribute(ERTS_CACHE_LINE_SIZE);
 
-/*
- * erts_get_approx_time() returns an *approximate* time
- * in seconds. NOTE that this time may jump backwards!!!
- */
-erts_approx_time_t
-erts_get_approx_time(void)
-{
-    ErtsSystemTime stime = erts_os_system_time();
-    return (erts_approx_time_t) ERTS_MONOTONIC_TO_SEC(stime);
-}
-
 static ERTS_INLINE void
 init_time_offset(ErtsMonotonicTime offset)
 {
-- 
cgit v1.2.3


From 31e6c79f956f71e80d8f6be48e5ab8df87f2b85c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 19 Oct 2017 20:44:54 +0200
Subject: Prevent hipe_bs_validate_unicode from doing GC

Fix for x86_64 only.

The calling native code can not handle a GC
as it has a raw pointer where to write the binary data.

If a GC happens the data (utf32) will be written
to the old deallocated heap.
---
 erts/emulator/hipe/hipe_amd64_bifs.m4 | 37 +++++++++++++++++++++++++++++++++++
 erts/emulator/hipe/hipe_bif_list.m4   |  4 ++++
 2 files changed, 41 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_amd64_bifs.m4 b/erts/emulator/hipe/hipe_amd64_bifs.m4
index dca3887564..aff10f1528 100644
--- a/erts/emulator/hipe/hipe_amd64_bifs.m4
+++ b/erts/emulator/hipe/hipe_amd64_bifs.m4
@@ -462,6 +462,43 @@ ASYM($1):
 	TYPE_FUNCTION(ASYM($1))
 #endif')
 
+/*
+ * nogc_bif_interface_1(nbif_name, cbif_name)
+ *
+ * Generate native interface for a bif with implicit P
+ * The bif can fail but cannot do GC.
+ */
+
+define(nogc_bif_interface_1,
+`
+#ifndef HAVE_$1
+#`define' HAVE_$1
+	TEXT
+	.align	4
+	GLOBAL(ASYM($1))
+ASYM($1):
+	/* set up the parameters */
+	movq	P, %rdi
+	NBIF_ARG(%rsi,1,0)
+
+	/* make the call on the C stack */
+	SWITCH_ERLANG_TO_C
+	pushq	%rsi
+	movq	%rsp, %rsi	/* Eterm* BIF__ARGS */
+	sub	$(8), %rsp	/* stack frame 16-byte alignment */
+	CALL_BIF($2)
+	add	$(1*8 + 8), %rsp
+	SWITCH_C_TO_ERLANG
+
+	/* throw exception if failure, otherwise return */
+	TEST_GOT_EXN
+	jz	nbif_1_simple_exception
+	NBIF_RET(1)
+	SET_SIZE(ASYM($1))
+	TYPE_FUNCTION(ASYM($1))
+#endif')
+
+
 /*
  * noproc_primop_interface_0(nbif_name, cbif_name)
  * noproc_primop_interface_1(nbif_name, cbif_name)
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index f034c4700c..ada3ab2c83 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -247,7 +247,11 @@ nofail_primop_interface_3(nbif_bs_get_float_2, erts_bs_get_float_2)
 standard_bif_interface_3(nbif_bs_put_utf8, hipe_bs_put_utf8)
 standard_bif_interface_3(nbif_bs_put_utf16be, hipe_bs_put_utf16be)
 standard_bif_interface_3(nbif_bs_put_utf16le, hipe_bs_put_utf16le)
+ifdef(`nogc_bif_interface_1',`
+nogc_bif_interface_1(nbif_bs_validate_unicode, hipe_bs_validate_unicode)
+',`
 standard_bif_interface_1(nbif_bs_validate_unicode, hipe_bs_validate_unicode)
+')
 
 /*
  * Bit-syntax primops without any P parameter.
-- 
cgit v1.2.3


From 5369e34a892bfd8ab5aa98df330e3bbf19497b71 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 3 Nov 2017 16:11:00 +0100
Subject: Fix bug in hipe for <<X/utf32>>

by introducing new primop 'is_unicode'
with no exception (ab)use and no GC.

Replaces bs_validate_unicode which is kept for backward compat for now.
---
 erts/emulator/hipe/hipe_bif0.tab     | 1 +
 erts/emulator/hipe/hipe_bif_list.m4  | 1 +
 erts/emulator/hipe/hipe_native_bif.c | 6 ++++++
 erts/emulator/hipe/hipe_native_bif.h | 2 ++
 erts/emulator/hipe/hipe_primops.h    | 1 +
 5 files changed, 11 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 4038ca7ef8..4f73770d24 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -140,3 +140,4 @@ atom bs_validate_unicode_retract
 atom emulate_fpe
 atom emasculate_binary
 atom is_divisible
+atom is_unicode
\ No newline at end of file
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index ada3ab2c83..b86f2dafdc 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -223,6 +223,7 @@ standard_bif_interface_3(nbif_find_na_or_make_stub, hipe_find_na_or_make_stub)
 standard_bif_interface_2(nbif_nonclosure_address, hipe_nonclosure_address)
 nocons_nofail_primop_interface_0(nbif_fclearerror_error, hipe_fclearerror_error)
 standard_bif_interface_2(nbif_is_divisible, hipe_is_divisible)
+noproc_primop_interface_1(nbif_is_unicode, hipe_is_unicode)
 
 /*
  * Mbox primops with implicit P parameter.
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index d8044fe6da..e1c22701d0 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -495,6 +495,12 @@ BIF_RETTYPE nbif_impl_hipe_bs_validate_unicode(NBIF_ALIST_1)
     return NIL;
 }
 
+Uint hipe_is_unicode(Eterm arg)
+{
+    return (Uint) validate_unicode(arg);
+}
+
+
 int hipe_bs_validate_unicode_retract(ErlBinMatchBuffer* mb, Eterm arg)
 {
     if (!validate_unicode(arg)) {
diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h
index 38f874888b..1127d4ac56 100644
--- a/erts/emulator/hipe/hipe_native_bif.h
+++ b/erts/emulator/hipe/hipe_native_bif.h
@@ -67,6 +67,7 @@ AEXTERN(Eterm,nbif_bs_put_utf16be,(Process*,Eterm,byte*,unsigned int));
 AEXTERN(Eterm,nbif_bs_put_utf16le,(Process*,Eterm,byte*,unsigned int));
 AEXTERN(Eterm,nbif_bs_get_utf16,(void));
 AEXTERN(Eterm,nbif_bs_validate_unicode,(Process*,Eterm));
+AEXTERN(Uint,nbif_is_unicode,(Eterm));
 AEXTERN(Eterm,nbif_bs_validate_unicode_retract,(void));
 AEXTERN(void,nbif_is_divisible,(Process*,Uint,Uint));
 
@@ -92,6 +93,7 @@ Eterm hipe_bs_utf16_size(Eterm);
 BIF_RETTYPE nbif_impl_hipe_bs_put_utf16be(NBIF_ALIST_3);
 BIF_RETTYPE nbif_impl_hipe_bs_put_utf16le(NBIF_ALIST_3);
 BIF_RETTYPE nbif_impl_hipe_bs_validate_unicode(NBIF_ALIST_1);
+Uint hipe_is_unicode(Eterm);
 struct erl_bin_match_buffer;
 int hipe_bs_validate_unicode_retract(struct erl_bin_match_buffer*, Eterm);
 BIF_RETTYPE nbif_impl_hipe_is_divisible(NBIF_ALIST_2);
diff --git a/erts/emulator/hipe/hipe_primops.h b/erts/emulator/hipe/hipe_primops.h
index 4fcbc9df38..77f0dfe7e5 100644
--- a/erts/emulator/hipe/hipe_primops.h
+++ b/erts/emulator/hipe/hipe_primops.h
@@ -66,6 +66,7 @@ PRIMOP_LIST(am_bs_put_utf16be, &nbif_bs_put_utf16be)
 PRIMOP_LIST(am_bs_put_utf16le, &nbif_bs_put_utf16le)
 PRIMOP_LIST(am_bs_get_utf16, &nbif_bs_get_utf16)
 PRIMOP_LIST(am_bs_validate_unicode, &nbif_bs_validate_unicode)
+PRIMOP_LIST(am_is_unicode, &nbif_is_unicode)
 PRIMOP_LIST(am_bs_validate_unicode_retract, &nbif_bs_validate_unicode_retract)
 
 PRIMOP_LIST(am_is_divisible, &nbif_is_divisible)
-- 
cgit v1.2.3


From 64101035e025fc44dff5ba3d6ed4668c1e3a698f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 3 Nov 2017 19:52:50 +0100
Subject: erts: Fix lock checking in do_call_trace

which was lost in merge 3ec66701f91eba6a7a12a813b2283c2e733f62c1
---
 erts/emulator/beam/beam_bp.c | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index fe1e15701b..0832b3f374 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -998,7 +998,9 @@ do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg,
 
     fixup_cp_before_trace(c_p, &return_to_trace);
 
+    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
     flags = erts_call_trace(c_p, info, ms, reg, local, &tracer);
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
 
     /* restore cp after potential fixup */
     c_p->cp = cp_save;
-- 
cgit v1.2.3


From e58e6c2db1f8b01f0adca778456ff5e1f9475038 Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikpelinux@gmail.com>
Date: Sun, 5 Nov 2017 10:19:09 +0100
Subject: fix output formatting in hipe_bifs:show_heap/1

When the code switches from printf() to erts_printf() the output
becomes garbled.  Fixed by fflush()ing stdout first.

Fixed formatting of the "H E A P" banner for 64-bit systems.
---
 erts/emulator/hipe/hipe_debug.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_debug.c b/erts/emulator/hipe/hipe_debug.c
index 222a11db3d..cfe60b379e 100644
--- a/erts/emulator/hipe/hipe_debug.c
+++ b/erts/emulator/hipe/hipe_debug.c
@@ -135,7 +135,9 @@ static void print_heap(Eterm *pos, Eterm *end)
     printf("From: 0x%0*lx to 0x%0*lx\n\r",
 	   2*(int)sizeof(long), (unsigned long)pos,
 	   2*(int)sizeof(long), (unsigned long)end);
-    printf(" |         H E A P         |\r\n");
+    printf(" | %*s H E A P %*s |\r\n",
+	   2*(int)sizeof(long)-1, "",
+	   2*(int)sizeof(long)-1, "");
     printf(" | %*s | %*s |\r\n",
 	   2+2*(int)sizeof(long), "Address",
 	   2+2*(int)sizeof(long), "Contents");
@@ -158,8 +160,10 @@ static void print_heap(Eterm *pos, Eterm *end)
 		++pos;
 		--ari;
 	    }
-	} else
+	} else {
+	    fflush(stdout);
 	    erts_printf("%.30T", val);
+	}
 	printf("\r\n");
     }
     printf(" |%s|%s|\r\n", dashes, dashes);
-- 
cgit v1.2.3


From 354f6d1c29ecaabab8a5aa3f1c95a4188a0aa542 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Magnus=20L=C3=A5ng?= <margnus1@telia.com>
Date: Sun, 9 Apr 2017 13:27:51 +0200
Subject: HiPE: Make is_divisible a primop

Since gcunsafe values are live over is_divisible calls (although only
the happy path, which never GCd), it should be a primop so there cannot
be any GCs.
---
 erts/emulator/hipe/hipe_bif_list.m4  |  2 +-
 erts/emulator/hipe/hipe_native_bif.c | 10 +++-------
 erts/emulator/hipe/hipe_native_bif.h |  4 ++--
 3 files changed, 6 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index 08c1ab9318..23b6709cd0 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -222,7 +222,7 @@ standard_bif_interface_2(nbif_rethrow, hipe_rethrow)
 standard_bif_interface_3(nbif_find_na_or_make_stub, hipe_find_na_or_make_stub)
 standard_bif_interface_2(nbif_nonclosure_address, hipe_nonclosure_address)
 nocons_nofail_primop_interface_0(nbif_fclearerror_error, hipe_fclearerror_error)
-standard_bif_interface_2(nbif_is_divisible, hipe_is_divisible)
+noproc_primop_interface_2(nbif_is_divisible, hipe_is_divisible)
 noproc_primop_interface_1(nbif_is_unicode, hipe_is_unicode)
 
 /*
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index 4d20fbdb90..e444e3dc5d 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -506,16 +506,12 @@ int hipe_bs_validate_unicode_retract(ErlBinMatchBuffer* mb, Eterm arg)
     return 1;
 }
 
-/* Called via standard_bif_interface_2 */
-BIF_RETTYPE nbif_impl_hipe_is_divisible(NBIF_ALIST_2)
+Uint hipe_is_divisible(Uint dividend, Uint divisor)
 {
-    /* Arguments are Eterm-sized unsigned integers */
-    Uint dividend = BIF_ARG_1;
-    Uint divisor = BIF_ARG_2;
     if (dividend % divisor) {
-	BIF_ERROR(BIF_P, BADARG);
+        return 0;
     } else {
-	return NIL;
+        return 1;
     }
 }
 
diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h
index aa961ab6d0..71f63875a4 100644
--- a/erts/emulator/hipe/hipe_native_bif.h
+++ b/erts/emulator/hipe/hipe_native_bif.h
@@ -69,7 +69,7 @@ AEXTERN(Eterm,nbif_bs_get_utf16,(void));
 AEXTERN(Eterm,nbif_bs_validate_unicode,(Process*,Eterm));
 AEXTERN(Uint,nbif_is_unicode,(Eterm));
 AEXTERN(Eterm,nbif_bs_validate_unicode_retract,(void));
-AEXTERN(void,nbif_is_divisible,(Process*,Uint,Uint));
+AEXTERN(Uint,nbif_is_divisible,(Uint,Uint));
 
 AEXTERN(void,nbif_select_msg,(Process*));
 AEXTERN(Eterm,nbif_cmp_2,(void));
@@ -96,7 +96,7 @@ BIF_RETTYPE nbif_impl_hipe_bs_validate_unicode(NBIF_ALIST_1);
 Uint hipe_is_unicode(Eterm);
 struct erl_bin_match_buffer;
 int hipe_bs_validate_unicode_retract(struct erl_bin_match_buffer*, Eterm);
-BIF_RETTYPE nbif_impl_hipe_is_divisible(NBIF_ALIST_2);
+Uint hipe_is_divisible(Uint, Uint);
 
 #ifdef NO_FPE_SIGNALS
 AEXTERN(void,nbif_emulate_fpe,(Process*));
-- 
cgit v1.2.3


From 7503303b0cba1e02320bdf604e0d0997b7c2ff59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Magnus=20L=C3=A5ng?= <margnus1@telia.com>
Date: Sat, 8 Apr 2017 10:48:01 +0200
Subject: HiPE: Support literal tags

Literal tags are used by the VM as an alternative to reserving a large
virtual memory space in order to be able to quickly identify which terms
are literals. The use of literal tags harms performance, but is useful
to support systems where allocating a large amount of virtual memory is
not an option.
---
 erts/emulator/beam/erl_term.h        | 3 ---
 erts/emulator/hipe/hipe_mkliterals.c | 9 +++++++++
 2 files changed, 9 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 5ec6b6b44b..7d42d1b396 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -55,9 +55,6 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 #if defined(ARCH_64)
 #  define TAG_PTR_MASK__	0x7
 #  if !defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
-#    ifdef HIPE
-#      error Hipe on 64-bit needs a real mmap as it does not support the literal tag
-#    endif
 #    define TAG_LITERAL_PTR	0x4
 #  else
 #    undef TAG_LITERAL_PTR
diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c
index 6ea120c65c..de00994d64 100644
--- a/erts/emulator/hipe/hipe_mkliterals.c
+++ b/erts/emulator/hipe/hipe_mkliterals.c
@@ -460,6 +460,15 @@ static const struct rts_param rts_params[] = {
       1
 #else
       0
+#endif
+    },
+    /* This flag is always defined, but its value is configuration-dependent. */
+    { 17, "ERTS_USE_LITERAL_TAG",
+      1,
+#if defined(TAG_LITERAL_PTR)
+      1
+#else
+      0
 #endif
     },
     /* This parameter is always defined, but its value depends on ERTS_SMP. */
-- 
cgit v1.2.3


From fe35a4f6a26a1242fef0b7d728403d162812679c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 3 Nov 2017 10:13:56 +0100
Subject: Clean up definition of built-in macros

Make it easier to define new macros.
---
 erts/emulator/utils/beam_makeops | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 02227bbdfd..296fbc9d5f 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -306,9 +306,14 @@ if ($wordsize == 64) {
 # Add placeholders for built-in macros.
 #
 
-$c_code{'IS_PACKED'} = ['$Expr',"built-in macro",('Expr')];
-$c_code{'ARG_POSITION'} = ['$Expr',"built-in macro",('Expr')];
-foreach my $name (keys %c_code) {
+my %predef_macros =
+    (IS_PACKED => ['Expr'],
+     ARG_POSITION => ['Expr'],
+    );
+foreach my $name (keys %predef_macros) {
+    my @args = @{$predef_macros{$name}};
+    my $body = join(':', map { '$' . $_ } @args);
+    $c_code{$name} = [$body,"built-in macro",@args],
     $c_code_used{$name} = 1;
 }
 
-- 
cgit v1.2.3


From 83b87392042cb138c3761ad009e30784fb551b76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 3 Nov 2017 10:15:57 +0100
Subject: Rename the built-in macro ARG_SIZE() to OPERAND_SIZE()

Although the terminology "arguments" is used in the comments
for beam_makeops, the documentation in beam_makeops.md consistenly
use the term "operand".  Since the name of the macro is user-facing,
it should be consistent with the documentation.
---
 erts/emulator/utils/beam_makeops | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 296fbc9d5f..452e070482 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -308,8 +308,8 @@ if ($wordsize == 64) {
 
 my %predef_macros =
     (IS_PACKED => ['Expr'],
-     ARG_POSITION => ['Expr'],
-    );
+     OPERAND_POSITION => ['Expr'],
+);
 foreach my $name (keys %predef_macros) {
     my @args = @{$predef_macros{$name}};
     my $body = join(':', map { '$' . $_ } @args);
@@ -1651,7 +1651,7 @@ sub expand_macro {
     }
 
     # Handle built-in macros.
-    if ($name eq 'ARG_POSITION') {
+    if ($name eq 'OPERAND_POSITION') {
         if ($body =~ /^I\[(\d+)\]$/) {
             $body = $1;
         } else {
-- 
cgit v1.2.3


From 6808aa38b1845bf4bc92ed1cbf6005911e95385a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 3 Nov 2017 10:51:36 +0100
Subject: Add the built-in macro $IF() to handle conditionals

Add an $IF() macro to conditionally expand macros.  Use it like
this:

    $IF(Expression, IfTrue, IfFalse)

Expression is a Perl expression that can be evaulated at macro
expansion time.  If the expression evaluates to 0, the result will be
IfFalse, otherwise IfTrue.
---
 erts/emulator/utils/beam_makeops | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 452e070482..dc4da0cca0 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -309,7 +309,8 @@ if ($wordsize == 64) {
 my %predef_macros =
     (IS_PACKED => ['Expr'],
      OPERAND_POSITION => ['Expr'],
-);
+     IF => ['Expr','IfTrue','IfFalse'],
+    );
 foreach my $name (keys %predef_macros) {
     my @args = @{$predef_macros{$name}};
     my $body = join(':', map { '$' . $_ } @args);
@@ -1659,9 +1660,17 @@ sub expand_macro {
         }
     } elsif ($name eq 'IS_PACKED') {
         $body = ($body =~ /^I\[\d+\]$/) ? 0 : 1;
+    } elsif ($name eq 'IF') {
+        my $expr = $new_bindings{Expr};
+        my $bool = eval $expr;
+        if ($@ ne '') {
+            &error("bad expression '$expr' in \$IF()");
+        }
+        my $part = $bool ? 'IfTrue' : 'IfFalse';
+        $body = $new_bindings{$part};
     }
 
-    # Wrap body if needed and return resul.t
+    # Wrap body if needed and return result.
     $body = "do {\n$body\n} while (0)"
         if needs_do_wrapper($body);
     ($body,$rest);
-- 
cgit v1.2.3


From 0c88933a6ea89250460532847529699774a60b2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 6 Nov 2017 11:49:03 +0100
Subject: Consistently use the "erts_gc_" prefix for functions that do GC

In beam_emu, use "erts_gc_" for any function that does garbage
collection, as preparation for adding more sanity checks.
---
 erts/emulator/beam/beam_emu.c     | 21 ++++++++++++---------
 erts/emulator/beam/map_instrs.tab |  8 ++++----
 2 files changed, 16 insertions(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 60d0008d8f..bc95ccec52 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -400,12 +400,13 @@ static BeamInstr* apply_fun(Process* p, Eterm fun,
 			    Eterm args, Eterm* reg) NOINLINE;
 static Eterm new_fun(Process* p, Eterm* reg,
 		     ErlFunEntry* fe, int num_free) NOINLINE;
-static Eterm new_map(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* ptr) NOINLINE;
-static Eterm new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal,
+static Eterm erts_gc_new_map(Process* p, Eterm* reg, Uint live,
+                             Uint n, BeamInstr* ptr) NOINLINE;
+static Eterm erts_gc_new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal,
                                Uint live, BeamInstr* ptr) NOINLINE;
-static Eterm update_map_assoc(Process* p, Eterm* reg, Uint live,
+static Eterm erts_gc_update_map_assoc(Process* p, Eterm* reg, Uint live,
                               Uint n, BeamInstr* new_p) NOINLINE;
-static Eterm update_map_exact(Process* p, Eterm* reg, Uint live,
+static Eterm erts_gc_update_map_exact(Process* p, Eterm* reg, Uint live,
                               Uint n, Eterm* new_p) NOINLINE;
 static Eterm get_map_element(Eterm map, Eterm key);
 static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx);
@@ -2755,7 +2756,7 @@ do {						\
 
 
 static Eterm
-new_map(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* ptr)
+erts_gc_new_map(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* ptr)
 {
     Uint i;
     Uint need = n + 1 /* hdr */ + 1 /*size*/ + 1 /* ptr */ + 1 /* arity */;
@@ -2812,7 +2813,8 @@ new_map(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* ptr)
 }
 
 static Eterm
-new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal, Uint live, BeamInstr* ptr)
+erts_gc_new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal,
+                          Uint live, BeamInstr* ptr)
 {
     Eterm* keys = tuple_val(keys_literal);
     Uint n = arityval(*keys);
@@ -2846,7 +2848,8 @@ new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal, Uint live, BeamIns
 }
 
 static Eterm
-update_map_assoc(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* new_p)
+erts_gc_update_map_assoc(Process* p, Eterm* reg, Uint live,
+                         Uint n, BeamInstr* new_p)
 {
     Uint num_old;
     Uint num_updates;
@@ -2892,7 +2895,7 @@ update_map_assoc(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* new_p)
      */
 
     if (num_old == 0) {
-	return new_map(p, reg, live, n, new_p);
+	return erts_gc_new_map(p, reg, live, n, new_p);
     }
 
     /*
@@ -3048,7 +3051,7 @@ update_map_assoc(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* new_p)
  */
 
 static Eterm
-update_map_exact(Process* p, Eterm* reg, Uint live, Uint n, Eterm* new_p)
+erts_gc_update_map_exact(Process* p, Eterm* reg, Uint live, Uint n, Eterm* new_p)
 {
     Uint i;
     Uint num_old;
diff --git a/erts/emulator/beam/map_instrs.tab b/erts/emulator/beam/map_instrs.tab
index bbb2f49b66..c594a87298 100644
--- a/erts/emulator/beam/map_instrs.tab
+++ b/erts/emulator/beam/map_instrs.tab
@@ -31,7 +31,7 @@ new_map(Dst, Live, N) {
     Eterm res;
 
     HEAVY_SWAPOUT;
-    res = new_map(c_p, reg, $Live, $N, $NEXT_INSTRUCTION);
+    res = erts_gc_new_map(c_p, reg, $Live, $N, $NEXT_INSTRUCTION);
     HEAVY_SWAPIN;
     $REFRESH_GEN_DEST();
     $Dst = res;
@@ -44,7 +44,7 @@ i_new_small_map_lit(Dst, Live, Keys) {
     Eterm keys = $Keys;
 
     HEAVY_SWAPOUT;
-    res = new_small_map_lit(c_p, reg, keys, $Live, $NEXT_INSTRUCTION);
+    res = erts_gc_new_small_map_lit(c_p, reg, keys, $Live, $NEXT_INSTRUCTION);
     HEAVY_SWAPIN;
     $REFRESH_GEN_DEST();
     $Dst = res;
@@ -133,7 +133,7 @@ update_map_assoc(Src, Dst, Live, N) {
 
     reg[live] = $Src;
     HEAVY_SWAPOUT;
-    res = update_map_assoc(c_p, reg, live, $N, $NEXT_INSTRUCTION);
+    res = erts_gc_update_map_assoc(c_p, reg, live, $N, $NEXT_INSTRUCTION);
     HEAVY_SWAPIN;
     ASSERT(is_value(res));
     $REFRESH_GEN_DEST();
@@ -147,7 +147,7 @@ update_map_exact(Fail, Src, Dst, Live, N) {
 
     reg[live] = $Src;
     HEAVY_SWAPOUT;
-    res = update_map_exact(c_p, reg, live, $N, $NEXT_INSTRUCTION);
+    res = erts_gc_update_map_exact(c_p, reg, live, $N, $NEXT_INSTRUCTION);
     HEAVY_SWAPIN;
     if (is_value(res)) {
         $REFRESH_GEN_DEST();
-- 
cgit v1.2.3


From cfb75380fcba60058825815068eac8e402d10e40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 3 Nov 2017 11:02:53 +0100
Subject: Warn when $REFRESH_GEN_DEST() is not used after a GC

It is easy to to forget to use $REFRESH_GEN_DEST() in an instruction
that has a general destionation ('d').  Add a heuristic that should
catch most if not all such problems.
---
 erts/emulator/beam/macros.tab    |  9 +++----
 erts/emulator/utils/beam_makeops | 54 ++++++++++++++++++++++++++++++++++------
 2 files changed, 51 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab
index e0b5f56b53..494fe8961e 100644
--- a/erts/emulator/beam/macros.tab
+++ b/erts/emulator/beam/macros.tab
@@ -20,13 +20,12 @@
 //
 
 //
-// Use if there is a garbage collection before storing to a
-// general destination (either X or Y register).
+// Define a regular expression that will match instructions that
+// perform GC.  That will allow beam_makeops to check for instructions
+// that don't use $REFRESH_GEN_DEST() when they should.
 //
 
-REFRESH_GEN_DEST() {
-    dst_ptr = REG_TARGET_PTR(dst);
-}
+GC_REGEXP=erts_garbage_collect|erts_gc|GcBifFunction;
 
 // $Offset is relative to the start of the instruction (not to the
 // location of the failure label reference). Since combined
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index dc4da0cca0..3e1116ef52 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -19,7 +19,7 @@
 # %CopyrightEnd%
 #
 use strict;
-use vars qw($BEAM_FORMAT_NUMBER);
+use vars qw($BEAM_FORMAT_NUMBER $GC_REGEXP);
 use constant COLD => 0;
 use constant WARM => 1;
 use constant HOT => 2;
@@ -36,6 +36,7 @@ use constant PACK_CMD_LOOSE => '3';
 use constant PACK_CMD_WIDE => '4';
 
 $BEAM_FORMAT_NUMBER = undef;
+$GC_REGEXP = undef;
 
 my $target = \&emulator_output;
 my $outdir = ".";		# Directory for output files.
@@ -310,6 +311,7 @@ my %predef_macros =
     (IS_PACKED => ['Expr'],
      OPERAND_POSITION => ['Expr'],
      IF => ['Expr','IfTrue','IfFalse'],
+     REFRESH_GEN_DEST => [],
     );
 foreach my $name (keys %predef_macros) {
     my @args = @{$predef_macros{$name}};
@@ -392,8 +394,10 @@ while (<>) {
     #
     if (/^([\w_][\w\d_]+)=(.*)/) {
 	no strict 'refs';
-	my($name) = $1;
-	$$name = $2;
+	my $name = $1;
+        my $value = $2;
+        $value =~ s/;\s*$//;
+	$$name = $value;
 	next;
     }
 
@@ -1366,7 +1370,9 @@ sub cg_basic {
 #
 
 sub cg_combined_size {
-    my %params = (@_, pack_options => \@basic_pack_options);
+    my %params = (@_,
+                  pack_options => \@basic_pack_options,
+                  size_only => 1);
     $params{pack_options} = \@extended_pack_options
         if $params{first};
     my($size) = code_gen(%params);
@@ -1394,6 +1400,7 @@ sub code_gen {
     my %params = (extra_comments => '',
                   offset => 0,
                   inc => 0,
+                  size_only => 0,
                   @_);
     my $name = $params{name};
     my $extra_comments = $params{extra_comments};
@@ -1426,6 +1433,7 @@ sub code_gen {
 
     my $need_block = 0;
     my $arg_offset = $offset;
+    my $has_gen_dest = 0;
     @args = map { s/[?]$//g; $_ } @args;
     foreach (@args) {
 	my($this_size) = $arg_size{$_};
@@ -1436,6 +1444,7 @@ sub code_gen {
                     "Eterm* dst_ptr = REG_TARGET_PTR(dst);\n";
                 push(@f, "*dst_ptr");
                 $this_size = $1;
+                $has_gen_dest = 1;
                 last SWITCH;
             };
 	    /^packed:[a-zA-z]:(\d):(.*)/ and do {
@@ -1468,6 +1477,7 @@ sub code_gen {
                 $var_decls .= "Eterm dst = " . arg_offset($arg_offset) . ";\n" .
                     "Eterm* dst_ptr = REG_TARGET_PTR(dst);\n";
                 push(@f, "*dst_ptr");
+                $has_gen_dest = 1;
                 last SWITCH;
             };
             defined $arg_size{$_} and do {
@@ -1482,10 +1492,10 @@ sub code_gen {
     }
 
     #
-    # If the implementation is in beam_emu.c, there is nothing
-    # more to do.
+    # If the implementation is in beam_emu.c or if
+    # the caller only wants the size, we are done.
     #
-    unless (defined $c_code_ref) {
+    if (not defined $c_code_ref or $params{size_only}) {
         return ($size+1, undef, '');
     }
 
@@ -1550,9 +1560,36 @@ sub code_gen {
                     "{",
                     "$var_decls$body",
                     "}", "");
+
+    # Make sure that $REFRESH_GEN_DEST() is used when a
+    # general destination ('d') may have been clobbered by
+    # a GC.
+    my $gc_error = verify_gc_code($code, $has_gen_dest);
+    if (defined $gc_error) {
+        warn $gc_error;
+        error("... from the body of $name at $where");
+    }
+
+    # Done.
     ($size+1, $code, $pack_spec);
 }
 
+sub verify_gc_code {
+    my $code = shift;
+    my $has_gen_dest = shift;
+
+    return unless $has_gen_dest;
+
+    if ($code =~ /$GC_REGEXP/o) {
+        my $code_after_gc = substr($code, $+[0]);
+        unless ($code_after_gc =~ /dst_ptr = REG_TARGET_PTR/) {
+            return "pointer to destination register is invalid after GC -- " .
+                "use \$REFRESH_GEN_DEST()\n";
+        }
+    }
+    return undef;
+}
+
 sub arg_offset {
     my $offset = shift;
     "I[" . ($offset+1) . "]";
@@ -1668,8 +1705,11 @@ sub expand_macro {
         }
         my $part = $bool ? 'IfTrue' : 'IfFalse';
         $body = $new_bindings{$part};
+    } elsif ($name eq 'REFRESH_GEN_DEST') {
+        $body = "dst_ptr = REG_TARGET_PTR(dst)";
     }
 
+
     # Wrap body if needed and return result.
     $body = "do {\n$body\n} while (0)"
         if needs_do_wrapper($body);
-- 
cgit v1.2.3


From 6d810d71f4f2e2ee0be749a53e328c3719db00f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 6 Nov 2017 12:55:07 +0100
Subject: Remove redundant built-in macro $IS_PACKED()

It can easily be implemented as $OPERAND_POSITION($Operand) == 0.
---
 erts/emulator/utils/beam_makeops | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 3e1116ef52..d5cb127246 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -308,8 +308,7 @@ if ($wordsize == 64) {
 #
 
 my %predef_macros =
-    (IS_PACKED => ['Expr'],
-     OPERAND_POSITION => ['Expr'],
+    (OPERAND_POSITION => ['Expr'],
      IF => ['Expr','IfTrue','IfFalse'],
      REFRESH_GEN_DEST => [],
     );
@@ -1695,8 +1694,6 @@ sub expand_macro {
         } else {
             $body = 0;
         }
-    } elsif ($name eq 'IS_PACKED') {
-        $body = ($body =~ /^I\[\d+\]$/) ? 0 : 1;
     } elsif ($name eq 'IF') {
         my $expr = $new_bindings{Expr};
         my $bool = eval $expr;
-- 
cgit v1.2.3


From d563fdd7f3bfa5df218e3965ca45a0348514a2a7 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 6 Nov 2017 17:59:28 +0100
Subject: erts: Fix bug in systask scheduling

when request id is an immediate.

Ex:
erlang:garbage_collect(P, [{async,Immediate}]).
may crash the VM.
---
 erts/emulator/beam/erl_process.c     |  2 +-
 erts/emulator/test/process_SUITE.erl | 13 ++++++++++---
 2 files changed, 11 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b4b97d7df1..fd02f10540 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9824,7 +9824,7 @@ erts_internal_request_system_task_3(BIF_ALIST_3)
 	    goto badarg;
 	req_type = tp[1];
 	req_id = tp[2];
-	req_id_sz = is_immed(req_id) ? req_id : size_object(req_id);
+	req_id_sz = is_immed(req_id) ? 0 : size_object(req_id);
 	tot_sz = req_id_sz;
 	for (i = 0; i < ERTS_MAX_PROC_SYS_TASK_ARGS; i++) {
 	    int tix = 3 + i;
diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index bf31655066..932d0f87f1 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -2365,8 +2365,13 @@ system_task_on_suspended(Config) when is_list(Config) ->
     end.
 
 gc_request_when_gc_disabled(Config) when is_list(Config) ->
-    Master = self(),
     AIS = erts_debug:set_internal_state(available_internal_state, true),
+    gc_request_when_gc_disabled_do(ref),
+    gc_request_when_gc_disabled_do(immed),
+    erts_debug:set_internal_state(available_internal_state, AIS).
+
+gc_request_when_gc_disabled_do(ReqIdType) ->
+    Master = self(),
     {P, M} = spawn_opt(fun () ->
 			       true = erts_debug:set_internal_state(gc_state,
 								    false),
@@ -2378,7 +2383,10 @@ gc_request_when_gc_disabled(Config) when is_list(Config) ->
 			       receive after 100 -> ok end
 		       end, [monitor, link]),
     receive {P, gc_state, false} -> ok end,
-    ReqId = make_ref(),
+    ReqId = case ReqIdType of
+                ref -> make_ref();
+                immed -> immed
+            end,
     async = garbage_collect(P, [{async, ReqId}]),
     receive
 	{garbage_collect, ReqId, Result} ->
@@ -2387,7 +2395,6 @@ gc_request_when_gc_disabled(Config) when is_list(Config) ->
 	    ok
     end,
     receive {garbage_collect, ReqId, true} -> ok end,
-    erts_debug:set_internal_state(available_internal_state, AIS),
     receive {'DOWN', M, process, P, _Reason} -> ok end,
     ok.
 
-- 
cgit v1.2.3


From 54fd69c2887d1a76cae3bf43e31e611dbcf152fa Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 6 Nov 2017 18:45:13 +0100
Subject: erts: Remove obsolete hipe primop bs_validate_unicode

which was replaced by 'is_unicode' in 5369e34a892bfd8ab5aa98df330e3bbf19497b71
but kept for ABI compatibility in OTP-20.*.
---
 erts/emulator/hipe/hipe_amd64_bifs.m4 | 36 -----------------------------------
 erts/emulator/hipe/hipe_bif0.tab      |  1 -
 erts/emulator/hipe/hipe_bif_list.m4   |  5 -----
 erts/emulator/hipe/hipe_native_bif.c  |  9 ---------
 erts/emulator/hipe/hipe_native_bif.h  |  2 --
 erts/emulator/hipe/hipe_primops.h     |  1 -
 6 files changed, 54 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_amd64_bifs.m4 b/erts/emulator/hipe/hipe_amd64_bifs.m4
index 6d998c4b55..cf4c59c9af 100644
--- a/erts/emulator/hipe/hipe_amd64_bifs.m4
+++ b/erts/emulator/hipe/hipe_amd64_bifs.m4
@@ -462,42 +462,6 @@ ASYM($1):
 	TYPE_FUNCTION(ASYM($1))
 #endif')
 
-/*
- * nogc_bif_interface_1(nbif_name, cbif_name)
- *
- * Generate native interface for a bif with implicit P
- * The bif can fail but cannot do GC.
- */
-
-define(nogc_bif_interface_1,
-`
-#ifndef HAVE_$1
-#`define' HAVE_$1
-	TEXT
-	.align	4
-	GLOBAL(ASYM($1))
-ASYM($1):
-	/* set up the parameters */
-	movq	P, %rdi
-	NBIF_ARG(%rsi,1,0)
-
-	/* make the call on the C stack */
-	SWITCH_ERLANG_TO_C
-	pushq	%rsi
-	movq	%rsp, %rsi	/* Eterm* BIF__ARGS */
-	sub	$(8), %rsp	/* stack frame 16-byte alignment */
-	CALL_BIF($2)
-	add	$(1*8 + 8), %rsp
-	SWITCH_C_TO_ERLANG
-
-	/* throw exception if failure, otherwise return */
-	TEST_GOT_EXN
-	jz	nbif_1_simple_exception
-	NBIF_RET(1)
-	SET_SIZE(ASYM($1))
-	TYPE_FUNCTION(ASYM($1))
-#endif')
-
 
 /*
  * noproc_primop_interface_0(nbif_name, cbif_name)
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 4f73770d24..0380e8c795 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -135,7 +135,6 @@ atom bs_utf16_size
 atom bs_put_utf16be
 atom bs_put_utf16le
 atom bs_get_utf16
-atom bs_validate_unicode
 atom bs_validate_unicode_retract
 atom emulate_fpe
 atom emasculate_binary
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index 08c1ab9318..7e3851f20e 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -248,11 +248,6 @@ nofail_primop_interface_3(nbif_bs_get_float_2, erts_bs_get_float_2)
 standard_bif_interface_3(nbif_bs_put_utf8, hipe_bs_put_utf8)
 standard_bif_interface_3(nbif_bs_put_utf16be, hipe_bs_put_utf16be)
 standard_bif_interface_3(nbif_bs_put_utf16le, hipe_bs_put_utf16le)
-ifdef(`nogc_bif_interface_1',`
-nogc_bif_interface_1(nbif_bs_validate_unicode, hipe_bs_validate_unicode)
-',`
-standard_bif_interface_1(nbif_bs_validate_unicode, hipe_bs_validate_unicode)
-')
 
 /*
  * Bit-syntax primops without any P parameter.
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index 4d20fbdb90..f5471285c2 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -482,15 +482,6 @@ static int validate_unicode(Eterm arg)
     return 1;
 }
 
-BIF_RETTYPE nbif_impl_hipe_bs_validate_unicode(NBIF_ALIST_1)
-{
-    Process *p = BIF_P;
-    Eterm arg = BIF_ARG_1;
-    if (!validate_unicode(arg))
-	BIF_ERROR(p, BADARG);
-    return NIL;
-}
-
 Uint hipe_is_unicode(Eterm arg)
 {
     return (Uint) validate_unicode(arg);
diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h
index aa961ab6d0..a4919f4886 100644
--- a/erts/emulator/hipe/hipe_native_bif.h
+++ b/erts/emulator/hipe/hipe_native_bif.h
@@ -66,7 +66,6 @@ AEXTERN(Eterm,nbif_bs_utf16_size,(Eterm));
 AEXTERN(Eterm,nbif_bs_put_utf16be,(Process*,Eterm,byte*,unsigned int));
 AEXTERN(Eterm,nbif_bs_put_utf16le,(Process*,Eterm,byte*,unsigned int));
 AEXTERN(Eterm,nbif_bs_get_utf16,(void));
-AEXTERN(Eterm,nbif_bs_validate_unicode,(Process*,Eterm));
 AEXTERN(Uint,nbif_is_unicode,(Eterm));
 AEXTERN(Eterm,nbif_bs_validate_unicode_retract,(void));
 AEXTERN(void,nbif_is_divisible,(Process*,Uint,Uint));
@@ -92,7 +91,6 @@ BIF_RETTYPE nbif_impl_hipe_bs_put_utf8(NBIF_ALIST_3);
 Eterm hipe_bs_utf16_size(Eterm);
 BIF_RETTYPE nbif_impl_hipe_bs_put_utf16be(NBIF_ALIST_3);
 BIF_RETTYPE nbif_impl_hipe_bs_put_utf16le(NBIF_ALIST_3);
-BIF_RETTYPE nbif_impl_hipe_bs_validate_unicode(NBIF_ALIST_1);
 Uint hipe_is_unicode(Eterm);
 struct erl_bin_match_buffer;
 int hipe_bs_validate_unicode_retract(struct erl_bin_match_buffer*, Eterm);
diff --git a/erts/emulator/hipe/hipe_primops.h b/erts/emulator/hipe/hipe_primops.h
index 49e6bdb026..a6abd3e011 100644
--- a/erts/emulator/hipe/hipe_primops.h
+++ b/erts/emulator/hipe/hipe_primops.h
@@ -63,7 +63,6 @@ PRIMOP_LIST(am_bs_utf16_size, &nbif_bs_utf16_size)
 PRIMOP_LIST(am_bs_put_utf16be, &nbif_bs_put_utf16be)
 PRIMOP_LIST(am_bs_put_utf16le, &nbif_bs_put_utf16le)
 PRIMOP_LIST(am_bs_get_utf16, &nbif_bs_get_utf16)
-PRIMOP_LIST(am_bs_validate_unicode, &nbif_bs_validate_unicode)
 PRIMOP_LIST(am_is_unicode, &nbif_is_unicode)
 PRIMOP_LIST(am_bs_validate_unicode_retract, &nbif_bs_validate_unicode_retract)
 
-- 
cgit v1.2.3


From 1f8177e950715b3444dadfb1624875afe228195b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 7 Nov 2017 05:12:54 +0100
Subject: Strengthen tests of definition of specific instructions

Don't allow defining an specific operation more than once with
the exact same operands.  Don't allow a specific operation to be
defined with different arities.
---
 erts/emulator/beam/ops.tab       |  2 --
 erts/emulator/utils/beam_makeops | 20 ++++++++++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 7a2c39b3a8..a560bde920 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -511,8 +511,6 @@ put_list y x x
 put_list y y x
 put_list x y x
 
-put_list y x x
-
 # put_list SrcReg Constant Dst
 
 put_list x c x
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index d5cb127246..da994fae3e 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -78,6 +78,10 @@ my %num_specific;
 my %gen_to_spec;
 my %specific_op;
 
+# The following hashes are used for error checking.
+my %print_name;
+my %specific_op_arity;
+
 # Information about each specific operator. Key is the print name (e.g. get_list_xxy).
 # Value is a hash.
 my %spec_op_info;
@@ -1055,6 +1059,22 @@ sub parse_specific_op {
     my $key = "$name/$arity";
     foreach my $args_ref (@res) {
         @args = @$args_ref;
+        my $arity = @args;
+        my $loc = "$ARGV($.)";
+        if (defined $specific_op_arity{$name}) {
+            my($prev_arity,$loc) = @{$specific_op_arity{$name}};
+            if ($arity != $prev_arity) {
+                error("$name defined with arity $arity, " .
+                      "but previously defined with arity $prev_arity at $loc");
+            }
+        }
+        $specific_op_arity{$name} = [$arity,$loc];
+        my $print_name = print_name($name, @args);
+        if (defined $print_name{$print_name}) {
+            error("$name @args: already defined at " .
+                  $print_name{$print_name});
+        }
+        $print_name{$print_name} = $loc;
         push @{$specific_op{$key}}, [$name,$hotness,@args];
     }
 
-- 
cgit v1.2.3


From b1ce8325624d22fa34b76407932947a543754962 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 7 Nov 2017 07:16:31 +0100
Subject: Emasculate writable binaries on entering an iovec

The lack of this caused serious data corruption when a binary was
altered after entering the queue. This went unnoticed because it
was never used without erlang:iolist_to_iovec, which always
emasculates binaries.
---
 erts/emulator/beam/erl_nif.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index ac4ecd77e5..8342070521 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3488,6 +3488,10 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) {
     if (thing_subtag(*parent_header) == REFC_BINARY_SUBTAG) {
         ProcBin *pb = (ProcBin*)parent_header;
 
+        if (pb->flags & (PB_IS_WRITABLE | PB_ACTIVE_WRITER)) {
+            erts_emasculate_writable_binary(pb);
+        }
+
         ASSERT(pb->val != NULL);
         ASSERT(byte_offset < pb->size);
         ASSERT(&pb->bytes[byte_offset] >= (byte*)(pb->val)->orig_bytes);
-- 
cgit v1.2.3


From c221ada91c0027bbd74e3e5bff2865df7765a2ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 7 Nov 2017 11:30:27 +0100
Subject: Ignore empty binaries in enif_inspect_iovec

---
 erts/emulator/beam/erl_nif.c | 49 ++++++++++++++++++++++----------------------
 1 file changed, 25 insertions(+), 24 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 8342070521..3ade17b10d 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3426,36 +3426,38 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul
         size = binary_size(binary);
         binary_header = binary_val(binary);
 
-        /* If we're a sub-binary we'll need to check our underlying binary to
-         * determine whether we're on-heap or not. */
-        if(thing_subtag(*binary_header) == SUB_BINARY_SUBTAG) {
-            ErlSubBin *sb = (ErlSubBin*)binary_header;
-
-            /* Reject bitstrings */
-            if((sb->bitoffs + sb->bitsize) > 0) {
-                return 0;
+        if (size > 0) {
+            /* If we're a sub-binary we'll need to check our underlying binary
+             * to determine whether we're on-heap or not. */
+            if (thing_subtag(*binary_header) == SUB_BINARY_SUBTAG) {
+                ErlSubBin *sb = (ErlSubBin*)binary_header;
+
+                /* Reject bitstrings */
+                if((sb->bitoffs + sb->bitsize) > 0) {
+                    return 0;
+                }
+
+                ASSERT(size <= binary_size(sb->orig));
+                binary_header = binary_val(sb->orig);
             }
 
-            ASSERT(size <= binary_size(sb->orig));
-            binary_header = binary_val(sb->orig);
-        }
-
-        if(thing_subtag(*binary_header) == HEAP_BINARY_SUBTAG) {
-            ASSERT(size <= ERL_ONHEAP_BIN_LIMIT);
+            if (thing_subtag(*binary_header) == HEAP_BINARY_SUBTAG) {
+                ASSERT(size <= ERL_ONHEAP_BIN_LIMIT);
 
-            result->iovec_len += 1;
-            result->onheap_size += size;
-        } else {
-            ASSERT(thing_subtag(*binary_header) == REFC_BINARY_SUBTAG);
+                result->iovec_len += 1;
+                result->onheap_size += size;
+            } else {
+                ASSERT(thing_subtag(*binary_header) == REFC_BINARY_SUBTAG);
 
-            result->iovec_len += 1 + size / MAX_SYSIOVEC_IOVLEN;
-            result->offheap_size += size;
+                result->iovec_len += 1 + size / MAX_SYSIOVEC_IOVLEN;
+                result->offheap_size += size;
+            }
         }
 
         result->sublist_length += 1;
         lookahead = CDR(cell);
 
-        if(result->sublist_length >= max_length) {
+        if (result->sublist_length >= max_length) {
             break;
         }
     }
@@ -3535,7 +3537,7 @@ static int fill_iovec_with_slice(ErlNifEnv *env,
 
         /* If this isn't a refc binary, copy its contents to the onheap buffer
          * and reference that instead. */
-        if (raw_data.ref_bin == NULL) {
+        if (raw_data.size > 0 && raw_data.ref_bin == NULL) {
             ASSERT(onheap_offset < onheap_data.size);
             ASSERT(slice->onheap_size > 0);
 
@@ -3546,12 +3548,11 @@ static int fill_iovec_with_slice(ErlNifEnv *env,
             raw_data.ref_bin = onheap_data.ref_bin;
         }
 
-        ASSERT(raw_data.ref_bin != NULL);
-
         while (raw_data.size > 0) {
             UWord chunk_len = MIN(raw_data.size, MAX_SYSIOVEC_IOVLEN);
 
             ASSERT(iovec_idx < iovec->iovcnt);
+            ASSERT(raw_data.ref_bin != NULL);
 
             iovec->iov[iovec_idx].iov_base = raw_data.data;
             iovec->iov[iovec_idx].iov_len = chunk_len;
-- 
cgit v1.2.3


From 8cfe9496873fa36eccda939e56bf7a922e945ca9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 7 Nov 2017 17:24:55 +0100
Subject: Fix deflateParams on zlib 1.2.11

1.2.11 started bailing when avail_out==0 regardless of whether
there's anything to flush or not, and there's no point in adapting
the old method since it was vulnerable to bugs in other zlib
versions which updated the deflate parameters even on failure.

The api_deflateParams test has been expanded accordingly, and two
white-box cases in zip_usage has been updated to make fewer
assumptions about the output; the validity of the compressed data
is what matters, not whether it's exactly the same as the test
vector.
---
 erts/emulator/nifs/common/zlib_nif.c | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/nifs/common/zlib_nif.c b/erts/emulator/nifs/common/zlib_nif.c
index fa29b4fb71..9565a5a059 100644
--- a/erts/emulator/nifs/common/zlib_nif.c
+++ b/erts/emulator/nifs/common/zlib_nif.c
@@ -717,7 +717,9 @@ static ERL_NIF_TERM zlib_deflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM
 
 static ERL_NIF_TERM zlib_deflateParams(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
     zlib_data_t *d;
+
     int res, level, strategy;
+    Bytef dummy_buffer;
 
     if(argc != 3 || !get_zlib_data(env, argv[0], &d)
                  || !enif_get_int(env, argv[1], &level)
@@ -729,12 +731,27 @@ static ERL_NIF_TERM zlib_deflateParams(ErlNifEnv *env, int argc, const ERL_NIF_T
         return enif_raise_exception(env, am_not_initialized);
     }
 
-    /* deflateParams will flush everything currently in the stream, corrupting
-     * the heap unless it's empty. We therefore pretend to have a full output
-     * buffer, forcing a Z_BUF_ERROR if there's anything left to be flushed. */
-    d->s.avail_out = 0;
+    /* This is a bit of a hack; deflateParams flushes with Z_BLOCK which won't
+     * stop at a byte boundary, so we can't split this operation up, and we
+     * can't allocate a buffer large enough to fit it in one go since we have
+     * to support zlib versions that lack deflatePending.
+     *
+     * We therefore flush everything prior to this call to ensure that we are
+     * stopped on a byte boundary and have no pending data. We then hand it a
+     * dummy buffer to detect when this assumption doesn't hold (Hopefully
+     * never), and to smooth over an issue with zlib 1.2.11 which always
+     * returns Z_BUF_ERROR when d->s.avail_out is 0, regardless of whether
+     * there's any pending data or not. */
+
+    d->s.next_out = &dummy_buffer;
+    d->s.avail_out = 1;
+
     res = deflateParams(&d->s, level, strategy);
 
+    if(d->s.avail_out == 0) {
+        return zlib_return(env, Z_STREAM_ERROR);
+    }
+
     return zlib_return(env, res);
 }
 
-- 
cgit v1.2.3


From c133f397f30662d567216e38ef29b73b19f6bd3c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 8 Nov 2017 20:23:16 +0100
Subject: erts: Fix distribution_SUITE:bad_dist_ext_size

for "+hmqd off_heap"
---
 erts/emulator/test/distribution_SUITE.erl | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index 4a0b299e03..917717f2f5 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -1683,13 +1683,16 @@ bad_dist_ext_size(Config) when is_list(Config) ->
     start_node_monitors([Offender,Victim]),
 
     Parent = self(),
-    P = spawn_link(Victim,
+    P = spawn_opt(Victim,
                    fun () ->
                            Parent ! {self(), started},
                            receive check_msgs -> ok end,  %% DID CRASH HERE
                            bad_dist_ext_check_msgs([one]),
                            Parent ! {self(), messages_checked}
-                   end),
+                   end,
+                 [link,
+                  %% on_heap to force total_heap_size to inspect msg queue
+                  {message_queue_data, on_heap}]),
 
     receive {P, started} -> ok end,
     P ! one,
@@ -1712,6 +1715,7 @@ bad_dist_ext_size(Config) when is_list(Config) ->
 
     verify_still_up(Offender, Victim),
 
+    %% Let process_info(P, total_heap_size) find bad msg and disconnect
     rpc:call(Victim, erlang, process_info, [P, total_heap_size]),
 
     verify_down(Offender, connection_closed, Victim, killed),
-- 
cgit v1.2.3


From 869ca895b339a6cab9d0fb24e78842ee1d89c6b9 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 9 Nov 2017 17:59:45 +0100
Subject: erts: Fix race in distribution_SUITE:bad_dist_struct

Symptom: random rpc net_adm:ping returned pang

Use monitor_node to wait for failed connection
before we try to connect again.
---
 erts/emulator/test/distribution_SUITE.erl | 56 ++++++++++++-------------------
 1 file changed, 22 insertions(+), 34 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index 917717f2f5..e731b68f2f 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -1363,81 +1363,59 @@ bad_dist_structure(Config) when is_list(Config) ->
     start_monitor(Offender,P),
     P ! one,
     send_bad_structure(Offender, P,{?DOP_MONITOR_P_EXIT,'replace',P,normal},2),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
+
     start_monitor(Offender,P),
     send_bad_structure(Offender, P,{?DOP_MONITOR_P_EXIT,'replace',P,normal,normal},2),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
+
     start_link(Offender,P),
     send_bad_structure(Offender, P,{?DOP_LINK},0),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
+
     start_link(Offender,P),
     send_bad_structure(Offender, P,{?DOP_UNLINK,'replace'},2),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
+
     start_link(Offender,P),
     send_bad_structure(Offender, P,{?DOP_UNLINK,'replace',make_ref()},2),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
+
     start_link(Offender,P),
     send_bad_structure(Offender, P,{?DOP_UNLINK,make_ref(),P},0),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
+
     start_link(Offender,P),
     send_bad_structure(Offender, P,{?DOP_UNLINK,normal,normal},0),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
+
     start_monitor(Offender,P),
     send_bad_structure(Offender, P,{?DOP_MONITOR_P,'replace',P},2),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
+
     start_monitor(Offender,P),
     send_bad_structure(Offender, P,{?DOP_MONITOR_P,'replace',P,normal},2),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
+
     start_monitor(Offender,P),
     send_bad_structure(Offender, P,{?DOP_DEMONITOR_P,'replace',P},2),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
+
     start_monitor(Offender,P),
     send_bad_structure(Offender, P,{?DOP_DEMONITOR_P,'replace',P,normal},2),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
+
     send_bad_structure(Offender, P,{?DOP_EXIT,'replace',P},2),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_EXIT,make_ref(),normal,normal},0),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_EXIT_TT,'replace',token,P},2),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_EXIT_TT,make_ref(),token,normal,normal},0),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_EXIT2,'replace',P},2),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_EXIT2,make_ref(),normal,normal},0),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_EXIT2_TT,'replace',token,P},2),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_EXIT2_TT,make_ref(),token,normal,normal},0),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace'},2),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace','atomic'},2),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace',P},0),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_REG_SEND_TT,'replace','',name},2,{message}),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_REG_SEND_TT,'replace','',name,token},0,{message}),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace',''},2,{message}),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',P},0,{message}),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',name},0,{message}),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',name,{token}},2,{message}),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_SEND_TT,'',P},0,{message}),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_SEND_TT,'',name,token},0,{message}),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_SEND,''},0,{message}),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_SEND,'',name},0,{message}),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     send_bad_structure(Offender, P,{?DOP_SEND,'',P,{token}},0,{message}),
-    pong = rpc:call(Victim, net_adm, ping, [Offender]),
     P ! two,
     P ! check_msgs,
     receive
@@ -1796,10 +1774,11 @@ send_bad_structure(Offender,Victim,Bad,WhereToPutSelf) ->
 send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) ->
     Parent = self(),
     Done = make_ref(),
-    spawn(Offender,
+    spawn_link(Offender,
           fun () ->
                   Node = node(Victim),
                   pong = net_adm:ping(Node),
+                  erlang:monitor_node(Node, true),
                   DPrt = dport(Node),
                   Bad1 = case WhereToPutSelf of
                              0 ->
@@ -1813,7 +1792,16 @@ send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) ->
                       [] -> [];
                       _Other -> [dmsg_ext(PayLoad)]
                   end,
+
+                  receive {nodedown, Node} -> exit("premature nodedown")
+                  after 10 -> ok
+                  end,
+
                   port_command(DPrt, DData),
+
+                  receive {nodedown, Node} -> ok
+                  after 5000 -> exit("missing nodedown")
+                  end,
                   Parent ! {DData,Done}
           end),
     receive
-- 
cgit v1.2.3


From d71c5604e7ba5e30391c493fb46e67a0284cef76 Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikpelinux@gmail.com>
Date: Sat, 11 Nov 2017 14:29:16 +0100
Subject: fix output formatting in several HiPE debug BIFs

Fix formatting in hipe_bifs:show_pcb/1, hipe_bifs:show_estack/1,
and hipe_bifs:show_nstack/1.

fflush(stdout) before switching from printf() to erts_printf() to
avoid garbled output.

Adjust field lengths to work on both 64- and 32-bit systems.

Tested on Linux/x86_64 (64-bit) and Linux/ARMv7 (32-bit).
---
 erts/emulator/hipe/hipe_debug.c      | 24 +++++++++++++++++-------
 erts/emulator/hipe/hipe_risc_stack.c |  8 ++++++--
 erts/emulator/hipe/hipe_x86_stack.c  |  8 ++++++--
 3 files changed, 29 insertions(+), 11 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_debug.c b/erts/emulator/hipe/hipe_debug.c
index cfe60b379e..929b2a9432 100644
--- a/erts/emulator/hipe/hipe_debug.c
+++ b/erts/emulator/hipe/hipe_debug.c
@@ -63,12 +63,13 @@ static void print_beam_pc(BeamInstr *pc)
 	printf("normal-process-exit");
     } else {
 	ErtsCodeMFA *cmfa = find_function_from_pc(pc);
-	if (cmfa)
+	if (cmfa) {
+	    fflush(stdout);
 	    erts_printf("%T:%T/%bpu + 0x%bpx",
 			cmfa->module, cmfa->function,
                         cmfa->arity,
                         pc - erts_codemfa_to_code(cmfa));
-	else
+	} else
 	    printf("?");
     }
 }
@@ -116,6 +117,7 @@ static void print_stack(Eterm *sp, Eterm *end)
 	    printf(" | 0x%0*lx | 0x%0*lx | ",
 		   2*(int)sizeof(long), (unsigned long)sp,
 		   2*(int)sizeof(long), (unsigned long)val);
+	    fflush(stdout);
 	    erts_printf("%.30T", val);
 	    printf("\r\n");
 	}
@@ -126,7 +128,9 @@ static void print_stack(Eterm *sp, Eterm *end)
 
 void hipe_print_estack(Process *p)
 {
-    printf(" |       BEAM  STACK       |\r\n");
+    printf(" | %*s BEAM   STACK %*s |\r\n",
+	   2*(int)sizeof(long)-3, "",
+	   2*(int)sizeof(long)-4, "");
     print_stack(p->stop, STACK_START(p));
 }
 
@@ -177,11 +181,15 @@ void hipe_print_heap(Process *p)
 void hipe_print_pcb(Process *p)
 {
     printf("P: 0x%0*lx\r\n", 2*(int)sizeof(long), (unsigned long)p);
-    printf("-----------------------------------------------\r\n");
-    printf("Offset| Name        | Value      | *Value     |\r\n");
+    printf("%.*s\r\n",
+           6+1+13+1+2*(int)sizeof(long)+4+1+2*(int)sizeof(long)+4+1,
+           "---------------------------------------------------------------");
+    printf("Offset| Name        | Value %*s | *Value %*s |\r\n",
+           2*(int)sizeof(long)-4, "",
+           2*(int)sizeof(long)-5, "");
 #undef U
 #define U(n,x) \
-    printf(" % 4d | %s | 0x%0*lx |            |\r\n", (int)offsetof(Process,x), n, 2*(int)sizeof(long), (unsigned long)p->x)
+    printf(" % 4d | %s | 0x%0*lx | %*s |\r\n", (int)offsetof(Process,x), n, 2*(int)sizeof(long), (unsigned long)p->x, 2*(int)sizeof(long)+2, "")
 #undef P
 #define P(n,x) \
     printf(" % 4d | %s | 0x%0*lx | 0x%0*lx |\r\n", (int)offsetof(Process,x), n, 2*(int)sizeof(long), (unsigned long)p->x, 2*(int)sizeof(long), p->x ? (unsigned long)*(p->x) : -1UL)
@@ -245,5 +253,7 @@ void hipe_print_pcb(Process *p)
 #endif	/* HIPE */
 #undef U
 #undef P
-    printf("-----------------------------------------------\r\n");
+    printf("%.*s\r\n",
+           6+1+14+1+2*(int)sizeof(long)+4+1+2*(int)sizeof(long)+4+1,
+           "---------------------------------------------------------------");
 }
diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c
index 4001bedeb6..bb93a918a2 100644
--- a/erts/emulator/hipe/hipe_risc_stack.c
+++ b/erts/emulator/hipe/hipe_risc_stack.c
@@ -47,8 +47,10 @@ static void print_slot(Eterm *sp, unsigned int live)
     printf(" | 0x%0*lx | 0x%0*lx | ",
 	   2*(int)sizeof(long), (unsigned long)sp,
 	   2*(int)sizeof(long), val);
-    if (live)
+    if (live) {
+	fflush(stdout);
 	erts_printf("%.30T", val);
+    }
     printf("\r\n");
 }
 
@@ -68,7 +70,9 @@ void hipe_print_nstack(Process *p)
 	[0 ... 2*sizeof(long)+3] = '-'
     };
 
-    printf(" |      NATIVE  STACK      |\r\n");
+    printf(" | %*s NATIVE   STACK %*s |\r\n",
+	   2*(int)sizeof(long)-5, "",
+	   2*(int)sizeof(long)-4, "");
     printf(" |%s|%s|\r\n", dashes, dashes);
     printf(" | %*s | 0x%0*lx |\r\n",
 	   2+2*(int)sizeof(long), "heap",
diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c
index 31582b3a2e..615e07917a 100644
--- a/erts/emulator/hipe/hipe_x86_stack.c
+++ b/erts/emulator/hipe/hipe_x86_stack.c
@@ -43,8 +43,10 @@ static void print_slot(Eterm *sp, unsigned int live)
     printf(" | 0x%0*lx | 0x%0*lx | ",
 	   2*(int)sizeof(long), (unsigned long)sp,
 	   2*(int)sizeof(long), val);
-    if (live)
+    if (live) {
+	fflush(stdout);
 	erts_printf("%.30T", val);
+    }
     printf("\r\n");
 }
 
@@ -74,7 +76,9 @@ void hipe_print_nstack(Process *p)
     sdesc0.livebits[0] = ~1;
     sdesc = &sdesc0;
 
-    printf(" |      NATIVE  STACK      |\r\n");
+    printf(" | %*s NATIVE   STACK %*s |\r\n",
+	   2*(int)sizeof(long)-5, "",
+	   2*(int)sizeof(long)-4, "");
     printf(" |%s|%s|\r\n", dashes, dashes);
     printf(" | %*s | 0x%0*lx |\r\n",
 	   2+2*(int)sizeof(long), "heap",
-- 
cgit v1.2.3


From 6c2b0ff5eb6009efdfdb3fa7b4ee794804e01bf0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 10 Nov 2017 14:43:13 +0100
Subject: Refuse to load "literals" that can be confused with registers

The 's' operand overloads the tags for pids and ports to represent
X and Y registers, respectively.  At load time, refuse to load the
module if the "literal" term is not a pid or port, as it would be
interpreted as a register.

This does not happen with normally compiled code, but it can happen
if the compiler (or beam_asm) is abused like in the following example:

make_bad() ->
    Pid = self(),
    Forms = [{attribute, 0, module, bad_s_operand},
             {attribute, 0, export, [{test, 0}]},
             {function, 0, test, 0,
              [{clause, 0, [], [],
                [{call,0,{atom,0,tuple_size},[{integer, 0, Pid}]}]}]}],
    {ok, Module, Bin} = compile:forms(Forms, [no_copt,no_postopt,report_errors]),
    code:load_binary(Module, "bad_s_operand.erl", Bin).

With this commit applied, the following message will be printed
when make_bad() is run:

=ERROR REPORT==== 10-Nov-2017::14:47:59 ===
Loading of bad_s_operand.erl failed: badfile

=ERROR REPORT==== 10-Nov-2017::14:47:59 ===
beam/beam_load.c(2396): Error loading function bad_s_operand:test/0: op bif1_body: bsd:
  the term '<0.60.0>' would be confused with a register
---
 erts/emulator/beam/beam_load.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 7331c331a6..4fcfea527c 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2384,8 +2384,18 @@ load_code(LoaderState* stp)
 		    code[ci++] = NIL;
 		    break;
 		case TAG_q:
-		    new_literal_patch(stp, ci);
-		    code[ci++] = tmp_op->a[arg].val;
+                    {
+                        BeamInstr val = tmp_op->a[arg].val;
+                        Eterm term = stp->literals[val].term;
+                        new_literal_patch(stp, ci);
+                        code[ci++] = val;
+                        switch (loader_tag(term)) {
+                        case LOADER_X_REG:
+                        case LOADER_Y_REG:
+                            LoadError1(stp, "the term '%T' would be confused "
+                                       "with a register", term);
+                        }
+                    }
 		    break;
 		default:
 		    LoadError1(stp, "bad tag %d for general source",
-- 
cgit v1.2.3


From 773ded1a02898c4a1f9f395d8df1b25444631706 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 27 Oct 2017 13:07:13 +0200
Subject: Document beam_makeops

---
 erts/emulator/internal_doc/beam_makeops.md | 1846 ++++++++++++++++++++++++++++
 1 file changed, 1846 insertions(+)
 create mode 100644 erts/emulator/internal_doc/beam_makeops.md

(limited to 'erts/emulator')

diff --git a/erts/emulator/internal_doc/beam_makeops.md b/erts/emulator/internal_doc/beam_makeops.md
new file mode 100644
index 0000000000..1da8d2ab05
--- /dev/null
+++ b/erts/emulator/internal_doc/beam_makeops.md
@@ -0,0 +1,1846 @@
+The beam\_makeops script
+=======================
+
+This document describes the **beam\_makeops** script.
+
+Introduction
+------------
+
+The **beam\_makeops** Perl script is used at build-time by both the
+compiler and runtime system.  Given a number of input files (all with
+the extension `.tab`), it will generate source files used by the
+Erlang compiler and by the runtime system to load and execute BEAM
+instructions.
+
+Essentially those `.tab` files define:
+
+* External generic BEAM instructions.  They are the instructions that
+are known to both the compiler and the runtime system.  Generic
+instructions are stable between releases.  New generic instructions
+with high numbers than previous instructions can be added in major
+releases.  The OTP 20 release has 159 external generic instructions.
+
+* Internal generic instructions.  They are known only to the runtime
+system and can be changed at any time without compatibility issues.
+They are created by transformation rules (described next).
+
+* Rules for transforming one or more generic instructions to other
+generic instructions.  The transformation rules allow combining,
+splitting, and removal of instructions, as well as shuffling operands.
+Because of the transformation rules, the runtime can have many
+internal generic instructions that are only known to runtime system.
+
+* Specific BEAM instructions.  The specific instructions are the
+instructions that are actually executed by the runtime system.  They
+can be changed at any time without causing compatibility issues.
+The loader translates generic instructions to specific instructions.
+In general, for each generic instruction, there exists a family of
+specific instructions.  The OTP 20 release has 389 specific
+instructions.
+
+* The implementation of specific instructions.
+
+Generic instructions have typed operands. Here are a few examples of
+operands for `move/2`:
+
+    {move,{atom,id},{x,5}}.
+    {move,{x,3},{x,0}}.
+    {move,{x,2},{y,1}}.
+
+When those instructions are loaded, the loader rewrites them
+to specific instructions:
+
+    move_cx id 5
+    move_xx 3 0
+    move_xy 2 1
+
+Corresponding to each generic instruction, there is a family of
+specific instructions.  The types that an instance of a specific
+instruction can handle are encoded in the instruction names.  For
+example, `move_xy` takes an X register number as the first operand and
+a Y register number as the second operand.  `move_cx` takes a tagged
+Erlang term as the first operand and an X register number as the
+second operand.
+
+An example: the move instruction
+--------------------------------
+
+Using the `move` instruction as an example, we will give a quick
+tour to show the main features of **beam\_makeops**.
+
+In the `compiler` application, in the file `genop.tab`, there is the
+following line:
+
+    64: move/2
+
+This is a definition of an external generic BEAM instruction. Most
+importantly it specifices that the opcode is 64.  It also defines that
+it has two operands.  The BEAM assembler will use the opcode when
+creating `.beam` files.  The compiler does not really need the arity,
+but it will use it as an internal sanity check when assembling the
+BEAM code.
+
+Let's have a look at `ops.tab` in `erts/emulator/beam`, where the
+specific `move` instructions are defined.  Here are a few of them:
+
+    move x x
+    move x y
+    move c x
+
+Each specific instructions is defined by following the name of the
+instruction with the types for each operand.  An operand type is a
+single letter.  For example, `x` means an X register, `y`
+means a Y register, and `c` is a "constant" (a tagged term such as
+an integer, an atom, or a literal).
+
+Now let's look at the implementation of the `move` instruction.  There
+are multiple files containing implementations of instructions in the
+`erts/emulator/beam` directory.  The `move` instruction is defined in
+`instrs.tab`.  It looks like this:
+
+    move(Src, Dst) {
+        $Dst = $Src;
+    }
+
+The implementation for an instruction largely follows the C syntax,
+except that the variables in the function head don't have any types.
+The `$` before an identifier denotes a macro expansion.  Thus,
+`$Src` will expand to the code to pick up the source operand for
+the instruction and `$Dst` to the code for the destination register.
+
+We will look at the code for each specific instruction in turn.  To
+make the code easier to understand, let's first look at the memory
+layout for the instruction `{move,{atom,id},{x,5}}`:
+
+         +--------------------+--------------------+
+    I -> |                 40 |       &&lb_move_cx |
+         +--------------------+--------------------+
+         |                        Tagged atom 'id' |
+         +--------------------+--------------------+
+
+This example and all other examples in the document assumes a 64-bit
+archictecture, and furthermore that pointers to C code fit in 32 bits.
+
+`I` in the BEAM virtual machine is the instruction pointer.  When BEAM
+executes an instruction, `I` points to the first word of the
+instruction.
+
+`&&lb_move_cx` is the address to C code that implements `move_cx`.  It
+is stored in the lower 32 bits of the word.  In the upper 32 bits is
+the byte offset to the X register; the register number 5 has been
+multiplied by the word size size 8.
+
+In the next word the tagged atom `id` is stored.
+
+With that background, we can look at the generated code for `move_cx`
+in `beam_hot.h`:
+
+    OpCase(move_cx):
+    {
+      BeamInstr next_pf = BeamCodeAddr(I[2]);
+      xb(BeamExtraData(I[0])) = I[1];
+      I += 2;
+      ASSERT(VALID_INSTR(next_pf));
+      GotoPF(next_pf);
+    }
+
+We will go through each line in turn.
+
+* `OpCase(move_cx):` defines a label for the instruction.  The
+`OpCase()` macro is defined in `beam_emu.c`.  It will expand this line
+to `lb_move_cx:`.
+
+* `BeamInstr next_pf = BeamCodeAddr(I[2]);` fetches the pointer to
+code for the next instruction to be executed.  The `BeamCodeAddr()`
+macro extracts the pointer from the lower 32 bits of the instruction
+word.
+
+* `xb(BeamExtraData(I[0])) = I[1];` is the expansion of `$Dst = $Src`.
+`BeamExtraData()` is a macro that will extract the upper 32 bits from
+the instruction word.  In this example, it will return 40 which is the
+byte offset for X register 5.  The `xb()` macro will cast a byte
+pointer to an `Eterm` pointer and dereference it.  The `I[1]` on
+the right side of the `=` fetches an Erlang term (the atom `id` in
+this case).
+
+* `I += 2` advances the instruction pointer to the next
+instruction.
+
+* In a debug-compiled emulator, `ASSERT(VALID_INSTR(next_pf));` makes
+sure that `next_pf` is a valid instruction (that is, that it points
+within the `process_main()` function in `beam_emu.c`).
+
+* `GotoPF(next_pf);` transfers control to the next instruction.
+
+Now let's look at the implementation of `move_xx`:
+
+    OpCase(move_xx):
+    {
+      Eterm tmp_packed1 = BeamExtraData(I[0]);
+      BeamInstr next_pf = BeamCodeAddr(I[1]);
+      xb((tmp_packed1>>BEAM_TIGHT_SHIFT)) = xb(tmp_packed1&BEAM_TIGHT_MASK);
+      I += 1;
+      ASSERT(VALID_INSTR(next_pf));
+      GotoPF(next_pf);
+    }
+
+We will go through the lines that are new or have changed compared to
+`move_cx`.
+
+* `Eterm tmp_packed1 = BeamExtraData(I[0]);` picks up both X register
+numbers packed into the upper 32 bits of the instruction word.
+
+* `BeamInstr next_pf = BeamCodeAddr(I[1]);` pre-fetches the address of
+the next instruction. Note that because both X registers operands fits
+into the instruction word, the next instruction is in the very next
+word.
+
+* `xb((tmp_packed1>>BEAM_TIGHT_SHIFT)) = xb(tmp_packed1&BEAM_TIGHT_MASK);`
+copies the source to the destination.  (For a 64-bit architecture,
+`BEAM_TIGHT_SHIFT` is 16 and `BEAM_TIGHT_MASK` is `0xFFFF`.)
+
+* `I += 1;` advances the instruction pointer to the next instruction.
+
+`move_xy` is almost identical to `move_xx`.  The only difference is
+the use of the `yb()` macro instead of `xb()` to reference the
+destination register:
+
+    OpCase(move_xy):
+    {
+      Eterm tmp_packed1 = BeamExtraData(I[0]);
+      BeamInstr next_pf = BeamCodeAddr(I[1]);
+      yb((tmp_packed1>>BEAM_TIGHT_SHIFT)) = xb(tmp_packed1&BEAM_TIGHT_MASK);
+      I += 1;
+      ASSERT(VALID_INSTR(next_pf));
+      GotoPF(next_pf);
+    }
+
+### Transformation rules ###
+
+Next let's look at how we can do some optimizations using transformation
+rules.  For simple instructions such as `move/2`, the instruction dispatch
+overhead can be substantial.  A simple optimization is to combine common
+instructions sequences to a single instruction.  One such common sequence
+is multiple `move` instructions moving X registers to Y registers.
+
+Using the following rule we can combine two `move` instructions
+to a `move2` instruction:
+
+    move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2
+
+The left side of the arrow (`=>`) is a pattern.  If the pattern
+matches, the matching instructions will be replaced by the
+instructions on the right side.  Variables in a pattern must start
+with an uppercase letter just as in Erlang.  A pattern variable may be
+followed `=` and one or more type letters to constrain the match to
+one of those types.  The variables that are bound on the left side can
+be used on the right side.
+
+We will also need to define a specific instruction and an implementation:
+
+    # In ops.tab
+    move2 x y x y
+
+    // In instrs.tab
+    move2(S1, D1, S2, D2) {
+        Eterm V1, V2;
+        V1 = $S1;
+        V2 = $S2;
+        $D1 = V1;
+        $D2 = V2;
+    }
+
+When the loader has found a match and replaced the matched instructions,
+it will match the new instructions against the transformation rules.
+Because of that, we can define the rule for a `move3/6` instruction
+as follows:
+
+    move2 X1=x Y1=y X2=x Y2=y | move X3=x Y3=y => \
+          move3 X1 Y1 X2 Y2 X3 Y3
+
+(A `\` before a newline can be used to break a long line for readability.)
+
+It would also be possible to define it like this:
+
+    move X1=x Y1=y | move X2=x Y2=y | move X3=x Y3=y => \
+         move3 X1 Y1 X2 Y2 X3 Y3
+
+but in that case it must be defined before the rule for `move2/4`
+because the first matching rule will be applied.
+
+One must be careful not to create infinite loops.  For example, if we
+for some reason would want to reverse the operand order for the `move`
+instruction, we must not do like this:
+
+    move Src Dst => move Dst Src
+
+The loader would swap the operands forever.  To avoid the loop, we must
+rename the instruction.  For example:
+
+    move Src Dst => assign Dst Src
+
+This concludes the quick tour of the features of **beam\_makeops**.
+
+Short overview of instruction loading
+-------------------------------------
+
+To give some background to the rest of this document, here follows a
+quick overview of how instructions are loaded.
+
+* The loader reads and decodes one instruction at a time from the BEAM
+code and creates a generic instruction.  Many transformation rules
+must look at multiple instructions, so the loader will
+keep multiple generic instructions in a linked list.
+
+* The loader tries to apply transformation rules against the
+generic instructions in the linked list.  If a rule matches, the
+matched instructions will be removed and replaced with new
+generic instructions constructed from the right side of the
+transformation.
+
+* If a transformation rule matched, the loader applies the
+transformation rules again.
+
+* If no transformation rule match, the loader will begin rewriting
+the first of generic instructions to a specific instruction.
+
+* First the loader will search for a specific operation where the
+types for all operands match the type for the generic instruction.
+The first matching instruction will be selected.  **beam\_makeops**
+has ordered the specific instructions so that instructions with more
+specific operands comes before instructions with less specific
+operands.  For example, `move_nx` is more specific than `move_cx`.  If
+the first operand is `[]` (NIL), `move_nx` will be selected.
+
+* Given the opcode for the selected specific instruction, the loader
+looks up the pointer to the C code for the instruction and stores
+in the code area for the module being loaded.
+
+* The loader translates each operand to a machine word and stores it
+in the code area.  The operand type for the selected specific
+instruction guides the translation.  For example, if the type is `e`,
+the value of the operand is an index into an arry of external
+functions and will be translated to a pointer to the export entry for
+the function to call.  If the type is `x`, the number of the X
+register will be multiplied by the word size to produce a byte offset.
+
+* The loader runs the packing engine to pack multiple operands into a
+single word.  The packing engine is controlled by a small program,
+which is a string where each character is an instruction.  For
+example, the code to pack the operands for `move_xy` is `"22#"` (on a
+64-bit machine).  That program will pack the byte offsets for both
+registers into the same word as the pointer to C code.
+
+Running beam_makeops
+--------------------
+
+**beam\_makeops** is found in `$ERL_TOP/erts/emulator/utils`.  Options
+start with a hyphen (`-`).  The options are followed by the name of
+the input files.  By convention, all input files have the extension
+`.tab`, but is not enforced by **beam\_makeops**.
+
+### The -outdir option ###
+
+The option `-outdir Directory` specifies the output directory for
+the generated files.  Default is the current working directory.
+
+### Running beam_makeops for the compiler ###
+
+Give the option `-compiler` to produce output files for the compiler.
+The following files will be written to the output directory:
+
+* `beam_opcodes.erl` - Used primarily by `beam_asm` and `beam_diasm`.
+
+* `beam_opcode.hrl` - Used by `beam_asm`.  It contains tag definitions
+used for encoding instruction operands.
+
+The input file should only contain the definition of BEAM_FORMAT_NUMBER
+and external generic instructions.  (Everything else would be ignored.)
+
+### Running beam_makeops for the emulator ###
+
+Give the option `-emulator` to produce output files for the emulator.
+The following output files will be generated in the output directory.
+
+* `beam_hot.h`, `beam_warm.h`, `beam_cold.`h - Implementation of
+instructions.  Included inside the `process_main()` function in
+`beam_emu.c`.
+
+* `beam_opcodes.c` - Defines static data used by the loader
+(`beam_load.c`).  Data about generic instructions, specific
+instructions (including how to pack their operands), and
+transformation rules are all part of this file.
+
+* `beam_opcodes.h` - Miscellanous preprocessor definitions, mainly
+used by `beam_load.c` but also by `beam_{hot,warm,cold}.h`.
+
+* `beam_pred_funcs.h` - Included by `beam_load.c`.  Contains defines
+needed to call guard constraints in transformation rules.
+
+* `beam_tr_funcs.h` - Included by `beam_load.c`.  Contains defines
+needed to call a C function to the right of a transformation rule.
+
+The following options can be given:
+
+* `wordsize 32|64` - Defines the word size.  Default is 32.
+
+* `code-model Model` - The code model as given to `-mcmodel` option
+for GCC.  Default is `unknown`.  If the code model is `small` (and
+the word size is 64 bits), **beam\_makeops** will pack operands
+into the upper 32 bits of the instruction word.
+
+* `DSymbol=0|1` - Defines the value for a symbol.  The symbol can be
+used in `%if` and `%unless` directives.
+
+Syntax of .tab files
+--------------------
+
+### Comments ###
+
+Any line starting with `#` is a comment and is ignored.
+
+A line with `//` is also a comment.  It is recommended to only
+use this style of comments in files that define implementations of
+instructions.
+
+A long line can be broken into shorter lines by a placing a`\` before
+the newline.
+
+### Variable definitions ###
+
+A variable definition binds a variable to a Perl variable.  It is only
+meaningful to add a new definition if **beam\_makeops** is updated
+at the same time to use the variable.  A variable definition looks this:
+
+*name*=*value*[;]
+
+where *name* is the name of a Perl variable in **beam\_makeops**,
+and *value* is the value to be given to the variable.  The line
+can optionally end with a `;` (to avoid messing up the
+C indentation mode in Emacs).
+
+Here follows a description of the variables that are defined.
+
+#### BEAM\_FORMAT\_NUMBER ####
+
+`genop.tab` has the following definition:
+
+    BEAM_FORMAT_NUMBER=0
+
+It defines the version of the instruction set (which will be
+included in the code header in the BEAM code).  Theoretically,
+the version could be bumped, and all instructions changed.
+In practice, we would have two support two instruction sets
+in the runtime system for at least two releases, so it will
+probably never happen in practice.
+
+#### GC\_REGEXP ####
+
+In `macros.tab`, there is a definition of `GC_REGEXP`.
+It will be described in [a later section](#the-gc_regexp-definition).
+
+### Directives ###
+
+There are directives to classify specific instructions depending
+on how frequently used they are:
+
+* `%hot` - Implementation will be placed in `beam_hot.h`. Frequently
+executed instructions.
+
+* `%warm` - Implementation will be placed in `beam_warm.h`.  Binary
+syntax instructions.
+
+* `%cold` - Implementation will be placed in `beam_cold.h`. Trace
+instructions and infrequently used instructions.
+
+Default is `%hot`.  The directives will be applied to declarations
+of the specific instruction that follow.  Here is an example:
+
+    %cold
+    is_number f? xy
+    %hot
+
+#### Conditional compilation directives ####
+
+The `%if` directive includes a range of lines if a condition is
+true.  For example:
+
+    %if ARCH_64
+    i_bs_get_integer_32 x f? x
+    %endif
+
+The specific instruction `i_bs_get_integer_32` will only be defined
+on a 64-bit machine.
+
+The condition can be inverted by using `%unless` instead of `%if`:
+
+    %unless NO_FPE_SIGNALS
+    fcheckerror p => i_fcheckerror
+    i_fcheckerror
+    fclearerror
+    %endif
+
+It is also possible to add an `%else` clause:
+
+    %if ARCH_64
+    BS_SAFE_MUL(A, B, Fail, Dst) {
+        Uint64 res = ($A) * ($B);
+        if (res / $B != $A) {
+            $Fail;
+        }
+        $Dst = res;
+    }
+    %else
+    BS_SAFE_MUL(A, B, Fail, Dst) {
+        Uint64 res = (Uint64)($A) * (Uint64)($B);
+        if ((res >> (8*sizeof(Uint))) != 0) {
+            $Fail;
+        }
+        $Dst = res;
+    }
+    %endif
+
+#### Symbols that are defined in directives ####
+
+The following symbols are always defined.
+
+* `ARCH_64` - is 1 for a 64-bit machine, and 0 otherwise.
+* `ARCH_32` - is 1 for 32-bit machine, and 1 otherwise.
+
+The `Makefile` for building the emulator currently defines the
+following symbols by using the `-D` option on the command line for
+**beam\_makeops**.
+
+* `NO_FPE_SIGNALS` - 1 if FPE signals are not enable in runtime system,
+0 otherwise.
+* `USE_VM_PROBES` - 1 if the runtime system is compiled to use VM probes (support for dtrace or systemtap), 0 otherwise.
+
+### Defining external generic instructions ###
+
+External generic BEAM instructions are known to both the compiler and
+the runtime system.  They remain stable between releases.  A new major
+release may add more external generic instructions, but must not change
+the semantics for a previously defined instruction.
+
+The syntax for an external generic instruction is as follows:
+
+*opcode*: [-]*name*/*arity*
+
+*opcode* is an integer greater than or equal to 1.
+
+*name* is an identifier starting with a lowercase letter.  *arity* is
+an integer denoting the number of operands.
+
+*name* can optionally be preceded by `-` to indicate that it has been
+obsoleted.  The compiler is not allowed to generate BEAM files that
+use obsolete instructions and the loader will refuse to load BEAM
+files that use obsolete instructions.
+
+It only makes sense to define external generic instructions in the
+file `genop.tab` in `lib/compiler/src`, because the compiler must
+know about them in order to use them.
+
+New instructions must be added at the end of the file, with higher
+numbers than the previous instructions.
+
+### Defining internal generic instructions ###
+
+Internal generic instructions are known only to the runtime
+system and can be changed at any time without compatibility issues.
+
+There are two ways to define internal generic instructions:
+
+* Implicitly when a specific instruction is defined.  This is by far
+the most common way.  Whenever a specific instruction is created,
+**beam\_makeops** automatically creates an internal generic instruction
+if it does not previously exist.
+
+* Explicitly.  This is necessary only when a generic instruction does
+not have any corresponding specific instruction.
+
+The syntax for an internal generic instruction is as follows:
+
+*name*/*arity*
+
+*name* is an identifier starting with a lowercase letter.  *arity* is
+an integer denoting the number of operands.
+
+### About generic instructions in general ###
+
+Each generic instruction has an opcode.  The opcode is an integer,
+greater than or equal to 1. For an external generic instruction, it
+must be explicitly given `genop.tab`, while internal generic
+instructions are automatically numbered by **beam\_makeops**.
+
+The identity of a generic instruction is its name combined with its
+arity.  That means that it is allowed to define two distinct generic
+instructions having the same name but with different arities.  For
+example:
+
+    move_window/5
+    move_window/6
+
+Each operand of a generic instruction is tagged with its type.  A generic
+instruction can have one of the following types:
+
+* `x` - X register.
+
+* `y` - Y register.
+
+* `l` - Floating point register number.
+
+* `i` - Tagged literal integer.
+
+* `a` - Tagged literal atom.
+
+* `n` - NIL (`[]`, the empty list).
+
+* `q` - Literal that don't fit in a word, that is an object stored on
+the heap such as a list or tuple.  Any heap object type is supported,
+even types that don't have real literals such as external references.
+
+* `f` - Non-zero failure label.
+
+* `p` - Zero failure label.
+
+* `u` - Untagged integer that fits in a machine word.  It is used for many
+different purposes, such as the number of live registers in `test_heap/2`,
+as a reference to the export for `call_ext/2`, and as the flags operand for
+binary syntax instructions.  When the generic instruction is translated to a
+specific instruction, the type for the operand in the specific operation will
+tell the loader how to treat the operand.
+
+* `o` - Overflow.  If the value for an `u` operand does not fit in a machine
+word, the type of the operand will be changed to `o` (with no associated
+value).  Currently only used internally in the loader in the guard constraint
+function `binary_too_big()`.
+
+* `v` - Arity value.  Only used internally in the loader.
+
+
+### Defining specific instructions ###
+
+The specific instructions are known only to the runtime system and
+are the instructions that are actually executed.  They can be changed
+at any time without causing compatibility issues.
+
+A specific instruction can have at most 6 operands.
+
+A specific instruction is defined by first giving its name followed by
+the types for each operand.  For example:
+
+     move x y
+
+Internally, for example in the generated code and in the output from
+the BEAM disassembler, the instruction `move x y` will be called `move_xy`.
+
+The name for a specific instruction is an identifier starting with a
+lowercase letter.  A type is an lowercase or uppercase letter.
+
+All specific instructions with a given name must have the same number
+of operands. That is, the following is **not** allowed:
+
+     move x x
+     move x y x y
+
+Here follows the type letters that more or less directly corresponds
+to the types for generic instructions.
+
+* `x` - X register.  Will be loaded as a byte offset to the X register
+relative to the base of X register array.  (Can be packed with other
+operands.)
+
+* `y` - Y register.  Will be loaded as a byte offset to the Y register
+relative to the stack frame. (Can be packed with other operands.)
+
+* `r` - X register 0.  An implicit operand that will not be stored in
+the loaded code.
+
+* `l` - Floating point register number.  (Can be packed with other
+operands.)
+
+* `i` - Tagged literal integer (a SMALL that will fit in one word).
+
+* `a` - Tagged atom.
+
+* `n` - NIL or the empty list.  (Will not be stored in the loaded code.)
+
+* `q` - Tagged CONS or BOXED pointer.  That is, a term such as a list
+or tuple.  Any heap object type is supported, even types that don't
+have real literals such as external references.
+
+* `f` - Failure label (non-zero).  The target for a branch
+or call instruction.
+
+* `p` - The 0 failure label, meaning that an exception should be raised
+if the instruction fails.  (Will not be stored in the loaded code.)
+
+* `c` - Any literal term; that is, immediate literals such as SMALL,
+and CONS or BOXED pointers to literals.  (Can be used where the
+operand in the generic instruction has one of the types `i`, `a`, `n`,
+or `q`.)
+
+The types that follow do a type test of the operand at runtime; thus,
+they are generally more expensive in terms of runtime than the types
+described earlier.  However, those operand types are needed to avoid a
+combinatorial explosion in the number of specific instructions and
+overall code size of `process_main()`.
+
+* `s` - Tagged source: X register, Y register, or a literal term.  The
+tag will be tested at runtime to retrieve the value from an X
+register, a Y register, or simply use the value as a tagged Erlang
+term.  (Implementation note: An X register is tagged as a pid, and a Y
+register as a port.  Therefore the literal term must not contain a
+port or pid.)
+
+* `S` - Tagged source register (X or Y).  The tag will be tested at
+runtime to retrieve the value from an X register or a Y register.  Slighly
+cheaper than `s`.
+
+* `d` - Tagged destination register (X or Y).  The tag will be tested
+at runtime to set up a pointer to the destination register.  If the
+instrution performs a garbarge collection, it must use the
+`$REFRESH_GEN_DEST()` macro to refresh the pointer before storing to
+it (there are more details about that in a later section).
+
+* `j` - A failure label (combination of `f` and `p`).  If the branch target 0,
+an exception will be raised if instruction fails, otherwise control will be
+transfered to the target address.
+
+The types that follows are all applied to an operand that has the `u`
+type.
+
+* `t` - An untagged integer that will fit in 12 bits (0-4096).  It can be
+packed with other operands in a word.  Most often used as the number
+of live registers in instructions such as `test_heap`.
+
+* `I` - An untagged integer that will fit in 32 bits.  It can be
+packed with other operands in a word on a 64-bit system.
+
+* `W` - Untagged integer or pointer.  Not possible to pack with other
+operands.
+
+* `e` - Pointer to an export entry.  Use by call instructions that call
+other modules, such as `call_ext`.
+
+* `L` - A label.  Only used by the `label/1` instruction.
+
+* `b` - Pointer to BIF.  Used by instructions that BIFs, such as
+`call_bif`.
+
+* `A` - A tagged arityvalue.  Used in instructions that test the arity
+of a tuple.
+
+* `P` - A byte offset into a tuple.
+
+* `Q` - A byte offset into the stack.  Used for updating the frame
+pointer register.  Can be packed with other operands.
+
+When the loader translates a generic instruction a specific
+instruction, it will choose the most specific instruction that will
+fit the types.  Consider the following two instructions:
+
+    move c x
+    move n x
+
+The `c` operand can encode any literal value, including NIL.  The
+`n` operand only works for NIL.  If we have the generic instruction
+`{move,nil,{x,1}}`, the loader will translate it to `move_nx 1`
+because `move n x` is more specific.  `move_nx` could be slightly
+faster or smaller (depending on the architecture), because the `[]`
+is not stored explicitly as an operand.
+
+#### Syntactic sugar for specific instructions ####
+
+It is possible to specify more than one type letter for each operand.
+Here is an example:
+
+    move cxy xy
+
+This is syntactic sugar for:
+
+    move c x
+    move c y
+    move x x
+    move x y
+    move y x
+    move y y
+
+Note the difference between `move c xy` and `move c d`.  Note that `move c xy`
+is equivalent to the following two definitions:
+
+    move c x
+    move c y
+
+On the other hand, `move c d` is a single instruction.  At runtime,
+the `d` operand will be tested to see whether it refers to an X
+register or a Y register, and a pointer to the register will be set
+up.
+
+#### The '?' type modifier ####
+
+The character `?` can be added to the end of an operand to indicate
+that the operand will not be used every time the instruction is executed.
+For example:
+
+    allocate_heap t I t?
+    is_eq_exact f? x xy
+
+In `allocate_heap`, the last operand is the number of live registers.
+It will only be used if there is not enough heap space and a garbage
+collection must be performed.
+
+In `is_eq_exact`, the failure address (the first operand) will only be
+used if the two register operands are not equal.
+
+Knowing that an operand is not always used can improve how packing
+is done for some instructions.
+
+For the `allocate_heap` instruction, without the `?` the packing would
+be done like this:
+
+         +--------------------+--------------------+
+    I -> |       Stack needed | &&lb_allocate_heap +
+         +--------------------+--------------------+
+         |        Heap needed | Live registers     +
+         +--------------------+--------------------+
+
+"Stack needed" and "Heap needed" are always used, but they are in
+different words.  Thus, at runtime the `allocate_heap` instruction
+must read both words from memory even though it will not always use
+"Live registers".
+
+With the `?`, the operands will be packed like this:
+
+         +--------------------+--------------------+
+    I -> |     Live registers | &&lb_allocate_heap +
+         +--------------------+--------------------+
+         |        Heap needed |       Stack needed +
+         +--------------------+--------------------+
+
+Now "Stack needed" and "Heap needed" are in the same word.
+
+### Defining transformation rules ###
+
+Transformation rules are used to rewrite generic instructions to other
+generic instructions.  The transformations rules are applied
+repeatedly until no rule match.  At that point, the first instruction
+in the resulting instruction sequence will be converted to a specific
+instruction and added to the code for the module being loaded.  Then
+the transformation rules for the remaining instructions are run in the
+same way.
+
+A rule is recognized by its right-pointer arrow: `=>`.  To the left of
+the arrow is one or more instruction patterns, separated by `|`.  To
+the right of the arrow is zero or more instructions, separated by `|`.
+If the instructions from the BEAM code matches the instruction
+patterns on the left side, they will be replaced with instructions on
+the right side (or removed if there are no instructions on the right).
+
+#### Defining instruction patterns ####
+
+We will start looking at the patterns on the left side of the arrow.
+
+A pattern for an instruction consists of its name, followed by a pattern
+for each of its operands.  The operand patterns are separated by spaces.
+
+The simplest possible pattern is a variable.  Just like in Erlang,
+a variable must begin with an uppercase letter.  If the same variable is
+used in multiple operands, the pattern will only match if the operands
+are equal.  For example:
+
+    move Same Same =>
+
+This pattern will match if the operands for `move` are the same.  If
+the pattern match, the instruction will be removed.  (That used to be an
+actual rule a long time ago when the compiler would occasionally produce
+instructions such as `{move,{x,2},{x,2}}`.)
+
+Variables that have been bound on the left side can be used on the
+right side.  For example, this rule will rewrite all `move` instructions
+to `assign` instructions with the operands swapped:
+
+    move Src Dst => assign Dst Src
+
+If we only want to match operands of a certain type, we can
+use a type constraint.  A type constraint consists of one or more
+lowercase letters, each specifying a type.  For example:
+
+    is_integer Fail an => jump Fail
+
+The second operand pattern, `an`, will match if the second operand is
+either an atom or NIL (the empty list).  In case of a match, the
+`is_integer/2` instruction will be replaced with a `jump/1`
+instruction.
+
+An operand pattern can bind a variable and constrain the type at the
+same time by following the variable with a `=` and the constraint.
+For example:
+
+    is_eq_exact Fail=f R=xy C=q => i_is_eq_exact_literal Fail R C
+
+Here the `is_eq_exact` instruction is replaced with a specialized instruction
+that only compares literals, but only if the first operand is a register and
+the second operand is a literal.
+
+#### Further constraining patterns ####
+
+In addition to specifying a type letter, the actual value for the type can
+be specified.  For example:
+
+    move C=c x==1 => move_x1 C
+
+Here the second operand of `move` is constrained to be X register 1.
+
+When specifying an atom constraint, the atom is written as it would be
+in the C source code.  That is, it needs an `am_` prefix, and it must
+be listed in `atom.names`.  For example:
+
+    is_boolean Fail=f a==am_true =>
+    is_boolean Fail=f a==am_false =>
+
+There are several constraints available for testing whether a call is to a BIF
+or a function.
+
+The constraint `u$is_bif` will test whether the given operand refers to a BIF.
+For example:
+
+    call_ext u Bif=u$is_bif => call_bif Bif
+    call_ext u Func         => i_call_ext Func
+
+The `call_ext` instruction can be used to call functions written in
+Erlang as well as BIFs (or more properly called SNIFs).  The
+`u$is_bif` constraint will match if the operand refers to a BIF (that
+is, if it is listed in the file `bif.tab`).  Note that `u$is_bif`
+should only be applied to operands that are known to contain an index
+to the import table chunk in the BEAM file (such operands have the
+type `b` or `e` in the corresponding specific instruction).  If
+applied to other `u` operands, it will at best return a nonsense
+result.
+
+The `u$is_not_bif` constraint matches if the operand does not refer to
+a BIF (not listed in `bif.tab`).  For example:
+
+    move S X0=x==0 | line Loc | call_ext_last Ar Func=u$is_not_bif D => \
+         move S X0 | call_ext_last Ar Func D
+
+The `u$bif:Module:Name/Arity` constraint tests whether the given
+operand refers to a specific BIF.  Note that `Module:Name/Arity`
+**must** be an existing BIF defined in `bif.tab`, or there will
+be a compilation error.  It is useful when a call to a specific BIF
+should be replaced with an instruction as in this example:
+
+    gc_bif2 Fail Live u$bif:erlang:splus/2 S1 S2 Dst => \
+         gen_plus Fail Live S1 S2 Dst
+
+Here the call to the GC BIF `'+'/2` will be replaced with the instruction
+`gen_plus/5`.  Note that the same name as used in the C source code must be
+used for the BIF, which in this case is `splus`.  It is defined like this
+in `bit.tab`:
+
+    ubif erlang:'+'/2 splus_2
+
+The `u$func:Module:Name/Arity` will test whether the given operand is a
+a specific function.  Here is an example:
+
+    bif1 Fail u$func:erlang:is_constant/1 Src Dst => too_old_compiler
+
+`is_constant/1` used to be a BIF a long time ago.  The transformation
+replaces the call with the `too_old_compiler` instruction which will produce
+a nicer error message than the default error would be for a missing guard BIF.
+
+#### Type constraints allowed in patterns ####
+
+Here are all type letters that are allowed on the left side of a transformation
+rule.
+
+* `u` - An untagged integer that fits in a machine word.
+
+* `x` - X register.
+
+* `y` - Y register.
+
+* `l` - Floating point register number.
+
+* `i` - Tagged literal integer.
+
+* `a` - Tagged literal atom.
+
+* `n` - NIL (`[]`, the empty list).
+
+* `q` - Literals that don't fit in a word, such as list or tuples.
+
+* `f` - Non-zero failure label.
+
+* `p` - The zero failure label.
+
+* `j` - Any label.  Equivalent to `fp`.
+
+* `c` - Any literal term.  Equivalent to `ainq`.
+
+* `s` - X register, Y register, or any literal term.  Equivalent to `xyc`.
+
+* `d` - X or Y register.  Equivalent to `xy`.  (In a pattern `d` will
+match both source and destination registers.  As an operand in a specific
+instruction, it must only be used for a destination register.)
+
+* `o` - Overflow.  An untagged integer that does not fit in a machine word.
+
+#### Guard constraints ####
+
+If the constraints described so far is not enough, additional
+constraints can be written in C in `beam_load.c` and be called as a
+guard function on the left side of the transformation.  If the guard
+function returns a non-zero value, the matching of the rule will
+continue, otherwise the match will fail.  For example:
+
+    ensure_map Lit=q | literal_is_map(Lit) =>
+
+The guard test `literal_is_map/1` tests whether the given literal is a map.
+If the literal is a map, the instruction is unnecessary and can be removed.
+
+It is outside the scope for this document to describe in detail how such
+guard functions are written, but for the curious here is the implementation
+of `literal_is_map()`:
+
+    static int
+    literal_is_map(LoaderState* stp, GenOpArg Lit)
+    {
+        Eterm term;
+
+        ASSERT(Lit.type == TAG_q);
+        term = stp->literals[Lit.val].term;
+        return is_map(term);
+    }
+
+#### Handling instruction with variable number of operands ####
+
+Some instructions, such as `select_val/3`, essentially has a variable
+number of operands.  Such instructions have a `{list,[...]}` operand
+as their last operand in the BEAM assembly code.  For example:
+
+    {select_val,{x,0},
+                {f,1},
+                {list,[{atom,b},{f,4},{atom,a},{f,5}]}}.
+
+The loader will convert a `{list,[...]}` operand to an `u` operand whose
+value is the number of elements in the list, followed by each element in
+the list.  The instruction above would be translated to the following
+generic instruction:
+
+    {select_val,{x,0},{f,1},{u,4},{atom,b},{f,4},{atom,a},{f,5}}
+
+To match a variable number of arguments we need to use the special
+operand type `*` like this:
+
+    select_val Src=aiq Fail=f Size=u List=* => \
+        i_const_select_val Src Fail Size List
+
+This transformation renames a `select_val/3` instruction
+with a constant source operand to `i_const_select_val/3`.
+
+#### Constructing new instructions on the right side ####
+
+The most common operand on the right side is a variable that was bound while
+matching the left side.  For example:
+
+    trim N Remaining => i_trim N
+
+An operand can also be a type letter to construct an operand of that type.
+Each type has a default value.  For example, the type `x` has the default
+value 1023, which is the highest X register.  That makes `x` on the right
+side a convenient shortcut for a temporary X register.  For example:
+
+    is_number Fail Literal=q => move Literal x | is_number Fail x
+
+If the second operand for `is_number/2` is a literal, it will be moved to
+X register 1023.  Then `is_number/2` will test whether the value stored in
+X register 1023 is a number.
+
+This kind of transformation is useful when it is rare that an operand can
+be anything else but a register.  In the case of `is_number/2`, the second
+operand is always a register unless the compiler optimizations have been
+disabled.
+
+If the default value is not suitable, the type letter can be followed
+by `=` and a value.  Most types take an integer value.  The value for
+an atom is written the same way as in the C source code.  For example,
+the atom `false` is written as `am_false`.  The atom must be listed in
+`atom.names`.
+
+Here is an example showing how values can be specified:
+
+    bs_put_utf32 Fail=j Flags=u Src=s => \
+        i_bs_validate_unicode Fail Src | \
+        bs_put_integer Fail i=32 u=1 Flags Src
+
+#### Type letters on the right side ####
+
+Here follows all types that are allowed to be used in operands for
+instructions being constructed on the right side of a transformation
+rule.
+
+* `u` - Construct an untagged integer.  The default value is 0.
+
+* `x` - X register.  The default value is 1023.  That makes `x` convenient to
+use as a temporary X register.
+
+* `y` - Y register.  The default value is 0.
+
+* `l` - Foating point register number.  The default value is 0.
+
+* `i` - Tagged literal integer.  The default value is 0.
+
+* `a` - Tagged atom.  The default value is the empty atom (`am_Empty`).
+
+* `n` - NIL (`[]`, the empty list).
+
+#### Function call on the right side ####
+
+Transformations that are not possible to describe with the rule
+language as described here can be written as a C function in
+`beam_load.c` and called from the right side of a transformation.  The
+left side of the transformation will perform the match and bind
+operands to variables.  The variables can then be passed to a
+generator function on the right side.  For example:
+
+    bif2 Fail=j u$bif:erlang:element/2 Index=s Tuple=xy Dst=d => \
+        gen_element(Jump, Index, Tuple, Dst)
+
+This transformation rule matches a call to the BIF `element/2`.
+The operands will be captured and the function `gen_element()` will
+be called.
+
+`gen_element()` will produce one of two instructions depending
+on `Index`.  If `Index` is an integer in the range from 1 up to
+the maximum tuple size, the instruction `i_fast_element/2` will
+be produced, otherwise the instruction `i_element/4` will be
+produced.  The corresponding specific instructions are:
+
+    i_fast_element xy j? I d
+    i_element xy j? s d
+
+The `i_fast_element/2` instruction is faster because the tuple is
+already an untagged integer.  It also knows that the index is at least
+1, so it does not have to test for that.  The `i_element/4`
+instruction will have to fetch the index from a register, test that it
+is an integer, and untag the integer.
+
+It is outside the scope of this document to describe in detail how
+generator functions are written, but for the curious, here is the
+implementation of `gen_element()`:
+
+    static GenOp*
+    gen_element(LoaderState* stp, GenOpArg Fail,
+       GenOpArg Index, GenOpArg Tuple, GenOpArg Dst)
+    {
+        GenOp* op;
+
+        NEW_GENOP(stp, op);
+        op->arity = 4;
+        op->next = NULL;
+
+        if (Index.type == TAG_i && Index.val > 0 &&
+           Index.val <= ERTS_MAX_TUPLE_SIZE &&
+           (Tuple.type == TAG_x || Tuple.type == TAG_y)) {
+            op->op = genop_i_fast_element_4;
+            op->a[0] = Tuple;
+            op->a[1] = Fail;
+            op->a[2].type = TAG_u;
+            op->a[2].val = Index.val;
+            op->a[3] = Dst;
+        } else {
+            op->op = genop_i_element_4;
+            op->a[0] = Tuple;
+            op->a[1] = Fail;
+            op->a[2] = Index;
+            op->a[3] = Dst;
+        }
+
+        return op;
+    }
+}
+
+### Defining the implementation ###
+
+The actual implementation of instructions are also defined in `.tab`
+files processed by **beam\_makeops**.  For practical reasons,
+instruction definitions are stored in several files, at the time of
+writing in the following files:
+
+    bif_instrs.tab
+    arith_instrs.tab
+    bs_instrs.tab
+    float_instrs.tab
+    instrs.tab
+    map_instrs.tab
+    msg_instrs.tab
+    select_instrs.tab
+    trace_instrs.tab
+
+There is also a file that only contains macro definitions:
+
+    macros.tab
+
+The syntax of each file is similar to C code.  In fact, most of
+the contents *is* C code, interspersed with macro invocations.
+
+To allow Emacs to auto-indent the code, each file starts with the
+following line:
+
+    // -*- c -*-
+
+To avoid messing up the indentation, all comments are written
+as C++ style comments (`//`) instead of `#`.  Note that a comment
+must start at the beginning of a line.
+
+The meat of an instruction definition file are macro definitions.
+We have seen this macro definition before:
+
+    move(Src, Dst) {
+        $Dst = $Src;
+    }
+
+A macro definitions must start at the beginning of the line (no spaces
+allowed), the opening curly bracket must be on the same line, and the
+finishing curly bracket must be at the beginning of a line.  It is
+recommended that the macro body is properly indented.
+
+As a convention, the macro arguments in the head all start with an
+uppercase letter.  In the body, the macro arguments can be expanded
+by preceding them with `$`.
+
+A macro definition whose name and arity matches a family of
+specific instructions is assumed to be the implementation of that
+instruction.
+
+A macro can also be invoked from within another macro.  For example,
+`move_deallocate_return/2` avoids repeating code by invoking
+`$deallocate_return()` as a macro:
+
+    move_deallocate_return(Src, Deallocate) {
+        x(0) = $Src;
+        $deallocate_return($Deallocate);
+    }
+
+Here is the definition of `deallocate_return/1`:
+
+    deallocate_return(Deallocate) {
+        //| -no_next
+        int words_to_pop = $Deallocate;
+        SET_I((BeamInstr *) cp_val(*E));
+        E = ADD_BYTE_OFFSET(E, words_to_pop);
+        CHECK_TERM(x(0));
+        DispatchReturn;
+    }
+
+The expanded code for `move_deallocate_return` will look this:
+
+    OpCase(move_deallocate_return_cQ):
+    {
+      x(0) = I[1];
+      do {
+        int words_to_pop = Qb(BeamExtraData(I[0]));
+        SET_I((BeamInstr *) cp_val(*E));
+        E = ADD_BYTE_OFFSET(E, words_to_pop);
+        CHECK_TERM(x(0));
+        DispatchReturn;
+      } while (0);
+    }
+
+When expanding macros, **beam\_makeops** wraps the expansion in a
+`do`/`while` wrapper unless **beam\_makeops** can clearly see that no
+wrapper is needed.  In this case, the wrapper is needed.
+
+Note that arguments for macros cannot be complex expressions, because
+the arguments are split on `,`.  For example, the following would
+not work because **beam\_makeops** would split the expression into
+two arguments:
+
+    $deallocate_return(get_deallocation(y, $Deallocate));
+
+#### Code generation directives ####
+
+Within macro definitions, `//` comments are in general not treated
+specially.  They will be copied to the file with the generated code
+along with the rest of code in the body.
+
+However, there is an exception. Within a macro definition, a line that
+starts with whitespace followed by `//|` is treated specially.  The
+rest of the line is assumed to contain directives to control code
+generation.
+
+Currently, two code generation directives are recognized:
+
+* `-no_prefetch`
+* `-no_next`
+
+##### The -no_prefetch directive #####
+
+To see what `-no_prefetch` does, let's first look at the default code
+generation.  Here is the code generated for `move_cx`:
+
+    OpCase(move_cx):
+    {
+      BeamInstr next_pf = BeamCodeAddr(I[2]);
+      xb(BeamExtraData(I[0])) = I[1];
+      I += 2;
+      ASSERT(VALID_INSTR(next_pf));
+      GotoPF(next_pf);
+    }
+
+Note that the very first thing done is to fetch the address to the
+next instruction.  The reason is that it usually improves performance.
+
+Just as a demonstration, we can add a `-no_prefetch` directive to
+the `move/2` instruction:
+
+    move(Src, Dst) {
+        //| -no_prefetch
+        $Dst = $Src;
+    }
+
+We can see that the prefetch is no longer done:
+
+    OpCase(move_cx):
+    {
+      xb(BeamExtraData(I[0])) = I[1];
+      I += 2;
+      ASSERT(VALID_INSTR(*I));
+      Goto(*I);
+    }
+
+When would we want to turn off the prefetch in practice?
+
+In instructions that will not always execute the next instruction.
+For example:
+
+    is_atom(Fail, Src) {
+        if (is_not_atom($Src)) {
+            $FAIL($Fail);
+        }
+    }
+
+    // From macros.tab
+    FAIL(Fail) {
+        //| -no_prefetch
+        $SET_I_REL($Fail);
+        Goto(*I);
+    }
+
+`is_atom/2` may either execute the next instruction (if the second
+operand is an atom) or branch to the failure label.
+
+The generated code looks like this:
+
+    OpCase(is_atom_fx):
+    {
+      if (is_not_atom(xb(I[1]))) {
+        ASSERT(VALID_INSTR(*(I + (fb(BeamExtraData(I[0]))) + 0)));
+        I += fb(BeamExtraData(I[0])) + 0;;
+        Goto(*I);;
+      }
+      I += 2;
+      ASSERT(VALID_INSTR(*I));
+      Goto(*I);
+    }
+
+##### The -no_next directive #####
+
+Next we will look at when the `-no_next` directive can be used.  Here
+is the `jump/1` instruction:
+
+    jump(Fail) {
+        $JUMP($Fail);
+    }
+
+    // From macros.tab
+    JUMP(Fail) {
+        //| -no_next
+        $SET_I_REL($Fail);
+        Goto(*I);
+    }
+
+The generated code looks like this:
+
+    OpCase(jump_f):
+    {
+      ASSERT(VALID_INSTR(*(I + (fb(BeamExtraData(I[0]))) + 0)));
+      I += fb(BeamExtraData(I[0])) + 0;;
+      Goto(*I);;
+    }
+
+If we remove the `-no_next` directive, the code would look like this:
+
+    OpCase(jump_f):
+    {
+      BeamInstr next_pf = BeamCodeAddr(I[1]);
+      ASSERT(VALID_INSTR(*(I + (fb(BeamExtraData(I[0]))) + 0)));
+      I += fb(BeamExtraData(I[0])) + 0;;
+      Goto(*I);;
+      I += 1;
+      ASSERT(VALID_INSTR(next_pf));
+      GotoPF(next_pf);
+    }
+
+In the end, the C compiler will probably optimize this code to the
+same native code as the first version, but the first version is certainly
+much easier to read for human readers.
+
+#### Macros in the macros.tab file ####
+
+The file `macros.tab` contains many useful macros.  When implementing
+new instructions it is good practice to look through `macros.tab` to
+see if any of existing macros can be used rather than re-inventing
+the wheel.
+
+We will describe a few of the most useful macros here.
+
+##### The GC_REGEXP definition #####
+
+The following line defines a regular expression that will recognize
+a call to a function that does a garbage collection:
+
+     GC_REGEXP=erts_garbage_collect|erts_gc|GcBifFunction;
+
+The purpose is that **beam\_makeops** can verify that an instruction
+that does a garbage collection and has an `d` operand uses the
+`$REFRESH_GEN_DEST()` macro.
+
+If you need to define a new function that does garbage collection,
+you should give it the prefix `erts_gc_`.  If that is not possible
+you should update the regular expression so that it will match your
+new function.
+
+##### FAIL(Fail) #####
+
+Branch to `$Fail`.  Will suppress prefetch (`-no_prefetch`).  Typical use:
+
+    is_nonempty_list(Fail, Src) {
+        if (is_not_list($Src)) {
+            $FAIL($Fail);
+        }
+    }
+
+##### JUMP(Fail) #####
+
+Branch to `$Fail`.  Suppresses generation of dispatch of the next
+instruction (`-no_next`).  Typical use:
+
+    jump(Fail) {
+        $JUMP($Fail);
+    }
+
+##### GC_TEST(NeedStack, NeedHeap, Live) #####
+
+`$GC_TEST(NeedStack, NeedHeap, Live)` tests that given amount of
+stack space and heap space is available.  If not it will do a
+garbage collection.  Typical use:
+
+    test_heap(Nh, Live) {
+        $GC_TEST(0, $Nh, $Live);
+    }
+
+##### AH(NeedStack, NeedHeap, Live) #####
+
+`AH(NeedStack, NeedHeap, Live)` allocates a stack frame and
+optionally additional heap space.
+
+#### Pre-defined macros and variables ####
+
+**beam\_makeops** defines several built-in macros and pre-bound variables.
+
+##### The NEXT_INSTRUCTION pre-bound variable #####
+
+The NEXT_INSTRUCTION is a pre-bound variable that is available in
+all instructions.  It expands to the address of the next instruction.
+
+Here is an example:
+
+    i_call(CallDest) {
+        SET_CP(c_p, $NEXT_INSTRUCTION);
+        $DISPATCH_REL($CallDest);
+    }
+
+When calling a function, the return address is first stored in `c_p->cp`
+(using the `SET_CP()` macro defined in `beam_emu.c`), and then control is
+transferred to the callee.  Here is the generated code:
+
+    OpCase(i_call_f):
+    {
+      SET_CP(c_p, I+1);
+      ASSERT(VALID_INSTR(*(I + (fb(BeamExtraData(I[0]))) + 0)));
+      I += fb(BeamExtraData(I[0])) + 0;;
+      DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I));
+      Dispatch();;
+    }
+
+We can see that that `$NEXT_INSTRUCTION` has been expanded to `I+1`.
+That makes sense since the size of the `i_call_f/1` instruction is
+one word.
+
+##### The IP_ADJUSTMENT pre-bound variable #####
+
+`$IP_ADJUSTMENT` is usually 0.  In a few combined instructions
+(described below) it can be non-zero.  It is used like this
+in `macros.tab`:
+
+    SET_I_REL(Offset) {
+        ASSERT(VALID_INSTR(*(I + ($Offset) + $IP_ADJUSTMENT)));
+        I += $Offset + $IP_ADJUSTMENT;
+    }
+
+Avoid using `IP_ADJUSTMENT` directly.  Use `SET_I_REL()` or
+one of the macros that invoke such as `FAIL()` or `JUMP()`
+defined in `macros.tab`.
+
+#### Pre-defined macro functions ####
+
+##### The IF() macro #####
+
+`$IF(Expr, IfTrue, IfFalse)` evaluates `Expr`, which must be a valid
+Perl expression (which for simple numeric expressions have the same
+syntax as C).  If `Expr` evaluates to 0, the entire `IF()` expression will be
+replaced with `IfFalse`, otherwise it will be replaced with `IfTrue`.
+
+See the description of `OPERAND_POSITION()` for an example.
+
+##### The OPERAND\_POSITION() macro #####
+
+`$OPERAND_POSITION(Expr)` returns the position for `Expr`, if
+`Expr` is an operand that is not packed.  The first operand is
+at position 1.
+
+Returns 0 otherwise.
+
+This macro could be used like this in order to share code:
+
+    FAIL(Fail) {
+        //| -no_prefetch
+        $IF($OPERAND_POSITION($Fail) == 1 && $IP_ADJUSTMENT == 0,
+            goto common_jump,
+            $DO_JUMP($Fail));
+    }
+
+    DO_JUMP(Fail) {
+        $SET_I_REL($Fail);
+        Goto(*I));
+    }
+
+    // In beam_emu.c:
+    common_jump:
+       I += I[1];
+       Goto(*I));
+
+
+#### The $REFRESH\_GEN\_DEST() macro ####
+
+When a specific instruction has a `d` operand, early during execution
+of the instruction, a pointer will be initialized to point to the X or
+Y register in question.
+
+If there is a garbage collection before the result is stored,
+the stack will move and if the `d` operand refered to a Y
+register, the pointer will no longer be valid.  (Y registers are
+stored on the stack.)
+
+In those circumstances, `$REFRESH_GEN_DEST()` must be invoked
+to set up the pointer again.  **beam\_makeops** will notice
+if there is a call to a function that does a garbage collection and
+`$REFRESH_GEN_DEST()` is not called.
+
+Here is a complete example.  The `new_map` instruction is defined
+like this:
+
+    new_map d t I
+
+It is implemented like this:
+
+    new_map(Dst, Live, N) {
+        Eterm res;
+
+        HEAVY_SWAPOUT;
+        res = erts_gc_new_map(c_p, reg, $Live, $N, $NEXT_INSTRUCTION);
+        HEAVY_SWAPIN;
+        $REFRESH_GEN_DEST();
+        $Dst = res;
+        $NEXT($NEXT_INSTRUCTION+$N);
+    }
+
+If we have forgotten the `$REFRESH_GEN_DEST()` there would be a message
+similar to this:
+
+    pointer to destination register is invalid after GC -- use $REFRESH_GEN_DEST()
+    ... from the body of new_map at beam/map_instrs.tab(30)
+
+#### Combined instructions ####
+
+**Problem**: For frequently executed instructions we want to use
+"fast" operands types such as `x` and `y`, as opposed to `s` or `S`.
+To avoid an explosion in code size, we want to share most of the
+implementation between the instructions.  Here are the specific
+instructions for `i_increment/5`:
+
+    i_increment r W t d
+    i_increment x W t d
+    i_increment y W t d
+
+The `i_increment` instruction is implemented like this:
+
+    i_increment(Source, IncrementVal, Live, Dst) {
+        Eterm increment_reg_source = $Source;
+        Eterm increment_val = $IncrementVal;
+        Uint live;
+        Eterm result;
+
+        if (ERTS_LIKELY(is_small(increment_reg_val))) {
+            Sint i = signed_val(increment_reg_val) + increment_val;
+            if (ERTS_LIKELY(IS_SSMALL(i))) {
+                $Dst = make_small(i);
+                $NEXT0();
+            }
+        }
+        live = $Live;
+        HEAVY_SWAPOUT;
+        reg[live] = increment_reg_val;
+        reg[live+1] = make_small(increment_val);
+        result = erts_gc_mixed_plus(c_p, reg, live);
+        HEAVY_SWAPIN;
+        ERTS_HOLE_CHECK(c_p);
+        if (ERTS_LIKELY(is_value(result))) {
+            $REFRESH_GEN_DEST();
+            $Dst = result;
+            $NEXT0();
+        }
+        ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue));
+        goto find_func_info;
+    }
+
+There will be three almost identical copies of the code.  Given the
+size of the code, that could be too high cost to pay.
+
+To avoid the three copies of the code, we could use only one specific
+instruction:
+
+    i_increment S W t d
+
+(The same implementation as above will work.)
+
+That reduces the code size, but is slower because `S` means that
+there will be extra code to test whether the operand refers to an X
+register or a Y register.
+
+**Solution**: We can use "combined instructions".  Combined
+instructions are combined from instruction fragments.  The
+bulk of the code can be shared.
+
+Here we will show how `i_increment` can be implemented as a combined
+instruction.  We will show each individual fragment first, and then
+show how to connect them together.  First we will need a variable that
+we can store the value fetched from the register in:
+
+    increment.head() {
+        Eterm increment_reg_val;
+    }
+
+The name `increment` is the name of the group that the fragment
+belongs to.  Note that it does not need to have the same
+name as the instruction.  The group name is followed by `.` and
+the name of the fragment.  The name `head` is pre-defined.
+The code in it will be placed at the beginning of a block, so
+that all fragments in the group can access it.
+
+Next we define the fragment that will pick up the value from the
+register from the first operand:
+
+    increment.fetch(Src) {
+        increment_reg_val = $Src;
+    }
+
+We call this fragment `fetch`.  This fragment will be duplicated three
+times, one for each value of the first operand (`r`, `x`, and `y`).
+
+Next we define the main part of the code that do the actual incrementing.
+
+    increment.execute(IncrementVal, Live, Dst) {
+        Eterm increment_val = $IncrementVal;
+        Uint live;
+        Eterm result;
+
+        if (ERTS_LIKELY(is_small(increment_reg_val))) {
+            Sint i = signed_val(increment_reg_val) + increment_val;
+            if (ERTS_LIKELY(IS_SSMALL(i))) {
+                $Dst = make_small(i);
+                $NEXT0();
+            }
+        }
+        live = $Live;
+        HEAVY_SWAPOUT;
+        reg[live] = increment_reg_val;
+        reg[live+1] = make_small(increment_val);
+        result = erts_gc_mixed_plus(c_p, reg, live);
+        HEAVY_SWAPIN;
+        ERTS_HOLE_CHECK(c_p);
+        if (ERTS_LIKELY(is_value(result))) {
+            $REFRESH_GEN_DEST();
+            $Dst = result;
+            $NEXT0();
+        }
+        ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue));
+        goto find_func_info;
+    }
+
+We call this fragment `execute`.  It will handle the three remaining
+operands (`W t d`).  There will only be one copy of this fragment.
+
+Now that we have defined the fragments, we need to inform
+**beam\_makeops** how they should be connected:
+
+    i_increment := increment.fetch.execute;
+
+To the left of the `:=` is the name of the specific instruction that
+should be implemented by the fragments, in this case `i_increment`.
+To the right of `:=` is the name of the group with the fragments,
+followed by a `.`.  Then the name of the fragments in the group are
+listed in the order they should be executed.  Note that the `head`
+fragment is not listed.
+
+The line ends in `;` (to avoid messing up the indentation in Emacs).
+
+(Note that in practice the `:=` line is usually placed before the
+fragments.)
+
+The generated code looks like this:
+
+    {
+      Eterm increment_reg_val;
+      OpCase(i_increment_rWtd):
+      {
+        increment_reg_val = r(0);
+      }
+      goto increment__execute;
+
+      OpCase(i_increment_xWtd):
+      {
+        increment_reg_val = xb(BeamExtraData(I[0]));
+      }
+      goto increment__execute;
+
+      OpCase(i_increment_yWtd):
+      {
+        increment_reg_val = yb(BeamExtraData(I[0]));
+      }
+      goto increment__execute;
+
+      increment__execute:
+      {
+        // Here follows the code from increment.execute()
+        .
+        .
+        .
+    }
+
+##### Some notes about combined instructions #####
+
+The operands that are different must be at
+the beginning of the instruction.  All operands in the last
+fragment must have the same operands in all variants of
+the specific instruction.
+
+As an example, the following specific instructions cannot be
+implemented as a combined instruction:
+
+    i_times j? t x x d
+    i_times j? t x y d
+    i_times j? t s s d
+
+We would have to change the order of the operands so that the
+two operands that are different are placed first:
+
+    i_times x x j? t d
+    i_times x y j? t d
+    i_times s s j? t d
+
+We can then define:
+
+    i_times := times.fetch.execute;
+
+    times.head {
+        Eterm op1, op2;
+    }
+
+    times.fetch(Src1, Src2) {
+        op1 = $Src1;
+        op2 = $Src2;
+    }
+
+    times.execute(Fail, Live, Dst) {
+        // Multiply op1 and op2.
+        .
+        .
+        .
+    }
+
+Several instructions can share a group.  As an example, the following
+instructions have different names, but in the end they all create a
+binary.  The last two operands are common for all of them:
+
+    i_bs_init_fail       xy j? t? x
+    i_bs_init_fail_heap s I j? t? x
+    i_bs_init                W t? x
+    i_bs_init_heap         W I t? x
+
+The instructions are defined like this (formatted with extra
+spaces for clarity):
+
+    i_bs_init_fail_heap := bs_init . fail_heap . verify . execute;
+    i_bs_init_fail      := bs_init . fail      . verify . execute;
+    i_bs_init           := bs_init .           .  plain . execute;
+    i_bs_init_heap      := bs_init .               heap . execute;
+
+Note that the first two instruction have three fragments, while the
+other two only have two fragments.  Here are the fragments:
+
+    bs_init_bits.head() {
+        Eterm num_bits_term;
+        Uint num_bits;
+        Uint alloc;
+    }
+
+    bs_init_bits.plain(NumBits) {
+        num_bits = $NumBits;
+        alloc = 0;
+    }
+
+    bs_init_bits.heap(NumBits, Alloc) {
+        num_bits = $NumBits;
+        alloc = $Alloc;
+    }
+
+    bs_init_bits.fail(NumBitsTerm) {
+        num_bits_term = $NumBitsTerm;
+        alloc = 0;
+    }
+
+    bs_init_bits.fail_heap(NumBitsTerm, Alloc) {
+        num_bits_term = $NumBitsTerm;
+        alloc = $Alloc;
+    }
+
+    bs_init_bits.verify(Fail) {
+        // Verify the num_bits_term, fail using $FAIL
+        // if there is a problem.
+	.
+	.
+	.
+    }
+
+    bs_init_bits.execute(Live, Dst) {
+       // Long complicated code to a create a binary.
+       .
+       .
+       .
+    }
+
+The full definitions of those instructions can be found in `bs_instrs.tab`.
+The generated code can be found in `beam_warm.h`.
-- 
cgit v1.2.3


From b9c13d5cba56d267bfe7b7c9bc3e4f681a53e33a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 13 Nov 2017 15:27:09 +0100
Subject: Remove invalid EINTR loop around close(2)

Retrying close(2) on anything other than HP-UX is likely to close
something entirely different. POSIX says that the state of the file
descriptor is unspecified, and Linux/BSD guarantee that it's closed
on return.
---
 erts/emulator/drivers/unix/unix_efile.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c
index 3ff68a8859..e4fccd954b 100644
--- a/erts/emulator/drivers/unix/unix_efile.c
+++ b/erts/emulator/drivers/unix/unix_efile.c
@@ -463,7 +463,7 @@ efile_may_openfile(Efile_error* errInfo, char *name) {
 void
 efile_closefile(int fd)
 {
-    while((close(fd) < 0) && (errno == EINTR));
+    close(fd);
 }
 
 int
-- 
cgit v1.2.3


From 3679444fb654e9cba1252c6df0be5170e5388639 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 10 Nov 2017 05:09:46 +0100
Subject: Fix broken receive optimization
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When a ref is created before performing a receive that will only
receive message containing that ref, there is a compiler optimization
to avoid scanning messages that can't possible contain the newly
created ref.

Magnus Lång pointed out that the implementation of the optimization
is flawed. Exceptions or recursive calls could cause the receive
operation to scan the receive queue from a position beyond the expected
message (that is, the message containing the ref would never be
matched out). See the receive_opt_exception/1 and receive_opt_recursion/1
test cases in receive_SUITE.

It turns out that we can simplify the implementation of the
optimization while fixing the bug (suggested by Magnus Lång). We
actually don't need the c_p->msg.mark field. It is enough to have
c_p->msg.saved_pos; if it is non-zero, it is a valid position in the
message qeueue.  All we need to do is to ensure that we clear
c_p->msg.saved_pos when a receive is exited normally or abnormally.

We can clear c_p->msg.saved_pos in JOIN_MESSAGE(), since it is called
both when leaving a receive because a message matched and because there
was a timeout and the 'after' clause was executed. In addition, we
need to clear c_p->msg.saved_pos when an exception is caught.

https://bugs.erlang.org/browse/ERL-511
---
 erts/emulator/beam/beam_emu.c        |  1 +
 erts/emulator/beam/erl_message.h     | 16 ++++++----
 erts/emulator/beam/msg_instrs.tab    | 21 ++++++------
 erts/emulator/beam/ops.tab           |  7 +++-
 erts/emulator/test/receive_SUITE.erl | 62 ++++++++++++++++++++++++++++++++++--
 5 files changed, 85 insertions(+), 22 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index bc95ccec52..ef9abcde08 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1454,6 +1454,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, ErtsCodeMFA *bif_mfa)
 	reg[3] = c_p->ftrace;
         if ((new_pc = next_catch(c_p, reg))) {
 	    c_p->cp = 0;	/* To avoid keeping stale references. */
+            c_p->msg.saved_last = 0;  /* No longer safe to use this position */
 	    return new_pc;
 	}
 	if (c_p->catches > 0) erts_exit(ERTS_ERROR_EXIT, "Catch not found");
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 9c8cf84e43..a14f4f51d8 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -167,10 +167,9 @@ typedef struct {
     Sint len;            /* queue length */
 
     /*
-     * The following two fields are used by the recv_mark/1 and
+     * The following field is used by the recv_mark/1 and
      * recv_set/1 instructions.
      */
-    BeamInstr* mark;		/* address to rec_loop/2 instruction */
     ErtsMessage** saved_last;	/* saved last pointer */
 } ErlMessageQueue;
 
@@ -236,12 +235,17 @@ typedef struct erl_trace_message_queue__ {
      (p)->msg.len--; \
      if (__mp == NULL) \
          (p)->msg.last = (p)->msg.save; \
-     (p)->msg.mark = 0; \
 } while(0)
 
-/* Reset message save point (after receive match) */
-#define JOIN_MESSAGE(p) \
-     (p)->msg.save = &(p)->msg.first
+/*
+ * Reset message save point (after receive match).
+ * Also invalidate the saved position since it may no
+ * longer be safe to use.
+ */
+#define JOIN_MESSAGE(p) do {                    \
+    (p)->msg.save = &(p)->msg.first;            \
+    (p)->msg.saved_last = 0;                    \
+} while(0)
 
 /* Save current message */
 #define SAVE_MESSAGE(p) \
diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab
index 8055a8616f..d6d4d2fb49 100644
--- a/erts/emulator/beam/msg_instrs.tab
+++ b/erts/emulator/beam/msg_instrs.tab
@@ -43,27 +43,23 @@
 //  *
 //  */
 
-recv_mark(Dest) {
+i_recv_mark() {
     /*
-     * Save the current position in message buffer and the
-     * the label for the loop_rec/2 instruction for the
-     * the receive statement.
+     * Save the current position in message buffer.
      */
-    $SET_REL_I(c_p->msg.mark, $Dest);
     c_p->msg.saved_last = c_p->msg.last;
 }
 
 i_recv_set() {
     /*
-     * If the mark is valid (points to the loop_rec/2
-     * instruction that follows), we know that the saved
-     * position points to the first message that could
-     * possibly be matched out.
+     * If c_p->msg.saved_last is non-zero, it points to the first
+     * message that could possibly be matched out.
      *
-     * If the mark is invalid, we do nothing, meaning that
-     * we will look through all messages in the message queue.
+     * If c_p->msg.saved_last is zero, it means that it was invalidated
+     * because another receive was executed before this i_recv_set()
+     * instruction was reached.
      */
-    if (c_p->msg.mark == (BeamInstr *) ($NEXT_INSTRUCTION)) {
+    if (c_p->msg.saved_last) {
         c_p->msg.save = c_p->msg.saved_last;
     }
     SET_I($NEXT_INSTRUCTION);
@@ -131,6 +127,7 @@ i_loop_rec(Dest) {
             ASSERT(HTOP == c_p->htop && E == c_p->stop);
             /* TODO: Add DTrace probe for this bad message situation? */
             UNLINK_MESSAGE(c_p, msgp);
+            c_p->msg.saved_last = 0; /* Better safe than sorry. */
             msgp->next = NULL;
             erts_cleanup_messages(msgp);
             goto loop_rec__;
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index a560bde920..3df91056cb 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1565,7 +1565,12 @@ on_load
 #
 # R14A.
 #
-recv_mark f
+# Modified in OTP 21 because it turns out that we don't need the
+# label after all.
+#
+
+recv_mark f => i_recv_mark
+i_recv_mark
 
 recv_set Fail | label Lbl | loop_rec Lf Reg => \
    i_recv_set | label Lbl | loop_rec Lf Reg
diff --git a/erts/emulator/test/receive_SUITE.erl b/erts/emulator/test/receive_SUITE.erl
index a12019ec83..1fe11428b4 100644
--- a/erts/emulator/test/receive_SUITE.erl
+++ b/erts/emulator/test/receive_SUITE.erl
@@ -25,14 +25,16 @@
 -include_lib("common_test/include/ct.hrl").
 
 -export([all/0, suite/0,
-	 call_with_huge_message_queue/1,receive_in_between/1]).
+	 call_with_huge_message_queue/1,receive_in_between/1,
+         receive_opt_exception/1,receive_opt_recursion/1]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
      {timetrap, {minutes, 3}}].
 
-all() -> 
-    [call_with_huge_message_queue, receive_in_between].
+all() ->
+    [call_with_huge_message_queue, receive_in_between,
+     receive_opt_exception, receive_opt_recursion].
 
 call_with_huge_message_queue(Config) when is_list(Config) ->
     Pid = spawn_link(fun echo_loop/0),
@@ -113,6 +115,60 @@ receive_one() ->
 	dummy -> ok
     end.
 
+receive_opt_exception(_Config) ->
+    Recurse = fun() ->
+                      %% Overwrite with the same mark,
+                      %% and never consume it.
+                      ThrowFun = fun() -> throw(aborted) end,
+                      aborted = (catch do_receive_opt_exception(ThrowFun)),
+                      ok
+              end,
+    do_receive_opt_exception(Recurse),
+
+    %% Eat the second message.
+    receive
+        Ref when is_reference(Ref) -> ok
+    end.
+
+do_receive_opt_exception(Disturber) ->
+    %% Create a receive mark.
+    Ref = make_ref(),
+    self() ! Ref,
+    Disturber(),
+    receive
+        Ref ->
+            ok
+    after 0 ->
+            error(the_expected_message_was_not_there)
+    end.
+
+receive_opt_recursion(_Config) ->
+    Recurse = fun() ->
+                      %% Overwrite with the same mark,
+                      %% and never consume it.
+                      NoOp = fun() -> ok end,
+                      BlackHole = spawn(NoOp),
+                      expected = do_receive_opt_recursion(BlackHole, NoOp, true),
+                      ok
+              end,
+    do_receive_opt_recursion(self(), Recurse, false),
+    ok.
+
+do_receive_opt_recursion(Recipient, Disturber, IsInner) ->
+    Ref = make_ref(),
+    Recipient ! Ref,
+    Disturber(),
+    receive
+        Ref -> ok
+    after 0 ->
+            case IsInner of
+                true ->
+                    expected;
+                false ->
+                    error(the_expected_message_was_not_there)
+            end
+    end.
+
 %%%
 %%% Common helpers.
 %%%
-- 
cgit v1.2.3


From 567cdef0b7b0192f0bb9b7a552ce6868795efae0 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 15 Nov 2017 13:27:06 +0100
Subject: erts: Fix NIF debug readonly check of binaries

when done by enif_free_env or enif_clear_env.

Do check before we free heap fragments.
---
 erts/emulator/beam/erl_nif.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 3ade17b10d..f67b67325d 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -550,6 +550,9 @@ void enif_clear_env(ErlNifEnv* env)
     ASSERT(p == menv->env.proc);
     ASSERT(p->common.id == ERTS_INVALID_PID);
     ASSERT(MBUF(p) == menv->env.heap_frag);
+
+    free_tmp_objs(env);
+
     if (MBUF(p) != NULL) {
 	erts_cleanup_offheap(&MSO(p));
 	clear_offheap(&MSO(p));
@@ -561,7 +564,6 @@ void enif_clear_env(ErlNifEnv* env)
     menv->env.hp = menv->env.hp_end = HEAP_TOP(p);
     
     ASSERT(!is_offheap(&MSO(p)));
-    free_tmp_objs(env);
 }
 
 #ifdef ERTS_SMP
-- 
cgit v1.2.3


From afd92e272aa6fa6836e6e5e48e8a68ba519202e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 15 Nov 2017 13:03:29 +0100
Subject: Fix NIF API version for enif_ioq

---
 erts/emulator/beam/erl_nif.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index d195721054..7fb447e4a8 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -50,10 +50,11 @@
 ** 2.9: 18.2 enif_getenv
 ** 2.10: Time API
 ** 2.11: 19.0 enif_snprintf 
-** 2.12: 20.0 add enif_queue
+** 2.12: 20.0 add enif_select, enif_open_resource_type_x
+** 2.13: 20.1 add enif_ioq
 */
 #define ERL_NIF_MAJOR_VERSION 2
-#define ERL_NIF_MINOR_VERSION 12
+#define ERL_NIF_MINOR_VERSION 13
 
 /*
  * The emulator will refuse to load a nif-lib with a major version
-- 
cgit v1.2.3


From 19e8943798f8ef282a8c2b4da26176f07c3652d5 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 15 Nov 2017 19:29:46 +0100
Subject: Fix triggering of node monitors

---
 erts/emulator/beam/dist.c | 34 ++++++++++++++++++----------------
 1 file changed, 18 insertions(+), 16 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 3611eeee02..5566055de0 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -383,22 +383,24 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp)
 	if (!rp)
 	    goto done;
 	erts_smp_proc_lock(rp, rp_locks);
-	rlnk = erts_remove_link(&ERTS_P_LINKS(rp), name);
-	if (rlnk != NULL) {
-	    ASSERT(is_atom(rlnk->pid) && (rlnk->type == LINK_NODE));
-	    erts_destroy_link(rlnk);
-	}
-	n = ERTS_LINK_REFC(lnk);
-	for (i = 0; i < n; ++i) {
-	    Eterm tup;
-	    Eterm *hp;
-	    ErtsMessage *msgp;
-
-	    msgp = erts_alloc_message_heap(rp, &rp_locks,
-					   3, &hp, &ohp);
-	    tup = TUPLE2(hp, am_nodedown, name);
-	    erts_queue_message(rp, rp_locks, msgp, tup, am_system);
-	}
+        if (!ERTS_PROC_IS_EXITING(rp)) {
+            rlnk = erts_remove_link(&ERTS_P_LINKS(rp), name);
+            if (rlnk != NULL) {
+                ASSERT(is_atom(rlnk->pid) && (rlnk->type == LINK_NODE));
+                erts_destroy_link(rlnk);
+            }
+            n = ERTS_LINK_REFC(lnk);
+            for (i = 0; i < n; ++i) {
+                Eterm tup;
+                Eterm *hp;
+                ErtsMessage *msgp;
+
+                msgp = erts_alloc_message_heap(rp, &rp_locks,
+                                               3, &hp, &ohp);
+                tup = TUPLE2(hp, am_nodedown, name);
+                erts_queue_message(rp, rp_locks, msgp, tup, am_system);
+            }
+        }
 	erts_smp_proc_unlock(rp, rp_locks);
     }
  done:
-- 
cgit v1.2.3


From b17db5ae892a8b9c48f8f94f7219c60fe122f629 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 21 Sep 2017 19:21:18 +0200
Subject: erts: Fix bug in DistEntry refc dance

to handle "lookup without refc++" correctly
which was introduced in 4dcb2ae7810a507b701a30072b2f514cab7ebbdb.

When decrementing refc to zero (in try_delete or prepare_try_delete)
we must always wait thread progress to make sure no thread has done
lookup without refc++ and is just about to do refc++ and thereby revive
the DistEntry. That is, we wait for a potential other thread to either
do refc++ or drop its pointer to the DistEntry.

And if that other thread does refc++ (in erts_ref_dist_entry)
it must also do the extra refc++ for the scheduled pending delete.
---
 erts/emulator/beam/erl_node_tables.c        | 103 +++++++++++++++++-----------
 erts/emulator/test/node_container_SUITE.erl |   2 +
 2 files changed, 66 insertions(+), 39 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 0f3dfa797c..1ab096ab69 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -101,7 +101,9 @@ void
 erts_ref_dist_entry(DistEntry *dep)
 {
     ASSERT(dep);
-    de_refc_inc(dep, 1);
+    if (de_refc_inc_read(dep, 1) == 1) {
+        de_refc_inc(dep, 2);  /* Pending delete */
+    }
 }
 
 void
@@ -396,44 +398,77 @@ erts_make_dhandle(Process *c_p, DistEntry *dep)
     return erts_mk_magic_ref(&hp, &c_p->off_heap, bin);
 }
 
-static void try_delete_dist_entry(void *vbin);
+static void start_timer_delete_dist_entry(void *vdep);
+static void prepare_try_delete_dist_entry(void *vdep);
+static void try_delete_dist_entry(DistEntry*);
+
+static void schedule_delete_dist_entry(DistEntry* dep)
+{
+    /*
+     * Here we need thread progress to wait for other threads, that may have
+     * done lookup without refc++, to do either refc++ or drop their refs.
+     *
+     * Note that timeouts do not guarantee thread progress.
+     */
+    erts_schedule_thr_prgr_later_op(start_timer_delete_dist_entry,
+                                    dep, &dep->later_op);
+}
 
 static void
-prepare_try_delete_dist_entry(void *vbin)
+start_timer_delete_dist_entry(void *vdep)
 {
-    Binary *bin = (Binary *) vbin;
-    DistEntry *dep = ErtsBin2DistEntry(bin);
-    Uint size;
+    if (node_tab_delete_delay == 0) {
+        prepare_try_delete_dist_entry(vdep);
+    }
+    else {
+        ASSERT(node_tab_delete_delay > 0);
+        erts_start_timer_callback(node_tab_delete_delay,
+                                  prepare_try_delete_dist_entry,
+                                  vdep);
+    }
+}
+
+static void
+prepare_try_delete_dist_entry(void *vdep)
+{
+    DistEntry *dep = vdep;
     erts_aint_t refc;
 
+    /*
+     * Time has passed since we decremented refc to zero and DistEntry may
+     * have been revived. Do a fast check without table lock first.
+     */
     refc = de_refc_read(dep, 0);
-    if (refc > 0)
-        return;
-
-    size = ERTS_MAGIC_BIN_SIZE(sizeof(DistEntry));
-    erts_schedule_thr_prgr_later_cleanup_op(try_delete_dist_entry,
-                                            vbin, &dep->later_op, size);
+    if (refc == 0) {
+        try_delete_dist_entry(dep);
+    }
+    else {
+        /*
+         * Someone has done lookup and done refc++ for us.
+         */
+        refc = de_refc_dec_read(dep, 0);
+        if (refc == 0)
+            schedule_delete_dist_entry(dep);
+    }
 }
 
-static void try_delete_dist_entry(void *vbin)
+static void try_delete_dist_entry(DistEntry* dep)
 {
-    Binary *bin = (Binary *) vbin;
-    DistEntry *dep = ErtsBin2DistEntry(bin);
     erts_aint_t refc;
 
     erts_rwmtx_rwlock(&erts_dist_table_rwmtx);
     /*
      * Another thread might have looked up this dist entry after
      * we decided to delete it (refc became zero). If so, the other
-     * thread incremented refc twice. Once for the new reference
-     * and once for this thread.
+     * thread incremented refc one extra step for this thread.
      *
-     * If refc reach -1, no one has used the entry since we
-     * set up the timer. Delete the entry.
+     * If refc reach -1, no one has done lookup and no one can do lookup
+     * as we have table lock. Delete the entry.
      *
-     * If refc reach 0, the entry is currently not in use
-     * but has been used since we set up the timer. Set up a
-     * new timer.
+     * If refc reach 0, someone raced us and either
+     *  (1) did lookup with own refc++ and already released it again
+     *  (2) did lookup without own refc++
+     * Schedule new delete operation.
      *
      * If refc > 0, the entry is in use. Keep the entry.
      */
@@ -443,12 +478,7 @@ static void try_delete_dist_entry(void *vbin)
     erts_rwmtx_rwunlock(&erts_dist_table_rwmtx);
 
     if (refc == 0) {
-        if (node_tab_delete_delay == 0)
-            prepare_try_delete_dist_entry(vbin);
-        else if (node_tab_delete_delay > 0)
-            erts_start_timer_callback(node_tab_delete_delay,
-                                      prepare_try_delete_dist_entry,
-                                      vbin);
+        schedule_delete_dist_entry(dep);
     }
 }
 
@@ -462,12 +492,7 @@ int erts_dist_entry_destructor(Binary *bin)
     if (refc == -1)
         return 1; /* Allow deallocation of structure... */
 
-    if (node_tab_delete_delay == 0)
-        prepare_try_delete_dist_entry((void *) bin);
-    else if (node_tab_delete_delay > 0)
-        erts_start_timer_callback(node_tab_delete_delay,
-                                  prepare_try_delete_dist_entry,
-                                  (void *) bin);
+    schedule_delete_dist_entry(dep);
 
     return 0;
 }
@@ -1463,9 +1488,9 @@ insert_delayed_delete_node(void *state,
 }
 
 static void
-insert_thr_prgr_delete_dist_entry(void *arg, ErtsThrPrgrVal thr_prgr, void *vbin)
+insert_thr_prgr_delete_dist_entry(void *arg, ErtsThrPrgrVal thr_prgr, void *vdep)
 {
-    DistEntry *dep = ErtsBin2DistEntry(vbin);
+    DistEntry *dep = vdep;
     Eterm heap[3];
     insert_dist_entry(dep,
 		      SYSTEM_REF,
@@ -1476,9 +1501,9 @@ insert_thr_prgr_delete_dist_entry(void *arg, ErtsThrPrgrVal thr_prgr, void *vbin
 static void
 insert_delayed_delete_dist_entry(void *state,
 				 ErtsMonotonicTime timeout_pos,
-				 void *vbin)
+				 void *vdep)
 {
-    DistEntry *dep = ErtsBin2DistEntry(vbin);
+    DistEntry *dep = vdep;
     Eterm heap[3];
     insert_dist_entry(dep,
 		      SYSTEM_REF,
@@ -1520,7 +1545,7 @@ setup_reference_table(void)
     erts_debug_callback_timer_foreach(prepare_try_delete_dist_entry,
 				      insert_delayed_delete_dist_entry,
 				      NULL);
-    erts_debug_later_op_foreach(try_delete_dist_entry,
+    erts_debug_later_op_foreach(start_timer_delete_dist_entry,
                                 insert_thr_prgr_delete_dist_entry,
                                 NULL);
 
diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl
index be90f929df..7df001fec5 100644
--- a/erts/emulator/test/node_container_SUITE.erl
+++ b/erts/emulator/test/node_container_SUITE.erl
@@ -966,6 +966,8 @@ check_refc(ThisNodeName,ThisCreation,Table,EntryList) when is_list(EntryList) ->
                         {case Referrer of
                              {system,delayed_delete_timer} ->
                                  true;
+                             {system,thread_progress_delete_timer} ->
+                                 true;
                              _ ->
                                  DDT
                          end,
-- 
cgit v1.2.3


From 50f08826065ad29040a6c218addcf32bf12945b1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 22 Sep 2017 16:24:45 +0200
Subject: erts TEST: Add missing ref to DistEntry from send context

---
 erts/emulator/beam/erl_node_tables.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 1ab096ab69..9931686cbe 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1325,6 +1325,11 @@ insert_offheap2(ErlOffHeap *oh, void *arg)
     (((Bin)->intern.flags & BIN_FLAG_MAGIC)                             \
      && ERTS_MAGIC_BIN_DESTRUCTOR((Bin)) == erts_dist_entry_destructor)
 
+#define IsSendCtxBinary(Bin)                                            \
+    (((Bin)->intern.flags & BIN_FLAG_MAGIC)                             \
+     && ERTS_MAGIC_BIN_DESTRUCTOR((Bin)) == erts_dsend_context_dtor)
+
+
 static void
 insert_offheap(ErlOffHeap *oh, int type, Eterm id)
 {
@@ -1361,7 +1366,12 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id)
 		    inserted_bins = nib;
 		    UnUseTmpHeapNoproc(BIG_UINT_HEAP_SIZE);
 		}
-	    }		
+	    }
+            else if (IsSendCtxBinary(u.mref->mb)) {
+                ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(u.mref->mb);
+                if (ctx->deref_dep)
+                    insert_dist_entry(ctx->dep, type, id, 0);
+            }
 	    break;
 	case REFC_BINARY_SUBTAG:
 	case FUN_SUBTAG:
-- 
cgit v1.2.3


From adc71259e87805990ea2a91bc0b5c17861984a47 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 14 Jul 2017 15:08:03 +0200
Subject: erts: Remove obsolete code for latin1 in atom cache

Distribution flag DFLAG_UTF8_ATOMS is supported since R16
and mandatory since 20.0.
---
 erts/emulator/beam/external.c | 49 +++++++++++--------------------------------
 1 file changed, 12 insertions(+), 37 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 6666c42778..9761a69ad1 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -220,23 +220,10 @@ erts_destroy_atom_cache_map(ErtsAtomCacheMap *acmp)
 static ERTS_INLINE void
 insert_acache_map(ErtsAtomCacheMap *acmp, Eterm atom, Uint32 dflags)
 {
-    /*
-     * If the receiver do not understand utf8 atoms
-     * and this atom cannot be represented in latin1,
-     * we are not allowed to cache it.
-     *
-     * In this case all atoms are assumed to have
-     * latin1 encoding in the cache. By refusing it
-     * in the cache we will instead encode it using
-     * ATOM_UTF8_EXT/SMALL_ATOM_UTF8_EXT which the
-     * receiver do not recognize and tear down the
-     * connection.
-     */
-    if (acmp && acmp->sz < ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES
-	&& ((dflags & DFLAG_UTF8_ATOMS)
-	    || atom_tab(atom_val(atom))->latin1_chars >= 0)) {
+    if (acmp && acmp->sz < ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES) {
 	int ix;
 	ASSERT(acmp->hdr_sz < 0);
+        ASSERT(dflags & DFLAG_UTF8_ATOMS);
 	ix = atom2cix(atom);
 	if (acmp->cache[ix].iix < 0) {
 	    acmp->cache[ix].iix = acmp->sz;
@@ -256,9 +243,7 @@ get_iix_acache_map(ErtsAtomCacheMap *acmp, Eterm atom, Uint32 dflags)
 	ASSERT(is_atom(atom));
 	ix = atom2cix(atom);
 	if (acmp->cache[ix].iix < 0) {
-	    ASSERT(acmp->sz == ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES
-		   || (!(dflags & DFLAG_UTF8_ATOMS)
-		       && atom_tab(atom_val(atom))->latin1_chars < 0));
+	    ASSERT(acmp->sz == ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES);
 	    return -1;
 	}
 	else {
@@ -272,7 +257,6 @@ void
 erts_finalize_atom_cache_map(ErtsAtomCacheMap *acmp, Uint32 dflags)
 {
     if (acmp) {
-	int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS);
 	int long_atoms = 0; /* !0 if one or more atoms are longer than 255. */
 	int i;
 	int sz;
@@ -283,6 +267,7 @@ erts_finalize_atom_cache_map(ErtsAtomCacheMap *acmp, Uint32 dflags)
 	    + 1 /* number of internal cache entries */
 	    ;
 	int min_sz;
+        ASSERT(dflags & DFLAG_UTF8_ATOMS);
 	ASSERT(acmp->hdr_sz < 0);
 	/* Make sure cache update instructions fit */
 	min_sz = fix_sz+(2+4)*acmp->sz;
@@ -294,7 +279,7 @@ erts_finalize_atom_cache_map(ErtsAtomCacheMap *acmp, Uint32 dflags)
 	    atom = acmp->cache[acmp->cix[i]].atom;
 	    ASSERT(is_atom(atom));
 	    a = atom_tab(atom_val(atom));
-	    len = (int) (utf8_atoms ? a->len : a->latin1_chars);
+	    len = (int) a->len;
 	    ASSERT(len >= 0);
 	    if (!long_atoms && len > 255)
 		long_atoms = 1;
@@ -371,8 +356,8 @@ byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache, Uint
     int ci, sz;
     byte dist_hdr_flags;
     int long_atoms;
-    int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS);
     register byte *ep = ext;
+    ASSERT(dflags & DFLAG_UTF8_ATOMS);
     ASSERT(ep[0] == VERSION_MAGIC);
     if (ep[1] != DIST_HEADER)
 	return ext;
@@ -445,17 +430,9 @@ byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache, Uint
 		Atom *a;
 		cache->out_arr[cix] = atom;
 		a = atom_tab(atom_val(atom));
-		if (utf8_atoms) {
-		    sz = a->len;
-		    ep -= sz;
-		    sys_memcpy((void *) ep, (void *) a->name, sz);
-		}
-		else {
-		    ASSERT(0 <= a->latin1_chars && a->latin1_chars <= MAX_ATOM_CHARACTERS);
-		    ep -= a->latin1_chars;
-		    sz = erts_utf8_to_latin1(ep, a->name, a->len);
-		    ASSERT(a->latin1_chars == sz);
-		}
+                sz = a->len;
+                ep -= sz;
+                sys_memcpy((void *) ep, (void *) a->name, sz);
 		if (long_atoms) {
 		    ep -= 2;
 		    put_int16(sz, ep);
@@ -641,7 +618,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
 #endif
 
     register byte *ep = ext;
-    int utf8_atoms = (int) (dep->flags & DFLAG_UTF8_ATOMS);
+    ASSERT(dep->flags & DFLAG_UTF8_ATOMS);
 
     edep->heap_size = -1;
     edep->ext_endp = ext+size;
@@ -804,9 +781,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
 		    CHKSIZE(len);
 		    atom = erts_atom_put((byte *) ep,
 					 len,
-					 (utf8_atoms
-					  ? ERTS_ATOM_ENC_UTF8
-					  : ERTS_ATOM_ENC_LATIN1),
+                                         ERTS_ATOM_ENC_UTF8,
 					 0);
 		    if (is_non_value(atom))
 			ERTS_EXT_HDR_FAIL;
@@ -2135,7 +2110,7 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags)
 {
     int iix;
     int len;
-    int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS);
+    const int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS);
 
     ASSERT(is_atom(atom));
 
-- 
cgit v1.2.3


From 8f3cb95bb60db55728bfea339b94ebc164b4dbdb Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 14 Jul 2017 19:35:26 +0200
Subject: erts: Remove some dead code

---
 erts/emulator/beam/erl_node_tables.h |  3 ---
 erts/emulator/beam/external.h        | 22 ----------------------
 2 files changed, 25 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index ee8277b5ea..2cfd18f22f 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -60,9 +60,6 @@
 #define ERTS_DE_SFLG_CONNECTED			(((Uint32) 1) <<  0)
 #define ERTS_DE_SFLG_EXITING			(((Uint32) 1) <<  1)
 
-#define ERTS_DE_SFLGS_ALL			(ERTS_DE_SFLG_CONNECTED \
-						 | ERTS_DE_SFLG_EXITING)
-
 #define ERTS_DE_QFLG_BUSY			(((erts_aint32_t) 1) <<  0)
 #define ERTS_DE_QFLG_EXIT			(((erts_aint32_t) 1) <<  1)
 #define ERTS_DE_QFLG_REQ_INFO			(((erts_aint32_t) 1) <<  2)
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index 3c61d013da..fb7eec1a31 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -133,11 +133,6 @@ typedef struct {
     ErtsAtomTranslationTable attab;
 } ErtsDistExternal;
 
-typedef struct {
-    int have_header;
-    int cache_entries;
-} ErtsDistHeaderPeek;
-
 #define ERTS_DIST_EXT_SIZE(EDEP) \
   (sizeof(ErtsDistExternal) \
    - (((EDEP)->flags & ERTS_DIST_EXT_ATOM_TRANS_TAB) \
@@ -177,9 +172,6 @@ Uint erts_encode_ext_size_ets(Eterm);
 void erts_encode_ext(Eterm, byte **);
 byte* erts_encode_ext_ets(Eterm, byte *, struct erl_off_heap_header** ext_off_heap);
 
-#ifdef ERTS_WANT_EXTERNAL_TAGS
-ERTS_GLB_INLINE void erts_peek_dist_header(ErtsDistHeaderPeek *, byte *, Uint);
-#endif
 ERTS_GLB_INLINE void erts_free_dist_ext_copy(ErtsDistExternal *);
 ERTS_GLB_INLINE void *erts_dist_ext_trailer(ErtsDistExternal *);
 ErtsDistExternal *erts_make_dist_ext_copy(ErtsDistExternal *, Uint);
@@ -210,20 +202,6 @@ int erts_debug_atom_to_out_cache_index(Eterm);
 
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
-#ifdef ERTS_WANT_EXTERNAL_TAGS
-ERTS_GLB_INLINE void
-erts_peek_dist_header(ErtsDistHeaderPeek *dhpp, byte *ext, Uint sz)
-{
-    if (ext[0] == VERSION_MAGIC
-	|| ext[1] != DIST_HEADER
-	|| sz < (1+1+1))
-	dhpp->have_header = 0;
-    else {
-	dhpp->have_header = 1;
-	dhpp->cache_entries = (int) get_int8(&ext[2]);
-    }
-}
-#endif
 
 ERTS_GLB_INLINE void
 erts_free_dist_ext_copy(ErtsDistExternal *edep)
-- 
cgit v1.2.3


From 1b8515868ee4cadd7e93b281cea8e59078c27be8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 14 Aug 2017 19:38:21 +0200
Subject: erts: Change to #ifndef from #if !defined

---
 erts/emulator/beam/erl_thr_progress.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h
index fa936b5707..8c029bcf99 100644
--- a/erts/emulator/beam/erl_thr_progress.h
+++ b/erts/emulator/beam/erl_thr_progress.h
@@ -28,7 +28,7 @@
  * Author: 	Rickard Green
  */
 
-#if !defined(ERL_THR_PROGRESS_H__TSD_TYPE__)
+#ifndef ERL_THR_PROGRESS_H__TSD_TYPE__
 #define ERL_THR_PROGRESS_H__TSD_TYPE__
 
 #include "sys.h"
-- 
cgit v1.2.3


From 8fa8ec018cf7d6ff6f4e1711fdb71f34fcea6db1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 16 Aug 2017 20:14:26 +0200
Subject: erts: Refactoring in distribution_SUITE

for set/get_internal_state calls.
---
 erts/emulator/test/distribution_SUITE.erl | 44 +++++++++++++++++++------------
 1 file changed, 27 insertions(+), 17 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index e2914cbc92..c7d63f9feb 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -1158,8 +1158,6 @@ contended_atom_cache_entry_test(Config, Type) ->
     spawn_link(
       SNode,
       fun () ->
-              erts_debug:set_internal_state(available_internal_state,
-                                            true),
               Master = self(),
               CIX = get_cix(),
               TestAtoms = case Type of
@@ -1244,7 +1242,7 @@ get_cix(CIX) when is_integer(CIX), CIX < 0 ->
 get_cix(CIX) when is_integer(CIX) ->
     get_cix(CIX,
             unwanted_cixs(),
-            erts_debug:get_internal_state(max_atom_out_cache_index)).
+            get_internal_state(max_atom_out_cache_index)).
 
 get_cix(CIX, Unwanted, MaxCIX) when CIX > MaxCIX ->
     get_cix(0, Unwanted, MaxCIX);
@@ -1256,8 +1254,8 @@ get_cix(CIX, Unwanted, MaxCIX) ->
 
 unwanted_cixs() ->
     lists:map(fun (Node) ->
-                      erts_debug:get_internal_state({atom_out_cache_index,
-                                                     Node})
+                      get_internal_state({atom_out_cache_index,
+                                          Node})
               end,
               nodes()).
 
@@ -1266,7 +1264,7 @@ get_conflicting_atoms(_CIX, 0) ->
     [];
 get_conflicting_atoms(CIX, N) ->
     Atom = list_to_atom("atom" ++ integer_to_list(erlang:unique_integer([positive]))),
-    case erts_debug:get_internal_state({atom_out_cache_index, Atom}) of
+    case get_internal_state({atom_out_cache_index, Atom}) of
         CIX ->
             [Atom|get_conflicting_atoms(CIX, N-1)];
         _ ->
@@ -1277,7 +1275,7 @@ get_conflicting_unicode_atoms(_CIX, 0) ->
     [];
 get_conflicting_unicode_atoms(CIX, N) ->
     Atom = string_to_atom([16#1f608] ++ "atom" ++ integer_to_list(erlang:unique_integer([positive]))),
-    case erts_debug:get_internal_state({atom_out_cache_index, Atom}) of
+    case get_internal_state({atom_out_cache_index, Atom}) of
         CIX ->
             [Atom|get_conflicting_unicode_atoms(CIX, N-1)];
         _ ->
@@ -1885,11 +1883,26 @@ send_bad_dhdr(BadNode, ToNode) when is_atom(BadNode), is_atom(ToNode) ->
     receive Done -> ok end.
 
 dctrl(Node) when is_atom(Node) ->
-    case catch erts_debug:get_internal_state(available_internal_state) of
-        true -> true;
-        _ -> erts_debug:set_internal_state(available_internal_state, true)
-    end,
-    erts_debug:get_internal_state({dist_ctrl, Node}).
+    get_internal_state({dist_ctrl, Node}).
+
+get_internal_state(Op) ->
+    try erts_debug:get_internal_state(Op) of
+        R -> R
+    catch
+        error:undef ->
+            erts_debug:set_internal_state(available_internal_state, true),
+            erts_debug:get_internal_state(Op)
+    end.
+
+set_internal_state(Op, Val) ->
+    try erts_debug:set_internal_state(Op, Val) of
+        R -> R
+    catch
+        error:undef ->
+            erts_debug:set_internal_state(available_internal_state, true),
+            erts_debug:set_internal_state(Op, Val)
+    end.
+
 
 dmsg_hdr() ->
     [131, % Version Magic
@@ -2032,11 +2045,9 @@ freeze_node(Node, MS) ->
     Freezer = self(),
     spawn_link(Node,
                fun () ->
-                       erts_debug:set_internal_state(available_internal_state,
-                                                     true),
                        dctrl_dop_send(Freezer, DoingIt),
                        receive after Own -> ok end,
-                       erts_debug:set_internal_state(block, MS+Own)
+                       set_internal_state(block, MS+Own)
                end),
     receive DoingIt -> ok end,
     receive after Own -> ok end.
@@ -2240,8 +2251,7 @@ forever(Fun) ->
     forever(Fun).
 
 abort(Why) ->
-    erts_debug:set_internal_state(available_internal_state, true),
-    erts_debug:set_internal_state(abort, Why).
+    set_internal_state(abort, Why).
 
 
 start_busy_dist_port_tracer() ->
-- 
cgit v1.2.3


From 92d383cac5ddcb3856432e653ef84b0a3f84cc0a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 22 Aug 2017 16:48:33 +0200
Subject: erts: Make DFLAGS_NEW_FUN_TAGS mandatory

and remove ugly encoding fallback as {fun, ...}

DFLAGS_NEW_FUN_TAGS has been supported by vm/erl_interface/jinterface
since R13 or even older.

Renamed test case obsolete_funs to term2bin_tuple_fallbacks
and removed test for {fun,...} fallback
and added missing test for bitstring fallback {Binary, Bits}.
---
 erts/emulator/beam/erl_bif_info.c   |  8 ++--
 erts/emulator/beam/external.c       | 91 ++++++++++---------------------------
 erts/emulator/test/binary_SUITE.erl | 51 +++++++++------------
 3 files changed, 49 insertions(+), 101 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 27bbf70c0b..9d05680723 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3830,10 +3830,10 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 		    BIF_RET(res);
 		}
 	    }
-	    else if (ERTS_IS_ATOM_STR("term_to_binary_no_funs", tp[1])) {
-		Uint dflags = (DFLAG_EXTENDED_REFERENCES |
-			       DFLAG_EXTENDED_PIDS_PORTS |
-			       DFLAG_BIT_BINARIES);
+	    else if (ERTS_IS_ATOM_STR("term_to_binary_tuple_fallbacks", tp[1])) {
+		Uint dflags = (TERM_TO_BINARY_DFLAGS
+                               & ~DFLAG_EXPORT_PTR_TAG
+                               & ~DFLAG_BIT_BINARIES);
 		BIF_RET(erts_term_to_binary(BIF_P, tp[2], 0, dflags));
 	    }
 	    else if (ERTS_IS_ATOM_STR("dist_ctrl", tp[1])) {
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 9761a69ad1..9230e1d8ab 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2878,55 +2878,24 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		ErlFunThing* funp = (ErlFunThing *) fun_val(obj);
 		int ei;
 
-		if ((dflags & DFLAG_NEW_FUN_TAGS) != 0) {
-		    *ep++ = NEW_FUN_EXT;
-		    WSTACK_PUSH2(s, ENC_PATCH_FUN_SIZE,
-				 (UWord) ep); /* Position for patching in size */
-		    ep += 4;
-		    *ep = funp->arity;
-		    ep += 1;
-		    sys_memcpy(ep, funp->fe->uniq, 16);
-		    ep += 16;
-		    put_int32(funp->fe->index, ep);
-		    ep += 4;
-		    put_int32(funp->num_free, ep);
-		    ep += 4;
-		    ep = enc_atom(acmp, funp->fe->module, ep, dflags);
-		    ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags, off_heap);
-		    ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags, off_heap);
-		    ep = enc_pid(acmp, funp->creator, ep, dflags);
-		} else {
-		    /*
-		     * Communicating with an obsolete erl_interface or
-		     * jinterface node. Convert the fun to a tuple to
-		     * avoid crasching.
-		     */
-		
-		    /* Tag, arity */
-		    *ep++ = SMALL_TUPLE_EXT;
-		    put_int8(5, ep);
-		    ep += 1;
-		
-		    /* 'fun' */
-		    ep = enc_atom(acmp, am_fun, ep, dflags);
-		
-		    /* Module name */
-		    ep = enc_atom(acmp, funp->fe->module, ep, dflags);
-		
-		    /* Index, Uniq */
-		    *ep++ = INTEGER_EXT;
-		    put_int32(funp->fe->old_index, ep);
-		    ep += 4;
-		    *ep++ = INTEGER_EXT;
-		    put_int32(funp->fe->old_uniq, ep);
-		    ep += 4;
-		
-		    /* Environment sub-tuple arity */
-		    ASSERT(funp->num_free < MAX_ARG);
-		    *ep++ = SMALL_TUPLE_EXT;
-		    put_int8(funp->num_free, ep);
-		    ep += 1;
-		}
+		ASSERT(dflags & DFLAG_NEW_FUN_TAGS);
+                *ep++ = NEW_FUN_EXT;
+                WSTACK_PUSH2(s, ENC_PATCH_FUN_SIZE,
+                             (UWord) ep); /* Position for patching in size */
+                ep += 4;
+                *ep = funp->arity;
+                ep += 1;
+                sys_memcpy(ep, funp->fe->uniq, 16);
+                ep += 16;
+                put_int32(funp->fe->index, ep);
+                ep += 4;
+                put_int32(funp->num_free, ep);
+                ep += 4;
+                ep = enc_atom(acmp, funp->fe->module, ep, dflags);
+                ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags, off_heap);
+                ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags, off_heap);
+                ep = enc_pid(acmp, funp->creator, ep, dflags);
+
 		for (ei = funp->num_free-1; ei > 0; ei--) {
 		    WSTACK_PUSH2(s, ENC_TERM, (UWord) funp->env[ei]);
 		}
@@ -4281,24 +4250,12 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
 	    {
 		ErlFunThing* funp = (ErlFunThing *) fun_val(obj);
 		
-		if ((dflags & DFLAG_NEW_FUN_TAGS) != 0) {
-		    result += 20+1+1+4;	/* New ID + Tag */
-		    result += 4; /* Length field (number of free variables */
-		    result += encode_size_struct2(acmp, funp->creator, dflags);
-		    result += encode_size_struct2(acmp, funp->fe->module, dflags);
-		    result += 2 * (1+4);	/* Index, Uniq */
-		} else {
-		    /*
-		     * Size when fun is mapped to a tuple.
-		     */
-		    result += 1 + 1; /* Tuple tag, arity */
-		    result += 1 + 1 + 2 +
-			atom_tab(atom_val(am_fun))->len; /* 'fun' */
-		    result += 1 + 1 + 2 +
-			atom_tab(atom_val(funp->fe->module))->len; /* Module name */
-		    result += 2 * (1 + 4); /* Index + Uniq */
-		    result += 1 + (funp->num_free < 0x100 ? 1 : 4);
-		}
+                ASSERT(dflags & DFLAG_NEW_FUN_TAGS);
+                result += 20+1+1+4;	/* New ID + Tag */
+                result += 4; /* Length field (number of free variables */
+                result += encode_size_struct2(acmp, funp->creator, dflags);
+                result += encode_size_struct2(acmp, funp->fe->module, dflags);
+                result += 2 * (1+4);	/* Index, Uniq */
 		if (funp->num_free > 1) {
 		    WSTACK_PUSH2(s, (UWord) (funp->env + 1),
 				    (UWord) TERM_ARRAY_OP(funp->num_free-1));
diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index cbc2d8fae5..b59c22f125 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -58,7 +58,9 @@
 	 otp_5484/1,otp_5933/1,
 	 ordering/1,unaligned_order/1,gc_test/1,
 	 bit_sized_binary_sizes/1,
-	 otp_6817/1,deep/1,obsolete_funs/1,robustness/1,otp_8117/1,
+	 otp_6817/1,deep/1,
+         term2bin_tuple_fallbacks/1,
+         robustness/1,otp_8117/1,
 	 otp_8180/1, trapping/1, large/1,
 	 error_after_yield/1, cmp_old_impl/1]).
 
@@ -79,7 +81,8 @@ all() ->
      bad_term_to_binary, more_bad_terms, otp_5484, otp_5933,
      ordering, unaligned_order, gc_test,
      bit_sized_binary_sizes, otp_6817, otp_8117, deep,
-     obsolete_funs, robustness, otp_8180, trapping, large,
+     term2bin_tuple_fallbacks,
+     robustness, otp_8180, trapping, large,
      error_after_yield, cmp_old_impl].
 
 groups() -> 
@@ -1300,40 +1303,28 @@ deep_roundtrip(T) ->
     B = term_to_binary(T),
     T = binary_to_term(B).
 
-obsolete_funs(Config) when is_list(Config) ->
+term2bin_tuple_fallbacks(Config) when is_list(Config) ->
     erts_debug:set_internal_state(available_internal_state, true),
 
-    X = id({1,2,3}),
-    Y = id([a,b,c,d]),
-    Z = id({x,y,z}),
-    obsolete_fun(fun() -> ok end),
-    obsolete_fun(fun() -> X end),
-    obsolete_fun(fun(A) -> {A,X} end),
-    obsolete_fun(fun() -> {X,Y} end),
-    obsolete_fun(fun() -> {X,Y,Z} end),
-
-    obsolete_fun(fun ?MODULE:all/1),
+    term2bin_tf(fun ?MODULE:all/1),
+    term2bin_tf(<<1:1>>),
+    term2bin_tf(<<90,80:7>>),
 
     erts_debug:set_internal_state(available_internal_state, false),
     ok.
 
-obsolete_fun(Fun) ->
-    Tuple = case erlang:fun_info(Fun, type) of
-		{type,external} ->
-		    {module,M} = erlang:fun_info(Fun, module),
-		    {name,F} = erlang:fun_info(Fun, name),
-		    {M,F};
-		{type,local} ->
-		    {module,M} = erlang:fun_info(Fun, module),
-		    {index,I} = erlang:fun_info(Fun, index),
-		    {uniq,U} = erlang:fun_info(Fun, uniq),
-		    {env,E} = erlang:fun_info(Fun, env),
-		    {'fun',M,I,U,list_to_tuple(E)}
-	    end,
-    Tuple = no_fun_roundtrip(Fun).
-
-no_fun_roundtrip(Term) ->
-    binary_to_term_stress(erts_debug:get_internal_state({term_to_binary_no_funs,Term})).
+term2bin_tf(Term) ->
+    Tuple = case Term of
+                Fun when is_function(Fun) ->
+                    {type, external} = erlang:fun_info(Fun, type),
+                    {module,M} = erlang:fun_info(Fun, module),
+                    {name,F} = erlang:fun_info(Fun, name),
+                    {M,F};
+                BS when bit_size(BS) rem 8 =/= 0 ->
+                    Bits = bit_size(BS) rem 8,
+                    {<<BS/bitstring, 0:(8-Bits)>>, Bits}
+            end,
+    Tuple = binary_to_term_stress(erts_debug:get_internal_state({term_to_binary_tuple_fallbacks,Term})).
 
 %% Test non-standard encodings never generated by term_to_binary/1
 %% but recognized by binary_to_term/1.
-- 
cgit v1.2.3


From fe720f6b2051c9bf8ff303f857c3db0a84b1c050 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 18 Sep 2017 17:15:19 +0200
Subject: erts: Refactor connection_id in ErtsDistExternal

Break out from 'flags' into new dedicated 'connection_id'
just for simplicity.

Also changed flags to low bits
and that affected enif_binary_to_term.
---
 erts/emulator/beam/erl_nif.c  | 11 ++++++-----
 erts/emulator/beam/external.c |  4 ++--
 erts/emulator/beam/external.h | 22 +++++++++-------------
 3 files changed, 17 insertions(+), 20 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index c21b139cfa..d1018bab26 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1201,11 +1201,12 @@ size_t enif_binary_to_term(ErlNifEnv *dst_env,
     Sint size;
     ErtsHeapFactory factory;
     byte *bp = (byte*) data;
+    Uint32 flags = 0;
 
-    ERTS_CT_ASSERT(ERL_NIF_BIN2TERM_SAFE == ERTS_DIST_EXT_BTT_SAFE);
-
-    if (opts & ~ERL_NIF_BIN2TERM_SAFE) {
-        return 0;
+    switch ((Uint32)opts) {
+    case 0: break;
+    case ERL_NIF_BIN2TERM_SAFE: flags = ERTS_DIST_EXT_BTT_SAFE; break;
+    default: return 0;
     }
     if ((size = erts_decode_ext_size(bp, data_sz)) < 0)
         return 0;
@@ -1217,7 +1218,7 @@ size_t enif_binary_to_term(ErlNifEnv *dst_env,
         erts_factory_dummy_init(&factory);
     }
 
-    *term = erts_decode_ext(&factory, &bp, (Uint32)opts);
+    *term = erts_decode_ext(&factory, &bp, flags);
 
     if (is_non_value(*term)) {
         return 0;
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 9230e1d8ab..3b851087d1 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -654,7 +654,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
         edep->flags |= ERTS_DIST_EXT_DFLAG_HDR;
 
     *connection_id = dep->connection_id;
-    edep->flags |= (dep->connection_id & ERTS_DIST_EXT_CON_ID_MASK);
+    edep->connection_id = dep->connection_id;
 
     if (ep[1] != DIST_HEADER) {
 	if (edep->flags & ERTS_DIST_EXT_DFLAG_HDR)
@@ -868,7 +868,7 @@ bad_dist_ext(ErtsDistExternal *edep)
 		erts_dsprintf(dsbufp, ", %d=%T", i, edep->attab.atom[i]);
 	}
 	erts_send_warning_to_logger_nogl(dsbufp);
-	erts_kill_dist_connection(dep, ERTS_DIST_EXT_CON_ID(edep));
+	erts_kill_dist_connection(dep, edep->connection_id);
     }
 }
 
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index fb7eec1a31..d6416edbc3 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -109,26 +109,22 @@ typedef struct {
 } ErtsAtomTranslationTable;
 
 /*
- * These flags are tagged onto the high bits of a connection ID and stored in
- * the ErtsDistExternal structure's flags field.  They are used to indicate
- * various bits of state necessary to decode binaries in a variety of
- * scenarios. The mask ERTS_DIST_EXT_CON_ID_MASK is used later to separate the
- * connection ID from the flags. Be careful to ensure that the mask does not
- * overlap any of the bits used for flags, or ERTS will leak flags bits into
- * connection IDs and leak connection ID bits into the flags.
+ * These flags are stored in the ErtsDistExternal structure's flags field.
+ * They are used to indicate various bits of state necessary to decode binaries
+ * in a variety of scenarios.
  */
-#define ERTS_DIST_EXT_DFLAG_HDR      ((Uint32) 0x80000000)
-#define ERTS_DIST_EXT_ATOM_TRANS_TAB ((Uint32) 0x40000000)
-#define ERTS_DIST_EXT_BTT_SAFE       ((Uint32) 0x20000000)
-#define ERTS_DIST_EXT_CON_ID_MASK    ((Uint32) 0x1fffffff)
+#define ERTS_DIST_EXT_DFLAG_HDR      ((Uint32) 0x1)
+#define ERTS_DIST_EXT_ATOM_TRANS_TAB ((Uint32) 0x2)
+#define ERTS_DIST_EXT_BTT_SAFE       ((Uint32) 0x4)
+
+#define ERTS_DIST_CON_ID_MASK    ((Uint32) 0x00ffffff)
 
-#define ERTS_DIST_EXT_CON_ID(DIST_EXTP) \
-  ((DIST_EXTP)->flags & ERTS_DIST_EXT_CON_ID_MASK)
 typedef struct {
     DistEntry *dep;
     byte *extp;
     byte *ext_endp;
     Sint heap_size;
+    Uint32 connection_id;
     Uint32 flags;
     ErtsAtomTranslationTable attab;
 } ErtsDistExternal;
-- 
cgit v1.2.3


From f89fb92384280e2939414287a2ecb8f86a199318 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 14 Jul 2017 19:34:54 +0200
Subject: erts: Introduce asynchronous auto-connect

---
 erts/emulator/beam/atom.names        |   1 +
 erts/emulator/beam/bif.c             | 121 ++++++++++------
 erts/emulator/beam/bif.tab           |   8 +
 erts/emulator/beam/dist.c            | 274 ++++++++++++++++++++++++++---------
 erts/emulator/beam/dist.h            | 122 +++++++++++++---
 erts/emulator/beam/erl_node_tables.c |  20 ++-
 erts/emulator/beam/erl_node_tables.h |   7 +-
 erts/emulator/beam/erl_process.c     |  40 +++--
 erts/emulator/beam/external.c        |  41 +++++-
 erts/emulator/beam/external.h        |   2 +-
 erts/emulator/beam/io.c              |   3 +-
 11 files changed, 474 insertions(+), 165 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index fc55b687d4..a534dd44fb 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -108,6 +108,7 @@ atom asynchronous
 atom atom
 atom atom_used
 atom attributes
+atom auto_connect
 atom await_microstate_accounting_modifications
 atom await_port_send_result
 atom await_proc_exit
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 4b11884f38..34f5afae78 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -227,17 +227,40 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
 		goto res_no_proc;
 	    }
 
-	    code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_DSP_RLOCK, 0);
+	    code = erts_dsig_prepare(&dsd, &dep, BIF_P,
+				     (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK),
+				     ERTS_DSP_RLOCK, 0, 1);
 	    switch (code) {
 	    case ERTS_DSIG_PREP_NOT_ALIVE:
-		/* Let the dlink trap handle it */
-	    case ERTS_DSIG_PREP_NOT_CONNECTED:
-		erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
-		BIF_TRAP1(dlink_trap, BIF_P, BIF_ARG_1);
-
+	    case ERTS_DSIG_PREP_NOT_CONNECTED: {
+		ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK;
+		erts_aint32_t state;
+		erts_proc_lock(BIF_P, (ERTS_PROC_LOCKS_ALL & ~locks));
+		locks = ERTS_PROC_LOCKS_ALL;
+		erts_send_exit_signal(BIF_P, BIF_ARG_1, BIF_P, &locks,
+				      am_noconnection, NIL, NULL, 0);
+		erts_proc_unlock(BIF_P, locks & ERTS_PROC_LOCKS_ALL_MINOR);
+
+		/*
+		 * Copy-paste from old dist_exit_3, not sure if we really
+		 * need erts_handle_pending_exit when exit_2 does not.
+		 */
+		state = erts_atomic32_read_acqb(&BIF_P->state);
+		if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
+#ifdef ERTS_SMP
+		    if (state & ERTS_PSFLG_PENDING_EXIT)
+			erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
+#endif
+		    ERTS_BIF_EXITED(BIF_P);
+		}
+		BIF_RET(am_true);
+	    }
+            case ERTS_DSIG_PREP_PENDING:
 	    case ERTS_DSIG_PREP_CONNECTED:
-		/* We are connected. Setup link and send link signal */
-
+                /*
+                 * We have (pending) connection.
+                 * Setup link and enqueue link signal.
+                 */
 		erts_de_links_lock(dep);
 
 		erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, BIF_ARG_1);
@@ -256,8 +279,7 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
 		    ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
 		BIF_RET(am_true);
 	    default:
-		ASSERT(! "Invalid dsig prepare result");
-		BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
+		ERTS_ASSERT(! "Invalid dsig prepare result");
 	    }
 	}
     }
@@ -292,7 +314,8 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
     ERTS_LC_ASSERT((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK)
 		       == erts_proc_lc_my_proc_locks(c_p));
 
-    code = erts_dsig_prepare(&dsd, dep, c_p, ERTS_DSP_RLOCK, 0);
+    code = erts_dsig_prepare(&dsd, &dep, c_p, ERTS_PROC_LOCK_MAIN,
+			     ERTS_DSP_RLOCK, 0, 0);
     switch (code) {
     case ERTS_DSIG_PREP_NOT_ALIVE:
     case ERTS_DSIG_PREP_NOT_CONNECTED:
@@ -313,6 +336,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
         res = am_true;
 	break;
 
+    case ERTS_DSIG_PREP_PENDING:
     case ERTS_DSIG_PREP_CONNECTED:
 
 	erts_de_links_lock(dep);
@@ -347,8 +371,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
 	}
 	break;
     default:
-	ASSERT(! "Invalid dsig prepare result");
-        return am_internal_error;
+	ERTS_ASSERT(! "Invalid dsig prepare result");
     }
 
 
@@ -768,10 +791,18 @@ remote_monitor(Process *p, Eterm bifarg1, Eterm bifarg2,
     int code;
 
     erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
-    code = erts_dsig_prepare(&dsd, dep, p, ERTS_DSP_RLOCK, 0);
+    code = erts_dsig_prepare(&dsd, &dep,
+			     p, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK),
+			     ERTS_DSP_RLOCK, 0, 0);
     switch (code) {
+    case ERTS_DSIG_PREP_PENDING:
+	/*
+	 * Must wait for connection to know if node supports monitor.
+	 * Damn these synchronous errors.
+	 */
+	erts_smp_de_runlock(dep);
+	/* fall through */
     case ERTS_DSIG_PREP_NOT_ALIVE:
-	/* Let the dmonitor_p trap handle it */
     case ERTS_DSIG_PREP_NOT_CONNECTED:
 	erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 	ERTS_BIF_PREP_TRAP2(ret, dmonitor_p_trap, p, bifarg1, bifarg2);
@@ -818,9 +849,7 @@ remote_monitor(Process *p, Eterm bifarg1, Eterm bifarg2,
 	}
 	break;
     default:
-	ASSERT(! "Invalid dsig prepare result");
-	ERTS_BIF_PREP_ERROR(ret, p, EXC_INTERNAL_ERROR);
-	break;
+	ERTS_ASSERT(! "Invalid dsig prepare result");
     }
 
     BIF_RET(ret);
@@ -1158,7 +1187,8 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 	    BIF_RET(am_true);
 	}
 
-	code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_DSP_NO_LOCK, 0);
+	code = erts_dsig_prepare(&dsd, &dep, BIF_P, ERTS_PROC_LOCK_MAIN,
+				 ERTS_DSP_NO_LOCK, 0, 0);
 	switch (code) {
 	case ERTS_DSIG_PREP_NOT_ALIVE:
 	case ERTS_DSIG_PREP_NOT_CONNECTED:
@@ -1173,6 +1203,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 	    BIF_TRAP1(dunlink_trap, BIF_P, BIF_ARG_1);
 #endif
 
+	case ERTS_DSIG_PREP_PENDING:
 	case ERTS_DSIG_PREP_CONNECTED:
 	    erts_remove_dist_link(&dld, BIF_P->common.id, BIF_ARG_1, dep);
 	    code = erts_dsig_send_unlink(&dsd, BIF_P->common.id, BIF_ARG_1);
@@ -1545,22 +1576,24 @@ BIF_RETTYPE exit_2(BIF_ALIST_2)
 	 DistEntry *dep;
 
 	 dep = external_pid_dist_entry(BIF_ARG_1);
+	 ERTS_ASSERT(dep);
 	 if(dep == erts_this_dist_entry)
 	     BIF_RET(am_true);
 
-	 code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_DSP_NO_LOCK, 0);
+	 code = erts_dsig_prepare(&dsd, &dep, BIF_P, ERTS_PROC_LOCK_MAIN,
+				  ERTS_DSP_NO_LOCK, 0, 1);
 	 switch (code) {
 	 case ERTS_DSIG_PREP_NOT_ALIVE:
 	 case ERTS_DSIG_PREP_NOT_CONNECTED:
-	     BIF_TRAP2(dexit_trap, BIF_P, BIF_ARG_1, BIF_ARG_2);
+	     BIF_RET(am_true);
+	 case ERTS_DSIG_PREP_PENDING:
 	 case ERTS_DSIG_PREP_CONNECTED:
 	     code = erts_dsig_send_exit2(&dsd, BIF_P->common.id, BIF_ARG_1, BIF_ARG_2);
 	     if (code == ERTS_DSIG_SEND_YIELD)
 		 ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
 	     BIF_RET(am_true);
 	 default:
-	     ASSERT(! "Invalid dsig prepare result");
-	     BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
+	     ERTS_ASSERT(! "Invalid dsig prepare result");
 	 }
      }
      else if (is_not_internal_pid(BIF_ARG_1)) {
@@ -1964,7 +1997,7 @@ ebif_bang_2(BIF_ALIST_2)
  * Send a message to Process, Port or Registered Process.
  * Returns non-negative reduction bump or negative result code.
  */
-#define SEND_TRAP		(-1)
+#define SEND_NOCONNECT		(-1)
 #define SEND_YIELD		(-2)
 #define SEND_YIELD_RETURN	(-3)
 #define SEND_BADARG		(-4)
@@ -1980,20 +2013,22 @@ static Sint remote_send(Process *p, DistEntry *dep,
 {
     Sint res;
     int code;
-
     ASSERT(is_atom(to) || is_external_pid(to));
 
     ctx->dep = dep;
-    code = erts_dsig_prepare(&ctx->dsd, dep, p, ERTS_DSP_NO_LOCK, !ctx->suspend);
+    code = erts_dsig_prepare(&ctx->dsd, &dep, p, ERTS_PROC_LOCK_MAIN,
+			     ERTS_DSP_NO_LOCK,
+			     !ctx->suspend, ctx->connect);
     switch (code) {
     case ERTS_DSIG_PREP_NOT_ALIVE:
     case ERTS_DSIG_PREP_NOT_CONNECTED:
-	res = SEND_TRAP;
+	res = SEND_NOCONNECT;
 	break;
     case ERTS_DSIG_PREP_WOULD_SUSPEND:
 	ASSERT(!ctx->suspend);
 	res = SEND_YIELD;
 	break;
+    case ERTS_DSIG_PREP_PENDING:
     case ERTS_DSIG_PREP_CONNECTED: {
 
 	if (is_atom(to))
@@ -2205,12 +2240,14 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx)
 	    }
 	    return 0;
 	}
+	ctx->dsd.node = tp[2];
 
 	ret = remote_send(p, dep, tp[1], to, msg, ctx);
 	if (ret == SEND_YIELD_CONTINUE) {
-            if (dep)
+            if (dep) {
                 erts_ref_dist_entry(dep);
-	    ctx->dep_to_deref = dep;
+		ctx->deref_dep = 1;
+	    }
 	}
 	return ret;
     } else {
@@ -2251,7 +2288,6 @@ BIF_RETTYPE send_3(BIF_ALIST_3)
     Eterm msg = BIF_ARG_2;
     Eterm opts = BIF_ARG_3;
 
-    int connect = !0;
     Eterm l = opts;
     Sint result;
 
@@ -2262,14 +2298,15 @@ BIF_RETTYPE send_3(BIF_ALIST_3)
     UseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), BIF_P);
 
     ctx->suspend = !0;
-    ctx->dep_to_deref = NULL;
+    ctx->connect = !0;
+    ctx->deref_dep = 0;
     ctx->return_term = am_ok;
     ctx->dss.reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR);
     ctx->dss.phase = ERTS_DSIG_SEND_PHASE_INIT;
 
     while (is_list(l)) {
 	if (CAR(list_val(l)) == am_noconnect) {
-	    connect = 0;
+	    ctx->connect = 0;
 	} else if (CAR(list_val(l)) == am_nosuspend) {
 	    ctx->suspend = 0;
 	} else {
@@ -2306,9 +2343,9 @@ BIF_RETTYPE send_3(BIF_ALIST_3)
 	    goto yield_return;
 	ERTS_BIF_PREP_RET(retval, am_ok);
 	break;
-    case SEND_TRAP:
-	if (connect) {
-	    ERTS_BIF_PREP_TRAP3(retval, dsend3_trap, p, to, msg, opts);
+    case SEND_NOCONNECT:
+	if (ctx->connect) {
+	    ERTS_BIF_PREP_RET(retval, am_ok);
 	} else {
 	    ERTS_BIF_PREP_RET(retval, am_noconnect);
 	}
@@ -2412,7 +2449,8 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg)
     ref = NIL;
 #endif
     ctx->suspend = !0;
-    ctx->dep_to_deref = NULL;
+    ctx->connect = !0;
+    ctx->deref_dep = 0;
     ctx->return_term = msg;
     ctx->dss.reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR);
     ctx->dss.phase = ERTS_DSIG_SEND_PHASE_INIT;
@@ -2436,8 +2474,8 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg)
 	    goto yield_return;
 	ERTS_BIF_PREP_RET(retval, msg);
 	break;
-    case SEND_TRAP:
-	ERTS_BIF_PREP_TRAP2(retval, dsend2_trap, p, to, msg);
+    case SEND_NOCONNECT:
+	ERTS_BIF_PREP_RET(retval, msg);
 	break;
     case SEND_YIELD:
 	ERTS_BIF_PREP_YIELD2(retval, bif_export[BIF_send_2], p, to, msg);
@@ -4387,20 +4425,21 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2)
 	if(dep == erts_this_dist_entry)
 	    BIF_ERROR(BIF_P, BADARG);
 
-	code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_DSP_NO_LOCK, 0);
+	code = erts_dsig_prepare(&dsd, &dep, BIF_P, ERTS_PROC_LOCK_MAIN,
+				 ERTS_DSP_NO_LOCK, 0, 0);
 	switch (code) {
 	case ERTS_DSIG_PREP_NOT_ALIVE:
 	    BIF_RET(am_true);
 	case ERTS_DSIG_PREP_NOT_CONNECTED:
 	    BIF_TRAP2(dgroup_leader_trap, BIF_P, BIF_ARG_1, BIF_ARG_2);
+	case ERTS_DSIG_PREP_PENDING:
 	case ERTS_DSIG_PREP_CONNECTED:
 	    code = erts_dsig_send_group_leader(&dsd, BIF_ARG_1, BIF_ARG_2);
 	    if (code == ERTS_DSIG_SEND_YIELD)
 		ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
 	    BIF_RET(am_true);
 	default:
-	    ASSERT(! "Invalid dsig prepare result");
-	    BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
+	    ERTS_ASSERT(! "Invalid dsig prepare result");
 	}
     }
     else if (is_internal_pid(BIF_ARG_2)) {
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index f7b4451890..66469a011d 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -691,3 +691,11 @@ bif erts_internal:maps_to_list/2
 #
 
 bif erlang:iolist_to_iovec/1
+
+#
+# New in 21.0
+#
+
+bif erlang:new_connection_id/1
+bif erlang:abort_connection_id/2
+
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index bc168fc58d..0fa399cf53 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -103,8 +103,6 @@ dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz)
 
 
 
-#define PASS_THROUGH 'p'        /* This code should go */
-
 int erts_is_alive; /* System must be blocked on change */
 int erts_dist_buf_busy_limit;
 
@@ -684,31 +682,11 @@ size_obuf(ErtsDistOutputBuf *obuf)
     return bin->orig_size;
 }
 
-static void clear_dist_entry(DistEntry *dep)
+static ErtsDistOutputBuf* clear_de_out_queues(DistEntry* dep)
 {
-    Sint obufsize = 0;
-    ErtsAtomCache *cache;
-    ErtsProcList *suspendees;
     ErtsDistOutputBuf *obuf;
 
-    erts_de_rwlock(dep);
-    erts_atomic_set_nob(&dep->input_handler,
-                        (erts_aint_t) NIL);
-    cache = dep->cache;
-    dep->cache = NULL;
-
-#ifdef DEBUG
-    erts_de_links_lock(dep);
-    ASSERT(!dep->nlinks);
-    ASSERT(!dep->node_links);
-    ASSERT(!dep->monitors);
-    erts_de_links_unlock(dep);
-#endif
-
-    erts_mtx_lock(&dep->qlock);
-
-    erts_atomic64_set_nob(&dep->in, 0);
-    erts_atomic64_set_nob(&dep->out, 0);
+    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&dep->qlock));
 
     if (!dep->out_queue.last)
 	obuf = dep->finalized_out_queue.first;
@@ -728,17 +706,12 @@ static void clear_dist_entry(DistEntry *dep)
     dep->tmp_out_queue.last = NULL;
     dep->finalized_out_queue.first = NULL;
     dep->finalized_out_queue.last = NULL;
-    dep->status = 0;
-    suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL);
-
-    erts_mtx_unlock(&dep->qlock);
-    erts_atomic_set_nob(&dep->dist_cmd_scheduled, 0);
-    dep->send = NULL;
-    erts_de_rwunlock(dep);
-
-    erts_resume_processes(suspendees);
+    return obuf;
+}
 
-    delete_cache(cache);
+static void free_de_out_queues(DistEntry* dep, ErtsDistOutputBuf *obuf)
+{
+    Sint obufsize = 0;
 
     while (obuf) {
 	ErtsDistOutputBuf *fobuf;
@@ -750,13 +723,54 @@ static void clear_dist_entry(DistEntry *dep)
 
     if (obufsize) {
 	erts_mtx_lock(&dep->qlock);
-	ASSERT(erts_atomic_read_nob(&dep->qsize) >= obufsize);
+        ASSERT(erts_atomic_read_nob(&dep->qsize) >= obufsize);
         erts_atomic_add_nob(&dep->qsize,
                             (erts_aint_t) -obufsize);
 	erts_mtx_unlock(&dep->qlock);
     }
 }
 
+static void clear_dist_entry(DistEntry *dep)
+{
+    ErtsAtomCache *cache;
+    ErtsProcList *suspendees;
+    ErtsDistOutputBuf *obuf;
+
+    erts_de_rwlock(dep);
+    erts_atomic_set_nob(&dep->input_handler,
+                        (erts_aint_t) NIL);
+    cache = dep->cache;
+    dep->cache = NULL;
+
+#ifdef DEBUG
+    erts_de_links_lock(dep);
+    ASSERT(!dep->nlinks);
+    ASSERT(!dep->node_links);
+    ASSERT(!dep->monitors);
+    erts_de_links_unlock(dep);
+#endif
+
+    erts_mtx_lock(&dep->qlock);
+
+    erts_atomic64_set_nob(&dep->in, 0);
+    erts_atomic64_set_nob(&dep->out, 0);
+
+    obuf = clear_de_out_queues(dep);
+    dep->status = 0;
+    suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL);
+
+    erts_mtx_unlock(&dep->qlock);
+    erts_atomic_set_nob(&dep->dist_cmd_scheduled, 0);
+    dep->send = NULL;
+    erts_de_rwunlock(dep);
+
+    erts_resume_processes(suspendees);
+
+    delete_cache(cache);
+
+    free_de_out_queues(dep, obuf);
+}
+
 int erts_dsend_context_dtor(Binary* ctx_bin)
 {
     ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(ctx_bin);
@@ -772,8 +786,8 @@ int erts_dsend_context_dtor(Binary* ctx_bin)
     if (ctx->dss.phase >= ERTS_DSIG_SEND_PHASE_ALLOC && ctx->dss.obuf) {
 	free_dist_obuf(ctx->dss.obuf);
     }
-    if (ctx->dep_to_deref)
-	erts_deref_dist_entry(ctx->dep_to_deref);
+    if (ctx->deref_dep)
+	erts_deref_dist_entry(ctx->dep);
 
     return 1;
 }
@@ -1324,7 +1338,7 @@ int erts_net_message(Port *prt,
 	    /* This is tricky (we MUST force a distributed send) */
 	    ErtsDSigData dsd;
 	    int code;
-	    code = erts_dsig_prepare(&dsd, dep, NULL, ERTS_DSP_NO_LOCK, 0);
+	    code = erts_dsig_prepare(&dsd, &dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0);
 	    if (code == ERTS_DSIG_PREP_CONNECTED) {
 		code = erts_dsig_send_exit(&dsd, to, from, am_noproc);
 		ASSERT(code == ERTS_DSIG_SEND_OK);
@@ -1416,7 +1430,7 @@ int erts_net_message(Port *prt,
 	if (!rp) {
 	    ErtsDSigData dsd;
 	    int code;
-	    code = erts_dsig_prepare(&dsd, dep, NULL, ERTS_DSP_NO_LOCK, 0);
+	    code = erts_dsig_prepare(&dsd, &dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0);
 	    if (code == ERTS_DSIG_PREP_CONNECTED) {
 		code = erts_dsig_send_m_exit(&dsd, watcher, watched, ref,
 					     am_noproc);
@@ -1883,7 +1897,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 	    }
 	    else {
 		ctx->acmp = NULL;
-		ctx->pass_through_size = 1;
+		ctx->pass_through_size = 3; /* SVERK rename */
 	    }
 
     #ifdef ERTS_DIST_MSG_DBG
@@ -1961,9 +1975,9 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 	    ctx->obuf->next = NULL;
 	    erts_de_rlock(dep);
 	    cid = dep->cid;
-	    if (cid != dsdp->cid
-		|| dep->connection_id != dsdp->connection_id
-		|| dep->status & ERTS_DE_SFLG_EXITING) {
+	    if (!(dep->status & (ERTS_DE_SFLG_PENDING | ERTS_DE_SFLG_CONNECTED))
+                || dep->status & ERTS_DE_SFLG_EXITING
+                || dep->connection_id != dsdp->connection_id) {
 		/* Not the same connection as when we started; drop message... */
 		erts_de_runlock(dep);
 		free_dist_obuf(ctx->obuf);
@@ -2037,8 +2051,13 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 		}
 
 		erts_mtx_unlock(&dep->qlock);
-                if (is_internal_port(dep->cid))
-                    erts_schedule_dist_command(NULL, dep);
+		if (!(dep->status & ERTS_DE_SFLG_PENDING)) {
+                    if (is_internal_port(dep->cid))
+                        erts_schedule_dist_command(NULL, dep);
+                }
+                else {
+                    notify_proc = NIL;
+                }
 		erts_de_runlock(dep);
                 if (is_internal_pid(notify_proc))
                     notify_dist_data(ctx->c_p, notify_proc);
@@ -2306,9 +2325,6 @@ erts_dist_command(Port *prt, int reds_limit)
 		ob->extp = erts_encode_ext_dist_header_finalize(ob->extp,
 								dep->cache,
 								flags);
-		if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
-		    *--ob->extp = PASS_THROUGH; /* Old node; 'pass through'
-						   needed */
 		ASSERT(&ob->data[0] <= ob->extp && ob->extp < ob->ext_endp);
 		reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE;
 		preempt = reds > reds_limit;
@@ -2346,21 +2362,18 @@ erts_dist_command(Port *prt, int reds_limit)
         int de_busy;
 	int preempt = 0;
 	while (oq.first && !preempt) {
-            ErtsDistOutputBuf *fob;
-            Uint size;
-            oq.first->extp
-                = erts_encode_ext_dist_header_finalize(oq.first->extp,
-                                                       dep->cache,
-                                                       flags);
-            reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE;
-            if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
-                *--oq.first->extp = PASS_THROUGH; /* Old node; 'pass through'
-                                                     needed */
-            ASSERT(&oq.first->data[0] <= oq.first->extp
-                   && oq.first->extp < oq.first->ext_endp);
-            size = (*send)(prt, oq.first);
+	    ErtsDistOutputBuf *fob;
+	    Uint size;
+	    oq.first->extp
+		= erts_encode_ext_dist_header_finalize(oq.first->extp,
+						       dep->cache,
+						       flags);
+	    reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE;
+	    ASSERT(&oq.first->data[0] <= oq.first->extp
+		   && oq.first->extp < oq.first->ext_endp);
+	    size = (*send)(prt, oq.first);
             erts_atomic64_inc_nob(&dep->out);
-            esdp->io.out += (Uint64) size;
+	    esdp->io.out += (Uint64) size;
 #ifdef ERTS_RAW_DIST_MSG_DBG
             erts_fprintf(stderr, ">> ");
             bw(oq.first->extp, size);
@@ -2482,11 +2495,13 @@ erts_dist_command(Port *prt, int reds_limit)
 	foq.first = NULL;
 	foq.last = NULL;
 
+/* SVERK Hmmm....
 #ifdef DEBUG
 	erts_mtx_lock(&dep->qlock);
 	ASSERT(erts_atomic_read_nob(&dep->qsize) == obufsize);
 	erts_mtx_unlock(&dep->qlock);
 #endif
+*/
     }
     else {
 	if (oq.first) {
@@ -3218,6 +3233,8 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     ErtsProcLocks proc_unlock = 0;
     Process *proc;
     Port *pp = NULL;
+    Eterm notify_proc;
+    erts_aint32_t qflgs;
 
     /*
      * Check and pick out arguments
@@ -3245,15 +3262,14 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     if (!is_atom(ic) || !is_atom(oc))
 	goto badarg;
 
-    /* DFLAG_EXTENDED_REFERENCES is compulsory from R9 and forward */
-    if (!(DFLAG_EXTENDED_REFERENCES & flags)) {
+    if (~flags & DFLAG_DIST_MANDATORY) {
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
 	erts_dsprintf(dsbufp, "%T", BIF_P->common.id);
 	if (BIF_P->common.u.alive.reg)
 	    erts_dsprintf(dsbufp, " (%T)", BIF_P->common.u.alive.reg->name);
 	erts_dsprintf(dsbufp,
 		      " attempted to enable connection to node %T "
-		      "which is not able to handle extended references.\n",
+		      "which does not support all mandatory capabilities.\n",
 		      BIF_ARG_1);
 	erts_send_error_to_logger(BIF_P->group_leader, dsbufp);
 	goto badarg;
@@ -3376,7 +3392,8 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     dep->creation = 0;
 
 #ifdef DEBUG
-    ASSERT(erts_atomic_read_nob(&dep->qsize) == 0);
+    ASSERT(erts_atomic_read_nob(&dep->qsize) == 0 
+           || (dep->status & ERTS_DE_SFLG_PENDING));
 #endif
 
     if (flags & DFLAG_DIST_HDR_ATOM_CACHE)
@@ -3384,7 +3401,26 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
 
     erts_set_dist_entry_connected(dep, BIF_ARG_2, flags);
 
+    notify_proc = NIL;
+    if (erts_atomic_read_nob(&dep->qsize)) {
+        if (is_internal_port(dep->cid)) {
+            erts_schedule_dist_command(NULL, dep);
+        }
+        else {
+            qflgs = erts_atomic32_read_nob(&dep->qflgs);
+            if (qflgs & ERTS_DE_QFLG_REQ_INFO) {
+                qflgs = erts_atomic32_read_band_mb(&dep->qflgs,
+                                                   ~ERTS_DE_QFLG_REQ_INFO);
+                if (qflgs & ERTS_DE_QFLG_REQ_INFO) {
+                    notify_proc = dep->cid;
+                    ASSERT(is_internal_pid(notify_proc));
+                }
+            }
+        }
+    }
     erts_de_rwunlock(dep);
+    if (is_internal_pid(notify_proc))
+        notify_dist_data(BIF_P, notify_proc);
 
     ERTS_BIF_PREP_RET(ret, erts_make_dhandle(BIF_P, dep));
 
@@ -3426,6 +3462,112 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     goto done;
 }
 
+BIF_RETTYPE new_connection_id_1(BIF_ALIST_1)
+{
+    DistEntry* dep;
+    Uint32 conn_id;
+
+    if (is_not_atom(BIF_ARG_1)) {
+	BIF_ERROR(BIF_P, BADARG);
+    }
+    dep = erts_find_or_insert_dist_entry(BIF_ARG_1);
+    ASSERT(dep != erts_this_dist_entry); /* SVERK: What to do? */
+
+    erts_de_rwlock(dep);
+
+    if (ERTS_DE_IS_CONNECTED(dep) || dep->status & ERTS_DE_SFLG_PENDING)
+	conn_id = dep->connection_id;
+    else if (dep->status == 0) {
+	dep->status = ERTS_DE_SFLG_PENDING;
+	dep->flags = DFLAG_DIST_MANDATORY | DFLAG_PENDING_CONNECTION;
+	dep->connection_id++;
+	dep->connection_id &= ERTS_DIST_CON_ID_MASK;
+	conn_id = dep->connection_id;
+    }
+    else {
+	ASSERT(!"SVERK: What to do?");
+	conn_id = dep->connection_id;
+    }
+    erts_de_rwunlock(dep);
+    BIF_RET(make_small(conn_id));
+}
+
+BIF_RETTYPE abort_connection_id_2(BIF_ALIST_2)
+{
+    DistEntry* dep;
+
+    if (is_not_atom(BIF_ARG_1) || is_not_small(BIF_ARG_2)) {
+	BIF_ERROR(BIF_P, BADARG);
+    }
+    dep = erts_find_dist_entry(BIF_ARG_1);
+    ASSERT(dep != erts_this_dist_entry); /* SVERK: What to do? */
+
+    if (!dep) {
+	BIF_RET(am_false);
+    }
+
+    erts_de_rwlock(dep);
+
+    if (dep->status == ERTS_DE_SFLG_PENDING
+	&& dep->connection_id == unsigned_val(BIF_ARG_2)) {
+
+	NetExitsContext nec = {dep};
+	ErtsLink *nlinks;
+	ErtsLink *node_links;
+	ErtsMonitor *monitors;
+	ErtsAtomCache *cache;
+	ErtsDistOutputBuf *obuf;
+        ErtsProcList *resume_procs;
+        Sint reds = 0;
+
+	ASSERT(is_nil(dep->cid));
+
+	erts_de_links_lock(dep);
+	monitors	= dep->monitors;
+	nlinks		= dep->nlinks;
+	node_links	= dep->node_links;
+	dep->monitors	= NULL;
+	dep->nlinks	= NULL;
+	dep->node_links	= NULL;
+	erts_de_links_unlock(dep);
+
+	cache = dep->cache;
+	dep->cache = NULL;
+	dep->status = 0;
+	dep->flags = 0;
+	erts_mtx_lock(&dep->qlock);
+        obuf = dep->out_queue.first;
+        dep->out_queue.first = NULL;
+        dep->out_queue.last = NULL;
+        ASSERT(!dep->tmp_out_queue.first);
+        ASSERT(!dep->finalized_out_queue.first);
+        resume_procs = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL);
+	erts_mtx_unlock(&dep->qlock);
+	erts_atomic_set_nob(&dep->dist_cmd_scheduled, 0);
+	dep->send = NULL;
+	erts_de_rwunlock(dep);
+
+	erts_sweep_monitors(monitors, &doit_monitor_net_exits, &nec);
+	erts_sweep_links(nlinks, &doit_link_net_exits, &nec);
+	erts_sweep_links(node_links, &doit_node_link_net_exits, &nec);
+
+        if (resume_procs) {
+            int resumed = erts_resume_processes(resume_procs);
+            reds += resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED;
+        }
+
+	delete_cache(cache);
+
+	free_de_out_queues(dep, obuf);
+
+	BIF_RET2(am_true, reds);
+    }
+
+    erts_de_rwunlock(dep);
+
+    BIF_RET(am_false);
+}
+
 
 /**********************************************************************/
 /* dist_exit(Local, Term, Remote) -> Bool */
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index d4765c50b8..26432b21e9 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -45,6 +45,13 @@
 #define DFLAG_MAP_TAG             0x20000
 #define DFLAG_BIG_CREATION        0x40000
 #define DFLAG_SEND_SENDER         0x80000
+#define DFLAG_PENDING_CONNECTION  0x100000
+
+/* Mandatory flags for distribution (sync with dist_util.erl) */
+#define DFLAG_DIST_MANDATORY (DFLAG_EXTENDED_REFERENCES         \
+                              | DFLAG_EXTENDED_PIDS_PORTS       \
+			      | DFLAG_UTF8_ATOMS                \
+			      | DFLAG_NEW_FUN_TAGS)
 
 /* All flags that should be enabled when term_to_binary/1 is used. */
 #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES	\
@@ -98,6 +105,7 @@ typedef enum {
 typedef struct {
     Process *proc;
     DistEntry *dep;
+    Eterm node;   /* used if dep == NULL */
     Eterm cid;
     Eterm connection_id;
     int no_suspend;
@@ -117,14 +125,10 @@ extern int erts_is_alive;
 
 /*
  * erts_dsig_prepare() prepares a send of a distributed signal.
- * One of the values defined below are returned. If the returned
- * value is another than ERTS_DSIG_PREP_CONNECTED, the
- * distributed signal cannot be sent before appropriate actions
- * have been taken. Appropriate actions would typically be setting
- * up the connection.
+ * One of the values defined below are returned.
  */
 
-/* Connected; signal can be sent. */
+/* Connected; signals can be enqueued and sent. */
 #define ERTS_DSIG_PREP_CONNECTED	0
 /* Not connected; connection needs to be set up. */
 #define ERTS_DSIG_PREP_NOT_CONNECTED	1
@@ -132,11 +136,15 @@ extern int erts_is_alive;
 #define ERTS_DSIG_PREP_WOULD_SUSPEND	2
 /* System not alive (distributed) */
 #define ERTS_DSIG_PREP_NOT_ALIVE	3
+/* Pending connection; signals can be enqueued */
+#define ERTS_DSIG_PREP_PENDING	        4
 
 ERTS_GLB_INLINE int erts_dsig_prepare(ErtsDSigData *,
-				      DistEntry *,
+				      DistEntry **,
 				      Process *,
+                                      ErtsProcLocks,
 				      ErtsDSigPrepLock,
+				      int,
 				      int);
 
 ERTS_GLB_INLINE
@@ -146,29 +154,100 @@ void erts_schedule_dist_command(Port *, DistEntry *);
 
 ERTS_GLB_INLINE int 
 erts_dsig_prepare(ErtsDSigData *dsdp,
-		  DistEntry *dep,
+		  DistEntry **depp,
 		  Process *proc,
+                  ErtsProcLocks proc_locks,
 		  ErtsDSigPrepLock dspl,
-		  int no_suspend)
+		  int no_suspend,
+		  int connect)
 {
-    int failure;
+    DistEntry* dep = *depp;
+    int res;
+
     if (!erts_is_alive)
 	return ERTS_DSIG_PREP_NOT_ALIVE;
-    if (!dep)
-	return ERTS_DSIG_PREP_NOT_CONNECTED;
+    if (!dep) {
+	if (!connect)
+	    return ERTS_DSIG_PREP_NOT_CONNECTED;
+
+	dep = erts_find_or_insert_dist_entry(dsdp->node);
+	ASSERT(dep != erts_this_dist_entry);  /* SVERK: What to do? */
+    }
+
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    if (connect) {
+        erts_proc_lc_might_unlock(proc, proc_locks);
+    }
+#endif
+
+retry:
     if (dspl == ERTS_DSP_RWLOCK)
 	erts_de_rwlock(dep);
     else
 	erts_de_rlock(dep);
-    if (ERTS_DE_IS_NOT_CONNECTED(dep)) {
-	failure = ERTS_DSIG_PREP_NOT_CONNECTED;
+
+    if (ERTS_DE_IS_CONNECTED(dep)) {
+	res = ERTS_DSIG_PREP_CONNECTED;
+    }
+    else if (dep->status & ERTS_DE_SFLG_PENDING) {
+	res = ERTS_DSIG_PREP_PENDING;
+    }
+    else if (dep->status & ERTS_DE_SFLG_EXITING) {
+	/* SVERK is this ok, or should we trigger another connection setup */
+	res = ERTS_DSIG_PREP_NOT_CONNECTED;
+	goto fail;
+    }
+    else if (connect) {
+        ASSERT(dep->status == 0);
+	if (dspl != ERTS_DSP_RWLOCK) {
+	    erts_de_runlock(dep);
+	    erts_de_rwlock(dep);
+	}
+	if (dep->status == 0) {
+	    Process* net_kernel;
+	    ErtsProcLocks nk_locks = ERTS_PROC_LOCK_MSGQ;
+	    Eterm *hp;
+	    ErlOffHeap *ohp;
+	    ErtsMessage *mp;
+	    Eterm msg, conn_id;
+
+	    dep->status = ERTS_DE_SFLG_PENDING;
+	    dep->flags = DFLAG_DIST_MANDATORY | DFLAG_PENDING_CONNECTION;
+	    dep->connection_id++;
+	    dep->connection_id &= ERTS_DIST_CON_ID_MASK;
+            conn_id = make_small(dep->connection_id);
+	    erts_de_rwunlock(dep);
+
+	    net_kernel = erts_whereis_process(proc, proc_locks,
+					      am_net_kernel, nk_locks, 0);
+	    if (!net_kernel) {
+		if (!*depp) {
+		    erts_deref_dist_entry(dep);
+		}
+		return ERTS_DSIG_PREP_NOT_ALIVE;
+	    }
+
+	    /* Send {auto_connect, Node, ConnId} to net_kernel */
+	    mp = erts_alloc_message_heap(net_kernel, &nk_locks, 4, &hp, &ohp);
+	    msg = TUPLE3(hp, am_auto_connect, dep->sysname, conn_id);
+	    erts_queue_message(net_kernel, nk_locks, mp, msg, proc->common.id);
+	    erts_proc_unlock(net_kernel, nk_locks);
+	}
+	else
+	    erts_de_rwunlock(dep);
+	goto retry;
+    }
+    else {
+        ASSERT(dep->status == 0);
+	res = ERTS_DSIG_PREP_NOT_CONNECTED;
 	goto fail;
     }
+
     if (no_suspend) {
-        if (erts_atomic32_read_acqb(&dep->qflgs) & ERTS_DE_QFLG_BUSY) {
-	    failure = ERTS_DSIG_PREP_WOULD_SUSPEND;
+	if (erts_atomic32_read_acqb(&dep->qflgs) & ERTS_DE_QFLG_BUSY) {
+	    res = ERTS_DSIG_PREP_WOULD_SUSPEND;
 	    goto fail;
-        }
+	}
     }
     dsdp->proc = proc;
     dsdp->dep = dep;
@@ -177,15 +256,15 @@ erts_dsig_prepare(ErtsDSigData *dsdp,
     dsdp->no_suspend = no_suspend;
     if (dspl == ERTS_DSP_NO_LOCK)
 	erts_de_runlock(dep);
-    return ERTS_DSIG_PREP_CONNECTED;
+    *depp = dep;
+    return res;
 
  fail:
     if (dspl == ERTS_DSP_RWLOCK)
 	erts_de_rwunlock(dep);
     else
 	erts_de_runlock(dep);
-    return failure;
-
+    return res;
 }
 
 ERTS_GLB_INLINE
@@ -346,11 +425,12 @@ struct erts_dsig_send_context {
 
 typedef struct {
     int suspend;
+    int connect;
 
     Eterm ctl_heap[6];
     ErtsDSigData dsd;
-    DistEntry* dep_to_deref;
     DistEntry *dep;
+    int deref_dep;
     struct erts_dsig_send_context dss;
 
     Eterm return_term;
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 9931686cbe..dd607f438e 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -597,6 +597,8 @@ erts_set_dist_entry_not_connected(DistEntry *dep)
 void
 erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags)
 {
+    erts_aint32_t set_qflgs;
+
     ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep));
     erts_rwmtx_rwlock(&erts_dist_table_rwmtx);
 
@@ -619,22 +621,26 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags)
     ASSERT(erts_no_of_not_connected_dist_entries > 0);
     erts_no_of_not_connected_dist_entries--;
 
+    if (dep->status & ERTS_DE_SFLG_PENDING) {
+	dep->status &= ~ERTS_DE_SFLG_PENDING;
+    } else {
+	dep->connection_id++;
+	dep->connection_id &= ERTS_DIST_CON_ID_MASK;
+    }
     dep->status |= ERTS_DE_SFLG_CONNECTED;
-    dep->flags = flags;
+    dep->flags = flags & ~DFLAG_PENDING_CONNECTION;
     dep->cid = cid;
     erts_atomic_set_nob(&dep->input_handler,
                             (erts_aint_t) cid);
 
-    dep->connection_id++;
-    dep->connection_id &= ERTS_DIST_EXT_CON_ID_MASK;
     dep->prev = NULL;
 
     erts_atomic64_set_nob(&dep->in, 0);
     erts_atomic64_set_nob(&dep->out, 0);
-    erts_atomic32_set_nob(&dep->qflgs,
-                          (is_internal_port(cid)
-                           ? ERTS_DE_QFLG_PORT_CTRL
-                           : ERTS_DE_QFLG_PROC_CTRL));
+    set_qflgs = (is_internal_port(cid) ?
+                 ERTS_DE_QFLG_PORT_CTRL : ERTS_DE_QFLG_PROC_CTRL);
+    erts_atomic32_read_bor_nob(&dep->qflgs, set_qflgs);
+
     if(flags & DFLAG_PUBLISHED) {
 	dep->next = erts_visible_dist_entries;
 	if(erts_visible_dist_entries) {
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 2cfd18f22f..d012f4b2cf 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -57,8 +57,9 @@
  
 #define ERST_INTERNAL_CHANNEL_NO 0
 
-#define ERTS_DE_SFLG_CONNECTED			(((Uint32) 1) <<  0)
-#define ERTS_DE_SFLG_EXITING			(((Uint32) 1) <<  1)
+#define ERTS_DE_SFLG_PENDING			(((Uint32) 1) <<  0)
+#define ERTS_DE_SFLG_CONNECTED			(((Uint32) 1) <<  1)
+#define ERTS_DE_SFLG_EXITING			(((Uint32) 1) <<  2)
 
 #define ERTS_DE_QFLG_BUSY			(((erts_aint32_t) 1) <<  0)
 #define ERTS_DE_QFLG_EXIT			(((erts_aint32_t) 1) <<  1)
@@ -106,8 +107,6 @@ struct ErtsProcList_;
  *   unlock mutexes with higher numbers before mutexes with higher numbers.
  */
 
-struct erl_link;
-
 typedef struct dist_entry_ {
     HashBucket hash_bucket;     /* Hash bucket */
     struct dist_entry_ *next;	/* Next entry in dist_table (not sorted) */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 3c0a126fe2..2d13cd92b2 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -12659,9 +12659,11 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		erts_de_links_unlock(dep);
 		if (rmon) {
 		    ErtsDSigData dsd;
-		    int code = erts_dsig_prepare(&dsd, dep, NULL,
-						 ERTS_DSP_NO_LOCK, 0);
-		    if (code == ERTS_DSIG_PREP_CONNECTED) {
+		    int code = erts_dsig_prepare(&dsd, &dep, NULL, 0,
+						 ERTS_DSP_NO_LOCK, 0, 0);
+		    if (code == ERTS_DSIG_PREP_CONNECTED ||
+                        code == ERTS_DSIG_PREP_PENDING) {
+
 			code = erts_dsig_send_demonitor(&dsd,
 							rmon->u.pid,
 							mon->name,
@@ -12705,9 +12707,11 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		    erts_de_links_unlock(dep);
 		    if (rmon) {
 			ErtsDSigData dsd;
-			int code = erts_dsig_prepare(&dsd, dep, NULL,
-						     ERTS_DSP_NO_LOCK, 0);
-			if (code == ERTS_DSIG_PREP_CONNECTED) {
+			int code = erts_dsig_prepare(&dsd, &dep, NULL, 0,
+						     ERTS_DSP_NO_LOCK, 0, 0);
+			if (code == ERTS_DSIG_PREP_CONNECTED ||
+                            code == ERTS_DSIG_PREP_PENDING) {
+
 			    code = erts_dsig_send_demonitor(&dsd,
 							    rmon->u.pid,
 							    mon->u.pid,
@@ -12764,8 +12768,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		erts_de_links_unlock(dep);
 		if (rmon) {
 		    ErtsDSigData dsd;
-		    int code = erts_dsig_prepare(&dsd, dep, NULL,
-						 ERTS_DSP_NO_LOCK, 0);
+		    int code = erts_dsig_prepare(&dsd, &dep, NULL, 0,
+						 ERTS_DSP_NO_LOCK, 0, 0);
 		    if (code == ERTS_DSIG_PREP_CONNECTED) {
 			code = erts_dsig_send_m_exit(&dsd,
 						     mon->u.pid,
@@ -12887,14 +12891,18 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
 		int code;
 		ErtsDistLinkData dld;
 		erts_remove_dist_link(&dld, p->common.id, item, dep);
-		erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
-		code = erts_dsig_prepare(&dsd, dep, p, ERTS_DSP_NO_LOCK, 0);
-		if (code == ERTS_DSIG_PREP_CONNECTED) {
-		    code = erts_dsig_send_exit_tt(&dsd, p->common.id, item,
-						  reason, SEQ_TRACE_TOKEN(p));
-		    ASSERT(code == ERTS_DSIG_SEND_OK);
-		}
-		erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
+                if (dld.d_lnk) {
+                    erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+                    code = erts_dsig_prepare(&dsd, &dep, p, 0, ERTS_DSP_NO_LOCK, 0, 0);
+                    if (code == ERTS_DSIG_PREP_CONNECTED ||
+                        code == ERTS_DSIG_PREP_PENDING) {
+
+                        code = erts_dsig_send_exit_tt(&dsd, p->common.id, item,
+                                                      reason, SEQ_TRACE_TOKEN(p));
+                        ASSERT(code == ERTS_DSIG_SEND_OK);
+                    }
+                    erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
+                }
 		erts_destroy_dist_link(&dld);
 	    }
 	}
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 3b851087d1..c49e2f617a 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -349,6 +349,8 @@ byte *erts_encode_ext_dist_header_setup(byte *ctl_ext, ErtsAtomCacheMap *acmp)
     }
 }
 
+#define PASS_THROUGH 'p'        /* This code should go */
+
 byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache, Uint32 dflags)
 {
     byte *ip;
@@ -358,9 +360,33 @@ byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache, Uint
     int long_atoms;
     register byte *ep = ext;
     ASSERT(dflags & DFLAG_UTF8_ATOMS);
-    ASSERT(ep[0] == VERSION_MAGIC);
-    if (ep[1] != DIST_HEADER)
-	return ext;
+
+    if (ep[0] != VERSION_MAGIC) {
+        ASSERT(ep[0] == SMALL_TUPLE_EXT || ep[0] == LARGE_TUPLE_EXT);
+        if (dflags & DFLAG_DIST_HDR_ATOM_CACHE) {
+            /*
+             * Encoded without atom cache (toward pending connection)
+             * but receiver wants dist header. Let's prepend an empty one.
+             */
+            *--ep = 0; /* NumberOfAtomCacheRefs */
+            *--ep = DIST_HEADER;
+            *--ep = VERSION_MAGIC;
+        }
+        else {
+            /* Node without atom cache, 'pass through' needed */
+
+            ASSERT(!"SVERK: Must insert VERSION_MAGIC's");
+            *--ep = PASS_THROUGH;
+        }
+        return ep;
+    }
+    else if (ep[1] != DIST_HEADER) {
+        ASSERT(ep[1] == SMALL_TUPLE_EXT || ep[1] == LARGE_TUPLE_EXT);
+        ASSERT(!(dflags & DFLAG_DIST_HDR_ATOM_CACHE));
+        /* Node without atom cache, 'pass through' needed */
+        *--ep = PASS_THROUGH;
+        return ep;
+    }
 
     dist_hdr_flags = ep[2];
     long_atoms = ERTS_DIST_HDR_LONG_ATOMS_FLG & ((int) dist_hdr_flags);
@@ -511,7 +537,7 @@ int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx
 	return -1;
     } else {
 #ifndef ERTS_DEBUG_USE_DIST_SEP
-	if (!(ctx->flags & DFLAG_DIST_HDR_ATOM_CACHE))
+	if (!(ctx->flags & (DFLAG_DIST_HDR_ATOM_CACHE | DFLAG_PENDING_CONNECTION)))
 #endif
 	    sz++ /* VERSION_MAGIC */;
 
@@ -543,7 +569,7 @@ int erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap
 {
     if (!ctx || !ctx->wstack.wstart) {
     #ifndef ERTS_DEBUG_USE_DIST_SEP
-	if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
+	if (!(flags & (DFLAG_DIST_HDR_ATOM_CACHE | DFLAG_PENDING_CONNECTION)))
     #endif
 	    *(*ext)++ = VERSION_MAGIC;
     }
@@ -644,12 +670,11 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
 
     erts_de_rlock(dep);
 
-    if ((dep->status & (ERTS_DE_SFLG_EXITING|ERTS_DE_SFLG_CONNECTED))
-        != ERTS_DE_SFLG_CONNECTED) {
+    if (dep->status != ERTS_DE_SFLG_CONNECTED &&
+	dep->status != ERTS_DE_SFLG_PENDING) {
         erts_de_runlock(dep);
         return ERTS_PREP_DIST_EXT_CLOSED;
     }
-
     if (dep->flags & DFLAG_DIST_HDR_ATOM_CACHE)
         edep->flags |= ERTS_DIST_EXT_DFLAG_HDR;
 
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index d6416edbc3..49950c4aad 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -117,7 +117,7 @@ typedef struct {
 #define ERTS_DIST_EXT_ATOM_TRANS_TAB ((Uint32) 0x2)
 #define ERTS_DIST_EXT_BTT_SAFE       ((Uint32) 0x4)
 
-#define ERTS_DIST_CON_ID_MASK    ((Uint32) 0x00ffffff)
+#define ERTS_DIST_CON_ID_MASK ((Uint32) 0x00ffffff) /* also in net_kernel.erl */
 
 typedef struct {
     DistEntry *dep;
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 85013af3ad..6cd1aa5e79 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -3829,11 +3829,12 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc)
 	    ErtsDistLinkData dld;
 	    ErtsDSigData dsd;
 	    int code;
-	    code = erts_dsig_prepare(&dsd, dep, NULL, ERTS_DSP_NO_LOCK, 0);
+	    code = erts_dsig_prepare(&dsd, &dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0);
 	    switch (code) {
 	    case ERTS_DSIG_PREP_NOT_ALIVE:
 	    case ERTS_DSIG_PREP_NOT_CONNECTED:
 		break;
+	    case ERTS_DSIG_PREP_PENDING:
 	    case ERTS_DSIG_PREP_CONNECTED:
 		erts_remove_dist_link(&dld, port_id, lnk->pid, dep);
 		erts_destroy_dist_link(&dld);
-- 
cgit v1.2.3


From 3dda5298322a76f2787c6c4dd1cce78416c20dc3 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 16 Aug 2017 16:56:12 +0200
Subject: erts: Async auto-connect for monitor_node

Removed distribution_SUITE:applied_monitor_node
as it seems to test apply of trapping BIF
and monitor_node does not trap anymore.
---
 erts/emulator/beam/dist.c                 | 124 ++++++++++++++++++++----------
 erts/emulator/test/distribution_SUITE.erl |  24 +-----
 erts/emulator/test/erl_link_SUITE.erl     |  13 ++--
 3 files changed, 94 insertions(+), 67 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 0fa399cf53..724cb6b24d 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -3789,11 +3789,17 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options)
     DistEntry *dep;
     ErtsLink *lnk;
     Eterm l;
+    int async_connect = 1;
 
     for (l = Options; l != NIL && is_list(l); l = CDR(list_val(l))) {
 	Eterm t = CAR(list_val(l));
-	/* allow_passive_connect the only available option right now */
-	if (t != am_allow_passive_connect) {
+	if (t == am_allow_passive_connect) {
+	    /*
+	     * Handle this horrible feature by falling back on old synchronous
+	     * auto-connect (if needed)
+	     */
+	    async_connect = 0;
+	} else {
 	    BIF_ERROR(p, BADARG);
 	}
     }
@@ -3807,47 +3813,85 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options)
 	 && (Node != erts_this_node->sysname))) {
 	BIF_ERROR(p, BADARG);
     }
-    dep = erts_sysname_to_connected_dist_entry(Node);
-    if (!dep) {
-    do_trap:
-	BIF_TRAP3(dmonitor_node_trap, p, Node, Bool, Options);
-    }
-    if (dep == erts_this_dist_entry)
-	goto done;
-
-    erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
-    erts_de_rlock(dep);
-    if (ERTS_DE_IS_NOT_CONNECTED(dep)) {
-	erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
-	erts_de_runlock(dep);
-	goto do_trap;
-    }
-    erts_de_links_lock(dep);
-    erts_de_runlock(dep);
 
     if (Bool == am_true) {
-	ASSERT(dep->cid != NIL);
-	lnk = erts_add_or_lookup_link(&(dep->node_links), LINK_NODE, 
-				      p->common.id);
-	++ERTS_LINK_REFC(lnk);
-	lnk = erts_add_or_lookup_link(&ERTS_P_LINKS(p), LINK_NODE, Node);
-	++ERTS_LINK_REFC(lnk);
-    }
-    else  {
-	lnk = erts_lookup_link(dep->node_links, p->common.id);
-	if (lnk != NULL) {
-	    if ((--ERTS_LINK_REFC(lnk)) == 0) {
-		erts_destroy_link(erts_remove_link(&(dep->node_links), 
-						   p->common.id));
-	    }
-	}
-	lnk = erts_lookup_link(ERTS_P_LINKS(p), Node);
-	if (lnk != NULL) {
-	    if ((--ERTS_LINK_REFC(lnk)) == 0) {
-		erts_destroy_link(erts_remove_link(&ERTS_P_LINKS(p),
-						   Node));
+        ErtsDSigData dsd;
+        dsd.node = Node;
+        dep = erts_find_or_insert_dist_entry(Node);
+        if (dep == erts_this_dist_entry)
+            goto done;
+
+        erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
+
+        switch (erts_dsig_prepare(&dsd, &dep, p,
+                                  (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK),
+                                  ERTS_DSP_RLOCK, 0, async_connect)) {
+        case ERTS_DSIG_PREP_NOT_ALIVE:
+	case ERTS_DSIG_PREP_NOT_CONNECTED:
+	    /* Trap to either send 'nodedown' or do passive connection attempt */
+	trap:
+            erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+	    erts_deref_dist_entry(dep);
+            BIF_TRAP3(dmonitor_node_trap, p, Node, Bool, Options);
+	case ERTS_DSIG_PREP_PENDING:
+	    if (!async_connect) {
+		/*
+		 * Pending connection may fail, so we must trap
+		 * to ensure passive connection attempt
+		 */
+		erts_de_runlock(dep);
+		goto trap;
 	    }
-	}
+	    /*fall through*/
+        case ERTS_DSIG_PREP_CONNECTED:
+            erts_de_links_lock(dep);
+            erts_de_runlock(dep);
+            lnk = erts_add_or_lookup_link(&(dep->node_links), LINK_NODE,
+                                          p->common.id);
+            ++ERTS_LINK_REFC(lnk);
+            lnk = erts_add_or_lookup_link(&ERTS_P_LINKS(p), LINK_NODE, Node);
+            ++ERTS_LINK_REFC(lnk);
+            break;
+        default:
+            ERTS_ASSERT(! "Invalid dsig prepare result");
+        }
+    }
+    else { /* Bool == false */
+        dep = erts_sysname_to_connected_dist_entry(Node);
+        if (!dep) {
+	    /*
+	     * Before OTP-21 this case triggered auto-connect
+	     * and a 'nodedown' message if that failed.
+	     * Now it's a simple no-op which feels more reasonable.
+	     */
+            BIF_RET(am_true);
+        }
+        if (dep == erts_this_dist_entry)
+            goto done;
+
+        erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
+        erts_de_rlock(dep);
+        if (!(dep->status & (ERTS_DE_SFLG_PENDING | ERTS_DE_SFLG_CONNECTED))) {
+            erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+            erts_de_runlock(dep);
+            goto done;
+        }
+        erts_de_links_lock(dep);
+        erts_de_runlock(dep);
+        lnk = erts_lookup_link(dep->node_links, p->common.id);
+        if (lnk != NULL) {
+            if ((--ERTS_LINK_REFC(lnk)) == 0) {
+                erts_destroy_link(erts_remove_link(&(dep->node_links),
+                                                   p->common.id));
+            }
+        }
+        lnk = erts_lookup_link(ERTS_P_LINKS(p), Node);
+        if (lnk != NULL) {
+            if ((--ERTS_LINK_REFC(lnk)) == 0) {
+                erts_destroy_link(erts_remove_link(&ERTS_P_LINKS(p),
+                                                   Node));
+            }
+        }
     }
 
     erts_de_links_unlock(dep);
diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index c7d63f9feb..98c39dc6dd 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -41,7 +41,7 @@
          local_send_small/1, local_send_big/1,
          local_send_legal/1, link_to_busy/1, exit_to_busy/1,
          lost_exit/1, link_to_dead/1, link_to_dead_new_node/1,
-         applied_monitor_node/1, ref_port_roundtrip/1, nil_roundtrip/1,
+         ref_port_roundtrip/1, nil_roundtrip/1,
          trap_bif_1/1, trap_bif_2/1, trap_bif_3/1,
          stop_dist/1,
          dist_auto_connect_never/1, dist_auto_connect_once/1,
@@ -75,7 +75,7 @@ suite() ->
 all() ->
     [ping, {group, bulk_send}, {group, local_send},
      link_to_busy, exit_to_busy, lost_exit, link_to_dead,
-     link_to_dead_new_node, applied_monitor_node,
+     link_to_dead_new_node,
      ref_port_roundtrip, nil_roundtrip, stop_dist,
      {group, trap_bif}, {group, dist_auto_connect},
      dist_parallel_send, atom_roundtrip, unicode_atom_roundtrip,
@@ -639,26 +639,6 @@ link_to_dead_new_node(Config) when is_list(Config) ->
     end,
     ok.
 
-%% Test that monitor_node/2 works when applied.
-applied_monitor_node(Config) when is_list(Config) ->
-    NonExisting = list_to_atom("__non_existing__@" ++ hostname()),
-
-    %% Tail-recursive call to apply (since the node is non-existing,
-    %% there will be a trap).
-
-    true = tail_apply(erlang, monitor_node, [NonExisting, true]),
-    [{nodedown, NonExisting}] = test_server:messages_get(),
-
-    %% Ordinary call (with trap).
-
-    true = apply(erlang, monitor_node, [NonExisting, true]),
-    [{nodedown, NonExisting}] = test_server:messages_get(),
-
-    ok.
-
-tail_apply(M, F, A) ->
-    apply(M, F, A).
-
 %% Test that sending a port or reference to another node and back again
 %% doesn't correct them in any way.
 ref_port_roundtrip(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/erl_link_SUITE.erl b/erts/emulator/test/erl_link_SUITE.erl
index d8c5b663e3..a66ca7a57d 100644
--- a/erts/emulator/test/erl_link_SUITE.erl
+++ b/erts/emulator/test/erl_link_SUITE.erl
@@ -200,13 +200,16 @@ monitor_nodes(Config) when is_list(Config) ->
     monitor_node(A, true),
     check_monitor_node(self(), A, 1),
     check_monitor_node(self(), B, 3),
+    ok = receive {nodedown, C} -> ok after 1000 -> timeout end,
+    %%OTP-21: monitor_node(_,false) does not trigger auto-connect anymore
+    %%        and therefore no nodedown if it fails.
+    %%ok = receive {nodedown, C} -> ok after 1000 -> timeout end,
+    ok = receive {nodedown, C} -> ok after 1000 -> timeout end,
+    ok = receive {nodedown, D} -> ok after 1000 -> timeout end,
+    ok = receive {nodedown, D} -> ok after 1000 -> timeout end,
     check_monitor_node(self(), C, 0),
     check_monitor_node(self(), D, 0),
-    receive {nodedown, C} -> ok end,
-    receive {nodedown, C} -> ok end,
-    receive {nodedown, C} -> ok end,
-    receive {nodedown, D} -> ok end,
-    receive {nodedown, D} -> ok end,
+
     stop_node(A),
     receive {nodedown, A} -> ok end,
     check_monitor_node(self(), A, 0),
-- 
cgit v1.2.3


From 3f5f22656a33aec1ba4ed01249732e32d08a8c50 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 18 Aug 2017 16:56:21 +0200
Subject: erts: Async auto-connect for group_leader/2

---
 erts/emulator/beam/bif.c                  |  6 ++--
 erts/emulator/test/distribution_SUITE.erl | 50 +++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 34f5afae78..2583a5f8d6 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4422,16 +4422,16 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2)
 	int code;
 	ErtsDSigData dsd;
 	dep = external_pid_dist_entry(BIF_ARG_2);
+	ERTS_ASSERT(dep);
 	if(dep == erts_this_dist_entry)
 	    BIF_ERROR(BIF_P, BADARG);
 
 	code = erts_dsig_prepare(&dsd, &dep, BIF_P, ERTS_PROC_LOCK_MAIN,
-				 ERTS_DSP_NO_LOCK, 0, 0);
+				 ERTS_DSP_NO_LOCK, 0, 1);
 	switch (code) {
 	case ERTS_DSIG_PREP_NOT_ALIVE:
-	    BIF_RET(am_true);
 	case ERTS_DSIG_PREP_NOT_CONNECTED:
-	    BIF_TRAP2(dgroup_leader_trap, BIF_P, BIF_ARG_1, BIF_ARG_2);
+	    BIF_RET(am_true);
 	case ERTS_DSIG_PREP_PENDING:
 	case ERTS_DSIG_PREP_CONNECTED:
 	    code = erts_dsig_send_group_leader(&dsd, BIF_ARG_1, BIF_ARG_2);
diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index 98c39dc6dd..fa683f363f 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -35,8 +35,12 @@
 
 -include_lib("common_test/include/ct.hrl").
 
+%-define(Line, erlang:display({line,?LINE}),).
+-define(Line,).
+
 -export([all/0, suite/0, groups/0,
          ping/1, bulk_send_small/1,
+         group_leader/1,
          bulk_send_big/1, bulk_send_bigbig/1,
          local_send_small/1, local_send_big/1,
          local_send_legal/1, link_to_busy/1, exit_to_busy/1,
@@ -61,6 +65,7 @@
 
 %% Internal exports.
 -export([sender/3, receiver2/2, dummy_waiter/0, dead_process/0,
+         group_leader_1/1,
          roundtrip/1, bounce/1, do_dist_auto_connect/1, inet_rpc_server/1,
          dist_parallel_sender/3, dist_parallel_receiver/0,
          dist_evil_parallel_receiver/0]).
@@ -74,6 +79,7 @@ suite() ->
 
 all() ->
     [ping, {group, bulk_send}, {group, local_send},
+     group_leader,
      link_to_busy, exit_to_busy, lost_exit, link_to_dead,
      link_to_dead_new_node,
      ref_port_roundtrip, nil_roundtrip, stop_dist,
@@ -124,6 +130,50 @@ ping(Config) when is_list(Config) ->
 
     ok.
 
+%% Test erlang:group_leader(_, ExternalPid), i.e. DOP_GROUP_LEADER
+group_leader(Config) when is_list(Config) ->
+    ?Line Sock = start_relay_node(group_leader_1, []),
+    ?Line Sock2 = start_relay_node(group_leader_2, []),
+    try
+        ?Line Node2 = inet_rpc_nodename(Sock2),
+        ?Line {ok, ok} = do_inet_rpc(Sock, ?MODULE, group_leader_1, [Node2])
+    after
+        ?Line stop_relay_node(Sock),
+        ?Line stop_relay_node(Sock2)
+    end,
+    ok.
+
+group_leader_1(Node2) ->
+    ?Line ExtPid = spawn(Node2, fun F() ->
+                                        receive {From, group_leader} ->
+                                                From ! {self(), group_leader, group_leader()}
+                                        end,
+                                        F()
+                                end),
+    ?Line GL1 = self(),
+    ?Line group_leader(GL1, ExtPid),
+    ?Line ExtPid ! {self(), group_leader},
+    ?Line {ExtPid, group_leader, GL1} = receive_one(),
+
+    %% Kill connection and repeat test when group_leader/2 triggers auto-connect
+    ?Line net_kernel:monitor_nodes(true),
+    ?Line net_kernel:disconnect(Node2),
+    ?Line {nodedown, Node2} = receive_one(),
+    ?Line GL2 = spawn(fun() -> dummy end),
+    ?Line group_leader(GL2, ExtPid),
+    ?Line {nodeup, Node2} = receive_one(),
+    ?Line ExtPid ! {self(), group_leader},
+    ?Line {ExtPid, group_leader, GL2} = receive_one(),
+    ok.
+
+receive_one() ->
+    receive M -> M after 1000 -> timeout end.
+
+
+
+
+
+
 bulk_send_small(Config) when is_list(Config) ->
     bulk_send(64, 32).
 
-- 
cgit v1.2.3


From 17e198d6ee60f7dec9abfed272cf4226aea44535 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 18 Aug 2017 19:56:02 +0200
Subject: erts: Async auto-connect for monitor/2

---
 erts/emulator/beam/bif.c | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 2583a5f8d6..2af61e6ebc 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -790,31 +790,20 @@ remote_monitor(Process *p, Eterm bifarg1, Eterm bifarg2,
     BIF_RETTYPE ret;
     int code;
 
+    ASSERT(dep);
     erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
     code = erts_dsig_prepare(&dsd, &dep,
 			     p, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK),
-			     ERTS_DSP_RLOCK, 0, 0);
+			     ERTS_DSP_RLOCK, 0, 1);
     switch (code) {
-    case ERTS_DSIG_PREP_PENDING:
-	/*
-	 * Must wait for connection to know if node supports monitor.
-	 * Damn these synchronous errors.
-	 */
-	erts_smp_de_runlock(dep);
-	/* fall through */
     case ERTS_DSIG_PREP_NOT_ALIVE:
     case ERTS_DSIG_PREP_NOT_CONNECTED:
 	erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 	ERTS_BIF_PREP_TRAP2(ret, dmonitor_p_trap, p, bifarg1, bifarg2);
 	break;
+    case ERTS_DSIG_PREP_PENDING:
     case ERTS_DSIG_PREP_CONNECTED:
-	if (!(dep->flags & DFLAG_DIST_MONITOR)
-	    || (byname && !(dep->flags & DFLAG_DIST_MONITOR_NAME))) {
-	    erts_de_runlock(dep);
-	    erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
-	    ERTS_BIF_PREP_ERROR(ret, p, BADARG);
-	}
-	else {
+	{
 	    Eterm p_trgt, p_name, d_name, mon_ref;
 
 	    mon_ref = erts_make_ref(p);
@@ -917,17 +906,17 @@ local_port:
 	if (!erts_is_alive && remote_node != am_Noname) {
             goto badarg; /* Remote monitor from (this) undistributed node */
 	}
-	dep = erts_sysname_to_connected_dist_entry(remote_node);
+	dep = erts_find_or_insert_dist_entry(remote_node);
 	if (dep == erts_this_dist_entry) {
             ret = local_name_monitor(BIF_P, BIF_ARG_1, name);
 	} else {
 	    ret = remote_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2, dep, name, 1);
 	}
+	erts_deref_dist_entry(dep);
     } else {
 badarg:
 	ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
     }
-
     return ret;
 }
 
-- 
cgit v1.2.3


From e8df66a934045a1b0dc1edf0a695b6dbcf4cca4b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 24 Jul 2017 16:05:36 +0200
Subject: Remove obsolete erlang:dsend

---
 erts/emulator/beam/atom.names |  1 -
 erts/emulator/beam/dist.c     | 11 +----------
 erts/emulator/beam/dist.h     |  2 --
 3 files changed, 1 insertion(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index a534dd44fb..dd3e11dc26 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -231,7 +231,6 @@ atom dollar_endonly
 atom dotall
 atom driver
 atom driver_options
-atom dsend
 atom dsend_continue_trap
 atom dunlink
 atom duplicate_bag
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 724cb6b24d..7c44545de3 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -108,9 +108,6 @@ int erts_dist_buf_busy_limit;
 
 
 /* distribution trap functions */
-Export* dsend2_trap = NULL;
-Export* dsend3_trap = NULL;
-/*Export* dsend_nosuspend_trap = NULL;*/
 Export* dlink_trap = NULL;
 Export* dunlink_trap = NULL;
 Export* dmonitor_node_trap = NULL;
@@ -636,9 +633,6 @@ void init_dist(void)
     erts_atomic_init_nob(&no_caches, 0);
 
     /* Lookup/Install all references to trap functions */
-    dsend2_trap = trap_function(am_dsend,2);
-    dsend3_trap = trap_function(am_dsend,3);
-    /*    dsend_nosuspend_trap = trap_function(am_dsend_nosuspend,2);*/
     dlink_trap = trap_function(am_dlink,1);
     dunlink_trap = trap_function(am_dunlink,1);
     dmonitor_node_trap = trap_function(am_dmonitor_node,3);
@@ -3148,10 +3142,7 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2)
 	goto error;
 
     /* Check that all trap functions are defined !! */
-    if (dsend2_trap->addressv[0] == NULL ||
-	dsend3_trap->addressv[0] == NULL ||
-	/*	dsend_nosuspend_trap->address == NULL ||*/
-	dlink_trap->addressv[0] == NULL ||
+    if (dlink_trap->addressv[0] == NULL ||
 	dunlink_trap->addressv[0] == NULL ||
 	dmonitor_node_trap->addressv[0] == NULL ||
 	dgroup_leader_trap->addressv[0] == NULL ||
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 26432b21e9..8bb915c76b 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -86,8 +86,6 @@
 #define DOP_SEND_SENDER_TT      23
 
 /* distribution trap functions */
-extern Export* dsend2_trap;
-extern Export* dsend3_trap;
 extern Export* dlink_trap;
 extern Export* dunlink_trap;
 extern Export* dmonitor_node_trap;
-- 
cgit v1.2.3


From 5ad822ccfd841400bc44cb53acc6a0889ca3f128 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 15 Aug 2017 16:26:34 +0200
Subject: Remove obsolete erlang:dlink/1, dunlink/1 and dist_exit/3

---
 erts/emulator/beam/atom.names |  2 -
 erts/emulator/beam/bif.c      |  9 -----
 erts/emulator/beam/bif.tab    |  2 -
 erts/emulator/beam/dist.c     | 87 +------------------------------------------
 erts/emulator/beam/dist.h     |  2 -
 5 files changed, 1 insertion(+), 101 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index dd3e11dc26..3c06c26ba0 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -222,7 +222,6 @@ atom dist_ctrl_put_data
 atom dist_data
 atom Div='/'
 atom div
-atom dlink
 atom dmonitor_node
 atom dmonitor_p
 atom DollarDollar='$$'
@@ -232,7 +231,6 @@ atom dotall
 atom driver
 atom driver_options
 atom dsend_continue_trap
-atom dunlink
 atom duplicate_bag
 atom duplicated
 atom dupnames
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 2af61e6ebc..8eb4d9e85b 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1181,16 +1181,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 	switch (code) {
 	case ERTS_DSIG_PREP_NOT_ALIVE:
 	case ERTS_DSIG_PREP_NOT_CONNECTED:
-#if 1
 	    BIF_RET(am_true);
-#else
-	    /*
-	     * This is how we used to do it, but the link is obviously not
-	     * active, so I see no point in setting up a connection.
-	     * /Rickard
-	     */
-	    BIF_TRAP1(dunlink_trap, BIF_P, BIF_ARG_1);
-#endif
 
 	case ERTS_DSIG_PREP_PENDING:
 	case ERTS_DSIG_PREP_CONNECTED:
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 66469a011d..5a591caac9 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -153,14 +153,12 @@ bif erlang:whereis/1
 bif erlang:spawn_opt/1
 bif erlang:setnode/2
 bif erlang:setnode/3
-bif erlang:dist_exit/3
 bif erlang:dist_get_stat/1
 bif erlang:dist_ctrl_input_handler/2
 bif erlang:dist_ctrl_put_data/2
 bif erlang:dist_ctrl_get_data/1
 bif erlang:dist_ctrl_get_data_notification/1
 
-
 # Static native functions in erts_internal
 bif erts_internal:port_info/1
 bif erts_internal:port_info/2
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 7c44545de3..6e08c72289 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -108,8 +108,6 @@ int erts_dist_buf_busy_limit;
 
 
 /* distribution trap functions */
-Export* dlink_trap = NULL;
-Export* dunlink_trap = NULL;
 Export* dmonitor_node_trap = NULL;
 Export* dgroup_leader_trap = NULL;
 Export* dexit_trap = NULL;
@@ -633,8 +631,6 @@ void init_dist(void)
     erts_atomic_init_nob(&no_caches, 0);
 
     /* Lookup/Install all references to trap functions */
-    dlink_trap = trap_function(am_dlink,1);
-    dunlink_trap = trap_function(am_dunlink,1);
     dmonitor_node_trap = trap_function(am_dmonitor_node,3);
     dgroup_leader_trap = trap_function(am_dgroup_leader,2);
     dexit_trap = trap_function(am_dexit, 2);
@@ -3100,7 +3096,6 @@ int distribution_info(fmtfn_t to, void *arg)	/* Called by break handler */
 	    monitor_node  -- turn on/off node monitoring
 
             node controller only:
-            dist_exit/3       -- send exit signals from remote to local process
             dist_link/2       -- link a remote process to a local
             dist_unlink/2     -- unlink a remote from a local
 ****************************************************************************/
@@ -3111,9 +3106,6 @@ int distribution_info(fmtfn_t to, void *arg)	/* Called by break handler */
  ** Set the node name of current node fail if node already is set.
  ** setnode(name@host, Creation)
  ** loads functions pointer to trap_functions from module erlang.
- **    erlang:dsend/2
- **    erlang:dlink/1
- **    erlang:dunlink/1
  **    erlang:dmonitor_node/3
  **    erlang:dgroup_leader/2
  **    erlang:dexit/2
@@ -3142,9 +3134,7 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2)
 	goto error;
 
     /* Check that all trap functions are defined !! */
-    if (dlink_trap->addressv[0] == NULL ||
-	dunlink_trap->addressv[0] == NULL ||
-	dmonitor_node_trap->addressv[0] == NULL ||
+    if (dmonitor_node_trap->addressv[0] == NULL ||
 	dgroup_leader_trap->addressv[0] == NULL ||
 	dmonitor_p_trap->addressv[0] == NULL ||
 	dexit_trap->addressv[0] == NULL) {
@@ -3559,81 +3549,6 @@ BIF_RETTYPE abort_connection_id_2(BIF_ALIST_2)
     BIF_RET(am_false);
 }
 
-
-/**********************************************************************/
-/* dist_exit(Local, Term, Remote) -> Bool */
-
-BIF_RETTYPE dist_exit_3(BIF_ALIST_3)
-{
-    Eterm local;
-    Eterm remote;
-    DistEntry *rdep;
-
-    local = BIF_ARG_1;
-    remote = BIF_ARG_3;
-
-    /* Check that remote is a remote process */
-    if (is_not_external_pid(remote))
-	goto error;
-
-    rdep = external_dist_entry(remote);
-    
-    if(rdep == erts_this_dist_entry)
-	goto error;
-
-    /* Check that local is local */
-    if (is_internal_pid(local)) {
-	Process *lp;
-	ErtsProcLocks lp_locks;
-	if (BIF_P->common.id == local) {
-	    lp_locks = ERTS_PROC_LOCKS_ALL;
-	    lp = BIF_P;
-	    erts_proc_lock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR);
-	}
-	else {
-	    lp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
-	    lp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
-			       local, lp_locks);
-	    if (!lp) {
-		BIF_RET(am_true); /* ignore */
-	    }
-	}
-	
-	(void) erts_send_exit_signal(BIF_P,
-				     remote,
-				     lp,
-				     &lp_locks,
-				     BIF_ARG_2,
-				     NIL,
-				     NULL,
-				     0);
-	if (lp == BIF_P)
-	    lp_locks &= ~ERTS_PROC_LOCK_MAIN;
-	erts_proc_unlock(lp, lp_locks);
-	if (lp == BIF_P) {
-	    erts_aint32_t state = erts_atomic32_read_acqb(&BIF_P->state);
-	    /*
-	     * We may have exited current process and may have to take action.
-	     */
-	    if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
-		if (state & ERTS_PSFLG_PENDING_EXIT)
-		    erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
-		ERTS_BIF_EXITED(BIF_P);
-	    }
-	}
-    }
-    else if (is_external_pid(local)
-	     && external_dist_entry(local) == erts_this_dist_entry) {
-	BIF_RET(am_true); /* ignore */
-    }
-    else
-	goto error;
-    BIF_RET(am_true);
-
- error:
-    BIF_ERROR(BIF_P, BADARG);
-}
-
 /**********************************************************************/
 /* node(Object) -> Node */
 
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 8bb915c76b..50e72ac42c 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -86,8 +86,6 @@
 #define DOP_SEND_SENDER_TT      23
 
 /* distribution trap functions */
-extern Export* dlink_trap;
-extern Export* dunlink_trap;
 extern Export* dmonitor_node_trap;
 extern Export* dgroup_leader_trap;
 extern Export* dexit_trap;
-- 
cgit v1.2.3


From da2935ce340cef5db1b5f589778eb20044796610 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 15 Aug 2017 17:12:02 +0200
Subject: Remove obsolete erlang:dexit/2

---
 erts/emulator/beam/atom.names | 1 -
 erts/emulator/beam/dist.c     | 8 +-------
 erts/emulator/beam/dist.h     | 1 -
 3 files changed, 1 insertion(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 3c06c26ba0..d755b5393b 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -199,7 +199,6 @@ atom debug_flags
 atom decimals
 atom default
 atom delay_trap
-atom dexit
 atom depth
 atom dgroup_leader
 atom dictionary
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 6e08c72289..e36ea9ba94 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -110,7 +110,6 @@ int erts_dist_buf_busy_limit;
 /* distribution trap functions */
 Export* dmonitor_node_trap = NULL;
 Export* dgroup_leader_trap = NULL;
-Export* dexit_trap = NULL;
 Export* dmonitor_p_trap = NULL;
 
 /* local variables */
@@ -633,7 +632,6 @@ void init_dist(void)
     /* Lookup/Install all references to trap functions */
     dmonitor_node_trap = trap_function(am_dmonitor_node,3);
     dgroup_leader_trap = trap_function(am_dgroup_leader,2);
-    dexit_trap = trap_function(am_dexit, 2);
     dmonitor_p_trap = trap_function(am_dmonitor_p, 2);
     dist_ctrl_put_data_trap = erts_export_put(am_erts_internal,
                                               am_dist_ctrl_put_data,
@@ -3108,9 +3106,6 @@ int distribution_info(fmtfn_t to, void *arg)	/* Called by break handler */
  ** loads functions pointer to trap_functions from module erlang.
  **    erlang:dmonitor_node/3
  **    erlang:dgroup_leader/2
- **    erlang:dexit/2
- **  -- are these needed ?
- **    dexit/1
  ***********************************************************************/
 
 BIF_RETTYPE setnode_2(BIF_ALIST_2)
@@ -3136,8 +3131,7 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2)
     /* Check that all trap functions are defined !! */
     if (dmonitor_node_trap->addressv[0] == NULL ||
 	dgroup_leader_trap->addressv[0] == NULL ||
-	dmonitor_p_trap->addressv[0] == NULL ||
-	dexit_trap->addressv[0] == NULL) {
+	dmonitor_p_trap->addressv[0] == NULL) {
 	goto error;
     }
 
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 50e72ac42c..339e4839ea 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -88,7 +88,6 @@
 /* distribution trap functions */
 extern Export* dmonitor_node_trap;
 extern Export* dgroup_leader_trap;
-extern Export* dexit_trap;
 extern Export* dmonitor_p_trap;
 
 typedef enum {
-- 
cgit v1.2.3


From 389e11b8b8a476ca73ca03a39ad7ec298dc99e83 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 18 Aug 2017 17:08:27 +0200
Subject: Remove obsolete erlang:dgroup_leader

---
 erts/emulator/beam/atom.names | 1 -
 erts/emulator/beam/dist.c     | 6 ------
 erts/emulator/beam/dist.h     | 1 -
 3 files changed, 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index d755b5393b..75a70c3716 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -200,7 +200,6 @@ atom decimals
 atom default
 atom delay_trap
 atom depth
-atom dgroup_leader
 atom dictionary
 atom dirty_bif_exception
 atom dirty_bif_result
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index e36ea9ba94..cd4125e404 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -109,7 +109,6 @@ int erts_dist_buf_busy_limit;
 
 /* distribution trap functions */
 Export* dmonitor_node_trap = NULL;
-Export* dgroup_leader_trap = NULL;
 Export* dmonitor_p_trap = NULL;
 
 /* local variables */
@@ -631,7 +630,6 @@ void init_dist(void)
 
     /* Lookup/Install all references to trap functions */
     dmonitor_node_trap = trap_function(am_dmonitor_node,3);
-    dgroup_leader_trap = trap_function(am_dgroup_leader,2);
     dmonitor_p_trap = trap_function(am_dmonitor_p, 2);
     dist_ctrl_put_data_trap = erts_export_put(am_erts_internal,
                                               am_dist_ctrl_put_data,
@@ -3103,9 +3101,6 @@ int distribution_info(fmtfn_t to, void *arg)	/* Called by break handler */
 /**********************************************************************
  ** Set the node name of current node fail if node already is set.
  ** setnode(name@host, Creation)
- ** loads functions pointer to trap_functions from module erlang.
- **    erlang:dmonitor_node/3
- **    erlang:dgroup_leader/2
  ***********************************************************************/
 
 BIF_RETTYPE setnode_2(BIF_ALIST_2)
@@ -3130,7 +3125,6 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2)
 
     /* Check that all trap functions are defined !! */
     if (dmonitor_node_trap->addressv[0] == NULL ||
-	dgroup_leader_trap->addressv[0] == NULL ||
 	dmonitor_p_trap->addressv[0] == NULL) {
 	goto error;
     }
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 339e4839ea..49f3eac340 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -87,7 +87,6 @@
 
 /* distribution trap functions */
 extern Export* dmonitor_node_trap;
-extern Export* dgroup_leader_trap;
 extern Export* dmonitor_p_trap;
 
 typedef enum {
-- 
cgit v1.2.3


From 9271fd39c48a121b6be79e7a44a524c883fafc41 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 8 Sep 2017 17:11:20 +0200
Subject: erts: Let send(_,_,[noconnect]) enqueue msg on pending connection.

The least bad behavior I think:

* We cannot return 'noconnect' as caller might already have enqueued
  monitor/link that never triggers.

* We cannot block waiting for connection as that can ruin latency
  when 'noconnect' is used to avoid blocking auto-connect
  (see gen_server and gen_statem).

But there might be users getting more cases of bad latency
waiting for a pendig connection, instead of a fast 'noconnect'.
---
 erts/emulator/beam/bif.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 8eb4d9e85b..30278cbe36 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2192,11 +2192,10 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx)
 	if (is_not_atom(tp[1]) || is_not_atom(tp[2]))
 	    return SEND_BADARG;
 	
-	/* sysname_to_connected_dist_entry will return NULL if there
-	   is no dist_entry or the dist_entry has no port,
+	/* erts_find_dist_entry will return NULL if there is no dist_entry
 	   but remote_send() will handle that. */
 
-	dep = erts_sysname_to_connected_dist_entry(tp[2]);
+	dep = erts_find_dist_entry(tp[2]);
 
 	if (dep == erts_this_dist_entry) {
 	    Eterm id;
-- 
cgit v1.2.3


From 5857245406584b8bfcbdf0be450ad494a992d5c8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 8 Sep 2017 17:11:43 +0200
Subject: erts: Fix auto-connect toward erl_interface/jinterface

---
 erts/emulator/beam/dist.c            | 63 +++++++++++++++++-------------------
 erts/emulator/beam/dist.h            |  2 +-
 erts/emulator/beam/erl_node_tables.h |  1 +
 erts/emulator/beam/external.c        | 44 +++++++++++++++++++------
 erts/emulator/beam/external.h        |  2 +-
 5 files changed, 66 insertions(+), 46 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index cd4125e404..6d2e907f18 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1879,33 +1879,32 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 
 	    if (ctx->flags & DFLAG_DIST_HDR_ATOM_CACHE) {
 		ctx->acmp = erts_get_atom_cache_map(ctx->c_p);
-		ctx->pass_through_size = 0;
+		ctx->max_finalize_prepend = 0;
 	    }
 	    else {
 		ctx->acmp = NULL;
-		ctx->pass_through_size = 3; /* SVERK rename */
+		ctx->max_finalize_prepend = 3;
 	    }
 
     #ifdef ERTS_DIST_MSG_DBG
-	    erts_fprintf(stderr, ">>%s CTL: %T\n", ctx->pass_through_size ? "P" : " ", ctx->ctl);
-        if (is_value(ctx->msg))
-            erts_fprintf(stderr, "    MSG: %T\n", ctx->msg);
+            erts_fprintf(stderr, ">> CTL: %T\n", ctx->ctl);
+            if (is_value(ctx->msg))
+                erts_fprintf(stderr, "    MSG: %T\n", ctx->msg);
     #endif
 
-	    ctx->data_size = ctx->pass_through_size;
+	    ctx->data_size = ctx->max_finalize_prepend;
 	    erts_reset_atom_cache_map(ctx->acmp);
 	    erts_encode_dist_ext_size(ctx->ctl, ctx->flags, ctx->acmp, &ctx->data_size);
 
-	    if (is_value(ctx->msg)) {
-		ctx->u.sc.wstack.wstart = NULL;
-		ctx->u.sc.flags = ctx->flags;
-		ctx->u.sc.level = 0;
-		ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_SIZE;
-	    } else {
-		ctx->phase = ERTS_DSIG_SEND_PHASE_ALLOC;
-	    }
-	    break;
+	    if (is_non_value(ctx->msg)) {
+                ctx->phase = ERTS_DSIG_SEND_PHASE_ALLOC;
+                break;
+            }
+            ctx->u.sc.wstack.wstart = NULL;
+            ctx->u.sc.flags = ctx->flags;
+            ctx->u.sc.level = 0;
 
+            ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_SIZE;
 	case ERTS_DSIG_SEND_PHASE_MSG_SIZE:
 	    if (erts_encode_dist_ext_size_int(ctx->msg, ctx, &ctx->data_size)) {
 		retval = ERTS_DSIG_SEND_CONTINUE;
@@ -1920,23 +1919,24 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 	    ctx->data_size += ctx->dhdr_ext_size;
 
 	    ctx->obuf = alloc_dist_obuf(ctx->data_size);
-	    ctx->obuf->ext_endp = &ctx->obuf->data[0] + ctx->pass_through_size + ctx->dhdr_ext_size;
+	    ctx->obuf->ext_endp = &ctx->obuf->data[0] + ctx->max_finalize_prepend + ctx->dhdr_ext_size;
 
 	    /* Encode internal version of dist header */
 	    ctx->obuf->extp = erts_encode_ext_dist_header_setup(ctx->obuf->ext_endp, ctx->acmp);
 	    /* Encode control message */
 	    erts_encode_dist_ext(ctx->ctl, &ctx->obuf->ext_endp, ctx->flags, ctx->acmp, NULL, NULL);
-	    if (is_value(ctx->msg)) {
-		ctx->u.ec.flags = ctx->flags;
-		ctx->u.ec.level = 0;
-		ctx->u.ec.wstack.wstart = NULL;
-		ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_ENCODE;
-	    } else {
-		ctx->phase = ERTS_DSIG_SEND_PHASE_FIN;
-	    }
-	    break;
+	    if (is_non_value(ctx->msg)) {
+                ctx->obuf->msg_start = NULL;
+                ctx->phase = ERTS_DSIG_SEND_PHASE_FIN;
+                break;
+            }
+            ctx->u.ec.flags = ctx->flags;
+            ctx->u.ec.level = 0;
+            ctx->u.ec.wstack.wstart = NULL;
+            ctx->obuf->msg_start = ctx->obuf->ext_endp;
 
-	case ERTS_DSIG_SEND_PHASE_MSG_ENCODE:
+            ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_ENCODE;
+        case ERTS_DSIG_SEND_PHASE_MSG_ENCODE:
 	    if (erts_encode_dist_ext(ctx->msg, &ctx->obuf->ext_endp, ctx->flags, ctx->acmp, &ctx->u.ec, &ctx->reds)) {
 		retval = ERTS_DSIG_SEND_CONTINUE;
 		goto done;
@@ -1949,7 +1949,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 	    int resume = 0;
 
 	    ASSERT(ctx->obuf->extp < ctx->obuf->ext_endp);
-	    ASSERT(&ctx->obuf->data[0] <= ctx->obuf->extp - ctx->pass_through_size);
+	    ASSERT(&ctx->obuf->data[0] <= ctx->obuf->extp - ctx->max_finalize_prepend);
 	    ASSERT(ctx->obuf->ext_endp <= &ctx->obuf->data[0] + ctx->data_size);
 
 	    ctx->data_size = ctx->obuf->ext_endp - ctx->obuf->extp;
@@ -2308,9 +2308,7 @@ erts_dist_command(Port *prt, int reds_limit)
 	    ob = oq.first;
 	    ASSERT(ob);
 	    do {
-		ob->extp = erts_encode_ext_dist_header_finalize(ob->extp,
-								dep->cache,
-								flags);
+		erts_encode_ext_dist_header_finalize(ob, dep->cache, flags);
 		ASSERT(&ob->data[0] <= ob->extp && ob->extp < ob->ext_endp);
 		reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE;
 		preempt = reds > reds_limit;
@@ -2350,10 +2348,7 @@ erts_dist_command(Port *prt, int reds_limit)
 	while (oq.first && !preempt) {
 	    ErtsDistOutputBuf *fob;
 	    Uint size;
-	    oq.first->extp
-		= erts_encode_ext_dist_header_finalize(oq.first->extp,
-						       dep->cache,
-						       flags);
+            erts_encode_ext_dist_header_finalize(oq.first, dep->cache, flags);
 	    reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE;
 	    ASSERT(&oq.first->data[0] <= oq.first->extp
 		   && oq.first->extp < oq.first->ext_endp);
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 49f3eac340..505d510473 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -405,7 +405,7 @@ struct erts_dsig_send_context {
     Eterm ctl;
     Eterm msg;
     int force_busy;
-    Uint32 pass_through_size;
+    Uint32 max_finalize_prepend;
     Uint data_size, dhdr_ext_size;
     ErtsAtomCacheMap *acmp;
     ErtsDistOutputBuf *obuf;
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index d012f4b2cf..af42814b8b 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -87,6 +87,7 @@ struct ErtsDistOutputBuf_ {
     ErtsDistOutputBuf *next;
     byte *extp;
     byte *ext_endp;
+    byte *msg_start;
     byte data[1];
 };
 
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index c49e2f617a..d8f52ceb57 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -351,41 +351,63 @@ byte *erts_encode_ext_dist_header_setup(byte *ctl_ext, ErtsAtomCacheMap *acmp)
 
 #define PASS_THROUGH 'p'        /* This code should go */
 
-byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache, Uint32 dflags)
+void erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob,
+                                          ErtsAtomCache *cache,
+                                          Uint32 dflags)
 {
     byte *ip;
     byte instr_buf[(2+4)*ERTS_ATOM_CACHE_SIZE];
     int ci, sz;
     byte dist_hdr_flags;
     int long_atoms;
-    register byte *ep = ext;
+    register byte *ep = ob->extp;
     ASSERT(dflags & DFLAG_UTF8_ATOMS);
 
+    /*
+     * The buffer can have three different layouts at this point depending on
+     * what was known when encoded:
+     *
+     * Pending connection: CtrlTerm [, MsgTerm]
+     * With atom cache   : VERSION_MAGIC, DIST_HEADER, ..., CtrlTerm [, MsgTerm]
+     * No atom cache     : VERSION_MAGIC, CtrlTerm [, VERSION_MAGIC, MsgTerm]
+     */
+
     if (ep[0] != VERSION_MAGIC) {
+        /*
+         * Was encoded without atom cache toward pending connection.
+         */
         ASSERT(ep[0] == SMALL_TUPLE_EXT || ep[0] == LARGE_TUPLE_EXT);
         if (dflags & DFLAG_DIST_HDR_ATOM_CACHE) {
             /*
-             * Encoded without atom cache (toward pending connection)
-             * but receiver wants dist header. Let's prepend an empty one.
+             * Receiver wants dist header. Let's prepend an empty one.
              */
             *--ep = 0; /* NumberOfAtomCacheRefs */
             *--ep = DIST_HEADER;
             *--ep = VERSION_MAGIC;
         }
         else {
-            /* Node without atom cache, 'pass through' needed */
-
-            ASSERT(!"SVERK: Must insert VERSION_MAGIC's");
+            /*
+             * Primitive receiver without atom cache (erl_interface/jinterface).
+             * Must prepend VERSION_MAGIC to both ctrl and message
+             * And add PASSTHROUGH.
+             */
+            if (ob->msg_start) {
+                ASSERT(ep < ob->msg_start && ob->msg_start < ob->ext_endp);
+                sys_memmove(ep-1, ep, (ob->msg_start - ep));
+                ob->msg_start[-1] = VERSION_MAGIC;
+                --ep;
+            }
+            *--ep = VERSION_MAGIC;
             *--ep = PASS_THROUGH;
         }
-        return ep;
+        goto done;
     }
     else if (ep[1] != DIST_HEADER) {
         ASSERT(ep[1] == SMALL_TUPLE_EXT || ep[1] == LARGE_TUPLE_EXT);
         ASSERT(!(dflags & DFLAG_DIST_HDR_ATOM_CACHE));
         /* Node without atom cache, 'pass through' needed */
         *--ep = PASS_THROUGH;
-        return ep;
+        goto done;
     }
 
     dist_hdr_flags = ep[2];
@@ -510,7 +532,9 @@ byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache, Uint
     put_int8(ci, ep);
     *--ep = DIST_HEADER;
     *--ep = VERSION_MAGIC;
-    return ep;
+done:
+    ASSERT(ep >= ob->data);
+    ob->extp = ep;
 }
 
 int erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp,
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index 49950c4aad..11044b17ef 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -154,7 +154,7 @@ void erts_finalize_atom_cache_map(ErtsAtomCacheMap *, Uint32);
 
 Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *);
 byte *erts_encode_ext_dist_header_setup(byte *, ErtsAtomCacheMap *);
-byte *erts_encode_ext_dist_header_finalize(byte *, ErtsAtomCache *, Uint32);
+void erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf*, ErtsAtomCache *, Uint32);
 struct erts_dsig_send_context;
 int erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap*, Uint* szp);
 int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp);
-- 
cgit v1.2.3


From d736e87ff94ab8191f33dca55516e6c1d440b915 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 24 Aug 2017 17:20:21 +0200
Subject: Add optimistic DFLAG_DIST_HOPEFULLY for pending connections

to avoid tuple fallbacks for export funs and bitstrings.

ToDo: Re-encode if receiver turn out to be erl_interface/jinterface.
---
 erts/emulator/beam/dist.c                 |  2 +-
 erts/emulator/beam/dist.h                 |  9 ++++--
 erts/emulator/beam/erl_node_tables.c      |  2 +-
 erts/emulator/beam/external.c             |  4 +--
 erts/emulator/test/distribution_SUITE.erl | 53 +++++++++++++++++++++++++++++--
 5 files changed, 62 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 6d2e907f18..7877865ad4 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -3443,7 +3443,7 @@ BIF_RETTYPE new_connection_id_1(BIF_ALIST_1)
 	conn_id = dep->connection_id;
     else if (dep->status == 0) {
 	dep->status = ERTS_DE_SFLG_PENDING;
-	dep->flags = DFLAG_DIST_MANDATORY | DFLAG_PENDING_CONNECTION;
+	dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY);
 	dep->connection_id++;
 	dep->connection_id &= ERTS_DIST_CON_ID_MASK;
 	conn_id = dep->connection_id;
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 505d510473..f9a2037687 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -45,7 +45,7 @@
 #define DFLAG_MAP_TAG             0x20000
 #define DFLAG_BIG_CREATION        0x40000
 #define DFLAG_SEND_SENDER         0x80000
-#define DFLAG_PENDING_CONNECTION  0x100000
+#define DFLAG_NO_MAGIC            0x100000
 
 /* Mandatory flags for distribution (sync with dist_util.erl) */
 #define DFLAG_DIST_MANDATORY (DFLAG_EXTENDED_REFERENCES         \
@@ -53,6 +53,11 @@
 			      | DFLAG_UTF8_ATOMS                \
 			      | DFLAG_NEW_FUN_TAGS)
 
+/* Additional optimistic flags when encoding toward pending connection */
+#define DFLAG_DIST_HOPEFULLY (DFLAG_NO_MAGIC                    \
+                              | DFLAG_EXPORT_PTR_TAG            \
+                              | DFLAG_BIT_BINARIES)
+
 /* All flags that should be enabled when term_to_binary/1 is used. */
 #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES	\
 			       | DFLAG_NEW_FUN_TAGS		\
@@ -206,7 +211,7 @@ retry:
 	    Eterm msg, conn_id;
 
 	    dep->status = ERTS_DE_SFLG_PENDING;
-	    dep->flags = DFLAG_DIST_MANDATORY | DFLAG_PENDING_CONNECTION;
+	    dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY);
 	    dep->connection_id++;
 	    dep->connection_id &= ERTS_DIST_CON_ID_MASK;
             conn_id = make_small(dep->connection_id);
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index dd607f438e..2a33766ac8 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -628,7 +628,7 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags)
 	dep->connection_id &= ERTS_DIST_CON_ID_MASK;
     }
     dep->status |= ERTS_DE_SFLG_CONNECTED;
-    dep->flags = flags & ~DFLAG_PENDING_CONNECTION;
+    dep->flags = flags & ~DFLAG_NO_MAGIC;
     dep->cid = cid;
     erts_atomic_set_nob(&dep->input_handler,
                             (erts_aint_t) cid);
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index d8f52ceb57..bfac48580c 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -561,7 +561,7 @@ int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx
 	return -1;
     } else {
 #ifndef ERTS_DEBUG_USE_DIST_SEP
-	if (!(ctx->flags & (DFLAG_DIST_HDR_ATOM_CACHE | DFLAG_PENDING_CONNECTION)))
+	if (!(ctx->flags & (DFLAG_DIST_HDR_ATOM_CACHE | DFLAG_NO_MAGIC)))
 #endif
 	    sz++ /* VERSION_MAGIC */;
 
@@ -593,7 +593,7 @@ int erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap
 {
     if (!ctx || !ctx->wstack.wstart) {
     #ifndef ERTS_DEBUG_USE_DIST_SEP
-	if (!(flags & (DFLAG_DIST_HDR_ATOM_CACHE | DFLAG_PENDING_CONNECTION)))
+	if (!(flags & (DFLAG_DIST_HDR_ATOM_CACHE | DFLAG_NO_MAGIC)))
     #endif
 	    *(*ext)++ = VERSION_MAGIC;
     }
diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index fa683f363f..f1831b7c45 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -41,6 +41,7 @@
 -export([all/0, suite/0, groups/0,
          ping/1, bulk_send_small/1,
          group_leader/1,
+         optimistic_dflags/1,
          bulk_send_big/1, bulk_send_bigbig/1,
          local_send_small/1, local_send_big/1,
          local_send_legal/1, link_to_busy/1, exit_to_busy/1,
@@ -66,6 +67,7 @@
 %% Internal exports.
 -export([sender/3, receiver2/2, dummy_waiter/0, dead_process/0,
          group_leader_1/1,
+         optimistic_dflags_echo/0, optimistic_dflags_sender/1,
          roundtrip/1, bounce/1, do_dist_auto_connect/1, inet_rpc_server/1,
          dist_parallel_sender/3, dist_parallel_receiver/0,
          dist_evil_parallel_receiver/0]).
@@ -80,6 +82,7 @@ suite() ->
 all() ->
     [ping, {group, bulk_send}, {group, local_send},
      group_leader,
+     optimistic_dflags,
      link_to_busy, exit_to_busy, lost_exit, link_to_dead,
      link_to_dead_new_node,
      ref_port_roundtrip, nil_roundtrip, stop_dist,
@@ -166,12 +169,58 @@ group_leader_1(Node2) ->
     ?Line {ExtPid, group_leader, GL2} = receive_one(),
     ok.
 
-receive_one() ->
-    receive M -> M after 1000 -> timeout end.
+%% Test optimistic distribution flags toward pending connections (DFLAG_DIST_HOPEFULLY)
+optimistic_dflags(Config) when is_list(Config) ->
+    ?Line Sender = start_relay_node(optimistic_dflags_sender, []),
+    ?Line Echo = start_relay_node(optimistic_dflags_echo, []),
+    try
+        ?Line {ok, ok} = do_inet_rpc(Echo, ?MODULE, optimistic_dflags_echo, []),
+
+        ?Line EchoNode = inet_rpc_nodename(Echo),
+        ?Line {ok, ok} = do_inet_rpc(Sender, ?MODULE, optimistic_dflags_sender, [EchoNode])
+    after
+        ?Line stop_relay_node(Sender),
+        ?Line stop_relay_node(Echo)
+    end,
+    ok.
+
+optimistic_dflags_echo() ->
+    P = spawn(fun F() ->
+                      receive {From, Term} ->
+                              From ! {self(), Term}
+                      end,
+                      F()
+              end),
+    register(optimistic_dflags_echo, P),
+    optimistic_dflags_echo ! {self(), hello},
+    {P, hello} = receive_one(),
+    ok.
+
+optimistic_dflags_sender(EchoNode) ->
+    ?Line net_kernel:monitor_nodes(true),
 
+    optimistic_dflags_do(EchoNode, <<1:1>>),
+    optimistic_dflags_do(EchoNode, fun lists:map/2),
+    ok.
 
+optimistic_dflags_do(EchoNode, Term) ->
+    ?Line {optimistic_dflags_echo, EchoNode} ! {self(), Term},
+    ?Line {nodeup, EchoNode} = receive_one(),
+    ?Line {EchoPid, Term} = receive_one(),
+    %% repeat with pid destination
+    ?Line net_kernel:disconnect(EchoNode),
+    ?Line {nodedown, EchoNode} = receive_one(),
+    ?Line EchoPid ! {self(), Term},
+    ?Line {nodeup, EchoNode} = receive_one(),
+    ?Line {EchoPid, Term} = receive_one(),
+
+    ?Line net_kernel:disconnect(EchoNode),
+    ?Line {nodedown, EchoNode} = receive_one(),
+    ok.
 
 
+receive_one() ->
+    receive M -> M after 1000 -> timeout end.
 
 
 bulk_send_small(Config) when is_list(Config) ->
-- 
cgit v1.2.3


From 42dadd12ac6b6977513a6edd73ff6137488c4a4a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 6 Sep 2017 20:38:52 +0200
Subject: erts: Ensure enc_term_int() always do progress

even when reds <= 1

Removed micro optimization for first fun variable
to make things simpler.
---
 erts/emulator/beam/external.c | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index bfac48580c..8408d7dd12 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2554,8 +2554,6 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	    break;
 	}
 
-    L_jump_start:
-
 	if (ctx && --r <= 0) {
 	    *reds = 0;
 	    ctx->obj = obj;
@@ -2563,6 +2561,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 	    WSTACK_SAVE(s, &ctx->wstack);
 	    return -1;
 	}
+
+    L_jump_start:
 	switch(tag_val_def(obj)) {
 	case NIL_DEF:
 	    *ep++ = NIL_EXT;
@@ -2945,13 +2945,9 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
                 ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags, off_heap);
                 ep = enc_pid(acmp, funp->creator, ep, dflags);
 
-		for (ei = funp->num_free-1; ei > 0; ei--) {
+		for (ei = funp->num_free-1; ei >= 0; ei--) {
 		    WSTACK_PUSH2(s, ENC_TERM, (UWord) funp->env[ei]);
 		}
-		if (funp->num_free != 0) {
-		    obj = funp->env[0];
-		    goto L_jump_start;
-		}
 	    }
 	    break;
 	}
-- 
cgit v1.2.3


From e4d28f70133af9b3b8c320fe5e70e2086830874d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 25 Aug 2017 18:46:58 +0200
Subject: erts: Transcode tuple fallbacks

When finalizing outgoing distribution messages
we transcode them into using tuple fallbacks if the
receiver does not support bitstrings and export-funs.

This can only happen if the message was first encoded toward
a pending connection when the receiver was unknown.
It's an optimistic approach optmimized for modern beam nodes,
that expect real bitstrings and funs (since <R13).

Only erl_interface/jinterface lack this support.
---
 erts/emulator/beam/dist.c            | 159 ++++++++++++------------
 erts/emulator/beam/erl_alloc.types   |   1 +
 erts/emulator/beam/erl_node_tables.c |   1 +
 erts/emulator/beam/erl_node_tables.h |   3 +
 erts/emulator/beam/external.c        | 227 +++++++++++++++++++++++++++++++----
 erts/emulator/beam/external.h        |   4 +-
 6 files changed, 289 insertions(+), 106 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 7877865ad4..a5df92bf9a 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -648,6 +648,7 @@ alloc_dist_obuf(Uint size)
     obuf = (ErtsDistOutputBuf *) &bin->orig_bytes[0];
 #ifdef DEBUG
     obuf->dbg_pattern = ERTS_DIST_OUTPUT_BUF_DBG_PATTERN;
+    obuf->alloc_endp = obuf->data + size;
     ASSERT(bin == ErtsDistOutputBuf2Binary(obuf));
 #endif
     return obuf;
@@ -692,6 +693,7 @@ static ErtsDistOutputBuf* clear_de_out_queues(DistEntry* dep)
     dep->tmp_out_queue.last = NULL;
     dep->finalized_out_queue.first = NULL;
     dep->finalized_out_queue.last = NULL;
+
     return obuf;
 }
 
@@ -755,6 +757,8 @@ static void clear_dist_entry(DistEntry *dep)
     delete_cache(cache);
 
     free_de_out_queues(dep, obuf);
+    if (dep->transcode_ctx)
+        transcode_free_ctx(dep);
 }
 
 int erts_dsend_context_dtor(Binary* ctx_bin)
@@ -2208,7 +2212,6 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf)
 #endif
 
 #define ERTS_PORT_REDS_DIST_CMD_START 5
-#define ERTS_PORT_REDS_DIST_CMD_FINALIZE 3
 #define ERTS_PORT_REDS_DIST_CMD_EXIT 200
 #define ERTS_PORT_REDS_DIST_CMD_RESUMED 5
 #define ERTS_PORT_REDS_DIST_CMD_DATA(SZ) \
@@ -2217,9 +2220,9 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf)
    : ((((Sint) (SZ)) >> 10) & ((Sint) ERTS_PORT_REDS_MASK__)))
 
 int
-erts_dist_command(Port *prt, int reds_limit)
+erts_dist_command(Port *prt, int initial_reds)
 {
-    Sint reds = ERTS_PORT_REDS_DIST_CMD_START;
+    Sint reds = initial_reds - ERTS_PORT_REDS_DIST_CMD_START;
     Uint32 status;
     Uint32 flags;
     Sint qsize, obufsize = 0;
@@ -2241,9 +2244,12 @@ erts_dist_command(Port *prt, int reds_limit)
 
     if (status & ERTS_DE_SFLG_EXITING) {
 	erts_deliver_port_exit(prt, prt->common.id, am_killed, 0, 1);
-	return reds + ERTS_PORT_REDS_DIST_CMD_EXIT;
+        reds -= ERTS_PORT_REDS_DIST_CMD_EXIT;
+	return initial_reds - reds;
     }
 
+    ASSERT(!(status & ERTS_DE_SFLG_PENDING));
+
     ASSERT(send);
 
     /*
@@ -2268,7 +2274,7 @@ erts_dist_command(Port *prt, int reds_limit)
 
     sched_flags = erts_atomic32_read_nob(&prt->sched.flags);
 
-    if (reds > reds_limit)
+    if (reds < 0)
 	goto preempted;
 
     if (!(sched_flags & ERTS_PTS_FLG_BUSY_PORT) && foq.first) {
@@ -2283,13 +2289,13 @@ erts_dist_command(Port *prt, int reds_limit)
             erts_fprintf(stderr, ">> ");
             bw(foq.first->extp, size);
 #endif
-            reds += ERTS_PORT_REDS_DIST_CMD_DATA(size);
-            fob = foq.first;
-            obufsize += size_obuf(fob);
-            foq.first = foq.first->next;
-            free_dist_obuf(fob);
+	    reds -= ERTS_PORT_REDS_DIST_CMD_DATA(size);
+	    fob = foq.first;
+	    obufsize += size_obuf(fob);
+	    foq.first = foq.first->next;
+	    free_dist_obuf(fob);
 	    sched_flags = erts_atomic32_read_nob(&prt->sched.flags);
-	    preempt = reds > reds_limit || (sched_flags & ERTS_PTS_FLG_EXIT);
+	    preempt = reds < 0 || (sched_flags & ERTS_PTS_FLG_EXIT);
 	    if (sched_flags & ERTS_PTS_FLG_BUSY_PORT)
 		break;
 	} while (foq.first && !preempt);
@@ -2302,44 +2308,42 @@ erts_dist_command(Port *prt, int reds_limit)
     if (sched_flags & ERTS_PTS_FLG_BUSY_PORT) {
 	if (oq.first) {
 	    ErtsDistOutputBuf *ob;
-	    int preempt;
+            ErtsDistOutputBuf *last_finalized = NULL;
 	finalize_only:
-	    preempt = 0;
 	    ob = oq.first;
 	    ASSERT(ob);
 	    do {
-		erts_encode_ext_dist_header_finalize(ob, dep->cache, flags);
-		ASSERT(&ob->data[0] <= ob->extp && ob->extp < ob->ext_endp);
-		reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE;
-		preempt = reds > reds_limit;
-		if (preempt)
-		    break;
-		ob = ob->next;
+		reds = erts_encode_ext_dist_header_finalize(ob, dep, flags, reds);
+                if (reds >= 0) {
+                    last_finalized  = ob;
+                    ob = ob->next;
+                }
 	    } while (ob);
-	    /*
-	     * At least one buffer was finalized; if we got preempted,
-	     * ob points to the last buffer that we finalized.
-	     */
-	    if (foq.last)
-		foq.last->next = oq.first;
-	    else
-		foq.first = oq.first;
-	    if (!preempt) {
-		/* All buffers finalized */
-		foq.last = oq.last;
-		oq.first = oq.last = NULL;
-	    }
-	    else {
-		/* Not all buffers finalized; split oq. */
-		foq.last = ob;
-		oq.first = ob->next;
-		if (oq.first)
-		    ob->next = NULL;
-		else
-		    oq.last = NULL;
-	    }
-	    if (preempt)
-		goto preempted;
+            if (last_finalized) {
+                /*
+                 * At least one buffer was finalized; if we got preempted,
+                 * ob points to the next buffer to continue finalize.
+                 */
+                if (foq.last)
+                    foq.last->next = oq.first;
+                else
+                    foq.first = oq.first;
+                foq.last = last_finalized;
+                if (!ob) {
+                    /* All buffers finalized */
+                    ASSERT(foq.last == oq.last);
+                    ASSERT(foq.last->next == NULL);
+                    oq.first = oq.last = NULL;
+                }
+                else {
+                    /* Not all buffers finalized; split oq. */
+                    ASSERT(foq.last->next == ob);
+                    foq.last->next = NULL;
+                    oq.first = ob;
+                }
+            }
+            if (reds <= 0)
+                goto preempted;
 	}
     }
     else {
@@ -2348,8 +2352,11 @@ erts_dist_command(Port *prt, int reds_limit)
 	while (oq.first && !preempt) {
 	    ErtsDistOutputBuf *fob;
 	    Uint size;
-            erts_encode_ext_dist_header_finalize(oq.first, dep->cache, flags);
-	    reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE;
+            reds = erts_encode_ext_dist_header_finalize(oq.first, dep, flags, reds);
+            if (reds < 0) {
+                preempt = 1;
+                break;
+            }
 	    ASSERT(&oq.first->data[0] <= oq.first->extp
 		   && oq.first->extp < oq.first->ext_endp);
 	    size = (*send)(prt, oq.first);
@@ -2359,13 +2366,13 @@ erts_dist_command(Port *prt, int reds_limit)
             erts_fprintf(stderr, ">> ");
             bw(oq.first->extp, size);
 #endif
-            reds += ERTS_PORT_REDS_DIST_CMD_DATA(size);
-            fob = oq.first;
-            obufsize += size_obuf(fob);
-            oq.first = oq.first->next;
-            free_dist_obuf(fob);
+	    reds -= ERTS_PORT_REDS_DIST_CMD_DATA(size);
+	    fob = oq.first;
+	    obufsize += size_obuf(fob);
+	    oq.first = oq.first->next;
+	    free_dist_obuf(fob);
 	    sched_flags = erts_atomic32_read_nob(&prt->sched.flags);
-	    preempt = reds > reds_limit || (sched_flags & ERTS_PTS_FLG_EXIT);
+	    preempt = reds <= 0 || (sched_flags & ERTS_PTS_FLG_EXIT);
 	    if ((sched_flags & ERTS_PTS_FLG_BUSY_PORT) && oq.first && !preempt)
 		goto finalize_only;
 	}
@@ -2405,7 +2412,7 @@ erts_dist_command(Port *prt, int reds_limit)
 	    erts_mtx_unlock(&dep->qlock);
 
 	    resumed = erts_resume_processes(suspendees);
-	    reds += resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED;
+	    reds -= resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED;
 	}
 	else
 	    erts_mtx_unlock(&dep->qlock);
@@ -2428,8 +2435,7 @@ erts_dist_command(Port *prt, int reds_limit)
 	erts_mtx_unlock(&dep->qlock);
     }
 
-    ASSERT(foq.first || !foq.last);
-    ASSERT(!foq.first || foq.last);
+    ASSERT(!!foq.first == !!foq.last);
     ASSERT(!dep->finalized_out_queue.first);
     ASSERT(!dep->finalized_out_queue.last);
 
@@ -2439,10 +2445,10 @@ erts_dist_command(Port *prt, int reds_limit)
     }
 
      /* Avoid wrapping reduction counter... */
-    if (reds > INT_MAX/2)
-	reds = INT_MAX/2;
+    if (reds < INT_MIN/2)
+	reds = INT_MIN/2;
 
-    return reds;
+    return initial_reds - reds;
 
  preempted:
     /*
@@ -2450,8 +2456,7 @@ erts_dist_command(Port *prt, int reds_limit)
      * since last call to driver.
      */
 
-    ASSERT(oq.first || !oq.last);
-    ASSERT(!oq.first || oq.last);
+    ASSERT(!!oq.first == !!oq.last);
 
     if (sched_flags & ERTS_PTS_FLG_EXIT) {
 	/*
@@ -2475,14 +2480,11 @@ erts_dist_command(Port *prt, int reds_limit)
 
 	foq.first = NULL;
 	foq.last = NULL;
-
-/* SVERK Hmmm....
 #ifdef DEBUG
 	erts_mtx_lock(&dep->qlock);
 	ASSERT(erts_atomic_read_nob(&dep->qsize) == obufsize);
 	erts_mtx_unlock(&dep->qlock);
 #endif
-*/
     }
     else {
 	if (oq.first) {
@@ -2772,7 +2774,8 @@ BIF_RETTYPE
 dist_ctrl_get_data_1(BIF_ALIST_1)
 {
     DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P);
-    int reds = 1;
+    const Sint initial_reds = ERTS_BIF_REDS_LEFT(BIF_P);
+    Sint reds = initial_reds;
     ErtsDistOutputBuf *obuf;
     Eterm *hp;
     ProcBin *pb;
@@ -2803,6 +2806,7 @@ dist_ctrl_get_data_1(BIF_ALIST_1)
     {
         if (!dep->tmp_out_queue.first) {
             ASSERT(!dep->tmp_out_queue.last);
+            ASSERT(!dep->transcode_ctx);
             qsize = erts_atomic_read_acqb(&dep->qsize);
             if (qsize > 0) {
                 erts_mtx_lock(&dep->qlock);
@@ -2820,21 +2824,18 @@ dist_ctrl_get_data_1(BIF_ALIST_1)
             erts_de_runlock(dep);
             BIF_RET(am_none);
         }
-        else {
-            obuf = dep->tmp_out_queue.first;
-            dep->tmp_out_queue.first = obuf->next;
-            if (!obuf->next)
-                dep->tmp_out_queue.last = NULL;
+
+        obuf = dep->tmp_out_queue.first;
+        reds = erts_encode_ext_dist_header_finalize(obuf, dep, dep->flags, reds);
+        if (reds < 0) {
+            erts_de_runlock(dep);
+            ERTS_BIF_YIELD1(bif_export[BIF_dist_ctrl_get_data_1],
+                            BIF_P, BIF_ARG_1);
         }
 
-        obuf->extp = erts_encode_ext_dist_header_finalize(obuf->extp,
-                                                          dep->cache,
-                                                          dep->flags);
-        reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE;
-        if (!(dep->flags & DFLAG_DIST_HDR_ATOM_CACHE))
-            *--obuf->extp = PASS_THROUGH; /* 'pass through' needed */
-        ASSERT(&obuf->data[0] <= obuf->extp
-               && obuf->extp < obuf->ext_endp);
+        dep->tmp_out_queue.first = obuf->next;
+        if (!obuf->next)
+            dep->tmp_out_queue.last = NULL;
     }
 
     erts_atomic64_inc_nob(&dep->out);
@@ -2862,11 +2863,11 @@ dist_ctrl_get_data_1(BIF_ALIST_1)
         erts_mtx_unlock(&dep->qlock);
         if (resume_procs) {
             int resumed = erts_resume_processes(resume_procs);
-            reds += resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED;
+            reds -= resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED;
         }
     }
 
-    BIF_RET2(make_binary(pb), reds);
+    BIF_RET2(make_binary(pb), (initial_reds - reds));
 }
 
 void
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 2960272eab..c6cc5c78b3 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -261,6 +261,7 @@ type	MINDIRECTION	FIXED_SIZE	SYSTEM		magic_indirection
 type	BINARY_FIND	SHORT_LIVED	PROCESSES	binary_find
 type  	OPEN_PORT_ENV 	TEMPORARY	SYSTEM		open_port_env
 type	CRASH_DUMP	STANDARD	SYSTEM		crash_dump
+type	DIST_TRANSCODE  SHORT_LIVED	SYSTEM		dist_transcode_context
 
 type	THR_Q_EL	STANDARD   	SYSTEM		thr_q_element
 type	THR_Q_EL_SL	FIXED_SIZE	SYSTEM		sl_thr_q_element
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 2a33766ac8..00e8306510 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -204,6 +204,7 @@ dist_table_alloc(void *dep_tmpl)
     erts_port_task_handle_init(&dep->dist_cmd);
     dep->send				= NULL;
     dep->cache				= NULL;
+    dep->transcode_ctx                  = NULL;
 
     /* Link in */
 
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index af42814b8b..afa83f8b46 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -83,6 +83,7 @@ typedef struct ErtsDistOutputBuf_ ErtsDistOutputBuf;
 struct ErtsDistOutputBuf_ {
 #ifdef DEBUG
     Uint dbg_pattern;
+    byte *alloc_endp;
 #endif
     ErtsDistOutputBuf *next;
     byte *extp;
@@ -156,6 +157,8 @@ typedef struct dist_entry_ {
     struct cache* cache;	/* The atom cache */
 
     ErtsThrPrgrLaterOp later_op;
+
+    struct transcode_context* transcode_ctx;
 } DistEntry;
 
 typedef struct erl_node_ {
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 8408d7dd12..f2e6399ad7 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -122,6 +122,9 @@ static int encode_size_struct_int(struct TTBSizeContext_*, ErtsAtomCacheMap *acm
 
 static Export binary_to_term_trap_export;
 static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1);
+static Sint transcode_dist_obuf(ErtsDistOutputBuf*, DistEntry*, Uint32 dflags, Sint reds);
+
+
 
 void erts_init_external(void) {
     erts_init_trap_export(&term_to_binary_trap_export,
@@ -349,11 +352,13 @@ byte *erts_encode_ext_dist_header_setup(byte *ctl_ext, ErtsAtomCacheMap *acmp)
     }
 }
 
-#define PASS_THROUGH 'p'        /* This code should go */
 
-void erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob,
-                                          ErtsAtomCache *cache,
-                                          Uint32 dflags)
+#define PASS_THROUGH 'p'
+
+Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob,
+                                          DistEntry* dep,
+                                          Uint32 dflags,
+                                          Sint reds)
 {
     byte *ip;
     byte instr_buf[(2+4)*ERTS_ATOM_CACHE_SIZE];
@@ -364,7 +369,7 @@ void erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob,
     ASSERT(dflags & DFLAG_UTF8_ATOMS);
 
     /*
-     * The buffer can have three different layouts at this point depending on
+     * The buffer can have different layouts at this point depending on
      * what was known when encoded:
      *
      * Pending connection: CtrlTerm [, MsgTerm]
@@ -372,34 +377,28 @@ void erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob,
      * No atom cache     : VERSION_MAGIC, CtrlTerm [, VERSION_MAGIC, MsgTerm]
      */
 
-    if (ep[0] != VERSION_MAGIC) {
+    if (ep[0] != VERSION_MAGIC || dep->transcode_ctx) {
         /*
          * Was encoded without atom cache toward pending connection.
          */
         ASSERT(ep[0] == SMALL_TUPLE_EXT || ep[0] == LARGE_TUPLE_EXT);
+
+        if (~dflags & (DFLAG_BIT_BINARIES | DFLAG_EXPORT_PTR_TAG
+                       | DFLAG_DIST_HDR_ATOM_CACHE)) {
+            reds = transcode_dist_obuf(ob, dep, dflags, reds);
+            if (reds < 0)
+                return reds;
+            ep = ob->extp;
+        }
         if (dflags & DFLAG_DIST_HDR_ATOM_CACHE) {
             /*
-             * Receiver wants dist header. Let's prepend an empty one.
+             * Encoding was done without atom caching but receiver expects
+             * a dist header, so we prepend an empty one.
              */
             *--ep = 0; /* NumberOfAtomCacheRefs */
             *--ep = DIST_HEADER;
             *--ep = VERSION_MAGIC;
         }
-        else {
-            /*
-             * Primitive receiver without atom cache (erl_interface/jinterface).
-             * Must prepend VERSION_MAGIC to both ctrl and message
-             * And add PASSTHROUGH.
-             */
-            if (ob->msg_start) {
-                ASSERT(ep < ob->msg_start && ob->msg_start < ob->ext_endp);
-                sys_memmove(ep-1, ep, (ob->msg_start - ep));
-                ob->msg_start[-1] = VERSION_MAGIC;
-                --ep;
-            }
-            *--ep = VERSION_MAGIC;
-            *--ep = PASS_THROUGH;
-        }
         goto done;
     }
     else if (ep[1] != DIST_HEADER) {
@@ -436,6 +435,7 @@ void erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob,
 			 / sizeof(Uint32))+1];
 	register Uint32 flgs;
 	int iix, flgs_bytes, flgs_buf_ix, used_half_bytes;
+        ErtsAtomCache* cache = dep->cache;
 #ifdef DEBUG
 	int tot_used_half_bytes;
 #endif
@@ -527,14 +527,16 @@ void erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob,
 		break;
 	    }
 	}
+        reds -= 3; /*was ERTS_PORT_REDS_DIST_CMD_FINALIZE*/
     }
     --ep;
     put_int8(ci, ep);
     *--ep = DIST_HEADER;
     *--ep = VERSION_MAGIC;
 done:
-    ASSERT(ep >= ob->data);
     ob->extp = ep;
+    ASSERT(&ob->data[0] <= ob->extp && ob->extp < ob->ext_endp);
+    return reds < 0 ? 0 : reds;
 }
 
 int erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp,
@@ -545,7 +547,7 @@ int erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp,
 	return -1;
     } else {
 #ifndef ERTS_DEBUG_USE_DIST_SEP
-	if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
+	if (!(flags & (DFLAG_DIST_HDR_ATOM_CACHE | DFLAG_NO_MAGIC)))
 #endif
 	    sz++ /* VERSION_MAGIC */;
 
@@ -3285,6 +3287,7 @@ dec_term_atom_common:
             n--;
 	    if (ctx) {
                 if (reds < n) {
+                    ASSERT(reds > 0);
 		    ctx->state = B2TDecodeList;
 		    ctx->u.dc.remaining_n = n - reds;
 		    n = reds;
@@ -4388,7 +4391,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx)
         }
     }
     else
-        reds = 0; /* not used but compiler warns anyway */
+        ERTS_UNDEF(reds, 0);
 
     heap_size = 0;
     terms = 1;
@@ -4698,3 +4701,177 @@ error:
 #undef SKIP2
 #undef CHKSIZE
 }
+
+
+struct transcode_context {
+    enum {
+        TRANSCODE_DEC_MSG_SIZE,
+        TRANSCODE_DEC_MSG,
+        TRANSCODE_ENC_CTL,
+        TRANSCODE_ENC_MSG
+    }state;
+    Eterm ctl_term;
+    Eterm* ctl_heap;
+    ErtsHeapFactory ctl_factory;
+    Eterm* msg_heap;
+    B2TContext b2t;
+    TTBEncodeContext ttb;
+#ifdef DEBUG
+    ErtsDistOutputBuf* dbg_ob;
+#endif
+};
+
+void transcode_free_ctx(DistEntry* dep)
+{
+    struct transcode_context* ctx = dep->transcode_ctx;
+
+    erts_factory_close(&ctx->ctl_factory);
+    erts_free(ERTS_ALC_T_DIST_TRANSCODE, ctx->ctl_heap);
+
+    if (ctx->msg_heap) {
+        erts_factory_close(&ctx->b2t.u.dc.factory);
+        erts_free(ERTS_ALC_T_DIST_TRANSCODE, ctx->msg_heap);
+    }
+    erts_free(ERTS_ALC_T_DIST_TRANSCODE, ctx);
+    dep->transcode_ctx = NULL;
+}
+
+Sint transcode_dist_obuf(ErtsDistOutputBuf* ob,
+                         DistEntry* dep,
+                         Uint32 dflags,
+                         Sint reds)
+{
+    Sint hsz;
+    byte* decp;
+    const int have_msg = !!ob->msg_start;
+    int i;
+    struct transcode_context* ctx = dep->transcode_ctx;
+
+    if (!ctx) { /* first call for 'ob' */
+
+        if (~dflags & (DFLAG_BIT_BINARIES | DFLAG_EXPORT_PTR_TAG)) {
+            /*
+             * Receiver does not support bitstrings and/or export funs.
+             * We need to transcode control and message terms to use tuple fallbacks.
+             */
+            ctx = erts_alloc(ERTS_ALC_T_DIST_TRANSCODE, sizeof(struct transcode_context));
+            dep->transcode_ctx = ctx;
+        #ifdef DEBUG
+            ctx->dbg_ob = ob;
+        #endif
+
+            hsz = decoded_size(ob->extp, ob->ext_endp, 0, NULL);
+            ctx->ctl_heap = erts_alloc(ERTS_ALC_T_DIST_TRANSCODE, hsz*sizeof(Eterm));
+            erts_factory_tmp_init(&ctx->ctl_factory, ctx->ctl_heap, hsz, ERTS_ALC_T_DIST_TRANSCODE);
+            ctx->msg_heap = NULL;
+
+            decp = dec_term(NULL, &ctx->ctl_factory, ob->extp, &ctx->ctl_term, NULL);
+            if (have_msg) {
+                ASSERT(decp == ob->msg_start); (void)decp;
+                ctx->b2t.u.sc.ep = NULL;
+                ctx->b2t.state = B2TSize;
+                ctx->b2t.aligned_alloc = NULL;
+                ctx->b2t.b2ts.exttmp = 0;
+                ctx->state = TRANSCODE_DEC_MSG_SIZE;
+            }
+            else {
+                ASSERT(decp == ob->ext_endp);
+                ctx->state = TRANSCODE_ENC_CTL;
+            }
+        }
+        else {
+            /*
+             * No need for full transcoding, but primitive receiver (erl_/jinterface)
+             * expects VERSION_MAGIC before both control and message terms.
+             */
+            if (ob->msg_start) {
+                Sint ctl_bytes = ob->msg_start - ob->extp;
+                ASSERT(ob->extp < ob->msg_start && ob->msg_start < ob->ext_endp);
+                /* Move control term back 1 byte to make room */
+                sys_memmove(ob->extp-1, ob->extp, ctl_bytes);
+                *--(ob->msg_start) = VERSION_MAGIC;
+                --(ob->extp);
+                reds -= ctl_bytes / (B2T_BYTES_PER_REDUCTION * B2T_MEMCPY_FACTOR);
+            }
+            *--(ob->extp) = VERSION_MAGIC;
+            goto done;
+        }
+    }
+    else {
+        ASSERT(ctx->dbg_ob == ob);
+    }
+    ctx->b2t.reds = reds * B2T_BYTES_PER_REDUCTION;
+
+    switch (ctx->state) {
+    case TRANSCODE_DEC_MSG_SIZE:
+        hsz = decoded_size(ob->msg_start, ob->ext_endp, 0, &ctx->b2t);
+        if (ctx->b2t.state == B2TSize) {
+            return -1;
+        }
+        ASSERT(ctx->b2t.state == B2TDecodeInit);
+        ctx->msg_heap = erts_alloc(ERTS_ALC_T_DIST_TRANSCODE, hsz*sizeof(Eterm));
+        ctx->b2t.u.dc.ep = ob->msg_start;
+        ctx->b2t.u.dc.res = (Eterm) NULL;
+        ctx->b2t.u.dc.next = &ctx->b2t.u.dc.res;
+        erts_factory_tmp_init(&ctx->b2t.u.dc.factory,
+                              ctx->msg_heap, hsz, ERTS_ALC_T_DIST_TRANSCODE);
+        ctx->b2t.u.dc.flat_maps.wstart = NULL;
+        ctx->b2t.u.dc.hamt_array.pstart = NULL;
+        ctx->b2t.state = B2TDecode;
+
+        ctx->state = TRANSCODE_DEC_MSG;
+    case TRANSCODE_DEC_MSG:
+        if (ctx->b2t.reds <= 0)
+            ctx->b2t.reds = 1;
+        decp = dec_term(NULL, NULL, NULL, NULL, &ctx->b2t);
+        if (ctx->b2t.state < B2TDone) {
+            return -1;
+        }
+        ASSERT(ctx->b2t.state == B2TDone);
+        ASSERT(decp && decp <= ob->ext_endp);
+        reds = ctx->b2t.reds / B2T_BYTES_PER_REDUCTION;
+        b2t_destroy_context(&ctx->b2t);
+
+        ctx->state = TRANSCODE_ENC_CTL;
+    case TRANSCODE_ENC_CTL:
+        if (!(dflags & DFLAG_DIST_HDR_ATOM_CACHE)) {
+            ASSERT(!(dflags & DFLAG_NO_MAGIC));
+            ob->extp -= 2; /* VERSION_MAGIC x 2 */
+        }
+        ob->ext_endp = ob->extp;
+        i = erts_encode_dist_ext(ctx->ctl_term, &ob->ext_endp, dflags,
+                                 NULL, NULL, NULL);
+        ASSERT(i == 0); (void)i;
+        ASSERT(ob->ext_endp <= ob->alloc_endp);
+
+        if (!have_msg) {
+            break;
+        }
+        ob->msg_start = ob->ext_endp;
+        ctx->ttb.wstack.wstart = NULL;
+        ctx->ttb.flags = dflags;
+        ctx->ttb.level = 0;
+
+        ctx->state = TRANSCODE_ENC_MSG;
+    case TRANSCODE_ENC_MSG:
+        reds *= TERM_TO_BINARY_LOOP_FACTOR;
+        if (erts_encode_dist_ext(ctx->b2t.u.dc.res, &ob->ext_endp, dflags, NULL,
+                                 &ctx->ttb, &reds)) {
+            return -1;
+        }
+        reds /= TERM_TO_BINARY_LOOP_FACTOR;
+
+        ASSERT(ob->ext_endp <= ob->alloc_endp);
+
+    }
+    transcode_free_ctx(dep);
+
+done:
+    if (!(dflags & DFLAG_DIST_HDR_ATOM_CACHE))
+        *--(ob->extp) = PASS_THROUGH;
+
+    if (reds < 0)
+        reds = 0;
+
+    return reds;
+}
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index 11044b17ef..f9f8abcc27 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -154,7 +154,7 @@ void erts_finalize_atom_cache_map(ErtsAtomCacheMap *, Uint32);
 
 Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *);
 byte *erts_encode_ext_dist_header_setup(byte *, ErtsAtomCacheMap *);
-void erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf*, ErtsAtomCache *, Uint32);
+Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf*, DistEntry *, Uint32 dflags, Sint reds);
 struct erts_dsig_send_context;
 int erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap*, Uint* szp);
 int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp);
@@ -195,7 +195,7 @@ void erts_binary2term_abort(ErtsBinary2TermState *);
 Eterm erts_binary2term_create(ErtsBinary2TermState *, ErtsHeapFactory*);
 int erts_debug_max_atom_out_cache_index(void);
 int erts_debug_atom_to_out_cache_index(Eterm);
-
+void transcode_free_ctx(DistEntry* dep);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-- 
cgit v1.2.3


From 56ab31904f7157d7b4282c2afd474c7bed7b9c75 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 19 Sep 2017 17:44:47 +0200
Subject: Remove faulty assert

Send may have failed, port exit with dist_entry cleaned up
and then new pending connection with queued messages.
---
 erts/emulator/beam/dist.c | 5 -----
 1 file changed, 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index a5df92bf9a..474b0e8364 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -2480,11 +2480,6 @@ erts_dist_command(Port *prt, int initial_reds)
 
 	foq.first = NULL;
 	foq.last = NULL;
-#ifdef DEBUG
-	erts_mtx_lock(&dep->qlock);
-	ASSERT(erts_atomic_read_nob(&dep->qsize) == obufsize);
-	erts_mtx_unlock(&dep->qlock);
-#endif
     }
     else {
 	if (oq.first) {
-- 
cgit v1.2.3


From 08040e5c5cd329395d9d756f1fdf8d66ffbbe705 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 21 Sep 2017 19:50:22 +0200
Subject: Allow DistEntries in ETS

---
 erts/emulator/beam/erl_node_tables.c | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 00e8306510..17d410e42b 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1089,6 +1089,7 @@ typedef struct {
 typedef struct dist_referrer_ {
     struct dist_referrer_ *next;
     int heap_ref;
+    int ets_ref;
     int node_ref;
     int ctrl_ref;
     int system_ref;
@@ -1207,10 +1208,11 @@ insert_dist_referrer(ReferredDist *referred_dist,
 	else {
 	    Uint *hp = &drp->id_heap[0];
 	    ASSERT(is_tuple(id));
-	    drp->id = copy_struct(id, size_object(id), &hp, NULL);
+            drp->id = copy_struct(id, size_object(id), &hp, NULL);
 	}
 	drp->creation = creation;
 	drp->heap_ref = 0;
+        drp->ets_ref = 0;
 	drp->node_ref = 0;
 	drp->ctrl_ref = 0;
 	drp->system_ref = 0;
@@ -1220,6 +1222,7 @@ insert_dist_referrer(ReferredDist *referred_dist,
     case NODE_REF:	drp->node_ref++;	break;
     case CTRL_REF:	drp->ctrl_ref++;	break;
     case HEAP_REF:	drp->heap_ref++;	break;
+    case ETS_REF:	drp->ets_ref++;	        break;
     case SYSTEM_REF:	drp->system_ref++;	break;
     default:		ASSERT(0);
     }
@@ -1897,6 +1900,10 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp)
 		tup = MK_2TUP(AM_heap, MK_UINT(drp->heap_ref));
 		drl = MK_CONS(tup, drl);
 	    }
+            if(drp->ets_ref) {
+                tup = MK_2TUP(AM_ets, MK_UINT(drp->ets_ref));
+                drl = MK_CONS(tup, drl);
+            }
 	    if(drp->system_ref) {
 		tup = MK_2TUP(AM_system, MK_UINT(drp->system_ref));
 		drl = MK_CONS(tup, drl);
@@ -1913,12 +1920,17 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp)
 	    else if (is_tuple(drp->id)) {
 		Eterm *t;
 		ASSERT(drp->system_ref && !drp->node_ref
-		       && !drp->ctrl_ref && !drp->heap_ref);
+		       && !drp->ctrl_ref && !drp->heap_ref && !drp->ets_ref);
 		t = tuple_val(drp->id);
 		ASSERT(2 == arityval(t[0]));
 		tup = MK_2TUP(t[1], t[2]);
 	    }
-	    else {
+	    else if (drp->ets_ref) {
+                ASSERT(!drp->heap_ref && !drp->node_ref &&
+                       !drp->ctrl_ref && !drp->system_ref);
+                tup = MK_2TUP(AM_ets, drp->id);
+            }
+            else {
 		ASSERT(!drp->ctrl_ref && drp->node_ref);
 		ASSERT(is_atom(drp->id));
 		tup = MK_2TUP(drp->id, MK_UINT(drp->creation));
-- 
cgit v1.2.3


From 771abbd23709b5a03416278595588931889ab7c5 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 20 Sep 2017 15:06:07 +0200
Subject: erts: Keep magic ref to DistEntry in net_kernel

to make sure it's kept alive.
---
 erts/emulator/beam/dist.c            | 22 +++++++++++++++++-----
 erts/emulator/beam/dist.h            | 24 ++++++++++++++++--------
 erts/emulator/beam/erl_node_tables.c | 30 ++++++++++++++----------------
 erts/emulator/beam/erl_node_tables.h |  1 +
 4 files changed, 48 insertions(+), 29 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 474b0e8364..c16579037c 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -3426,6 +3426,8 @@ BIF_RETTYPE new_connection_id_1(BIF_ALIST_1)
 {
     DistEntry* dep;
     Uint32 conn_id;
+    Eterm* hp;
+    Eterm dhandle;
 
     if (is_not_atom(BIF_ARG_1)) {
 	BIF_ERROR(BIF_P, BADARG);
@@ -3449,17 +3451,25 @@ BIF_RETTYPE new_connection_id_1(BIF_ALIST_1)
 	conn_id = dep->connection_id;
     }
     erts_de_rwunlock(dep);
-    BIF_RET(make_small(conn_id));
+    hp = HAlloc(BIF_P, 3 + ERTS_MAGIC_REF_THING_SIZE);
+    dhandle = erts_build_dhandle(&hp, &BIF_P->off_heap, dep);
+    erts_deref_dist_entry(dep);
+    BIF_RET(TUPLE2(hp, make_small(conn_id), dhandle));
 }
 
 BIF_RETTYPE abort_connection_id_2(BIF_ALIST_2)
 {
     DistEntry* dep;
+    Eterm* tp;
 
-    if (is_not_atom(BIF_ARG_1) || is_not_small(BIF_ARG_2)) {
+    if (is_not_atom(BIF_ARG_1) || is_not_tuple_arity(BIF_ARG_2, 2)) {
 	BIF_ERROR(BIF_P, BADARG);
     }
-    dep = erts_find_dist_entry(BIF_ARG_1);
+    tp = tuple_val(BIF_ARG_2);
+    dep = erts_dhandle_to_dist_entry(tp[2]);
+    if (is_not_small(tp[1]) || dep != erts_find_dist_entry(BIF_ARG_1)) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
     ASSERT(dep != erts_this_dist_entry); /* SVERK: What to do? */
 
     if (!dep) {
@@ -3469,7 +3479,7 @@ BIF_RETTYPE abort_connection_id_2(BIF_ALIST_2)
     erts_de_rwlock(dep);
 
     if (dep->status == ERTS_DE_SFLG_PENDING
-	&& dep->connection_id == unsigned_val(BIF_ARG_2)) {
+	&& dep->connection_id == unsigned_val(tp[1])) {
 
 	NetExitsContext nec = {dep};
 	ErtsLink *nlinks;
@@ -3736,10 +3746,12 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options)
             ++ERTS_LINK_REFC(lnk);
             lnk = erts_add_or_lookup_link(&ERTS_P_LINKS(p), LINK_NODE, Node);
             ++ERTS_LINK_REFC(lnk);
+            erts_de_links_unlock(dep);
             break;
         default:
             ERTS_ASSERT(! "Invalid dsig prepare result");
         }
+        erts_deref_dist_entry(dep);
     }
     else { /* Bool == false */
         dep = erts_sysname_to_connected_dist_entry(Node);
@@ -3777,9 +3789,9 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options)
                                                    Node));
             }
         }
+        erts_de_links_unlock(dep);
     }
 
-    erts_de_links_unlock(dep);
     erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 
  done:
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index f9a2037687..2ead588ee9 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -161,6 +161,7 @@ erts_dsig_prepare(ErtsDSigData *dsdp,
 		  int connect)
 {
     DistEntry* dep = *depp;
+    int deref_dep = 0;
     int res;
 
     if (!erts_is_alive)
@@ -171,6 +172,7 @@ erts_dsig_prepare(ErtsDSigData *dsdp,
 
 	dep = erts_find_or_insert_dist_entry(dsdp->node);
 	ASSERT(dep != erts_this_dist_entry);  /* SVERK: What to do? */
+        deref_dep = 1;
     }
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -208,7 +210,7 @@ retry:
 	    Eterm *hp;
 	    ErlOffHeap *ohp;
 	    ErtsMessage *mp;
-	    Eterm msg, conn_id;
+	    Eterm msg, conn_id, dhandle;
 
 	    dep->status = ERTS_DE_SFLG_PENDING;
 	    dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY);
@@ -220,16 +222,18 @@ retry:
 	    net_kernel = erts_whereis_process(proc, proc_locks,
 					      am_net_kernel, nk_locks, 0);
 	    if (!net_kernel) {
-		if (!*depp) {
-		    erts_deref_dist_entry(dep);
-		}
+                if (deref_dep)
+                    erts_deref_dist_entry(dep);
 		return ERTS_DSIG_PREP_NOT_ALIVE;
 	    }
 
-	    /* Send {auto_connect, Node, ConnId} to net_kernel */
-	    mp = erts_alloc_message_heap(net_kernel, &nk_locks, 4, &hp, &ohp);
-	    msg = TUPLE3(hp, am_auto_connect, dep->sysname, conn_id);
-	    erts_queue_message(net_kernel, nk_locks, mp, msg, proc->common.id);
+	    /* Send {auto_connect, Node, ConnId, DHandle} to net_kernel */
+	    mp = erts_alloc_message_heap(net_kernel, &nk_locks,
+                                         5 + ERTS_MAGIC_REF_THING_SIZE,
+                                         &hp, &ohp);
+            dhandle = erts_build_dhandle(&hp, ohp, dep);
+	    msg = TUPLE4(hp, am_auto_connect, dep->sysname, conn_id, dhandle);
+            erts_queue_message(net_kernel, nk_locks, mp, msg, proc->common.id);
 	    erts_proc_unlock(net_kernel, nk_locks);
 	}
 	else
@@ -255,6 +259,8 @@ retry:
     dsdp->no_suspend = no_suspend;
     if (dspl == ERTS_DSP_NO_LOCK)
 	erts_de_runlock(dep);
+    if (deref_dep)
+        erts_deref_dist_entry(dep);
     *depp = dep;
     return res;
 
@@ -263,6 +269,8 @@ retry:
 	erts_de_rwunlock(dep);
     else
 	erts_de_runlock(dep);
+    if (deref_dep)
+        erts_deref_dist_entry(dep);
     return res;
 }
 
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 17d410e42b..7c75ec60e9 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -141,9 +141,7 @@ dist_table_cmp(void *dep1, void *dep2)
 static void*
 dist_table_alloc(void *dep_tmpl)
 {
-#ifdef DEBUG
     erts_aint_t refc;
-#endif
     Eterm sysname;
     Binary *bin;
     DistEntry *dep;
@@ -160,13 +158,8 @@ dist_table_alloc(void *dep_tmpl)
 
     dist_entries++;
 
-#ifdef DEBUG
-    refc =
-#else
-    (void)
-#endif
-        de_refc_dec_read(dep, -1);
-    ASSERT(refc == -1);
+    refc = de_refc_dec_read(dep, -1);
+    ASSERT(refc == -1); (void)refc;
 
     dep->prev				= NULL;
     erts_rwmtx_init_opt(&dep->rwmtx, &rwmtx_opt, "dist_entry", sysname,
@@ -227,6 +220,8 @@ dist_table_free(void *vdep)
 {
     DistEntry *dep = (DistEntry *) vdep;
 
+    ASSERT(de_refc_read(dep, -1) == -1);
+    ASSERT(dep->status == 0);
     ASSERT(is_nil(dep->cid));
     ASSERT(dep->nlinks == NULL);
     ASSERT(dep->node_links == NULL);
@@ -387,16 +382,19 @@ erts_dhandle_to_dist_entry(Eterm dhandle)
 }
 
 Eterm
-erts_make_dhandle(Process *c_p, DistEntry *dep)
+erts_build_dhandle(Eterm **hpp, ErlOffHeap* ohp, DistEntry *dep)
 {
-    Binary *bin;
-    Eterm *hp;
-
-    bin = ErtsDistEntry2Bin(dep);
+    Binary *bin = ErtsDistEntry2Bin(dep);
     ASSERT(bin);
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == erts_dist_entry_destructor);
-    hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE);
-    return erts_mk_magic_ref(&hp, &c_p->off_heap, bin);
+    return erts_mk_magic_ref(hpp, ohp, bin);
+}
+
+Eterm
+erts_make_dhandle(Process *c_p, DistEntry *dep)
+{
+    Eterm *hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE);
+    return erts_build_dhandle(&hp, &c_p->off_heap, dep);
 }
 
 static void start_timer_delete_dist_entry(void *vdep);
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index afa83f8b46..83d0678bbc 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -210,6 +210,7 @@ int erts_lc_is_de_rlocked(DistEntry *);
 #endif
 int erts_dist_entry_destructor(Binary *bin);
 DistEntry *erts_dhandle_to_dist_entry(Eterm dhandle);
+Eterm erts_build_dhandle(Eterm **hpp, ErlOffHeap*, DistEntry*);
 Eterm erts_make_dhandle(Process *c_p, DistEntry *dep);
 void erts_ref_dist_entry(DistEntry *dep);
 void erts_deref_dist_entry(DistEntry *dep);
-- 
cgit v1.2.3


From 23ba06691aa64d7f05781bc9b2ea448c3710a812 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 25 Sep 2017 20:23:06 +0200
Subject: Remove unused ERTS_DSP_RWLOCK

---
 erts/emulator/beam/dist.h | 19 +++++--------------
 1 file changed, 5 insertions(+), 14 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 2ead588ee9..b9b39de3b6 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -96,8 +96,7 @@ extern Export* dmonitor_p_trap;
 
 typedef enum {
     ERTS_DSP_NO_LOCK,
-    ERTS_DSP_RLOCK,
-    ERTS_DSP_RWLOCK
+    ERTS_DSP_RLOCK
 } ErtsDSigPrepLock;
 
 
@@ -182,10 +181,7 @@ erts_dsig_prepare(ErtsDSigData *dsdp,
 #endif
 
 retry:
-    if (dspl == ERTS_DSP_RWLOCK)
-	erts_de_rwlock(dep);
-    else
-	erts_de_rlock(dep);
+    erts_de_rlock(dep);
 
     if (ERTS_DE_IS_CONNECTED(dep)) {
 	res = ERTS_DSIG_PREP_CONNECTED;
@@ -200,10 +196,8 @@ retry:
     }
     else if (connect) {
         ASSERT(dep->status == 0);
-	if (dspl != ERTS_DSP_RWLOCK) {
-	    erts_de_runlock(dep);
-	    erts_de_rwlock(dep);
-	}
+        erts_de_runlock(dep);
+        erts_de_rwlock(dep);
 	if (dep->status == 0) {
 	    Process* net_kernel;
 	    ErtsProcLocks nk_locks = ERTS_PROC_LOCK_MSGQ;
@@ -265,10 +259,7 @@ retry:
     return res;
 
  fail:
-    if (dspl == ERTS_DSP_RWLOCK)
-	erts_de_rwunlock(dep);
-    else
-	erts_de_runlock(dep);
+    erts_de_runlock(dep);
     if (deref_dep)
         erts_deref_dist_entry(dep);
     return res;
-- 
cgit v1.2.3


From 25f9d0469e2b1da0aec73b55e54561be8b573634 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 26 Sep 2017 16:01:29 +0200
Subject: Refactor auto_connect into an outline function

and abort_pending_connection into own utility function.
---
 erts/emulator/beam/dist.c | 92 ++++++++++++++++++++++++++++++++++-------------
 erts/emulator/beam/dist.h | 42 ++++------------------
 2 files changed, 75 insertions(+), 59 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index c16579037c..3e8c4c65b5 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -120,6 +120,7 @@ static void clear_dist_entry(DistEntry*);
 static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy);
 static void send_nodes_mon_msgs(Process *, Eterm, Eterm, Eterm, Eterm);
 static void init_nodes_monitors(void);
+static Sint abort_pending_connection(DistEntry* dep, Uint32 conn_id);
 
 static erts_atomic_t no_caches;
 static erts_atomic_t no_nodes;
@@ -3457,30 +3458,16 @@ BIF_RETTYPE new_connection_id_1(BIF_ALIST_1)
     BIF_RET(TUPLE2(hp, make_small(conn_id), dhandle));
 }
 
-BIF_RETTYPE abort_connection_id_2(BIF_ALIST_2)
+static Sint abort_pending_connection(DistEntry* dep, Uint32 conn_id)
 {
-    DistEntry* dep;
-    Eterm* tp;
-
-    if (is_not_atom(BIF_ARG_1) || is_not_tuple_arity(BIF_ARG_2, 2)) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
-    tp = tuple_val(BIF_ARG_2);
-    dep = erts_dhandle_to_dist_entry(tp[2]);
-    if (is_not_small(tp[1]) || dep != erts_find_dist_entry(BIF_ARG_1)) {
-        BIF_ERROR(BIF_P, BADARG);
-    }
-    ASSERT(dep != erts_this_dist_entry); /* SVERK: What to do? */
-
-    if (!dep) {
-	BIF_RET(am_false);
-    }
+    Sint reds = 0;
 
     erts_de_rwlock(dep);
 
-    if (dep->status == ERTS_DE_SFLG_PENDING
-	&& dep->connection_id == unsigned_val(tp[1])) {
-
+    if (dep->status != ERTS_DE_SFLG_PENDING || dep->connection_id != conn_id) {
+        erts_de_rwunlock(dep);
+    }
+    else {
 	NetExitsContext nec = {dep};
 	ErtsLink *nlinks;
 	ErtsLink *node_links;
@@ -3488,7 +3475,6 @@ BIF_RETTYPE abort_connection_id_2(BIF_ALIST_2)
 	ErtsAtomCache *cache;
 	ErtsDistOutputBuf *obuf;
         ErtsProcList *resume_procs;
-        Sint reds = 0;
 
 	ASSERT(is_nil(dep->cid));
 
@@ -3529,13 +3515,71 @@ BIF_RETTYPE abort_connection_id_2(BIF_ALIST_2)
 	delete_cache(cache);
 
 	free_de_out_queues(dep, obuf);
+    }
+    return reds;
+}
 
-	BIF_RET2(am_true, reds);
+BIF_RETTYPE abort_connection_id_2(BIF_ALIST_2)
+{
+    DistEntry* dep;
+    Eterm* tp;
+
+    if (is_not_atom(BIF_ARG_1) || is_not_tuple_arity(BIF_ARG_2, 2)) {
+	BIF_ERROR(BIF_P, BADARG);
+    }
+    tp = tuple_val(BIF_ARG_2);
+    dep = erts_dhandle_to_dist_entry(tp[2]);
+    if (is_not_small(tp[1]) || dep != erts_find_dist_entry(BIF_ARG_1)) {
+        BIF_ERROR(BIF_P, BADARG);
     }
+    ASSERT(dep != erts_this_dist_entry); /* SVERK: What to do? */
 
-    erts_de_rwunlock(dep);
+    if (dep) {
+        Sint reds = abort_pending_connection(dep, unsigned_val(tp[1]));
+        BUMP_REDS(BIF_P, reds);
+    }
+    BIF_RET(am_true);
+}
 
-    BIF_RET(am_false);
+int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks)
+{
+    erts_de_rwlock(dep);
+    if (dep->status != 0) {
+        erts_de_rwunlock(dep);
+    }
+    else {
+        Process* net_kernel;
+        ErtsProcLocks nk_locks = ERTS_PROC_LOCK_MSGQ;
+        Eterm *hp;
+        ErlOffHeap *ohp;
+        ErtsMessage *mp;
+        Eterm msg, dhandle;
+        Uint32 conn_id;
+
+        dep->status = ERTS_DE_SFLG_PENDING;
+        dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY);
+        dep->connection_id = (dep->connection_id + 1) & ERTS_DIST_CON_ID_MASK;
+        conn_id = dep->connection_id;
+        erts_de_rwunlock(dep);
+
+        net_kernel = erts_whereis_process(proc, proc_locks,
+                                          am_net_kernel, nk_locks, 0);
+        if (!net_kernel) {
+            return 0;
+        }
+
+        /* Send {auto_connect, Node, ConnId, DHandle} to net_kernel */
+        mp = erts_alloc_message_heap(net_kernel, &nk_locks,
+                                     5 + ERTS_MAGIC_REF_THING_SIZE,
+                                     &hp, &ohp);
+        dhandle = erts_build_dhandle(&hp, ohp, dep);
+        msg = TUPLE4(hp, am_auto_connect, dep->sysname, make_small(conn_id),
+                     dhandle);
+        erts_queue_message(net_kernel, nk_locks, mp, msg, proc->common.id);
+        erts_proc_unlock(net_kernel, nk_locks);
+    }
+
+    return 1;
 }
 
 /**********************************************************************/
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index b9b39de3b6..2685a55b97 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -148,6 +148,8 @@ ERTS_GLB_INLINE int erts_dsig_prepare(ErtsDSigData *,
 ERTS_GLB_INLINE
 void erts_schedule_dist_command(Port *, DistEntry *);
 
+int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks);
+
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_INLINE int 
@@ -197,41 +199,11 @@ retry:
     else if (connect) {
         ASSERT(dep->status == 0);
         erts_de_runlock(dep);
-        erts_de_rwlock(dep);
-	if (dep->status == 0) {
-	    Process* net_kernel;
-	    ErtsProcLocks nk_locks = ERTS_PROC_LOCK_MSGQ;
-	    Eterm *hp;
-	    ErlOffHeap *ohp;
-	    ErtsMessage *mp;
-	    Eterm msg, conn_id, dhandle;
-
-	    dep->status = ERTS_DE_SFLG_PENDING;
-	    dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY);
-	    dep->connection_id++;
-	    dep->connection_id &= ERTS_DIST_CON_ID_MASK;
-            conn_id = make_small(dep->connection_id);
-	    erts_de_rwunlock(dep);
-
-	    net_kernel = erts_whereis_process(proc, proc_locks,
-					      am_net_kernel, nk_locks, 0);
-	    if (!net_kernel) {
-                if (deref_dep)
-                    erts_deref_dist_entry(dep);
-		return ERTS_DSIG_PREP_NOT_ALIVE;
-	    }
-
-	    /* Send {auto_connect, Node, ConnId, DHandle} to net_kernel */
-	    mp = erts_alloc_message_heap(net_kernel, &nk_locks,
-                                         5 + ERTS_MAGIC_REF_THING_SIZE,
-                                         &hp, &ohp);
-            dhandle = erts_build_dhandle(&hp, ohp, dep);
-	    msg = TUPLE4(hp, am_auto_connect, dep->sysname, conn_id, dhandle);
-            erts_queue_message(net_kernel, nk_locks, mp, msg, proc->common.id);
-	    erts_proc_unlock(net_kernel, nk_locks);
-	}
-	else
-	    erts_de_rwunlock(dep);
+        if (!erts_auto_connect(dep, proc, proc_locks)) {
+            if (deref_dep)
+                erts_deref_dist_entry(dep);
+            return ERTS_DSIG_PREP_NOT_ALIVE;
+        }
 	goto retry;
     }
     else {
-- 
cgit v1.2.3


From 6b48ef0226084a7f471d391abd4c1c399068840a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 27 Sep 2017 19:02:10 +0200
Subject: erts: Put pending DistrEntry in separate list

---
 erts/emulator/beam/dist.c            |  29 +++++---
 erts/emulator/beam/erl_node_tables.c | 128 +++++++++++++++++++++++++----------
 erts/emulator/beam/erl_node_tables.h |   3 +
 3 files changed, 113 insertions(+), 47 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 3e8c4c65b5..44f56ebe28 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -3062,6 +3062,10 @@ int distribution_info(fmtfn_t to, void *arg)	/* Called by break handler */
       info_dist_entry(to, arg, dep, 0, 1);
     }
 
+    for (dep = erts_pending_dist_entries; dep; dep = dep->next) {
+        info_dist_entry(to, arg, dep, 0, 0);
+    }
+
     for (dep = erts_not_connected_dist_entries; dep; dep = dep->next) {
         if (dep != erts_this_dist_entry) {
             info_dist_entry(to, arg, dep, 0, 0);
@@ -3441,10 +3445,7 @@ BIF_RETTYPE new_connection_id_1(BIF_ALIST_1)
     if (ERTS_DE_IS_CONNECTED(dep) || dep->status & ERTS_DE_SFLG_PENDING)
 	conn_id = dep->connection_id;
     else if (dep->status == 0) {
-	dep->status = ERTS_DE_SFLG_PENDING;
-	dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY);
-	dep->connection_id++;
-	dep->connection_id &= ERTS_DIST_CON_ID_MASK;
+        erts_set_dist_entry_pending(dep);
 	conn_id = dep->connection_id;
     }
     else {
@@ -3489,8 +3490,6 @@ static Sint abort_pending_connection(DistEntry* dep, Uint32 conn_id)
 
 	cache = dep->cache;
 	dep->cache = NULL;
-	dep->status = 0;
-	dep->flags = 0;
 	erts_mtx_lock(&dep->qlock);
         obuf = dep->out_queue.first;
         dep->out_queue.first = NULL;
@@ -3501,6 +3500,9 @@ static Sint abort_pending_connection(DistEntry* dep, Uint32 conn_id)
 	erts_mtx_unlock(&dep->qlock);
 	erts_atomic_set_nob(&dep->dist_cmd_scheduled, 0);
 	dep->send = NULL;
+
+        erts_set_dist_entry_not_connected(dep);
+
 	erts_de_rwunlock(dep);
 
 	erts_sweep_monitors(monitors, &doit_monitor_net_exits, &nec);
@@ -3556,9 +3558,7 @@ int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks)
         Eterm msg, dhandle;
         Uint32 conn_id;
 
-        dep->status = ERTS_DE_SFLG_PENDING;
-        dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY);
-        dep->connection_id = (dep->connection_id + 1) & ERTS_DIST_CON_ID_MASK;
+        erts_set_dist_entry_pending(dep);
         conn_id = dep->connection_id;
         erts_de_rwunlock(dep);
 
@@ -3655,9 +3655,11 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
 
     ASSERT(erts_no_of_not_connected_dist_entries > 0);
     ASSERT(erts_no_of_hidden_dist_entries >= 0);
+    ASSERT(erts_no_of_pending_dist_entries >= 0);
     ASSERT(erts_no_of_visible_dist_entries >= 0);
     if(not_connected)
-      length += (erts_no_of_not_connected_dist_entries - 1);
+      length += ((erts_no_of_not_connected_dist_entries - 1)
+                 + erts_no_of_pending_dist_entries);
     if(hidden)
       length += erts_no_of_hidden_dist_entries;
     if(visible)
@@ -3677,13 +3679,18 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
 #ifdef DEBUG
     endp = hp + length*2;
 #endif
-    if(not_connected)
+    if(not_connected) {
       for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) {
           if (dep != erts_this_dist_entry) {
             result = CONS(hp, dep->sysname, result);
             hp += 2;
           }
+        }
+      for(dep = erts_pending_dist_entries; dep; dep = dep->next) {
+          result = CONS(hp, dep->sysname, result);
+          hp += 2;
       }
+    }
     if(hidden)
       for(dep = erts_hidden_dist_entries; dep; dep = dep->next) {
 	result = CONS(hp, dep->sysname, result);
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 7c75ec60e9..eaf133f5c0 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -39,9 +39,11 @@ erts_rwmtx_t erts_node_table_rwmtx;
 
 DistEntry *erts_hidden_dist_entries;
 DistEntry *erts_visible_dist_entries;
+DistEntry *erts_pending_dist_entries;
 DistEntry *erts_not_connected_dist_entries; /* including erts_this_dist_entry */
 Sint erts_no_of_hidden_dist_entries;
 Sint erts_no_of_visible_dist_entries;
+Sint erts_no_of_pending_dist_entries;
 Sint erts_no_of_not_connected_dist_entries; /* including erts_this_dist_entry */
 
 DistEntry *erts_this_dist_entry;
@@ -522,12 +524,17 @@ erts_dist_table_size(void)
 	i++;
     ASSERT(i == erts_no_of_hidden_dist_entries);
     i = 0;
+    for(dep = erts_pending_dist_entries; dep; dep = dep->next)
+        i++;
+    ASSERT(i == erts_no_of_pending_dist_entries);
+    i = 0;
     for(dep = erts_not_connected_dist_entries; dep; dep = dep->next)
 	i++;
     ASSERT(i == erts_no_of_not_connected_dist_entries);
 
     ASSERT(dist_entries == (erts_no_of_visible_dist_entries
 			    + erts_no_of_hidden_dist_entries
+                            + erts_no_of_pending_dist_entries
 			    + erts_no_of_not_connected_dist_entries));
 #endif
 
@@ -542,43 +549,46 @@ erts_dist_table_size(void)
 void
 erts_set_dist_entry_not_connected(DistEntry *dep)
 {
+    DistEntry** head;
+
     ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep));
     erts_rwmtx_rwlock(&erts_dist_table_rwmtx);
 
     ASSERT(dep != erts_this_dist_entry);
-    ASSERT(is_internal_port(dep->cid) || is_internal_pid(dep->cid));
 
-    if(dep->flags & DFLAG_PUBLISHED) {
-	if(dep->prev) {
-	    ASSERT(is_in_de_list(dep, erts_visible_dist_entries));
-	    dep->prev->next = dep->next;
-	}
-	else {
-	    ASSERT(erts_visible_dist_entries == dep);
-	    erts_visible_dist_entries = dep->next;
-	}
-
-	ASSERT(erts_no_of_visible_dist_entries > 0);
-	erts_no_of_visible_dist_entries--;
+    if (dep->status & ERTS_DE_SFLG_PENDING) {
+        ASSERT(is_nil(dep->cid));
+        ASSERT(erts_no_of_pending_dist_entries > 0);
+        erts_no_of_pending_dist_entries--;
+        head = &erts_pending_dist_entries;
     }
     else {
-	if(dep->prev) {
-	    ASSERT(is_in_de_list(dep, erts_hidden_dist_entries));
-	    dep->prev->next = dep->next;
-	}
-	else {
-	    ASSERT(erts_hidden_dist_entries == dep);
-	    erts_hidden_dist_entries = dep->next;
-	}
-
-	ASSERT(erts_no_of_hidden_dist_entries > 0);
-	erts_no_of_hidden_dist_entries--;
+        ASSERT(dep->status != 0);
+        ASSERT(is_internal_port(dep->cid) || is_internal_pid(dep->cid));
+        if (dep->flags & DFLAG_PUBLISHED) {
+            ASSERT(erts_no_of_visible_dist_entries > 0);
+            erts_no_of_visible_dist_entries--;
+            head = &erts_visible_dist_entries;
+        }
+        else {
+            ASSERT(erts_no_of_hidden_dist_entries > 0);
+            erts_no_of_hidden_dist_entries--;
+            head = &erts_hidden_dist_entries;
+        }
     }
 
+    if(dep->prev) {
+        ASSERT(is_in_de_list(dep, *head));
+        dep->prev->next = dep->next;
+    }
+    else {
+        ASSERT(*head == dep);
+        *head = dep->next;
+    }
     if(dep->next)
 	dep->next->prev = dep->prev;
 
-    dep->status &= ~ERTS_DE_SFLG_CONNECTED;
+    dep->status &= ~(ERTS_DE_SFLG_PENDING | ERTS_DE_SFLG_CONNECTED);
     dep->flags = 0;
     dep->prev = NULL;
     dep->cid = NIL;
@@ -593,6 +603,45 @@ erts_set_dist_entry_not_connected(DistEntry *dep)
     erts_rwmtx_rwunlock(&erts_dist_table_rwmtx);
 }
 
+void
+erts_set_dist_entry_pending(DistEntry *dep)
+{
+    ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep));
+    erts_rwmtx_rwlock(&erts_dist_table_rwmtx);
+
+    ASSERT(dep != erts_this_dist_entry);
+    ASSERT(dep->status == 0);
+    ASSERT(is_nil(dep->cid));
+
+    if(dep->prev) {
+        ASSERT(is_in_de_list(dep, erts_not_connected_dist_entries));
+        dep->prev->next = dep->next;
+    }
+    else {
+        ASSERT(dep == erts_not_connected_dist_entries);
+        erts_not_connected_dist_entries = dep->next;
+    }
+
+    if(dep->next)
+	dep->next->prev = dep->prev;
+
+    erts_no_of_not_connected_dist_entries--;
+
+    dep->status = ERTS_DE_SFLG_PENDING;
+    dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY);
+    dep->connection_id = (dep->connection_id + 1) & ERTS_DIST_CON_ID_MASK;
+
+    dep->prev = NULL;
+    dep->next = erts_pending_dist_entries;
+    if(erts_pending_dist_entries) {
+	ASSERT(erts_pending_dist_entries->prev == NULL);
+	erts_pending_dist_entries->prev = dep;
+    }
+    erts_pending_dist_entries = dep;
+    erts_no_of_pending_dist_entries++;
+    erts_rwmtx_rwunlock(&erts_dist_table_rwmtx);
+}
+
 void
 erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags)
 {
@@ -603,29 +652,25 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags)
 
     ASSERT(dep != erts_this_dist_entry);
     ASSERT(is_nil(dep->cid));
+    ASSERT(dep->status & ERTS_DE_SFLG_PENDING);
     ASSERT(is_internal_port(cid) || is_internal_pid(cid));
 
     if(dep->prev) {
-	ASSERT(is_in_de_list(dep, erts_not_connected_dist_entries));
+	ASSERT(is_in_de_list(dep, erts_pending_dist_entries));
 	dep->prev->next = dep->next;
     }
     else {
-	ASSERT(erts_not_connected_dist_entries == dep);
-	erts_not_connected_dist_entries = dep->next;
+	ASSERT(erts_pending_dist_entries == dep);
+	erts_pending_dist_entries = dep->next;
     }
 
     if(dep->next)
 	dep->next->prev = dep->prev;
 
-    ASSERT(erts_no_of_not_connected_dist_entries > 0);
-    erts_no_of_not_connected_dist_entries--;
+    ASSERT(erts_no_of_pending_dist_entries > 0);
+    erts_no_of_pending_dist_entries--;
 
-    if (dep->status & ERTS_DE_SFLG_PENDING) {
-	dep->status &= ~ERTS_DE_SFLG_PENDING;
-    } else {
-	dep->connection_id++;
-	dep->connection_id &= ERTS_DIST_CON_ID_MASK;
-    }
+    dep->status &= ~ERTS_DE_SFLG_PENDING;
     dep->status |= ERTS_DE_SFLG_CONNECTED;
     dep->flags = flags & ~DFLAG_NO_MAGIC;
     dep->cid = cid;
@@ -959,9 +1004,11 @@ void erts_init_node_tables(int dd_sec)
 
     erts_hidden_dist_entries				= NULL;
     erts_visible_dist_entries				= NULL;
+    erts_pending_dist_entries			        = NULL;
     erts_not_connected_dist_entries			= NULL;
     erts_no_of_hidden_dist_entries			= 0;
     erts_no_of_visible_dist_entries			= 0;
+    erts_no_of_pending_dist_entries			= 0;
     erts_no_of_not_connected_dist_entries		= 0;
 
     node_tmpl.sysname = am_Noname;
@@ -1729,6 +1776,15 @@ setup_reference_table(void)
 	    insert_monitors(dep->monitors, dep->sysname);
     }
 
+    for(dep = erts_pending_dist_entries; dep; dep = dep->next) {
+        if(dep->nlinks)
+            insert_links2(dep->nlinks, dep->sysname);
+        if(dep->node_links)
+            insert_links(dep->node_links, dep->sysname);
+        if(dep->monitors)
+            insert_monitors(dep->monitors, dep->sysname);
+    }
+
     /* Not connected dist entries should not have any links,
        but inspect them anyway */
     for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) {
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 83d0678bbc..8d29c83e15 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -177,9 +177,11 @@ extern erts_rwmtx_t erts_node_table_rwmtx;
 
 extern DistEntry *erts_hidden_dist_entries;
 extern DistEntry *erts_visible_dist_entries;
+extern DistEntry *erts_pending_dist_entries;
 extern DistEntry *erts_not_connected_dist_entries;
 extern Sint erts_no_of_hidden_dist_entries;
 extern Sint erts_no_of_visible_dist_entries;
+extern Sint erts_no_of_pending_dist_entries;
 extern Sint erts_no_of_not_connected_dist_entries;
 
 extern DistEntry *erts_this_dist_entry;
@@ -195,6 +197,7 @@ void erts_schedule_delete_dist_entry(DistEntry *);
 Uint erts_dist_table_size(void);
 void erts_dist_table_info(fmtfn_t, void *);
 void erts_set_dist_entry_not_connected(DistEntry *);
+void erts_set_dist_entry_pending(DistEntry *);
 void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint);
 ErlNode *erts_find_or_insert_node(Eterm, Uint32);
 void erts_schedule_delete_node(ErlNode *);
-- 
cgit v1.2.3


From 0a6ccd3d1fb1d9cacd7369533d6e5aa1fc2ded77 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 27 Sep 2017 19:06:23 +0200
Subject: Abort all pending connections if net_kernel terminates

---
 erts/emulator/beam/dist.c | 66 ++++++++++++++++++++++++++++++-----------------
 1 file changed, 43 insertions(+), 23 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 44f56ebe28..1b5b730764 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -476,41 +476,54 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 
     if (dep == erts_this_dist_entry) {  /* Net kernel has died (clean up!!) */
 	DistEntry *tdep;
-	int no_dist_ctrl = 0;
+	int no_dist_ctrl;
+        int no_pending;
 	Eterm nd_reason = (reason == am_no_network
 			   ? am_no_network
 			   : am_net_kernel_terminated);
+        int i = 0;
+        Eterm *dist_ctrl;
+        DistEntry** pending;
+
+        ERTS_UNDEF(dist_ctrl, NULL);
+        ERTS_UNDEF(pending, NULL);
+
 	erts_rwmtx_rlock(&erts_dist_table_rwmtx);
 
-	for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next)
-	    no_dist_ctrl++;
-	for (tdep = erts_visible_dist_entries; tdep; tdep = tdep->next)
-	    no_dist_ctrl++;
+        no_dist_ctrl = (erts_no_of_hidden_dist_entries +
+                        erts_no_of_visible_dist_entries);
+        no_pending = erts_no_of_pending_dist_entries;
 
 	/* KILL all port controllers */
-	if (no_dist_ctrl == 0)
-	    erts_rwmtx_runlock(&erts_dist_table_rwmtx);
-	else {
-	    Eterm def_buf[128];
-	    int i = 0;
-	    Eterm *dist_ctrl;
-
-	    if (no_dist_ctrl <= sizeof(def_buf)/sizeof(def_buf[0]))
-		dist_ctrl = &def_buf[0];
-	    else
-		dist_ctrl = erts_alloc(ERTS_ALC_T_TMP,
-				       sizeof(Eterm)*no_dist_ctrl);
+	if (no_dist_ctrl) {
+            dist_ctrl = erts_alloc(ERTS_ALC_T_TMP,
+                                   sizeof(Eterm)*no_dist_ctrl);
 	    for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next) {
 		ASSERT(is_internal_port(tdep->cid) || is_internal_pid(tdep->cid));
+                ASSERT(i < no_dist_ctrl);
 		dist_ctrl[i++] = tdep->cid;
 	    }
 	    for (tdep = erts_visible_dist_entries; tdep; tdep = tdep->next) {
 		ASSERT(is_internal_port(tdep->cid) || is_internal_pid(tdep->cid));
+                ASSERT(i < no_dist_ctrl);
 		dist_ctrl[i++] = tdep->cid;
 	    }
-	    erts_rwmtx_runlock(&erts_dist_table_rwmtx);
+            ASSERT(i == no_dist_ctrl);
+        }
+        if (no_pending) {
+            pending = erts_alloc(ERTS_ALC_T_TMP, sizeof(DistEntry*)*no_pending);
+            i = 0;
+            for (tdep = erts_pending_dist_entries; tdep; tdep = tdep->next) {
+                ASSERT(is_nil(tdep->cid));
+                ASSERT(i < no_pending);
+                pending[i++] = tdep;
+            }
+            ASSERT(i == no_pending);
+        }
+        erts_rwmtx_runlock(&erts_dist_table_rwmtx);
 
-	    for (i = 0; i < no_dist_ctrl; i++) {
+        if (no_dist_ctrl) {
+            for (i = 0; i < no_dist_ctrl; i++) {
                 if (is_internal_pid(dist_ctrl[i]))
                     schedule_kill_dist_ctrl_proc(dist_ctrl[i]);
                 else {
@@ -524,11 +537,17 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
                                        prt, dist_ctrl[i], nd_reason, NULL);
                     }
                 }
-	    }
+            }
+            erts_free(ERTS_ALC_T_TMP, dist_ctrl);
+        }
+
+        if (no_pending) {
+            for (i = 0; i < no_pending; i++) {
+                abort_pending_connection(pending[i], pending[i]->connection_id);
+            }
+            erts_free(ERTS_ALC_T_TMP, pending);
+        }
 
-	    if (dist_ctrl != &def_buf[0])
-		erts_free(ERTS_ALC_T_TMP, dist_ctrl);
-	}
 
 	/*
 	 * When last dist ctrl exits, node will be taken
@@ -3565,6 +3584,7 @@ int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks)
         net_kernel = erts_whereis_process(proc, proc_locks,
                                           am_net_kernel, nk_locks, 0);
         if (!net_kernel) {
+            abort_pending_connection(dep, conn_id);
             return 0;
         }
 
-- 
cgit v1.2.3


From 6c4a3094d0263e46827c6b1869a6de0c033b6b64 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 27 Sep 2017 19:45:42 +0200
Subject: Improve connection aborting

---
 erts/emulator/beam/dist.c | 120 +++++++++++++++++++++++++++++++---------------
 erts/emulator/beam/dist.h |   3 +-
 2 files changed, 82 insertions(+), 41 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 1b5b730764..4a25fbdd4c 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -120,7 +120,7 @@ static void clear_dist_entry(DistEntry*);
 static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy);
 static void send_nodes_mon_msgs(Process *, Eterm, Eterm, Eterm, Eterm);
 static void init_nodes_monitors(void);
-static Sint abort_pending_connection(DistEntry* dep, Uint32 conn_id);
+static Sint abort_connection(DistEntry* dep, Uint32 conn_id);
 
 static erts_atomic_t no_caches;
 static erts_atomic_t no_nodes;
@@ -517,6 +517,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
                 ASSERT(is_nil(tdep->cid));
                 ASSERT(i < no_pending);
                 pending[i++] = tdep;
+                erts_ref_dist_entry(tdep);
             }
             ASSERT(i == no_pending);
         }
@@ -543,7 +544,8 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 
         if (no_pending) {
             for (i = 0; i < no_pending; i++) {
-                abort_pending_connection(pending[i], pending[i]->connection_id);
+                abort_connection(pending[i], pending[i]->connection_id);
+                erts_deref_dist_entry(pending[i]);
             }
             erts_free(ERTS_ALC_T_TMP, pending);
         }
@@ -624,7 +626,6 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 			    reason == am_normal ? am_connection_closed : reason);
 
 	clear_dist_entry(dep);
-
     }
 
     dec_no_nodes();
@@ -2904,24 +2905,31 @@ erts_dist_port_not_busy(Port *prt)
     erts_schedule_dist_command(prt, NULL);
 }
 
+static void kill_connection(DistEntry *dep)
+{
+    ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep));
+    ASSERT(dep->status == ERTS_DE_SFLG_CONNECTED);
+
+    dep->status |= ERTS_DE_SFLG_EXITING;
+    erts_mtx_lock(&dep->qlock);
+    ASSERT(!(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT));
+    erts_atomic32_read_bor_nob(&dep->qflgs, ERTS_DE_QFLG_EXIT);
+    erts_mtx_unlock(&dep->qlock);
+
+    if (is_internal_port(dep->cid))
+        erts_schedule_dist_command(NULL, dep);
+    else if (is_internal_pid(dep->cid))
+        schedule_kill_dist_ctrl_proc(dep->cid);
+}
+
 void
 erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id)
 {
     erts_de_rwlock(dep);
     if (connection_id == dep->connection_id
-	&& !(dep->status & ERTS_DE_SFLG_EXITING)) {
-
-	dep->status |= ERTS_DE_SFLG_EXITING;
-
-	erts_mtx_lock(&dep->qlock);
-	ASSERT(!(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT));
-	erts_atomic32_read_bor_nob(&dep->qflgs, ERTS_DE_QFLG_EXIT);
-	erts_mtx_unlock(&dep->qlock);
+        && dep->status == ERTS_DE_SFLG_CONNECTED) {
 
-        if (is_internal_port(dep->cid))
-            erts_schedule_dist_command(NULL, dep);
-        else if (is_internal_pid(dep->cid))
-            schedule_kill_dist_ctrl_proc(dep->cid);
+        kill_connection(dep);
     }
     erts_de_rwunlock(dep);
 }
@@ -3259,6 +3267,11 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
 	goto badarg;
     }
 
+    /*
+     * ToDo: Should we not pass connection_id as well
+     *       to make sure it's the right connection we commit.
+     */
+
     /*
      * Arguments seem to be in order.
      */
@@ -3302,6 +3315,23 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
             goto badarg;
         }
 
+        if (dep->status & ERTS_DE_SFLG_EXITING) {
+            /* Suspend on dist entry waiting for the exit to finish */
+            ErtsProcList *plp = erts_proclist_create(BIF_P);
+            plp->next = NULL;
+            erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
+            erts_mtx_lock(&dep->qlock);
+            erts_proclist_store_last(&dep->suspended, plp);
+            erts_mtx_unlock(&dep->qlock);
+            goto yield;
+        }
+        if (dep->status != ERTS_DE_SFLG_PENDING) {
+            if (dep->status == 0)
+                erts_set_dist_entry_pending(dep);
+            else
+                goto badarg;
+        }
+
         if (is_not_nil(dep->cid))
             goto badarg;
 
@@ -3344,8 +3374,12 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
             erts_mtx_unlock(&dep->qlock);
             goto yield;
         }
-
-        ASSERT(!(dep->status & ERTS_DE_SFLG_EXITING));
+        if (dep->status != ERTS_DE_SFLG_PENDING) {
+            if (dep->status == 0)
+                erts_set_dist_entry_pending(dep);
+            else
+                goto badarg;
+        }
 
         if (pp->dist_entry || is_not_nil(dep->cid))
             goto badarg;
@@ -3457,7 +3491,11 @@ BIF_RETTYPE new_connection_id_1(BIF_ALIST_1)
 	BIF_ERROR(BIF_P, BADARG);
     }
     dep = erts_find_or_insert_dist_entry(BIF_ARG_1);
-    ASSERT(dep != erts_this_dist_entry); /* SVERK: What to do? */
+
+    if (dep == erts_this_dist_entry) {
+        erts_deref_dist_entry(dep);
+        BIF_ERROR(BIF_P, BADARG);
+    }
 
     erts_de_rwlock(dep);
 
@@ -3468,8 +3506,8 @@ BIF_RETTYPE new_connection_id_1(BIF_ALIST_1)
 	conn_id = dep->connection_id;
     }
     else {
-	ASSERT(!"SVERK: What to do?");
-	conn_id = dep->connection_id;
+        ASSERT(dep->status & ERTS_DE_SFLG_EXITING);
+        conn_id = (dep->connection_id + 1) & ERTS_DIST_CON_ID_MASK;
     }
     erts_de_rwunlock(dep);
     hp = HAlloc(BIF_P, 3 + ERTS_MAGIC_REF_THING_SIZE);
@@ -3478,23 +3516,24 @@ BIF_RETTYPE new_connection_id_1(BIF_ALIST_1)
     BIF_RET(TUPLE2(hp, make_small(conn_id), dhandle));
 }
 
-static Sint abort_pending_connection(DistEntry* dep, Uint32 conn_id)
+static Sint abort_connection(DistEntry* dep, Uint32 conn_id)
 {
-    Sint reds = 0;
-
     erts_de_rwlock(dep);
 
-    if (dep->status != ERTS_DE_SFLG_PENDING || dep->connection_id != conn_id) {
-        erts_de_rwunlock(dep);
+    if (dep->connection_id != conn_id)
+        ;
+    else if (dep->status == ERTS_DE_SFLG_CONNECTED) {
+        kill_connection(dep);
     }
-    else {
-	NetExitsContext nec = {dep};
-	ErtsLink *nlinks;
-	ErtsLink *node_links;
-	ErtsMonitor *monitors;
-	ErtsAtomCache *cache;
-	ErtsDistOutputBuf *obuf;
+    else if (dep->status == ERTS_DE_SFLG_PENDING) {
+        NetExitsContext nec = {dep};
+        ErtsLink *nlinks;
+        ErtsLink *node_links;
+        ErtsMonitor *monitors;
+        ErtsAtomCache *cache;
+        ErtsDistOutputBuf *obuf;
         ErtsProcList *resume_procs;
+        Sint reds = 0;
 
 	ASSERT(is_nil(dep->cid));
 
@@ -3534,10 +3573,11 @@ static Sint abort_pending_connection(DistEntry* dep, Uint32 conn_id)
         }
 
 	delete_cache(cache);
-
 	free_de_out_queues(dep, obuf);
+        return reds;
     }
-    return reds;
+    erts_de_rwunlock(dep);
+    return 0;
 }
 
 BIF_RETTYPE abort_connection_id_2(BIF_ALIST_2)
@@ -3550,13 +3590,13 @@ BIF_RETTYPE abort_connection_id_2(BIF_ALIST_2)
     }
     tp = tuple_val(BIF_ARG_2);
     dep = erts_dhandle_to_dist_entry(tp[2]);
-    if (is_not_small(tp[1]) || dep != erts_find_dist_entry(BIF_ARG_1)) {
+    if (is_not_small(tp[1]) || dep != erts_find_dist_entry(BIF_ARG_1)
+        || dep == erts_this_dist_entry) {
         BIF_ERROR(BIF_P, BADARG);
     }
-    ASSERT(dep != erts_this_dist_entry); /* SVERK: What to do? */
 
     if (dep) {
-        Sint reds = abort_pending_connection(dep, unsigned_val(tp[1]));
+        Sint reds = abort_connection(dep, unsigned_val(tp[1]));
         BUMP_REDS(BIF_P, reds);
     }
     BIF_RET(am_true);
@@ -3584,11 +3624,13 @@ int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks)
         net_kernel = erts_whereis_process(proc, proc_locks,
                                           am_net_kernel, nk_locks, 0);
         if (!net_kernel) {
-            abort_pending_connection(dep, conn_id);
+            abort_connection(dep, conn_id);
             return 0;
         }
 
-        /* Send {auto_connect, Node, ConnId, DHandle} to net_kernel */
+        /*
+         * Send {auto_connect, Node, ConnId, DHandle} to net_kernel
+         */
         mp = erts_alloc_message_heap(net_kernel, &nk_locks,
                                      5 + ERTS_MAGIC_REF_THING_SIZE,
                                      &hp, &ohp);
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 2685a55b97..a205ce0cb5 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -172,7 +172,7 @@ erts_dsig_prepare(ErtsDSigData *dsdp,
 	    return ERTS_DSIG_PREP_NOT_CONNECTED;
 
 	dep = erts_find_or_insert_dist_entry(dsdp->node);
-	ASSERT(dep != erts_this_dist_entry);  /* SVERK: What to do? */
+	ASSERT(dep != erts_this_dist_entry);
         deref_dep = 1;
     }
 
@@ -192,7 +192,6 @@ retry:
 	res = ERTS_DSIG_PREP_PENDING;
     }
     else if (dep->status & ERTS_DE_SFLG_EXITING) {
-	/* SVERK is this ok, or should we trigger another connection setup */
 	res = ERTS_DSIG_PREP_NOT_CONNECTED;
 	goto fail;
     }
-- 
cgit v1.2.3


From 2f7a0afbbf34699ac1cc1733e3634c1cbeb94c5d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 11 Oct 2017 20:38:33 +0200
Subject: Refactor erts_dsig_prepare argument dep(p)

Don't need to be pointer-pointer
---
 erts/emulator/beam/bif.c         | 28 +++++++++++++++++-----------
 erts/emulator/beam/dist.c        |  6 +++---
 erts/emulator/beam/dist.h        | 21 ++++-----------------
 erts/emulator/beam/erl_process.c |  8 ++++----
 erts/emulator/beam/io.c          |  2 +-
 5 files changed, 29 insertions(+), 36 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 30278cbe36..50699eac31 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -227,7 +227,7 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
 		goto res_no_proc;
 	    }
 
-	    code = erts_dsig_prepare(&dsd, &dep, BIF_P,
+	    code = erts_dsig_prepare(&dsd, dep, BIF_P,
 				     (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK),
 				     ERTS_DSP_RLOCK, 0, 1);
 	    switch (code) {
@@ -314,7 +314,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
     ERTS_LC_ASSERT((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK)
 		       == erts_proc_lc_my_proc_locks(c_p));
 
-    code = erts_dsig_prepare(&dsd, &dep, c_p, ERTS_PROC_LOCK_MAIN,
+    code = erts_dsig_prepare(&dsd, dep, c_p, ERTS_PROC_LOCK_MAIN,
 			     ERTS_DSP_RLOCK, 0, 0);
     switch (code) {
     case ERTS_DSIG_PREP_NOT_ALIVE:
@@ -792,7 +792,7 @@ remote_monitor(Process *p, Eterm bifarg1, Eterm bifarg2,
 
     ASSERT(dep);
     erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
-    code = erts_dsig_prepare(&dsd, &dep,
+    code = erts_dsig_prepare(&dsd, dep,
 			     p, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK),
 			     ERTS_DSP_RLOCK, 0, 1);
     switch (code) {
@@ -1176,7 +1176,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 	    BIF_RET(am_true);
 	}
 
-	code = erts_dsig_prepare(&dsd, &dep, BIF_P, ERTS_PROC_LOCK_MAIN,
+	code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_PROC_LOCK_MAIN,
 				 ERTS_DSP_NO_LOCK, 0, 0);
 	switch (code) {
 	case ERTS_DSIG_PREP_NOT_ALIVE:
@@ -1560,7 +1560,7 @@ BIF_RETTYPE exit_2(BIF_ALIST_2)
 	 if(dep == erts_this_dist_entry)
 	     BIF_RET(am_true);
 
-	 code = erts_dsig_prepare(&dsd, &dep, BIF_P, ERTS_PROC_LOCK_MAIN,
+	 code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_PROC_LOCK_MAIN,
 				  ERTS_DSP_NO_LOCK, 0, 1);
 	 switch (code) {
 	 case ERTS_DSIG_PREP_NOT_ALIVE:
@@ -1996,7 +1996,7 @@ static Sint remote_send(Process *p, DistEntry *dep,
     ASSERT(is_atom(to) || is_external_pid(to));
 
     ctx->dep = dep;
-    code = erts_dsig_prepare(&ctx->dsd, &dep, p, ERTS_PROC_LOCK_MAIN,
+    code = erts_dsig_prepare(&ctx->dsd, dep, p, ERTS_PROC_LOCK_MAIN,
 			     ERTS_DSP_NO_LOCK,
 			     !ctx->suspend, ctx->connect);
     switch (code) {
@@ -2185,6 +2185,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx)
 	}
 	return ret_val;
     } else if (is_tuple(to)) { /* Remote send */
+        int deref_dep = 0;
 	int ret;
 	tp = tuple_val(to);
 	if (*tp != make_arityval(2))
@@ -2219,15 +2220,20 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx)
 	    }
 	    return 0;
 	}
+        if (dep == NULL) {
+            dep = erts_find_or_insert_dist_entry(tp[2]);
+            ASSERT(dep != erts_this_dist_entry);
+            deref_dep = 1;
+        }
 	ctx->dsd.node = tp[2];
 
 	ret = remote_send(p, dep, tp[1], to, msg, ctx);
 	if (ret == SEND_YIELD_CONTINUE) {
-            if (dep) {
-                erts_ref_dist_entry(dep);
-		ctx->deref_dep = 1;
-	    }
+            erts_ref_dist_entry(ctx->dep);
+            ctx->deref_dep = 1;
 	}
+        if (deref_dep)
+            erts_deref_dist_entry(dep);
 	return ret;
     } else {
 	if (IS_TRACED_FL(p, F_TRACE_SEND))
@@ -4405,7 +4411,7 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2)
 	if(dep == erts_this_dist_entry)
 	    BIF_ERROR(BIF_P, BADARG);
 
-	code = erts_dsig_prepare(&dsd, &dep, BIF_P, ERTS_PROC_LOCK_MAIN,
+	code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_PROC_LOCK_MAIN,
 				 ERTS_DSP_NO_LOCK, 0, 1);
 	switch (code) {
 	case ERTS_DSIG_PREP_NOT_ALIVE:
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 4a25fbdd4c..ae64f394c8 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1349,7 +1349,7 @@ int erts_net_message(Port *prt,
 	    /* This is tricky (we MUST force a distributed send) */
 	    ErtsDSigData dsd;
 	    int code;
-	    code = erts_dsig_prepare(&dsd, &dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0);
+	    code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0);
 	    if (code == ERTS_DSIG_PREP_CONNECTED) {
 		code = erts_dsig_send_exit(&dsd, to, from, am_noproc);
 		ASSERT(code == ERTS_DSIG_SEND_OK);
@@ -1441,7 +1441,7 @@ int erts_net_message(Port *prt,
 	if (!rp) {
 	    ErtsDSigData dsd;
 	    int code;
-	    code = erts_dsig_prepare(&dsd, &dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0);
+	    code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0);
 	    if (code == ERTS_DSIG_PREP_CONNECTED) {
 		code = erts_dsig_send_m_exit(&dsd, watcher, watched, ref,
 					     am_noproc);
@@ -3831,7 +3831,7 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options)
 
         erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
 
-        switch (erts_dsig_prepare(&dsd, &dep, p,
+        switch (erts_dsig_prepare(&dsd, dep, p,
                                   (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK),
                                   ERTS_DSP_RLOCK, 0, async_connect)) {
         case ERTS_DSIG_PREP_NOT_ALIVE:
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index a205ce0cb5..a96e39be89 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -138,7 +138,7 @@ extern int erts_is_alive;
 #define ERTS_DSIG_PREP_PENDING	        4
 
 ERTS_GLB_INLINE int erts_dsig_prepare(ErtsDSigData *,
-				      DistEntry **,
+				      DistEntry*,
 				      Process *,
                                       ErtsProcLocks,
 				      ErtsDSigPrepLock,
@@ -154,26 +154,20 @@ int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks);
 
 ERTS_GLB_INLINE int 
 erts_dsig_prepare(ErtsDSigData *dsdp,
-		  DistEntry **depp,
+		  DistEntry *dep,
 		  Process *proc,
                   ErtsProcLocks proc_locks,
 		  ErtsDSigPrepLock dspl,
 		  int no_suspend,
 		  int connect)
 {
-    DistEntry* dep = *depp;
-    int deref_dep = 0;
     int res;
 
     if (!erts_is_alive)
 	return ERTS_DSIG_PREP_NOT_ALIVE;
     if (!dep) {
-	if (!connect)
-	    return ERTS_DSIG_PREP_NOT_CONNECTED;
-
-	dep = erts_find_or_insert_dist_entry(dsdp->node);
-	ASSERT(dep != erts_this_dist_entry);
-        deref_dep = 1;
+        ASSERT(!connect);
+        return ERTS_DSIG_PREP_NOT_CONNECTED;
     }
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -199,8 +193,6 @@ retry:
         ASSERT(dep->status == 0);
         erts_de_runlock(dep);
         if (!erts_auto_connect(dep, proc, proc_locks)) {
-            if (deref_dep)
-                erts_deref_dist_entry(dep);
             return ERTS_DSIG_PREP_NOT_ALIVE;
         }
 	goto retry;
@@ -224,15 +216,10 @@ retry:
     dsdp->no_suspend = no_suspend;
     if (dspl == ERTS_DSP_NO_LOCK)
 	erts_de_runlock(dep);
-    if (deref_dep)
-        erts_deref_dist_entry(dep);
-    *depp = dep;
     return res;
 
  fail:
     erts_de_runlock(dep);
-    if (deref_dep)
-        erts_deref_dist_entry(dep);
     return res;
 }
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 2d13cd92b2..a807d60ec7 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -12659,7 +12659,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		erts_de_links_unlock(dep);
 		if (rmon) {
 		    ErtsDSigData dsd;
-		    int code = erts_dsig_prepare(&dsd, &dep, NULL, 0,
+		    int code = erts_dsig_prepare(&dsd, dep, NULL, 0,
 						 ERTS_DSP_NO_LOCK, 0, 0);
 		    if (code == ERTS_DSIG_PREP_CONNECTED ||
                         code == ERTS_DSIG_PREP_PENDING) {
@@ -12707,7 +12707,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		    erts_de_links_unlock(dep);
 		    if (rmon) {
 			ErtsDSigData dsd;
-			int code = erts_dsig_prepare(&dsd, &dep, NULL, 0,
+			int code = erts_dsig_prepare(&dsd, dep, NULL, 0,
 						     ERTS_DSP_NO_LOCK, 0, 0);
 			if (code == ERTS_DSIG_PREP_CONNECTED ||
                             code == ERTS_DSIG_PREP_PENDING) {
@@ -12768,7 +12768,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		erts_de_links_unlock(dep);
 		if (rmon) {
 		    ErtsDSigData dsd;
-		    int code = erts_dsig_prepare(&dsd, &dep, NULL, 0,
+		    int code = erts_dsig_prepare(&dsd, dep, NULL, 0,
 						 ERTS_DSP_NO_LOCK, 0, 0);
 		    if (code == ERTS_DSIG_PREP_CONNECTED) {
 			code = erts_dsig_send_m_exit(&dsd,
@@ -12893,7 +12893,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
 		erts_remove_dist_link(&dld, p->common.id, item, dep);
                 if (dld.d_lnk) {
                     erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
-                    code = erts_dsig_prepare(&dsd, &dep, p, 0, ERTS_DSP_NO_LOCK, 0, 0);
+                    code = erts_dsig_prepare(&dsd, dep, p, 0, ERTS_DSP_NO_LOCK, 0, 0);
                     if (code == ERTS_DSIG_PREP_CONNECTED ||
                         code == ERTS_DSIG_PREP_PENDING) {
 
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 6cd1aa5e79..9933c8dda4 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -3829,7 +3829,7 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc)
 	    ErtsDistLinkData dld;
 	    ErtsDSigData dsd;
 	    int code;
-	    code = erts_dsig_prepare(&dsd, &dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0);
+	    code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0);
 	    switch (code) {
 	    case ERTS_DSIG_PREP_NOT_ALIVE:
 	    case ERTS_DSIG_PREP_NOT_CONNECTED:
-- 
cgit v1.2.3


From 19c9f3a61fbef3682056bab3db4787cd763bd06e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 15 Nov 2017 19:44:52 +0100
Subject: Move new|abort_connection_id to erts_internal

and drop _id suffix.
---
 erts/emulator/beam/bif.tab | 4 ++--
 erts/emulator/beam/dist.c  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 5a591caac9..f739f414de 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -694,6 +694,6 @@ bif erlang:iolist_to_iovec/1
 # New in 21.0
 #
 
-bif erlang:new_connection_id/1
-bif erlang:abort_connection_id/2
+bif erts_internal:new_connection/1
+bif erts_internal:abort_connection/2
 
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index ae64f394c8..16a7bd9eb3 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -3480,7 +3480,7 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     goto done;
 }
 
-BIF_RETTYPE new_connection_id_1(BIF_ALIST_1)
+BIF_RETTYPE erts_internal_new_connection_1(BIF_ALIST_1)
 {
     DistEntry* dep;
     Uint32 conn_id;
@@ -3580,7 +3580,7 @@ static Sint abort_connection(DistEntry* dep, Uint32 conn_id)
     return 0;
 }
 
-BIF_RETTYPE abort_connection_id_2(BIF_ALIST_2)
+BIF_RETTYPE erts_internal_abort_connection_2(BIF_ALIST_2)
 {
     DistEntry* dep;
     Eterm* tp;
-- 
cgit v1.2.3


From 8e8380865bb31c119e7f11fbdbbb14ea58ebbef2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Magnus=20L=C3=A5ng?= <margnus1@telia.com>
Date: Sun, 5 Nov 2017 16:12:42 +0100
Subject: HiPE: Unique ref receive opt

---
 erts/emulator/hipe/hipe_mkliterals.c | 3 +++
 erts/emulator/hipe/hipe_native_bif.c | 2 ++
 2 files changed, 5 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c
index de00994d64..84889b3376 100644
--- a/erts/emulator/hipe/hipe_mkliterals.c
+++ b/erts/emulator/hipe/hipe_mkliterals.c
@@ -531,6 +531,9 @@ static const struct rts_param rts_params[] = {
       1, offsetof(struct process, hipe.gc_is_unsafe)
 #endif
     },
+
+    { 54, "P_MSG_LAST", 1, offsetof(struct process, msg.last) },
+    { 55, "P_MSG_SAVED_LAST", 1, offsetof(struct process, msg.saved_last) },
 };
 
 #define NR_PARAMS	ARRAY_SIZE(rts_params)
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index 99c34532b9..d6358eabf4 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -254,6 +254,8 @@ void hipe_handle_exception(Process *c_p)
     /* Synthesized to avoid having to generate code for it. */
     c_p->def_arg_reg[0] = exception_tag[GET_EXC_CLASS(c_p->freason)];
 
+    c_p->msg.saved_last = 0;  /* No longer safe to use this position */
+
     hipe_find_handler(c_p);
 }
 
-- 
cgit v1.2.3


From 745c99e35283686a3a636c82e44304cd180db232 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 15 Nov 2017 19:29:46 +0100
Subject: Fix triggering of node monitors

---
 erts/emulator/beam/dist.c | 34 ++++++++++++++++++----------------
 1 file changed, 18 insertions(+), 16 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 09fdb897f5..9fddac8980 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -385,22 +385,24 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp)
 	if (!rp)
 	    goto done;
 	erts_smp_proc_lock(rp, rp_locks);
-	rlnk = erts_remove_link(&ERTS_P_LINKS(rp), name);
-	if (rlnk != NULL) {
-	    ASSERT(is_atom(rlnk->pid) && (rlnk->type == LINK_NODE));
-	    erts_destroy_link(rlnk);
-	}
-	n = ERTS_LINK_REFC(lnk);
-	for (i = 0; i < n; ++i) {
-	    Eterm tup;
-	    Eterm *hp;
-	    ErtsMessage *msgp;
-
-	    msgp = erts_alloc_message_heap(rp, &rp_locks,
-					   3, &hp, &ohp);
-	    tup = TUPLE2(hp, am_nodedown, name);
-	    erts_queue_message(rp, rp_locks, msgp, tup, am_system);
-	}
+        if (!ERTS_PROC_IS_EXITING(rp)) {
+            rlnk = erts_remove_link(&ERTS_P_LINKS(rp), name);
+            if (rlnk != NULL) {
+                ASSERT(is_atom(rlnk->pid) && (rlnk->type == LINK_NODE));
+                erts_destroy_link(rlnk);
+            }
+            n = ERTS_LINK_REFC(lnk);
+            for (i = 0; i < n; ++i) {
+                Eterm tup;
+                Eterm *hp;
+                ErtsMessage *msgp;
+
+                msgp = erts_alloc_message_heap(rp, &rp_locks,
+                                               3, &hp, &ohp);
+                tup = TUPLE2(hp, am_nodedown, name);
+                erts_queue_message(rp, rp_locks, msgp, tup, am_system);
+            }
+        }
 	erts_smp_proc_unlock(rp, rp_locks);
     }
  done:
-- 
cgit v1.2.3


From 1107083935fbdb8f8a7ece9549d8203652c4fec6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 17 Nov 2017 14:20:05 +0100
Subject: Avoid using the efile driver in test suites

The efile driver will soon be reimplemented as a BIF.

Instead of opening a port based on efile, use hd(erlang:ports()).  It
is a reasonable safe assumption that the runtime will continue to use
use at least some ports.
---
 erts/emulator/test/binary_SUITE.erl       | 2 +-
 erts/emulator/test/distribution_SUITE.erl | 5 ++++-
 erts/emulator/test/guard_SUITE.erl        | 5 ++++-
 3 files changed, 9 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index b59c22f125..c71267a6d1 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -1202,7 +1202,7 @@ very_big_num(0, Result) ->
     Result.
 
 make_port() ->
-    open_port({spawn, efile}, [eof]).
+    hd(erlang:ports()).
 
 make_pid() ->
     spawn_link(?MODULE, sleeper, []).
diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index f1831b7c45..e40d346e10 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -742,7 +742,7 @@ link_to_dead_new_node(Config) when is_list(Config) ->
 %% doesn't correct them in any way.
 ref_port_roundtrip(Config) when is_list(Config) ->
     process_flag(trap_exit, true),
-    Port = open_port({spawn, efile}, []),
+    Port = make_port(),
     Ref = make_ref(),
     {ok, Node} = start_node(ref_port_roundtrip),
     net_adm:ping(Node),
@@ -763,6 +763,9 @@ ref_port_roundtrip(Config) when is_list(Config) ->
     end,
     ok.
 
+make_port() ->
+    hd(erlang:ports()).
+
 roundtrip(Term) ->
     exit(Term).
 
diff --git a/erts/emulator/test/guard_SUITE.erl b/erts/emulator/test/guard_SUITE.erl
index 1a93a9f5c2..f2c1595392 100644
--- a/erts/emulator/test/guard_SUITE.erl
+++ b/erts/emulator/test/guard_SUITE.erl
@@ -500,7 +500,7 @@ all_types() ->
      {atom, xxxx},
      {ref, make_ref()},
      {pid, self()},
-     {port, open_port({spawn, efile}, [])},
+     {port, make_port()},
      {function, fun(_) -> "" end},
      {function, fun erlang:abs/1},
      {binary, list_to_binary([])},
@@ -551,4 +551,7 @@ type_test(bitstring, X) when is_bitstring(X) ->
 type_test(function, X) when is_function(X) ->
     function.
 
+make_port() ->
+    hd(erlang:ports()).
+
 id(I) -> I.
-- 
cgit v1.2.3


From a1c796e7f6b86b4b506492ae6354382c565278d1 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 12 Oct 2017 16:00:50 +0200
Subject: erts: Implement batching maps:iterator

This iterator implementation fetches multiple elements to
iterate over in one call to erts_internal:maps_next instead
of one at a time. This means that the memory usage will go
up for the iterator as we are buffering elements, but the
usage is still bounded.

In this implementation the max memory usage is 1000 words.

Using this approach makes the iterator as fast as using
maps:to_list, so maps:iterator/2 has been removed.
---
 erts/emulator/beam/atom.names    |   1 +
 erts/emulator/beam/bif.tab       |   3 +-
 erts/emulator/beam/erl_map.c     | 375 ++++++++++++++++++++++-----------------
 erts/emulator/test/map_SUITE.erl |  57 ++----
 4 files changed, 237 insertions(+), 199 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index fc55b687d4..42a6991c6e 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -352,6 +352,7 @@ atom instruction_counts
 atom invalid
 atom is_constant
 atom is_seq_trace
+atom iterator
 atom io
 atom keypos
 atom kill
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index f2c8331850..b7b66e2ee7 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -629,7 +629,6 @@ bif re:inspect/2
 
 ubif erlang:is_map/1
 gcbif erlang:map_size/1
-bif maps:to_list/1
 bif maps:find/2
 bif maps:get/2
 bif maps:from_list/1
@@ -695,4 +694,4 @@ bif erlang:iolist_to_iovec/1
 # New in 21.0
 #
 
-bif erts_internal:map_next/2
+bif erts_internal:map_next/3
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index a5fda7b86d..135053dd18 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -139,35 +139,6 @@ BIF_RETTYPE map_size_1(BIF_ALIST_1) {
     BIF_ERROR(BIF_P, BADMAP);
 }
 
-/* maps:to_list/1 */
-
-BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) {
-    if (is_flatmap(BIF_ARG_1)) {
-	Uint n;
-	Eterm* hp;
-	Eterm *ks,*vs, res, tup;
-	flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1);
-
-	ks  = flatmap_get_keys(mp);
-	vs  = flatmap_get_values(mp);
-	n   = flatmap_get_size(mp);
-	hp  = HAlloc(BIF_P, (2 + 3) * n);
-	res = NIL;
-
-	while(n--) {
-	    tup = TUPLE2(hp, ks[n], vs[n]); hp += 3;
-	    res = CONS(hp, tup, res); hp += 2;
-	}
-
-	BIF_RET(res);
-    } else if (is_hashmap(BIF_ARG_1)) {
-	return hashmap_to_list(BIF_P, BIF_ARG_1, -1);
-    }
-
-    BIF_P->fvalue = BIF_ARG_1;
-    BIF_ERROR(BIF_P, BADMAP);
-}
-
 /* erts_internal:maps_to_list/2
  *
  * This function should be removed once iterators are in place.
@@ -1986,21 +1957,31 @@ static Eterm hashmap_to_list(Process *p, Eterm node, Sint m) {
     return res;
 }
 
-void hashmap_iterator_init(ErtsWStack* s, Eterm node, int reverse) {
-    Eterm hdr = *hashmap_val(node);
+static ERTS_INLINE
+Uint hashmap_node_size(Eterm hdr, Eterm **nodep)
+{
     Uint sz;
 
     switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
     case HAMT_SUBTAG_HEAD_ARRAY:
 	sz = 16;
+        if (nodep) ++*nodep;
 	break;
     case HAMT_SUBTAG_HEAD_BITMAP:
+        if (nodep) ++*nodep;
     case HAMT_SUBTAG_NODE_BITMAP:
         sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+        ASSERT(sz < 17);
 	break;
     default:
 	erts_exit(ERTS_ABORT_EXIT, "bad header");
     }
+    return sz;
+}
+
+void hashmap_iterator_init(ErtsWStack* s, Eterm node, int reverse) {
+    Eterm hdr = *hashmap_val(node);
+    Uint sz = hashmap_node_size(hdr, NULL);
 
     WSTACK_PUSH3((*s), (UWord)THE_NON_VALUE,  /* end marker */
 		 (UWord)(!reverse ? 0 : sz+1),
@@ -2024,20 +2005,7 @@ Eterm* hashmap_iterator_next(ErtsWStack* s) {
 	    ptr = boxed_val(node);
 	    hdr = *ptr;
 	    ASSERT(is_header(hdr));
-	    switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_HEAD_ARRAY:
-		ptr++;
-		sz = 16;
-		break;
-	    case HAMT_SUBTAG_HEAD_BITMAP:
-		ptr++;
-	    case HAMT_SUBTAG_NODE_BITMAP:
-		sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
-		ASSERT(sz < 17);
-		break;
-	    default:
-		erts_exit(ERTS_ABORT_EXIT, "bad header");
-	    }
+            sz = hashmap_node_size(hdr, &ptr);
 
 	    idx++;
 
@@ -2074,20 +2042,7 @@ Eterm* hashmap_iterator_prev(ErtsWStack* s) {
 	    ptr = boxed_val(node);
 	    hdr = *ptr;
 	    ASSERT(is_header(hdr));
-	    switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_HEAD_ARRAY:
-		ptr++;
-		sz = 16;
-		break;
-	    case HAMT_SUBTAG_HEAD_BITMAP:
-		ptr++;
-	    case HAMT_SUBTAG_NODE_BITMAP:
-		sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
-		ASSERT(sz < 17);
-		break;
-	    default:
-		erts_exit(ERTS_ERROR_EXIT, "bad header");
-	    }
+            sz = hashmap_node_size(hdr, &ptr);
 
             if (idx > sz)
 		idx = sz;
@@ -3061,15 +3016,7 @@ static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[])
 }
 
 
-/*
- * The iterator work in about the same way for both
- * flat- and hash-maps. It consists of a Path, Map
- * term where Path describes where to find the
- * term to get.
- *
- * In flatmap Path is just the index of the element
- * in the flatmap array.
- *
+/**
  * In hashmap the Path is a bit pattern that describes
  * which slot we should traverse in each hashmap node.
  * Since each hashmap node can only be up to 16 elements
@@ -3079,17 +3026,35 @@ static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[])
  * slot in the head node, and then the 1:st slot in the
  * resulting node and then finally the 1:st slot in the
  * node beneath. If that slot is not a leaf, then the path
- * continues down the 0:th slot until it finds a leaf and
- * returns that leaf.
+ * continues down the 0:th slot until it finds a leaf.
  *
- * Once the leaf has been found, the Path to the next leaf
- * is built using a stack that was created while traversing
- * the tree down.
+ * Once the leaf has been found, the return value is created
+ * by traversing the tree using the the stack that was built
+ * when searching for the first leaf to return.
  *
  * The index can become a bignum, which complicates the code
  * a bit. However it should be very rare that this happens
  * even on a 32bit system as you would need a tree of depth
  * 7 or more.
+ *
+ * If the number of elements remaining in the map is greater
+ * than how many we want to return, we build a new Path, using
+ * the stack, that points to the next leaf.
+ *
+ * The third argument to this function controls how the data
+ * is returned.
+ *
+ * iterator: The key-value associations are to be used by
+ *           maps:iterator. The return has this format:
+ *             {K1,V1,{K2,V2,none | [Path | Map]}}
+ *           this makes the maps:next function very simple
+ *           and performant.
+ *
+ * list(): The key-value associations are to be used by
+ *         maps:to_list. The return has this format:
+ *             [Path, Map | [{K1,V1},{K2,V2} | BIF_ARG_3]]
+ *                 or if no more associations remain
+ *             [{K1,V1},{K2,V2} | BIF_ARG_3]
  */
 
 #define PATH_ELEM_SIZE 4
@@ -3097,10 +3062,24 @@ static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[])
 #define PATH_ELEM(PATH) ((PATH) & PATH_ELEM_MASK)
 #define PATH_ELEMS_PER_DIGIT (sizeof(ErtsDigit) * 8 / PATH_ELEM_SIZE)
 
-BIF_RETTYPE erts_internal_map_next_2(BIF_ALIST_2) {
+/* How many elements we return in one call depends on the number of reductions
+   that the process has left to run. In debug we return fewer elements to test
+   the Path implementation better. */
+#if defined(DEBUG)
+#define FCALLS_ELEMS(BIF_P) ((BIF_P->fcalls / 4) & 0xF)
+#else
+#define FCALLS_ELEMS(BIF_P) (BIF_P->fcalls / 4)
+#endif
+
+#define NUM_ELEMS_TO_RETURN(BIF_P, map)                   \
+    ((MAX(FCALLS_ELEMS(BIF_P), 1) < hashmap_size(map)) ?  \
+     MAX(FCALLS_ELEMS(BIF_P), 1) : hashmap_size(map))
 
-    Eterm *hp = NULL, *lst,
-        iter, path, map, k, v;
+
+BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
+
+    Eterm path, map;
+    enum { iterator, list } type;
 
     path = BIF_ARG_1;
     map  = BIF_ARG_2;
@@ -3108,41 +3087,65 @@ BIF_RETTYPE erts_internal_map_next_2(BIF_ALIST_2) {
     if (!is_map(map))
         BIF_ERROR(BIF_P, BADARG);
 
-    if (path == am_none)
-        BIF_RET(am_none);
+    if (BIF_ARG_3 == am_iterator) {
+        type = iterator;
+    } else if (is_nil(BIF_ARG_3) || is_list(BIF_ARG_3)) {
+        type = list;
+    } else {
+        BIF_ERROR(BIF_P, BADARG);
+    }
 
     if (is_flatmap(map)) {
-        flatmap_t *mp = (flatmap_t *)flatmap_val(map);
-        Uint idx, sz = flatmap_get_size(mp);
-
-        if (is_not_small(path))
-            BIF_ERROR(BIF_P, BADARG);
+        Uint n;
+	Eterm *ks,*vs, res, *hp;
+	flatmap_t *mp = (flatmap_t*)flatmap_val(map);
 
-        idx = unsigned_val(path);
+	ks  = flatmap_get_keys(mp);
+	vs  = flatmap_get_values(mp);
+	n   = flatmap_get_size(mp);
 
-        if (sz == 0) {
-            if (idx == 0)
-                BIF_RET(am_none);
-            else
-                BIF_ERROR(BIF_P, BADARG);
-        }
+        if (!is_small(BIF_ARG_1) || n < unsigned_val(BIF_ARG_1))
+            BIF_ERROR(BIF_P, BADARG);
 
-        k = flatmap_get_keys(mp)[idx];
-        v = flatmap_get_values(mp)[idx];
+        if (type == iterator) {
+            hp  = HAlloc(BIF_P, 4 * n);
+            res = am_none;
 
-        if (idx + 1 < sz) {
-            path = make_small(idx + 1);
-        } else if (idx < sz) {
-            path = am_none;
+            while(n--) {
+                res = TUPLE3(hp, ks[n], vs[n], res); hp += 4;
+            }
         } else {
-            BIF_ERROR(BIF_P, BADARG);
+            hp  = HAlloc(BIF_P, (2 + 3) * n);
+            res = BIF_ARG_3;
+
+            while(n--) {
+                Eterm tup = TUPLE2(hp, ks[n], vs[n]); hp += 3;
+                res = CONS(hp, tup, res); hp += 2;
+            }
         }
+
+	BIF_RET(res);
     } else {
         Uint curr_path;
         Uint path_length = 0;
         Uint *path_rest = NULL;
-        int i;
-        Eterm node = map;
+        int i, elems = NUM_ELEMS_TO_RETURN(BIF_P, map);
+        Eterm node = map, res, *path_ptr = NULL, *hp;
+
+        /* A stack WSTACK is used when traversing the hashmap.
+         * It contains: node, idx, sz, ptr
+         *
+         * `node` is not really needed, but it is very nice to
+         * have when debugging.
+         *
+         * `idx` always points to the next un-explored entry in
+         * a node. If there are no more un-explored entries,
+         * `idx` is equal to `sz`.
+         *
+         * `sz` is the number of elements in the node.
+         *
+         * `ptr` is a pointer to where the elements of the node begins.
+         */
         DECLARE_WSTACK(stack);
 
         ASSERT(is_hashmap(node));
@@ -3152,7 +3155,7 @@ BIF_RETTYPE erts_internal_map_next_2(BIF_ALIST_2) {
         } else if (is_big(path)) {
             Eterm *big = big_val(path);
             if (bignum_header_is_neg(*big))
-                BIF_ERROR(BIF_P, BADARG);
+                goto badarg;
             path_length = BIG_ARITY(big) - 1;
             curr_path = BIG_DIGIT(big,  0);
             path_rest = BIG_V(big) + 1;
@@ -3160,42 +3163,45 @@ BIF_RETTYPE erts_internal_map_next_2(BIF_ALIST_2) {
             BIF_ERROR(BIF_P, BADARG);
         }
 
-        /* First we look for the K and V to return using the
+        if (type == iterator) {
+            /* iterator uses the format {K, V, {K, V, {K, V, [Path | Map]}}},
+             * so each element is 4 words large */
+            hp = HAlloc(BIF_P, 4 * NUM_ELEMS_TO_RETURN(BIF_P, map));
+            res = am_none;
+        } else {
+            /* list used the format [Path, Map, {K,V}, {K,V} | BIF_ARG_3],
+             * so each element is 2+3 words large */
+            hp = HAlloc(BIF_P, (2 + 3) * NUM_ELEMS_TO_RETURN(BIF_P, map));
+            res = BIF_ARG_3;
+        }
+
+        /* First we look for the leaf to start at using the
            path given. While doing so, we push each map node
-           and the index onto the stack so use later when
-           building the next index. */
+           and the index onto the stack to use later. */
         for (i = 1; ; i++) {
             Eterm *ptr = hashmap_val(node),
-                hdr = *ptr;
+                hdr = *ptr++;
             Uint sz;
 
-            ptr++;
-
-            switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_HEAD_ARRAY:
-		ptr++;
-		sz = 16;
-		break;
-	    case HAMT_SUBTAG_HEAD_BITMAP:
-		ptr++;
-	    case HAMT_SUBTAG_NODE_BITMAP:
-		sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
-		ASSERT(sz < 17);
-		break;
-            default:
-		erts_exit(ERTS_ABORT_EXIT, "bad header");
-	    }
+            sz = hashmap_node_size(hdr, &ptr);
 
             if (PATH_ELEM(curr_path) >= sz)
-                BIF_ERROR(BIF_P, BADARG);
+                goto badarg;
 
-            WSTACK_PUSH2(stack, node, PATH_ELEM(curr_path));
+            WSTACK_PUSH4(stack, node, PATH_ELEM(curr_path)+1, sz, (UWord)ptr);
 
-            /* We have found a leaf, return it and calculate the path to next */
+            /* We have found a leaf, return it and the next X elements */
             if (is_list(ptr[PATH_ELEM(curr_path)])) {
-                lst = list_val(ptr[PATH_ELEM(curr_path)]);
-                k = CAR(lst);
-                v = CDR(lst);
+                Eterm *lst = list_val(ptr[PATH_ELEM(curr_path)]);
+                if (type == iterator) {
+                    res = TUPLE3(hp, CAR(lst), CDR(lst), res); hp += 4;
+                    /* Note where we should patch the Iterator is needed */
+                    path_ptr = hp-1;
+                } else {
+                    Eterm tup = TUPLE2(hp, CAR(lst), CDR(lst)); hp += 3;
+                    res = CONS(hp, tup, res); hp += 2;
+                }
+                elems--;
                 break;
             }
 
@@ -3217,48 +3223,70 @@ BIF_RETTYPE erts_internal_map_next_2(BIF_ALIST_2) {
             }
         }
 
-        /* We pop the stack until we come to the next leaf to return */
-        while (1) {
+        /* We traverse the hashmap and return at most `elems` elements */
+        while(1) {
+            Eterm *ptr = (Eterm*)WSTACK_POP(stack);
+            Uint sz = (Uint)WSTACK_POP(stack);
             Uint idx = (Uint)WSTACK_POP(stack);
             Eterm node = (Eterm)WSTACK_POP(stack);
-            Eterm *ptr = hashmap_val(node),
-                hdr = *ptr;
-            Uint sz;
 
-            ptr++;
-
-            switch(hdr & _HEADER_MAP_SUBTAG_MASK) {
-	    case HAMT_SUBTAG_HEAD_ARRAY:
-		ptr++;
-		sz = 16;
-		break;
-	    case HAMT_SUBTAG_HEAD_BITMAP:
-		ptr++;
-	    case HAMT_SUBTAG_NODE_BITMAP:
-		sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
-		ASSERT(sz < 17);
-		break;
-            default:
-		erts_exit(ERTS_ABORT_EXIT, "bad header");
-	    }
+            while (idx < sz && elems != 0 && is_list(ptr[idx])) {
+                Eterm *lst = list_val(ptr[idx]);
+                if (type == iterator) {
+                    res = TUPLE3(hp, CAR(lst), CDR(lst), res); hp += 4;
+                } else {
+                    Eterm tup = TUPLE2(hp, CAR(lst), CDR(lst)); hp += 3;
+                    res = CONS(hp, tup, res); hp += 2;
+                }
+                elems--;
+                idx++;
+            }
 
-            if (idx + 1 < sz) {
-                curr_path = idx + 1;
+            if (elems == 0) {
+                if (idx < sz) {
+                    /* There are more elements in this node to explore */
+                    WSTACK_PUSH4(stack, node, idx+1, sz, (UWord)ptr);
+                } else {
+                    /* pop stack to find the next value */
+                    while (!WSTACK_ISEMPTY(stack)) {
+                        Eterm *ptr = (Eterm*)WSTACK_POP(stack);
+                        Uint sz = (Uint)WSTACK_POP(stack);
+                        Uint idx = (Uint)WSTACK_POP(stack);
+                        Eterm node = (Eterm)WSTACK_POP(stack);
+                        if (idx < sz) {
+                            WSTACK_PUSH4(stack, node, idx+1, sz, (UWord)ptr);
+                            break;
+                        }
+                    }
+                }
                 break;
+            } else {
+                if (idx < sz) {
+                    Eterm hdr;
+                    /* Push next idx in current node */
+                    WSTACK_PUSH4(stack, node, idx+1, sz, (UWord)ptr);
+
+                    /* Push first idx in child node */
+                    node = ptr[idx];
+                    ptr = hashmap_val(ptr[idx]);
+                    hdr = *ptr++;
+                    sz = hashmap_node_size(hdr, &ptr);
+                    WSTACK_PUSH4(stack, node, 0, sz, (UWord)ptr);
+                }
             }
 
-            /* We have reached the end of the iteration */
+            /* There are no more element in the hashmap */
             if (WSTACK_ISEMPTY(stack)) {
-                path = am_none;
                 break;
             }
 
         }
 
-        if (path != am_none) {
-            Uint depth = WSTACK_COUNT(stack) / 2 + 1;
+        if (!WSTACK_ISEMPTY(stack)) {
+            Uint depth = WSTACK_COUNT(stack) / 4 + 1;
             /* +1 because we already have the first element in curr_path */
             Eterm *path_digits = NULL;
+            Uint curr_path = 0;
 
             /* If the path cannot fit in a small, we allocate a bignum */
             if (depth >= PATH_ELEMS_PER_DIGIT) {
@@ -3269,13 +3297,21 @@ BIF_RETTYPE erts_internal_map_next_2(BIF_ALIST_2) {
                 path_digits = hp + big_size - 1;
             }
 
+
             /* Pop the stack to create the complete path to the next leaf */
             while(!WSTACK_ISEMPTY(stack)) {
-                Uint idx = (Uint)WSTACK_POP(stack);
+                Uint idx;
+
+                (void)WSTACK_POP(stack);
+                (void)WSTACK_POP(stack);
+                idx = (Uint)WSTACK_POP(stack)-1;
+                /* idx - 1 because idx in the stack is pointing to
+                   the next element to fetch. */
                 (void)WSTACK_POP(stack);
 
                 depth--;
                 if (depth % PATH_ELEMS_PER_DIGIT == 0) {
+                    /* Switch to next bignum element */
                     path_digits[0] = curr_path;
                     path_digits--;
                     curr_path = 0;
@@ -3292,15 +3328,36 @@ BIF_RETTYPE erts_internal_map_next_2(BIF_ALIST_2) {
                 /* The Uint could be too large for a small */
                 path = erts_make_integer(curr_path, BIF_P);
             }
+
+            if (type == iterator) {
+                hp = HAlloc(BIF_P, 2);
+                *path_ptr = CONS(hp, path, map); hp += 2;
+            } else {
+                hp = HAlloc(BIF_P, 4);
+                res = CONS(hp, map, res); hp += 2;
+                res = CONS(hp, path, res); hp += 2;
+            }
+        } else {
+            if (type == iterator) {
+                HRelease(BIF_P, hp + 4 * elems, hp);
+            } else {
+                HRelease(BIF_P, hp + (2+3) * elems, hp);
+            }
         }
+        BIF_P->fcalls -= 4 * (NUM_ELEMS_TO_RETURN(BIF_P, map) - elems);
+        DESTROY_WSTACK(stack);
+        BIF_RET(res);
 
+    badarg:
+        if (type == iterator) {
+            HRelease(BIF_P, hp + 4 * elems, hp);
+        } else {
+            HRelease(BIF_P, hp + (2+3) * elems, hp);
+        }
+        BIF_P->fcalls -= 4 * (NUM_ELEMS_TO_RETURN(BIF_P, map) - elems);
         DESTROY_WSTACK(stack);
+        BIF_ERROR(BIF_P, BADARG);
     }
-
-    hp = HAlloc(BIF_P, 4 + 3 /* 3-tuple + 2-tuple */);
-    iter = TUPLE2(hp, path, map);
-    hp += 3;
-    BIF_RET(TUPLE3(hp, k, v, iter));
 }
 
 /* implementation of builtin emulations */
diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl
index 3cf071b590..1e70d31b16 100644
--- a/erts/emulator/test/map_SUITE.erl
+++ b/erts/emulator/test/map_SUITE.erl
@@ -2366,41 +2366,6 @@ t_bif_map_from_list(Config) when is_list(Config) ->
     {'EXIT', {badarg,_}} = (catch maps:from_list(id(42))),
     ok.
 
-t_bif_erts_internal_maps_to_list(Config) when is_list(Config) ->
-    %% small maps
-    [] = erts_internal:maps_to_list(#{},-1),
-    [] = erts_internal:maps_to_list(#{},-2),
-    [] = erts_internal:maps_to_list(#{},10),
-    [{a,1},{b,2}] = lists:sort(erts_internal:maps_to_list(#{a=>1,b=>2}, 2)),
-    [{a,1},{b,2}] = lists:sort(erts_internal:maps_to_list(#{a=>1,b=>2}, -1)),
-    [{_,_}] = erts_internal:maps_to_list(#{a=>1,b=>2}, 1),
-    [{a,1},{b,2},{c,3}] = lists:sort(erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},-2)),
-    [{a,1},{b,2},{c,3}] = lists:sort(erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},3)),
-    [{a,1},{b,2},{c,3}] = lists:sort(erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},5)),
-    [{_,_},{_,_}] = erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},2),
-    [{_,_}] = erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},1),
-    [] = erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},0),
-
-    %% big maps
-    M = maps:from_list([{I,ok}||I <- lists:seq(1,500)]),
-    [] = erts_internal:maps_to_list(M,0),
-    [{_,_}] = erts_internal:maps_to_list(M,1),
-    [{_,_},{_,_}] = erts_internal:maps_to_list(M,2),
-    Ls1 = erts_internal:maps_to_list(M,10),
-    10 = length(Ls1),
-    Ls2 = erts_internal:maps_to_list(M,20),
-    20 = length(Ls2),
-    Ls3 = erts_internal:maps_to_list(M,120),
-    120 = length(Ls3),
-    Ls4 = erts_internal:maps_to_list(M,-1),
-    500 = length(Ls4),
-
-    %% error cases
-    {'EXIT', {{badmap,[{a,b},b]},_}} = (catch erts_internal:maps_to_list(id([{a,b},b]),id(1))),
-    {'EXIT', {badarg,_}} = (catch erts_internal:maps_to_list(id(#{}),id(a))),
-    {'EXIT', {badarg,_}} = (catch erts_internal:maps_to_list(id(#{1=>2}),id(<<>>))),
-    ok.
-
 t_bif_map_next(Config) when is_list(Config) ->
 
     erts_debug:set_internal_state(available_internal_state, true),
@@ -2420,9 +2385,25 @@ t_bif_map_next(Config) when is_list(Config) ->
         {'EXIT', {badarg,_}} = (catch maps:next(id(a))),
         {'EXIT', {badarg,_}} = (catch maps:next(id([a|FM]))),
         {'EXIT', {badarg,_}} = (catch maps:next(id([1|#{}]))),
-        {'EXIT', {badarg,_}} = (catch maps:next(id([16#FFFFFFF|FM]))),
-        {'EXIT', {badarg,_}} = (catch maps:next(id([16#F0F0F0F|FM]))),
-        {'EXIT', {badarg,_}} = (catch maps:next(id([16#1234567890ABCDEF|FM]))),
+        {'EXIT', {badarg,_}} = (catch maps:next(id([-1|#{}]))),
+        {'EXIT', {badarg,_}} = (catch maps:next(id([-1|FM]))),
+        {'EXIT', {badarg,_}} = (catch maps:next(id([16#FFFFFFFFFFFFFFFF|FM]))),
+        {'EXIT', {badarg,_}} = (catch maps:next(id([-16#FFFFFFFFFFFFFFFF|FM]))),
+
+        %% This us a whitebox test that the error code works correctly.
+        %% It uses a path for a tree of depth 4 and tries to do next on
+        %% each of those paths.
+        (fun F(0) -> ok;
+             F(N) ->
+                 try maps:next([N|FM]) of
+                     none ->
+                         F(N-1);
+                     {_K,_V,_I} ->
+                         F(N-1)
+                 catch error:badarg ->
+                         F(N-1)
+                 end
+         end)(16#FFFF),
 
         ok
     after
-- 
cgit v1.2.3


From 488049bd59352670ba4df17df7cc8a288b273b63 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 12 Oct 2017 16:01:43 +0200
Subject: erts: Remove erts_internal:maps_to_list/2

This function is no longer needed as maps:iterator has
now been implemented.
---
 erts/emulator/beam/bif.tab       |  1 -
 erts/emulator/beam/erl_map.c     | 70 ----------------------------------------
 erts/emulator/test/map_SUITE.erl |  2 --
 3 files changed, 73 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index b7b66e2ee7..5d5ea3cdc4 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -683,7 +683,6 @@ bif math:floor/1
 bif math:ceil/1
 bif math:fmod/2
 bif os:set_signal/2
-bif erts_internal:maps_to_list/2
 
 #
 # New in 20.1
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 135053dd18..cbbd62f1a2 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -91,7 +91,6 @@ static BIF_RETTYPE hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_
 static Export hashmap_merge_trap_export;
 static BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1);
 static Uint hashmap_subtree_size(Eterm node);
-static Eterm hashmap_to_list(Process *p, Eterm map, Sint n);
 static Eterm hashmap_keys(Process *p, Eterm map);
 static Eterm hashmap_values(Process *p, Eterm map);
 static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node, Eterm *value);
@@ -139,51 +138,6 @@ BIF_RETTYPE map_size_1(BIF_ALIST_1) {
     BIF_ERROR(BIF_P, BADMAP);
 }
 
-/* erts_internal:maps_to_list/2
- *
- * This function should be removed once iterators are in place.
- * Never document it.
- * Never encourage its usage.
- *
- * A negative value in ARG 2 means the entire map.
- */
-
-BIF_RETTYPE erts_internal_maps_to_list_2(BIF_ALIST_2) {
-    Sint m;
-    if (term_to_Sint(BIF_ARG_2, &m)) {
-        if (is_flatmap(BIF_ARG_1)) {
-            Uint n;
-            Eterm* hp;
-            Eterm *ks,*vs, res, tup;
-            flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1);
-
-            ks  = flatmap_get_keys(mp);
-            vs  = flatmap_get_values(mp);
-            n   = flatmap_get_size(mp);
-
-            if (m >= 0) {
-                n = m < n ? m : n;
-            }
-
-            hp  = HAlloc(BIF_P, (2 + 3) * n);
-            res = NIL;
-
-            while(n--) {
-                tup = TUPLE2(hp, ks[n], vs[n]); hp += 3;
-                res = CONS(hp, tup, res); hp += 2;
-            }
-
-            BIF_RET(res);
-        } else if (is_hashmap(BIF_ARG_1)) {
-            return hashmap_to_list(BIF_P, BIF_ARG_1, m);
-        }
-        BIF_P->fvalue = BIF_ARG_1;
-        BIF_ERROR(BIF_P, BADMAP);
-    }
-    BIF_ERROR(BIF_P, BADARG);
-}
-
-
 /* maps:find/2
  * return value if key *matches* a key in the map
  */
@@ -1933,30 +1887,6 @@ BIF_RETTYPE maps_values_1(BIF_ALIST_1) {
     BIF_ERROR(BIF_P, BADMAP);
 }
 
-static Eterm hashmap_to_list(Process *p, Eterm node, Sint m) {
-    DECLARE_WSTACK(stack);
-    Eterm *hp, *kv;
-    Eterm tup, res = NIL;
-    Uint n = hashmap_size(node);
-
-    if (m >= 0) {
-        n = m < n ? m : n;
-    }
-
-    hp  = HAlloc(p, n * (2 + 3));
-    hashmap_iterator_init(&stack, node, 0);
-    while (n--) {
-        kv  = hashmap_iterator_next(&stack);
-        ASSERT(kv != NULL);
-	tup = TUPLE2(hp, CAR(kv), CDR(kv));
-	hp += 3;
-	res = CONS(hp, tup, res);
-	hp += 2;
-    }
-    DESTROY_WSTACK(stack);
-    return res;
-}
-
 static ERTS_INLINE
 Uint hashmap_node_size(Eterm hdr, Eterm **nodep)
 {
diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl
index 1e70d31b16..c9e971af8a 100644
--- a/erts/emulator/test/map_SUITE.erl
+++ b/erts/emulator/test/map_SUITE.erl
@@ -52,7 +52,6 @@
          t_bif_map_values/1,
          t_bif_map_to_list/1,
          t_bif_map_from_list/1,
-         t_bif_erts_internal_maps_to_list/1,
          t_bif_map_next/1,
 
          %% erlang
@@ -120,7 +119,6 @@ all() -> [t_build_and_match_literals, t_build_and_match_literals_large,
           t_bif_map_update,
           t_bif_map_values,
           t_bif_map_to_list, t_bif_map_from_list,
-          t_bif_erts_internal_maps_to_list,
           t_bif_map_next,
 
           %% erlang
-- 
cgit v1.2.3


From 10912210641fe7d784c4ba6398c24504200ed339 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 18 Oct 2017 15:34:10 +0200
Subject: erts: Limit size of first iterator for hashmaps

---
 erts/emulator/beam/erl_map.c | 53 +++++++++++++++++++++++++++-----------------
 1 file changed, 33 insertions(+), 20 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index cbbd62f1a2..8047a9567f 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -2992,20 +2992,6 @@ static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[])
 #define PATH_ELEM(PATH) ((PATH) & PATH_ELEM_MASK)
 #define PATH_ELEMS_PER_DIGIT (sizeof(ErtsDigit) * 8 / PATH_ELEM_SIZE)
 
-/* How many elements we return in one call depends on the number of reductions
-   that the process has left to run. In debug we return fewer elements to test
-   the Path implementation better. */
-#if defined(DEBUG)
-#define FCALLS_ELEMS(BIF_P) ((BIF_P->fcalls / 4) & 0xF)
-#else
-#define FCALLS_ELEMS(BIF_P) (BIF_P->fcalls / 4)
-#endif
-
-#define NUM_ELEMS_TO_RETURN(BIF_P, map)                   \
-    ((MAX(FCALLS_ELEMS(BIF_P), 1) < hashmap_size(map)) ?  \
-     MAX(FCALLS_ELEMS(BIF_P), 1) : hashmap_size(map))
-
-
 BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
 
     Eterm path, map;
@@ -3059,7 +3045,7 @@ BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
         Uint curr_path;
         Uint path_length = 0;
         Uint *path_rest = NULL;
-        int i, elems = NUM_ELEMS_TO_RETURN(BIF_P, map);
+        int i, elems, orig_elems;
         Eterm node = map, res, *path_ptr = NULL, *hp;
 
         /* A stack WSTACK is used when traversing the hashmap.
@@ -3080,12 +3066,37 @@ BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
 
         ASSERT(is_hashmap(node));
 
+/* How many elements we return in one call depends on the number of reductions
+ * that the process has left to run. In debug we return fewer elements to test
+ * the Path implementation better.
+ *
+ * Also, when the path is 0 (i.e. for the first call) we limit the number of
+ * elements to MAP_SMALL_MAP_LIMIT in order to not use a huge amount of heap
+ * when only the first X associations in the hashmap was needed.
+ */
+#if defined(DEBUG)
+#define FCALLS_ELEMS(BIF_P) ((BIF_P->fcalls / 4) & 0xF)
+#else
+#define FCALLS_ELEMS(BIF_P) (BIF_P->fcalls / 4)
+#endif
+
+        if (MAX(FCALLS_ELEMS(BIF_P), 1) < hashmap_size(map))
+            elems = MAX(FCALLS_ELEMS(BIF_P), 1);
+        else
+            elems = hashmap_size(map);
+
+#undef FCALLS_ELEMS
+
         if (is_small(path)) {
             curr_path = unsigned_val(path);
+
+            if (curr_path == 0 && elems > MAP_SMALL_MAP_LIMIT) {
+                elems = MAP_SMALL_MAP_LIMIT;
+            }
         } else if (is_big(path)) {
             Eterm *big = big_val(path);
             if (bignum_header_is_neg(*big))
-                goto badarg;
+                BIF_ERROR(BIF_P, BADARG);
             path_length = BIG_ARITY(big) - 1;
             curr_path = BIG_DIGIT(big,  0);
             path_rest = BIG_V(big) + 1;
@@ -3096,15 +3107,17 @@ BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
         if (type == iterator) {
             /* iterator uses the format {K, V, {K, V, {K, V, [Path | Map]}}},
              * so each element is 4 words large */
-            hp = HAlloc(BIF_P, 4 * NUM_ELEMS_TO_RETURN(BIF_P, map));
+            hp = HAlloc(BIF_P, 4 * elems);
             res = am_none;
         } else {
             /* list used the format [Path, Map, {K,V}, {K,V} | BIF_ARG_3],
              * so each element is 2+3 words large */
-            hp = HAlloc(BIF_P, (2 + 3) * NUM_ELEMS_TO_RETURN(BIF_P, map));
+            hp = HAlloc(BIF_P, (2 + 3) * elems);
             res = BIF_ARG_3;
         }
 
+        orig_elems = elems;
+
         /* First we look for the leaf to start at using the
            path given. While doing so, we push each map node
            and the index onto the stack to use later. */
@@ -3274,7 +3287,7 @@ BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
                 HRelease(BIF_P, hp + (2+3) * elems, hp);
             }
         }
-        BIF_P->fcalls -= 4 * (NUM_ELEMS_TO_RETURN(BIF_P, map) - elems);
+        BIF_P->fcalls -= 4 * (orig_elems - elems);
         DESTROY_WSTACK(stack);
         BIF_RET(res);
 
@@ -3284,7 +3297,7 @@ BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
         } else {
             HRelease(BIF_P, hp + (2+3) * elems, hp);
         }
-        BIF_P->fcalls -= 4 * (NUM_ELEMS_TO_RETURN(BIF_P, map) - elems);
+        BIF_P->fcalls -= 4 * (orig_elems - elems);
         DESTROY_WSTACK(stack);
         BIF_ERROR(BIF_P, BADARG);
     }
-- 
cgit v1.2.3


From 23e511d470e6795799c29b5ccb2ab6a17589e543 Mon Sep 17 00:00:00 2001
From: Boshan Sun <boshan@subsplash.com>
Date: Mon, 20 Nov 2017 10:21:29 -0800
Subject: Fix integer overflow when set a large maximum value for atom table

When setting maximum atom table size using +t option, there will be a
integer overflow for a large size.

	$ erl +t2147482625
	ll_alloc: Cannot allocate 18446744073692774400 bytes of memory
	(of type "atom_tab").

The overflow is caused by the arithmetic operations on int type.

When 2147482625 + 1024 it will become -2147483647 due to the signed
integerger overflow. Then the result will be resized to Uint type, which
is a unsigned long type, the negative int will first be expand to 64
bits long via sign extension, then change to unsigned type, which
becomes 18446744073692774400.

The fix is done by convert `limit` to Uint type before doing any
arithmetic operation. This will expand variable to 64 bits long type via
zero extension, then the following operation are all positive, therefore
no overflow will happen.

Note: here we assume the int `limit` passed in is always positive. If
some future change cause the `limit` passed in maybe negative, then the
current fix will also cause overflow.
---
 erts/emulator/beam/index.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c
index a1f6f54543..7bf1a032c1 100644
--- a/erts/emulator/beam/index.c
+++ b/erts/emulator/beam/index.c
@@ -58,7 +58,7 @@ IndexTable*
 erts_index_init(ErtsAlcType_t type, IndexTable* t, char* name,
 		int size, int limit, HashFunctions fun)
 {
-    Uint base_size = ((limit+INDEX_PAGE_SIZE-1)/INDEX_PAGE_SIZE)*sizeof(IndexSlot*);
+    Uint base_size = (((Uint)limit+INDEX_PAGE_SIZE-1)/INDEX_PAGE_SIZE)*sizeof(IndexSlot*);
     hash_init(type, &t->htable, name, 3*size/4, fun);
 
     t->size = 0;
-- 
cgit v1.2.3


From 864362749246d15355e904cf95248a634eebdfb7 Mon Sep 17 00:00:00 2001
From: Boshan Sun <boshan@subsplash.com>
Date: Mon, 20 Nov 2017 10:41:48 -0800
Subject: Fix max atom size overflow on 64-bits Erlang by lowering the
 MAX_ATOM_TABLE_SIZE

Currently, the max atom size on 64-bits Erlang is

	((UWORD_CONSTANT(1) << 32) = 4294967296

This number will cause the range of atom size to be displayed as
[8192-0].

Also, the +t option for max atom size will be parsed as a long type, and
assigned to a int variable erts_atom_table_size (erl_init.c),
which will cause integer overflow if the number is larger than the
maximum value a 4-bytes signed integer can hold

	((1 << 31) - 1) = 2147483647

Therefore, during the comparison

	erts_atom_table_size < MIN_ATOM_TABLE_SIZE

any number above 2147483647 will be come negative, and causing
the condition to be true, which then errored out as bad atom table size.

Hence, the actual max atom size is same as the max signed int value.
---
 erts/emulator/beam/atom.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h
index be998a46bd..385120a8d9 100644
--- a/erts/emulator/beam/atom.h
+++ b/erts/emulator/beam/atom.h
@@ -36,7 +36,7 @@
 /* Internal atom cache needs MAX_ATOM_TABLE_SIZE to be less than an
    unsigned 32 bit integer. See external.c(erts_encode_ext_dist_header_setup)
    for more details. */
-#define MAX_ATOM_TABLE_SIZE ((MAX_ATOM_INDEX + 1 < (UWORD_CONSTANT(1) << 32)) ? MAX_ATOM_INDEX + 1 : (UWORD_CONSTANT(1) << 32))
+#define MAX_ATOM_TABLE_SIZE ((MAX_ATOM_INDEX + 1 < (UWORD_CONSTANT(1) << 32)) ? MAX_ATOM_INDEX + 1 : ((UWORD_CONSTANT(1) << 31) - 1)) /* Here we use maximum signed interger value to avoid integer overflow */
 #else
 #define MAX_ATOM_TABLE_SIZE (MAX_ATOM_INDEX + 1)
 #endif
-- 
cgit v1.2.3


From 3b964e8dbaa0cd73ca7a983b3ce948e0dbd2c35c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 16 Oct 2017 15:12:06 +0200
Subject: Use base64 encoding in crash dumps

This will reduce the size of crash dumps, especially if
there are large binaries.
---
 erts/emulator/beam/break.c            | 31 ++++++++++++++++++++++----
 erts/emulator/beam/erl_process_dump.c | 42 +++++++++++++++++------------------
 erts/emulator/beam/global.h           |  1 +
 3 files changed, 48 insertions(+), 26 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index e57be5a595..e4c2801ea2 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -492,9 +492,7 @@ loaded(fmtfn_t to, void *to_arg)
 static void
 dump_attributes(fmtfn_t to, void *to_arg, byte* ptr, int size)
 {
-    while (size-- > 0) {
-	erts_print(to, to_arg, "%02X", *ptr++);
-    }
+    erts_print_base64(to, to_arg, ptr, size);
     erts_print(to, to_arg, "\n");
 }
 
@@ -858,7 +856,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     }
 
     time(&now);
-    erts_cbprintf(to, to_arg, "=erl_crash_dump:0.4\n%s", ctime(&now));
+    erts_cbprintf(to, to_arg, "=erl_crash_dump:0.5\n%s", ctime(&now));
 
     if (file != NULL)
        erts_cbprintf(to, to_arg, "The error occurred in file %s, line %d\n", file, line);
@@ -975,3 +973,28 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     erts_fprintf(stderr,"done\n");
 }
 
+void
+erts_print_base64(fmtfn_t to, void *to_arg, byte* src, Uint size)
+{
+    static const byte base64_chars[] =
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+    while (size >= 3) {
+        erts_putc(to, to_arg, base64_chars[src[0] >> 2]);
+        erts_putc(to, to_arg, base64_chars[((src[0] & 0x03) << 4) | (src[1] >> 4)]);
+        erts_putc(to, to_arg, base64_chars[((src[1] & 0x0f) << 2) | (src[2] >> 6)]);
+        erts_putc(to, to_arg, base64_chars[src[2] & 0x3f]);
+        size -= 3;
+        src += 3;
+    }
+    if (size == 1) {
+        erts_putc(to, to_arg, base64_chars[src[0] >> 2]);
+        erts_putc(to, to_arg, base64_chars[(src[0] & 0x03) << 4]);
+        erts_print(to, to_arg, "==");
+    } else if (size == 2) {
+        erts_putc(to, to_arg, base64_chars[src[0] >> 2]);
+        erts_putc(to, to_arg, base64_chars[((src[0] & 0x03) << 4) | (src[1] >> 4)]);
+        erts_putc(to, to_arg, base64_chars[(src[1] & 0x0f) << 2]);
+        erts_putc(to, to_arg, '=');
+    }
+}
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 5641195458..e0b795fbf3 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -51,6 +51,8 @@ static void stack_trace_dump(fmtfn_t to, void *to_arg, Eterm* sp);
 static void print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x);
 static void heap_dump(fmtfn_t to, void *to_arg, Eterm x);
 static void dump_binaries(fmtfn_t to, void *to_arg, Binary* root);
+void erts_print_base64(fmtfn_t to, void *to_arg,
+                       byte* src, Uint size);
 static void dump_externally(fmtfn_t to, void *to_arg, Eterm term);
 static void mark_literal(Eterm* ptr);
 static void init_literal_areas(void);
@@ -193,6 +195,7 @@ dump_dist_ext(fmtfn_t to, void *to_arg, ErtsDistExternal *edep)
     else {
 	byte *e;
 	size_t sz;
+
 	if (!(edep->flags & ERTS_DIST_EXT_ATOM_TRANS_TAB))
 	    erts_print(to, to_arg, "D0:");
 	else {
@@ -210,12 +213,18 @@ dump_dist_ext(fmtfn_t to, void *to_arg, ErtsDistExternal *edep)
 	else {
 	    ASSERT(*e == VERSION_MAGIC);
 	}
-
 	erts_print(to, to_arg, "E%X:", sz);
-	if (edep->flags & ERTS_DIST_EXT_DFLAG_HDR)
-	    erts_print(to, to_arg, "%02X", VERSION_MAGIC);
-	while (e < edep->ext_endp)
-	    erts_print(to, to_arg, "%02X", *e++);
+        if (edep->flags & ERTS_DIST_EXT_DFLAG_HDR) {
+            byte sbuf[3];
+            int i = 0;
+
+            sbuf[i++] = VERSION_MAGIC;
+            while (i < sizeof(sbuf) && e < edep->ext_endp) {
+                sbuf[i++] = *e++;
+            }
+            erts_print_base64(to, to_arg, sbuf, i);
+        }
+        erts_print_base64(to, to_arg, e, edep->ext_endp - e);
     }
 }
 
@@ -444,16 +453,13 @@ heap_dump(fmtfn_t to, void *to_arg, Eterm x)
 		} else if (is_binary_header(hdr)) {
 		    Uint tag = thing_subtag(hdr);
 		    Uint size = binary_size(x);
-		    Uint i;
 
 		    if (tag == HEAP_BINARY_SUBTAG) {
 			byte* p;
 
 			erts_print(to, to_arg, "Yh%X:", size);
 			p = binary_bytes(x);
-			for (i = 0; i < size; i++) {
-			    erts_print(to, to_arg, "%02X", p[i]);
-			}
+                        erts_print_base64(to, to_arg, p, size);
 		    } else if (tag == REFC_BINARY_SUBTAG) {
 			ProcBin* pb = (ProcBin *) binary_val(x);
 			Binary* val = pb->val;
@@ -596,16 +602,13 @@ static void
 dump_binaries(fmtfn_t to, void *to_arg, Binary* current)
 {
     while (current) {
-	long i;
-	long size = current->orig_size;
+	SWord size = current->orig_size;
 	byte* bytes = (byte*) current->orig_bytes;
 
 	erts_print(to, to_arg, "=binary:" PTR_FMT "\n", current);
 	erts_print(to, to_arg, "%X:", size);
-	for (i = 0; i < size; i++) {
-	    erts_print(to, to_arg, "%02X", bytes[i]);
-	}
-	erts_putc(to, to_arg, '\n');
+        erts_print_base64(to, to_arg, bytes, size);
+        erts_putc(to, to_arg, '\n');
 	current = (Binary *) current->intern.flags;
     }
 }
@@ -644,9 +647,7 @@ dump_externally(fmtfn_t to, void *to_arg, Eterm term)
     s = p = sbuf;
     erts_encode_ext(term, &p);
     erts_print(to, to_arg, "E%X:", p-s);
-    while (s < p) {
-	erts_print(to, to_arg, "%02X", *s++);
-    }
+    erts_print_base64(to, to_arg, sbuf, p-s);
 }
 
 /*
@@ -802,16 +803,13 @@ dump_module_literals(fmtfn_t to, void *to_arg, ErtsLiteralArea* lit_area)
             } else if (is_binary_header(w)) {
                 Uint tag = thing_subtag(w);
                 Uint size = binary_size(term);
-                Uint i;
 
                 if (tag == HEAP_BINARY_SUBTAG) {
                     byte* p;
 
                     erts_print(to, to_arg, "Yh%X:", size);
                     p = binary_bytes(term);
-                    for (i = 0; i < size; i++) {
-                        erts_print(to, to_arg, "%02X", p[i]);
-                    }
+                    erts_print_base64(to, to_arg, p, size);
                 } else if (tag == REFC_BINARY_SUBTAG) {
                     ProcBin* pb = (ProcBin *) binary_val(term);
                     Binary* val = pb->val;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 440723aea6..9505942307 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -966,6 +966,7 @@ void process_info(fmtfn_t, void *);
 void print_process_info(fmtfn_t, void *, Process*);
 void info(fmtfn_t, void *);
 void loaded(fmtfn_t, void *);
+void erts_print_base64(fmtfn_t to, void *to_arg, byte* src, Uint size);
 
 /* sighandler sys.c */
 int erts_set_signal(Eterm signal, Eterm type);
-- 
cgit v1.2.3


From f3014f40a1e81e2f83c261e541ade39b7dcda24d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 21 Nov 2017 15:40:22 +0100
Subject: erts: Remove obsolete comments

---
 erts/emulator/beam/erl_binary.h | 7 -------
 1 file changed, 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 05007e864e..46653a8580 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -363,8 +363,6 @@ erts_free_aligned_binary_bytes(byte* buf)
 #  define CHICKEN_PAD (sizeof(void*) - 1)
 #endif
 
-/* Caller must initialize 'refc'
-*/
 ERTS_GLB_INLINE Binary *
 erts_bin_drv_alloc_fnf(Uint size)
 {
@@ -383,8 +381,6 @@ erts_bin_drv_alloc_fnf(Uint size)
     return res;
 }
 
-/* Caller must initialize 'refc'
-*/
 ERTS_GLB_INLINE Binary *
 erts_bin_drv_alloc(Uint size)
 {
@@ -401,9 +397,6 @@ erts_bin_drv_alloc(Uint size)
     return res;
 }
 
-
-/* Caller must initialize 'refc'
-*/
 ERTS_GLB_INLINE Binary *
 erts_bin_nrml_alloc(Uint size)
 {
-- 
cgit v1.2.3


From a5aaba7c9a316327940d0b861dd300d59aa99209 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 21 Nov 2017 14:40:06 +0100
Subject: erts: Fix preemption bug in erts_dist_command

We should break out when out of reductions.
---
 erts/emulator/beam/dist.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 16a7bd9eb3..ee288dc0bd 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -2335,10 +2335,10 @@ erts_dist_command(Port *prt, int initial_reds)
 	    ASSERT(ob);
 	    do {
 		reds = erts_encode_ext_dist_header_finalize(ob, dep, flags, reds);
-                if (reds >= 0) {
-                    last_finalized  = ob;
-                    ob = ob->next;
-                }
+                if (reds < 0)
+                    break;
+                last_finalized  = ob;
+                ob = ob->next;
 	    } while (ob);
             if (last_finalized) {
                 /*
-- 
cgit v1.2.3


From 8ed0d75c186d9da24bd6cfb85732487b17a3b054 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 23 Nov 2017 15:05:34 +0100
Subject: erts: Fix erlang:monitor toward c-nodes

by suppressing DOP_MONITOR_P, DOP_MONITOR_P_EXIT and DOP_DEMONITOR_P
if not supported by the remote node.

In 17e198d6ee60f7dec9abfed272cf4226aea44535
I changed the behavior of erlang:monitor
to not raise badarg for c-nodes but instead create
a monitor to only supervise the connection.

But I forgot to prevent DOP_MONITOR_P and friends from being
sent to the node that does not expect them.

Note: We test both DFLAG_DIST_MONITOR and DFLAG_DIST_MONITOR_NAME
      for the node to support process monitoring. This is because
      erl_interface is buggy as it sets DFLAG_DIST_MONITOR without
      really supporting it.
ToDo: Should erl_interface stop setting DFLAG_DIST_MONITOR
      or should we change the meaning of these flags?
---
 erts/emulator/beam/dist.c     | 26 +++++++++++++++++++++++++-
 erts/emulator/beam/dist.h     |  4 +++-
 erts/emulator/beam/external.c | 15 +++++++++++++++
 3 files changed, 43 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index ee288dc0bd..8548dbb577 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -877,6 +877,13 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
     DeclareTmpHeapNoproc(ctl_heap,6);
     int res;
 
+    if (~dsdp->dep->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) {
+        /*
+         * Receiver does not support DOP_MONITOR_P_EXIT (see dsig_send_monitor)
+         */
+        return ERTS_DSIG_SEND_OK;
+    }
+
     UseTmpHeapNoproc(6);
 
     ctl = TUPLE5(&ctl_heap[0], make_small(DOP_MONITOR_P_EXIT),
@@ -904,6 +911,16 @@ erts_dsig_send_monitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
     DeclareTmpHeapNoproc(ctl_heap,5);
     int res;
 
+    if (~dsdp->dep->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) {
+        /*
+         * Receiver does not support DOP_MONITOR_P.
+         * Just avoid sending it and by doing that reduce this monitor
+         * to only supervise the connection. This will work for simple c-nodes
+         * with a 1-to-1 relation between "Erlang process" and OS-process.
+         */
+        return ERTS_DSIG_SEND_OK;
+    }
+
     UseTmpHeapNoproc(5);
     ctl = TUPLE4(&ctl_heap[0],
 		 make_small(DOP_MONITOR_P),
@@ -926,6 +943,13 @@ erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher,
     DeclareTmpHeapNoproc(ctl_heap,5);
     int res;
 
+    if (~dsdp->dep->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) {
+        /*
+         * Receiver does not support DOP_DEMONITOR_P (see dsig_send_monitor)
+         */
+        return ERTS_DSIG_SEND_OK;
+    }
+
     UseTmpHeapNoproc(5);
     ctl = TUPLE4(&ctl_heap[0],
 		 make_small(DOP_DEMONITOR_P),
@@ -2379,7 +2403,7 @@ erts_dist_command(Port *prt, int initial_reds)
                 break;
             }
 	    ASSERT(&oq.first->data[0] <= oq.first->extp
-		   && oq.first->extp < oq.first->ext_endp);
+		   && oq.first->extp <= oq.first->ext_endp);
 	    size = (*send)(prt, oq.first);
             erts_atomic64_inc_nob(&dep->out);
 	    esdp->io.out += (Uint64) size;
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index a96e39be89..ea4697815f 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -56,7 +56,9 @@
 /* Additional optimistic flags when encoding toward pending connection */
 #define DFLAG_DIST_HOPEFULLY (DFLAG_NO_MAGIC                    \
                               | DFLAG_EXPORT_PTR_TAG            \
-                              | DFLAG_BIT_BINARIES)
+                              | DFLAG_BIT_BINARIES              \
+                              | DFLAG_DIST_MONITOR              \
+                              | DFLAG_DIST_MONITOR_NAME)
 
 /* All flags that should be enabled when term_to_binary/1 is used. */
 #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES	\
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index f2e6399ad7..9cc5e71afa 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -383,6 +383,21 @@ Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob,
          */
         ASSERT(ep[0] == SMALL_TUPLE_EXT || ep[0] == LARGE_TUPLE_EXT);
 
+        if (~dflags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)
+            && ep[0] == SMALL_TUPLE_EXT
+            && ep[1] == 4
+            && ep[2] == SMALL_INTEGER_EXT
+            && (ep[3] == DOP_MONITOR_P ||
+                ep[3] == DOP_MONITOR_P_EXIT ||
+                ep[3] == DOP_DEMONITOR_P)) {
+            /*
+             * Receiver does not support process monitoring.
+             * Suppress monitor control msg (see erts_dsig_send_monitor)
+             * by converting it to an empty (tick) packet.
+             */
+            ob->ext_endp = ob->extp;
+            return reds;
+        }
         if (~dflags & (DFLAG_BIT_BINARIES | DFLAG_EXPORT_PTR_TAG
                        | DFLAG_DIST_HDR_ATOM_CACHE)) {
             reds = transcode_dist_obuf(ob, dep, dflags, reds);
-- 
cgit v1.2.3


From 3105c314beff5c9b226d8e863d32c4329706353c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 24 Nov 2017 12:50:25 +0100
Subject: Fix purging of modules with "fake literals"

When compiling Erlang source code, the literal area for the
module can only contain data types that have a literal
syntax.

However, it is possible to sneak in other data types
(such as references) in the literal pool by compiling from
abstract or assembly code. Those "fake literals" would work
fine, but would crash the runtime system when the module containing
the literals was purged.

Although fake literals are not officially supported, the
runtime should not crash when attempting to use them.
Therefore, fix the garbage collection of literals and releasing
of literal areas.

https://bugs.erlang.org/browse/ERL-508
---
 erts/emulator/beam/beam_load.c    | 35 +++++++++++++++++++----
 erts/emulator/beam/erl_gc.c       | 47 +++++++++++++++++++++++-------
 erts/emulator/test/code_SUITE.erl | 60 ++++++++++++++++++++++++++++++++++++++-
 3 files changed, 124 insertions(+), 18 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 23258dbe9c..adf8779f11 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -5665,13 +5665,36 @@ erts_release_literal_area(ErtsLiteralArea* literal_area)
 	return;
 
     oh = literal_area->off_heap;
-	
+
     while (oh) {
-	Binary* bptr;
-	ASSERT(thing_subtag(oh->thing_word) == REFC_BINARY_SUBTAG);
-	bptr = ((ProcBin*)oh)->val;
-        erts_bin_release(bptr);
-	oh = oh->next;
+        switch (thing_subtag(oh->thing_word)) {
+        case REFC_BINARY_SUBTAG:
+            {
+                Binary* bptr = ((ProcBin*)oh)->val;
+                erts_bin_release(bptr);
+                break;
+            }
+        case FUN_SUBTAG:
+            {
+                ErlFunEntry* fe = ((ErlFunThing*)oh)->fe;
+                if (erts_smp_refc_dectest(&fe->refc, 0) == 0) {
+                    erts_erase_fun_entry(fe);
+                }
+                break;
+            }
+        case REF_SUBTAG:
+            {
+                ErtsMagicBinary *bptr;
+                ASSERT(is_magic_ref_thing(oh));
+                bptr = ((ErtsMRefThing *) oh)->mb;
+                erts_bin_release((Binary *) bptr);
+                break;
+            }
+        default:
+            ASSERT(is_external_header(oh->thing_word));
+            erts_deref_node_entry(((ExternalThing*)oh)->node);
+        }
+        oh = oh->next;
     }
     erts_free(ERTS_ALC_T_LITERAL, literal_area);
 }
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 8cb977a7f3..6a87136463 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1234,8 +1234,8 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
     p->old_htop = old_htop;
 
     /*
-     * Prepare to sweep binaries. Since all MSOs on the new heap
-     * must be come before MSOs on the old heap, find the end of
+     * Prepare to sweep off-heap objects. Since all MSOs on the new
+     * heap must be come before MSOs on the old heap, find the end of
      * current MSO list and use that as a starting point.
      */
 
@@ -1247,25 +1247,50 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
     }
 
     /*
-     * Sweep through all binaries in the temporary literal area.
+     * Sweep through all off-heap objects in the temporary literal area.
      */
 
     while (oh) {
 	if (IS_MOVED_BOXED(oh->thing_word)) {
-	    Binary* bptr;
 	    struct erl_off_heap_header* ptr;
 
-	    ptr = (struct erl_off_heap_header*) boxed_val(oh->thing_word);
-	    ASSERT(thing_subtag(ptr->thing_word) == REFC_BINARY_SUBTAG);
-	    bptr = ((ProcBin*)ptr)->val;
-
-	    /*
-	     * This binary has been copied to the heap.
+            /*
+	     * This off-heap object has been copied to the heap.
 	     * We must increment its reference count and
 	     * link it into the MSO list for the process.
 	     */
 
-	    erts_refc_inc(&bptr->intern.refc, 1);
+	    ptr = (struct erl_off_heap_header*) boxed_val(oh->thing_word);
+            switch (thing_subtag(ptr->thing_word)) {
+            case REFC_BINARY_SUBTAG:
+                {
+                    Binary* bptr = ((ProcBin*)ptr)->val;
+                    erts_refc_inc(&bptr->intern.refc, 1);
+                    break;
+                }
+            case FUN_SUBTAG:
+                {
+                    ErlFunEntry* fe = ((ErlFunThing*)ptr)->fe;
+                    erts_refc_inc(&fe->refc, 1);
+                    break;
+                }
+            case REF_SUBTAG:
+                {
+                    ErtsMagicBinary *bptr;
+                    ASSERT(is_magic_ref_thing(ptr));
+                    bptr = ((ErtsMRefThing *) ptr)->mb;
+                    erts_refc_inc(&bptr->intern.refc, 1);
+                    break;
+                }
+            default:
+                {
+                    ExternalThing *etp;
+                    ASSERT(is_external_header(ptr->thing_word));
+                    etp = (ExternalThing *) ptr;
+                    erts_smp_refc_inc(&etp->node->refc, 1);
+                    break;
+                }
+            }
 	    *prev = ptr;
 	    prev = &ptr->next;
 	}
diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index 77321aa50f..dca600bc7b 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -25,6 +25,7 @@
          multi_proc_purge/1, t_check_old_code/1,
          external_fun/1,get_chunk/1,module_md5/1,
          constant_pools/1,constant_refc_binaries/1,
+         fake_literals/1,
          false_dependency/1,coverage/1,fun_confusion/1,
          t_copy_literals/1, t_copy_literals_frags/1]).
 
@@ -38,7 +39,8 @@ all() ->
      call_purged_fun_code_reload, call_purged_fun_code_there,
      multi_proc_purge, t_check_old_code, external_fun, get_chunk,
      module_md5,
-     constant_pools, constant_refc_binaries, false_dependency,
+     constant_pools, constant_refc_binaries, fake_literals,
+     false_dependency,
      coverage, fun_confusion, t_copy_literals, t_copy_literals_frags].
 
 init_per_suite(Config) ->
@@ -554,6 +556,62 @@ wait_for_memory_deallocations() ->
             wait_for_memory_deallocations()
     end.
 
+fake_literals(_Config) ->
+    Mod = fake__literals__module,
+    try
+        do_fake_literals(Mod)
+    after
+        _ = code:purge(Mod),
+        _ = code:delete(Mod),
+        _ = code:purge(Mod),
+        _ = code:delete(Mod)
+    end,
+    ok.
+
+do_fake_literals(Mod) ->
+    Tid = ets:new(test, []),
+    ExtTerms = get_external_terms(),
+    Term0 = {self(),make_ref(),Tid,fun() -> ok end,ExtTerms},
+    Terms = [begin
+                 make_literal_module(Mod, Term0),
+                 Mod:term()
+             end || _ <- lists:seq(1, 10)],
+    verify_lit_terms(Terms, Term0),
+    true = ets:delete(Tid),
+    ok.
+
+make_literal_module(Mod, Term) ->
+    Exp = [{term,0}],
+    Attr = [],
+    Fs = [{function,term,0,2,
+           [{label,1},
+            {line,[]},
+            {func_info,{atom,Mod},{atom,term},0},
+            {label,2},
+            {move,{literal,Term},{x,0}},
+            return]}],
+    Asm = {Mod,Exp,Attr,Fs,2},
+    {ok,Mod,Beam} = compile:forms(Asm, [from_asm,binary,report]),
+    code:load_binary(Mod, atom_to_list(Mod), Beam).
+
+verify_lit_terms([H|T], Term) ->
+    case H =:= Term of
+        true ->
+            verify_lit_terms(T, Term);
+        false ->
+            error({bad_term,H})
+    end;
+verify_lit_terms([], _) ->
+    ok.
+
+get_external_terms() ->
+    {ok,Node} =	test_server:start_node(?FUNCTION_NAME, slave, []),
+    Ref = rpc:call(Node, erlang, make_ref, []),
+    Ports = rpc:call(Node, erlang, ports, []),
+    Pid = rpc:call(Node, erlang, self, []),
+    _ = test_server:stop_node(Node),
+    {Ref,hd(Ports),Pid}.
+
 %% OTP-7559: c_p->cp could contain garbage and create a false dependency
 %% to a module in a process. (Thanks to Richard Carlsson.)
 false_dependency(Config) when is_list(Config) ->
-- 
cgit v1.2.3


From 5833bc53f9b785146a58d17fc171a5f049ac1cb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 27 Nov 2017 14:45:02 +0100
Subject: erts: Fix bad merge of PR #1644

---
 erts/emulator/beam/beam_load.c | 2 +-
 erts/emulator/beam/erl_gc.c    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 82a8ddd78c..e43f7fda2c 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -6040,7 +6040,7 @@ erts_release_literal_area(ErtsLiteralArea* literal_area)
         case FUN_SUBTAG:
             {
                 ErlFunEntry* fe = ((ErlFunThing*)oh)->fe;
-                if (erts_smp_refc_dectest(&fe->refc, 0) == 0) {
+                if (erts_refc_dectest(&fe->refc, 0) == 0) {
                     erts_erase_fun_entry(fe);
                 }
                 break;
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index a75856bfc9..0da4468f9c 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1263,7 +1263,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
                     ExternalThing *etp;
                     ASSERT(is_external_header(ptr->thing_word));
                     etp = (ExternalThing *) ptr;
-                    erts_smp_refc_inc(&etp->node->refc, 1);
+                    erts_refc_inc(&etp->node->refc, 1);
                     break;
                 }
             }
-- 
cgit v1.2.3


From 67fd015394185302f769378c2c5e47bddbdc22ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 29 Nov 2017 15:13:05 +0100
Subject: Stop trying to maximize the use of x(0)

X register 0 used to be mapped to a hardware register, and therefore
faster than the other registers. Because of that, the compiler
tried to use x(0) as much as possible as a temporary register.

That was changed a few releases ago. X register 0 is now placed
in the array of all X registers and has no special speed
advantage compared to the other registers.

Remove the code in the compiler that attempts to use x(0) as
much as possible. As a result, the following type of instruction
will be much less frequent:

 {put_list,Src,{x,0},{x,0}}

Instead, the following type of instruction will be more frequent:

 {put_list,Src,{x,X},{x,X}}

(Where X is an arbitrary X register.)

Update the runtime system to specialize that kind of put_list
instruction.
---
 erts/emulator/beam/instrs.tab | 7 +++++++
 erts/emulator/beam/ops.tab    | 7 +++++--
 2 files changed, 12 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index c17d1a8f69..d934abb217 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -542,6 +542,13 @@ put_list(Hd, Tl, Dst) {
     HTOP += 2;
 }
 
+update_list(Hd, Dst) {
+    HTOP[0] = $Hd;
+    HTOP[1] = $Dst;
+    $Dst = make_list(HTOP);
+    HTOP += 2;
+}
+
 i_put_tuple := i_put_tuple.make.fill;
 
 i_put_tuple.make(Dst) {
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 3df91056cb..c30af029ce 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -503,6 +503,10 @@ i_put_tuple xy I
 #
 put_list Const=c n Dst => move Const x | put_list x n Dst
 
+put_list Src Dst=x Dst => update_list Src Dst
+
+update_list xyc x
+
 put_list x n x
 put_list y n x
 put_list x x x
@@ -525,8 +529,6 @@ put_list c y x
 
 # The following put_list instructions using x(0) are frequently used.
 
-put_list y r r
-put_list x r r
 put_list r n r
 put_list r n x
 put_list r x x
@@ -537,6 +539,7 @@ put_list x x r
 put_list s s d
 %hot
 
+
 #
 # Some more only used by the emulator
 #
-- 
cgit v1.2.3


From ef2e55c707a6c3d1f54f2acb9c8bfbe3fb7a8659 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 14 Nov 2017 09:05:31 +0100
Subject: Add syntax in try/catch to retrieve the stacktrace directly

This commit adds a new syntax for retrieving the stacktrace
without calling erlang:get_stacktrace/0. That allow us to
deprecate erlang:get_stacktrace/0 and ultimately remove it.

The problem with erlang:get_stacktrace/0 is that it can keep huge
terms in a process for an indefinite time after an exception. The
stacktrace can be huge after a 'function_clause' exception or a failed
call to a BIF or operator, because the arguments for the call will be
included in the stacktrace. For example:

1> catch abs(lists:seq(1, 1000)).
{'EXIT',{badarg,[{erlang,abs,
                         [[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20|...]],
                         []},
                 {erl_eval,do_apply,6,[{file,"erl_eval.erl"},{line,674}]},
                 {erl_eval,expr,5,[{file,"erl_eval.erl"},{line,431}]},
                 {shell,exprs,7,[{file,"shell.erl"},{line,687}]},
                 {shell,eval_exprs,7,[{file,"shell.erl"},{line,642}]},
                 {shell,eval_loop,3,[{file,"shell.erl"},{line,627}]}]}}
2> erlang:get_stacktrace().
[{erlang,abs,
         [[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,
           23,24|...]],
         []},
 {erl_eval,do_apply,6,[{file,"erl_eval.erl"},{line,674}]},
 {erl_eval,expr,5,[{file,"erl_eval.erl"},{line,431}]},
 {shell,exprs,7,[{file,"shell.erl"},{line,687}]},
 {shell,eval_exprs,7,[{file,"shell.erl"},{line,642}]},
 {shell,eval_loop,3,[{file,"shell.erl"},{line,627}]}]
3>

We can extend the syntax for clauses in try/catch to optionally bind
the stacktrace to a variable.

Here is an example using the current syntax:

    try
      Expr
    catch C:E ->
      Stk = erlang:get_stacktrace(),
      .
      .
      .

In the new syntax, it would look like:

    try
      Expr
    catch
      C:E:Stk ->
       .
       .
       .

Only a variable (not a pattern) is allowed in the stacktrace position,
to discourage matching of the stacktrace. (Matching would also be
expensive, because the raw format of the stacktrace would have to be
converted to the cooked form before matching.)

Note that:

    try
      Expr
    catch E ->
      .
      .
      .

is a shorthand for:

    try
      Expr
    catch throw:E ->
      .
      .
      .

If the stacktrace is to be retrieved for a throw, the 'throw:'
prefix must be explicitly included:

    try
      Expr
    catch throw:E:Stk ->
      .
      .
      .
---
 erts/emulator/beam/instrs.tab        | 5 +++++
 erts/emulator/beam/ops.tab           | 6 ++++++
 erts/emulator/hipe/hipe_bif2.c       | 5 +++++
 erts/emulator/hipe/hipe_bif2.tab     | 1 +
 erts/emulator/hipe/hipe_bif_list.m4  | 1 +
 erts/emulator/hipe/hipe_native_bif.h | 3 +++
 erts/emulator/hipe/hipe_primops.h    | 1 +
 7 files changed, 22 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index c17d1a8f69..f4acf6167d 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -924,3 +924,8 @@ i_raise() {
     //| -no_next
 }
 
+build_stacktrace() {
+    SWAPOUT;
+    x(0) = build_stacktrace(c_p, x(0));
+    SWAPIN;
+}
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 3df91056cb..ba4ca67e5e 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1575,3 +1575,9 @@ i_recv_mark
 recv_set Fail | label Lbl | loop_rec Lf Reg => \
    i_recv_set | label Lbl | loop_rec Lf Reg
 i_recv_set
+
+#
+# OTP 21.
+#
+
+build_stacktrace
diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c
index 9ebbb22846..df377b2153 100644
--- a/erts/emulator/hipe/hipe_bif2.c
+++ b/erts/emulator/hipe/hipe_bif2.c
@@ -190,3 +190,8 @@ BIF_RETTYPE hipe_bifs_llvm_fix_pinned_regs_0(BIF_ALIST_0)
 {
     BIF_RET(am_ok);
 }
+
+BIF_RETTYPE hipe_bifs_build_stacktrace_1(BIF_ALIST_1)
+{
+    BIF_RET(build_stacktrace(BIF_P, BIF_ARG_1));
+}
diff --git a/erts/emulator/hipe/hipe_bif2.tab b/erts/emulator/hipe/hipe_bif2.tab
index bbcb577be0..c4da44606a 100644
--- a/erts/emulator/hipe/hipe_bif2.tab
+++ b/erts/emulator/hipe/hipe_bif2.tab
@@ -32,3 +32,4 @@ bif hipe_bifs:modeswitch_debug_on/0
 bif hipe_bifs:modeswitch_debug_off/0
 bif hipe_bifs:debug_native_called/2
 bif hipe_bifs:llvm_fix_pinned_regs/0
+bif hipe_bifs:build_stacktrace/1
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index 625d8486fd..4f0a655508 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -186,6 +186,7 @@ gc_bif_interface_1(nbif_erase_1, erase_1)
 gc_bif_interface_1(nbif_erts_internal_garbage_collect_1, erts_internal_garbage_collect_1)
 gc_nofail_primop_interface_1(nbif_gc_1, hipe_gc)
 gc_bif_interface_2(nbif_put_2, put_2)
+gc_bif_interface_2(nbif_hipe_bifs_build_stacktrace, hipe_bifs_build_stacktrace_1)
 
 /*
  * Debug BIFs that need read access to the full state.
diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h
index d5081b8438..6dffec1d8d 100644
--- a/erts/emulator/hipe/hipe_native_bif.h
+++ b/erts/emulator/hipe/hipe_native_bif.h
@@ -104,6 +104,9 @@ void hipe_emulate_fpe(Process*);
 AEXTERN(void,nbif_emasculate_binary,(Eterm));
 void hipe_emasculate_binary(Eterm);
 
+AEXTERN(BIF_RETTYPE,nbif_hipe_bifs_build_stacktrace,(Process*,Eterm));
+BIF_RETTYPE hipe_bifs_build_stacktrace_1(BIF_ALIST_1);
+
 /*
  * Stuff that is different in SMP and non-SMP.
  */
diff --git a/erts/emulator/hipe/hipe_primops.h b/erts/emulator/hipe/hipe_primops.h
index a6abd3e011..d6fd10bdff 100644
--- a/erts/emulator/hipe/hipe_primops.h
+++ b/erts/emulator/hipe/hipe_primops.h
@@ -83,6 +83,7 @@ PRIMOP_LIST(am_emulate_fpe, &nbif_emulate_fpe)
 #endif
 PRIMOP_LIST(am_emasculate_binary, &nbif_emasculate_binary)
 PRIMOP_LIST(am_debug_native_called, &nbif_hipe_bifs_debug_native_called)
+PRIMOP_LIST(am_build_stacktrace, &nbif_hipe_bifs_build_stacktrace)
 
 #if defined(__sparc__)
 #include "hipe_sparc_primops.h"
-- 
cgit v1.2.3


From 38e4dfbd281cae0fb58211b63861b037ad121231 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 16 Nov 2017 05:27:05 +0100
Subject: Use the new syntax in more test suites

---
 erts/emulator/test/binary_SUITE.erl     |  4 +-
 erts/emulator/test/call_trace_SUITE.erl |  4 +-
 erts/emulator/test/dirty_bif_SUITE.erl  | 65 +++++++++++++--------------------
 erts/emulator/test/dirty_nif_SUITE.erl  | 14 +++----
 erts/emulator/test/driver_SUITE.erl     |  4 +-
 erts/emulator/test/exception_SUITE.erl  |  4 +-
 erts/emulator/test/nif_SUITE.erl        |  9 ++---
 7 files changed, 44 insertions(+), 60 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index c71267a6d1..4bc1838139 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -1472,13 +1472,13 @@ error_after_yield(Type, M, F, AN, AFun, TrapFunc) ->
 					   apply(M, F, A),
 					   exit({unexpected_success, {M, F, A}})
 				       catch
-					   error:Type ->
+					   error:Type:Stk ->
 					       erlang:trace(self(),false,[running,{tracer,Tracer}]),
 					       %% We threw the exception from the native
 					       %% function we trapped to, but we want
 					       %% the BIF that originally was called
 					       %% to appear in the stack trace.
-					       [{M, F, A, _} | _] = erlang:get_stacktrace()
+					       [{M, F, A, _} | _] = Stk
 				       end
 			       end),
     receive
diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl
index 1251d644ae..d19f7f81ad 100644
--- a/erts/emulator/test/call_trace_SUITE.erl
+++ b/erts/emulator/test/call_trace_SUITE.erl
@@ -1116,8 +1116,8 @@ get_deep_4_loc(Arg) ->
         deep_4(Arg),
         ct:fail(should_not_return_to_here)
     catch
-        _:_ ->
-            [{?MODULE,deep_4,1,Loc0}|_] = erlang:get_stacktrace(),
+        _:_:Stk ->
+            [{?MODULE,deep_4,1,Loc0}|_] = Stk,
             Loc0
     end.
 
diff --git a/erts/emulator/test/dirty_bif_SUITE.erl b/erts/emulator/test/dirty_bif_SUITE.erl
index 981ec4d48d..fcdfc9220e 100644
--- a/erts/emulator/test/dirty_bif_SUITE.erl
+++ b/erts/emulator/test/dirty_bif_SUITE.erl
@@ -108,90 +108,80 @@ dirty_bif_exception(Config) when is_list(Config) ->
 			      erts_debug:dirty_cpu(error, Error),
 			      ct:fail(expected_exception)
 			  catch
-			      error:ErrorType ->
-				  [{erts_debug,dirty_cpu,[error, Error],_}|_]
-				      = erlang:get_stacktrace(),
+			      error:ErrorType:Stk1 ->
+				  [{erts_debug,dirty_cpu,[error, Error],_}|_] = Stk1,
 				  ok
 			  end,
 			  try
 			      apply(erts_debug,dirty_cpu,[error, Error]),
 			      ct:fail(expected_exception)
 			  catch
-			      error:ErrorType ->
-				  [{erts_debug,dirty_cpu,[error, Error],_}|_]
-				      = erlang:get_stacktrace(),
+			      error:ErrorType:Stk2 ->
+				  [{erts_debug,dirty_cpu,[error, Error],_}|_] = Stk2,
 				  ok
 			  end,
 			  try
 			      erts_debug:dirty_io(error, Error),
 			      ct:fail(expected_exception)
 			  catch
-			      error:ErrorType ->
-				  [{erts_debug,dirty_io,[error, Error],_}|_]
-				      = erlang:get_stacktrace(),
+			      error:ErrorType:Stk3 ->
+				  [{erts_debug,dirty_io,[error, Error],_}|_] = Stk3,
 				  ok
 			  end,
 			  try
 			      apply(erts_debug,dirty_io,[error, Error]),
 			      ct:fail(expected_exception)
 			  catch
-			      error:ErrorType ->
-				  [{erts_debug,dirty_io,[error, Error],_}|_]
-				      = erlang:get_stacktrace(),
+			      error:ErrorType:Stk4 ->
+				  [{erts_debug,dirty_io,[error, Error],_}|_] = Stk4,
 				  ok
 			  end,
 			  try
 			      erts_debug:dirty(normal, error, Error),
 			      ct:fail(expected_exception)
 			  catch
-			      error:ErrorType ->
-				  [{erts_debug,dirty,[normal, error, Error],_}|_]
-				      = erlang:get_stacktrace(),
+			      error:ErrorType:Stk5 ->
+				  [{erts_debug,dirty,[normal, error, Error],_}|_] = Stk5,
 				  ok
 			  end,
 			  try
 			      apply(erts_debug,dirty,[normal, error, Error]),
 			      ct:fail(expected_exception)
 			  catch
-			      error:ErrorType ->
-				  [{erts_debug,dirty,[normal, error, Error],_}|_]
-				      = erlang:get_stacktrace(),
+			      error:ErrorType:Stk6 ->
+				  [{erts_debug,dirty,[normal, error, Error],_}|_] = Stk6,
 				  ok
 			  end,
 			  try
 			      erts_debug:dirty(dirty_cpu, error, Error),
 			      ct:fail(expected_exception)
 			  catch
-			      error:ErrorType ->
-				  [{erts_debug,dirty,[dirty_cpu, error, Error],_}|_]
-				      = erlang:get_stacktrace(),
+			      error:ErrorType:Stk7 ->
+				  [{erts_debug,dirty,[dirty_cpu, error, Error],_}|_] = Stk7,
 				  ok
 			  end,
 			  try
 			      apply(erts_debug,dirty,[dirty_cpu, error, Error]),
 			      ct:fail(expected_exception)
 			  catch
-			      error:ErrorType ->
-				  [{erts_debug,dirty,[dirty_cpu, error, Error],_}|_]
-				      = erlang:get_stacktrace(),
+			      error:ErrorType:Stk8 ->
+				  [{erts_debug,dirty,[dirty_cpu, error, Error],_}|_] = Stk8,
 				  ok
 			  end,
 			  try
 			      erts_debug:dirty(dirty_io, error, Error),
 			      ct:fail(expected_exception)
 			  catch
-			      error:ErrorType ->
-				  [{erts_debug,dirty,[dirty_io, error, Error],_}|_]
-				      = erlang:get_stacktrace(),
+			      error:ErrorType:Stk9 ->
+				  [{erts_debug,dirty,[dirty_io, error, Error],_}|_] = Stk9,
 				  ok
 			  end,
 			  try
 			      apply(erts_debug,dirty,[dirty_io, error, Error]),
 			      ct:fail(expected_exception)
 			  catch
-			      error:ErrorType ->
-				  [{erts_debug,dirty,[dirty_io, error, Error],_}|_]
-				      = erlang:get_stacktrace(),
+			      error:ErrorType:Stk10 ->
+				  [{erts_debug,dirty,[dirty_io, error, Error],_}|_] = Stk10,
 				  ok
 			  end
 		  end,
@@ -204,25 +194,22 @@ dirty_bif_multischedule_exception(Config) when is_list(Config) ->
     try
 	erts_debug:dirty_cpu(reschedule,1001)
     catch
-	error:badarg ->
-	    [{erts_debug,dirty_cpu,[reschedule, 1001],_}|_]
-		= erlang:get_stacktrace(),
+	error:badarg:Stk1 ->
+	    [{erts_debug,dirty_cpu,[reschedule, 1001],_}|_] = Stk1,
 	    ok
     end,
     try
 	erts_debug:dirty_io(reschedule,1001)
     catch
-	error:badarg ->
-	    [{erts_debug,dirty_io,[reschedule, 1001],_}|_]
-		= erlang:get_stacktrace(),
+	error:badarg:Stk2 ->
+	    [{erts_debug,dirty_io,[reschedule, 1001],_}|_] = Stk2,
 	    ok
     end,
     try
 	erts_debug:dirty(normal,reschedule,1001)
     catch
-	error:badarg ->
-	    [{erts_debug,dirty,[normal,reschedule,1001],_}|_]
-		= erlang:get_stacktrace(),
+	error:badarg:Stk3 ->
+	    [{erts_debug,dirty,[normal,reschedule,1001],_}|_] = Stk3,
 	    ok
     end.
 
diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl
index 13806fd5c4..aa01435758 100644
--- a/erts/emulator/test/dirty_nif_SUITE.erl
+++ b/erts/emulator/test/dirty_nif_SUITE.erl
@@ -109,9 +109,8 @@ dirty_nif_exception(Config) when is_list(Config) ->
 	call_dirty_nif_exception(1),
 	ct:fail(expected_badarg)
     catch
-	error:badarg ->
-	    [{?MODULE,call_dirty_nif_exception,[1],_}|_] =
-		erlang:get_stacktrace(),
+	error:badarg:Stk1 ->
+	    [{?MODULE,call_dirty_nif_exception,[1],_}|_] = Stk1,
 	    ok
     end,
     try
@@ -121,9 +120,8 @@ dirty_nif_exception(Config) when is_list(Config) ->
 	call_dirty_nif_exception(0),
 	ct:fail(expected_badarg)
     catch
-	error:badarg ->
-	    [{?MODULE,call_dirty_nif_exception,[0],_}|_] =
-		erlang:get_stacktrace(),
+	error:badarg:Stk2 ->
+	    [{?MODULE,call_dirty_nif_exception,[0],_}|_] = Stk2,
 	    ok
     end,
     %% this checks that a dirty NIF can raise various terms as
@@ -138,8 +136,8 @@ nif_raise_exceptions(NifFunc) ->
                             erlang:apply(?MODULE,NifFunc,[Term]),
                             ct:fail({expected,Term})
                         catch
-                            error:Term ->
-                                [{?MODULE,NifFunc,[Term],_}|_] = erlang:get_stacktrace(),
+                            error:Term:Stk ->
+                                [{?MODULE,NifFunc,[Term],_}|_] = Stk,
                                 ok
                         end
                 end, ok, ExcTerms).
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 475e03087a..6446d5f6d2 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -2614,8 +2614,8 @@ rpc(Config, Fun) ->
                                 Result
                                     = try Fun() of
                                           Res -> Res
-                                      catch E:R ->
-                                              {'EXIT',E,R,erlang:get_stacktrace()}
+                                      catch E:R:Stk ->
+                                              {'EXIT',E,R,Stk}
                                       end,
                                 Self ! {Ref, Result}
                         end),
diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl
index be9b63d534..da0292f385 100644
--- a/erts/emulator/test/exception_SUITE.erl
+++ b/erts/emulator/test/exception_SUITE.erl
@@ -395,12 +395,12 @@ raise(Conf) when is_list(Conf) ->
     try 
         try foo({'div',{1,0}}) 
         catch
-            error:badarith ->
+            error:badarith:A0 ->
                 put(raise, A0 = erlang:get_stacktrace()),
                 erlang:raise(error, badarith, A0)
         end
     catch
-        error:badarith ->
+        error:badarith:A1 ->
             A1 = erlang:get_stacktrace(),
             A1 = get(raise)
     end,
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index bec2291867..b4b7953ec8 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -2257,9 +2257,8 @@ nif_schedule(Config) when is_list(Config) ->
     {B,A} = call_nif_schedule(A, B),
     ok = try call_nif_schedule(1, 2)
 	 catch
-	     error:badarg ->
-		 [{?MODULE,call_nif_schedule,[1,2],_}|_] =
-		     erlang:get_stacktrace(),
+	     error:badarg:Stk ->
+		 [{?MODULE,call_nif_schedule,[1,2],_}|_] = Stk,
 		 ok
 	 end,
     ok.
@@ -2429,8 +2428,8 @@ nif_raise_exceptions(NifFunc) ->
                             erlang:apply(?MODULE,NifFunc,[Term]),
                             ct:fail({expected,Term})
                         catch
-                            error:Term ->
-                                [{?MODULE,NifFunc,[Term],_}|_] = erlang:get_stacktrace(),
+                            error:Term:Stk ->
+                                [{?MODULE,NifFunc,[Term],_}|_] = Stk,
                                 ok
                         end
                 end, ok, ExcTerms).
-- 
cgit v1.2.3


From 5036bf7d5006a6f1a4294b4a3b1f4120d39113ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 7 Nov 2017 09:31:24 +0100
Subject: Add enif_ioq_peek_head

This introduces a way to retrieve erlang terms from NIF IO queues
without having to resort to copying.

OTP-14797
---
 erts/emulator/beam/erl_nif.c                  | 49 +++++++++++++++++++++++++++
 erts/emulator/beam/erl_nif.h                  |  3 +-
 erts/emulator/beam/erl_nif_api_funcs.h        |  2 ++
 erts/emulator/test/nif_SUITE.erl              | 23 +++++++++++++
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c |  9 +++++
 5 files changed, 85 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index dbcc894ac9..abb490cf9c 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3605,6 +3605,55 @@ int enif_ioq_deq(ErlNifIOQueue *q, size_t elems, size_t *size)
     return 1;
 }
 
+int enif_ioq_peek_head(ErlNifEnv *env, ErlNifIOQueue *q, size_t *size, ERL_NIF_TERM *bin_term) {
+    SysIOVec *iov_entry;
+    Binary *ref_bin;
+
+    if (q->size == 0) {
+        return 0;
+    }
+
+    ASSERT(q->b_head != q->b_tail && q->v_head != q->v_tail);
+
+    ref_bin = &q->b_head[0]->nif;
+    iov_entry = &q->v_head[0];
+
+    if (size != NULL) {
+        *size = iov_entry->iov_len;
+    }
+
+    if (iov_entry->iov_len > ERL_ONHEAP_BIN_LIMIT) {
+        ProcBin *pb = (ProcBin*)alloc_heap(env, PROC_BIN_SIZE);
+
+        pb->thing_word = HEADER_PROC_BIN;
+        pb->next = MSO(env->proc).first;
+        pb->val = ref_bin;
+        pb->flags = 0;
+
+        ASSERT((byte*)iov_entry->iov_base >= (byte*)ref_bin->orig_bytes);
+        ASSERT(iov_entry->iov_len <= ref_bin->orig_size);
+
+        pb->bytes = (byte*)iov_entry->iov_base;
+        pb->size = iov_entry->iov_len;
+
+        MSO(env->proc).first = (struct erl_off_heap_header*) pb;
+        OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm));
+
+        erts_refc_inc(&ref_bin->intern.refc, 2);
+        *bin_term = make_binary(pb);
+    } else {
+        ErlHeapBin* hb = (ErlHeapBin*)alloc_heap(env, heap_bin_size(iov_entry->iov_len));
+
+        hb->thing_word = header_heap_bin(iov_entry->iov_len);
+        hb->size = iov_entry->iov_len;
+
+        sys_memcpy(hb->data, iov_entry->iov_base, iov_entry->iov_len);
+        *bin_term = make_binary(hb);
+    }
+
+    return 1;
+}
+
 SysIOVec *enif_ioq_peek(ErlNifIOQueue *q, int *iovlen)
 {
     return erts_ioq_peekq(q, iovlen);
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 7fb447e4a8..053f7673c4 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -52,9 +52,10 @@
 ** 2.11: 19.0 enif_snprintf 
 ** 2.12: 20.0 add enif_select, enif_open_resource_type_x
 ** 2.13: 20.1 add enif_ioq
+** 2.14: 21.0 add enif_ioq_peek_head
 */
 #define ERL_NIF_MAJOR_VERSION 2
-#define ERL_NIF_MINOR_VERSION 13
+#define ERL_NIF_MINOR_VERSION 14
 
 /*
  * The emulator will refuse to load a nif-lib with a major version
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 9e573307d8..3750fd9b68 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -198,6 +198,7 @@ ERL_NIF_API_FUNC_DECL(SysIOVec*,enif_ioq_peek,(ErlNifIOQueue *q, int *iovlen));
 ERL_NIF_API_FUNC_DECL(int,enif_inspect_iovec,(ErlNifEnv *env, size_t max_length, ERL_NIF_TERM iovec_term, ERL_NIF_TERM *tail, ErlNifIOVec **iovec));
 ERL_NIF_API_FUNC_DECL(void,enif_free_iovec,(ErlNifIOVec *iov));
 
+ERL_NIF_API_FUNC_DECL(int,enif_ioq_peek_head,(ErlNifEnv *env, ErlNifIOQueue *q, size_t *size, ERL_NIF_TERM *head));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
@@ -373,6 +374,7 @@ ERL_NIF_API_FUNC_DECL(void,enif_free_iovec,(ErlNifIOVec *iov));
 #  define enif_ioq_peek ERL_NIF_API_FUNC_MACRO(enif_ioq_peek)
 #  define enif_inspect_iovec ERL_NIF_API_FUNC_MACRO(enif_inspect_iovec)
 #  define enif_free_iovec ERL_NIF_API_FUNC_MACRO(enif_free_iovec)
+#  define enif_ioq_peek_head ERL_NIF_API_FUNC_MACRO(enif_ioq_peek_head)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index bec2291867..2afeb7e4cf 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -3037,14 +3037,17 @@ nif_ioq(Config) ->
          {enqbraw,a},
          {enqbraw,a, 5},
          {peek,   a},
+         {peek_head,  a},
          {deq,    a, 42},
 
          %% Test enqv
          {enqv,   a, 2, 100},
+         {peek_head,  a},
          {deq,    a, all},
 
          %% This skips all elements but one in the iolist
          {enqv,   a, 5, iolist_size(nif_ioq_payload(5)) - 1},
+         {peek_head,  a},
          {peek,   a},
 
          %% Test to enqueue a bunch of refc binaries
@@ -3074,9 +3077,13 @@ nif_ioq(Config) ->
 
     Q = ioq_nif(create),
 
+    false = ioq_nif(peek_head, Q),
+
     {'EXIT', {badarg, _}} = (catch ioq_nif(deq, Q, 1)),
     {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, 1, 1234)),
 
+    false = ioq_nif(peek_head, Q),
+
     {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [atom_in_list], 0)),
     {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [make_ref()], 0)),
     {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [256], 0)),
@@ -3085,6 +3092,8 @@ nif_ioq(Config) ->
     {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [1 bsl 64], 0)),
     {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [{tuple}], 0)),
 
+    false = ioq_nif(peek_head, Q),
+
     {'EXIT', {badarg, _}} = (catch ioq_nif(inspect,  [atom_in_list], use_stack)),
     {'EXIT', {badarg, _}} = (catch ioq_nif(inspect,  [make_ref()], no_stack)),
     {'EXIT', {badarg, _}} = (catch ioq_nif(inspect,  [256], use_stack)),
@@ -3144,6 +3153,20 @@ nif_ioq_run([{peek, Name} = H|T], State) ->
     Data = ioq_nif(peek, IOQ, ioq_nif(size, IOQ)),
 
     true = iolist_to_binary(B) == iolist_to_binary(Data),
+    nif_ioq_run(T, State);
+nif_ioq_run([{peek_head, Name} = H|T], State) ->
+    #{ q := IOQ, b := B } = maps:get(Name, State),
+    RefData = iolist_to_binary(B),
+
+    ct:log("~p", [H]),
+
+    {true, QueueHead} = ioq_nif(peek_head, IOQ),
+    true = byte_size(QueueHead) > 0,
+
+    {RefHead, _Tail} = split_binary(RefData, byte_size(QueueHead)),
+
+    true = QueueHead =:= RefHead,
+
     nif_ioq_run(T, State);
 nif_ioq_run([{deq, Name, all}|T], State) ->
     #{ q := IOQ, b := B } = maps:get(Name, State),
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 79560a38aa..fa9ae1015c 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -3403,6 +3403,15 @@ static ERL_NIF_TERM ioq(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 		return enif_make_badarg(env);
 	    else
 		return enif_make_atom(env, "true");
+        } else if (enif_is_identical(argv[0], enif_make_atom(env, "peek_head"))) {
+            ERL_NIF_TERM head_term;
+
+            if(enif_ioq_peek_head(env, ioq->q, NULL, &head_term)) {
+                return enif_make_tuple2(env,
+                    enif_make_atom(env, "true"), head_term);
+            }
+
+            return enif_make_atom(env, "false");
         } else if (enif_is_identical(argv[0], enif_make_atom(env, "peek"))) {
             int iovlen, num, i, off = 0;
             SysIOVec *iov = enif_ioq_peek(ioq->q, &iovlen);
-- 
cgit v1.2.3


From 53a035457ea419d552c9a8ec5797a6f44d6eb381 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 31 Oct 2017 07:07:32 +0100
Subject: Change resource_monitors' lock order

If a NIF monitor fired while the resource was present in an ETS
table, the lock checker would erroneously report a lock order
violation.

This has no effect outside of debug builds.
---
 erts/emulator/beam/erl_lock_check.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 4cdef0200f..88716d478d 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -86,6 +86,10 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"hipe_mfait_lock",			NULL			},
 #endif
     {	"nodes_monitors",			NULL			},
+    {	"meta_name_tab",	         	"address"		},
+    {	"db_tab",				"address"		},
+    {	"db_tab_fix",				"address"		},
+    {	"db_hash_slot",				"address"		},
     {	"resource_monitors",			"address"	        },
     {   "driver_list",                          NULL                    },
     {	"proc_link",				"pid"			},
@@ -95,12 +99,8 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"dist_entry_links",			"address"		},
     {   "code_write_permission",                NULL                    },
     {	"purge_state",		      		NULL			},
-    {	"meta_name_tab",	         	"address"		},
-    {	"db_tab",				"address"		},
     {	"proc_status",				"pid"			},
     {	"proc_trace",				"pid"			},
-    {	"db_tab_fix",				"address"		},
-    {	"db_hash_slot",				"address"		},
     {	"node_table",				NULL			},
     {	"dist_table",				NULL			},
     {	"sys_tracers",				NULL			},
-- 
cgit v1.2.3


From 6cb62e44eba1db8d1917ebb0db84298e91582c4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Fri, 8 Sep 2017 15:01:01 +0200
Subject: Add a mutable binary buffer type (prim_buffer)

---
 erts/emulator/Makefile.in                   |   6 +-
 erts/emulator/nifs/common/prim_buffer_nif.c | 512 ++++++++++++++++++++++++++++
 2 files changed, 516 insertions(+), 2 deletions(-)
 create mode 100644 erts/emulator/nifs/common/prim_buffer_nif.c

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index fdaf253fe5..9a09a0f6fa 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -644,7 +644,8 @@ PRELOAD_BEAM =		$(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/erts_internal.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_code_checker.beam
+			$(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_code_checker.beam \
+			$(ERL_TOP)/erts/preloaded/ebin/prim_buffer.beam
 
 ifeq ($(TARGET),win32)
 # On windows the preloaded objects are in a resource object.
@@ -873,7 +874,8 @@ RUN_OBJS += \
 LTTNG_OBJS = $(OBJDIR)/erlang_lttng.o
 NIF_OBJS = \
 	   $(OBJDIR)/erl_tracer_nif.o \
-	   $(OBJDIR)/zlib_nif.o
+	   $(OBJDIR)/zlib_nif.o \
+	   $(OBJDIR)/prim_buffer_nif.o
 
 ifeq ($(TARGET),win32)
 DRV_OBJS = \
diff --git a/erts/emulator/nifs/common/prim_buffer_nif.c b/erts/emulator/nifs/common/prim_buffer_nif.c
new file mode 100644
index 0000000000..a8ef5fc355
--- /dev/null
+++ b/erts/emulator/nifs/common/prim_buffer_nif.c
@@ -0,0 +1,512 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson 2017. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#define STATIC_ERLANG_NIF 1
+
+#include "erl_nif.h"
+#include "config.h"
+#include "sys.h"
+
+#ifdef VALGRIND
+#  include <valgrind/memcheck.h>
+#endif
+
+#define ACCUMULATOR_SIZE (2 << 10)
+
+#define FIND_NIF_RESCHEDULE_SIZE (1 << 20)
+
+/* NIF interface declarations */
+static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info);
+static int upgrade(ErlNifEnv *env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info);
+static void unload(ErlNifEnv *env, void* priv_data);
+
+static ErlNifResourceType *rtype_buffer;
+
+static ERL_NIF_TERM am_ok;
+static ERL_NIF_TERM am_error;
+
+static ERL_NIF_TERM am_lock_order_violation;
+
+static ERL_NIF_TERM am_acquired;
+static ERL_NIF_TERM am_busy;
+
+static ERL_NIF_TERM am_continue;
+
+static ERL_NIF_TERM am_out_of_memory;
+static ERL_NIF_TERM am_not_found;
+
+typedef struct {
+#ifdef DEBUG
+    erts_atomic32_t concurrent_users;
+#endif
+
+    ErlNifBinary accumulator;
+    size_t accumulated_bytes;
+    int accumulator_present;
+
+    ErlNifIOQueue *queue;
+
+    erts_atomic32_t external_lock;
+} buffer_data_t;
+
+static ERL_NIF_TERM new_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+static ERL_NIF_TERM peek_head_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM skip_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM size_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+static ERL_NIF_TERM write_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM copying_read_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+static ERL_NIF_TERM find_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+static ERL_NIF_TERM trylock_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM unlock_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+static ErlNifFunc nif_funcs[] = {
+    {"new", 0, new_nif},
+    {"size", 1, size_nif},
+    {"peek_head", 1, peek_head_nif},
+    {"copying_read", 2, copying_read_nif},
+    {"write", 2, write_nif},
+    {"skip", 2, skip_nif},
+    {"find_byte_index", 2, find_nif},
+    {"try_lock", 1, trylock_nif},
+    {"unlock", 1, unlock_nif},
+};
+
+ERL_NIF_INIT(prim_buffer, nif_funcs, load, NULL, upgrade, unload)
+
+static void gc_buffer(ErlNifEnv *env, void* data);
+
+static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info)
+{
+    am_ok = enif_make_atom(env, "ok");
+    am_error = enif_make_atom(env, "error");
+
+    am_lock_order_violation = enif_make_atom(env, "lock_order_violation");
+    am_acquired = enif_make_atom(env, "acquired");
+    am_busy = enif_make_atom(env, "busy");
+
+    am_continue = enif_make_atom(env, "continue");
+
+    am_out_of_memory = enif_make_atom(env, "out_of_memory");
+    am_not_found = enif_make_atom(env, "not_found");
+
+    rtype_buffer = enif_open_resource_type(env, NULL, "gc_buffer", gc_buffer,
+        ERL_NIF_RT_CREATE, NULL);
+
+    *priv_data = NULL;
+
+    return 0;
+}
+
+static void unload(ErlNifEnv *env, void* priv_data)
+{
+
+}
+
+static int upgrade(ErlNifEnv *env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
+{
+    if(*old_priv_data != NULL) {
+        return -1; /* Don't know how to do that */
+    }
+
+    if(*priv_data != NULL) {
+        return -1; /* Don't know how to do that */
+    }
+
+    if(load(env, priv_data, load_info)) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static void gc_buffer(ErlNifEnv *env, void* data) {
+    buffer_data_t *buffer = (buffer_data_t*)data;
+
+    if(buffer->accumulator_present) {
+        enif_release_binary(&buffer->accumulator);
+    }
+
+    enif_ioq_destroy(buffer->queue);
+}
+
+static int get_buffer_data(ErlNifEnv *env, ERL_NIF_TERM opaque, buffer_data_t **buffer) {
+    return enif_get_resource(env, opaque, rtype_buffer, (void **)buffer);
+}
+
+/* Copies a number of bytes from the head of the iovec, skipping "vec_skip"
+ * vector elements followed by "byte_skip" bytes on the target vector. */
+static void copy_from_iovec(SysIOVec *iovec, int vec_len, int vec_skip,
+        size_t byte_skip, size_t size, char *data) {
+
+    size_t bytes_copied, skip_offset;
+    int vec_index;
+
+    skip_offset = byte_skip;
+    vec_index = vec_skip;
+    bytes_copied = 0;
+
+    while(bytes_copied < size) {
+        size_t block_size, copy_size;
+        char *block_start;
+
+        ASSERT(vec_index < vec_len);
+
+        block_start = (char*)iovec[vec_index].iov_base;
+        block_size = iovec[vec_index].iov_len;
+
+        copy_size = MIN(size - bytes_copied, block_size - skip_offset);
+        sys_memcpy(&data[bytes_copied], &block_start[skip_offset], copy_size);
+
+        bytes_copied += copy_size;
+        skip_offset = 0;
+
+        vec_index++;
+    }
+}
+
+/* Convenience function for copy_from_iovec over queues. */
+static void copy_from_queue(ErlNifIOQueue *queue, int queue_skip,
+    size_t byte_skip, size_t size, char *data) {
+
+    SysIOVec *queued_data;
+    int queue_length;
+
+    queued_data = enif_ioq_peek(queue, &queue_length);
+    ASSERT(queue_skip < queue_length);
+
+    copy_from_iovec(queued_data, queue_length, queue_skip, byte_skip, size, data);
+}
+
+static int enqueue_write_accumulator(buffer_data_t *buffer) {
+    ASSERT(!buffer->accumulator_present ^ (buffer->accumulated_bytes > 0));
+
+    if(buffer->accumulator_present && buffer->accumulated_bytes > 0) {
+        if(!enif_realloc_binary(&buffer->accumulator, buffer->accumulated_bytes)) {
+            return 0;
+        } else if(!enif_ioq_enq_binary(buffer->queue, &buffer->accumulator, 0)) {
+            return 0;
+        }
+
+        /* The queue owns the accumulator now. */
+        buffer->accumulator_present = 0;
+        buffer->accumulated_bytes = 0;
+    }
+
+    return 1;
+}
+
+static int combine_small_writes(buffer_data_t *buffer, ErlNifIOVec *iovec) {
+    ASSERT(!buffer->accumulator_present ^ (buffer->accumulated_bytes > 0));
+
+    if(buffer->accumulated_bytes + iovec->size >= ACCUMULATOR_SIZE) {
+        if(iovec->size >= (ACCUMULATOR_SIZE / 2)) {
+            return 0;
+        }
+
+        if(!enqueue_write_accumulator(buffer)) {
+            return 0;
+        }
+    }
+
+    if(!buffer->accumulator_present) {
+        if(!enif_alloc_binary(ACCUMULATOR_SIZE, &buffer->accumulator)) {
+            return 0;
+        }
+
+        buffer->accumulator_present = 1;
+    }
+
+    copy_from_iovec(iovec->iov, iovec->iovcnt, 0, 0, iovec->size,
+            (char*)&buffer->accumulator.data[buffer->accumulated_bytes]);
+    buffer->accumulated_bytes += iovec->size;
+
+    return 1;
+}
+
+/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */
+
+static ERL_NIF_TERM new_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    buffer_data_t *buffer;
+    ERL_NIF_TERM result;
+
+    buffer = (buffer_data_t*)enif_alloc_resource(rtype_buffer, sizeof(buffer_data_t));
+    buffer->queue = enif_ioq_create(ERL_NIF_IOQ_NORMAL);
+
+    if(buffer->queue != NULL) {
+#ifdef DEBUG
+        erts_atomic32_init_nob(&buffer->concurrent_users, 0);
+#endif
+        erts_atomic32_init_nob(&buffer->external_lock, 0);
+
+        buffer->accumulator_present = 0;
+        buffer->accumulated_bytes = 0;
+
+        result = enif_make_resource(env, buffer);
+    } else {
+        result = enif_raise_exception(env, am_out_of_memory);
+    }
+
+    enif_release_resource(buffer);
+
+    return result;
+}
+
+static ERL_NIF_TERM size_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    buffer_data_t *buffer;
+
+    size_t total_size;
+
+    if(argc != 1 || !get_buffer_data(env, argv[0], &buffer)) {
+        return enif_make_badarg(env);
+    }
+
+    ASSERT(erts_atomic32_inc_read_acqb(&buffer->concurrent_users) == 1);
+
+    total_size = enif_ioq_size(buffer->queue);
+
+    if(buffer->accumulator_present) {
+        total_size += buffer->accumulated_bytes;
+    } else {
+        ASSERT(buffer->accumulated_bytes == 0);
+    }
+
+    ASSERT(erts_atomic32_dec_read_relb(&buffer->concurrent_users) == 0);
+
+    return enif_make_uint64(env, total_size);
+}
+
+static ERL_NIF_TERM copying_read_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    buffer_data_t *buffer;
+
+    ERL_NIF_TERM result;
+    unsigned char *data;
+    Uint64 block_size;
+
+    if(argc != 2 || !get_buffer_data(env, argv[0], &buffer)
+                 || !enif_get_uint64(env, argv[1], &block_size)) {
+        return enif_make_badarg(env);
+    }
+
+    ASSERT(erts_atomic32_inc_read_acqb(&buffer->concurrent_users) == 1);
+
+    if(!enqueue_write_accumulator(buffer)) {
+        return enif_raise_exception(env, am_out_of_memory);
+    }
+
+    if(enif_ioq_size(buffer->queue) < block_size) {
+        return enif_make_badarg(env);
+    }
+
+    data = enif_make_new_binary(env, block_size, &result);
+
+    if(block_size > 0) {
+        copy_from_queue(buffer->queue, 0, 0, block_size, (char*)data);
+        enif_ioq_deq(buffer->queue, block_size, NULL);
+    }
+
+    ASSERT(erts_atomic32_dec_read_relb(&buffer->concurrent_users) == 0);
+
+    return result;
+}
+
+static ERL_NIF_TERM write_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    buffer_data_t *buffer;
+
+    ErlNifIOVec vec, *iovec = &vec;
+    ERL_NIF_TERM tail;
+
+    if(argc != 2 || !get_buffer_data(env, argv[0], &buffer)
+                 || !enif_inspect_iovec(env, 64, argv[1], &tail, &iovec)) {
+        return enif_make_badarg(env);
+    }
+
+    ASSERT(erts_atomic32_inc_read_acqb(&buffer->concurrent_users) == 1);
+
+    if(!combine_small_writes(buffer, iovec)) {
+        if(!enqueue_write_accumulator(buffer) || !enif_ioq_enqv(buffer->queue, iovec, 0)) {
+            return enif_raise_exception(env, am_out_of_memory);
+        }
+    }
+
+    ASSERT(erts_atomic32_dec_read_relb(&buffer->concurrent_users) == 0);
+
+    if(!enif_is_empty_list(env, tail)) {
+        const ERL_NIF_TERM new_argv[2] = {argv[0], tail};
+
+        return enif_schedule_nif(env, "write", 0, &write_nif, argc, new_argv);
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM peek_head_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    buffer_data_t *buffer;
+
+    ERL_NIF_TERM result;
+
+    if(argc != 1 || !get_buffer_data(env, argv[0], &buffer)) {
+        return enif_make_badarg(env);
+    }
+
+    ASSERT(erts_atomic32_inc_read_acqb(&buffer->concurrent_users) == 1);
+
+    if(!enqueue_write_accumulator(buffer)) {
+        return enif_raise_exception(env, am_out_of_memory);
+    }
+
+    if(!enif_ioq_peek_head(env, buffer->queue, NULL, &result)) {
+        return enif_make_badarg(env);
+    }
+
+    ASSERT(erts_atomic32_dec_read_relb(&buffer->concurrent_users) == 0);
+
+    return result;
+}
+
+static ERL_NIF_TERM skip_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    buffer_data_t *buffer;
+
+    Uint64 block_size;
+
+    if(argc != 2 || !get_buffer_data(env, argv[0], &buffer)
+                 || !enif_get_uint64(env, argv[1], &block_size)) {
+        return enif_make_badarg(env);
+    }
+
+    ASSERT(erts_atomic32_inc_read_acqb(&buffer->concurrent_users) == 1);
+
+    if(!enqueue_write_accumulator(buffer)) {
+        return enif_raise_exception(env, am_out_of_memory);
+    } else if(enif_ioq_size(buffer->queue) < block_size) {
+        return enif_make_badarg(env);
+    }
+
+    enif_ioq_deq(buffer->queue, block_size, NULL);
+
+    ASSERT(erts_atomic32_dec_read_relb(&buffer->concurrent_users) == 0);
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM find_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    buffer_data_t *buffer;
+
+    int queue_length, queue_index;
+    SysIOVec *queued_data;
+    size_t queue_size;
+
+    size_t search_offset;
+    int needle;
+
+    if(argc != 2 || !get_buffer_data(env, argv[0], &buffer)
+                 || !enif_get_int(env, argv[1], &needle)) {
+        return enif_make_badarg(env);
+    }
+
+    ASSERT(erts_atomic32_inc_read_acqb(&buffer->concurrent_users) == 1);
+
+    if(!enqueue_write_accumulator(buffer)) {
+        return enif_raise_exception(env, am_out_of_memory);
+    } else if(needle < 0 || needle > 255) {
+        return enif_make_badarg(env);
+    }
+
+    queued_data = enif_ioq_peek(buffer->queue, &queue_length);
+    queue_size = enif_ioq_size(buffer->queue);
+    queue_index = 0;
+
+    search_offset = 0;
+
+    if(queue_size > (FIND_NIF_RESCHEDULE_SIZE / 100)) {
+        if(enif_thread_type() == ERL_NIF_THR_NORMAL_SCHEDULER) {
+            int timeslice_percent;
+
+            if(queue_size >= FIND_NIF_RESCHEDULE_SIZE) {
+                ASSERT(erts_atomic32_dec_read_relb(&buffer->concurrent_users) == 0);
+
+                return enif_schedule_nif(env, "find",
+                    ERL_NIF_DIRTY_JOB_CPU_BOUND, &find_nif, argc, argv);
+            }
+
+            timeslice_percent = (queue_size * 100) / FIND_NIF_RESCHEDULE_SIZE;
+            enif_consume_timeslice(env, timeslice_percent);
+        }
+    }
+
+    while(queue_index < queue_length) {
+        char *needle_address;
+        char *block_start;
+        size_t block_size;
+
+        block_start = queued_data[queue_index].iov_base;
+        block_size = queued_data[queue_index].iov_len;
+
+        needle_address = memchr(block_start, needle, block_size);
+
+        if(needle_address != NULL) {
+            size_t result = search_offset + (needle_address - block_start);
+
+            ASSERT(erts_atomic32_dec_read_relb(&buffer->concurrent_users) == 0);
+
+            return enif_make_tuple2(env, am_ok, enif_make_uint64(env, result));
+        }
+
+        search_offset += block_size;
+        queue_index++;
+    }
+
+    ASSERT(erts_atomic32_dec_read_relb(&buffer->concurrent_users) == 0);
+
+    return am_not_found;
+}
+
+/* */
+
+static ERL_NIF_TERM trylock_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    buffer_data_t *buffer;
+
+    if(argc != 1 || !get_buffer_data(env, argv[0], &buffer)) {
+        return enif_make_badarg(env);
+    }
+
+    if(erts_atomic32_cmpxchg_acqb(&buffer->external_lock, 1, 0) == 0) {
+        return am_acquired;
+    }
+
+    return am_busy;
+}
+
+static ERL_NIF_TERM unlock_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    buffer_data_t *buffer;
+
+    if(argc != 1 || !get_buffer_data(env, argv[0], &buffer)) {
+        return enif_make_badarg(env);
+    }
+
+    if(erts_atomic32_cmpxchg_relb(&buffer->external_lock, 0, 1) == 0) {
+        return enif_raise_exception(env, am_lock_order_violation);
+    }
+
+    return am_ok;
+}
-- 
cgit v1.2.3


From ebbd26eeea4115c946d1254d94acd50f150b4455 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Fri, 3 Nov 2017 11:49:27 +0100
Subject: Reimplement efile_drv as a dirty NIF

This improves the latency of file operations as dirty schedulers
are a bit more eager to run jobs than async threads, and use a
single global queue rather than per-thread queues, eliminating the
risk of a job stalling behind a long-running job on the same thread
while other async threads sit idle.

There's no such thing as a free lunch though; the lowered latency
comes at the cost of increased busy-waiting which may have an
adverse effect on some applications. This behavior can be tweaked
with the +sbwt flag, but unfortunately it affects all types of
schedulers and not just dirty ones. We plan to add type-specific
flags at a later stage.

sendfile has been moved to inet_drv to lessen the effect of a nasty
race; the cooperation between inet_drv and efile has never been
airtight and the socket dying at the wrong time (Regardless of
reason) could result in fd aliasing. Moving it to the inet driver
makes it impossible to trigger this by closing the socket in the
middle of a sendfile operation, while still allowing it to be
aborted -- something that can't be done if it stays in the file
driver.

The race still occurs if the controlling process dies in the short
window between dispatching the sendfile operation and the dup(2)
call in the driver, but it's much less likely to happen now.

A proper fix is in the works.

--

Notable functional differences:

* The use_threads option for file:sendfile/5 no longer has any
  effect.

* The file-specific DTrace probes have been removed. The same
  effect can be achieved with normal tracing together with the
  nif__entry/nif__return probes to track scheduling.

--

OTP-14256
---
 erts/emulator/Makefile.in                 |   34 +-
 erts/emulator/beam/erl_lock_check.c       |    4 -
 erts/emulator/drivers/common/efile_drv.c  | 4295 -----------------------------
 erts/emulator/drivers/common/erl_efile.h  |  176 --
 erts/emulator/drivers/common/gzio.c       |  712 +----
 erts/emulator/drivers/common/gzio.h       |    8 -
 erts/emulator/drivers/common/inet_drv.c   |  382 ++-
 erts/emulator/drivers/unix/unix_efile.c   | 1102 --------
 erts/emulator/drivers/win32/win_efile.c   | 2058 --------------
 erts/emulator/nifs/common/prim_file_nif.c | 1237 +++++++++
 erts/emulator/nifs/common/prim_file_nif.h |  240 ++
 erts/emulator/nifs/unix/unix_prim_file.c  |  957 +++++++
 erts/emulator/nifs/win32/win_prim_file.c  | 1427 ++++++++++
 13 files changed, 4258 insertions(+), 8374 deletions(-)
 delete mode 100644 erts/emulator/drivers/common/efile_drv.c
 delete mode 100644 erts/emulator/drivers/common/erl_efile.h
 delete mode 100644 erts/emulator/drivers/unix/unix_efile.c
 delete mode 100644 erts/emulator/drivers/win32/win_efile.c
 create mode 100644 erts/emulator/nifs/common/prim_file_nif.c
 create mode 100644 erts/emulator/nifs/common/prim_file_nif.h
 create mode 100644 erts/emulator/nifs/unix/unix_prim_file.c
 create mode 100644 erts/emulator/nifs/win32/win_prim_file.c

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 9a09a0f6fa..15b65f1c71 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -634,6 +634,7 @@ GENERATE += $(TTF_DIR)/driver_tab.c
 PRELOAD_BEAM =		$(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/init.beam \
+			$(ERL_TOP)/erts/preloaded/ebin/prim_buffer.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/prim_eval.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \
@@ -644,8 +645,7 @@ PRELOAD_BEAM =		$(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/erts_internal.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_code_checker.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/prim_buffer.beam
+			$(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_code_checker.beam
 
 ifeq ($(TARGET),win32)
 # On windows the preloaded objects are in a resource object.
@@ -786,6 +786,9 @@ $(OBJDIR)/%.o: drivers/$(ERLANG_OSTYPE)/%.c
 $(OBJDIR)/%.o: nifs/common/%.c
 	$(V_CC) $(CFLAGS) -DLIBSCTP=$(LIBSCTP) $(INCLUDES) -Inifs/common -Inifs/$(ERLANG_OSTYPE) -c $< -o $@
 
+$(OBJDIR)/%.o: nifs/$(ERLANG_OSTYPE)/%.c
+	$(V_CC) $(CFLAGS) $(INCLUDES) -Inifs/common -Inifs/$(ERLANG_OSTYPE) -I../etc/$(ERLANG_OSTYPE) -c $< -o $@
+
 # ----------------------------------------------------------------------
 # Specials
 #
@@ -874,18 +877,17 @@ RUN_OBJS += \
 LTTNG_OBJS = $(OBJDIR)/erlang_lttng.o
 NIF_OBJS = \
 	   $(OBJDIR)/erl_tracer_nif.o \
-	   $(OBJDIR)/zlib_nif.o \
-	   $(OBJDIR)/prim_buffer_nif.o
+	   $(OBJDIR)/prim_buffer_nif.o \
+	   $(OBJDIR)/prim_file_nif.o \
+	   $(OBJDIR)/zlib_nif.o
 
 ifeq ($(TARGET),win32)
 DRV_OBJS = \
 	$(OBJDIR)/registry_drv.o \
-	$(OBJDIR)/efile_drv.o \
 	$(OBJDIR)/inet_drv.o \
 	$(OBJDIR)/ram_file_drv.o \
 	$(OBJDIR)/ttsl_drv.o
 OS_OBJS = \
-	$(OBJDIR)/win_efile.o \
 	$(OBJDIR)/win_con.o \
 	$(OBJDIR)/dll_sys.o \
 	$(OBJDIR)/driver_tab.o \
@@ -894,7 +896,8 @@ OS_OBJS = \
 	$(OBJDIR)/sys_time.o \
 	$(OBJDIR)/sys_interrupt.o \
 	$(OBJDIR)/sys_env.o \
-	$(OBJDIR)/dosmap.o
+	$(OBJDIR)/dosmap.o \
+	$(OBJDIR)/win_prim_file.o
 
 else
 OS_OBJS = \
@@ -902,14 +905,13 @@ OS_OBJS = \
 	$(OBJDIR)/sys_drivers.o \
 	$(OBJDIR)/sys_uds.o \
 	$(OBJDIR)/driver_tab.o \
-	$(OBJDIR)/unix_efile.o \
+	$(OBJDIR)/elib_memmove.o \
 	$(OBJDIR)/gzio.o \
-	$(OBJDIR)/elib_memmove.o
+	$(OBJDIR)/unix_prim_file.o
 
 	OS_OBJS += $(OBJDIR)/sys_float.o \
                    $(OBJDIR)/sys_time.o
 DRV_OBJS = \
-	$(OBJDIR)/efile_drv.o \
 	$(OBJDIR)/inet_drv.o \
 	$(OBJDIR)/ram_file_drv.o \
 	$(OBJDIR)/ttsl_drv.o
@@ -1137,6 +1139,7 @@ BEAM_SRC=$(wildcard beam/*.c)
 DRV_COMMON_SRC=$(wildcard drivers/common/*.c)
 DRV_OSTYPE_SRC=$(wildcard drivers/$(ERLANG_OSTYPE)/*.c)
 NIF_COMMON_SRC=$(wildcard nifs/common/*.c)
+NIF_OSTYPE_SRC=$(wildcard nifs/$(ERLANG_OSTYPE)/*.c)
 ALL_SYS_SRC=$(wildcard sys/$(ERLANG_OSTYPE)/*.c) $(wildcard sys/common/*.c)
 # We use $(shell ls) here instead of wildcard as $(wildcard ) resolved at
 # loadtime of the makefile and at that time these files are not generated yet.
@@ -1149,7 +1152,10 @@ ifeq ($(TARGET),win32)
 
 #DEP_CC=$(EMU_CC)
 DEP_CC=$(CC)
-DEP_FLAGS=-MM  $(subst -O2,,$(CFLAGS)) $(INCLUDES) -I../etc/win32 -Idrivers/common -Idrivers/$(ERLANG_OSTYPE)
+DEP_FLAGS=-MM  $(subst -O2,,$(CFLAGS)) $(INCLUDES) -I../etc/win32 \
+	-Idrivers/common -Idrivers/$(ERLANG_OSTYPE) \
+	-Inifs/common -Inifs/$(ERLANG_OSTYPE)
+
 # ifeq (@MIXED_CYGWIN_VC@,yes)
 # VC++ used for compiling. If __GNUC__ is defined we will include
 # other headers then when compiling which will result in faulty
@@ -1169,7 +1175,9 @@ MG_FLAG=-MG
 endif
 
 DEP_CC=$(CC)
-DEP_FLAGS=-MM $(MG_FLAG) $(CFLAGS) $(INCLUDES) -Inifs/common -Idrivers/common -Idrivers/$(ERLANG_OSTYPE)
+DEP_FLAGS=-MM $(MG_FLAG) $(CFLAGS) $(INCLUDES) \
+	-Idrivers/common -Idrivers/$(ERLANG_OSTYPE) \
+	-Inifs/common -Inifs/$(ERLANG_OSTYPE)
 SYS_SRC=$(ALL_SYS_SRC)
 endif
 
@@ -1200,6 +1208,8 @@ $(TTF_DIR)/depend.mk: $(TTF_DIR)/GENERATED $(PRELOAD_SRC)
 		| $(SED_DEPEND) >> $(TTF_DIR)/depend.mk
 	$(V_at)$(DEP_CC) $(DEP_FLAGS) $(NIF_COMMON_SRC) \
 		| $(SED_DEPEND) >> $(TTF_DIR)/depend.mk
+	$(V_at)$(DEP_CC) $(DEP_FLAGS) -I../etc/$(ERLANG_OSTYPE) $(NIF_OSTYPE_SRC) \
+		| $(SED_DEPEND) >> $(TTF_DIR)/depend.mk
 	$(V_at)$(DEP_CC) $(DEP_FLAGS) $(SYS_SRC) \
 		| $(SED_DEPEND) >> $(TTF_DIR)/depend.mk
 	$(V_at)$(DEP_CC) $(DEP_FLAGS) $(TARGET_SRC) \
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 88716d478d..64950fc252 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -109,7 +109,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"fun_tab",				NULL			},
     {	"environ",				NULL			},
     {	"release_literal_areas",		NULL			},
-    {	"efile_drv",				"address"		},
     {	"drv_ev_state_grow",			NULL,   		},
     {	"drv_ev_state",				"address"		},
     {	"safe_hash",				"address"		},
@@ -169,9 +168,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
 #ifdef DEBUG
     {   "save_ops_lock",                        NULL                    },
 #endif
-#endif
-#ifdef	USE_VM_PROBES
-    {   "efile_drv dtrace mutex",               NULL                    },
 #endif
     {	"mtrace_buf",				NULL			},
     {	"os_monotonic_time",			NULL			},
diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c
deleted file mode 100644
index 4e1d2f0d7f..0000000000
--- a/erts/emulator/drivers/common/efile_drv.c
+++ /dev/null
@@ -1,4295 +0,0 @@
-/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * %CopyrightEnd%
- */
-/*
- * Purpose: Provides file and directory operations.
- *
- * This file is generic, and does the work of decoding the commands
- * and encoding the responses.  System-specific functions are found in
- * the unix_efile.c and win_efile.c files.
- */
-
-/* Operations */
-
-#define FILE_OPEN		 1 /* Essential for startup */
-#define FILE_READ		 2
-#define FILE_LSEEK		 3
-#define FILE_WRITE		 4
-#define FILE_FSTAT		 5 /* Essential for startup */
-#define FILE_PWD                 6 /* Essential for startup */
-#define FILE_READDIR             7 /* Essential for startup */
-#define FILE_CHDIR               8
-#define FILE_FSYNC               9
-#define FILE_MKDIR              10
-#define FILE_DELETE             11
-#define FILE_RENAME             12
-#define FILE_RMDIR              13
-#define FILE_TRUNCATE           14
-#define FILE_READ_FILE          15 /* Essential for startup */
-#define FILE_WRITE_INFO		16
-#define FILE_LSTAT            	19
-#define FILE_READLINK        	20
-#define FILE_LINK             	21
-#define FILE_SYMLINK          	22
-#define FILE_CLOSE		23
-#define FILE_PWRITEV		24
-#define FILE_PREADV		25
-#define FILE_SETOPT		26
-#define FILE_IPREAD             27
-#define FILE_ALTNAME            28
-#define FILE_READ_LINE          29
-#define FILE_FDATASYNC          30
-#define FILE_FADVISE            31
-#define FILE_SENDFILE           32
-#define FILE_FALLOCATE          33
-#define FILE_CLOSE_ON_PORT_EXIT 34
-/* Return codes */
-
-#define FILE_RESP_OK         0
-#define FILE_RESP_ERROR      1
-#define FILE_RESP_DATA       2
-#define FILE_RESP_NUMBER     3
-#define FILE_RESP_INFO       4
-#define FILE_RESP_NUMERR     5
-#define FILE_RESP_LDATA      6
-#define FILE_RESP_N2DATA     7
-#define FILE_RESP_EOF        8
-#define FILE_RESP_FNAME      9
-#define FILE_RESP_ALL_DATA  10
-#define FILE_RESP_LFNAME    11
-
-/* Options */
-
-#define FILE_OPT_DELAYED_WRITE 0
-#define FILE_OPT_READ_AHEAD    1
-
-/* IPREAD variants */
-
-#define IPREAD_S32BU_P32BU 0
-
-/* Limits */
-
-#define FILE_SEGMENT_READ  (256*1024)
-#define FILE_SEGMENT_WRITE (256*1024)
-
-/* Internal */
-
-/* Set to 1 to test having read_ahead implicitly for read_line */ 
-#define ALWAYS_READ_LINE_AHEAD 0
-
-
-/* Must not be possible to get from malloc()! */
-#define FILE_FD_INVALID ((Sint)(-1))
-
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-
-#include <ctype.h>
-#include <sys/types.h>
-#include <stdlib.h>
-
-/* Need (NON)BLOCKING macros for sendfile */
-#ifndef WANT_NONBLOCKING
-#define WANT_NONBLOCKING
-#endif
-
-#include "sys.h"
-
-#include "erl_driver.h"
-#include "erl_efile.h"
-#include "erl_threads.h"
-#include "gzio.h"
-#include "dtrace-wrapper.h" 
-
-
-static ErlDrvSysInfo sys_info;
-
-/* For explanation of this var, see comment for same var in erl_async.c */
-static unsigned gcc_optimizer_hack = 0;
-
-#ifdef  USE_VM_PROBES
-
-#define DTRACE_EFILE_BUFSIZ 128
-
-#define DTRACE_INVOKE_SETUP(op) \
-    do { DTRACE3(efile_drv_int_entry, d->sched_i1, d->sched_i2, op); } while (0)
-#define DTRACE_INVOKE_SETUP_BY_NAME(op) \
-    struct t_data *d = (struct t_data *) data ; \
-    DTRACE_INVOKE_SETUP(op)
-#define DTRACE_INVOKE_RETURN(op) \
-    do { DTRACE3(efile_drv_int_return, d->sched_i1, d->sched_i2, \
-                 op); } while (0) ; gcc_optimizer_hack++ ;
-
-/* Assign human-friendlier id numbers to scheduler & I/O worker threads */
-int             dt_driver_idnum = 0;
-int             dt_driver_io_worker_base = 5000;
-erts_mtx_t      dt_driver_mutex;
-pthread_key_t   dt_driver_key;
-
-typedef struct {
-    int         thread_num;
-    Uint64      tag;
-} dt_private;
-
-dt_private *get_dt_private(int);
-#else  /* USE_VM_PROBES */
-#define DTRACE_INVOKE_SETUP(op)            do {} while (0)
-#define DTRACE_INVOKE_SETUP_BY_NAME(op)    do {} while (0)
-#define DTRACE_INVOKE_RETURN(op)           do {} while (0)
-#endif  /* USE_VM_PROBES */
-
-/* #define TRACE 1 */
-#ifdef TRACE
-#    define TRACE_C(c) do { putchar(c); fflush(stdout); } while (0)
-#    define TRACE_S(s) do { fputs((s), stdout); fflush(stdout); } while (0)
-#    define TRACE_F(args) do { printf args ;fflush(stdout); } while (0)
-#else
-#    define TRACE_C(c) ((void)(0))
-#    define TRACE_S(s) ((void)(0))
-#    define TRACE_F(args) ((void)(0))
-#endif
-
-
-#define THRDS_AVAILABLE (sys_info.async_threads > 0)
-#ifdef HARDDEBUG /* HARDDEBUG in io.c is expected too */
-#define TRACE_DRIVER fprintf(stderr, "Efile: ")
-#else
-#define TRACE_DRIVER
-#endif
-#define MUTEX_INIT(m, p) do { IF_THRDS { TRACE_DRIVER; (m = driver_pdl_create(p)); } } while (0)
-#define MUTEX_LOCK(m)    do { IF_THRDS { TRACE_DRIVER; driver_pdl_lock(m);   } } while (0)
-#define MUTEX_UNLOCK(m)  do { IF_THRDS { TRACE_DRIVER; driver_pdl_unlock(m); } } while (0)
-#define IF_THRDS if (THRDS_AVAILABLE)
-
-
-#define SENDFILE_FLGS_USE_THREADS (1 << 0)
-/**
- * On DARWIN sendfile can deadlock with close if called in
- * different threads. So until Apple fixes so that sendfile
- * is not buggy we disable usage of the async pool for
- * DARWIN. The testcase t_sendfile_crashduring reproduces
- * this error when using +A 10 and enabling SENDFILE_FLGS_USE_THREADS.
- */
-#if defined(__APPLE__) && defined(__MACH__)
-#define USE_THRDS_FOR_SENDFILE(DATA) 0
-#else
-#define USE_THRDS_FOR_SENDFILE(DATA) (DATA->flags & SENDFILE_FLGS_USE_THREADS)
-#endif /* defined(__APPLE__) && defined(__MACH__) */
-
-
-
-#if 0
-/* Experimental, for forcing all file operations to use the same thread. */
-   static unsigned file_fixed_key = 1;
-#  define KEY(desc) (&file_fixed_key)
-#else
-#  define KEY(desc) (&(desc)->key)
-#endif
-
-#ifndef MAX
-#  define MAX(x, y) (((x) > (y)) ? (x) : (y))
-#endif
-
-#ifdef FILENAMES_16BIT
-#ifdef USE_VM_PROBES
-#error 16bit characters in filenames and dtrace in combination is not supported.
-#endif
-#  define FILENAME_BYTELEN(Str) filename_len_16bit(Str)
-#  define FILENAME_COPY(To,From) filename_cpy_16bit((To),(From)) 
-#  define FILENAME_CHARSIZE 2
-
-   static int filename_len_16bit(char *str) 
-   {
-       char *p = str;
-       while(*p != '\0' || p[1] != '\0') {
-	   p += 2;
-       }
-       return (p - str);
-   }
-
-   static void filename_cpy_16bit(char *to, char *from) 
-   {
-       while(*from != '\0' || from[1] != '\0') {
-	   *to++ = *from++;
-	   *to++ = *from++;
-       }
-       *to++ = *from++;
-       *to++ = *from++;
-   }
-
-#else
-#  define FILENAME_BYTELEN(Str) strlen(Str)
-#  define FILENAME_COPY(To,From) strcpy(To,From) 
-#  define FILENAME_CHARSIZE 1
-#endif
-
-#if     (MAXPATHLEN+1)*FILENAME_CHARSIZE+1 > BUFSIZ
-#  define    RESBUFSIZE  ((MAXPATHLEN+1)*FILENAME_CHARSIZE+1)
-#else
-#  define    RESBUFSIZE  BUFSIZ
-#endif
-
-#define READDIR_CHUNKS (5)
-
-
-
-#if ALWAYS_READ_LINE_AHEAD
-#define DEFAULT_LINEBUF_SIZE 2048
-#else
-#define DEFAULT_LINEBUF_SIZE 512 /* Small, it's usually discarded anyway */ 
-#endif
-
-typedef unsigned char uchar;
-
-static ErlDrvData file_start(ErlDrvPort port, char* command);
-static int file_init(void);
-static void file_stop(ErlDrvData);
-static void file_output(ErlDrvData, char* buf, ErlDrvSizeT len);
-static ErlDrvSSizeT file_control(ErlDrvData, unsigned int command,
-				 char* buf, ErlDrvSizeT len,
-				 char **rbuf, ErlDrvSizeT rlen);
-static void file_timeout(ErlDrvData);
-static void file_outputv(ErlDrvData, ErlIOVec*);
-static void file_async_ready(ErlDrvData, ErlDrvThreadData);
-static void file_flush(ErlDrvData);
-
-#ifdef HAVE_SENDFILE
-static void file_ready_output(ErlDrvData data, ErlDrvEvent event);
-static void file_stop_select(ErlDrvEvent event, void* _);
-#endif /* HAVE_SENDFILE */
-
-
-enum e_timer {timer_idle, timer_again, timer_write};
-#ifdef HAVE_SENDFILE
-enum e_sendfile {sending, not_sending};
-#define SENDFILE_USE_THREADS (1 << 0)
-#endif /* HAVE_SENDFILE */
-
-struct t_data;
-
-typedef struct {
-    SWord           fd;
-    ErlDrvPort      port;
-    unsigned int    key;      /* Async queue key */
-    unsigned        flags;    /* Original flags from FILE_OPEN. */
-    void          (*invoke)(void *);
-    struct t_data  *d;
-    void          (*free)(void *);
-    struct t_data  *cq_head;  /* Queue of incoming commands */
-    struct t_data  *cq_tail;  /* -""- */
-    enum e_timer    timer_state;
-#ifdef HAVE_SENDFILE
-    enum e_sendfile sendfile_state;
-#endif /* HAVE_SENDFILE */
-    size_t          read_bufsize;
-    ErlDrvBinary   *read_binp;
-    size_t          read_offset;
-    size_t          read_size;
-    size_t          write_bufsize;
-    unsigned long   write_delay;
-    int             write_error;
-    Efile_error     write_errInfo;
-    ErlDrvPDL       q_mtx;    /* Mutex for the driver queue, known by the emulator. Also used for
-				 mutual exclusion when accessing field(s) below. */
-    size_t          write_buffered;
-#ifdef USE_VM_PROBES
-    int             idnum;      /* Unique ID # for this driver thread/desc */
-    char            port_str[DTRACE_TERM_BUF_SIZE];
-#endif
-} file_descriptor;
-
-
-static int reply_error(file_descriptor*, Efile_error* errInfo);
-
-struct erl_drv_entry efile_driver_entry = {
-    file_init,
-    file_start,
-    file_stop,
-    file_output,
-    NULL,
-#ifdef HAVE_SENDFILE
-    file_ready_output,
-#else
-    NULL,
-#endif /* HAVE_SENDFILE */
-    "efile",
-    NULL,
-    NULL,
-    file_control,
-    file_timeout,
-    file_outputv,
-    file_async_ready,
-    file_flush,
-    NULL,
-    NULL,
-    ERL_DRV_EXTENDED_MARKER,
-    ERL_DRV_EXTENDED_MAJOR_VERSION,
-    ERL_DRV_EXTENDED_MINOR_VERSION,
-    ERL_DRV_FLAG_USE_PORT_LOCKING,
-    NULL,
-    NULL,
-#ifdef HAVE_SENDFILE
-    file_stop_select
-#else
-    NULL
-#endif /* HAVE_SENDFILE */
-};
-
-
-
-static int thread_short_circuit;
-
-#define DRIVER_ASYNC(level, desc, f_invoke, data, f_free) \
-if (thread_short_circuit >= (level)) { \
-    (*(f_invoke))(data); \
-    file_async_ready((ErlDrvData)(desc), (data)); \
-} else { \
-    driver_async((desc)->port, KEY(desc), (f_invoke), (data), (f_free)); \
-}
-
-
-
-struct t_pbuf_spec {
-    Sint64 offset;
-    size_t size;
-};
-
-struct t_pwritev {
-    ErlDrvPort         port;
-    ErlDrvPDL          q_mtx;
-    size_t             size;
-    unsigned           cnt;
-    unsigned           n;
-    struct t_pbuf_spec specs[1];
-};
-
-struct t_preadv {
-    ErlIOVec eiov;
-    unsigned n;
-    unsigned cnt;
-    size_t   size;
-    Sint64   offsets[1];
-};
-
-#define READDIR_BUFSIZE (8*1024)*READDIR_CHUNKS
-#if READDIR_BUFSIZE < (1 + (2 + MAXPATHLEN)*FILENAME_CHARSIZE*READDIR_CHUNKS)
-#  undef READDIR_BUFSIZE
-#  define READDIR_BUFSIZE (1 + (2 + MAXPATHLEN)*FILENAME_CHARSIZE*READDIR_CHUNKS)
-#endif
-
-struct t_readdir_buf {
-     struct t_readdir_buf *next;
-     size_t n;
-     char buf[READDIR_BUFSIZE];
-};
-
-struct t_data
-{
-    struct t_data *next;
-    int            command;
-    int            level;
-    void         (*invoke)(void *);
-    void         (*free)(void *);
-    void           *data_to_free; /* used by FILE_CLOSE_ON_PORT_EXIT only */
-    int            again;
-    int            reply;
-#ifdef  USE_VM_PROBES
-    int               sched_i1;
-    Uint64            sched_i2;
-    char              sched_utag[DTRACE_EFILE_BUFSIZ+1];
-#endif
-    int            result_ok;
-    Efile_error    errInfo;
-    int            flags;
-    SWord          fd;
-    int            is_fd_unused;
-    /**/
-    Efile_info        info;
-    EFILE_DIR_HANDLE  dir_handle; /* Handle to open directory. */
-    ErlDrvBinary     *bin;
-    int               drive;
-    size_t            n;
-    /*off_t             offset;*/
-    /*size_t            bytesRead; Bytes read from the file. */
-    /**/
-    union {
-	struct {
-	    Sint64 offset;
-	    int    origin;
-	    Sint64 location;
-	} lseek;
-	struct {
-	    ErlDrvPort    port;
-	    ErlDrvPDL     q_mtx;
-	    size_t        size;
-	    size_t        reply_size;
-	} writev;
-	struct t_pwritev pwritev;
-	struct t_preadv  preadv;
-	struct {
-	    ErlDrvBinary *binp;
-	    size_t        bin_offset;
-	    size_t        bin_size;
-	    size_t        size;
-	} read;
-	struct {
-	    ErlDrvBinary *binp; /* in - out */
-	    size_t        read_offset; /* in - out */
-	    size_t        read_size; /* in - out */
-	    size_t        nl_pos; /* out */
-	    short         nl_skip; /* out, 0 or 1 */
-#if !ALWAYS_READ_LINE_AHEAD
-	    short         read_ahead; /* in, bool */
-#endif
-	} read_line;
-	struct {
-	    ErlDrvBinary *binp;
-	    int           size;
-	    int           offset;
-	} read_file;
-	struct {
-	    struct t_readdir_buf *first_buf;
-	    struct t_readdir_buf *last_buf;
-	} read_dir;
-	struct {
-	    Sint64 offset;
-	    Sint64 length;
-	    int advise;
-	} fadvise;
-#ifdef HAVE_SENDFILE
-	struct {
-	    ErlDrvPort port;
-	    ErlDrvPDL q_mtx;
-	    int out_fd;
-	    off_t offset;
-	    Uint64 nbytes;
-	    Uint64 written;
-	} sendfile;
-#endif /* HAVE_SENDFILE */
-	struct {
-	    Sint64 offset;
-	    Sint64 length;
-	} fallocate;
-    } c;
-    char b[1];
-};
-
-#define EF_ALLOC(S)		driver_alloc((S))
-#define EF_REALLOC(P, S)	driver_realloc((P), (S))
-#define EF_SAFE_ALLOC(S)	ef_safe_alloc((S))
-#define EF_SAFE_REALLOC(P, S)	ef_safe_realloc((P), (S))
-#define EF_FREE(P)		do { if((P)) driver_free((P)); } while(0)
-
-static void *ef_safe_alloc(Uint s)
-{
-    void *p = EF_ALLOC(s);
-    if (!p) erts_exit(ERTS_ERROR_EXIT, "efile drv: Can't allocate %lu bytes of memory\n", (unsigned long)s);
-    return p;
-}
-
-/*********************************************************************
- * ErlIOVec manipulation functions.
- */
-
-/* char EV_CHAR_P(ErlIOVec *ev, int p, int q) */
-#define EV_CHAR_P(ev, p, q)			\
-    (((char *)(ev)->iov[q].iov_base) + (p))
-
-/* int EV_GET_CHAR(ErlIOVec *ev, char *p, int *pp, int *qp) */
-#define EV_GET_CHAR(ev, p, pp, qp) efile_ev_get_char(ev, p ,pp, qp)
-static int
-efile_ev_get_char(ErlIOVec *ev, char *p, size_t *pp, size_t *qp) {
-    if (*pp + 1 <= ev->iov[*qp].iov_len) {
-	*p = *EV_CHAR_P(ev, *pp, *qp);
-	if (*pp + 1 < ev->iov[*qp].iov_len)
-	    *pp += 1;
-	else {
-	    *qp += 1;
-	    *pp = 0;
-	}
-	return !0;
-    }
-    return 0;
-}
-
-/* Uint32 EV_UINT32(ErlIOVec *ev, int p, int q)*/
-#define EV_UINT32(ev, p, q)						\
-    ((Uint32) ((unsigned char *)(ev)->iov[q].iov_base)[p])
-
-/* int EV_GET_UINT32(ErlIOVec *ev, Uint32 *p, int *pp, int *qp) */
-#define EV_GET_UINT32(ev, p, pp, qp) efile_ev_get_uint32(ev, p, pp, qp)
-static int
-efile_ev_get_uint32(ErlIOVec *ev, Uint32 *p, size_t *pp, size_t *qp) {
-    if (*pp + 4 <= ev->iov[*qp].iov_len) {
-	*p = (EV_UINT32(ev, *pp,   *qp) << 24)
-	    | (EV_UINT32(ev, *pp + 1, *qp) << 16)
-	    | (EV_UINT32(ev, *pp + 2, *qp) << 8)
-	    | (EV_UINT32(ev, *pp + 3, *qp));
-	if (*pp + 4 < ev->iov[*qp].iov_len)
-	    *pp += 4;
-	else {
-	    *qp += 1;
-	    *pp = 0;
-	}
-	return !0;
-    }
-    return 0;
-}
-
-/* Uint64 EV_UINT64(ErlIOVec *ev, int p, int q)*/
-#define EV_UINT64(ev, p, q)						\
-    ((Uint64) ((unsigned char *)(ev)->iov[q].iov_base)[p])
-
-/* int EV_GET_UINT64(ErlIOVec *ev, Uint64 *p, int *pp, int *qp) */
-#define EV_GET_UINT64(ev, p, pp, qp) efile_ev_get_uint64(ev, p, pp, qp)
-static int
-efile_ev_get_uint64(ErlIOVec *ev, Uint64 *p, size_t *pp, size_t *qp) {
-    if (*pp + 8 <= ev->iov[*qp].iov_len) {
-	*p = (EV_UINT64(ev, *pp, *qp) << 56)
-	    | (EV_UINT64(ev, *pp + 1, *qp) << 48)
-	    | (EV_UINT64(ev, *pp + 2, *qp) << 40)
-	    | (EV_UINT64(ev, *pp + 3, *qp) << 32)
-	    | (EV_UINT64(ev, *pp + 4, *qp) << 24)
-	    | (EV_UINT64(ev, *pp + 5, *qp) << 16)
-	    | (EV_UINT64(ev, *pp + 6, *qp) << 8)
-	    | (EV_UINT64(ev, *pp + 7, *qp));
-	if (*pp + 8 < ev->iov[*qp].iov_len)
-	    *pp += 8;
-	else {
-	    *qp += 1;
-	    *pp = 0;
-	}
-	return !0;
-    }
-    return 0;
-}
-
-/* int EV_GET_SINT64(ErlIOVec *ev, Uint64 *p, int *pp, int *qp) */
-#define EV_GET_SINT64(ev, p, pp, qp) efile_ev_get_sint64(ev, p, pp, qp)
-static int
-efile_ev_get_sint64(ErlIOVec *ev, Sint64 *p, size_t *pp, size_t *qp) {
-    Uint64 *tmp = (Uint64*)p;
-    return EV_GET_UINT64(ev, tmp, pp, qp);
-}
-
-#if 0
-
-static void ev_clear(ErlIOVec *ev) {
-    ASSERT(ev);
-    ev->size = 0;
-    ev->vsize = 0;
-    ev->iov = NULL;
-    ev->binv = NULL;
-}
-
-/* Assumes that ->iov and ->binv were allocated with sys_alloc().
- */
-static void ev_free(ErlIOVec *ev) {
-    if (! ev) {
-	return;
-    }
-    if (ev->vsize > 0) {
-	int i;
-	ASSERT(ev->iov);
-	ASSERT(ev->binv);
-	for (i = 0; i < ev->vsize; i++) {
-	    if (ev->binv[i]) {
-		driver_free_binary(ev->binv[i]);
-	    }
-	}
-	EF_FREE(ev->iov);
-	EF_FREE(ev->binv);
-    }
-}
-
-/* Copy the contents from source to dest.
- * Data in binaries is not copied, just the pointers; 
- * and refc is incremented.
- */
-static ErlIOVec *ev_copy(ErlIOVec *dest, ErlIOVec *source) {
-    int *ip;
-    ASSERT(dest);
-    ASSERT(source);
-    if (source->vsize == 0) {
-	/* Empty source */
-	ev_clear(dest);
-	return dest;
-    }
-    /* Allocate ->iov and ->binv */
-    dest->iov = EF_ALLOC(sizeof(*dest->iov) * source->vsize);
-    if (! dest->iov) {
-	return NULL;
-    }
-    dest->binv = EF_ALLOC(sizeof(*dest->binv) * source->vsize);
-    if (! dest->binv) {
-	EF_FREE(dest->iov);
-	return NULL;
-    }
-    dest->size = source->size;
-    /* Copy one vector element at the time. 
-     * Use *ip as an alias for dest->vsize to improve readabiliy.
-     * Keep dest consistent in every iteration by using 
-     * dest->vsize==*ip as loop variable.
-     */
-    for (ip = &dest->vsize, *ip = 0;  *ip < source->vsize;  (*ip)++) {
-	if (source->iov[*ip].iov_len == 0) {
-	    /* Empty vector element */
-	    dest->iov[*ip].iov_len = 0;
-	    dest->iov[*ip].iov_base = NULL;
-	    dest->binv[*ip] = NULL;
-	} else {
-	    /* Non empty vector element */
-	    if (source->binv[*ip]) {
-		/* Contents in binary - copy pointers and increment refc */
-		dest->iov[*ip] = source->iov[*ip];
-		dest->binv[*ip] = source->binv[*ip];
-		driver_binary_inc_refc(source->binv[*ip]);
-	    } else {
-		/* Contents not in binary - allocate new binary and copy data */
-		if (! (dest->binv[*ip] = 
-		       driver_alloc_binary(source->iov[*ip].iov_len))) {
-		    goto failed;
-		}
-		sys_memcpy(dest->binv[*ip]->orig_bytes,
-			   source->iov[*ip].iov_base,
-			   source->iov[*ip].iov_len);
-		dest->iov[*ip].iov_base = dest->binv[*ip]->orig_bytes;
-		dest->iov[*ip].iov_len = source->iov[*ip].iov_len;
-	    }
-	}
-    }
-    return dest;
- failed:
-    ev_free(dest);
-    return NULL;
-}
-
-#endif
-
-
-
-/*********************************************************************
- * Command queue functions
- */
-
-static void cq_enq(file_descriptor *desc, struct t_data *d) {
-    ASSERT(d);
-    if (desc->cq_head) {
-	ASSERT(desc->cq_tail);
-	ASSERT(!desc->cq_tail->next);
-	desc->cq_tail = desc->cq_tail->next = d;
-    } else {
-	ASSERT(desc->cq_tail == NULL);
-	desc->cq_head = desc->cq_tail = d;
-    }
-    d->next = NULL;
-}
-
-static struct t_data *cq_deq(file_descriptor *desc) {
-    struct t_data *d = desc->cq_head;
-    ASSERT(d || (!d && !desc->cq_tail));
-    if (d) {
-	ASSERT(!d->next || (d->next && desc->cq_tail != d));
-	if ((desc->cq_head = d->next) == NULL) {
-	    ASSERT(desc->cq_tail == d);
-	    desc->cq_tail = NULL;
-	}
-    }	
-    return d;
-}
-
-
-/*********************************************************************
- * Driver entry point -> init
- */
-static int 
-file_init(void)
-{
-    char buf[21]; /* enough to hold any 64-bit integer */
-    size_t bufsz = sizeof(buf);
-    thread_short_circuit = (erl_drv_getenv("ERL_EFILE_THREAD_SHORT_CIRCUIT",
-					   buf,
-					   &bufsz) == 0
-			    ? atoi(buf)
-			    : 0);
-    driver_system_info(&sys_info, sizeof(ErlDrvSysInfo));
-
-    /* run initiation of efile_driver if needed */
-    efile_init();
-
-#ifdef  USE_VM_PROBES
-    erts_mtx_init(&dt_driver_mutex, "efile_drv dtrace mutex", NIL,
-        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
-    pthread_key_create(&dt_driver_key, NULL);
-#endif  /* USE_VM_PROBES */
-
-    return 0;
-}
-
-
-/*********************************************************************
- * Driver entry point -> start
- */
-static ErlDrvData 
-file_start(ErlDrvPort port, char* command) 
-
-{
-    file_descriptor* desc;
-
-    if ((desc = (file_descriptor*) EF_ALLOC(sizeof(file_descriptor)))
-	== NULL) {
-	errno = ENOMEM;
-	return ERL_DRV_ERROR_ERRNO;
-    }
-    desc->fd = FILE_FD_INVALID;
-    desc->port = port;
-    desc->key = driver_async_port_key(port);
-    desc->flags = 0;
-    desc->invoke = NULL;
-    desc->d = NULL;
-    desc->free = NULL;
-    desc->cq_head = NULL;
-    desc->cq_tail = NULL;
-    desc->timer_state = timer_idle;
-#ifdef HAVE_SENDFILE
-    desc->sendfile_state = not_sending;
-#endif
-    desc->read_bufsize = 0;
-    desc->read_binp = NULL;
-    desc->read_offset = 0;
-    desc->read_size = 0;
-    desc->write_delay = 0L;
-    desc->write_bufsize = 0;
-    desc->write_error = 0;
-    MUTEX_INIT(desc->q_mtx, port); /* Refc is one, referenced by emulator now */
-    desc->write_buffered = 0;
-#ifdef  USE_VM_PROBES
-    dtrace_drvport_str(port, desc->port_str);
-    get_dt_private(0);           /* throw away return value */
-#endif  /* USE_VM_PROBES */
-    return (ErlDrvData) desc;
-}
-
-static void do_close(int flags, SWord fd) {
-    if (flags & EFILE_COMPRESSED) {
-	erts_gzclose((ErtsGzFile)(fd));
-    } else {
-	efile_closefile((int) fd);
-    }
-}
-
-static void invoke_close(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    DTRACE_INVOKE_SETUP(FILE_CLOSE);
-    d->again = 0;
-    do_close(d->flags, d->fd);
-    DTRACE_INVOKE_RETURN(FILE_CLOSE);
-}
-
-static void free_data(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-
-    switch (d->command) {
-    case FILE_OPEN:
-        if (d->is_fd_unused && d->fd != FILE_FD_INVALID) {
-            /* This is OK to do in scheduler thread because there can be no async op
-               ongoing for this fd here, as we exited during async open.
-               Ideally, this close should happen in an async thread too, but that would
-               require a substantial rewrite, as we are here because of a dead port and
-               cannot schedule async jobs for that port any more... */
-            do_close(d->flags, d->fd);
-        }
-        break;
-    case FILE_CLOSE_ON_PORT_EXIT:
-        EF_FREE(d->data_to_free);
-        break;
-    }
-
-    EF_FREE(data);
-}
-
-
-/*
- * Sends back an error reply to Erlang.
- */
-
-static void reply_posix_error(file_descriptor *desc, int posix_errno) {
-    char response[256];		/* Response buffer. */
-    char* s;
-    char* t;
-    
-    /*
-     * Contents of buffer sent back:
-     *
-     * +-----------------------------------------+
-     * | FILE_RESP_ERROR | Posix error id string |
-     * +-----------------------------------------+
-     */
-
-    TRACE_C('E');
-
-    response[0] = FILE_RESP_ERROR;
-    for (s = erl_errno_id(posix_errno), t = response+1; *s; s++, t++)
-	*t = tolower(*s);
-    driver_output2(desc->port, response, t-response, NULL, 0);
-}
-
-static void reply_Uint_posix_error(file_descriptor *desc, Uint num, 
-				   int posix_errno) {
-    char response[256];		/* Response buffer. */
-    char* s;
-    char* t;
-    
-    /*
-     * Contents of buffer sent back:
-     *
-     * +----------------------------------------------------------------------+
-     * | FILE_RESP_NUMERR | 64-bit number (big-endian) | Posix error id string |
-     * +----------------------------------------------------------------------+
-     */
-
-    TRACE_C('N');
-
-    response[0] = FILE_RESP_NUMERR;
-#if SIZEOF_VOID_P == 4
-    put_int32(0, response+1);
-#else
-    put_int32(num>>32, response+1);
-#endif
-    put_int32((Uint32)num, response+1+4);
-    for (s = erl_errno_id(posix_errno), t = response+1+4+4; *s; s++, t++)
-	*t = tolower(*s);
-    driver_output2(desc->port, response, t-response, NULL, 0);
-}
-
-#ifdef HAVE_SENDFILE
-static void reply_string_error(file_descriptor *desc, char* str) {
-    char response[256];		/* Response buffer. */
-    char* s;
-    char* t;
-
-    response[0] = FILE_RESP_ERROR;
-    for (s = str, t = response+1; *s; s++, t++)
-	*t = tolower(*s);
-    driver_output2(desc->port, response, t-response, NULL, 0);
-}
-#endif
-
-static int reply_error(file_descriptor *desc, 
-		       Efile_error *errInfo) /* The error codes. */
-{
-    reply_posix_error(desc, errInfo->posix_errno);
-    return 0;
-}
-
-static int reply_Uint_error(file_descriptor *desc, Uint num, 
-			    Efile_error *errInfo) /* The error codes. */
-{
-    reply_Uint_posix_error(desc, num, errInfo->posix_errno);
-    return 0;
-}
-
-static int reply_ok(file_descriptor *desc) {
-    char c = FILE_RESP_OK;
-
-    driver_output2(desc->port, &c, 1, NULL, 0);
-    return 0;
-}
-
-static int reply(file_descriptor *desc, int ok, Efile_error *errInfo) {
-    if (!ok) {
-	reply_error(desc, errInfo);
-    } else {
-	TRACE_C('K');
-	reply_ok(desc);
-    }
-    return 0;
-}
-
-static int reply_Uint(file_descriptor *desc, Uint result) {
-    char tmp[1+4+4];
-
-    /*
-     * Contents of buffer sent back:
-     *
-     * +-----------------------------------------------+
-     * | FILE_RESP_NUMBER | 64-bit number (big-endian) |
-     * +-----------------------------------------------+
-     */
-
-    TRACE_C('R');
-
-    tmp[0] = FILE_RESP_NUMBER;
-#if SIZEOF_VOID_P == 4
-    put_int32(0, tmp+1);
-#else
-    put_int32(result>>32, tmp+1);
-#endif
-    put_int32((Uint32)result, tmp+1+4);
-    driver_output2(desc->port, tmp, sizeof(tmp), NULL, 0);
-    return 0;
-}
-
-static int reply_Sint64(file_descriptor *desc, Sint64 result) {
-    char tmp[1+4+4];
-
-    /*
-     * Contents of buffer sent back:
-     *
-     * +-----------------------------------------------+
-     * | FILE_RESP_NUMBER | 64-bit number (big-endian) |
-     * +-----------------------------------------------+
-     */
-
-    TRACE_C('R');
-
-    tmp[0] = FILE_RESP_NUMBER;
-    put_int64(result, tmp+1);
-    driver_output2(desc->port, tmp, sizeof(tmp), NULL, 0);
-    return 0;
-}
-
-#if 0
-static void reply_again(file_descriptor *desc) {
-    char tmp[1];
-    tmp[0] = FILE_RESP_AGAIN;
-    driver_output2(desc->port, tmp, sizeof(tmp), NULL, 0);
-}
-#endif
-
-static void reply_ev(file_descriptor *desc, char response, ErlIOVec *ev) {
-    char tmp[1];
-    /* Data arriving at the Erlang process:
-     * [Response, Binary0, Binary1, .... | BinaryN-1]
-     */
-    tmp[0] = response;
-    driver_outputv(desc->port, tmp, sizeof(tmp), ev, 0);
-}
-
-static void reply_data(file_descriptor *desc, 
-		       ErlDrvBinary *binp, size_t offset, size_t len) {
-    char header[1+4+4];
-    /* Data arriving at the Erlang process:
-     * [?FILE_RESP_DATA, 64-bit length (big-endian) | Data]
-     */
-    header[0] = FILE_RESP_DATA;
-#if SIZEOF_SIZE_T == 4
-    put_int32(0, header+1);
-#else
-    put_int32(len>>32, header+1);
-#endif
-    put_int32((Uint32)len, header+1+4);
-    driver_output_binary(desc->port, header, sizeof(header),
-			 binp, offset, len);
-}
-
-static void reply_buf(file_descriptor *desc, char *buf, size_t len) {
-    char header[1+4+4];
-    /* Data arriving at the Erlang process:
-     * [?FILE_RESP_DATA, 64-bit length (big-endian) | Data]
-     */
-    header[0] = FILE_RESP_DATA;
-#if SIZEOF_SIZE_T == 4
-    put_int32(0, header+1);
-#else
-    put_int32(len>>32, header+1);
-#endif
-    put_int32((Uint32)len, header+1+4);
-    driver_output2(desc->port, header, sizeof(header), buf, len);
-}
-
-static int reply_eof(file_descriptor *desc) {
-    char c = FILE_RESP_EOF;
-
-    driver_output2(desc->port, &c, 1, NULL, 0);
-    return 0;
-}
- 
-static void invoke_name(void *data, int (*f)(Efile_error *, char *))
-{
-    struct t_data *d = (struct t_data *) data;
-    char *name = (char *) d->b;
-
-    d->again = 0;
-    d->result_ok = (*f)(&d->errInfo, name);
-}
-
-static void invoke_mkdir(void *data)
-{
-    DTRACE_INVOKE_SETUP_BY_NAME(FILE_MKDIR);
-    invoke_name(data, efile_mkdir);
-    DTRACE_INVOKE_RETURN(FILE_MKDIR);
-}
-
-static void invoke_rmdir(void *data)
-{
-    DTRACE_INVOKE_SETUP_BY_NAME(FILE_RMDIR);
-    invoke_name(data, efile_rmdir);
-    DTRACE_INVOKE_RETURN(FILE_RMDIR);
-}
-
-static void invoke_delete_file(void *data)
-{
-    DTRACE_INVOKE_SETUP_BY_NAME(FILE_DELETE);
-    invoke_name(data, efile_delete_file);
-    DTRACE_INVOKE_RETURN(FILE_DELETE);
-}
-
-static void invoke_chdir(void *data)
-{
-    DTRACE_INVOKE_SETUP_BY_NAME(FILE_CHDIR);
-    invoke_name(data, efile_chdir);
-    DTRACE_INVOKE_RETURN(FILE_CHDIR);
-}
-
-static void invoke_fdatasync(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    int fd = (int) d->fd;
-    DTRACE_INVOKE_SETUP(FILE_FDATASYNC);
-
-    d->again = 0;
-    d->result_ok = efile_fdatasync(&d->errInfo, fd);
-    DTRACE_INVOKE_RETURN(FILE_FDATASYNC);
-}
-
-static void invoke_fsync(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    int fd = (int) d->fd;
-    DTRACE_INVOKE_SETUP(FILE_FSYNC);
-
-    d->again = 0;
-    d->result_ok = efile_fsync(&d->errInfo, fd);
-    DTRACE_INVOKE_RETURN(FILE_FSYNC);
-}
-
-static void invoke_truncate(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    int fd = (int) d->fd;
-    DTRACE_INVOKE_SETUP(FILE_TRUNCATE);
-
-    d->again = 0;
-    d->result_ok = efile_truncate_file(&d->errInfo, &fd, d->flags);
-    DTRACE_INVOKE_RETURN(FILE_TRUNCATE);
-}
-
-static void invoke_read(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    int status, segment;
-    size_t size, read_size;
-    DTRACE_INVOKE_SETUP(FILE_READ);
-
-    segment = d->again && d->c.read.bin_size >= 2*FILE_SEGMENT_READ;
-    if (segment) {
-	size = FILE_SEGMENT_READ;
-    } else {
-	size = d->c.read.bin_size;
-    }
-    read_size = size;
-    if (d->flags & EFILE_COMPRESSED) {
-	read_size = erts_gzread((ErtsGzFile)d->fd,
-				d->c.read.binp->orig_bytes + d->c.read.bin_offset,
-				size);
-	status = (read_size != (size_t) -1);
-	if (!status) {
-	    d->errInfo.posix_errno = EIO;
-	}
-    } else {
-	status = efile_read(&d->errInfo, d->flags, (int) d->fd,
-			    d->c.read.binp->orig_bytes + d->c.read.bin_offset,
-			    size,
-			    &read_size);
-    }
-    if ( (d->result_ok = status)) {
-	ASSERT(read_size <= size);
-	d->c.read.bin_offset += read_size;
-	if (read_size < size || !segment) {
-	    d->c.read.bin_size = 0;
-	    d->again = 0;
-	} else {
-	    d->c.read.bin_size -= read_size;
-	}
-    } else {
-	d->again = 0;
-    }
-    DTRACE_INVOKE_RETURN(FILE_READ);
-}
-
-static void free_read(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-
-    driver_free_binary(d->c.read.binp);
-    EF_FREE(d);
-}
-
-static void invoke_read_line(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    int status;
-    size_t read_size = 0;
-    int local_loop = (d->again == 0);
-    DTRACE_INVOKE_SETUP(FILE_READ_LINE);
-
-    do {
-	size_t size = (d->c.read_line.binp)->orig_size - 
-	    d->c.read_line.read_offset - d->c.read_line.read_size;
-	if (size == 0) {
-	    /* Need more place */
-	    ErlDrvSizeT need = (d->c.read_line.read_size >= DEFAULT_LINEBUF_SIZE) ?
-		d->c.read_line.read_size + DEFAULT_LINEBUF_SIZE : DEFAULT_LINEBUF_SIZE;
-	    ErlDrvBinary   *newbin;
-#if !ALWAYS_READ_LINE_AHEAD
-	    /* Use read_ahead size if need does not exceed it */
-	    if (need < (d->c.read_line.binp)->orig_size && 
-		d->c.read_line.read_ahead)
-	      need = (d->c.read_line.binp)->orig_size;
-#endif
-	    newbin = driver_alloc_binary(need);
-	    if (newbin == NULL) {
-		d->result_ok = 0;
-		d->errInfo.posix_errno = ENOMEM;
-		d->again = 0;
-		break;
-	    }
-	    memcpy(newbin->orig_bytes, (d->c.read_line.binp)->orig_bytes + d->c.read_line.read_offset,  
-		   d->c.read_line.read_size);
-	    driver_free_binary(d->c.read_line.binp);
-	    d->c.read_line.binp = newbin;
-	    d->c.read_line.read_offset = 0;
-	    size = need - d->c.read_line.read_size;
-	}
-	if (d->flags & EFILE_COMPRESSED) {
-	    read_size = erts_gzread((ErtsGzFile)d->fd,
-				    d->c.read_line.binp->orig_bytes + 
-				    d->c.read_line.read_offset + d->c.read_line.read_size,
-				    size);
-	    status = (read_size != (size_t) -1);
-	    if (!status) {
-		d->errInfo.posix_errno = EIO;
-	    }
-	} else {
-	    status = efile_read(&d->errInfo, d->flags, (int) d->fd,
-				d->c.read_line.binp->orig_bytes + 
-				d->c.read_line.read_offset + d->c.read_line.read_size,
-				size,
-				&read_size);
-	}
-	if ( (d->result_ok = status)) {
-	    void *nl_ptr = memchr((d->c.read_line.binp)->orig_bytes + 
-				  d->c.read_line.read_offset + d->c.read_line.read_size,'\n',read_size);
-	    ASSERT(read_size <= size);
-	    d->c.read_line.read_size += read_size;
-	    if (nl_ptr != NULL) {
-		/* If found, we're done */
-		d->c.read_line.nl_pos = ((char *) nl_ptr) - 
-		    ((char *) ((d->c.read_line.binp)->orig_bytes)) + 1;
-		if (d->c.read_line.nl_pos > 1 &&
-		    *(((char *) nl_ptr) - 1) == '\r') {
-		    --d->c.read_line.nl_pos;
-		    *(((char *) nl_ptr) - 1) = '\n';
-		    d->c.read_line.nl_skip = 1;
-		} else {
-		    d->c.read_line.nl_skip = 0;
-		}
-		d->again = 0;
-#if !ALWAYS_READ_LINE_AHEAD
-		if (!(d->c.read_line.read_ahead)) {
-		    /* Ouch! Undo buffering... */
-		    size_t too_much = d->c.read_line.read_size - d->c.read_line.nl_skip - 
-			(d->c.read_line.nl_pos - d->c.read_line.read_offset);
-		    d->c.read_line.read_size -= too_much;
-		    ASSERT(d->c.read_line.read_size >= 0);
-		    if (d->flags & EFILE_COMPRESSED) {
-			Sint64 location = erts_gzseek((ErtsGzFile)d->fd,
-						      -((Sint64) too_much), EFILE_SEEK_CUR);
-			if (location == -1) {
-			    d->result_ok = 0;
-			    d->errInfo.posix_errno = errno;
-			}
-		    } else {
-			Sint64 location;
-			d->result_ok = efile_seek(&d->errInfo, (int) d->fd, 
-						-((Sint64) too_much), EFILE_SEEK_CUR,
-						&location);
-		    }
-		}
-#endif
-		break;
-	    } else if (read_size == 0) {
-		d->c.read_line.nl_pos = 
-		    d->c.read_line.read_offset + d->c.read_line.read_size;
-		d->c.read_line.nl_skip = 0;
-		d->again = 0;
-		break;
-	    }
-	} else {
-	    d->again = 0;
-	    break;
-	}
-    } while (local_loop);
-    DTRACE_INVOKE_RETURN(FILE_READ_LINE);
-}
-
-static void free_read_line(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-
-    driver_free_binary(d->c.read_line.binp);
-    EF_FREE(d);
-}
-
-void read_file_zero_size(struct t_data* d);
-#define ZERO_FILE_CHUNK (64 * 1024)
-
-/* [ERL-327] Some special files like /proc/... have reported size 0 */
-void read_file_zero_size(struct t_data* d) {
-    size_t total_read_size = 0;
-    size_t allocated_size = ZERO_FILE_CHUNK; /* allocd in invoke_read_file */
-    for (;;) {
-        size_t read_result;
-
-        /* Read until we hit EOF (read less than FILE_SEGMENT_READ) */
-        d->result_ok = efile_read(&d->errInfo,
-                                  EFILE_MODE_READ,
-                                  (int) d->fd,
-                                  (d->c.read_file.binp->orig_bytes +
-                                   total_read_size),
-                                  ZERO_FILE_CHUNK,
-                                  &read_result);
-        if (!d->result_ok) {
-            break;
-        }
-
-        total_read_size += read_result;
-        d->c.read_file.offset += read_result;
-        if (read_result < ZERO_FILE_CHUNK) {
-            break;
-        }
-
-        /* Grow before the next read call */
-        allocated_size = total_read_size + ZERO_FILE_CHUNK;
-        d->c.read_file.binp = driver_realloc_binary(d->c.read_file.binp,
-                                                    allocated_size);
-    }
-
-    /* Finalize the memory usage. Hopefully it was read fully on the first
-     * go, so the binary allocation overhead becomes:
-     * alloc ZERO_FILE_CHUNK (64kb) -> realloc real_size */
-    if (allocated_size != total_read_size) {
-        d->c.read_file.binp = driver_realloc_binary(d->c.read_file.binp,
-                                                    total_read_size);
-    }
-    d->again = 0;
-}
-
-static void invoke_read_file(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    size_t read_size;
-    int chop;
-    DTRACE_INVOKE_SETUP(FILE_READ_FILE);
-    
-    if (! d->c.read_file.binp) { /* First invocation only */
-	int fd;
-	Sint64 size;
-	
-	if (! (d->result_ok = 
-	       efile_openfile(&d->errInfo, d->b, 
-			      EFILE_MODE_READ, &fd, &size))) {
-	    goto done;
-	}
-	d->fd = fd;
-	d->c.read_file.size = (int) size;
-
-        /* For zero sized files allocate a reasonable chunk to attempt reading
-         * anyway. Note: This will eat ZERO_FILE_CHUNK bytes for any 0 file
-         * and free them immediately after (if the file was empty). */
-        ERTS_ASSERT(size >= 0);
-        d->c.read_file.binp = driver_alloc_binary(size != 0 ? (size_t)size
-                                                            : ZERO_FILE_CHUNK);
-
-	if (size < 0 || size != d->c.read_file.size || !d->c.read_file.binp) {
-	    d->result_ok = 0;
-	    d->errInfo.posix_errno = ENOMEM;
-	    goto close;
-	}
-	d->c.read_file.offset = 0;
-    }
-    /* Invariant: d->c.read_file.size >= d->c.read_file.offset */
-    
-    if (d->c.read_file.size == 0) {
-        read_file_zero_size(d);
-        goto close;
-    }
-
-    read_size = (size_t) (d->c.read_file.size - d->c.read_file.offset);
-    if (! read_size) goto close;
-    chop = d->again && read_size >= FILE_SEGMENT_READ*2;
-    if (chop) read_size = FILE_SEGMENT_READ;
-    d->result_ok = 
-	efile_read(&d->errInfo, 
-		   EFILE_MODE_READ, 
-		   (int) d->fd, 
-		   d->c.read_file.binp->orig_bytes + d->c.read_file.offset,
-		   read_size, 
-		   &read_size);
-    if (d->result_ok) {
-	d->c.read_file.offset += read_size;
-	if (chop) goto chop_done; /* again */
-    }
- close:
-    efile_closefile((int) d->fd);
- done:
-    d->again = 0;
- chop_done:
-    DTRACE_INVOKE_RETURN(FILE_READ_FILE);
-}
-
-static void free_read_file(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-
-    if (d->c.read_file.binp) driver_free_binary(d->c.read_file.binp);
-    EF_FREE(d);
-}
-
-
-
-static void invoke_preadv(void *data)
-{
-    struct t_data   *d = (struct t_data *) data;
-    struct t_preadv *c = &d->c.preadv;
-    ErlIOVec        *ev = &c->eiov;
-    size_t           bytes_read_so_far = 0;
-    unsigned char   *p = (unsigned char *)ev->iov[0].iov_base + 4+4+8*c->cnt;
-    DTRACE_INVOKE_SETUP(FILE_PREADV);
-
-    while (c->cnt < c->n) {
-	size_t read_size = ev->iov[1 + c->cnt].iov_len - c->size;
-	size_t bytes_read = 0;
-	int chop = d->again 
-	    && bytes_read_so_far + read_size >= 2*FILE_SEGMENT_READ;
-	if (chop) {
-	    ASSERT(bytes_read_so_far < FILE_SEGMENT_READ);
-	    read_size = FILE_SEGMENT_READ + FILE_SEGMENT_READ/2
-		- bytes_read_so_far;
-	}
-	if ( (d->result_ok 
-	      = efile_pread(&d->errInfo, 
-			    (int) d->fd,
-			    c->offsets[c->cnt] + c->size,
-			    ((char *)ev->iov[1 + c->cnt].iov_base) + c->size,
-			    read_size,
-			    &bytes_read))) {
-	    bytes_read_so_far += bytes_read;
-	    if (chop && bytes_read == read_size) {
-		c->size += bytes_read;
-		goto done;
-	    }
-	    ASSERT(bytes_read <= read_size);
-	    ev->iov[1 + c->cnt].iov_len = bytes_read + c->size;
-	    ev->size += bytes_read + c->size;
-	    put_int64(bytes_read + c->size, p); p += 8;
-	    c->size = 0;
-	    c->cnt++;
-	    if (d->again 
-		&& bytes_read_so_far >= FILE_SEGMENT_READ
-		&& c->cnt < c->n) {
-		goto done;
-	    }
-	} else {
-	    /* In case of a read error, ev->size will not be correct,
-	     * which does not matter since no read data is returned
-	     * to Erlang.
-	     */
-	    break;
-	}
-    }					
-    d->again = 0;
- done:
-    DTRACE_INVOKE_RETURN(FILE_PREADV);
-}
-
-static void free_preadv(void *data) {
-    struct t_data *d = data;
-    int            i;
-    ErlIOVec      *ev = &d->c.preadv.eiov;
-    
-    for(i = 0; i < ev->vsize; i++) {
-	driver_free_binary(ev->binv[i]);
-    }
-    EF_FREE(d);
-}
-
-static void invoke_ipread(void *data)
-{
-    struct t_data   *d = data;
-    struct t_preadv *c = &d->c.preadv;
-    ErlIOVec        *ev = &c->eiov;
-    size_t bytes_read = 0;
-    char buf[2*sizeof(Uint32)];
-    Uint32 offset, size;
-    DTRACE_INVOKE_SETUP(FILE_IPREAD);
-    
-    /* Read indirection header */
-    if (! efile_pread(&d->errInfo, (int) d->fd, c->offsets[0], 
-		      buf, sizeof(buf), &bytes_read)) {
-	goto error;
-    }
-    if (bytes_read != sizeof(buf)) goto done; /* eof */
-    size = get_int32(buf);
-    offset = get_int32(buf+4);
-    if (size > c->size) goto done; /* eof */
-    c->n = 1;
-    c->cnt = 0;
-    c->size = 0;
-    c->offsets[0] = offset;
-    if (! (ev->binv[0] = driver_alloc_binary(3*8))) {
-	d->errInfo.posix_errno = ENOMEM;
-	goto error;
-    }
-    ev->vsize = 1;
-    ev->iov[0].iov_len = 3*8;
-    ev->iov[0].iov_base = ev->binv[0]->orig_bytes;
-    ev->size = ev->iov[0].iov_len;
-    put_int64(offset, ev->iov[0].iov_base);
-    put_int64(size, ((char *)ev->iov[0].iov_base) + 2*8);
-    if (size == 0) {
-	put_int64(size, ((char *)ev->iov[0].iov_base) + 8);
-	goto done;
-    }
-    if (! (ev->binv[1] = driver_alloc_binary(size))) {
-	d->errInfo.posix_errno = ENOMEM;
-	goto error;
-    }
-    ev->vsize = 2;
-    ev->iov[1].iov_len = size;
-    ev->iov[1].iov_base = ev->binv[1]->orig_bytes;
-    /* Read data block */
-    d->invoke = invoke_preadv;
-    invoke_preadv(data);
-    DTRACE_INVOKE_RETURN(FILE_IPREAD);
-    return;
- error:
-    d->result_ok = 0;
-    d->again = 0;
-    DTRACE_INVOKE_RETURN(FILE_IPREAD);
-    return;
- done:
-    d->result_ok = !0;
-    d->again = 0;
-    DTRACE_INVOKE_RETURN(FILE_IPREAD);
-}
-
-/* invoke_writev and invoke_pwritev are the only thread functions that
- * access non-thread data i.e the port queue and a mutex in the port
- * structure that is used to lock the port queue.
- *
- * The port will normally not be terminated until the port queue is
- * empty, but if the port is killed, i.e., exit(Port, kill) is called,
- * it will terminate regardless of the port queue state. When the
- * port is invalid driver_peekq() returns NULL and set the size to -1,
- * and driver_sizeq() returns -1.
- */
-
-static void invoke_writev(void *data) {
-    struct t_data *d = (struct t_data *) data;
-    SysIOVec      *iov0;
-    SysIOVec      *iov;
-    int            iovlen;
-    int            iovcnt;
-    size_t         size;
-    size_t         p;
-    int            segment;
-    DTRACE_INVOKE_SETUP(FILE_WRITE);
-
-    segment = d->again && d->c.writev.size >= 2*FILE_SEGMENT_WRITE;
-    if (segment) {
-	size = FILE_SEGMENT_WRITE;
-    } else {
-	size = d->c.writev.size;
-    }
-
-    /* Copy the io vector to avoid locking the port que while writing,
-     * also, both we and efile_writev might/will change the SysIOVec
-     * when segmenting or due to partial write and we do not want to
-     * tamper with the actual queue that we get from driver_peekq
-     */
-    MUTEX_LOCK(d->c.writev.q_mtx); /* Lock before accessing the port queue */
-    iov0 = driver_peekq(d->c.writev.port, &iovlen);
-
-    /* Calculate iovcnt */
-    for (p = 0, iovcnt = 0;
-	 p < size && iovcnt < iovlen;
-	 p += iov0[iovcnt++].iov_len)
-	;
-    iov = EF_SAFE_ALLOC(sizeof(SysIOVec)*iovcnt);
-    memcpy(iov,iov0,iovcnt*sizeof(SysIOVec));
-    MUTEX_UNLOCK(d->c.writev.q_mtx);
-    /* Let go of lock until we deque from original vector */
-
-    if (iovlen > 0) {
-	ASSERT(iov[iovcnt-1].iov_len > p - size);
-	iov[iovcnt-1].iov_len -= p - size;
-	if (d->flags & EFILE_COMPRESSED) {
-	    int i, status = 1;
-	    for (i = 0; i < iovcnt; i++) {
-		if (iov[i].iov_base && iov[i].iov_len > 0) {
-		    /* Just in case, I do not know what gzwrite does
-		     * with errno.
-		     */
-		    errno = EINVAL; 
-		    status = erts_gzwrite((ErtsGzFile)d->fd,
-					  iov[i].iov_base,
-					  iov[i].iov_len) == iov[i].iov_len;
-		    if (! status) {
-			d->errInfo.posix_errno =
-			    d->errInfo.os_errno = errno; /* XXX Correct? */
-			break;
-		    }
-		}
-	    }
-	    d->result_ok = status;
-	} else {
-	    d->result_ok = efile_writev(&d->errInfo, 
-					d->flags, (int) d->fd,
-					iov, iovcnt);
-	}
-    } else if (iovlen == 0) {
-	d->result_ok = 1;
-    }
-    else { /* Port has terminated */
-	d->result_ok = 0;
-	d->errInfo.posix_errno = d->errInfo.os_errno = EINVAL;
-    }
-    EF_FREE(iov);
-
-    if (! d->result_ok) {
-	d->again = 0;
-	MUTEX_LOCK(d->c.writev.q_mtx);
-	driver_deq(d->c.writev.port, d->c.writev.size);
-	MUTEX_UNLOCK(d->c.writev.q_mtx);
-    } else {
-	if (! segment) {
-	    d->again = 0;
-	}
-	d->c.writev.size -= size;
-	TRACE_F(("w%lu", (unsigned long)size));
-	MUTEX_LOCK(d->c.writev.q_mtx);
-	driver_deq(d->c.writev.port, size);
-	MUTEX_UNLOCK(d->c.writev.q_mtx);
-    }
-
-
-    DTRACE_INVOKE_RETURN(FILE_WRITE);
-}
-
-static void invoke_pwd(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    DTRACE_INVOKE_SETUP(FILE_PWD);
-
-    d->again = 0;
-    d->result_ok = efile_getdcwd(&d->errInfo,d->drive, d->b+1,
-				 RESBUFSIZE-1);
-    DTRACE_INVOKE_RETURN(FILE_PWD);
-}
-
-static void invoke_readlink(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    char resbuf[RESBUFSIZE];	/* Result buffer. */
-    DTRACE_INVOKE_SETUP(FILE_READLINK);
-
-    d->again = 0;
-    d->result_ok = efile_readlink(&d->errInfo, d->b, resbuf+1,
-				  RESBUFSIZE-1);
-    if (d->result_ok != 0)
-	FILENAME_COPY((char *) d->b + 1, resbuf+1);
-    DTRACE_INVOKE_RETURN(FILE_READLINK);
-}
-
-static void invoke_altname(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    char resbuf[RESBUFSIZE];	/* Result buffer. */
-    DTRACE_INVOKE_SETUP(FILE_ALTNAME);
-
-    d->again = 0;
-    d->result_ok = efile_altname(&d->errInfo, d->b, resbuf+1,
-				  RESBUFSIZE-1);
-    if (d->result_ok != 0)
-	FILENAME_COPY((char *) d->b + 1, resbuf+1);
-    DTRACE_INVOKE_RETURN(FILE_ALTNAME);
-}
-
-static void invoke_pwritev(void *data) {
-    struct t_data* const d = (struct t_data *) data;
-    struct t_pwritev * const c = &d->c.pwritev;
-    SysIOVec         *iov0;
-    SysIOVec         *iov;
-    int               iovlen;
-    int               iovcnt;
-    size_t            p;
-    int               segment;
-    size_t            size, write_size, written;
-    DTRACE_INVOKE_SETUP(FILE_PWRITEV);
-
-    segment = d->again && c->size >= 2*FILE_SEGMENT_WRITE;
-    if (segment) {
-	size = FILE_SEGMENT_WRITE;
-    } else {
-	size = c->size;
-    }
-    d->result_ok = !0;
-    p = 0;
-    /* Lock the queue just for a while, we don't want it locked during write */
-    MUTEX_LOCK(c->q_mtx);
-    iov0 = driver_peekq(c->port, &iovlen);
-    iov = EF_SAFE_ALLOC(sizeof(SysIOVec)*iovlen);
-    memcpy(iov,iov0,sizeof(SysIOVec)*iovlen);
-    MUTEX_UNLOCK(c->q_mtx);
-
-    if (iovlen < 0)
-	goto error; /* Port terminated */
-    for (iovcnt = 0, written = 0;
-	 c->cnt < c->n && iovcnt < iovlen && written < size;
-	 c->cnt++) {
-	int chop;
-	write_size = c->specs[c->cnt].size;
-	if (iov[iovcnt].iov_len - p < write_size) {
-	    goto error;
-	}
-	chop = segment && written + write_size >= 2*FILE_SEGMENT_WRITE;
-	if (chop) {
-	    ASSERT(written < FILE_SEGMENT_WRITE);
-	    write_size = FILE_SEGMENT_WRITE + FILE_SEGMENT_WRITE/2 
-		- written;
-	}
-	d->result_ok = efile_pwrite(&d->errInfo, (int) d->fd,
-				    (char *)(iov[iovcnt].iov_base) + p,
-				    write_size,
-				    c->specs[c->cnt].offset);
-	if (! d->result_ok) {
-	    d->again = 0;
-	    goto deq_error;
-	}
-	written += write_size; 
-	c->size -= write_size;
-	if (chop) { 
-	    c->specs[c->cnt].offset += write_size;
-	    c->specs[c->cnt].size -= write_size;
-	    /* Schedule out (d->again != 0) */
-	    break;
-	}
-	/* Move forward in buffer */
-	p += write_size;
-	ASSERT(iov[iovcnt].iov_len >= p);
-	if (iov[iovcnt].iov_len == p) {
-	    /* Move to next iov[], we trust that it is not a 
-	     * zero length vector, and thereby depend on that
-	     * such are not queued.
-	     */
-	    iovcnt++; p = 0;
-	}
-    }
-    if (! segment) {
-	if (c->cnt != c->n) {
-	    /* Mismatch between number of 
-	     * pos/size specs vs number of queued buffers .
-	     */
-	error:
-	    d->errInfo.posix_errno = EINVAL;
-	    d->result_ok = 0;
-	    d->again = 0;
-	deq_error:
-	    MUTEX_LOCK(c->q_mtx);
-	    driver_deq(c->port, c->size);
-	    MUTEX_UNLOCK(c->q_mtx);
-
-	    goto done;
-	} else {
-	    ASSERT(written == size);
-	    d->again = 0;
-	}
-    } else {
-      ASSERT(written >= FILE_SEGMENT_WRITE);
-    }
-      
-    MUTEX_LOCK(c->q_mtx);
-    driver_deq(c->port, written);
-    MUTEX_UNLOCK(c->q_mtx);
- done:
-    EF_FREE(iov); /* Free our copy of the vector, nothing to restore */
-    
-    DTRACE_INVOKE_RETURN(FILE_PWRITEV);
-}
-
-static void invoke_flstat(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-
-    DTRACE3(efile_drv_int_entry, d->sched_i1, d->sched_i2,
-            d->command == FILE_LSTAT ? FILE_LSTAT : FILE_FSTAT);
-    d->again = 0;
-    d->result_ok = efile_fileinfo(&d->errInfo, &d->info,
-				  d->b, d->command == FILE_LSTAT);
-    DTRACE3(efile_drv_int_entry, d->sched_i1, d->sched_i2,
-            d->command == FILE_LSTAT ? FILE_LSTAT : FILE_FSTAT);
-    gcc_optimizer_hack++;
-}
-
-static void invoke_link(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    char *name = d->b;
-    char *new_name;
-    DTRACE_INVOKE_SETUP(FILE_LINK);
-
-    d->again = 0;
-    new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE;
-    d->result_ok = efile_link(&d->errInfo, name, new_name);
-    DTRACE_INVOKE_RETURN(FILE_LINK);
-}
-
-static void invoke_symlink(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    char *name = d->b;
-    char *new_name;
-    DTRACE_INVOKE_SETUP(FILE_SYMLINK);
-
-    d->again = 0;
-    new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE;
-    d->result_ok = efile_symlink(&d->errInfo, name, new_name);
-    DTRACE_INVOKE_RETURN(FILE_SYMLINK);
-}
-
-static void invoke_rename(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    char *name = d->b;
-    char *new_name;
-    DTRACE_INVOKE_SETUP(FILE_RENAME);
-
-    d->again = 0;
-    new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE;
-    d->result_ok = efile_rename(&d->errInfo, name, new_name);
-    DTRACE_INVOKE_RETURN(FILE_RENAME);
-}
-
-static void invoke_write_info(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    DTRACE_INVOKE_SETUP(FILE_WRITE_INFO);
-
-    d->again = 0;
-    d->result_ok = efile_write_info(&d->errInfo, &d->info, d->b);
-    DTRACE_INVOKE_RETURN(FILE_WRITE_INFO);
-}
-
-static void invoke_lseek(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    int status;
-    DTRACE_INVOKE_SETUP(FILE_LSEEK);
-
-    d->again = 0;
-    if (d->flags & EFILE_COMPRESSED) {
-	int offset = (int) d->c.lseek.offset;
-	
-	if (offset != d->c.lseek.offset) {
-	    d->errInfo.posix_errno = EINVAL;
-	    status = 0;
-	} else {
-	    d->c.lseek.location = erts_gzseek((ErtsGzFile)d->fd,
-					      offset, d->c.lseek.origin);
-	    if (d->c.lseek.location == -1) {
-		d->errInfo.posix_errno = errno;
-		status = 0;
-	    } else {
-		status = 1;
-	    }
-	}
-    } else {
-	status = efile_seek(&d->errInfo, (int) d->fd, 
-			    d->c.lseek.offset, d->c.lseek.origin,
-			    &d->c.lseek.location);
-    }
-    d->result_ok = status;
-    DTRACE_INVOKE_RETURN(FILE_LSEEK);
-}
-
-static void invoke_readdir(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    char *p = NULL;
-    size_t file_bs;
-    size_t n = 0, total = 0;
-    struct t_readdir_buf *b = NULL;
-    int res = 0;
-    DTRACE_INVOKE_SETUP(FILE_READDIR);
-
-    d->again = 0;
-    d->errInfo.posix_errno = 0;
-
-    do {
-	total   = READDIR_BUFSIZE;
-	n       = 1;
-	b       = EF_SAFE_ALLOC(sizeof(struct t_readdir_buf));
-	b->next = NULL;
-	
-	if (d->c.read_dir.last_buf) {
-	    d->c.read_dir.last_buf->next = b;
-	} else {
-	    d->c.read_dir.first_buf = b;
-	}
-	d->c.read_dir.last_buf = b;
-
-	p       = &b->buf[0];
-	p[0]    = FILE_RESP_LFNAME;
-	file_bs = READDIR_BUFSIZE - n;
-
-	do {
-	    res = efile_readdir(&d->errInfo, d->b, &d->dir_handle, p + n + 2, &file_bs);
-
-	    if (res) {
-		put_int16((Uint16)file_bs, p + n);
-		n += 2 + file_bs;
-		file_bs = READDIR_BUFSIZE - n;
-	    }
-	} while( res && ((total - n - 2) >= MAXPATHLEN*FILENAME_CHARSIZE));
-
-	b->n = n;
-    } while(res);
-
-    d->result_ok = (d->errInfo.posix_errno == 0);
-    DTRACE_INVOKE_RETURN(FILE_READDIR);
-}
-
-static void invoke_open(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    int status = 1;		/* Status of open call. */
-    DTRACE_INVOKE_SETUP(FILE_OPEN);
-
-    d->again = 0;
-    if ((d->flags & EFILE_COMPRESSED) == 0) {
-	int fd;
-	status = efile_openfile(&d->errInfo, d->b, d->flags, &fd, NULL);
-	d->fd = fd;
-    } else {
-	char* mode = NULL;
-
-	if (((d->flags & (EFILE_MODE_READ_WRITE)) == EFILE_MODE_READ_WRITE) ||
-	    (d->flags & EFILE_MODE_APPEND)) {
-	    status = 0;
-	    d->errInfo.posix_errno = EINVAL;
-	} else {
-	    status = efile_may_openfile(&d->errInfo, d->b);
-	    if (status || (d->errInfo.posix_errno != EISDIR)) {
-		mode = (d->flags & EFILE_MODE_READ) ? "rb" : "wb";
-		d->fd = (SWord) erts_gzopen(d->b, mode);
-		if ((ErtsGzFile)d->fd) {
-		    status = 1;
-		} else {
-		    if (errno == 0) {
-			errno = ENOMEM;
-		    }
-		    d->errInfo.posix_errno = errno;
-		    status = 0;
-		}
-	    }
-	}
-    }
-
-    d->result_ok = status;
-    if (!status) {
-        d->fd = FILE_FD_INVALID;
-    }
-    DTRACE_INVOKE_RETURN(FILE_OPEN);
-}
-
-static void invoke_fadvise(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    int fd = (int) d->fd;
-    off_t offset = (off_t) d->c.fadvise.offset;
-    off_t length = (off_t) d->c.fadvise.length;
-    int advise = (int) d->c.fadvise.advise;
-    DTRACE_INVOKE_SETUP(FILE_FADVISE);
-
-    d->again = 0;
-    d->result_ok = efile_fadvise(&d->errInfo, fd, offset, length, advise);
-    DTRACE_INVOKE_RETURN(FILE_FADVISE);
-}
-
-#ifdef HAVE_SENDFILE
-static void invoke_sendfile(void *data)
-{
-    struct t_data *d = (struct t_data *)data;
-    int fd = d->fd;
-    int out_fd = (int)d->c.sendfile.out_fd;
-    Uint64 nbytes = d->c.sendfile.nbytes;
-    int result = 0;
-    d->again = 0;
-
-    result = efile_sendfile(&d->errInfo, fd, out_fd, &d->c.sendfile.offset, &nbytes, NULL);
-
-    d->c.sendfile.written += nbytes;
-
-    if (result == 1 || (result == 0 && USE_THRDS_FOR_SENDFILE(d))) {
-      d->result_ok = 0;
-    } else if (result == 0 && (d->errInfo.posix_errno == EAGAIN
-				 || d->errInfo.posix_errno == EINTR)) {
-      if ((d->c.sendfile.nbytes - nbytes) != 0) {
-	d->result_ok = 1;
-	if (d->c.sendfile.nbytes != 0)
-	  d->c.sendfile.nbytes -= nbytes;
-      } else if (nbytes == 0 && d->c.sendfile.nbytes == 0) {
-	d->result_ok = 1;
-      } else
-	d->result_ok = 0;
-    } else {
-	d->result_ok = -1;
-    }
-}
-
-static void free_sendfile(void *data) {
-    struct t_data *d = (struct t_data *)data;
-    if (USE_THRDS_FOR_SENDFILE(d)) {
-	SET_NONBLOCKING(d->c.sendfile.out_fd);
-    } else {
-	MUTEX_LOCK(d->c.sendfile.q_mtx);
-	driver_deq(d->c.sendfile.port,1);
-	MUTEX_UNLOCK(d->c.sendfile.q_mtx);
-	driver_select(d->c.sendfile.port, (ErlDrvEvent)(long)d->c.sendfile.out_fd,
-                      ERL_DRV_USE_NO_CALLBACK|ERL_DRV_WRITE, 0);
-    }
-    EF_FREE(data);
-}
-
-static void file_ready_output(ErlDrvData data, ErlDrvEvent event)
-{
-    file_descriptor* fd = (file_descriptor*) data;
-
-    switch (fd->d->command) {
-    case FILE_SENDFILE:
-	driver_select(fd->d->c.sendfile.port, event,
-		      (int)ERL_DRV_WRITE,(int) 0);
-	invoke_sendfile((void *)fd->d);
-	file_async_ready(data, (ErlDrvThreadData)fd->d);
-	break;
-    default:
-	break;
-    }
-}
-
-static void file_stop_select(ErlDrvEvent event, void* _)
-{
-
-}
-
-static int flush_sendfile(file_descriptor *desc,void *_) {
-    if (desc->sendfile_state == sending) {
-	desc->d->result_ok = -1;
-	desc->d->errInfo.posix_errno = ECONNABORTED;
-	file_async_ready((ErlDrvData)desc,(ErlDrvThreadData)desc->d);
-    }
-    return 1;
-}
-#endif /* HAVE_SENDFILE */
-
-
-static void invoke_fallocate(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    int fd = (int) d->fd;
-    Sint64 offset = d->c.fallocate.offset;
-    Sint64 length = d->c.fallocate.length;
-
-    d->again = 0;
-    d->result_ok = efile_fallocate(&d->errInfo, fd, offset, length);
-}
-
-static void free_readdir(void *data)
-{
-    struct t_data *d = (struct t_data *) data;
-    struct t_readdir_buf *b1 = d->c.read_dir.first_buf;
-
-    while (b1) {
-	struct t_readdir_buf *b2 = b1;
-	b1 = b1->next;
-	EF_FREE(b2);
-    }
-    EF_FREE(d);
-}
-
-
-
-static void try_free_read_bin(file_descriptor *desc) {
-    if ((desc->read_size == 0)
-	&& (desc->read_offset >= desc->read_binp->orig_size)) {
-	ASSERT(desc->read_offset == desc->read_binp->orig_size);
-	driver_free_binary(desc->read_binp);
-	desc->read_binp = NULL;
-	desc->read_offset = 0;
-	desc->read_size = 0;
-    }
-}
-
-
-
-static int try_again(file_descriptor *desc, struct t_data *d) {
-    if (! d->again)
-	return 0;
-    if (desc->timer_state != timer_idle) {
-	driver_cancel_timer(desc->port);
-    }
-    desc->timer_state = timer_again;
-    desc->invoke = d->invoke;
-    desc->d = d;
-    desc->free = d->free;
-    driver_set_timer(desc->port, 0L);
-    return !0;
-}
-
-
-
-static void cq_execute(file_descriptor *desc) {
-    struct t_data *d;
-    register void *void_ptr; /* Soft cast variable */
-    if (desc->timer_state == timer_again)
-	return;
-#ifdef HAVE_SENDFILE
-    if (desc->sendfile_state == sending)
-	return;
-#endif
-    if (! (d = cq_deq(desc)))
-	return;
-    TRACE_F(("x%i", (int) d->command));
-    d->again = sys_info.async_threads == 0;
-    DRIVER_ASYNC(d->level, desc, d->invoke, void_ptr=d, d->free);
-}
-
-static struct t_data *async_write(file_descriptor *desc, int *errp,
-		       int reply, Uint32 reply_size
-#ifdef USE_VM_PROBES
-		       ,Sint64 *dt_i1, Sint64 *dt_i2, Sint64 *dt_i3
-#endif
-) {
-    struct t_data *d;
-    if (! (d = EF_ALLOC(sizeof(struct t_data) - 1))) {
-	if (errp) *errp = ENOMEM;
-	return NULL;
-    }
-    TRACE_F(("w%lu", (unsigned long)desc->write_buffered));
-    d->command = FILE_WRITE;
-    d->fd = desc->fd;
-    d->flags = desc->flags;
-    d->c.writev.port = desc->port;
-    d->c.writev.q_mtx = desc->q_mtx;
-    d->c.writev.size = desc->write_buffered;
-#ifdef USE_VM_PROBES
-    if (dt_i1 != NULL) {
-        *dt_i1 = d->fd;
-        *dt_i2 = d->flags;
-        *dt_i3 = d->c.writev.size;
-    }
-#endif
-    d->reply = reply;
-    d->c.writev.reply_size = reply_size;
-    d->invoke = invoke_writev;
-    d->free = free_data;
-    d->level = 1;
-    cq_enq(desc, d);
-    desc->write_buffered = 0;
-    return d;
-}
-
-static int flush_write(file_descriptor *desc, int *errp
-#ifdef USE_VM_PROBES
-                       , dt_private *dt_priv, char *dt_utag
-#endif
-) {
-    int    result = 0;
-#ifdef USE_VM_PROBES
-    Sint64 dt_i1 = 0, dt_i2 = 0, dt_i3 = 0;
-#endif
-    struct t_data *d = NULL;
-
-    MUTEX_LOCK(desc->q_mtx);
-    if (desc->write_buffered > 0) {
-	if ((d = async_write(desc, errp, 0, 0
-#ifdef USE_VM_PROBES
-			     ,&dt_i1, &dt_i2, &dt_i3
-#endif
-			     )) == NULL) {
-            result = -1;
-        }
-    }
-    MUTEX_UNLOCK(desc->q_mtx);
-#ifdef USE_VM_PROBES
-    if (d != NULL) {
-        d->sched_i1 = dt_priv->thread_num;
-        d->sched_i2 = dt_priv->tag;
-        d->sched_utag[0] = '\0';
-        if (dt_utag != NULL) {
-            if (dt_utag[0] == '\0') {
-                dt_utag = NULL;
-            } else {
-                strncpy(d->sched_utag, dt_utag, sizeof(d->sched_utag) - 1);
-                d->sched_utag[sizeof(d->sched_utag) - 1] = '\0';
-            }
-        }
-        DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag++,
-                 dt_utag, FILE_WRITE,
-                 NULL, NULL, dt_i1, dt_i2, dt_i3, 0, desc->port_str);
-    }
-#endif /* USE_VM_PROBES */
-    return result;
-}
-
-static int check_write_error(file_descriptor *desc, int *errp) {
-    if (desc->write_error) {
-	if (errp) *errp = desc->write_errInfo.posix_errno;
-	desc->write_error = 0;
-	return -1;
-    }
-    return 0;
-}
-
-static int flush_write_check_error(file_descriptor *desc, int *errp
-#ifdef USE_VM_PROBES
-                                   , dt_private *dt_priv, char *dt_utag
-#endif
-				   ) {
-    int r;
-    if ( (r = flush_write(desc, errp
-#ifdef USE_VM_PROBES
-			  , dt_priv, dt_utag
-#endif
-			  )) != 0) {
-	check_write_error(desc, NULL);
-	return r;
-    } else {
-	return check_write_error(desc, errp);
-    }
-}
-
-static struct t_data *async_lseek(file_descriptor *desc, int *errp, int reply,
-				  Sint64 offset, int origin
-#ifdef USE_VM_PROBES
-				  , Sint64 *dt_i1, Sint64 *dt_i2, Sint64 *dt_i3
-#endif
-				  ) {
-    struct t_data *d;
-    if (! (d = EF_ALLOC(sizeof(struct t_data)))) {
-	*errp = ENOMEM;
-	return NULL;
-    }
-    d->flags = desc->flags;
-    d->fd = desc->fd;
-    d->command = FILE_LSEEK;
-    d->reply = reply;
-    d->c.lseek.offset = offset;
-    d->c.lseek.origin = origin;
-#ifdef USE_VM_PROBES
-    if (dt_i1 != NULL) {
-        *dt_i1 = d->fd;
-        *dt_i2 = d->c.lseek.offset;
-        *dt_i3 = d->c.lseek.origin;
-    }
-#endif
-    d->invoke = invoke_lseek;
-    d->free = free_data;
-    d->level = 1;
-    cq_enq(desc, d);
-    return d;
-}
-
-static void flush_read(file_descriptor *desc) {
-    desc->read_offset = 0;
-    desc->read_size = 0;
-    if (desc->read_binp) {
-	driver_free_binary(desc->read_binp);
-	desc->read_binp = NULL;
-    }
-}
-
-static int lseek_flush_read(file_descriptor *desc, int *errp
-#ifdef USE_VM_PROBES
-			    ,dt_private *dt_priv, char *dt_utag
-#endif
-			    ) {
-    int r = 0;
-    size_t read_size = desc->read_size;
-#ifdef USE_VM_PROBES
-    Sint64 dt_i1 = 0, dt_i2 = 0, dt_i3 = 0;
-#endif
-    struct t_data *d;
-
-    flush_read(desc);
-    if (read_size != 0) {
-	if ((d = async_lseek(desc, errp, 0,
-                             -((ssize_t)read_size), EFILE_SEEK_CUR
-#ifdef USE_VM_PROBES
-			     , &dt_i1, &dt_i2, &dt_i3
-#endif
-			     )) == NULL) {
-            r = -1;
-        } else {
-#ifdef USE_VM_PROBES
-            d->sched_i1 = dt_priv->thread_num;
-            d->sched_i2 = dt_priv->tag;
-            d->sched_utag[0] = '\0';
-            if (dt_utag != NULL) {
-                if (dt_utag[0] == '\0') {
-                    dt_utag = NULL;
-                } else {
-                    strncpy(d->sched_utag, dt_utag, sizeof(d->sched_utag) - 1);
-                    d->sched_utag[sizeof(d->sched_utag) - 1] = '\0';
-                }
-            }
-            DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag++,
-                     dt_utag, FILE_LSEEK,
-                     NULL, NULL, dt_i1, dt_i2, dt_i3, 0, desc->port_str);
-#endif /* USE_VM_PROBES */
-        }
-    }
-    return r;
-}
-
-
-/*********************************************************************
- * Driver entry point -> stop
- * The close has to be scheduled on async thread, so that currently active
- * async operation does not suddenly have the ground disappearing under their feet...
- */
-static void 
-file_stop(ErlDrvData e)
-{
-    file_descriptor* desc = (file_descriptor*)e;
-
-    TRACE_C('p');
-
-    IF_THRDS {
-	flush_read(desc);
-	if (desc->fd != FILE_FD_INVALID) {
-	    struct t_data *d = EF_SAFE_ALLOC(sizeof(struct t_data));
-	    d->command = FILE_CLOSE_ON_PORT_EXIT;
-	    d->reply = !0;
-	    d->fd = desc->fd;
-	    d->flags = desc->flags;
-	    d->invoke = invoke_close;
-	    d->free = free_data;
-	    d->level = 2;
-	    d->data_to_free = (void *) desc;
-	    cq_enq(desc, d);
-	    desc->fd = FILE_FD_INVALID;
-	    desc->flags = 0;
-	    cq_execute(desc);
-	} else {
-	    EF_FREE(desc);
-	}
-    } else {
-	if (desc->fd != FILE_FD_INVALID) {
-	    do_close(desc->flags, desc->fd);
-	    desc->fd = FILE_FD_INVALID;
-	    desc->flags = 0;
-	}
-	if (desc->read_binp) {
-	    driver_free_binary(desc->read_binp);
-	}
-	EF_FREE(desc);
-    }
-}
-
-/*********************************************************************
- * Driver entry point -> ready_async
- */
-static void 
-file_async_ready(ErlDrvData e, ErlDrvThreadData data)
-{
-    file_descriptor *desc = (file_descriptor*)e;
-    struct t_data *d = (struct t_data *) data;
-    char header[5];		/* result code + count */
-    char resbuf[RESBUFSIZE];	/* Result buffer. */
-#ifdef  USE_VM_PROBES
-    int sched_i1 = d->sched_i1, sched_i2 = d->sched_i2, command = d->command,
-        result_ok = d->result_ok,
-        posix_errno = d->result_ok ? 0 : d->errInfo.posix_errno;
-    DTRACE_CHARBUF(sched_utag, DTRACE_EFILE_BUFSIZ+1);
-
-    sched_utag[0] = '\0';
-    if (DTRACE_ENABLED(efile_drv_return)) {
-        strncpy(sched_utag, d->sched_utag, DTRACE_EFILE_BUFSIZ);
-        sched_utag[DTRACE_EFILE_BUFSIZ] = '\0';
-    }
-#endif  /* USE_VM_PROBES */
-
-    TRACE_C('r');
-
-    if (try_again(desc, d)) {
-        /* DTRACE TODO: what kind of probe makes sense here? */
-	return;
-    }
-
-    switch (d->command)
-    {
-    case FILE_READ:
-	if (!d->result_ok) {
-	    reply_error(desc, &d->errInfo);
-	} else {
-	    size_t available_bytes = 
-		d->c.read.bin_offset + d->c.read.bin_size - desc->read_offset;
-	    if (available_bytes < d->c.read.size) {
-		d->c.read.size = available_bytes;
-	    }
-	    TRACE_C('D');
-	    reply_data(desc, d->c.read.binp, 
-		       desc->read_offset, d->c.read.size);
-	    desc->read_offset += d->c.read.size;
-	    desc->read_size = 
-		d->c.read.bin_offset + d->c.read.bin_size - desc->read_offset;
-	    try_free_read_bin(desc);
-	}
-	free_read(data);
-	break;
-      case FILE_READ_LINE:
-	  /* The read_line structure differs from the read structure.
-	     The data->read_offset and d->c.read_line.read_offset are copies, as are
-	     data->read_size and d->c.read_line.read_size
-             The read_line function does not kniow in advance how large the binary has to be,
-	     why new allocation (but not reallocation of the old binary, for obvious reasons) 
-	     may happen in the worker thread. */
-	if (!d->result_ok) {
-	    reply_error(desc, &d->errInfo);
-	} else {
-	    size_t len = d->c.read_line.nl_pos - d->c.read_line.read_offset;
-	    TRACE_C('L');
-	    reply_data(desc, d->c.read_line.binp, 
-		       d->c.read_line.read_offset, len);
-	    desc->read_offset = d->c.read_line.read_offset + d->c.read_line.nl_skip + len;
-	    desc->read_size = 
-		d->c.read_line.read_size - d->c.read_line.nl_skip - len;
-	    if (desc->read_binp != d->c.read_line.binp) { /* New binary allocated */
-		driver_free_binary(desc->read_binp);
-		desc->read_binp =  d->c.read_line.binp;
-		driver_binary_inc_refc(desc->read_binp);
-	    }
-#if !ALWAYS_READ_LINE_AHEAD
-	    ASSERT(desc->read_bufsize > 0 || desc->read_size == 0);
-	    if (desc->read_bufsize == 0) {
-		desc->read_offset = desc->read_binp->orig_size; /* triggers cleanup */
-	    }
-#endif
-	    try_free_read_bin(desc);
-	}
-	free_read_line(data);
-	break;
-      case FILE_READ_FILE:
-	if (!d->result_ok)
-	    reply_error(desc, &d->errInfo);
-	else {
-	    header[0] = FILE_RESP_ALL_DATA;
-	    TRACE_C('R');
-	    driver_output_binary(desc->port, header, 1,
-				 d->c.read_file.binp,
-				 0, d->c.read_file.offset);
-	}
-	free_read_file(data);
-	break;
-      case FILE_WRITE:
-	  if (d->reply) {
-	      if (! d->result_ok) {
-		  reply_error(desc, &d->errInfo);
-	      } else {
-		  reply_Uint(desc, d->c.writev.reply_size);
-	      }
-	  } else {
-	      if (! d->result_ok) {
-		  desc->write_error = !0;
-		  desc->write_errInfo = d->errInfo;
-	      }
-	  }
-	  free_data(data);
-	  break;
-      case FILE_LSEEK:
-	  if (d->reply) {
-	      if (d->result_ok)
-		  reply_Sint64(desc, d->c.lseek.location);
-	      else
-		  reply_error(desc, &d->errInfo);
-	  }
-	  free_data(data);
-	  break;
-      case FILE_MKDIR:
-      case FILE_RMDIR:
-      case FILE_CHDIR:
-      case FILE_DELETE:
-      case FILE_FDATASYNC:
-      case FILE_FSYNC:
-      case FILE_TRUNCATE:
-      case FILE_LINK:
-      case FILE_SYMLINK:
-      case FILE_RENAME:
-      case FILE_WRITE_INFO:
-      case FILE_FADVISE:
-      case FILE_FALLOCATE:
-	reply(desc, d->result_ok, &d->errInfo);
-	free_data(data);
-	break;
-      case FILE_ALTNAME:
-      case FILE_PWD:
-      case FILE_READLINK:
-        {
-	    int length;
-	    char *resbuf = d->b;
-
-	    if (!d->result_ok)
-		reply_error(desc, &d->errInfo);
-	    else {
-		resbuf[0] = FILE_RESP_FNAME;
-		length = 1+FILENAME_BYTELEN((char*) resbuf+1);
-		TRACE_C('R');
-		driver_output2(desc->port, resbuf, 1, resbuf+1, length-1);
-	    }
-	    free_data(data);
-	    break;
-	}
-      case FILE_OPEN:
-	if (!d->result_ok) {
-	    reply_error(desc, &d->errInfo);
-	} else {
-	    ASSERT(d->is_fd_unused);
-	    desc->fd = d->fd;
-	    desc->flags = d->flags;
-	    d->is_fd_unused = 0;
-	    reply_Uint(desc, d->fd);
-	}
-	free_data(data);
-	break;
-      case FILE_FSTAT:
-      case FILE_LSTAT:
-        {
-	    if (d->result_ok) {
-		resbuf[0] = FILE_RESP_INFO;
-
-		put_int32(d->info.size_high,         &resbuf[1 + ( 0 * 4)]);
-		put_int32(d->info.size_low,          &resbuf[1 + ( 1 * 4)]);
-		put_int32(d->info.type,              &resbuf[1 + ( 2 * 4)]);
-
-		/* Note 64 bit indexing in resbuf here */
-		put_int64(d->info.accessTime,        &resbuf[1 + ( 3 * 4)]);
-		put_int64(d->info.modifyTime,        &resbuf[1 + ( 5 * 4)]);
-		put_int64(d->info.cTime,             &resbuf[1 + ( 7 * 4)]);
-
-		put_int32(d->info.mode,              &resbuf[1 + ( 9 * 4)]);
-		put_int32(d->info.links,             &resbuf[1 + (10 * 4)]);
-		put_int32(d->info.major_device,      &resbuf[1 + (11 * 4)]);
-		put_int32(d->info.minor_device,      &resbuf[1 + (12 * 4)]);
-		put_int32(d->info.inode,             &resbuf[1 + (13 * 4)]);
-		put_int32(d->info.uid,               &resbuf[1 + (14 * 4)]);
-		put_int32(d->info.gid,               &resbuf[1 + (15 * 4)]);
-		put_int32(d->info.access,            &resbuf[1 + (16 * 4)]);
-
-#define RESULT_SIZE (1 + (17 * 4))
-		TRACE_C('R');
-		driver_output2(desc->port, resbuf, RESULT_SIZE, NULL, 0);
-#undef RESULT_SIZE
-	    } else
-		reply_error(desc, &d->errInfo);
-	}
-	free_data(data);
-	break;
-      case FILE_READDIR:
-	if (!d->result_ok) {
-	    reply_error(desc, &d->errInfo);
-	} else {
-	    struct t_readdir_buf *b1 = d->c.read_dir.first_buf;
-	    char   op = FILE_RESP_LFNAME;
-
-	    TRACE_C('R');
-	    ASSERT(b1);
-
-	    while (b1) {
-		struct t_readdir_buf *b2 = b1;
-		char *p = &b1->buf[0];
-		driver_output2(desc->port, p, 1, p + 1, b1->n - 1);
-		b1 = b1->next;
-		EF_FREE(b2);
-	    }
-	    driver_output2(desc->port, &op, 1, NULL, 0);
-    
-	    d->c.read_dir.first_buf = NULL;
-	    d->c.read_dir.last_buf = NULL;
-	}
-	free_readdir(data);
-	break;
-      case FILE_CLOSE:
-	  if (d->reply) {
-	      TRACE_C('K');
-	      reply_ok(desc);
-#ifdef USE_VM_PROBES
-              result_ok = 1;
-#endif
-	  }
-	  free_data(data);
-	  break;
-      case FILE_PWRITEV:
-	  if (!d->result_ok) {
-	      reply_Uint_error(desc, d->c.pwritev.cnt, &d->errInfo);
-	  } else {
-	      reply_Uint(desc, d->c.pwritev.n);
-	  }
-	  free_data(data);
-	  break;
-      case FILE_PREADV:
-	  if (!d->result_ok) {
-	      reply_error(desc, &d->errInfo);
-	  } else {
-	      reply_ev(desc, FILE_RESP_LDATA, &d->c.preadv.eiov);
-	  }
-	  free_preadv(data);
-	  break;
-      case FILE_IPREAD:
-	  if (!d->result_ok) {
-	      reply_error(desc, &d->errInfo);
-	  } else if (!d->c.preadv.eiov.vsize) {
-	      reply_eof(desc);
-	  } else {
-	      reply_ev(desc, FILE_RESP_N2DATA, &d->c.preadv.eiov);
-	  }
-	  free_preadv(data);
-	  break;
-#ifdef HAVE_SENDFILE
-      case FILE_SENDFILE:
-	  if (d->result_ok == -1) {
-	      if (d->errInfo.posix_errno == ECONNRESET ||
-		  d->errInfo.posix_errno == ENOTCONN ||
-		  d->errInfo.posix_errno == EPIPE)
-		  reply_string_error(desc,"closed");
-	      else
-		  reply_error(desc, &d->errInfo);
-	      desc->sendfile_state = not_sending;
-	      free_sendfile(data);
-	  } else if (d->result_ok == 0) {
-	      reply_Sint64(desc, d->c.sendfile.written);
-	      desc->sendfile_state = not_sending;
-	      free_sendfile(data);
-	  } else if (d->result_ok == 1) { /* If we are using select to send the rest of the data */
-	      desc->sendfile_state = sending;
-	      desc->d = d;
-	      driver_select(desc->port, (ErlDrvEvent)(long)d->c.sendfile.out_fd,
-			    ERL_DRV_USE|ERL_DRV_WRITE, 1);
-	  }
-	  break;
-#endif
-      case FILE_CLOSE_ON_PORT_EXIT:
-	  /* See file_stop. However this is never invoked after the port is killed. */
-	  free_data(data);
-	  desc = NULL;
-	  /* This is it for this port, so just send dtrace and return, avoid doing anything to the freed data */
-	  DTRACE6(efile_drv_return, sched_i1, sched_i2, sched_utag,
-		  command, result_ok, posix_errno);
-	  return;
-      default:
-	abort();
-    }
-    DTRACE6(efile_drv_return, sched_i1, sched_i2, sched_utag,
-            command, result_ok, posix_errno);
-    if (desc->write_buffered != 0 && desc->timer_state == timer_idle ) {
-	desc->timer_state = timer_write;
-	driver_set_timer(desc->port, desc->write_delay);
-    }
-    cq_execute(desc);
-
-}
-
-
-/*********************************************************************
- * Driver entry point -> output
- */
-static void 
-file_output(ErlDrvData e, char* buf, ErlDrvSizeT count)
-{
-    file_descriptor* desc = (file_descriptor*)e;
-    Efile_error errInfo;	/* The error codes for the last operation. */
-    Sint fd;			/* The file descriptor for this port, if any,
-				 * -1 if none.
-				 */
-    char* name;			/* Points to the filename in buf. */
-    int command;
-    struct t_data *d = NULL;
-#ifdef  USE_VM_PROBES
-    char *dt_utag = NULL;
-    char *dt_s1 = NULL, *dt_s2 = NULL;
-    Sint64 dt_i1 = 0;
-    Sint64 dt_i2 = 0;
-    Sint64 dt_i3 = 0;
-    Sint64 dt_i4 = 0;
-    dt_private *dt_priv = get_dt_private(0);
-#endif  /* USE_VM_PROBES */
-
-    TRACE_C('o');
-
-    fd  = desc->fd;
-    name = buf+1;
-    command = *(uchar*)buf++;
-
-    switch(command) {
-
-    case FILE_MKDIR:
-    {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE);
-	
-	FILENAME_COPY(d->b, name);
-#ifdef USE_VM_PROBES
-	dt_s1 = d->b;
-	dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
-#endif
-	d->command = command;
-	d->invoke = invoke_mkdir;
-	d->free = free_data;
-	d->level = 2;
-	goto done;
-    }
-    case FILE_RMDIR:
-    {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE);
-	
-	FILENAME_COPY(d->b, name);
-#ifdef USE_VM_PROBES
-	dt_s1 = d->b;
-	dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
-#endif
-	d->command = command;
-	d->invoke = invoke_rmdir;
-	d->free = free_data;
-	d->level = 2;
-	goto done;
-    }
-    case FILE_DELETE:
-    {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE);
-	
-	FILENAME_COPY(d->b, name);
-#ifdef USE_VM_PROBES
-	dt_s1 = d->b;
-	dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
-#endif
-	d->command = command;
-	d->invoke = invoke_delete_file;
-	d->free = free_data;
-	d->level = 2;
-	goto done;
-    }
-    case FILE_RENAME:
-	{
-	    char* new_name;
-	    int namelen = FILENAME_BYTELEN(name)+FILENAME_CHARSIZE;
-	    new_name = name+namelen;
-	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1
-			      + namelen
-			      + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE);
-	
-	    FILENAME_COPY(d->b, name);
-	    FILENAME_COPY(d->b + namelen, new_name);
-#ifdef USE_VM_PROBES
-	    dt_s1 = d->b;
-	    dt_s2 = d->b + namelen;
-	    dt_utag = buf + namelen + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE;
-#endif
-	    d->flags = desc->flags;
-	    d->fd = fd;
-	    d->command = command;
-	    d->invoke = invoke_rename;
-	    d->free = free_data;
-	    d->level = 2;
-	    goto done;
-	}
-    case FILE_CHDIR:
-    {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE);
-	
-	FILENAME_COPY(d->b, name);
-#ifdef USE_VM_PROBES
-	dt_s1 = d->b;
-	dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
-#endif
-	d->command = command;
-	d->invoke = invoke_chdir;
-	d->free = free_data;
-	d->level = 2;
-	goto done;
-    }
-    case FILE_PWD:
-        {
-	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1);
-	
-	    d->drive = *(uchar*)buf;
-#ifdef USE_VM_PROBES
-	    dt_utag = buf + 1;
-#endif
-	    d->command = command;
-	    d->invoke = invoke_pwd;
-	    d->free = free_data;
-	    d->level = 2;
-	    goto done;
-	}
-
-    case FILE_READDIR: 
-	if (sys_info.async_threads > 0)
-	{
-	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + 
-			      FILENAME_CHARSIZE);
-	
-	    FILENAME_COPY(d->b, name);
-#ifdef USE_VM_PROBES
-	    dt_s1 = d->b;
-	    dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
-#endif
-	    d->dir_handle = NULL;
-	    d->command = command;
-	    d->invoke = invoke_readdir;
-	    d->free = free_readdir;
-	    d->level = 2;
-	    d->c.read_dir.first_buf = NULL;
-	    d->c.read_dir.last_buf = NULL;
-	    goto done;
-	}
-	else   
-	{
-	    size_t resbufsize;
-	    size_t n = 0, total = 0;
-	    int res = 0;
-	    char resbuf[READDIR_BUFSIZE];
-
-	    EFILE_DIR_HANDLE dir_handle; /* Handle to open directory. */
-
-	    total               = READDIR_BUFSIZE;
-	    errInfo.posix_errno = 0;
-	    dir_handle          = NULL;
-	    resbuf[0]           = FILE_RESP_LFNAME;
-
-#ifdef USE_VM_PROBES
-	    dt_s1 = name;
-	    dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
-#endif
-	    /* Fill the buffer with multiple directory listings before sending it to the
-	     * receiving process. READDIR_CHUNKS is minimum number of files sent to the
-	     * receiver.
-	     * Format for each driver_output2:
-	     * ------------------------------------
-	     * | Type   | Len     | Filename  | ...
-	     * | 1 byte | 2 bytes | Len bytes | ...
-	     * ------------------------------------
-	     */
-
-	    do {
-		n = 1;
-		resbufsize = READDIR_BUFSIZE - n;
-
-		do {
-		    res = efile_readdir(&errInfo, name, &dir_handle, resbuf + n + 2, &resbufsize);
-
-		    if (res) {
-			put_int16((Uint16)resbufsize, resbuf + n);
-			n += 2 + resbufsize;
-			resbufsize = READDIR_BUFSIZE - n;
-		    }
-		} while( res && ((total - n - 2) >= MAXPATHLEN*FILENAME_CHARSIZE));
-
-		if (n > 1) {
-		    driver_output2(desc->port, resbuf, 1, resbuf + 1, n - 1);
-		}
-	    } while(res);
-
-	    if (errInfo.posix_errno != 0) {
-		reply_error(desc, &errInfo);
-		return;
-	    }
-#ifdef USE_VM_PROBES
-	    if (dt_utag != NULL && dt_utag[0] == '\0') {
-                dt_utag = NULL;
-            } 
-
-	    DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag,
-		     dt_utag, command, name, dt_s2,
-		     dt_i1, dt_i2, dt_i3, dt_i4, desc->port_str);
-	    DTRACE6(efile_drv_return, dt_priv->thread_num, dt_priv->tag++, 
-		    dt_utag, command, 1, 0);
-#endif
-	    TRACE_C('R');
-	    driver_output2(desc->port, resbuf, 1, NULL, 0);
-	    return;
-	}
-    case FILE_OPEN:
-	{
-	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(buf+4) + 
-			      FILENAME_CHARSIZE);
-	
-	    d->flags = get_int32((uchar*)buf);
-	    name = buf+4;
-	    FILENAME_COPY(d->b, name);
-#ifdef USE_VM_PROBES
-	    dt_i1 = d->flags;
-	    dt_s1 = d->b;
-	    dt_utag = name + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE;
-#endif
-	    d->command = command;
-	    d->invoke = invoke_open;
-	    d->free = free_data;
-	    d->level = 2;
-	    d->is_fd_unused = 1;
-	    goto done;
-	}
-
-    case FILE_FDATASYNC:
-	{
-	    d = EF_SAFE_ALLOC(sizeof(struct t_data));
-	    
-	    d->fd = fd;
-#ifdef USE_VM_PROBES
-	    dt_utag = name;
-	    dt_i1 = fd;
-#endif
-	    d->command = command;
-	    d->invoke = invoke_fdatasync;
-	    d->free = free_data;
-	    d->level = 2;
-	    goto done;
-	}
-
-    case FILE_FSYNC:
-	{
-	    d = EF_SAFE_ALLOC(sizeof(struct t_data));
-	    
-	    d->fd = fd;
-#ifdef USE_VM_PROBES
-	    dt_utag = name;
-	    dt_i1 = fd;
-#endif
-	    d->command = command;
-	    d->invoke = invoke_fsync;
-	    d->free = free_data;
-	    d->level = 2;
-	    goto done;
-	}
-
-
-    case FILE_FSTAT: 
-    case FILE_LSTAT:
-	{
-	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + 
-			      FILENAME_CHARSIZE);
-	    
-	    FILENAME_COPY(d->b, name);
-	    d->fd = fd;
-#ifdef USE_VM_PROBES
-	    dt_utag = name + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE;
-	    if (command == FILE_LSTAT) {
-		dt_s1 = d->b;
-	    } else {
-		dt_i1 = fd;
-	    }
-#endif
-	    d->command = command;
-	    d->invoke = invoke_flstat;
-	    d->free = free_data;
-	    d->level = 2;
-	    goto done;
-	}
-	
-    case FILE_TRUNCATE:
-        {
-	    d = EF_SAFE_ALLOC(sizeof(struct t_data));
-	    
-	    d->flags = desc->flags;
-	    d->fd = fd;
-#ifdef USE_VM_PROBES
-	    dt_utag = name;
-	    dt_i1 = fd;
-	    dt_i2 = d->flags;
-#endif
-	    d->command = command;
-	    d->invoke = invoke_truncate;
-	    d->free = free_data;
-	    d->level = 2;
-	    goto done;
-	}
-
-    case FILE_WRITE_INFO:
-	{
-	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1
-			      + FILENAME_BYTELEN(buf + 9*4) + FILENAME_CHARSIZE);
-	    
-	    d->info.mode       = get_int32(buf + 0 * 4);
-	    d->info.uid        = get_int32(buf + 1 * 4);
-	    d->info.gid        = get_int32(buf + 2 * 4);
-	    d->info.accessTime = get_int64(buf + 3 * 4);
-	    d->info.modifyTime = get_int64(buf + 5 * 4);
-	    d->info.cTime      = get_int64(buf + 7 * 4);
-
-	    FILENAME_COPY(d->b, buf + 9*4);
-#ifdef USE_VM_PROBES
-	    dt_i1              = d->info.mode;
-	    dt_i2              = d->info.uid;
-	    dt_i3              = d->info.gid;
-	    dt_s1 = d->b;
-	    dt_utag = buf + 9 * 4 + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE;
-#endif
-	    d->command = command;
-	    d->invoke = invoke_write_info;
-	    d->free = free_data;
-	    d->level = 2;
-	    goto done;
-	}
-
-    case FILE_READLINK:
-	{
-	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + 
-			      MAX(RESBUFSIZE, (FILENAME_BYTELEN(name) +  
-					       FILENAME_CHARSIZE))  + 1);
-	    FILENAME_COPY(d->b, name);
-#ifdef USE_VM_PROBES
-	    dt_s1 = d->b;
-	    dt_utag = name + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE;
-#endif
-	    d->command = command;
-	    d->invoke = invoke_readlink;
-	    d->free = free_data;
-	    d->level = 2;
-	    goto done;
-	}
-
-    case FILE_ALTNAME:
-	{
-	     d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + 
-			       MAX(RESBUFSIZE, (FILENAME_BYTELEN(name) +  
-						FILENAME_CHARSIZE))  + 1);
-	    FILENAME_COPY(d->b, name);
-#ifdef USE_VM_PROBES
-	    dt_s1 = d->b;
-	    dt_utag = name + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE;
-#endif
-	    d->command = command;
-	    d->invoke = invoke_altname;
-	    d->free = free_data;
-	    d->level = 2;
-	    goto done;
-	}
-
-
-    case FILE_LINK:
-	{
-	    char* new_name;
-	    int namelen = FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
-
-	    new_name = name+namelen;
-	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1
-			      + namelen
-			      + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE);
-	
-	    FILENAME_COPY(d->b, name);
-	    FILENAME_COPY(d->b + namelen, new_name);
-#ifdef USE_VM_PROBES
-	    dt_s1 = d->b;
-	    dt_s2 = d->b + namelen;
-	    dt_utag = buf + namelen + FILENAME_BYTELEN(dt_s2) + FILENAME_CHARSIZE;
-#endif
-	    d->flags = desc->flags;
-	    d->fd = fd;
-	    d->command = command;
-	    d->invoke = invoke_link;
-	    d->free = free_data;
-	    d->level = 2;
-	    goto done;
-	}
-
-    case FILE_SYMLINK:
-	{
-	    char* new_name;
-	    int namelen = FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
-
-	    new_name = name+namelen;
-	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1
-			      + namelen
-			      + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE);
-	
-	    FILENAME_COPY(d->b, name);
-	    FILENAME_COPY(d->b + namelen, new_name);
-#ifdef USE_VM_PROBES
-	    dt_s1 = d->b;
-	    dt_s2 = d->b + namelen;
-	    dt_utag = buf + namelen + FILENAME_BYTELEN(dt_s2) + FILENAME_CHARSIZE;
-#endif
-	    d->flags = desc->flags;
-	    d->fd = fd;
-	    d->command = command;
-	    d->invoke = invoke_symlink;
-	    d->free = free_data;
-	    d->level = 2;
-	    goto done;
-	}
-
-    case FILE_FADVISE:
-    {
-        d = EF_SAFE_ALLOC(sizeof(struct t_data));
-
-        d->fd = fd;
-        d->command = command;
-        d->invoke = invoke_fadvise;
-        d->free = free_data;
-        d->level = 2;
-        d->c.fadvise.offset = get_int64((uchar*) buf);
-        d->c.fadvise.length = get_int64(((uchar*) buf) + sizeof(Sint64));
-        d->c.fadvise.advise = get_int32(((uchar*) buf) + 2 * sizeof(Sint64));
-#ifdef USE_VM_PROBES
-        dt_i1 = d->fd;
-        dt_i2 = d->c.fadvise.offset;
-        dt_i3 = d->c.fadvise.length;
-        dt_i4 = d->c.fadvise.advise;
-        dt_utag = buf + 3 * sizeof(Sint64);
-#endif
-        goto done;
-    }
-
-    case FILE_FALLOCATE:
-    {
-        d = EF_SAFE_ALLOC(sizeof(struct t_data));
-
-        d->fd = fd;
-        d->command = command;
-        d->invoke = invoke_fallocate;
-        d->free = free_data;
-        d->level = 2;
-        d->c.fallocate.offset = get_int64((uchar*) buf);
-        d->c.fallocate.length = get_int64(((uchar*) buf) + sizeof(Sint64));
-        goto done;
-    }
-
-    }
-
-    /*
-     * Ignore anything else -- let the caller hang.
-     */
-     
-    return;
-
- done:
-    if (d) {
-#ifdef USE_VM_PROBES
-	d->sched_i1 = dt_priv->thread_num;
-	d->sched_i2 = dt_priv->tag;
-	d->sched_utag[0] = '\0';
-	if (dt_utag != NULL) {
-	    if (dt_utag[0] == '\0') {
-		dt_utag = NULL;
-	    } else {
-		strncpy(d->sched_utag, dt_utag, sizeof(d->sched_utag) - 1);
-		d->sched_utag[sizeof(d->sched_utag) - 1] = '\0';
-	    }
-	}
-	DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag++,
-		 dt_utag, command, dt_s1, dt_s2,
-		 dt_i1, dt_i2, dt_i3, dt_i4, desc->port_str);
-#endif
-	cq_enq(desc, d);
-    }
-}
-
-/*********************************************************************
- * Driver entry point -> flush
- */
-static void 
-file_flush(ErlDrvData e) {
-    file_descriptor *desc = (file_descriptor *)e;
-#ifdef DEBUG
-    int r;
-#endif
-#ifdef  USE_VM_PROBES
-    dt_private *dt_priv = get_dt_private(dt_driver_io_worker_base);
-#endif
-
-    TRACE_C('f');
-
-#ifdef HAVE_SENDFILE
-    flush_sendfile(desc, NULL);
-#endif
-
-#ifdef DEBUG
-    r = 
-#endif
-         flush_write(desc, NULL
-#ifdef USE_VM_PROBES
-		     , dt_priv, (desc->d == NULL) ? NULL : desc->d->sched_utag
-#endif
-		     );
-    /* Only possible reason for bad return value is ENOMEM, and 
-     * there is nobody to tell...
-     */
-#ifdef DEBUG
-    ASSERT(r == 0); 
-#endif
-    cq_execute(desc);
-}
-
-
-
-/*********************************************************************
- * Driver entry point -> control
- * Only debug functionality...
- */
-static ErlDrvSSizeT
-file_control(ErlDrvData e, unsigned int command, 
-	     char* buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen) {
-    file_descriptor *desc = (file_descriptor *)e;
-    switch (command) {
-    case 'K' :
-	if (rlen < 4) {
-	    *rbuf = EF_ALLOC(4);
-	}
-	(*rbuf)[0] = ((desc->key) >> 24) & 0xFF;
-	(*rbuf)[1] = ((desc->key) >> 16) & 0xFF;
-	(*rbuf)[2] = ((desc->key) >> 8) & 0xFF;
-	(*rbuf)[3] = (desc->key) & 0xFF;
-	return 4;
-    default:
-	return 0;
-    }
-}
-
-/*********************************************************************
- * Driver entry point -> timeout
- */
-static void 
-file_timeout(ErlDrvData e) {
-    file_descriptor *desc = (file_descriptor *)e;
-    enum e_timer timer_state = desc->timer_state;
-#ifdef  USE_VM_PROBES
-    dt_private *dt_priv = get_dt_private(dt_driver_io_worker_base);
-#endif
-
-    TRACE_C('t');
-
-    desc->timer_state = timer_idle;
-    switch (timer_state) {
-    case timer_idle:
-	ASSERT(0);
-	break;
-    case timer_again:
-	ASSERT(desc->invoke);
-	ASSERT(desc->free);
-	driver_async(desc->port, KEY(desc), desc->invoke, desc->d, desc->free);
-	break;
-    case timer_write: {
-#ifdef DEBUG
-	int r = 
-#endif
-	         flush_write(desc, NULL
-#ifdef USE_VM_PROBES
-			     , dt_priv, (desc->d == NULL) ? NULL : desc->d->sched_utag
-#endif
-			     );
-	/* Only possible reason for bad return value is ENOMEM, and 
-	 * there is nobody to tell...
-	 */
-	ASSERT(r == 0); 
-	cq_execute(desc);
-    } break;
-    } /* case */
-}
-
-
-
-/*********************************************************************
- * Driver entry point -> outputv
- */
-static void 
-file_outputv(ErlDrvData e, ErlIOVec *ev) {
-    file_descriptor* desc = (file_descriptor*)e;
-    char command;
-    size_t p, q;
-    int err;
-    struct t_data *d = NULL;
-#ifdef USE_VM_PROBES
-    Sint64 dt_i1 = 0, dt_i2 = 0, dt_i3 = 0;
-    Sint64 dt_i4 = 0;
-    char *dt_utag = NULL;
-    char *dt_s1 = NULL;
-    dt_private *dt_priv = get_dt_private(dt_driver_io_worker_base);
-#endif
-
-    TRACE_C('v');
-
-    p = 0; q = 1;
-    if (! EV_GET_CHAR(ev, &command, &p, &q)) {
-	/* Empty command */
-	reply_posix_error(desc, EINVAL);
-	goto done;
-    }
-    /* 'command' contains the decoded command number,
-     * 'p' and 'q' point out the next byte in the command:
-     * ((char *)ev->iov[q].iov_base) + p;
-     */
-    
-    TRACE_F(("%i", (int) command));
-
-    switch (command) {
-
-    case FILE_CLOSE: {
-#ifdef USE_VM_PROBES
-	dt_utag = EV_CHAR_P(ev, p, q);
-#endif
-	flush_read(desc);
-	if (flush_write_check_error(desc, &err
-#ifdef USE_VM_PROBES
-				    , dt_priv, dt_utag
-#endif
-				    ) < 0) {
-	    reply_posix_error(desc, err);
-	    goto done;
-	}
-	if (desc->fd != FILE_FD_INVALID) {
-	    if (! (d = EF_ALLOC(sizeof(struct t_data)))) {
-		reply_posix_error(desc, ENOMEM);
-	    } else {
-		d->command = command;
-		d->reply = !0;
-		d->fd = desc->fd;
-		d->flags = desc->flags;
-#ifdef USE_VM_PROBES
-		dt_i1 = d->fd;
-		dt_i2 = d->flags;
-#endif
-		d->invoke = invoke_close;
-		d->free = free_data;
-		d->level = 2;
-		cq_enq(desc, d);
-		desc->fd = FILE_FD_INVALID;
-		desc->flags = 0;
-	    }
-	} else {
-	    reply_posix_error(desc, EBADF);
-	}
-    } goto done;
-
-    case FILE_READ: {
-	Uint32 sizeH, sizeL;
-	size_t size, alloc_size;
-
-	if (!EV_GET_UINT32(ev, &sizeH, &p, &q)
-	    || !EV_GET_UINT32(ev, &sizeL, &p, &q)) {
-	    /* Wrong buffer length to contain the read count */
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
-#ifdef USE_VM_PROBES
-	dt_utag = EV_CHAR_P(ev, p, q);
-#endif
-	if (flush_write_check_error(desc, &err
-#ifdef USE_VM_PROBES
-				    , dt_priv, dt_utag
-#endif
-				    ) < 0) {
-	    reply_posix_error(desc, err);
-	    goto done;
-	}
-#if ALWAYS_READ_LINE_AHEAD
-	if (desc->read_bufsize == 0 && desc->read_binp != NULL && desc->read_size > 0) {
-	    /* We have allocated a buffer for line mode but should not really have a 
-	       read-ahead buffer... */
-	    if (lseek_flush_read(desc, &err
-#ifdef USE_VM_PROBES
-				 , dt_priv, dt_utag
-#endif
-				 ) < 0) {
-		reply_posix_error(desc, err);
-		goto done;
-	    }
-	}
-#endif
-#if SIZEOF_SIZE_T == 4
-	if (sizeH != 0) {
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
-	size = sizeL;
-#else
-	size = ((size_t)sizeH << 32) | sizeL;
-#endif
-	if ((desc->fd == FILE_FD_INVALID)
-	    || (! (desc->flags & EFILE_MODE_READ)) ) {
-	    reply_posix_error(desc, EBADF);
-	    goto done;
-	}
-	if (size == 0) {
-	    reply_buf(desc, &command, 0);
-	    goto done;
-	}
-	if (desc->read_size >= size) {
-	    /* We already have all data */
-	    TRACE_C('D');
-	    reply_data(desc, desc->read_binp, desc->read_offset, size);
-	    desc->read_offset += size;
-	    desc->read_size -= size;
-	    try_free_read_bin(desc);
-	    goto done;
-	}
-	/* We may have some of the data 
-	 */
-	/* Justification for the following strange formula:
-	 * If the read request is for such a large block as more than 
-	 * half the buffer size it may lead to a lot of unnecessary copying, 
-	 * since the tail of the old buffer is copied to the head of the
-	 * new, and if the tail is almost half the buffer it is a lot
-	 * to copy. Therefore allocate the exact amount needed in 
-	 * this case, giving no lingering tail. */
-	alloc_size = 
-	    size > (desc->read_bufsize>>1) ? 
-	    size : desc->read_bufsize;
-	if (! desc->read_binp) {
-	    /* Need to allocate a new binary for the result */
-	    if (! (desc->read_binp = driver_alloc_binary(alloc_size))) {
-		reply_posix_error(desc, ENOMEM);
-		goto done;
-	    }
-	} else {
-	    /* We already have a buffer */
-	    if (desc->read_binp->orig_size - desc->read_offset < size) {
-		/* Need to allocate a new binary for the result */
-		ErlDrvBinary *binp;
-		if (! (binp = driver_alloc_binary(alloc_size))) {
-		    reply_posix_error(desc, ENOMEM);
-		    goto done;
-		}
-		/* Move data we already have to the new binary */
-		sys_memcpy(binp->orig_bytes, 
-			   desc->read_binp->orig_bytes + desc->read_offset,
-			   desc->read_size);
-		driver_free_binary(desc->read_binp);
-		desc->read_offset = 0;
-		desc->read_binp = binp;
-	    }
-	} 
-	if (! (d = EF_ALLOC(sizeof(struct t_data)))) {
-	    reply_posix_error(desc, ENOMEM);
-	    goto done;
-	}
-	d->command = command;
-	d->reply = !0;
-	d->fd = desc->fd;
-	d->flags = desc->flags;
-	d->c.read.binp = desc->read_binp;
-	d->c.read.bin_offset = desc->read_offset + desc->read_size;
-	d->c.read.bin_size = desc->read_binp->orig_size - d->c.read.bin_offset;
-	d->c.read.size = size;
-#ifdef USE_VM_PROBES
-	dt_i1 = d->fd;
-	dt_i2 = d->flags;
-	dt_i3 = d->c.read.size;
-#endif
-	driver_binary_inc_refc(d->c.read.binp);
-	d->invoke = invoke_read;
-	d->free = free_read;
-	d->level = 1;
-	cq_enq(desc, d);
-    } goto done; /* case FILE_READ: */
-
-    case FILE_READ_LINE: {
-	/*
-	 * Icky little creature... We do mostly as ordinary file read, but with a few differences.
-	 * 1) We have to scan for proper newline sequence if there is a buffer already, we cannot know 
-	 *    in advance if the buffer contains a whole line without scanning.
-	 * 2) We do not know how large the buffer needs to be in advance. We give a default buffer,
-	 *    but the worker may need to allocate a new one. Freeing the old and rereferencing a newly 
-	 *    allocated binary + dealing with offsets and lengts are done in file_async ready
-	 *    for this OP.
-	 */
-#ifdef USE_VM_PROBES
-	dt_utag = EV_CHAR_P(ev, p, q);
-#endif
-	if (flush_write_check_error(desc, &err
-#ifdef USE_VM_PROBES
-				    , dt_priv, dt_utag
-#endif
-				    ) < 0) {
-	    reply_posix_error(desc, err);
-	    goto done;
-	}
-	if (ev->size != 1
-#ifdef USE_VM_PROBES
-	    + FILENAME_BYTELEN(dt_utag) + FILENAME_CHARSIZE
-#endif
-	    ) {
-	    /* Wrong command length */
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
-	if ((desc->fd == FILE_FD_INVALID)
-	    || (! (desc->flags & EFILE_MODE_READ)) ) {
-	    reply_posix_error(desc, EBADF);
-	    goto done;
-	}
-	if (desc->read_size > 0) {
-	    /* look for '\n' in what we'we already got */
-	    void *nl_ptr = memchr(desc->read_binp->orig_bytes + desc->read_offset,'\n',desc->read_size);
-	    if (nl_ptr != NULL) {
-		/* If found, we're done */
-		int skip = 0;
-		size_t size = ((char *) nl_ptr) - 
-		    ((char *) (desc->read_binp->orig_bytes + desc->read_offset)) + 1;
-		if (size > 1 &&
-		    *(((char *) nl_ptr) - 1) == '\r') {
-		    *(((char *) nl_ptr) - 1) = '\n';		    
-		    skip = 1;
-		    --size;
-		} 
-		reply_data(desc, desc->read_binp, desc->read_offset, size);
-		desc->read_offset += (size + skip);
-		desc->read_size -= (size + skip);
-		try_free_read_bin(desc);
-		goto done;
-	    }
-	}
-	/* Now, it's up to the thread to work out the need for more buffers and such, it's
-	   no use doing it in this thread as we do not have the information required anyway. 
-	   Even a NULL buffer could be handled by the thread, but code is simplified by us 
-	   allocating it */
-	if (! desc->read_binp) {
-	    int alloc_size = (desc->read_bufsize > DEFAULT_LINEBUF_SIZE) ? desc->read_bufsize : 
-		DEFAULT_LINEBUF_SIZE;
-	    /* Allocate a new binary for the result */
-	    if (! (desc->read_binp = driver_alloc_binary(alloc_size))) {
-		reply_posix_error(desc, ENOMEM);
-		goto done;
-	    }
-	}	
-	if (! (d = EF_ALLOC(sizeof(struct t_data)))) {
-	    reply_posix_error(desc, ENOMEM);
-	    goto done;
-	}
-
-	d->command = command;
-	d->reply = !0;
-	d->fd = desc->fd;
-	d->flags = desc->flags;
-	d->c.read_line.binp = desc->read_binp;
-	d->c.read_line.read_offset = desc->read_offset;
-	d->c.read_line.read_size = desc->read_size;
-#ifdef USE_VM_PROBES
-	dt_i1 = d->fd;
-	dt_i2 = d->flags;
-	dt_i3 = d->c.read_line.read_offset;
-#endif
-#if !ALWAYS_READ_LINE_AHEAD
-	d->c.read_line.read_ahead = (desc->read_bufsize > 0);
-#ifdef USE_VM_PROBES
-	dt_i4 = d->c.read_line.read_ahead;
-#endif
-#endif 
-	driver_binary_inc_refc(d->c.read.binp);
-	d->invoke = invoke_read_line;
-	d->free = free_read_line;
-	d->level = 1;
-	cq_enq(desc, d);
-    } goto done;
-    case FILE_WRITE: { /* Dtrace: The dtrace user tag is not last in message, 
-			  but follows the message tag directly. 
-			  This is handled specially in prim_file.erl */
-	ErlDrvSizeT skip = 1;
-	ErlDrvSizeT size = ev->size - skip;
-
-#ifdef USE_VM_PROBES
-	dt_utag = EV_CHAR_P(ev, p, q);
-	skip += FILENAME_BYTELEN(dt_utag) + FILENAME_CHARSIZE;
-	size = ev->size - skip;
-#endif
-	if (lseek_flush_read(desc, &err
-#ifdef USE_VM_PROBES
-			     , dt_priv, dt_utag
-#endif
-			     ) < 0) {
-	    reply_posix_error(desc, err);
-	    goto done;
-	}
-	if (! (desc->flags & EFILE_MODE_WRITE)) {
-	    reply_posix_error(desc, EBADF);
-	    goto done;
-	}
-	if (size == 0) {
-	    reply_Uint(desc, size);
-	    goto done;
-	}
-	MUTEX_LOCK(desc->q_mtx);
-	if (driver_enqv(desc->port, ev, skip)) {
-	    MUTEX_UNLOCK(desc->q_mtx);
-	    reply_posix_error(desc, ENOMEM);
-	    goto done;
-	}
-	desc->write_buffered += size;
-	if (desc->write_buffered < desc->write_bufsize) {
-	    MUTEX_UNLOCK(desc->q_mtx);
-	    reply_Uint(desc, size);
-	    if (desc->timer_state == timer_idle) {
-		desc->timer_state = timer_write;
-		driver_set_timer(desc->port, desc->write_delay);
-	    }
-	} else {
-	    if ((d = async_write(desc, &err, !0, size
-#ifdef USE_VM_PROBES
-				 , &dt_i1, &dt_i2, &dt_i3
-#endif
-				 )) == NULL) {
-		MUTEX_UNLOCK(desc->q_mtx);
-		reply_posix_error(desc, err);
-		goto done;
-	    } else {
-		MUTEX_UNLOCK(desc->q_mtx);
-	    }
-	}
-    } goto done; /* case FILE_WRITE */
-
-    case FILE_PWRITEV: { /* Dtrace: The dtrace user tag is not last in message, 
-			   but follows the message tag directly. 
-			   This is handled specially in prim_file.erl */
-	Uint32 i, j, n; 
-	size_t total;
-#ifdef USE_VM_PROBES
-	char dt_tmp;
-	int dt_utag_bytes = 1;
-
-	dt_utag = EV_CHAR_P(ev, p, q);
-	/* This will work for UTF-8, but not for UTF-16 - extra reminder here */
-#ifdef FILENAMES_16BIT 
-#error 16bit characters in filenames and dtrace in combination is not supported.
-#endif
-	while (EV_GET_CHAR(ev, &dt_tmp, &p, &q) && dt_tmp != '\0') {
-	    dt_utag_bytes++;
-	}
-#endif
-	if (ev->size < 1+4
-#ifdef USE_VM_PROBES
-	    + dt_utag_bytes
-#endif
-	    || !EV_GET_UINT32(ev, &n, &p, &q)) {
-	    /* Buffer too short to contain even the number of pos/size specs */
-	    reply_Uint_posix_error(desc, 0, EINVAL);
-	    goto done;
-	}
-	if (lseek_flush_read(desc, &err
-#ifdef USE_VM_PROBES
-			     , dt_priv, dt_utag
-#endif
-			     ) < 0) {
-	    reply_Uint_posix_error(desc, 0, err);
-	    goto done;
-	}
-	if (flush_write_check_error(desc, &err
-#ifdef USE_VM_PROBES
-				    , dt_priv, dt_utag
-#endif
-				    ) < 0) {
-	    reply_Uint_posix_error(desc, 0, err);
-	    goto done;
-	}
-	if (n == 0) {
-	    /* Trivial case - nothing to write */
-	    if (ev->size != 1+4) {
-		reply_posix_error(desc, err);
-	    } else {
-		reply_Uint(desc, 0);
-	    }
-	    goto done;
-	}
-	if (ev->size < 1+4+8*(2*n)
-#ifdef USE_VM_PROBES
-	    + dt_utag_bytes
-#endif
-	    ) {
-	    /* Buffer too short to contain even the pos/size specs */
-	    reply_Uint_posix_error(desc, 0, EINVAL);
-	    goto done;
-	}
-	d = EF_ALLOC(sizeof(struct t_data) 
-		     + (n * sizeof(struct t_pbuf_spec)));
-	if (! d) {
-	    reply_Uint_posix_error(desc, 0, ENOMEM);
-	    goto done;
-	}
-	d->command = command;
-	d->reply = !0;
-	d->fd = desc->fd;
-	d->flags = desc->flags;
-#ifdef USE_VM_PROBES
-	dt_i1 = d->fd;
-	dt_i2 = d->flags;
-#endif
-	d->c.pwritev.port = desc->port;
-	d->c.pwritev.q_mtx = desc->q_mtx;
-	d->c.pwritev.n = n;
-	d->c.pwritev.cnt = 0;
-	total = 0;
-	j = 0;
-	/* Create pos/size specs in the thread data structure
-	 * for all non-zero size binaries. Calculate total size.
-	 */
-	for(i = 0; i < n; i++) {
-	    Uint32 sizeH, sizeL;
-	    size_t size;
-	    if (   !EV_GET_SINT64(ev, &d->c.pwritev.specs[i].offset, &p, &q)
-		|| !EV_GET_UINT32(ev, &sizeH, &p, &q)
-		|| !EV_GET_UINT32(ev, &sizeL, &p, &q)) {
-		/* Misalignment in buffer */
-		reply_Uint_posix_error(desc, 0, EINVAL);
-		EF_FREE(d);
-		goto done;
-	    }
-#if SIZEOF_SIZE_T == 4
-	    if (sizeH != 0) {
-		reply_Uint_posix_error(desc, 0, EINVAL);
-		EF_FREE(d);
-		goto done;
-	    }
-	    size = sizeL;
-#else
-	    size = ((size_t)sizeH<<32) | sizeL;
-#endif
-	    if (size > 0) {
-		total += size;
-		d->c.pwritev.specs[j].size = size;
-		j++;
-	    }
-	}
-	d->c.pwritev.size = total;
-#ifdef USE_VM_PROBES
-	dt_i3 = d->c.pwritev.size;
-#endif
-	if (j == 0) {
-	    /* Trivial case - nothing to write */
-	    EF_FREE(d);
-	    reply_Uint(desc, 0);
-	} else {
-	    ErlDrvSizeT skip = 1 + 4 + 8 * (2*n) 
-#ifdef USE_VM_PROBES
-		+ dt_utag_bytes
-#endif
-		;
-	    if (skip + total != ev->size) {
-		/* Actual amount of data does not match 
-		 * total of all pos/size specs
-		 */
-		EF_FREE(d);
-		reply_Uint_posix_error(desc, 0, EINVAL);
-	    } else {
-		/* Enqueue the data */
-		MUTEX_LOCK(desc->q_mtx);
-		driver_enqv(desc->port, ev, skip);
-		MUTEX_UNLOCK(desc->q_mtx);
-		/* Execute the command */
-		d->invoke = invoke_pwritev;
-		d->free = free_data;
-		d->level = 1;
-		cq_enq(desc, d);
-	    }
-	}
-    } goto done; /* case FILE_PWRITEV: */
-
-    case FILE_PREADV: { /* Dtrace: The dtrace user tag is not last in message, 
-			   but follows the message tag directly. 
-			   This is handled specially in prim_file.erl */
-	register void * void_ptr;
-	Uint32 i, n;
-	ErlIOVec *res_ev;
-#ifdef USE_VM_PROBES
-	char dt_tmp;
-	int dt_utag_bytes = 1;
-	/* This will work for UTF-8, but not for UTF-16 - extra reminder here */
-#ifdef FILENAMES_16BIT 
-#error 16bit characters in filenames and dtrace in combination is not supported.
-#endif
-	dt_utag = EV_CHAR_P(ev, p, q);
-	while (EV_GET_CHAR(ev, &dt_tmp, &p, &q) && dt_tmp != '\0') {
-	    dt_utag_bytes++;
-	}
-#endif
-	if (lseek_flush_read(desc, &err
-#ifdef USE_VM_PROBES
-			     , dt_priv, dt_utag
-#endif
-			     ) < 0) {
-	    reply_posix_error(desc, err);
-	    goto done;
-	}
-	if (flush_write_check_error(desc, &err
-#ifdef USE_VM_PROBES
-				    , dt_priv, dt_utag
-#endif
-				    ) < 0) {
-	    reply_posix_error(desc, err);
-	    goto done;
-	}
-	if (ev->size < 1+8
-#ifdef USE_VM_PROBES
-	    + dt_utag_bytes
-#endif
-	    || !EV_GET_UINT32(ev, &n, &p, &q)
-	    || !EV_GET_UINT32(ev, &n, &p, &q)) {
-	    /* Buffer too short to contain even the number of pos/size specs */
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
-	if (ev->size < 1+8+8*(2*n)
-#ifdef USE_VM_PROBES
-	    + dt_utag_bytes
-#endif
-	    ) {
-	    /* Buffer wrong length to contain the pos/size specs */
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
-	/* Create the thread data structure with the contained ErlIOVec 
-	 * and corresponding binaries for the response 
-	 */
-	d = EF_ALLOC(sizeof(*d)
-		     + (n * sizeof(*d->c.preadv.offsets))
-		     + ((1+n) * (sizeof(*res_ev->iov)
-				 + sizeof(*res_ev->binv))));
-	if (! d) {
-	    reply_posix_error(desc, ENOMEM);
-	    goto done;
-	}
-	d->command = command;
-	d->reply = !0;
-	d->fd = desc->fd;
-	d->flags = desc->flags;
-#ifdef USE_VM_PROBES
-	dt_i1 = d->fd;
-	dt_i2 = d->flags;
-#endif
-	d->c.preadv.n = n;
-	d->c.preadv.cnt = 0;
-	d->c.preadv.size = 0;
-	res_ev = &d->c.preadv.eiov;
-	/* XXX possible alignment problems here for weird machines */
-	res_ev->vsize = 1+d->c.preadv.n;
-	res_ev->iov = void_ptr = &d->c.preadv.offsets[d->c.preadv.n];
-	res_ev->binv = void_ptr = &res_ev->iov[res_ev->vsize];
-	/* Read in the pos/size specs and allocate binaries for the results */
-	for (i = 1; i < 1+n; i++) {
-	    Uint32 sizeH, sizeL;
-	    size_t size;
-	    if (   !EV_GET_SINT64(ev, &d->c.preadv.offsets[i-1], &p, &q)
-		|| !EV_GET_UINT32(ev, &sizeH, &p, &q)
-		|| !EV_GET_UINT32(ev, &sizeL, &p, &q)) {
-		reply_posix_error(desc, EINVAL);
-		break;
-	    }
-#if SIZEOF_SIZE_T == 4
-	    if (sizeH != 0) {
-		reply_posix_error(desc, EINVAL);
-		break;
-	    }
-	    size = sizeL;
-#else
-	    size = ((size_t)sizeH<<32) | sizeL;
-#endif
-#ifdef USE_VM_PROBES
-	    dt_i3 += size;
-#endif
-	    if (! (res_ev->binv[i] = driver_alloc_binary(size))) {
-		reply_posix_error(desc, ENOMEM);
-		break;
-	    } else {
-		res_ev->iov[i].iov_len  = size;
-		res_ev->iov[i].iov_base = res_ev->binv[i]->orig_bytes;
-	    }
-	}
-	if (i < 1+n) {
-	    for (i--; i > 0; i--) {
-		driver_free_binary(res_ev->binv[i]);
-	    }
-	    EF_FREE(d);
-	    goto done;
-	}
-	/* Allocate the header binary (index 0) */
-	res_ev->binv[0] = driver_alloc_binary(4+4+8*n);
-	if (! res_ev->binv[0]) {
-	    reply_posix_error(desc, ENOMEM);
-	    for (i = 1; i < 1+n; i++) {
-		driver_free_binary(res_ev->binv[i]);
-	    }
-	    EF_FREE(d);
-	    goto done;
-	}
-	res_ev->iov[0].iov_len = 4+4+8*n;
-	res_ev->iov[0].iov_base = res_ev->binv[0]->orig_bytes;
-	/* Fill in the number of buffers in the header */
-	put_int32(0, res_ev->iov[0].iov_base);
-	put_int32(n, (char *)(res_ev->iov[0].iov_base) + 4);
-	/**/
-	res_ev->size = res_ev->iov[0].iov_len;
-	if (n == 0) {
-	    /* Trivial case - nothing to read */
-	    reply_ev(desc, FILE_RESP_LDATA, res_ev);
-	    free_preadv(d);
-	    goto done;
-	} else {
-	    d->invoke = invoke_preadv;
-	    d->free = free_preadv;
-	    d->level = 1;
-	    cq_enq(desc, d);
-	}
-    } goto done; /* case FILE_PREADV: */
-
-    case FILE_LSEEK: {
-	Sint64 offset;		/* Offset for seek */
-	Uint32 origin;		/* Origin of seek. */
-
-	if (ev->size < 1+8+4
-	    || !EV_GET_SINT64(ev, &offset, &p, &q)
-	    || !EV_GET_UINT32(ev, &origin, &p, &q)) {
-	    /* Wrong length of buffer to contain offset and origin */
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
-#ifdef USE_VM_PROBES
-	dt_utag = EV_CHAR_P(ev, p, q);
-#endif
-	if (lseek_flush_read(desc, &err
-#ifdef USE_VM_PROBES
-			     , dt_priv, dt_utag
-#endif
-			     ) < 0) {
-	    reply_posix_error(desc, err);
-	    goto done;
-	}
-	if (flush_write_check_error(desc, &err
-#ifdef USE_VM_PROBES
-				    , dt_priv, dt_utag
-#endif
-				    ) < 0) {
-	    reply_posix_error(desc, err);
-	    goto done;
-	}
-	if ((d = async_lseek(desc, &err, !0, offset, origin
-#ifdef USE_VM_PROBES
-			     , &dt_i1, &dt_i2, &dt_i3
-#endif
-			     )) == NULL) {
-	    reply_posix_error(desc, err);
-	    goto done;
-	}
-    } goto done;
-
-    case FILE_READ_FILE: {
-	char *filename;
-	if (ev->size < 1+1) {
-	    /* Buffer contains empty name */
-	    reply_posix_error(desc, ENOENT);
-	    goto done;
-	}
-#ifndef USE_VM_PROBES
-	/* In the dtrace case, the iov has an extra element, the dtrace utag - we will need 
-	   another test to see that
-	   the filename is in a single buffer: */
-	if (ev->size-1 != ev->iov[q].iov_len-p) {
-	    /* Name not in one single buffer */
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
-#else
-	if (((byte *)ev->iov[q].iov_base)[ev->iov[q].iov_len-1] != '\0') {
-	    /* Name not in one single buffer */
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}	
-#endif
-	filename = EV_CHAR_P(ev, p, q);
-	d = EF_ALLOC(sizeof(struct t_data) -1 + FILENAME_BYTELEN(filename) + FILENAME_CHARSIZE);
-	if (! d) {
-	    reply_posix_error(desc, ENOMEM);
-	    goto done;
-	}
-	d->command = command;
-	d->reply = !0;
-	/* Copy name */
-	FILENAME_COPY(d->b, filename);
-#ifdef USE_VM_PROBES
-	{
-	    char dt_tmp;
-
-	    /* This will work for UTF-8, but not for UTF-16 - extra reminder here */
-#ifdef FILENAMES_16BIT 
-#error 16bit characters in filenames and dtrace in combination is not supported.
-#endif
-	    while (EV_GET_CHAR(ev, &dt_tmp, &p, &q) && dt_tmp != '\0') 
-		;
-	    dt_s1 = d->b;
-	    dt_utag = EV_CHAR_P(ev, p, q);
-	}
-#endif
-	d->c.read_file.binp = NULL;
-	d->invoke = invoke_read_file;
-	d->free = free_read_file;
-	d->level = 2;
-	cq_enq(desc, d);
-    } goto done;
-
-    case FILE_IPREAD: {
-	/* This operation cheets by using invoke_preadv() and free_preadv()
-	 * plus its own invoke_ipread. Therefore the result format is 
-	 * a bit awkward - the header binary contains one extra 64 bit
-	 * field that invoke_preadv() fortunately ignores,
-	 * and the first 64 bit field does not contain the number of 
-	 * data binaries which invoke_preadv() also ignores.
-	 */
-	register void * void_ptr;
-	char mode;
-	Sint64 hdr_offset;
-	Uint32 max_size;
-	ErlIOVec *res_ev;
-	int vsize;
-	if (! EV_GET_CHAR(ev, &mode, &p, &q)) {
-	    /* Empty command */
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
-	if (mode != IPREAD_S32BU_P32BU) {
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
-	if (ev->size < 1+1+8+4
-	    || !EV_GET_SINT64(ev, &hdr_offset, &p, &q)
-	    || !EV_GET_UINT32(ev, &max_size, &p, &q)) {
-	    /* Buffer too short to contain 
-	     * the header offset and max size spec */
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
-#ifdef USE_VM_PROBES
-	dt_utag = EV_CHAR_P(ev, p, q);
-#endif
-	if (lseek_flush_read(desc, &err
-#ifdef USE_VM_PROBES
-			     , dt_priv, dt_utag
-#endif
-			     ) < 0) {
-	    reply_posix_error(desc, err);
-	    goto done;
-	}
-	if (flush_write_check_error(desc, &err
-#ifdef USE_VM_PROBES
-				    , dt_priv, dt_utag
-#endif
-				    ) < 0) {
-	    reply_posix_error(desc, err);
-	    goto done;
-	}
-	/* Create the thread data structure with the contained ErlIOVec 
-	 * and corresponding binaries for the response 
-	 */
-	vsize = 2;
-	d = EF_ALLOC(sizeof(*d) + 
-		     vsize*(sizeof(*res_ev->iov) + sizeof(*res_ev->binv)));
-	if (! d) {
-	    reply_posix_error(desc, ENOMEM);
-	    goto done;
-	}
-	d->command = command;
-	d->reply = !0;
-	d->fd = desc->fd;
-	d->flags = desc->flags;
-	d->c.preadv.offsets[0] = hdr_offset;
-	d->c.preadv.size = max_size;
-#ifdef USE_VM_PROBES
-	dt_i1 = d->fd;
-	dt_i2 = d->flags;
-	dt_i3 = d->c.preadv.offsets[0];
-	dt_i4 = d->c.preadv.size;
-#endif
-	res_ev = &d->c.preadv.eiov;
-	/* XXX possible alignment problems here for weird machines */
-	res_ev->iov = void_ptr = d + 1;
-	res_ev->binv = void_ptr = res_ev->iov + vsize;
-	res_ev->size = 0;
-	res_ev->vsize = 0;
-	d->invoke = invoke_ipread;
-	d->free = free_preadv;
-	d->level = 1;
-	cq_enq(desc, d);
-    } goto done; /* case FILE_IPREAD: */
-
-    case FILE_SETOPT: {
-	char opt;
-
-	if (ev->size < 1+1
-	    || !EV_GET_CHAR(ev, &opt, &p, &q)) {
-	    /* Buffer too short to contain even the option type */
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
-#ifdef USE_VM_PROBES
-	dt_i1 = opt;
-	dt_utag = EV_CHAR_P(ev, p, q);
-#endif
-	switch (opt) {
-	case FILE_OPT_DELAYED_WRITE: {
-	    Uint32 sizeH, sizeL, delayH, delayL;
-	    if (ev->size != 1+1+4*sizeof(Uint32)
-#ifdef USE_VM_PROBES
-		+ FILENAME_BYTELEN(dt_utag) + FILENAME_CHARSIZE
-#endif
-		|| !EV_GET_UINT32(ev, &sizeH, &p, &q)
-		|| !EV_GET_UINT32(ev, &sizeL, &p, &q)
-		|| !EV_GET_UINT32(ev, &delayH, &p, &q)
-		|| !EV_GET_UINT32(ev, &delayL, &p, &q)) {
-		/* Buffer has wrong length to contain the option values */
-		reply_posix_error(desc, EINVAL);
-		goto done;
-	    }
-#if SIZEOF_SIZE_T == 4
-	    if (sizeH != 0) {
-		reply_posix_error(desc, EINVAL);
-		goto done;
-	    }
-	    desc->write_bufsize = sizeL;
-#else
-	    desc->write_bufsize = ((size_t)sizeH << 32) | sizeL;
-#endif
-#if SIZEOF_LONG == 4
-	    if (delayH != 0) {
-		reply_posix_error(desc, EINVAL);
-		goto done;
-	    }
-	    desc->write_delay = delayL;
-#else
-	    desc->write_delay = ((unsigned long)delayH << 32) | delayL;
-#endif
-#ifdef USE_VM_PROBES
-	    dt_i2 = desc->write_delay;
-#endif
-	    TRACE_C('K');
-	    reply_ok(desc);
-	} goto done;
-	case FILE_OPT_READ_AHEAD: {
-	    Uint32 sizeH, sizeL;
-	    if (ev->size != 1+1+2*sizeof(Uint32)
-#ifdef USE_VM_PROBES
-		+ FILENAME_BYTELEN(dt_utag)+FILENAME_CHARSIZE
-#endif
-		|| !EV_GET_UINT32(ev, &sizeH, &p, &q)
-		|| !EV_GET_UINT32(ev, &sizeL, &p, &q)) {
-		/* Buffer has wrong length to contain the option values */
-		reply_posix_error(desc, EINVAL);
-		goto done;
-	    }
-#if SIZEOF_SIZE_T == 4
-	    if (sizeH != 0) {
-		reply_posix_error(desc, EINVAL);
-		goto done;
-	    }
-	    desc->read_bufsize = sizeL;
-#else
-	    desc->read_bufsize = ((size_t)sizeH << 32) | sizeL;
-#endif
-#ifdef USE_VM_PROBES
-	    dt_i2 = desc->read_bufsize;
-#endif
-	    TRACE_C('K');
-	    reply_ok(desc);
-	} goto done;
-	default:
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	} /* case FILE_OPT_DELAYED_WRITE: */
-    } ASSERT(0); goto done; /* case FILE_SETOPT: */
-
-    case FILE_SENDFILE: {
-
-#ifdef HAVE_SENDFILE
-        struct t_data *d;
-	Uint32 out_fd, offsetH, offsetL, hd_len, tl_len;
-	Uint64 nbytes;
-	char flags;
-
-	if (ev->size < 1 + 7 * sizeof(Uint32) + sizeof(char)
-		|| !EV_GET_UINT32(ev, &out_fd, &p, &q)
-		|| !EV_GET_CHAR(ev, &flags, &p, &q)
-		|| !EV_GET_UINT32(ev, &offsetH, &p, &q)
-		|| !EV_GET_UINT32(ev, &offsetL, &p, &q)
-		|| !EV_GET_UINT64(ev, &nbytes, &p, &q)
-		|| !EV_GET_UINT32(ev, &hd_len, &p, &q)
-		|| !EV_GET_UINT32(ev, &tl_len, &p, &q)) {
-	    /* Buffer has wrong length to contain all the needed values */
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
-
-	if (hd_len != 0 || tl_len != 0) {
-	    /* We do not allow header, trailers */
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
-
-	
-	if (flags & SENDFILE_FLGS_USE_THREADS && !THRDS_AVAILABLE) {
-	    /* We do not allow use_threads flag on a system where
-	       no threads are available. */
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
-
-	d = EF_SAFE_ALLOC(sizeof(struct t_data));
-	d->fd = desc->fd;
-	d->command = command;
-	d->invoke = invoke_sendfile;
-	d->free = free_sendfile;
-	d->flags = flags;
-	d->level = 2;
-
-	d->c.sendfile.out_fd = (int) out_fd;
-	d->c.sendfile.written = 0;
-	d->c.sendfile.port = desc->port;
-	d->c.sendfile.q_mtx = desc->q_mtx;
-
-    #if SIZEOF_OFF_T == 4
-	if (offsetH != 0) {
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
-	d->c.sendfile.offset = (off_t) offsetL;
-    #else
-	d->c.sendfile.offset = ((off_t) offsetH << 32) | offsetL;
-    #endif
-
-	d->c.sendfile.nbytes = nbytes;
-
-	if (USE_THRDS_FOR_SENDFILE(d)) {
-	    SET_BLOCKING(d->c.sendfile.out_fd);
-	} else {
-	    /**
-	     * Write a place holder to queue in order to force file_flush
-	     * to be called before the driver is closed.
-	     */
-	    char tmp[1] = "";
-	    MUTEX_LOCK(d->c.sendfile.q_mtx);
-	    if (driver_enq(d->c.sendfile.port, tmp, 1)) {
-	        MUTEX_UNLOCK(d->c.sendfile.q_mtx);
-		reply_posix_error(desc, ENOMEM);
-		goto done;
-	    }
-	    MUTEX_UNLOCK(d->c.sendfile.q_mtx);
-	}
-
-	cq_enq(desc, d);
-#else
-	reply_posix_error(desc, ENOTSUP);
-#endif
-	goto done;
-        } /* case FILE_SENDFILE: */
-
-    } /* switch(command) */
-
-    if (lseek_flush_read(desc, &err
-#ifdef USE_VM_PROBES
-			 , dt_priv, dt_utag
-#endif
-			 ) < 0) {
-	reply_posix_error(desc, err);
-	goto done;
-    }
-    if (flush_write_check_error(desc, &err
-#ifdef USE_VM_PROBES
-				, dt_priv, dt_utag
-#endif
-				) < 0) {
-	reply_posix_error(desc, err);
-	goto done;
-    } else {
-	/* Flatten buffer and send it to file_output(desc, buf, len) */
-	int len = ev->size;
-	char *buf = EF_ALLOC(len);
-	if (! buf) {
-	    reply_posix_error(desc, ENOMEM);
-	    goto done;
-	}
-	driver_vec_to_buf(ev, buf, len);
-	file_output((ErlDrvData) desc, buf, len);
-	EF_FREE(buf);
-	goto done;
-    }
-
- done:
-    if (d != NULL) {
-#ifdef USE_VM_PROBES
-	/*
-	 * If d == NULL, then either:
-	 *    1). There was an error of some sort, or
-	 *    2). The command given to us is actually implemented
-	 *	  by file_output() instead.
-	 *
-	 * Case #1 is probably a TODO item, perhaps?
-	 * Case #2 we definitely don't want to activate a probe.
-	 */
-	d->sched_i1 = dt_priv->thread_num;
-	d->sched_i2 = dt_priv->tag;
-	d->sched_utag[0] = '\0';
-	if (dt_utag != NULL) {
-	    if (dt_utag[0] == '\0') {
-                dt_utag = NULL;
-            } else {
-		strncpy(d->sched_utag, dt_utag, sizeof(d->sched_utag) - 1);
-		d->sched_utag[sizeof(d->sched_utag) - 1] = '\0';
-	    }
-	}
-	DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag++,
-		 dt_utag, command, dt_s1, NULL, dt_i1, dt_i2, dt_i3, dt_i4,
-		 desc->port_str);
-#endif
-    }
-    cq_execute(desc);
-}
-
-#ifdef  USE_VM_PROBES
-dt_private *
-get_dt_private(int base)
-{
-    dt_private *dt_priv = (dt_private *) pthread_getspecific(dt_driver_key);
-
-    if (dt_priv == NULL) {
-	dt_priv = EF_SAFE_ALLOC(sizeof(dt_private));
-	erts_mtx_lock(&dt_driver_mutex);
-	dt_priv->thread_num = (base + dt_driver_idnum++);
-	erts_mtx_unlock(&dt_driver_mutex);
-	dt_priv->tag = 0;
-	pthread_setspecific(dt_driver_key, dt_priv);
-    }
-    return dt_priv;
-}
-#endif  /* USE_VM_PROBES */
diff --git a/erts/emulator/drivers/common/erl_efile.h b/erts/emulator/drivers/common/erl_efile.h
deleted file mode 100644
index b7f063b4f2..0000000000
--- a/erts/emulator/drivers/common/erl_efile.h
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 1997-2016. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * %CopyrightEnd%
- */
-/*
- * Defines the interfaces between the generic efile driver and its
- * operating-system dependent helpers.
- */
-
-#include "sys.h"
-#include "erl_driver.h"
-
-/*
- * Open modes for efile_openfile().
- */
-#define EFILE_MODE_READ 1
-#define EFILE_MODE_WRITE 2	/* Implies truncating file when used alone. */
-#define EFILE_MODE_READ_WRITE 	3
-#define EFILE_MODE_APPEND	4
-#define EFILE_COMPRESSED 	8
-#define EFILE_MODE_EXCL        16
-#define EFILE_NO_TRUNCATE      32 /* Special for reopening on VxWorks */
-#define EFILE_MODE_SYNC        64
-
-/*
- * Seek modes for efile_seek().
- */
-#define	EFILE_SEEK_SET	0
-#define	EFILE_SEEK_CUR	1
-#define	EFILE_SEEK_END	2
-
-/*
- * File types returned by efile_fileinfo().
- */
-#define FT_DEVICE 	1
-#define FT_DIRECTORY 	2
-#define FT_REGULAR	3
-#define FT_SYMLINK	4
-#define FT_OTHER	5
-
-/*
- * Access attributes returned by efile_fileinfo() (the bits can be ORed
- * together).
- */
-#define FA_NONE    	0
-#define FA_WRITE 	1
-#define FA_READ		2
-
-/* Some OS'es (i.e. Windows) has filenames in wide charaqcters. That requires special handling */
-/* Note that we do *not* honor alignment in the communication to the OS specific driver, */
-/* which is not a problem on x86, but might be on other platforms. The OS specific efile */
-/* implementation is expected to align if needed */
-#ifdef __WIN32__
-#define FILENAMES_16BIT 1
-#endif
-
-/* We use sendfilev if it exist on solaris */
-#if !defined(HAVE_SENDFILE) && defined(HAVE_SENDFILEV)
-#define HAVE_SENDFILE
-#endif
-
-/*
- * An handle to an open directory.  To be cast to the correct type
- * in the system-dependent directory functions.
- */
-
-typedef struct _Efile_Dir_Handle* EFILE_DIR_HANDLE;
-
-/*
- * Error information from the last call.
- */  
-typedef struct _Efile_error {
-    int posix_errno;		/* Posix error number, as in <errno.h>. */
-    int os_errno;		/* Os-dependent error number (not used). */
-} Efile_error;
-
-/*
- * Describes what is returned by file:file_info/1.
- */
-
-typedef struct _Efile_info {
-    Uint32 size_low;		/* Size of file, lower 32 bits.. */
-    Uint32 size_high;		/* Size of file, higher 32 bits. */
-    Uint32 type;		/* Type of file -- one of FT_*. */
-    Uint32 access;		/* Access to file -- one of FA_*. */
-    Uint32 mode;		/* Access permissions -- bit field. */
-    Uint32 links;		/* Number of links to file. */
-    Uint32 major_device;	/* Major device or file system. */
-    Uint32 minor_device;	/* Minor device (for devices). */
-    Uint32 inode;		/* Inode number. */
-    Uint32 uid;			/* User id of owner. */
-    Uint32 gid;			/* Group id of owner. */
-    Sint64 accessTime;		/* Last time the file was accessed. */
-    Sint64 modifyTime;		/* Last time the file was modified. */
-    Sint64 cTime;		/* Creation time (Windows) or last
-				 * inode change (Unix).
-				 */
-} Efile_info;
-
-
-#ifdef HAVE_SENDFILE
-/*
- * Describes the structure of headers/trailers for sendfile
- */
-struct t_sendfile_hdtl {
-    SysIOVec *headers;
-    int hdr_cnt;
-    SysIOVec *trailers;
-    int trl_cnt;
-};
-#endif /* HAVE_SENDFILE */
-
-/*
- * Functions.
- */
-int efile_init(void);
-int efile_mkdir(Efile_error* errInfo, char* name);
-int efile_rmdir(Efile_error* errInfo, char* name);
-int efile_delete_file(Efile_error* errInfo, char* name);
-int efile_rename(Efile_error* errInfo, char* src, char* dst);
-int efile_chdir(Efile_error* errInfo, char* name);
-int efile_getdcwd(Efile_error* errInfo, int drive,
-		  char* buffer, size_t size);
-int efile_readdir(Efile_error* errInfo, char* name, 
-		  EFILE_DIR_HANDLE* dir_handle,
-		  char* buffer, size_t *size);
-int efile_openfile(Efile_error* errInfo, char* name, int flags,
-		   int* pfd, Sint64* pSize);
-void efile_closefile(int fd);
-int efile_fdatasync(Efile_error* errInfo, int fd);
-int efile_fsync(Efile_error* errInfo, int fd);
-int efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
-		   char *name, int info_for_link);
-int efile_write_info(Efile_error* errInfo, Efile_info* pInfo, char *name);
-int efile_write(Efile_error* errInfo, int flags, int fd, 
-		char* buf, size_t count);
-int efile_writev(Efile_error* errInfo, int flags, int fd, 
-		 SysIOVec* iov, int iovcnt);
-int efile_read(Efile_error* errInfo, int flags, int fd, 
-	       char* buf, size_t count, size_t* pBytesRead);
-int efile_seek(Efile_error* errInfo, int fd, 
-	       Sint64 offset, int origin, Sint64* new_location);
-int efile_truncate_file(Efile_error* errInfo, int *fd, int flags);
-int efile_pwrite(Efile_error* errInfo, int fd, 
-		 char* buf, size_t count, Sint64 offset);
-int efile_pread(Efile_error* errInfo, int fd, 
-		Sint64 offset, char* buf, size_t count, size_t* pBytesRead);
-int efile_readlink(Efile_error* errInfo, char *name, 
-		   char* buffer, size_t size);
-int efile_altname(Efile_error* errInfo, char *name, 
-		  char* buffer, size_t size);
-int efile_link(Efile_error* errInfo, char* old, char* new);
-int efile_symlink(Efile_error* errInfo, char* old, char* new);
-int efile_may_openfile(Efile_error* errInfo, char *name);
-int efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length,
-		  int advise);
-#ifdef HAVE_SENDFILE
-int efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd,
-		      off_t *offset, Uint64 *nbytes, struct t_sendfile_hdtl *hdtl);
-#endif /* HAVE_SENDFILE */
-int efile_fallocate(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length);
diff --git a/erts/emulator/drivers/common/gzio.c b/erts/emulator/drivers/common/gzio.c
index f60c781894..86c3b07cea 100644
--- a/erts/emulator/drivers/common/gzio.c
+++ b/erts/emulator/drivers/common/gzio.c
@@ -19,726 +19,16 @@
 #include <unistd.h>
 #endif
 #include <ctype.h>
+
 #include "erl_driver.h"
-#include "erl_efile.h"
 #include "sys.h"
 
-#ifdef __WIN32__
-#ifndef HAVE_CONFLICTING_FREAD_DECLARATION
-#define HAVE_CONFLICTING_FREAD_DECLARATION
-#endif
-#define FILENAMES_16BIT 1
-#endif
-
-#ifdef STDC
-#  define zstrerror(errnum) strerror(errnum)
-#else
-#  define zstrerror(errnum) ""
-#endif
-
 #include "gzio_zutil.h"
 #include "erl_zlib.h"
 #include "gzio.h"
 
-/********struct internal_state {int dummy;}; / * for buggy compilers */
-
-#define Z_BUFSIZE 4096
-
-#define ALLOC(size) driver_alloc(size)
-#define TRYFREE(p) {if (p) driver_free(p);}
-
 static int gz_magic[2] = {0x1f, 0x8b}; /* gzip magic header */
 
-/* gzip flag byte */
-#define ASCII_FLAG   0x01 /* bit 0 set: file probably ascii text */
-#define HEAD_CRC     0x02 /* bit 1 set: header CRC present */
-#define EXTRA_FIELD  0x04 /* bit 2 set: extra field present */
-#define ORIG_NAME    0x08 /* bit 3 set: original file name present */
-#define COMMENT      0x10 /* bit 4 set: file comment present */
-#define RESERVED     0xE0 /* bits 5..7: reserved */
-
-typedef struct gz_stream {
-    z_stream stream;
-    int      z_err;   /* error code for last stream operation */
-    int      z_eof;   /* set if end of input file */
-#ifdef UNIX
-    int      file;    /* .gz file descriptor */
-#else
-    FILE     *file;   /* .gz file */
-#endif
-    Byte     *inbuf;  /* input buffer */
-    Byte     *outbuf; /* output buffer */
-    uLong    crc;     /* crc32 of uncompressed data */
-    char     *msg;    /* error message */
-    char     *path;   /* path name for debugging only */
-    int      transparent; /* 1 if input file is not a .gz file */
-    char     mode;    /* 'w' or 'r' */
-    int      position; /* Position (for seek) */
-    int (*destroy)(struct gz_stream*); /* Function to destroy
-					    *  this structure. */
-} gz_stream;
-
-local ErtsGzFile gz_open      (const char *path, const char *mode);
-local int    get_byte     (gz_stream *s);
-local void   check_header (gz_stream *s);
-local int    destroy      (gz_stream *s);
-local uLong  getLong      (gz_stream *s);
-
-#ifdef UNIX
-/*
- * In Solaris 8 and earlier, fopen() and its friends cannot handle 
- * file descriptors larger than 255. Therefore, we use read()/write()
- * on all Unix systems.
- */
-# define ERTS_GZWRITE(File, Buf, Count) write((File), (Buf), (Count))
-# define ERTS_GZREAD(File, Buf, Count) read((File), (Buf), (Count))
-#else
-/*
- * On all other operating systems, using fopen(), fread()/fwrite(), since
- * there is not guaranteed to exist any read()/write() (not part of
- * ANSI/ISO-C).
- */
-# define ERTS_GZWRITE(File, Buf, Count) fwrite((Buf), 1, (Count), (File))
-# define ERTS_GZREAD(File, Buf, Count) fread((Buf), 1, (Count), (File))
-#endif
-
-/*
- * Ripped from efile_drv.c
- */
-
-#ifdef FILENAMES_16BIT
-#  define FILENAME_BYTELEN(Str) filename_len_16bit(Str)
-#  define FILENAME_COPY(To,From) filename_cpy_16bit((To),(From)) 
-#  define FILENAME_CHARSIZE 2
-
-   static int filename_len_16bit(const char *str) 
-   {
-       const char *p = str;
-       while(*p != '\0' || p[1] != '\0') {
-	   p += 2;
-       }
-       return (p - str);
-   }
-
-   static void filename_cpy_16bit(char *to, const char *from) 
-   {
-       while(*from != '\0' || from[1] != '\0') {
-	   *to++ = *from++;
-	   *to++ = *from++;
-       }
-       *to++ = *from++;
-       *to++ = *from++;
-   }
-
-#else
-#  define FILENAME_BYTELEN(Str) strlen(Str)
-#  define FILENAME_COPY(To,From) strcpy(To,From) 
-#  define FILENAME_CHARSIZE 1
-#endif
-
-/* ===========================================================================
-     Opens a gzip (.gz) file for reading or writing. The mode parameter
-   is as in fopen ("rb" or "wb"). The file is given either by file descriptor
-   or path name (if fd == -1).
-     gz_open return NULL if the file could not be opened or if there was
-   insufficient memory to allocate the (de)compression state; errno
-   can be checked to distinguish the two cases (if errno is zero, the
-   zlib error is Z_MEM_ERROR).
-*/
-local ErtsGzFile gz_open (path, mode)
-    const char *path;
-    const char *mode;
-{
-    int err;
-    int level = Z_DEFAULT_COMPRESSION; /* compression level */
-    char *p = (char*)mode;
-    gz_stream *s;
-    char fmode[80]; /* copy of mode, without the compression level */
-    char *m = fmode;
-
-    if (!path || !mode) return Z_NULL;
-
-    s = (gz_stream *)ALLOC(sizeof(gz_stream));
-    if (!s) return Z_NULL;
-
-    erl_zlib_alloc_init(&s->stream);
-    s->stream.next_in = s->inbuf = Z_NULL;
-    s->stream.next_out = s->outbuf = Z_NULL;
-    s->stream.avail_in = s->stream.avail_out = 0;
-#ifdef UNIX
-    s->file = -1;
-#else
-    s->file = NULL;
-#endif
-    s->z_err = Z_OK;
-    s->z_eof = 0;
-    s->crc = crc32(0L, Z_NULL, 0);
-    s->msg = NULL;
-    s->transparent = 0;
-    s->position = 0;
-    s->destroy = destroy;
-
-    s->path = (char*)ALLOC(FILENAME_BYTELEN(path)+FILENAME_CHARSIZE);
-    if (s->path == NULL) {
-        return s->destroy(s), (ErtsGzFile)Z_NULL;
-    }
-    FILENAME_COPY(s->path, path); /* do this early for debugging */
-
-    s->mode = '\0';
-    do {
-        if (*p == 'r')
-	    s->mode = 'r';
-	if (*p == 'w' || *p == 'a')
-	    s->mode = 'w';
-	if (isdigit((int)*p)) {
-	    level = *p - '0';
-	} else {
-	    *m++ = *p;		/* Copy the mode */
-	}
-    } while (*p++ && m < fmode + sizeof(fmode) - 1);
-    *m = '\0';
-    if (s->mode == '\0')
-	return s->destroy(s), (ErtsGzFile)Z_NULL;
-    
-    if (s->mode == 'w') {
-        err = deflateInit2(&(s->stream), level,
-                           Z_DEFLATED, MAX_WBITS+16, DEF_MEM_LEVEL, 0);
-        /* windowBits is passed < 0 to suppress zlib header */
-
-        s->stream.next_out = s->outbuf = (Byte*)ALLOC(Z_BUFSIZE);
-
-        if (err != Z_OK || s->outbuf == Z_NULL) {
-            return s->destroy(s), (ErtsGzFile)Z_NULL;
-        }
-    } else {
-	/*
-	 * It is tempting to use the built-in support in zlib
-	 * for handling GZIP headers, but unfortunately it
-	 * cannot handle multiple GZIP headers (which occur when
-	 * several GZIP files have been concatenated).
-	 */
-
-	err = inflateInit2(&(s->stream), -MAX_WBITS);
-        s->stream.next_in  = s->inbuf = (Byte*)ALLOC(Z_BUFSIZE);
-
-        if (err != Z_OK || s->inbuf == Z_NULL) {
-            return s->destroy(s), (ErtsGzFile)Z_NULL;
-        }
-    }
-    s->stream.avail_out = Z_BUFSIZE;
-
-    errno = 0;
-#if defined(FILENAMES_16BIT)
-    {
-	FILE* efile_wfopen(const WCHAR* name, const WCHAR* mode);
-	WCHAR wfmode[80];
-	int i = 0;
-	int j;
-	for(j = 0; fmode[j] != '\0'; ++j) {
-	    wfmode[i++] = (WCHAR) fmode[j];
-	}
-	wfmode[i++] = L'\0';
-	s->file = efile_wfopen((WCHAR *)path, wfmode);
-	if (s->file == NULL) {
-	    return s->destroy(s), (ErtsGzFile)Z_NULL;
-	}
-    }
-#elif defined(UNIX)
-    if (s->mode == 'r') {
-	s->file = open(path, O_RDONLY);
-    } else {
-	s->file = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0666);
-    }
-    if (s->file == -1) {
-        return s->destroy(s), (ErtsGzFile)Z_NULL;
-    }
-#else
-    s->file = fopen(path, fmode);
-    if (s->file == NULL) {
-        return s->destroy(s), (ErtsGzFile)Z_NULL;
-    }
-#endif
-    if (s->mode == 'r') {
-	check_header(s); /* skip the .gz header */
-    }
-    return (ErtsGzFile)s;
-}
-
-/* ===========================================================================
-   Rewind a gzfile back to the beginning.
-*/
-
-local int gz_rewind (gz_stream *s)
-{
-    TRYFREE(s->msg);
-
-#ifdef UNIX
-    lseek(s->file, 0L, SEEK_SET);
-#else
-    fseek(s->file, 0L, SEEK_SET);
-#endif
-    inflateReset(&(s->stream));
-    s->stream.next_in = Z_NULL;
-    s->stream.next_out = Z_NULL;
-    s->stream.avail_in = s->stream.avail_out = 0;
-    s->z_err = Z_OK;
-    s->z_eof = 0;
-    s->crc = crc32(0L, Z_NULL, 0);
-    s->msg = NULL;
-    s->position = 0;
-    s->stream.next_in = s->inbuf;
-
-    s->stream.avail_out = Z_BUFSIZE;
-
-    check_header(s);		/* skip the .gz header */
-    return 1;
-}
-
-/* ===========================================================================
-     Opens a gzip (.gz) file for reading or writing.
-*/
-ErtsGzFile erts_gzopen (path, mode)
-    const char *path;
-    const char *mode;
-{
-    return gz_open (path, mode);
-}
-
-
-/* ===========================================================================
-     Read a byte from a gz_stream; update next_in and avail_in. Return EOF
-   for end of file.
-   IN assertion: the stream s has been successfully opened for reading.
-*/
-local int get_byte(s)
-    gz_stream *s;
-{
-    if (s->z_eof) return EOF;
-    if (s->stream.avail_in == 0) {
-#ifdef UNIX
-	ssize_t res;
-	errno = 0;
-	res = ERTS_GZREAD(s->file, s->inbuf, Z_BUFSIZE);
-	if (res == 0) {
-	    s->stream.avail_in = 0;
-	    s->z_eof = 1;
-	    return EOF;
-	} else if (res < 0) {
-	    s->stream.avail_in = 0;
-	    s->z_eof = 1;
-	    s->z_err = Z_ERRNO;
-	    return EOF;
-	} else {
-	    s->stream.avail_in = (uInt) res;
-	}
-#else
-	errno = 0;
-	s->stream.avail_in = ERTS_GZREAD(s->file, s->inbuf, Z_BUFSIZE);
-	if (s->stream.avail_in == 0) {
-	    s->z_eof = 1;
-	    if (s->file && ferror(s->file))
-		s->z_err = Z_ERRNO;
-	    return EOF;
-	}
-#endif
-	s->stream.next_in = s->inbuf;
-    }
-    s->stream.avail_in--;
-    return *(s->stream.next_in)++;
-}
-
-/* ===========================================================================
-      Check the gzip header of a gz_stream opened for reading. Set the stream
-    mode to transparent if the gzip magic header is not present; set s->err
-    to Z_DATA_ERROR if the magic header is present but the rest of the header
-    is incorrect.
-    IN assertion: the stream s has already been created sucessfully;
-       s->stream.avail_in is zero for the first time, but may be non-zero
-       for concatenated .gz files.
-*/
-local void check_header(s)
-    gz_stream *s;
-{
-    int method; /* method byte */
-    int flags;  /* flags byte */
-    uInt len;
-    int c;
-
-    /* Check the gzip magic header */
-    for (len = 0; len < 2; len++) {
-	c = get_byte(s);
-	if (c != gz_magic[len]) {
-	    if (len != 0) s->stream.avail_in++, s->stream.next_in--;
-	    if (c != EOF) {
-		s->stream.avail_in++, s->stream.next_in--;
-		s->transparent = 1;
-	    }
-	    s->z_err = s->stream.avail_in != 0 ? Z_OK : Z_STREAM_END;
-	    return;
-	}
-    }
-    method = get_byte(s);
-    flags = get_byte(s);
-    if (method != Z_DEFLATED || (flags & RESERVED) != 0) {
-	s->z_err = Z_DATA_ERROR;
-	return;
-    }
-
-    /* Discard time, xflags and OS code: */
-    for (len = 0; len < 6; len++) (void)get_byte(s);
-
-    if ((flags & EXTRA_FIELD) != 0) { /* skip the extra field */
-	len  =  (uInt)get_byte(s);
-	len += ((uInt)get_byte(s))<<8;
-	/* len is garbage if EOF but the loop below will quit anyway */
-	while (len-- != 0 && get_byte(s) != EOF) ;
-    }
-    if ((flags & ORIG_NAME) != 0) { /* skip the original file name */
-	while ((c = get_byte(s)) != 0 && c != EOF) ;
-    }
-    if ((flags & COMMENT) != 0) {   /* skip the .gz file comment */
-	while ((c = get_byte(s)) != 0 && c != EOF) ;
-    }
-    if ((flags & HEAD_CRC) != 0) {  /* skip the header crc */
-	for (len = 0; len < 2; len++) (void)get_byte(s);
-    }
-    s->z_err = s->z_eof ? Z_DATA_ERROR : Z_OK;
-}
-
- /* ===========================================================================
- * Cleanup then free the given gz_stream. Return a zlib error code.
-   Try freeing in the reverse order of allocations.
- */
-local int destroy (s)
-    gz_stream *s;
-{
-    int err = Z_OK;
-
-    if (!s) return Z_STREAM_ERROR;
-
-    TRYFREE(s->msg);
-
-    if (s->stream.state != NULL) {
-       if (s->mode == 'w') {
-           err = deflateEnd(&(s->stream));
-       } else if (s->mode == 'r') {
-           err = inflateEnd(&(s->stream));
-       }
-    }
-#ifdef UNIX
-    if (s->file != -1 && close(s->file)) {
-        err = Z_ERRNO;
-    }
-#else
-    if (s->file != NULL && fclose(s->file)) {
-        err = Z_ERRNO;
-    }
-#endif
-    if (s->z_err < 0) err = s->z_err;
-
-    TRYFREE(s->inbuf);
-    TRYFREE(s->outbuf);
-    TRYFREE(s->path);
-    TRYFREE(s);
-    return err;
-}
-
-/* ===========================================================================
-     Reads the given number of uncompressed bytes from the compressed file.
-   gzread returns the number of bytes actually read (0 for end of file).
-*/
-int
-erts_gzread(ErtsGzFile file, voidp buf, unsigned len)
-{
-    gz_stream *s = (gz_stream*)file;
-    Bytef *start = buf; /* starting point for crc computation */
-    Byte  *next_out; /* == stream.next_out but not forced far (for MSDOS) */
-
-    if (s == NULL || s->mode != 'r') return Z_STREAM_ERROR;
-
-    if (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO) return -1;
-    if (s->z_err == Z_STREAM_END) return 0;  /* EOF */
-
-    s->stream.next_out = next_out = buf;
-    s->stream.avail_out = len;
-
-    while (s->stream.avail_out != 0) {
-
-	if (s->transparent) {
-	    /* Copy first the lookahead bytes: */
-	    uInt n = s->stream.avail_in;
-	    if (n > s->stream.avail_out) n = s->stream.avail_out;
-	    if (n > 0) {
-		zmemcpy(s->stream.next_out, s->stream.next_in, n);
-		next_out += n;
-		s->stream.next_out = next_out;
-		s->stream.next_in   += n;
-		s->stream.avail_out -= n;
-		s->stream.avail_in  -= n;
-	    }
-	    if (s->stream.avail_out > 0) {
-		s->stream.avail_out -= ERTS_GZREAD(s->file, next_out,
-						   s->stream.avail_out);
-	    }
-	    len -= s->stream.avail_out;
-	    s->stream.total_in  += (uLong)len;
-	    s->stream.total_out += (uLong)len;
-            if (len == 0) s->z_eof = 1;
-	    s->position += (int)len;
-	    return (int)len;
-	}
-        if (s->stream.avail_in == 0 && !s->z_eof) {
-#ifdef UNIX
-	    ssize_t res;
-	    errno = 0;
-	    res = ERTS_GZREAD(s->file, s->inbuf, Z_BUFSIZE);
-	    if (res == 0) {
-		s->stream.avail_in = 0;
-		s->z_eof = 1;
-		return EOF;
-	    } else if (res < 0) {
-		s->stream.avail_in = 0;
-		s->z_eof = 1;
-		s->z_err = Z_ERRNO;
-		return EOF;
-	    } else {
-		s->stream.avail_in = (uInt) res;
-	    }
-#else
-	    errno = 0;
-            s->stream.avail_in = ERTS_GZREAD(s->file, s->inbuf, Z_BUFSIZE);
-            if (s->stream.avail_in == 0) {
-                s->z_eof = 1;
-		if (s->file && ferror(s->file)) {
-		    s->z_err = Z_ERRNO;
-		    break;
-		}
-            }
-#endif
-            s->stream.next_in = s->inbuf;
-        }
-        s->z_err = inflate(&(s->stream), Z_NO_FLUSH);
-
-	if (s->z_err == Z_STREAM_END) {
-	    /* Check CRC and original size */
-	    s->crc = crc32(s->crc, start, (uInt)(s->stream.next_out - start));
-	    start = s->stream.next_out;
-
-	    if (getLong(s) != s->crc) {
-		s->z_err = Z_DATA_ERROR;
-	    } else {
-	        (void)getLong(s);
-                /* The uncompressed length returned by above getlong() may
-                 * be different from s->stream.total_out) in case of
-		 * concatenated .gz files. Check for such files:
-		 */
-		check_header(s);
-		if (s->z_err == Z_OK) {
-		    uLong total_in = s->stream.total_in;
-		    uLong total_out = s->stream.total_out;
-
-		    inflateReset(&(s->stream));
-		    s->stream.total_in = total_in;
-		    s->stream.total_out = total_out;
-		    s->crc = crc32(0L, Z_NULL, 0);
-		}
-	    }
-	}
-	if (s->z_err != Z_OK || s->z_eof) break;
-    }
-    s->crc = crc32(s->crc, start, (uInt)(s->stream.next_out - start));
-
-    s->position += (int)(len - s->stream.avail_out);
-
-    return (int)(len - s->stream.avail_out);
-}
-
-/* ===========================================================================
-     Writes the given number of uncompressed bytes into the compressed file.
-   gzwrite returns the number of bytes actually written (0 in case of error).
-*/
-int
-erts_gzwrite(ErtsGzFile file, voidp buf, unsigned len)
-{
-    gz_stream *s = (gz_stream*)file;
-
-    if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR;
-
-    s->stream.next_in = buf;
-    s->stream.avail_in = len;
-
-    while (s->stream.avail_in != 0) {
-
-        if (s->stream.avail_out == 0) {
-
-            s->stream.next_out = s->outbuf;
-            if (ERTS_GZWRITE(s->file, s->outbuf, Z_BUFSIZE) != Z_BUFSIZE) {
-                s->z_err = Z_ERRNO;
-                break;
-            }
-            s->stream.avail_out = Z_BUFSIZE;
-        }
-        s->z_err = deflate(&(s->stream), Z_NO_FLUSH);
-        if (s->z_err != Z_OK) break;
-    }
-    s->position += (int)(len - s->stream.avail_in);
-    return (int)(len - s->stream.avail_in);
-}
-
-/*
- * For use by Erlang file driver.
- *
- * XXX Limitations:
- *  - SEEK_END is not allowed (length of file is not known).
- *  - When writing, only forward seek is supported.
- */
-
-int
-erts_gzseek(ErtsGzFile file, int offset, int whence)
-{
-    int pos;
-    gz_stream* s = (gz_stream *) file;
-
-    switch (whence) {
-    case EFILE_SEEK_SET: whence = SEEK_SET; break;
-    case EFILE_SEEK_CUR: whence = SEEK_CUR; break;
-    case EFILE_SEEK_END: whence = SEEK_END; break;
-    default:
-	errno = EINVAL;
-	return -1;
-    }
-
-    if (s == NULL) {
-	errno = EINVAL;
-	return -1;
-    }
-    if (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO) {
-	errno = EIO;
-	return -1;
-    }
-
-    switch (whence) {
-    case SEEK_SET: pos = offset; break;
-    case SEEK_CUR: pos = s->position+offset; break;
-    case SEEK_END: 
-    default:
-      errno = EINVAL; return -1;
-    }
-
-    if (pos == s->position) {
-	return pos;
-    }
-
-    if (pos < s->position) {
-	if (s->mode == 'w') {
-	    errno = EINVAL;
-	    return -1;
-	}
-	gz_rewind(s);
-    }
-
-    while (s->position < pos) {
-	char buf[512];
-	int n;
-	int save_pos = s->position;
-
-	n = pos - s->position;
-	if (n > sizeof(buf))
-	    n = sizeof(buf);
-
-	if (s->mode == 'r') {
-	    erts_gzread(file, buf, n);
-	} else {
-	    memset(buf, '\0', n);
-	    erts_gzwrite(file, buf, n);
-	}
-	if (save_pos == s->position) break;
-    }
-
-    return s->position;
-}
-
-/* ===========================================================================
-     Flushes all pending output into the compressed file. The parameter
-   flush is as in the deflate() function.
-     gzflush should be called only when strictly necessary because it can
-   degrade compression.
-*/
-int
-erts_gzflush(ErtsGzFile file, int flush)
-{
-    uInt len;
-    int done = 0;
-    gz_stream *s = (gz_stream*)file;
-
-    if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR;
-
-    s->stream.avail_in = 0; /* should be zero already anyway */
-
-    for (;;) {
-        len = Z_BUFSIZE - s->stream.avail_out;
-
-        if (len != 0) {
-            if ((uInt)ERTS_GZWRITE(s->file, s->outbuf, len) != len) {
-                s->z_err = Z_ERRNO;
-                return Z_ERRNO;
-            }
-            s->stream.next_out = s->outbuf;
-            s->stream.avail_out = Z_BUFSIZE;
-        }
-        if (done) break;
-        s->z_err = deflate(&(s->stream), flush);
-
-        /* deflate has finished flushing only when it hasn't used up
-         * all the available space in the output buffer: 
-         */
-        done = (s->stream.avail_out != 0 || s->z_err == Z_STREAM_END);
- 
-        if (s->z_err != Z_OK && s->z_err != Z_STREAM_END) break;
-    }
-#ifndef UNIX
-    fflush(s->file);
-#endif
-    return  s->z_err == Z_STREAM_END ? Z_OK : s->z_err;
-}
-
-/* ===========================================================================
-   Reads a long in LSB order from the given gz_stream. Sets 
-*/
-local uLong getLong (s)
-    gz_stream *s;
-{
-    uLong x = (uLong)get_byte(s);
-    int c;
-
-    x += ((uLong)get_byte(s))<<8;
-    x += ((uLong)get_byte(s))<<16;
-    c = get_byte(s);
-    if (c == EOF) s->z_err = Z_DATA_ERROR;
-    x += ((uLong)c)<<24;
-    return x;
-}
-
-/* ===========================================================================
-     Flushes all pending output if necessary, closes the compressed file
-   and deallocates all the (de)compression state.
-*/
-int
-erts_gzclose(ErtsGzFile file)
-{
-    int err;
-    gz_stream *s = (gz_stream*)file;
-
-    if (s == NULL) return Z_STREAM_ERROR;
-
-    if (s->mode == 'w') {
-        err = erts_gzflush (file, Z_FINISH);
-        if (err != Z_OK) return s->destroy(s);
-    }
-    return s->destroy(s);
-}
-
-
 /* ===========================================================================
    Uncompresses the buffer given and returns a pointer to a binary.
    If the buffer was not compressed with gzip, the buffer contents
diff --git a/erts/emulator/drivers/common/gzio.h b/erts/emulator/drivers/common/gzio.h
index ee0ebe7bd8..e331b5208b 100644
--- a/erts/emulator/drivers/common/gzio.h
+++ b/erts/emulator/drivers/common/gzio.h
@@ -20,13 +20,5 @@
 
 #include "zlib.h"
 
-typedef struct erts_gzFile* ErtsGzFile;
-
-ErtsGzFile erts_gzopen (const char *path, const char *mode);
-int erts_gzread(ErtsGzFile file, voidp buf, unsigned len);
-int erts_gzwrite(ErtsGzFile file, voidp buf, unsigned len);
-int erts_gzseek(ErtsGzFile, int, int);
-int erts_gzflush(ErtsGzFile file, int flush);
-int erts_gzclose(ErtsGzFile file);
 ErlDrvBinary* erts_gzinflate_buffer(char*, uLong);
 ErlDrvBinary* erts_gzdeflate_buffer(char*, uLong);
diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 95d61fcc5d..4294fb4f46 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -63,6 +63,20 @@
 #include <sys/un.h>
 #endif
 
+#ifdef HAVE_SENDFILE
+#if defined(__linux__) || (defined(__sun) && defined(__SVR4))
+    #include <sys/sendfile.h>
+#elif defined(__FreeBSD__) || defined(__DragonFly__)
+    /* Need to define __BSD_VISIBLE in order to expose prototype of sendfile */
+    #define __BSD_VISIBLE 1
+    #include <sys/socket.h>
+#endif
+#endif
+
+#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__)
+    #define __DARWIN__ 1
+#endif
+
 /* All platforms fail on malloc errors. */
 #define FATAL_MALLOC
 
@@ -701,6 +715,7 @@ static size_t my_strnlen(const char *s, size_t maxlen)
 #define TCP_REQ_RECV           42
 #define TCP_REQ_UNRECV         43
 #define TCP_REQ_SHUTDOWN       44
+#define TCP_REQ_SENDFILE       45
 /* UDP and SCTP requests */
 #define PACKET_REQ_RECV        60 /* Common for UDP and SCTP         */
 /* #define SCTP_REQ_LISTEN       61 MERGED Different from TCP; not for UDP */
@@ -723,6 +738,7 @@ static size_t my_strnlen(const char *s, size_t maxlen)
 #define TCP_ADDF_DELAYED_ECONNRESET 128 /* An ECONNRESET error occurred on send or shutdown */
 #define TCP_ADDF_SHUTDOWN_WR_DONE 256 /* A shutdown(sock, SHUT_WR) or SHUT_RDWR was made */
 #define TCP_ADDF_LINGER_ZERO 	  512 /* Discard driver queue on port close */
+#define TCP_ADDF_SENDFILE         1024 /* Send from an fd instead of the driver queue */
 
 /* *_REQ_* replies */
 #define INET_REP_ERROR       0
@@ -1235,6 +1251,21 @@ typedef struct {
     inet_async_multi_op *multi_first;/* NULL == no multi-accept-queue, op is in ordinary queue */
     inet_async_multi_op *multi_last;
     MultiTimerData *mtd;        /* Timer structures for multiple accept */
+#ifdef HAVE_SENDFILE
+    struct {
+        ErlDrvSizeT ioq_skip;   /* The number of bytes in the queue at the time
+                                 * sendfile was issued, which must be sent
+                                 * before issuing the sendfile call itself. */
+        int dup_file_fd;        /* The file handle to send from; this is
+                                 * duplicated when sendfile is issued to
+                                 * reduce (but not eliminate) the impact of a
+                                 * nasty race, so we have to remember to close
+                                 * it. */
+        Uint64 bytes_sent;
+        Uint64 offset;
+        Uint64 length;
+    } sendfile;
+#endif
 } tcp_descriptor;
 
 /* send function */
@@ -1245,6 +1276,8 @@ static int tcp_deliver(tcp_descriptor* desc, int len);
 
 static int tcp_shutdown_error(tcp_descriptor* desc, int err);
 
+static int tcp_inet_sendfile(tcp_descriptor* desc);
+
 static int tcp_inet_output(tcp_descriptor* desc, HANDLE event);
 static int tcp_inet_input(tcp_descriptor* desc, HANDLE event);
 
@@ -1329,6 +1362,9 @@ static ErlDrvTermData am_ipv6_v6only;
 static ErlDrvTermData am_netns;
 static ErlDrvTermData am_bind_to_device;
 #endif
+#ifdef HAVE_SENDFILE
+static ErlDrvTermData am_sendfile;
+#endif
 
 static char str_eafnosupport[] = "eafnosupport";
 static char str_einval[] = "einval";
@@ -3875,6 +3911,10 @@ static int inet_init()
     INIT_ATOM(https);
     INIT_ATOM(scheme);
 
+#ifdef HAVE_SENDFILE
+    INIT_ATOM(sendfile);
+#endif
+
     /* add TCP, UDP and SCTP drivers */
     add_driver_entry(&tcp_inet_driver_entry);
 #ifdef HAVE_UDP
@@ -9270,6 +9310,13 @@ static void tcp_inet_stop(ErlDrvData e)
  * will be freed through tcp_inet_stop later on. */
 static void tcp_desc_close(tcp_descriptor* desc)
 {
+#ifdef HAVE_SENDFILE
+    if(desc->tcp_add_flags & TCP_ADDF_SENDFILE) {
+        desc->tcp_add_flags &= ~TCP_ADDF_SENDFILE;
+        close(desc->sendfile.dup_file_fd);
+    }
+#endif
+
     tcp_clear_input(desc);
     tcp_clear_output(desc);
 
@@ -9608,6 +9655,60 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd,
 	    return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
 	}
     }
+
+    case TCP_REQ_SENDFILE: {
+#ifdef HAVE_SENDFILE
+        const ErlDrvSizeT required_len =
+            sizeof(desc->sendfile.dup_file_fd) +
+            sizeof(Uint64) * 2;
+
+        int raw_file_fd;
+
+        DEBUGF(("tcp_inet_ctl(%ld): SENDFILE\r\n", (long)desc->inet.port));
+
+        if (len != required_len) {
+            return ctl_error(EINVAL, rbuf, rsize);
+        } else if (!IS_CONNECTED(INETP(desc))) {
+            return ctl_error(ENOTCONN, rbuf, rsize);
+        }
+
+        sys_memcpy(&raw_file_fd, buf, sizeof(raw_file_fd));
+        buf += sizeof(raw_file_fd);
+
+        desc->sendfile.dup_file_fd = dup(raw_file_fd);
+
+        if(desc->sendfile.dup_file_fd == -1) {
+            return ctl_error(errno, rbuf, rsize);
+        }
+
+        desc->sendfile.offset = get_int64(buf);
+        buf += sizeof(Uint64);
+
+        desc->sendfile.length = get_int64(buf);
+        buf += sizeof(Uint64);
+
+        ASSERT(desc->sendfile.offset >= 0);
+        ASSERT(desc->sendfile.length >= 0);
+
+        desc->sendfile.ioq_skip = driver_sizeq(desc->inet.port);
+        desc->sendfile.bytes_sent = 0;
+
+        desc->inet.caller = driver_caller(desc->inet.port);
+        desc->tcp_add_flags |= TCP_ADDF_SENDFILE;
+
+        /* See if we can finish sending without selecting & rescheduling. */
+        tcp_inet_sendfile(desc);
+
+        if(desc->sendfile.length > 0) {
+            sock_select(INETP(desc), FD_WRITE, 1);
+        }
+
+        return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
+#else
+        return ctl_error(ENOTSUP, rbuf, rsize);
+#endif
+    }
+
     default:
 	DEBUGF(("tcp_inet_ctl(%ld): %u\r\n", (long)desc->inet.port, cmd)); 
 	return inet_ctl(INETP(desc), cmd, buf, len, rbuf, rsize);
@@ -9747,12 +9848,27 @@ static void tcp_inet_commandv(ErlDrvData e, ErlIOVec* ev)
 static void tcp_inet_flush(ErlDrvData e)
 {
     tcp_descriptor* desc = (tcp_descriptor*)e;
-    if (!(desc->inet.event_mask & FD_WRITE)) {
-	/* Discard send queue to avoid hanging port (OTP-7615) */
-	tcp_clear_output(desc);
+    int discard_output;
+
+    /* Discard send queue to avoid hanging port (OTP-7615) */
+    discard_output = !(desc->inet.event_mask & FD_WRITE);
+
+    discard_output |= desc->tcp_add_flags & TCP_ADDF_LINGER_ZERO;
+
+#ifdef HAVE_SENDFILE
+    /* The old file driver aborted when it was stopped during sendfile, so
+     * we'll clear the flag and discard all output. */
+    if(desc->tcp_add_flags & TCP_ADDF_SENDFILE) {
+        desc->tcp_add_flags &= ~TCP_ADDF_SENDFILE;
+        close(desc->sendfile.dup_file_fd);
+
+        discard_output = 1;
+    }
+#endif
+
+    if (discard_output) {
+        tcp_clear_output(desc);
     }
-    if (desc->tcp_add_flags & TCP_ADDF_LINGER_ZERO)
-	tcp_clear_output(desc);
 }
 
 static void tcp_inet_process_exit(ErlDrvData e, ErlDrvMonitor *monitorp) 
@@ -10647,7 +10763,9 @@ static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev)
 	ev->size += h_len;
     }
 
-    if ((sz = driver_sizeq(ix)) > 0) {
+    sz = driver_sizeq(ix);
+
+    if ((desc->tcp_add_flags & TCP_ADDF_SENDFILE) || sz > 0) {
 	driver_enqv(ix, ev, 0);
 	if (sz+ev->size >= desc->high) {
 	    DEBUGF(("tcp_sendv(%ld): s=%d, sender forced busy\r\n",
@@ -10741,8 +10859,9 @@ static int tcp_send(tcp_descriptor* desc, char* ptr, ErlDrvSizeT len)
 
     inet_output_count(INETP(desc), len+h_len);
 
+    sz = driver_sizeq(ix);
 
-    if ((sz = driver_sizeq(ix)) > 0) {
+    if ((desc->tcp_add_flags & TCP_ADDF_SENDFILE) || sz > 0) {
 	if (h_len > 0)
 	    driver_enq(ix, buf, h_len);
 	driver_enq(ix, ptr, len);
@@ -10832,6 +10951,246 @@ static void tcp_inet_drv_input(ErlDrvData data, ErlDrvEvent event)
     (void)tcp_inet_input((tcp_descriptor*)data, (HANDLE)event);
 }
 
+#ifdef HAVE_SENDFILE
+static int tcp_sendfile_completed(tcp_descriptor* desc) {
+    ErlDrvTermData spec[LOAD_PORT_CNT + LOAD_TUPLE_CNT * 2 +
+        LOAD_ATOM_CNT * 2 + LOAD_UINT_CNT * 2];
+    Uint32 sent_low, sent_high;
+    int i;
+
+    desc->tcp_add_flags &= ~TCP_ADDF_SENDFILE;
+    close(desc->sendfile.dup_file_fd);
+
+    /* While we flushed the output queue prior to sending the file, we've
+     * deferred clearing busy status until now as there's no point in doing so
+     * while we still have a file to send.
+     *
+     * The watermark is checked since more data may have been added while we
+     * were sending the file. */
+
+    if (driver_sizeq(desc->inet.port) <= desc->low) {
+        if (IS_BUSY(INETP(desc))) {
+            desc->inet.caller = desc->inet.busy_caller;
+            desc->inet.state &= ~INET_F_BUSY;
+
+            set_busy_port(desc->inet.port, 0);
+
+            /* if we have a timer then cancel and send ok to client */
+            if (desc->busy_on_send) {
+                driver_cancel_timer(desc->inet.port);
+                desc->busy_on_send = 0;
+            }
+
+            inet_reply_ok(INETP(desc));
+        }
+    }
+
+    if (driver_sizeq(desc->inet.port) == 0) {
+        sock_select(INETP(desc), FD_WRITE, 0);
+        send_empty_out_q_msgs(INETP(desc));
+
+        if (desc->tcp_add_flags & TCP_ADDF_PENDING_SHUTDOWN) {
+            tcp_shutdown_async(desc);
+        }
+    }
+
+    sent_low = ((Uint64)desc->sendfile.bytes_sent >> 0) & 0xFFFFFFFF;
+    sent_high = ((Uint64)desc->sendfile.bytes_sent >> 32) & 0xFFFFFFFF;
+
+    i = LOAD_ATOM(spec, 0, am_sendfile);
+    i = LOAD_PORT(spec, i, desc->inet.dport);
+    i = LOAD_ATOM(spec, i, am_ok);
+    i = LOAD_UINT(spec, i, sent_low);
+    i = LOAD_UINT(spec, i, sent_high);
+    i = LOAD_TUPLE(spec, i, 3);
+    i = LOAD_TUPLE(spec, i, 3);
+
+    ASSERT(i == sizeof(spec)/sizeof(*spec));
+
+    return erl_drv_output_term(desc->inet.dport, spec, i);
+}
+
+static int tcp_sendfile_aborted(tcp_descriptor* desc, int socket_error) {
+    ErlDrvTermData spec[LOAD_PORT_CNT + LOAD_TUPLE_CNT * 2 + LOAD_ATOM_CNT * 3];
+    int i;
+
+    /* We don't clean up sendfile state here, as that's done in tcp_desc_close
+     * following normal error handling. All we do here is report the failure. */
+
+    i = LOAD_ATOM(spec, 0, am_sendfile);
+    i = LOAD_PORT(spec, i, desc->inet.dport);
+    i = LOAD_ATOM(spec, i, am_error);
+
+    switch (socket_error) {
+    case ECONNRESET:
+    case ENOTCONN:
+    case EPIPE:
+        i = LOAD_ATOM(spec, i, am_closed);
+        break;
+    default:
+        i = LOAD_ATOM(spec, i, error_atom(socket_error));
+    }
+
+    i = LOAD_TUPLE(spec, i, 2);
+    i = LOAD_TUPLE(spec, i, 3);
+
+    ASSERT(i == sizeof(spec)/sizeof(*spec));
+
+    return erl_drv_output_term(desc->inet.dport, spec, i);
+}
+
+static int tcp_inet_sendfile(tcp_descriptor* desc) {
+    ErlDrvPort ix = desc->inet.port;
+    int result = 0;
+    ssize_t n;
+
+    DEBUGF(("tcp_inet_sendfile(%ld) {s=%d\r\n", (long)ix, desc->inet.s));
+
+    /* If there was any data in the queue by the time sendfile was issued,
+     * we'll need to skip it first. Note that we don't clear busy status until
+     * we're finished sending the file. */
+    while (desc->sendfile.ioq_skip > 0) {
+        ssize_t bytes_to_send;
+        SysIOVec* iov;
+        int vsize;
+
+        ASSERT(driver_sizeq(ix) >= desc->sendfile.ioq_skip);
+
+        if ((iov = driver_peekq(ix, &vsize)) == NULL) {
+            ERTS_INTERNAL_ERROR("ioq empty when sendfile.ioq_skip > 0");
+        }
+
+        bytes_to_send = MIN(desc->sendfile.ioq_skip, iov[0].iov_len);
+        n = sock_send(desc->inet.s, iov[0].iov_base, bytes_to_send, 0);
+
+        if (!IS_SOCKET_ERROR(n)) {
+            desc->sendfile.ioq_skip -= n;
+            driver_deq(ix, n);
+        } else if (sock_errno() == ERRNO_BLOCK) {
+#ifdef __WIN32__
+            desc->inet.send_would_block = 1;
+#endif
+            goto done;
+        } else if (sock_errno() != EINTR) {
+            goto socket_error;
+        }
+    }
+
+    while (desc->sendfile.length > 0) {
+        /* For some reason the maximum ssize_t cannot be used as the max size.
+         * 1GB seems to work on all platforms */
+        const Sint64 SENDFILE_CHUNK_SIZE = ((1UL << 30) - 1);
+
+        ssize_t bytes_to_send = MIN(SENDFILE_CHUNK_SIZE, desc->sendfile.length);
+        off_t offset = desc->sendfile.offset;
+
+#if defined(__linux__)
+        n = sendfile(desc->inet.s, desc->sendfile.dup_file_fd, &offset,
+            bytes_to_send);
+#elif defined(__FreeBSD__) || defined(__DragonFly__) || defined(__DARWIN__)
+        {
+            off_t bytes_sent;
+            int error;
+
+    #if defined(__DARWIN__)
+            bytes_sent = bytes_to_send;
+
+            error = sendfile(desc->sendfile.dup_file_fd, desc->inet.s, offset,
+                &bytes_sent, NULL, 0);
+            n = bytes_sent;
+    #else
+            error = sendfile(desc->sendfile.dup_file_fd, desc->inet.s, offset,
+                bytes_to_send, NULL, &bytes_sent, 0);
+            n = bytes_sent;
+    #endif
+
+            if(error < 0) {
+                /* EAGAIN/EINTR report partial success by setting bytes_sent,
+                 * so we have to skip error handling if nonzero, and skip EOF
+                 * handling if zero, as it's possible that we didn't manage to
+                 * send anything at all before being interrupted by a
+                 * signal. */
+                if((errno != EAGAIN && errno != EINTR) || bytes_sent == 0) {
+                    n = -1;
+                }
+            }
+        }
+#elif defined(__sun) && defined(__SVR4) && defined(HAVE_SENDFILEV)
+        {
+            sendfilevec_t sfvec[1];
+            size_t bytes_sent;
+            ssize_t error;
+
+            sfvec[0].sfv_fd = desc->sendfile.dup_file_fd;
+            sfvec[0].sfv_len = bytes_to_send;
+            sfvec[0].sfv_off = offset;
+            sfvec[0].sfv_flag = 0;
+
+            error = sendfilev(desc->inet.s, sfvec, 1, &bytes_sent);
+            n = bytes_sent;
+
+            if(error < 0) {
+                if(errno == EINVAL) {
+                    /* On some solaris versions (I've seen it on SunOS 5.10),
+                     * using a sfv_len larger than the filesize will result in
+                     * a (-1 && errno == EINVAL). We translate this to a
+                     * successful send of the data.*/
+                } else {
+                    /* EAGAIN/EINTR behavior is identical to *BSD. */
+                    if((errno != EAGAIN && errno != EINTR) || bytes_sent == 0) {
+                        n = -1;
+                    }
+                }
+            }
+        }
+#else
+        #error "Unsupported sendfile syscall; update configure test."
+#endif
+
+        if (n > 0) {
+            desc->sendfile.bytes_sent += n;
+            desc->sendfile.offset += n;
+            desc->sendfile.length -= n;
+        } else if (n == 0) {
+            /* EOF. */
+            desc->sendfile.length = 0;
+            break;
+        } else if (IS_SOCKET_ERROR(n) && sock_errno() != EINTR) {
+            if (sock_errno() != ERRNO_BLOCK) {
+                goto socket_error;
+            }
+
+#ifdef __WIN32__
+            desc->inet.send_would_block = 1;
+#endif
+            break;
+        }
+    }
+
+    if (desc->sendfile.length == 0) {
+        tcp_sendfile_completed(desc);
+    }
+
+    goto done;
+
+socket_error: {
+        int socket_errno = sock_errno();
+
+        DEBUGF(("tcp_inet_sendfile(%ld): send errno = %d (errno %d)\r\n",
+            (long)desc->inet.port, socket_errno, errno));
+
+        result = tcp_send_error(desc, socket_errno);
+        tcp_sendfile_aborted(desc, socket_errno);
+
+        goto done;
+    }
+
+done:
+    DEBUGF(("tcp_inet_sendfile(%ld) }\r\n", (long)desc->inet.port));
+    return result;
+}
+#endif /* HAVE_SENDFILE */
+
 /* socket ready for ouput:
 ** 1. INET_STATE_CONNECTING => non block connect ?
 ** 2. INET_STATE_CONNECTED  => write output
@@ -10892,7 +11251,14 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event)
 	async_ok(INETP(desc));
     }
     else if (IS_CONNECTED(INETP(desc))) {
-	for (;;) {
+
+#ifdef HAVE_SENDFILE
+        if(desc->tcp_add_flags & TCP_ADDF_SENDFILE) {
+            return tcp_inet_sendfile(desc);
+        }
+#endif
+
+        for (;;) {
 	    int vsize;
 	    ssize_t n;
 	    SysIOVec* iov;
diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c
deleted file mode 100644
index 33e4d75ef7..0000000000
--- a/erts/emulator/drivers/unix/unix_efile.c
+++ /dev/null
@@ -1,1102 +0,0 @@
-/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 1997-2017. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * %CopyrightEnd%
- */
-/*
- * Purpose: Provides file and directory operations for Unix.
- */
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-#if defined(HAVE_POSIX_FALLOCATE) && !defined(__sun) && !defined(__sun__)
-#define _XOPEN_SOURCE 600
-#endif
-#if !defined(_GNU_SOURCE) && defined(HAVE_LINUX_FALLOC_H)
-#define _GNU_SOURCE
-#endif
-#include "sys.h"
-#include "erl_driver.h"
-#include "erl_efile.h"
-#include <utime.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SYS_UIO_H
-#include <sys/types.h>
-#include <sys/uio.h>
-#if defined(HAVE_SENDFILE) && (defined(__FreeBSD__) || defined(__DragonFly__))
-/* Need to define __BSD_VISIBLE in order to expose prototype of sendfile */
-#define __BSD_VISIBLE 1
-#include <sys/socket.h>
-#endif
-#endif
-#if defined(HAVE_SENDFILE) && (defined(__linux__) || (defined(__sun) && defined(__SVR4)))
-#include <sys/sendfile.h>
-#endif
-
-#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__)
-#define __DARWIN__ 1
-#endif
-
-#if defined(__DARWIN__) || defined(HAVE_LINUX_FALLOC_H) || defined(HAVE_POSIX_FALLOCATE)
-#include <fcntl.h>
-#endif
-
-#ifdef HAVE_LINUX_FALLOC_H
-#include <linux/falloc.h>
-#endif
-
-#ifdef SUNOS4
-#  define getcwd(buf, size) getwd(buf)
-#endif
-
-/* Find a definition of MAXIOV, that is used in the code later. */
-#if defined IOV_MAX
-#define MAXIOV IOV_MAX
-#elif defined UIO_MAXIOV
-#define MAXIOV UIO_MAXIOV
-#else
-#define MAXIOV 16
-#endif
-
-
-/*
- * Macros for testing file types.
- */
-
-#define ISDIR(st) (S_ISDIR((st).st_mode))
-#define ISREG(st) (S_ISREG((st).st_mode))
-#define ISDEV(st) (S_ISCHR((st).st_mode) || S_ISBLK((st).st_mode))
-#define ISLNK(st) (S_ISLNK((st).st_mode))
-#ifdef NO_UMASK
-#define FILE_MODE 0644
-#define DIR_MODE  0755
-#else
-#define FILE_MODE 0666
-#define DIR_MODE  0777
-#endif
-
-#define IS_DOT_OR_DOTDOT(s) \
-    (s[0] == '.' && (s[1] == '\0' || (s[1] == '.' && s[2] == '\0')))
-
-static int check_error(int result, Efile_error* errInfo);
-
-static int
-check_error(int result, Efile_error *errInfo)
-{
-    if (result < 0) {
-	errInfo->posix_errno = errInfo->os_errno = errno;
-	return 0;
-    }
-    return 1;
-}
-
-int
-efile_init() {
-   return 1;
-}
-
-int
-efile_mkdir(Efile_error* errInfo,	/* Where to return error codes. */
-	    char* name)			/* Name of directory to create. */
-{
-#ifdef NO_MKDIR_MODE
-    return check_error(mkdir(name), errInfo);
-#else
-    return check_error(mkdir(name, DIR_MODE), errInfo);
-#endif
-}
-
-int
-efile_rmdir(Efile_error* errInfo,	/* Where to return error codes. */
-	    char* name)			/* Name of directory to delete. */
-{
-    if (rmdir(name) == 0) {
-	return 1;
-    }
-    if (errno == ENOTEMPTY) {
-	errno = EEXIST;
-    }
-    if (errno == EEXIST) {
-	int saved_errno = errno;
-	struct stat file_stat;
-	struct stat cwd_stat;
-
-	/*
-	 *  The error code might be wrong if this is the current directory.
-	 */
-
-	if (stat(name, &file_stat) == 0 && stat(".", &cwd_stat) == 0 &&
-	    file_stat.st_ino == cwd_stat.st_ino &&
-	    file_stat.st_dev == cwd_stat.st_dev) {
-	    saved_errno = EINVAL;
-	}
-	errno = saved_errno;
-    }
-    return check_error(-1, errInfo);
-}
-
-int
-efile_delete_file(Efile_error* errInfo,	/* Where to return error codes. */
-		  char* name)		/* Name of file to delete. */
-{
-    if (unlink(name) == 0) {
-	return 1;
-    }
-    if (errno == EISDIR) {	/* Linux sets the wrong error code. */
-	errno = EPERM;
-    }
-    return check_error(-1, errInfo);
-}
-
-/*
- *---------------------------------------------------------------------------
- *
- *      Changes the name of an existing file or directory, from src to dst.
- *	If src and dst refer to the same file or directory, does nothing
- *	and returns success.  Otherwise if dst already exists, it will be
- *	deleted and replaced by src subject to the following conditions:
- *	    If src is a directory, dst may be an empty directory.
- *	    If src is a file, dst may be a file.
- *	In any other situation where dst already exists, the rename will
- *	fail.  
- *
- * Results:
- *	If the directory was successfully created, returns 1.
- *	Otherwise the return value is 0 and errno is set to
- *	indicate the error.  Some possible values for errno are:
- *
- *	EACCES:     src or dst parent directory can't be read and/or written.
- *	EEXIST:	    dst is a non-empty directory.
- *	EINVAL:	    src is a root directory or dst is a subdirectory of src.
- *	EISDIR:	    dst is a directory, but src is not.
- *	ENOENT:	    src doesn't exist, or src or dst is "".
- *	ENOTDIR:    src is a directory, but dst is not.  
- *	EXDEV:	    src and dst are on different filesystems.
- *	
- * Side effects:
- *	The implementation of rename may allow cross-filesystem renames,
- *	but the caller should be prepared to emulate it with copy and
- *	delete if errno is EXDEV.
- *
- *---------------------------------------------------------------------------
- */
-
-int
-efile_rename(Efile_error* errInfo,	/* Where to return error codes. */
-	     char* src,		        /* Original name. */
-	     char* dst)			/* New name. */
-{
-    if (rename(src, dst) == 0) {
-	return 1;
-    }
-    if (errno == ENOTEMPTY) {
-	errno = EEXIST;
-    }
-#if defined (sparc)
-    /*
-     * SunOS 4.1.4 reports overwriting a non-empty directory with a
-     * directory as EINVAL instead of EEXIST (first rule out the correct
-     * EINVAL result code for moving a directory into itself).  Must be
-     * conditionally compiled because realpath() is only defined on SunOS.
-     */
-
-    if (errno == EINVAL) {
-	char srcPath[MAXPATHLEN], dstPath[MAXPATHLEN];
-	DIR *dirPtr;
-	struct dirent *dirEntPtr;
-
-#ifdef PURIFY
-	memset(srcPath, '\0', sizeof(srcPath));
-	memset(dstPath, '\0', sizeof(dstPath));
-#endif
-
-	if ((realpath(src, srcPath) != NULL)
-		&& (realpath(dst, dstPath) != NULL)
-		&& (strncmp(srcPath, dstPath, strlen(srcPath)) != 0)) {
-	    dirPtr = opendir(dst);
-	    if (dirPtr != NULL) {
-		while ((dirEntPtr = readdir(dirPtr)) != NULL) {
-		    if ((strcmp(dirEntPtr->d_name, ".") != 0) &&
-			    (strcmp(dirEntPtr->d_name, "..") != 0)) {
-			errno = EEXIST;
-			closedir(dirPtr);
-			return check_error(-1, errInfo);
-		    }
-		}
-		closedir(dirPtr);
-	    }
-	}
-	errno = EINVAL;
-    }
-#endif	/* sparc */
-
-    if (strcmp(src, "/") == 0) {
-	/*
-	 * Alpha reports renaming / as EBUSY and Linux reports it as EACCES,
-	 * instead of EINVAL.
-	 */
-	 
-	errno = EINVAL;
-    }
-
-    /*
-     * DEC Alpha OSF1 V3.0 returns EACCES when attempting to move a
-     * file across filesystems and the parent directory of that file is
-     * not writable.  Most other systems return EXDEV.  Does nothing to
-     * correct this behavior.
-     */
-
-    return check_error(-1, errInfo);
-}
-
-int
-efile_chdir(Efile_error* errInfo,   /* Where to return error codes. */
-	    char* name)		    /* Name of directory to make current. */
-{
-    return check_error(chdir(name), errInfo);
-}
-
-
-int
-efile_getdcwd(Efile_error* errInfo,	/* Where to return error codes. */
-	      int drive,		/* 0 - current, 1 - A, 2 - B etc. */
-	      char* buffer,		/* Where to return the current 
-					   directory. */
-	      size_t size)		/* Size of buffer. */
-{
-    if (drive == 0) {
-	if (getcwd(buffer, size) == NULL)
-	    return check_error(-1, errInfo);
-	
-#ifdef SIMSPARCSOLARIS
-	/* We get "host:" prepended to the dirname - remove!. */
-	{
-	  int i = 0;
-	  int j = 0;
-	  while ((buffer[i] != ':') && (buffer[i] != '\0')) i++;
-	  if (buffer[i] == ':') {
-	    i++;
-	    while ((buffer[j++] = buffer[i++]) != '\0');
-	  }
-	}
-#endif	      
-	return 1;
-    }
-
-    /*
-     * Drives other than 0 is not supported on Unix.
-     */
-
-    errno = ENOTSUP;
-    return check_error(-1, errInfo);
-}
-
-int
-efile_readdir(Efile_error* errInfo,	/* Where to return error codes. */
-	      char* name,		/* Name of directory to open. */
-	      EFILE_DIR_HANDLE* p_dir_handle,	/* Pointer to directory 
-						   handle of
-						   open directory.*/
-	      char* buffer,		/* Pointer to buffer for 
-					   one filename. */
-	      size_t *size)		/* in-out Size of buffer, length
-					   of name. */
-{
-    DIR *dp;			/* Pointer to directory structure. */
-    struct dirent* dirp;	/* Pointer to directory entry. */
-
-    /*
-     * If this is the first call, we must open the directory.
-     */
-
-    if (*p_dir_handle == NULL) {
-	dp = opendir(name);
-	if (dp == NULL)
-	    return check_error(-1, errInfo);
-	*p_dir_handle = (EFILE_DIR_HANDLE) dp;
-    }
-
-    /*
-     * Retrieve the name of the next file using the directory handle.
-     */
-
-    dp = *((DIR **)((void *)p_dir_handle));
-    for (;;) {
-	dirp = readdir(dp);
-	if (dirp == NULL) {
-	    closedir(dp);
-	    return 0;
-	}
-	if (IS_DOT_OR_DOTDOT(dirp->d_name))
-	    continue;
-	buffer[0] = '\0';
-	strncat(buffer, dirp->d_name, (*size)-1);
-	*size = strlen(dirp->d_name);
-	return 1;
-    }
-}
-
-int
-efile_openfile(Efile_error* errInfo,	/* Where to return error codes. */
-	       char* name,		/* Name of directory to open. */
-	       int flags,		/* Flags to user for opening. */
-	       int* pfd,		/* Where to store the file 
-					   descriptor. */
-	       Sint64 *pSize)		/* Where to store the size of the 
-					   file. */
-{
-    struct stat statbuf;
-    int fd;
-    int mode;			/* Open mode. */
-
-    switch (flags & (EFILE_MODE_READ|EFILE_MODE_WRITE)) {
-    case EFILE_MODE_READ:
-	mode = O_RDONLY;
-	break;
-    case EFILE_MODE_WRITE:
-	if (flags & EFILE_NO_TRUNCATE)
-	    mode = O_WRONLY | O_CREAT;
-	else
-	    mode = O_WRONLY | O_CREAT | O_TRUNC;
-	break;
-    case EFILE_MODE_READ_WRITE:
-	mode = O_RDWR | O_CREAT;
-	break;
-    default:
-	errno = EINVAL;
-	return check_error(-1, errInfo);
-    }
-
-    if (flags & EFILE_MODE_APPEND) {
-	mode &= ~O_TRUNC;
-	mode |= O_APPEND;
-    }
-    if (flags & EFILE_MODE_EXCL) {
-	mode |= O_EXCL;
-    }
-    if (flags & EFILE_MODE_SYNC) {
-#ifdef O_SYNC
-	mode |= O_SYNC;
-#else
-	errno = ENOTSUP;
-	return check_error(-1, errInfo);
-#endif
-    }
-
-#ifdef HAVE_FSTAT
-    while (((fd = open(name, mode, FILE_MODE)) < 0) && (errno == EINTR));
-    if (!check_error(fd, errInfo)) return 0;
-#endif
-
-    if (
-#ifdef HAVE_FSTAT
-        fstat(fd, &statbuf) < 0
-#else
-        stat(name, &statbuf) < 0
-#endif
-        ) {
-        /* statbuf is undefined: if the caller depends on it,
-           i.e. invoke_read_file(), fail the call immediately */
-        if (pSize && flags == EFILE_MODE_READ) {
-            check_error(-1, errInfo);
-#ifdef HAVE_FSTAT
-            efile_closefile(fd);
-#endif
-            return 0;
-        }
-    }
-    else if (! ISREG(statbuf)) {
-        struct stat nullstatbuf;
-	/*
-	 * For UNIX only, here is some ugly code to allow
-	 * /dev/null to be opened as a file.
-	 */
-        if ( (stat("/dev/null", &nullstatbuf) < 0)
-             || (statbuf.st_ino != nullstatbuf.st_ino)
-             || (statbuf.st_dev != nullstatbuf.st_dev) ) {
-#ifdef HAVE_FSTAT
-            efile_closefile(fd);
-#endif
-	    errno = EISDIR;
-	    return check_error(-1, errInfo);
-        }
-    }
-
-#ifndef HAVE_FSTAT
-    while (((fd = open(name, mode, FILE_MODE)) < 0) && (errno == EINTR));
-    if (!check_error(fd, errInfo)) return 0;
-#endif
-
-    *pfd = fd;
-    if (pSize) *pSize = statbuf.st_size;
-    return 1;
-}
-
-int 
-efile_may_openfile(Efile_error* errInfo, char *name) {
-    struct stat statbuf;	/* Information about the file */
-    int result;
-    
-    result = stat(name, &statbuf);
-    if (!check_error(result, errInfo))
-	return 0;
-    if (!ISREG(statbuf)) {
-	errno = EISDIR;
-	return check_error(-1, errInfo);
-    }
-    return 1;
-}
-
-void
-efile_closefile(int fd)
-{
-    close(fd);
-}
-
-int
-efile_fdatasync(Efile_error *errInfo, /* Where to return error codes. */
-	    int fd)               /* File descriptor for file to sync data. */
-{
-#if defined(HAVE_FDATASYNC) && !defined(__DARWIN__)
-    return check_error(fdatasync(fd), errInfo);
-#else
-    return efile_fsync(errInfo, fd);
-#endif
-}
-
-int
-efile_fsync(Efile_error *errInfo, /* Where to return error codes. */
-	    int fd)               /* File descriptor for file to sync. */
-{
-#ifdef NO_FSYNC
-  undefined fsync /* XXX: Really? */
-#else
-#if defined(__DARWIN__) && defined(F_FULLFSYNC)
-    return check_error(fcntl(fd, F_FULLFSYNC), errInfo);
-#else
-    return check_error(fsync(fd), errInfo);
-#endif /* __DARWIN__ */
-#endif /* NO_FSYNC */
-}
-
-int
-efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, 
-	       char* name, int info_for_link)
-{
-    struct stat statbuf;	/* Information about the file */
-    int result;
-
-    if (info_for_link) {
-	result = lstat(name, &statbuf);
-    } else {
-	result = stat(name, &statbuf);
-    }	
-    if (!check_error(result, errInfo)) {
-	return 0;
-    }
-
-#if SIZEOF_OFF_T == 4
-    pInfo->size_high = 0;
-#else
-    pInfo->size_high = (Uint32)(statbuf.st_size >> 32);
-#endif
-    pInfo->size_low = (Uint32)statbuf.st_size;
-
-#ifdef NO_ACCESS
-    /* Just look at read/write access for owner. */
-
-    pInfo->access = ((statbuf.st_mode >> 6) & 07) >> 1;
-
-#else
-    pInfo->access = FA_NONE;
-    if (access(name, R_OK) == 0)
-	pInfo->access |= FA_READ;
-    if (access(name, W_OK) == 0)
-	pInfo->access |= FA_WRITE;
-
-#endif	
-
-    if (ISDEV(statbuf))
-	pInfo->type = FT_DEVICE;
-    else if (ISDIR(statbuf))
-	pInfo->type = FT_DIRECTORY;
-    else if (ISREG(statbuf))
-	pInfo->type = FT_REGULAR;
-    else if (ISLNK(statbuf))
-	pInfo->type = FT_SYMLINK;
-    else
-	pInfo->type = FT_OTHER;
-
-    pInfo->accessTime   = (Sint64)statbuf.st_atime;
-    pInfo->modifyTime   = (Sint64)statbuf.st_mtime;
-    pInfo->cTime        = (Sint64)statbuf.st_ctime;
-
-    pInfo->mode         = statbuf.st_mode;
-    pInfo->links        = statbuf.st_nlink;
-    pInfo->major_device = statbuf.st_dev;
-    pInfo->minor_device = statbuf.st_rdev;
-    pInfo->inode        = statbuf.st_ino;
-    pInfo->uid          = statbuf.st_uid;
-    pInfo->gid          = statbuf.st_gid;
-
-    return 1;
-}
-
-int
-efile_write_info(Efile_error *errInfo, Efile_info *pInfo, char *name)
-{
-    struct utimbuf tval;
-
-    /*
-     * On some systems chown will always fail for a non-root user unless
-     * POSIX_CHOWN_RESTRICTED is not set.  Others will succeed as long as 
-     * you don't try to chown a file to someone besides youself.
-     */
-
-    if (chown(name, pInfo->uid, pInfo->gid) && errno != EPERM) {
-	return check_error(-1, errInfo);
-    }
-
-    if (pInfo->mode != -1) {
-	mode_t newMode = pInfo->mode & (S_ISUID | S_ISGID |
-					S_IRWXU | S_IRWXG | S_IRWXO);
-	if (chmod(name, newMode)) {
-	    newMode &= ~(S_ISUID | S_ISGID);
-	    if (chmod(name, newMode)) {
-		return check_error(-1, errInfo);
-	    }
-	}
-    }
-
-    tval.actime  = (time_t)pInfo->accessTime;
-    tval.modtime = (time_t)pInfo->modifyTime;
-
-    return check_error(utime(name, &tval), errInfo);
-}
-
-
-int
-efile_write(Efile_error* errInfo,	/* Where to return error codes. */
-	    int flags,			/* Flags given when file was 
-					   opened. */
-	    int fd,			/* File descriptor to write to. */
-	    char* buf,			/* Buffer to write. */
-	    size_t count)		/* Number of bytes to write. */
-{
-    ssize_t written;			/* Bytes written in last operation. */
-
-    while (count > 0) {
-	if ((written = write(fd, buf, count)) < 0) {
-	    if (errno != EINTR)
-		return check_error(-1, errInfo);
-	    else
-		written = 0;
-	}
-	ASSERT(written <= count);
-	buf += written;
-	count -= written;
-    }
-    return 1;
-}
-
-int
-efile_writev(Efile_error* errInfo,   /* Where to return error codes */
-	     int flags,              /* Flags given when file was
-				      * opened */
-	     int fd,                 /* File descriptor to write to */
-	     SysIOVec* iov,          /* Vector of buffer structs.
-				      * The structs may be changed i.e.
-				      * due to incomplete writes */
-	     int iovcnt)             /* Number of structs in vector */
-{
-    int cnt = 0;                     /* Buffers so far written */
-
-    ASSERT(iovcnt >= 0);
-    
-    while (cnt < iovcnt) {
-	if ((! iov[cnt].iov_base) || (iov[cnt].iov_len <= 0)) {
-	    /* Empty buffer - skip */
-	    cnt++;
-	} else { /* Non-empty buffer */
-	    ssize_t w;                   /* Bytes written in this call */
-#ifdef HAVE_WRITEV
-	    int b = iovcnt - cnt;        /* Buffers to write */
-	    /* Use as many buffers as MAXIOV allows */
-	    if (b > MAXIOV)
-		b = MAXIOV;
-	    if (b > 1) {
-		do {
-		    w = writev(fd, &iov[cnt], b);
-		} while (w < 0 && errno == EINTR);
-		if (w < 0 && errno == EINVAL) {
-		    goto single_write;
-		}
-	    } else
-	    single_write:
-		/* Degenerated io vector - use regular write */
-#endif
-		{
-		    do {
-			size_t iov_len = iov[cnt].iov_len;
-			size_t limit = 1024*1024*1024; /* 1GB */
-			if (iov_len > limit) {
-			    iov_len = limit;
-			}
-			w = write(fd, iov[cnt].iov_base, iov_len);
-		    } while (w < 0 && errno == EINTR);
-		    ASSERT(w <= iov[cnt].iov_len ||
-			   (w == -1 && errno != EINTR));
-		}
-	    if (w < 0) return check_error(-1, errInfo);
-	    /* Move forward to next buffer to write */
-	    for (; cnt < iovcnt && w > 0; cnt++) {
-		if (iov[cnt].iov_base && iov[cnt].iov_len > 0) {
-		    if (w < iov[cnt].iov_len) {
-			/* Adjust the buffer for next write */
-			iov[cnt].iov_len -= w;
-			iov[cnt].iov_base = ((char *)iov[cnt].iov_base) + w;
-			w = 0;
-			break;
-		    } else {
-			w -= iov[cnt].iov_len;
-		    }
-		}
-	    }
-	    ASSERT(w == 0);
-	} /* else Non-empty buffer */
-    } /* while (cnt< iovcnt) */
-    return 1;
-}
-
-int
-efile_read(Efile_error* errInfo,     /* Where to return error codes. */
-	   int flags,		     /* Flags given when file was opened. */
-	   int fd,		     /* File descriptor to read from. */
-	   char* buf,		     /* Buffer to read into. */
-	   size_t count,	     /* Number of bytes to read. */
-	   size_t *pBytesRead)	     /* Where to return number of 
-					bytes read. */
-{
-    ssize_t n;
-
-    for (;;) {
-	if ((n = read(fd, buf, count)) >= 0)
-	    break;
-	else if (errno != EINTR)
-	    return check_error(-1, errInfo);
-    }
-    *pBytesRead = (size_t) n;
-    return 1;
-}
-
-
-/* pread() and pwrite()                                                   */
-/* Some unix systems, notably Solaris has these syscalls                  */
-/* It is especially nice for i.e. the dets module to have support         */
-/* for this, even if the underlying OS dosn't support it, it is           */
-/* reasonably easy to work around by first calling seek, and then         */
-/* calling read().                                                        */
-/* This later strategy however changes the file pointer, which pread()    */
-/* does not do. We choose to ignore this and say that the location        */
-/* of the file pointer is undefined after a call to any of the p functions*/
-
-
-int
-efile_pread(Efile_error* errInfo,     /* Where to return error codes. */
-	    int fd,		      /* File descriptor to read from. */
-	    Sint64 offset,            /* Offset in bytes from BOF. */
-	    char* buf,		      /* Buffer to read into. */
-	    size_t count,	      /* Number of bytes to read. */
-	    size_t *pBytesRead)	      /* Where to return 
-					 number of bytes read. */
-{
-#if defined(HAVE_PREAD) && defined(HAVE_PWRITE)
-    ssize_t n;
-    off_t off = (off_t) offset;
-    if (off != offset) {
-	errno = EINVAL;
-	return check_error(-1, errInfo);
-    }
-    for (;;) {
-	if ((n = pread(fd, buf, count, offset)) >= 0)
-	    break;
-	else if (errno != EINTR)
-	    return check_error(-1, errInfo);
-    }
-    *pBytesRead = (size_t) n;
-    return 1;
-#else
-    {
-	int res = efile_seek(errInfo, fd, offset, EFILE_SEEK_SET, NULL);
-	if (res) {
-	    return efile_read(errInfo, 0, fd, buf, count, pBytesRead);
-	} else {
-	    return res;
-	}
-    }
-#endif
-}
-
-
-
-int
-efile_pwrite(Efile_error* errInfo,  /* Where to return error codes. */
-	     int fd,		    /* File descriptor to write to. */
-	     char* buf,		    /* Buffer to write. */
-	     size_t count,	    /* Number of bytes to write. */
-	     Sint64 offset)	    /* where to write it */
-{ 
-#if defined(HAVE_PREAD) && defined(HAVE_PWRITE)
-    ssize_t written;		    /* Bytes written in last operation. */
-    off_t off = (off_t) offset;
-    if (off != offset) {
-	errno = EINVAL;
-	return check_error(-1, errInfo);
-    }
-    
-    while (count > 0) {
-	if ((written = pwrite(fd, buf, count, offset)) < 0) {
-	    if (errno != EINTR)
-		return check_error(-1, errInfo);
-	    else
-		written = 0;
-	}
-	ASSERT(written <= count);
-	buf += written;
-	count -= written;
-	offset += written;
-    }
-    return 1;
-#else  /* For unix systems that don't support pread() and pwrite() */    
-    {
-	int res = efile_seek(errInfo, fd, offset, EFILE_SEEK_SET, NULL);
-	
-	if (res) {
-	    return efile_write(errInfo, 0, fd, buf, count);
-	} else {
-	    return res;
-	}
-    }
-#endif
-}
-
-
-int
-efile_seek(Efile_error* errInfo,      /* Where to return error codes. */
-	   int fd,                    /* File descriptor to do the seek on. */
-	   Sint64 offset,             /* Offset in bytes from the given 
-					 origin. */ 
-	   int origin,                /* Origin of seek (SEEK_SET, SEEK_CUR,
-				         SEEK_END). */ 
-	   Sint64 *new_location)      /* Resulting new location in file. */
-{
-    off_t off, result;
-
-    switch (origin) {
-    case EFILE_SEEK_SET: origin = SEEK_SET; break;
-    case EFILE_SEEK_CUR: origin = SEEK_CUR; break;
-    case EFILE_SEEK_END: origin = SEEK_END; break;
-    default:
-	errno = EINVAL;
-	return check_error(-1, errInfo);
-    }
-    off = (off_t) offset;
-    if (off != offset) {
-	errno = EINVAL;
-	return check_error(-1, errInfo);
-    }
-    
-    errno = 0;
-    result = lseek(fd, off, origin);
-
-    /*
-     * Note that the man page for lseek (on SunOs 5) says:
-     * 
-     * "if fildes is a remote file  descriptor  and  offset  is
-     * negative,  lseek()  returns  the  file pointer even if it is
-     * negative."
-     */
-
-    if (result < 0 && errno == 0)
-	errno = EINVAL;
-    if (result < 0)
-	return check_error(-1, errInfo);
-    if (new_location) {
-	*new_location = result;
-    }
-    return 1;
-}
-
-
-int
-efile_truncate_file(Efile_error* errInfo, int *fd, int flags)
-{
-#ifndef NO_FTRUNCATE
-    off_t offset;
-
-    return check_error((offset = lseek(*fd, 0, 1)) >= 0 &&
-		       ftruncate(*fd, offset) == 0 ? 1 : -1,
-		       errInfo);
-#else
-    return 1;
-#endif
-}
-
-int
-efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size)
-{
-    int len;
-    ASSERT(size > 0);
-    len = readlink(name, buffer, size-1);
-    if (len == -1) {
-	return check_error(-1, errInfo);
-    }
-    buffer[len] = '\0';
-    return 1;
-}
-
-int
-efile_altname(Efile_error* errInfo, char* name, char* buffer, size_t size)
-{
-    errno = ENOTSUP;
-    return check_error(-1, errInfo);
-}
-
-int
-efile_link(Efile_error* errInfo, char* old, char* new)
-{
-    return check_error(link(old, new), errInfo);
-}
-
-int
-efile_symlink(Efile_error* errInfo, char* old, char* new)
-{
-    return check_error(symlink(old, new), errInfo);
-}
-
-int
-efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset,
-		  Sint64 length, int advise)
-{
-#ifdef HAVE_POSIX_FADVISE
-    return check_error(posix_fadvise(fd, offset, length, advise), errInfo);
-#else
-    return check_error(0, errInfo);
-#endif
-}
-
-#ifdef HAVE_SENDFILE
-/* For some reason the maximum size_t cannot be used as the max size
-   3GB seems to work on all platforms */
-#define SENDFILE_CHUNK_SIZE ((1UL << 30) -1)
-
-/*
- * sendfile: The implementation of the sendfile system call varies
- * a lot on different *nix platforms so to make the api similar in all
- * we have to emulate some things in linux and play with variables on
- * bsd/darwin.
- *
- * All of the calls will split a command which tries to send more than
- * SENDFILE_CHUNK_SIZE of data at once.
- *
- * On platforms where *nbytes of 0 does not mean the entire file, this is
- * simulated.
- *
- * It could be possible to implement header/trailer in sendfile. Though
- * you would have to emulate it in linux and on BSD/Darwin some complex
- * calculations have to be made when using a non blocking socket to figure
- * out how much of the header/file/trailer was sent in each command.
- *
- * The semantics of the API is this:
- * Return value: 1 if all data was sent and the function does not need to
- * be called again. 0 if an error occures OR if there is more data which
- * has to be sent (EAGAIN or EINTR will be set appropriately)
- *
- * The amount of data written in a call is returned through nbytes.
- *
- */
-
-int
-efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd,
-	       off_t *offset, Uint64 *nbytes, struct t_sendfile_hdtl* hdtl)
-{
-    Uint64 written = 0;
-#if defined(__linux__)
-    ssize_t retval;
-    do {
-      /* check if *nbytes is 0 or greater than chunk size */
-      if (*nbytes == 0 || *nbytes > SENDFILE_CHUNK_SIZE)
-	retval = sendfile(out_fd, in_fd, offset, SENDFILE_CHUNK_SIZE);
-      else
-	retval = sendfile(out_fd, in_fd, offset, *nbytes);
-      if (retval > 0) {
-	written += retval;
-	*nbytes -= retval;
-      }
-    } while (retval == SENDFILE_CHUNK_SIZE);
-    if (written != 0) {
-      /* -1 is not returned by the linux API so we have to simulate it */
-      retval = -1;
-      errno = EAGAIN;
-    }
-#elif defined(__sun) && defined(__SVR4) && defined(HAVE_SENDFILEV)
-    ssize_t retval;
-    size_t len;
-    sendfilevec_t fdrec;
-    fdrec.sfv_fd = in_fd;
-    fdrec.sfv_flag = 0;
-    do {
-      fdrec.sfv_off = *offset;
-      len = 0;
-      /* check if *nbytes is 0 or greater than chunk size */
-      if (*nbytes == 0 || *nbytes > SENDFILE_CHUNK_SIZE)
-	fdrec.sfv_len = SENDFILE_CHUNK_SIZE;
-      else
-	fdrec.sfv_len = *nbytes;
-
-      retval = sendfilev(out_fd, &fdrec, 1, &len);
-
-      if (retval == -1 && errno == EINVAL) {
-          /* On some solaris versions (I've seen it on SunOS 5.10),
-             using a sfv_len larger then a filesize will result in
-             a -1 && errno == EINVAL return. We translate this so
-             a successful send of the data.*/
-          retval = len;
-      }
-
-      if (retval != -1 || errno == EAGAIN || errno == EINTR) {
-        *offset += len;
-	*nbytes -= len;
-	written += len;
-      }
-    } while (len == SENDFILE_CHUNK_SIZE);
-#elif defined(__DARWIN__)
-    int retval;
-    off_t len;
-    do {
-      /* check if *nbytes is 0 or greater than chunk size */
-      if(*nbytes > SENDFILE_CHUNK_SIZE)
-	len = SENDFILE_CHUNK_SIZE;
-      else
-	len = *nbytes;
-      retval = sendfile(in_fd, out_fd, *offset, &len, NULL, 0);
-      if (retval != -1 || errno == EAGAIN || errno == EINTR) {
-        *offset += len;
-	*nbytes -= len;
-	written += len;
-      }
-    } while (len == SENDFILE_CHUNK_SIZE);
-#elif defined(__FreeBSD__) || defined(__DragonFly__)
-    off_t len;
-    int retval;
-    do {
-      if (*nbytes > SENDFILE_CHUNK_SIZE)
-	retval = sendfile(in_fd, out_fd, *offset, SENDFILE_CHUNK_SIZE,
-			  NULL, &len, 0);
-      else
-	retval = sendfile(in_fd, out_fd, *offset, *nbytes, NULL, &len, 0);
-      if (retval != -1 || errno == EAGAIN || errno == EINTR) {
-	*offset += len;
-	*nbytes -= len;
-	written += len;
-      }
-    } while(len == SENDFILE_CHUNK_SIZE);
-#endif
-    *nbytes = written;
-    return check_error(retval, errInfo);
-}
-#endif /* HAVE_SENDFILE */
-
-#ifdef HAVE_POSIX_FALLOCATE
-static int
-call_posix_fallocate(int fd, Sint64 offset, Sint64 length)
-{
-    int ret;
-
-    /*
-     * On Linux and Solaris for example, posix_fallocate() returns
-     * a positive error number on error and it does not set errno.
-     * On FreeBSD however (9.0 at least), it returns -1 on error
-     * and it sets errno.
-     */
-    do {
-        ret = posix_fallocate(fd, (off_t) offset, (off_t) length);
-        if (ret > 0) {
-            errno = ret;
-            ret = -1;
-        }
-    } while (ret != 0 && errno == EINTR);
-
-    return ret;
-}
-#endif /* HAVE_POSIX_FALLOCATE */
-
-int
-efile_fallocate(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length)
-{
-#if defined HAVE_FALLOCATE
-    /* Linux specific, more efficient than posix_fallocate. */
-    int ret;
-
-    do {
-        ret = fallocate(fd, FALLOC_FL_KEEP_SIZE, (off_t) offset, (off_t) length);
-    } while (ret != 0 && errno == EINTR);
-
-#if defined HAVE_POSIX_FALLOCATE
-    /* Fallback to posix_fallocate if available. */
-    if (ret != 0) {
-        ret = call_posix_fallocate(fd, offset, length);
-    }
-#endif
-
-    return check_error(ret, errInfo);
-#elif defined F_PREALLOCATE
-    /* Mac OS X specific, equivalent to posix_fallocate. */
-    int ret;
-    fstore_t fs;
-
-    memset(&fs, 0, sizeof(fs));
-    fs.fst_flags = F_ALLOCATECONTIG;
-    fs.fst_posmode = F_VOLPOSMODE;
-    fs.fst_offset = (off_t) offset;
-    fs.fst_length = (off_t) length;
-
-    ret = fcntl(fd, F_PREALLOCATE, &fs);
-
-    if (-1 == ret) {
-        fs.fst_flags = F_ALLOCATEALL;
-        ret = fcntl(fd, F_PREALLOCATE, &fs);
-
-#if defined HAVE_POSIX_FALLOCATE
-        /* Fallback to posix_fallocate if available. */
-        if (-1 == ret) {
-            ret = call_posix_fallocate(fd, offset, length);
-        }
-#endif
-    }
-
-    return check_error(ret, errInfo);
-#elif defined HAVE_POSIX_FALLOCATE
-    /* Other Unixes, use posix_fallocate if available. */
-    return check_error(call_posix_fallocate(fd, offset, length), errInfo);
-#else
-    errno = ENOTSUP;
-    return check_error(-1, errInfo);
-#endif
-}
diff --git a/erts/emulator/drivers/win32/win_efile.c b/erts/emulator/drivers/win32/win_efile.c
deleted file mode 100644
index 2d366b5833..0000000000
--- a/erts/emulator/drivers/win32/win_efile.c
+++ /dev/null
@@ -1,2058 +0,0 @@
-/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 1997-2016. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * %CopyrightEnd%
- */
-/*
- * Purpose: Provides file and directory operations for Windows.
- */
-
-#include <windows.h>
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-#include "sys.h"
-#include <ctype.h>
-#include <wchar.h>
-#include "erl_efile.h"
-
-#define DBG_TRACE_MASK 0
-/* 1 = file name ops
- * 2 = file descr ops
- * 4 = errors
- * 8 = path name conversion
- */
-#if !DBG_TRACE_MASK
-#  define DBG_TRACE(M,S)
-#  define DBG_TRACE1(M,FMT,A)
-#  define DBG_TRACE2(M,FMT,A,B)
-#else
-#  define DBG_TRACE(M,S) do { if ((M)&DBG_TRACE_MASK) fwprintf(stderr, L"DBG_TRACE %d: %s\r\n", __LINE__, (WCHAR*)(S)); }while(0)
-#  define DBG_TRACE1(M,FMT,A) do { if ((M)&DBG_TRACE_MASK) fwprintf(stderr, L"DBG_TRACE %d: " L##FMT L"\r\n", __LINE__, (A)); }while(0)
-#  define DBG_TRACE2(M,FMT,A,B) do { if ((M)&DBG_TRACE_MASK) fwprintf(stderr, L"DBG_TRACE %d: " L##FMT L"\r\n", __LINE__, (A), (B)); }while(0)
-#endif
-
-/*
- * Microsoft-specific function to map a WIN32 error code to a Posix errno.
- */
-
-#define ISSLASH(a)  ((a) == L'\\' || (a) == L'/')
-#define ISDIR(st) (((st).st_mode&S_IFMT) == S_IFDIR)
-#define ISREG(st) (((st).st_mode&S_IFMT) == S_IFREG)
-
-#define IS_DOT_OR_DOTDOT(s) \
-    ((s)[0] == L'.' && ((s)[1] == L'\0' || ((s)[1] == L'.' && (s)[2] == L'\0')))
-
-#define FILE_SHARE_FLAGS (FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
-
-#ifndef INVALID_FILE_ATTRIBUTES
-#define INVALID_FILE_ATTRIBUTES ((DWORD) 0xFFFFFFFF)
-#endif
-
-#define TICKS_PER_SECOND (10000000ULL)
-#define EPOCH_DIFFERENCE (11644473600LL)
-
-#define FILETIME_TO_EPOCH(epoch, ft) \
-    do { \
-	ULARGE_INTEGER ull; \
-	ull.LowPart  = (ft).dwLowDateTime; \
-	ull.HighPart = (ft).dwHighDateTime; \
-	(epoch) = ((ull.QuadPart / TICKS_PER_SECOND) - EPOCH_DIFFERENCE); \
-    } while(0)
-
-#define EPOCH_TO_FILETIME(ft, epoch) \
-    do { \
-	ULARGE_INTEGER ull; \
-	ull.QuadPart = (((epoch) + EPOCH_DIFFERENCE) * TICKS_PER_SECOND); \
-	(ft).dwLowDateTime = ull.LowPart; \
-	(ft).dwHighDateTime = ull.HighPart; \
-    } while(0)
-
-
-static int check_error(int result, Efile_error* errInfo);
-static int set_error(Efile_error* errInfo);
-static int set_os_errno(Efile_error* errInfo, DWORD os_errno);
-static int is_root_unc_name(const WCHAR *path);
-static int extract_root(WCHAR *name);
-static unsigned short dos_to_posix_mode(int attr, const WCHAR *name);
-
-
-struct wpath_tmp_buffer {
-    struct wpath_tmp_buffer* next;
-    WCHAR buffer[1];
-};
-
-typedef struct {
-    Efile_error* errInfo;
-    struct wpath_tmp_buffer* buf_list;
-}Efile_call_state;
-
-static void call_state_init(Efile_call_state* state, Efile_error* errInfo)
-{
-    state->errInfo = errInfo;
-    state->buf_list = NULL;
-}
-static WCHAR* wpath_tmp_alloc(Efile_call_state* state, size_t len)
-{
-    size_t sz = offsetof(struct wpath_tmp_buffer, buffer)
-	+ (len+1)*sizeof(WCHAR);
-    struct wpath_tmp_buffer* p = driver_alloc(sz);
-    p->next = state->buf_list;
-    state->buf_list = p;
-    return p->buffer;
-}
-static void call_state_free(Efile_call_state* state)
-{
-    while(state->buf_list) {
-	struct wpath_tmp_buffer* next = state->buf_list->next;
-	driver_free(state->buf_list);
-	state->buf_list = next;
-    }
-}
-static WCHAR* get_cwd_wpath_tmp(Efile_call_state* state)
-{
-    WCHAR dummy;
-    DWORD size = GetCurrentDirectoryW(0, &dummy);
-    WCHAR* ret = NULL;
-
-    if (size) {
-	ret = wpath_tmp_alloc(state, size);
-	if (!GetCurrentDirectoryW(size, ret)) {
-	    ret = NULL;
-	}
-    }
-    return ret;
-}
-static WCHAR* get_full_wpath_tmp(Efile_call_state* state,
-                                 const WCHAR* file,
-				 WCHAR** file_part,
-				 DWORD extra)
-{
-    WCHAR dummy;
-    DWORD size = GetFullPathNameW(file, 0, &dummy, NULL);
-    WCHAR* ret = NULL;
-
-    if (size) {
-	int ok;
-	ret = wpath_tmp_alloc(state, size + extra);
-	if (file_part) {
-	    ok = (GetFullPathNameW(file, size, ret, file_part) != 0);
-	}
-	else {
-	    ok = (_wfullpath(ret, file, size) != NULL);
-	}
-	if (!ok) {
-	    ret = NULL;
-	}
-    }
-    return ret;
-}
-
-static void ensure_wpath_max(Efile_call_state* state, WCHAR** pathp, size_t max);
-static int do_rmdir(Efile_call_state*, char* name);
-static int do_rename(Efile_call_state*, char* src, char* dst);
-static int do_readdir(Efile_call_state*, char* name, EFILE_DIR_HANDLE*, char* buffer, size_t *size);
-static int do_fileinfo(Efile_call_state*, Efile_info*, char* orig_name, int info_for_link);
-static char* do_readlink(Efile_call_state*, char* name, char* buffer, size_t size);
-static int do_altname(Efile_call_state*, char* orig_name, char* buffer, size_t size);
-
-
-static int errno_map(DWORD last_error) {
-
-    switch (last_error) {
-    case ERROR_SUCCESS:
-	return 0;
-    case ERROR_INVALID_FUNCTION:
-    case ERROR_INVALID_DATA:
-    case ERROR_INVALID_PARAMETER:
-    case ERROR_INVALID_TARGET_HANDLE:
-    case ERROR_INVALID_CATEGORY:
-    case ERROR_NEGATIVE_SEEK:
-	return EINVAL;
-    case ERROR_DIR_NOT_EMPTY:
-	return EEXIST;
-    case ERROR_BAD_FORMAT:
-	return ENOEXEC;
-    case ERROR_PATH_NOT_FOUND:
-    case ERROR_FILE_NOT_FOUND:
-    case ERROR_NO_MORE_FILES:
-	return ENOENT;
-    case ERROR_TOO_MANY_OPEN_FILES:
-	return EMFILE;
-    case ERROR_ACCESS_DENIED:
-    case ERROR_INVALID_ACCESS:
-    case ERROR_CURRENT_DIRECTORY:
-    case ERROR_SHARING_VIOLATION:
-    case ERROR_LOCK_VIOLATION:
-    case ERROR_INVALID_PASSWORD:
-    case ERROR_DRIVE_LOCKED:
-	return EACCES;
-    case ERROR_INVALID_HANDLE:
-	return EBADF;
-    case ERROR_NOT_ENOUGH_MEMORY:
-    case ERROR_OUTOFMEMORY:
-    case ERROR_OUT_OF_STRUCTURES:
-	return ENOMEM;
-    case ERROR_INVALID_DRIVE:
-    case ERROR_BAD_UNIT:
-    case ERROR_NOT_READY:
-    case ERROR_REM_NOT_LIST:
-    case ERROR_DUP_NAME:
-    case ERROR_BAD_NETPATH:
-    case ERROR_NETWORK_BUSY:
-    case ERROR_DEV_NOT_EXIST:
-    case ERROR_BAD_NET_NAME:
-	return ENXIO;
-    case ERROR_NOT_SAME_DEVICE:
-	return EXDEV;
-    case ERROR_WRITE_PROTECT:
-	return EROFS;
-    case ERROR_BAD_LENGTH:
-    case ERROR_BUFFER_OVERFLOW:
-	return E2BIG;
-    case ERROR_SEEK:
-    case ERROR_SECTOR_NOT_FOUND:
-	return ESPIPE;
-    case ERROR_NOT_DOS_DISK:
-	return ENODEV;
-    case ERROR_GEN_FAILURE:
-	return ENODEV;
-    case ERROR_SHARING_BUFFER_EXCEEDED:
-    case ERROR_NO_MORE_SEARCH_HANDLES:
-	return EMFILE;
-    case ERROR_HANDLE_EOF:
-    case ERROR_BROKEN_PIPE:
-	return EPIPE;
-    case ERROR_HANDLE_DISK_FULL:
-    case ERROR_DISK_FULL:
-	return ENOSPC;
-    case ERROR_NOT_SUPPORTED:
-	return ENOTSUP;
-    case ERROR_FILE_EXISTS:
-    case ERROR_ALREADY_EXISTS:
-    case ERROR_CANNOT_MAKE:
-	return EEXIST;
-    case ERROR_ALREADY_ASSIGNED:
-	return EBUSY;
-    case ERROR_NO_PROC_SLOTS:
-	return EAGAIN;
-    case ERROR_CANT_RESOLVE_FILENAME:
-	return EMLINK;
-    case ERROR_PRIVILEGE_NOT_HELD:
-	return EPERM;
-    case ERROR_ARENA_TRASHED:
-    case ERROR_INVALID_BLOCK:
-    case ERROR_BAD_ENVIRONMENT:
-    case ERROR_BAD_COMMAND:
-    case ERROR_CRC:
-    case ERROR_OUT_OF_PAPER:
-    case ERROR_READ_FAULT:
-    case ERROR_WRITE_FAULT:
-    case ERROR_WRONG_DISK:
-    case ERROR_NET_WRITE_FAULT:
-	return EIO;
-    default: /* not to do with files I expect. */
-	return EIO;
-    }	
-}
-
-static int
-check_error(int result, Efile_error* errInfo)
-{
-    if (result < 0) {
-	errInfo->posix_errno = errno;
-	errInfo->os_errno = GetLastError();
-	DBG_TRACE2(4, "ERROR os_error=%d errno=%d @@@@@@@@@@@@@@@@@@@@@@@@@@@@",
-	        errInfo->os_errno, errInfo->posix_errno);
-	return 0;
-    }
-    return 1;
-}
-
-static void
-save_last_error(Efile_error* errInfo)
-{
-    errInfo->posix_errno = errno;
-    errInfo->os_errno = GetLastError();
-    DBG_TRACE2(4, "ERROR os_error=%d errno=%d $$$$$$$$$$$$$$$$$$$$$$$$$$$$$",
-	errInfo->os_errno, errInfo->posix_errno);
-}
-
-
-/*
- * Fills the provided error information structure with information
- * with the error code given by GetLastError() and its corresponding
- * Posix error number.
- *
- * Returns 0.
- */
-
-static int
-set_error(Efile_error* errInfo)
-{
-    set_os_errno(errInfo, GetLastError());
-    return 0;
-}
-
-
-static int
-set_os_errno(Efile_error* errInfo, DWORD os_errno)
-{
-    errInfo->os_errno = os_errno;
-    errInfo->posix_errno = errno_map(os_errno);
-    DBG_TRACE2(4, "ERROR os_error=%d errno=%d ############################",
-	    errInfo->os_errno, errInfo->posix_errno);
-    return 0;
-}
-
-int
-efile_init() {
-   return 1;
-}
-
-/*
- * A writev with Unix semantics, but with Windows arguments 
- */
-static int 
-win_writev(Efile_error* errInfo,
-	   HANDLE fd,                  /* handle to file */
-	   FILE_SEGMENT_ELEMENT iov[], /* array of buffer pointers */
-	   DWORD *size)                /* number of bytes to write */
-{
-    OVERLAPPED ov;
-    ov.Offset = 0L;
-    ov.OffsetHigh = 0L;
-    ov.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
-    if (ov.hEvent == NULL)
-	return set_error(errInfo);
-    if (! write_file_gather(fd, iov, *size, NULL, &ov))
-	return set_error(errInfo);
-    if (WaitForSingleObject(ov.hEvent, INFINITE) != WAIT_OBJECT_0)
-	return set_error(errInfo);
-    if (! GetOverlappedResult(fd, &ov, size, FALSE))
-	return set_error(errInfo);
-    return 1;
-}
-
-
-/* Check '*pathp' and convert it if needed to something that windows will accept.
- * Typically use UNC path with \\?\ prefix if absolute path is longer than 260.
- */
-static void ensure_wpath(Efile_call_state* state, WCHAR** pathp)
-{
-    ensure_wpath_max(state, pathp, MAX_PATH);
-}
-
-static void ensure_wpath_max(Efile_call_state* state, WCHAR** pathp, size_t max)
-{
-    WCHAR* path = *pathp;
-    WCHAR* p;
-    size_t len = wcslen(path);
-    int unc_fixup = 0;
-
-    if (path[0] == 0) {
-	DBG_TRACE(8, L"Let empty path pass through");
-	return;
-    }
-
-    DBG_TRACE1(8,"IN: %s", path);
-
-    if (path[1] == L':' && ISSLASH(path[2])) { /* absolute path */
-	if (len >= max) {
-	    WCHAR *src, *dst;
-
-	    *pathp = wpath_tmp_alloc(state, 4+len+1);
-	    dst = *pathp;
-	    wcscpy(dst, L"\\\\?\\");
-	    for (src=path,dst+=4; *src; src++) {
-		if (*src == L'/') {
-		    if (dst[-1] != L'\\') {
-			*dst++ =  L'\\';
-		    }
-		    /*else ignore redundant slashes */
-		}
-		else
-		    *dst++ = *src;
-	    }
-	    *dst = 0;
-	    unc_fixup = 1;
-	}
-    }
-    else if (!(ISSLASH(path[0]) && ISSLASH(path[1]))) { /* relative path */
-	DWORD cwdLen = GetCurrentDirectoryW(0, NULL);
-	DWORD absLen = cwdLen + 1 + len;
-	if (absLen >= max) {
-	    WCHAR *fullPath = wpath_tmp_alloc(state, 4+4+absLen);
-	    DWORD fullLen;
-
-	    fullLen = GetFullPathNameW(path, 4 + absLen, fullPath+4, NULL);
-	    if (fullLen >= 4+absLen) {
-		*pathp = path;
-		DBG_TRACE2(8,"ensure_wpath FAILED absLen=%u %s", (int)absLen, path);
-		return;
-	    }
-	    /* GetFullPathNameW can return paths longer than MAX_PATH without the \\?\ prefix.
-	     * At least seen on Windows 7. Go figure...
-	     */
-	    if (fullLen >= max && wcsncmp(fullPath+4, L"\\\\?\\", 4) != 0) {
-		wcsncpy(fullPath, L"\\\\?\\", 4);
-		*pathp = fullPath;
-	    }
-	    else {
-		*pathp = fullPath + 4;
-	    }
-	}
-    }
-
-    if (unc_fixup) {
-	WCHAR* endp;
-
-	p = *pathp;
-	len = wcslen(p);
-	endp = p + len;
-	if (len > 4) {
-	    p += 4;
-	    while (*p) {
-		if (p[0] == L'\\' && p[1] == L'.') {
-		    if (p[2] == L'\\' || !p[2]) { /* single dot */
-			wmemmove(p, p+2, (&endp[1] - &p[2]));
-			endp -= 2;
-		    }
-		    else if (p[2] == L'.' && (p[3] == L'\\' || !p[3])) { /* double dot */
-			WCHAR* r;
-			for (r=p-1; *r == L'\\'; --r)
-			    /*skip redundant slashes*/;
-			for (; *r != L'\\'; --r)
-			    /*find start of prev directory*/;
-			if (r < *pathp + 6)
-			    break;
-			wmemmove(r, p+3, (&endp[1] - &p[3]));
-			p = r;
-		    }
-		    else p += 3;
-		}
-		else ++p;
-	    }
-	}
-    }
-    DBG_TRACE1(8,"OUT: %s", *pathp);
-}
-
-int
-efile_mkdir(Efile_error* errInfo,	/* Where to return error codes. */
-	    char* name)			/* Name of directory to create. */
-{
-    Efile_call_state state;
-    WCHAR* wname = (WCHAR*)name;
-    int ret;
-
-    DBG_TRACE(1, name);
-    call_state_init(&state, errInfo);
-    ensure_wpath_max(&state, &wname, 248); /* Yes, 248 limit for normal paths */
-
-    ret = (int) CreateDirectoryW(wname, NULL);
-    if (!ret)
-	set_error(errInfo);
-
-    call_state_free(&state);
-    return ret;
-}
-
-int
-efile_rmdir(Efile_error* errInfo,	/* Where to return error codes. */
-	    char* name)			/* Name of directory to delete. */
-{
-    Efile_call_state state;
-    int ret;
-
-    DBG_TRACE(1, name);
-    call_state_init(&state, errInfo);
-    ret = do_rmdir(&state, name);
-    call_state_free(&state);
-    return ret;
-}
-
-static int do_rmdir(Efile_call_state* state, char* name)
-{
-    OSVERSIONINFO os;
-    DWORD attr;
-    WCHAR *wname = (WCHAR *) name;
-    WCHAR *buffer = NULL;
-
-    ensure_wpath(state, &wname);
-
-    if (RemoveDirectoryW(wname) != FALSE) {
-	return 1;
-    }
-    errno = errno_map(GetLastError());
-    if (errno == EACCES) {
-	attr = GetFileAttributesW(wname);
-	if (attr != (DWORD) -1) {
-	    if ((attr & FILE_ATTRIBUTE_DIRECTORY) == 0) {
-		/* 
-		 * Windows 95 reports calling RemoveDirectory on a file as an 
-		 * EACCES, not an ENOTDIR.
-		 */
-		
-		errno = ENOTDIR;
-		goto end;
-	    }
-
-	    /* 
-	     * Windows 95 reports removing a non-empty directory as
-	     * an EACCES, not an EEXIST.  If the directory is not empty,
-	     * change errno so caller knows what's going on.
-	     */
-
-	    os.dwOSVersionInfoSize = sizeof(os);
-	    GetVersionEx(&os);
-	    if (os.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS) {
-		HANDLE handle;
-		WIN32_FIND_DATAW data;
-		int len = wcslen(wname);
-
-		buffer = wpath_tmp_alloc(state, len + 4);
-		wcscpy(buffer, wname);
-		if (buffer[0] && buffer[len-1] != L'\\' && buffer[len-1] != L'/') {
-		    wcscat(buffer, L"\\");
-		}
-		wcscat(buffer, L"*.*");
-		handle = FindFirstFileW(buffer, &data);
-		if (handle != INVALID_HANDLE_VALUE) {
-		    while (1) {
-			if ((wcscmp(data.cFileName, L".") != 0)
-				&& (wcscmp(data.cFileName, L"..") != 0)) {
-			    /*
-			     * Found something in this directory.
-			     */
-
-			    errno = EEXIST;
-			    break;
-			}
-			if (FindNextFileW(handle, &data) == FALSE) {
-			    break;
-			}
-		    }
-		    FindClose(handle);
-		}
-	    }
-	}
-    }
-
-    if (errno == ENOTEMPTY) {
-	/* 
-	 * Posix allows both EEXIST or ENOTEMPTY, but we'll always
-	 * return EEXIST to allow easy matching in Erlang code.
-	 */
-
-	errno = EEXIST;
-    }
-
- end:
-    save_last_error(state->errInfo);
-    return 0;
-}
-
-int
-efile_delete_file(Efile_error* errInfo,		/* Where to return error codes. */
-		  char* name)			/* Name of file to delete. */
-{
-    Efile_call_state state;
-    int ret;
-    DBG_TRACE(1, name);
-    call_state_init(&state, errInfo);
-    ret = do_delete_file(&state, name);
-    call_state_free(&state);
-    return ret;
-}
-
-static int do_delete_file(Efile_call_state* state, char* name)
-{
-    DWORD attr;
-    WCHAR *wname = (WCHAR *) name;
-
-    ensure_wpath(state, &wname);
-
-    if (DeleteFileW(wname) != FALSE) {
-	return 1;
-    }
-
-    errno = errno_map(GetLastError());
-    if (errno == EACCES) {
-        attr = GetFileAttributesW(wname);
-	if (attr != (DWORD) -1) {
-	    if (attr & FILE_ATTRIBUTE_DIRECTORY) {
-		/*
-		 * Windows NT reports removing a directory as EACCES instead
-		 * of EPERM.
-		 */
-
-		errno = EPERM;
-	    }
-	}
-    } else if (errno == ENOENT) {
-        attr = GetFileAttributesW(wname);
-	if (attr != (DWORD) -1) {
-	    if (attr & FILE_ATTRIBUTE_DIRECTORY) {
-	    	/*
-		 * Windows 95 reports removing a directory as ENOENT instead 
-		 * of EPERM.
-		 */
-
-		errno = EPERM;
-	    }
-	}
-    } else if (errno == EINVAL) {
-	/*
-	 * Windows NT reports removing a char device as EINVAL instead of
-	 * EACCES.
-	 */
-	
-	errno = EACCES;
-    }
-
-    return check_error(-1, state->errInfo);
-}
-
-/*
- *---------------------------------------------------------------------------
- *
- *      Changes the name of an existing file or directory, from src to dst.
- *	If src and dst refer to the same file or directory, does nothing
- *	and returns success.  Otherwise if dst already exists, it will be
- *	deleted and replaced by src subject to the following conditions:
- *	    If src is a directory, dst may be an empty directory.
- *	    If src is a file, dst may be a file.
- *	In any other situation where dst already exists, the rename will
- *	fail.  
- *
- *	Some possible error codes:
- *
- *	EACCES:     src or dst parent directory can't be read and/or written.
- *	EEXIST:	    dst is a non-empty directory.
- *	EINVAL:	    src is a root directory or dst is a subdirectory of src.
- *	EISDIR:	    dst is a directory, but src is not.
- *	ENOENT:	    src doesn't exist, or src or dst is "".
- *	ENOTDIR:    src is a directory, but dst is not.  
- *	EXDEV:	    src and dst are on different filesystems.
- *	
- * Side effects:
- *	The implementation of rename may allow cross-filesystem renames,
- *	but the caller should be prepared to emulate it with copy and
- *	delete if errno is EXDEV.
- *
- *---------------------------------------------------------------------------
- */
-
-int
-efile_rename(Efile_error* errInfo, char* src, char* dst)
-{
-    Efile_call_state state;
-    int ret;
-    DBG_TRACE(1, src);
-    call_state_init(&state, errInfo);
-    ret = do_rename(&state, src, dst);
-    call_state_free(&state);
-    return ret;
-}
-
-static int
-do_rename(Efile_call_state* state,
-	  char* src,			/* Original name. */
-	  char* dst)			/* New name. */
-{
-    DWORD srcAttr, dstAttr;
-    WCHAR *wsrc = (WCHAR *) src;
-    WCHAR *wdst = (WCHAR *) dst;
-
-    ensure_wpath(state, &wsrc);
-    ensure_wpath(state, &wdst);
-
-    if (MoveFileW(wsrc, wdst) != FALSE) {
-	return 1;
-    }
-
-    errno = errno_map(GetLastError());
-    srcAttr = GetFileAttributesW(wsrc);
-    dstAttr = GetFileAttributesW(wdst);
-    if (srcAttr == (DWORD) -1) {
-	srcAttr = 0;
-    }
-    if (dstAttr == (DWORD) -1) {
-	dstAttr = 0;
-    }
-
-    if (errno == EBADF) {
-	errno = EACCES;
-	return check_error(-1, state->errInfo);
-    }
-    if (errno == EACCES) {
-	decode:
-	if (srcAttr & FILE_ATTRIBUTE_DIRECTORY) {
-	    WCHAR *srcPath, *dstPath;
-	    WCHAR *srcRest, *dstRest;
-	    int size;
-
-	    srcPath = get_full_wpath_tmp(state, wsrc, &srcRest, 0);
-	    if (!srcPath) {
-		save_last_error(state->errInfo);
-		return 0;
-	    }
-
-	    dstPath = get_full_wpath_tmp(state, wdst, &dstRest, 0);
-	    if (!dstPath) {
-		save_last_error(state->errInfo);
-		return 0;
-	    }
-
-	    if (srcRest == NULL) {
-		srcRest = srcPath + wcslen(srcPath);
-	    }
-	    if (_wcsnicmp(srcPath, dstPath, srcRest - srcPath) == 0) {
-		/*
-		 * Trying to move a directory into itself.
-		 */
-
-		errno = EINVAL;
-	    }
-	    if (extract_root(srcPath)) {
-		/*
-		 * Attempt to move a root directory.  Never allowed.
-		 */
-		errno = EINVAL;
-	    }
-
-	    (void) extract_root(dstPath);
-	    if (dstPath[0] == L'\0') {
-		/*
-		 * The filename was invalid.  (Don't know why,
-		 * but play it safe.)
-		 */
-		errno = EINVAL;
-	    }
-	    if (_wcsicmp(srcPath, dstPath) != 0) {
-		/*
-		 * If src is a directory and dst filesystem != src
-		 * filesystem, errno should be EXDEV.  It is very
-		 * important to get this behavior, so that the caller
-		 * can respond to a cross filesystem rename by
-		 * simulating it with copy and delete.  The MoveFile
-		 * system call already handles the case of moving a
-		 * *file* between filesystems.
-		 */
-
-		errno = EXDEV;
-	    }
-	}
-
-	/*
-	 * Other types of access failure is that dst is a read-only
-	 * filesystem, that an open file referred to src or dest, or that
-	 * src or dest specified the current working directory on the
-	 * current filesystem.  EACCES is returned for those cases.
-	 */
-
-    } else if (errno == EEXIST) {
-	/*
-	 * Reports EEXIST any time the target already exists.  If it makes
-	 * sense, remove the old file and try renaming again.
-	 */
-
-	if (srcAttr & FILE_ATTRIBUTE_DIRECTORY) {
-	    if (dstAttr & FILE_ATTRIBUTE_DIRECTORY) {
-		/*
-		 * Overwrite empty dst directory with src directory.  The
-		 * following call will remove an empty directory.  If it
-		 * fails, it's because it wasn't empty.
-		 */
-
-		if (RemoveDirectoryW(wdst)) {
-		    /*
-		     * Now that that empty directory is gone, we can try
-		     * renaming again.  If that fails, we'll put this empty
-		     * directory back, for completeness.
-		     */
-
-		    if (MoveFileW(wsrc, wdst) != FALSE) {
-			return 1;
-		    }
-
-		    /*
-		     * Some new error has occurred.  Don't know what it
-		     * could be, but report this one.
-		     */
-
-		    errno = errno_map(GetLastError());
-		    CreateDirectoryW(wdst, NULL);
-		    SetFileAttributesW(wdst, dstAttr);
-		    if (errno == EACCES) {
-			/*
-			 * Decode the EACCES to a more meaningful error.
-			 */
-
-			goto decode;
-		    }
-		}
-	    } else {	/* (dstAttr & FILE_ATTRIBUTE_DIRECTORY) == 0 */
-		errno = ENOTDIR;
-	    }
-	} else {    /* (srcAttr & FILE_ATTRIBUTE_DIRECTORY) == 0 */
-	    if (dstAttr & FILE_ATTRIBUTE_DIRECTORY) {
-		errno = EISDIR;
-	    } else {
-		/*
-		 * Overwrite existing file by:
-		 * 
-		 * 1. Rename existing file to temp name.
-		 * 2. Rename old file to new name.
-		 * 3. If success, delete temp file.  If failure,
-		 *    put temp file back to old name.
-		 */
-
-		WCHAR *tempName;
-		int result;
-		WCHAR *rest;
-		
-		tempName = get_full_wpath_tmp(state, wdst, &rest, 14);
-		if (!tempName || !rest) {
-		    save_last_error(state->errInfo);
-		    return 0;
-		}
-
-		*rest = L'\0';
-		result = -1;
-		if (GetTempFileNameW(tempName, L"erlr", 0, tempName) != 0) {
-		    /*
-		     * Strictly speaking, need the following DeleteFile and
-		     * MoveFile to be joined as an atomic operation so no
-		     * other app comes along in the meantime and creates the
-		     * same temp file.
-		     */
-		     
-		    DeleteFileW(tempName);
-		    if (MoveFileW(wdst, tempName) != FALSE) {
-			if (MoveFileW(wsrc, wdst) != FALSE) {
-			    SetFileAttributesW(tempName, FILE_ATTRIBUTE_NORMAL);
-			    DeleteFileW(tempName);
-			    return 1;
-			} else {
-			    DeleteFileW(wdst);
-			    MoveFileW(tempName, wdst);
-			}
-		    } 
-
-		    /*
-		     * Can't backup dst file or move src file.  Return that
-		     * error.  Could happen if an open file refers to dst.
-		     */
-
-		    errno = errno_map(GetLastError());
-		    if (errno == EACCES) {
-			/*
-			 * Decode the EACCES to a more meaningful error.
-			 */
-			goto decode;
-		    }
-		}
-		return result;
-	    }
-	}
-    }
-    return check_error(-1, state->errInfo);
-}
-
-int
-efile_chdir(Efile_error* errInfo,	/* Where to return error codes. */
-	    char* name)			/* Name of directory to make current. */
-{
-    /* We don't even try to handle long paths here
-     * as current working directory is always limited to MAX_PATH
-     * even if we use UNC paths and SetCurrentDirectoryW()
-     */
-    int success = check_error(_wchdir((WCHAR *) name), errInfo);
-    if (!success && errInfo->posix_errno == EINVAL)
-	/* POSIXification of errno */
-	errInfo->posix_errno = ENOENT;
-    return success;
-}
-
-int
-efile_getdcwd(Efile_error* errInfo,		/* Where to return error codes. */
-	      int drive,			/* 0 - current, 1 - A, 2 - B etc. */
-	      char* buffer,			/* Where to return the current directory. */
-	      size_t size)			/* Size of buffer. */
-{
-    WCHAR *wbuffer = (WCHAR *) buffer;
-    size_t wbuffer_size = size / 2; 
-    DBG_TRACE(1, L"#getdcwd#");
-    if (_wgetdcwd(drive, wbuffer, wbuffer_size) == NULL) {
-	return check_error(-1, errInfo);
-    }
-    DBG_TRACE1(8, "getdcwd OS=%s", wbuffer);
-    if (wcsncmp(wbuffer, L"\\\\?\\", 4) == 0) {
-	wmemmove(wbuffer, wbuffer+4, wcslen(wbuffer+4)+1);
-    }
-    for ( ; *wbuffer; wbuffer++) 
-	if (*wbuffer == L'\\')
-	    *wbuffer = L'/';
-    DBG_TRACE1(8, "getdcwd ERLANG=%s", (WCHAR*)buffer);
-    return 1;
-}
-
-int
-efile_readdir(Efile_error* errInfo, char* name, EFILE_DIR_HANDLE* dir_handle,
-	      char* buffer, size_t *size)
-{
-    Efile_call_state state;
-    int ret;
-    DBG_TRACE(dir_handle?2:1, name);
-    call_state_init(&state, errInfo);
-    ret = do_readdir(&state, name, dir_handle, buffer, size);
-    call_state_free(&state);
-    return ret;
-}
-
-static int do_readdir(Efile_call_state* state,
-                      char* name,                   /* Name of directory to list */
-		      EFILE_DIR_HANDLE* dir_handle, /* Handle of opened directory or NULL */
-		      char* buffer,                 /* Buffer to put one filename in */
-		      size_t *size)                 /* in-out size of buffer/size of filename excluding zero
-					               termination in bytes*/
-{
-    HANDLE dir;			/* Handle to directory. */
-    WIN32_FIND_DATAW findData;	/* Data found by FindFirstFile() or FindNext(). */
-    /* Alignment is not honored, this works on x86 because of alignment fixup by processor.
-       Not perfect, but faster than alinging by hand (really) */
-    WCHAR *wbuffer = (WCHAR *) buffer;
-
-    /*
-     * First time we must setup everything.
-     */
-
-    if (*dir_handle == NULL) {
-	WCHAR *wname = (WCHAR *) name;
-	WCHAR* wildcard;
-	int length;
-	WCHAR* s;
-
-	ensure_wpath_max(state, &wname, MAX_PATH-2);
-	length = wcslen(wname);
-
-	wildcard = wpath_tmp_alloc(state, length+3);
-
-	wcscpy(wildcard, wname);
-	s = wildcard+length-1;
-	if (*s != L'/' && *s != L'\\')
-	    *++s = L'\\';
-	*++s = L'*';
-	*++s = L'\0';
-	DEBUGF(("Reading %ws\n", wildcard));
-	dir = FindFirstFileW(wildcard, &findData);
-	if (dir == INVALID_HANDLE_VALUE) {
-	    set_error(state->errInfo);
-	    return 0;
-	}
-	*dir_handle = (EFILE_DIR_HANDLE) dir;
-
-	if (!IS_DOT_OR_DOTDOT(findData.cFileName)) {
-	    wcscpy(wbuffer, findData.cFileName);
-	    *size = wcslen(wbuffer)*2;
-	    return 1;
-	}
-    }
-
-    /*
-     * Retrieve the name of the next file using the directory handle.
-     */
-
-    dir = (HANDLE) *dir_handle;
-
-    for (;;) {
-	if (FindNextFileW(dir, &findData)) {
-	    if (IS_DOT_OR_DOTDOT(findData.cFileName))
-		continue;
-	    wcscpy(wbuffer, findData.cFileName);
-	    *size = wcslen(wbuffer)*2;
-	    return 1;
-	}
-
-	if (GetLastError() == ERROR_NO_MORE_FILES) {
-	    state->errInfo->posix_errno = state->errInfo->os_errno = 0;
-	}
-	else {
-	    set_error(state->errInfo);
-	}
-	FindClose(dir);
-	return 0;
-    }
-}
-
-int
-efile_openfile(Efile_error* errInfo, char* name, int flags, int* pfd, Sint64* pSize)
-{
-    Efile_call_state state;
-    int ret;
-    DBG_TRACE1(1, "openfile(%s)", name);
-    call_state_init(&state, errInfo);
-    ret = do_openfile(&state, name, flags, pfd, pSize);
-    call_state_free(&state);
-    return ret;
-}
-
-static
-int do_openfile(Efile_call_state* state,        /* Where to return error codes. */
-	        char* name,			/* Name of directory to open. */
-	        int flags,			/* Flags to use for opening. */
-	        int* pfd,			/* Where to store the file descriptor. */
-	        Sint64* pSize)			/* Where to store the size of the file. */
-{
-    Efile_error* errInfo = state->errInfo;
-    BY_HANDLE_FILE_INFORMATION fileInfo; /* File information from a handle. */
-    HANDLE fd;			/* Handle to open file. */
-    DWORD access;		/* Access mode: GENERIC_READ, GENERIC_WRITE. */
-    DWORD crFlags;
-    DWORD flagsAndAttrs = FILE_ATTRIBUTE_NORMAL;
-    WCHAR *wname = (WCHAR *) name;
-
-    switch (flags & (EFILE_MODE_READ|EFILE_MODE_WRITE)) {
-    case EFILE_MODE_READ:
-	access = GENERIC_READ;
-	crFlags = OPEN_EXISTING;
-	break;
-    case EFILE_MODE_WRITE:
-	access = GENERIC_WRITE;
-	crFlags = CREATE_ALWAYS;
-	break;
-    case EFILE_MODE_READ_WRITE:
-	access = GENERIC_READ|GENERIC_WRITE;
-	crFlags = OPEN_ALWAYS;
-	break;
-    default:
-	errno = EINVAL;
-	check_error(-1, errInfo);
-	return 0;
-    }
-
-    if (flags & EFILE_MODE_SYNC) {
-        flagsAndAttrs = FILE_FLAG_WRITE_THROUGH;
-    }
-
-    if (flags & EFILE_MODE_APPEND) {
-	crFlags = OPEN_ALWAYS;
-    }
-    if (flags & EFILE_MODE_EXCL) {
-	crFlags = CREATE_NEW;
-    }
-    ensure_wpath(state, &wname);
-    fd = CreateFileW(wname, access,
-		    FILE_SHARE_FLAGS,
-		    NULL, crFlags, flagsAndAttrs, NULL);
-
-    /*
-     * Check for errors.
-     */
-
-    if (fd == INVALID_HANDLE_VALUE) {
-	DWORD attr;
-
-	set_error(errInfo);
-
-	/*
-	 * If the error is EACESS, the reason could be that we tried to
-	 * open a directory.  In that case, we'll change the error code
-	 * to EISDIR.
-	 */
-	if (errInfo->posix_errno && 
-	    (attr = GetFileAttributesW(wname)) != INVALID_FILE_ATTRIBUTES && 
-	    (attr & FILE_ATTRIBUTE_DIRECTORY)) {
-	    errInfo->posix_errno = EISDIR;
-	}
-	return 0;
-    }
-
-    /*
-     * Get and return the length of the open file.
-     */
-       
-    if (!GetFileInformationByHandle(fd, &fileInfo))
-	return set_error(errInfo);
-    *pfd = (int) fd;
-    if (pSize) {
-	*pSize = (Sint64)
-	    (((Uint64)fileInfo.nFileSizeHigh << 32) | 
-	     (Uint64)fileInfo.nFileSizeLow);
-    }
-    return 1;
-}
-
-int 
-efile_may_openfile(Efile_error* errInfo, char *name)
-{
-    Efile_call_state state;
-    WCHAR *wname = (WCHAR *) name;
-    DWORD attr;
-    int ret;
-
-    DBG_TRACE(1, name);
-    call_state_init(&state, errInfo);
-    ensure_wpath(&state, &wname);
-    if ((attr = GetFileAttributesW(wname)) == INVALID_FILE_ATTRIBUTES) {
-	errno = ENOENT;
-	ret = check_error(-1, errInfo);
-    }
-    else if (attr & FILE_ATTRIBUTE_DIRECTORY) {
-	errno = EISDIR;
-	ret = check_error(-1, errInfo);
-    }
-    else ret = 1;
-
-    call_state_free(&state);
-    return ret;
-}
-
-void
-efile_closefile(fd)
-int fd;				/* File descriptor for file to close. */
-{
-    DBG_TRACE(2, L"");
-    CloseHandle((HANDLE) fd);
-}
-
-FILE* efile_wfopen(const WCHAR* name, const WCHAR* mode)
-{
-    Efile_call_state state;
-    Efile_error dummy;
-    FILE* f;
-    call_state_init(&state, &dummy);
-    ensure_wpath(&state, (WCHAR**)&name);
-    f = _wfopen(name, mode);
-    call_state_free(&state);
-    return f;
-}
-
-int
-efile_fdatasync(errInfo, fd)
-Efile_error* errInfo;		/* Where to return error codes. */
-int fd;				/* File descriptor for file to sync. */
-{
-    DBG_TRACE(2, L"");
-    /* Not available in Windows, just call regular fsync */
-    return efile_fsync(errInfo, fd);
-}
-
-int
-efile_fsync(errInfo, fd)
-Efile_error* errInfo;		/* Where to return error codes. */
-int fd;				/* File descriptor for file to sync. */
-{
-    DBG_TRACE(2, L"");
-    if (!FlushFileBuffers((HANDLE) fd)) {
-	return check_error(-1, errInfo);
-    }
-    return 1;
-}
-
-int
-efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
-	       char* orig_name, int info_for_link)
-{
-    Efile_call_state state;
-    int ret;
-    DBG_TRACE(1, L"");
-    call_state_init(&state, errInfo);
-    ret = do_fileinfo(&state, pInfo, orig_name, info_for_link);
-    call_state_free(&state);
-    return ret;
-}
-
-static int
-do_fileinfo(Efile_call_state* state, Efile_info* pInfo,
-	    char* orig_name, int info_for_link)
-{
-    Efile_error* errInfo = state->errInfo;
-    HANDLE findhandle;		/* Handle returned by FindFirstFile(). */
-    WIN32_FIND_DATAW findbuf;	/* Data return by FindFirstFile(). */
-    WCHAR* name = NULL;
-    WCHAR* win_path;
-    int name_len;
-    int drive;			/* Drive for filename (1 = A:, 2 = B: etc). */
-    WCHAR *worig_name = (WCHAR *) orig_name;
-
-    ensure_wpath(state, &worig_name);
-    /* Don't allow wildcards to be interpreted by system */
-
-
-    /*
-     * Move the name to a buffer and make sure to remove a trailing
-     * slash, because it causes FindFirstFile() to fail on Win95.
-     */
-
-    name_len = wcslen(worig_name);
-
-    name = wpath_tmp_alloc(state, name_len+1);
-    wcscpy(name, worig_name);
-    if (name_len > 2 && ISSLASH(name[name_len-1]) &&
-	name[name_len-2] != L':') {
-	name[name_len-1] = L'\0';
-    }
-
-    win_path = name;
-    if (wcsncmp(name, L"\\\\?\\", 4) == 0) {
-	win_path += 4;
-    }
-
-    if (wcspbrk(win_path, L"?*")) {
-    enoent:
-	errInfo->posix_errno = ENOENT;
-	errInfo->os_errno = ERROR_FILE_NOT_FOUND;
-        return 0;
-    }
-
-    /* Try to get disk from name.  If none, get current disk.  */
-
-    if (win_path[1] != L':') {
-	WCHAR* cwd_path = get_cwd_wpath_tmp(state);
-        drive = 0;
-	if (cwd_path[1] == L':') {
-	    drive = towlower(cwd_path[0]) - L'a' + 1;
-	}
-    } else if (*win_path && win_path[2] == L'\0') {
-	/*
-	 * X: and nothing more is an error.
-	 */
-	errInfo->posix_errno = ENOENT;
-	errInfo->os_errno = ERROR_FILE_NOT_FOUND;
-	return 0;
-    } else {
-        drive = towlower(*win_path) - L'a' + 1;
-    }
-
-    findhandle = FindFirstFileW(name, &findbuf);
-    if (findhandle == INVALID_HANDLE_VALUE) {
-	WCHAR* path = NULL;
-
-        if (!(wcspbrk(name, L"./\\") &&
-	      (path = get_full_wpath_tmp(state, name, NULL, 0)) &&
-	      /* root dir. ('C:\') or UNC root dir. ('\\server\share\') */
-	      ((wcslen(path) == 3) || is_root_unc_name(path)) &&
-	      (GetDriveTypeW(path) > 1)   ) ) {
-
-	    errInfo->posix_errno = ENOENT;
-	    errInfo->os_errno = ERROR_FILE_NOT_FOUND;
-	    return 0;
-	}
-
-        /*
-         * Root directories (such as C:\ or \\server\share\ are fabricated.
-         */
-	
-        findbuf.dwFileAttributes = FILE_ATTRIBUTE_DIRECTORY;
-        findbuf.nFileSizeHigh = 0;
-        findbuf.nFileSizeLow = 0;
-        findbuf.cFileName[0] = L'\0';
-
-	pInfo->links = 1;
-	pInfo->cTime = pInfo->accessTime = pInfo->modifyTime = 0;
-    } else {
-	SYSTEMTIME SystemTime;
-        FILETIME LocalFTime;
-
-	/*first check if we are a symlink */
-	if (!info_for_link && (findbuf.dwFileAttributes &
-			       FILE_ATTRIBUTE_REPARSE_POINT)){
-	    /*
-	     * given that we know this is a symlink,
-	     we should be able to find its target */
-	    WCHAR* target_name = (WCHAR*) do_readlink(state, (char *) name, NULL, 0);
-	    if (target_name) {
-		FindClose(findhandle);
-		return do_fileinfo(state, pInfo,
-				   (char *) target_name, info_for_link);
-	    }
-	}
-
-	/* number of links: */
-	{
-	    HANDLE handle;	/* Handle returned by CreateFile() */
-	    BY_HANDLE_FILE_INFORMATION fileInfo; /* from  CreateFile() */
-
-            /* We initialise nNumberOfLinks as GetFileInformationByHandle
-               does not always initialise this field */
-            fileInfo.nNumberOfLinks = 1;
-	    if (handle = CreateFileW(name, GENERIC_READ, FILE_SHARE_FLAGS, NULL,
-				    OPEN_EXISTING, 0, NULL)) {
-		GetFileInformationByHandle(handle, &fileInfo);
-		pInfo->links = fileInfo.nNumberOfLinks;
-		CloseHandle(handle);
-	    } else {
-		pInfo->links = 1;
-	    }	
-	}
-
-        FILETIME_TO_EPOCH(pInfo->modifyTime, findbuf.ftLastWriteTime);
-
-        if (findbuf.ftLastAccessTime.dwLowDateTime == 0 &&
-	    findbuf.ftLastAccessTime.dwHighDateTime == 0) {
-	    pInfo->accessTime = pInfo->modifyTime;
-	} else {
-	    FILETIME_TO_EPOCH(pInfo->accessTime, findbuf.ftLastAccessTime);
-	}
-
-        if (findbuf.ftCreationTime.dwLowDateTime == 0 &&
-	    findbuf.ftCreationTime.dwHighDateTime == 0) {
-	    pInfo->cTime = pInfo->modifyTime;
-	} else {
-	    FILETIME_TO_EPOCH(pInfo->cTime ,findbuf.ftCreationTime);
-	}
-        FindClose(findhandle);
-    }
-
-    pInfo->size_low = findbuf.nFileSizeLow;
-    pInfo->size_high = findbuf.nFileSizeHigh;
-	
-    if (info_for_link && (findbuf.dwFileAttributes &
-			  FILE_ATTRIBUTE_REPARSE_POINT))
-	pInfo->type = FT_SYMLINK;
-    else if (findbuf.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
-	pInfo->type = FT_DIRECTORY;
-    else
-	pInfo->type = FT_REGULAR;
-
-    if (findbuf.dwFileAttributes & FILE_ATTRIBUTE_READONLY)
-	pInfo->access = FA_READ;
-    else
-	pInfo->access = FA_READ|FA_WRITE;
-
-    pInfo->mode = dos_to_posix_mode(findbuf.dwFileAttributes, name);
-    pInfo->major_device = drive;
-    pInfo->minor_device = 0;
-    pInfo->inode = 0;
-    pInfo->uid = 0;
-    pInfo->gid = 0;
-    
-    return 1;
-}
-
-int
-efile_write_info(Efile_error* errInfo,
-		 Efile_info* pInfo,
-		 char* name)
-{
-    Efile_call_state state;
-    int ret;
-    call_state_init(&state, errInfo);
-    ret = do_write_info(&state, pInfo, name);
-    call_state_free(&state);
-    return ret;
-}
-
-static int
-do_write_info(Efile_call_state* state,
-              Efile_info* pInfo,
-	      char* name)
-{
-    Efile_error* errInfo = state->errInfo;
-    SYSTEMTIME timebuf;
-    FILETIME ModifyFileTime;
-    FILETIME AccessFileTime;
-    FILETIME CreationFileTime;
-    HANDLE fd;
-    DWORD attr;
-    DWORD tempAttr;
-    WCHAR *wname = (WCHAR *) name;
-
-    DBG_TRACE(1, name);
-
-    ensure_wpath(state, &wname);
-
-    /*
-     * Get the attributes for the file.
-     */
-
-    tempAttr = attr = GetFileAttributesW(wname);
-    if (attr == 0xffffffff) {
-	return set_error(errInfo);
-    }
-    if (pInfo->mode != -1) {
-	if (pInfo->mode & _S_IWRITE) {
-	    /* clear read only bit */
-	    attr &= ~FILE_ATTRIBUTE_READONLY;
-	} else {
-	    /* set read only bit */
-	    attr |= FILE_ATTRIBUTE_READONLY;
-	}
-    }
-
-    /*
-     * Construct all file times.
-     */
-
-    EPOCH_TO_FILETIME(ModifyFileTime,   pInfo->modifyTime);
-    EPOCH_TO_FILETIME(AccessFileTime,   pInfo->accessTime);
-    EPOCH_TO_FILETIME(CreationFileTime, pInfo->cTime);
-
-    /*
-     * If necessary, set the file times.
-     */
-
-    /*
-     * If the has read only access, we must temporarily turn on
-     * write access (this is necessary for native filesystems,
-     * but not for NFS filesystems).
-     */
-
-    if (tempAttr & FILE_ATTRIBUTE_READONLY) {
-	tempAttr &= ~FILE_ATTRIBUTE_READONLY;
-	if (!SetFileAttributesW(wname, tempAttr)) {
-	    return set_error(errInfo);
-	}
-    }
-
-    fd = CreateFileW(wname, GENERIC_READ|GENERIC_WRITE,
-	    FILE_SHARE_FLAGS,
-	    NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
-    if (fd != INVALID_HANDLE_VALUE) {
-	BOOL result = SetFileTime(fd, &CreationFileTime, &AccessFileTime, &ModifyFileTime);
-	if (!result) {
-	    return set_error(errInfo);
-	}
-	CloseHandle(fd);
-    }
-
-    /*
-     * If the file doesn't have the correct attributes, set them now.
-     * (It could have been done before setting the file times, above).
-     */
-
-    if (tempAttr != attr) {
-	if (!SetFileAttributesW(wname, attr)) {
-	    return set_error(errInfo);
-	}
-    }
-    return 1;
-}
-
-
-int
-efile_pwrite(errInfo, fd, buf, count, offset)
-Efile_error* errInfo;		/* Where to return error codes. */
-int fd;				/* File descriptor to write to. */
-char* buf;			/* Buffer to write. */
-size_t count;			/* Number of bytes to write. */
-Sint64 offset;			/* where to write it */
-{
-    int res;
-    DBG_TRACE(2, L"");
-    res = efile_seek(errInfo, fd, offset, EFILE_SEEK_SET, NULL);
-    if (res) {
-	return efile_write(errInfo, EFILE_MODE_WRITE, fd, buf, count);
-    } else {
-	return res;
-    }
-}
-
-/* position and read/write as a single atomic op */
-int
-efile_pread(errInfo, fd, offset, buf, count, pBytesRead)
-Efile_error* errInfo;		/* Where to return error codes. */
-int fd;				/* File descriptor to read from. */
-Sint64 offset;			/* Offset in bytes from BOF. */
-char* buf;			/* Buffer to read into. */
-size_t count;			/* Number of bytes to read. */
-size_t* pBytesRead;		/* Where to return number of bytes read. */
-{
-    int res;
-    DBG_TRACE(2, L"");
-    res = efile_seek(errInfo, fd, offset, EFILE_SEEK_SET, NULL);
-    if (res) {
-	return efile_read(errInfo, EFILE_MODE_READ, fd, buf, count, pBytesRead);
-    } else {
-	return res;
-    }
-}
-
-
-
-int
-efile_write(errInfo, flags, fd, buf, count)
-Efile_error* errInfo;		/* Where to return error codes. */
-int flags;			/* Flags given when file was opened. */
-int fd;				/* File descriptor to write to. */
-char* buf;			/* Buffer to write. */
-size_t count;			/* Number of bytes to write. */
-{
-    DWORD written;		/* Bytes written in last operation. */
-    OVERLAPPED overlapped;
-    OVERLAPPED* pOverlapped = NULL;
-
-    DBG_TRACE(2, L"");
-    if (flags & EFILE_MODE_APPEND) {
-	memset(&overlapped, 0, sizeof(overlapped));
-	overlapped.Offset = 0xffffffff;
-	overlapped.OffsetHigh = 0xffffffff;
-	pOverlapped = &overlapped;
-    }
-    while (count > 0) {
-	if (!WriteFile((HANDLE) fd, buf, count, &written, pOverlapped))
-	    return set_error(errInfo);
-	buf += written;
-	count -= written;
-    }
-    return 1;
-}
-
-int
-efile_writev(Efile_error* errInfo,   /* Where to return error codes */
-	     int flags,              /* Flags given when file was
-				      * opened */
-	     int fd,                 /* File descriptor to write to */
-	     SysIOVec* iov,          /* Vector of buffer structs.
-				      * The structs are unchanged 
-				      * after the call */
-	     int iovcnt)             /* Number of structs in vector */
-{
-    int cnt;                         /* Buffers so far written */
-    OVERLAPPED overlapped;
-    OVERLAPPED* pOverlapped = NULL;
-
-    DBG_TRACE(2, L"");
-    ASSERT(iovcnt >= 0);
-    
-    if (flags & EFILE_MODE_APPEND) {
-	memset(&overlapped, 0, sizeof(overlapped));
-	overlapped.Offset = 0xffffffff;
-	overlapped.OffsetHigh = 0xffffffff;
-	pOverlapped = &overlapped;
-    }
-    for (cnt = 0; cnt < iovcnt; cnt++) {
-	if (iov[cnt].iov_base && iov[cnt].iov_len > 0) {
-	    /* Non-empty buffer */
-	    int p;                   /* Position in buffer */
-	    int w = iov[cnt].iov_len;/* Bytes written in this call */
-	    for (p = 0; p < iov[cnt].iov_len; p += w) {
-		if (!WriteFile((HANDLE) fd, 
-			       iov[cnt].iov_base + p, 
-			       iov[cnt].iov_len - p, 
-			       &w, 
-			       pOverlapped))
-		    return set_error(errInfo);
-	    }
-	}
-    }	
-    return 1;
-}
-
-int
-efile_read(errInfo, flags, fd, buf, count, pBytesRead)
-Efile_error* errInfo;		/* Where to return error codes. */
-int flags;			/* Flags given when file was opened. */
-int fd;				/* File descriptor to read from. */
-char* buf;			/* Buffer to read into. */
-size_t count;			/* Number of bytes to read. */
-size_t* pBytesRead;		/* Where to return number of bytes read. */
-{
-    DWORD nbytes = 0;
-
-    DBG_TRACE(2, L"");
-    if (!ReadFile((HANDLE) fd, buf, count, &nbytes, NULL))
-	return set_error(errInfo);
-
-    *pBytesRead = nbytes;
-    return 1;
-}
-
-int
-efile_seek(errInfo, fd, offset, origin, new_location)
-Efile_error* errInfo;		/* Where to return error codes. */
-int fd;				/* File descriptor to do the seek on. */
-Sint64 offset;			/* Offset in bytes from the given origin. */
-int origin;			/* Origin of seek (SEEK_SET, SEEK_CUR,
-				 * SEEK_END).
-				 */
-Sint64* new_location;		/* Resulting new location in file. */
-{
-    LARGE_INTEGER off, new_loc;
-    
-    DBG_TRACE(2, L"");
-    switch (origin) {
-    case EFILE_SEEK_SET: origin = FILE_BEGIN; break;
-    case EFILE_SEEK_CUR: origin = FILE_CURRENT; break;
-    case EFILE_SEEK_END: origin = FILE_END; break;
-    default:
-	errno = EINVAL;
-	check_error(-1, errInfo);
-	break;
-    }
-    
-    off.QuadPart = offset;
-    if (! SetFilePointerEx((HANDLE) fd, off,
-	new_location ? &new_loc : NULL, origin)) {
-	return set_error(errInfo);
-    }
-    if (new_location) {
-	*new_location = new_loc.QuadPart;
-	DEBUGF(("efile_seek(offset=%ld, origin=%d) -> %ld\n", 
-		(long) offset, origin, (long) *new_location));
-    } else {
-	DEBUGF(("efile_seek(offset=%ld, origin=%d)\n", (long) offset, origin));
-    }
-    return 1;
-}
-
-int
-efile_truncate_file(errInfo, fd, flags)
-Efile_error* errInfo;		/* Where to return error codes. */
-int *fd;				/* File descriptor for file to truncate. */
-int flags;
-{
-    DBG_TRACE(2, L"");
-    if (!SetEndOfFile((HANDLE) (*fd)))
-	return set_error(errInfo);
-    return 1;
-}
-
-
-/*
- * is_root_unc_name - returns TRUE if the argument is a UNC name specifying
- *      a root share.  That is, if it is of the form \\server\share\.
- *      This routine will also return true if the argument is of the
- *      form \\server\share (no trailing slash) but Win32 currently
- *      does not like that form.
- *
- *      Forward slashes ('/') may be used instead of backslashes ('\').
- */
-
-static int
-is_root_unc_name(const WCHAR *path)
-{
-    /*
-     * If a root UNC name, path will start with 2 (but not 3) slashes
-     */
-
-    if ((wcslen(path) >= 5) /* minimum string is "//x/y" */
-	&& ISSLASH(path[0]) && ISSLASH(path[1]))
-    {
-        const WCHAR *p = path + 2;
-
-        /*
-         * find the slash between the server name and share name
-         */
-        while ( * ++ p )
-            if ( ISSLASH(*p) )
-                break ;
-
-        if ( *p && p[1] )
-        {
-            /*
-             * is there a further slash?
-             */
-            while ( * ++ p )
-                if ( ISSLASH(*p) )
-                    break ;
-
-            /*
-             * just final slash (or no final slash)
-             */
-            if ( !*p || !p[1])
-                return 1;
-        }
-    }
-
-    return 0 ;
-}
-
-/*
- * Extracts the root part of an absolute filename (by modifying the string
- * pointed to by the name argument).  The name can start
- * with either a driver letter (for example, C:\), or a UNC name
- * (for example, \\guinness\bjorn).
- *
- * If the name is invalid, the buffer will be modified to point to
- * an empty string.
- *
- * Returns: 1 if the name consists of just the root part, 0 if
- * 	    the name was longer.
- */
-
-static int
-extract_root(WCHAR* name)
-{
-    int len = wcslen(name);
-
-    if (iswalpha(name[0]) && name[1] == L':' && ISSLASH(name[2])) {
-	WCHAR c = name[3];
-	name[3] = L'\0';
-	return c == L'\0';
-    } else if (len < 5 || !ISSLASH(name[0]) || !ISSLASH(name[1])) {
-	goto error;
-    } else {			/* Try to find the end of the UNC name. */
-	WCHAR* p;
-	WCHAR c;
-
-        /*
-         * Find the slash between the server name and share name.
-         */
-
-	for (p = name + 2; *p; p++)
-            if (ISSLASH(*p))
-                break;
-	if (*p == L'\0')
-	    goto error;
-
-	/*
-	 * Find the slash after the share name.
-	 */
-
-	for (p++; *p; p++)
-            if (ISSLASH(*p))
-                break;
-	c = *p;
-	*p = L'\0';
-	return c == L'\0' || p[1] == L'\0';
-    }
-
- error:
-    *name = L'\0';
-    return 1;
-}
-
-static unsigned short
-dos_to_posix_mode(int attr, const WCHAR *name)
-{
-    register unsigned short uxmode;
-    unsigned dosmode;
-    register const WCHAR *p;
-
-    dosmode = attr & 0xff;
-    if ((p = name)[1] == L':')
-	p += 2;
-
-    /* check to see if this is a directory - note we must make a special
-     * check for the root, which DOS thinks is not a directory
-     */
-
-    uxmode = (unsigned short)
-	(((ISSLASH(*p) && !p[1]) || (dosmode & FILE_ATTRIBUTE_DIRECTORY) ||
-	       *p == L'\0') ? _S_IFDIR|_S_IEXEC : _S_IFREG);
-
-    /* If attribute byte does not have read-only bit, it is read-write */
-
-    uxmode |= (dosmode & FILE_ATTRIBUTE_READONLY) ?
-	_S_IREAD : (_S_IREAD|_S_IWRITE);
-
-    /* see if file appears to be executable - check extension of name */
-
-    if (p = wcsrchr(name, L'.')) {
-        if (!_wcsicmp(p, L".exe") ||
-	    !_wcsicmp(p, L".cmd") ||
-	    !_wcsicmp(p, L".bat") ||
-	    !_wcsicmp(p, L".com"))
-            uxmode |= _S_IEXEC;
-    }
-
-    /* propagate user read/write/execute bits to group/other fields */
-
-    uxmode |= (uxmode & 0700) >> 3;
-    uxmode |= (uxmode & 0700) >> 6;
-
-    return uxmode;
-}
-
-
-int
-efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size)
-{
-    Efile_call_state state;
-    int ret;
-    DBG_TRACE(1, name);
-    call_state_init(&state, errInfo);
-    ret = !!do_readlink(&state, name, buffer, size);
-    call_state_free(&state);
-    return ret;
-}
-
-/* If buffer==0, return buffer allocated by wpath_tmp_allocate
-*/
-static char*
-do_readlink(Efile_call_state* state, char* name, char* buffer, size_t size)
-{
-    /*
-     * load dll and see if we have CreateSymbolicLink at runtime:
-     * (Vista only)
-     */
-    HINSTANCE hModule = NULL;
-    WCHAR *wname = (WCHAR *) name;
-    WCHAR *wbuffer = (WCHAR *) buffer;
-    DWORD wsize = size / sizeof(WCHAR);
-    char* ret = NULL;
-
-    if ((hModule = LoadLibrary("kernel32.dll")) != NULL) {
-	typedef DWORD (WINAPI * GETFINALPATHNAMEBYHANDLEPTR)(
-							     HANDLE hFile,
-							     LPCWSTR lpFilePath,
-							     DWORD cchFilePath,
-							     DWORD dwFlags);
-
-	GETFINALPATHNAMEBYHANDLEPTR pGetFinalPathNameByHandle =
-	    (GETFINALPATHNAMEBYHANDLEPTR)GetProcAddress(hModule, "GetFinalPathNameByHandleW");
-
-	if (pGetFinalPathNameByHandle != NULL) {
-	    DWORD fileAttributes;
-	    ensure_wpath(state, &wname);
-	    /* first check if file is a symlink; {error, einval} otherwise */
-	    fileAttributes = GetFileAttributesW(wname);
-	    if ((fileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) {
-		DWORD success = 0;
-		HANDLE h = CreateFileW(wname, GENERIC_READ, FILE_SHARE_FLAGS, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL);
-		int len;
-		if(h != INVALID_HANDLE_VALUE) {
-		    if (!wbuffer) { /* dynamic allocation */
-			WCHAR dummy;
-			wsize = pGetFinalPathNameByHandle(h, &dummy, 0, 0);
-			if (wsize) {
-			    wbuffer = wpath_tmp_alloc(state, wsize);
-			}
-		    }
-		    if (wbuffer
-			&& (success = pGetFinalPathNameByHandle(h, wbuffer, wsize, 0))
-			&& success < wsize) {
-			WCHAR* wp;
-
-			/* GetFinalPathNameByHandle prepends path with "\\?\": */
-			len = wcslen(wbuffer);
-			wmemmove(wbuffer,wbuffer+4,len-3);
-			if (len - 4 >= 2 && wbuffer[1] == L':' && wbuffer[0] >= L'A' &&
-			    wbuffer[0] <= L'Z') {
-			    wbuffer[0] = wbuffer[0] + L'a' - L'A';
-			}
-			
-			for (wp=wbuffer ; *wp; wp++)
-			    if (*wp == L'\\')
-				*wp = L'/';
-		    }
-		    CloseHandle(h);
-		}
-		if (success) {
-		    ret = (char*) wbuffer;
-		} else {
-		    set_error(state->errInfo);
-		}
-	    } else {
-		errno = EINVAL;
-		save_last_error(state->errInfo);
-	    }
-	    goto done;
-	}
-    }
-    errno = ENOTSUP;
-    save_last_error(state->errInfo);
-
-done:
-    if (hModule)
-	FreeLibrary(hModule);
-    return ret;
-}
-
-
-int
-efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size)
-{
-    Efile_call_state state;
-    int ret;
-    DBG_TRACE(1, orig_name);
-    call_state_init(&state, errInfo);
-    ret = do_altname(&state, orig_name, buffer, size);
-    call_state_free(&state);
-    return ret;
-}
-
-static int
-do_altname(Efile_call_state* state, char* orig_name, char* buffer, size_t size)
-{
-    WIN32_FIND_DATAW wfd;
-    HANDLE fh;
-    WCHAR* name;
-    int name_len;
-    WCHAR* full_path = NULL;
-    WCHAR *worig_name = (WCHAR *) orig_name;
-    WCHAR *wbuffer = (WCHAR *) buffer;
-    int drive;			/* Drive for filename (1 = A:, 2 = B: etc). */
-
-    /* Don't allow wildcards to be interpreted by system */
-
-    if (wcspbrk(worig_name, L"?*")) {
-    enoent:
-	state->errInfo->posix_errno = ENOENT;
-	state->errInfo->os_errno = ERROR_FILE_NOT_FOUND;
-        return 0;
-    }
-
-    /*
-     * Move the name to a buffer and make sure to remove a trailing
-     * slash, because it causes FindFirstFile() to fail on Win95.
-     */
-    ensure_wpath(state, &worig_name);
-    name_len = wcslen(worig_name);
-
-    name = wpath_tmp_alloc(state, name_len + 1);
-    wcscpy(name, worig_name);
-    if (name_len > 2 && ISSLASH(name[name_len-1]) &&
-	name[name_len-2] != L':') {
-	name[name_len-1] = L'\0';
-    }
-    
-    /* Try to get disk from name.  If none, get current disk.  */
-
-    if (name[1] != L':') {
-	WCHAR* cwd_path = get_cwd_wpath_tmp(state);
-        drive = 0;
-        if (cwd_path[1] == L':') {
-	    drive = towlower(cwd_path[0]) - L'a' + 1;
-	}
-    } else if (*name && name[2] == L'\0') {
-	/*
-	 * X: and nothing more is an error.
-	 */
-	goto enoent;
-    } else {
-        drive = towlower(*name) - L'a' + 1;
-    }
-    fh = FindFirstFileW(name,&wfd);
-    if (fh == INVALID_HANDLE_VALUE) {
-	DWORD fff_error = GetLastError();
-        if (!(wcspbrk(name, L"./\\") &&
-	      (full_path = get_full_wpath_tmp(state, name, NULL, 0)) &&
-	      /* root dir. ('C:\') or UNC root dir. ('\\server\share\') */
-	      ((wcslen(full_path) == 3) || is_root_unc_name(full_path)) &&
-	      (GetDriveTypeW(full_path) > 1)   ) ) {
-
-	    set_os_errno(state->errInfo, fff_error);
-	    return 0;
-	}
-        /*
-         * Root directories (such as C:\ or \\server\share\ are fabricated.
-         */
-	wcscpy(wbuffer,name);
-	return 1;
-    }
-	
-    wcscpy(wbuffer,wfd.cAlternateFileName);
-    if (!*wbuffer) {
-	wcscpy(wbuffer,wfd.cFileName);
-    }
-    FindClose(fh);
-    return 1;
-}
-
-
-int
-efile_link(Efile_error* errInfo, char* old, char* new)
-{
-    Efile_call_state state;
-    WCHAR *wold = (WCHAR *) old;
-    WCHAR *wnew = (WCHAR *) new;
-    int ret;
-    DBG_TRACE(1, old);
-    call_state_init(&state, errInfo);
-    ensure_wpath(&state, &wold);
-    ensure_wpath(&state, &wnew);
-    if(!CreateHardLinkW(wnew, wold, NULL)) {
-	ret = set_error(errInfo);
-    }
-    else ret =1;
-    call_state_free(&state);
-    return ret;
-}
-
-int
-efile_symlink(Efile_error* errInfo, char* old, char* new)
-{
-    Efile_call_state state;
-    int ret;
-    DBG_TRACE2(1, "symlink(%s <- %s)", old, new);
-    call_state_init(&state, errInfo);
-    ret = do_symlink(&state, old, new);
-    call_state_free(&state);
-    return ret;
-}
-
-static int
-do_symlink(Efile_call_state* state, char* old, char* new)
-{
-    /*
-     * Load dll and see if we have CreateSymbolicLink at runtime:
-     * (Vista only)
-     */
-    HINSTANCE hModule = NULL;
-    WCHAR *wold = (WCHAR *) old;
-    WCHAR *wnew = (WCHAR *) new;
-
-    DBG_TRACE(1, old);
-    if ((hModule = LoadLibrary("kernel32.dll")) != NULL) {
-	typedef BOOLEAN (WINAPI  * CREATESYMBOLICLINKFUNCPTR) (
-	     LPCWSTR lpSymlinkFileName,
-	     LPCWSTR lpTargetFileName,
-	     DWORD dwFlags);
-
-	CREATESYMBOLICLINKFUNCPTR pCreateSymbolicLink =
-	    (CREATESYMBOLICLINKFUNCPTR) GetProcAddress(hModule,
-						       "CreateSymbolicLinkW");
-	/* A for MBCS, W for UNICODE... char* above implies 'W'! */
-	if (pCreateSymbolicLink != NULL) {
-	  ensure_wpath(state, &wold);
-	  ensure_wpath(state, &wnew);
-	  {
-	    DWORD attr = GetFileAttributesW(wold);
-	    int flag = (attr != INVALID_FILE_ATTRIBUTES &&
-			attr & FILE_ATTRIBUTE_DIRECTORY) ? 1 : 0;
-	    /*  SYMBOLIC_LINK_FLAG_DIRECTORY = 1 */
-	    BOOLEAN success = pCreateSymbolicLink(wnew, wold, flag);
-	    FreeLibrary(hModule);
-
-	    if (success) {
-		return 1;
-	    } else {
-		return set_error(state->errInfo);
-	    }
-	  }
-	} else
-	    FreeLibrary(hModule);
-    }
-    errno = ENOTSUP;
-    return check_error(-1, state->errInfo);
-}
-
-int
-efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset,
-	    Sint64 length, int advise)
-{
-    DBG_TRACE(2, L"");
-    /* posix_fadvise is not available on Windows, do nothing */
-    errno = ERROR_SUCCESS;
-    return check_error(0, errInfo);
-}
-
-int
-efile_fallocate(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length)
-{
-    DBG_TRACE(2, L"");
-    /* No file preallocation method available in Windows. */
-    errno = errno_map(ERROR_NOT_SUPPORTED);
-    SetLastError(ERROR_NOT_SUPPORTED);
-
-    return check_error(-1, errInfo);
-}
diff --git a/erts/emulator/nifs/common/prim_file_nif.c b/erts/emulator/nifs/common/prim_file_nif.c
new file mode 100644
index 0000000000..6874f41d75
--- /dev/null
+++ b/erts/emulator/nifs/common/prim_file_nif.c
@@ -0,0 +1,1237 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson 2017. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#define STATIC_ERLANG_NIF 1
+
+#include "erl_nif.h"
+#include "config.h"
+#include "sys.h"
+
+#ifdef VALGRIND
+#  include <valgrind/memcheck.h>
+#endif
+
+#include "erl_driver.h"
+#include "prim_file_nif.h"
+
+/* NIF interface declarations */
+static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info);
+static int upgrade(ErlNifEnv *env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info);
+static void unload(ErlNifEnv *env, void* priv_data);
+
+static ErlNifResourceType *efile_resource_type;
+
+static ERL_NIF_TERM am_ok;
+static ERL_NIF_TERM am_error;
+static ERL_NIF_TERM am_continue;
+
+static ERL_NIF_TERM am_file_info;
+
+/* File modes */
+static ERL_NIF_TERM am_read;
+static ERL_NIF_TERM am_write;
+static ERL_NIF_TERM am_exclusive;
+static ERL_NIF_TERM am_append;
+static ERL_NIF_TERM am_sync;
+static ERL_NIF_TERM am_skip_type_check;
+
+/* enum efile_access_t; read and write are defined above.*/
+static ERL_NIF_TERM am_read_write;
+static ERL_NIF_TERM am_none;
+
+/* enum efile_advise_t */
+static ERL_NIF_TERM am_normal;
+static ERL_NIF_TERM am_random;
+static ERL_NIF_TERM am_sequential;
+static ERL_NIF_TERM am_will_need;
+static ERL_NIF_TERM am_dont_need;
+static ERL_NIF_TERM am_no_reuse;
+
+/* enum efile_filetype_t */
+static ERL_NIF_TERM am_device;
+static ERL_NIF_TERM am_directory;
+static ERL_NIF_TERM am_regular;
+static ERL_NIF_TERM am_symlink;
+static ERL_NIF_TERM am_other;
+
+/* enum efile_seek_t, 'eof' marker. */
+static ERL_NIF_TERM am_bof;
+static ERL_NIF_TERM am_cur;
+static ERL_NIF_TERM am_eof;
+
+static ERL_NIF_TERM read_info_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM set_permissions_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM set_owner_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM set_time_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+static ERL_NIF_TERM read_link_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM list_dir_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+static ERL_NIF_TERM make_hard_link_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM make_soft_link_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM rename_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM make_dir_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM del_file_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM del_dir_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM get_device_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM get_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM set_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+static ERL_NIF_TERM read_file_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+static ERL_NIF_TERM get_handle_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM altname_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+static ERL_NIF_TERM open_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+/* All file handle operations are passed through a wrapper that handles state
+ * transitions, marking it as busy during the course of the operation, and
+ * closing on completion if the owner died in the middle of an operation.
+ *
+ * This is pretty ugly but required as there's no way to tell when it's safe to
+ * asynchronously close a file; the event could have fired just before landing
+ * in a system call which will fail with EBADF at best or alias a newly opened
+ * fd at worst.
+ *
+ * The old driver got away with enqueueing the close operation on the same
+ * async queue as all of its other operations, but since dirty schedulers use a
+ * single global queue there's no natural way to schedule an asynchronous close
+ * "behind" other operations.
+ *
+ * The states may transition as follows:
+ *
+ * IDLE ->
+ *      BUSY (file_handle_wrapper) |
+ *      CLOSED (owner_death_callback)
+ *
+ * BUSY ->
+ *      IDLE (file_handle_wrapper)
+ *      CLOSED (close_nif_impl)
+ *      CLOSE_PENDING (owner_death_callback)
+ *
+ * CLOSE_PENDING ->
+ *      CLOSED (file_handle_wrapper)
+ */
+
+typedef ERL_NIF_TERM (*file_op_impl_t)(efile_data_t *d, ErlNifEnv *env,
+    int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM file_handle_wrapper(file_op_impl_t operation, ErlNifEnv *env,
+    int argc, const ERL_NIF_TERM argv[]);
+
+#define WRAP_FILE_HANDLE_EXPORT(name) \
+    static ERL_NIF_TERM name ## _impl (efile_data_t *d, ErlNifEnv *env, \
+        int argc, const ERL_NIF_TERM argv[]);\
+    static ERL_NIF_TERM name(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { \
+        return file_handle_wrapper( name ## _impl , env, argc, argv); \
+    }
+
+WRAP_FILE_HANDLE_EXPORT(close_nif)
+WRAP_FILE_HANDLE_EXPORT(read_nif)
+WRAP_FILE_HANDLE_EXPORT(write_nif)
+WRAP_FILE_HANDLE_EXPORT(pread_nif)
+WRAP_FILE_HANDLE_EXPORT(pwrite_nif)
+WRAP_FILE_HANDLE_EXPORT(seek_nif)
+WRAP_FILE_HANDLE_EXPORT(sync_nif)
+WRAP_FILE_HANDLE_EXPORT(truncate_nif)
+WRAP_FILE_HANDLE_EXPORT(allocate_nif)
+WRAP_FILE_HANDLE_EXPORT(advise_nif)
+WRAP_FILE_HANDLE_EXPORT(get_handle_nif)
+WRAP_FILE_HANDLE_EXPORT(ipread_s32bu_p32bu_nif)
+
+static ErlNifFunc nif_funcs[] = {
+    /* File handle ops */
+    {"open_nif", 2, open_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"close_nif", 1, close_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"read_nif", 2, read_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"write_nif", 2, write_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"pread_nif", 3, pread_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"pwrite_nif", 3, pwrite_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"seek_nif", 3, seek_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"sync_nif", 2, sync_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"truncate_nif", 1, truncate_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"allocate_nif", 3, allocate_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"advise_nif", 4, advise_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+
+    /* Filesystem ops */
+    {"make_hard_link_nif", 2, make_hard_link_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"make_soft_link_nif", 2, make_soft_link_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"rename_nif", 2, rename_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"read_info_nif", 2, read_info_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"set_permissions_nif", 2, set_permissions_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"set_owner_nif", 3, set_owner_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"set_time_nif", 4, set_time_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"read_link_nif", 1, read_link_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"list_dir_nif", 1, list_dir_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"make_dir_nif", 1, make_dir_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"del_file_nif", 1, del_file_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"del_dir_nif", 1, del_dir_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"get_device_cwd_nif", 1, get_device_cwd_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"set_cwd_nif", 1, set_cwd_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"get_cwd_nif", 0, get_cwd_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+
+    /* These operations are equivalent to chained calls of other operations,
+     * but have been moved down to avoid excessive rescheduling. */
+    {"ipread_s32bu_p32bu_nif", 3, ipread_s32bu_p32bu_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"read_file_nif", 1, read_file_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+
+    /* Internal ops. */
+    {"get_handle_nif", 1, get_handle_nif},
+    {"altname_nif", 1, altname_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+};
+
+ERL_NIF_INIT(prim_file, nif_funcs, load, NULL, upgrade, unload)
+
+static void owner_death_callback(ErlNifEnv* env, void* obj, ErlNifPid* pid, ErlNifMonitor* mon);
+static void gc_callback(ErlNifEnv *env, void* data);
+
+static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info)
+{
+    ErlNifResourceTypeInit callbacks;
+
+    am_ok = enif_make_atom(env, "ok");
+    am_error = enif_make_atom(env, "error");
+    am_continue = enif_make_atom(env, "continue");
+
+    am_read = enif_make_atom(env, "read");
+    am_write = enif_make_atom(env, "write");
+    am_exclusive = enif_make_atom(env, "exclusive");
+    am_append = enif_make_atom(env, "append");
+    am_sync = enif_make_atom(env, "sync");
+    am_skip_type_check = enif_make_atom(env, "skip_type_check");
+
+    am_read_write = enif_make_atom(env, "read_write");
+    am_none = enif_make_atom(env, "none");
+
+    am_normal = enif_make_atom(env, "normal");
+    am_random = enif_make_atom(env, "random");
+    am_sequential = enif_make_atom(env, "sequential");
+    am_will_need = enif_make_atom(env, "will_need");
+    am_dont_need = enif_make_atom(env, "dont_need");
+    am_no_reuse = enif_make_atom(env, "no_reuse");
+
+    am_device = enif_make_atom(env, "device");
+    am_directory = enif_make_atom(env, "directory");
+    am_regular = enif_make_atom(env, "regular");
+    am_symlink = enif_make_atom(env, "symlink");
+    am_other = enif_make_atom(env, "other");
+
+    am_file_info = enif_make_atom(env, "file_info");
+
+    am_bof = enif_make_atom(env, "bof");
+    am_cur = enif_make_atom(env, "cur");
+    am_eof = enif_make_atom(env, "eof");
+
+    callbacks.down = owner_death_callback;
+    callbacks.dtor = gc_callback;
+    callbacks.stop = NULL;
+
+    efile_resource_type = enif_open_resource_type_x(env, "efile", &callbacks,
+        ERL_NIF_RT_CREATE, NULL);
+
+    *priv_data = NULL;
+
+    return 0;
+}
+
+static void unload(ErlNifEnv *env, void* priv_data)
+{
+
+}
+
+static int upgrade(ErlNifEnv *env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
+{
+    if(*old_priv_data != NULL) {
+        return -1; /* Don't know how to do that */
+    }
+    if(*priv_data != NULL) {
+        return -1; /* Don't know how to do that */
+    }
+    if(load(env, priv_data, load_info)) {
+        return -1;
+    }
+    return 0;
+}
+
+static ERL_NIF_TERM posix_error_to_tuple(ErlNifEnv *env, posix_errno_t posix_errno) {
+    ERL_NIF_TERM error = enif_make_atom(env, erl_errno_id(posix_errno));
+    return enif_make_tuple2(env, am_error, error);
+}
+
+static int get_file_data(ErlNifEnv *env, ERL_NIF_TERM opaque, efile_data_t **d) {
+    return enif_get_resource(env, opaque, efile_resource_type, (void **)d);
+}
+
+static ERL_NIF_TERM file_handle_wrapper(file_op_impl_t operation, ErlNifEnv *env,
+        int argc, const ERL_NIF_TERM argv[]) {
+
+    efile_data_t *d;
+
+    enum efile_state_t previous_state;
+    ERL_NIF_TERM result;
+
+    if(argc < 1 || !get_file_data(env, argv[0], &d)) {
+        return enif_make_badarg(env);
+    }
+
+    previous_state = erts_atomic32_cmpxchg_acqb(&d->state,
+        EFILE_STATE_BUSY, EFILE_STATE_IDLE);
+
+    if(previous_state == EFILE_STATE_IDLE) {
+        result = operation(d, env, argc - 1, &argv[1]);
+
+        previous_state = erts_atomic32_cmpxchg_relb(&d->state,
+            EFILE_STATE_IDLE, EFILE_STATE_BUSY);
+
+        ASSERT(previous_state != EFILE_STATE_IDLE);
+
+        if(previous_state == EFILE_STATE_CLOSE_PENDING) {
+            /* This is the only point where a change from CLOSE_PENDING is
+             * possible, and we're running synchronously, so we can't race with
+             * anything else here. */
+            erts_atomic32_set_acqb(&d->state, EFILE_STATE_CLOSED);
+            efile_close(d);
+        }
+    } else {
+        /* CLOSE_PENDING should be impossible at this point since it requires
+         * a transition from BUSY; the only valid state here is CLOSED. */
+        ASSERT(previous_state == EFILE_STATE_CLOSED);
+
+        result = posix_error_to_tuple(env, EINVAL);
+    }
+
+    return result;
+}
+
+static void owner_death_callback(ErlNifEnv* env, void* obj, ErlNifPid* pid, ErlNifMonitor* mon) {
+    efile_data_t *d = (efile_data_t*)obj;
+
+    (void)env;
+    (void)pid;
+    (void)mon;
+
+    for(;;) {
+        enum efile_state_t previous_state;
+
+        previous_state = erts_atomic32_cmpxchg_acqb(&d->state,
+            EFILE_STATE_CLOSED, EFILE_STATE_IDLE);
+
+        switch(previous_state) {
+        case EFILE_STATE_IDLE:
+            efile_close(d);
+            return;
+        case EFILE_STATE_CLOSE_PENDING:
+        case EFILE_STATE_CLOSED:
+            /* We're either already closed or managed to mark ourselves for
+             * closure in the previous iteration. */
+            return;
+        case EFILE_STATE_BUSY:
+            /* Schedule ourselves to be closed once the current operation
+             * finishes, retrying the [IDLE -> CLOSED] transition in case we
+             * narrowly passed the [BUSY -> IDLE] one. */
+            erts_atomic32_cmpxchg_nob(&d->state,
+                EFILE_STATE_CLOSE_PENDING, EFILE_STATE_BUSY);
+            break;
+        }
+    }
+}
+
+static void gc_callback(ErlNifEnv *env, void* data) {
+    efile_data_t *d = (efile_data_t*)data;
+
+    enum efile_state_t previous_state;
+
+    (void)env;
+
+    previous_state = erts_atomic32_cmpxchg_acqb(&d->state,
+        EFILE_STATE_CLOSED, EFILE_STATE_IDLE);
+
+    ASSERT(previous_state != EFILE_STATE_CLOSE_PENDING &&
+        previous_state != EFILE_STATE_BUSY);
+
+    if(previous_state == EFILE_STATE_IDLE) {
+        efile_close(d);
+    }
+}
+
+static ERL_NIF_TERM efile_filetype_to_atom(enum efile_filetype_t type) {
+    switch(type) {
+        case EFILE_FILETYPE_DEVICE: return am_device;
+        case EFILE_FILETYPE_DIRECTORY: return am_directory;
+        case EFILE_FILETYPE_REGULAR: return am_regular;
+        case EFILE_FILETYPE_SYMLINK: return am_symlink;
+        case EFILE_FILETYPE_OTHER: return am_other;
+    }
+
+    return am_other;
+}
+
+static ERL_NIF_TERM efile_access_to_atom(enum efile_access_t type) {
+    if(type & EFILE_ACCESS_READ && !(type & EFILE_ACCESS_WRITE)) {
+        return am_read;
+    } else if(type & EFILE_ACCESS_WRITE && !(type & EFILE_ACCESS_READ)) {
+        return am_write;
+    } else if(type & EFILE_ACCESS_READ_WRITE) {
+        return am_read_write;
+    }
+
+    return am_none;
+}
+
+static enum efile_modes_t efile_translate_modelist(ErlNifEnv *env, ERL_NIF_TERM list) {
+    enum efile_modes_t modes;
+    ERL_NIF_TERM head, tail;
+
+    modes = 0;
+
+    while(enif_get_list_cell(env, list, &head, &tail)) {
+        if(enif_is_identical(head, am_read)) {
+            modes |= EFILE_MODE_READ;
+        } else if(enif_is_identical(head, am_write)) {
+            modes |= EFILE_MODE_WRITE;
+        } else if(enif_is_identical(head, am_exclusive)) {
+            modes |= EFILE_MODE_EXCLUSIVE;
+        } else if(enif_is_identical(head, am_append)) {
+            modes |= EFILE_MODE_APPEND;
+        } else if(enif_is_identical(head, am_sync)) {
+            modes |= EFILE_MODE_SYNC;
+        } else if(enif_is_identical(head, am_skip_type_check)) {
+            modes |= EFILE_MODE_SKIP_TYPE_CHECK;
+        } else {
+            /* Modes like 'raw', 'ram', 'delayed_writes' etc are handled
+             * further up the chain. */
+        }
+
+        list = tail;
+    }
+
+    if(modes & (EFILE_MODE_APPEND | EFILE_MODE_EXCLUSIVE)) {
+        /* 'append' and 'exclusive' are documented as "open for writing." */
+        modes |= EFILE_MODE_WRITE;
+    } else if(!(modes & EFILE_MODE_READ_WRITE)) {
+        /* Defaulting to read if !(W|R) is undocumented, but specifically
+         * tested against in file_SUITE. */
+        modes |= EFILE_MODE_READ;
+    }
+
+    return modes;
+}
+
+static ERL_NIF_TERM open_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+    efile_data_t *d;
+
+    ErlNifPid controlling_process;
+    enum efile_modes_t modes;
+    ERL_NIF_TERM result;
+    efile_path_t path;
+
+    if(argc != 2 || !enif_is_list(env, argv[1])) {
+        return enif_make_badarg(env);
+    }
+
+    modes = efile_translate_modelist(env, argv[1]);
+
+    if((posix_errno = efile_marshal_path(env, argv[0], &path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_open(&path, modes, efile_resource_type, &d))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    result = enif_make_resource(env, d);
+    enif_release_resource(d);
+
+    enif_self(env, &controlling_process);
+
+    if(enif_monitor_process(env, d, &controlling_process, &d->monitor)) {
+        return posix_error_to_tuple(env, EINVAL);
+    }
+
+    return enif_make_tuple2(env, am_ok, result);
+}
+
+static ERL_NIF_TERM close_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    enum efile_state_t previous_state;
+
+    if(argc != 0) {
+        return enif_make_badarg(env);
+    }
+
+    previous_state = erts_atomic32_cmpxchg_acqb(&d->state,
+        EFILE_STATE_CLOSED, EFILE_STATE_BUSY);
+
+    ASSERT(previous_state == EFILE_STATE_CLOSE_PENDING ||
+        previous_state == EFILE_STATE_BUSY);
+
+    if(previous_state == EFILE_STATE_BUSY) {
+        enif_demonitor_process(env, d, &d->monitor);
+
+        if(!efile_close(d)) {
+            return posix_error_to_tuple(env, d->posix_errno);
+        }
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM read_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    Sint64 bytes_read, block_size;
+    SysIOVec read_vec[1];
+    ErlNifBinary result;
+
+    if(argc != 1 || !enif_is_number(env, argv[0])) {
+        return enif_make_badarg(env);
+    }
+
+    if(!enif_get_int64(env, argv[0], &block_size) || block_size < 0) {
+        return posix_error_to_tuple(env, EINVAL);
+    }
+
+    if(!enif_alloc_binary(block_size, &result)) {
+        return posix_error_to_tuple(env, ENOMEM);
+    }
+
+    read_vec[0].iov_base = result.data;
+    read_vec[0].iov_len = result.size;
+
+    bytes_read = efile_readv(d, read_vec, 1);
+    ASSERT(bytes_read <= block_size);
+
+    if(bytes_read < 0) {
+        return posix_error_to_tuple(env, d->posix_errno);
+    } else if(bytes_read == 0) {
+        enif_release_binary(&result);
+        return am_eof;
+    }
+
+    if(bytes_read < block_size && !enif_realloc_binary(&result, bytes_read)) {
+        ERTS_INTERNAL_ERROR("Failed to shrink read result.");
+    }
+
+    return enif_make_tuple2(env, am_ok, enif_make_binary(env, &result));
+}
+
+static ERL_NIF_TERM write_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    ErlNifIOVec vec, *input = &vec;
+    Sint64 bytes_written;
+    ERL_NIF_TERM tail;
+
+    if(argc != 1 || !enif_inspect_iovec(env, 64, argv[0], &tail, &input)) {
+        return enif_make_badarg(env);
+    }
+
+    bytes_written = efile_writev(d, input->iov, input->iovcnt);
+
+    if(bytes_written < 0) {
+        return posix_error_to_tuple(env, d->posix_errno);
+    }
+
+    if(!enif_is_empty_list(env, tail)) {
+        ASSERT(bytes_written > 0);
+        return enif_make_tuple2(env, am_continue, tail);
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM pread_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    Sint64 bytes_read, block_size, offset;
+    SysIOVec read_vec[1];
+    ErlNifBinary result;
+
+    if(argc != 2 || !enif_is_number(env, argv[0])
+                 || !enif_is_number(env, argv[1])) {
+        return enif_make_badarg(env);
+    }
+
+    if(!enif_get_int64(env, argv[0], &offset) ||
+       !enif_get_int64(env, argv[1], &block_size) ||
+       (offset < 0 || block_size < 0)) {
+        return posix_error_to_tuple(env, EINVAL);
+    }
+
+    if(!enif_alloc_binary(block_size, &result)) {
+        return posix_error_to_tuple(env, ENOMEM);
+    }
+
+    read_vec[0].iov_base = result.data;
+    read_vec[0].iov_len = result.size;
+
+    bytes_read = efile_preadv(d, offset, read_vec, 1);
+
+    if(bytes_read < 0) {
+        return posix_error_to_tuple(env, d->posix_errno);
+    } else if(bytes_read == 0) {
+        enif_release_binary(&result);
+        return am_eof;
+    }
+
+    if(bytes_read < block_size && !enif_realloc_binary(&result, bytes_read)) {
+        ERTS_INTERNAL_ERROR("Failed to shrink pread result.");
+    }
+
+    return enif_make_tuple2(env, am_ok, enif_make_binary(env, &result));
+}
+
+static ERL_NIF_TERM pwrite_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    ErlNifIOVec vec, *input = &vec;
+    Sint64 bytes_written, offset;
+    ERL_NIF_TERM tail;
+
+    if(argc != 2 || !enif_is_number(env, argv[0])
+                 || !enif_inspect_iovec(env, 64, argv[1], &tail, &input)) {
+        return enif_make_badarg(env);
+    }
+
+    if(!enif_get_int64(env, argv[0], &offset) || offset < 0) {
+        return posix_error_to_tuple(env, EINVAL);
+    }
+
+    bytes_written = efile_pwritev(d, offset, input->iov, input->iovcnt);
+
+    if(bytes_written < 0) {
+        return posix_error_to_tuple(env, d->posix_errno);
+    }
+
+    if(!enif_is_empty_list(env, tail)) {
+        ASSERT(bytes_written > 0);
+        return enif_make_tuple3(env, am_continue,
+            enif_make_int64(env, bytes_written), tail);
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM seek_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    Sint64 new_position, offset;
+    enum efile_seek_t seek;
+
+    if(argc != 2 || !enif_get_int64(env, argv[1], &offset)) {
+        return enif_make_badarg(env);
+    }
+
+    if(enif_is_identical(argv[0], am_bof)) {
+        seek = EFILE_SEEK_BOF;
+    } else if(enif_is_identical(argv[0], am_cur)) {
+        seek = EFILE_SEEK_CUR;
+    } else if(enif_is_identical(argv[0], am_eof)) {
+        seek = EFILE_SEEK_EOF;
+    } else {
+        return enif_make_badarg(env);
+    }
+
+    if(!efile_seek(d, seek, offset, &new_position)) {
+        return posix_error_to_tuple(env, d->posix_errno);
+    }
+
+    return enif_make_tuple2(env, am_ok, enif_make_uint64(env, new_position));
+}
+
+static ERL_NIF_TERM sync_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    int data_only;
+
+    if(argc != 1 || !enif_get_int(env, argv[0], &data_only)) {
+        return enif_make_badarg(env);
+    }
+
+    if(!efile_sync(d, data_only)) {
+        return posix_error_to_tuple(env, d->posix_errno);
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM truncate_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    if(argc != 0) {
+        return enif_make_badarg(env);
+    }
+
+    if(!efile_truncate(d)) {
+        return posix_error_to_tuple(env, d->posix_errno);
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM allocate_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    Sint64 offset, length;
+
+    if(argc != 2 || !enif_is_number(env, argv[0])
+                 || !enif_is_number(env, argv[1])) {
+        return enif_make_badarg(env);
+    }
+
+    if(!enif_get_int64(env, argv[0], &offset) ||
+       !enif_get_int64(env, argv[1], &length) ||
+       (offset < 0 || length < 0)) {
+        return posix_error_to_tuple(env, EINVAL);
+    }
+
+    if(!efile_allocate(d, offset, length)) {
+        return posix_error_to_tuple(env, d->posix_errno);
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM advise_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    enum efile_advise_t advise;
+    Sint64 offset, length;
+
+    if(argc != 3 || !enif_is_number(env, argv[0])
+                 || !enif_is_number(env, argv[1])) {
+        return enif_make_badarg(env);
+    }
+
+    if(!enif_get_int64(env, argv[0], &offset) ||
+       !enif_get_int64(env, argv[1], &length) ||
+       (offset < 0 || length < 0)) {
+        return posix_error_to_tuple(env, EINVAL);
+    }
+
+    if(enif_is_identical(argv[2], am_normal)) {
+        advise = EFILE_ADVISE_NORMAL;
+    } else if(enif_is_identical(argv[2], am_random)) {
+        advise = EFILE_ADVISE_RANDOM;
+    } else if(enif_is_identical(argv[2], am_sequential)) {
+        advise = EFILE_ADVISE_SEQUENTIAL;
+    } else if(enif_is_identical(argv[2], am_will_need)) {
+        advise = EFILE_ADVISE_WILL_NEED;
+    } else if(enif_is_identical(argv[2], am_dont_need)) {
+        advise = EFILE_ADVISE_DONT_NEED;
+    } else if(enif_is_identical(argv[2], am_no_reuse)) {
+        advise = EFILE_ADVISE_NO_REUSE;
+    } else {
+        /* The tests check for EINVAL instead of badarg. Sigh. */
+        return posix_error_to_tuple(env, EINVAL);
+    }
+
+    if(!efile_advise(d, offset, length, advise)) {
+        return posix_error_to_tuple(env, d->posix_errno);
+    }
+
+    return am_ok;
+}
+
+/* This undocumented function reads a pointer and then reads the data block
+ * described by said pointer. It was reverse-engineered from the old
+ * implementation so while all tests pass it may not be entirely correct. Our
+ * current understanding is as follows:
+ *
+ * Pointer layout:
+ *
+ *     <<Size:1/integer-unit:32, Offset:1/integer-unit:32>>
+ *
+ * Where Offset is the -absolute- address to the data block.
+ *
+ * *) If we fail to read the pointer block in its entirety, we return eof.
+ * *) If the provided max_payload_size is larger than Size, we return eof.
+ * *) If we fail to read any data whatsoever at Offset, we return
+ *    {ok, {Size, Offset, eof}}
+ * *) Otherwise, we return {ok, {Size, Offset, Data}}. Note that the size
+ *    of Data may be smaller than Size if we encounter EOF before we could
+ *    read the entire block.
+ *
+ * On errors we'll return {error, posix()} regardless of whether they
+ * happened before or after reading the pointer block. */
+static ERL_NIF_TERM ipread_s32bu_p32bu_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    Sint64 payload_offset, payload_size;
+
+    SysIOVec read_vec[1];
+    Sint64 bytes_read;
+
+    ErlNifBinary payload;
+
+    if(argc != 2 || !enif_is_number(env, argv[0])
+                 || !enif_is_number(env, argv[1])) {
+        return enif_make_badarg(env);
+    }
+
+    {
+        Sint64 max_payload_size, pointer_offset;
+        unsigned char pointer_block[8];
+
+        if(!enif_get_int64(env, argv[0], &pointer_offset) ||
+           !enif_get_int64(env, argv[1], &max_payload_size) ||
+           (pointer_offset < 0 || max_payload_size >= 1u << 31)) {
+            return posix_error_to_tuple(env, EINVAL);
+        }
+
+        read_vec[0].iov_base = pointer_block;
+        read_vec[0].iov_len = sizeof(pointer_block);
+
+        bytes_read = efile_preadv(d, pointer_offset, read_vec, 1);
+
+        if(bytes_read < 0) {
+            return posix_error_to_tuple(env, d->posix_errno);
+        } else if(bytes_read < sizeof(pointer_block)) {
+            return am_eof;
+        }
+
+        payload_size = (Uint32)get_int32(&pointer_block[0]);
+        payload_offset = (Uint32)get_int32(&pointer_block[4]);
+
+        if(payload_size > max_payload_size) {
+            return am_eof;
+        }
+    }
+
+    if(!enif_alloc_binary(payload_size, &payload)) {
+        return posix_error_to_tuple(env, ENOMEM);
+    }
+
+    read_vec[0].iov_base = payload.data;
+    read_vec[0].iov_len = payload.size;
+
+    bytes_read = efile_preadv(d, payload_offset, read_vec, 1);
+
+    if(bytes_read < 0) {
+        return posix_error_to_tuple(env, d->posix_errno);
+    } else if(bytes_read == 0) {
+        enif_release_binary(&payload);
+
+        return enif_make_tuple2(env, am_ok,
+                enif_make_tuple3(env,
+                    enif_make_uint(env, payload_size),
+                    enif_make_uint(env, payload_offset),
+                    am_eof));
+    }
+
+    if(bytes_read < payload.size && !enif_realloc_binary(&payload, bytes_read)) {
+        ERTS_INTERNAL_ERROR("Failed to shrink ipread payload.");
+    }
+
+    return enif_make_tuple2(env, am_ok,
+        enif_make_tuple3(env,
+            enif_make_uint(env, payload_size),
+            enif_make_uint(env, payload_offset),
+            enif_make_binary(env, &payload)));
+}
+
+static ERL_NIF_TERM get_handle_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    if(argc != 0) {
+        return enif_make_badarg(env);
+    }
+
+    return efile_get_handle(env, d);
+}
+
+static ERL_NIF_TERM read_info_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+
+    efile_fileinfo_t info = {0};
+    efile_path_t path;
+    int follow_links;
+
+    if(argc != 2 || !enif_get_int(env, argv[1], &follow_links)) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_marshal_path(env, argv[0], &path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_read_info(&path, follow_links, &info))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    /* #file_info as declared in file.hrl */
+    return enif_make_tuple(env, 14,
+        am_file_info,
+        enif_make_uint64(env, info.size),
+        efile_filetype_to_atom(info.type),
+        efile_access_to_atom(info.access),
+        enif_make_int64(env, MAX(EFILE_MIN_FILETIME, info.a_time)),
+        enif_make_int64(env, MAX(EFILE_MIN_FILETIME, info.m_time)),
+        enif_make_int64(env, MAX(EFILE_MIN_FILETIME, info.c_time)),
+        enif_make_uint(env, info.mode),
+        enif_make_uint(env, info.links),
+        enif_make_uint(env, info.major_device),
+        enif_make_uint(env, info.minor_device),
+        enif_make_uint(env, info.inode),
+        enif_make_uint(env, info.uid),
+        enif_make_uint(env, info.gid)
+    );
+}
+
+static ERL_NIF_TERM set_permissions_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+
+    efile_path_t path;
+    Uint32 permissions;
+
+    if(argc != 2 || !enif_get_uint(env, argv[1], &permissions)) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_marshal_path(env, argv[0], &path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_set_permissions(&path, permissions))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM set_owner_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+
+    efile_path_t path;
+    Uint32 uid, gid;
+
+    if(argc != 3 || !enif_get_uint(env, argv[1], &uid)
+                 || !enif_get_uint(env, argv[2], &gid)) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_marshal_path(env, argv[0], &path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_set_owner(&path, uid, gid))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM set_time_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+
+    Sint64 accessed, modified, created;
+    efile_path_t path;
+
+    if(argc != 4 || !enif_get_int64(env, argv[1], &accessed)
+                 || !enif_get_int64(env, argv[2], &modified)
+                 || !enif_get_int64(env, argv[3], &created)) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_marshal_path(env, argv[0], &path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_set_time(&path, accessed, modified, created))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM read_link_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+
+    efile_path_t path;
+    ERL_NIF_TERM result;
+
+    if(argc != 1) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_marshal_path(env, argv[0], &path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_read_link(env, &path, &result))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    return enif_make_tuple2(env, am_ok, result);
+}
+
+static ERL_NIF_TERM list_dir_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+
+    efile_path_t path;
+    ERL_NIF_TERM result;
+
+    if(argc != 1) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_marshal_path(env, argv[0], &path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_list_dir(env, &path, &result))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    return enif_make_tuple2(env, am_ok, result);
+}
+
+static ERL_NIF_TERM rename_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+
+    efile_path_t existing_path, new_path;
+
+    if(argc != 2) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_marshal_path(env, argv[0], &existing_path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_marshal_path(env, argv[1], &new_path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_rename(&existing_path, &new_path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM make_hard_link_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+
+    efile_path_t existing_path, new_path;
+
+    if(argc != 2) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_marshal_path(env, argv[0], &existing_path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_marshal_path(env, argv[1], &new_path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_make_hard_link(&existing_path, &new_path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM make_soft_link_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+
+    efile_path_t existing_path, new_path;
+
+    if(argc != 2) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_marshal_path(env, argv[0], &existing_path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_marshal_path(env, argv[1], &new_path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_make_soft_link(&existing_path, &new_path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM make_dir_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+
+    efile_path_t path;
+
+    if(argc != 1) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_marshal_path(env, argv[0], &path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_make_dir(&path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM del_file_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+
+    efile_path_t path;
+
+    if(argc != 1) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_marshal_path(env, argv[0], &path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_del_file(&path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM del_dir_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+
+    efile_path_t path;
+
+    if(argc != 1) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_marshal_path(env, argv[0], &path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_del_dir(&path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    return am_ok;
+}
+
+static ERL_NIF_TERM get_device_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+
+    ERL_NIF_TERM result;
+    int device_index;
+
+    if(argc != 1 || !enif_get_int(env, argv[0], &device_index)) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_get_device_cwd(env, device_index, &result))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    return enif_make_tuple2(env, am_ok, result);
+}
+
+static ERL_NIF_TERM get_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+    ERL_NIF_TERM result;
+
+    if(argc != 0) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_get_cwd(env, &result))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    return enif_make_tuple2(env, am_ok, result);
+}
+
+static ERL_NIF_TERM set_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+
+    efile_path_t path;
+
+    if(argc != 1) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_marshal_path(env, argv[0], &path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_set_cwd(&path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    return am_ok;
+}
+
+/** @brief Reads an entire file into \c result, stopping after \c size bytes or
+ * EOF. It will read until EOF if size is 0. */
+static posix_errno_t read_file(efile_data_t *d, size_t size, ErlNifBinary *result) {
+    size_t initial_buffer_size;
+    ssize_t bytes_read;
+
+    if(size == 0) {
+        initial_buffer_size = 16 << 10;
+    } else {
+        initial_buffer_size = size;
+    }
+
+    if(!enif_alloc_binary(initial_buffer_size, result)) {
+        return ENOMEM;
+    }
+
+    bytes_read = 0;
+
+    for(;;) {
+        ssize_t block_bytes_read;
+        SysIOVec read_vec[1];
+
+        read_vec[0].iov_base = result->data + bytes_read;
+        read_vec[0].iov_len = result->size - bytes_read;
+
+        block_bytes_read = efile_readv(d, read_vec, 1);
+
+        if(block_bytes_read < 0) {
+            enif_release_binary(result);
+            return d->posix_errno;
+        }
+
+        bytes_read += block_bytes_read;
+
+        if(block_bytes_read < (result->size - bytes_read)) {
+            /* EOF */
+            break;
+        } else if(bytes_read == size) {
+            break;
+        }
+
+        if(!enif_realloc_binary(result, bytes_read * 2)) {
+            enif_release_binary(result);
+            return ENOMEM;
+        }
+    }
+
+    /* The file may have shrunk since we queried its size, so we have to do
+     * this even when the size is known. */
+    if(bytes_read < result->size && !enif_realloc_binary(result, bytes_read)) {
+        ERTS_INTERNAL_ERROR("Failed to shrink read_file result.");
+    }
+
+    return 0;
+}
+
+static ERL_NIF_TERM read_file_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+
+    efile_fileinfo_t info = {0};
+    efile_path_t path;
+    efile_data_t *d;
+
+    ErlNifBinary result;
+
+    if(argc != 1) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_marshal_path(env, argv[0], &path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_read_info(&path, 1, &info))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_open(&path, EFILE_MODE_READ, efile_resource_type, &d))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    posix_errno = read_file(d, info.size, &result);
+    enif_release_resource(d);
+
+    if(posix_errno) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    return enif_make_tuple2(env, am_ok, enif_make_binary(env, &result));
+}
+
+static ERL_NIF_TERM altname_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t posix_errno;
+
+    efile_path_t path;
+    ERL_NIF_TERM result;
+
+    if(argc != 1) {
+        return enif_make_badarg(env);
+    }
+
+    if((posix_errno = efile_marshal_path(env, argv[0], &path))) {
+        return posix_error_to_tuple(env, posix_errno);
+    } else if((posix_errno = efile_altname(env, &path, &result))) {
+        return posix_error_to_tuple(env, posix_errno);
+    }
+
+    return enif_make_tuple2(env, am_ok, result);
+}
diff --git a/erts/emulator/nifs/common/prim_file_nif.h b/erts/emulator/nifs/common/prim_file_nif.h
new file mode 100644
index 0000000000..cc9bc8f5c3
--- /dev/null
+++ b/erts/emulator/nifs/common/prim_file_nif.h
@@ -0,0 +1,240 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson 2017. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+typedef int posix_errno_t;
+
+enum efile_modes_t {
+    EFILE_MODE_READ = (1 << 0),
+    EFILE_MODE_WRITE = (1 << 1), /* Implies truncating file when used alone. */
+    EFILE_MODE_APPEND = (1 << 2),
+    EFILE_MODE_EXCLUSIVE = (1 << 3),
+    EFILE_MODE_SYNC = (1 << 4),
+
+    EFILE_MODE_SKIP_TYPE_CHECK = (1 << 5), /* Special for device files on Unix. */
+    EFILE_MODE_NO_TRUNCATE = (1 << 6), /* Special for reopening on VxWorks. */
+
+    EFILE_MODE_READ_WRITE = EFILE_MODE_READ | EFILE_MODE_WRITE
+};
+
+enum efile_access_t {
+    EFILE_ACCESS_NONE = 0,
+    EFILE_ACCESS_READ = 1,
+    EFILE_ACCESS_WRITE = 2,
+    EFILE_ACCESS_READ_WRITE = EFILE_ACCESS_READ | EFILE_ACCESS_WRITE
+};
+
+enum efile_seek_t {
+    EFILE_SEEK_BOF,
+    EFILE_SEEK_CUR,
+    EFILE_SEEK_EOF
+};
+
+enum efile_filetype_t {
+    EFILE_FILETYPE_DEVICE,
+    EFILE_FILETYPE_DIRECTORY,
+    EFILE_FILETYPE_REGULAR,
+    EFILE_FILETYPE_SYMLINK,
+    EFILE_FILETYPE_OTHER
+};
+
+enum efile_advise_t {
+    EFILE_ADVISE_NORMAL,
+    EFILE_ADVISE_RANDOM,
+    EFILE_ADVISE_SEQUENTIAL,
+    EFILE_ADVISE_WILL_NEED,
+    EFILE_ADVISE_DONT_NEED,
+    EFILE_ADVISE_NO_REUSE
+};
+
+enum efile_state_t {
+    EFILE_STATE_IDLE = 0,
+    EFILE_STATE_BUSY = 1,
+    EFILE_STATE_CLOSE_PENDING = 2,
+    EFILE_STATE_CLOSED = 3
+};
+
+typedef struct {
+    Sint64 size;            /* Size of file */
+    Uint32 type;            /* Type of file -- one of EFILE_FILETYPE_*. */
+    Uint32 access;          /* Access to file -- one of EFILE_ACCESS_*. */
+    Uint32 mode;            /* Access permissions -- bit field. */
+    Uint32 links;           /* Number of links to file. */
+    Uint32 major_device;    /* Major device or file system. */
+    Uint32 minor_device;    /* Minor device (for devices). */
+    Uint32 inode;           /* Inode number. */
+    Uint32 uid;             /* User id of owner. */
+    Uint32 gid;             /* Group id of owner. */
+    Sint64 a_time;          /* Last time the file was accessed. */
+    Sint64 m_time;          /* Last time the file was modified. */
+    Sint64 c_time;          /* Windows: creation time, Unix: last inode
+                             * change. */
+} efile_fileinfo_t;
+
+/* The smallest value that can be converted freely between universal, local,
+ * and POSIX time, as required by read_file_info/2. Corresponds to
+ * {{1902,1,1},{0,0,0}} */
+#define EFILE_MIN_FILETIME -2145916800
+
+/* Initializes an efile_data_t; must be used in efile_open on success. */
+#define EFILE_INIT_RESOURCE(__d, __modes) do { \
+        erts_atomic32_init_acqb(&(__d)->state, EFILE_STATE_IDLE); \
+        (__d)->posix_errno = 0; \
+        (__d)->modes = __modes; \
+    } while(0)
+
+typedef struct {
+    erts_atomic32_t state;
+
+    posix_errno_t posix_errno;
+    enum efile_modes_t modes;
+
+    ErlNifMonitor monitor;
+} efile_data_t;
+
+typedef ErlNifBinary efile_path_t;
+
+/* @brief Translates the given "raw name" into the format expected by the APIs
+ * used by the underlying implementation. The result is transient and does not
+ * need to be released.
+ *
+ * This may change the structure of the path and its results should never be
+ * passed on to the user. Refer to the OS-specific implementation for details.
+ *
+ * @param path The term to translate; it must have been encoded with
+ * prim_file:internal_native2name for compatibility reasons. */
+posix_errno_t efile_marshal_path(ErlNifEnv *env, ERL_NIF_TERM path, efile_path_t *result);
+
+/* @brief Returns the underlying handle as an implementation-defined term.
+ *
+ * This is an internal function intended to support tests and tricky
+ * operations like sendfile(2). */
+ERL_NIF_TERM efile_get_handle(ErlNifEnv *env, efile_data_t *d);
+
+/* @brief Read until EOF or the given iovec has been filled.
+ *
+ * @return -1 on failure, or the number of bytes read on success. The return
+ * value will be 0 if no bytes could be read before EOF or the end of the
+ * iovec. */
+Sint64 efile_readv(efile_data_t *d, SysIOVec *iov, int iovlen);
+
+/* @brief Write the entirety of the given iovec.
+ *
+ * @return -1 on failure, or the number of bytes written on success. "Partial"
+ * failures will be reported with -1 and not the number of bytes we managed to
+ * write to disk before the failure. */
+Sint64 efile_writev(efile_data_t *d, SysIOVec *iov, int iovlen);
+
+/* @brief As \c efile_readv, but starting from a file offset. */
+Sint64 efile_preadv(efile_data_t *d, Sint64 offset, SysIOVec *iov, int iovlen);
+
+/* @brief As \c efile_writev, but starting from a file offset. */
+Sint64 efile_pwritev(efile_data_t *d, Sint64 offset, SysIOVec *iov, int iovlen);
+
+int efile_seek(efile_data_t *d, enum efile_seek_t seek, Sint64 offset, Sint64 *new_position);
+
+int efile_sync(efile_data_t *d, int data_only);
+
+int efile_advise(efile_data_t *d, Sint64 offset, Sint64 length, enum efile_advise_t advise);
+int efile_allocate(efile_data_t *d, Sint64 offset, Sint64 length);
+int efile_truncate(efile_data_t *d);
+
+posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes,
+        ErlNifResourceType *nif_type, efile_data_t **d);
+
+/** @brief Closes a file. The file must have entered the CLOSED state prior to
+ * calling this to prevent double close. */
+int efile_close(efile_data_t *d);
+
+/* **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** */
+
+posix_errno_t efile_read_info(const efile_path_t *path, int follow_link, efile_fileinfo_t *result);
+
+/** @brief Sets the file times to the given values. Refer to efile_fileinfo_t
+ * for a description of each. */
+posix_errno_t efile_set_time(const efile_path_t *path, Sint64 a_time, Sint64 m_time, Sint64 c_time);
+
+/** @brief On Unix, this sets the file permissions according to the docs for
+ * file:write_file_info/2. On Windows it uses the "owner write permission" flag
+ * to toggle whether the file is read-only or not. */
+posix_errno_t efile_set_permissions(const efile_path_t *path, Uint32 permissions);
+
+/** @brief On Unix, this will set the owner/group to the given values. It will
+ * do nothing on other platforms. */
+posix_errno_t efile_set_owner(const efile_path_t *path, Uint32 owner, Uint32 group);
+
+/** @brief Resolves the final path of the given link. */
+posix_errno_t efile_read_link(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result);
+
+/** @brief Lists the contents of the given directory.
+ * @param result [out] A list of all the directory/file names contained in the
+ * given directory. */
+posix_errno_t efile_list_dir(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result);
+
+/** @brief Changes the name of an existing file or directory, from old_path
+ * to new_path.
+ *
+ * If old_path and new_path refer to the same file or directory, it does
+ * nothing and returns success. Otherwise if new_path already exists, it will
+ * be deleted and replaced by src subject to the following conditions:
+ *
+ *     If old_path is a directory, new_path may be an empty directory.
+ *     If old_path is a file, new_path may be a file.
+ *
+ * Neither of these are guaranteed to be atomic. In any other situation where
+ * new_path already exists, the rename will fail.
+ *
+ * Some possible error codes:
+ *
+ * - EACCES:     Either paths or one of their parent directories can't be read
+ *               and/or written.
+ * - EEXIST:     new_path is a non-empty directory.
+ * - EINVAL:     old_path is a root directory or new_path is a subdirectory
+ *               of new_path.
+ * - EISDIR:     new_path is a directory, but old_path is not.
+ * - ENOTDIR:    old_path is a directory, but new_path is not.
+ * - ENOENT:     old_path doesn't exist, or either path is "".
+ * - EXDEV:      The paths are on different filesystems.
+ *
+ * The implementation of rename may allow cross-filesystem renames,
+ * but the caller should be prepared to emulate it with copy and
+ * delete if errno is EXDEV. */
+posix_errno_t efile_rename(const efile_path_t *old_path, const efile_path_t *new_path);
+
+posix_errno_t efile_make_hard_link(const efile_path_t *existing_path, const efile_path_t *new_path);
+posix_errno_t efile_make_soft_link(const efile_path_t *existing_path, const efile_path_t *new_path);
+posix_errno_t efile_make_dir(const efile_path_t *path);
+
+posix_errno_t efile_del_file(const efile_path_t *path);
+posix_errno_t efile_del_dir(const efile_path_t *path);
+
+posix_errno_t efile_get_cwd(ErlNifEnv *env, ERL_NIF_TERM *result);
+posix_errno_t efile_set_cwd(const efile_path_t *path);
+
+/** @brief A Windows-specific function for returning the working directory of a
+ * given device.
+ *
+ * @param device_index The drive index; 1 for A, 2 for B, etc.
+ * @param result [out] The working directory of the given device
+ */
+posix_errno_t efile_get_device_cwd(ErlNifEnv *env, int device_index, ERL_NIF_TERM *result);
+
+/** @brief A Windows-specific function for returning the 8.3-name of a given
+ * file or directory. */
+posix_errno_t efile_altname(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result);
diff --git a/erts/emulator/nifs/unix/unix_prim_file.c b/erts/emulator/nifs/unix/unix_prim_file.c
new file mode 100644
index 0000000000..57c8ef62e1
--- /dev/null
+++ b/erts/emulator/nifs/unix/unix_prim_file.c
@@ -0,0 +1,957 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson 2017. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "erl_nif.h"
+#include "config.h"
+#include "sys.h"
+
+#ifdef VALGRIND
+#  include <valgrind/memcheck.h>
+#endif
+
+#include "prim_file_nif.h"
+
+#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__)
+#define __DARWIN__ 1
+#endif
+
+#if defined(__DARWIN__) || defined(HAVE_LINUX_FALLOC_H) || defined(HAVE_POSIX_FALLOCATE)
+#include <fcntl.h>
+#endif
+
+#ifdef HAVE_LINUX_FALLOC_H
+#include <linux/falloc.h>
+#endif
+
+#include <utime.h>
+
+/* Macros for testing file types. */
+#ifdef NO_UMASK
+#define FILE_MODE 0644
+#define DIR_MODE  0755
+#else
+#define FILE_MODE 0666
+#define DIR_MODE  0777
+#endif
+
+/* Old platforms might not have IOV_MAX defined. */
+#if !defined(IOV_MAX) && defined(UIO_MAXIOV)
+#define IOV_MAX UIO_MAXIOV
+#elif !defined(IOV_MAX)
+#define IOV_MAX 16
+#endif
+
+typedef struct {
+    efile_data_t common;
+    int fd;
+} efile_unix_t;
+
+static int has_invalid_null_termination(const ErlNifBinary *path) {
+    const char *null_pos, *end_pos;
+
+    null_pos = memchr(path->data, '\0', path->size);
+    end_pos = (const char*)&path->data[path->size] - 1;
+
+    if(null_pos == NULL) {
+        return 1;
+    }
+
+    /* prim_file:internal_name2native sometimes feeds us data that is "doubly"
+     * NUL-terminated, so we'll accept any number of trailing NULs so long as
+     * they aren't interrupted by anything else. */
+    while(null_pos < end_pos && (*null_pos) == '\0') {
+        null_pos++;
+    }
+
+    return null_pos != end_pos;
+}
+
+posix_errno_t efile_marshal_path(ErlNifEnv *env, ERL_NIF_TERM path, efile_path_t *result) {
+    if(!enif_inspect_binary(env, path, result)) {
+        return EINVAL;
+    }
+
+    if(has_invalid_null_termination(result)) {
+        return EINVAL;
+    }
+
+    return 0;
+}
+
+ERL_NIF_TERM efile_get_handle(ErlNifEnv *env, efile_data_t *d) {
+    efile_unix_t *u = (efile_unix_t*)d;
+
+    ERL_NIF_TERM result;
+    unsigned char *bits;
+
+    bits = enif_make_new_binary(env, sizeof(u->fd), &result);
+    memcpy(bits, &u->fd, sizeof(u->fd));
+
+    return result;
+}
+
+static int open_file_type_check(const efile_path_t *path, int fd) {
+    struct stat file_info;
+    int error;
+
+#ifndef HAVE_FSTAT
+    error = stat((const char*)path->data, &file_info);
+    (void)fd;
+#else
+    error = fstat(fd, &file_info);
+    (void)path;
+#endif
+
+    if(error < 0) {
+        /* If we failed to stat assume success and let the next call handle the
+         * error. The old driver checked whether the file was to be used
+         * immediately in a read within the call, but the new implementation
+         * never does that. */
+         return 1;
+    } else {
+        /* The old driver tolerated opening /dev/null despite the "no devices"
+         * limitation. It provided no explanation for this but we still need
+         * to match the behavior. We're checking through stat(2) instead of
+         * comparing the name to account for links. */
+        struct stat null_device_info;
+        int is_dev_null;
+
+        is_dev_null = (stat("/dev/null", &null_device_info) == 0);
+        is_dev_null &= (file_info.st_ino == null_device_info.st_ino);
+        is_dev_null &= (file_info.st_dev == null_device_info.st_dev);
+
+        if(is_dev_null) {
+            return 1;
+        }
+    }
+
+    if(!S_ISREG(file_info.st_mode)) {
+        return 0;
+    }
+
+    return 1;
+}
+
+posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes,
+        ErlNifResourceType *nif_type, efile_data_t **d) {
+
+    int flags, fd;
+
+    flags = 0;
+
+    if(modes & EFILE_MODE_READ && !(modes & EFILE_MODE_WRITE)) {
+        flags |= O_RDONLY;
+    } else if(modes & EFILE_MODE_WRITE && !(modes & EFILE_MODE_READ)) {
+        if(!(modes & EFILE_MODE_NO_TRUNCATE)) {
+            flags |= O_TRUNC;
+        }
+
+        flags |= O_WRONLY | O_CREAT;
+    } else if(modes & EFILE_MODE_READ_WRITE) {
+        flags |= O_RDWR | O_CREAT;
+    } else {
+        return EINVAL;
+    }
+
+    if(modes & EFILE_MODE_APPEND) {
+        flags &= ~O_TRUNC;
+        flags |= O_APPEND;
+    }
+
+    if(modes & EFILE_MODE_EXCLUSIVE) {
+        flags |= O_EXCL;
+    }
+
+    if(modes & EFILE_MODE_SYNC) {
+#ifndef O_SYNC
+        return ENOTSUP;
+#else
+        flags |= O_SYNC;
+#endif
+    }
+
+    do {
+        fd = open((const char*)path->data, flags, FILE_MODE);
+    } while(fd == -1 && errno == EINTR);
+
+    if(fd != -1) {
+        efile_unix_t *u;
+
+        if(!(modes & EFILE_MODE_SKIP_TYPE_CHECK) && !open_file_type_check(path, fd)) {
+            close(fd);
+
+            /* This is blatantly incorrect, but we're documented as returning
+             * this for everything that isn't a file. */
+            return EISDIR;
+        }
+
+        u = (efile_unix_t*)enif_alloc_resource(nif_type, sizeof(efile_unix_t));
+        u->fd = fd;
+
+        EFILE_INIT_RESOURCE(&u->common, modes);
+        (*d) = &u->common;
+
+        return 0;
+    }
+
+    (*d) = NULL;
+    return errno;
+}
+
+int efile_close(efile_data_t *d) {
+    efile_unix_t *u = (efile_unix_t*)d;
+    int fd;
+
+    ASSERT(erts_atomic32_read_nob(&d->state) == EFILE_STATE_CLOSED);
+    ASSERT(u->fd != -1);
+
+    fd = u->fd;
+    u->fd = -1;
+
+    /* close(2) either always closes (*BSD, Linux) or leaves the fd in an
+     * undefined state (POSIX 2008, Solaris), so we must not retry on EINTR. */
+
+    if(close(fd) < 0) {
+        u->common.posix_errno = errno;
+        return 0;
+    }
+
+    return 1;
+}
+
+static void shift_iov(SysIOVec **iov, int *iovlen, ssize_t shift) {
+    SysIOVec *head_vec = (*iov);
+
+    ASSERT(shift >= 0);
+
+    while(shift > 0) {
+        ASSERT(head_vec < &(*iov)[*iovlen]);
+
+        if(shift < head_vec->iov_len) {
+            head_vec->iov_base = (char*)head_vec->iov_base + shift;
+            head_vec->iov_len -= shift;
+            break;
+        } else {
+            shift -= head_vec->iov_len;
+            head_vec++;
+        }
+    }
+
+    (*iovlen) -= head_vec - (*iov);
+    (*iov) = head_vec;
+}
+
+Sint64 efile_readv(efile_data_t *d, SysIOVec *iov, int iovlen) {
+    efile_unix_t *u = (efile_unix_t*)d;
+
+    Sint64 bytes_read;
+    ssize_t result;
+
+    bytes_read = 0;
+
+    do {
+        int use_fallback = 0;
+
+        if(iovlen < 1) {
+            result = 0;
+            break;
+        }
+
+        /* writev(2) implies readv(2) */
+#ifdef HAVE_WRITEV
+        result = readv(u->fd, iov, MIN(IOV_MAX, iovlen));
+
+        /* Fall back to using read(2) if readv(2) reports that the combined
+         * size of iov is greater than SSIZE_T_MAX. */
+        use_fallback = (result < 0 && errno == EINVAL);
+#else
+        use_fallback = 1;
+#endif
+
+        if(use_fallback) {
+            result = read(u->fd, iov->iov_base, iov->iov_len);
+        }
+
+        if(result > 0) {
+            shift_iov(&iov, &iovlen, result);
+            bytes_read += result;
+        }
+    } while(result > 0 || (result < 0 && errno == EINTR));
+
+    u->common.posix_errno = errno;
+
+    if(result == 0 && bytes_read > 0) {
+        return bytes_read;
+    }
+
+    return result;
+}
+
+Sint64 efile_writev(efile_data_t *d, SysIOVec *iov, int iovlen) {
+    efile_unix_t *u = (efile_unix_t*)d;
+
+    Sint64 bytes_written;
+    ssize_t result;
+
+    bytes_written = 0;
+
+    do {
+        int use_fallback = 0;
+
+        if(iovlen < 1) {
+            result = 0;
+            break;
+        }
+
+#ifdef HAVE_WRITEV
+        result = writev(u->fd, iov, MIN(IOV_MAX, iovlen));
+
+        /* Fall back to using write(2) if writev(2) reports that the combined
+         * size of iov is greater than SSIZE_T_MAX. */
+        use_fallback = (result < 0 && errno == EINVAL);
+#else
+        use_fallback = 1;
+#endif
+
+        if(use_fallback) {
+            result = write(u->fd, iov->iov_base, iov->iov_len);
+        }
+
+        if(result > 0) {
+            shift_iov(&iov, &iovlen, result);
+            bytes_written += result;
+        }
+    } while(result > 0 || (result < 0 && errno == EINTR));
+
+    u->common.posix_errno = errno;
+
+    if(result == 0 && bytes_written > 0) {
+        return bytes_written;
+    }
+
+    return result;
+}
+
+Sint64 efile_preadv(efile_data_t *d, Sint64 offset, SysIOVec *iov, int iovlen) {
+    efile_unix_t *u = (efile_unix_t*)d;
+
+    Uint64 bytes_read;
+    Sint64 result;
+
+#if !defined(HAVE_PREADV) && !defined(HAVE_PREAD)
+    /* This function is documented as leaving the file position undefined, but
+     * the old driver always reset it so there's probably code in the wild that
+     * relies on this behavior. */
+    off_t original_position = lseek(u->fd, 0, SEEK_CUR);
+
+    if(original_position < 0 || lseek(u->fd, offset, SEEK_SET) < 0) {
+        u->common.posix_errno = errno;
+        return -1;
+    }
+#endif
+
+    bytes_read = 0;
+
+    do {
+        if(iovlen < 1) {
+            result = 0;
+            break;
+        }
+
+#if defined(HAVE_PREADV)
+        result = preadv(u->fd, iov, MIN(IOV_MAX, iovlen), offset);
+#elif defined(HAVE_PREAD)
+        result = pread(u->fd, iov->iov_base, iov->iov_len, offset);
+#else
+        result = read(u->fd, iov->iov_base, iov->iov_len);
+#endif
+
+        if(result > 0) {
+            shift_iov(&iov, &iovlen, result);
+            bytes_read += result;
+            offset += result;
+        }
+    } while(result > 0 || (result < 0 && errno == EINTR));
+
+    u->common.posix_errno = errno;
+
+#if !defined(HAVE_PREADV) && !defined(HAVE_PREAD)
+    if(result >= 0) {
+        if(lseek(u->fd, original_position, SEEK_SET) < 0) {
+            u->common.posix_errno = errno;
+            return -1;
+        }
+    }
+#endif
+
+    if(result == 0 && bytes_read > 0) {
+        return bytes_read;
+    }
+
+    return result;
+}
+
+Sint64 efile_pwritev(efile_data_t *d, Sint64 offset, SysIOVec *iov, int iovlen) {
+    efile_unix_t *u = (efile_unix_t*)d;
+
+    Sint64 bytes_written;
+    ssize_t result;
+
+#if !defined(HAVE_PWRITEV) && !defined(HAVE_PWRITE)
+    off_t original_position = lseek(u->fd, 0, SEEK_CUR);
+
+    if(original_position < 0 || lseek(u->fd, offset, SEEK_SET) < 0) {
+        u->common.posix_errno = errno;
+        return -1;
+    }
+#endif
+
+    bytes_written = 0;
+
+    do {
+        if(iovlen < 1) {
+            result = 0;
+            break;
+        }
+
+#if defined(HAVE_PWRITEV)
+        result = pwritev(u->fd, iov, MIN(IOV_MAX, iovlen), offset);
+#elif defined(HAVE_PWRITE)
+        result = pwrite(u->fd, iov->iov_base, iov->iov_len, offset);
+#else
+        result = write(u->fd, iov->iov_base, iov->iov_len);
+#endif
+
+        if(result > 0) {
+            shift_iov(&iov, &iovlen, result);
+            bytes_written += result;
+            offset += result;
+        }
+    } while(result > 0 || (result < 0 && errno == EINTR));
+
+    u->common.posix_errno = errno;
+
+#if !defined(HAVE_PWRITEV) && !defined(HAVE_PWRITE)
+    if(result >= 0) {
+        if(lseek(u->fd, original_position, SEEK_SET) < 0) {
+            u->common.posix_errno = errno;
+            return -1;
+        }
+    }
+#endif
+
+    if(result == 0 && bytes_written > 0) {
+        return bytes_written;
+    }
+
+    return result;
+}
+
+int efile_seek(efile_data_t *d, enum efile_seek_t seek, Sint64 offset, Sint64 *new_position) {
+    efile_unix_t *u = (efile_unix_t*)d;
+    off_t result;
+    int whence;
+
+    switch(seek) {
+        case EFILE_SEEK_BOF: whence = SEEK_SET; break;
+        case EFILE_SEEK_CUR: whence = SEEK_CUR; break;
+        case EFILE_SEEK_EOF: whence = SEEK_END; break;
+        default: ERTS_INTERNAL_ERROR("Invalid seek parameter");
+    }
+
+    result = lseek(u->fd, offset, whence);
+
+    /*
+     * The man page for lseek (on SunOs 5) says:
+     *
+     * "if fildes is a remote file descriptor and offset is negative, lseek()
+     * returns the file pointer even if it is negative."
+     */
+    if(result < 0 && errno == 0) {
+        errno = EINVAL;
+    }
+
+    if(result < 0) {
+        u->common.posix_errno = errno;
+        return 0;
+    }
+
+    (*new_position) = result;
+
+    return 1;
+}
+
+int efile_sync(efile_data_t *d, int data_only) {
+    efile_unix_t *u = (efile_unix_t*)d;
+
+#if defined(HAVE_FDATASYNC) && !defined(__DARWIN__)
+    if(data_only) {
+        if(fdatasync(u->fd) < 0) {
+            u->common.posix_errno = errno;
+            return 0;
+        }
+
+        return 1;
+    }
+#endif
+
+#if defined(__DARWIN__) && defined(F_FULLFSYNC)
+    if(fcntl(u->fd, F_FULLFSYNC) < 0) {
+#else
+    if(fsync(u->fd) < 0) {
+#endif
+        u->common.posix_errno = errno;
+        return 0;
+    }
+
+    return 1;
+}
+
+int efile_advise(efile_data_t *d, Sint64 offset, Sint64 length, enum efile_advise_t advise) {
+    efile_unix_t *u = (efile_unix_t*)d;
+#ifdef HAVE_POSIX_FADVISE
+    int p_advise;
+
+    switch(advise) {
+        case EFILE_ADVISE_NORMAL: p_advise = POSIX_FADV_NORMAL; break;
+        case EFILE_ADVISE_RANDOM: p_advise = POSIX_FADV_RANDOM; break;
+        case EFILE_ADVISE_SEQUENTIAL: p_advise = POSIX_FADV_SEQUENTIAL; break;
+        case EFILE_ADVISE_WILL_NEED: p_advise = POSIX_FADV_WILLNEED; break;
+        case EFILE_ADVISE_DONT_NEED: p_advise = POSIX_FADV_DONTNEED; break;
+        case EFILE_ADVISE_NO_REUSE: p_advise = POSIX_FADV_NOREUSE; break;
+        default:
+            u->common.posix_errno = EINVAL;
+            return 0;
+    }
+
+    if(posix_fadvise(u->fd, offset, length, p_advise) < 0) {
+        u->common.posix_errno = errno;
+        return 0;
+    }
+
+    return 1;
+#else
+    /* We'll pretend to support this syscall as it's only a recommendation even
+     * on systems that do support it. */
+    return 1;
+#endif
+}
+
+int efile_allocate(efile_data_t *d, Sint64 offset, Sint64 length) {
+    efile_unix_t *u = (efile_unix_t*)d;
+    int ret = -1;
+
+    /* We prefer OS-specific methods, but fall back to posix_fallocate on
+     * failure. It's unclear whether this has any practical benefit on
+     * modern systems, but the old driver did it. */
+
+#if defined(HAVE_FALLOCATE)
+    /* Linux-specific */
+    do {
+        ret = fallocate(u->fd, FALLOC_FL_KEEP_SIZE, offset, length);
+    } while(ret < 0 && errno == EINTR);
+#elif defined(F_PREALLOCATE)
+    /* Mac-specific */
+    fstore_t fs = {};
+
+    fs.fst_flags = F_ALLOCATECONTIG;
+    fs.fst_posmode = F_VOLPOSMODE;
+    fs.fst_offset = offset;
+    fs.fst_length = length;
+
+    ret = fcntl(u->fd, F_PREALLOCATE, &fs);
+    if(ret < 0) {
+        fs.fst_flags = F_ALLOCATEALL;
+        ret = fcntl(u->fd, F_PREALLOCATE, &fs);
+    }
+#elif !defined(HAVE_POSIX_FALLOCATE)
+    u->common.posix_errno = ENOTSUP;
+    return 0;
+#endif
+
+#ifdef HAVE_POSIX_FALLOCATE
+    if(ret < 0) {
+        do {
+            ret = posix_fallocate(u->fd, offset, length);
+
+            /* On Linux and Solaris for example, posix_fallocate() returns a
+             * positive error number on error and it does not set errno. On
+             * FreeBSD however (9.0 at least), it returns -1 on error and it
+             * sets errno. */
+            if (ret > 0) {
+                errno = ret;
+                ret = -1;
+            }
+        } while(ret < 0 && errno == EINTR);
+    }
+#endif
+
+    if(ret < 0) {
+        u->common.posix_errno = errno;
+        return 0;
+    }
+
+    return 1;
+}
+
+int efile_truncate(efile_data_t *d) {
+    efile_unix_t *u = (efile_unix_t*)d;
+    off_t offset;
+
+    offset = lseek(u->fd, 0, SEEK_CUR);
+
+    if(offset < 0) {
+        u->common.posix_errno = errno;
+        return 0;
+    }
+
+    if(ftruncate(u->fd, offset) < 0) {
+        u->common.posix_errno = errno;
+        return 0;
+    }
+
+    return 1;
+}
+
+posix_errno_t efile_read_info(const efile_path_t *path, int follow_links, efile_fileinfo_t *result) {
+    struct stat data;
+
+    if(follow_links) {
+        if(stat((const char*)path->data, &data) < 0) {
+            return errno;
+        }
+    } else {
+        if(lstat((const char*)path->data, &data) < 0) {
+            return errno;
+        }
+    }
+
+    if(S_ISCHR(data.st_mode) || S_ISBLK(data.st_mode)) {
+        result->type = EFILE_FILETYPE_DEVICE;
+    } else if(S_ISDIR(data.st_mode)) {
+        result->type = EFILE_FILETYPE_DIRECTORY;
+    } else if(S_ISREG(data.st_mode)) {
+        result->type = EFILE_FILETYPE_REGULAR;
+    } else if(S_ISLNK(data.st_mode)) {
+        result->type = EFILE_FILETYPE_SYMLINK;
+    } else {
+        result->type = EFILE_FILETYPE_OTHER;
+    }
+
+    result->a_time = (Sint64)data.st_atime;
+    result->m_time = (Sint64)data.st_mtime;
+    result->c_time = (Sint64)data.st_ctime;
+    result->size = data.st_size;
+
+    result->major_device = data.st_dev;
+    result->minor_device = data.st_rdev;
+    result->links = data.st_nlink;
+    result->inode = data.st_ino;
+    result->mode = data.st_mode;
+    result->uid = data.st_uid;
+    result->gid = data.st_gid;
+
+#ifndef NO_ACCESS
+    result->access = EFILE_ACCESS_NONE;
+
+    if(access((const char*)path->data, R_OK) == 0) {
+        result->access |= EFILE_ACCESS_READ;
+    }
+    if(access((const char*)path->data, W_OK) == 0) {
+        result->access |= EFILE_ACCESS_WRITE;
+    }
+#else
+    /* Just look at read/write access for owner. */
+    result->access = ((data.st_mode >> 6) & 07) >> 1;
+#endif
+
+    return 0;
+}
+
+posix_errno_t efile_set_permissions(const efile_path_t *path, Uint32 permissions) {
+    const mode_t MUTABLE_MODES = (S_ISUID | S_ISGID | S_IRWXU | S_IRWXG | S_IRWXO);
+    mode_t new_modes = permissions & MUTABLE_MODES;
+
+    if(chmod((const char*)path->data, new_modes) < 0) {
+        new_modes &= ~(S_ISUID | S_ISGID);
+
+        if (chmod((const char*)path->data, new_modes) < 0) {
+            return errno;
+        }
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_set_owner(const efile_path_t *path, Uint32 owner, Uint32 group) {
+    if(chown((const char*)path->data, owner, group) < 0) {
+        return errno;
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_set_time(const efile_path_t *path, Sint64 a_time, Sint64 m_time, Sint64 c_time) {
+    struct utimbuf tval;
+
+    tval.actime = (time_t)a_time;
+    tval.modtime = (time_t)m_time;
+
+    (void)c_time;
+
+    if(utime((const char*)path->data, &tval) < 0) {
+        return errno;
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_read_link(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result) {
+    ErlNifBinary result_bin;
+
+    if(!enif_alloc_binary(256, &result_bin)) {
+        return ENOMEM;
+    }
+
+    for(;;) {
+        ssize_t bytes_copied;
+
+        bytes_copied = readlink((const char*)path->data, (char*)result_bin.data,
+            result_bin.size);
+
+        if(bytes_copied <= 0) {
+            posix_errno_t saved_errno = errno;
+            enif_release_binary(&result_bin);
+            return saved_errno;
+        } else if(bytes_copied < result_bin.size) {
+            if(!enif_realloc_binary(&result_bin, bytes_copied)) {
+                enif_release_binary(&result_bin);
+                return ENOMEM;
+            }
+
+            (*result) = enif_make_binary(env, &result_bin);
+
+            return 0;
+        }
+
+        /* The result didn't fit into the buffer, so we'll try again with a
+         * larger one. */
+
+        if(!enif_realloc_binary(&result_bin, result_bin.size * 2)) {
+            enif_release_binary(&result_bin);
+            return ENOMEM;
+        }
+    }
+}
+
+static int is_ignored_name(int name_length, const char *name) {
+    if(name_length == 1 && name[0] == '.') {
+        return 1;
+    } else if(name_length == 2 && memcmp(name, "..", 2) == 0) {
+        return 1;
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_list_dir(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result) {
+    ERL_NIF_TERM list_head;
+
+    struct dirent *dir_entry;
+    DIR *dir_stream;
+
+    dir_stream = opendir((const char*)path->data);
+    if(dir_stream == NULL) {
+        posix_errno_t saved_errno = errno;
+        *result = enif_make_list(env, 0);
+        return saved_errno;
+    }
+
+    list_head = enif_make_list(env, 0);
+    dir_entry = readdir(dir_stream);
+
+    while(dir_entry != NULL) {
+        int name_length = strlen(dir_entry->d_name);
+
+        if(!is_ignored_name(name_length, dir_entry->d_name)) {
+            unsigned char *name_bytes;
+            ERL_NIF_TERM name_term;
+
+            name_bytes = enif_make_new_binary(env, name_length, &name_term);
+            sys_memcpy(name_bytes, dir_entry->d_name, name_length);
+
+            list_head = enif_make_list_cell(env, name_term, list_head);
+        }
+
+        dir_entry = readdir(dir_stream);
+    }
+
+    (*result) = list_head;
+    closedir(dir_stream);
+
+    return 0;
+}
+
+posix_errno_t efile_rename(const efile_path_t *old_path, const efile_path_t *new_path) {
+    if(rename((const char*)old_path->data, (const char*)new_path->data) < 0) {
+        if(errno == ENOTEMPTY) {
+            return EEXIST;
+        }
+
+        if(strcmp((const char*)old_path->data, "/") == 0) {
+            /* Alpha reports renaming / as EBUSY and Linux reports it as EACCES
+             * instead of EINVAL.*/
+             return EINVAL;
+        }
+
+        return errno;
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_make_hard_link(const efile_path_t *existing_path, const efile_path_t *new_path) {
+    if(link((const char*)existing_path->data, (const char*)new_path->data) < 0) {
+        return errno;
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_make_soft_link(const efile_path_t *existing_path, const efile_path_t *new_path) {
+    if(symlink((const char*)existing_path->data, (const char*)new_path->data) < 0) {
+        return errno;
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_make_dir(const efile_path_t *path) {
+#ifdef NO_MKDIR_MODE
+    if(mkdir((const char*)path->data) < 0) {
+#else
+    if(mkdir((const char*)path->data, DIR_MODE) < 0) {
+#endif
+        return errno;
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_del_file(const efile_path_t *path) {
+    if(unlink((const char*)path->data) < 0) {
+        /* Linux sets the wrong error code. */
+        if(errno == EISDIR) {
+            return EPERM;
+        }
+
+        return errno;
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_del_dir(const efile_path_t *path) {
+    if(rmdir((const char*)path->data) < 0) {
+        posix_errno_t saved_errno = errno;
+
+        if(saved_errno == ENOTEMPTY) {
+            saved_errno = EEXIST;
+        }
+
+        /* The error code might be wrong if we're trying to delete the current
+         * directory. */
+        if(saved_errno == EEXIST) {
+            struct stat path_stat, cwd_stat;
+            int has_stat;
+
+            has_stat = (stat((const char*)path->data, &path_stat) == 0);
+            has_stat &= (stat(".", &cwd_stat) == 0);
+
+            if(has_stat && path_stat.st_ino == cwd_stat.st_ino) {
+                if(path_stat.st_dev == cwd_stat.st_dev) {
+                    return EINVAL;
+                }
+            }
+        }
+
+        return saved_errno;
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_set_cwd(const efile_path_t *path) {
+    if(chdir((const char*)path->data) < 0) {
+        return errno;
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_get_device_cwd(ErlNifEnv *env, int device_index, ERL_NIF_TERM *result) {
+    (void)device_index;
+    (void)result;
+    (void)env;
+
+    return ENOTSUP;
+}
+
+posix_errno_t efile_get_cwd(ErlNifEnv *env, ERL_NIF_TERM *result) {
+    ErlNifBinary result_bin;
+    size_t bytes_copied;
+
+    if(!enif_alloc_binary(256, &result_bin)) {
+        return ENOMEM;
+    }
+
+    while(getcwd((char*)result_bin.data, result_bin.size) == NULL) {
+        posix_errno_t saved_errno = errno;
+
+        if(saved_errno != ERANGE) {
+            enif_release_binary(&result_bin);
+            return saved_errno;
+        } else {
+            if(!enif_realloc_binary(&result_bin, result_bin.size * 2)) {
+                enif_release_binary(&result_bin);
+                return ENOMEM;
+            }
+        }
+    }
+
+    /* getcwd(2) guarantees null-termination. */
+    bytes_copied = strlen((const char*)result_bin.data);
+
+    if(!enif_realloc_binary(&result_bin, bytes_copied)) {
+        enif_release_binary(&result_bin);
+        return ENOMEM;
+    }
+
+    (*result) = enif_make_binary(env, &result_bin);
+
+    return 0;
+}
+
+posix_errno_t efile_altname(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result) {
+    (void)path;
+    (void)result;
+
+    return ENOTSUP;
+}
diff --git a/erts/emulator/nifs/win32/win_prim_file.c b/erts/emulator/nifs/win32/win_prim_file.c
new file mode 100644
index 0000000000..9f993f1d24
--- /dev/null
+++ b/erts/emulator/nifs/win32/win_prim_file.c
@@ -0,0 +1,1427 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson 2017. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "erl_nif.h"
+#include "config.h"
+#include "sys.h"
+
+#include "prim_file_nif.h"
+
+#include <windows.h>
+#include <strsafe.h>
+#include <wchar.h>
+
+#define IS_SLASH(a)  ((a) == L'\\' || (a) == L'/')
+
+#define FILE_SHARE_FLAGS (FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
+
+#define LP_PREFIX L"\\\\?\\"
+#define LP_PREFIX_SIZE (sizeof(LP_PREFIX) - sizeof(WCHAR))
+#define LP_PREFIX_LENGTH (LP_PREFIX_SIZE / sizeof(WCHAR))
+
+#define PATH_LENGTH(path) (path->size / sizeof(WCHAR) - 1)
+
+#define ASSERT_PATH_FORMAT(path) \
+    do { \
+        ASSERT(PATH_LENGTH(path) >= 4 && \
+            !memcmp(path->data, LP_PREFIX, LP_PREFIX_SIZE)); \
+        ASSERT(PATH_LENGTH(path) == wcslen((WCHAR*)path->data)); \
+    } while(0)
+
+#define TICKS_PER_SECOND (10000000ULL)
+#define EPOCH_DIFFERENCE (11644473600LL)
+
+#define FILETIME_TO_EPOCH(epoch, ft) \
+    do { \
+        ULARGE_INTEGER ull; \
+        ull.LowPart  = (ft).dwLowDateTime; \
+        ull.HighPart = (ft).dwHighDateTime; \
+        (epoch) = ((ull.QuadPart / TICKS_PER_SECOND) - EPOCH_DIFFERENCE); \
+    } while(0)
+
+#define EPOCH_TO_FILETIME(ft, epoch) \
+    do { \
+        ULARGE_INTEGER ull; \
+        ull.QuadPart = (((epoch) + EPOCH_DIFFERENCE) * TICKS_PER_SECOND); \
+        (ft).dwLowDateTime = ull.LowPart; \
+        (ft).dwHighDateTime = ull.HighPart; \
+    } while(0)
+
+typedef struct {
+    efile_data_t common;
+    HANDLE handle;
+} efile_win_t;
+
+static int windows_to_posix_errno(DWORD last_error);
+
+static int has_invalid_null_termination(const ErlNifBinary *path) {
+    const WCHAR *null_pos, *end_pos;
+
+    null_pos = wmemchr((const WCHAR*)path->data, L'\0', path->size);
+    end_pos = (const WCHAR*)&path->data[path->size] - 1;
+
+    if(null_pos == NULL) {
+        return 1;
+    }
+
+    /* prim_file:internal_name2native sometimes feeds us data that is "doubly"
+     * NUL-terminated, so we'll accept any number of trailing NULs so long as
+     * they aren't interrupted by anything else. */
+    while(null_pos < end_pos && (*null_pos) == L'\0') {
+        null_pos++;
+    }
+
+    return null_pos != end_pos;
+}
+
+static posix_errno_t get_full_path(ErlNifEnv *env, WCHAR *input, efile_path_t *result) {
+    DWORD maximum_length, actual_length;
+    int add_long_prefix;
+
+    maximum_length = GetFullPathNameW(input, 0, NULL, NULL);
+    add_long_prefix = 0;
+
+    if(maximum_length == 0) {
+        /* POSIX doesn't have the concept of a "path error" in the same way
+         * Windows does, so we'll return ENOENT since that's what most POSIX
+         * APIs would return if they were fed such garbage. */
+        return ENOENT;
+    }
+
+    maximum_length += LP_PREFIX_LENGTH;
+
+    if(!enif_alloc_binary(maximum_length * sizeof(WCHAR), result)) {
+        return ENOMEM;
+    }
+
+    actual_length = GetFullPathNameW(input, maximum_length, (WCHAR*)result->data, NULL);
+
+    if(actual_length < maximum_length) {
+        int has_long_path_prefix;
+        WCHAR *path_start;
+
+        /* Make sure we have a long-path prefix; GetFullPathNameW only adds one
+         * if the path is relative. */
+        has_long_path_prefix = actual_length >= LP_PREFIX_LENGTH &&
+            !sys_memcmp(result->data, LP_PREFIX, LP_PREFIX_SIZE);
+
+        if(!has_long_path_prefix) {
+            sys_memmove(result->data + LP_PREFIX_SIZE, result->data,
+                (actual_length + 1) * sizeof(WCHAR));
+            sys_memcpy(result->data, LP_PREFIX, LP_PREFIX_SIZE);
+            actual_length += LP_PREFIX_LENGTH;
+        }
+
+        path_start = (WCHAR*)result->data;
+
+        /* We're removing trailing slashes since quite a few APIs refuse to
+         * work with them, and none require them. We only check the last
+         * character since GetFullPathNameW folds slashes together. */
+        if(IS_SLASH(path_start[actual_length - 1])) {
+            if(path_start[actual_length - 2] != L':') {
+                path_start[actual_length - 1] = L'\0';
+                actual_length--;
+            }
+        }
+
+        if(!enif_realloc_binary(result, (actual_length + 1) * sizeof(WCHAR))) {
+            enif_release_binary(result);
+            return ENOMEM;
+        }
+
+        enif_make_binary(env, result);
+        return 0;
+    }
+
+    /* We may end up here if the current directory changes to something longer
+     * between/during GetFullPathName. There's nothing sensible we can do about
+     * this. */
+
+    enif_release_binary(result);
+
+    return EINVAL;
+}
+
+posix_errno_t efile_marshal_path(ErlNifEnv *env, ERL_NIF_TERM path, efile_path_t *result) {
+    ErlNifBinary raw_path;
+
+    if(!enif_inspect_binary(env, path, &raw_path)) {
+        return EINVAL;
+    } else if(raw_path.size % sizeof(WCHAR)) {
+        return EINVAL;
+    }
+
+    if(has_invalid_null_termination(&raw_path)) {
+        return EINVAL;
+    }
+
+    return get_full_path(env, (WCHAR*)raw_path.data, result);
+}
+
+ERL_NIF_TERM efile_get_handle(ErlNifEnv *env, efile_data_t *d) {
+    efile_win_t *w = (efile_win_t*)d;
+
+    ERL_NIF_TERM result;
+    unsigned char *bits;
+
+    bits = enif_make_new_binary(env, sizeof(w->handle), &result);
+    memcpy(bits, &w->handle, sizeof(w->handle));
+
+    return result;
+}
+
+/** @brief Converts a native path to the preferred form in "erlang space,"
+ * without path-prefixes, forward-slashes, or NUL terminators. */
+static int normalize_path_result(ErlNifBinary *path) {
+    WCHAR *path_iterator, *path_start, *path_end;
+    int length;
+
+    path_start = (WCHAR*)path->data;
+    length = wcslen(path_start);
+
+    ASSERT(length < path->size / sizeof(WCHAR));
+
+    /* Get rid of the long-path prefix, if present. */
+    if(length >= LP_PREFIX_LENGTH) {
+        if(!sys_memcmp(path_start, LP_PREFIX, LP_PREFIX_SIZE)) {
+            length -= LP_PREFIX_LENGTH;
+
+            sys_memmove(path_start, &path_start[LP_PREFIX_LENGTH],
+                length * sizeof(WCHAR));
+        }
+    }
+
+    path_end = &path_start[length];
+    path_iterator = path_start;
+
+    /* Convert drive letters to lowercase, if present. */
+    if(length >= 2 && path_start[1] == L':') {
+        WCHAR drive_letter = path_start[0];
+
+        if(drive_letter >= L'A' && drive_letter <= L'Z') {
+            path_start[0] = drive_letter - L'A' + L'a';
+        }
+    }
+
+    while(path_iterator < path_end) {
+        if(*path_iterator == L'\\') {
+            *path_iterator = L'/';
+        }
+
+        path_iterator++;
+    }
+
+    /* Truncate the result to its actual length; we don't want to include the
+     * NUL terminator. */
+    return enif_realloc_binary(path, length * sizeof(WCHAR));
+}
+
+/* @brief Checks whether all the given attributes are set on the object at the
+ * given path. Note that it assumes false on errors. */
+static int has_file_attributes(const efile_path_t *path, DWORD mask) {
+    DWORD attributes = GetFileAttributesW((WCHAR*)path->data);
+
+    if(attributes == INVALID_FILE_ATTRIBUTES) {
+        return 0;
+    }
+
+    return !!((attributes & mask) == mask);
+}
+
+static int is_ignored_name(int name_length, const WCHAR *name) {
+    if(name_length == 1 && name[0] == L'.') {
+        return 1;
+    } else if(name_length == 2 && !sys_memcmp(name, L"..", 2 * sizeof(WCHAR))) {
+        return 1;
+    }
+
+    return 0;
+}
+
+static int get_drive_number(const efile_path_t *path) {
+    const WCHAR *path_start;
+    int length;
+
+    ASSERT_PATH_FORMAT(path);
+
+    path_start = (WCHAR*)path->data + LP_PREFIX_LENGTH;
+    length = PATH_LENGTH(path) - LP_PREFIX_LENGTH;
+
+    if(length >= 2 && path_start[1] == L':') {
+        WCHAR drive_letter = path_start[0];
+
+        if(drive_letter >= L'A' && drive_letter <= L'Z') {
+            return drive_letter - L'A' + 1;
+        } else if(drive_letter >= L'a' && drive_letter <= L'z') {
+            return drive_letter - L'a' + 1;
+        }
+    }
+
+    return -1;
+}
+
+/* @brief Checks whether two *paths* are on the same mount point; they don't
+ * have to refer to existing or accessible files/directories. */
+static int has_same_mount_point(const efile_path_t *path_a, const efile_path_t *path_b) {
+    WCHAR *mount_a, *mount_b;
+    int result = 0;
+
+    mount_a = enif_alloc(path_a->size);
+    mount_b = enif_alloc(path_b->size);
+
+    if(mount_a != NULL && mount_b != NULL) {
+        int length_a, length_b;
+
+        length_a = PATH_LENGTH(path_a);
+        length_b = PATH_LENGTH(path_b);
+
+        if(GetVolumePathNameW((WCHAR*)path_a->data, mount_a, length_a)) {
+            ASSERT(wcslen(mount_a) <= length_a);
+
+            if(GetVolumePathNameW((WCHAR*)path_b->data, mount_b, length_b)) {
+                ASSERT(wcslen(mount_b) <= length_b);
+
+                result = !_wcsicmp(mount_a, mount_b);
+            }
+        }
+    }
+
+    if(mount_b != NULL) {
+        enif_free(mount_b);
+    }
+
+    if(mount_a != NULL) {
+        enif_free(mount_a);
+    }
+
+    return result;
+}
+
+/* Mirrors the PathIsRootW function of the shell API, but doesn't choke on
+ * paths longer than MAX_PATH. */
+static int is_path_root(const efile_path_t *path) {
+    const WCHAR *path_start, *path_end;
+    int length;
+
+    ASSERT_PATH_FORMAT(path);
+
+    path_start = (WCHAR*)path->data + LP_PREFIX_LENGTH;
+    length = PATH_LENGTH(path) - LP_PREFIX_LENGTH;
+
+    path_end = &path_start[length];
+
+    if(length == 1) {
+        /* A single \ refers to the root of the current working directory. */
+        return IS_SLASH(path_start[0]);
+    } else if(length == 3 && iswalpha(path_start[0]) && path_start[1] == L':') {
+        /* Drive letter. */
+        return IS_SLASH(path_start[2]);
+    } else if(length >= 4) {
+        /* Check whether we're a UNC root, eg. \\server, \\server\share */
+        const WCHAR *path_iterator;
+
+        if(!IS_SLASH(path_start[0]) || !IS_SLASH(path_start[1])) {
+            return 0;
+        }
+
+        path_iterator = path_start + 2;
+
+        /* Slide to the slash between the server and share names, if present. */
+        while(path_iterator < path_end && !IS_SLASH(*path_iterator)) {
+            path_iterator++;
+        }
+
+        /* Slide past the end of the string, stopping at the first slash we
+         * encounter. */
+        do {
+            path_iterator++;
+        }  while(path_iterator < path_end && !IS_SLASH(*path_iterator));
+
+        /* If we're past the end of the string and it didnt't end with a slash,
+         * then we're a root path. */
+        return path_iterator >= path_end && !IS_SLASH(path_start[length - 1]);
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes,
+        ErlNifResourceType *nif_type, efile_data_t **d) {
+
+    DWORD attributes, access_flags, open_mode;
+    HANDLE handle;
+
+    ASSERT_PATH_FORMAT(path);
+
+    access_flags = 0;
+    open_mode = 0;
+
+    if(modes & EFILE_MODE_READ && !(modes & EFILE_MODE_WRITE)) {
+        access_flags = GENERIC_READ;
+        open_mode = OPEN_EXISTING;
+    } else if(modes & EFILE_MODE_WRITE && !(modes & EFILE_MODE_READ)) {
+        access_flags = GENERIC_WRITE;
+        open_mode = CREATE_ALWAYS;
+    } else if(modes & EFILE_MODE_READ_WRITE) {
+        access_flags = GENERIC_READ | GENERIC_WRITE;
+        open_mode = OPEN_ALWAYS;
+    } else {
+        return EINVAL;
+    }
+
+    if(modes & EFILE_MODE_APPEND) {
+        access_flags |= FILE_APPEND_DATA;
+        open_mode = OPEN_ALWAYS;
+    }
+
+    if(modes & EFILE_MODE_EXCLUSIVE) {
+        open_mode = CREATE_NEW;
+    }
+
+    if(modes & EFILE_MODE_SYNC) {
+        attributes = FILE_FLAG_WRITE_THROUGH;
+    } else {
+        attributes = FILE_ATTRIBUTE_NORMAL;
+    }
+
+    handle = CreateFileW((WCHAR*)path->data, access_flags,
+        FILE_SHARE_FLAGS, NULL, open_mode, attributes, NULL);
+
+    if(handle != INVALID_HANDLE_VALUE) {
+        efile_win_t *w;
+
+        w = (efile_win_t*)enif_alloc_resource(nif_type, sizeof(efile_win_t));
+        w->handle = handle;
+
+        EFILE_INIT_RESOURCE(&w->common, modes);
+        (*d) = &w->common;
+
+        return 0;
+    } else {
+        DWORD last_error = GetLastError();
+
+        /* Rewrite all failures on directories to EISDIR to match the old
+         * driver. */
+        if(has_file_attributes(path, FILE_ATTRIBUTE_DIRECTORY)) {
+            return EISDIR;
+        }
+
+        return windows_to_posix_errno(last_error);
+    }
+}
+
+int efile_close(efile_data_t *d) {
+    efile_win_t *w = (efile_win_t*)d;
+    HANDLE handle;
+
+    ASSERT(erts_atomic32_read_nob(&d->state) == EFILE_STATE_CLOSED);
+    ASSERT(w->handle != INVALID_HANDLE_VALUE);
+
+    handle = w->handle;
+    w->handle = INVALID_HANDLE_VALUE;
+
+    if(!CloseHandle(handle)) {
+        w->common.posix_errno = windows_to_posix_errno(GetLastError());
+        return 0;
+    }
+
+    return 1;
+}
+
+static void shift_overlapped(OVERLAPPED *overlapped, DWORD shift) {
+    LARGE_INTEGER offset;
+
+    ASSERT(shift >= 0);
+
+    offset.HighPart = overlapped->OffsetHigh;
+    offset.LowPart = overlapped->Offset;
+
+    /* ~(Uint64)0 is a magic value ("append to end of file") which needs to be
+     * preserved. Other positions resulting in overflow would have errored out
+     * just prior to this point. */
+    if(offset.QuadPart != ERTS_UINT64_MAX) {
+        offset.QuadPart += shift;
+    }
+
+    /* All unused fields must be zeroed for the next call. */
+    sys_memset(overlapped, 0, sizeof(*overlapped));
+    overlapped->OffsetHigh = offset.HighPart;
+    overlapped->Offset = offset.LowPart;
+}
+
+static void shift_iov(SysIOVec **iov, int *iovlen, DWORD shift) {
+    SysIOVec *head_vec = (*iov);
+
+    ASSERT(shift >= 0);
+
+    while(shift > 0) {
+        ASSERT(head_vec < &(*iov)[*iovlen]);
+
+        if(shift < head_vec->iov_len) {
+            head_vec->iov_base = (char*)head_vec->iov_base + shift;
+            head_vec->iov_len -= shift;
+            break;
+        } else {
+            shift -= head_vec->iov_len;
+            head_vec++;
+        }
+    }
+
+    (*iovlen) -= head_vec - (*iov);
+    (*iov) = head_vec;
+}
+
+typedef BOOL (WINAPI *io_op_t)(HANDLE, LPVOID, DWORD, LPDWORD, LPOVERLAPPED);
+
+static Sint64 internal_sync_io(efile_win_t *w, io_op_t operation,
+        SysIOVec *iov, int iovlen, OVERLAPPED *overlapped) {
+
+    Sint64 bytes_processed = 0;
+
+    for(;;) {
+        DWORD block_bytes_processed, last_error;
+        BOOL succeeded;
+
+        if(iovlen < 1) {
+            return bytes_processed;
+        }
+
+        succeeded = operation(w->handle, iov->iov_base, iov->iov_len,
+            &block_bytes_processed, overlapped);
+        last_error = GetLastError();
+
+        if(!succeeded && (last_error != ERROR_HANDLE_EOF)) {
+            w->common.posix_errno = windows_to_posix_errno(last_error);
+            return -1;
+        } else if(block_bytes_processed == 0) {
+            /* EOF */
+            return bytes_processed;
+        }
+
+        if(overlapped != NULL) {
+            shift_overlapped(overlapped, block_bytes_processed);
+        }
+
+        shift_iov(&iov, &iovlen, block_bytes_processed);
+
+        bytes_processed += block_bytes_processed;
+    }
+}
+
+Sint64 efile_readv(efile_data_t *d, SysIOVec *iov, int iovlen) {
+    efile_win_t *w = (efile_win_t*)d;
+
+    return internal_sync_io(w, ReadFile, iov, iovlen, NULL);
+}
+
+Sint64 efile_writev(efile_data_t *d, SysIOVec *iov, int iovlen) {
+    efile_win_t *w = (efile_win_t*)d;
+
+    OVERLAPPED __overlapped, *overlapped;
+    Uint64 bytes_written;
+
+    if(w->common.modes & EFILE_MODE_APPEND) {
+        overlapped = &__overlapped;
+
+        sys_memset(overlapped, 0, sizeof(*overlapped));
+        overlapped->OffsetHigh = 0xFFFFFFFF;
+        overlapped->Offset = 0xFFFFFFFF;
+    } else {
+        overlapped = NULL;
+    }
+
+    return internal_sync_io(w, WriteFile, iov, iovlen, overlapped);
+}
+
+Sint64 efile_preadv(efile_data_t *d, Sint64 offset, SysIOVec *iov, int iovlen) {
+    efile_win_t *w = (efile_win_t*)d;
+
+    OVERLAPPED overlapped;
+
+    sys_memset(&overlapped, 0, sizeof(overlapped));
+    overlapped.OffsetHigh = (offset >> 32) & 0xFFFFFFFF;
+    overlapped.Offset = offset & 0xFFFFFFFF;
+
+    return internal_sync_io(w, ReadFile, iov, iovlen, &overlapped);
+}
+
+Sint64 efile_pwritev(efile_data_t *d, Sint64 offset, SysIOVec *iov, int iovlen) {
+    efile_win_t *w = (efile_win_t*)d;
+
+    OVERLAPPED overlapped;
+
+    sys_memset(&overlapped, 0, sizeof(overlapped));
+    overlapped.OffsetHigh = (offset >> 32) & 0xFFFFFFFF;
+    overlapped.Offset = offset & 0xFFFFFFFF;
+
+    return internal_sync_io(w, WriteFile, iov, iovlen, &overlapped);
+}
+
+int efile_seek(efile_data_t *d, enum efile_seek_t seek, Sint64 offset, Sint64 *new_position) {
+    efile_win_t *w = (efile_win_t*)d;
+
+    LARGE_INTEGER large_offset, large_new_position;
+    DWORD whence;
+
+    switch(seek) {
+        case EFILE_SEEK_BOF: whence = FILE_BEGIN; break;
+        case EFILE_SEEK_CUR: whence = FILE_CURRENT; break;
+        case EFILE_SEEK_EOF: whence = FILE_END; break;
+        default: ERTS_INTERNAL_ERROR("Invalid seek parameter");
+    }
+
+    large_offset.QuadPart = offset;
+
+    if(!SetFilePointerEx(w->handle, large_offset, &large_new_position, whence)) {
+        w->common.posix_errno = windows_to_posix_errno(GetLastError());
+        return 0;
+    }
+
+    (*new_position) = large_new_position.QuadPart;
+
+    return 1;
+}
+
+int efile_sync(efile_data_t *d, int data_only) {
+    efile_win_t *w = (efile_win_t*)d;
+
+    /* Windows doesn't support data-only syncing. */
+    (void)data_only;
+
+    if(!FlushFileBuffers(w->handle)) {
+        w->common.posix_errno = windows_to_posix_errno(GetLastError());
+        return 0;
+    }
+
+    return 1;
+}
+
+int efile_advise(efile_data_t *d, Sint64 offset, Sint64 length, enum efile_advise_t advise) {
+    /* Windows doesn't support this, but we'll pretend it does since the call
+     * is only a recommendation even on systems that do support it. */
+
+    (void)d;
+    (void)offset;
+    (void)length;
+    (void)advise;
+
+    return 1;
+}
+
+int efile_allocate(efile_data_t *d, Sint64 offset, Sint64 length) {
+    efile_win_t *w = (efile_win_t*)d;
+
+    (void)d;
+    (void)offset;
+    (void)length;
+
+    w->common.posix_errno = ENOTSUP;
+
+    return 0;
+}
+
+int efile_truncate(efile_data_t *d) {
+    efile_win_t *w = (efile_win_t*)d;
+
+    if(!SetEndOfFile(w->handle)) {
+        w->common.posix_errno = windows_to_posix_errno(GetLastError());
+        return 0;
+    }
+
+    return 1;
+}
+
+static int is_executable_file(const efile_path_t *path) {
+    /* We're using the file extension in order to be quirks-compliant with the
+     * old driver, which never bothered to check the actual permissions. We
+     * could easily do so now (cf. GetNamedSecurityInfo) but the execute
+     * permission is only relevant for files that are started with the default
+     * loader, and batch files run just fine with read permission alone. */
+
+    int length = PATH_LENGTH(path);
+
+    if(length >= 4) {
+        const WCHAR *last_four = &((WCHAR*)path->data)[length - 4];
+
+        if (!_wcsicmp(last_four, L".exe") ||
+            !_wcsicmp(last_four, L".cmd") ||
+            !_wcsicmp(last_four, L".bat") ||
+            !_wcsicmp(last_four, L".com")) {
+            return 1;
+        }
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_read_info(const efile_path_t *path, int follow_links, efile_fileinfo_t *result) {
+    BY_HANDLE_FILE_INFORMATION native_file_info;
+    DWORD attributes;
+
+    sys_memset(&native_file_info, 0, sizeof(native_file_info));
+
+    attributes = GetFileAttributesW((WCHAR*)path->data);
+
+    if(attributes == INVALID_FILE_ATTRIBUTES) {
+        DWORD last_error = GetLastError();
+
+        /* Querying a network share root fails with ERROR_BAD_NETPATH, so we'll
+         * fake it as a directory just like local roots. */
+        if(!is_path_root(path) || last_error != ERROR_BAD_NETPATH) {
+            return windows_to_posix_errno(last_error);
+        }
+
+        attributes = FILE_ATTRIBUTE_DIRECTORY;
+    } else if(is_path_root(path)) {
+        /* Local (or mounted) roots can be queried with GetFileAttributesW but
+         * lack support for GetFileInformationByHandle, so we'll skip that
+         * part. */
+    } else {
+        HANDLE handle;
+
+        if(follow_links && (attributes & FILE_ATTRIBUTE_REPARSE_POINT)) {
+            posix_errno_t posix_errno;
+            efile_path_t resolved_path;
+
+            posix_errno = internal_read_link(path, &resolved_path);
+
+            if(posix_errno != 0) {
+                return posix_errno;
+            }
+
+            return efile_read_info(&resolved_path, 0, result);
+        }
+
+        handle = CreateFileW((const WCHAR*)path->data, GENERIC_READ,
+            FILE_SHARE_FLAGS, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS,
+            NULL);
+
+        /* The old driver never cared whether this succeeded. */
+        if(handle != INVALID_HANDLE_VALUE) {
+            GetFileInformationByHandle(handle, &native_file_info);
+            CloseHandle(handle);
+        }
+
+        FILETIME_TO_EPOCH(result->m_time, native_file_info.ftLastWriteTime);
+        FILETIME_TO_EPOCH(result->a_time, native_file_info.ftLastAccessTime);
+        FILETIME_TO_EPOCH(result->c_time, native_file_info.ftCreationTime);
+
+        if(result->m_time == -EPOCH_DIFFERENCE) {
+            /* Default to 1970 just like the old driver. */
+            result->m_time = 0;
+        }
+
+        if(result->a_time == -EPOCH_DIFFERENCE) {
+            result->a_time = result->m_time;
+        }
+
+        if(result->c_time == -EPOCH_DIFFERENCE) {
+            result->c_time = result->m_time;
+        }
+    }
+
+    if(attributes & FILE_ATTRIBUTE_REPARSE_POINT) {
+        result->type = EFILE_FILETYPE_SYMLINK;
+        /* This should be _S_IFLNK, but the old driver always set
+         * non-directories to _S_IFREG. */
+        result->mode |= _S_IFREG;
+    } else if(attributes & FILE_ATTRIBUTE_DIRECTORY) {
+        result->type = EFILE_FILETYPE_DIRECTORY;
+        result->mode |= _S_IFDIR | _S_IEXEC;
+    } else {
+        if(is_executable_file(path)) {
+            result->mode |= _S_IEXEC;
+        }
+
+        result->type = EFILE_FILETYPE_REGULAR;
+        result->mode |= _S_IFREG;
+    }
+
+    if(!(attributes & FILE_ATTRIBUTE_READONLY)) {
+        result->access = EFILE_ACCESS_READ | EFILE_ACCESS_WRITE;
+        result->mode |= _S_IREAD | _S_IWRITE;
+    } else {
+        result->access = EFILE_ACCESS_READ;
+        result->mode |= _S_IREAD;
+    }
+
+    /* Propagate user mode-bits to group/other fields */
+    result->mode |= (result->mode & 0700) >> 3;
+    result->mode |= (result->mode & 0700) >> 6;
+
+    result->size =
+        ((Uint64)native_file_info.nFileSizeHigh << 32ull) |
+        (Uint64)native_file_info.nFileSizeLow;
+
+    result->links = MAX(1, native_file_info.nNumberOfLinks);
+
+    result->major_device = get_drive_number(path);
+    result->minor_device = 0;
+    result->inode = 0;
+    result->uid = 0;
+    result->gid = 0;
+
+    return 0;
+}
+
+posix_errno_t efile_set_permissions(const efile_path_t *path, Uint32 permissions) {
+    DWORD attributes = GetFileAttributesW((WCHAR*)path->data);
+
+    if(attributes == INVALID_FILE_ATTRIBUTES) {
+        return windows_to_posix_errno(GetLastError());
+    }
+
+    if(permissions & _S_IWRITE) {
+        attributes &= ~FILE_ATTRIBUTE_READONLY;
+    } else {
+        attributes |= FILE_ATTRIBUTE_READONLY;
+    }
+
+    if(SetFileAttributesW((WCHAR*)path->data, attributes)) {
+        return 0;
+    }
+
+    return windows_to_posix_errno(GetLastError());
+}
+
+posix_errno_t efile_set_owner(const efile_path_t *path, Uint32 owner, Uint32 group) {
+    (void)path;
+    (void)owner;
+    (void)group;
+
+    return 0;
+}
+
+posix_errno_t efile_set_time(const efile_path_t *path, Sint64 a_time, Sint64 m_time, Sint64 c_time) {
+    FILETIME accessed, modified, created;
+    DWORD last_error, attributes;
+    HANDLE handle;
+
+    attributes = GetFileAttributesW((WCHAR*)path->data);
+
+    if(attributes == INVALID_FILE_ATTRIBUTES) {
+        return windows_to_posix_errno(GetLastError());
+    }
+
+    /* If the file is read-only, we have to make it temporarily writable while
+     * setting new metadata. */
+    if(attributes & FILE_ATTRIBUTE_READONLY) {
+        DWORD without_readonly = attributes & ~FILE_ATTRIBUTE_READONLY;
+
+        if(!SetFileAttributesW((WCHAR*)path->data, without_readonly)) {
+            return windows_to_posix_errno(GetLastError());
+        }
+    }
+
+    EPOCH_TO_FILETIME(modified, m_time);
+    EPOCH_TO_FILETIME(accessed, a_time);
+    EPOCH_TO_FILETIME(created, c_time);
+
+    handle = CreateFileW((WCHAR*)path->data, GENERIC_READ | GENERIC_WRITE,
+        FILE_SHARE_FLAGS, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL);
+    last_error = GetLastError();
+
+    if(handle != INVALID_HANDLE_VALUE) {
+        if(SetFileTime(handle, &created, &accessed, &modified)) {
+            last_error = ERROR_SUCCESS;
+        } else {
+            last_error = GetLastError();
+        }
+
+        CloseHandle(handle);
+    }
+
+    if(attributes & FILE_ATTRIBUTE_READONLY) {
+        SetFileAttributesW((WCHAR*)path->data, attributes);
+    }
+
+    return windows_to_posix_errno(last_error);
+}
+
+static posix_errno_t internal_read_link(const efile_path_t *path, efile_path_t *result) {
+    DWORD required_length, actual_length;
+    HANDLE link_handle;
+    DWORD last_error;
+
+    link_handle = CreateFileW((WCHAR*)path->data, GENERIC_READ,
+        FILE_SHARE_FLAGS, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL);
+    last_error = GetLastError();
+
+    if(link_handle == INVALID_HANDLE_VALUE) {
+        return windows_to_posix_errno(last_error);
+    }
+
+    required_length = GetFinalPathNameByHandleW(link_handle, NULL, 0, 0);
+    last_error = GetLastError();
+
+    if(required_length <= 0) {
+        CloseHandle(link_handle);
+        return windows_to_posix_errno(last_error);
+    }
+
+    /* Unlike many other path functions (eg. GetFullPathNameW), this one
+     * includes the NUL terminator in its required length. */
+    if(!enif_alloc_binary(required_length * sizeof(WCHAR), result)) {
+        CloseHandle(link_handle);
+        return ENOMEM;
+    }
+
+    actual_length = GetFinalPathNameByHandleW(link_handle,
+        (WCHAR*)result->data, required_length, 0);
+    last_error = GetLastError();
+
+    CloseHandle(link_handle);
+
+    if(actual_length == 0 || actual_length >= required_length) {
+        enif_release_binary(result);
+        return windows_to_posix_errno(last_error);
+    }
+
+    /* GetFinalPathNameByHandle always prepends with "\\?\" and NUL-terminates,
+     * so we never have to touch-up the resulting path. */
+
+    ASSERT_PATH_FORMAT(result);
+
+    return 0;
+}
+
+posix_errno_t efile_read_link(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result) {
+    posix_errno_t posix_errno;
+    ErlNifBinary result_bin;
+    DWORD attributes;
+
+    ASSERT_PATH_FORMAT(path);
+
+    attributes = GetFileAttributesW((WCHAR*)path->data);
+
+    if(attributes == INVALID_FILE_ATTRIBUTES) {
+        return windows_to_posix_errno(GetLastError());
+    } else if(!(attributes & FILE_ATTRIBUTE_REPARSE_POINT)) {
+        return EINVAL;
+    }
+
+    posix_errno = internal_read_link(path, &result_bin);
+
+    if(posix_errno == 0) {
+        if(!normalize_path_result(&result_bin)) {
+            enif_release_binary(&result_bin);
+            return ENOMEM;
+        }
+
+        (*result) = enif_make_binary(env, &result_bin);
+    }
+
+    return posix_errno;
+}
+
+posix_errno_t efile_list_dir(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result) {
+    ERL_NIF_TERM list_head;
+    WIN32_FIND_DATAW data;
+    HANDLE search_handle;
+    WCHAR *search_path;
+    DWORD last_error;
+
+    ASSERT_PATH_FORMAT(path);
+
+    search_path = enif_alloc(path->size + 2 * sizeof(WCHAR));
+
+    if(search_path == NULL) {
+        return ENOMEM;
+    }
+
+    sys_memcpy(search_path, path->data, path->size);
+    search_path[PATH_LENGTH(path) + 0] = L'\\';
+    search_path[PATH_LENGTH(path) + 1] = L'*';
+    search_path[PATH_LENGTH(path) + 2] = L'\0';
+
+    search_handle = FindFirstFileW(search_path, &data);
+    last_error = GetLastError();
+
+    enif_free(search_path);
+
+    if(search_handle == INVALID_HANDLE_VALUE) {
+        return windows_to_posix_errno(last_error);
+    }
+
+    list_head = enif_make_list(env, 0);
+
+    do {
+        int name_length = wcslen(data.cFileName);
+
+        if(!is_ignored_name(name_length, data.cFileName)) {
+            unsigned char *name_bytes;
+            ERL_NIF_TERM name_term;
+            size_t name_size;
+
+            name_size = name_length * sizeof(WCHAR);
+
+            name_bytes = enif_make_new_binary(env, name_size, &name_term);
+            sys_memcpy(name_bytes, data.cFileName, name_size);
+
+            list_head = enif_make_list_cell(env, name_term, list_head);
+        }
+    } while(FindNextFileW(search_handle, &data));
+
+    FindClose(search_handle);
+    (*result) = list_head;
+
+    return 0;
+}
+
+posix_errno_t efile_rename(const efile_path_t *old_path, const efile_path_t *new_path) {
+    BOOL old_is_directory, new_is_directory;
+    DWORD move_flags, last_error;
+
+    ASSERT_PATH_FORMAT(old_path);
+    ASSERT_PATH_FORMAT(new_path);
+
+    move_flags = MOVEFILE_COPY_ALLOWED | MOVEFILE_WRITE_THROUGH;
+
+    if(MoveFileExW((WCHAR*)old_path->data, (WCHAR*)new_path->data, move_flags)) {
+        return 0;
+    }
+
+    last_error = GetLastError();
+
+    old_is_directory = has_file_attributes(old_path, FILE_ATTRIBUTE_DIRECTORY);
+    new_is_directory = has_file_attributes(new_path, FILE_ATTRIBUTE_DIRECTORY);
+
+    switch(last_error) {
+    case ERROR_SHARING_VIOLATION:
+    case ERROR_ACCESS_DENIED:
+        if(old_is_directory) {
+            BOOL moved_into_itself;
+
+            moved_into_itself = (old_path->size <= new_path->size) &&
+                !_wcsnicmp((WCHAR*)old_path->data, (WCHAR*)new_path->data,
+                    PATH_LENGTH(old_path));
+
+            if(moved_into_itself) {
+                return EINVAL;
+            } else if(is_path_root(old_path)) {
+                return EINVAL;
+            }
+
+            /* Renaming a directory across volumes needs to be rewritten as
+             * EXDEV so that the caller can respond by simulating it with
+             * copy/delete operations.
+             *
+             * Files are handled through MOVEFILE_COPY_ALLOWED. */
+            if(!has_same_mount_point(old_path, new_path)) {
+                return EXDEV;
+            }
+        }
+        break;
+    case ERROR_PATH_NOT_FOUND:
+    case ERROR_FILE_NOT_FOUND:
+        return ENOENT;
+    case ERROR_ALREADY_EXISTS:
+    case ERROR_FILE_EXISTS:
+        if(old_is_directory && !new_is_directory) {
+            return ENOTDIR;
+        } else if(!old_is_directory && new_is_directory) {
+            return EISDIR;
+        } else if(old_is_directory && new_is_directory) {
+            /* This will fail if the destination isn't empty. */
+            if(RemoveDirectoryW((WCHAR*)new_path->data)) {
+                return efile_rename(old_path, new_path);
+            }
+
+            return EEXIST;
+        } else if(!old_is_directory && !new_is_directory) {
+            /* This is pretty iffy; the public documentation says that the
+             * operation may EACCES on some systems when either file is open,
+             * which gives us room to use MOVEFILE_REPLACE_EXISTING and be done
+             * with it, but the old implementation simulated Unix semantics and
+             * there's a lot of code that relies on that.
+             *
+             * The simulation renames the destination to a scratch name to get
+             * around the fact that it's impossible to open (and by extension
+             * rename) a file that's been deleted while open. It has a few
+             * drawbacks though;
+             *
+             * 1) It's not atomic as there's a small window where there's no
+             *    file at all on the destination path.
+             * 2) It will confuse applications that subscribe to folder
+             *    changes.
+             * 3) It will fail if we lack general permission to write in the
+             *    same folder. */
+
+            WCHAR *swap_path = enif_alloc(new_path->size + sizeof(WCHAR) * 64);
+
+            if(swap_path == NULL) {
+                return ENOMEM;
+            } else {
+                static LONGLONG unique_counter = 0;
+                WCHAR *swap_path_end;
+
+                /* We swap in the same folder as the destination to be
+                 * reasonably sure that it's on the same volume. Note that
+                 * we're avoiding GetTempFileNameW as it will fail on long
+                 * paths. */
+
+                sys_memcpy(swap_path, (WCHAR*)new_path->data, new_path->size);
+                swap_path_end = swap_path + PATH_LENGTH(new_path);
+
+                while(!IS_SLASH(*swap_path_end)) {
+                    ASSERT(swap_path_end > swap_path);
+                    swap_path_end--;
+                }
+
+                StringCchPrintfW(&swap_path_end[1], 64, L"erl-%lx-%llx.tmp",
+                    GetCurrentProcessId(), unique_counter);
+                InterlockedIncrement64(&unique_counter);
+            }
+
+            if(MoveFileExW((WCHAR*)new_path->data, swap_path, MOVEFILE_REPLACE_EXISTING)) {
+                if(MoveFileExW((WCHAR*)old_path->data, (WCHAR*)new_path->data, move_flags)) {
+                    last_error = ERROR_SUCCESS;
+                    DeleteFileW(swap_path);
+                } else {
+                    last_error = GetLastError();
+                    MoveFileW(swap_path, (WCHAR*)new_path->data);
+                }
+            } else {
+                last_error = GetLastError();
+                DeleteFileW(swap_path);
+            }
+
+            enif_free(swap_path);
+
+            return windows_to_posix_errno(last_error);
+        }
+
+        return EEXIST;
+    }
+
+    return windows_to_posix_errno(last_error);
+}
+
+posix_errno_t efile_make_hard_link(const efile_path_t *existing_path, const efile_path_t *new_path) {
+    ASSERT_PATH_FORMAT(existing_path);
+    ASSERT_PATH_FORMAT(new_path);
+
+    if(!CreateHardLinkW((WCHAR*)new_path->data, (WCHAR*)existing_path->data, NULL)) {
+        return windows_to_posix_errno(GetLastError());
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_make_soft_link(const efile_path_t *existing_path, const efile_path_t *new_path) {
+    DWORD link_flags;
+
+    ASSERT_PATH_FORMAT(existing_path);
+    ASSERT_PATH_FORMAT(new_path);
+
+    if(has_file_attributes(existing_path, FILE_ATTRIBUTE_DIRECTORY)) {
+        link_flags = SYMBOLIC_LINK_FLAG_DIRECTORY;
+    } else {
+        link_flags = 0;
+    }
+
+    if(!CreateSymbolicLinkW((WCHAR*)new_path->data, (WCHAR*)existing_path->data, link_flags)) {
+        return windows_to_posix_errno(GetLastError());
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_make_dir(const efile_path_t *path) {
+    ASSERT_PATH_FORMAT(path);
+
+    if(!CreateDirectoryW((WCHAR*)path->data, NULL)) {
+        return windows_to_posix_errno(GetLastError());
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_del_file(const efile_path_t *path) {
+    ASSERT_PATH_FORMAT(path);
+
+    if(!DeleteFileW((WCHAR*)path->data)) {
+        DWORD last_error = GetLastError();
+
+        switch(last_error) {
+        case ERROR_INVALID_NAME:
+            /* Attempted to delete a device or similar. */
+            return EACCES;
+        case ERROR_ACCESS_DENIED:
+            /* Windows NT reports removing a directory as EACCES instead of
+             * EPERM. */
+            if(has_file_attributes(path, FILE_ATTRIBUTE_DIRECTORY)) {
+                return EPERM;
+            }
+            break;
+        }
+
+        return windows_to_posix_errno(last_error);
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_del_dir(const efile_path_t *path) {
+    ASSERT_PATH_FORMAT(path);
+
+    if(!RemoveDirectoryW((WCHAR*)path->data)) {
+        DWORD last_error = GetLastError();
+
+        if(last_error == ERROR_DIRECTORY) {
+            return ENOTDIR;
+        }
+
+        return windows_to_posix_errno(last_error);
+    }
+
+    return 0;
+}
+
+posix_errno_t efile_set_cwd(const efile_path_t *path) {
+    const WCHAR *path_start;
+
+    ASSERT_PATH_FORMAT(path);
+
+    /* We have to use _wchdir since that's the only function that updates the
+     * per-drive working directory, but it naively assumes that all paths
+     * starting with \\ are UNC paths, so we have to skip the \\?\-prefix. */
+    path_start = (WCHAR*)path->data + LP_PREFIX_LENGTH;
+
+    if(_wchdir(path_start)) {
+        return windows_to_posix_errno(GetLastError());
+    }
+
+    return 0;
+}
+
+static int is_valid_drive(int device_index) {
+    WCHAR drive_path[4] = {L'?', L':', L'\\', L'\0'};
+
+    if(device_index == 0) {
+        /* Default drive; always valid. */
+        return 1;
+    } else if(device_index > (L'Z' - L'A' + 1)) {
+        return 0;
+    }
+
+    drive_path[0] = device_index + L'A' - 1;
+
+    switch(GetDriveTypeW(drive_path)) {
+    case DRIVE_NO_ROOT_DIR:
+    case DRIVE_UNKNOWN:
+        return 0;
+    }
+
+    return 1;
+}
+
+posix_errno_t efile_get_device_cwd(ErlNifEnv *env, int device_index, ERL_NIF_TERM *result) {
+    ErlNifBinary result_bin;
+
+    /* _wgetdcwd might crash the entire emulator on debug builds since the CRT
+     * invalid parameter handler asserts if passed a non-existent drive (Or
+     * simply one that has been unmounted), so we check it ourselves to avoid
+     * that. */
+    if(!is_valid_drive(device_index)) {
+        return EACCES;
+    }
+
+    if(!enif_alloc_binary(MAX_PATH * sizeof(WCHAR), &result_bin)) {
+        return ENOMEM;
+    }
+
+    if(_wgetdcwd(device_index, (WCHAR*)result_bin.data, MAX_PATH) == NULL) {
+        enif_release_binary(&result_bin);
+        return EACCES;
+    }
+
+    if(!normalize_path_result(&result_bin)) {
+        enif_release_binary(&result_bin);
+        return ENOMEM;
+    }
+
+    (*result) = enif_make_binary(env, &result_bin);
+
+    return 0;
+}
+
+posix_errno_t efile_get_cwd(ErlNifEnv *env, ERL_NIF_TERM *result) {
+    return efile_get_device_cwd(env, 0, result);
+}
+
+posix_errno_t efile_altname(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result) {
+    ErlNifBinary result_bin;
+
+    ASSERT_PATH_FORMAT(path);
+
+    if(is_path_root(path)) {
+        /* Root paths can't be queried so we'll just return them as they are. */
+        if(!enif_alloc_binary(path->size, &result_bin)) {
+            return ENOMEM;
+        }
+
+        sys_memcpy(result_bin.data, path->data, path->size);
+    } else {
+        WIN32_FIND_DATAW data;
+        HANDLE handle;
+
+        WCHAR *name_buffer;
+        int name_length;
+
+        /* Reject path wildcards. */
+        if(wcspbrk(&((const WCHAR*)path->data)[4], L"?*")) {
+            return ENOENT;
+        }
+
+        handle = FindFirstFileW((const WCHAR*)path->data, &data);
+
+        if(handle == INVALID_HANDLE_VALUE) {
+            return windows_to_posix_errno(GetLastError());
+        }
+
+        FindClose(handle);
+
+        name_length = wcslen(data.cAlternateFileName);
+
+        if(name_length > 0) {
+            name_buffer = data.cAlternateFileName;
+        } else {
+            name_length = wcslen(data.cFileName);
+            name_buffer = data.cFileName;
+        }
+
+        /* Include NUL-terminator; it will be removed after normalization. */
+        name_length += 1;
+
+        if(!enif_alloc_binary(name_length * sizeof(WCHAR), &result_bin)) {
+            return ENOMEM;
+        }
+
+        sys_memcpy(result_bin.data, name_buffer, name_length * sizeof(WCHAR));
+    }
+
+    if(!normalize_path_result(&result_bin)) {
+        enif_release_binary(&result_bin);
+        return ENOMEM;
+    }
+
+    (*result) = enif_make_binary(env, &result_bin);
+
+    return 0;
+}
+
+static int windows_to_posix_errno(DWORD last_error) {
+    switch(last_error) {
+    case ERROR_SUCCESS:
+        return 0;
+    case ERROR_INVALID_FUNCTION:
+    case ERROR_INVALID_DATA:
+    case ERROR_INVALID_PARAMETER:
+    case ERROR_INVALID_TARGET_HANDLE:
+    case ERROR_INVALID_CATEGORY:
+    case ERROR_NEGATIVE_SEEK:
+        return EINVAL;
+    case ERROR_DIR_NOT_EMPTY:
+        return EEXIST;
+    case ERROR_BAD_FORMAT:
+        return ENOEXEC;
+    case ERROR_PATH_NOT_FOUND:
+    case ERROR_FILE_NOT_FOUND:
+    case ERROR_NO_MORE_FILES:
+        return ENOENT;
+    case ERROR_TOO_MANY_OPEN_FILES:
+        return EMFILE;
+    case ERROR_ACCESS_DENIED:
+    case ERROR_INVALID_ACCESS:
+    case ERROR_CURRENT_DIRECTORY:
+    case ERROR_SHARING_VIOLATION:
+    case ERROR_LOCK_VIOLATION:
+    case ERROR_INVALID_PASSWORD:
+    case ERROR_DRIVE_LOCKED:
+        return EACCES;
+    case ERROR_INVALID_HANDLE:
+        return EBADF;
+    case ERROR_NOT_ENOUGH_MEMORY:
+    case ERROR_OUTOFMEMORY:
+    case ERROR_OUT_OF_STRUCTURES:
+        return ENOMEM;
+    case ERROR_INVALID_DRIVE:
+    case ERROR_BAD_UNIT:
+    case ERROR_NOT_READY:
+    case ERROR_REM_NOT_LIST:
+    case ERROR_DUP_NAME:
+    case ERROR_BAD_NETPATH:
+    case ERROR_NETWORK_BUSY:
+    case ERROR_DEV_NOT_EXIST:
+    case ERROR_BAD_NET_NAME:
+        return ENXIO;
+    case ERROR_NOT_SAME_DEVICE:
+        return EXDEV;
+    case ERROR_WRITE_PROTECT:
+        return EROFS;
+    case ERROR_BAD_LENGTH:
+    case ERROR_BUFFER_OVERFLOW:
+        return E2BIG;
+    case ERROR_SEEK:
+    case ERROR_SECTOR_NOT_FOUND:
+        return ESPIPE;
+    case ERROR_NOT_DOS_DISK:
+        return ENODEV;
+    case ERROR_GEN_FAILURE:
+        return ENODEV;
+    case ERROR_SHARING_BUFFER_EXCEEDED:
+    case ERROR_NO_MORE_SEARCH_HANDLES:
+        return EMFILE;
+    case ERROR_HANDLE_EOF:
+    case ERROR_BROKEN_PIPE:
+        return EPIPE;
+    case ERROR_HANDLE_DISK_FULL:
+    case ERROR_DISK_FULL:
+        return ENOSPC;
+    case ERROR_NOT_SUPPORTED:
+        return ENOTSUP;
+    case ERROR_FILE_EXISTS:
+    case ERROR_ALREADY_EXISTS:
+    case ERROR_CANNOT_MAKE:
+        return EEXIST;
+    case ERROR_ALREADY_ASSIGNED:
+        return EBUSY;
+    case ERROR_NO_PROC_SLOTS:
+        return EAGAIN;
+    case ERROR_CANT_RESOLVE_FILENAME:
+        return EMLINK;
+    case ERROR_PRIVILEGE_NOT_HELD:
+        return EPERM;
+    case ERROR_ARENA_TRASHED:
+    case ERROR_INVALID_BLOCK:
+    case ERROR_BAD_ENVIRONMENT:
+    case ERROR_BAD_COMMAND:
+    case ERROR_CRC:
+    case ERROR_OUT_OF_PAPER:
+    case ERROR_READ_FAULT:
+    case ERROR_WRITE_FAULT:
+    case ERROR_WRONG_DISK:
+    case ERROR_NET_WRITE_FAULT:
+        return EIO;
+    default: /* not to do with files I expect. */
+        return EIO;
+    }
+}
-- 
cgit v1.2.3


From 7769c7a29806cd2c7a9bbff76b2fe32f5401e9ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 9 Oct 2017 20:56:25 +0200
Subject: Remove efile_SUITE:async_dist

This test is irrelevant as the new implementation doesn't use async
threads.
---
 erts/emulator/test/efile_SUITE.erl | 86 ++------------------------------------
 1 file changed, 3 insertions(+), 83 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl
index 08d5597d78..98d7aa739e 100644
--- a/erts/emulator/test/efile_SUITE.erl
+++ b/erts/emulator/test/efile_SUITE.erl
@@ -19,12 +19,9 @@
 
 -module(efile_SUITE).
 -export([all/0, suite/0]).
--export([async_dist/1,
-         iter_max_files/1,
-         proc_zero_sized_files/1
-        ]).
+-export([iter_max_files/1, proc_zero_sized_files/1]).
 
--export([do_iter_max_files/2, do_async_dist/1]).
+-export([do_iter_max_files/2]).
 
 -include_lib("common_test/include/ct.hrl").
 -include_lib("stdlib/include/assert.hrl").
@@ -32,84 +29,7 @@
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
 all() -> 
-    [iter_max_files, async_dist, proc_zero_sized_files].
-
-do_async_dist(Dir) ->
-    X = 100,
-    AT = erlang:system_info(thread_pool_size),
-    Keys = file_keys(Dir,AT*X,[],[]),
-    Tab = ets:new(x,[ordered_set]),
-    [ ets:insert(Tab,{N,0}) || N <- lists:seq(0,AT-1) ],
-    [ ets:update_counter(Tab,(N rem AT),1) || N <- Keys ],
-    Res = [ V || {_,V} <- ets:tab2list(Tab) ],
-    ets:delete(Tab),
-    {Res, sdev(Res)/X}.
-
-sdev(List) ->
-    Len = length(List),
-    Mean = lists:sum(List)/Len,
-    math:sqrt(lists:sum([ (X - Mean) * (X - Mean) || X <- List ]) / Len).
-
-file_keys(_,0,FdList,FnList) ->
-    [ file:close(FD) || FD <- FdList ],
-    [ file:delete(FN) || FN <- FnList ],
-    [];
-file_keys(Dir,Num,FdList,FnList) ->
-    Name = "dummy"++integer_to_list(Num),
-    FN = filename:join([Dir,Name]),
-    case file:open(FN,[write,raw]) of
-        {ok,FD} ->
-            {file_descriptor,prim_file,{Port,_}} = FD,
-            <<X:32/integer-big>> = 
-            iolist_to_binary(erlang:port_control(Port,$K,[])),
-            [X | file_keys(Dir,Num-1,[FD|FdList],[FN|FnList])];
-        {error,_} ->
-            % Try freeing up FD's if there are any
-            case FdList of 
-                [] ->
-                    exit({cannot_open_file,FN});
-                _ ->
-                    [ file:close(FD) || FD <- FdList ],
-                    [ file:delete(F) || F <- FnList ],
-                    file_keys(Dir,Num,[],[])
-            end
-    end.
-
-%% Check that the distribution of files over async threads is fair
-async_dist(Config) when is_list(Config) ->
-    DataDir = proplists:get_value(data_dir,Config),
-    Dir = filename:dirname(code:which(?MODULE)),
-    AsyncSizes = [7,10,100,255,256,64,63,65],
-    Max = 0.5,
-
-    lists:foreach(fun(Size) ->
-                          {ok,Node} = 
-                          test_server:start_node
-                          (test_iter_max_files,slave,
-                           [{args,
-                             "+A "++integer_to_list(Size)++
-                             " -pa " ++ Dir}]),
-                          {Distr,SD} = rpc:call(Node,?MODULE,do_async_dist,
-                                                [DataDir]),
-                          test_server:stop_node(Node),
-                          if
-                              SD > Max ->
-                                  io:format("Bad async queue distribution for "
-                                            "~p async threads:~n"
-                                            "    Standard deviation is ~p~n"
-                                            "    Key distribution:~n    ~lp~n",
-                                            [Size,SD,Distr]),
-                                  exit({bad_async_dist,Size,SD,Distr});
-                              true ->
-                                  io:format("OK async queue distribution for "
-                                            "~p async threads:~n"
-                                            "    Standard deviation is ~p~n"
-                                            "    Key distribution:~n    ~lp~n",
-                                            [Size,SD,Distr]),
-                                  ok
-                          end
-                  end, AsyncSizes),
-    ok.
+    [iter_max_files, proc_zero_sized_files].
 
 %%
 %% Open as many files as possible. Do this several times and check 
-- 
cgit v1.2.3


From 08901d99137014bd53b999a9fbb0aac893d854e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 25 Oct 2017 16:29:37 +0200
Subject: Fix dirty_*if_SUITE after file rewrite

Code loading is done through dirty IO now, causing the
dirty_scheduler_exit tests to fail as they block their own progress
by invoking erts_debug:dirty_io(wait, _); the spawned processes
will exit normally before we have a chance to kill them.

To get around this, we perform a dry run to ensure that all required
code is loaded. It isn't particularly pretty (or fast) but it saves
us the hassle of maintaining a module list (cf. embedded mode).
---
 erts/emulator/test/dirty_bif_SUITE.erl | 20 ++++++++++++--------
 erts/emulator/test/dirty_nif_SUITE.erl | 20 ++++++++++++--------
 2 files changed, 24 insertions(+), 16 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/dirty_bif_SUITE.erl b/erts/emulator/test/dirty_bif_SUITE.erl
index 981ec4d48d..f1279a0bd2 100644
--- a/erts/emulator/test/dirty_bif_SUITE.erl
+++ b/erts/emulator/test/dirty_bif_SUITE.erl
@@ -230,6 +230,11 @@ dirty_scheduler_exit(Config) when is_list(Config) ->
     {ok, Node} = start_node(Config, "+SDio 1"),
     [ok] = mcall(Node,
                  [fun() ->
+                          %% Perform a dry run to ensure that all required code
+                          %% is loaded. Otherwise the test will fail since code
+                          %% loading is done through dirty IO and it won't make
+                          %% any progress during this test.
+                          _DryRun = test_dirty_scheduler_exit(),
 			  Start = erlang:monotonic_time(millisecond),
                           ok = test_dirty_scheduler_exit(),
 			  End = erlang:monotonic_time(millisecond),
@@ -250,19 +255,18 @@ test_dse(N,Pids) ->
     test_dse(N-1,[Pid|Pids]).
 
 kill_dse([],Killed) ->
-    wait_dse(Killed);
+    wait_dse(Killed, ok);
 kill_dse([Pid|Pids],AlreadyKilled) ->
     exit(Pid,kill),
     kill_dse(Pids,[Pid|AlreadyKilled]).
 
-wait_dse([]) ->
-    ok;
-wait_dse([Pid|Pids]) ->
+wait_dse([], Result) ->
+    Result;
+wait_dse([Pid|Pids], Result) ->
     receive
-        {'EXIT',Pid,Reason} ->
-	    killed = Reason
-    end,
-    wait_dse(Pids).
+        {'EXIT', Pid, killed} -> wait_dse(Pids, Result);
+        {'EXIT', Pid, _Other} -> wait_dse(Pids, failed)
+    end.
 
 dirty_call_while_terminated(Config) when is_list(Config) ->
     Me = self(),
diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl
index 13806fd5c4..711ecbfe95 100644
--- a/erts/emulator/test/dirty_nif_SUITE.erl
+++ b/erts/emulator/test/dirty_nif_SUITE.erl
@@ -151,6 +151,11 @@ dirty_scheduler_exit(Config) when is_list(Config) ->
     [ok] = mcall(Node,
                  [fun() ->
                           ok = erlang:load_nif(NifLib, []),
+                          %% Perform a dry run to ensure that all required code
+                          %% is loaded. Otherwise the test will fail since code
+                          %% loading is done through dirty IO and it won't make
+                          %% any progress during this test.
+                          _DryRun = test_dirty_scheduler_exit(),
 			  Start = erlang:monotonic_time(millisecond),
                           ok = test_dirty_scheduler_exit(),
 			  End = erlang:monotonic_time(millisecond),
@@ -171,19 +176,18 @@ test_dse(N,Pids) ->
     test_dse(N-1,[Pid|Pids]).
 
 kill_dse([],Killed) ->
-    wait_dse(Killed);
+    wait_dse(Killed, ok);
 kill_dse([Pid|Pids],AlreadyKilled) ->
     exit(Pid,kill),
     kill_dse(Pids,[Pid|AlreadyKilled]).
 
-wait_dse([]) ->
-    ok;
-wait_dse([Pid|Pids]) ->
+wait_dse([], Result) ->
+    Result;
+wait_dse([Pid|Pids], Result) ->
     receive
-        {'EXIT',Pid,Reason} ->
-	    killed = Reason
-    end,
-    wait_dse(Pids).
+        {'EXIT', Pid, killed} -> wait_dse(Pids, Result);
+        {'EXIT', Pid, _Other} -> wait_dse(Pids, failed)
+    end.
 
 dirty_call_while_terminated(Config) when is_list(Config) ->
     Me = self(),
-- 
cgit v1.2.3


From b5149c7073896f3ddb8c7995b2b0992007a1f0e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 25 Oct 2017 19:16:45 +0200
Subject: Tighten timings in dirty_*if_SUITE:dirty_scheduler_exit

There doesn't seem to be any science behind the long delays, and
the (newly introduced) dry run forces us to eat them twice, so
they've been shortened to more reasonable values.
---
 erts/emulator/test/dirty_bif_SUITE.erl                    |  2 +-
 erts/emulator/test/dirty_nif_SUITE.erl                    | 12 ++++++------
 erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/dirty_bif_SUITE.erl b/erts/emulator/test/dirty_bif_SUITE.erl
index f1279a0bd2..c8f0cbf42d 100644
--- a/erts/emulator/test/dirty_bif_SUITE.erl
+++ b/erts/emulator/test/dirty_bif_SUITE.erl
@@ -251,7 +251,7 @@ test_dse(0,Pids) ->
     timer:sleep(100),
     kill_dse(Pids,[]);
 test_dse(N,Pids) ->
-    Pid = spawn_link(fun () -> erts_debug:dirty_io(wait, 5000) end),
+    Pid = spawn_link(fun () -> erts_debug:dirty_io(wait, 1000) end),
     test_dse(N-1,[Pid|Pids]).
 
 kill_dse([],Killed) ->
diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl
index 711ecbfe95..d36113c3e3 100644
--- a/erts/emulator/test/dirty_nif_SUITE.erl
+++ b/erts/emulator/test/dirty_nif_SUITE.erl
@@ -291,9 +291,9 @@ access_dirty_heap(Dirty, RGL, N, R) ->
 %% dirty NIF where the main lock is needed for that access do not get
 %% blocked. Each test passes its pid to dirty_sleeper, which sends a
 %% 'ready' message when it's running on a dirty scheduler and just before
-%% it starts a 6 second sleep. When it receives the message, it verifies
+%% it starts a 2 second sleep. When it receives the message, it verifies
 %% that access to the dirty process is as it expects.  After the dirty
-%% process finishes its 6 second sleep but before it returns from the dirty
+%% process finishes its 2 second sleep but before it returns from the dirty
 %% scheduler, it sends a 'done' message. If the tester already received
 %% that message, the test fails because it means attempting to access the
 %% dirty process waited for that process to return to a regular scheduler,
@@ -357,7 +357,7 @@ dirty_process_trace(Config) when is_list(Config) ->
 			      error(missing_trace_return_message)
 		      end
 	      after
-		  6500 ->
+		  2500 ->
 		      error(missing_done_message)
 	      end,
 	      ok
@@ -384,7 +384,7 @@ code_purge(Config) when is_list(Config) ->
     Start = erlang:monotonic_time(),
     {Pid1, Mon1} = spawn_monitor(fun () ->
 				       dirty_code_test:func(fun () ->
-								    %% Sleep for 6 seconds
+								    %% Sleep for 2 seconds
 								    %% in dirty nif...
 								    dirty_sleeper()
 							    end)
@@ -392,7 +392,7 @@ code_purge(Config) when is_list(Config) ->
     {module, dirty_code_test} = erlang:load_module(dirty_code_test, Bin),
     {Pid2, Mon2} = spawn_monitor(fun () ->
 				       dirty_code_test:func(fun () ->
-								    %% Sleep for 6 seconds
+								    %% Sleep for 2 seconds
 								    %% in dirty nif...
 								    dirty_sleeper()
 							    end)
@@ -494,7 +494,7 @@ test_dirty_process_access(Start, Test, Finish) ->
 			 ok
 		 end
 	 after
-	     3000 ->
+	     1000 ->
 		 error(timeout)
 	 end,
     ok = Finish(NifPid).
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
index 0321b9898f..2a8b999307 100644
--- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
+++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
@@ -217,9 +217,9 @@ dirty_sleeper(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     }
 
 #ifdef __WIN32__
-    Sleep(6000);
+    Sleep(2000);
 #else
-    sleep(6);
+    sleep(2);
 #endif
 
     if (argc == 1) {
-- 
cgit v1.2.3


From 186018a4ce8f3e5ad3f7c10959964d3b7669bb64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Fri, 27 Oct 2017 10:23:50 +0200
Subject: Skip efile_SUITE:iter_max_files on Windows

---
 erts/emulator/test/efile_SUITE.erl | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl
index 98d7aa739e..16d581a567 100644
--- a/erts/emulator/test/efile_SUITE.erl
+++ b/erts/emulator/test/efile_SUITE.erl
@@ -37,6 +37,12 @@ all() ->
 %%
 
 iter_max_files(Config) when is_list(Config) ->
+    case os:type() of
+        {win32, _} -> {skip, "Windows lacks a hard limit on file handles"};
+        _ -> iter_max_files_1(Config)
+    end.
+
+iter_max_files_1(Config) ->
     DataDir = proplists:get_value(data_dir,Config),
     TestFile = filename:join(DataDir, "existing_file"),
     N = 10,
-- 
cgit v1.2.3


From e40f9c8fac00476819d003bedbf899a52bbf09d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 1 Nov 2017 14:55:34 +0100
Subject: Remove all mention of efile_drv from DTrace docs

efile_drv is gone and so is the need for file-specific DTrace. The
new implementation works fine with the normal tracing mechanism so
there's nothing preventing anyone from making an erl_tracer nif
that forward these events to DTrace.
---
 erts/emulator/beam/erlang_dtrace.d | 66 --------------------------------------
 1 file changed, 66 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erlang_dtrace.d b/erts/emulator/beam/erlang_dtrace.d
index 237889e0f5..d0b10e0306 100644
--- a/erts/emulator/beam/erlang_dtrace.d
+++ b/erts/emulator/beam/erlang_dtrace.d
@@ -634,72 +634,6 @@ provider erlang {
      */
     probe aio_pool__get(char *, int);
 
-    /* Probes for efile_drv.c */
-
-    /**
-     * Entry into the efile_drv.c file I/O driver
-     *
-     * For a list of command numbers used by this driver, see the section
-     * "Guide to efile_drv.c probe arguments" in ../../../HOWTO/DTRACE.md.
-     * That section also contains explanation of the various integer and
-     * string arguments that may be present when any particular probe fires.
-     *
-     * NOTE: Not all Linux platforms (using SystemTap) can support
-     *       arguments beyond arg9.
-     *
-     *
-     * TODO: Adding the port string, args[10], is a pain.  Making that
-     *       port string available to all the other efile_drv.c probes
-     *       will be more pain.  Is the pain worth it?  If yes, then
-     *       add them everywhere else and grit our teeth.  If no, then
-     *       rip it out.
-     *
-     * @param thread-id number of the scheduler Pthread                   arg0
-     * @param tag number: {thread-id, tag} uniquely names a driver operation
-     * @param user-tag string                                             arg2
-     * @param command number                                              arg3
-     * @param string argument 1                                           arg4
-     * @param string argument 2                                           arg5
-     * @param integer argument 1                                          arg6
-     * @param integer argument 2                                          arg7
-     * @param integer argument 3                                          arg8
-     * @param integer argument 4                                          arg9
-     * @param port the port ID of the busy port                       args[10]
-     */
-    probe efile_drv__entry(int, int, char *, int, char *, char *,
-                           int64_t, int64_t, int64_t, int64_t, char *);
-
-    /**
-     * Entry into the driver's internal work function.  Computation here
-     * is performed by a async worker pool Pthread.
-     *
-     * @param thread-id number
-     * @param tag number
-     * @param command number
-     */
-    probe efile_drv__int_entry(int, int, int);
-
-    /**
-     * Return from the driver's internal work function.
-     *
-     * @param thread-id number
-     * @param tag number
-     * @param command number
-     */
-    probe efile_drv__int_return(int, int, int);
-
-    /**
-     * Return from the efile_drv.c file I/O driver
-     *
-     * @param thread-id number                                            arg0
-     * @param tag number                                                  arg1
-     * @param user-tag string                                             arg2
-     * @param command number                                              arg3
-     * @param Success? 1 is success, 0 is failure                         arg4
-     * @param If failure, the errno of the error.                         arg5
-     */
-    probe efile_drv__return(int, int, char *, int, int, int);
-
 
 /*
  * The set of probes called by the erlang tracer nif backend. In order
-- 
cgit v1.2.3


From 0278c9873ffc277c9a940d6f5cae1508a6685c7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 22 Nov 2017 16:27:53 +0100
Subject: Don't assume efile driver is present

---
 erts/emulator/test/ddll_SUITE.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/ddll_SUITE.erl b/erts/emulator/test/ddll_SUITE.erl
index 031b05790d..4998fc08be 100644
--- a/erts/emulator/test/ddll_SUITE.erl
+++ b/erts/emulator/test/ddll_SUITE.erl
@@ -775,7 +775,7 @@ errors(Config) when is_list(Config) ->
     {error, bad_driver_name} = erl_ddll:load_driver(Path, wrongname_drv),
 
     %% We assume that there is a statically linked driver named "ddll":
-    {error, linked_in_driver} = erl_ddll:unload_driver(efile),
+    {error, linked_in_driver} = erl_ddll:unload_driver(ram_file_drv),
     {error, not_loaded} = erl_ddll:unload_driver("__pucko_driver__"),
 
     case os:type() of
-- 
cgit v1.2.3


From fb99e17d04bddfcb43f54388c93302ea32d60b4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 4 Dec 2017 10:29:26 +0100
Subject: Fix process name resolution in lcnt results

---
 erts/emulator/beam/erl_bif_info.c |  2 +-
 erts/emulator/test/lcnt_SUITE.erl | 28 ++++++++++++++++++++++++++--
 2 files changed, 27 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 0547b4d75c..80adca0072 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -4578,7 +4578,7 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s
 static Eterm lcnt_pretty_print_lock_id(erts_lcnt_lock_info_t *info) {
     Eterm id = info->id;
 
-    if((info->flags & ERTS_LOCK_FLAGS_MASK_TYPE) == ERTS_LOCK_TYPE_PROCLOCK) {
+    if((info->flags & ERTS_LOCK_FLAGS_MASK_TYPE) == ERTS_LOCK_FLAGS_TYPE_PROCLOCK) {
         /* Use registered names as id's for process locks if available. Thread
          * progress is delayed since we may be running on a dirty scheduler. */
         ErtsThrPrgrDelayHandle delay_handle;
diff --git a/erts/emulator/test/lcnt_SUITE.erl b/erts/emulator/test/lcnt_SUITE.erl
index 504b9b54cf..4e52c2813c 100644
--- a/erts/emulator/test/lcnt_SUITE.erl
+++ b/erts/emulator/test/lcnt_SUITE.erl
@@ -28,14 +28,16 @@
      init_per_testcase/2, end_per_testcase/2]).
 
 -export(
-    [toggle_lock_counting/1, error_on_invalid_category/1, preserve_locks/1]).
+    [toggle_lock_counting/1, error_on_invalid_category/1, preserve_locks/1,
+     registered_processes/1, registered_db_tables/1]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
      {timetrap, {seconds, 10}}].
 
 all() ->
-    [toggle_lock_counting, error_on_invalid_category, preserve_locks].
+    [toggle_lock_counting, error_on_invalid_category, preserve_locks,
+     registered_processes, registered_db_tables].
 
 init_per_suite(Config) ->
     case erlang:system_info(lock_counting) of
@@ -154,3 +156,25 @@ preserve_locks(Config) when is_list(Config) ->
 error_on_invalid_category(Config) when is_list(Config) ->
     {error, badarg, q_invalid} = erts_debug:lcnt_control(mask, [q_invalid]),
     ok.
+
+registered_processes(Config) when is_list(Config) ->
+    %% There ought to be at least one registered process (init/code_server)
+    erts_debug:lcnt_control(mask, [process]),
+    [_, {locks, ProcLocks}] = erts_debug:lcnt_collect(),
+    true = lists:any(
+        fun
+            ({proc_main, RegName, _, _}) when is_atom(RegName) -> true;
+            (_Lock) -> false
+        end, ProcLocks),
+    ok.
+
+registered_db_tables(Config) when is_list(Config) ->
+    %% There ought to be at least one registered table (code)
+    erts_debug:lcnt_control(mask, [db]),
+    [_, {locks, DbLocks}] = erts_debug:lcnt_collect(),
+    true = lists:any(
+        fun
+            ({db_tab, RegName, _, _}) when is_atom(RegName) -> true;
+            (_Lock) -> false
+        end, DbLocks),
+    ok.
-- 
cgit v1.2.3


From 3ba4c2551e2ee06f2df9784f097484f8621820f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 6 Dec 2017 08:48:00 +0100
Subject: Add translation for ERROR_INVALID_NAME

Most functions return this if they're given an invalid path, eg. if
they contain "<" or ">". ENOENT may seem like a strange translation,
but that's what open(2) returns when fed garbage, so we'll roll with
that.
---
 erts/emulator/nifs/win32/win_prim_file.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/nifs/win32/win_prim_file.c b/erts/emulator/nifs/win32/win_prim_file.c
index 9f993f1d24..9b79182f2c 100644
--- a/erts/emulator/nifs/win32/win_prim_file.c
+++ b/erts/emulator/nifs/win32/win_prim_file.c
@@ -1346,6 +1346,7 @@ static int windows_to_posix_errno(DWORD last_error) {
     case ERROR_PATH_NOT_FOUND:
     case ERROR_FILE_NOT_FOUND:
     case ERROR_NO_MORE_FILES:
+    case ERROR_INVALID_NAME:
         return ENOENT;
     case ERROR_TOO_MANY_OPEN_FILES:
         return EMFILE;
-- 
cgit v1.2.3


From d577018a76db8382421bcc6e7ed2dd2717867302 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 6 Dec 2017 16:18:31 +0100
Subject: "Fix" tracing of dirty NIFs on debug builds

When a dirty NIF is executed a "schedule in" trace event is
generated, which may in turn result in a generic system task being
created, causing the process to be scheduled out as it can't run
dirty with pending tasks.

This is usually fine since said system task is seldom created, but
ERTS_FORCE_ENIF_SEND_DELAY was de-facto always on for debug builds,
causing the process to bounce between dirty and normal schedulers
forever.

This commit is not a complete fix and it can go off the rails even
on normal builds; if there's a lot of dirty jobs lined up and the
receiver's msgq lock happens to be busy at the wrong time, the
additional trace messages generated through this will hammer the
lock and keep everything bouncing.
---
 erts/emulator/beam/erl_nif.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index abb490cf9c..0a5fe026db 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -562,7 +562,7 @@ void enif_clear_env(ErlNifEnv* env)
 
 #ifdef DEBUG
 static int enif_send_delay = 0;
-#define ERTS_FORCE_ENIF_SEND_DELAY() (enif_send_delay++ % 2 == 0)
+#define ERTS_FORCE_ENIF_SEND_DELAY() (enif_send_delay++ % 32 == 0)
 #else
 #ifdef ERTS_PROC_LOCK_OWN_IMPL
 #define ERTS_FORCE_ENIF_SEND_DELAY() 0
-- 
cgit v1.2.3


From 7bbfd17a838e5c056b4ed8ff13af5635fc192969 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 12 Dec 2017 14:54:29 +0100
Subject: erts: Remove some straggling ERTS_SMP

---
 erts/emulator/beam/bif.c             | 2 --
 erts/emulator/beam/break.c           | 2 +-
 erts/emulator/hipe/hipe_mkliterals.c | 1 -
 3 files changed, 1 insertion(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 50699eac31..0c88c39cbc 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -247,10 +247,8 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
 		 */
 		state = erts_atomic32_read_acqb(&BIF_P->state);
 		if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
-#ifdef ERTS_SMP
 		    if (state & ERTS_PSFLG_PENDING_EXIT)
 			erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
-#endif
 		    ERTS_BIF_EXITED(BIF_P);
 		}
 		BIF_RET(am_true);
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 2bfb481771..2cff05c7f3 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -358,7 +358,7 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p)
 static void
 print_garb_info(fmtfn_t to, void *to_arg, Process* p)
 {
-    /* ERTS_SMP: A scheduler is probably concurrently doing gc... */
+    /* A scheduler is probably concurrently doing gc... */
     if (!ERTS_IS_CRASH_DUMPING)
       return;
     erts_print(to, to_arg, "New heap start: %bpX\n", p->heap);
diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c
index 84889b3376..d115dfaeea 100644
--- a/erts/emulator/hipe/hipe_mkliterals.c
+++ b/erts/emulator/hipe/hipe_mkliterals.c
@@ -471,7 +471,6 @@ static const struct rts_param rts_params[] = {
       0
 #endif
     },
-    /* This parameter is always defined, but its value depends on ERTS_SMP. */
     { 19, "MSG_MESSAGE",
       1, offsetof(struct erl_mesg, m[0])
     },
-- 
cgit v1.2.3


From a72a33e51218b1584f87629a9725665c22f9eb09 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 12 Dec 2017 21:58:25 +0100
Subject: erts: Trim driver_SUITE:smp_select for slow valgrind

Symptom: "Closing pipe in state Waiting. Event lost?"

Snake oil:
Do erlang:yield() instead of busy spinning in "Waiting" state.
---
 erts/emulator/test/driver_SUITE.erl              |  5 ++++-
 erts/emulator/test/driver_SUITE_data/chkio_drv.c | 17 ++++++++++-------
 2 files changed, 14 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 6810729285..c31ceb4d4b 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -1673,7 +1673,10 @@ smp_select0(Config) ->
 smp_select_loop(_, 0) ->
     ok;
 smp_select_loop(Port, N) ->
-    "ok" = erlang:port_control(Port, ?CHKIO_SMP_SELECT, []),
+    case erlang:port_control(Port, ?CHKIO_SMP_SELECT, []) of
+        "yield" -> erlang:yield();
+        "ok" -> ok
+    end,
     receive
         stop -> 
             io:format("Worker ~p stopped with ~p laps left\n",[self(), N]), 
diff --git a/erts/emulator/test/driver_SUITE_data/chkio_drv.c b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
index 8e5e81665c..e55b9e10ba 100644
--- a/erts/emulator/test/driver_SUITE_data/chkio_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
@@ -1225,7 +1225,6 @@ chkio_drv_control(ErlDrvData drv_data,
 	break;
     }
     case CHKIO_SMP_SELECT: {
-	int rounds = 1; /*rand(); */
 	ChkioSmpSelect* pip = (ChkioSmpSelect*) cddp->test_data;
 	if (pip == NULL) {
 	    erl_drv_mutex_lock(smp_pipes_mtx);
@@ -1242,7 +1241,8 @@ chkio_drv_control(ErlDrvData drv_data,
 	    }
 	    erl_drv_mutex_unlock(smp_pipes_mtx);	    
 	}
-	while (rounds--) {
+        res_str = NULL;
+	{
 	    int op = rand_r(&pip->rand_state);
 	    switch (pip->state) {
 	    case Closed: {
@@ -1252,7 +1252,6 @@ chkio_drv_control(ErlDrvData drv_data,
 		    fcntl(fds[0], F_SETFL, flags|O_NONBLOCK) < 0) {
 
 		    driver_failure_posix(cddp->port, errno);
-		    rounds = 0;
 		    break;
 		}
 		TRACEF(("%T: Created pipe [%d->%d]\n", cddp->id, fds[1], fds[0]));
@@ -1332,7 +1331,9 @@ chkio_drv_control(ErlDrvData drv_data,
 		    pip->next_write++;		    
 		}
 		break;
-	    case Waiting:
+            case Waiting:
+                res_str = "yield";
+                res_len = -1;
 		break;
 	    default:
 		fprintf(stderr, "Strange state %d\n", pip->state);
@@ -1348,9 +1349,11 @@ chkio_drv_control(ErlDrvData drv_data,
 	    else {
 		cddp->test_data = pip;
 	    }
-	}	
-	res_str = "ok";
-	res_len = -1;
+	}
+        if (!res_str) {
+            res_str = "ok";
+            res_len = -1;
+        }
 	break;
     }
     case CHKIO_DRV_USE:
-- 
cgit v1.2.3


From 7f6ac587f47bd115604c7c9d1504e66cafffd265 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 28 Nov 2017 14:17:24 +0100
Subject: Redirect system_flag(scheduler_wall_time,_) to kernel_refc

---
 erts/emulator/beam/atom.names     |  1 +
 erts/emulator/beam/bif.c          | 32 +++++++++++++++++++-------------
 erts/emulator/beam/bif.tab        |  2 ++
 erts/emulator/beam/erl_bif_info.c |  2 +-
 4 files changed, 23 insertions(+), 14 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index cef633bd93..cb4fab51f1 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -641,6 +641,7 @@ atom suspending
 atom sys_misc
 atom system
 atom system_error
+atom system_flag_scheduler_wall_time
 atom system_limit
 atom system_version
 atom system_architecture
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 50699eac31..6684803bb8 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -59,6 +59,7 @@ Export *erts_convert_time_unit_trap = NULL;
 static Export *await_msacc_mod_trap = NULL;
 static erts_atomic32_t msacc;
 
+static Export *system_flag_scheduler_wall_time_trap;
 static Export *await_sched_wall_time_mod_trap;
 static erts_atomic32_t sched_wall_time;
 
@@ -4496,7 +4497,7 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2)
 	BIF_ERROR(BIF_P, BADARG);
     }
 }
-    
+
 BIF_RETTYPE system_flag_2(BIF_ALIST_2)    
 {
     Sint n;
@@ -4714,17 +4715,9 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 
 	BIF_RET(am_true);
     } else if (BIF_ARG_1 == am_scheduler_wall_time) {
-	if (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) {
-	    erts_aint32_t new = BIF_ARG_2 == am_true ? 1 : 0;
-	    erts_aint32_t old = erts_atomic32_xchg_nob(&sched_wall_time,
-							   new);
-	    Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new, 0, 0);
-	    ASSERT(is_value(ref));
-	    BIF_TRAP2(await_sched_wall_time_mod_trap,
-		      BIF_P,
-		      ref,
-		      old ? am_true : am_false);
-	}
+	if (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false)
+            BIF_TRAP1(system_flag_scheduler_wall_time_trap,
+                      BIF_P, BIF_ARG_2);
     } else if (BIF_ARG_1 == am_dirty_cpu_schedulers_online) {
 	Sint old_no;
 	if (!is_small(BIF_ARG_2))
@@ -4836,6 +4829,17 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
     BIF_ERROR(BIF_P, BADARG);
 }
 
+BIF_RETTYPE erts_internal_scheduler_wall_time_1(BIF_ALIST_1)
+{
+    erts_aint32_t new = BIF_ARG_1 == am_true ? 1 : 0;
+    erts_aint32_t old = erts_atomic32_xchg_nob(&sched_wall_time,
+                                               new);
+    Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new, 0, 0);
+    ASSERT(is_value(ref));
+    BIF_TRAP2(await_sched_wall_time_mod_trap,
+              BIF_P, ref, old ? am_true : am_false);
+}
+
 /**********************************************************************/
 
 BIF_RETTYPE phash_2(BIF_ALIST_2)
@@ -5094,8 +5098,10 @@ void erts_init_bif(void)
     await_proc_exit_trap = erts_export_put(am_erlang,am_await_proc_exit,3);
     await_port_send_result_trap
 	= erts_export_put(am_erts_internal, am_await_port_send_result, 3);
+    system_flag_scheduler_wall_time_trap
+        = erts_export_put(am_erts_internal, am_system_flag_scheduler_wall_time, 1);
     await_sched_wall_time_mod_trap
-        = erts_export_put(am_erlang, am_await_sched_wall_time_modifications, 2);
+        = erts_export_put(am_erts_internal, am_await_sched_wall_time_modifications, 2);
     await_msacc_mod_trap
 	= erts_export_put(am_erts_internal, am_await_microstate_accounting_modifications, 3);
 
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 1bd4acbf95..33f6eef652 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -185,6 +185,8 @@ bif erts_internal:system_check/1
 
 bif erts_internal:release_literal_area_switch/0
 
+bif erts_internal:scheduler_wall_time/1
+
 # inet_db support
 bif erlang:port_set_data/2
 bif erlang:port_get_data/1
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 9d05680723..b9d8f9d6e9 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -4758,7 +4758,7 @@ erts_bif_info_init(void)
     alloc_info_trap = erts_export_put(am_erlang, am_alloc_info, 1);
     alloc_sizes_trap = erts_export_put(am_erlang, am_alloc_sizes, 1);
     gather_sched_wall_time_res_trap
-	= erts_export_put(am_erlang, am_gather_sched_wall_time_result, 1);
+	= erts_export_put(am_erts_internal, am_gather_sched_wall_time_result, 1);
     gather_gc_info_res_trap
 	= erts_export_put(am_erlang, am_gather_gc_info_result, 1);
     gather_io_bytes_trap
-- 
cgit v1.2.3


From abb61a8b4039bb9059cb6a8cf78feaad5f2cdc28 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 8 Dec 2017 18:52:32 +0100
Subject: erts: Refactor carrier dealloc migration

by adding a dedicated 'homecoming_dd_block' to use in dd-queue.

+ Preparation for dd-queue-migration of non-empty carriers.

+ Get rid of ugly hack where blk->carrier pointer is overwritten
  by dd-queue and then have to be restored.
---
 erts/emulator/beam/erl_alloc_util.c |  23 +++-----
 erts/emulator/beam/erl_alloc_util.h | 112 ++++++++++++++++++++----------------
 2 files changed, 69 insertions(+), 66 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 230ca6ccbb..f0597e778a 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -2083,27 +2083,14 @@ handle_delayed_dealloc(Allctr_t *allctr,
 	res = 1;
 
 	blk = UMEM2BLK(ptr);
-	if (IS_FREE_LAST_MBC_BLK(blk)) {
+	if (blk->bhdr == HOMECOMING_MBC_BLK_HDR) {
 	    /*
 	     * A multiblock carrier that previously has been migrated away
 	     * from us and now is back to be deallocated. For more info
 	     * see schedule_dealloc_carrier().
-	     *
-	     * Note that we cannot use FBLK_TO_MBC(blk) since it
-	     * data has been overwritten by the queue.
 	     */
-	    Carrier_t *crr = FIRST_BLK_TO_MBC(allctr, blk);
-
-	     /* Restore word overwritten by the dd-queue as it will be read
-	      * if this carrier is pulled from dc_list by cpool_fetch()
-	      */
-	    ERTS_ALC_CPOOL_ASSERT(FBLK_TO_MBC(blk) != crr);
-	    ERTS_CT_ASSERT(sizeof(ErtsAllctrDDBlock_t) == sizeof(void*));
-#ifdef MBC_ABLK_OFFSET_BITS
-	    blk->u.carrier = crr;
-#else
-	    blk->carrier = crr;
-#endif
+	    Carrier_t *crr = ErtsContainerStruct(blk, Carrier_t,
+                                                 cpool.homecoming_dd.blk);
 
 	    ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr));
 	    ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr);
@@ -2116,6 +2103,7 @@ handle_delayed_dealloc(Allctr_t *allctr,
 	    schedule_dealloc_carrier(allctr, crr);
 	}
 	else {
+            ASSERT(IS_SBC_BLK(blk) || !IS_FREE_BLK(blk));
 
 	    INC_CC(allctr->calls.this_free);
 
@@ -3543,6 +3531,8 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
 			      == (erts_smp_atomic_read_nob(&crr->allctr)
 				  & ~ERTS_CRR_ALCTR_FLG_MASK));
 
+        blk = &crr->cpool.homecoming_dd.blk;
+        ERTS_ALC_CPOOL_ASSERT(blk->bhdr == HOMECOMING_MBC_BLK_HDR);
 	if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(blk), cinit))
 	    erts_alloc_notify_delayed_dealloc(orig_allctr->ix);
 	return;
@@ -3581,6 +3571,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
 static ERTS_INLINE void
 cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr)
 {
+    crr->cpool.homecoming_dd.blk.bhdr = HOMECOMING_MBC_BLK_HDR;
     erts_atomic_init_nob(&crr->cpool.next, ERTS_AINT_NULL);
     erts_atomic_init_nob(&crr->cpool.prev, ERTS_AINT_NULL);
     crr->cpool.orig_allctr = allctr;
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 81180382af..1291a6c10e 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -305,45 +305,7 @@ void  erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int supe
 
 typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t;
 
-#ifdef ERTS_SMP
-
-typedef struct ErtsDoubleLink_t_ {
-    struct ErtsDoubleLink_t_ *next;
-    struct ErtsDoubleLink_t_ *prev;
-}ErtsDoubleLink_t;
-
-typedef struct {
-    erts_atomic_t next;
-    erts_atomic_t prev;
-    Allctr_t *orig_allctr;      /* read-only while carrier is alive */
-    ErtsThrPrgrVal thr_prgr;
-    erts_atomic_t max_size;
-    UWord abandon_limit;
-    UWord blocks;
-    UWord blocks_size;
-    ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */
-} ErtsAlcCPoolData_t;
-
-#endif
-
 typedef struct Carrier_t_ Carrier_t;
-struct Carrier_t_ {
-    UWord chdr;
-    Carrier_t *next;
-    Carrier_t *prev;
-    erts_smp_atomic_t allctr;
-#ifdef ERTS_SMP
-    ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */
-#endif
-};
-
-#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \
-  ((Allctr_t *) (erts_smp_atomic_read_nob(&(C)->allctr) & ~FLG_MASK))
-
-typedef struct {
-    Carrier_t *first;
-    Carrier_t *last;
-} CarrierList_t;
 
 typedef struct {
     UWord bhdr;
@@ -357,6 +319,22 @@ typedef struct {
 #endif
 } Block_t;
 
+typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t;
+
+union ErtsAllctrDDBlock_t_ {
+    erts_atomic_t atmc_next;
+    ErtsAllctrDDBlock_t *ptr_next;
+};
+
+typedef struct {
+    Block_t blk;
+#if !MBC_ABLK_OFFSET_BITS
+    ErtsAllctrDDBlock_t umem_;
+#endif
+} ErtsFakeDDBlock_t;
+
+
+
 #define THIS_FREE_BLK_HDR_FLG 	(((UWord) 1) << 0)
 #define PREV_FREE_BLK_HDR_FLG 	(((UWord) 1) << 1)
 #define LAST_BLK_HDR_FLG 	(((UWord) 1) << 2)
@@ -365,14 +343,13 @@ typedef struct {
     (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)
 
 /*
- * FREE_LAST_MBC_BLK_HDR_FLGS is a special flag combo used for
- * distinguishing empty mbc's from allocated blocks in
- * handle_delayed_dealloc().
+ * HOMECOMING_MBC_BLK_HDR is a special block header combo used for
+ * distinguishing MBC's from allocated blocks in handle_delayed_dealloc().
  */
-#define FREE_LAST_MBC_BLK_HDR_FLGS (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)
+#define HOMECOMING_MBC_BLK_HDR (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)
 
 #define IS_FREE_LAST_MBC_BLK(B) \
-    (((B)->bhdr & FLG_MASK) == FREE_LAST_MBC_BLK_HDR_FLGS)
+    (((B)->bhdr & FLG_MASK) == (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG))
 
 #define IS_SBC_BLK(B) (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG)
 #define IS_MBC_BLK(B) (!IS_SBC_BLK((B)))
@@ -396,6 +373,48 @@ typedef struct {
 
 typedef UWord FreeBlkFtr_t; /* Footer of a free block */
 
+
+#ifdef ERTS_SMP
+
+typedef struct ErtsDoubleLink_t_ {
+    struct ErtsDoubleLink_t_ *next;
+    struct ErtsDoubleLink_t_ *prev;
+}ErtsDoubleLink_t;
+
+typedef struct {
+    ErtsFakeDDBlock_t homecoming_dd;
+    erts_atomic_t next;
+    erts_atomic_t prev;
+    Allctr_t *orig_allctr;      /* read-only while carrier is alive */
+    ErtsThrPrgrVal thr_prgr;
+    erts_atomic_t max_size;
+    UWord abandon_limit;
+    UWord blocks;
+    UWord blocks_size;
+    ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */
+} ErtsAlcCPoolData_t;
+
+#endif
+
+struct Carrier_t_ {
+    UWord chdr;
+    Carrier_t *next;
+    Carrier_t *prev;
+    erts_smp_atomic_t allctr;
+#ifdef ERTS_SMP
+    ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */
+#endif
+};
+
+#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \
+  ((Allctr_t *) (erts_smp_atomic_read_nob(&(C)->allctr) & ~FLG_MASK))
+
+typedef struct {
+    Carrier_t *first;
+    Carrier_t *last;
+} CarrierList_t;
+
+
 typedef Uint64 CallCounter_t;
 
 typedef struct {
@@ -433,13 +452,6 @@ typedef struct {
 
 #ifdef ERTS_SMP
 
-typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t;
-
-union ErtsAllctrDDBlock_t_ {
-    erts_atomic_t atmc_next;
-    ErtsAllctrDDBlock_t *ptr_next;
-};
-
 typedef struct {
     ErtsAllctrDDBlock_t marker;
     erts_atomic_t last;
-- 
cgit v1.2.3


From 97152092fd4e5fe827a4dac42f3b51ae634ba1ff Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 8 Dec 2017 19:02:04 +0100
Subject: erts: Improve alloc_SUITE:migration

to mix it up with some realloc calls.
---
 erts/emulator/beam/erl_alloc.c                     |   6 +-
 erts/emulator/test/alloc_SUITE.erl                 |  58 +++++++----
 .../test/alloc_SUITE_data/allocator_test.h         |   5 +-
 erts/emulator/test/alloc_SUITE_data/migration.c    | 112 ++++++++++++++++-----
 4 files changed, 132 insertions(+), 49 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 214fb1f2af..b3522a9dbf 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -3722,7 +3722,9 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
 
 	case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0;
 
-	case 0xf16: {
+        case 0xf16: return (UWord) erts_realloc(ERTS_ALC_T_TEST, (void*)a1, (Uint)a2);
+
+	case 0xf17: {
             Uint extra_hdr_sz = UNIT_CEILING((Uint)a1);
 	    ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST];
 	    Uint offset = ts->allctr[0]->mbc_header_size;
@@ -3749,7 +3751,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
 	    *(void**)a3 = orig_destroying_mbc;
 	    return offset;
 	}
-	case 0xf17: {
+	case 0xf18: {
 	    ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST];
 	    return ts->allctr[0]->largest_mbc_size;
 	}
diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl
index 3a721095e2..518e0c974a 100644
--- a/erts/emulator/test/alloc_SUITE.erl
+++ b/erts/emulator/test/alloc_SUITE.erl
@@ -114,7 +114,7 @@ erts_mmap_do(Config, SCO, SCRPM, SCRFSD) ->
 	       0 -> O1;
 	       _ -> O1 ++ " +MMscrfsd"++integer_to_list(SCRFSD)
 	   end,
-    {ok, Node} = start_node(Config, Opts),
+    {ok, Node} = start_node(Config, Opts, []),
     Self = self(),
     Ref = make_ref(),
     F = fun() ->
@@ -155,7 +155,9 @@ drv_case(Config) ->
 drv_case(Config, Mode, NodeOpts) when is_list(Config) ->
     case os:type() of
 	{Family, _} when Family == unix; Family == win32 ->
-	    {ok, Node} = start_node(Config, NodeOpts),
+            %%Prog = {prog,"/my/own/otp/bin/cerl -debug"},
+            Prog = [],
+	    {ok, Node} = start_node(Config, NodeOpts, Prog),
 	    Self = self(),
 	    Ref = make_ref(),
 	    spawn_link(Node,
@@ -221,19 +223,35 @@ wait_for_memory_deallocations() ->
     end.
 
 print_stats(migration) ->
-    {Btot,Ctot} = lists:foldl(fun({instance,Inr,Istats}, {Bacc,Cacc}) ->
-				      {mbcs,MBCS} = lists:keyfind(mbcs, 1, Istats),
-				      Btup = lists:keyfind(blocks, 1, MBCS),
-				      Ctup = lists:keyfind(carriers, 1, MBCS),
-				      io:format("{instance,~p,~p,~p}\n", [Inr, Btup, Ctup]),
-				      {tuple_add(Bacc,Btup),tuple_add(Cacc,Ctup)};
-				 (_, Acc) -> Acc
-			      end,
-			      {{blocks,0,0,0},{carriers,0,0,0}},
-			      erlang:system_info({allocator,test_alloc})),
-
+    IFun = fun({instance,Inr,Istats}, {Bacc,Cacc,Pacc}) ->
+                   {mbcs,MBCS} = lists:keyfind(mbcs, 1, Istats),
+                   Btup = lists:keyfind(blocks, 1, MBCS),
+                   Ctup = lists:keyfind(carriers, 1, MBCS),
+
+                   Ptup = case lists:keyfind(mbcs_pool, 1, Istats) of
+                              {mbcs_pool,POOL} ->
+                                  {blocks, Bpool} = lists:keyfind(blocks, 1, POOL),
+                                  {carriers, Cpool} = lists:keyfind(carriers, 1, POOL),
+                                  {pool, Bpool, Cpool};
+                              false ->
+                                  {pool, 0, 0}
+                          end,
+                   io:format("{instance,~p,~p,~p,~p}}\n",
+                             [Inr, Btup, Ctup, Ptup]),
+                   {tuple_add(Bacc,Btup),tuple_add(Cacc,Ctup),
+                    tuple_add(Pacc,Ptup)};
+              (_, Acc) -> Acc
+           end,
+
+    {Btot,Ctot,Ptot} = lists:foldl(IFun,
+                                   {{blocks,0,0,0},{carriers,0,0,0},{pool,0,0}},
+                                   erlang:system_info({allocator,test_alloc})),
+
+    {pool, PBtot, PCtot} = Ptot,
     io:format("Number of blocks  : ~p\n", [Btot]),
-    io:format("Number of carriers: ~p\n", [Ctot]);
+    io:format("Number of carriers: ~p\n", [Ctot]),
+    io:format("Number of pooled blocks  : ~p\n", [PBtot]),
+    io:format("Number of pooled carriers: ~p\n", [PCtot]);
 print_stats(_) -> ok.
 
 tuple_add(T1, T2) ->
@@ -330,13 +348,13 @@ handle_result(_State, Result0) ->
 	    continue
     end.
 
-start_node(Config, Opts) when is_list(Config), is_list(Opts) ->
+start_node(Config, Opts, Prog) when is_list(Config), is_list(Opts) ->
     case proplists:get_value(debug,Config) of
 	true -> {ok, node()};
-	_ -> start_node_1(Config, Opts)
+	_ -> start_node_1(Config, Opts, Prog)
     end.
 
-start_node_1(Config, Opts) ->
+start_node_1(Config, Opts, Prog) ->
     Pa = filename:dirname(code:which(?MODULE)),
     Name = list_to_atom(atom_to_list(?MODULE)
 			++ "-"
@@ -345,7 +363,11 @@ start_node_1(Config, Opts) ->
 			++ integer_to_list(erlang:system_time(second))
 			++ "-"
 			++ integer_to_list(erlang:unique_integer([positive]))),
-    test_server:start_node(Name, slave, [{args, Opts++" -pa "++Pa}]).
+    ErlArg = case Prog of
+                 [] -> [];
+                 _ -> [{erl,[Prog]}]
+             end,
+    test_server:start_node(Name, slave, [{args, Opts++" -pa "++Pa} | ErlArg]).
 
 stop_node(Node) when Node =:= node() -> ok;
 stop_node(Node) ->
diff --git a/erts/emulator/test/alloc_SUITE_data/allocator_test.h b/erts/emulator/test/alloc_SUITE_data/allocator_test.h
index 97ee58cdad..5272f86c98 100644
--- a/erts/emulator/test/alloc_SUITE_data/allocator_test.h
+++ b/erts/emulator/test/alloc_SUITE_data/allocator_test.h
@@ -156,7 +156,8 @@ typedef void* erts_cond;
 #define IS_SMP_ENABLED		((int)		ALC_TEST0(0xf13))
 #define ALLOC_TEST(S)		((void*)	ALC_TEST1(0xf14, (S)))
 #define FREE_TEST(P)		((void)		ALC_TEST1(0xf15, (P)))
-#define SET_TEST_MBC_USER_HEADER(SZ,CMBC,DMBC) ((int)ALC_TEST3(0xf16, (SZ), (CMBC), (DMBC)))
-#define GET_TEST_MBC_SIZE()     ((int)          ALC_TEST0(0xf17))
+#define REALLOC_TEST(P,S)	((void*)	ALC_TEST2(0xf16, (P), (S)))
+#define SET_TEST_MBC_USER_HEADER(SZ,CMBC,DMBC) ((int)ALC_TEST3(0xf17, (SZ), (CMBC), (DMBC)))
+#define GET_TEST_MBC_SIZE()     ((int)          ALC_TEST0(0xf18))
 
 #endif
diff --git a/erts/emulator/test/alloc_SUITE_data/migration.c b/erts/emulator/test/alloc_SUITE_data/migration.c
index b9a4de03b3..1d974225fc 100644
--- a/erts/emulator/test/alloc_SUITE_data/migration.c
+++ b/erts/emulator/test/alloc_SUITE_data/migration.c
@@ -223,6 +223,42 @@ static int rand_int(MigrationState* state, int low, int high)
     return low + (x % (high+1-low));
 }
 
+enum Operation
+{
+    ALLOCATE_OP,
+    FREE_OP,
+    REALLOC_OP,
+    CLEANUP_OP
+};
+
+static enum Operation rand_op(MigrationState* state)
+{
+    int r = rand_int(state, 1, 100);
+    switch (state->phase) {
+    case GROWING:
+        FATAL_ASSERT(state->nblocks < state->max_nblocks);
+        if (r > 10 || state->nblocks == 0)
+            return ALLOCATE_OP;
+        else if (r > 5)
+            return FREE_OP;
+        else
+            return REALLOC_OP;
+
+    case SHRINKING:
+        FATAL_ASSERT(state->nblocks > 0);
+        if (r > 10 || state->nblocks == state->max_nblocks)
+            return FREE_OP;
+        else if (r > 5)
+            return ALLOCATE_OP;
+        else
+            return REALLOC_OP;
+
+    case CLEANUP:
+        return CLEANUP_OP;
+    default:
+        FATAL_ASSERT(!"Invalid op phase");
+    }
+}
 
 static void do_cleanup(TestCaseState_t *tcs, MigrationState* state)
 {
@@ -275,53 +311,75 @@ testcase_run(TestCaseState_t *tcs)
 	state->goal_nblocks = rand_int(state, 1, state->max_nblocks);
     }
 
-    switch (state->phase) {
-    case GROWING: {
+    switch (rand_op(state)) {
+    case ALLOCATE_OP: {
 	MyBlock* p;
 	FATAL_ASSERT(!state->blockv[state->nblocks]);
-	p = ALLOC_TEST(rand_int(state, state->block_size/2, state->block_size));
+        p = ALLOC_TEST(rand_int(state, state->block_size/2, state->block_size));
 	FATAL_ASSERT(p);
 	add_block(p, state);
-	state->blockv[state->nblocks] = p;
-	if (++state->nblocks >= state->goal_nblocks) {
-	    /*testcase_printf(tcs, "%d: Grown to %d blocks", tcs->thr_nr, state->nblocks);*/
-	    state->phase = SHRINKING;
-	    state->goal_nblocks = rand_int(state, 0, state->goal_nblocks-1);
-	}
-	else
-	    FATAL_ASSERT(!state->blockv[state->nblocks]);
+	state->blockv[state->nblocks++] = p;
 	break;
     }
-    case SHRINKING: {
+    case FREE_OP: {
 	int ix = rand_int(state, 0, state->nblocks-1);
 	FATAL_ASSERT(state->blockv[ix]);
 	remove_block(state->blockv[ix]);
 	FREE_TEST(state->blockv[ix]);
 	state->blockv[ix] = state->blockv[--state->nblocks];
 	state->blockv[state->nblocks] = NULL;
-
-	if (state->nblocks <= state->goal_nblocks) {
-	    /*testcase_printf(tcs, "%d: Shrunk to %d blocks", tcs->thr_nr, state->nblocks);*/
-	    if (++state->round >= MAX_ROUNDS) {
-		state->phase = CLEANUP;
-	    } else {
-		state->phase = GROWING;
-		state->goal_nblocks = rand_int(state, state->goal_nblocks+1, state->max_nblocks);
-	    }
-	}
 	break;
     }
+    case REALLOC_OP: {
+        int ix = rand_int(state, 0, state->nblocks-1);
+        MyBlock* p;
+        FATAL_ASSERT(state->blockv[ix]);
+        remove_block(state->blockv[ix]);
+        p = REALLOC_TEST(state->blockv[ix], rand_int(state, state->block_size/2, state->block_size));
+        FATAL_ASSERT(p);
+        add_block(p, state);
+        state->blockv[ix] = p;
+        break;
+    }
+    case CLEANUP_OP:
+        do_cleanup(tcs, state);
+        break;
+    default:
+	FATAL_ASSERT(!"Invalid operation");
+    }
+
+    switch (state->phase) {
+    case GROWING: {
+        if (state->nblocks >= state->goal_nblocks) {
+            /*testcase_printf(tcs, "%d: Grown to %d blocks", tcs->thr_nr, state->nblocks);*/
+            state->phase = SHRINKING;
+            state->goal_nblocks = rand_int(state, 0, state->goal_nblocks-1);
+        }
+        else
+            FATAL_ASSERT(!state->blockv[state->nblocks]);
+        break;
+    }
+    case SHRINKING: {
+        if (state->nblocks <= state->goal_nblocks) {
+            /*testcase_printf(tcs, "%d: Shrunk to %d blocks", tcs->thr_nr, state->nblocks);*/
+            if (++state->round >= MAX_ROUNDS) {
+                state->phase = CLEANUP;
+            } else {
+                state->phase = GROWING;
+                state->goal_nblocks = rand_int(state, state->goal_nblocks+1, state->max_nblocks);
+            }
+        }
+        break;
+    }
     case CLEANUP:
-	do_cleanup(tcs, state);
-	break;
+    case DONE:
+        break;
 
     default:
 	FATAL_ASSERT(!"Invalid phase");
     }
 
-    if (state->phase == DONE) {
-    }
-    else {
+    if (state->phase != DONE) {
 	testcase_continue(tcs);
     }
 }
-- 
cgit v1.2.3


From a7f87e104e769cb7fed65076193ef0bc4c9f08fd Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 8 Dec 2017 19:02:15 +0100
Subject: erts: Improve carrier pool search

* Give back carrier to owner when put in pool with use of dd-queue.
* Replace pooled_list with pooled_tree for more efficient search
  of all owned pooled carriers.
* Remove traitor_list as it does not serve much purpose anymore.
* Add HOMECOMING bit flag in crr->allctr atomic to
  (1) avoid double enqueue into dd-enqueue.
  (2) trigger read barrier in get_used_allctr for newly received carriers.
---
 erts/emulator/beam/erl_alloc_util.c            | 659 +++++++++++++------------
 erts/emulator/beam/erl_alloc_util.h            |  39 +-
 erts/emulator/beam/erl_ao_firstfit_alloc.c     | 126 +++--
 erts/emulator/beam/erl_ao_firstfit_alloc.h     |   1 -
 erts/emulator/internal_doc/CarrierMigration.md | 138 +++---
 5 files changed, 539 insertions(+), 424 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index f0597e778a..dbf27f1d2f 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -375,8 +375,10 @@ do {										\
 
 #define ERTS_CRR_ALCTR_FLG_IN_POOL	(((erts_aint_t) 1) << 0)
 #define ERTS_CRR_ALCTR_FLG_BUSY		(((erts_aint_t) 1) << 1)
+#define ERTS_CRR_ALCTR_FLG_HOMECOMING	(((erts_aint_t) 1) << 2)
 #define ERTS_CRR_ALCTR_FLG_MASK (ERTS_CRR_ALCTR_FLG_IN_POOL | \
-                                 ERTS_CRR_ALCTR_FLG_BUSY)
+                                 ERTS_CRR_ALCTR_FLG_BUSY |    \
+                                 ERTS_CRR_ALCTR_FLG_HOMECOMING)
 
 #ifdef ERTS_SMP
 #define SBC_HEADER_SIZE	   						\
@@ -583,7 +585,7 @@ do {									\
     DEBUG_CHECK_CARRIER_NO_SZ((AP));					\
 } while (0)
 
-#define STAT_MBC_CPOOL_INSERT(AP, CRR)					\
+#define STAT_MBC_ABANDON(AP, CRR)					\
 do {									\
     UWord csz__ = CARRIER_SZ((CRR));					\
     if (IS_MSEG_CARRIER((CRR)))						\
@@ -1143,90 +1145,25 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr)
 	ASSERT(crr->next);
 	crr->next->prev = crr->prev;
     }
-}
-
-#ifdef ERTS_SMP
-
 #ifdef DEBUG
-static int is_in_list(ErtsDoubleLink_t* sentinel, ErtsDoubleLink_t* node)
-{
-    ErtsDoubleLink_t* p;
-
-    ASSERT(node != sentinel);
-    for (p = sentinel->next; p != sentinel; p = p->next) {
-	if (p == node)
-	    return 1;
-    }
-    return 0;
-}
-#endif /* DEBUG */
-
-static ERTS_INLINE void
-link_edl_after(ErtsDoubleLink_t* after_me, ErtsDoubleLink_t* node)
-{
-    ErtsDoubleLink_t* before_me = after_me->next;
-    ASSERT(node != after_me && node != before_me);
-    node->next = before_me;
-    node->prev = after_me;
-    before_me->prev = node;
-    after_me->next = node;
-}
-
-static ERTS_INLINE void
-link_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node)
-{
-    ErtsDoubleLink_t* after_me = before_me->prev;
-    ASSERT(node != before_me && node != after_me);
-    node->next = before_me;
-    node->prev = after_me;
-    before_me->prev = node;
-    after_me->next = node;
-}
-
-static ERTS_INLINE void
-unlink_edl(ErtsDoubleLink_t* node)
-{
-    node->next->prev = node->prev;
-    node->prev->next = node->next;
+    crr->next = crr;
+    crr->prev = crr;
+#endif
 }
 
-static ERTS_INLINE void
-relink_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node)
-{
-    if (node != before_me && node != before_me->prev) {
-	unlink_edl(node);
-	link_edl_before(before_me, node);
-    }
-}
+#ifdef ERTS_SMP
 
 static ERTS_INLINE int is_abandoned(Carrier_t *crr)
 {
-    return crr->cpool.abandoned.next != NULL;
-}
-
-static ERTS_INLINE void
-link_abandoned_carrier(ErtsDoubleLink_t* list, Carrier_t *crr)
-{
-    ASSERT(!is_abandoned(crr));
-
-    link_edl_after(list, &crr->cpool.abandoned);
-
-    ASSERT(crr->cpool.abandoned.next != &crr->cpool.abandoned);
-    ASSERT(crr->cpool.abandoned.prev != &crr->cpool.abandoned);
+    return crr->cpool.state != ERTS_MBC_IS_HOME;
 }
 
 static ERTS_INLINE void
 unlink_abandoned_carrier(Carrier_t *crr)
 {
-    ASSERT(is_in_list(&crr->cpool.orig_allctr->cpool.pooled_list,
-		      &crr->cpool.abandoned) ||
-	   is_in_list(&crr->cpool.orig_allctr->cpool.traitor_list,
-		      &crr->cpool.abandoned));
-
-    unlink_edl(&crr->cpool.abandoned);
-
-    crr->cpool.abandoned.next = NULL;
-    crr->cpool.abandoned.prev = NULL;
+    if (crr->cpool.state == ERTS_MBC_WAS_POOLED) {
+        aoff_remove_pooled_mbc(crr->cpool.orig_allctr, crr);
+    }
 }
 
 static ERTS_INLINE void
@@ -1234,24 +1171,19 @@ clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr)
 {
     if (crr) {
 	erts_aint_t max_size;
-	erts_aint_t new_val;
+	erts_aint_t iallctr;
 
 	max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr);
 	erts_atomic_set_nob(&crr->cpool.max_size, max_size);
 
-	new_val = (((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL);
-
-#ifdef ERTS_ALC_CPOOL_DEBUG
-	{
-	    erts_aint_t old_val = new_val|ERTS_CRR_ALCTR_FLG_BUSY;
+        iallctr = erts_smp_atomic_read_nob(&crr->allctr);
+        ERTS_ALC_CPOOL_ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING)
+                              == ((erts_aint_t)allctr |
+                                  ERTS_CRR_ALCTR_FLG_IN_POOL |
+                                  ERTS_CRR_ALCTR_FLG_BUSY));
 
-	    ERTS_ALC_CPOOL_ASSERT(old_val
-				  == erts_smp_atomic_xchg_relb(&crr->allctr,
-							       new_val));
-	}
-#else
-	erts_smp_atomic_set_relb(&crr->allctr, new_val);
-#endif
+	iallctr &= ~ERTS_CRR_ALCTR_FLG_BUSY;
+	erts_smp_atomic_set_relb(&crr->allctr, iallctr);
     }
 }
 
@@ -1667,6 +1599,11 @@ dealloc_mbc(Allctr_t *allctr, Carrier_t *crr)
 
 #ifdef ERTS_SMP
 
+static void set_new_allctr_abandon_limit(Allctr_t*);
+static void abandon_carrier(Allctr_t*, Carrier_t*);
+static void poolify_my_carrier(Allctr_t*, Carrier_t*);
+static void enqueue_homecoming(Allctr_t*, Carrier_t*);
+
 static ERTS_INLINE Allctr_t*
 get_pref_allctr(void *extra)
 {
@@ -1733,9 +1670,23 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep,
 		erts_aint_t act;
 
 		ERTS_ALC_CPOOL_ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY));
-		act = erts_smp_atomic_cmpxchg_ddrb(&crr->allctr,
-						   iallctr|ERTS_CRR_ALCTR_FLG_BUSY,
-						   iallctr);
+                if (iallctr & ERTS_CRR_ALCTR_FLG_HOMECOMING) {
+                    /*
+                     * This carrier has just been given back to us by writing
+                     * to crr->allctr with a write barrier (see abandon_carrier).
+                     *
+                     * We need a mathing read barrier to guarantee a correct view
+                     * of the carrier for deallocation work.
+                     */
+                    act = erts_smp_atomic_cmpxchg_rb(&crr->allctr,
+                                                     iallctr|ERTS_CRR_ALCTR_FLG_BUSY,
+                                                     iallctr);
+                }
+                else {
+                    act = erts_smp_atomic_cmpxchg_ddrb(&crr->allctr,
+                                                       iallctr|ERTS_CRR_ALCTR_FLG_BUSY,
+                                                       iallctr);
+                }
 		if (act == iallctr) {
 		    *busy_pcrr_pp = crr;
 		    break;
@@ -1751,13 +1702,6 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep,
 		    erts_mtx_unlock(&pref_allctr->mutex);
 		}
 	    }
-
-	    ERTS_ALC_CPOOL_ASSERT(
-		(((iallctr & ~ERTS_CRR_ALCTR_FLG_MASK) == (erts_aint_t) pref_allctr)
-		 ? (((iallctr & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL)
-		    || ((iallctr & ERTS_CRR_ALCTR_FLG_MASK) == 0))
-		 : 1));
-
 	    return used_allctr;
 	}
     }
@@ -2009,9 +1953,9 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr)
 	    /* Carrier migrated; need to redirect block to new owner... */
 	    int cinit = used_allctr->dd.ix - allctr->dd.ix;
 
-	    ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p);
+            ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p);
 
-	    DEC_CC(allctr->calls.this_free);
+            DEC_CC(allctr->calls.this_free);
 
 	    ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type;
 	    if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit))
@@ -2020,8 +1964,9 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr)
     }
 }
 
-static void
-schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr);
+static void schedule_dealloc_carrier(Allctr_t*, Carrier_t*);
+static void dealloc_my_carrier(Allctr_t*, Carrier_t*);
+
 
 static ERTS_INLINE int
 handle_delayed_dealloc(Allctr_t *allctr,
@@ -2086,24 +2031,58 @@ handle_delayed_dealloc(Allctr_t *allctr,
 	if (blk->bhdr == HOMECOMING_MBC_BLK_HDR) {
 	    /*
 	     * A multiblock carrier that previously has been migrated away
-	     * from us and now is back to be deallocated. For more info
-	     * see schedule_dealloc_carrier().
+             * from us, was sent back to us either because
+             * - it became empty and we need to deallocated it, or
+             * - it was inserted into the pool and we need to update our pooled_tree
 	     */
 	    Carrier_t *crr = ErtsContainerStruct(blk, Carrier_t,
                                                  cpool.homecoming_dd.blk);
+            Block_t* first_blk = MBC_TO_FIRST_BLK(allctr, crr);
+            erts_aint_t iallctr;
 
 	    ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr));
 	    ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr);
-	    ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr)
-				  != (erts_smp_atomic_read_nob(&crr->allctr)
-				      & ~ERTS_CRR_ALCTR_FLG_MASK));
 
-	    erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
-
-	    schedule_dealloc_carrier(allctr, crr);
+            iallctr = erts_smp_atomic_read_nob(&crr->allctr);
+            ASSERT(iallctr & ERTS_CRR_ALCTR_FLG_HOMECOMING);
+            while (1) {
+                if ((iallctr & (~ERTS_CRR_ALCTR_FLG_MASK |
+                                ERTS_CRR_ALCTR_FLG_IN_POOL))
+                    == (erts_aint_t)allctr) {
+                    /*
+                     * Carrier is home (mine and not in pool)
+                     */
+                    ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY));
+                    erts_smp_atomic_set_nob(&crr->allctr, (erts_aint_t)allctr);
+                    if (IS_FREE_LAST_MBC_BLK(first_blk))
+                        dealloc_my_carrier(allctr, crr);
+                    else
+                        ASSERT(crr->cpool.state == ERTS_MBC_IS_HOME);
+                }
+                else {
+                    erts_aint_t exp = iallctr;
+                    erts_aint_t want = iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING;
+
+                    iallctr = erts_smp_atomic_cmpxchg_nob(&crr->allctr,
+                                                          want,
+                                                          exp);
+                    if (iallctr != exp)
+                        continue; /* retry */
+
+                    ASSERT(crr->cpool.state != ERTS_MBC_IS_HOME);
+                    unlink_abandoned_carrier(crr);
+                    if (iallctr & ERTS_CRR_ALCTR_FLG_IN_POOL)
+                        poolify_my_carrier(allctr, crr);
+                    else
+                        crr->cpool.state = ERTS_MBC_WAS_TRAITOR;
+                }
+                break;
+            }
 	}
 	else {
-            ASSERT(IS_SBC_BLK(blk) || !IS_FREE_BLK(blk));
+            ASSERT(IS_SBC_BLK(blk) || (ABLK_TO_MBC(blk) !=
+                                       ErtsContainerStruct(blk, Carrier_t,
+                                                           cpool.homecoming_dd.blk)));
 
 	    INC_CC(allctr->calls.this_free);
 
@@ -2145,23 +2124,26 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type,
 	erts_alloc_notify_delayed_dealloc(allctr->ix);
 }
 
-#endif
-
-#ifdef ERTS_SMP
-static void
-set_new_allctr_abandon_limit(Allctr_t *allctr);
-static void
-abandon_carrier(Allctr_t *allctr, Carrier_t *crr);
-
+static ERTS_INLINE void
+update_pooled_tree(Allctr_t *allctr, Carrier_t *crr, Uint blk_sz)
+{
+    if (allctr == crr->cpool.orig_allctr && crr->cpool.state == ERTS_MBC_WAS_POOLED) {
+	/*
+	 * Update pooled_tree with a potentially new (larger) max_sz
+         */
+        AOFF_RBTree_t* crr_node = &crr->cpool.pooled;
+        if (blk_sz > crr_node->hdr.bhdr) {
+            crr_node->hdr.bhdr = blk_sz;
+            erts_aoff_larger_max_size(crr_node);
+        }
+    }
+}
 
 static ERTS_INLINE void
 check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp)
 {
     Carrier_t *crr;
 
-    if (busy_pcrr_pp && *busy_pcrr_pp)
-	return;
-
     if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
 	return;
 
@@ -2243,6 +2225,7 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_
     else {
 	Carrier_t *busy_pcrr_p;
 	Allctr_t *used_allctr;
+
 	used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr,
 				      NULL, &busy_pcrr_p);
 	if (used_allctr == allctr) {
@@ -2259,10 +2242,10 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_
 	    /* Carrier migrated; need to redirect block to new owner... */
 	    int cinit = used_allctr->dd.ix - allctr->dd.ix;
 
-	    ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p);
+            ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p);
 
-	    if (dec_cc_on_redirect)
-		DEC_CC(allctr->calls.this_free);
+            if (dec_cc_on_redirect)
+                DEC_CC(allctr->calls.this_free);
 	    if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit))
 		erts_alloc_notify_delayed_dealloc(used_allctr->ix);
 	}
@@ -2507,16 +2490,17 @@ mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp)
     ASSERT(blk_sz % sizeof(Unit_t) == 0);
     ASSERT(IS_MBC_BLK(blk));
 
-    if (is_first_blk
-	&& is_last_blk
-	&& allctr->main_carrier != FIRST_BLK_TO_MBC(allctr, blk)) {
-	destroy_carrier(allctr, blk, busy_pcrr_pp);
+    if (is_first_blk && is_last_blk && crr != allctr->main_carrier) {
+        destroy_carrier(allctr, blk, busy_pcrr_pp);
     }
     else {
 	(*allctr->link_free_block)(allctr, blk);
 	HARD_CHECK_BLK_CARRIER(allctr, blk);
 #ifdef ERTS_SMP
-	check_abandon_carrier(allctr, blk, busy_pcrr_pp);
+        if (busy_pcrr_pp && *busy_pcrr_pp)
+            update_pooled_tree(allctr, crr, blk_sz);
+        else
+            check_abandon_carrier(allctr, blk, busy_pcrr_pp);
 #endif
     }
 }
@@ -2552,8 +2536,19 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs,
 #else /* !MBC_REALLOC_ALWAYS_MOVES */
 
 #ifdef ERTS_SMP
-    if (busy_pcrr_pp && *busy_pcrr_pp)
-	goto realloc_move; /* Don't want to use carrier in pool */
+    if (busy_pcrr_pp && *busy_pcrr_pp) {
+        /*
+         * Don't want to use carrier in pool
+         */
+        new_p = mbc_alloc(allctr, size);
+        if (!new_p)
+            return NULL;
+        new_blk = UMEM2BLK(new_p);
+        ASSERT(!(IS_MBC_BLK(new_blk) && ABLK_TO_MBC(new_blk) == *busy_pcrr_pp));
+        sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ));
+        mbc_free(allctr, p, busy_pcrr_pp);
+        return new_p;
+    }
 #endif
 
     get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size);
@@ -2789,9 +2784,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs,
 
     if (cand_blk_sz < get_blk_sz) {
 	/* We wont fit in cand_blk get a new one */
-#ifdef ERTS_SMP
-    realloc_move:
-#endif
+
 #endif /* !MBC_REALLOC_ALWAYS_MOVES */
 
 	new_p = mbc_alloc(allctr, size);
@@ -2896,8 +2889,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs,
 #ifdef ERTS_SMP
 
 #define ERTS_ALC_MAX_DEALLOC_CARRIER		10
-#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT	20
-#define ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT	10
+#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT	100
 #define ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT	100
 #define ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS	3
 
@@ -3061,19 +3053,18 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr)
     ErtsAlcCPoolData_t *cpd1p, *cpd2p;
     erts_aint_t val;
     ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel;
+    Allctr_t *orig_allctr = crr->cpool.orig_allctr;
 
     ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */
 			  || erts_thr_progress_is_managed_thread());
-    ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr)
-			  == (erts_aint_t) allctr);
 
-    erts_atomic_add_nob(&allctr->cpool.stat.blocks_size,
+    erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size,
 			(erts_aint_t) crr->cpool.blocks_size);
-    erts_atomic_add_nob(&allctr->cpool.stat.no_blocks,
+    erts_atomic_add_nob(&orig_allctr->cpool.stat.no_blocks,
 			(erts_aint_t) crr->cpool.blocks);
-    erts_atomic_add_nob(&allctr->cpool.stat.carriers_size,
+    erts_atomic_add_nob(&orig_allctr->cpool.stat.carriers_size,
 			(erts_aint_t) CARRIER_SZ(crr));
-    erts_atomic_inc_nob(&allctr->cpool.stat.no_carriers);
+    erts_atomic_inc_nob(&orig_allctr->cpool.stat.no_carriers);
 
     /*
      * We search in 'next' direction and begin by passing
@@ -3134,8 +3125,6 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr)
 			 (erts_aint_t) &crr->cpool,
 			 (erts_aint_t) cpd1p);
 
-    erts_smp_atomic_set_wb(&crr->allctr,
-			   ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL);
     LTTNG3(carrier_pool_put, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, CARRIER_SZ(crr));
 }
 
@@ -3237,130 +3226,119 @@ cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr)
 static Carrier_t *
 cpool_fetch(Allctr_t *allctr, UWord size)
 {
-    int i, i_stop, has_passed_sentinel;
+    enum { IGNORANT, HAS_SEEN_SENTINEL, THE_LAST_ONE } loop_state;
+    int i;
     Carrier_t *crr;
+    Carrier_t *reinsert_crr = NULL;
     ErtsAlcCPoolData_t *cpdp;
-    ErtsAlcCPoolData_t *cpool_entrance;
+    ErtsAlcCPoolData_t *cpool_entrance = NULL;
     ErtsAlcCPoolData_t *sentinel;
-    ErtsDoubleLink_t* dl;
-    ErtsDoubleLink_t* first_old_traitor;
 
     ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */
 			  || erts_thr_progress_is_managed_thread());
 
     i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT;
-    first_old_traitor = allctr->cpool.traitor_list.next;
-    cpool_entrance = NULL;
 
     LTTNG3(carrier_pool_get, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, (unsigned long)size);
     /*
-     * Search my own pooled_list,
+     * Search my own pooled_tree,
      * i.e my abandoned carriers that were in the pool last time I checked.
      */
+    do {
+        erts_aint_t exp, act;
+
+        crr = aoff_lookup_pooled_mbc(allctr, size);
+        if (!crr)
+            break;
+
+        ASSERT(crr->cpool.state == ERTS_MBC_WAS_POOLED);
+        ASSERT(crr->cpool.orig_allctr == allctr);
+
+        aoff_remove_pooled_mbc(allctr, crr);
+
+        exp = erts_smp_atomic_read_nob(&crr->allctr);
+        if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) {
+            ASSERT((exp & ~ERTS_CRR_ALCTR_FLG_MASK) == (erts_aint_t)allctr);
+            if (erts_atomic_read_nob(&crr->cpool.max_size) < size) {
+                /*
+                 * This carrier has been fetched and inserted back again
+                 * by a foreign allocator. That's why it has a stale search size.
+                 */
+                ASSERT(exp & ERTS_CRR_ALCTR_FLG_HOMECOMING);
+                crr->cpool.pooled.hdr.bhdr = erts_atomic_read_nob(&crr->cpool.max_size);
+                aoff_add_pooled_mbc(allctr, crr);
+                continue;
+            }
+            else if (exp & ERTS_CRR_ALCTR_FLG_BUSY) {
+                /*
+                 * This must be our own carrier as part of a realloc call.
+                 * Skip it to make things simpler.
+                 * Must wait to re-insert to not be found again by lookup.
+                 */
+                ASSERT(!reinsert_crr);
+                reinsert_crr = crr;
+                continue;
+            }
+
+            /* Try to fetch it... */
+            act = erts_smp_atomic_cmpxchg_mb(&crr->allctr,
+                                             exp & ~ERTS_CRR_ALCTR_FLG_IN_POOL,
+                                             exp);
+            if (act == exp) {
+                cpool_delete(allctr, allctr, crr);
+                crr->cpool.state = ERTS_MBC_IS_HOME;
+
+                if (reinsert_crr)
+                    aoff_add_pooled_mbc(allctr, reinsert_crr);
+                return crr;
+            }
+            exp = act;
+        }
 
-    dl = allctr->cpool.pooled_list.next;
-    while(dl != &allctr->cpool.pooled_list) {
-	erts_aint_t exp, act;
-	crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned));
+        /* Not in pool anymore */
+        ASSERT(!(exp & ERTS_CRR_ALCTR_FLG_BUSY));
+        crr->cpool.state = ERTS_MBC_WAS_TRAITOR;
 
-	ASSERT(!is_in_list(&allctr->cpool.traitor_list, dl));
-	ASSERT(crr->cpool.orig_allctr == allctr);
-	dl = dl->next;
-	exp = erts_smp_atomic_read_rb(&crr->allctr);
-	if ((exp & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL
-	    && erts_atomic_read_nob(&crr->cpool.max_size) >= size) {
-	    /* Try to fetch it... */
-	    act = erts_smp_atomic_cmpxchg_mb(&crr->allctr,
-					     (erts_aint_t) allctr,
-					     exp);
-	    if (act == exp) {
-		cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr);
-		unlink_abandoned_carrier(crr);
+    }while (--i > 0);
 
-		/* Move sentinel to continue next search from here */
-		relink_edl_before(dl, &allctr->cpool.pooled_list);
-		return crr;
-	    }
-	    exp = act;
-	}
-	if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) {
-	    if (!cpool_entrance)
-		cpool_entrance = &crr->cpool;
-	}
-	else { /* Not in pool, move to traitor_list */
-	    unlink_abandoned_carrier(crr);
-	    link_abandoned_carrier(&allctr->cpool.traitor_list, crr);
-	}
-	if (--i <= 0) {
-	    /* Move sentinel to continue next search from here */
-	    relink_edl_before(dl, &allctr->cpool.pooled_list);
-	    return NULL;
-	}
-    }
+    if (reinsert_crr)
+        aoff_add_pooled_mbc(allctr, reinsert_crr);
 
-    /* Now search traitor_list.
-     * i.e carriers employed by other allocators last time I checked.
-     * They might have been abandoned since then.
+    /*
+     * Try find a nice cpool_entrance
      */
+    while (allctr->cpool.pooled_tree) {
+        erts_aint_t iallctr;
+
+        crr = ErtsContainerStruct(allctr->cpool.pooled_tree, Carrier_t, cpool.pooled);
+        iallctr = erts_smp_atomic_read_nob(&crr->allctr);
+        if (iallctr & ERTS_CRR_ALCTR_FLG_IN_POOL) {
+            cpool_entrance = &crr->cpool;
+            break;
+        }
+        /* Not in pool anymore */
+        ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY));
+        aoff_remove_pooled_mbc(allctr, crr);
+        crr->cpool.state = ERTS_MBC_WAS_TRAITOR;
 
-    i_stop = (i < ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT ?
-	      0 : i - ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT);
-    dl = first_old_traitor;
-    while(dl != &allctr->cpool.traitor_list) {
-	erts_aint_t exp, act;
-	crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned));
-	ASSERT(dl != &allctr->cpool.pooled_list);
-	ASSERT(crr->cpool.orig_allctr == allctr);
-	dl = dl->next;
-	exp = erts_smp_atomic_read_rb(&crr->allctr);
-	if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) {
-	    if (!(exp & ERTS_CRR_ALCTR_FLG_BUSY)
-		&& erts_atomic_read_nob(&crr->cpool.max_size) >= size) {
-		/* Try to fetch it... */
-		act = erts_smp_atomic_cmpxchg_mb(&crr->allctr,
-						 (erts_aint_t) allctr,
-						 exp);
-		if (act == exp) {
-		    cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr);
-		    unlink_abandoned_carrier(crr);
-
-		    /* Move sentinel to continue next search from here */
-		    relink_edl_before(dl, &allctr->cpool.traitor_list);
-		    return crr;
-		}
-		exp = act;
-	    }
-	    if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) {
-		if (!cpool_entrance)
-		    cpool_entrance = &crr->cpool;
-
-		/* Move to pooled_list */
-		unlink_abandoned_carrier(crr);
-		link_abandoned_carrier(&allctr->cpool.pooled_list, crr);
-	    }
-	}
-	if (--i <= i_stop) {
-	    /* Move sentinel to continue next search from here */
-	    relink_edl_before(dl, &allctr->cpool.traitor_list);
-	    if (i > 0)
-		break;
-	    else
-		return NULL;
-	}
+        if (--i <= 0)
+            return NULL;
     }
 
+
     /*
      * Finally search the shared pool and try employ foreign carriers
      */
-
     sentinel = &carrier_pool[allctr->alloc_no].sentinel;
     if (cpool_entrance) {
-	/* We saw a pooled carried above, use it as entrance into the pool
+        /*
+         * We saw a pooled carried above, use it as entrance into the pool
 	 */
 	cpdp = cpool_entrance;
     }
     else {
-	/* No pooled carried seen above. Start search at cpool sentinel,
+        /*
+         * No pooled carried seen above. Start search at cpool sentinel,
 	 * but begin by passing one element before trying to fetch.
 	 * This in order to avoid contention with threads inserting elements.
 	 */
@@ -3370,8 +3348,8 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 	    goto check_dc_list;
     }
 
-    has_passed_sentinel = 0;
-    while (1) {
+    loop_state = IGNORANT;
+    do {
 	erts_aint_t exp;
 	cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev));
 	if (cpdp == cpool_entrance) {
@@ -3380,38 +3358,41 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 		if (cpdp == sentinel)
 		    break;
 	    }
-	    i = 0; /* Last one to inspect */
+            loop_state = THE_LAST_ONE;
 	}
 	else if (cpdp == sentinel) {
-	    if (has_passed_sentinel) {
+	    if (loop_state == HAS_SEEN_SENTINEL) {
 		/* We been here before. cpool_entrance must have been removed */
 		break;
 	    }
 	    cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev));
 	    if (cpdp == sentinel)
                 break;
-	    has_passed_sentinel = 1;
+            loop_state = HAS_SEEN_SENTINEL;
 	}
-	crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool));
+	crr = ErtsContainerStruct(cpdp, Carrier_t, cpool);
 	exp = erts_smp_atomic_read_rb(&crr->allctr);
-	if (((exp & (ERTS_CRR_ALCTR_FLG_MASK)) == ERTS_CRR_ALCTR_FLG_IN_POOL)
-	    && (erts_atomic_read_nob(&cpdp->max_size) >= size)) {
+	if (((exp & (ERTS_CRR_ALCTR_FLG_IN_POOL |
+                     ERTS_CRR_ALCTR_FLG_BUSY)) == ERTS_CRR_ALCTR_FLG_IN_POOL)
+            && (erts_atomic_read_nob(&cpdp->max_size) >= size)) {
 	    erts_aint_t act;
-	    /* Try to fetch it... */
-	    act = erts_smp_atomic_cmpxchg_mb(&crr->allctr,
-					     (erts_aint_t) allctr,
-					     exp);
+            erts_aint_t want = (((erts_aint_t) allctr)
+                                | (exp & ERTS_CRR_ALCTR_FLG_HOMECOMING));
+            /* Try to fetch it... */
+	    act = erts_smp_atomic_cmpxchg_mb(&crr->allctr, want, exp);
 	    if (act == exp) {
 		cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr);
 		if (crr->cpool.orig_allctr == allctr) {
 		    unlink_abandoned_carrier(crr);
-		}
+                    crr->cpool.state = ERTS_MBC_IS_HOME;
+                }
 		return crr;
 	    }
 	}
+
 	if (--i <= 0)
 	    return NULL;
-    }
+    }while (loop_state != THE_LAST_ONE);
 
 check_dc_list:
     /* Last; check our own pending dealloc carrier list... */
@@ -3420,13 +3401,8 @@ check_dc_list:
 	if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) {
 	    Block_t* blk;
 	    unlink_carrier(&allctr->cpool.dc_list, crr);
-#ifdef ERTS_ALC_CPOOL_DEBUG
-	    ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr,
-							   ((erts_aint_t) allctr))
-				  == (((erts_aint_t) allctr) & ~ERTS_CRR_ALCTR_FLG_MASK));
-#else
-	    erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
-#endif
+	    ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr)
+                                  == ((erts_aint_t) allctr));
 	    blk = MBC_TO_FIRST_BLK(allctr, crr);
 	    ASSERT(FBLK_TO_MBC(blk) == crr);
 	    allctr->link_free_block(allctr, blk);
@@ -3491,9 +3467,6 @@ static void
 schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
 {
     Allctr_t *orig_allctr;
-    Block_t *blk;
-    int check_pending_dealloc;
-    erts_aint_t max_size;
 
     ASSERT(IS_MB_CARRIER(crr));
 
@@ -3504,9 +3477,17 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
 
     orig_allctr = crr->cpool.orig_allctr;
 
-    if (allctr != orig_allctr) {
-	int cinit = orig_allctr->dd.ix - allctr->dd.ix;
-
+    if (allctr == orig_allctr) {
+        if (!(erts_smp_atomic_read_nob(&crr->allctr) & ERTS_CRR_ALCTR_FLG_HOMECOMING)) {
+            dealloc_my_carrier(allctr, crr);
+        }
+        /*else
+         * Carrier was abandoned earlier by other thread and
+         * is still waiting for us in dd-queue.
+         * handle_delayed_dealloc() will handle it when crr is dequeued.
+         */
+    }
+    else {
 	/*
 	 * We send the carrier to its origin for deallocation.
 	 * This in order:
@@ -3515,31 +3496,39 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr)
 	 * - to ensure that we always only reuse empty carriers
 	 *   originating from our own thread specific mseg_alloc
 	 *   instance which is beneficial on NUMA systems.
-	 *
-	 * The receiver will recognize that this is a carrier to
-	 * deallocate (and not a block which is the common case)
-	 * since the block is an mbc block that is free and last
-	 * in the carrier.
 	 */
-	blk = MBC_TO_FIRST_BLK(allctr, crr);
-	ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk));
-
-	ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk));
-	ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(blk));
-	ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, blk));
-	ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr)
-			      == (erts_smp_atomic_read_nob(&crr->allctr)
-				  & ~ERTS_CRR_ALCTR_FLG_MASK));
-
-        blk = &crr->cpool.homecoming_dd.blk;
-        ERTS_ALC_CPOOL_ASSERT(blk->bhdr == HOMECOMING_MBC_BLK_HDR);
-	if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(blk), cinit))
-	    erts_alloc_notify_delayed_dealloc(orig_allctr->ix);
-	return;
+        erts_aint_t iallctr;
+#ifdef ERTS_ALC_CPOOL_DEBUG
+	Block_t* first_blk = MBC_TO_FIRST_BLK(allctr, crr);
+	ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(first_blk));
+
+	ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, first_blk));
+	ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(first_blk));
+	ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, first_blk));
+	ERTS_ALC_CPOOL_ASSERT((erts_smp_atomic_read_nob(&crr->allctr)
+                               & ~ERTS_CRR_ALCTR_FLG_HOMECOMING)
+                              == (erts_aint_t) allctr);
+#endif
+
+        iallctr = (erts_aint_t)orig_allctr | ERTS_CRR_ALCTR_FLG_HOMECOMING;
+        if (!(erts_smp_atomic_xchg_nob(&crr->allctr, iallctr)
+              & ERTS_CRR_ALCTR_FLG_HOMECOMING)) {
+            enqueue_homecoming(allctr, crr);
+        }
     }
+}
 
-    if (is_abandoned(crr))
-	unlink_abandoned_carrier(crr);
+static void dealloc_my_carrier(Allctr_t *allctr, Carrier_t *crr)
+{
+    Block_t *blk;
+    int check_pending_dealloc;
+    erts_aint_t max_size;
+
+    ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr);
+    if (is_abandoned(crr)) {
+        unlink_abandoned_carrier(crr);
+        crr->cpool.state = ERTS_MBC_IS_HOME;
+    }
 
     if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID
 	|| erts_thr_progress_has_reached(crr->cpool.thr_prgr)) {
@@ -3590,8 +3579,7 @@ cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr)
 	    limit = (csz/100)*allctr->cpool.util_limit;
 	crr->cpool.abandon_limit = limit;
     }
-    crr->cpool.abandoned.next = NULL;
-    crr->cpool.abandoned.prev = NULL;
+    crr->cpool.state = ERTS_MBC_IS_HOME;
 }
 
 static void
@@ -3618,22 +3606,64 @@ static void
 abandon_carrier(Allctr_t *allctr, Carrier_t *crr)
 {
     erts_aint_t max_size;
+    erts_aint_t iallctr;
 
-    STAT_MBC_CPOOL_INSERT(allctr, crr);
+    STAT_MBC_ABANDON(allctr, crr);
 
     unlink_carrier(&allctr->mbc_list, crr);
-    if (crr->cpool.orig_allctr == allctr) {
-	link_abandoned_carrier(&allctr->cpool.pooled_list, crr);
-    }
-
     allctr->remove_mbc(allctr, crr);
+    set_new_allctr_abandon_limit(allctr);
 
     max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr);
     erts_atomic_set_nob(&crr->cpool.max_size, max_size);
-
     cpool_insert(allctr, crr);
 
-    set_new_allctr_abandon_limit(allctr);
+
+    iallctr = erts_smp_atomic_read_nob(&crr->allctr);
+    if (allctr == crr->cpool.orig_allctr) {
+        /* preserve HOMECOMING flag */
+        ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) == (erts_aint_t)allctr);
+        erts_smp_atomic_set_wb(&crr->allctr, iallctr | ERTS_CRR_ALCTR_FLG_IN_POOL);
+        poolify_my_carrier(allctr, crr);
+    }
+    else {
+        ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) == (erts_aint_t)allctr);
+        iallctr = ((erts_aint_t)crr->cpool.orig_allctr |
+                   ERTS_CRR_ALCTR_FLG_HOMECOMING |
+                   ERTS_CRR_ALCTR_FLG_IN_POOL);
+        if (!(erts_smp_atomic_xchg_wb(&crr->allctr, iallctr)
+              & ERTS_CRR_ALCTR_FLG_HOMECOMING)) {
+
+            enqueue_homecoming(allctr, crr);
+        }
+    }
+}
+
+static void
+enqueue_homecoming(Allctr_t* allctr, Carrier_t* crr)
+{
+    Allctr_t* orig_allctr = crr->cpool.orig_allctr;
+    const int cinit = orig_allctr->dd.ix - allctr->dd.ix;
+    Block_t* dd_blk = &crr->cpool.homecoming_dd.blk;
+
+    /*
+     * The receiver will recognize this as a carrier
+     * (and not a block which is the common case)
+     * since the block header is HOMECOMING_MBC_BLK_HDR.
+     */
+    ASSERT(dd_blk->bhdr == HOMECOMING_MBC_BLK_HDR);
+    if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(dd_blk), cinit))
+        erts_alloc_notify_delayed_dealloc(orig_allctr->ix);
+}
+
+static void
+poolify_my_carrier(Allctr_t *allctr, Carrier_t *crr)
+{
+    ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr);
+
+    crr->cpool.pooled.hdr.bhdr = erts_atomic_read_nob(&crr->cpool.max_size);
+    aoff_add_pooled_mbc(allctr, crr);
+    crr->cpool.state = ERTS_MBC_WAS_POOLED;
 }
 
 static void
@@ -4153,16 +4183,21 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
 
 #ifdef ERTS_SMP
 	if (busy_pcrr_pp && *busy_pcrr_pp) {
+            erts_aint_t iallctr = erts_smp_atomic_read_nob(&crr->allctr);
 	    ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr);
-	    *busy_pcrr_pp = NULL;
-	    ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr)
-				  == (((erts_aint_t) allctr)
-				      | ERTS_CRR_ALCTR_FLG_IN_POOL
-				      | ERTS_CRR_ALCTR_FLG_BUSY));
-	    erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr));
+            ERTS_ALC_CPOOL_ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING)
+                                  == (((erts_aint_t) allctr)
+                                      | ERTS_CRR_ALCTR_FLG_IN_POOL
+                                      | ERTS_CRR_ALCTR_FLG_BUSY));
+            ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr);
+
+            *busy_pcrr_pp = NULL;
+	    erts_smp_atomic_set_nob(&crr->allctr,
+                                    (iallctr & ~(ERTS_CRR_ALCTR_FLG_IN_POOL |
+                                                 ERTS_CRR_ALCTR_FLG_BUSY)));
 	    cpool_delete(allctr, allctr, crr);
 	}
-	else
+        else
 #endif
 	{
 	    unlink_carrier(&allctr->mbc_list, crr);
@@ -5565,12 +5600,13 @@ erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p)
 	pref_allctr = get_pref_allctr(extra);
 	used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_IF_USED,
 				      p, NULL, &busy_pcrr_p);
-	if (pref_allctr != used_allctr)
+	if (pref_allctr != used_allctr) {
 	    enqueue_dealloc_other_instance(type,
-					   used_allctr,
-					   p,
-					   (used_allctr->dd.ix
-					    - pref_allctr->dd.ix));
+                                           used_allctr,
+                                           p,
+                                           (used_allctr->dd.ix
+                                            - pref_allctr->dd.ix));
+        }
 	else {
 	    ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr);
 	    do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p);
@@ -6040,10 +6076,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 	    allctr->min_block_size = sz;
     }
 
-    allctr->cpool.pooled_list.next  = &allctr->cpool.pooled_list;
-    allctr->cpool.pooled_list.prev  = &allctr->cpool.pooled_list;
-    allctr->cpool.traitor_list.next = &allctr->cpool.traitor_list;
-    allctr->cpool.traitor_list.prev = &allctr->cpool.traitor_list;
+    allctr->cpool.pooled_tree = NULL;
     allctr->cpool.dc_list.first = NULL;
     allctr->cpool.dc_list.last = NULL;
     allctr->cpool.abandon_limit = 0;
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 1291a6c10e..e5bf494e97 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -307,6 +307,7 @@ typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t;
 
 typedef struct Carrier_t_ Carrier_t;
 
+
 typedef struct {
     UWord bhdr;
 #if !MBC_ABLK_OFFSET_BITS
@@ -373,13 +374,24 @@ typedef struct {
 
 typedef UWord FreeBlkFtr_t; /* Footer of a free block */
 
-
+/* This AOFF stuff really belong in erl_ao_firstfit_alloc.h */
+typedef struct AOFF_RBTree_t_ AOFF_RBTree_t;
+struct AOFF_RBTree_t_ {
+    Block_t hdr;
+    AOFF_RBTree_t *parent;
+    AOFF_RBTree_t *left;
+    AOFF_RBTree_t *right;
+    Uint32 flags;
+    Uint32 max_sz;  /* of all blocks in this sub-tree */
+};
 #ifdef ERTS_SMP
+void aoff_add_pooled_mbc(Allctr_t*, Carrier_t*);
+void aoff_remove_pooled_mbc(Allctr_t*, Carrier_t*);
+Carrier_t* aoff_lookup_pooled_mbc(Allctr_t*, Uint size);
+void erts_aoff_larger_max_size(AOFF_RBTree_t *node);
+#endif
 
-typedef struct ErtsDoubleLink_t_ {
-    struct ErtsDoubleLink_t_ *next;
-    struct ErtsDoubleLink_t_ *prev;
-}ErtsDoubleLink_t;
+#ifdef ERTS_SMP
 
 typedef struct {
     ErtsFakeDDBlock_t homecoming_dd;
@@ -391,7 +403,12 @@ typedef struct {
     UWord abandon_limit;
     UWord blocks;
     UWord blocks_size;
-    ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */
+    enum {
+        ERTS_MBC_IS_HOME,
+        ERTS_MBC_WAS_POOLED,
+        ERTS_MBC_WAS_TRAITOR
+    } state;
+    AOFF_RBTree_t pooled;  /* node in pooled_tree */
 } ErtsAlcCPoolData_t;
 
 #endif
@@ -566,15 +583,14 @@ struct Allctr_t_ {
     UWord               crr_set_flgs;
     UWord               crr_clr_flgs;
 
-    /* Carriers */
+    /* Carriers *employed* by this allocator */
     CarrierList_t	mbc_list;
     CarrierList_t	sbc_list;
 #ifdef ERTS_SMP
     struct {
-	/* pooled_list, traitor list and dc_list contain only
-           carriers _created_ by this allocator */
-	ErtsDoubleLink_t pooled_list;
-	ErtsDoubleLink_t traitor_list;
+	/* pooled_tree and dc_list contain only
+           carriers *created* by this allocator */
+	AOFF_RBTree_t*   pooled_tree;
 	CarrierList_t	 dc_list;
 
 	UWord		abandon_limit;
@@ -686,7 +702,6 @@ void erts_alcu_assert_failed(char* expr, char* file, int line, char *func);
 int is_sbc_blk(Block_t*);
 #endif
 
-
 #endif /* #if defined(GET_ERL_ALLOC_UTIL_IMPL)
 	      && !defined(ERL_ALLOC_UTIL_IMPL__) */
 
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index 05ba1f9891..b781db152f 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -92,18 +92,6 @@
 #define RBT_ASSERT(x)
 #endif
 
-
-/* Types... */
-typedef struct AOFF_RBTree_t_ AOFF_RBTree_t;
-
-struct AOFF_RBTree_t_ {
-    Block_t hdr;
-    AOFF_RBTree_t *parent;
-    AOFF_RBTree_t *left;
-    AOFF_RBTree_t *right;
-    Uint32 flags;
-    Uint32 max_sz;  /* of all blocks in this sub-tree */
-};
 #define AOFF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr)
 
 /* BF block nodes keeps list of all with equal size
@@ -120,7 +108,7 @@ typedef struct AOFF_Carrier_t_ AOFF_Carrier_t;
 
 struct AOFF_Carrier_t_ {
     Carrier_t crr;
-    AOFF_RBTree_t rbt_node;     /* My node in the carrier tree */
+    AOFF_RBTree_t rbt_node;   /* My node in the carrier tree */
     AOFF_RBTree_t* root;        /* Root of my block tree */
 };
 #define RBT_NODE_TO_MBC(PTR) ErtsContainerStruct((PTR), AOFF_Carrier_t, rbt_node)
@@ -136,8 +124,9 @@ struct AOFF_Carrier_t_ {
 */
 
 #ifdef HARD_DEBUG
-#  define HARD_CHECK_IS_MEMBER(ROOT,NODE) rbt_assert_is_member(ROOT,NODE)
+#  define HARD_CHECK_IS_MEMBER(ROOT,NODE) ASSERT(rbt_is_member(ROOT,NODE))
 #  define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) check_tree(CRR, FLV, ROOT, SZ)
+static int rbt_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node);
 static AOFF_RBTree_t * check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint);
 #else
 #  define HARD_CHECK_IS_MEMBER(ROOT,NODE)
@@ -179,6 +168,27 @@ static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node,
     else ASSERT(new_max == old_max);
 }
 
+#ifdef ERTS_SMP
+/*
+ * Set possibly new larger 'max_sz' of node and propagate change toward root
+ */
+void erts_aoff_larger_max_size(AOFF_RBTree_t *node)
+{
+    AOFF_RBTree_t* x = node;
+    const Uint new_sz = node->hdr.bhdr;
+
+    ASSERT(!x->left  || x->left->max_sz  <= x->max_sz);
+    ASSERT(!x->right || x->right->max_sz <= x->max_sz);
+
+    while (new_sz > x->max_sz) {
+        x->max_sz = new_sz;
+        x = x->parent;
+        if (!x)
+            break;
+    }
+}
+#endif
+
 static ERTS_INLINE SWord cmp_blocks(enum AOFF_Flavor flavor,
 				    AOFF_RBTree_t* lhs, AOFF_RBTree_t* rhs)
 {
@@ -220,9 +230,6 @@ static UWord aoff_largest_fblk_in_mbc(Allctr_t*, Carrier_t*);
 static void rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del);
 static void rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk);
 static AOFF_RBTree_t* rbt_search(AOFF_RBTree_t* root, Uint size);
-#ifdef HARD_DEBUG
-static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node);
-#endif
 
 static Eterm info_options(Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *);
 static void init_atoms(void);
@@ -719,19 +726,20 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block)
 
     rbt_insert(alc->flavor, &blk_crr->root, blk);
 
-    /* Update the carrier tree with a potentially new (larger) max_sz
-    */
+    /*
+     * Update carrier tree with a potentially new (larger) max_sz
+     */
     crr_node = &blk_crr->rbt_node;
     if (blk_sz > crr_node->hdr.bhdr) {
-	ASSERT(blk_sz == blk_crr->root->max_sz);
-	crr_node->hdr.bhdr = blk_sz;
-	while (blk_sz > crr_node->max_sz) {
-	    crr_node->max_sz = blk_sz;
-	    crr_node = crr_node->parent;
-	    if (!crr_node) break;
-	}
+        ASSERT(blk_sz == blk_crr->root->max_sz);
+        crr_node->hdr.bhdr = blk_sz;
+        while (blk_sz > crr_node->max_sz) {
+            crr_node->max_sz = blk_sz;
+            crr_node = crr_node->parent;
+            if (!crr_node) break;
+        }
     }
-    HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0);
+    HARD_CHECK_TREE(NULL, AOFF_AOFF, alc->mbc_root, 0);
 }
 
 static void
@@ -826,6 +834,18 @@ rbt_search(AOFF_RBTree_t* root, Uint size)
     }
 }
 
+#ifdef ERTS_SMP
+Carrier_t* aoff_lookup_pooled_mbc(Allctr_t* allctr, Uint size)
+{
+    AOFF_RBTree_t* node;
+
+    if (!allctr->cpool.pooled_tree)
+	return NULL;
+    node = rbt_search(allctr->cpool.pooled_tree, size);
+    return node ? ErtsContainerStruct(node, Carrier_t, cpool.pooled) : NULL;
+}
+#endif
+
 static Block_t *
 aoff_get_free_block(Allctr_t *allctr, Uint size,
 		    Block_t *cand_blk, Uint cand_size)
@@ -920,16 +940,31 @@ static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier)
     HARD_CHECK_TREE(NULL, 0, *root, 0);
 }
 
+#ifdef ERTS_SMP
+void aoff_add_pooled_mbc(Allctr_t *allctr, Carrier_t *crr)
+{
+    AOFF_RBTree_t **root = &allctr->cpool.pooled_tree;
+
+    ASSERT(allctr == crr->cpool.orig_allctr);
+    HARD_CHECK_TREE(NULL, 0, *root, 0);
+
+    /* Link carrier in address order tree
+     */
+    rbt_insert(AOFF_AOFF, root, &crr->cpool.pooled);
+
+    HARD_CHECK_TREE(NULL, 0, *root, 0);
+}
+#endif
+
 static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier)
 {
-    AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr;
-    AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier;
-    AOFF_RBTree_t **root = &alc->mbc_root;
+    AOFF_RBTree_t **root = &((AOFFAllctr_t*)allctr)->mbc_root;
+    AOFF_Carrier_t *crr = (AOFF_Carrier_t*)carrier;
 
     ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier));
 
     if (!IS_CRR_IN_TREE(crr,*root))
-        return;
+	return;
 
     HARD_CHECK_TREE(NULL, 0, *root, 0);
 
@@ -942,6 +977,26 @@ static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier)
     HARD_CHECK_TREE(NULL, 0, *root, 0);
 }
 
+#ifdef ERTS_SMP
+void aoff_remove_pooled_mbc(Allctr_t *allctr, Carrier_t *crr)
+{
+    ASSERT(allctr == crr->cpool.orig_allctr);
+
+    HARD_CHECK_TREE(NULL, 0, allctr->cpool.pooled_tree, 0);
+
+    rbt_delete(&allctr->cpool.pooled_tree, &crr->cpool.pooled);
+#ifdef DEBUG
+    crr->cpool.pooled.parent = NULL;
+    crr->cpool.pooled.left = NULL;
+    crr->cpool.pooled.right = NULL;
+    crr->cpool.pooled.max_sz = 0;
+#endif
+    HARD_CHECK_TREE(NULL, 0, allctr->cpool.pooled_tree, 0);
+
+}
+#endif
+
+
 static UWord aoff_largest_fblk_in_mbc(Allctr_t* allctr, Carrier_t* carrier)
 {
     AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier;
@@ -1085,12 +1140,13 @@ erts_aoffalc_test(UWord op, UWord a1, UWord a2)
 
 
 #ifdef HARD_DEBUG
-
-static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node)
+static int rbt_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node)
 {
     while (node != root) {
-	ASSERT(node->parent);
-	ASSERT(node->parent->left == node || node->parent->right == node);
+        if (!node->parent || (node->parent->left != node &&
+                              node->parent->right != node)) {
+            return 0;
+        }
 	node = node->parent;
     }    
     return 1;
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h
index 7349c6ab19..a3fca7d3e3 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.h
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h
@@ -53,7 +53,6 @@ Allctr_t *erts_aoffalc_start(AOFFAllctr_t *, AOFFAllctrInit_t*, AllctrInit_t *);
 #define GET_ERL_ALLOC_UTIL_IMPL
 #include "erl_alloc_util.h"
 
-
 struct AOFFAllctr_t_ {
     Allctr_t		allctr; /* Has to be first! */
 
diff --git a/erts/emulator/internal_doc/CarrierMigration.md b/erts/emulator/internal_doc/CarrierMigration.md
index 2a9594db25..3a796d11b7 100644
--- a/erts/emulator/internal_doc/CarrierMigration.md
+++ b/erts/emulator/internal_doc/CarrierMigration.md
@@ -3,17 +3,17 @@ Carrier Migration
 
 The ERTS memory allocators manage memory blocks in two types of raw
 memory chunks. We call these chunks of raw memory
-*carriers*. Singleblock carriers which only contain one large block,
-and multiblock carriers which contain multiple blocks. A carrier is
+*carriers*. Single-block carriers which only contain one large block,
+and multi-block carriers which contain multiple blocks. A carrier is
 typically created using `mmap()` on unix systems. However, how a
 carrier is created is of minor importance. An allocator instance
-typically manages a mixture of single- and multiblock carriers.
+typically manages a mixture of single- and multi-block carriers.
 
 Problem
 -------
 
 When a carrier is empty, i.e. contains only one large free block, it
-is deallocated. Since multiblock carriers can contain both allocated
+is deallocated. Since multi-block carriers can contain both allocated
 blocks and free blocks at the same time, an allocator instance might
 be stuck with a large amount of poorly utilized carriers if the memory
 load decreases. After a peak in memory usage it is expected that not
@@ -23,9 +23,9 @@ can usually be reused if the memory load increases again. However,
 since each scheduler thread manages its own set of allocator
 instances, and memory load is not necessarily correlated to CPU load, we
 might get into a situation where there are lots of poorly utilized
-multiblock carriers on some allocator instances while we need to
-allocate new multiblock carriers on other allocator instances. In
-scenarios like this, the demand for multiblock carriers in the system
+multi-block carriers on some allocator instances while we need to
+allocate new multi-block carriers on other allocator instances. In
+scenarios like this, the demand for multi-block carriers in the system
 might increase at the same time as the actual memory demand in the
 system has decreased which is both unwanted and quite unexpected for
 the end user.
@@ -34,7 +34,7 @@ Solution
 --------
 
 In order to prevent scenarios like this we've implemented support for
-migration of multiblock carriers between allocator instances of the
+migration of multi-block carriers between allocator instances of the
 same type.
 
 ### Management of Free Blocks ###
@@ -44,7 +44,7 @@ and add it to another we need to be able to move references to the
 free blocks of the carrier between the allocator instances. The
 allocator instance specific data structure referring to the free
 blocks it manages often refers to the same carrier from multiple
-places. For example, when the address order bestfit strategy is used
+places. For example, when the address order best-fit strategy is used
 this data structure is a binary search tree spanning all carriers that
 the allocator instance manages. Free blocks in one specific carrier
 can be referred to from potentially every other carrier that is
@@ -135,7 +135,7 @@ carriers between scheduler specific allocator instances of the same
 allocator type.
 
 Each allocator instance keeps track of the current utilization of its
-multiblock carriers. When the total utilization falls below the "abandon
+multi-block carriers. When the total utilization falls below the "abandon
 carrier utilization limit" it starts to inspect the utilization of the
 current carrier when deallocations are made. If also the utilization
 of the carrier falls below the "abandon carrier utilization limit" it
@@ -144,31 +144,45 @@ and inserts the carrier into the pool.
 
 Since the carrier has been unlinked from the data structure of
 available free blocks, no more allocations will be made in the
-carrier. The allocator instance putting the carrier into the pool,
-however, still has the responsibility of performing deallocations in
-it while it remains in the pool. The allocator instance with this
-deallocation responsibility is here called the **employer**.
-
-Each carrier has a flag field containing information about the
-employing allocator instance, a flag indicating if the carrier is in
-the pool or not, and a flag indicating if it is busy or not. When the
-carrier is in the pool, the employing allocator instance needs to mark it
-as busy while operating on it. If another thread inspects it in order
-to try to fetch it from the pool, it will skip it if it is busy. When
-fetching the carrier from the pool, employment will change and further
+carrier.
+
+The allocator instance that created a carrier is called its **owner**.
+Ownership never changes.
+
+The allocator instance that has the responsibility to perform deallocations in a
+carrier is called its **employer**. The employer may also perform allocations if
+the carrier is not in the pool. Employment may change when a carrier is fetched from
+or inserted into the pool.
+
+Deallocations in a carrier, while it remains in the pool, is always performed
+the owner. That is, all pooled carriers are employed by their owners.
+
+Each carrier has an atomic word containing a pointer to the employing allocator
+instance and three bit flags; IN_POOL, BUSY and HOMECOMING.
+
+When fetching a carrier from the pool, employment may change and further
 deallocations in the carrier will be redirected to the new
 employer using the delayed dealloc functionality.
 
-If a carrier in the pool becomes empty, it will be withdrawn from the
-pool. All carriers that become empty are also always passed to its
-**owning** allocator instance for deallocation using the delayed
-dealloc functionality. Since carriers this way always will be
-deallocated by the owner that allocated the carrier, the
+When a foreign allocator instance abandons a carrier back into the pool, it will
+also pass it back to its **owner** using the delayed dealloc queue. When doing
+this it will set the HOMECOMING bit flag to mark it as "enqueued". The owner
+will later clear the HOMECOMING bit when the carrier is dequeued. This mechanism
+prevents a carrier from being enqueued again before it has been dequeued.
+
+When a carrier becomes empty, it will be deallocated. Carrier deallocation is
+always done by the owner that allocated the carrier. By doing this, the
 underlying functionality of allocating and deallocating carriers can
 remain simple and doesn't have to bother about multiple threads. In a
 NUMA system we will also not mix carriers originating from multiple
 NUMA nodes.
 
+If a carrier in the pool becomes empty, it will be withdrawn from the
+pool and be deallocated by the owner which already employs it.
+
+If a carrier employed by a foreign allocator becomes empty, it will be passed
+back to the owner for deallocation using the delayed dealloc functionality.
+
 In short:
 
 * The allocator instance that created a carrier **owns** it.
@@ -177,34 +191,31 @@ In short:
 * The allocator instance that uses a carrier **employs** it.
 * An **employer** can abandon a carrier into the pool.
 * Pooled carriers are not allocated from.
-* Deallocation in a pooled carrier is still performed by its **employer**.
-* **Employment** can only change when a carrier is fetched from the pool.
+* Pooled carriers are always **employed** by their **owner**.
+* **Employment** can only change from **owner** to a foreign allocator
+  when a carrier is fetched from the pool.
+
 
 ### Searching the pool ###
 
+When an allocator instance needs more carrier space, it inspects the pool. If no
+carrier could be fetched from the pool, it will allocate a new
+carrier. Regardless of where the allocator instance gets the carrier from, it
+just links in the carrier into its data structure of free blocks.
+
 To harbor real time characteristics, searching the pool is
 limited. We only inspect a limited number of carriers. If none of
 those carriers had a free block large enough to satisfy the allocation
-request, the search will fail. A carrier in the pool can also be busy
+request, the search will fail. A carrier in the pool can also be BUSY
 if another thread is currently doing block deallocation work on the
-carrier. A busy carrier will also be skipped by the search as it can
+carrier. A BUSY carrier will also be skipped by the search as it can
 not satisfy the request. The pool is lock-free and we do not want to
 block, waiting for the other thread to finish.
 
-#### Before OTP 17.4 ####
+### The bad cluster problem ###
 
-When an allocator instance needs more carrier space, it always begins
-by inspecting its own carriers that are waiting for thread progress
-before they can be deallocated. If no such carrier could be found, it
-then inspects the pool. If no carrier could be fetched from the pool,
-it will allocate a new carrier. Regardless of where the allocator
-instance gets the carrier from it the just links in the carrier into
-its data structure of free blocks.
-
-#### After OTP 17.4 ####
-
-The old search algorithm had a problem as the search always started at
-the same position in the pool, the sentinel. This could lead to
+Before OTP-17.4 the search algorithm had a problem as the search always started
+at the same position in the pool, the sentinel. This could lead to
 contention from concurrent searching processes. But even worse, it
 could lead to a "bad" state when searches fail with a high rate
 leading to new carriers instead being allocated. These new carriers
@@ -236,26 +247,27 @@ The result is that we prefer carriers created by the thread itself,
 which is good for NUMA performance. And we get more entry points when
 searching the pool, which will ease contention and clustering.
 
+### Our own pooled tree ###
+
 To do the first search among own carriers, every allocator instance
-has two new lists: `pooled_list` and `traitor_list`. These lists are only
-accessed by the allocator itself and they only contain the allocator's
-own carriers. When an owned carrier is abandoned and put in the
-pool, it is also linked into `pooled_list`. When we search our
-`pooled_list` and find a carrier that is no longer in the pool, we
-move that carrier from `pooled_list` to `traitor_list` as it is now
-employed by another allocator. If searching `pooled_list` fails, we
-also do a limited search of `traitor_list`. When finding an abandoned
-carrier in `traitor_list` it is either employed or moved back to
-`pooled_list` if it could not satisfy the allocation request.
-
-When searching `pooled_list` and `traitor_list` we always start at the
-point where the last search ended. This to avoid clustering
-problems and increase the probability to find a "good" carrier. As
-`pooled_list` and `traitor_list` are only accessed by the owning
-allocator instance, they need no thread synchronization at all.
+has a `pooled_tree` of carriers. This tree is only accessed by the allocator
+itself and can only contain its own carriers. When a carrier is
+abandoned and put in the pool, it is also inserted into `pooled_tree`. This is
+either done direct, if the carrier was already employed by its owner, or by
+first passing it back to the owner via the delayed dealloc queue.
+
+When we search our `pooled_tree` and find a carrier that is no longer in the
+pool, we remove that carrier from `pooled_tree` and mark it as TRAITOR, as it is
+now employed by a foreign allocator. We will not find any carriers in
+`pooled_tree` that are marked as BUSY by other threads.
+
+If no carrier in `pooled_tree` had a large enough free block, we search it again
+to find any carrier that may act as an entry point into the shared list of all
+pooled carriers. This in order to, if possible, avoid starting at the sentinel
+and thereby ease the "bad clustering" problem.
 
 Furthermore, the search for own carriers that are scheduled
-for deallocation is now done as the last search option. The idea is
+for deallocation is done as the last search option. The idea is
 that it is better to reuse a poorly utilized carrier than to
 resurrect an empty carrier that was just about to be released back to
 the OS.
@@ -271,14 +283,14 @@ load did not.
 When using the `aoffcaobf` or `aoff` strategies compared to `gf` or
 `bf`, we loose some performance since we get more modifications in the
 data structure of free blocks. This performance penalty is however
-reduced using the `aoffcbf` strategy. A tradeoff between memory
+reduced using the `aoffcbf` strategy. A trade off between memory
 consumption and performance is however inevitable, and it is up to
 the user to decide what is most important. 
 
 Further work
 ------------
 
-It would be quite easy to extend this to allow migration of multiblock
+It would be quite easy to extend this to allow migration of multi-block
 carriers between all allocator types. More or less the only obstacle
 is maintenance of the statistics information.
 
-- 
cgit v1.2.3


From 6789b20533b4a848953fa7af34d9ba3a4c89c5ca Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 13 Dec 2017 20:14:56 +0100
Subject: erts: Fix alloc_SUITE:migration

It crashed due to recursive calls to alloc_util
in carrier initialization test callback.
---
 erts/emulator/test/alloc_SUITE.erl | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl
index 518e0c974a..022cda88e8 100644
--- a/erts/emulator/test/alloc_SUITE.erl
+++ b/erts/emulator/test/alloc_SUITE.erl
@@ -67,7 +67,10 @@ cpool(Cfg) -> drv_case(Cfg).
 migration(Cfg) ->
     case erlang:system_info(smp_support) of
 	true ->
-	    drv_case(Cfg, concurrent, "+MZe true");
+            %% Enable test_alloc.
+            %% Disable driver_alloc to avoid recursive alloc_util calls
+            %% through enif_mutex_create() in my_creating_mbc().
+	    drv_case(Cfg, concurrent, "+MZe true +MRe false");
 	false ->
 	    {skipped, "No smp"}
     end.
-- 
cgit v1.2.3


From d74796ecb17a68d442e846c4032a57cb2c083686 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 18 Dec 2017 18:41:38 +0100
Subject: erts: Add more stats for mbcs_pool

similar to the ones in OTP-19.2.3.1
---
 erts/emulator/beam/erl_alloc_util.c | 105 +++++++++++++++++++++++++++++++++---
 erts/emulator/beam/erl_alloc_util.h |  11 ++++
 2 files changed, 110 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index dbf27f1d2f..970facce09 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -3267,6 +3267,7 @@ cpool_fetch(Allctr_t *allctr, UWord size)
                 ASSERT(exp & ERTS_CRR_ALCTR_FLG_HOMECOMING);
                 crr->cpool.pooled.hdr.bhdr = erts_atomic_read_nob(&crr->cpool.max_size);
                 aoff_add_pooled_mbc(allctr, crr);
+                INC_CC(allctr->cpool.stat.skip_size);
                 continue;
             }
             else if (exp & ERTS_CRR_ALCTR_FLG_BUSY) {
@@ -3277,6 +3278,7 @@ cpool_fetch(Allctr_t *allctr, UWord size)
                  */
                 ASSERT(!reinsert_crr);
                 reinsert_crr = crr;
+                INC_CC(allctr->cpool.stat.skip_busy);
                 continue;
             }
 
@@ -3293,7 +3295,10 @@ cpool_fetch(Allctr_t *allctr, UWord size)
                 return crr;
             }
             exp = act;
+            INC_CC(allctr->cpool.stat.skip_race);
         }
+        else
+            INC_CC(allctr->cpool.stat.skip_not_pooled);
 
         /* Not in pool anymore */
         ASSERT(!(exp & ERTS_CRR_ALCTR_FLG_BUSY));
@@ -3321,8 +3326,10 @@ cpool_fetch(Allctr_t *allctr, UWord size)
         aoff_remove_pooled_mbc(allctr, crr);
         crr->cpool.state = ERTS_MBC_WAS_TRAITOR;
 
-        if (--i <= 0)
+        if (--i <= 0) {
+            INC_CC(allctr->cpool.stat.fail_pooled);
             return NULL;
+        }
     }
 
 
@@ -3363,6 +3370,7 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 	else if (cpdp == sentinel) {
 	    if (loop_state == HAS_SEEN_SENTINEL) {
 		/* We been here before. cpool_entrance must have been removed */
+                INC_CC(allctr->cpool.stat.entrance_removed);
 		break;
 	    }
 	    cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev));
@@ -3372,9 +3380,12 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 	}
 	crr = ErtsContainerStruct(cpdp, Carrier_t, cpool);
 	exp = erts_smp_atomic_read_rb(&crr->allctr);
-	if (((exp & (ERTS_CRR_ALCTR_FLG_IN_POOL |
-                     ERTS_CRR_ALCTR_FLG_BUSY)) == ERTS_CRR_ALCTR_FLG_IN_POOL)
-            && (erts_atomic_read_nob(&cpdp->max_size) >= size)) {
+
+        if (erts_atomic_read_nob(&cpdp->max_size) < size) {
+            INC_CC(allctr->cpool.stat.skip_size);
+        }
+        else if ((exp & (ERTS_CRR_ALCTR_FLG_IN_POOL | ERTS_CRR_ALCTR_FLG_BUSY))
+                  == ERTS_CRR_ALCTR_FLG_IN_POOL) {
 	    erts_aint_t act;
             erts_aint_t want = (((erts_aint_t) allctr)
                                 | (exp & ERTS_CRR_ALCTR_FLG_HOMECOMING));
@@ -3390,8 +3401,15 @@ cpool_fetch(Allctr_t *allctr, UWord size)
 	    }
 	}
 
-	if (--i <= 0)
+        if (exp & ERTS_CRR_ALCTR_FLG_BUSY)
+            INC_CC(allctr->cpool.stat.skip_busy);
+        else
+            INC_CC(allctr->cpool.stat.skip_race);
+
+	if (--i <= 0) {
+            INC_CC(allctr->cpool.stat.fail_shared);
 	    return NULL;
+        }
     }while (loop_state != THE_LAST_ONE);
 
 check_dc_list:
@@ -3409,10 +3427,15 @@ check_dc_list:
 	    return crr;
 	}
 	crr = crr->prev;
-	if (--i <= 0)
+	if (--i <= 0) {
+            INC_CC(allctr->cpool.stat.fail_pend_dealloc);
 	    return NULL;
+        }
     }
 
+    if (i != ERTS_ALC_CPOOL_MAX_FETCH_INSPECT)
+        INC_CC(allctr->cpool.stat.fail);
+
     return NULL;
 }
 
@@ -3822,6 +3845,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 	crr = cpool_fetch(allctr, blk_sz);
 	if (crr) {
 	    STAT_MBC_CPOOL_FETCH(allctr, crr);
+            INC_CC(allctr->cpool.stat.fetch);
 	    link_carrier(&allctr->mbc_list, crr);
 	    (*allctr->add_mbc)(allctr, crr);
 	    blk = (*allctr->get_free_block)(allctr, blk_sz, NULL, 0);
@@ -4278,6 +4302,17 @@ static struct {
     Eterm mbcs;
 #ifdef ERTS_SMP
     Eterm mbcs_pool;
+    Eterm fetch;
+    Eterm fail_pooled;
+    Eterm fail_shared;
+    Eterm fail_pend_dealloc;
+    Eterm fail;
+    Eterm skip_size;
+    Eterm skip_busy;
+    Eterm skip_not_pooled;
+    Eterm skip_homecoming;
+    Eterm skip_race;
+    Eterm entrance_removed;
 #endif
     Eterm sbcs;
 
@@ -4368,6 +4403,17 @@ init_atoms(Allctr_t *allctr)
 	AM_INIT(mbcs);
 #ifdef ERTS_SMP
 	AM_INIT(mbcs_pool);
+	AM_INIT(fetch);
+        AM_INIT(fail_pooled);
+        AM_INIT(fail_shared);
+        AM_INIT(fail_pend_dealloc);
+        AM_INIT(fail);
+        AM_INIT(skip_size);
+        AM_INIT(skip_busy);
+        AM_INIT(skip_not_pooled);
+        AM_INIT(skip_homecoming);
+        AM_INIT(skip_race);
+        AM_INIT(entrance_removed);
 #endif
 	AM_INIT(sbcs);
 
@@ -4653,9 +4699,56 @@ info_cpool(Allctr_t *allctr,
 
     if (hpp || szp) {
 	res = NIL;
+
+      if (!sz_only) {
+        add_3tup(hpp, szp, &res, am.fail_pooled,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_pooled)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_pooled)));
+
+        add_3tup(hpp, szp, &res, am.fail_shared,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_shared)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_shared)));
+
+        add_3tup(hpp, szp, &res, am.fail_pend_dealloc,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_pend_dealloc)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_pend_dealloc)));
+
+        add_3tup(hpp, szp, &res, am.fail,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail)));
+
+        add_3tup(hpp, szp, &res, am.fetch,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fetch)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fetch)));
+
+        add_3tup(hpp, szp, &res, am.skip_size,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_size)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_size)));
+
+        add_3tup(hpp, szp, &res, am.skip_busy,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_busy)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_busy)));
+
+        add_3tup(hpp, szp, &res, am.skip_not_pooled,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_not_pooled)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_not_pooled)));
+
+        add_3tup(hpp, szp, &res, am.skip_homecoming,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_homecoming)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_homecoming)));
+
+        add_3tup(hpp, szp, &res, am.skip_race,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_race)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_race)));
+
+        add_3tup(hpp, szp, &res, am.entrance_removed,
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.entrance_removed)),
+                 bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.entrance_removed)));
+
 	add_2tup(hpp, szp, &res,
 		 am.carriers_size,
 		 bld_unstable_uint(hpp, szp, csz));
+      }
 	if (!sz_only)
 	    add_2tup(hpp, szp, &res,
 		     am.carriers,
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index e5bf494e97..19165d83d9 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -602,6 +602,17 @@ struct Allctr_t_ {
 	    erts_atomic_t	no_blocks;
 	    erts_atomic_t	carriers_size;
 	    erts_atomic_t	no_carriers;
+            CallCounter_t       fail_pooled;
+            CallCounter_t       fail_shared;
+            CallCounter_t       fail_pend_dealloc;
+            CallCounter_t       fail;
+            CallCounter_t       fetch;
+	    CallCounter_t       skip_size;
+	    CallCounter_t       skip_busy;
+	    CallCounter_t       skip_not_pooled;
+	    CallCounter_t       skip_homecoming;
+	    CallCounter_t       skip_race;
+	    CallCounter_t       entrance_removed;
 	} stat;
     } cpool;
 #endif
-- 
cgit v1.2.3


From c2d70945dce9cb09d5d7120d6e9ddf7faac8d230 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 22 Nov 2017 13:19:57 +0100
Subject: Replace the libc environment with a thread-safe emulation

putenv(3) and friends aren't thread-safe regardless of how you slice
it; a global lock around all environment operations (like before)
keeps things safe as far as our own operations go, but we have
absolutely no control over what libc or a library dragged in by a
driver/NIF does -- they're free to call getenv(3) or putenv(3)
without honoring our lock.

This commit solves this by setting up an "emulated" environment which
can't be touched without going through our interfaces. Third-party
libraries can still shoot themselves in the foot but benign uses of
os:putenv/2 will no longer risk crashing the emulator.
---
 erts/emulator/Makefile.in                         |   4 +-
 erts/emulator/beam/bif.tab                        |   8 +-
 erts/emulator/beam/break.c                        |  16 +-
 erts/emulator/beam/dist.c                         |   2 +-
 erts/emulator/beam/erl_alloc.types                |   6 +-
 erts/emulator/beam/erl_bif_info.c                 |   2 +-
 erts/emulator/beam/erl_bif_os.c                   | 199 +++-----
 erts/emulator/beam/erl_bif_port.c                 | 193 +++-----
 erts/emulator/beam/erl_init.c                     |  13 +-
 erts/emulator/beam/erl_unicode.c                  |  16 +-
 erts/emulator/beam/global.h                       |   2 +-
 erts/emulator/beam/io.c                           |  18 +-
 erts/emulator/beam/sys.h                          |  37 +-
 erts/emulator/beam/utils.c                        |  19 +-
 erts/emulator/sys/common/erl_osenv.c              | 396 ++++++++++++++++
 erts/emulator/sys/common/erl_osenv.h              | 121 +++++
 erts/emulator/sys/unix/erl_unix_sys.h             |   2 +-
 erts/emulator/sys/unix/sys.c                      | 127 +-----
 erts/emulator/sys/unix/sys_drivers.c              | 183 +++-----
 erts/emulator/sys/unix/sys_env.c                  | 133 ++++++
 erts/emulator/sys/win32/sys.c                     |  87 +++-
 erts/emulator/sys/win32/sys_env.c                 | 531 +++++++++-------------
 erts/emulator/test/driver_SUITE.erl               |  47 ++
 erts/emulator/test/driver_SUITE_data/Makefile.src |   3 +-
 erts/emulator/test/driver_SUITE_data/env_drv.c    | 108 +++++
 25 files changed, 1333 insertions(+), 940 deletions(-)
 create mode 100644 erts/emulator/sys/common/erl_osenv.c
 create mode 100644 erts/emulator/sys/common/erl_osenv.h
 create mode 100644 erts/emulator/sys/unix/sys_env.c
 create mode 100644 erts/emulator/test/driver_SUITE_data/env_drv.c

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 15b65f1c71..9c79bf3da0 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -903,6 +903,7 @@ else
 OS_OBJS = \
 	$(OBJDIR)/sys.o \
 	$(OBJDIR)/sys_drivers.o \
+	$(OBJDIR)/sys_env.o \
 	$(OBJDIR)/sys_uds.o \
 	$(OBJDIR)/driver_tab.o \
 	$(OBJDIR)/elib_memmove.o \
@@ -944,7 +945,8 @@ endif
 OS_OBJS +=	$(OBJDIR)/erl_poll.o \
 		$(OBJDIR)/erl_check_io.o \
 		$(OBJDIR)/erl_mseg.o \
-		$(OBJDIR)/erl_mmap.o \
+		$(OBJDIR)/erl_mmap.o  \
+		$(OBJDIR)/erl_osenv.o \
 		$(OBJDIR)/erl_$(ERLANG_OSTYPE)_sys_ddll.o \
 		$(OBJDIR)/erl_mtrace_sys_wrap.o \
 		$(OBJDIR)/erl_sys_common_misc.o \
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 1bd4acbf95..1c5e19cd88 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -374,9 +374,10 @@ bif ets:match_spec_run_r/3
 # Bifs in os module.
 #
 
-bif os:putenv/2
-bif os:getenv/0
-bif os:getenv/1
+bif os:get_env_var/1
+bif os:set_env_var/2
+bif os:unset_env_var/1
+bif os:list_env_vars/0
 bif os:getpid/0
 bif os:timestamp/0
 bif os:system_time/0
@@ -617,7 +618,6 @@ bif erlang:float_to_binary/2
 bif erlang:binary_to_float/1
 
 bif io:printable_range/0
-bif os:unsetenv/1
 
 #
 # New in 17.0
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 2bfb481771..cb8812c455 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -775,16 +775,16 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
      * - write dump until alarm or file is written completely
      */
 	
-    if (erts_sys_getenv__("ERL_CRASH_DUMP_SECONDS", env, &envsz) != 0) {
-	env_erl_crash_dump_seconds_set = 0;
-	secs = -1;
+    if (erts_sys_explicit_8bit_getenv("ERL_CRASH_DUMP_SECONDS", env, &envsz) == 1) {
+        env_erl_crash_dump_seconds_set = 1;
+        secs = atoi(env);
     } else {
-	env_erl_crash_dump_seconds_set = 1;
-	secs = atoi(env);
+        env_erl_crash_dump_seconds_set = 0;
+        secs = -1;
     }
 
     if (secs == 0) {
-	return;
+        return;
     }
 
     /* erts_sys_prepare_crash_dump returns 1 if heart port is found, otherwise 0
@@ -800,7 +800,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
 
     crash_dump_limit = ERTS_SINT64_MAX;
     envsz = sizeof(env);
-    if (erts_sys_getenv__("ERL_CRASH_DUMP_BYTES", env, &envsz) == 0) {
+    if (erts_sys_explicit_8bit_getenv("ERL_CRASH_DUMP_BYTES", env, &envsz) == 1) {
         Sint64 limit;
         char* endptr;
         errno = 0;
@@ -813,7 +813,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
         }
     }
 
-    if (erts_sys_getenv__("ERL_CRASH_DUMP",&dumpnamebuf[0],&dumpnamebufsize) != 0)
+    if (erts_sys_explicit_8bit_getenv("ERL_CRASH_DUMP",&dumpnamebuf[0],&dumpnamebufsize) != 1)
 	dumpname = "erl_crash.dump";
     else
 	dumpname = &dumpnamebuf[0];
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 7ff7462bf6..30390cdb5e 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1212,7 +1212,7 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote)
 #  define PURIFY_MSG(msg)                                                    \
     do {								     \
 	char buf__[1]; size_t bufsz__ = sizeof(buf__);			     \
-	if (erts_sys_getenv_raw("VALGRIND_LOG_XML", buf__, &bufsz__) >= 0) { \
+	if (erts_sys_explicit_8bit_getenv("VALGRIND_LOG_XML", buf__, &bufsz__) >= 0) { \
 	    VALGRIND_PRINTF_XML("<erlang_error_log>"			     \
 			    "%s, line %d: %s</erlang_error_log>\n",	     \
 			    __FILE__, __LINE__, msg);			     \
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index c6cc5c78b3..8107f133aa 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -283,6 +283,8 @@ type	THR_PRGR_DATA	LONG_LIVED	SYSTEM		thr_prgr_data
 type	T_THR_PRGR_DATA	SHORT_LIVED	SYSTEM		temp_thr_prgr_data
 type	RELEASE_LAREA	SHORT_LIVED	SYSTEM		release_literal_area
 
+type	ENVIRONMENT	SYSTEM		SYSTEM		environment
+
 #
 # Types used for special emulators
 #
@@ -370,8 +372,6 @@ type	SYS_READ_BUF	TEMPORARY	SYSTEM		sys_read_buf
 type	FD_TAB		LONG_LIVED	SYSTEM		fd_tab
 type	FD_ENTRY_BUF	STANDARD	SYSTEM		fd_entry_buf
 type	CS_PROG_PATH	LONG_LIVED	SYSTEM		cs_prog_path
-type	ENVIRONMENT	TEMPORARY	SYSTEM		environment
-type	PUTENV_STR	SYSTEM		SYSTEM		putenv_string
 type	PRT_REP_EXIT	STANDARD	SYSTEM		port_report_exit
 type	SYS_BLOCKING	STANDARD	SYSTEM		sys_blocking
 
@@ -383,9 +383,7 @@ type	SYS_WRITE_BUF	TEMPORARY	SYSTEM		sys_write_buf
 
 type	DRV_DATA_BUF	SYSTEM		SYSTEM		drv_data_buf
 type	PRELOADED	LONG_LIVED	SYSTEM		preloaded
-type	PUTENV_STR	SYSTEM		SYSTEM		putenv_string
 type	WAITER_OBJ	LONG_LIVED	SYSTEM		waiter_object
-type	ENVIRONMENT	SYSTEM		SYSTEM		environment
 type	CON_VPRINTF_BUF	TEMPORARY	SYSTEM		con_vprintf_buf
 
 +endif
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 64639e157b..9aafab86c0 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1754,7 +1754,7 @@ static int check_if_xml(void)
 {
     char buf[1];
     size_t bufsz = sizeof(buf);
-    return erts_sys_getenv_raw("VALGRIND_LOG_XML", buf, &bufsz) >= 0;
+    return erts_sys_explicit_8bit_getenv("VALGRIND_LOG_XML", buf, &bufsz) >= 0;
 }
 #else
 #define check_if_xml() 0
diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c
index 910325a2f4..ce2b27409b 100644
--- a/erts/emulator/beam/erl_bif_os.c
+++ b/erts/emulator/beam/erl_bif_os.c
@@ -36,8 +36,7 @@
 #include "big.h"
 #include "dist.h"
 #include "erl_version.h"
-
-static int check_env_name(char *name);
+#include "erl_osenv.h"
 
 /*
  * Return the pid for the Erlang process in the host OS.
@@ -67,148 +66,78 @@ BIF_RETTYPE os_getpid_0(BIF_ALIST_0)
      BIF_RET(buf_to_intlist(&hp, pid_string, n, NIL));
 }
 
-BIF_RETTYPE os_getenv_0(BIF_ALIST_0)
+static void os_getenv_foreach(Process *process, Eterm *result, Eterm key, Eterm value)
 {
-    GETENV_STATE state;
-    char *cp;
-    Eterm* hp;
-    Eterm ret;
-    Eterm str;
+    Eterm kvp_term, *hp;
 
-    init_getenv_state(&state);
+    hp = HAlloc(process, 5);
+    kvp_term = TUPLE2(hp, key, value);
+    hp += 3;
 
-    ret = NIL;
-    while ((cp = getenv_string(&state)) != NULL) {
-	str = erts_convert_native_to_filename(BIF_P,(byte *)cp);
-	hp = HAlloc(BIF_P, 2);
-	ret = CONS(hp, str, ret);
-    }
+    (*result) = CONS(hp, kvp_term, (*result));
+}
 
-    fini_getenv_state(&state);
+BIF_RETTYPE os_list_env_vars_0(BIF_ALIST_0)
+{
+    const erts_osenv_t *global_env;
+    Eterm result = NIL;
+
+    global_env = erts_sys_rlock_global_osenv();
+    erts_osenv_foreach_term(global_env, BIF_P, &result, (void*)&os_getenv_foreach);
+    erts_sys_runlock_global_osenv();
 
-    return ret;
+    return result;
 }
 
-#define STATIC_BUF_SIZE 1024
-BIF_RETTYPE os_getenv_1(BIF_ALIST_1)
+BIF_RETTYPE os_get_env_var_1(BIF_ALIST_1)
 {
-    Process* p = BIF_P;
-    Eterm str;
-    Sint len;
-    int res;
-    char *key_str, *val;
-    char buf[STATIC_BUF_SIZE];
-    size_t val_size = sizeof(buf);
-
-    key_str = erts_convert_filename_to_native(BIF_ARG_1,buf,STATIC_BUF_SIZE,
-					      ERTS_ALC_T_TMP,1,0,&len);
-
-    if (!check_env_name(key_str)) {
-        if (key_str && key_str != &buf[0])
-            erts_free(ERTS_ALC_T_TMP, key_str);
-        BIF_ERROR(p, BADARG);
-    }
+    const erts_osenv_t *global_env;
+    Eterm out_term;
+    int error;
 
-    if (key_str != &buf[0])
-	val = &buf[0];
-    else {
-	/* len includes zero byte */
-	val_size -= len;
-	val = &buf[len];
-    }
-    res = erts_sys_getenv(key_str, val, &val_size);
-
-    if (res < 0) {
-    no_var:
-	str = am_false;
-    } else {
-	if (res > 0) {
-	    val = erts_alloc(ERTS_ALC_T_TMP, val_size);
-	    while (1) {
-		res = erts_sys_getenv(key_str, val, &val_size);
-		if (res == 0)
-		    break;
-		else if (res < 0)
-		    goto no_var;
-		else
-		    val = erts_realloc(ERTS_ALC_T_TMP, val, val_size);
-	    }
-	}
-	str = erts_convert_native_to_filename(p,(byte *)val);
-    }
-    if (key_str != &buf[0])
-	erts_free(ERTS_ALC_T_TMP, key_str);
-    if (val < &buf[0] || &buf[sizeof(buf)-1] < val)
-	erts_free(ERTS_ALC_T_TMP, val);
-    BIF_RET(str);
+    global_env = erts_sys_rlock_global_osenv();
+    error = erts_osenv_get_term(global_env, BIF_P, BIF_ARG_1, &out_term);
+    erts_sys_runlock_global_osenv();
+
+    if (error == 0) {
+        return am_false;
+    } else if (error < 0) {
+        BIF_ERROR(BIF_P, BADARG);
+    } 
+
+    return out_term;
 }
 
-BIF_RETTYPE os_putenv_2(BIF_ALIST_2)
+BIF_RETTYPE os_set_env_var_2(BIF_ALIST_2)
 {
-    char def_buf_key[STATIC_BUF_SIZE];
-    char def_buf_value[STATIC_BUF_SIZE];
-    char *key_buf = NULL, *value_buf = NULL;
-
-    key_buf = erts_convert_filename_to_native(BIF_ARG_1,def_buf_key,
-					      STATIC_BUF_SIZE,
-					      ERTS_ALC_T_TMP,0,0,NULL);
-    if (!check_env_name(key_buf))
-        goto badarg;
-
-    value_buf = erts_convert_filename_to_native(BIF_ARG_2,def_buf_value,
-						STATIC_BUF_SIZE,
-						ERTS_ALC_T_TMP,1,0,
-						NULL);
-    if (!value_buf)
-        goto badarg;
-
-    if (erts_sys_putenv(key_buf, value_buf)) {
-	if (key_buf != def_buf_key) {
-	    erts_free(ERTS_ALC_T_TMP, key_buf);
-	}
-	if (value_buf != def_buf_value) {
-	    erts_free(ERTS_ALC_T_TMP, value_buf);
-	}
-	BIF_ERROR(BIF_P, BADARG);
-    }
-    if (key_buf != def_buf_key) {
-	erts_free(ERTS_ALC_T_TMP, key_buf);
-    }
-    if (value_buf != def_buf_value) {
-	erts_free(ERTS_ALC_T_TMP, value_buf);
+    erts_osenv_t *global_env;
+    int error;
+
+    global_env = erts_sys_rwlock_global_osenv();
+    error = erts_osenv_put_term(global_env, BIF_ARG_1, BIF_ARG_2);
+    erts_sys_rwunlock_global_osenv();
+
+    if (error < 0) {
+        BIF_ERROR(BIF_P, BADARG);
     }
-    BIF_RET(am_true);
 
-badarg:
-    if (key_buf && key_buf != def_buf_key)
-	erts_free(ERTS_ALC_T_TMP, key_buf);
-    if (value_buf && value_buf != def_buf_value)
-	erts_free(ERTS_ALC_T_TMP, value_buf);
-    BIF_ERROR(BIF_P, BADARG);
+    BIF_RET(am_true);
 }
 
-BIF_RETTYPE os_unsetenv_1(BIF_ALIST_1)
+BIF_RETTYPE os_unset_env_var_1(BIF_ALIST_1)
 {
-    char *key_buf;
-    char buf[STATIC_BUF_SIZE];
+    erts_osenv_t *global_env;
+    int error;
 
-    key_buf = erts_convert_filename_to_native(BIF_ARG_1,buf,STATIC_BUF_SIZE,
-					      ERTS_ALC_T_TMP,0,0,NULL);
-    if (!check_env_name(key_buf))
-        goto badarg;
+    global_env = erts_sys_rwlock_global_osenv();
+    error = erts_osenv_unset_term(global_env, BIF_ARG_1);
+    erts_sys_rwunlock_global_osenv();
 
-    if (erts_sys_unsetenv(key_buf))
-        goto badarg;
-
-    if (key_buf != buf) {
-	erts_free(ERTS_ALC_T_TMP, key_buf);
+    if (error < 0) {
+        BIF_ERROR(BIF_P, BADARG);
     }
-    BIF_RET(am_true);
 
-badarg:
-    if (key_buf && key_buf != buf)
-        erts_free(ERTS_ALC_T_TMP, key_buf);
-    BIF_ERROR(BIF_P, BADARG);    
+    BIF_RET(am_true);
 }
 
 BIF_RETTYPE os_set_signal_2(BIF_ALIST_2) {
@@ -224,27 +153,3 @@ BIF_RETTYPE os_set_signal_2(BIF_ALIST_2) {
 error:
     BIF_ERROR(BIF_P, BADARG);
 }
-
-static int
-check_env_name(char *raw_name)
-{
-    byte *c = (byte *) raw_name;
-    int encoding;
-
-    if (!c)
-        return 0;
-    
-    encoding = erts_get_native_filename_encoding();
-
-    if (erts_raw_env_char_is_7bit_ascii_char('\0', c, encoding))
-        return 0; /* Do not allow empty name... */
-
-    /* Verify no '=' characters in variable name... */
-    do {
-        if (erts_raw_env_char_is_7bit_ascii_char('=', c, encoding))
-            return 0;
-        c = erts_raw_env_next_char(c, encoding);
-    } while (!erts_raw_env_char_is_7bit_ascii_char('\0', c, encoding));
-
-    return 1; /* Seems ok... */
-}
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index c4a4dd5863..9f0c90ff7b 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -45,7 +45,7 @@
 #include "dtrace-wrapper.h"
 
 static Port *open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump);
-static char* convert_environment(Eterm env);
+static int merge_global_environment(erts_osenv_t *env, Eterm key_value_pairs);
 static char **convert_args(Eterm);
 static void free_args(char **);
 
@@ -651,6 +651,7 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1)
 static Port *
 open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
 {
+    int merged_environment = 0;
     Sint i;
     Eterm option;
     Uint arity;
@@ -672,12 +673,13 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
     opts.read_write = 0;
     opts.hide_window = 0;
     opts.wd = NULL;
-    opts.envir = NULL;
     opts.exit_status = 0;
     opts.overlapped_io = 0; 
     opts.spawn_type = ERTS_SPAWN_ANY; 
     opts.argv = NULL;
     opts.parallelism = erts_port_parallelism;
+    erts_osenv_init(&opts.envir);
+
     linebuf = 0;
 
     *err_nump = 0;
@@ -718,11 +720,16 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
 			goto badarg;
 		    }
 		} else if (option == am_env) {
-                    if (opts.envir) /* ignore previous env option... */
-                        erts_free(ERTS_ALC_T_OPEN_PORT_ENV, opts.envir);
-                    opts.envir = convert_environment(*tp);
-                    if (!opts.envir)
-			goto badarg;
+		    if (merged_environment) {
+		        /* Ignore previous env option */
+		        erts_osenv_clear(&opts.envir);
+		    }
+
+		    merged_environment = 1;
+
+		    if (merge_global_environment(&opts.envir, *tp)) {
+		        goto badarg;
+		    }
 		} else if (option == am_args) {
 		    char **av;
 		    char **oav = opts.argv;
@@ -807,6 +814,12 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
     if((linebuf && opts.packet_bytes) || 
        (opts.redir_stderr && !opts.use_stdio)) {
 	goto badarg;
+}
+
+    /* If we lacked an env option, fill in the global environment without
+     * changes. */
+    if (!merged_environment) {
+        merge_global_environment(&opts.envir, NIL);
     }
 
     /*
@@ -956,8 +969,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
 	erts_atomic32_read_bor_relb(&port->state, sflgs);
  
  do_return:
-    if (opts.envir)
-        erts_free(ERTS_ALC_T_OPEN_PORT_ENV, opts.envir);
+    erts_osenv_clear(&opts.envir);
     if (name_buf)
 	erts_free(ERTS_ALC_T_TMP, (void *) name_buf);
     if (opts.argv) {
@@ -977,6 +989,45 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
     goto do_return;
 }
 
+/* Merges the the global environment and the given {Key, Value} list into env,
+ * unsetting all keys whose value is either 'false' or NIL. The behavior on
+ * NIL is undocumented and perhaps surprising, but the previous implementation
+ * worked in this manner. */
+static int merge_global_environment(erts_osenv_t *env, Eterm key_value_pairs) {
+    const erts_osenv_t *global_env = erts_sys_rlock_global_osenv();
+    erts_osenv_merge(env, global_env, 0);
+    erts_sys_runlock_global_osenv();
+
+    while (is_list(key_value_pairs)) {
+        Eterm *cell, *tuple;
+
+        cell = list_val(key_value_pairs);
+
+        if(!is_tuple_arity(CAR(cell), 2)) {
+            return -1;
+        }
+
+        tuple = tuple_val(CAR(cell));
+        key_value_pairs = CDR(cell);
+
+        if(is_nil(tuple[2]) || tuple[2] == am_false) {
+            if(erts_osenv_unset_term(env, tuple[1]) < 0) {
+                return -1;
+            }
+        } else {
+            if(erts_osenv_put_term(env, tuple[1], tuple[2]) < 0) {
+                return -1;
+            }
+        }
+    }
+
+    if(!is_nil(key_value_pairs)) {
+        return -1;
+    }
+
+    return 0;
+}
+
 /* Arguments can be given i unicode and as raw binaries, convert filename is used to convert */
 static char **convert_args(Eterm l)
 {
@@ -1024,130 +1075,6 @@ static void free_args(char **av)
     erts_free(ERTS_ALC_T_TMP, av);
 }
 
-#ifdef DEBUG
-#define ERTS_CONV_ENV_BUF_EXTRA 2
-#else
-#define ERTS_CONV_ENV_BUF_EXTRA 1024
-#endif
-
-static char* convert_environment(Eterm env)
-{
-    /*
-     * Returns environment buffer in memory allocated
-     * as ERTS_ALC_T_OPEN_PORT_ENV. Caller *needs*
-     * to deallocate...
-     */
-
-    Sint size, alloc_size;
-    byte* bytes;
-    int encoding = erts_get_native_filename_encoding();
-
-    alloc_size = ERTS_CONV_ENV_BUF_EXTRA;
-    bytes = erts_alloc(ERTS_ALC_T_OPEN_PORT_ENV,
-                       alloc_size);
-    size = 0;
-
-    /* ERTS_CONV_ENV_BUF_EXTRA >= for end delimiter... */
-    ERTS_CT_ASSERT(ERTS_CONV_ENV_BUF_EXTRA >= 2);
-
-    while (is_list(env)) {
-        Sint var_sz, val_sz, need;
-        byte *str, *limit;
-	Eterm tmp, *tp, *consp;
-
-        consp = list_val(env);
-	tmp = CAR(consp);
-	if (is_not_tuple_arity(tmp, 2))
-	    goto error;
-
-	tp = tuple_val(tmp);
-
-        /* Check encoding of env variable... */
-        if (is_not_list(tp[1]))
-            goto error;
-        var_sz = erts_native_filename_need(tp[1], encoding);
-        if (var_sz <= 0)
-            goto error;
-        /* Check encoding of value... */
-        if (tp[2] == am_false || is_nil(tp[2]))
-            val_sz = 0;
-        else if (is_not_list(tp[2]))
-            goto error;
-        else {
-            val_sz = erts_native_filename_need(tp[2], encoding);
-            if (val_sz < 0)
-                goto error;
-        }
-
-        /* Ensure enough memory... */
-        need = size;
-        need += var_sz + val_sz;
-        /* '=' and '\0' */
-        need += 2 * erts_raw_env_7bit_ascii_char_need(encoding);
-        if (need > alloc_size) {
-            alloc_size = (need - alloc_size) + alloc_size;
-            alloc_size += ERTS_CONV_ENV_BUF_EXTRA;
-            bytes = erts_realloc(ERTS_ALC_T_OPEN_PORT_ENV,
-                                 bytes, alloc_size);
-        }
-
-        /* Write environment variable name... */
-        str = bytes + size;
-        erts_native_filename_put(tp[1], encoding, str);
-        /* empty variable name is not allowed... */
-        if (erts_raw_env_char_is_7bit_ascii_char('\0', str, encoding))
-            goto error;
-
-        /*
-         * Drop null characters at the end and verify that we do
-         * not have any '=' characters in the name...
-         */
-        limit = str + var_sz;
-        while (str < limit) {
-            if (erts_raw_env_char_is_7bit_ascii_char('\0', str, encoding))
-                break;
-            if (erts_raw_env_char_is_7bit_ascii_char('=', str, encoding))
-                goto error;
-            str = erts_raw_env_next_char(str, encoding);
-        }
-
-        /* Write the equals sign... */
-        str = erts_raw_env_7bit_ascii_char_put('=', str, encoding);
-
-        /* Write the value... */
-        if (val_sz > 0) {
-            limit = str + val_sz;
-            erts_native_filename_put(tp[2], encoding, str);
-            while (str < limit) {
-                if (erts_raw_env_char_is_7bit_ascii_char('\0', str, encoding))
-                    break;
-                str = erts_raw_env_next_char(str, encoding);
-            }
-        }
-
-        /* Delimit... */
-        str = erts_raw_env_7bit_ascii_char_put('\0', str, encoding);
-
-        size = str - bytes;
-        ASSERT(size <= alloc_size);
-
-        env = CDR(consp);
-    }
-
-    /* End delimit... */
-    (void) erts_raw_env_7bit_ascii_char_put('\0', &bytes[size], encoding);
-
-    if (is_nil(env))
-        return (char *) bytes;
-
-error:
-
-    if (bytes)
-        erts_free(ERTS_ALC_T_OPEN_PORT_ENV, bytes);
-
-    return (char *) NULL; /* error... */
-}
-
 /* ------------ decode_packet() and friends: */
 
 struct packet_callback_args
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 6cef9bd0e3..f52eed41d5 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -50,7 +50,7 @@
 #define ERTS_WANT_TIMER_WHEEL_API
 #include "erl_time.h"
 #include "erl_check_io.h"
-
+#include "erl_osenv.h"
 #ifdef HIPE
 #include "hipe_mode_switch.h"	/* for hipe_mode_switch_init() */
 #include "hipe_signal.h"	/* for hipe_signal_init() */
@@ -803,8 +803,9 @@ early_init(int *argc, char **argv) /*
 
     envbufsz = sizeof(envbuf);
 
-    /* erts_sys_getenv(_raw)() not initialized yet; need erts_sys_getenv__() */
-    if (erts_sys_getenv__("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 0)
+    /* erts_osenv hasn't been initialized yet, so we need to fall back to
+     * erts_sys_explicit_host_getenv() */
+    if (erts_sys_explicit_host_getenv("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 1)
 	erts_async_max_threads = atoi(envbuf);
     else
 	erts_async_max_threads = ERTS_DEFAULT_NO_ASYNC_THREADS;
@@ -1210,20 +1211,20 @@ erl_start(int argc, char **argv)
 			 &time_warp_mode);
 
     envbufsz = sizeof(envbuf);
-    if (erts_sys_getenv_raw(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 0)
+    if (erts_sys_explicit_8bit_getenv(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 1)
 	user_requested_db_max_tabs = atoi(envbuf);
     else
 	user_requested_db_max_tabs = 0;
 
     envbufsz = sizeof(envbuf);
-    if (erts_sys_getenv_raw("ERL_FULLSWEEP_AFTER", envbuf, &envbufsz) == 0) {
+    if (erts_sys_explicit_8bit_getenv("ERL_FULLSWEEP_AFTER", envbuf, &envbufsz) == 1) {
 	Uint16 max_gen_gcs = atoi(envbuf);
 	erts_atomic32_set_nob(&erts_max_gen_gcs,
 				  (erts_aint32_t) max_gen_gcs);
     }
 
     envbufsz = sizeof(envbuf);
-    if (erts_sys_getenv_raw("ERL_MAX_PORTS", envbuf, &envbufsz) == 0) {
+    if (erts_sys_explicit_8bit_getenv("ERL_MAX_PORTS", envbuf, &envbufsz) == 1) {
 	port_tab_sz = atoi(envbuf);
 	port_tab_sz_ignore_files = 1;
     }
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index b7a5c45fea..bd5439ba24 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -2083,18 +2083,9 @@ char* erts_convert_filename_to_wchar(byte* bytes, Uint size,
     return name_buf;
 }
 
-
-static int filename_len_16bit(byte *str) 
-{
-    byte *p = str;
-    while(*p != '\0' || p[1] != '\0') {
-	p += 2;
-    }
-    return (p - str);
-}
-Eterm erts_convert_native_to_filename(Process *p, byte *bytes)
+Eterm erts_convert_native_to_filename(Process *p, size_t size, byte *bytes)
 {
-    Uint size,num_chars;
+    Uint num_chars;
     Eterm *hp;
     byte *err_pos;
     Uint num_built; /* characters */
@@ -2108,7 +2099,6 @@ Eterm erts_convert_native_to_filename(Process *p, byte *bytes)
     case ERL_FILENAME_UTF8_MAC:
 	mac = 1;
     case ERL_FILENAME_UTF8:
-	size = strlen((char *) bytes);
 	if (size == 0)
 	    return NIL;
 	if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) {
@@ -2123,7 +2113,6 @@ Eterm erts_convert_native_to_filename(Process *p, byte *bytes)
 	} 
 	return ret;
     case ERL_FILENAME_WIN_WCHAR:
-	size=filename_len_16bit(bytes);
 	if ((size % 2) != 0) { /* Panic fixup to avoid crashing the emulator */
 	    size--;
 	    hp = HAlloc(p, size+2);
@@ -2146,7 +2135,6 @@ Eterm erts_convert_native_to_filename(Process *p, byte *bytes)
 	goto noconvert;
     }
  noconvert:
-    size = strlen((char *) bytes);
     hp = HAlloc(p, 2 * size);
     return erts_bin_bytes_to_list(NIL, hp, bytes, size, 0);
 }
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 09207364eb..09500c5bc0 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1264,7 +1264,7 @@ char* erts_convert_filename_to_wchar(byte* bytes, Uint size,
                                      char *statbuf, size_t statbuf_size,
                                      ErtsAlcType_t alloc_type, Sint* used,
                                      Uint extra_wchars);
-Eterm erts_convert_native_to_filename(Process *p, byte *bytes);
+Eterm erts_convert_native_to_filename(Process *p, size_t size, byte *bytes);
 Eterm erts_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, Uint left,
 			Uint *num_built, Uint *num_eaten, Eterm tail);
 int erts_utf8_to_latin1(byte* dest, const byte* source, int slen);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 9933c8dda4..6158e9613a 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -7715,13 +7715,27 @@ int null_func(void)
 int
 erl_drv_putenv(const char *key, char *value)
 {
-    return erts_sys_putenv_raw((char*)key, value);
+    switch (erts_sys_explicit_8bit_putenv((char*)key, value)) {
+    case -1: /* Insufficient buffer space */
+        return 1;
+    case 1: /* Success */
+        return 0;
+    default: /* Not found */
+        return -1;
+    }
 }
 
 int
 erl_drv_getenv(const char *key, char *value, size_t *value_size)
 {
-    return erts_sys_getenv_raw((char*)key, value, value_size);
+    switch (erts_sys_explicit_8bit_getenv((char*)key, value, value_size)) {
+    case -1: /* Insufficient buffer space */
+        return 1;
+    case 1: /* Success */
+        return 0;
+    default: /* Not found */
+        return -1;
+    }
 }
 
 /* get heart_port
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index bf7d310568..290e0b209a 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -636,6 +636,8 @@ typedef struct preload {
  */
 typedef Eterm ErtsTracer;
 
+#include "erl_osenv.h"
+
 /*
  * This structure contains options to all built in drivers.
  * None of the drivers use all of the fields.
@@ -651,8 +653,7 @@ typedef struct _SysDriverOpts {
     int hide_window;		/* Hide this windows (Windows). */
     int exit_status;		/* Report exit status of subprocess. */
     int overlapped_io;          /* Only has effect on windows NT et al */
-    char *envir;		/* Environment of the port process, */
-				/* in Windows format. */
+    erts_osenv_t envir;		/* Environment of the port process */
     char **argv;                /* Argument vector in Unix'ish format. */
     char *wd;			/* Working directory. */
     unsigned spawn_type;        /* Bitfield of ERTS_SPAWN_DRIVER | 
@@ -782,9 +783,6 @@ void set_break_quit(void (*)(void), void (*)(void));
 
 void os_flavor(char*, unsigned);
 void os_version(int*, int*, int*);
-void init_getenv_state(GETENV_STATE *);
-char * getenv_string(GETENV_STATE *);
-void fini_getenv_state(GETENV_STATE *);
 
 #define HAVE_ERTS_CHECK_IO_DEBUG
 typedef struct {
@@ -805,20 +803,21 @@ int sys_double_to_chars_ext(double, char*, size_t, size_t);
 int sys_double_to_chars_fast(double, char*, int, int, int);
 void sys_get_pid(char *, size_t);
 
-/* erts_sys_putenv() returns, 0 on success and a value != 0 on failure. */
-int erts_sys_putenv(char *key, char *value);
-/* Simple variant used from drivers, raw eightbit interface */
-int erts_sys_putenv_raw(char *key, char *value);
-/* erts_sys_getenv() returns 0 on success (length of value string in
-   *size), a value > 0 if value buffer is too small (*size is set to needed
-   size), and a value < 0 on failure. */
-int erts_sys_getenv(char *key, char *value, size_t *size);
-/* Simple variant used from drivers, raw eightbit interface */
-int erts_sys_getenv_raw(char *key, char *value, size_t *size);
-/* erts_sys_getenv__() is only allowed to be used in early init phase */
-int erts_sys_getenv__(char *key, char *value, size_t *size);
-/* erst_sys_unsetenv() returns 0 on success and a value != 0 on failure. */
-int erts_sys_unsetenv(char *key);
+/* erl_drv_get/putenv have been implicitly 8-bit for so long that we can't
+ * change them without breaking things on Windows. Their return values are
+ * identical to erts_osenv_get/putenv */
+int erts_sys_explicit_8bit_getenv(char *key, char *value, size_t *size);
+int erts_sys_explicit_8bit_putenv(char *key, char *value);
+
+/* This is identical to erts_sys_explicit_8bit_getenv but falls down to the
+ * host OS implementation instead of erts_osenv. */
+int erts_sys_explicit_host_getenv(char *key, char *value, size_t *size);
+
+const erts_osenv_t *erts_sys_rlock_global_osenv(void);
+void erts_sys_runlock_global_osenv(void);
+
+erts_osenv_t *erts_sys_rwlock_global_osenv(void);
+void erts_sys_rwunlock_global_osenv(void);
 
 /* Easier to use, but not as efficient, environment functions */
 char *erts_read_env(char *key);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 993585be10..fe9f1c7606 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -4360,15 +4360,20 @@ erts_read_env(char *key)
     char *value = erts_alloc(ERTS_ALC_T_TMP, value_len);
     int res;
     while (1) {
-	res = erts_sys_getenv_raw(key, value, &value_len);
-	if (res <= 0)
-	    break;
-	value = erts_realloc(ERTS_ALC_T_TMP, value, value_len);
+        res = erts_sys_explicit_8bit_getenv(key, value, &value_len);
+
+        if (res >= 0) {
+            break;
+        }
+
+        value = erts_realloc(ERTS_ALC_T_TMP, value, value_len);
     }
-    if (res != 0) {
-	erts_free(ERTS_ALC_T_TMP, value);
-	return NULL;
+
+    if (res != 1) {
+        erts_free(ERTS_ALC_T_TMP, value);
+        return NULL;
     }
+
     return value;
 }
 
diff --git a/erts/emulator/sys/common/erl_osenv.c b/erts/emulator/sys/common/erl_osenv.c
new file mode 100644
index 0000000000..9f54d1dff0
--- /dev/null
+++ b/erts/emulator/sys/common/erl_osenv.c
@@ -0,0 +1,396 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2017. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "erl_osenv.h"
+
+#include "global.h"
+#include "erl_alloc.h"
+#include "erl_process.h"
+
+#define STACKBUF_SIZE (512)
+
+typedef struct __env_rbtnode_t {
+    struct __env_rbtnode_t *parent;
+    struct __env_rbtnode_t *left;
+    struct __env_rbtnode_t *right;
+
+    int is_red;
+
+    erts_osenv_data_t key;
+    erts_osenv_data_t value;
+} env_rbtnode_t;
+
+#define ERTS_RBT_PREFIX env
+#define ERTS_RBT_T env_rbtnode_t
+#define ERTS_RBT_KEY_T erts_osenv_data_t
+#define ERTS_RBT_FLAGS_T int
+#define ERTS_RBT_INIT_EMPTY_TNODE(T) \
+    do { \
+        (T)->parent = NULL; \
+        (T)->left = NULL; \
+        (T)->right = NULL; \
+        (T)->is_red = 0; \
+    } while(0)
+#define ERTS_RBT_IS_RED(T) ((T)->is_red)
+#define ERTS_RBT_SET_RED(T) ((T)->is_red = 1)
+#define ERTS_RBT_IS_BLACK(T) (!ERTS_RBT_IS_RED(T))
+#define ERTS_RBT_SET_BLACK(T) ((T)->is_red = 0)
+#define ERTS_RBT_GET_FLAGS(T) ((T)->is_red)
+#define ERTS_RBT_SET_FLAGS(T, F) ((T)->is_red = F)
+#define ERTS_RBT_GET_PARENT(T) ((T)->parent)
+#define ERTS_RBT_SET_PARENT(T, P) ((T)->parent = P)
+#define ERTS_RBT_GET_RIGHT(T) ((T)->right)
+#define ERTS_RBT_SET_RIGHT(T, R) ((T)->right = (R))
+#define ERTS_RBT_GET_LEFT(T) ((T)->left)
+#define ERTS_RBT_SET_LEFT(T, L) ((T)->left = (L))
+#define ERTS_RBT_GET_KEY(T) ((T)->key)
+#define ERTS_RBT_IS_LT(KX, KY) (compare_env_keys(KX, KY) < 0)
+#define ERTS_RBT_IS_EQ(KX, KY) (compare_env_keys(KX, KY) == 0)
+#define ERTS_RBT_WANT_FOREACH_DESTROY
+#define ERTS_RBT_WANT_FOREACH
+#define ERTS_RBT_WANT_REPLACE
+#define ERTS_RBT_WANT_DELETE
+#define ERTS_RBT_WANT_INSERT
+#define ERTS_RBT_WANT_LOOKUP
+
+static int compare_env_keys(const erts_osenv_data_t a, const erts_osenv_data_t b);
+
+#include "erl_rbtree.h"
+
+static int compare_env_keys(const erts_osenv_data_t a, const erts_osenv_data_t b) {
+    int relation = sys_memcmp(a.data, b.data, MIN(a.length, b.length));
+
+    if(relation != 0) {
+        return relation;
+    }
+
+    if(a.length < b.length) {
+        return -1;
+    } else if(a.length == b.length) {
+        return 0;
+    } else {
+        return 1;
+    }
+}
+
+static void *convert_value_to_native(Eterm term, char *stackbuf,
+        int stackbuf_size, Sint *length) {
+    int encoding;
+    void *result;
+
+    if(is_atom(term)) {
+        return NULL;
+    }
+
+    encoding = erts_get_native_filename_encoding();
+    *length = erts_native_filename_need(term, encoding);
+
+    if(*length < 0) {
+        return NULL;
+    } else if(*length >= stackbuf_size) {
+        result = erts_alloc(ERTS_ALC_T_TMP, *length);
+    } else {
+        result = stackbuf;
+    }
+
+    erts_native_filename_put(term, encoding, (byte*)result);
+
+    return result;
+}
+
+static void *convert_key_to_native(Eterm term, char *stackbuf,
+        int stackbuf_size, Sint *length) {
+    byte *name_iterator, *name_end;
+    void *result;
+    int encoding;
+
+    result = convert_value_to_native(term, stackbuf, stackbuf_size, length);
+
+    if(result == NULL || length == 0) {
+        return NULL;
+    }
+
+    encoding = erts_get_native_filename_encoding();
+
+    name_iterator = (byte*)result;
+    name_end = &name_iterator[*length];
+
+#ifdef __WIN32__
+    /* Windows stores per-drive working directories as variables starting with
+     * '=', so we skip the first character to tolerate that. */
+    name_iterator = erts_raw_env_next_char(name_iterator, encoding);
+#endif
+
+    while(name_iterator < name_end) {
+        if(erts_raw_env_char_is_7bit_ascii_char('=', name_iterator, encoding)) {
+            if(result != stackbuf) {
+                erts_free(ERTS_ALC_T_TMP, result);
+            }
+
+            return NULL;
+        }
+
+        name_iterator = erts_raw_env_next_char(name_iterator, encoding);
+    }
+
+    return result;
+}
+
+void erts_osenv_init(erts_osenv_t *env) {
+    env->variable_count = 0;
+    env->content_size = 0;
+    env->tree = NULL;
+}
+
+static void destroy_foreach(env_rbtnode_t *node, void *_state) {
+    erts_free(ERTS_ALC_T_ENVIRONMENT, node);
+    (void)_state;
+}
+
+void erts_osenv_clear(erts_osenv_t *env) {
+    env_rbt_foreach_destroy(&env->tree, &destroy_foreach, NULL);
+    erts_osenv_init(env);
+}
+
+struct __env_merge {
+    int overwrite_existing;
+    erts_osenv_t *env;
+};
+
+static void merge_foreach(env_rbtnode_t *node, void *_state) {
+    struct __env_merge *state = (struct __env_merge*)(_state);
+    env_rbtnode_t *existing_node;
+
+    existing_node = env_rbt_lookup(state->env->tree, node->key);
+
+    if(existing_node == NULL || state->overwrite_existing) {
+        erts_osenv_put_native(state->env, &node->key, &node->value);
+    }
+}
+
+void erts_osenv_merge(erts_osenv_t *env, const erts_osenv_t *with, int overwrite) {
+    struct __env_merge merge_state;
+
+    merge_state.overwrite_existing = overwrite;
+    merge_state.env = env;
+
+    env_rbt_foreach(with->tree, merge_foreach, &merge_state);
+}
+
+struct __env_foreach_term {
+    erts_osenv_foreach_term_cb_t user_callback;
+    struct process *process;
+    void *user_state;
+};
+
+static void foreach_term_wrapper(env_rbtnode_t *node, void *_state) {
+    struct __env_foreach_term *state = (struct __env_foreach_term*)_state;
+    Eterm key, value;
+
+    key = erts_convert_native_to_filename(state->process,
+        node->key.length, (byte*)node->key.data);
+    value = erts_convert_native_to_filename(state->process,
+        node->value.length, (byte*)node->value.data);
+
+    state->user_callback(state->process, state->user_state, key, value);
+}
+
+void erts_osenv_foreach_term(const erts_osenv_t *env, struct process *process,
+        void *state, erts_osenv_foreach_term_cb_t callback) {
+    struct __env_foreach_term wrapper_state;
+
+    wrapper_state.user_callback = callback;
+    wrapper_state.user_state = state;
+    wrapper_state.process = process;
+
+    env_rbt_foreach(env->tree, foreach_term_wrapper, &wrapper_state);
+}
+
+int erts_osenv_get_term(const erts_osenv_t *env, Process *process,
+        Eterm key_term, Eterm *out_term) {
+    char key_stackbuf[STACKBUF_SIZE];
+    erts_osenv_data_t key;
+    int result;
+
+    key.data = convert_key_to_native(key_term, key_stackbuf,
+        STACKBUF_SIZE, &key.length);
+    result = -1;
+
+    if(key.data != NULL) {
+        env_rbtnode_t *node;
+
+        node = env_rbt_lookup(env->tree, key);
+        result = 0;
+
+        if(node != NULL) {
+            (*out_term) = erts_convert_native_to_filename(process,
+                node->value.length, (byte*)node->value.data);
+            result = 1;
+        }
+
+        if(key.data != key_stackbuf) {
+            erts_free(ERTS_ALC_T_TMP, key.data);
+        }
+    }
+
+    return result;
+}
+
+int erts_osenv_put_term(erts_osenv_t *env, Eterm key_term, Eterm value_term) {
+    char key_stackbuf[STACKBUF_SIZE], value_stackbuf[STACKBUF_SIZE];
+    erts_osenv_data_t key, value;
+    int result;
+
+    key.data = convert_key_to_native(key_term, key_stackbuf,
+        STACKBUF_SIZE, &key.length);
+    value.data = convert_value_to_native(value_term, value_stackbuf,
+        STACKBUF_SIZE, &value.length);
+    result = -1;
+
+    if(value.data != NULL && key.data != NULL) {
+        result = erts_osenv_put_native(env, &key, &value);
+    }
+
+    if(value.data != NULL && value.data != value_stackbuf) {
+        erts_free(ERTS_ALC_T_TMP, value.data);
+    }
+
+    if(key.data != NULL && key.data != key_stackbuf) {
+        erts_free(ERTS_ALC_T_TMP, key.data);
+    }
+
+    return result;
+}
+
+int erts_osenv_unset_term(erts_osenv_t *env, Eterm key_term) {
+    char key_stackbuf[STACKBUF_SIZE];
+    erts_osenv_data_t key;
+    int result;
+
+    key.data = convert_key_to_native(key_term, key_stackbuf,
+        STACKBUF_SIZE, &key.length);
+    result = -1;
+
+    if(key.data != NULL) {
+        result = erts_osenv_unset_native(env, &key);
+
+        if(key.data != key_stackbuf) {
+            erts_free(ERTS_ALC_T_TMP, key.data);
+        }
+    }
+
+    return result;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+struct __env_foreach_native {
+    erts_osenv_foreach_native_cb_t user_callback;
+    void *user_state;
+};
+
+static void foreach_native_wrapper(env_rbtnode_t *node, void *_state) {
+    struct __env_foreach_native *state = (struct __env_foreach_native*)_state;
+
+    state->user_callback(state->user_state, &node->key, &node->value);
+}
+
+void erts_osenv_foreach_native(const erts_osenv_t *env, void *state,
+        erts_osenv_foreach_native_cb_t callback) {
+    struct __env_foreach_native wrapper_state;
+
+    wrapper_state.user_callback = callback;
+    wrapper_state.user_state = state;
+
+    env_rbt_foreach(env->tree, foreach_native_wrapper, &wrapper_state);
+}
+
+int erts_osenv_get_native(const erts_osenv_t *env,
+        const erts_osenv_data_t *key,
+        erts_osenv_data_t *value) {
+    env_rbtnode_t *node = env_rbt_lookup(env->tree, *key);
+
+    if(node != NULL) {
+        if(value != NULL) {
+            if(node->value.length > value->length) {
+                return -1;
+            }
+
+            sys_memcpy(value->data, node->value.data, node->value.length);
+            value->length = node->value.length;
+        }
+
+        return 1;
+    }
+
+    return 0;
+}
+
+int erts_osenv_put_native(erts_osenv_t *env, const erts_osenv_data_t *key,
+        const erts_osenv_data_t *value) {
+    env_rbtnode_t *old_node, *new_node;
+
+    new_node = erts_alloc(ERTS_ALC_T_ENVIRONMENT, sizeof(env_rbtnode_t) +
+        key->length + value->length);
+
+    new_node->key.data = (char*)(&new_node[1]);
+    new_node->key.length = key->length;
+    new_node->value.data = &((char*)new_node->key.data)[key->length];
+    new_node->value.length = value->length;
+
+    sys_memcpy(new_node->key.data, key->data, key->length);
+    sys_memcpy(new_node->value.data, value->data, value->length);
+
+    old_node = env_rbt_lookup(env->tree, *key);
+
+    if(old_node != NULL) {
+        env->content_size -= old_node->value.length;
+        env->content_size -= old_node->key.length;
+        env_rbt_replace(&env->tree, old_node, new_node);
+    } else {
+        env_rbt_insert(&env->tree, new_node);
+        env->variable_count++;
+    }
+
+    env->content_size += new_node->value.length;
+    env->content_size += new_node->key.length;
+
+    if(old_node != NULL) {
+        erts_free(ERTS_ALC_T_ENVIRONMENT, old_node);
+    }
+
+    return 1;
+}
+
+int erts_osenv_unset_native(erts_osenv_t *env, const erts_osenv_data_t *key) {
+    env_rbtnode_t *old_node = env_rbt_lookup(env->tree, *key);
+
+    if(old_node != NULL) {
+        env->content_size -= old_node->value.length;
+        env->content_size -= old_node->key.length;
+        env->variable_count -= 1;
+
+        env_rbt_delete(&env->tree, old_node);
+        erts_free(ERTS_ALC_T_ENVIRONMENT, old_node);
+        return 1;
+    }
+
+    return 0;
+}
diff --git a/erts/emulator/sys/common/erl_osenv.h b/erts/emulator/sys/common/erl_osenv.h
new file mode 100644
index 0000000000..4777f2148a
--- /dev/null
+++ b/erts/emulator/sys/common/erl_osenv.h
@@ -0,0 +1,121 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2017. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/* This is a replacement for getenv(3) and friends, operating on instances so
+ * we can keep a common implementation for both the global and local (per-port)
+ * environments.
+ *
+ * The instances are not thread-safe on their own but unlike getenv(3) we're
+ * guaranteed to be the only user, so placing locks around all our accesses
+ * will suffice.
+ *
+ * Use erts_sys_rwlock_global_osenv to access the global environment. */
+
+#ifndef __ERL_OSENV_H__
+#define __ERL_OSENV_H__
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+typedef struct __erts_osenv_data_t erts_osenv_data_t;
+
+typedef struct __erts_osenv_t {
+    struct __env_rbtnode_t *tree;
+    int variable_count;
+    int content_size;
+} erts_osenv_t;
+
+#include "sys.h"
+
+struct __erts_osenv_data_t {
+    Sint length;
+    void *data;
+};
+
+void erts_osenv_init(erts_osenv_t *env);
+void erts_osenv_clear(erts_osenv_t *env);
+
+/* @brief Merges \c with into \c env
+ *
+ * @param overwrite Whether to overwrite existing entries or keep them as they
+ * are. */
+void erts_osenv_merge(erts_osenv_t *env, const erts_osenv_t *with, int overwrite);
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/* @brief Copies env[key] into \c value
+ *
+ * @return 1 on success, 0 if the key couldn't be found, and -1 if the input
+ * was invalid. */
+int erts_osenv_get_term(const erts_osenv_t *env, struct process *process,
+    Eterm key, Eterm *value);
+
+/* @brief Copies \c value into \c env[key]
+ *
+ * @return 1 on success, -1 if the input was invalid. */
+int erts_osenv_put_term(erts_osenv_t *env, Eterm key, Eterm value);
+
+/* @brief Removes \c env[key]
+ *
+ * @return 1 on success, 0 if the key couldn't be found, and -1 if the input
+ * was invalid. */
+int erts_osenv_unset_term(erts_osenv_t *env, Eterm key);
+
+/* @brief Copies env[key] into \c value
+ *
+ * @param value [in,out] The buffer to copy the value into, may be NULL if you
+ * only wish to query presence.
+ *
+ * @return 1 on success, 0 if the key couldn't be found, and -1 if if the value
+ * didn't fit into the buffer. */
+int erts_osenv_get_native(const erts_osenv_t *env, const erts_osenv_data_t *key,
+    erts_osenv_data_t *value);
+
+/* @brief Copies \c value into \c env[key]
+ *
+ * @return 1 on success, -1 on failure. */
+int erts_osenv_put_native(erts_osenv_t *env, const erts_osenv_data_t *key,
+    const erts_osenv_data_t *value);
+
+/* @brief Removes \c key from the env.
+ *
+ * @return 1 on success, 0 if the key couldn't be found. */
+int erts_osenv_unset_native(erts_osenv_t *env, const erts_osenv_data_t *key);
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+typedef void (*erts_osenv_foreach_term_cb_t)(struct process *process,
+    void *state, Eterm key, Eterm value);
+
+typedef void (*erts_osenv_foreach_native_cb_t)(void *state,
+    const erts_osenv_data_t *key,
+    const erts_osenv_data_t *value);
+
+/* @brief Walks through all environment variables, calling \c callback for each
+ * one. It's unsafe to modify \c env within the callback. */
+void erts_osenv_foreach_term(const erts_osenv_t *env, struct process *process,
+    void *state, erts_osenv_foreach_term_cb_t callback);
+
+/* @copydoc erts_osenv_foreach_term */
+void erts_osenv_foreach_native(const erts_osenv_t *env, void *state,
+    erts_osenv_foreach_native_cb_t callback);
+
+#endif
diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index b6f5b319ee..e367d565a7 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -133,7 +133,7 @@
 #define  ERTS_SYS_CONTINOUS_FD_NUMBERS
 
 
-typedef void *GETENV_STATE;
+void erts_sys_env_init(void);
 
 /*
 ** For the erl_timer_sup module.
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 6315135151..189ca083d7 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -62,9 +62,6 @@
 
 #include "erl_mseg.h"
 
-extern char **environ;
-erts_rwmtx_t environ_rwmtx;
-
 #define MAX_VSIZE 16		/* Max number of entries allowed in an I/O
 				 * vector sock_sendv().
 				 */
@@ -77,7 +74,7 @@ erts_rwmtx_t environ_rwmtx;
 
 #include "erl_check_io.h"
 #include "erl_cpu_topology.h"
-
+#include "erl_osenv.h"
 extern int  driver_interrupt(int, int);
 extern void do_break(void);
 
@@ -454,10 +451,10 @@ prepare_crash_dump(int secs)
     close(crashdump_companion_cube_fd);
 
     envsz = sizeof(env);
-    i = erts_sys_getenv__("ERL_CRASH_DUMP_NICE", env, &envsz);
+    i = erts_sys_explicit_8bit_getenv("ERL_CRASH_DUMP_NICE", env, &envsz);
     if (i >= 0) {
 	int nice_val;
-	nice_val = i != 0 ? 0 : atoi(env);
+	nice_val = i != 1 ? 0 : atoi(env);
 	if (nice_val > 39) {
 	    nice_val = 39;
 	}
@@ -749,34 +746,6 @@ void os_version(int *pMajor, int *pMinor, int *pBuild) {
     *pBuild = get_number(&release); /* Pointer to build number. */
 }
 
-void init_getenv_state(GETENV_STATE *state)
-{
-   erts_rwmtx_rlock(&environ_rwmtx);
-   *state = NULL;
-}
-
-char *getenv_string(GETENV_STATE *state0)
-{
-   char **state = (char **) *state0;
-   char *cp;
-
-   ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&environ_rwmtx));
-
-   if (state == NULL)
-      state = environ;
-
-   cp = *state++;
-   *state0 = (GETENV_STATE) state;
-
-   return cp;
-}
-
-void fini_getenv_state(GETENV_STATE *state)
-{
-   *state = NULL;
-   erts_rwmtx_runlock(&environ_rwmtx);
-}
-
 void erts_do_break_handling(void)
 {
     struct termios temp_mode;
@@ -830,90 +799,6 @@ void sys_get_pid(char *buffer, size_t buffer_size){
     erts_snprintf(buffer, buffer_size, "%lu",(unsigned long) p);
 }
 
-int
-erts_sys_putenv_raw(char *key, char *value) {
-    return erts_sys_putenv(key, value);
-}
-int
-erts_sys_putenv(char *key, char *value)
-{
-    int res;
-    char *env;
-    Uint need = strlen(key) + strlen(value) + 2;
-
-#ifdef HAVE_COPYING_PUTENV
-    env = erts_alloc(ERTS_ALC_T_TMP, need);
-#else
-    env = erts_alloc(ERTS_ALC_T_PUTENV_STR, need);
-    erts_atomic_add_nob(&sys_misc_mem_sz, need);
-#endif
-    strcpy(env,key);
-    strcat(env,"=");
-    strcat(env,value);
-    erts_rwmtx_rwlock(&environ_rwmtx);
-    res = putenv(env);
-    erts_rwmtx_rwunlock(&environ_rwmtx);
-#ifdef HAVE_COPYING_PUTENV
-    erts_free(ERTS_ALC_T_TMP, env);
-#endif
-    return res;
-}
-
-int
-erts_sys_getenv__(char *key, char *value, size_t *size)
-{
-    int res;
-    char *orig_value = getenv(key);
-    if (!orig_value)
-	res = -1;
-    else {
-	size_t len = sys_strlen(orig_value);
-	if (len >= *size) {
-	    *size = len + 1;
-	    res = 1;
-	}
-	else {
-	    *size = len;
-	    sys_memcpy((void *) value, (void *) orig_value, len+1);
-	    res = 0;
-	}
-    }
-    return res;
-}
-
-int
-erts_sys_getenv_raw(char *key, char *value, size_t *size) {
-    return erts_sys_getenv(key, value, size);
-}
-
-/*
- * erts_sys_getenv
- * returns:
- *  -1, if environment key is not set with a value
- *   0, if environment key is set and value fits into buffer size
- *   1, if environment key is set but does not fit into buffer size
- *      size is set with the needed buffer size value
- */
-
-int
-erts_sys_getenv(char *key, char *value, size_t *size)
-{
-    int res;
-    erts_rwmtx_rlock(&environ_rwmtx);
-    res = erts_sys_getenv__(key, value, size);
-    erts_rwmtx_runlock(&environ_rwmtx);
-    return res;
-}
-
-int
-erts_sys_unsetenv(char *key)
-{
-    int res;
-    erts_rwmtx_rwlock(&environ_rwmtx);
-    res = unsetenv(key);
-    erts_rwmtx_rwunlock(&environ_rwmtx);
-    return res;
-}
 
 void sys_init_io(void) { }
 void erts_sys_alloc_init(void) { }
@@ -1260,14 +1145,9 @@ erts_sys_main_thread(void)
     }
 }
 
-
 void
 erl_sys_args(int* argc, char** argv)
 {
-
-    erts_rwmtx_init(&environ_rwmtx, "environ", NIL,
-        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
-
     ASSERT(argc && argv);
 
     max_files = erts_check_io_max_files();
@@ -1275,4 +1155,5 @@ erl_sys_args(int* argc, char** argv)
     init_smp_sig_notify();
     init_smp_sig_suspend();
 
+    erts_sys_env_init();
 }
diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c
index 0228e1af54..b7ac89d89a 100644
--- a/erts/emulator/sys/unix/sys_drivers.c
+++ b/erts/emulator/sys/unix/sys_drivers.c
@@ -55,9 +55,6 @@
 
 #include "erl_threads.h"
 
-extern char **environ;
-extern erts_rwmtx_t environ_rwmtx;
-
 extern erts_atomic_t sys_misc_mem_sz;
 
 static Eterm forker_port;
@@ -180,7 +177,7 @@ erl_sys_late_init(void)
     opts.read_write = 0;
     opts.hide_window = 0;
     opts.wd = NULL;
-    opts.envir = NULL;
+    erts_osenv_init(&opts.envir);
     opts.exit_status = 0;
     opts.overlapped_io = 0;
     opts.spawn_type = ERTS_SPAWN_ANY;
@@ -443,85 +440,55 @@ static void close_pipes(int ifd[2], int ofd[2])
     close(ofd[1]);
 }
 
-static char **build_unix_environment(char *block)
+struct __add_spawn_env_state {
+    struct iovec *iov;
+    int *iov_index;
+
+    Sint32 *payload_size;
+    char *env_block;
+};
+
+static void add_spawn_env_block_foreach(void *_state,
+                                        const erts_osenv_data_t *key,
+                                        const erts_osenv_data_t *value)
 {
-    int i;
-    int j;
-    int len;
-    char *cp;
-    char **cpp;
-    char** old_env;
-    
-    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&environ_rwmtx));
-
-    cp = block;
-    len = 0;
-    while (*cp != '\0') {
-	cp += strlen(cp) + 1;
-	len++;
-    }
-    old_env = environ;
-    while (*old_env++ != NULL) {
-	len++;
-    }
-    
-    cpp = (char **) erts_alloc_fnf(ERTS_ALC_T_ENVIRONMENT,
-				   sizeof(char *) * (len+1));
-    if (cpp == NULL) {
-	return NULL;
-    }
+    struct __add_spawn_env_state *state;
+    struct iovec *iov;
 
-    cp = block;
-    len = 0;
-    while (*cp != '\0') {
-	cpp[len] = cp;
-	cp += strlen(cp) + 1;
-	len++;
-    }
-    
-    i = len;
-    for (old_env = environ; *old_env; old_env++) {
-	char* old = *old_env;
-
-	for (j = 0; j < len; j++) {
-	    char *s, *t;
-
-	    /* check if cpp[j] equals old
-	       before the = sign,
-	       i.e.
-	       "TMPDIR=/tmp/" */
-	    s = cpp[j];
-	    t = old;
-	    while (*s == *t && *s != '=') {
-		s++, t++;
-	    }
-	    if (*s == '=' && *t == '=') {
-		break;
-	    }
-	}
+    state = (struct __add_spawn_env_state*)(_state);
+    iov = &state->iov[*state->iov_index];
 
-	if (j == len) {		/* New version not found */
-	    cpp[len++] = old;
-	}
-    }
+    iov->iov_base = state->env_block;
 
-    for (j = 0; j < i; ) {
-        size_t last = strlen(cpp[j])-1;
-	if (cpp[j][last] == '=' && strchr(cpp[j], '=') == cpp[j]+last) {
-	    cpp[j] = cpp[--len];
-	    if (len < i) {
-		i--;
-	    } else {
-		j++;
-	    }
-	}
-	else {
-	    j++;
-	}
-    }
+    sys_memcpy(state->env_block, key->data, key->length);
+    state->env_block += key->length;
+    *state->env_block++ = '=';
+    sys_memcpy(state->env_block, value->data, value->length);
+    state->env_block += value->length;
+    *state->env_block++ = '\0';
 
-    cpp[len] = NULL;
-    return cpp;
+    iov->iov_len = state->env_block - (char*)iov->iov_base;
+
+    (*state->payload_size) += iov->iov_len;
+    (*state->iov_index)++;
+}
+
+static void *add_spawn_env_block(const erts_osenv_t *env, struct iovec *iov,
+                                  int *iov_index, Sint32 *payload_size) {
+    struct __add_spawn_env_state add_state;
+    char *env_block;
+
+    env_block = erts_alloc(ERTS_ALC_T_TMP, env->content_size +
+        env->variable_count * sizeof("=\0"));
+
+    add_state.iov = iov;
+    add_state.iov_index = iov_index;
+    add_state.env_block = env_block;
+    add_state.payload_size = payload_size;
+
+    erts_osenv_foreach_native(env, &add_state, add_spawn_env_block_foreach);
+
+    return env_block;
 }
 
 static ErlDrvData spawn_start(ErlDrvPort port_num, char* name,
@@ -531,7 +498,6 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name,
 #define CMD_LINE_PREFIX_STR_SZ (sizeof(CMD_LINE_PREFIX_STR) - 1)
 
     int len;
-    char **new_environ;
     ErtsSysDriverData *dd;
     char *cmd_line;
     char wd_buff[MAXPATHLEN+1];
@@ -598,19 +564,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name,
 	memcpy((void *) (cmd_line + CMD_LINE_PREFIX_STR_SZ), (void *) name, len);
 	cmd_line[CMD_LINE_PREFIX_STR_SZ + len] = '\0';
 	len = CMD_LINE_PREFIX_STR_SZ + len + 1;
-    }
-
-    erts_rwmtx_rlock(&environ_rwmtx);
-
-    if (opts->envir == NULL) {
-	new_environ = environ;
-    } else if ((new_environ = build_unix_environment(opts->envir)) == NULL) {
-	erts_rwmtx_runlock(&environ_rwmtx);
-        close_pipes(ifd, ofd);
-	erts_free(ERTS_ALC_T_TMP, (void *) cmd_line);
-	errno = ENOMEM;
-	return ERL_DRV_ERROR_ERRNO;
-    }
+}
 
     if ((cwd = getcwd(wd_buff, MAXPATHLEN+1)) == NULL) {
         /* on some OSs this call opens a fd in the
@@ -619,9 +573,6 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name,
         int err = errno;
         close_pipes(ifd, ofd);
         erts_free(ERTS_ALC_T_TMP, (void *) cmd_line);
-        if (new_environ != environ)
-            erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ);
-        erts_rwmtx_runlock(&environ_rwmtx);
         errno = err;
         return ERL_DRV_ERROR_ERRNO;
     }
@@ -629,6 +580,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name,
     wd = opts->wd;
 
     {
+        void *environment_block;
         struct iovec *io_vector;
         int iov_len = 5;
         char nullbuff[] = "\0";
@@ -641,10 +593,8 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name,
 
         if (wd) iov_len++;
 
-        /* count number of elements in environment */
-        while(new_environ[env_len] != NULL)
-            env_len++;
-        iov_len += 1 + env_len; /* num envs including size int */
+        /* num envs including size int */
+        iov_len += 1 + opts->envir.variable_count;
 
         /* count number of element in argument list */
         if (opts->spawn_type == ERTS_SPAWN_EXECUTABLE) {
@@ -661,10 +611,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name,
 
         if (!io_vector) {
             close_pipes(ifd, ofd);
-            erts_rwmtx_runlock(&environ_rwmtx);
             erts_free(ERTS_ALC_T_TMP, (void *) cmd_line);
-            if (new_environ != environ)
-                erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ);
             errno = ENOMEM;
             return ERL_DRV_ERROR_ERRNO;
         }
@@ -699,16 +646,13 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name,
         io_vector[i++].iov_len = 1;
         buffsz += io_vector[i-1].iov_len;
 
+        env_len = htonl(opts->envir.variable_count);
         io_vector[i].iov_base = (void*)&env_len;
-        env_len = htonl(env_len);
         io_vector[i++].iov_len = sizeof(env_len);
         buffsz += io_vector[i-1].iov_len;
 
-        for (j = 0; new_environ[j] != NULL; j++) {
-            io_vector[i].iov_base = new_environ[j];
-            io_vector[i++].iov_len = strlen(new_environ[j]) + 1;
-            buffsz += io_vector[i-1].iov_len;
-        }
+        environment_block = add_spawn_env_block(&opts->envir, io_vector, &i,
+            &buffsz);
 
         /* only append arguments if this was a spawn_executable */
         if (opts->spawn_type == ERTS_SPAWN_EXECUTABLE) {
@@ -744,9 +688,6 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name,
                 int err = errno;
                 close_pipes(ifd, ofd);
                 erts_free(ERTS_ALC_T_TMP, io_vector);
-                if (new_environ != environ)
-                    erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ);
-                erts_rwmtx_runlock(&environ_rwmtx);
                 erts_free(ERTS_ALC_T_TMP, (void *) cmd_line);
                 errno = err;
                 return ERL_DRV_ERROR_ERRNO;
@@ -767,16 +708,12 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name,
             driver_select(port_num, ofd[1], ERL_DRV_WRITE|ERL_DRV_USE, 1);
         }
 
+        erts_free(ERTS_ALC_T_TMP, environment_block);
         erts_free(ERTS_ALC_T_TMP, io_vector);
     }
 
     erts_free(ERTS_ALC_T_TMP, (void *) cmd_line);
 
-    if (new_environ != environ)
-	erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ);
-
-    erts_rwmtx_runlock(&environ_rwmtx);
-
     dd = create_driver_data(port_num, ifd[0], ofd[1], opts->packet_bytes,
                              DO_WRITE | DO_READ, opts->exit_status,
                              0, 0);
@@ -1652,15 +1589,13 @@ static ErlDrvData forker_start(ErlDrvPort port_num, char* name,
 
     forker_port = erts_drvport2id(port_num);
 
-    res = erts_sys_getenv_raw("BINDIR", bindir, &bindirsz);
-    if (res != 0) {
-        if (res < 0)
-            erts_exit(1,
-                     "Environment variable BINDIR is not set\n");
-        if (res > 0)
-            erts_exit(1,
-                     "Value of environment variable BINDIR is too large\n");
+    res = erts_sys_explicit_8bit_getenv("BINDIR", bindir, &bindirsz);
+    if (res == 0) {
+        erts_exit(1, "Environment variable BINDIR is not set\n");
+    } else if(res < 0) {
+        erts_exit(1, "Value of environment variable BINDIR is too large\n");
     }
+
     if (bindir[0] != DIR_SEPARATOR_CHAR)
         erts_exit(1,
                  "Environment variable BINDIR does not contain an"
diff --git a/erts/emulator/sys/unix/sys_env.c b/erts/emulator/sys/unix/sys_env.c
new file mode 100644
index 0000000000..4d8301f985
--- /dev/null
+++ b/erts/emulator/sys/unix/sys_env.c
@@ -0,0 +1,133 @@
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "sys.h"
+#include "erl_osenv.h"
+#include "erl_alloc.h"
+
+#include "erl_thr_progress.h"
+
+static erts_osenv_t sysenv_global_env;
+static erts_rwmtx_t sysenv_rwmtx;
+
+extern char **environ;
+
+static void import_initial_env(void);
+
+void erts_sys_env_init() {
+    erts_rwmtx_init(&sysenv_rwmtx, "environ", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
+
+    erts_osenv_init(&sysenv_global_env);
+    import_initial_env();
+}
+
+const erts_osenv_t *erts_sys_rlock_global_osenv() {
+    erts_rwmtx_rlock(&sysenv_rwmtx);
+    return &sysenv_global_env;
+}
+
+erts_osenv_t *erts_sys_rwlock_global_osenv() {
+    erts_rwmtx_rwlock(&sysenv_rwmtx);
+    return &sysenv_global_env;
+}
+
+void erts_sys_rwunlock_global_osenv() {
+    erts_rwmtx_rwunlock(&sysenv_rwmtx);
+}
+
+void erts_sys_runlock_global_osenv() {
+    erts_rwmtx_runlock(&sysenv_rwmtx);
+}
+
+int erts_sys_explicit_8bit_putenv(char *key, char *value) {
+    erts_osenv_data_t env_key, env_value;
+    int result;
+
+    env_key.length = sys_strlen(key);
+    env_key.data = key;
+
+    env_value.length = sys_strlen(value);
+    env_value.data = value;
+
+    {
+        erts_osenv_t *env = erts_sys_rwlock_global_osenv();
+        result = erts_osenv_put_native(env, &env_key, &env_value);
+        erts_sys_rwunlock_global_osenv();
+    }
+
+    return result;
+}
+
+int erts_sys_explicit_8bit_getenv(char *key, char *value, size_t *size) {
+    erts_osenv_data_t env_key, env_value;
+    int result;
+
+    env_key.length = sys_strlen(key);
+    env_key.data = key;
+
+    /* Reserve space for NUL termination. */
+    env_value.length = *size - 1;
+    env_value.data = value;
+
+    {
+        const erts_osenv_t *env = erts_sys_rlock_global_osenv();
+        result = erts_osenv_get_native(env, &env_key, &env_value);
+        erts_sys_runlock_global_osenv();
+    }
+
+    if(result == 1) {
+        value[env_value.length] = '\0';
+    }
+
+    *size = env_value.length;
+
+    return result;
+}
+
+int erts_sys_explicit_host_getenv(char *key, char *value, size_t *size) {
+    char *orig_value;
+    size_t length;
+    
+    orig_value = getenv(key);
+
+    if(orig_value == NULL) {
+        return 0;
+    }
+
+    length = sys_strlen(orig_value);
+
+    if (length >= *size) {
+        *size = length + 1;
+        return -1;
+    }
+
+    sys_memcpy((void*)value, (void*)orig_value, length + 1);
+    *size = length;
+
+    return 1;
+}
+
+static void import_initial_env(void) {
+    char **environ_iterator, *environ_variable;
+
+    environ_iterator = environ;
+
+    while ((environ_variable = *(environ_iterator++)) != NULL) {
+        char *separator_index = strchr(environ_variable, '=');
+
+        if (separator_index != NULL) {
+            erts_osenv_data_t env_key, env_value;
+
+            env_key.length = separator_index - environ_variable;
+            env_key.data = environ_variable;
+    
+            env_value.length = sys_strlen(separator_index) - 1;
+            env_value.data = separator_index + 1;
+
+            erts_osenv_put_native(&sysenv_global_env, &env_key, &env_value);
+        }
+    }
+}
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index 0598a12351..a1c630d68a 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -77,6 +77,7 @@ static int create_pipe(LPHANDLE, LPHANDLE, BOOL, BOOL);
 static int application_type(const wchar_t* originalName, wchar_t fullPath[MAX_PATH],
 			   BOOL search_in_path, BOOL handle_quotes,
 			   int *error_return);
+static void *build_env_block(const erts_osenv_t *env);
 
 HANDLE erts_service_event;
 
@@ -1190,7 +1191,6 @@ spawn_start(ErlDrvPort port_num, char* utf8_name, SysDriverOpts* opts)
     int ok;
     int neededSelects = 0;
     SECURITY_ATTRIBUTES sa = {sizeof(SECURITY_ATTRIBUTES), NULL, TRUE};
-    char* envir = opts->envir;
     int errno_return = -1;
     wchar_t *name;
     int len;
@@ -1265,29 +1265,33 @@ spawn_start(ErlDrvPort port_num, char* utf8_name, SysDriverOpts* opts)
 	name[i] = L'\0';
     }
     DEBUGF(("Spawning \"%S\"\n", name));
-    envir = win_build_environment(envir); /* Always a unicode environment */ 
-    ok = create_child_process(name,
-			      hChildStdin,
-			      hChildStdout,
-			      hChildStderr,
-			      &dp->port_pid,
-			      &pid,
-			      opts->hide_window,
-			      (LPVOID) envir,
-			      (wchar_t *) opts->wd,
-			      opts->spawn_type,
-			      (wchar_t **) opts->argv,
-			      &errno_return);
-    CloseHandle(hChildStdin);
-    CloseHandle(hChildStdout);
-    if (close_child_stderr && hChildStderr != INVALID_HANDLE_VALUE &&
-	hChildStderr != 0) {
-	CloseHandle(hChildStderr);
-    }
-    erts_free(ERTS_ALC_T_TMP, name);
-
-    if (envir != NULL) {
-	erts_free(ERTS_ALC_T_ENVIRONMENT, envir);
+
+    {
+        void *environment_block = build_env_block(&opts->envir);
+
+        ok = create_child_process(name,
+                                  hChildStdin,
+                                  hChildStdout,
+                                  hChildStderr,
+                                  &dp->port_pid,
+                                  &pid,
+                                  opts->hide_window,
+                                  environment_block,
+                                  (wchar_t *) opts->wd,
+                                  opts->spawn_type,
+                                  (wchar_t **) opts->argv,
+                                  &errno_return);
+
+        CloseHandle(hChildStdin);
+        CloseHandle(hChildStdout);
+
+        if (close_child_stderr && hChildStderr != INVALID_HANDLE_VALUE &&
+            hChildStderr != 0) {
+            CloseHandle(hChildStderr);
+        }
+
+        erts_free(ERTS_ALC_T_TMP, environment_block);
+        erts_free(ERTS_ALC_T_TMP, name);
     }
 
     if (!ok) {
@@ -1338,6 +1342,41 @@ spawn_start(ErlDrvPort port_num, char* utf8_name, SysDriverOpts* opts)
     return retval;
 }
 
+struct __build_env_state {
+    WCHAR *next_variable;
+};
+
+static void build_env_foreach(void *_state, const erts_osenv_data_t *key,
+                              const erts_osenv_data_t *value)
+{
+    struct __build_env_state *state = (struct __build_env_state*)(_state);
+
+    sys_memcpy(state->next_variable, key->data, key->length);
+    state->next_variable += (int)key->length / sizeof(WCHAR);
+    *state->next_variable++ = L'=';
+
+    sys_memcpy(state->next_variable, value->data, value->length);
+    state->next_variable += (int)value->length / sizeof(WCHAR);
+    *state->next_variable++ = L'\0';
+}
+
+/* Builds an environment block suitable for CreateProcessW. */
+static void *build_env_block(const erts_osenv_t *env) {
+    struct __build_env_state build_state;
+    WCHAR *env_block;
+
+    env_block = erts_alloc(ERTS_ALC_T_TMP, env->content_size +
+        (env->variable_count * sizeof(L"=\0") + sizeof(L'\0')));
+
+    build_state.next_variable = env_block;
+
+    erts_osenv_foreach_native(env, &build_state, build_env_foreach);
+
+    (*build_state.next_variable) = L'\0';
+
+    return env_block;
+}
+
 static int
 create_file_thread(AsyncIo* aio, int mode)
 {
diff --git a/erts/emulator/sys/win32/sys_env.c b/erts/emulator/sys/win32/sys_env.c
index 5792816267..c78161b344 100644
--- a/erts/emulator/sys/win32/sys_env.c
+++ b/erts/emulator/sys/win32/sys_env.c
@@ -1,319 +1,212 @@
-/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 2002-2016. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * %CopyrightEnd%
- */
-
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-
-#include "sys.h"
-#include "erl_sys_driver.h"
-#include "erl_alloc.h"
-
-static WCHAR *merge_environment(WCHAR *current, WCHAR *add);
-static WCHAR *arg_to_env(WCHAR **arg);
-static WCHAR **env_to_arg(WCHAR *env);
-static WCHAR **find_arg(WCHAR **arg, WCHAR *str);
-static int compare(const void *a, const void *b);
-
-static erts_rwmtx_t environ_rwmtx;
-
-void
-erts_sys_env_init(void)
-{
-    erts_rwmtx_init(&environ_rwmtx, "environ", NIL,
-        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
-}
-
-int
-erts_sys_putenv_raw(char *key, char *value)
-{
-    int res;
-    erts_rwmtx_rwlock(&environ_rwmtx);
-    res = (SetEnvironmentVariable((LPCTSTR) key,
-				  (LPCTSTR) value) ? 0 : 1);
-    erts_rwmtx_rwunlock(&environ_rwmtx);
-    return res;
-}
-
-int
-erts_sys_putenv(char *key, char *value)
-{
-    int res;
-    WCHAR *wkey = (WCHAR *) key;
-    WCHAR *wvalue = (WCHAR *) value;
-    erts_rwmtx_rwlock(&environ_rwmtx);
-    res = (SetEnvironmentVariableW(wkey,
-				   wvalue) ? 0 : 1);
-    erts_rwmtx_rwunlock(&environ_rwmtx);
-    return res;
-}
-
-int
-erts_sys_getenv(char *key, char *value, size_t *size)
-{
-    size_t req_size = 0;
-    int res = 0;
-    DWORD new_size;
-    WCHAR *wkey = (WCHAR *) key;
-    WCHAR *wvalue = (WCHAR *) value;
-    DWORD wsize = *size / (sizeof(WCHAR) / sizeof(char));
-
-    SetLastError(0);
-    erts_rwmtx_rlock(&environ_rwmtx);
-    new_size = GetEnvironmentVariableW(wkey,
-				       wvalue,
-				      (DWORD) wsize);
-    res = !new_size && GetLastError() == ERROR_ENVVAR_NOT_FOUND ? -1 : 0;
-    erts_rwmtx_runlock(&environ_rwmtx);
-    if (res < 0)
-	return res;
-    res = new_size > wsize ? 1 : 0;
-    *size = new_size * (sizeof(WCHAR) / sizeof(char));
-    return res;
-}
-int
-erts_sys_getenv__(char *key, char *value, size_t *size)
-{
-    size_t req_size = 0;
-    int res = 0;
-    DWORD new_size;
-
-    SetLastError(0);
-    new_size = GetEnvironmentVariable((LPCTSTR) key,
-				      (LPTSTR) value,
-				      (DWORD) *size);
-    res = !new_size && GetLastError() == ERROR_ENVVAR_NOT_FOUND ? -1 : 0;
-    if (res < 0)
-	return res;
-    res = new_size > *size ? 1 : 0;
-    *size = new_size;
-    return res;
-}
-
-int
-erts_sys_getenv_raw(char *key, char *value, size_t *size)
-{
-    int res;
-    erts_rwmtx_rlock(&environ_rwmtx);
-    res = erts_sys_getenv__(key, value, size);
-    erts_rwmtx_runlock(&environ_rwmtx);
-    return res;
-}
-
-void init_getenv_state(GETENV_STATE *state)
-{
-    erts_rwmtx_rlock(&environ_rwmtx);
-    state->environment_strings = GetEnvironmentStringsW();
-    state->next_string = state->environment_strings;
-}
-
-char *getenv_string(GETENV_STATE *state)
-{
-    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&environ_rwmtx));
-    if (state->next_string[0] == L'\0') {
-	return NULL;
-    } else {
-	WCHAR *res = state->next_string;
-	state->next_string += wcslen(res) + 1;
-	return (char *) res;
-    }
-}
-
-void fini_getenv_state(GETENV_STATE *state)
-{
-    FreeEnvironmentStringsW(state->environment_strings);
-    state->environment_strings = state->next_string = NULL;
-    erts_rwmtx_runlock(&environ_rwmtx);
-}
-
-int erts_sys_unsetenv(char *key)
-{
-    int res = 0;
-    WCHAR *wkey = (WCHAR *) key;
-
-    SetLastError(0);
-    erts_rwmtx_rlock(&environ_rwmtx);
-    GetEnvironmentVariableW(wkey,
-                            NULL,
-                            0);
-    if (GetLastError() != ERROR_ENVVAR_NOT_FOUND) {
-        res = (SetEnvironmentVariableW(wkey,
-                                       NULL) ? 0 : 1);
-    }
-    erts_rwmtx_runlock(&environ_rwmtx);
-    return res;
-}
-
-char*
-win_build_environment(char* new_env)
-{
-    if (new_env == NULL) {
-	return NULL;
-    } else {
-	WCHAR *tmp, *merged, *tmp_new;
-
-	tmp_new = (WCHAR *) new_env;
-	
-	erts_rwmtx_rlock(&environ_rwmtx);
-	tmp = GetEnvironmentStringsW();
-	merged = merge_environment(tmp, tmp_new);
-
-	FreeEnvironmentStringsW(tmp);
-	erts_rwmtx_runlock(&environ_rwmtx);
-	return (char *) merged;
-    }
-}
-
-static WCHAR *
-merge_environment(WCHAR *old, WCHAR *add)
-{
-    WCHAR **a_arg = env_to_arg(add);
-    WCHAR **c_arg = env_to_arg(old);
-    WCHAR *ret;
-    int i, j;
-    
-    for(i = 0; c_arg[i] != NULL; ++i)
-	;
-
-    for(j = 0; a_arg[j] != NULL; ++j)
-	;
-
-    c_arg = erts_realloc(ERTS_ALC_T_TMP,
-			 c_arg, (i+j+1) * sizeof(WCHAR *));
-
-    for(j = 0; a_arg[j] != NULL; ++j){
-	WCHAR **tmp;
-	WCHAR *current = a_arg[j];
-	WCHAR *eq_p = wcschr(current,L'=');
-	int unset = (eq_p!=NULL && eq_p[1]==L'\0');
-
-	if ((tmp = find_arg(c_arg, current)) != NULL) {
-	    if (!unset) {
-		*tmp = current;
-	    } else {
-		*tmp = c_arg[--i];
-		c_arg[i] = NULL;
-	    }
-	} else if (!unset) {
-	    c_arg[i++] = current;
-	    c_arg[i] = NULL;
-	}
-    }
-    ret = arg_to_env(c_arg);
-    erts_free(ERTS_ALC_T_TMP, c_arg);
-    erts_free(ERTS_ALC_T_TMP, a_arg);
-    return ret;
-}
-
-static WCHAR**
-find_arg(WCHAR **arg, WCHAR *str)
-{
-    WCHAR *tmp;
-    int len;
-
-    if ((tmp = wcschr(str, L'=')) != NULL) {
-	tmp++;
-	len = tmp - str;
-	while (*arg != NULL){
-	    if (_wcsnicmp(*arg, str, len) == 0){
-		return arg;
-	    }
-	    ++arg;
-	}
-    }
-    return NULL;
-}
-
-static int
-compare(const void *a, const void *b)
-{
-    WCHAR *s1 = *((WCHAR **) a);
-    WCHAR *s2 = *((WCHAR **) b);
-    WCHAR *e1 = wcschr(s1,L'=');
-    WCHAR *e2 = wcschr(s2,L'=');
-    int ret;
-    int len;
-  
-    if(!e1)
-	e1 = s1 + wcslen(s1);
-    if(!e2)
-	e2 = s2 + wcslen(s2);
-  
-    if((e1 - s1) > (e2 - s2))
-	len = (e2 - s2);
-    else
-	len = (e1 - s1);
-  
-    ret = _wcsnicmp(s1,s2,len);
-    if (ret == 0)
-	return ((e1 - s1) - (e2 - s2));
-    else
-	return ret;
-}
-
-static WCHAR**
-env_to_arg(WCHAR *env)
-{
-    WCHAR **ret;
-    WCHAR *tmp;
-    int i;
-    int num_strings = 0;
-
-    for(tmp = env; *tmp != '\0'; tmp += wcslen(tmp)+1) {
-	++num_strings;
-    }
-    ret = erts_alloc(ERTS_ALC_T_TMP, sizeof(WCHAR *) * (num_strings + 1));
-    i = 0;
-    for(tmp = env; *tmp != '\0'; tmp += wcslen(tmp)+1){
-	ret[i++] = tmp;
-    }
-    ret[i] = NULL;
-    return ret;
-}
-
-static WCHAR *
-arg_to_env(WCHAR **arg)
-{
-    WCHAR *block;
-    WCHAR *ptr;
-    int i;
-    int totlen = 1;		/* extra '\0' */
-
-    for(i = 0; arg[i] != NULL; ++i) {
-	totlen += wcslen(arg[i])+1;
-    }
-
-    /* sort the environment vector */
-    qsort(arg, i, sizeof(WCHAR *), &compare);
-
-    if (totlen == 1){
-	block = erts_alloc(ERTS_ALC_T_ENVIRONMENT, 2 * sizeof(WCHAR));
-	block[0] = block[1] = '\0'; 
-    } else {
-	block = erts_alloc(ERTS_ALC_T_ENVIRONMENT, totlen * sizeof(WCHAR));
-	ptr = block;
-	for(i=0; arg[i] != NULL; ++i){
-	    wcscpy(ptr, arg[i]);
-	    ptr += wcslen(ptr)+1;
-	}
-	*ptr = '\0';
-    }
-    return block;
-}
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2002-2017. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "sys.h"
+#include "erl_sys_driver.h"
+#include "erl_alloc.h"
+
+static erts_osenv_t sysenv_global_env;
+static erts_rwmtx_t sysenv_rwmtx;
+
+static void import_initial_env(void);
+
+void erts_sys_env_init() {
+    erts_rwmtx_init(&sysenv_rwmtx, "environ", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
+
+    erts_osenv_init(&sysenv_global_env);
+    import_initial_env();
+}
+
+const erts_osenv_t *erts_sys_rlock_global_osenv() {
+    erts_rwmtx_rlock(&sysenv_rwmtx);
+    return &sysenv_global_env;
+}
+
+erts_osenv_t *erts_sys_rwlock_global_osenv() {
+    erts_rwmtx_rwlock(&sysenv_rwmtx);
+    return &sysenv_global_env;
+}
+
+void erts_sys_runlock_global_osenv() {
+    erts_rwmtx_runlock(&sysenv_rwmtx);
+}
+
+void erts_sys_rwunlock_global_osenv() {
+    erts_rwmtx_rwunlock(&sysenv_rwmtx);
+}
+
+int erts_sys_explicit_host_getenv(char *key, char *value, size_t *size) {
+    size_t new_size = GetEnvironmentVariableA(key, value, (DWORD)*size);
+
+    if(new_size == 0 && GetLastError() == ERROR_ENVVAR_NOT_FOUND) {
+        return 0;
+    } else if(new_size > *size) {
+        return -1;
+    }
+
+    *size = new_size;
+    return 1;
+}
+
+int erts_sys_explicit_8bit_putenv(char *key, char *value) {
+    WCHAR *wide_key, *wide_value;
+    int key_length, value_length;
+    int result;
+
+    /* Note that we do *NOT* honor the filename encoding flags (+fnu/+fnl)
+     * here; the previous implementation used SetEnvironmentVariableA and
+     * things may break if we step away from that. */
+
+    key_length = MultiByteToWideChar(CP_ACP, 0, key, -1, NULL, 0);
+    value_length = MultiByteToWideChar(CP_ACP, 0, value, -1, NULL, 0);
+
+    /* Report "not found" if either string isn't convertible. */
+    if(key_length == 0 || value_length == 0) {
+        return 0;
+    }
+
+    wide_key = erts_alloc(ERTS_ALC_T_TMP, key_length * sizeof(WCHAR));
+    wide_value = erts_alloc(ERTS_ALC_T_TMP, value_length * sizeof(WCHAR));
+
+    MultiByteToWideChar(CP_ACP, 0, key, -1, wide_key, key_length);
+    MultiByteToWideChar(CP_ACP, 0, value, -1, wide_value, value_length);
+
+    {
+        erts_osenv_data_t env_key, env_value;
+        erts_osenv_t *env;
+
+        env = erts_sys_rwlock_global_osenv();
+
+        /* -1 to exclude the NUL terminator. */
+        env_key.length = (key_length - 1) * sizeof(WCHAR);
+        env_key.data = wide_key;
+
+        env_value.length = (value_length - 1) * sizeof(WCHAR);
+        env_value.data = wide_value;
+
+        result = erts_osenv_put_native(env, &env_key, &env_value);
+        erts_sys_rwunlock_global_osenv();
+    }
+
+    erts_free(ERTS_ALC_T_TMP, wide_key);
+    erts_free(ERTS_ALC_T_TMP, wide_value);
+
+    return result;
+}
+
+int erts_sys_explicit_8bit_getenv(char *key, char *value, size_t *size) {
+    erts_osenv_data_t env_key, env_value;
+    int key_length, value_length, result;
+    WCHAR *wide_key, *wide_value;
+
+    key_length = MultiByteToWideChar(CP_ACP, 0, key, -1, NULL, 0);
+
+    /* Report "not found" if the string isn't convertible. */
+    if(key_length == 0) {
+        return 0;
+    }
+
+    wide_key = erts_alloc(ERTS_ALC_T_TMP, key_length * sizeof(WCHAR));
+    MultiByteToWideChar(CP_ACP, 0, key, -1, wide_key, key_length);
+
+    /* We assume that the worst possible size is twice the output buffer width,
+     * as we could theoretically be on a code page that requires surrogates. */
+    value_length = (*size) * 2;
+    wide_value = erts_alloc(ERTS_ALC_T_TMP, value_length * sizeof(WCHAR));
+
+    {
+        const erts_osenv_t *env = erts_sys_rlock_global_osenv();
+
+        /* -1 to exclude the NUL terminator. */
+        env_key.length = (key_length - 1) * sizeof(WCHAR);
+        env_key.data = wide_key;
+
+        env_value.length = value_length * sizeof(WCHAR);
+        env_value.data = wide_value;
+
+        result = erts_osenv_get_native(env, &env_key, &env_value);
+        erts_sys_runlock_global_osenv();
+    }
+
+    if(result == 1 && env_value.length > 0) {
+        /* This function doesn't NUL-terminate if the provided size is >= 0,
+         * so we pass (*size - 1) to reserve space for it and then do it
+         * manually. */
+        *size = WideCharToMultiByte(CP_ACP, 0, env_value.data,
+            env_value.length / sizeof(WCHAR), value, *size - 1, NULL, NULL);
+
+        if(*size == 0) {
+            if(GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
+                result = -1;
+            } else {
+                result = 0;
+            }
+        }
+    } else {
+        *size = 0;
+    }
+
+    if(*size > 0) {
+        value[*size] = '\0';
+    }
+
+    erts_free(ERTS_ALC_T_TMP, wide_key);
+    erts_free(ERTS_ALC_T_TMP, wide_value);
+
+    return result;
+}
+
+static void import_initial_env(void) {
+    WCHAR *environment_block, *current_variable;
+
+    environment_block = GetEnvironmentStringsW();
+    current_variable = environment_block;
+
+    while(wcslen(current_variable) > 0) {
+        WCHAR *separator_index = wcschr(current_variable, L'=');
+
+        /* We tolerate environment variables starting with '=' as the per-drive
+         * working directories are stored this way. */
+        if(separator_index == current_variable) {
+            separator_index = wcschr(separator_index + 1, L'=');
+        }
+
+        if(separator_index != NULL && separator_index != current_variable) {
+            erts_osenv_data_t env_key, env_value;
+
+            env_key.length = (separator_index - current_variable) * sizeof(WCHAR);
+            env_key.data = current_variable;
+
+            env_value.length = (wcslen(separator_index) - 1) * sizeof(WCHAR);
+            env_value.data = separator_index + 1;
+
+            erts_osenv_put_native(&sysenv_global_env, &env_key, &env_value);
+        }
+
+        current_variable += wcslen(current_variable) + 1;
+    }
+
+    FreeEnvironmentStringsW(environment_block);
+}
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 475e03087a..884f697384 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -80,6 +80,7 @@
          async_blast/1,
          thr_msg_blast/1,
          consume_timeslice/1,
+         env/1,
          z_test/1]).
 
 -export([bin_prefix/2]).
@@ -166,6 +167,7 @@ all() -> %% Keep a_test first and z_test last...
      async_blast,
      thr_msg_blast,
      consume_timeslice,
+     env,
      z_test].
 
 groups() -> 
@@ -2360,6 +2362,51 @@ count_proc_sched(Ps, PNs) ->
               PNs
     end.
 
+%%
+%% Tests whether erl_drv_putenv reflects in os:getenv and vice versa.
+%%
+env(Config) when is_list(Config) ->
+    ok = load_driver(proplists:get_value(data_dir, Config), env_drv),
+    Port = open_port({spawn_driver, env_drv}, []),
+    true = is_port(Port),
+
+    Keys = ["env_drv_a_key", "env_drv_b_key", "env_drv_c_key"],
+    Values = ["a_value", "b_value", "c_value"],
+
+    [env_put_test(Port, Key, Value) || Key <- Keys, Value <- Values],
+    [env_get_test(Port, Key, Value) || Key <- Keys, Value <- Values],
+    [env_oversize_test(Port, Key) || Key <- Keys],
+    [env_notfound_test(Port, Key) || Key <- Keys],
+
+    true = port_close(Port),
+    erl_ddll:unload_driver(env_drv),
+    ok.
+
+env_control(Port, Command, Key, Value) ->
+    KeyBin = list_to_binary(Key),
+    ValueBin = list_to_binary(Value),
+    Header = <<(byte_size(KeyBin)), (byte_size(ValueBin))>>,
+    Payload = <<KeyBin/binary, ValueBin/binary>>,
+    port_control(Port, Command, <<Header/binary, Payload/binary>>).
+
+env_put_test(Port, Key, Value) ->
+    os:unsetenv(Key),
+    [0] = env_control(Port, 0, Key, Value),
+    Value = os:getenv(Key).
+
+env_get_test(Port, Key, ExpectedValue) ->
+    true = os:putenv(Key, ExpectedValue),
+    [0] = env_control(Port, 1, Key, ExpectedValue).
+
+env_oversize_test(Port, Key) ->
+    os:putenv(Key, [$A || _ <- lists:seq(1, 1024)]),
+    [127] = env_control(Port, 1, Key, "").
+
+env_notfound_test(Port, Key) ->
+    true = os:unsetenv(Key),
+    [255] = env_control(Port, 1, Key, "").
+
+
 a_test(Config) when is_list(Config) ->
     rpc(Config, fun check_io_debug/0).
 
diff --git a/erts/emulator/test/driver_SUITE_data/Makefile.src b/erts/emulator/test/driver_SUITE_data/Makefile.src
index 1fedd72200..bcabaa689d 100644
--- a/erts/emulator/test/driver_SUITE_data/Makefile.src
+++ b/erts/emulator/test/driver_SUITE_data/Makefile.src
@@ -16,7 +16,8 @@ MISC_DRVS =		outputv_drv@dll@ \
 			thr_free_drv@dll@ \
 			async_blast_drv@dll@ \
 			thr_msg_blast_drv@dll@ \
-			consume_timeslice_drv@dll@
+			consume_timeslice_drv@dll@ \
+			env_drv@dll@
 
 SYS_INFO_DRVS = 	sys_info_base_drv@dll@ \
 			sys_info_prev_drv@dll@ \
diff --git a/erts/emulator/test/driver_SUITE_data/env_drv.c b/erts/emulator/test/driver_SUITE_data/env_drv.c
new file mode 100644
index 0000000000..0e910eeb84
--- /dev/null
+++ b/erts/emulator/test/driver_SUITE_data/env_drv.c
@@ -0,0 +1,108 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2017. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/* Tests whether erl_drv_putenv/erl_drv_getenv work correctly and reflect
+ * changes to os:putenv/getenv. */
+
+#include <string.h>
+#include <stdio.h>
+
+#include "erl_driver.h"
+
+static ErlDrvSSizeT env_drv_ctl(ErlDrvData drv_data, unsigned int cmd,
+        char* buf, ErlDrvSizeT len, char** rbuf, ErlDrvSizeT rsize);
+
+static ErlDrvEntry env_drv_entry = { 
+    NULL /* init */,
+    NULL /* start */,
+    NULL /* stop */,
+    NULL /* output */,
+    NULL /* ready_input */,
+    NULL /* ready_output */,
+    "env_drv",
+    NULL /* finish */,
+    NULL /* handle */,
+    env_drv_ctl,
+    NULL /* timeout */,
+    NULL /* outputv*/,
+    NULL /* ready_async */,
+    NULL /* flush */,
+    NULL /* call*/,
+    NULL /* event */,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    ERL_DRV_FLAG_USE_PORT_LOCKING,
+    NULL /* handle2 */,
+    NULL /* handle_monitor */
+};
+
+DRIVER_INIT(env_drv) {
+    return &env_drv_entry;
+}
+
+static int test_putenv(ErlDrvData drv_data, char *buf, ErlDrvSizeT len) {
+    char key[256], value[256];
+    int key_len, value_len;
+
+    key_len = buf[0];
+    value_len = buf[1];
+
+    sprintf(key, "%.*s", key_len, &buf[2]);
+    sprintf(value, "%.*s", value_len, &buf[2 + key_len]);
+
+    return erl_drv_putenv(key, value);
+}
+
+static int test_getenv(ErlDrvData drv_data, char *buf, ErlDrvSizeT len) {
+    char expected_value[256], stored_value[256], key[256];
+    int expected_value_len, key_len;
+    size_t stored_value_len;
+    int res;
+
+    key_len = buf[0];
+    sprintf(key, "%.*s", key_len, &buf[2]);
+
+    expected_value_len = buf[1];
+    sprintf(expected_value, "%.*s", expected_value_len, &buf[2 + key_len]);
+
+    stored_value_len = sizeof(stored_value);
+    res = erl_drv_getenv(key, stored_value, &stored_value_len);
+
+    if(res == 0) {
+        return strcmp(stored_value, expected_value) != 0;
+    } else if(res == 1) {
+        return 127;
+    }
+
+    return 255;
+}
+
+static ErlDrvSSizeT env_drv_ctl(ErlDrvData drv_data, unsigned int cmd,
+        char* buf, ErlDrvSizeT len, char** rbuf, ErlDrvSizeT rsize) {
+
+    if(cmd == 0) {
+        (**rbuf) = (char)test_putenv(drv_data, buf, len);
+    } else {
+        (**rbuf) = (char)test_getenv(drv_data, buf, len);
+    }
+
+    return 1;
+}
-- 
cgit v1.2.3


From 65df0ce5cec5fdee60c80409f322a58092526537 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 29 Nov 2017 12:54:21 +0100
Subject: Alter erl_ddll:load dependency search path on Windows

The standard search order does not include the directory that the
loaded DLL resides in, requiring dirty $PATH hacks to get things
working, which is not an option now that os:putenv/2 is divorced
from the OS environment.
---
 erts/emulator/sys/win32/erl_win32_sys_ddll.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/win32/erl_win32_sys_ddll.c b/erts/emulator/sys/win32/erl_win32_sys_ddll.c
index 274133a346..fc2179328f 100644
--- a/erts/emulator/sys/win32/erl_win32_sys_ddll.c
+++ b/erts/emulator/sys/win32/erl_win32_sys_ddll.c
@@ -46,9 +46,17 @@ static TWinDynDriverCallbacks wddc;
 static TWinDynNifCallbacks nif_callbacks;
 
 void erl_sys_ddll_init(void) {
+    WCHAR cwd_buffer[MAX_PATH];
+
     tls_index = TlsAlloc();
     ERL_INIT_CALLBACK_STRUCTURE(wddc);
 
+    /* LOAD_WITH_ALTERED_SEARCH_PATH removes the startup directory from the
+     * search path, so we add it separately to be backwards compatible. */
+    if (GetCurrentDirectoryW(sizeof(cwd_buffer), cwd_buffer)) {
+        SetDllDirectoryW(cwd_buffer);
+    }
+
 #define ERL_NIF_API_FUNC_DECL(RET,NAME,ARGS) nif_callbacks.NAME = NAME
 #include "erl_nif_api_funcs.h"
 #undef ERL_NIF_API_FUNC_DECL
@@ -81,7 +89,10 @@ int erts_sys_ddll_open(const char *full_name, void **handle, ErtsSysDdllError* e
 						   ERTS_ALC_T_TMP, &used, EXT_LEN);
     wcscpy(&wcp[used/2 - 1], FILE_EXT_WCHAR);
 
-    if ((hinstance = LoadLibraryW(wcp)) == NULL) {
+    /* LOAD_WITH_ALTERED_SEARCH_PATH adds the specified DLL's directory to the
+     * dependency search path. This also removes the directory we started in,
+     * but we've explicitly added that in in erl_sys_ddll_init. */
+    if ((hinstance = LoadLibraryExW(wcp, NULL, LOAD_WITH_ALTERED_SEARCH_PATH)) == NULL) {
 	code = ERL_DE_DYNAMIC_ERROR_OFFSET - GetLastError();
 	if (err != NULL) {
 	    err->str = erts_sys_ddll_error(code);
-- 
cgit v1.2.3


From d469368b9e14b9834017a7cf318f02950a4aadcb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Fri, 1 Dec 2017 12:22:53 +0100
Subject: Disallow NULs in filename-encoded strings

Previously we accepted trailing NULs, which was backwards compatible
as such usage never resulted in misbehavior in the first place. The
downside is that it prevented erts_native_filename_need from
returning an accurate number of *actual characters*, needlessly
complicating encoding-agnostic code like erts_osenv.
---
 erts/emulator/beam/erl_unicode.c | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index bd5439ba24..e5c7a9502b 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -2146,7 +2146,6 @@ Sint erts_native_filename_need(Eterm ioterm, int encoding)
     Eterm obj;
     DECLARE_ESTACK(stack);
     Sint need = 0;
-    int seen_null = 0;
 
     if (is_atom(ioterm)) {
 	Atom* ap;
@@ -2191,9 +2190,7 @@ Sint erts_native_filename_need(Eterm ioterm, int encoding)
             byte *name = ap->name;
             int len = ap->len;
             for (i = 0; i < len; i++) {
-                if (name[i] == 0)
-                    seen_null = 1;
-                else if (seen_null) {
+                if (name[i] == 0) {
                     need = -1;
                     break;
                 }
@@ -2233,9 +2230,7 @@ L_Again:   /* Restart with sublist, old listend was pushed on stack */
                          * Do not allow null in
                          * the middle of filenames
                          */
-                        if (x == 0)
-                            seen_null = 1;
-                        else if (seen_null) {
+                        if (x == 0) {
                             DESTROY_ESTACK(stack);
                             return ((Sint) -1);
                         }
@@ -2568,7 +2563,6 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
 	BIF_ERROR(BIF_P,BADARG);
     }
     if (is_binary(BIF_ARG_1)) {
-        int seen_null = 0;
 	byte *temp_alloc = NULL;
 	byte *bytes;
 	byte *err_pos;
@@ -2585,8 +2579,6 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
             for (i = 0; i < size; i++) {
                 /* Don't allow null in the middle of filenames... */
                 if (bytes[i] == 0)
-                    seen_null = 1;
-                else if (seen_null)
                     goto bin_name_error;
                 bin_p[i] = bytes[i];
             }
@@ -2605,8 +2597,6 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
 	    while (size--) {
                 /* Don't allow null in the middle of filenames... */
                 if (*bytes == 0)
-                    seen_null = 1;
-                else if (seen_null)
                     goto bin_name_error;
 		*bin_p++ = *bytes++;
 		*bin_p++ = 0;
-- 
cgit v1.2.3


From 3d21f793538927ae88f78504a11dd898e8ca1a7a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 20 Dec 2017 17:18:31 +0100
Subject: Fix bug in hipe primop bs_put_utf8

by preventing it from doing GC, which generated code relies on.
---
 erts/emulator/hipe/hipe_bif_list.m4  | 2 +-
 erts/emulator/hipe/hipe_native_bif.c | 9 +++------
 erts/emulator/hipe/hipe_native_bif.h | 2 +-
 3 files changed, 5 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index b86f2dafdc..b2fccdadef 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -245,7 +245,7 @@ noproc_primop_interface_2(nbif_eq_2, eq)
 nofail_primop_interface_3(nbif_bs_get_integer_2, erts_bs_get_integer_2)
 nofail_primop_interface_3(nbif_bs_get_binary_2, erts_bs_get_binary_2)
 nofail_primop_interface_3(nbif_bs_get_float_2, erts_bs_get_float_2)
-standard_bif_interface_3(nbif_bs_put_utf8, hipe_bs_put_utf8)
+nocons_nofail_primop_interface_3(nbif_bs_put_utf8, hipe_bs_put_utf8)
 standard_bif_interface_3(nbif_bs_put_utf16be, hipe_bs_put_utf16be)
 standard_bif_interface_3(nbif_bs_put_utf16le, hipe_bs_put_utf16le)
 ifdef(`nogc_bif_interface_1',`
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index e1c22701d0..6ab7a9e1de 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -398,12 +398,8 @@ Eterm hipe_bs_utf8_size(Eterm arg)
 	return make_small(4);
 }
 
-BIF_RETTYPE nbif_impl_hipe_bs_put_utf8(NBIF_ALIST_3)
+Eterm hipe_bs_put_utf8(Process* p, Eterm arg, byte* base, Uint offset)
 {
-    Process* p = BIF_P;
-    Eterm arg = BIF_ARG_1;
-    byte* base = (byte*) BIF_ARG_2;
-    Uint offset = (Uint) BIF_ARG_3;
     byte *save_bin_buf;
     Uint save_bin_offset;
     int res;
@@ -419,7 +415,8 @@ BIF_RETTYPE nbif_impl_hipe_bs_put_utf8(NBIF_ALIST_3)
     erts_current_bin = save_bin_buf;
     erts_bin_offset = save_bin_offset;
     if (res == 0)
-	BIF_ERROR(p, BADARG);
+        return 0;
+    ASSERT(new_offset != 0);
     return new_offset;
 }
 
diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h
index 1127d4ac56..6321e66e7a 100644
--- a/erts/emulator/hipe/hipe_native_bif.h
+++ b/erts/emulator/hipe/hipe_native_bif.h
@@ -88,7 +88,7 @@ Binary *hipe_bs_reallocate(Binary*, int);
 int hipe_bs_put_small_float(Process*, Eterm, Uint, byte*, unsigned, unsigned);
 void hipe_bs_put_bits(Eterm, Uint, byte*, unsigned, unsigned);
 Eterm hipe_bs_utf8_size(Eterm);
-BIF_RETTYPE nbif_impl_hipe_bs_put_utf8(NBIF_ALIST_3);
+Eterm hipe_bs_put_utf8(Process*, Eterm arg, byte* base, Uint offset);
 Eterm hipe_bs_utf16_size(Eterm);
 BIF_RETTYPE nbif_impl_hipe_bs_put_utf16be(NBIF_ALIST_3);
 BIF_RETTYPE nbif_impl_hipe_bs_put_utf16le(NBIF_ALIST_3);
-- 
cgit v1.2.3


From 898352fd68022432c1e6f8d9b7a926394d3e8899 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 21 Dec 2017 12:00:28 +0100
Subject: Add PRIMOP_ABI_VSN to erts checksum

in order to detect incompatible changes in primop interface
(which we just did for bs_put_utf8) and refuse hipe loading.
---
 erts/emulator/hipe/hipe_mkliterals.c | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c
index 4573980e1e..1ebe4e1188 100644
--- a/erts/emulator/hipe/hipe_mkliterals.c
+++ b/erts/emulator/hipe/hipe_mkliterals.c
@@ -535,6 +535,11 @@ static const struct rts_param rts_params[] = {
 static unsigned int literals_crc;
 static unsigned int system_crc;
 
+/*
+ * Change this version value to detect incompatible changes in primop interface.
+ */
+#define PRIMOP_ABI_VSN 0x090300  /* erts-9.3 */
+
 static void compute_crc(void)
 {
     unsigned int crc_value;
@@ -550,6 +555,8 @@ static void compute_crc(void)
     for (i = 0; i < NR_PARAMS; ++i)
 	if (rts_params[i].is_defined)
 	    crc_value = crc_update_int(crc_value, &rts_params[i].value);
+
+    crc_value ^= PRIMOP_ABI_VSN;
     crc_value &= 0x07FFFFFF;
     system_crc = crc_value;
 }
-- 
cgit v1.2.3


From 434e4774a4f7e70437ee8a50c7b99ff3bda67282 Mon Sep 17 00:00:00 2001
From: Frank Hunleth <fhunleth@troodon-software.com>
Date: Wed, 3 Jan 2018 18:10:12 -0500
Subject: Fail if ':' is passed to binary_to_integer/2

Before:

1> binary_to_integer(<<":">>, 16).
3

After:

1> binary_to_integer(<<":">>, 16).
** exception error: bad argument
     in function  binary_to_integer/2
        called as binary_to_integer(<<":">>,16)

Prior to this change, both list_to_integer/2 and binary_to_integer/2
would convert strings with values between ASCII '9' up to '0'+base for
base > 10. For example, when converting in base 16, you could pass ':',
';', '<', '=', '>', and '?' without getting an exception.  This was due
to a missing check in c2int_is_invalid_char().

This change adds the missing check and a regression test for passing
':'. It also simplifies the code and tightens up an out-of-bounds check
to make it off-by-one rather than off-by-two.
---
 erts/emulator/beam/big.c             | 15 ++++++++++-----
 erts/emulator/test/num_bif_SUITE.erl |  2 +-
 2 files changed, 11 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 5eaf262cd8..c5cb268f09 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -2549,12 +2549,17 @@ int term_equals_2pow32(Eterm x)
     }
 }
 
+static ERTS_INLINE int c2int_is_valid_char(byte ch, int base) {
+    if (base <= 10)
+        return (ch >= '0' && ch < ('0' + base));
+    else
+        return (ch >= '0' && ch <= '9')
+            || (ch >= 'A' && ch < ('A' + base - 10))
+            || (ch >= 'a' && ch < ('a' + base - 10));
+}
+
 static ERTS_INLINE int c2int_is_invalid_char(byte ch, int base) {
-    return (ch < '0'
-            || (ch > ('0' + base - 1)
-                && !(base > 10
-                     && ((ch >= 'a' && ch < ('a' + base - 10))
-                         || (ch >= 'A' && ch < ('A' + base - 10))))));
+    return !c2int_is_valid_char(ch, base);
 }
 
 static ERTS_INLINE byte c2int_digit_from_base(byte ch) {
diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl
index 1c76eb8019..ac1a667185 100644
--- a/erts/emulator/test/num_bif_SUITE.erl
+++ b/erts/emulator/test/num_bif_SUITE.erl
@@ -491,7 +491,7 @@ t_string_to_integer(Config) when is_list(Config) ->
 				       list_to_binary(Value),Base)),
 			  {'EXIT', {badarg, _}} =
 			      (catch erlang:list_to_integer(Value,Base))
-		  end,[{" 1",1},{" 1",37},{"2",2},{"C",11},
+		  end,[{" 1",1},{" 1",37},{"2",2},{"B",11},{"b",11},{":", 16},
 		       {"1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111z",16},
 		       {"1z111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111",16},
 		       {"111z11111111",16}]),
-- 
cgit v1.2.3


From 19f84054b621aaa9ad05748cc72057726faeda01 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 4 Jan 2018 20:39:11 +0100
Subject: erts: Refactor ProcBin creation

into utility functions.
---
 erts/emulator/beam/binary.c | 63 +++++++++++++++++++--------------------------
 erts/emulator/beam/global.h |  1 +
 2 files changed, 27 insertions(+), 37 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index ca3e48e205..d4db1a4188 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -60,14 +60,36 @@ erts_init_binary(void)
 
 }
 
+static ERTS_INLINE
+Eterm build_proc_bin(ErlOffHeap* ohp, Eterm* hp, Binary* bptr)
+{
+    ProcBin* pb = (ProcBin *) hp;
+    pb->thing_word = HEADER_PROC_BIN;
+    pb->size = bptr->orig_size;
+    pb->next = ohp->first;
+    ohp->first = (struct erl_off_heap_header*)pb;
+    pb->val = bptr;
+    pb->bytes = (byte*) bptr->orig_bytes;
+    pb->flags = 0;
+    OH_OVERHEAD(ohp, pb->size / sizeof(Eterm));
+
+    return make_binary(pb);
+}
+
+/** @brief Initiate a ProcBin for a full Binary.
+ *  @param hp must point to PROC_BIN_SIZE available heap words.
+ */
+Eterm erts_build_proc_bin(ErlOffHeap* ohp, Eterm* hp, Binary* bptr)
+{
+    return build_proc_bin(ohp, hp, bptr);
+}
+
 /*
  * Create a brand new binary from scratch.
  */
-
 Eterm
 new_binary(Process *p, byte *buf, Uint len)
 {
-    ProcBin* pb;
     Binary* bptr;
 
     if (len <= ERL_ONHEAP_BIN_LIMIT) {
@@ -88,23 +110,7 @@ new_binary(Process *p, byte *buf, Uint len)
 	sys_memcpy(bptr->orig_bytes, buf, len);
     }
 
-    /*
-     * Now allocate the ProcBin on the heap.
-     */
-    pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
-    pb->thing_word = HEADER_PROC_BIN;
-    pb->size = len;
-    pb->next = MSO(p).first;
-    MSO(p).first = (struct erl_off_heap_header*)pb;
-    pb->val = bptr;
-    pb->bytes = (byte*) bptr->orig_bytes;
-    pb->flags = 0;
-
-    /*
-     * Miscellaneous updates. Return the tagged binary.
-     */
-    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
-    return make_binary(pb);
+    return build_proc_bin(&MSO(p), HAlloc(p, PROC_BIN_SIZE), bptr);
 }
 
 /* 
@@ -113,7 +119,6 @@ new_binary(Process *p, byte *buf, Uint len)
 
 Eterm erts_new_mso_binary(Process *p, byte *buf, Uint len)
 {
-    ProcBin* pb;
     Binary* bptr;
 
     /*
@@ -124,23 +129,7 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, Uint len)
 	sys_memcpy(bptr->orig_bytes, buf, len);
     }
 
-    /*
-     * Now allocate the ProcBin on the heap.
-     */
-    pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
-    pb->thing_word = HEADER_PROC_BIN;
-    pb->size = len;
-    pb->next = MSO(p).first;
-    MSO(p).first = (struct erl_off_heap_header*)pb;
-    pb->val = bptr;
-    pb->bytes = (byte*) bptr->orig_bytes;
-    pb->flags = 0;
-
-    /*
-     * Miscellaneous updates. Return the tagged binary.
-     */
-    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
-    return make_binary(pb);
+    return build_proc_bin(&MSO(p), HAlloc(p, PROC_BIN_SIZE), bptr);
 }
 
 /*
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 9505942307..87777d14e9 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -867,6 +867,7 @@ Eterm erts_new_heap_binary(Process *p, byte *buf, int len, byte** datap);
 Eterm erts_new_mso_binary(Process*, byte*, Uint);
 Eterm new_binary(Process*, byte*, Uint);
 Eterm erts_realloc_binary(Eterm bin, size_t size);
+Eterm erts_build_proc_bin(ErlOffHeap*, Eterm*, Binary*);
 
 /* erl_bif_info.c */
 
-- 
cgit v1.2.3


From 22c2151b2823e8d74567e04c63ed2678f85f83cf Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 4 Jan 2018 16:30:04 +0100
Subject: erts: Add non fatal big binary creation for test purpose

---
 erts/emulator/beam/erl_bif_info.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 80adca0072..4050fb6146 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -4447,6 +4447,19 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
                 BIF_RET(res);
             }
         }
+        else if (ERTS_IS_ATOM_STR("binary", BIF_ARG_1)) {
+            Sint64 size;
+            if (term_to_Sint64(BIF_ARG_2, &size)) {
+                Binary* refbin = erts_bin_drv_alloc_fnf(size);
+                if (!refbin)
+                    BIF_RET(am_false);
+                sys_memset(refbin->orig_bytes, 0, size);
+                BIF_RET(erts_build_proc_bin(&MSO(BIF_P),
+                                            HAlloc(BIF_P, PROC_BIN_SIZE),
+                                            refbin));
+            }
+        }
+
     }
 
     BIF_ERROR(BIF_P, BADARG);
-- 
cgit v1.2.3


From f0123ff032ea2f9b4aa30322ad476e3f79757e79 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 4 Jan 2018 19:04:09 +0100
Subject: erts: Refactor more usage of erts_build_proc_bin

---
 erts/emulator/beam/erl_bif_binary.c | 23 +++++---------
 erts/emulator/beam/erl_io_queue.c   | 19 +++---------
 erts/emulator/beam/erl_nif.c        | 16 ++--------
 erts/emulator/beam/external.c       | 62 +++++++++----------------------------
 erts/emulator/beam/io.c             | 44 +++++---------------------
 5 files changed, 37 insertions(+), 127 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index dcffde5777..7bea1630cb 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -2731,28 +2731,21 @@ BIF_RETTYPE binary_copy_trap(BIF_ALIST_2)
 	BIF_TRAP2(&binary_copy_trap_export, BIF_P, BIF_ARG_1, BIF_ARG_2);
     } else {
 	Binary *save;
-	ProcBin* pb;
+        Eterm resbin;
 	Uint target_size = cbs->result->orig_size;
 	while (pos < target_size) {
 	    memcpy(t+pos,cbs->source, size);
 	    pos += size;
 	}
-	save =  cbs->result;
+	save = cbs->result;
 	cbs->result = NULL;
 	cleanup_copy_bin_state(mb); /* now cbs is dead */
-	pb = (ProcBin *) HAlloc(BIF_P, PROC_BIN_SIZE);
-	pb->thing_word = HEADER_PROC_BIN;
-	pb->size = target_size;
-	pb->next = MSO(BIF_P).first;
-	MSO(BIF_P).first = (struct erl_off_heap_header*) pb;
-	pb->val = save;
-	pb->bytes = t;
-	pb->flags = 0;
-
-	OH_OVERHEAD(&(MSO(BIF_P)), target_size / sizeof(Eterm));
-	BUMP_REDS(BIF_P,(pos - opos) / BINARY_COPY_LOOP_FACTOR);
-
-	BIF_RET(make_binary(pb));
+
+        resbin = erts_build_proc_bin(&MSO(BIF_P),
+                                     HAlloc(BIF_P, PROC_BIN_SIZE),
+                                     save);
+        BUMP_REDS(BIF_P,(pos - opos) / BINARY_COPY_LOOP_FACTOR);
+	BIF_RET(resbin);
     }
 }
 
diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c
index 190ba6bbb9..f678889f81 100644
--- a/erts/emulator/beam/erl_io_queue.c
+++ b/erts/emulator/beam/erl_io_queue.c
@@ -817,24 +817,15 @@ static Eterm iol2v_make_sub_bin(iol2v_state_t *state, Eterm bin_term,
 }
 
 static Eterm iol2v_promote_acc(iol2v_state_t *state) {
-    ProcBin *pb;
-
-    state->acc = erts_bin_realloc(state->acc, state->acc_size);
-
-    pb = (ProcBin*)HAlloc(state->process, PROC_BIN_SIZE);
-    pb->thing_word = HEADER_PROC_BIN;
-    pb->size = state->acc_size;
-    pb->val = state->acc;
-    pb->bytes = (byte*)(state->acc)->orig_bytes;
-    pb->flags = 0;
-    pb->next = MSO(state->process).first;
-    OH_OVERHEAD(&(MSO(state->process)), pb->size / sizeof(Eterm));
-    MSO(state->process).first = (struct erl_off_heap_header*)pb;
+    Eterm bin;
 
+    bin = erts_build_proc_bin(&MSO(state->process),
+                              HAlloc(state->process, PROC_BIN_SIZE),
+                              erts_bin_realloc(state->acc, state->acc_size));
     state->acc_size = 0;
     state->acc = NULL;
 
-    return make_binary(pb);
+    return bin;
 }
 
 /* Destructively enqueues a term to the result list, saving us the hassle of
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index f67b67325d..682df1021c 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1343,21 +1343,11 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin)
     }
     else if (bin->ref_bin != NULL) {
 	Binary* bptr = bin->ref_bin;
-	ProcBin* pb;
 	Eterm bin_term;
 	
-	/* !! Copy-paste from new_binary() !! */
-	pb = (ProcBin *) alloc_heap(env, PROC_BIN_SIZE);
-	pb->thing_word = HEADER_PROC_BIN;
-	pb->size = bptr->orig_size;
-	pb->next = MSO(env->proc).first;
-	MSO(env->proc).first = (struct erl_off_heap_header*) pb;
-	pb->val = bptr;
-	pb->bytes = (byte*) bptr->orig_bytes;
-	pb->flags = 0;
-	
-	OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm));
-	bin_term = make_binary(pb);	
+        bin_term = erts_build_proc_bin(&MSO(env->proc),
+                                       alloc_heap(env, PROC_BIN_SIZE),
+                                       bptr);
 	if (erts_refc_read(&bptr->intern.refc, 1) == 1) {
 	    /* Total ownership transfer */
 	    bin->ref_bin = NULL;
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index c0a3838d42..050c47c1c3 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1949,23 +1949,14 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 		level = context->s.ec.level;
 		BUMP_REDS(p, (initial_reds - reds) / TERM_TO_BINARY_LOOP_FACTOR);
 		if (level == 0 || real_size < 6) { /* We are done */
-		    ProcBin* pb;
 		return_normal:
 		    context->s.ec.result_bin = NULL;
 		    context->alive = 0;
-		    pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
-		    pb->thing_word = HEADER_PROC_BIN;
-		    pb->size = real_size;
-		    pb->next = MSO(p).first;
-		    MSO(p).first = (struct erl_off_heap_header*)pb;
-		    pb->val = result_bin;
-		    pb->bytes = (byte*) result_bin->orig_bytes;
-		    pb->flags = 0;
-		    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
 		    if (context_b && erts_refc_read(&context_b->intern.refc,0) == 0) {
 			erts_bin_free(context_b);
 		    }
-		    return make_binary(pb);
+		    return erts_build_proc_bin(&MSO(p), HAlloc(p, PROC_BIN_SIZE),
+                                               result_bin);
 		}
 		/* Continue with compression... */
 		/* To make absolutely sure that zlib does not barf on a reallocated context, 
@@ -2022,16 +2013,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 			result_bin = erts_bin_realloc(context->s.cc.destination_bin,
 						      context->s.cc.dest_len+6);
 			context->s.cc.destination_bin = NULL;
-			pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
-			pb->thing_word = HEADER_PROC_BIN;
-			pb->size = context->s.cc.dest_len+6;
-			pb->next = MSO(p).first;
-			MSO(p).first = (struct erl_off_heap_header*)pb;
-			pb->val = result_bin;
 			ASSERT(erts_refc_read(&result_bin->intern.refc, 1));
-			pb->bytes = (byte*) result_bin->orig_bytes;
-			pb->flags = 0;
-			OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
 			erts_bin_free(context->s.cc.result_bin);
 			context->s.cc.result_bin = NULL;
 			context->alive = 0;
@@ -2039,7 +2021,9 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 			if (context_b && erts_refc_read(&context_b->intern.refc,0) == 0) {
 			    erts_bin_free(context_b);
 			}
-			return make_binary(pb);
+			return erts_build_proc_bin(&MSO(p),
+						   HAlloc(p, PROC_BIN_SIZE),
+                                                   result_bin);
 		    }
 		default: /* Compression error, revert to uncompressed binary (still in 
 			    context) */
@@ -3535,18 +3519,9 @@ dec_term_atom_common:
 		    *objp = make_binary(hb);
 		} else {
 		    Binary* dbin = erts_bin_nrml_alloc(n);
-		    ProcBin* pb;
-		    pb = (ProcBin *) hp;
+
+		    *objp = erts_build_proc_bin(factory->off_heap, hp, dbin);
 		    hp += PROC_BIN_SIZE;
-		    pb->thing_word = HEADER_PROC_BIN;
-		    pb->size = n;
-		    pb->next = factory->off_heap->first;
-		    factory->off_heap->first = (struct erl_off_heap_header*)pb;
-		    OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm));
-		    pb->val = dbin;
-		    pb->bytes = (byte*) dbin->orig_bytes;
-		    pb->flags = 0;
-		    *objp = make_binary(pb);
                     if (ctx) {
                         int n_limit = reds * B2T_MEMCPY_FACTOR;
                         if (n > n_limit) {
@@ -3586,18 +3561,9 @@ dec_term_atom_common:
                     ep += n;
 		} else {
 		    Binary* dbin = erts_bin_nrml_alloc(n);
-		    ProcBin* pb;
+		    Uint n_copy = n;
 
-		    pb = (ProcBin *) hp;
-		    pb->thing_word = HEADER_PROC_BIN;
-		    pb->size = n;
-		    pb->next = factory->off_heap->first;
-		    factory->off_heap->first = (struct erl_off_heap_header*)pb;
-		    OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm));
-		    pb->val = dbin;
-		    pb->bytes = (byte*) dbin->orig_bytes;
-		    pb->flags = 0;
-		    bin = make_binary(pb);
+		    bin = erts_build_proc_bin(factory->off_heap, hp, dbin);
 		    hp += PROC_BIN_SIZE;
                     if (ctx) {
                         int n_limit = reds * B2T_MEMCPY_FACTOR;
@@ -3605,15 +3571,15 @@ dec_term_atom_common:
                             ctx->state = B2TDecodeBinary;
                             ctx->u.dc.remaining_n = n - n_limit;
                             ctx->u.dc.remaining_bytes = dbin->orig_bytes + n_limit;
-                            n = n_limit;
+                            n_copy = n_limit;
                             reds = 0;
                         }
-                        else
+                        else {
                             reds -= n / B2T_MEMCPY_FACTOR;
+			}
                     }
-                    sys_memcpy(dbin->orig_bytes, ep, n);
-                    ep += n;
-                    n = pb->size;
+                    sys_memcpy(dbin->orig_bytes, ep, n_copy);
+                    ep += n_copy;
                 }
 
 		if (bitsize == 8 || n == 0) {
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 3a5ddde5f4..101b8c3db3 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -3434,24 +3434,11 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
     if ((state & ERTS_PORT_SFLG_BINARY_IO) == 0) {
 	listp = buf_to_intlist(&hp, buf, len, listp);
     } else if (buf != NULL) {
-	ProcBin* pb;
-	Binary* bptr;
-
-	bptr = erts_bin_nrml_alloc(len);
+	Binary* bptr = erts_bin_nrml_alloc(len);
 	sys_memcpy(bptr->orig_bytes, buf, len);
 
-	pb = (ProcBin *) hp;
-	pb->thing_word = HEADER_PROC_BIN;
-	pb->size = len;
-	pb->next = ohp->first;
-	ohp->first = (struct erl_off_heap_header*)pb;
-	pb->val = bptr;
-	pb->bytes = (byte*) bptr->orig_bytes;
-	pb->flags = 0;
+        listp = erts_build_proc_bin(ohp, hp, bptr);
 	hp += PROC_BIN_SIZE;
-
-	OH_OVERHEAD(ohp, pb->size / sizeof(Eterm));
-	listp = make_binary(pb);
     }
 
     /* Prepend the header */
@@ -4257,17 +4244,9 @@ write_port_control_result(int control_flags,
 	else {
 	    dbin = (ErlDrvBinary *) resp_bufp;
 	    if (dbin->orig_size > ERL_ONHEAP_BIN_LIMIT) {
-		ProcBin* pb = (ProcBin *) *hpp;
+                res = erts_build_proc_bin(ohp, *hpp, ErlDrvBinary2Binary(dbin));
 		*hpp += PROC_BIN_SIZE;
-		pb->thing_word = HEADER_PROC_BIN;
-		pb->size = dbin->orig_size;
-		pb->next = ohp->first;
-		ohp->first = (struct erl_off_heap_header *) pb;
-		pb->val = ErlDrvBinary2Binary(dbin);
-		pb->bytes = (byte*) dbin->orig_bytes;
-		pb->flags = 0;
-		OH_OVERHEAD(ohp, dbin->orig_size / sizeof(Eterm));
-		return make_binary(pb);
+                return res;
 	    }
 	    resp_bufp = dbin->orig_bytes;
 	    resp_size = dbin->orig_size;
@@ -6040,21 +6019,12 @@ driver_deliver_term(Port *prt, Eterm to, ErlDrvTermData* data, int len)
 		mess = make_binary(hbp);
 	    }
 	    else {
-		ProcBin* pbp;
+		Eterm* hp;
 		Binary* bp = erts_bin_nrml_alloc(size);
 		ASSERT(bufp);
 		sys_memcpy((void *) bp->orig_bytes, (void *) bufp, size);
-		pbp = (ProcBin *) erts_produce_heap(&factory,
-						    PROC_BIN_SIZE, HEAP_EXTRA);
-		pbp->thing_word = HEADER_PROC_BIN;
-		pbp->size = size;
-		pbp->next = factory.off_heap->first;
-		factory.off_heap->first = (struct erl_off_heap_header*)pbp;
-		pbp->val = bp;
-		pbp->bytes = (byte*) bp->orig_bytes;
-		pbp->flags = 0;
-		OH_OVERHEAD(factory.off_heap, pbp->size / sizeof(Eterm));
-		mess = make_binary(pbp);
+                hp = erts_produce_heap(&factory, PROC_BIN_SIZE, HEAP_EXTRA);
+                mess = erts_build_proc_bin(factory.off_heap, hp, bp);
 	    }
 	    ptr += 2;
 	    break;
-- 
cgit v1.2.3


From 2e601a2efc19d64ed0628a5973596e6331ddcc7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 8 Jan 2018 11:45:33 +0100
Subject: Remove sender punishment

The reduction cost of sending messages is now constant and will no
longer scale according to the length of the receiving process'
message queue.
---
 erts/emulator/beam/bif.c      | 29 ++++++-----------------------
 erts/emulator/beam/erl_init.c |  7 -------
 erts/emulator/beam/global.h   |  1 -
 3 files changed, 6 insertions(+), 31 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 0e5f7bb22b..f086c434ea 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2244,20 +2244,15 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx)
     
  send_message: {
 	ErtsProcLocks rp_locks = 0;
-	Sint res;
 	if (p == rp)
 	    rp_locks |= ERTS_PROC_LOCK_MAIN;
 	/* send to local process */
-	res = erts_send_message(p, rp, &rp_locks, msg, 0);
-	if (erts_use_sender_punish)
-	    res *= 4;
-	else
-	    res = 0;
+	erts_send_message(p, rp, &rp_locks, msg, 0);
 	erts_proc_unlock(rp,
 			     p == rp
 			     ? (rp_locks & ~ERTS_PROC_LOCK_MAIN)
 			     : rp_locks);
-	return res;
+	return 0;
     }
 }
 
@@ -2312,8 +2307,8 @@ BIF_RETTYPE send_3(BIF_ALIST_3)
     result = do_send(p, to, msg, &ref, ctx);
     ERTS_MSACC_POP_STATE_M_X();
 
-    if (result > 0) {
-	ERTS_VBUMP_REDS(p, result);
+    if (result >= 0) {
+	ERTS_VBUMP_REDS(p, 4);
 	if (ERTS_IS_PROC_OUT_OF_REDS(p))
 	    goto yield_return;
 	ERTS_BIF_PREP_RET(retval, am_ok);
@@ -2321,12 +2316,6 @@ BIF_RETTYPE send_3(BIF_ALIST_3)
     }
 
     switch (result) {
-    case 0:
-	/* May need to yield even though we do not bump reds here... */
-	if (ERTS_IS_PROC_OUT_OF_REDS(p))
-	    goto yield_return;
-	ERTS_BIF_PREP_RET(retval, am_ok);
-	break;
     case SEND_NOCONNECT:
 	if (ctx->connect) {
 	    ERTS_BIF_PREP_RET(retval, am_ok);
@@ -2443,8 +2432,8 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg)
 
     ERTS_MSACC_POP_STATE_M_X();
 
-    if (result > 0) {
-	ERTS_VBUMP_REDS(p, result);
+    if (result >= 0) {
+	ERTS_VBUMP_REDS(p, 4);
 	if (ERTS_IS_PROC_OUT_OF_REDS(p))
 	    goto yield_return;
 	ERTS_BIF_PREP_RET(retval, msg);
@@ -2452,12 +2441,6 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg)
     }
 
     switch (result) {
-    case 0:
-	/* May need to yield even though we do not bump reds here... */
-	if (ERTS_IS_PROC_OUT_OF_REDS(p))
-	    goto yield_return;
-	ERTS_BIF_PREP_RET(retval, msg);
-	break;
     case SEND_NOCONNECT:
 	ERTS_BIF_PREP_RET(retval, msg);
 	break;
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index f52eed41d5..4846ccd2d3 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -155,9 +155,6 @@ erts_atomic32_t erts_writing_erl_crash_dump;
 erts_tsd_key_t erts_is_crash_dumping_key;
 int erts_initialized = 0;
 
-
-int erts_use_sender_punish;
-
 /*
  * Configurable parameters.
  */
@@ -758,8 +755,6 @@ early_init(int *argc, char **argv) /*
 
     erts_initialized = 0;
 
-    erts_use_sender_punish = 1;
-
     erts_pre_early_init_cpu_topology(&max_reader_groups,
 				     &ncpu,
 				     &ncpuonln,
@@ -1765,8 +1760,6 @@ erl_start(int argc, char **argv)
 		    erts_usage();
 		}
 	    }
-	    else if (sys_strcmp("nsp", sub_param) == 0)
-		erts_use_sender_punish = 0;
 	    else if (has_prefix("tbt", sub_param)) {
 		arg = get_arg(sub_param+3, argv[i+1], &i);
 		res = erts_init_scheduler_bind_type_string(arg);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 09500c5bc0..d16ab75178 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1122,7 +1122,6 @@ extern int erts_no_line_info;
 extern Eterm erts_error_logger_warnings;
 extern int erts_initialized;
 extern int erts_compat_rel;
-extern int erts_use_sender_punish;
 void erl_start(int, char**);
 void erts_usage(void);
 Eterm erts_preloaded(Process* p);
-- 
cgit v1.2.3


From c3ddb0fc41efcada961920a26f5286f12793614e Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 27 Dec 2017 19:17:37 +0100
Subject: Fix encoding of filenames in stacktraces

---
 erts/emulator/beam/beam_load.c                 |   8 +-
 erts/emulator/beam/beam_ranges.c               |   6 +-
 erts/emulator/beam/erl_unicode.c               | 152 +++++++++++++++++++------
 erts/emulator/beam/erl_unicode.h               |   5 +-
 erts/emulator/test/code_SUITE.erl              |  50 +++++++-
 erts/emulator/test/code_SUITE_data/erl_544.erl |  35 ++++++
 6 files changed, 210 insertions(+), 46 deletions(-)
 create mode 100644 erts/emulator/test/code_SUITE_data/erl_544.erl

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index adf8779f11..494b7543ec 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -40,6 +40,7 @@
 #include "erl_zlib.h"
 #include "erl_map.h"
 #include "erl_process_dict.h"
+#include "erl_unicode.h"
 
 #ifdef HIPE
 #include "hipe_bif0.h"
@@ -1792,7 +1793,7 @@ read_line_table(LoaderState* stp)
 
 	    GetInt(stp, 2, n);
 	    GetString(stp, fname, n);
-	    stp->fname[i] = erts_atom_put(fname, n, ERTS_ATOM_ENC_LATIN1, 1);
+	    stp->fname[i] = erts_atom_put(fname, n, ERTS_ATOM_ENC_UTF8, 1);
 	}
     }
 
@@ -5891,8 +5892,7 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p)
 	    file_term = buf_to_intlist(&hp, ".erl", 4, NIL);
 	    file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, file_term);
 	} else {
-	    Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1]));
-	    file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, NIL);
+            file_term = erts_atom_to_string(&hp, (fi->fname_ptr)[file-1]);
 	}
 
 	tuple = TUPLE2(hp, am_line, make_small(line));
diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c
index fac4289271..c7b17d58f3 100644
--- a/erts/emulator/beam/beam_ranges.c
+++ b/erts/emulator/beam/beam_ranges.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,6 +26,7 @@
 #include "erl_vm.h"
 #include "global.h"
 #include "beam_load.h"
+#include "erl_unicode.h"
 
 typedef struct {
     BeamInstr* start;		/* Pointer to start of module. */
@@ -341,8 +342,7 @@ lookup_loc(FunctionInfo* fi, const BeamInstr* pc,
 		Atom* mod_atom = atom_tab(atom_val(fi->mfa->module));
 		fi->needed += 2*(mod_atom->len+4);
 	    } else {
-		Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1]));
-		fi->needed += 2*ap->len;
+                fi->needed += 2*erts_atom_to_string_length((fi->fname_ptr)[file-1]);
 	    }
 	    return;
 	} else {
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 2d1d1443a7..c5fc6f9815 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -1158,14 +1158,15 @@ static ERTS_INLINE int
 analyze_utf8(byte *source, Uint size, byte **err_pos, Uint *num_chars, int *left,
 	     Sint *num_latin1_chars, Uint max_chars)
 {
+    int res = ERTS_UTF8_OK;
     Uint latin1_count;
     int is_latin1;
+    Uint nchars = 0;
     *err_pos = source;
     if (num_latin1_chars) {
 	is_latin1 = 1;
 	latin1_count = 0;
     }
-    *num_chars = 0;
     while (size) {
 	if (((*source) & ((byte) 0x80)) == 0) {
 	    source++;
@@ -1174,11 +1175,13 @@ analyze_utf8(byte *source, Uint size, byte **err_pos, Uint *num_chars, int *left
 		latin1_count++;
 	} else if (((*source) & ((byte) 0xE0)) == 0xC0) {
 	    if (size < 2) {
-		return ERTS_UTF8_INCOMPLETE;
+		res = ERTS_UTF8_INCOMPLETE;
+                break;
 	    }
 	    if (((source[1] & ((byte) 0xC0)) != 0x80) ||
 		((*source) < 0xC2) /* overlong */) {
-		return ERTS_UTF8_ERROR;
+		res = ERTS_UTF8_ERROR;
+                break;
 	    }
 	    if (num_latin1_chars) {
 		latin1_count++;
@@ -1189,16 +1192,19 @@ analyze_utf8(byte *source, Uint size, byte **err_pos, Uint *num_chars, int *left
 	    size -= 2;
 	} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
 	    if (size < 3) {
-		return ERTS_UTF8_INCOMPLETE;
+		res = ERTS_UTF8_INCOMPLETE;
+                break;
 	    }
 	    if (((source[1] & ((byte) 0xC0)) != 0x80) ||
 		((source[2] & ((byte) 0xC0)) != 0x80) ||
 		(((*source) == 0xE0) && (source[1] < 0xA0)) /* overlong */ ) {
-		return ERTS_UTF8_ERROR;
+		res = ERTS_UTF8_ERROR;
+                break;
 	    }
 	    if ((((*source) & ((byte) 0xF)) == 0xD) && 
 		((source[1] & 0x20) != 0)) {
-		return ERTS_UTF8_ERROR;
+		res = ERTS_UTF8_ERROR;
+                break;
 	    }
 	    source += 3;
 	    size -= 3;
@@ -1206,37 +1212,47 @@ analyze_utf8(byte *source, Uint size, byte **err_pos, Uint *num_chars, int *left
 		is_latin1 = 0;
 	} else if (((*source) & ((byte) 0xF8)) == 0xF0) {
 	    if (size < 4) {
-		return ERTS_UTF8_INCOMPLETE;
+		res = ERTS_UTF8_INCOMPLETE;
+                break;
 	    }
 	    if (((source[1] & ((byte) 0xC0)) != 0x80) ||
 		((source[2] & ((byte) 0xC0)) != 0x80) ||
 		((source[3] & ((byte) 0xC0)) != 0x80) ||
 		(((*source) == 0xF0) && (source[1] < 0x90)) /* overlong */) {
-		return ERTS_UTF8_ERROR;
+		res = ERTS_UTF8_ERROR;
+                break;
 	    }
 	    if ((((*source) & ((byte)0x7)) > 0x4U) ||
 		((((*source) & ((byte)0x7)) == 0x4U) && 
 		 ((source[1] & ((byte)0x3F)) > 0xFU))) {
-		return ERTS_UTF8_ERROR;
+		res = ERTS_UTF8_ERROR;
+                break;
 	    }
 	    source += 4;
 	    size -= 4; 
 	    if (num_latin1_chars)
 		is_latin1 = 0;
 	} else {
-	    return ERTS_UTF8_ERROR;
+	    res = ERTS_UTF8_ERROR;
+            break;
 	}
-	++(*num_chars);
+	++nchars;
 	*err_pos = source;
-	if (max_chars && size > 0 && *num_chars == max_chars)
-	    return ERTS_UTF8_OK_MAX_CHARS;
+	if (max_chars && size > 0 && nchars == max_chars) {
+	    res = ERTS_UTF8_OK_MAX_CHARS;
+            break;
+        }
 	if (left && --(*left) <= 0 && size) {
-	    return ERTS_UTF8_ANALYZE_MORE;
+	    res = ERTS_UTF8_ANALYZE_MORE;
+            break;
 	}
     }
+
+    *num_chars = nchars;
     if (num_latin1_chars)
 	*num_latin1_chars = is_latin1 ? latin1_count : -1;
-    return ERTS_UTF8_OK;
+
+    return res;
 }
 
 int erts_analyze_utf8(byte *source, Uint size, 
@@ -1252,29 +1268,18 @@ int erts_analyze_utf8_x(byte *source, Uint size,
     return analyze_utf8(source, size, err_pos, num_chars, left, num_latin1_chars, max_chars);
 }
 
-/*
- * No errors should be able to occur - no overlongs, no malformed, no nothing
- */
-static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, 
-			     Uint left,
-			     Uint *num_built, Uint *num_eaten, Eterm tail)
+static ERTS_INLINE Eterm
+make_list_from_utf8_buf(Eterm **hpp, Uint num,
+                        byte *bytes, Uint sz, 
+                        Uint *num_built, Uint *num_eaten,
+                        Eterm tail)
 {
     Eterm *hp;
     Eterm ret;
+    Uint left = num;
     byte *source, *ssource;
     Uint unipoint;
-
-    ASSERT(num > 0);
-    if (left < num) {
-	if (left > 0)
-	    num = left;
-	else
-	    num = 1;
-    }
-    
-    *num_built = num; /* Always */
-
-    hp = HAlloc(p,num * 2);
+    hp = *hpp;
     ret = tail;
     source = bytes + sz;
     ssource = source;
@@ -1302,20 +1307,97 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz,
 	}
 	ret = CONS(hp,make_small(unipoint),ret);
 	hp += 2;
-	if (--num <= 0) {
+	if (--left <= 0) {
 	    break;
 	}
     }
+    *hpp = hp;
+    *num_built = num; /* Always */
     *num_eaten = (ssource - source);
     return ret;
 }
 
+/*
+ * No errors should be able to occur - no overlongs, no malformed, no nothing
+ */
+static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, 
+			     Uint left,
+			     Uint *num_built, Uint *num_eaten, Eterm tail)
+{
+    Eterm *hp;
+
+    ASSERT(num > 0);
+    if (left < num) {
+	if (left > 0)
+	    num = left;
+	else
+	    num = 1;
+    }
+    
+    hp = HAlloc(p,num * 2);
+
+    return make_list_from_utf8_buf(&hp, num, bytes, sz,
+                                   num_built, num_eaten,
+                                   tail);
+}
 Eterm erts_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, Uint left,
 			Uint *num_built, Uint *num_eaten, Eterm tail)
 {
     return do_utf8_to_list(p, num, bytes, sz, left, num_built, num_eaten, tail);
 }
 
+Uint erts_atom_to_string_length(Eterm atom)
+{
+    Atom *ap;
+
+    ASSERT(is_atom(atom));
+    ap = atom_tab(atom_val(atom));
+
+    if (ap->latin1_chars >= 0)
+        return (Uint) ap->len;
+    else {
+        byte* err_pos;
+        Uint num_chars;
+#ifdef DEBUG
+        int ares =
+#endif
+            erts_analyze_utf8(ap->name, ap->len, &err_pos, &num_chars, NULL);
+        ASSERT(ares == ERTS_UTF8_OK);
+
+        return num_chars;
+    }
+}
+
+Eterm erts_atom_to_string(Eterm **hpp, Eterm atom)
+{
+    Atom *ap;
+
+    ASSERT(is_atom(atom));
+    ap = atom_tab(atom_val(atom));
+    if (ap->latin1_chars >= 0)
+        return buf_to_intlist(hpp, (char*)ap->name, ap->len, NIL);
+    else {
+        Eterm res;
+        byte* err_pos;
+        Uint num_chars, num_built, num_eaten;
+#ifdef DEBUG
+        Eterm *hp_start = *hpp;
+        int ares =
+#endif
+            erts_analyze_utf8(ap->name, ap->len, &err_pos, &num_chars, NULL);
+        ASSERT(ares == ERTS_UTF8_OK);
+
+        res = make_list_from_utf8_buf(hpp, num_chars, ap->name, ap->len,
+                                      &num_built, &num_eaten, NIL);
+
+        ASSERT(num_built == num_chars);
+        ASSERT(num_eaten == ap->len);
+        ASSERT(*hpp - hp_start == 2*num_chars);
+
+        return res;
+    }
+}
+
 static int is_candidate(Uint cp)
 {
     int index,pos;
diff --git a/erts/emulator/beam/erl_unicode.h b/erts/emulator/beam/erl_unicode.h
index e01eaa787e..31369fc8f9 100644
--- a/erts/emulator/beam/erl_unicode.h
+++ b/erts/emulator/beam/erl_unicode.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2008-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,4 +21,7 @@
 #ifndef _ERL_UNICODE_H
 #define _ERL_UNICODE_H
 
+Uint erts_atom_to_string_length(Eterm atom);
+Eterm erts_atom_to_string(Eterm **hpp, Eterm atom);
+
 #endif /* _ERL_UNICODE_H */
diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index dca600bc7b..fcef070f08 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2017. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -27,7 +27,8 @@
          constant_pools/1,constant_refc_binaries/1,
          fake_literals/1,
          false_dependency/1,coverage/1,fun_confusion/1,
-         t_copy_literals/1, t_copy_literals_frags/1]).
+         t_copy_literals/1, t_copy_literals_frags/1,
+         erl_544/1]).
 
 -define(line_trace, 1).
 -include_lib("common_test/include/ct.hrl").
@@ -41,7 +42,8 @@ all() ->
      module_md5,
      constant_pools, constant_refc_binaries, fake_literals,
      false_dependency,
-     coverage, fun_confusion, t_copy_literals, t_copy_literals_frags].
+     coverage, fun_confusion, t_copy_literals, t_copy_literals_frags,
+     erl_544].
 
 init_per_suite(Config) ->
     erts_debug:set_internal_state(available_internal_state, true),
@@ -918,6 +920,48 @@ reloader(Mod,Code,Time) ->
               reloader(Mod,Code,Time)
     end.
 
+erl_544(Config) when is_list(Config) ->
+    case file:native_name_encoding() of
+        utf8 ->
+            {ok, CWD} = file:get_cwd(),
+            try
+                Mod = erl_544,
+                FileName = atom_to_list(Mod) ++ ".erl",
+                Priv = proplists:get_value(priv_dir, Config),
+                Data = proplists:get_value(data_dir, Config),
+                {ok, FileContent} = file:read_file(filename:join(Data,
+                                                                 FileName)),
+                Dir = filename:join(Priv, [16#2620,16#2620,16#2620]),
+                File = filename:join(Dir, FileName),
+                io:format("~ts~n", [File]),
+                ok = file:make_dir(Dir),
+                ok = file:set_cwd(Dir),
+                ok = file:write_file(File, [FileContent]),
+                {ok, Mod} = compile:file(File),
+                Res1 = (catch Mod:err()),
+                io:format("~p~n", [Res1]),
+                {'EXIT', {err, [{Mod, err, 0, Info1}|_]}} = Res1,
+                File = proplists:get_value(file, Info1),
+                Me = self(),
+                Go = make_ref(),
+                Tester = spawn_link(fun () ->
+                                            Mod:wait(Me, Go),
+                                            Mod:err()
+                                    end),
+                receive Go -> ok end,
+                Res2 = process_info(Tester, current_stacktrace),
+                io:format("~p~n", [Res2]),
+                {current_stacktrace,
+                 [{Mod, wait, 2, Info2}|_]} = Res2,
+                File = proplists:get_value(file, Info2),
+                ok
+            after
+                ok = file:set_cwd(CWD)
+            end,
+            ok;
+        _Enc ->
+            {skipped, "Only run when native file name encoding is utf8"}
+    end.
 
 %% Utilities.
 
diff --git a/erts/emulator/test/code_SUITE_data/erl_544.erl b/erts/emulator/test/code_SUITE_data/erl_544.erl
new file mode 100644
index 0000000000..c93f3ef5bc
--- /dev/null
+++ b/erts/emulator/test/code_SUITE_data/erl_544.erl
@@ -0,0 +1,35 @@
+%%
+%% %CopyrightBegin%
+%% 
+%% Copyright Ericsson AB 2018. All Rights Reserved.
+%% 
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%% 
+%% %CopyrightEnd%
+%%
+
+-module(erl_544).
+
+-export([err/0, wait/2]).
+
+err() ->
+    erlang:error(err).
+
+wait(Pid, Msg) ->
+    erlang:yield(),
+    Pid ! Msg,
+    receive
+    after infinity ->
+            ok
+    end,
+    err().
-- 
cgit v1.2.3


From ff65e1164cc16739c51c456b4e350e966cc2b1ef Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Fri, 12 Jan 2018 16:43:55 +0100
Subject: stdlib: Handle Unicode when formatting stacktraces

See also ERL-553 and ERL-544 (commit c3ddb0f).
---
 erts/emulator/test/code_SUITE.erl | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index fcef070f08..661a2ee6c9 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -951,9 +951,14 @@ erl_544(Config) when is_list(Config) ->
                 receive Go -> ok end,
                 Res2 = process_info(Tester, current_stacktrace),
                 io:format("~p~n", [Res2]),
-                {current_stacktrace,
-                 [{Mod, wait, 2, Info2}|_]} = Res2,
+                {current_stacktrace, Stack} = Res2,
+                [{Mod, wait, 2, Info2}|_] = Stack,
                 File = proplists:get_value(file, Info2),
+                StackFun = fun(_, _, _) -> false end,
+                FormatFun = fun (Term, _) -> io_lib:format("~tp", [Term]) end,
+                Formated =
+                    lib:format_stacktrace(1, Stack, StackFun, FormatFun),
+                true = is_list(Formated),
                 ok
             after
                 ok = file:set_cwd(CWD)
-- 
cgit v1.2.3


From c81c01da7b4870aa56cc38d865cd081f9bb800bc Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 15 Jan 2018 19:13:15 +0100
Subject: erts: Fix float_to_list(F, [{decimals,D}]).

Example symptom:
1> float_to_list(0.145, [{decimals,1}]).
"0.2"

There were two problems in sys_double_to_chars_fast

1. Most serious was adding 0.55555555 / (10^D) instead of 0.5 / (10^D)
   which imposed a 5.5% risk of a faulty rounding up.

2. Using fixpoint for frac_part which lost significant bits if F < 0.5
---
 erts/emulator/sys/common/erl_sys_common_misc.c | 28 +++++++---
 erts/emulator/test/num_bif_SUITE.erl           | 71 +++++++++++++++++++++++---
 2 files changed, 84 insertions(+), 15 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c
index 79f87eb3a9..ddb414580a 100644
--- a/erts/emulator/sys/common/erl_sys_common_misc.c
+++ b/erts/emulator/sys/common/erl_sys_common_misc.c
@@ -160,7 +160,7 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
 {
     /* Note that some C compilers don't support "static const" propagation
      * so we use a defines */
-    #define SYS_DOUBLE_RND_CONST 0.55555555555555555
+    #define SYS_DOUBLE_RND_CONST 0.5
     #define FRAC_SIZE            52
     #define EXP_SIZE             11
     #define EXP_MASK             ((1ll << EXP_SIZE) - 1)
@@ -256,11 +256,14 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
         return p - buffer;
     } else if (exp >= FRAC_SIZE) {
         int_part  = mantissa << (exp - FRAC_SIZE);
+        exp = 0;
     } else if (exp >= 0) {
         int_part  = mantissa >> (FRAC_SIZE - exp);
         frac_part = (mantissa << (exp + 1)) & FRAC_MASK2;
+        exp = 0;
     } else /* if (exp < 0) */ {
-        frac_part = (mantissa & FRAC_MASK2) >> -(exp + 1);
+        frac_part = mantissa;
+        exp = -(exp+1);
     }
 
     if (!int_part) {
@@ -298,11 +301,22 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
         max = decimals;
 
         for (i = 0; i < max; i++) {
-            /* frac_part *= 10; */
-            frac_part = (frac_part << 3) + (frac_part << 1);
-
-            *p++ = (char)((frac_part >> (FRAC_SIZE + 1)) + '0');
-            frac_part &= FRAC_MASK2;
+            if (frac_part > (ERTS_UINT64_MAX/16)) {
+                frac_part /= 16;
+                exp -= 4;
+            }
+            if (FRAC_SIZE + 1 + exp >= 64) {
+                *p++ = '0';
+                frac_part *= 5;
+                exp--;
+            }
+            else
+            {
+                frac_part *= 10;
+
+                *p++ = (char)((frac_part >> (FRAC_SIZE + 1 + exp)) + '0');
+                frac_part &= (1ll << (FRAC_SIZE + 1 + exp)) - 1;
+            }
         }
 
         /* Delete trailing zeroes */
diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl
index 1c76eb8019..b5ff940e00 100644
--- a/erts/emulator/test/num_bif_SUITE.erl
+++ b/erts/emulator/test/num_bif_SUITE.erl
@@ -118,6 +118,7 @@ t_float(Config) when is_list(Config) ->
 %% Tests float_to_list/1, float_to_list/2, float_to_binary/1, float_to_binary/2
 
 t_float_to_string(Config) when is_list(Config) ->
+    rand_seed(),
     test_fts("0.00000000000000000000e+00", 0.0),
     test_fts("2.50000000000000000000e+01", 25.0),
     test_fts("2.50000000000000000000e+00", 2.5),
@@ -167,8 +168,8 @@ t_float_to_string(Config) when is_list(Config) ->
     test_fts("1.12300",1.123, [{decimals, 5}]),
     test_fts("1.123",1.123, [{decimals, 5}, compact]),
     test_fts("1.1234",1.1234,[{decimals, 6}, compact]),
-    test_fts("1.01",1.005, [{decimals, 2}]),
-    test_fts("-1.01",-1.005,[{decimals, 2}]),
+    test_fts("1.00",1.005, [{decimals, 2}]),  %% 1.005 is really 1.0049999999...
+    test_fts("-1.00",-1.005,[{decimals, 2}]),
     test_fts("0.999",0.999, [{decimals, 3}]),
     test_fts("-0.999",-0.999,[{decimals, 3}]),
     test_fts("1.0",0.999, [{decimals, 2}, compact]),
@@ -184,6 +185,9 @@ t_float_to_string(Config) when is_list(Config) ->
     test_fts("123000000000000000000.0",1.23e20, [{decimals,   10}, compact]),
     test_fts("1.2300000000e+20",1.23e20, [{scientific, 10}, compact]),
     test_fts("1.23000000000000000000e+20",1.23e20, []),
+
+    fts_rand_float_decimals(1000),
+
     ok.
 
 test_fts(Expect, Float) ->
@@ -197,6 +201,49 @@ test_fts(Expect, Float, Args) ->
     BinExpect = float_to_binary(Float,Args).
 
 
+rand_float_reasonable() ->
+    F = rand_float(),
+    case abs(F) > 1.0e238 of
+        true -> rand_float_reasonable();
+        false -> F
+    end.
+
+fts_rand_float_decimals(0) -> ok;
+fts_rand_float_decimals(N) ->
+    [begin
+         F0 = rand_float_reasonable(),
+         L0 = float_to_list(F0, [{decimals, D}]),
+         L1 = case D of
+                  0 -> L0 ++ ".0";
+                  _ -> L0
+              end,
+         F1 = list_to_float(L1),
+         Diff = abs(F0-F1),
+         MaxDiff = max_diff_decimals(F0, D),
+         ok = case Diff =< MaxDiff of
+                  true -> ok;
+                  false ->
+                      io:format("F0 = ~w ~w\n",  [F0, <<F0/float>>]),
+                      io:format("L1 = ~s\n",  [L1]),
+                      io:format("F1 = ~w ~w\n",  [F1, <<F1/float>>]),
+                      io:format("Diff = ~w, MaxDiff = ~w\n", [Diff, MaxDiff]),
+                      error
+              end
+     end
+     || D <- lists:seq(0,15)],
+
+    fts_rand_float_decimals(N-1).
+
+max_diff_decimals(F, D) ->
+    IntBits = floor(math:log2(abs(F))) + 1,
+    FracBits = (52 - IntBits),
+    Log10_2 = 0.3010299956639812,  % math:log10(2)
+    MaxDec = floor(FracBits * Log10_2),
+
+    Resolution = math:pow(2, IntBits - 53),
+
+    (math:pow(10, -min(D,MaxDec)) / 2) + Resolution.
+
 %% Tests list_to_float/1.
 
 t_string_to_float_safe(Config) when is_list(Config) ->
@@ -331,18 +378,26 @@ t_trunc_and_friends(_Config) ->
     -18446744073709551616 = trunc_and_friends(-float(1 bsl 64)),
 
     %% Random.
+    rand_seed(),
     t_trunc_and_friends_rand(100),
     ok.
 
+rand_seed() ->
+    rand:seed(exrop),
+    io:format("\n*** rand:export_seed() = ~w\n\n", [rand:export_seed()]),
+    ok.
+
+rand_float() ->
+    F0 = rand:uniform() * math:pow(10, 50*rand:normal()),
+    case rand:uniform() of
+        U when U < 0.5 -> -F0;
+        _ -> F0
+    end.
+
 t_trunc_and_friends_rand(0) ->
     ok;
 t_trunc_and_friends_rand(N) ->
-    F0 = rand:uniform() * math:pow(10, 50*rand:normal()),
-    F = case rand:uniform() of
-	    U when U < 0.5 -> -F0;
-	    _ -> F0
-	end,
-    _ = trunc_and_friends(F),
+    _ = trunc_and_friends(rand_float()),
     t_trunc_and_friends_rand(N-1).
 
 trunc_and_friends(F) ->
-- 
cgit v1.2.3


From b76d188778655394641f87ac0367fefecf233268 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 15 Jan 2018 19:13:25 +0100
Subject: erts: Refactor and cleanup sys_double_to_chars_fast

Replace long long with Uint64
---
 erts/emulator/sys/common/erl_sys_common_misc.c | 106 ++++++++++++-------------
 1 file changed, 49 insertions(+), 57 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c
index ddb414580a..ba03a11405 100644
--- a/erts/emulator/sys/common/erl_sys_common_misc.c
+++ b/erts/emulator/sys/common/erl_sys_common_misc.c
@@ -158,59 +158,50 @@ int
 sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
 			 int compact)
 {
-    /* Note that some C compilers don't support "static const" propagation
-     * so we use a defines */
     #define SYS_DOUBLE_RND_CONST 0.5
     #define FRAC_SIZE            52
     #define EXP_SIZE             11
-    #define EXP_MASK             ((1ll << EXP_SIZE) - 1)
+    #define EXP_MASK             (((Uint64)1 << EXP_SIZE) - 1)
     #define MAX_DECIMALS         (sizeof(cs_sys_double_pow10) \
 				   / sizeof(cs_sys_double_pow10[0]))
-    #define FRAC_MASK            ((1ll << FRAC_SIZE) - 1)
-    #define FRAC_MASK2           ((1ll << (FRAC_SIZE + 1)) - 1)
-    #define MAX_FLOAT            (1ll << (FRAC_SIZE+1))
+    #define FRAC_MASK            (((Uint64)1 << FRAC_SIZE) - 1)
+    #define FRAC_MASK2           (((Uint64)1 << (FRAC_SIZE + 1)) - 1)
+    #define MAX_FLOAT            ((Uint64)1 << (FRAC_SIZE+1))
 
     static const double cs_sys_double_pow10[] = {
-        SYS_DOUBLE_RND_CONST / 1ll,
-        SYS_DOUBLE_RND_CONST / 10ll,
-        SYS_DOUBLE_RND_CONST / 100ll,
-        SYS_DOUBLE_RND_CONST / 1000ll,
-        SYS_DOUBLE_RND_CONST / 10000ll,
-        SYS_DOUBLE_RND_CONST / 100000ll,
-        SYS_DOUBLE_RND_CONST / 1000000ll,
-        SYS_DOUBLE_RND_CONST / 10000000ll,
-        SYS_DOUBLE_RND_CONST / 100000000ll,
-        SYS_DOUBLE_RND_CONST / 1000000000ll,
-        SYS_DOUBLE_RND_CONST / 10000000000ll,
-        SYS_DOUBLE_RND_CONST / 100000000000ll,
-        SYS_DOUBLE_RND_CONST / 1000000000000ll,
-        SYS_DOUBLE_RND_CONST / 10000000000000ll,
-        SYS_DOUBLE_RND_CONST / 100000000000000ll,
-        SYS_DOUBLE_RND_CONST / 1000000000000000ll,
-        SYS_DOUBLE_RND_CONST / 10000000000000000ll,
-        SYS_DOUBLE_RND_CONST / 100000000000000000ll,
-        SYS_DOUBLE_RND_CONST / 1000000000000000000ll
+        SYS_DOUBLE_RND_CONST / 1e0,
+        SYS_DOUBLE_RND_CONST / 1e1,
+        SYS_DOUBLE_RND_CONST / 1e2,
+        SYS_DOUBLE_RND_CONST / 1e3,
+        SYS_DOUBLE_RND_CONST / 1e4,
+        SYS_DOUBLE_RND_CONST / 1e5,
+        SYS_DOUBLE_RND_CONST / 1e6,
+        SYS_DOUBLE_RND_CONST / 1e7,
+        SYS_DOUBLE_RND_CONST / 1e8,
+        SYS_DOUBLE_RND_CONST / 1e9,
+        SYS_DOUBLE_RND_CONST / 1e10,
+        SYS_DOUBLE_RND_CONST / 1e11,
+        SYS_DOUBLE_RND_CONST / 1e12,
+        SYS_DOUBLE_RND_CONST / 1e13,
+        SYS_DOUBLE_RND_CONST / 1e14,
+        SYS_DOUBLE_RND_CONST / 1e15,
+        SYS_DOUBLE_RND_CONST / 1e16,
+        SYS_DOUBLE_RND_CONST / 1e17,
+        SYS_DOUBLE_RND_CONST / 1e18
     };
 
-    long long mantissa, int_part = 0, frac_part = 0;
-    short exp;
+    Uint64 mantissa, int_part, frac_part;
+    int exp;
+    int fbits;
     int max;
     int neg;
     double fr;
-    union { long long L; double F; } x;
+    union { Uint64 L; double F; } x;
     char *p = buffer;
 
     if (decimals < 0)
         return -1;
 
-    /* Round the number to given decimal places. The number of 5's in the
-     * SYS_DOUBLE_RND_CONST constant is chosen such that adding any more 5's doesn't
-     * change the double precision of the number, i.e.:
-     * 1> term_to_binary(0.55555555555555555, [{minor_version, 1}]).
-     * <<131,70,63,225,199,28,113,199,28,114>>
-     * 2> term_to_binary(0.5555555555555555555, [{minor_version, 1}]).
-     * <<131,70,63,225,199,28,113,199,28,114>>
-     */
     if (f >= 0) {
         neg = 0;
         fr  = decimals < MAX_DECIMALS ? (f + cs_sys_double_pow10[decimals]) : f;
@@ -241,7 +232,7 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
     }
 
     exp      -= EXP_MASK >> 1;
-    mantissa |= (1ll << FRAC_SIZE);
+    mantissa |= ((Uint64)1 << FRAC_SIZE);
 
     /* Don't bother with optimizing too large numbers or too large precision */
     if (x.F > MAX_FLOAT || decimals >= MAX_DECIMALS) {
@@ -256,14 +247,16 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
         return p - buffer;
     } else if (exp >= FRAC_SIZE) {
         int_part  = mantissa << (exp - FRAC_SIZE);
-        exp = 0;
+        frac_part = 0;
+        fbits = FRAC_SIZE;  /* not important as frac_part==0 */
     } else if (exp >= 0) {
-        int_part  = mantissa >> (FRAC_SIZE - exp);
-        frac_part = (mantissa << (exp + 1)) & FRAC_MASK2;
-        exp = 0;
+        fbits = FRAC_SIZE - exp;
+        int_part  = mantissa >> fbits;
+        frac_part = mantissa & (((Uint64)1 << fbits) -1);
     } else /* if (exp < 0) */ {
+        int_part = 0;
         frac_part = mantissa;
-        exp = -(exp+1);
+        fbits = FRAC_SIZE - exp;
     }
 
     if (!int_part) {
@@ -273,9 +266,8 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
     } else {
         int ret, i, n;
         while (int_part != 0) {
-            long long j = int_part / 10;
-            *p++ = (char)(int_part - ((j << 3) + (j << 1)) + '0');
-            int_part = j;
+            *p++ = (char)((int_part % 10) + '0');
+            int_part /= 10;
         }
         if (neg)
             *p++ = '-';
@@ -301,21 +293,21 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
         max = decimals;
 
         for (i = 0; i < max; i++) {
-            if (frac_part > (ERTS_UINT64_MAX/16)) {
-                frac_part /= 16;
-                exp -= 4;
+            if (frac_part > (ERTS_UINT64_MAX/5)) {
+                frac_part >>= 3;
+                fbits -= 3;
             }
-            if (FRAC_SIZE + 1 + exp >= 64) {
+
+            /* Multiply by 10 (5*2) to extract decimal digit as integer part */
+            frac_part *= 5;
+            fbits--;
+
+            if (fbits >= 64) {
                 *p++ = '0';
-                frac_part *= 5;
-                exp--;
             }
-            else
-            {
-                frac_part *= 10;
-
-                *p++ = (char)((frac_part >> (FRAC_SIZE + 1 + exp)) + '0');
-                frac_part &= (1ll << (FRAC_SIZE + 1 + exp)) - 1;
+            else {
+                *p++ = (char)((frac_part >> fbits) + '0');
+                frac_part &= ((Uint64)1 << fbits) - 1;
             }
         }
 
-- 
cgit v1.2.3


From d57d91427e1ddb09cbf55cac2014a368fe717b46 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 17 Jan 2018 20:00:49 +0100
Subject: Fix slow hipe execution

particularly slow erlc when compiler is hipe compiled.

hipe_unified_loader:load did not patch external call sites
and instead caused a double hipe mode switch per call.

hipe_unified_loader:load is only used
for early modules first loaded as beam
and by code:atomic_load and friends.
---
 erts/emulator/beam/beam_load.c | 2 ++
 erts/emulator/hipe/hipe_bif0.c | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index adf8779f11..5b1aa6f128 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -6621,6 +6621,8 @@ int erts_commit_hipe_patch_load(Eterm hipe_magic_bin)
     hipe_stp->new_hipe_refs = NULL;
     hipe_stp->new_hipe_sdesc = NULL;
 
+    hipe_redirect_to_module(modp);
+
     return 1;
 }
 
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 94bc563fda..d8427a4c36 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1773,7 +1773,8 @@ void hipe_redirect_to_module(Module* modp)
     struct hipe_mfa_info *p;
     struct hipe_ref_head* refh;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking() ||
+                       erts_is_multi_scheduling_blocked());
 
     for (p = mod2mfa_get(modp); p; p = p->next_in_mod) {
         if (p->new_address) {
-- 
cgit v1.2.3


From 1aa1833be4e9ee0d48351564f4bf9d1f77c1e634 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 18 Jan 2018 12:58:23 +0100
Subject: beam_debug: Print the MFA in the i_make_fun/2 instruction

---
 erts/emulator/beam/beam_debug.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 509aa2a84f..5aff1c3cb8 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -570,6 +570,16 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 		    }
 		    break;
 		}
+	    case op_i_make_fun_Wt:
+                if (*sign == 'W') {
+                    ErlFunEntry* fe = (ErlFunEntry *) *ap;
+                    ErtsCodeMFA* cmfa = find_function_from_pc(fe->address);
+		    erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module,
+                               cmfa->function, cmfa->arity);
+                } else {
+                    erts_print(to, to_arg, "%d", *ap);
+                }
+                break;
 	    default:
 		erts_print(to, to_arg, "%d", *ap);
 	    }
-- 
cgit v1.2.3


From 1f802279f79ff3a7f1ded5604d6433a6cd448448 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 18 Jan 2018 13:33:06 +0100
Subject: beam_debug: Print out strings for bs_match_string/bs_put_string

---
 erts/emulator/beam/beam_debug.c | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 5aff1c3cb8..188cbbe915 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -55,6 +55,7 @@ static int print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif);
 static BeamInstr* f_to_addr(BeamInstr* base, int op, BeamInstr* ap);
 static BeamInstr* f_to_addr_packed(BeamInstr* base, int op, Sint32* ap);
+static void print_byte_string(fmtfn_t to, void *to_arg, byte* str, Uint bytes);
 
 BIF_RETTYPE
 erts_debug_same_2(BIF_ALIST_2)
@@ -396,6 +397,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
     BeamInstr args[8];		/* Arguments for this instruction. */
     BeamInstr* ap;			/* Pointer to arguments. */
     BeamInstr* unpacked;		/* Unpacked arguments */
+    BeamInstr* first_arg;               /* First argument */
 
     start_prog = opc[op].pack;
 
@@ -480,6 +482,8 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	ap = args;
     }
 
+    first_arg = ap;
+
     /*
      * Print the name and all operands of the instructions.
      */
@@ -580,6 +584,25 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
                     erts_print(to, to_arg, "%d", *ap);
                 }
                 break;
+	    case op_i_bs_match_string_xfWW:
+                if (ap - first_arg < 3) {
+                    erts_print(to, to_arg, "%d", *ap);
+                } else {
+                    Uint bits = ap[-1];
+                    Uint bytes = (bits+7)/8;
+                    byte* str = (byte *) *ap;
+                    print_byte_string(to, to_arg, str, bytes);
+                }
+                break;
+	    case op_bs_put_string_WW:
+                if (ap - first_arg == 0) {
+                    erts_print(to, to_arg, "%d", *ap);
+                } else {
+                    Uint bytes = ap[-1];
+                    byte* str = (byte *) ap[0];
+                    print_byte_string(to, to_arg, str, bytes);
+                }
+                break;
 	    default:
 		erts_print(to, to_arg, "%d", *ap);
 	    }
@@ -858,6 +881,14 @@ static BeamInstr* f_to_addr_packed(BeamInstr* base, int op, Sint32* ap)
     return base - 1 + opc[op].adjust + *ap;
 }
 
+static void print_byte_string(fmtfn_t to, void *to_arg, byte* str, Uint bytes)
+{
+    Uint i;
+
+    for (i = 0; i < bytes; i++) {
+        erts_print(to, to_arg, "%02X", str[i]);
+    }
+}
 
 /*
  * Dirty BIF testing.
-- 
cgit v1.2.3


From 825edd6dbaf1987cee2e6571765376ff3e8c0180 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 19 Jan 2018 06:06:48 +0100
Subject: beam_debug: Fix printing of f operand for catch_yf

---
 erts/emulator/beam/beam_debug.c | 32 ++++++++++++++++++++------------
 1 file changed, 20 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 188cbbe915..8f02d509a9 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -40,6 +40,7 @@
 #include "erl_binary.h"
 #include "erl_thr_progress.h"
 #include "erl_nfunc_sched.h"
+#include "beam_catches.h"
 
 #ifdef ARCH_64
 # define HEXF "%016bpX"
@@ -609,18 +610,25 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	    ap++;
 	    break;
 	case 'f':		/* Destination label */
-	    {
-                BeamInstr* target = f_to_addr(addr, op, ap);
-		ErtsCodeMFA* cmfa = find_function_from_pc(target);
-		if (!cmfa || erts_codemfa_to_code(cmfa) != target) {
-		    erts_print(to, to_arg, "f(" HEXF ")", target);
-		} else {
-		    erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module,
-                               cmfa->function, cmfa->arity);
-		}
-		ap++;
-	    }
-	    break;
+            switch (op) {
+            case op_catch_yf:
+                erts_print(to, to_arg, "f(" HEXF ")", catch_pc((BeamInstr)*ap));
+                break;
+            default:
+                {
+                    BeamInstr* target = f_to_addr(addr, op, ap);
+                    ErtsCodeMFA* cmfa = find_function_from_pc(target);
+                    if (!cmfa || erts_codemfa_to_code(cmfa) != target) {
+                        erts_print(to, to_arg, "f(" HEXF ")", target);
+                    } else {
+                        erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module,
+                                   cmfa->function, cmfa->arity);
+                    }
+                    ap++;
+                }
+                break;
+            }
+            break;
 	case 'p':		/* Pointer (to label) */
 	    {
                 BeamInstr* target = f_to_addr(addr, op, ap);
-- 
cgit v1.2.3


From c2a9915a451a3c6ad618f55a2871a43403333b7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 17 Jan 2018 16:01:28 +0100
Subject: Optimize matching of an 'utf8' segment in the binary syntax

Matching out an 8-bit integer is faster than matching out
an utf8-encoded code point, even if the value of the code
point is less than 128. The reason is that matching out
an 8-bit integer is specially optimized to avoid a function
call. Do a similar optimization for matching out an utf8
segment.
---
 erts/emulator/beam/bs_instrs.tab | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab
index 9f03b19731..b11903a47b 100644
--- a/erts/emulator/beam/bs_instrs.tab
+++ b/erts/emulator/beam/bs_instrs.tab
@@ -919,9 +919,23 @@ i_bs_get_integer(Fail, Live, FlagsAndUnit, Ms, Sz, Dst) {
 }
 
 i_bs_get_utf8(Ctx, Fail, Dst) {
+    Eterm result;
     ErlBinMatchBuffer* mb = ms_matchbuffer($Ctx);
-    Eterm result = erts_bs_get_utf8(mb);
 
+    if (mb->size - mb->offset < 8) {
+        $FAIL($Fail);
+    }
+    if (BIT_OFFSET(mb->offset) != 0) {
+        result = erts_bs_get_utf8(mb);
+    } else {
+        byte b = mb->base[BYTE_OFFSET(mb->offset)];
+        if (b < 128) {
+            result = make_small(b);
+            mb->offset += 8;
+        } else {
+            result = erts_bs_get_utf8(mb);
+        }
+    }
     if (is_non_value(result)) {
         $FAIL($Fail);
     }
-- 
cgit v1.2.3


From b20447803c80740e685feb1e266f8afce4c26cf8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 16 Jan 2018 20:45:58 +0100
Subject: erts: Optimize move_cons & move_boxed

Replace double pointer with return that can mostly be ignored.

Use restrict pointers.
---
 erts/emulator/beam/copy.c    |  4 ++--
 erts/emulator/beam/erl_gc.c  | 49 ++++++++++++++++++++++----------------------
 erts/emulator/beam/erl_gc.h  | 25 +++++++++++-----------
 erts/emulator/hipe/hipe_gc.c | 16 +++++++--------
 4 files changed, 47 insertions(+), 47 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 10bf197405..8a82154a08 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -1985,7 +1985,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite
 	if (is_header(val)) {
 	    struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp;
 	    ASSERT(ptr + header_arity(val) < end);
-	    move_boxed(&ptr, val, &hp, &dummy_ref);
+	    ptr = move_boxed(ptr, val, &hp, &dummy_ref);
 	    switch (val & _HEADER_SUBTAG_MASK) {
 	    case REF_SUBTAG:
 		if (is_ordinary_ref_thing(hdr))
@@ -2002,7 +2002,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite
 	}
 	else { /* must be a cons cell */
 	    ASSERT(ptr+1 < end);
-	    move_cons(&ptr, val, &hp, &dummy_ref);
+	    move_cons(ptr, val, &hp, &dummy_ref);
 	    ptr += 2;
 	}
     }
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 0da4468f9c..238ac4ad68 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1172,7 +1172,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
 		    ASSERT(is_boxed(val));
                     *g_ptr++ = val;
 		} else if (ErtsInArea(ptr, area, area_size)) {
-                    move_boxed(&ptr,val,&old_htop,g_ptr++);
+                    move_boxed(ptr,val,&old_htop,g_ptr++);
 		} else {
 		    g_ptr++;
 		}
@@ -1183,7 +1183,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
                 if (IS_MOVED_CONS(val)) { /* Moved */
                     *g_ptr++ = ptr[1];
 		} else if (ErtsInArea(ptr, area, area_size)) {
-                    move_cons(&ptr,val,&old_htop,g_ptr++);
+                    move_cons(ptr,val,&old_htop,g_ptr++);
                 } else {
 		    g_ptr++;
 		}
@@ -1509,9 +1509,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
 		    ASSERT(is_boxed(val));
                     *g_ptr++ = val;
                 } else if (ErtsInArea(ptr, mature, mature_size)) {
-                    move_boxed(&ptr,val,&old_htop,g_ptr++);
+                    move_boxed(ptr,val,&old_htop,g_ptr++);
                 } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
-                    move_boxed(&ptr,val,&n_htop,g_ptr++);
+                    move_boxed(ptr,val,&n_htop,g_ptr++);
                 } else {
 		    g_ptr++;
 		}
@@ -1524,9 +1524,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
                 if (IS_MOVED_CONS(val)) { /* Moved */
                     *g_ptr++ = ptr[1];
                 } else if (ErtsInArea(ptr, mature, mature_size)) {
-                    move_cons(&ptr,val,&old_htop,g_ptr++);
+                    move_cons(ptr,val,&old_htop,g_ptr++);
                 } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
-                    move_cons(&ptr,val,&n_htop,g_ptr++);
+                    move_cons(ptr,val,&n_htop,g_ptr++);
                 } else {
 		    g_ptr++;
 		}
@@ -1568,9 +1568,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
 		    ASSERT(is_boxed(val));
 		    *n_hp++ = val;
 		} else if (ErtsInArea(ptr, mature, mature_size)) {
-		    move_boxed(&ptr,val,&old_htop,n_hp++);
+		    move_boxed(ptr,val,&old_htop,n_hp++);
 		} else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
-		    move_boxed(&ptr,val,&n_htop,n_hp++);
+		    move_boxed(ptr,val,&n_htop,n_hp++);
 		} else {
 		    n_hp++;
 		}
@@ -1582,9 +1582,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
 		if (IS_MOVED_CONS(val)) {
 		    *n_hp++ = ptr[1];
 		} else if (ErtsInArea(ptr, mature, mature_size)) {
-		    move_cons(&ptr,val,&old_htop,n_hp++);
+		    move_cons(ptr,val,&old_htop,n_hp++);
 		} else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
-		    move_cons(&ptr,val,&n_htop,n_hp++);
+		    move_cons(ptr,val,&n_htop,n_hp++);
 		} else {
 		    n_hp++;
 		}
@@ -1604,10 +1604,10 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
 			    *origptr = val;
 			    mb->base = binary_bytes(val);
 			} else if (ErtsInArea(ptr, mature, mature_size)) {
-			    move_boxed(&ptr,val,&old_htop,origptr);
+			    move_boxed(ptr,val,&old_htop,origptr);
 			    mb->base = binary_bytes(mb->orig);
 			} else if (ErtsInYoungGen(*origptr, ptr, oh, oh_size)) {
-			    move_boxed(&ptr,val,&n_htop,origptr);
+			    move_boxed(ptr,val,&n_htop,origptr);
 			    mb->base = binary_bytes(mb->orig);
 			}
 		    }
@@ -1832,7 +1832,7 @@ full_sweep_heaps(Process *p,
 		    ASSERT(is_boxed(val));
 		    *g_ptr++ = val;
 		} else if (!erts_is_literal(gval, ptr)) {
-		    move_boxed(&ptr,val,&n_htop,g_ptr++);
+		    move_boxed(ptr,val,&n_htop,g_ptr++);
 		} else {
 		    g_ptr++;
 		}
@@ -1845,7 +1845,7 @@ full_sweep_heaps(Process *p,
 		if (IS_MOVED_CONS(val)) {
 		    *g_ptr++ = ptr[1];
 		} else if (!erts_is_literal(gval, ptr)) {
-		    move_cons(&ptr,val,&n_htop,g_ptr++);
+		    move_cons(ptr,val,&n_htop,g_ptr++);
 		} else {
 		    g_ptr++;
 		}
@@ -2134,7 +2134,7 @@ sweep(Eterm *n_hp, Eterm *n_htop,
 		ASSERT(is_boxed(val));
 		*n_hp++ = val;
 	    } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) {
-		move_boxed(&ptr,val,&n_htop,n_hp++);
+		move_boxed(ptr,val,&n_htop,n_hp++);
 	    } else {
 		n_hp++;
 	    }
@@ -2146,7 +2146,7 @@ sweep(Eterm *n_hp, Eterm *n_htop,
 	    if (IS_MOVED_CONS(val)) {
 		*n_hp++ = ptr[1];
 	    } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) {
-		move_cons(&ptr,val,&n_htop,n_hp++);
+		move_cons(ptr,val,&n_htop,n_hp++);
 	    } else {
 		n_hp++;
 	    }
@@ -2167,7 +2167,7 @@ sweep(Eterm *n_hp, Eterm *n_htop,
 			*origptr = val;
 			mb->base = binary_bytes(*origptr);
 		    } else if (ERTS_IS_IN_SWEEP_AREA(*origptr, ptr)) {
-			move_boxed(&ptr,val,&n_htop,origptr);
+			move_boxed(ptr,val,&n_htop,origptr);
 			mb->base = binary_bytes(*origptr);
 		    }
 		}
@@ -2230,7 +2230,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
 		ASSERT(is_boxed(val));
 		*heap_ptr++ = val;
 	    } else if (ErtsInArea(ptr, src, src_size)) {
-		move_boxed(&ptr,val,&htop,heap_ptr++);
+		move_boxed(ptr,val,&htop,heap_ptr++);
 	    } else {
 		heap_ptr++;
 	    }
@@ -2242,7 +2242,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
 	    if (IS_MOVED_CONS(val)) {
 		*heap_ptr++ = ptr[1];
 	    } else if (ErtsInArea(ptr, src, src_size)) {
-		move_cons(&ptr,val,&htop,heap_ptr++);
+		move_cons(ptr,val,&htop,heap_ptr++);
 	    } else {
 		heap_ptr++;
 	    }
@@ -2263,7 +2263,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
 			*origptr = val;
 			mb->base = binary_bytes(*origptr);
 		    } else if (ErtsInArea(ptr, src, src_size)) {
-			move_boxed(&ptr,val,&htop,origptr);
+			move_boxed(ptr,val,&htop,origptr);
 			mb->base = binary_bytes(*origptr);
 		    }
 		}
@@ -2296,11 +2296,11 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size)
 	ASSERT(val != ERTS_HOLE_MARKER);
 	if (is_header(val)) {
 	    ASSERT(ptr + header_arity(val) < end);
-	    move_boxed(&ptr, val, &n_htop, &dummy_ref);
+	    ptr = move_boxed(ptr, val, &n_htop, &dummy_ref);
 	}
 	else { /* must be a cons cell */
 	    ASSERT(ptr+1 < end);
-	    move_cons(&ptr, val, &n_htop, &dummy_ref);
+	    move_cons(ptr, val, &n_htop, &dummy_ref);
 	    ptr += 2;
 	}
     }
@@ -3281,8 +3281,7 @@ reply_gc_info(void *vgcirp)
 	gcireq_free(vgcirp);
 }
 
-void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig) {
-    Eterm *ptr = *pp;
+Eterm* erts_sub_binary_to_heap_binary(Eterm *ptr, Eterm **hpp, Eterm *orig) {
     Eterm *htop = *hpp;
     Eterm gval;
     ErlSubBin *sb = (ErlSubBin *)ptr;
@@ -3310,7 +3309,7 @@ void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig) {
     htop += heap_bin_size(sb->size);
 
     *hpp = htop;
-    *pp  = ptr;
+    return ptr;
 }
 
 
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 6a529b8443..dec0ab1143 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -33,14 +33,15 @@
 
 #define IS_MOVED_BOXED(x)	(!is_header((x)))
 #define IS_MOVED_CONS(x)	(is_non_value((x)))
-void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig);
+Eterm* erts_sub_binary_to_heap_binary(Eterm *ptr, Eterm **hpp, Eterm *orig);
 
-ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig);
+ERTS_GLB_INLINE void move_cons(Eterm *ERTS_RESTRICT ptr, Eterm car, Eterm **hpp,
+                               Eterm *orig);
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
-ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig)
+ERTS_GLB_INLINE void move_cons(Eterm *ERTS_RESTRICT ptr, Eterm car, Eterm **hpp,
+                               Eterm *orig)
 {
-    Eterm *ptr  = *pp;
-    Eterm *htop = *hpp;
+    Eterm *ERTS_RESTRICT htop = *hpp;
     Eterm gval;
 
     htop[0] = car;               /* copy car */
@@ -53,14 +54,15 @@ ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig)
 }
 #endif
 
-ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig);
+ERTS_GLB_INLINE Eterm* move_boxed(Eterm *ERTS_RESTRICT ptr, Eterm hdr, Eterm **hpp,
+                                  Eterm *orig);
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
-ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig)
+ERTS_GLB_INLINE Eterm* move_boxed(Eterm *ERTS_RESTRICT ptr, Eterm hdr, Eterm **hpp,
+                                  Eterm *orig)
 {
     Eterm gval;
     Sint nelts;
-    Eterm *ptr = *pp;
-    Eterm *htop = *hpp;
+    Eterm *ERTS_RESTRICT htop = *hpp;
 
     ASSERT(is_header(hdr));
     nelts = header_arity(hdr);
@@ -71,8 +73,7 @@ ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig)
             /* convert sub-binary to heap-binary if applicable */
             if (sb->bitsize == 0 && sb->bitoffs == 0 &&
                 sb->is_writable == 0 && sb->size <= sizeof(Eterm) * 3) {
-                erts_sub_binary_to_heap_binary(pp, hpp, orig);
-                return;
+                return erts_sub_binary_to_heap_binary(ptr, hpp, orig);
             }
         }
         nelts++;
@@ -90,7 +91,7 @@ ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig)
     while (nelts--) *htop++ = *ptr++;
 
     *hpp = htop;
-    *pp  = ptr;
+    return ptr;
 }
 #endif
 
diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c
index 1a4a4c7952..aaedba1afd 100644
--- a/erts/emulator/hipe/hipe_gc.c
+++ b/erts/emulator/hipe/hipe_gc.c
@@ -91,7 +91,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
 			ASSERT(is_boxed(val));
 			*nsp_i = val;
 		    } else if (!erts_is_literal(gval, ptr)) {
-			move_boxed(&ptr, val, &n_htop, nsp_i);
+			move_boxed(ptr, val, &n_htop, nsp_i);
 		    }
 		} else if (is_list(gval)) {
 		    Eterm *ptr = list_val(gval);
@@ -100,7 +100,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
 			*nsp_i = ptr[1];
 		    } else if (!erts_is_literal(gval, ptr)) {
 			ASSERT(erts_dbg_within_proc(ptr, p, NULL));
-			move_cons(&ptr, val, &n_htop, nsp_i);
+			move_cons(ptr, val, &n_htop, nsp_i);
 		    }
 		}
 	    }
@@ -206,10 +206,10 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
 			ASSERT(is_boxed(val));
 			*nsp_i = val;
 		    } else if (ErtsInArea(ptr, mature, mature_size)) {
-			move_boxed(&ptr, val, &old_htop, nsp_i);
+			move_boxed(ptr, val, &old_htop, nsp_i);
 		    } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
 			ASSERT(erts_dbg_within_proc(ptr, p, NULL));
-			move_boxed(&ptr, val, &n_htop, nsp_i);
+			move_boxed(ptr, val, &n_htop, nsp_i);
 		    }
 		} else if (is_list(gval)) {
 		    Eterm *ptr = list_val(gval);
@@ -217,10 +217,10 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
 		    if (IS_MOVED_CONS(val)) {
 			*nsp_i = ptr[1];
 		    } else if (ErtsInArea(ptr, mature, mature_size)) {
-			move_cons(&ptr, val, &old_htop, nsp_i);
+			move_cons(ptr, val, &old_htop, nsp_i);
 		    } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
 			ASSERT(erts_dbg_within_proc(ptr, p, NULL));
-			move_cons(&ptr, val, &n_htop, nsp_i);
+			move_cons(ptr, val, &n_htop, nsp_i);
 		    }
 		}
 	    }
@@ -278,7 +278,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area,
 			ASSERT(is_boxed(val));
 			*nsp_i = val;
 		    } else if (ErtsInArea(ptr, area, area_size)) {
-			move_boxed(&ptr, val, &old_htop, nsp_i);
+			move_boxed(ptr, val, &old_htop, nsp_i);
 		    }
 		} else if (is_list(gval)) {
 		    Eterm *ptr = list_val(gval);
@@ -286,7 +286,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area,
 		    if (IS_MOVED_CONS(val)) {
 			*nsp_i = ptr[1];
 		    } else if (ErtsInArea(ptr, area, area_size)) {
-			move_cons(&ptr, val, &old_htop, nsp_i);
+			move_cons(ptr, val, &old_htop, nsp_i);
 		    }
 		}
 	    }
-- 
cgit v1.2.3


From 17cdb9694b97854ced0ae257f97a9713d0dcefae Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 16 Jan 2018 21:44:34 +0100
Subject: erts: Polish GC rootset interation

---
 erts/emulator/beam/erl_gc.c | 65 +++++++++++++++++----------------------------
 1 file changed, 24 insertions(+), 41 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 238ac4ad68..a5a59c8e74 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1161,7 +1161,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
 
 	roots++;
 
-        while (g_sz--) {
+        for ( ; g_sz--; g_ptr++) {
             Eterm gval = *g_ptr;
 
             switch (primary_tag(gval)) {
@@ -1170,26 +1170,21 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
 		val = *ptr;
                 if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
-                    *g_ptr++ = val;
+                    *g_ptr = val;
 		} else if (ErtsInArea(ptr, area, area_size)) {
-                    move_boxed(ptr,val,&old_htop,g_ptr++);
-		} else {
-		    g_ptr++;
+                    move_boxed(ptr,val,&old_htop,g_ptr);
 		}
 		break;
 	    case TAG_PRIMARY_LIST:
                 ptr = list_val(gval);
                 val = *ptr;
                 if (IS_MOVED_CONS(val)) { /* Moved */
-                    *g_ptr++ = ptr[1];
+                    *g_ptr = ptr[1];
 		} else if (ErtsInArea(ptr, area, area_size)) {
-                    move_cons(ptr,val,&old_htop,g_ptr++);
-                } else {
-		    g_ptr++;
-		}
+                    move_cons(ptr,val,&old_htop,g_ptr);
+                }
 		break;
 	    default:
-                g_ptr++;
 		break;
 	    }
 	}
@@ -1497,7 +1492,7 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
         Uint g_sz = roots->sz;
 
 	roots++;
-        while (g_sz--) {
+        for ( ; g_sz--; g_ptr++) {
             gval = *g_ptr;
 
             switch (primary_tag(gval)) {
@@ -1507,14 +1502,12 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
                 val = *ptr;
                 if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
-                    *g_ptr++ = val;
+                    *g_ptr = val;
                 } else if (ErtsInArea(ptr, mature, mature_size)) {
-                    move_boxed(ptr,val,&old_htop,g_ptr++);
+                    move_boxed(ptr,val,&old_htop,g_ptr);
                 } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
-                    move_boxed(ptr,val,&n_htop,g_ptr++);
-                } else {
-		    g_ptr++;
-		}
+                    move_boxed(ptr,val,&n_htop,g_ptr);
+                }
                 break;
 	    }
 
@@ -1522,19 +1515,15 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
                 ptr = list_val(gval);
                 val = *ptr;
                 if (IS_MOVED_CONS(val)) { /* Moved */
-                    *g_ptr++ = ptr[1];
+                    *g_ptr = ptr[1];
                 } else if (ErtsInArea(ptr, mature, mature_size)) {
-                    move_cons(ptr,val,&old_htop,g_ptr++);
+                    move_cons(ptr,val,&old_htop,g_ptr);
                 } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
-                    move_cons(ptr,val,&n_htop,g_ptr++);
-                } else {
-		    g_ptr++;
-		}
+                    move_cons(ptr,val,&n_htop,g_ptr);
+                }
 		break;
 	    }
-
 	    default:
-                g_ptr++;
 		break;
             }
         }
@@ -1818,7 +1807,7 @@ full_sweep_heaps(Process *p,
 	Eterm g_sz = roots->sz;
 
 	roots++;
-	while (g_sz--) {
+	for ( ; g_sz--; g_ptr++) {
 	    Eterm* ptr;
 	    Eterm val;
 	    Eterm gval = *g_ptr;
@@ -1830,32 +1819,26 @@ full_sweep_heaps(Process *p,
 		val = *ptr;
 		if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
-		    *g_ptr++ = val;
+		    *g_ptr = val;
 		} else if (!erts_is_literal(gval, ptr)) {
-		    move_boxed(ptr,val,&n_htop,g_ptr++);
-		} else {
-		    g_ptr++;
+		    move_boxed(ptr,val,&n_htop,g_ptr);
 		}
-		continue;
+                break;
 	    }
 
 	    case TAG_PRIMARY_LIST: {
 		ptr = list_val(gval);
 		val = *ptr;
 		if (IS_MOVED_CONS(val)) {
-		    *g_ptr++ = ptr[1];
+		    *g_ptr = ptr[1];
 		} else if (!erts_is_literal(gval, ptr)) {
-		    move_cons(ptr,val,&n_htop,g_ptr++);
-		} else {
-		    g_ptr++;
+		    move_cons(ptr,val,&n_htop,g_ptr);
 		}
-		continue;
+                break;
 	    }
 
-	    default: {
-		g_ptr++;
-		continue;
-	    }
+            default:
+                break;
 	    }
 	}
     }
-- 
cgit v1.2.3


From 9b0122b65bdcafbae2a3cfd3299903da0948acab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Tue, 28 Nov 2017 07:28:25 +0100
Subject: Don't build a stacktrace if it's only passed to erlang:raise/3

Consider the following function:

    function({function,Name,Arity,CLabel,Is0}, Lc0) ->
      try
        %% Optimize the code for the function.
      catch
        Class:Error:Stack ->
          io:format("Function: ~w/~w\n", [Name,Arity]),
          erlang:raise(Class, Error, Stack)
      end.

The stacktrace is retrieved, but it is only used in the call
to erlang:raise/3. There is no need to build a stacktrace
in this function. We can avoid the building if we introduce
an instruction called raw_raise/3 that works exactly like
the erlang:raise/3 BIF except that its third argument must
be a raw stacktrace.
---
 erts/emulator/beam/instrs.tab        | 25 +++++++++++++++++++++++++
 erts/emulator/beam/ops.tab           |  1 +
 erts/emulator/hipe/hipe_bif0.tab     |  1 +
 erts/emulator/hipe/hipe_bif_list.m4  |  1 +
 erts/emulator/hipe/hipe_native_bif.c | 26 ++++++++++++++++++++++++++
 erts/emulator/hipe/hipe_native_bif.h |  2 ++
 erts/emulator/hipe/hipe_primops.h    |  1 +
 7 files changed, 57 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index b92152238e..f19027b1ec 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -936,3 +936,28 @@ build_stacktrace() {
     x(0) = build_stacktrace(c_p, x(0));
     SWAPIN;
 }
+
+raw_raise() {
+    Eterm class = x(0);
+    Eterm value = x(1);
+    Eterm stacktrace = x(2);
+
+    if (class == am_error) {
+        c_p->freason = EXC_ERROR & ~EXF_SAVETRACE;
+        c_p->fvalue = value;
+        c_p->ftrace = stacktrace;
+        goto find_func_info;
+    } else if (class == am_exit) {
+        c_p->freason = EXC_EXIT & ~EXF_SAVETRACE;
+        c_p->fvalue = value;
+        c_p->ftrace = stacktrace;
+        goto find_func_info;
+    } else if (class == am_throw) {
+        c_p->freason = EXC_THROWN & ~EXF_SAVETRACE;
+        c_p->fvalue = value;
+        c_p->ftrace = stacktrace;
+        goto find_func_info;
+    } else {
+        x(0) = am_badarg;
+    }
+}
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index fd1b3b9c74..1f4a8eadb0 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1584,3 +1584,4 @@ i_recv_set
 #
 
 build_stacktrace
+raw_raise
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 0380e8c795..6728e20123 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -109,6 +109,7 @@ atom suspend_0
 atom gc_1
 atom hipe_apply
 atom rethrow
+atom raw_raise
 atom find_na_or_make_stub
 atom nonclosure_address
 atom atomic_inc
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index 0562d676ae..33b3cc1ee5 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -220,6 +220,7 @@ standard_bif_interface_1(nbif_bnot_1, bnot_1)
 standard_bif_interface_1(nbif_set_timeout, hipe_set_timeout)
 standard_bif_interface_1(nbif_conv_big_to_float, hipe_conv_big_to_float)
 standard_bif_interface_2(nbif_rethrow, hipe_rethrow)
+standard_bif_interface_3(nbif_raw_raise, hipe_raw_raise)
 standard_bif_interface_3(nbif_find_na_or_make_stub, hipe_find_na_or_make_stub)
 standard_bif_interface_2(nbif_nonclosure_address, hipe_nonclosure_address)
 nocons_nofail_primop_interface_0(nbif_fclearerror_error, hipe_fclearerror_error)
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index ac1480d2ed..498b43ac6b 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -314,6 +314,32 @@ BIF_RETTYPE nbif_impl_hipe_rethrow(NBIF_ALIST_2)
     }
 }
 
+/* Called via standard_bif_interface_3 */
+BIF_RETTYPE nbif_impl_hipe_raw_raise(NBIF_ALIST_3)
+{
+    Process *c_p = BIF_P;
+    Eterm class = BIF_ARG_1;
+    Eterm value = BIF_ARG_2;
+    Eterm stacktrace = BIF_ARG_3;
+    Eterm reason;
+
+    if (class == am_error) {
+	c_p->fvalue = value;
+	reason = EXC_ERROR;
+    } else if (class == am_exit) {
+	c_p->fvalue = value;
+	reason = EXC_EXIT;
+    } else if (class == am_throw) {
+	c_p->fvalue = value;
+	reason = EXC_THROWN;
+    } else {
+        return am_badarg;
+    }
+    reason &= ~EXF_SAVETRACE;
+    c_p->ftrace = stacktrace;
+    BIF_ERROR(c_p, reason);
+}
+
 /*
  * Support for compiled binary syntax operations.
  */
diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h
index 5711594b1e..ba42b126be 100644
--- a/erts/emulator/hipe/hipe_native_bif.h
+++ b/erts/emulator/hipe/hipe_native_bif.h
@@ -36,6 +36,7 @@ AEXTERN(int,nbif_suspend_msg,(void));
 AEXTERN(int,nbif_suspend_msg_timeout,(void));
 
 AEXTERN(Eterm,nbif_rethrow,(Process*, Eterm, Eterm));
+AEXTERN(Eterm,nbif_raw_raise,(Process*, Eterm, Eterm, Eterm));
 AEXTERN(Eterm,nbif_set_timeout,(Process*, Eterm));
 
 AEXTERN(Eterm,nbif_gc_1,(void));
@@ -82,6 +83,7 @@ void hipe_gc(Process*, Eterm);
 BIF_RETTYPE nbif_impl_hipe_set_timeout(NBIF_ALIST_1);
 void hipe_handle_exception(Process*);
 BIF_RETTYPE nbif_impl_hipe_rethrow(NBIF_ALIST_2);
+BIF_RETTYPE nbif_impl_hipe_raw_raise(NBIF_ALIST_3);
 char *hipe_bs_allocate(int);
 Binary *hipe_bs_reallocate(Binary*, int);
 int hipe_bs_put_small_float(Process*, Eterm, Uint, byte*, unsigned, unsigned);
diff --git a/erts/emulator/hipe/hipe_primops.h b/erts/emulator/hipe/hipe_primops.h
index d6fd10bdff..c5f10672f3 100644
--- a/erts/emulator/hipe/hipe_primops.h
+++ b/erts/emulator/hipe/hipe_primops.h
@@ -46,6 +46,7 @@ PRIMOP_LIST(am_clear_timeout, &nbif_clear_timeout)
 PRIMOP_LIST(am_select_msg, &nbif_select_msg)
 PRIMOP_LIST(am_set_timeout, &nbif_set_timeout)
 PRIMOP_LIST(am_rethrow, &nbif_rethrow)
+PRIMOP_LIST(am_raw_raise, &nbif_raw_raise)
 
 
 PRIMOP_LIST(am_bs_get_integer_2, &nbif_bs_get_integer_2)
-- 
cgit v1.2.3


From cd1253b9d13b7f0e7cdae995d35bf10c3d537053 Mon Sep 17 00:00:00 2001
From: Peer Stritzinger <peer@stritzinger.com>
Date: Mon, 26 Jun 2017 19:18:10 +0200
Subject: Make sure ERTS_WRITE_UNLIKELY section is also set on declaration

Otherwise on targets which have small data area with short addressing
like on PowerPC ther will be linking errors due to the mismatch of
declaration/usage and definition.
---
 erts/emulator/beam/erl_alloc.h   |  6 ++++--
 erts/emulator/beam/erl_msacc.h   |  4 ++--
 erts/emulator/beam/erl_process.h | 22 +++++++++++-----------
 3 files changed, 17 insertions(+), 15 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 117f96a4ad..efde36e47a 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -126,8 +126,10 @@ typedef struct {
     void *extra;
 } ErtsAllocatorFunctions_t;
 
-extern ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1];
-extern ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1];
+extern ErtsAllocatorFunctions_t
+    ERTS_WRITE_UNLIKELY(erts_allctrs[ERTS_ALC_A_MAX+1]);
+extern ErtsAllocatorInfo_t
+    ERTS_WRITE_UNLIKELY(erts_allctrs_info[ERTS_ALC_A_MAX+1]);
 
 typedef struct {
     int enabled;
diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h
index 2588dec903..895b1ae319 100644
--- a/erts/emulator/beam/erl_msacc.h
+++ b/erts/emulator/beam/erl_msacc.h
@@ -159,12 +159,12 @@ struct erl_msacc_t_ {
 
 #ifdef ERTS_ENABLE_MSACC
 
-extern erts_tsd_key_t erts_msacc_key;
+extern erts_tsd_key_t ERTS_WRITE_UNLIKELY(erts_msacc_key);
 
 #ifdef ERTS_MSACC_ALWAYS_ON
 #define erts_msacc_enabled 1
 #else
-extern int erts_msacc_enabled;
+extern int ERTS_WRITE_UNLIKELY(erts_msacc_enabled);
 #endif
 
 #define ERTS_MSACC_TSD_GET() erts_tsd_get(erts_msacc_key)
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 66d7848f89..76dea2ce37 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -105,12 +105,12 @@ struct saved_calls {
 };
 
 extern Export exp_send, exp_receive, exp_timeout;
-extern int erts_sched_compact_load;
-extern int erts_sched_balance_util;
-extern Uint erts_no_schedulers;
-extern Uint erts_no_total_schedulers;
-extern Uint erts_no_dirty_cpu_schedulers;
-extern Uint erts_no_dirty_io_schedulers;
+extern int ERTS_WRITE_UNLIKELY(erts_sched_compact_load);
+extern int ERTS_WRITE_UNLIKELY(erts_sched_balance_util);
+extern Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers);
+extern Uint ERTS_WRITE_UNLIKELY(erts_no_total_schedulers);
+extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers);
+extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers);
 extern Uint erts_no_run_queues;
 extern int erts_sched_thread_suggested_stack_size;
 extern int erts_dcpu_sched_thread_suggested_stack_size;
@@ -1270,7 +1270,7 @@ void erts_check_for_holes(Process* p);
 #define SPO_OFF_HEAP_MSGQ 16
 #define SPO_ON_HEAP_MSGQ 32
 
-extern int erts_default_spo_flags;
+extern int ERTS_WRITE_UNLIKELY(erts_default_spo_flags);
 
 /*
  * The following struct contains options for a process to be spawned.
@@ -1326,10 +1326,10 @@ extern erts_rwmtx_t erts_cpu_bind_rwmtx;
 ** erts_system_monitor must be != NIL, to allow testing on just
 ** the erts_system_monitor_* variables.
 */
-extern Eterm erts_system_monitor;
-extern Uint erts_system_monitor_long_gc;
-extern Uint erts_system_monitor_long_schedule;
-extern Uint erts_system_monitor_large_heap;
+extern Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor);
+extern Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_long_gc);
+extern Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_long_schedule);
+extern Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_large_heap);
 struct erts_system_monitor_flags_t {
 	 unsigned int busy_port : 1;
     unsigned int busy_dist_port : 1;
-- 
cgit v1.2.3


From 7b3c7a72843b9ff65d2a99f5ac73756ee3b291d1 Mon Sep 17 00:00:00 2001
From: Sebastien Merle <s.merle@gmail.com>
Date: Tue, 23 Jan 2018 20:14:25 +0100
Subject: Fix more missing ERTS_WRITE_UNLIKELY section on declaration

---
 erts/emulator/beam/erl_process.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 76dea2ce37..55c020d47b 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -111,7 +111,7 @@ extern Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers);
 extern Uint ERTS_WRITE_UNLIKELY(erts_no_total_schedulers);
 extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers);
 extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers);
-extern Uint erts_no_run_queues;
+extern Uint ERTS_WRITE_UNLIKELY(erts_no_run_queues);
 extern int erts_sched_thread_suggested_stack_size;
 extern int erts_dcpu_sched_thread_suggested_stack_size;
 extern int erts_dio_sched_thread_suggested_stack_size;
@@ -522,7 +522,7 @@ typedef union {
     char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))];
 } ErtsAlignedRunQueue;
 
-extern ErtsAlignedRunQueue *erts_aligned_run_queues;
+extern ErtsAlignedRunQueue * ERTS_WRITE_UNLIKELY(erts_aligned_run_queues);
 
 #define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\
 do {								\
@@ -675,9 +675,9 @@ typedef union {
     char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))];
 } ErtsAlignedSchedulerData;
 
-extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
-extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data;
-extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
+extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_scheduler_data);
+extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_cpu_scheduler_data);
+extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_io_scheduler_data);
 
 
 #if defined(ERTS_ENABLE_LOCK_CHECK)
-- 
cgit v1.2.3


From 58632af80fd43955ec58c021e5c0d04caa1840de Mon Sep 17 00:00:00 2001
From: Peer Stritzinger <peer@stritzinger.com>
Date: Sat, 4 Mar 2017 15:08:41 +0100
Subject: erts_allctr_wrappers should be declared extern in header

---
 erts/emulator/beam/erl_alloc.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index efde36e47a..0438b7cd8c 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -146,7 +146,7 @@ typedef struct ErtsAllocatorWrapper_t_ {
     void (*unlock)(void);
     struct ErtsAllocatorWrapper_t_* next;
 }ErtsAllocatorWrapper_t;
-ErtsAllocatorWrapper_t *erts_allctr_wrappers;
+extern ErtsAllocatorWrapper_t *erts_allctr_wrappers;
 extern int erts_allctr_wrapper_prelocked;
 extern erts_tsd_key_t erts_allctr_prelock_tsd_key;
 void erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t* wrapper);
-- 
cgit v1.2.3


From 2207e5248bbaf2961f179ddcc1dabbab292ca3c9 Mon Sep 17 00:00:00 2001
From: Sebastien Merle <s.merle@gmail.com>
Date: Tue, 23 Jan 2018 20:14:58 +0100
Subject: Add missing extern storage class in global.h header

---
 erts/emulator/beam/global.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 86e2c351af..ebe673cd86 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -948,8 +948,8 @@ void erts_update_ranges(BeamInstr* code, Uint size);
 void erts_remove_from_ranges(BeamInstr* code);
 UWord erts_ranges_sz(void);
 void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info);
-ErtsLiteralArea** erts_dump_lit_areas;
-Uint erts_dump_num_lit_areas;
+extern ErtsLiteralArea** erts_dump_lit_areas;
+extern Uint erts_dump_num_lit_areas;
 
 /* break.c */
 void init_break_handler(void);
-- 
cgit v1.2.3


From fbcff5a137e37edd80aca9c5fe18ce6880648169 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 25 Jan 2018 10:09:59 +0100
Subject: Eliminate get_list/3 internally in the compiler

Instructions that produce more than one result complicate
optimizations. get_list/3 is one of two instructions that
produce multiple results (get_map_elements/3 is the other).

Introduce the get_hd/2 and get_tl/2 instructions
that return the head and tail of a cons cell, respectively,
and use it internally in all optimization passes.

For efficiency, we still want to use get_list/3 if both
head and tail are used, so we will translate matching pairs
of get_hd and get_tl back to get_list instructions.
---
 erts/emulator/beam/instrs.tab | 10 ++++++++++
 erts/emulator/beam/ops.tab    |  3 +++
 2 files changed, 13 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index f19027b1ec..6a531fcc09 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -322,6 +322,16 @@ get_list(Src, Hd, Tl) {
     $Tl = tl;
 }
 
+get_hd(Src, Hd) {
+    Eterm* tmp_ptr = list_val($Src);
+    $Hd = CAR(tmp_ptr);
+}
+
+get_tl(Src, Tl) {
+    Eterm* tmp_ptr = list_val($Src);
+    $Tl = CDR(tmp_ptr);
+}
+
 i_get(Src, Dst) {
     $Dst = erts_pd_hash_get(c_p, $Src);
 }
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 1f4a8eadb0..77e375f2c0 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -182,6 +182,9 @@ get_list r x y
 get_list r y r
 get_list r x r
 
+get_hd xy xy
+get_tl xy xy
+
 # Old-style catch.
 catch y f
 catch_end y
-- 
cgit v1.2.3


From 38a99af36f044459db40b76be2cc72c638eb6d98 Mon Sep 17 00:00:00 2001
From: bhuztez <bhuztez@gmail.com>
Date: Sun, 31 Dec 2017 14:07:49 +0800
Subject: make HiPE work on x86_64 when PIE is enabled

Currently HiPE amd64 assumes the runtime system code is loaded into
the low 2G of the address space. However, this is not the case when
PIE is enabled, it is loaded into a random location. So trampolines
are required to call BIFs, and also we have first to load the address
of sse2_fnegate_mask to a regisiter before xorpd in fchs.
---
 erts/emulator/hipe/hipe_amd64.c | 106 ++++++++++++++++++++++++++++++++--------
 erts/emulator/hipe/hipe_bif0.c  |   6 +--
 2 files changed, 88 insertions(+), 24 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c
index e3cff4a4ba..bdeae23c7c 100644
--- a/erts/emulator/hipe/hipe_amd64.c
+++ b/erts/emulator/hipe/hipe_amd64.c
@@ -28,6 +28,7 @@
 #include "error.h"
 #include "bif.h"
 #include "big.h"	/* term_to_Sint() */
+#include "erl_binary.h"
 
 #include "hipe_arch.h"
 #include "hipe_bif0.h"
@@ -52,9 +53,9 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type)
     switch (type) {
       case am_closure:
       case am_constant:
+      case am_c_const:
 	*(Uint64*)address = value;
 	break;
-      case am_c_const:
       case am_atom:
 	/* check that value fits in an unsigned imm32 */
 	/* XXX: are we sure it's not really a signed imm32? */
@@ -71,14 +72,19 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type)
 
 int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
 {
-    Sint rel32;
+    Sint64 destOffset = (Sint64)destAddress - (Sint64)callAddress - 4;
 
-    ASSERT(trampoline == NULL);
+    if ((destOffset < -0x80000000L) || (destOffset >= 0x80000000L)) {
+        destOffset = ((Sint64)trampoline - 2) - (Sint64)callAddress - 4;
 
-    rel32 = (Sint)destAddress - (Sint)callAddress - 4;
-    if ((Sint)(Sint32)rel32 != rel32)
-	return -1;
-    *(Uint32*)callAddress = (Uint32)rel32;
+        if ((destOffset < -0x80000000L) || (destOffset >= 0x80000000L))
+            return -1;
+
+        *(void**)trampoline = destAddress;
+        hipe_flush_icache_word(trampoline);
+    }
+
+    *(Uint32*)callAddress = (Uint32)destOffset;
     hipe_flush_icache_word(callAddress);
     return 0;
 }
@@ -96,12 +102,70 @@ static void *alloc_code(unsigned int alloc_bytes)
     return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes);
 }
 
+static int check_callees(Eterm callees)
+{
+    Eterm *tuple;
+    Uint arity;
+    Uint i;
+
+    if (is_not_tuple(callees))
+	return -1;
+    tuple = tuple_val(callees);
+    arity = arityval(tuple[0]);
+    for (i = 1; i <= arity; ++i) {
+	Eterm mfa = tuple[i];
+	if (is_atom(mfa))
+	    continue;
+	if (is_not_tuple(mfa) ||
+	    tuple_val(mfa)[0] != make_arityval(3) ||
+	    is_not_atom(tuple_val(mfa)[1]) ||
+	    is_not_atom(tuple_val(mfa)[2]) ||
+	    is_not_small(tuple_val(mfa)[3]) ||
+	    unsigned_val(tuple_val(mfa)[3]) > 255)
+	    return -1;
+    }
+    return arity;
+}
+
+#define TRAMPOLINE_BYTES 12
+
+static void generate_trampolines(unsigned char *address,
+                                 int nrcallees, Eterm callees,
+                                 unsigned char **trampvec)
+{
+    unsigned char *trampoline = address;
+    int i;
+
+    for(i = 0; i < nrcallees; ++i) {
+        trampoline[0] = 0x48;           /* movabsq $..., %rax; */
+        trampoline[1] = 0xb8;
+        *((Uint64*)(trampoline+2)) = 0; /* callee's address */
+        trampoline[10] = 0xff;          /* jmpq *%rax */
+        trampoline[11] = 0xe0;
+        trampvec[i] = trampoline+2;
+        trampoline += TRAMPOLINE_BYTES;
+    }
+    hipe_flush_icache_range(address, nrcallees*TRAMPOLINE_BYTES);
+}
+
 void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
 {
-    if (is_not_nil(callees))
+    int nrcallees;
+    Eterm trampvecbin;
+    unsigned char **trampvec;
+    unsigned char *address;
+
+    nrcallees = check_callees(callees);
+    if (nrcallees < 0)
 	return NULL;
-    *trampolines = NIL;
-    return alloc_code(nrbytes);
+
+    trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned char*));
+    trampvec = (unsigned char **)binary_bytes(trampvecbin);
+
+    address = alloc_code(nrbytes + nrcallees*TRAMPOLINE_BYTES);
+    generate_trampolines(address + nrbytes, nrcallees, callees, trampvec);
+    *trampolines = trampvecbin;
+    return address;
 }
 
 void hipe_free_code(void* code, unsigned int bytes)
@@ -129,10 +193,9 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
      */
     unsigned int codeSize;
     unsigned char *code, *codep;
-    unsigned int callEmuOffset;
 
-    codeSize =	/* 23, 26, 29, or 32 bytes */
-      23 +	/* 23 when all offsets are 8-bit */
+    codeSize =	/* 30, 33, 36, or 39 bytes */
+      30 +	/* 30 when all offsets are 8-bit */
       (P_CALLEE_EXP >= 128 ? 3 : 0) +
       ((P_CALLEE_EXP + 4) >= 128 ? 3 : 0) +
       (P_ARITY >= 128 ? 3 : 0);
@@ -197,14 +260,15 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
     codep[0] = beamArity;
     codep += 1;
 
-    /* jmp callemu; 5 bytes */
-    callEmuOffset = (unsigned char*)nbif_callemu - (code + codeSize);
-    codep[0] = 0xe9;
-    codep[1] =  callEmuOffset        & 0xFF;
-    codep[2] = (callEmuOffset >>  8) & 0xFF;
-    codep[3] = (callEmuOffset >> 16) & 0xFF;
-    codep[4] = (callEmuOffset >> 24) & 0xFF;
-    codep += 5;
+    /* jmp callemu; 12 bytes */
+    codep[0] = 0x48;
+    codep[1] = 0xb8;
+    codep += 2;
+    *(Uint64*)codep = (Uint64)nbif_callemu;
+    codep += 8;
+    codep[0] = 0xff;
+    codep[1] = 0xe0;
+    codep += 2;
 
     ASSERT(codep == code + codeSize);
 
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index e477c4cdea..a8be64e08d 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1112,7 +1112,7 @@ static struct hipe_mfa_info* mod2mfa_put(struct hipe_mfa_info* mfa)
 struct hipe_ref {
     struct hipe_ref_head head;    /* list of refs to same calleee */
     void *address;
-#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
+#if defined(__x86_64__) || defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
     void *trampoline;
 #endif
     unsigned int flags;
@@ -1543,7 +1543,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
 
     ref = erts_alloc(ERTS_ALC_T_HIPE_LL, sizeof(struct hipe_ref));
     ref->address = address;
-#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
+#if defined(__x86_64__) || defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
     ref->trampoline = trampoline;
 #endif
     ref->flags = flags;
@@ -1819,7 +1819,7 @@ void hipe_redirect_to_module(Module* modp)
 	    if (ref->flags & REF_FLAG_IS_LOAD_MFA)
 		res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa);
 	    else {
-#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
+#if defined(__x86_64__) || defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
                 void* trampoline = ref->trampoline;
 #else
                 void* trampoline = NULL;
-- 
cgit v1.2.3


From 627a958104c50c9cb4a5022b29239b7d63fb5d76 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 29 Jan 2018 17:51:06 +0100
Subject: erts: Refactor hipe x86_64 trampolines

---
 erts/emulator/hipe/hipe_amd64.c | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c
index bdeae23c7c..f23f341e6d 100644
--- a/erts/emulator/hipe/hipe_amd64.c
+++ b/erts/emulator/hipe/hipe_amd64.c
@@ -39,6 +39,8 @@
 #undef ERL_FUN_SIZE
 #include "hipe_literals.h"
 
+static void patch_trampoline(void *trampoline, void *destAddress);
+
 const Uint sse2_fnegate_mask[2] = {0x8000000000000000,0};
 
 void hipe_patch_load_fe(Uint64 *address, Uint64 value)
@@ -75,13 +77,12 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
     Sint64 destOffset = (Sint64)destAddress - (Sint64)callAddress - 4;
 
     if ((destOffset < -0x80000000L) || (destOffset >= 0x80000000L)) {
-        destOffset = ((Sint64)trampoline - 2) - (Sint64)callAddress - 4;
+        destOffset = (Sint64)trampoline - (Sint64)callAddress - 4;
 
         if ((destOffset < -0x80000000L) || (destOffset >= 0x80000000L))
             return -1;
 
-        *(void**)trampoline = destAddress;
-        hipe_flush_icache_word(trampoline);
+        patch_trampoline(trampoline, destAddress);
     }
 
     *(Uint32*)callAddress = (Uint32)destOffset;
@@ -139,15 +140,25 @@ static void generate_trampolines(unsigned char *address,
     for(i = 0; i < nrcallees; ++i) {
         trampoline[0] = 0x48;           /* movabsq $..., %rax; */
         trampoline[1] = 0xb8;
-        *((Uint64*)(trampoline+2)) = 0; /* callee's address */
+        *(void**)(trampoline+2) = NULL; /* callee's address */
         trampoline[10] = 0xff;          /* jmpq *%rax */
         trampoline[11] = 0xe0;
-        trampvec[i] = trampoline+2;
+        trampvec[i] = trampoline;
         trampoline += TRAMPOLINE_BYTES;
     }
     hipe_flush_icache_range(address, nrcallees*TRAMPOLINE_BYTES);
 }
 
+static void patch_trampoline(void *trampoline, void *destAddress)
+{
+    unsigned char *tp = (unsigned char*) trampoline;
+
+    ASSERT(tp[0] == 0x48 && tp[1] == 0xb8);
+
+    *(void**)(tp+2) = destAddress; /* callee's address */
+    hipe_flush_icache_word(tp+2);
+}
+
 void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
 {
     int nrcallees;
-- 
cgit v1.2.3


From 6a09d85b1762b87d0dc6cf503fcb20080237339b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 29 Jan 2018 19:10:20 +0100
Subject: More ERTS_RESTRICT usage

---
 erts/emulator/beam/copy.c   | 5 +++--
 erts/emulator/beam/global.h | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 8a82154a08..7769a914db 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -611,7 +611,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint
     Eterm* htop;
     Eterm* hbot;
     Eterm* hp;
-    Eterm* objp;
+    Eterm* ERTS_RESTRICT objp;
     Eterm* tp;
     Eterm  res;
     Eterm  elem;
@@ -1821,7 +1821,8 @@ all_clean:
  *
  * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr).
  */
-Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
+Eterm copy_shallow(Eterm* ERTS_RESTRICT ptr, Uint sz, Eterm** hpp,
+                   ErlOffHeap* off_heap)
 {
     Eterm* tp = ptr;
     Eterm* hp = *hpp;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 86e2c351af..132ef878c3 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1066,7 +1066,7 @@ Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint*, erts_literal_area_
 #define copy_struct_litopt(Obj,Sz,HPP,OH,LitArea) \
     copy_struct_x(Obj,Sz,HPP,OH,NULL,LitArea)
 
-Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*);
+Eterm copy_shallow(Eterm* ERTS_RESTRICT, Uint, Eterm**, ErlOffHeap*);
 
 void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first,
 			   Eterm* refs, unsigned nrefs, int literals);
-- 
cgit v1.2.3


From 250f6d74e645968d036bad14d5b92907ff4d7342 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 30 Jan 2018 21:53:39 +0100
Subject: erts: Remove low memory demand for hipe amd64 code

---
 erts/emulator/sys/common/erl_mmap.c | 26 --------------------------
 1 file changed, 26 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index a9c6e72c5f..95e6c6fbd0 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -1371,32 +1371,6 @@ os_mmap_virtual(char *ptr, UWord size, int exec)
     int flags = ERTS_MMAP_VIRTUAL_FLAGS;
     void* res;
 
-#ifdef ERTS_ALC_A_EXEC
-    if (exec) {
-        ASSERT(!ptr);
-        /* OTP-19.0: Nice hack below cut-and-pasted from hipe_amd64.c */
-
-#  ifdef MAP_32BIT
-        /* If we got MAP_32BIT (Linux), then use that to ask for low memory */
-        flags |= MAP_32BIT;
-#  else
-        /* FreeBSD doesn't have MAP_32BIT, and it doesn't respect
-           a plain map_hint (returns high mappings even though the
-           hint refers to a free area), so we have to use both map_hint
-           and MAP_FIXED to get addresses below the 2GB boundary.
-           This is even worse than the Linux/ppc64 case.
-           Similarly, Solaris 10 doesn't have MAP_32BIT,
-           and it doesn't respect a plain map_hint. */
-        ptr = (char*)(512*1024*1024); /* 0.5GB */
-
-#    if defined(__FreeBSD__) || defined(__sun__)
-        flags |= MAP_FIXED;
-#    endif
-#  endif /* !MAP_32BIT */
-    }
-#else /* !ERTS_ALC_A_EXEC */
-    ASSERT(!exec);
-#endif
     res = mmap((void *) ptr, (size_t) size, ERTS_MMAP_VIRTUAL_PROT,
                flags, ERTS_MMAP_FD, 0);
     if (res == (void *) MAP_FAILED)
-- 
cgit v1.2.3


From ad29b4d489b5a3f1139d3307a67c544b93995e26 Mon Sep 17 00:00:00 2001
From: Philip Stears <philip@philipstears.com>
Date: Tue, 6 Feb 2018 13:59:00 +0100
Subject: Optimize non-strict equality check of binaries differing in size

This commit brings the perform of a non-strict equality check of
binaries into the same range as strict ones.

Currently, these checks can be significantly slower because they go
through the general comparison code which doesn't have an optimization
to check the size before comparing contents in the case that it's
checking equality.
---
 erts/emulator/beam/utils.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index fe9f1c7606..4bf60619ba 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3152,6 +3152,9 @@ tailrecur_ne:
 		    int cmp;
 		    byte* a_ptr;
 		    byte* b_ptr;
+		    if (eq_only && a_size != b_size) {
+		        RETURN_NEQ(a_size - b_size);
+		    }
 		    ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize);
 		    ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize);
 		    if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) {
-- 
cgit v1.2.3


From 443b2e0fabe9dfbe78d7fe857b29e74f7533da39 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 23 Jan 2018 22:26:44 +0100
Subject: erts: Add migration options "acnl" and "acfml"

acnl:  Abandon Carrier Nr Limit
acfml: Abandon Carrier Free block Min Limit
---
 erts/emulator/beam/erl_alloc.c      | 36 ++++++++++++++++---------
 erts/emulator/beam/erl_alloc_util.c | 52 +++++++++++++++++++++++++++----------
 erts/emulator/beam/erl_alloc_util.h | 12 +++++++--
 3 files changed, 72 insertions(+), 28 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 214fb1f2af..b51cf5bdfc 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1372,18 +1372,30 @@ handle_au_arg(struct au_init *auip,
 
     switch (sub_param[0]) {
     case 'a':
-	if (has_prefix("acul", sub_param)) {
-	    if (!auip->carrier_migration_allowed) {
-		if (!u_switch)
-		    goto bad_switch;
-		else {
-		    /* ignore */
-		    (void) get_acul_value(auip, sub_param + 4, argv, ip);
-		    break;
-		}
-	    }
-	    auip->init.util.acul = get_acul_value(auip, sub_param + 4, argv, ip);
-	}
+        if (sub_param[1] == 'c') { /* Migration parameters "ac*" */
+            UWord value;
+            UWord* wp;
+            if (!auip->carrier_migration_allowed && !u_switch)
+                goto bad_switch;
+
+            if (has_prefix("acul", sub_param)) {
+                value = get_acul_value(auip, sub_param + 4, argv, ip);
+                wp = &auip->init.util.acul;
+            }
+            else if (has_prefix("acnl", sub_param)) {
+                value = get_amount_value(sub_param + 4, argv, ip);
+                wp = &auip->init.util.acnl;
+            }
+            else if (has_prefix("acfml", sub_param)) {
+                value = get_amount_value(sub_param + 5, argv, ip);
+                wp = &auip->init.util.acfml;
+            }
+            else
+                goto bad_switch;
+
+            if (auip->carrier_migration_allowed)
+                *wp = value;
+        }
 	else if(has_prefix("asbcst", sub_param)) {
 	    auip->init.util.asbcst = get_kb_value(sub_param + 6, argv, ip);
 	}
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 230ca6ccbb..016d85fe2a 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -2170,6 +2170,7 @@ static ERTS_INLINE void
 check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp)
 {
     Carrier_t *crr;
+    UWord ncrr_in_pool, largest_fblk;
 
     if (busy_pcrr_pp && *busy_pcrr_pp)
 	return;
@@ -2181,8 +2182,7 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp)
     if (--allctr->cpool.check_limit_count <= 0)
 	set_new_allctr_abandon_limit(allctr);
 
-    if (!erts_thr_progress_is_managed_thread())
-	return;
+    ASSERT(erts_thr_progress_is_managed_thread());
 
     if (allctr->cpool.disable_abandon)
 	return;
@@ -2190,6 +2190,9 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp)
     if (allctr->mbcs.blocks.curr.size > allctr->cpool.abandon_limit)
 	return;
 
+    ncrr_in_pool = erts_atomic_read_nob(&allctr->cpool.stat.no_carriers);
+    if (ncrr_in_pool >= allctr->cpool.in_pool_limit)
+        return;
 
     crr = FBLK_TO_MBC(fblk);
 
@@ -2200,9 +2203,14 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp)
 	return;
 
     if (crr->cpool.thr_prgr != ERTS_THR_PRGR_INVALID
-	&& !erts_thr_progress_has_reached(crr->cpool.thr_prgr))
-	return;
+        && !erts_thr_progress_has_reached(crr->cpool.thr_prgr))
+        return;
+
+    largest_fblk = allctr->largest_fblk_in_mbc(allctr, crr);
+    if (largest_fblk < allctr->cpool.fblk_min_limit)
+        return;
 
+    erts_atomic_set_nob(&crr->cpool.max_size, largest_fblk);
     abandon_carrier(allctr, crr);
 }
 
@@ -3626,8 +3634,6 @@ set_new_allctr_abandon_limit(Allctr_t *allctr)
 static void
 abandon_carrier(Allctr_t *allctr, Carrier_t *crr)
 {
-    erts_aint_t max_size;
-
     STAT_MBC_CPOOL_INSERT(allctr, crr);
 
     unlink_carrier(&allctr->mbc_list, crr);
@@ -3637,9 +3643,6 @@ abandon_carrier(Allctr_t *allctr, Carrier_t *crr)
 
     allctr->remove_mbc(allctr, crr);
 
-    max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr);
-    erts_atomic_set_nob(&crr->cpool.max_size, max_size);
-
     cpool_insert(allctr, crr);
 
     set_new_allctr_abandon_limit(allctr);
@@ -4240,6 +4243,8 @@ static struct {
     Eterm smbcs;
     Eterm mbcgs;
     Eterm acul;
+    Eterm acnl;
+    Eterm acfml;
 
 #if HAVE_ERTS_MSEG
     Eterm mmc;
@@ -4330,6 +4335,8 @@ init_atoms(Allctr_t *allctr)
 	AM_INIT(smbcs);
 	AM_INIT(mbcgs);
 	AM_INIT(acul);
+        AM_INIT(acnl);
+        AM_INIT(acfml);
 
 #if HAVE_ERTS_MSEG
 	AM_INIT(mmc);
@@ -4889,7 +4896,7 @@ info_options(Allctr_t *allctr,
 	     Uint *szp)
 {
     Eterm res = THE_NON_VALUE;
-    int acul;
+    UWord acul, acnl, acfml;
 
     if (!allctr) {
 	if (print_to_p)
@@ -4903,8 +4910,10 @@ info_options(Allctr_t *allctr,
 
 #ifdef ERTS_SMP
     acul = allctr->cpool.util_limit;
+    acnl = allctr->cpool.in_pool_limit;
+    acfml = allctr->cpool.fblk_min_limit;
 #else
-    acul = 0;
+    acul = 0; acnl = 0; acfml = 0;
 #endif
 
     if (print_to_p) {
@@ -4933,7 +4942,7 @@ info_options(Allctr_t *allctr,
 		   "option lmbcs: %beu\n"
 		   "option smbcs: %beu\n"
 		   "option mbcgs: %beu\n"
-		   "option acul: %d\n",
+		   "option acul: %bpu\n",
 		   topt,
 		   allctr->ramv ? "true" : "false",
 		   allctr->sbc_threshold,
@@ -4958,9 +4967,15 @@ info_options(Allctr_t *allctr,
 				  hpp, szp);
 
     if (hpp || szp) {
+        add_2tup(hpp, szp, &res,
+                 am.acfml,
+                 bld_uint(hpp, szp, acfml));
+        add_2tup(hpp, szp, &res,
+                 am.acnl,
+                 bld_uint(hpp, szp, acnl));
 	add_2tup(hpp, szp, &res,
 		 am.acul,
-		 bld_uint(hpp, szp, (UWord) acul));
+		 bld_uint(hpp, szp, acul));
 	add_2tup(hpp, szp, &res,
 		 am.mbcgs,
 		 bld_uint(hpp, szp, allctr->mbc_growth_stages));
@@ -6062,7 +6077,16 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
     erts_atomic_init_nob(&allctr->cpool.stat.carriers_size, 0);
     erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0);
     allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT;
-    allctr->cpool.util_limit = init->ts ? 0 : init->acul;
+    if (!init->ts && init->acul && init->acnl) {
+        allctr->cpool.util_limit = init->acul;
+        allctr->cpool.in_pool_limit = init->acnl;
+        allctr->cpool.fblk_min_limit = init->acfml;
+    }
+    else {
+        allctr->cpool.util_limit = 0;
+        allctr->cpool.in_pool_limit = 0;
+        allctr->cpool.fblk_min_limit = 0;
+    }
 #endif
 
     allctr->sbc_threshold		= init->sbct;
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 81180382af..d2cdc3a320 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -63,7 +63,9 @@ typedef struct {
     UWord lmbcs;
     UWord smbcs;
     UWord mbcgs;
-    int acul;
+    UWord acul;
+    UWord acnl;
+    UWord acfml;
 
     void *fix;
     size_t *fix_type_size;
@@ -118,6 +120,8 @@ typedef struct {
     1024*1024,		/* (bytes)  smbcs:  smallest mbc size            */\
     10,			/* (amount) mbcgs:  mbc growth stages            */\
     0,			/* (%)      acul:  abandon carrier utilization limit */\
+    1000,		/* (amount) acnl:  abandoned carriers number limit */\
+    0,			/* (bytes)  acfml: abandoned carrier fblk min limit */\
     /* --- Data not options -------------------------------------------- */\
     NULL,		/* (ptr)    fix                                  */\
     NULL		/* (ptr)    fix_type_size                        */\
@@ -151,6 +155,8 @@ typedef struct {
     128*1024,		/* (bytes)  smbcs:  smallest mbc size            */\
     10,			/* (amount) mbcgs:  mbc growth stages            */\
     0,			/* (%)      acul:  abandon carrier utilization limit */\
+    1000,		/* (amount) acnl:  abandoned carriers number limit */\
+    0,			/* (bytes)  acfml: abandoned carrier fblk min limit */\
     /* --- Data not options -------------------------------------------- */\
     NULL,		/* (ptr)    fix                                  */\
     NULL		/* (ptr)    fix_type_size                        */\
@@ -568,7 +574,9 @@ struct Allctr_t_ {
 	UWord		abandon_limit;
 	int		disable_abandon;
 	int		check_limit_count;
-	int		util_limit;
+	UWord		util_limit;       /* acul */
+        UWord           in_pool_limit;    /* acnl */
+        UWord           fblk_min_limit;   /* acmfl */
 	struct {
 	    erts_atomic_t	blocks_size;
 	    erts_atomic_t	no_blocks;
-- 
cgit v1.2.3


From 3d8612cd57efd1e96601c5ef9cc986676b76fbf3 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 26 Jan 2018 17:37:38 +0100
Subject: erts: Refactor erl_ao_firstfit_alloc

In preparation for carrier age order.

Change 'flavor' to 'blk_order' and 'crr_order'.
---
 erts/emulator/beam/erl_alloc.c             | 33 +++++-----
 erts/emulator/beam/erl_ao_firstfit_alloc.c | 96 ++++++++++++++++--------------
 erts/emulator/beam/erl_ao_firstfit_alloc.h | 14 +++--
 3 files changed, 77 insertions(+), 66 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index b51cf5bdfc..4c9fb466eb 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -167,7 +167,7 @@ enum allctr_type {
     GOODFIT,
     BESTFIT,
     AFIT,
-    AOFIRSTFIT
+    FIRSTFIT
 };
 
 struct au_init {
@@ -500,8 +500,9 @@ set_default_test_alloc_opts(struct au_init *ip)
     SET_DEFAULT_ALLOC_OPTS(ip);
     ip->enable			= 0; /* Disabled by default */
     ip->thr_spec		= -1 * erts_no_schedulers;
-    ip->atype			= AOFIRSTFIT;
-    ip->init.aoff.flavor        = AOFF_BF;
+    ip->atype			= FIRSTFIT;
+    ip->init.aoff.crr_order     = FF_AOFF;
+    ip->init.aoff.blk_order     = FF_BF;
     ip->init.util.name_prefix	= "test_";
     ip->init.util.alloc_no	= ERTS_ALC_A_TEST;
     ip->init.util.mmbcs 	= 0; /* Main carrier size */
@@ -606,7 +607,7 @@ strategy_support_carrier_migration(struct au_init *auip)
      * Currently only aoff, aoffcbf and aoffcaobf support carrier
      * migration, i.e, type AOFIRSTFIT.
      */
-    return auip->atype == AOFIRSTFIT;
+    return auip->atype == FIRSTFIT;
 }
 
 static ERTS_INLINE void
@@ -622,8 +623,9 @@ adjust_carrier_migration_support(struct au_init *auip)
 	 */
 	if (!strategy_support_carrier_migration(auip)) {
 	    /* Default to aoffcbf */
-	    auip->atype = AOFIRSTFIT;
-	    auip->init.aoff.flavor = AOFF_BF;
+	    auip->atype = FIRSTFIT;
+	    auip->init.aoff.crr_order = FF_AOFF;
+	    auip->init.aoff.blk_order = FF_BF;
 	}
     }
 #else
@@ -1176,7 +1178,7 @@ start_au_allocator(ErtsAlcType_t alctr_n,
 					   &init->init.af,
 					   &init->init.util);
 	    break;
-    	case AOFIRSTFIT:
+	case FIRSTFIT:
 	    as = erts_aoffalc_start((AOFFAllctr_t *) as0,
 					     &init->init.aoff,
 					     &init->init.util);
@@ -1416,16 +1418,19 @@ handle_au_arg(struct au_init *auip,
 		auip->atype = AFIT;
 	    }
 	    else if (strcmp("aoff", alg) == 0) {
-		auip->atype = AOFIRSTFIT;
-		auip->init.aoff.flavor = AOFF_AOFF;
+		auip->atype = FIRSTFIT;
+		auip->init.aoff.crr_order = FF_AOFF;
+		auip->init.aoff.blk_order = FF_AOFF;
 	    }
 	    else if (strcmp("aoffcbf", alg) == 0) {
-		auip->atype = AOFIRSTFIT;
-		auip->init.aoff.flavor = AOFF_BF;
+		auip->atype = FIRSTFIT;
+		auip->init.aoff.crr_order = FF_AOFF;
+		auip->init.aoff.blk_order = FF_BF;
 	    }
 	    else if (strcmp("aoffcaobf", alg) == 0) {
-		auip->atype = AOFIRSTFIT;
-		auip->init.aoff.flavor = AOFF_AOBF;
+		auip->atype = FIRSTFIT;
+		auip->init.aoff.crr_order = FF_AOFF;
+		auip->init.aoff.blk_order = FF_AOBF;
 	    }
 	    else {
 		bad_value(param, sub_param + 1, alg);
@@ -3634,7 +3639,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3)
 					  &init.init.af,
 					  &init.init.util);
 		break;
-	    case AOFIRSTFIT:
+	    case FIRSTFIT:
 		allctr = erts_aoffalc_start((AOFFAllctr_t *)
 					  erts_alloc(ERTS_ALC_T_UNDEF,
 						     sizeof(AOFFAllctr_t)),
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index 05ba1f9891..8808a4f0a4 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -137,11 +137,11 @@ struct AOFF_Carrier_t_ {
 
 #ifdef HARD_DEBUG
 #  define HARD_CHECK_IS_MEMBER(ROOT,NODE) rbt_assert_is_member(ROOT,NODE)
-#  define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) check_tree(CRR, FLV, ROOT, SZ)
-static AOFF_RBTree_t * check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint);
+#  define HARD_CHECK_TREE(CRR,ORDER,ROOT,SZ) check_tree(CRR, ORDER, ROOT, SZ)
+static AOFF_RBTree_t * check_tree(Carrier_t*, enum AOFFSortOrder, AOFF_RBTree_t*, Uint);
 #else
 #  define HARD_CHECK_IS_MEMBER(ROOT,NODE)
-#  define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ)
+#  define HARD_CHECK_TREE(CRR,ORDER,ROOT,SZ)
 #endif
 
 
@@ -179,25 +179,27 @@ static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node,
     else ASSERT(new_max == old_max);
 }
 
-static ERTS_INLINE SWord cmp_blocks(enum AOFF_Flavor flavor,
+static ERTS_INLINE SWord cmp_blocks(enum AOFFSortOrder order,
 				    AOFF_RBTree_t* lhs, AOFF_RBTree_t* rhs)
 {
     ASSERT(lhs != rhs);
-    ASSERT(flavor == AOFF_AOFF || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr));
-    if (flavor != AOFF_AOFF) {
-	SWord diff = (SWord)AOFF_BLK_SZ(lhs) - (SWord)AOFF_BLK_SZ(rhs);
-	if (diff || flavor == AOFF_BF) return diff;
+    {
+	ASSERT(order == FF_AOFF || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr));
+	if (order != FF_AOFF) {
+	    SWord diff = (SWord)AOFF_BLK_SZ(lhs) - (SWord)AOFF_BLK_SZ(rhs);
+	    if (diff || order == FF_BF) return diff;
+	}
     }
     return (char*)lhs - (char*)rhs;
 }
 
-static ERTS_INLINE SWord cmp_cand_blk(enum AOFF_Flavor flavor,
+static ERTS_INLINE SWord cmp_cand_blk(enum AOFFSortOrder order,
 				      Block_t* cand_blk, AOFF_RBTree_t* rhs)
 {
-    if (flavor != AOFF_AOFF) {
+    if (order != FF_AOFF) {
 	if (BLK_TO_MBC(cand_blk) == FBLK_TO_MBC(&rhs->hdr)) {
 	    SWord diff = (SWord)MBC_BLK_SZ(cand_blk) - (SWord)MBC_FBLK_SZ(&rhs->hdr);
-	    if (diff || flavor == AOFF_BF) return diff;
+	    if (diff || order == FF_BF) return diff;
 	}
     }
     return (char*)cand_blk - (char*)rhs;
@@ -218,7 +220,7 @@ static UWord aoff_largest_fblk_in_mbc(Allctr_t*, Carrier_t*);
 
 /* Generic tree functions used by both carrier and block trees. */
 static void rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del);
-static void rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk);
+static void rbt_insert(enum AOFFSortOrder, AOFF_RBTree_t** root, AOFF_RBTree_t* blk);
 static AOFF_RBTree_t* rbt_search(AOFF_RBTree_t* root, Uint size);
 #ifdef HARD_DEBUG
 static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node);
@@ -254,11 +256,12 @@ erts_aoffalc_start(AOFFAllctr_t *alc,
 
     sys_memcpy((void *) alc, (void *) &zero.allctr, sizeof(AOFFAllctr_t));
 
-    alc->flavor                         = aoffinit->flavor;
+    alc->blk_order                      = aoffinit->blk_order;
+    alc->crr_order                      = aoffinit->crr_order;
     allctr->mbc_header_size		= sizeof(AOFF_Carrier_t);
     allctr->min_mbc_size		= MIN_MBC_SZ;
     allctr->min_mbc_first_free_size	= MIN_MBC_FIRST_FREE_SZ;
-    allctr->min_block_size = (aoffinit->flavor == AOFF_BF ?
+    allctr->min_block_size = (aoffinit->blk_order == FF_BF ?
 			      sizeof(AOFF_RBTreeList_t):sizeof(AOFF_RBTree_t));
 
     allctr->vsn_str			= ERTS_ALC_AOFF_ALLOC_VSN_STR;
@@ -487,9 +490,9 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk)
     AOFF_Carrier_t *crr = (AOFF_Carrier_t*) FBLK_TO_MBC(&del->hdr);
 
     ASSERT(crr->rbt_node.hdr.bhdr == crr->root->max_sz);
-    HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0);
+    HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0);
 
-    if (alc->flavor == AOFF_BF) {
+    if (alc->blk_order == FF_BF) {
 	ASSERT(del->flags & IS_BF_FLG);
 	if (IS_LIST_ELEM(del)) {
 	    /* Remove from list */
@@ -510,14 +513,14 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk)
 	    
 	    replace(&crr->root, (AOFF_RBTree_t*)del, LIST_NEXT(del));
 	    
-	    HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0);
+	    HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0);
 	    return;
 	}
     }
 
     rbt_delete(&crr->root, (AOFF_RBTree_t*)del);
 
-    HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0);
+    HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0);
 
     /* Update the carrier tree with a potentially new (lower) max_sz
      */    
@@ -715,9 +718,9 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block)
 
     ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(&blk_crr->crr));
     ASSERT(blk_crr->rbt_node.hdr.bhdr == (blk_crr->root ? blk_crr->root->max_sz : 0));
-    HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0);
+    HARD_CHECK_TREE(&blk_crr->crr, alc->blk_order, blk_crr->root, 0);
 
-    rbt_insert(alc->flavor, &blk_crr->root, blk);
+    rbt_insert(alc->blk_order, &blk_crr->root, blk);
 
     /* Update the carrier tree with a potentially new (larger) max_sz
     */
@@ -731,16 +734,16 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block)
 	    if (!crr_node) break;
 	}
     }
-    HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0);
+    HARD_CHECK_TREE(&blk_crr->crr, alc->blk_order, blk_crr->root, 0);
 }
 
 static void
-rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk)
+rbt_insert(enum AOFFSortOrder order, AOFF_RBTree_t** root, AOFF_RBTree_t* blk)
 {
     Uint blk_sz = AOFF_BLK_SZ(blk);
 
 #ifdef DEBUG
-    blk->flags  = (flavor == AOFF_BF) ? IS_BF_FLG : 0; 
+    blk->flags  = (order == FF_BF) ? IS_BF_FLG : 0;
 #else
     blk->flags  = 0; 
 #endif
@@ -760,7 +763,7 @@ rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk)
 	    if (x->max_sz < blk_sz) {
 		x->max_sz = blk_sz;
 	    }
-	    diff = cmp_blocks(flavor, blk, x);
+	    diff = cmp_blocks(order, blk, x);
 	    if (diff < 0) {
 		if (!x->left) {
 		    blk->parent = x;
@@ -778,7 +781,7 @@ rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk)
 		x = x->right;
 	    }
 	    else {
-		ASSERT(flavor == AOFF_BF);
+		ASSERT(order == FF_BF);
 		ASSERT(blk->flags & IS_BF_FLG);			    
 		ASSERT(x->flags & IS_BF_FLG);			    
 		SET_LIST_ELEM(blk);
@@ -798,7 +801,7 @@ rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk)
 	if (IS_RED(blk->parent))
 	    tree_insert_fixup(root, blk);
     }
-    if (flavor == AOFF_BF) {
+    if (order == FF_BF) {
 	SET_TREE_NODE(blk);
 	LIST_NEXT(blk) = NULL;
     }
@@ -850,7 +853,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size,
     /* Get block within carrier tree
      */
 #ifdef HARD_DEBUG
-    dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, size);
+    dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, size);
 #endif
 
     blk = rbt_search(crr->root, size);
@@ -863,7 +866,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size,
     if (!blk)
 	return NULL;
 
-    if (cand_blk && cmp_cand_blk(alc->flavor, cand_blk, blk) < 0) {
+    if (cand_blk && cmp_cand_blk(alc->blk_order, cand_blk, blk) < 0) {
 	return NULL; /* cand_blk was better */
     }
 
@@ -878,17 +881,17 @@ static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier)
     AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier;
     AOFF_RBTree_t **root = &alc->mbc_root;
 
-    HARD_CHECK_TREE(NULL, 0, *root, 0);
+    HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0);
 
     /* Link carrier in address order tree
      */
     crr->rbt_node.hdr.bhdr = 0;
-    rbt_insert(AOFF_AOFF, root, &crr->rbt_node);
+    rbt_insert(alc->crr_order, root, &crr->rbt_node);
 
     /* aoff_link_free_block will add free block later */
     crr->root = NULL;
 
-    HARD_CHECK_TREE(NULL, 0, *root, 0);
+    HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0);
 }
 
 #define IS_CRR_IN_TREE(CRR,ROOT) \
@@ -911,13 +914,13 @@ static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier)
     AOFF_RBTree_t **root = &alc->mbc_root;
 
     ASSERT(!IS_CRR_IN_TREE(crr, *root));
-    HARD_CHECK_TREE(NULL, 0, *root, 0);   
+    HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0);   
 
     /* Link carrier in address order tree
      */
-    rbt_insert(AOFF_AOFF, root, &crr->rbt_node);
+    rbt_insert(alc->crr_order, root, &crr->rbt_node);
 
-    HARD_CHECK_TREE(NULL, 0, *root, 0);
+    HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0);
 }
 
 static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier)
@@ -931,7 +934,7 @@ static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier)
     if (!IS_CRR_IN_TREE(crr,*root))
         return;
 
-    HARD_CHECK_TREE(NULL, 0, *root, 0);
+    HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0);
 
     rbt_delete(root, &crr->rbt_node);
     crr->rbt_node.parent = NULL;
@@ -939,7 +942,7 @@ static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier)
     crr->rbt_node.right = NULL;
     crr->rbt_node.max_sz = crr->rbt_node.hdr.bhdr;
 
-    HARD_CHECK_TREE(NULL, 0, *root, 0);
+    HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0);
 }
 
 static UWord aoff_largest_fblk_in_mbc(Allctr_t* allctr, Carrier_t* carrier)
@@ -1029,7 +1032,7 @@ info_options(Allctr_t *allctr,
 		   print_to_arg,
 		   "%sas: %s\n",
 		   prefix,
-		   flavor_str[alc->flavor]);
+		   flavor_str[alc->blk_order-1]);
     }
 
     if (hpp || szp) {
@@ -1039,7 +1042,7 @@ info_options(Allctr_t *allctr,
 		     __FILE__, __LINE__);;
 
 	res = NIL;
-	add_2tup(hpp, szp, &res, am.as, flavor_atom[alc->flavor]);
+	add_2tup(hpp, szp, &res, am.as, flavor_atom[alc->blk_order-1]);
     }
 
     return res;
@@ -1057,7 +1060,7 @@ UWord
 erts_aoffalc_test(UWord op, UWord a1, UWord a2)
 {
     switch (op) {
-    case 0x500: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_AOBF;
+    case 0x500: return (UWord) ((AOFFAllctr_t *) a1)->blk_order == FF_AOBF;
     case 0x501: {
 	AOFF_RBTree_t *node = ((AOFFAllctr_t *) a1)->mbc_root; 
 	Uint size = (Uint) a2;
@@ -1072,7 +1075,7 @@ erts_aoffalc_test(UWord op, UWord a1, UWord a2)
     case 0x507:	return (UWord) IS_TREE_NODE((AOFF_RBTree_t *) a1);
     case 0x508: return (UWord) 0; /* IS_BF_ALGO */
     case 0x509: return (UWord) ((AOFF_RBTree_t *) a1)->max_sz;
-    case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_BF;
+    case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->blk_order == FF_BF;
     case 0x50b:	return (UWord) LIST_PREV(a1);
     default:	ASSERT(0); return ~((UWord) 0);
     }
@@ -1132,7 +1135,7 @@ static void print_tree(AOFF_RBTree_t*);
  */
 
 static AOFF_RBTree_t *
-check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint size)
+check_tree(Carrier_t* within_crr, enum AOFFSortOrder order, AOFF_RBTree_t* root, Uint size)
 {
     AOFF_RBTree_t *res = NULL;
     Sint blacks;
@@ -1144,7 +1147,8 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root,
 #ifdef PRINT_TREE
     print_tree(root);
 #endif
-    ASSERT(within_crr || flavor == AOFF_AOFF);
+    ASSERT((within_crr && order >= FF_AOFF) ||
+           (!within_crr && order <= FF_AOFF));
 
     if (!root)
 	return res;
@@ -1202,7 +1206,7 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root,
 	    ASSERT(((char*)x + AOFF_BLK_SZ(x)) <= ((char*)crr + CARRIER_SZ(crr)));
 
 	}
-	if (flavor == AOFF_BF) {
+	if (order == FF_BF) {
 	    AOFF_RBTree_t* y = x;
 	    AOFF_RBTree_t* nxt = LIST_NEXT(y);
 	    ASSERT(IS_TREE_NODE(x));
@@ -1225,13 +1229,13 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root,
 
 	if (x->left) {
 	    ASSERT(x->left->parent == x);
-	    ASSERT(cmp_blocks(flavor, x->left, x) < 0);
+	    ASSERT(cmp_blocks(order, x->left, x) < 0);
 	    ASSERT(x->left->max_sz <= x->max_sz);	    
 	}
 
 	if (x->right) {
 	    ASSERT(x->right->parent == x);
-	    ASSERT(cmp_blocks(flavor, x->right, x) > 0);
+	    ASSERT(cmp_blocks(order, x->right, x) > 0);
 	    ASSERT(x->right->max_sz <= x->max_sz);	    
 	}
 	ASSERT(x->max_sz >= AOFF_BLK_SZ(x));
@@ -1240,7 +1244,7 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root,
 	       || x->max_sz == (x->right ? x->right->max_sz : 0));
 
 	if (size && AOFF_BLK_SZ(x) >= size) {
-	    if (!res || cmp_blocks(flavor, x, res) < 0) {
+	    if (!res || cmp_blocks(order, x, res) < 0) {
 		res = x;
 	    }
 	}
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h
index 7349c6ab19..7864f6c914 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.h
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h
@@ -28,14 +28,15 @@
 
 typedef struct AOFFAllctr_t_ AOFFAllctr_t;
 
-enum AOFF_Flavor {
-    AOFF_AOFF = 0,
-    AOFF_AOBF = 1,
-    AOFF_BF   = 2
+enum AOFFSortOrder {
+    FF_AOFF  = 1,
+    FF_AOBF  = 2,
+    FF_BF    = 3
 };
 
 typedef struct {
-    enum AOFF_Flavor flavor;
+    enum AOFFSortOrder blk_order;
+    enum AOFFSortOrder crr_order;
 } AOFFAllctrInit_t;
 
 #define ERTS_DEFAULT_AOFF_ALLCTR_INIT {0/*dummy*/}
@@ -58,7 +59,8 @@ struct AOFFAllctr_t_ {
     Allctr_t		allctr; /* Has to be first! */
 
     struct AOFF_RBTree_t_* mbc_root;
-    enum AOFF_Flavor flavor;
+    enum AOFFSortOrder blk_order;
+    enum AOFFSortOrder crr_order;
 };
 
 UWord erts_aoffalc_test(UWord, UWord, UWord);
-- 
cgit v1.2.3


From 1b3b4386c220e81395c04215d15c727cfdacb65b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 12 Feb 2018 09:22:19 +0100
Subject: Make sure that the overwrite in bs_context_to_binary is safe

---
 erts/emulator/beam/bs_instrs.tab | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab
index b11903a47b..94e0000c8b 100644
--- a/erts/emulator/beam/bs_instrs.tab
+++ b/erts/emulator/beam/bs_instrs.tab
@@ -990,6 +990,9 @@ ctx_to_bin.execute() {
     Uint hole_size;
     Uint orig = mb->orig;
     ErlSubBin* sb = (ErlSubBin *) boxed_val(context);
+    /* Since we're going to overwrite the match state with the result, an
+     * ErlBinMatchState must be at least as large as an ErlSubBin. */
+    ERTS_CT_ASSERT(sizeof(ErlSubBin) <= sizeof(ErlBinMatchState));
     hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE;
     sb->thing_word = HEADER_SUB_BIN;
     sb->size = BYTE_OFFSET(size);
-- 
cgit v1.2.3


From defd43985282606e841e2bcb29ad7414080d5a80 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 26 Jan 2018 18:42:32 +0100
Subject: erts: Add age order first fit allocator strategies

ageffcaoff: Age First Fit Carrier, Address Order First Fit (within carrier)
ageffcbf  : Age First Fit Carrier, Best Fit (within carrier)
ageffcaobf: Age First Fit Carrier, Address Order Best Fit (within carrier)

Prefer old carriers, the older the better.
---
 erts/emulator/beam/erl_alloc.c             | 17 +++++-
 erts/emulator/beam/erl_ao_firstfit_alloc.c | 94 +++++++++++++++++++-----------
 erts/emulator/beam/erl_ao_firstfit_alloc.h |  1 +
 erts/emulator/test/alloc_SUITE.erl         |  3 +-
 4 files changed, 80 insertions(+), 35 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 4c9fb466eb..67cd3afb25 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -604,7 +604,7 @@ static ERTS_INLINE int
 strategy_support_carrier_migration(struct au_init *auip)
 {
     /*
-     * Currently only aoff, aoffcbf and aoffcaobf support carrier
+     * Currently only aoff* and ageff* support carrier
      * migration, i.e, type AOFIRSTFIT.
      */
     return auip->atype == FIRSTFIT;
@@ -1432,6 +1432,21 @@ handle_au_arg(struct au_init *auip,
 		auip->init.aoff.crr_order = FF_AOFF;
 		auip->init.aoff.blk_order = FF_AOBF;
 	    }
+            else if (strcmp("ageffcaoff", alg) == 0) {
+                auip->atype = FIRSTFIT;
+		auip->init.aoff.crr_order = FF_AGEFF;
+		auip->init.aoff.blk_order = FF_AOFF;
+            }
+            else if (strcmp("ageffcbf", alg) == 0) {
+                auip->atype = FIRSTFIT;
+		auip->init.aoff.crr_order = FF_AGEFF;
+		auip->init.aoff.blk_order = FF_BF;
+            }
+            else if (strcmp("ageffcaobf", alg) == 0) {
+                auip->atype = FIRSTFIT;
+		auip->init.aoff.crr_order = FF_AGEFF;
+		auip->init.aoff.blk_order = FF_AOBF;
+            }
 	    else {
 		bad_value(param, sub_param + 1, alg);
 	    }
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index 8808a4f0a4..ad34fb389a 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -20,7 +20,7 @@
 
 
 /*
- * Description:	An "address order first fit" allocator
+ * Description:	A family of "first fit" allocator strategies
  *              based on a Red-Black (binary search) Tree. The search,
  *              insert, and delete operations are all O(log n) operations
  *              on a Red-Black Tree.
@@ -40,6 +40,10 @@
  *              sorting order. Blocks within the same carrier are sorted
  *              wrt size instead of address. The 'max_sz' field is maintained
  *              in order to dismiss entire carriers with too small blocks. 
+ * Age Order:
+ *      	Carriers are ordered by creation time instead of address.
+ *      	Oldest carrier with a large enough free block is chosen.
+ *      	No age order supported for blocks.
  *
  * Authors: 	Rickard Green/Sverker Eriksson
  */
@@ -53,10 +57,12 @@
 #include "erl_ao_firstfit_alloc.h"
 
 #ifdef DEBUG
+# define IS_DEBUG 1
 #if 0
 #define HARD_DEBUG
 #endif
 #else
+# define IS_DEBUG 0
 #undef HARD_DEBUG
 #endif
 
@@ -121,6 +127,7 @@ typedef struct AOFF_Carrier_t_ AOFF_Carrier_t;
 struct AOFF_Carrier_t_ {
     Carrier_t crr;
     AOFF_RBTree_t rbt_node;     /* My node in the carrier tree */
+    Sint64 birth_time;
     AOFF_RBTree_t* root;        /* Root of my block tree */
 };
 #define RBT_NODE_TO_MBC(PTR) ErtsContainerStruct((PTR), AOFF_Carrier_t, rbt_node)
@@ -179,11 +186,26 @@ static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node,
     else ASSERT(new_max == old_max);
 }
 
+/* Compare nodes for both carrier and block trees */
 static ERTS_INLINE SWord cmp_blocks(enum AOFFSortOrder order,
 				    AOFF_RBTree_t* lhs, AOFF_RBTree_t* rhs)
 {
     ASSERT(lhs != rhs);
-    {
+    if (order == FF_AGEFF) {
+	AOFF_Carrier_t* lc = RBT_NODE_TO_MBC(lhs);
+	AOFF_Carrier_t* rc = RBT_NODE_TO_MBC(rhs);
+	Sint64 diff = lc->birth_time - rc->birth_time;
+ #ifdef ARCH_64
+        if (diff)
+            return diff;
+ #else
+        if (diff < 0)
+            return -1;
+        else if (diff > 0)
+            return 1;
+ #endif
+    }
+    else {
 	ASSERT(order == FF_AOFF || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr));
 	if (order != FF_AOFF) {
 	    SWord diff = (SWord)AOFF_BLK_SZ(lhs) - (SWord)AOFF_BLK_SZ(rhs);
@@ -193,9 +215,11 @@ static ERTS_INLINE SWord cmp_blocks(enum AOFFSortOrder order,
     return (char*)lhs - (char*)rhs;
 }
 
+/* Compare candidate block. Only for block tree */
 static ERTS_INLINE SWord cmp_cand_blk(enum AOFFSortOrder order,
 				      Block_t* cand_blk, AOFF_RBTree_t* rhs)
 {
+    ASSERT(order != FF_AGEFF);
     if (order != FF_AOFF) {
 	if (BLK_TO_MBC(cand_blk) == FBLK_TO_MBC(&rhs->hdr)) {
 	    SWord diff = (SWord)MBC_BLK_SZ(cand_blk) - (SWord)MBC_FBLK_SZ(&rhs->hdr);
@@ -232,10 +256,17 @@ static void init_atoms(void);
 
 static int atoms_initialized = 0;
 
+#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+static erts_atomic64_t birth_time_counter;
+#endif
+
 void
 erts_aoffalc_init(void)
 {
     atoms_initialized = 0;
+#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+    erts_atomic64_init_nob(&birth_time_counter, 0);
+#endif
 }
 
 Allctr_t *
@@ -875,6 +906,15 @@ aoff_get_free_block(Allctr_t *allctr, Uint size,
     return (Block_t *) blk;
 }
 
+static ERTS_INLINE Sint64 get_birth_time(void)
+{
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+    return (Sint64) erts_os_monotonic_time();
+#else
+    return (Sint64) erts_atomic64_inc_read_nob(&birth_time_counter);
+#endif
+}
+
 static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier)
 {
     AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr;
@@ -883,9 +923,9 @@ static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier)
 
     HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0);
 
-    /* Link carrier in address order tree
-     */
     crr->rbt_node.hdr.bhdr = 0;
+    if (alc->crr_order == FF_AGEFF || IS_DEBUG)
+	crr->birth_time = get_birth_time();
     rbt_insert(alc->crr_order, root, &crr->rbt_node);
 
     /* aoff_link_free_block will add free block later */
@@ -916,8 +956,6 @@ static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier)
     ASSERT(!IS_CRR_IN_TREE(crr, *root));
     HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0);   
 
-    /* Link carrier in address order tree
-     */
     rbt_insert(alc->crr_order, root, &crr->rbt_node);
 
     HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0);
@@ -958,17 +996,17 @@ static UWord aoff_largest_fblk_in_mbc(Allctr_t* allctr, Carrier_t* carrier)
  * info_options()
  */
 
+static const char* flavor_str[2][3] = {
+    {"ageffcaoff", "ageffcaobf", "ageffcbf"},
+    {      "aoff",  "aoffcaobf",  "aoffcbf"}
+};
+static Eterm flavor_atoms[2][3];
+
 static struct {
     Eterm as;
-    Eterm aoff;
-    Eterm aoffcaobf;
-    Eterm aoffcbf;
-#ifdef DEBUG
-    Eterm end_of_atoms;
-#endif
 } am;
 
-static void ERTS_INLINE atom_init(Eterm *atom, char *name)
+static void ERTS_INLINE atom_init(Eterm *atom, const char *name)
 {
     *atom = am_atom_put(name, strlen(name));
 }
@@ -977,28 +1015,16 @@ static void ERTS_INLINE atom_init(Eterm *atom, char *name)
 static void
 init_atoms(void)
 {
-#ifdef DEBUG
-    Eterm *atom;
-#endif
+    int i, j;
 
     if (atoms_initialized)
 	return;
 
-#ifdef DEBUG
-    for (atom = (Eterm *) &am; atom <= &am.end_of_atoms; atom++) {
-	*atom = THE_NON_VALUE;
-    }
-#endif
     AM_INIT(as);
-    AM_INIT(aoff);
-    AM_INIT(aoffcaobf);
-    AM_INIT(aoffcbf);
 
-#ifdef DEBUG
-    for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) {
-	ASSERT(*atom != THE_NON_VALUE);
-    }
-#endif
+    for (i = 0; i < 2; i++)
+        for (j = 0; j < 3; j++)
+            atom_init(&flavor_atoms[i][j], flavor_str[i][j]);
 
     atoms_initialized = 1;
 }
@@ -1024,15 +1050,16 @@ info_options(Allctr_t *allctr,
 {
     AOFFAllctr_t* alc = (AOFFAllctr_t*) allctr;
     Eterm res = THE_NON_VALUE;
-    const char* flavor_str[3] = {"aoff", "aoffcaobf", "aoffcbf"};
-    Eterm flavor_atom[3] = {am.aoff, am.aoffcaobf, am.aoffcbf};
+
+    ASSERT(alc->crr_order >= 0 && alc->crr_order <= 1);
+    ASSERT(alc->blk_order >= 1 && alc->blk_order <= 3);
 
     if (print_to_p) {
 	erts_print(*print_to_p,
 		   print_to_arg,
 		   "%sas: %s\n",
 		   prefix,
-		   flavor_str[alc->blk_order-1]);
+		   flavor_str[alc->crr_order][alc->blk_order-1]);
     }
 
     if (hpp || szp) {
@@ -1042,7 +1069,8 @@ info_options(Allctr_t *allctr,
 		     __FILE__, __LINE__);;
 
 	res = NIL;
-	add_2tup(hpp, szp, &res, am.as, flavor_atom[alc->blk_order-1]);
+	add_2tup(hpp, szp, &res, am.as,
+                 flavor_atoms[alc->crr_order][alc->blk_order-1]);
     }
 
     return res;
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h
index 7864f6c914..b5492551e6 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.h
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h
@@ -29,6 +29,7 @@
 typedef struct AOFFAllctr_t_ AOFFAllctr_t;
 
 enum AOFFSortOrder {
+    FF_AGEFF = 0,
     FF_AOFF  = 1,
     FF_AOBF  = 2,
     FF_BF    = 3
diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl
index 3a721095e2..61d2adcbca 100644
--- a/erts/emulator/test/alloc_SUITE.erl
+++ b/erts/emulator/test/alloc_SUITE.erl
@@ -67,7 +67,8 @@ cpool(Cfg) -> drv_case(Cfg).
 migration(Cfg) ->
     case erlang:system_info(smp_support) of
 	true ->
-	    drv_case(Cfg, concurrent, "+MZe true");
+	    drv_case(Cfg, concurrent, "+MZe true"),
+	    drv_case(Cfg, concurrent, "+MZe true +MZas ageffcbf");
 	false ->
 	    {skipped, "No smp"}
     end.
-- 
cgit v1.2.3


From d1e89f8df4be7197fdab36a3e1662183a7dfe6ae Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 25 Jan 2018 19:05:50 +0100
Subject: erts: Add system_flags(erts_alloc,"+M?sbct *")

to change sbct limit in runtime for chosen allocator type.

With great power comes great responsibility.
---
 erts/emulator/beam/atom.names          |  1 +
 erts/emulator/beam/bif.c               |  2 +
 erts/emulator/beam/erl_alloc.c         | 81 +++++++++++++++++++++++++++++++---
 erts/emulator/beam/erl_alloc.h         |  2 +
 erts/emulator/beam/erl_alloc_util.c    | 56 +++++++++++++++--------
 erts/emulator/beam/erl_alloc_util.h    |  4 ++
 erts/emulator/beam/erl_goodfit_alloc.c | 12 +++++
 erts/emulator/test/alloc_SUITE.erl     | 78 ++++++++++++++++++++++++++++++++
 8 files changed, 211 insertions(+), 25 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 2b66bf6f4e..6adb244eb7 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -560,6 +560,7 @@ atom running_procs
 atom runtime
 atom safe
 atom save_calls
+atom sbct
 atom scheduler 
 atom scheduler_id
 atom schedulers_online
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index d9048065c8..3c44573c24 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4706,6 +4706,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	    "scheduled for removal in Erlang/OTP 18. For more\n"
 	    "information see the erlang:system_flag/2 documentation.\n");
 	return erts_bind_schedulers(BIF_P, BIF_ARG_2);
+    } else if (ERTS_IS_ATOM_STR("erts_alloc", BIF_ARG_1)) {
+        return erts_alloc_set_dyn_param(BIF_P, BIF_ARG_2);
     }
     error:
     BIF_ERROR(BIF_P, BADARG);
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 67cd3afb25..625aa98edf 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1272,22 +1272,32 @@ get_bool_value(char *param_end, char** argv, int* ip)
     return -1;
 }
 
+static Uint kb_to_bytes(Sint kb, Uint *bytes)
+{
+    const Uint max = ((~((Uint) 0))/1024) + 1;
+
+    if (kb < 0 || (Uint)kb > max)
+        return 0;
+    if ((Uint)kb == max)
+        *bytes = ~((Uint) 0);
+    else
+        *bytes = ((Uint) kb)*1024;
+    return 1;
+}
+
 static Uint
 get_kb_value(char *param_end, char** argv, int* ip)
 {
     Sint tmp;
-    Uint max = ((~((Uint) 0))/1024) + 1;
+    Uint bytes = 0;
     char *rest;
     char *param = argv[*ip]+1;
     char *value = get_value(param_end, argv, ip);
     errno = 0;
     tmp = (Sint) ErtsStrToSint(value, &rest, 10);
-    if (errno != 0 || rest == value || tmp < 0 || max < ((Uint) tmp))
+    if (errno != 0 || rest == value || !kb_to_bytes(tmp, &bytes))
 	bad_value(param, param_end, value);
-    if (max == (Uint) tmp)
-	return ~((Uint) 0);
-    else
-	return ((Uint) tmp)*1024;
+    return bytes;
 }
 
 static UWord
@@ -3486,6 +3496,65 @@ erts_request_alloc_info(struct process *c_p,
     return 1;
 }
 
+Eterm erts_alloc_set_dyn_param(Process* c_p, Eterm tuple)
+{
+    ErtsAllocatorThrSpec_t *tspec;
+    ErtsAlcType_t ai;
+    Allctr_t* allctr;
+    Eterm* tp;
+    Eterm res;
+
+    if (!is_tuple_arity(tuple, 3))
+        goto badarg;
+
+    tp = tuple_val(tuple);
+
+    /*
+     * Ex: {ets_alloc, sbct, 256000}
+     */
+    if (!is_atom(tp[1]) || !is_atom(tp[2]) || !is_integer(tp[3]))
+        goto badarg;
+
+    for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++)
+        if (erts_is_atom_str(erts_alc_a2ad[ai], tp[1], 0))
+            break;
+
+    if (ai > ERTS_ALC_A_MAX)
+        goto badarg;
+
+    if (!erts_allctrs_info[ai].enabled ||
+        !erts_allctrs_info[ai].alloc_util) {
+        return am_notsup;
+    }
+
+    if (tp[2] == am_sbct) {
+        Uint sbct;
+        int i, ok;
+
+        if (!term_to_Uint(tp[3], &sbct))
+            goto badarg;
+
+        tspec = &erts_allctr_thr_spec[ai];
+        if (tspec->enabled) {
+            ok = 0;
+            for (i = 0; i < tspec->size; i++) {
+                allctr = tspec->allctr[i];
+                ok |= allctr->try_set_dyn_param(allctr, am_sbct, sbct);
+            }
+        }
+        else {
+            allctr = erts_allctrs_info[ai].extra;
+            ok = allctr->try_set_dyn_param(allctr, am_sbct, sbct);
+        }
+        return ok ? am_ok : am_notsup;
+    }
+    return am_notsup;
+
+badarg:
+    ERTS_BIF_PREP_ERROR(res, c_p, EXC_BADARG);
+    return res;
+}
+
 /* 
  * The allocator wrapper prelocking stuff below is about the locking order.
  * It only affects wrappers (erl_mtrace.c and erl_instrument.c) that keep locks
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 56a3b73bf9..33a9fa0bac 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -173,6 +173,8 @@ __decl_noreturn void erts_realloc_n_enomem(ErtsAlcType_t,void*,Uint)
 __decl_noreturn void erts_alc_fatal_error(int,int,ErtsAlcType_t,...)	
      __noreturn;
 
+Eterm erts_alloc_set_dyn_param(struct process*, Eterm);
+
 /* --- DO *NOT* USE THESE DEPRECATED FUNCTIONS ---    Instead use:       */
 void *safe_alloc(Uint)               __deprecated; /* erts_alloc()       */
 void *safe_realloc(void *, Uint)     __deprecated; /* erts_realloc()     */
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 016d85fe2a..0bf95a65be 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -4226,7 +4226,6 @@ static struct {
     Eterm e;
     Eterm t;
     Eterm ramv;
-    Eterm sbct;
 #if HAVE_ERTS_MSEG
     Eterm asbcst;
     Eterm rsbcst;
@@ -4318,7 +4317,6 @@ init_atoms(Allctr_t *allctr)
 	AM_INIT(e);
 	AM_INIT(t);
 	AM_INIT(ramv);
-	AM_INIT(sbct);
 #if HAVE_ERTS_MSEG
 	AM_INIT(asbcst);
 	AM_INIT(rsbcst);
@@ -5011,7 +5009,7 @@ info_options(Allctr_t *allctr,
 		 bld_uint(hpp, szp, allctr->mseg_opt.abs_shrink_th));
 #endif
 	add_2tup(hpp, szp, &res,
-		 am.sbct,
+		 am_sbct,
 		 bld_uint(hpp, szp, allctr->sbc_threshold));
 	add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false);
 	add_2tup(hpp, szp, &res, am.t, (allctr->t ? am_true : am_false));
@@ -5974,6 +5972,37 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra,
 
 #endif
 
+static Uint adjust_sbct(Allctr_t* allctr, Uint sbct)
+{
+#ifndef ARCH_64
+    if (sbct > 0) {
+	Uint max_mbc_block_sz = UNIT_CEILING(sbct - 1 + ABLK_HDR_SZ);
+	if (max_mbc_block_sz + UNIT_FLOOR(allctr->min_block_size - 1) > MBC_ABLK_SZ_MASK
+	    || max_mbc_block_sz < sbct) { /* wrap around */
+	    /*
+	     * By limiting sbc_threshold to (hard limit - min_block_size)
+	     * we avoid having to split off free "residue blocks"
+	     * smaller than min_block_size.
+	     */
+	    max_mbc_block_sz = MBC_ABLK_SZ_MASK - UNIT_FLOOR(allctr->min_block_size - 1);
+	    sbct = max_mbc_block_sz - ABLK_HDR_SZ + 1;
+	}
+    }
+#endif
+    return sbct;
+}
+
+int erts_alcu_try_set_dyn_param(Allctr_t* allctr, Eterm param, Uint value)
+{
+    const Uint MIN_DYN_SBCT = 4000;  /* a lame catastrophe prevention */
+
+    if (param == am_sbct && value >= MIN_DYN_SBCT) {
+        allctr->sbc_threshold = adjust_sbct(allctr, value);
+        return 1;
+    }
+    return 0;
+}
+
 /* ------------------------------------------------------------------------- */
 
 int
@@ -6089,22 +6118,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
     }
 #endif
 
-    allctr->sbc_threshold		= init->sbct;
-#ifndef ARCH_64
-    if (allctr->sbc_threshold > 0) {
-	Uint max_mbc_block_sz = UNIT_CEILING(allctr->sbc_threshold - 1 + ABLK_HDR_SZ); 
-	if (max_mbc_block_sz + UNIT_FLOOR(allctr->min_block_size - 1) > MBC_ABLK_SZ_MASK
-	    || max_mbc_block_sz < allctr->sbc_threshold) { /* wrap around */
-	    /* 
-	     * By limiting sbc_threshold to (hard limit - min_block_size)
-	     * we avoid having to split off free "residue blocks"
-	     * smaller than min_block_size.
-	     */
-	    max_mbc_block_sz = MBC_ABLK_SZ_MASK - UNIT_FLOOR(allctr->min_block_size - 1);
-	    allctr->sbc_threshold = max_mbc_block_sz - ABLK_HDR_SZ + 1;
-	}
-    }
-#endif
+    allctr->sbc_threshold = adjust_sbct(allctr, init->sbct);
 
 #if HAVE_ERTS_MSEG
     if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100)
@@ -6167,6 +6181,9 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
         allctr->sys_realloc = &erts_alcu_sys_realloc;
         allctr->sys_dealloc = &erts_alcu_sys_dealloc;
     }
+
+    allctr->try_set_dyn_param = &erts_alcu_try_set_dyn_param;
+
 #if HAVE_ERTS_MSEG
     if (init->mseg_alloc) {
         ASSERT(init->mseg_realloc && init->mseg_dealloc);
@@ -6181,6 +6198,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
         allctr->mseg_realloc = &erts_alcu_mseg_realloc;
         allctr->mseg_dealloc = &erts_alcu_mseg_dealloc;
     }
+
     /* If a custom carrier alloc function is specified, make sure it's used */
     if (init->mseg_alloc && !init->sys_alloc) {
         allctr->crr_set_flgs = CFLG_FORCE_MSEG;
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index d2cdc3a320..6c0c5ca86a 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -227,6 +227,8 @@ void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint
 void  erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign);
 #endif
 
+int erts_alcu_try_set_dyn_param(Allctr_t*, Eterm param, Uint value);
+
 #endif /* !ERL_ALLOC_UTIL__ */
 
 #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__)
@@ -616,6 +618,8 @@ struct Allctr_t_ {
     void*               (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign);
     void                (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign);
 
+    int                 (*try_set_dyn_param)(Allctr_t*, Eterm param, Uint value);
+
     void		(*init_atoms)		(void);
 
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c
index 50aa41b4d2..a38f6c7daf 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.c
+++ b/erts/emulator/beam/erl_goodfit_alloc.c
@@ -170,6 +170,7 @@ static void		unlink_free_block	(Allctr_t *, Block_t *);
 static void		update_last_aux_mbc	(Allctr_t *, Carrier_t *);
 static Eterm		info_options		(Allctr_t *, char *, fmtfn_t *,
 						 void *, Uint **, Uint *);
+static int        gfalc_try_set_dyn_param(Allctr_t*, Eterm param, Uint value);
 static void		init_atoms		(void);
 
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
@@ -250,6 +251,8 @@ erts_gfalc_start(GFAllctr_t *gfallctr,
     if (!erts_alcu_start(allctr, init))
 	return NULL;
 
+    allctr->try_set_dyn_param = gfalc_try_set_dyn_param;
+
     if (allctr->min_block_size != MIN_BLK_SZ)
 	return NULL;
 
@@ -584,6 +587,15 @@ info_options(Allctr_t *allctr,
     return res;
 }
 
+static int gfalc_try_set_dyn_param(Allctr_t* allctr, Eterm param, Uint value)
+{
+    if (param == am_sbct) {
+        /* Cannot change 'sbct' without rearranging buckets */
+        return 0;
+    }
+    return erts_alcu_try_set_dyn_param(allctr, param, value);
+}
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
  * NOTE:  erts_gfalc_test() is only supposed to be used for testing.         *
  *                                                                           *
diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl
index 61d2adcbca..e4b4465c60 100644
--- a/erts/emulator/test/alloc_SUITE.erl
+++ b/erts/emulator/test/alloc_SUITE.erl
@@ -31,6 +31,7 @@
 	 mseg_clear_cache/1,
 	 erts_mmap/1,
 	 cpool/1,
+         set_dyn_param/1,
 	 migration/1]).
 
 -include_lib("common_test/include/ct.hrl").
@@ -41,6 +42,7 @@ suite() ->
 
 all() -> 
     [basic, coalesce, threads, realloc_copy, bucket_index,
+     set_dyn_param,
      bucket_mask, rbtree, mseg_clear_cache, erts_mmap, cpool, migration].
 
 init_per_testcase(Case, Config) when is_list(Config) ->
@@ -145,6 +147,82 @@ erts_mmap_do(Config, SCO, SCRPM, SCRFSD) ->
     Result.
 
 
+%% Test erlang:system_flag(erts_alloc, ...)
+set_dyn_param(_Config) ->
+    {_, _, _, AlcList} = erlang:system_info(allocator),
+
+    {Enabled, Disabled, Others} =
+        lists:foldl(fun({sys_alloc,_}, {Es, Ds, Os}) ->
+                            {Es, [sys_alloc | Ds], Os};
+
+                       ({AT, Opts}, {Es, Ds, Os}) when is_list(Opts) ->
+                            case lists:keyfind(e, 1, Opts) of
+                                {e, true} ->
+                                    {[AT | Es], Ds, Os};
+                                {e, false} ->
+                                    {Es, [AT | Ds], Os};
+                                false ->
+                                    {Es, Ds, [AT | Os]}
+                            end;
+
+                       (_, Acc) -> Acc
+                    end,
+                    {[], [], []},
+                    AlcList),
+
+    Param = sbct,
+    lists:foreach(fun(AT) -> set_dyn_param_enabled(AT, Param) end,
+                  Enabled),
+
+    lists:foreach(fun(AT) ->
+                          Tpl = {AT, Param, 12345},
+                          io:format("~p\n", [Tpl]),
+                          notsup = erlang:system_flag(erts_alloc, Tpl)
+                  end,
+                  Disabled),
+
+    lists:foreach(fun(AT) ->
+                          Tpl = {AT, Param, 12345},
+                          io:format("~p\n", [Tpl]),
+                          {'EXIT',{badarg,_}} =
+                              (catch erlang:system_flag(erts_alloc, Tpl))
+                  end,
+                  Others),
+    ok.
+
+set_dyn_param_enabled(AT, Param) ->
+    OldVal = get_alc_param(AT, Param),
+
+    Val1 = OldVal div 2,
+    Tuple = {AT, Param, Val1},
+    io:format("~p\n", [Tuple]),
+    ok = erlang:system_flag(erts_alloc, Tuple),
+    Val1 = get_alc_param(AT, Param),
+
+    ok = erlang:system_flag(erts_alloc, {AT, Param, OldVal}),
+    OldVal = get_alc_param(AT, Param),
+    ok.
+
+get_alc_param(AT, Param) ->
+    lists:foldl(fun({instance,_,Istats}, Acc) ->
+                        {options,Opts} = lists:keyfind(options, 1, Istats),
+                        {Param,Val} = lists:keyfind(Param, 1, Opts),
+                        {as,Strategy} = lists:keyfind(as, 1, Opts),
+
+                        case {param_for_strat(Param, Strategy), Acc} of
+                            {false, _} -> Acc;
+                            {true, undefined} -> Val;
+                            {true, _} ->
+                                Val = Acc
+                        end
+                end,
+                undefined,
+                erlang:system_info({allocator, AT})).
+
+param_for_strat(sbct, gf) -> false;
+param_for_strat(_, _) -> true.
+
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%                                                                        %%
 %% Internal functions                                                     %%
-- 
cgit v1.2.3


From 0f5dfb1695108383a903d4f0062b96edc8eaa957 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 13 Feb 2018 14:16:40 +0100
Subject: Use smaller inputs in iovec tests

Huge inputs weren't particularly useful and took forever to run, so
this commit winds it down to a more sane level that still causes
lots of yielding.
---
 erts/emulator/test/iovec_SUITE.erl | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/iovec_SUITE.erl b/erts/emulator/test/iovec_SUITE.erl
index 49dc64b0d2..b65d13aaa8 100644
--- a/erts/emulator/test/iovec_SUITE.erl
+++ b/erts/emulator/test/iovec_SUITE.erl
@@ -98,14 +98,7 @@ cons_bomb(Config) when is_list(Config) ->
     BinBase = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]),
     MixBase = gen_variations([<<12, 45, 78>>, lists:seq(1, 255)]),
 
-    Rounds =
-        case system_mem_size() of
-            Mem when Mem >= (16 bsl 30) -> 32;
-            Mem when Mem >= (3 bsl 30) -> 28;
-            _ -> 20
-        end,
-
-    Variations = gen_variations([IntBase, BinBase, MixBase], Rounds),
+    Variations = gen_variations([IntBase, BinBase, MixBase], 16),
     equivalence_test(fun erlang:iolist_to_iovec/1, Variations).
 
 iolist_to_iovec_idempotence(Config) when is_list(Config) ->
@@ -149,7 +142,7 @@ is_iolist_equal(A, B) ->
 %% few times. The lists only differ by their structure, so their reduction to
 %% a simpler format should yield the same result.
 gen_variations(Base) ->
-    gen_variations(Base, 16).
+    gen_variations(Base, 12).
 gen_variations(Base, N) ->
     [gen_flat_list(Base, N),
      gen_nested_list(Base, N),
@@ -169,8 +162,3 @@ gen_nasty_list_1([Head | Base], Result) when is_list(Head) ->
     gen_nasty_list_1(Base, [[Result], [gen_nasty_list_1(Head, [])]]);
 gen_nasty_list_1([Head | Base], Result) ->
     gen_nasty_list_1(Base, [[Result], [Head]]).
-
-system_mem_size() ->
-    application:ensure_all_started(os_mon),
-    {Tot,_Used,_}  = memsup:get_memory_data(),
-    Tot.
-- 
cgit v1.2.3


From 45f2c76f3f4f83e82d6475536e488d4efa724a91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 15 Feb 2018 13:24:35 +0100
Subject: Test badarg when an improper list tail contains a bitstring

This has always worked but we lacked test coverage for it.
---
 erts/emulator/test/iovec_SUITE.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/iovec_SUITE.erl b/erts/emulator/test/iovec_SUITE.erl
index b65d13aaa8..baf1174a14 100644
--- a/erts/emulator/test/iovec_SUITE.erl
+++ b/erts/emulator/test/iovec_SUITE.erl
@@ -78,7 +78,7 @@ illegal_lists(Config) when is_list(Config) ->
     BitStrs = gen_variations(["gurka", <<1:1>>, "gaffel"]),
     BadInts = gen_variations(["gurka", 890, "gaffel"]),
     Atoms = gen_variations([gurka, "gaffel"]),
-    BadTails = [["test" | 0], ["gurka", gaffel]],
+    BadTails = [["test" | 0], ["gurka" | gaffel], ["gaffel" | <<1:1>>]],
 
     Variations =
         BitStrs ++ BadInts ++ Atoms ++ BadTails,
-- 
cgit v1.2.3


From ca603d7ba3e537670331754539430c704779d5ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 15 Feb 2018 16:34:04 +0100
Subject: badarg on iolist_to_iovec(Bitstring)

When supplied without an enclosing list, bitstrings were silently
truncated to [] instead of badarging.
---
 erts/emulator/beam/erl_io_queue.c  |  5 ++++-
 erts/emulator/test/iovec_SUITE.erl | 13 +++++++++++--
 2 files changed, 15 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c
index 190ba6bbb9..945ee1b6b5 100644
--- a/erts/emulator/beam/erl_io_queue.c
+++ b/erts/emulator/beam/erl_io_queue.c
@@ -1188,7 +1188,10 @@ BIF_RETTYPE iolist_to_iovec_1(BIF_ALIST_1) {
     if (is_nil(BIF_ARG_1)) {
         BIF_RET(NIL);
     } else if (is_binary(BIF_ARG_1)) {
-        if (binary_size(BIF_ARG_1) != 0) {
+        if (binary_bitsize(BIF_ARG_1) != 0) {
+            ASSERT(!(BIF_P->flags & F_DISABLE_GC));
+            BIF_ERROR(BIF_P, BADARG);
+        } else if (binary_size(BIF_ARG_1) != 0) {
             Eterm *hp = HAlloc(BIF_P, 2);
 
             BIF_RET(CONS(hp, BIF_ARG_1, NIL));
diff --git a/erts/emulator/test/iovec_SUITE.erl b/erts/emulator/test/iovec_SUITE.erl
index 49dc64b0d2..abe10f7c20 100644
--- a/erts/emulator/test/iovec_SUITE.erl
+++ b/erts/emulator/test/iovec_SUITE.erl
@@ -25,7 +25,7 @@
 -export([integer_lists/1, binary_lists/1, empty_lists/1, empty_binary_lists/1,
          mixed_lists/1, improper_lists/1, illegal_lists/1, cons_bomb/1,
          sub_binary_lists/1, iolist_to_iovec_idempotence/1,
-         iolist_to_iovec_correctness/1]).
+         iolist_to_iovec_correctness/1, direct_binary_arg/1]).
 
 -include_lib("common_test/include/ct.hrl").
 
@@ -36,7 +36,8 @@ suite() ->
 all() ->
     [integer_lists, binary_lists, empty_lists, empty_binary_lists, mixed_lists,
      sub_binary_lists, illegal_lists, improper_lists, cons_bomb,
-     iolist_to_iovec_idempotence, iolist_to_iovec_correctness].
+     iolist_to_iovec_idempotence, iolist_to_iovec_correctness,
+     direct_binary_arg].
 
 init_per_suite(Config) ->
     Config.
@@ -130,6 +131,14 @@ iolist_to_iovec_correctness(Config) when is_list(Config) ->
     true = is_iolist_equal(Optimized, Variations),
     ok.
 
+direct_binary_arg(Config) when is_list(Config) ->
+    {'EXIT',{badarg, _}} = (catch erlang:iolist_to_iovec(<<1:1>>)),
+
+    [<<1>>] = erlang:iolist_to_iovec(<<1>>),
+    [] = erlang:iolist_to_iovec(<<>>),
+
+    ok.
+
 illegality_test(Fun, Variations) ->
     [{'EXIT',{badarg, _}} = (catch Fun(Variation)) || Variation <- Variations],
     ok.
-- 
cgit v1.2.3


From 13497e652a9b9495fbefd52396620bfccc6d7a7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 13 Feb 2018 12:34:08 +0100
Subject: Handle unaligned binaries in erlang:iolist_to_iovec/1

A binary is a binary as long as its size in bits is evenly divisible
by 8, regardless of whether it has a bit offset or not.
---
 erts/emulator/beam/erl_io_queue.c  | 31 +++++++++++++++++++++++--------
 erts/emulator/test/iovec_SUITE.erl | 21 +++++++++++++++++++--
 2 files changed, 42 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c
index 190ba6bbb9..990c0c1cac 100644
--- a/erts/emulator/beam/erl_io_queue.c
+++ b/erts/emulator/beam/erl_io_queue.c
@@ -801,12 +801,11 @@ static Eterm iol2v_make_sub_bin(iol2v_state_t *state, Eterm bin_term,
     ERTS_GET_REAL_BIN(bin_term, orig_pb_term,
         byte_offset, bit_offset, bit_size);
 
-    (void)bit_offset;
-    (void)bit_size;
+    ASSERT(bit_size == 0);
 
     sb->thing_word = HEADER_SUB_BIN;
+    sb->bitoffs = bit_offset;
     sb->bitsize = 0;
-    sb->bitoffs = 0;
     sb->orig = orig_pb_term;
     sb->is_writable = 0;
 
@@ -984,7 +983,7 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) {
     parent_header = binary_val(parent_binary);
     binary_size = binary_size(bin_term);
 
-    if (bit_offset != 0 || bit_size != 0) {
+    if (bit_size != 0) {
         return 0;
     } else if (binary_size == 0) {
         state->bytereds_spent += 1;
@@ -1026,8 +1025,16 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) {
          * then just copy it into the accumulator. */
         iol2v_expand_acc(state, binary_size);
 
-        sys_memcpy(&(state->acc)->orig_bytes[state->acc_size],
-            binary_data, binary_size);
+        if (ERTS_LIKELY(bit_offset == 0)) {
+            sys_memcpy(&(state->acc)->orig_bytes[state->acc_size],
+                binary_data, binary_size);
+        } else {
+            ASSERT(binary_size <= ERTS_UWORD_MAX / 8);
+
+            erts_copy_bits(binary_data, bit_offset, 1,
+                (byte*)&(state->acc)->orig_bytes[state->acc_size], 0, 1,
+                binary_size * 8);
+        }
 
         state->acc_size += binary_size;
     } else {
@@ -1038,8 +1045,16 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) {
 
         iol2v_expand_acc(state, spill);
 
-        sys_memcpy(&(state->acc)->orig_bytes[state->acc_size],
-            binary_data, spill);
+        if (ERTS_LIKELY(bit_offset == 0)) {
+            sys_memcpy(&(state->acc)->orig_bytes[state->acc_size],
+                binary_data, spill);
+        } else {
+            ASSERT(binary_size <= ERTS_UWORD_MAX / 8);
+
+            erts_copy_bits(binary_data, bit_offset, 1,
+                (byte*)&(state->acc)->orig_bytes[state->acc_size], 0, 1,
+                spill * 8);
+        }
 
         state->acc_size += spill;
 
diff --git a/erts/emulator/test/iovec_SUITE.erl b/erts/emulator/test/iovec_SUITE.erl
index baf1174a14..76adbe4b5e 100644
--- a/erts/emulator/test/iovec_SUITE.erl
+++ b/erts/emulator/test/iovec_SUITE.erl
@@ -25,7 +25,7 @@
 -export([integer_lists/1, binary_lists/1, empty_lists/1, empty_binary_lists/1,
          mixed_lists/1, improper_lists/1, illegal_lists/1, cons_bomb/1,
          sub_binary_lists/1, iolist_to_iovec_idempotence/1,
-         iolist_to_iovec_correctness/1]).
+         iolist_to_iovec_correctness/1, unaligned_sub_binaries/1]).
 
 -include_lib("common_test/include/ct.hrl").
 
@@ -36,7 +36,8 @@ suite() ->
 all() ->
     [integer_lists, binary_lists, empty_lists, empty_binary_lists, mixed_lists,
      sub_binary_lists, illegal_lists, improper_lists, cons_bomb,
-     iolist_to_iovec_idempotence, iolist_to_iovec_correctness].
+     iolist_to_iovec_idempotence, iolist_to_iovec_correctness,
+     unaligned_sub_binaries].
 
 init_per_suite(Config) ->
     Config.
@@ -123,6 +124,15 @@ iolist_to_iovec_correctness(Config) when is_list(Config) ->
     true = is_iolist_equal(Optimized, Variations),
     ok.
 
+unaligned_sub_binaries(Config) when is_list(Config) ->
+    UnalignedBins = [gen_unaligned_binary(I) || I <- lists:seq(32, 4 bsl 10, 512)],
+    UnalignedVariations = gen_variations(UnalignedBins),
+
+    Optimized = erlang:iolist_to_iovec(UnalignedVariations),
+
+    true = is_iolist_equal(Optimized, UnalignedVariations),
+    ok.
+
 illegality_test(Fun, Variations) ->
     [{'EXIT',{badarg, _}} = (catch Fun(Variation)) || Variation <- Variations],
     ok.
@@ -138,6 +148,13 @@ equivalence_test(Fun, [Head | _] = Variations) ->
 is_iolist_equal(A, B) ->
     iolist_to_binary(A) =:= iolist_to_binary(B).
 
+gen_unaligned_binary(Size) ->
+    Bin0 = << <<I>> || I <- lists:seq(1, Size) >>,
+    <<0:3,Bin:Size/binary,31:5>> = id(<<0:3,Bin0/binary,31:5>>),
+    Bin.
+
+id(I) -> I.
+
 %% Generates a bunch of lists whose contents will be equal to Base repeated a
 %% few times. The lists only differ by their structure, so their reduction to
 %% a simpler format should yield the same result.
-- 
cgit v1.2.3


From c47c8ed7da8c222c5d3b99d0f109fe126a86c632 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 13 Feb 2018 16:12:30 +0100
Subject: Handle unaligned binaries in enif_inspect_iovec

---
 erts/emulator/beam/erl_nif.c     | 124 ++++++++++++++++++++++-----------------
 erts/emulator/test/nif_SUITE.erl |  30 +++++++++-
 2 files changed, 99 insertions(+), 55 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index f67b67325d..b96db5d054 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3396,8 +3396,8 @@ typedef struct {
     Eterm sublist_start;
     Eterm sublist_end;
 
-    UWord offheap_size;
-    UWord onheap_size;
+    UWord referenced_size;
+    UWord copied_size;
 
     UWord iovec_len;
 } iovec_slice_t;
@@ -3407,16 +3407,16 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul
 
     result->sublist_start = list;
     result->sublist_length = 0;
-    result->offheap_size = 0;
-    result->onheap_size = 0;
+    result->referenced_size = 0;
+    result->copied_size = 0;
     result->iovec_len = 0;
 
     lookahead = result->sublist_start;
 
     while (is_list(lookahead)) {
-        Eterm *binary_header, binary;
+        UWord byte_size;
+        Eterm binary;
         Eterm *cell;
-        UWord size;
 
         cell = list_val(lookahead);
         binary = CAR(cell);
@@ -3425,35 +3425,36 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul
             return 0;
         }
 
-        size = binary_size(binary);
-        binary_header = binary_val(binary);
+        byte_size = binary_size(binary);
 
-        if (size > 0) {
-            /* If we're a sub-binary we'll need to check our underlying binary
-             * to determine whether we're on-heap or not. */
-            if (thing_subtag(*binary_header) == SUB_BINARY_SUBTAG) {
-                ErlSubBin *sb = (ErlSubBin*)binary_header;
+        if (byte_size > 0) {
+            int bit_offset, bit_size;
+            Eterm parent_binary;
+            UWord byte_offset;
 
-                /* Reject bitstrings */
-                if((sb->bitoffs + sb->bitsize) > 0) {
-                    return 0;
-                }
+            int requires_copying;
 
-                ASSERT(size <= binary_size(sb->orig));
-                binary_header = binary_val(sb->orig);
+            ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset,
+                bit_offset, bit_size);
+
+            (void)byte_offset;
+
+            if (bit_size != 0) {
+                return 0;
             }
 
-            if (thing_subtag(*binary_header) == HEAP_BINARY_SUBTAG) {
-                ASSERT(size <= ERL_ONHEAP_BIN_LIMIT);
+            /* If we're unaligned or an on-heap binary we'll need to copy
+             * ourselves over to a temporary buffer. */
+            requires_copying = (bit_offset != 0) ||
+                thing_subtag(*binary_val(parent_binary)) == HEAP_BINARY_SUBTAG;
 
-                result->iovec_len += 1;
-                result->onheap_size += size;
+            if (requires_copying) {
+                result->copied_size += byte_size;
             } else {
-                ASSERT(thing_subtag(*binary_header) == REFC_BINARY_SUBTAG);
-
-                result->iovec_len += 1 + size / MAX_SYSIOVEC_IOVLEN;
-                result->offheap_size += size;
+                result->referenced_size += byte_size;
             }
+
+            result->iovec_len += 1 + byte_size / MAX_SYSIOVEC_IOVLEN;
         }
 
         result->sublist_length += 1;
@@ -3473,7 +3474,9 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul
     return 1;
 }
 
-static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) {
+static void marshal_iovec_binary(Eterm binary, ErlNifBinary *copy_buffer,
+        UWord *copy_offset, ErlNifBinary *result) {
+
     Eterm *parent_header;
     Eterm parent_binary;
 
@@ -3484,6 +3487,8 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) {
 
     ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset, bit_offset, bit_size);
 
+    ASSERT(bit_size == 0);
+
     parent_header = binary_val(parent_binary);
 
     result->size = binary_size(binary);
@@ -3510,24 +3515,50 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) {
         result->data = &((unsigned char*)&hb->data)[byte_offset];
         result->ref_bin = NULL;
     }
+
+    /* If this isn't an *aligned* refc binary, copy its contents to the buffer
+     * and reference that instead. */
+
+    if (result->ref_bin == NULL || bit_offset != 0) {
+        ASSERT(result->size <= (copy_buffer->size - *copy_offset));
+
+        if (bit_offset == 0) {
+            sys_memcpy(&copy_buffer->data[*copy_offset],
+                result->data, result->size);
+        } else {
+            erts_copy_bits(result->data, bit_offset, 1,
+                (byte*)&copy_buffer->data[*copy_offset], 0, 1,
+                result->size * 8);
+        }
+
+        result->data = &copy_buffer->data[*copy_offset];
+        result->ref_bin = copy_buffer->ref_bin;
+
+        *copy_offset += result->size;
+    }
 }
 
 static int fill_iovec_with_slice(ErlNifEnv *env,
                                  iovec_slice_t *slice,
                                  ErlNifIOVec *iovec) {
-    UWord onheap_offset, iovec_idx;
-    ErlNifBinary onheap_data;
+    UWord copy_offset, iovec_idx;
+    ErlNifBinary copy_buffer;
     Eterm sublist_iterator;
 
-    /* Set up a common refc binary for all on-heap binaries. */
-    if (slice->onheap_size > 0) {
-        if (!enif_alloc_binary(slice->onheap_size, &onheap_data)) {
+    /* Set up a common refc binary for all on-heap and unaligned binaries. */
+    if (slice->copied_size > 0) {
+        if (!enif_alloc_binary(slice->copied_size, &copy_buffer)) {
             return 0;
         }
+    } else {
+#ifdef DEBUG
+        copy_buffer.data = NULL;
+        copy_buffer.size = 0;
+#endif
     }
 
     sublist_iterator = slice->sublist_start;
-    onheap_offset = 0;
+    copy_offset = 0;
     iovec_idx = 0;
 
     while (sublist_iterator != slice->sublist_end) {
@@ -3535,20 +3566,7 @@ static int fill_iovec_with_slice(ErlNifEnv *env,
         Eterm *cell;
 
         cell = list_val(sublist_iterator);
-        inspect_raw_binary_data(CAR(cell), &raw_data);
-
-        /* If this isn't a refc binary, copy its contents to the onheap buffer
-         * and reference that instead. */
-        if (raw_data.size > 0 && raw_data.ref_bin == NULL) {
-            ASSERT(onheap_offset < onheap_data.size);
-            ASSERT(slice->onheap_size > 0);
-
-            sys_memcpy(&onheap_data.data[onheap_offset],
-                       raw_data.data, raw_data.size);
-
-            raw_data.data = &onheap_data.data[onheap_offset];
-            raw_data.ref_bin = onheap_data.ref_bin;
-        }
+        marshal_iovec_binary(CAR(cell), &copy_buffer, &copy_offset, &raw_data);
 
         while (raw_data.size > 0) {
             UWord chunk_len = MIN(raw_data.size, MAX_SYSIOVEC_IOVLEN);
@@ -3579,16 +3597,16 @@ static int fill_iovec_with_slice(ErlNifEnv *env,
             erts_refc_inc(&refc_binary->intern.refc, 1);
         }
 
-        if (slice->onheap_size > 0) {
+        if (slice->copied_size > 0) {
             /* Transfer ownership to the iovec; we've taken references to it in
              * the above loop. */
-            enif_release_binary(&onheap_data);
+            enif_release_binary(&copy_buffer);
         }
     } else {
-        if (slice->onheap_size > 0) {
+        if (slice->copied_size > 0) {
             /* Attach the binary to our environment and let the GC take care of
              * it after returning. */
-            enif_make_binary(env, &onheap_data);
+            enif_make_binary(env, &copy_buffer);
         }
     }
 
@@ -3635,7 +3653,7 @@ static int create_iovec_from_slice(ErlNifEnv *env,
         iovec->flags = 0;
     }
 
-    iovec->size = slice->offheap_size + slice->onheap_size;
+    iovec->size = slice->referenced_size + slice->copied_size;
     iovec->iovcnt = slice->iovec_len;
 
     if(!fill_iovec_with_slice(env, slice, iovec)) {
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 4811244b98..ac2b136a38 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -2983,8 +2983,24 @@ nif_ioq(Config) ->
          {enqv,   a, 5, iolist_size(nif_ioq_payload(5)) - 1},
          {peek,   a},
 
-         %% Test to enqueue a bunch of refc binaries
+         %% Ensure that enqueued refc binaries are intact after a roundtrip.
+         %%
+         %% This test and the ones immediately following it does not go through
+         %% erlang:iolist_to_iovec/1
          {enqv,   a, [nif_ioq_payload(refcbin) || _ <- lists:seq(1,20)], 0},
+         {peek,   a},
+
+         %% ... heap binaries
+         {enqv,   a, [nif_ioq_payload(heapbin) || _ <- lists:seq(1,20)], 0},
+         {peek,   a},
+
+         %% ... plain sub-binaries
+         {enqv,   a, [nif_ioq_payload(subbin) || _ <- lists:seq(1,20)], 0},
+         {peek,   a},
+
+         %% ... unaligned binaries
+         {enqv,   a, [nif_ioq_payload(unaligned_bin) || _ <- lists:seq(1,20)], 0},
+         {peek,   a},
 
          %% Enq stuff to destroy with data in queue
          {enqv,   a, 2, 100},
@@ -3120,13 +3136,16 @@ nif_ioq_payload(N) when is_integer(N) ->
     Tail = if N > 3 -> nif_ioq_payload(N-3); true -> [] end,
     Head = element(1, lists:split(N,[nif_ioq_payload(subbin),
                                      nif_ioq_payload(heapbin),
-                                     nif_ioq_payload(refcbin) | Tail])),
+                                     nif_ioq_payload(refcbin),
+                                     nif_ioq_payload(unaligned_bin) | Tail])),
     erlang:iolist_to_iovec(Head);
 nif_ioq_payload(subbin) ->
     Bin = nif_ioq_payload(refcbin),
     Sz = size(Bin) - 1,
     <<_:8,SubBin:Sz/binary,_/bits>> = Bin,
     SubBin;
+nif_ioq_payload(unaligned_bin) ->
+    make_unaligned_binary(<< <<I>> || I <- lists:seq(1, 255) >>);
 nif_ioq_payload(heapbin) ->
     <<"a literal heap binary">>;
 nif_ioq_payload(refcbin) ->
@@ -3134,6 +3153,13 @@ nif_ioq_payload(refcbin) ->
 nif_ioq_payload(Else) ->
     Else.
 
+make_unaligned_binary(Bin0) ->
+    Size = byte_size(Bin0),
+    <<0:3,Bin:Size/binary,31:5>> = id(<<0:3,Bin0/binary,31:5>>),
+    Bin.
+
+id(I) -> I.
+
 %% The NIFs:
 lib_version() -> undefined.
 call_history() -> ?nif_stub.
-- 
cgit v1.2.3


From 443fbb24248516523512c9bc591252eafafd1311 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 15 Feb 2018 17:06:36 +0100
Subject: badarg on iolist_to_binary(Bitstring)

When supplied without an enclosing list, bitstrings were returned
as-is instead of badarging.
---
 erts/emulator/beam/binary.c         | 5 ++++-
 erts/emulator/test/binary_SUITE.erl | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index ca3e48e205..cf7dcb13d5 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -964,7 +964,10 @@ HIPE_WRAPPER_BIF_DISABLE_GC(iolist_to_binary, 1)
 BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1)
 {
     if (is_binary(BIF_ARG_1)) {
-	BIF_RET(BIF_ARG_1);
+        if (binary_bitsize(BIF_ARG_1) == 0) {
+            BIF_RET(BIF_ARG_1);
+        }
+        BIF_ERROR(BIF_P, BADARG);
     }
     return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_iolist_to_binary_1]);
 }
diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index 61536bacd7..374f91e487 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -257,6 +257,7 @@ test_deep_bitstr(List) ->
     {Bin,bitstring_to_list(Bin)}.
 
 bad_list_to_binary(Config) when is_list(Config) ->
+    test_bad_bin(<<1:1>>),
     test_bad_bin(atom),
     test_bad_bin(42),
     test_bad_bin([1|2]),
-- 
cgit v1.2.3


From cff8dce026edfa86d0c96aebfbb8af53fee238a6 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 16 Feb 2018 09:55:08 +0100
Subject: erts: Ignore SIGTERM in erl_child_setup

---
 erts/emulator/sys/unix/erl_child_setup.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/erl_child_setup.c b/erts/emulator/sys/unix/erl_child_setup.c
index 69fc6c2879..57973b10d7 100644
--- a/erts/emulator/sys/unix/erl_child_setup.c
+++ b/erts/emulator/sys/unix/erl_child_setup.c
@@ -437,6 +437,21 @@ main(int argc, char *argv[])
         exit(1);
     }
 
+    /* Ignore SIGTERM.
+       Some container environments send SIGTERM to all processes
+       when terminating. We don't want erl_child_setup to terminate
+       in these cases as that will prevent beam from properly
+       cleaning up.
+    */
+    sa.sa_handler = SIG_IGN;
+    sigemptyset(&sa.sa_mask);
+    sa.sa_flags = 0;
+
+    if (sigaction(SIGTERM, &sa, 0) == -1) {
+        perror(NULL);
+        exit(1);
+    }
+
     forker_hash_init();
 
     SET_CLOEXEC(uds_fd);
-- 
cgit v1.2.3


From 2d3b4d0945190589098d42375b4738b004e7373a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 15 Feb 2018 16:41:28 +0100
Subject: erts: Add binary vheap sizes to crash dump

---
 erts/emulator/beam/break.c | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index e4c2801ea2..127c14e21b 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -336,6 +336,12 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p)
     erts_print(to, to_arg, "Heap unused: %bpu\n", (p->hend - p->htop));
     erts_print(to, to_arg, "OldHeap unused: %bpu\n",
 	       (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HTOP(p)) );
+    erts_print(to, to_arg, "BinVHeap: %b64u\n", p->off_heap.overhead);
+    erts_print(to, to_arg, "OldBinVHeap: %b64u\n", BIN_OLD_VHEAP(p));
+    erts_print(to, to_arg, "BinVHeap unused: %b64u\n",
+               BIN_VHEAP_SZ(p) - p->off_heap.overhead);
+    erts_print(to, to_arg, "OldBinVHeap unused: %b64u\n",
+               BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p));
     erts_print(to, to_arg, "Memory: %beu\n", erts_process_memory(p, !0));
 
     if (garbing) {
-- 
cgit v1.2.3


From 283e669d971cdd493226d33ae89e507acf84ea7b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 15 Feb 2018 18:50:14 +0100
Subject: erts: Remove unused args to collect_live_heap_frags

---
 erts/emulator/beam/erl_gc.c | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 6a87136463..015ff0a106 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -142,7 +142,7 @@ static Eterm* sweep_literal_area(Eterm* n_hp, Eterm* n_htop,
 static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
 					 char* src, Uint src_size);
 static Eterm* collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end,
-				      Eterm* heap, Eterm* htop, Eterm* objv, int nobj);
+				      Eterm* htop);
 static int adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj);
 static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj);
 static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj);
@@ -1508,8 +1508,7 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
 	 * Move heap frags that we know are completely live
 	 * directly into the new young heap generation.
 	 */
-	n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop,
-					 objv, nobj);
+	n_htop = collect_live_heap_frags(p, live_hf_end, n_htop);
     }
 
     n = setup_rootset(p, objv, nobj, &rootset);
@@ -1762,8 +1761,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
 	 * Move heap frags that we know are completely live
 	 * directly into the heap.
 	 */
-	n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop,
-					 objv, nobj);
+	n_htop = collect_live_heap_frags(p, live_hf_end, n_htop);
     }
 
     n_htop = full_sweep_heaps(p, 0, n_heap, n_htop, oh, oh_size, objv, nobj);
@@ -2337,9 +2335,7 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size)
  */
 
 static Eterm*
-collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end,
-			Eterm* n_hstart, Eterm* n_htop,
-			Eterm* objv, int nobj)
+collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, Eterm* n_htop)
 {
     ErlHeapFragment* qb;
     char* frag_begin;
-- 
cgit v1.2.3


From fac2f04b8e294b137d8bc4292fcaefb597faf9e8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 15 Feb 2018 20:31:17 +0100
Subject: erts: Include failing garbing process in crash dump

Exclude garbing processes, EXCEPT if run by crash dumping thread
in which case we assume the heap is healthy
without any move markers yet/left.

Switched order between (allocating) setup_rootset()
and (move marking) collect_live_heap_frags().
---
 erts/emulator/beam/erl_gc.c           | 38 ++++++++++++++++++++++++-----------
 erts/emulator/beam/erl_process_dump.c | 13 ++++++++++--
 2 files changed, 37 insertions(+), 14 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 015ff0a106..154b9b8dac 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -116,6 +116,7 @@ typedef struct {
 static Uint setup_rootset(Process*, Eterm*, int, Rootset*);
 static void cleanup_rootset(Rootset *rootset);
 static Eterm *full_sweep_heaps(Process *p,
+                               ErlHeapFragment *live_hf_end,
 			       int hibernate,
 			       Eterm *n_heap, Eterm* n_htop,
 			       char *oh, Uint oh_size,
@@ -939,6 +940,7 @@ garbage_collect_hibernate(Process* p, int check_long_gc)
     htop = heap;
 
     htop = full_sweep_heaps(p,
+                            ERTS_INVALID_HFRAG_PTR,
 			    1,
 			    heap,
 			    htop,
@@ -1503,6 +1505,14 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
     n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP,
 					       sizeof(Eterm)*new_sz);
 
+    n = setup_rootset(p, objv, nobj, &rootset);
+    roots = rootset.roots;
+
+    /*
+     * All allocations done. Start defile heap with move markers.
+     * A crash dump due to allocation failure above will see a healthy heap.
+     */
+
     if (live_hf_end != ERTS_INVALID_HFRAG_PTR) {
 	/*
 	 * Move heap frags that we know are completely live
@@ -1511,9 +1521,6 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
 	n_htop = collect_live_heap_frags(p, live_hf_end, n_htop);
     }
 
-    n = setup_rootset(p, objv, nobj, &rootset);
-    roots = rootset.roots;
-
     GENSWEEP_NSTACK(p, old_htop, n_htop);
     while (n--) {
         Eterm* g_ptr = roots->v;
@@ -1756,15 +1763,8 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
     n_htop = n_heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP,
 						sizeof(Eterm)*new_sz);
 
-    if (live_hf_end != ERTS_INVALID_HFRAG_PTR) {
-	/*
-	 * Move heap frags that we know are completely live
-	 * directly into the heap.
-	 */
-	n_htop = collect_live_heap_frags(p, live_hf_end, n_htop);
-    }
-
-    n_htop = full_sweep_heaps(p, 0, n_heap, n_htop, oh, oh_size, objv, nobj);
+    n_htop = full_sweep_heaps(p, live_hf_end, 0, n_heap, n_htop, oh, oh_size,
+                              objv, nobj);
 
     /* Move the stack to the end of the heap */
     stk_sz = HEAP_END(p) - p->stop;
@@ -1811,6 +1811,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
 
 static Eterm *
 full_sweep_heaps(Process *p,
+                 ErlHeapFragment *live_hf_end,
 		 int hibernate,
 		 Eterm *n_heap, Eterm* n_htop,
 		 char *oh, Uint oh_size,
@@ -1827,6 +1828,19 @@ full_sweep_heaps(Process *p,
 
     n = setup_rootset(p, objv, nobj, &rootset);
 
+    /*
+     * All allocations done. Start defile heap with move markers.
+     * A crash dump due to allocation failure above will see a healthy heap.
+     */
+
+    if (live_hf_end != ERTS_INVALID_HFRAG_PTR) {
+        /*
+         * Move heap frags that we know are completely live
+         * directly into the heap.
+         */
+        n_htop = collect_live_heap_frags(p, live_hf_end, n_htop);
+    }
+
 #ifdef HIPE
     if (hibernate)
 	hipe_empty_nstack(p);
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index e0b795fbf3..c30a1e48ec 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -78,8 +78,17 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg)
 	Process *p = erts_pix2proc(i);
 	if (p && p->i != ENULL) {
 	    erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
-	    if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_GC)))
-		dump_process_info(to, to_arg, p);
+	    if (state & ERTS_PSFLG_EXITING)
+                continue;
+            if (state & ERTS_PSFLG_GC) {
+                ErtsSchedulerData *sdp = erts_get_scheduler_data();
+                if (!sdp || p != sdp->current_process)
+                    continue;
+
+                /* We want to dump the garbing process that caused the dump */
+            }
+
+            dump_process_info(to, to_arg, p);
        }
     }
 
-- 
cgit v1.2.3


From 43edeef1d224ace49dac13d6a5778cd0b50f1d25 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 15 Feb 2018 20:32:44 +0100
Subject: erts: Cleanup dump_process_info()

by testing F_SENSITIVE only once.
---
 erts/emulator/beam/erl_process_dump.c | 59 +++++++++++++++++------------------
 1 file changed, 29 insertions(+), 30 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index c30a1e48ec..00fe30dfcb 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -144,9 +144,12 @@ dump_process_info(fmtfn_t to, void *to_arg, Process *p)
     ErtsMessage* mp;
     int yreg = -1;
 
+    if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE)
+        return;
+
     ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
 
-    if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0 && p->msg.first) {
+    if (p->msg.first) {
 	erts_print(to, to_arg, "=proc_messages:%T\n", p->common.id);
 	for (mp = p->msg.first; mp != NULL; mp = mp->next) {
 	    Eterm mesg = ERL_MESSAGE_TERM(mp);
@@ -161,38 +164,34 @@ dump_process_info(fmtfn_t to, void *to_arg, Process *p)
 	}
     }
 
-    if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) {
-	if (p->dictionary) {
-	    erts_print(to, to_arg, "=proc_dictionary:%T\n", p->common.id);
-	    erts_deep_dictionary_dump(to, to_arg,
-				      p->dictionary, dump_element_nl);
-	}
+    if (p->dictionary) {
+        erts_print(to, to_arg, "=proc_dictionary:%T\n", p->common.id);
+        erts_deep_dictionary_dump(to, to_arg,
+                                  p->dictionary, dump_element_nl);
     }
 
-    if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) {
-	erts_print(to, to_arg, "=proc_stack:%T\n", p->common.id);
-	for (sp = p->stop; sp < STACK_START(p); sp++) {
-	    yreg = stack_element_dump(to, to_arg, sp, yreg);
-	}
+    erts_print(to, to_arg, "=proc_stack:%T\n", p->common.id);
+    for (sp = p->stop; sp < STACK_START(p); sp++) {
+        yreg = stack_element_dump(to, to_arg, sp, yreg);
+    }
 
-	erts_print(to, to_arg, "=proc_heap:%T\n", p->common.id);
-	for (sp = p->stop; sp < STACK_START(p); sp++) {
-	    Eterm term = *sp;
-	    
-	    if (!is_catch(term) && !is_CP(term)) {
-		heap_dump(to, to_arg, term);
-	    }
-	}
-	for (mp = p->msg.first; mp != NULL; mp = mp->next) {
-	    Eterm mesg = ERL_MESSAGE_TERM(mp);
-	    if (is_value(mesg))
-		heap_dump(to, to_arg, mesg);
-	    mesg = ERL_MESSAGE_TOKEN(mp);
-	    heap_dump(to, to_arg, mesg);
-	}
-	if (p->dictionary) {
-	    erts_deep_dictionary_dump(to, to_arg, p->dictionary, heap_dump);
-	}
+    erts_print(to, to_arg, "=proc_heap:%T\n", p->common.id);
+    for (sp = p->stop; sp < STACK_START(p); sp++) {
+        Eterm term = *sp;
+
+        if (!is_catch(term) && !is_CP(term)) {
+            heap_dump(to, to_arg, term);
+        }
+    }
+    for (mp = p->msg.first; mp != NULL; mp = mp->next) {
+        Eterm mesg = ERL_MESSAGE_TERM(mp);
+        if (is_value(mesg))
+            heap_dump(to, to_arg, mesg);
+        mesg = ERL_MESSAGE_TOKEN(mp);
+        heap_dump(to, to_arg, mesg);
+    }
+    if (p->dictionary) {
+        erts_deep_dictionary_dump(to, to_arg, p->dictionary, heap_dump);
     }
 }
 
-- 
cgit v1.2.3


From cff79c76c3fb574addb5c9364f7c4c7d48a75907 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 19 Feb 2018 11:59:12 +0100
Subject: erts,observer: Add more port info to crash dump

---
 erts/emulator/beam/erl_bif_port.c | 21 +++++++++++++++++++++
 erts/emulator/beam/erl_port.h     |  1 +
 erts/emulator/beam/io.c           |  8 ++++++++
 3 files changed, 30 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index ff03151619..d05507118e 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -647,6 +647,27 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1)
     BIF_RET(res);
 }
 
+Eterm erts_port_data_read(Port* prt)
+{
+    Eterm res;
+    erts_aint_t data;
+
+    data = erts_smp_atomic_read_ddrb(&prt->data);
+    if (data == (erts_aint_t)NULL)
+        return am_undefined;  /* Port terminated by racing thread */
+
+    if ((data & 0x3) != 0) {
+	res = (Eterm) (UWord) data;
+	ASSERT(is_immed(res));
+    }
+    else {
+	ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data;
+	res = pdhp->data;
+    }
+    return res;
+}
+
+
 /* 
  * Open a port. Most of the work is not done here but rather in
  * the file io.c.
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index b64de624dd..c30bbbca06 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -187,6 +187,7 @@ void erts_init_port_data(Port *);
 void erts_cleanup_port_data(Port *);
 Uint erts_port_data_size(Port *);
 ErlOffHeap *erts_port_data_offheap(Port *);
+Eterm erts_port_data_read(Port* prt);
 
 #define ERTS_PORT_GET_CONNECTED(PRT) \
     ((Eterm) erts_atomic_read_nob(&(PRT)->connected))
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 3a5ddde5f4..c6444fd440 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -5199,6 +5199,14 @@ print_port_info(Port *p, fmtfn_t to, void *arg)
     } else {
 	erts_print(to, arg, "Port controls linked-in driver: %s\n",p->name);
     }
+    erts_print(to, arg, "Input: %beu\n", p->bytes_in);
+    erts_print(to, arg, "Output: %beu\n", p->bytes_out);
+    erts_print(to, arg, "Queue: %beu\n", erts_ioq_size(&p->ioq));
+    {
+        Eterm port_data = erts_port_data_read(p);
+        if (port_data != am_undefined)
+            erts_print(to, arg, "Port Data: %T\n", port_data);
+    }
 }
 
 void
-- 
cgit v1.2.3


From 083973ccb86d1cf826821832285f1443200f6668 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 20 Feb 2018 20:45:52 +0100
Subject: erts,observer: Add dirty schedulers to crash dump

---
 erts/emulator/beam/break.c       | 17 +++++++++++
 erts/emulator/beam/erl_process.c | 63 ++++++++++++++++++++++++++++++++--------
 erts/emulator/beam/erl_process.h |  1 +
 3 files changed, 69 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 127c14e21b..5ace997344 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -911,6 +911,23 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
             erts_print_scheduler_info(to, to_arg, ERTS_SCHEDULER_IX(i)),
             erts_cbprintf(to, to_arg, "** crashed **\n"));
     }
+#ifdef ERTS_DIRTY_SCHEDULERS
+    for (i = 0; i < erts_no_dirty_cpu_schedulers; i++) {
+        ERTS_SYS_TRY_CATCH(
+            erts_print_scheduler_info(to, to_arg, ERTS_DIRTY_CPU_SCHEDULER_IX(i)),
+            erts_cbprintf(to, to_arg, "** crashed **\n"));
+    }
+    erts_cbprintf(to, to_arg, "=dirty_cpu_run_queue\n");
+    erts_print_run_queue_info(to, to_arg, ERTS_DIRTY_CPU_RUNQ);
+
+    for (i = 0; i < erts_no_dirty_io_schedulers; i++) {
+        ERTS_SYS_TRY_CATCH(
+            erts_print_scheduler_info(to, to_arg, ERTS_DIRTY_IO_SCHEDULER_IX(i)),
+            erts_cbprintf(to, to_arg, "** crashed **\n"));
+    }
+    erts_cbprintf(to, to_arg, "=dirty_io_run_queue\n");
+    erts_print_run_queue_info(to, to_arg, ERTS_DIRTY_IO_RUNQ);
+#endif /* ERTS_DIRTY_SCHEDULERS */
 #endif
 
 #ifdef ERTS_SMP
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 63e9275ac1..5fcb100fae 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -14259,16 +14259,35 @@ stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg)
     return yreg;
 }
 
+static void print_current_process_info(fmtfn_t, void *to_arg, ErtsSchedulerData*);
+
 /*
  * Print scheduler information
  */
 void
-erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
+erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp)
+{
     int i;
     erts_aint32_t flg;
-    Process *p;
 
-    erts_print(to, to_arg, "=scheduler:%u\n", esdp->no);
+    switch (esdp->type) {
+    case ERTS_SCHED_NORMAL:
+        erts_print(to, to_arg, "=scheduler:%u\n", esdp->no);
+        break;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    case ERTS_SCHED_DIRTY_CPU:
+        erts_print(to, to_arg, "=dirty_cpu_scheduler:%u\n",
+                   (esdp->dirty_no + erts_no_schedulers));
+        break;
+    case ERTS_SCHED_DIRTY_IO:
+        erts_print(to, to_arg, "=dirty_io_scheduler:%u\n",
+                   (esdp->dirty_no + erts_no_schedulers + erts_no_dirty_cpu_schedulers));
+        break;
+#endif
+    default:
+        erts_print(to, to_arg, "=unknown_scheduler_type:%u\n", esdp->type);
+        break;
+    }
 
 #ifdef ERTS_SMP
     flg = erts_smp_atomic32_read_dirty(&esdp->ssi->flags);
@@ -14316,10 +14335,24 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
     }
     erts_print(to, to_arg, "\n");
 
-    erts_print(to, to_arg, "Current Port: ");
-    if (esdp->current_port)
-        erts_print(to, to_arg, "%T", esdp->current_port->common.id);
-    erts_print(to, to_arg, "\n");
+    if (esdp->type == ERTS_SCHED_NORMAL) {
+        erts_print(to, to_arg, "Current Port: ");
+        if (esdp->current_port)
+            erts_print(to, to_arg, "%T", esdp->current_port->common.id);
+        erts_print(to, to_arg, "\n");
+
+        erts_print_run_queue_info(to, to_arg, esdp->run_queue);
+    }
+
+    /* This *MUST* to be the last information in scheduler block */
+    print_current_process_info(to, to_arg, esdp);
+}
+
+void erts_print_run_queue_info(fmtfn_t to, void *to_arg,
+                               ErtsRunQueue *run_queue)
+{
+    erts_aint32_t flg;
+    int i;
 
     for (i = 0; i < ERTS_NO_PROC_PRIO_LEVELS; i++) {
         erts_print(to, to_arg, "Run Queue ");
@@ -14341,12 +14374,12 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
             break;
         }
         erts_print(to, to_arg, "Length: %d\n",
-                   erts_smp_atomic32_read_dirty(&esdp->run_queue->procs.prio_info[i].len));
+                   erts_smp_atomic32_read_dirty(&run_queue->procs.prio_info[i].len));
     }
     erts_print(to, to_arg, "Run Queue Port Length: %d\n",
-               erts_smp_atomic32_read_dirty(&esdp->run_queue->ports.info.len));
+               erts_smp_atomic32_read_dirty(&run_queue->ports.info.len));
 
-    flg = erts_smp_atomic32_read_dirty(&esdp->run_queue->flags);
+    flg = erts_smp_atomic32_read_dirty(&run_queue->flags);
     erts_print(to, to_arg, "Run Queue Flags: ");
     for (i = 0; i < ERTS_RUNQ_FLG_MAX && flg; i++) {
         erts_aint32_t chk = (1 << i);
@@ -14413,9 +14446,15 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
         }
     }
     erts_print(to, to_arg, "\n");
+}
+
+
+static void print_current_process_info(fmtfn_t to, void *to_arg,
+                                       ErtsSchedulerData* esdp)
+{
+    Process *p = esdp->current_process;
+    erts_aint32_t flg;
 
-    /* This *MUST* to be the last information in scheduler block */
-    p = esdp->current_process;
     erts_print(to, to_arg, "Current Process: ");
     if (esdp->current_process && !(ERTS_TRACE_FLAGS(p) & F_SENSITIVE)) {
 	flg = erts_smp_atomic32_read_dirty(&p->state);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 639818c20c..dd89e30f82 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1868,6 +1868,7 @@ void erts_stack_dump(fmtfn_t to, void *to_arg, Process *);
 void erts_limited_stack_trace(fmtfn_t to, void *to_arg, Process *);
 void erts_program_counter_info(fmtfn_t to, void *to_arg, Process *);
 void erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp);
+void erts_print_run_queue_info(fmtfn_t, void *to_arg, ErtsRunQueue*);
 void erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg);
 void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg);
 
-- 
cgit v1.2.3


From 9ca2c14787f42c5fe98c9c00364b508844059438 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 21 Feb 2018 17:23:49 +0100
Subject: erts: Refactor erlang:put/2

---
 erts/emulator/beam/erl_process_dict.c | 49 ++++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 21 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 3c80f0e0f6..1ec81f50a3 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -577,11 +577,12 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
 { 
     unsigned int hval;
     Eterm *hp;
+    Eterm *tp;
     Eterm tpl;
     Eterm old;
     Eterm tmp;
     int needed;
-    int i = 0;
+    int key_at = 0;
 #ifdef DEBUG
     Eterm *hp_limit;
 #endif
@@ -603,21 +604,26 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
      */
     needed = 3;			/* {Key,Value} tuple */
     if (is_boxed(old)) {
-	/*
-	 * We don't want to compare keys twice, so we'll always
-	 * reserve the space for two CONS cells.
-	 */
-	needed += 2+2;
+        ASSERT(is_tuple(old));
+        tp = tuple_val(old);
+        if (EQ(tp[1], id)) {
+            key_at = 1;
+        }
+        else {
+            needed += 2+2;
+        }
     } else if (is_list(old)) {
-	i = 0;
-	for (tmp = old; tmp != NIL && !EQ(tuple_val(TCAR(tmp))[1], id); tmp = TCDR(tmp)) {
-	    ++i;
-	}
-	if (is_nil(tmp)) {
-	    i = -1;
-	    needed += 2;
-	} else {
-	    needed += 2*(i+1);
+        int i = 1;
+
+        needed += 2;
+	for (tmp = old; tmp != NIL; tmp = TCDR(tmp)) {
+            tp = tuple_val(TCAR(tmp));
+            if (EQ(tp[1], id)) {
+                key_at = i;
+                needed += 2*(key_at-1);
+                break;
+            }
+            ++i;
 	}
     }
     if (HeapWordsLeft(p) < needed) {
@@ -648,7 +654,8 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
 	++(p->dictionary->numElements);
     } else if (is_boxed(old)) {
 	ASSERT(is_tuple(old));
-	if (EQ(tuple_val(old)[1],id)) {
+	if (key_at) {
+            ASSERT(EQ(tuple_val(old)[1],id));
 	    ARRAY_PUT(p->dictionary, hval, tpl);
 	    return tuple_val(old)[2];
 	} else {
@@ -661,7 +668,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
 	    ASSERT(hp <= hp_limit);
 	}
     } else if (is_list(old)) {
-	if (i == -1) {
+	if (!key_at) {
 	    /*
 	     * New key. Simply prepend the tuple to the beginning of the list.
 	     */
@@ -672,7 +679,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
 	    ++(p->dictionary->numElements);
 	} else {
 	    /*
-	     * i = Number of CDRs to skip to reach the changed element in the list.
+	     * key_at = Key position in the list.
 	     *
 	     * Replace old value in list. To avoid pointers from the old generation
 	     * to the new, we must rebuild the list from the beginning up to and
@@ -681,17 +688,17 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
 	    Eterm nlist;
 	    int j;
 
-	    hp = HeapOnlyAlloc(p, (i+1)*2);
+	    hp = HeapOnlyAlloc(p, key_at*2);
 	    
 	    /* Find the list element to change. */
-	    for (j = 0, nlist = old; j < i; j++, nlist = TCDR(nlist)) {
+	    for (j = 1, nlist = old; j < key_at; j++, nlist = TCDR(nlist)) {
 		;
 	    }
 	    ASSERT(EQ(tuple_val(TCAR(nlist))[1], id));
 	    nlist = TCDR(nlist); /* Unchanged part of list. */
 
 	    /* Rebuild list before the updated element. */
-	    for (tmp = old; i-- > 0; tmp = TCDR(tmp)) {
+	    for (tmp = old; --key_at > 0; tmp = TCDR(tmp)) {
 		nlist = CONS(hp, TCAR(tmp), nlist);
 		hp += 2;
 	    }
-- 
cgit v1.2.3


From 4d547dfb2a012ba1cf8fb9dd3cdc4d9df933a37f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 21 Feb 2018 22:05:10 +0100
Subject: erts: Optimize erlang:put/2 for immed values

Do destructive write of immed value if key exists.

This is an optimization at the expense of get/0
which must now copy all (mutable) key-value tuples.
---
 erts/emulator/beam/erl_process_dict.c | 41 +++++++++++++++++++++++++++++------
 1 file changed, 34 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 1ec81f50a3..87b440093b 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -92,7 +92,7 @@ static void pd_hash_erase_all(Process *p);
 static Eterm pd_hash_get_with_hval(Process *p, Eterm bucket, Eterm id);
 static Eterm pd_hash_get_keys(Process *p, Eterm value);
 static Eterm pd_hash_get_all_keys(Process *p, ProcDict *pd);
-static Eterm pd_hash_get_all(Process *p, ProcDict *pd);
+static Eterm pd_hash_get_all(Process *p, ProcDict *pd, int keep_dict);
 static Eterm pd_hash_put(Process *p, Eterm id, Eterm value);
 
 static void shrink(Process *p, Eterm* ret); 
@@ -281,7 +281,7 @@ BIF_RETTYPE get_0(BIF_ALIST_0)
 {
     Eterm ret;
     PD_CHECK(BIF_P->dictionary);
-    ret = pd_hash_get_all(BIF_P, BIF_P->dictionary);
+    ret = pd_hash_get_all(BIF_P, BIF_P->dictionary, 1);
     PD_CHECK(BIF_P->dictionary);
     BIF_RET(ret);
 }
@@ -329,7 +329,7 @@ BIF_RETTYPE erase_0(BIF_ALIST_0)
 {
     Eterm ret;
     PD_CHECK(BIF_P->dictionary);
-    ret = pd_hash_get_all(BIF_P, BIF_P->dictionary);
+    ret = pd_hash_get_all(BIF_P, BIF_P->dictionary, 0);
     pd_hash_erase_all(BIF_P);
     PD_CHECK(BIF_P->dictionary);
     BIF_RET(ret);
@@ -541,29 +541,46 @@ static Eterm pd_hash_get_keys(Process *p, Eterm value)
 	
 
 static Eterm
-pd_hash_get_all(Process *p, ProcDict *pd)
+pd_hash_get_all(Process *p, ProcDict *pd, int keep_dict)
 {
     Eterm* hp;
+    Eterm* tp;
     Eterm res = NIL;
     Eterm tmp, tmp2;
     unsigned int i;
     unsigned int num;
+    Uint need;
 
     if (pd == NULL) {
 	return res;
     }
     num = HASH_RANGE(pd);
-    hp = HAlloc(p, pd->numElements * 2);
-    
+
+    /*
+     * If this is not erase/0, then must copy all key-value tuples
+     * as they may be mutated by put/2.
+     */
+    need = pd->numElements * (keep_dict ? 2+3 : 2);
+    hp = HAlloc(p, need);
+
     for (i = 0; i < num; ++i) {
 	tmp = ARRAY_GET(pd, i);
 	if (is_boxed(tmp)) {
-	    ASSERT(is_tuple(tmp));
+            if (keep_dict) {
+                tp = tuple_val(tmp);
+                tmp = TUPLE2(hp, tp[1], tp[2]);
+                hp += 3;
+            }
 	    res = CONS(hp, tmp, res);
 	    hp += 2;
 	} else if (is_list(tmp)) {
 	    while (tmp != NIL) {
 		tmp2 = TCAR(tmp);
+                if (keep_dict) {
+                    tp = tuple_val(tmp2);
+                    tmp2 = TUPLE2(hp, tp[1], tp[2]);
+                    hp += 3;
+                }
 		res = CONS(hp, tmp2, res);
 		hp += 2;
 		tmp = TCDR(tmp);
@@ -607,6 +624,11 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
         ASSERT(is_tuple(old));
         tp = tuple_val(old);
         if (EQ(tp[1], id)) {
+            if (is_immed(value)) {
+                Eterm old_val = tp[2];
+                tp[2] = value;
+                return old_val;
+            }
             key_at = 1;
         }
         else {
@@ -619,6 +641,11 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
 	for (tmp = old; tmp != NIL; tmp = TCDR(tmp)) {
             tp = tuple_val(TCAR(tmp));
             if (EQ(tp[1], id)) {
+                if (is_immed(value)) {
+                    Eterm old_val = tp[2];
+                    tp[2] = value;
+                    return old_val;
+                }
                 key_at = i;
                 needed += 2*(key_at-1);
                 break;
-- 
cgit v1.2.3


From 8f3536583a6338e3a61dbf008d674f54a77e401c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 21 Feb 2018 12:57:25 +0100
Subject: Add ets:whereis/1 for resolving table names -> tid()

---
 erts/emulator/beam/bif.tab  |  1 +
 erts/emulator/beam/erl_db.c | 28 +++++++++++++++++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index c0d5e8ce74..b5725e4185 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -696,3 +696,4 @@ bif erlang:iolist_to_iovec/1
 bif erts_internal:new_connection/1
 bif erts_internal:abort_connection/2
 bif erts_internal:map_next/3
+bif ets:whereis/1
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 3ba0886464..1ab1c4a363 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1753,6 +1753,28 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     BIF_RET(ret);
 }
 
+/*
+** Retrieves the tid() of a named ets table.
+*/
+BIF_RETTYPE ets_whereis_1(BIF_ALIST_1)
+{
+    DbTable* tb;
+    Eterm res;
+
+    if (is_not_atom(BIF_ARG_1)) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+
+    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL) {
+        BIF_RET(am_undefined);
+    }
+
+    res = make_tid(BIF_P, tb);
+    db_unlock(tb, LCK_READ);
+
+    BIF_RET(res);
+}
+
 /* 
 ** The lookup BIF 
 */
@@ -3126,7 +3148,8 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1)
     static Eterm fields[] = {am_protection, am_keypos, am_type, am_named_table,
                              am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed,
                              am_write_concurrency,
-                             am_read_concurrency};
+                             am_read_concurrency,
+                             am_id};
     Eterm results[sizeof(fields)/sizeof(Eterm)];
     DbTable* tb;
     Eterm res;
@@ -4016,7 +4039,10 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	ret = is_table_named(tb) ? am_true : am_false;
     } else if (What == am_compressed) {
 	ret = tb->common.compress ? am_true : am_false;
+    } else if (What == am_id) {
+        ret = make_tid(p, tb);
     }
+
     /*
      * For debugging purposes
      */
-- 
cgit v1.2.3


From 105c2f9d8e2846857b4aa37ebec70439e429c497 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 22 Feb 2018 17:28:51 +0100
Subject: erts,observer: Add port states and flags to crash dump

---
 erts/emulator/beam/io.c | 89 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 89 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index c6444fd440..f08fd2ac2d 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -5151,6 +5151,93 @@ static void prt_one_lnk(ErtsLink *lnk, void *vprtd)
     erts_print(prtd->to, prtd->arg, "%T", lnk->pid);
 }
 
+static void dump_port_state(fmtfn_t to, void *arg, erts_aint32_t state)
+{
+    erts_aint32_t rest;
+    int unknown = 0;
+    char delim = ' ';
+
+    erts_print(to, arg, "State:");
+
+    rest = state;
+    while (rest) {
+        erts_aint32_t chk = (rest ^ (rest-1)) & rest;  /* lowest set bit */
+        char* s;
+
+        rest &= ~chk;
+        switch (chk) {
+        case ERTS_PORT_SFLG_CONNECTED:          s = "CONNECTED"; break;
+        case ERTS_PORT_SFLG_EXITING:            s = "EXITING"; break;
+        case ERTS_PORT_SFLG_DISTRIBUTION:       s = "DISTR"; break;
+        case ERTS_PORT_SFLG_BINARY_IO:          s = "BINARY_IO"; break;
+        case ERTS_PORT_SFLG_SOFT_EOF:           s = "SOFT_EOF"; break;
+        case ERTS_PORT_SFLG_CLOSING:            s = "CLOSING"; break;
+        case ERTS_PORT_SFLG_SEND_CLOSED:        s = "SEND_CLOSED"; break;
+        case ERTS_PORT_SFLG_LINEBUF_IO:         s = "LINEBUF_IO"; break;
+        case ERTS_PORT_SFLG_FREE:               s = "FREE"; break;
+        case ERTS_PORT_SFLG_INITIALIZING:       s = "INITIALIZING"; break;
+        case ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK: s = "PORT_LOCK"; break;
+        case ERTS_PORT_SFLG_INVALID:            s = "INVALID"; break;
+        case ERTS_PORT_SFLG_HALT:               s = "HALT"; break;
+#ifdef DEBUG
+        case ERTS_PORT_SFLG_PORT_DEBUG:         s = "DEBUG"; break;
+#endif
+        default:
+            unknown = 1;
+            continue;
+        }
+        erts_print(to, arg, "%c%s", delim, s);
+        delim = '|';
+    }
+    if (unknown || !state)
+        erts_print(to, arg, "%c0x%x\n", delim, state);
+    else
+        erts_print(to, arg, "\n");
+}
+
+static void dump_port_task_flags(fmtfn_t to, void *arg, Port* p)
+{
+    erts_aint32_t flags = erts_smp_atomic32_read_nob(&p->sched.flags);
+    erts_aint32_t unknown = 0;
+    char delim = ' ';
+
+    if (!flags)
+	return;
+
+    erts_print(to, arg, "Task Flags:");
+
+    while (flags) {
+        erts_aint32_t chk = (flags ^ (flags-1)) & flags;  /* lowest set bit */
+        char* s;
+
+        flags &= ~chk;
+        switch (chk) {
+	case ERTS_PTS_FLG_IN_RUNQ:               s = "IN_RUNQ"; break;
+	case ERTS_PTS_FLG_EXEC:                  s = "EXEC"; break;
+	case ERTS_PTS_FLG_HAVE_TASKS:            s = "HAVE_TASKS"; break;
+	case ERTS_PTS_FLG_EXIT:                  s = "EXIT"; break;
+	case ERTS_PTS_FLG_BUSY_PORT:             s = "BUSY_PORT"; break;
+	case ERTS_PTS_FLG_BUSY_PORT_Q:           s = "BUSY_Q"; break;
+	case ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q: s = "CHK_UNSET_BUSY_Q"; break;
+	case ERTS_PTS_FLG_HAVE_BUSY_TASKS:       s = "BUSY_TASKS"; break;
+	case ERTS_PTS_FLG_HAVE_NS_TASKS:         s = "NS_TASKS"; break;
+	case ERTS_PTS_FLG_PARALLELISM:           s = "PARALLELISM"; break;
+	case ERTS_PTS_FLG_FORCE_SCHED:           s = "FORCE_SCHED"; break;
+	case ERTS_PTS_FLG_EXITING:               s = "EXITING"; break;
+	case ERTS_PTS_FLG_EXEC_IMM:              s = "EXEC_IMM"; break;
+        default:
+            unknown |= chk;
+            continue;
+        }
+        erts_print(to, arg, "%c%s", delim, s);
+        delim = '|';
+    }
+    if (unknown)
+        erts_print(to, arg, "%cUNKNOWN(0x%x)\n", delim, unknown);
+    else
+        erts_print(to, arg, "\n");
+}
+
 void
 print_port_info(Port *p, fmtfn_t to, void *arg)
 {
@@ -5160,6 +5247,8 @@ print_port_info(Port *p, fmtfn_t to, void *arg)
 	return;
 
     erts_print(to, arg, "=port:%T\n", p->common.id);
+    dump_port_state(to, arg, state);
+    dump_port_task_flags(to, arg, p);
     erts_print(to, arg, "Slot: %d\n", internal_port_index(p->common.id));
     if (state & ERTS_PORT_SFLG_CONNECTED) {
 	erts_print(to, arg, "Connected: %T", ERTS_PORT_GET_CONNECTED(p));
-- 
cgit v1.2.3


From 2e2d1ea0affa72b613d291f4d9002292d878fc2b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 22 Feb 2018 20:25:04 +0100
Subject: erts,observer: Add port-suspended pids to crash dump

---
 erts/emulator/beam/erl_process.c | 14 ++++++++++++++
 erts/emulator/beam/erl_process.h |  1 +
 erts/emulator/beam/io.c          |  4 ++++
 3 files changed, 19 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 5fcb100fae..9b22f024b0 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1609,6 +1609,20 @@ erts_proclist_destroy(ErtsProcList *plp)
     proclist_destroy(plp);
 }
 
+void
+erts_proclist_dump(fmtfn_t to, void *to_arg, ErtsProcList *plp)
+{
+    ErtsProcList *first = plp;
+
+    while (plp) {
+        erts_print(to, to_arg, "%T", plp->pid);
+        plp = plp->next;
+        if (plp == first)
+            break;
+    }
+    erts_print(to, to_arg, "\n");
+}
+
 void *
 erts_psd_set_init(Process *p, int ix, void *data)
 {
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index dd89e30f82..5cac939771 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1614,6 +1614,7 @@ Uint64 erts_step_proc_interval(void);
 ErtsProcList *erts_proclist_create(Process *);
 ErtsProcList *erts_proclist_copy(ErtsProcList *);
 void erts_proclist_destroy(ErtsProcList *);
+void erts_proclist_dump(fmtfn_t to, void *to_arg, ErtsProcList*);
 
 ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *);
 ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index f08fd2ac2d..13b8125a99 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -5271,6 +5271,10 @@ print_port_info(Port *p, fmtfn_t to, void *arg)
 	erts_doforall_monitors(ERTS_P_MONITORS(p), &prt_one_monitor, &prtd);
 	erts_print(to, arg, "\n");
     }
+    if (p->suspended) {
+	erts_print(to, arg, "Suspended: ");
+	erts_proclist_dump(to, arg, p->suspended);
+    }
 
     if (p->common.u.alive.reg != NULL)
 	erts_print(to, arg, "Registered as: %T\n", p->common.u.alive.reg->name);
-- 
cgit v1.2.3


From 8c3f3303f20ab5f1031731af6df5c2ef1499cd36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 20 Feb 2018 11:30:09 +0100
Subject: Add a helper function for attaching tmp objects to NIF environments

---
 erts/emulator/beam/erl_nif.c | 96 ++++++++++++++++++++++++++------------------
 1 file changed, 58 insertions(+), 38 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 7db4af8919..6ccf37e793 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -237,9 +237,11 @@ static void cache_env(ErlNifEnv* env);
 static void full_flush_env(ErlNifEnv *env);
 static void flush_env(ErlNifEnv* env);
 
-/* Temporary object header, auto-deallocated when NIF returns
- * or when independent environment is cleared.
- */
+/* Temporary object header, auto-deallocated when NIF returns or when
+ * independent environment is cleared.
+ *
+ * The payload can be accessed with &tmp_obj_ptr[1] but keep in mind that its
+ * first element must not require greater alignment than `next`. */
 struct enif_tmp_obj_t {
     struct enif_tmp_obj_t* next;
     void (*dtor)(struct enif_tmp_obj_t*);
@@ -256,6 +258,46 @@ static ERTS_INLINE void free_tmp_objs(ErlNifEnv* env)
     }
 }
 
+/* Whether the given environment is bound to a process and will be cleaned up
+ * when the NIF returns. It's safe to use temp_alloc for objects in
+ * env->tmp_obj_list when this is true. */
+static ERTS_INLINE int is_proc_bound(ErlNifEnv *env)
+{
+    return env->mod_nif != NULL;
+}
+
+/* Allocates and attaches an object to the given environment, running its
+ * destructor when the environment is cleared. To avoid temporary variables the
+ * address of the allocated object is returned instead of the enif_tmp_obj_t.
+ *
+ * The destructor *must* call `erts_free(tmp_obj->allocator, tmp_obj)` to free
+ * the object. If the destructor needs to refer to the allocated object its
+ * address will be &tmp_obj[1]. */
+static ERTS_INLINE void *alloc_tmp_obj(ErlNifEnv *env, size_t size,
+                                       void (*dtor)(struct enif_tmp_obj_t*)) {
+    struct enif_tmp_obj_t *tmp_obj;
+    ErtsAlcType_t allocator;
+
+    allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF;
+
+    tmp_obj = erts_alloc(allocator, sizeof(struct enif_tmp_obj_t) + MAX(1, size));
+
+    tmp_obj->next = env->tmp_obj_list;
+    tmp_obj->allocator = allocator;
+    tmp_obj->dtor = dtor;
+
+    env->tmp_obj_list = tmp_obj;
+
+    return (void*)&tmp_obj[1];
+}
+
+/* Generic destructor for objects allocated through alloc_tmp_obj that don't
+ * care about their payload. */
+static void tmp_alloc_dtor(struct enif_tmp_obj_t *tmp_obj)
+{
+    erts_free(tmp_obj->allocator, tmp_obj);
+}
+
 void erts_post_nif(ErlNifEnv* env)
 {
     erts_unblock_fpe(env->fpe_was_unmasked);
@@ -1019,11 +1061,6 @@ int enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term)
     return is_number(term);
 }
 
-static ERTS_INLINE int is_proc_bound(ErlNifEnv* env)
-{
-    return env->mod_nif != NULL;
-}
-
 static void aligned_binary_dtor(struct enif_tmp_obj_t* obj)
 {
     erts_free_aligned_binary_bytes_extra((byte*)obj, obj->allocator);
@@ -1065,15 +1102,8 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin)
     return 1;
 }
 
-static void tmp_alloc_dtor(struct enif_tmp_obj_t* obj)
-{
-    erts_free(obj->allocator,  obj);
-}
-
 int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin)
 {
-    struct enif_tmp_obj_t* tobj;
-    ErtsAlcType_t allocator;
     ErlDrvSizeT sz;
     if (is_binary(term)) {
 	return enif_inspect_binary(env,term,bin);
@@ -1089,14 +1119,7 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin)
 	return 0;
     }
 
-    allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF;
-    tobj = erts_alloc(allocator, sz + sizeof(struct enif_tmp_obj_t));
-    tobj->allocator = allocator;
-    tobj->next = env->tmp_obj_list;
-    tobj->dtor = &tmp_alloc_dtor;
-    env->tmp_obj_list = tobj;
-
-    bin->data = (unsigned char*) &tobj[1]; 
+    bin->data = alloc_tmp_obj(env, sz, &tmp_alloc_dtor);
     bin->size = sz;
     bin->bin_term = THE_NON_VALUE;
     bin->ref_bin = NULL;
@@ -4236,34 +4259,31 @@ static unsigned calc_checksum(unsigned char* ptr, unsigned size);
 
 struct readonly_check_t
 {
-    struct enif_tmp_obj_t hdr;
     unsigned char* ptr;
     unsigned size;
     unsigned checksum;
 };
 static void add_readonly_check(ErlNifEnv* env, unsigned char* ptr, unsigned sz)
 {
-    ErtsAlcType_t allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF;
-    struct readonly_check_t* obj = erts_alloc(allocator, 
-					      sizeof(struct readonly_check_t));
-    obj->hdr.allocator = allocator;
-    obj->hdr.next = env->tmp_obj_list;
-    env->tmp_obj_list = &obj->hdr;
-    obj->hdr.dtor = &readonly_check_dtor;
+    struct readonly_check_t* obj;
+
+    obj = alloc_tmp_obj(env, sizeof(struct readonly_check_t),
+        &readonly_check_dtor);
+
     obj->ptr = ptr;
     obj->size = sz;
-    obj->checksum = calc_checksum(ptr, sz);    
+    obj->checksum = calc_checksum(ptr, sz);
 }
-static void readonly_check_dtor(struct enif_tmp_obj_t* o)
+static void readonly_check_dtor(struct enif_tmp_obj_t* tmp_obj)
 {
-    struct readonly_check_t* obj = (struct readonly_check_t*) o;
-    unsigned chksum = calc_checksum(obj->ptr, obj->size);
-    if (chksum != obj->checksum) { 
+    struct readonly_check_t* ro_check = (struct readonly_check_t*)&tmp_obj[1];
+    unsigned chksum = calc_checksum(ro_check->ptr, ro_check->size);
+    if (chksum != ro_check->checksum) { 
 	fprintf(stderr, "\r\nReadonly data written by NIF, checksums differ"
-		" %x != %x\r\nABORTING\r\n", chksum, obj->checksum);
+		" %x != %x\r\nABORTING\r\n", chksum, ro_check->checksum);
 	abort();
     }
-    erts_free(obj->hdr.allocator,  obj);
+    erts_free(tmp_obj->allocator, tmp_obj);
 }
 static unsigned calc_checksum(unsigned char* ptr, unsigned size)
 {
-- 
cgit v1.2.3


From 7fb9b59ed1fc6cbff581effaea7504fd2dc90f4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 19 Feb 2018 17:06:35 +0100
Subject: Free temporary iovecs through the tmp object list

Attaching to a ProcBin is the fastest way to delay the release of an
already allocated Binary, but alloc_tmp_obj is a lot faster when
starting from scratch since it usually uses temp_alloc.
---
 erts/emulator/beam/erl_nif.c | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 6ccf37e793..28b68c9e7d 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3543,19 +3543,14 @@ static int create_iovec_from_slice(ErlNifEnv *env,
         alloc_size = binv_offset;
         alloc_size += slice->iovec_len * sizeof(Binary*);
 
-        /* If we have an environment we'll attach the allocated data to it. The
-         * GC will take care of releasing it later on. */
+        /* When the user passes an environment, we attach the iovec to it so
+         * the user won't have to bother managing it (similar to
+         * enif_inspect_binary). It'll disappear once the environment is
+         * cleaned up. */
         if (env != NULL) {
-            ErlNifBinary gc_bin;
-
-            if (!enif_alloc_binary(alloc_size, &gc_bin)) {
-                return 0;
-            }
-
-            alloc_base = (char*)gc_bin.data;
-            enif_make_binary(env, &gc_bin);
+            alloc_base = alloc_tmp_obj(env, alloc_size, &tmp_alloc_dtor);
         } else {
-            alloc_base = enif_alloc(alloc_size);
+            alloc_base = erts_alloc(ERTS_ALC_T_NIF, alloc_size);
         }
 
         iovec = (ErlNifIOVec*)alloc_base;
@@ -3569,7 +3564,7 @@ static int create_iovec_from_slice(ErlNifEnv *env,
 
     if(!fill_iovec_with_slice(env, slice, iovec)) {
         if (env == NULL && !(iovec->flags & ERL_NIF_IOVEC_FLAGS_PREALLOC)) {
-            enif_free(iovec);
+            erts_free(ERTS_ALC_T_NIF, iovec);
         }
 
         return 0;
-- 
cgit v1.2.3


From 97ac7f37c93b36617fe09b81f64a967b0d341673 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 19 Feb 2018 18:04:01 +0100
Subject: Tweak asserts and explain why the copy buf doesn't use the tmp obj
 list

---
 erts/emulator/beam/erl_nif.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 28b68c9e7d..72bff49f32 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3431,6 +3431,7 @@ static void marshal_iovec_binary(Eterm binary, ErlNifBinary *copy_buffer,
      * and reference that instead. */
 
     if (result->ref_bin == NULL || bit_offset != 0) {
+        ASSERT(copy_buffer->ref_bin != NULL && copy_buffer->data != NULL);
         ASSERT(result->size <= (copy_buffer->size - *copy_offset));
 
         if (bit_offset == 0) {
@@ -3452,8 +3453,8 @@ static void marshal_iovec_binary(Eterm binary, ErlNifBinary *copy_buffer,
 static int fill_iovec_with_slice(ErlNifEnv *env,
                                  iovec_slice_t *slice,
                                  ErlNifIOVec *iovec) {
+    ErlNifBinary copy_buffer = {0};
     UWord copy_offset, iovec_idx;
-    ErlNifBinary copy_buffer;
     Eterm sublist_iterator;
 
     /* Set up a common refc binary for all on-heap and unaligned binaries. */
@@ -3461,11 +3462,8 @@ static int fill_iovec_with_slice(ErlNifEnv *env,
         if (!enif_alloc_binary(slice->copied_size, &copy_buffer)) {
             return 0;
         }
-    } else {
-#ifdef DEBUG
-        copy_buffer.data = NULL;
-        copy_buffer.size = 0;
-#endif
+
+        ASSERT(copy_buffer.ref_bin != NULL);
     }
 
     sublist_iterator = slice->sublist_start;
@@ -3515,9 +3513,11 @@ static int fill_iovec_with_slice(ErlNifEnv *env,
         }
     } else {
         if (slice->copied_size > 0) {
-            /* Attach the binary to our environment and let the GC take care of
-             * it after returning. */
-            enif_make_binary(env, &copy_buffer);
+            /* Attach the binary to our environment and let the next minor GC
+             * get rid of it. This is slightly faster than using the tmp object
+             * list since it avoids off-heap allocations. */
+            erts_build_proc_bin(&MSO(env->proc),
+                alloc_heap(env, PROC_BIN_SIZE), copy_buffer.ref_bin);
         }
     }
 
-- 
cgit v1.2.3


From 89de89e4c962aac2ff0c55f420ef6d510533f02a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 19 Feb 2018 18:04:01 +0100
Subject: Unconditionally transfer ownership to the created term

This fixes two corner-cases:

1) We will no longer return an invalid term when a binary inspected
   on environment A is used in enif_make_binary on environment B

2) A double-free in this sequence of events:

   * enif_alloc_binary(size, &bin);
   * enif_ioq_enq_binary(ioq, &bin, skip);
   * enif_make_binary(env, &bin);
   * enif_make_binary(env, &bin);

OTP-14931
OTP-14932
---
 erts/emulator/beam/erl_nif.c | 42 +++++++++++++++---------------------------
 erts/emulator/beam/erl_nif.h |  3 ++-
 2 files changed, 17 insertions(+), 28 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 72bff49f32..42a90f706b 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1095,7 +1095,6 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin)
 	u.tmp->dtor = &aligned_binary_dtor;
 	env->tmp_obj_list = u.tmp;
     }
-    bin->bin_term = bin_term;
     bin->size = binary_size(bin_term);
     bin->ref_bin = NULL;
     ADD_READONLY_CHECK(env, bin->data, bin->size);
@@ -1111,7 +1110,6 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin)
     if (is_nil(term)) {
 	bin->data = (unsigned char*) &bin->data; /* dummy non-NULL */
 	bin->size = 0;
-	bin->bin_term = THE_NON_VALUE;
 	bin->ref_bin = NULL;
 	return 1;
     }
@@ -1121,7 +1119,6 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin)
 
     bin->data = alloc_tmp_obj(env, sz, &tmp_alloc_dtor);
     bin->size = sz;
-    bin->bin_term = THE_NON_VALUE;
     bin->ref_bin = NULL;
     erts_iolist_to_buf(term, (char*) bin->data, sz);
     ADD_READONLY_CHECK(env, bin->data, bin->size); 
@@ -1139,7 +1136,6 @@ int enif_alloc_binary(size_t size, ErlNifBinary* bin)
 
     bin->size = size;
     bin->data = (unsigned char*) refbin->orig_bytes;
-    bin->bin_term = THE_NON_VALUE;
     bin->ref_bin = refbin;
     return 1;
 }
@@ -1173,12 +1169,10 @@ void enif_release_binary(ErlNifBinary* bin)
 {
     if (bin->ref_bin != NULL) {
 	Binary* refbin = bin->ref_bin;
-	ASSERT(bin->bin_term == THE_NON_VALUE);
         erts_bin_release(refbin);
     }
 #ifdef DEBUG
     bin->data = NULL;
-    bin->bin_term = THE_NON_VALUE;
     bin->ref_bin = NULL;
 #endif
 }
@@ -1335,29 +1329,24 @@ int enif_get_string(ErlNifEnv *env, ERL_NIF_TERM list, char* buf, unsigned len,
 
 Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin)
 {
-    if (bin->bin_term != THE_NON_VALUE) {
-	return bin->bin_term;
-    }
-    else if (bin->ref_bin != NULL) {
-	Binary* bptr = bin->ref_bin;
-	Eterm bin_term;
-	
+    Eterm bin_term;
+
+    if (bin->ref_bin != NULL) {
+        Binary* binary = bin->ref_bin;
+
         bin_term = erts_build_proc_bin(&MSO(env->proc),
                                        alloc_heap(env, PROC_BIN_SIZE),
-                                       bptr);
-	if (erts_refc_read(&bptr->intern.refc, 1) == 1) {
-	    /* Total ownership transfer */
-	    bin->ref_bin = NULL;
-	    bin->bin_term = bin_term;
-	}
-	return bin_term;
-    }
-    else {
-	flush_env(env);
-	bin->bin_term = new_binary(env->proc, bin->data, bin->size);
-	cache_env(env);
-	return bin->bin_term;
+                                       binary);
+
+        /* Our (possibly shared) ownership has been transferred to the term. */
+        bin->ref_bin = NULL;
+    } else {
+        flush_env(env);
+        bin_term = new_binary(env->proc, bin->data, bin->size);
+        cache_env(env);
     }
+
+    return bin_term;
 }
 
 Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term,
@@ -3403,7 +3392,6 @@ static void marshal_iovec_binary(Eterm binary, ErlNifBinary *copy_buffer,
     parent_header = binary_val(parent_binary);
 
     result->size = binary_size(binary);
-    result->bin_term = binary;
 
     if (thing_subtag(*parent_header) == REFC_BINARY_SUBTAG) {
         ProcBin *pb = (ProcBin*)parent_header;
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 053f7673c4..eb69a8255c 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -137,8 +137,9 @@ typedef struct
     unsigned char* data;
 
     /* Internals (avert your eyes) */
-    ERL_NIF_TERM bin_term;
     void* ref_bin;
+    /* for future additions to be ABI compatible (same struct size) */
+    void* __spare__[1];
 }ErlNifBinary;
 
 #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
-- 
cgit v1.2.3


From 24b67adabd6e31ae8f2bb529d4f1cd5d6fdf1b1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 20 Feb 2018 19:07:46 +0100
Subject: Expand ErlNifBinary by another word for future additions

When fixing OTP-14931/14932 we had to settle for a solution that
decreased performance in a few corner cases like multiple calls to
enif_make_binary, as the other options on the table required extra
fields which would have broken ABI compatibility.

This is ABI-compatible since the fields are unused and at the end.
---
 erts/emulator/beam/erl_nif.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index eb69a8255c..a99b4db705 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -139,7 +139,7 @@ typedef struct
     /* Internals (avert your eyes) */
     void* ref_bin;
     /* for future additions to be ABI compatible (same struct size) */
-    void* __spare__[1];
+    void* __spare__[2];
 }ErlNifBinary;
 
 #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
-- 
cgit v1.2.3


From a92c200af89b9b1f35db26ae8e13333af5638e15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 19 Feb 2018 18:04:01 +0100
Subject: Make enif_make_binary return heap binaries if possible

This does not avoid allocating a ProcBin since we still need to
release the binary somehow (and using the tmp obj list is slightly
slower since it's an unavoidable off-heap allocation), but it does give
us a hard limit on how long said ProcBin will live, which is a lot
better than before.

OTP-14845
---
 erts/emulator/beam/erl_nif.c | 34 +++++++++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 42a90f706b..d2000aa71e 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -488,6 +488,7 @@ static void cache_env(ErlNifEnv* env)
 	env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size;
     }
 }
+
 void* enif_priv_data(ErlNifEnv* env)
 {
     return env->mod_nif->priv_data;
@@ -1334,9 +1335,36 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin)
     if (bin->ref_bin != NULL) {
         Binary* binary = bin->ref_bin;
 
-        bin_term = erts_build_proc_bin(&MSO(env->proc),
-                                       alloc_heap(env, PROC_BIN_SIZE),
-                                       binary);
+        /* If the binary is smaller than the heap binary limit we'll return a
+         * heap binary to reduce the number of small refc binaries in the
+         * system. We can't simply release the refc binary right away however;
+         * the documentation states that the binary should be considered
+         * read-only from this point on, which implies that it should still be
+         * readable.
+         *
+         * We could keep it alive until we return by adding it to the temporary
+         * object list, but that requires an off-heap allocation which is
+         * potentially quite slow, so we create a dummy ProcBin instead and
+         * rely on the next minor GC to get rid of it. */
+        if (bin->size <= ERL_ONHEAP_BIN_LIMIT) {
+            ErlHeapBin* hb;
+
+            hb = (ErlHeapBin*)alloc_heap(env, heap_bin_size(bin->size));
+            hb->thing_word = header_heap_bin(bin->size);
+            hb->size = bin->size;
+
+            sys_memcpy(hb->data, bin->data, bin->size);
+
+            erts_build_proc_bin(&MSO(env->proc),
+                                alloc_heap(env, PROC_BIN_SIZE),
+                                binary);
+
+            bin_term = make_binary(hb);
+        } else {
+            bin_term = erts_build_proc_bin(&MSO(env->proc),
+                                           alloc_heap(env, PROC_BIN_SIZE),
+                                           binary);
+        }
 
         /* Our (possibly shared) ownership has been transferred to the term. */
         bin->ref_bin = NULL;
-- 
cgit v1.2.3


From 6f120d4ab4fab2231f7475256bcc1eeac2bfe6e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 26 Feb 2018 10:28:26 +0100
Subject: Allow opening device files and FIFOs with file:open/2

To the best of our knowledge this was introduced since file operations
on device files/FIFO:s could hang the emulator forever back when the
emulator was single-threaded and lacked IO threads; a read operation
could block all progress preventing the write operation it waited for
from occurring.

Granted, this could still happen through starving all dirty IO
schedulers, but the same issue can arise with NFS files which we've
always allowed.

Removing this restriction also lets us remove a stat(2) call that was
added to specifically allow `/dev/null`.
---
 erts/emulator/nifs/unix/unix_prim_file.c | 19 +++----------------
 1 file changed, 3 insertions(+), 16 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/nifs/unix/unix_prim_file.c b/erts/emulator/nifs/unix/unix_prim_file.c
index 57c8ef62e1..4a6c476882 100644
--- a/erts/emulator/nifs/unix/unix_prim_file.c
+++ b/erts/emulator/nifs/unix/unix_prim_file.c
@@ -125,24 +125,11 @@ static int open_file_type_check(const efile_path_t *path, int fd) {
          * immediately in a read within the call, but the new implementation
          * never does that. */
          return 1;
-    } else {
-        /* The old driver tolerated opening /dev/null despite the "no devices"
-         * limitation. It provided no explanation for this but we still need
-         * to match the behavior. We're checking through stat(2) instead of
-         * comparing the name to account for links. */
-        struct stat null_device_info;
-        int is_dev_null;
-
-        is_dev_null = (stat("/dev/null", &null_device_info) == 0);
-        is_dev_null &= (file_info.st_ino == null_device_info.st_ino);
-        is_dev_null &= (file_info.st_dev == null_device_info.st_dev);
-
-        if(is_dev_null) {
-            return 1;
-        }
     }
 
-    if(!S_ISREG(file_info.st_mode)) {
+    /* Allow everything that isn't a directory, and error out on the next call
+     * if it's unsupported. */
+    if(S_ISDIR(file_info.st_mode)) {
         return 0;
     }
 
-- 
cgit v1.2.3


From 94c94e901b69e9320011570041cf16ecd960dea2 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 2 Jan 2018 14:38:09 +0100
Subject: erts: Delay cleanup of removed tracer on dirty scheds

It is not simple to do the correct de-allocation on
a dirty schedulers, so we just delay it until this
process runs on a normal scheduler.
---
 erts/emulator/beam/erl_trace.c | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index db7d0ac449..9fda4ff6ff 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -3009,24 +3009,28 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks,
         ASSERT(0);
     }
 
-    /* Only remove tracer on self() and ports */
+    /* Only remove tracer on (self() or ports) AND we are on a normal scheduler */
     if (is_internal_port(t_p->id) || (c_p && c_p->common.id == t_p->id)) {
+        ErtsSchedulerData *esdp = erts_get_scheduler_data();
         ErtsProcLocks c_p_xlocks = 0;
-        if (is_internal_pid(t_p->id)) {
-            ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN);
-            if (c_p_locks != ERTS_PROC_LOCKS_ALL) {
-                c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL;
-                if (erts_smp_proc_trylock(c_p, c_p_xlocks) == EBUSY) {
-                    erts_smp_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN);
-                    erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+        if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+            if (is_internal_pid(t_p->id)) {
+                ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN);
+                if (c_p_locks != ERTS_PROC_LOCKS_ALL) {
+                    c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL;
+                    if (erts_smp_proc_trylock(c_p, c_p_xlocks) == EBUSY) {
+                        erts_smp_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN);
+                        erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+                    }
                 }
             }
-        }
-        erts_tracer_replace(t_p, erts_tracer_nil);
-        t_p->trace_flags &= ~TRACEE_FLAGS;
 
-        if (c_p_xlocks)
-            erts_smp_proc_unlock(c_p, c_p_xlocks);
+            erts_tracer_replace(t_p, erts_tracer_nil);
+            t_p->trace_flags &= ~TRACEE_FLAGS;
+
+            if (c_p_xlocks)
+                erts_smp_proc_unlock(c_p, c_p_xlocks);
+        }
     }
 
     return 0;
-- 
cgit v1.2.3


From 2627297ad239697c7df297b8fde35f9827fa962f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 26 Feb 2018 16:27:39 +0100
Subject: Replace binary:bin_to_list CIF implementation with binary_to_list

binary:bin_to_list had a poor implementation that resulted in
excessive garbage collection. binary_to_list is almost identical and
has a generally better implementation, so I've replaced
binary:bin_to_list's CIF with a thin wrapper around binary_to_list.

Granted, binary_to_list has a deprecated indexing scheme, but we're
unlikely to ever remote it entirely and it's somewhat easy to move
it to the 'binary' module later on.
---
 erts/emulator/beam/atom.names       |   1 -
 erts/emulator/beam/bif.tab          |   3 -
 erts/emulator/beam/erl_bif_binary.c | 191 ------------------------------------
 3 files changed, 195 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index b6ec3e7ed2..42a368cdd8 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -124,7 +124,6 @@ atom big
 atom bif_return_trap
 atom bif_timer_server
 atom binary
-atom binary_bin_to_list_trap
 atom binary_copy_trap
 atom binary_find_trap
 atom binary_longest_prefix_trap
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index b5725e4185..0d1166f6ed 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -551,9 +551,6 @@ bif binary:last/1
 bif binary:at/2
 bif binary:part/2 binary_binary_part_2
 bif binary:part/3 binary_binary_part_3
-bif binary:bin_to_list/1
-bif binary:bin_to_list/2
-bif binary:bin_to_list/3
 bif binary:list_to_bin/1
 bif binary:copy/1
 bif binary:copy/2
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 41c2ae08d3..33bc189182 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -61,8 +61,6 @@ static Export binary_longest_prefix_trap_export;
 static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3);
 static Export binary_longest_suffix_trap_export;
 static BIF_RETTYPE binary_longest_suffix_trap(BIF_ALIST_3);
-static Export binary_bin_to_list_trap_export;
-static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3);
 static Export binary_copy_trap_export;
 static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2);
 static Uint max_loop_limit;
@@ -86,10 +84,6 @@ void erts_init_bif_binary(void)
 			  am_erlang, am_binary_longest_suffix_trap, 3,
 			  &binary_longest_suffix_trap);
 
-    erts_init_trap_export(&binary_bin_to_list_trap_export,
-			  am_erlang, am_binary_bin_to_list_trap, 3,
-			  &binary_bin_to_list_trap);
-
     erts_init_trap_export(&binary_copy_trap_export,
 			  am_erlang, am_binary_copy_trap, 2,
 			  &binary_copy_trap);
@@ -2440,191 +2434,6 @@ BIF_RETTYPE binary_at_2(BIF_ALIST_2)
     BIF_ERROR(BIF_P,BADARG);
 }
 
-#define BIN_TO_LIST_OK 0
-#define BIN_TO_LIST_TRAP 1
-/* No badarg, checked before call */
-
-#define BIN_TO_LIST_LOOP_FACTOR 10
-
-static int do_bin_to_list(Process *p, byte *bytes, Uint bit_offs,
-			  Uint start, Sint *lenp, Eterm *termp)
-{
-    Uint reds = get_reds(p, BIN_TO_LIST_LOOP_FACTOR); /* reds can never be 0 */
-    Uint len = *lenp;
-    Uint loops;
-    Eterm *hp;
-    Eterm term = *termp;
-    Uint n;
-
-    ASSERT(reds > 0);
-
-    loops = MIN(reds,len);
-
-    BUMP_REDS(p, loops / BIN_TO_LIST_LOOP_FACTOR);
-
-    hp = HAlloc(p,2*loops);
-    while (loops--) {
-	--len;
-	if (bit_offs) {
-	    n = ((((Uint) bytes[start+len]) << bit_offs) |
-		 (((Uint) bytes[start+len+1]) >> (8-bit_offs))) & 0xFF;
-	} else {
-	    n = bytes[start+len];
-	}
-
-	term = CONS(hp,make_small(n),term);
-	hp +=2;
-    }
-    *termp = term;
-    *lenp = len;
-    if (len) {
-	BUMP_ALL_REDS(p);
-	return BIN_TO_LIST_TRAP;
-    }
-    return BIN_TO_LIST_OK;
-}
-
-
-static BIF_RETTYPE do_trap_bin_to_list(Process *p, Eterm binary,
-				       Uint start, Sint len, Eterm sofar)
-{
-    Eterm *hp;
-    Eterm blob;
-
-    hp = HAlloc(p,3);
-    hp[0] = make_pos_bignum_header(2);
-    hp[1] = start;
-    hp[2] = (Uint) len;
-    blob = make_big(hp);
-    BIF_TRAP3(&binary_bin_to_list_trap_export, p, binary, blob, sofar);
-}
-
-static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3)
-{
-    Eterm *ptr;
-    Uint start;
-    Sint len;
-    byte *bytes;
-    Uint bit_offs;
-    Uint bit_size;
-    Eterm res = BIF_ARG_3;
-
-    ptr  = big_val(BIF_ARG_2);
-    start = ptr[1];
-    len = (Sint) ptr[2];
-
-    ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size);
-    if (do_bin_to_list(BIF_P, bytes, bit_offs, start, &len, &res) ==
-	BIN_TO_LIST_OK) {
-	BIF_RET(res);
-    }
-    return do_trap_bin_to_list(BIF_P,BIF_ARG_1,start,len,res);
-}
-
-static BIF_RETTYPE binary_bin_to_list_common(Process *p,
-					     Eterm bin,
-					     Eterm epos,
-					     Eterm elen)
-{
-    Uint pos;
-    Sint len;
-    size_t sz;
-    byte *bytes;
-    Uint bit_offs;
-    Uint bit_size;
-    Eterm res = NIL;
-
-    if (is_not_binary(bin)) {
-	goto badarg;
-    }
-    if (!term_to_Uint(epos, &pos)) {
-	goto badarg;
-    }
-    if (!term_to_Sint(elen, &len)) {
-	goto badarg;
-    }
-    if (len < 0) {
-	Uint lentmp = -(Uint)len;
-	/* overflow */
-	if ((Sint)lentmp < 0) {
-	    goto badarg;
-	}
-	len = lentmp;
-	if (len > pos) {
-	    goto badarg;
-	}
-	pos -= len;
-    }
-    /* overflow */
-    if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) {
-	goto badarg;
-    }
-    sz = binary_size(bin);
-
-    if (pos+len > sz) {
-	goto badarg;
-    }
-    ERTS_GET_BINARY_BYTES(bin,bytes,bit_offs,bit_size);
-    if (bit_size != 0) {
-	goto badarg;
-    }
-    if(do_bin_to_list(p, bytes, bit_offs, pos, &len, &res) ==
-       BIN_TO_LIST_OK) {
-	BIF_RET(res);
-    }
-    return do_trap_bin_to_list(p,bin,pos,len,res);
-
- badarg:
-    BIF_ERROR(p,BADARG);
-}
-
-BIF_RETTYPE binary_bin_to_list_3(BIF_ALIST_3)
-{
-    return binary_bin_to_list_common(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3);
-}
-
-BIF_RETTYPE binary_bin_to_list_2(BIF_ALIST_2)
-{
-    Eterm *tp;
-
-    if (is_not_tuple(BIF_ARG_2)) {
-	goto badarg;
-    }
-    tp = tuple_val(BIF_ARG_2);
-    if (arityval(*tp) != 2) {
-	goto badarg;
-    }
-    return binary_bin_to_list_common(BIF_P,BIF_ARG_1,tp[1],tp[2]);
- badarg:
-    BIF_ERROR(BIF_P,BADARG);
-}
-
-BIF_RETTYPE binary_bin_to_list_1(BIF_ALIST_1)
-{
-    Uint pos = 0;
-    Sint len;
-    byte *bytes;
-    Uint bit_offs;
-    Uint bit_size;
-    Eterm res = NIL;
-
-    if (is_not_binary(BIF_ARG_1)) {
-	goto badarg;
-    }
-    len = binary_size(BIF_ARG_1);
-    ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size);
-    if (bit_size != 0) {
-	goto badarg;
-    }
-    if(do_bin_to_list(BIF_P, bytes, bit_offs, pos, &len, &res) ==
-       BIN_TO_LIST_OK) {
-	BIF_RET(res);
-    }
-    return do_trap_bin_to_list(BIF_P,BIF_ARG_1,pos,len,res);
- badarg:
-    BIF_ERROR(BIF_P,BADARG);
-}
-
 HIPE_WRAPPER_BIF_DISABLE_GC(binary_list_to_bin, 1)
 
 BIF_RETTYPE binary_list_to_bin_1(BIF_ALIST_1)
-- 
cgit v1.2.3


From ae3e6b6d18019a8f8d1c5804e48cddffc9cc3c2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 1 Mar 2018 14:50:12 +0100
Subject: Make efile_SUITE:iter_max_files more stable

---
 erts/emulator/test/efile_SUITE.erl | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl
index 16d581a567..821381bf0d 100644
--- a/erts/emulator/test/efile_SUITE.erl
+++ b/erts/emulator/test/efile_SUITE.erl
@@ -46,14 +46,14 @@ iter_max_files_1(Config) ->
     DataDir = proplists:get_value(data_dir,Config),
     TestFile = filename:join(DataDir, "existing_file"),
     N = 10,
-    %% Run on a different node in order to set the max ports
+    %% Run on a different node in order to make the test more stable.
     Dir = filename:dirname(code:which(?MODULE)),
     {ok,Node} = test_server:start_node(test_iter_max_files,slave,
-                                       [{args,"+Q 1524 -pa " ++ Dir}]),
+                                       [{args,"-pa " ++ Dir}]),
     L = rpc:call(Node,?MODULE,do_iter_max_files,[N, TestFile]),
     test_server:stop_node(Node),
     io:format("Number of files opened in each test:~n~w\n", [L]),
-    all_equal(L),
+    verify_max_files(L),
     Head = hd(L),
     if  Head >= 2 -> ok;
         true -> ct:fail(too_few_files)
@@ -65,12 +65,15 @@ do_iter_max_files(N, Name) when N > 0 ->
 do_iter_max_files(_, _) ->
     [].
 
-all_equal([E, E| T]) ->
-    all_equal([E| T]);
-all_equal([_]) ->
-    ok;
-all_equal([]) ->
-    ok.
+%% The attempts shouldn't vary too much; we used to require that they were all
+%% exactly equal, but after we reimplemented the file driver as a NIF we
+%% noticed that the only reason it was stable on Darwin was because the port
+%% limit was hit before ulimit.
+verify_max_files(Attempts) ->
+    N = length(Attempts),
+    Mean = lists:sum(Attempts) / N,
+    Variance = lists:sum([(X - Mean) * (X - Mean) || X <- Attempts]) / N,
+    true = math:sqrt(Variance) =< 1 + (Mean / 1000).
 
 max_files(Name) ->
     Fds = open_files(Name),
-- 
cgit v1.2.3


From 45a1da4e9dfdb912ea0bdff69d772618d8ce460f Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 2 Mar 2018 14:32:44 +0100
Subject: erts: Correctly ignore +secio true

This is a fix for a bug introduced in 22cde2bda
---
 erts/emulator/beam/erl_init.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 4846ccd2d3..e8048cfdfc 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -1746,6 +1746,7 @@ erl_start(int argc, char **argv)
 	    }
             else if (has_prefix("ecio", sub_param)) {
                 /* ignore argument, eager check io no longer used */
+                arg = get_arg(sub_param+4, argv[i+1], &i);
             }
 	    else if (has_prefix("pp", sub_param)) {
 		arg = get_arg(sub_param+2, argv[i+1], &i);
-- 
cgit v1.2.3


From 337919abe6ed9a97e15d5f471ef8f79d44cb8363 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 26 Feb 2018 15:36:34 +0100
Subject: erts,kernel: Add erts_internal:get_dflags/0

for kernel to ask erts about distribution flags
and keep this info in one place.
---
 erts/emulator/beam/atom.names        |  1 +
 erts/emulator/beam/bif.tab           |  1 +
 erts/emulator/beam/dist.c            | 21 ++++++++++++++++++++
 erts/emulator/beam/dist.h            | 37 ++++++++++++++++++++++++++++++------
 erts/emulator/beam/erl_node_tables.c |  2 +-
 5 files changed, 55 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 42a368cdd8..b40d9a325b 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -250,6 +250,7 @@ atom error_handler
 atom error_logger
 atom erts_code_purger
 atom erts_debug
+atom erts_dflags
 atom erts_internal
 atom ets
 atom ETS_TRANSFER='ETS-TRANSFER'
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 0d1166f6ed..be653ee2a0 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -690,6 +690,7 @@ bif erlang:iolist_to_iovec/1
 # New in 21.0
 #
 
+bif erts_internal:get_dflags/0
 bif erts_internal:new_connection/1
 bif erts_internal:abort_connection/2
 bif erts_internal:map_next/3
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 30390cdb5e..6f122273dc 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -641,6 +641,13 @@ trap_function(Eterm func, int arity)
     return erts_export_put(am_erlang, func, arity);
 }
 
+/*
+ * Sync with dist_util.erl:
+ *
+ * -record(erts_dflags, {default, mandatory, addable, rejectable}).
+ */
+static Eterm erts_dflags_record;
+
 void init_dist(void)
 {
     init_nodes_monitors();
@@ -657,6 +664,15 @@ void init_dist(void)
     dist_ctrl_put_data_trap = erts_export_put(am_erts_internal,
                                               am_dist_ctrl_put_data,
                                               2);
+    {
+        Eterm* hp = erts_alloc(ERTS_ALC_T_LITERAL, (1+5)*sizeof(Eterm));
+        erts_dflags_record = TUPLE5(hp, am_erts_dflags,
+                                    make_small(DFLAG_DIST_DEFAULT),
+                                    make_small(DFLAG_DIST_MANDATORY),
+                                    make_small(DFLAG_DIST_ADDABLE),
+                                    make_small(DFLAG_DIST_REJECTABLE));
+        erts_set_literal_tag(&erts_dflags_record, hp, (1+5));
+    }
 }
 
 #define ErtsDistOutputBuf2Binary(OB) \
@@ -3506,6 +3522,11 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     goto done;
 }
 
+BIF_RETTYPE erts_internal_get_dflags_0(BIF_ALIST_0)
+{
+    return erts_dflags_record;
+}
+
 BIF_RETTYPE erts_internal_new_connection_1(BIF_ALIST_1)
 {
     DistEntry* dep;
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index ea4697815f..202457bb3b 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -40,26 +40,51 @@
 #define DFLAG_UNICODE_IO          0x1000
 #define DFLAG_DIST_HDR_ATOM_CACHE 0x2000
 #define DFLAG_SMALL_ATOM_TAGS     0x4000
-#define DFLAG_INTERNAL_TAGS       0x8000
+#define DFLAG_INTERNAL_TAGS       0x8000   /* used by ETS 'compressed' option */
 #define DFLAG_UTF8_ATOMS          0x10000
 #define DFLAG_MAP_TAG             0x20000
 #define DFLAG_BIG_CREATION        0x40000
 #define DFLAG_SEND_SENDER         0x80000
-#define DFLAG_NO_MAGIC            0x100000
+#define DFLAG_NO_MAGIC            0x100000 /* internal for pending connection */
 
-/* Mandatory flags for distribution (sync with dist_util.erl) */
+/* Mandatory flags for distribution */
 #define DFLAG_DIST_MANDATORY (DFLAG_EXTENDED_REFERENCES         \
                               | DFLAG_EXTENDED_PIDS_PORTS       \
 			      | DFLAG_UTF8_ATOMS                \
 			      | DFLAG_NEW_FUN_TAGS)
 
-/* Additional optimistic flags when encoding toward pending connection */
-#define DFLAG_DIST_HOPEFULLY (DFLAG_NO_MAGIC                    \
-                              | DFLAG_EXPORT_PTR_TAG            \
+/*
+ * Additional optimistic flags when encoding toward pending connection.
+ * If remote node does not supporting these (erl_interface)
+ * then we will need to transcode all messages enqueued before
+ * connection setup was finished.
+ */
+#define DFLAG_DIST_HOPEFULLY (DFLAG_EXPORT_PTR_TAG              \
                               | DFLAG_BIT_BINARIES              \
                               | DFLAG_DIST_MONITOR              \
                               | DFLAG_DIST_MONITOR_NAME)
 
+/* Our preferred set of flags. Used for connection setup handshake */
+#define DFLAG_DIST_DEFAULT (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY \
+                            | DFLAG_FUN_TAGS                  \
+                            | DFLAG_NEW_FLOATS                \
+                            | DFLAG_UNICODE_IO                \
+                            | DFLAG_DIST_HDR_ATOM_CACHE       \
+                            | DFLAG_SMALL_ATOM_TAGS           \
+                            | DFLAG_UTF8_ATOMS                \
+                            | DFLAG_MAP_TAG                   \
+                            | DFLAG_BIG_CREATION              \
+                            | DFLAG_SEND_SENDER)
+
+/* Flags addable by local distr implementations */
+#define DFLAG_DIST_ADDABLE    DFLAG_DIST_DEFAULT
+
+/* Flags rejectable by local distr implementation */
+#define DFLAG_DIST_REJECTABLE (DFLAG_DIST_HDR_ATOM_CACHE         \
+                               | DFLAG_HIDDEN_ATOM_CACHE         \
+                               | DFLAG_ATOM_CACHE)
+
+
 /* All flags that should be enabled when term_to_binary/1 is used. */
 #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES	\
 			       | DFLAG_NEW_FUN_TAGS		\
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index eaf133f5c0..088b087ebb 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -628,7 +628,7 @@ erts_set_dist_entry_pending(DistEntry *dep)
     erts_no_of_not_connected_dist_entries--;
 
     dep->status = ERTS_DE_SFLG_PENDING;
-    dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY);
+    dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY | DFLAG_NO_MAGIC);
     dep->connection_id = (dep->connection_id + 1) & ERTS_DIST_CON_ID_MASK;
 
     dep->prev = NULL;
-- 
cgit v1.2.3


From 7fb3ed7d2731050186eb5224fe8e6050e4909341 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 26 Feb 2018 20:40:51 +0100
Subject: erts,kernel: Add dist_util:strict_order_flags/0

to replace DFLAGS_STRICT_ORDER_DELIVERY
and remove that compile time dependency.
---
 erts/emulator/beam/dist.c | 12 +++++++-----
 erts/emulator/beam/dist.h |  2 ++
 2 files changed, 9 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 6f122273dc..c08a8ec832 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -644,7 +644,8 @@ trap_function(Eterm func, int arity)
 /*
  * Sync with dist_util.erl:
  *
- * -record(erts_dflags, {default, mandatory, addable, rejectable}).
+ * -record(erts_dflags,
+ *         {default, mandatory, addable, rejectable, strict_order}).
  */
 static Eterm erts_dflags_record;
 
@@ -665,13 +666,14 @@ void init_dist(void)
                                               am_dist_ctrl_put_data,
                                               2);
     {
-        Eterm* hp = erts_alloc(ERTS_ALC_T_LITERAL, (1+5)*sizeof(Eterm));
-        erts_dflags_record = TUPLE5(hp, am_erts_dflags,
+        Eterm* hp = erts_alloc(ERTS_ALC_T_LITERAL, (1+6)*sizeof(Eterm));
+        erts_dflags_record = TUPLE6(hp, am_erts_dflags,
                                     make_small(DFLAG_DIST_DEFAULT),
                                     make_small(DFLAG_DIST_MANDATORY),
                                     make_small(DFLAG_DIST_ADDABLE),
-                                    make_small(DFLAG_DIST_REJECTABLE));
-        erts_set_literal_tag(&erts_dflags_record, hp, (1+5));
+                                    make_small(DFLAG_DIST_REJECTABLE),
+                                    make_small(DFLAG_DIST_STRICT_ORDER));
+        erts_set_literal_tag(&erts_dflags_record, hp, (1+6));
     }
 }
 
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 202457bb3b..e8dcdb669d 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -84,6 +84,8 @@
                                | DFLAG_HIDDEN_ATOM_CACHE         \
                                | DFLAG_ATOM_CACHE)
 
+/* Flags for all features needing strict order delivery */
+#define DFLAG_DIST_STRICT_ORDER DFLAG_DIST_HDR_ATOM_CACHE
 
 /* All flags that should be enabled when term_to_binary/1 is used. */
 #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES	\
-- 
cgit v1.2.3


From c10e3e1fc83cbccd1c8c3b377e2309ea8fa27572 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 1 Mar 2018 12:27:48 +0100
Subject: erts: Optimize dist transcoding toward erl_/jinterface

to only transcode if output buffer actually contains
unsupported BIT_BINARY_EXT or EXPORT_EXT.
---
 erts/emulator/beam/dist.c            |  2 ++
 erts/emulator/beam/dist.h            |  6 +++---
 erts/emulator/beam/erl_node_tables.h |  1 +
 erts/emulator/beam/external.c        | 23 ++++++++++++++++-------
 4 files changed, 22 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index c08a8ec832..132a0b9fba 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -2000,6 +2000,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
                 break;
             }
             ctx->u.ec.flags = ctx->flags;
+            ctx->u.ec.hopefull_flags = 0;
             ctx->u.ec.level = 0;
             ctx->u.ec.wstack.wstart = NULL;
             ctx->obuf->msg_start = ctx->obuf->ext_endp;
@@ -2023,6 +2024,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 
 	    ctx->data_size = ctx->obuf->ext_endp - ctx->obuf->extp;
 
+            ctx->obuf->hopefull_flags = ctx->u.ec.hopefull_flags;
 	    /*
 	     * Signal encoded; now verify that the connection still exists,
 	     * and if so enqueue the signal and schedule it for send.
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index e8dcdb669d..000c66a00f 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -55,9 +55,8 @@
 
 /*
  * Additional optimistic flags when encoding toward pending connection.
- * If remote node does not supporting these (erl_interface)
- * then we will need to transcode all messages enqueued before
- * connection setup was finished.
+ * If remote node (erl_interface) does not supporting these then we may need
+ * to transcode messages enqueued before connection setup was finished.
  */
 #define DFLAG_DIST_HOPEFULLY (DFLAG_EXPORT_PTR_TAG              \
                               | DFLAG_BIT_BINARIES              \
@@ -355,6 +354,7 @@ typedef struct TTBSizeContext_ {
 
 typedef struct TTBEncodeContext_ {
     Uint flags;
+    Uint hopefull_flags;
     int level;
     byte* ep;
     Eterm obj;
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 8d29c83e15..5822f97f55 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -86,6 +86,7 @@ struct ErtsDistOutputBuf_ {
     byte *alloc_endp;
 #endif
     ErtsDistOutputBuf *next;
+    Uint hopefull_flags;
     byte *extp;
     byte *ext_endp;
     byte *msg_start;
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index b12a021e41..b358685cc0 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2873,6 +2873,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		    ep[j] = 0;	/* Zero unused bits at end of binary */
 		    data_dst = ep;
 		    ep += j + 1;
+                    if (ctx)
+                        ctx->hopefull_flags |= DFLAG_BIT_BINARIES;
 		} else {
 		    /*
 		     * Bit-level binary, but the receiver doesn't support it.
@@ -2908,6 +2910,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
 		    ep = enc_atom(acmp, exp->info.mfa.function, ep, dflags);
 		    ep = enc_term(acmp, make_small(exp->info.mfa.arity),
                                   ep, dflags, off_heap);
+                    if (ctx)
+                        ctx->hopefull_flags |= DFLAG_EXPORT_PTR_TAG;
 		} else {
 		    /* Tag, arity */
 		    *ep++ = SMALL_TUPLE_EXT;
@@ -4729,11 +4733,13 @@ Sint transcode_dist_obuf(ErtsDistOutputBuf* ob,
     struct transcode_context* ctx = dep->transcode_ctx;
 
     if (!ctx) { /* first call for 'ob' */
-
-        if (~dflags & (DFLAG_BIT_BINARIES | DFLAG_EXPORT_PTR_TAG)) {
+	ASSERT(!(ob->hopefull_flags & ~(Uint)(DFLAG_BIT_BINARIES |
+					      DFLAG_EXPORT_PTR_TAG)));
+        if (~dflags & ob->hopefull_flags) {
             /*
-             * Receiver does not support bitstrings and/or export funs.
-             * We need to transcode control and message terms to use tuple fallbacks.
+             * Receiver does not support bitstrings and/or export funs
+             * and output buffer contains such message tags (hopefull_flags).
+             * Must transcode control and message terms to use tuple fallbacks.
              */
             ctx = erts_alloc(ERTS_ALC_T_DIST_TRANSCODE, sizeof(struct transcode_context));
             dep->transcode_ctx = ctx;
@@ -4760,7 +4766,7 @@ Sint transcode_dist_obuf(ErtsDistOutputBuf* ob,
                 ctx->state = TRANSCODE_ENC_CTL;
             }
         }
-        else {
+        else if (!(dflags & DFLAG_DIST_HDR_ATOM_CACHE)) {
             /*
              * No need for full transcoding, but primitive receiver (erl_/jinterface)
              * expects VERSION_MAGIC before both control and message terms.
@@ -4777,8 +4783,10 @@ Sint transcode_dist_obuf(ErtsDistOutputBuf* ob,
             *--(ob->extp) = VERSION_MAGIC;
             goto done;
         }
+        else
+            goto done;
     }
-    else {
+    else {  /* continue after yield */
         ASSERT(ctx->dbg_ob == ob);
     }
     ctx->b2t.reds = reds * B2T_BYTES_PER_REDUCTION;
@@ -4831,6 +4839,7 @@ Sint transcode_dist_obuf(ErtsDistOutputBuf* ob,
         ob->msg_start = ob->ext_endp;
         ctx->ttb.wstack.wstart = NULL;
         ctx->ttb.flags = dflags;
+        ctx->ttb.hopefull_flags = 0;
         ctx->ttb.level = 0;
 
         ctx->state = TRANSCODE_ENC_MSG;
@@ -4843,7 +4852,7 @@ Sint transcode_dist_obuf(ErtsDistOutputBuf* ob,
         reds /= TERM_TO_BINARY_LOOP_FACTOR;
 
         ASSERT(ob->ext_endp <= ob->alloc_endp);
-
+        ASSERT(!ctx->ttb.hopefull_flags);
     }
     transcode_free_ctx(dep);
 
-- 
cgit v1.2.3


From b31750711daa35011c62898d2eb424332e170bcc Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 1 Mar 2018 16:36:17 +0100
Subject: erts: Refactor DistEntry.status flags into a state

Just to simplify and get 4 distinctive states
IDLE, PENDING, CONNECTED and EXITING.

The old possible flag combos were:
0
PENDING
CONNECTED
CONNECTED|EXITING
EXITING

The two EXITING states did not serve any purpose
other then as a slight optimization in monitor_node(_,false,_)
to shortcut EXITING when there can be no monitors.
---
 erts/emulator/beam/dist.c            | 66 ++++++++++++++++++++----------------
 erts/emulator/beam/dist.h            | 18 +++-------
 erts/emulator/beam/erl_node_tables.c | 23 +++++++------
 erts/emulator/beam/erl_node_tables.h | 11 +++---
 erts/emulator/beam/external.c        |  4 +--
 5 files changed, 63 insertions(+), 59 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 132a0b9fba..f5491fd92a 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -588,13 +588,13 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
                 erts_port_task_abort(&dep->dist_cmd);
         }
 
-	if (dep->status & ERTS_DE_SFLG_EXITING) {
+	if (dep->state == ERTS_DE_STATE_EXITING) {
 #ifdef DEBUG
 	    ASSERT(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT);
 #endif
 	}
 	else {
-	    dep->status |= ERTS_DE_SFLG_EXITING;
+	    dep->state = ERTS_DE_STATE_EXITING;
 	    erts_mtx_lock(&dep->qlock);
 	    ASSERT(!(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT));
 	    erts_atomic32_read_bor_relb(&dep->qflgs, ERTS_DE_QFLG_EXIT);
@@ -785,7 +785,7 @@ static void clear_dist_entry(DistEntry *dep)
     erts_atomic64_set_nob(&dep->out, 0);
 
     obuf = clear_de_out_queues(dep);
-    dep->status = 0;
+    dep->state = ERTS_DE_STATE_IDLE;
     suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL);
 
     erts_mtx_unlock(&dep->qlock);
@@ -2032,8 +2032,8 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 	    ctx->obuf->next = NULL;
 	    erts_de_rlock(dep);
 	    cid = dep->cid;
-	    if (!(dep->status & (ERTS_DE_SFLG_PENDING | ERTS_DE_SFLG_CONNECTED))
-                || dep->status & ERTS_DE_SFLG_EXITING
+	    if (dep->state == ERTS_DE_STATE_EXITING
+                || dep->state == ERTS_DE_STATE_IDLE
                 || dep->connection_id != dsdp->connection_id) {
 		/* Not the same connection as when we started; drop message... */
 		erts_de_runlock(dep);
@@ -2108,7 +2108,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
 		}
 
 		erts_mtx_unlock(&dep->qlock);
-		if (!(dep->status & ERTS_DE_SFLG_PENDING)) {
+		if (dep->state != ERTS_DE_STATE_PENDING) {
                     if (is_internal_port(dep->cid))
                         erts_schedule_dist_command(NULL, dep);
                 }
@@ -2290,7 +2290,7 @@ int
 erts_dist_command(Port *prt, int initial_reds)
 {
     Sint reds = initial_reds - ERTS_PORT_REDS_DIST_CMD_START;
-    Uint32 status;
+    enum dist_entry_state state;
     Uint32 flags;
     Sint qsize, obufsize = 0;
     ErtsDistOutputQueue oq, foq;
@@ -2305,17 +2305,17 @@ erts_dist_command(Port *prt, int initial_reds)
 
     erts_de_rlock(dep);
     flags = dep->flags;
-    status = dep->status;
+    state = dep->state;
     send = dep->send;
     erts_de_runlock(dep);
 
-    if (status & ERTS_DE_SFLG_EXITING) {
+    if (state == ERTS_DE_STATE_EXITING) {
 	erts_deliver_port_exit(prt, prt->common.id, am_killed, 0, 1);
         reds -= ERTS_PORT_REDS_DIST_CMD_EXIT;
 	return initial_reds - reds;
     }
 
-    ASSERT(!(status & ERTS_DE_SFLG_PENDING));
+    ASSERT(state != ERTS_DE_STATE_PENDING);
 
     ASSERT(send);
 
@@ -2851,7 +2851,7 @@ dist_ctrl_get_data_1(BIF_ALIST_1)
 
     erts_de_rlock(dep);
 
-    if (dep->status & ERTS_DE_SFLG_EXITING)
+    if (dep->state == ERTS_DE_STATE_EXITING)
         goto return_none;
 
     ASSERT(dep->cid == BIF_P->common.id);
@@ -2954,9 +2954,9 @@ erts_dist_port_not_busy(Port *prt)
 static void kill_connection(DistEntry *dep)
 {
     ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep));
-    ASSERT(dep->status == ERTS_DE_SFLG_CONNECTED);
+    ASSERT(dep->state == ERTS_DE_STATE_CONNECTED);
 
-    dep->status |= ERTS_DE_SFLG_EXITING;
+    dep->state = ERTS_DE_STATE_EXITING;
     erts_mtx_lock(&dep->qlock);
     ASSERT(!(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT));
     erts_atomic32_read_bor_nob(&dep->qflgs, ERTS_DE_QFLG_EXIT);
@@ -2973,7 +2973,7 @@ erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id)
 {
     erts_de_rwlock(dep);
     if (connection_id == dep->connection_id
-        && dep->status == ERTS_DE_SFLG_CONNECTED) {
+        && dep->state == ERTS_DE_STATE_CONNECTED) {
 
         kill_connection(dep);
     }
@@ -3361,7 +3361,7 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
             goto badarg;
         }
 
-        if (dep->status & ERTS_DE_SFLG_EXITING) {
+        if (dep->state == ERTS_DE_STATE_EXITING) {
             /* Suspend on dist entry waiting for the exit to finish */
             ErtsProcList *plp = erts_proclist_create(BIF_P);
             plp->next = NULL;
@@ -3371,8 +3371,8 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
             erts_mtx_unlock(&dep->qlock);
             goto yield;
         }
-        if (dep->status != ERTS_DE_SFLG_PENDING) {
-            if (dep->status == 0)
+        if (dep->state != ERTS_DE_STATE_PENDING) {
+            if (dep->state == ERTS_DE_STATE_IDLE)
                 erts_set_dist_entry_pending(dep);
             else
                 goto badarg;
@@ -3410,7 +3410,7 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
             goto done; /* Already set */
         }
 
-        if (dep->status & ERTS_DE_SFLG_EXITING) {
+        if (dep->state == ERTS_DE_STATE_EXITING) {
             /* Suspend on dist entry waiting for the exit to finish */
             ErtsProcList *plp = erts_proclist_create(BIF_P);
             plp->next = NULL;
@@ -3420,8 +3420,8 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
             erts_mtx_unlock(&dep->qlock);
             goto yield;
         }
-        if (dep->status != ERTS_DE_SFLG_PENDING) {
-            if (dep->status == 0)
+        if (dep->state != ERTS_DE_STATE_PENDING) {
+            if (dep->state == ERTS_DE_STATE_IDLE)
                 erts_set_dist_entry_pending(dep);
             else
                 goto badarg;
@@ -3457,7 +3457,7 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
 
 #ifdef DEBUG
     ASSERT(erts_atomic_read_nob(&dep->qsize) == 0 
-           || (dep->status & ERTS_DE_SFLG_PENDING));
+           || (dep->state == ERTS_DE_STATE_PENDING));
 #endif
 
     if (flags & DFLAG_DIST_HDR_ATOM_CACHE)
@@ -3550,15 +3550,20 @@ BIF_RETTYPE erts_internal_new_connection_1(BIF_ALIST_1)
 
     erts_de_rwlock(dep);
 
-    if (ERTS_DE_IS_CONNECTED(dep) || dep->status & ERTS_DE_SFLG_PENDING)
+    switch (dep->state) {
+    case ERTS_DE_STATE_PENDING:
+    case ERTS_DE_STATE_CONNECTED:
 	conn_id = dep->connection_id;
-    else if (dep->status == 0) {
+        break;
+    case ERTS_DE_STATE_IDLE:
         erts_set_dist_entry_pending(dep);
 	conn_id = dep->connection_id;
-    }
-    else {
-        ASSERT(dep->status & ERTS_DE_SFLG_EXITING);
+        break;
+    case ERTS_DE_STATE_EXITING:
         conn_id = (dep->connection_id + 1) & ERTS_DIST_CON_ID_MASK;
+        break;
+    default:
+        erts_exit(ERTS_ABORT_EXIT, "Invalid dep->state (%d)\n", dep->state);
     }
     erts_de_rwunlock(dep);
     hp = HAlloc(BIF_P, 3 + ERTS_MAGIC_REF_THING_SIZE);
@@ -3573,10 +3578,10 @@ static Sint abort_connection(DistEntry* dep, Uint32 conn_id)
 
     if (dep->connection_id != conn_id)
         ;
-    else if (dep->status == ERTS_DE_SFLG_CONNECTED) {
+    else if (dep->state == ERTS_DE_STATE_CONNECTED) {
         kill_connection(dep);
     }
-    else if (dep->status == ERTS_DE_SFLG_PENDING) {
+    else if (dep->state == ERTS_DE_STATE_PENDING) {
         NetExitsContext nec = {dep};
         ErtsLink *nlinks;
         ErtsLink *node_links;
@@ -3656,7 +3661,7 @@ BIF_RETTYPE erts_internal_abort_connection_2(BIF_ALIST_2)
 int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks)
 {
     erts_de_rwlock(dep);
-    if (dep->status != 0) {
+    if (dep->state != ERTS_DE_STATE_IDLE) {
         erts_de_rwunlock(dep);
     }
     else {
@@ -3932,7 +3937,8 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options)
 
         erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
         erts_de_rlock(dep);
-        if (!(dep->status & (ERTS_DE_SFLG_PENDING | ERTS_DE_SFLG_CONNECTED))) {
+        if (dep->state == ERTS_DE_STATE_IDLE) {
+            ASSERT(!dep->node_links);
             erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
             erts_de_runlock(dep);
             goto done;
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 000c66a00f..b1b7ce9c78 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -137,14 +137,6 @@ typedef struct {
     int no_suspend;
 } ErtsDSigData;
 
-#define ERTS_DE_IS_NOT_CONNECTED(DEP) \
-  (ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&(DEP)->rwmtx) \
-		      || erts_lc_rwmtx_is_rwlocked(&(DEP)->rwmtx)), \
-   (is_nil((DEP)->cid) || ((DEP)->status & ERTS_DE_SFLG_EXITING)))
-
-#define ERTS_DE_IS_CONNECTED(DEP) \
-  (!ERTS_DE_IS_NOT_CONNECTED((DEP)))
-
 #define ERTS_DE_BUSY_LIMIT (1024*1024)
 extern int erts_dist_buf_busy_limit;
 extern int erts_is_alive;
@@ -207,18 +199,18 @@ erts_dsig_prepare(ErtsDSigData *dsdp,
 retry:
     erts_de_rlock(dep);
 
-    if (ERTS_DE_IS_CONNECTED(dep)) {
+    if (dep->state == ERTS_DE_STATE_CONNECTED) {
 	res = ERTS_DSIG_PREP_CONNECTED;
     }
-    else if (dep->status & ERTS_DE_SFLG_PENDING) {
+    else if (dep->state == ERTS_DE_STATE_PENDING) {
 	res = ERTS_DSIG_PREP_PENDING;
     }
-    else if (dep->status & ERTS_DE_SFLG_EXITING) {
+    else if (dep->state == ERTS_DE_STATE_EXITING) {
 	res = ERTS_DSIG_PREP_NOT_CONNECTED;
 	goto fail;
     }
     else if (connect) {
-        ASSERT(dep->status == 0);
+        ASSERT(dep->state == ERTS_DE_STATE_IDLE);
         erts_de_runlock(dep);
         if (!erts_auto_connect(dep, proc, proc_locks)) {
             return ERTS_DSIG_PREP_NOT_ALIVE;
@@ -226,7 +218,7 @@ retry:
 	goto retry;
     }
     else {
-        ASSERT(dep->status == 0);
+        ASSERT(dep->state == ERTS_DE_STATE_IDLE);
 	res = ERTS_DSIG_PREP_NOT_CONNECTED;
 	goto fail;
     }
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 088b087ebb..9871965ba6 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -170,7 +170,7 @@ dist_table_alloc(void *dep_tmpl)
     dep->cid				= NIL;
     erts_atomic_init_nob(&dep->input_handler, (erts_aint_t) NIL);
     dep->connection_id			= 0;
-    dep->status				= 0;
+    dep->state				= ERTS_DE_STATE_IDLE;
     dep->flags				= 0;
     dep->version			= 0;
 
@@ -223,7 +223,7 @@ dist_table_free(void *vdep)
     DistEntry *dep = (DistEntry *) vdep;
 
     ASSERT(de_refc_read(dep, -1) == -1);
-    ASSERT(dep->status == 0);
+    ASSERT(dep->state == ERTS_DE_STATE_IDLE);
     ASSERT(is_nil(dep->cid));
     ASSERT(dep->nlinks == NULL);
     ASSERT(dep->node_links == NULL);
@@ -556,14 +556,18 @@ erts_set_dist_entry_not_connected(DistEntry *dep)
 
     ASSERT(dep != erts_this_dist_entry);
 
-    if (dep->status & ERTS_DE_SFLG_PENDING) {
+    if (dep->state == ERTS_DE_STATE_PENDING) {
         ASSERT(is_nil(dep->cid));
         ASSERT(erts_no_of_pending_dist_entries > 0);
         erts_no_of_pending_dist_entries--;
         head = &erts_pending_dist_entries;
+
+        // Todo: Is this really ok? Must be not wait for links and monitors
+        // to be fired before we can allow another connection.
+        dep->state = ERTS_DE_STATE_IDLE;
     }
     else {
-        ASSERT(dep->status != 0);
+        ASSERT(dep->state != ERTS_DE_STATE_IDLE);
         ASSERT(is_internal_port(dep->cid) || is_internal_pid(dep->cid));
         if (dep->flags & DFLAG_PUBLISHED) {
             ASSERT(erts_no_of_visible_dist_entries > 0);
@@ -575,6 +579,7 @@ erts_set_dist_entry_not_connected(DistEntry *dep)
             erts_no_of_hidden_dist_entries--;
             head = &erts_hidden_dist_entries;
         }
+        dep->state = ERTS_DE_STATE_EXITING;
     }
 
     if(dep->prev) {
@@ -588,7 +593,6 @@ erts_set_dist_entry_not_connected(DistEntry *dep)
     if(dep->next)
 	dep->next->prev = dep->prev;
 
-    dep->status &= ~(ERTS_DE_SFLG_PENDING | ERTS_DE_SFLG_CONNECTED);
     dep->flags = 0;
     dep->prev = NULL;
     dep->cid = NIL;
@@ -610,7 +614,7 @@ erts_set_dist_entry_pending(DistEntry *dep)
     erts_rwmtx_rwlock(&erts_dist_table_rwmtx);
 
     ASSERT(dep != erts_this_dist_entry);
-    ASSERT(dep->status == 0);
+    ASSERT(dep->state == ERTS_DE_STATE_IDLE);
     ASSERT(is_nil(dep->cid));
 
     if(dep->prev) {
@@ -627,7 +631,7 @@ erts_set_dist_entry_pending(DistEntry *dep)
 
     erts_no_of_not_connected_dist_entries--;
 
-    dep->status = ERTS_DE_SFLG_PENDING;
+    dep->state = ERTS_DE_STATE_PENDING;
     dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY | DFLAG_NO_MAGIC);
     dep->connection_id = (dep->connection_id + 1) & ERTS_DIST_CON_ID_MASK;
 
@@ -652,7 +656,7 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags)
 
     ASSERT(dep != erts_this_dist_entry);
     ASSERT(is_nil(dep->cid));
-    ASSERT(dep->status & ERTS_DE_SFLG_PENDING);
+    ASSERT(dep->state == ERTS_DE_STATE_PENDING);
     ASSERT(is_internal_port(cid) || is_internal_pid(cid));
 
     if(dep->prev) {
@@ -670,8 +674,7 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags)
     ASSERT(erts_no_of_pending_dist_entries > 0);
     erts_no_of_pending_dist_entries--;
 
-    dep->status &= ~ERTS_DE_SFLG_PENDING;
-    dep->status |= ERTS_DE_SFLG_CONNECTED;
+    dep->state = ERTS_DE_STATE_CONNECTED;
     dep->flags = flags & ~DFLAG_NO_MAGIC;
     dep->cid = cid;
     erts_atomic_set_nob(&dep->input_handler,
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 5822f97f55..58279017c8 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -57,9 +57,12 @@
  
 #define ERST_INTERNAL_CHANNEL_NO 0
 
-#define ERTS_DE_SFLG_PENDING			(((Uint32) 1) <<  0)
-#define ERTS_DE_SFLG_CONNECTED			(((Uint32) 1) <<  1)
-#define ERTS_DE_SFLG_EXITING			(((Uint32) 1) <<  2)
+enum dist_entry_state {
+    ERTS_DE_STATE_IDLE,
+    ERTS_DE_STATE_PENDING,
+    ERTS_DE_STATE_CONNECTED,
+    ERTS_DE_STATE_EXITING
+};
 
 #define ERTS_DE_QFLG_BUSY			(((erts_aint32_t) 1) <<  0)
 #define ERTS_DE_QFLG_EXIT			(((erts_aint32_t) 1) <<  1)
@@ -122,7 +125,7 @@ typedef struct dist_entry_ {
     Eterm cid;			/* connection handler (pid or port),
                                    NIL == free */
     Uint32 connection_id;	/* Connection id incremented on connect */
-    Uint32 status;		/* Slot status, like exiting reserved etc */
+    enum dist_entry_state state;
     Uint32 flags;		/* Distribution flags, like hidden, 
 				   atom cache etc. */
     unsigned long version;	/* Protocol version */
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index b358685cc0..314005e18d 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -711,8 +711,8 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
 
     erts_de_rlock(dep);
 
-    if (dep->status != ERTS_DE_SFLG_CONNECTED &&
-	dep->status != ERTS_DE_SFLG_PENDING) {
+    if (dep->state != ERTS_DE_STATE_CONNECTED &&
+	dep->state != ERTS_DE_STATE_PENDING) {
         erts_de_runlock(dep);
         return ERTS_PREP_DIST_EXT_CLOSED;
     }
-- 
cgit v1.2.3


From 96aba63465135d5e2537b42e5167b2863957915c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 1 Mar 2018 17:56:01 +0100
Subject: erts: Postpone idle DistEntry until abort is completed

---
 erts/emulator/beam/dist.c            | 11 +++++++++++
 erts/emulator/beam/erl_node_tables.c |  6 +-----
 2 files changed, 12 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index f5491fd92a..cd799e04b8 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -3630,6 +3630,17 @@ static Sint abort_connection(DistEntry* dep, Uint32 conn_id)
 
 	delete_cache(cache);
 	free_de_out_queues(dep, obuf);
+
+        /*
+         * We wait to make DistEntry idle and accept new connection attempts
+         * until all is cleared and deallocated. This to get some back pressure
+         * against repeated failing connection attempts saturating all CPUs
+         * with cleanup jobs.
+         */
+        erts_de_rwlock(dep);
+        ASSERT(dep->state == ERTS_DE_STATE_EXITING);
+        dep->state = ERTS_DE_STATE_IDLE;
+        erts_de_rwunlock(dep);
         return reds;
     }
     erts_de_rwunlock(dep);
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 9871965ba6..e8901a652f 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -561,10 +561,6 @@ erts_set_dist_entry_not_connected(DistEntry *dep)
         ASSERT(erts_no_of_pending_dist_entries > 0);
         erts_no_of_pending_dist_entries--;
         head = &erts_pending_dist_entries;
-
-        // Todo: Is this really ok? Must be not wait for links and monitors
-        // to be fired before we can allow another connection.
-        dep->state = ERTS_DE_STATE_IDLE;
     }
     else {
         ASSERT(dep->state != ERTS_DE_STATE_IDLE);
@@ -579,7 +575,6 @@ erts_set_dist_entry_not_connected(DistEntry *dep)
             erts_no_of_hidden_dist_entries--;
             head = &erts_hidden_dist_entries;
         }
-        dep->state = ERTS_DE_STATE_EXITING;
     }
 
     if(dep->prev) {
@@ -593,6 +588,7 @@ erts_set_dist_entry_not_connected(DistEntry *dep)
     if(dep->next)
 	dep->next->prev = dep->prev;
 
+    dep->state = ERTS_DE_STATE_EXITING;
     dep->flags = 0;
     dep->prev = NULL;
     dep->cid = NIL;
-- 
cgit v1.2.3


From 12e6a19037afe9bcab53b1429b68e35c4eb7bafb Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 23 Oct 2017 19:21:07 +0200
Subject: Remove ERTS_PSFLG_ON_HEAP_MSGQ

---
 erts/emulator/beam/erl_message.c      | 6 +-----
 erts/emulator/beam/erl_process.c      | 6 ------
 erts/emulator/beam/erl_process.h      | 2 +-
 erts/emulator/beam/erl_process_dump.c | 2 --
 4 files changed, 2 insertions(+), 14 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index abf194cf94..6f7c71ef98 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -616,7 +616,7 @@ erts_try_alloc_message_on_heap(Process *pp,
     }
     else {
     in_message_fragment:
-	if (!((*psp) & ERTS_PSFLG_ON_HEAP_MSGQ)) {
+	if ((*psp) & ERTS_PSFLG_OFF_HEAP_MSGQ) {
 	    mp = erts_alloc_message(sz, hpp);
 	    *ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap;
 	}
@@ -1079,8 +1079,6 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state)
 	case am_on_heap:
 	    c_p->flags |= F_ON_HEAP_MSGQ;
 	    c_p->flags &= ~F_OFF_HEAP_MSGQ;
-	    erts_atomic32_read_bor_nob(&c_p->state,
-					   ERTS_PSFLG_ON_HEAP_MSGQ);
 	    /*
 	     * We are not allowed to clear ERTS_PSFLG_OFF_HEAP_MSGQ
 	     * if a off heap change is ongoing. It will be adjusted
@@ -1106,8 +1104,6 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state)
 	    break;
 	case am_off_heap:
 	    c_p->flags &= ~F_ON_HEAP_MSGQ;
-	    erts_atomic32_read_band_nob(&c_p->state,
-					    ~ERTS_PSFLG_ON_HEAP_MSGQ);
 	    goto change_to_off_heap;
 	default:
 	    res = THE_NON_VALUE; /* badarg */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 3baca6ce79..ea60013c52 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11593,7 +11593,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	flags |= F_OFF_HEAP_MSGQ;
     }
     else if (so->flags & SPO_ON_HEAP_MSGQ) {
-	state |= ERTS_PSFLG_ON_HEAP_MSGQ;
 	flags |= F_ON_HEAP_MSGQ;
     }
 
@@ -11611,11 +11610,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	goto error;
     }
 
-    ASSERT((erts_atomic32_read_nob(&p->state)
-	    & ERTS_PSFLG_ON_HEAP_MSGQ)
-	   || (erts_atomic32_read_nob(&p->state)
-	       & ERTS_PSFLG_OFF_HEAP_MSGQ));
-
 #ifdef SHCOPY_SPAWN
     arg_size = copy_shared_calculate(args, &info);
 #else
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 4f200b13ec..56bed00b40 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1175,7 +1175,7 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
 #define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
 #define ERTS_PSFLG_OFF_HEAP_MSGQ	ERTS_PSFLG_BIT(18)
-#define ERTS_PSFLG_ON_HEAP_MSGQ		ERTS_PSFLG_BIT(19)
+/* #define ERTS_PSFLG_			ERTS_PSFLG_BIT(19) */
 #define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(20)
 #define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(21)
 #define ERTS_PSFLG_DIRTY_ACTIVE_SYS	ERTS_PSFLG_BIT(22)
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 0d76a4ef53..01a1965836 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -1023,8 +1023,6 @@ erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg)
                 erts_print(to, to_arg, "DELAYED_SYS"); break;
             case ERTS_PSFLG_OFF_HEAP_MSGQ:
                 erts_print(to, to_arg, "OFF_HEAP_MSGQ"); break;
-            case ERTS_PSFLG_ON_HEAP_MSGQ:
-                erts_print(to, to_arg, "ON_HEAP_MSGQ"); break;
             case ERTS_PSFLG_DIRTY_CPU_PROC:
                 erts_print(to, to_arg, "DIRTY_CPU_PROC"); break;
             case ERTS_PSFLG_DIRTY_IO_PROC:
-- 
cgit v1.2.3


From fbb10ebc4a37555c7ea7f99e14286d862993976a Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 24 Oct 2017 14:24:45 +0200
Subject: Replace usage of ERTS_PSFLG_BOUND

---
 erts/emulator/beam/bif.c              |  15 ++--
 erts/emulator/beam/erl_port.h         |  38 +++++++--
 erts/emulator/beam/erl_port_task.c    |  25 ++----
 erts/emulator/beam/erl_process.c      | 155 +++++++++++-----------------------
 erts/emulator/beam/erl_process.h      | 150 ++++++++++++++++++++++++++++++--
 erts/emulator/beam/erl_process_dump.c |   2 -
 erts/emulator/beam/io.c               |  16 ++--
 7 files changed, 253 insertions(+), 148 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index d68ccc3028..bfd572335f 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1744,7 +1744,6 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
    else if (BIF_ARG_1 == am_scheduler) {
        ErtsRunQueue *old, *new, *curr;
        Sint sched;
-       erts_aint32_t state;
 
        if (!is_small(BIF_ARG_2))
 	   goto error;
@@ -1753,23 +1752,23 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
 	   goto error;
 
        if (sched == 0) {
+           old = erts_bind_runq_proc(BIF_P, 0);
 	   new = NULL;
-	   state = erts_atomic32_read_band_mb(&BIF_P->state,
-						  ~ERTS_PSFLG_BOUND);
        }
        else {
+           int bound = !0;
 	   new = erts_schedid2runq(sched);
-	   erts_atomic_set_nob(&BIF_P->run_queue, (erts_aint_t) new);
-	   state = erts_atomic32_read_bor_mb(&BIF_P->state,
-						 ERTS_PSFLG_BOUND);
+           old = erts_set_runq_proc(BIF_P, new, &bound);
+           if (!bound)
+               old = NULL;
        }
 
+       old_value = old ? make_small(old->ix+1) : make_small(0);
+
        curr = erts_proc_sched_data(BIF_P)->run_queue;
-       old = (ERTS_PSFLG_BOUND & state) ? curr : NULL;
 
        ASSERT(!old || old == curr);
 
-       old_value = old ? make_small(old->ix+1) : make_small(0);
        if (new && new != curr)
 	   ERTS_BIF_YIELD_RETURN_X(BIF_P, old_value, am_scheduler);
        else
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index 6b90187d60..0d148ee048 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -196,26 +196,52 @@ struct erl_drv_port_data_lock {
     Port *prt;
 };
 
+ERTS_GLB_INLINE void erts_init_runq_port(Port *prt, ErtsRunQueue *runq);
+ERTS_GLB_INLINE void erts_set_runq_port(Port *prt, ErtsRunQueue *runq);
+ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_port(Port *prt);
 ERTS_GLB_INLINE ErtsRunQueue *erts_port_runq(Port *prt);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
+ERTS_GLB_INLINE void
+erts_init_runq_port(Port *prt, ErtsRunQueue *runq)
+{
+    if (!runq)
+        ERTS_INTERNAL_ERROR("Missing run-queue");
+    erts_atomic_init_nob(&prt->run_queue, (erts_aint_t) runq);
+}
+
+ERTS_GLB_INLINE void
+erts_set_runq_port(Port *prt, ErtsRunQueue *runq)
+{
+    if (!runq)
+        ERTS_INTERNAL_ERROR("Missing run-queue");
+    erts_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq);
+}
+
+ERTS_GLB_INLINE ErtsRunQueue *
+erts_get_runq_port(Port *prt)
+{
+    ErtsRunQueue *runq;
+    runq = (ErtsRunQueue *) erts_atomic_read_nob(&prt->run_queue);
+    if (!runq)
+        ERTS_INTERNAL_ERROR("Missing run-queue");
+    return runq;
+}
+
+
 ERTS_GLB_INLINE ErtsRunQueue *
 erts_port_runq(Port *prt)
 {
     ErtsRunQueue *rq1, *rq2;
-    rq1 = (ErtsRunQueue *) erts_atomic_read_nob(&prt->run_queue);
-    if (!rq1)
-	return NULL;
+    rq1 = erts_get_runq_port(prt);
     while (1) {
 	erts_runq_lock(rq1);
-	rq2 = (ErtsRunQueue *) erts_atomic_read_nob(&prt->run_queue);
+	rq2 = erts_get_runq_port(prt);
 	if (rq1 == rq2)
 	    return rq1;
 	erts_runq_unlock(rq1);
 	rq1 = rq2;
-	if (!rq1)
-	    return NULL;
     }
 }
 
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index a588477320..4a3671df0c 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -84,11 +84,10 @@ static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_q
 #define LTTNG_DRIVER(TRACEPOINT, PP) do {} while(0)
 #endif
 
-#define ERTS_LC_VERIFY_RQ(RQ, PP)					\
-    do {								\
+#define ERTS_LC_VERIFY_RQ(RQ, PP)				\
+    do {							\
 	ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq));		\
-	ERTS_LC_ASSERT((RQ) == ((ErtsRunQueue *)			\
-				    erts_atomic_read_nob(&(PP)->run_queue))); \
+	ERTS_LC_ASSERT((RQ) == erts_get_runq_port((PP)));       \
     } while (0)
 
 #define ERTS_PT_STATE_SCHEDULED		0
@@ -1520,19 +1519,15 @@ erts_port_task_schedule(Eterm id,
     /* Enqueue port on run-queue */
 
     runq = erts_port_runq(pp);
-    if (!runq)
-	ERTS_INTERNAL_ERROR("Missing run-queue");
 
     xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL);
     ERTS_LC_ASSERT(runq != xrunq);
     ERTS_LC_VERIFY_RQ(runq, pp);
     if (xrunq) {
 	/* Emigrate port ... */
-	erts_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq);
+        erts_set_runq_port(pp, xrunq);
 	erts_runq_unlock(runq);
 	runq = erts_port_runq(pp);
-	if (!runq)
-	    ERTS_INTERNAL_ERROR("Missing run-queue");
     }
 
     enqueue_port(runq, pp);
@@ -1593,8 +1588,6 @@ erts_port_task_free_port(Port *pp)
     ASSERT(!(erts_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD));
 
     runq = erts_port_runq(pp);
-    if (!runq)
-	ERTS_INTERNAL_ERROR("Missing run-queue");
     erts_port_task_sched_lock(&pp->sched);
     flags = erts_atomic32_read_bor_relb(&pp->sched.flags,
 					    ERTS_PTS_FLG_EXIT);
@@ -1805,7 +1798,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     erts_unblock_fpe(fpe_was_unmasked);
     ERTS_MSACC_POP_STATE_M();
 
-    ASSERT(runq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue));
+    ASSERT(runq == erts_get_runq_port(pp));
 
     active = finalize_exec(pp, &execq, processing_busy_q);
 
@@ -1831,11 +1824,10 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	}
 	else {
 	    /* Emigrate port... */
-	    erts_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq);
+            erts_set_runq_port(pp, xrunq);
 	    erts_runq_unlock(runq);
 
 	    xrunq = erts_port_runq(pp);
-	    ASSERT(xrunq);
 	    enqueue_port(xrunq, pp);
 	    erts_runq_unlock(xrunq);
 	    erts_notify_inc_runq(xrunq);
@@ -2069,7 +2061,7 @@ void
 erts_enqueue_port(ErtsRunQueue *rq, Port *pp)
 {
     ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
-    ASSERT(rq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue));
+    ASSERT(rq == erts_get_runq_port(pp));
     ASSERT(erts_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_IN_RUNQ);
     enqueue_port(rq, pp);
 }
@@ -2080,8 +2072,7 @@ erts_dequeue_port(ErtsRunQueue *rq)
     Port *pp;
     ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
     pp = pop_port(rq);
-    ASSERT(!pp
-	   || rq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue));
+    ASSERT(!pp || rq == erts_get_runq_port(pp));
     ASSERT(!pp || (erts_atomic32_read_nob(&pp->sched.flags)
 		   & ERTS_PTS_FLG_IN_RUNQ));
     return pp;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index ea60013c52..b19659f496 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -137,36 +137,6 @@ runq_got_work_to_execute(ErtsRunQueue *rq)
     return runq_got_work_to_execute_flags(ERTS_RUNQ_FLGS_GET_NOB(rq));
 }
 
-#undef RUNQ_READ_RQ
-#undef RUNQ_SET_RQ
-#define RUNQ_READ_RQ(X) ((ErtsRunQueue *) erts_atomic_read_nob((X)))
-#define RUNQ_SET_RQ(X, RQ) erts_atomic_set_nob((X), (erts_aint_t) (RQ))
-
-#ifdef DEBUG
-#  if defined(ARCH_64)
-#    define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \
-    (RUNQ_SET_RQ((RQP), (0xdeadbeefdead0003LL | ((N) << 4)))
-#  define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \
-do { \
-    ASSERT((RQP) != NULL); \
-    ASSERT(((((Uint) (RQP)) & ((Uint) 0x3))) == ((Uint) 0)); \
-    ASSERT((((Uint) (RQP)) & ~((Uint) 0xffff)) != ((Uint) 0xdeadbeefdead0000LL));\
-} while (0)
-#  else
-#    define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \
-    (RUNQ_SET_RQ((RQP), (0xdead0003 | ((N) << 4))))
-#  define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \
-do { \
-    ASSERT((RQP) != NULL); \
-    ASSERT(((((UWord) (RQP)) & ((UWord) 1))) == ((UWord) 0)); \
-    ASSERT((((UWord) (RQP)) & ~((UWord) 0xffff)) != ((UWord) 0xdead0000)); \
-} while (0)
-#  endif
-#else
-#  define ERTS_DBG_SET_INVALID_RUNQP(RQP, N)
-#  define ERTS_DBG_VERIFY_VALID_RUNQP(RQP)
-#endif
-
 const Process erts_invalid_process = {{ERTS_INVALID_PID}};
 
 extern BeamInstr beam_apply[];
@@ -3940,21 +3910,16 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp)
 		Port *prt;
 		prt = erts_dequeue_port(rq);
 		if (prt)
-		    RUNQ_SET_RQ(&prt->run_queue, c_rq);
+                    erts_set_runq_port(prt, c_rq);
 		erts_runq_unlock(rq);
 		if (prt) {
-		    /* port might terminate while we have no lock... */
 		    rq = erts_port_runq(prt);
-		    if (rq) {
-			if (rq != c_rq)
-			    erts_exit(ERTS_ABORT_EXIT,
-				     "%s:%d:%s(): Internal error",
-				     __FILE__, __LINE__, __func__);
-			erts_enqueue_port(c_rq, prt);
-			if (!iflag)
-			    return; /* done */
-			erts_runq_unlock(c_rq);
-		    }
+                    if (rq != c_rq)
+                        ERTS_INTERNAL_ERROR("Unexpected run-queue");
+                    erts_enqueue_port(c_rq, prt);
+                    if (!iflag)
+                        return; /* done */
+                    erts_runq_unlock(c_rq);
 		}
 	    }
 	    else {
@@ -3968,12 +3933,11 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp)
 		while (proc) {
 		    erts_aint32_t state;
 		    state = erts_atomic32_read_acqb(&proc->state);
-		    if (!(ERTS_PSFLG_BOUND & state)
-			&& (prio == (int) ERTS_PSFLGS_GET_PRQ_PRIO(state))) {
+		    if (prio == (int) ERTS_PSFLGS_GET_PRQ_PRIO(state)
+                        && erts_try_change_runq_proc(proc, c_rq)) {
 			ErtsRunQueueInfo *rqi = &rq->procs.prio_info[prio];
 			unqueue_process(rq, rpq, rqi, prio, prev_proc, proc);
 			erts_runq_unlock(rq);
-			RUNQ_SET_RQ(&proc->run_queue, c_rq);
 			rq_locked = 0;
 
 			erts_runq_lock(c_rq);
@@ -4154,21 +4118,13 @@ evacuate_run_queue(ErtsRunQueue *rq,
 	while (prt) {
 	    ErtsRunQueue *prt_rq;
 	    prt = erts_dequeue_port(rq);
-	    RUNQ_SET_RQ(&prt->run_queue, to_rq);
+            erts_set_runq_port(prt, to_rq);
 	    erts_runq_unlock(rq);
-	    /*
-	     * The port might terminate while
-	     * we have no lock on it...
-	     */
 	    prt_rq = erts_port_runq(prt);
-	    if (prt_rq) {
-		if (prt_rq != to_rq)
-		    erts_exit(ERTS_ABORT_EXIT,
-			     "%s:%d:%s() internal error\n",
-			     __FILE__, __LINE__, __func__);
-		erts_enqueue_port(to_rq, prt);
-		erts_runq_unlock(to_rq);
-	    }
+            if (prt_rq != to_rq)
+                ERTS_INTERNAL_ERROR("Unexpected run-queue");
+            erts_enqueue_port(to_rq, prt);
+            erts_runq_unlock(to_rq);
 	    erts_runq_lock(rq);
 	    prt = rq->ports.start;
 	}
@@ -4191,14 +4147,13 @@ evacuate_run_queue(ErtsRunQueue *rq,
 	while (proc) {
 	    Process *real_proc;
 	    int prio;
-	    erts_aint32_t max_qbit, qbit, real_state;
+	    erts_aint32_t max_qbit, qbit;
 
 	    prio = ERTS_PSFLGS_GET_PRQ_PRIO(state);
 	    qbit = ((erts_aint32_t) 1) << prio;
 
 	    if (!(state & ERTS_PSFLG_PROXY)) {
 		real_proc = proc;
-		real_state = state;
 	    }
 	    else {
 		real_proc = erts_proc_lookup_raw(proc->common.id);
@@ -4206,7 +4161,6 @@ evacuate_run_queue(ErtsRunQueue *rq,
 		    free_proxy_proc(proc);
 		    goto handle_next_proc;
 		}
-		real_state =  erts_atomic32_read_acqb(&real_proc->state);
 	    }
 
 	    max_qbit = (state >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET);
@@ -4241,7 +4195,7 @@ evacuate_run_queue(ErtsRunQueue *rq,
 		goto handle_next_proc;
 	    }
 
-	    if (ERTS_PSFLG_BOUND & real_state) {
+	    if (!erts_try_change_runq_proc(proc, to_rq)) {
 		/* Bound processes get stuck here... */
 		proc->next = NULL;
 		if (sbpp->last)
@@ -4255,7 +4209,6 @@ evacuate_run_queue(ErtsRunQueue *rq,
 		erts_runq_unlock(rq);
 
                 to_rq = mp->prio[prio].runq;
-		RUNQ_SET_RQ(&proc->run_queue, to_rq);
 
 		erts_runq_lock(to_rq);
 		enqueue_process(to_rq, prio, proc);
@@ -4323,14 +4276,13 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq,
 	proc = rpq->first;
 
 	while (proc) {
-	    erts_aint32_t state = erts_atomic32_read_acqb(&proc->state);
-	    if (!(ERTS_PSFLG_BOUND & state)) {
+	    if (erts_try_change_runq_proc(proc, rq)) {
+                erts_aint32_t state = erts_atomic32_read_acqb(&proc->state);
 		/* Steal process */
 		int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state);
 		ErtsRunQueueInfo *rqi = &vrq->procs.prio_info[prio];
 		unqueue_process(vrq, rpq, rqi, prio, prev_proc, proc);
 		erts_runq_unlock(vrq);
-		RUNQ_SET_RQ(&proc->run_queue, rq);
 
 		erts_runq_lock(rq);
 		*rq_lockedp = 1;
@@ -4355,26 +4307,14 @@ no_procs:
     if (vrq->ports.start) {
 	ErtsRunQueue *prt_rq;
 	Port *prt = erts_dequeue_port(vrq);
-	RUNQ_SET_RQ(&prt->run_queue, rq);
+	erts_set_runq_port(prt, rq);
 	erts_runq_unlock(vrq);
-
-	/*
-	 * The port might terminate while
-	 * we have no lock on it...
-	 */
-
 	prt_rq = erts_port_runq(prt);
-	if (!prt_rq)
-	    return 0;
-	else {
-	    if (prt_rq != rq)
-		erts_exit(ERTS_ABORT_EXIT,
-			 "%s:%d:%s() internal error\n",
-			 __FILE__, __LINE__, __func__);
-	    *rq_lockedp = 1;
-	    erts_enqueue_port(rq, prt);
-	    return !0;
-	}
+        if (prt_rq != rq)
+            ERTS_INTERNAL_ERROR("Unexpected run-queue");
+        *rq_lockedp = 1;
+        erts_enqueue_port(rq, prt);
+        return !0;
     }
 
     erts_runq_unlock(vrq);
@@ -6130,7 +6070,8 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
 {
     erts_aint32_t state;
     Process *proxy;
-    ErtsRunQueue *rq = RUNQ_READ_RQ(&proc->run_queue);
+    int bound;
+    ErtsRunQueue *rq = erts_get_runq_proc(proc, &bound);
 
     state = (ERTS_PSFLG_PROXY
 	     | ERTS_PSFLG_IN_RUNQ
@@ -6143,7 +6084,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
 	proxy = prev_proxy;
 	ASSERT(erts_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY);
 	erts_atomic32_set_nob(&proxy->state, state);
-	RUNQ_SET_RQ(&proc->run_queue, rq);
+        (void) erts_set_runq_proc(proc, rq, &bound);
     }
     else {
 	proxy = erts_alloc(ERTS_ALC_T_PROC, sizeof(Process));
@@ -6156,8 +6097,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
 	}
 #endif
 	erts_atomic32_init_nob(&proxy->state, state);
-	erts_atomic_init_nob(&proxy->run_queue,
-				 erts_atomic_read_nob(&proc->run_queue));
+        erts_init_runq_proc(proc, rq, bound);
     }
 
     proxy->common.id = proc->common.id;
@@ -6348,18 +6288,21 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st
 
     default: {
 	ErtsRunQueue* runq;
+        int bound;
 
 	ASSERT(enqueue == ERTS_ENQUEUE_NORMAL_QUEUE
 	       || enqueue == -ERTS_ENQUEUE_NORMAL_QUEUE);
 
-	runq = erts_get_runq_proc(p);
+	runq = erts_get_runq_proc(p, &bound);
 
-	if (!(ERTS_PSFLG_BOUND & state)) {
+	if (!bound) {
 	    ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio);
-	    if (new_runq) {
-		RUNQ_SET_RQ(&p->run_queue, new_runq);
-		runq = new_runq;
-	    }
+            if (new_runq) {
+                if (erts_try_change_runq_proc(p, new_runq))
+                    runq = new_runq;
+                else
+                    runq = erts_get_runq_proc(p, NULL);
+            }
 	}
 
 	ASSERT(runq);
@@ -11476,6 +11419,7 @@ typedef struct {
     Process *proc;
     erts_aint32_t state;
     ErtsRunQueue *run_queue;
+    int bound;
 } ErtsEarlyProcInit;
 
 static void early_init_process_struct(void *varg, Eterm data)
@@ -11486,10 +11430,9 @@ static void early_init_process_struct(void *varg, Eterm data)
     proc->common.id = make_internal_pid(data);
     erts_atomic32_init_nob(&proc->dirty_state, 0);
     proc->dirty_sys_tasks = NULL;
+    erts_init_runq_proc(proc, arg->run_queue, arg->bound);
     erts_atomic32_init_relb(&proc->state, arg->state);
 
-    RUNQ_SET_RQ(&proc->run_queue, arg->run_queue);
-
     erts_proc_lock_init(proc); /* All locks locked */
 
 }
@@ -11498,7 +11441,7 @@ static void early_init_process_struct(void *varg, Eterm data)
 ** Allocate process and find out where to place next process.
 */
 static Process*
-alloc_process(ErtsRunQueue *rq, erts_aint32_t state)
+alloc_process(ErtsRunQueue *rq, int bound, erts_aint32_t state)
 {
     ErtsEarlyProcInit init_arg;
     Process *p;
@@ -11507,9 +11450,12 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state)
     if (!p)
 	return NULL;
 
+    ASSERT(rq);
+
     init_arg.proc = (Process *) p;
-    init_arg.run_queue = rq;
     init_arg.state = state;
+    init_arg.run_queue = rq;
+    init_arg.bound = bound;
 
     ERTS_CT_ASSERT(offsetof(Process,common) == 0);
 
@@ -11544,6 +11490,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 		   Eterm args,	/* Arguments for function (must be well-formed list). */
 		   ErlSpawnOpts* so) /* Options for spawn. */
 {
+    int bound = 0;
     Uint flags = 0;
     ErtsRunQueue *rq = NULL;
     Process *p;
@@ -11580,7 +11527,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	    ASSERT(0 <= ix && ix < erts_no_run_queues);
 	    rq = ERTS_RUNQ_IX(ix);
 	    /* Unsupported feature... */
-	    state |= ERTS_PSFLG_BOUND;
+            bound = !0;
 	}
 	prio = (erts_aint32_t) so->priority;
     }
@@ -11599,10 +11546,10 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     ASSERT((flags & F_ON_HEAP_MSGQ) || (flags & F_OFF_HEAP_MSGQ));
 
     if (!rq)
-	rq = erts_get_runq_proc(parent);
+	rq = erts_get_runq_proc(parent, NULL);
 
-    p = alloc_process(rq, state); /* All proc locks are locked by this thread
-				     on success */
+    p = alloc_process(rq, bound, state); /* All proc locks are locked by this thread
+                                            on success */
     if (!p) {
 	erts_send_error_to_logger_str(parent->group_leader,
 				      "Too many processes\n");
@@ -11984,7 +11931,7 @@ void erts_init_empty_process(Process *p)
     p->pending_exit.bp = NULL;
     erts_proc_lock_init(p);
     erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
-    RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0));
+    erts_init_runq_proc(p, ERTS_RUNQ_IX(0), 0);
 
 #if !defined(NO_FPE_SIGNALS) || defined(HIPE)
     p->fp_exception = 0;
@@ -12306,7 +12253,7 @@ save_pending_exiter(Process *p, ErtsProcList *plp)
 
     ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
 
-    rq = RUNQ_READ_RQ(&p->run_queue);
+    rq = erts_get_runq_proc(p, NULL);
     ASSERT(rq && !ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
 
     if (!plp)
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 56bed00b40..e585fabb51 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -244,6 +244,9 @@ extern int erts_dio_sched_thread_suggested_stack_size;
 					       (erts_aint32_t) (MSK),	\
 					       (erts_aint32_t) (FLGS)))
 
+#define ERTS_RUNQ_POINTER_MASK  (~((erts_aint_t) 3))
+#define ERTS_RUNQ_BOUND_FLAG    ((erts_aint_t) 1)
+
 typedef enum {
     ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED,
     ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED,
@@ -1168,7 +1171,7 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLG_RUNNING		ERTS_PSFLG_BIT(9)
 #define ERTS_PSFLG_SUSPENDED		ERTS_PSFLG_BIT(10)
 #define ERTS_PSFLG_GC			ERTS_PSFLG_BIT(11)
-#define ERTS_PSFLG_BOUND		ERTS_PSFLG_BIT(12)
+/* #define ERTS_PSFLG_		        ERTS_PSFLG_BIT(12) */
 #define ERTS_PSFLG_TRAP_EXIT		ERTS_PSFLG_BIT(13)
 #define ERTS_PSFLG_ACTIVE_SYS		ERTS_PSFLG_BIT(14)
 #define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
@@ -2169,7 +2172,12 @@ ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp);
 ERTS_GLB_INLINE Process *erts_get_current_process(void);
 ERTS_GLB_INLINE Eterm erts_get_current_pid(void);
 ERTS_GLB_INLINE Uint erts_get_scheduler_id(void);
-ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p);
+ERTS_GLB_INLINE void erts_init_runq_proc(Process *p, ErtsRunQueue *rq, int bnd);
+ERTS_GLB_INLINE ErtsRunQueue *erts_set_runq_proc(Process *p, ErtsRunQueue *rq, int *boundp);
+ERTS_GLB_INLINE int erts_try_change_runq_proc(Process *p, ErtsRunQueue *rq);
+ERTS_GLB_INLINE ErtsRunQueue *erts_bind_runq_proc(Process *p, int bind);
+ERTS_GLB_INLINE int erts_proc_runq_is_bound(Process *p);
+ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p, int *boundp);
 ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp);
 ERTS_GLB_INLINE void erts_runq_lock(ErtsRunQueue *rq);
 ERTS_GLB_INLINE int erts_runq_trylock(ErtsRunQueue *rq);
@@ -2249,11 +2257,143 @@ Uint erts_get_scheduler_id(void)
 	return esdp ? esdp->no : (Uint) 0;
 }
 
+/**
+ * Init run-queue of process.
+ *
+ * @param p[in,out]     Process
+ * @param rq[in]        Run-queue that process will be assigned to
+ * @param bnd[in,out]   If non-zero binds process to run-queue.
+ */
+
+ERTS_GLB_INLINE void
+erts_init_runq_proc(Process *p, ErtsRunQueue *rq, int bnd)
+{
+    erts_aint_t rqint = (erts_aint_t) rq;
+    if (bnd)
+        rqint |= ERTS_RUNQ_BOUND_FLAG;
+    erts_atomic_init_nob(&p->run_queue, rqint);
+}
+
+/**
+ * Forcibly set run-queue of process.
+ *
+ * @param p[in,out]     Process
+ * @param rq[in]        Run-queue that process will be assigned to
+ * @param bndp[in,out]  Pointer to integer. On input non-zero
+ *                      value causes the process to be bound to
+ *                      the run-queue. On output, indicating
+ *                      wether process previously was bound or
+ *                      not.
+ * @return              Previous run-queue.
+ */
+
+ERTS_GLB_INLINE ErtsRunQueue *
+erts_set_runq_proc(Process *p, ErtsRunQueue *rq, int *bndp)
+{
+    erts_aint_t rqint = (erts_aint_t) rq;
+    ASSERT(bndp);
+    if (*bndp)
+        rqint |= ERTS_RUNQ_BOUND_FLAG;
+    rqint = erts_atomic_xchg_nob(&p->run_queue, rqint);
+    *bndp = (int) (rqint & ERTS_RUNQ_BOUND_FLAG);
+    return (ErtsRunQueue *) (rqint & ERTS_RUNQ_POINTER_MASK);
+}
+
+/**
+ * Try to change run-queue assignment of a process.
+ *
+ * @param p[in,out]     Process
+ * @param rq[int]       Run-queue that process will be assigned to
+ * @return              Non-zero if the run-queue assignment was
+ *                      successfully changed.
+ */
+
+ERTS_GLB_INLINE int
+erts_try_change_runq_proc(Process *p, ErtsRunQueue *rq)
+{
+    erts_aint_t old_rqint, new_rqint;
+
+    new_rqint = (erts_aint_t) rq;
+    old_rqint = (erts_aint_t) erts_atomic_read_nob(&p->run_queue);
+    while (1) {
+        erts_aint_t act_rqint;
+
+        if (old_rqint & ERTS_RUNQ_BOUND_FLAG)
+            return 0;
+
+        act_rqint = erts_atomic_cmpxchg_nob(&p->run_queue,
+                                            new_rqint,
+                                            old_rqint);
+        if (act_rqint == old_rqint)
+            return !0;
+    }
+}
+
+/**
+ *
+ * Bind or unbind process to/from currently used run-queue.
+ *
+ * @param p             Process
+ * @param bind          Bind if non-zero; otherwise unbind
+ * @return              Pointer to previously bound run-queue,
+ *                      or NULL if previously unbound
+ */
+
+ERTS_GLB_INLINE ErtsRunQueue *
+erts_bind_runq_proc(Process *p, int bind)
+{
+    erts_aint_t rqint;
+    if (bind)
+        rqint = erts_atomic_read_bor_nob(&p->run_queue,
+                                         ERTS_RUNQ_BOUND_FLAG);
+    else
+        rqint = erts_atomic_read_band_nob(&p->run_queue,
+                                          ~ERTS_RUNQ_BOUND_FLAG);
+    if (rqint & ERTS_RUNQ_BOUND_FLAG)
+        return (ErtsRunQueue *) (rqint & ERTS_RUNQ_POINTER_MASK);
+    else
+        return NULL;
+}
+
+/**
+ * Determine wether a process is bound to a run-queue or not.
+ *
+ * @return              Returns a non-zero value if bound,
+ *                      and zero of not bound.
+ */
+
+ERTS_GLB_INLINE int
+erts_proc_runq_is_bound(Process *p)
+{
+    erts_aint_t rqint = erts_atomic_read_nob(&p->run_queue);
+    return (int) (rqint & ERTS_RUNQ_BOUND_FLAG);
+}
+
+/**
+ * Set run-queue of process.
+ *
+ * @param p[in,out]     Process
+ * @param bndp[out]     Pointer to integer. If non-NULL pointer,
+ *                      the integer will be set to a non-zero
+ *                      value if the process is bound to the
+ *                      run-queue.
+ * @return              Pointer to the normal run-queue that
+ *                      the process currently is assigend to.
+ *                      A process is always assigned to a
+ *                      normal run-queue.
+ */
+
 ERTS_GLB_INLINE ErtsRunQueue *
-erts_get_runq_proc(Process *p)
+erts_get_runq_proc(Process *p, int *bndp)
 {
-    ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue));
-    return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue);
+    erts_aint_t rqint = erts_atomic_read_nob(&p->run_queue);
+    ErtsRunQueue *rq;
+    if (bndp)
+        *bndp = (int) (rqint & ERTS_RUNQ_BOUND_FLAG);
+    rqint &= ERTS_RUNQ_POINTER_MASK;
+    rq = (ErtsRunQueue *) rqint;
+    ASSERT(rq);
+    return rq;
 }
 
 ERTS_GLB_INLINE ErtsRunQueue *
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 01a1965836..05e7bcdea2 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -1009,8 +1009,6 @@ erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg)
                 erts_print(to, to_arg, "SUSPENDED"); break;
             case ERTS_PSFLG_GC:
                 erts_print(to, to_arg, "GC"); break;
-            case ERTS_PSFLG_BOUND:
-                erts_print(to, to_arg, "BOUND"); break;
             case ERTS_PSFLG_TRAP_EXIT:
                 erts_print(to, to_arg, "TRAP_EXIT"); break;
             case ERTS_PSFLG_ACTIVE_SYS:
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 4bb2c0cb01..3e8f6263bb 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -298,7 +298,6 @@ static Port *create_port(char *name,
     erts_aint32_t state = ERTS_PORT_SFLG_CONNECTED;
     erts_aint32_t x_pts_flgs = 0;
 
-    ErtsRunQueue *runq;
     if (!driver_lock) {
 	/* Align size for mutex following port struct */
 	port_size = size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port));
@@ -347,11 +346,16 @@ static Port *create_port(char *name,
 	p += sizeof(erts_mtx_t);
 	state |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK;
     }
-    if (erts_get_scheduler_data())
-        runq = erts_get_runq_current(NULL);
-    else
-        runq = ERTS_RUNQ_IX(0);
-    erts_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq);
+
+    {
+        ErtsRunQueue *runq;
+        ErtsSchedulerData *esdp = erts_get_scheduler_data();
+        if (esdp)
+            runq = erts_get_runq_current(esdp);
+        else
+            runq = ERTS_RUNQ_IX(0);
+        erts_init_runq_port(prt, runq);
+    }
 
     prt->xports = NULL;
     
-- 
cgit v1.2.3


From a091bcc99dd1c8ab5dc5e5f5e80ef254383170f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 5 Mar 2018 14:03:29 +0100
Subject: Assert that sz <= MAX_ARITYVAL in make_arityval(sz)

---
 erts/emulator/beam/bif.c      | 2 +-
 erts/emulator/beam/erl_term.h | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index d7a25adccb..92d49a6def 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3171,7 +3171,7 @@ BIF_RETTYPE list_to_integer_2(BIF_ALIST_2)
 static int do_float_to_charbuf(Process *p, Eterm efloat, Eterm list, 
 			char *fbuf, int sizeof_fbuf) {
 
-    const static int arity_two = make_arityval(2);
+    Eterm arity_two = make_arityval(2);
     int decimals = SYS_DEFAULT_FLOAT_DECIMALS;
     int compact = 0;
     enum fmt_type_ {
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 842802f8d9..b263be0738 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -317,7 +317,8 @@ _ET_DECLARE_CHECKED(Uint,header_arity,Eterm)
 #define MAX_ARITYVAL            ((((Uint)1) << 24) - 1)
 #define ERTS_MAX_TUPLE_SIZE     MAX_ARITYVAL
 
-#define make_arityval(sz)	_make_header((sz),_TAG_HEADER_ARITYVAL)
+#define make_arityval(sz)	(ASSERT((sz) <= MAX_ARITYVAL), \
+                                 _make_header((sz),_TAG_HEADER_ARITYVAL))
 #define is_arity_value(x)	(((x) & _TAG_HEADER_MASK) == _TAG_HEADER_ARITYVAL)
 #define is_sane_arity_value(x)	((((x) & _TAG_HEADER_MASK) == _TAG_HEADER_ARITYVAL) && \
 				 (((x) >> _HEADER_ARITY_OFFS) <= MAX_ARITYVAL))
-- 
cgit v1.2.3


From 8fd6e3353b094aa94e37e257ee20cd5e0e5dddcb Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 30 Jan 2018 22:19:00 +0100
Subject: erts: Remove hipe amd64 code super carrier (exec_mmap)

---
 erts/emulator/beam/erl_alloc.c      | 34 ++--------------------------------
 erts/emulator/beam/erl_alloc_util.c |  8 ++++----
 erts/emulator/beam/erl_alloc_util.h |  2 +-
 erts/emulator/sys/common/erl_mmap.c |  4 ----
 erts/emulator/sys/common/erl_mmap.h | 14 --------------
 erts/emulator/sys/common/erl_mseg.c |  6 ------
 erts/emulator/sys/common/erl_mseg.h |  2 --
 7 files changed, 7 insertions(+), 63 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 88285d8be6..f519dd49ef 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -366,16 +366,9 @@ set_default_exec_alloc_opts(struct au_init *ip)
     ip->init.util.rmbcmt	= 0;
     ip->init.util.acul		= 0;
 
-# ifdef ERTS_HAVE_EXEC_MMAPPER
-    ip->init.util.mseg_alloc    = &erts_alcu_mmapper_mseg_alloc;
-    ip->init.util.mseg_realloc  = &erts_alcu_mmapper_mseg_realloc;
-    ip->init.util.mseg_dealloc  = &erts_alcu_mmapper_mseg_dealloc;
-    ip->init.util.mseg_mmapper  = &erts_exec_mmapper;
-# else
     ip->init.util.mseg_alloc    = &erts_alcu_exec_mseg_alloc;
     ip->init.util.mseg_realloc  = &erts_alcu_exec_mseg_realloc;
     ip->init.util.mseg_dealloc  = &erts_alcu_exec_mseg_dealloc;
-# endif
 }
 #endif /* ERTS_ALC_A_EXEC */
 
@@ -1526,10 +1519,8 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 		    break;
                 case 'X':
                     if (has_prefix("scs", argv[i]+3)) {
-#ifdef ERTS_HAVE_EXEC_MMAPPER
-                        init->mseg.exec_mmap.scs =
-#endif
-                            get_mb_value(argv[i]+6, argv, &i);
+                        /* Ignore obsolete */
+                        (void) get_mb_value(argv[i]+6, argv, &i);
                     }
                     else
                         handle_au_arg(&init->exec_alloc, &argv[i][3], argv, &i, 0);
@@ -2790,10 +2781,6 @@ erts_allocator_info(fmtfn_t to, void *arg)
 #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
         erts_print(to, arg, "=allocator:erts_mmap.literal_mmap\n");
         erts_mmap_info(&erts_literal_mmapper, &to, arg, NULL, NULL, &emis);
-#endif
-#ifdef ERTS_HAVE_EXEC_MMAPPER
-        erts_print(to, arg, "=allocator:erts_mmap.exec_mmap\n");
-        erts_mmap_info(&erts_exec_mmapper, &to, arg, NULL, NULL, &emis);
 #endif
     }
 #endif
@@ -2948,9 +2935,6 @@ erts_allocator_options(void *proc)
 #endif
 #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
     terms[length++] = ERTS_MAKE_AM("literal_mmap");
-#endif
-#ifdef ERTS_HAVE_EXEC_MMAPPER
-    terms[length++] = ERTS_MAKE_AM("exec_mmap");
 #endif
     features = length ? erts_bld_list(hpp, szp, length, terms) : NIL;
 
@@ -3041,9 +3025,6 @@ reply_alloc_info(void *vair)
 # if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
     struct erts_mmap_info_struct mmap_info_literal;
 # endif
-# ifdef ERTS_HAVE_EXEC_MMAPPER
-    struct erts_mmap_info_struct mmap_info_exec;
-# endif
 #endif
     int i;
     Eterm (*info_func)(Allctr_t *,
@@ -3172,17 +3153,6 @@ reply_alloc_info(void *vair)
                                             erts_bld_atom(hpp,szp,"literal_mmap"),
                                             ainfo);
 #  endif
-#  ifdef ERTS_HAVE_EXEC_MMAPPER
-                    ai_list = erts_bld_cons(hpp, szp,
-                                            ainfo, ai_list);
-                    ainfo = (air->only_sz ? NIL :
-                             erts_mmap_info(&erts_exec_mmapper, NULL, NULL,
-                                            hpp, szp, &mmap_info_exec));
-                    ainfo = erts_bld_tuple3(hpp, szp,
-                                            alloc_atom,
-                                            erts_bld_atom(hpp,szp,"exec_mmap"),
-                                            ainfo);
-#  endif
 #else  /* !HAVE_ERTS_MMAP */
                     ainfo = erts_bld_tuple2(hpp, szp, alloc_atom,
                                             am_false);
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 4d4bddb93f..4acb07e19d 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -872,7 +872,7 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
 #elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
 
 /* For allocators that have their own mmapper (super carrier),
- * like literal_alloc and exec_alloc on amd64
+ * like literal_alloc.
  */
 void*
 erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
@@ -914,10 +914,10 @@ erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
 }
 #endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */
 
-#if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER)
+#if defined(ERTS_ALC_A_EXEC)
 
 /*
- * For exec_alloc on non-amd64 that just need memory with PROT_EXEC
+ * For exec_alloc that need memory with PROT_EXEC
  */
 void*
 erts_alcu_exec_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
@@ -956,7 +956,7 @@ erts_alcu_exec_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags)
     ASSERT(r == 0); (void)r;
     erts_alcu_mseg_dealloc(allctr, seg, size, flags);
 }
-#endif /* ERTS_ALC_A_EXEC && !ERTS_HAVE_EXEC_MMAPPER */
+#endif /* ERTS_ALC_A_EXEC */
 
 #endif /* HAVE_ERTS_MSEG */
 
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index faeb5ef368..31fc755803 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -199,7 +199,7 @@ void* erts_alcu_mmapper_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *
 void  erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
 # endif
 
-# if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER)
+# if defined(ERTS_ALC_A_EXEC)
 void* erts_alcu_exec_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
 void* erts_alcu_exec_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
 void  erts_alcu_exec_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index 95e6c6fbd0..90ade5265f 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -358,10 +358,6 @@ char* erts_literals_start;
 UWord erts_literals_size;
 #endif
 
-#ifdef ERTS_HAVE_EXEC_MMAPPER
-ErtsMemMapper erts_exec_mmapper;
-#endif
-
 
 #define ERTS_MMAP_SIZE_SC_SA_INC(SZ) 						\
     do {									\
diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h
index 2a07d93c8c..efb12e7737 100644
--- a/erts/emulator/sys/common/erl_mmap.h
+++ b/erts/emulator/sys/common/erl_mmap.h
@@ -93,11 +93,6 @@ typedef struct {
 #define ERTS_MMAP_INIT_LITERAL_INITER \
     {{NULL, NULL}, {NULL, NULL}, ERTS_LITERAL_VIRTUAL_AREA_SIZE, 1, (1 << 10), 0}
 
-#define ERTS_HIPE_EXEC_VIRTUAL_AREA_SIZE (UWORD_CONSTANT(512)*1024*1024)
-
-#define ERTS_MMAP_INIT_HIPE_EXEC_INITER \
-    {{NULL, NULL}, {NULL, NULL}, ERTS_HIPE_EXEC_VIRTUAL_AREA_SIZE, 1, (1 << 10), 0}
-
 
 #define ERTS_SUPERALIGNED_SIZE \
     (1 << ERTS_MMAP_SUPERALIGNED_BITS)
@@ -165,15 +160,6 @@ extern ErtsMemMapper erts_dflt_mmapper;
 extern ErtsMemMapper erts_literal_mmapper;
 #  endif
 
-#  if defined(ERTS_ALC_A_EXEC) && defined(__x86_64__)
-   /*
-    * On x86_64, exec_alloc employs its own super carrier 'erts_exec_mmaper'
-    * to ensure low memory for HiPE AMD64 small code model.
-    */
-#   define ERTS_HAVE_EXEC_MMAPPER
-extern ErtsMemMapper erts_exec_mmapper;
-#  endif
-
 # endif /* ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */
 #endif /* ERTS_WANT_MEM_MAPPERS */
 
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index bf6de9b13a..a782161a7c 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -1406,12 +1406,6 @@ erts_mseg_init(ErtsMsegInit_t *init)
     erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
-#ifdef ERTS_HAVE_EXEC_MMAPPER
-    /* Initialize erts_exec_mapper *FIRST*, to increase probability
-     * of getting low memory for HiPE AMD64's small code model.
-     */
-    erts_mmap_init(&erts_exec_mmapper, &init->exec_mmap, 1);
-#endif
     erts_mmap_init(&erts_dflt_mmapper, &init->dflt_mmap, 0);
 #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
     erts_mmap_init(&erts_literal_mmapper, &init->literal_mmap, 0);
diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h
index bba0dec499..af275c18be 100644
--- a/erts/emulator/sys/common/erl_mseg.h
+++ b/erts/emulator/sys/common/erl_mseg.h
@@ -61,7 +61,6 @@ typedef struct {
     Uint nos;
     ErtsMMapInit dflt_mmap;
     ErtsMMapInit literal_mmap;
-    ErtsMMapInit exec_mmap;
 } ErtsMsegInit_t;
 
 #define ERTS_MSEG_INIT_DEFAULT_INITIALIZER				\
@@ -72,7 +71,6 @@ typedef struct {
     1000,		/* cci:   Cache check interval		*/	\
     ERTS_MMAP_INIT_DEFAULT_INITER,					\
     ERTS_MMAP_INIT_LITERAL_INITER,                                      \
-    ERTS_MMAP_INIT_HIPE_EXEC_INITER                                     \
 }
 
 typedef struct {
-- 
cgit v1.2.3


From a2f9a1e05ef8dd1fb8a280c66434529fec580a25 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 1 Mar 2018 11:47:54 +0100
Subject: erts: Remove unused "executable" feature from ErtsMemMapper

No longer need for super carrier allocating executable memory.
---
 erts/emulator/sys/common/erl_mmap.c | 73 +++++++++++++++----------------------
 erts/emulator/sys/common/erl_mmap.h |  2 +-
 erts/emulator/sys/common/erl_mseg.c |  4 +-
 3 files changed, 32 insertions(+), 47 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index 90ade5265f..5dadd8a5a6 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -296,11 +296,10 @@ typedef struct {
 }ErtsFreeSegMap;
 
 struct ErtsMemMapper_ {
-    int (*reserve_physical)(char *, UWord, int exec);
+    int (*reserve_physical)(char *, UWord);
     void (*unreserve_physical)(char *, UWord);
     int supercarrier;
     int no_os_mmap;
-    int executable;   /* is client a native code allocator? */
     /*
      * Super unaligned area is located above super aligned
      * area. That is, `sa.bot` is beginning of the super
@@ -1236,7 +1235,6 @@ Eterm build_free_seg_list(Process* p, ErtsFreeSegMap* map)
 
 #if HAVE_MMAP
 #  define ERTS_MMAP_PROT		(PROT_READ|PROT_WRITE)
-#  define ERTS_MMAP_PROT_EXEC		(PROT_READ|PROT_WRITE|PROT_EXEC)
 #  if defined(MAP_ANONYMOUS)
 #    define ERTS_MMAP_FLAGS		(MAP_ANON|MAP_PRIVATE)
 #    define ERTS_MMAP_FD		(-1)
@@ -1250,26 +1248,24 @@ Eterm build_free_seg_list(Process* p, ErtsFreeSegMap* map)
 #endif
 
 static ERTS_INLINE void *
-os_mmap(void *hint_ptr, UWord size, int try_superalign, int executable)
+os_mmap(void *hint_ptr, UWord size, int try_superalign)
 {
 #if HAVE_MMAP
-    const int prot = executable ? ERTS_MMAP_PROT_EXEC : ERTS_MMAP_PROT;
     void *res;
 #ifdef MAP_ALIGN
     if (try_superalign)
-	res = mmap((void *) ERTS_SUPERALIGNED_SIZE, size, prot,
+	res = mmap((void *) ERTS_SUPERALIGNED_SIZE, size, ERTS_MMAP_PROT,
 		   ERTS_MMAP_FLAGS|MAP_ALIGN, ERTS_MMAP_FD, 0);
     else
 #endif
-	res = mmap((void *) hint_ptr, size, prot,
+	res = mmap((void *) hint_ptr, size, ERTS_MMAP_PROT,
 		   ERTS_MMAP_FLAGS, ERTS_MMAP_FD, 0);
     if (res == MAP_FAILED)
 	return NULL;
     return res;
 #elif HAVE_VIRTUALALLOC
-    const DWORD prot = executable ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE;
     return (void *) VirtualAlloc(NULL, (SIZE_T) size,
-				 MEM_COMMIT|MEM_RESERVE, prot);
+				 MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE);
 #else
 #  error "missing mmap() or similar"
 #endif
@@ -1326,7 +1322,6 @@ os_mremap(void *ptr, UWord old_size, UWord new_size, int try_superalign)
 #if HAVE_MMAP
 
 #define ERTS_MMAP_RESERVE_PROT		(ERTS_MMAP_PROT)
-#define ERTS_MMAP_RESERVE_PROT_EXEC	(ERTS_MMAP_PROT_EXEC)
 #define ERTS_MMAP_RESERVE_FLAGS		(ERTS_MMAP_FLAGS|MAP_FIXED)
 #define ERTS_MMAP_UNRESERVE_PROT	(PROT_NONE)
 #if defined(__FreeBSD__)
@@ -1342,10 +1337,9 @@ os_mremap(void *ptr, UWord old_size, UWord new_size, int try_superalign)
 #endif /* __FreeBSD__ */
 
 static int
-os_reserve_physical(char *ptr, UWord size, int exec)
+os_reserve_physical(char *ptr, UWord size)
 {
-    const int prot = exec ? ERTS_MMAP_RESERVE_PROT_EXEC : ERTS_MMAP_RESERVE_PROT;
-    void *res = mmap((void *) ptr, (size_t) size, prot,
+    void *res = mmap((void *) ptr, (size_t) size, ERTS_MMAP_RESERVE_PROT,
 		     ERTS_MMAP_RESERVE_FLAGS, ERTS_MMAP_FD, 0);
     if (res == (void *) MAP_FAILED)
 	return 0;
@@ -1362,7 +1356,7 @@ os_unreserve_physical(char *ptr, UWord size)
 }
 
 static void *
-os_mmap_virtual(char *ptr, UWord size, int exec)
+os_mmap_virtual(char *ptr, UWord size)
 {
     int flags = ERTS_MMAP_VIRTUAL_FLAGS;
     void* res;
@@ -1381,7 +1375,7 @@ os_mmap_virtual(char *ptr, UWord size, int exec)
 
 #endif /* ERTS_HAVE_OS_MMAP */
 
-static int reserve_noop(char *ptr, UWord size, int exec)
+static int reserve_noop(char *ptr, UWord size)
 {
 #ifdef ERTS_MMAP_DEBUG_FILL_AREAS
     Uint32 *uip, *end = (Uint32 *) (ptr + size);
@@ -1424,7 +1418,7 @@ alloc_desc_insert_free_seg(ErtsMemMapper* mm,
 
 #if ERTS_HAVE_OS_MMAP
     if (!mm->no_os_mmap) {
-        ptr = os_mmap(mm->desc.new_area_hint, ERTS_PAGEALIGNED_SIZE, 0, 0);
+        ptr = os_mmap(mm->desc.new_area_hint, ERTS_PAGEALIGNED_SIZE, 0);
         if (ptr) {
             mm->desc.new_area_hint = ptr+ERTS_PAGEALIGNED_SIZE;
             ERTS_MMAP_SIZE_OS_INC(ERTS_PAGEALIGNED_SIZE);
@@ -1443,7 +1437,7 @@ alloc_desc_insert_free_seg(ErtsMemMapper* mm,
     da_map = &mm->sua.map;
     desc = lookup_free_seg(da_map, ERTS_PAGEALIGNED_SIZE);
     if (desc) {
-	if (mm->reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE, 0))
+	if (mm->reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE))
 	    ERTS_MMAP_SIZE_SC_SUA_INC(ERTS_PAGEALIGNED_SIZE);
 	else
 	    desc = NULL;
@@ -1453,7 +1447,7 @@ alloc_desc_insert_free_seg(ErtsMemMapper* mm,
 	da_map = &mm->sa.map;
 	desc = lookup_free_seg(da_map, ERTS_PAGEALIGNED_SIZE);
 	if (desc) {
-	    if (mm->reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE, 0))
+	    if (mm->reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE))
 		ERTS_MMAP_SIZE_SC_SA_INC(ERTS_PAGEALIGNED_SIZE);
 	    else
 		desc = NULL;
@@ -1514,7 +1508,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep)
 	    if (desc) {
 		seg = desc->start;
 		end = seg+asize;
-		if (!mm->reserve_physical(seg, asize, mm->executable))
+		if (!mm->reserve_physical(seg, asize))
 		    goto supercarrier_reserve_failure;
 		if (desc->end == end) {
 		    delete_free_seg(&mm->sua.map, desc);
@@ -1529,8 +1523,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep)
 	    }
 
 	    if (asize <= mm->sua.bot - mm->sa.top) {
-		if (!mm->reserve_physical(mm->sua.bot - asize, asize,
-                                          mm->executable))
+		if (!mm->reserve_physical(mm->sua.bot - asize, asize))
 		    goto supercarrier_reserve_failure;
 		mm->sua.bot -= asize;
 		seg = mm->sua.bot;
@@ -1546,8 +1539,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep)
 	    char *start = seg = desc->start;
 	    seg = (char *) ERTS_SUPERALIGNED_CEILING(seg);
 	    end = seg+asize;
-	    if (!mm->reserve_physical(start, (UWord) (end - start),
-                                      mm->executable))
+	    if (!mm->reserve_physical(start, (UWord) (end - start)))
 		goto supercarrier_reserve_failure;
 	    ERTS_MMAP_SIZE_SC_SA_INC(asize);
 	    if (desc->end == end) {
@@ -1579,8 +1571,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep)
 
 	    if (asize + (seg - start) <= mm->sua.bot - start) {
 		end = seg + asize;
-		if (!mm->reserve_physical(start, (UWord) (end - start),
-                                          mm->executable))
+		if (!mm->reserve_physical(start, (UWord) (end - start)))
 		    goto supercarrier_reserve_failure;
 		mm->sa.top = end;
 		ERTS_MMAP_SIZE_SC_SA_INC(asize);
@@ -1602,8 +1593,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep)
 
 		seg = (char *) ERTS_SUPERALIGNED_CEILING(org_start);
 		end = seg + asize;
-		if (!mm->reserve_physical(seg, (UWord) (org_end - seg),
-                                          mm->executable))
+		if (!mm->reserve_physical(seg, (UWord) (org_end - seg)))
 		    goto supercarrier_reserve_failure;
 		ERTS_MMAP_SIZE_SC_SUA_INC(asize);
 		if (org_start != seg) {
@@ -1636,13 +1626,13 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep)
     /* Map using OS primitives */
     if (!(ERTS_MMAPFLG_SUPERCARRIER_ONLY & flags) && !mm->no_os_mmap) {
 	if (!(ERTS_MMAPFLG_SUPERALIGNED & flags)) {
-	    seg = os_mmap(NULL, asize, 0, mm->executable);
+	    seg = os_mmap(NULL, asize, 0);
 	    if (!seg)
 		goto failure;
 	}
 	else {
 	    asize = ERTS_SUPERALIGNED_CEILING(*sizep);
-	    seg = os_mmap(NULL, asize, 1, mm->executable);
+	    seg = os_mmap(NULL, asize, 1);
 	    if (!seg)
 		goto failure;
 
@@ -1652,8 +1642,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep)
 
 		os_munmap(seg, asize);
 
-		ptr = os_mmap(NULL, asize + ERTS_SUPERALIGNED_SIZE, 1,
-                              mm->executable);
+		ptr = os_mmap(NULL, asize + ERTS_SUPERALIGNED_SIZE, 1);
 		if (!ptr)
 		    goto failure;
 
@@ -1988,8 +1977,7 @@ erts_mremap(ErtsMemMapper* mm,
 
 	    if (next && new_end <= next->end) {
 		if (!mm->reserve_physical(((char *) ptr) + old_size,
-                                          asize - old_size,
-                                          mm->executable))
+                                          asize - old_size))
 		    goto supercarrier_reserve_failure;
 		if (new_end < next->end)
 		    resize_free_seg(&mm->sua.map, next, new_end, next->end);
@@ -2007,8 +1995,7 @@ erts_mremap(ErtsMemMapper* mm,
 	    if (end == mm->sa.top) {
 		if (new_end <= mm->sua.bot) {
 		    if (!mm->reserve_physical(((char *) ptr) + old_size,
-                                              asize - old_size,
-                                              mm->executable))
+                                              asize - old_size))
 			goto supercarrier_reserve_failure;
 		    mm->sa.top = new_end;
 		    new_ptr = ptr;
@@ -2020,8 +2007,7 @@ erts_mremap(ErtsMemMapper* mm,
 		adjacent_free_seg(&mm->sa.map, start, end, &prev, &next);
 		if (next && new_end <= next->end) {
 		    if (!mm->reserve_physical(((char *) ptr) + old_size,
-                                              asize - old_size,
-                                              mm->executable))
+                                              asize - old_size))
 			goto supercarrier_reserve_failure;
 		    if (new_end < next->end)
 			resize_free_seg(&mm->sa.map, next, new_end, next->end);
@@ -2141,7 +2127,7 @@ static void hard_dbg_mseg_init(void);
 #endif
 
 void
-erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable)
+erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init)
 {
     static int is_first_call = 1;
     int virtual_map = 0;
@@ -2173,7 +2159,6 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable)
     mm->supercarrier = 0;
     mm->reserve_physical = reserve_noop;
     mm->unreserve_physical = unreserve_noop;
-    mm->executable = executable;
 
 #if HAVE_MMAP && !defined(MAP_ANON)
     mm->mmap_fd = open("/dev/zero", O_RDWR);
@@ -2195,7 +2180,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable)
 	ptr = (char *) ERTS_PAGEALIGNED_CEILING(init->virtual_range.start);
 	end = (char *) ERTS_PAGEALIGNED_FLOOR(init->virtual_range.end);
 	sz = end - ptr;
-	start = os_mmap_virtual(ptr, sz, executable);
+	start = os_mmap_virtual(ptr, sz);
 	if (!start || start > ptr || start >= end)
 	    erts_exit(1,
 		     "erts_mmap: Failed to create virtual range for super carrier\n");
@@ -2220,7 +2205,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable)
 	sz = ERTS_PAGEALIGNED_CEILING(init->scs);
 #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
 	if (!init->scrpm) {
-	    start = os_mmap_virtual(NULL, sz, executable);
+	    start = os_mmap_virtual(NULL, sz);
 	    mm->reserve_physical = os_reserve_physical;
 	    mm->unreserve_physical = os_unreserve_physical;
 	    virtual_map = 1;
@@ -2232,7 +2217,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable)
 	     * The whole supercarrier will by physically
 	     * reserved all the time.
 	     */
-	    start = os_mmap(NULL, sz, 1, executable);
+	    start = os_mmap(NULL, sz, 1);
 	}
 	if (!start)
 	    erts_exit(1,
@@ -2306,7 +2291,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable)
 	    mm->sua.top -= ERTS_PAGEALIGNED_SIZE;
 	    mm->size.supercarrier.used.total += ERTS_PAGEALIGNED_SIZE;
 #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
-	    if (!virtual_map || os_reserve_physical(mm->sua.top, ERTS_PAGEALIGNED_SIZE, 0))
+	    if (!virtual_map || os_reserve_physical(mm->sua.top, ERTS_PAGEALIGNED_SIZE))
 #endif
 		add_free_desc_area(mm, mm->sua.top, end);
             mm->desc.reserved += (end - mm->sua.top) / sizeof(ErtsFreeSegDesc);
@@ -2319,7 +2304,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable)
 	 * will be used for free segment descritors.
 	 */
 #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
-	if (virtual_map && !os_reserve_physical(start, mm->sa.bot - start, 0))
+	if (virtual_map && !os_reserve_physical(start, mm->sa.bot - start))
 	    erts_exit(1, "erts_mmap: Failed to reserve physical memory for descriptors\n");
 #endif
 	mm->desc.unused_start = start;
diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h
index efb12e7737..c1f9668b18 100644
--- a/erts/emulator/sys/common/erl_mmap.h
+++ b/erts/emulator/sys/common/erl_mmap.h
@@ -135,7 +135,7 @@ void *erts_mmap(ErtsMemMapper*, Uint32 flags, UWord *sizep);
 void erts_munmap(ErtsMemMapper*, Uint32 flags, void *ptr, UWord size);
 void *erts_mremap(ErtsMemMapper*, Uint32 flags, void *ptr, UWord old_size, UWord *sizep);
 int erts_mmap_in_supercarrier(ErtsMemMapper*, void *ptr);
-void erts_mmap_init(ErtsMemMapper*, ErtsMMapInit*, int executable);
+void erts_mmap_init(ErtsMemMapper*, ErtsMMapInit*);
 struct erts_mmap_info_struct
 {
     UWord sizes[6];
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index a782161a7c..ced3d61525 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -1406,9 +1406,9 @@ erts_mseg_init(ErtsMsegInit_t *init)
     erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
-    erts_mmap_init(&erts_dflt_mmapper, &init->dflt_mmap, 0);
+    erts_mmap_init(&erts_dflt_mmapper, &init->dflt_mmap);
 #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
-    erts_mmap_init(&erts_literal_mmapper, &init->literal_mmap, 0);
+    erts_mmap_init(&erts_literal_mmapper, &init->literal_mmap);
 #endif
 
     if (!IS_2POW(GET_PAGE_SIZE))
-- 
cgit v1.2.3


From 847c465d9fae3630c011d928743f262288072057 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 5 Mar 2018 17:54:01 +0100
Subject: erts: Optimize erlang:put/2 for hash collision lists

Instead of rebuilding all cons cells before key,
just unlink key cell from list with a destructive heap write op.

This is safe as these lists never leak out and any new-to-old-heap-refs
are preserved.
---
 erts/emulator/beam/erl_process_dict.c | 102 ++++++++++++----------------------
 1 file changed, 37 insertions(+), 65 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 87b440093b..aee88841ae 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -79,6 +79,8 @@
 /* Array access macro */ 
 #define ARRAY_GET(PDict, Index) (ASSERT((Index) < (PDict)->arraySize), \
 				 (PDict)->data[Index])
+#define ARRAY_GET_PTR(PDict, Index) (ASSERT((Index) < (PDict)->arraySize), \
+				     &(PDict)->data[Index])
 #define ARRAY_PUT(PDict, Index, Val) (ASSERT((Index) < (PDict)->arraySize), \
                                       (PDict)->data[Index] = (Val))
 
@@ -595,11 +597,13 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
     unsigned int hval;
     Eterm *hp;
     Eterm *tp;
+    Eterm *bucket;
     Eterm tpl;
     Eterm old;
+    Eterm old_val = am_undefined;
     Eterm tmp;
     int needed;
-    int key_at = 0;
+    int new_key = 1;
 #ifdef DEBUG
     Eterm *hp_limit;
 #endif
@@ -613,7 +617,8 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
         p->dictionary->numElements = 0;
     }	
     hval = pd_hash_value(p->dictionary, id);
-    old = ARRAY_GET(p->dictionary, hval);
+    bucket = ARRAY_GET_PTR(p->dictionary, hval);
+    old = *bucket;
 
     /*
      * Calculate the number of heap words needed and garbage
@@ -624,44 +629,46 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
         ASSERT(is_tuple(old));
         tp = tuple_val(old);
         if (EQ(tp[1], id)) {
+            old_val = tp[2];
             if (is_immed(value)) {
-                Eterm old_val = tp[2];
-                tp[2] = value;
+                tp[2] = value;     /* DESTRUCTIVE HEAP ASSIGNMENT */
                 return old_val;
             }
-            key_at = 1;
+            new_key = 0;
         }
         else {
             needed += 2+2;
         }
     } else if (is_list(old)) {
-        int i = 1;
+        Eterm* prev_cdr = bucket;
 
         needed += 2;
-	for (tmp = old; tmp != NIL; tmp = TCDR(tmp)) {
+	for (tmp = old; tmp != NIL; prev_cdr = &TCDR(tmp), tmp = *prev_cdr) {
             tp = tuple_val(TCAR(tmp));
             if (EQ(tp[1], id)) {
+                old_val = tp[2];
                 if (is_immed(value)) {
-                    Eterm old_val = tp[2];
-                    tp[2] = value;
+                    tp[2] = value;     /* DESTRUCTIVE HEAP ASSIGNMENT */
                     return old_val;
                 }
-                key_at = i;
-                needed += 2*(key_at-1);
+                new_key = 0;
+                /* Unlink old {Key,Value} from list */
+                *prev_cdr = TCDR(tmp);  /* maybe DESTRUCTIVE HEAP ASSIGNMENT */
                 break;
             }
-            ++i;
 	}
     }
     if (HeapWordsLeft(p) < needed) {
 	Eterm root[3];
 	root[0] = id;
 	root[1] = value;
-	root[2] = old;
+        root[2] = old_val;
 	erts_garbage_collect(p, needed, root, 3);
 	id = root[0];
 	value = root[1];
-	old = root[2];
+        old_val = root[2];
+        ASSERT(bucket == ARRAY_GET_PTR(p->dictionary, hval));
+        old = *bucket;
     }
 #ifdef DEBUG
     hp_limit = p->htop + needed;
@@ -677,67 +684,29 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
      * Update the dictionary.
      */
     if (is_nil(old)) {
-	ARRAY_PUT(p->dictionary, hval, tpl);
-	++(p->dictionary->numElements);
+	*bucket = tpl;
     } else if (is_boxed(old)) {
 	ASSERT(is_tuple(old));
-	if (key_at) {
+	if (!new_key) {
             ASSERT(EQ(tuple_val(old)[1],id));
-	    ARRAY_PUT(p->dictionary, hval, tpl);
-	    return tuple_val(old)[2];
+	    *bucket = tpl;
+	    return old_val;
 	} else {
 	    hp = HeapOnlyAlloc(p, 4);
 	    tmp = CONS(hp, old, NIL);
 	    hp += 2;
-	    ++(p->dictionary->numElements);
-	    ARRAY_PUT(p->dictionary, hval, CONS(hp, tpl, tmp));
+	    *bucket = CONS(hp, tpl, tmp);
 	    hp += 2;
 	    ASSERT(hp <= hp_limit);
 	}
     } else if (is_list(old)) {
-	if (!key_at) {
-	    /*
-	     * New key. Simply prepend the tuple to the beginning of the list.
-	     */
-	    hp = HeapOnlyAlloc(p, 2);
-	    ARRAY_PUT(p->dictionary, hval, CONS(hp, tpl, old));
-	    hp += 2;
-	    ASSERT(hp <= hp_limit);
-	    ++(p->dictionary->numElements);
-	} else {
-	    /*
-	     * key_at = Key position in the list.
-	     *
-	     * Replace old value in list. To avoid pointers from the old generation
-	     * to the new, we must rebuild the list from the beginning up to and
-	     * including the changed element.
-	     */
-	    Eterm nlist;
-	    int j;
-
-	    hp = HeapOnlyAlloc(p, key_at*2);
-	    
-	    /* Find the list element to change. */
-	    for (j = 1, nlist = old; j < key_at; j++, nlist = TCDR(nlist)) {
-		;
-	    }
-	    ASSERT(EQ(tuple_val(TCAR(nlist))[1], id));
-	    nlist = TCDR(nlist); /* Unchanged part of list. */
-
-	    /* Rebuild list before the updated element. */
-	    for (tmp = old; --key_at > 0; tmp = TCDR(tmp)) {
-		nlist = CONS(hp, TCAR(tmp), nlist);
-		hp += 2;
-	    }
-	    ASSERT(EQ(tuple_val(TCAR(tmp))[1], id));
-
-	    /* Put the updated element first in the new list. */
-	    nlist = CONS(hp, tpl, nlist);
-	    hp += 2;
-	    ASSERT(hp <= hp_limit);
-	    ARRAY_PUT(p->dictionary, hval, nlist);
-	    return tuple_val(TCAR(tmp))[2];
-	}
+        /*
+	 * Simply prepend the tuple to the beginning of the list.
+	 */
+	hp = HeapOnlyAlloc(p, 2);
+        *bucket = CONS(hp, tpl, *bucket);
+	hp += 2;
+	ASSERT(hp <= hp_limit);
     } else {
 #ifdef DEBUG
 	erts_fprintf(stderr,
@@ -748,10 +717,13 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
 
 	erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during put/2.");
     }
+
+    p->dictionary->numElements += new_key;
+
     if (HASH_RANGE(p->dictionary) <= p->dictionary->numElements) {
 	grow(p);
     }
-    return am_undefined;
+    return old_val;
 }
 
 /*
-- 
cgit v1.2.3


From 9ef6b9a923b25ceb487eea4431cdf730972fc489 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 7 Mar 2018 16:18:24 +0100
Subject: Fix for bug introduced when replacing ERTS_PSFLG_BOUND

Bug introduced in commit fbb10ebc4a37555c7ea7f99e14286d862993976a
---
 erts/emulator/beam/erl_process.c | 17 +++++++++--------
 erts/emulator/beam/erl_process.h |  1 +
 2 files changed, 10 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b19659f496..23fe353495 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -4135,8 +4135,6 @@ evacuate_run_queue(ErtsRunQueue *rq,
     for (prio_q = 0; prio_q < ERTS_NO_PROC_PRIO_QUEUES; prio_q++) {
 	erts_aint32_t state;
 	Process *proc;
-	int notify = 0;
-	to_rq = NULL;
 
         if (!mp->prio[prio_q].runq)
             return;
@@ -4195,6 +4193,12 @@ evacuate_run_queue(ErtsRunQueue *rq,
 		goto handle_next_proc;
 	    }
 
+            prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state);
+            to_rq = mp->prio[prio].runq;
+
+            if (!to_rq)
+                goto handle_next_proc;
+
 	    if (!erts_try_change_runq_proc(proc, to_rq)) {
 		/* Bound processes get stuck here... */
 		proc->next = NULL;
@@ -4205,15 +4209,13 @@ evacuate_run_queue(ErtsRunQueue *rq,
 		sbpp->last = proc;
 	    }
 	    else {
-		int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state);
 		erts_runq_unlock(rq);
 
-                to_rq = mp->prio[prio].runq;
-
 		erts_runq_lock(to_rq);
 		enqueue_process(to_rq, prio, proc);
 		erts_runq_unlock(to_rq);
-		notify = 1;
+
+                smp_notify_inc_runq(to_rq);
 
 		erts_runq_lock(rq);
 	    }
@@ -4221,8 +4223,7 @@ evacuate_run_queue(ErtsRunQueue *rq,
 	handle_next_proc:
 	    proc = dequeue_process(rq, prio_q, &state);
 	}
-	if (notify)
-	    smp_notify_inc_runq(to_rq);
+
     }
 }
 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index e585fabb51..8581c6a639 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -2292,6 +2292,7 @@ erts_set_runq_proc(Process *p, ErtsRunQueue *rq, int *bndp)
 {
     erts_aint_t rqint = (erts_aint_t) rq;
     ASSERT(bndp);
+    ASSERT(rq);
     if (*bndp)
         rqint |= ERTS_RUNQ_BOUND_FLAG;
     rqint = erts_atomic_xchg_nob(&p->run_queue, rqint);
-- 
cgit v1.2.3


From ca9e3cea6cfdad0d99dcef149cb0ba1d1e1e98f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 5 Mar 2018 11:49:34 +0100
Subject: Check the arguments to sys_memcpy and friends

Passing NULL is undefined behavior and unconditionally executing these
may result in the compiler optimizing away a later NULL check. It can
often work since the pointer isn't touched when the length is 0, but
it's a major footgun. See ERL-573.
---
 erts/emulator/beam/sys.h | 40 ++++++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 290e0b209a..13ae80e4a5 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -996,16 +996,36 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-#define sys_memcpy(s1,s2,n)  memcpy(s1,s2,n)
-#define sys_memmove(s1,s2,n) memmove(s1,s2,n)
-#define sys_memcmp(s1,s2,n)  memcmp(s1,s2,n)
-#define sys_memset(s,c,n)    memset(s,c,n)
-#define sys_memzero(s, n)    memset(s,'\0',n)
-#define sys_strcmp(s1,s2)    strcmp(s1,s2)
-#define sys_strncmp(s1,s2,n) strncmp(s1,s2,n)
-#define sys_strcpy(s1,s2)    strcpy(s1,s2)
-#define sys_strncpy(s1,s2,n) strncpy(s1,s2,n)
-#define sys_strlen(s)        strlen(s)
+/* Thin wrappers around memcpy and friends, which should always be used in
+ * place of plain memcpy, memset, etc.
+ *
+ * Passing NULL to any of these functions is undefined behavior even though it
+ * may seemingly work when the length (if any) is zero; a compiler can take
+ * this as a hint that the passed operand may *never* be NULL and then optimize
+ * based on that information.
+ *
+ * (The weird casts in the assertions silence an "always evaluates to true"
+ * warning when an operand is the address of an lvalue) */
+#define sys_memcpy(s1,s2,n) \
+    (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), memcpy(s1,s2,n))
+#define sys_memmove(s1,s2,n) \
+    (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), memmove(s1,s2,n))
+#define sys_memcmp(s1,s2,n) \
+    (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), memcmp(s1,s2,n))
+#define sys_memset(s,c,n) \
+    (ASSERT((void*)(s) != NULL), memset(s,c,n))
+#define sys_memzero(s, n) \
+    (ASSERT((void*)(s) != NULL), memset(s,'\0',n))
+#define sys_strcmp(s1,s2) \
+    (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strcmp(s1,s2))
+#define sys_strncmp(s1,s2,n) \
+    (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strncmp(s1,s2,n))
+#define sys_strcpy(s1,s2) \
+    (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strcpy(s1,s2))
+#define sys_strncpy(s1,s2,n) \
+    (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strncpy(s1,s2,n))
+#define sys_strlen(s) \
+    (ASSERT((void*)(s) != NULL), strlen(s))
 
 /* define function symbols (needed in sys_drv_api) */
 #define sys_fp_alloc     sys_alloc
-- 
cgit v1.2.3


From 4d4629605ab7d3c3a5268502dadcf639151d6c42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 5 Mar 2018 12:55:13 +0100
Subject: Always use sys_memcpy/cmp/etc instead of plain memcpy/cmp/etc

---
 erts/emulator/beam/atom.c                  |  2 +-
 erts/emulator/beam/beam_debug.c            |  6 +--
 erts/emulator/beam/beam_emu.c              |  4 +-
 erts/emulator/beam/beam_load.c             | 18 ++++----
 erts/emulator/beam/bif.c                   |  6 +--
 erts/emulator/beam/break.c                 |  2 +-
 erts/emulator/beam/erl_afit_alloc.c        |  2 +-
 erts/emulator/beam/erl_alloc.c             | 66 +++++++++++++++---------------
 erts/emulator/beam/erl_alloc.h             |  2 +-
 erts/emulator/beam/erl_alloc_util.c        | 14 +++----
 erts/emulator/beam/erl_ao_firstfit_alloc.c |  2 +-
 erts/emulator/beam/erl_bestfit_alloc.c     |  2 +-
 erts/emulator/beam/erl_bif_binary.c        | 20 ++++-----
 erts/emulator/beam/erl_bif_chksum.c        |  6 +--
 erts/emulator/beam/erl_bif_ddll.c          | 12 +++---
 erts/emulator/beam/erl_bif_info.c          | 38 ++++++++---------
 erts/emulator/beam/erl_bif_port.c          |  8 ++--
 erts/emulator/beam/erl_bif_re.c            | 18 ++++----
 erts/emulator/beam/erl_bif_unique.h        |  4 +-
 erts/emulator/beam/erl_db.c                |  2 +-
 erts/emulator/beam/erl_db_util.c           | 14 +++----
 erts/emulator/beam/erl_drv_thread.c        |  8 ++--
 erts/emulator/beam/erl_goodfit_alloc.c     |  2 +-
 erts/emulator/beam/erl_init.c              | 38 ++++++++---------
 erts/emulator/beam/erl_instrument.c        |  2 +-
 erts/emulator/beam/erl_lock_check.c        | 12 +++---
 erts/emulator/beam/erl_msacc.c             |  4 +-
 erts/emulator/beam/erl_mtrace.c            | 22 +++++-----
 erts/emulator/beam/erl_nif.c               |  6 +--
 erts/emulator/beam/erl_trace.c             |  4 +-
 erts/emulator/beam/erl_unicode.c           | 12 +++---
 erts/emulator/beam/erl_utils.h             |  2 +-
 erts/emulator/beam/erl_vm.h                |  2 +-
 erts/emulator/beam/external.c              |  2 +-
 erts/emulator/beam/global.h                |  4 +-
 erts/emulator/beam/io.c                    |  8 ++--
 erts/emulator/beam/lttng-wrapper.h         |  4 +-
 erts/emulator/beam/packet_parser.c         |  8 ++--
 38 files changed, 194 insertions(+), 194 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index bbe1cb3e11..e5b7616a0d 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -452,7 +452,7 @@ init_atom_table(void)
     /* Ordinary atoms */
     for (i = 0; erl_atom_names[i] != 0; i++) {
 	int ix;
-	a.len = strlen(erl_atom_names[i]);
+	a.len = sys_strlen(erl_atom_names[i]);
 	a.latin1_chars = a.len;
 	a.name = (byte*)erl_atom_names[i];
 	a.slot.index = i;
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 8f02d509a9..5eb68b817e 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -228,11 +228,11 @@ erts_debug_instructions_0(BIF_ALIST_0)
     Eterm res = NIL;
 
     for (i = 0; i < num_instructions; i++) {
-	needed += 2*strlen(opc[i].name);
+	needed += 2*sys_strlen(opc[i].name);
     }
     hp = HAlloc(BIF_P, needed);
     for (i = num_instructions-1; i >= 0; i--) {
-	Eterm s = erts_bld_string_n(&hp, 0, opc[i].name, strlen(opc[i].name));
+	Eterm s = erts_bld_string_n(&hp, 0, opc[i].name, sys_strlen(opc[i].name));
 	res = erts_bld_cons(&hp, 0, s, res);
     }
     return res;
@@ -431,7 +431,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	 * the packing program backwards and in reverse.
 	 */
 
-	prog = start_prog + strlen(start_prog);
+	prog = start_prog + sys_strlen(start_prog);
 	while (start_prog < prog) {
 	    prog--;
 	    switch (*prog) {
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index ef9abcde08..fbd0e38735 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -749,7 +749,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
             dtrace_proc_str(c_p, process_buf);
 
             if (ERTS_PROC_IS_EXITING(c_p)) {
-                strcpy(fun_buf, "<exiting>");
+                sys_strcpy(fun_buf, "<exiting>");
             } else {
                 ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i);
                 if (cmfa) {
@@ -1229,7 +1229,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
             dtrace_proc_str(c_p, process_buf);
 
             if (ERTS_PROC_IS_EXITING(c_p)) {
-                strcpy(fun_buf, "<exiting>");
+                sys_strcpy(fun_buf, "<exiting>");
             } else {
                 ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i);
                 if (cmfa) {
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index e242fe9140..50498cb6cf 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -459,7 +459,7 @@ typedef struct LoaderState {
 
 #ifdef DEBUG
 # define GARBAGE 0xCC
-# define DEBUG_INIT_GENOP(Dst) memset(Dst, GARBAGE, sizeof(GenOp))
+# define DEBUG_INIT_GENOP(Dst) sys_memset(Dst, GARBAGE, sizeof(GenOp))
 #else
 # define DEBUG_INIT_GENOP(Dst)
 #endif
@@ -1422,7 +1422,7 @@ load_atom_table(LoaderState* stp, ErtsAtomEncoding enc)
 	Atom* ap;
 
 	ap = atom_tab(atom_val(stp->atom[1]));
-	memcpy(sbuf, ap->name, ap->len);
+	sys_memcpy(sbuf, ap->name, ap->len);
 	sbuf[ap->len] = '\0';
 	LoadError1(stp, "module name in object code is %s", sbuf);
     }
@@ -2089,7 +2089,7 @@ load_code(LoaderState* stp)
 			    erts_alloc(ERTS_ALC_T_LOADER_TMP,
 				       (arity+last_op->a[arg].val)
 				       *sizeof(GenOpArg));
-			memcpy(last_op->a, last_op->def_args,
+			sys_memcpy(last_op->a, last_op->def_args,
 			       arity*sizeof(GenOpArg));
 			arity += last_op->a[arg].val;
 			break;
@@ -4915,7 +4915,7 @@ freeze_code(LoaderState* stp)
 	line_items[i] = codev + stp->ci - 1;
 
 	line_tab->fname_ptr = (Eterm*) &line_items[i + 1];
-	memcpy(line_tab->fname_ptr, stp->fname, stp->num_fnames*sizeof(Eterm));
+	sys_memcpy(line_tab->fname_ptr, stp->fname, stp->num_fnames*sizeof(Eterm));
 
 	line_tab->loc_size = stp->loc_size;
 	if (stp->loc_size == 2) {
@@ -5498,8 +5498,8 @@ transform_engine(LoaderState* st)
 	case TOP_store_rest_args:
 	    {
 		GENOP_ARITY(instr, instr->arity+num_rest_args);
-		memcpy(instr->a, instr->def_args, ap*sizeof(GenOpArg));
-		memcpy(instr->a+ap, rest_args, num_rest_args*sizeof(GenOpArg));
+		sys_memcpy(instr->a, instr->def_args, ap*sizeof(GenOpArg));
+		sys_memcpy(instr->a+ap, rest_args, num_rest_args*sizeof(GenOpArg));
 		ap += num_rest_args;
 	    }
 	    break;
@@ -6447,7 +6447,7 @@ stub_copy_info(LoaderState* stp,
     Sint decoded_size;
     Uint size = stp->chunks[chunk].size;
     if (size != 0) {
-	memcpy(info, stp->chunks[chunk].start, size);
+	sys_memcpy(info, stp->chunks[chunk].start, size);
 	*ptr_word = info;
 	decoded_size = erts_decode_ext_size(info, size);
 	if (decoded_size < 0) {
@@ -7017,8 +7017,8 @@ void dbg_set_traced_mfa(const char* m, const char* f, Uint a)
 {
     unsigned i = dbg_trace_ix++;
     ASSERT(i < MFA_MAX);
-    dbg_trace_m[i] = am_atom_put(m, strlen(m));
-    dbg_trace_f[i] = am_atom_put(f, strlen(f));
+    dbg_trace_m[i] = am_atom_put(m, sys_strlen(m));
+    dbg_trace_f[i] = am_atom_put(f, sys_strlen(f));
     dbg_trace_a[i] = a;
 }
 
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index d68ccc3028..f6f1bb07e0 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3466,7 +3466,7 @@ BIF_RETTYPE binary_to_float_1(BIF_ALIST_1)
     if (bit_offs)
       erts_copy_bits(bytes, bit_offs, 1, buf, 0, 1, size*8);
     else
-      memcpy(buf, bytes, size);
+      sys_memcpy(buf, bytes, size);
     
     buf[size] = '\0';
     
@@ -4200,10 +4200,10 @@ BIF_RETTYPE list_to_port_1(BIF_ALIST_1)
     buf[i] = '\0';		/* null terminal */
 
     cp = &buf[0];
-    if (strncmp("#Port<", cp, 6) != 0)
+    if (sys_strncmp("#Port<", cp, 6) != 0)
         goto bad;
 
-    cp += 6; /* strlen("#Port<") */
+    cp += 6; /* sys_strlen("#Port<") */
 
     if (sscanf(cp, "%u.%u>", (unsigned int*)&n, (unsigned int*)&p) < 2)
         goto bad;
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index b9b70cd8ef..fe0ec1925b 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -509,7 +509,7 @@ do_break(void)
     /* check if we're in console mode and, if so,
        halt immediately if break is called */
     mode = erts_read_env("ERL_CONSOLE_MODE");
-    if (mode && strcmp(mode, "window") != 0)
+    if (mode && sys_strcmp(mode, "window") != 0)
 	erts_exit(0, "");
     erts_free_read_env(mode);
 #endif /* __WIN32__ */
diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c
index 4ebe37ee1d..23efe3bba4 100644
--- a/erts/emulator/beam/erl_afit_alloc.c
+++ b/erts/emulator/beam/erl_afit_alloc.c
@@ -181,7 +181,7 @@ static struct {
 
 static void ERTS_INLINE atom_init(Eterm *atom, char *name)
 {
-    *atom = am_atom_put(name, strlen(name));
+    *atom = am_atom_put(name, sys_strlen(name));
 }
 #define AM_INIT(AM) atom_init(&am.AM, #AM)
 
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 239dda6bcf..fa49096d2c 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1219,9 +1219,9 @@ get_bool_value(char *param_end, char** argv, int* ip)
 {
     char *param = argv[*ip]+1;
     char *value = get_value(param_end, argv, ip);
-    if (strcmp(value, "true") == 0)
+    if (sys_strcmp(value, "true") == 0)
 	return 1;
-    else if (strcmp(value, "false") == 0)
+    else if (sys_strcmp(value, "false") == 0)
 	return 0;
     else
 	bad_value(param, param_end, value);
@@ -1369,46 +1369,46 @@ handle_au_arg(struct au_init *auip,
 	}
 	else if(has_prefix("as", sub_param)) {
 	    char *alg = get_value(sub_param + 2, argv, ip);
-	    if (strcmp("bf", alg) == 0) {
+	    if (sys_strcmp("bf", alg) == 0) {
 		auip->atype = BESTFIT;
 		auip->init.bf.ao = 0;
 	    }
-	    else if (strcmp("aobf", alg) == 0) {
+	    else if (sys_strcmp("aobf", alg) == 0) {
 		auip->atype = BESTFIT;
 		auip->init.bf.ao = 1;
 	    }
-	    else if (strcmp("gf", alg) == 0) {
+	    else if (sys_strcmp("gf", alg) == 0) {
 		auip->atype = GOODFIT;
 	    }
-	    else if (strcmp("af", alg) == 0) {
+	    else if (sys_strcmp("af", alg) == 0) {
 		auip->atype = AFIT;
 	    }
-	    else if (strcmp("aoff", alg) == 0) {
+	    else if (sys_strcmp("aoff", alg) == 0) {
 		auip->atype = FIRSTFIT;
 		auip->init.aoff.crr_order = FF_AOFF;
 		auip->init.aoff.blk_order = FF_AOFF;
 	    }
-	    else if (strcmp("aoffcbf", alg) == 0) {
+	    else if (sys_strcmp("aoffcbf", alg) == 0) {
 		auip->atype = FIRSTFIT;
 		auip->init.aoff.crr_order = FF_AOFF;
 		auip->init.aoff.blk_order = FF_BF;
 	    }
-	    else if (strcmp("aoffcaobf", alg) == 0) {
+	    else if (sys_strcmp("aoffcaobf", alg) == 0) {
 		auip->atype = FIRSTFIT;
 		auip->init.aoff.crr_order = FF_AOFF;
 		auip->init.aoff.blk_order = FF_AOBF;
 	    }
-            else if (strcmp("ageffcaoff", alg) == 0) {
+            else if (sys_strcmp("ageffcaoff", alg) == 0) {
                 auip->atype = FIRSTFIT;
 		auip->init.aoff.crr_order = FF_AGEFF;
 		auip->init.aoff.blk_order = FF_AOFF;
             }
-            else if (strcmp("ageffcbf", alg) == 0) {
+            else if (sys_strcmp("ageffcbf", alg) == 0) {
                 auip->atype = FIRSTFIT;
 		auip->init.aoff.crr_order = FF_AGEFF;
 		auip->init.aoff.blk_order = FF_BF;
             }
-            else if (strcmp("ageffcaobf", alg) == 0) {
+            else if (sys_strcmp("ageffcaobf", alg) == 0) {
                 auip->atype = FIRSTFIT;
 		auip->init.aoff.crr_order = FF_AGEFF;
 		auip->init.aoff.blk_order = FF_AOBF;
@@ -1688,7 +1688,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 		    }
 		    else if (has_prefix("e", param+2)) {
 			arg = get_value(param+3, argv, &i);
-			if (strcmp("true", arg) != 0)
+			if (sys_strcmp("true", arg) != 0)
 			    bad_value(param, param+3, arg);
 		    }
 		    else
@@ -1700,20 +1700,20 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 		    case 'a': {
 			int a;
 			arg = get_value(argv[i]+4, argv, &i);
-			if (strcmp("min", arg) == 0) {
+			if (sys_strcmp("min", arg) == 0) {
 			    for (a = 0; a < aui_sz; a++)
 				aui[a]->enable = 0;
 			}
-			else if (strcmp("max", arg) == 0) {
+			else if (sys_strcmp("max", arg) == 0) {
 			    for (a = 0; a < aui_sz; a++)
 				aui[a]->enable = 1;
 			}
-			else if (strcmp("config", arg) == 0) {
+			else if (sys_strcmp("config", arg) == 0) {
 			    init->erts_alloc_config = 1;
 			}
-			else if (strcmp("r9c", arg) == 0
-				 || strcmp("r10b", arg) == 0
-				 || strcmp("r11b", arg) == 0) {
+			else if (sys_strcmp("r9c", arg) == 0
+				 || sys_strcmp("r10b", arg) == 0
+				 || sys_strcmp("r11b", arg) == 0) {
 			    set_default_sl_alloc_opts(&init->sl_alloc);
 			    set_default_std_alloc_opts(&init->std_alloc);
 			    set_default_ll_alloc_opts(&init->ll_alloc);
@@ -1725,7 +1725,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 			    set_default_driver_alloc_opts(&init->fix_alloc);
 
 			    init->driver_alloc.enable = 0;
-			    if (strcmp("r9c", arg) == 0) {
+			    if (sys_strcmp("r9c", arg) == 0) {
 				init->sl_alloc.enable = 0;
 				init->std_alloc.enable = 0;
 				init->binary_alloc.enable = 0;
@@ -1752,18 +1752,18 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 		    switch (argv[i][3]) {
 		    case 's':
 			arg = get_value(argv[i]+4, argv, &i);
-			if (strcmp("true", arg) == 0)
+			if (sys_strcmp("true", arg) == 0)
 			    init->instr.stat = 1;
-			else if (strcmp("false", arg) == 0)
+			else if (sys_strcmp("false", arg) == 0)
 			    init->instr.stat = 0;
 			else
 			    bad_value(param, param+3, arg);
 			break;
 		    case 'm':
 			arg = get_value(argv[i]+4, argv, &i);
-			if (strcmp("true", arg) == 0)
+			if (sys_strcmp("true", arg) == 0)
 			    init->instr.map = 1;
-			else if (strcmp("false", arg) == 0)
+			else if (sys_strcmp("false", arg) == 0)
 			    init->instr.map = 0;
 			else
 			    bad_value(param, param+3, arg);
@@ -1778,9 +1778,9 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 		case 'l':
 		    if (has_prefix("pm", param+2)) {
 			arg = get_value(argv[i]+5, argv, &i);
-			if (strcmp("all", arg) == 0)
+			if (sys_strcmp("all", arg) == 0)
 			    lock_all_physical_memory = 1;
-			else if (strcmp("no", arg) == 0)
+			else if (sys_strcmp("no", arg) == 0)
 			    lock_all_physical_memory = 0;
 			else
 			    bad_value(param, param+4, arg);
@@ -1830,8 +1830,8 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 			/* || init->instr.stat
 			   || init->instr.map */) {
 			while (i < *argc) {
-			    if(strcmp(argv[i], "-sname") == 0
-			       || strcmp(argv[i], "-name") == 0) {
+			    if(sys_strcmp(argv[i], "-sname") == 0
+			       || sys_strcmp(argv[i], "-name") == 0) {
 				if (i + 1 <*argc) {
 				    init->instr.nodename = argv[i+1];
 				    break;
@@ -2692,7 +2692,7 @@ erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc)
 		Eterm atom;
 		if (hpp)
 		    atom = am_atom_put(values[i].name,
-				       (int) strlen(values[i].name));
+				       (int) sys_strlen(values[i].name));
 		else
 		    atom = am_true;
 
@@ -2883,7 +2883,7 @@ erts_allocator_options(void *proc)
     for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) {
 	Eterm tmp = NIL;
 	atoms[length] = am_atom_put((char *) ERTS_ALC_A2AD(a),
-				    strlen(ERTS_ALC_A2AD(a)));
+				    sys_strlen(ERTS_ALC_A2AD(a)));
 	if (erts_allctrs_info[a].enabled) {
 	    if (erts_allctrs_info[a].alloc_util) {
 		Allctr_t *allctr;
@@ -2977,7 +2977,7 @@ erts_allocator_options(void *proc)
     for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) {
 	if (erts_allctrs_info[a].enabled) {
 	    terms[length++] = am_atom_put((char *) ERTS_ALC_A2AD(a),
-					  strlen(ERTS_ALC_A2AD(a)));
+					  sys_strlen(ERTS_ALC_A2AD(a)));
 	}
     }
 
@@ -3961,7 +3961,7 @@ set_memory_fence(void *ptr, Uint sz, ErtsAlcType_t n)
 
     *(ui_ptr++) = sz;
     *(ui_ptr++) = pattern;
-    memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord));
+    sys_memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord));
 
 #ifdef HARD_DEBUG
     *mblkpp = hdbg_alloc((void *) ui_ptr, sz, n);
@@ -4001,7 +4001,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
 		     (UWord) ptr);
     }
 
-    memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord));
+    sys_memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord));
 
     if (post_pattern != MK_PATTERN(n)
 	|| pre_pattern != post_pattern) {
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 174bf7a80a..578a3717d9 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -434,7 +434,7 @@ NAME##_free(TYPE *p)							\
 
 #ifdef DEBUG
 #define ERTS_PRE_ALLOC_SIZE(SZ) ((SZ) < 1000 ? (SZ)/10 + 10 : 100)
-#define ERTS_PRE_ALLOC_CLOBBER(P, T) memset((void *) (P), 0xfd, sizeof(T))
+#define ERTS_PRE_ALLOC_CLOBBER(P, T) sys_memset((void *) (P), 0xfd, sizeof(T))
 #else
 #define ERTS_PRE_ALLOC_SIZE(SZ) ((SZ) > 1 ? (SZ) : 1)
 #define ERTS_PRE_ALLOC_CLOBBER(P, T)
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index fa97ead908..d178c2c2c2 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -4315,7 +4315,7 @@ static Eterm fix_type_atoms[ERTS_ALC_NO_FIXED_SIZES];
 
 static ERTS_INLINE void atom_init(Eterm *atom, char *name)
 {
-    *atom = am_atom_put(name, strlen(name));
+    *atom = am_atom_put(name, sys_strlen(name));
 }
 #define AM_INIT(AM) atom_init(&am.AM, #AM)
 
@@ -4416,7 +4416,7 @@ init_atoms(Allctr_t *allctr)
 	for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) {
 	    ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix;
 	    char *name = (char *) ERTS_ALC_N2TD(n);
-	    size_t len = strlen(name);
+	    size_t len = sys_strlen(name);
 	    fix_type_atoms[ix] = am_atom_put(name, len);
 	}
     }
@@ -4854,20 +4854,20 @@ make_name_atoms(Allctr_t *allctr)
     char realloc[] = "realloc";
     char free[] = "free";
     char buf[MAX_ATOM_CHARACTERS];
-    size_t prefix_len = strlen(allctr->name_prefix);
+    size_t prefix_len = sys_strlen(allctr->name_prefix);
 
     if (prefix_len > MAX_ATOM_CHARACTERS + sizeof(realloc) - 1)
 	erts_exit(ERTS_ERROR_EXIT,"Too long allocator name: %salloc\n",allctr->name_prefix);
 
-    memcpy((void *) buf, (void *) allctr->name_prefix, prefix_len);
+    sys_memcpy((void *) buf, (void *) allctr->name_prefix, prefix_len);
 
-    memcpy((void *) &buf[prefix_len], (void *) alloc, sizeof(alloc) - 1);
+    sys_memcpy((void *) &buf[prefix_len], (void *) alloc, sizeof(alloc) - 1);
     allctr->name.alloc = am_atom_put(buf, prefix_len + sizeof(alloc) - 1);
 
-    memcpy((void *) &buf[prefix_len], (void *) realloc, sizeof(realloc) - 1);
+    sys_memcpy((void *) &buf[prefix_len], (void *) realloc, sizeof(realloc) - 1);
     allctr->name.realloc = am_atom_put(buf, prefix_len + sizeof(realloc) - 1);
 
-    memcpy((void *) &buf[prefix_len], (void *) free, sizeof(free) - 1);
+    sys_memcpy((void *) &buf[prefix_len], (void *) free, sizeof(free) - 1);
     allctr->name.free = am_atom_put(buf, prefix_len + sizeof(free) - 1);
 
 }
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index f8a6101b95..0c5545401a 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -1054,7 +1054,7 @@ static struct {
 
 static void ERTS_INLINE atom_init(Eterm *atom, const char *name)
 {
-    *atom = am_atom_put(name, strlen(name));
+    *atom = am_atom_put(name, sys_strlen(name));
 }
 #define AM_INIT(AM) atom_init(&am.AM, #AM)
 
diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c
index 6173c408e1..85fc4c3a85 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.c
+++ b/erts/emulator/beam/erl_bestfit_alloc.c
@@ -875,7 +875,7 @@ static struct {
 
 static void ERTS_INLINE atom_init(Eterm *atom, char *name)
 {
-    *atom = am_atom_put(name, strlen(name));
+    *atom = am_atom_put(name, sys_strlen(name));
 }
 #define AM_INIT(AM) atom_init(&am.AM, #AM)
 
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 33bc189182..469f6a1ea8 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -365,7 +365,7 @@ static ACTrie *create_acdata(MyAllocator *my, Uint len,
     acn->d = 0;
     acn->final = 0;
     acn->h = NULL;
-    memset(acn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE);
+    sys_memset(acn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE);
 #ifdef HARDDEBUG
     act->idc = 0;
     acn->id = 0;
@@ -388,7 +388,7 @@ static BMData *create_bmdata(MyAllocator *my, byte *x, Uint len,
     init_my_allocator(my, datasize, data);
     bmd = my_alloc(my, sizeof(BMData));
     bmd->x = my_alloc(my,len);
-    memcpy(bmd->x,x,len);
+    sys_memcpy(bmd->x,x,len);
     bmd->len = len;
     bmd->goodshift = my_alloc(my,sizeof(Uint) * len);
     *the_bin = mb;
@@ -433,7 +433,7 @@ static void ac_add_one_pattern(MyAllocator *my, ACTrie *act, byte *x, Uint len)
 	    nn->d = i+1;
 	    nn->h = act->root;
 	    nn->final = 0;
-	    memset(nn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE);
+	    sys_memset(nn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE);
 	    acn->g[x[i]] = nn;
 	    ++i;
 	    acn = nn;
@@ -2264,7 +2264,7 @@ static BIF_RETTYPE do_longest_common(Process *p, Eterm list, int direction)
 	    if (cd[i].type == CL_TYPE_HEAP_NOALLOC) {
 		unsigned char *tmp = cd[i].buff;
 		cd[i].buff = erts_alloc(ERTS_ALC_T_BINARY_BUFFER, cd[i].bufflen);
-		memcpy(cd[i].buff,tmp,cd[i].bufflen);
+		sys_memcpy(cd[i].buff,tmp,cd[i].bufflen);
 		cd[i].type = CL_TYPE_HEAP;
 	    }
 	}
@@ -2556,7 +2556,7 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en)
 						    0);
 	    if (!(cbs->temp_alloc)) { /* alignment not needed, need to copy */
 		byte *tmp = erts_alloc(ERTS_ALC_T_BINARY_BUFFER,size);
-		memcpy(tmp,cbs->source,size);
+		sys_memcpy(tmp,cbs->source,size);
 		cbs->source = tmp;
 		cbs->source_type = BC_TYPE_HEAP;
 	    } else {
@@ -2569,7 +2569,7 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en)
 	pos = 0;
 	i = 0;
 	while (pos < reds) {
-	    memcpy(t+pos,cbs->source, size);
+	    sys_memcpy(t+pos,cbs->source, size);
 	    pos += size;
 	    ++i;
 	}
@@ -2594,7 +2594,7 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en)
 	}
 	pos = 0;
 	while (pos < target_size) {
-	    memcpy(t+pos,source, size);
+	    sys_memcpy(t+pos,source, size);
 	    pos += size;
 	}
 	erts_free_aligned_binary_bytes(temp_alloc);
@@ -2624,7 +2624,7 @@ BIF_RETTYPE binary_copy_trap(BIF_ALIST_2)
     if ((n-1) * size >= reds) {
 	Uint i = 0;
 	while ((pos - opos) < reds) {
-	    memcpy(t+pos,cbs->source, size);
+	    sys_memcpy(t+pos,cbs->source, size);
 	    pos += size;
 	    ++i;
 	}
@@ -2637,7 +2637,7 @@ BIF_RETTYPE binary_copy_trap(BIF_ALIST_2)
         Eterm resbin;
 	Uint target_size = cbs->result->orig_size;
 	while (pos < target_size) {
-	    memcpy(t+pos,cbs->source, size);
+	    sys_memcpy(t+pos,cbs->source, size);
 	    pos += size;
 	}
 	save = cbs->result;
@@ -3029,7 +3029,7 @@ static void dump_bm_data(BMData *bm)
 static void dump_ac_node(ACNode *node, int indent, int ch) {
     int i;
     char *spaces = erts_alloc(ERTS_ALC_T_TMP, 10 * indent + 1);
-    memset(spaces,' ',10*indent);
+    sys_memset(spaces,' ',10*indent);
     spaces[10*indent] = '\0';
     erts_printf("%s-> %c\n",spaces,ch);
     erts_printf("%sId: %u\n",spaces,(unsigned) node->id);
diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c
index 9417803e14..9095bcd380 100644
--- a/erts/emulator/beam/erl_bif_chksum.c
+++ b/erts/emulator/beam/erl_bif_chksum.c
@@ -516,7 +516,7 @@ md5_2(BIF_ALIST_2)
     /* No need to check context, this function cannot be called with unaligned
        or badly sized context as it's always trapped to. */
     bytes = binary_bytes(BIF_ARG_1);
-    memcpy(&context,bytes,sizeof(MD5_CTX));
+    sys_memcpy(&context,bytes,sizeof(MD5_CTX));
     rest = do_chksum(&md5_wrap,BIF_P,BIF_ARG_2,100,(void *) &context,&res,
 		     &err);
     if (err != 0) {
@@ -564,7 +564,7 @@ md5_update_2(BIF_ALIST_2)
 	erts_free_aligned_binary_bytes(temp_alloc);
 	BIF_ERROR(BIF_P, BADARG);
     }
-    memcpy(&context,bytes,sizeof(MD5_CTX));
+    sys_memcpy(&context,bytes,sizeof(MD5_CTX));
     erts_free_aligned_binary_bytes(temp_alloc);
     rest = do_chksum(&md5_wrap,BIF_P,BIF_ARG_2,100,(void *) &context,&res,
 		     &err);
@@ -599,7 +599,7 @@ md5_final_1(BIF_ALIST_1)
 	goto error;
     }
     bin = erts_new_heap_binary(BIF_P, (byte *)NULL, 16, &result);
-    memcpy(&ctx_copy, context, sizeof(MD5_CTX));
+    sys_memcpy(&ctx_copy, context, sizeof(MD5_CTX));
     erts_free_aligned_binary_bytes(temp_alloc);
     MD5Final(result, &ctx_copy);
     BIF_RET(bin);
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index f673ef3194..579e9b12f4 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1412,7 +1412,7 @@ static Eterm copy_ref(Eterm ref, Eterm *hp)
     ErtsORefThing *ptr;
     ASSERT(is_internal_ordinary_ref(ref));
     ptr = ordinary_ref_thing_ptr(ref);
-    memcpy(hp, ptr, sizeof(ErtsORefThing));
+    sys_memcpy(hp, ptr, sizeof(ErtsORefThing));
     return (make_internal_ref(hp));
 }
 
@@ -1454,9 +1454,9 @@ static void set_driver_reloading(DE_Handle *dh, Process *proc, char *path, char
     dh->procs = p;
     dh->status = ERL_DE_RELOAD;
     dh->reload_full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1);
-    strcpy(dh->reload_full_path,path);
+    sys_strcpy(dh->reload_full_path,path);
     dh->reload_driver_name = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(name) + 1);
-    strcpy(dh->reload_driver_name,name);
+    sys_strcpy(dh->reload_driver_name,name);
     dh->reload_flags = flags;
 }
 
@@ -1501,7 +1501,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name)
 	goto error;
     }
 
-    if (strcmp(name, dp->driver_name) != 0) {
+    if (sys_strcmp(name, dp->driver_name) != 0) {
 	res = ERL_DE_LOAD_ERROR_BAD_NAME;
 	goto error;
     }
@@ -1799,7 +1799,7 @@ static char *pick_list_or_atom(Eterm name_term)
 	    goto error;
 	}
 	name = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, ap->len + 1);
-	memcpy(name,ap->name,ap->len);
+	sys_memcpy(name,ap->name,ap->len);
 	name[ap->len] = '\0';
     } else {
 	if (erts_iolist_size(name_term, &name_len)) {
@@ -1870,7 +1870,7 @@ static erts_driver_t *lookup_driver(char *name)
 {
     erts_driver_t *drv;
     assert_drv_list_locked();
-    for (drv = driver_list; drv != NULL && strcmp(drv->name, name); drv = drv->next)
+    for (drv = driver_list; drv != NULL && sys_strcmp(drv->name, name); drv = drv->next)
 	;
     return drv;
 }
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 903f54e2fb..6475f04c56 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2378,12 +2378,12 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(os_version_tuple);
     }
     else if (BIF_ARG_1 == am_version) {
-	int n = strlen(ERLANG_VERSION);
+	int n = sys_strlen(ERLANG_VERSION);
 	hp = HAlloc(BIF_P, ((sizeof ERLANG_VERSION)-1) * 2);
 	BIF_RET(buf_to_intlist(&hp, ERLANG_VERSION, n, NIL));
     }
     else if (BIF_ARG_1 == am_machine) {
-	int n = strlen(EMULATOR);
+	int n = sys_strlen(EMULATOR);
 	hp = HAlloc(BIF_P, n*2);
 	BIF_RET(buf_to_intlist(&hp, EMULATOR, n, NIL));
     }
@@ -2409,7 +2409,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	    res = erts_bld_cons(hpp, hszp,
 				erts_bld_tuple(hpp, hszp, 2,
 					       erts_atom_put((byte *)opc[i].name,
-							     strlen(opc[i].name),
+							     sys_strlen(opc[i].name),
 							     ERTS_ATOM_ENC_LATIN1,
 							     1),
 					       erts_bld_uint(hpp, hszp,
@@ -2805,21 +2805,21 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	Uint  sz;
 	Eterm res = NIL, tup, text;
 	Eterm *hp = HAlloc(BIF_P, 3*(2 + 3) + /* three 2-tuples and three cons */
-		2*(strlen(erts_build_flags_CONFIG_H) +
-		   strlen(erts_build_flags_CFLAGS) +
-		   strlen(erts_build_flags_LDFLAGS)));
+		2*(sys_strlen(erts_build_flags_CONFIG_H) +
+		   sys_strlen(erts_build_flags_CFLAGS) +
+		   sys_strlen(erts_build_flags_LDFLAGS)));
 
-	sz   = strlen(erts_build_flags_CONFIG_H);
+	sz   = sys_strlen(erts_build_flags_CONFIG_H);
 	text = buf_to_intlist(&hp, erts_build_flags_CONFIG_H, sz, NIL);
 	tup  = TUPLE2(hp, am_config_h, text); hp += 3;
 	res  = CONS(hp, tup, res); hp += 2;
 
-	sz   = strlen(erts_build_flags_CFLAGS);
+	sz   = sys_strlen(erts_build_flags_CFLAGS);
 	text = buf_to_intlist(&hp, erts_build_flags_CFLAGS, sz, NIL);
 	tup  = TUPLE2(hp, am_cflags, text); hp += 3;
 	res  = CONS(hp, tup, res); hp += 2;
 
-	sz   = strlen(erts_build_flags_LDFLAGS);
+	sz   = sys_strlen(erts_build_flags_LDFLAGS);
 	text = buf_to_intlist(&hp, erts_build_flags_LDFLAGS, sz, NIL);
 	tup  = TUPLE2(hp, am_ldflags, text); hp += 3;
 	res  = CONS(hp, tup, res); hp += 2;
@@ -3883,7 +3883,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 		while (ix >= atom_table_size()) {
 		    char tmp[20];
 		    erts_snprintf(tmp, sizeof(tmp), "am%x", atom_table_size());
-		    erts_atom_put((byte *) tmp, strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1);
+		    erts_atom_put((byte *) tmp, sys_strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1);
 		}
 		return make_atom(ix);
 	    }
@@ -4449,7 +4449,7 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s
 
     file = stats->file ? stats->file : "undefined";
 
-    af    = erts_atom_put((byte *)file, strlen(file), ERTS_ATOM_ENC_LATIN1, 1);
+    af    = erts_atom_put((byte *)file, sys_strlen(file), ERTS_ATOM_ENC_LATIN1, 1);
     uil   = erts_bld_uint( hpp, szp, stats->line);
     tloc  = erts_bld_tuple(hpp, szp, 2, af, uil);
 
@@ -4495,7 +4495,7 @@ static Eterm lcnt_pretty_print_lock_id(erts_lcnt_lock_info_t *info) {
     } else if(info->flags & ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR) {
         if(is_small(id) && !sys_strcmp(info->name, "alcu_allocator")) {
             const char *name = (const char*)ERTS_ALC_A2AD(signed_val(id));
-            id = erts_atom_put((byte*)name, strlen(name), ERTS_ATOM_ENC_LATIN1, 1);
+            id = erts_atom_put((byte*)name, sys_strlen(name), ERTS_ATOM_ENC_LATIN1, 1);
         }
     }
 
@@ -4515,8 +4515,8 @@ static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, lcnt_sample_t *sample,
     
     lock_desc = erts_lock_flags_get_type_name(info->flags);
 
-    type  = erts_atom_put((byte*)lock_desc, strlen(lock_desc), ERTS_ATOM_ENC_LATIN1, 1);
-    name  = erts_atom_put((byte*)info->name, strlen(info->name), ERTS_ATOM_ENC_LATIN1, 1);
+    type  = erts_atom_put((byte*)lock_desc, sys_strlen(lock_desc), ERTS_ATOM_ENC_LATIN1, 1);
+    name  = erts_atom_put((byte*)info->name, sys_strlen(info->name), ERTS_ATOM_ENC_LATIN1, 1);
 
     /* Only attempt to resolve ids when actually emitting the term. This ought
      * to be safe since all immediates are the same size. */
@@ -4552,11 +4552,11 @@ static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_time_t *du
     dtns = bld_unstable_uint64(hpp, szp, duration->ns);
     tdt  = erts_bld_tuple(hpp, szp, 2, dts, dtns);
 
-    adur = erts_atom_put((byte *)str_duration, strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1);
+    adur = erts_atom_put((byte *)str_duration, sys_strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1);
     tdur = erts_bld_tuple(hpp, szp, 2, adur, tdt);
    
     /* lock tuple */
-    aloc = erts_atom_put((byte *)str_locks, strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1);
+    aloc = erts_atom_put((byte *)str_locks, sys_strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1);
 
     for(i = 0; i < current_locks->size; i++) {
         lloc = lcnt_build_lock_term(hpp, szp, &current_locks->elements[i], lloc);
@@ -4610,7 +4610,7 @@ static Eterm lcnt_build_category_list(Eterm **hpp, Uint *szp, erts_lock_flags_t
     for(i = 0; lcnt_category_map[i].name != NULL; i++) {
         if(mask & lcnt_category_map[i].flag) {
             Eterm category = erts_atom_put((byte*)lcnt_category_map[i].name,
-                                           strlen(lcnt_category_map[i].name),
+                                           sys_strlen(lcnt_category_map[i].name),
                                            ERTS_ATOM_ENC_UTF8, 0);
 
             res = erts_bld_cons(hpp, szp, category, res);
@@ -4740,14 +4740,14 @@ BIF_RETTYPE erts_debug_lcnt_control_2(BIF_ALIST_2)
 
 static void os_info_init(void)
 {
-    Eterm type = erts_atom_put((byte *) os_type, strlen(os_type), ERTS_ATOM_ENC_LATIN1, 1);
+    Eterm type = erts_atom_put((byte *) os_type, sys_strlen(os_type), ERTS_ATOM_ENC_LATIN1, 1);
     Eterm flav;
     int major, minor, build;
     char* buf = erts_alloc(ERTS_ALC_T_TMP, 1024); /* More than enough */
     Eterm* hp;
 
     os_flavor(buf, 1024);
-    flav = erts_atom_put((byte *) buf, strlen(buf), ERTS_ATOM_ENC_LATIN1, 1);
+    flav = erts_atom_put((byte *) buf, sys_strlen(buf), ERTS_ATOM_ENC_LATIN1, 1);
     erts_free(ERTS_ALC_T_TMP, (void *) buf);
     hp = erts_alloc(ERTS_ALC_T_LITERAL, (3+4)*sizeof(Eterm));
     os_type_tuple = TUPLE2(hp, type, flav);
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index b184adedee..1feb892b93 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -71,7 +71,7 @@ BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2)
                 BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
 	} else if (err_type == -2) {
 	    str = erl_errno_id(err_num);
-            res = erts_atom_put((byte *) str, strlen(str), ERTS_ATOM_ENC_LATIN1, 1);
+            res = erts_atom_put((byte *) str, sys_strlen(str), ERTS_ATOM_ENC_LATIN1, 1);
 	} else {
 	    res = am_einval;
 	}
@@ -1145,7 +1145,7 @@ http_bld_string(struct packet_callback_args* pca, Uint **hpp, Uint *szp,
 		ErlHeapBin* bin = (ErlHeapBin*) *hpp;
 		bin->thing_word = header_heap_bin(len);
 		bin->size = len;
-		memcpy(bin->data, str, len);
+		sys_memcpy(bin->data, str, len);
 		*hpp += size;
 	    }
 	}
@@ -1323,9 +1323,9 @@ int ssl_tls_erl(void* arg, unsigned type, unsigned major, unsigned minor,
     Eterm bin = new_binary(pca->p, NULL, plen+len);
     byte* bin_ptr = binary_bytes(bin);
 
-    memcpy(bin_ptr+plen, buf, len);
+    sys_memcpy(bin_ptr+plen, buf, len);
     if (plen) {
-        memcpy(bin_ptr, prefix, plen);
+        sys_memcpy(bin_ptr, prefix, plen);
     }
 
     /* {ssl_tls,NIL,ContentType,{Major,Minor},Bin} */
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index bc819505e7..4d769c2d46 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -514,7 +514,7 @@ re_version_0(BIF_ALIST_0)
     Eterm ret;
     size_t version_size = 0;
     byte *version = (byte *) erts_pcre_version();
-    version_size = strlen((const char *) version);
+    version_size = sys_strlen((const char *) version);
     ret = new_binary(BIF_P, version, version_size);
     BIF_RET(ret);
 }
@@ -998,7 +998,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
 	    has_dupnames = ((options & PCRE_DUPNAMES) != 0);
 
 	    for(i=0;i<top;++i) {
-		if (last == NULL || !has_dupnames || strcmp((char *) last+2,(char *) nametable+2)) {
+		if (last == NULL || !has_dupnames || sys_strcmp((char *) last+2,(char *) nametable+2)) {
 		    ASSERT(ri->num_spec >= 0);
 		    ++(ri->num_spec);
 		    if(ri->num_spec > sallocated) {
@@ -1048,7 +1048,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
 						    (tmpbsiz = ap->len + 1));
 			    }
 			}
-			memcpy(tmpb,ap->name,ap->len);
+			sys_memcpy(tmpb,ap->name,ap->len);
 			tmpb[ap->len] = '\0';
 		    } else {
 			ErlDrvSizeT slen;
@@ -1210,7 +1210,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 	    erts_pcre_fullinfo(result, NULL, PCRE_INFO_CAPTURECOUNT, &capture_count);
 	    ovsize = 3*(capture_count+1);
 	    restart.code = erts_alloc(ERTS_ALC_T_RE_SUBJECT, code_size);
-	    memcpy(restart.code, result, code_size);
+	    sys_memcpy(restart.code, result, code_size);
 	    erts_pcre_free(result);
 	    erts_free(ERTS_ALC_T_RE_TMP_BUF, expr);
 	    /*unicode = (pflags & PARSE_FLAG_UNICODE) ? 1 : 0;*/
@@ -1252,7 +1252,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 	    BIF_ERROR(p, BADARG);
 	}
 	restart.code = erts_alloc(ERTS_ALC_T_RE_SUBJECT, code_size);
-	memcpy(restart.code, code_tmp, code_size);
+	sys_memcpy(restart.code, code_tmp, code_size);
 	erts_free_aligned_binary_bytes(temp_alloc);
 
     }
@@ -1364,7 +1364,7 @@ handle_iolist:
 	RestartContext *restartp = ERTS_MAGIC_BIN_DATA(mbp);
 	Eterm magic_ref;
 	Eterm *hp;
-	memcpy(restartp,&restart,sizeof(RestartContext));
+	sys_memcpy(restartp,&restart,sizeof(RestartContext));
 	BUMP_ALL_REDS(p);
 	hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
 	magic_ref = erts_mk_magic_ref(&hp, &MSO(p), mbp);
@@ -1517,7 +1517,7 @@ re_inspect_2(BIF_ALIST_2)
     last = NULL;
     name = nametable;
     for(i=0;i<top;++i) {
-	if (last == NULL || !has_dupnames || strcmp((char *) last+2,
+	if (last == NULL || !has_dupnames || sys_strcmp((char *) last+2,
 						    (char *) name+2)) {
 	    ++num_names;
 	}
@@ -1531,9 +1531,9 @@ re_inspect_2(BIF_ALIST_2)
     name = nametable;
     j = 0;
     for(i=0;i<top;++i) {
-	if (last == NULL || !has_dupnames || strcmp((char *) last+2,
+	if (last == NULL || !has_dupnames || sys_strcmp((char *) last+2,
 						    (char *) name+2)) {
-	    tmp_vec[j++] = new_binary(BIF_P, (byte *) name+2, strlen((char *) name+2));
+	    tmp_vec[j++] = new_binary(BIF_P, (byte *) name+2, sys_strlen((char *) name+2));
 	}
 	last = name;
 	name += entrysize;
diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h
index 9aa631fde9..6af9d4ac8c 100644
--- a/erts/emulator/beam/erl_bif_unique.h
+++ b/erts/emulator/beam/erl_bif_unique.h
@@ -307,7 +307,7 @@ erts_iref_storage_clean(ErtsIRefStorage *iref)
     if (iref->is_magic && erts_refc_dectest(&iref->u.mb->intern.refc, 0) == 0)
 	erts_ref_bin_free(iref->u.mb);
 #ifdef DEBUG
-    memset((void *) iref, 0xf, sizeof(ErtsIRefStorage));
+    sys_memset((void *) iref, 0xf, sizeof(ErtsIRefStorage));
 #endif
 }
 
@@ -342,7 +342,7 @@ erts_iref_storage_make_ref(ErtsIRefStorage *iref,
 
 #ifdef DEBUG
     if (clean_storage)
-	memset((void *) iref, 0xf, sizeof(ErtsIRefStorage));
+	sys_memset((void *) iref, 0xf, sizeof(ErtsIRefStorage));
 #endif
 
     return make_internal_ref(hp);
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 1ab1c4a363..a76d769283 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -4261,7 +4261,7 @@ erts_ets_colliding_names(Process* p, Eterm name, Uint cnt)
             while (index >= atom_table_size()) {
                 char tmp[20];
                 erts_snprintf(tmp, sizeof(tmp), "am%x", atom_table_size());
-                erts_atom_put((byte *) tmp, strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1);
+                erts_atom_put((byte *) tmp, sys_strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1);
             }
             list = CONS(hp, make_atom(index), list);
             hp += 2;
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index e017b9552b..956662aefa 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -91,7 +91,7 @@ do {									\
 	(On).siz *= 2;							\
 	(On).data							\
 	    = (((On).def == (On).data)					\
-	       ? memcpy(erts_alloc(ERTS_ALC_T_DB_MC_STK,		\
+	       ? sys_memcpy(erts_alloc(ERTS_ALC_T_DB_MC_STK,		\
 				   (On).siz*sizeof(*((On).data))),	\
 			(On).def,					\
 			DMC_DEFAULT_SIZE*sizeof(*((On).data)))		\
@@ -1518,7 +1518,7 @@ restart:
 	 context.current_match < num_progs; 
 	 ++context.current_match) { /* This loop is long, 
 				       too long */
-	memset(heap.vars, 0, heap.size * sizeof(*heap.vars));
+	sys_memset(heap.vars, 0, heap.size * sizeof(*heap.vars));
 	t = context.matchexpr[context.current_match];
 	context.stack_used = 0;
 	structure_checked = 0;
@@ -2140,7 +2140,7 @@ restart:
 	case matchEqFloat:
 	    if (!is_float(*ep))
 		FAIL();
-	    if (memcmp(float_val(*ep) + 1, pc, sizeof(double)))
+	    if (sys_memcmp(float_val(*ep) + 1, pc, sizeof(double)))
 		FAIL();
 	    pc += TermWords(2);
 	    ++ep;
@@ -2742,8 +2742,8 @@ Eterm db_format_dmc_err_info(Process *p, DMCErrInfo *ei)
 	if (vnum >= 0)
 	    erts_snprintf(buff,sizeof(buff)+20,tmp->error_string, vnum);
 	else
-	    strcpy(buff,tmp->error_string);
-	sl = strlen(buff);
+	    sys_strcpy(buff,tmp->error_string);
+	sl = sys_strlen(buff);
 	shp = HAlloc(p, sl * 2 + 5);
 	sev = (tmp->severity == dmcWarning) ? 
 	    am_atom_put("warning",7) :
@@ -5044,7 +5044,7 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info)
 	    ASSERT(j < x);
 	    erts_snprintf(buff+1, sizeof(buff) - 1, "%u", (unsigned) j);
 	    /* Yes, writing directly into terms, they ARE off heap */
-	    *p = erts_atom_put((byte *) buff, strlen(buff),
+	    *p = erts_atom_put((byte *) buff, sys_strlen(buff),
 			       ERTS_ATOM_ENC_LATIN1, 1);
 	}
 	++p;
@@ -5505,7 +5505,7 @@ void db_match_dis(Binary *bp)
 	    ++t;
 	    {
 		double num;
-		memcpy(&num,t,sizeof(double));
+		sys_memcpy(&num,t,sizeof(double));
 		t += TermWords(2);
 		erts_printf("EqFloat\t%f\n", num);
 	    }
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 71d4534ef9..4cf42fce57 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -439,7 +439,7 @@ erl_drv_tsd_key_create(char *name, ErlDrvTSDKey *key)
 	name_copy = no_name;
     else {
 	name_copy = erts_alloc_fnf(ERTS_ALC_T_DRV_TSD,
-				   sizeof(char)*(strlen(name) + 1));
+				   sizeof(char)*(sys_strlen(name) + 1));
 	if (!name_copy) {
 	    *key = -1;
 	    return ENOMEM;
@@ -750,7 +750,7 @@ erl_drv_steal_main_thread(char *name,
 			   + (name ? sys_strlen(name) + 1 : 0)));
     if (!dtid)
 	return ENOMEM;
-    memset(dtid,0,sizeof(ErlDrvTid_));
+    sys_memset(dtid,0,sizeof(ErlDrvTid_));
     dtid->tid = (void * ) -1;
     dtid->drv_thr = 1;
     dtid->func = func;
@@ -763,8 +763,8 @@ erl_drv_steal_main_thread(char *name,
     *tid = NULL;
     /* Ignore options and name... */
     
-    memcpy(buff,&func,sizeof(void* (*)(void*)));
-    memcpy(buff + sizeof(void* (*)(void*)),&arg,sizeof(void *));
+    sys_memcpy(buff,&func,sizeof(void* (*)(void*)));
+    sys_memcpy(buff + sizeof(void* (*)(void*)),&arg,sizeof(void *));
     write(erts_darwin_main_thread_pipe[1],buff,buff_sz);
     return 0;
 }
diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c
index a38f6c7daf..e3ba67f0af 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.c
+++ b/erts/emulator/beam/erl_goodfit_alloc.c
@@ -508,7 +508,7 @@ static struct {
 
 static void ERTS_INLINE atom_init(Eterm *atom, char *name)
 {
-    *atom = am_atom_put(name, strlen(name));
+    *atom = am_atom_put(name, sys_strlen(name));
 }
 #define AM_INIT(AM) atom_init(&am.AM, #AM)
 
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 4846ccd2d3..c1760a2e65 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -238,7 +238,7 @@ progname(char *fullname)
 {
     int i;
     
-    i = strlen(fullname);
+    i = sys_strlen(fullname);
     while (i >= 0) {
 	if ((fullname[i] != '/') && (fullname[i] != '\\')) 
 	    i--;
@@ -810,7 +810,7 @@ early_init(int *argc, char **argv) /*
     if (argc && argv) {
 	int i = 1;
 	while (i < *argc) {
-	    if (strcmp(argv[i], "--") == 0) { /* end of emulator options */
+	    if (sys_strcmp(argv[i], "--") == 0) { /* end of emulator options */
 		i++;
 		break;
 	    }
@@ -892,7 +892,7 @@ early_init(int *argc, char **argv) /*
 		    else if (argv[i][2] == 'D') {
 			char *arg;
 			char *type = argv[i]+3;
-			if (strncmp(type, "Pcpu", 4) == 0) {
+			if (sys_strncmp(type, "Pcpu", 4) == 0) {
 			    int ptot, ponln;
 			    arg = get_arg(argv[i]+7, argv[i+1], &i);
 			    switch (sscanf(arg, "%d:%d", &ptot, &ponln)) {
@@ -929,7 +929,7 @@ early_init(int *argc, char **argv) /*
 			    VERBOSE(DEBUG_SYSTEM,
 				    ("using %d:%d dirty CPU scheduler percentages\n",
 				     dirty_cpu_scheds_pctg, dirty_cpu_scheds_onln_pctg));
-			} else if (strncmp(type, "cpu", 3) == 0) {
+			} else if (sys_strncmp(type, "cpu", 3) == 0) {
 			    int tot, onln;
 			    arg = get_arg(argv[i]+6, argv[i+1], &i);
 			    switch (sscanf(arg, "%d:%d", &tot, &onln)) {
@@ -980,7 +980,7 @@ early_init(int *argc, char **argv) /*
 			    }
 			    VERBOSE(DEBUG_SYSTEM,
 				    ("using %d:%d dirty CPU scheduler(s)\n", tot, onln));
-			} else if (strncmp(type, "io", 2) == 0) {
+			} else if (sys_strncmp(type, "io", 2) == 0) {
 			    arg = get_arg(argv[i]+5, argv[i+1], &i);
 			    dirty_io_scheds = atoi(arg);
 			    if (dirty_io_scheds < 0 ||
@@ -1242,7 +1242,7 @@ erl_start(int argc, char **argv)
 	if (argv[i][0] != '-') {
 	    erts_usage();
 	}
-	if (strcmp(argv[i], "--") == 0) { /* end of emulator options */
+	if (sys_strcmp(argv[i], "--") == 0) { /* end of emulator options */
 	    i++;
 	    break;
 	}
@@ -1270,12 +1270,12 @@ erl_start(int argc, char **argv)
                     ("using display items %d\n",display_items));
 	    break;
 	case 'p':
-	    if (!strncmp(argv[i],"-pc",3)) {
+	    if (!sys_strncmp(argv[i],"-pc",3)) {
 		int printable_chars = ERL_PRINTABLE_CHARACTERS_LATIN1;
 		arg = get_arg(argv[i]+3, argv[i+1], &i);
-		if (!strcmp(arg,"unicode")) {
+		if (!sys_strcmp(arg,"unicode")) {
 		    printable_chars = ERL_PRINTABLE_CHARACTERS_UNICODE;
-		} else if (strcmp(arg,"latin1")) {
+		} else if (sys_strcmp(arg,"latin1")) {
 		    erts_fprintf(stderr, "bad range of printable "
 				 "characters: %s\n", arg);
 		    erts_usage();
@@ -1287,7 +1287,7 @@ erl_start(int argc, char **argv)
 		erts_usage();
 	    }
 	case 'f':
-	    if (!strncmp(argv[i],"-fn",3)) {
+	    if (!sys_strncmp(argv[i],"-fn",3)) {
 		int warning_type =  ERL_FILENAME_WARNING_WARNING;
 		arg = get_arg(argv[i]+3, argv[i+1], &i);
 		switch (*arg) {
@@ -1470,9 +1470,9 @@ erl_start(int argc, char **argv)
 		}
             } else if (has_prefix("maxk", sub_param)) {
 		arg = get_arg(sub_param+4, argv[i+1], &i);
-		if (strcmp(arg,"true") == 0) {
+		if (sys_strcmp(arg,"true") == 0) {
                     H_MAX_FLAGS |= MAX_HEAP_SIZE_KILL;
-                } else if (strcmp(arg,"false") == 0) {
+                } else if (sys_strcmp(arg,"false") == 0) {
                     H_MAX_FLAGS &= ~MAX_HEAP_SIZE_KILL;
                 } else {
 		    erts_fprintf(stderr, "bad max heap kill %s\n", arg);
@@ -1481,9 +1481,9 @@ erl_start(int argc, char **argv)
 		VERBOSE(DEBUG_SYSTEM, ("using max heap kill %d\n", H_MAX_FLAGS));
             } else if (has_prefix("maxel", sub_param)) {
 		arg = get_arg(sub_param+5, argv[i+1], &i);
-		if (strcmp(arg,"true") == 0) {
+		if (sys_strcmp(arg,"true") == 0) {
                     H_MAX_FLAGS |= MAX_HEAP_SIZE_LOG;
-                } else if (strcmp(arg,"false") == 0) {
+                } else if (sys_strcmp(arg,"false") == 0) {
                     H_MAX_FLAGS &= ~MAX_HEAP_SIZE_LOG;
                 } else {
 		    erts_fprintf(stderr, "bad max heap error logger %s\n", arg);
@@ -1601,7 +1601,7 @@ erl_start(int argc, char **argv)
 
 	case 'P': /* set maximum number of processes */
 	    arg = get_arg(argv[i]+2, argv[i+1], &i);
-	    if (strcmp(arg, "legacy") == 0)
+	    if (sys_strcmp(arg, "legacy") == 0)
 		legacy_proc_tab = 1;
 	    else {
 		errno = 0;
@@ -1617,7 +1617,7 @@ erl_start(int argc, char **argv)
 
 	case 'Q': /* set maximum number of ports */
 	    arg = get_arg(argv[i]+2, argv[i+1], &i);
-	    if (strcmp(arg, "legacy") == 0)
+	    if (sys_strcmp(arg, "legacy") == 0)
 		legacy_port_tab = 1;
 	    else {
 		errno = 0;
@@ -1635,11 +1635,11 @@ erl_start(int argc, char **argv)
 	case 'S' : /* Was handled in early_init() just read past it */
 	    if (argv[i][2] == 'D') {
 		char* type = argv[i]+3;
-		if (strcmp(type, "Pcpu") == 0)
+		if (sys_strcmp(type, "Pcpu") == 0)
 		    (void) get_arg(argv[i]+7, argv[i+1], &i);
-		if (strcmp(type, "cpu") == 0)
+		if (sys_strcmp(type, "cpu") == 0)
 		    (void) get_arg(argv[i]+6, argv[i+1], &i);
-		else if (strcmp(type, "io") == 0)
+		else if (sys_strcmp(type, "io") == 0)
 		    (void) get_arg(argv[i]+5, argv[i+1], &i);
 	    } else if (argv[i][2] == 'P')
 		(void) get_arg(argv[i]+3, argv[i+1], &i);
diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c
index 634509f880..2f70e7996e 100644
--- a/erts/emulator/beam/erl_instrument.c
+++ b/erts/emulator/beam/erl_instrument.c
@@ -103,7 +103,7 @@ static struct {
 
 static void ERTS_INLINE atom_init(Eterm *atom, const char *name)
 {
-    *atom = am_atom_put((char *) name, strlen(name));
+    *atom = am_atom_put((char *) name, sys_strlen(name));
 }
 #define AM_INIT(AM) atom_init(&am.AM, #AM)
 
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 64950fc252..773b138d92 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -259,7 +259,7 @@ static ERTS_INLINE void lc_free(void *p)
 {
     erts_lc_free_block_t *fb = (erts_lc_free_block_t *) p;
 #ifdef DEBUG
-    memset((void *) p, 0xdf, sizeof(erts_lc_free_block_t));
+    sys_memset((void *) p, 0xdf, sizeof(erts_lc_free_block_t));
 #endif
     lc_lock();
     fb->next = free_blocks;
@@ -289,12 +289,12 @@ static void *lc_core_alloc(void)
     }
     for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) {
 #ifdef DEBUG
-	memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t));
+	sys_memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t));
 #endif
 	fbs[i].next = &fbs[i+1];
     }
 #ifdef DEBUG
-    memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1],
+    sys_memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1],
 	   0xdf, sizeof(erts_lc_free_block_t));
 #endif
     lc_lock();
@@ -695,7 +695,7 @@ erts_lc_get_lock_order_id(char *name)
 	erts_fprintf(stderr, "Missing lock name\n");
     else {
 	for (i = 0; i < ERTS_LOCK_ORDER_SIZE; i++)
-	    if (strcmp(erts_lock_order[i].name, name) == 0)
+	    if (sys_strcmp(erts_lock_order[i].name, name) == 0)
 		return i;
 	erts_fprintf(stderr,
 		     "Lock name '%s' missing in lock order "
@@ -1322,12 +1322,12 @@ erts_lc_init(void)
     static erts_lc_free_block_t fbs[ERTS_LC_FB_CHUNK_SIZE];
     for (i = 0; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) {
 #ifdef DEBUG
-	memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t));
+	sys_memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t));
 #endif
 	fbs[i].next = &fbs[i+1];
     }
 #ifdef DEBUG
-    memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1],
+    sys_memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1],
 	   0xdf, sizeof(erts_lc_free_block_t));
 #endif
     fbs[ERTS_LC_FB_CHUNK_SIZE-1].next = NULL;
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index d659842b7e..d13d6080e1 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -81,7 +81,7 @@ void erts_msacc_init(void) {
                                         sizeof(Eterm)*ERTS_MSACC_STATE_COUNT);
     for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) {
         erts_msacc_state_atoms[i] = am_atom_put(erts_msacc_states[i],
-                                                strlen(erts_msacc_states[i]));
+                                                sys_strlen(erts_msacc_states[i]));
     }
 }
 
@@ -191,7 +191,7 @@ Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, ErtsHeapFactory *factory) {
     map->keys = key;
     hp[0] = state_map;
     hp[1] = msacc->id;
-    hp[2] = am_atom_put(msacc->type,strlen(msacc->type));
+    hp[2] = am_atom_put(msacc->type,sys_strlen(msacc->type));
 
     return make_flatmap(map);
 }
diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c
index f2a660f085..2807b443a1 100644
--- a/erts/emulator/beam/erl_mtrace.c
+++ b/erts/emulator/beam/erl_mtrace.c
@@ -314,7 +314,7 @@ disable_trace(int error, char *reason, int eno)
 	erts_fprintf(stderr, "%s: %s\n", mt_dis, reason);
     else {
 	eno_str = erl_errno_id(eno);
-	if (strcmp(eno_str, "unknown") == 0)
+	if (sys_strcmp(eno_str, "unknown") == 0)
 	    erts_fprintf(stderr, "%s: %s: %d\n", mt_dis, reason, eno);
 	else
 	    erts_fprintf(stderr, "%s: %s: %s\n", mt_dis, reason, eno_str);
@@ -401,9 +401,9 @@ write_trace_header(char *nodename, char *pid, char *hostname)
     PUT_UI32(tracep, ERTS_MT_MAJOR_VSN);
     PUT_UI32(tracep, ERTS_MT_MINOR_VSN);
 
-    n_len = strlen(nodename);
-    h_len = strlen(hostname);
-    p_len = strlen(pid);
+    n_len = sys_strlen(nodename);
+    h_len = sys_strlen(hostname);
+    p_len = sys_strlen(pid);
     hdr_prolog_len = (2*UI32_SZ
 		      + 3*UI16_SZ
 		      + 3*UI32_SZ
@@ -436,15 +436,15 @@ write_trace_header(char *nodename, char *pid, char *hostname)
     PUT_UI32(tracep, start_time.usec);
 
     PUT_UI8(tracep, (byte) n_len);
-    memcpy((void *) tracep, (void *) nodename, n_len);
+    sys_memcpy((void *) tracep, (void *) nodename, n_len);
     tracep += n_len;
 
     PUT_UI8(tracep, (byte) h_len);
-    memcpy((void *) tracep, (void *) hostname, h_len);
+    sys_memcpy((void *) tracep, (void *) hostname, h_len);
     tracep += h_len;
 
     PUT_UI8(tracep, (byte) p_len);
-    memcpy((void *) tracep, (void *) pid, p_len);
+    sys_memcpy((void *) tracep, (void *) pid, p_len);
     tracep += p_len;
 
     ASSERT(startp + hdr_prolog_len == tracep);
@@ -472,7 +472,7 @@ write_trace_header(char *nodename, char *pid, char *hostname)
 
 	str = ERTS_ALC_A2AD(i);
 	ASSERT(str);
-	str_len = strlen(str);
+	str_len = sys_strlen(str);
 	if (str_len >= (1 << 8)) {
 	    disable_trace(1, "Excessively large allocator string", 0);
 	    return 0;
@@ -493,7 +493,7 @@ write_trace_header(char *nodename, char *pid, char *hostname)
 	PUT_UI16(tracep, aflags);
 	PUT_UI16(tracep, (Uint16) i);
 	PUT_UI8( tracep, (byte) str_len);
-	memcpy((void *) tracep, (void *) str, str_len);
+	sys_memcpy((void *) tracep, (void *) str, str_len);
 	tracep += str_len;
 	if (erts_allctrs_info[i].alloc_util) {
 	    PUT_UI8(tracep, 2);
@@ -519,7 +519,7 @@ write_trace_header(char *nodename, char *pid, char *hostname)
 	str = ERTS_ALC_N2TD(i);
 	ASSERT(str);
 
-	str_len = strlen(str);
+	str_len = sys_strlen(str);
 	if (str_len >= (1 << 8)) {
 	    disable_trace(1, "Excessively large type string", 0);
 	    return 0;
@@ -543,7 +543,7 @@ write_trace_header(char *nodename, char *pid, char *hostname)
 	PUT_UI16(tracep, nflags);
 	PUT_UI16(tracep, (Uint16) i);
 	PUT_UI8(tracep, (byte) str_len);
-	memcpy((void *) tracep, (void *) str, str_len);
+	sys_memcpy((void *) tracep, (void *) str, str_len);
 	tracep += str_len;
 	PUT_UI16(tracep, no);
 	ASSERT(startp + entry_sz == tracep);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index d2000aa71e..c60cc7fecf 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -529,7 +529,7 @@ setup_nif_env(struct enif_msg_environment_t* msg_env,
     msg_env->env.tmp_obj_list = NULL;
     msg_env->env.proc = &msg_env->phony_proc;
     msg_env->env.exception_thrown = 0;
-    memset(&msg_env->phony_proc, 0, sizeof(Process));
+    sys_memset(&msg_env->phony_proc, 0, sizeof(Process));
     HEAP_START(&msg_env->phony_proc) = phony_heap;
     HEAP_TOP(&msg_env->phony_proc) = phony_heap;
     HEAP_LIMIT(&msg_env->phony_proc) = phony_heap;
@@ -4369,7 +4369,7 @@ static void get_string_maybe(ErlNifEnv *env, const ERL_NIF_TERM term,
         str_bin.size > bufsiz) {
         *ptr = NULL;
     } else {
-        memcpy(buf, (char *) str_bin.data, str_bin.size);
+        sys_memcpy(buf, (char *) str_bin.data, str_bin.size);
         buf[str_bin.size] = '\0';
         *ptr = buf;
     }
@@ -4386,7 +4386,7 @@ ERL_NIF_TERM erl_nif_user_trace_s1(ErlNifEnv* env, int argc,
 	    message_bin.size > MESSAGE_BUFSIZ) {
 	    return am_badarg;
 	}
-	memcpy(messagebuf, (char *) message_bin.data, message_bin.size);
+	sys_memcpy(messagebuf, (char *) message_bin.data, message_bin.size);
         messagebuf[message_bin.size] = '\0';
 	DTRACE1(user_trace_s1, messagebuf);
 	return am_true;
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index d2495479ab..018894f685 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -2533,7 +2533,7 @@ load_tracer_nif(const ErtsTracer tracer)
 
     for(i = 0; i < num_of_funcs; i++) {
         for (j = 0; j < NIF_TRACER_TYPES; j++) {
-            if (strcmp(tracers[j].name, funcs[i].name) == 0 && tracers[j].arity == funcs[i].arity) {
+            if (sys_strcmp(tracers[j].name, funcs[i].name) == 0 && tracers[j].arity == funcs[i].arity) {
                 tracers[j].cb = &(funcs[i]);
                 break;
             }
@@ -3019,7 +3019,7 @@ static void *tracer_alloc_fun(void* tmpl)
     ErtsTracerNif *obj = erts_alloc(ERTS_ALC_T_TRACER_NIF,
                                     sizeof(ErtsTracerNif) +
                                     sizeof(ErtsThrPrgrLaterOp));
-    memcpy(obj, tmpl, sizeof(*obj));
+    sys_memcpy(obj, tmpl, sizeof(*obj));
     return obj;
 }
 
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 604f0be127..8673e029e6 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -144,7 +144,7 @@ static Eterm make_magic_bin_for_restart(Process *p, RestartContext *rc)
 					   cleanup_restart_context_bin);
     RestartContext *restartp = ERTS_MAGIC_BIN_DATA(mbp);
     Eterm *hp;
-    memcpy(restartp,rc,sizeof(RestartContext));
+    sys_memcpy(restartp,rc,sizeof(RestartContext));
     hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
     return erts_mk_magic_ref(&hp, &MSO(p), mbp);
 }
@@ -254,12 +254,12 @@ static Uint copy_utf8_bin(byte *target, byte *source, Uint size,
 	ASSERT(need > 0);
 	ASSERT(from_source > 0);
 	if (size < from_source) {
-	    memcpy(leftover + (*num_leftovers), source, size);
+	    sys_memcpy(leftover + (*num_leftovers), source, size);
 	    *num_leftovers += size;
 	    return 0;
 	}
 	/* leftover has room for four bytes (see bif) */
-	memcpy(leftover + (*num_leftovers),source,from_source);
+	sys_memcpy(leftover + (*num_leftovers),source,from_source);
 	c = copy_utf8_bin(target, leftover, need, NULL, NULL, &tmp_err_pos, characters);
 	if (tmp_err_pos != 0) {
 	    *err_pos = source;
@@ -291,7 +291,7 @@ static Uint copy_utf8_bin(byte *target, byte *source, Uint size,
 	    size -= 2; copied += 2;
 	} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
 	    if (leftover && size < 3) {
-		memcpy(leftover, source, (int) size);
+		sys_memcpy(leftover, source, (int) size);
 		*num_leftovers = (int) size;
 		break;
 	    }
@@ -313,7 +313,7 @@ static Uint copy_utf8_bin(byte *target, byte *source, Uint size,
 	    size -= 3; copied += 3;
 	} else if (((*source) & ((byte) 0xF8)) == 0xF0) {
 	    if (leftover && size < 4) {
-		memcpy(leftover, source, (int) size);
+		sys_memcpy(leftover, source, (int) size);
 		*num_leftovers = (int) size;
 		break;
 	    }
@@ -2108,7 +2108,7 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu
 	    } else {
 		name_buf = statbuf;
 	    }
-	    memcpy(name_buf,bytes,size);
+	    sys_memcpy(name_buf,bytes,size);
 	    name_buf[size]=0;
 	} else {
             name_buf = erts_convert_filename_to_wchar(bytes, size,
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 44d8c85867..e4087e0ac8 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -91,7 +91,7 @@ Eterm erts_bld_tuple(Uint **hpp, Uint *szp, Uint arity, ...);
 #define erts_bld_tuple5(H,S,E1,E2,E3,E4,E5) erts_bld_tuple(H,S,5,E1,E2,E3,E4,E5)
 Eterm erts_bld_tuplev(Uint **hpp, Uint *szp, Uint arity, Eterm terms[]);
 Eterm erts_bld_string_n(Uint **hpp, Uint *szp, const char *str, Sint len);
-#define erts_bld_string(hpp,szp,str) erts_bld_string_n(hpp,szp,str,strlen(str))
+#define erts_bld_string(hpp,szp,str) erts_bld_string_n(hpp,szp,str,sys_strlen(str))
 Eterm erts_bld_list(Uint **hpp, Uint *szp, Sint length, Eterm terms[]);
 Eterm erts_bld_2tup_list(Uint **hpp, Uint *szp,
 			 Sint length, Eterm terms1[], Uint terms2[]);
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 76980b5871..f8391fb665 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -69,7 +69,7 @@
 #  ifdef CHECK_FOR_HOLES
 #    define INIT_HEAP_MEM(p,sz) erts_set_hole_marker(HEAP_TOP(p), (sz))
 #  else
-#    define INIT_HEAP_MEM(p,sz) memset(HEAP_TOP(p),0x01,(sz)*sizeof(Eterm*))
+#    define INIT_HEAP_MEM(p,sz) sys_memset(HEAP_TOP(p),0x01,(sz)*sizeof(Eterm*))
 #  endif
 #else
 #  define INIT_HEAP_MEM(p,sz) ((void)0)
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index b12a021e41..a5d752a3d0 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1953,7 +1953,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 	    context_b = erts_create_magic_binary(sizeof(TTBContext),    \
                                                  ttb_context_destructor);   \
 	    context =  ERTS_MAGIC_BIN_DATA(context_b);			\
-	    memcpy(context,&c_buff,sizeof(TTBContext));			\
+	    sys_memcpy(context,&c_buff,sizeof(TTBContext));			\
 	}								\
     } while (0)
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index d757651933..0f23027752 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -368,7 +368,7 @@ do {\
 	UWord _wsz = ESTACK_COUNT(s);\
 	(dst)->start = erts_alloc((s).alloc_type,\
 				  DEF_ESTACK_SIZE * sizeof(Eterm));\
-	memcpy((dst)->start, (s).start,_wsz*sizeof(Eterm));\
+	sys_memcpy((dst)->start, (s).start,_wsz*sizeof(Eterm));\
 	(dst)->sp = (dst)->start + _wsz;\
 	(dst)->end = (dst)->start + DEF_ESTACK_SIZE;\
         (dst)->edefault = NULL;\
@@ -536,7 +536,7 @@ do {\
 	UWord _wsz = WSTACK_COUNT(s);\
 	(dst)->wstart = erts_alloc(s.alloc_type,\
 				  DEF_WSTACK_SIZE * sizeof(UWord));\
-	memcpy((dst)->wstart, s.wstart,_wsz*sizeof(UWord));\
+	sys_memcpy((dst)->wstart, s.wstart,_wsz*sizeof(UWord));\
 	(dst)->wsp = (dst)->wstart + _wsz;\
 	(dst)->wend = (dst)->wstart + DEF_WSTACK_SIZE;\
         (dst)->wdefault = NULL;\
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 4bb2c0cb01..1ae914c5bf 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -584,7 +584,7 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 	     */
 	    
 	    for (d = driver_list; d; d = d->next) {
-		if (strcmp(d->name, name) == 0 && 
+		if (sys_strcmp(d->name, name) == 0 && 
 		    erts_ddll_driver_ok(d->handle)) {
 		    driver = d;
 		    break;
@@ -634,7 +634,7 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 	trace_port_open(port,
 			pid,
 			erts_atom_put((byte *) port->name,
-				      strlen(port->name),
+				      sys_strlen(port->name),
 				      ERTS_ATOM_ENC_LATIN1,
 				      1));
     }
@@ -2925,7 +2925,7 @@ erl_drv_init_ack(ErlDrvPort ix, ErlDrvData res) {
             break;
         case -2: {
             char *str = erl_errno_id(errno);
-            resp = erts_atom_put((byte *) str, strlen(str),
+            resp = erts_atom_put((byte *) str, sys_strlen(str),
                                  ERTS_ATOM_ENC_LATIN1, 1);
             break;
         }
@@ -7320,7 +7320,7 @@ int driver_failure_atom(ErlDrvPort ix, char* string)
 {
     return driver_failure_term(ix,
 			       erts_atom_put((byte *) string,
-					     strlen(string),
+					     sys_strlen(string),
 					     ERTS_ATOM_ENC_LATIN1,
 					     1),
 			       0);
diff --git a/erts/emulator/beam/lttng-wrapper.h b/erts/emulator/beam/lttng-wrapper.h
index 0bc75c1552..e7f2971bf7 100644
--- a/erts/emulator/beam/lttng-wrapper.h
+++ b/erts/emulator/beam/lttng-wrapper.h
@@ -61,13 +61,13 @@
 #define lttng_proc_to_mfa_str(p, Name)                              \
     do {                                                            \
         if (ERTS_PROC_IS_EXITING((p))) {                            \
-            strcpy(Name, "<exiting>");                              \
+            sys_strcpy(Name, "<exiting>");                              \
         } else {                                                    \
             BeamInstr *_fptr = find_function_from_pc((p)->i);       \
             if (_fptr) {                                            \
                 lttng_mfa_to_str(_fptr[0],_fptr[1],_fptr[2], Name); \
             } else {                                                \
-                strcpy(Name, "<unknown>");                          \
+                sys_strcpy(Name, "<unknown>");                          \
             }                                                       \
         }                                                           \
     } while(0)
diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c
index f14910bc72..de1d481105 100644
--- a/erts/emulator/beam/packet_parser.c
+++ b/erts/emulator/beam/packet_parser.c
@@ -200,7 +200,7 @@ static int http_init(void)
     for (i = 0; i < HTTP_HDR_HASH_SIZE; i++)
         http_hdr_hash[i] = NULL;
     for (i = 0; http_hdr_strings[i] != NULL; i++) {
-        ASSERT(strlen(http_hdr_strings[i]) <= HTTP_MAX_NAME_LEN);
+        ASSERT(sys_strlen(http_hdr_strings[i]) <= HTTP_MAX_NAME_LEN);
         http_hdr_table[i].index = i;
         http_hash_insert(http_hdr_strings[i], 
                          &http_hdr_table[i], 
@@ -516,7 +516,7 @@ static http_atom_t* http_hash_lookup(const char* name, int len,
 
     while (ap != NULL) {
         if ((ap->h == h) && (ap->len == len) && 
-            (strncmp(ap->name, name, len) == 0))
+            (sys_strncmp(ap->name, name, len) == 0))
             return ap;
         ap = ap->next;
     }
@@ -656,7 +656,7 @@ int packet_parse_http(const char* buf, int len, int* statep,
     if (*statep == 0) {
         /* start-line = Request-Line | Status-Line */
 
-        if (n >= 5 && (strncmp(buf, "HTTP/", 5) == 0)) {
+        if (n >= 5 && (sys_strncmp(buf, "HTTP/", 5) == 0)) {
             int major  = 0;
             int minor  = 0;
             int status = 0;
@@ -750,7 +750,7 @@ int packet_parse_http(const char* buf, int len, int* statep,
             }
             if (n < 8)
                 return -1;
-            if (strncmp(ptr, "HTTP/", 5) != 0)
+            if (sys_strncmp(ptr, "HTTP/", 5) != 0)
                 return -1;
             ptr += 5;
             n   -= 5;
-- 
cgit v1.2.3


From 9f1d9a497ba42df8c122bc1ac2c1add6f64a164c Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikpelinux@gmail.com>
Date: Sat, 10 Mar 2018 16:19:26 +0100
Subject: make erlang:process_info/1 not retrieve messages

process_info/1 retrieves a number of properties related to a process,
including the list of messages in its mailbox.  This is potentially
unsafe if the target process has a large number of queued messages:

- there is no a priori upper bound on the amount of memory being
  allocated to hold that list, and
- the loop to retrieve the messages is uninterruptible, so the
  Erlang scheduler where this executes blocks for the duration

We've seen process_info/1 bring down heavily loaded nodes on more
than one occasion.  At least once it appeared to have blocked the
Erlang heart process from executing, causing the external heart to
kill the VM.

Consequently this removes 'messages' from the list of process_info
tags to retrieve for process_info/1.  Note that process_info/1 still
retrieves 'message_queue_len', and process_info/2 can still retrieve
'messages' when asked to.

A few places in the OTP libraries need minor adjustments, since they
want 'message_queue_len' but compute it from the length of the list
of messages.
---
 erts/emulator/beam/erl_bif_info.c | 1 -
 1 file changed, 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 4050fb6146..c5f498c3b1 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -717,7 +717,6 @@ static Eterm pi_1_keys[] = {
     am_initial_call,
     am_status,
     am_message_queue_len,
-    am_messages,
     am_links,
     am_dictionary,
     am_trap_exit,
-- 
cgit v1.2.3


From 620c4456a3b9d58fa2551cb13f1fe20dc55f4513 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 13 Mar 2018 17:21:20 +0100
Subject: erts: Remove unused atoms

---
 erts/emulator/beam/atom.names | 39 ++-------------------------------------
 1 file changed, 2 insertions(+), 37 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index b6ec3e7ed2..2a00fedb65 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -48,7 +48,7 @@ atom Empty=''
 #
 # Used in the Beam emulator loop. (Smaller literals usually means tighter code.)
 #
-atom fun infinity timeout normal call return
+atom infinity timeout normal call return
 atom throw error exit
 atom undefined
 
@@ -69,11 +69,9 @@ atom DOWN='DOWN'
 atom UP='UP'
 atom EXIT='EXIT'
 atom abort
-atom aborted
 atom abs_path
 atom absoluteURI
 atom ac
-atom accessor
 atom active
 atom active_tasks
 atom active_tasks_all
@@ -88,7 +86,6 @@ atom allocated_areas
 atom allocator
 atom allocator_sizes
 atom alloc_util_allocators
-atom allow_gc
 atom allow_passive_connect
 atom already_loaded
 atom amd64
@@ -122,7 +119,6 @@ atom bag
 atom band
 atom big
 atom bif_return_trap
-atom bif_timer_server
 atom binary
 atom binary_bin_to_list_trap
 atom binary_copy_trap
@@ -179,7 +175,6 @@ atom convert_time_unit
 atom connect
 atom connected
 atom connection_closed
-atom cons
 atom const
 atom context_switches
 atom control
@@ -199,7 +194,6 @@ atom debug_flags
 atom decimals
 atom default
 atom delay_trap
-atom depth
 atom dictionary
 atom dirty_bif_exception
 atom dirty_bif_result
@@ -246,7 +240,6 @@ atom Eqeq='=='
 atom erl_tracer
 atom erlang
 atom erl_signal_server
-atom ERROR='ERROR'
 atom error_handler
 atom error_logger
 atom erts_code_purger
@@ -254,7 +247,6 @@ atom erts_debug
 atom erts_internal
 atom ets
 atom ETS_TRANSFER='ETS-TRANSFER'
-atom event
 atom exact_reductions
 atom exception_from
 atom exception_trace
@@ -281,27 +273,20 @@ atom force
 atom format_cpu_topology
 atom free
 atom fullsweep_after
-atom fullsweep_if_old_binaries
-atom fun
-atom function
 atom functions
 atom function_clause
 atom garbage_collecting
 atom garbage_collection
 atom garbage_collection_info
-atom gc_end
 atom gc_major_end
 atom gc_major_start
 atom gc_max_heap_size
 atom gc_minor_end
 atom gc_minor_start
-atom gc_start
 atom Ge='>='
 atom generational
-atom get_data
 atom get_seq_token
 atom get_tcw
-atom getenv
 atom gather_gc_info_result
 atom gather_io_bytes
 atom gather_microstate_accounting_result
@@ -343,7 +328,6 @@ atom initial_call
 atom input
 atom internal
 atom internal_error
-atom internal_status
 atom instruction_counts
 atom invalid
 atom is_constant
@@ -389,14 +373,12 @@ atom match_spec_result
 atom max
 atom maximum
 atom max_heap_size
-atom max_tables max_processes
 atom mbuf_size
 atom md5
 atom memory
 atom memory_internal
 atom memory_types
 atom message
-atom message_binary
 atom message_queue_data
 atom message_queue_len
 atom messages
@@ -443,21 +425,18 @@ atom new_processes
 atom new_ports
 atom new_uniq
 atom newline
-atom next
 atom no
 atom nomatch
 atom none
 atom no_auto_capture
 atom noconnect
 atom noconnection
-atom nocookie
 atom node
 atom node_type
 atom nodedown
 atom nodedown_reason
 atom nodeup
 atom noeol
-atom nofile
 atom noproc
 atom normal
 atom nosuspend
@@ -479,7 +458,6 @@ atom notempty_atstart
 atom notify
 atom notsup
 atom nouse_stdio
-atom objects
 atom off_heap
 atom offset
 atom ok
@@ -549,7 +527,6 @@ atom re_run_trap
 atom read_concurrency
 atom ready_input
 atom ready_output
-atom ready_async
 atom reason
 atom receive
 atom recent_size
@@ -596,7 +573,6 @@ atom sequential_trace_token
 atom serial
 atom set
 atom set_cpu_topology
-atom set_data
 atom set_on_first_link
 atom set_on_first_spawn
 atom set_on_link
@@ -604,8 +580,6 @@ atom set_on_spawn
 atom set_seq_token
 atom set_tcw
 atom set_tcw_fake
-atom separate
-atom shared
 atom sighup
 atom sigterm
 atom sigusr1
@@ -621,7 +595,6 @@ atom sigtstp
 atom sigquit
 atom silent
 atom size
-atom sl_alloc
 atom spawn_executable
 atom spawn_driver
 atom spawned
@@ -629,7 +602,6 @@ atom ssl_tls
 atom stack_size
 atom start
 atom status
-atom static
 atom stderr_to_stdout
 atom stop
 atom stream
@@ -639,14 +611,11 @@ atom sunrm
 atom suspend
 atom suspended
 atom suspending
-atom sys_misc
 atom system
-atom system_error
 atom system_flag_scheduler_wall_time
 atom system_limit
 atom system_version
 atom system_architecture
-atom SYSTEM='SYSTEM'
 atom table
 atom term_to_binary_trap
 atom this
@@ -664,7 +633,7 @@ atom total_heap_size
 atom total_run_queue_lengths
 atom total_run_queue_lengths_all
 atom tpkt
-atom trace trace_ts traced
+atom trace traced
 atom trace_control_word
 atom trace_status
 atom tracer
@@ -673,7 +642,6 @@ atom trim
 atom trim_all
 atom try_clause
 atom true
-atom tuple
 atom type
 atom ucompile
 atom ucp
@@ -690,14 +658,11 @@ atom unblock_normal
 atom uniq
 atom unless_suspending
 atom unloaded
-atom unloading
 atom unloaded_only
 atom unload_cancelled
 atom value
-atom values
 atom version
 atom visible
-atom wait
 atom waiting
 atom wall_clock
 atom warning
-- 
cgit v1.2.3


From 4cf4044313ae5a1a349fcedd3d2472c3b6ed3fe7 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 8 Mar 2018 11:29:14 +0100
Subject: Force 64-bit alignment for pre-allocators unless x86

---
 erts/emulator/beam/erl_sched_spec_pre_alloc.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
index ab204303d7..4a6e02281a 100644
--- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
@@ -47,6 +47,15 @@ erts_sspa_create(size_t blk_sz, int pa_size, int nthreads, const char* name)
     int cix;
     int no_blocks = pa_size;
     int no_blocks_per_chunk;
+    size_t aligned_blk_sz;
+
+#if !defined(ERTS_STRUCTURE_ALIGNED_ALLOC)
+    /* Force 64-bit alignment... */
+    aligned_blk_sz = ((blk_sz - 1) / 8) * 8 + 8;
+#else
+    /* Alignment of structure is enough... */
+    aligned_blk_sz = blk_sz;
+#endif
 
     if (!name) { /* schedulers only variant */
         ASSERT(!nthreads);
@@ -68,7 +77,7 @@ erts_sspa_create(size_t blk_sz, int pa_size, int nthreads, const char* name)
     }
     no_blocks = no_blocks_per_chunk * nthreads;
     chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_chunk_header_t));
-    chunk_mem_size += blk_sz * no_blocks_per_chunk;
+    chunk_mem_size += aligned_blk_sz * no_blocks_per_chunk;
     chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size);
     tot_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t));
     tot_size += chunk_mem_size * nthreads;
@@ -115,7 +124,7 @@ erts_sspa_create(size_t blk_sz, int pa_size, int nthreads, const char* name)
 	blk = (erts_sspa_blk_t *) p;
 	for (i = 0; i < no_blocks_per_chunk; i++) {
 	    blk = (erts_sspa_blk_t *) p;
-	    p += blk_sz;
+	    p += aligned_blk_sz;
 	    blk->next_ptr = (erts_sspa_blk_t *) p;
 	}
 
-- 
cgit v1.2.3


From d0573e3e497337e13f4685de4b455817dfb601b7 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 12 Mar 2018 15:41:16 +0100
Subject: Reschedule on ordinary scheduler if dirty work is gone

---
 erts/emulator/beam/erl_process.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 63e9275ac1..bcf68e80ba 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10816,6 +10816,10 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    p->scheduler_data = esdp;
 	}
 	else {
+            if (!(state & ERTS_PSFLGS_DIRTY_WORK)) {
+                /* Dirty work completed... */
+		goto sunlock_sched_out_proc;
+            }
 	    if (state & (ERTS_PSFLG_ACTIVE_SYS
 			 | ERTS_PSFLG_PENDING_EXIT
 			 | ERTS_PSFLG_EXITING)) {
@@ -10966,6 +10970,23 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    ERTS_PTMR_CLEAR(p);
 	}
 
+#ifdef DEBUG
+        if (is_normal_sched) {
+	    if (state & ERTS_PSFLGS_DIRTY_WORK)
+                ERTS_INTERNAL_ERROR("Executing dirty code on normal scheduler");
+        }
+        else {
+	    if (!(state & ERTS_PSFLGS_DIRTY_WORK)) {
+                if (esdp->type == ERTS_SCHED_DIRTY_CPU)
+                    ERTS_INTERNAL_ERROR("Executing normal code on dirty CPU scheduler");
+                else if (esdp->type == ERTS_SCHED_DIRTY_IO)
+                    ERTS_INTERNAL_ERROR("Executing normal code on dirty IO scheduler");
+                else
+                    ERTS_INTERNAL_ERROR("Executing normal code on dirty UNKNOWN scheduler");
+            }
+        }
+#endif
+
 	return p;
     }
 }
-- 
cgit v1.2.3


From eaf20e4f60ba537733752637c7952eb52e225473 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 14 Mar 2018 17:42:46 +0100
Subject: erts: Fix faulty sys_memcpy of 0 bytes

---
 erts/emulator/beam/beam_load.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 50498cb6cf..0184c567f1 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4915,7 +4915,9 @@ freeze_code(LoaderState* stp)
 	line_items[i] = codev + stp->ci - 1;
 
 	line_tab->fname_ptr = (Eterm*) &line_items[i + 1];
-	sys_memcpy(line_tab->fname_ptr, stp->fname, stp->num_fnames*sizeof(Eterm));
+        if (stp->num_fnames)
+            sys_memcpy(line_tab->fname_ptr, stp->fname,
+                       stp->num_fnames*sizeof(Eterm));
 
 	line_tab->loc_size = stp->loc_size;
 	if (stp->loc_size == 2) {
-- 
cgit v1.2.3


From f14a5306622994a6b49b25c63ec882a1551398ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 19 Mar 2018 13:18:08 +0100
Subject: Fix file:change_group/change_owner

It wasn't possible to change group/owner separately, and our test
suite lacked coverage for that.

ERL-589
---
 erts/emulator/nifs/common/prim_file_nif.c | 6 +++---
 erts/emulator/nifs/common/prim_file_nif.h | 2 +-
 erts/emulator/nifs/unix/unix_prim_file.c  | 2 +-
 erts/emulator/nifs/win32/win_prim_file.c  | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/nifs/common/prim_file_nif.c b/erts/emulator/nifs/common/prim_file_nif.c
index 6874f41d75..bbd9becb47 100644
--- a/erts/emulator/nifs/common/prim_file_nif.c
+++ b/erts/emulator/nifs/common/prim_file_nif.c
@@ -891,10 +891,10 @@ static ERL_NIF_TERM set_owner_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM a
     posix_errno_t posix_errno;
 
     efile_path_t path;
-    Uint32 uid, gid;
+    Sint32 uid, gid;
 
-    if(argc != 3 || !enif_get_uint(env, argv[1], &uid)
-                 || !enif_get_uint(env, argv[2], &gid)) {
+    if(argc != 3 || !enif_get_int(env, argv[1], &uid)
+                 || !enif_get_int(env, argv[2], &gid)) {
         return enif_make_badarg(env);
     }
 
diff --git a/erts/emulator/nifs/common/prim_file_nif.h b/erts/emulator/nifs/common/prim_file_nif.h
index cc9bc8f5c3..4194cdc7d9 100644
--- a/erts/emulator/nifs/common/prim_file_nif.h
+++ b/erts/emulator/nifs/common/prim_file_nif.h
@@ -177,7 +177,7 @@ posix_errno_t efile_set_permissions(const efile_path_t *path, Uint32 permissions
 
 /** @brief On Unix, this will set the owner/group to the given values. It will
  * do nothing on other platforms. */
-posix_errno_t efile_set_owner(const efile_path_t *path, Uint32 owner, Uint32 group);
+posix_errno_t efile_set_owner(const efile_path_t *path, Sint32 owner, Sint32 group);
 
 /** @brief Resolves the final path of the given link. */
 posix_errno_t efile_read_link(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result);
diff --git a/erts/emulator/nifs/unix/unix_prim_file.c b/erts/emulator/nifs/unix/unix_prim_file.c
index 4a6c476882..1637f9cb71 100644
--- a/erts/emulator/nifs/unix/unix_prim_file.c
+++ b/erts/emulator/nifs/unix/unix_prim_file.c
@@ -687,7 +687,7 @@ posix_errno_t efile_set_permissions(const efile_path_t *path, Uint32 permissions
     return 0;
 }
 
-posix_errno_t efile_set_owner(const efile_path_t *path, Uint32 owner, Uint32 group) {
+posix_errno_t efile_set_owner(const efile_path_t *path, Sint32 owner, Sint32 group) {
     if(chown((const char*)path->data, owner, group) < 0) {
         return errno;
     }
diff --git a/erts/emulator/nifs/win32/win_prim_file.c b/erts/emulator/nifs/win32/win_prim_file.c
index 9b79182f2c..8058350b25 100644
--- a/erts/emulator/nifs/win32/win_prim_file.c
+++ b/erts/emulator/nifs/win32/win_prim_file.c
@@ -801,7 +801,7 @@ posix_errno_t efile_set_permissions(const efile_path_t *path, Uint32 permissions
     return windows_to_posix_errno(GetLastError());
 }
 
-posix_errno_t efile_set_owner(const efile_path_t *path, Uint32 owner, Uint32 group) {
+posix_errno_t efile_set_owner(const efile_path_t *path, Sint32 owner, Sint32 group) {
     (void)path;
     (void)owner;
     (void)group;
-- 
cgit v1.2.3


From 2371741165a0f6c39893fe9f586d7586fd7d02a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 16 Mar 2018 16:13:08 +0100
Subject: Correctly handle get_map_elements with a literal map

A get_map_elements instruction that has a literal map operand
would never be translated to a i_get_map_element instruction.
That would be a problem for the following instruction:

    get_map_elements Fail #{} {x,0}, {x,1}

Since the key is not a literal, get_map_element must be used,
since get_map_elements requires that a hash value can be
calculated for each element.

When the instruction is translated to i_get_map_element, the
hash value will be set to 0 and an assertion will trigger in
the debug build.
---
 erts/emulator/beam/ops.tab | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 77e375f2c0..bc765a8c94 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1413,7 +1413,7 @@ has_map_fields Fail Src Size Rest=* => \
 
 ## Transform get_map_elements(s) #{ K1 := V1, K2 := V2 }
 
-get_map_elements Fail Src=xy Size=u==2 Rest=* => \
+get_map_elements Fail Src Size=u==2 Rest=* => \
     gen_get_map_element(Fail, Src, Size, Rest)
 get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \
    gen_get_map_elements(Fail, Src, Size, Rest)
@@ -1423,8 +1423,12 @@ i_get_map_elements f? s I
 i_get_map_element Fail Src=xy Key=y Dst => \
     move Key x | i_get_map_element Fail Src x Dst
 
+i_get_map_element_hash Fail Src=c Key Hash Dst => \
+    move Src x | i_get_map_element_hash Fail x Key Hash Dst
 i_get_map_element_hash f? xy c I xy
 
+i_get_map_element Fail Src=c Key Dst => \
+    move Src x | i_get_map_element Fail x Key Dst
 i_get_map_element f? xy x xy
 
 #
-- 
cgit v1.2.3


From 836b90884f9abff3a82b2cee8bada16b23f7ddf7 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 15 Mar 2018 17:21:04 +0100
Subject: erts: Make sys_memcpy and friends inline functions

to avoid argument-evaluated-twice bugs
like in macro DMC_PUSH.

Had to shuffle around some #include and #define
to make erl_child_setup build debug target.
---
 erts/emulator/beam/hash.c                |  4 +-
 erts/emulator/beam/sys.h                 | 88 ++++++++++++++++++++++++--------
 erts/emulator/sys/unix/erl_child_setup.c |  2 +
 erts/emulator/sys/unix/sys_drivers.c     |  3 ++
 erts/emulator/sys/unix/sys_uds.c         | 36 +++++++++++++
 erts/emulator/sys/unix/sys_uds.h         | 14 -----
 6 files changed, 110 insertions(+), 37 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c
index 8548e30e8b..6a31489473 100644
--- a/erts/emulator/beam/hash.c
+++ b/erts/emulator/beam/hash.c
@@ -152,7 +152,7 @@ Hash* hash_init(int type, Hash* h, char* name, int size, HashFunctions fun)
 
     h->bucket = (HashBucket**) fun.meta_alloc(h->meta_alloc_type, sz);
 
-    sys_memzero(h->bucket, sz);
+    memzero(h->bucket, sz);
     h->is_allocated = 0;
     h->name = name;
     h->fun = fun;
@@ -224,7 +224,7 @@ static void rehash(Hash* h, int grow)
     sz = h->size*sizeof(HashBucket*);
 
     new_bucket = (HashBucket **) h->fun.meta_alloc(h->meta_alloc_type, sz);
-    sys_memzero(new_bucket, sz);
+    memzero(new_bucket, sz);
 
     for (i = 0; i < old_size; i++) {
 	HashBucket* b = h->bucket[i];
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 13ae80e4a5..152da8c9e1 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -994,7 +994,8 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
     return val;
 }
 
-#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+#endif  /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
 
 /* Thin wrappers around memcpy and friends, which should always be used in
  * place of plain memcpy, memset, etc.
@@ -1006,26 +1007,71 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
  *
  * (The weird casts in the assertions silence an "always evaluates to true"
  * warning when an operand is the address of an lvalue) */
-#define sys_memcpy(s1,s2,n) \
-    (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), memcpy(s1,s2,n))
-#define sys_memmove(s1,s2,n) \
-    (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), memmove(s1,s2,n))
-#define sys_memcmp(s1,s2,n) \
-    (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), memcmp(s1,s2,n))
-#define sys_memset(s,c,n) \
-    (ASSERT((void*)(s) != NULL), memset(s,c,n))
-#define sys_memzero(s, n) \
-    (ASSERT((void*)(s) != NULL), memset(s,'\0',n))
-#define sys_strcmp(s1,s2) \
-    (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strcmp(s1,s2))
-#define sys_strncmp(s1,s2,n) \
-    (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strncmp(s1,s2,n))
-#define sys_strcpy(s1,s2) \
-    (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strcpy(s1,s2))
-#define sys_strncpy(s1,s2,n) \
-    (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strncpy(s1,s2,n))
-#define sys_strlen(s) \
-    (ASSERT((void*)(s) != NULL), strlen(s))
+ERTS_GLB_INLINE void *sys_memcpy(void *dest, const void *src, size_t n);
+ERTS_GLB_INLINE void *sys_memmove(void *dest, const void *src, size_t n);
+ERTS_GLB_INLINE int sys_memcmp(const void *s1, const void *s2, size_t n);
+ERTS_GLB_INLINE void *sys_memset(void *s, int c, size_t n);
+ERTS_GLB_INLINE void *sys_memzero(void *s, size_t n);
+ERTS_GLB_INLINE int sys_strcmp(const char *s1, const char *s2);
+ERTS_GLB_INLINE int sys_strncmp(const char *s1, const char *s2, size_t n);
+ERTS_GLB_INLINE char *sys_strcpy(char *dest, const char *src);
+ERTS_GLB_INLINE char *sys_strncpy(char *dest, const char *src, size_t n);
+ERTS_GLB_INLINE size_t sys_strlen(const char *s);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE void *sys_memcpy(void *dest, const void *src, size_t n)
+{
+    ASSERT(dest != NULL && src != NULL);
+    return memcpy(dest,src,n);
+}
+ERTS_GLB_INLINE void *sys_memmove(void *dest, const void *src, size_t n)
+{
+    ASSERT(dest != NULL && src != NULL);
+    return memmove(dest,src,n);
+}
+ERTS_GLB_INLINE int sys_memcmp(const void *s1, const void *s2, size_t n)
+{
+    ASSERT(s1 != NULL && s2 != NULL);
+    return memcmp(s1,s2,n);
+}
+ERTS_GLB_INLINE void *sys_memset(void *s, int c, size_t n)
+{
+    ASSERT(s != NULL);
+    return memset(s,c,n);
+}
+ERTS_GLB_INLINE void *sys_memzero(void *s, size_t n)
+{
+    ASSERT(s != NULL);
+    return memset(s,'\0',n);
+}
+ERTS_GLB_INLINE int sys_strcmp(const char *s1, const char *s2)
+{
+    ASSERT(s1 != NULL && s2 != NULL);
+    return strcmp(s1,s2);
+}
+ERTS_GLB_INLINE int sys_strncmp(const char *s1, const char *s2, size_t n)
+{
+    ASSERT(s1 != NULL && s2 != NULL);
+    return strncmp(s1,s2,n);
+}
+ERTS_GLB_INLINE char *sys_strcpy(char *dest, const char *src)
+{
+    ASSERT(dest != NULL && src != NULL);
+    return strcpy(dest,src);
+
+}
+ERTS_GLB_INLINE char *sys_strncpy(char *dest, const char *src, size_t n)
+{
+    ASSERT(dest != NULL && src != NULL);
+    return strncpy(dest,src,n);
+}
+ERTS_GLB_INLINE size_t sys_strlen(const char *s)
+{
+    ASSERT(s != NULL);
+    return strlen(s);
+}
+#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 /* define function symbols (needed in sys_drv_api) */
 #define sys_fp_alloc     sys_alloc
diff --git a/erts/emulator/sys/unix/erl_child_setup.c b/erts/emulator/sys/unix/erl_child_setup.c
index 57973b10d7..10601529a4 100644
--- a/erts/emulator/sys/unix/erl_child_setup.c
+++ b/erts/emulator/sys/unix/erl_child_setup.c
@@ -56,6 +56,8 @@
 #include <stdio.h>
 #include <stdarg.h>
 #include <sys/wait.h>
+#include <sys/types.h>
+#include <sys/socket.h>
 
 #define WANT_NONBLOCKING
 
diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c
index b7ac89d89a..117855acf0 100644
--- a/erts/emulator/sys/unix/sys_drivers.c
+++ b/erts/emulator/sys/unix/sys_drivers.c
@@ -50,6 +50,9 @@
 #include <sys/ioctl.h>
 #endif
 
+#include <sys/types.h>
+#include <sys/socket.h>
+
 #define WANT_NONBLOCKING    /* must define this to pull in defs from sys.h */
 #include "sys.h"
 
diff --git a/erts/emulator/sys/unix/sys_uds.c b/erts/emulator/sys/unix/sys_uds.c
index dd0a3b03ff..278c6b6ba1 100644
--- a/erts/emulator/sys/unix/sys_uds.c
+++ b/erts/emulator/sys/unix/sys_uds.c
@@ -18,6 +18,42 @@
  * %CopyrightEnd%
  */
 
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#if defined(__sun__) && !defined(_XOPEN_SOURCE)
+#define _XOPEN_SOURCE 500
+#endif
+
+#include <limits.h>
+
+#include <sys/types.h>
+#include <sys/socket.h>
+
+#ifdef HAVE_SYS_SOCKETIO_H
+#   include <sys/socketio.h>
+#endif
+#ifdef HAVE_SYS_SOCKIO_H
+#   include <sys/sockio.h>
+#endif
+
+#ifdef HAVE_NET_ERRNO_H
+#include <net/errno.h>
+#endif
+
+#ifdef HAVE_DIRENT_H
+#  include <dirent.h>
+#endif
+
+#ifdef HAVE_UNISTD_H
+#   include <unistd.h>
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
 #include "sys_uds.h"
 
 int
diff --git a/erts/emulator/sys/unix/sys_uds.h b/erts/emulator/sys/unix/sys_uds.h
index a598102d5c..26c91d6a00 100644
--- a/erts/emulator/sys/unix/sys_uds.h
+++ b/erts/emulator/sys/unix/sys_uds.h
@@ -21,18 +21,6 @@
 #ifndef _ERL_UNIX_UDS_H
 #define _ERL_UNIX_UDS_H
 
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-
-#if defined(__sun__) && !defined(_XOPEN_SOURCE)
-#define _XOPEN_SOURCE 500
-#endif
-
-#include <limits.h>
-
-#include <sys/types.h>
-#include <sys/socket.h>
 #include <sys/uio.h>
 
 #if defined IOV_MAX
@@ -43,8 +31,6 @@
 #define MAXIOV 16
 #endif
 
-#include "sys.h"
-
 int sys_uds_readv(int fd, struct iovec *iov, size_t iov_len,
                   int *fds, int fd_count, int flags);
 int sys_uds_read(int fd, char *buff, size_t len,
-- 
cgit v1.2.3


From 2bdd0ba8f6046301778ea3673716d8df9fdd42fa Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 15 Mar 2018 20:10:39 +0100
Subject: erts: Optimize macro DMC_PUSH

to call a common static function dmc_stack_grow()
and reduce the code bloat.

and did a combined DMC_PUSH2
---
 erts/emulator/beam/erl_db_util.c | 102 ++++++++++++++++++++-------------------
 1 file changed, 53 insertions(+), 49 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 956662aefa..f21db4b65b 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -74,12 +74,35 @@ DBIF_TABLE_GUARD | DBIF_TABLE_BODY | DBIF_TRACE_GUARD | DBIF_TRACE_BODY
 typedef struct DMC_STACK_TYPE(Type) {		\
     int pos;					\
     int siz;					\
-    Type def[DMC_DEFAULT_SIZE];		        \
+    int bytes;                                  \
     Type *data;					\
+    Type def[DMC_DEFAULT_SIZE];		        \
 } DMC_STACK_TYPE(Type)
+
+
+typedef int Dummy;
+DMC_DECLARE_STACK_TYPE(Dummy);
+
+static void dmc_stack_grow(DMC_Dummy_stack* s)
+{
+    int was_bytes = s->bytes;
+    s->siz *= 2;
+    s->bytes *= 2;
+    if (s->data == s->def) {
+        s->data = erts_alloc(ERTS_ALC_T_DB_MC_STK, s->bytes);
+        sys_memcpy(s->data, s->def, was_bytes);
+    }
+    else {
+        s->data = erts_realloc(ERTS_ALC_T_DB_MC_STK, s->data, s->bytes);
+    }
+}
     
-#define DMC_INIT_STACK(Name) \
-     (Name).pos = 0; (Name).siz = DMC_DEFAULT_SIZE; (Name).data = (Name).def
+#define DMC_INIT_STACK(Name) do {       \
+    (Name).pos = 0;                     \
+    (Name).siz = DMC_DEFAULT_SIZE;      \
+    (Name).bytes = sizeof((Name).def);  \
+    (Name).data = (Name).def;           \
+} while (0)
 
 #define DMC_STACK_DATA(Name) (Name).data
 
@@ -87,21 +110,19 @@ typedef struct DMC_STACK_TYPE(Type) {		\
 
 #define DMC_PUSH(On, What)						\
 do {									\
-    if ((On).pos >= (On).siz) {						\
-	(On).siz *= 2;							\
-	(On).data							\
-	    = (((On).def == (On).data)					\
-	       ? sys_memcpy(erts_alloc(ERTS_ALC_T_DB_MC_STK,		\
-				   (On).siz*sizeof(*((On).data))),	\
-			(On).def,					\
-			DMC_DEFAULT_SIZE*sizeof(*((On).data)))		\
-	       : erts_realloc(ERTS_ALC_T_DB_MC_STK,			\
-			      (void *) (On).data,			\
-			      (On).siz*sizeof(*((On).data))));		\
-    }									\
+    if ((On).pos >= (On).siz)  						\
+        dmc_stack_grow((DMC_Dummy_stack*)&(On));                        \
     (On).data[(On).pos++] = What;					\
 } while (0)
 
+#define DMC_PUSH2(On, A, B)						\
+do {									\
+    if ((On).pos+1 >= (On).siz)  					\
+        dmc_stack_grow((DMC_Dummy_stack*)&(On));                        \
+    (On).data[(On).pos++] = A;					        \
+    (On).data[(On).pos++] = B;					        \
+} while (0)
+
 #define DMC_POP(From) (From).data[--(From).pos]
 
 #define DMC_TOP(From) (From).data[(From).pos - 1]
@@ -1537,8 +1558,7 @@ restart:
                 if (is_flatmap(t)) {
                     num_iters = flatmap_get_size(flatmap_val(t));
                     if (!structure_checked) {
-                        DMC_PUSH(text, matchMap);
-                        DMC_PUSH(text, num_iters);
+                        DMC_PUSH2(text, matchMap, num_iters);
                     }
                     structure_checked = 0;
                     for (i = 0; i < num_iters; ++i) {
@@ -1558,8 +1578,7 @@ restart:
                             }
                             goto error;
                         }
-                        DMC_PUSH(text, matchKey);
-                        DMC_PUSH(text, dmc_private_copy(&context, key));
+                        DMC_PUSH2(text, matchKey, dmc_private_copy(&context, key));
                         {
                             int old_stack = ++(context.stack_used);
                             Eterm value = flatmap_get_values(flatmap_val(t))[i];
@@ -1587,8 +1606,7 @@ restart:
                     Eterm *kv;
                     num_iters = hashmap_size(t);
                     if (!structure_checked) {
-                        DMC_PUSH(text, matchMap);
-                        DMC_PUSH(text, num_iters);
+                        DMC_PUSH2(text, matchMap, num_iters);
                     }
                     structure_checked = 0;
 
@@ -1614,8 +1632,7 @@ restart:
                             DESTROY_WSTACK(wstack);
                             goto error;
                         }
-                        DMC_PUSH(text, matchKey);
-                        DMC_PUSH(text, dmc_private_copy(&context, key));
+                        DMC_PUSH2(text, matchKey, dmc_private_copy(&context, key));
                         {
                             int old_stack = ++(context.stack_used);
                             res = dmc_one_term(&context, &heap, &stack, &text,
@@ -1645,8 +1662,7 @@ restart:
 		num_iters = arityval(*tuple_val(t));
 		if (!structure_checked) { /* i.e. we did not 
 					     pop it */
-		    DMC_PUSH(text,matchTuple);
-		    DMC_PUSH(text,num_iters);
+		    DMC_PUSH2(text, matchTuple, num_iters);
 		}
 		structure_checked = 0;
 		for (i = 1; i <= num_iters; ++i) {
@@ -3535,20 +3551,17 @@ static DMCRet dmc_one_term(DMCContext *context,
 		return retRestart;
 	    }
 	    if (heap->vars[n].is_bound) {
-		DMC_PUSH(*text,matchCmp);
-		DMC_PUSH(*text,n);
+		DMC_PUSH2(*text, matchCmp, n);
 	    } else { /* Not bound, bind! */
 		if (n >= heap->vars_used)
 		    heap->vars_used = n + 1;
-		DMC_PUSH(*text,matchBind);
-		DMC_PUSH(*text,n);
+		DMC_PUSH2(*text, matchBind, n);
 		heap->vars[n].is_bound = 1;
 	    }
 	} else if (c == am_Underscore) {
 	    DMC_PUSH(*text, matchSkip);
 	} else { /* Any immediate value */
-	    DMC_PUSH(*text, matchEq);
-	    DMC_PUSH(*text, (Uint) c);
+	    DMC_PUSH2(*text, matchEq, (Uint) c);
 	}
 	break;
     case TAG_PRIMARY_LIST:
@@ -3561,9 +3574,8 @@ static DMCRet dmc_one_term(DMCContext *context,
 	switch ((hdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) {
 	case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE):    
 	    n = arityval(*tuple_val(c));
-	    DMC_PUSH(*text, matchPushT);
+	    DMC_PUSH2(*text, matchPushT, n);
 	    ++(context->stack_used);
-	    DMC_PUSH(*text, n);
 	    DMC_PUSH(*stack, c);
 	    break;
         case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE):
@@ -3571,9 +3583,8 @@ static DMCRet dmc_one_term(DMCContext *context,
                 n = flatmap_get_size(flatmap_val(c));
             else
                 n = hashmap_size(c);
-            DMC_PUSH(*text, matchPushM);
+            DMC_PUSH2(*text, matchPushM, n);
             ++(context->stack_used);
-            DMC_PUSH(*text, n);
             DMC_PUSH(*stack, c);
             break;
 	case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE):
@@ -3598,8 +3609,7 @@ static DMCRet dmc_one_term(DMCContext *context,
 	    break;
 	}
 	case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE):
-	    DMC_PUSH(*text,matchEqFloat);
-	    DMC_PUSH(*text, (Uint) float_val(c)[1]);
+	    DMC_PUSH2(*text, matchEqFloat, (Uint) float_val(c)[1]);
 #ifdef ARCH_64
 	    DMC_PUSH(*text, (Uint) 0);
 #else
@@ -3607,8 +3617,7 @@ static DMCRet dmc_one_term(DMCContext *context,
 #endif
 	    break;
 	default: /* BINARY, FUN, VECTOR, or EXTERNAL */
-	    DMC_PUSH(*text, matchEqBin);
-            DMC_PUSH(*text, dmc_private_copy(context, c));
+	    DMC_PUSH2(*text, matchEqBin, dmc_private_copy(context, c));
 	    break;
 	}
 	break;
@@ -3658,8 +3667,7 @@ static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text,
 	    emb->next = context->save;
 	    context->save = emb;
 	}
-	DMC_PUSH(*text,matchPushC);
-	DMC_PUSH(*text,(Uint) tmp);
+	DMC_PUSH2(*text, matchPushC, (Uint)tmp);
 	if (++context->stack_used > context->stack_need)
 	    context->stack_need = context->stack_used;
 }
@@ -3797,8 +3805,7 @@ dmc_tuple(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text,
         *constant = 1;
         return retOk;
     }
-    DMC_PUSH(*text, matchMkTuple);
-    DMC_PUSH(*text, nelems);
+    DMC_PUSH2(*text, matchMkTuple, nelems);
     context->stack_used -= (nelems - 1);
     *constant = 0;
     return retOk;
@@ -3825,13 +3832,11 @@ dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text,
             *constant = 1;
             return retOk;
         }
-        DMC_PUSH(*text, matchPushC);
-        DMC_PUSH(*text, dmc_private_copy(context, m->keys));
+        DMC_PUSH2(*text, matchPushC, dmc_private_copy(context, m->keys));
         if (++context->stack_used > context->stack_need) {
             context->stack_need = context->stack_used;
         }
-        DMC_PUSH(*text, matchMkFlatMap);
-        DMC_PUSH(*text, nelems);
+        DMC_PUSH2(*text, matchMkFlatMap, nelems);
         context->stack_used -= nelems;
         *constant = 0;
         return retOk;
@@ -3883,8 +3888,7 @@ dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text,
                 do_emit_constant(context, text, CDR(kv));
             }
         }
-        DMC_PUSH(*text, matchMkHashMap);
-        DMC_PUSH(*text, nelems);
+        DMC_PUSH2(*text, matchMkHashMap, nelems);
         context->stack_used -= nelems;
         DESTROY_WSTACK(wstack);
         return retOk;
-- 
cgit v1.2.3


From 1ac8caa8d22f3767e84e7be749afbd5ed17a11b4 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 20 Mar 2018 16:48:26 +0100
Subject: erts: Fix some zero size sys_memcpy

---
 erts/emulator/beam/erl_bits.h         | 2 +-
 erts/emulator/beam/sys.h              | 4 +---
 erts/emulator/hipe/hipe_mode_switch.c | 3 ++-
 3 files changed, 4 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h
index b9d141d585..a3816fa820 100644
--- a/erts/emulator/beam/erl_bits.h
+++ b/erts/emulator/beam/erl_bits.h
@@ -111,7 +111,7 @@ typedef struct erl_bin_match_struct{
 #define copy_binary_to_buffer(DstBuffer, DstBufOffset, SrcBuffer, SrcBufferOffset, NumBits) \
   do {											    \
     if (BIT_OFFSET(DstBufOffset) == 0 && (SrcBufferOffset == 0) &&			    \
-        (BIT_OFFSET(NumBits)==0)) {							    \
+        (BIT_OFFSET(NumBits)==0) && (NumBits != 0)) {					    \
       sys_memcpy(DstBuffer+BYTE_OFFSET(DstBufOffset),					    \
 		 SrcBuffer, NBYTES(NumBits));						    \
     } else {										    \
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 152da8c9e1..c21acadd8d 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1004,9 +1004,7 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
  * may seemingly work when the length (if any) is zero; a compiler can take
  * this as a hint that the passed operand may *never* be NULL and then optimize
  * based on that information.
- *
- * (The weird casts in the assertions silence an "always evaluates to true"
- * warning when an operand is the address of an lvalue) */
+ */
 ERTS_GLB_INLINE void *sys_memcpy(void *dest, const void *src, size_t n);
 ERTS_GLB_INLINE void *sys_memmove(void *dest, const void *src, size_t n);
 ERTS_GLB_INLINE int sys_memcmp(const void *s1, const void *s2, size_t n);
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index 8b497c9970..f2befb07e3 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -662,7 +662,8 @@ void hipe_inc_nstack(Process *p)
     Eterm *new_nstack = erts_alloc(ERTS_ALC_T_HIPE_STK, new_size*sizeof(Eterm));
     unsigned used_size = p->hipe.nstend - p->hipe.nsp;
 
-    sys_memcpy(new_nstack+new_size-used_size, p->hipe.nsp, used_size*sizeof(Eterm));
+    if (used_size)
+        sys_memcpy(new_nstack+new_size-used_size, p->hipe.nsp, used_size*sizeof(Eterm));
     if (p->hipe.nstgraylim)
 	p->hipe.nstgraylim = new_nstack + new_size - (p->hipe.nstend - p->hipe.nstgraylim);
     if (p->hipe.nstblacklim)
-- 
cgit v1.2.3


From 902e1df69542e07e7c363f5b599ac1551b8fbb64 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 20 Mar 2018 15:27:43 +0100
Subject: erts: Add dynamic loaded drivers to list of "taints"

---
 erts/emulator/beam/erl_bif_ddll.c   |  9 +++++++++
 erts/emulator/beam/erl_nif.c        | 40 +++++++++++++++++++++++++------------
 erts/emulator/beam/global.h         |  1 +
 erts/emulator/test/driver_SUITE.erl | 12 ++++++++++-
 4 files changed, 48 insertions(+), 14 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 579e9b12f4..8220ba97a2 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1505,6 +1505,15 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name)
 	res = ERL_DE_LOAD_ERROR_BAD_NAME;
 	goto error;
     }
+
+    {
+        Eterm name_atom = erts_atom_put((byte*)name, sys_strlen(name),
+                                        ERTS_ATOM_ENC_LATIN1, 0);
+        if (is_non_value(name_atom))
+            goto error;
+        erts_add_taint(name_atom);
+    }
+
     erts_atomic_init_nob(&(dh->refc), (erts_aint_t) 0);
     erts_atomic32_init_nob(&dh->port_count, 0);
     dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index c60cc7fecf..121a7d943f 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3726,16 +3726,26 @@ static Eterm mkatom(const char *str)
     return am_atom_put(str, sys_strlen(str));
 }
 
-static struct tainted_module_t
+struct tainted_module_t
 {
     struct tainted_module_t* next;
     Eterm module_atom;
-}*first_tainted_module = NULL;
+};
+
+erts_atomic_t first_taint; /* struct tainted_module_t* */
 
-static void add_taint(Eterm mod_atom)
+void erts_add_taint(Eterm mod_atom)
 {
-    struct tainted_module_t* t;
-    for (t=first_tainted_module ; t!=NULL; t=t->next) {
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    extern erts_rwmtx_t erts_driver_list_lock; /* Mutex for driver list */
+#endif
+    struct tainted_module_t *first, *t;
+
+    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_driver_list_lock)
+                   || erts_thr_progress_is_blocking());
+
+    first = (struct tainted_module_t*) erts_atomic_read_nob(&first_taint);
+    for (t=first ; t; t=t->next) {
 	if (t->module_atom == mod_atom) {
 	    return;
 	}
@@ -3743,22 +3753,24 @@ static void add_taint(Eterm mod_atom)
     t = erts_alloc_fnf(ERTS_ALC_T_TAINT, sizeof(*t));
     if (t != NULL) {
 	t->module_atom = mod_atom;
-	t->next = first_tainted_module;
-	first_tainted_module = t;
+	t->next = first;
+        erts_atomic_set_nob(&first_taint, (erts_aint_t)t);
     }
 }
 
 Eterm erts_nif_taints(Process* p)
 {
-    struct tainted_module_t* t;
+    struct tainted_module_t *first, *t;
     unsigned cnt = 0;
     Eterm list = NIL;
     Eterm* hp;
-    for (t=first_tainted_module ; t!=NULL; t=t->next) {
+
+    first = (struct tainted_module_t*) erts_atomic_read_nob(&first_taint);
+    for (t=first ; t!=NULL; t=t->next) {
 	cnt++;
     }
     hp = HAlloc(p,cnt*2);
-    for (t=first_tainted_module ; t!=NULL; t=t->next) {
+    for (t=first ; t!=NULL; t=t->next) {
 	list = CONS(hp, t->module_atom, list);
 	hp += 2;
     }
@@ -3767,9 +3779,11 @@ Eterm erts_nif_taints(Process* p)
 
 void erts_print_nif_taints(fmtfn_t to, void* to_arg)
 {
-    struct tainted_module_t* t;
+    struct tainted_module_t *t;
     const char* delim = "";
-    for (t=first_tainted_module ; t!=NULL; t=t->next) {
+
+    t = (struct tainted_module_t*) erts_atomic_read_nob(&first_taint);
+    for ( ; t; t = t->next) {
 	const Atom* atom = atom_tab(atom_val(t->module_atom));
 	erts_cbprintf(to,to_arg,"%s%.*s", delim, atom->len, atom->name);
 	delim = ",";
@@ -3964,7 +3978,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 			      " function: '%s'", errdesc.str);
 	
     }
-    else if ((add_taint(mod_atom),
+    else if ((erts_add_taint(mod_atom),
 	      (entry = erts_sys_ddll_call_nif_init(init_func)) == NULL)) {
 	ret = load_nif_error(BIF_P, bad_lib, "Library init-call unsuccessful");
     }
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 0f23027752..3c98ccfef3 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -117,6 +117,7 @@ extern void erts_pre_nif(struct enif_environment_t*, Process*,
 extern void erts_post_nif(struct enif_environment_t* env);
 extern void erts_resource_stop(ErtsResource*, ErlNifEvent, int is_direct_call);
 void erts_fire_nif_monitor(ErtsResource*, Eterm pid, Eterm ref);
+extern void erts_add_taint(Eterm mod_atom);
 extern Eterm erts_nif_taints(Process* p);
 extern void erts_print_nif_taints(fmtfn_t to, void* to_arg);
 void erts_unload_nif(struct erl_module_nif* nif);
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 294c42780d..40c7cc11e1 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -2589,8 +2589,18 @@ stop_driver(Port, Name) ->
     ok = erl_ddll:stop().
 
 load_driver(Dir, Driver) ->
+    Before = erlang:system_info(taints),
     case erl_ddll:load_driver(Dir, Driver) of
-        ok -> ok;
+        ok ->
+            After = erlang:system_info(taints),
+            case lists:member(Driver, Before) of
+                true ->
+                    After = Before;
+                false ->
+                    true = lists:member(Driver, After),
+                    Before = lists:delete(Driver, After)
+            end,
+            ok;
         {error, Error} = Res ->
             io:format("~s\n", [erl_ddll:format_error(Error)]),
             Res
-- 
cgit v1.2.3


From 5b557bccac579291301a7a4d78a3d992b4e9373d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 20 Mar 2018 18:55:51 +0100
Subject: erts: Refactor erts_static_nif_get_nif_init

to return pointer to ErtsStaticNifEntry.
---
 erts/emulator/beam/erl_nif.c        | 12 ++++++++----
 erts/emulator/beam/global.h         |  6 +++++-
 erts/emulator/utils/make_driver_tab | 15 +++------------
 3 files changed, 16 insertions(+), 17 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 121a7d943f..0720796b53 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3938,10 +3938,14 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     ASSERT(module_p != NULL);
 
     mod_atomp = atom_tab(atom_val(mod_atom));
-    init_func = erts_static_nif_get_nif_init((char*)mod_atomp->name, mod_atomp->len);
-    if (init_func != NULL)
-      handle = init_func;
-
+    {
+        ErtsStaticNifEntry* sne;
+        sne = erts_static_nif_get_nif_init((char*)mod_atomp->name, mod_atomp->len);
+        if (sne != NULL) {
+            init_func = sne->nif_init;
+            handle = init_func;
+        }
+    }
     this_mi = &module_p->curr;
     prev_mi = &module_p->old;
     if (in_area(caller, module_p->old.code_hdr, module_p->old.code_length)) {
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 3c98ccfef3..ae9fe0cc62 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1178,7 +1178,11 @@ void erts_lcnt_update_port_locks(int enable);
 
 /* driver_tab.c */
 typedef void *(*ErtsStaticNifInitFPtr)(void);
-ErtsStaticNifInitFPtr erts_static_nif_get_nif_init(const char *name, int len);
+typedef struct ErtsStaticNifEntry_ {
+    const char *nif_name;
+    ErtsStaticNifInitFPtr nif_init;
+} ErtsStaticNifEntry;
+ErtsStaticNifEntry* erts_static_nif_get_nif_init(const char *name, int len);
 int erts_is_static_nif(void *handle);
 void erts_init_static_drivers(void);
 
diff --git a/erts/emulator/utils/make_driver_tab b/erts/emulator/utils/make_driver_tab
index ffb5f58ebf..6f28a21f81 100755
--- a/erts/emulator/utils/make_driver_tab
+++ b/erts/emulator/utils/make_driver_tab
@@ -115,15 +115,6 @@ foreach (@static_drivers) {
 
 print "}\n";
 
-print <<EOF;
-
-typedef struct ErtsStaticNifEntry_ {
-    const char *nif_name;
-    ErtsStaticNifInitFPtr nif_init;
-} ErtsStaticNifEntry;
-
-EOF
-
 # prototypes
 foreach (@static_nifs) {
     my $d = ${_};
@@ -137,17 +128,17 @@ print "static ErtsStaticNifEntry static_nif_tab[] =\n{\n";
 foreach (@static_nifs) {
     my $d = ${_};
     $d =~ s/\.debug//; # strip .debug
-    print "{\"${_}\",&".$d."_nif_init},\n";
+    print "    {\"${_}\",&".$d."_nif_init},\n";
 }
 
 print "    {NULL,NULL}\n};\n";
 
 print <<EOF;
-ErtsStaticNifInitFPtr erts_static_nif_get_nif_init(const char *name, int len) {
+ErtsStaticNifEntry* erts_static_nif_get_nif_init(const char *name, int len) {
     ErtsStaticNifEntry* p;
     for (p = static_nif_tab; p->nif_name != NULL; p++)
         if (strncmp(p->nif_name, name, len) == 0 && p->nif_name[len] == 0)
-            return p->nif_init;
+            return p;
     return NULL;
 }
 
-- 
cgit v1.2.3


From 8fea289244d8758c69b8c8443679b2d73cb2f70d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 20 Mar 2018 19:24:42 +0100
Subject: erts: Remove our own NIF modules from "taints"

Dynamic NIF libs and those added with config option
--enable-static-nifs are considered as taints.
---
 erts/emulator/beam/erl_nif.c        |  4 +++-
 erts/emulator/beam/global.h         |  1 +
 erts/emulator/utils/make_driver_tab | 15 +++++++++++++--
 3 files changed, 17 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 0720796b53..8dcecd79f0 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3892,6 +3892,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT;
     Eterm ret = am_ok;
     int veto;
+    int taint = 1;
     struct erl_module_nif* lib = NULL;
     struct erl_module_instance* this_mi;
     struct erl_module_instance* prev_mi;
@@ -3944,6 +3945,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
         if (sne != NULL) {
             init_func = sne->nif_init;
             handle = init_func;
+            taint = sne->taint;
         }
     }
     this_mi = &module_p->curr;
@@ -3982,7 +3984,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 			      " function: '%s'", errdesc.str);
 	
     }
-    else if ((erts_add_taint(mod_atom),
+    else if ((taint ? erts_add_taint(mod_atom) : 0,
 	      (entry = erts_sys_ddll_call_nif_init(init_func)) == NULL)) {
 	ret = load_nif_error(BIF_P, bad_lib, "Library init-call unsuccessful");
     }
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index ae9fe0cc62..8a746ea4b1 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1181,6 +1181,7 @@ typedef void *(*ErtsStaticNifInitFPtr)(void);
 typedef struct ErtsStaticNifEntry_ {
     const char *nif_name;
     ErtsStaticNifInitFPtr nif_init;
+    int taint;
 } ErtsStaticNifEntry;
 ErtsStaticNifEntry* erts_static_nif_get_nif_init(const char *name, int len);
 int erts_is_static_nif(void *handle);
diff --git a/erts/emulator/utils/make_driver_tab b/erts/emulator/utils/make_driver_tab
index 6f28a21f81..cefb3e2504 100755
--- a/erts/emulator/utils/make_driver_tab
+++ b/erts/emulator/utils/make_driver_tab
@@ -30,6 +30,7 @@ use File::Basename;
 my $file = "";
 my $nif = "";
 my @emu_drivers = ();
+my @emu_nifs = ();
 my @static_drivers = ();
 my @static_nifs = ();
 my $mode = 1;
@@ -61,7 +62,7 @@ while (@ARGV) {
     } elsif ($mode == 2) {
         $d = basename $d;
         $d =~ s/_nif(\..*|)$//; # strip nif.* or just nif
-        push(@static_nifs, $d);
+        push(@emu_nifs, $d);
         next;
     }
     $d = basename $d;
@@ -116,6 +117,11 @@ foreach (@static_drivers) {
 print "}\n";
 
 # prototypes
+foreach (@emu_nifs) {
+    my $d = ${_};
+    $d =~ s/\.debug//; # strip .debug
+    print "void *".$d."_nif_init(void);\n";
+}
 foreach (@static_nifs) {
     my $d = ${_};
     $d =~ s/\.debug//; # strip .debug
@@ -125,10 +131,15 @@ foreach (@static_nifs) {
 # The array itself
 print "static ErtsStaticNifEntry static_nif_tab[] =\n{\n";
 
+foreach (@emu_nifs) {
+    my $d = ${_};
+    $d =~ s/\.debug//; # strip .debug
+    print "    {\"${_}\", &".$d."_nif_init, 0},\n";
+}
 foreach (@static_nifs) {
     my $d = ${_};
     $d =~ s/\.debug//; # strip .debug
-    print "    {\"${_}\",&".$d."_nif_init},\n";
+    print "    {\"${_}\", &".$d."_nif_init, 1},\n";
 }
 
 print "    {NULL,NULL}\n};\n";
-- 
cgit v1.2.3


From 5fcb8fe88e26938537de72462dc77e91a450ee57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 21 Mar 2018 10:09:00 +0100
Subject: Fix a misleading comment

---
 erts/emulator/test/module_info_SUITE.erl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/module_info_SUITE.erl b/erts/emulator/test/module_info_SUITE.erl
index ba9b564fdc..0341e63f13 100644
--- a/erts/emulator/test/module_info_SUITE.erl
+++ b/erts/emulator/test/module_info_SUITE.erl
@@ -62,7 +62,8 @@ exports(Config) when is_list(Config) ->
     All = lists:sort(?MODULE:module_info(exports)),
     ok.
 
-%% Test that the list of exported functions from this module is correct.
+%% Test that the list of local and exported functions from this module is
+%% correct.
 functions(Config) when is_list(Config) ->
     All = all_functions(),
     All = lists:sort(?MODULE:module_info(functions)),
-- 
cgit v1.2.3


From 4bc282d812cc2c49aa3e2d073e96c720f16aa270 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 7 Mar 2018 01:17:21 +0100
Subject: Implementation of true asynchronous signaling between processes

Communication between Erlang processes has conceptually always been
performed through asynchronous signaling. The runtime system
implementation has however previously preformed most operation
synchronously. In a system with only one true thread of execution, this
is not problematic (often the opposite). In a system with multiple threads
of execution (as current runtime system implementation with SMP support)
it becomes problematic. This since it often involves locking of structures
when updating them which in turn cause resource contention. Utilizing
true asynchronous communication often avoids these resource contention
issues.

The case that triggered this change was contention on the link lock due
to frequent updates of the monitor trees during communication with a
frequently used server. The signal order delivery guarantees of the
language makes it hard to change the implementation of only some signals
to use true asynchronous signaling. Therefore the implementations
of (almost) all signals have been changed.

Currently the following signals have been implemented as true
asynchronous signals:
- Message signals
- Exit signals
- Monitor signals
- Demonitor signals
- Monitor triggered signals (DOWN, CHANGE, etc)
- Link signals
- Unlink signals
- Group leader signals

All of the above already defined as asynchronous signals in the
language. The implementation of messages signals was quite
asynchronous to begin with, but had quite strict delivery constraints
due to the ordering guarantees of signals between a pair of processes.

The previously used message queue partitioned into two halves has been
replaced by a more general signal queue partitioned into three parts
that service all kinds of signals. More details regarding the signal
queue can be found in comments in the erl_proc_sig_queue.h file.

The monitor and link implementations have also been completely replaced
in order to fit the new asynchronous signaling implementation as good
as possible. More details regarding the new monitor and link
implementations can be found in the erl_monitor_link.h file.
---
 erts/emulator/Makefile.in                          |   10 +-
 erts/emulator/beam/atom.names                      |    4 +-
 erts/emulator/beam/beam_bif_load.c                 |  101 +-
 erts/emulator/beam/beam_emu.c                      |   13 +-
 erts/emulator/beam/bif.c                           | 1733 ++++-----
 erts/emulator/beam/bif.h                           |    8 +-
 erts/emulator/beam/bif.tab                         |   15 +-
 erts/emulator/beam/break.c                         |  137 +-
 erts/emulator/beam/dist.c                          | 1768 ++++-----
 erts/emulator/beam/dist.h                          |   58 +-
 erts/emulator/beam/erl_alloc.c                     |   23 +-
 erts/emulator/beam/erl_alloc.types                 |   15 +-
 erts/emulator/beam/erl_bif_info.c                  |  916 +++--
 erts/emulator/beam/erl_bif_port.c                  |   66 +-
 erts/emulator/beam/erl_bif_trace.c                 |    3 +-
 erts/emulator/beam/erl_db_util.c                   |    4 +-
 erts/emulator/beam/erl_debug.c                     |   36 +-
 erts/emulator/beam/erl_gc.c                        |  247 +-
 erts/emulator/beam/erl_gc.h                        |    4 +-
 erts/emulator/beam/erl_hl_timer.c                  |   40 +-
 erts/emulator/beam/erl_init.c                      |   66 +-
 erts/emulator/beam/erl_lock_check.c                |    3 +-
 erts/emulator/beam/erl_message.c                   |  327 +-
 erts/emulator/beam/erl_message.h                   |  259 +-
 erts/emulator/beam/erl_monitor_link.c              | 1341 +++++++
 erts/emulator/beam/erl_monitor_link.h              | 2326 ++++++++++++
 erts/emulator/beam/erl_monitors.c                  | 1073 ------
 erts/emulator/beam/erl_monitors.h                  |  187 -
 erts/emulator/beam/erl_nif.c                       |  373 +-
 erts/emulator/beam/erl_node_container_utils.h      |    2 -
 erts/emulator/beam/erl_node_tables.c               |  408 ++-
 erts/emulator/beam/erl_node_tables.h               |   52 +-
 erts/emulator/beam/erl_port.h                      |   48 +-
 erts/emulator/beam/erl_proc_sig_queue.c            | 3772 ++++++++++++++++++++
 erts/emulator/beam/erl_proc_sig_queue.h            |  665 ++++
 erts/emulator/beam/erl_process.c                   | 1910 +++++-----
 erts/emulator/beam/erl_process.h                   |   83 +-
 erts/emulator/beam/erl_process_dump.c              |  112 +-
 erts/emulator/beam/erl_process_lock.c              |   98 +-
 erts/emulator/beam/erl_process_lock.h              |   56 +-
 erts/emulator/beam/erl_ptab.h                      |    6 +-
 erts/emulator/beam/erl_rbtree.h                    |  181 +-
 erts/emulator/beam/erl_term.h                      |    7 +-
 erts/emulator/beam/erl_time.h                      |    8 +-
 erts/emulator/beam/erl_time_sup.c                  |   84 +-
 erts/emulator/beam/external.h                      |    5 +-
 erts/emulator/beam/global.h                        |   13 +-
 erts/emulator/beam/io.c                            |  755 ++--
 erts/emulator/beam/msg_instrs.tab                  |   95 +-
 erts/emulator/hipe/hipe_mkliterals.c               |   10 +-
 erts/emulator/hipe/hipe_mode_switch.c              |    6 +-
 erts/emulator/hipe/hipe_native_bif.c               |   80 +-
 erts/emulator/sys/common/erl_check_io.c            |    4 +-
 erts/emulator/test/bif_SUITE.erl                   |  131 +-
 .../test/dirty_nif_SUITE_data/dirty_nif_SUITE.c    |   31 +-
 erts/emulator/test/erl_link_SUITE.erl              |  266 +-
 erts/emulator/test/monitor_SUITE.erl               |   10 +-
 erts/emulator/test/port_SUITE.erl                  |    9 +-
 erts/emulator/test/port_trace_SUITE.erl            |   14 +-
 erts/emulator/test/process_SUITE.erl               |   28 +-
 erts/emulator/test/signal_SUITE.erl                |  412 +--
 erts/emulator/test/trace_SUITE.erl                 |   91 +-
 62 files changed, 13239 insertions(+), 7369 deletions(-)
 create mode 100644 erts/emulator/beam/erl_monitor_link.c
 create mode 100644 erts/emulator/beam/erl_monitor_link.h
 delete mode 100644 erts/emulator/beam/erl_monitors.c
 delete mode 100644 erts/emulator/beam/erl_monitors.h
 create mode 100644 erts/emulator/beam/erl_proc_sig_queue.c
 create mode 100644 erts/emulator/beam/erl_proc_sig_queue.h

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 9c79bf3da0..92cf40c1ae 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1996-2017. All Rights Reserved.
+# Copyright Ericsson AB 1996-2018. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -645,7 +645,7 @@ PRELOAD_BEAM =		$(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/erts_internal.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_code_checker.beam
+			$(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam
 
 ifeq ($(TARGET),win32)
 # On windows the preloaded objects are in a resource object.
@@ -844,7 +844,7 @@ RUN_OBJS += \
 	$(OBJDIR)/utils.o		$(OBJDIR)/bif.o \
 	$(OBJDIR)/io.o 			$(OBJDIR)/erl_printf_term.o\
 	$(OBJDIR)/erl_debug.o		$(OBJDIR)/erl_md5.o \
-	$(OBJDIR)/erl_message.o		 \
+	$(OBJDIR)/erl_message.o		$(OBJDIR)/erl_proc_sig_queue.o \
 	$(OBJDIR)/erl_process_dict.o	$(OBJDIR)/erl_process_lock.o \
 	$(OBJDIR)/erl_port_task.o	$(OBJDIR)/erl_arith.o \
 	$(OBJDIR)/time.o		$(OBJDIR)/erl_time_sup.o  \
@@ -858,11 +858,11 @@ RUN_OBJS += \
 	$(OBJDIR)/register.o		$(OBJDIR)/break.o \
 	$(OBJDIR)/erl_async.o		$(OBJDIR)/erl_lock_check.o \
 	$(OBJDIR)/erl_gc.o 		$(OBJDIR)/erl_lock_count.o \
-	$(OBJDIR)/erl_posix_str.o \
+	$(OBJDIR)/erl_posix_str.o	 \
 	$(OBJDIR)/erl_bits.o 		$(OBJDIR)/erl_math.o \
 	$(OBJDIR)/erl_fun.o             $(OBJDIR)/erl_bif_port.o \
 	$(OBJDIR)/erl_term.o 		$(OBJDIR)/erl_node_tables.o \
-	$(OBJDIR)/erl_monitors.o	$(OBJDIR)/erl_process_dump.o \
+	$(OBJDIR)/erl_monitor_link.o	$(OBJDIR)/erl_process_dump.o \
 	$(OBJDIR)/erl_hl_timer.o	$(OBJDIR)/erl_cpu_topology.o \
 	$(OBJDIR)/erl_drv_thread.o      $(OBJDIR)/erl_bif_chksum.o \
 	$(OBJDIR)/erl_bif_re.o		$(OBJDIR)/erl_unicode.o \
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 38b5f0c5e3..7963386e1d 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1996-2017. All Rights Reserved.
+# Copyright Ericsson AB 1996-2018. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -106,6 +106,7 @@ atom atom
 atom atom_used
 atom attributes
 atom auto_connect
+atom await_exit
 atom await_microstate_accounting_modifications
 atom await_port_send_result
 atom await_proc_exit
@@ -214,7 +215,6 @@ atom dist_data
 atom Div='/'
 atom div
 atom dmonitor_node
-atom dmonitor_p
 atom DollarDollar='$$'
 atom DollarUnderscore='$_'
 atom dollar_endonly
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 87367b44ab..5c76aafae7 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -37,6 +37,7 @@
 #include "erl_bits.h"
 #include "erl_thr_progress.h"
 #include "erl_nfunc_sched.h"
+#include "erl_proc_sig_queue.h"
 #ifdef HIPE
 #  include "hipe_bif0.h"
 #  define IF_HIPE(X) (X)
@@ -65,7 +66,6 @@ static struct {
 
 Process *erts_code_purger = NULL;
 
-Process *erts_dirty_process_code_checker;
 erts_atomic_t erts_copy_literal_area__;
 #define ERTS_SET_COPY_LITERAL_AREA(LA)			\
     erts_atomic_set_nob(&erts_copy_literal_area__,	\
@@ -607,7 +607,9 @@ BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2)
     int reds = 0;
     Eterm res;
 
-    if (BIF_P != erts_dirty_process_code_checker)
+    if (BIF_P != erts_dirty_process_signal_handler
+        && BIF_P != erts_dirty_process_signal_handler_high
+        && BIF_P != erts_dirty_process_signal_handler_max)
 	BIF_ERROR(BIF_P, EXC_NOTSUP);
 
     if (is_not_internal_pid(BIF_ARG_1))
@@ -901,15 +903,59 @@ static void hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp,
                                Eterm *start, Eterm *end,
                                char *lit_start, Uint lit_size);
 
+static ERTS_INLINE void
+msg_copy_literal_area(ErtsMessage *msgp, int *redsp,
+                      char *literals, Uint lit_bsize)
+{
+    ErlHeapFragment *hfrag, *hf;
+    Uint lit_sz = 0;
+
+    *redsp += 1;
+
+    if (!ERTS_SIG_IS_INTERNAL_MSG(msgp) || !msgp->data.attached)
+        return;
+
+    if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG)
+        hfrag = &msgp->hfrag;
+    else
+        hfrag = msgp->data.heap_frag;
+
+    for (hf = hfrag; hf; hf = hf->next) {
+        lit_sz += hfrag_literal_size(&hf->mem[0],
+                                     &hf->mem[hf->used_size],
+                                     literals, lit_bsize);
+        *redsp += 1;
+    }
+
+    *redsp += lit_sz / 16; /* Better value needed... */
+    if (lit_sz > 0) {
+        ErlHeapFragment *bp = new_message_buffer(lit_sz);
+        Eterm *hp = bp->mem;
+
+        for (hf = hfrag; hf; hf = hf->next) {
+            hfrag_literal_copy(&hp, &bp->off_heap,
+                               &hf->mem[0],
+                               &hf->mem[hf->used_size],
+                               literals, lit_bsize);
+            hfrag = hf;
+        }
+
+        /* link new hfrag last */
+        ASSERT(hfrag->next == NULL);
+        hfrag->next = bp;
+        bp->next = NULL;
+    }
+}
+
 Eterm
 erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed)
 {
     ErtsLiteralArea *la;
-    ErtsMessage *msgp;
     struct erl_off_heap_header* oh;
     char *literals;
     Uint lit_bsize;
     ErlHeapFragment *hfrag;
+    ErtsMessage *mfp;
 
     la = ERTS_COPY_LITERAL_AREA();
     if (!la)
@@ -927,46 +973,13 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed
      */
 
     erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
-    ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
+    erts_proc_sig_fetch(c_p);
     erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
 
-    for (msgp = c_p->msg.first; msgp; msgp = msgp->next) {
-	ErlHeapFragment *hf;
-	Uint lit_sz = 0;
-
-	*redsp += 1;
-
-	if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG)
-	    hfrag = &msgp->hfrag;
-	else if (is_value(ERL_MESSAGE_TERM(msgp)) && msgp->data.heap_frag)
-	    hfrag = msgp->data.heap_frag;
-	else
-	    continue; /* Content on heap or in external term format... */
-
-	for (hf = hfrag; hf; hf = hf->next) {
-	    lit_sz += hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size],
-					literals, lit_bsize);
-	    *redsp += 1;
-	}
-
-	*redsp += lit_sz / 16; /* Better value needed... */
-	if (lit_sz > 0) {
-	    ErlHeapFragment *bp = new_message_buffer(lit_sz);
-	    Eterm *hp = bp->mem;
-
-	    for (hf = hfrag; hf; hf = hf->next) {
-		hfrag_literal_copy(&hp, &bp->off_heap,
-				   &hf->mem[0], &hf->mem[hf->used_size],
-				   literals, lit_bsize);
-		hfrag = hf;
-	    }
-
-	    /* link new hfrag last */
-	    ASSERT(hfrag->next == NULL);
-	    hfrag->next = bp;
-	    bp->next = NULL;
-        }
-    }
+    ERTS_FOREACH_SIG_PRIVQS(c_p, msgp, msg_copy_literal_area(msgp,
+                                                             redsp,
+                                                             literals,
+                                                             lit_bsize));
 
     if (gc_allowed) {
 	/*
@@ -1041,8 +1054,8 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed
      *    process off heap structure.
      *  - Check for literals
      */
-    for (msgp = c_p->msg_frag; msgp; msgp = msgp->next) {
-	hfrag = erts_message_to_heap_frag(msgp);
+    for (mfp = c_p->msg_frag; mfp; mfp = mfp->next) {
+	hfrag = erts_message_to_heap_frag(mfp);
 	for (; hfrag; hfrag = hfrag->next) {
 	    Eterm *hp, *hp_end;
 
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index fbd0e38735..fb87be3f17 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -44,6 +44,7 @@
 #include "hipe_bif1.h"
 #endif
 #include "dtrace-wrapper.h"
+#include "erl_proc_sig_queue.h"
 
 /* #define HARDDEBUG 1 */
 
@@ -1454,7 +1455,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, ErtsCodeMFA *bif_mfa)
 	reg[3] = c_p->ftrace;
         if ((new_pc = next_catch(c_p, reg))) {
 	    c_p->cp = 0;	/* To avoid keeping stale references. */
-            c_p->msg.saved_last = 0;  /* No longer safe to use this position */
+            ERTS_RECV_MARK_CLEAR(c_p); /* No longer safe to use this position */
 	    return new_pc;
 	}
 	if (c_p->catches > 0) erts_exit(ERTS_ERROR_EXIT, "Catch not found");
@@ -2419,8 +2420,8 @@ erts_hibernate(Process* c_p, Eterm* reg)
      * shrink the heap. 
      */
     erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
-    ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
-    if (!c_p->msg.len) {
+    erts_proc_sig_fetch(c_p);
+    if (!c_p->sig_qs.len) {
 	erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
 	c_p->fvalue = NIL;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2428,8 +2429,8 @@ erts_hibernate(Process* c_p, Eterm* reg)
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
-        ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
-	if (!c_p->msg.len)
+        erts_proc_sig_fetch(c_p);
+	if (!c_p->sig_qs.len)
 	    erts_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
     }
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 652b95105f..adea7d007e 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -46,8 +46,10 @@
 #include "erl_bif_unique.h"
 #include "erl_map.h"
 #include "erl_msacc.h"
+#include "erl_proc_sig_queue.h"
 
 Export *erts_await_result;
+static Export await_exit_trap;
 static Export* flush_monitor_messages_trap = NULL;
 static Export* set_cpu_topology_trap = NULL;
 static Export* await_proc_exit_trap = NULL;
@@ -92,83 +94,51 @@ BIF_RETTYPE spawn_3(BIF_ALIST_3)
 /* Utility to add a new link between processes p and another internal
  * process (rpid). Process p must be the currently executing process.
  */
-static int insert_internal_link(Process* p, Eterm rpid)
-{
-    Process *rp;
-    ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK;
-
-    ASSERT(is_internal_pid(rpid));
-
-    if (IS_TRACED(p)
-	&& (ERTS_TRACE_FLAGS(p) & (F_TRACE_SOL|F_TRACE_SOL1))) {
-	rp_locks = ERTS_PROC_LOCKS_ALL;
-    }
-
-    erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
-
-    /* get a pointer to the process struct of the linked process */
-    rp = erts_pid2proc_opt(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK,
-			   rpid, rp_locks,
-			   ERTS_P2P_FLG_ALLOW_OTHER_X);
-
-    if (!rp) {
-	erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
-	return 0;
-    }
-
-    if (p != rp) {
-	erts_add_link(&ERTS_P_LINKS(p), LINK_PID, rp->common.id);
-	erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, p->common.id);
-
-	ASSERT(IS_TRACER_VALID(ERTS_TRACER(p)));
-
-	if (IS_TRACED(p)) {
-	    if (ERTS_TRACE_FLAGS(p) & (F_TRACE_SOL|F_TRACE_SOL1))  {
-		ERTS_TRACE_FLAGS(rp) |= (ERTS_TRACE_FLAGS(p) & TRACEE_FLAGS);
-                erts_tracer_replace(&rp->common, ERTS_TRACER(p));
-		if (ERTS_TRACE_FLAGS(p) & F_TRACE_SOL1) { /* maybe override */
-		    ERTS_TRACE_FLAGS(rp) &= ~(F_TRACE_SOL1 | F_TRACE_SOL);
-		    ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOL1 | F_TRACE_SOL);
-		}
-	    }
-	}
-    }
-    if (IS_TRACED_FL(rp, F_TRACE_PROCS))
-	trace_proc(p, p == rp ? rp_locks : ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK,
-                   rp, am_getting_linked, p->common.id);
-
-    if (p == rp)
-	erts_proc_unlock(p, rp_locks & ~ERTS_PROC_LOCK_MAIN);
-    else {
-	erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
-	erts_proc_unlock(rp, rp_locks);
-    }
-
-    return 1;
-}
-
 
 /* create a link to the process */
 BIF_RETTYPE link_1(BIF_ALIST_1)
 {
-    DistEntry *dep;
-
     if (IS_TRACED_FL(BIF_P, F_TRACE_PROCS)) {
 	trace_proc(BIF_P, ERTS_PROC_LOCK_MAIN, BIF_P, am_link, BIF_ARG_1);
     }
     /* check that the pid or port which is our argument is OK */
 
     if (is_internal_pid(BIF_ARG_1)) {
-	if (insert_internal_link(BIF_P, BIF_ARG_1)) {
-	    BIF_RET(am_true);
-	}
-	else {
-	    goto res_no_proc;
-	}
+        int created;
+        ErtsLinkData *ldp;
+        ErtsLink *lnk;
+
+        if (BIF_P->common.id == BIF_ARG_1)
+            BIF_RET(am_true);
+
+        if (!erts_proc_lookup(BIF_ARG_1))
+            goto res_no_proc;
+
+        lnk = erts_link_tree_lookup_create(&ERTS_P_LINKS(BIF_P),
+                                           &created,
+                                           ERTS_LNK_TYPE_PROC,
+                                           BIF_P->common.id,
+                                           BIF_ARG_1);
+        if (!created)
+            BIF_RET(am_true);
+
+        ldp = erts_link_to_data(lnk);
+        
+
+        if (erts_proc_sig_send_link(BIF_P, BIF_ARG_1, &ldp->b))
+            BIF_RET(am_true);
+
+        erts_link_tree_delete(&ERTS_P_LINKS(BIF_P), lnk);
+        erts_link_release_both(ldp);
+        goto res_no_proc;
     }
 
     if (is_internal_port(BIF_ARG_1)) {
-	int send_link_signal = 0;
+        int created;
+        ErtsLinkData *ldp;
+        ErtsLink *lnk;
+        Eterm ref;
+        Eterm *refp;
 	Port *prt = erts_port_lookup(BIF_ARG_1,
 				     (erts_port_synchronous_ops
 				      ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
@@ -177,31 +147,31 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
 	    goto res_no_proc;
 	}
 
-	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
-	
-	if (erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, BIF_ARG_1) >= 0)
-	    send_link_signal = 1;
-	/* else: already linked */
-
-	erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
+        lnk = erts_link_tree_lookup_create(&ERTS_P_LINKS(BIF_P),
+                                           &created,
+                                           ERTS_LNK_TYPE_PORT,
+                                           BIF_P->common.id,
+                                           BIF_ARG_1);
+        if (!created)
+            BIF_RET(am_true);
 
-	if (send_link_signal) {
-	    Eterm ref;
-	    Eterm *refp = erts_port_synchronous_ops ? &ref : NULL;
+        ldp = erts_link_to_data(lnk);
+        refp = erts_port_synchronous_ops ? &ref : NULL;
 
-	    switch (erts_port_link(BIF_P, prt, BIF_P->common.id, refp)) {
-	    case ERTS_PORT_OP_DROPPED:
-	    case ERTS_PORT_OP_BADARG:
-		goto res_no_proc;
-	    case ERTS_PORT_OP_SCHEDULED:
-		if (refp) {
-		    ASSERT(is_internal_ordinary_ref(ref));
-		    BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true);
-		}
-	    default:
-		break;
-	    }
-	}
+        switch (erts_port_link(BIF_P, prt, &ldp->b, refp)) {
+        case ERTS_PORT_OP_DROPPED:
+        case ERTS_PORT_OP_BADARG:
+            erts_link_tree_delete(&ERTS_P_LINKS(BIF_P), lnk);
+            erts_link_release_both(ldp);
+            goto res_no_proc;
+        case ERTS_PORT_OP_SCHEDULED:
+            if (refp) {
+                ASSERT(is_internal_ordinary_ref(ref));
+                BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true);
+            }
+        default:
+            break;
+        }
 	BIF_RET(am_true);
     }
     else if (is_external_port(BIF_ARG_1)
@@ -210,317 +180,203 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
     }
 
     if (is_external_pid(BIF_ARG_1)) {
+        ErtsLinkData *ldp;
+        int created;
+        DistEntry *dep;
+        ErtsLink *lnk;
+        int code;
+        ErtsDSigData dsd;
+
+        dep = external_pid_dist_entry(BIF_ARG_1);
+        if (dep == erts_this_dist_entry)
+            goto res_no_proc;
 
-	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
-
-	/* We may earn time by checking first that we're not linked already */
-	if (erts_lookup_link(ERTS_P_LINKS(BIF_P), BIF_ARG_1) != NULL) {
-	    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
-	    BIF_RET(am_true);
-	}
-	else {
-	    ErtsLink *lnk;
-	    int code;
-	    ErtsDSigData dsd;
-	    dep = external_pid_dist_entry(BIF_ARG_1);
-	    if (dep == erts_this_dist_entry) {
-		erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
-		goto res_no_proc;
-	    }
-
-	    code = erts_dsig_prepare(&dsd, dep, BIF_P,
-				     (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK),
-				     ERTS_DSP_RLOCK, 0, 1);
-	    switch (code) {
-	    case ERTS_DSIG_PREP_NOT_ALIVE:
-	    case ERTS_DSIG_PREP_NOT_CONNECTED: {
-		ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK;
-		erts_aint32_t state;
-		erts_proc_lock(BIF_P, (ERTS_PROC_LOCKS_ALL & ~locks));
-		locks = ERTS_PROC_LOCKS_ALL;
-		erts_send_exit_signal(BIF_P, BIF_ARG_1, BIF_P, &locks,
-				      am_noconnection, NIL, NULL, 0);
-		erts_proc_unlock(BIF_P, locks & ERTS_PROC_LOCKS_ALL_MINOR);
-
-		/*
-		 * Copy-paste from old dist_exit_3, not sure if we really
-		 * need erts_handle_pending_exit when exit_2 does not.
-		 */
-		state = erts_atomic32_read_acqb(&BIF_P->state);
-		if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
-		    if (state & ERTS_PSFLG_PENDING_EXIT)
-			erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
-		    ERTS_BIF_EXITED(BIF_P);
-		}
-		BIF_RET(am_true);
-	    }
-            case ERTS_DSIG_PREP_PENDING:
-	    case ERTS_DSIG_PREP_CONNECTED:
-                /*
-                 * We have (pending) connection.
-                 * Setup link and enqueue link signal.
-                 */
-		erts_de_links_lock(dep);
-
-		erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, BIF_ARG_1);
-		lnk = erts_add_or_lookup_link(&(dep->nlinks),
-					      LINK_PID,
-					      BIF_P->common.id);
-		ASSERT(lnk != NULL);
-		erts_add_link(&ERTS_LINK_ROOT(lnk), LINK_PID, BIF_ARG_1);
-
-		erts_de_links_unlock(dep);
-		erts_de_runlock(dep);
-		erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
-
-		code = erts_dsig_send_link(&dsd, BIF_P->common.id, BIF_ARG_1);
-		if (code == ERTS_DSIG_SEND_YIELD)
-		    ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
-		BIF_RET(am_true);
-	    default:
-		ERTS_ASSERT(! "Invalid dsig prepare result");
-	    }
-	}
+        lnk = erts_link_tree_lookup_create(&ERTS_P_LINKS(BIF_P),
+                                           &created,
+                                           ERTS_LNK_TYPE_DIST_PROC,
+                                           BIF_P->common.id,
+                                           BIF_ARG_1);
+
+        if (!created)
+            BIF_RET(am_true); /* Already present... */
+
+        ldp = erts_link_to_data(lnk);
+
+        code = erts_dsig_prepare(&dsd, dep, BIF_P,
+                                 ERTS_PROC_LOCK_MAIN,
+                                 ERTS_DSP_RLOCK, 0, 1);
+        switch (code) {
+        case ERTS_DSIG_PREP_NOT_ALIVE:
+        case ERTS_DSIG_PREP_NOT_CONNECTED:
+            erts_link_set_dead_dist(&ldp->b, dep->sysname);
+            erts_proc_sig_send_link_exit(NULL, BIF_ARG_1, &ldp->b,
+                                         am_noconnection, NIL);
+            BIF_RET(am_true);
+
+        case ERTS_DSIG_PREP_PENDING:
+        case ERTS_DSIG_PREP_CONNECTED: {
+            /*
+             * We have (pending) connection.
+             * Setup link and enqueue link signal.
+             */
+#ifdef DEBUG
+            int inserted =
+#endif
+                erts_link_dist_insert(&ldp->b, dep->mld);
+            ASSERT(inserted);
+            erts_de_runlock(dep);
+
+            code = erts_dsig_send_link(&dsd, BIF_P->common.id, BIF_ARG_1);
+            if (code == ERTS_DSIG_SEND_YIELD)
+                ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
+            BIF_RET(am_true);
+            break;
+        }
+        default:
+            ERTS_ASSERT(! "Invalid dsig prepare result");
+        }
     }
 
     BIF_ERROR(BIF_P, BADARG);
 
-res_no_proc: {
-	erts_aint32_t state = erts_atomic32_read_nob(&BIF_P->state);
-	if (state & ERTS_PSFLG_TRAP_EXIT) {
-	    ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN;
-	    erts_deliver_exit_message(BIF_ARG_1, BIF_P, &locks, am_noproc, NIL);
-	    erts_proc_unlock(BIF_P, ~ERTS_PROC_LOCK_MAIN & locks);
-	    BIF_RET(am_true);
-	}
-	else
-	    BIF_ERROR(BIF_P, EXC_NOPROC);
+res_no_proc:
+    if (BIF_P->flags & F_TRAP_EXIT) {
+        ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN;
+        erts_deliver_exit_message(BIF_ARG_1, BIF_P, &locks, am_noproc, NIL);
+        erts_proc_unlock(BIF_P, ~ERTS_PROC_LOCK_MAIN & locks);
+        BIF_RET(am_true);
+    }
+    else {
+        /*
+         * This behaviour is *really* sad but link/1 has
+         * behaved like this for ages (and this behaviour is
+         * actually documented)... :'-(
+         *
+         * The proper behavior would have been to
+         * send calling process an exit signal..
+         */
+        BIF_ERROR(BIF_P, EXC_NOPROC);
     }
 }
 
-/* This function is allowed to return range of values handled by demonitor/1-2
- * Namely: atoms true, false, yield, internal_error, badarg or THE_NON_VALUE
- */
 static Eterm
-remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to)
+demonitor(Process *c_p, Eterm ref, Eterm *multip)
 {
-    ErtsDSigData dsd;
-    ErtsMonitor *dmon;
-    ErtsMonitor *mon;
-    int code;
-    Eterm res = am_false;
-
-    ERTS_LC_ASSERT((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK)
-		       == erts_proc_lc_my_proc_locks(c_p));
+    ErtsMonitor  *mon;  /* The monitor entry to delete */
 
-    code = erts_dsig_prepare(&dsd, dep, c_p, ERTS_PROC_LOCK_MAIN,
-			     ERTS_DSP_RLOCK, 0, 0);
-    switch (code) {
-    case ERTS_DSIG_PREP_NOT_ALIVE:
-    case ERTS_DSIG_PREP_NOT_CONNECTED:
-	/*
-	 * In the smp case this is possible if the node goes
-	 * down just before the call to demonitor.
-	 */
-	if (dep) {
-	    erts_de_links_lock(dep);
-	    dmon = erts_remove_monitor(&dep->monitors, ref);
-	    erts_de_links_unlock(dep);
-	    if (dmon)
-		erts_destroy_monitor(dmon);
-	}
-	mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref);
-	erts_proc_unlock(c_p, ERTS_PROC_LOCK_LINK);
-
-        res = am_true;
-	break;
-
-    case ERTS_DSIG_PREP_PENDING:
-    case ERTS_DSIG_PREP_CONNECTED:
-
-	erts_de_links_lock(dep);
-	mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref);
-	dmon = erts_remove_monitor(&dep->monitors, ref);
-	erts_de_links_unlock(dep);
-	erts_de_runlock(dep);
-	erts_proc_unlock(c_p, ERTS_PROC_LOCK_LINK);
-
-	if (!dmon) {
-	    /*
-	     * This is possible when smp support is enabled.
-	     * 'DOWN' message just arrived.
-	     */
-            res = am_true;
-	}
-	else {
-	    /*
-	     * Soft (no force) send, use ->data in dist slot 
-	     * monitor list since in case of monitor name 
-	     * the atom is stored there. Yield if necessary.
-	     */
-	    code = erts_dsig_send_demonitor(&dsd,
-					    c_p->common.id, 
-					    (mon->name != NIL
-					     ? mon->name
-					     : mon->u.pid), 
-					    ref,
-					    0);
-            res = (code == ERTS_DSIG_SEND_YIELD ? am_yield : am_true);
-	    erts_destroy_monitor(dmon);
-	}
-	break;
-    default:
-	ERTS_ASSERT(! "Invalid dsig prepare result");
-    }
-
-
-    /*
-     * We aren't allowed to destroy 'mon' until now, since 'to'
-     * may refer into 'mon' (external pid).
-     */
-    ASSERT(mon); /* Since link lock wasn't released between
-		    lookup and remove */
-    erts_destroy_monitor(mon);
-
-    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
-    return res;
-}
+   *multip = am_false;
 
-static ERTS_INLINE void
-demonitor_local_process(Process *c_p, Eterm ref, Eterm to, Eterm *res)
-{
-    Process *rp = erts_pid2proc_opt(c_p,
-                           ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK,
-                           to,
-                           ERTS_PROC_LOCK_LINK,
-                           ERTS_P2P_FLG_ALLOW_OTHER_X);
-    ErtsMonitor *mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref);
+   if (is_not_internal_ref(ref)) {
+       if (is_external_ref(ref)
+           && (erts_this_dist_entry
+               == external_ref_dist_entry(ref))) {
+           return am_false;
+       }
+       return am_badarg; /* Not monitored by this monitor's ref */
+   }
 
-    if (!mon)
-        *res = am_false;
-    else
-    {
-        *res = am_true;
-        erts_destroy_monitor(mon);
-    }
-    if (rp) {
-        ErtsMonitor *rmon;
-        rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref);
-        if (rp != c_p)
-            erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-        if (rmon != NULL)
-            erts_destroy_monitor(rmon);
-    }
-    else {
-        ERTS_ASSERT_IS_NOT_EXITING(c_p);
-    }
-}
+   mon = erts_monitor_tree_lookup(ERTS_P_MONITORS(c_p), ref);
+   if (!mon)
+       return am_false;
 
-static ERTS_INLINE BIF_RETTYPE
-demonitor_local_port(Process *origin, Eterm ref, Eterm target)
-{
-    BIF_RETTYPE res = am_false;
-    Port        *port = erts_port_lookup_raw(target);
+   if (!erts_monitor_is_origin(mon))
+       return am_badarg;
 
-    if (!port) {
-        BIF_ERROR(origin, BADARG);
-    }
-    erts_proc_unlock(origin, ERTS_PROC_LOCK_LINK);
+   erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), mon);
 
-    if (port) {
-        Eterm trap_ref;
-        switch (erts_port_demonitor(origin, ERTS_PORT_DEMONITOR_NORMAL,
-                                    port, ref, &trap_ref)) {
-        case ERTS_PORT_OP_DROPPED:
-        case ERTS_PORT_OP_BADARG:
-            break;
-        case ERTS_PORT_OP_SCHEDULED:
-            BIF_TRAP3(await_port_send_result_trap, origin, trap_ref,
-                      am_busy_port, am_true);
-            /* the busy_port atom will never be returned, because it cannot be
-             * returned from erts_port_(de)monitor, but just in case if in future
-             * internal API changes - you may see this atom */
-        default:
-            break;
-        }
-    }
-    else {
-        ERTS_ASSERT_IS_NOT_EXITING(origin);
-    }
-    BIF_RET(res);
-}
+   switch (mon->type) {
 
-/* Can return atom true, false, yield, internal_error, badarg or
- * THE_NON_VALUE if error occurred or trap has been set up
- */
-static
-BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip)
-{
-   ErtsMonitor  *mon = NULL;  /* The monitor entry to delete */
-   Eterm        to = NIL;     /* Monitor link traget */
-   DistEntry    *dep = NULL;  /* Target's distribution entry */
-   BIF_RETTYPE  res = am_false;
-   int          unlock_link = 1;
+   case ERTS_MON_TYPE_TIME_OFFSET:
+       *multip = am_true;
+       erts_demonitor_time_offset(mon);
+       return am_true;
+
+   case ERTS_MON_TYPE_PORT: {
+       Port *prt;
+       ASSERT(is_internal_port(mon->other.item));
+       prt = erts_port_lookup(mon->other.item, ERTS_PORT_SFLGS_DEAD);
+       if (!prt || erts_port_demonitor(c_p, prt, mon) == ERTS_PORT_OP_DROPPED)
+           erts_monitor_release(mon);
+       return am_true;
+   }
 
-   erts_proc_lock(c_p, ERTS_PROC_LOCK_LINK);
+   case ERTS_MON_TYPE_PROC:
+       erts_proc_sig_send_demonitor(mon);
+       return am_true;
 
-   if (is_not_internal_ref(ref)) {
-       res = am_badarg;
-       goto done; /* Cannot be this monitor's ref */
-   }
+   case ERTS_MON_TYPE_DIST_PROC: {
+       ErtsMonitorData *mdp = erts_monitor_to_data(mon);
+       Eterm to = mon->other.item;
+       DistEntry *dep;
+       int code = ERTS_DSIG_SEND_OK;
+       int deleted;
+       ErtsDSigData dsd;
 
-   mon = erts_lookup_monitor(ERTS_P_MONITORS(c_p), ref);
-   if (!mon) {
-       goto done;
-   }
+       ASSERT(is_external_pid(to) || is_node_name_atom(to));
 
-   switch (mon->type) {
-   case MON_TIME_OFFSET:
-       *multip = am_true;
-       erts_demonitor_time_offset(ref);
-       res = am_true;
-       break;
-   case MON_ORIGIN:
-       to = mon->u.pid;
-       *multip = am_false;
-       if (is_atom(to)) {
+       if (is_external_pid(to))
+           dep = external_pid_dist_entry(to);
+       else {
            /* Monitoring a name at node to */
-           ASSERT(is_node_name_atom(to));
            dep = erts_sysname_to_connected_dist_entry(to);
            ASSERT(dep != erts_this_dist_entry);
-       } else if (is_port(to)) {
-           if (port_dist_entry(to) != erts_this_dist_entry) {
-               goto badarg;
+           if (!dep) {
+               erts_monitor_release(mon);
+               return am_false;
            }
-           res = demonitor_local_port(c_p, ref, to);
-           unlock_link = 0;
-           goto done;
-       } else {
-           ASSERT(is_pid(to));
-           dep = pid_dist_entry(to);
        }
-       if (dep != erts_this_dist_entry) {
-           res = remote_demonitor(c_p, dep, ref, to);
-           /* remote_demonitor() unlocks link lock on c_p */
-           unlock_link = 0;
+
+       code = erts_dsig_prepare(&dsd, dep, c_p, ERTS_PROC_LOCK_MAIN,
+                                ERTS_DSP_RLOCK, 0, 0);
+
+       deleted = erts_monitor_dist_delete(&mdp->target);
+
+       switch (code) {
+       case ERTS_DSIG_PREP_NOT_ALIVE:
+       case ERTS_DSIG_PREP_NOT_CONNECTED:
+           /*
+            * In the smp case this is possible if the node goes
+            * down just before the call to demonitor.
+            */
+           break;
+
+       case ERTS_DSIG_PREP_PENDING:
+       case ERTS_DSIG_PREP_CONNECTED: {
+           Eterm watched;
+
+           erts_de_runlock(dep);
+
+           if (mon->flags & ERTS_ML_FLG_NAME)
+               watched = ((ErtsMonitorDataExtended *) mdp)->u.name;
+           else
+               watched = to;
+
+           /*
+            * Soft (no force) send, use ->data in dist slot 
+            * monitor list since in case of monitor name 
+            * the atom is stored there. Yield if necessary.
+            */
+           code = erts_dsig_send_demonitor(&dsd, c_p->common.id,
+                                           watched, mdp->ref, 0);
+           break;
        }
-       else { /* Local monitor */
-           demonitor_local_process(c_p, ref, to, &res);
+
+       default:
+           ERTS_INTERNAL_ERROR("invalid result from erts_dsig_prepare()");
+           break;
        }
-       break;
-    default /* case */ :
-badarg:
-       res = am_badarg; /* will be converted to error by caller */
-       *multip = am_false;
-       break;
-   }
-done:
 
-   if (unlock_link)
-       erts_proc_unlock(c_p, ERTS_PROC_LOCK_LINK);
+       if (deleted)
+           erts_monitor_release(&mdp->target);
 
-   ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
-   BIF_RET(res);
+       erts_monitor_release(mon);
+       return code == ERTS_DSIG_SEND_YIELD ? am_yield : am_true;
+   }
+
+   default:
+       ERTS_INTERNAL_ERROR("Unexpected monitor type");
+       return am_false;
+   }
 }
 
 BIF_RETTYPE demonitor_1(BIF_ALIST_1)
@@ -528,25 +384,23 @@ BIF_RETTYPE demonitor_1(BIF_ALIST_1)
     Eterm multi;
     switch (demonitor(BIF_P, BIF_ARG_1, &multi)) {
     case am_false:
-    case am_true:       BIF_RET(am_true);
-    case THE_NON_VALUE: BIF_RET(THE_NON_VALUE);
-    case am_yield:      ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
-    case am_badarg:     BIF_ERROR(BIF_P, BADARG);
-
-    case am_internal_error:
+    case am_true:
+        BIF_RET(am_true);
+    case am_yield:
+        ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
+    case am_badarg:
     default:
-	ASSERT(! "demonitor(): internal error");
-	BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
+        BIF_ERROR(BIF_P, BADARG);
     }
 }
 
 BIF_RETTYPE demonitor_2(BIF_ALIST_2)
 {
-    BIF_RETTYPE res = am_true;
-    Eterm       multi = am_false;
-    int         info = 0;
-    int         flush = 0;
-    Eterm       list = BIF_ARG_2;
+    BIF_RETTYPE res;
+    Eterm multi = am_false;
+    int info = 0;
+    int flush = 0;
+    Eterm list = BIF_ARG_2;
 
     while (is_list(list)) {
 	Eterm* consp = list_val(list);
@@ -566,10 +420,9 @@ BIF_RETTYPE demonitor_2(BIF_ALIST_2)
     if (is_not_nil(list))
 	goto badarg;
 
+    res = am_true;
     switch (demonitor(BIF_P, BIF_ARG_1, &multi)) {
-    case THE_NON_VALUE:
-        /* If other error occurred or trap has been set up - pass through */
-        BIF_RET(THE_NON_VALUE);
+
     case am_false:
 	if (info)
 	    res = am_false;
@@ -578,20 +431,29 @@ flush_messages:
 	    BIF_TRAP3(flush_monitor_messages_trap, BIF_P,
 		      BIF_ARG_1, multi, res);
 	}
+        /* Fall through... */
+
     case am_true:
 	if (multi == am_true && flush)
 	    goto flush_messages;
 	BIF_RET(res);
+
     case am_yield:
-	ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
+        /* return true after yield... */
+        if (flush) {
+            ERTS_VBUMP_ALL_REDS(BIF_P);
+            goto flush_messages;
+        }
+        ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
+
     case am_badarg:
-badarg:
-	BIF_ERROR(BIF_P, BADARG);
-    case am_internal_error:
     default:
-	ASSERT(! "demonitor(): internal error");
-	BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
+        break;
+
     }
+
+badarg:
+    BIF_ERROR(BIF_P, BADARG);
 }
 
 /* Type must be atomic object! */
@@ -631,292 +493,212 @@ erts_queue_monitor_message(Process *p,
     erts_queue_message(p, *p_locksp, msgp, tup, am_system);
 }
 
-static Eterm
-local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int boolean)
+BIF_RETTYPE monitor_2(BIF_ALIST_2)
 {
-    Eterm         ret = mon_ref;
-    Process      *rp;
-    ErtsProcLocks p_locks = ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK;
-
-    if (target == p->common.id) {
-	return ret;
-    }
+    Eterm target = BIF_ARG_2;
+    Eterm tmp_heap[3];
+    Eterm ref, id, name;
+    ErtsMonitorData *mdp;
+
+    if (BIF_ARG_1 == am_process) {
+        DistEntry *dep;
+        int byname;
+
+        if (is_internal_pid(target)) {
+            name = NIL;
+            id = target;
+
+        local_process:
+
+            ref = erts_make_ref(BIF_P);
+            if (id != BIF_P->common.id) {
+                mdp = erts_monitor_create(ERTS_MON_TYPE_PROC,
+                                          ref, BIF_P->common.id,
+                                          id, name);
+                erts_monitor_tree_insert(&ERTS_P_MONITORS(BIF_P),
+                                         &mdp->origin);
+
+                if (!erts_proc_sig_send_monitor(&mdp->target, id))
+                    erts_proc_sig_send_monitor_down(&mdp->target,
+                                                    am_noproc);
+            }
+            BIF_RET(ref);
+        }
 
-    erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
-    rp = erts_pid2proc_opt(p, p_locks,
-			   target, ERTS_PROC_LOCK_LINK,
-			   ERTS_P2P_FLG_ALLOW_OTHER_X);
-    if (!rp) {
-	erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
-	p_locks &= ~ERTS_PROC_LOCK_LINK;
-	if (boolean)
-	    ret = am_false;
-	else
-	    erts_queue_monitor_message(p, &p_locks,
-				       mon_ref, am_process, target, am_noproc);
-    }
-    else {
-	ASSERT(rp != p);
+        if (is_atom(target)) {
+        local_named_process:
+            name = target;
+            id = erts_whereis_name_to_id(BIF_P, target);
+            if (is_internal_pid(id))
+                goto local_process;
+            target = TUPLE2(&tmp_heap[0], name,
+                            erts_this_dist_entry->sysname);
+            goto noproc;
+        }
 
-	if (boolean)
-	    ret = am_true;
+        if (is_external_pid(target)) {
+            ErtsDSigData dsd;
+            int code;
+
+            dep = external_pid_dist_entry(target);
+            if (dep == erts_this_dist_entry)
+                goto noproc;
+
+            id = target;
+            name = NIL;
+            byname = 0;
+
+        remote_process:
+
+            ref = erts_make_ref(BIF_P);
+            mdp = erts_monitor_create(ERTS_MON_TYPE_DIST_PROC, ref,
+                                      BIF_P->common.id, id, name);
+            erts_monitor_tree_insert(&ERTS_P_MONITORS(BIF_P), &mdp->origin);
+
+            code = erts_dsig_prepare(&dsd, dep,
+                                     BIF_P, ERTS_PROC_LOCK_MAIN,
+                                     ERTS_DSP_RLOCK, 0, 1);
+            switch (code) {
+            case ERTS_DSIG_PREP_NOT_ALIVE:
+            case ERTS_DSIG_PREP_NOT_CONNECTED:
+                erts_monitor_set_dead_dist(&mdp->target, dep->sysname);
+                erts_proc_sig_send_monitor_down(&mdp->target, am_noconnection);
+                code = ERTS_DSIG_SEND_OK;
+                break;
 
-	erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, target, NIL);
-	erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, mon_ref, p->common.id, NIL);
+            case ERTS_DSIG_PREP_PENDING:
+            case ERTS_DSIG_PREP_CONNECTED: {
+#ifdef DEBUG
+                int inserted =
+#endif
 
-	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-    }
+                erts_monitor_dist_insert(&mdp->target, dep->mld);
+                ASSERT(inserted);
+                erts_de_runlock(dep);
 
-    erts_proc_unlock(p, p_locks & ~ERTS_PROC_LOCK_MAIN);
+                code = erts_dsig_send_monitor(&dsd, BIF_P->common.id, target, ref);
+                break;
+            }
 
-    return ret;
-}
+            default:
+                ERTS_ASSERT(! "Invalid dsig prepare result");
+                code = ERTS_DSIG_SEND_OK;
+                break;
+            }
 
-static BIF_RETTYPE
-local_port_monitor(Process *origin, Eterm target)
-{
-    BIF_RETTYPE ref = erts_make_ref(origin);
-    Port *port = erts_sig_lookup_port(origin, target);
-    ErtsProcLocks p_locks = ERTS_PROC_LOCK_MAIN;
+            if (byname)
+                erts_deref_dist_entry(dep);
 
-    if (!port) {
-res_no_proc:
-        /* Send the DOWN message immediately. Ref is made on the fly because
-         * caller has never seen it yet. */
-        erts_queue_monitor_message(origin, &p_locks, ref,
-                                   am_port, target, am_noproc);
-    }
-    else {
-        switch (erts_port_monitor(origin, port, target, &ref)) {
-        case ERTS_PORT_OP_DROPPED:
-        case ERTS_PORT_OP_BADARG:
-            goto res_no_proc;
-        case ERTS_PORT_OP_SCHEDULED:
-            BIF_TRAP3(await_port_send_result_trap, origin, ref,
-                      am_busy_port, ref);
-            /* the busy_port atom will never be returned, because it cannot be
-             * returned from erts_port_monitor, but just in case if in future
-             * internal API changes - you may see this atom */
-        default:
-            break;
+            if (code == ERTS_DSIG_SEND_YIELD)
+                ERTS_BIF_YIELD_RETURN(BIF_P, ref);
+            BIF_RET(ref);
         }
-    }
-    erts_proc_unlock(origin, p_locks & ~ERTS_PROC_LOCK_MAIN);
-    BIF_RET(ref);
-}
 
-/* Type = process | port :: atom(), 1st argument passed to erlang:monitor/2
- */
-static BIF_RETTYPE
-local_name_monitor(Process *self, Eterm type, Eterm target_name)
-{
-    BIF_RETTYPE ret = erts_make_ref(self);
+        if (is_tuple(target)) {
+            Eterm *tpl = tuple_val(target);
+            if (arityval(tpl[0]) != 2)
+                goto badarg;
+            if (is_not_atom(tpl[1]) || is_not_atom(tpl[2]))
+                goto badarg;
+            if (!erts_is_alive && tpl[2] != am_Noname)
+                goto badarg;
+            target = tpl[1];
+            dep = erts_find_or_insert_dist_entry(tpl[2]);
+            if (dep == erts_this_dist_entry) {
+                erts_deref_dist_entry(dep);
+                goto local_named_process;
+            }
 
-    ErtsProcLocks p_locks = ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK;
-    Process     *proc = NULL;
-    Port        *port = NULL;
+            id = dep->sysname;
+            name = target;
+            byname = 1;
+            goto remote_process;
+        }
 
-    erts_proc_lock(self, ERTS_PROC_LOCK_LINK);
+        /* badarg... */
+    }
+    else if (BIF_ARG_1 == am_port) {
+
+        if (is_internal_port(target)) {
+            Port *prt;
+            name = NIL;
+            id = target;
+        local_port:
+            ref = erts_make_ref(BIF_P);
+            mdp = erts_monitor_create(ERTS_MON_TYPE_PORT, ref,
+                                      BIF_P->common.id, id, name);
+            erts_monitor_tree_insert(&ERTS_P_MONITORS(BIF_P), &mdp->origin);
+            prt = erts_port_lookup(id, ERTS_PORT_SFLGS_INVALID_LOOKUP);
+            if (!prt || erts_port_monitor(BIF_P, prt, &mdp->target) == ERTS_PORT_OP_DROPPED)
+                erts_proc_sig_send_monitor_down(&mdp->target, am_noproc);
+            BIF_RET(ref);
+        }
 
-    erts_whereis_name(self, p_locks, target_name,
-                      &proc, ERTS_PROC_LOCK_LINK,
-                      ERTS_P2P_FLG_ALLOW_OTHER_X,
-                      &port, 0);
+        if (is_atom(target)) {
+        local_named_port:
+            name = target;
+            id = erts_whereis_name_to_id(BIF_P, target);
+            if (is_internal_port(id))
+                goto local_port;
+            target = TUPLE2(&tmp_heap[0], name,
+                            erts_this_dist_entry->sysname);
+            goto noproc;
+        }
 
-    /* If the name is not registered,
-     * or if we asked for proc and got a port,
-     * or if we asked for port and got a proc,
-     * we just send the 'DOWN' message.
-     */
-    if ((!proc && !port) ||
-        (type == am_process && port) ||
-        (type == am_port && proc)) {
-        DeclareTmpHeap(lhp,3,self);
-	Eterm item;
-        UseTmpHeap(3,self);
-
-        erts_proc_unlock(self, ERTS_PROC_LOCK_LINK);
-	p_locks &= ~ERTS_PROC_LOCK_LINK;
-
-	item = TUPLE2(lhp, target_name, erts_this_dist_entry->sysname);
-        erts_queue_monitor_message(self, &p_locks,
-                                   ret,
-                                   type, /* = process|port :: atom() */
-                                   item, am_noproc);
-        UnUseTmpHeap(3,self);
-    }
-    else if (port) {
-        erts_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN);
-        p_locks &= ~ERTS_PROC_LOCK_MAIN;
-
-        switch (erts_port_monitor(self, port, target_name, &ret)) {
-        case ERTS_PORT_OP_DONE:
-            return ret;
-        case ERTS_PORT_OP_SCHEDULED: { /* Scheduled a signal */
-            ASSERT(is_internal_ordinary_ref(ret));
-            BIF_TRAP3(await_port_send_result_trap, self,
-                      ret, am_true, ret);
-            /* bif_trap returns */
-        } break;
-        default:
+        if (is_external_port(target)) {
+            if (erts_this_dist_entry == external_port_dist_entry(target))
+                goto noproc;
             goto badarg;
         }
-    }
-    else if (proc != self) {
-        erts_add_monitor(&ERTS_P_MONITORS(self), MON_ORIGIN, ret,
-                         proc->common.id, target_name);
-        erts_add_monitor(&ERTS_P_MONITORS(proc), MON_TARGET, ret,
-                         self->common.id, target_name);
-        erts_proc_unlock(proc, ERTS_PROC_LOCK_LINK);
-    }
-
-    if (p_locks) {
-        erts_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN);
-    }
-    BIF_RET(ret);
-badarg:
-    if (p_locks) {
-        erts_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN);
-    }
-    BIF_ERROR(self, BADARG);
-}
-
-static BIF_RETTYPE
-remote_monitor(Process *p, Eterm bifarg1, Eterm bifarg2,
-	       DistEntry *dep, Eterm target, int byname)
-{
-    ErtsDSigData dsd;
-    BIF_RETTYPE ret;
-    int code;
 
-    ASSERT(dep);
-    erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
-    code = erts_dsig_prepare(&dsd, dep,
-			     p, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK),
-			     ERTS_DSP_RLOCK, 0, 1);
-    switch (code) {
-    case ERTS_DSIG_PREP_NOT_ALIVE:
-    case ERTS_DSIG_PREP_NOT_CONNECTED:
-	erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
-	ERTS_BIF_PREP_TRAP2(ret, dmonitor_p_trap, p, bifarg1, bifarg2);
-	break;
-    case ERTS_DSIG_PREP_PENDING:
-    case ERTS_DSIG_PREP_CONNECTED:
-	{
-	    Eterm p_trgt, p_name, d_name, mon_ref;
+        if (is_tuple(target)) {
+            Eterm *tpl = tuple_val(target);
+            if (arityval(tpl[0]) != 2)
+                goto badarg;
+            if (is_not_atom(tpl[1]) || is_not_atom(tpl[2]))
+                goto badarg;
+            if (tpl[2] == erts_this_dist_entry->sysname) {
+                target = tpl[1];
+                goto local_named_port;
+            }
+        }
 
-	    mon_ref = erts_make_ref(p);
+        /* badarg... */
+    }
+    else if (BIF_ARG_1 == am_time_offset) {
 
-	    if (byname) {
-		p_trgt = dep->sysname;
-		p_name = target;
-		d_name = target;
-	    }
-	    else {
-		p_trgt = target;
-		p_name = NIL;
-		d_name = NIL;
-	    }
+        if (target != am_clock_service)
+            goto badarg;
+	ref = erts_make_ref(BIF_P);
+        mdp = erts_monitor_create(ERTS_MON_TYPE_TIME_OFFSET,
+                                  ref, BIF_P->common.id,
+                                  am_clock_service, NIL);
+        erts_monitor_tree_insert(&ERTS_P_MONITORS(BIF_P), &mdp->origin);
 
-	    erts_de_links_lock(dep);
+	erts_monitor_time_offset(&mdp->target);
 
-	    erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, p_trgt,
-			     p_name);
-	    erts_add_monitor(&(dep->monitors), MON_TARGET, mon_ref, p->common.id,
-			     d_name);
+        BIF_RET(ref);
+    }
 
-	    erts_de_links_unlock(dep);
-	    erts_de_runlock(dep);
-	    erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+badarg:
 
-	    code = erts_dsig_send_monitor(&dsd, p->common.id, target, mon_ref);
-	    if (code == ERTS_DSIG_SEND_YIELD)
-		ERTS_BIF_PREP_YIELD_RETURN(ret, p, mon_ref);
-	    else
-		ERTS_BIF_PREP_RET(ret, mon_ref);
-	}
-	break;
-    default:
-	ERTS_ASSERT(! "Invalid dsig prepare result");
-    }
+    BIF_ERROR(BIF_P, BADARG);
 
-    BIF_RET(ret);
-}
-	
-BIF_RETTYPE monitor_2(BIF_ALIST_2)
-{
-    Eterm target = BIF_ARG_2;
-    BIF_RETTYPE ret;
-    DistEntry  *dep = NULL; 
+noproc: {
+        ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN;
 
-    /* Only process monitors are implemented */
-    switch (BIF_ARG_1) {
-    case am_time_offset: {
-	Eterm ref;
-        if (BIF_ARG_2 != am_clock_service) {
-            goto badarg;
-        }
-	ref = erts_make_ref(BIF_P);
-	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
-	erts_add_monitor(&ERTS_P_MONITORS(BIF_P), MON_TIME_OFFSET,
-			 ref, am_clock_service, NIL);
-	erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
-	erts_monitor_time_offset(BIF_P->common.id, ref);
-	BIF_RET(ref);
-    }
-    case am_process:
-    case am_port:
-	break;
-    default:
-        goto badarg;
-    }
+        ref = erts_make_ref(BIF_P);
+        erts_queue_monitor_message(BIF_P,
+                                   &locks,
+                                   ref,
+                                   BIF_ARG_1,
+                                   target,
+                                   am_noproc);
+        if (locks != ERTS_PROC_LOCK_MAIN)
+            erts_proc_unlock(BIF_P, locks & ~ERTS_PROC_LOCK_MAIN);
 
-    if (is_internal_pid(target) && BIF_ARG_1 == am_process) {
-local_pid:
-        ret = local_pid_monitor(BIF_P, target, erts_make_ref(BIF_P), 0);
-    } else if (is_external_pid(target) && BIF_ARG_1 == am_process) {
-	dep = external_pid_dist_entry(target);
-	if (dep == erts_this_dist_entry)
-	    goto local_pid;
-	ret = remote_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2, dep, target, 0);
-    } else if (is_internal_port(target) && BIF_ARG_1 == am_port) {
-local_port:
-        ret = local_port_monitor(BIF_P, target);
-    } else if (is_external_port(target) && BIF_ARG_1 == am_port) {
-        dep = external_port_dist_entry(target);
-        if (dep == erts_this_dist_entry) {
-            goto local_port;
-        }
-        goto badarg; /* No want remote port */
-    } else if (is_atom(target)) {
-        ret = local_name_monitor(BIF_P, BIF_ARG_1, target);
-    } else if (is_tuple(target)) {
-	Eterm *tp = tuple_val(target);
-	Eterm remote_node;
-	Eterm name;
-        if (arityval(*tp) != 2) {
-            goto badarg;
-        }
-	remote_node = tp[2];
-	name = tp[1];
-	if (!is_atom(remote_node) || !is_atom(name)) {
-            goto badarg;
-	}
-	if (!erts_is_alive && remote_node != am_Noname) {
-            goto badarg; /* Remote monitor from (this) undistributed node */
-	}
-	dep = erts_find_or_insert_dist_entry(remote_node);
-	if (dep == erts_this_dist_entry) {
-            ret = local_name_monitor(BIF_P, BIF_ARG_1, name);
-	} else {
-	    ret = remote_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2, dep, name, 1);
-	}
-	erts_deref_dist_entry(dep);
-    } else {
-badarg:
-	ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
+        BIF_RET(ref);
     }
-    return ret;
 }
 
 /**********************************************************************/
@@ -1089,91 +871,74 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1)
 /* remove a link from a process */
 BIF_RETTYPE unlink_1(BIF_ALIST_1)
 {
-    Process *rp;
-    DistEntry *dep;
-    ErtsLink *l = NULL, *rl = NULL;
-    ErtsProcLocks cp_locks = ERTS_PROC_LOCK_MAIN;
-
-    /*
-     * SMP specific note concerning incoming exit signals:
-     *   We have to have at least the status lock during removal of
-     *   the link half on current process, and check for and handle
-     *   a present pending exit while the status lock is held. This
-     *   in order to ensure that we wont be exited by a link after
-     *   it has been removed.
-     *
-     *   (We also have to have the link lock, of course, in order to
-     *    be allowed to remove the link...)
-     */
-
     if (IS_TRACED_FL(BIF_P, F_TRACE_PROCS)) {
-	trace_proc(BIF_P, cp_locks, BIF_P, am_unlink, BIF_ARG_1);
+        trace_proc(BIF_P, ERTS_PROC_LOCK_MAIN,
+                   BIF_P, am_unlink, BIF_ARG_1);
     }
 
-    if (is_internal_port(BIF_ARG_1)) {
-	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
-	if (ERTS_PROC_PENDING_EXIT(BIF_P))
-	    goto handle_pending_exit;
-
-	l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1);
+    if (is_internal_pid(BIF_ARG_1)) {
+        ErtsLink *lnk = erts_link_tree_lookup(ERTS_P_LINKS(BIF_P), BIF_ARG_1);
+        if (lnk) {
+            erts_link_tree_delete(&ERTS_P_LINKS(BIF_P), lnk);
+            erts_proc_sig_send_unlink(BIF_P, lnk);
+        }
+        BIF_RET(am_true);
+    }
 
-	erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
+    if (is_internal_port(BIF_ARG_1)) {
+        ErtsLink *lnk = erts_link_tree_lookup(ERTS_P_LINKS(BIF_P), BIF_ARG_1);
 
-	if (l) {
+	if (lnk) {
+            Eterm ref;
+            Eterm *refp = erts_port_synchronous_ops ? &ref : NULL;
+            ErtsPortOpResult res = ERTS_PORT_OP_DROPPED;
 	    Port *prt;
 
-	    erts_destroy_link(l);
+            erts_link_tree_delete(&ERTS_P_LINKS(BIF_P), lnk);
 
 	    /* Send unlink signal */
 	    prt = erts_port_lookup(BIF_ARG_1, ERTS_PORT_SFLGS_DEAD);
 	    if (prt) {
-		ErtsPortOpResult res;
-		Eterm ref;
-		Eterm *refp = erts_port_synchronous_ops ? &ref : NULL;
 #ifdef DEBUG
 		ref = NIL;
 #endif
-		res = erts_port_unlink(BIF_P, prt, BIF_P->common.id, refp);
+		res = erts_port_unlink(BIF_P, prt, lnk, refp);
 
-		if (refp && res == ERTS_PORT_OP_SCHEDULED) {
-		    ASSERT(is_internal_ordinary_ref(ref));
-		    BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true);
-		}
 	    }
+
+            if (res == ERTS_PORT_OP_DROPPED)
+                erts_link_release(lnk);
+            else if (refp && res == ERTS_PORT_OP_SCHEDULED) {
+                ASSERT(is_internal_ordinary_ref(ref));
+                BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true);
+            }
 	}
 
 	BIF_RET(am_true);
     }
-    else if (is_external_port(BIF_ARG_1)
-	     && external_port_dist_entry(BIF_ARG_1) == erts_this_dist_entry) {
-	BIF_RET(am_true);
-    }
-
-    if (is_not_pid(BIF_ARG_1))
-	BIF_ERROR(BIF_P, BADARG);
 
     if (is_external_pid(BIF_ARG_1)) {
-	ErtsDistLinkData dld;
+        ErtsLink *lnk, *dlnk;
+        ErtsLinkData *ldp;
+        DistEntry *dep;
 	int code;
 	ErtsDSigData dsd;
-	/* Blind removal, we might have trapped or anything, this leaves
-	   us in a state where monitors might be inconsistent, but the dist
-	   code should take care of it. */
-	erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
-	if (ERTS_PROC_PENDING_EXIT(BIF_P))
-	    goto handle_pending_exit;
-	l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1);
-
-	erts_proc_unlock(BIF_P,
-			     ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
-
-	if (l)
-	    erts_destroy_link(l);
 
 	dep = external_pid_dist_entry(BIF_ARG_1);
-	if (dep == erts_this_dist_entry) {
+	if (dep == erts_this_dist_entry)
 	    BIF_RET(am_true);
-	}
+
+        lnk = erts_link_tree_lookup(ERTS_P_LINKS(BIF_P), BIF_ARG_1);
+        if (!lnk)
+            BIF_RET(am_true);
+
+        erts_link_tree_delete(&ERTS_P_LINKS(BIF_P), lnk);
+        dlnk = erts_link_to_other(lnk, &ldp);
+
+        if (erts_link_dist_delete(dlnk))
+            erts_link_release_both(ldp);
+        else
+            erts_link_release(lnk);
 
 	code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_PROC_LOCK_MAIN,
 				 ERTS_DSP_NO_LOCK, 0, 0);
@@ -1181,74 +946,27 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 	case ERTS_DSIG_PREP_NOT_ALIVE:
 	case ERTS_DSIG_PREP_NOT_CONNECTED:
 	    BIF_RET(am_true);
-
 	case ERTS_DSIG_PREP_PENDING:
 	case ERTS_DSIG_PREP_CONNECTED:
-	    erts_remove_dist_link(&dld, BIF_P->common.id, BIF_ARG_1, dep);
 	    code = erts_dsig_send_unlink(&dsd, BIF_P->common.id, BIF_ARG_1);
-	    erts_destroy_dist_link(&dld);
 	    if (code == ERTS_DSIG_SEND_YIELD)
 		ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
-	    BIF_RET(am_true);
-
+            break;
 	default:
 	    ASSERT(! "Invalid dsig prepare result");
 	    BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR);
 	}
-    }
-
-    /* Internal pid... */
-
-    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
-
-    cp_locks |= ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS;
-
-    /* get process struct */
-    rp = erts_pid2proc_opt(BIF_P, cp_locks,
-			   BIF_ARG_1, ERTS_PROC_LOCK_LINK,
-			   ERTS_P2P_FLG_ALLOW_OTHER_X);
-
-    if (ERTS_PROC_PENDING_EXIT(BIF_P)) {
-	if (rp && rp != BIF_P)
-	    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-	goto handle_pending_exit;
-    }
-
-    /* unlink and ignore errors */
-    l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1);
-    if (l != NULL)
-	erts_destroy_link(l);
 
-    if (!rp) {
-	ERTS_ASSERT_IS_NOT_EXITING(BIF_P);
+        BIF_RET(am_true);
     }
-    else {
-	rl = erts_remove_link(&ERTS_P_LINKS(rp), BIF_P->common.id);
-	if (rl != NULL)
-	    erts_destroy_link(rl);
-
-	if (IS_TRACED_FL(rp, F_TRACE_PROCS) && rl != NULL) {
-            erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS);
-            cp_locks &= ~ERTS_PROC_LOCK_STATUS;
-	    trace_proc(BIF_P, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK),
-                       rp, am_getting_unlinked, BIF_P->common.id);
-	}
 
-	if (rp != BIF_P)
-	    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+    if (is_external_port(BIF_ARG_1)) {
+        if (external_port_dist_entry(BIF_ARG_1) == erts_this_dist_entry)
+            BIF_RET(am_true);
+        /* Links to Remote ports not supported... */
     }
- 
-    erts_proc_unlock(BIF_P, cp_locks & ~ERTS_PROC_LOCK_MAIN);
 
-    BIF_RET(am_true);
-
- handle_pending_exit:
-    erts_handle_pending_exit(BIF_P, (ERTS_PROC_LOCK_MAIN
-				     | ERTS_PROC_LOCK_LINK
-				     | ERTS_PROC_LOCK_STATUS));
-    ASSERT(ERTS_PROC_IS_EXITING(BIF_P)); 
-    erts_proc_unlock(BIF_P,	ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
-    ERTS_BIF_EXITED(BIF_P);
+    BIF_ERROR(BIF_P, BADARG);
 }
 
 BIF_RETTYPE hibernate_3(BIF_ALIST_3)
@@ -1493,22 +1211,69 @@ BIF_RETTYPE raise_3(BIF_ALIST_3)
     return am_badarg;
 }
 
+static BIF_RETTYPE
+erts_internal_await_exit_trap(BIF_ALIST_0)
+{
+    /*
+     * We have sent ourselves an exit signal which will
+     * terminate ourselves. Handle all signals until
+     * terminated in order to ensure that signal order
+     * is preserved. Yield if necessary.
+     */
+    erts_aint32_t state;
+    int reds = ERTS_BIF_REDS_LEFT(BIF_P);
+    (void) erts_proc_sig_handle_incoming(BIF_P, &state, &reds,
+                                         reds, !0);
+    BUMP_REDS(BIF_P, reds);
+    if (state & ERTS_PSFLG_EXITING)
+        ERTS_BIF_EXITED(BIF_P);
+
+    ERTS_BIF_YIELD0(&await_exit_trap, BIF_P);
+}
+
 /**********************************************************************/
-/* send an exit message to another process (if trapping exits) or
-   exit the other process */
+/* send an exit signal to another process */
 
-BIF_RETTYPE exit_2(BIF_ALIST_2)
+static BIF_RETTYPE send_exit_signal_bif(Process *c_p, Eterm id, Eterm reason, int exit2)
 {
-     Process *rp;
+    BIF_RETTYPE ret_val;
 
-     /*
-      * If the first argument is not a pid, or a local port it is an error.
-      */
+    /*
+     * 'id' not a process id, nor a local port id is a 'badarg' error.
+     */
 
-     if (is_internal_port(BIF_ARG_1)) {
+     if (is_internal_pid(id)) {
+         /*
+          * Preserve the very old and *very strange* behaviour
+          * of erlang:exit/2...
+          *
+          * - terminate ourselves even though exit reason
+          *   is normal (unless we trap exit)
+          * - terminate ourselves before exit/2 return
+          */
+         int exit2_suicide = (exit2
+                              && c_p->common.id == id
+                              && (reason == am_kill
+                                  || !(c_p->flags & F_TRAP_EXIT)));
+         erts_proc_sig_send_exit(c_p, c_p->common.id, id,
+                                 reason, NIL, exit2_suicide);
+         if (!exit2_suicide)
+             ERTS_BIF_PREP_RET(ret_val, am_true);
+         else {
+             erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
+             erts_proc_sig_fetch(c_p);
+             erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
+             ERTS_BIF_PREP_TRAP0(ret_val, &await_exit_trap, c_p);
+         }
+     }
+     else if (is_internal_port(id)) {
 	 Eterm ref, *refp;
 	 Uint32 invalid_flags;
 	 Port *prt;
+         ErtsPortOpResult res = ERTS_PORT_OP_DONE;
+#ifdef DEBUG
+         ref = NIL;
+#endif
 
 	 if (erts_port_synchronous_ops) {
 	     refp = &ref;
@@ -1519,108 +1284,75 @@ BIF_RETTYPE exit_2(BIF_ALIST_2)
 	     invalid_flags = ERTS_PORT_SFLGS_INVALID_LOOKUP;
 	 }
 
-	 prt = erts_port_lookup(BIF_ARG_1, invalid_flags);
-
-	 if (prt) {
-	     ErtsPortOpResult res;
-
-#ifdef DEBUG
-	     ref = NIL;
-#endif
-
-	     res = erts_port_exit(BIF_P, 0, prt, BIF_P->common.id, BIF_ARG_2, refp);
-
-	     ERTS_BIF_CHK_EXITED(BIF_P);
-
-	     if (refp && res == ERTS_PORT_OP_SCHEDULED) {
-		 ASSERT(is_internal_ordinary_ref(ref));
-		 BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true);
-	     }
-
-	 }
-
-	 BIF_RET(am_true);
+	 prt = erts_port_lookup(id, invalid_flags);
+	 if (prt)
+	     res = erts_port_exit(c_p, 0, prt, c_p->common.id, reason, refp);
+
+         if (!refp || res != ERTS_PORT_OP_SCHEDULED)
+             ERTS_BIF_PREP_RET(ret_val, am_true);
+         else {
+             ASSERT(is_internal_ordinary_ref(ref));
+             ERTS_BIF_PREP_TRAP3(ret_val, await_port_send_result_trap,
+                                 c_p, ref, am_true, am_true);
+         }
      }
-     else if(is_external_port(BIF_ARG_1)
-	     && external_port_dist_entry(BIF_ARG_1) == erts_this_dist_entry)
-	 BIF_RET(am_true);
-     
-     /*
-      * If it is a remote pid, send a signal to the remote node.
-      */
-
-     if (is_external_pid(BIF_ARG_1)) {
-	 int code;
-	 ErtsDSigData dsd;
-	 DistEntry *dep;
-
-	 dep = external_pid_dist_entry(BIF_ARG_1);
-	 ERTS_ASSERT(dep);
-	 if(dep == erts_this_dist_entry)
-	     BIF_RET(am_true);
-
-	 code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_PROC_LOCK_MAIN,
-				  ERTS_DSP_NO_LOCK, 0, 1);
-	 switch (code) {
-	 case ERTS_DSIG_PREP_NOT_ALIVE:
-	 case ERTS_DSIG_PREP_NOT_CONNECTED:
-	     BIF_RET(am_true);
-	 case ERTS_DSIG_PREP_PENDING:
-	 case ERTS_DSIG_PREP_CONNECTED:
-	     code = erts_dsig_send_exit2(&dsd, BIF_P->common.id, BIF_ARG_1, BIF_ARG_2);
-	     if (code == ERTS_DSIG_SEND_YIELD)
-		 ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
-	     BIF_RET(am_true);
-	 default:
-	     ERTS_ASSERT(! "Invalid dsig prepare result");
-	 }
+     else if (is_external_pid(id)) {
+	 DistEntry *dep = external_pid_dist_entry(id);
+	 if (dep == erts_this_dist_entry)
+             ERTS_BIF_PREP_RET(ret_val, am_true); /* Old incarnation of this node... */
+         else {
+             int code;
+             ErtsDSigData dsd;
+
+             code = erts_dsig_prepare(&dsd, dep, c_p, ERTS_PROC_LOCK_MAIN,
+                                      ERTS_DSP_NO_LOCK, 0, 1);
+             switch (code) {
+             case ERTS_DSIG_PREP_NOT_ALIVE:
+             case ERTS_DSIG_PREP_NOT_CONNECTED:
+                 ERTS_BIF_PREP_RET(ret_val, am_true);
+                 break;
+             case ERTS_DSIG_PREP_PENDING:
+             case ERTS_DSIG_PREP_CONNECTED:
+                 code = erts_dsig_send_exit2(&dsd, c_p->common.id, id, reason);
+                 if (code == ERTS_DSIG_SEND_YIELD)
+                     ERTS_BIF_PREP_YIELD_RETURN(ret_val, c_p, am_true);
+                 else
+                     ERTS_BIF_PREP_RET(ret_val, am_true);
+                 break;
+             default:
+                 ASSERT(! "Invalid dsig prepare result");
+                 ERTS_BIF_PREP_ERROR(ret_val, c_p, EXC_INTERNAL_ERROR);
+                 break;
+             }
+         }
      }
-     else if (is_not_internal_pid(BIF_ARG_1)) {
-       BIF_ERROR(BIF_P, BADARG);
+     else if (is_external_port(id)) {
+	 DistEntry *dep = external_port_dist_entry(id);
+	 if(dep == erts_this_dist_entry)
+             ERTS_BIF_PREP_RET(ret_val, am_true); /* Old incarnation of this node... */
+         else
+             ERTS_BIF_PREP_ERROR(ret_val, c_p, BADARG);
      }
      else {
-	 /*
-	  * The pid is internal.  Verify that it refers to an existing process.
-	  */
-	 ErtsProcLocks rp_locks;
-
-	 if (BIF_ARG_1 == BIF_P->common.id) {
-	     rp_locks = ERTS_PROC_LOCKS_ALL;
-	     rp = BIF_P;
-	     erts_proc_lock(rp, ERTS_PROC_LOCKS_ALL_MINOR);
-	 }
-	 else {
-	     rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
-	     rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
-				BIF_ARG_1, rp_locks);
-	     if (!rp) {
-		 BIF_RET(am_true);
-	     }
-	 }
+         /* Not an id of a process or a port... */
 
-	 /*
-	  * Send an exit signal.
-	  */
-	 erts_send_exit_signal(BIF_P,
-			       BIF_P->common.id,
-			       rp,
-			       &rp_locks,
-			       BIF_ARG_2,
-			       NIL,
-			       NULL,
-			       BIF_P == rp ? ERTS_XSIG_FLG_NO_IGN_NORMAL : 0);
-	 if (rp == BIF_P)
-	     rp_locks &= ~ERTS_PROC_LOCK_MAIN;
-	 if (rp_locks)
-	     erts_proc_unlock(rp, rp_locks);
-	 /*
-	  * We may have exited ourselves and may have to take action.
-	  */
-	 ERTS_BIF_CHK_EXITED(BIF_P);
-	 BIF_RET(am_true);
+         ERTS_BIF_PREP_ERROR(ret_val, c_p, BADARG);
      }
+
+     return ret_val;
 }
 
+BIF_RETTYPE exit_2(BIF_ALIST_2)
+{
+    return send_exit_signal_bif(BIF_P, BIF_ARG_1, BIF_ARG_2, !0);
+}
+
+BIF_RETTYPE exit_signal_2(BIF_ALIST_2)
+{
+    return send_exit_signal_bif(BIF_P, BIF_ARG_1, BIF_ARG_2, 0);
+}
+
+
 /**********************************************************************/
 /* this sets some process info- trapping exits or the error handler */
 
@@ -1709,36 +1441,13 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
        BIF_RET(old_value);
    }
    else if (BIF_ARG_1 == am_trap_exit) {
-       erts_aint32_t state;
-       Uint trap_exit;
-       if (BIF_ARG_2 == am_true) {
-	   trap_exit = 1;
-       } else if (BIF_ARG_2 == am_false) {
-	   trap_exit = 0;
-       } else {
-	   goto error;
-       }
-       /*
-	* NOTE: It is important that we check for pending exit signals
-	*       and handle them before returning if trap_exit is set to
-	*       true. For more info, see implementation of
-	*       erts_send_exit_signal().
-	*/
-       erts_proc_lock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND);
-       if (trap_exit)
-	   state = erts_atomic32_read_bor_mb(&BIF_P->state,
-						 ERTS_PSFLG_TRAP_EXIT);
+       old_value = (BIF_P->flags & F_TRAP_EXIT) ? am_true : am_false;
+       if (BIF_ARG_2 == am_true)
+           BIF_P->flags |= F_TRAP_EXIT;
+       else if (BIF_ARG_2 == am_false)
+           BIF_P->flags &= ~F_TRAP_EXIT;
        else
-	   state = erts_atomic32_read_band_mb(&BIF_P->state,
-						  ~ERTS_PSFLG_TRAP_EXIT);
-       erts_proc_unlock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND);
-
-       if (state & ERTS_PSFLG_PENDING_EXIT) {
-	   erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
-	   ERTS_BIF_EXITED(BIF_P);
-       }
-
-       old_value = (state & ERTS_PSFLG_TRAP_EXIT) ? am_true : am_false;
+	   goto error;
        BIF_RET(old_value);
    }
    else if (BIF_ARG_1 == am_scheduler) {
@@ -2142,9 +1851,6 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx)
             }
 
 	    switch (erts_port_command(p, ps_flags, pt, msg, refp)) {
-	    case ERTS_PORT_OP_CALLER_EXIT:
-		/* We are exiting... */
-		return SEND_USER_ERROR;
 	    case ERTS_PORT_OP_BUSY:
 		/* Nothing has been sent */
 		if (ctx->suspend)
@@ -4373,14 +4079,88 @@ BIF_RETTYPE group_leader_0(BIF_ALIST_0)
 }
 
 /**********************************************************************/
-/* arg1 == leader, arg2 == new member */
+/* set group leader */
 
-BIF_RETTYPE group_leader_2(BIF_ALIST_2)
+int
+erts_set_group_leader(Process *proc, Eterm new_gl)
 {
-    Process* new_member;
+    
+    erts_aint32_t state;
 
-    if (is_not_pid(BIF_ARG_1)) {
-	BIF_ERROR(BIF_P, BADARG);
+    ASSERT(is_pid(new_gl));
+
+    state = erts_atomic32_read_nob(&proc->state);
+
+    if (state & ERTS_PSFLG_EXITING)
+        return 0;
+        
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(proc));
+
+    if (!(state & ERTS_PSFLG_DIRTY_RUNNING))
+        proc->group_leader = STORE_NC_IN_PROC(proc, new_gl);
+    else {
+        ErlHeapFragment *bp;
+        Eterm *hp;
+        /*
+         * Currently executing on a dirty scheduler,
+         * so we are not allowed to write to its heap.
+         * Store group leader pid in heap fragment.
+         */
+        bp = new_message_buffer(NC_HEAP_SIZE(new_gl));
+        hp = bp->mem;
+        proc->group_leader = STORE_NC(&hp,
+                                            &proc->off_heap,
+                                            new_gl);
+        bp->next = proc->mbuf;
+        proc->mbuf = bp;
+        proc->mbuf_sz += bp->used_size;
+    }
+
+    return !0;
+}
+
+BIF_RETTYPE erts_internal_group_leader_3(BIF_ALIST_3)
+{
+    if (is_not_pid(BIF_ARG_1))
+        BIF_ERROR(BIF_P, BADARG);
+    if (is_not_internal_pid(BIF_ARG_2))
+        BIF_ERROR(BIF_P, BADARG);
+    if (is_not_internal_ref(BIF_ARG_3))
+        BIF_ERROR(BIF_P, BADARG);
+
+    erts_proc_sig_send_group_leader(BIF_P,
+                                    BIF_ARG_2,
+                                    BIF_ARG_1,
+                                    BIF_ARG_3);
+    BIF_RET(am_ok);
+}
+
+BIF_RETTYPE erts_internal_group_leader_2(BIF_ALIST_2)
+{
+    if (is_not_pid(BIF_ARG_1))
+        BIF_RET(am_badarg);
+
+    if (is_internal_pid(BIF_ARG_2)) {
+        Process *rp;
+        int res;
+
+        if (BIF_ARG_2 == BIF_P->common.id)
+            rp = BIF_P;
+        else {
+            rp = erts_try_lock_sig_free_proc(BIF_ARG_2,
+                                             ERTS_PROC_LOCK_MAIN);
+            if (!rp)
+                BIF_RET(am_badarg);
+            if (rp == ERTS_PROC_LOCK_BUSY)
+                BIF_RET(am_false);
+        }
+
+        res = erts_set_group_leader(rp, BIF_ARG_1);
+
+        if (rp != BIF_P)
+            erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
+
+        BIF_RET(res ? am_true : am_badarg);
     }
 
     if (is_external_pid(BIF_ARG_2)) {
@@ -4408,74 +4188,8 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2)
 	    ERTS_ASSERT(! "Invalid dsig prepare result");
 	}
     }
-    else if (is_internal_pid(BIF_ARG_2)) {
-	int await_x;
-	ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS;
-	new_member = erts_pid2proc_nropt(BIF_P, ERTS_PROC_LOCK_MAIN,
-					 BIF_ARG_2, locks);
-	if (!new_member)
-	    BIF_ERROR(BIF_P, BADARG);
-
-	if (new_member == ERTS_PROC_LOCK_BUSY)
-	    ERTS_BIF_YIELD2(bif_export[BIF_group_leader_2], BIF_P,
-			    BIF_ARG_1, BIF_ARG_2);
-
-	await_x = (new_member != BIF_P
-		   && ERTS_PROC_PENDING_EXIT(new_member));
-	if (!await_x) {
-	    if (is_immed(BIF_ARG_1))
-		new_member->group_leader = BIF_ARG_1;
-	    else {
-		locks &= ~ERTS_PROC_LOCK_STATUS;
-		erts_proc_unlock(new_member, ERTS_PROC_LOCK_STATUS);
-		if (new_member == BIF_P
-		    || !(erts_atomic32_read_nob(&new_member->state)
-			 & ERTS_PSFLG_DIRTY_RUNNING)) {
-		    new_member->group_leader = STORE_NC_IN_PROC(new_member,
-								BIF_ARG_1);
-		}
-		else {
-		    ErlHeapFragment *bp;
-		    Eterm *hp;
-		    /*
-		     * Other process executing on a dirty scheduler,
-		     * so we are not allowed to write to its heap.
-		     * Store in heap fragment.
-		     */
-
-		    bp = new_message_buffer(NC_HEAP_SIZE(BIF_ARG_1));
-		    hp = bp->mem;
-		    new_member->group_leader = STORE_NC(&hp,
-							&new_member->off_heap,
-							BIF_ARG_1);
-		    bp->next = new_member->mbuf;
-		    new_member->mbuf = bp;
-		    new_member->mbuf_sz += bp->used_size;
-		}
-	    }
-	}
-
-	if (new_member == BIF_P)
-	    locks &= ~ERTS_PROC_LOCK_MAIN;
-	if (locks)
-	    erts_proc_unlock(new_member, locks);
-
-	if (await_x) {
-	    /* Wait for new_member to terminate; then badarg */
-	    Eterm args[2] = {BIF_ARG_1, BIF_ARG_2};
-	    ERTS_BIF_AWAIT_X_APPLY_TRAP(BIF_P,
-					BIF_ARG_2,
-					am_erlang,
-					am_group_leader,
-					args,
-					2);
-	}
 
-	BIF_RET(am_true);
-    }
-    else {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    BIF_RET(am_badarg);
 }
 
 BIF_RETTYPE system_flag_2(BIF_ALIST_2)    
@@ -4662,7 +4376,6 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
         BIF_RET(ret);
     } else if (BIF_ARG_1 == make_small(1)) {
 	int i, max;
-	ErtsMessage* mp;
 	erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	erts_thr_progress_block();
 
@@ -4677,16 +4390,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 #endif
 		p->seq_trace_clock = 0;
 		p->seq_trace_lastcnt = 0;
-		ERTS_MSGQ_MV_INQ2PRIVQ(p);
-		mp = p->msg.first;
-		while(mp != NULL) {
-#ifdef USE_VM_PROBES
-		    ERL_MESSAGE_TOKEN(mp) = (ERL_MESSAGE_DT_UTAG(mp) != NIL) ? am_have_dt_utag : NIL;
-#else
-		    ERL_MESSAGE_TOKEN(mp) = NIL;
-#endif
-		    mp = mp->next;
-		}
+
+                erts_proc_sig_clear_seq_trace_tokens(p);
 	    }
 	}
 
@@ -4949,58 +4654,17 @@ static BIF_RETTYPE bif_return_trap(BIF_ALIST_2)
     BIF_RET(res);
 }
 
-/*
- * NOTE: The erts_bif_prep_await_proc_exit_*() functions are
- * tightly coupled with the implementation of erlang:await_proc_exit/3.
- * The erts_bif_prep_await_proc_exit_*() functions can safely call
- * skip_current_msgq() since they know that erlang:await_proc_exit/3
- * unconditionally will do a monitor and then unconditionally will
- * wait for the corresponding 'DOWN' message in a receive, and no other
- * receive is done before this receive. This optimization removes an
- * unnecessary scan of the currently existing message queue (which
- * can be large). If the erlang:await_proc_exit/3 implementation
- * is changed so that the above isn't true, nasty bugs in later
- * receives, etc, may appear.
- */
-
-static ERTS_INLINE int
-skip_current_msgq(Process *c_p)
-{
-    int res;
-#if defined(ERTS_ENABLE_LOCK_CHECK)
-    erts_proc_lc_chk_only_proc_main(c_p);
-#endif
-
-    erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-    if (ERTS_PROC_PENDING_EXIT(c_p)) {
-	KILL_CATCHES(c_p);
-	c_p->freason = EXC_EXIT;
-	res = 0;
-    }
-    else {
-        ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
-	c_p->msg.save = c_p->msg.last;
-	res = 1;
-    }
-    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-    return res;
-}
-
 void
 erts_bif_prep_await_proc_exit_data_trap(Process *c_p, Eterm pid, Eterm ret)
 {
-    if (skip_current_msgq(c_p)) {
-	ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, pid, am_data, ret);
-    }
+    ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, pid, am_data, ret);
 }
 
 void
 erts_bif_prep_await_proc_exit_reason_trap(Process *c_p, Eterm pid)
 {
-    if (skip_current_msgq(c_p)) {
-	ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p,
-			    pid, am_reason, am_undefined);
-    }
+    ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p,
+                               pid, am_reason, am_undefined);
 }
 
 void
@@ -5011,21 +4675,19 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
 					 Eterm args[],
 					 int nargs)
 {
+    Eterm term;
+    Eterm *hp;
+    int i;
     ASSERT(is_atom(module) && is_atom(function));
-    if (skip_current_msgq(c_p)) {
-	Eterm term;
-	Eterm *hp;
-	int i;
 
-	hp = HAlloc(c_p, 4+2*nargs);
-	term = NIL;
-	for (i = nargs-1; i >= 0; i--) {
-	    term = CONS(hp, args[i], term);
-	    hp += 2;
-	}
-	term = TUPLE3(hp, module, function, term);
-	ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, pid, am_apply, term);
+    hp = HAlloc(c_p, 4+2*nargs);
+    term = NIL;
+    for (i = nargs-1; i >= 0; i--) {
+        term = CONS(hp, args[i], term);
+        hp += 2;
     }
+    term = TUPLE3(hp, module, function, term);
+    ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, pid, am_apply, term);
 }
 
 Export bif_return_trap_export;
@@ -5063,6 +4725,9 @@ void erts_init_bif(void)
 			  am_erts_internal, am_dsend_continue_trap, 1,
 			  dsend_continue_trap_1);
 
+    erts_init_trap_export(&await_exit_trap, am_erts_internal,
+                          am_await_exit, 0, erts_internal_await_exit_trap);
+
     flush_monitor_messages_trap = erts_export_put(am_erts_internal,
 						  am_flush_monitor_messages,
 						  3);
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index a2bc883dbe..dca53686f4 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -306,7 +306,7 @@ do {							\
     (Proc)->freason = TRAP;				\
 } while (0)
 
-#define BIF_TRAP0(p, Trap_) do {		\
+#define BIF_TRAP0(Trap_, p) do {                  \
       (p)->arity = 0;				\
       (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]);	\
       (p)->freason = TRAP;			\
@@ -404,7 +404,7 @@ do {									\
 #define ERTS_BIF_YIELD0(TRP, P)						\
 do {									\
     ERTS_VBUMP_ALL_REDS((P));						\
-    BIF_TRAP0((TRP), (P));						\
+    BIF_TRAP0((TRP), (P));                                              \
 } while (0)
 
 #define ERTS_BIF_YIELD1(TRP, P, A0)					\
@@ -428,7 +428,7 @@ do {									\
 #define ERTS_BIF_EXITED(PROC)		\
 do {					\
     KILL_CATCHES((PROC));		\
-    BIF_ERROR((PROC), EXC_EXIT);	\
+    BIF_ERROR((PROC), EXTAG_EXIT);	\
 } while (0)
 
 #define ERTS_BIF_CHK_EXITED(PROC)	\
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index be653ee2a0..93613ac2eb 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1996-2017. All Rights Reserved.
+# Copyright Ericsson AB 1996-2018. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -62,6 +62,7 @@ bif erlang:erase/0
 bif erlang:erase/1
 bif erlang:exit/1
 bif erlang:exit/2
+bif erlang:exit_signal/2
 bif erlang:external_size/1
 bif erlang:external_size/2
 gcbif erlang:float/1
@@ -73,7 +74,8 @@ bif erlang:get/0
 bif erlang:get/1
 bif erlang:get_keys/1
 bif erlang:group_leader/0
-bif erlang:group_leader/2
+bif erts_internal:group_leader/2
+bif erts_internal:group_leader/3
 bif erlang:halt/2
 bif erlang:phash/2
 bif erlang:phash2/1
@@ -187,6 +189,8 @@ bif erts_internal:release_literal_area_switch/0
 
 bif erts_internal:scheduler_wall_time/1
 
+bif erts_internal:dirty_process_handle_signals/1
+
 # inet_db support
 bif erlang:port_set_data/2
 bif erlang:port_get_data/1
@@ -434,13 +438,6 @@ bif erts_debug:dirty_cpu/2
 bif erts_debug:dirty_io/2
 bif erts_debug:dirty/3
 
-#
-# Monitor testing bif's...
-#
-bif erts_debug:dump_monitors/1
-bif erts_debug:dump_links/1
-
-
 #
 # Lock counter bif's
 #
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 3967f7f7fc..4dabac3512 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -39,6 +39,7 @@
 #include "erl_instrument.h"
 #include "erl_hl_timer.h"
 #include "erl_thr_progress.h"
+#include "erl_proc_sig_queue.h"
 
 /* Forward declarations -- should really appear somewhere else */
 static void process_killer(void);
@@ -107,36 +108,11 @@ process_killer(void)
 		if ((j = sys_get_key(0)) <= 0)
 		    erts_exit(0, "");
 		switch(j) {
-		case 'k': {
-		    ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
-		    erts_aint32_t state;
-		    erts_proc_inc_refc(rp);
-		    erts_proc_lock(rp, rp_locks);
-		    state = erts_atomic32_read_acqb(&rp->state);
-		    if (state & (ERTS_PSFLG_FREE
-				 | ERTS_PSFLG_EXITING
-				 | ERTS_PSFLG_ACTIVE
-				 | ERTS_PSFLG_ACTIVE_SYS
-				 | ERTS_PSFLG_IN_RUNQ
-				 | ERTS_PSFLG_RUNNING
-				 | ERTS_PSFLG_RUNNING_SYS
-				 | ERTS_PSFLG_DIRTY_RUNNING
-				 | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
-			erts_printf("Can only kill WAITING processes this way\n");
-		    }
-		    else {
-			(void) erts_send_exit_signal(NULL,
-						     NIL,
-						     rp,
-						     &rp_locks,
-						     am_kill,
-						     NIL,
-						     NULL,
-						     0);
-		    }
-		    erts_proc_unlock(rp, rp_locks);
-		    erts_proc_dec_refc(rp);
-		}
+		case 'k':
+                    /* Send a 'kill' exit signal from init process */
+                    erts_proc_sig_send_exit(NULL, erts_init_process_id,
+                                            rp->common.id, am_kill, NIL,
+                                            0);
 		case 'n': br = 1; break;
 		case 'r': return;
 		default: return;
@@ -161,47 +137,64 @@ static void doit_print_link(ErtsLink *lnk, void *vpcontext)
 
     if (pcontext->is_first) {
 	pcontext->is_first = 0;
-	erts_print(to, to_arg, "%T", lnk->pid);
+	erts_print(to, to_arg, "%T", lnk->other.item);
     } else {
-	erts_print(to, to_arg, ", %T", lnk->pid);
+	erts_print(to, to_arg, ", %T", lnk->other.item);
     }
 }
     
 
 static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext)
 {
+    ErtsMonitorData *mdp;
     PrintMonitorContext *pcontext = vpcontext;
     fmtfn_t to = pcontext->to;
     void *to_arg = pcontext->to_arg;
     char *prefix = ", ";
  
-    if (pcontext->is_first) {
-	pcontext->is_first = 0;
-	prefix = "";
-    }
-
+    mdp = erts_monitor_to_data(mon);
     switch (mon->type) {
-    case MON_ORIGIN:
-	if (is_atom(mon->u.pid)) { /* dist by name */
-	    ASSERT(is_node_name_atom(mon->u.pid));
-	    erts_print(to, to_arg, "%s{to,{%T,%T},%T}", prefix, mon->name,
-		       mon->u.pid, mon->ref);
-	} else if (is_atom(mon->name)){ /* local by name */
-	    erts_print(to, to_arg, "%s{to,{%T,%T},%T}", prefix, mon->name,
-		       erts_this_dist_entry->sysname, mon->ref);
-	} else { /* local and distributed by pid */
-	    erts_print(to, to_arg, "%s{to,%T,%T}", prefix, mon->u.pid, mon->ref);
-	}
-	break;
-    case MON_TARGET:
-	erts_print(to, to_arg, "%s{from,%T,%T}", prefix, mon->u.pid, mon->ref);
-	break;
-    case MON_NIF_TARGET: {
-        ErtsResource* rsrc = mon->u.resource;
-        erts_print(to, to_arg, "%s{from,{%T,%T},%T}", prefix, rsrc->type->module,
-                   rsrc->type->name, mon->ref);
-	break;
-    }
+    case ERTS_MON_TYPE_PROC:
+    case ERTS_MON_TYPE_PORT:
+    case ERTS_MON_TYPE_TIME_OFFSET:
+    case ERTS_MON_TYPE_DIST_PROC:
+    case ERTS_MON_TYPE_RESOURCE:
+    case ERTS_MON_TYPE_NODE:
+
+        if (pcontext->is_first) {
+            pcontext->is_first = 0;
+            prefix = "";
+        }
+
+        if (erts_monitor_is_target(mon)) {
+            if (mon->type != ERTS_MON_TYPE_RESOURCE)
+                erts_print(to, to_arg, "%s{from,%T,%T}", prefix, mon->other.item, mdp->ref);
+            else {
+                ErtsResource* rsrc = mon->other.ptr;
+                erts_print(to, to_arg, "%s{from,{%T,%T},%T}", prefix, rsrc->type->module,
+                           rsrc->type->name, mdp->ref);
+            }
+        }
+        else {
+            if (!(mon->flags & ERTS_ML_FLG_NAME))
+                erts_print(to, to_arg, "%s{to,%T,%T}", prefix, mon->other.item, mdp->ref);
+            else {
+                ErtsMonitorDataExtended *mdep = (ErtsMonitorDataExtended *) mdp;
+                Eterm node;
+                if (mdep->dist)
+                    node = mdep->dist->nodename;
+                else
+                    node = erts_this_dist_entry->sysname;
+                erts_print(to, to_arg, "%s{to,{%T,%T},%T}", prefix, mdep->u.name,
+                           node, mdp->ref);
+            }
+        }
+
+        break;
+
+    default:
+        /* ignore other monitors... */
+        break;
     }
 }
 			       
@@ -257,15 +250,22 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p)
     }
 
     erts_print(to, to_arg, "Spawned by: %T\n", p->parent);
-    ERTS_MSGQ_MV_INQ2PRIVQ(p);
-    erts_print(to, to_arg, "Message queue length: %d\n", p->msg.len);
+
+    erts_proc_lock(p, ERTS_PROC_LOCK_MSGQ);
+    erts_proc_sig_fetch(p);
+    erts_proc_unlock(p, ERTS_PROC_LOCK_MSGQ);
+    erts_print(to, to_arg, "Message queue length: %d\n", p->sig_qs.len);
 
     /* display the message queue only if there is anything in it */
-    if (!ERTS_IS_CRASH_DUMPING && p->msg.first != NULL && !garbing) {
-	ErtsMessage* mp;
+    if (!ERTS_IS_CRASH_DUMPING && p->sig_qs.first != NULL && !garbing) {
 	erts_print(to, to_arg, "Message queue: [");
-	for (mp = p->msg.first; mp; mp = mp->next)
-	    erts_print(to, to_arg, mp->next ? "%T," : "%T", ERL_MESSAGE_TERM(mp));
+        ERTS_FOREACH_SIG_PRIVQS(
+            p, mp,
+            {
+                if (ERTS_SIG_IS_NON_MSG((ErtsSignal *) mp))
+                    erts_print(to, to_arg, mp->next ? "%T," : "%T",
+                               ERL_MESSAGE_TERM(mp));
+            });
 	erts_print(to, to_arg, "]\n");
     }
 
@@ -306,11 +306,12 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p)
     }
 
     /* display the links only if there are any*/
-    if (ERTS_P_LINKS(p) || ERTS_P_MONITORS(p)) {
+    if (ERTS_P_LINKS(p) || ERTS_P_MONITORS(p) || ERTS_P_LT_MONITORS(p)) {
 	PrintMonitorContext context = {1, to, to_arg};
 	erts_print(to, to_arg,"Link list: [");
-	erts_doforall_links(ERTS_P_LINKS(p), &doit_print_link, &context);	
-	erts_doforall_monitors(ERTS_P_MONITORS(p), &doit_print_monitor, &context);
+	erts_link_tree_foreach(ERTS_P_LINKS(p), doit_print_link, &context);	
+	erts_monitor_tree_foreach(ERTS_P_MONITORS(p), doit_print_monitor, &context);
+	erts_monitor_list_foreach(ERTS_P_LT_MONITORS(p), doit_print_monitor, &context);
 	erts_print(to, to_arg,"]\n");
     }
 
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index cd799e04b8..fdf307da1b 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -45,6 +45,7 @@
 #include "erl_binary.h"
 #include "erl_thr_progress.h"
 #include "dtrace-wrapper.h"
+#include "erl_proc_sig_queue.h"
 
 #define DIST_CTL_DEFAULT_SIZE 64
 
@@ -109,7 +110,6 @@ int erts_dist_buf_busy_limit;
 
 /* distribution trap functions */
 Export* dmonitor_node_trap = NULL;
-Export* dmonitor_p_trap = NULL;
 
 /* local variables */
 static Export *dist_ctrl_put_data_trap;
@@ -184,6 +184,161 @@ get_suspended_on_de(DistEntry *dep, erts_aint32_t unset_qflgs)
     }
 }
 
+#define ERTS_MON_LNK_FIRE_LIMIT 100
+
+static void monitor_connection_down(ErtsMonitor *mon, void *unused)
+{
+    if (erts_monitor_is_origin(mon))
+        erts_proc_sig_send_demonitor(mon);
+    else
+        erts_proc_sig_send_monitor_down(mon, am_noconnection);
+}
+
+static void link_connection_down(ErtsLink *lnk, void *vdist)
+{
+    erts_proc_sig_send_link_exit(NULL, THE_NON_VALUE, lnk,
+                                 am_noconnection, NIL);
+}
+
+typedef enum {
+    ERTS_CML_CLEANUP_STATE_LINKS,
+    ERTS_CML_CLEANUP_STATE_MONITORS,
+    ERTS_CML_CLEANUP_STATE_ONAME_MONITORS,
+    ERTS_CML_CLEANUP_STATE_NODE_MONITORS
+} ErtsConMonLnkCleaupState;
+
+typedef struct {
+    ErtsConMonLnkCleaupState state;
+    ErtsMonLnkDist *dist;
+    void *yield_state;
+    int trigger_node_monitors;
+    Eterm nodename;
+    Eterm visability;
+    Eterm reason;
+    ErlOffHeap oh;
+    Eterm heap[1];
+} ErtsConMonLnkCleanup;
+
+static void
+con_monitor_link_cleanup(void *vcmlcp)
+{
+    ErtsConMonLnkCleanup *cmlcp = vcmlcp;
+    ErtsMonLnkDist *dist = cmlcp->dist;
+    ErtsSchedulerData *esdp;
+    int yield;
+
+    switch (cmlcp->state) {
+    case ERTS_CML_CLEANUP_STATE_LINKS:
+        yield = erts_link_list_foreach_delete_yielding(&dist->links,
+                                                       link_connection_down,
+                                                       NULL, &cmlcp->yield_state,
+                                                       ERTS_MON_LNK_FIRE_LIMIT);
+        if (yield)
+            break;
+
+        ASSERT(!cmlcp->yield_state);
+        cmlcp->state = ERTS_CML_CLEANUP_STATE_MONITORS;
+    case ERTS_CML_CLEANUP_STATE_MONITORS:
+        yield = erts_monitor_list_foreach_delete_yielding(&dist->monitors,
+                                                          monitor_connection_down,
+                                                          NULL, &cmlcp->yield_state,
+                                                          ERTS_MON_LNK_FIRE_LIMIT);
+        if (yield)
+            break;
+
+        ASSERT(!cmlcp->yield_state);
+        cmlcp->state = ERTS_CML_CLEANUP_STATE_ONAME_MONITORS;
+    case ERTS_CML_CLEANUP_STATE_ONAME_MONITORS:
+        yield = erts_monitor_tree_foreach_delete_yielding(&dist->orig_name_monitors,
+                                                          monitor_connection_down,
+                                                          NULL, &cmlcp->yield_state,
+                                                          ERTS_MON_LNK_FIRE_LIMIT/2);
+        if (yield)
+            break;
+
+        cmlcp->dist = NULL;
+        erts_mon_link_dist_dec_refc(dist);
+
+        ASSERT(!cmlcp->yield_state);
+        cmlcp->state = ERTS_CML_CLEANUP_STATE_NODE_MONITORS;
+    case ERTS_CML_CLEANUP_STATE_NODE_MONITORS:
+        if (cmlcp->trigger_node_monitors) {
+            send_nodes_mon_msgs(NULL,
+                                am_nodedown,
+                                cmlcp->nodename,
+                                cmlcp->visability,
+                                cmlcp->reason);
+        }
+        erts_cleanup_offheap(&cmlcp->oh);
+        erts_free(ERTS_ALC_T_CML_CLEANUP, vcmlcp);
+        return; /* done */
+    }
+
+    /* yield... */
+
+    esdp = erts_get_scheduler_data();
+    ASSERT(esdp && esdp->type == ERTS_SCHED_NORMAL);
+    erts_schedule_misc_aux_work((int) esdp->no,
+                                con_monitor_link_cleanup,
+                                (void *) cmlcp);
+}
+
+static void
+schedule_con_monitor_link_cleanup(ErtsMonLnkDist *dist,
+                                  Eterm nodename,
+                                  Eterm visability,
+                                  Eterm reason)
+{
+    if (dist || is_value(nodename)) {
+        ErtsSchedulerData *esdp;
+        ErtsConMonLnkCleanup *cmlcp;
+        Uint rsz, size;
+
+        size = sizeof(ErtsConMonLnkCleanup);
+
+        if (is_non_value(reason) || is_immed(reason)) {
+            rsz = 0;
+            size -= sizeof(Eterm);
+        }
+        else {
+            rsz = size_object(reason);
+            size += sizeof(Eterm) * (rsz - 1);
+        }
+
+        cmlcp = erts_alloc(ERTS_ALC_T_CML_CLEANUP, size);
+
+        ERTS_INIT_OFF_HEAP(&cmlcp->oh);
+
+        cmlcp->yield_state = NULL;
+        cmlcp->dist = dist;
+        if (!dist)
+            cmlcp->state = ERTS_CML_CLEANUP_STATE_NODE_MONITORS;
+        else {
+            cmlcp->state = ERTS_CML_CLEANUP_STATE_LINKS;
+            erts_mtx_lock(&dist->mtx);
+            ASSERT(dist->alive);
+            dist->alive = 0;
+            erts_mtx_unlock(&dist->mtx);
+        }
+
+        cmlcp->trigger_node_monitors = is_value(nodename);
+        cmlcp->nodename = nodename;
+        cmlcp->visability = visability;
+        if (rsz == 0)
+            cmlcp->reason = reason;
+        else {
+            Eterm *hp = &cmlcp->heap[0];
+            cmlcp->reason = copy_struct(reason, rsz, &hp, &cmlcp->oh);
+        }
+
+        esdp = erts_get_scheduler_data();
+        ASSERT(esdp && esdp->type == ERTS_SCHED_NORMAL);
+        erts_schedule_misc_aux_work((int) esdp->no,
+                                    con_monitor_link_cleanup,
+                                    (void *) cmlcp);
+    }
+}
+
 /*
 ** A full node name constists of a "n@h"
 **
@@ -235,170 +390,6 @@ int is_node_name_atom(Eterm a)
     return is_node_name((char*)atom_tab(i)->name, atom_tab(i)->len);
 }
 
-typedef struct {
-    DistEntry *dep;
-    Eterm *lhp;
-} NetExitsContext;
-
-/* 
-** This function is called when a distribution 
-** port or process terminates
-*/
-static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp)
-{
-    Process *rp;
-    ErtsMonitor *rmon;
-    DistEntry *dep = ((NetExitsContext *) vnecp)->dep;
-    ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK;
-
-    rp = erts_pid2proc(NULL, 0, mon->u.pid, rp_locks);
-    if (!rp)
-	goto done;
-
-    if (mon->type == MON_ORIGIN) {
-	/* local pid is being monitored */
-	rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref);
-	/* ASSERT(rmon != NULL); nope, can happen during process exit */
-	if (rmon != NULL) {
-	    erts_destroy_monitor(rmon);
-	}
-    } else {
-	DeclareTmpHeapNoproc(lhp,3);
-	Eterm watched;
-	UseTmpHeapNoproc(3);
-	ASSERT(mon->type == MON_TARGET);
-	rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref);
-	/* ASSERT(rmon != NULL); can happen during process exit */
-	if (rmon != NULL) {
-            ASSERT(rmon->type == MON_ORIGIN);
-	    ASSERT(is_atom(rmon->name) || is_nil(rmon->name));
-	    watched = (is_atom(rmon->name)
-		       ? TUPLE2(lhp, rmon->name, dep->sysname)
-		       : rmon->u.pid);
-	    rp_locks |= ERTS_PROC_LOCKS_MSG_SEND;
-	    erts_proc_lock(rp, ERTS_PROC_LOCKS_MSG_SEND);
-	    erts_queue_monitor_message(rp, &rp_locks, mon->ref, am_process, 
-				       watched, am_noconnection);
-	    erts_destroy_monitor(rmon);
-	}
-	UnUseTmpHeapNoproc(3);
-    }
-    erts_proc_unlock(rp, rp_locks);
- done:
-    erts_destroy_monitor(mon);
-}
-	
-typedef struct {
-    NetExitsContext *necp;
-    ErtsLink *lnk;
-} LinkNetExitsContext;
-
-/* 
-** This is the function actually doing the job of sending exit messages
-** for links in a dist entry upon net_exit (the node goes down), NB,
-** only process links, not node monitors are handled here, 
-** they reside in a separate tree....
-*/
-static void doit_link_net_exits_sub(ErtsLink *sublnk, void *vlnecp)
-{
-    ErtsLink *lnk = ((LinkNetExitsContext *) vlnecp)->lnk; /* the local pid */
-    ErtsLink *rlnk;
-    Process *rp;
-
-    ASSERT(lnk->type == LINK_PID);
-    if (is_internal_pid(lnk->pid)) {
-	int xres;
-	ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCKS_XSIG_SEND;
-
-	rp = erts_pid2proc(NULL, 0, lnk->pid, rp_locks);
-	if (!rp) {
-	    goto done;
-	}
-
-	rlnk = erts_remove_link(&ERTS_P_LINKS(rp), sublnk->pid);
-	xres = erts_send_exit_signal(NULL,
-				     sublnk->pid,
-				     rp,
-				     &rp_locks,
-				     am_noconnection,
-				     NIL,
-				     NULL,
-				     0);
-
-	if (rlnk) {
-	    erts_destroy_link(rlnk);
-	    if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) {
-		/* We didn't exit the process and it is traced */
-		trace_proc(NULL, 0, rp, am_getting_unlinked, sublnk->pid);
-	    }
-	}
-	erts_proc_unlock(rp, rp_locks);
-    }
- done:
-    erts_destroy_link(sublnk);
-
-}
-    
-
-
-
-
-/* 
-** This function is called when a distribution 
-** port or process terminates, once for each link on the high level, 
-** it in turn traverses the link subtree for the specific link node...
-*/
-static void doit_link_net_exits(ErtsLink *lnk, void *vnecp)
-{
-    LinkNetExitsContext lnec = {(NetExitsContext *) vnecp, lnk};
-    ASSERT(lnk->type == LINK_PID);
-    erts_sweep_links(ERTS_LINK_ROOT(lnk), &doit_link_net_exits_sub, (void *) &lnec);
-#ifdef DEBUG
-    ERTS_LINK_ROOT(lnk) = NULL;
-#endif
-    erts_destroy_link(lnk);
-}
-
-
-static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp)
-{
-    DistEntry *dep = ((NetExitsContext *) vnecp)->dep;
-    Eterm name = dep->sysname;
-    Process *rp;
-    ErtsLink *rlnk;
-    Uint i,n;
-    ASSERT(lnk->type == LINK_NODE);
-    if (is_internal_pid(lnk->pid)) {
-	ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK;
-	ErlOffHeap *ohp;
-	rp = erts_proc_lookup(lnk->pid);
-	if (!rp)
-	    goto done;
-	erts_proc_lock(rp, rp_locks);
-        if (!ERTS_PROC_IS_EXITING(rp)) {
-            rlnk = erts_remove_link(&ERTS_P_LINKS(rp), name);
-            if (rlnk != NULL) {
-                ASSERT(is_atom(rlnk->pid) && (rlnk->type == LINK_NODE));
-                erts_destroy_link(rlnk);
-            }
-            n = ERTS_LINK_REFC(lnk);
-            for (i = 0; i < n; ++i) {
-                Eterm tup;
-                Eterm *hp;
-                ErtsMessage *msgp;
-
-                msgp = erts_alloc_message_heap(rp, &rp_locks,
-                                               3, &hp, &ohp);
-                tup = TUPLE2(hp, am_nodedown, name);
-                erts_queue_message(rp, rp_locks, msgp, tup, am_system);
-            }
-        }
-	erts_proc_unlock(rp, rp_locks);
-    }
- done:
-    erts_destroy_link(lnk);
-}
-
 static void
 set_node_not_alive(void *unused)
 {
@@ -444,15 +435,8 @@ inc_no_nodes(void)
 static void
 kill_dist_ctrl_proc(void *vpid)
 {
-    Eterm pid = (Eterm) vpid;
-    ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
-    Process *rp = erts_pid2proc(NULL, 0, pid, rp_locks);
-    if (rp) {
-        erts_send_exit_signal(NULL, rp->common.id, rp, &rp_locks,
-                              am_kill, NIL, NULL, 0);
-        if (rp_locks)
-            erts_proc_unlock(rp, rp_locks);
-    }
+    erts_proc_sig_send_exit(NULL, (Eterm) vpid, (Eterm) vpid,
+                            am_kill, NIL, 0);
 }
 
 static void
@@ -572,10 +556,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 	}
     }
     else { /* Call from distribution controller (port/process) */
-	NetExitsContext nec = {dep};
-	ErtsLink *nlinks;
-	ErtsLink *node_links;
-	ErtsMonitor *monitors;
+	ErtsMonLnkDist *mld;
 	Uint32 flags;
 
 	erts_atomic_set_mb(&dep->dist_cmd_scheduled, 1);
@@ -601,14 +582,8 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 	    erts_mtx_unlock(&dep->qlock);
 	}
 
-	erts_de_links_lock(dep);
-	monitors	= dep->monitors;
-        nlinks		= dep->nlinks;
-	node_links	= dep->node_links;
-	dep->monitors	= NULL;
-        dep->nlinks	= NULL;
-	dep->node_links	= NULL;
-	erts_de_links_unlock(dep);
+        mld = dep->mld;
+        dep->mld = NULL;
 
 	nodename = dep->sysname;
 	flags = dep->flags;
@@ -617,15 +592,14 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 
 	erts_de_rwunlock(dep);
 
-	erts_sweep_monitors(monitors, &doit_monitor_net_exits, (void *) &nec);
-	erts_sweep_links(nlinks, &doit_link_net_exits, (void *) &nec);
-	erts_sweep_links(node_links, &doit_node_link_net_exits, (void *) &nec);
-
-	send_nodes_mon_msgs(NULL,
-			    am_nodedown,
-			    nodename,
-			    flags & DFLAG_PUBLISHED ? am_visible : am_hidden,
-			    reason == am_normal ? am_connection_closed : reason);
+        schedule_con_monitor_link_cleanup(mld,
+                                          nodename,
+                                          (flags & DFLAG_PUBLISHED
+                                           ? am_visible
+                                           : am_hidden),
+                                          (reason == am_normal
+                                           ? am_connection_closed
+                                           : reason));
 
 	clear_dist_entry(dep);
     }
@@ -661,7 +635,6 @@ void init_dist(void)
 
     /* Lookup/Install all references to trap functions */
     dmonitor_node_trap = trap_function(am_dmonitor_node,3);
-    dmonitor_p_trap = trap_function(am_dmonitor_p, 2);
     dist_ctrl_put_data_trap = erts_export_put(am_erts_internal,
                                               am_dist_ctrl_put_data,
                                               2);
@@ -771,14 +744,6 @@ static void clear_dist_entry(DistEntry *dep)
     cache = dep->cache;
     dep->cache = NULL;
 
-#ifdef DEBUG
-    erts_de_links_lock(dep);
-    ASSERT(!dep->nlinks);
-    ASSERT(!dep->node_links);
-    ASSERT(!dep->monitors);
-    erts_de_links_unlock(dep);
-#endif
-
     erts_mtx_lock(&dep->qlock);
 
     erts_atomic64_set_nob(&dep->in, 0);
@@ -887,8 +852,7 @@ erts_dsig_send_unlink(ErtsDSigData *dsdp, Eterm local, Eterm remote)
 
 
 /* A local process that's being monitored by a remote one exits. We send:
-   {DOP_MONITOR_P_EXIT, Local pid or name, Remote pid, ref, reason},
-   which is rather sad as only the ref is needed, no pid's... */
+   {DOP_MONITOR_P_EXIT, Local pid or name, Remote pid, ref, reason} */
 int
 erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, 
 			  Eterm ref, Eterm reason)
@@ -909,12 +873,6 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
     ctl = TUPLE5(&ctl_heap[0], make_small(DOP_MONITOR_P_EXIT),
 		 watched, watcher, ref, reason);
 
-#ifdef DEBUG
-    erts_de_links_lock(dsdp->dep);
-    ASSERT(!erts_lookup_monitor(dsdp->dep->monitors, ref));
-    erts_de_links_unlock(dsdp->dep);
-#endif
-
     res = dsig_send_ctl(dsdp, ctl, 1);
     UnUseTmpHeapNoproc(6);
     return res;
@@ -953,8 +911,7 @@ erts_dsig_send_monitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
 
 /* A local process monitoring a remote one wants to stop monitoring, either 
    because of a demonitor bif call or because the local process died. We send
-   {DOP_DEMONITOR_P, Local pid, Remote pid or name, ref}, which is once again
-   rather redundant as only the ref will be needed on the other side... */
+   {DOP_DEMONITOR_P, Local pid, Remote pid or name, ref} */
 int
 erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher,
 			 Eterm watched, Eterm ref, int force)
@@ -1276,8 +1233,6 @@ int erts_net_message(Port *prt,
     Sint type;
     Eterm token;
     Eterm token_size;
-    ErtsMonitor *mon;
-    ErtsLink *lnk;
     Uint tuple_arity;
     int res;
     Uint32 connection_id;
@@ -1375,88 +1330,89 @@ int erts_net_message(Port *prt,
     token = NIL;
 
     switch (type = unsigned_val(tuple[1])) {
-    case DOP_LINK:
+    case DOP_LINK: {
+        ErtsDSigData dsd;
+        int code;
+
 	if (tuple_arity != 3) {
 	    goto invalid_message;
 	}
 	from = tuple[2];
 	to   = tuple[3];  /* local proc to link to */
 
-	if (is_not_pid(from) || is_not_pid(to)) {
-	    goto invalid_message;
-	}
+	if (is_not_external_pid(from))
+            goto invalid_message;
 
-	rp = erts_pid2proc_opt(NULL, 0,
-			       to, ERTS_PROC_LOCK_LINK,
-			       ERTS_P2P_FLG_ALLOW_OTHER_X);
-	if (!rp) {
-	    /* This is tricky (we MUST force a distributed send) */
-	    ErtsDSigData dsd;
-	    int code;
-	    code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0);
-	    if (code == ERTS_DSIG_PREP_CONNECTED) {
-		code = erts_dsig_send_exit(&dsd, to, from, am_noproc);
-		ASSERT(code == ERTS_DSIG_SEND_OK);
-	    }
-	    break;
-	}
+        if (dep != external_pid_dist_entry(from))
+            goto invalid_message;
+
+        if (is_external_pid(to)) {
+            if (external_pid_dist_entry(to) != erts_this_dist_entry)
+                goto invalid_message;
+            /* old incarnation of node; reply noproc... */
+        }
+        else if (is_internal_pid(to)) {
+            ErtsLinkData *ldp = erts_link_create(ERTS_LNK_TYPE_DIST_PROC,
+                                                 from, to);
+            ASSERT(ldp->a.other.item == to);
+            ASSERT(eq(ldp->b.other.item, from));
+#ifdef DEBUG
+            code =
+#endif
+                erts_link_dist_insert(&ldp->a, dep->mld);
+            ASSERT(code);
 
-	erts_de_links_lock(dep);
-	res = erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, from);
+            if (erts_proc_sig_send_link(NULL, to, &ldp->b))
+                break; /* done */
 
-	if (res < 0) {
-	    /* It was already there! Lets skip the rest... */
-	    erts_de_links_unlock(dep);
-	    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-	    break;
-	}
-	lnk = erts_add_or_lookup_link(&(dep->nlinks), LINK_PID, rp->common.id);
-	erts_add_link(&(ERTS_LINK_ROOT(lnk)), LINK_PID, from);
-	erts_de_links_unlock(dep);
+            /* Failed to send signal; cleanup and reply noproc... */
+
+#ifdef DEBUG
+            code =
+#endif
+                erts_link_dist_delete(&ldp->a);
+            ASSERT(code);
+            erts_link_release_both(ldp);
+        }
 
-	if (IS_TRACED_FL(rp, F_TRACE_PROCS))
-	    trace_proc(NULL, 0, rp, am_getting_linked, from);
+        code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0);
+        if (code == ERTS_DSIG_PREP_CONNECTED) {
+            code = erts_dsig_send_exit(&dsd, to, from, am_noproc);
+            ASSERT(code == ERTS_DSIG_SEND_OK);
+        }
 
-	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 	break;
+    }
 
     case DOP_UNLINK: {
-	ErtsDistLinkData dld;
 	if (tuple_arity != 3) {
 	    goto invalid_message;
 	}
 	from = tuple[2];
 	to = tuple[3];
-	if (is_not_pid(from) || is_not_pid(to)) {
+	if (is_not_external_pid(from))
+	    goto invalid_message;
+        if (dep != external_pid_dist_entry(from))
 	    goto invalid_message;
-	}
-	
-	rp = erts_pid2proc_opt(NULL, 0,
-			       to, ERTS_PROC_LOCK_LINK,
-			       ERTS_P2P_FLG_ALLOW_OTHER_X);
-	if (!rp)
-	    break;
-
-	lnk = erts_remove_link(&ERTS_P_LINKS(rp), from);
 
-	if (IS_TRACED_FL(rp, F_TRACE_PROCS) && lnk != NULL) {
-	    trace_proc(NULL, 0, rp, am_getting_unlinked, from);
-	}
+        if (is_external_pid(to)
+            && erts_this_dist_entry == external_pid_dist_entry(from))
+            break;
 
-	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+        if (is_not_internal_pid(to))
+            goto invalid_message;
 
-	erts_remove_dist_link(&dld, to, from, dep);
-	erts_destroy_dist_link(&dld);
-	if (lnk)
-	    erts_destroy_link(lnk);
+        erts_proc_sig_send_dist_unlink(dep, from, to);
 	break;
     }
     
     case DOP_MONITOR_P: {
 	/* A remote process wants to monitor us, we get:
 	   {DOP_MONITOR_P, Remote pid, local pid or name, ref} */
-	Eterm name;
-	
+	Eterm pid, name;
+        ErtsDSigData dsd;
+        int code;
+
 	if (tuple_arity != 4) {
 	    goto invalid_message;
 	}
@@ -1465,44 +1421,64 @@ int erts_net_message(Port *prt,
 	watched = tuple[3];  /* local proc to monitor */
 	ref     = tuple[4];
 
-	if (is_not_ref(ref)) {
+        if (is_not_external_pid(watcher))
+            goto invalid_message;
+        else if (external_pid_dist_entry(watcher) != dep)
+            goto invalid_message;
+
+	if (is_not_ref(ref))
 	    goto invalid_message;
-	}
 
-	if (is_atom(watched)) {
-	    name = watched;
-	    rp = erts_whereis_process(NULL, 0,
-				      watched, ERTS_PROC_LOCK_LINK,
-				      ERTS_P2P_FLG_ALLOW_OTHER_X);
-	}
-	else {
-	    name = NIL;
-	    rp = erts_pid2proc_opt(NULL, 0,
-				   watched, ERTS_PROC_LOCK_LINK,
-				   ERTS_P2P_FLG_ALLOW_OTHER_X);
-	}
+        if (is_internal_pid(watched)) {
+            name = NIL;
+            pid = watched;
+        }
+        else if (is_atom(watched)) {
+            name = watched;
+            pid = erts_whereis_name_to_id(NULL, watched);
+            /* if port or undefined; reply noproc... */
+        }
+        else if (is_external_pid(watched)
+                 && external_pid_dist_entry(watched) == erts_this_dist_entry) {
+            name = NIL;
+            pid = am_undefined; /* old incarnation; reply noproc... */
+        }
+        else
+            goto invalid_message;
 
-	if (!rp) {
-	    ErtsDSigData dsd;
-	    int code;
-	    code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0);
-	    if (code == ERTS_DSIG_PREP_CONNECTED) {
-		code = erts_dsig_send_m_exit(&dsd, watcher, watched, ref,
-					     am_noproc);
-		ASSERT(code == ERTS_DSIG_SEND_OK);
-	    }
-	}
-	else {
-	    if (is_atom(watched))
-		watched = rp->common.id;
-	    erts_de_links_lock(dep);
-	    erts_add_monitor(&(dep->monitors), MON_ORIGIN, ref, watched, name);
-	    erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, ref, watcher, name);
-	    erts_de_links_unlock(dep);
-	    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-	}
+        if (is_internal_pid(pid)) {
+            ErtsMonitorData *mdp;
+            mdp = erts_monitor_create(ERTS_MON_TYPE_DIST_PROC,
+                                      ref, watcher, pid, name);
 
-	break;
+#ifdef DEBUG
+            code =
+#endif
+                erts_monitor_dist_insert(&mdp->origin, dep->mld);
+            ASSERT(code);
+
+            if (erts_proc_sig_send_monitor(&mdp->target, pid))
+                break; /* done */
+
+            /* Failed to send to local proc; cleanup reply noproc... */
+
+#ifdef DEBUG
+            code =
+#endif
+                erts_monitor_dist_delete(&mdp->origin);
+            ASSERT(code);
+            erts_monitor_release_both(mdp);
+
+        }
+
+        code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0);
+        if (code == ERTS_DSIG_PREP_CONNECTED) {
+            code = erts_dsig_send_m_exit(&dsd, watcher, watched, ref,
+                                         am_noproc);
+            ASSERT(code == ERTS_DSIG_SEND_OK);
+        }
+
+        break;
     }
 
     case DOP_DEMONITOR_P:
@@ -1513,36 +1489,43 @@ int erts_net_message(Port *prt,
 	if (tuple_arity != 4) {
 	    goto invalid_message;
 	}
-	/* watcher = tuple[2]; */
-	/* watched = tuple[3]; May be an atom in case of monitor name */
+
+	watcher = tuple[2];
+	watched = tuple[3];
 	ref = tuple[4];
 
-	if(is_not_ref(ref)) {
+	if (is_not_ref(ref)) {
 	    goto invalid_message;
 	}
 
-	erts_de_links_lock(dep);
-	mon = erts_remove_monitor(&(dep->monitors),ref);
-	erts_de_links_unlock(dep);
-	/* ASSERT(mon != NULL); can happen in case of broken dist message */
-	if (mon == NULL) {
-	    break;
-	}
-	watched = mon->u.pid;
-	erts_destroy_monitor(mon);
-	rp = erts_pid2proc_opt(NULL, 0,
-			       watched, ERTS_PROC_LOCK_LINK,
-			       ERTS_P2P_FLG_ALLOW_OTHER_X);
-	if (!rp) {
-	    break;
-	}
-	mon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref);
-	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-	ASSERT(mon != NULL);
-	if (mon == NULL) {
-	    break;
-	}
-	erts_destroy_monitor(mon);
+        if (is_not_external_pid(watcher) || external_pid_dist_entry(watcher) != dep)
+            goto invalid_message;
+
+        if (is_internal_pid(watched))
+            erts_proc_sig_send_dist_demonitor(watched, ref);
+        else if (is_external_pid(watched)
+                 && external_pid_dist_entry(watched) == erts_this_dist_entry) {
+            /* old incarnation; ignore it */
+            ;
+        }
+        else if (is_atom(watched)) {
+            ErtsMonLnkDist *mld = dep->mld;
+            ErtsMonitor *mon;
+
+            erts_mtx_lock(&mld->mtx);
+
+            mon = erts_monitor_tree_lookup(mld->orig_name_monitors, ref);
+            if (mon)
+                erts_monitor_tree_delete(&mld->orig_name_monitors, mon);
+
+            erts_mtx_unlock(&mld->mtx);
+
+            if (mon)
+                erts_proc_sig_send_demonitor(mon);
+        }
+        else
+            goto invalid_message;
+
 	break;
 
     case DOP_REG_SEND_TT:
@@ -1669,68 +1652,36 @@ int erts_net_message(Port *prt,
 	/* We are monitoring a process on the remote node which dies, we get
 	   {DOP_MONITOR_P_EXIT, Remote pid or name, Local pid, ref, reason} */
 	   
-
-	DeclareTmpHeapNoproc(lhp,3);
-	Eterm sysname;
-	ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_MSG_SEND|ERTS_PROC_LOCK_LINK;
-
 	if (tuple_arity != 5) {
 	    goto invalid_message;
 	}
 
-	/* watched = tuple[2]; */  /* remote proc which died */
-	/* watcher = tuple[3]; */
+	watched = tuple[2];  /* remote proc or name which died */
+	watcher = tuple[3];
 	ref     = tuple[4];
 	reason  = tuple[5];
 
-	if(is_not_ref(ref)) {
+	if (is_not_ref(ref))
 	    goto invalid_message;
-	}
-
-	erts_de_links_lock(dep);
-	sysname = dep->sysname;
-	mon = erts_remove_monitor(&(dep->monitors), ref);
-	/*
-	 * If demonitor was performed at the same time as the
-	 * monitored process exits, monitoring side will have
-	 * removed info about monitor. In this case, do nothing
-	 * and everything will be as it should.
-	 */
-	erts_de_links_unlock(dep);
-	if (mon == NULL) {
-	    break;
-	}
-	rp = erts_pid2proc(NULL, 0, mon->u.pid, rp_locks);
 
-	erts_destroy_monitor(mon);
-	if (rp == NULL) {
-	    break;
-	}
+        if (is_not_external_pid(watched) && is_not_atom(watched))
+            goto invalid_message;
 
-	mon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref);
+        if (is_not_internal_pid(watcher)) {
+            if (!is_external_pid(watcher))
+                goto invalid_message;
+            if (erts_this_dist_entry == external_pid_dist_entry(watcher))
+                break;
+            goto invalid_message;
+        }
 
-	if (mon == NULL) {
-	    erts_proc_unlock(rp, rp_locks);
-	    break;
-	}
-	UseTmpHeapNoproc(3);
-	
-	watched = (is_not_nil(mon->name)
-		   ? TUPLE2(&lhp[0], mon->name, sysname)
-		   : mon->u.pid);
-	
-	erts_queue_monitor_message(rp, &rp_locks,
-				   ref, am_process, watched, reason);
-	erts_proc_unlock(rp, rp_locks);
-	erts_destroy_monitor(mon);
-	UnUseTmpHeapNoproc(3);
+        erts_proc_sig_send_dist_monitor_down(dep, ref, watched,
+                                             watcher, reason);
 	break;
     }
 
     case DOP_EXIT_TT:
     case DOP_EXIT: {
-	ErtsDistLinkData dld;
-	ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCKS_XSIG_SEND;
 	/* 'from', which 'to' is linked to, died */
 	if (type == DOP_EXIT) {
 	    if (tuple_arity != 4) {
@@ -1750,56 +1701,19 @@ int erts_net_message(Port *prt,
 	    token = tuple[4];
 	    reason = tuple[5];
 	}
-	if (is_not_pid(from) || is_not_internal_pid(to)) {
+	if (is_not_external_pid(from)
+            || dep != external_pid_dist_entry(from)
+            || is_not_internal_pid(to)) {
 	    goto invalid_message;
 	}
 
-	rp = erts_pid2proc(NULL, 0, to, rp_locks);
-	if (!rp)
-	    lnk = NULL;
-	else {
-	    lnk = erts_remove_link(&ERTS_P_LINKS(rp), from);
-
-	    /* If lnk == NULL, we have unlinked on this side, i.e.
-	     * ignore exit.
-	     */
-	    if (lnk) {
-		int xres;
-#if 0
-		/* Arndt: Maybe it should never be 'kill', but it can be,
-		   namely when a linked process does exit(kill). Until we know
-		   whether that is incorrect and what should happen instead,
-		   we leave the assertion out. */
-		ASSERT(reason != am_kill); /* should never be kill (killed) */
-#endif
-		xres = erts_send_exit_signal(NULL,
-					     from,
-					     rp,
-					     &rp_locks, 
-					     reason,
-					     token,
-					     NULL,
-					     ERTS_XSIG_FLG_IGN_KILL);
-		if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) {
-		    /* We didn't exit the process and it is traced */
-                    if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) {
-                        erts_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND);
-                        rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND;
-                    }
-		    trace_proc(NULL, 0, rp, am_getting_unlinked, from);
-		}
-	    }
-	    erts_proc_unlock(rp, rp_locks);
-	}
-	erts_remove_dist_link(&dld, to, from, dep);
-	if (lnk)
-	    erts_destroy_link(lnk);
-	erts_destroy_dist_link(&dld);
+        erts_proc_sig_send_dist_link_exit(dep,
+                                          from, to,
+                                          reason, token);
 	break;
     }
     case DOP_EXIT2_TT:
-    case DOP_EXIT2: {
-	ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
+    case DOP_EXIT2:
 	/* 'from' is send an exit signal to 'to' */
 	if (type == DOP_EXIT2) {
 	    if (tuple_arity != 4) {
@@ -1821,20 +1735,10 @@ int erts_net_message(Port *prt,
 	if (is_not_pid(from) || is_not_internal_pid(to)) {
 	    goto invalid_message;
 	}
-	rp = erts_pid2proc(NULL, 0, to, rp_locks);
-	if (rp) {
-	    (void) erts_send_exit_signal(NULL,
-					 from,
-					 rp,
-					 &rp_locks,
-					 reason,
-					 token,
-					 NULL,
-					 0);
-	    erts_proc_unlock(rp, rp_locks);
-	}
+
+        erts_proc_sig_send_exit(NULL, from, to, reason, token, 0);
 	break;
-    }
+
     case DOP_GROUP_LEADER:
 	if (tuple_arity != 3) {
 	    goto invalid_message;
@@ -1845,11 +1749,7 @@ int erts_net_message(Port *prt,
 	    goto invalid_message;
 	}
 
-	rp = erts_pid2proc(NULL, 0, to, ERTS_PROC_LOCK_MAIN);
-	if (!rp)
-	    break;
-	rp->group_leader = STORE_NC_IN_PROC(rp, from);
-	erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
+        (void) erts_proc_sig_send_group_leader(NULL, to, from, NIL);
 	break;
 
     default: 
@@ -2989,57 +2889,55 @@ static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp)
 {
     fmtfn_t to = ((struct print_to_data *) vptdp)->to;
     void *arg = ((struct print_to_data *) vptdp)->arg;
-    Process *rp;
-    ErtsMonitor *rmon;
-    rp = erts_proc_lookup(mon->u.pid);
-    if (!rp || (rmon = erts_lookup_monitor(ERTS_P_MONITORS(rp), mon->ref)) == NULL) {
-	erts_print(to, arg, "Warning, stray monitor for: %T\n", mon->u.pid);
-    } else if (mon->type == MON_ORIGIN) {
-	/* Local pid is being monitored */
+    ErtsMonitorDataExtended *mdep;
+
+    ASSERT(mon->flags & ERTS_ML_FLG_EXTENDED);
+
+    mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon);
+
+    ASSERT(mdep->dist);
+
+    if (erts_monitor_is_origin(mon)) {
 	erts_print(to, arg, "Remotely monitored by: %T %T\n",
-		   mon->u.pid, rmon->u.pid);
-    } else {
-	erts_print(to, arg, "Remote monitoring: %T ", mon->u.pid);
-	if (is_not_atom(rmon->u.pid))
-	    erts_print(to, arg, "%T\n", rmon->u.pid);
-	else
-	    erts_print(to, arg, "{%T, %T}\n",
-		       rmon->name,
-		       rmon->u.pid); /* which in this case is the 
-				      remote system name... */
+		   mon->other.item, mdep->md.target.other.item);
+    }
+    else {
+	erts_print(to, arg, "Remote monitoring: %T ", mon->other.item);
+        if (mon->flags & ERTS_ML_FLG_NAME)
+	    erts_print(to, arg, "{%T, %T}\n", mdep->u.name, mdep->dist->nodename);
+        else
+	    erts_print(to, arg, "%T\n", mdep->md.origin.other.item);
     }
 }    
 
-static void print_monitor_info(fmtfn_t to, void *arg, ErtsMonitor *mon)
+static void print_monitor_info(fmtfn_t to, void *arg, DistEntry *dep)
 {
     struct print_to_data ptd = {to, arg};
-    erts_doforall_monitors(mon,&doit_print_monitor_info,&ptd);
-}
-
-typedef struct {
-    struct print_to_data *ptdp;
-    Eterm from;
-} PrintLinkContext;
-
-static void doit_print_link_info2(ErtsLink *lnk, void *vpplc)
-{
-    PrintLinkContext *pplc = (PrintLinkContext *) vpplc;
-    erts_print(pplc->ptdp->to, pplc->ptdp->arg, "Remote link: %T %T\n",
-	       pplc->from, lnk->pid);
+    if (dep->mld) {
+        erts_monitor_list_foreach(dep->mld->monitors,
+                                  doit_print_monitor_info,
+                                  (void *) &ptd);
+        erts_monitor_tree_foreach(dep->mld->orig_name_monitors,
+                                  doit_print_monitor_info,
+                                  (void *) &ptd);
+    }
 }
 
 static void doit_print_link_info(ErtsLink *lnk, void *vptdp)
 {
-    if (is_internal_pid(lnk->pid) && erts_proc_lookup(lnk->pid)) {
-	PrintLinkContext plc = {(struct print_to_data *) vptdp, lnk->pid};
-	erts_doforall_links(ERTS_LINK_ROOT(lnk), &doit_print_link_info2, &plc);
-    } 
+    struct print_to_data *ptdp = vptdp;
+    ErtsLink *lnk2 = erts_link_to_other(lnk, NULL);
+    erts_print(ptdp->to, ptdp->arg, "Remote link: %T %T\n",
+	       lnk2->other.item, lnk->other.item);
 }
 
-static void print_link_info(fmtfn_t to, void *arg, ErtsLink *lnk)
+static void print_link_info(fmtfn_t to, void *arg, DistEntry *dep)
 {
     struct print_to_data ptd = {to, arg};
-    erts_doforall_links(lnk, &doit_print_link_info, (void *) &ptd);
+    if (dep->mld)
+        erts_link_list_foreach(dep->mld->links,
+                               doit_print_link_info,
+                               (void *) &ptd);
 }
 
 typedef struct {
@@ -3047,23 +2945,6 @@ typedef struct {
     Eterm sysname;
 } PrintNodeLinkContext;
     
-
-static void doit_print_nodelink_info(ErtsLink *lnk, void *vpcontext)
-{
-    PrintNodeLinkContext *pcontext = vpcontext;
-
-    if (is_internal_pid(lnk->pid) && erts_proc_lookup(lnk->pid))
-	erts_print(pcontext->ptd.to, pcontext->ptd.arg,
-		   "Remote monitoring: %T %T\n", lnk->pid, pcontext->sysname);
-}
-
-static void print_nodelink_info(fmtfn_t to, void *arg, ErtsLink *lnk, Eterm sysname)
-{
-    PrintNodeLinkContext context = {{to, arg}, sysname};
-    erts_doforall_links(lnk, &doit_print_nodelink_info, &context);
-}
-
-
 static int
 info_dist_entry(fmtfn_t to, void *arg, DistEntry *dep, int visible, int connected)
 {
@@ -3094,8 +2975,8 @@ info_dist_entry(fmtfn_t to, void *arg, DistEntry *dep, int visible, int connecte
   erts_print(to, arg, "Name: %T", dep->sysname);
   erts_print(to, arg, "\n");
   if (!connected && is_nil(dep->cid)) {
-    if (dep->nlinks) {
-      erts_print(to, arg, "Error: Got links to not connected node:%T\n",
+    if (dep->mld) {
+      erts_print(to, arg, "Error: Got links/monitors to not connected node:%T\n",
 		 dep->sysname);
     }
     return 0;
@@ -3104,9 +2985,8 @@ info_dist_entry(fmtfn_t to, void *arg, DistEntry *dep, int visible, int connecte
   erts_print(to, arg, "Controller: %T\n", dep->cid, to);
 
   erts_print_node_info(to, arg, dep->sysname, NULL, NULL);
-  print_monitor_info(to, arg, dep->monitors);
-  print_link_info(to, arg, dep->nlinks);
-  print_nodelink_info(to, arg, dep->node_links, dep->sysname);
+  print_monitor_info(to, arg, dep);
+  print_link_info(to, arg, dep);
 
   return 0;
     
@@ -3193,8 +3073,7 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2)
 	goto error;
 
     /* Check that all trap functions are defined !! */
-    if (dmonitor_node_trap->addressv[0] == NULL ||
-	dmonitor_p_trap->addressv[0] == NULL) {
+    if (dmonitor_node_trap->addressv[0] == NULL) {
 	goto error;
     }
 
@@ -3582,25 +3461,16 @@ static Sint abort_connection(DistEntry* dep, Uint32 conn_id)
         kill_connection(dep);
     }
     else if (dep->state == ERTS_DE_STATE_PENDING) {
-        NetExitsContext nec = {dep};
-        ErtsLink *nlinks;
-        ErtsLink *node_links;
-        ErtsMonitor *monitors;
         ErtsAtomCache *cache;
         ErtsDistOutputBuf *obuf;
         ErtsProcList *resume_procs;
         Sint reds = 0;
+        ErtsMonLnkDist *mld;
 
 	ASSERT(is_nil(dep->cid));
 
-	erts_de_links_lock(dep);
-	monitors	= dep->monitors;
-	nlinks		= dep->nlinks;
-	node_links	= dep->node_links;
-	dep->monitors	= NULL;
-	dep->nlinks	= NULL;
-	dep->node_links	= NULL;
-	erts_de_links_unlock(dep);
+        mld = dep->mld;
+        dep->mld = NULL;
 
 	cache = dep->cache;
 	dep->cache = NULL;
@@ -3619,9 +3489,8 @@ static Sint abort_connection(DistEntry* dep, Uint32 conn_id)
 
 	erts_de_rwunlock(dep);
 
-	erts_sweep_monitors(monitors, &doit_monitor_net_exits, &nec);
-	erts_sweep_links(nlinks, &doit_link_net_exits, &nec);
-	erts_sweep_links(node_links, &doit_node_link_net_exits, &nec);
+        schedule_con_monitor_link_cleanup(mld, THE_NON_VALUE,
+                                          THE_NON_VALUE, THE_NON_VALUE);
 
         if (resume_procs) {
             int resumed = erts_resume_processes(resume_procs);
@@ -3861,8 +3730,8 @@ BIF_RETTYPE is_alive_0(BIF_ALIST_0)
 static BIF_RETTYPE
 monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options)
 {
-    DistEntry *dep;
-    ErtsLink *lnk;
+    BIF_RETTYPE ret;
+    DistEntry *dep = NULL;
     Eterm l;
     int async_connect = 1;
 
@@ -3881,33 +3750,71 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options)
     if (l != NIL) {
 	BIF_ERROR(p, BADARG);
     }
+    if (l != NIL)
+        goto badarg;
 
-    if (is_not_atom(Node) ||
-	((Bool != am_true) && (Bool != am_false)) ||
-	((erts_this_node->sysname == am_Noname)
-	 && (Node != erts_this_node->sysname))) {
-	BIF_ERROR(p, BADARG);
+    if (is_not_atom(Node))
+        goto badarg;
+
+    if (erts_this_node->sysname == am_Noname && Node != am_Noname)
+	goto badarg;
+
+    switch (Bool) {
+
+    case am_false: {
+        ErtsMonitor *mon;
+        /*
+         * Before OTP-21, monitor_node(Node, false) triggered
+         * auto-connect and a 'nodedown' message if that failed.
+         * Now it's a simple no-op which feels more reasonable.
+         */
+        mon = erts_monitor_tree_lookup(ERTS_P_MONITORS(p), Node);
+        if (mon) {
+            ErtsMonitorDataExtended *mdep;
+            ASSERT(erts_monitor_is_origin(mon));
+
+            mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon);
+
+            ASSERT((mdep->u.refc > 0));
+            if (--mdep->u.refc == 0) {
+                if (!mdep->uptr.node_monitors)
+                    erts_monitor_tree_delete(&ERTS_P_MONITORS(p), mon);
+                else {
+                    ErtsMonitor *sub_mon;
+                    ErtsMonitorDataExtended *sub_mdep;
+                    sub_mon = erts_monitor_list_last(mdep->uptr.node_monitors);
+                    erts_monitor_list_delete(&mdep->uptr.node_monitors, sub_mon);
+                    sub_mon->flags &= ~ERTS_ML_FLG_IN_SUBTABLE;
+                    sub_mdep = ((ErtsMonitorDataExtended *)
+                                erts_monitor_to_data(sub_mon));
+                    sub_mdep->uptr.node_monitors = mdep->uptr.node_monitors;
+                    mdep->uptr.node_monitors = NULL;
+                    erts_monitor_tree_replace(&ERTS_P_MONITORS(p), mon, sub_mon);
+                }
+                if (erts_monitor_dist_delete(&mdep->md.target))
+                    erts_monitor_release_both((ErtsMonitorData *) mdep);
+                else
+                    erts_monitor_release(mon);
+            }
+        }
+        break;
     }
 
-    if (Bool == am_true) {
+    case am_true: {
         ErtsDSigData dsd;
         dsd.node = Node;
+
         dep = erts_find_or_insert_dist_entry(Node);
         if (dep == erts_this_dist_entry)
-            goto done;
-
-        erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
+            break;
 
         switch (erts_dsig_prepare(&dsd, dep, p,
-                                  (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK),
+                                  ERTS_PROC_LOCK_MAIN,
                                   ERTS_DSP_RLOCK, 0, async_connect)) {
         case ERTS_DSIG_PREP_NOT_ALIVE:
 	case ERTS_DSIG_PREP_NOT_CONNECTED:
 	    /* Trap to either send 'nodedown' or do passive connection attempt */
-	trap:
-            erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
-	    erts_deref_dist_entry(dep);
-            BIF_TRAP3(dmonitor_node_trap, p, Node, Bool, Options);
+            goto do_trap;
 	case ERTS_DSIG_PREP_PENDING:
 	    if (!async_connect) {
 		/*
@@ -3915,68 +3822,87 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options)
 		 * to ensure passive connection attempt
 		 */
 		erts_de_runlock(dep);
-		goto trap;
+		goto do_trap;
 	    }
 	    /*fall through*/
-        case ERTS_DSIG_PREP_CONNECTED:
-            erts_de_links_lock(dep);
-            erts_de_runlock(dep);
-            lnk = erts_add_or_lookup_link(&(dep->node_links), LINK_NODE,
-                                          p->common.id);
-            ++ERTS_LINK_REFC(lnk);
-            lnk = erts_add_or_lookup_link(&ERTS_P_LINKS(p), LINK_NODE, Node);
-            ++ERTS_LINK_REFC(lnk);
-            erts_de_links_unlock(dep);
-            break;
-        default:
-            ERTS_ASSERT(! "Invalid dsig prepare result");
-        }
-        erts_deref_dist_entry(dep);
-    }
-    else { /* Bool == false */
-        dep = erts_sysname_to_connected_dist_entry(Node);
-        if (!dep) {
-	    /*
-	     * Before OTP-21 this case triggered auto-connect
-	     * and a 'nodedown' message if that failed.
-	     * Now it's a simple no-op which feels more reasonable.
-	     */
-            BIF_RET(am_true);
+        case ERTS_DSIG_PREP_CONNECTED: {
+            ErtsMonitor *mon;
+            ErtsMonitorDataExtended *mdep;
+            int created;
+
+            mon = erts_monitor_tree_lookup_create(&ERTS_P_MONITORS(p),
+                                                  &created,
+                                                  ERTS_MON_TYPE_NODE,
+                                                  p->common.id,
+                                                  Node);
+            mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon);
+            if (created) {
+#ifdef DEBUG
+                int inserted =
+#endif
+                    erts_monitor_dist_insert(&mdep->md.target, dep->mld);
+                ASSERT(inserted);
+                ASSERT(mdep->dist->connection_id == dep->connection_id);
+            }
+            else if (mdep->dist->connection_id != dep->connection_id) {
+                ErtsMonitorDataExtended *mdep2;
+                ErtsMonitor *mon2;
+#ifdef DEBUG
+                int inserted;
+#endif
+                mdep2 = ((ErtsMonitorDataExtended *)
+                         erts_monitor_create(ERTS_MON_TYPE_NODE, NIL,
+                                             p->common.id, Node, NIL));
+                mon2 = &mdep2->md.origin;
+#ifdef DEBUG
+                inserted =
+#endif
+                    erts_monitor_dist_insert(&mdep->md.target, dep->mld);
+                ASSERT(inserted);
+                ASSERT(mdep2->dist->connection_id == dep->connection_id);
+
+                mdep2->uptr.node_monitors = mdep->uptr.node_monitors;
+                mdep->uptr.node_monitors = NULL;
+                erts_monitor_tree_replace(&ERTS_P_MONITORS(p), mon, mon2);
+                erts_monitor_list_insert(&mdep2->uptr.node_monitors, mon);
+                mon->flags |= ERTS_ML_FLG_IN_SUBTABLE;
+                mdep = mdep2;
+            }
+
+            mdep->u.refc++;
+
+            break;
         }
-        if (dep == erts_this_dist_entry)
-            goto done;
 
-        erts_proc_lock(p, ERTS_PROC_LOCK_LINK);
-        erts_de_rlock(dep);
-        if (dep->state == ERTS_DE_STATE_IDLE) {
-            ASSERT(!dep->node_links);
-            erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
-            erts_de_runlock(dep);
-            goto done;
+        default:
+            ERTS_ASSERT(! "Invalid dsig prepare result");
         }
-        erts_de_links_lock(dep);
+
         erts_de_runlock(dep);
-        lnk = erts_lookup_link(dep->node_links, p->common.id);
-        if (lnk != NULL) {
-            if ((--ERTS_LINK_REFC(lnk)) == 0) {
-                erts_destroy_link(erts_remove_link(&(dep->node_links),
-                                                   p->common.id));
-            }
-        }
-        lnk = erts_lookup_link(ERTS_P_LINKS(p), Node);
-        if (lnk != NULL) {
-            if ((--ERTS_LINK_REFC(lnk)) == 0) {
-                erts_destroy_link(erts_remove_link(&ERTS_P_LINKS(p),
-                                                   Node));
-            }
-        }
-        erts_de_links_unlock(dep);
+
+        break;
     }
 
-    erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+    default:
+        goto badarg;
+    }
 
- done:
-    BIF_RET(am_true);
+    ERTS_BIF_PREP_RET(ret, am_true);
+
+do_return:
+
+    if (dep)
+        erts_deref_dist_entry(dep);
+
+    return ret;
+
+do_trap:
+    ERTS_BIF_PREP_TRAP3(ret, dmonitor_node_trap, p, Node, Bool, Options);
+    goto do_return;
+
+badarg:
+    ERTS_BIF_PREP_ERROR(ret, p, BADARG);
+    goto do_return;
 }
 
 BIF_RETTYPE monitor_node_3(BIF_ALIST_3)
@@ -4027,18 +3953,9 @@ BIF_RETTYPE net_kernel_dflag_unicode_io_1(BIF_ALIST_1)
 #define ERTS_NODES_MON_OPT_TYPES \
   (ERTS_NODES_MON_OPT_TYPE_VISIBLE|ERTS_NODES_MON_OPT_TYPE_HIDDEN)
 
-typedef struct ErtsNodesMonitor_ ErtsNodesMonitor;
-struct ErtsNodesMonitor_ {
-    ErtsNodesMonitor *prev;
-    ErtsNodesMonitor *next;
-    Process *proc;
-    Uint16 opts;
-    Uint16 no;
-};
-
 static erts_mtx_t nodes_monitors_mtx;
-static ErtsNodesMonitor *nodes_monitors;
-static ErtsNodesMonitor *nodes_monitors_end;
+static ErtsMonitor *nodes_monitors;
+static Uint no_nodes_monitors;
 
 /*
  * Nodes monitors are stored in a double linked list. 'nodes_monitors'
@@ -4057,109 +3974,169 @@ init_nodes_monitors(void)
     erts_mtx_init(&nodes_monitors_mtx, "nodes_monitors", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
     nodes_monitors = NULL;
-    nodes_monitors_end = NULL;
+    no_nodes_monitors = 0;
 }
 
-static ERTS_INLINE Uint
-nodes_mon_msg_sz(ErtsNodesMonitor *nmp, Eterm what, Eterm reason)
+Eterm
+erts_monitor_nodes(Process *c_p, Eterm on, Eterm olist)
 {
-    Uint sz;
-    if (!nmp->opts) {
-	sz = 3;
-    }
-    else {
-	sz = 0;
+    Eterm key, old_value, opts_list = olist;
+    Uint opts = (Uint) 0;
+
+    ASSERT(c_p);
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
 
-	if (nmp->opts & ERTS_NODES_MON_OPT_TYPES)
-	    sz += 2 + 3;
+    if (on != am_true && on != am_false)
+	return THE_NON_VALUE;
+
+    if (is_not_nil(opts_list)) {
+	int all = 0, visible = 0, hidden = 0;
 
-	if (what == am_nodedown
-	    && (nmp->opts & ERTS_NODES_MON_OPT_DOWN_REASON)) {
-	    if (is_not_immed(reason))
-		sz += size_object(reason);
-	    sz += 2 + 3;
+	while (is_list(opts_list)) {
+	    Eterm *cp = list_val(opts_list);
+	    Eterm opt = CAR(cp);
+	    opts_list = CDR(cp);
+	    if (opt == am_nodedown_reason)
+		opts |= ERTS_NODES_MON_OPT_DOWN_REASON;
+	    else if (is_tuple(opt)) {
+		Eterm* tp = tuple_val(opt);
+		if (arityval(tp[0]) != 2)
+		    return THE_NON_VALUE;
+		switch (tp[1]) {
+		case am_node_type:
+		    switch (tp[2]) {
+		    case am_visible:
+			if (hidden || all)
+			    return THE_NON_VALUE;
+			opts |= ERTS_NODES_MON_OPT_TYPE_VISIBLE;
+			visible = 1;
+			break;
+		    case am_hidden:
+			if (visible || all)
+			    return THE_NON_VALUE;
+			opts |= ERTS_NODES_MON_OPT_TYPE_HIDDEN;
+			hidden = 1;
+			break;
+		    case am_all:
+			if (visible || hidden)
+			    return THE_NON_VALUE;
+			opts |= ERTS_NODES_MON_OPT_TYPES;
+			all = 1;
+			break;
+		    default:
+			return THE_NON_VALUE;
+		    }
+		    break;
+		default:
+		    return THE_NON_VALUE;
+		}
+	    }
+	    else {
+		return THE_NON_VALUE;
+	    }
 	}
 
-	sz += 4;
+	if (is_not_nil(opts_list))
+	    return THE_NON_VALUE;
+    }
+
+    key = make_small(opts);
+
+    if (on == am_true) {
+        ErtsMonitorDataExtended *mdep;
+        ErtsMonitor *omon;
+        int created;
+        omon = erts_monitor_tree_lookup_create(&ERTS_P_MONITORS(c_p),
+                                               &created,
+                                               ERTS_MON_TYPE_NODES,
+                                               c_p->common.id,
+                                               key);
+        mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(omon);
+        if (created) {
+            erts_mtx_lock(&nodes_monitors_mtx);
+            no_nodes_monitors++;
+            erts_monitor_list_insert(&nodes_monitors, &mdep->md.target);
+            erts_mtx_unlock(&nodes_monitors_mtx);
+        }
+        old_value = mdep->u.refc;
+        mdep->u.refc++;
+    }
+    else {
+        ErtsMonitorDataExtended *mdep;
+        ErtsMonitor *omon;
+        omon = erts_monitor_tree_lookup(ERTS_P_MONITORS(c_p), key);
+        if (!omon)
+            old_value = 0;
+        else {
+            mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(omon);
+            old_value = mdep->u.refc;
+            ASSERT(mdep->u.refc > 0);
+            erts_mtx_lock(&nodes_monitors_mtx);
+            ASSERT(no_nodes_monitors > 0);
+            no_nodes_monitors--;
+            ASSERT(erts_monitor_is_in_table(&mdep->md.target));
+            erts_monitor_list_delete(&nodes_monitors, &mdep->md.target);
+            erts_mtx_unlock(&nodes_monitors_mtx);
+            erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), omon);
+            erts_monitor_release_both((ErtsMonitorData *) mdep);
+        }
     }
-    return sz;
+
+    return erts_make_integer(old_value, c_p);
 }
 
-static ERTS_INLINE void
-send_nodes_mon_msg(Process *rp,
-		   ErtsProcLocks *rp_locksp,
-		   ErtsNodesMonitor *nmp,
-		   Eterm node,
-		   Eterm what,
-		   Eterm type,
-		   Eterm reason,
-		   Uint sz)
+void
+erts_monitor_nodes_delete(ErtsMonitor *omon)
 {
-    Eterm msg;
-    Eterm *hp;
-    ErtsMessage *mp;
-    ErlOffHeap *ohp;
-#ifdef DEBUG
-    Eterm *hend;
-#endif
+    ErtsMonitorData *mdp;
 
-    mp = erts_alloc_message_heap(rp, rp_locksp, sz, &hp, &ohp);
-#ifdef DEBUG
-    hend = hp + sz;
-#endif
+    ASSERT(omon->type == ERTS_MON_TYPE_NODES);
+    ASSERT(erts_monitor_is_origin(omon));
 
-    if (!nmp->opts) {
-	msg = TUPLE2(hp, what, node);
-#ifdef DEBUG
-	hp += 3;
-#endif
-    }
-    else {
-	Eterm tup;
-	Eterm info = NIL;
+    mdp = erts_monitor_to_data(omon);
 
-	if (nmp->opts & (ERTS_NODES_MON_OPT_TYPE_VISIBLE
-			 | ERTS_NODES_MON_OPT_TYPE_HIDDEN)) {
+    erts_mtx_lock(&nodes_monitors_mtx);
+    ASSERT(erts_monitor_is_in_table(&mdp->target));
+    ASSERT(no_nodes_monitors > 0);
+    no_nodes_monitors--;
+    erts_monitor_list_delete(&nodes_monitors, &mdp->target);
+    erts_mtx_unlock(&nodes_monitors_mtx);
+    erts_monitor_release_both(mdp);
+}
 
-	    tup = TUPLE2(hp, am_node_type, type);
-	    hp += 3;
-	    info = CONS(hp, tup, info);
-	    hp += 2;
-	}
+typedef struct {
+    Eterm pid;
+    Eterm options;
+} ErtsNodesMonitorData;
 
-	if (what == am_nodedown
-	    && (nmp->opts & ERTS_NODES_MON_OPT_DOWN_REASON)) {
-	    Eterm rsn_cpy;
-	    
-	    if (is_immed(reason))
-		rsn_cpy = reason;
-	    else {
-		Eterm rsn_sz = size_object(reason);
-		rsn_cpy = copy_struct(reason, rsn_sz, &hp, ohp);
-	    }
+typedef struct {
+    ErtsNodesMonitorData *nmdp;
+    Uint i;
+} ErtsNodesMonitorContext;
 
-	    tup = TUPLE2(hp, am_nodedown_reason, rsn_cpy);
-	    hp += 3;
-	    info = CONS(hp, tup, info);
-	    hp += 2;
-	}
+static void
+save_nodes_monitor(ErtsMonitor *mon, void *vctxt)
+{
+    ErtsNodesMonitorContext *ctxt = vctxt;
+    ErtsMonitorData *mdp = erts_monitor_to_data(mon);
 
-	msg = TUPLE3(hp, what, node, info);
-#ifdef DEBUG
-	hp += 4;
-#endif
-    }
+    ASSERT(erts_monitor_is_target(mon));
+    ASSERT(mon->type == ERTS_MON_TYPE_NODES);
+
+    ctxt->nmdp[ctxt->i].pid = mon->other.item;
+    ctxt->nmdp[ctxt->i].options = mdp->origin.other.item;
 
-    ASSERT(hend == hp);
-    erts_queue_message(rp, *rp_locksp, mp, msg, am_system);
+    ctxt->i++;
 }
 
 static void
 send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reason)
 {
-    ErtsNodesMonitor *nmp;
-    ErtsProcLocks rp_locks = 0; /* Init to shut up false warning */
-    Process *rp = NULL;
+    Uint opts;
+    Uint i, no, reason_size;
+    ErtsNodesMonitorData def_buf[100];
+    ErtsNodesMonitorData *nmdp = &def_buf[0];
+    ErtsNodesMonitorContext ctxt;
 
     ASSERT(is_immed(what));
     ASSERT(is_immed(node));
@@ -4180,31 +4157,44 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas
     }
 #endif
 
-    ERTS_LC_ASSERT(!c_p
-		       || (erts_proc_lc_my_proc_locks(c_p)
-			   == ERTS_PROC_LOCK_MAIN));
+    ctxt.i = 0;
+
+    reason_size = is_immed(reason) ? 0 : size_object(reason);
+
     erts_mtx_lock(&nodes_monitors_mtx);
+    if (no_nodes_monitors > sizeof(def_buf)/sizeof(def_buf[0]))
+        nmdp = erts_alloc(ERTS_ALC_T_TMP,
+                          no_nodes_monitors*sizeof(ErtsNodesMonitorData));
+    ctxt.nmdp = nmdp;
+    erts_monitor_list_foreach(nodes_monitors,
+                              save_nodes_monitor,
+                              (void *) &ctxt);
+    
+    ASSERT(ctxt.i == no_nodes_monitors);
+    no = no_nodes_monitors;
 
-    for (nmp = nodes_monitors; nmp; nmp = nmp->next) {
-	int i;
-	Uint16 no;
-	Uint sz;
+    erts_mtx_unlock(&nodes_monitors_mtx);
 
-	ASSERT(nmp->proc != NULL);
+    for (i = 0; i < no; i++) {
+        Eterm tmp_heap[3+2+3+2+4 /* max need */];
+        Eterm *hp, msg;
+        Uint hsz;
 
-	if (!nmp->opts) {
+        ASSERT(is_small(nmdp[i].options));
+        opts = (Uint) signed_val(nmdp[i].options);
+	if (!opts) {
 	    if (type != am_visible)
 		continue;
 	}
 	else {
 	    switch (type) {
 	    case am_hidden:
-		if (!(nmp->opts & ERTS_NODES_MON_OPT_TYPE_HIDDEN))
+		if (!(opts & ERTS_NODES_MON_OPT_TYPE_HIDDEN))
 		    continue;
 		break;
 	    case am_visible:
-		if ((nmp->opts & ERTS_NODES_MON_OPT_TYPES)
-		    && !(nmp->opts & ERTS_NODES_MON_OPT_TYPE_VISIBLE))
+		if ((opts & ERTS_NODES_MON_OPT_TYPES)
+		    && !(opts & ERTS_NODES_MON_OPT_TYPE_VISIBLE))
 		    continue;
 		break;
 	    default:
@@ -4212,342 +4202,162 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas
 	    }
 	}
 
-	if (rp != nmp->proc) {
-	    if (rp) {
-		if (rp == c_p)
-		    rp_locks &= ~ERTS_PROC_LOCK_MAIN;
-		erts_proc_unlock(rp, rp_locks);
-	    }
-
-	    rp = nmp->proc;
-	    rp_locks = 0;
-	    if (rp == c_p)
-		rp_locks |= ERTS_PROC_LOCK_MAIN;
-	}
+        hsz = 0;
+        hp = &tmp_heap[0];
 
-	ASSERT(rp);
-
-	sz = nodes_mon_msg_sz(nmp, what, reason);
+        if (!opts) {
+            msg = TUPLE2(hp, what, node);
+            hp += 3;
+        }
+        else {
+            Eterm tup;
+            Eterm info = NIL;
 
-	for (i = 0, no = nmp->no; i < no; i++)
-	    send_nodes_mon_msg(rp,
-			       &rp_locks,
-			       nmp,
-			       node,
-			       what,
-			       type,
-			       reason,
-			       sz);
-    }
+            if (opts & (ERTS_NODES_MON_OPT_TYPE_VISIBLE
+                        | ERTS_NODES_MON_OPT_TYPE_HIDDEN)) {
 
-    if (rp) {
-	if (rp == c_p)
-	    rp_locks &= ~ERTS_PROC_LOCK_MAIN;
-	erts_proc_unlock(rp, rp_locks);
-    }
-
-    erts_mtx_unlock(&nodes_monitors_mtx);
-}
+                tup = TUPLE2(hp, am_node_type, type);
+                hp += 3;
+                info = CONS(hp, tup, info);
+                hp += 2;
+            }
 
-static Eterm
-insert_nodes_monitor(Process *c_p, Uint32 opts)
-{
-    Uint16 no = 1;
-    Eterm res = am_false;
-    ErtsNodesMonitor *xnmp, *nmp;
+            if (what == am_nodedown
+                && (opts & ERTS_NODES_MON_OPT_DOWN_REASON)) {
+                hsz += reason_size;
+                tup = TUPLE2(hp, am_nodedown_reason, reason);
+                hp += 3;
+                info = CONS(hp, tup, info);
+                hp += 2;
+            }
 
-    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&nodes_monitors_mtx));
-    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN);
+            msg = TUPLE3(hp, what, node, info);
+            hp += 4;
+        }
 
-    xnmp = c_p->nodes_monitors;
-    if (xnmp) {
-	ASSERT(!xnmp->prev || xnmp->prev->proc != c_p);
+        ASSERT(hp - &tmp_heap[0] <= sizeof(tmp_heap)/sizeof(tmp_heap[0]));
 
-	while (1) {
-	    ASSERT(xnmp->proc == c_p);
-	    if (xnmp->opts == opts)
-		break;
-	    if (!xnmp->next || xnmp->next->proc != c_p)
-		break;
-	    xnmp = xnmp->next;
-	}
-	ASSERT(xnmp);
-	ASSERT(xnmp->proc == c_p);
-	ASSERT(xnmp->opts == opts
-	       || !xnmp->next
-	       || xnmp->next->proc != c_p);
-
-	if (xnmp->opts != opts)
-	    goto alloc_new;
-	else {
-	    res = am_true;
-	    no = xnmp->no++;
-	    if (!xnmp->no) {
-		/*
-		 * 'no' wrapped; transfer all prevous monitors to new
-		 * element (which will be the next element in the list)
-		 * and set this to one...
-		 */
-		xnmp->no = 1;
-		goto alloc_new;
-	    }
-	}
-    }
-    else {
-    alloc_new:
-	nmp = erts_alloc(ERTS_ALC_T_NODES_MON, sizeof(ErtsNodesMonitor));
-	nmp->proc = c_p;
-	nmp->opts = opts;
-	nmp->no = no;
-
-	if (xnmp) {
-	    ASSERT(nodes_monitors);
-	    ASSERT(c_p->nodes_monitors);
-	    nmp->next = xnmp->next;
-	    nmp->prev = xnmp;
-	    xnmp->next = nmp;
-	    if (nmp->next) {
-		ASSERT(nodes_monitors_end != xnmp);
-		ASSERT(nmp->next->prev == xnmp);
-		nmp->next->prev = nmp;
-	    }
-	    else {
-		ASSERT(nodes_monitors_end == xnmp);
-		nodes_monitors_end = nmp;
-	    }
-	}
-	else {
-	    ASSERT(!c_p->nodes_monitors);
-	    c_p->nodes_monitors = nmp;
-	    nmp->next = NULL;
-	    nmp->prev = nodes_monitors_end;
-	    if (nodes_monitors_end) {
-		ASSERT(nodes_monitors);
-		nodes_monitors_end->next = nmp;
-	    }
-	    else {
-		ASSERT(!nodes_monitors);
-		nodes_monitors = nmp;
-	    }
-	    nodes_monitors_end = nmp;
-	}
-    }
-    return res;
-}
+        hsz += hp - &tmp_heap[0];
 
-static Eterm
-remove_nodes_monitors(Process *c_p, Uint32 opts, int all)
-{
-    Eterm res = am_false;
-    ErtsNodesMonitor *nmp;
-
-    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&nodes_monitors_mtx));
-    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN);
-
-    nmp = c_p->nodes_monitors;
-    ASSERT(!nmp || !nmp->prev || nmp->prev->proc != c_p);
-
-    while (nmp && nmp->proc == c_p) {
-	if (!all && nmp->opts != opts)
-	    nmp = nmp->next;
-	else { /* if (all || nmp->opts == opts) */
-	    ErtsNodesMonitor *free_nmp;
-	    res = am_true;
-	    if (nmp->prev) {
-		ASSERT(nodes_monitors != nmp);
-		nmp->prev->next = nmp->next;
-	    }
-	    else {
-		ASSERT(nodes_monitors == nmp);
-		nodes_monitors = nmp->next;
-	    }
-	    if (nmp->next) {
-		ASSERT(nodes_monitors_end != nmp);
-		nmp->next->prev = nmp->prev;
-	    }
-	    else {
-		ASSERT(nodes_monitors_end == nmp);
-		nodes_monitors_end = nmp->prev;
-	    }
-	    free_nmp = nmp;
-	    nmp = nmp->next;
-	    if (c_p->nodes_monitors == free_nmp)
-		c_p->nodes_monitors = nmp && nmp->proc == c_p ? nmp : NULL;
-	    erts_free(ERTS_ALC_T_NODES_MON, free_nmp);
-	}
+        erts_proc_sig_send_persistent_monitor_msg(ERTS_MON_TYPE_NODES,
+                                                  nmdp[i].options,
+                                                  am_system,
+                                                  nmdp[i].pid,
+                                                  msg,
+                                                  hsz);
     }
-    
-    ASSERT(!all || !c_p->nodes_monitors);
-    return res;
-}
 
-void
-erts_delete_nodes_monitors(Process *c_p, ErtsProcLocks locks)
-{
-#if defined(ERTS_ENABLE_LOCK_CHECK)
-    if (c_p) {
-	ErtsProcLocks might_unlock = locks & ~ERTS_PROC_LOCK_MAIN;
-	if (might_unlock)
-	    erts_proc_lc_might_unlock(c_p, might_unlock);
-    }
-#endif
-    if (erts_mtx_trylock(&nodes_monitors_mtx) == EBUSY) {
-	ErtsProcLocks unlock_locks = locks & ~ERTS_PROC_LOCK_MAIN;
-	if (c_p && unlock_locks)
-	    erts_proc_unlock(c_p, unlock_locks);
-	erts_mtx_lock(&nodes_monitors_mtx);
-	if (c_p && unlock_locks)
-	    erts_proc_lock(c_p, unlock_locks);
-    }
-    remove_nodes_monitors(c_p, 0, 1);
-    erts_mtx_unlock(&nodes_monitors_mtx);
+    if (nmdp != &def_buf[0])
+        erts_free(ERTS_ALC_T_TMP, nmdp);
 }
+                           
 
-Eterm
-erts_monitor_nodes(Process *c_p, Eterm on, Eterm olist)
-{
+typedef struct {
+    Eterm **hpp;
+    Uint *szp;
     Eterm res;
-    Eterm opts_list = olist;
-    Uint16 opts = (Uint16) 0;
-
-    ASSERT(c_p);
-    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
+} ErtsNodesMonitorInfoContext;
 
-    if (on != am_true && on != am_false)
-	return THE_NON_VALUE;
 
-    if (is_not_nil(opts_list)) {
-	int all = 0, visible = 0, hidden = 0;
-
-	while (is_list(opts_list)) {
-	    Eterm *cp = list_val(opts_list);
-	    Eterm opt = CAR(cp);
-	    opts_list = CDR(cp);
-	    if (opt == am_nodedown_reason)
-		opts |= ERTS_NODES_MON_OPT_DOWN_REASON;
-	    else if (is_tuple(opt)) {
-		Eterm* tp = tuple_val(opt);
-		if (arityval(tp[0]) != 2)
-		    return THE_NON_VALUE;
-		switch (tp[1]) {
-		case am_node_type:
-		    switch (tp[2]) {
-		    case am_visible:
-			if (hidden || all)
-			    return THE_NON_VALUE;
-			opts |= ERTS_NODES_MON_OPT_TYPE_VISIBLE;
-			visible = 1;
-			break;
-		    case am_hidden:
-			if (visible || all)
-			    return THE_NON_VALUE;
-			opts |= ERTS_NODES_MON_OPT_TYPE_HIDDEN;
-			hidden = 1;
-			break;
-		    case am_all:
-			if (visible || hidden)
-			    return THE_NON_VALUE;
-			opts |= ERTS_NODES_MON_OPT_TYPES;
-			all = 1;
-			break;
-		    default:
-			return THE_NON_VALUE;
-		    }
-		    break;
-		default:
-		    return THE_NON_VALUE;
-		}
-	    }
-	    else {
-		return THE_NON_VALUE;
-	    }
-	}
-
-	if (is_not_nil(opts_list))
-	    return THE_NON_VALUE;
-    }
-
-    erts_mtx_lock(&nodes_monitors_mtx);
-
-    if (on == am_true)
-	res = insert_nodes_monitor(c_p, opts);
-    else
-	res = remove_nodes_monitors(c_p, opts, 0);
-
-    erts_mtx_unlock(&nodes_monitors_mtx);
-
-    return res;
+static void
+nodes_monitor_info(ErtsMonitor *mon, void *vctxt)
+{
+    ErtsMonitorDataExtended *mdep;
+    ErtsNodesMonitorInfoContext *ctxt = vctxt;
+    Uint no, i, opts, *szp;
+    Eterm **hpp, res;
+
+    hpp = ctxt->hpp;
+    szp = ctxt->szp;
+    res = ctxt->res;
+
+    ASSERT(erts_monitor_is_target(mon));
+    ASSERT(mon->type == ERTS_MON_TYPE_NODES);
+    mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon);
+    no = mdep->u.refc;
+
+    ASSERT(is_small(mdep->md.origin.other.item));
+    opts = (Uint) signed_val(mdep->md.origin.other.item);
+
+    for (i = 0; i < no; i++) {
+        Eterm olist = NIL;
+        if (opts & ERTS_NODES_MON_OPT_TYPES) {
+            Eterm type;
+            switch (opts & ERTS_NODES_MON_OPT_TYPES) {
+            case ERTS_NODES_MON_OPT_TYPES:        type = am_all;     break;
+            case ERTS_NODES_MON_OPT_TYPE_VISIBLE: type = am_visible; break;
+            case ERTS_NODES_MON_OPT_TYPE_HIDDEN:  type = am_hidden;  break;
+            default: erts_exit(ERTS_ABORT_EXIT, "Bad node type found\n");
+            }
+            olist = erts_bld_cons(hpp, szp, 
+                                  erts_bld_tuple(hpp, szp, 2,
+                                                 am_node_type,
+                                                 type),
+                                  olist);
+        }
+        if (opts & ERTS_NODES_MON_OPT_DOWN_REASON)
+            olist = erts_bld_cons(hpp, szp, am_nodedown_reason, olist);
+        res = erts_bld_cons(hpp, szp,
+                            erts_bld_tuple(hpp, szp, 2,
+                                           mon->other.item,
+                                           olist),
+                            res);
+    }
+
+    ctxt->hpp = hpp;
+    ctxt->szp = szp;
+    ctxt->res = res;
 }
 
-/*
- * Note, this function is only used for debuging.
- */
-
 Eterm
 erts_processes_monitoring_nodes(Process *c_p)
 {
-    ErtsNodesMonitor *nmp;
-    Eterm res;
+    /*
+     * Note, this function is only used for debugging.
+     */
+    ErtsNodesMonitorInfoContext ctxt;
     Eterm *hp;
-    Eterm **hpp;
     Uint sz;
-    Uint *szp;
 #ifdef DEBUG
     Eterm *hend;
 #endif
 
     ASSERT(c_p);
     ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
+
+    erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+    erts_thr_progress_block();
+
     erts_mtx_lock(&nodes_monitors_mtx);
 
     sz = 0;
-    szp = &sz;
-    hpp = NULL;
+    ctxt.szp = &sz;
+    ctxt.hpp = NULL;
 
- bld_result:
-    res = NIL;
-
-    for (nmp = nodes_monitors_end; nmp; nmp = nmp->prev) {
-	Uint16 i;
-	for (i = 0; i < nmp->no; i++) {
-	    Eterm olist = NIL;
-	    if (nmp->opts & ERTS_NODES_MON_OPT_TYPES) {
-		Eterm type;
-		switch (nmp->opts & ERTS_NODES_MON_OPT_TYPES) {
-		case ERTS_NODES_MON_OPT_TYPES:        type = am_all;     break;
-		case ERTS_NODES_MON_OPT_TYPE_VISIBLE: type = am_visible; break;
-		case ERTS_NODES_MON_OPT_TYPE_HIDDEN:  type = am_hidden;  break;
-		default: erts_exit(ERTS_ABORT_EXIT, "Bad node type found\n");
-		}
-		olist = erts_bld_cons(hpp, szp, 
-				      erts_bld_tuple(hpp, szp, 2,
-						     am_node_type,
-						     type),
-				      olist);
-	    }
-	    if (nmp->opts & ERTS_NODES_MON_OPT_DOWN_REASON)
-		olist = erts_bld_cons(hpp, szp, am_nodedown_reason, olist);
-	    res = erts_bld_cons(hpp, szp,
-				erts_bld_tuple(hpp, szp, 2,
-					       nmp->proc->common.id,
-					       olist),
-				res);
-	}
-    }
+    while (1) {
+        ctxt.res = NIL;
+
+        erts_monitor_list_foreach(nodes_monitors,
+                                  nodes_monitor_info,
+                                  (void *) &ctxt);
+
+        if (ctxt.hpp)
+            break;
 
-    if (!hpp) {
 	hp = HAlloc(c_p, sz);
 #ifdef DEBUG
 	hend = hp + sz;
 #endif
-	hpp = &hp;
-	szp = NULL;
-	goto bld_result;
+	ctxt.hpp = &hp;
+	ctxt.szp = NULL;
     }
 
     ASSERT(hp == hend);
 
     erts_mtx_unlock(&nodes_monitors_mtx);
 
-    return res;
+    erts_thr_progress_unblock();
+    erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+
+    return ctxt.res;
 }
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index b1b7ce9c78..c608fef816 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -120,7 +120,6 @@
 
 /* distribution trap functions */
 extern Export* dmonitor_node_trap;
-extern Export* dmonitor_p_trap;
 
 typedef enum {
     ERTS_DSP_NO_LOCK,
@@ -274,58 +273,13 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry)
 
 #endif
 
-typedef struct {
-    ErtsLink *d_lnk;
-    ErtsLink *d_sub_lnk;
-} ErtsDistLinkData;
-
-ERTS_GLB_INLINE void erts_remove_dist_link(ErtsDistLinkData *,
-					   Eterm,
-					   Eterm,
-					   DistEntry *);
-ERTS_GLB_INLINE int erts_was_dist_link_removed(ErtsDistLinkData *);
-ERTS_GLB_INLINE void erts_destroy_dist_link(ErtsDistLinkData *);
-
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-
-ERTS_GLB_INLINE void
-erts_remove_dist_link(ErtsDistLinkData *dldp,
-		      Eterm lid,
-		      Eterm rid,
-		      DistEntry *dep)
-{
-    erts_de_links_lock(dep);
-    dldp->d_lnk = erts_lookup_link(dep->nlinks, lid);
-    if (!dldp->d_lnk)
-	dldp->d_sub_lnk = NULL;
-    else {
-	dldp->d_sub_lnk = erts_remove_link(&ERTS_LINK_ROOT(dldp->d_lnk), rid);
-	dldp->d_lnk = (ERTS_LINK_ROOT(dldp->d_lnk)
-		       ? NULL
-		       : erts_remove_link(&dep->nlinks, lid));
-    }
-    erts_de_links_unlock(dep);
-}
-
-ERTS_GLB_INLINE int
-erts_was_dist_link_removed(ErtsDistLinkData *dldp)
-{
-    return dldp->d_sub_lnk != NULL;
-}
-
-ERTS_GLB_INLINE void
-erts_destroy_dist_link(ErtsDistLinkData *dldp)
-{
-    if (dldp->d_lnk)
-	erts_destroy_link(dldp->d_lnk);
-    if (dldp->d_sub_lnk)
-	erts_destroy_link(dldp->d_sub_lnk);	
-}
-
+#ifdef DEBUG
+#define ERTS_DBG_CHK_NO_DIST_LNK(D, R, L) \
+    erts_dbg_chk_no_dist_proc_link((D), (R), (L))
+#else
+#define ERTS_DBG_CHK_NO_DIST_LNK(D, R, L)
 #endif
 
-
-
 /* Define for testing */
 /* #define EXTREME_TTB_TRAPPING 1 */
 
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index fa49096d2c..562225c6ca 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -40,7 +40,7 @@
 #include "erl_bits.h"
 #include "erl_instrument.h"
 #include "erl_mseg.h"
-#include "erl_monitors.h"
+#include "erl_monitor_link.h"
 #include "erl_hl_timer.h"
 #include "erl_cpu_topology.h"
 #include "erl_thr_queue.h"
@@ -643,10 +643,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_PROC)]
 	= sizeof(Process);
-    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR_SH)]
-	= ERTS_MONITOR_SH_SIZE * sizeof(Uint);
-    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)]
-	= ERTS_LINK_SH_SIZE * sizeof(Uint);
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR)]
+	= sizeof(ErtsMonitorDataHeap);
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_LINK)]
+	= sizeof(ErtsLinkData);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)]
 	= sizeof(ErtsDrvSelectDataState);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_SEL_D_STATE)]
@@ -2379,7 +2379,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	}
 	tmp += erts_ptab_mem_size(&erts_proc);
 	tmp += erts_bif_timer_memory_size();
-	tmp += erts_tot_link_lh_size();
 
 	size.processes = size.processes_used = tmp;
 
@@ -2390,12 +2389,11 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	add_fix_values(&size.processes,
 		       &size.processes_used,
 		       fi,
-		       ERTS_ALC_T_MONITOR_SH);
-
+		       ERTS_ALC_T_MONITOR);
 	add_fix_values(&size.processes,
 		       &size.processes_used,
 		       fi,
-		       ERTS_ALC_T_NLINK_SH);
+		       ERTS_ALC_T_LINK);
 	add_fix_values(&size.processes,
 		       &size.processes_used,
 		       fi,
@@ -2625,11 +2623,6 @@ erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc)
     values[i].ui[0] = erts_bif_timer_memory_size();
     i++;
 
-    values[i].arity = 2;
-    values[i].name = "link_lh";
-    values[i].ui[0] = erts_tot_link_lh_size();
-    i++;
-
     values[i].arity = 2;
     values[i].name = "process_table";
     values[i].ui[0] = erts_ptab_mem_size(&erts_proc);
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 8107f133aa..4a6a19b210 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 2003-2017. All Rights Reserved.
+# Copyright Ericsson AB 2003-2018. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -282,6 +282,11 @@ type	THR_PRGR_IDATA	LONG_LIVED	SYSTEM		thr_prgr_internal_data
 type	THR_PRGR_DATA	LONG_LIVED	SYSTEM		thr_prgr_data
 type	T_THR_PRGR_DATA	SHORT_LIVED	SYSTEM		temp_thr_prgr_data
 type	RELEASE_LAREA	SHORT_LIVED	SYSTEM		release_literal_area
+type	SIG_DATA	SHORT_LIVED	PROCESSES	signal_data
+type	DIST_DEMONITOR	SHORT_LIVED	PROCESSES	dist_demonitor
+type	CML_CLEANUP	SHORT_LIVED	SYSTEM		connection_ml_cleanup
+type	ML_YIELD_STATE	SHORT_LIVED	SYSTEM		monitor_link_yield_state
+type	ML_DIST		STANDARD	SYSTEM		monitor_link_dist
 
 type	ENVIRONMENT	SYSTEM		SYSTEM		environment
 
@@ -326,8 +331,8 @@ type    LCNT_VECTOR SHORT_LIVED SYSTEM lcnt_sample_vector
 type	DEBUG	        SHORT_LIVED	SYSTEM          debugging
 
 type	DDLL_PROCESS	STANDARD	SYSTEM		ddll_processes
-type	MONITOR_LH	STANDARD	PROCESSES	monitor_lh
-type	NLINK_LH	STANDARD	PROCESSES	nlink_lh
+type	MONITOR_EXT	STANDARD	PROCESSES	monitor_extended
+type	LINK_EXT	STANDARD	PROCESSES	link_extended
 type	CODE		LONG_LIVED	CODE		code
 type	LITERAL		LITERAL 	CODE		literal
 type	LITERAL_REF	SHORT_LIVED	CODE		literal_area_ref
@@ -340,8 +345,8 @@ type  	LL_TEMP_TERM 	LONG_LIVED	SYSTEM		ll_temp_term
 type	NIF_TRAP_EXPORT	STANDARD	PROCESSES	nif_trap_export_entry
 type	NIF_EXP_TRACE	FIXED_SIZE	PROCESSES	nif_export_trace
 type	EXPORT		LONG_LIVED	CODE		export_entry
-type	MONITOR_SH	FIXED_SIZE	PROCESSES	monitor_sh
-type	NLINK_SH	FIXED_SIZE	PROCESSES	nlink_sh
+type	MONITOR		FIXED_SIZE	PROCESSES	monitor
+type	LINK		FIXED_SIZE	PROCESSES	link
 type	AINFO_REQ	SHORT_LIVED	SYSTEM		alloc_info_request
 type	SCHED_WTIME_REQ	SHORT_LIVED	SYSTEM		sched_wall_time_request
 type	GC_INFO_REQ	SHORT_LIVED	SYSTEM		gc_info_request
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 6475f04c56..9a3c058e82 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -50,6 +50,7 @@
 #define ERTS_PTAB_WANT_DEBUG_FUNCS__
 #include "erl_ptab.h"
 #include "erl_time.h"
+#include "erl_proc_sig_queue.h"
 #ifdef HIPE
 #include "hipe_arch.h"
 #endif
@@ -211,21 +212,27 @@ bld_magic_ref_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh)
   make_monitor_list:
   returns a list of records..
   -record(erl_monitor, {
-            type, % MON_ORIGIN or MON_TARGET (1 or 3)
-	    ref,
+            type, % process | port | time_offset | dist_process | resource
+                  % | node | nodes | suspend
+            dir, % origin | target
+	    ref, % reference or []
 	    pid, % Process or nodename
-	    name % registered name or []
+	    extra % registered name, integer or []
           }).
 */
 
 static void do_calc_mon_size(ErtsMonitor *mon, void *vpsz)
 {
+    ErtsMonitorData *mdp = erts_monitor_to_data(mon);
     Uint *psz = vpsz;
-    *psz += NC_HEAP_SIZE(mon->ref);
-    *psz += (mon->type == MON_NIF_TARGET ?
-	     erts_resource_ref_size(mon->u.resource) :
-	     (is_immed(mon->u.pid) ? 0 : NC_HEAP_SIZE(mon->u.pid)));
-    *psz += 8; /* CONS + 5-tuple */ 
+    *psz += is_immed(mdp->ref) ? 0 : NC_HEAP_SIZE(mdp->ref);
+
+    if (mon->type == ERTS_MON_TYPE_RESOURCE && erts_monitor_is_target(mon))
+        *psz += erts_resource_ref_size(mon->other.ptr);
+    else
+        *psz += is_immed(mon->other.item) ? 0 : NC_HEAP_SIZE(mon->other.item);
+
+    *psz += 9; /* CONS + 6-tuple */ 
 }
 
 typedef struct {
@@ -237,35 +244,97 @@ typedef struct {
 
 static void do_make_one_mon_element(ErtsMonitor *mon, void * vpmlc)
 {
+    ErtsMonitorData *mdp = erts_monitor_to_data(mon);
     MonListContext *pmlc = vpmlc;
-    Eterm tup;
-    Eterm r = STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->ref);
-    Eterm p = (mon->type == MON_NIF_TARGET ?
-	       erts_bld_resource_ref(&(pmlc->hp), &MSO(pmlc->p), mon->u.resource)
-	       : (is_immed(mon->u.pid) ? mon->u.pid
-		  : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->u.pid)));
-    tup = TUPLE5(pmlc->hp, pmlc->tag, make_small(mon->type), r, p, mon->name);
-    pmlc->hp += 6;
+    Eterm tup, t, d, r, p, x;
+
+    r = is_immed(mdp->ref) ? mdp->ref : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mdp->ref);
+    if (mon->type == ERTS_MON_TYPE_RESOURCE && erts_monitor_is_target(mon))
+        p = erts_bld_resource_ref(&(pmlc->hp), &MSO(pmlc->p), mon->other.ptr);
+    else
+        p = (is_immed(mon->other.item)
+             ? mon->other.item
+             : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->other.item));
+
+    if (mon->flags & ERTS_ML_FLG_NAME)
+        x = ((ErtsMonitorDataExtended *) mdp)->u.name;
+    else if (erts_monitor_is_target(mon))
+        x = NIL;
+    else if (mon->type == ERTS_MON_TYPE_NODE || mon->type == ERTS_MON_TYPE_NODES)
+        x = make_small(((ErtsMonitorDataExtended *) mdp)->u.refc);
+    else
+        x = NIL;
+
+    switch (mon->type) {
+    case ERTS_MON_TYPE_PROC:
+        t = am_process;
+        break;
+    case ERTS_MON_TYPE_PORT:
+        t = am_port;
+        break;
+    case ERTS_MON_TYPE_TIME_OFFSET:
+        t = am_time_offset;
+        break;
+    case ERTS_MON_TYPE_DIST_PROC: {
+        ERTS_DECL_AM(dist_process);
+        t = AM_dist_process;
+        break;
+    }
+    case ERTS_MON_TYPE_RESOURCE: {
+        ERTS_DECL_AM(resource);
+        t = AM_resource;
+        break;
+    }
+    case ERTS_MON_TYPE_NODE:
+        t = am_node;
+        break;
+    case ERTS_MON_TYPE_NODES: {
+        ERTS_DECL_AM(nodes);
+        t = AM_nodes;
+        break;
+    }
+    case ERTS_MON_TYPE_SUSPEND:
+        t = am_suspend;
+        break;
+    default:
+        ERTS_INTERNAL_ERROR("Unknown monitor type");
+        t = am_error;
+        break;
+    }
+    if (erts_monitor_is_target(mon)) {
+        ERTS_DECL_AM(target);
+        d = AM_target;
+    }
+    else {
+        ERTS_DECL_AM(origin);
+        d = AM_origin;
+    }
+    tup = TUPLE6(pmlc->hp, pmlc->tag, t, d, r, p, x);
+    pmlc->hp += 7;
     pmlc->res = CONS(pmlc->hp, tup, pmlc->res);
     pmlc->hp += 2;
 }
 
 static Eterm 
-make_monitor_list(Process *p, ErtsMonitor *root)
+make_monitor_list(Process *p, int tree, ErtsMonitor *root, Eterm tail)
 {
     DECL_AM(erl_monitor);
     Uint sz = 0;
     MonListContext mlc;
+    void (*foreach)(ErtsMonitor *,
+                    void (*)(ErtsMonitor *, void *),
+                    void *);
 
-    erts_doforall_monitors(root, &do_calc_mon_size, &sz);
-    if (sz == 0) {
-	return NIL;
-    }
+    foreach = tree ? erts_monitor_tree_foreach : erts_monitor_list_foreach;
+
+    (*foreach)(root, do_calc_mon_size, &sz);
+    if (sz == 0)
+	return tail;
     mlc.p = p;
     mlc.hp = HAlloc(p,sz);
-    mlc.res = NIL;
+    mlc.res = tail;
     mlc.tag = AM_erl_monitor;
-    erts_doforall_monitors(root, &do_make_one_mon_element, &mlc);
+    (*foreach)(root, do_make_one_mon_element, &mlc);
     return mlc.res;
 }
 
@@ -273,20 +342,22 @@ make_monitor_list(Process *p, ErtsMonitor *root)
   make_link_list:
   returns a list of records..
   -record(erl_link, {
-            type, % LINK_NODE or LINK_PID (1 or 3)
-	    pid, % Process or nodename
-	    targets % List of erl_link's or nil
+            type, % process | port | dist_process
+	    pid, % Process or port
+            id % (address)
           }).
 */
 
-static void do_calc_lnk_size(ErtsLink *lnk, void *vpsz)
+static void calc_lnk_size(ErtsLink *lnk, void *vpsz)
 {
     Uint *psz = vpsz;
-    *psz += is_immed(lnk->pid) ? 0 : NC_HEAP_SIZE(lnk->pid);
-    if (lnk->type != LINK_NODE && ERTS_LINK_ROOT(lnk) != NULL) { 
-	/* Node links use this pointer as ref counter... */
-	erts_doforall_links(ERTS_LINK_ROOT(lnk),&do_calc_lnk_size,vpsz);
-    }
+    Uint sz = 0;
+    ErtsLinkData *ldp = erts_link_to_data(lnk);
+
+    (void) erts_bld_uword(NULL, &sz, (UWord) ldp);
+
+    *psz += sz;
+    *psz += is_immed(lnk->other.item) ? 0 : size_object(lnk->other.item);
     *psz += 7; /* CONS + 4-tuple */ 
 }
 
@@ -297,37 +368,58 @@ typedef struct {
     Eterm tag;
 } LnkListContext;
 
-static void do_make_one_lnk_element(ErtsLink *lnk, void * vpllc)
+static void make_one_lnk_element(ErtsLink *lnk, void * vpllc)
 {
     LnkListContext *pllc = vpllc;
-    Eterm tup;
-    Eterm old_res, targets = NIL;
-    Eterm p = (is_immed(lnk->pid)
-	       ? lnk->pid
-	       : STORE_NC(&(pllc->hp), &MSO(pllc->p), lnk->pid));
-    if (lnk->type == LINK_NODE) {
-	targets = make_small(ERTS_LINK_REFC(lnk));
-    } else if (ERTS_LINK_ROOT(lnk) != NULL) {
-	old_res = pllc->res;
-	pllc->res = NIL;
-	erts_doforall_links(ERTS_LINK_ROOT(lnk),&do_make_one_lnk_element, vpllc);
-	targets = pllc->res;
-	pllc->res = old_res;
-    }
-    tup = TUPLE4(pllc->hp, pllc->tag, make_small(lnk->type), p, targets);
+    Eterm tup, t, pid, id;
+    ErtsLinkData *ldp = erts_link_to_data(lnk);
+
+    id = erts_bld_uword(&pllc->hp, NULL, (UWord) ldp);
+
+    if (is_immed(lnk->other.item))
+        pid = lnk->other.item;
+    else {
+        Uint sz = size_object(lnk->other.item);
+        pid = copy_struct(lnk->other.item, sz, &(pllc->hp), &MSO(pllc->p));
+    }
+
+    switch (lnk->type) {
+    case ERTS_LNK_TYPE_PROC:
+        t = am_process;
+        break;
+    case ERTS_LNK_TYPE_PORT:
+        t = am_port;
+        break;
+    case ERTS_LNK_TYPE_DIST_PROC: {
+        ERTS_DECL_AM(dist_process);
+        t = AM_dist_process;
+        break;
+    }
+    default:
+        ERTS_INTERNAL_ERROR("Unkown link type");
+        t = am_undefined;
+        break;
+    }
+
+    tup = TUPLE4(pllc->hp, pllc->tag, t, pid, id);
     pllc->hp += 5;
     pllc->res = CONS(pllc->hp, tup, pllc->res);
     pllc->hp += 2;
 }
 
 static Eterm 
-make_link_list(Process *p, ErtsLink *root, Eterm tail)
+make_link_list(Process *p, int tree, ErtsLink *root, Eterm tail)
 {
     DECL_AM(erl_link);
     Uint sz = 0;
     LnkListContext llc;
+    void (*foreach)(ErtsLink *,
+                    void (*)(ErtsLink *, void *),
+                    void *);
 
-    erts_doforall_links(root, &do_calc_lnk_size, &sz);
+    foreach = tree ? erts_link_tree_foreach : erts_link_list_foreach;
+
+    (*foreach)(root, calc_lnk_size, (void *) &sz);
     if (sz == 0) {
 	return tail;
     }
@@ -335,7 +427,7 @@ make_link_list(Process *p, ErtsLink *root, Eterm tail)
     llc.hp = HAlloc(p,sz);
     llc.res = tail;
     llc.tag = AM_erl_link;
-    erts_doforall_links(root, &do_make_one_lnk_element, &llc);
+    (*foreach)(root, make_one_lnk_element, (void *) &llc);
     return llc.res;
 }
 
@@ -381,6 +473,8 @@ typedef struct {
 	Eterm term;
 	ErtsResource* resource;
     }entity;
+    int named;
+    Uint16 type;
     Eterm node;
     /* pid is actual target being monitored, no matter pid/port or name */
     Eterm pid;
@@ -423,78 +517,103 @@ static void collect_one_link(ErtsLink *lnk, void *vmicp)
 {
     MonitorInfoCollection *micp = vmicp;
     EXTEND_MONITOR_INFOS(micp);
-    if (!(lnk->type == LINK_PID)) {
-	return;
-    }
-    micp->mi[micp->mi_i].entity.term = lnk->pid;
-    micp->sz += 2 + NC_HEAP_SIZE(lnk->pid);
+    micp->mi[micp->mi_i].entity.term = lnk->other.item;
+    micp->sz += 2 + NC_HEAP_SIZE(lnk->other.item);
     micp->mi_i++;
 } 
 
 static void collect_one_origin_monitor(ErtsMonitor *mon, void *vmicp)
 {
-    MonitorInfoCollection *micp = vmicp;
+    if (erts_monitor_is_origin(mon)) {
+        MonitorInfoCollection *micp = vmicp;
  
-    if (mon->type != MON_ORIGIN) {
-	return;
-    }
-    EXTEND_MONITOR_INFOS(micp);
-    if (is_atom(mon->u.pid)) { /* external by name */
-	micp->mi[micp->mi_i].entity.term = mon->name;
-        micp->mi[micp->mi_i].node = mon->u.pid;
-        micp->sz += 3; /* need one 2-tuple */
-    } else if (is_external_pid(mon->u.pid)) { /* external by pid */
-	micp->mi[micp->mi_i].entity.term = mon->u.pid;
-        micp->mi[micp->mi_i].node = NIL;
-        micp->sz += NC_HEAP_SIZE(mon->u.pid);
-    } else if (!is_nil(mon->name)) { /* internal by name */
-	micp->mi[micp->mi_i].entity.term = mon->name;
-        micp->mi[micp->mi_i].node = erts_this_dist_entry->sysname;
-        micp->sz += 3; /* need one 2-tuple */
-    } else { /* internal by pid */
-	micp->mi[micp->mi_i].entity.term = mon->u.pid;
-        micp->mi[micp->mi_i].node = NIL;
-	/* no additional heap space needed */
-    }
-
-    /* have always pid at hand, to assist with figuring out if its a port or
-     * a process, when we monitored by name and process_info is requested.
-     * See: erl_bif_info.c:process_info_aux section for am_monitors */
-    micp->mi[micp->mi_i].pid = mon->u.pid;
+        EXTEND_MONITOR_INFOS(micp);
+
+        micp->mi[micp->mi_i].type = mon->type;
+
+        switch (mon->type) {
+        case ERTS_MON_TYPE_PROC:
+        case ERTS_MON_TYPE_PORT:
+        case ERTS_MON_TYPE_DIST_PROC:
+        case ERTS_MON_TYPE_TIME_OFFSET:
+            if (!(mon->flags & ERTS_ML_FLG_NAME)) {
+                micp->mi[micp->mi_i].named = 0;
+                micp->mi[micp->mi_i].entity.term = mon->other.item;
+                micp->mi[micp->mi_i].node = NIL;
+                if (is_not_atom(mon->other.item))
+                    micp->sz += NC_HEAP_SIZE(mon->other.item);
+            }
+            else {
+                ErtsMonitorDataExtended *mdep;
+                micp->mi[micp->mi_i].named = !0;
+                mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon);
+                micp->mi[micp->mi_i].entity.term = mdep->u.name;
+                if (mdep->dist)
+                    micp->mi[micp->mi_i].node = mdep->dist->nodename;
+                else
+                    micp->mi[micp->mi_i].node = erts_this_dist_entry->sysname;
+                micp->sz += 3; /* need one 2-tuple */
+            }
 
-    micp->mi_i++;
-    micp->sz += 2 + 3; /* For a cons cell and a 2-tuple */
+            /* have always pid at hand, to assist with figuring out if its a port or
+             * a process, when we monitored by name and process_info is requested.
+             * See: erl_bif_info.c:process_info_aux section for am_monitors */
+            micp->mi[micp->mi_i].pid = mon->other.item;
+
+            micp->mi_i++;
+            micp->sz += 2 + 3; /* For a cons cell and a 2-tuple */
+            break;
+        default:
+            break;
+        }
+    }
 }
 
 static void collect_one_target_monitor(ErtsMonitor *mon, void *vmicp)
 {
     MonitorInfoCollection *micp = vmicp;
  
-    if (mon->type != MON_TARGET && mon->type != MON_NIF_TARGET) {
-        return;
-    }
+    if (erts_monitor_is_target(mon)) {
 
-    EXTEND_MONITOR_INFOS(micp);
+        EXTEND_MONITOR_INFOS(micp);
   
+        micp->mi[micp->mi_i].type = mon->type;
+        micp->mi[micp->mi_i].named = !!(mon->flags & ERTS_ML_FLG_NAME);
+        switch (mon->type) {
+
+        case ERTS_MON_TYPE_PROC:
+        case ERTS_MON_TYPE_PORT:
+        case ERTS_MON_TYPE_DIST_PROC:
+
+            micp->mi[micp->mi_i].entity.term = mon->other.item;
+            micp->mi[micp->mi_i].node = NIL;
+            micp->sz += NC_HEAP_SIZE(mon->other.item);
+
+            micp->sz += 2; /* cons */;
+            micp->mi_i++;
+            break;
+
+        case ERTS_MON_TYPE_RESOURCE:
+
+            micp->mi[micp->mi_i].entity.resource = mon->other.ptr;
+            micp->mi[micp->mi_i].node = NIL;
+            micp->sz += erts_resource_ref_size(mon->other.ptr);
+
+            micp->sz += 2; /* cons */;
+            micp->mi_i++;
+            break;
+
+        default:
+            break;
+        }
 
-    if (mon->type == MON_NIF_TARGET) {
-	micp->mi[micp->mi_i].entity.resource = mon->u.resource;
-        micp->mi[micp->mi_i].node = make_small(MON_NIF_TARGET);
-        micp->sz += erts_resource_ref_size(mon->u.resource);
-    }
-    else {
-	micp->mi[micp->mi_i].entity.term = mon->u.pid;
-        micp->mi[micp->mi_i].node = NIL;
-        micp->sz += NC_HEAP_SIZE(mon->u.pid);
     }
-    micp->sz += 2; /* cons */;
-    micp->mi_i++;
 }
 
 typedef struct {
     Process *c_p;
     ErtsProcLocks c_p_locks;
-    ErtsSuspendMonitor **smi;
+    ErtsMonitorSuspend **smi;
     Uint smi_i;
     Uint smi_max;
     int sz;
@@ -517,10 +636,10 @@ do {									\
 				       (SMICP)->smi,			\
 				       ((SMICP)->smi_max		\
 					+ ERTS_SMI_INC)			\
-				       * sizeof(ErtsSuspendMonitor *))	\
+				       * sizeof(ErtsMonitorSuspend *))	\
 			: erts_alloc(ERTS_ALC_T_TMP,			\
 				     ERTS_SMI_INC			\
-				     * sizeof(ErtsSuspendMonitor *)));	\
+				     * sizeof(ErtsMonitorSuspend *)));	\
 	(SMICP)->smi_max += ERTS_SMI_INC;				\
     }									\
  } while (0)
@@ -533,12 +652,13 @@ do {									\
  } while (0)
 
 static void
-collect_one_suspend_monitor(ErtsSuspendMonitor *smon, void *vsmicp)
+collect_one_suspend_monitor(ErtsMonitor *mon, void *vsmicp)
 {
+    ErtsMonitorSuspend *smon = erts_monitor_suspend(mon);
     ErtsSuspendMonitorInfoCollection *smicp = vsmicp;
     Process *suspendee = erts_pid2proc(smicp->c_p,
 				       smicp->c_p_locks,
-				       smon->pid,
+				       mon->other.item,
 				       0);
     if (suspendee) { /* suspendee is alive */
 	Sint a, p;
@@ -572,29 +692,23 @@ collect_one_suspend_monitor(ErtsSuspendMonitor *smon, void *vsmicp)
 
 #define ERTS_PI_FAIL_TYPE_BADARG		0
 #define ERTS_PI_FAIL_TYPE_YIELD			1
-#define ERTS_PI_FAIL_TYPE_AWAIT_EXIT		2
+#define ERTS_PI_FAIL_TYPE_EXITED                2
 
 static ERTS_INLINE ErtsProcLocks
 pi_locks(Eterm info)
 {
     switch (info) {
-    case am_status:
     case am_priority:
-    case am_trap_exit:
-	return ERTS_PROC_LOCK_STATUS;
-    case am_links:
-    case am_monitors:
-    case am_monitored_by:
-    case am_suspending:
-	return ERTS_PROC_LOCK_LINK;
+    case am_status:
+        return 0;
+    case am_suspended:
+        return ERTS_PROC_LOCK_STATUS;
     case am_messages:
     case am_message_queue_len:
     case am_total_heap_size:
-	return ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_MSGQ;
-    case am_memory:
-	return ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_MSGQ;
+        return ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_MSGQ;
     default:
-	return ERTS_PROC_LOCK_MAIN;
+        return ERTS_PROC_LOCK_MAIN;
     }
 }
 
@@ -743,7 +857,7 @@ process_info_init(void)
 }
 
 static ERTS_INLINE Process *
-pi_pid2proc(Process *c_p, Eterm pid, ErtsProcLocks info_locks)
+pi_lookup_proc(Process *c_p, Eterm pid, ErtsProcLocks *locks)
 {
     /*
      * If the main lock is needed, we use erts_pid2proc_not_running()
@@ -758,23 +872,79 @@ pi_pid2proc(Process *c_p, Eterm pid, ErtsProcLocks info_locks)
      * is currently running.
      */
 
-    if (info_locks & ERTS_PROC_LOCK_MAIN)
-	return erts_pid2proc_not_running(c_p, ERTS_PROC_LOCK_MAIN,
-					 pid, info_locks);
-    else
-	return erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN,
-			     pid, info_locks);
+    if ((*locks) & ERTS_PROC_LOCK_MAIN) {
+        erts_aint32_t state;
+        int local_only, done;
+        Process *rp;
+        ErtsProcLocks more_locks;
+
+        rp = erts_pid2proc_not_running(c_p, ERTS_PROC_LOCK_MAIN,
+                                       pid, ERTS_PROC_LOCK_MAIN);
+
+        if (!rp || rp == ERTS_PROC_LOCK_BUSY)
+            return rp;
+
+        if ((*locks) & ERTS_PROC_LOCK_MSGQ) {
+            /*
+             * Move in queue into private queue and
+             * release msgq lock, enabling others to
+             * send messages to the process while it
+             * is being inspected...
+             */
+            erts_proc_lock(rp, ERTS_PROC_LOCK_MSGQ);
+            erts_proc_sig_fetch(rp);
+            erts_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ);
+            (*locks) &= ~ERTS_PROC_LOCK_MSGQ;
+	}
+
+        /*
+         * Handle all signals received up to this point
+         * in order to preserve signal order.
+         * 
+         * FIX ME: Should be done yielding...
+         */
+        local_only = 0;
+        do {
+            int r = CONTEXT_REDS;
+            done = erts_proc_sig_handle_incoming(rp, &state, &r,
+                                                 CONTEXT_REDS,
+                                                 local_only);
+            local_only = !0;
+            BUMP_REDS(c_p, r);
+        } while (!done && !(state & ERTS_PSFLG_EXITING));
+
+        if (state & ERTS_PSFLG_EXITING) {
+            if (rp != c_p)
+                erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
+            return NULL;
+        }
+        more_locks = (*locks) & ~ERTS_PROC_LOCK_MAIN;
+        if (more_locks)
+            erts_proc_lock(rp, more_locks);
+        return rp;
+    }
+
+    ASSERT(!((*locks) & ERTS_PROC_LOCK_MSGQ));
+
+    if (*locks)
+        return erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN,
+			     pid, *locks);
+
+    return erts_proc_lookup(pid);
 }
 
 
 
+
+
 static BIF_RETTYPE
 process_info_aux(Process *BIF_P,
 		 Process *rp,
 		 ErtsProcLocks rp_locks,
 		 Eterm rpid,
 		 Eterm item,
-		 int always_wrap);
+		 int always_wrap,
+                 int *reds);
 
 #define ERTS_PI_RES_ELEM_IX_BUF_INC 1024
 #define ERTS_PI_DEF_RES_ELEM_IX_BUF_SZ ERTS_PI_ARGS
@@ -794,6 +964,7 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap,
     ErtsProcLocks locks = (ErtsProcLocks) 0;
     int res_len, ix;
     Process *rp = NULL;
+    int reds = 0;
 
     *fail_type = ERTS_PI_FAIL_TYPE_BADARG;
 
@@ -845,9 +1016,14 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap,
 
     ASSERT(res_len > 0);
 
-    rp = pi_pid2proc(c_p, pid, locks|ERTS_PROC_LOCK_STATUS);
+    rp = pi_lookup_proc(c_p, pid, &locks);
     if (!rp) {
-	res = am_undefined;
+        if (c_p->common.id != pid)
+            res = am_undefined;
+        else {
+            *fail_type = ERTS_PI_FAIL_TYPE_EXITED;
+            res = THE_NON_VALUE;
+        }
 	goto done;
     }
     else if (rp == ERTS_PROC_LOCK_BUSY) {
@@ -856,38 +1032,9 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap,
 	*fail_type = ERTS_PI_FAIL_TYPE_YIELD;
 	goto done;
     }
-    else if (c_p != rp && ERTS_PROC_PENDING_EXIT(rp)) {
-	locks |= ERTS_PROC_LOCK_STATUS;
-	res = THE_NON_VALUE;
-	*fail_type = ERTS_PI_FAIL_TYPE_AWAIT_EXIT;
-	goto done;
-    }
-    else {
-	ErtsProcLocks unlock_locks = 0;
-
-	if (c_p == rp)
-	    locks |= ERTS_PROC_LOCK_MAIN;
 
-	if (!(locks & ERTS_PROC_LOCK_STATUS))
-	    unlock_locks |= ERTS_PROC_LOCK_STATUS;
-
-	if (locks & ERTS_PROC_LOCK_MSGQ) {
-	    /*
-	     * Move in queue into private queue and
-	     * release msgq lock, enabling others to
-	     * send messages to the process while it
-	     * is being inspected...
-	     */
-	    ASSERT(locks & ERTS_PROC_LOCK_MAIN);
-	    ERTS_MSGQ_MV_INQ2PRIVQ(rp);
-	    locks &= ~ERTS_PROC_LOCK_MSGQ;
-	    unlock_locks |= ERTS_PROC_LOCK_MSGQ;
-	}
-
-	if (unlock_locks)
-	    erts_proc_unlock(rp, unlock_locks);
-
-    }
+    if (c_p == rp)
+        locks |= ERTS_PROC_LOCK_MAIN;
 
     /*
      * We always handle 'messages' first if it should be part
@@ -899,16 +1046,23 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap,
     if (want_messages) {
 	ix = pi_arg2ix(am_messages);
 	ASSERT(part_res[ix] == THE_NON_VALUE);
-	part_res[ix] = process_info_aux(c_p, rp, locks, pid, am_messages, always_wrap);
-	ASSERT(part_res[ix] != THE_NON_VALUE);
+	res = process_info_aux(c_p, rp, locks, pid, am_messages,
+                               always_wrap, &reds);
+	ASSERT(res != am_undefined);
+	ASSERT(res != THE_NON_VALUE);
+        part_res[ix] = res;
     }
 
     for (; res_elem_ix_ix >= 0; res_elem_ix_ix--) {
 	ix = res_elem_ix[res_elem_ix_ix];
 	if (part_res[ix] == THE_NON_VALUE) {
 	    arg = pi_ix2arg(ix);
-	    part_res[ix] = process_info_aux(c_p, rp, locks, pid, arg, always_wrap);
-	    ASSERT(part_res[ix] != THE_NON_VALUE);
+	    res = process_info_aux(c_p, rp, locks, pid, arg,
+                                   always_wrap, &reds);
+            if (res == am_undefined)
+                goto done;
+	    ASSERT(res != THE_NON_VALUE);
+            part_res[ix] = res;
 	}
     }
 
@@ -948,6 +1102,8 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap,
     if (res_elem_ix != &def_res_elem_ix_buf[0])
 	erts_free(ERTS_ALC_T_TMP, res_elem_ix);
 
+    BUMP_REDS(c_p, reds);
+
     return res;
 }
 
@@ -971,8 +1127,8 @@ BIF_RETTYPE process_info_1(BIF_ALIST_1)
 	    BIF_ERROR(BIF_P, BADARG);
 	case ERTS_PI_FAIL_TYPE_YIELD:
 	    ERTS_BIF_YIELD1(bif_export[BIF_process_info_1], BIF_P, BIF_ARG_1);
-	case ERTS_PI_FAIL_TYPE_AWAIT_EXIT:
-	    ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined);
+        case ERTS_PI_FAIL_TYPE_EXITED:
+            ERTS_BIF_EXITED(BIF_P);
 	default:
 	    erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__);
 	}
@@ -989,7 +1145,7 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2)
     Process *rp;
     Eterm pid = BIF_ARG_1;
     ErtsProcLocks info_locks;
-    int fail_type;
+    int fail_type, reds = 0;
 
     if (is_external_pid(pid)
 	&& external_pid_dist_entry(pid) == erts_this_dist_entry)
@@ -1011,8 +1167,8 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2)
 	    case ERTS_PI_FAIL_TYPE_YIELD:
 		ERTS_BIF_YIELD2(bif_export[BIF_process_info_2], BIF_P,
 				BIF_ARG_1, BIF_ARG_2);
-	    case ERTS_PI_FAIL_TYPE_AWAIT_EXIT:
-		ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined);
+            case ERTS_PI_FAIL_TYPE_EXITED:
+                ERTS_BIF_EXITED(BIF_P);
 	    default:
 		erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error",
 			 __FILE__, __LINE__);
@@ -1027,44 +1183,23 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2)
 
     info_locks = pi_locks(BIF_ARG_2); 
 
-    rp = pi_pid2proc(BIF_P, pid, info_locks|ERTS_PROC_LOCK_STATUS);
-    if (!rp)
-	res = am_undefined;
+    rp = pi_lookup_proc(BIF_P, pid, &info_locks);
+    if (!rp) {
+        if (BIF_P->common.id == pid)
+            ERTS_BIF_EXITED(BIF_P);
+	BIF_RET(am_undefined);
+    }
     else if (rp == ERTS_PROC_LOCK_BUSY)
 	ERTS_BIF_YIELD2(bif_export[BIF_process_info_2], BIF_P,
 			BIF_ARG_1, BIF_ARG_2);
-    else if (rp != BIF_P && ERTS_PROC_PENDING_EXIT(rp)) {
-	erts_proc_unlock(rp, info_locks|ERTS_PROC_LOCK_STATUS);
-	ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined);
-    }
-    else {
-	ErtsProcLocks unlock_locks = 0;
-
-	if (BIF_P == rp)
-	    info_locks |= ERTS_PROC_LOCK_MAIN;
 
-	if (!(info_locks & ERTS_PROC_LOCK_STATUS))
-	    unlock_locks |= ERTS_PROC_LOCK_STATUS;
-
-	if (info_locks & ERTS_PROC_LOCK_MSGQ) {
-	    /*
-	     * Move in queue into private queue and
-	     * release msgq lock, enabling others to
-	     * send messages to the process while it
-	     * is being inspected...
-	     */
-	    ASSERT(info_locks & ERTS_PROC_LOCK_MAIN);
-	    ERTS_MSGQ_MV_INQ2PRIVQ(rp);
-	    info_locks &= ~ERTS_PROC_LOCK_MSGQ;
-	    unlock_locks |= ERTS_PROC_LOCK_MSGQ;
-	}
+    if (BIF_P == rp)
+        info_locks |= ERTS_PROC_LOCK_MAIN;
 
-	if (unlock_locks)
-	    erts_proc_unlock(rp, unlock_locks);
+    res = process_info_aux(BIF_P, rp, info_locks, pid, BIF_ARG_2,
+                           0, &reds);
 
-	res = process_info_aux(BIF_P, rp, info_locks, pid, BIF_ARG_2, 0);
-    }
-    ASSERT(is_value(res));
+    BUMP_REDS(BIF_P, reds);
 
     if (BIF_P == rp)
 	info_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -1081,11 +1216,14 @@ process_info_aux(Process *BIF_P,
 		 ErtsProcLocks rp_locks,
 		 Eterm rpid,
 		 Eterm item,
-		 int always_wrap)
+		 int always_wrap,
+                 int *reds)
 {
     Eterm *hp;
     Eterm res = NIL;
 
+    (*reds)++;
+
     ASSERT(rp);
 
     /*
@@ -1144,14 +1282,15 @@ process_info_aux(Process *BIF_P,
 	break;
 
     case am_status:
-	res = erts_process_status(rp, rpid);
-	ASSERT(res != am_undefined);
+        res = erts_process_state2status(erts_atomic32_read_nob(&rp->state));
+        if (res == am_exiting || res == am_free)
+            return am_undefined;
 	hp = HAlloc(BIF_P, 3);
 	break;
 
     case am_messages: {
 
-	if (rp->msg.len == 0 || ERTS_TRACE_FLAGS(rp) & F_SENSITIVE) {
+	if (rp->sig_qs.len == 0 || ERTS_TRACE_FLAGS(rp) & F_SENSITIVE) {
 	    hp = HAlloc(BIF_P, 3);
 	} else {
 	    ErtsMessageInfo *mip;
@@ -1162,16 +1301,17 @@ process_info_aux(Process *BIF_P,
 #endif
 
 	    mip = erts_alloc(ERTS_ALC_T_TMP,
-			     rp->msg.len*sizeof(ErtsMessageInfo));
+			     rp->sig_qs.len*sizeof(ErtsMessageInfo));
 
 	    /*
 	     * Note that message queue may shrink when calling
-	     * erts_prep_msgq_for_inspection() since it removes
+	     * erts_proc_sig_prep_msgq_for_inspection() since it removes
 	     * corrupt distribution messages.
 	     */
-	    heap_need = erts_prep_msgq_for_inspection(BIF_P, rp, rp_locks, mip);
+	    heap_need = erts_proc_sig_prep_msgq_for_inspection(BIF_P, rp,
+                                                               rp_locks, mip);
 	    heap_need += 3; /* top 2-tuple */
-	    heap_need += rp->msg.len*2; /* Cons cells */
+	    heap_need += rp->sig_qs.len*2; /* Cons cells */
 
 	    hp = HAlloc(BIF_P, heap_need); /* heap_need is exact */
 #ifdef DEBUG
@@ -1179,7 +1319,7 @@ process_info_aux(Process *BIF_P,
 #endif
 
 	    /* Build list of messages... */
-	    for (i = rp->msg.len - 1, res = NIL; i >= 0; i--) {
+	    for (i = rp->sig_qs.len - 1, res = NIL; i >= 0; i--) {
 		Eterm msg = ERL_MESSAGE_TERM(mip[i].msgp);
 		Uint sz = mip[i].size;
 
@@ -1192,6 +1332,11 @@ process_info_aux(Process *BIF_P,
 
 	    ASSERT(hp_end == hp + 3);
 
+            if (rp->sig_qs.len > CONTEXT_REDS*4)
+                *reds += CONTEXT_REDS*4;
+            else
+                *reds += rp->sig_qs.len / 4;
+
 	    erts_free(ERTS_ALC_T_TMP, mip);
 	}
 	break;
@@ -1199,7 +1344,7 @@ process_info_aux(Process *BIF_P,
 
     case am_message_queue_len:
 	hp = HAlloc(BIF_P, 3);
-	res = make_small(rp->msg.len);
+	res = make_small(rp->sig_qs.len);
 	break;
 
     case am_links: {
@@ -1209,7 +1354,7 @@ process_info_aux(Process *BIF_P,
 
 	INIT_MONITOR_INFOS(mic);
 
-	erts_doforall_links(ERTS_P_LINKS(rp),&collect_one_link,&mic);
+	erts_link_tree_foreach(ERTS_P_LINKS(rp), collect_one_link, (void *) &mic);
 
 	hp = HAlloc(BIF_P, 3 + mic.sz);
 	res = NIL;
@@ -1218,6 +1363,12 @@ process_info_aux(Process *BIF_P,
 	    res = CONS(hp, item, res);
 	    hp += 2;
 	}
+
+        if (mic.mi_i > CONTEXT_REDS*4)
+            *reds += CONTEXT_REDS*4;
+        else
+            *reds += mic.mi_i / 4;
+
 	DESTROY_MONITOR_INFOS(mic);
 	break;
     }
@@ -1227,12 +1378,14 @@ process_info_aux(Process *BIF_P,
         int i;
 
 	INIT_MONITOR_INFOS(mic);
-        erts_doforall_monitors(ERTS_P_MONITORS(rp),
-                               &collect_one_origin_monitor, &mic);
+        erts_monitor_tree_foreach(ERTS_P_MONITORS(rp),
+                                  collect_one_origin_monitor,
+                                  (void *) &mic);
+
         hp = HAlloc(BIF_P, 3 + mic.sz);
 	res = NIL;
 	for (i = 0; i < mic.mi_i; i++) {
-	    if (is_atom(mic.mi[i].entity.term)) {
+	    if (mic.mi[i].named) {
 		/* Monitor by name. 
                  * Build {process|port, {Name, Node}} and cons it.
 		 */
@@ -1252,11 +1405,28 @@ process_info_aux(Process *BIF_P,
                 hp += 2;
 	    }
 	    else {
-                /* Monitor by pid. Build {process|port, Pid} and cons it. */
+                /* Build {process|port|time_offset, Pid|clock_service} and cons it. */
 		Eterm t;
-		Eterm pid = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity.term);
+		Eterm pid;
+                Eterm m_type;
+
+                if (is_atom(mic.mi[i].entity.term))
+                    pid = mic.mi[i].entity.term;
+                else
+                    pid = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity.term);
+
+                switch (mic.mi[i].type) {
+                case ERTS_MON_TYPE_PORT:
+                    m_type = am_port;
+                    break;
+                case ERTS_MON_TYPE_TIME_OFFSET:
+                    m_type = am_time_offset;
+                    break;
+                default:
+                    m_type = am_process;
+                    break;
+                }
 
-                Eterm m_type = is_port(mic.mi[i].pid) ? am_port : am_process;
                 ASSERT(is_pid(mic.mi[i].pid)
                     || is_port(mic.mi[i].pid));
 
@@ -1266,6 +1436,12 @@ process_info_aux(Process *BIF_P,
                 hp += 2;
 	    }
 	}
+
+        if (mic.mi_i > CONTEXT_REDS*4)
+            *reds += CONTEXT_REDS*4;
+        else
+            *reds += mic.mi_i / 4;
+
         DESTROY_MONITOR_INFOS(mic);
 	break;
     }
@@ -1276,20 +1452,34 @@ process_info_aux(Process *BIF_P,
 	Eterm item;
 
 	INIT_MONITOR_INFOS(mic);
-	erts_doforall_monitors(ERTS_P_MONITORS(rp),&collect_one_target_monitor,&mic);
+        erts_monitor_list_foreach(ERTS_P_LT_MONITORS(rp),
+                                  collect_one_target_monitor,
+                                  (void *) &mic);
+        erts_monitor_tree_foreach(ERTS_P_MONITORS(rp),
+                                  collect_one_target_monitor,
+                                  (void *) &mic);
+
 	hp = HAlloc(BIF_P, 3 + mic.sz);
 
 	res = NIL;
 	for (i = 0; i < mic.mi_i; ++i) {
-            if (mic.mi[i].node == make_small(MON_NIF_TARGET)) {
-                item = erts_bld_resource_ref(&hp, &MSO(BIF_P), mic.mi[i].entity.resource);
-            }
-            else {
-                item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity.term);
-            }
+            if (mic.mi[i].type == ERTS_MON_TYPE_RESOURCE)
+                item = erts_bld_resource_ref(&hp,
+                                             &MSO(BIF_P),
+                                             mic.mi[i].entity.resource);
+            else
+                item = STORE_NC(&hp,
+                                &MSO(BIF_P),
+                                mic.mi[i].entity.term);
 	    res = CONS(hp, item, res);
 	    hp += 2;
 	}
+
+        if (mic.mi_i > CONTEXT_REDS*4)
+            *reds += CONTEXT_REDS*4;
+        else
+            *reds += mic.mi_i / 4;
+
 	DESTROY_MONITOR_INFOS(mic);
 	break;
     }
@@ -1306,11 +1496,11 @@ process_info_aux(Process *BIF_P,
 					BIF_P,
 					(BIF_P == rp
 					 ? ERTS_PROC_LOCK_MAIN
-					 : 0) | ERTS_PROC_LOCK_LINK);
+					 : 0) | ERTS_PROC_LOCK_STATUS);
 
-	erts_doforall_suspend_monitors(rp->suspend_monitors,
-				       &collect_one_suspend_monitor,
-				       &smic);
+	erts_monitor_tree_foreach(rp->suspend_monitors,
+                                  &collect_one_suspend_monitor,
+                                  &smic);
 	hp = HAlloc(BIF_P, 3 + smic.sz);
 #ifdef DEBUG
 	hp_end = hp + smic.sz;
@@ -1334,12 +1524,17 @@ process_info_aux(Process *BIF_P,
 		pending = small_to_big(p, hp);
 		hp += BIG_UINT_HEAP_SIZE;
 	    }
-	    item = TUPLE3(hp, smic.smi[i]->pid, active, pending);
+	    item = TUPLE3(hp, smic.smi[i]->mon.other.item, active, pending);
 	    hp += 4;
 	    res = CONS(hp, item, res);
 	    hp += 2;
 	}
 
+        if (smic.smi_i > CONTEXT_REDS*4)
+            *reds += CONTEXT_REDS*4;
+        else
+            *reds += smic.smi_i / 4;
+
 	ERTS_DESTROY_SUSPEND_MONITOR_INFOS(smic);
 	ASSERT(hp == hp_end);
 
@@ -1347,18 +1542,22 @@ process_info_aux(Process *BIF_P,
     }
 
     case am_dictionary:
-	if (ERTS_TRACE_FLAGS(rp) & F_SENSITIVE) {
+	if (!rp->dictionary || (ERTS_TRACE_FLAGS(rp) & F_SENSITIVE)) {
 	    res = NIL;
 	} else {
+            Uint num = rp->dictionary->numElements;
 	    res = erts_dictionary_copy(BIF_P, rp->dictionary);
+            if (num > CONTEXT_REDS*4)
+                *reds += CONTEXT_REDS*4;
+            else
+                *reds += (int) num / 4;
 	}
 	hp = HAlloc(BIF_P, 3);
 	break;
 
     case am_trap_exit: {
-	erts_aint32_t state = erts_atomic32_read_nob(&rp->state);
 	hp = HAlloc(BIF_P, 3);
-	if (state & ERTS_PSFLG_TRAP_EXIT)
+        if (rp->flags & F_TRAP_EXIT)
 	    res = am_true;
 	else
 	    res = am_false;
@@ -1415,7 +1614,6 @@ process_info_aux(Process *BIF_P,
     }
 
     case am_total_heap_size: {
-	ErtsMessage *mp;
 	Uint total_heap_size;
 	Uint hsz = 3;
 
@@ -1425,10 +1623,19 @@ process_info_aux(Process *BIF_P,
 
 	total_heap_size += rp->mbuf_sz;
 
-        if (rp->flags & F_ON_HEAP_MSGQ)
-            for (mp = rp->msg.first; mp; mp = mp->next)
-                if (mp->data.attached)
-                    total_heap_size += erts_msg_attached_data_size(mp);
+        if (rp->flags & F_ON_HEAP_MSGQ) {
+            ERTS_FOREACH_SIG_PRIVQS(
+                rp, mp,
+                {
+                    if (ERTS_SIG_IS_MSG(mp) && mp->data.attached)
+                        total_heap_size += erts_msg_attached_data_size(mp);
+                });
+
+            if (rp->sig_qs.len > CONTEXT_REDS*4)
+                *reds += CONTEXT_REDS*4;
+            else
+                *reds += rp->sig_qs.len / 4;
+        }
 
 	(void) erts_bld_uint(NULL, &hsz, total_heap_size);
 	hp = HAlloc(BIF_P, hsz);
@@ -1451,6 +1658,12 @@ process_info_aux(Process *BIF_P,
 	(void) erts_bld_uint(NULL, &hsz, size);
 	hp = HAlloc(BIF_P, hsz);
 	res = erts_bld_uint(&hp, NULL, size);
+
+        if (rp->sig_qs.len > CONTEXT_REDS*4)
+            *reds += CONTEXT_REDS*4;
+        else
+            *reds += rp->sig_qs.len / 4;
+
 	break;
     }
 
@@ -1523,10 +1736,14 @@ process_info_aux(Process *BIF_P,
 	break;
     }
 
-    case am_priority:
+    case am_priority: {
+        erts_aint32_t state = erts_atomic32_read_nob(&rp->state);
+        if (ERTS_PSFLG_EXITING & state)
+            return am_undefined;
 	hp = HAlloc(BIF_P, 3);
-	res = erts_get_process_priority(rp);
+	res = erts_get_process_priority(state);
 	break;
+    }
 
     case am_trace:
 	hp = HAlloc(BIF_P, 3);
@@ -1629,6 +1846,8 @@ process_info_aux(Process *BIF_P,
 	(void) bld_magic_ref_bin_list(NULL, &sz, &MSO(rp));
 	hp = HAlloc(BIF_P, sz);
 	res = bld_magic_ref_bin_list(&hp, NULL, &MSO(rp));
+
+        *reds += 10;
 	break;
     }
 
@@ -2863,6 +3082,16 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     BIF_ERROR(BIF_P, BADARG);
 }
 
+static void monitor_size(ErtsMonitor *mon, void *vsz)
+{
+    *((Uint *) vsz) = erts_monitor_size(mon);
+}
+
+static void link_size(ErtsMonitor *lnk, void *vsz)
+{
+    *((Uint *) vsz) = erts_link_size(lnk);
+}
+
 /**********************************************************************/ 
 /* Return information on ports */
 /* Info:
@@ -2898,7 +3127,7 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt,
 
 	INIT_MONITOR_INFOS(mic);
 
-	erts_doforall_links(ERTS_P_LINKS(prt), &collect_one_link, &mic);
+	erts_link_tree_foreach(ERTS_P_LINKS(prt), collect_one_link, (void *) &mic);
 
 	if (szp)
 	    *szp += mic.sz;
@@ -2922,11 +3151,11 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt,
     else if (item == am_monitors) {
 	MonitorInfoCollection mic;
 	int i;
-	Eterm item;
 
 	INIT_MONITOR_INFOS(mic);
-        erts_doforall_monitors(ERTS_P_MONITORS(prt),
-                               &collect_one_origin_monitor, &mic);
+        erts_monitor_tree_foreach(ERTS_P_MONITORS(prt),
+                                  collect_one_origin_monitor,
+                                  (void *) &mic);
 
 	if (szp)
 	    *szp += mic.sz;
@@ -2935,11 +3164,10 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt,
 	    res = NIL;
 	    for (i = 0; i < mic.mi_i; i++) {
 		Eterm t;
-                Eterm m_type;
 
-                item = STORE_NC(hpp, ohp, mic.mi[i].entity.term);
-                m_type = is_port(item) ? am_port : am_process;
-                t = TUPLE2(*hpp, m_type, item);
+                ASSERT(mic.mi[i].type == ERTS_MON_TYPE_PORT);
+                ASSERT(is_internal_pid(mic.mi[i].entity.term));
+                t = TUPLE2(*hpp, am_process, mic.mi[i].entity.term);
 		*hpp += 3;
 		res = CONS(*hpp, t, res);
 		*hpp += 2;
@@ -2958,15 +3186,19 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt,
         Eterm item;
 
         INIT_MONITOR_INFOS(mic);
-        erts_doforall_monitors(ERTS_P_MONITORS(prt),
-                               &collect_one_target_monitor, &mic);
+        erts_monitor_list_foreach(ERTS_P_LT_MONITORS(prt),
+                                  collect_one_target_monitor,
+                                  (void *) &mic);
+        erts_monitor_tree_foreach(ERTS_P_MONITORS(prt),
+                                  collect_one_target_monitor,
+                                  (void *) &mic);
         if (szp)
             *szp += mic.sz;
 
         if (hpp) {
             res = NIL;
             for (i = 0; i < mic.mi_i; ++i) {
-                ASSERT(mic.mi[i].node == NIL);
+                ASSERT(mic.mi[i].type != ERTS_MON_TYPE_RESOURCE);
                 item = STORE_NC(hpp, ohp, mic.mi[i].entity.term);
                 res = CONS(*hpp, item, res);
                 *hpp += 2;
@@ -3043,7 +3275,12 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt,
 	 */
 	Uint size = 0;
 
-	erts_doforall_links(ERTS_P_LINKS(prt), &erts_one_link_size, &size);
+        erts_link_tree_foreach(ERTS_P_LINKS(prt),
+                               link_size, (void *) &size);
+        erts_monitor_tree_foreach(ERTS_P_MONITORS(prt),
+                                  monitor_size, (void *) &size);
+        erts_monitor_list_foreach(ERTS_P_LT_MONITORS(prt),
+                                  monitor_size, (void *) &size);
 
 	size += erts_port_data_size(prt);
 
@@ -3278,8 +3515,7 @@ BIF_RETTYPE is_process_alive_1(BIF_ALIST_1)
 	   BIF_RET(am_false);
        }
        else {
-	   if (erts_atomic32_read_acqb(&rp->state)
-		 & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING))
+	   if (erts_atomic32_read_acqb(&rp->state) & ERTS_PSFLG_EXITING)
 	       ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_false);
 	   else
 	       BIF_RET(am_true);
@@ -3311,16 +3547,6 @@ BIF_RETTYPE process_display_2(BIF_ALIST_2)
    if (rp == ERTS_PROC_LOCK_BUSY)
        ERTS_BIF_YIELD2(bif_export[BIF_process_display_2], BIF_P,
 		       BIF_ARG_1, BIF_ARG_2);
-   if (rp != BIF_P && ERTS_PROC_PENDING_EXIT(rp)) {
-       Eterm args[2] = {BIF_ARG_1, BIF_ARG_2};
-       erts_proc_unlock(rp, ERTS_PROC_LOCKS_ALL);
-       ERTS_BIF_AWAIT_X_APPLY_TRAP(BIF_P,
-				   BIF_ARG_1,
-				   am_erlang,
-				   am_process_display,
-				   args,
-				   2);
-   }
    erts_stack_dump(ERTS_PRINT_STDERR, NULL, rp);
    erts_proc_unlock(rp, (BIF_P == rp
 			     ? ERTS_PROC_LOCKS_ALL_MINOR
@@ -3693,22 +3919,59 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 		    BIF_RET(erts_process_status(NULL, tp[2]));
 		}
 	    }
+            else if (ERTS_IS_ATOM_STR("connection_id", tp[1])) {
+                DistEntry *dep;
+                Eterm *hp, res;
+                Uint con_id, hsz = 0;
+                if (!is_atom(tp[2]))
+                    BIF_ERROR(BIF_P, BADARG);
+                dep = erts_sysname_to_connected_dist_entry(tp[2]);
+                if (!dep)
+                    BIF_ERROR(BIF_P, BADARG);
+                erts_de_rlock(dep);
+                con_id = (Uint) dep->connection_id;
+                erts_de_runlock(dep);
+                (void) erts_bld_uint(NULL, &hsz, con_id);
+                hp = hsz ? HAlloc(BIF_P, hsz) : NULL;
+                res = erts_bld_uint(&hp, NULL, con_id);
+                BIF_RET(res);
+            }
 	    else if (ERTS_IS_ATOM_STR("link_list", tp[1])) {
 		/* Used by erl_link_SUITE (emulator) */
 		if(is_internal_pid(tp[2])) {
+                    erts_aint32_t state;
 		    Eterm res;
 		    Process *p;
+                    int sigs_done, local_only;
 
 		    p = erts_pid2proc(BIF_P,
 				      ERTS_PROC_LOCK_MAIN,
 				      tp[2],
-				      ERTS_PROC_LOCK_LINK);
+				      ERTS_PROC_LOCK_MAIN);
 		    if (!p) {
 			ERTS_ASSERT_IS_NOT_EXITING(BIF_P);
 			BIF_RET(am_undefined);
 		    }
-		    res = make_link_list(BIF_P, ERTS_P_LINKS(p), NIL);
-		    erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+                    
+                    local_only = 0;
+                    do {
+                        int reds = CONTEXT_REDS;
+                        sigs_done = erts_proc_sig_handle_incoming(p,
+                                                                  &state,
+                                                                  &reds,
+                                                                  CONTEXT_REDS,
+                                                                  local_only);
+                        local_only = !0;
+                    } while (!sigs_done && !(state & ERTS_PSFLG_EXITING));
+
+                    if (!(state & ERTS_PSFLG_EXITING))
+                        res = make_link_list(BIF_P, 1, ERTS_P_LINKS(p), NIL);
+                    else if (BIF_P == p)
+                        ERTS_BIF_EXITED(BIF_P);
+                    else
+                        res = am_undefined;
+                    if (BIF_P != p)
+                        erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
 		    BIF_RET(res);
 		}
 		else if(is_internal_port(tp[2])) {
@@ -3719,19 +3982,20 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 						 ERTS_PORT_SFLGS_INVALID_LOOKUP);
 		    if(!p)
 			BIF_RET(am_undefined);
-		    res = make_link_list(BIF_P, ERTS_P_LINKS(p), NIL);
+		    res = make_link_list(BIF_P, 1, ERTS_P_LINKS(p), NIL);
 		    erts_port_release(p);
 		    BIF_RET(res);
 		}
 		else if(is_node_name_atom(tp[2])) {
 		    DistEntry *dep = erts_find_dist_entry(tp[2]);
 		    if(dep) {
-			Eterm subres;
-			erts_de_links_lock(dep);
-			subres = make_link_list(BIF_P, dep->nlinks, NIL);
-			subres = make_link_list(BIF_P, dep->node_links, subres);
-			erts_de_links_unlock(dep);
-			BIF_RET(subres);
+			Eterm res = NIL;
+                        if (dep->mld) {
+                            erts_mtx_lock(&dep->mld->mtx);
+                            res = make_link_list(BIF_P, 0, dep->mld->links, NIL);
+                            erts_mtx_unlock(&dep->mld->mtx);
+                        }
+			BIF_RET(res);
 		    } else {
 			BIF_RET(am_undefined);
 		    }
@@ -3740,27 +4004,54 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    else if (ERTS_IS_ATOM_STR("monitor_list", tp[1])) {
 		/* Used by erl_link_SUITE (emulator) */
 		if(is_internal_pid(tp[2])) {
+                    erts_aint32_t state;
 		    Process *p;
 		    Eterm res;
+                    int sigs_done, local_only;
 
 		    p = erts_pid2proc(BIF_P,
 				      ERTS_PROC_LOCK_MAIN,
 				      tp[2],
-				      ERTS_PROC_LOCK_LINK);
+				      ERTS_PROC_LOCK_MAIN);
 		    if (!p) {
 			ERTS_ASSERT_IS_NOT_EXITING(BIF_P);
 			BIF_RET(am_undefined);
 		    }
-		    res = make_monitor_list(BIF_P, ERTS_P_MONITORS(p));
-		    erts_proc_unlock(p, ERTS_PROC_LOCK_LINK);
+                    
+                    local_only = 0;
+                    do {
+                        int reds = CONTEXT_REDS;
+                        sigs_done = erts_proc_sig_handle_incoming(p,
+                                                                  &state,
+                                                                  &reds,
+                                                                  CONTEXT_REDS,
+                                                                  local_only);
+                        local_only = !0;
+                    } while (!sigs_done && !(state & ERTS_PSFLG_EXITING));
+
+                    if (!(state & ERTS_PSFLG_EXITING)) {
+                        res = make_monitor_list(BIF_P, 1, ERTS_P_MONITORS(p), NIL);
+                        res = make_monitor_list(BIF_P, 0, ERTS_P_LT_MONITORS(p), res);
+                    }
+                    else {
+                        if (BIF_P == p)
+                            ERTS_BIF_EXITED(BIF_P);
+                        else
+                            res = am_undefined;
+                    }
+                    if (BIF_P != p)
+                        erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
 		    BIF_RET(res);
 		} else if(is_node_name_atom(tp[2])) {
 		    DistEntry *dep = erts_find_dist_entry(tp[2]);
 		    if(dep) {
-			Eterm ml;
-			erts_de_links_lock(dep);
-			ml = make_monitor_list(BIF_P, dep->monitors);
-			erts_de_links_unlock(dep);
+			Eterm ml = NIL;
+                        if (dep->mld) {
+                            erts_mtx_lock(&dep->mld->mtx);
+                            ml = make_monitor_list(BIF_P, 1, dep->mld->orig_name_monitors, NIL);
+                            ml = make_monitor_list(BIF_P, 0, dep->mld->monitors, ml);
+                            erts_mtx_unlock(&dep->mld->mtx);
+                        }
 			BIF_RET(ml);
 		    } else {
 			BIF_RET(am_undefined);
@@ -3778,18 +4069,6 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 		}
 		BIF_RET(res);
 	    }
-	    else if (ERTS_IS_ATOM_STR("have_pending_exit", tp[1])) {
-		Process *rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
-					    tp[2], ERTS_PROC_LOCK_STATUS);
-		if (!rp) {
-		    BIF_RET(am_undefined);
-		}
-		else {
-		    Eterm res = ERTS_PROC_PENDING_EXIT(rp) ? am_true : am_false;
-		    erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
-		    BIF_RET(res);
-		}
-	    }
 	    else if (ERTS_IS_ATOM_STR("binary_info", tp[1])) {
 		Eterm bin = tp[2];
 		if (is_binary(bin)) {
@@ -4116,57 +4395,6 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	    erts_set_gc_state(BIF_P, enable);
 	    BIF_RET(res);
 	}
-	else if (ERTS_IS_ATOM_STR("send_fake_exit_signal", BIF_ARG_1)) {
-	    /* Used by signal_SUITE (emulator) */
-
-	    /* Testcases depend on the exit being received via
-	       a pending exit when the receiver is the same as
-	       the caller.  */
-	    if (is_tuple(BIF_ARG_2)) {
-		Eterm* tp = tuple_val(BIF_ARG_2);
-		if (arityval(tp[0]) == 3
-		    && (is_pid(tp[1]) || is_port(tp[1]))
-		    && is_internal_pid(tp[2])) {
-		    int xres;
-		    ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
-		    Process *rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
-						tp[2], rp_locks);
-		    if (!rp) {
-			DECL_AM(dead);
-			BIF_RET(AM_dead);
-		    }
-
-		    if (BIF_P == rp)
-			rp_locks |= ERTS_PROC_LOCK_MAIN;
-		    xres = erts_send_exit_signal(NULL, /* NULL in order to
-							  force a pending exit
-							  when we send to our
-							  selves. */
-						 tp[1],
-						 rp,
-						 &rp_locks,
-						 tp[3],
-						 NIL,
-						 NULL,
-						 0);
-		    if (BIF_P == rp)
-			rp_locks &= ~ERTS_PROC_LOCK_MAIN;
-		    erts_proc_unlock(rp, rp_locks);
-		    if (xres > 1) {
-			DECL_AM(message);
-			BIF_RET(AM_message);
-		    }
-		    else if (xres == 0) {
-			DECL_AM(unaffected);
-			BIF_RET(AM_unaffected);
-		    }
-		    else {
-			DECL_AM(exit);
-			BIF_RET(AM_exit);
-		    }
-		}
-	    }
-	}
         else if (ERTS_IS_ATOM_STR("colliding_names", BIF_ARG_1)) {
 	    /* Used by ets_SUITE (stdlib) */
 	    if (is_tuple(BIF_ARG_2)) {
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 1feb892b93..7fe4e02782 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -43,6 +43,7 @@
 #include "erl_bits.h"
 #include "erl_bif_unique.h"
 #include "dtrace-wrapper.h"
+#include "erl_proc_sig_queue.h"
 
 static Port *open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump);
 static int merge_global_environment(erts_osenv_t *env, Eterm key_value_pairs);
@@ -53,10 +54,13 @@ char *erts_default_arg0 = "default";
 
 BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2)
 {
+    BIF_RETTYPE ret;
     Port *port;
     Eterm res;
     char *str;
     int err_type, err_num;
+    ErtsLinkData *ldp;
+    ErtsLink *lnk;
 
     port = open_port(BIF_P, BIF_ARG_1, BIF_ARG_2, &err_type, &err_num);
     if (!port) {
@@ -78,6 +82,16 @@ BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2)
         BIF_RET(res);
     }
 
+    ldp = erts_link_create(ERTS_LNK_TYPE_PORT, BIF_P->common.id, port->common.id);
+    ASSERT(ldp->a.other.item == port->common.id);
+    ASSERT(ldp->b.other.item == BIF_P->common.id);
+    /*
+     * This link should not already be present, but can potentially
+     * due to id wrapping...
+     */
+    lnk = erts_link_tree_lookup_insert(&ERTS_P_LINKS(BIF_P), &ldp->a);
+    erts_link_tree_insert(&ERTS_P_LINKS(port), &ldp->b);
+
     if (port->drv_ptr->flags & ERL_DRV_FLAG_USE_INIT_ACK) {
 
         /* Copied from erl_port_task.c */
@@ -86,39 +100,30 @@ BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2)
         erts_make_ref_in_array(port->async_open_port->ref);
         port->async_open_port->to = BIF_P->common.id;
 
-        erts_proc_lock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK);
-        if (ERTS_PROC_PENDING_EXIT(BIF_P)) {
-	    /* need to exit caller instead */
-	    erts_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK);
-	    KILL_CATCHES(BIF_P);
-	    BIF_P->freason = EXC_EXIT;
-            erts_port_release(port);
-            BIF_RET(am_badarg);
-	}
-
-	ERTS_MSGQ_MV_INQ2PRIVQ(BIF_P);
-	BIF_P->msg.save = BIF_P->msg.last;
-
-	erts_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE);
+        /*
+         * We unconditionaly *must* do a receive on a message
+         * containing the reference after this...
+         */
+        ERTS_RECV_MARK_SAVE(BIF_P);
+        ERTS_RECV_MARK_SET(BIF_P);
 
         res = erts_proc_store_ref(BIF_P, port->async_open_port->ref);
     } else {
         res = port->common.id;
-        erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
     }
 
-    erts_add_link(&ERTS_P_LINKS(port), LINK_PID, BIF_P->common.id);
-    erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, port->common.id);
-
     if (IS_TRACED_FL(BIF_P, F_TRACE_PROCS))
-        trace_proc(BIF_P, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, BIF_P,
+        trace_proc(BIF_P, ERTS_PROC_LOCK_MAIN, BIF_P,
                    am_link, port->common.id);
 
-    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
+    ERTS_BIF_PREP_RET(ret, res);
 
     erts_port_release(port);
 
-    BIF_RET(res);
+    if (lnk)
+        erts_link_release(lnk);
+
+    return ret;
 }
 
 static ERTS_INLINE Port *
@@ -195,7 +200,6 @@ BIF_RETTYPE erts_internal_port_command_3(BIF_ALIST_3)
 #endif
 
     switch (erts_port_output(BIF_P, flags, prt, prt->common.id, BIF_ARG_2, &ref)) {
-    case ERTS_PORT_OP_CALLER_EXIT:
     case ERTS_PORT_OP_BADARG:
     case ERTS_PORT_OP_DROPPED:
  	ERTS_BIF_PREP_RET(res, am_badarg);
@@ -254,7 +258,6 @@ BIF_RETTYPE erts_internal_port_call_3(BIF_ALIST_3)
     op = (unsigned int) uint_op;
 
     switch (erts_port_call(BIF_P, prt, op, BIF_ARG_3, &retval)) {
-    case ERTS_PORT_OP_CALLER_EXIT:
     case ERTS_PORT_OP_DROPPED:
     case ERTS_PORT_OP_BADARG:
 	retval = am_badarg;
@@ -272,11 +275,8 @@ BIF_RETTYPE erts_internal_port_call_3(BIF_ALIST_3)
     }
 
     state = erts_atomic32_read_acqb(&BIF_P->state);
-    if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
-	if (state & ERTS_PSFLG_PENDING_EXIT)
-	    erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
+    if (state & ERTS_PSFLG_EXITING)
 	ERTS_BIF_EXITED(BIF_P);
-    }
 
     BIF_RET(retval);
 }
@@ -302,7 +302,6 @@ BIF_RETTYPE erts_internal_port_control_3(BIF_ALIST_3)
     op = (unsigned int) uint_op;
 
     switch (erts_port_control(BIF_P, prt, op, BIF_ARG_3, &retval)) {
-    case ERTS_PORT_OP_CALLER_EXIT:
     case ERTS_PORT_OP_BADARG:
     case ERTS_PORT_OP_DROPPED:
 	retval = am_badarg;
@@ -320,11 +319,8 @@ BIF_RETTYPE erts_internal_port_control_3(BIF_ALIST_3)
     }
 
     state = erts_atomic32_read_acqb(&BIF_P->state);
-    if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
-	if (state & ERTS_PSFLG_PENDING_EXIT)
-	    erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
+    if (state & ERTS_PSFLG_EXITING)
 	ERTS_BIF_EXITED(BIF_P);
-    }
 
     BIF_RET(retval);
 }
@@ -347,7 +343,6 @@ BIF_RETTYPE erts_internal_port_close_1(BIF_ALIST_1)
 	BIF_RET(am_badarg);
 
     switch (erts_port_exit(BIF_P, 0, prt, BIF_P->common.id, am_normal, &ref)) {
-    case ERTS_PORT_OP_CALLER_EXIT:
     case ERTS_PORT_OP_BADARG:
     case ERTS_PORT_OP_DROPPED:
 	BIF_RET(am_badarg);
@@ -380,7 +375,6 @@ BIF_RETTYPE erts_internal_port_connect_2(BIF_ALIST_2)
 #endif
 
     switch (erts_port_connect(BIF_P, 0, prt, BIF_P->common.id, BIF_ARG_2, &ref)) {
-    case ERTS_PORT_OP_CALLER_EXIT:
     case ERTS_PORT_OP_BADARG:
     case ERTS_PORT_OP_DROPPED:
 	BIF_RET(am_badarg);
@@ -418,7 +412,6 @@ BIF_RETTYPE erts_internal_port_info_1(BIF_ALIST_1)
     }
 
     switch (erts_port_info(BIF_P, prt, THE_NON_VALUE, &retval)) {
-    case ERTS_PORT_OP_CALLER_EXIT:
     case ERTS_PORT_OP_BADARG:
 	BIF_RET(am_badarg);
     case ERTS_PORT_OP_DROPPED:
@@ -457,7 +450,6 @@ BIF_RETTYPE erts_internal_port_info_2(BIF_ALIST_2)
     }
 
     switch (erts_port_info(BIF_P, prt, BIF_ARG_2, &retval)) {
-    case ERTS_PORT_OP_CALLER_EXIT:
     case ERTS_PORT_OP_BADARG:
 	BIF_RET(am_badarg);
     case ERTS_PORT_OP_DROPPED:
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 22942b40c4..a076f0bf54 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -40,6 +40,7 @@
 #include "erl_binary.h"
 #include "erl_thr_progress.h"
 #include "erl_bif_unique.h"
+#include "erl_proc_sig_queue.h"
 
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 956662aefa..3982b87e34 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -38,6 +38,7 @@
 #include "erl_binary.h"
 #include "erl_map.h"
 #include "erl_thr_progress.h"
+#include "erl_proc_sig_queue.h"
 
 #include "erl_db_util.h"
 
@@ -155,6 +156,7 @@ set_tracee_flags(Process *tracee_p, ErtsTracer tracer,
 	   : am_false);
     erts_tracer_replace(&tracee_p->common, tracer);
     ERTS_TRACE_FLAGS(tracee_p) = flags;
+
     return ret;
 }
 /*
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index bf8244564a..db78378257 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -404,15 +404,16 @@ void verify_process(Process *p)
         erts_exit(ERTS_ERROR_EXIT,"Wild pointer found in " name " of %T!\n",p->common.id); }
 
 
-    ErtsMessage* mp = p->msg.first;
-
     VERBOSE(DEBUG_MEMORY,("Verify process: %T...\n",p->common.id));
 
-    while (mp != NULL) {
-        VERIFY_ETERM("message term",ERL_MESSAGE_TERM(mp));
-        VERIFY_ETERM("message token",ERL_MESSAGE_TOKEN(mp));
-        mp = mp->next;
-    }
+    ERTS_FOREACH_SIG_PRIVQS(
+        p, mp,
+        {
+            if (ERTS_SIG_IS_MSG(mp)) {
+                VERIFY_ETERM("message term",ERL_MESSAGE_TERM(mp));
+                VERIFY_ETERM("message token",ERL_MESSAGE_TOKEN(mp));
+            }
+        });
 
     erts_check_stack(p);
     erts_check_heap(p);
@@ -532,16 +533,15 @@ static void print_process_memory(Process *p)
     erts_printf("-- %-*s ---%s-%s-%s-%s--\n",
                 PTR_SIZE, "PCB", dashes, dashes, dashes, dashes);
 
-    if (p->msg.first != NULL) {
-        ErtsMessage* mp;
-        erts_printf("  Message Queue:\n");
-        mp = p->msg.first;
-        while (mp != NULL) {
-            erts_printf("| 0x%0*lx | 0x%0*lx |\n",PTR_SIZE,
-                        ERL_MESSAGE_TERM(mp),PTR_SIZE,ERL_MESSAGE_TOKEN(mp));
-            mp = mp->next;
-        }
-    }
+
+    erts_printf("  Message Queue:\n");
+    ERTS_FOREACH_SIG_PRIVQS(
+        p, mp,
+        {
+            if (ERTS_SIG_IS_MSG(mp))
+                erts_printf("| 0x%0*lx | 0x%0*lx |\n",PTR_SIZE,
+                            ERL_MESSAGE_TERM(mp),PTR_SIZE,ERL_MESSAGE_TOKEN(mp));
+        });
 
     if (p->dictionary != NULL) {
         int n = ERTS_PD_SIZE(p->dictionary);
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 1c64644efc..5da8e3eda5 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -43,6 +43,7 @@
 #include "erl_bif_unique.h"
 #include "dist.h"
 #include "erl_nfunc_sched.h"
+#include "erl_proc_sig_queue.h"
 
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20
@@ -154,7 +155,6 @@ static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size,
 			   Eterm* objv, int nobj);
 static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size);
 static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size);
-static void move_msgq_to_heap(Process *p);
 static int reached_max_heap_size(Process *p, Uint total_heap_size,
                                  Uint extra_heap_size, Uint extra_old_heap_size);
 static void init_gc_info(ErtsGCInfo *gcip);
@@ -189,6 +189,21 @@ static struct {
     ErtsGCInfo info;
 } dirty_gc;
 
+static void move_msgs_to_heap(Process *c_p)
+{
+    Uint64 pre_oh, post_oh;
+
+    pre_oh = c_p->off_heap.overhead;
+    erts_proc_sig_move_msgs_to_heap(c_p);
+    post_oh = c_p->off_heap.overhead;
+
+    if (pre_oh != post_oh) {
+	/* Got new binaries; update bin vheap size... */
+        c_p->bin_vheap_sz = next_vheap_size(c_p, post_oh,
+                                            c_p->bin_vheap_sz);
+    }
+}
+
 static ERTS_INLINE int
 gc_cost(Uint gc_moved_live_words, Uint resize_moved_words)
 {
@@ -565,20 +580,26 @@ young_gen_usage(Process *p)
     hsz = p->mbuf_sz;
 
     if (p->flags & F_ON_HEAP_MSGQ) {
-	ErtsMessage *mp;
-	for (mp = p->msg.first; mp; mp = mp->next) {
-	    /*
-	     * We leave not yet decoded distribution messages
-	     * as they are in the queue since it is not
-	     * possible to determine a maximum size until
-	     * actual decoding. However, we use their estimated
-	     * size when calculating need, and by this making
-	     * it more likely that they will fit on the heap
-	     * when actually decoded.
-	     */
-	    if (mp->data.attached)
-		hsz += erts_msg_attached_data_size(mp);
-	}
+        ERTS_FOREACH_SIG_PRIVQS(
+            p, mp,
+            {
+                /*
+                 * We leave not yet decoded distribution messages
+                 * as they are in the queue since it is not
+                 * possible to determine a maximum size until
+                 * actual decoding. However, we use their estimated
+                 * size when calculating need, and by this making
+                 * it more likely that they will fit on the heap
+                 * when actually decoded.
+                 *
+                 * We however ignore off heap messages...
+                 */
+                if (ERTS_SIG_IS_MSG(mp)
+                    && mp->data.attached
+                    && mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
+                    hsz += erts_msg_attached_data_size(mp);
+                }
+            });
     }
 
     hsz += p->htop - p->heap;
@@ -621,7 +642,7 @@ check_for_possibly_long_gc(Process *p, Uint ygen_usage)
     sz = ygen_usage;
     sz += p->hend - p->stop;
     if (p->flags & F_ON_HEAP_MSGQ)
-        sz += p->msg.len;
+        sz += p->sig_qs.len;
     if (major)
 	sz += p->old_htop - p->old_heap;
 
@@ -761,13 +782,9 @@ do_major_collection:
        long_gc/large_gc triggers when this happens as process was
        killed before a GC could be done. */
     if (reds == -2) {
-        ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL;
         int res;
 
-        erts_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
-        erts_send_exit_signal(p, p->common.id, p, &locks,
-                              am_kill, NIL, NULL, 0);
-        erts_proc_unlock(p, locks & ERTS_PROC_LOCKS_ALL_MINOR);
+        erts_set_self_exiting(p, am_killed);
 
     delay_gc_after_start:
         /* erts_send_exit_signal looks for ERTS_PSFLG_GC, so
@@ -1375,7 +1392,7 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
 		 new_sz, objv, nobj);
 
 	if (p->flags & F_ON_HEAP_MSGQ)
-	    move_msgq_to_heap(p);
+	    move_msgs_to_heap(p);
 
 	new_mature = p->old_htop - prev_old_htop;
 
@@ -1760,7 +1777,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
     HIGH_WATER(p) = HEAP_TOP(p);
 
     if (p->flags & F_ON_HEAP_MSGQ)
-	move_msgq_to_heap(p);
+	move_msgs_to_heap(p);
 
     ErtsGcQuickSanityCheck(p);
 
@@ -2332,9 +2349,9 @@ collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, Eterm* n_htop)
     return n_htop;
 }
 
-static ERTS_INLINE void
-copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap,
-	      ErlHeapFragment *bp, Eterm *refs, int nrefs)
+void
+erts_copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap,
+                   ErlHeapFragment *bp, Eterm *refs, int nrefs)
 {
     Uint sz;
     int i;
@@ -2440,61 +2457,6 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap,
     bp->off_heap.first = NULL;
 }
 
-static void
-move_msgq_to_heap(Process *p)
-{
-    
-    ErtsMessage **mpp = &p->msg.first;
-    Uint64 pre_oh = MSO(p).overhead;
-
-    while (*mpp) {
-	ErtsMessage *mp = *mpp;
-
-	if (mp->data.attached) {
-	    ErlHeapFragment *bp;
-
-	    /*
-	     * We leave not yet decoded distribution messages
-	     * as they are in the queue since it is not
-	     * possible to determine a maximum size until
-	     * actual decoding...
-	     */
-	    if (is_value(ERL_MESSAGE_TERM(mp))) {
-
-                bp = erts_message_to_heap_frag(mp);
-
-		if (bp->next)
-		    erts_move_multi_frags(&p->htop, &p->off_heap, bp,
-					  mp->m, ERL_MESSAGE_REF_ARRAY_SZ, 0);
-		else
-		    copy_one_frag(&p->htop, &p->off_heap, bp,
-				  mp->m, ERL_MESSAGE_REF_ARRAY_SZ);
-
-		if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
-		    mp->data.heap_frag = NULL;
-		    free_message_buffer(bp);
-		}
-		else {
-		    ErtsMessage *new_mp = erts_alloc_message(0, NULL);
-		    sys_memcpy((void *) new_mp->m, (void *) mp->m,
-			       sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ);
-		    erts_msgq_replace_msg_ref(&p->msg, new_mp, mpp);
-		    mp->next = NULL;
-		    erts_cleanup_messages(mp);
-		    mp = new_mp;
-		}
-	    }
-	}
-
-	mpp = &(*mpp)->next;
-    }
-
-    if (pre_oh != MSO(p).overhead) {
-	/* Got new binaries; update vheap size... */
-	BIN_VHEAP_SZ(p) = next_vheap_size(p, MSO(p).overhead, BIN_VHEAP_SZ(p));
-    }
-}
-
 static Uint
 setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
 {
@@ -2583,11 +2545,10 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
 	 * We do not have off heap message queue enabled, i.e. we
 	 * need to add message queue to rootset...
 	 */
-	ErtsMessage *mp;
 
 	/* Ensure large enough rootset... */
-	if (n + p->msg.len > rootset->size) {
-	    Uint new_size = n + p->msg.len;
+	if (n + p->sig_qs.len > rootset->size) {
+	    Uint new_size = n + p->sig_qs.len;
 	    ERTS_GC_ASSERT(roots == rootset->def);
 	    roots = erts_alloc(ERTS_ALC_T_ROOTSET,
 			       new_size*sizeof(Roots));
@@ -2595,18 +2556,19 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
 	    rootset->size = new_size;
 	}
 
-	for (mp = p->msg.first; mp; mp = mp->next) {
-
-	    if (!mp->data.attached) {
-		/*
-		 * Message may refer data on heap;
-		 * add it to rootset...
-		 */
-		roots[n].v = mp->m;
-		roots[n].sz = ERL_MESSAGE_REF_ARRAY_SZ;
-		n++;
-	    }
-	}
+        ERTS_FOREACH_SIG_PRIVQS(
+            p, mp,
+            {
+                if (ERTS_SIG_IS_INTERNAL_MSG(mp) && !mp->data.attached) {
+                    /*
+                     * Message may refer data on heap;
+                     * add it to rootset...
+                     */
+                    roots[n].v = mp->m;
+                    roots[n].sz = ERL_MESSAGE_REF_ARRAY_SZ;
+                    n++;
+                }
+            });
 	break;
     }
     }
@@ -3117,51 +3079,59 @@ offset_off_heap(Process* p, Sint offs, char* area, Uint area_size)
     }
 }
 
+#ifndef USE_VM_PROBES
+#define ERTS_OFFSET_DT_UTAG(MP, A, ASZ, O)
+#else
+#define ERTS_OFFSET_DT_UTAG(MP, A, ASZ, O)                                      \
+    do {                                                                        \
+        Eterm utag__ = ERL_MESSAGE_DT_UTAG((MP));                               \
+        if (is_boxed(utag__) && ErtsInArea(ptr_val(utag__), (A), (ASZ))) {      \
+            ERL_MESSAGE_DT_UTAG((MP)) = offset_ptr(utag__, (O));                \
+        }                                                                       \
+    } while (0)
+#endif
+
+static ERTS_INLINE void
+offset_message(ErtsMessage *mp, Sint offs, char* area, Uint area_size)
+{
+    Eterm mesg = ERL_MESSAGE_TERM(mp);
+    if (ERTS_SIG_IS_MSG_TAG(mesg)) {
+        if (ERTS_SIG_IS_INTERNAL_MSG_TAG(mesg)) {
+            switch (primary_tag(mesg)) {
+            case TAG_PRIMARY_LIST:
+            case TAG_PRIMARY_BOXED:
+                if (ErtsInArea(ptr_val(mesg), area, area_size)) {
+                    ERL_MESSAGE_TERM(mp) = offset_ptr(mesg, offs);
+                }
+                break;
+            }
+        }
+        mesg = ERL_MESSAGE_TOKEN(mp);
+        if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) {
+            ERL_MESSAGE_TOKEN(mp) = offset_ptr(mesg, offs);
+        }
+
+        ERTS_OFFSET_DT_UTAG(mp, area, area_size, offs);
+	
+        ASSERT((is_nil(ERL_MESSAGE_TOKEN(mp)) ||
+                is_tuple(ERL_MESSAGE_TOKEN(mp)) ||
+                is_atom(ERL_MESSAGE_TOKEN(mp))));
+    }
+}
+
 /*
  * Offset pointers in message queue.
  */
 static void
 offset_mqueue(Process *p, Sint offs, char* area, Uint area_size)
 {
-    ErtsMessage* mp = p->msg.first;
-
-    if ((p->flags & (F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG)) != F_OFF_HEAP_MSGQ) {
-
-	while (mp != NULL) {
-	    Eterm mesg = ERL_MESSAGE_TERM(mp);
-	    if (is_value(mesg)) {
-		switch (primary_tag(mesg)) {
-		case TAG_PRIMARY_LIST:
-		case TAG_PRIMARY_BOXED:
-		    if (ErtsInArea(ptr_val(mesg), area, area_size)) {
-			ERL_MESSAGE_TERM(mp) = offset_ptr(mesg, offs);
-		    }
-		    break;
-		}
-	    }
-	    mesg = ERL_MESSAGE_TOKEN(mp);
-	    if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) {
-		ERL_MESSAGE_TOKEN(mp) = offset_ptr(mesg, offs);
-	    }
-#ifdef USE_VM_PROBES
-	    mesg = ERL_MESSAGE_DT_UTAG(mp);
-	    if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) {
-		ERL_MESSAGE_DT_UTAG(mp) = offset_ptr(mesg, offs);
-	    }
-#endif	
-	
-	    ASSERT((is_nil(ERL_MESSAGE_TOKEN(mp)) ||
-		    is_tuple(ERL_MESSAGE_TOKEN(mp)) ||
-		    is_atom(ERL_MESSAGE_TOKEN(mp))));
-	    mp = mp->next;
-	}
-
-    }
+    if ((p->flags & (F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG)) != F_OFF_HEAP_MSGQ)
+        ERTS_FOREACH_SIG_PRIVQS(p, mp, offset_message(mp, offs, area, area_size));
 }
 
 static void ERTS_INLINE
 offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size,
-	       Eterm* objv, int nobj)
+                   Eterm* objv, int nobj)
 {
     Eterm *v;
     Uint sz;
@@ -3378,7 +3348,6 @@ erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp,
     };
 
     Eterm res = THE_NON_VALUE;
-    ErtsMessage *mp;
 
     ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(*tags));
     ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == ERTS_PROCESS_GC_INFO_MAX_TERMS);
@@ -3397,9 +3366,15 @@ erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp,
            be the same as adding old_heap_block_size + heap_block_size
            + mbuf_size.
         */
-        for (mp = p->msg.first; mp; mp = mp->next)
-            if (mp->data.attached)
-                values[2] += erts_msg_attached_data_size(mp);
+        ERTS_FOREACH_SIG_PRIVQS(
+            p, mp,
+            {
+                if (ERTS_SIG_IS_MSG(mp)
+                    && mp->data.attached
+                    && mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
+                    values[2] += erts_msg_attached_data_size(mp);
+                }
+            });
     }
 
     res = erts_bld_atom_uword_2tup_list(hpp,
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index dec0ab1143..63d03cdf8b 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2007-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -181,6 +181,8 @@ void erts_free_heap_frags(struct process* p);
 Eterm erts_max_heap_size_map(Sint, Uint, Eterm **, Uint *);
 int erts_max_heap_size(Eterm, Uint *, Uint *);
 void erts_deallocate_young_generation(Process *c_p);
+void erts_copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap,
+                        ErlHeapFragment *bp, Eterm *refs, int nrefs);
 #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG)
 int erts_dbg_within_proc(Eterm *ptr, Process *p, Eterm* real_htop);
 #endif
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index bda2c9b94d..6ec6f8065e 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2015-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2015-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -38,6 +38,7 @@
 #define ERTS_WANT_TIMER_WHEEL_API
 #include "erl_time.h"
 #include "erl_hl_timer.h"
+#include "erl_proc_sig_queue.h"
 #ifdef ERTS_MAGIC_REF_BIF_TIMERS
 #include "erl_binary.h"
 #endif
@@ -2470,28 +2471,21 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info)
 	    req->rrefn[1] = rrefn[1];
 	    req->rrefn[2] = rrefn[2];
 
-	    erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-
-	    if (ERTS_PROC_PENDING_EXIT(c_p))
-		ERTS_VBUMP_ALL_REDS(c_p);
-	    else {
-		/*
-		 * Caller needs to wait for a message containing
-		 * the ref that we just created. No such message
-		 * can exist in callers message queue at this time.
-		 * We therefore move the save pointer of the
-		 * callers message queue to the end of the queue.
-		 *
-		 * NOTE: It is of vital importance that the caller
-		 *       immediately do a receive unconditionaly
-		 *       waiting for the message with the reference;
-		 *       otherwise, next receive will *not* work
-		 *       as expected!
-		 */
-		ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
-		c_p->msg.save = c_p->msg.last;
-	    }
-	    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+            /*
+             * Caller needs to wait for a message containing
+             * the ref that we just created. No such message
+             * can exist in callers message queue at this time.
+             * We therefore move the save pointer of the
+             * callers message queue to the end of the queue.
+             *
+             * NOTE: It is of vital importance that the caller
+             *       immediately do a receive unconditionaly
+             *       waiting for the message with the reference;
+             *       otherwise, next receive will *not* work
+             *       as expected!
+             */
+            ERTS_RECV_MARK_SAVE(c_p);
+            ERTS_RECV_MARK_SET(c_p);
 
 	    ERTS_BIF_PREP_TRAP1(ret, erts_await_result, c_p, rref);
 	}
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 8430a5559b..83ed2e4cd6 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -51,6 +51,8 @@
 #include "erl_time.h"
 #include "erl_check_io.h"
 #include "erl_osenv.h"
+#include "erl_proc_sig_queue.h"
+
 #ifdef HIPE
 #include "hipe_mode_switch.h"	/* for hipe_mode_switch_init() */
 #include "hipe_signal.h"	/* for hipe_signal_init() */
@@ -126,6 +128,8 @@ const Eterm etp_hole_marker = 0;
 
 static int modified_sched_thread_suggested_stack_size = 0;
 
+Eterm erts_init_process_id;
+
 /*
  * Note about VxWorks: All variables must be initialized by executable code,
  * not by an initializer. Otherwise a new instance of the emulator will
@@ -304,8 +308,9 @@ erl_init(int ncpu,
 	 int node_tab_delete_delay,
 	 ErtsDbSpinCount db_spin_count)
 {
+    erts_monitor_link_init();
+    erts_proc_sig_queue_init();
     erts_bif_unique_init();
-    erts_init_monitors();
     erts_init_time(time_correction, time_warp_mode);
     erts_init_sys_common_misc();
     erts_init_process(ncpu, proc_tab_sz, legacy_proc_tab);
@@ -413,7 +418,7 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char**
 }
 
 static Eterm
-erl_system_process_otp(Eterm parent_pid, char* modname, int off_heap_msgq)
+erl_system_process_otp(Eterm parent_pid, char* modname, int off_heap_msgq, int prio)
 {
     Eterm start_mod;
     Process* parent;
@@ -428,9 +433,16 @@ erl_system_process_otp(Eterm parent_pid, char* modname, int off_heap_msgq)
 
     parent = erts_pid2proc(NULL, 0, parent_pid, ERTS_PROC_LOCK_MAIN);
 
-    so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC;
+    so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC|SPO_USE_ARGS;
     if (off_heap_msgq)
         so.flags |= SPO_OFF_HEAP_MSGQ;
+    so.min_heap_size    = H_MIN_SIZE;
+    so.min_vheap_size   = BIN_VH_MIN_SIZE;
+    so.max_heap_size    = H_MAX_SIZE;
+    so.max_heap_flags   = H_MAX_FLAGS;
+    so.priority         = prio;
+    so.max_gen_gcs      = (Uint16) erts_atomic32_read_nob(&erts_max_gen_gcs);
+    so.scheduler        = 0;
     res = erl_create_process(parent, start_mod, am_start, NIL, &so);
     erts_proc_unlock(parent, ERTS_PROC_LOCK_MAIN);
     return res;
@@ -1200,7 +1212,6 @@ erl_start(int argc, char **argv)
     ErtsTimeWarpMode time_warp_mode;
     int node_tab_delete_delay = ERTS_NODE_TAB_DELAY_GC_DEFAULT;
     ErtsDbSpinCount db_spin_count = ERTS_DB_SPNCNT_NORMAL;
-    Eterm otp_ring0_pid;
 
     set_default_time_adj(&time_correction,
 			 &time_warp_mode);
@@ -2191,8 +2202,8 @@ erl_start(int argc, char **argv)
 
     erts_initialized = 1;
 
-    otp_ring0_pid = erl_first_process_otp("otp_ring0", NULL, 0,
-					  boot_argc, boot_argv);
+    erts_init_process_id = erl_first_process_otp("otp_ring0", NULL, 0,
+                                                 boot_argc, boot_argv);
 
     {
 	/*
@@ -2202,14 +2213,18 @@ erl_start(int argc, char **argv)
 	 */
 	Eterm pid;
 
-	pid = erl_system_process_otp(otp_ring0_pid, "erts_code_purger", !0);
+	pid = erl_system_process_otp(erts_init_process_id,
+                                     "erts_code_purger", !0,
+                                     PRIORITY_NORMAL);
 	erts_code_purger
 	    = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc,
 						    internal_pid_index(pid));
 	ASSERT(erts_code_purger && erts_code_purger->common.id == pid);
 	erts_proc_inc_refc(erts_code_purger); 
 
-	pid = erl_system_process_otp(otp_ring0_pid, "erts_literal_area_collector", !0);
+	pid = erl_system_process_otp(erts_init_process_id,
+                                     "erts_literal_area_collector",
+                                     !0, PRIORITY_NORMAL);
 	erts_literal_area_collector
 	    = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc,
 						    internal_pid_index(pid));
@@ -2217,14 +2232,35 @@ erl_start(int argc, char **argv)
 	       && erts_literal_area_collector->common.id == pid);
 	erts_proc_inc_refc(erts_literal_area_collector);
 
-	pid = erl_system_process_otp(otp_ring0_pid, "erts_dirty_process_code_checker", !0);
-	erts_dirty_process_code_checker
+	pid = erl_system_process_otp(erts_init_process_id,
+                                     "erts_dirty_process_signal_handler",
+                                     !0, PRIORITY_NORMAL);
+        erts_dirty_process_signal_handler
 	    = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc,
 						    internal_pid_index(pid));
-	ASSERT(erts_dirty_process_code_checker
-	       && erts_dirty_process_code_checker->common.id == pid);
-	erts_proc_inc_refc(erts_dirty_process_code_checker);
-
+	ASSERT(erts_dirty_process_signal_handler
+	       && erts_dirty_process_signal_handler->common.id == pid);
+	erts_proc_inc_refc(erts_dirty_process_signal_handler);
+
+	pid = erl_system_process_otp(erts_init_process_id,
+                                     "erts_dirty_process_signal_handler",
+                                     !0, PRIORITY_HIGH);
+        erts_dirty_process_signal_handler_high
+	    = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc,
+						    internal_pid_index(pid));
+	ASSERT(erts_dirty_process_signal_handler_high
+	       && erts_dirty_process_signal_handler_high->common.id == pid);
+	erts_proc_inc_refc(erts_dirty_process_signal_handler_high);
+
+	pid = erl_system_process_otp(erts_init_process_id,
+                                     "erts_dirty_process_signal_handler",
+                                     !0, PRIORITY_MAX);
+        erts_dirty_process_signal_handler_max
+	    = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc,
+						    internal_pid_index(pid));
+	ASSERT(erts_dirty_process_signal_handler_max
+	       && erts_dirty_process_signal_handler_max->common.id == pid);
+	erts_proc_inc_refc(erts_dirty_process_signal_handler_max);
     }
 
     erts_start_schedulers();
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 773b138d92..0ced5ec310 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -92,7 +92,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"db_hash_slot",				"address"		},
     {	"resource_monitors",			"address"	        },
     {   "driver_list",                          NULL                    },
-    {	"proc_link",				"pid"			},
     {	"proc_msgq",				"pid"			},
     {	"proc_btm",				"pid"			},
     {	"dist_entry",				"address"		},
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 6f7c71ef98..ed7a9d37c2 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -33,6 +33,7 @@
 #include "erl_binary.h"
 #include "dtrace-wrapper.h"
 #include "beam_bp.h"
+#include "erl_proc_sig_queue.h"
 
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message_ref,
 				 ErtsMessageRef,
@@ -207,7 +208,7 @@ erts_cleanup_messages(ErtsMessage *msgp)
     while (mp) {
 	ErtsMessage *fmp;
 	ErlHeapFragment *bp;
-	if (is_non_value(ERL_MESSAGE_TERM(mp))) {
+	if (ERTS_SIG_IS_EXTERNAL_MSG(mp)) {
 	    if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) {
 		bp = (ErlHeapFragment *) mp->data.dist_ext->ext_endp;
 		erts_cleanup_offheap(&bp->off_heap);
@@ -215,10 +216,13 @@ erts_cleanup_messages(ErtsMessage *msgp)
 	    if (mp->data.dist_ext)
 		erts_free_dist_ext_copy(mp->data.dist_ext);
 	}
-	else {
-	    if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG)
+        else {
+	    if (ERTS_SIG_IS_INTERNAL_MSG(mp)
+                && mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
 		bp = mp->data.heap_frag;
+            }
 	    else {
+                mp->data.attached = ERTS_MSG_COMBINED_HFRAG;
 		bp = mp->hfrag.next;
 		erts_cleanup_offheap(&mp->hfrag.off_heap);
 	    }
@@ -272,6 +276,7 @@ erts_queue_dist_message(Process *rcvr,
     mp = erts_alloc_message(0, NULL);
     mp->data.dist_ext = dist_ext;
 
+    ERL_MESSAGE_FROM(mp) = dist_ext->dep->sysname;
     ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
 #ifdef USE_VM_PROBES
     ERL_MESSAGE_DT_UTAG(mp) = NIL;
@@ -294,46 +299,15 @@ erts_queue_dist_message(Process *rcvr,
 	}
     }
 
+
     state = erts_atomic32_read_acqb(&rcvr->state);
-    if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
+    if (state & ERTS_PSFLG_EXITING) {
 	if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
 	    erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
 	/* Drop message if receiver is exiting or has a pending exit ... */
 	erts_cleanup_messages(mp);
     }
-    else
-    if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) {
-        if (from == am_Empty)
-            from = dist_ext->dep->sysname;
-
-	/* Ahh... need to decode it in order to trace it... */
-	if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
-	    erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
-	if (!erts_decode_dist_message(rcvr, rcvr_locks, mp, 0))
-	    erts_free_message(mp);
-	else {
-	    Eterm msg = ERL_MESSAGE_TERM(mp);
-	    token = ERL_MESSAGE_TOKEN(mp);
-#ifdef USE_VM_PROBES
-	    if (DTRACE_ENABLED(message_queued)) {
-		DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
-
-		dtrace_proc_str(rcvr, receiver_name);
-                if (have_seqtrace(token)) {
-		    tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
-		    tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
-		    tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
-		}
-		DTRACE6(message_queued,
-			receiver_name, size_object(msg), rcvr->msg.len,
-			tok_label, tok_lastcnt, tok_serial);
-	    }
-#endif
-	    erts_queue_message(rcvr, rcvr_locks, mp, msg, from);
-	}
-    }
     else {
-	/* Enqueue message on external format */
 
 #ifdef USE_VM_PROBES
         if (DTRACE_ENABLED(message_queued)) {
@@ -359,9 +333,7 @@ erts_queue_dist_message(Process *rcvr,
 	if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
 	    erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
 
-	erts_proc_notify_new_message(rcvr,
-				     rcvr_locks
-	    );
+	erts_proc_notify_new_message(rcvr, rcvr_locks);
     }
 }
 
@@ -372,15 +344,14 @@ queue_messages(Process* receiver,
                ErtsProcLocks receiver_locks,
                ErtsMessage* first,
                ErtsMessage** last,
-               Uint len,
-               Eterm from)
+               Uint len)
 {
-    ErtsTracingEvent* te;
     Sint res;
     int locked_msgq = 0;
     erts_aint32_t state;
 
     ASSERT(is_value(ERL_MESSAGE_TERM(first)));
+    ASSERT(is_value(ERL_MESSAGE_FROM(first)));
     ASSERT(ERL_MESSAGE_TOKEN(first) == am_undefined ||
            ERL_MESSAGE_TOKEN(first) == NIL ||
            is_tuple(ERL_MESSAGE_TOKEN(first)));
@@ -398,7 +369,7 @@ queue_messages(Process* receiver,
 		state = *receiver_state;
 	    else
 		state = erts_atomic32_read_nob(&receiver->state);
-	    if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
+	    if (state & ERTS_PSFLG_EXITING)
 		goto exiting;
 
             need_locks = receiver_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
@@ -414,7 +385,7 @@ queue_messages(Process* receiver,
 
     state = erts_atomic32_read_nob(&receiver->state);
 
-    if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) {
+    if (state & ERTS_PSFLG_EXITING) {
     exiting:
 	/* Drop message if receiver is exiting or has a pending exit... */
 	if (locked_msgq)
@@ -423,7 +394,7 @@ queue_messages(Process* receiver,
 	return 0;
     }
 
-    res = receiver->msg.len;
+    res = receiver->sig_qs.len;
     if (receiver_locks & ERTS_PROC_LOCK_MAIN) {
 	/*
 	 * We move 'in queue' to 'private queue' and place
@@ -433,8 +404,8 @@ queue_messages(Process* receiver,
 	 * we don't need to include the 'in queue' in
 	 * the root set when garbage collecting.
 	 */
-	res += receiver->msg_inq.len;
-	ERTS_MSGQ_MV_INQ2PRIVQ(receiver);
+	res += receiver->sig_inq.len;
+        erts_proc_sig_fetch(receiver);
         LINK_MESSAGE_PRIVQ(receiver, first, last, len);
     }
     else
@@ -442,38 +413,6 @@ queue_messages(Process* receiver,
 	LINK_MESSAGE(receiver, first, last, len);
     }
 
-    if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)
-        && (te = &erts_receive_tracing[erts_active_bp_ix()],
-            te->on)) {
-
-        ErtsMessage *msg = first;
-
-#ifdef USE_VM_PROBES
-        if (DTRACE_ENABLED(message_queued)) {
-            DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
-            Sint tok_label = 0;
-            Sint tok_lastcnt = 0;
-            Sint tok_serial = 0;
-            Eterm seq_trace_token = ERL_MESSAGE_TOKEN(msg);
-
-            dtrace_proc_str(receiver, receiver_name);
-            if (seq_trace_token != NIL && is_tuple(seq_trace_token)) {
-                tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token));
-                tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token));
-                tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token));
-            }
-            DTRACE6(message_queued,
-                    receiver_name, size_object(ERL_MESSAGE_TERM(msg)),
-                    receiver->msg.len,
-                    tok_label, tok_lastcnt, tok_serial);
-        }
-#endif
-        while (msg) {
-            trace_receive(receiver, from, ERL_MESSAGE_TERM(msg), te);
-            msg = msg->next;
-        }
-
-    }
     if (locked_msgq) {
 	erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
     }
@@ -489,8 +428,9 @@ queue_message(Process* receiver,
               ErtsMessage* mp, Eterm msg, Eterm from)
 {
     ERL_MESSAGE_TERM(mp) = msg;
+    ERL_MESSAGE_FROM(mp) = from;
     return queue_messages(receiver, receiver_state, receiver_locks,
-                          mp, &mp->next, 1, from);
+                          mp, &mp->next, 1);
 }
 
 Sint
@@ -503,11 +443,10 @@ erts_queue_message(Process* receiver, ErtsProcLocks receiver_locks,
 
 Sint
 erts_queue_messages(Process* receiver, ErtsProcLocks receiver_locks,
-                    ErtsMessage* first, ErtsMessage** last, Uint len,
-                    Eterm from)
+                    ErtsMessage* first, ErtsMessage** last, Uint len)
 {
     return queue_messages(receiver, NULL, receiver_locks,
-                          first, last, len, from);
+                          first, last, len);
 }
 
 void
@@ -922,70 +861,77 @@ Sint
 erts_move_messages_off_heap(Process *c_p)
 {
     int reds = 1;
+    int i;
+    ErtsMessage *msgq[] = {c_p->sig_qs.first, c_p->sig_qs.cont};
     /*
      * Move all messages off heap. This *only* occurs when the
      * process had off heap message disabled and just enabled
      * it...
      */
-    ErtsMessage *mp;
 
-    reds += c_p->msg.len / 10;
+    reds += c_p->sig_qs.len / 10;
 
     ASSERT(erts_atomic32_read_nob(&c_p->state)
 	   & ERTS_PSFLG_OFF_HEAP_MSGQ);
     ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG);
 
-    for (mp = c_p->msg.first; mp; mp = mp->next) {
-	Uint msg_sz, token_sz;
+    for (i = 0; i < sizeof(msgq)/sizeof(msgq[0]); i++) {
+        ErtsMessage *mp;
+        for (mp = msgq[i]; mp; mp = mp->next) {
+            Uint msg_sz, token_sz;
 #ifdef USE_VM_PROBES
-	Uint utag_sz;
+            Uint utag_sz;
 #endif
-	Eterm *hp;
-	ErlHeapFragment *hfrag;
+            Eterm *hp;
+            ErlHeapFragment *hfrag;
+
+            if (!ERTS_SIG_IS_INTERNAL_MSG(mp))
+                continue;
 
-	if (mp->data.attached)
-	    continue;
+            if (mp->data.attached)
+                continue;
 
-	if (is_immed(ERL_MESSAGE_TERM(mp))
+            if (is_immed(ERL_MESSAGE_TERM(mp))
 #ifdef USE_VM_PROBES
-	    && is_immed(ERL_MESSAGE_DT_UTAG(mp))
+                && is_immed(ERL_MESSAGE_DT_UTAG(mp))
 #endif
-	    && is_not_immed(ERL_MESSAGE_TOKEN(mp)))
-	    continue;
+                && is_not_immed(ERL_MESSAGE_TOKEN(mp)))
+                continue;
 
-	/*
-	 * The message refers into the heap. Copy the message
-	 * from the heap into a heap fragment and attach
-	 * it to the message...
-	 */
-	msg_sz = size_object(ERL_MESSAGE_TERM(mp));
+            /*
+             * The message refers into the heap. Copy the message
+             * from the heap into a heap fragment and attach
+             * it to the message...
+             */
+            msg_sz = size_object(ERL_MESSAGE_TERM(mp));
 #ifdef USE_VM_PROBES
-	utag_sz = size_object(ERL_MESSAGE_DT_UTAG(mp));
+            utag_sz = size_object(ERL_MESSAGE_DT_UTAG(mp));
 #endif
-	token_sz = size_object(ERL_MESSAGE_TOKEN(mp));
+            token_sz = size_object(ERL_MESSAGE_TOKEN(mp));
 
-	hfrag = new_message_buffer(msg_sz
+            hfrag = new_message_buffer(msg_sz
 #ifdef USE_VM_PROBES
-				   + utag_sz
+                                       + utag_sz
 #endif
-				   + token_sz);
-	hp = hfrag->mem;
-	if (is_not_immed(ERL_MESSAGE_TERM(mp)))
-	    ERL_MESSAGE_TERM(mp) = copy_struct(ERL_MESSAGE_TERM(mp),
-                                               msg_sz, &hp,
-                                               &hfrag->off_heap);
-	if (is_not_immed(ERL_MESSAGE_TOKEN(mp)))
-	    ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp),
-						token_sz, &hp,
-						&hfrag->off_heap);
+                                       + token_sz);
+            hp = hfrag->mem;
+            if (is_not_immed(ERL_MESSAGE_TERM(mp)))
+                ERL_MESSAGE_TERM(mp) = copy_struct(ERL_MESSAGE_TERM(mp),
+                                                   msg_sz, &hp,
+                                                   &hfrag->off_heap);
+            if (is_not_immed(ERL_MESSAGE_TOKEN(mp)))
+                ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp),
+                                                    token_sz, &hp,
+                                                    &hfrag->off_heap);
 #ifdef USE_VM_PROBES
-	if (is_not_immed(ERL_MESSAGE_DT_UTAG(mp)))
-	    ERL_MESSAGE_DT_UTAG(mp) = copy_struct(ERL_MESSAGE_DT_UTAG(mp),
-						  utag_sz, &hp,
-						  &hfrag->off_heap);
+            if (is_not_immed(ERL_MESSAGE_DT_UTAG(mp)))
+                ERL_MESSAGE_DT_UTAG(mp) = copy_struct(ERL_MESSAGE_DT_UTAG(mp),
+                                                      utag_sz, &hp,
+                                                      &hfrag->off_heap);
 #endif
-	mp->data.heap_frag = hfrag;
-	reds += 1;
+            mp->data.heap_frag = hfrag;
+            reds += 1;
+        }
     }
 
     return reds;
@@ -1015,7 +961,7 @@ erts_complete_off_heap_message_queue_change(Process *c_p)
     else {
 	reds += 2;
 	erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
-	ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
+        erts_proc_sig_fetch(c_p);
 	erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
 	reds += erts_move_messages_off_heap(c_p);
     }
@@ -1225,139 +1171,6 @@ erts_decode_dist_message(Process *proc, ErtsProcLocks proc_locks,
     return 1;
 }
 
-/*
- * ERTS_INSPECT_MSGQ_KEEP_OH_MSGS == 0 will move off heap messages
- * into the heap of the inspected process if off_heap_message_queue
- * is false when process_info(_, messages) is called. That is, the
- * following GC will have more data in the rootset compared to the
- * scenario when process_info(_, messages) had not been called.
- *
- * ERTS_INSPECT_MSGQ_KEEP_OH_MSGS != 0 will keep off heap messages
- * off heap when process_info(_, messages) is called regardless of
- * the off_heap_message_queue setting of the process. That is, it
- * will change the following execution of the process as little as
- * possible.
- */
-#define ERTS_INSPECT_MSGQ_KEEP_OH_MSGS 1
-
-Uint
-erts_prep_msgq_for_inspection(Process *c_p, Process *rp,
-			      ErtsProcLocks rp_locks, ErtsMessageInfo *mip)
-{
-    Uint tot_heap_size;
-    ErtsMessage* mp;
-    Sint i;
-    int self_on_heap;
-    
-    /*
-     * Prepare the message queue for inspection
-     * by process_info().
-     *
-     *
-     * - Decode all messages on external format
-     * - Remove all corrupt dist messages from queue
-     * - Save pointer to, and heap size need of each
-     *   message in the mip array.
-     * - Return total heap size need for all messages
-     *   that needs to be copied.
-     *
-     * If ERTS_INSPECT_MSGQ_KEEP_OH_MSGS == 0:
-     * - In case off heap messages is disabled and
-     *   we are inspecting our own queue, move all
-     *   off heap data into the heap.
-     */
-
-    self_on_heap = c_p == rp && !(c_p->flags & F_OFF_HEAP_MSGQ);
-
-    tot_heap_size = 0;
-    i = 0;
-    mp = rp->msg.first;
-    while (mp) {
-	Eterm msg = ERL_MESSAGE_TERM(mp);
-
-	mip[i].size = 0;
-
-	if (is_non_value(msg)) {
-	    /* Dist message on external format; decode it... */
-	    if (mp->data.attached)
-		erts_decode_dist_message(rp, rp_locks, mp,
-					 ERTS_INSPECT_MSGQ_KEEP_OH_MSGS);
-
-	    msg = ERL_MESSAGE_TERM(mp);
-
-	    if (is_non_value(msg)) {
-		ErtsMessage **mpp;
-		ErtsMessage *bad_mp = mp;
-		/*
-		 * Bad distribution message; remove
-		 * it from the queue...
-		 */
-		ASSERT(!mp->data.attached);
-
-		mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next;
-
-		ASSERT(*mpp == bad_mp);
-
-		erts_msgq_update_internal_pointers(&rp->msg, mpp, &bad_mp->next);
-
-		mp = mp->next;
-		*mpp = mp;
-		rp->msg.len--;
-		bad_mp->next = NULL;
-		erts_cleanup_messages(bad_mp);
-		continue;
-	    }
-	}
-
-	ASSERT(is_value(msg));
-
-#if ERTS_INSPECT_MSGQ_KEEP_OH_MSGS
-	if (is_not_immed(msg) && (!self_on_heap || mp->data.attached)) {
-	    Uint sz = size_object(msg);
-	    mip[i].size = sz;
-	    tot_heap_size += sz;
-	}
-#else
-	if (self_on_heap) {
-	    if (mp->data.attached) {
-		ErtsMessage *tmp = NULL;
-		if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
-		    erts_link_mbuf_to_proc(rp, mp->data.heap_frag);
-		    mp->data.attached = NULL;
-		}
-		else {
-		    /*
-		     * Need to replace the message reference since
-		     * we will get references to the message data
-		     * from the heap...
-		     */
-		    ErtsMessage **mpp;
-		    tmp = erts_alloc_message(0, NULL);
-		    sys_memcpy((void *) tmp->m, (void *) mp->m,
-			       sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ); 
-		    mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next;
-		    erts_msgq_replace_msg_ref(&rp->msg, tmp, mpp);
-		    erts_save_message_in_proc(rp, mp);
-		    mp = tmp;
-		}
-	    }
-	}
-	else if (is_not_immed(msg)) {
-	    Uint sz = size_object(msg);
-	    mip[i].size = sz;
-	    tot_heap_size += sz;
-	}
-
-#endif
-
-	mip[i].msgp = mp;
-	i++;
-	mp = mp->next;
-    }
-
-    return tot_heap_size;
-}
-
 void erts_factory_proc_init(ErtsHeapFactory* factory,
 			    Process* p)
 {
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index a14f4f51d8..f56a252aef 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,6 +21,11 @@
 #ifndef __ERL_MESSAGE_H__
 #define __ERL_MESSAGE_H__
 
+#include "sys.h"
+#define ERTS_PROC_SIG_QUEUE_TYPE_ONLY
+#include "erl_proc_sig_queue.h"
+#undef ERTS_PROC_SIG_QUEUE_TYPE_ONLY
+
 struct proc_bin;
 struct external_thing_;
 
@@ -118,15 +123,16 @@ struct erl_heap_fragment {
 };
 
 /* m[0] = message, m[1] = seq trace token */
-#define ERL_MESSAGE_REF_ARRAY_SZ 2
+#define ERL_MESSAGE_REF_ARRAY_SZ 3
 #define ERL_MESSAGE_TERM(mp) ((mp)->m[0])
 #define ERL_MESSAGE_TOKEN(mp) ((mp)->m[1])
+#define ERL_MESSAGE_FROM(mp) ((mp)->m[2])
 
 #ifdef USE_VM_PROBES
 /* m[2] = dynamic trace user tag */
 #undef ERL_MESSAGE_REF_ARRAY_SZ
-#define ERL_MESSAGE_REF_ARRAY_SZ 3
-#define ERL_MESSAGE_DT_UTAG(mp) ((mp)->m[2])
+#define ERL_MESSAGE_REF_ARRAY_SZ 4
+#define ERL_MESSAGE_DT_UTAG(mp) ((mp)->m[3])
 #else
 #endif
 
@@ -157,28 +163,79 @@ struct erl_mesg {
     ErlHeapFragment hfrag;
 };
 
+/*
+ * The ErtsMessage struct is only one special type
+ * of signal. All signal structs have a common
+ * begining and can be differentiated by looking
+ * at the ErtsSignal 'common.tag' field.
+ *
+ * - An ordinary message will have a value
+ * - A distribution message that has not been
+ *   decoded yet will have the non-value.
+ * - Other signals will have an external pid
+ *   header tag. In order to differentiate
+ *   between those signals one needs to look
+ *   at the arity part of the header (see
+ *   erts_proc_sig_queue.h).
+ */
+
+#define ERTS_SIG_IS_NON_MSG_TAG(Tag) \
+    (is_external_pid_header((Tag)))
+
+#define ERTS_SIG_IS_NON_MSG(Sig) \
+    ERTS_SIG_IS_NON_MSG_TAG(((ErtsSignal *) (Sig))->common.tag)
+
+#define ERTS_SIG_IS_INTERNAL_MSG_TAG(Tag) \
+    (!is_header((Tag)))
+#define ERTS_SIG_IS_INTERNAL_MSG(Sig) \
+    ERTS_SIG_IS_INTERNAL_MSG_TAG(((ErtsSignal *) (Sig))->common.tag)
+
+#define ERTS_SIG_IS_EXTERNAL_MSG_TAG(Tag) \
+    ((Tag) == THE_NON_VALUE)
+#define ERTS_SIG_IS_EXTERNAL_MSG(Sig) \
+    ERTS_SIG_IS_EXTERNAL_MSG_TAG(((ErtsSignal *) (Sig))->common.tag)
+
+#define ERTS_SIG_IS_MSG_TAG(Tag) \
+    (!ERTS_SIG_IS_NON_MSG_TAG(Tag))
+#define ERTS_SIG_IS_MSG(Sig) \
+    ERTS_SIG_IS_MSG_TAG(((ErtsSignal *) (Sig))->common.tag)
+
+typedef union {
+    ErtsSignalCommon common;
+    ErtsMessageRef msg;
+} ErtsSignal;
+
+typedef struct {
+    /* pointers to next pointers pointing to... */
+    ErtsMessage **next; /* ... next (non-message) signal */
+    ErtsMessage **last; /* ... next (non-message) signal */
+} ErtsMsgQNMSigs;
+
 /* Size of default message buffer (erl_message.c) */
 #define ERL_MESSAGE_BUF_SZ 500
 
 typedef struct {
-    ErtsMessage* first;
-    ErtsMessage** last;  /* point to the last next pointer */
-    ErtsMessage** save;
-    Sint len;            /* queue length */
+    /* inner queue */
+    ErtsMessage *first;
+    ErtsMessage **last;  /* point to the last next pointer */
+    ErtsMessage **save;
 
-    /*
-     * The following field is used by the recv_mark/1 and
-     * recv_set/1 instructions.
-     */
-    ErtsMessage** saved_last;	/* saved last pointer */
-} ErlMessageQueue;
+    /* middle queue */
+    ErtsMessage *cont;
+    ErtsMessage **cont_last;
+    ErtsMsgQNMSigs nmsigs;
 
+    /* Common for inner and middle queue */
+    ErtsMessage **saved_last;	/* saved last pointer */
+    Sint len; /* message queue length (inner+middle) */
+} ErtsSignalPrivQueues;
 
 typedef struct {
     ErtsMessage* first;
     ErtsMessage** last;  /* point to the last next pointer */
     Sint len;            /* queue length */
-} ErlMessageInQueue;
+    ErtsMsgQNMSigs nmsigs;
+} ErtsSignalInQueue;
 
 typedef struct erl_trace_message_queue__ {
     struct erl_trace_message_queue__ *next; /* point to the next receiver */
@@ -188,9 +245,46 @@ typedef struct erl_trace_message_queue__ {
     Sint len;            /* queue length */
 } ErlTraceMessageQueue;
 
+#define ERTS_RECV_MARK_SAVE(P)                                          \
+    do {                                                                \
+        erts_proc_lock((P), ERTS_PROC_LOCK_MSGQ);                       \
+        if ((P)->sig_inq.first)                                         \
+            erts_proc_sig_fetch((P));                                   \
+        erts_proc_unlock((P), ERTS_PROC_LOCK_MSGQ);                     \
+        if ((P)->sig_qs.cont) {                                         \
+            (P)->sig_qs.saved_last = (P)->sig_qs.cont_last;             \
+            (P)->flags |= F_DEFERRED_SAVED_LAST;                        \
+        }                                                               \
+        else {                                                          \
+            (P)->sig_qs.saved_last = (P)->sig_qs.last;                  \
+            (P)->flags &= ~F_DEFERRED_SAVED_LAST;                       \
+        }                                                               \
+    } while (0)
+
+#define ERTS_RECV_MARK_SET(P)                                           \
+    do {                                                                \
+        if ((P)->sig_qs.saved_last) {                                   \
+            if ((P)->flags & F_DEFERRED_SAVED_LAST) {                   \
+                /* Points to middle queue; use end of inner */          \
+                (P)->sig_qs.save = (P)->sig_qs.last;                    \
+                ASSERT(!PEEK_MESSAGE((P)));                             \
+            }                                                           \
+            else {                                                      \
+                /* Points to inner queue; safe to use */                \
+                (P)->sig_qs.save = (P)->sig_qs.saved_last;              \
+            }                                                           \
+        }                                                               \
+    } while (0)
+
+#define ERTS_RECV_MARK_CLEAR(P)                                         \
+    do {                                                                \
+        (P)->sig_qs.saved_last = NULL;                                  \
+        (P)->flags &= ~F_DEFERRED_SAVED_LAST;                           \
+    } while (0)
+
 
 /* Get "current" message */
-#define PEEK_MESSAGE(p)  (*(p)->msg.save)
+#define PEEK_MESSAGE(p)  (*(p)->sig_qs.save)
 
 #ifdef USE_VM_PROBES
 #define LINK_MESSAGE_DTAG(mp, dt) ERL_MESSAGE_DT_UTAG(mp) = dt
@@ -198,58 +292,69 @@ typedef struct erl_trace_message_queue__ {
 #define LINK_MESSAGE_DTAG(mp, dt)
 #endif
 
-#define LINK_MESSAGE_IMPL(p, first_msg, last_msg, num_msgs, where) do { \
-        *(p)->where.last = (first_msg);                                 \
-        (p)->where.last = (last_msg);                                   \
-        (p)->where.len += (num_msgs);                                   \
-    } while(0)
+#ifdef USE_VM_PROBES
+#  define ERTS_MSG_RECV_TRACED(P)                                       \
+    ((ERTS_TRACE_FLAGS((P)) & F_TRACE_RECEIVE)                          \
+     || DTRACE_ENABLED(message_queued))
+#else
+#  define ERTS_MSG_RECV_TRACED(P)                                       \
+    (ERTS_TRACE_FLAGS((P)) & F_TRACE_RECEIVE)
+
+#endif
 
 
 /* Add message last in private message queue */
-#define LINK_MESSAGE_PRIVQ(p, first_msg, last_msg, len)                 \
+#define LINK_MESSAGE_PRIVQ(p, first_msg, last_msg, num_msgs)            \
     do {                                                                \
-        LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg);            \
+        ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((p), "before");             \
+        if ((p)->sig_qs.cont || ERTS_MSG_RECV_TRACED((p))) {            \
+            *(p)->sig_qs.cont_last = (first_msg);                       \
+            (p)->sig_qs.cont_last = (last_msg);                         \
+        }                                                               \
+        else {                                                          \
+            *(p)->sig_qs.last = (first_msg);                            \
+            (p)->sig_qs.last = (last_msg);                              \
+        }                                                               \
+        (p)->sig_qs.len += (num_msgs);                                  \
+        ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((p), "after");              \
     } while (0)
 
 /* Add message last_msg in message queue */
-#define LINK_MESSAGE(p, first_msg, last_msg, len)       \
-    LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg_inq)
-
-#define ERTS_MSGQ_MV_INQ2PRIVQ(p)                   \
-    do {                                                \
-        if (p->msg_inq.first) {                         \
-            *p->msg.last = p->msg_inq.first;            \
-            p->msg.last = p->msg_inq.last;              \
-            p->msg.len += p->msg_inq.len;               \
-            p->msg_inq.first = NULL;                    \
-            p->msg_inq.last = &p->msg_inq.first;        \
-            p->msg_inq.len = 0;                         \
-        }                                               \
-    } while (0)
-
+#define LINK_MESSAGE(p, first_msg, last_msg, num_msgs) \
+    do {                                                                \
+        ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__((p), "before");               \
+        *(p)->sig_inq.last = (first_msg);                               \
+        (p)->sig_inq.last = (last_msg);                                 \
+        (p)->sig_inq.len += (num_msgs);                                 \
+        ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__((p), "before");               \
+    } while(0)
 
 /* Unlink current message */
-#define UNLINK_MESSAGE(p,msgp) do { \
-     ErtsMessage* __mp = (msgp)->next; \
-     *(p)->msg.save = __mp; \
-     (p)->msg.len--; \
-     if (__mp == NULL) \
-         (p)->msg.last = (p)->msg.save; \
-} while(0)
+#define UNLINK_MESSAGE(p,msgp)                                          \
+    do {                                                                \
+        ErtsMessage *mp__ = (msgp)->next;                               \
+        ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((p), "before");             \
+        *(p)->sig_qs.save = mp__;                                       \
+        (p)->sig_qs.len--;                                              \
+        if (mp__ == NULL)                                               \
+            (p)->sig_qs.last = (p)->sig_qs.save;                        \
+        ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((p), "after");              \
+    } while(0)
 
 /*
  * Reset message save point (after receive match).
  * Also invalidate the saved position since it may no
  * longer be safe to use.
  */
-#define JOIN_MESSAGE(p) do {                    \
-    (p)->msg.save = &(p)->msg.first;            \
-    (p)->msg.saved_last = 0;                    \
-} while(0)
+#define JOIN_MESSAGE(p)                                                 \
+   do {                                                                 \
+       (p)->sig_qs.save = &(p)->sig_qs.first;                           \
+       ERTS_RECV_MARK_CLEAR((p));                                       \
+   } while(0)
 
 /* Save current message */
 #define SAVE_MESSAGE(p) \
-     (p)->msg.save = &(*(p)->msg.save)->next
+     (p)->sig_qs.save = &(*(p)->sig_qs.save)->next
 
 #define ERTS_SND_FLG_NO_SEQ_TRACE		(((unsigned) 1) << 0)
 
@@ -276,6 +381,7 @@ typedef struct erl_trace_message_queue__ {
         (MP)->next = NULL;                              \
         ERL_MESSAGE_TERM(MP) = THE_NON_VALUE;           \
         ERL_MESSAGE_TOKEN(MP) = NIL;                    \
+        ERL_MESSAGE_FROM(MP) = NIL;                     \
         ERL_MESSAGE_DT_UTAG_INIT(MP);                   \
         MP->data.attached = NULL;                       \
     } while (0)
@@ -288,7 +394,7 @@ void free_message_buffer(ErlHeapFragment *);
 void erts_queue_dist_message(Process*, ErtsProcLocks, ErtsDistExternal *, Eterm, Eterm);
 Sint erts_queue_message(Process*, ErtsProcLocks,ErtsMessage*, Eterm, Eterm);
 Sint erts_queue_messages(Process*, ErtsProcLocks,
-                         ErtsMessage*, ErtsMessage**, Uint, Eterm);
+                         ErtsMessage*, ErtsMessage**, Uint);
 void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm);
 Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned);
 void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp);
@@ -305,16 +411,6 @@ int erts_decode_dist_message(Process *, ErtsProcLocks, ErtsMessage *, int);
 
 void erts_cleanup_messages(ErtsMessage *mp);
 
-typedef struct {
-    Uint size;
-    ErtsMessage *msgp;
-} ErtsMessageInfo;
-
-Uint erts_prep_msgq_for_inspection(Process *c_p,
-				   Process *rp,
-				   ErtsProcLocks rp_locks,
-				   ErtsMessageInfo *mip);
-
 void *erts_alloc_message_ref(void);
 void erts_free_message_ref(void *);
 
@@ -356,12 +452,6 @@ ERTS_GLB_FORCE_INLINE ErtsMessage *erts_shrink_message(ErtsMessage *mp, Uint sz,
 ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp);
 ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*);
 ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg);
-ERTS_GLB_INLINE void erts_msgq_update_internal_pointers(ErlMessageQueue *msgq,
-							ErtsMessage **newpp,
-							ErtsMessage **oldpp);
-ERTS_GLB_INLINE void erts_msgq_replace_msg_ref(ErlMessageQueue *msgq,
-					       ErtsMessage *newp,
-					       ErtsMessage **oldpp);
 
 #define ERTS_MSG_COMBINED_HFRAG ((void *) 0x1)
 
@@ -465,28 +555,6 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg)
     }
 }
 
-ERTS_GLB_INLINE void
-erts_msgq_update_internal_pointers(ErlMessageQueue *msgq,
-				   ErtsMessage **newpp,
-				   ErtsMessage **oldpp)
-{
-    if (msgq->save == oldpp)
-	msgq->save = newpp;
-    if (msgq->last == oldpp)
-	msgq->last = newpp;
-    if (msgq->saved_last == oldpp)
-	msgq->saved_last = newpp;
-}
-
-ERTS_GLB_INLINE void
-erts_msgq_replace_msg_ref(ErlMessageQueue *msgq, ErtsMessage *newp, ErtsMessage **oldpp)
-{
-    ErtsMessage *oldp = *oldpp;
-    newp->next = oldp->next;
-    erts_msgq_update_internal_pointers(msgq, &newp->next, &oldp->next);
-    *oldpp = newp;
-}
-
 #endif
 
 Uint erts_mbuf_size(Process *p);
@@ -500,4 +568,17 @@ Uint erts_mbuf_size(Process *p);
 #  define ERTS_CHK_MBUF_SZ(P) ((void) 1)
 #endif
 
+#define ERTS_FOREACH_SIG_PRIVQS(PROC, MVAR, CODE)                       \
+    do {                                                                \
+        int i__;                                                        \
+        ErtsMessage *msgs__[] = {(PROC)->sig_qs.first,                  \
+                                 (PROC)->sig_qs.cont};                  \
+        for (i__ = 0; i__ < sizeof(msgs__)/sizeof(msgs__[0]); i__++) {  \
+            ErtsMessage *MVAR;                                          \
+            for (MVAR = msgs__[i__]; MVAR; MVAR = MVAR->next) {         \
+                CODE;                                                   \
+            }                                                           \
+        }                                                               \
+    } while (0)
+
 #endif
diff --git a/erts/emulator/beam/erl_monitor_link.c b/erts/emulator/beam/erl_monitor_link.c
new file mode 100644
index 0000000000..70f36fb6b7
--- /dev/null
+++ b/erts/emulator/beam/erl_monitor_link.c
@@ -0,0 +1,1341 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2018. All Rights Reserved.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description:	Monitor and link implementation.
+ *
+ * Author: 	Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "sys.h"
+#include <stddef.h>
+#include "global.h"
+#include "erl_node_tables.h"
+#include "erl_monitor_link.h"
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Red-black tree implementation used for monitors and links.                *
+\*                                                                           */
+
+static ERTS_INLINE Eterm
+ml_get_key(ErtsMonLnkNode *mln)
+{
+    char *ptr = (char *) mln;
+    ptr += mln->key_offset;
+
+#ifdef ERTS_ML_DEBUG
+    switch (mln->type) {
+    case ERTS_MON_TYPE_PROC:
+    case ERTS_MON_TYPE_PORT:
+    case ERTS_MON_TYPE_DIST_PROC:
+    case ERTS_MON_TYPE_TIME_OFFSET:
+    case ERTS_MON_TYPE_RESOURCE: {
+        ErtsMonitorData *mdp = erts_monitor_to_data(mln);
+        ERTS_ML_ASSERT(&mdp->ref == (Eterm *) ptr);
+        break;
+    }
+    case ERTS_LNK_TYPE_PROC:
+    case ERTS_LNK_TYPE_DIST_PROC:
+    case ERTS_LNK_TYPE_PORT:
+    case ERTS_MON_TYPE_NODE:
+    case ERTS_MON_TYPE_NODES:
+    case ERTS_MON_TYPE_SUSPEND:
+        ERTS_ML_ASSERT(&mln->other.item == (Eterm *) ptr);
+        break;
+    default:
+        ERTS_INTERNAL_ERROR("Invalid type");
+        break;
+    }
+#endif
+
+    return *((Eterm *) ptr);
+}
+
+/*
+ * Comparison functions for valid *and only valid* keys. That
+ * is, if the set of valid keys is changed, this function needs
+ * to be updated...
+ *
+ * Note that this function does *not* order terms according to
+ * term order, and that the order may vary between emulator
+ * restarts...
+ */
+static int
+ml_cmp_keys(Eterm key1, Eterm key2)
+{
+    /*
+     * A monitor key is either an internal pid, a (nodename) atom,
+     * a small (monitor nodes bitmask), an ordinary internal ref,
+     * or an external ref.
+     *
+     * Order between monitors with keys of different types:
+     *  internal pid < atom < small < internal ref < external ref
+     *
+     * A link key is either a pid, an internal port
+     *
+     * Order between links with keys of different types:
+     *  internal pid < internal port < external pid
+     *
+     */
+    ERTS_ML_ASSERT(is_internal_pid(key1)
+                   || is_internal_port(key1)
+                   || is_atom(key1)
+                   || is_small(key1)
+                   || is_external_pid(key1)
+                   || is_internal_ordinary_ref(key1)
+                   || is_external_ref(key1));
+
+    ERTS_ML_ASSERT(is_internal_pid(key2)
+                   || is_internal_port(key2)
+                   || is_atom(key2)
+                   || is_small(key2)
+                   || is_external_pid(key2)
+                   || is_internal_ordinary_ref(key2)
+                   || is_external_ref(key2));
+
+    if (is_immed(key1)) {
+        int key1_tag, key2_tag;
+
+        ERTS_ML_ASSERT(is_internal_pid(key1)
+                       || is_internal_port(key1)
+                       || is_atom(key1)
+                       || is_small(key1));
+
+        if (key1 == key2)
+            return 0;
+
+        if (is_boxed(key2))
+            return -1;
+
+        ERTS_ML_ASSERT(is_internal_pid(key2)
+                       || is_internal_port(key2)
+                       || is_atom(key2)
+                       || is_small(key2));
+
+        key1_tag = (int) (key1 & _TAG_IMMED1_MASK);
+        key2_tag = (int) (key2 & _TAG_IMMED1_MASK);
+
+        if (key1_tag != key2_tag)
+            return key1_tag - key2_tag;
+
+        ASSERT((is_atom(key1) && is_atom(key2))
+               || (is_small(key1) && is_small(key2))
+               || (is_internal_pid(key1) && is_internal_pid(key2))
+               || (is_internal_port(key1) && is_internal_port(key2)));
+
+        return key1 < key2 ? -1 : 1;
+    }
+    else {
+        Eterm *w1, hdr1;
+
+        ERTS_ML_ASSERT(is_boxed(key1));
+
+        w1 = boxed_val(key1);
+        hdr1 = *w1;
+
+        if ((hdr1 & _TAG_HEADER_MASK) == _TAG_HEADER_REF) {
+            Eterm *w2;
+
+            if (!is_internal_ref(key2))
+                return is_immed(key2) ? 1 : -1;
+
+            w2 = internal_ref_val(key2);
+
+            ERTS_ML_ASSERT(w1[0] == ERTS_REF_THING_HEADER);
+            ERTS_ML_ASSERT(w2[0] == ERTS_REF_THING_HEADER);
+
+            return sys_memcmp((void *) &w1[1], (void *) &w2[1],
+                              (ERTS_REF_THING_SIZE - 1)*sizeof(Eterm));
+        }
+
+        ERTS_ML_ASSERT(is_external(key1));
+
+        if (is_not_external(key2))
+            return 1;
+        else {
+            Uint ndw1, ndw2;
+            ExternalThing *et1, *et2;
+            ErlNode *n1, *n2;
+
+            ERTS_ML_ASSERT((is_external_ref(key1) && is_external_ref(key2))
+                           || (is_external_pid(key1) && is_external_pid(key2)));
+
+            et1 = (ExternalThing *) w1;
+            et2 = (ExternalThing *) external_val(key2);
+
+            n1 = et1->node;
+            n2 = et2->node;
+
+            if (n1 != n2) {
+                if (n1->sysname != n2->sysname)
+                    return n1->sysname < n2->sysname ? -1 : 1;
+                ASSERT(n1->creation != n2->creation);
+                return n1->creation < n2->creation ? -1 : 1;
+            }
+
+            ndw1 = external_thing_data_words(et1);
+            ndw2 = external_thing_data_words(et1);
+            if (ndw1 != ndw2)
+                return ndw1 < ndw2 ? -1 : 1;
+
+            return sys_memcmp((void *) &et1->data.ui[0],
+                              (void *) &et2->data.ui[0],
+                              ndw1*sizeof(Eterm));
+        }
+    }
+}
+
+#define ERTS_ML_TPFLG_RED               (((UWord) 1) << 0)
+
+#define ERTS_ML_TPFLGS_MASK             (ERTS_ML_TPFLG_RED)
+
+#define ERTS_RBT_PREFIX mon_lnk
+#define ERTS_RBT_T ErtsMonLnkNode
+#define ERTS_RBT_KEY_T Eterm
+#define ERTS_RBT_FLAGS_T UWord
+#define ERTS_RBT_INIT_EMPTY_TNODE(T)					\
+    do {								\
+	(T)->node.tree.parent = (UWord) NULL;                           \
+	(T)->node.tree.right = NULL;                                    \
+	(T)->node.tree.left = NULL;                                     \
+    } while (0)
+#define ERTS_RBT_IS_RED(T)						\
+    (!!((T)->node.tree.parent & ERTS_ML_TPFLG_RED))
+#define ERTS_RBT_SET_RED(T)						\
+    ((T)->node.tree.parent |= ERTS_ML_TPFLG_RED)
+#define ERTS_RBT_IS_BLACK(T)						\
+    (!ERTS_RBT_IS_RED((T)))
+#define ERTS_RBT_SET_BLACK(T)						\
+    ((T)->node.tree.parent &= ~ERTS_ML_TPFLG_RED)
+#define ERTS_RBT_GET_FLAGS(T)						\
+    ((T)->node.tree.parent & ERTS_ML_TPFLGS_MASK)
+#define ERTS_RBT_SET_FLAGS(T, F)					\
+    do {								\
+	ERTS_ML_ASSERT((((UWord) (F)) & ~ERTS_ML_TPFLGS_MASK) == 0);	\
+	(T)->node.tree.parent &= ~ERTS_ML_TPFLGS_MASK;                 \
+	(T)->node.tree.parent |= (F);                                   \
+    } while (0)
+#define ERTS_RBT_GET_PARENT(T)						\
+    ((ERTS_RBT_T *) ((T)->node.tree.parent & ~ERTS_ML_TPFLGS_MASK))
+#define ERTS_RBT_SET_PARENT(T, P)					\
+    do {								\
+	ERTS_ML_ASSERT((((UWord) (P)) & ERTS_ML_TPFLGS_MASK) == 0);	\
+	(T)->node.tree.parent &= ERTS_ML_TPFLGS_MASK;                  \
+	(T)->node.tree.parent |= (UWord) (P);                           \
+    } while (0)
+#define ERTS_RBT_GET_RIGHT(T) ((T)->node.tree.right)
+#define ERTS_RBT_SET_RIGHT(T, R) ((T)->node.tree.right = (R))
+#define ERTS_RBT_GET_LEFT(T) ((T)->node.tree.left)
+#define ERTS_RBT_SET_LEFT(T, L) ((T)->node.tree.left = (L))
+#define ERTS_RBT_GET_KEY(T) (ml_get_key((T)))
+#define ERTS_RBT_CMP_KEYS(KX, KY) (ml_cmp_keys((KX), (KY)))
+#define ERTS_RBT_WANT_DELETE
+#define ERTS_RBT_WANT_LOOKUP
+#define ERTS_RBT_WANT_LOOKUP_INSERT
+#define ERTS_RBT_WANT_LOOKUP_CREATE
+#define ERTS_RBT_WANT_INSERT
+#define ERTS_RBT_WANT_REPLACE
+#define ERTS_RBT_WANT_FOREACH
+#define ERTS_RBT_WANT_FOREACH_YIELDING
+#define ERTS_RBT_WANT_FOREACH_DESTROY
+#define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING
+#define ERTS_RBT_UNDEF
+
+#include "erl_rbtree.h"
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Tree Operations                                                           *
+\*                                                                           */
+
+static ErtsMonLnkNode *
+ml_rbt_lookup(ErtsMonLnkNode *root, Eterm ref)
+{
+    ErtsMonLnkNode *ml = mon_lnk_rbt_lookup(root, ref);
+    ASSERT(!ml || (ml->flags & ERTS_ML_FLG_IN_TABLE));
+    return ml;
+}
+
+static ErtsMonLnkNode *
+ml_rbt_lookup_insert(ErtsMonLnkNode **root, ErtsMonLnkNode *ml)
+{
+    ErtsMonLnkNode *res;
+    ERTS_ML_ASSERT(!(ml->flags & ERTS_ML_FLG_IN_TABLE));
+    res = mon_lnk_rbt_lookup_insert(root, ml);
+    if (!res)
+        ml->flags |= ERTS_ML_FLG_IN_TABLE;
+    return res;
+}
+
+static ErtsMonLnkNode *
+ml_rbt_lookup_create(ErtsMonLnkNode ** root, Eterm key,
+                     ErtsMonLnkNode *(*create)(Eterm, void *),
+                     void *carg, int *created)
+{
+    ErtsMonLnkNode *ml;
+    ml = mon_lnk_rbt_lookup_create(root, key, create, carg, created);
+    if (*created)
+        ml->flags |= ERTS_ML_FLG_IN_TABLE;
+    return ml;
+}
+
+static void
+ml_rbt_insert(ErtsMonLnkNode **root, ErtsMonLnkNode *ml)
+{
+#ifdef ERTS_ML_DEBUG
+    ErtsMonLnkNode *res;
+    ERTS_ML_ASSERT(!(ml->flags & ERTS_ML_FLG_IN_TABLE));
+    res = ml_rbt_lookup(*root, ml_get_key(ml));
+    ERTS_ML_ASSERT(res == NULL);
+#endif
+
+    mon_lnk_rbt_insert(root, ml);
+    ml->flags |= ERTS_ML_FLG_IN_TABLE;
+}
+
+static void
+ml_rbt_replace(ErtsMonLnkNode **root, ErtsMonLnkNode *old, ErtsMonLnkNode *new)
+{
+    ERTS_ML_ASSERT(old->flags & ERTS_ML_FLG_IN_TABLE);
+    ERTS_ML_ASSERT(!(new->flags & ERTS_ML_FLG_IN_TABLE));
+
+    mon_lnk_rbt_replace(root, old, new);
+    old->flags &= ~ERTS_ML_FLG_IN_TABLE;
+    new->flags |= ERTS_ML_FLG_IN_TABLE;
+}
+
+static void
+ml_rbt_delete(ErtsMonLnkNode **root, ErtsMonLnkNode *ml)
+{
+    ERTS_ML_ASSERT(ml->flags & ERTS_ML_FLG_IN_TABLE);
+    mon_lnk_rbt_delete(root, ml);
+    ml->flags &= ~ERTS_ML_FLG_IN_TABLE;
+}
+
+
+static void
+ml_rbt_foreach(ErtsMonLnkNode *root,
+               void (*func)(ErtsMonLnkNode *, void *),
+               void *arg)
+{
+    mon_lnk_rbt_foreach(root, func, arg);
+}
+
+typedef struct {
+    ErtsMonLnkNode *root;
+    mon_lnk_rbt_yield_state_t rbt_ystate;
+} ErtsMonLnkYieldState;
+
+static int
+ml_rbt_foreach_yielding(ErtsMonLnkNode *root,
+                        void (*func)(ErtsMonLnkNode *, void *),
+                        void *arg,
+                        void **vyspp,
+                        Sint limit)
+{
+    int res;
+    ErtsMonLnkYieldState ys = {root, ERTS_RBT_YIELD_STAT_INITER};
+    ErtsMonLnkYieldState *ysp;
+
+    ysp = (ErtsMonLnkYieldState *) *vyspp;
+    if (!ysp)
+	ysp = &ys;
+    res = mon_lnk_rbt_foreach_yielding(ysp->root, func, arg,
+                                       &ysp->rbt_ystate, limit);
+    if (res == 0) {
+	if (ysp != &ys)
+	    erts_free(ERTS_ALC_T_ML_YIELD_STATE, ysp);
+	*vyspp = NULL;
+    }
+    else {
+
+	if (ysp == &ys) {
+	    ysp = erts_alloc(ERTS_ALC_T_ML_YIELD_STATE,
+			     sizeof(ErtsMonLnkYieldState));
+	    sys_memcpy((void *) ysp, (void *) &ys,
+		       sizeof(ErtsMonLnkYieldState));
+	}
+
+	*vyspp = (void *) ysp;
+    }
+
+    return res;
+}
+
+typedef struct {
+    void (*func)(ErtsMonLnkNode *, void *);
+    void *arg;
+} ErtsMonLnkForeachDeleteContext;
+
+static void
+rbt_wrap_foreach_delete(ErtsMonLnkNode *ml, void *vctxt)
+{
+    ErtsMonLnkForeachDeleteContext *ctxt = vctxt;
+    ERTS_ML_ASSERT(ml->flags & ERTS_ML_FLG_IN_TABLE);
+    ml->flags &= ~ERTS_ML_FLG_IN_TABLE;
+    ctxt->func(ml, ctxt->arg);
+}
+
+static void
+ml_rbt_foreach_delete(ErtsMonLnkNode **root,
+                      void (*func)(ErtsMonLnkNode *, void *),
+                      void *arg)
+{
+    ErtsMonLnkForeachDeleteContext ctxt;
+    ctxt.func = func;
+    ctxt.arg = arg;
+    mon_lnk_rbt_foreach_destroy(root,
+                                rbt_wrap_foreach_delete,
+                                (void *) &ctxt);
+}
+
+static int
+ml_rbt_foreach_delete_yielding(ErtsMonLnkNode **root,
+                               void (*func)(ErtsMonLnkNode *, void *),
+                               void *arg,
+                               void **vyspp,
+                               Sint limit)
+{
+    int res;
+    ErtsMonLnkYieldState ys = {*root, ERTS_RBT_YIELD_STAT_INITER};
+    ErtsMonLnkYieldState *ysp;
+    ErtsMonLnkForeachDeleteContext ctxt;
+    ctxt.func = func;
+    ctxt.arg = arg;
+
+    ysp = (ErtsMonLnkYieldState *) *vyspp;
+    if (!ysp) {
+        *root = NULL;
+	ysp = &ys;
+    }
+    res = mon_lnk_rbt_foreach_destroy_yielding(&ysp->root,
+                                               rbt_wrap_foreach_delete,
+                                               (void *) &ctxt,
+                                               &ysp->rbt_ystate,
+                                               limit);
+    if (res == 0) {
+	if (ysp != &ys)
+	    erts_free(ERTS_ALC_T_ML_YIELD_STATE, ysp);
+	*vyspp = NULL;
+    }
+    else {
+
+	if (ysp == &ys) {
+	    ysp = erts_alloc(ERTS_ALC_T_ML_YIELD_STATE,
+			     sizeof(ErtsMonLnkYieldState));
+	    sys_memcpy((void *) ysp, (void *) &ys,
+		       sizeof(ErtsMonLnkYieldState));
+	}
+
+	*vyspp = (void *) ysp;
+    }
+
+    return res;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * List Operations                                                           *
+\*                                                                           */
+
+static int
+ml_dl_list_foreach_yielding(ErtsMonLnkNode *list,
+                            void (*func)(ErtsMonLnkNode *, void *),
+                            void *arg,
+                            void **vyspp,
+                            Sint limit)
+{
+    Sint cnt = 0;
+    ErtsMonLnkNode *ml = (ErtsMonLnkNode *) *vyspp;
+
+    ERTS_ML_ASSERT(!ml || list);
+
+    if (!ml)
+        ml = list;
+
+    if (ml) {
+        do {
+            ERTS_ML_ASSERT(ml->flags & ERTS_ML_FLG_IN_TABLE);
+            func(ml, arg);
+            ml = ml->node.list.next;
+            cnt++;
+        } while (ml != list && cnt < limit);
+        if (ml != list) {
+            *vyspp = (void *) ml;
+            return 1; /* yield */
+        }
+    }
+
+    *vyspp = NULL;
+    return 0; /* done */
+}
+
+static int
+ml_dl_list_foreach_delete_yielding(ErtsMonLnkNode **list,
+                                   void (*func)(ErtsMonLnkNode *, void *),
+                                   void *arg,
+                                   void **vyspp,
+                                   Sint limit)
+{
+    Sint cnt = 0;
+    ErtsMonLnkNode *first = *list;
+    ErtsMonLnkNode *ml = (ErtsMonLnkNode *) *vyspp;
+
+    ERTS_ML_ASSERT(!ml || first);
+
+    if (!ml)
+        ml = first;
+
+    if (ml) {
+        do {
+            ErtsMonLnkNode *next = ml->node.list.next;
+            ERTS_ML_ASSERT(ml->flags & ERTS_ML_FLG_IN_TABLE);
+            ml->flags &= ~ERTS_ML_FLG_IN_TABLE;
+            func(ml, arg);
+            ml = next;
+            cnt++;
+        } while (ml != first && cnt < limit);
+        if (ml != first) {
+            *vyspp = (void *) ml;
+            return 1; /* yield */
+        }
+    }
+
+    *vyspp = NULL;
+    *list = NULL;
+    return 0; /* done */
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Misc                                                                      *
+\*                                                                           */
+
+ErtsMonLnkDist *
+erts_mon_link_dist_create(Eterm nodename)
+{
+    ErtsMonLnkDist *mld = erts_alloc(ERTS_ALC_T_ML_DIST,
+                                     sizeof(ErtsMonLnkDist));
+    mld->nodename = nodename;
+    mld->connection_id = ~((Uint32) 0);
+    erts_atomic_init_nob(&mld->refc, 1);
+    erts_mtx_init(&mld->mtx, "dist_entry_links", nodename,
+                  ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
+    mld->alive = !0;
+    mld->links = NULL;
+    mld->monitors = NULL;
+    mld->orig_name_monitors = NULL;
+    return mld;
+}
+
+void
+erts_mon_link_dist_destroy__(ErtsMonLnkDist *mld)
+{
+    ERTS_ML_ASSERT(erts_atomic_read_nob(&mld->refc) == 0);
+    ERTS_ML_ASSERT(!mld->alive);
+    ERTS_ML_ASSERT(!mld->links);
+    ERTS_ML_ASSERT(!mld->monitors);
+    ERTS_ML_ASSERT(!mld->orig_name_monitors);
+
+    erts_mtx_destroy(&mld->mtx);
+    erts_free(ERTS_ALC_T_ML_DIST, mld);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Monitor Operations                                                        *
+\*                                                                           */
+
+#ifdef ERTS_ML_DEBUG
+size_t erts_monitor_origin_offset;
+size_t erts_monitor_origin_key_offset;
+size_t erts_monitor_target_offset;
+size_t erts_monitor_target_key_offset;
+size_t erts_monitor_node_key_offset;
+#endif
+
+static ERTS_INLINE void
+monitor_init(void)
+{
+#ifdef ERTS_ML_DEBUG
+    erts_monitor_origin_offset = offsetof(ErtsMonitorData, origin);
+    erts_monitor_origin_key_offset = offsetof(ErtsMonitorData, ref);
+    ASSERT(erts_monitor_origin_key_offset >= erts_monitor_origin_offset);
+    erts_monitor_origin_key_offset -= erts_monitor_origin_offset;
+    erts_monitor_target_offset = offsetof(ErtsMonitorData, target);
+    erts_monitor_target_key_offset = offsetof(ErtsMonitorData, ref);
+    ASSERT(erts_monitor_target_key_offset >= erts_monitor_target_offset);
+    erts_monitor_target_key_offset -= erts_monitor_target_offset;
+    erts_monitor_node_key_offset = offsetof(ErtsMonitor, other.item);
+#endif
+}
+
+ErtsMonitor *
+erts_monitor_tree_lookup(ErtsMonitor *root, Eterm key)
+{
+    ERTS_ML_ASSERT(is_internal_ordinary_ref(key)
+                   || is_external_ref(key)
+                   || is_atom(key)
+                   || is_small(key)
+                   || is_internal_pid(key));
+    return (ErtsMonitor *) ml_rbt_lookup((ErtsMonLnkNode *) root, key);
+}
+
+ErtsMonitor *
+erts_monotor_tree_lookup_insert(ErtsMonitor **root, ErtsMonitor *mon)
+{
+    return (ErtsMonitor *) ml_rbt_lookup_insert((ErtsMonLnkNode **) root,
+                                                (ErtsMonLnkNode *) mon);
+}
+
+typedef struct {
+    Uint16 type;
+    Eterm origin;
+} ErtsMonitorCreateCtxt;
+
+static ErtsMonLnkNode *
+create_monitor(Eterm target, void *vcctxt)
+{
+    ErtsMonitorCreateCtxt *cctxt = vcctxt;
+    ErtsMonitorData *mdp = erts_monitor_create(cctxt->type,
+                                               NIL,
+                                               cctxt->origin,
+                                               target,
+                                               NIL);
+    ERTS_ML_ASSERT(ml_cmp_keys(ml_get_key(&mdp->origin), target) == 0);
+    return (ErtsMonLnkNode *) &mdp->origin;
+}
+
+ErtsMonitor *
+erts_monitor_tree_lookup_create(ErtsMonitor **root, int *created, Uint16 type,
+                                Eterm origin, Eterm target)
+{
+    ErtsMonitor *res;
+    ErtsMonitorCreateCtxt cctxt = {type, origin};
+
+    ERTS_ML_ASSERT(type == ERTS_MON_TYPE_NODE || type == ERTS_MON_TYPE_NODES);
+
+    res = (ErtsMonitor *) ml_rbt_lookup_create((ErtsMonLnkNode **) root,
+                                               target, create_monitor,
+                                               (void *) &cctxt,
+                                               created);
+
+    ERTS_ML_ASSERT(res && erts_monitor_is_origin(res));
+
+    return res;
+}
+
+void
+erts_monitor_tree_insert(ErtsMonitor **root, ErtsMonitor *mon)
+{
+    ml_rbt_insert((ErtsMonLnkNode **) root, (ErtsMonLnkNode *) mon);
+}
+
+void
+erts_monitor_tree_replace(ErtsMonitor **root, ErtsMonitor *old, ErtsMonitor *new)
+{
+    ml_rbt_replace((ErtsMonLnkNode **) root,
+                   (ErtsMonLnkNode *) old,
+                   (ErtsMonLnkNode *) new);
+}
+
+void
+erts_monitor_tree_delete(ErtsMonitor **root, ErtsMonitor *mon)
+{
+    ml_rbt_delete((ErtsMonLnkNode **) root, (ErtsMonLnkNode *) mon);
+}
+
+void
+erts_monitor_tree_foreach(ErtsMonitor *root,
+                          void (*func)(ErtsMonitor *, void *),
+                          void *arg)
+{
+    ml_rbt_foreach((ErtsMonLnkNode *) root,
+                   (void (*)(ErtsMonLnkNode*, void*)) func,
+                   arg);
+}
+
+int
+erts_monitor_tree_foreach_yielding(ErtsMonitor *root,
+                                   void (*func)(ErtsMonitor *, void *),
+                                   void *arg,
+                                   void **vyspp,
+                                   Sint limit)
+{
+    return ml_rbt_foreach_yielding((ErtsMonLnkNode *) root,
+                                   (void (*)(ErtsMonLnkNode*, void*)) func,
+                                   arg, vyspp, limit);
+}
+
+void
+erts_monitor_tree_foreach_delete(ErtsMonitor **root,
+                                 void (*func)(ErtsMonitor *, void *),
+                                 void *arg)
+{
+    ml_rbt_foreach_delete((ErtsMonLnkNode **) root,
+                          (void (*)(ErtsMonLnkNode*, void*)) func,
+                          arg);
+}
+
+int
+erts_monitor_tree_foreach_delete_yielding(ErtsMonitor **root,
+                                          void (*func)(ErtsMonitor *, void *),
+                                          void *arg,
+                                          void **vyspp,
+                                          Sint limit)
+{
+    return ml_rbt_foreach_delete_yielding((ErtsMonLnkNode **) root,
+                                          (void (*)(ErtsMonLnkNode*, void*)) func,
+                                          arg, vyspp, limit);
+}
+
+void
+erts_monitor_list_foreach(ErtsMonitor *list,
+                          void (*func)(ErtsMonitor *, void *),
+                          void *arg)
+{
+    void *ystate = NULL;
+    while (ml_dl_list_foreach_yielding((ErtsMonLnkNode *) list,
+                                       (void (*)(ErtsMonLnkNode *, void *)) func,
+                                       arg, &ystate, (Sint) INT_MAX));
+}
+
+int
+erts_monitor_list_foreach_yielding(ErtsMonitor *list,
+                                   void (*func)(ErtsMonitor *, void *),
+                                   void *arg,
+                                   void **vyspp,
+                                   Sint limit)
+{
+    return ml_dl_list_foreach_yielding((ErtsMonLnkNode *) list,
+                                       (void (*)(ErtsMonLnkNode *, void *)) func,
+                                       arg, vyspp, limit);
+}
+
+void
+erts_monitor_list_foreach_delete(ErtsMonitor **list,
+                                 void (*func)(ErtsMonitor *, void *),
+                                 void *arg)
+{
+    void *ystate = NULL;
+    while (ml_dl_list_foreach_delete_yielding((ErtsMonLnkNode **) list,
+                                              (void (*)(ErtsMonLnkNode*, void*)) func,
+                                              arg, &ystate, (Sint) INT_MAX));
+}
+
+int
+erts_monitor_list_foreach_delete_yielding(ErtsMonitor **list,
+                                          void (*func)(ErtsMonitor *, void *),
+                                          void *arg,
+                                          void **vyspp,
+                                          Sint limit)
+{
+    return ml_dl_list_foreach_delete_yielding((ErtsMonLnkNode **) list,
+                                              (void (*)(ErtsMonLnkNode*, void*)) func,
+                                              arg, vyspp, limit);
+}
+
+ErtsMonitorData *
+erts_monitor_create(Uint16 type, Eterm ref, Eterm orgn, Eterm trgt, Eterm name)
+{
+    ErtsMonitorData *mdp;
+
+    switch (type) {
+    case ERTS_MON_TYPE_PROC:
+    case ERTS_MON_TYPE_PORT:
+    case ERTS_MON_TYPE_TIME_OFFSET:
+        if (is_nil(name)) {
+            ErtsMonitorDataHeap *mdhp;
+            ErtsORefThing *ortp;
+
+            ERTS_ML_ASSERT(is_immed(orgn) && is_immed(trgt));
+            ERTS_ML_ASSERT(is_internal_ordinary_ref(ref));
+        
+            mdhp = erts_alloc(ERTS_ALC_T_MONITOR, sizeof(ErtsMonitorDataHeap));
+            mdp = &mdhp->md;
+            ERTS_ML_ASSERT(((void *) mdp) == ((void *) mdhp));
+
+            ortp = (ErtsORefThing *) (char *) internal_ref_val(ref);
+            mdhp->oref_thing = *ortp;
+            mdp->ref = make_internal_ref(&mdhp->oref_thing.header);
+
+            mdp->origin.other.item = trgt;
+            mdp->origin.offset = (Uint16) offsetof(ErtsMonitorData, origin);
+            mdp->origin.key_offset = (Uint16) offsetof(ErtsMonitorData, ref);
+            ERTS_ML_ASSERT(mdp->origin.key_offset >= mdp->origin.offset);
+            mdp->origin.key_offset -= mdp->origin.offset;
+            mdp->origin.flags = (Uint16) 0;
+            mdp->origin.type = type;
+
+            mdp->target.other.item = orgn;
+            mdp->target.offset = (Uint16) offsetof(ErtsMonitorData, target);
+            mdp->target.key_offset = (Uint16) offsetof(ErtsMonitorData, ref);
+            ERTS_ML_ASSERT(mdp->target.key_offset >= mdp->target.offset);
+            mdp->target.key_offset -= mdp->target.offset;
+            mdp->target.flags = ERTS_ML_FLG_TARGET;
+            mdp->target.type = type;
+            break;
+        }
+    case ERTS_MON_TYPE_DIST_PROC:
+    case ERTS_MON_TYPE_RESOURCE:
+    case ERTS_MON_TYPE_NODE:
+    case ERTS_MON_TYPE_NODES: {
+        ErtsMonitorDataExtended *mdep;
+        Uint size = sizeof(ErtsMonitorDataExtended) - sizeof(Eterm);
+        Uint rsz, osz, tsz;
+        Eterm *hp;
+        ErlOffHeap oh;
+        Uint16 name_flag = is_nil(name) ? ((Uint16) 0) : ERTS_ML_FLG_NAME;
+
+        rsz = is_immed(ref) ? 0 : size_object(ref);
+        tsz = is_immed(trgt) ? 0 : size_object(trgt);
+        if (type == ERTS_MON_TYPE_RESOURCE)
+            osz = 0;
+        else
+            osz = is_immed(orgn) ? 0 : size_object(orgn);
+
+        size += (rsz + osz + tsz) * sizeof(Eterm);
+
+        mdep = erts_alloc(ERTS_ALC_T_MONITOR_EXT, size);
+
+        ERTS_INIT_OFF_HEAP(&oh);
+
+        hp = &mdep->heap[0];
+
+        mdp = &mdep->md;
+        ERTS_ML_ASSERT(((void *) mdp) == ((void *) mdep));
+
+        mdp->ref = rsz ? copy_struct(ref, rsz, &hp, &oh) : ref;
+
+        mdp->origin.other.item = tsz ? copy_struct(trgt, tsz, &hp, &oh) : trgt;
+        mdp->origin.offset = (Uint16) offsetof(ErtsMonitorData, origin);
+        mdp->origin.flags = ERTS_ML_FLG_EXTENDED|name_flag;
+        mdp->origin.type = type;
+
+        if (type == ERTS_MON_TYPE_RESOURCE)
+            mdp->target.other.ptr = (void *) orgn;
+        else
+            mdp->target.other.item = osz ? copy_struct(orgn, osz, &hp, &oh) : orgn;
+        mdp->target.offset = (Uint16) offsetof(ErtsMonitorData, target);
+        mdp->target.flags = ERTS_ML_FLG_TARGET|ERTS_ML_FLG_EXTENDED|name_flag;
+        mdp->target.type = type;
+
+        if (type == ERTS_MON_TYPE_NODE || type == ERTS_MON_TYPE_NODES) {
+            mdep->u.refc = 0;
+            mdp->origin.key_offset = (Uint16) offsetof(ErtsMonitor, other.item);
+            mdp->target.key_offset = (Uint16) offsetof(ErtsMonitor, other.item);
+            ERTS_ML_ASSERT(!oh.first);
+            mdep->uptr.node_monitors = NULL;
+        }
+        else {
+            mdep->u.name = name;
+
+            mdp->origin.key_offset = (Uint16) offsetof(ErtsMonitorData, ref);
+            ERTS_ML_ASSERT(mdp->origin.key_offset >= mdp->origin.offset);
+            mdp->origin.key_offset -= mdp->origin.offset;
+
+            mdp->target.key_offset = (Uint16) offsetof(ErtsMonitorData, ref);
+            ERTS_ML_ASSERT(mdp->target.key_offset >= mdp->target.offset);
+            mdp->target.key_offset -= mdp->target.offset;
+
+            mdep->uptr.ohhp = oh.first;
+        }
+        mdep->dist = NULL;
+        break;
+    }
+    case ERTS_MON_TYPE_SUSPEND:
+        ERTS_INTERNAL_ERROR("Use erts_monitor_suspend_create() instead...");
+        mdp = NULL;
+        break;
+    default:
+        ERTS_INTERNAL_ERROR("Invalid monitor type");
+        mdp = NULL;
+        break;
+    }
+
+    erts_atomic32_init_nob(&mdp->refc, 2);
+
+    return mdp;
+}
+
+/*
+ * erts_monitor_destroy__() should only be called from
+ * erts_monitor_release() or erts_monitor_release_both().
+ */
+void
+erts_monitor_destroy__(ErtsMonitorData *mdp)
+{
+    ERTS_ML_ASSERT(erts_atomic32_read_nob(&mdp->refc) == 0);
+    ERTS_ML_ASSERT(!(mdp->origin.flags & ERTS_ML_FLG_IN_TABLE));
+    ERTS_ML_ASSERT(!(mdp->target.flags & ERTS_ML_FLG_IN_TABLE));
+    ERTS_ML_ASSERT((mdp->origin.flags & ERTS_ML_FLGS_SAME)
+                   == (mdp->target.flags & ERTS_ML_FLGS_SAME));
+    ERTS_ML_ASSERT(mdp->origin.type != ERTS_MON_TYPE_SUSPEND);
+
+    if (!(mdp->origin.flags & ERTS_ML_FLG_EXTENDED))
+        erts_free(ERTS_ALC_T_MONITOR, mdp);
+    else {
+        ErtsMonitorDataExtended *mdep = (ErtsMonitorDataExtended *) mdp;
+        ErlOffHeap oh;
+        if (mdp->origin.type == ERTS_MON_TYPE_NODE)
+            ERTS_ML_ASSERT(!mdep->uptr.node_monitors);
+        else if (mdep->uptr.ohhp) {
+            ERTS_INIT_OFF_HEAP(&oh);
+            oh.first = mdep->uptr.ohhp;
+            erts_cleanup_offheap(&oh);
+        }
+        if (mdep->dist)
+            erts_mon_link_dist_dec_refc(mdep->dist);
+        erts_free(ERTS_ALC_T_MONITOR_EXT, mdp);
+    }
+}
+
+void
+erts_monitor_set_dead_dist(ErtsMonitor *mon, Eterm nodename)
+{
+    ErtsMonitorDataExtended *mdep;
+    mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon);
+
+    ERTS_ML_ASSERT(mon->flags & ERTS_ML_FLG_EXTENDED);
+    ERTS_ML_ASSERT(mon->type == ERTS_MON_TYPE_DIST_PROC);
+    ERTS_ML_ASSERT(!mdep->dist);
+
+    mdep->dist = erts_mon_link_dist_create(nodename);
+    mdep->dist->alive = 0;
+}
+
+Uint
+erts_monitor_size(ErtsMonitor *mon)
+{
+    Uint size, refc;
+    ErtsMonitorData *mdp = erts_monitor_to_data(mon);
+
+    ERTS_ML_ASSERT(mon->type != ERTS_MON_TYPE_SUSPEND);
+
+    if (!(mon->flags & ERTS_ML_FLG_EXTENDED))
+        size = sizeof(ErtsMonitorDataHeap);
+    else {
+        ErtsMonitorDataExtended *mdep;
+        Uint hsz = 0;
+
+        mdep = (ErtsMonitorDataExtended *) mdp;
+
+        if (mon->type != ERTS_MON_TYPE_NODE
+            && mon->type != ERTS_MON_TYPE_NODES) {
+            if (!is_immed(mdep->md.ref))
+                hsz += NC_HEAP_SIZE(mdep->md.ref);
+            if (mon->type == ERTS_MON_TYPE_DIST_PROC) {
+                if (!is_immed(mdep->md.origin.other.item))
+                    hsz += NC_HEAP_SIZE(mdep->md.origin.other.item);
+                if (!is_immed(mdep->md.target.other.item))
+                    hsz += NC_HEAP_SIZE(mdep->md.target.other.item);
+            }
+        }
+        size = sizeof(ErtsMonitorDataExtended) + (hsz - 1)*sizeof(Eterm);
+    }
+    
+    refc = (Uint) erts_atomic32_read_nob(&mdp->refc);
+    ASSERT(refc > 0);
+
+    return size / refc;
+}
+
+
+/* suspend monitors... */
+
+ErtsMonitorSuspend *
+erts_monitor_suspend_create(Eterm pid)
+{
+    ErtsMonitorSuspend *msp;
+
+    ERTS_ML_ASSERT(is_internal_pid(pid));
+
+    msp = erts_alloc(ERTS_ALC_T_SUSPEND_MON,
+                     sizeof(ErtsMonitorSuspend));
+    msp->mon.offset = (Uint16) offsetof(ErtsMonitorSuspend, mon);
+    msp->mon.key_offset = (Uint16) offsetof(ErtsMonitor, other.item);
+    msp->mon.other.item = pid;
+    msp->mon.flags = 0;
+    msp->mon.type = ERTS_MON_TYPE_SUSPEND;
+    msp->pending = 0;
+    msp->active = 0;
+    return msp;
+}
+
+static ErtsMonLnkNode *
+create_monitor_suspend(Eterm pid, void *unused)
+{
+    ErtsMonitorSuspend *msp = erts_monitor_suspend_create(pid);
+    return (ErtsMonLnkNode *) &msp->mon;
+}
+
+ErtsMonitorSuspend *
+erts_monitor_suspend_tree_lookup_create(ErtsMonitor **root, int *created,
+                                        Eterm pid)
+{
+    ErtsMonitor *mon;
+    mon = (ErtsMonitor *) ml_rbt_lookup_create((ErtsMonLnkNode **) root,
+                                               pid, create_monitor_suspend,
+                                               NULL,
+                                               created);
+    return erts_monitor_suspend(mon);
+}
+
+void
+erts_monitor_suspend_destroy(ErtsMonitorSuspend *msp)
+{
+    ERTS_ML_ASSERT(!(msp->mon.flags & ERTS_ML_FLG_IN_TABLE));
+    erts_free(ERTS_ALC_T_SUSPEND_MON, msp);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Link Operations                                                           *
+ *                                                                           *
+\*                                                                           */
+
+#ifdef ERTS_ML_DEBUG
+size_t erts_link_a_offset;
+size_t erts_link_b_offset;
+size_t erts_link_key_offset;
+#endif
+
+static ERTS_INLINE void
+link_init(void)
+{
+#ifdef ERTS_ML_DEBUG
+    erts_link_a_offset = offsetof(ErtsLinkData, a);
+    erts_link_b_offset = offsetof(ErtsLinkData, b);
+    erts_link_key_offset = offsetof(ErtsLink, other.item);
+#endif
+}
+
+ErtsLink *
+erts_link_tree_lookup(ErtsLink *root, Eterm key)
+{
+    ASSERT(is_pid(key) || is_internal_port(key));
+    return (ErtsLink *) ml_rbt_lookup((ErtsMonLnkNode *) root, key);
+}
+
+ErtsLink *
+erts_link_tree_lookup_insert(ErtsLink **root, ErtsLink *lnk)
+{
+    return (ErtsLink *) ml_rbt_lookup_insert((ErtsMonLnkNode **) root,
+                                             (ErtsMonLnkNode *) lnk);
+}
+
+typedef struct {
+    Uint16 type;
+    Eterm a;
+} ErtsLinkCreateCtxt;
+
+static ErtsMonLnkNode *
+create_link(Eterm b, void *vcctxt)
+{
+    ErtsLinkCreateCtxt *cctxt = vcctxt;
+    ErtsLinkData *ldp = erts_link_create(cctxt->type, cctxt->a, b);
+    ERTS_ML_ASSERT(ml_cmp_keys(ldp->a.other.item, b) == 0);
+    return (ErtsMonLnkNode *) &ldp->a;
+}
+
+ErtsLink *erts_link_tree_lookup_create(ErtsLink **root, int *created,
+                                       Uint16 type, Eterm this,
+                                       Eterm other)
+{
+    ErtsLinkCreateCtxt cctxt;
+    cctxt.type = type;
+    cctxt.a = this;
+    return (ErtsLink *) ml_rbt_lookup_create((ErtsMonLnkNode **) root,
+                                             other, create_link,
+                                             (void *) &cctxt,
+                                             created);
+}
+
+void
+erts_link_tree_insert(ErtsLink **root, ErtsLink *lnk)
+{
+    ml_rbt_insert((ErtsMonLnkNode **) root, (ErtsMonLnkNode *) lnk);
+}
+
+void
+erts_link_tree_replace(ErtsLink **root, ErtsLink *old, ErtsLink *new)
+{
+    ml_rbt_replace((ErtsMonLnkNode **) root,
+                   (ErtsMonLnkNode *) old,
+                   (ErtsMonLnkNode *) new);
+}
+
+void
+erts_link_tree_delete(ErtsLink **root, ErtsLink *lnk)
+{
+    ml_rbt_delete((ErtsMonLnkNode **) root, (ErtsMonLnkNode *) lnk);
+}
+
+void
+erts_link_tree_foreach(ErtsLink *root,
+                       void (*func)(ErtsLink *, void *),
+                       void *arg)
+{
+    ml_rbt_foreach((ErtsMonLnkNode *) root,
+                   (void (*)(ErtsMonLnkNode*, void*)) func,
+                   arg);
+
+}
+
+int
+erts_link_tree_foreach_yielding(ErtsLink *root,
+                                void (*func)(ErtsLink *, void *),
+                                void *arg,
+                                void **vyspp,
+                                Sint limit)
+{
+    return ml_rbt_foreach_yielding((ErtsMonLnkNode *) root,
+                                   (void (*)(ErtsMonLnkNode*, void*)) func,
+                                   arg, vyspp, limit);
+}
+
+void
+erts_link_tree_foreach_delete(ErtsLink **root,
+                              void (*func)(ErtsLink *, void *),
+                              void *arg)
+{
+    ml_rbt_foreach_delete((ErtsMonLnkNode **) root,
+                          (void (*)(ErtsMonLnkNode*, void*)) func,
+                          arg);
+}
+
+int
+erts_link_tree_foreach_delete_yielding(ErtsLink **root,
+                                       void (*func)(ErtsLink *, void *),
+                                       void *arg,
+                                       void **vyspp,
+                                       Sint limit)
+{
+    return ml_rbt_foreach_delete_yielding((ErtsMonLnkNode **) root,
+                                          (void (*)(ErtsMonLnkNode*, void*)) func,
+                                          arg, vyspp, limit);
+}
+
+void
+erts_link_list_foreach(ErtsLink *list,
+                       void (*func)(ErtsLink *, void *),
+                       void *arg)
+{
+    void *ystate = NULL;
+    while (ml_dl_list_foreach_yielding((ErtsMonLnkNode *) list,
+                                       (void (*)(ErtsMonLnkNode *, void *)) func,
+                                       arg, &ystate, (Sint) INT_MAX));
+}
+
+int
+erts_link_list_foreach_yielding(ErtsLink *list,
+                                void (*func)(ErtsLink *, void *),
+                                void *arg,
+                                void **vyspp,
+                                Sint limit)
+{
+    return ml_dl_list_foreach_yielding((ErtsMonLnkNode *) list,
+                                       (void (*)(ErtsMonLnkNode *, void *)) func,
+                                       arg, vyspp, limit);
+}
+
+void
+erts_link_list_foreach_delete(ErtsLink **list,
+                               void (*func)(ErtsLink *, void *),
+                               void *arg)
+{
+    void *ystate = NULL;
+    while (ml_dl_list_foreach_delete_yielding((ErtsMonLnkNode **) list,
+                                              (void (*)(ErtsMonLnkNode*, void*)) func,
+                                              arg, &ystate, (Sint) INT_MAX));
+}
+
+int
+erts_link_list_foreach_delete_yielding(ErtsLink **list,
+                                       void (*func)(ErtsLink *, void *),
+                                       void *arg,
+                                       void **vyspp,
+                                       Sint limit)
+{
+    return ml_dl_list_foreach_delete_yielding((ErtsMonLnkNode **) list,
+                                              (void (*)(ErtsMonLnkNode*, void*)) func,
+                                              arg, vyspp, limit);
+}
+
+ErtsLinkData *
+erts_link_create(Uint16 type, Eterm a, Eterm b)
+{
+    ErtsLinkData *ldp;
+
+#ifdef ERTS_ML_DEBUG
+    switch (type) {
+    case ERTS_LNK_TYPE_PROC:
+        ERTS_ML_ASSERT(is_internal_pid(a) && is_internal_pid(a));
+        break;
+    case ERTS_LNK_TYPE_PORT:
+        ERTS_ML_ASSERT(is_internal_pid(a) || is_internal_pid(b));
+        ERTS_ML_ASSERT(is_internal_port(a) || is_internal_port(b));
+        break;
+    case ERTS_LNK_TYPE_DIST_PROC:
+        ERTS_ML_ASSERT(is_internal_pid(a) || is_internal_pid(b));
+        ERTS_ML_ASSERT(is_external_pid(a) || is_external_pid(b));
+        break;
+    default:
+        ERTS_INTERNAL_ERROR("Invalid link type");
+        break;
+    }
+#endif
+
+    if (type != ERTS_LNK_TYPE_DIST_PROC) {
+        ldp = erts_alloc(ERTS_ALC_T_LINK, sizeof(ErtsLinkData));
+
+        ldp->a.other.item = b;
+        ldp->a.flags = (Uint16) 0;
+
+        ldp->b.other.item = a;
+        ldp->b.flags = (Uint16) 0;
+    }
+    else {
+        ErtsLinkDataExtended *ldep;
+        Uint size, hsz;
+        Eterm *hp;
+        ErlOffHeap oh;
+
+        if (is_internal_pid(a))
+            hsz = NC_HEAP_SIZE(b);
+        else 
+            hsz = NC_HEAP_SIZE(a);
+        ERTS_ML_ASSERT(hsz > 0);
+
+        size = sizeof(ErtsLinkDataExtended) - sizeof(Eterm);
+        size += hsz*sizeof(Eterm);
+
+        ldp = erts_alloc(ERTS_ALC_T_LINK_EXT, size);
+
+        ldp->a.flags = ERTS_ML_FLG_EXTENDED;
+        ldp->b.flags = ERTS_ML_FLG_EXTENDED;
+
+        ldep = (ErtsLinkDataExtended *) ldp;
+        hp = &ldep->heap[0];
+
+        ERTS_INIT_OFF_HEAP(&oh);
+
+        if (is_internal_pid(a)) {
+            ldp->a.other.item = STORE_NC(&hp, &oh, b);
+            ldp->b.other.item = a;
+        }
+        else {
+            ldp->a.other.item = b;
+            ldp->b.other.item = STORE_NC(&hp, &oh, a);
+        }
+
+        ldep->ohhp = oh.first;
+        ldep->dist = NULL;
+    }
+
+    erts_atomic32_init_nob(&ldp->refc, 2);
+
+    ldp->a.key_offset = (Uint16) offsetof(ErtsLink, other.item);
+    ldp->a.offset = (Uint16) offsetof(ErtsLinkData, a);
+    ldp->a.type = type;
+
+    ldp->b.key_offset = (Uint16) offsetof(ErtsLink, other.item);
+    ldp->b.offset = (Uint16) offsetof(ErtsLinkData, b);
+    ldp->b.type = type;
+
+    return ldp;
+}
+
+/*
+ * erts_link_destroy__() should only be called from
+ * erts_link_release() or erts_link_release_both().
+ */
+void
+erts_link_destroy__(ErtsLinkData *ldp)
+{
+    ERTS_ML_ASSERT(erts_atomic32_read_nob(&ldp->refc) == 0);
+    ERTS_ML_ASSERT(!(ldp->a.flags & ERTS_ML_FLG_IN_TABLE));
+    ERTS_ML_ASSERT(!(ldp->b.flags & ERTS_ML_FLG_IN_TABLE));
+    ERTS_ML_ASSERT((ldp->a.flags & ERTS_ML_FLGS_SAME)
+                   == (ldp->b.flags & ERTS_ML_FLGS_SAME));
+
+    if (!(ldp->a.flags & ERTS_ML_FLG_EXTENDED))
+        erts_free(ERTS_ALC_T_LINK, ldp);
+    else {
+        ErtsLinkDataExtended *ldep = (ErtsLinkDataExtended *) ldp;
+        ErlOffHeap oh;
+        if (ldep->ohhp) {
+            ERTS_INIT_OFF_HEAP(&oh);
+            oh.first = ldep->ohhp;
+            erts_cleanup_offheap(&oh);
+        }
+        if (ldep->dist)
+            erts_mon_link_dist_dec_refc(ldep->dist);
+        erts_free(ERTS_ALC_T_LINK_EXT, ldep);
+    }
+}
+
+void
+erts_link_set_dead_dist(ErtsLink *lnk, Eterm nodename)
+{
+    ErtsLinkDataExtended *ldep;
+    ldep = (ErtsLinkDataExtended *) erts_link_to_data(lnk);
+
+    ERTS_ML_ASSERT(lnk->flags & ERTS_ML_FLG_EXTENDED);
+    ERTS_ML_ASSERT(lnk->type == ERTS_LNK_TYPE_DIST_PROC);
+    ERTS_ML_ASSERT(!ldep->dist);
+
+    ldep->dist = erts_mon_link_dist_create(nodename);
+    ldep->dist->alive = 0;
+}
+
+Uint
+erts_link_size(ErtsLink *lnk)
+{
+    Uint size, refc;
+    ErtsLinkData *ldp = erts_link_to_data(lnk);
+
+    if (!(lnk->flags & ERTS_ML_FLG_EXTENDED))
+        size = sizeof(ErtsLinkData);
+    else {
+        ErtsLinkDataExtended *ldep = (ErtsLinkDataExtended *) ldp;
+
+        ASSERT(lnk->type == ERTS_LNK_TYPE_DIST_PROC);
+        ASSERT(is_external_pid_header(ldep->heap[0]));
+
+        size = sizeof(ErtsLinkDataExtended);
+        size += thing_arityval(ldep->heap[0])*sizeof(Eterm);
+    }
+    
+    refc = (Uint) erts_atomic32_read_nob(&ldp->refc);
+    ASSERT(refc > 0);
+
+    return size / refc;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Misc                                                                      *
+\*                                                                           */
+
+void
+erts_monitor_link_init(void)
+{
+    monitor_init();
+    link_init();
+}
diff --git a/erts/emulator/beam/erl_monitor_link.h b/erts/emulator/beam/erl_monitor_link.h
new file mode 100644
index 0000000000..603aead8cc
--- /dev/null
+++ b/erts/emulator/beam/erl_monitor_link.h
@@ -0,0 +1,2326 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2018. All Rights Reserved.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description:	Monitor and link implementation.
+ *
+ *              === Monitors ==================================================
+ *
+ *              The monitor data structure contains:
+ *              - an 'origin' part that should be inserted in a data structure
+ *                of the origin entity, i.e., the entity monitoring another
+ *                entity
+ *              - a 'target' part that should be inserted in a data structure
+ *                of the target entity, i.e., the entity being monitored by
+ *                another entity
+ *              - a shared part that contains information shared between both
+ *                origin and target entities
+ *
+ *              That is, the two halves of the monitor as well as shared data
+ *              are allocated in one single continuous memory block. The
+ *              origin and target parts can separately each be inserted in
+ *              either a (red-black) tree, a (circular double linked) list, or
+ *              in a process signal queue.
+ *
+ *              Each process and port contains:
+ *              - a monitor list for local target monitors that is accessed
+ *                via the ERTS_P_LT_MONITORS() macro, and
+ *              - a monitor tree for other monitors that is accessed via the
+ *                ERTS_P_MONITORS() macro
+ *
+ *              These fields of processes/ports are protected by the main lock
+ *              of the process/port. These are only intended to be accessed by
+ *              the process/port itself. When setting up or tearing down a
+ *              monitor one should *only* operate on the monitor tree/list of
+ *              the currently executing process/port and send signals to the
+ *              other involved process/port so it can modify its own monitor
+ *              tree/list by itself (see erl_proc_sig_queue.h). One should
+ *              absolutely *not* acquire the lock of the other involved
+ *              process/port and operate on its monitor tree/list directly.
+ *
+ *              Each dist entry contains a monitor/link dist structure that
+ *              contains:
+ *              - a monitor tree for origin named monitors that is accessed via
+ *                the field 'orig_name_monitors', and
+ *              - a monitor list for other monitors that is accessed via the
+ *                'monitors' field.
+ *              Monitors in these fields contain information about all monitors
+ *              over this specific connection.
+ *
+ *              The fields of the dist structure are protected by a mutex in
+ *              the same dist structure. Operations on these fields are
+ *              normally performed by the locally involved process only,
+ *              except when a connection is taken down. However in the case
+ *              of distributed named monitors that originates from another
+ *              node this is not possible. That is this operation is also
+ *              performed from another context that the locally involved
+ *              process.
+ *
+ *              Access to monitor trees are performed using the
+ *              erts_monitor_tree_* functions below. Access to monitor lists
+ *              are performed using the erts_monitor_list_* functions below.
+ *
+ *
+ *              The different monitor types:
+ *
+ *              --- ERTS_MON_TYPE_PROC ----------------------------------------
+ *
+ *              A local process (origin) monitors another local process
+ *              (target).
+ *
+ *              Origin:
+ *                      Other Item:     Target process identifier
+ *              Target:
+ *                      Other Item:     Origin process identifier
+ *              Shared:
+ *                      Key:            Reference
+ *                      Name:           Name (atom) if by name
+ *
+ *              Valid keys are only ordinary internal references.
+ *
+ *              Origin part of the monitor is stored in the monitor tree of
+ *              origin process and target part of the monitor is stored in
+ *              monitor list for local targets on the target process.
+ *
+ *              --- ERTS_MON_TYPE_PORT ----------------------------------------
+ *
+ *              A local process (origin) monitors a local port (target), or a
+ *              local port (origin) monitors a local process (target).
+ *
+ *              Origin:
+ *                      Other Item:     Target process/port identifier
+ *              Target:
+ *                      Other Item:     Origin process/port identifier
+ *              Shared:
+ *                      Key:            Reference
+ *                      Name:           Name (atom) if by name
+ *
+ *              Valid keys are only ordinary internal references.
+ *
+ *              Origin part of the monitor is stored in the monitor tree of
+ *              origin process/port and target part of the monitor is stored
+ *              in monitor list for local targets on the target process/port.
+ *
+ *
+ *              --- ERTS_MON_TYPE_TIME_OFFSET ---------------------------------
+ *
+ *              A local process (origin) monitors time offset (target)
+ *
+ *              Origin:
+ *                      Other Item:     clock_service
+ *              Target:
+ *                      Other Item:     Origin process identifier
+ *              Shared:
+ *                      Key:            Reference
+ *
+ *              Valid keys are only ordinary internal references.
+ *
+ *              Origin part of the monitor is stored in the monitor tree of
+ *              origin process and target part of the monitor is stored in
+ *              monitor list referred by the variable 'time_offset_monitors'
+ *              (see erl_time_sup.c).
+ *
+ *
+ *              --- ERTS_MON_TYPE_DIST_PROC -----------------------------------
+ *
+ *              A local process (origin) monitors a remote process (target).
+ *              Origin node on local process and target node on dist entry.
+ *
+ *              Origin:
+ *                      Other Item:     Remote process identifier/Node name
+ *                                      if by name
+ *              Target:
+ *                      Other Item:     Local process identifier
+ *              Shared:
+ *                      Key:            Reference
+ *                      Name:           Name (atom) if by name
+ *                      Dist:           Pointer to dist structure
+ *
+ *              Valid keys are only ordinary internal references.
+ *
+ *              Origin part of the monitor is stored in the monitor tree of
+ *              origin process and target part of the monitor is stored in
+ *              monitor list referred by 'monitors' field of the dist
+ *              structure.
+ *
+ *
+ *              A remote process (origin) monitors a local process (target).
+ *              Origin node on dist entry and target node on local process.
+ *
+ *              Origin:
+ *                      Other Item:     Local process identifier
+ *              Target:
+ *                      Other Item:     Remote process identifier
+ *              Shared:
+ *                      Key:            Reference
+ *                      Name:           Name (atom) if by name
+ *
+ *              Valid keys are only external references.
+ *
+ *              If monitor by name, the origin part of the monitor is stored
+ *              in the monitor tree referred by 'orig_name_monitors' field in
+ *              dist structure; otherwise in the monitor list referred by
+ *              'monitors' field in dist structure. The target part of the
+ *              monitor is stored in the monitor tree of the local target
+ *              process.
+ *
+ *
+ *              --- ERTS_MON_TYPE_RESOURCE ------------------------------------
+ *
+ *              A NIF resource (origin) monitors a process (target).
+ *
+ *              Origin:
+ *                      Other Item:     Target process identifier
+ *              Target:
+ *                      Other Ptr:      Pointer to resource
+ *              Shared:
+ *                      Key:            Reference
+ *
+ *              Valid keys are only ordinary internal references.
+ *
+ *              Origin part of the monitor is stored in the monitor tree of
+ *              origin resource (see erl_nif.c) and target part of the
+ *              monitor is stored in monitor list for local targets on the
+ *              target process.
+ *
+ *              --- ERTS_MON_TYPE_NODE ----------------------------------------
+ *
+ *              A local process (origin) monitors a distribution connection
+ *              (target) via erlang:monitor_node().
+ *
+ *              Origin:
+ *                      Other Item:     Node name (atom)
+ *                      Key:            Node name
+ *              Target:
+ *                      Other Item:     Origin process identifier
+ *                      Key:            Origin process identifier
+ *              Shared:
+ *                      Refc:           Number of invocations
+ *
+ *              Valid keys are only node-name atoms and internal process
+ *              identifiers.
+ *
+ *              Origin part of the monitor is stored in the monitor tree of
+ *              origin process and target part of the monitor is stored in
+ *              monitor list referred by 'monitors' field of the dist
+ *              structure.
+ *
+ *              --- ERTS_MON_TYPE_NODES ---------------------------------------
+ *
+ *              A local process (origin) monitors all connections (target),
+ *              via net_kernel:monitor_nodes().
+ *
+ *              Origin:
+ *                      Other Item:     Bit mask (small)
+ *                      Key:            Bit mask
+ *              Target:
+ *                      Other Item:     Origin process identifier
+ *                      Key:            Origin process identifier
+ *              Shared:
+ *                      Refc:           Number of invocations
+ *
+ *              Valid keys are only small integers and internal process
+ *              identifiers.
+ *
+ *              Origin part of the monitor is stored in the monitor tree of
+ *              origin process and target part of the monitor is stored in
+ *              monitor list referred by the variable 'nodes_monitors' (see
+ *              dist.c).
+ *
+ *              --- ERTS_MON_TYPE_SUSPEND -------------------------------------
+ *
+ *              Suspend monitor.
+ *
+ *              Other Item:         Suspendee process identifier
+ *              Key:                Suspendee process identifier
+ *
+ *              Valid keys are only ordinary internal references.
+ *
+ *              This type of monitor is a bit strange and the whole process
+ *              suspend functionality should be improved...
+ *
+ *
+ *
+ *              === Links =====================================================
+ *
+ *              The link data structure contains:
+ *              - an 'a' part that should be inserted in a data structure of
+ *                one entity and contains the identifier of the other involved
+ *                entity (b)
+ *              - a 'b' part that should be inserted in a data structure of
+ *                the other involved entity and contains the identifier of the
+ *                other involved entity (a)
+ *              - shared part that contains information shared between both
+ *                involved entities
+ *
+ *              That is, the two halves of the link as well as shared data
+ *              are allocated in one single continuous memory block. The 'a'
+ *              and the 'b' parts can separately each be inserted in either
+ *              a (red-black) tree, a (circular double linked) list, or in a
+ *              process signal queue.
+ *
+ *              Each process and port contains:
+ *              - a link tree for links that is accessed via the
+ *                ERTS_P_LINKS() macro
+ *
+ *              This field of processes/ports is protected by the main lock of
+ *              the process/port. It is only intended to be accessed by the
+ *              process/port itself. When setting up or tearing down a link
+ *              one should *only* operate on the link tree of the currently
+ *              executing process/port and send signals to the other involved
+ *              process/port so it can modify its own monitor tree by itself
+ *              (see erl_proc_sig_queue.h). One should absolutely *not*
+ *              acquire the lock of the other involved process/port and
+ *              operate on its link tree directly.
+ *
+ *              Each dist entry contains a monitor/link dist structure that
+ *              contains:
+ *              - a link list for links via the 'links' field.
+ *              Links in this field contain information about all links over
+ *              this specific connection.
+ *
+ *              The fields of the dist structure are protected by a mutex in
+ *              the same dist structure. Operation on the 'links' fields are
+ *              normally performed by the locally involved process only,
+ *              except when a connection is taken down.
+ *
+ *              Access to link trees are performed using the erts_link_tree_*
+ *              functions below. Access to link lists are performed using the
+ *              erts_link_list_* functions below.
+ *
+ *              There can only be one link between the same pair of
+ *              processes/ports. Since a link can be simultaneously initiated
+ *              from both ends we always save the link data structure with the
+ *              lowest address if multiple links should appear between the
+ *              same pair of processes/ports.
+ *
+ *
+ *              The different link types:
+ *
+ *              --- ERTS_LNK_TYPE_PROC -----------------------------------------
+ *
+ *              A link between a local process A and a local process B.
+ *
+ *              A:
+ *                      Other Item:     B process identifier
+ *                      Key:            B process identifier
+ *              B:
+ *                      Other Item:     A process identifier
+ *                      Key:            A process identifier
+ *
+ *              Valid keys are only internal process identifiers.
+ *
+ *              'A' part of the link stored in the link tree of process A and
+ *              'B' part of the link is stored in link tree of process B.
+ *
+ *              --- ERTS_LNK_TYPE_PORT -----------------------------------------
+ *
+ *              A link between a local process/port A and a local process/port
+ *              B.
+ *
+ *              A:
+ *                      Other Item:     B process/port identifier
+ *                      Key:            B process/port identifier
+ *              B:
+ *                      Other Item:     A process/port identifier
+ *                      Key:            A process/port identifier
+ *
+ *              Valid keys are internal process identifiers and internal port
+ *              identifiers.
+ *
+ *              'A' part of the link stored in the link tree of process/port
+ *              A and 'B' part of the link is stored in link tree of
+ *              process/port B.
+ *
+ *              --- ERTS_LNK_TYPE_DIST_PROC ------------------------------------
+ *
+ *              A link between a local process and a remote process. Either of
+ *              the processes can be used as A or B.
+ *
+ *              A:
+ *                      Other Item:     B process identifier
+ *                      Key:            B process identifier
+ *              B:
+ *                      Other Item:     A process identifier
+ *                      Key:            A process identifier
+ *              Shared:
+ *                      Dist:           Pointer to dist structure
+ *
+ *              Valid keys are internal and external process identifiers.
+ *
+ *              The part of the link with a remote pid as "other item" is
+ *              stored in the link tree of the local process. The part of
+ *              the link with a local pid as "other item" is stored in the
+ *              links list of the dist structure.
+ *
+ *              ===============================================================
+ *
+ * Author: 	Rickard Green
+ *
+ */
+
+#ifndef ERL_MONITOR_LINK_H__
+#define ERL_MONITOR_LINK_H__
+
+#define ERTS_PROC_SIG_QUEUE_TYPE_ONLY
+#include "erl_proc_sig_queue.h"
+#undef ERTS_PROC_SIG_QUEUE_TYPE_ONLY
+
+#if defined(DEBUG) || 0
+#  define ERTS_ML_DEBUG
+#else
+#  undef ERTS_ML_DEBUG
+#endif
+
+#ifdef ERTS_ML_DEBUG
+#  define ERTS_ML_ASSERT ERTS_ASSERT
+#else
+#  define ERTS_ML_ASSERT(E) ((void) 1)
+#endif
+
+#define ERTS_MON_TYPE_MAX               ((Uint16) 7)
+
+#define ERTS_MON_TYPE_PROC              ((Uint16) 0)
+#define ERTS_MON_TYPE_PORT              ((Uint16) 1)
+#define ERTS_MON_TYPE_TIME_OFFSET       ((Uint16) 2)
+#define ERTS_MON_TYPE_DIST_PROC         ((Uint16) 3)
+#define ERTS_MON_TYPE_RESOURCE          ((Uint16) 4)
+#define ERTS_MON_TYPE_NODE              ((Uint16) 5)
+#define ERTS_MON_TYPE_NODES             ((Uint16) 6)
+#define ERTS_MON_TYPE_SUSPEND           ERTS_MON_TYPE_MAX
+
+#define ERTS_MON_LNK_TYPE_MAX           (ERTS_MON_TYPE_MAX + ((Uint16) 3))
+#define ERTS_LNK_TYPE_MAX               ERTS_MON_LNK_TYPE_MAX
+
+#define ERTS_LNK_TYPE_PROC              (ERTS_MON_TYPE_MAX + ((Uint16) 1))
+#define ERTS_LNK_TYPE_PORT              (ERTS_MON_TYPE_MAX + ((Uint16) 2))
+#define ERTS_LNK_TYPE_DIST_PROC         ERTS_LNK_TYPE_MAX
+
+#define ERTS_ML_FLG_TARGET              (((Uint16) 1) << 0)
+#define ERTS_ML_FLG_IN_TABLE            (((Uint16) 1) << 1)
+#define ERTS_ML_FLG_IN_SUBTABLE         (((Uint16) 1) << 2)
+#define ERTS_ML_FLG_NAME                (((Uint16) 1) << 3)
+#define ERTS_ML_FLG_EXTENDED            (((Uint16) 1) << 4)
+
+#define ERTS_ML_FLG_DBG_VISITED         (((Uint16) 1) << 15)
+
+/* Flags that should be the same on both monitor/link halves */
+#define ERTS_ML_FLGS_SAME \
+    (ERTS_ML_FLG_EXTENDED|ERTS_ML_FLG_NAME)
+
+typedef struct ErtsMonLnkNode__ ErtsMonLnkNode;
+
+typedef struct {
+    UWord parent; /* Parent ptr and flags... */
+    ErtsMonLnkNode *right;
+    ErtsMonLnkNode *left;
+} ErtsMonLnkTreeNode;
+
+typedef struct {
+    ErtsMonLnkNode *next;
+    ErtsMonLnkNode *prev;
+} ErtsMonLnkListNode;
+
+struct ErtsMonLnkNode__ {
+    union {
+        ErtsSignalCommon signal;
+        ErtsMonLnkTreeNode tree;
+        ErtsMonLnkListNode list;
+    } node;
+    union {
+        Eterm item;
+        void *ptr;
+    } other;
+    Uint16 offset; /* offset from monitor/link data to this structure (node) */
+    Uint16 key_offset; /* offset from this structure (node) to key */
+    Uint16 flags;
+    Uint16 type;
+};
+
+typedef struct {
+    Eterm nodename;
+    Uint32 connection_id;
+    erts_atomic_t refc;
+    erts_mtx_t mtx;
+    int alive;
+    ErtsMonLnkNode *links; /* Link double linked circular list */
+    ErtsMonLnkNode *monitors; /* Monitor double linked circular list */
+    ErtsMonLnkNode *orig_name_monitors; /* Origin named monitors
+                                           read-black tree */
+} ErtsMonLnkDist;
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Misc                                                                      *
+\*                                                                           */
+
+/**
+ *
+ * @brief Initialize monitor/link implementation
+ *
+ */
+void erts_monitor_link_init(void);
+
+/**
+ *
+ * @brief Create monitor/link dist structure to attach to dist entry
+ *
+ * Create dist structure containing monitor and link containers. This
+ * structure is to be attached to a connected dist entry.
+ *
+ * @param[in]     nodename      Node name as an atom
+ *
+ * @returns                     Pointer to dist structure
+ *
+ */
+ErtsMonLnkDist *erts_mon_link_dist_create(Eterm nodename);
+
+/**
+ *
+ * @brief Increase reference count of monitor/link dist structure
+ *
+ * @param[in]     mld           Pointer to dist structure
+ *
+ */
+ERTS_GLB_INLINE void erts_mon_link_dist_inc_refc(ErtsMonLnkDist *mld);
+
+/**
+ *
+ * @brief Decrease reference count of monitor/link dist structure
+ *
+ * @param[in]     mld           Pointer to dist structure
+ *
+ */
+ERTS_GLB_INLINE void erts_mon_link_dist_dec_refc(ErtsMonLnkDist *mld);
+
+/* internal functions... */
+ERTS_GLB_INLINE void erts_ml_dl_list_insert__(ErtsMonLnkNode **list,
+                                              ErtsMonLnkNode *ml);
+ERTS_GLB_INLINE void erts_ml_dl_list_delete__(ErtsMonLnkNode **list,
+                                              ErtsMonLnkNode *ml);
+ERTS_GLB_INLINE ErtsMonLnkNode *erts_ml_dl_list_first__(ErtsMonLnkNode *list);
+ERTS_GLB_INLINE ErtsMonLnkNode *erts_ml_dl_list_last__(ErtsMonLnkNode *list);
+void erts_mon_link_dist_destroy__(ErtsMonLnkDist *mld);
+ERTS_GLB_INLINE void *erts_ml_node_to_main_struct__(ErtsMonLnkNode *mln);
+
+/* implementations for globally inlined misc functions... */
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE void
+erts_mon_link_dist_inc_refc(ErtsMonLnkDist *mld)
+{
+    ERTS_ML_ASSERT(erts_atomic_read_nob(&mld->refc) > 0);
+    erts_atomic_inc_nob(&mld->refc);
+}
+
+ERTS_GLB_INLINE void
+erts_mon_link_dist_dec_refc(ErtsMonLnkDist *mld)
+{
+    ERTS_ML_ASSERT(erts_atomic_read_nob(&mld->refc) > 0);
+    if (erts_atomic_dec_read_nob(&mld->refc) == 0)
+        erts_mon_link_dist_destroy__(mld);
+}
+
+ERTS_GLB_INLINE void *
+erts_ml_node_to_main_struct__(ErtsMonLnkNode *mln)
+{
+    return (void *) (((char *) mln) - ((size_t) mln->offset));
+}
+
+ERTS_GLB_INLINE void
+erts_ml_dl_list_insert__(ErtsMonLnkNode **list, ErtsMonLnkNode *ml)
+{
+    ErtsMonLnkNode *first = *list;
+    ERTS_ML_ASSERT(!(ml->flags & ERTS_ML_FLG_IN_TABLE));
+    if (!first) {
+        ml->node.list.next = ml->node.list.prev = ml;
+        *list = ml;
+    }
+    else {
+        ERTS_ML_ASSERT(first->node.list.prev->node.list.next == first);
+        ERTS_ML_ASSERT(first->node.list.next->node.list.prev == first);
+        ml->node.list.next = first;
+        ml->node.list.prev = first->node.list.prev;
+        first->node.list.prev = ml;
+        ml->node.list.prev->node.list.next = ml;
+    }
+    ml->flags |= ERTS_ML_FLG_IN_TABLE;
+}
+
+ERTS_GLB_INLINE void
+erts_ml_dl_list_delete__(ErtsMonLnkNode **list, ErtsMonLnkNode *ml)
+{
+    ERTS_ML_ASSERT(ml->flags & ERTS_ML_FLG_IN_TABLE);
+    if (ml->node.list.next == ml) {
+        ERTS_ML_ASSERT(ml->node.list.prev == ml);
+        ERTS_ML_ASSERT(*list == ml);
+
+        *list = NULL;
+    }
+    else {
+        ERTS_ML_ASSERT(ml->node.list.prev->node.list.next == ml);
+        ERTS_ML_ASSERT(ml->node.list.prev != ml);
+        ERTS_ML_ASSERT(ml->node.list.next->node.list.prev == ml);
+        ERTS_ML_ASSERT(ml->node.list.next != ml);
+
+        if (*list == ml)
+            *list = ml->node.list.next;
+        ml->node.list.prev->node.list.next = ml->node.list.next;
+        ml->node.list.next->node.list.prev = ml->node.list.prev;
+    }
+    ml->flags &= ~ERTS_ML_FLG_IN_TABLE;
+}
+
+ERTS_GLB_INLINE ErtsMonLnkNode *
+erts_ml_dl_list_first__(ErtsMonLnkNode *list)
+{
+    return list;
+}
+
+ERTS_GLB_INLINE ErtsMonLnkNode *
+erts_ml_dl_list_last__(ErtsMonLnkNode *list)
+{
+    if (!list)
+        return NULL;
+    return list->node.list.prev;
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Monitor Operations                                                        *
+\*                                                                           */
+
+
+typedef struct ErtsMonLnkNode__ ErtsMonitor;
+
+typedef struct {
+    ErtsMonitor origin;
+    ErtsMonitor target;
+    Eterm ref;
+    erts_atomic32_t refc;
+} ErtsMonitorData;
+
+typedef struct {
+    ErtsMonitorData md;
+    ErtsORefThing oref_thing;
+} ErtsMonitorDataHeap;
+
+typedef struct ErtsMonitorDataExtended__ ErtsMonitorDataExtended;
+
+struct ErtsMonitorDataExtended__ {
+    ErtsMonitorData md;
+    union {
+        Eterm name;
+        Uint refc;
+    } u;
+    union {
+        struct erl_off_heap_header *ohhp;
+        ErtsMonitor *node_monitors;
+    } uptr;
+    ErtsMonLnkDist *dist;
+    Eterm heap[1]; /* heap start... */
+};
+
+typedef struct {
+    ErtsMonitor mon;
+    int pending;
+    int active;
+} ErtsMonitorSuspend;
+
+/* 
+ * --- Monitor tree operations ---
+ */
+
+/**
+ *
+ * @brief Lookup a monitor in a monitor tree
+ *
+ *
+ * @param[in]     root          Pointer to root of monitor tree
+ *
+ * @param[in]     key           Key of monitor to lookup
+ *
+ * @returns                     Pointer to a monitor with the
+ *                              key 'key', or NULL if no such
+ *                              monitor was found
+ *
+ */
+ErtsMonitor *erts_monitor_tree_lookup(ErtsMonitor *root, Eterm key);
+
+/**
+ *
+ * @brief Lookup or insert a monitor in a monitor tree
+ *
+ * When the funcion is called it is assumed that:
+ * - 'mon' monitor is not part of any tree or list
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in,out] root          Pointer to pointer to root of monitor tree
+ *
+ * @param[in]     mon           Monitor to insert if no monitor
+ *                              with the same key already exists
+ *
+ * @returns                     Pointer to a monitor with the
+ *                              key 'key'. If no monitor with the key
+ *                              'key' was found and 'mon' was inserted
+ *                              'mon' is returned.
+ *
+ */
+ErtsMonitor *erts_monotor_tree_lookup_insert(ErtsMonitor **root,
+                                             ErtsMonitor *mon);
+
+/**
+ *
+ * @brief Lookup or create a node or a nodes monitor in a monitor tree.
+ *
+ * Looks up an origin monitor with the key 'target' in the monitor tree.
+ * If it is not found, creates a monitor and returns a pointer to the
+ * origin monitor.
+ *
+ * When the funcion is called it is assumed that:
+ * - no target monitors with the key 'target' exists in the tree.
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in,out] root          Pointer to pointer to root of monitor tree
+ *
+ * @param[out]    created       Pointer to integer. The integer is set to
+ *                              a non-zero value if no monitor with key
+ *                              'target' was found, and a new monitor
+ *                              was created. If a monitor was found, it
+ *                              is set to zero.
+ *
+ * @param[in]     type          ERTS_MON_TYPE_NODE | ERTS_MON_TYPE_NODES
+ *
+ * @param[in]     origin        The key of the origin
+ *
+ * @param[in]     target        The key of the target
+ *
+ */
+ErtsMonitor *erts_monitor_tree_lookup_create(ErtsMonitor **root, int *created,
+                                             Uint16 type, Eterm origin,
+                                             Eterm target);
+
+/**
+ *
+ * @brief Insert a monitor in a monitor tree
+ *
+ * When the funcion is called it is assumed that:
+ * - no monitors with the same key that 'mon' exist in the tree
+ * - 'mon' is not part of any list of tree
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in,out] root          Pointer to pointer to root of monitor tree
+ *
+ * @param[in]     mon           Monitor to insert.
+ *
+ */
+void erts_monitor_tree_insert(ErtsMonitor **root, ErtsMonitor *mon);
+
+/**
+ *
+ * @brief Replace a monitor in a monitor tree
+ *
+ * When the funcion is called it is assumed that:
+ * - 'old' monitor and 'new' monitor have exactly the same key
+ * - 'old' monitor is part of the tree
+ * - 'new' monitor is not part of any tree or list
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in,out] root          Pointer to pointer to root of monitor tree
+ *
+ * @param[in]     old           Monitor to remove from the tree
+ *
+ * @param[in]     new           Monitor to insert into the tree
+ *
+ */
+void erts_monitor_tree_replace(ErtsMonitor **root, ErtsMonitor *old,
+                               ErtsMonitor *new);
+
+/**
+ *
+ * @brief Delete a monitor from a monitor tree
+ *
+ * When the funcion is called it is assumed that:
+ * - 'mon' monitor is part of the tree
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in,out] root          Pointer to pointer to root of monitor tree
+ *
+ * @param[in]     mon           Monitor to remove from the tree
+ *
+ */
+void erts_monitor_tree_delete(ErtsMonitor **root, ErtsMonitor *mon);
+
+/**
+ *
+ * @brief Call a function for each monitor in a monitor tree
+ *
+ * The funcion 'func' will be called with a pointer to a monitor
+ * as first argument and 'arg' as second argument for each monitor
+ * in the tree referred to by 'root'.
+ *
+ * @param[in]     root          Pointer to root of monitor tree
+ *
+ * @param[in]     func          Pointer to function to call
+ *
+ * @param[in]     arg           Argument to pass as second argument in
+ *                              calls to 'func'
+ *
+ */
+void erts_monitor_tree_foreach(ErtsMonitor *root,
+                               void (*func)(ErtsMonitor *, void *),
+                               void *arg);
+
+/**
+ *
+ * @brief Call a function for each monitor in a monitor tree. Yield
+ *        if lots of monitors exist.
+ *
+ * The funcion 'func' will be called with a pointer to a monitor
+ * as first argument and 'arg' as second argument for each monitor
+ * in the tree referred to by 'root'.
+ *
+ * It is assumed that:
+ * - *yspp equals NULL on first call
+ * - this function is repetedly called with *yspp set
+ *   as set when previous call returned until a non-zero
+ *   value is returned.
+ * - no modifications are made on the tree between first call
+ *   and the call that returns a non-zero value
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in]     root          Pointer to root of monitor tree
+ *
+ * @param[in]     func          Pointer to function to call
+ *
+ * @param[in]     arg           Argument to pass as second argument in
+ *                              calls to 'func'
+ *
+ * @param[in,out] vyspp         Pointer to a pointer to an internal state
+ *                              used by this function. At initial call
+ *                              *yspp should be NULL. When done *yspp
+ *                              will be NULL.
+ *
+ * @param[in]     limit         Maximum amount of monitors to process
+ *                              before yielding.
+ *
+ * @returns                     A non-zero value when all monitors has been
+ *                              processed, and zero when more work is needed.
+ *
+ */
+int erts_monitor_tree_foreach_yielding(ErtsMonitor *root,
+                                       void (*func)(ErtsMonitor *, void *),
+                                       void *arg,
+                                       void **vyspp,
+                                       Sint limit);
+
+/**
+ *
+ * @brief Delete all monitors from a monitor tree and call a function for
+ *        each monitor
+ *
+ * The funcion 'func' will be called with a pointer to a monitor
+ * as first argument and 'arg' as second argument for each monitor
+ * in the tree referred to by 'root'.
+ *
+ * @param[in,out] root          Pointer to pointer to root of monitor tree
+ *
+ * @param[in]     func          Pointer to function to call
+ *
+ * @param[in]     arg           Argument to pass as second argument in
+ *                              calls to 'func'
+ *
+ */
+void erts_monitor_tree_foreach_delete(ErtsMonitor **root,
+                                      void (*func)(ErtsMonitor *, void *),
+                                      void *arg);
+
+/**
+ *
+ * @brief Delete all monitors from a monitor tree and call a function for
+ *        each monitor
+ *
+ * The funcion 'func' will be called with a pointer to a monitor
+ * as first argument and 'arg' as second argument for each monitor
+ * in the tree referred to by 'root'.
+ *
+ * It is assumed that:
+ * - *yspp equals NULL on first call
+ * - this function is repetededly called with *yspp set
+ *   as set when previous call returned until a non-zero
+ *   value is returned.
+ * - no modifications are made on the tree between first call
+ *   and the call that returns a non-zero value
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in,out] root          Pointer to pointer to root of monitor tree
+ *
+ * @param[in]     func          Pointer to function to call
+ *
+ * @param[in]     arg           Argument to pass as second argument in
+ *                              calls to 'func'
+ *
+ * @param[in,out] vyspp         Pointer to a pointer to an internal state
+ *                              used by this function. At initial call
+ *                              *yspp should be NULL. When done *yspp
+ *                              will be NULL.
+ *
+ * @param[in]     limit         Maximum amount of monitors to process
+ *                              before yielding.
+ *
+ * @returns                     A non-zero value when all monitors has been
+ *                              processed, and zero when more work is needed.
+ *
+ */
+int erts_monitor_tree_foreach_delete_yielding(ErtsMonitor **root,
+                                              void (*func)(ErtsMonitor *, void *),
+                                              void *arg,
+                                              void **vyspp,
+                                              Sint limit);
+
+/*
+ * --- Monitor list operations --
+ */
+
+/**
+ *
+ * @brief Insert a monitor in a monitor list
+ *
+ * When the funcion is called it is assumed that:
+ * - 'mon' monitor is not part of any list or tree
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in,out] list          Pointer to pointer to monitor list
+ *
+ * @param[in]     mon           Monitor to insert
+ *
+ */
+ERTS_GLB_INLINE void erts_monitor_list_insert(ErtsMonitor **list, ErtsMonitor *mon);
+
+/**
+ *
+ * @brief Delete a monitor from a monitor list
+ *
+ * When the funcion is called it is assumed that:
+ * - 'mon' monitor is part of the list
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in,out] list          Pointer to pointer to monitor list
+ *
+ * @param[in]     mon           Monitor to remove from the list
+ *
+ */
+ERTS_GLB_INLINE void erts_monitor_list_delete(ErtsMonitor **list, ErtsMonitor *mon);
+
+/**
+ *
+ * @brief Get a pointer to first monitor in a monitor list
+ *
+ * The monitor will still remain in the list after the return
+ *
+ * @param[in] list              Pointer to monitor list
+ *
+ * @returns                     Pointer to first monitor in list if
+ *                              list is not empty. If list is empty
+ *                              NULL is returned.
+ *
+ */
+ERTS_GLB_INLINE ErtsMonitor *erts_monitor_list_first(ErtsMonitor *list);
+
+/**
+ *
+ * @brief Get a pointer to last monitor in a monitor list
+ *
+ * The monitor will still remain in the list after the return
+ *
+ * @param[in] list              Pointer to monitor list
+ *
+ * @returns                     Pointer to last monitor in list if
+ *                              list is not empty. If list is empty
+ *                              NULL is returned.
+ *
+ */
+ERTS_GLB_INLINE ErtsMonitor *erts_monitor_list_last(ErtsMonitor *list);
+
+/**
+ *
+ * @brief Call a function for each monitor in a monitor list
+ *
+ * The funcion 'func' will be called with a pointer to a monitor
+ * as first argument and 'arg' as second argument for each monitor
+ * in the tree referred to by 'list'.
+ *
+ * @param[in]     list          Pointer to root of monitor list
+ *
+ * @param[in]     func          Pointer to function to call
+ *
+ * @param[in]     arg           Argument to pass as second argument in
+ *                              calls to 'func'
+ *
+ */
+void erts_monitor_list_foreach(ErtsMonitor *list,
+                               void (*func)(ErtsMonitor *, void *),
+                               void *arg);
+
+/**
+ *
+ * @brief Call a function for each monitor in a monitor list. Yield
+ *        if lots of monitors exist.
+ *
+ * The funcion 'func' will be called with a pointer to a monitor
+ * as first argument and 'arg' as second argument for each monitor
+ * in the tree referred to by 'root'.
+ *
+ * It is assumed that:
+ * - *yspp equals NULL on first call
+ * - this function is repetedly called with *yspp set
+ *   as set when previous call returned until a non-zero
+ *   value is returned.
+ * - no modifications are made on the tree between first call
+ *   and the call that returns a non-zero value
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in]     list          Pointer to monitor list
+ *
+ * @param[in]     func          Pointer to function to call
+ *
+ * @param[in]     arg           Argument to pass as second argument in
+ *                              calls to 'func'
+ *
+ * @param[in,out] vyspp         Pointer to a pointer to an internal state
+ *                              used by this function. At initial call
+ *                              *yspp should be NULL. When done *yspp
+ *                              will be NULL.
+ *
+ * @param[in]     limit         Maximum amount of monitors to process
+ *                              before yielding.
+ *
+ * @returns                     A non-zero value when all monitors has been
+ *                              processed, and zero when more work is needed.
+ *
+ */
+int erts_monitor_list_foreach_yielding(ErtsMonitor *list,
+                                       void (*func)(ErtsMonitor *, void *),
+                                       void *arg,
+                                       void **vyspp,
+                                       Sint limit);
+
+/**
+ *
+ * @brief Delete all monitors from a monitor list and call a function for
+ *        each monitor
+ *
+ * The funcion 'func' will be called with a pointer to a monitor
+ * as first argument and 'arg' as second argument for each monitor
+ * in the tree referred to by 'root'.
+ *
+ * @param[in,out] list          Pointer to pointer to monitor list
+ *
+ * @param[in]     func          Pointer to function to call
+ *
+ * @param[in]     arg           Argument to pass as second argument in
+ *                              calls to 'func'
+ *
+ */
+void erts_monitor_list_foreach_delete(ErtsMonitor **list,
+                                      void (*func)(ErtsMonitor *, void *),
+                                      void *arg);
+
+/**
+ *
+ * @brief Delete all monitors from a monitor list and call a function for
+ *        each monitor
+ *
+ * The funcion 'func' will be called with a pointer to a monitor
+ * as first argument and 'arg' as second argument for each monitor
+ * in the tree referred to by 'root'.
+ *
+ * It is assumed that:
+ * - *yspp equals NULL on first call
+ * - this function is repetededly called with *yspp set
+ *   as set when previous call returned until a non-zero
+ *   value is returned.
+ * - no modifications are made on the tree between first
+ *   and the call that returns a non-zero value
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in,out] list          Pointer to pointer to monitor list
+ *
+ * @param[in]     func          Pointer to function to call
+ *
+ * @param[in]     arg           Argument to pass as second argument in
+ *                              calls to 'func'
+ *
+ * @param[in,out] vyspp         Pointer to a pointer to an internal state
+ *                              used by this function. At initial call
+ *                              *yspp should be NULL. When done *yspp
+ *                              will be NULL.
+ *
+ * @param[in]     limit         Maximum amount of monitors to process
+ *                              before yielding.
+ *
+ * @returns                     A non-zero value when all monitors has been
+ *                              processed, and zero when more work is needed.
+ *
+ */
+int erts_monitor_list_foreach_delete_yielding(ErtsMonitor **list,
+                                              void (*func)(ErtsMonitor *, void *),
+                                              void *arg,
+                                              void **vyspp,
+                                              Sint limit);
+
+/*
+ * --- Misc monitor operations ---
+ */
+
+/**
+ *
+ * @brief Create a monitor
+ *
+ * Can create all types of monitors exept for suspend monitors
+ *
+ * When the funcion is called it is assumed that:
+ * - 'ref' is an internal ordinary reference if type is ERTS_MON_TYPE_PROC,
+ *   ERTS_MON_TYPE_PORT, ERTS_MON_TYPE_TIME_OFFSET, or ERTS_MON_TYPE_RESOURCE
+ * - 'ref' is NIL if type is ERTS_MON_TYPE_NODE or ERTS_MON_TYPE_NODES
+ * - 'ref' is and ordinary internal reference or an external reference if
+ *   type is ERTS_MON_TYPE_DIST_PROC
+ * - 'name' is an atom or NIL if type is ERTS_MON_TYPE_PROC,
+ *   ERTS_MON_TYPE_PORT, or ERTS_MON_TYPE_DIST_PROC
+ * - 'name is NIL if type is ERTS_MON_TYPE_TIME_OFFSET, ERTS_MON_TYPE_RESOURCE,
+ *   ERTS_MON_TYPE_NODE, or ERTS_MON_TYPE_NODES
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in]     type          ERTS_MON_TYPE_PROC, ERTS_MON_TYPE_PORT,
+ *                              ERTS_MON_TYPE_TIME_OFFSET, ERTS_MON_TYPE_DIST_PROC,
+ *                              ERTS_MON_TYPE_RESOURCE, ERTS_MON_TYPE_NODE,
+ *                              or ERTS_MON_TYPE_NODES
+ *
+ * @param[in]     ref           A reference or NIL depending on type
+ *
+ * @param[in]     origin        The key of the origin
+ *
+ * @param[in]     target        The key of the target
+ *
+ */
+ErtsMonitorData *erts_monitor_create(Uint16 type, Eterm ref, Eterm origin,
+                                     Eterm target, Eterm name);
+
+/**
+ *
+ * @brief Get pointer to monitor data structure
+ *
+ * @param[in]    mon            Pointer to monitor
+ *
+ * @returns                     Pointer to monitor data structure
+ *
+ */
+ERTS_GLB_INLINE ErtsMonitorData *erts_monitor_to_data(ErtsMonitor *mon);
+
+/**
+ *
+ * @brief Check if monitor is a target monitor
+ *
+ * @param[in]    mon            Pointer to monitor to check
+ *
+ * @returns                     A non-zero value if target monitor;
+ *                              otherwise zero
+ *
+ */
+ERTS_GLB_INLINE int erts_monitor_is_target(ErtsMonitor *mon);
+
+/**
+ *
+ * @brief Check if monitor is an origin monitor
+ *
+ * @param[in]    mon            Pointer to monitor to check
+ *
+ * @returns                     A non-zero value if origin monitor;
+ *                              otherwise zero
+ *
+ */
+ERTS_GLB_INLINE int erts_monitor_is_origin(ErtsMonitor *mon);
+
+/**
+ *
+ * @brief Check if monitor is in tree or list
+ *
+ * @param[in]    mon            Pointer to monitor to check
+ *
+ * @returns                     A non-zero value if in tree or list;
+ *                              otherwise zero
+ *
+ */
+ERTS_GLB_INLINE int erts_monitor_is_in_table(ErtsMonitor *mon);
+
+/**
+ *
+ * @brief Release monitor
+ *
+ * When both the origin and the target part of the monitor have
+ * been released the monitor structure will be deallocated.
+ *
+ * When the funcion is called it is assumed that:
+ * - 'mon' monitor is not part of any list or tree
+ * - 'mon' is not referred to by any other structures
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in]    mon            Pointer to monitor
+ *
+ */
+ERTS_GLB_INLINE void erts_monitor_release(ErtsMonitor *mon);
+
+/**
+ *
+ * @brief Release both target and origin monitor structures simultaneously
+ *
+ * Release both the origin and target parts of the monitor
+ * simultaneously and deallocate the structure.
+ *
+ * When the funcion is called it is assumed that:
+ * - Neither the origin part nor the target part of the monitor
+ *   are not part of any list or tree
+ * - Neither the origin part nor the target part of the monitor
+ *   are referred to by any other structures
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in]    mdp            Pointer to monitor data structure
+ *
+ */
+ERTS_GLB_INLINE void erts_monitor_release_both(ErtsMonitorData *mdp);
+
+/**
+ *
+ * @brief Insert monitor in dist monitor tree or list
+ *
+ * When the funcion is called it is assumed that:
+ * - 'mon' monitor is not part of any list or tree
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in]    mon            Pointer to monitor
+ *
+ * @param[in]    dist           Pointer to dist structure
+ *
+ * @returns                     A non-zero value if inserted;
+ *                              otherwise, zero. The monitor
+ *                              is not inserted if the dist
+ *                              structure has been set in a
+ *                              dead state.
+ *
+ */
+ERTS_GLB_INLINE int erts_monitor_dist_insert(ErtsMonitor *mon, ErtsMonLnkDist *dist);
+
+/**
+ *
+ * @brief Delete monitor from dist monitor tree or list
+ *
+ * When the funcion is called it is assumed that:
+ * - 'mon' monitor earler has been inserted into 'dist'
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in]    mon            Pointer to monitor
+ *
+ * @param[in]    dist           Pointer to dist structure
+ *
+ * @returns                     A non-zero value if deleted;
+ *                              otherwise, zero. The monitor
+ *                              is not deleted if the dist
+ *                              structure has been set in a
+ *                              dead state or if it has already
+ *                              been deleted.
+ *
+ */
+ERTS_GLB_INLINE int erts_monitor_dist_delete(ErtsMonitor *mon);
+
+/**
+ *
+ * @brief Set dead dist structure on monitor
+ *
+ * @param[in]    mon            Pointer to monitor
+ *
+ * @param[in]    nodename       Name of remote node
+ *
+ */
+void
+erts_monitor_set_dead_dist(ErtsMonitor *mon, Eterm nodename);
+
+/**
+ *
+ * @brief Get charged size of monitor
+ *
+ * If the other side of the monitor has been released, the
+ * whole size of the monitor data structure is returned; otherwise,
+ * half of the size is returned.
+ *
+ * When the funcion is called it is assumed that:
+ * - 'mon' has not been released
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in]    mon            Pointer to monitor
+ *
+ * @returns                     Charged size in bytes
+ *
+ */
+Uint erts_monitor_size(ErtsMonitor *mon);
+
+
+/* internal function... */
+void erts_monitor_destroy__(ErtsMonitorData *mdp);
+
+/* implementations for globally inlined monitor functions... */
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE int
+erts_monitor_is_target(ErtsMonitor *mon)
+{
+    return !!(mon->flags & ERTS_ML_FLG_TARGET);
+}
+
+ERTS_GLB_INLINE int
+erts_monitor_is_origin(ErtsMonitor *mon)
+{
+    return !(mon->flags & ERTS_ML_FLG_TARGET);
+}
+
+ERTS_GLB_INLINE int
+erts_monitor_is_in_table(ErtsMonitor *mon)
+{
+    return !!(mon->flags & ERTS_ML_FLG_IN_TABLE);
+}
+
+ERTS_GLB_INLINE void
+erts_monitor_list_insert(ErtsMonitor **list, ErtsMonitor *mon)
+{
+    erts_ml_dl_list_insert__((ErtsMonLnkNode **) list, (ErtsMonLnkNode *) mon);
+}
+
+ERTS_GLB_INLINE void
+erts_monitor_list_delete(ErtsMonitor **list, ErtsMonitor *mon)
+{
+    erts_ml_dl_list_delete__((ErtsMonLnkNode **) list, (ErtsMonLnkNode *) mon);
+}
+
+ERTS_GLB_INLINE ErtsMonitor *
+erts_monitor_list_first(ErtsMonitor *list)
+{
+    return (ErtsMonitor *) erts_ml_dl_list_first__((ErtsMonLnkNode *) list);
+}
+
+ERTS_GLB_INLINE ErtsMonitor *
+erts_monitor_list_last(ErtsMonitor *list)
+{
+    return (ErtsMonitor *) erts_ml_dl_list_last__((ErtsMonLnkNode *) list);
+}
+
+#ifdef ERTS_ML_DEBUG
+extern size_t erts_monitor_origin_offset;
+extern size_t erts_monitor_origin_key_offset;
+extern size_t erts_monitor_target_offset;
+extern size_t erts_monitor_target_key_offset;
+extern size_t erts_monitor_node_key_offset;
+#endif
+
+ERTS_GLB_INLINE ErtsMonitorData *
+erts_monitor_to_data(ErtsMonitor *mon)
+{
+    ErtsMonitorData *mdp = erts_ml_node_to_main_struct__((ErtsMonLnkNode *) mon);
+
+#ifdef ERTS_ML_DEBUG
+    ERTS_ML_ASSERT(!(mdp->origin.flags & ERTS_ML_FLG_TARGET));
+    ERTS_ML_ASSERT(erts_monitor_origin_offset == (size_t) mdp->origin.offset);
+    ERTS_ML_ASSERT(!!(mdp->target.flags & ERTS_ML_FLG_TARGET));
+    ERTS_ML_ASSERT(erts_monitor_target_offset == (size_t) mdp->target.offset);
+    if (mon->type == ERTS_MON_TYPE_NODE || mon->type == ERTS_MON_TYPE_NODES) {
+        ERTS_ML_ASSERT(erts_monitor_node_key_offset == (size_t) mdp->origin.key_offset);
+        ERTS_ML_ASSERT(erts_monitor_node_key_offset == (size_t) mdp->target.key_offset);
+    }
+    else {
+        ERTS_ML_ASSERT(erts_monitor_origin_key_offset == (size_t) mdp->origin.key_offset);
+        ERTS_ML_ASSERT(erts_monitor_target_key_offset == (size_t) mdp->target.key_offset);
+    }
+#endif
+
+    return mdp;
+}
+
+ERTS_GLB_INLINE void
+erts_monitor_release(ErtsMonitor *mon)
+{
+    ErtsMonitorData *mdp = erts_monitor_to_data(mon);
+    ERTS_ML_ASSERT(!(mon->flags & ERTS_ML_FLG_IN_TABLE));
+    ERTS_ML_ASSERT(erts_atomic32_read_nob(&mdp->refc) > 0);
+
+    if (erts_atomic32_dec_read_nob(&mdp->refc) == 0)
+        erts_monitor_destroy__(mdp);
+}
+
+ERTS_GLB_INLINE void
+erts_monitor_release_both(ErtsMonitorData *mdp)
+{
+    ERTS_ML_ASSERT((mdp->origin.flags & ERTS_ML_FLGS_SAME)
+                   == (mdp->target.flags & ERTS_ML_FLGS_SAME));
+    ERTS_ML_ASSERT(!(mdp->origin.flags & ERTS_ML_FLG_IN_TABLE));
+    ERTS_ML_ASSERT(!(mdp->target.flags & ERTS_ML_FLG_IN_TABLE));
+    ERTS_ML_ASSERT(erts_atomic32_read_nob(&mdp->refc) >= 2);
+
+    if (erts_atomic32_add_read_nob(&mdp->refc, (erts_aint32_t) -2) == 0)
+        erts_monitor_destroy__(mdp);
+}
+
+ERTS_GLB_INLINE int
+erts_monitor_dist_insert(ErtsMonitor *mon, ErtsMonLnkDist *dist)
+{
+    ErtsMonitorDataExtended *mdep;
+    int insert;
+
+    ERTS_ML_ASSERT(mon->flags & ERTS_ML_FLG_EXTENDED);
+    ERTS_ML_ASSERT(mon->type == ERTS_MON_TYPE_DIST_PROC
+                   || mon->type == ERTS_MON_TYPE_NODE);
+
+    mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon);
+
+    ERTS_ML_ASSERT(!mdep->dist);
+    ERTS_ML_ASSERT(dist);
+    mdep->dist = dist;
+
+    erts_mon_link_dist_inc_refc(dist);
+
+    erts_mtx_lock(&dist->mtx);
+
+    insert = dist->alive;
+    if (insert) {
+        if ((mon->flags & (ERTS_ML_FLG_NAME
+                           | ERTS_ML_FLG_TARGET)) == ERTS_ML_FLG_NAME)
+            erts_monitor_tree_insert(&dist->orig_name_monitors, mon);
+        else
+            erts_monitor_list_insert(&dist->monitors, mon);
+    }
+
+    erts_mtx_unlock(&dist->mtx);
+
+    return insert;
+}
+
+ERTS_GLB_INLINE int
+erts_monitor_dist_delete(ErtsMonitor *mon)
+{
+    ErtsMonitorDataExtended *mdep;
+    ErtsMonLnkDist *dist;
+    Uint16 flags;
+    int delete;
+
+    ERTS_ML_ASSERT(mon->flags & ERTS_ML_FLG_EXTENDED);
+    ERTS_ML_ASSERT(mon->type == ERTS_MON_TYPE_DIST_PROC
+                   || mon->type == ERTS_MON_TYPE_NODE);
+
+    mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon);
+    dist = mdep->dist;
+    ERTS_ML_ASSERT(dist);
+
+    erts_mtx_lock(&dist->mtx);
+
+    flags = mon->flags;
+    delete = !!dist->alive & !!(flags & ERTS_ML_FLG_IN_TABLE);
+    if (delete) {
+        if ((flags & (ERTS_ML_FLG_NAME
+                      | ERTS_ML_FLG_TARGET)) == ERTS_ML_FLG_NAME)
+            erts_monitor_tree_delete(&dist->orig_name_monitors, mon);
+        else
+            erts_monitor_list_delete(&dist->monitors, mon);
+    }
+
+    erts_mtx_unlock(&dist->mtx);
+
+    return delete;
+}
+
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+/* suspend monitors... */
+ErtsMonitorSuspend *erts_monitor_suspend_create(Eterm pid);
+ErtsMonitorSuspend *erts_monitor_suspend_tree_lookup_create(ErtsMonitor **root,
+                                                            int *created,
+                                                            Eterm pid);
+void erts_monitor_suspend_destroy(ErtsMonitorSuspend *msp);
+
+ERTS_GLB_INLINE ErtsMonitorSuspend *erts_monitor_suspend(ErtsMonitor *mon);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE ErtsMonitorSuspend *erts_monitor_suspend(ErtsMonitor *mon)
+{
+    ERTS_ML_ASSERT(!mon || mon->type == ERTS_MON_TYPE_SUSPEND);
+    return (ErtsMonitorSuspend *) mon;
+}
+
+#endif
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Link Operations                                                           *
+\*                                                                           */
+
+typedef struct ErtsMonLnkNode__ ErtsLink;
+
+typedef struct {
+    ErtsLink a;
+    ErtsLink b;
+    erts_atomic32_t refc;
+} ErtsLinkData;
+
+typedef struct {
+    ErtsLinkData ld;
+    struct erl_off_heap_header *ohhp;
+    ErtsMonLnkDist *dist;
+    Eterm heap[1]; /* heap start... */
+} ErtsLinkDataExtended;
+
+/*
+ * --- Link tree operations ---
+ */
+
+/**
+ *
+ * @brief Lookup a link in a link tree
+ *
+ *
+ * @param[in]     root          Pointer to root of link tree
+ *
+ * @param[in]     key           Key of link to lookup
+ *
+ * @returns                     Pointer to a link with the
+ *                              key 'key', or NULL if no such
+ *                              link was found
+ *
+ */
+ErtsLink *erts_link_tree_lookup(ErtsLink *root, Eterm item);
+
+/**
+ *
+ * @brief Lookup or insert a link in a link tree
+ *
+ * When the funcion is called it is assumed that:
+ * - 'lnk' link is not part of any tree or list
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in,out] root          Pointer to pointer to root of link tree
+ *
+ * @param[in]     lnk           Link to insert if no link
+ *                              with the same key already exists
+ *
+ * @returns                     Pointer to a link with the
+ *                              key 'key'. If no link with the key
+ *                              'key' was found and 'lnk' was inserted
+ *                              'lnk' is returned.
+ *
+ */
+ErtsLink *erts_link_tree_lookup_insert(ErtsLink **root, ErtsLink *lnk);
+
+/**
+ *
+ * @brief Lookup or create a link in a link tree.
+ *
+ * Looks up a link with the key 'other' in the link tree. If it is not
+ * found, creates and insert a link with the key 'other'.
+ *
+ * @param[in,out] root          Pointer to pointer to root of link tree
+ *
+ * @param[out]    created       Pointer to integer. The integer is set to
+ *                              a non-zero value if no link with key
+ *                              'other' was found, and a new link
+ *                              was created. If a link was found, it
+ *                              is set to zero.
+ *
+ * @param[in]     type          Type of link
+ *
+ * @param[in]     this          Id of this entity
+ *
+ * @param[in]     other         Id of other entity
+ *
+ */
+ErtsLink *erts_link_tree_lookup_create(ErtsLink **root, int *created,
+                                       Uint16 type, Eterm this, Eterm other);
+
+/**
+ *
+ * @brief Insert a link in a link tree
+ *
+ * When the funcion is called it is assumed that:
+ * - no links with the same key that 'lnk' exist in the tree
+ * - 'lnk' is not part of any list of tree
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in,out] root          Pointer to pointer to root of link tree
+ *
+ * @param[in]     lnk           Link to insert.
+ *
+ */
+void erts_link_tree_insert(ErtsLink **root, ErtsLink *lnk);
+
+/**
+ *
+ * @brief Replace a link in a link tree
+ *
+ * When the funcion is called it is assumed that:
+ * - 'old' link and 'new' link have exactly the same key
+ * - 'old' link is part of the tree
+ * - 'new' link is not part of any tree or list
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in,out] root          Pointer to pointer to root of link tree
+ *
+ * @param[in]     old           Link to remove from the tree
+ *
+ * @param[in]     new           Link to insert into the tree
+ *
+ */
+void erts_link_tree_replace(ErtsLink **root, ErtsLink *old, ErtsLink *new);
+
+/**
+ *
+ * @brief Replace a link in a link tree if key already exist based on adress
+ *
+ * Inserts the link 'lnk' in the tree if no link with the same key
+ * already exists in tree. If a link with the same key exists in
+ * the tree and 'lnk' has a lower address than the link in the
+ * tree, the existing link in the tree is replaced by 'lnk'.
+ *
+ * When the funcion is called it is assumed that:
+ * - 'lnk' link is not part of any tree or list
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in,out] root          Pointer to pointer to root of link tree
+ *
+ * @param[in]     lnk           Link to insert into the tree
+ *
+ * @returns                     A pointer to the link that is not part
+ *                              of the tree after this operation.
+ *
+ */
+ERTS_GLB_INLINE ErtsLink *erts_link_tree_insert_addr_replace(ErtsLink **root,
+                                                             ErtsLink *lnk);
+
+/**
+ *
+ * @brief Delete a link from a link tree
+ *
+ * When the funcion is called it is assumed that:
+ * - 'lnk' link is part of the tree
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in,out] root          Pointer to pointer to root of link tree
+ *
+ * @param[in]     lnk           Link to remove from the tree
+ *
+ */
+void erts_link_tree_delete(ErtsLink **root, ErtsLink *lnk);
+
+/**
+ *
+ * @brief Delete a link from a link tree based on key
+ *
+ * If link 'lnk' is in the tree, 'lnk' is deleted from the tree.
+ * If link 'lnk' is not in the tree, another link with the same
+ * key as 'lnk' is deleted from the tree if such a link exist.
+ *
+ * When the funcion is called it is assumed that:
+ * - if 'lnk' link is part of a tree or list, it is part of this tree
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in,out] root          Pointer to pointer to root of link tree
+ *
+ * @param[in]     lnk           Link to remove from the tree
+ *
+ * @returns                     A pointer to the link that was deleted
+ *                              from the tree, or NULL in case no link
+ *                              was deleted from the tree
+ *
+ */
+ERTS_GLB_INLINE ErtsLink *erts_link_tree_key_delete(ErtsLink **root, ErtsLink *lnk);
+
+/**
+ *
+ * @brief Call a function for each link in a link tree
+ *
+ * The funcion 'func' will be called with a pointer to a link
+ * as first argument and 'arg' as second argument for each link
+ * in the tree referred to by 'root'.
+ *
+ * @param[in]     root          Pointer to root of link tree
+ *
+ * @param[in]     func          Pointer to function to call
+ *
+ * @param[in]     arg           Argument to pass as second argument in
+ *                              calls to 'func'
+ *
+ */
+void erts_link_tree_foreach(ErtsLink *root,
+                            void (*func)(ErtsLink *, void *),
+                            void *arg);
+
+/**
+ *
+ * @brief Call a function for each link in a link tree. Yield if lots
+ *        of links exist.
+ *
+ * The funcion 'func' will be called with a pointer to a link
+ * as first argument and 'arg' as second argument for each link
+ * in the tree referred to by 'root'.
+ *
+ * It is assumed that:
+ * - *yspp equals NULL on first call
+ * - this function is repetedly called with *yspp set
+ *   as set when previous call returned until a non-zero
+ *   value is returned.
+ * - no modifications are made on the tree between first call
+ *   and the call that returns a non-zero value
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in]     root          Pointer to root of link tree
+ *
+ * @param[in]     func          Pointer to function to call
+ *
+ * @param[in]     arg           Argument to pass as second argument in
+ *                              calls to 'func'
+ *
+ * @param[in,out] vyspp         Pointer to a pointer to an internal state
+ *                              used by this function. At initial call
+ *                              *yspp should be NULL. When done *yspp
+ *                              will be NULL.
+ *
+ * @param[in]     limit         Maximum amount of links to process
+ *                              before yielding.
+ *
+ * @returns                     A non-zero value when all links has been
+ *                              processed, and zero when more work is needed.
+ *
+ */
+int erts_link_tree_foreach_yielding(ErtsLink *root,
+                                    void (*func)(ErtsLink *, void *),
+                                    void *arg,
+                                    void **vyspp,
+                                    Sint limit);
+
+/**
+ *
+ * @brief Delete all links from a link tree and call a function for
+ *        each link
+ *
+ * The funcion 'func' will be called with a pointer to a link
+ * as first argument and 'arg' as second argument for each link
+ * in the tree referred to by 'root'.
+ *
+ * @param[in,out] root          Pointer to pointer to root of link tree
+ *
+ * @param[in]     func          Pointer to function to call
+ *
+ * @param[in]     arg           Argument to pass as second argument in
+ *                              calls to 'func'
+ *
+ */
+void erts_link_tree_foreach_delete(ErtsLink **root,
+                                   void (*func)(ErtsLink *, void *),
+                                   void *arg);
+
+/**
+ *
+ * @brief Delete all links from a link tree and call a function for
+ *        each link
+ *
+ * The funcion 'func' will be called with a pointer to a link
+ * as first argument and 'arg' as second argument for each link
+ * in the tree referred to by 'root'.
+ *
+ * It is assumed that:
+ * - *yspp equals NULL on first call
+ * - this function is repetededly called with *yspp set
+ *   as set when previous call returned until a non-zero
+ *   value is returned.
+ * - no modifications are made on the tree between first call
+ *   and the call that returns a non-zero value
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in,out] root          Pointer to pointer to root of link tree
+ *
+ * @param[in]     func          Pointer to function to call
+ *
+ * @param[in]     arg           Argument to pass as second argument in
+ *                              calls to 'func'
+ *
+ * @param[in,out] vyspp         Pointer to a pointer to an internal state
+ *                              used by this function. At initial call
+ *                              *yspp should be NULL. When done *yspp
+ *                              will be NULL.
+ *
+ * @param[in]     limit         Maximum amount of links to process
+ *                              before yielding.
+ *
+ * @returns                     A non-zero value when all links has been
+ *                              processed, and zero when more work is needed.
+ *
+ */
+int erts_link_tree_foreach_delete_yielding(ErtsLink **root,
+                                           void (*func)(ErtsLink *, void *),
+                                           void *arg,
+                                           void **vyspp,
+                                           Sint limit);
+
+/* 
+ * --- Link list operations ---
+ */
+
+/**
+ *
+ * @brief Insert a link in a link list
+ *
+ * When the funcion is called it is assumed that:
+ * - 'lnk' link is not part of any list or tree
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in,out] list          Pointer to pointer to link list
+ *
+ * @param[in]     lnk           Link to insert
+ *
+ */
+ERTS_GLB_INLINE void erts_link_list_insert(ErtsLink **list, ErtsLink *lnk);
+
+/**
+ *
+ * @brief Delete a link from a link list
+ *
+ * When the funcion is called it is assumed that:
+ * - 'lnk' link is part of the list
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in,out] list          Pointer to pointer to link list
+ *
+ * @param[in]     lnk           Link to remove from the list
+ *
+ */
+ERTS_GLB_INLINE void erts_link_list_delete(ErtsLink **list, ErtsLink *lnk);
+
+/**
+ *
+ * @brief Get a pointer to first link in a link list
+ *
+ * The link will still remain in the list after the return
+ *
+ * @param[in] list              Pointer to link list
+ *
+ * @returns                     Pointer to first link in list if
+ *                              list is not empty. If list is empty
+ *                              NULL is returned.
+ *
+ */
+ERTS_GLB_INLINE ErtsLink *erts_link_list_first(ErtsLink *list);
+
+/**
+ *
+ * @brief Get a pointer to last link in a link list
+ *
+ * The link will still remain in the list after the return
+ *
+ * @param[in] list              Pointer to link list
+ *
+ * @returns                     Pointer to last link in list if
+ *                              list is not empty. If list is empty
+ *                              NULL is returned.
+ *
+ */
+ERTS_GLB_INLINE ErtsLink *erts_link_list_last(ErtsLink *list);
+
+/**
+ *
+ * @brief Call a function for each link in a link list
+ *
+ * The funcion 'func' will be called with a pointer to a link
+ * as first argument and 'arg' as second argument for each link
+ * in the tree referred to by 'list'.
+ *
+ * @param[in]     list          Pointer to root of link list
+ *
+ * @param[in]     func          Pointer to function to call
+ *
+ * @param[in]     arg           Argument to pass as second argument in
+ *                              calls to 'func'
+ *
+ */
+void erts_link_list_foreach(ErtsLink *list,
+                            void (*func)(ErtsLink *, void *),
+                            void *arg);
+
+/**
+ *
+ * @brief Call a function for each link in a link list. Yield
+ *        if lots of links exist.
+ *
+ * The funcion 'func' will be called with a pointer to a link
+ * as first argument and 'arg' as second argument for each link
+ * in the tree referred to by 'root'.
+ *
+ * It is assumed that:
+ * - *yspp equals NULL on first call
+ * - this function is repetedly called with *yspp set
+ *   as set when previous call returned until a non-zero
+ *   value is returned.
+ * - no modifications are made on the tree between first call
+ *   and the call that returns a non-zero value
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in]     list          Pointer to link list
+ *
+ * @param[in]     func          Pointer to function to call
+ *
+ * @param[in]     arg           Argument to pass as second argument in
+ *                              calls to 'func'
+ *
+ * @param[in,out] vyspp         Pointer to a pointer to an internal state
+ *                              used by this function. At initial call
+ *                              *yspp should be NULL. When done *yspp
+ *                              will be NULL.
+ *
+ * @param[in]     limit         Maximum amount of links to process
+ *                              before yielding.
+ *
+ * @returns                     A non-zero value when all links has been
+ *                              processed, and zero when more work is needed.
+ *
+ */
+int erts_link_list_foreach_yielding(ErtsLink *list,
+                                    void (*func)(ErtsLink *, void *),
+                                    void *arg,
+                                    void **vyspp,
+                                    Sint limit);
+
+/**
+ *
+ * @brief Delete all links from a link list and call a function for
+ *        each link
+ *
+ * The funcion 'func' will be called with a pointer to a link
+ * as first argument and 'arg' as second argument for each link
+ * in the tree referred to by 'root'.
+ *
+ * @param[in,out] list          Pointer to pointer to link list
+ *
+ * @param[in]     func          Pointer to function to call
+ *
+ * @param[in]     arg           Argument to pass as second argument in
+ *                              calls to 'func'
+ *
+ */
+void erts_link_list_foreach_delete(ErtsLink **list,
+                                   void (*func)(ErtsLink *, void *),
+                                   void *arg);
+
+/**
+ *
+ * @brief Delete all links from a link list and call a function for
+ *        each link
+ *
+ * The funcion 'func' will be called with a pointer to a link
+ * as first argument and 'arg' as second argument for each link
+ * in the tree referred to by 'root'.
+ *
+ * It is assumed that:
+ * - *yspp equals NULL on first call
+ * - this function is repetededly called with *yspp set
+ *   as set when previous call returned until a non-zero
+ *   value is returned.
+ * - no modifications are made on the tree between first
+ *   and the call that returns a non-zero value
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in,out] list          Pointer to pointer to link list
+ *
+ * @param[in]     func          Pointer to function to call
+ *
+ * @param[in]     arg           Argument to pass as second argument in
+ *                              calls to 'func'
+ *
+ * @param[in,out] vyspp         Pointer to a pointer to an internal state
+ *                              used by this function. At initial call
+ *                              *yspp should be NULL. When done *yspp
+ *                              will be NULL.
+ *
+ * @param[in]     limit         Maximum amount of links to process
+ *                              before yielding.
+ *
+ * @returns                     A non-zero value when all links has been
+ *                              processed, and zero when more work is needed.
+ *
+ */
+int erts_link_list_foreach_delete_yielding(ErtsLink **list,
+                                           void (*func)(ErtsLink *, void *),
+                                           void *arg,
+                                           void **vyspp,
+                                           Sint limit);
+
+/*
+ * --- Misc link operations ---
+ */
+
+/**
+ *
+ * @brief Create a link
+ *
+ * Can create all types of links
+ *
+ * When the funcion is called it is assumed that:
+ * - 'ref' is an internal ordinary reference if type is ERTS_MON_TYPE_PROC,
+ *   ERTS_MON_TYPE_PORT, ERTS_MON_TYPE_TIME_OFFSET, or ERTS_MON_TYPE_RESOURCE
+ * - 'ref' is NIL if type is ERTS_MON_TYPE_NODE or ERTS_MON_TYPE_NODES
+ * - 'ref' is and ordinary internal reference or an external reference if
+ *   type is ERTS_MON_TYPE_DIST_PROC
+ * - 'name' is an atom or NIL if type is ERTS_MON_TYPE_PROC,
+ *   ERTS_MON_TYPE_PORT, or ERTS_MON_TYPE_DIST_PROC
+ * - 'name is NIL if type is ERTS_MON_TYPE_TIME_OFFSET, ERTS_MON_TYPE_RESOURCE,
+ *   ERTS_MON_TYPE_NODE, or ERTS_MON_TYPE_NODES
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in]     type          ERTS_MON_TYPE_PROC, ERTS_MON_TYPE_PORT,
+ *                              ERTS_MON_TYPE_TIME_OFFSET, ERTS_MON_TYPE_DIST_PROC,
+ *                              ERTS_MON_TYPE_RESOURCE, ERTS_MON_TYPE_NODE,
+ *                              or ERTS_MON_TYPE_NODES
+ *
+ * @param[in]     a             The key of entity a. Link structure a will
+ *                              have field other.item set to 'b'.
+ *
+ * @param[in]     b             The key of entity b. Link structure b will
+ *                              have field other.item set to 'a'.
+ *
+ */
+ErtsLinkData *erts_link_create(Uint16 type, Eterm a, Eterm b);
+
+/**
+ *
+ * @brief Get pointer to link data structure
+ *
+ * @param[in]    lnk            Pointer to link
+ *
+ * @returns                     Pointer to link data structure
+ *
+ */
+ERTS_GLB_INLINE ErtsLinkData *erts_link_to_data(ErtsLink *lnk);
+
+/**
+ *
+ * @brief Get pointer to the other link structure part of the link
+ *
+ * @param[in]    lnk            Pointer to link
+ *
+ * @param[out]   ldpp           Pointer to pointer to link data structure,
+ *                              if a non-NULL value is passed in the call
+ *
+ * @returns                     Pointer to other link
+ *
+ */
+ERTS_GLB_INLINE ErtsLink *erts_link_to_other(ErtsLink *lnk, ErtsLinkData **ldpp);
+
+/**
+ *
+ * @brief Check if link is in tree or list
+ *
+ * @param[in]    lnk            Pointer to lnk to check
+ *
+ * @returns                     A non-zero value if in tree or list;
+ *                              otherwise zero
+ *
+ */
+ERTS_GLB_INLINE int erts_link_is_in_table(ErtsLink *lnk);
+
+/**
+ *
+ * @brief Release link
+ *
+ * When both link halves part of the link have been released the link
+ * structure will be deallocated.
+ *
+ * When the funcion is called it is assumed that:
+ * - 'lnk' link is not part of any list or tree
+ * - 'lnk' is not referred to by any other structures
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in]    lnk            Pointer to link
+ *
+ */
+ERTS_GLB_INLINE void erts_link_release(ErtsLink *lnk);
+
+/**
+ *
+ * @brief Release both link halves of a link simultaneously
+ *
+ * Release both halves of a link simultaneously and deallocate
+ * the structure.
+ *
+ * When the funcion is called it is assumed that:
+ * - Neither of the parts of the link are part of any list or tree
+ * - Neither of the parts of the link or the link data structure
+ *   are referred to by any other structures
+ * If the above are not true, bad things will happen.
+ *
+ * @param[in]    mdp            Pointer to link data structure
+ *
+ */
+ERTS_GLB_INLINE void erts_link_release_both(ErtsLinkData *ldp);
+
+/**
+ *
+ * @brief Insert link in dist link list
+ *
+ * When the funcion is called it is assumed that:
+ * - 'lnk' link is not part of any list or tree
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in]    lnk            Pointer to link
+ *
+ * @param[in]    dist           Pointer to dist structure
+ *
+ * @returns                     A non-zero value if inserted;
+ *                              otherwise, zero. The link
+ *                              is not inserted if the dist
+ *                              structure has been set in a
+ *                              dead state.
+ *
+ */
+ERTS_GLB_INLINE int erts_link_dist_insert(ErtsLink *lnk, ErtsMonLnkDist *dist);
+
+/**
+ *
+ * @brief Delete link from dist link list
+ *
+ * When the funcion is called it is assumed that:
+ * - 'lnk' link earler has been inserted into 'dist'
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in]    lnk            Pointer to link
+ *
+ * @param[in]    dist           Pointer to dist structure
+ *
+ * @returns                     A non-zero value if deleted;
+ *                              otherwise, zero. The link
+ *                              is not deleted if the dist
+ *                              structure has been set in a
+ *                              dead state or if it has already
+ *                              been deleted.
+ *
+ */
+ERTS_GLB_INLINE int erts_link_dist_delete(ErtsLink *lnk);
+
+/**
+ *
+ * @brief Set dead dist structure on link
+ *
+ * @param[in]    lnk            Pointer to link
+ *
+ * @param[in]    nodename       Name of remote node
+ *
+ */
+void
+erts_link_set_dead_dist(ErtsLink *lnk, Eterm nodename);
+
+/**
+ *
+ * @brief Get charged size of link
+ *
+ * If the other side of the link has been released, the
+ * whole size of the link data structure is returned; otherwise,
+ * half of the size is returned.
+ *
+ * When the funcion is called it is assumed that:
+ * - 'lnk' has not been released
+ * If the above is not true, bad things will happen.
+ *
+ * @param[in]    lnk            Pointer to link
+ *
+ * @returns                     Charged size in bytes
+ *
+ */
+Uint erts_link_size(ErtsLink *lnk);
+
+/* internal function... */
+void erts_link_destroy__(ErtsLinkData *ldp);
+
+/* implementations for globally inlined link functions... */
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+#ifdef ERTS_ML_DEBUG
+extern size_t erts_link_a_offset;
+extern size_t erts_link_b_offset;
+extern size_t erts_link_key_offset;
+#endif
+
+ERTS_GLB_INLINE ErtsLinkData *
+erts_link_to_data(ErtsLink *lnk)
+{
+    ErtsLinkData *ldp = erts_ml_node_to_main_struct__((ErtsMonLnkNode *) lnk);
+
+#ifdef ERTS_ML_DEBUG
+    ERTS_ML_ASSERT(erts_link_a_offset == (size_t) ldp->a.offset);
+    ERTS_ML_ASSERT(erts_link_key_offset == (size_t) ldp->a.key_offset);
+    ERTS_ML_ASSERT(erts_link_b_offset == (size_t) ldp->b.offset);
+    ERTS_ML_ASSERT(erts_link_key_offset == (size_t) ldp->b.key_offset);
+#endif
+
+    return ldp;
+}
+
+ERTS_GLB_INLINE ErtsLink *
+erts_link_to_other(ErtsLink *lnk, ErtsLinkData **ldpp)
+{
+    ErtsLinkData *ldp = erts_link_to_data(lnk);
+    if (ldpp)
+        *ldpp = ldp;
+    return lnk == &ldp->a ? &ldp->b : &ldp->a;
+}
+
+ERTS_GLB_INLINE int
+erts_link_is_in_table(ErtsLink *lnk)
+{
+    return !!(lnk->flags & ERTS_ML_FLG_IN_TABLE);
+}
+
+ERTS_GLB_INLINE void
+erts_link_list_insert(ErtsLink **list, ErtsLink *lnk)
+{
+    erts_ml_dl_list_insert__((ErtsMonLnkNode **) list, (ErtsMonLnkNode *) lnk);
+}
+
+ERTS_GLB_INLINE void
+erts_link_list_delete(ErtsLink **list, ErtsLink *lnk)
+{
+    erts_ml_dl_list_delete__((ErtsMonLnkNode **) list, (ErtsMonLnkNode *) lnk);
+}
+
+ERTS_GLB_INLINE ErtsLink *
+erts_link_list_first(ErtsLink *list)
+{
+    return (ErtsLink *) erts_ml_dl_list_first__((ErtsMonLnkNode *) list);
+}
+
+ERTS_GLB_INLINE ErtsLink *
+erts_link_list_last(ErtsLink *list)
+{
+    return (ErtsLink *) erts_ml_dl_list_last__((ErtsMonLnkNode *) list);
+}
+
+ERTS_GLB_INLINE void
+erts_link_release(ErtsLink *lnk)
+{
+    ErtsLinkData *ldp = erts_link_to_data(lnk);
+    ERTS_ML_ASSERT(!(lnk->flags & ERTS_ML_FLG_IN_TABLE));
+    ERTS_ML_ASSERT(erts_atomic32_read_nob(&ldp->refc) > 0);
+    if (erts_atomic32_dec_read_nob(&ldp->refc) == 0)
+        erts_link_destroy__(ldp);
+}
+
+ERTS_GLB_INLINE void
+erts_link_release_both(ErtsLinkData *ldp)
+{
+    ERTS_ML_ASSERT(!(ldp->a.flags & ERTS_ML_FLG_IN_TABLE));
+    ERTS_ML_ASSERT(!(ldp->b.flags & ERTS_ML_FLG_IN_TABLE));
+    ERTS_ML_ASSERT(erts_atomic32_read_nob(&ldp->refc) >= 2);
+    if (erts_atomic32_add_read_nob(&ldp->refc, (erts_aint32_t) -2) == 0)
+        erts_link_destroy__(ldp);
+}
+
+ERTS_GLB_INLINE ErtsLink *
+erts_link_tree_insert_addr_replace(ErtsLink **root, ErtsLink *lnk)
+{
+    ErtsLink *lnk2 = erts_link_tree_lookup_insert(root, lnk);
+    if (!lnk2)
+        return NULL;
+    if (lnk2 < lnk)
+        return lnk;
+    erts_link_tree_replace(root, lnk2, lnk);
+    return lnk2;
+}
+
+ERTS_GLB_INLINE ErtsLink *
+erts_link_tree_key_delete(ErtsLink **root, ErtsLink *lnk)
+{
+    ErtsLink *dlnk;
+    if (erts_link_is_in_table(lnk))
+        dlnk = lnk;
+    else
+        dlnk = erts_link_tree_lookup(*root, lnk->other.item);
+    if (dlnk)
+        erts_link_tree_delete(root, dlnk);
+    return dlnk;
+}
+
+ERTS_GLB_INLINE int
+erts_link_dist_insert(ErtsLink *lnk, ErtsMonLnkDist *dist)
+{
+    ErtsLinkDataExtended *ldep;
+    int insert;
+
+    ERTS_ML_ASSERT(lnk->flags & ERTS_ML_FLG_EXTENDED);
+    ERTS_ML_ASSERT(lnk->type == ERTS_LNK_TYPE_DIST_PROC);
+
+    ldep = (ErtsLinkDataExtended *) erts_link_to_data(lnk);
+
+    ERTS_ML_ASSERT(!ldep->dist);
+    ERTS_ML_ASSERT(dist);
+    ldep->dist = dist;
+
+    erts_mon_link_dist_inc_refc(dist);
+
+    erts_mtx_lock(&dist->mtx);
+
+    insert = dist->alive;
+    if (insert)
+        erts_link_list_insert(&dist->links, lnk);
+
+    erts_mtx_unlock(&dist->mtx);
+
+    return insert;
+}
+
+ERTS_GLB_INLINE int
+erts_link_dist_delete(ErtsLink *lnk)
+{
+    ErtsLinkDataExtended *ldep;
+    ErtsMonLnkDist *dist;
+    int delete;
+
+    ERTS_ML_ASSERT(lnk->flags & ERTS_ML_FLG_EXTENDED);
+    ERTS_ML_ASSERT(lnk->type == ERTS_LNK_TYPE_DIST_PROC);
+
+    ldep = (ErtsLinkDataExtended *) erts_link_to_data(lnk);
+    dist = ldep->dist;
+    if (!dist)
+        return -1;
+
+    erts_mtx_lock(&dist->mtx);
+
+    delete = !!dist->alive & !!(lnk->flags & ERTS_ML_FLG_IN_TABLE);
+    if (delete)
+        erts_link_list_delete(&dist->links, lnk);
+
+    erts_mtx_unlock(&dist->mtx);
+
+    return delete;
+}
+
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#endif /* ERL_MONITOR_LINK_H__ */
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
deleted file mode 100644
index 1c840d89f6..0000000000
--- a/erts/emulator/beam/erl_monitors.c
+++ /dev/null
@@ -1,1073 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2017. All Rights Reserved.
- * 
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * 
- * %CopyrightEnd%
- */
-
-/**************************************************************************
- * Monitors and links data structure manipulation.
- * Monitors and links are organized as AVL trees with the reference as 
- * key in the monitor case and the pid of the linked process as key in the 
- * link case. Lookups the order of the references is somewhat special. Local 
- * references are strictly smaller than remote references and are sorted 
- * by inlined comparison functionality. Remote references are handled by the
- * usual cmp function.
- * Each Monitor is tagged with different tags depending on which end of the 
- * monitor it is.
- * A monitor is removed either explicitly by reference or all monitors are 
- * removed when the process exits. No need to access the monitor by pid.
- **************************************************************************/ 
-
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-
-#include "sys.h"
-#include "erl_vm.h"
-#include "global.h"
-#include "erl_process.h"
-#include "error.h"
-#include "erl_db.h"
-#include "bif.h"
-#include "big.h"
-#include "erl_monitors.h"
-#include "erl_bif_unique.h"
-
-#define STACK_NEED 50
-#define MAX_MONITORS 0xFFFFFFFFUL
-
-#define DIR_LEFT 0
-#define DIR_RIGHT 1
-#define DIR_END 2 
-
-static erts_atomic_t tot_link_lh_size;
-
-/* Implements the sort order in monitor trees, which is different from 
-   the ordinary term order.
-   No short local ref's should ever exist (the ref is created by the bif's 
-   in runtime), therefore:
-   All local ref's are less than external ref's
-   Local ref's are inline-compared,
-   External ref's are compared by cmp */
-
-#if 0   
-#define CMP_MON_REF(Ref1,Ref2) \
-cmp((Ref1),(Ref2)) /* XXX, the inline comparision yet to be done */
-#else
-#define CMP_MON_REF(Ref1,Ref2) cmp_mon_ref((Ref1),(Ref2))
-#endif
-
-static ERTS_INLINE int cmp_mon_ref(Eterm ref1, Eterm ref2) 
-{
-    Eterm *b1, *b2;
-
-
-    b1 = boxed_val(ref1);
-    b2 = boxed_val(ref2);
-    if (is_ref_thing_header(*b1)) {
-	if (is_ref_thing_header(*b2)) {
-	    Uint32 *num1, *num2;
-	    if (is_ordinary_ref_thing(b1)) {
-		ErtsORefThing *rtp = (ErtsORefThing *) b1;
-		num1 = rtp->num;
-	    }
-	    else {
-		ErtsMRefThing *mrtp = (ErtsMRefThing *) b1;
-		num1 = mrtp->mb->refn;
-	    }
-	    if (is_ordinary_ref_thing(b2)) {
-		ErtsORefThing *rtp = (ErtsORefThing *) b2;
-		num2 = rtp->num;
-	    }
-	    else {
-		ErtsMRefThing *mrtp = (ErtsMRefThing *) b2;
-		num2 = mrtp->mb->refn;
-	    }
-	    return erts_internal_ref_number_cmp(num1, num2);
-	}
-	return -1;
-    }
-    if (is_ref_thing_header(*b2)) {
-	return 1;
-    }
-    return CMP(ref1,ref2);
-}
-	    
-#define CP_LINK_VAL(To, Hp, From)				\
-do {								\
-    if (is_immed(From))						\
-	(To) = (From);						\
-    else {							\
-	Uint i__;						\
-	Uint len__;						\
-	ASSERT((Hp));						\
-	ASSERT(is_internal_ordinary_ref((From))                 \
-               || is_external((From)));                         \
-	(To) = make_boxed((Hp));				\
-	len__ = thing_arityval(*boxed_val((From))) + 1;		\
-	for(i__ = 0; i__ < len__; i__++)			\
-	    (*((Hp)++)) = boxed_val((From))[i__];		\
-	if (is_external((To))) {				\
-	    external_thing_ptr((To))->next = NULL;		\
-	    erts_refc_inc(&(external_thing_ptr((To))->node->refc), 2);\
-	}							\
-    }								\
-} while (0)
-
-static ErtsMonitor *create_monitor(Uint type, Eterm ref, UWord entity, Eterm name)
-{
-     Uint mon_size = ERTS_MONITOR_SIZE;
-     ErtsMonitor *n;
-     Eterm *hp;
-
-     mon_size += NC_HEAP_SIZE(ref);
-     if (type != MON_NIF_TARGET && is_not_immed(entity)) {
-	 mon_size += NC_HEAP_SIZE(entity);
-     }
-
-     if (mon_size <= ERTS_MONITOR_SH_SIZE) {
-	 n = (ErtsMonitor *) erts_alloc(ERTS_ALC_T_MONITOR_SH,
-					mon_size*sizeof(Uint));
-     } else {
-	 n = (ErtsMonitor *) erts_alloc(ERTS_ALC_T_MONITOR_LH,
-					mon_size*sizeof(Uint));
-	 erts_atomic_add_nob(&tot_link_lh_size, mon_size*sizeof(Uint));
-     } 
-     hp = n->heap;
-
-
-     n->left = n->right = NULL; /* Always the same initial value*/
-     n->type = (Uint16) type;
-     n->balance = 0;            /* Always the same initial value */
-     n->name = name; /* atom() or [] */
-     CP_LINK_VAL(n->ref, hp, ref); /*XXX Unnecessary check, never immediate*/
-     if (type == MON_NIF_TARGET)
-         n->u.resource = (ErtsResource*)entity;
-     else
-         CP_LINK_VAL(n->u.pid, hp, (Eterm)entity);
-
-     return n;
-}
-
-static ErtsLink *create_link(Uint type, Eterm pid)
-{
-     Uint lnk_size = ERTS_LINK_SIZE;
-     ErtsLink *n;
-     Eterm *hp;
-
-     if (is_not_immed(pid)) {
-	 lnk_size += NC_HEAP_SIZE(pid);
-     }
-
-     if (lnk_size <= ERTS_LINK_SH_SIZE) {
-	 n = (ErtsLink *) erts_alloc(ERTS_ALC_T_NLINK_SH,
-				     lnk_size*sizeof(Uint));
-     } else {
-	 n = (ErtsLink *) erts_alloc(ERTS_ALC_T_NLINK_LH,
-				     lnk_size*sizeof(Uint));
-	 erts_atomic_add_nob(&tot_link_lh_size, lnk_size*sizeof(Uint));
-     } 
-     hp = n->heap;
-
-
-     n->left = n->right = NULL; /* Always the same initial value*/
-     n->type = (Uint16) type;
-     n->balance = 0;            /* Always the same initial value */
-     if (n->type == LINK_NODE) {
-	 ERTS_LINK_REFC(n) = 0;
-     } else {
-	 ERTS_LINK_ROOT(n) = NULL; 
-     }
-     CP_LINK_VAL(n->pid, hp, pid);
-
-     return n;
-}
-
-#undef CP_LINK_VAL
-
-static ErtsSuspendMonitor *create_suspend_monitor(Eterm pid)
-{
-    ErtsSuspendMonitor *smon = erts_alloc(ERTS_ALC_T_SUSPEND_MON,
-					  sizeof(ErtsSuspendMonitor));
-    smon->left = smon->right = NULL; /* Always the same initial value */
-    smon->balance = 0;               /* Always the same initial value */
-    smon->pending = 0;
-    smon->active = 0;
-    smon->pid = pid;
-    return smon;
-}
-
-void
-erts_init_monitors(void)
-{
-    erts_atomic_init_nob(&tot_link_lh_size, 0);
-}
-
-Uint
-erts_tot_link_lh_size(void)
-{
-    return (Uint) erts_atomic_read_nob(&tot_link_lh_size);
-}
-
-void erts_destroy_monitor(ErtsMonitor *mon)
-{
-    Uint mon_size = ERTS_MONITOR_SIZE;
-    ErlNode *node;
-
-    ASSERT(is_not_immed(mon->ref));
-    mon_size +=  NC_HEAP_SIZE(mon->ref);
-    if (is_external(mon->ref)) {
-	node = external_thing_ptr(mon->ref)->node;
-	erts_deref_node_entry(node);
-    }
-    if (mon->type != MON_NIF_TARGET && is_not_immed(mon->u.pid)) {
-	mon_size += NC_HEAP_SIZE(mon->u.pid);
-	if (is_external(mon->u.pid)) {
-	    node = external_thing_ptr(mon->u.pid)->node;
-	    erts_deref_node_entry(node);
-	}
-    }
-    if (mon_size <= ERTS_MONITOR_SH_SIZE) {
-	erts_free(ERTS_ALC_T_MONITOR_SH, (void *) mon);
-    } else {
-	erts_free(ERTS_ALC_T_MONITOR_LH, (void *) mon);
-	erts_atomic_add_nob(&tot_link_lh_size, -1*mon_size*sizeof(Uint));
-    }
-}
-    
-void erts_destroy_link(ErtsLink *lnk)
-{
-    Uint lnk_size = ERTS_LINK_SIZE;
-    ErlNode *node;
-
-    ASSERT(lnk->type == LINK_NODE || ERTS_LINK_ROOT(lnk) == NULL);
-
-    if (is_not_immed(lnk->pid)) {
-	lnk_size += NC_HEAP_SIZE(lnk->pid);
-	if (is_external(lnk->pid)) {
-	    node = external_thing_ptr(lnk->pid)->node;
-	    erts_deref_node_entry(node);
-	}
-    }
-    if (lnk_size <= ERTS_LINK_SH_SIZE) {
-	erts_free(ERTS_ALC_T_NLINK_SH, (void *) lnk);
-    } else {
-	erts_free(ERTS_ALC_T_NLINK_LH, (void *) lnk);
-	erts_atomic_add_nob(&tot_link_lh_size, -1*lnk_size*sizeof(Uint));
-    }
-}
-
-void erts_destroy_suspend_monitor(ErtsSuspendMonitor *smon)
-{
-    erts_free(ERTS_ALC_T_SUSPEND_MON, smon);
-}
-     
-static void insertion_rotation(int dstack[], int dpos, 
-			       void *tstack[], int tpos, 
-			       int state) {
-    
-    ErtsMonitorOrLink **this;
-    ErtsMonitorOrLink *p1, *p2, *p;
-    int dir;
-
-    while (state && ( dir = dstack[--dpos] ) != DIR_END) {
-	this = tstack[--tpos];
-	p = *this;
-	if (dir == DIR_LEFT) {
-	    switch (p->balance) {
-	    case 1:
-		p->balance = 0;
-		state = 0;
-		break;
-	    case 0:
-		p->balance = -1;
-		break;
-	    case -1: /* The icky case */
-		p1 = p->left;
-		if (p1->balance == -1) { /* Single LL rotation */
-		    p->left = p1->right;
-		    p1->right = p;
-		    p->balance = 0;
-		    (*this) = p1;
-		} else { /* Double RR rotation */
-		    p2 = p1->right;
-		    p1->right = p2->left;
-		    p2->left = p1;
-		    p->left = p2->right;
-		    p2->right = p;
-		    p->balance = (p2->balance == -1) ? +1 : 0;
-		    p1->balance = (p2->balance == 1) ? -1 : 0;
-		    (*this) = p2;
-		}
-		(*this)->balance = 0;
-		state = 0;
-		break;
-	    }
-	} else { /* dir == DIR_RIGHT */
-	    switch (p->balance) {
-	    case -1:
-		p->balance = 0;
-		state = 0;
-		break;
-	    case 0:
-		p->balance = 1;
-		break;
-	    case 1:
-		p1 = p->right;
-		if (p1->balance == 1) { /* Single RR rotation */
-		    p->right = p1->left;
-		    p1->left = p;
-		    p->balance = 0;
-		    (*this) = p1;
-		} else { /* Double RL rotation */
-		    p2 = p1->left;
-		    p1->left = p2->right;
-		    p2->right = p1;
-		    p->right = p2->left;
-		    p2->left = p;
-		    p->balance = (p2->balance == 1) ? -1 : 0;
-		    p1->balance = (p2->balance == -1) ? 1 : 0;
-		    (*this) = p2;
-		}
-		(*this)->balance = 0; 
-		state = 0;
-		break;
-	    }
-	}
-    }
-}
-
-void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, UWord entity, 
-		      Eterm name)
-{
-    void *tstack[STACK_NEED];
-    int tpos = 0;
-    int dstack[STACK_NEED+1];
-    int dpos = 1;
-    int state = 0;
-    ErtsMonitor **this = root;
-    Sint c;
-
-    ASSERT(is_internal_ordinary_ref(ref) || is_external_ref(ref));
-  
-    dstack[0] = DIR_END;
-    for (;;) {
-	if (!*this) { /* Found our place */
-	    state = 1;
-	    *this = create_monitor(type,ref,entity,name);
-	    break;
-	} else if ((c = CMP_MON_REF(ref,(*this)->ref)) < 0) { 
-	    /* go left */
-	    dstack[dpos++] = DIR_LEFT;
-	    tstack[tpos++] = this;
-	    this = &((*this)->left);
-	} else if (c > 0) { /* go right */
-	    dstack[dpos++] = DIR_RIGHT;
-	    tstack[tpos++] = this;
-	    this = &((*this)->right);
-	} else { /* Equal key is an error for monitors */
-	    erts_exit(ERTS_ERROR_EXIT,"Insertion of already present monitor!");
-	    break;
-	}
-    }
-    insertion_rotation(dstack, dpos, tstack, tpos, state);
-}
-
-
-/* Returns 0 if OK, < 0 if already present */
-int erts_add_link(ErtsLink **root, Uint type, Eterm pid)
-{
-    void *tstack[STACK_NEED];
-    int tpos = 0;
-    int dstack[STACK_NEED+1];
-    int dpos = 1;
-    int state = 0;
-    ErtsLink **this = root;
-    Sint c;
-  
-    dstack[0] = DIR_END;
-    for (;;) {
-	if (!*this) { /* Found our place */
-	    state = 1;
-	    *this = create_link(type,pid);
-	    break;
-	} else if ((c = CMP(pid,(*this)->pid)) < 0) {
-	    /* go left */
-	    dstack[dpos++] = DIR_LEFT;
-	    tstack[tpos++] = this;
-	    this = &((*this)->left);
-	} else if (c > 0) { /* go right */
-	    dstack[dpos++] = DIR_RIGHT;
-	    tstack[tpos++] = this;
-	    this = &((*this)->right);
-	} else { /* Equal key is an error for monitors */
-	    return -1;
-	}
-    }
-    insertion_rotation(dstack, dpos, tstack, tpos, state);
-    return 0;
-}
-
-ErtsSuspendMonitor *
-erts_add_or_lookup_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid)
-{
-    void *tstack[STACK_NEED];
-    int tpos = 0;
-    int dstack[STACK_NEED+1];
-    int dpos = 1;
-    int state = 0;
-    ErtsSuspendMonitor **this = root;
-    ErtsSuspendMonitor *res;
-    Sint c;
-  
-    dstack[0] = DIR_END;
-    for (;;) {
-	if (!*this) { /* Found our place */
-	    state = 1;
-	    res = *this = create_suspend_monitor(pid);
-	    break;
-	} else if ((c = CMP(pid,(*this)->pid)) < 0) {
-	    /* go left */
-	    dstack[dpos++] = DIR_LEFT;
-	    tstack[tpos++] = this;
-	    this = &((*this)->left);
-	} else if (c > 0) { /* go right */
-	    dstack[dpos++] = DIR_RIGHT;
-	    tstack[tpos++] = this;
-	    this = &((*this)->right);
-	} else { /* Already here... */
-	    ASSERT((*this)->pid == pid);
-	    return *this;
-	}
-    }
-    insertion_rotation(dstack, dpos, tstack, tpos, state);
-    return res;
-}
-
-
-/* Returns the new or old link structure */
-ErtsLink *erts_add_or_lookup_link(ErtsLink **root, Uint type, Eterm pid)
-{
-    void *tstack[STACK_NEED];
-    int tpos = 0;
-    int dstack[STACK_NEED+1];
-    int dpos = 1;
-    int state = 0;
-    ErtsLink **this = root;
-    Sint c;
-    ErtsLink *ret = NULL;
-
-    dstack[0] = DIR_END;
-    for (;;) {
-	if (!*this) { /* Found our place */
-	    state = 1;
-	    *this = create_link(type,pid);
-	    ret = *this;
-	    break;
-	} else if ((c = CMP(pid,(*this)->pid)) < 0) {
-	    /* go left */
-	    dstack[dpos++] = DIR_LEFT;
-	    tstack[tpos++] = this;
-	    this = &((*this)->left);
-	} else if (c > 0) { /* go right */
-	    dstack[dpos++] = DIR_RIGHT;
-	    tstack[tpos++] = this;
-	    this = &((*this)->right);
-	} else { /* Equal key is an error for monitors */
-	    return *this;
-	}
-    }
-    insertion_rotation(dstack, dpos, tstack, tpos, state);
-    return ret;
-}
-
-
-/*
- * Deletion helpers
- */
-static int balance_left(ErtsMonitorOrLink **this) 
-{
-    ErtsMonitorOrLink *p, *p1, *p2;
-    int b1, b2, h = 1;
-    
-    p = *this;
-    switch (p->balance) {
-    case -1:
-	p->balance = 0;
-	break;
-    case 0:
-	p->balance = 1;
-	h = 0;
-	break;
-    case 1:
-	p1 = p->right;
-	b1 = p1->balance;
-	if (b1 >= 0) { /* Single RR rotation */
-	    p->right = p1->left;
-	    p1->left = p;
-	    if (b1 == 0) {
-		p->balance = 1;
-		p1->balance = -1;
-		h = 0;
-	    } else {
-		p->balance = p1->balance = 0;
-	    }
-	    (*this) = p1;
-	} else { /* Double RL rotation */
-	    p2 = p1->left;
-	    b2 = p2->balance;
-	    p1->left = p2->right;
-	    p2->right = p1;
-	    p->right = p2->left;
-	    p2->left = p;
-	    p->balance = (b2 == 1) ? -1 : 0;
-	    p1->balance = (b2 == -1) ? 1 : 0;
-	    p2->balance = 0;
-	    (*this) = p2;
-	}
-	break;
-    }
-    return h;
-}
-
-static int balance_right(ErtsMonitorOrLink **this) 
-{
-    ErtsMonitorOrLink *p, *p1, *p2;
-    int b1, b2, h = 1;
-    
-    p = *this;
-    switch (p->balance) {
-    case 1:
-	p->balance = 0;
-	break;
-    case 0:
-	p->balance = -1;
-	h = 0;
-	break;
-    case -1:
-	p1 = p->left;
-	b1 = p1->balance;
-	if (b1 <= 0) { /* Single LL rotation */
-	    p->left = p1->right;
-	    p1->right = p;
-	    if (b1 == 0) {
-		p->balance = -1;
-		p1->balance = 1;
-		h = 0;
-	    } else {
-		p->balance = p1->balance = 0;
-	    }
-	    (*this) = p1;
-	} else { /* Double LR rotation */
-	    p2 = p1->right;
-	    b2 = p2->balance;
-	    p1->right = p2->left;
-	    p2->left = p1;
-	    p->left = p2->right;
-	    p2->right = p;
-	    p->balance = (b2 == -1) ? 1 : 0;
-	    p1->balance = (b2 == 1) ? -1 : 0;
-	    p2->balance = 0;
-	    (*this) = p2;
-	}
-    }
-    return h;
-}
-
-static int delsub(ErtsMonitorOrLink **this) 
-{
-    ErtsMonitorOrLink **tstack[STACK_NEED];
-    int tpos = 0;
-    ErtsMonitorOrLink *q = (*this);
-    ErtsMonitorOrLink **r = &(q->left);
-    int h;
-
-    /*
-     * Walk down the tree to the right and search 
-     * for a void right child, pick that child out
-     * and return it to be put in the deleted 
-     * object's place.
-     */
-    
-    while ((*r)->right != NULL) {
-	tstack[tpos++] = r;
-	r = &((*r)->right);
-    }
-    *this = *r;
-    *r = (*r)->left;
-    (*this)->left = q->left;
-    (*this)->right = q->right;
-    (*this)->balance = q->balance;
-    tstack[0] = &((*this)->left);
-    h = 1;
-    while (tpos && h) {
-	r = tstack[--tpos];
-	h = balance_right(r);
-    }
-    return h;
-}
-
-ErtsMonitor *erts_remove_monitor(ErtsMonitor **root, Eterm ref) 
-{
-    ErtsMonitor **tstack[STACK_NEED];
-    int tpos = 0;
-    int dstack[STACK_NEED+1];
-    int dpos = 1;
-    int state = 0;
-    ErtsMonitor **this = root;
-    Sint c;
-    int dir;
-    ErtsMonitor *q = NULL;
-
-    dstack[0] = DIR_END;
-    for (;;) {
-	if (!*this) { /* Failure */
-	    return NULL;
-	} else if ((c = CMP_MON_REF(ref,(*this)->ref)) < 0) { 
-	    dstack[dpos++] = DIR_LEFT;
-	    tstack[tpos++] = this;
-	    this = &((*this)->left);
-	} else if (c > 0) { /* go right */
-	    dstack[dpos++] = DIR_RIGHT;
-	    tstack[tpos++] = this;
-	    this = &((*this)->right);
-	} else { /* Equal key, found the one to delete */
-	    q = (*this);
-	    if (q->right == NULL) {
-		(*this) = q->left;
-		state = 1;
-	    } else if (q->left == NULL) {
-		(*this) = q->right;
-		state = 1;
-	    } else {
-		dstack[dpos++] = DIR_LEFT;
-		tstack[tpos++] = this;
-		state = delsub((ErtsMonitorOrLink **) this);
-	    }
-	    break;
-	}
-    }
-    while (state && ( dir = dstack[--dpos] ) != DIR_END) {
-	this = tstack[--tpos];
-	if (dir == DIR_LEFT) {
-	    state = balance_left((ErtsMonitorOrLink **) this);
-	} else {
-	    state = balance_right((ErtsMonitorOrLink **) this);
-	}
-    }
-    return q;
-}
-
-ErtsLink *erts_remove_link(ErtsLink **root, Eterm pid) 
-{
-    ErtsLink **tstack[STACK_NEED];
-    int tpos = 0;
-    int dstack[STACK_NEED+1];
-    int dpos = 1;
-    int state = 0;
-    ErtsLink **this = root;
-    Sint c;
-    int dir;
-    ErtsLink *q = NULL;
-
-    dstack[0] = DIR_END;
-    for (;;) {
-	if (!*this) { /* Failure */
-	    return NULL;
-	} else if ((c = CMP(pid,(*this)->pid)) < 0) {
-	    dstack[dpos++] = DIR_LEFT;
-	    tstack[tpos++] = this;
-	    this = &((*this)->left);
-	} else if (c > 0) { /* go right */
-	    dstack[dpos++] = DIR_RIGHT;
-	    tstack[tpos++] = this;
-	    this = &((*this)->right);
-	} else { /* Equal key, found the one to delete */
-	    q = (*this);
-	    if (q->right == NULL) {
-		(*this) = q->left;
-		state = 1;
-	    } else if (q->left == NULL) {
-		(*this) = q->right;
-		state = 1;
-	    } else {
-		dstack[dpos++] = DIR_LEFT;
-		tstack[tpos++] = this;
-		state = delsub((ErtsMonitorOrLink **) this);
-	    }
-	    break;
-	}
-    }
-    while (state && ( dir = dstack[--dpos] ) != DIR_END) {
-	this = tstack[--tpos];
-	if (dir == DIR_LEFT) {
-	    state = balance_left((ErtsMonitorOrLink **) this);
-	} else {
-	    state = balance_right((ErtsMonitorOrLink **) this);
-	}
-    }
-    return q;
-}
-
-void
-erts_delete_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid)
-{
-    ErtsSuspendMonitor **tstack[STACK_NEED];
-    int tpos = 0;
-    int dstack[STACK_NEED+1];
-    int dpos = 1;
-    int state = 0;
-    ErtsSuspendMonitor **this = root;
-    Sint c;
-    int dir;
-    ErtsSuspendMonitor *q = NULL;
-
-    dstack[0] = DIR_END;
-    for (;;) {
-	if (!*this) { /* Nothing found */
-	    return;
-	} else if ((c = CMP(pid,(*this)->pid)) < 0) {
-	    dstack[dpos++] = DIR_LEFT;
-	    tstack[tpos++] = this;
-	    this = &((*this)->left);
-	} else if (c > 0) { /* go right */
-	    dstack[dpos++] = DIR_RIGHT;
-	    tstack[tpos++] = this;
-	    this = &((*this)->right);
-	} else { /* Equal key, found the one to delete */
-	    q = (*this);
-	    ASSERT(q->pid == pid);
-	    if (q->right == NULL) {
-		(*this) = q->left;
-		state = 1;
-	    } else if (q->left == NULL) {
-		(*this) = q->right;
-		state = 1;
-	    } else {
-		dstack[dpos++] = DIR_LEFT;
-		tstack[tpos++] = this;
-		state = delsub((ErtsMonitorOrLink **) this);
-	    }
-	    erts_destroy_suspend_monitor(q);
-	    break;
-	}
-    }
-    while (state && ( dir = dstack[--dpos] ) != DIR_END) {
-	this = tstack[--tpos];
-	if (dir == DIR_LEFT) {
-	    state = balance_left((ErtsMonitorOrLink **) this);
-	} else {
-	    state = balance_right((ErtsMonitorOrLink **) this);
-	}
-    }
-}
-
-ErtsMonitor *erts_lookup_monitor(ErtsMonitor *root, Eterm ref) 
-{
-    Sint c;
-
-    for (;;) {
-	if (root == NULL || (c = CMP_MON_REF(ref,root->ref)) == 0) {
-	    return root;
-	} else if (c < 0) { 
-	    root = root->left;
-	} else { /* c > 0 */ 
-	    root = root->right;
-	} 
-    }
-}
-
-ErtsLink *erts_lookup_link(ErtsLink *root, Eterm pid) 
-{
-    Sint c;
-
-    for (;;) {
-	if (root == NULL || (c = CMP(pid,root->pid)) == 0) {
-	    return root;
-	} else if (c < 0) { 
-	    root = root->left;
-	} else { /* c > 0 */ 
-	    root = root->right;
-	} 
-    }
-}
-
-ErtsSuspendMonitor *
-erts_lookup_suspend_monitor(ErtsSuspendMonitor *root, Eterm pid) 
-{
-    Sint c;
-
-    for (;;) {
-	if (root == NULL || (c = CMP(pid,root->pid)) == 0) {
-	    return root;
-	} else if (c < 0) { 
-	    root = root->left;
-	} else { /* c > 0 */ 
-	    root = root->right;
-	} 
-    }
-}
-
-void erts_sweep_monitors(ErtsMonitor *root, 
-			 void (*doit)(ErtsMonitor *, void *),
-			 void *context) 
-{
-    ErtsMonitor *tstack[STACK_NEED];
-    int tpos = 0;
-    int dstack[STACK_NEED+1];
-    int dpos = 1;
-    int dir;
-    
-    dstack[0] = DIR_END;
-
-    for (;;) {
-	if (root == NULL) {
-	    if ((dir = dstack[dpos-1]) == DIR_END) {
-		return;
-	    }
-	    if (dir == DIR_LEFT) {
-		/* Still has DIR_RIGHT to do */
-		dstack[dpos-1] = DIR_RIGHT;
-		root = (tstack[tpos-1])->right;
-	    } else {
-		/* stacktop is an object to be deleted */
-		(*doit)(tstack[--tpos],context); /* expeted to do the 
-						    deletion */
-		--dpos;
-		root = NULL;
-	    }
-	} else {
-	    dstack[dpos++] = DIR_LEFT;
-	    tstack[tpos++] = root;
-	    root = root->left;
-	}
-    }
-}
-
-void erts_sweep_links(ErtsLink *root, 
-		      void (*doit)(ErtsLink *, void *),
-		      void *context) 
-{
-    ErtsLink *tstack[STACK_NEED];
-    int tpos = 0;
-    int dstack[STACK_NEED+1];
-    int dpos = 1;
-    int dir;
-    
-    dstack[0] = DIR_END;
-
-    for (;;) {
-	if (root == NULL) {
-	    if ((dir = dstack[dpos-1]) == DIR_END) {
-		return;
-	    }
-	    if (dir == DIR_LEFT) {
-		/* Still has DIR_RIGHT to do */
-		dstack[dpos-1] = DIR_RIGHT;
-		root = (tstack[tpos-1])->right;
-	    } else {
-		/* stacktop is an object to be deleted */
-		(*doit)(tstack[--tpos],context); /* expeted to do the 
-						    deletion */
-		--dpos;
-		root = NULL;
-	    }
-	} else {
-	    dstack[dpos++] = DIR_LEFT;
-	    tstack[tpos++] = root;
-	    root = root->left;
-	}
-    }
-}
-
-void erts_sweep_suspend_monitors(ErtsSuspendMonitor *root,
-				 void (*doit)(ErtsSuspendMonitor *, void *),
-				 void *context)
-{
-    ErtsSuspendMonitor *tstack[STACK_NEED];
-    int tpos = 0;
-    int dstack[STACK_NEED+1];
-    int dpos = 1;
-    int dir;
-    
-    dstack[0] = DIR_END;
-
-    for (;;) {
-	if (root == NULL) {
-	    if ((dir = dstack[dpos-1]) == DIR_END) {
-		return;
-	    }
-	    if (dir == DIR_LEFT) {
-		/* Still has DIR_RIGHT to do */
-		dstack[dpos-1] = DIR_RIGHT;
-		root = (tstack[tpos-1])->right;
-	    } else {
-		/* stacktop is an object to be deleted */
-		(*doit)(tstack[--tpos],context); /* expeted to do the 
-						    deletion */
-		--dpos;
-		root = NULL;
-	    }
-	} else {
-	    dstack[dpos++] = DIR_LEFT;
-	    tstack[tpos++] = root;
-	    root = root->left;
-	}
-    }
-}
-				
-
-/* Debug BIF, always present, but undocumented... */
-	    
-static void erts_dump_monitors(ErtsMonitor *root, int indent)
-{
-    if (root == NULL)
-	return;
-    erts_dump_monitors(root->right,indent+2);
-    erts_printf("%*s[%b16d:%b16u:%T:%T", indent, "", root->balance,
-		root->type, root->ref, root->name);
-    if (root->type == MON_NIF_TARGET)
-	erts_printf(":%p]\n", root->u.resource);
-    else
-	erts_printf(":%T]\n", root->u.pid);
-    erts_dump_monitors(root->left,indent+2);
-}
-
-static void erts_dump_links_aux(ErtsLink *root, int indent,
-				erts_dsprintf_buf_t *dsbufp)
-{
-    if (root == NULL)
-	return;
-    erts_dump_links_aux(root->right, indent+2, dsbufp);
-    dsbufp->str_len = 0;
-    erts_dsprintf(dsbufp, "%*s[%b16d:%b16u:%T:%p]", indent, "",
-		  root->balance, root->type, root->pid, ERTS_LINK_ROOT(root));
-    if (ERTS_LINK_ROOT(root) != NULL) {
-	ErtsLink *sub = ERTS_LINK_ROOT(root);
-	int len = dsbufp->str_len;
-	erts_dump_links_aux(sub->right, indent+len+5, dsbufp);
-	erts_dsprintf(dsbufp, "-> %*s[%b16d:%b16u:%T:%p]", indent, "",
-		      sub->balance, sub->type, sub->pid, ERTS_LINK_ROOT(sub));
-	erts_printf("%s\n", dsbufp->str);
-	erts_dump_links_aux(sub->left, indent+len+5, dsbufp);
-    } else {
-	erts_printf("%s\n", dsbufp->str);
-    }
-    erts_dump_links_aux(root->left, indent+2, dsbufp);
-}
-
-static void erts_dump_links(ErtsLink *root, int indent)
-{
-    erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0);
-    erts_dump_links_aux(root, indent, dsbufp);
-    erts_destroy_tmp_dsbuf(dsbufp);
-}
-
-Eterm erts_debug_dump_monitors_1(BIF_ALIST_1)
-{
-    Process *p = BIF_P;
-    Eterm pid = BIF_ARG_1;
-    Process *rp;
-    DistEntry *dep;
-    rp = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, pid, ERTS_PROC_LOCK_LINK);
-    if (!rp) {
-	ERTS_ASSERT_IS_NOT_EXITING(p);
-	if (is_atom(pid) && is_node_name_atom(pid) &&
-	    (dep = erts_find_dist_entry(pid)) != NULL) {
-	    erts_printf("Dumping dist monitors-------------------\n");
-	    erts_de_links_lock(dep);
-	    erts_dump_monitors(dep->monitors,0);
-	    erts_de_links_unlock(dep);
-	    erts_printf("Monitors dumped-------------------------\n");
-	    BIF_RET(am_true);
-	} else {
-	    BIF_ERROR(p,BADARG);
-	}
-    } else {
-	erts_printf("Dumping pid monitors--------------------\n");
-	erts_dump_monitors(ERTS_P_MONITORS(rp),0);
-	erts_printf("Monitors dumped-------------------------\n");
-	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-	BIF_RET(am_true);
-    }
-}
-
-Eterm erts_debug_dump_links_1(BIF_ALIST_1)
-{
-    Process *p = BIF_P;
-    Eterm pid = BIF_ARG_1;
-    Process *rp;
-    DistEntry *dep;
-    if (is_internal_port(pid)) {
-	Port *rport = erts_id2port_sflgs(pid,
-					 p,
-					 ERTS_PROC_LOCK_MAIN,
-					 ERTS_PORT_SFLGS_INVALID_LOOKUP);
-	if (rport) {
-	    erts_printf("Dumping port links----------------------\n");
-	    erts_dump_links(ERTS_P_LINKS(rport), 0);
-	    erts_printf("Links dumped----------------------------\n");
-	    erts_port_release(rport);
-	    BIF_RET(am_true);
-	} else {
-	    BIF_ERROR(p,BADARG);
-	}
-    } else {
-	rp = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, pid, ERTS_PROC_LOCK_LINK);
-	if (!rp) {
-	    ERTS_ASSERT_IS_NOT_EXITING(p);
-	    if (is_atom(pid) && is_node_name_atom(pid) &&
-		(dep = erts_find_dist_entry(pid)) != NULL) {
-		erts_printf("Dumping dist links----------------------\n");
-		erts_de_links_lock(dep);
-		erts_dump_links(dep->nlinks,0);
-		erts_de_links_unlock(dep);
-		erts_printf("Links dumped----------------------------\n");
-		BIF_RET(am_true);
-	    } else {
-		BIF_ERROR(p,BADARG);
-	    }
-
-	} else {
-	    erts_printf("Dumping pid links-----------------------\n");
-	    erts_dump_links(ERTS_P_LINKS(rp), 0);
-	    erts_printf("Links dumped----------------------------\n");
-	    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-	    BIF_RET(am_true);
-	}
-    }
-}
-
-void erts_one_link_size(ErtsLink *lnk, void *vpu)
-{
-    Uint *pu = vpu;
-    *pu += ERTS_LINK_SIZE*sizeof(Uint);
-    if(is_not_immed(lnk->pid))
-	*pu += NC_HEAP_SIZE(lnk->pid)*sizeof(Uint);
-    if (lnk->type != LINK_NODE && ERTS_LINK_ROOT(lnk) != NULL) {
-	erts_doforall_links(ERTS_LINK_ROOT(lnk),&erts_one_link_size,vpu);
-    }
-}
-void erts_one_mon_size(ErtsMonitor *mon, void *vpu)
-{
-    Uint *pu = vpu;
-    *pu += ERTS_MONITOR_SIZE*sizeof(Uint);
-    if(mon->type != MON_NIF_TARGET && is_not_immed(mon->u.pid))
-	*pu += NC_HEAP_SIZE(mon->u.pid)*sizeof(Uint);
-    if(is_not_immed(mon->ref))
-	*pu += NC_HEAP_SIZE(mon->ref)*sizeof(Uint);
-}
diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h
deleted file mode 100644
index 1cacecd7e9..0000000000
--- a/erts/emulator/beam/erl_monitors.h
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2017. All Rights Reserved.
- * 
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * 
- * %CopyrightEnd%
- */
-
-/**********************************************************************
- * Header for monitors and links data structures.
- * Monitors are kept in an AVL tree and the data structures for
- * the four different types of monitors are like this:
- **********************************************************************
- * Local monitor by pid/port: 
- * (Ref is always same in all involved data structures)
- **********************************************************************
- *        Process/Port X     Process Y
- *       +-------------+    +-------------+
- * Type: | MON_ORIGIN  |    | MON_TARGET  |
- *       +-------------+    +-------------+
- * Pid:  | Pid(Y)      |    | Pid/Port(X) |
- *       +-------------+    +-------------+
- * Name: | []          |    | []          |
- *       +-------------+    +-------------+
- **********************************************************************
- * Local monitor by name: (Ref is always same in all involved data structures)
- **********************************************************************
- *        Process X          Process Y (name foo)
- *       +-------------+    +-------------+
- * Type: | MON_ORIGIN  |    | MON_TARGET  |
- *       +-------------+    +-------------+
- * Pid:  | Pid(Y)      |    | Pid(X)      |
- *       +-------------+    +-------------+
- * Name: | Atom(foo)   |    | Atom(foo)   |
- *       +-------------+    +-------------+
- **********************************************************************
- * Remote monitor by pid: (Ref is always same in all involved data structures)
- **********************************************************************
- *                    Node A              |           Node B
- *       ---------------------------------+----------------------------------
- *        Process X (@A)   Distentry @A       Distentry @B     Process Y (@B)
- *                         for node B         for node A 
- *       +-------------+  +-------------+    +-------------+  +-------------+
- * Type: | MON_ORIGIN  |  | MON_TARGET  |    | MON_ORIGIN  |  | MON_TARGET  |
- *       +-------------+  +-------------+    +-------------+  +-------------+
- * Pid:  | Pid(Y)      |  | Pid(X)      |    | Pid(Y)      |  | Pid(X)      |
- *       +-------------+  +-------------+    +-------------+  +-------------+
- * Name: | []          |  | []          |    | []          |  | []          |
- *       +-------------+  +-------------+    +-------------+  +-------------+
- **********************************************************************
- * Remote monitor by name: (Ref is always same in all involved data structures)
- **********************************************************************
- *                    Node A              |           Node B
- *       ---------------------------------+----------------------------------
- *        Process X (@A)   Distentry @A       Distentry @B     Process Y (@B)
- *                         for node B         for node A       (name foo)
- *       +-------------+  +-------------+    +-------------+  +-------------+
- * Type: | MON_ORIGIN  |  | MON_TARGET  |    | MON_ORIGIN  |  | MON_TARGET  |
- *       +-------------+  +-------------+    +-------------+  +-------------+
- * Pid:  | Atom(node B)|  | Pid(X)      |    | Pid(Y)      |  | Pid(X)      |
- *       +-------------+  +-------------+    +-------------+  +-------------+
- * Name: | Atom(foo)   |  | Atom(foo)   |    | Atom(foo)   |  | Atom(foo)   |
- *       +-------------+  +-------------+    +-------------+  +-------------+
- * The reason for the node atom in X->pid is that we don't know the actual
- * pid of the monitored process on the other node when setting the monitor
- * (which is done asyncronously).
- **********************************************************************/
-#ifndef _ERL_MONITORS_H
-#define _ERL_MONITORS_H
-
-/* Type tags for monitors */
-#define MON_ORIGIN 1
-#define MON_TARGET 2
-#define MON_NIF_TARGET 3
-#define MON_TIME_OFFSET 4
-
-/* Type tags for links */
-#define LINK_PID 1   /* ...Or port */
-#define LINK_NODE 3  /* "Node monitor" */  
-
-/* Size of a monitor without heap, in words (fixalloc) */
-#define ERTS_MONITOR_SIZE ((sizeof(ErtsMonitor) - sizeof(Uint))/sizeof(Uint))
-#define ERTS_MONITOR_SH_SIZE (ERTS_MONITOR_SIZE + ERTS_REF_THING_SIZE)
-#define ERTS_LINK_SIZE ((sizeof(ErtsLink) - sizeof(Uint))/sizeof(Uint))
-#define ERTS_LINK_SH_SIZE ERTS_LINK_SIZE /* Size of fix-alloced links */
-
-/* ErtsMonitor and ErtsLink *need* to begin in a similar way as 
-   ErtsMonitorOrLink */
-typedef struct erts_monitor_or_link {
-    struct erts_monitor_or_link *left, *right;
-    Sint16 balance;    
-} ErtsMonitorOrLink;
-
-typedef struct erts_monitor {
-    struct erts_monitor *left, *right; 
-    Sint16 balance;
-    Uint16 type;  /* MON_ORIGIN | MON_TARGET | MON_NIF_TARGET | MON_TIME_OFFSET */
-    Eterm ref;
-    union {
-        Eterm pid;  /* In case of distributed named monitor, this is the
-		     * nodename atom in MON_ORIGIN process, otherwise a pid or,
-		     * in case of a MON_TARGET, a port
-                     */
-        struct ErtsResource_* resource; /* MON_NIF_TARGET */
-    }u;
-    Eterm name;   /* When monitoring a named process: atom() else [] */
-    Uint heap[1]; /* Larger in reality */
-} ErtsMonitor;
-
-typedef struct erts_link {
-    struct erts_link *left, *right;
-    Sint16 balance;
-    Uint16 type;             /* LINK_PID | LINK_NODE */
-    Eterm pid;               /* When node monitor, 
-				the node atom is here instead */
-    union {
-	struct erts_link *root;  /* Used only in dist entries */
-	Uint refc;
-    } shared;
-    Uint heap[1];            /* Larger in reality */
-} ErtsLink;   
-
-typedef struct erts_suspend_monitor {
-    struct erts_suspend_monitor *left, *right;
-    Sint16 balance;
-
-    int pending;
-    int active;
-    Eterm pid;
-} ErtsSuspendMonitor;
-
-#define ERTS_LINK_ROOT(Linkp) ((Linkp)->shared.root)
-#define ERTS_LINK_REFC(Linkp) ((Linkp)->shared.refc) 
-
-Uint erts_tot_link_lh_size(void);
-
-
-/* Prototypes */
-void erts_destroy_monitor(ErtsMonitor *mon);
-void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, UWord entity,
-		      Eterm name);
-ErtsMonitor *erts_remove_monitor(ErtsMonitor **root, Eterm ref);
-ErtsMonitor *erts_lookup_monitor(ErtsMonitor *root, Eterm ref);
-void erts_sweep_monitors(ErtsMonitor *root, 
-			 void (*doit)(ErtsMonitor *, void *),
-			 void *context);
-
-void erts_destroy_link(ErtsLink *lnk);
-/* Returns 0 if OK, < 0 if already present */
-int erts_add_link(ErtsLink **root, Uint type, Eterm pid);
-ErtsLink *erts_add_or_lookup_link(ErtsLink **root, Uint type, Eterm pid);
-ErtsLink *erts_remove_link(ErtsLink **root, Eterm pid);
-ErtsLink *erts_lookup_link(ErtsLink *root, Eterm pid);
-void erts_sweep_links(ErtsLink *root, 
-		      void (*doit)(ErtsLink *, void *),
-		      void *context);
-
-void erts_destroy_suspend_monitor(ErtsSuspendMonitor *sproc);
-void erts_sweep_suspend_monitors(ErtsSuspendMonitor *root,
-				 void (*doit)(ErtsSuspendMonitor *, void *),
-				 void *context);
-ErtsSuspendMonitor *erts_add_or_lookup_suspend_monitor(ErtsSuspendMonitor **root,
-						       Eterm pid);
-ErtsSuspendMonitor *erts_lookup_suspend_monitor(ErtsSuspendMonitor *root,
-						Eterm pid);
-void erts_delete_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid);
-void erts_init_monitors(void);
-void erts_one_link_size(ErtsLink *lnk, void *vpu);
-void erts_one_mon_size(ErtsMonitor *mon, void *vpu);
-
-#define erts_doforall_monitors erts_sweep_monitors
-#define erts_doforall_links erts_sweep_links
-#define erts_doforall_suspend_monitors erts_sweep_suspend_monitors
-
-#endif /* _ERL_MONITORS_H */
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index c60cc7fecf..cd4fd7b635 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -57,6 +57,7 @@
 #include "erl_bif_unique.h"
 #include "erl_utils.h"
 #include "erl_io_queue.h"
+#include "erl_proc_sig_queue.h"
 #undef ERTS_WANT_NFUNC_SCHED_INTERNALS__
 #define ERTS_WANT_NFUNC_SCHED_INTERNALS__
 #include "erl_nfunc_sched.h"
@@ -658,7 +659,7 @@ int erts_flush_trace_messages(Process *c_p, ErtsProcLocks c_p_locks)
             rp_locks = 0;
             if (rp->common.id == c_p->common.id)
                 rp_locks = c_p_locks;
-            erts_queue_messages(rp, rp_locks, first, last, len, c_p->common.id);
+            erts_queue_messages(rp, rp_locks, first, last, len);
             if (rp->common.id == c_p->common.id)
                 rp_locks &= ~c_p_locks;
             if (rp_locks)
@@ -700,6 +701,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
     Process* rp;
     Process* c_p;
     ErtsMessage *mp;
+    Eterm from;
     Eterm receiver = to_pid->pid;
     int scheduler;
 
@@ -778,7 +780,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
         msg = copy_struct_litopt(msg, sz, &hp, ohp, &litarea);
     }
 
-    ERL_MESSAGE_TERM(mp) = msg;
+    from = c_p ? c_p->common.id : am_undefined;
 
     if (!env || !env->tracee) {
 
@@ -797,7 +799,6 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
         ErlTraceMessageQueue *msgq;
         Process *t_p = env->tracee;
 
-
         erts_proc_lock(t_p, ERTS_PROC_LOCK_TRACE);
 
         msgq = t_p->trace_msg_q;
@@ -817,6 +818,9 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
             rp_locks & ERTS_PROC_LOCK_MSGQ ||
             erts_proc_trylock(rp, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
 
+            ERL_MESSAGE_TERM(mp) = msg;
+            ERL_MESSAGE_FROM(mp) = from;
+
             if (!msgq) {
                 msgq = erts_alloc(ERTS_ALC_T_TRACE_MSG_QUEUE,
                                   sizeof(ErlTraceMessageQueue));
@@ -846,8 +850,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
         }
     }
 
-    erts_queue_message(rp, rp_locks, mp, msg,
-                       c_p ? c_p->common.id : am_undefined);
+    erts_queue_message(rp, rp_locks, mp, msg, from);
 
 done:
     if (c_p == rp)
@@ -2206,71 +2209,61 @@ static void rollback_opened_resource_types(void)
     }
 }
 
-struct destroy_monitor_ctx
-{
-    ErtsResource* resource;
-    int exiting_procs;
-    int scheduler;
-};
+#ifdef ARCH_64
+#  define ERTS_RESOURCE_DYING_FLAG (((Uint) 1) << 63)
+#else
+#  define ERTS_RESOURCE_DYING_FLAG (((Uint) 1) << 31)
+#endif
+#define ERTS_RESOURCE_REFC_MASK (~ERTS_RESOURCE_DYING_FLAG)
 
-static void destroy_one_monitor(ErtsMonitor* mon, void* context)
+static ERTS_INLINE void
+rmon_set_dying(ErtsResourceMonitors *rms)
 {
-    struct destroy_monitor_ctx* ctx = (struct destroy_monitor_ctx*) context;
-    Process* rp;
-    ErtsMonitor *rmon = NULL;
-    int is_exiting;
+    rms->refc |= ERTS_RESOURCE_DYING_FLAG;
+}
 
-    ASSERT(mon->type == MON_ORIGIN);
-    ASSERT(is_internal_pid(mon->u.pid));
-    ASSERT(is_internal_ref(mon->ref));
+static ERTS_INLINE int
+rmon_is_dying(ErtsResourceMonitors *rms)
+{
+    return !!(rms->refc & ERTS_RESOURCE_DYING_FLAG);
+}
 
-    if (ctx->scheduler > 0) { /* Normal scheduler */
-        rp = erts_proc_lookup(mon->u.pid);
-    }
-    else {
-        rp = erts_proc_lookup_inc_refc(mon->u.pid);
-    }
+static ERTS_INLINE void
+rmon_refc_inc(ErtsResourceMonitors *rms)
+{
+    rms->refc++;
+}
 
-    if (!rp) {
-        is_exiting = 1;
-    }
-    if (rp) {
-        erts_proc_lock(rp, ERTS_PROC_LOCK_LINK);
-        if (ERTS_PROC_IS_EXITING(rp)) {
-            is_exiting = 1;
-        } else {
-            rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref);
-            ASSERT(rmon);
-            is_exiting = 0;
-        }
-        erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-        if (ctx->scheduler <= 0)
-            erts_proc_dec_refc(rp);
-    }
-    if (is_exiting) {
-        ctx->resource->monitors->pending_failed_fire++;
-    }
+static ERTS_INLINE Uint
+rmon_refc_dec_read(ErtsResourceMonitors *rms)
+{
+    Uint res;
+    ASSERT((rms->refc & ERTS_RESOURCE_REFC_MASK) != 0);
+    res = --rms->refc;
+    return res & ERTS_RESOURCE_REFC_MASK;
+}
 
-    /* ToDo: Delay destruction after monitor_locks */
-    if (rmon) {
-        ASSERT(rmon->type == MON_NIF_TARGET);
-        ASSERT(rmon->u.resource == ctx->resource);
-        erts_destroy_monitor(rmon);
-    }
-    erts_destroy_monitor(mon);
+static ERTS_INLINE void
+rmon_refc_dec(ErtsResourceMonitors *rms)
+{
+    ASSERT((rms->refc & ERTS_RESOURCE_REFC_MASK) != 0);
+    --rms->refc;
 }
 
-static void destroy_all_monitors(ErtsMonitor* monitors, ErtsResource* resource)
+static ERTS_INLINE Uint
+rmon_refc_read(ErtsResourceMonitors *rms)
 {
-    struct destroy_monitor_ctx ctx;
+    return rms->refc & ERTS_RESOURCE_REFC_MASK;
+}
 
-    execution_state(NULL, NULL, &ctx.scheduler);
+static void dtor_demonitor(ErtsMonitor* mon, void* context)
+{
+    ASSERT(erts_monitor_is_origin(mon));
+    ASSERT(is_internal_pid(mon->other.item));
 
-    ctx.resource = resource;
-    erts_sweep_monitors(monitors, &destroy_one_monitor, &ctx);
+    erts_proc_sig_send_demonitor(mon);
 }
 
-
 #  define NIF_RESOURCE_DTOR &nif_resource_dtor
 
 static int nif_resource_dtor(Binary* bin)
@@ -2281,34 +2274,36 @@ static int nif_resource_dtor(Binary* bin)
 
     if (resource->monitors) {
         ErtsResourceMonitors* rm = resource->monitors;
+        int kill;
+        ErtsMonitor *root;
+        Uint refc;
 
         ASSERT(type->down);
         erts_mtx_lock(&rm->lock);
         ASSERT(erts_refc_read(&bin->intern.refc, 0) == 0);
-        if (rm->root) {
-            ASSERT(!rm->is_dying);
-            destroy_all_monitors(rm->root, resource);
+        kill = !rmon_is_dying(rm);
+        if (kill) {
+            rmon_set_dying(rm);
+            root = rm->root;
             rm->root = NULL;
         }
-        if (rm->pending_failed_fire) {
-            /*
-             * Resource death struggle prolonged to serve exiting process(es).
-             * Destructor will be called again when last exiting process
-             * tries to fire its MON_NIF_TARGET monitor (and fails).
-             *
-             * This resource is doomed. It has no "real" references and
-             * should get not get called upon to do anything except the
-             * final destructor call.
-             *
-             * We keep refc at 0 and use a separate counter for exiting
-             * processes to avoid resource getting revived by "dec_term".
-             */
-            ASSERT(!rm->is_dying);
-            rm->is_dying = 1;
-            erts_mtx_unlock(&rm->lock);
-            return 0;
-        }
+        refc = rmon_refc_read(rm);
         erts_mtx_unlock(&rm->lock);
+
+        if (kill)
+            erts_monitor_tree_foreach_delete(&root,
+                                             dtor_demonitor,
+                                             NULL);
+
+        /*
+         * If resource->monitors->refc != 0 there are
+         * outstanding references to the resource from
+         * monitors that has not been removed yet.
+         * nif_resource_dtor() will be called again this
+         * reference count reach zero.
+         */
+        if (refc != 0)
+            return 0; /* we'll be back... */
         erts_mtx_destroy(&rm->lock);
     }
 
@@ -2338,54 +2333,82 @@ void erts_resource_stop(ErtsResource* resource, ErlNifEvent e,
     post_nif_noproc(&msg_env);
 }
 
-void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref)
+void erts_nif_demonitored(ErtsResource* resource)
 {
-    ErtsMonitor* rmon;
+    ErtsResourceMonitors* rmp = resource->monitors;
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
+    int free_me;
+
+    ASSERT(rmp);
+    ASSERT(resource->type->down);
+
+    erts_mtx_lock(&rmp->lock);
+    free_me = ((rmon_refc_dec_read(rmp) == 0) & !!rmon_is_dying(rmp));
+    erts_mtx_unlock(&rmp->lock);
+
+    if (free_me)
+        erts_bin_free(&bin->binary);
+}
+
+void erts_fire_nif_monitor(ErtsMonitor *tmon)
+{
+    ErtsResource* resource;
+    ErtsMonitorData *mdp;
+    ErtsMonitor *omon;
+    ErtsBinary* bin;
     struct enif_msg_environment_t msg_env;
     ErlNifPid nif_pid;
     ErlNifMonitor nif_monitor;
-    ErtsResourceMonitors* rmp = resource->monitors;
+    ErtsResourceMonitors* rmp;
+    Uint mrefc, brefc;
+    int active, is_dying;
+
+    ASSERT(tmon->type == ERTS_MON_TYPE_RESOURCE);
+    ASSERT(erts_monitor_is_target(tmon));
+
+    resource = tmon->other.ptr;
+    bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
+    rmp = resource->monitors;
+
+    mdp = erts_monitor_to_data(tmon);
+    omon = &mdp->origin;
 
     ASSERT(rmp);
     ASSERT(resource->type->down);
 
     erts_mtx_lock(&rmp->lock);
-    rmon = erts_remove_monitor(&rmp->root, ref);
-    if (!rmon) {
-        int free_me = (--rmp->pending_failed_fire == 0) && rmp->is_dying;
-        ASSERT(rmp->pending_failed_fire >= 0);
-        erts_mtx_unlock(&rmp->lock);
-
-        if (free_me) {
-            ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) == 0);
-            erts_bin_free(&bin->binary);
-        }
-        return;
+
+    mrefc = rmon_refc_dec_read(rmp);
+    is_dying = rmon_is_dying(rmp);
+    active = !is_dying && erts_monitor_is_in_table(omon);
+
+    if (active) {
+        erts_monitor_tree_delete(&rmp->root, omon);
+        brefc = (Uint) erts_refc_inc_unless(&bin->binary.intern.refc, 0, 0);
     }
-    ASSERT(!rmp->is_dying);
-    if (erts_refc_inc_unless(&bin->binary.intern.refc, 0, 0) == 0) {
-        /*
-         * Racing resource destruction. 
-         * To avoid a more complex refc-dance with destructing thread
-         * we avoid calling 'down' and just silently remove the monitor.
-         * This can happen even for non smp as destructor calls may be scheduled.
-         */
-        erts_mtx_unlock(&rmp->lock);
+
+    erts_mtx_unlock(&rmp->lock);
+
+    if (!active) {
+        ASSERT(!is_dying || erts_refc_read(&bin->binary.intern.refc, 0) == 0);
+        if (is_dying && mrefc == 0)
+            erts_bin_free(&bin->binary);
+        erts_monitor_release(tmon);
     }
     else {
-        erts_mtx_unlock(&rmp->lock);
-
-        ASSERT(rmon->u.pid == pid);
-        erts_ref_to_driver_monitor(ref, &nif_monitor);
-        nif_pid.pid = pid;
-        pre_nif_noproc(&msg_env, resource->type->owner, NULL);
-        resource->type->down(&msg_env.env, resource->data, &nif_pid, &nif_monitor);
-        post_nif_noproc(&msg_env);
+        if (brefc > 0) {
+            ASSERT(is_internal_pid(omon->other.item));
+            erts_ref_to_driver_monitor(mdp->ref, &nif_monitor);
+            nif_pid.pid = omon->other.item;
+            pre_nif_noproc(&msg_env, resource->type->owner, NULL);
+            resource->type->down(&msg_env.env, resource->data, &nif_pid, &nif_monitor);
+            post_nif_noproc(&msg_env);
+
+            erts_bin_release(&bin->binary);
+        }
 
-        erts_bin_release(&bin->binary);
+        erts_monitor_release_both(mdp);
     }
-    erts_destroy_monitor(rmon);
 }
 
 void* enif_alloc_resource(ErlNifResourceType* type, size_t data_sz)
@@ -2422,8 +2445,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t data_sz)
         erts_mtx_init(&resource->monitors->lock, "resource_monitors", NIL,
             ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
         resource->monitors->root = NULL;
-        resource->monitors->pending_failed_fire = 0;
-        resource->monitors->is_dying = 0;
+        resource->monitors->refc = 0;
         resource->monitors->user_data_sz = data_sz;
     }
     else {
@@ -2438,7 +2460,7 @@ void enif_release_resource(void* obj)
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
 
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == NIF_RESOURCE_DTOR);
-    ASSERT(!(resource->monitors && resource->monitors->is_dying));
+    ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) != 0);
 #ifdef DEBUG
     erts_refc_dec(&resource->nif_refc, 0);
 #endif
@@ -2451,7 +2473,7 @@ void enif_keep_resource(void* obj)
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
 
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == NIF_RESOURCE_DTOR);
-    ASSERT(!(resource->monitors && resource->monitors->is_dying));
+    ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) != 0);
 #ifdef DEBUG
     erts_refc_inc(&resource->nif_refc, 1);
 #endif
@@ -2461,7 +2483,7 @@ void enif_keep_resource(void* obj)
 Eterm erts_bld_resource_ref(Eterm** hpp, ErlOffHeap* oh, ErtsResource* resource)
 {
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
-    ASSERT(!(resource->monitors && resource->monitors->is_dying));
+    ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) != 0);
     return erts_mk_magic_ref(hpp, oh, &bin->binary);
 }
 
@@ -2470,7 +2492,7 @@ ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj)
     ErtsResource* resource = DATA_TO_RESOURCE(obj);
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource);
     Eterm* hp = alloc_heap(env, ERTS_MAGIC_REF_THING_SIZE);
-    ASSERT(!(resource->monitors && resource->monitors->is_dying));
+    ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) != 0);
     return erts_mk_magic_ref(&hp, &MSO(env->proc), &bin->binary);
 }
 
@@ -3150,123 +3172,88 @@ int enif_map_iterator_get_pair(ErlNifEnv *env,
 int enif_monitor_process(ErlNifEnv* env, void* obj, const ErlNifPid* target_pid,
                          ErlNifMonitor* monitor)
 {
-    int scheduler;
     ErtsResource* rsrc = DATA_TO_RESOURCE(obj);
-    Process *rp;
     Eterm tmp[ERTS_REF_THING_SIZE];
     Eterm ref;
-    int retval;
+    ErtsResourceMonitors *rm;
+    ErtsMonitorData *mdp;
 
     ASSERT(ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc)->magic_binary.destructor
            == NIF_RESOURCE_DTOR);
-    ASSERT(!(rsrc->monitors && rsrc->monitors->is_dying));
+    ASSERT(erts_refc_read(&ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc)->binary.intern.refc, 0) != 0);
     ASSERT(!rsrc->monitors == !rsrc->type->down);
 
-
-    if (!rsrc->monitors) {
+    rm = rsrc->monitors;
+    if (!rm) {
         ASSERT(!rsrc->type->down);
         return -1;
     }
     ASSERT(rsrc->type->down);
 
-    execution_state(env, NULL, &scheduler);
+    ref = erts_make_ref_in_buffer(tmp);
 
-    if (scheduler > 0) /* Normal scheduler */
-        rp = erts_proc_lookup_raw(target_pid->pid);
-    else
-        rp = erts_proc_lookup_raw_inc_refc(target_pid->pid);
+    mdp = erts_monitor_create(ERTS_MON_TYPE_RESOURCE, ref,
+                              (Eterm) rsrc, target_pid->pid, NIL);
+    erts_mtx_lock(&rm->lock);
+    ASSERT(!rmon_is_dying(rm));
+    erts_monitor_tree_insert(&rm->root, &mdp->origin);
+    rmon_refc_inc(rm);
+    erts_mtx_unlock(&rm->lock);
 
-    if (!rp)
-        return 1;
+    if (!erts_proc_sig_send_monitor(&mdp->target, target_pid->pid)) {
+        /* Failed to send monitor signal; cleanup... */
+#ifdef DEBUG
+        ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc);
+#endif
 
-    ref = erts_make_ref_in_buffer(tmp);
+        erts_mtx_lock(&rm->lock);
+        ASSERT(!rmon_is_dying(rm));
+        erts_monitor_tree_delete(&rm->root, &mdp->origin);
+        rmon_refc_dec(rm);
+        ASSERT(erts_refc_read(&bin->binary.intern.refc, 1) != 0);
+        erts_mtx_unlock(&rm->lock);
+        erts_monitor_release_both(mdp);
 
-    erts_mtx_lock(&rsrc->monitors->lock);
-    erts_proc_lock(rp, ERTS_PROC_LOCK_LINK);
-    if (ERTS_PSFLG_FREE & erts_atomic32_read_nob(&rp->state)) {
-        retval = 1;
-    }
-    else {
-        erts_add_monitor(&rsrc->monitors->root, MON_ORIGIN, ref, rp->common.id, NIL);
-        erts_add_monitor(&ERTS_P_MONITORS(rp), MON_NIF_TARGET, ref, (UWord)rsrc, NIL);
-        retval = 0;
+        return 1;
     }
-    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-    erts_mtx_unlock(&rsrc->monitors->lock);
 
-    if (scheduler <= 0)
-        erts_proc_dec_refc(rp);
     if (monitor)
         erts_ref_to_driver_monitor(ref,monitor);
 
-    return retval;
+    return 0;
 }
 
 int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monitor)
 {
-    int scheduler;
     ErtsResource* rsrc = DATA_TO_RESOURCE(obj);
 #ifdef DEBUG
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(rsrc);
 #endif
-    Process *rp;
+    ErtsResourceMonitors *rm;
     ErtsMonitor *mon;
-    ErtsMonitor *rmon = NULL;
     Eterm ref_heap[ERTS_REF_THING_SIZE];
     Eterm ref;
-    int is_exiting;
 
     ASSERT(bin->magic_binary.destructor == NIF_RESOURCE_DTOR);
-    ASSERT(!(rsrc->monitors && rsrc->monitors->is_dying));
-
-    execution_state(env, NULL, &scheduler);
+    ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) != 0);
 
     ref = erts_driver_monitor_to_ref(ref_heap, monitor);
 
-    erts_mtx_lock(&rsrc->monitors->lock);
-    mon = erts_remove_monitor(&rsrc->monitors->root, ref);
+    rm = rsrc->monitors;
+    erts_mtx_lock(&rm->lock);
+    ASSERT(!rmon_is_dying(rm));
+    mon = erts_monitor_tree_lookup(rm->root, ref);
+    if (mon)
+        erts_monitor_tree_delete(&rm->root, mon);
+    erts_mtx_unlock(&rm->lock);
 
-    if (mon == NULL) {
-        erts_mtx_unlock(&rsrc->monitors->lock);
+    if (!mon)
         return 1;
-    }
-
-    ASSERT(mon->type == MON_ORIGIN);
-    ASSERT(is_internal_pid(mon->u.pid));
-
-    if (scheduler > 0) /* Normal scheduler */
-        rp = erts_proc_lookup(mon->u.pid);
-    else
-        rp = erts_proc_lookup_inc_refc(mon->u.pid);
-
-    if (!rp) {
-        is_exiting = 1;
-    }
-    else {
-        erts_proc_lock(rp, ERTS_PROC_LOCK_LINK);
-        if (ERTS_PROC_IS_EXITING(rp)) {
-            is_exiting = 1;
-        } else {
-            rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref);
-            ASSERT(rmon);
-            is_exiting = 0;
-        }
-        erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 
-        if (scheduler <= 0)
-            erts_proc_dec_refc(rp);
-    }
-    if (is_exiting) {
-        rsrc->monitors->pending_failed_fire++;
-    }
-    erts_mtx_unlock(&rsrc->monitors->lock);
+    ASSERT(erts_monitor_is_origin(mon));
+    ASSERT(is_internal_pid(mon->other.item));
 
-    if (rmon) {
-        ASSERT(rmon->type == MON_NIF_TARGET);
-        ASSERT(rmon->u.resource == rsrc);
-        erts_destroy_monitor(rmon);
-    }
-    erts_destroy_monitor(mon);
+    erts_proc_sig_send_demonitor(mon);
 
     return 0;
 }
diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h
index 6ec428e282..99e938266b 100644
--- a/erts/emulator/beam/erl_node_container_utils.h
+++ b/erts/emulator/beam/erl_node_container_utils.h
@@ -318,5 +318,3 @@ extern ErtsPTab erts_port;
 #define is_not_ref(x)			(!is_ref(x))
 
 #endif
-
-
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index e8901a652f..ca83e70046 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -31,6 +31,7 @@
 #include "dtrace-wrapper.h"
 #include "erl_binary.h"
 #include "erl_bif_unique.h"
+#include "erl_proc_sig_queue.h"
 
 Hash erts_dist_table;
 Hash erts_node_table;
@@ -174,11 +175,7 @@ dist_table_alloc(void *dep_tmpl)
     dep->flags				= 0;
     dep->version			= 0;
 
-    erts_mtx_init(&dep->lnk_mtx, "dist_entry_links", sysname,
-        ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
-    dep->node_links			= NULL;
-    dep->nlinks				= NULL;
-    dep->monitors			= NULL;
+    dep->mld                            = NULL;
 
     erts_mtx_init(&dep->qlock, "dist_entry_out_queue", sysname,
         ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
@@ -225,9 +222,7 @@ dist_table_free(void *vdep)
     ASSERT(de_refc_read(dep, -1) == -1);
     ASSERT(dep->state == ERTS_DE_STATE_IDLE);
     ASSERT(is_nil(dep->cid));
-    ASSERT(dep->nlinks == NULL);
-    ASSERT(dep->node_links == NULL);
-    ASSERT(dep->monitors == NULL);
+    ASSERT(dep->mld == NULL);
 
     /* Link out */
 
@@ -250,7 +245,6 @@ dist_table_free(void *vdep)
 
     ASSERT(!dep->cache);
     erts_rwmtx_destroy(&dep->rwmtx);
-    erts_mtx_destroy(&dep->lnk_mtx);
     erts_mtx_destroy(&dep->qlock);
 
 #ifdef DEBUG
@@ -606,7 +600,9 @@ erts_set_dist_entry_not_connected(DistEntry *dep)
 void
 erts_set_dist_entry_pending(DistEntry *dep)
 {
+    ErtsMonLnkDist *mld = erts_mon_link_dist_create(dep->sysname);
     ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep));
+    
     erts_rwmtx_rwlock(&erts_dist_table_rwmtx);
 
     ASSERT(dep != erts_this_dist_entry);
@@ -631,6 +627,10 @@ erts_set_dist_entry_pending(DistEntry *dep)
     dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY | DFLAG_NO_MAGIC);
     dep->connection_id = (dep->connection_id + 1) & ERTS_DIST_CON_ID_MASK;
 
+    ASSERT(!dep->mld);
+    mld->connection_id = dep->connection_id;
+    dep->mld = mld;
+
     dep->prev = NULL;
     dep->next = erts_pending_dist_entries;
     if(erts_pending_dist_entries) {
@@ -647,6 +647,8 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags)
 {
     erts_aint32_t set_qflgs;
 
+    ASSERT(dep->mld);
+
     ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep));
     erts_rwmtx_rwlock(&erts_dist_table_rwmtx);
 
@@ -1099,6 +1101,7 @@ static Eterm AM_system;
 static Eterm AM_timer;
 static Eterm AM_delayed_delete_timer;
 static Eterm AM_thread_progress_delete_timer;
+static Eterm AM_signal;
 
 static void setup_reference_table(void);
 static Eterm reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp);
@@ -1120,6 +1123,7 @@ typedef struct node_referrer_ {
     int bin_ref;
     int timer_ref;
     int system_ref;
+    int signal_ref;
     Eterm id;
     Uint id_heap[ID_HEAP_SIZE];
     ErlOffHeap off_heap;
@@ -1137,6 +1141,7 @@ typedef struct dist_referrer_ {
     int node_ref;
     int ctrl_ref;
     int system_ref;
+    int signal_ref;
     Eterm id;
     Uint creation;
     Uint id_heap[ID_HEAP_SIZE];
@@ -1190,6 +1195,7 @@ erts_get_node_and_dist_references(struct process *proc)
 	INIT_AM(system);
 	INIT_AM(delayed_delete_timer);
         INIT_AM(thread_progress_delete_timer);
+	INIT_AM(signal);
 	references_atoms_need_init = 0;
     }
 
@@ -1226,6 +1232,7 @@ erts_get_node_and_dist_references(struct process *proc)
 #define MONITOR_REF 7
 #define TIMER_REF 8
 #define SYSTEM_REF 9
+#define SIGNAL_REF 10
 
 #define INC_TAB_SZ 10
 
@@ -1260,6 +1267,7 @@ insert_dist_referrer(ReferredDist *referred_dist,
 	drp->node_ref = 0;
 	drp->ctrl_ref = 0;
 	drp->system_ref = 0;
+        drp->signal_ref = 0;
     }
 
     switch (type) {
@@ -1268,6 +1276,7 @@ insert_dist_referrer(ReferredDist *referred_dist,
     case HEAP_REF:	drp->heap_ref++;	break;
     case ETS_REF:	drp->ets_ref++;	        break;
     case SYSTEM_REF:	drp->system_ref++;	break;
+    case SIGNAL_REF:	drp->signal_ref++;	break;
     default:		ASSERT(0);
     }
 }
@@ -1321,6 +1330,7 @@ insert_node_referrer(ReferredNode *referred_node, int type, Eterm id)
 	nrp->bin_ref = 0;
 	nrp->timer_ref = 0;
 	nrp->system_ref = 0;
+	nrp->signal_ref = 0;
     }
 
     switch (type) {
@@ -1331,6 +1341,7 @@ insert_node_referrer(ReferredNode *referred_node, int type, Eterm id)
     case MONITOR_REF:   nrp->monitor_ref++;     break;
     case TIMER_REF:	nrp->timer_ref++;	break;
     case SYSTEM_REF:	nrp->system_ref++;	break;
+    case SIGNAL_REF:	nrp->signal_ref++;	break;
     default:		ASSERT(0);
     }
 }
@@ -1438,49 +1449,145 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id)
     }
 }
 
-static void doit_insert_monitor(ErtsMonitor *monitor, void *p)
+static void insert_monitor_data(ErtsMonitor *mon, int type, Eterm id)
+{
+    ErtsMonitorData *mdp = erts_monitor_to_data(mon);
+    if ((mdp->origin.flags & (ERTS_ML_FLG_DBG_VISITED
+                              | ERTS_ML_FLG_EXTENDED)) == ERTS_ML_FLG_EXTENDED) {
+        if (mon->type != ERTS_MON_TYPE_NODE) {
+            ErtsMonitorDataExtended *mdep = (ErtsMonitorDataExtended *) mdp;
+            ASSERT(mon->flags & ERTS_ML_FLG_EXTENDED);
+            if (mdep->uptr.ohhp) {
+                ErlOffHeap oh;
+                ERTS_INIT_OFF_HEAP(&oh);
+                oh.first = mdep->uptr.ohhp;
+                insert_offheap(&oh, type, id);
+            }
+        }
+    }
+    mdp->origin.flags |= ERTS_ML_FLG_DBG_VISITED;
+}
+
+static void insert_monitor(ErtsMonitor *mon, void *idp)
+{
+    Eterm id = *((Eterm *) idp);
+    insert_monitor_data(mon, MONITOR_REF, id);
+}
+
+static void clear_visited_monitor(ErtsMonitor *mon, void *p)
+{
+    ErtsMonitorData *mdp = erts_monitor_to_data(mon);
+    mdp->origin.flags &= ~ERTS_ML_FLG_DBG_VISITED;
+}
+
+static void
+insert_p_monitors(ErtsPTabElementCommon *p)
+{
+    Eterm id = p->id;
+    erts_monitor_tree_foreach(p->u.alive.monitors,
+                              insert_monitor,
+                              (void *) &id);
+    erts_monitor_list_foreach(p->u.alive.lt_monitors,
+                              insert_monitor,
+                              (void *) &id);
+}
+
+static void
+insert_dist_monitors(DistEntry *dep)
+{
+    if (dep->mld) {
+        erts_monitor_list_foreach(dep->mld->monitors,
+                                  insert_monitor,
+                                  (void *) &dep->sysname);
+        erts_monitor_tree_foreach(dep->mld->orig_name_monitors,
+                                  insert_monitor,
+                                  (void *) &dep->sysname);
+    }
+}
+
+static void
+clear_visited_p_monitors(ErtsPTabElementCommon *p)
+{
+    erts_monitor_tree_foreach(p->u.alive.monitors,
+                              clear_visited_monitor,
+                              NULL);
+    erts_monitor_list_foreach(p->u.alive.lt_monitors,
+                              clear_visited_monitor,
+                              NULL);
+}
+
+static void
+clear_visited_dist_monitors(DistEntry *dep)
+{
+    if (dep->mld) {
+        erts_monitor_list_foreach(dep->mld->monitors,
+                                  clear_visited_monitor,
+                                  NULL);
+        erts_monitor_tree_foreach(dep->mld->orig_name_monitors,
+                                  clear_visited_monitor,
+                                  NULL);
+    }
+}
+
+static void insert_link_data(ErtsLink *lnk, int type, Eterm id)
 {
-    Eterm *idp = p;
-    if(monitor->type != MON_NIF_TARGET && is_external(monitor->u.pid))
-	insert_node(external_thing_ptr(monitor->u.pid)->node, MONITOR_REF, *idp);
-    if(is_external(monitor->ref))
-	insert_node(external_thing_ptr(monitor->ref)->node, MONITOR_REF, *idp);
+    ErtsLinkData *ldp = erts_link_to_data(lnk);
+    if ((ldp->a.flags & (ERTS_ML_FLG_DBG_VISITED
+                         | ERTS_ML_FLG_EXTENDED)) == ERTS_ML_FLG_EXTENDED) {
+        ErtsLinkDataExtended *ldep = (ErtsLinkDataExtended *) ldp;
+        if (ldep->ohhp) {
+            ErlOffHeap oh;
+            ERTS_INIT_OFF_HEAP(&oh);
+            oh.first = ldep->ohhp;
+            insert_offheap(&oh, type, id);
+        }
+    }
+    ldp->a.flags |= ERTS_ML_FLG_DBG_VISITED;
 }
 
-static void doit_insert_link(ErtsLink *lnk, void *p)
+static void insert_link(ErtsLink *lnk, void *idp)
 {
-    Eterm *idp = p;
-    if(is_external(lnk->pid))
-	insert_node(external_thing_ptr(lnk->pid)->node, LINK_REF, 
-		    *idp);
+    Eterm id = *((Eterm *) idp);
+    insert_link_data(lnk, LINK_REF, id);
 }
 
+static void clear_visited_link(ErtsLink *lnk, void *p)
+{
+    ErtsLinkData *ldp = erts_link_to_data(lnk);
+    ldp->a.flags &= ~ERTS_ML_FLG_DBG_VISITED;
+}
 
 static void
-insert_monitors(ErtsMonitor *monitors, Eterm id)
+insert_p_links(ErtsPTabElementCommon *p)
 {
-    erts_doforall_monitors(monitors,&doit_insert_monitor,&id);
+    Eterm id = p->id;
+    erts_link_tree_foreach(p->u.alive.links, insert_link, (void *) &id);
 }
 
 static void
-insert_links(ErtsLink *lnk, Eterm id)
+insert_dist_links(DistEntry *dep)
 {
-    erts_doforall_links(lnk,&doit_insert_link,&id);
+    if (dep->mld)
+        erts_link_list_foreach(dep->mld->links,
+                               insert_link,
+                               (void *) &dep->sysname);
 }
 
-static void doit_insert_link2(ErtsLink *lnk, void *p)
+static void
+clear_visited_p_links(ErtsPTabElementCommon *p)
 {
-    Eterm *idp = p;
-    if(is_external(lnk->pid))
-	insert_node(external_thing_ptr(lnk->pid)->node, LINK_REF, 
-		    *idp);
-    insert_links(ERTS_LINK_ROOT(lnk), *idp);
+    erts_link_tree_foreach(p->u.alive.links,
+                           clear_visited_link,
+                           NULL);
 }
 
 static void
-insert_links2(ErtsLink *lnk, Eterm id)
+clear_visited_dist_links(DistEntry *dep)
 {
-    erts_doforall_links(lnk,&doit_insert_link2,&id);
+    if (dep->mld)
+        erts_link_list_foreach(dep->mld->links,
+                               clear_visited_link,
+                               NULL);
 }
 
 static void
@@ -1575,6 +1682,60 @@ insert_delayed_delete_dist_entry(void *state,
 		      0);
 }
 
+static void
+insert_message(ErtsMessage *msg, int type, Process *proc)
+{
+    ErlHeapFragment *heap_frag = NULL;
+
+    ASSERT(ERTS_SIG_IS_MSG(msg));
+    if (msg->data.attached) {
+        if (msg->data.attached == ERTS_MSG_COMBINED_HFRAG)
+            heap_frag = &msg->hfrag;
+        else if (ERTS_SIG_IS_INTERNAL_MSG(msg))
+            heap_frag = msg->data.heap_frag;
+        else {
+            if (msg->data.dist_ext->dep)
+                insert_dist_entry(msg->data.dist_ext->dep,
+                                  type, proc->common.id, 0);
+            if (is_not_nil(ERL_MESSAGE_TOKEN(msg)))
+                heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
+        }
+    }
+    while (heap_frag) {
+        insert_offheap(&(heap_frag->off_heap),
+                       type,
+                       proc->common.id);
+        heap_frag = heap_frag->next;
+    }
+}
+
+static void
+insert_sig_msg(ErtsMessage *msg, void *arg)
+{
+    insert_message(msg, SIGNAL_REF, (Process *) arg);
+}
+
+static void
+insert_sig_offheap(ErlOffHeap *ohp, void *arg)
+{
+    Process *proc = arg;
+    insert_offheap(ohp, SIGNAL_REF, proc->common.id);
+}
+
+static void
+insert_sig_monitor(ErtsMonitor *mon, void *arg)
+{
+    Process *proc = arg;
+    insert_monitor_data(mon, SIGNAL_REF, proc->common.id);
+}
+
+static void
+insert_sig_link(ErtsLink *lnk, void *arg)
+{
+    Process *proc = arg;
+    insert_link_data(lnk, SIGNAL_REF, proc->common.id);
+}
+
 static void
 setup_reference_table(void)
 {
@@ -1630,10 +1791,7 @@ setup_reference_table(void)
 	Process *proc = erts_pix2proc(i);
 	if (proc) {
 	    int mli;
-	    ErtsMessage *msg_list[] = {
-		proc->msg.first,
-		proc->msg_inq.first,
-		proc->msg_frag};
+	    ErtsMessage *msg_list[] = {proc->msg_frag};
 
 	    /* Insert Heap */
 	    insert_offheap(&(proc->off_heap),
@@ -1648,34 +1806,24 @@ setup_reference_table(void)
 	    /* Insert msg buffers */
 	    for (mli = 0; mli < sizeof(msg_list)/sizeof(msg_list[0]); mli++) {
 		ErtsMessage *msg;
-		for (msg = msg_list[mli]; msg; msg = msg->next) {
-		    ErlHeapFragment *heap_frag = NULL;
-		    if (msg->data.attached) {
-			if (msg->data.attached == ERTS_MSG_COMBINED_HFRAG)
-			    heap_frag = &msg->hfrag;
-			else if (is_value(ERL_MESSAGE_TERM(msg)))
-			    heap_frag = msg->data.heap_frag;
-			else {
-			    if (msg->data.dist_ext->dep)
-				insert_dist_entry(msg->data.dist_ext->dep,
-						  HEAP_REF, proc->common.id, 0);
-			    if (is_not_nil(ERL_MESSAGE_TOKEN(msg)))
-				heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
-			}
-		    }
-		    while (heap_frag) {
-			insert_offheap(&(heap_frag->off_heap),
-				       HEAP_REF,
-				       proc->common.id);
-			heap_frag = heap_frag->next;
-		    }
-		}
+		for (msg = msg_list[mli]; msg; msg = msg->next)
+                    insert_message(msg, HEAP_REF, proc);
 	    }
+
+            /* Insert signal queue */
+            erts_proc_sig_debug_foreach_sig(proc,
+                                            insert_sig_msg,
+                                            insert_sig_offheap,
+                                            insert_sig_monitor,
+                                            insert_sig_link,
+                                            (void *) proc);
+
 	    /* Insert links */
-	    if (ERTS_P_LINKS(proc))
-		insert_links(ERTS_P_LINKS(proc), proc->common.id);
-	    if (ERTS_P_MONITORS(proc))
-		insert_monitors(ERTS_P_MONITORS(proc), proc->common.id);
+            insert_p_links(&proc->common);
+
+            /* Insert monitors */
+            insert_p_monitors(&proc->common);
+
             {
                 DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc);
                 if (dep)
@@ -1705,11 +1853,9 @@ setup_reference_table(void)
 	    continue;
 
 	/* Insert links */
-	if (ERTS_P_LINKS(prt))
-	    insert_links(ERTS_P_LINKS(prt), prt->common.id);
+        insert_p_links(&prt->common);
 	/* Insert monitors */
-	if (ERTS_P_MONITORS(prt))
-	    insert_monitors(ERTS_P_MONITORS(prt), prt->common.id);
+        insert_p_monitors(&prt->common);
 	/* Insert port data */
 	ohp = erts_port_data_offheap(prt);
 	if (ohp)
@@ -1758,41 +1904,25 @@ setup_reference_table(void)
     /* Insert all dist links */
 
     for(dep = erts_visible_dist_entries; dep; dep = dep->next) {
-	if(dep->nlinks)
-	    insert_links2(dep->nlinks, dep->sysname);
-	if(dep->node_links)
-	    insert_links(dep->node_links, dep->sysname);
-	if(dep->monitors)
-	    insert_monitors(dep->monitors, dep->sysname);
+        insert_dist_links(dep);
+        insert_dist_monitors(dep);
     }
 
     for(dep = erts_hidden_dist_entries; dep; dep = dep->next) {
-	if(dep->nlinks)
-	    insert_links2(dep->nlinks, dep->sysname);
-	if(dep->node_links)
-	    insert_links(dep->node_links, dep->sysname);
-	if(dep->monitors)
-	    insert_monitors(dep->monitors, dep->sysname);
+        insert_dist_links(dep);
+        insert_dist_monitors(dep);
     }
 
     for(dep = erts_pending_dist_entries; dep; dep = dep->next) {
-        if(dep->nlinks)
-            insert_links2(dep->nlinks, dep->sysname);
-        if(dep->node_links)
-            insert_links(dep->node_links, dep->sysname);
-        if(dep->monitors)
-            insert_monitors(dep->monitors, dep->sysname);
+        insert_dist_links(dep);
+        insert_dist_monitors(dep);
     }
 
     /* Not connected dist entries should not have any links,
        but inspect them anyway */
     for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) {
-	if(dep->nlinks)
-	    insert_links2(dep->nlinks, dep->sysname);
-	if(dep->node_links)
-	    insert_links(dep->node_links, dep->sysname);
-	if(dep->monitors)
-	    insert_monitors(dep->monitors, dep->sysname);
+        insert_dist_links(dep);
+        insert_dist_monitors(dep);
     }
 
     /* Insert all ets tables */
@@ -1877,6 +2007,10 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp)
 		tup = MK_2TUP(AM_system, MK_UINT(nrp->system_ref));
 		nrl = MK_CONS(tup, nrl);
 	    }
+	    if(nrp->signal_ref) {
+		tup = MK_2TUP(AM_signal, MK_UINT(nrp->signal_ref));
+		nrl = MK_CONS(tup, nrl);
+	    }
 
 	    nrid = nrp->id;
 	    if (!IS_CONST(nrp->id)) {
@@ -1900,24 +2034,25 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp)
 	    }
 	    else if(is_internal_port(nrid)) {
 		ASSERT(!nrp->heap_ref && !nrp->ets_ref && !nrp->bin_ref
-		       && !nrp->timer_ref && !nrp->system_ref);
+		       && !nrp->timer_ref && !nrp->system_ref && !nrp->signal_ref);
 		tup = MK_2TUP(AM_port, nrid);
 	    }
 	    else if(nrp->ets_ref) {
 		ASSERT(!nrp->heap_ref && !nrp->link_ref && 
 		       !nrp->monitor_ref && !nrp->bin_ref
-		       && !nrp->timer_ref && !nrp->system_ref);
+		       && !nrp->timer_ref && !nrp->system_ref && !nrp->signal_ref);
 		tup = MK_2TUP(AM_ets, nrid);
 	    }
 	    else if(nrp->bin_ref) {
 		ASSERT(is_small(nrid) || is_big(nrid));
 		ASSERT(!nrp->heap_ref && !nrp->ets_ref && !nrp->link_ref && 
 		       !nrp->monitor_ref && !nrp->timer_ref
-		       && !nrp->system_ref);
+		       && !nrp->system_ref && !nrp->signal_ref);
 		tup = MK_2TUP(AM_match_spec, nrid);
 	    }
 	    else  {
-		ASSERT(!nrp->heap_ref && !nrp->ets_ref && !nrp->bin_ref);
+		ASSERT(!nrp->heap_ref && !nrp->ets_ref && !nrp->bin_ref
+                       && !nrp->signal_ref);
 		ASSERT(is_atom(nrid));
 		tup = MK_2TUP(AM_dist, nrid);
 	    }
@@ -1961,30 +2096,36 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp)
 		tup = MK_2TUP(AM_system, MK_UINT(drp->system_ref));
 		drl = MK_CONS(tup, drl);
 	    }
+	    if(drp->signal_ref) {
+		tup = MK_2TUP(AM_signal, MK_UINT(drp->signal_ref));
+		drl = MK_CONS(tup, drl);
+	    }
 
 	    if (is_internal_pid(drp->id)) {
 		ASSERT(!drp->node_ref);
 		tup = MK_2TUP(AM_process, drp->id);
 	    }
 	    else if(is_internal_port(drp->id)) {
-		ASSERT(drp->ctrl_ref && !drp->node_ref);
+		ASSERT(drp->ctrl_ref && !drp->node_ref && !drp->signal_ref);
 		tup = MK_2TUP(AM_port, drp->id);
 	    }
 	    else if (is_tuple(drp->id)) {
 		Eterm *t;
 		ASSERT(drp->system_ref && !drp->node_ref
-		       && !drp->ctrl_ref && !drp->heap_ref && !drp->ets_ref);
+		       && !drp->ctrl_ref && !drp->heap_ref && !drp->ets_ref
+                       && !drp->signal_ref);
 		t = tuple_val(drp->id);
 		ASSERT(2 == arityval(t[0]));
 		tup = MK_2TUP(t[1], t[2]);
 	    }
 	    else if (drp->ets_ref) {
                 ASSERT(!drp->heap_ref && !drp->node_ref &&
-                       !drp->ctrl_ref && !drp->system_ref);
+                       !drp->ctrl_ref && !drp->system_ref
+                       && !drp->signal_ref);
                 tup = MK_2TUP(AM_ets, drp->id);
             }
-            else {
-		ASSERT(!drp->ctrl_ref && drp->node_ref);
+	    else {
+		ASSERT(!drp->ctrl_ref && drp->node_ref && !drp->signal_ref);
 		ASSERT(is_atom(drp->id));
 		tup = MK_2TUP(drp->id, MK_UINT(drp->creation));
 		tup = MK_2TUP(AM_node, tup);
@@ -2019,10 +2160,21 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp)
 
 }
 
+static void noop_sig_msg(ErtsMessage *msg, void *arg)
+{
+
+}
+
+static void noop_sig_offheap(ErlOffHeap *oh, void *arg)
+{
+
+}
+
 static void
 delete_reference_table(void)
 {
-    Uint i;
+    DistEntry *dep;
+    int i, max;
     for(i = 0; i < no_referred_nodes; i++) {
 	NodeReferrer *nrp;
 	NodeReferrer *tnrp;
@@ -2054,6 +2206,60 @@ delete_reference_table(void)
 	inserted_bins = inserted_bins->next;
 	erts_free(ERTS_ALC_T_NC_TMP, (void *)ib);
     }
+
+    /* Cleanup... */
+
+    max = erts_ptab_max(&erts_proc);
+    for (i = 0; i < max; i++) {
+	Process *proc = erts_pix2proc(i);
+	if (proc) {
+            clear_visited_p_links(&proc->common);
+            clear_visited_p_monitors(&proc->common);
+            erts_proc_sig_debug_foreach_sig(proc,
+                                            noop_sig_msg,
+                                            noop_sig_offheap,
+                                            clear_visited_monitor,
+                                            clear_visited_link,
+                                            (void *) proc);
+        }
+    }
+
+    max = erts_ptab_max(&erts_port);
+    for (i = 0; i < max; i++) {
+	erts_aint32_t state;
+	Port *prt;
+
+	prt = erts_pix2port(i);
+	if (!prt)
+	    continue;
+
+	state = erts_atomic32_read_nob(&prt->state);
+	if (state & ERTS_PORT_SFLGS_DEAD)
+	    continue;
+
+        clear_visited_p_links(&prt->common);
+        clear_visited_p_monitors(&prt->common);
+    }
+
+    for(dep = erts_visible_dist_entries; dep; dep = dep->next) {
+        clear_visited_dist_links(dep);
+        clear_visited_dist_monitors(dep);
+    }
+
+    for(dep = erts_hidden_dist_entries; dep; dep = dep->next) {
+        clear_visited_dist_links(dep);
+        clear_visited_dist_monitors(dep);
+    }
+
+    for(dep = erts_pending_dist_entries; dep; dep = dep->next) {
+        clear_visited_dist_links(dep);
+        clear_visited_dist_monitors(dep);
+    }
+
+    for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) {
+        clear_visited_dist_links(dep);
+        clear_visited_dist_monitors(dep);
+    }
 }
 
 void
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 58279017c8..9a792b10b1 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,7 +18,17 @@
  * %CopyrightEnd%
  */
 
-#ifndef ERL_NODE_TABLES_H__
+#ifndef ERL_NODE_TABLES_BASIC__
+#define ERL_NODE_TABLES_BASIC__
+
+typedef struct dist_entry_ DistEntry;
+typedef struct ErtsDistOutputBuf_ ErtsDistOutputBuf;
+void erts_ref_dist_entry(DistEntry *dep);
+void erts_deref_dist_entry(DistEntry *dep);
+
+#endif
+
+#if !defined(ERL_NODE_TABLES_BASIC_ONLY) && !defined(ERL_NODE_TABLES_H__)
 #define ERL_NODE_TABLES_H__
 
 /*
@@ -42,14 +52,14 @@
 #include "sys.h"
 #include "hash.h"
 #include "erl_alloc.h"
-#include "erl_process.h"
-#include "erl_monitors.h"
 #define ERTS_PORT_TASK_ONLY_BASIC_TYPES__
 #include "erl_port_task.h"
 #undef ERTS_PORT_TASK_ONLY_BASIC_TYPES__
+#include "erl_process.h"
 #define ERTS_BINARY_TYPES_ONLY__
 #include "erl_binary.h"
 #undef ERTS_BINARY_TYPES_ONLY__
+#include "erl_monitor_link.h"
 
 #define ERTS_NODE_TAB_DELAY_GC_DEFAULT (60)
 #define ERTS_NODE_TAB_DELAY_GC_MAX (100*1000*1000)
@@ -82,7 +92,6 @@ enum dist_entry_state {
 #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713)
 #endif
 
-typedef struct ErtsDistOutputBuf_ ErtsDistOutputBuf;
 struct ErtsDistOutputBuf_ {
 #ifdef DEBUG
     Uint dbg_pattern;
@@ -113,7 +122,7 @@ struct ErtsProcList_;
  *   unlock mutexes with higher numbers before mutexes with higher numbers.
  */
 
-typedef struct dist_entry_ {
+struct dist_entry_ {
     HashBucket hash_bucket;     /* Hash bucket */
     struct dist_entry_ *next;	/* Next entry in dist_table (not sorted) */
     struct dist_entry_ *prev;	/* Previous entry in dist_table (not sorted) */
@@ -130,18 +139,7 @@ typedef struct dist_entry_ {
 				   atom cache etc. */
     unsigned long version;	/* Protocol version */
 
-
-    erts_mtx_t lnk_mtx;         /* Protects node_links, nlinks, and
-				   monitors. */
-    ErtsLink *node_links;       /* In a dist entry, node links are kept 
-				   in a separate tree, while they are 
-				   colocted with the ordinary link tree
-				   for processes. It's not due to confusion,
-				   it's because the link tree for the dist 
-				   entry is in two levels, see erl_monitors.h 
-				*/
-    ErtsLink *nlinks;           /* Link tree with subtrees */
-    ErtsMonitor *monitors;      /* Monitor tree */
+    ErtsMonLnkDist *mld;        /* Monitors and links */
 
     erts_mtx_t qlock;           /* Protects qflgs and out_queue */
     erts_atomic32_t qflgs;
@@ -163,7 +161,7 @@ typedef struct dist_entry_ {
     ErtsThrPrgrLaterOp later_op;
 
     struct transcode_context* transcode_ctx;
-} DistEntry;
+};
 
 typedef struct erl_node_ {
   HashBucket hash_bucket;	/* Hash bucket */
@@ -219,16 +217,12 @@ int erts_dist_entry_destructor(Binary *bin);
 DistEntry *erts_dhandle_to_dist_entry(Eterm dhandle);
 Eterm erts_build_dhandle(Eterm **hpp, ErlOffHeap*, DistEntry*);
 Eterm erts_make_dhandle(Process *c_p, DistEntry *dep);
-void erts_ref_dist_entry(DistEntry *dep);
-void erts_deref_dist_entry(DistEntry *dep);
 
 ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np);
 ERTS_GLB_INLINE void erts_de_rlock(DistEntry *dep);
 ERTS_GLB_INLINE void erts_de_runlock(DistEntry *dep);
 ERTS_GLB_INLINE void erts_de_rwlock(DistEntry *dep);
 ERTS_GLB_INLINE void erts_de_rwunlock(DistEntry *dep);
-ERTS_GLB_INLINE void erts_de_links_lock(DistEntry *dep);
-ERTS_GLB_INLINE void erts_de_links_unlock(DistEntry *dep);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
@@ -264,18 +258,6 @@ erts_de_rwunlock(DistEntry *dep)
     erts_rwmtx_rwunlock(&dep->rwmtx);
 }
 
-ERTS_GLB_INLINE void
-erts_de_links_lock(DistEntry *dep)
-{
-    erts_mtx_lock(&dep->lnk_mtx);
-}
-
-ERTS_GLB_INLINE void
-erts_de_links_unlock(DistEntry *dep)
-{
-    erts_mtx_unlock(&dep->lnk_mtx);
-}
-
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 void erts_debug_test_node_tab_delayed_delete(Sint64 millisecs);
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index 0d148ee048..2a98a6f00b 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -373,7 +373,7 @@ Eterm erts_request_io_bytes(Process *c_p);
 
 void print_port_info(Port *, fmtfn_t, void *);
 void erts_port_free(Port *);
-void erts_fire_port_monitor(Port *prt, Eterm ref);
+void erts_fire_port_monitor(Port *prt, ErtsMonitor *tmon);
 int erts_port_handle_xports(Port *);
 
 #if defined(ERTS_ENABLE_LOCK_CHECK)
@@ -887,20 +887,20 @@ struct ErtsProc2PortSigData_ {
 	    Eterm item;
 	} info;
 	struct {
-	    Eterm port;
-	    Eterm to;
+            Eterm port_id;
+	    ErtsLink *lnk;
 	} link;
 	struct {
-	    Eterm from;
+            Eterm port_id;
+	    ErtsLink *lnk;
 	} unlink;
         struct {
-            Eterm origin;   /* who receives monitor event, pid */
-            Eterm name;     /* either name for named monitor, or port id */
+            Eterm port_id;
+            ErtsMonitor *mon;
         } monitor;
         struct {
-            Eterm origin;   /* who is at the other end of the monitor, pid */
-            Eterm name;     /* port id */
-            Uint32 ref[ERTS_MAX_REF_NUMBERS]; /* box contents of a ref */
+            Eterm port_id;
+            ErtsMonitor *mon;
         } demonitor;
     } u;
 } ;
@@ -946,7 +946,6 @@ typedef int (*ErtsProc2PortSigCallback)(Port *,
 
 typedef enum {
     ERTS_PORT_OP_BADARG,
-    ERTS_PORT_OP_CALLER_EXIT,
     ERTS_PORT_OP_BUSY,
     ERTS_PORT_OP_BUSY_SCHEDULED,
     ERTS_PORT_OP_SCHEDULED,
@@ -982,32 +981,13 @@ ErtsPortOpResult erts_port_command(Process *, int, Port *, Eterm, Eterm *);
 ErtsPortOpResult erts_port_output(Process *, int, Port *, Eterm, Eterm, Eterm *);
 ErtsPortOpResult erts_port_exit(Process *, int, Port *, Eterm, Eterm, Eterm *);
 ErtsPortOpResult erts_port_connect(Process *, int, Port *, Eterm, Eterm, Eterm *);
-ErtsPortOpResult erts_port_link(Process *, Port *, Eterm, Eterm *);
-ErtsPortOpResult erts_port_unlink(Process *, Port *, Eterm, Eterm *);
+ErtsPortOpResult erts_port_link(Process *, Port *, ErtsLink *, Eterm *);
+ErtsPortOpResult erts_port_unlink(Process *, Port *, ErtsLink *, Eterm *);
 ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm *);
 ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *);
 ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *);
-
-/* Creates monitor between Origin and Target. Ref must be initialized to
- * a reference (ref may be rewritten to be used to serve additionally as a
- * signal id). Name is atom if user monitors port by name or NIL */
-ErtsPortOpResult erts_port_monitor(Process *origin, Port *target, Eterm name,
-                                   Eterm *ref);
-
-typedef enum {
-    /* Normal demonitor rules apply with locking and reductions bump */
-    ERTS_PORT_DEMONITOR_NORMAL = 1,
-    /* Relaxed demonitor rules when process is about to die, which means that
-     * pid lookup won't work, locks won't work, no reductions bump. */
-    ERTS_PORT_DEMONITOR_ORIGIN_ON_DEATHBED = 2,
-} ErtsDemonitorMode;
-
-/* Removes monitor between origin and target, identified by ref.
- * origin_is_dying can be 0 (false, normal locking rules and reductions bump
- * apply) or 1 (true, in case when we avoid origin locking) */
-ErtsPortOpResult erts_port_demonitor(Process *origin, ErtsDemonitorMode mode,
-                                    Port *target, Eterm ref,
-                                    Eterm *trap_ref);
+ErtsPortOpResult erts_port_monitor(Process *, Port *, ErtsMonitor *);
+ErtsPortOpResult erts_port_demonitor(Process *, Port *, ErtsMonitor *);
 /* defined in erl_bif_port.c */
 Port *erts_sig_lookup_port(Process *c_p, Eterm id_or_name);
 
diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
new file mode 100644
index 0000000000..1b5cbb1919
--- /dev/null
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -0,0 +1,3772 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2018. All Rights Reserved.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description:	Process signal queue implementation.
+ *
+ * Author: 	Rickard Green
+ */
+
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "sys.h"
+#include "global.h"
+#include "dist.h"
+#include "erl_process.h"
+#include "erl_port_task.h"
+#include "erl_trace.h"
+#include "beam_bp.h"
+#include "big.h"
+#include "erl_gc.h"
+#include "bif.h"
+#include "erl_proc_sig_queue.h"
+
+#define ERTS_SIG_REDS_CNT_FACTOR 4
+#define ERTS_PROC_SIG_TRACE_COUNT_LIMIT 200
+
+/*
+ * Note that not all signal are handled using this functionality!
+ */
+
+#define ERTS_SIG_Q_OP_MAX 9
+
+#define ERTS_SIG_Q_OP_EXIT                      0
+#define ERTS_SIG_Q_OP_EXIT_LINKED               1
+#define ERTS_SIG_Q_OP_MONITOR_DOWN              2
+#define ERTS_SIG_Q_OP_MONITOR                   3
+#define ERTS_SIG_Q_OP_DEMONITOR                 4
+#define ERTS_SIG_Q_OP_LINK                      5
+#define ERTS_SIG_Q_OP_UNLINK                    6
+#define ERTS_SIG_Q_OP_GROUP_LEADER              7
+#define ERTS_SIG_Q_OP_TRACE_CHANGE_STATE        8
+#define ERTS_SIG_Q_OP_PERSISTENT_MON_MSG        ERTS_SIG_Q_OP_MAX
+
+#define ERTS_SIG_Q_TYPE_MAX (ERTS_MON_LNK_TYPE_MAX + 5)
+
+#define ERTS_SIG_Q_TYPE_UNDEFINED \
+    (ERTS_MON_LNK_TYPE_MAX + 1)
+#define ERTS_SIG_Q_TYPE_DIST_LINK \
+    (ERTS_MON_LNK_TYPE_MAX + 2)
+#define ERTS_SIG_Q_TYPE_GEN_EXIT \
+    (ERTS_MON_LNK_TYPE_MAX + 3)
+#define ERTS_SIG_Q_TYPE_DIST_PROC_DEMONITOR \
+    (ERTS_MON_LNK_TYPE_MAX + 4)
+#define ERTS_SIG_Q_TYPE_ADJUST_TRACE_INFO \
+    ERTS_SIG_Q_TYPE_MAX
+
+
+#define ERTS_SIG_Q_OP_BITS      8                      
+#define ERTS_SIG_Q_OP_SHIFT     0
+#define ERTS_SIG_Q_OP_MASK      ((1 << ERTS_SIG_Q_OP_BITS) - 1)
+
+#define ERTS_SIG_Q_TYPE_BITS    8
+#define ERTS_SIG_Q_TYPE_SHIFT   ERTS_SIG_Q_OP_BITS
+#define ERTS_SIG_Q_TYPE_MASK    ((1 << ERTS_SIG_Q_TYPE_BITS) - 1)
+
+#define ERTS_SIG_Q_NON_X_BITS__ (_HEADER_ARITY_OFFS \
+                                 + ERTS_SIG_Q_OP_BITS \
+                                 + ERTS_SIG_Q_TYPE_BITS)
+
+#define ERTS_SIG_Q_XTRA_BITS    (32 - ERTS_SIG_Q_NON_X_BITS__)
+#define ERTS_SIG_Q_XTRA_SHIFT   (ERTS_SIG_Q_OP_BITS \
+                                 + ERTS_SIG_Q_TYPE_BITS)
+#define ERTS_SIG_Q_XTRA_MASK    ((1 << ERTS_SIG_Q_XTRA_BITS) - 1)
+
+#define ERTS_PROC_SIG_OP(Tag) \
+    ((int) (_unchecked_thing_arityval((Tag)) \
+            >> ERTS_SIG_Q_OP_SHIFT) & ERTS_SIG_Q_OP_MASK)
+
+#define ERTS_PROC_SIG_TYPE(Tag) \
+    ((Uint16) (_unchecked_thing_arityval((Tag)) \
+               >> ERTS_SIG_Q_TYPE_SHIFT) & ERTS_SIG_Q_TYPE_MASK)
+
+#define ERTS_PROC_SIG_XTRA(Tag) \
+    ((Uint32) (_unchecked_thing_arityval((Tag)) \
+               >> ERTS_SIG_Q_XTRA_SHIFT) & ERTS_SIG_Q_XTRA_MASK)
+
+#define ERTS_PROC_SIG_MAKE_TAG(Op, Type, Xtra)                  \
+    (ASSERT(0 <= (Xtra) && (Xtra) <= ERTS_SIG_Q_XTRA_MASK),     \
+     _make_header((((Type) & ERTS_SIG_Q_TYPE_MASK)              \
+                   << ERTS_SIG_Q_TYPE_SHIFT)                    \
+                  | (((Op) & ERTS_SIG_Q_OP_MASK)                \
+                     << ERTS_SIG_Q_OP_SHIFT)                    \
+                  | (((Xtra) & ERTS_SIG_Q_XTRA_MASK)            \
+                     << ERTS_SIG_Q_XTRA_SHIFT),                 \
+                  _TAG_HEADER_EXTERNAL_PID))
+
+Process *ERTS_WRITE_UNLIKELY(erts_dirty_process_signal_handler);
+Process *ERTS_WRITE_UNLIKELY(erts_dirty_process_signal_handler_high);
+Process *ERTS_WRITE_UNLIKELY(erts_dirty_process_signal_handler_max);
+
+void
+erts_proc_sig_queue_init(void)
+{
+    ERTS_CT_ASSERT(ERTS_SIG_Q_OP_MASK >= ERTS_SIG_Q_OP_MAX);
+    ERTS_CT_ASSERT(ERTS_SIG_Q_TYPE_MASK >= ERTS_SIG_Q_TYPE_MAX);
+}
+
+typedef struct {
+    int active;
+    int procs;
+    struct {
+        int active;
+#if defined(USE_VM_PROBES)
+        int vm_probes;
+        char receiver_name[DTRACE_TERM_BUF_SIZE];
+#endif
+        int receive_trace;
+        int bp_ix;
+        ErtsMessage **next;
+        ErtsTracingEvent *event;
+    } messages;
+} ErtsSigRecvTracing;
+
+typedef struct {
+    Eterm message;
+    Eterm from;
+    Eterm reason;
+    union {
+        Eterm ref;
+        int normal_kills;
+    } u;
+} ErtsExitSignalData;
+
+typedef struct {
+    Eterm message;
+    Eterm key;
+} ErtsPersistMonMsg;
+
+typedef struct {
+    ErtsSignalCommon common;
+    Eterm local; /* internal pid (immediate) */
+    Eterm remote; /* external pid (heap for it follow) */
+    Eterm heap[EXTERNAL_THING_HEAD_SIZE + 1];
+} ErtsSigDistLinkOp;
+
+typedef struct {
+    ErtsSignalCommon common;
+    Uint flags_on;
+    Uint flags_off;
+    Eterm tracer;
+} ErtsSigTraceInfo;
+
+#define ERTS_SIG_GL_FLG_ACTIVE          (((erts_aint_t) 1) << 0)
+#define ERTS_SIG_GL_FLG_RECEIVER        (((erts_aint_t) 1) << 1)
+#define ERTS_SIG_GL_FLG_SENDER          (((erts_aint_t) 1) << 2)
+
+typedef struct {
+    ErtsSignalCommon common;
+    erts_atomic_t flags;
+    Eterm group_leader;
+    Eterm reply_to;
+    Eterm ref;
+    ErlOffHeap oh;
+    Eterm heap[1];
+} ErtsSigGroupLeader;
+
+static int handle_msg_tracing(Process *c_p,
+                              ErtsSigRecvTracing *tracing,
+                              ErtsMessage ***next_nm_sig);
+static int handle_trace_change_state(Process *c_p,
+                                     ErtsSigRecvTracing *tracing,
+                                     Uint16 type,
+                                     ErtsMessage *sig,
+                                     ErtsMessage ***next_nm_sig);
+static void getting_unlinked(Process *c_p, Eterm unlinker);
+static void getting_linked(Process *c_p, Eterm linker);
+static void group_leader_reply(Process *c_p, Eterm to,
+                               Eterm ref, int success);
+static int stretch_limit(Process *c_p, ErtsSigRecvTracing *tp,
+                         int abs_lim, int *limp);
+
+#ifdef ERTS_PROC_SIG_HARD_DEBUG
+#define ERTS_PROC_SIG_HDBG_PRIV_CHKQ(P, T, NMN)                 \
+    do {                                                        \
+        ErtsMessage **nm_next__ = *(NMN);                       \
+        ErtsMessage **nm_last__ = (P)->sig_qs.nmsigs.last;      \
+        if (!nm_next__ || !*nm_next__) {                        \
+            nm_next__ = NULL;                                   \
+            nm_last__ = NULL;                                   \
+        }                                                       \
+        proc_sig_hdbg_check_queue((P),                          \
+                                  1,                            \
+                                  &(P)->sig_qs.cont,            \
+                                  (P)->sig_qs.cont_last,        \
+                                  nm_next__,                    \
+                                  nm_last__,                    \
+                                  (T),                          \
+                                  NULL,                         \
+                                  ERTS_PSFLG_FREE);             \
+    } while (0);
+static Sint
+proc_sig_hdbg_check_queue(Process *c_p,
+                          int privq,
+                          ErtsMessage **sig_next,
+                          ErtsMessage **sig_last,
+                          ErtsMessage **sig_nm_next,
+                          ErtsMessage **sig_nm_last,
+                          ErtsSigRecvTracing *tracing,
+                          int *found_saved_last_p,
+                          erts_aint32_t sig_psflg);
+#else
+#define ERTS_PROC_SIG_HDBG_PRIV_CHKQ(P, T, NMN)
+#endif
+
+typedef struct {
+    ErtsSignalCommon common;
+    Eterm ref;
+    Eterm heap[1];
+} ErtsSigDistProcDemonitor;
+
+static void
+destroy_dist_proc_demonitor(ErtsSigDistProcDemonitor *dmon)
+{
+    Eterm ref = dmon->ref;
+    if (is_external(ref)) {
+        ExternalThing *etp = external_thing_ptr(ref);
+        erts_deref_node_entry(etp->node);
+    }
+    erts_free(ERTS_ALC_T_DIST_DEMONITOR, dmon);
+}
+
+static ERTS_INLINE ErtsSigDistLinkOp *
+make_sig_dist_link_op(int op, Eterm local, Eterm remote)
+{
+    Eterm *hp;
+    ErlOffHeap oh = {0};
+    ErtsSigDistLinkOp *sdlnk = erts_alloc(ERTS_ALC_T_SIG_DATA,
+                                          sizeof(ErtsSigDistLinkOp));
+    ASSERT(is_internal_pid(local));
+    ASSERT(is_external_pid(remote));
+
+    hp = &sdlnk->heap[0];
+
+    sdlnk->common.tag = ERTS_PROC_SIG_MAKE_TAG(op,
+                                               ERTS_SIG_Q_TYPE_DIST_LINK,
+                                               0);
+    sdlnk->local = local;
+    sdlnk->remote = STORE_NC(&hp, &oh, remote);
+
+    ASSERT(&sdlnk->heap[0] < hp);
+    ASSERT(hp <= &sdlnk->heap[0] + sizeof(sdlnk->heap)/sizeof(sdlnk->heap[0]));
+    ASSERT(boxed_val(sdlnk->remote) == &sdlnk->heap[0]);
+
+    return sdlnk;
+}
+
+static ERTS_INLINE void
+destroy_sig_dist_link_op(ErtsSigDistLinkOp *sdlnk)
+{
+    ASSERT(is_external_pid(sdlnk->remote));
+    ASSERT(boxed_val(sdlnk->remote) == &sdlnk->heap[0]);
+    erts_deref_node_entry(((ExternalThing *) &sdlnk->heap[0])->node);
+    erts_free(ERTS_ALC_T_SIG_DATA, sdlnk);
+}
+
+static ERTS_INLINE ErtsExitSignalData *
+get_exit_signal_data(ErtsMessage *xsig)
+{
+    ASSERT(ERTS_SIG_IS_NON_MSG(xsig));
+    ASSERT((ERTS_PROC_SIG_OP(((ErtsSignal *) xsig)->common.tag)
+            == ERTS_SIG_Q_OP_EXIT)
+           || (ERTS_PROC_SIG_OP(((ErtsSignal *) xsig)->common.tag)
+                == ERTS_SIG_Q_OP_EXIT_LINKED)
+           || (ERTS_PROC_SIG_OP(((ErtsSignal *) xsig)->common.tag)
+               == ERTS_SIG_Q_OP_MONITOR_DOWN));
+    ASSERT(xsig->hfrag.alloc_size > xsig->hfrag.used_size);
+    ASSERT((xsig->hfrag.alloc_size - xsig->hfrag.used_size)*sizeof(UWord)
+           >= sizeof(ErtsExitSignalData));
+    return (ErtsExitSignalData *) (char *) (&xsig->hfrag.mem[0]
+                                            + xsig->hfrag.used_size);
+}
+
+static ERTS_INLINE void
+destroy_trace_info(ErtsSigTraceInfo *ti)
+{
+    if (is_value(ti->tracer))
+        erts_tracer_update(&ti->tracer, NIL);
+    erts_free(ERTS_ALC_T_SIG_DATA, ti);
+}
+
+static void
+destroy_sig_group_leader(ErtsSigGroupLeader *sgl)
+{
+    erts_cleanup_offheap(&sgl->oh);
+    erts_free(ERTS_ALC_T_SIG_DATA, sgl);
+}
+
+static ERTS_INLINE void
+sig_enqueue_trace(Process *c_p, ErtsMessage *sig, int op,
+                  Process *rp, ErtsMessage **first,
+                  ErtsMessage **last, ErtsMessage ***last_next)
+{
+    switch (op) {
+    case ERTS_SIG_Q_OP_LINK:
+        if (c_p
+            && ((!!IS_TRACED(c_p))
+                & (ERTS_TRACE_FLAGS(c_p) & (F_TRACE_SOL
+                                            | F_TRACE_SOL1)))) {
+            ErtsSigTraceInfo *ti;
+            Eterm tag;
+            /*
+             * Set on link enabled.
+             *
+             * Prepend a trace-change-state signal before the
+             * link signal...
+             */
+
+            tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_TRACE_CHANGE_STATE,
+                                         ERTS_SIG_Q_TYPE_ADJUST_TRACE_INFO,
+                                         0);
+            ti = erts_alloc(ERTS_ALC_T_SIG_DATA, sizeof(ErtsSigTraceInfo));
+            ti->common.next = *last;
+            ti->common.specific.next = &ti->common.next;
+            ti->common.tag = tag;
+            ti->flags_on = ERTS_TRACE_FLAGS(c_p) & TRACEE_FLAGS;
+            if (!(ti->flags_on & F_TRACE_SOL1))
+                ti->flags_off = 0;
+            else {
+                ti->flags_off = F_TRACE_SOL1|F_TRACE_SOL;
+                erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+                ERTS_TRACE_FLAGS(c_p) &= ~(F_TRACE_SOL1|F_TRACE_SOL);
+                erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+            }
+            erts_tracer_update(&ti->tracer, ERTS_TRACER(c_p));
+            *first = (ErtsMessage *) ti;
+            *last_next = &ti->common.next;
+        }
+        break;
+
+#ifdef USE_VM_PROBES
+    case ERTS_SIG_Q_OP_EXIT:
+    case ERTS_SIG_Q_OP_EXIT_LINKED: {
+        ErtsExitSignalData *xsigd = get_exit_signal_data(sig);
+        if(DTRACE_ENABLED(process_exit_signal) && is_pid(xsigd->from)) {
+            DTRACE_CHARBUF(sender_str, DTRACE_TERM_BUF_SIZE);
+            DTRACE_CHARBUF(receiver_str, DTRACE_TERM_BUF_SIZE);
+            DTRACE_CHARBUF(reason_buf, DTRACE_TERM_BUF_SIZE);
+
+            dtrace_pid_str(from, sender_str);
+            dtrace_proc_str(rp, receiver_str);
+            erts_snprintf(reason_buf, sizeof(DTRACE_CHARBUF_NAME(reason_buf)) - 1, "%T", reason);
+            DTRACE3(process_exit_signal, sender_str, receiver_str, reason_buf);
+        }
+        break;
+    }
+#endif
+
+    default:
+        break;
+    }
+}
+
+static void
+sig_enqueue_trace_cleanup(ErtsMessage *first, ErtsSignal *sig, ErtsMessage *last)
+{
+    ErtsMessage *tmp;
+
+    /* The usual case; no tracing signals... */
+    if (sig == (ErtsSignal *) first && sig == (ErtsSignal *) last) {
+        sig->common.next = NULL;
+        return;
+    }
+
+    /* Got trace signals to clean up... */
+
+    tmp = first;
+
+    while (tmp) {
+        ErtsMessage *tmp_free = tmp;
+        tmp = tmp->next;
+        if (sig != (ErtsSignal *) tmp_free) {
+            switch (ERTS_PROC_SIG_OP(((ErtsSignal *) tmp_free)->common.tag)) {
+            case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE:
+                destroy_trace_info((ErtsSigTraceInfo *) tmp_free);
+                break;
+            default:
+                ERTS_INTERNAL_ERROR("Unexpected signal op");
+                break;
+            }
+        }
+    }
+}
+
+static ERTS_INLINE erts_aint32_t
+enqueue_signals(Process *rp, ErtsMessage *first,
+                ErtsMessage *last, ErtsMessage **last_next,
+                erts_aint32_t in_state)
+{
+    erts_aint32_t state = in_state;
+    ErtsMessage **this = rp->sig_inq.last;
+
+    ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE(rp);
+
+    ASSERT(!*this);
+    *this = first;
+    rp->sig_inq.last = &last->next;
+
+    if (!rp->sig_inq.nmsigs.next) {
+        ASSERT(!rp->sig_inq.nmsigs.last);
+        rp->sig_inq.nmsigs.next = this;
+        state = erts_atomic32_read_bor_nob(&rp->state,
+                                           ERTS_PSFLG_SIG_IN_Q);
+        ASSERT(!(state & ERTS_PSFLG_SIG_IN_Q));
+    }
+    else {
+        ErtsSignal *sig;
+        ASSERT(rp->sig_inq.nmsigs.last);
+
+        sig = (ErtsSignal *) *rp->sig_inq.nmsigs.last;
+
+        ASSERT(sig && !sig->common.specific.next);
+        ASSERT(state & ERTS_PSFLG_SIG_IN_Q);
+        sig->common.specific.next = this;
+    }
+
+    if (last_next) {
+        ASSERT(first != last);
+        rp->sig_inq.nmsigs.last = last_next;
+    }
+    else {
+        ASSERT(first == last);
+        rp->sig_inq.nmsigs.last = this;
+    }
+
+    ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE(rp);
+
+    return state;
+}
+
+static ERTS_INLINE void
+ensure_dirty_proc_handled(Eterm pid,
+                          erts_aint32_t state,
+                          erts_aint32_t prio)
+{
+    if (state & (ERTS_PSFLG_DIRTY_RUNNING
+                 | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
+        Eterm *hp;
+        ErtsMessage *mp;
+        Process *sig_handler;
+
+        if (prio < 0)
+            prio = (int) ERTS_PSFLGS_GET_USR_PRIO(state);
+
+        switch (prio) {
+        case PRIORITY_MAX:
+            sig_handler = erts_dirty_process_signal_handler_max;
+            break;
+        case PRIORITY_HIGH:
+            sig_handler = erts_dirty_process_signal_handler_high;
+            break;
+        default:
+            sig_handler = erts_dirty_process_signal_handler;
+            break;
+        }
+
+        /* Make sure signals are handled... */
+        mp = erts_alloc_message(0, &hp);
+        erts_queue_message(sig_handler, 0, mp, pid, am_system);
+    }
+}
+
+static int
+proc_queue_signal(Process *c_p, Eterm pid, ErtsSignal *sig, int op)
+{
+    int res;
+    Process *rp;
+    ErtsMessage *first, *last, **last_next;
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    int is_normal_sched = !!esdp && esdp->type == ERTS_SCHED_NORMAL;
+    erts_aint32_t state;
+
+    if (is_normal_sched)
+        rp = erts_proc_lookup_raw(pid);
+    else
+        rp = erts_proc_lookup_raw_inc_refc(pid);
+
+    if (!rp)
+        return 0;
+
+    sig->common.specific.next = NULL;
+    first = last = (ErtsMessage *) sig;
+    last_next = NULL;
+
+    /* may add signals before and/or after sig */
+    sig_enqueue_trace(c_p, first, op, rp,
+                      &first, &last, &last_next);
+
+    last->next = NULL;
+
+    erts_proc_lock(rp, ERTS_PROC_LOCK_MSGQ);
+
+    state = erts_atomic32_read_nob(&rp->state);
+
+    if (ERTS_PSFLG_FREE & state)
+        res = 0;
+    else {
+        state = enqueue_signals(rp, first, last, last_next, state);
+        res = !0;
+    }
+
+    erts_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ);
+
+    if (res == 0)
+        sig_enqueue_trace_cleanup(first, sig, last);
+
+    if (!(state & (ERTS_PSFLG_EXITING
+                   | ERTS_PSFLG_ACTIVE_SYS
+                   | ERTS_PSFLG_SIG_IN_Q))) {
+        /* Schedule process... */
+        state = erts_proc_sys_schedule(rp, state, 0);
+    }
+
+    ensure_dirty_proc_handled(rp->common.id, state, -1);
+
+    if (!is_normal_sched)
+        erts_proc_dec_refc(rp);
+
+    return res;
+}
+
+void
+erts_proc_sig_fetch(Process *proc)
+{
+#ifdef ERTS_PROC_SIG_HARD_DEBUG
+    ErtsSignalPrivQueues sig_qs = proc->sig_qs;
+    ErtsSignalInQueue sig_inq = proc->sig_inq;
+#endif
+
+    ERTS_LC_ASSERT(erts_thr_progress_is_blocking()
+                   || ERTS_PROC_IS_EXITING(proc)
+                   || ((erts_proc_lc_my_proc_locks(proc)
+                        & (ERTS_PROC_LOCK_MAIN
+                           | ERTS_PROC_LOCK_MSGQ))
+                       == (ERTS_PROC_LOCK_MAIN
+                           | ERTS_PROC_LOCK_MSGQ)));
+
+    if (!proc->sig_inq.first) {
+        ASSERT(proc->sig_inq.last == &proc->sig_inq.first);
+        ASSERT(proc->sig_inq.len == 0);
+        ASSERT(!proc->sig_inq.nmsigs.next);
+        ASSERT(!proc->sig_inq.nmsigs.last);
+        return;
+    }
+
+    ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE(proc);
+    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(proc);
+
+    if (!proc->sig_inq.nmsigs.next) {
+        ASSERT(!(ERTS_PSFLG_SIG_IN_Q
+                 & erts_atomic32_read_nob(&proc->state)));
+        ASSERT(!proc->sig_inq.nmsigs.last);
+
+        if (proc->sig_qs.cont || ERTS_MSG_RECV_TRACED(proc)) {
+            *proc->sig_qs.cont_last = proc->sig_inq.first;
+            proc->sig_qs.cont_last = proc->sig_inq.last;
+        }
+        else {
+            *proc->sig_qs.last = proc->sig_inq.first;
+            proc->sig_qs.last = proc->sig_inq.last;
+        }
+    }
+    else {
+#ifdef DEBUG
+        erts_aint32_t s;
+#endif
+        ASSERT(proc->sig_inq.nmsigs.last);
+         if (!proc->sig_qs.nmsigs.last) {
+            ASSERT(!proc->sig_qs.nmsigs.next);
+            if (proc->sig_inq.nmsigs.next == &proc->sig_inq.first)
+                proc->sig_qs.nmsigs.next = proc->sig_qs.cont_last;
+            else
+                proc->sig_qs.nmsigs.next = proc->sig_inq.nmsigs.next;
+
+#ifdef DEBUG
+            s =
+#endif
+                erts_atomic32_read_bset_nob(&proc->state,
+                                            (ERTS_PSFLG_SIG_Q
+                                             | ERTS_PSFLG_SIG_IN_Q),
+                                            ERTS_PSFLG_SIG_Q);
+
+            ASSERT((s & (ERTS_PSFLG_SIG_Q|ERTS_PSFLG_SIG_IN_Q))
+                   == ERTS_PSFLG_SIG_IN_Q);
+        }
+        else {
+            ErtsSignal *sig;
+            ASSERT(proc->sig_qs.nmsigs.next);
+            sig = ((ErtsSignal *) *proc->sig_qs.nmsigs.last);
+            ASSERT(ERTS_SIG_IS_NON_MSG(sig));
+            ASSERT(!sig->common.specific.next);
+            if (proc->sig_inq.nmsigs.next == &proc->sig_inq.first)
+                sig->common.specific.next = proc->sig_qs.cont_last;
+            else
+                sig->common.specific.next = proc->sig_inq.nmsigs.next;
+
+#ifdef DEBUG
+            s =
+#endif
+                erts_atomic32_read_band_nob(&proc->state,
+                                            ~ERTS_PSFLG_SIG_IN_Q);
+
+            ASSERT((s & (ERTS_PSFLG_SIG_Q|ERTS_PSFLG_SIG_IN_Q))
+                   == (ERTS_PSFLG_SIG_Q|ERTS_PSFLG_SIG_IN_Q));
+        }
+        if (proc->sig_inq.nmsigs.last == &proc->sig_inq.first)
+            proc->sig_qs.nmsigs.last = proc->sig_qs.cont_last;
+        else
+            proc->sig_qs.nmsigs.last = proc->sig_inq.nmsigs.last;
+        proc->sig_inq.nmsigs.next = NULL;
+        proc->sig_inq.nmsigs.last = NULL;
+
+        *proc->sig_qs.cont_last = proc->sig_inq.first;
+        proc->sig_qs.cont_last = proc->sig_inq.last;
+    }
+
+    proc->sig_qs.len += proc->sig_inq.len;
+
+    proc->sig_inq.first = NULL;
+    proc->sig_inq.last = &proc->sig_inq.first;
+    proc->sig_inq.len = 0;
+
+    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(proc);
+}
+
+void do_seq_trace_output(Eterm to, Eterm token, Eterm msg);
+
+static void
+send_gen_exit_signal(Process *c_p, Eterm from_tag,
+                     Eterm from, Eterm to,
+                     Sint16 op, Eterm reason, Eterm ref,
+                     Eterm token, int normal_kills)
+{
+    ErtsExitSignalData *xsigd;
+    Eterm *hp, *start_hp, s_reason, s_ref, s_message, s_token, s_from;
+    ErtsMessage *mp;
+    ErlHeapFragment *hfrag;
+    ErlOffHeap *ohp;
+    Uint hsz, from_sz, reason_sz, ref_sz, token_sz;
+    int seq_trace;
+#ifdef USE_VM_PROBES
+    Eterm s_utag, utag;
+    Uint utag_sz;
+#endif
+
+    ASSERT(is_immed(from_tag));
+
+    hsz = sizeof(ErtsExitSignalData)/sizeof(Uint);
+
+    seq_trace = c_p && have_seqtrace(token);
+    if (seq_trace)
+        seq_trace_update_send(c_p);
+
+#ifdef USE_VM_PROBES
+    if (c_p && token != NIL && (DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING)) {
+        utag_sz = size_object(DT_UTAG(c_p));
+        utag = DT_UTAG(c_p);
+    }
+    else if (token == am_have_dt_utag) {
+        utag_sz = 0;
+        utag = token = NIL;
+    }
+    hsz += utag_sz;
+#endif
+
+    token_sz = is_immed(token) ? 0 : size_object(token);
+    hsz += token_sz;
+
+    from_sz = is_immed(from) ? 0 : size_object(from);
+    hsz += from_sz;
+
+    reason_sz = is_immed(reason) ? 0 : size_object(reason);
+    hsz += reason_sz;
+
+    switch (op) {
+    case ERTS_SIG_Q_OP_EXIT:
+    case ERTS_SIG_Q_OP_EXIT_LINKED: {
+        /* {'EXIT', From, Reason} */
+        hsz += 4; /* 3-tuple */
+        ref_sz = 0;
+        break;
+    }
+    case ERTS_SIG_Q_OP_MONITOR_DOWN: {
+        /* {'DOWN', Ref, process, From, Reason} */
+        hsz += 6; /* 5-tuple */
+        ref_sz = NC_HEAP_SIZE(ref);
+        hsz += ref_sz;
+        break;
+    }
+    default:
+        ERTS_INTERNAL_ERROR("Invalid exit signal op");
+        break;
+    }
+
+    /*
+     * Allocate message combined with heap fragment...
+     */
+    mp = erts_alloc_message(hsz, &hp);
+    hfrag = &mp->hfrag;
+    mp->next = NULL;
+    ohp = &hfrag->off_heap;
+    start_hp = hp;
+
+    s_token = (is_immed(token)
+                ? token
+                : copy_struct(token, token_sz, &hp, ohp));
+
+    s_reason = (is_immed(reason)
+                ? reason
+                : copy_struct(reason, reason_sz, &hp, ohp));
+
+    s_from = (is_immed(from)
+              ? from
+              : copy_struct(from, from_sz, &hp, ohp));
+
+    if (!ref_sz)
+        s_ref = NIL;
+    else
+        s_ref = STORE_NC(&hp, ohp, ref);
+
+    switch (op) {
+    case ERTS_SIG_Q_OP_EXIT:
+    case ERTS_SIG_Q_OP_EXIT_LINKED:
+        /* {'EXIT', From, Reason} */
+        s_message = TUPLE3(hp, am_EXIT, s_from, s_reason);
+        hp += 4;
+        break;
+    case ERTS_SIG_Q_OP_MONITOR_DOWN:
+        /* {'DOWN', Ref, process, From, Reason} */
+        s_message = TUPLE5(hp, am_DOWN, s_ref, am_process, s_from, s_reason);
+        hp += 6;
+        break;
+    }
+
+#ifdef USE_VM_PROBES
+    s_utag = (is_immed(utag)
+              ? utag
+              : copy_struct(utag, utag_sz, &hp, ohp));
+    ERL_MESSAGE_DT_UTAG(mp) = utag;
+#endif
+
+    ERL_MESSAGE_TERM(mp) = ERTS_PROC_SIG_MAKE_TAG(op,
+                                                  ERTS_SIG_Q_TYPE_GEN_EXIT,
+                                                  0);
+    ERL_MESSAGE_TOKEN(mp) = s_token;
+    ERL_MESSAGE_FROM(mp) = from_tag; /* immediate... */
+
+    hfrag->used_size = hp - start_hp;
+
+    xsigd = (ErtsExitSignalData *) (char *) hp;
+
+    xsigd->message = s_message;
+    xsigd->from = s_from;
+    xsigd->reason = s_reason;
+    if (is_nil(s_ref))
+        xsigd->u.normal_kills = normal_kills;
+    else {
+        ASSERT(is_ref(s_ref));
+        xsigd->u.ref = s_ref;
+    }
+
+    if (seq_trace)
+        do_seq_trace_output(to, s_token, s_message);
+
+    if (!proc_queue_signal(c_p, to, (ErtsSignal *) mp, op)) {
+        mp->next = NULL;
+        erts_cleanup_messages(mp);
+    }
+}
+
+void
+do_seq_trace_output(Eterm to, Eterm token, Eterm msg)
+{
+    /*
+     * We could do this when enqueuing the signal and avoid some
+     * locking. However, the enqueuing code would then always
+     * have the penalty of this seq-tracing code which we do not
+     * want...
+     */
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    int is_normal_sched = !!esdp && esdp->type == ERTS_SCHED_NORMAL;
+    Process *rp;
+
+    if (is_normal_sched)
+        rp = erts_proc_lookup_raw(to);
+    else
+        rp = erts_proc_lookup_raw_inc_refc(to);
+
+    erts_proc_lock(rp, ERTS_PROC_LOCK_MSGQ);
+
+    if (!ERTS_PROC_IS_EXITING(rp))
+        seq_trace_output(token, msg, SEQ_TRACE_SEND, to, rp);
+
+    erts_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ);
+
+    if (!is_normal_sched)
+        erts_proc_dec_refc(rp);
+}
+
+void
+erts_proc_sig_send_persistent_monitor_msg(Uint16 type, Eterm key,
+                                          Eterm from, Eterm to,
+                                          Eterm msg, Uint msg_sz)
+{
+    ErtsPersistMonMsg *prst_mon;
+    ErtsMessage *mp;
+    ErlHeapFragment *hfrag;
+    Eterm *hp, *start_hp, message;
+    ErlOffHeap *ohp;
+    Uint hsz = sizeof(ErtsPersistMonMsg) + msg_sz;
+
+    /*
+     * Allocate message combined with heap fragment...
+     */
+    mp = erts_alloc_message(hsz, &hp);
+    hfrag = &mp->hfrag;
+    mp->next = NULL;
+    ohp = &hfrag->off_heap;
+    start_hp = hp;
+
+    ASSERT(msg_sz == size_object(msg));
+    message = copy_struct(msg, msg_sz, &hp, ohp);
+    hfrag->used_size = hp - start_hp;
+
+    prst_mon = (ErtsPersistMonMsg *) (char *) hp;
+    prst_mon->message = message;
+
+    switch (type) {
+    case ERTS_MON_TYPE_NODES:
+        ASSERT(is_small(key));
+        prst_mon->key = key;
+        break;
+
+    case ERTS_MON_TYPE_TIME_OFFSET:
+        ASSERT(is_internal_ref(key));
+        ASSERT(is_tuple_arity(message, 5));
+
+        prst_mon->key = tuple_val(message)[2];
+
+        ASSERT(eq(prst_mon->key, key));
+        break;
+
+    default:
+        ERTS_INTERNAL_ERROR("Invalid persistent monitor type");
+        prst_mon->key = key;
+        break;
+    }
+
+    ASSERT(is_immed(from));
+
+    ERL_MESSAGE_TERM(mp) = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_PERSISTENT_MON_MSG,
+                                                  type, 0);
+    ERL_MESSAGE_FROM(mp) = from;
+    ERL_MESSAGE_TOKEN(mp) = NIL;
+#ifdef USE_VM_PROBES
+    ERL_MESSAGE_DT_UTAG(mp) = NIL;
+#endif
+
+    if (!proc_queue_signal(NULL, to, (ErtsSignal *) mp,
+                           ERTS_SIG_Q_OP_PERSISTENT_MON_MSG)) {
+        mp->next = NULL;
+        erts_cleanup_messages(mp);
+    }
+}
+
+static ERTS_INLINE Eterm
+get_persist_mon_msg(ErtsMessage *sig, Eterm *msg)
+{
+    ErtsPersistMonMsg *prst_mon;
+    prst_mon = ((ErtsPersistMonMsg *)
+                (char *) (&sig->hfrag.mem[0]
+                          + sig->hfrag.used_size));
+    *msg = prst_mon->message;
+    return prst_mon->key;
+}
+
+void
+erts_proc_sig_send_exit(Process *c_p, Eterm from, Eterm to,
+                        Eterm reason, Eterm token,
+                        int normal_kills)
+{
+    Eterm from_tag;
+    ASSERT(!c_p || c_p->common.id == from);
+    if (is_immed(from)) {
+        ASSERT(is_internal_pid(from) || is_internal_port(from));
+        from_tag = from;
+    }
+    else {
+        DistEntry *dep;
+        ASSERT(is_external_pid(from));
+        dep = external_pid_dist_entry(from);
+        from_tag = dep->sysname;
+    }
+    send_gen_exit_signal(c_p, from_tag, from, to, ERTS_SIG_Q_OP_EXIT,
+                         reason, NIL, token, normal_kills);
+}
+
+void
+erts_proc_sig_send_link_exit(Process *c_p, Eterm from, ErtsLink *lnk,
+                             Eterm reason, Eterm token)
+{
+    Eterm to;
+    ASSERT(!c_p || c_p->common.id == from);
+    ASSERT(lnk);
+    to = lnk->other.item;
+    if (is_not_immed(reason) || is_not_nil(token)) {
+        ASSERT(is_internal_pid(from) || is_internal_port(from));
+        send_gen_exit_signal(c_p, from, from, to, ERTS_SIG_Q_OP_EXIT_LINKED,
+                             reason, NIL, token, 0);
+    }
+    else {
+        /* Pass signal using old link structure... */
+        ErtsSignal *sig = (ErtsSignal *) lnk;
+        lnk->other.item = reason; /* pass reason via this other.item */
+        sig->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_EXIT_LINKED,
+                                                 lnk->type, 0);
+        if (proc_queue_signal(c_p, to, sig, ERTS_SIG_Q_OP_EXIT_LINKED))
+            return; /* receiver will destroy lnk structure */
+    }
+    if (lnk)
+        erts_link_release(lnk);
+}
+
+int
+erts_proc_sig_send_link(Process *c_p, Eterm to, ErtsLink *lnk)
+{
+    ErtsSignal *sig;
+    Uint16 type = lnk->type;
+
+    ASSERT(!c_p || c_p->common.id == lnk->other.item);
+    ASSERT(lnk);
+    ASSERT(is_internal_pid(to));
+
+    sig = (ErtsSignal *) lnk;
+    sig->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_LINK,
+                                             type, 0);
+
+    return proc_queue_signal(c_p, to, sig, ERTS_SIG_Q_OP_LINK);
+}
+
+void
+erts_proc_sig_send_unlink(Process *c_p, ErtsLink *lnk)
+{
+    ErtsSignal *sig;
+    Eterm to;
+
+    ASSERT(lnk);
+
+    sig = (ErtsSignal *) lnk;
+    to = lnk->other.item;
+
+    ASSERT(is_internal_pid(to));
+
+    sig->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_UNLINK,
+                                             lnk->type, 0);
+
+    if (!proc_queue_signal(c_p, to, sig, ERTS_SIG_Q_OP_UNLINK))
+        erts_link_release(lnk);
+}
+
+void
+erts_proc_sig_send_dist_link_exit(DistEntry *dep,
+                                  Eterm from, Eterm to,
+                                  Eterm reason, Eterm token)
+{
+    send_gen_exit_signal(NULL, dep->sysname, from, to, ERTS_SIG_Q_OP_EXIT_LINKED,
+                         reason, NIL, token, 0);
+}
+
+void
+erts_proc_sig_send_dist_unlink(DistEntry *dep, Eterm from, Eterm to)
+{
+    ErtsSignal *sig;
+
+    ASSERT(is_internal_pid(to));
+    ASSERT(is_external_pid(from));
+    ASSERT(dep == external_pid_dist_entry(from));
+
+    sig = (ErtsSignal *) make_sig_dist_link_op(ERTS_SIG_Q_OP_UNLINK,
+                                               to, from);
+
+    if (!proc_queue_signal(NULL, to, sig, ERTS_SIG_Q_OP_UNLINK))
+        destroy_sig_dist_link_op((ErtsSigDistLinkOp *) sig);
+}
+
+void
+erts_proc_sig_send_dist_monitor_down(DistEntry *dep, Eterm ref,
+                                     Eterm from, Eterm to,
+                                     Eterm reason)
+{
+    Eterm monitored, heap[3];
+    if (is_atom(from)) 
+        monitored = TUPLE2(&heap[0], from, dep->sysname);
+    else
+        monitored = from;
+    send_gen_exit_signal(NULL, dep->sysname, monitored,
+                         to, ERTS_SIG_Q_OP_MONITOR_DOWN,
+                         reason, ref, NIL, 0);    
+}
+
+void
+erts_proc_sig_send_monitor_down(ErtsMonitor *mon, Eterm reason)
+{
+    Eterm to;
+
+    ASSERT(erts_monitor_is_target(mon));
+    ASSERT(!erts_monitor_is_in_table(mon));
+
+    to = mon->other.item;
+    ASSERT(is_internal_pid(to));
+
+    if (is_immed(reason)) {
+        /* Pass signal using old monitor structure... */
+        ErtsSignal *sig;
+
+        mon->other.item = reason; /* Pass immed reason via other.item... */
+        sig = (ErtsSignal *) mon;
+        sig->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_MONITOR_DOWN,
+                                                 mon->type, 0);
+        if (proc_queue_signal(NULL, to, sig, ERTS_SIG_Q_OP_MONITOR_DOWN))
+            return; /* receiver will destroy mon structure */
+    }
+    else {
+        ErtsMonitorData *mdp = erts_monitor_to_data(mon);
+        Eterm from_tag, monitored, heap[3];
+
+        if (!(mon->flags & ERTS_ML_FLG_NAME)) {
+            from_tag = monitored = mdp->origin.other.item;
+            if (is_external_pid(from_tag)) {
+                DistEntry *dep = external_pid_dist_entry(from_tag);
+                from_tag = dep->sysname;
+            }
+        }
+        else {
+            ErtsMonitorDataExtended *mdep;
+            Eterm name, node;
+            mdep = (ErtsMonitorDataExtended *) mdp;
+            name = mdep->u.name;
+            ASSERT(is_atom(name));
+            if (mdep->dist) {
+                node = mdep->dist->nodename;
+                from_tag = node;
+            }
+            else {
+                node = erts_this_dist_entry->sysname;
+                from_tag = mdp->origin.other.item;
+            }
+            ASSERT(is_internal_port(from_tag)
+                   || is_internal_pid(from_tag)
+                   || is_atom(from_tag));
+            monitored = TUPLE2(&heap[0], name, node);
+        }    
+        send_gen_exit_signal(NULL, from_tag, monitored,
+                             to, ERTS_SIG_Q_OP_MONITOR_DOWN,
+                             reason, mdp->ref, NIL, 0);
+    }
+    erts_monitor_release(mon);
+}
+
+void
+erts_proc_sig_send_dist_demonitor(Eterm to, Eterm ref)
+{
+    ErtsSigDistProcDemonitor *dmon;
+    ErtsSignal *sig;
+    Eterm *hp;
+    ErlOffHeap oh;
+    size_t size;
+
+    ERTS_INIT_OFF_HEAP(&oh);
+
+    ASSERT(is_internal_pid(to));
+
+    size = sizeof(ErtsSigDistProcDemonitor) - sizeof(Eterm);
+    ASSERT(is_ref(ref));
+    size += NC_HEAP_SIZE(ref)*sizeof(Eterm);
+
+    dmon = erts_alloc(ERTS_ALC_T_DIST_DEMONITOR, size);
+
+    hp = &dmon->heap[0];
+    dmon->ref = STORE_NC(&hp, &oh, ref);
+    sig = (ErtsSignal *) dmon;
+
+    sig->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_DEMONITOR,
+                                             ERTS_SIG_Q_TYPE_DIST_PROC_DEMONITOR,
+                                             0);
+
+    if (!proc_queue_signal(NULL, to, sig, ERTS_SIG_Q_OP_DEMONITOR))
+        destroy_dist_proc_demonitor(dmon);
+}
+
+void
+erts_proc_sig_send_demonitor(ErtsMonitor *mon)
+{
+    ErtsSignal *sig = (ErtsSignal *) mon;
+    Uint16 type = mon->type;
+    Eterm to = mon->other.item;
+
+    ASSERT(is_internal_pid(to));
+    ASSERT(erts_monitor_is_origin(mon));
+    ASSERT(!erts_monitor_is_in_table(mon));
+
+    sig->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_DEMONITOR,
+                                             type, 0);
+    
+    if (!proc_queue_signal(NULL, to, sig, ERTS_SIG_Q_OP_DEMONITOR))
+        erts_monitor_release(mon);
+}
+
+int
+erts_proc_sig_send_monitor(ErtsMonitor *mon, Eterm to)
+{
+    ErtsSignal *sig = (ErtsSignal *) mon;
+    Uint16 type = mon->type;
+
+    ASSERT(is_internal_pid(to) || to == am_undefined);
+    ASSERT(erts_monitor_is_target(mon));
+
+    sig->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_MONITOR,
+                                             type, 0);
+    
+    return proc_queue_signal(NULL, to, sig, ERTS_SIG_Q_OP_MONITOR);
+}
+
+void
+erts_proc_sig_send_trace_change(Eterm to, Uint on, Uint off, Eterm tracer)
+{
+    ErtsSigTraceInfo *ti;
+    Eterm tag;
+
+    ti = erts_alloc(ERTS_ALC_T_SIG_DATA, sizeof(ErtsSigTraceInfo));
+    tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_TRACE_CHANGE_STATE,
+                                 ERTS_SIG_Q_TYPE_ADJUST_TRACE_INFO,
+                                 0);
+
+    ti->common.tag = tag;
+    ti->flags_off = off;
+    ti->flags_on = on;
+    ti->tracer = NIL;
+    if (is_not_nil(tracer))
+        erts_tracer_update(&ti->tracer, tracer);
+
+    if (!proc_queue_signal(NULL, to, (ErtsSignal *) ti,
+                           ERTS_SIG_Q_OP_TRACE_CHANGE_STATE))
+        destroy_trace_info(ti);
+}
+
+void
+erts_proc_sig_send_group_leader(Process *c_p, Eterm to, Eterm gl, Eterm ref)
+{
+    int res;
+    ErtsSigGroupLeader *sgl;
+    Eterm *hp;
+    Uint gl_sz, ref_sz, size;
+    erts_aint_t init_flags = ERTS_SIG_GL_FLG_ACTIVE|ERTS_SIG_GL_FLG_RECEIVER;
+    if (c_p)
+        init_flags |= ERTS_SIG_GL_FLG_SENDER;
+
+    ASSERT(c_p ? is_internal_ref(ref) : ref == NIL);
+
+    gl_sz = is_immed(gl) ? 0 : size_object(gl);
+    ref_sz = is_immed(ref) ? 0 : size_object(ref);
+    
+    size = sizeof(ErtsSigGroupLeader);
+
+    size += (gl_sz + ref_sz - 1) * sizeof(Eterm);
+
+    sgl = erts_alloc(ERTS_ALC_T_SIG_DATA, size);
+
+    erts_atomic_init_nob(&sgl->flags, init_flags);
+
+    ERTS_INIT_OFF_HEAP(&sgl->oh);
+
+    hp = &sgl->heap[0];
+
+    sgl->group_leader = is_immed(gl) ? gl : copy_struct(gl, gl_sz, &hp, &sgl->oh);
+    sgl->reply_to = c_p ? c_p->common.id : NIL;
+    sgl->ref = is_immed(ref) ? ref : copy_struct(ref, ref_sz, &hp, &sgl->oh);
+
+    sgl->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_GROUP_LEADER,
+                                             ERTS_SIG_Q_TYPE_UNDEFINED,
+                                             0);
+
+    res = proc_queue_signal(c_p, to, (ErtsSignal *) sgl,
+                            ERTS_SIG_Q_OP_GROUP_LEADER);
+
+    if (!res)
+        destroy_sig_group_leader(sgl);
+    else if (c_p) {
+        int prio_res = !0;
+        erts_aint_t flags, rm_flags = ERTS_SIG_GL_FLG_SENDER;
+        Process *rp;
+        erts_aint32_t state, my_prio, other_prio;
+
+        state = erts_atomic32_read_nob(&c_p->state);
+        my_prio = ERTS_PSFLGS_GET_USR_PRIO(state);
+
+        rp = erts_proc_lookup_raw(to);
+        if (!rp)
+            prio_res = 0;
+        else {
+            state = erts_atomic32_read_nob(&rp->state);
+            other_prio = ERTS_PSFLGS_GET_USR_PRIO(state);
+
+            if (other_prio > my_prio) {
+                /* Others prio is lower than mine; elevate it... */
+                prio_res = erts_sig_prio(to, my_prio);
+                if (prio_res) {
+                    state = erts_atomic32_read_nob(&rp->state);
+                    ensure_dirty_proc_handled(to, state, my_prio);
+                }
+            }
+        }
+        if (!prio_res)
+            rm_flags |= ERTS_SIG_GL_FLG_ACTIVE;
+
+        flags = erts_atomic_read_band_nob(&sgl->flags, ~rm_flags);
+        if (!prio_res && (flags & ERTS_SIG_GL_FLG_ACTIVE))
+            res = 0; /* We deactivated signal... */
+        if ((flags & ~rm_flags) == 0)
+            destroy_sig_group_leader(sgl);
+    }
+
+    if (!res && c_p)
+        group_leader_reply(c_p, c_p->common.id, ref, 0);
+}
+
+static ERTS_INLINE void
+adjust_tracing_state(Process *c_p, ErtsSigRecvTracing *tracing, int setup)
+{
+    if (!IS_TRACED(c_p) || (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE)) {
+        tracing->messages.active = 0;
+        tracing->messages.receive_trace = 0;
+        tracing->messages.event = NULL;
+        tracing->messages.next = NULL;
+        tracing->procs = 0;
+        tracing->active = 0;
+    }
+    else {
+        Uint flgs = ERTS_TRACE_FLAGS(c_p);
+        int procs_trace = !!(flgs & F_TRACE_PROCS);
+        int recv_trace = !!(flgs & F_TRACE_RECEIVE);
+        /* procs tracing enabled? */
+
+        tracing->procs = procs_trace;
+
+        /* message receive tracing enabled? */
+        tracing->messages.receive_trace = recv_trace;
+        if (!recv_trace)
+            tracing->messages.event = NULL;
+        else {
+            if (tracing->messages.bp_ix < 0)
+                tracing->messages.bp_ix = erts_active_bp_ix();
+            tracing->messages.event = &erts_receive_tracing[tracing->messages.bp_ix];
+        }
+        if (setup) {
+            if (recv_trace)
+                tracing->messages.next = &c_p->sig_qs.cont;
+            else
+                tracing->messages.next = NULL;
+        }
+        tracing->messages.active = recv_trace;
+        tracing->active = recv_trace | procs_trace;
+    }
+
+#if defined(USE_VM_PROBES)
+    /* vm probe message_queued enabled? */
+
+    tracing->messages.vm_probes = DTRACE_ENABLED(message_queued);
+    if (tracing->messages.vm_probes) {
+        dtrace_proc_str(c_p, tracing->messages.receiver_name);
+        tracing->messages.active = !0;
+        tracing->active = !0;
+        if (setup && !tracing->messages.next)
+            tracing->messages.next = &c_p->sig_qs.cont;
+    }
+
+#endif
+}
+
+static ERTS_INLINE void
+setup_tracing_state(Process *c_p, ErtsSigRecvTracing *tracing)
+{
+    tracing->messages.bp_ix = -1;
+    adjust_tracing_state(c_p, tracing, !0);
+}
+
+static ERTS_INLINE void
+remove_iq_sig(Process *c_p, ErtsMessage *sig, ErtsMessage **next_sig)
+{
+    /*
+     * Remove signal from inner queue.
+     */
+    ASSERT(c_p->sig_qs.cont_last != &sig->next);
+    ASSERT(c_p->sig_qs.nmsigs.next != &sig->next);
+    ASSERT(c_p->sig_qs.nmsigs.last != &sig->next);
+
+    if (c_p->sig_qs.save == &sig->next)
+        c_p->sig_qs.save = next_sig;
+    if (c_p->sig_qs.last == &sig->next)
+        c_p->sig_qs.last = next_sig;
+    if (c_p->sig_qs.saved_last == &sig->next)
+        c_p->sig_qs.saved_last = next_sig;
+
+    *next_sig = sig->next;
+}
+
+static ERTS_INLINE void
+remove_mq_sig(Process *c_p, ErtsMessage *sig,
+           ErtsMessage **next_sig, ErtsMessage ***next_nm_sig)
+{
+    /*
+     * Remove signal from middle queue.
+     */
+    ASSERT(c_p->sig_qs.save != &sig->next);
+    ASSERT(c_p->sig_qs.last != &sig->next);
+
+    if (c_p->sig_qs.cont_last == &sig->next)
+	c_p->sig_qs.cont_last = next_sig;
+    if (c_p->sig_qs.saved_last == &sig->next)
+        c_p->sig_qs.saved_last = next_sig;
+    if (*next_nm_sig == &sig->next)
+        *next_nm_sig = next_sig;
+    if (c_p->sig_qs.nmsigs.last == &sig->next)
+        c_p->sig_qs.nmsigs.last = next_sig;
+
+    *next_sig = sig->next;
+}
+
+static ERTS_INLINE void
+remove_nm_sig(Process *c_p, ErtsMessage *sig, ErtsMessage ***next_nm_sig)
+{
+    ErtsMessage **next_sig = *next_nm_sig;
+    ASSERT(ERTS_SIG_IS_NON_MSG(sig));
+    ASSERT(*next_sig == sig);
+    *next_nm_sig = ((ErtsSignal *) sig)->common.specific.next;
+    remove_mq_sig(c_p, sig, next_sig, next_nm_sig);
+}
+
+static ERTS_INLINE void
+convert_to_msg(Process *c_p, ErtsMessage *sig, ErtsMessage *msg,
+               ErtsMessage ***next_nm_sig)
+{
+    ErtsMessage **next_sig = *next_nm_sig;
+    ASSERT(ERTS_SIG_IS_NON_MSG(sig));
+    *next_nm_sig = ((ErtsSignal *) sig)->common.specific.next;
+    c_p->sig_qs.len++;
+    *next_sig = msg;
+    remove_mq_sig(c_p, sig, &msg->next, next_nm_sig);
+}
+
+static ERTS_INLINE void
+convert_to_msgs(Process *c_p, ErtsMessage *sig, Uint no_msgs,
+                ErtsMessage *first_msg, ErtsMessage *last_msg,
+                ErtsMessage ***next_nm_sig)
+{
+    ErtsMessage **next_sig = *next_nm_sig;
+    ASSERT(ERTS_SIG_IS_NON_MSG(sig));
+    *next_nm_sig = ((ErtsSignal *) sig)->common.specific.next;
+    c_p->sig_qs.len += no_msgs;
+    *next_sig = first_msg;
+    remove_mq_sig(c_p, sig, &last_msg->next, next_nm_sig);
+}
+
+static ERTS_INLINE void
+insert_messages(Process *c_p, ErtsMessage **next, ErtsMessage *first,
+                ErtsMessage *last, Uint no_msgs, ErtsMessage ***next_nm_sig)
+{
+    last->next = *next;
+    if (c_p->sig_qs.cont_last == next)
+	c_p->sig_qs.cont_last = &last->next;
+    if (*next_nm_sig == next)
+        *next_nm_sig = &last->next;
+    if (c_p->sig_qs.nmsigs.last == next)
+        c_p->sig_qs.nmsigs.last = &last->next;
+    c_p->sig_qs.len += no_msgs;
+    *next = first;
+}
+
+static ERTS_INLINE void
+remove_mq_m_sig(Process *c_p, ErtsMessage *sig, ErtsMessage **next_sig, ErtsMessage ***next_nm_sig)
+{
+    /* Removing message... */
+    ASSERT(!ERTS_SIG_IS_NON_MSG(sig));
+    ASSERT(c_p->sig_qs.len > 0);
+    c_p->sig_qs.len--;
+    remove_mq_sig(c_p, sig, next_sig, next_nm_sig);
+}
+
+static ERTS_INLINE void
+remove_iq_m_sig(Process *c_p, ErtsMessage *sig, ErtsMessage **next_sig)
+{
+    /* Removing message... */
+    ASSERT(!ERTS_SIG_IS_NON_MSG(sig));
+    ASSERT(c_p->sig_qs.len > 0);
+    c_p->sig_qs.len--;
+    remove_iq_sig(c_p, sig, next_sig);
+}
+
+static ERTS_INLINE void
+convert_prepared_sig_to_msg(Process *c_p, ErtsMessage *sig, Eterm msg,
+                            ErtsMessage ***next_nm_sig)
+{
+    /*
+     * Everything is already there except for the reference to
+     * the message and the combined hfrag marker that needs to be
+     * restored...
+     */
+    *next_nm_sig = ((ErtsSignal *) sig)->common.specific.next;
+    sig->data.attached = ERTS_MSG_COMBINED_HFRAG;
+    ERL_MESSAGE_TERM(sig) = msg;
+    c_p->sig_qs.len++;
+}
+
+static ERTS_INLINE int
+handle_exit_signal(Process *c_p, ErtsSigRecvTracing *tracing,
+                   ErtsMessage *sig, ErtsMessage ***next_nm_sig,
+                   int *exited)
+{
+    ErtsMessage *conv_msg = NULL;
+    ErtsExitSignalData *xsigd = NULL;
+    ErtsLinkData *ldp;
+    ErtsLink *dlnk;
+    Eterm tag = ((ErtsSignal *) sig)->common.tag;
+    Uint16 type = ERTS_PROC_SIG_TYPE(tag);
+    int op = ERTS_PROC_SIG_OP(tag);
+    int destroy = 0;
+    int ignore = 0;
+    int save = 0;
+    int exit = 0;
+    int cnt = 1;
+    Eterm reason;
+    Eterm from;
+
+    if (type == ERTS_SIG_Q_TYPE_GEN_EXIT) {
+        xsigd = get_exit_signal_data(sig);
+        from = xsigd->from;
+        reason = xsigd->reason;
+        if (op != ERTS_SIG_Q_OP_EXIT_LINKED)
+            ignore = 0;
+        else {
+            ErtsLink *llnk = erts_link_tree_lookup(ERTS_P_LINKS(c_p), from);
+            if (!llnk) {
+                /* Link no longer active; ignore... */
+                ignore = !0;
+                destroy = !0;
+                ldp = NULL; /* Avoid erroneous warning... */
+                dlnk = NULL; /* Avoid erroneous warning... */
+            }
+            else {
+                ignore = 0;
+                erts_link_tree_delete(&ERTS_P_LINKS(c_p), llnk);
+                if (llnk->type != ERTS_LNK_TYPE_DIST_PROC)
+                    erts_link_release(llnk);
+                else {
+                    dlnk = erts_link_to_other(llnk, &ldp);
+                    if (erts_link_dist_delete(dlnk))
+                        erts_link_release_both(ldp);
+                    else
+                        erts_link_release(llnk);
+                }
+            }
+        }
+
+        if (!ignore) {
+
+            if ((op != ERTS_SIG_Q_OP_EXIT || reason != am_kill)
+                && (c_p->flags & F_TRAP_EXIT)) {
+                convert_prepared_sig_to_msg(c_p, sig,
+                                            xsigd->message, next_nm_sig);
+                conv_msg = sig;
+            }
+            else if (reason == am_normal && !xsigd->u.normal_kills) {
+                /* Ignore it... */
+                destroy = !0;
+                ignore = !0;
+            }
+            else {
+                /* Terminate... */
+                save = !0;
+                exit = !0;
+                if (op == ERTS_SIG_Q_OP_EXIT && reason == am_kill)
+                    reason = am_killed;
+            }
+        }
+    }
+    else { /* Link exit */
+        ErtsLink *slnk = (ErtsLink *) sig;
+        ErtsLink *llnk = erts_link_to_other(slnk, &ldp);
+
+        ASSERT(type == ERTS_LNK_TYPE_PROC
+               || type == ERTS_LNK_TYPE_PORT
+               || type == ERTS_LNK_TYPE_DIST_PROC);
+
+        from = llnk->other.item;
+        reason = slnk->other.item; /* reason in other.item ... */
+        ASSERT(is_pid(from) || is_internal_port(from));
+        ASSERT(is_immed(reason));
+        ASSERT(op == ERTS_SIG_Q_OP_EXIT_LINKED);
+        dlnk = erts_link_tree_key_delete(&ERTS_P_LINKS(c_p), llnk);
+        if (!dlnk) {
+            ignore = !0; /* Link no longer active; ignore... */
+            ldp = NULL;
+        }
+        else {
+            Eterm pid;
+            ErtsMessage *mp;
+            ErtsProcLocks locks;
+            Uint hsz;
+            Eterm *hp;
+            ErlOffHeap *ohp;
+            ignore = 0;
+            if (dlnk == llnk)
+                dlnk = NULL;
+            else
+                ldp = NULL;
+
+            ASSERT(is_immed(reason));
+
+            if (!(c_p->flags & F_TRAP_EXIT)) {
+                if (reason == am_normal)
+                    ignore = !0; /* Ignore it... */
+                else
+                    exit = !0; /* Terminate... */
+            }
+            else {
+
+                /*
+                 * Create and EXIT message and replace
+                 * the original signal with the message...
+                 */
+
+                locks = ERTS_PROC_LOCK_MAIN;
+
+                hsz = 4 + NC_HEAP_SIZE(from);
+
+                mp = erts_alloc_message_heap(c_p, &locks, hsz, &hp, &ohp);
+
+                if (locks != ERTS_PROC_LOCK_MAIN)
+                    erts_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN);
+
+                pid = STORE_NC(&hp, ohp, from);
+
+                ERL_MESSAGE_TERM(mp) = TUPLE3(hp, am_EXIT, pid, reason);
+                ERL_MESSAGE_TOKEN(mp) = NIL;
+#ifdef USE_VM_PROBES
+                ERL_MESSAGE_DT_UTAG(mp) = NIL;
+#endif
+                if (is_immed(pid))
+                    ERL_MESSAGE_FROM(mp) = pid;
+                else {
+                    DistEntry *dep;
+                    ASSERT(is_external_pid(pid));
+                    dep = external_pid_dist_entry(pid);
+                    ERL_MESSAGE_FROM(mp) = dep->sysname;
+                }
+
+                /* Replace original signal with the exit message... */
+                convert_to_msg(c_p, sig, mp, next_nm_sig);
+
+                cnt += 4;
+
+                conv_msg = mp;
+            }
+        }
+        destroy = !0;
+    }
+
+    if (ignore|exit) {
+        remove_nm_sig(c_p, sig, next_nm_sig);
+        if (exit) {
+            if (save) {
+                sig->data.attached = ERTS_MSG_COMBINED_HFRAG;
+                ERL_MESSAGE_TERM(sig) = xsigd->message;
+                erts_save_message_in_proc(c_p, sig);
+            }
+            /* Exit process... */
+            erts_set_self_exiting(c_p, reason);
+
+            cnt++;
+        }
+    }
+
+    if (!exit) {
+        if (conv_msg)
+            erts_proc_notify_new_message(c_p, ERTS_PROC_LOCK_MAIN);
+        if (op == ERTS_SIG_Q_OP_EXIT_LINKED && tracing->procs)
+            getting_unlinked(c_p, from);
+    }
+
+    if (destroy) {
+        cnt++;
+        if (type == ERTS_SIG_Q_TYPE_GEN_EXIT) {
+            sig->next = NULL;
+            erts_cleanup_messages(sig);
+        }
+        else {
+            if (ldp)
+                erts_link_release_both(ldp);
+            else {
+                if (dlnk)
+                    erts_link_release(dlnk);
+                erts_link_release((ErtsLink *) sig);
+            }
+        }
+    }
+
+    *exited = exit;
+
+    return cnt;
+}
+
+static ERTS_INLINE int
+convert_prepared_down_message(Process *c_p, ErtsMessage *sig,
+                              Eterm msg, ErtsMessage ***next_nm_sig)
+{
+    convert_prepared_sig_to_msg(c_p, sig, msg, next_nm_sig);
+    erts_proc_notify_new_message(c_p, ERTS_PROC_LOCK_MAIN);
+    return 1;
+}
+
+static int
+convert_to_down_message(Process *c_p,
+                        ErtsMessage *sig,
+                        ErtsMonitorData *mdp,
+                        Uint16 mon_type,
+                        ErtsMessage ***next_nm_sig)
+{
+    /*
+     * Create a 'DOWN' message and replace the signal
+     * with it...
+     */
+    int cnt = 0;
+    Eterm node = am_undefined;
+    ErtsMessage *mp;
+    ErtsProcLocks locks;
+    Uint hsz;
+    Eterm *hp, ref, from, type, reason;
+    ErlOffHeap *ohp;
+
+    ASSERT(mdp);
+    ASSERT((mdp->origin.flags & ERTS_ML_FLGS_SAME)
+           == (mdp->target.flags & ERTS_ML_FLGS_SAME));
+
+    hsz = 6; /* 5-tuple */
+
+    if (mdp->origin.flags & ERTS_ML_FLG_NAME)
+        hsz += 3;  /* reg name 2-tuple */
+    else {
+        ASSERT(is_pid(mdp->origin.other.item)
+               || is_internal_port(mdp->origin.other.item));
+        hsz += NC_HEAP_SIZE(mdp->origin.other.item);
+    }
+
+    ASSERT(is_ref(mdp->ref));
+    hsz += NC_HEAP_SIZE(mdp->ref);
+
+    locks = ERTS_PROC_LOCK_MAIN;
+
+    /* reason is mdp->target.other.item */
+    reason = mdp->target.other.item;
+    ASSERT(is_immed(reason));
+
+    mp = erts_alloc_message_heap(c_p, &locks, hsz, &hp, &ohp);
+
+    if (locks != ERTS_PROC_LOCK_MAIN)
+        erts_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN);
+
+    cnt += 4;
+
+    ref = STORE_NC(&hp, ohp, mdp->ref);
+
+    if (!(mdp->origin.flags & ERTS_ML_FLG_NAME)) {
+        from = STORE_NC(&hp, ohp, mdp->origin.other.item);
+    }
+    else {
+        ErtsMonitorDataExtended *mdep;
+        ASSERT(mdp->origin.flags & ERTS_ML_FLG_EXTENDED);
+        mdep = (ErtsMonitorDataExtended *) mdp;
+        ASSERT(is_atom(mdep->u.name));
+        if (mdep->dist)
+            node = mdep->dist->nodename;
+        else
+            node = erts_this_dist_entry->sysname;
+        from = TUPLE2(hp, mdep->u.name, node);
+        hp += 3;
+    }
+
+    ASSERT(mdp->origin.type == mon_type);
+    switch (mon_type) {
+    case ERTS_MON_TYPE_PORT:
+        type = am_port;
+        if (mdp->origin.other.item == am_undefined) {
+            /* failed by name... */
+            ERL_MESSAGE_FROM(mp) = am_system;
+        }
+        else {
+            ASSERT(is_internal_port(mdp->origin.other.item));
+            ERL_MESSAGE_FROM(mp) = mdp->origin.other.item;
+        }
+        break;
+    case ERTS_MON_TYPE_PROC:
+        type = am_process;
+        if (mdp->origin.other.item == am_undefined) {
+            /* failed by name... */
+            ERL_MESSAGE_FROM(mp) = am_system;
+        }
+        else {
+            ASSERT(is_internal_pid(mdp->origin.other.item));
+            ERL_MESSAGE_FROM(mp) = mdp->origin.other.item;
+        }
+        break;
+    case ERTS_MON_TYPE_DIST_PROC:
+        type = am_process;
+        if (node == am_undefined) {
+            ErtsMonitorDataExtended *mdep;
+            ASSERT(mdp->origin.flags & ERTS_ML_FLG_EXTENDED);
+            mdep = (ErtsMonitorDataExtended *) mdp;
+            ASSERT(mdep->dist);
+            node = mdep->dist->nodename;
+        }
+        ASSERT(is_atom(node) && node != am_undefined);
+        ERL_MESSAGE_FROM(mp) = node;
+        break;
+    default:
+        ERTS_INTERNAL_ERROR("Unexpected monitor type");
+        type = am_undefined;
+        ERL_MESSAGE_FROM(mp) = am_undefined;
+        break;
+    }
+
+    ERL_MESSAGE_TERM(mp) = TUPLE5(hp, am_DOWN, ref,
+                                  type, from, reason);
+    hp += 6;
+
+    ERL_MESSAGE_TOKEN(mp) = NIL;
+#ifdef USE_VM_PROBES
+    ERL_MESSAGE_DT_UTAG(mp) = NIL;
+#endif
+    /* Replace original signal with the exit message... */
+    convert_to_msg(c_p, sig, mp, next_nm_sig);
+
+    cnt += 4;
+
+    erts_proc_notify_new_message(c_p, ERTS_PROC_LOCK_MAIN);
+
+    return cnt;
+}
+
+static ERTS_INLINE int
+convert_to_nodedown_messages(Process *c_p,
+                             ErtsMessage *sig,
+                             ErtsMonitorData *mdp,
+                             ErtsMessage ***next_nm_sig)
+{
+    int cnt = 1;
+    Uint n;
+    ErtsMonitorDataExtended *mdep = (ErtsMonitorDataExtended *) mdp;
+
+    ASSERT((mdp->origin.flags & ERTS_ML_FLGS_SAME)
+           == (mdp->target.flags & ERTS_ML_FLGS_SAME));
+    ASSERT(mdp->origin.flags & ERTS_ML_FLG_EXTENDED);
+
+    n = mdep->u.refc;
+
+    if (n == 0)
+        remove_nm_sig(c_p, sig, next_nm_sig);
+    else {
+        Uint i;
+        ErtsMessage *nd_first = NULL;
+        ErtsMessage *nd_last = NULL;
+        ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN;
+        Eterm node = mdep->dist->nodename;
+
+        ASSERT(is_atom(node));
+        ASSERT(n > 0);
+
+        for (i = 0; i < n; i++) {
+            ErtsMessage *mp;
+            ErlOffHeap *ohp;
+            Eterm *hp;
+
+            mp = erts_alloc_message_heap(c_p, &locks, 3, &hp, &ohp);
+
+            ERL_MESSAGE_TERM(mp) = TUPLE2(hp, am_nodedown, node);
+            ERL_MESSAGE_FROM(mp) = am_system;
+            ERL_MESSAGE_TOKEN(mp) = NIL;
+#ifdef USE_VM_PROBES
+            ERL_MESSAGE_DT_UTAG(mp) = NIL;
+#endif
+            mp->next = nd_first;
+            nd_first = mp;
+            if (!nd_last)
+                nd_last = mp;
+            cnt++;
+        }
+
+        if (locks != ERTS_PROC_LOCK_MAIN)
+            erts_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN);
+
+        /* Replace signal with 'nodedown' messages */
+        convert_to_msgs(c_p, sig, n, nd_first, nd_last, next_nm_sig);
+
+        erts_proc_notify_new_message(c_p, ERTS_PROC_LOCK_MAIN);
+    }
+    return cnt;
+}
+
+static int 
+handle_nodedown(Process *c_p,
+                ErtsMessage *sig,
+                ErtsMonitorData *mdp,
+                ErtsMessage ***next_nm_sig)
+{
+    ErtsMonitorDataExtended *mdep = (ErtsMonitorDataExtended *) mdp;
+    ErtsMonitor *omon = &mdp->origin;
+    int not_in_subtab = !(omon->flags & ERTS_ML_FLG_IN_SUBTABLE);
+    int cnt = 1;
+
+    ASSERT(erts_monitor_is_in_table(omon));
+
+    if (not_in_subtab & !mdep->uptr.node_monitors)
+        erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), omon);
+    else if (not_in_subtab) {
+        ErtsMonitor *sub_mon;
+        ErtsMonitorDataExtended *sub_mdep;
+        sub_mon = erts_monitor_list_last(mdep->uptr.node_monitors);
+        ASSERT(sub_mon);
+        erts_monitor_list_delete(&mdep->uptr.node_monitors, sub_mon);
+        sub_mon->flags &= ~ERTS_ML_FLG_IN_SUBTABLE;
+        sub_mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(sub_mon);
+        ASSERT(!sub_mdep->uptr.node_monitors);
+        sub_mdep->uptr.node_monitors = mdep->uptr.node_monitors;
+        mdep->uptr.node_monitors = NULL;
+        erts_monitor_tree_replace(&ERTS_P_MONITORS(c_p), omon, sub_mon);
+        cnt += 2;
+    }
+    else {
+        ErtsMonitorDataExtended *top_mdep;
+        ErtsMonitor *top_mon;
+        ASSERT(is_atom(omon->other.item));
+        ASSERT(!mdep->uptr.node_monitors);
+        top_mon = erts_monitor_tree_lookup(ERTS_P_MONITORS(c_p),
+                                           omon->other.item);
+        ASSERT(top_mon);
+        top_mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(top_mon);
+        ASSERT(top_mdep->uptr.node_monitors);
+        erts_monitor_list_delete(&top_mdep->uptr.node_monitors, omon);
+        omon->flags &= ~ERTS_ML_FLG_IN_SUBTABLE;
+        cnt += 3;
+    }
+
+    return cnt + convert_to_nodedown_messages(c_p, sig, mdp, next_nm_sig);
+}
+
+static void
+handle_persistent_mon_msg(Process *c_p, Uint16 type,
+                          ErtsMonitor *mon, ErtsMessage *sig,
+                          Eterm msg, ErtsMessage ***next_nm_sig)
+{
+    convert_prepared_sig_to_msg(c_p, sig, msg, next_nm_sig);
+
+    switch (type) {
+
+    case ERTS_MON_TYPE_TIME_OFFSET:
+        ASSERT(mon->type == ERTS_MON_TYPE_TIME_OFFSET);
+        break;
+
+    case ERTS_MON_TYPE_NODES: {
+        ErtsMonitorDataExtended *mdep;
+        Uint n;
+        ASSERT(mon->type == ERTS_MON_TYPE_NODES);
+        mdep = (ErtsMonitorDataExtended *) erts_monitor_to_data(mon);
+        ERTS_ML_ASSERT(mdep->u.refc > 0);
+        n = mdep->u.refc;
+        n--;
+        if (n > 0) {
+            ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN;
+            ErtsMessage *first = NULL, *prev, *last;
+            Uint hsz = size_object(msg);
+            Uint i;
+
+            for (i = 0; i < n; i++) {
+                Eterm *hp;
+                ErlOffHeap *ohp;
+
+                last = erts_alloc_message_heap(c_p, &locks, hsz, &hp, &ohp);
+
+                if (!first)
+                    first = last;
+                else
+                    prev->next = last;
+                prev = last;
+
+                ERL_MESSAGE_TERM(last) = copy_struct(msg, hsz, &hp, ohp);
+
+#ifdef USE_VM_PROBES
+                ASSERT(is_immed(ERL_MESSAGE_DT_UTAG(sig)));
+                ERL_MESSAGE_DT_UTAG(last) = ERL_MESSAGE_DT_UTAG(sig);
+#endif
+                ASSERT(is_immed(ERL_MESSAGE_TOKEN(sig)));
+                ERL_MESSAGE_TOKEN(last) = ERL_MESSAGE_TOKEN(sig);
+                ASSERT(is_immed(ERL_MESSAGE_FROM(sig)));
+                ERL_MESSAGE_FROM(last) = ERL_MESSAGE_FROM(sig);
+
+            }
+            if (locks != ERTS_PROC_LOCK_MAIN)
+                erts_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN);
+            insert_messages(c_p, &sig->next, first, last, n, next_nm_sig);
+        }
+        break;
+    }
+
+    default:
+        ERTS_INTERNAL_ERROR("Invalid type");
+        break;
+    }
+
+    erts_proc_notify_new_message(c_p, ERTS_PROC_LOCK_MAIN);
+}
+
+static void
+group_leader_reply(Process *c_p, Eterm to, Eterm ref, int success)
+{
+    Process *rp = erts_proc_lookup(to);
+
+    if (rp) {
+        ErtsProcLocks locks;
+        Uint sz;
+        Eterm *hp, msg, ref_cpy, result;
+        ErlOffHeap *ohp;
+        ErtsMessage *mp;
+
+        ASSERT(is_internal_ref(ref));
+
+        locks = c_p == rp ? ERTS_PROC_LOCK_MAIN : 0;
+        sz = size_object(ref);
+
+        mp = erts_alloc_message_heap(rp, &locks, sz+3,
+                                     &hp, &ohp);
+
+        ref_cpy = copy_struct(ref, sz, &hp, ohp);
+        result = success ? am_true : am_badarg;
+        msg = TUPLE2(hp, ref_cpy, result);
+
+        erts_queue_message(rp, locks, mp, msg, am_system);
+
+        if (c_p == rp)
+            locks &= ~ERTS_PROC_LOCK_MAIN;
+
+        if (locks)
+            erts_proc_unlock(rp, locks);
+    }
+}
+
+static void
+handle_group_leader(Process *c_p, ErtsSigGroupLeader *sgl)
+{
+    erts_aint_t flags;
+
+    flags = erts_atomic_read_band_nob(&sgl->flags, ~ERTS_SIG_GL_FLG_ACTIVE);
+    if (flags & ERTS_SIG_GL_FLG_ACTIVE) {
+        int res = erts_set_group_leader(c_p, sgl->group_leader);
+        if (is_internal_pid(sgl->reply_to))
+            group_leader_reply(c_p, sgl->reply_to, sgl->ref, res);
+    }
+
+    flags = erts_atomic_read_band_nob(&sgl->flags, ~ERTS_SIG_GL_FLG_RECEIVER);
+    if ((flags & ~ERTS_SIG_GL_FLG_RECEIVER) == 0)
+        destroy_sig_group_leader(sgl);
+}
+
+
+/*
+ * Called in order to handle incoming signals.
+ */
+
+int
+erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep,
+                              int *redsp, int max_reds, int local_only)
+{
+    Eterm tag;
+    erts_aint32_t state;
+    int cnt, limit, abs_lim, msg_tracing;
+    ErtsMessage *sig, ***next_nm_sig;
+    ErtsSigRecvTracing tracing;
+
+    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p);
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
+
+    if (local_only)
+        state = -1; /* can never be a valid state... */
+    else {
+        state = erts_atomic32_read_nob(&c_p->state);
+        if (ERTS_PSFLG_SIG_IN_Q & state) {
+            erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
+            erts_proc_sig_fetch(c_p);
+            erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
+        }
+    }
+
+    limit = *redsp;
+    *redsp = 0;
+
+    if (!c_p->sig_qs.cont) {
+        if (state == -1)
+            *statep = erts_atomic32_read_nob(&c_p->state);
+        else
+            *statep = state;
+        return !0;
+    }
+
+    next_nm_sig = &c_p->sig_qs.nmsigs.next;
+
+    setup_tracing_state(c_p, &tracing);
+    msg_tracing = tracing.messages.active;
+
+    limit *= ERTS_SIG_REDS_CNT_FACTOR;
+    abs_lim = ERTS_SIG_REDS_CNT_FACTOR*max_reds;
+    if (limit > abs_lim)
+        limit = abs_lim;
+
+    cnt = 0;
+
+    do {
+
+        if (msg_tracing) {
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            if (handle_msg_tracing(c_p, &tracing, next_nm_sig) != 0) {
+                ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+                break; /* tracing limit or end... */
+            }
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+        }
+
+        if (!*next_nm_sig)
+            break;
+
+        sig = **next_nm_sig;
+
+        ASSERT(sig);
+        ASSERT(ERTS_SIG_IS_NON_MSG(sig));
+
+        tag = ((ErtsSignal *) sig)->common.tag;
+ 
+        switch (ERTS_PROC_SIG_OP(tag)) {
+
+        case ERTS_SIG_Q_OP_EXIT:
+        case ERTS_SIG_Q_OP_EXIT_LINKED: {
+            int exited;
+
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+
+            cnt += handle_exit_signal(c_p, &tracing, sig,
+                                      next_nm_sig, &exited);
+
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+
+            if (exited)
+                goto stop; /* terminated by signal */
+            /* ignored or converted to exit message... */
+            break;
+        }
+
+        case ERTS_SIG_Q_OP_MONITOR_DOWN: {
+            Uint16 type = ERTS_PROC_SIG_TYPE(tag);
+            ErtsExitSignalData *xsigd = NULL;
+            ErtsMonitorData *mdp = NULL;
+            ErtsMonitor *omon = NULL, *tmon = NULL;
+
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+
+            switch (type) {
+            case ERTS_MON_TYPE_DIST_PROC:
+            case ERTS_MON_TYPE_PROC:
+            case ERTS_MON_TYPE_PORT:
+                tmon = (ErtsMonitor *) sig;
+                ASSERT(erts_monitor_is_target(tmon));
+                ASSERT(!erts_monitor_is_in_table(tmon));
+                mdp = erts_monitor_to_data(tmon);
+                if (erts_monitor_is_in_table(&mdp->origin)) {
+                    omon = &mdp->origin;
+                    erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p),
+                                             omon);
+                    cnt += convert_to_down_message(c_p, sig, mdp,
+                                                   type, next_nm_sig);
+                }
+                break;
+            case ERTS_SIG_Q_TYPE_GEN_EXIT:
+                xsigd = get_exit_signal_data(sig);
+                omon = erts_monitor_tree_lookup(ERTS_P_MONITORS(c_p),
+                                                xsigd->u.ref);
+                if (omon) {
+                    ASSERT(erts_monitor_is_origin(omon));
+                    if (omon->type == ERTS_MON_TYPE_DIST_PROC) {
+                        mdp = erts_monitor_to_data(omon);
+                        if (erts_monitor_dist_delete(&mdp->target))
+                            tmon = &mdp->target;
+                    }
+                    erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p),
+                                             omon);
+                    cnt += convert_prepared_down_message(c_p, sig,
+                                                         xsigd->message,
+                                                         next_nm_sig);
+                }
+                break;
+            case ERTS_MON_TYPE_NODE:
+                tmon = (ErtsMonitor *) sig;
+                ASSERT(erts_monitor_is_target(tmon));
+                ASSERT(!erts_monitor_is_in_table(tmon));
+                mdp = erts_monitor_to_data(tmon);
+                if (erts_monitor_is_in_table(&mdp->origin)) {
+                    omon = &mdp->origin;
+                    cnt += handle_nodedown(c_p, sig, mdp, next_nm_sig);
+                }
+                break;
+            default:
+                ERTS_INTERNAL_ERROR("invalid monitor type");
+                break;
+            }
+
+            if (omon) {
+                if (tmon)
+                    erts_monitor_release_both(mdp);
+                else
+                    erts_monitor_release(omon);
+            }
+            else {
+                remove_nm_sig(c_p, sig, next_nm_sig);
+                if (xsigd) {
+                    sig->next = NULL;
+                    erts_cleanup_messages(sig);
+                }
+                if (tmon)
+                    erts_monitor_release(tmon);
+            }
+
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            break;
+        }
+
+        case ERTS_SIG_Q_OP_PERSISTENT_MON_MSG: {
+            Uint16 type = ERTS_PROC_SIG_TYPE(tag);
+            ErtsMonitor *mon;
+            Eterm msg;
+            Eterm key;
+
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+
+            key = get_persist_mon_msg(sig, &msg);
+
+            cnt++;
+            mon = erts_monitor_tree_lookup(ERTS_P_MONITORS(c_p), key);
+            if (mon) {
+                ASSERT(erts_monitor_is_origin(mon));
+                handle_persistent_mon_msg(c_p, type, mon, sig,
+                                          msg, next_nm_sig);
+            }
+            else {
+                cnt++;
+                remove_nm_sig(c_p, sig, next_nm_sig);
+                sig->next = NULL;
+                erts_cleanup_messages(sig);
+            }
+
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            break;
+        }
+
+        case ERTS_SIG_Q_OP_MONITOR: {
+            ErtsMonitor *mon = (ErtsMonitor *) sig;
+
+            ASSERT(erts_monitor_is_target(mon));
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+
+            remove_nm_sig(c_p, sig, next_nm_sig);
+
+            if (mon->type == ERTS_MON_TYPE_DIST_PROC)
+                erts_monitor_tree_insert(&ERTS_P_MONITORS(c_p), mon);
+            else
+                erts_monitor_list_insert(&ERTS_P_LT_MONITORS(c_p), mon);
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            break;
+        }
+
+        case ERTS_SIG_Q_OP_DEMONITOR: {
+            Uint16 type = ERTS_PROC_SIG_TYPE(tag);
+
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+
+            remove_nm_sig(c_p, sig, next_nm_sig);
+
+            if (type == ERTS_SIG_Q_TYPE_DIST_PROC_DEMONITOR) {
+                ErtsMonitor *tmon;
+                ErtsSigDistProcDemonitor *dmon;
+                dmon = (ErtsSigDistProcDemonitor *) sig;
+                tmon = erts_monitor_tree_lookup(ERTS_P_MONITORS(c_p), dmon->ref);
+                destroy_dist_proc_demonitor(dmon);
+                cnt++;
+                if (tmon) {
+                    ErtsMonitorData *mdp = erts_monitor_to_data(tmon);
+                    ASSERT(erts_monitor_is_target(tmon));
+                    erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), tmon);
+                    if (!erts_monitor_dist_delete(&mdp->origin))
+                        erts_monitor_release(tmon);
+                    else
+                        erts_monitor_release_both(mdp);
+                    cnt += 2;
+                }
+            }
+            else {
+                ErtsMonitor *omon = (ErtsMonitor *) sig;
+                ErtsMonitorData *mdp = erts_monitor_to_data(omon);
+                ASSERT(omon->type == type);
+                ASSERT(erts_monitor_is_origin(omon));
+                ASSERT(!erts_monitor_is_in_table(omon));
+                if (!erts_monitor_is_in_table(&mdp->target))
+                    erts_monitor_release(omon);
+                else {
+                    ErtsMonitor *tmon = &mdp->target;
+                    ASSERT(tmon->type == type);
+                    if (type == ERTS_MON_TYPE_DIST_PROC)
+                        erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), tmon);
+                    else {
+                        erts_monitor_list_delete(&ERTS_P_LT_MONITORS(c_p), tmon);
+                        if (type == ERTS_MON_TYPE_RESOURCE) {
+                            erts_nif_demonitored((ErtsResource *) tmon->other.ptr);
+                            cnt++;
+                        }
+                    }
+                    erts_monitor_release_both(mdp);
+                    cnt++;
+                }
+                cnt++;
+            }
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            break;
+        }
+
+        case ERTS_SIG_Q_OP_LINK: {
+            ErtsLink *rlnk, *lnk = (ErtsLink *) sig;
+
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+
+            remove_nm_sig(c_p, sig, next_nm_sig);
+            rlnk = erts_link_tree_insert_addr_replace(&ERTS_P_LINKS(c_p),
+                                                      lnk);
+            if (!rlnk) {
+                if (tracing.procs)
+                    getting_linked(c_p, lnk->other.item);
+            }
+            else {
+                if (rlnk->type != ERTS_LNK_TYPE_DIST_PROC)
+                    erts_link_release(rlnk);
+                else {
+                    ErtsLinkData *ldp;
+                    ErtsLink *dlnk = erts_link_to_other(rlnk, &ldp);
+                    if (erts_link_dist_delete(dlnk))
+                        erts_link_release_both(ldp);
+                    else
+                        erts_link_release(rlnk);
+                }
+            }
+
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            break;
+        }
+
+        case ERTS_SIG_Q_OP_UNLINK: {
+            Uint16 type = ERTS_PROC_SIG_TYPE(tag);
+            ErtsLinkData *ldp;
+            ErtsLink *llnk;
+
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+
+            remove_nm_sig(c_p, sig, next_nm_sig);
+            if (type == ERTS_SIG_Q_TYPE_DIST_LINK) {
+                ErtsSigDistLinkOp *sdlnk = (ErtsSigDistLinkOp *) sig;
+                ASSERT(type == ERTS_SIG_Q_TYPE_DIST_LINK);
+                ASSERT(is_external_pid(sdlnk->remote));
+                llnk = erts_link_tree_lookup(ERTS_P_LINKS(c_p), sdlnk->remote);
+                if (llnk) {
+                    ErtsLink *dlnk = erts_link_to_other(llnk, &ldp);
+                    erts_link_tree_delete(&ERTS_P_LINKS(c_p), llnk);
+                    if (erts_link_dist_delete(dlnk))
+                        erts_link_release_both(ldp);
+                    else
+                        erts_link_release(llnk);
+                    cnt += 8;
+                    if (tracing.procs)
+                        getting_unlinked(c_p, sdlnk->remote);
+                }
+                destroy_sig_dist_link_op(sdlnk);
+                cnt++;
+            }
+            else {
+                ErtsLinkData *ldp;
+                ErtsLink *dlnk, *slnk;
+                slnk = (ErtsLink *) sig;
+                llnk = erts_link_to_other(slnk, &ldp);
+                dlnk = erts_link_tree_key_delete(&ERTS_P_LINKS(c_p), llnk);
+                if (!dlnk)
+                    erts_link_release(slnk);
+                else {
+                    if (tracing.procs)
+                        getting_unlinked(c_p, llnk->other.item);
+                    if (dlnk == llnk)
+                        erts_link_release_both(ldp);
+                    else {
+                        erts_link_release(slnk);
+                        erts_link_release(dlnk);
+                    }
+                }
+                cnt += 2;
+            }
+
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            break;
+        }
+
+        case ERTS_SIG_Q_OP_GROUP_LEADER: {
+            ErtsSigGroupLeader *sgl = (ErtsSigGroupLeader *) sig;
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            remove_nm_sig(c_p, sig, next_nm_sig);
+            handle_group_leader(c_p, sgl);
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            break;
+        }
+
+        case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE: {
+            Uint16 type = ERTS_PROC_SIG_TYPE(tag);
+
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            msg_tracing = handle_trace_change_state(c_p, &tracing,
+                                                    type, sig,
+                                                    next_nm_sig);
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            
+            break;
+        }
+
+        default:
+            ERTS_INTERNAL_ERROR("Unknown signal");
+            break;
+        }
+
+        cnt++;
+
+    } while (cnt <= limit || stretch_limit(c_p, &tracing, abs_lim, &limit));
+
+stop: {
+        int deferred_save, deferred_saved_last, res;
+
+        deferred_saved_last = !!(c_p->flags & F_DEFERRED_SAVED_LAST);
+        deferred_save = 0;
+
+        if (!deferred_saved_last)
+            deferred_save = 0;
+        else {
+            if (c_p->sig_qs.saved_last == &c_p->sig_qs.cont) {
+                c_p->sig_qs.saved_last = c_p->sig_qs.last;
+                c_p->flags &= ~F_DEFERRED_SAVED_LAST;
+                deferred_saved_last = deferred_save = 0;
+            }
+            else {
+                if (c_p->sig_qs.save == c_p->sig_qs.last)
+                    deferred_save = !0;
+                else
+                    deferred_save = 0;
+            }
+        }
+
+        ASSERT(c_p->sig_qs.saved_last != &c_p->sig_qs.cont);
+
+        if (ERTS_UNLIKELY(msg_tracing != 0)) {
+            /*
+             * All messages that has been traced should
+             * be moved to inner queue. Next signal in
+             * middle queue should either be next message
+             * to trace or next non-message signal.
+             */
+            ASSERT(tracing.messages.next);
+            if (*next_nm_sig) {
+                if (*next_nm_sig == tracing.messages.next)
+                    *next_nm_sig = &c_p->sig_qs.cont;
+                if (c_p->sig_qs.nmsigs.last == tracing.messages.next)
+                    c_p->sig_qs.nmsigs.last = &c_p->sig_qs.cont;
+                *statep = erts_atomic32_read_nob(&c_p->state);
+            }
+            else {
+                ASSERT(!c_p->sig_qs.nmsigs.next);
+                c_p->sig_qs.nmsigs.last = NULL;
+                state = erts_atomic32_read_band_nob(&c_p->state,
+                                                    ~ERTS_PSFLG_SIG_Q);
+                state &= ~ERTS_PSFLG_SIG_Q;
+                *statep = state;
+            }
+
+            if (tracing.messages.next != &c_p->sig_qs.cont) {
+                *c_p->sig_qs.last = c_p->sig_qs.cont;
+                c_p->sig_qs.last = tracing.messages.next;
+
+                c_p->sig_qs.cont = *tracing.messages.next;
+                if (!c_p->sig_qs.cont)
+                    c_p->sig_qs.cont_last = &c_p->sig_qs.cont;
+                *c_p->sig_qs.last = NULL;
+            }
+
+            res = !c_p->sig_qs.cont;
+        }
+        else if (*next_nm_sig) {
+            /*
+             * All messages prior to next non-message
+             * signal should be moved to inner queue.
+             * Next non-message signal to handle should
+             * be first in middle queue.
+             */
+            ASSERT(**next_nm_sig);
+            if (*next_nm_sig != &c_p->sig_qs.cont) {
+                *c_p->sig_qs.last = c_p->sig_qs.cont;
+                c_p->sig_qs.last = *next_nm_sig;
+
+                c_p->sig_qs.cont = **next_nm_sig;
+                if (c_p->sig_qs.nmsigs.last == *next_nm_sig)
+                    c_p->sig_qs.nmsigs.last = &c_p->sig_qs.cont;
+                *next_nm_sig = &c_p->sig_qs.cont;
+                *c_p->sig_qs.last = NULL;
+            }
+
+            ASSERT(c_p->sig_qs.cont);
+
+            *statep = erts_atomic32_read_nob(&c_p->state);
+
+            res = 0;
+        }
+        else {
+            /*
+             * All non-message signals handled. All
+             * messages should be moved to inner queue.
+             * Middle queue should be empty.
+             */
+            ASSERT(!c_p->sig_qs.nmsigs.next);
+            c_p->sig_qs.nmsigs.last = NULL;
+
+            if (c_p->sig_qs.cont_last != &c_p->sig_qs.cont) {
+                ASSERT(!*c_p->sig_qs.last);
+                *c_p->sig_qs.last = c_p->sig_qs.cont;
+                c_p->sig_qs.last = c_p->sig_qs.cont_last;
+                ASSERT(!*c_p->sig_qs.last);
+
+                c_p->sig_qs.cont_last = &c_p->sig_qs.cont;
+                c_p->sig_qs.cont = NULL;
+            }
+
+            ASSERT(!c_p->sig_qs.cont);
+
+            state = erts_atomic32_read_band_nob(&c_p->state,
+                                                ~ERTS_PSFLG_SIG_Q);
+            state &= ~ERTS_PSFLG_SIG_Q;
+            *statep = state;
+            res = !0;
+        }
+
+        if (deferred_saved_last
+            && (c_p->sig_qs.saved_last == &c_p->sig_qs.cont)) {
+            c_p->sig_qs.saved_last = c_p->sig_qs.last;
+            c_p->flags &= ~F_DEFERRED_SAVED_LAST;
+            if (deferred_save)
+                c_p->sig_qs.save = c_p->sig_qs.saved_last;
+        }
+        else if (!res) {
+            if (deferred_save) {
+                c_p->sig_qs.save = c_p->sig_qs.last;
+                ASSERT(!PEEK_MESSAGE(c_p));
+            }
+        }
+        else {
+            c_p->flags &= ~F_DEFERRED_SAVED_LAST;
+            if (deferred_save)
+                c_p->sig_qs.save = c_p->sig_qs.saved_last;
+        }
+
+        ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p);
+
+        *redsp = cnt/4 + 1;
+
+        return res;
+    }
+}
+
+static int
+stretch_limit(Process *c_p, ErtsSigRecvTracing *tp,
+              int abs_lim, int *limp)
+{
+    int lim;
+    /*
+     * Stretch limit up to a maximum of 'abs_lim' if
+     * there currently are no messages available to
+     * inspect by 'receive' and it might be possible
+     * to get messages available by processing
+     * signals (or trace messages).
+     */
+
+    lim = *limp;
+    ASSERT(abs_lim >= lim);
+    if (abs_lim == lim)
+        return 0;
+
+    if (!(c_p->flags & F_DEFERRED_SAVED_LAST)) {
+        ErtsSignal *sig;
+
+        if (PEEK_MESSAGE(c_p))
+            return 0;
+        sig = (ErtsSignal *) c_p->sig_qs.cont;
+        if (!sig)
+            return 0; /* No signals to process available... */
+        if (ERTS_SIG_IS_MSG(sig) && tp->messages.next != &c_p->sig_qs.cont)
+            return 0;
+    }
+
+    lim += ERTS_SIG_REDS_CNT_FACTOR*100;
+    if (lim > abs_lim)
+        lim = abs_lim;
+    *limp = lim;
+    return !0;
+}
+
+
+int
+erts_proc_sig_handle_exit(Process *c_p, int *redsp)
+{
+    int cnt, limit;
+    ErtsMessage *sig, ***next_nm_sig;
+
+    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p);
+    ERTS_LC_ASSERT(!erts_proc_lc_my_proc_locks(c_p));
+
+    ASSERT(!(ERTS_PSFLG_SIG_IN_Q & erts_atomic32_read_nob(&c_p->state)));
+
+    limit = *redsp;
+    limit *= ERTS_SIG_REDS_CNT_FACTOR;
+
+    *redsp = 1;
+
+    next_nm_sig = &c_p->sig_qs.nmsigs.next;
+
+    if (!*next_nm_sig) {
+        ASSERT(!c_p->sig_qs.nmsigs.last);
+        return !0; /* done... */
+    }
+
+    cnt = 0;
+
+    ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, NULL, next_nm_sig);
+
+    do {
+        Eterm tag;
+        Uint16 type;
+        int op;
+
+        sig = **next_nm_sig;
+
+        ASSERT(sig);
+        ASSERT(ERTS_SIG_IS_NON_MSG(sig));
+
+        tag = ((ErtsSignal *) sig)->common.tag;
+        type = ERTS_PROC_SIG_TYPE(tag);
+        op = ERTS_PROC_SIG_OP(tag);
+
+        remove_nm_sig(c_p, sig, next_nm_sig);
+ 
+        ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, NULL, next_nm_sig);
+
+        cnt++;
+
+        switch (op) {
+
+        case ERTS_SIG_Q_OP_EXIT:
+        case ERTS_SIG_Q_OP_EXIT_LINKED:
+        case ERTS_SIG_Q_OP_MONITOR_DOWN:
+            switch (type) {
+            case ERTS_SIG_Q_TYPE_GEN_EXIT:
+                sig->next = NULL;
+                erts_cleanup_messages(sig);
+                break;
+            case ERTS_LNK_TYPE_PORT:
+            case ERTS_LNK_TYPE_PROC:
+            case ERTS_LNK_TYPE_DIST_PROC:
+                erts_link_release((ErtsLink *) sig);
+                break;
+            case ERTS_MON_TYPE_PORT:
+            case ERTS_MON_TYPE_PROC:
+            case ERTS_MON_TYPE_DIST_PROC:
+            case ERTS_MON_TYPE_NODE:
+                erts_monitor_release((ErtsMonitor *) sig);
+                break;
+            default:
+                ERTS_INTERNAL_ERROR("Unexpected sig type");
+                break;
+            }
+            break;
+
+        case ERTS_SIG_Q_OP_PERSISTENT_MON_MSG:
+            sig->next = NULL;
+            erts_cleanup_messages(sig);
+            break;
+
+        case ERTS_SIG_Q_OP_MONITOR: {
+            ErtsProcExitContext pectxt = {c_p, am_noproc};
+            erts_proc_exit_handle_monitor((ErtsMonitor *) sig,
+                                          (void *) &pectxt);
+            cnt += 4;
+            break;
+        }
+
+        case ERTS_SIG_Q_OP_DEMONITOR:
+            if (type == ERTS_SIG_Q_TYPE_DIST_PROC_DEMONITOR)
+                destroy_dist_proc_demonitor((ErtsSigDistProcDemonitor *) sig);
+            else
+                erts_monitor_release((ErtsMonitor *) sig);
+            break;
+
+        case ERTS_SIG_Q_OP_LINK: {
+            ErtsProcExitContext pectxt = {c_p, am_noproc};
+            erts_proc_exit_handle_link((ErtsLink *) sig, (void *) &pectxt);
+            break;
+        }
+
+        case ERTS_SIG_Q_OP_UNLINK:
+            if (type == ERTS_SIG_Q_TYPE_DIST_LINK)
+                destroy_sig_dist_link_op((ErtsSigDistLinkOp *) sig);
+            else
+                erts_link_release((ErtsLink *) sig);
+            break;
+
+        case ERTS_SIG_Q_OP_GROUP_LEADER: {
+            ErtsSigGroupLeader *sgl = (ErtsSigGroupLeader *) sig;
+            handle_group_leader(c_p, sgl);
+            break;
+        }
+
+        case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE:
+            destroy_trace_info((ErtsSigTraceInfo *) sig);
+            break;
+
+        default:
+            ERTS_INTERNAL_ERROR("Unknown signal");
+            break;
+        }
+
+    } while (cnt >= limit && *next_nm_sig);
+
+    *redsp += cnt / ERTS_SIG_REDS_CNT_FACTOR;
+
+    if (*next_nm_sig)
+        return 0;
+
+    ASSERT(!c_p->sig_qs.nmsigs.next);
+    c_p->sig_qs.nmsigs.last = NULL;
+    (void) erts_atomic32_read_band_nob(&c_p->state,
+                                       ~ERTS_PSFLG_SIG_Q);
+    return !0;
+}
+
+#ifdef USE_VM_PROBES
+#  define ERTS_CLEAR_SEQ_TOKEN(MP)                                      \
+    ERL_MESSAGE_TOKEN((MP)) = ((ERL_MESSAGE_DT_UTAG((MP)) != NIL)       \
+                               ? am_have_dt_utag : NIL)
+#else
+#  define ERTS_CLEAR_SEQ_TOKEN(MP)                                      \
+    ERL_MESSAGE_TOKEN((MP)) = NIL
+#endif
+
+static ERTS_INLINE void
+clear_seq_trace_token(ErtsMessage *sig)
+{
+    if (ERTS_SIG_IS_MSG((ErtsSignal *) sig))
+        ERTS_CLEAR_SEQ_TOKEN(sig);
+    else {
+        Uint tag;
+        Uint16 op, type;
+
+        tag = ((ErtsSignal *) sig)->common.tag;
+        type = ERTS_PROC_SIG_TYPE(tag);
+        op = ERTS_PROC_SIG_OP(tag);
+
+        switch (op) {
+
+        case ERTS_SIG_Q_OP_EXIT:
+        case ERTS_SIG_Q_OP_EXIT_LINKED:
+        case ERTS_SIG_Q_OP_MONITOR_DOWN:
+            switch (type) {
+            case ERTS_SIG_Q_TYPE_GEN_EXIT:
+                ERTS_CLEAR_SEQ_TOKEN(sig);
+                break;
+            case ERTS_LNK_TYPE_PORT:
+            case ERTS_LNK_TYPE_PROC:
+            case ERTS_LNK_TYPE_DIST_PROC:
+            case ERTS_MON_TYPE_PORT:
+            case ERTS_MON_TYPE_PROC:
+            case ERTS_MON_TYPE_DIST_PROC:
+            case ERTS_MON_TYPE_NODE:
+                break;
+            default:
+                ERTS_INTERNAL_ERROR("Unexpected sig type");
+                break;
+            }
+            break;
+
+        case ERTS_SIG_Q_OP_PERSISTENT_MON_MSG:
+            ERTS_CLEAR_SEQ_TOKEN(sig);
+            break;
+
+        case ERTS_SIG_Q_OP_MONITOR:
+        case ERTS_SIG_Q_OP_DEMONITOR:
+        case ERTS_SIG_Q_OP_LINK:
+        case ERTS_SIG_Q_OP_UNLINK:
+        case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE:
+            break;
+
+        default:
+            ERTS_INTERNAL_ERROR("Unknown signal");
+            break;
+        }
+    }
+}
+
+void
+erts_proc_sig_clear_seq_trace_tokens(Process *c_p)
+{
+    ASSERT(erts_thr_progress_is_blocking());
+    erts_proc_sig_fetch(c_p);
+    ERTS_FOREACH_SIG_PRIVQS(c_p, sig, clear_seq_trace_token(sig));
+}
+
+Uint
+erts_proc_sig_signal_size(ErtsSignal *sig)
+{
+    Eterm tag;
+    Uint16 type;
+    int op;
+    Uint size = 0;
+
+    ASSERT(sig);
+    ASSERT(ERTS_SIG_IS_NON_MSG(sig));
+
+    tag = sig->common.tag;
+    type = ERTS_PROC_SIG_TYPE(tag);
+    op = ERTS_PROC_SIG_OP(tag);
+
+    switch (op) {
+    case ERTS_SIG_Q_OP_EXIT:
+    case ERTS_SIG_Q_OP_EXIT_LINKED:
+    case ERTS_SIG_Q_OP_MONITOR_DOWN:
+        switch (type) {
+        case ERTS_SIG_Q_TYPE_GEN_EXIT:
+            size = ((ErtsMessage *) sig)->hfrag.alloc_size;
+            size *= sizeof(Eterm);
+            size += sizeof(ErtsMessage) - sizeof(Eterm);
+            break;
+        case ERTS_LNK_TYPE_PORT:
+        case ERTS_LNK_TYPE_PROC:
+        case ERTS_LNK_TYPE_DIST_PROC:
+            size = erts_link_size((ErtsLink *) sig);
+            break;
+        case ERTS_MON_TYPE_PORT:
+        case ERTS_MON_TYPE_PROC:
+        case ERTS_MON_TYPE_DIST_PROC:
+        case ERTS_MON_TYPE_NODE:
+            size = erts_monitor_size((ErtsMonitor *) sig);
+        default:
+            ERTS_INTERNAL_ERROR("Unexpected sig type");
+            break;
+        }
+        break;
+
+    case ERTS_SIG_Q_OP_PERSISTENT_MON_MSG:
+        size = ((ErtsMessage *) sig)->hfrag.alloc_size;
+        size *= sizeof(Eterm);
+        size += sizeof(ErtsMessage) - sizeof(Eterm);
+        break;
+
+    case ERTS_SIG_Q_OP_DEMONITOR:
+        if (type == ERTS_SIG_Q_TYPE_DIST_PROC_DEMONITOR) {
+            size = NC_HEAP_SIZE(((ErtsSigDistProcDemonitor *) sig)->ref);
+            size--;
+            size *= sizeof(Eterm);
+            size += sizeof(ErtsSigDistProcDemonitor);
+            break;
+        }
+        /* Fall through... */
+
+    case ERTS_SIG_Q_OP_MONITOR:
+        size = erts_monitor_size((ErtsMonitor *) sig);
+        break;
+
+    case ERTS_SIG_Q_OP_UNLINK:
+        if (type == ERTS_SIG_Q_TYPE_DIST_LINK) {
+            size = NC_HEAP_SIZE(((ErtsSigDistLinkOp *) sig)->remote);
+            size--;
+            size *= sizeof(Eterm);
+            size += sizeof(ErtsSigDistLinkOp);
+            break;
+        }
+        /* Fall through... */
+
+    case ERTS_SIG_Q_OP_LINK:
+        size = erts_link_size((ErtsLink *) sig);
+        break;
+
+    case ERTS_SIG_Q_OP_GROUP_LEADER: {
+        ErtsSigGroupLeader *sgl = (ErtsSigGroupLeader *) sig;
+        size = size_object(sgl->group_leader);
+        size += size_object(sgl->ref);
+        size *= sizeof(Eterm);
+        size += sizeof(ErtsSigGroupLeader) - sizeof(Eterm);
+        break;
+    }
+
+    case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE:
+        size = sizeof(ErtsSigTraceInfo);
+        break;
+
+    default:
+        ERTS_INTERNAL_ERROR("Unknown signal");
+        break;
+    }
+
+    return size;
+}
+
+int
+erts_proc_sig_receive_helper(Process *c_p,
+                             int fcalls,
+                             int neg_o_reds,
+                             ErtsMessage **msgpp,
+                             int *get_outp)
+{
+    ErtsMessage *msgp;
+    int reds, consumed_reds, left_reds, max_reds;
+
+    /*
+     * Called from the loop-rec instruction when receive
+     * has reached end of inner (private) queue. This function
+     * tries to move more messages into the inner queue
+     * for the receive to handle. This by, processing the
+     * middle (private) queue and/or moving signals from
+     * the outer (public) queue into the middle queue.
+     *
+     * If this function succeeds in making more messages
+     * available in the inner queue, *msgpp points to next
+     * message. If *msgpp is set to NULL when:
+     * -- process became exiting. *get_outp is set to a
+     *    value greater than zero.
+     * -- process needs to yield. *get_outp is set to a
+     *    value less than zero.
+     * -- no more messages exist in any of the queues.
+     *    *get_outp is set to zero and the message queue
+     *    lock remains locked. This so the process can
+     *    make its way to the appropriate wait instruction
+     *    without racing with new incoming messages.
+     */
+
+    ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
+    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+    ASSERT(!*msgpp);
+
+    left_reds = fcalls - neg_o_reds;
+    consumed_reds = 0;
+
+    while (!0) {
+        erts_aint32_t state;
+
+        if (!c_p->sig_qs.cont) {
+
+            consumed_reds += 4;
+            left_reds -= 4;
+            erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
+            if (c_p->sig_inq.first)
+                erts_proc_sig_fetch(c_p);
+            /*
+             * Messages may have been moved directly to
+             * inner queue...
+             */
+            msgp = PEEK_MESSAGE(c_p);
+            if (msgp) {
+                erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
+                *get_outp = 0;
+                *msgpp = msgp;
+                return consumed_reds;
+            }
+
+            if (!c_p->sig_qs.cont) {
+                /*
+                 * No messages! Return with message queue
+                 * locked and let the process continue
+                 * to wait instruction...
+                 */
+                *get_outp = 0;
+                *msgpp = NULL;
+                return consumed_reds;
+            }
+            erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
+
+            if (left_reds <= 0) {
+                *get_outp = -1; /* yield */
+                *msgpp = NULL;
+
+                ASSERT(consumed_reds >= (fcalls - neg_o_reds));
+                return consumed_reds;
+            }
+
+            /* handle newly arrived signals... */
+        }
+
+        reds = ERTS_SIG_HANDLE_REDS_MAX_PREFERED;
+#ifdef DEBUG
+        /* test that it works also with very few reds */
+        max_reds = left_reds;
+        if (reds > left_reds)
+            reds = left_reds;
+#else
+        /* At least work preferred amount of reds... */
+        max_reds = left_reds;
+        if (max_reds < reds)
+            max_reds = reds;
+#endif
+        (void) erts_proc_sig_handle_incoming(c_p, &state, &reds,
+                                             max_reds, !0);
+        consumed_reds += reds;
+        left_reds -= reds;
+        /* we may have exited by an incoming signal... */
+        if (state & ERTS_PSFLG_EXITING) {
+            /*
+             * Process need to schedule out in order
+             * to terminate. Prepare this a bit...
+             */
+            ASSERT(c_p->flags & F_DELAY_GC);
+
+            c_p->flags &= ~F_DELAY_GC;
+            c_p->arity = 0;
+            c_p->current = NULL;
+            *get_outp = 1;
+            *msgpp = NULL;
+            return consumed_reds;
+        }
+
+        msgp = PEEK_MESSAGE(c_p);
+        if (msgp) {
+            *get_outp = 0;
+            *msgpp = msgp;
+            return consumed_reds;
+        }
+
+        if (left_reds <= 0) {
+            *get_outp = -1; /* yield */
+            *msgpp = NULL;
+
+            ASSERT(consumed_reds >= (fcalls - neg_o_reds));
+            return consumed_reds;
+        }
+
+        ASSERT(!c_p->sig_qs.cont);
+        /* Go fetch again... */
+    }
+}
+
+static int
+handle_trace_change_state(Process *c_p,
+                          ErtsSigRecvTracing *tracing,
+                          Uint16 type,
+                          ErtsMessage *sig,
+                          ErtsMessage ***next_nm_sig)
+{
+    ErtsSigTraceInfo *trace_info = (ErtsSigTraceInfo *) sig;
+    ErtsMessage **next = *next_nm_sig;
+    int msgs_active, old_msgs_active = !!tracing->messages.active;
+
+    ASSERT(sig == *next);
+
+    erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+
+    ERTS_TRACE_FLAGS(c_p) |= trace_info->flags_on;
+    ERTS_TRACE_FLAGS(c_p) &= ~trace_info->flags_off;
+    if (is_value(trace_info->tracer))
+        erts_tracer_replace(&c_p->common, trace_info->tracer);
+
+    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+
+    remove_nm_sig(c_p, sig, next_nm_sig);
+    destroy_trace_info(trace_info);
+    /*
+     * Adjust tracing state according to modifications made by
+     * the trace info signal...
+     */
+    adjust_tracing_state(c_p, tracing, 0);
+    msgs_active = !!tracing->messages.active;
+
+    if (old_msgs_active ^ msgs_active) {
+        if (msgs_active) {
+            ASSERT(!tracing->messages.next);
+            tracing->messages.next = next;
+        }
+        else {
+            ASSERT(tracing->messages.next);
+            tracing->messages.next = NULL;
+        }
+    }
+
+    ASSERT(!msgs_active || tracing->messages.next);
+
+    return msgs_active;
+}
+
+static void
+getting_unlinked(Process *c_p, Eterm unlinker)
+{
+    trace_proc(c_p, ERTS_PROC_LOCK_MAIN, c_p,
+               am_getting_unlinked, unlinker);
+}
+
+static void
+getting_linked(Process *c_p, Eterm linker)
+{
+    trace_proc(c_p, ERTS_PROC_LOCK_MAIN, c_p,
+               am_getting_linked, linker);
+}
+
+static ERTS_INLINE void
+handle_message_enqueued_tracing(Process *c_p,
+                                ErtsSigRecvTracing *tracing,
+                                ErtsMessage *msg)
+{
+    ASSERT(ERTS_SIG_IS_INTERNAL_MSG(msg));
+
+#if defined(USE_VM_PROBES)
+    if (tracing->messages.vm_probes && DTRACE_ENABLED(message_queued)) {
+        Sint tok_label = 0;
+        Sint tok_lastcnt = 0;
+        Sint tok_serial = 0;
+        Eterm seq_trace_token = ERL_MESSAGE_TOKEN(msg);
+
+        if (seq_trace_token != NIL && is_tuple(seq_trace_token)) {
+            tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token));
+            tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token));
+            tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token));
+        }
+        /* Message intentionally not passed... */
+        DTRACE6(message_queued,
+                tracing->messages.receiver_name,
+                size_object(ERL_MESSAGE_TERM(msg)),
+                c_p->sig_qs.len,
+                tok_label, tok_lastcnt, tok_serial);
+    }
+#endif
+
+    if (tracing->messages.receive_trace && tracing->messages.event->on) {
+        ASSERT(IS_TRACED(c_p));
+        trace_receive(c_p,
+                      ERL_MESSAGE_FROM(msg),
+                      ERL_MESSAGE_TERM(msg),
+                      tracing->messages.event);
+    }
+}
+
+static int
+handle_msg_tracing(Process *c_p, ErtsSigRecvTracing *tracing,
+                   ErtsMessage ***next_nm_sig)
+{
+    ErtsMessage **next_sig, *sig;
+    int cnt = 0, limit = ERTS_PROC_SIG_TRACE_COUNT_LIMIT;
+
+    ASSERT(tracing->messages.next);
+    next_sig = tracing->messages.next;
+    sig = *next_sig;
+
+    /*
+     * Receive tracing active. Handle all messages
+     * until next non-message signal...
+     */
+
+    while (sig && ERTS_SIG_IS_MSG(sig)) {
+        if (cnt > limit) {
+            tracing->messages.next = next_sig;
+            return -1; /* Yield... */
+        }
+        if (ERTS_SIG_IS_EXTERNAL_MSG(sig)) {
+            cnt++;
+            if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN,
+                                          sig, 0)) {
+                /* Bad dist message; remove it... */
+                remove_mq_m_sig(c_p, sig, next_sig, next_nm_sig);
+                sig = *next_sig;
+                continue;
+            }
+        }
+        handle_message_enqueued_tracing(c_p, tracing, sig);
+        cnt++;
+
+        next_sig = &(*next_sig)->next;
+        sig = *next_sig;
+    }
+
+    tracing->messages.next = next_sig;
+
+    if (!sig) {
+        ASSERT(!*next_nm_sig);
+        return 1; /* end... */
+    }
+
+    ASSERT(*next_nm_sig);
+    ASSERT(**next_nm_sig == sig);
+
+    /* Next signal a non-message signal... */
+    ASSERT(ERTS_SIG_IS_NON_MSG(sig));
+
+    /*
+     * Return and handle the non-message signal...
+     */
+
+    return 0;
+}
+
+/*
+ * ERTS_INSPECT_MSGQ_KEEP_OH_MSGS == 0 will move off heap messages
+ * into the heap of the inspected process if off_heap_message_queue
+ * is false when process_info(_, messages) is called. That is, the
+ * following GC will have more data in the rootset compared to the
+ * scenario when process_info(_, messages) had not been called.
+ *
+ * ERTS_INSPECT_MSGQ_KEEP_OH_MSGS != 0 will keep off heap messages
+ * off heap when process_info(_, messages) is called regardless of
+ * the off_heap_message_queue setting of the process. That is, it
+ * will change the following execution of the process as little as
+ * possible.
+ */
+#define ERTS_INSPECT_MSGQ_KEEP_OH_MSGS 1
+
+Uint
+erts_proc_sig_prep_msgq_for_inspection(Process *c_p, Process *rp,
+                                       ErtsProcLocks rp_locks,
+                                       ErtsMessageInfo *mip)
+{
+    Uint tot_heap_size;
+    ErtsMessage *mp, **mpp;
+    Sint i;
+    int self_on_heap;
+
+    /*
+     * Prepare the message queue for inspection
+     * by process_info().
+     *
+     *
+     * - Decode all messages on external format
+     * - Remove all corrupt dist messages from queue
+     * - Save pointer to, and heap size need of each
+     *   message in the mip array.
+     * - Return total heap size need for all messages
+     *   that needs to be copied.
+     *
+     * If ERTS_INSPECT_MSGQ_KEEP_OH_MSGS == 0:
+     * - In case off heap messages is disabled and
+     *   we are inspecting our own queue, move all
+     *   off heap data into the heap.
+     */
+
+    /*
+     * All non-message signals *need* to have been
+     * handled before calling this functions...
+     */
+    ASSERT(!rp->sig_qs.cont);
+    ASSERT(!rp->sig_qs.nmsigs.next && !rp->sig_qs.nmsigs.last);
+
+    self_on_heap = c_p == rp && !(c_p->flags & F_OFF_HEAP_MSGQ);
+
+    tot_heap_size = 0;
+    i = 0;
+    mpp = &rp->sig_qs.first;
+    mp = rp->sig_qs.first;
+    while (mp) {
+	Eterm msg = ERL_MESSAGE_TERM(mp);
+
+	mip[i].size = 0;
+
+	if (ERTS_SIG_IS_EXTERNAL_MSG(mp)) {
+	    /* decode it... */
+	    if (mp->data.attached)
+		erts_decode_dist_message(rp, rp_locks, mp,
+					 ERTS_INSPECT_MSGQ_KEEP_OH_MSGS);
+
+	    msg = ERL_MESSAGE_TERM(mp);
+
+	    if (is_non_value(msg)) {
+		ErtsMessage *bad_mp = mp;
+		/*
+		 * Bad distribution message; remove
+		 * it from the queue...
+		 */
+		ASSERT(!mp->data.attached);
+
+		ASSERT(*mpp == bad_mp);
+
+                remove_iq_m_sig(rp, mp, mpp);
+
+                mp = *mpp;
+
+		bad_mp->next = NULL;
+		erts_cleanup_messages(bad_mp);
+		continue;
+	    }
+	}
+
+	ASSERT(is_value(msg));
+
+#if ERTS_INSPECT_MSGQ_KEEP_OH_MSGS
+	if (is_not_immed(msg) && (!self_on_heap || mp->data.attached)) {
+	    Uint sz = size_object(msg);
+	    mip[i].size = sz;
+	    tot_heap_size += sz;
+	}
+#else
+	if (self_on_heap) {
+	    if (mp->data.attached) {
+		ErtsMessage *tmp = NULL;
+		if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
+		    erts_link_mbuf_to_proc(rp, mp->data.heap_frag);
+		    mp->data.attached = NULL;
+		}
+		else {
+		    /*
+		     * Need to replace the message reference since
+		     * we will get references to the message data
+		     * from the heap...
+		     */
+		    ErtsMessage **mpp;
+		    tmp = erts_alloc_message(0, NULL);
+		    sys_memcpy((void *) tmp->m, (void *) mp->m,
+			       sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ); 
+		    mpp = i == 0 ? &rp->sig_qs.first : &mip[i-1].msgp->next;
+		    erts_msgq_replace_msg_ref(&rp->msg, tmp, mpp);
+		    erts_save_message_in_proc(rp, mp);
+		    mp = tmp;
+		}
+	    }
+	}
+	else if (is_not_immed(msg)) {
+	    Uint sz = size_object(msg);
+	    mip[i].size = sz;
+	    tot_heap_size += sz;
+	}
+
+#endif
+
+	mip[i].msgp = mp;
+	i++;
+        mpp = &mp->next;
+	mp = mp->next;
+    }
+
+    return tot_heap_size;
+}
+
+static ERTS_INLINE void
+move_msg_to_heap(Process *c_p, ErtsMessage *mp)
+{
+    /*
+     * We leave not yet decoded distribution messages
+     * as they are in the queue since it is not
+     * possible to determine a maximum size until
+     * actual decoding...
+     *
+     * We also leave combined messages as they are...
+     */
+    if (ERTS_SIG_IS_INTERNAL_MSG(mp)
+        && mp->data.attached
+        && mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
+        ErlHeapFragment *bp;
+
+        bp = erts_message_to_heap_frag(mp);
+
+        if (bp->next)
+            erts_move_multi_frags(&c_p->htop, &c_p->off_heap, bp,
+                                  mp->m, ERL_MESSAGE_REF_ARRAY_SZ, 0);
+        else
+            erts_copy_one_frag(&c_p->htop, &c_p->off_heap, bp,
+                               mp->m, ERL_MESSAGE_REF_ARRAY_SZ);
+
+        mp->data.heap_frag = NULL;
+        free_message_buffer(bp);
+    }
+}
+
+void
+erts_proc_sig_move_msgs_to_heap(Process *c_p)
+{
+    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p);
+
+    ERTS_FOREACH_SIG_PRIVQS(c_p, sig, move_msg_to_heap(c_p, sig));
+
+    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p);
+}
+
+
+BIF_RETTYPE
+erts_internal_dirty_process_handle_signals_1(BIF_ALIST_1)
+{
+    erts_aint32_t state, dirty, noproc;
+    int busy;
+    Process *rp;
+
+    if (BIF_P != erts_dirty_process_signal_handler
+        && BIF_P != erts_dirty_process_signal_handler_high
+        && BIF_P != erts_dirty_process_signal_handler_max)
+        BIF_ERROR(BIF_P, EXC_NOTSUP);
+
+    if (is_not_internal_pid(BIF_ARG_1))
+        BIF_RET(am_false);
+
+    rp = erts_proc_lookup_raw(BIF_ARG_1);
+    if (!rp)
+        BIF_RET(am_noproc);
+
+    state = erts_atomic32_read_nob(&rp->state);
+    dirty = (state & (ERTS_PSFLG_DIRTY_RUNNING
+                      | ERTS_PSFLG_DIRTY_RUNNING_SYS));
+    if (!dirty)
+        BIF_RET(am_normal);
+
+    busy = erts_proc_trylock(rp, ERTS_PROC_LOCK_MAIN) == EBUSY;
+
+    state = erts_atomic32_read_mb(&rp->state);
+    noproc = (state & ERTS_PSFLG_FREE);
+    dirty = (state & (ERTS_PSFLG_DIRTY_RUNNING
+                      | ERTS_PSFLG_DIRTY_RUNNING_SYS));
+
+    if (busy) {
+        if (noproc)
+            BIF_RET(am_noproc);
+        if (dirty)
+            BIF_RET(am_more); /* try again... */
+        BIF_RET(am_normal); /* will handle signals itself... */
+    }
+    else {
+        erts_aint32_t state;
+        int done;
+        Eterm res = am_false;
+        int reds = 0;
+
+        if (noproc)
+            res = am_noproc;
+        else if (!dirty)
+            res = am_normal; /* will handle signals itself... */
+        else {
+            reds = ERTS_BIF_REDS_LEFT(BIF_P);
+            done = erts_proc_sig_handle_incoming(rp, &state, &reds,
+                                                 reds, 0);
+            if (done || (state & ERTS_PSFLG_EXITING))
+                res = am_ok;
+            else
+                res = am_more;
+        }
+
+        erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
+
+        if (reds)
+            BUMP_REDS(BIF_P, reds);
+
+        BIF_RET(res);
+    }
+}
+
+static void
+debug_foreach_sig_heap_frags(ErlHeapFragment *hfrag,
+                             void (*oh_func)(ErlOffHeap *, void *),
+                             void *arg)
+{
+    ErlHeapFragment *hf = hfrag;
+    while (hf) {
+        oh_func(&(hf->off_heap), arg);
+        hf = hf->next;
+    }
+}
+
+static void
+debug_foreach_sig_fake_oh(Eterm term,
+                          void (*oh_func)(ErlOffHeap *, void *),
+                          void *arg)
+{
+    if (is_external(term)) {
+        ErlOffHeap oh;
+        oh.overhead = 0;
+        oh.first = ((struct erl_off_heap_header *)
+                    (char *) external_thing_ptr(term));
+        ASSERT(!oh.first->next);
+        oh_func(&oh, arg);
+    }
+
+}
+
+void
+erts_proc_sig_debug_foreach_sig(Process *c_p,
+                                void (*msg_func)(ErtsMessage *, void *),
+                                void (*oh_func)(ErlOffHeap *, void *),
+                                void (*mon_func)(ErtsMonitor *, void *),
+                                void (*lnk_func)(ErtsLink *, void *),
+                                void *arg)
+{
+    ErtsMessage *queue[] = {c_p->sig_qs.first, c_p->sig_qs.cont, c_p->sig_inq.first};
+    int qix;
+
+    for (qix = 0; qix < sizeof(queue)/sizeof(queue[0]); qix++) {
+        ErtsMessage *sig;
+        for (sig = queue[qix]; sig; sig = sig->next) {
+            
+            if (ERTS_SIG_IS_MSG(sig))
+                msg_func(sig, arg);
+            else {
+                Eterm tag;
+                Uint16 type;
+                int op;
+
+                ASSERT(sig);
+                ASSERT(ERTS_SIG_IS_NON_MSG(sig));
+
+                tag = ((ErtsSignal *) sig)->common.tag;
+                type = ERTS_PROC_SIG_TYPE(tag);
+                op = ERTS_PROC_SIG_OP(tag);
+
+                switch (op) {
+
+                case ERTS_SIG_Q_OP_EXIT:
+                case ERTS_SIG_Q_OP_EXIT_LINKED:
+                case ERTS_SIG_Q_OP_MONITOR_DOWN:
+                    switch (type) {
+                    case ERTS_SIG_Q_TYPE_GEN_EXIT:
+                        debug_foreach_sig_heap_frags(&sig->hfrag, oh_func, arg);
+                        break;
+                    case ERTS_LNK_TYPE_PORT:
+                    case ERTS_LNK_TYPE_PROC:
+                    case ERTS_LNK_TYPE_DIST_PROC:
+                        lnk_func((ErtsLink *) sig, arg);
+                        break;
+                    case ERTS_MON_TYPE_PORT:
+                    case ERTS_MON_TYPE_PROC:
+                    case ERTS_MON_TYPE_DIST_PROC:
+                    case ERTS_MON_TYPE_NODE:
+                        mon_func((ErtsMonitor *) sig, arg);
+                        break;
+                    default:
+                        ERTS_INTERNAL_ERROR("Unexpected sig type");
+                        break;
+                    }
+                    break;
+
+                case ERTS_SIG_Q_OP_PERSISTENT_MON_MSG:
+                    debug_foreach_sig_heap_frags(&sig->hfrag, oh_func, arg);
+                    break;
+
+                case ERTS_SIG_Q_OP_DEMONITOR:
+                    if (type == ERTS_SIG_Q_TYPE_DIST_PROC_DEMONITOR) {
+                        debug_foreach_sig_fake_oh(((ErtsSigDistProcDemonitor *) sig)->ref,
+                                                  oh_func, arg);
+                        break;
+                    }
+                    /* Fall through... */
+
+                case ERTS_SIG_Q_OP_MONITOR:
+                    mon_func((ErtsMonitor *) sig, arg);
+                    break;
+
+                case ERTS_SIG_Q_OP_UNLINK:
+                    if (type == ERTS_SIG_Q_TYPE_DIST_LINK) {
+                        debug_foreach_sig_fake_oh(((ErtsSigDistLinkOp *) sig)->remote,
+                                                  oh_func, arg);
+                        break;
+                    }
+                    /* Fall through... */
+
+                case ERTS_SIG_Q_OP_LINK:
+                    lnk_func((ErtsLink *) sig, arg);
+                    break;
+
+                case ERTS_SIG_Q_OP_GROUP_LEADER: {
+                    ErtsSigGroupLeader *sgl = (ErtsSigGroupLeader *) sig;
+                    oh_func(&sgl->oh, arg);
+                    break;
+                }
+
+                case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE:
+                    break;
+
+                default:
+                    ERTS_INTERNAL_ERROR("Unknown signal");
+                    break;
+                }
+
+            }
+        }
+    }
+}
+
+#ifdef ERTS_PROC_SIG_HARD_DEBUG
+
+static void
+chk_eterm(Process *c_p, int privq, ErtsMessage *mp, Eterm term)
+{
+    ErlHeapFragment *bp;
+    Eterm *ptr = NULL;
+
+    switch (primary_tag(term)) {
+    case TAG_PRIMARY_IMMED1:
+        return;
+    case TAG_PRIMARY_LIST:
+        ptr = list_val(term);
+        ERTS_ASSERT(!is_header(CAR(ptr)));
+        ERTS_ASSERT(!is_header(CDR(ptr)));
+        break;
+    case TAG_PRIMARY_BOXED:
+        ptr = boxed_val(term);
+        ERTS_ASSERT(is_header(*ptr));
+        break;
+    case TAG_PRIMARY_HEADER:
+    default:
+        ERTS_INTERNAL_ERROR("Not valid term");
+        break;
+    }
+
+    if (erts_is_literal(term, ptr))
+        return;
+
+    for (bp = erts_message_to_heap_frag(mp); bp; bp = bp->next) {
+        if (bp->mem <= ptr && ptr < bp->mem + bp->used_size)
+            return;
+        bp = bp->next;
+    }
+
+    ERTS_ASSERT(privq);
+    ASSERT(erts_dbg_within_proc(ptr, c_p, NULL));
+}
+
+static Sint
+proc_sig_hdbg_check_queue(Process *proc,
+                          int privq,
+                          ErtsMessage **sig_next,
+                          ErtsMessage **sig_last,
+                          ErtsMessage **sig_nm_next,
+                          ErtsMessage **sig_nm_last,
+                          ErtsSigRecvTracing *tracing,
+                          int *found_saved_last_p,
+                          erts_aint32_t sig_psflg)
+{
+    ErtsMessage **next, *sig, **nm_next, **nm_last;
+    int last_nm_sig_found, nm_sigs = 0, found_next_trace = 0,
+        found_save = 0, last_sig_found = 0, found_saved_last = 0;
+    Sint msg_len = 0;
+    ErtsMessage **next_trace = tracing ? tracing->messages.next : NULL;
+    ErtsMessage **save = proc->sig_qs.save;
+    ErtsMessage **saved_last = proc->sig_qs.saved_last;
+
+
+    nm_next = sig_nm_next;
+    nm_last = sig_nm_last;
+    next = sig_next;
+    sig = *sig_next;
+
+    last_nm_sig_found = !nm_last;
+    if (last_nm_sig_found)
+        ERTS_ASSERT(!nm_next);
+    else
+        ERTS_ASSERT(nm_next);
+
+    while (1) {
+        ErtsSignal *nm_sig;
+
+        if (next == sig_last) {
+            ASSERT(!*next);
+            last_sig_found = 1;
+        }
+
+        if (next == save)
+            found_save = 1;
+
+        if (next == saved_last)
+            found_saved_last = 1;
+
+        if (next == next_trace) {
+            found_next_trace = 1;
+            ERTS_ASSERT(nm_sigs == 0);
+        }
+
+        while (sig && ERTS_SIG_IS_MSG(sig)) {
+            int i;
+            if (ERTS_SIG_IS_EXTERNAL_MSG(sig))
+                i = 1;
+            else
+                i = 0;
+            for (; i < ERL_MESSAGE_REF_ARRAY_SZ; i++)
+                chk_eterm(proc, privq, sig, sig->m[i]);
+
+            msg_len++;
+            next = &sig->next;
+            sig = sig->next;
+
+            if (next == sig_last) {
+                ASSERT(!*next);
+                last_sig_found = 1;
+            }
+
+            if (next == save)
+                found_save = 1;
+
+            if (next == saved_last)
+                found_saved_last = 1;
+
+            if (next == next_trace) {
+                found_next_trace = 1;
+                ERTS_ASSERT(nm_sigs == 0);
+            }
+        }
+
+        if (!sig)
+            break;
+
+        nm_sigs++;
+
+        ERTS_ASSERT(!last_nm_sig_found);
+        ERTS_ASSERT(ERTS_SIG_IS_NON_MSG(sig));
+
+        nm_sig = (ErtsSignal *) sig;
+
+        ERTS_ASSERT(nm_next == next);
+
+        if (nm_last == next) {
+            ASSERT(!nm_sig->common.specific.next);
+            last_nm_sig_found = 1;
+        }
+
+        nm_next = nm_sig->common.specific.next;
+        next = &nm_sig->common.next;
+        sig = nm_sig->common.next;
+
+    }
+
+    if (!privq) {
+        /* outer queue */
+        ERTS_ASSERT(!found_save);
+        ERTS_ASSERT(!found_saved_last);
+    }
+    else if (privq > 0) {
+        /* middle queue */
+        ERTS_ASSERT(!next_trace || found_next_trace);
+        ERTS_ASSERT(!found_save);
+        if (!found_saved_last_p) {
+            ERTS_ASSERT(!found_saved_last
+                        || (proc->flags & F_DEFERRED_SAVED_LAST));
+        }
+        else {
+            if (*found_saved_last_p) {
+                ERTS_ASSERT(!found_saved_last);
+                ERTS_ASSERT(!(proc->flags & F_DEFERRED_SAVED_LAST));
+            }
+            else if (saved_last) {
+                ERTS_ASSERT(found_saved_last);
+                ERTS_ASSERT(proc->flags & F_DEFERRED_SAVED_LAST);
+            }
+            *found_saved_last_p |= found_saved_last;
+        }
+    }
+    else {
+        /* inner queue */
+        ERTS_ASSERT(!found_next_trace);
+        ERTS_ASSERT(nm_sigs == 0);
+        ERTS_ASSERT(found_save);
+        ERTS_ASSERT(!saved_last
+                    || (found_saved_last
+                        || (proc->flags & F_DEFERRED_SAVED_LAST)));
+        if (found_saved_last_p)
+            *found_saved_last_p |= found_saved_last;
+    }
+
+    ERTS_ASSERT(last_nm_sig_found);
+    ERTS_ASSERT(last_sig_found);
+
+    if (sig_psflg != ERTS_PSFLG_FREE) {
+        erts_aint32_t state = erts_atomic32_read_nob(&proc->state);
+        ERTS_ASSERT(nm_sigs ? !!(state & sig_psflg) : !(state & sig_psflg));
+    }
+
+    return msg_len;
+}
+
+void
+erts_proc_sig_hdbg_check_priv_queue(Process *p, char *what, char *file, int line)
+{
+    int found_saved_last = 0;
+    Sint len1, len2;
+    ERTS_LC_ASSERT(erts_thr_progress_is_blocking()
+                   || ERTS_PROC_IS_EXITING(p)
+                   || (ERTS_PROC_LOCK_MAIN
+                       & erts_proc_lc_my_proc_locks(p)));
+    len1 = proc_sig_hdbg_check_queue(p,
+                                     -1,
+                                     &p->sig_qs.first,
+                                     p->sig_qs.last,
+                                     NULL,
+                                     NULL,
+                                     NULL,
+                                     &found_saved_last,
+                                     ERTS_PSFLG_FREE);
+    len2 = proc_sig_hdbg_check_queue(p,
+                                     1,
+                                     &p->sig_qs.cont,
+                                     p->sig_qs.cont_last,
+                                     p->sig_qs.nmsigs.next,
+                                     p->sig_qs.nmsigs.last,
+                                     NULL,
+                                     &found_saved_last,
+                                     ERTS_PSFLG_SIG_Q);
+    if (p->sig_qs.saved_last)
+        ERTS_ASSERT(found_saved_last);
+    ERTS_ASSERT(p->sig_qs.len == len1 + len2);
+}
+
+void
+erts_proc_sig_hdbg_check_in_queue(Process *p, char *what, char *file, int line)
+{
+    Sint len;
+    ERTS_LC_ASSERT(erts_thr_progress_is_blocking()
+                   || ERTS_PROC_IS_EXITING(p)
+                   || (ERTS_PROC_LOCK_MSGQ
+                       & erts_proc_lc_my_proc_locks(p)));
+    len = proc_sig_hdbg_check_queue(p,
+                                    0,
+                                    &p->sig_inq.first,
+                                    p->sig_inq.last,
+                                    p->sig_inq.nmsigs.next,
+                                    p->sig_inq.nmsigs.last,
+                                    NULL,
+                                    NULL,
+                                    ERTS_PSFLG_SIG_IN_Q);
+    ASSERT(p->sig_inq.len == len);
+}
+
+#endif
diff --git a/erts/emulator/beam/erl_proc_sig_queue.h b/erts/emulator/beam/erl_proc_sig_queue.h
new file mode 100644
index 0000000000..433e30ce4a
--- /dev/null
+++ b/erts/emulator/beam/erl_proc_sig_queue.h
@@ -0,0 +1,665 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2018. All Rights Reserved.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description:	Process signal queue implementation.
+ *
+ *              Currently the following signals are handled:
+ *              - Messages
+ *              - Exit
+ *              - Monitor
+ *              - Demonitor
+ *              - Monitor down
+ *              - Persistent monitor message
+ *              - Link
+ *              - Unlink
+ *              - Group leader
+ *              - Trace change
+ *
+ *              The signal queue consists of three parts:
+ *              - Outer queue (sig_inq field in process struct)
+ *              - Middle queue (sig_qs field in process struct)
+ *              - Inner queue (sig_qs field in process struct)
+ *
+ *              Incoming signals are placed in the outer queue
+ *              by other processes, ports, or by the runtime system
+ *              itself. This queue is protected by the msgq process
+ *              lock and may be accessed by any other entity. While
+ *              a signal is located in the outer queue, it is still
+ *              in transit between sender and receiver.
+ *
+ *              The middle and the inner queues are private to the
+ *              receiving process and can only be accessed while
+ *              holding the main process lock. The signal changes
+ *              from being in transit to being received while in
+ *              the middle queue. Non-message signals are handled
+ *              immediately upon reception while message signals
+ *              are moved into the inner queue.
+ *
+ *              In the outer and middle queues both message signals
+ *              and non-message signals are mixed. Signals in these
+ *              queues are referenced using two single linked lists.
+ *              One single linked list that go through all signals
+ *              in the queue and another single linked list that
+ *              goes through only non-message signals. The list
+ *              through the non-message signals is used for fast
+ *              access to these signals in the middle queue, since
+ *              these should be handled immediately upon reception.
+ *
+ *              The inner queue consists only of one single linked
+ *              list through the message signals. A receive
+ *              expression can only operate on messages once they
+ *              have entered the inner queue.
+ *
+ * Author: 	Rickard Green
+ */
+
+#ifndef ERTS_PROC_SIG_QUEUE_H_TYPE__
+#define ERTS_PROC_SIG_QUEUE_H_TYPE__
+
+#if 0
+#  define ERTS_PROC_SIG_HARD_DEBUG
+#endif
+
+struct erl_mesg;
+
+typedef struct {
+    struct erl_mesg *next;
+    union {
+        struct erl_mesg **next;
+        void *attachment;
+    } specific;
+    Eterm tag;
+} ErtsSignalCommon;
+
+#define ERTS_SIG_HANDLE_REDS_MAX_PREFERED (CONTEXT_REDS/40)
+
+#ifdef ERTS_PROC_SIG_HARD_DEBUG
+#  define ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE(P)       \
+    ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__((P), "")
+#  define ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(P) \
+    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((P), "")
+#  define ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__(P, What)   \
+    erts_proc_sig_hdbg_check_in_queue((P), (What), __FILE__, __LINE__)
+#  define ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__(P, What)   \
+    erts_proc_sig_hdbg_check_priv_queue((P), (What), __FILE__, __LINE__)
+struct process;
+void erts_proc_sig_hdbg_check_priv_queue(struct process *c_p, char *what,
+                                         char *file, int line);
+void erts_proc_sig_hdbg_check_in_queue(struct process *c_p, char *what,
+                                       char *file, int line);
+#else
+#  define ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE(P)
+#  define ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(P)
+#  define ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__(P, What)
+#define ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__(P, What)
+#endif
+
+#endif
+
+#if !defined(ERTS_PROC_SIG_QUEUE_H__) && !defined(ERTS_PROC_SIG_QUEUE_TYPE_ONLY)
+#define ERTS_PROC_SIG_QUEUE_H__
+
+struct dist_entry_;
+
+/*
+ * Send operations of currently supported process signals follow...
+ */
+
+/**
+ *
+ * @brief Send an exit signal to a process.
+ *
+ *
+ * @param[in]     c_p           Pointer to process struct of
+ *                              currently executing process.
+ *
+ * @param[in]     from          Identifier of sender.
+ *
+ * @param[in]     to            Identifier of local process
+ *                              to send signal to.
+ *
+ * @param[in]     reason        Exit reason.
+ *
+ * @param[in]     token         Seq trace token.
+ *
+ * @param[in]     normal_kills  If non-zero, also normal exit
+ *                              reason will kill the receiver
+ *                              if it is not trapping exit.
+ *
+ */
+void
+erts_proc_sig_send_exit(Process *c_p, Eterm from, Eterm to,
+                        Eterm reason, Eterm token, int normal_kills);
+
+/**
+ *
+ * @brief Send an exit signal due to broken link to a process.
+ *
+ *
+ * @param[in]     c_p           Pointer to process struct of
+ *                              currently executing process.
+ *
+ * @param[in]     from          Identifier of sender.
+ *
+ * @param[in]     lnk           Pointer to link structure
+ *                              from the sending side. It
+ *                              should contain information
+ *                              about receiver.
+ *
+ * @param[in]     reason        Exit reason.
+ *
+ * @param[in]     token         Seq trace token.
+ *
+ */
+void
+erts_proc_sig_send_link_exit(Process *c_p, Eterm from, ErtsLink *lnk,
+                             Eterm reason, Eterm token);
+
+/**
+ *
+ * @brief Send an link signal to a process.
+ *
+ *
+ * @param[in]     c_p           Pointer to process struct of
+ *                              currently executing process.
+ *
+ * @param[in]     to            Identifier of receiver.
+ *
+ * @param[in]     lnk           Pointer to link structure to
+ *                              insert on receiver side.
+ *
+ * @return                      A non-zero value if
+ *                              signal was successfully
+ *                              sent. If a zero, value
+ *                              the signal was not sent
+ *                              due to the receiver not
+ *                              existing. The sender
+ *                              needs to deallocate the
+ *                              link structure.
+ *
+ */
+int
+erts_proc_sig_send_link(Process *c_p, Eterm to, ErtsLink *lnk);
+
+/**
+ *
+ * @brief Send an unlink signal to a process.
+ *
+ *
+ * @param[in]     c_p           Pointer to process struct of
+ *                              currently executing process.
+ *
+ * @param[in]     lnk           Pointer to link structure from
+ *                              the sending side. It should
+ *                              contain information about
+ *                              receiver.
+ */
+void
+erts_proc_sig_send_unlink(Process *c_p, ErtsLink *lnk);
+
+/**
+ *
+ * @brief Send an exit signal due to broken link to a process.
+ *
+ * This function is used instead of erts_proc_sig_send_link_exit()
+ * when the signal arrives via the distribution and
+ * no link structure is available.
+ *
+ * @param[in]     dep           Distribution entry of channel
+ *                              that the signal arrived on.
+ *
+ * @param[in]     from          Identifier of sender.
+ *
+ * @param[in]     to            Identifier of receiver.
+ *
+ * @param[in]     reason        Exit reason.
+ *
+ * @param[in]     token         Seq trace token.
+ *
+ */
+void
+erts_proc_sig_send_dist_link_exit(struct dist_entry_ *dep,
+                                  Eterm from, Eterm to,
+                                  Eterm reason, Eterm token);
+
+/**
+ *
+ * @brief Send an unlink signal to a process.
+ *
+ * This function is used instead of erts_proc_sig_send_unlink()
+ * when the signal arrives via the distribution and
+ * no link structure is available.
+ *
+ * @param[in]     dep           Distribution entry of channel
+ *                              that the signal arrived on.
+ *
+ * @param[in]     from          Identifier of sender.
+ *
+ * @param[in]     to            Identifier of receiver.
+ *
+ */
+void
+erts_proc_sig_send_dist_unlink(struct dist_entry_ *dep,
+                               Eterm from, Eterm to);
+
+/**
+ *
+ * @brief Send a monitor down signal to a process.
+ *
+ * @param[in]     mon           Pointer to target monitor
+ *                              structure from the sending
+ *                              side. It should contain
+ *                              information about receiver.
+ *
+ * @param[in]     reason        Exit reason.
+ *
+ */
+void
+erts_proc_sig_send_monitor_down(ErtsMonitor *mon, Eterm reason);
+
+/**
+ *
+ * @brief Send a demonitor signal to a process.
+ *
+ * @param[in]     mon           Pointer to origin monitor
+ *                              structure from the sending
+ *                              side. It should contain
+ *                              information about receiver.
+ *
+ * @param[in]     reason        Exit reason.
+ *
+ */
+void
+erts_proc_sig_send_demonitor(ErtsMonitor *mon);
+
+/**
+ *
+ * @brief Send a monitor signal to a process.
+ *
+ * @param[in]     mon           Pointer to target monitor
+ *                              structure to insert on
+ *                              receiver side.
+ *
+ * @param[in]     to            Identifier of receiver.
+ *
+ * @return                      A non-zero value if
+ *                              signal was successfully
+ *                              sent. If a zero, value
+ *                              the signal was not sent
+ *                              due to the receiver not
+ *                              existing. The sender
+ *                              needs to deallocate the
+ *                              monitor structure.
+ *
+ */
+int
+erts_proc_sig_send_monitor(ErtsMonitor *mon, Eterm to);
+
+/**
+ *
+ * @brief Send a monitor down signal to a process.
+ *
+ * This function is used instead of erts_proc_sig_send_monitor_down()
+ * when the signal arrives via the distribution and
+ * no link structure is available.
+ *
+ * @param[in]     dep           Pointer to distribution entry
+ *                              of channel that the signal
+ *                              arrived on.
+ *
+ * @param[in]     ref           Reference identifying the monitor.
+ *
+ * @param[in]     from          Identifier of sender.
+ *
+ * @param[in]     to            Identifier of receiver.
+ *
+ * @param[in]     reason        Exit reason.
+ *
+ */
+void
+erts_proc_sig_send_dist_monitor_down(DistEntry *dep, Eterm ref,
+                                     Eterm from, Eterm to,
+                                     Eterm reason);
+
+/**
+ *
+ * @brief Send a demonitor signal to a process.
+ *
+ * This function is used instead of erts_proc_sig_send_demonitor()
+ * when the signal arrives via the distribution and
+ * no monitor structure is available.
+ *
+ * @param[in]     to            Identifier of receiver.
+ *
+ * @param[in]     ref           Reference identifying the monitor.
+ *
+ */
+void
+erts_proc_sig_send_dist_demonitor(Eterm to, Eterm ref);
+
+/**
+ *
+ * @brief Send a persistent monitor triggered signal to a process.
+ *
+ * Used by monitors that are not auto disabled such as for
+ * example 'time_offset' monitors.
+ *
+ * @param[in]     type          Monitor type.
+ *
+ * @param[in]     key           Monitor key.
+ *
+ * @param[in]     from          Identifier of sender.
+ *
+ * @param[in]     to            Identifier of receiver.
+ *
+ * @param[in]     msg           Message template.
+ *
+ * @param[in]     msg_sz        Heap size of message template.
+ *
+ */
+void
+erts_proc_sig_send_persistent_monitor_msg(Uint16 type, Eterm key,
+                                          Eterm from, Eterm to,
+                                          Eterm msg, Uint msg_sz);
+
+/**
+ *
+ * @brief Send a trace change signal to a process.
+ *
+ * @param[in]     to            Identifier of receiver.
+ *
+ * @param[in]     on            Trace flags to enable.
+ *
+ * @param[in]     off           Trace flags to disable.
+ *
+ * @param[in]     tracer        Tracer to set. If the non-value,
+ *                              tracer will not be changed.
+ *
+ */
+void
+erts_proc_sig_send_trace_change(Eterm to, Uint on, Uint off,
+                                Eterm tracer);
+
+/**
+ *
+ * @brief Send a group leader signal to a process.
+ *
+ * Set group-leader of receiving process. If sent locally,
+ * a response message '{Ref, Result}' is sent to the original
+ * sender when performed where Ref is the reference passed
+ * as 'ref' argument, and Result is either 'true' or 'badarg'.
+ *
+ * @param[in]     c_p           Pointer to process struct of
+ *                              currently executing process.
+ *                              NULL if signal arrived via
+ *                              distribution.
+ *
+ * @param[in]     to            Identifier of receiver.
+ *
+ * @param[in]     gl            Identifier of new group leader.
+ *
+ * @param[in]     ref           Reference to use in response
+ *                              message to locally sending
+ *                              process (i.e., c_p when c_p
+ *                              is non-null).
+ *
+ */
+void
+erts_proc_sig_send_group_leader(Process *c_p, Eterm to, Eterm gl,
+                                Eterm ref);
+
+/*
+ * End of send operations of currently supported process signals.
+ */
+
+
+/**
+ *
+ * @brief Handle incoming signals.
+ *
+ * Called by an ordinary scheduler in order to handle incoming
+ * signals for a process. The work is done on the middle part
+ * of the signal queue. The maximum amount of signals handled
+ * is limited by the amount of reductions given when calling.
+ * Note that a reduction does not necessarily map to a signal.
+ *
+ * @param[in]     c_p           Pointer to process struct of
+ *                              currently executing process.
+ *
+ * @param[out]    statep        Pointer to process state after
+ *                              signal handling. May not be NULL.
+ *
+ * @param[in,out] redsp         Pointer to an integer containing
+ *                              reductions. On input, the amount
+ *                              of preferred reductions to be
+ *                              used by the call. On output, the
+ *                              amount of reductions consumed.
+ *
+ * @param[in]     max_reds      Absolute maximum of reductions
+ *                              to use. If the process cannot
+ *                              make progress after the preferred
+ *                              amount of reductions has been
+ *                              consumed, signal handling may
+ *                              proceed up to a maximum of
+ *                              'max_reds' in order to make
+ *                              the process able to proceed
+ *                              with other tasks after handling
+ *                              has finished.
+ *
+ * @param[in]     local_only    If is zero, new signals may be
+ *                              fetched from the outer queue and
+ *                              put in the middle queue before
+ *                              signal handling is performed. If
+ *                              non-zero, no new signals will be
+ *                              fetched before handling begins.
+ *
+ * @return                      Returns a non-zero value, when
+ *                              no more signals to handle in the
+ *                              middle queue remain. A zero
+ *                              return value means that there
+ *                              remains signals in the middle
+ *                              queue.
+ */
+int
+erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep,
+                              int *redsp, int max_reds,
+                              int local_only);
+
+/**
+ *
+ * @brief Handle remaining signals for an exiting process
+ *
+ * Called as part of termination of a process. It will handle
+ * remaining signals.
+ *
+ * @param[in]     c_p           Pointer to process struct of
+ *                              currently executing process.
+ *
+ * @param[in,out] redsp         Pointer to an integer containing
+ *                              reductions. On input, the amount
+ *                              of maximum reductions to be
+ *                              used by the call. On output, the
+ *                              amount of reductions consumed.
+ *
+ * @return                      Returns a non-zero value, when
+ *                              no more signals to handle in the
+ *                              middle queue remain. A zero
+ *                              return value means that there
+ *                              remains signals in the middle
+ *                              queue.
+ */
+int
+erts_proc_sig_handle_exit(Process *c_p, int *redsp);
+
+/**
+ *
+ * @brief Helper for loop_rec instruction.
+ *
+ * This function should only be called from the loop_rec
+ * instruction (or equivalents). It is called when loop_rec
+ * reach the end of the inner queue (which is the only
+ * part of the signal queue that receive is allowed to
+ * operate on). When called, this function tries to make
+ * more messages available in the inner queue. This by
+ * fetching signals from the outer queue to the middle
+ * queue and/or processing signals in the middle queue.
+ *
+ * @param[in]   c_p             Pointer to process struct of
+ *                              currently executing process.
+ *
+ * @param[in]   fcalls          Content of FCALLS in
+ *                              process_main()
+ *
+ * @param[in]   neg_o_reds      Content of neg_o_reds in
+ *                              process_main()
+ *
+ * @param[out]  msgpp           Pointer to pointer to next
+ *                              available message to process.
+ *                              If *msgpp == NULL, no more
+ *                              messages are available.
+ *
+ * @param[out]  get_outp        Pointer to an integer
+ *                              indicating how to respond
+ *                              if no more messages are
+ *                              available (msgpp). If integer
+ *                              is set to zero, loop_rec
+ *                              should jump to an appropriate
+ *                              wait instruction. If zero,
+ *                              the message queue lock remain
+ *                              locked since the test for
+ *                              more messages was done.
+ *                              If the integer is set to a
+ *                              value larger that zero, the
+ *                              process exited. If the integer
+ *                              is set to a value less than
+ *                              zero, the process is required
+ *                              to yield.
+ *
+ *
+ * @return                      The amount of reductions
+ *                              consumed.
+ *
+ */
+int
+erts_proc_sig_receive_helper(Process *c_p, int fcalls,
+                             int neg_o_reds, ErtsMessage **msgpp,
+                             int *get_outp);
+
+/**
+ *
+ * @brief Fetch signals from the outer queue
+ *
+ * Fetches signals from outer queue and places them in the
+ * middle queue ready for signal handling. If the middle
+ * queue is empty, only message signals were present in the
+ * outer queue, and no receive tracing has been enabled on
+ * the process, the middle queue is bypassed and messages
+ * are delivered directly to the inner queue instead.
+ *
+ * @param[in]   c_p             Pointer to process struct of
+ *                              currently executing process.
+ *
+ */
+void erts_proc_sig_fetch(Process *p);
+
+typedef struct {
+    Uint size;
+    ErtsMessage *msgp;
+} ErtsMessageInfo;
+
+/**
+ *
+ * @brief Prepare signal queue for inspection by process_info()
+ *
+ *
+ * @param[in]   c_p             Pointer to process struct of
+ *                              currently executing process.
+ *
+ * @param[in]   rp              Pointer to process struct of
+ *                              process to inspect.
+ *
+ * @param[in]   rp_locks        Process locks held on 'rp'.
+ *
+ * @param[in]   mip             Pointer to array of
+ *                              ErtsMessageInfo structures.
+ *                              
+ */
+
+Uint erts_proc_sig_prep_msgq_for_inspection(Process *c_p,
+                                            Process *rp,
+                                            ErtsProcLocks rp_locks,
+                                            ErtsMessageInfo *mip);
+
+
+/**
+ *
+ * @brief Move message data of messages in private queues to heap
+ *
+ * Move message data of messages in private queues to the heap.
+ * This is part of GC of processes that uses on-heap message
+ * data.
+ *
+ * @param[in]   c_p             Pointer to process struct of
+ *                              currently executing process.
+ *
+ */
+void erts_proc_sig_move_msgs_to_heap(Process *c_p);
+
+/**
+ *
+ * @brief Size of signal in bytes.
+ *
+ * @param[in]   sig             Signal to inspect.
+ *
+ */
+Uint erts_proc_sig_signal_size(ErtsSignal *sig);
+
+
+/**
+ *
+ * @brief Clear seq trace tokens on all signals
+ *
+ * Assumes thread progress has been blocked!
+ *
+ * @param[in]   c_p             Pointer to process
+ *
+ */
+void
+erts_proc_sig_clear_seq_trace_tokens(Process *c_p);
+
+/**
+ * @brief Initialize this functionality
+ */
+void erts_proc_sig_queue_init(void);
+
+void
+erts_proc_sig_debug_foreach_sig(Process *c_p,
+                                void (*msg_func)(ErtsMessage *, void *),
+                                void (*oh_func)(ErlOffHeap *, void *),
+                                void (*mon_func)(ErtsMonitor *, void *),
+                                void (*lnk_func)(ErtsLink *, void *),
+                                void *arg);
+
+extern Process *erts_dirty_process_signal_handler;
+extern Process *erts_dirty_process_signal_handler_high;
+extern Process *erts_dirty_process_signal_handler_max;
+
+#endif /* ERTS_PROC_SIG_QUEUE_H__ */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 0d02d10ac9..7969025f57 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -53,6 +53,7 @@
 #include "erl_nfunc_sched.h"
 #include "erl_check_io.h"
 #include "erl_poll.h"
+#include "erl_proc_sig_queue.h"
 
 #define ERTS_CHECK_TIME_REDS CONTEXT_REDS
 #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0)
@@ -431,7 +432,8 @@ typedef enum {
     ERTS_PSTT_CLA,	/* Copy Literal Area */
     ERTS_PSTT_COHMQ,    /* Change off heap message queue */
     ERTS_PSTT_FTMQ,     /* Flush trace msg queue */
-    ERTS_PSTT_ETS_FREE_FIXATION
+    ERTS_PSTT_ETS_FREE_FIXATION,
+    ERTS_PSTT_PRIO_SIG  /* Elevate prio on signal management */
 } ErtsProcSysTaskType;
 
 #define ERTS_MAX_PROC_SYS_TASK_ARGS 2
@@ -579,7 +581,6 @@ dbg_chk_aux_work_val(erts_aint32_t value)
     valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS;
     valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR;
     valid |= ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP;
-    valid |= ERTS_SSI_AUX_WORK_PENDING_EXITERS;
 #if HAVE_ERTS_MSEG
     valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK;
 #endif
@@ -601,7 +602,6 @@ dbg_chk_aux_work_val(erts_aint32_t value)
 #define ERTS_DBG_CHK_SSI_AUX_WORK(SSI)
 #endif
 
-static void do_handle_pending_exiters(ErtsProcList *);
 static void wake_scheduler(ErtsRunQueue *rq);
 
 #if defined(ERTS_ENABLE_LOCK_CHECK)
@@ -666,8 +666,6 @@ erts_pre_init_process(void)
 	= "MISC_THR_PRGR";
     erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_IX]
 	= "MISC";
-    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_PENDING_EXITERS_IX]
-	= "PENDING_EXITERS";
     erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_SET_TMO_IX]
 	= "SET_TMO";
     erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX]
@@ -2530,27 +2528,6 @@ handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiti
 #endif
 
 
-static ERTS_INLINE erts_aint32_t
-handle_pending_exiters(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
-{
-    ErtsProcList *pnd_xtrs;
-    ErtsRunQueue *rq;
-
-    rq = awdp->esdp->run_queue;
-    unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_PENDING_EXITERS);
-
-    erts_runq_lock(rq);
-    pnd_xtrs = rq->procs.pending_exiters;
-    rq->procs.pending_exiters = NULL;
-    erts_runq_unlock(rq);
-
-    if (erts_proclist_fetch(&pnd_xtrs, NULL))
-	do_handle_pending_exiters(pnd_xtrs);
-
-    return aux_work & ~ERTS_SSI_AUX_WORK_PENDING_EXITERS;
-}
-
-
 static ERTS_INLINE erts_aint32_t
 handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
 {
@@ -2632,9 +2609,6 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC,
 		    handle_misc_aux_work);
 
-    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_PENDING_EXITERS,
-		    handle_pending_exiters);
-
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO,
 		    handle_setup_aux_work_timer);
 
@@ -5793,7 +5767,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
 	rq->wakeup_other = 0;
 	rq->wakeup_other_reds = 0;
 
-	rq->procs.pending_exiters = NULL;
 	rq->procs.context_switches = 0;
 	rq->procs.reductions = 0;
 
@@ -6328,11 +6301,36 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
     int enqueue; /* < 0 -> use proxy */
     ErtsRunQueue* runq;
 
+    ASSERT(!(state & (ERTS_PSFLG_DIRTY_IO_PROC
+                      |ERTS_PSFLG_DIRTY_CPU_PROC))
+           || (BeamIsOpCode(*p->i, op_call_nif)
+               || BeamIsOpCode(*p->i, op_apply_bif)));
+
+    a = state;
+
+    /* Clear activ-sys if needed... */
+    while (1) {
+        n = e = a;
+        if (a & ERTS_PSFLG_ACTIVE_SYS) {
+            if (a & (ERTS_PSFLG_SIG_Q
+                     | ERTS_PSFLG_SIG_IN_Q
+                     | ERTS_PSFLG_SYS_TASKS))
+                break;
+            /* Clear active-sys */
+            n &= ~ERTS_PSFLG_ACTIVE_SYS;
+        }
+        a = erts_atomic32_cmpxchg_nob(&p->state, n, e);
+        if (a == e) {
+            a = n;
+            break;
+        }
+    }
+
     if (!is_normal_sched)
 	running_flgs = ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS;
     else {
 	running_flgs = ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS;
-        if (state & ERTS_PSFLG_DIRTY_ACTIVE_SYS
+        if ((a & ERTS_PSFLG_DIRTY_ACTIVE_SYS)
             && (p->flags & (F_DELAY_GC|F_DISABLE_GC))) {
             /*
              * Delay dirty GC; will be enabled automatically
@@ -6345,14 +6343,12 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
              */
             ASSERT(!(p->flags & (F_DIRTY_CLA | F_DIRTY_GC_HIBERNATE)));
 
-            state = erts_atomic32_read_band_nob(&p->state,
-                                                    ~ERTS_PSFLG_DIRTY_ACTIVE_SYS);
-            state &= ~ERTS_PSFLG_DIRTY_ACTIVE_SYS;
+            a = erts_atomic32_read_band_nob(&p->state,
+                                            ~ERTS_PSFLG_DIRTY_ACTIVE_SYS);
+            a &= ~ERTS_PSFLG_DIRTY_ACTIVE_SYS;
         }
     }
 
-    a = state;
-
     while (1) {
 	n = e = a;
 
@@ -6360,6 +6356,11 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
 
 	enqueue = ERTS_ENQUEUE_NOT;
 
+        ASSERT(((a & (ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE))
+                != ERTS_PSFLG_EXITING)
+               || ((a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED))
+                   == ERTS_PSFLG_ACTIVE));
+
 	n &= ~running_flgs;
 	if ((a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS))
 	    || (a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) {
@@ -6584,7 +6585,7 @@ change_proc_schedule_state(Process *p,
     }
 
 
-    *statep = a;
+    *statep = n;
 
     return enqueue;
 }
@@ -6609,6 +6610,95 @@ erts_schedule_process(Process *p, erts_aint32_t state, ErtsProcLocks locks)
     schedule_process(p, state, locks);
 }
 
+static ERTS_INLINE erts_aint32_t
+active_sys_enqueue(Process *p, erts_aint32_t state,
+                   erts_aint32_t enable_flags, int status_locked)
+{
+    /*
+     * This function may or may not be called with status locke held.
+     * It always returns without the status lock held!
+     */
+    unsigned int prof_runnable_procs = erts_system_profile_flags.runnable_procs;
+    erts_aint32_t n, a = state, enq_prio = -1;
+    int slocked = status_locked;
+    int enqueue; /* < 0 -> use proxy */
+
+    /* Status lock prevents out of order "runnable proc" trace msgs */
+    ERTS_LC_ASSERT(slocked || !(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)));
+    ERTS_LC_ASSERT(!slocked || (ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)));
+
+    if (!prof_runnable_procs) {
+        if (slocked) {
+            erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+            slocked = 0;
+        }
+    }
+    else {
+        if (!slocked) {
+            erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+            slocked = !0;
+        }
+    }
+
+    ASSERT(!(state & ERTS_PSFLG_PROXY));
+
+    while (1) {
+	erts_aint32_t e;
+	n = e = a;
+
+	if (a & ERTS_PSFLG_FREE)
+	    goto cleanup; /* We don't want to schedule free processes... */
+
+	enqueue = ERTS_ENQUEUE_NOT;
+        n |= enable_flags;
+	n |= ERTS_PSFLG_ACTIVE_SYS;
+	if (!(a & (ERTS_PSFLG_RUNNING
+		   | ERTS_PSFLG_RUNNING_SYS
+		   | ERTS_PSFLG_DIRTY_RUNNING
+		   | ERTS_PSFLG_DIRTY_RUNNING_SYS)))
+	    enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a);
+	a = erts_atomic32_cmpxchg_mb(&p->state, n, e);
+	if (a == e)
+	    break;
+	if (a == n && enqueue == ERTS_ENQUEUE_NOT)
+	    goto cleanup;
+    }
+
+    if (prof_runnable_procs) {
+
+	if (!(a & (ERTS_PSFLG_ACTIVE_SYS
+		   | ERTS_PSFLG_RUNNING
+		   | ERTS_PSFLG_RUNNING_SYS
+		   | ERTS_PSFLG_DIRTY_RUNNING
+		   | ERTS_PSFLG_DIRTY_RUNNING_SYS))
+	    && (!(a & ERTS_PSFLG_ACTIVE) || (a & ERTS_PSFLG_SUSPENDED))) {
+	    /* We activated a prevously inactive process */
+	    profile_runnable_proc(p, am_active);
+	}
+
+	erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+	slocked = 0;
+    }
+
+    add2runq(enqueue, enq_prio, p, n, NULL);
+
+cleanup:
+
+    if (slocked)
+	erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+
+    ERTS_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)));
+
+    return n;
+}
+
+erts_aint32_t
+erts_proc_sys_schedule(Process *p, erts_aint32_t state, erts_aint32_t enable_flag)
+{
+    /* We are not allowed to call this function with status lock held... */
+    return active_sys_enqueue(p, state, enable_flag, 0);
+}
+
 static int
 schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
 			  erts_aint32_t *fail_state_p)
@@ -6616,19 +6706,16 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
     int res;
     int locked;
     ErtsProcSysTaskQs *stqs, *free_stqs;
-    erts_aint32_t fail_state, state, a, n, enq_prio;
-    int enqueue; /* < 0 -> use proxy */
-    unsigned int prof_runnable_procs;
+    erts_aint32_t fail_state, state;
 
     fail_state = *fail_state_p;
 
     res = 1; /* prepare for success */
     st->next = st->prev = st; /* Prep for empty prio queue */
     state = erts_atomic32_read_nob(&p->state);
-    prof_runnable_procs = erts_system_profile_flags.runnable_procs;
     locked = 0;
     free_stqs = NULL;
-    if (state & ERTS_PSFLG_ACTIVE_SYS)
+    if (state & ERTS_PSFLG_SYS_TASKS)
 	stqs = NULL;
     else {
     alloc_qs:
@@ -6648,6 +6735,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
 
 	state = erts_atomic32_read_nob(&p->state);
 	if (state & fail_state) {
+            erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 	    *fail_state_p = (state & fail_state);
 	    free_stqs = stqs;
 	    res = 0;
@@ -6695,69 +6783,14 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
 	state = n;
     }
 
-
-    a = state;
-    enq_prio = -1;
-
-    /* Status lock prevents out of order "runnable proc" trace msgs */
-    ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
-
-    if (!prof_runnable_procs) {
-	erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-	locked = 0;
-    }
-
-    ASSERT(!(state & ERTS_PSFLG_PROXY));
-
-    while (1) {
-	erts_aint32_t e;
-	n = e = a;
-
-	if (a & ERTS_PSFLG_FREE)
-	    goto cleanup; /* We don't want to schedule free processes... */
-
-	enqueue = ERTS_ENQUEUE_NOT;
-	n |= ERTS_PSFLG_ACTIVE_SYS;
-	if (!(a & (ERTS_PSFLG_RUNNING
-		   | ERTS_PSFLG_RUNNING_SYS
-		   | ERTS_PSFLG_DIRTY_RUNNING
-		   | ERTS_PSFLG_DIRTY_RUNNING_SYS)))
-	    enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a);
-	a = erts_atomic32_cmpxchg_mb(&p->state, n, e);
-	if (a == e)
-	    break;
-	if (a == n && enqueue == ERTS_ENQUEUE_NOT)
-	    goto cleanup;
-    }
-
-    if (prof_runnable_procs) {
-
-	if (!(a & (ERTS_PSFLG_ACTIVE_SYS
-		   | ERTS_PSFLG_RUNNING
-		   | ERTS_PSFLG_RUNNING_SYS
-		   | ERTS_PSFLG_DIRTY_RUNNING
-		   | ERTS_PSFLG_DIRTY_RUNNING_SYS))
-	    && (!(a & ERTS_PSFLG_ACTIVE) || (a & ERTS_PSFLG_SUSPENDED))) {
-	    /* We activated a prevously inactive process */
-	    profile_runnable_proc(p, am_active);
-	}
-
-	erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-	locked = 0;
-    }
-
-    add2runq(enqueue, enq_prio, p, n, NULL);
+    /* active_sys_enqueue() always return with status lock unlocked */
+    (void) active_sys_enqueue(p, state, ERTS_PSFLG_SYS_TASKS, locked);
 
 cleanup:
 
-    if (locked)
-	erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-
     if (free_stqs)
 	proc_sys_task_queues_free(free_stqs);
 
-    ERTS_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)));
-
     return res;
 }
 
@@ -8536,11 +8569,13 @@ static ERTS_INLINE void
 cancel_suspend_of_suspendee(Process *p, ErtsProcLocks p_locks)
 {
     if (is_not_nil(p->suspendee)) {
+        ErtsMonitor *mon;
+        Eterm suspendee = p->suspendee;
 	Process *rp;
 	if (!(p_locks & ERTS_PROC_LOCK_STATUS))
 	    erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
 	rp = erts_pid2proc(p, p_locks|ERTS_PROC_LOCK_STATUS,
-			   p->suspendee, ERTS_PROC_LOCK_STATUS);
+			   suspendee, ERTS_PROC_LOCK_STATUS);
 	if (rp) {
 	    erts_resume(rp, ERTS_PROC_LOCK_STATUS);
 	    erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
@@ -8548,6 +8583,14 @@ cancel_suspend_of_suspendee(Process *p, ErtsProcLocks p_locks)
 	if (!(p_locks & ERTS_PROC_LOCK_STATUS))
 	    erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 	p->suspendee = NIL;
+
+        mon = erts_monitor_tree_lookup(p->suspend_monitors,
+                                       suspendee);
+        if (mon) {
+            erts_monitor_tree_delete(&p->suspend_monitors,
+                                     mon);
+            erts_monitor_suspend_destroy(erts_monitor_suspend(mon));
+        }
     }
 }
 
@@ -8706,8 +8749,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 	    {
 		erts_aint32_t state;
 		state = erts_atomic32_read_nob(&rp->state);
-		ASSERT((state & ERTS_PSFLG_PENDING_EXIT)
-		       || !(state & ERTS_PSFLG_RUNNING));
+		ASSERT(!(state & ERTS_PSFLG_RUNNING));
 	    }
 #endif
 
@@ -8762,7 +8804,7 @@ erts_pid2proc_nropt(Process *c_p, ErtsProcLocks c_p_locks,
 
 static ERTS_INLINE int
 do_bif_suspend_process(Process *c_p,
-		       ErtsSuspendMonitor *smon,
+		       ErtsMonitorSuspend *smon,
 		       Process *suspendee)
 {
     ASSERT(suspendee);
@@ -8795,21 +8837,25 @@ handle_pend_bif_sync_suspend(Process *suspendee,
     suspender = erts_pid2proc(suspendee,
 			      suspendee_locks,
 			      suspender_pid,
-			      ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
+			      ERTS_PROC_LOCK_STATUS);
     if (suspender) {
+        ErtsMonitorSuspend *smon;
+        ErtsMonitor *mon;
+        mon = erts_monitor_tree_lookup(suspender->suspend_monitors,
+                                       suspendee->common.id);
+        smon = erts_monitor_suspend(mon);
+
 	ASSERT(is_nil(suspender->suspendee));
-	if (!suspendee_alive)
-	    erts_delete_suspend_monitor(&suspender->suspend_monitors,
-					suspendee->common.id);
+	if (!suspendee_alive) {
+            if (mon) {
+                erts_monitor_tree_delete(&suspender->suspend_monitors,
+                                         mon);
+                erts_monitor_suspend_destroy(smon);
+            }
+        }
 	else {
 #ifdef DEBUG
-	    int res;
-#endif
-	    ErtsSuspendMonitor *smon;
-	    smon = erts_lookup_suspend_monitor(suspender->suspend_monitors,
-					       suspendee->common.id);
-#ifdef DEBUG
-	    res =
+	    int res =
 #endif
 		do_bif_suspend_process(suspendee, smon, suspendee);
 	    ASSERT(!smon || res != 0);
@@ -8818,9 +8864,8 @@ handle_pend_bif_sync_suspend(Process *suspendee,
 	/* suspender is suspended waiting for suspendee to suspend;
 	   resume suspender */
 	ASSERT(suspender != suspendee);
-	resume_process(suspender, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
-	erts_proc_unlock(suspender,
-			     ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
+	resume_process(suspender, ERTS_PROC_LOCK_STATUS);
+	erts_proc_unlock(suspender, ERTS_PROC_LOCK_STATUS);
     }
 }
 
@@ -8838,26 +8883,29 @@ handle_pend_bif_async_suspend(Process *suspendee,
     suspender = erts_pid2proc(suspendee,
 			      suspendee_locks,
 			      suspender_pid,
-			      ERTS_PROC_LOCK_LINK);
+			      ERTS_PROC_LOCK_STATUS);
     if (suspender) {
+        ErtsMonitorSuspend *smon;
+        ErtsMonitor *mon;
+        mon = erts_monitor_tree_lookup(suspender->suspend_monitors,
+                                       suspendee->common.id);
+        smon = erts_monitor_suspend(mon);
 	ASSERT(is_nil(suspender->suspendee));
-	if (!suspendee_alive)
-	    erts_delete_suspend_monitor(&suspender->suspend_monitors,
-					suspendee->common.id);
+	if (!suspendee_alive) {
+            if (mon) {
+                erts_monitor_tree_delete(&suspender->suspend_monitors,
+                                         mon);
+                erts_monitor_suspend_destroy(smon);
+            }
+        }
 	else {
 #ifdef DEBUG
-	    int res;
-#endif
-	    ErtsSuspendMonitor *smon;
-	    smon = erts_lookup_suspend_monitor(suspender->suspend_monitors,
-					       suspendee->common.id);
-#ifdef DEBUG
-	    res =
+	    int res =
 #endif
-	    do_bif_suspend_process(suspendee, smon, suspendee);
+                do_bif_suspend_process(suspendee, smon, suspendee);
 	    ASSERT(!smon || res != 0);
 	}
-	erts_proc_unlock(suspender, ERTS_PROC_LOCK_LINK);
+	erts_proc_unlock(suspender, ERTS_PROC_LOCK_STATUS);
     }
 }
 
@@ -8871,8 +8919,9 @@ suspend_process_2(BIF_ALIST_2)
 {
     Eterm res;
     Process* suspendee = NULL;
-    ErtsSuspendMonitor *smon;
+    ErtsMonitorSuspend *smon;
     ErtsProcLocks xlocks = (ErtsProcLocks) 0;
+    int created;
 
     /* Options and default values: */
     int asynchronous = 0;
@@ -8905,9 +8954,7 @@ suspend_process_2(BIF_ALIST_2)
 	    goto badarg;
     }
 
-    xlocks = ERTS_PROC_LOCK_LINK | (asynchronous
-				    ? (ErtsProcLocks) 0
-				    : ERTS_PROC_LOCK_STATUS);
+    xlocks = ERTS_PROC_LOCK_STATUS;
 
     erts_proc_lock(BIF_P, xlocks);
 
@@ -8918,15 +8965,14 @@ suspend_process_2(BIF_ALIST_2)
     if (!suspendee)
 	goto no_suspendee;
 
-    smon = erts_add_or_lookup_suspend_monitor(&BIF_P->suspend_monitors,
-					      BIF_ARG_1);
-
-    /* ... but a little trickier with SMP support ... */
+    smon = erts_monitor_suspend_tree_lookup_create(&BIF_P->suspend_monitors,
+                                                   &created,
+                                                   BIF_ARG_1);
 
     if (asynchronous) {
 	/* --- Asynchronous suspend begin ---------------------------------- */
 
-	ERTS_LC_ASSERT(ERTS_PROC_LOCK_LINK
+	ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS
 			   & erts_proc_lc_my_proc_locks(BIF_P));
 	ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS
 			   == erts_proc_lc_my_proc_locks(suspendee));
@@ -8968,9 +9014,9 @@ suspend_process_2(BIF_ALIST_2)
     else /* if (!asynchronous) */ {
 	/* --- Synchronous suspend begin ----------------------------------- */
 
-	ERTS_LC_ASSERT(((ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS)
+	ERTS_LC_ASSERT(((ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_STATUS)
 			    & erts_proc_lc_my_proc_locks(BIF_P))
-			   == (ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS));
+			   == (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_STATUS));
 	ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS
 			   == erts_proc_lc_my_proc_locks(suspendee));
 
@@ -9055,9 +9101,15 @@ suspend_process_2(BIF_ALIST_2)
     ERTS_BIF_PREP_ERROR(res, BIF_P, SYSTEM_LIMIT);
     goto do_return;
 
- no_suspendee:
-    BIF_P->suspendee = NIL;
-    erts_delete_suspend_monitor(&BIF_P->suspend_monitors, BIF_ARG_1);
+ no_suspendee: {
+        ErtsMonitor *mon;
+        BIF_P->suspendee = NIL;
+        mon = erts_monitor_tree_lookup(BIF_P->suspend_monitors, BIF_ARG_1);
+        if (mon) {
+            erts_monitor_tree_delete(&BIF_P->suspend_monitors, mon);
+            erts_monitor_suspend_destroy(erts_monitor_suspend(mon));
+        }
+    }
 
  badarg:
     ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
@@ -9084,15 +9136,17 @@ suspend_process_2(BIF_ALIST_2)
 BIF_RETTYPE
 resume_process_1(BIF_ALIST_1)
 {
-    ErtsSuspendMonitor *smon;
+    ErtsMonitor *mon;
+    ErtsMonitorSuspend *smon;
     Process *suspendee;
     int is_active;
  
     if (BIF_P->common.id == BIF_ARG_1)
 	BIF_ERROR(BIF_P, BADARG);
 
-    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
-    smon = erts_lookup_suspend_monitor(BIF_P->suspend_monitors, BIF_ARG_1);
+    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_STATUS);
+    mon = erts_monitor_tree_lookup(BIF_P->suspend_monitors, BIF_ARG_1);
+    smon = erts_monitor_suspend(mon);
 
     if (!smon) {
 	/* No previous suspend or dead suspendee */
@@ -9109,20 +9163,17 @@ resume_process_1(BIF_ALIST_1)
     }
     else if (smon->active) {
 	smon->active--;
-	ASSERT(smon->pending >= 0);
+	ASSERT(smon->pending == 0);
 	is_active = 1;
     }
     else {
 	/* No previous suspend or dead suspendee */
-	goto error;
+	goto no_suspendee;
     }
 
     if (smon->active || smon->pending || !is_active) {
 	/* Leave the suspendee as it is; just verify that it is still alive */
-	suspendee = erts_pid2proc(BIF_P,
-				  ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK,
-				  BIF_ARG_1,
-				  0);
+	suspendee = erts_proc_lookup(BIF_ARG_1);
 	if (!suspendee)
 	    goto no_suspendee;
 
@@ -9130,11 +9181,18 @@ resume_process_1(BIF_ALIST_1)
     else {
 	/* Resume */
 	suspendee = erts_pid2proc(BIF_P,
-				  ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK,
+				  ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS,
 				  BIF_ARG_1,
 				  ERTS_PROC_LOCK_STATUS);
-	if (!suspendee)
+	if (!suspendee) {
+            mon = erts_monitor_tree_lookup(BIF_P->suspend_monitors, BIF_ARG_1);
+            smon = erts_monitor_suspend(mon);
+            if (!mon)
+                goto error;
 	    goto no_suspendee;
+        }
+
+        ASSERT(mon == erts_monitor_tree_lookup(BIF_P->suspend_monitors, BIF_ARG_1));
 
 	ASSERT(ERTS_PSFLG_SUSPENDED
 	       & erts_atomic32_read_nob(&suspendee->state));
@@ -9144,19 +9202,24 @@ resume_process_1(BIF_ALIST_1)
 	erts_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
     }
 
-    if (!smon->active && !smon->pending)
-	erts_delete_suspend_monitor(&BIF_P->suspend_monitors, BIF_ARG_1);
+    if (!smon->active && !smon->pending) {
+        ASSERT(mon);
+        erts_monitor_tree_delete(&BIF_P->suspend_monitors, mon);
+        erts_monitor_suspend_destroy(smon);
+    }
 
-    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
+    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS);
 
     BIF_RET(am_true);
 
  no_suspendee:
     /* cleanup */
-    erts_delete_suspend_monitor(&BIF_P->suspend_monitors, BIF_ARG_1);
+    ASSERT(mon);
+    erts_monitor_tree_delete(&BIF_P->suspend_monitors, mon);
+    erts_monitor_suspend_destroy(smon);
 
  error:
-    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
+    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS);
     BIF_ERROR(BIF_P, BADARG);
 }
 
@@ -9375,9 +9438,8 @@ erts_resume_processes(ErtsProcList *list)
 }
 
 Eterm
-erts_get_process_priority(Process *p)
+erts_get_process_priority(erts_aint32_t state)
 {
-    erts_aint32_t state = erts_atomic32_read_nob(&p->state);
     switch (ERTS_PSFLGS_GET_USR_PRIO(state)) {
     case PRIORITY_MAX:		return am_max;
     case PRIORITY_HIGH:		return am_high;
@@ -9427,7 +9489,10 @@ erts_set_process_priority(Process *p, Eterm value)
 		}
 
 		max_qbit = 0;
-		if (a & ERTS_PSFLG_ACTIVE_SYS)
+                ASSERT((a & ERTS_PSFLG_SYS_TASKS)
+                       ? !!p->sys_task_qs
+                       : !p->sys_task_qs);
+		if (a & ERTS_PSFLG_SYS_TASKS)
 		    max_qbit |= p->sys_task_qs->qmask;
 		if (a & ERTS_PSFLG_DELAYED_SYS) {
 		    ErtsProcSysTaskQs *qs;
@@ -9450,8 +9515,8 @@ erts_set_process_priority(Process *p, Eterm value)
 		    aprio = PRIORITY_LOW;
 		    break;
 		default:
-		    ERTS_INTERNAL_ERROR("Invalid qmask");
-		    aprio = -1;
+                    aprio = nprio;
+                    break;
 		}
 
 		if (aprio > nprio) /* low value -> high prio */
@@ -9627,10 +9692,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
         /* have to re-read state after taking lock */
         state = erts_atomic32_read_nob(&p->state);
 
-	if (is_normal_sched && (state & ERTS_PSFLG_PENDING_EXIT))
-	    erts_handle_pending_exit(p, (ERTS_PROC_LOCK_MAIN
-					 | ERTS_PROC_LOCK_TRACE
-					 | ERTS_PROC_LOCK_STATUS));
 	if (p->pending_suspenders) 
 	    handle_pending_suspend(p, (ERTS_PROC_LOCK_MAIN
 				       | ERTS_PROC_LOCK_TRACE
@@ -9945,12 +10006,10 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 			       & ((state & (ERTS_PSFLG_SUSPENDED
 					    | ERTS_PSFLG_EXITING
 					    | ERTS_PSFLG_FREE
-					    | ERTS_PSFLG_PENDING_EXIT
 					    | ERTS_PSFLG_ACTIVE_SYS
 					    | ERTS_PSFLG_DIRTY_ACTIVE_SYS))
 				  != ERTS_PSFLG_SUSPENDED)
-                               & (!(state & (ERTS_PSFLG_EXITING
-                                             | ERTS_PSFLG_PENDING_EXIT))
+                               & (!(state & ERTS_PSFLG_EXITING)
                                   | (!!is_normal_sched))
                     );
 
@@ -10042,7 +10101,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		goto sunlock_sched_out_proc;
             }
 	    if (state & (ERTS_PSFLG_ACTIVE_SYS
-			 | ERTS_PSFLG_PENDING_EXIT
 			 | ERTS_PSFLG_EXITING)) {
 		/*
 		 * IMPORTANT! We need to take care of
@@ -10065,13 +10123,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		      && (state & ERTS_PSFLG_DIRTY_IO_PROC)));	    
 	}
 
-	if (state & ERTS_PSFLG_PENDING_EXIT) {
-	    erts_handle_pending_exit(p,
-				     ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
-	    state = erts_atomic32_read_nob(&p->state);
-	}
-
-
 	erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 
         /* Clear tracer if it has been removed */
@@ -10093,25 +10144,48 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	}
 
         if (is_normal_sched) {
-
             if (state & ERTS_PSFLG_RUNNING_SYS) {
-                /*
-                 * GC is normally never delayed when a process
-                 * is scheduled out, but might be when executing
-                 * hand written beam assembly in
-                 * prim_eval:'receive'. If GC is delayed we are
-                 * not allowed to execute system tasks.
-                 */
-                if (!(p->flags & F_DELAY_GC)) {
-                    int cost = execute_sys_tasks(p, &state, reds);
-                    calls += cost;
-                    reds -= cost;
-                    if (reds <= 0)
-                        goto sched_out_proc;
-                    if (state & ERTS_PSFLGS_DIRTY_WORK)
-                        goto sched_out_proc;
+                if (state & (ERTS_PSFLG_SIG_Q|ERTS_PSFLG_SIG_IN_Q)) {
+                    int local_only = !!(p->flags & F_LOCAL_SIGS_ONLY);
+                    if (!local_only || (state & ERTS_PSFLG_SIG_Q)) {
+                        int sig_reds;
+                        /*
+                         * If we have dirty work scheduled we allow
+                         * usage of all reductions since we need to
+                         * handle all signals before doing dirty
+                         * work...
+                         */
+                        if (state & ERTS_PSFLGS_DIRTY_WORK)
+                            sig_reds = reds;
+                        else
+                            sig_reds = ERTS_SIG_HANDLE_REDS_MAX_PREFERED;
+                        (void) erts_proc_sig_handle_incoming(p,
+                                                             &state,
+                                                             &sig_reds,
+                                                             sig_reds,
+                                                             local_only);
+                        reds -= sig_reds;
+                    }
+                }
+                if ((state & (ERTS_PSFLG_SYS_TASKS
+                              | ERTS_PSFLG_EXITING)) == ERTS_PSFLG_SYS_TASKS) {
+                    /*
+                     * GC is normally never delayed when a process
+                     * is scheduled out, but might be when executing
+                     * hand written beam assembly in
+                     * prim_eval:'receive'. If GC is delayed we are
+                     * not allowed to execute system tasks.
+                     */
+                    if (!(p->flags & F_DELAY_GC)) {
+                        int cost = execute_sys_tasks(p, &state, reds);
+                        calls += cost;
+                        reds -= cost;
+                    }
                 }
 
+                if (reds <= 0 || (state & ERTS_PSFLGS_DIRTY_WORK))
+                    goto sched_out_proc;
+
                 ASSERT(state & psflg_running_sys);
                 ASSERT(!(state & psflg_running));
 
@@ -10120,7 +10194,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
                     if (((state & (ERTS_PSFLG_SUSPENDED
                                    | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE)
-                        && !(state & ERTS_PSFLG_EXITING)) {
+                        & !(state & ERTS_PSFLG_EXITING)) {
                         goto sched_out_proc;
                     }
 
@@ -10162,15 +10236,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    
 	ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
-	/* Never run a suspended process */
-#ifdef DEBUG
-        {
-            erts_aint32_t dstate = erts_atomic32_read_nob(&p->state);
-            ASSERT(!(ERTS_PSFLG_SUSPENDED & dstate)
-                   || (ERTS_PSFLG_DIRTY_RUNNING_SYS & dstate));
-        }
-#endif
-
 	ASSERT(erts_proc_read_refc(p) > 0);
 
 	if (!(state & ERTS_PSFLG_EXITING) && ERTS_PTMR_IS_TIMED_OUT(p)) {
@@ -10183,6 +10248,11 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    ERTS_PTMR_CLEAR(p);
 	}
 
+        /* if exiting, we *shall* exit... */
+        ASSERT(!(state & ERTS_PSFLG_EXITING)
+               || p->i == (BeamInstr *) beam_exit
+               || p->i == (BeamInstr *) beam_continue_exit);
+
 #ifdef DEBUG
         if (is_normal_sched) {
 	    if (state & ERTS_PSFLGS_DIRTY_WORK)
@@ -10198,6 +10268,18 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
                     ERTS_INTERNAL_ERROR("Executing normal code on dirty UNKNOWN scheduler");
             }
         }
+        {
+            erts_aint32_t dstate = erts_atomic32_read_nob(&p->state);
+
+            /* Never run a suspended process */
+            ASSERT(!(ERTS_PSFLG_SUSPENDED & dstate)
+                   || (ERTS_PSFLG_DIRTY_RUNNING_SYS & dstate));
+
+            /* Do not execute on the wrong type of scheduler... */
+            ASSERT(is_normal_sched
+                   ? !(dstate & ERTS_PSFLGS_DIRTY_WORK)
+                   : !!(dstate & ERTS_PSFLGS_DIRTY_WORK));
+        }
 #endif
 
 	return p;
@@ -10399,7 +10481,7 @@ fetch_sys_task(Process *c_p, erts_aint32_t state, int *qmaskp, int *priop)
 	    n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET);
 
 	    if (!qmask)
-		n &= ~ERTS_PSFLG_ACTIVE_SYS;
+		n &= ~ERTS_PSFLG_SYS_TASKS;
 
 	    if (a == n)
 		break;
@@ -10439,12 +10521,8 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	int st_prio;
 	Eterm st_res;
 
-	if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
-	    if (state & ERTS_PSFLG_PENDING_EXIT)
-		erts_handle_pending_exit(c_p, ERTS_PROC_LOCK_MAIN);
-	    ASSERT(ERTS_PROC_IS_EXITING(c_p));
+	if (state & ERTS_PSFLG_EXITING)
 	    break;
-	}
 
 	st = fetch_sys_task(c_p, state, &qmask, &st_prio);
 	if (!st)
@@ -10540,6 +10618,70 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	    reds -= erts_db_execute_free_fixation(c_p, (DbFixation*)st->arg[0]);
 	    st_res = am_true;
 	    break;
+        case ERTS_PSTT_PRIO_SIG: {
+            erts_aint32_t fail_state, state;
+            int local_only, sig_res, sig_reds = reds;
+	    st_res = am_false;
+
+            if (st->arg[0] == am_true)
+                local_only = !0;
+            else
+                local_only = 0;
+
+            sig_reds = reds;
+            sig_res = erts_proc_sig_handle_incoming(c_p, &state, &sig_reds,
+                                                    reds, local_only);
+            reds -= sig_reds;
+
+            if (state & ERTS_PSFLG_EXITING)
+                goto perm_elevate_prio;
+
+            if (sig_res)
+                break;
+
+            st->arg[0] = am_true;
+
+            fail_state = ERTS_PSFLG_EXITING;
+
+            if (schedule_process_sys_task(c_p, st_prio, st, &fail_state)) {
+                /* Successfully rescheduled task... */
+                st = NULL;
+            }
+            else {
+                erts_aint32_t a;
+
+                state = erts_atomic32_read_nob(&c_p->state);                         
+
+            perm_elevate_prio:
+
+                /*
+                 * we are about to terminate; permanently elevate
+                 * prio in order to ensure high prio signal
+                 * handling...
+                 */
+
+                a = state;
+                while (1) {
+                    erts_aint32_t aprio, uprio, n, e;
+                    ASSERT(!(a & ERTS_PSFLG_FREE));
+                    aprio = ERTS_PSFLGS_GET_ACT_PRIO(a);
+                    uprio = ERTS_PSFLGS_GET_USR_PRIO(a);
+                    if (aprio >= uprio)
+                        break; /* user prio >= actual prio */
+                    /*
+                     * actual prio is higher than user prio; raise
+                     * user prio to actual prio...
+                     */
+                    n = e = a;
+                    n &= ~ERTS_PSFLGS_USR_PRIO_MASK;
+                    n |= aprio << ERTS_PSFLGS_USR_PRIO_OFFSET;
+                    a = erts_atomic32_cmpxchg_mb(&c_p->state, n, e);
+                    if (a == e)
+                        break;
+                }
+            }
+            break;
+        }
 	default:
 	    ERTS_INTERNAL_ERROR("Invalid process sys task type");
 	    st_res = am_false;
@@ -10593,6 +10735,7 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
 	case ERTS_PSTT_CPC:
         case ERTS_PSTT_COHMQ:
         case ERTS_PSTT_ETS_FREE_FIXATION:
+        case ERTS_PSTT_PRIO_SIG:
 	    st_res = am_false;
 	    break;
 	case ERTS_PSTT_CLA:
@@ -10645,8 +10788,8 @@ erts_execute_dirty_system_task(Process *c_p)
 
     if (c_p->flags & F_DIRTY_GC_HIBERNATE) {
 	erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
-	ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
-	if (c_p->msg.len)
+	erts_proc_sig_fetch(c_p);
+	if (c_p->sig_qs.len)
             c_p->flags &= ~F_DIRTY_GC_HIBERNATE; /* operation aborted... */
         else {
 	    erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
@@ -10718,7 +10861,7 @@ dispatch_system_task(Process *c_p, erts_aint_t fail_state,
 
     switch (st->type) {
     case ERTS_PSTT_CPC:
-	rp = erts_dirty_process_code_checker;
+	rp = erts_dirty_process_signal_handler;
 	ASSERT(fail_state & (ERTS_PSFLG_DIRTY_RUNNING
 			     | ERTS_PSFLG_DIRTY_RUNNING_SYS));
 	if (c_p == rp) {
@@ -10935,13 +11078,15 @@ erts_internal_request_system_task_4(BIF_ALIST_4)
 			       BIF_ARG_2, BIF_ARG_3, BIF_ARG_4);
 }
 
-static void
-erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type, void* arg)
+static int
+schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type,
+                          int prio, Eterm arg0, Eterm arg1)
 {
-    Process *rp = erts_proc_lookup(pid);
+    int res = 0;
+    Process *rp = erts_proc_lookup_raw(pid);
     if (rp) {
 	ErtsProcSysTask *st;
-	erts_aint32_t state, fail_state;
+	erts_aint32_t st_prio, fail_state;
 
 	st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK,
 			ERTS_PROC_SYS_TASK_SIZE(0));
@@ -10950,33 +11095,48 @@ erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type, void* arg)
 	st->reply_tag = NIL;
 	st->req_id = NIL;
 	st->req_id_sz = 0;
-        st->arg[0] = (Eterm)arg;
+        st->arg[0] = arg0;
+        st->arg[1] = arg1;
 	ERTS_INIT_OFF_HEAP(&st->off_heap);
-	state = erts_atomic32_read_nob(&rp->state);
 
-	fail_state = ERTS_PSFLG_EXITING;
-
-	if (!schedule_process_sys_task(rp, ERTS_PSFLGS_GET_USR_PRIO(state),
-				       st, &fail_state))
+        if (prio >= 0) {
+            st_prio = (erts_aint32_t) prio;
+            fail_state = ERTS_PSFLG_FREE;
+        }
+        else {
+            erts_aint32_t state = erts_atomic32_read_nob(&rp->state);
+            st_prio = ERTS_PSFLGS_GET_USR_PRIO(state);
+            fail_state = ERTS_PSFLG_EXITING;
+        }
+        res = schedule_process_sys_task(rp, st_prio, st, &fail_state);
+	if (!res)
 	    erts_free(ERTS_ALC_T_PROC_SYS_TSK, st);
     }
+    return res;
 }
 
-
 void
 erts_schedule_complete_off_heap_message_queue_change(Eterm pid)
 {
-    erts_schedule_generic_sys_task(pid, ERTS_PSTT_COHMQ, NULL);
+    schedule_generic_sys_task(pid, ERTS_PSTT_COHMQ,
+                              -1, NIL, NIL);
 }
 
 void
 erts_schedule_ets_free_fixation(Eterm pid, DbFixation* fix)
 {
-    erts_schedule_generic_sys_task(pid, ERTS_PSTT_ETS_FREE_FIXATION, fix);
+    schedule_generic_sys_task(pid, ERTS_PSTT_ETS_FREE_FIXATION,
+                              -1, (Eterm) fix, NIL);
 }
 
-
-static void
+int
+erts_sig_prio(Eterm pid, int prio)
+{
+    return schedule_generic_sys_task(pid, ERTS_PSTT_PRIO_SIG,
+                                     prio, am_false, NIL);
+}
+
+static void
 flush_dirty_trace_messages(void *vpid)
 {
     Process *proc;
@@ -11014,7 +11174,7 @@ erts_schedule_flush_trace_messages(Process *proc, int force_on_proc)
 
     dhndl = erts_thr_progress_unmanaged_delay();
 
-    erts_schedule_generic_sys_task(pid, ERTS_PSTT_FTMQ, NULL);
+    schedule_generic_sys_task(pid, ERTS_PSTT_FTMQ, -1, NIL, NIL);
 
     erts_thr_progress_unmanaged_continue(dhndl);
 
@@ -11210,9 +11370,11 @@ erts_set_gc_state(Process *c_p, int enable)
 #endif
 
 	    erts_atomic32_read_bset_nob(&c_p->state,
-					    (ERTS_PSFLG_DELAYED_SYS
-					     | ERTS_PSFLG_ACTIVE_SYS),
-					    ERTS_PSFLG_ACTIVE_SYS);
+                                        (ERTS_PSFLG_DELAYED_SYS
+                                         | ERTS_PSFLG_ACTIVE_SYS
+                                         | ERTS_PSFLG_SYS_TASKS),
+                                        (ERTS_PSFLG_ACTIVE_SYS
+                                         | ERTS_PSFLG_SYS_TASKS));
 
 #ifdef DEBUG
 	ASSERT(state & ERTS_PSFLG_DELAYED_SYS);
@@ -11675,7 +11837,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->common.u.alive.reg = NULL;
     ERTS_P_LINKS(p) = NULL;
     ERTS_P_MONITORS(p) = NULL;
-    p->nodes_monitors = NULL;
+    ERTS_P_LT_MONITORS(p) = NULL;
     p->suspend_monitors = NULL;
 
     ASSERT(is_pid(parent->group_leader));
@@ -11692,14 +11854,20 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 
     erts_get_default_proc_tracing(&ERTS_TRACE_FLAGS(p), &ERTS_TRACER(p));
 
-    p->msg.first = NULL;
-    p->msg.last = &p->msg.first;
-    p->msg.save = &p->msg.first;
-    p->msg.saved_last = &p->msg.first;
-    p->msg.len = 0;
-    p->msg_inq.first = NULL;
-    p->msg_inq.last = &p->msg_inq.first;
-    p->msg_inq.len = 0;
+    p->sig_qs.first = NULL;
+    p->sig_qs.last = &p->sig_qs.first;
+    p->sig_qs.cont = NULL;
+    p->sig_qs.cont_last = &p->sig_qs.cont;
+    p->sig_qs.save = &p->sig_qs.first;
+    p->sig_qs.saved_last = NULL;
+    p->sig_qs.len = 0;
+    p->sig_qs.nmsigs.next = NULL;
+    p->sig_qs.nmsigs.last = NULL;
+    p->sig_inq.first = NULL;
+    p->sig_inq.last = &p->sig_inq.first;
+    p->sig_inq.len = 0;
+    p->sig_inq.nmsigs.next = NULL;
+    p->sig_inq.nmsigs.last = NULL;
     p->bif_timers = NULL;
     p->mbuf = NULL;
     p->msg_frag = NULL;
@@ -11726,8 +11894,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->scheduler_data = NULL;
     p->suspendee = NIL;
     p->pending_suspenders = NULL;
-    p->pending_exit.reason = THE_NON_VALUE;
-    p->pending_exit.bp = NULL;
 
 #if !defined(NO_FPE_SIGNALS) || defined(HIPE)
     p->fp_exception = 0;
@@ -11781,30 +11947,33 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
      */
 
     if (so->flags & SPO_LINK) {
-#ifdef DEBUG
-	int ret;
-#endif
-#ifdef DEBUG
-	ret = erts_add_link(&ERTS_P_LINKS(parent),  LINK_PID, p->common.id);
-	ASSERT(ret == 0);
-	ret = erts_add_link(&ERTS_P_LINKS(p), LINK_PID, parent->common.id);
-	ASSERT(ret == 0);
-#else	
-	erts_add_link(&ERTS_P_LINKS(parent), LINK_PID, p->common.id);
-	erts_add_link(&ERTS_P_LINKS(p), LINK_PID, parent->common.id);
-#endif
-
+        ErtsLink *lnk;
+        ErtsLinkData *ldp = erts_link_create(ERTS_LNK_TYPE_PROC,
+                                             parent->common.id,
+                                             p->common.id);
+        lnk = erts_link_tree_lookup_insert(&ERTS_P_LINKS(parent), &ldp->a);
+        if (lnk) {
+            /*
+             * This should more or less never happen, but could
+             * potentially happen if pid:s wrap...
+             */
+            erts_link_release(lnk);
+        }
+        erts_link_tree_insert(&ERTS_P_LINKS(p), &ldp->b);
     }
 
     /*
      * Test whether this process should be initially monitored by its parent.
      */
     if (so->flags & SPO_MONITOR) {
-	Eterm mref;
-
-	mref = erts_make_ref(parent);
-	erts_add_monitor(&ERTS_P_MONITORS(parent), MON_ORIGIN, mref, p->common.id, NIL);
-	erts_add_monitor(&ERTS_P_MONITORS(p), MON_TARGET, mref, parent->common.id, NIL);
+	Eterm mref = erts_make_ref(parent);
+        ErtsMonitorData *mdp = erts_monitor_create(ERTS_MON_TYPE_PROC,
+                                                   mref,
+                                                   parent->common.id,
+                                                   p->common.id,
+                                                   NIL);
+        erts_monitor_tree_insert(&ERTS_P_MONITORS(parent), &mdp->origin);
+        erts_monitor_list_insert(&ERTS_P_LT_MONITORS(p), &mdp->target);
 	so->mref = mref;
     }
 
@@ -11889,13 +12058,23 @@ void erts_init_empty_process(Process *p)
     p->mbuf_sz = 0;
     erts_atomic_init_nob(&p->psd, (erts_aint_t) NULL);
     ERTS_P_MONITORS(p) = NULL;
+    ERTS_P_LT_MONITORS(p) = NULL;
     ERTS_P_LINKS(p) = NULL;         /* List of links */
-    p->nodes_monitors = NULL;
     p->suspend_monitors = NULL;
-    p->msg.first = NULL;
-    p->msg.last = &p->msg.first;
-    p->msg.save = &p->msg.first;
-    p->msg.len = 0;
+    p->sig_qs.first = NULL;
+    p->sig_qs.last = &p->sig_qs.first;
+    p->sig_qs.cont = NULL;
+    p->sig_qs.cont_last = &p->sig_qs.cont;
+    p->sig_qs.save = &p->sig_qs.first;
+    p->sig_qs.saved_last = NULL;
+    p->sig_qs.len = 0;
+    p->sig_qs.nmsigs.next = NULL;
+    p->sig_qs.nmsigs.last = NULL;
+    p->sig_inq.first = NULL;
+    p->sig_inq.last = &p->sig_inq.first;
+    p->sig_inq.len = 0;
+    p->sig_inq.nmsigs.next = NULL;
+    p->sig_inq.nmsigs.last = NULL;
     p->bif_timers = NULL;
     p->dictionary = NULL;
     p->seq_trace_clock = 0;
@@ -11942,13 +12121,8 @@ void erts_init_empty_process(Process *p)
     erts_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL);
 
     p->scheduler_data = NULL;
-    p->msg_inq.first = NULL;
-    p->msg_inq.last = &p->msg_inq.first;
-    p->msg_inq.len = 0;
     p->suspendee = NIL;
     p->pending_suspenders = NULL;
-    p->pending_exit.reason = THE_NON_VALUE;
-    p->pending_exit.bp = NULL;
     erts_proc_lock_init(p);
     erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
     erts_init_runq_proc(p, ERTS_RUNQ_IX(0), 0);
@@ -11984,11 +12158,11 @@ erts_debug_verify_clean_empty_process(Process* p)
     ASSERT(p->old_heap == NULL);
 
     ASSERT(ERTS_P_MONITORS(p) == NULL);
+    ASSERT(ERTS_P_LT_MONITORS(p) == NULL);
     ASSERT(ERTS_P_LINKS(p) == NULL);
-    ASSERT(p->nodes_monitors == NULL);
     ASSERT(p->suspend_monitors == NULL);
-    ASSERT(p->msg.first == NULL);
-    ASSERT(p->msg.len == 0);
+    ASSERT(p->sig_qs.first == NULL);
+    ASSERT(p->sig_qs.len == 0);
     ASSERT(p->bif_timers == NULL);
     ASSERT(p->dictionary == NULL);
     ASSERT(p->catches == 0);
@@ -11998,12 +12172,10 @@ erts_debug_verify_clean_empty_process(Process* p)
 
     ASSERT(p->parent == NIL);
 
-    ASSERT(p->msg_inq.first == NULL);
-    ASSERT(p->msg_inq.len == 0);
+    ASSERT(p->sig_inq.first == NULL);
+    ASSERT(p->sig_inq.len == 0);
     ASSERT(p->suspendee == NIL);
     ASSERT(p->pending_suspenders == NULL);
-    ASSERT(p->pending_exit.reason == THE_NON_VALUE);
-    ASSERT(p->pending_exit.bp == NULL);
 
     /* Thing that erts_cleanup_empty_process() cleans up */
 
@@ -12104,817 +12276,330 @@ delete_process(Process* p)
     erts_erase_dicts(p);
 
     /* free all pending messages */
-    erts_cleanup_messages(p->msg.first);
-    p->msg.first = NULL;
+    erts_cleanup_messages(p->sig_qs.first);
+    p->sig_qs.first = NULL;
+    erts_cleanup_messages(p->sig_qs.cont);
+    p->sig_qs.cont = NULL;
 
-    ASSERT(!p->nodes_monitors);
     ASSERT(!p->suspend_monitors);
 
     p->fvalue = NIL;
 }
 
 static ERTS_INLINE void
-set_proc_exiting(Process *p,
-		 erts_aint32_t in_state,
-		 Eterm reason,
-		 ErlHeapFragment *bp)
-{
-    erts_aint32_t state = in_state, enq_prio = -1;
-    int enqueue;
-    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCKS_ALL);
-
-    enqueue = change_proc_schedule_state(p,
-					 (ERTS_PSFLG_SUSPENDED
-					  | ERTS_PSFLG_PENDING_EXIT
-					  | ERTS_PSFLGS_DIRTY_WORK),
-					 ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE,
-					 &state,
-					 &enq_prio,
-					 ERTS_PROC_LOCKS_ALL);
-
-    p->fvalue = reason;
-    if (bp)
-	erts_link_mbuf_to_proc(p, bp);
-    /*
-     * We used to set freason to EXC_EXIT here, but there is no need to
-     * save the stack trace since this process irreversibly is going to
-     * exit.
-     */
-    p->freason = EXTAG_EXIT;
-    KILL_CATCHES(p);
-    p->i = (BeamInstr *) beam_exit;
-
-
-    add2runq(enqueue, enq_prio, p, state, NULL);
-}
-
-static ERTS_INLINE erts_aint32_t
-set_proc_self_exiting(Process *c_p)
+set_self_exiting(Process *c_p, Eterm reason, int *enqueue,
+                 erts_aint32_t *prio, erts_aint32_t *state)
 {
-#ifdef DEBUG
-    int enqueue;
-#endif
-    erts_aint32_t state, enq_prio = -1;
+    erts_aint32_t st, enq_prio = -1;
+    int enq;
 
     ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCKS_ALL);
 
-    state = erts_atomic32_read_nob(&c_p->state);
-    ASSERT(state & (ERTS_PSFLG_RUNNING
-		    |ERTS_PSFLG_RUNNING_SYS
-		    | ERTS_PSFLG_DIRTY_RUNNING
-		    | ERTS_PSFLG_DIRTY_RUNNING_SYS));
-
-#ifdef DEBUG
-    enqueue =
-#endif
-	change_proc_schedule_state(c_p,
-				   ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT,
-				   ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE,
-				   &state,
-				   &enq_prio,
-				   ERTS_PROC_LOCKS_ALL);
-
-    ASSERT(!enqueue);
-    return state;
-}
-
-
-void
-erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks)
-{
-    ErtsProcLocks xlocks;
-    ASSERT(is_value(c_p->pending_exit.reason));
-    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == locks);
-    ERTS_LC_ASSERT(locks & ERTS_PROC_LOCK_MAIN);
-    ERTS_LC_ASSERT(!((ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE)
-			 & erts_atomic32_read_nob(&c_p->state)));
-
-    /* Ensure that all locks on c_p are locked before proceeding... */
-    if (locks == ERTS_PROC_LOCKS_ALL)
-	xlocks = 0;
-    else {
-	xlocks = ~locks & ERTS_PROC_LOCKS_ALL;
-	if (erts_proc_trylock(c_p, xlocks) == EBUSY) {
-	    erts_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN);
-	    erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
-	}
-    }
-
-    set_proc_exiting(c_p,
-		     erts_atomic32_read_acqb(&c_p->state),
-		     c_p->pending_exit.reason,
-		     c_p->pending_exit.bp);
-    c_p->pending_exit.reason = THE_NON_VALUE;
-    c_p->pending_exit.bp = NULL;
+    c_p->fvalue = reason;
 
-    if (xlocks)
-	erts_proc_unlock(c_p, xlocks);
-}
+    st = erts_atomic32_read_nob(&c_p->state);
+    ASSERT(enqueue || (st & (ERTS_PSFLG_RUNNING
+                             |ERTS_PSFLG_RUNNING_SYS
+                             | ERTS_PSFLG_DIRTY_RUNNING
+                             | ERTS_PSFLG_DIRTY_RUNNING_SYS)));
 
-static void save_pending_exiter(Process *p, ErtsProcList *plp);
+    enq = change_proc_schedule_state(c_p,
+                                     (ERTS_PSFLG_SUSPENDED
+                                      | ERTS_PSFLGS_DIRTY_WORK),
+                                     ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE,
+                                     &st,
+                                     &enq_prio,
+                                     ERTS_PROC_LOCKS_ALL);
 
-static void
-do_handle_pending_exiters(ErtsProcList *pnd_xtrs)
-{
-    /* 'list' is expected to have been fetched (i.e. not a ring anymore) */
-    ErtsProcList *plp = pnd_xtrs;
-
-    while (plp) {
-	ErtsProcList *next_plp = plp->next;
-	Process *p = erts_proc_lookup(plp->pid);
-	if (p) {
-	    erts_aint32_t state;
-	    /*
-	     * If the process is running on a normal scheduler, the
-	     * pending exit will soon be detected and handled by the
-	     * scheduler running the process (at schedule in/out).
-	     */
-	    if (erts_proc_trylock(p, ERTS_PROC_LOCKS_ALL) != EBUSY) {
-		if (erts_proclist_same(plp, p)) {
-		    state = erts_atomic32_read_acqb(&p->state);
-		    if (!(state & (ERTS_PSFLG_RUNNING
-				   | ERTS_PSFLG_RUNNING_SYS
-				   | ERTS_PSFLG_EXITING))) {
-			ASSERT(state & ERTS_PSFLG_PENDING_EXIT);
-			erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL);
-		    }
-		}
-		erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
-	    }
-	    else {
-		erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
-		if (erts_proclist_same(plp, p)) {
-		    state = erts_atomic32_read_acqb(&p->state);
-		    if (!(state & (ERTS_PSFLG_RUNNING
-				   | ERTS_PSFLG_RUNNING_SYS
-				   | ERTS_PSFLG_EXITING))) {
-			/*
-			 * Save process and try to acquire all
-			 * locks at a later time...
-			 */
-			save_pending_exiter(p, plp);
-			plp = NULL;
-		    }
-		}
-		erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-	    }
-	}
-	if (plp)
-	    proclist_destroy(plp);
-	plp = next_plp;
-    }
+    ASSERT(enqueue || !enq);
+    if (enqueue)
+        *enqueue = enq;
+    if (prio)
+        *prio = enq_prio;
+    if (state)
+        *state = st;
 }
 
-static void
-save_pending_exiter(Process *p, ErtsProcList *plp)
+void
+erts_set_self_exiting(Process *c_p, Eterm reason)
 {
-    ErtsSchedulerSleepInfo *ssi;
-    ErtsRunQueue *rq;
-
-    ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
-
-    rq = erts_get_runq_proc(p, NULL);
-    ASSERT(rq && !ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
-
-    if (!plp)
-	plp = proclist_create(p);
-
-    erts_runq_lock(rq);
-
-    erts_proclist_store_last(&rq->procs.pending_exiters, plp);
-
-    non_empty_runq(rq);
-
-    ssi = rq->scheduler->ssi;
+    int enqueue;
+    erts_aint32_t enq_prio, state;
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
 
-    erts_runq_unlock(rq);
+    erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 
-    set_aux_work_flags_wakeup_nob(ssi, ERTS_SSI_AUX_WORK_PENDING_EXITERS);
-}
+    set_self_exiting(c_p, reason, &enqueue, &enq_prio, &state);
+    c_p->freason = EXTAG_EXIT;
+    KILL_CATCHES(c_p);
+    c_p->i = (BeamInstr *) beam_exit;
 
+    /* Always active when exiting... */
+    ASSERT(state & ERTS_PSFLG_ACTIVE);
 
-/*
- * This function delivers an EXIT message to a process
- * which is trapping EXITs.
- */
-
-static ERTS_INLINE void
-send_exit_message(Process *to, ErtsProcLocks *to_locksp,
-		  Eterm exit_term, Uint term_size, Eterm token)
-{
-    ErtsMessage *mp;
-    ErlOffHeap *ohp;
-    Eterm* hp;
-    Eterm mess;
-#ifdef SHCOPY_SEND
-    erts_shcopy_t info;
-#endif
+    /*
+     * If we are terminating a process that currently
+     * is executing on a dirty scheduler. It *should*
+     * be scheduled on a normal scheduler...
+     */
+    ASSERT(!(state & (ERTS_PSFLG_DIRTY_RUNNING
+                      | ERTS_PSFLG_DIRTY_RUNNING_SYS))
+           || enqueue == ERTS_ENQUEUE_NORMAL_QUEUE
+           || enqueue == -ERTS_ENQUEUE_NORMAL_QUEUE);
 
-    if (!have_seqtrace(token)) {
-#ifdef SHCOPY_SEND
-        INITIALIZE_SHCOPY(info);
-        term_size = copy_shared_calculate(exit_term, &info);
-	mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp);
-        mess = copy_shared_perform(exit_term, term_size, &info, &hp, ohp);
-        DESTROY_SHCOPY(info);
-#else
-	mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp);
-	mess = copy_struct(exit_term, term_size, &hp, ohp);
-#endif
-	erts_queue_message(to, *to_locksp, mp, mess, am_system);
-    } else {
-	Eterm temp_token;
-	Uint sz_token;
-
-	ASSERT(is_tuple(token));
-	sz_token = size_object(token);
-#ifdef SHCOPY_SEND
-        INITIALIZE_SHCOPY(info);
-        term_size = copy_shared_calculate(exit_term, &info);
-	mp = erts_alloc_message_heap(to, to_locksp, term_size+sz_token, &hp, &ohp);
-        mess = copy_shared_perform(exit_term, term_size, &info, &hp, ohp);
-        DESTROY_SHCOPY(info);
-#else
-	mp = erts_alloc_message_heap(to, to_locksp, term_size+sz_token, &hp, &ohp);
-	mess = copy_struct(exit_term, term_size, &hp, ohp);
-#endif
-	/* the trace token must in this case be updated by the caller */
-	seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, to);
-	temp_token = copy_struct(token, sz_token, &hp, ohp);
-        ERL_MESSAGE_TOKEN(mp) = temp_token;
-	erts_queue_message(to, *to_locksp, mp, mess, am_system);
-    }
+    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+    if (enqueue)
+        add2runq(enqueue, enq_prio, c_p, state, NULL);
 }
 
-/*
- *
- * *** Exit signal behavior ***
- *
- * Exit signals are asynchronous (truly asynchronous in the
- * SMP emulator). When the signal is received the receiver receives an
- * 'EXIT' message if it is trapping exits; otherwise, it will either
- * ignore the signal if the exit reason is normal, or go into an
- * exiting state (ERTS_PSFLG_EXITING). When a process has gone into the
- * exiting state it will not execute any more Erlang code, but it might
- * take a while before it actually exits. The exit signal is being
- * received when the 'EXIT' message is put in the message queue, the
- * signal is dropped, or when it changes state into exiting. The time it
- * is in the exiting state before actually exiting is undefined (it
- * might take a really long time under certain conditions). The
- * receiver of the exit signal does not break links or trigger monitors
- * until it actually exits.
- *
- * Exit signals and other signals, e.g. messages, have to be received
- * by a receiver in the same order as sent by a sender.
- *
- *
- *
- * Exit signal implementation in the SMP emulator:
- *
- * If the receiver is trapping exits, the signal is transformed
- * into an 'EXIT' message and sent as a normal message, if the
- * reason is normal the signal is dropped; otherwise, the process
- * is determined to be exited. The interesting case is when the
- * process is to be exited and this is what is described below.
- *
- * If it is possible, the receiver is set in the exiting state straight
- * away and we are done; otherwise, the sender places the exit reason
- * in the pending_exit field of the process struct and if necessary
- * adds the receiver to the run queue. It is typically not possible
- * to set a scheduled process or a process which we cannot get all locks
- * on without releasing locks on it in an exiting state straight away.
- *
- * The receiver will poll the pending_exit field when it reach certain
- * places during it's execution. When it discovers the pending exit
- * it will change state into the exiting state. If the receiver wasn't
- * scheduled when the pending exit was set, the first scheduler that
- * schedules a new process will set the receiving process in the exiting
- * state just before it schedules next process.
- * 
- * When the exit signal is placed in the pending_exit field, the signal
- * is considered as being in transit on the Erlang level. The signal is
- * actually in some kind of semi transit state, since we have already
- * determined how it should be received. It will exit the process no
- * matter what if it is received (the process may exit by itself before
- * reception of the exit signal). The signal is received when it is
- * discovered in the pending_exit field by the receiver.
- *
- * The receiver have to poll the pending_exit field at least before:
- * - moving messages from the message in queue to the private message
- *   queue. This in order to preserve signal order.
- * - unlink. Otherwise the process might get exited on a link that
- *   have been removed.
- * - changing the trap_exit flag to true. This in order to simplify the
- *   implementation; otherwise, we would have to transform the signal
- *   into an 'EXIT' message when setting the trap_exit flag to true. We
- *   would also have to maintain a queue of exit signals in transit.
- * - being scheduled in or out.
- */
-
-static ERTS_INLINE int
-send_exit_signal(Process *c_p,		/* current process if and only
-					   if reason is stored on it */
-		 Eterm from,		/* Id of sender of signal */
-		 Process *rp,		/* receiving process */
-		 ErtsProcLocks *rp_locks,/* current locks on receiver */
-		 Eterm reason,		/* exit reason */
-		 Eterm exit_tuple,	/* Prebuild exit tuple
-					   or THE_NON_VALUE */
-		 Uint exit_tuple_sz,	/* Size of prebuilt exit tuple
-					   (if exit_tuple != THE_NON_VALUE) */
-		 Eterm token,		/* token */
-		 Process *token_update, /* token updater */
-		 Uint32 flags		/* flags */
-    )		
-{
-    erts_aint32_t state = erts_atomic32_read_nob(&rp->state);
-    Eterm rsn = reason == am_kill ? am_killed : reason;
-
-    ERTS_LC_ASSERT(*rp_locks == erts_proc_lc_my_proc_locks(rp));
-    ERTS_LC_ASSERT((*rp_locks & ERTS_PROC_LOCKS_XSIG_SEND)
-		       == ERTS_PROC_LOCKS_XSIG_SEND);
-
-    ASSERT(reason != THE_NON_VALUE);
-
-#ifdef USE_VM_PROBES
-    if(DTRACE_ENABLED(process_exit_signal) && is_pid(from)) {
-        DTRACE_CHARBUF(sender_str, DTRACE_TERM_BUF_SIZE);
-        DTRACE_CHARBUF(receiver_str, DTRACE_TERM_BUF_SIZE);
-        DTRACE_CHARBUF(reason_buf, DTRACE_TERM_BUF_SIZE);
-
-        dtrace_pid_str(from, sender_str);
-        dtrace_proc_str(rp, receiver_str);
-        erts_snprintf(reason_buf, sizeof(DTRACE_CHARBUF_NAME(reason_buf)) - 1, "%T", reason);
-        DTRACE3(process_exit_signal, sender_str, receiver_str, reason_buf);
+void
+erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt)
+{
+    Process *c_p = ((ErtsProcExitContext *) vctxt)->c_p;
+    Eterm reason = ((ErtsProcExitContext *) vctxt)->reason;
+    ErtsMonitorData *mdp = NULL;
+
+    if (erts_monitor_is_target(mon)) {
+        /* We are being watched... */
+        switch (mon->type) {
+        case ERTS_MON_TYPE_PROC:
+            erts_proc_sig_send_monitor_down(mon, reason);
+            mon = NULL;
+            break;
+        case ERTS_MON_TYPE_PORT: {
+	    Port *prt;
+            ASSERT(is_internal_port(mon->other.item));
+            prt = erts_id2port(mon->other.item);
+	    if (prt) {
+                erts_fire_port_monitor(prt, mon);
+                erts_port_release(prt);
+                mon = NULL;
+            }
+            break;
+        }
+        case ERTS_MON_TYPE_RESOURCE:
+            erts_fire_nif_monitor(mon);
+            mon = NULL;
+            break;
+        case ERTS_MON_TYPE_DIST_PROC: {
+            ErtsMonLnkDist *dist;
+            DistEntry *dep;
+            ErtsDSigData dsd;
+            int code;
+            Eterm watcher;
+            Eterm watched;
+
+            mdp = erts_monitor_to_data(mon);
+
+            if (mon->flags & ERTS_ML_FLG_NAME)
+                watched = ((ErtsMonitorDataExtended *) mdp)->u.name;
+            else
+                watched = c_p->common.id;
+            ASSERT(is_internal_pid(watched) || is_atom(watched));
+
+            watcher = mon->other.item;
+	    ASSERT(is_external_pid(watcher));    
+	    dep = external_pid_dist_entry(watcher);
+            ASSERT(dep);
+            dist = ((ErtsMonitorDataExtended *) mdp)->dist;
+            ASSERT(dist);
+            code = erts_dsig_prepare(&dsd, dep, NULL, 0,
+                                     ERTS_DSP_NO_LOCK, 0, 0);
+            switch (code) {
+            case ERTS_DSIG_PREP_CONNECTED:
+            case ERTS_DSIG_PREP_PENDING:
+                if (dist->connection_id == dsd.connection_id) {
+                    code = erts_dsig_send_m_exit(&dsd,
+                                                 watcher,
+                                                 watched,
+                                                 mdp->ref,
+                                                 reason);
+                    ASSERT(code == ERTS_DSIG_SEND_OK);
+                }
+            default:
+                break;
+            }
+            if (!erts_monitor_dist_delete(&mdp->origin))
+                mdp = NULL;
+            break;
+        }
+        default:
+            ERTS_INTERNAL_ERROR("Invalid target monitor type");
+            break;
+        }
     }
-#endif
-
-    if ((state & ERTS_PSFLG_TRAP_EXIT)
-	&& (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) {
-        /* have to release the status and trace lock in order to send the exit message */
-        erts_proc_unlock(rp, *rp_locks & (ERTS_PROC_LOCKS_XSIG_SEND|ERTS_PROC_LOCK_TRACE));
-        *rp_locks &= ~(ERTS_PROC_LOCKS_XSIG_SEND|ERTS_PROC_LOCK_TRACE);
-        if (have_seqtrace(token) && token_update)
-	    seq_trace_update_send(token_update);
-	if (is_value(exit_tuple))
-	    send_exit_message(rp, rp_locks, exit_tuple, exit_tuple_sz, token);
-	else
-	    erts_deliver_exit_message(from, rp, rp_locks, rsn, token);
-	return 1; /* Receiver will get a message */
-    }
-    else if (reason != am_normal || (flags & ERTS_XSIG_FLG_NO_IGN_NORMAL)) {
-	if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))) {
-	    ASSERT(!rp->pending_exit.bp);
-
-	    if (rp == c_p && (*rp_locks & ERTS_PROC_LOCK_MAIN)) {
-		/* Ensure that all locks on c_p are locked before
-		   proceeding... */
-		if (*rp_locks != ERTS_PROC_LOCKS_ALL) {
-		    ErtsProcLocks need_locks = (~(*rp_locks)
-						& ERTS_PROC_LOCKS_ALL);
-		    if (erts_proc_trylock(c_p, need_locks) == EBUSY) {
-			erts_proc_unlock(c_p,
-					     *rp_locks & ~ERTS_PROC_LOCK_MAIN);
-			erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
-		    }
-		    *rp_locks = ERTS_PROC_LOCKS_ALL;
-		}
-		set_proc_exiting(c_p, state, rsn, NULL);
-	    }
-	    else if (!(state & (ERTS_PSFLG_RUNNING
-                                | ERTS_PSFLG_RUNNING_SYS
-                                | ERTS_PSFLG_DIRTY_RUNNING_SYS))) {
-		/* Process not running ... */
-		ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL;
-		ErlHeapFragment *bp = NULL;
-		Eterm rsn_cpy;
-		if (need_locks
-		    && erts_proc_trylock(rp, need_locks) == EBUSY) {
-		    /* ... but we havn't got all locks on it ... */
-		    save_pending_exiter(rp, NULL); 
-		    /*
-		     * The pending exit will be discovered when next
-		     * process is scheduled in
-		     */
-                    goto set_pending_exit;
-		}
-                /* ...and we have all locks on it... */
-                *rp_locks = ERTS_PROC_LOCKS_ALL;
-
-		state = erts_atomic32_read_nob(&rp->state);
-
-		if (is_immed(rsn))
-		    rsn_cpy = rsn;
-		else {
-		    Eterm *hp;
-		    ErlOffHeap *ohp;
-		    Uint rsn_sz = size_object(rsn);
-		    if (state & ERTS_PSFLG_DIRTY_RUNNING) {
-			bp = new_message_buffer(rsn_sz);
-			ohp = &bp->off_heap;
-			hp = &bp->mem[0];
-		    }
-		    else
-		    {
-			hp = HAlloc(rp, rsn_sz);
-			ohp = &rp->off_heap;
-		    }
-		    rsn_cpy = copy_struct(rsn, rsn_sz, &hp, ohp);
-		}
-
-		set_proc_exiting(rp, state, rsn_cpy, bp);
-	    }
-	    else { /* Process running... */
- 
-		/*
-		 * The pending exit will be discovered when the process
-		 * is scheduled out if not discovered earlier.
-		 */
-
-	    set_pending_exit:
-		if (is_immed(rsn)) {
-		    rp->pending_exit.reason = rsn;
-		}
-		else {
-		    Eterm *hp;
-		    Uint sz = size_object(rsn);
-		    ErlHeapFragment *bp = new_message_buffer(sz);
-
-		    hp = &bp->mem[0];
-		    rp->pending_exit.reason = copy_struct(rsn,
-							  sz,
-							  &hp,
-							  &bp->off_heap);
-		    rp->pending_exit.bp = bp;
-		}
-
-		/*
-		 * If no dirty work has been scheduled, pending exit will
-                 * be discovered when the process is scheduled. If dirty work
-                 * has been scheduled, we may need to add it to a normal run
-                 * queue...
-		 */
-                {
-                    erts_aint32_t a = erts_atomic32_read_nob(&rp->state);
-                    while (1) {
-                        erts_aint32_t n, e;
-                        int dwork;
-                        n = e = a;
-                        n |= ERTS_PSFLG_PENDING_EXIT;
-                        dwork = !!(n & ERTS_PSFLGS_DIRTY_WORK);
-                        n &= ~ERTS_PSFLGS_DIRTY_WORK;
-                        a = erts_atomic32_cmpxchg_mb(&rp->state, n, e);
-                        if (a == e) {
-                            if (dwork)
-                                erts_schedule_process(rp, n, *rp_locks);
-                            break;
-                        }
-                    }
+    else { /* Origin monitor */
+        /* We are watching someone else... */
+        switch (mon->type) {
+        case ERTS_MON_TYPE_PROC:
+            erts_proc_sig_send_demonitor(mon);
+            mon = NULL;
+            break;
+        case ERTS_MON_TYPE_TIME_OFFSET:
+            erts_demonitor_time_offset(mon);
+            mon = NULL;
+            break;
+        case ERTS_MON_TYPE_NODE:
+            mdp = erts_monitor_to_data(mon);
+            if (!erts_monitor_dist_delete(&mdp->target))
+                mdp = NULL;
+            break;
+        case ERTS_MON_TYPE_NODES:
+            erts_monitor_nodes_delete(mon);
+            mon = NULL;
+            break;
+        case ERTS_MON_TYPE_PORT: {
+            Port *prt;
+            ASSERT(is_internal_port(mon->other.item));
+            prt = erts_port_lookup_raw(mon->other.item);
+            if (prt) {
+                erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+                if (erts_port_demonitor(c_p, prt, mon) != ERTS_PORT_OP_DROPPED)
+                    mon = NULL;
+                erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+            }
+            break;
+        }
+        case ERTS_MON_TYPE_DIST_PROC: {
+            ErtsMonLnkDist *dist;
+            DistEntry *dep;
+            ErtsDSigData dsd;
+            int code;
+            Eterm watched;
+
+            mdp = erts_monitor_to_data(mon);
+            dist = ((ErtsMonitorDataExtended *) mdp)->dist;
+            ASSERT(dist);
+            if (mon->flags & ERTS_ML_FLG_NAME) {
+                watched = ((ErtsMonitorDataExtended *) mdp)->u.name;
+                ASSERT(is_atom(watched));
+                dep = erts_sysname_to_connected_dist_entry(dist->nodename);
+            }
+            else {
+                watched = mon->other.item;
+                ASSERT(is_external_pid(watched));
+		dep = external_pid_dist_entry(watched);
+            }
+            code = erts_dsig_prepare(&dsd, dep, NULL, 0,
+                                     ERTS_DSP_NO_LOCK, 0, 0);
+            switch (code) {
+            case ERTS_DSIG_PREP_CONNECTED:
+            case ERTS_DSIG_PREP_PENDING:
+                if (dist->connection_id == dsd.connection_id) {
+                    code = erts_dsig_send_demonitor(&dsd,
+                                                    c_p->common.id,
+                                                    watched,
+                                                    mdp->ref,
+                                                    1);
+                    ASSERT(code == ERTS_DSIG_SEND_OK);
                 }
-	    }
-	}
-	/* else:
-	 *
-	 *    The receiver already has a pending exit (or is exiting)
-	 *    so we drop this signal.
-	 *
-	 *    NOTE: dropping this exit signal is based on the assumption
-	 *          that the receiver *will* exit; either on the pending
-	 *          exit or by itself before seeing the pending exit.
-	 */
-	return -1; /* Receiver will exit */
+            default:
+                break;
+            }
+            if (!erts_monitor_dist_delete(&mdp->target))
+                mdp = NULL;
+            break;
+        }
+        default:
+            ERTS_INTERNAL_ERROR("Invalid origin monitor type");
+            break;
+        }
     }
 
-    return 0; /* Receiver unaffected */
-}
-
-
-int
-erts_send_exit_signal(Process *c_p,
-		      Eterm from,
-		      Process *rp,
-		      ErtsProcLocks *rp_locks,
-		      Eterm reason,
-		      Eterm token,
-		      Process *token_update,
-		      Uint32 flags)
-{
-    return send_exit_signal(c_p,
-			    from,
-			    rp,
-			    rp_locks,
-			    reason,
-			    THE_NON_VALUE,
-			    0,
-			    token,
-			    token_update,
-			    flags);
+    if (mdp)
+        erts_monitor_release_both(mdp);
+    else if (mon)
+        erts_monitor_release(mon);
 }
 
-typedef struct {
-    Eterm reason;
-    Process *p;
-} ExitMonitorContext;
-
-static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
-{
-    ExitMonitorContext *pcontext = vpcontext;
-    DistEntry *dep;
-    ErtsMonitor *rmon;
-
-    switch (mon->type) {
-    case MON_ORIGIN:
-	/* We are monitoring someone else, we need to demonitor that one.. */
-	if (is_atom(mon->u.pid)) { /* remote by name */
-	    ASSERT(is_node_name_atom(mon->u.pid));
-	    dep = erts_sysname_to_connected_dist_entry(mon->u.pid);
-	    if (dep) {
-		erts_de_links_lock(dep);
-		rmon = erts_remove_monitor(&(dep->monitors), mon->ref);
-		erts_de_links_unlock(dep);
-		if (rmon) {
-		    ErtsDSigData dsd;
-		    int code = erts_dsig_prepare(&dsd, dep, NULL, 0,
-						 ERTS_DSP_NO_LOCK, 0, 0);
-		    if (code == ERTS_DSIG_PREP_CONNECTED ||
-                        code == ERTS_DSIG_PREP_PENDING) {
-
-			code = erts_dsig_send_demonitor(&dsd,
-							rmon->u.pid,
-							mon->name,
-							mon->ref,
-							1);
-			ASSERT(code == ERTS_DSIG_SEND_OK);
-		    }
-		    erts_destroy_monitor(rmon);
-		}
-	    }
-	} else {
-            ASSERT(is_pid(mon->u.pid) || is_port(mon->u.pid));
-            /* if is local by pid or name */
-            if (is_internal_pid(mon->u.pid)) {
-                Process *rp = erts_pid2proc(NULL, 0, mon->u.pid, ERTS_PROC_LOCK_LINK);
-		if (!rp) {
-		    goto done;
-		}
-		rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref);
-		erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-		if (rmon == NULL) {
-		    goto done;
-		}
-		erts_destroy_monitor(rmon);
-            } else if (is_internal_port(mon->u.pid)) {
-                /* Is a local port */
-                Port *prt = erts_port_lookup_raw(mon->u.pid);
-                if (!prt) {
-                    goto done;
-                }
-                erts_port_demonitor(pcontext->p,
-                                    ERTS_PORT_DEMONITOR_ORIGIN_ON_DEATHBED,
-                                    prt, mon->ref, NULL);
-            } else { /* remote by pid */
-		ASSERT(is_external_pid(mon->u.pid));
-		dep = external_pid_dist_entry(mon->u.pid);
-		ASSERT(dep != NULL);
-		if (dep) {
-		    erts_de_links_lock(dep);
-		    rmon = erts_remove_monitor(&(dep->monitors), mon->ref);
-		    erts_de_links_unlock(dep);
-		    if (rmon) {
-			ErtsDSigData dsd;
-			int code = erts_dsig_prepare(&dsd, dep, NULL, 0,
-						     ERTS_DSP_NO_LOCK, 0, 0);
-			if (code == ERTS_DSIG_PREP_CONNECTED ||
-                            code == ERTS_DSIG_PREP_PENDING) {
-
-			    code = erts_dsig_send_demonitor(&dsd,
-							    rmon->u.pid,
-							    mon->u.pid,
-							    mon->ref,
-							    1);
-			    ASSERT(code == ERTS_DSIG_SEND_OK);
-			}
-			erts_destroy_monitor(rmon);
-		    }
-		}
-	    }
-	}
-	break;
-    case MON_TARGET:
-	ASSERT(is_pid(mon->u.pid) || is_internal_port(mon->u.pid));
-	if (is_internal_port(mon->u.pid)) {
-	    Port *prt = erts_id2port(mon->u.pid);
-	    if (prt == NULL) {
-		goto done;
-	    }
-	    erts_fire_port_monitor(prt, mon->ref);
-	    erts_port_release(prt); 
-	} else if (is_internal_pid(mon->u.pid)) {/* local by name or pid */
-	    Eterm watched;
-            Process *rp;
-	    DeclareTmpHeapNoproc(lhp,3);
-	    ErtsProcLocks rp_locks = (ERTS_PROC_LOCK_LINK
-				      | ERTS_PROC_LOCKS_MSG_SEND);
-	    rp = erts_pid2proc(NULL, 0, mon->u.pid, rp_locks);
-	    if (rp == NULL) {
-		goto done;
-	    }
-	    UseTmpHeapNoproc(3);
-	    rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref);
-	    if (rmon) {
-		erts_destroy_monitor(rmon);
-		watched = (is_atom(mon->name)
-			   ? TUPLE2(lhp, mon->name, 
-				    erts_this_dist_entry->sysname)
-			   : pcontext->p->common.id);
-		erts_queue_monitor_message(rp, &rp_locks, mon->ref, am_process, 
-					   watched, pcontext->reason);
-	    }
-	    UnUseTmpHeapNoproc(3);
-	    /* else: demonitor while we exited, i.e. do nothing... */
-	    erts_proc_unlock(rp, rp_locks);
-	} else { /* external by pid or name */
-	    ASSERT(is_external_pid(mon->u.pid));    
-	    dep = external_pid_dist_entry(mon->u.pid);
-	    ASSERT(dep != NULL);
-	    if (dep) {
-		erts_de_links_lock(dep);
-		rmon = erts_remove_monitor(&(dep->monitors), mon->ref);
-		erts_de_links_unlock(dep);
-		if (rmon) {
-		    ErtsDSigData dsd;
-		    int code = erts_dsig_prepare(&dsd, dep, NULL, 0,
-						 ERTS_DSP_NO_LOCK, 0, 0);
-		    if (code == ERTS_DSIG_PREP_CONNECTED) {
-			code = erts_dsig_send_m_exit(&dsd,
-						     mon->u.pid,
-						     (rmon->name != NIL
-						      ? rmon->name
-						      : rmon->u.pid),
-						     mon->ref,
-						     pcontext->reason);
-			ASSERT(code == ERTS_DSIG_SEND_OK);
-		    }
-		    erts_destroy_monitor(rmon);
-		}
-	    }
-	}
-	break;
-    case MON_NIF_TARGET:
-        erts_fire_nif_monitor(mon->u.resource,
-                              pcontext->p->common.id,
-                              mon->ref);
+void
+erts_proc_exit_handle_link(ErtsLink *lnk, void *vctxt)
+{
+    Process *c_p = ((ErtsProcExitContext *) vctxt)->c_p;
+    Eterm reason = ((ErtsProcExitContext *) vctxt)->reason;
+    ErtsLinkData *ldp = NULL;
+
+    switch (lnk->type) {
+    case ERTS_LNK_TYPE_PROC:
+        ASSERT(is_internal_pid(lnk->other.item));
+        erts_proc_sig_send_link_exit(c_p, c_p->common.id, lnk,
+                                     reason, SEQ_TRACE_TOKEN(c_p));
+        lnk = NULL;
+        break;
+    case ERTS_LNK_TYPE_PORT: {
+        Port *prt;
+        ASSERT(is_internal_port(lnk->other.item));
+        prt = erts_port_lookup(lnk->other.item,
+                               ERTS_PORT_SFLGS_INVALID_LOOKUP);
+        if (prt)
+            erts_port_exit(NULL,
+                           (ERTS_PORT_SIG_FLG_FORCE_SCHED
+                            | ERTS_PORT_SIG_FLG_BROKEN_LINK),
+                           prt,
+                           c_p->common.id,
+                           reason,
+                           NULL);
+        break;
+    }
+    case ERTS_LNK_TYPE_DIST_PROC: {
+        DistEntry *dep;
+        ErtsMonLnkDist *dist;
+        ErtsLink *dlnk;
+        ErtsDSigData dsd;
+        int code;
+
+        dlnk = erts_link_to_other(lnk, &ldp);
+        dist = ((ErtsLinkDataExtended *) ldp)->dist;
+
+        ASSERT(is_external_pid(lnk->other.item));
+        dep = external_pid_dist_entry(lnk->other.item);
+
+        ASSERT(dep != erts_this_dist_entry);
+
+        if (!erts_link_dist_delete(dlnk))
+            ldp = NULL;
+
+        erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+        code = erts_dsig_prepare(&dsd, dep, c_p, 0, ERTS_DSP_NO_LOCK, 0, 0);
+        switch (code) {
+        case ERTS_DSIG_PREP_CONNECTED:
+        case ERTS_DSIG_PREP_PENDING:
+            if (dist->connection_id == dsd.connection_id) {
+                code = erts_dsig_send_exit_tt(&dsd,
+                                              c_p->common.id,
+                                              lnk->other.item,
+                                              reason,
+                                              SEQ_TRACE_TOKEN(c_p));
+                ASSERT(code == ERTS_DSIG_SEND_OK);
+            }
+        }
+        erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
         break;
-    case MON_TIME_OFFSET:
-	erts_demonitor_time_offset(mon->ref);
-	break;
-    default:
-	ERTS_INTERNAL_ERROR("Invalid monitor type");
     }
- done:
-    /* As the monitors are previously removed from the process, 
-       distribution operations will not cause monitors to disappear,
-       we can safely delete it. */
-       
-    erts_destroy_monitor(mon);
-}
-
-typedef struct {
-    Process *p;
-    Eterm reason;
-    Eterm exit_tuple;
-    Uint exit_tuple_sz;
-} ExitLinkContext;
-
-static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
-{
-    ExitLinkContext *pcontext = vpcontext;
-    /* Unpack context, it's readonly */
-    Process *p = pcontext->p;
-    Eterm reason = pcontext->reason;
-    Eterm exit_tuple = pcontext->exit_tuple;
-    Uint exit_tuple_sz = pcontext->exit_tuple_sz;
-    Eterm item = lnk->pid;
-    ErtsLink *rlnk;
-    DistEntry *dep;
-    Process *rp;
-
-
-    switch(lnk->type) {
-    case LINK_PID:
-	if(is_internal_port(item)) {
-	    Port *prt = erts_port_lookup(item, ERTS_PORT_SFLGS_INVALID_LOOKUP);
-	    if (prt)
-		erts_port_exit(NULL,
-			       (ERTS_PORT_SIG_FLG_FORCE_SCHED
-				| ERTS_PORT_SIG_FLG_BROKEN_LINK),
-			       prt,
-			       p->common.id,
-			       reason,
-			       NULL);
-	}
-	else if(is_external_port(item)) {
-	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	    erts_dsprintf(dsbufp,
-			  "Erroneous link between %T and external port %T "
-			  "found\n",
-			  p->common.id,
-			  item);
-	    erts_send_error_to_logger_nogl(dsbufp);
-	    ASSERT(0); /* It isn't possible to setup such a link... */
-	}
-	else if (is_internal_pid(item)) {
-	    ErtsProcLocks rp_locks = (ERTS_PROC_LOCK_LINK
-				      | ERTS_PROC_LOCKS_XSIG_SEND);
-	    rp = erts_pid2proc(NULL, 0, item, rp_locks);
-	    if (rp) {
-		rlnk = erts_remove_link(&ERTS_P_LINKS(rp), p->common.id);
-		/* If rlnk == NULL, we got unlinked while exiting,
-		   i.e., do nothing... */
-		if (rlnk) {
-		    int xres;
-		    erts_destroy_link(rlnk);
-		    xres = send_exit_signal(NULL,
-					    p->common.id,
-					    rp,
-					    &rp_locks, 
-					    reason,
-					    exit_tuple,
-					    exit_tuple_sz,
-					    SEQ_TRACE_TOKEN(p),
-					    p,
-					    ERTS_XSIG_FLG_IGN_KILL);
-		    if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) {
-			/* We didn't exit the process and it is traced */
-			if (IS_TRACED_FL(rp, F_TRACE_PROCS)) {
-                            if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) {
-                                erts_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND);
-                                rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND;
-                            }
-			    trace_proc(NULL, 0, rp, am_getting_unlinked, p->common.id);
-			}
-		    }
-		}
-		ASSERT(rp != p);
-		erts_proc_unlock(rp, rp_locks);
-	    }
-	}
-	else if (is_external_pid(item)) {
-	    dep = external_pid_dist_entry(item);
-	    if(dep != erts_this_dist_entry) {
-		ErtsDSigData dsd;
-		int code;
-		ErtsDistLinkData dld;
-		erts_remove_dist_link(&dld, p->common.id, item, dep);
-                if (dld.d_lnk) {
-                    erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
-                    code = erts_dsig_prepare(&dsd, dep, p, 0, ERTS_DSP_NO_LOCK, 0, 0);
-                    if (code == ERTS_DSIG_PREP_CONNECTED ||
-                        code == ERTS_DSIG_PREP_PENDING) {
-
-                        code = erts_dsig_send_exit_tt(&dsd, p->common.id, item,
-                                                      reason, SEQ_TRACE_TOKEN(p));
-                        ASSERT(code == ERTS_DSIG_SEND_OK);
-                    }
-                    erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
-                }
-		erts_destroy_dist_link(&dld);
-	    }
-	}
-	break;
-    case LINK_NODE:
-	ASSERT(is_node_name_atom(item));
-	dep = erts_sysname_to_connected_dist_entry(item);
-	if(dep) {
-	    /* dist entries have node links in a separate structure to 
-	       avoid confusion */
-	    erts_de_links_lock(dep);
-	    rlnk = erts_remove_link(&(dep->node_links), p->common.id);
-	    erts_de_links_unlock(dep);
-	    if (rlnk)
-		erts_destroy_link(rlnk);
-	}
-	break;
-	
     default:
-	erts_exit(ERTS_ERROR_EXIT, "bad type in link list\n");
-	break;
+        ERTS_INTERNAL_ERROR("Unexpected link type");
+        break;
     }
-    erts_destroy_link(lnk);
+
+    if (ldp)
+        erts_link_release_both(ldp);
+    else if (lnk)
+        erts_link_release(lnk);
 }
 
 static void
-resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p)
+resume_suspend_monitor(ErtsMonitor *mon, void *vc_p)
 {
+    ErtsMonitorSuspend *smon = erts_monitor_suspend(mon);
     Process *suspendee = erts_pid2proc((Process *) vc_p, ERTS_PROC_LOCK_MAIN,
-				       smon->pid, ERTS_PROC_LOCK_STATUS);
+				       smon->mon.other.item, ERTS_PROC_LOCK_STATUS);
     if (suspendee) {
 	ASSERT(suspendee != vc_p);
 	if (smon->active)
 	    resume_process(suspendee, ERTS_PROC_LOCK_STATUS);
 	erts_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
     }
-    erts_destroy_suspend_monitor(smon);
+    erts_monitor_suspend_destroy(smon);
 }
 
 /* this function fishishes a process and propagates exit messages - called
@@ -12923,8 +12608,6 @@ void
 erts_do_exit_process(Process* p, Eterm reason)
 {
     p->arity = 0;		/* No live registers */
-    p->fvalue = reason;
-    
 
 #ifdef USE_VM_PROBES
     if (DTRACE_ENABLED(process_exit)) {
@@ -12948,18 +12631,11 @@ erts_do_exit_process(Process* p, Eterm reason)
        process from exiting until the lock has been released. */
     erts_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
 
-    if (ERTS_PSFLG_PENDING_EXIT & set_proc_self_exiting(p)) {
-	/* Process exited before pending exit was received... */
-	p->pending_exit.reason = THE_NON_VALUE;
-	if (p->pending_exit.bp) {
-	    free_message_buffer(p->pending_exit.bp);
-	    p->pending_exit.bp = NULL;
-	}
-    }
+    set_self_exiting(p, reason, NULL, NULL, NULL);
 
     cancel_suspend_of_suspendee(p, ERTS_PROC_LOCKS_ALL); 
 
-    ERTS_MSGQ_MV_INQ2PRIVQ(p);
+    erts_proc_sig_fetch(p);
 
     if (IS_TRACED(p)) {
 	if (IS_TRACED_FL(p, F_TRACE_CALLS))
@@ -12996,13 +12672,15 @@ erts_do_exit_process(Process* p, Eterm reason)
 void
 erts_continue_exit_process(Process *p)
 {
-    ErtsLink* lnk;
-    ErtsMonitor *mon;
+    ErtsLink *links;
+    ErtsMonitor *monitors;
+    ErtsMonitor *lt_monitors;
     ErtsProcLocks curr_locks = ERTS_PROC_LOCK_MAIN;
     Eterm reason = p->fvalue;
     DistEntry *dep = NULL;
     erts_aint32_t state;
     int delay_del_proc = 0;
+    ErtsProcExitContext pectxt;
 
 #ifdef DEBUG
     int yield_allowed = 1;
@@ -13074,7 +12752,7 @@ erts_continue_exit_process(Process *p)
 
     erts_set_gc_state(p, 1);
     state = erts_atomic32_read_acqb(&p->state);
-    if (state & ERTS_PSFLG_ACTIVE_SYS
+    if ((state & ERTS_PSFLG_SYS_TASKS)
         || p->dirty_sys_tasks
         ) {
 	if (cleanup_sys_tasks(p, state, CONTEXT_REDS) >= CONTEXT_REDS/2)
@@ -13094,18 +12772,10 @@ erts_continue_exit_process(Process *p)
 	p->flags &= ~F_USING_DDLL;
     }
 
-    if (p->nodes_monitors) {
-	erts_delete_nodes_monitors(p, ERTS_PROC_LOCK_MAIN);
-	p->nodes_monitors = NULL;
-    }
-	
-
-    if (p->suspend_monitors) {
-	erts_sweep_suspend_monitors(p->suspend_monitors,
-				    resume_suspend_monitor,
-				    p);
-	p->suspend_monitors = NULL;
-    }
+    if (p->suspend_monitors)
+        erts_monitor_tree_foreach_delete(&p->suspend_monitors,
+                                         resume_suspend_monitor,
+                                         p);
 
     /*
      * The registered name *should* be the last "erlang resource" to
@@ -13134,8 +12804,9 @@ erts_continue_exit_process(Process *p)
      * Note! The monitor and link fields will be overwritten 
      * by erts_ptab_delete_element() below.
      */
-    mon = ERTS_P_MONITORS(p);
-    lnk = ERTS_P_LINKS(p);
+    links = ERTS_P_LINKS(p);
+    monitors = ERTS_P_MONITORS(p);
+    lt_monitors = ERTS_P_LT_MONITORS(p);
 
     {
 	/* Do *not* use erts_get_runq_proc() */
@@ -13172,7 +12843,9 @@ erts_continue_exit_process(Process *p)
 	    n = e = a;
 	    ASSERT(a & ERTS_PSFLG_EXITING);
 	    n |= ERTS_PSFLG_FREE;
-	    n &= ~ERTS_PSFLG_ACTIVE;
+	    n &= ~(ERTS_PSFLG_ACTIVE
+                   | ERTS_PSFLG_ACTIVE_SYS
+                   | ERTS_PSFLG_DIRTY_ACTIVE_SYS);
 	    if ((n & ERTS_PSFLG_IN_RUNQ) && !refc_inced) {
 		erts_proc_inc_refc(p);
 		refc_inced = 1;
@@ -13207,33 +12880,40 @@ erts_continue_exit_process(Process *p)
         erts_deref_dist_entry(dep);
     }
 
-    /*
-     * Pre-build the EXIT tuple if there are any links.
-     */
-    if (lnk) {
-	DeclareTmpHeap(tmp_heap,4,p);
-	Eterm exit_tuple;
-	Uint exit_tuple_sz;
-	Eterm* hp;
-
-	UseTmpHeap(4,p);
-	hp = &tmp_heap[0];
+    pectxt.c_p = p;
+    pectxt.reason = reason;
 
-	exit_tuple = TUPLE3(hp, am_EXIT, p->common.id, reason);
+    if (links) {
+        erts_link_tree_foreach_delete(&links,
+                                      erts_proc_exit_handle_link,
+                                      (void *) &pectxt);
+        ASSERT(!links);
+    }
 
-	exit_tuple_sz = size_object(exit_tuple);
+    if (monitors) {
+        erts_monitor_tree_foreach_delete(&monitors,
+                                         erts_proc_exit_handle_monitor,
+                                         (void *) &pectxt);
+        ASSERT(!monitors);
+    }
 
-	{
-	    ExitLinkContext context = {p, reason, exit_tuple, exit_tuple_sz};
-	    erts_sweep_links(lnk, &doit_exit_link, &context);
-	}
-	UnUseTmpHeap(4,p);
+    if (lt_monitors) {
+        erts_monitor_list_foreach_delete(&lt_monitors,
+                                         erts_proc_exit_handle_monitor,
+                                         (void *) &pectxt);
+        ASSERT(!lt_monitors);
     }
 
-    {
-	ExitMonitorContext context = {reason, p};
-	erts_sweep_monitors(mon,&doit_exit_monitor,&context); /* Allocates TmpHeap, but we
-								 have none here */
+    erts_proc_sig_fetch(p);
+    /*
+     * erts_proc_sig_handle_exit() implements yielding.
+     * However, this function cannot handle it yet... loop
+     * until done...
+     */
+    while (!0) {
+        int reds = CONTEXT_REDS;
+        if (erts_proc_sig_handle_exit(p, &reds))
+            break;
     }
 
     erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
@@ -13272,6 +12952,40 @@ erts_continue_exit_process(Process *p)
     BUMP_ALL_REDS(p);
 }
 
+Process *
+erts_try_lock_sig_free_proc(Eterm pid, ErtsProcLocks locks)
+{
+    Process *rp = erts_proc_lookup_raw(pid);
+    erts_aint32_t state;
+
+    if (!rp)
+        return NULL;
+
+    ERTS_LC_ASSERT(!erts_proc_lc_my_proc_locks(rp));
+
+    state = erts_atomic32_read_nob(&rp->state);
+    if (state & ERTS_PSFLG_EXITING)
+        return NULL;
+
+    if (state & (ERTS_PSFLG_SIG_IN_Q|ERTS_PSFLG_SIG_Q))
+        return ERTS_PROC_LOCK_BUSY;
+    if (erts_proc_trylock(rp, locks) == EBUSY)
+        return ERTS_PROC_LOCK_BUSY;
+
+    state = erts_atomic32_read_nob(&rp->state);
+    if (state & ERTS_PSFLG_EXITING) {
+        erts_proc_unlock(rp, locks);
+        return NULL;
+    }
+
+    if (state & (ERTS_PSFLG_SIG_IN_Q|ERTS_PSFLG_SIG_Q)) {
+        erts_proc_unlock(rp, locks);
+        return ERTS_PROC_LOCK_BUSY;
+    }
+
+    return rp;
+}
+
 /*
  * Stack dump functions follow.
  */
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index ebf990c837..0550fb05b5 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -51,7 +51,7 @@ typedef struct process Process;
 #include "erl_process_dict.h"
 #include "erl_node_container_utils.h"
 #include "erl_node_tables.h"
-#include "erl_monitors.h"
+#include "erl_monitor_link.h"
 #include "erl_hl_timer.h"
 #include "erl_time.h"
 #include "erl_atom_table.h"
@@ -75,8 +75,6 @@ typedef struct process Process;
 #include "erl_thr_progress.h"
 #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
 
-struct ErtsNodesMonitor_;
-
 #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT	0
 #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT		0
 
@@ -311,7 +309,6 @@ typedef enum {
     ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX,
     ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX,
     ERTS_SSI_AUX_WORK_MISC_IX,
-    ERTS_SSI_AUX_WORK_PENDING_EXITERS_IX,
     ERTS_SSI_AUX_WORK_SET_TMO_IX,
     ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
     ERTS_SSI_AUX_WORK_YIELD_IX,
@@ -345,8 +342,6 @@ typedef enum {
     (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX)
 #define ERTS_SSI_AUX_WORK_MISC \
     (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX)
-#define ERTS_SSI_AUX_WORK_PENDING_EXITERS \
-    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_PENDING_EXITERS_IX)
 #define ERTS_SSI_AUX_WORK_SET_TMO \
     (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
 #define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
@@ -491,7 +486,6 @@ struct ErtsRunQueue_ {
     int wakeup_other_reds;
 
     struct {
-	ErtsProcList *pending_exiters;
 	Uint context_switches;
 	Uint reductions;
 
@@ -986,14 +980,10 @@ struct process {
 
     Process *next;		/* Pointer to next process in run queue */
 
-    struct ErtsNodesMonitor_ *nodes_monitors;
-
-    ErtsSuspendMonitor *suspend_monitors; /* Processes suspended by
-					     this process via
-					     erlang:suspend_process/1 */
-
-    ErlMessageQueue msg;	/* Message queue */
+    ErtsMonitor *suspend_monitors; /* Processes suspended by this process via
+                                      erlang:suspend_process/1 */
 
+    ErtsSignalPrivQueues sig_qs; /* Signal queues */
     ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
 
     ProcDict  *dictionary;       /* Process dictionary, may be NULL */
@@ -1052,9 +1042,8 @@ struct process {
     erts_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */
     erts_atomic32_t dirty_state; /* Process dirty state flags (see ERTS_PDSFLG_*) */
 
-    ErlMessageInQueue msg_inq;
+    ErtsSignalInQueue sig_inq;
     ErlTraceMessageQueue *trace_msg_q;
-    ErtsPendExit pending_exit;
     erts_proc_lock_t lock;
     ErtsSchedulerData *scheduler_data;
     Eterm suspendee;
@@ -1088,6 +1077,7 @@ struct process {
 #endif
 };
 
+extern Eterm erts_init_process_id; /* pid of init process */
 extern const Process erts_invalid_process;
 
 #ifdef CHECK_FOR_HOLES
@@ -1164,20 +1154,20 @@ void erts_check_for_holes(Process* p);
 #define ERTS_PSFLG_IN_PRQ_LOW 		ERTS_PSFLG_BIT(3)
 #define ERTS_PSFLG_FREE			ERTS_PSFLG_BIT(4)
 #define ERTS_PSFLG_EXITING		ERTS_PSFLG_BIT(5)
-#define ERTS_PSFLG_PENDING_EXIT		ERTS_PSFLG_BIT(6)
+#define ERTS_PSFLG_UNUSED		ERTS_PSFLG_BIT(6)
 #define ERTS_PSFLG_ACTIVE		ERTS_PSFLG_BIT(7)
 #define ERTS_PSFLG_IN_RUNQ		ERTS_PSFLG_BIT(8)
 #define ERTS_PSFLG_RUNNING		ERTS_PSFLG_BIT(9)
 #define ERTS_PSFLG_SUSPENDED		ERTS_PSFLG_BIT(10)
 #define ERTS_PSFLG_GC			ERTS_PSFLG_BIT(11)
-/* #define ERTS_PSFLG_		        ERTS_PSFLG_BIT(12) */
-#define ERTS_PSFLG_TRAP_EXIT		ERTS_PSFLG_BIT(13)
+#define ERTS_PSFLG_SYS_TASKS		ERTS_PSFLG_BIT(12)
+#define ERTS_PSFLG_SIG_IN_Q		ERTS_PSFLG_BIT(13)
 #define ERTS_PSFLG_ACTIVE_SYS		ERTS_PSFLG_BIT(14)
 #define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
 #define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
 #define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
 #define ERTS_PSFLG_OFF_HEAP_MSGQ	ERTS_PSFLG_BIT(18)
-/* #define ERTS_PSFLG_			ERTS_PSFLG_BIT(19) */
+#define ERTS_PSFLG_SIG_Q		ERTS_PSFLG_BIT(19)
 #define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(20)
 #define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(21)
 #define ERTS_PSFLG_DIRTY_ACTIVE_SYS	ERTS_PSFLG_BIT(22)
@@ -1196,7 +1186,6 @@ void erts_check_for_holes(Process* p);
 					 | ERTS_PSFLG_IN_PRQ_LOW)
 
 #define ERTS_PSFLGS_VOLATILE_HEAP	(ERTS_PSFLG_EXITING		\
-					 | ERTS_PSFLG_PENDING_EXIT	\
 					 | ERTS_PSFLG_DIRTY_RUNNING	\
 					 | ERTS_PSFLG_DIRTY_RUNNING_SYS)
 
@@ -1379,6 +1368,9 @@ extern int erts_system_profile_ts_type;
 #define F_DIRTY_MAJOR_GC     (1 << 23) /* Dirty major GC scheduled */
 #define F_DIRTY_MINOR_GC     (1 << 24) /* Dirty minor GC scheduled */
 #define F_HIBERNATED         (1 << 25) /* Hibernated */
+#define F_LOCAL_SIGS_ONLY    (1 << 26)
+#define F_TRAP_EXIT          (1 << 27) /* Trapping exit */
+#define F_DEFERRED_SAVED_LAST (1 << 28)
 
 /*
  * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent
@@ -1444,7 +1436,7 @@ extern int erts_system_profile_ts_type;
 		     | F_TRACE_ARITY_ONLY | F_TRACE_RETURN_TO \
                      | F_TRACE_SILENT | F_TRACE_SCHED_PROCS | F_TRACE_PORTS \
 		     | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \
-		     | F_TRACE_SCHED_EXIT)
+		     | F_TRACE_SCHED_EXIT )
 
 
 #define ERTS_TRACEE_MODIFIER_FLAGS \
@@ -1457,6 +1449,14 @@ extern int erts_system_profile_ts_type;
 
 #define SEQ_TRACE_FLAG(N)        (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))
 
+#define ERTS_SIG_ENABLE_TRACE_FLAGS \
+    ( F_TRACE_RECEIVE | F_TRACE_PROCS)
+
+/*
+ * F_TRACE_RECEIVE is always enabled/disable via signaling.
+ * F_TRACE_PROCS enable/disable F_TRACE_PROCS_SIG via signaling.
+ */
+
 /* Sequential trace flags */
 
 /* SEQ_TRACE_TIMESTAMP_MASK is a bit-field */
@@ -1483,10 +1483,6 @@ extern int erts_system_profile_ts_type;
 #define DT_UTAG_FLAGS(P)  ((P)->dt_utag_flags) 
 #endif
 
-/* Option flags to erts_send_exit_signal() */
-#define ERTS_XSIG_FLG_IGN_KILL		(((Uint32) 1) << 0)
-#define ERTS_XSIG_FLG_NO_IGN_NORMAL	(((Uint32) 1) << 1)
-
 #define CANCEL_TIMER(P)					\
     do {						\
 	if ((P)->flags & (F_INSLPQUEUE|F_TIMO)) {	\
@@ -1733,6 +1729,7 @@ struct db_fixation;
 void erts_schedule_ets_free_fixation(Eterm pid, struct db_fixation*);
 void erts_schedule_flush_trace_messages(Process *proc, int force_on_proc);
 int erts_flush_trace_messages(Process *c_p, ErtsProcLocks locks);
+int erts_sig_prio(Eterm pid, int prio);
 
 #if defined(ERTS_ENABLE_LOCK_CHECK)
 int erts_dbg_check_halloc_lock(Process *p);
@@ -1781,8 +1778,10 @@ ErtsRunQueue *erts_schedid2runq(Uint);
 Process *erts_schedule(ErtsSchedulerData *, Process*, int);
 void erts_schedule_misc_op(void (*)(void *), void *);
 Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
+void erts_set_self_exiting(Process *, Eterm);
 void erts_do_exit_process(Process*, Eterm);
 void erts_continue_exit_process(Process *);
+void erts_proc_exit_link(Process *, ErtsLink *, Uint16, Eterm, Eterm);
 /* Begin System profile */
 Uint erts_runnable_process_count(void);
 /* End System profile */
@@ -1799,7 +1798,14 @@ void erts_print_run_queue_info(fmtfn_t, void *to_arg, ErtsRunQueue*);
 void erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg);
 void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg);
 
-Eterm erts_get_process_priority(Process *p);
+typedef struct {
+    Process *c_p;
+    Eterm reason;
+} ErtsProcExitContext;
+void erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt);
+void erts_proc_exit_handle_link(ErtsLink *lnk, void *vctxt);
+
+Eterm erts_get_process_priority(erts_aint32_t state);
 Eterm erts_set_process_priority(Process *p, Eterm prio);
 
 Uint erts_get_total_context_switches(void);
@@ -1817,18 +1823,6 @@ void erts_suspend(Process*, ErtsProcLocks, Port*);
 void erts_resume(Process*, ErtsProcLocks);
 int erts_resume_processes(ErtsProcList *);
 
-int erts_send_exit_signal(Process *,
-			  Eterm,
-			  Process *,
-			  ErtsProcLocks *,
-			  Eterm,
-			  Eterm,
-			  Process *,
-			  Uint32);
-void erts_handle_pending_exit(Process *, ErtsProcLocks);
-#define ERTS_PROC_PENDING_EXIT(P) \
-    (ERTS_PSFLG_PENDING_EXIT & erts_atomic32_read_acqb(&(P)->state))
-
 void erts_deep_process_dump(fmtfn_t, void *);
 
 Eterm erts_get_reader_groups_map(Process *c_p);
@@ -1860,6 +1854,8 @@ do {										\
 ErtsSchedulerData *erts_get_scheduler_data(void);
 
 void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks);
+erts_aint32_t erts_proc_sys_schedule(Process *p, erts_aint32_t state,
+                                     erts_aint32_t enable_flag);
 
 ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks);
 ERTS_GLB_INLINE void erts_schedule_dirty_sys_execution(Process *c_p);
@@ -1891,8 +1887,7 @@ erts_schedule_dirty_sys_execution(Process *c_p)
     /* Don't set dirty-active-sys if we are about to exit... */
 
     while (!(a & (ERTS_PSFLG_DIRTY_ACTIVE_SYS
-                  | ERTS_PSFLG_EXITING
-                  | ERTS_PSFLG_PENDING_EXIT))) {
+                  | ERTS_PSFLG_EXITING))) {
         e = a;
         n = a | ERTS_PSFLG_DIRTY_ACTIVE_SYS;
         a = erts_atomic32_cmpxchg_mb(&c_p->state, n, e);
@@ -2313,6 +2308,8 @@ erts_try_change_runq_proc(Process *p, ErtsRunQueue *rq)
 {
     erts_aint_t old_rqint, new_rqint;
 
+    ASSERT(rq);
+
     new_rqint = (erts_aint_t) rq;
     old_rqint = (erts_aint_t) erts_atomic_read_nob(&p->run_queue);
     while (1) {
@@ -2577,6 +2574,8 @@ ERTS_TIME2REDS_IMPL__(ErtsMonotonicTime start, ErtsMonotonicTime end)
 }
 #endif
 
+Process *erts_try_lock_sig_free_proc(Eterm pid,
+                                     ErtsProcLocks locks);
 
 Process *erts_pid2proc_not_running(Process *,
 				   ErtsProcLocks,
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 05e7bcdea2..00659f9f49 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2003-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -35,6 +35,7 @@
 #include "erl_map.h"
 #define ERTS_WANT_EXTERNAL_TAGS
 #include "external.h"
+#include "erl_proc_sig_queue.h"
 
 #define PTR_FMT "%bpX"
 #define ETERM_FMT "%beX"
@@ -96,17 +97,35 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg)
     dump_binaries(to, to_arg, all_binaries);
 }
 
+static void
+monitor_size(ErtsMonitor *mon, void *vsize)
+{
+    *((Uint *) vsize) += erts_monitor_size(mon);
+}
+
+static void
+link_size(ErtsMonitor *lnk, void *vsize)
+{
+    *((Uint *) vsize) += erts_link_size(lnk);
+}
+
 Uint erts_process_memory(Process *p, int incl_msg_inq) {
-  ErtsMessage *mp;
   Uint size = 0;
   struct saved_calls *scb;
   size += sizeof(Process);
 
-  if (incl_msg_inq)
-      ERTS_MSGQ_MV_INQ2PRIVQ(p);
+  if (incl_msg_inq) {
+      erts_proc_lock(p, ERTS_PROC_LOCK_MSGQ);
+      erts_proc_sig_fetch(p);
+      erts_proc_unlock(p, ERTS_PROC_LOCK_MSGQ);
+  }
 
-  erts_doforall_links(ERTS_P_LINKS(p), &erts_one_link_size, &size);
-  erts_doforall_monitors(ERTS_P_MONITORS(p), &erts_one_mon_size, &size);
+  erts_link_tree_foreach(ERTS_P_LINKS(p),
+                         link_size, (void *) &size);
+  erts_monitor_tree_foreach(ERTS_P_MONITORS(p),
+                            monitor_size, (void *) &size);
+  erts_monitor_list_foreach(ERTS_P_LT_MONITORS(p),
+                            monitor_size, (void *) &size);
   size += (p->heap_sz + p->mbuf_sz) * sizeof(Eterm);
   if (p->abandoned_heap)
       size += (p->hend - p->heap) * sizeof(Eterm);
@@ -114,11 +133,16 @@ Uint erts_process_memory(Process *p, int incl_msg_inq) {
     size += (p->old_hend - p->old_heap) * sizeof(Eterm);
 
 
-  size += p->msg.len * sizeof(ErtsMessage);
+  size += p->sig_qs.len * sizeof(ErtsMessage);
 
-  for (mp = p->msg.first; mp; mp = mp->next)
-    if (mp->data.attached)
-      size += erts_msg_attached_data_size(mp)*sizeof(Eterm);
+  ERTS_FOREACH_SIG_PRIVQS(
+      p, mp,
+      {
+          if (ERTS_SIG_IS_NON_MSG((ErtsSignal *) mp))
+              size += erts_proc_sig_signal_size((ErtsSignal *) mp);
+          else if (mp->data.attached)
+              size += erts_msg_attached_data_size(mp) * sizeof(Eterm);
+      });
 
   if (p->arg_reg != p->def_arg_reg) {
     size += p->arity * sizeof(p->arg_reg[0]);
@@ -137,31 +161,48 @@ Uint erts_process_memory(Process *p, int incl_msg_inq) {
   return size;
 }
 
+static ERTS_INLINE void
+dump_msg(fmtfn_t to, void *to_arg, ErtsMessage *mp)
+{
+    if (ERTS_SIG_IS_MSG((ErtsSignal *) mp)) {
+        Eterm mesg = ERL_MESSAGE_TERM(mp);
+        if (is_value(mesg))
+            dump_element(to, to_arg, mesg);
+        else
+            dump_dist_ext(to, to_arg, mp->data.dist_ext);
+        mesg = ERL_MESSAGE_TOKEN(mp);
+        erts_print(to, to_arg, ":");
+        dump_element(to, to_arg, mesg);
+        erts_print(to, to_arg, "\n");
+    }
+}
+
+static ERTS_INLINE void
+heap_dump_msg(fmtfn_t to, void *to_arg, ErtsMessage *mp)
+{
+    if (ERTS_SIG_IS_MSG((ErtsSignal *) mp)) {
+        Eterm mesg = ERL_MESSAGE_TERM(mp);
+        if (is_value(mesg))
+            heap_dump(to, to_arg, mesg);
+        mesg = ERL_MESSAGE_TOKEN(mp);
+        heap_dump(to, to_arg, mesg);
+    }
+}
+
 static void
 dump_process_info(fmtfn_t to, void *to_arg, Process *p)
 {
     Eterm* sp;
-    ErtsMessage* mp;
     int yreg = -1;
 
     if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE)
         return;
 
-    ERTS_MSGQ_MV_INQ2PRIVQ(p);
+    erts_proc_sig_fetch(p);
 
-    if (p->msg.first) {
+    if (p->sig_qs.first || p->sig_qs.cont) {
 	erts_print(to, to_arg, "=proc_messages:%T\n", p->common.id);
-	for (mp = p->msg.first; mp != NULL; mp = mp->next) {
-	    Eterm mesg = ERL_MESSAGE_TERM(mp);
-	    if (is_value(mesg))
-		dump_element(to, to_arg, mesg);
-	    else
-		dump_dist_ext(to, to_arg, mp->data.dist_ext);
-	    mesg = ERL_MESSAGE_TOKEN(mp);
-	    erts_print(to, to_arg, ":");
-	    dump_element(to, to_arg, mesg);
-	    erts_print(to, to_arg, "\n");
-	}
+        ERTS_FOREACH_SIG_PRIVQS(p, mp, dump_msg(to, to_arg, mp));
     }
 
     if (p->dictionary) {
@@ -183,13 +224,10 @@ dump_process_info(fmtfn_t to, void *to_arg, Process *p)
             heap_dump(to, to_arg, term);
         }
     }
-    for (mp = p->msg.first; mp != NULL; mp = mp->next) {
-        Eterm mesg = ERL_MESSAGE_TERM(mp);
-        if (is_value(mesg))
-            heap_dump(to, to_arg, mesg);
-        mesg = ERL_MESSAGE_TOKEN(mp);
-        heap_dump(to, to_arg, mesg);
-    }
+
+    if (p->sig_qs.first || p->sig_qs.cont)
+        ERTS_FOREACH_SIG_PRIVQS(p, mp, heap_dump_msg(to, to_arg, mp));
+
     if (p->dictionary) {
         erts_deep_dictionary_dump(to, to_arg, p->dictionary, heap_dump);
     }
@@ -997,8 +1035,8 @@ erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg)
                 erts_print(to, to_arg, "FREE"); break;
             case ERTS_PSFLG_EXITING:
                 erts_print(to, to_arg, "EXITING"); break;
-            case ERTS_PSFLG_PENDING_EXIT:
-                erts_print(to, to_arg, "PENDING_EXIT"); break;
+            case ERTS_PSFLG_UNUSED:
+                erts_print(to, to_arg, "UNUSED"); break;
             case ERTS_PSFLG_ACTIVE:
                 erts_print(to, to_arg, "ACTIVE"); break;
             case ERTS_PSFLG_IN_RUNQ:
@@ -1009,8 +1047,10 @@ erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg)
                 erts_print(to, to_arg, "SUSPENDED"); break;
             case ERTS_PSFLG_GC:
                 erts_print(to, to_arg, "GC"); break;
-            case ERTS_PSFLG_TRAP_EXIT:
-                erts_print(to, to_arg, "TRAP_EXIT"); break;
+            case ERTS_PSFLG_SYS_TASKS:
+                erts_print(to, to_arg, "SYS_TASKS"); break;
+            case ERTS_PSFLG_SIG_IN_Q:
+                erts_print(to, to_arg, "SIG_IN_Q"); break;
             case ERTS_PSFLG_ACTIVE_SYS:
                 erts_print(to, to_arg, "ACTIVE_SYS"); break;
             case ERTS_PSFLG_RUNNING_SYS:
@@ -1021,6 +1061,8 @@ erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg)
                 erts_print(to, to_arg, "DELAYED_SYS"); break;
             case ERTS_PSFLG_OFF_HEAP_MSGQ:
                 erts_print(to, to_arg, "OFF_HEAP_MSGQ"); break;
+            case ERTS_PSFLG_SIG_Q:
+                erts_print(to, to_arg, "SIG_Q"); break;
             case ERTS_PSFLG_DIRTY_CPU_PROC:
                 erts_print(to, to_arg, "DIRTY_CPU_PROC"); break;
             case ERTS_PSFLG_DIRTY_IO_PROC:
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index 431867f27e..44c7892040 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2007-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -101,7 +101,6 @@ static void cleanup_tse(void);
 #ifdef ERTS_ENABLE_LOCK_CHECK
 static struct {
     Sint16 proc_lock_main;
-    Sint16 proc_lock_link;
     Sint16 proc_lock_msgq;
     Sint16 proc_lock_btm;
     Sint16 proc_lock_status;
@@ -140,7 +139,6 @@ erts_init_proc_lock(int cpus)
 #endif
 #ifdef ERTS_ENABLE_LOCK_CHECK
     lc_id.proc_lock_main	= erts_lc_get_lock_order_id("proc_main");
-    lc_id.proc_lock_link	= erts_lc_get_lock_order_id("proc_link");
     lc_id.proc_lock_msgq	= erts_lc_get_lock_order_id("proc_msgq");
     lc_id.proc_lock_btm		= erts_lc_get_lock_order_id("proc_btm");
     lc_id.proc_lock_status	= erts_lc_get_lock_order_id("proc_status");
@@ -1054,12 +1052,6 @@ erts_proc_lock_init(Process *p)
     ethr_mutex_lock(&p->lock.main.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.main.lc);
-#endif
-    erts_mtx_init(&p->lock.link, "proc_link", p->common.id,
-        ERTS_LOCK_FLAGS_CATEGORY_PROCESS);
-    ethr_mutex_lock(&p->lock.link.mtx);
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_trylock(1, &p->lock.link.lc);
 #endif
     erts_mtx_init(&p->lock.msgq, "proc_msgq", p->common.id,
         ERTS_LOCK_FLAGS_CATEGORY_PROCESS);
@@ -1102,7 +1094,6 @@ erts_proc_lock_fin(Process *p)
 {
 #if ERTS_PROC_LOCK_RAW_MUTEX_IMPL
     erts_mtx_destroy(&p->lock.main);
-    erts_mtx_destroy(&p->lock.link);
     erts_mtx_destroy(&p->lock.msgq);
     erts_mtx_destroy(&p->lock.btm);
     erts_mtx_destroy(&p->lock.status);
@@ -1144,8 +1135,6 @@ void erts_lcnt_enable_proc_lock_count(Process *proc, int enable) {
 
         erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN,
             "proc_main", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK);
-        erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK,
-            "proc_link", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK);
         erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ,
             "proc_msgq", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK);
         erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM,
@@ -1200,10 +1189,6 @@ erts_proc_lc_lock(Process *p, ErtsProcLocks locks, char *file, unsigned int line
 	lck.id = lc_id.proc_lock_main;
 	erts_lc_lock_x(&lck,file,line);
     }
-    if (locks & ERTS_PROC_LOCK_LINK) {
-	lck.id = lc_id.proc_lock_link;
-	erts_lc_lock_x(&lck,file,line);
-    }
     if (locks & ERTS_PROC_LOCK_MSGQ) {
 	lck.id = lc_id.proc_lock_msgq;
 	erts_lc_lock_x(&lck,file,line);
@@ -1233,10 +1218,6 @@ erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked,
 	lck.id = lc_id.proc_lock_main;
 	erts_lc_trylock_x(locked, &lck, file, line);
     }
-    if (locks & ERTS_PROC_LOCK_LINK) {
-	lck.id = lc_id.proc_lock_link;
-	erts_lc_trylock_x(locked, &lck, file, line);
-    }
     if (locks & ERTS_PROC_LOCK_MSGQ) {
 	lck.id = lc_id.proc_lock_msgq;
 	erts_lc_trylock_x(locked, &lck, file, line);
@@ -1277,10 +1258,6 @@ erts_proc_lc_unlock(Process *p, ErtsProcLocks locks)
 	lck.id = lc_id.proc_lock_msgq;
 	erts_lc_unlock(&lck);
     }
-    if (locks & ERTS_PROC_LOCK_LINK) {
-	lck.id = lc_id.proc_lock_link;
-	erts_lc_unlock(&lck);
-    }
     if (locks & ERTS_PROC_LOCK_MAIN) {
 	lck.id = lc_id.proc_lock_main;
 	erts_lc_unlock(&lck);
@@ -1312,10 +1289,6 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks)
 	lck.id = lc_id.proc_lock_msgq;
 	erts_lc_might_unlock(&lck);
     }
-    if (locks & ERTS_PROC_LOCK_LINK) {
-	lck.id = lc_id.proc_lock_link;
-	erts_lc_might_unlock(&lck);
-    }
     if (locks & ERTS_PROC_LOCK_MAIN) {
 	lck.id = lc_id.proc_lock_main;
 	erts_lc_might_unlock(&lck);
@@ -1323,8 +1296,6 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks)
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
     if (locks & ERTS_PROC_LOCK_MAIN)
 	erts_lc_might_unlock(&p->lock.main.lc);
-    if (locks & ERTS_PROC_LOCK_LINK)
-	erts_lc_might_unlock(&p->lock.link.lc);
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	erts_lc_might_unlock(&p->lock.msgq.lc);
     if (locks & ERTS_PROC_LOCK_BTM)
@@ -1348,10 +1319,6 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file,
 	lck.id = lc_id.proc_lock_main;
 	erts_lc_require_lock(&lck, file, line);
     }
-    if (locks & ERTS_PROC_LOCK_LINK) {
-	lck.id = lc_id.proc_lock_link;
-	erts_lc_require_lock(&lck, file, line);
-    }
     if (locks & ERTS_PROC_LOCK_MSGQ) {
 	lck.id = lc_id.proc_lock_msgq;
 	erts_lc_require_lock(&lck, file, line);
@@ -1371,8 +1338,6 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file,
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
     if (locks & ERTS_PROC_LOCK_MAIN)
 	erts_lc_require_lock(&p->lock.main.lc, file, line);
-    if (locks & ERTS_PROC_LOCK_LINK)
-	erts_lc_require_lock(&p->lock.link.lc, file, line);
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	erts_lc_require_lock(&p->lock.msgq.lc, file, line);
     if (locks & ERTS_PROC_LOCK_BTM)
@@ -1407,10 +1372,6 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks)
 	lck.id = lc_id.proc_lock_msgq;
 	erts_lc_unrequire_lock(&lck);
     }
-    if (locks & ERTS_PROC_LOCK_LINK) {
-	lck.id = lc_id.proc_lock_link;
-	erts_lc_unrequire_lock(&lck);
-    }
     if (locks & ERTS_PROC_LOCK_MAIN) {
 	lck.id = lc_id.proc_lock_main;
 	erts_lc_unrequire_lock(&lck);
@@ -1418,8 +1379,6 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks)
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
     if (locks & ERTS_PROC_LOCK_MAIN)
 	erts_lc_unrequire_lock(&p->lock.main.lc);
-    if (locks & ERTS_PROC_LOCK_LINK)
-	erts_lc_unrequire_lock(&p->lock.link.lc);
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	erts_lc_unrequire_lock(&p->lock.msgq.lc);
     if (locks & ERTS_PROC_LOCK_BTM)
@@ -1443,8 +1402,6 @@ erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks)
 
 	if (locks & ERTS_PROC_LOCK_MAIN)
 	    lck.id = lc_id.proc_lock_main;
-	else if (locks & ERTS_PROC_LOCK_LINK)
-	    lck.id = lc_id.proc_lock_link;
 	else if (locks & ERTS_PROC_LOCK_MSGQ)
 	    lck.id = lc_id.proc_lock_msgq;
 	else if (locks & ERTS_PROC_LOCK_BTM)
@@ -1487,10 +1444,6 @@ void erts_proc_lc_chk_only_proc(Process *p, ErtsProcLocks locks)
 	have_locks[have_locks_len].id = lc_id.proc_lock_main;
 	have_locks[have_locks_len++].extra = p->common.id;
     }
-    if (locks & ERTS_PROC_LOCK_LINK) {
-	have_locks[have_locks_len].id = lc_id.proc_lock_link;
-	have_locks[have_locks_len++].extra = p->common.id;
-    }
     if (locks & ERTS_PROC_LOCK_MSGQ) {
 	have_locks[have_locks_len].id = lc_id.proc_lock_msgq;
 	have_locks[have_locks_len++].extra = p->common.id;
@@ -1511,8 +1464,6 @@ void erts_proc_lc_chk_only_proc(Process *p, ErtsProcLocks locks)
     erts_lc_lock_t have_locks[6];
     if (locks & ERTS_PROC_LOCK_MAIN)
 	have_locks[have_locks_len++] = p->lock.main.lc;
-    if (locks & ERTS_PROC_LOCK_LINK)
-	have_locks[have_locks_len++] = p->lock.link.lc;
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	have_locks[have_locks_len++] = p->lock.msgq.lc;
     if (locks & ERTS_PROC_LOCK_BTM)
@@ -1540,10 +1491,6 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks)
 	have_locks[have_locks_len].id = lc_id.proc_lock_main;
 	have_locks[have_locks_len++].extra = p->common.id;
     }
-    if (locks & ERTS_PROC_LOCK_LINK) {
-	have_locks[have_locks_len].id = lc_id.proc_lock_link;
-	have_locks[have_locks_len++].extra = p->common.id;
-    }
     if (locks & ERTS_PROC_LOCK_MSGQ) {
 	have_locks[have_locks_len].id = lc_id.proc_lock_msgq;
 	have_locks[have_locks_len++].extra = p->common.id;
@@ -1564,8 +1511,6 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks)
     erts_lc_lock_t have_locks[6];
     if (locks & ERTS_PROC_LOCK_MAIN)
 	have_locks[have_locks_len++] = p->lock.main.lc;
-    if (locks & ERTS_PROC_LOCK_LINK)
-	have_locks[have_locks_len++] = p->lock.link.lc;
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	have_locks[have_locks_len++] = p->lock.msgq.lc;
     if (locks & ERTS_PROC_LOCK_BTM)
@@ -1603,14 +1548,6 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
 	have_not_locks[have_not_locks_len].id = lc_id.proc_lock_main;
 	have_not_locks[have_not_locks_len++].extra = p->common.id;
     }
-    if (locks & ERTS_PROC_LOCK_LINK) {
-	have_locks[have_locks_len].id = lc_id.proc_lock_link;
-	have_locks[have_locks_len++].extra = p->common.id;
-    }
-    else {
-	have_not_locks[have_not_locks_len].id = lc_id.proc_lock_link;
-	have_not_locks[have_not_locks_len++].extra = p->common.id;
-    }
     if (locks & ERTS_PROC_LOCK_MSGQ) {
 	have_locks[have_locks_len].id = lc_id.proc_lock_msgq;
 	have_locks[have_locks_len++].extra = p->common.id;
@@ -1651,10 +1588,6 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
 	have_locks[have_locks_len++] = p->lock.main.lc;
     else
 	have_not_locks[have_not_locks_len++] = p->lock.main.lc;
-    if (locks & ERTS_PROC_LOCK_LINK)
-	have_locks[have_locks_len++] = p->lock.link.lc;
-    else
-	have_not_locks[have_not_locks_len++] = p->lock.link.lc;
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	have_locks[have_locks_len++] = p->lock.msgq.lc;
     else
@@ -1680,13 +1613,10 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
 ErtsProcLocks
 erts_proc_lc_my_proc_locks(Process *p)
 {
-    int resv[6];
+    int resv[5];
     ErtsProcLocks res = 0;
 #if ERTS_PROC_LOCK_OWN_IMPL
-    erts_lc_lock_t locks[6] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main,
-						 p->common.id,
-						 ERTS_LOCK_TYPE_PROCLOCK),
-			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_link,
+    erts_lc_lock_t locks[5] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main,
 						 p->common.id,
 						 ERTS_LOCK_TYPE_PROCLOCK),
 			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_msgq,
@@ -1702,26 +1632,23 @@ erts_proc_lc_my_proc_locks(Process *p)
 						 p->common.id,
 						 ERTS_LOCK_TYPE_PROCLOCK)};
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
-    erts_lc_lock_t locks[6] = {p->lock.main.lc,
-			       p->lock.link.lc,
+    erts_lc_lock_t locks[5] = {p->lock.main.lc,
 			       p->lock.msgq.lc,
 			       p->lock.btm.lc,
 			       p->lock.status.lc,
 			       p->lock.trace.lc};
 #endif
 
-    erts_lc_have_locks(resv, locks, 6);
+    erts_lc_have_locks(resv, locks, 5);
     if (resv[0])
 	res |= ERTS_PROC_LOCK_MAIN;
     if (resv[1])
-	res |= ERTS_PROC_LOCK_LINK;
-    if (resv[2])
 	res |= ERTS_PROC_LOCK_MSGQ;
-    if (resv[3])
+    if (resv[2])
 	res |= ERTS_PROC_LOCK_BTM;
-    if (resv[4])
+    if (resv[3])
 	res |= ERTS_PROC_LOCK_STATUS;
-    if (resv[5])
+    if (resv[4])
 	res |= ERTS_PROC_LOCK_TRACE;
 
     return res;
@@ -1730,15 +1657,14 @@ erts_proc_lc_my_proc_locks(Process *p)
 void
 erts_proc_lc_chk_no_proc_locks(char *file, int line)
 {
-    int resv[6];
-    int ids[6] = {lc_id.proc_lock_main,
-		  lc_id.proc_lock_link,
+    int resv[5];
+    int ids[5] = {lc_id.proc_lock_main,
 		  lc_id.proc_lock_msgq,
 		  lc_id.proc_lock_btm,
 		  lc_id.proc_lock_status,
 		  lc_id.proc_lock_trace};
-    erts_lc_have_lock_ids(resv, ids, 6);
-    if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3] || resv[4] || resv[5])) {
+    erts_lc_have_lock_ids(resv, ids, 5);
+    if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3] || resv[4])) {
 	erts_lc_fail("%s:%d: Thread has process locks locked when expected "
 		     "not to have any process locks locked",
 		     file, line);
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 9d5691d3c4..43f396c547 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2007-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -66,7 +66,7 @@
 
 #endif
 
-#define ERTS_PROC_LOCK_MAX_BIT 5
+#define ERTS_PROC_LOCK_MAX_BIT 4
 
 typedef erts_aint32_t ErtsProcLocks;
 
@@ -82,19 +82,17 @@ typedef struct erts_proc_lock_t_ {
     /* Each erts_mtx_t has its own lock counter ^ */
 
     #define ERTS_LCNT_PROCLOCK_IDX_MAIN 0
-    #define ERTS_LCNT_PROCLOCK_IDX_LINK 1
-    #define ERTS_LCNT_PROCLOCK_IDX_MSGQ 2
-    #define ERTS_LCNT_PROCLOCK_IDX_BTM 3
-    #define ERTS_LCNT_PROCLOCK_IDX_STATUS 4
-    #define ERTS_LCNT_PROCLOCK_IDX_TRACE 5
+    #define ERTS_LCNT_PROCLOCK_IDX_MSGQ 1
+    #define ERTS_LCNT_PROCLOCK_IDX_BTM 2
+    #define ERTS_LCNT_PROCLOCK_IDX_STATUS 3
+    #define ERTS_LCNT_PROCLOCK_IDX_TRACE 4
 
-    #define ERTS_LCNT_PROCLOCK_COUNT 6
+    #define ERTS_LCNT_PROCLOCK_COUNT 5
 
     erts_lcnt_ref_t lcnt_carrier;
 #endif
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
     erts_mtx_t main;
-    erts_mtx_t link;
     erts_mtx_t msgq;
     erts_mtx_t btm;
     erts_mtx_t status;
@@ -117,28 +115,19 @@ typedef struct erts_proc_lock_t_ {
  */
 #define ERTS_PROC_LOCK_MAIN		(((ErtsProcLocks) 1) << 0)
 
-/*
- * Link lock:
- *   Protects the following fields in the process structure:
- *   * nlinks
- *   * monitors
- *   * suspend_monitors
- */
-#define ERTS_PROC_LOCK_LINK		(((ErtsProcLocks) 1) << 1)
-
 /*
  * Message queue lock:
  *   Protects the following fields in the process structure:
  *   * msg_inq
  */
-#define ERTS_PROC_LOCK_MSGQ		(((ErtsProcLocks) 1) << 2)
+#define ERTS_PROC_LOCK_MSGQ		(((ErtsProcLocks) 1) << 1)
 
 /*
  * Bif timer lock:
  *   Protects the following fields in the process structure:
  *   * bif_timers
  */
-#define ERTS_PROC_LOCK_BTM		(((ErtsProcLocks) 1) << 3)
+#define ERTS_PROC_LOCK_BTM		(((ErtsProcLocks) 1) << 2)
 
 /*
  * Status lock:
@@ -148,7 +137,7 @@ typedef struct erts_proc_lock_t_ {
  *   * sys_tasks
  *   * ...
  */
-#define ERTS_PROC_LOCK_STATUS		(((ErtsProcLocks) 1) << 4)
+#define ERTS_PROC_LOCK_STATUS		(((ErtsProcLocks) 1) << 3)
 
 /*
  * Trace message lock:
@@ -277,9 +266,6 @@ void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) {
         if (locks & ERTS_PROC_LOCK_MAIN) {
             erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN);
         }
-        if (locks & ERTS_PROC_LOCK_LINK) {
-            erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK);
-        }
         if (locks & ERTS_PROC_LOCK_MSGQ) {
             erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ);
         }
@@ -307,9 +293,6 @@ void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks,
         if (locks & ERTS_PROC_LOCK_MAIN) {
             erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN, file, line);
         }
-        if (locks & ERTS_PROC_LOCK_LINK) {
-            erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK, file, line);
-        }
         if (locks & ERTS_PROC_LOCK_MSGQ) {
             erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ, file, line);
         }
@@ -336,9 +319,6 @@ void erts_lcnt_proc_lock_unacquire(erts_proc_lock_t *lock, ErtsProcLocks locks)
         if (locks & ERTS_PROC_LOCK_MAIN) {
             erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN);
         }
-        if (locks & ERTS_PROC_LOCK_LINK) {
-            erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK);
-        }
         if (locks & ERTS_PROC_LOCK_MSGQ) {
             erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ);
         }
@@ -365,9 +345,6 @@ void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks) {
         if (locks & ERTS_PROC_LOCK_MAIN) {
             erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN);
         }
-        if (locks & ERTS_PROC_LOCK_LINK) {
-            erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK);
-        }
         if (locks & ERTS_PROC_LOCK_MSGQ) {
             erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ);
         }
@@ -394,9 +371,6 @@ void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res
         if (locks & ERTS_PROC_LOCK_MAIN) {
             erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN, res);
         }
-        if (locks & ERTS_PROC_LOCK_LINK) {
-            erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK, res);
-        }
         if (locks & ERTS_PROC_LOCK_MSGQ) {
             erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ, res);
         }
@@ -640,9 +614,6 @@ erts_proc_raw_trylock__(Process *p, ErtsProcLocks locks)
     if (locks & ERTS_PROC_LOCK_MAIN)
 	if (erts_mtx_trylock(&p->lock.main) == EBUSY)
 	    goto busy_main;
-    if (locks & ERTS_PROC_LOCK_LINK)
-	if (erts_mtx_trylock(&p->lock.link) == EBUSY)
-	    goto busy_link;
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	if (erts_mtx_trylock(&p->lock.msgq) == EBUSY)
 	    goto busy_msgq;
@@ -668,9 +639,6 @@ busy_btm:
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	erts_mtx_unlock(&p->lock.msgq);
 busy_msgq:
-    if (locks & ERTS_PROC_LOCK_LINK)
-	erts_mtx_unlock(&p->lock.link);
-busy_link:
     if (locks & ERTS_PROC_LOCK_MAIN)
 	erts_mtx_unlock(&p->lock.main);
 busy_main:
@@ -741,8 +709,6 @@ erts_proc_lock__(Process *p,
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
     if (locks & ERTS_PROC_LOCK_MAIN)
 	erts_mtx_lock(&p->lock.main);
-    if (locks & ERTS_PROC_LOCK_LINK)
-	erts_mtx_lock(&p->lock.link);
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	erts_mtx_lock(&p->lock.msgq);
     if (locks & ERTS_PROC_LOCK_BTM)
@@ -844,8 +810,6 @@ erts_proc_unlock__(Process *p,
 	erts_mtx_unlock(&p->lock.btm);
     if (locks & ERTS_PROC_LOCK_MSGQ)
 	erts_mtx_unlock(&p->lock.msgq);
-    if (locks & ERTS_PROC_LOCK_LINK)
-	erts_mtx_unlock(&p->lock.link);
     if (locks & ERTS_PROC_LOCK_MAIN)
 	erts_mtx_unlock(&p->lock.main);
 #endif
diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h
index 4858cc8ab8..94f0247492 100644
--- a/erts/emulator/beam/erl_ptab.h
+++ b/erts/emulator/beam/erl_ptab.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -35,7 +35,7 @@
 #include "erl_thr_progress.h"
 #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
 #include "erl_alloc.h"
-#include "erl_monitors.h"
+#include "erl_monitor_link.h"
 
 #define ERTS_TRACER(P)          ((P)->common.tracer)
 #define ERTS_TRACER_MODULE(T) 	(CAR(list_val(T)))
@@ -44,6 +44,7 @@
 
 #define ERTS_P_LINKS(P)		((P)->common.u.alive.links)
 #define ERTS_P_MONITORS(P)	((P)->common.u.alive.monitors)
+#define ERTS_P_LT_MONITORS(P)	((P)->common.u.alive.lt_monitors)
 
 #define IS_TRACED(p) \
     (ERTS_TRACER(p) != NIL)
@@ -68,6 +69,7 @@ typedef struct {
 	    struct reg_proc *reg;
 	    ErtsLink *links;
 	    ErtsMonitor *monitors;
+            ErtsMonitor *lt_monitors;
 	} alive;
 
 	/* --- While being released --- */
diff --git a/erts/emulator/beam/erl_rbtree.h b/erts/emulator/beam/erl_rbtree.h
index e59d6900b0..e50abf5cec 100644
--- a/erts/emulator/beam/erl_rbtree.h
+++ b/erts/emulator/beam/erl_rbtree.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2015-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2015-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -50,8 +50,14 @@
  * - ERTS_RBT_GET_LEFT(T) - Get left child node.
  * - ERTS_RBT_SET_LEFT(T, L) - Set left child node.
  * - ERTS_RBT_GET_KEY(T) - Get key of node.
- * - ERTS_RBT_IS_LT(KX, KY) - Is key KX less than key KY?
- * - ERTS_RBT_IS_EQ(KX, KY) - Is key KX equal to key KY?
+ * Either:
+ *   - ERTS_RBT_CMP_KEYS(KX, KY) - Compare keys...
+ * or:
+ *   - ERTS_RBT_IS_LT(KX, KY) - Is key KX less than key KY?
+ *   - ERTS_RBT_IS_EQ(KX, KY) - Is key KX equal to key KY?
+ *
+ * If ERTS_RBT_CMP_KEYS is defined ERTS_RBT_IS_LT and
+ * ERTS_RBT_IS_EQ will be redefined using ERTS_RBT_CMP_KEYS
  *
  * Optional defines:
  *
@@ -337,6 +343,15 @@
  *         Should only be used for debuging.
  */
 
+#ifdef ERTS_RBT_CMP_KEYS
+
+#  undef ERTS_RBT_IS_LT
+#  define ERTS_RBT_IS_LT(KX, KY) (ERTS_RBT_CMP_KEYS((KX), (KY)) < 0)
+
+#  undef ERTS_RBT_IS_EQ
+#  define ERTS_RBT_IS_EQ(KX, KY) (ERTS_RBT_CMP_KEYS((KX), (KY)) == 0)
+
+#endif
 
 /*
  * Check that we have all mandatory defines
@@ -396,6 +411,16 @@
 #  error Missing definition of ERTS_RBT_IS_EQ
 #endif
 
+#undef ERTS_RBT_IS_GT__
+#ifdef ERTS_RBT_CMP_KEYS
+#  define ERTS_RBT_IS_GT__(KX, KY) \
+    (ERTS_RBT_CMP_KEYS((KX), (KY)) > 0)
+#else
+#  define ERTS_RBT_IS_GT__(KX, KY) \
+    (!ERTS_RBT_IS_LT((KX), (KY)) && !ERTS_RBT_IS_EQ((KX), (KY)))
+
+#endif
+
 #if defined(ERTS_RBT_HARD_DEBUG) || defined(DEBUG)
 #  ifndef ERTS_RBT_DEBUG
 #    define ERTS_RBT_DEBUG 1
@@ -1007,19 +1032,30 @@ ERTS_RBT_FUNC__(insert_aux__)(ERTS_RBT_T **root, ERTS_RBT_T *n, int lookup)
 	ERTS_RBT_T *p, *x = *root;
 
 	while (1) {
-	    ERTS_RBT_KEY_T kx;
+	    ERTS_RBT_KEY_T kx = ERTS_RBT_GET_KEY(x);
 	    ERTS_RBT_T *c;
+            int kres;
+#ifdef ERTS_RBT_CMP_KEYS
+            int kcmp = ERTS_RBT_CMP_KEYS(kn, kx);
+            kres = kcmp == 0;
+#else
+            kres = ERTS_RBT_IS_EQ(kn, kx);
+#endif
 
-	    kx = ERTS_RBT_GET_KEY(x);
-
-	    if (lookup && ERTS_RBT_IS_EQ(kn, kx)) {
+	    if (lookup && kres) {
 
 		ERTS_RBT_HDBG_CHECK_TREE__(*root, NULL);
 
 		return x;
 	    }
 
-	    if (ERTS_RBT_IS_LT(kn, kx)) {
+#ifdef ERTS_RBT_CMP_KEYS
+            kres = kcmp < 0;
+#else
+            kres = ERTS_RBT_IS_LT(kn, kx);
+#endif
+
+	    if (kres) {
 		c = ERTS_RBT_GET_LEFT(x);
 		if (!c) {
 		    ERTS_RBT_SET_PARENT(n, x);
@@ -1075,6 +1111,101 @@ ERTS_RBT_FUNC__(insert)(ERTS_RBT_T **root, ERTS_RBT_T *n)
 
 #endif /* ERTS_RBT_WANT_INSERT */
 
+#ifdef ERTS_RBT_WANT_LOOKUP_CREATE
+static ERTS_INLINE ERTS_RBT_T *
+ERTS_RBT_FUNC__(lookup_create)(ERTS_RBT_T **root,
+                               ERTS_RBT_KEY_T kn,
+                               ERTS_RBT_T *(*create)(ERTS_RBT_KEY_T, void *),
+                               void *arg,
+                               int *created)
+{
+    ERTS_RBT_T *n;
+    ERTS_RBT_HDBG_CHECK_TREE__(*root, NULL);
+
+    if (!*root) {
+        n = (*create)(kn, arg);
+        ERTS_RBT_INIT_EMPTY_TNODE(n);
+        ERTS_RBT_ASSERT(ERTS_RBT_IS_EQ(ERTS_RBT_GET_KEY(n), kn));
+	ERTS_RBT_SET_BLACK(n);
+	*root = n;
+        *created = !0;
+#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT
+	ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(NULL, n);
+#endif
+    }
+    else {
+	ERTS_RBT_T *p, *x = *root;
+
+	while (1) {
+	    ERTS_RBT_KEY_T kx = ERTS_RBT_GET_KEY(x);
+	    ERTS_RBT_T *c;
+            int kres;
+#ifdef ERTS_RBT_CMP_KEYS
+            int kcmp = ERTS_RBT_CMP_KEYS(kn, kx);
+            kres = kcmp == 0;
+#else
+            kres = ERTS_RBT_IS_EQ(kn, kx);
+#endif
+
+	    if (kres) {
+
+		ERTS_RBT_HDBG_CHECK_TREE__(*root, NULL);
+
+                *created = 0;
+		return x;
+	    }
+
+#ifdef ERTS_RBT_CMP_KEYS
+            kres = kcmp < 0;
+#else
+            kres = ERTS_RBT_IS_LT(kn, kx);
+#endif
+
+	    if (kres) {
+		c = ERTS_RBT_GET_LEFT(x);
+		if (!c) {
+                    n = (*create)(kn, arg);
+                    ERTS_RBT_INIT_EMPTY_TNODE(n);
+                    ERTS_RBT_ASSERT(ERTS_RBT_IS_EQ(ERTS_RBT_GET_KEY(n), kn));
+                    *created = !0;
+		    ERTS_RBT_SET_PARENT(n, x);
+		    ERTS_RBT_SET_LEFT(x, n);
+		    p = x;
+		    break;
+		}
+	    }
+	    else {
+		c = ERTS_RBT_GET_RIGHT(x);
+		if (!c) {
+                    n = (*create)(kn, arg);
+                    ERTS_RBT_INIT_EMPTY_TNODE(n);
+                    ERTS_RBT_ASSERT(ERTS_RBT_IS_EQ(ERTS_RBT_GET_KEY(n), kn));
+                    *created = !0;
+		    ERTS_RBT_SET_PARENT(n, x);
+		    ERTS_RBT_SET_RIGHT(x, n);
+		    p = x;
+		    break;
+		}
+	    }
+
+	    x = c;
+	}
+
+        ERTS_RBT_ASSERT(ERTS_RBT_IS_EQ(ERTS_RBT_GET_KEY(n), kn));
+	ERTS_RBT_ASSERT(p);
+
+	ERTS_RBT_SET_RED(n);
+	if (ERTS_RBT_IS_RED(p))
+	    ERTS_RBT_FUNC__(insert_fixup__)(root, n);
+    }
+
+    ERTS_RBT_HDBG_CHECK_TREE__(*root, n);
+
+    return n;
+}
+
+#endif /* ERTS_RBT_WANT_LOOKUP_CREATE */
+
 #ifdef ERTS_RBT_WANT_LOOKUP
 
 static ERTS_RBT_API_INLINE__ ERTS_RBT_T *
@@ -1088,11 +1219,24 @@ ERTS_RBT_FUNC__(lookup)(ERTS_RBT_T *root, ERTS_RBT_KEY_T key)
     while (1) {
 	ERTS_RBT_KEY_T kx = ERTS_RBT_GET_KEY(x);
 	ERTS_RBT_T *c;
+        int kres;
+#ifdef ERTS_RBT_CMP_KEYS
+        int kcmp = ERTS_RBT_CMP_KEYS(key, kx);
+        kres = kcmp == 0;
+#else
+        kres = ERTS_RBT_IS_EQ(key, kx);
+#endif
 
-	if (ERTS_RBT_IS_EQ(key, kx))
+	if (kres)
 	    return x;
 
-	if (ERTS_RBT_IS_LT(key, kx)) {
+#ifdef ERTS_RBT_CMP_KEYS
+        kres = kcmp < 0;
+#else
+        kres = ERTS_RBT_IS_LT(key, kx);
+#endif
+
+	if (kres) {
 	    c = ERTS_RBT_GET_LEFT(x);
 	    if (!c)
 		return NULL;
@@ -1426,14 +1570,14 @@ ERTS_RBT_FUNC__(foreach_large)(ERTS_RBT_T *root,
 
 #ifdef ERTS_RBT_WANT_FOREACH_YIELDING
 
-static ERTS_RBT_API_INLINE__ void
+static ERTS_RBT_API_INLINE__ int
 ERTS_RBT_FUNC__(foreach_yielding)(ERTS_RBT_T *root,
 				  void (*op)(ERTS_RBT_T *, void *),
 				  void *arg,
 				  ERTS_RBT_YIELD_STATE_T__ *ystate,
 				  Sint ylimit)
 {
-    (void) ERTS_RBT_FUNC__(foreach_unordered__)(*root, 0, op, arg,
+    return ERTS_RBT_FUNC__(foreach_unordered__)(&root, 0, op, arg,
 						1, ystate, ylimit);
 }
 
@@ -1630,8 +1774,7 @@ ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root, ERTS_RBT_T *n)
 		kx = ERTS_RBT_GET_KEY(x);
 		kc = ERTS_RBT_GET_KEY(c);
 
-		ERTS_RBT_ASSERT(ERTS_RBT_IS_LT(kc, kx)
-				|| ERTS_RBT_IS_EQ(kc, kx));
+                ERTS_RBT_ASSERT(!ERTS_RBT_IS_GT__(kc, kx));
 
 		x = c;
 	    }
@@ -1649,8 +1792,8 @@ ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root, ERTS_RBT_T *n)
 	    kx = ERTS_RBT_GET_KEY(x);
 	    kc = ERTS_RBT_GET_KEY(c);
 
-	    ERTS_RBT_ASSERT(ERTS_RBT_IS_LT(kx, kc)
-			    || ERTS_RBT_IS_EQ(kx, kc));
+            ERTS_RBT_ASSERT(!ERTS_RBT_IS_GT__(kx, kc));
+
 	    x = c;
 	}
 
@@ -1672,8 +1815,8 @@ ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root, ERTS_RBT_T *n)
 		    kx = ERTS_RBT_GET_KEY(x);
 		    kc = ERTS_RBT_GET_KEY(c);
 
-		    ERTS_RBT_ASSERT(ERTS_RBT_IS_LT(kx, kc)
-				    || ERTS_RBT_IS_EQ(kx, kc));
+                    ERTS_RBT_ASSERT(!ERTS_RBT_IS_GT__(kx, kc));
+
 		    /* Go down tree of x's sibling... */
 		    x = c;
 		    break;
@@ -1707,6 +1850,7 @@ ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root, ERTS_RBT_T *n)
 #undef ERTS_RBT_NEED_FOREACH_ORDERED__
 #undef ERTS_RBT_NEED_HDBG_CHECK_TREE__
 #undef ERTS_RBT_HDBG_CHECK_TREE__
+#undef ERTS_RBT_IS_GT__
 
 #ifdef ERTS_RBT_UNDEF
 #  undef ERTS_RBT_PREFIX
@@ -1727,6 +1871,7 @@ ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root, ERTS_RBT_T *n)
 #  undef ERTS_RBT_GET_LEFT
 #  undef ERTS_RBT_SET_LEFT
 #  undef ERTS_RBT_GET_KEY
+#  undef ERTS_RBT_CMP_KEYS
 #  undef ERTS_RBT_IS_LT
 #  undef ERTS_RBT_IS_EQ
 #  undef ERTS_RBT_UNDEF
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 3dd1c2555c..18483fca35 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -345,6 +345,9 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm)
  *
  * To help find code which makes unwarranted assumptions about zero,
  * we now use a non-zero bit-pattern in debug mode.
+ *
+ * In order to be able to differentiata against values, the non-value
+ * needs to be tagged as a header of some sort.
  */
 #if ET_DEBUG
 # ifdef HIPE
@@ -355,7 +358,7 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm)
 #  define THE_NON_VALUE	_make_header(0,_TAG_HEADER_FLOAT)
 # endif
 #else
-#define THE_NON_VALUE	(0)
+#define THE_NON_VALUE	(TAG_PRIMARY_HEADER)
 #endif
 #define is_non_value(x)	((x) == THE_NON_VALUE)
 #define is_value(x)	((x) != THE_NON_VALUE)
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 65211e4e6f..968f21fd51 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,6 +21,8 @@
 #ifndef ERL_TIME_H__
 #define ERL_TIME_H__
 
+#include "erl_monitor_link.h"
+
 #if 0
 #  define ERTS_TW_DEBUG
 #endif
@@ -79,8 +81,8 @@ typedef ErtsMonotonicTime * ErtsNextTimeoutRef;
 extern SysTimeval erts_first_emu_time;
 
 
-void erts_monitor_time_offset(Eterm id, Eterm ref);
-int erts_demonitor_time_offset(Eterm ref);
+void erts_monitor_time_offset(ErtsMonitor *mon);
+void erts_demonitor_time_offset(ErtsMonitor *mon);
 
 int erts_init_time_sup(int, ErtsTimeWarpMode);
 void erts_late_init_time_sup(void);
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 8cbdf9fa0f..e5bb3cc15f 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -35,6 +35,7 @@
 #include "erl_time.h"
 #include "erl_driver.h"
 #include "erl_nif.h"
+#include "erl_proc_sig_queue.h"
  
 static erts_mtx_t erts_get_time_mtx;
 
@@ -1870,36 +1871,33 @@ void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) {
 #include "big.h"
 
 void
-erts_monitor_time_offset(Eterm id, Eterm ref)
+erts_monitor_time_offset(ErtsMonitor *mon)
 {
     erts_mtx_lock(&erts_get_time_mtx);
-    erts_add_monitor(&time_offset_monitors, MON_TIME_OFFSET, ref, id, NIL);
+    erts_monitor_list_insert(&time_offset_monitors, mon);
     no_time_offset_monitors++;
     erts_mtx_unlock(&erts_get_time_mtx);
 }
 
-int
-erts_demonitor_time_offset(Eterm ref)
+void
+erts_demonitor_time_offset(ErtsMonitor *mon)
 {
-    int res;
-    ErtsMonitor *mon;
-    ASSERT(is_internal_ref(ref));
+    ErtsMonitorData *mdp = erts_monitor_to_data(mon);
+    ASSERT(erts_monitor_is_origin(mon));
+    ASSERT(mon->type == ERTS_MON_TYPE_TIME_OFFSET);
+
     erts_mtx_lock(&erts_get_time_mtx);
-    if (is_internal_ordinary_ref(ref))
-        mon = erts_remove_monitor(&time_offset_monitors, ref);
-    else
-        mon = NULL;
-    if (!mon)
-	res = 0;
-    else {
-	ASSERT(no_time_offset_monitors > 0);
-	no_time_offset_monitors--;
-	res = 1;
-    }
+
+    ASSERT(erts_monitor_is_in_table(&mdp->target));
+
+    erts_monitor_list_delete(&time_offset_monitors, &mdp->target);
+
+    ASSERT(no_time_offset_monitors > 0);
+    no_time_offset_monitors--;
+
     erts_mtx_unlock(&erts_get_time_mtx);
-    if (res)
-	erts_destroy_monitor(mon);
-    return res;
+
+    erts_monitor_release_both(mdp);
 }
 
 typedef struct {
@@ -1917,17 +1915,19 @@ static void
 save_time_offset_monitor(ErtsMonitor *mon, void *vcntxt)
 {
     ErtsTimeOffsetMonitorContext *cntxt;
+    ErtsMonitorData *mdp = erts_monitor_to_data(mon);
     Eterm *from_hp, *to_hp;
     Uint mix;
     int hix;
 
     cntxt = (ErtsTimeOffsetMonitorContext *) vcntxt;
     mix = (cntxt->ix)++;
-    cntxt->to_mon_info[mix].pid = mon->u.pid;
+    ASSERT(is_internal_pid(mon->other.item));
+    cntxt->to_mon_info[mix].pid = mon->other.item;
     to_hp = &cntxt->to_mon_info[mix].heap[0];
 
-    ASSERT(is_internal_ordinary_ref(mon->ref));
-    from_hp = internal_ref_val(mon->ref);
+    ASSERT(is_internal_ordinary_ref(mdp->ref));
+    from_hp = internal_ref_val(mdp->ref);
     ASSERT(thing_arityval(*from_hp) + 1 == ERTS_REF_THING_SIZE);
 
     for (hix = 0; hix < ERTS_REF_THING_SIZE; hix++)
@@ -1972,9 +1972,9 @@ send_time_offset_changed_notifications(void *new_offsetp)
 	cntxt.ix = 0;
 	cntxt.to_mon_info = to_mon_info;
 
-	erts_doforall_monitors(time_offset_monitors,
-			       save_time_offset_monitor,
-			       &cntxt);
+        erts_monitor_list_foreach(time_offset_monitors,
+                                  save_time_offset_monitor,
+                                  &cntxt);
 
 	ASSERT(cntxt.ix == no_monitors);
     }
@@ -2008,26 +2008,14 @@ send_time_offset_changed_notifications(void *new_offsetp)
 	ASSERT(*patch_refp == THE_NON_VALUE);
 
 	for (mix = 0; mix < no_monitors; mix++) {
-	    Process *rp = erts_proc_lookup(to_mon_info[mix].pid);
-	    if (rp) {
-		Eterm ref = to_mon_info[mix].ref;
-		ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK;
-		erts_proc_lock(rp, ERTS_PROC_LOCK_LINK);
-		if (erts_lookup_monitor(ERTS_P_MONITORS(rp), ref)) {
-		    ErtsMessage *mp;
-		    ErlOffHeap *ohp;
-		    Eterm message;
-
-		    mp = erts_alloc_message_heap(rp, &rp_locks,
-						 hsz, &hp, &ohp);
-		    *patch_refp = ref;
-		    ASSERT(hsz == size_object(message_template));
-		    message = copy_struct(message_template, hsz, &hp, ohp);
-		    erts_queue_message(rp, rp_locks, mp, message, am_clock_service);
-		}
-		erts_proc_unlock(rp, rp_locks);
-	    }
-	}
+            *patch_refp = to_mon_info[mix].ref;
+            erts_proc_sig_send_persistent_monitor_msg(ERTS_MON_TYPE_TIME_OFFSET,
+                                                      *patch_refp,
+                                                      am_clock_service,
+                                                      to_mon_info[mix].pid,
+                                                      message_template,
+                                                      hsz);
+        }
 
 	erts_free(ERTS_ALC_T_TMP, tmp);
     }
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index f9f8abcc27..bbd9b4bad2 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -83,7 +83,10 @@
 #ifndef ERL_EXTERNAL_H__
 #define ERL_EXTERNAL_H__
 
+#define ERL_NODE_TABLES_BASIC_ONLY
 #include "erl_node_tables.h"
+#undef ERL_NODE_TABLES_BASIC_ONLY
+#include "erl_alloc.h"
 
 #define ERTS_ATOM_CACHE_SIZE 2048
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 0f23027752..d853b2e352 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -87,9 +87,7 @@ typedef struct
 {
     erts_mtx_t lock;
     ErtsMonitor* root;
-    int pending_failed_fire;
-    int is_dying;
-
+    Uint refc;
     size_t user_data_sz;
 } ErtsResourceMonitors;
 
@@ -116,7 +114,8 @@ extern void erts_pre_nif(struct enif_environment_t*, Process*,
 			 struct erl_module_nif*, Process* tracee);
 extern void erts_post_nif(struct enif_environment_t* env);
 extern void erts_resource_stop(ErtsResource*, ErlNifEvent, int is_direct_call);
-void erts_fire_nif_monitor(ErtsResource*, Eterm pid, Eterm ref);
+void erts_fire_nif_monitor(ErtsMonitor *tmon);
+void erts_nif_demonitored(ErtsResource* resource);
 extern Eterm erts_nif_taints(Process* p);
 extern void erts_print_nif_taints(fmtfn_t to, void* to_arg);
 void erts_unload_nif(struct erl_module_nif* nif);
@@ -886,6 +885,7 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a,
 			   Eterm (*bif)(Process*, Eterm*, BeamInstr*));
 void erts_init_bif(void);
 Eterm erl_send(Process *p, Eterm to, Eterm msg);
+int erts_set_group_leader(Process *proc, Eterm new_gl);
 
 /* erl_bif_op.c */
 
@@ -908,7 +908,6 @@ extern erts_atomic_t erts_copy_literal_area__;
 #define ERTS_COPY_LITERAL_AREA()					\
     ((ErtsLiteralArea *) erts_atomic_read_nob(&erts_copy_literal_area__))
 extern Process *erts_literal_area_collector;
-extern Process *erts_dirty_process_code_checker;
 
 extern Process *erts_code_purger;
 
@@ -1072,7 +1071,7 @@ void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first,
 			   Eterm* refs, unsigned nrefs, int literals);
 
 /* Utilities */
-extern void erts_delete_nodes_monitors(Process *, ErtsProcLocks);
+void erts_monitor_nodes_delete(ErtsMonitor *);
 extern Eterm erts_monitor_nodes(Process *, Eterm, Eterm);
 extern Eterm erts_processes_monitoring_nodes(Process *);
 extern int erts_do_net_exits(DistEntry*, Eterm);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index e4a5f2b6b6..b2afdc6bf2 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -53,6 +53,7 @@
 #include "erl_hl_timer.h"
 #include "erl_time.h"
 #include "erl_io_queue.h"
+#include "erl_proc_sig_queue.h"
 
 extern ErlDrvEntry fd_driver_entry;
 extern ErlDrvEntry vanilla_driver_entry;
@@ -366,6 +367,7 @@ static Port *create_port(char *name,
     prt->drv_ptr = driver;
     ERTS_P_LINKS(prt) = NULL;
     ERTS_P_MONITORS(prt) = NULL;
+    ERTS_P_LT_MONITORS(prt) = NULL;
     prt->linebuf = NULL;
     prt->suspended = NULL;
     erts_init_port_data(prt);
@@ -748,8 +750,8 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
     Port *creator_port;
     Port* port;
     erts_driver_t *driver;
-    Process *rp;
     erts_mtx_t *driver_lock = NULL;
+    ErtsLinkData *ldp;
 
     ERTS_CHK_NO_PROC_LOCKS;
 
@@ -761,17 +763,13 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
     if (creator_port == ERTS_INVALID_ERL_DRV_PORT)
 	return ERTS_INVALID_ERL_DRV_PORT;
 
-    rp = erts_proc_lookup(pid);
-    if (!rp)
-	return ERTS_INVALID_ERL_DRV_PORT;
-
     ERTS_LC_ASSERT(erts_lc_is_port_locked(creator_port));
 
     driver = creator_port->drv_ptr;
     erts_rwmtx_rlock(&erts_driver_list_lock);
     if (!erts_ddll_driver_ok(driver->handle)) {
 	erts_rwmtx_runlock(&erts_driver_list_lock);
-	return ERTS_INVALID_ERL_DRV_PORT;
+        return ERTS_INVALID_ERL_DRV_PORT;
     }
 
     if (driver->handle != NULL) {
@@ -799,9 +797,15 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
     }
     ERTS_LC_ASSERT(erts_lc_is_port_locked(port));
 
-    erts_proc_lock(rp, ERTS_PROC_LOCK_LINK);
-    if (ERTS_PROC_IS_EXITING(rp)) {
-	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+    ldp = erts_link_create(ERTS_LNK_TYPE_PORT,
+                           port->common.id, pid);
+    ASSERT(ldp->a.other.item == pid);
+    ASSERT(ldp->b.other.item == port->common.id);
+    erts_link_tree_insert(&ERTS_P_LINKS(port), &ldp->a);
+
+    if (!erts_proc_sig_send_link(NULL, pid, &ldp->b)) {
+        erts_link_tree_delete(&ERTS_P_LINKS(port), &ldp->a);
+        erts_link_release_both(ldp);
 	if (driver->handle) {
 	    erts_rwmtx_rlock(&erts_driver_list_lock);
 	    erts_ddll_decrement_port_count(driver->handle);
@@ -812,10 +816,6 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
 	return ERTS_INVALID_ERL_DRV_PORT;
     }
 
-    erts_add_link(&ERTS_P_LINKS(port), LINK_PID, pid);
-    erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, port->common.id);
-    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-
     if (!driver_lock) {
 	ErtsXPortsList *xplp = xports_list_alloc();
 	xplp->port = port;
@@ -1167,21 +1167,10 @@ erts_schedule_proc2port_signal(Process *c_p,
 	 *       otherwise, next receive will *not* work
 	 *       as expected!
 	 */
-	erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-
-	if (ERTS_PROC_PENDING_EXIT(c_p)) {
-	    /* need to exit caller instead */
-	    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-	    KILL_CATCHES(c_p);
-	    c_p->freason = EXC_EXIT;
-	    return ERTS_PORT_OP_CALLER_EXIT;
-	}
-
-	ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
-	c_p->msg.save = c_p->msg.last;
 
-        erts_proc_unlock(c_p, (ERTS_PROC_LOCKS_MSG_RECEIVE
-                                   | ERTS_PROC_LOCK_MAIN));
+        ERTS_RECV_MARK_SAVE(c_p);
+        ERTS_RECV_MARK_SET(c_p);
+        erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
     }
 
 
@@ -1239,29 +1228,12 @@ erts_schedule_port2port_signal(Eterm port_num, ErtsProc2PortSigData *sigdp,
 
 static ERTS_INLINE void
 send_badsig(Port *prt) {
-    ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
-    Process* rp;
     Eterm connected = ERTS_PORT_GET_CONNECTED(prt);
     ERTS_CHK_NO_PROC_LOCKS;
     ERTS_LC_ASSERT(erts_get_scheduler_id());
-
     ASSERT(is_internal_pid(connected));
-
-    rp = erts_proc_lookup_raw(connected);
-    if (rp) {
-	erts_proc_lock(rp, rp_locks);
-	if (!ERTS_PROC_IS_EXITING(rp))
-	    (void) erts_send_exit_signal(NULL,
-					 prt->common.id,
-					 rp,
-					 &rp_locks, 
-					 am_badsig,
-					 NIL,
-					 NULL,
-					 0);
-	if (rp_locks)
-	    erts_proc_unlock(rp, rp_locks);
-    } /* exit sent */
+    erts_proc_sig_send_exit(NULL, prt->common.id, connected,
+                            am_badsig, NIL, 0);
 } /* send_badsig */
 
 static void
@@ -2194,11 +2166,11 @@ call_deliver_port_exit(int bang_op,
     }
 
     if (broken_link) {
-	ErtsLink *lnk = erts_remove_link(&ERTS_P_LINKS(prt), from);
-	if (lnk)
-	    erts_destroy_link(lnk);
-	else
+	ErtsLink *lnk = erts_link_tree_lookup(ERTS_P_LINKS(prt), from);
+	if (!lnk) 
 	    return ERTS_PORT_OP_DROPPED;
+        erts_link_tree_delete(&ERTS_P_LINKS(prt), lnk);
+        erts_link_release(lnk);
     }
 
     if (IS_TRACED_FL(prt, F_TRACE_RECEIVE))
@@ -2367,27 +2339,30 @@ set_port_connected(int bang_op,
 #endif
     }
     else { /* Port BIF operation */
-	Process *rp = erts_proc_lookup_raw(connect);
-	if (!rp)
-	    return ERTS_PORT_OP_DROPPED;
-	erts_proc_lock(rp, ERTS_PROC_LOCK_LINK);
-	if (ERTS_PROC_IS_EXITING(rp)) {
-	    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-	    return ERTS_PORT_OP_DROPPED;
-	}
-
-	erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, prt->common.id);
-	erts_add_link(&ERTS_P_LINKS(prt), LINK_PID, connect);
-
-        if (IS_TRACED_FL(rp, F_TRACE_PROCS))
-            trace_proc(NULL, 0, rp, am_getting_linked, prt->common.id);
+        int created;
+        ErtsLink *lnk;
+
+        if (is_not_internal_pid(connect))
+            return ERTS_PORT_OP_DROPPED;
+
+        lnk = erts_link_tree_lookup_create(&ERTS_P_LINKS(prt), &created,
+                                           ERTS_LNK_TYPE_PORT, prt->common.id,
+                                           connect);
+        if (created) {
+            ErtsLinkData *ldp;
+            ErtsLink *olnk = erts_link_to_other(lnk, &ldp);
+            ASSERT(olnk->other.item == prt->common.id);
+            if (!erts_proc_sig_send_link(NULL, connect, olnk)) {
+                erts_link_tree_delete(&ERTS_P_LINKS(prt), lnk);
+                erts_link_release_both(ldp);
+                return ERTS_PORT_OP_DROPPED;
+            }
+            if (IS_TRACED_FL(prt, F_TRACE_PORTS))
+                trace_port(prt, am_getting_linked, connect);
+        }
 
 	ERTS_PORT_SET_CONNECTED(prt, connect);
 
-	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-
-        if (IS_TRACED_FL(prt, F_TRACE_PORTS))
-            trace_port(prt, am_getting_linked, connect);
         if (IS_TRACED_FL(prt, F_TRACE_RECEIVE))
             trace_port_receive(prt, from, am_connect, connect);
         if (IS_TRACED_FL(prt, F_TRACE_SEND)) {
@@ -2494,13 +2469,24 @@ erts_port_connect(Process *c_p,
 }
 
 static void
-port_unlink(Port *prt, Eterm from)
+port_unlink(Port *prt, ErtsLink *lnk)
 {
-    ErtsLink *lnk = erts_remove_link(&ERTS_P_LINKS(prt), from);
-    if (lnk) {
+    ErtsLinkData *ldp;
+    ErtsLink *dlnk, *llnk;
+
+    llnk = erts_link_to_other(lnk, &ldp);
+    dlnk = erts_link_tree_key_delete(&ERTS_P_LINKS(prt), llnk);
+    if (!dlnk)
+        erts_link_release(lnk);
+    else {
         if (IS_TRACED_FL(prt, F_TRACE_PORTS))
-            trace_port(prt, am_getting_unlinked, from);
-	erts_destroy_link(lnk);
+            trace_port(prt, am_getting_unlinked, llnk->other.item);
+        if (dlnk == llnk)
+            erts_link_release_both(ldp);
+        else {
+            erts_link_release(lnk);
+            erts_link_release(dlnk);
+        }
     }
 }
 
@@ -2508,14 +2494,14 @@ static int
 port_sig_unlink(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp)
 {
     if (op == ERTS_PROC2PORT_SIG_EXEC)
-	port_unlink(prt, sigdp->u.unlink.from);
+	port_unlink(prt, sigdp->u.unlink.lnk);
     if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
 	port_sched_op_reply(sigdp->caller, sigdp->ref, am_true, prt);
     return ERTS_PORT_REDS_UNLINK;
 }
 
 ErtsPortOpResult
-erts_port_unlink(Process *c_p, Port *prt, Eterm from, Eterm *refp)
+erts_port_unlink(Process *c_p, Port *prt, ErtsLink *lnk, Eterm *refp)
 {
     ErtsProc2PortSigData *sigdp;
     ErtsTryImmDrvCallState try_call_state
@@ -2528,7 +2514,7 @@ erts_port_unlink(Process *c_p, Port *prt, Eterm from, Eterm *refp)
 
     switch (try_imm_drv_call(&try_call_state)) {
     case ERTS_TRY_IMM_DRV_CALL_OK:
-	port_unlink(prt, from);
+	port_unlink(prt, lnk);
 	finalize_imm_drv_call(&try_call_state);
 	BUMP_REDS(c_p, ERTS_PORT_REDS_UNLINK);
 	return ERTS_PORT_OP_DONE;
@@ -2541,11 +2527,12 @@ erts_port_unlink(Process *c_p, Port *prt, Eterm from, Eterm *refp)
 
     sigdp = erts_port_task_alloc_p2p_sig_data();
     sigdp->flags = ERTS_P2P_SIG_TYPE_UNLINK;
-    sigdp->u.unlink.from = from;
-    
+    sigdp->u.unlink.port_id = prt->common.id;
+    sigdp->u.unlink.lnk = lnk;
+
     return erts_schedule_proc2port_signal(c_p,
 					  prt,
-					  c_p ? c_p->common.id : from,
+					  c_p->common.id,
 					  refp,
 					  sigdp,
 					  0,
@@ -2554,45 +2541,24 @@ erts_port_unlink(Process *c_p, Port *prt, Eterm from, Eterm *refp)
 }
 
 static void
-port_link_failure(Eterm port_id, Eterm linker)
+port_link_failure(Eterm port_id, ErtsLink *lnk)
 {
-    Process *rp;
-    ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCKS_XSIG_SEND;
-    ASSERT(is_internal_pid(linker));
-    rp = erts_pid2proc(NULL, 0, linker, rp_locks);
-    if (rp) {
-	ErtsLink *rlnk = erts_remove_link(&ERTS_P_LINKS(rp), port_id);
-	if (rlnk) {
-	    int xres = erts_send_exit_signal(NULL,
-					     port_id,
-					     rp,
-					     &rp_locks, 
-					     am_noproc,
-					     NIL,
-					     NULL,
-					     0);
-	    if (xres >= 0) {
-		/* We didn't exit the process and it is traced */
-		if (IS_TRACED_FL(rp, F_TRACE_PROCS))
-		    trace_proc(NULL, 0, rp, am_getting_unlinked, port_id);
-	    }
-	    if (rp_locks)
-		erts_proc_unlock(rp, rp_locks);
-	}
-    }
+    erts_proc_sig_send_link_exit(NULL, port_id, lnk, am_noproc, NIL);
 }
 
 static void
-port_link(Port *prt, erts_aint32_t state, Eterm to)
+port_link(Port *prt, erts_aint32_t state, ErtsLink *lnk)
 {
-    if (IS_TRACED_FL(prt, F_TRACE_PORTS))
-        trace_port(prt, am_getting_linked, to);
-    if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) {
-	erts_add_link(&ERTS_P_LINKS(prt), LINK_PID, to);
-    } else {
-	port_link_failure(prt->common.id, to);
-        if (IS_TRACED_FL(prt, F_TRACE_PORTS))
-            trace_port(prt, am_unlink, to);
+    if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP)
+	port_link_failure(prt->common.id, lnk);
+    else {
+        ErtsLink *rlnk;
+        rlnk = erts_link_tree_insert_addr_replace(&ERTS_P_LINKS(prt),
+                                                  lnk);
+        if (rlnk)
+            erts_link_release(rlnk);
+        else if (IS_TRACED_FL(prt, F_TRACE_PORTS))
+            trace_port(prt, am_getting_linked, lnk->other.item);
     }
 }
 
@@ -2600,17 +2566,16 @@ static int
 port_sig_link(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp)
 {
     if (op == ERTS_PROC2PORT_SIG_EXEC)
-	port_link(prt, state, sigdp->u.link.to);
-    else {
-	port_link_failure(sigdp->u.link.port, sigdp->u.link.to);
-    }
+	port_link(prt, state, sigdp->u.link.lnk);
+    else
+	port_link_failure(sigdp->u.link.port_id, sigdp->u.link.lnk);
     if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
 	port_sched_op_reply(sigdp->caller, sigdp->ref, am_true, prt);
     return ERTS_PORT_REDS_LINK;
 }
 
 ErtsPortOpResult
-erts_port_link(Process *c_p, Port *prt, Eterm to, Eterm *refp)
+erts_port_link(Process *c_p, Port *prt, ErtsLink *lnk, Eterm *refp)
 {
     ErtsProc2PortSigData *sigdp;
     ErtsTryImmDrvCallState try_call_state
@@ -2623,7 +2588,7 @@ erts_port_link(Process *c_p, Port *prt, Eterm to, Eterm *refp)
 
     switch (try_imm_drv_call(&try_call_state)) {
     case ERTS_TRY_IMM_DRV_CALL_OK:
-	port_link(prt, try_call_state.state, to);
+	port_link(prt, try_call_state.state, lnk);
 	finalize_imm_drv_call(&try_call_state);
 	BUMP_REDS(c_p, ERTS_PORT_REDS_LINK);
 	return ERTS_PORT_OP_DONE;
@@ -2636,12 +2601,12 @@ erts_port_link(Process *c_p, Port *prt, Eterm to, Eterm *refp)
 
     sigdp = erts_port_task_alloc_p2p_sig_data();
     sigdp->flags = ERTS_P2P_SIG_TYPE_LINK;
-    sigdp->u.link.port = prt->common.id;
-    sigdp->u.link.to = to;
+    sigdp->u.link.port_id = prt->common.id;
+    sigdp->u.link.lnk = lnk;
     
     return erts_schedule_proc2port_signal(c_p,
 					  prt,
-					  c_p ? c_p->common.id : to,
+					  c_p->common.id,
 					  refp,
 					  sigdp,
 					  0,
@@ -2650,51 +2615,23 @@ erts_port_link(Process *c_p, Port *prt, Eterm to, Eterm *refp)
 }
 
 static void
-port_monitor_failure(Eterm port_id, Eterm origin, Eterm ref_DOWN)
+port_monitor_failure(Eterm port_id, ErtsMonitor *mon)
 {
-    Process       *origin_p;
-    ErtsProcLocks p_locks = ERTS_PROC_LOCK_LINK;
-    ASSERT(is_internal_pid(origin));
-
-    origin_p = erts_pid2proc(NULL, 0, origin, p_locks);
-    if (! origin_p) { return; }
-
-    /* Send the DOWN message immediately. Ref is made on the fly because
-     * caller has never seen it yet. */
-    erts_queue_monitor_message(origin_p, &p_locks, ref_DOWN,
-                               am_port, port_id, am_noproc);
-    erts_proc_unlock(origin_p, p_locks);
+    erts_proc_sig_send_monitor_down(mon, am_noproc);
 }
 
 /* Origin wants to monitor port Prt. State contains possible error, which has
  * happened just before. Name is either NIL or an atom, if user monitors
  * a port by name. Ref is premade reference that will be returned to user */
 static void
-port_monitor(Port *prt, erts_aint32_t state, Eterm origin,
-             Eterm name, Eterm ref)
+port_monitor(Port *prt, erts_aint32_t state, ErtsMonitor *mon)
 {
-    Eterm name_or_nil = is_atom(name) ? name : NIL;
-
-    ASSERT(is_pid(origin));
-    ASSERT(is_atom(name) || is_port(name) || name == NIL);
-    ASSERT(is_internal_ordinary_ref(ref));
-
-    if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) {
-        ErtsProcLocks p_locks = ERTS_PROC_LOCK_LINK;
-
-        Process *origin_p = erts_pid2proc(NULL, 0, origin, p_locks);
-        if (! origin_p) {
-            goto failure;
-        }
-        erts_add_monitor(&ERTS_P_MONITORS(origin_p), MON_ORIGIN, ref,
-                         prt->common.id, name_or_nil);
-        erts_add_monitor(&ERTS_P_MONITORS(prt), MON_TARGET, ref,
-                         origin, name_or_nil);
-
-        erts_proc_unlock(origin_p, p_locks);
-    } else {
-failure:
-        port_monitor_failure(prt->common.id, origin, ref);
+    ASSERT(prt);
+    if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP)
+        port_monitor_failure(prt->common.id, mon);
+    else {
+        ASSERT(erts_monitor_is_target(mon));
+        erts_monitor_list_insert(&ERTS_P_LT_MONITORS(prt), mon);
     }
 }
 
@@ -2702,24 +2639,11 @@ static int
 port_sig_monitor(Port *prt, erts_aint32_t state, int op,
                  ErtsProc2PortSigData *sigdp)
 {
-    Eterm hp[ERTS_REF_THING_SIZE];
-    Eterm ref = make_internal_ref(&hp);
-    write_ref_thing(hp, sigdp->ref[0], sigdp->ref[1], sigdp->ref[2]);
-
-    if (op == ERTS_PROC2PORT_SIG_EXEC) {
-        /* erts_add_monitor call inside port_monitor will copy ref from hp */
-        port_monitor(prt, state,
-                     sigdp->u.monitor.origin,
-                     sigdp->u.monitor.name,
-                     ref);
-    } else {
-        port_monitor_failure(sigdp->u.monitor.name,
-                             sigdp->u.monitor.origin,
-                             ref);
-    }
-    if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) {
-        port_sched_op_reply(sigdp->caller, sigdp->ref, am_true, prt);
-    }
+    if (op == ERTS_PROC2PORT_SIG_EXEC)
+        port_monitor(prt, state, sigdp->u.monitor.mon);
+    else
+        port_monitor_failure(sigdp->u.monitor.port_id,
+                             sigdp->u.monitor.mon);
     return ERTS_PORT_REDS_MONITOR;
 }
 
@@ -2727,95 +2651,63 @@ port_sig_monitor(Port *prt, erts_aint32_t state, int op,
  * a reference (ref may be rewritten to be used to serve additionally as a
  * signal id). Name is atom if user monitors port by name or NIL */
 ErtsPortOpResult
-erts_port_monitor(Process *origin, Port *port, Eterm name, Eterm *refp)
+erts_port_monitor(Process *c_p, Port *port, ErtsMonitor *mon)
 {
     ErtsProc2PortSigData *sigdp;
     ErtsTryImmDrvCallState try_call_state
         = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(
-                origin, port, ERTS_PORT_SFLGS_INVALID_LOOKUP,
+                c_p,
+                port,
+                ERTS_PORT_SFLGS_INVALID_LOOKUP,
                 0,
-                0, /* trap_ref is always set so !trap_ref always is false */
+                !0,
                 am_monitor);
 
-    ASSERT(origin);
+    ASSERT(c_p);
     ASSERT(port);
-    ASSERT(is_atom(name) || is_port(name));
-    ASSERT(refp);
+    ASSERT(mon);
 
     switch (try_imm_drv_call(&try_call_state)) {
     case ERTS_TRY_IMM_DRV_CALL_OK:
-        port_monitor(port, try_call_state.state, origin->common.id, name, *refp);
+        port_monitor(port, try_call_state.state, mon);
         finalize_imm_drv_call(&try_call_state);
-        BUMP_REDS(origin, ERTS_PORT_REDS_MONITOR);
+        BUMP_REDS(c_p, ERTS_PORT_REDS_MONITOR);
         return ERTS_PORT_OP_DONE;
     case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
-        return ERTS_PORT_OP_BADARG;
+        return ERTS_PORT_OP_DROPPED;
     default:
         break; /* Schedule call instead... */
     }
 
     sigdp = erts_port_task_alloc_p2p_sig_data();
     sigdp->flags = ERTS_P2P_SIG_TYPE_MONITOR;
-    sigdp->u.monitor.origin = origin->common.id;
-    sigdp->u.monitor.name   = name; /* either named monitor, or port id */
+    sigdp->u.monitor.port_id = port->common.id;
+    sigdp->u.monitor.mon = mon;
 
     /* Ref contents will be initialized here */
-    return erts_schedule_proc2port_signal(origin, port, origin->common.id,
-                                          refp, sigdp, 0, NULL,
+    return erts_schedule_proc2port_signal(c_p,
+                                          port,
+                                          c_p->common.id,
+                                          NULL,
+                                          sigdp,
+                                          0,
+                                          NULL,
                                           port_sig_monitor);
 }
 
-static void
-port_demonitor_failure(Eterm port_id, Eterm origin, Eterm ref)
-{
-    Process       *origin_p;
-    ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK;
-    ErtsMonitor   *mon1;
-    ASSERT(is_internal_pid(origin));
-
-    origin_p = erts_pid2proc(NULL, 0, origin, rp_locks);
-    if (! origin_p) { return; }
-
-    /* do not send any DOWN messages, drop monitors on process */
-    mon1 = erts_remove_monitor(&ERTS_P_MONITORS(origin_p), ref);
-    if (mon1 != NULL) {
-        erts_destroy_monitor(mon1);
-    }
-
-    erts_proc_unlock(origin_p, rp_locks);
-}
-
 /* Origin wants to demonitor port Prt. State contains possible error, which has
  * happened just before. Ref is reference to monitor */
 static void
-port_demonitor(Port *port, erts_aint32_t state, Eterm origin, Eterm ref)
+port_demonitor(Port *port, erts_aint32_t state, ErtsMonitor *mon)
 {
-    ASSERT(port);
-    ASSERT(is_pid(origin));
-    ASSERT(is_internal_ref(ref));
-
-    if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) {
-        ErtsProcLocks p_locks = ERTS_PROC_LOCK_LINK;
-        Process *origin_p = erts_pid2proc(NULL, 0, origin, p_locks);
-        if (origin_p) {
-            ErtsMonitor *mon1 = erts_remove_monitor(&ERTS_P_MONITORS(origin_p),
-                                                    ref);
-            if (mon1 != NULL) {
-                erts_destroy_monitor(mon1);
-            }
-        }
-        if (1) {
-            ErtsMonitor *mon2 = erts_remove_monitor(&ERTS_P_MONITORS(port),
-                                                    ref);
-            if (mon2 != NULL) {
-                erts_destroy_monitor(mon2);
-            }
-        }
-        if (origin_p) { /* when origin is dying, it won't be found */
-            erts_proc_unlock(origin_p, p_locks);
-        }
-    } else {
-        port_demonitor_failure(port->common.id, origin, ref);
+    ErtsMonitorData *mdp = erts_monitor_to_data(mon);
+    ASSERT(port && mon);
+    ASSERT(erts_monitor_is_origin(mon));
+    if (!erts_monitor_is_in_table(&mdp->target))
+        erts_monitor_release(mon);
+    else {
+        erts_monitor_list_delete(&ERTS_P_LT_MONITORS(port), &mdp->target);
+        erts_monitor_release_both(mdp);
     }
 }
 
@@ -2823,73 +2715,47 @@ static int
 port_sig_demonitor(Port *prt, erts_aint32_t state, int op,
                    ErtsProc2PortSigData *sigdp)
 {
-    Eterm hp[ERTS_REF_THING_SIZE];
-    Eterm ref = make_internal_ref(&hp);
-    write_ref_thing(hp, sigdp->u.demonitor.ref[0],
-                    sigdp->u.demonitor.ref[1],
-                    sigdp->u.demonitor.ref[2]);
-    if (op == ERTS_PROC2PORT_SIG_EXEC) {
-        port_demonitor(prt, state, sigdp->u.demonitor.origin, ref);
-    } else {
-        port_demonitor_failure(sigdp->u.demonitor.name,
-                               sigdp->u.demonitor.origin,
-                               ref);
-    }
-    if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) {
-        port_sched_op_reply(sigdp->caller, sigdp->ref, am_true, prt);
-    }
+    if (op == ERTS_PROC2PORT_SIG_EXEC)
+        port_demonitor(prt, state, sigdp->u.demonitor.mon);
+    else
+        erts_monitor_release(sigdp->u.demonitor.mon);
     return ERTS_PORT_REDS_DEMONITOR;
 }
 
-/* Removes monitor between origin and target, identified by ref.
- * Mode defines normal or relaxed demonitor rules (process is at death) */
-ErtsPortOpResult erts_port_demonitor(Process *origin, ErtsDemonitorMode mode,
-                                     Port *target, Eterm ref,
-                                     Eterm *trap_ref)
+ErtsPortOpResult
+erts_port_demonitor(Process *c_p, Port *prt, ErtsMonitor *mon)
 {
-    Process *c_p = mode == ERTS_PORT_DEMONITOR_NORMAL ? origin : NULL;
     ErtsProc2PortSigData *sigdp;
     ErtsTryImmDrvCallState try_call_state
         = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(
                 c_p,
-                target, ERTS_PORT_SFLGS_INVALID_LOOKUP,
+                prt, ERTS_PORT_SFLGS_INVALID_LOOKUP,
                 0,
-                !trap_ref,
+                !0,
                 am_demonitor);
 
-    ASSERT(origin);
-    ASSERT(target);
-    ASSERT(is_internal_ref(ref));
+    ASSERT(c_p && prt && mon);
 
     switch (try_imm_drv_call(&try_call_state)) {
     case ERTS_TRY_IMM_DRV_CALL_OK:
-        port_demonitor(target, try_call_state.state, origin->common.id, ref);
+        port_demonitor(prt, try_call_state.state, mon);
         finalize_imm_drv_call(&try_call_state);
-        if (mode == ERTS_PORT_DEMONITOR_NORMAL) {
-            BUMP_REDS(origin, ERTS_PORT_REDS_DEMONITOR);
-        }
+        BUMP_REDS(c_p, ERTS_PORT_REDS_DEMONITOR);
         return ERTS_PORT_OP_DONE;
     case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
-        return ERTS_PORT_OP_BADARG;
+        return ERTS_PORT_OP_DROPPED;
     default:
         break; /* Schedule call instead... */
     }
 
     sigdp = erts_port_task_alloc_p2p_sig_data();
     sigdp->flags = ERTS_P2P_SIG_TYPE_DEMONITOR;
-    sigdp->u.demonitor.origin = origin->common.id;
-    sigdp->u.demonitor.name = target->common.id;
-    {
-	Uint32 *nums = internal_ref_numbers(ref);
-        /* Start from 1 skip ref arity */
-        sys_memcpy(sigdp->u.demonitor.ref,
-		   nums,
-                   sizeof(sigdp->u.demonitor.ref));
-    }
+    sigdp->u.demonitor.port_id = prt->common.id;
+    sigdp->u.demonitor.mon = mon;
 
     /* Ref contents will be initialized here */
-    return erts_schedule_proc2port_signal(c_p, target, origin->common.id,
-                                          trap_ref, sigdp, 0, NULL,
+    return erts_schedule_proc2port_signal(c_p, prt, c_p->common.id,
+                                          NULL, sigdp, 0, NULL,
                                           port_sig_demonitor);
 }
 
@@ -2898,11 +2764,13 @@ init_ack_send_reply(Port *port, Eterm resp)
 {
 
     if (!is_internal_port(resp)) {
-        Process *rp = erts_proc_lookup_raw(port->async_open_port->to);
-        erts_proc_lock(rp, ERTS_PROC_LOCK_LINK);
-        erts_remove_link(&ERTS_P_LINKS(port), port->async_open_port->to);
-        erts_remove_link(&ERTS_P_LINKS(rp), port->common.id);
-        erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+        Eterm proc = port->async_open_port->to;
+        ErtsLink *lnk = erts_link_tree_lookup(ERTS_P_LINKS(port),
+                                              proc);
+        if (lnk) {
+            erts_link_tree_delete(&ERTS_P_LINKS(port), lnk);
+            erts_proc_sig_send_unlink(NULL, lnk);
+        }
     }
     port_sched_op_reply(port->async_open_port->to,
                         port->async_open_port->ref,
@@ -3680,6 +3548,7 @@ terminate_port(Port *prt)
 
     ASSERT(!ERTS_P_LINKS(prt));
     ASSERT(!ERTS_P_MONITORS(prt));
+    ASSERT(!ERTS_P_LT_MONITORS(prt));
 
     /* state may be altered by kill_port() below */
     state = erts_atomic32_read_band_nob(&prt->state,
@@ -3767,146 +3636,25 @@ erts_terminate_port(Port *pp)
     terminate_port(pp);
 }
 
-static void port_fire_one_monitor(ErtsMonitor *mon, void *ctx0);
-static void sweep_one_monitor(ErtsMonitor *mon, void *vpsc)
-{
-    switch (mon->type) {
-    case MON_ORIGIN: {
-        ErtsMonitor *rmon;
-        Process *rp;
-
-        ASSERT(is_internal_pid(mon->u.pid));
-        rp = erts_pid2proc(NULL, 0, mon->u.pid, ERTS_PROC_LOCK_LINK);
-        if (!rp) {
-            goto done;
-        }
-        rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref);
-        erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-        if (rmon == NULL) {
-            goto done;
-        }
-        erts_destroy_monitor(rmon);
-    } break;
-    case MON_TARGET: {
-        port_fire_one_monitor(mon, vpsc); /* forward call */
-    } break;
-    }
- done:
-    erts_destroy_monitor(mon);
-}
-
-
-
 typedef struct {
-    Port *port;
+    Eterm port_id;
     Eterm reason;
-} SweepContext;
+} ErtsPortExitContext;
 
-static void sweep_one_link(ErtsLink *lnk, void *vpsc)
+static void link_port_exit(ErtsLink *lnk, void *vpectxt)
 {
-    SweepContext *psc = vpsc;
-    DistEntry *dep;
-    Process *rp;
-    Eterm port_id = psc->port->common.id;
-
-    ASSERT(lnk->type == LINK_PID);
-
-    if (IS_TRACED_FL(psc->port, F_TRACE_PORTS))
-        trace_port(psc->port, am_unlink, lnk->pid);
-
-    if (is_external_pid(lnk->pid)) {
-	dep = external_pid_dist_entry(lnk->pid);
-	if(dep != erts_this_dist_entry) {
-	    ErtsDistLinkData dld;
-	    ErtsDSigData dsd;
-	    int code;
-	    code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0);
-	    switch (code) {
-	    case ERTS_DSIG_PREP_NOT_ALIVE:
-	    case ERTS_DSIG_PREP_NOT_CONNECTED:
-		break;
-	    case ERTS_DSIG_PREP_PENDING:
-	    case ERTS_DSIG_PREP_CONNECTED:
-		erts_remove_dist_link(&dld, port_id, lnk->pid, dep);
-		erts_destroy_dist_link(&dld);
-		code = erts_dsig_send_exit(&dsd, port_id, lnk->pid,
-					   psc->reason);
-		ASSERT(code == ERTS_DSIG_SEND_OK);
-		break;
-	    default:
-		ASSERT(! "Invalid dsig prepare result");
-		break;
-	    }
-	}
-    } else {
-	ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCKS_XSIG_SEND;
-	ASSERT(is_internal_pid(lnk->pid));
-	rp = erts_pid2proc(NULL, 0, lnk->pid, rp_locks);
-	if (rp) {
-	    ErtsLink *rlnk = erts_remove_link(&ERTS_P_LINKS(rp), port_id);
-
-	    if (rlnk) {
-		int xres = erts_send_exit_signal(NULL,
-						 port_id,
-						 rp,
-						 &rp_locks, 
-						 psc->reason,
-						 NIL,
-						 NULL,
-						 0);
-		if (xres >= 0) {
-                    if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) {
-                        erts_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND);
-                        rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND;
-                    }
-		    /* We didn't exit the process and it is traced */
-		    if (IS_TRACED_FL(rp, F_TRACE_PROCS))
-			trace_proc(NULL, 0, rp, am_getting_unlinked, port_id);
-		}
-		erts_destroy_link(rlnk);
-	    }
-
-	    erts_proc_unlock(rp, rp_locks);
-	}
-    }
-    erts_destroy_link(lnk);
+    ErtsPortExitContext *pectxt = vpectxt;
+    erts_proc_sig_send_link_exit(NULL, pectxt->port_id,
+                                 lnk, pectxt->reason, NIL);
 }
 
-static void
-port_fire_one_monitor(ErtsMonitor *mon, void *ctx0)
+static void monitor_port_exit(ErtsMonitor *mon, void *vpectxt)
 {
-    Process      *origin;
-    ErtsProcLocks origin_locks;
-
-    if (mon->type != MON_TARGET || ! is_pid(mon->u.pid)) {
-        return;
-    }
-    /*
-     * Proceed here if someone monitors us, we (port) are the target and
-     * origin is some process
-     */
-    origin_locks = ERTS_PROC_LOCKS_MSG_SEND | ERTS_PROC_LOCK_LINK;
-
-    origin = erts_pid2proc(NULL, 0, mon->u.pid, origin_locks);
-    if (origin) {
-        DeclareTmpHeapNoproc(lhp,3);
-        SweepContext    *ctx = (SweepContext *)ctx0;
-        ErtsMonitor     *rmon;
-        Eterm           watched = (is_atom(mon->name)
-                       ? TUPLE2(lhp, mon->name, erts_this_dist_entry->sysname)
-                       : ctx->port->common.id);
-
-        erts_queue_monitor_message(origin, &origin_locks, mon->ref, am_port,
-                                   watched, ctx->reason);
-        UnUseTmpHeapNoproc(3);
-
-        rmon = erts_remove_monitor(&ERTS_P_MONITORS(origin), mon->ref);
-        erts_proc_unlock(origin, origin_locks);
-
-        if (rmon) {
-            erts_destroy_monitor(rmon);
-        }
-    }
+    ErtsPortExitContext *pectxt = vpectxt;
+    if (erts_monitor_is_target(mon))
+        erts_proc_sig_send_monitor_down(mon, pectxt->reason);
+    else
+        erts_proc_sig_send_demonitor(mon);
 }
 
 /* 'from' is sending 'this_port' an exit signal, (this_port must be internal).
@@ -3923,9 +3671,12 @@ int
 erts_deliver_port_exit(Port *prt, Eterm from, Eterm reason, int send_closed,
                        int drop_normal)
 {
-   ErtsLink *lnk;
+   ErtsLink *links;
+   ErtsMonitor *monitors;
+   ErtsMonitor *lt_monitors;
    Eterm modified_reason;
    erts_aint32_t state, set_state_flags;
+   ErtsPortExitContext pectxt;
 
    ERTS_CHK_NO_PROC_LOCKS;
    ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
@@ -3973,23 +3724,36 @@ erts_deliver_port_exit(Port *prt, Eterm from, Eterm reason, int send_closed,
 
    set_busy_port(ERTS_Port2ErlDrvPort(prt), 0);
 
+   links = ERTS_P_LINKS(prt);
+   ERTS_P_LINKS(prt) = NULL;
+   monitors = ERTS_P_MONITORS(prt);
+   ERTS_P_MONITORS(prt) = NULL;
+   lt_monitors = ERTS_P_LT_MONITORS(prt);
+   ERTS_P_LT_MONITORS(prt) = NULL;
+
    if (prt->common.u.alive.reg != NULL)
        (void) erts_unregister_name(NULL, 0, prt, prt->common.u.alive.reg->name);
 
-   {
-       SweepContext sc = {prt, modified_reason};
-       lnk = ERTS_P_LINKS(prt);
-       ERTS_P_LINKS(prt) = NULL;
-       erts_sweep_links(lnk, &sweep_one_link, &sc);
+   pectxt.port_id = prt->common.id;
+   pectxt.reason = modified_reason;
+
+   if (links)
+       erts_monitor_tree_foreach_delete(&links,
+                                        link_port_exit,
+                                        (void *) &pectxt);
+
+   if (monitors || lt_monitors) {
+       DRV_MONITOR_LOCK_PDL(prt);
+       if (monitors)
+           erts_monitor_tree_foreach_delete(&monitors,
+                                            monitor_port_exit,
+                                            (void *) &pectxt);
+       if (lt_monitors)
+           erts_monitor_list_foreach_delete(&lt_monitors,
+                                            monitor_port_exit,
+                                            (void *) &pectxt);
+       DRV_MONITOR_UNLOCK_PDL(prt);
    }
-   DRV_MONITOR_LOCK_PDL(prt);
-   {
-       SweepContext ctx = {prt, modified_reason};
-       ErtsMonitor *moni = ERTS_P_MONITORS(prt);
-       ERTS_P_MONITORS(prt) = NULL;
-       erts_sweep_monitors(moni, &sweep_one_monitor, &ctx);
-   } 
-   DRV_MONITOR_UNLOCK_PDL(prt);
 
    if ((state & ERTS_PORT_SFLG_DISTRIBUTION) && prt->dist_entry) {
        erts_do_net_exits(prt->dist_entry, modified_reason);
@@ -5069,14 +4833,18 @@ typedef struct {
 
 static void prt_one_monitor(ErtsMonitor *mon, void *vprtd)
 {
+    ErtsMonitorData *mdp = erts_monitor_to_data(mon);
     prt_one_lnk_data *prtd = (prt_one_lnk_data *) vprtd;
-    erts_print(prtd->to, prtd->arg, "(%T,%T)", mon->u.pid, mon->ref);
+    if (mon->type == ERTS_MON_TYPE_RESOURCE && erts_monitor_is_target(mon))
+        erts_print(prtd->to, prtd->arg, "(%p,%T)", mon->other.ptr, mdp->ref);
+    else
+        erts_print(prtd->to, prtd->arg, "(%T,%T)", mon->other.item, mdp->ref);
 }
 
 static void prt_one_lnk(ErtsLink *lnk, void *vprtd)
 {
     prt_one_lnk_data *prtd = (prt_one_lnk_data *) vprtd;
-    erts_print(prtd->to, prtd->arg, "%T", lnk->pid);
+    erts_print(prtd->to, prtd->arg, "%T", lnk->other.item);
 }
 
 static void dump_port_state(fmtfn_t to, void *arg, erts_aint32_t state)
@@ -5188,15 +4956,18 @@ print_port_info(Port *p, fmtfn_t to, void *arg)
 	prtd.to = to;
 	prtd.arg = arg;
 	erts_print(to, arg, "Links: ");
-	erts_doforall_links(ERTS_P_LINKS(p), &prt_one_lnk, &prtd);
+	erts_link_tree_foreach(ERTS_P_LINKS(p), prt_one_lnk, (void *) &prtd);
 	erts_print(to, arg, "\n");
     }
-    if (ERTS_P_MONITORS(p)) {
+    if (ERTS_P_MONITORS(p) || ERTS_P_LT_MONITORS(p)) {
 	prt_one_lnk_data prtd;
 	prtd.to = to;
 	prtd.arg = arg;
 	erts_print(to, arg, "Monitors: ");
-	erts_doforall_monitors(ERTS_P_MONITORS(p), &prt_one_monitor, &prtd);
+	erts_monitor_tree_foreach(ERTS_P_MONITORS(p), prt_one_monitor,
+                                  (void *) &prtd);
+	erts_monitor_list_foreach(ERTS_P_LT_MONITORS(p), prt_one_monitor,
+                                  (void *) &prtd);
 	erts_print(to, arg, "\n");
     }
     if (p->suspended) {
@@ -7033,24 +6804,23 @@ static int do_driver_monitor_process(Port *prt,
 				     ErlDrvMonitor *monitor)
 {
     Eterm buf[ERTS_REF_THING_SIZE];
-    Process *rp;
     Eterm ref;
+    ErtsMonitorData *mdp;
 
-    if (prt->drv_ptr->process_exit == NULL) {
+    if (!prt->drv_ptr->process_exit)
 	return -1;
-    }
-    rp = erts_pid2proc_opt(NULL, 0,
-			   (Eterm) process, ERTS_PROC_LOCK_LINK,
-			   ERTS_P2P_FLG_ALLOW_OTHER_X);
-    if (!rp) {
-	return 1;
-    }
 
     ref = erts_make_ref_in_buffer(buf);
-    erts_add_monitor(&ERTS_P_MONITORS(prt), MON_ORIGIN, ref, rp->common.id, NIL);
-    erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, ref, prt->common.id, NIL);
+    mdp = erts_monitor_create(ERTS_MON_TYPE_PORT, ref,
+                              prt->common.id, process, NIL);
+
+    if (!erts_proc_sig_send_monitor(&mdp->target, process)) {
+        erts_monitor_release_both(mdp);
+        return 1;
+    }
+
+    erts_monitor_tree_insert(&ERTS_P_MONITORS(prt), &mdp->origin);
 
-    erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
     erts_ref_to_driver_monitor(ref,monitor);
     return 0;
 }
@@ -7083,37 +6853,17 @@ int driver_monitor_process(ErlDrvPort drvport,
 static int do_driver_demonitor_process(Port *prt, const ErlDrvMonitor *monitor)
 {
     Eterm heap[ERTS_REF_THING_SIZE];
-    Process *rp;
     Eterm ref;
     ErtsMonitor *mon;
-    Eterm to;
 
     ref = erts_driver_monitor_to_ref(heap, monitor);
 
-    mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref);
-    if (mon == NULL) {
-	return 1;
-    }
-    ASSERT(mon->type == MON_ORIGIN);
-    to = mon->u.pid;
-    ASSERT(is_internal_pid(to));
-    rp = erts_pid2proc_opt(NULL,
-			   0,
-			   to,
-			   ERTS_PROC_LOCK_LINK,
-			   ERTS_P2P_FLG_ALLOW_OTHER_X);
-    mon = erts_remove_monitor(&ERTS_P_MONITORS(prt), ref);
-    if (mon) {
-	erts_destroy_monitor(mon);
-    }
-    if (rp) {
-	ErtsMonitor *rmon;
-	rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref);
-	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-	if (rmon != NULL) {
-	    erts_destroy_monitor(rmon);
-	}
-    } 
+    mon = erts_monitor_tree_lookup(ERTS_P_MONITORS(prt), ref);
+    if (!mon || !erts_monitor_is_origin(mon))
+        return 1;
+
+    erts_monitor_tree_delete(&ERTS_P_MONITORS(prt), mon);
+    erts_proc_sig_send_demonitor(mon);
     return 0;
 }
 
@@ -7142,19 +6892,16 @@ static ErlDrvTermData do_driver_get_monitored_process(Port *prt,const ErlDrvMoni
 {
     Eterm ref;
     ErtsMonitor *mon;
-    Eterm to;
     Eterm heap[ERTS_REF_THING_SIZE];
 
     ref = erts_driver_monitor_to_ref(heap, monitor);
 
-    mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref);
-    if (mon == NULL) {
+    mon = erts_monitor_tree_lookup(ERTS_P_MONITORS(prt), ref);
+    if (!mon || !erts_monitor_is_origin(mon))
 	return driver_term_nil;
-    }
-    ASSERT(mon->type == MON_ORIGIN);
-    to = mon->u.pid;
-    ASSERT(is_internal_pid(to));
-    return (ErlDrvTermData) to;
+
+    ASSERT(is_internal_pid(mon->other.item));
+    return (ErlDrvTermData) mon->other.item;
 }
 
 
@@ -7186,24 +6933,27 @@ int driver_compare_monitors(const ErlDrvMonitor *monitor1,
                       ERTS_REF_THING_SIZE*sizeof(Eterm));
 }
 
-void erts_fire_port_monitor(Port *prt, Eterm ref)
+void erts_fire_port_monitor(Port *prt, ErtsMonitor *tmon)
 {
-    ErtsMonitor *rmon;
+    ErtsMonitorData *mdp;
     void (*callback)(ErlDrvData drv_data, ErlDrvMonitor *monitor);
     ErlDrvMonitor drv_monitor;
     int fpe_was_unmasked;
     ERTS_MSACC_PUSH_STATE_M();
 
     ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
-    ASSERT(prt->drv_ptr != NULL);    
+    ASSERT(prt->drv_ptr != NULL);
+    ASSERT(erts_monitor_is_target(tmon));
+    mdp = erts_monitor_to_data(tmon);
     DRV_MONITOR_LOCK_PDL(prt);
-    if (erts_lookup_monitor(ERTS_P_MONITORS(prt), ref) == NULL) {
+    if (!erts_monitor_is_in_table(&mdp->origin)) {
 	DRV_MONITOR_UNLOCK_PDL(prt);
+        erts_monitor_release(tmon);
 	return;
     }
     callback = prt->drv_ptr->process_exit;
     ASSERT(callback != NULL);
-    erts_ref_to_driver_monitor(ref,&drv_monitor);
+    erts_ref_to_driver_monitor(mdp->ref,&drv_monitor);
     ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT);
     DRV_MONITOR_UNLOCK_PDL(prt);
 #ifdef USE_VM_PROBES
@@ -7227,11 +6977,9 @@ void erts_fire_port_monitor(Port *prt, Eterm ref)
     DRV_MONITOR_LOCK_PDL(prt);
     ERTS_MSACC_POP_STATE_M();
     /* remove monitor *after* callback */
-    rmon = erts_remove_monitor(&ERTS_P_MONITORS(prt), ref);
+    erts_monitor_tree_delete(&ERTS_P_MONITORS(prt), &mdp->origin);
     DRV_MONITOR_UNLOCK_PDL(prt);
-    if (rmon) {
-	erts_destroy_monitor(rmon);
-    }
+    erts_monitor_release_both(mdp);
 }
 
 
@@ -7276,8 +7024,7 @@ driver_failure_term(ErlDrvPort ix, Eterm term, int eof)
 int driver_exit(ErlDrvPort ix, int err)
 {
     Port* prt = erts_drvport2port(ix);
-    Process* rp;
-    ErtsLink *lnk, *rlnk = NULL;
+    ErtsLink *lnk;
     Eterm connected;
 
     ERTS_CHK_NO_PROC_LOCKS;
@@ -7286,22 +7033,10 @@ int driver_exit(ErlDrvPort ix, int err)
         return -1;
 
     connected = ERTS_PORT_GET_CONNECTED(prt);
-    rp = erts_pid2proc(NULL, 0, connected, ERTS_PROC_LOCK_LINK);
-    if (rp) {
-	rlnk = erts_remove_link(&ERTS_P_LINKS(rp),prt->common.id);
-    }
-
-    lnk = erts_remove_link(&ERTS_P_LINKS(prt), connected);
-
-    if (rp)
-	erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-
-    if (rlnk != NULL) {
-	erts_destroy_link(rlnk);
-    }
-
-    if (lnk != NULL) {
-	erts_destroy_link(lnk);
+    lnk = erts_link_tree_lookup(ERTS_P_LINKS(prt), connected);
+    if (lnk) {
+        erts_link_tree_delete(&ERTS_P_LINKS(prt), lnk);
+        erts_proc_sig_send_unlink(NULL, lnk);
     }
 
     if (err == 0)
diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab
index d6d4d2fb49..88d2ef9fa3 100644
--- a/erts/emulator/beam/msg_instrs.tab
+++ b/erts/emulator/beam/msg_instrs.tab
@@ -2,7 +2,7 @@
 //
 // %CopyrightBegin%
 //
-// Copyright Ericsson AB 2017. All Rights Reserved.
+// Copyright Ericsson AB 2017-2018. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -45,23 +45,16 @@
 
 i_recv_mark() {
     /*
-     * Save the current position in message buffer.
+     * Save the current end of message queue
      */
-    c_p->msg.saved_last = c_p->msg.last;
+    ERTS_RECV_MARK_SAVE(c_p);
 }
 
 i_recv_set() {
     /*
-     * If c_p->msg.saved_last is non-zero, it points to the first
-     * message that could possibly be matched out.
-     *
-     * If c_p->msg.saved_last is zero, it means that it was invalidated
-     * because another receive was executed before this i_recv_set()
-     * instruction was reached.
+     * If previously saved recv mark, set peek position to it
      */
-    if (c_p->msg.saved_last) {
-        c_p->msg.save = c_p->msg.saved_last;
-    }
+    ERTS_RECV_MARK_SET(c_p);
     SET_I($NEXT_INSTRUCTION);
     goto loop_rec_top__;
     //| -no_next
@@ -79,7 +72,6 @@ i_loop_rec(Dest) {
 
     /* Entry point from recv_set */
  loop_rec_top__:
-    ;
 
     /*
      * We need to disable GC while matching messages
@@ -89,34 +81,59 @@ i_loop_rec(Dest) {
     ASSERT(!(c_p->flags & F_DELAY_GC));
     c_p->flags |= F_DELAY_GC;
 
-    /* Entry point from loop_rec_end */
+    /* Entry point from loop_rec_end (and locally) */
  loop_rec__:
 
+    if (FCALLS <= 0 && FCALLS <= neg_o_reds) {
+        $SET_CP_I_ABS(I);
+        c_p->flags &= ~F_DELAY_GC;
+        SWAPOUT;
+        c_p->arity = 0;
+        c_p->current = NULL;
+        goto do_schedule;
+    }
+
+    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+
     PROCESS_MAIN_CHK_LOCKS(c_p);
 
     msgp = PEEK_MESSAGE(c_p);
 
-    if (!msgp) {
-        erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-        /* Make sure messages wont pass exit signals... */
-        if (ERTS_PROC_PENDING_EXIT(c_p)) {
-            erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-            SWAPOUT;
-            c_p->flags &= ~F_DELAY_GC;
-            c_p->arity = 0;
-            goto do_schedule; /* Will be rescheduled for exit */
-        }
-        ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
-        msgp = PEEK_MESSAGE(c_p);
-        if (msgp) {
-            erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-        } else {
+    if (ERTS_UNLIKELY(msgp == NULL)) {
+        int get_out;
+        SWAPOUT;
+        FCALLS -= erts_proc_sig_receive_helper(c_p, FCALLS, neg_o_reds,
+                                               &msgp, &get_out);
+        SWAPIN;
+        if (ERTS_UNLIKELY(msgp == NULL)) {
+            if (get_out) {
+                if (get_out < 0) {
+                    ASSERT(FCALLS <= 0 && FCALLS <= neg_o_reds);
+                    goto loop_rec__; /* yield */
+                }
+                else {
+                    ASSERT(ERTS_PROC_IS_EXITING(c_p));
+                    goto do_schedule; /* exit */
+                }
+            }
+
+            /*
+             * If there are no more messages in queue
+             * (and we are not yielding or exiting)
+             * erts_proc_sig_receive_helper()
+             * returns with message queue lock locked...
+             */
             c_p->flags &= ~F_DELAY_GC;
             $SET_I_REL($Dest);
             Goto(*I);		/* Jump to a wait or wait_timeout instruction */
         }
     }
-    if (is_non_value(ERL_MESSAGE_TERM(msgp))) {
+
+    ASSERT(msgp == PEEK_MESSAGE(c_p));
+    ASSERT(msgp && ERTS_SIG_IS_MSG(msgp));
+
+    if (ERTS_UNLIKELY(ERTS_SIG_IS_EXTERNAL_MSG(msgp))) {
+        FCALLS -= 10; /* FIXME: bump appropriate amount... */
         SWAPOUT; /* erts_decode_dist_message() may write to heap... */
         if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) {
             /*
@@ -127,13 +144,16 @@ i_loop_rec(Dest) {
             ASSERT(HTOP == c_p->htop && E == c_p->stop);
             /* TODO: Add DTrace probe for this bad message situation? */
             UNLINK_MESSAGE(c_p, msgp);
-            c_p->msg.saved_last = 0; /* Better safe than sorry. */
             msgp->next = NULL;
             erts_cleanup_messages(msgp);
             goto loop_rec__;
         }
         SWAPIN;
     }
+
+    ASSERT(msgp == PEEK_MESSAGE(c_p));
+    ASSERT(ERTS_SIG_IS_INTERNAL_MSG(msgp));
+
     r(0) = ERL_MESSAGE_TERM(msgp);
 }
 
@@ -252,17 +272,8 @@ loop_rec_end(Dest) {
 
     $SET_I_REL($Dest);
     SAVE_MESSAGE(c_p);
-    if (FCALLS > 0 || FCALLS > neg_o_reds) {
-        FCALLS--;
-        goto loop_rec__;
-    }
-
-    c_p->flags &= ~F_DELAY_GC;
-    $SET_CP_I_ABS(I);
-    SWAPOUT;
-    c_p->arity = 0;
-    c_p->current = NULL;
-    goto do_schedule;
+    FCALLS--;
+    goto loop_rec__;
 }
 
 timeout_locked() {
diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c
index 3323e8640b..74e793577c 100644
--- a/erts/emulator/hipe/hipe_mkliterals.c
+++ b/erts/emulator/hipe/hipe_mkliterals.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -519,8 +519,8 @@ static const struct rts_param rts_params[] = {
 	1, offsetof(struct process, hipe.bif_callee)
 #endif
     },
-    { 49, "P_MSG_FIRST", 1, offsetof(struct process, msg.first) },
-    { 50, "P_MSG_SAVE", 1, offsetof(struct process, msg.save) },
+    { 49, "P_MSG_FIRST", 1, offsetof(struct process, sig_qs.first) },
+    { 50, "P_MSG_SAVE", 1, offsetof(struct process, sig_qs.save) },
     { 51, "P_CALLEE_EXP", 1, offsetof(struct process, hipe.u.callee_exp) },
 
     { 52, "THE_NON_VALUE", 1, (int)THE_NON_VALUE },
@@ -531,8 +531,8 @@ static const struct rts_param rts_params[] = {
 #endif
     },
 
-    { 54, "P_MSG_LAST", 1, offsetof(struct process, msg.last) },
-    { 55, "P_MSG_SAVED_LAST", 1, offsetof(struct process, msg.saved_last) },
+    { 54, "P_MSG_LAST", 1, offsetof(struct process, sig_qs.last) },
+    { 55, "P_MSG_SAVED_LAST", 1, offsetof(struct process, sig_qs.saved_last) },
 };
 
 #define NR_PARAMS	ARRAY_SIZE(rts_params)
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index 8b497c9970..6af514d4ba 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -496,8 +496,10 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
 	      erts_proc_lock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	  p->i = hipe_beam_pc_resume;
 	  p->arity = 0;
-	  erts_atomic32_read_band_relb(&p->state,
-					   ~ERTS_PSFLG_ACTIVE);
+          if (erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_EXITING)
+              ASSERT(erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_ACTIVE);
+          else
+              erts_atomic32_read_band_relb(&p->state, ~ERTS_PSFLG_ACTIVE);
 	  erts_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
       do_schedule:
 	  {
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index 498b43ac6b..cf8c4139be 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -35,6 +35,7 @@
 #include "hipe_native_bif.h"
 #include "hipe_arch.h"
 #include "hipe_stack.h"
+#include "erl_proc_sig_queue.h"
 
 /*
  * These are wrappers for BIFs that may trigger a native
@@ -254,7 +255,7 @@ void hipe_handle_exception(Process *c_p)
     /* Synthesized to avoid having to generate code for it. */
     c_p->def_arg_reg[0] = exception_tag[GET_EXC_CLASS(c_p->freason)];
 
-    c_p->msg.saved_last = 0;  /* No longer safe to use this position */
+    ERTS_RECV_MARK_CLEAR(c_p);  /* No longer safe to use this position */
 
     hipe_find_handler(c_p);
 }
@@ -544,38 +545,57 @@ Eterm hipe_check_get_msg(Process *c_p)
     msgp = PEEK_MESSAGE(c_p);
 
     if (!msgp) {
-	erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-	/* Make sure messages wont pass exit signals... */
-	if (ERTS_PROC_PENDING_EXIT(c_p)) {
-	    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-	    return THE_NON_VALUE; /* Will be rescheduled for exit */
-	}
-	ERTS_MSGQ_MV_INQ2PRIVQ(c_p);
-	msgp = PEEK_MESSAGE(c_p);
-	if (msgp)
-	    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-	else {
-	    /* XXX: BEAM doesn't need this */
-	    c_p->hipe_smp.have_receive_locks = 1;
-	    c_p->flags &= ~F_DELAY_GC;
-	    return THE_NON_VALUE;
-	}
+        int get_out;
+        (void) erts_proc_sig_receive_helper(c_p, CONTEXT_REDS, 0,
+                                            &msgp, &get_out);
+        /* FIXME: Need to bump reductions... */
+        if (!msgp) {
+            if (get_out) {
+                if (get_out < 0) {
+                    /*
+                     * FIXME: We should get out yielding
+                     * here...
+                     */
+                    goto next_message;
+                }
+                /* Go exit... */
+                return THE_NON_VALUE;
+            }
+
+            /*
+             * If there are no more messages in queue
+             * (and we are not yielding or exiting)
+             * erts_proc_sig_receive_helper()
+             * returns with message queue lock locked...
+             */
+
+            /* XXX: BEAM doesn't need this */
+            c_p->hipe_smp.have_receive_locks = 1;
+            c_p->flags &= ~F_DELAY_GC;
+            return THE_NON_VALUE;
+        }
     }
 
-    if (is_non_value(ERL_MESSAGE_TERM(msgp))
-	&& !erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) {
-	/*
-	 * A corrupt distribution message that we weren't able to decode;
-	 * remove it...
-	 */
-	ASSERT(!msgp->data.attached);
-	UNLINK_MESSAGE(c_p, msgp);
-	msgp->next = NULL;
-	erts_cleanup_messages(msgp);
-	goto next_message;
+    ASSERT(msgp == PEEK_MESSAGE(c_p));
+    ASSERT(msgp && ERTS_SIG_IS_MSG(msgp));
+
+    if (ERTS_SIG_IS_EXTERNAL_MSG(msgp)) {
+        /* FIXME: bump appropriate amount... */
+        if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) {
+            /*
+             * A corrupt distribution message that we weren't able to decode;
+             * remove it...
+             */
+            /* TODO: Add DTrace probe for this bad message situation? */
+            UNLINK_MESSAGE(c_p, msgp);
+            msgp->next = NULL;
+            erts_cleanup_messages(msgp);
+            goto next_message;
+        }
     }
 
-    ASSERT(is_value(ERL_MESSAGE_TERM(msgp)));
+    ASSERT(msgp == PEEK_MESSAGE(c_p));
+    ASSERT(ERTS_SIG_IS_INTERNAL_MSG(msgp));
 
     return ERL_MESSAGE_TERM(msgp);
 }
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index f93d4c1557..6d531fdb76 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -885,7 +885,7 @@ enif_select(ErlNifEnv* env,
     ErtsDrvSelectDataState *free_select = NULL;
     ErtsNifSelectDataState *free_nif = NULL;
 
-    ASSERT(!(resource->monitors && resource->monitors->is_dying));
+    ASSERT(!resource->monitors);
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     if (!grow_drv_ev_state(fd)) {
diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl
index e1b42e5d85..d16c6a320d 100644
--- a/erts/emulator/test/bif_SUITE.erl
+++ b/erts/emulator/test/bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2005-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -33,7 +33,8 @@
 	 atom_to_binary/1,min_max/1, erlang_halt/1,
          erl_crash_dump_bytes/1,
 	 is_builtin/1, error_stacktrace/1,
-	 error_stacktrace_during_call_trace/1]).
+	 error_stacktrace_during_call_trace/1,
+         group_leader_prio/1, group_leader_prio_dirty/1]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
@@ -46,7 +47,8 @@ all() ->
      display, display_string, list_to_utf8_atom,
      atom_to_binary, binary_to_atom, binary_to_existing_atom,
      erl_crash_dump_bytes, min_max, erlang_halt, is_builtin,
-     error_stacktrace, error_stacktrace_during_call_trace].
+     error_stacktrace, error_stacktrace_during_call_trace,
+     group_leader_prio, group_leader_prio_dirty].
 
 %% Uses erlang:display to test that erts_printf does not do deep recursion
 display(Config) when is_list(Config) ->
@@ -825,7 +827,6 @@ error_stacktrace_during_call_trace(Config) when is_list(Config) ->
 	end
     end,
     ok.
-    
 
 error_stacktrace_test() ->
     Types = [apply_const_last, apply_const, apply_last,
@@ -963,9 +964,119 @@ do_error_1(call) ->
     erlang:error(id(oops)).
 
 
+group_leader_prio(Config) when is_list(Config) ->
+    group_leader_prio_test(false).
+
+group_leader_prio_dirty(Config) when is_list(Config) ->
+    group_leader_prio_test(true).
+
+group_leader_prio_test(Dirty) ->
+    %%
+    %% Unfortunately back in the days node local group_leader/2 was not
+    %% implemented as sending an asynchronous signal to the process to change
+    %% group leader for. Instead it has always been synchronously changed, and
+    %% nothing in the documentation have hinted otherwise... Therefore I do not
+    %% dare the change this.
+    %%
+    %% In order to prevent priority inversion, the priority of the receiver of
+    %% the group leader signal is elevated while handling incoming signals if
+    %% the sender has a higher priority than the receiver. This test tests that
+    %% the priority elevation actually works...
+    %%
+    Tester = self(),
+    Init = erlang:whereis(init),
+    GL = erlang:group_leader(),
+    process_flag(priority, max),
+    {TestProcFun, NTestProcs}
+        = case Dirty of
+              false ->
+                  %% These processes will handle all incoming signals
+                  %% by them selves...
+                  {fun () ->
+                           Tester ! {alive, self()},
+                           receive after infinity -> ok end
+                   end,
+                   100};
+              true ->
+                  %% These processes wont handle incoming signals by
+                  %% them selves since they are stuck on dirty schedulers
+                  %% when we try to change group leader. A dirty process
+                  %% signal handler process (system process) will be notified
+                  %% of the need to handle incoming signals for these processes,
+                  %% and will instead handle the signal for these processes...
+                  {fun () ->
+                           %% The following sends the message '{alive, self()}'
+                           %% to Tester once on a dirty io scheduler, then wait
+                           %% there until the process terminates...
+                           erts_debug:dirty_io(alive_waitexiting, Tester)
+                   end,
+                   erlang:system_info(dirty_io_schedulers)}
+          end,
+    TPs = lists:map(fun (_) ->
+                            spawn_opt(TestProcFun,
+                                      [link, {priority, normal}])
+                    end, lists:seq(1, NTestProcs)),
+    lists:foreach(fun (TP) -> receive {alive, TP} -> ok end end, TPs),
+    TLs = lists:map(fun (_) ->
+                            spawn_opt(fun () -> tok_loop() end,
+                                      [link, {priority, high}])
+                    end,
+                    lists:seq(1, 2*erlang:system_info(schedulers))),
+    %% Wait to ensure distribution of high prio processes over schedulers...
+    receive after 1000 -> ok end,
+    %%
+    %% Test that we can get group-leader signals through to normal prio
+    %% processes from a max prio process even though all schedulers are filled
+    %% with executing high prio processes.
+    %%
+    lists:foreach(fun (_) ->
+                          lists:foreach(fun (TP) ->
+                                                erlang:yield(),
+                                                %% whitebox -- Enqueue some signals on it
+                                                %% preventing us from hogging its main lock
+                                                %% and set group-leader directly....
+                                                erlang:demonitor(erlang:monitor(process, TP)),
+                                                true = erlang:group_leader(Init, TP),
+                                                {group_leader, Init} = process_info(TP, group_leader),
+                                                erlang:demonitor(erlang:monitor(process, TP)),
+                                                true = erlang:group_leader(GL, TP),
+                                                {group_leader, GL} = process_info(TP, group_leader)
+                                        end,
+                                        TPs)
+                  end,
+                  lists:seq(1,100)),
+    %%
+    %% Also test when it is exiting...
+    %%
+    lists:foreach(fun (TP) ->
+                          erlang:yield(),
+                          M = erlang:monitor(process, TP),
+                          unlink(TP),
+                          exit(TP, bang),
+                          badarg = try
+                                       true = erlang:group_leader(Init, TP)
+                                   catch
+                                       error : What -> What
+                                   end,
+                          receive
+                              {'DOWN', M, process, TP, Reason} ->
+                                  bang = Reason
+                          end
+                  end,
+                  TPs),
+    lists:foreach(fun (TL) ->
+                          M = erlang:monitor(process, TL),
+                          unlink(TL),
+                          exit(TL, bang),
+                          receive
+                              {'DOWN', M, process, TL, Reason} ->
+                                  bang = Reason
+                          end
+                  end,
+                  TLs),
+    ok.
 
-
-%% Helpers
+%% helpers
     
 id(I) -> I.
 
@@ -1005,3 +1116,11 @@ hostname([$@ | Hostname]) ->
     list_to_atom(Hostname);
 hostname([_C | Cs]) ->
     hostname(Cs).
+
+tok_loop() ->
+    tok_loop(hej).
+
+tok_loop(hej) ->
+    tok_loop(hopp);
+tok_loop(hopp) ->
+    tok_loop(hej).
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
index 2a8b999307..a94a2c0b02 100644
--- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
+++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -112,15 +112,12 @@ static ERL_NIF_TERM send_from_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_
 {
     ERL_NIF_TERM result;
     ErlNifPid pid;
-    ErlNifEnv* menv;
     int res;
 
     if (!enif_get_local_pid(env, argv[0], &pid))
 	return enif_make_badarg(env);
     result = enif_make_tuple2(env, enif_make_atom(env, "ok"), enif_make_pid(env, &pid));
-    menv = enif_alloc_env();
-    res = enif_send(env, &pid, menv, result);
-    enif_free_env(menv);
+    res = enif_send(env, &pid, NULL, result);
     if (!res)
 	return enif_make_badarg(env);
     else
@@ -131,15 +128,12 @@ static ERL_NIF_TERM send_wait_from_dirty_nif(ErlNifEnv* env, int argc, const ERL
 {
     ERL_NIF_TERM result;
     ErlNifPid pid;
-    ErlNifEnv* menv;
     int res;
 
     if (!enif_get_local_pid(env, argv[0], &pid))
 	return enif_make_badarg(env);
     result = enif_make_tuple2(env, enif_make_atom(env, "ok"), enif_make_pid(env, &pid));
-    menv = enif_alloc_env();
-    res = enif_send(env, &pid, menv, result);
-    enif_free_env(menv);
+    res = enif_send(env, &pid, NULL, result);
 
 #ifdef __WIN32__
     Sleep(2000);
@@ -211,10 +205,8 @@ dirty_sleeper(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     /* If we get a pid argument, it indicates a process involved in the
        test wants a message from us. Prior to the sleep we send a 'ready'
        message, and then after the sleep, send a 'done' message. */
-    if (argc == 1 && enif_get_local_pid(env, argv[0], &pid)) {
-        msg_env = enif_alloc_env();
-        enif_send(env, &pid, msg_env, enif_make_atom(msg_env, "ready"));
-    }
+    if (argc == 1 && enif_get_local_pid(env, argv[0], &pid))
+        enif_send(env, &pid, NULL, enif_make_atom(env, "ready"));
 
 #ifdef __WIN32__
     Sleep(2000);
@@ -222,11 +214,8 @@ dirty_sleeper(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     sleep(2);
 #endif
 
-    if (argc == 1) {
-        assert(msg_env != NULL);
-        enif_send(env, &pid, msg_env, enif_make_atom(msg_env, "done"));
-        enif_free_env(msg_env);
-    }
+    if (argc == 1)
+        enif_send(env, &pid, NULL, enif_make_atom(env, "done"));
 
     return enif_make_atom(env, "ok");
 }
@@ -247,8 +236,8 @@ static ERL_NIF_TERM dirty_call_while_terminated_nif(ErlNifEnv* env, int argc, co
 
     self_term = enif_make_pid(env, &self);
 
-    result = enif_make_tuple2(env, enif_make_atom(env, "dirty_alive"), self_term);
     menv = enif_alloc_env();
+    result = enif_make_tuple2(menv, enif_make_atom(menv, "dirty_alive"), self_term);
     res = enif_send(env, &to, menv, result);
     enif_free_env(menv);
     if (!res)
@@ -259,9 +248,7 @@ static ERL_NIF_TERM dirty_call_while_terminated_nif(ErlNifEnv* env, int argc, co
 	;
 
     result = enif_make_tuple2(env, enif_make_atom(env, "dirty_dead"), self_term);
-    menv = enif_alloc_env();
-    res = enif_send(env, &to, menv, result);
-    enif_free_env(menv);
+    res = enif_send(env, &to, NULL, result);
 
 #ifdef __WIN32__
     Sleep(1000);
diff --git a/erts/emulator/test/erl_link_SUITE.erl b/erts/emulator/test/erl_link_SUITE.erl
index a66ca7a57d..ed444f2599 100644
--- a/erts/emulator/test/erl_link_SUITE.erl
+++ b/erts/emulator/test/erl_link_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2001-2017. All Rights Reserved.
+%% Copyright Ericsson AB 2001-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -53,26 +53,17 @@
 -export([test_proc/0]).
 
 
--define(LINK_UNDEF, 0).
--define(LINK_PID,   1).
--define(LINK_NODE,  3).
-
-
-% These are to be kept in sync with erl_monitors.h 
--define(MON_ORIGIN, 1).
--define(MON_TARGET, 2).
-
-
--record(erl_link, {type = ?LINK_UNDEF,
+-record(erl_link, {type,           % process | port | dist_process
                    pid = [],
-                   targets = []}).
+                   id}).
 
 % This is to be kept in sync with erl_bif_info.c (make_monitor_list)
 
--record(erl_monitor, {type,        % MON_ORIGIN or MON_TARGET
-                      ref,
+-record(erl_monitor, {type,        % process | port | time_offset | dist_process | resource | node | nodes | suspend
+                      dir,         % origin | target
+                      ref,         % Reference | []
                       pid,         % Process or nodename
-                      name = []}). % registered name or []
+                      extra = []}). % registered name, integer or, []
 
 
 suite() ->
@@ -106,7 +97,7 @@ end_per_suite(_Config) ->
 links(Config) when is_list(Config) ->
     common_link_test(node(), node()),
     true = link(self()),
-    [] = find_erl_link(self(), ?LINK_PID, self()),
+    [] = find_erl_link(self(), process, self()),
     true = unlink(self()),
     ok.
 
@@ -191,6 +182,7 @@ monitor_nodes(Config) when is_list(Config) ->
     monitor_node(A, false),
     monitor_node(B, true),
     monitor_node(C, true),
+    receive after 1000 -> ok end,
     monitor_node(C, false),
     monitor_node(C, true),
     monitor_node(B, true),
@@ -304,7 +296,8 @@ run_common_process_monitors(TP1, TP2) ->
     wait_until(fun () -> is_proc_dead(TP2) end),
     ok = tp_call(TP1, fun () ->
                               receive
-                                  {'DOWN',R2,process,TP2O,bye} ->
+                                  {'DOWN',R2,process,TP2O,Reason1} ->
+                                      bye = Reason1,
                                       ok
                               end
                       end),
@@ -313,7 +306,8 @@ run_common_process_monitors(TP1, TP2) ->
     R3 = tp_call(TP1, fun () -> erlang:monitor(process, TP2) end),
     ok = tp_call(TP1, fun () ->
                               receive
-                                  {'DOWN',R3,process,TP2O,noproc} ->
+                                  {'DOWN',R3,process,TP2O,Reason2} ->
+                                      noproc = Reason2,
                                       ok
                               end
                       end),
@@ -698,22 +692,10 @@ test_proc() ->
     end,
     test_proc().
 
-expand_link_list([#erl_link{type = ?LINK_NODE, targets = N} = Rec | T]) ->
-    lists:duplicate(N,Rec#erl_link{targets = []}) ++ expand_link_list(T);
-expand_link_list([#erl_link{targets = [#erl_link{pid = Pid}]} = Rec | T]) ->
-    [Rec#erl_link{targets = [Pid]} | expand_link_list(T)];
-expand_link_list([#erl_link{targets = [#erl_link{pid = Pid}|TT]} = Rec | T]) ->
-    [ Rec#erl_link{targets = [Pid]} | expand_link_list( 
-                                        [Rec#erl_link{targets = TT} | T])]; 
-expand_link_list([#erl_link{targets = []} = Rec | T]) ->
-    [Rec | expand_link_list(T)];
-expand_link_list([]) ->
-    [].
-
 get_local_link_list(Obj) ->
     case catch erts_debug:get_internal_state({link_list, Obj}) of
         LL when is_list(LL) ->
-            expand_link_list(LL);
+            LL;
         _ ->
             []
     end.
@@ -722,7 +704,7 @@ get_remote_link_list(Node, Obj) ->
     case catch rpc:call(Node, erts_debug, get_internal_state,
                         [{link_list, Obj}]) of
         LL when is_list(LL) ->
-            expand_link_list(LL);
+            LL;
         _ ->
             []
     end.
@@ -776,82 +758,106 @@ get_monitor_list(undefined) ->
 
 
 find_erl_monitor(Pid, Ref) when is_reference(Ref) ->
+    MonitorList = get_monitor_list(Pid),
+    io:format("~p MonitorList: ~p~n", [Pid, MonitorList]),
     lists:foldl(fun (#erl_monitor{ref = R} = EL, Acc) when R == Ref ->
                         [EL|Acc];
                     (_, Acc) ->
                         Acc
                 end,
                 [],
-                get_monitor_list(Pid)).
-
-% find_erl_link(Obj, Ref) when is_reference(Ref) -> 
-%     lists:foldl(fun (#erl_link{ref = R} = EL, Acc) when R == Ref ->
-% 			      [EL|Acc];
-% 			  (_, Acc) ->
-% 			      Acc
-% 		      end,
-% 		      [],
-% 		      get_link_list(Obj)).
-
-find_erl_link(Obj, Type, [Item, Data]) when is_pid(Item);
-                                            is_port(Item);
-                                            is_atom(Item) -> 
-    lists:foldl(fun (#erl_link{type = T, pid = I, targets = D} = EL,
+                MonitorList);
+find_erl_monitor(Pid, Item) ->
+    MonitorList = get_monitor_list(Pid),
+    io:format("~p MonitorList: ~p~n", [Pid, MonitorList]),
+    lists:foldl(fun (#erl_monitor{pid = I} = EL, Acc) when I == Item ->
+                        [EL|Acc];
+                    (_, Acc) ->
+                        Acc
+                end,
+                [],
+                MonitorList).
+
+
+find_erl_link(Obj, Type, Item) when is_pid(Item); is_port(Item) -> 
+    LinkList = get_link_list(Obj),
+    io:format("~p LinkList: ~p~n", [Obj, LinkList]),
+    lists:foldl(fun (#erl_link{type = T, pid = I} = EL,
                      Acc) when T == Type, I == Item ->
-                        case Data of
-                            D ->
-                                [EL|Acc];
-                            [] ->
-                                [EL|Acc];
-                            _ ->
-                                Acc
-                        end;
+                        [EL|Acc];
+                    (_, Acc) ->
+                        Acc
+                end,
+                [],
+                LinkList);
+find_erl_link(Obj, Type, Id) when is_integer(Id) -> 
+    %% Find by Id
+    LinkList = get_link_list(Obj),
+    io:format("~p LinkList: ~p~n", [Obj, LinkList]),
+    lists:foldl(fun (#erl_link{type = T, id = I} = EL,
+                     Acc) when T == Type, I == Id ->
+                        [EL|Acc];
                     (_, Acc) ->
                         Acc
                 end,
                 [],
-                get_link_list(Obj));
-find_erl_link(Obj, Type, Item) when is_pid(Item); is_port(Item); is_atom(Item) ->
-    find_erl_link(Obj, Type, [Item, []]). 
+                LinkList).
 
 
+get_link_type(A, B) when is_port(A);
+                         is_port(B) ->
+    port;
+get_link_type(A, B) when is_pid(A),
+                         is_pid(B) ->
+    case node(A) == node(B) of
+        true ->
+            process;
+        false ->
+            dist_process
+    end.
 
+check_link(A, B) when node(A) == node(B) ->
+    LinkType = get_link_type(A, B),
+    [#erl_link{type = LinkType,
+               pid = B,
+               id = Id}] = find_erl_link(A, LinkType, B),
+    [#erl_link{type = LinkType,
+               pid = A,
+               id = Id}] = find_erl_link(B, LinkType, A),
+    [] = find_erl_link({node(A), node(B)},
+                       LinkType,
+                       A),
+    [] = find_erl_link({node(B), node(A)},
+                       LinkType,
+                       B),
+    ok;
 check_link(A, B) ->
-    [#erl_link{type = ?LINK_PID,
+    [#erl_link{type = dist_process,
                pid = B,
-               targets = []}] = find_erl_link(A, ?LINK_PID, B),
-    [#erl_link{type = ?LINK_PID,
+               id = IdA}] = find_erl_link(A, dist_process, B),
+    [#erl_link{type = dist_process,
                pid = A,
-               targets = []}] = find_erl_link(B, ?LINK_PID, A),
-    case node(A) == node(B) of
-        false ->
-            [#erl_link{type = ?LINK_PID,
-                       pid = A,
-                       targets = [B]}] = find_erl_link({node(A),
-                                                        node(B)},
-                                                       ?LINK_PID,
-                                                       [A, [B]]),
-            [#erl_link{type = ?LINK_PID,
-                       pid = B,
-                       targets = [A]}] = find_erl_link({node(B),
-                                                        node(A)},
-                                                       ?LINK_PID,
-                                                       [B, [A]]);
-        true ->
-            [] = find_erl_link({node(A), node(B)},
-                               ?LINK_PID,
-                               [A, [B]]),
-            [] = find_erl_link({node(B), node(A)},
-                               ?LINK_PID,
-                               [B, [A]])
-    end,
+               id = IdA}] = find_erl_link({node(A),
+                                           node(B)},
+                                          dist_process,
+                                          IdA),
+    [#erl_link{type = dist_process,
+               pid = A,
+               id = IdB}] = find_erl_link(B, dist_process, A),
+    [#erl_link{type = dist_process,
+               pid = B,
+               id = IdB}] = find_erl_link({node(B),
+                                           node(A)},
+                                          dist_process,
+                                          IdB),
     ok.
 
 check_unlink(A, B) ->
-    [] = find_erl_link(A, ?LINK_PID, B),
-    [] = find_erl_link(B, ?LINK_PID, A),
-    [] = find_erl_link({node(A), node(B)}, ?LINK_PID, [A, [B]]),
-    [] = find_erl_link({node(B), node(A)}, ?LINK_PID, [B, [A]]),
+    LinkType = get_link_type(A, B),
+    [] = find_erl_link(A, LinkType, B),
+    [] = find_erl_link(B, LinkType, A),
+    [] = find_erl_link({node(A), node(B)}, dist_process, A),
+    [] = find_erl_link({node(B), node(A)}, dist_process, B),
     ok.
 
 check_process_monitor(From, {Name, Node}, Ref) when is_pid(From),
@@ -864,22 +870,26 @@ check_process_monitor(From, {Name, Node}, Ref) when is_pid(From),
                                                     is_atom(Node),
                                                     is_reference(Ref) ->
     MonitoredPid = rpc:call(Node, erlang, whereis, [Name]),
-    [#erl_monitor{type = ?MON_ORIGIN,
+    [#erl_monitor{type = dist_process,
+                  dir = origin,
                   ref = Ref,
                   pid = Node,
-                  name = Name}] = find_erl_monitor(From, Ref),
-    [#erl_monitor{type = ?MON_TARGET,
+                  extra = Name}] = find_erl_monitor(From, Ref),
+    [#erl_monitor{type = dist_process,
+                  dir = target,
                   ref = Ref,
                   pid = From,
-                  name = Name}] = 	find_erl_monitor({node(From), Node}, Ref),
-    [#erl_monitor{type = ?MON_ORIGIN,
+                  extra = Name}] = find_erl_monitor({node(From), Node}, Ref),
+    [#erl_monitor{type = dist_process,
+                  dir = origin,
                   ref = Ref,
                   pid = MonitoredPid,
-                  name = Name}] = find_erl_monitor({Node, node(From)}, Ref),
-    [#erl_monitor{type = ?MON_TARGET,
+                  extra = Name}] = find_erl_monitor({Node, node(From)}, Ref),
+    [#erl_monitor{type = dist_process,
+                  dir = target,
                   ref = Ref,
                   pid = From,
-                  name = Name}] = find_erl_monitor(MonitoredPid, Ref),
+                  extra = Name}] = find_erl_monitor(MonitoredPid, Ref),
     ok;
 check_process_monitor(From, Name, Ref) when is_pid(From),
                                             is_atom(Name),
@@ -887,27 +897,36 @@ check_process_monitor(From, Name, Ref) when is_pid(From),
                                             is_reference(Ref) ->
     MonitoredPid = rpc:call(node(From), erlang, whereis, [Name]),
 
-    [#erl_monitor{type = ?MON_ORIGIN,
+    [#erl_monitor{type = process,
+                  dir = origin,
                   ref = Ref,
                   pid = MonitoredPid,
-                  name = Name}] = find_erl_monitor(From, Ref),
+                  extra = Name}] = find_erl_monitor(From, Ref),
 
 
-    [#erl_monitor{type = ?MON_TARGET,
+    [#erl_monitor{type = process,
+                  dir = target,
                   ref = Ref,
                   pid = From,
-                  name = Name}] = find_erl_monitor(MonitoredPid,Ref),
+                  extra = Name}] = find_erl_monitor(MonitoredPid,Ref),
     ok;
 check_process_monitor(From, To, Ref) when is_pid(From),
                                           is_pid(To),
                                           is_reference(Ref) ->
-    OriMon = [#erl_monitor{type = ?MON_ORIGIN,
+    MonType = case node(From) == node(To) of
+                  true -> process;
+                  false -> dist_process
+              end,
+
+    OriMon = [#erl_monitor{type = MonType,
+                           dir = origin,
                            ref = Ref,
                            pid = To}],
 
     OriMon = find_erl_monitor(From, Ref), 
 
-    TargMon = [#erl_monitor{type = ?MON_TARGET,
+    TargMon = [#erl_monitor{type = MonType,
+                            dir = target,
                             ref = Ref,
                             pid = From}],
     TargMon = find_erl_monitor(To, Ref),
@@ -915,7 +934,11 @@ check_process_monitor(From, To, Ref) when is_pid(From),
 
     case node(From) == node(To) of
         false ->
-            TargMon = find_erl_monitor({node(From), node(To)}, Ref),
+            DistTargMon = [#erl_monitor{type = dist_process,
+                                        dir = target,
+                                        ref = Ref,
+                                        pid = From}],
+            DistTargMon = find_erl_monitor({node(From), node(To)}, Ref),
             OriMon = find_erl_monitor({node(To), node(From)}, Ref);
         true ->
             [] = find_erl_monitor({node(From), node(From)}, Ref)
@@ -986,19 +1009,36 @@ check_process_demonitor(From, To, Ref) when is_pid(From),
     ok.
 
 no_of_monitor_node(From, Node) when is_pid(From), is_atom(Node) ->
-    length(find_erl_link(From, ?LINK_NODE, Node)).
+    case find_erl_monitor(From, Node) of
+        [] -> 0;
+        [#erl_monitor{type = node,
+                      dir = origin,
+                      pid = Node,
+                      extra = N}] -> N
+    end.
 
+check_monitor_node(From, Node, 0) when is_pid(From),
+                                        is_atom(Node) ->
+    [] = find_erl_monitor(From, Node),
+    [] = find_erl_monitor({node(From), Node}, From);
 check_monitor_node(From, Node, No) when is_pid(From),
                                         is_atom(Node),
                                         is_integer(No),
-                                        No >= 0 ->
-    LL = lists:duplicate(No, #erl_link{type = ?LINK_NODE, pid = Node}),
-    DLL = lists:duplicate(No, #erl_link{type = ?LINK_NODE, pid = From}),
-    LL = find_erl_link(From, ?LINK_NODE, Node),
-    DLL = find_erl_link({node(From), Node}, ?LINK_NODE, From),
-    ok.
-
-
+                                        No > 0 ->
+    [#erl_monitor{type = node,
+                  dir = origin,
+                  pid = Node,
+                  extra = No}] = find_erl_monitor(From, Node),
+    [#erl_monitor{type = node,
+                  dir = target,
+                  pid = From}] = find_erl_monitor({node(From), Node}, From).
+
+connection_id(Node) ->
+    try
+        erts_debug:get_internal_state({connection_id, Node})
+    catch
+        _:_ -> -1
+    end.
 
 hostname() ->
     from($@, atom_to_list(node())).
diff --git a/erts/emulator/test/monitor_SUITE.erl b/erts/emulator/test/monitor_SUITE.erl
index 9d772480d9..c7250a9d26 100644
--- a/erts/emulator/test/monitor_SUITE.erl
+++ b/erts/emulator/test/monitor_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2017. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -86,7 +86,7 @@ case_2(Config) when is_list(Config) ->
     R = erlang:monitor(process, B),
     B ! R,
     receive
-        {'EXIT', _} -> ok;
+        true -> ok;
         Other ->
             ct:fail({rec, Other})
     end,
@@ -98,7 +98,7 @@ case_2a(Config) when is_list(Config) ->
     {B,R} = spawn_monitor(?MODULE, y2, [self()]),
     B ! R,
     receive
-        {'EXIT', _} -> ok;
+        true -> ok;
         Other ->
             ct:fail({rec, Other})
     end,
@@ -182,7 +182,7 @@ demon_e_1(Config) when is_list(Config) ->
            end ),
     receive 
         {P2, ref, R2} -> 
-            demon_error(R2, badarg),
+            true = erlang:demonitor(R2),
             P2 ! {self(), stop};
         Other2 ->
             ct:fail({rec, Other2})
@@ -729,8 +729,8 @@ named_down(Config) when is_list(Config) ->
                            end),
     ?assertEqual(true, register(Name, NamedProc)),
     unlink(NamedProc),
-    exit(NamedProc, bang),
     Mon = erlang:monitor(process, Name),
+    exit(NamedProc, bang),
     receive {'DOWN',Mon, _, _, bang} -> ok
     after 3000 -> ?assert(false) end,
     ?assertEqual(true, register(Name, self())),
diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index e46665a881..5b39d05df8 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -109,7 +109,6 @@
     mon_port_pid_demonitor/1,
     mon_port_remote_on_remote/1,
     mon_port_driver_die/1,
-    mon_port_driver_die_demonitor/1,
     mul_basic/1,
     mul_slow_writes/1,
     name1/1,
@@ -180,8 +179,7 @@ all() ->
      mon_port_bad_named,
      mon_port_pid_demonitor,
      mon_port_name_demonitor,
-     mon_port_driver_die,
-     mon_port_driver_die_demonitor
+     mon_port_driver_die
     ].
 
 groups() ->
@@ -2783,7 +2781,7 @@ mon_port_driver_die(Config) ->
     end,
     ok.
 
-
+-ifdef(DISABLED_TESTCASE).
 %% 1. Spawn a port which will sleep 3 seconds
 %% 2. Monitor port
 %% 3. Port driver and dies horribly (via C driver_failure call). This should
@@ -2819,6 +2817,7 @@ mon_port_driver_die_demonitor(Config) ->
     after 5000 -> ?assert(false)
     end,
     ok.
+-endif.
 
 %% @doc Makes a controllable port for testing. Underlying mechanism of this
 %% port is not important, only important is our ability to close/kill it or
diff --git a/erts/emulator/test/port_trace_SUITE.erl b/erts/emulator/test/port_trace_SUITE.erl
index a1986397a8..eba8f194e0 100644
--- a/erts/emulator/test/port_trace_SUITE.erl
+++ b/erts/emulator/test/port_trace_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1999-2017. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -202,8 +202,7 @@ ports(_Config) ->
     erlang:port_close(Prt),
 
     [{trace,Prt,closed,normal},
-     {trace,Prt,unregister,port_trace_SUITE},
-     {trace,Prt,unlink,S}] = flush(),
+     {trace,Prt,unregister,port_trace_SUITE}] = flush(),
 
     ok.
 
@@ -475,8 +474,7 @@ failure_test(Failure, Reason) ->
         process_flag(trap_exit, false)
     end,
     [{trace, Prt, 'receive', {S, {command, Failure}}},
-     {trace, Prt, closed, Reason},
-     {trace, Prt, unlink, S}] = flush(),
+     {trace, Prt, closed, Reason}] = flush(),
 
     ok.
 
@@ -599,13 +597,11 @@ close(Prt, Flags) ->
 
     if Recv, Ports ->
             [{trace, Prt, 'receive', {S, close}},
-             {trace, Prt, closed, normal},
-             {trace, Prt, unlink, S}] = flush();
+             {trace, Prt, closed, normal}] = flush();
        Recv ->
             [{trace, Prt, 'receive', {S, close}}] = flush();
        Ports ->
-            [{trace, Prt, closed, normal},
-             {trace, Prt, unlink, S}] = flush();
+            [{trace, Prt, closed, normal}] = flush();
        true ->
             [] = flush()
     end.
diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index a8bcfac84d..7eff786e8b 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -43,7 +43,6 @@
 	 process_info_lock_reschedule3/1,
          process_info_garbage_collection/1,
 	 bump_reductions/1, low_prio/1, binary_owner/1, yield/1, yield2/1,
-	 process_status_exiting/1,
 	 otp_4725/1, bad_register/1, garbage_collect/1, otp_6237/1,
 	 process_info_messages/1, process_flag_badarg/1, process_flag_heap_size/1,
 	 spawn_opt_heap_size/1, spawn_opt_max_heap_size/1,
@@ -80,7 +79,6 @@ all() ->
      process_info_lock_reschedule2,
      process_info_lock_reschedule3,
      process_info_garbage_collection,
-     process_status_exiting,
      bump_reductions, low_prio, yield, yield2, otp_4725,
      bad_register, garbage_collect, process_info_messages,
      process_flag_badarg, process_flag_heap_size,
@@ -841,28 +839,6 @@ process_info_lock_reschedule3(Config) when is_list(Config) ->
 		  ct:fail(BadStatus)
 	  end.
 
-process_status_exiting(Config) when is_list(Config) ->
-    %% Make sure that erts_debug:get_internal_state({process_status,P})
-    %% returns exiting if it is in status P_EXITING.
-    erts_debug:set_internal_state(available_internal_state,true),
-    Prio = process_flag(priority, max),
-    P = spawn_opt(fun () -> receive after infinity -> ok end end,
-			[{priority, normal}]),
-    erlang:yield(),
-    %% The tok_loop processes are here to make it hard for the exiting
-    %% process to be scheduled in for exit...
-    TokLoops = lists:map(fun (_) ->
-		spawn_opt(fun tok_loop/0,
-		    [link,{priority, high}])
-	end, lists:seq(1, erlang:system_info(schedulers_online))),
-    exit(P, boom),
-    wait_until(fun() ->
-		exiting =:= erts_debug:get_internal_state({process_status,P})
-	end),
-    lists:foreach(fun (Tok) -> unlink(Tok), exit(Tok,bang) end, TokLoops),
-    process_flag(priority, Prio),
-    ok.
-
 otp_4725(Config) when is_list(Config) ->
     Tester = self(),
     Ref1 = make_ref(),
@@ -1000,7 +976,7 @@ gv(Key,List) ->
 %% Tests erlang:bump_reductions/1.
 bump_reductions(Config) when is_list(Config) ->
     erlang:garbage_collect(),
-    receive after 1 -> ok end,		% Clear reductions.
+    erlang:yield(),		% Clear reductions.
     {reductions,R1} = process_info(self(), reductions),
     true = erlang:bump_reductions(100),
     {reductions,R2} = process_info(self(), reductions),
diff --git a/erts/emulator/test/signal_SUITE.erl b/erts/emulator/test/signal_SUITE.erl
index 61a8617165..fab2f45f28 100644
--- a/erts/emulator/test/signal_SUITE.erl
+++ b/erts/emulator/test/signal_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2017. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -34,23 +34,9 @@
 -export([init_per_testcase/2, end_per_testcase/2]).
 
 % Test cases
--export([xm_sig_order/1,
-	 pending_exit_unlink_process/1,
-	 pending_exit_unlink_dist_process/1,
-	 pending_exit_unlink_port/1,
-	 pending_exit_trap_exit/1,
-	 pending_exit_receive/1,
-	 pending_exit_exit/1,
-	 pending_exit_gc/1,
-	 pending_exit_is_process_alive/1,
-	 pending_exit_process_display/1,
-	 pending_exit_process_info_1/1,
-	 pending_exit_process_info_2/1,
-	 pending_exit_group_leader/1,
-	 exit_before_pending_exit/1]).
+-export([xm_sig_order/1]).
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
-    available_internal_state(true),
     [{testcase, Func}|Config].
 
 end_per_testcase(_Func, _Config) ->
@@ -60,24 +46,14 @@ init_per_suite(Config) ->
     Config.
 
 end_per_suite(_Config) ->
-    available_internal_state(true),
-    catch erts_debug:set_internal_state(not_running_optimization, true),
-    available_internal_state(false).
+    ok.
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
      {timetrap, {minutes, 2}}].
 
 all() -> 
-    [xm_sig_order, pending_exit_unlink_process,
-     pending_exit_unlink_dist_process,
-     pending_exit_unlink_port, pending_exit_trap_exit,
-     pending_exit_receive, pending_exit_trap_exit,
-     pending_exit_gc, pending_exit_is_process_alive,
-     pending_exit_process_display,
-     pending_exit_process_info_1,
-     pending_exit_process_info_2, pending_exit_group_leader,
-     exit_before_pending_exit].
+    [xm_sig_order].
 
 
 %% Test that exit signals and messages are received in correct order
@@ -113,362 +89,9 @@ xm_sig_order_proc() ->
     end,
     xm_sig_order_proc().
 
-pending_exit_unlink_process(Config) when is_list(Config) ->
-    pending_exit_test(self(), unlink).
-
-pending_exit_unlink_dist_process(Config) when is_list(Config) ->
-    {ok, Node} = start_node(Config),
-    From = spawn(Node, fun () -> receive after infinity -> ok end end),
-    Res = pending_exit_test(From, unlink),
-    stop_node(Node),
-    Res.
-
-pending_exit_unlink_port(Config) when is_list(Config) ->
-    pending_exit_test(hd(erlang:ports()), unlink).
-
-pending_exit_trap_exit(Config) when is_list(Config) ->
-    pending_exit_test(self(), trap_exit).
-
-pending_exit_receive(Config) when is_list(Config) ->
-    pending_exit_test(self(), 'receive').
-
-pending_exit_exit(Config) when is_list(Config) ->
-    pending_exit_test(self(), exit).
-
-pending_exit_gc(Config) when is_list(Config) ->
-    pending_exit_test(self(), gc).
-
-pending_exit_test(From, Type) ->
-    OTE = process_flag(trap_exit, true),
-    Ref = make_ref(),
-    Master = self(),
-    ExitBySignal = case Type of
-                       gc ->
-                           lists:duplicate(10000,
-                                           exit_by_signal);
-                       _ ->
-                           exit_by_signal
-                   end,
-    Pid = spawn_link(
-            fun () ->
-                    receive go -> ok end,
-                    false = have_pending_exit(),
-                    exit = fake_exit(From,
-                                     self(),
-                                     ExitBySignal),
-                    true = have_pending_exit(),
-                    Master ! {self(), Ref, Type},
-                    case Type of
-                        gc ->
-                            force_gc(),
-                            erlang:yield();
-                        unlink ->
-                            unlink(From);
-                        trap_exit ->
-                            process_flag(trap_exit, true);
-                        'receive' ->
-                            receive _ -> ok
-                            after 0 -> ok
-                            end;
-                        exit ->
-                            ok
-                    end,
-                    exit(exit_by_myself)
-            end),
-    Mon = erlang:monitor(process, Pid),
-    Pid ! go,
-    Reason = receive
-                 {'DOWN', Mon, process, Pid, R} ->
-                     receive
-                         {Pid, Ref, Type} ->
-                             ok
-                     after 0 ->
-                             ct:fail(premature_exit)
-                     end,
-                     case Type of
-                         exit ->
-                             exit_by_myself = R;
-                         _ ->
-                             ExitBySignal = R
-                     end
-             end,
-    receive
-        {'EXIT', Pid, R2} ->
-            Reason = R2
-    end,
-    process_flag(trap_exit, OTE),
-    ok,
-    {comment, "Test only valid with current SMP emulator."}.
-
-
-
-exit_before_pending_exit(Config) when is_list(Config) ->
-    %% This is a testcase testcase very specific to the smp
-    %% implementation as it is of the time of writing.
-    %%
-    %% The testcase tries to check that a process can
-    %% exit by itself even though it has a pending exit.
-    OTE = process_flag(trap_exit, true),
-    Master = self(),
-    Tester = spawn_link(
-               fun () ->
-                       Opts = case {erlang:system_info(run_queues),
-                                    erlang:system_info(schedulers_online)} of
-                                  {RQ, SO} when RQ =:= 1; SO =:= 1 -> [];
-                                  _ ->
-                                      process_flag(scheduler, 1),
-                                      [{scheduler, 2}]
-                              end,
-                       P = self(),
-                       Exiter = spawn_opt(fun () ->
-                                                  receive
-                                                      {exit_me, P, R} ->
-                                                          exit(P, R)
-                                                  end
-                                          end, Opts),
-                       erlang:yield(),
-                       Exiter ! {exit_me, self(), exited_by_exiter},
-                       %% We want to get a pending exit
-                       %% before we exit ourselves. We
-                       %% don't want to be scheduled out
-                       %% since we will then see the
-                       %% pending exit.
-                       %%
-                       %% Do something that takes
-                       %% relatively long time but
-                       %% consumes few reductions...
-                       repeat(fun() -> erlang:system_info(procs) end,10),
-                       %% ... then exit.
-                       Master ! {self(),
-                                 pending_exit,
-                                 have_pending_exit()},
-                       exit(exited_by_myself)
-               end),
-    PendingExit = receive {Tester, pending_exit, PE} -> PE end,
-    receive
-	      {'EXIT', Tester, exited_by_myself} ->
-		  process_flag(trap_exit, OTE),
-		  ok;
-	      Msg ->
-		  ct:fail({unexpected_message, Msg})
-	  end,
-    NoScheds = integer_to_list(erlang:system_info(schedulers_online)),
-    {comment,
-     "Was "
-     ++ case PendingExit of
-	    true -> "";
-	    false ->"*not*"
-	end ++ " able to trigger a pending exit. "
-     ++ "Running on " ++ NoScheds ++ " scheduler(s). "
-     ++ "This test is only interesting with at least two schedulers."}.
-
--define(PE_INFO_REPEAT, 100).
-
-pending_exit_is_process_alive(Config) when is_list(Config) ->
-    S = exit_op_test_init(),
-    TestFun = fun (P) -> false = is_process_alive(P) end,
-    repeated_exit_op_test(TestFun, ?PE_INFO_REPEAT),
-    verify_pending_exit_success(S),
-    comment().
-
-pending_exit_process_info_1(Config) when is_list(Config) ->
-    S = exit_op_test_init(),
-    TestFun = fun (P) ->
-			    undefined = process_info(P)
-		    end,
-    repeated_exit_op_test(TestFun, ?PE_INFO_REPEAT),
-    verify_pending_exit_success(S),
-    comment().
-
-pending_exit_process_info_2(Config) when is_list(Config) ->
-    S0 = exit_op_test_init(),
-    repeated_exit_op_test(fun (P) ->
-					undefined = process_info(P, messages)
-				end, ?PE_INFO_REPEAT),
-    S1 = verify_pending_exit_success(S0),
-    repeated_exit_op_test(fun (P) ->
-					undefined = process_info(P, status)
-				end, ?PE_INFO_REPEAT),
-    S2 = verify_pending_exit_success(S1),
-    repeated_exit_op_test(fun (P) ->
-					undefined = process_info(P, links)
-				end, ?PE_INFO_REPEAT),
-    S3 = verify_pending_exit_success(S2),
-    repeated_exit_op_test(fun (P) ->
-					undefined = process_info(P, [messages])
-				end, ?PE_INFO_REPEAT),
-    S4 = verify_pending_exit_success(S3),
-    repeated_exit_op_test(fun (P) ->
-					undefined = process_info(P, [status])
-				end, ?PE_INFO_REPEAT),
-    S5 = verify_pending_exit_success(S4),
-    repeated_exit_op_test(fun (P) ->
-					undefined = process_info(P, [links])
-				end, ?PE_INFO_REPEAT),
-    S6 = verify_pending_exit_success(S5),
-    repeated_exit_op_test(fun (P) ->
-					undefined = process_info(P, [status,
-								     links])
-				end, ?PE_INFO_REPEAT),
-    S7 = verify_pending_exit_success(S6),
-    repeated_exit_op_test(fun (P) ->
-					undefined = process_info(P, [messages,
-								     status])
-				end, ?PE_INFO_REPEAT),
-    S8 = verify_pending_exit_success(S7),
-    repeated_exit_op_test(fun (P) ->
-					undefined = process_info(P, [messages,
-								     links])
-				end, ?PE_INFO_REPEAT),
-    S9 = verify_pending_exit_success(S8),
-    repeated_exit_op_test(
-	    fun (P) ->
-		    undefined = process_info(P, [message_queue_len,
-						 status])
-	    end, ?PE_INFO_REPEAT),
-    S10 = verify_pending_exit_success(S9),
-    repeated_exit_op_test(fun (P) ->
-					undefined = process_info(P, [messages,
-								     links,
-								     status])
-				end, ?PE_INFO_REPEAT),
-    verify_pending_exit_success(S10),
-    comment().
-
-pending_exit_process_display(Config) when is_list(Config) ->
-    S = exit_op_test_init(),
-    TestFun = fun (P) ->
-			    badarg = try
-					 erlang:process_display(P, backtrace)
-				     catch
-					 error:badarg -> badarg
-				     end
-		    end,
-    repeated_exit_op_test(TestFun, ?PE_INFO_REPEAT),
-    verify_pending_exit_success(S),
-    comment().
-
-pending_exit_group_leader(Config) when is_list(Config) ->
-    S = exit_op_test_init(),
-    TestFun = fun (P) ->
-			    badarg = try
-					 group_leader(self(), P)
-				     catch
-					 error:badarg -> badarg
-				     end
-		    end,
-    repeated_exit_op_test(TestFun, ?PE_INFO_REPEAT),
-    verify_pending_exit_success(S),
-    comment().
-
 %%
 %% -- Internal utils --------------------------------------------------------
 %%
-exit_op_test_init() ->
-    put(no_pending_exit_success, 0),
-    put(no_pending_exit_tries, 0),
-    {case {erlang:system_info(run_queues),
-	   erlang:system_info(schedulers_online)} of
-	 {RQ, SO} when RQ =:= 1; SO =:= 1 -> false;
-	 _ -> true
-     end, 0, 0}.
-
-verify_pending_exit_success({false, _, _} = S) ->
-    S;
-verify_pending_exit_success({true, S, T}) ->
-    NewS = get(no_pending_exit_success),
-    NewT = get(no_pending_exit_tries),
-    case NewT =:= T of
-	true -> ok;
-	_ -> case NewS > S of
-		 true -> ok;
-		 _ -> exit(no_pending_exits)
-	     end
-    end,
-    {true, NewS, NewT}.
-
-comment() ->
-    {comment,
-     "Pending exit trigger ratio "
-     ++ integer_to_list(get(no_pending_exit_success))
-     ++ "/"
-     ++ integer_to_list(get(no_pending_exit_tries))
-     ++ "."
-     ++ case get(not_running_opt_test) of
-	    true -> " No 'not running optimization' to disable.";
-	    _ -> ""
-	end}.
-
-repeated_exit_op_test(TestFun, N) ->
-    WorkFun0 = fun () ->
-		       lists:sort(lists:reverse(lists:seq(1, 1000)))
-	       end,
-    repeat(fun () -> exit_op_test(TestFun, WorkFun0) end, N),
-    try erts_debug:set_internal_state(not_running_optimization, false) of
-	Bool when Bool == true; Bool == false -> 
-	    WorkFun1 = fun () ->
-			       erts_debug:set_internal_state(sleep, 0),
-			       lists:sort(lists:reverse(lists:seq(1, 1000)))
-			    end,
-	    repeat(fun () ->
-			   exit_op_test(TestFun, WorkFun1)
-		   end, N)
-    catch
-	error:notsup -> put(not_running_opt_test, true)
-    after
-	catch erts_debug:set_internal_state(not_running_optimization, true)
-    end.
-
-exit_op_test(TestFun, WorkFun) ->
-    Opts = case {erlang:system_info(run_queues),
-		 erlang:system_info(schedulers_online)} of
-	       {RQ, SO} when RQ =:= 1; SO =:= 1 -> [];
-	       _ ->
-		   process_flag(scheduler, 1),
-		   [{scheduler, 2}]
-	   end,
-    Master = self(),
-    Going = make_ref(),
-    P = spawn_opt(fun () ->
-			  loop(10, WorkFun),
-			  Master ! Going,
-			  loop(infinity, WorkFun)
-		  end, Opts),
-    receive Going -> ok end,
-    loop(10, WorkFun),
-    erlang:yield(),
-    exit(P, bang),
-    PE0 = have_pending_exit(P),
-    TestFun(P),
-    PE = case PE0 of
-	     true -> true;
-	     _ -> false
-	 end,
-    case {PE, get(no_pending_exit_success), get(no_pending_exit_tries)} of
-	{true, undefined, undefined} ->
-	    put(no_pending_exit_success, 1),
-	    put(no_pending_exit_tries, 1);
-	{false, undefined, undefined} ->
-	    put(no_pending_exit_success, 0),
-	    put(no_pending_exit_tries, 1);
-	{true, S, T} ->
-	    put(no_pending_exit_success, S+1),
-	    put(no_pending_exit_tries, T+1);
-	{false, _S, T} ->
-	    put(no_pending_exit_tries, T+1)
-    end,
-    ok.
-
-loop(infinity, WorkFun) ->
-    do_loop(infinity, WorkFun);
-loop(0, _WorkFun) ->
-    ok;
-loop(N, WorkFun) when is_integer(N) ->
-    do_loop(N-1, WorkFun).
-
-do_loop(N, WorkFun) ->
-    WorkFun(),
-    loop(N, WorkFun).
 
 repeat(_Fun, N) when is_integer(N), N =< 0 ->
     ok;
@@ -486,30 +109,3 @@ start_node(Config) ->
 
 stop_node(Node) ->
     test_server:stop_node(Node).
-
-have_pending_exit() ->
-    have_pending_exit(self()).
-
-have_pending_exit(Pid) ->
-    erts_debug:get_internal_state({have_pending_exit, Pid}).
-
-force_gc() ->
-    erts_debug:set_internal_state(force_gc, self()).
-
-fake_exit(From, To, Reason) ->
-    erts_debug:set_internal_state(send_fake_exit_signal, {From, To, Reason}).
-
-available_internal_state(Bool) when Bool == true; Bool == false ->
-    case {Bool,
-          (catch erts_debug:get_internal_state(available_internal_state))} of
-        {true, true} ->
-            true;
-        {false, true} ->
-            erts_debug:set_internal_state(available_internal_state, false),
-            true;
-        {true, _} ->
-            erts_debug:set_internal_state(available_internal_state, true),
-            false;
-        {false, _} ->
-            false
-    end.
diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl
index a81aa64057..def25dba7d 100644
--- a/erts/emulator/test/trace_SUITE.erl
+++ b/erts/emulator/test/trace_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -235,7 +235,7 @@ link_receive_call_correlation(Config) when is_list(Config) ->
     1 = erlang:trace(Receiver, true, ['receive', procs, call, timestamp, scheduler_id]),
     1 = erlang:trace_pattern({?MODULE, receive_msg, '_'}, [], [local]),
 
-    Num = 100000,
+    Num = 100,
 
     (fun F(0) -> [];
          F(N) ->
@@ -255,7 +255,7 @@ link_receive_call_correlation(Config) when is_list(Config) ->
 
     Msgs = (fun F() -> receive M -> [M | F()] after 1 -> [] end end)(),
 
-    case check_consistent(Receiver, Num, Num, Num, Msgs) of
+    case check_consistent(Receiver, Num, Num, Num, Msgs, false, undefined) of
         ok ->
             ok;
         {error, Reason} ->
@@ -265,20 +265,63 @@ link_receive_call_correlation(Config) when is_list(Config) ->
 
 -define(schedid, , _).
 
-check_consistent(_Pid, Recv, Call, _LU, [Msg | _]) when Recv > Call ->
+check_consistent(_Pid, Recv, Call, _LU, [Msg | _], _Received, _LinkedN) when Recv > Call ->
     {error, Msg};
-check_consistent(Pid, Recv, Call, LU, [Msg | Msgs]) ->
+check_consistent(Pid, Recv, Call, LU, [Msg | Msgs], false, undefined) ->
 
     case Msg of
         {trace, Pid, 'receive', Recv ?schedid} ->
-            check_consistent(Pid,Recv - 1, Call, LU, Msgs);
+            check_consistent(Pid,Recv - 1, Call, LU, Msgs, true, undefined);
         {trace_ts, Pid, 'receive', Recv ?schedid, _} ->
-            check_consistent(Pid,Recv - 1, Call, LU, Msgs);
+            check_consistent(Pid,Recv - 1, Call, LU, Msgs, true, undefined);
 
         {trace, Pid, call, {?MODULE, receive_msg, [Call]} ?schedid} ->
-            check_consistent(Pid,Recv, Call - 1, LU, Msgs);
+            check_consistent(Pid,Recv, Call - 1, LU, Msgs, false, undefined);
         {trace_ts, Pid, call, {?MODULE, receive_msg, [Call]} ?schedid, _} ->
-            check_consistent(Pid,Recv, Call - 1, LU, Msgs);
+            check_consistent(Pid,Recv, Call - 1, LU, Msgs, false, undefined);
+
+        {trace, Pid, _, _Self ?schedid} ->
+            check_consistent(Pid, Recv, Call, LU, Msgs, false, undefined);
+        {trace_ts, Pid, _, _Self ?schedid, _} ->
+            check_consistent(Pid, Recv, Call, LU, Msgs, false, undefined);
+
+        Msg ->
+            {error, Msg}
+    end;
+check_consistent(Pid, Recv, Call, LU, [Msg | Msgs], true, undefined) ->
+
+    case Msg of
+        {trace, Pid, call, {?MODULE, receive_msg, [Call]} ?schedid} ->
+            check_consistent(Pid,Recv, Call - 1, LU, Msgs, true, undefined);
+        {trace_ts, Pid, call, {?MODULE, receive_msg, [Call]} ?schedid, _} ->
+            check_consistent(Pid,Recv, Call - 1, LU, Msgs, true, undefined);
+
+        {trace, Pid, getting_linked, _Self ?schedid} ->
+            check_consistent(Pid, Recv, Call, LU - 1, Msgs, true, Recv rem 2);
+        {trace_ts, Pid, getting_linked, _Self ?schedid, _} ->
+            check_consistent(Pid, Recv, Call, LU - 1, Msgs, true, Recv rem 2);
+
+        {trace, Pid, getting_unlinked, _Self ?schedid} ->
+            check_consistent(Pid, Recv, Call, LU - 1, Msgs, true, (Recv+1) rem 2);
+        {trace_ts, Pid, getting_unlinked, _Self ?schedid, _} ->
+            check_consistent(Pid, Recv, Call, LU - 1, Msgs, true, (Recv+1) rem 2);
+
+        Msg ->
+            {error, Msg}
+    end;
+check_consistent(Pid, Recv, Call, LU, [Msg | Msgs], true, LinkedN) ->
+    UnlinkedN = (LinkedN + 1) rem 2,
+
+    case Msg of
+        {trace, Pid, 'receive', Recv ?schedid} when Recv == LU ->
+            check_consistent(Pid,Recv - 1, Call, LU, Msgs, true, LinkedN);
+        {trace_ts, Pid, 'receive', Recv ?schedid, _} when Recv == LU ->
+            check_consistent(Pid,Recv - 1, Call, LU, Msgs, true, LinkedN);
+
+        {trace, Pid, call, {?MODULE, receive_msg, [Call]} ?schedid} ->
+            check_consistent(Pid,Recv, Call - 1, LU, Msgs, true, LinkedN);
+        {trace_ts, Pid, call, {?MODULE, receive_msg, [Call]} ?schedid, _} ->
+            check_consistent(Pid,Recv, Call - 1, LU, Msgs, true, LinkedN);
 
         %% We check that for each receive we have gotten a
         %% getting_linked or getting_unlinked message. Also
@@ -286,38 +329,38 @@ check_consistent(Pid, Recv, Call, LU, [Msg | Msgs]) ->
         %% message we expect to receive is an even number
         %% and odd number for getting_unlinked.
         {trace, Pid, getting_linked, _Self ?schedid}
-          when Recv rem 2 == 0, Recv == LU ->
-            check_consistent(Pid, Recv, Call, LU - 1, Msgs);
+          when Recv rem 2 == LinkedN ->
+            check_consistent(Pid, Recv, Call, LU - 1, Msgs, true, LinkedN);
         {trace_ts, Pid, getting_linked, _Self ?schedid, _}
-          when Recv rem 2 == 0, Recv == LU ->
-            check_consistent(Pid, Recv, Call, LU - 1, Msgs);
+          when Recv rem 2 == LinkedN ->
+            check_consistent(Pid, Recv, Call, LU - 1, Msgs, true, LinkedN);
 
         {trace, Pid, getting_unlinked, _Self ?schedid}
-          when Recv rem 2 == 1, Recv == LU ->
-            check_consistent(Pid, Recv, Call, LU - 1, Msgs);
+          when Recv rem 2 == UnlinkedN ->
+            check_consistent(Pid, Recv, Call, LU - 1, Msgs, true, LinkedN);
         {trace_ts, Pid, getting_unlinked, _Self ?schedid, _}
-          when Recv rem 2 == 1, Recv == LU ->
-            check_consistent(Pid, Recv, Call, LU - 1, Msgs);
+          when Recv rem 2 == UnlinkedN ->
+            check_consistent(Pid, Recv, Call, LU - 1, Msgs, true, LinkedN);
 
         {trace,Pid,'receive',Ignore ?schedid}
           when Ignore == stop; Ignore == timeout ->
-            check_consistent(Pid, Recv, Call, LU, Msgs);
+            check_consistent(Pid, Recv, Call, LU, Msgs, true, LinkedN);
         {trace_ts,Pid,'receive',Ignore ?schedid,_}
           when Ignore == stop; Ignore == timeout ->
-            check_consistent(Pid, Recv, Call, LU, Msgs);
+            check_consistent(Pid, Recv, Call, LU, Msgs, true, LinkedN);
 
         {trace, Pid, exit, normal ?schedid} ->
-            check_consistent(Pid, Recv, Call, LU, Msgs);
+            check_consistent(Pid, Recv, Call, LU, Msgs, true, LinkedN);
         {trace_ts, Pid, exit, normal  ?schedid, _} ->
-            check_consistent(Pid, Recv, Call, LU, Msgs);
+            check_consistent(Pid, Recv, Call, LU, Msgs, true, LinkedN);
         {'EXIT', Pid, normal} ->
-            check_consistent(Pid, Recv, Call, LU, Msgs);
+            check_consistent(Pid, Recv, Call, LU, Msgs, true, LinkedN);
         Msg ->
             {error, Msg}
     end;
-check_consistent(_, 0, 0, 0, []) ->
+check_consistent(_, 0, 0, 1, [], true, _) ->
     ok;
-check_consistent(_, Recv, Call, LU, []) ->
+check_consistent(_, Recv, Call, LU, [], _, _) ->
     {error,{Recv, Call, LU}}.
 
 receive_msg(M) ->
-- 
cgit v1.2.3


From f338d0011cbc83e82b4ff2264a7e37f52190828f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 21 Mar 2018 11:36:25 +0100
Subject: Change default async thread count to 1

All uses of async threads in the built-in drivers have been
replaced with dirty IO, so it no longer makes sense to leave the
default at 10.
---
 erts/emulator/beam/erl_init.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 8430a5559b..dfbf2c7651 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -60,7 +60,7 @@
 #  include <sys/resource.h>
 #endif
 
-#define ERTS_DEFAULT_NO_ASYNC_THREADS	10
+#define ERTS_DEFAULT_NO_ASYNC_THREADS	1
 
 #define ERTS_DEFAULT_SCHED_STACK_SIZE   128
 #define ERTS_DEFAULT_DCPU_SCHED_STACK_SIZE 40
-- 
cgit v1.2.3


From f5af4ec3aec18b7237c11ec6c8b7dd9367002d37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 21 Mar 2018 09:29:22 +0100
Subject: Add an option to ?MODULE:module_info/1 for listing NIFs

---
 erts/emulator/beam/atom.names            |  1 +
 erts/emulator/beam/beam_load.c           | 43 ++++++++++++++++++++++++++++++++
 erts/emulator/test/module_info_SUITE.erl | 15 +++++++++--
 3 files changed, 57 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 38b5f0c5e3..6aadde0d18 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -673,3 +673,4 @@ atom xor
 atom x86
 atom yes
 atom yield
+atom nifs
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 0184c567f1..af620d7432 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -554,6 +554,7 @@ static Eterm get_module_info(Process* p, ErtsCodeIndex code_ix,
 static Eterm exported_from_module(Process* p, ErtsCodeIndex code_ix,
                                   Eterm mod);
 static Eterm functions_in_module(Process* p, BeamCodeHeader*);
+static Eterm nifs_in_module(Process* p, Eterm module);
 static Eterm attributes_for_module(Process* p, BeamCodeHeader*);
 static Eterm compilation_info_for_module(Process* p, BeamCodeHeader*);
 static Eterm md5_of_module(Process* p, BeamCodeHeader*);
@@ -5954,6 +5955,8 @@ get_module_info(Process* p, ErtsCodeIndex code_ix, BeamCodeHeader* code_hdr,
 	return exported_from_module(p, code_ix, module);
     } else if (what == am_functions) {
 	return functions_in_module(p, code_hdr);
+    } else if (what == am_nifs) {
+	return nifs_in_module(p, module);
     } else if (what == am_attributes) {
 	return attributes_for_module(p, code_hdr);
     } else if (what == am_compile) {
@@ -6006,6 +6009,46 @@ functions_in_module(Process* p, /* Process whose heap to use. */
     return result;
 }
 
+/*
+ * Builds a list of all NIFs in the given module:
+ *     [{Name, Arity},...]
+ */
+Eterm
+nifs_in_module(Process* p, Eterm module)
+{
+    Eterm nif_list, *hp;
+    Module *mod;
+
+    mod = erts_get_module(module, erts_active_code_ix());
+    nif_list = NIL;
+
+    if (mod->curr.nif != NULL) {
+        int func_count, func_ix;
+        ErlNifFunc *funcs;
+
+        func_count = erts_nif_get_funcs(mod->curr.nif, &funcs);
+        hp = HAlloc(p, func_count * 5);
+
+        for (func_ix = func_count - 1; func_ix >= 0; func_ix--) {
+            Eterm name, arity, pair;
+            ErlNifFunc *func;
+
+            func = &funcs[func_ix];
+
+            name = am_atom_put(func->name, sys_strlen(func->name));
+            arity = make_small(func->arity);
+
+            pair = TUPLE2(hp, name, arity);
+            hp += 3;
+
+            nif_list = CONS(hp, pair, nif_list);
+            hp += 2;
+        }
+    }
+
+    return nif_list;
+}
+
 /*
  * Returns 'true' if mod has any native compiled functions, otherwise 'false'
  */
diff --git a/erts/emulator/test/module_info_SUITE.erl b/erts/emulator/test/module_info_SUITE.erl
index 0341e63f13..46a3bba732 100644
--- a/erts/emulator/test/module_info_SUITE.erl
+++ b/erts/emulator/test/module_info_SUITE.erl
@@ -23,7 +23,7 @@
 -include_lib("common_test/include/ct.hrl").
 
 -export([all/0, suite/0,
-	 exports/1,functions/1,deleted/1,native/1,info/1]).
+	 exports/1,functions/1,deleted/1,native/1,info/1,nifs/1]).
 
 %%-compile(native).
 
@@ -38,7 +38,7 @@ all() ->
     modules().
 
 modules() ->
-    [exports, functions, deleted, native, info].
+    [exports, functions, deleted, native, info, nifs].
 
 %% Should return all functions exported from this module. (local)
 all_exported() ->
@@ -69,6 +69,17 @@ functions(Config) when is_list(Config) ->
     All = lists:sort(?MODULE:module_info(functions)),
     ok.
 
+nifs(Config) when is_list(Config) ->
+    [] = ?MODULE:module_info(nifs),
+
+    %% erl_tracer is guaranteed to be present and contain these NIFs
+    TraceNIFs = erl_tracer:module_info(nifs),
+    true = lists:member({enabled, 3}, TraceNIFs),
+    true = lists:member({trace, 5}, TraceNIFs),
+    2 = length(TraceNIFs),
+
+    ok.
+
 %% Test that deleted modules cause badarg
 deleted(Config) when is_list(Config) ->
     Data = proplists:get_value(data_dir, Config),
-- 
cgit v1.2.3


From 2dc14df8ce5d56df3974aa57a61b3c1a0bfd3149 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 21 Mar 2018 17:34:38 +0100
Subject: Fix signal handling priority elevation

---
 erts/emulator/beam/erl_process.c | 104 +++++++++++++++++++++++++--------------
 1 file changed, 66 insertions(+), 38 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 7969025f57..374583ec47 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10501,6 +10501,8 @@ done:
     return st;
 }
 
+
+static void exit_permanent_prio_elevation(Process *c_p, erts_aint32_t state);
 static void save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio);
 static void save_dirty_task(Process *c_p, ErtsProcSysTask *st);
 
@@ -10633,8 +10635,10 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
                                                     reds, local_only);
             reds -= sig_reds;
 
-            if (state & ERTS_PSFLG_EXITING)
-                goto perm_elevate_prio;
+            if (state & ERTS_PSFLG_EXITING) {
+                exit_permanent_prio_elevation(c_p, state);
+                break;
+            }
 
             if (sig_res)
                 break;
@@ -10648,37 +10652,8 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
                 st = NULL;
             }
             else {
-                erts_aint32_t a;
-
                 state = erts_atomic32_read_nob(&c_p->state);                         
-
-            perm_elevate_prio:
-
-                /*
-                 * we are about to terminate; permanently elevate
-                 * prio in order to ensure high prio signal
-                 * handling...
-                 */
-
-                a = state;
-                while (1) {
-                    erts_aint32_t aprio, uprio, n, e;
-                    ASSERT(!(a & ERTS_PSFLG_FREE));
-                    aprio = ERTS_PSFLGS_GET_ACT_PRIO(a);
-                    uprio = ERTS_PSFLGS_GET_USR_PRIO(a);
-                    if (aprio >= uprio)
-                        break; /* user prio >= actual prio */
-                    /*
-                     * actual prio is higher than user prio; raise
-                     * user prio to actual prio...
-                     */
-                    n = e = a;
-                    n &= ~ERTS_PSFLGS_USR_PRIO_MASK;
-                    n |= aprio << ERTS_PSFLGS_USR_PRIO_OFFSET;
-                    a = erts_atomic32_cmpxchg_mb(&c_p->state, n, e);
-                    if (a == e)
-                        break;
-                }
+                exit_permanent_prio_elevation(c_p, state);
             }
             break;
         }
@@ -10730,12 +10705,15 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
 	}
 
 	switch (st->type) {
+        case ERTS_PSTT_PRIO_SIG:
+            state = erts_atomic32_read_nob(&c_p->state);                         
+            exit_permanent_prio_elevation(c_p, state);
+            /* fall through... */
         case ERTS_PSTT_GC_MAJOR:
         case ERTS_PSTT_GC_MINOR:
 	case ERTS_PSTT_CPC:
         case ERTS_PSTT_COHMQ:
         case ERTS_PSTT_ETS_FREE_FIXATION:
-        case ERTS_PSTT_PRIO_SIG:
 	    st_res = am_false;
 	    break;
 	case ERTS_PSTT_CLA:
@@ -10759,6 +10737,36 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
     return reds;
 }
 
+static void
+exit_permanent_prio_elevation(Process *c_p, erts_aint32_t state)
+{
+    erts_aint32_t a;
+    /*
+     * we are about to terminate; permanently elevate
+     * prio in order to ensure high prio signal
+     * handling...
+     */
+    a = state;
+    while (1) {
+        erts_aint32_t aprio, uprio, n, e;
+        ASSERT(a & ERTS_PSFLG_EXITING);
+        ASSERT(!(a & ERTS_PSFLG_FREE));
+        aprio = ERTS_PSFLGS_GET_ACT_PRIO(a);
+        uprio = ERTS_PSFLGS_GET_USR_PRIO(a);
+        if (aprio >= uprio)
+            break; /* user prio >= actual prio */
+        /*
+         * actual prio is higher than user prio; raise
+         * user prio to actual prio...
+         */
+        n = e = a;
+        n &= ~ERTS_PSFLGS_USR_PRIO_MASK;
+        n |= aprio << ERTS_PSFLGS_USR_PRIO_OFFSET;
+        a = erts_atomic32_cmpxchg_mb(&c_p->state, n, e);
+        if (a == e)
+            break;
+    }
+}
 
 void
 erts_execute_dirty_system_task(Process *c_p)
@@ -12752,16 +12760,13 @@ erts_continue_exit_process(Process *p)
 
     erts_set_gc_state(p, 1);
     state = erts_atomic32_read_acqb(&p->state);
-    if ((state & ERTS_PSFLG_SYS_TASKS)
-        || p->dirty_sys_tasks
-        ) {
+    if ((state & ERTS_PSFLG_SYS_TASKS) || p->dirty_sys_tasks) {
 	if (cleanup_sys_tasks(p, state, CONTEXT_REDS) >= CONTEXT_REDS/2)
 	    goto yield;
     }
 
 #ifdef DEBUG
     erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
-    ASSERT(p->sys_task_qs == NULL);
     ASSERT(ERTS_PROC_GET_DELAYED_GC_TASK_QS(p) == NULL);
     ASSERT(p->dirty_sys_tasks == NULL);
     erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
@@ -12873,7 +12878,30 @@ erts_continue_exit_process(Process *p)
            ? ERTS_PROC_SET_DIST_ENTRY(p, NULL)
            : NULL);
 
-    erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
+
+    /*
+     * It might show up signal prio elevation tasks until we
+     * have entered free state. Cleanup such tasks now.
+     */
+    state = erts_atomic32_read_acqb(&p->state);
+    if (!(state & ERTS_PSFLG_SYS_TASKS))
+        erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
+    else {
+        erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
+
+        do {
+            (void) cleanup_sys_tasks(p, state, CONTEXT_REDS);
+            state = erts_atomic32_read_acqb(&p->state);
+        } while (state & ERTS_PSFLG_SYS_TASKS);
+        
+        erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
+    }
+
+#ifdef DEBUG
+    erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+    ASSERT(p->sys_task_qs == NULL);
+    erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+#endif
 
     if (dep) {
         erts_do_net_exits(dep, (reason == am_kill) ? am_killed : reason);
-- 
cgit v1.2.3


From f78ed9b50effd3007aafef2979ae5921ffedf75b Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 21 Mar 2018 16:33:35 +0100
Subject: Fix signal order for is_process_alive

---
 erts/emulator/beam/bif.c                |  38 -------
 erts/emulator/beam/bif.h                |  61 -----------
 erts/emulator/beam/bif.tab              |   1 +
 erts/emulator/beam/erl_bif_info.c       |  74 ++++++++-----
 erts/emulator/beam/erl_proc_sig_queue.c | 180 +++++++++++++++++++++++++++-----
 erts/emulator/beam/erl_proc_sig_queue.h |  25 +++++
 erts/emulator/test/bif_SUITE.erl        |  27 ++++-
 7 files changed, 255 insertions(+), 151 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index adea7d007e..232597c5b6 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -52,7 +52,6 @@ Export *erts_await_result;
 static Export await_exit_trap;
 static Export* flush_monitor_messages_trap = NULL;
 static Export* set_cpu_topology_trap = NULL;
-static Export* await_proc_exit_trap = NULL;
 static Export* await_port_send_result_trap = NULL;
 Export* erts_format_cpu_topology_trap = NULL;
 static Export dsend_continue_trap_export;
@@ -4654,42 +4653,6 @@ static BIF_RETTYPE bif_return_trap(BIF_ALIST_2)
     BIF_RET(res);
 }
 
-void
-erts_bif_prep_await_proc_exit_data_trap(Process *c_p, Eterm pid, Eterm ret)
-{
-    ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, pid, am_data, ret);
-}
-
-void
-erts_bif_prep_await_proc_exit_reason_trap(Process *c_p, Eterm pid)
-{
-    ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p,
-                               pid, am_reason, am_undefined);
-}
-
-void
-erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
-					 Eterm pid,
-					 Eterm module,
-					 Eterm function,
-					 Eterm args[],
-					 int nargs)
-{
-    Eterm term;
-    Eterm *hp;
-    int i;
-    ASSERT(is_atom(module) && is_atom(function));
-
-    hp = HAlloc(c_p, 4+2*nargs);
-    term = NIL;
-    for (i = nargs-1; i >= 0; i--) {
-        term = CONS(hp, args[i], term);
-        hp += 2;
-    }
-    term = TUPLE3(hp, module, function, term);
-    ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, pid, am_apply, term);
-}
-
 Export bif_return_trap_export;
 
 void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a,
@@ -4742,7 +4705,6 @@ void erts_init_bif(void)
     erts_format_cpu_topology_trap = erts_export_put(am_erlang,
 						    am_format_cpu_topology,
 						    1);
-    await_proc_exit_trap = erts_export_put(am_erlang,am_await_proc_exit,3);
     await_port_send_result_trap
 	= erts_export_put(am_erts_internal, am_await_port_send_result, 3);
     system_flag_scheduler_wall_time_trap
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index dca53686f4..a33421d762 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -437,67 +437,6 @@ do {					\
 	ERTS_BIF_EXITED((PROC));	\
 } while (0)
 
-/*
- * The ERTS_BIF_*_AWAIT_X_*_TRAP makros either exits the caller, or
- * sets up a trap to erlang:await_proc_exit/3.
- *
- * The caller is acquired to hold the 'main' lock on C_P. No other locks
- * are allowed to be held.
- */
-
-#define ERTS_BIF_PREP_AWAIT_X_DATA_TRAP(RET, C_P, PID, DATA)		\
-do {									\
-    erts_bif_prep_await_proc_exit_data_trap((C_P), (PID), (DATA));	\
-    (RET) = THE_NON_VALUE;						\
-} while (0)
-
-#define ERTS_BIF_PREP_AWAIT_X_REASON_TRAP(RET, C_P, PID)		\
-do {									\
-    erts_bif_prep_await_proc_exit_reason_trap((C_P), (PID));		\
-    (RET) = THE_NON_VALUE;						\
-} while (0)
-
-#define ERTS_BIF_PREP_AWAIT_X_APPLY_TRAP(RET, C_P, PID, M, F, A, AN)	\
-do {									\
-    erts_bif_prep_await_proc_exit_apply_trap((C_P), (PID),		\
-					     (M), (F), (A), (AN));	\
-    (RET) = THE_NON_VALUE;						\
-} while (0)
-
-#define ERTS_BIF_AWAIT_X_DATA_TRAP(C_P, PID, DATA)			\
-do {									\
-    erts_bif_prep_await_proc_exit_data_trap((C_P), (PID), (DATA));	\
-    return THE_NON_VALUE;						\
-} while (0)
-
-#define ERTS_BIF_AWAIT_X_REASON_TRAP(C_P, PID)				\
-do {									\
-    erts_bif_prep_await_proc_exit_reason_trap((C_P), (PID));		\
-    return THE_NON_VALUE;						\
-} while (0)
-
-#define ERTS_BIF_AWAIT_X_APPLY_TRAP(C_P, PID, M, F, A, AN)		\
-do {									\
-    erts_bif_prep_await_proc_exit_apply_trap((C_P), (PID),		\
-					     (M), (F), (A), (AN));	\
-    return THE_NON_VALUE;						\
-} while (0)
-
-void
-erts_bif_prep_await_proc_exit_data_trap(Process *c_p,
-					Eterm pid,
-					Eterm data);
-void
-erts_bif_prep_await_proc_exit_reason_trap(Process *c_p,
-					  Eterm pid);
-void
-erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
-					 Eterm pid,
-					 Eterm module,
-					 Eterm function,
-					 Eterm args[],
-					 int nargs);
-
 int erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p,
 			BeamInstr *I, Eterm *reg);
 
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 93613ac2eb..687fd39d58 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -266,6 +266,7 @@ bif erlang:demonitor/1
 bif erlang:demonitor/2
 
 bif erlang:is_process_alive/1
+bif erts_internal:is_process_alive/2
 
 bif erlang:error/1		error_1
 bif erlang:error/2		error_2
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 9a3c058e82..e94544a678 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -73,6 +73,9 @@ static Export *gather_msacc_res_trap;
 static Export *gather_gc_info_res_trap;
 static Export *gather_system_check_res_trap;
 
+static Export *is_process_alive_trap;
+
+
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
 static char otp_version[] = ERLANG_OTP_VERSION;
@@ -3502,34 +3505,53 @@ fun_info_mfa_1(BIF_ALIST_1)
     BIF_ERROR(p, BADARG);
 }
 
+BIF_RETTYPE erts_internal_is_process_alive_2(BIF_ALIST_2)
+{
+    if (!is_internal_pid(BIF_ARG_1) || !is_internal_ordinary_ref(BIF_ARG_2))
+        BIF_ERROR(BIF_P, BADARG);
+    erts_proc_sig_send_is_alive_request(BIF_P, BIF_ARG_1, BIF_ARG_2);
+    BIF_RET(am_ok);
+}
+
 BIF_RETTYPE is_process_alive_1(BIF_ALIST_1) 
 {
-   if(is_internal_pid(BIF_ARG_1)) {
-       Process *rp;
-
-       if (BIF_ARG_1 == BIF_P->common.id)
-	   BIF_RET(am_true);
-
-       rp = erts_proc_lookup_raw(BIF_ARG_1);
-       if (!rp) {
-	   BIF_RET(am_false);
-       }
-       else {
-	   if (erts_atomic32_read_acqb(&rp->state) & ERTS_PSFLG_EXITING)
-	       ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_false);
-	   else
-	       BIF_RET(am_true);
-       }
-   }
-   else if(is_external_pid(BIF_ARG_1)) {
-       if(external_pid_dist_entry(BIF_ARG_1) == erts_this_dist_entry)
+    if (is_internal_pid(BIF_ARG_1)) {
+        erts_aint32_t state;
+        Process *rp;
+
+        if (BIF_ARG_1 == BIF_P->common.id)
+            BIF_RET(am_true);
+
+        rp = erts_proc_lookup_raw(BIF_ARG_1);
+        if (!rp)
+            BIF_RET(am_false);
+
+        state = erts_atomic32_read_acqb(&rp->state);
+        if (state & (ERTS_PSFLG_EXITING
+                     | ERTS_PSFLG_SIG_Q
+                     | ERTS_PSFLG_SIG_IN_Q)) {
+            /*
+             * If in exiting state, trap out and send 'is alive'
+             * request and wait for it to complete termination.
+             *
+             * If process has signals enqueued, we need to
+             * send it an 'is alive' request via its signal
+             * queue in order to ensure that signal order is
+             * preserved (we may earlier have sent it an
+             * exit signal that has not been processed yet).
+             */
+            BIF_TRAP1(is_process_alive_trap, BIF_P, BIF_ARG_1);
+        }
+
+        BIF_RET(am_true);
+    }
+
+   if (is_external_pid(BIF_ARG_1)) {
+       if (external_pid_dist_entry(BIF_ARG_1) == erts_this_dist_entry)
 	   BIF_RET(am_false); /* A pid from an old incarnation of this node */
-       else
-	   BIF_ERROR(BIF_P, BADARG);
-   }
-   else {
-      BIF_ERROR(BIF_P, BADARG);
    }
+
+   BIF_ERROR(BIF_P, BADARG);
 }
 
 BIF_RETTYPE process_display_2(BIF_ALIST_2)
@@ -5008,6 +5030,10 @@ erts_bif_info_init(void)
 	= erts_export_put(am_erts_internal, am_gather_microstate_accounting_result, 2);
     gather_system_check_res_trap
 	= erts_export_put(am_erts_internal, am_gather_system_check_result, 1);
+
+    is_process_alive_trap = erts_export_put(am_erts_internal, am_is_process_alive, 1);
+
+
     process_info_init();
     os_info_init();
 }
diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index 1b5cbb1919..489df08991 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -48,7 +48,7 @@
  * Note that not all signal are handled using this functionality!
  */
 
-#define ERTS_SIG_Q_OP_MAX 9
+#define ERTS_SIG_Q_OP_MAX 10
 
 #define ERTS_SIG_Q_OP_EXIT                      0
 #define ERTS_SIG_Q_OP_EXIT_LINKED               1
@@ -59,7 +59,8 @@
 #define ERTS_SIG_Q_OP_UNLINK                    6
 #define ERTS_SIG_Q_OP_GROUP_LEADER              7
 #define ERTS_SIG_Q_OP_TRACE_CHANGE_STATE        8
-#define ERTS_SIG_Q_OP_PERSISTENT_MON_MSG        ERTS_SIG_Q_OP_MAX
+#define ERTS_SIG_Q_OP_PERSISTENT_MON_MSG        9
+#define ERTS_SIG_Q_OP_IS_ALIVE                  ERTS_SIG_Q_OP_MAX
 
 #define ERTS_SIG_Q_TYPE_MAX (ERTS_MON_LNK_TYPE_MAX + 5)
 
@@ -184,6 +185,11 @@ typedef struct {
     Eterm heap[1];
 } ErtsSigGroupLeader;
 
+typedef struct {
+    Eterm message;
+    Eterm requester;
+} ErtsIsAliveRequest;
+
 static int handle_msg_tracing(Process *c_p,
                               ErtsSigRecvTracing *tracing,
                               ErtsMessage ***next_nm_sig);
@@ -548,6 +554,43 @@ proc_queue_signal(Process *c_p, Eterm pid, ErtsSignal *sig, int op)
     return res;
 }
 
+static int
+maybe_elevate_sig_handling_prio(Process *c_p, Eterm other)
+{
+    /*
+     * returns:
+     *  > 0 -> elevated prio; process alive or exiting
+     *  < 0 -> no elevation needed; process alive or exiting
+     *    0 -> process terminated (free)
+     */
+    int res;
+    Process *rp;
+    erts_aint32_t state, my_prio, other_prio;
+
+    rp = erts_proc_lookup_raw(other);
+    if (!rp)
+        res = 0;
+    else {
+        res = -1;
+        state = erts_atomic32_read_nob(&c_p->state);
+        my_prio = ERTS_PSFLGS_GET_USR_PRIO(state);
+
+        state = erts_atomic32_read_nob(&rp->state);
+        other_prio = ERTS_PSFLGS_GET_USR_PRIO(state);
+
+        if (other_prio > my_prio) {
+            /* Others prio is lower than mine; elevate it... */
+            res = !!erts_sig_prio(other, my_prio);
+            if (res) {
+                /* ensure handled if dirty executing... */
+                state = erts_atomic32_read_nob(&rp->state);
+                ensure_dirty_proc_handled(other, state, my_prio);
+            }
+        }
+    }
+    return res;
+}
+
 void
 erts_proc_sig_fetch(Process *proc)
 {
@@ -1215,33 +1258,10 @@ erts_proc_sig_send_group_leader(Process *c_p, Eterm to, Eterm gl, Eterm ref)
     if (!res)
         destroy_sig_group_leader(sgl);
     else if (c_p) {
-        int prio_res = !0;
         erts_aint_t flags, rm_flags = ERTS_SIG_GL_FLG_SENDER;
-        Process *rp;
-        erts_aint32_t state, my_prio, other_prio;
-
-        state = erts_atomic32_read_nob(&c_p->state);
-        my_prio = ERTS_PSFLGS_GET_USR_PRIO(state);
-
-        rp = erts_proc_lookup_raw(to);
-        if (!rp)
-            prio_res = 0;
-        else {
-            state = erts_atomic32_read_nob(&rp->state);
-            other_prio = ERTS_PSFLGS_GET_USR_PRIO(state);
-
-            if (other_prio > my_prio) {
-                /* Others prio is lower than mine; elevate it... */
-                prio_res = erts_sig_prio(to, my_prio);
-                if (prio_res) {
-                    state = erts_atomic32_read_nob(&rp->state);
-                    ensure_dirty_proc_handled(to, state, my_prio);
-                }
-            }
-        }
+        int prio_res = maybe_elevate_sig_handling_prio(c_p, to);
         if (!prio_res)
             rm_flags |= ERTS_SIG_GL_FLG_ACTIVE;
-
         flags = erts_atomic_read_band_nob(&sgl->flags, ~rm_flags);
         if (!prio_res && (flags & ERTS_SIG_GL_FLG_ACTIVE))
             res = 0; /* We deactivated signal... */
@@ -1253,6 +1273,99 @@ erts_proc_sig_send_group_leader(Process *c_p, Eterm to, Eterm gl, Eterm ref)
         group_leader_reply(c_p, c_p->common.id, ref, 0);
 }
 
+void
+erts_proc_sig_send_is_alive_request(Process *c_p, Eterm to, Eterm ref)
+{
+    ErlHeapFragment *hfrag;
+    Uint hsz;
+    Eterm *hp, *start_hp, ref_cpy, msg;
+    ErlOffHeap *ohp;
+    ErtsMessage *mp;
+    ErtsIsAliveRequest *alive_req;
+
+    ASSERT(is_internal_ordinary_ref(ref));
+
+    hsz = ERTS_REF_THING_SIZE + 3 + sizeof(ErtsIsAliveRequest)/sizeof(Eterm);
+
+    mp = erts_alloc_message(hsz, &hp);
+    hfrag = &mp->hfrag;
+    mp->next = NULL;
+    ohp = &hfrag->off_heap;
+    start_hp = hp;
+
+    ref_cpy = STORE_NC(&hp, ohp, ref);
+    msg = TUPLE2(hp, ref_cpy, am_false); /* default res 'false' */
+    hp += 3;
+
+    hfrag->used_size = hp - start_hp;
+
+    alive_req = (ErtsIsAliveRequest *) (char *) hp;
+    alive_req->message = msg;
+    alive_req->requester = c_p->common.id;
+
+    ERL_MESSAGE_TERM(mp) = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_IS_ALIVE,
+                                                  ERTS_SIG_Q_TYPE_UNDEFINED,
+                                                  0);
+    ERL_MESSAGE_TOKEN(mp) = NIL;
+    ERL_MESSAGE_FROM(mp) = am_system;
+#ifdef USE_VM_PROBES
+    ERL_MESSAGE_DT_UTAG(mp) = NIL;
+#endif
+
+    if (proc_queue_signal(c_p, to, (ErtsSignal *) mp, ERTS_SIG_Q_OP_IS_ALIVE))
+        (void) maybe_elevate_sig_handling_prio(c_p, to);
+    else {
+        /* It wasn't alive; reply to ourselves... */
+        mp->next = NULL;
+        mp->data.attached = ERTS_MSG_COMBINED_HFRAG;
+        erts_queue_message(c_p, ERTS_PROC_LOCK_MAIN,
+                           mp, msg, am_system);
+    }
+}
+
+static void
+is_alive_response(Process *c_p, ErtsMessage *mp, int is_alive)
+{
+    /*
+     * Sender prepared the message for us. Just patch
+     * the result if necessary. The default prepared
+     * result is 'false'.
+     */
+    Process *rp;
+    ErtsIsAliveRequest *alive_req;
+
+    alive_req = (ErtsIsAliveRequest *) (char *) (&mp->hfrag.mem[0]
+                                                 + mp->hfrag.used_size);
+
+
+    ASSERT(ERTS_SIG_IS_NON_MSG(mp));
+    ASSERT(ERTS_PROC_SIG_OP(((ErtsSignal *) mp)->common.tag)
+           == ERTS_SIG_Q_OP_IS_ALIVE);
+    ASSERT(mp->hfrag.alloc_size > mp->hfrag.used_size);
+    ASSERT((mp->hfrag.alloc_size - mp->hfrag.used_size)*sizeof(UWord)
+           >= sizeof(ErtsIsAliveRequest));
+    ASSERT(is_internal_pid(alive_req->requester));
+    ASSERT(alive_req->requester != c_p->common.id);
+    ASSERT(is_tuple_arity(alive_req->message, 2));
+    ASSERT(is_internal_ordinary_ref(tuple_val(alive_req->message)[1]));
+    ASSERT(tuple_val(alive_req->message)[2] == am_false);
+
+    ERL_MESSAGE_TERM(mp) = alive_req->message;
+    mp->data.attached = ERTS_MSG_COMBINED_HFRAG;
+    mp->next = NULL;
+
+    rp = erts_proc_lookup(alive_req->requester);
+    if (!rp)
+        erts_cleanup_messages(mp);
+    else {
+        if (is_alive) { /* patch result... */
+            Eterm *tp = tuple_val(alive_req->message);
+            tp[2] = am_true;
+        }
+        erts_queue_message(rp, 0, mp, alive_req->message, am_system);
+    }
+}
+
 static ERTS_INLINE void
 adjust_tracing_state(Process *c_p, ErtsSigRecvTracing *tracing, int setup)
 {
@@ -2355,6 +2468,13 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep,
             break;
         }
 
+        case ERTS_SIG_Q_OP_IS_ALIVE:
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            remove_nm_sig(c_p, sig, next_nm_sig);
+            is_alive_response(c_p, sig, !0);
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            break;
+
         case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE: {
             Uint16 type = ERTS_PROC_SIG_TYPE(tag);
 
@@ -2667,6 +2787,12 @@ erts_proc_sig_handle_exit(Process *c_p, int *redsp)
             break;
         }
 
+        case ERTS_SIG_Q_OP_IS_ALIVE:
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            is_alive_response(c_p, sig, 0);
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            break;
+
         case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE:
             destroy_trace_info((ErtsSigTraceInfo *) sig);
             break;
@@ -2803,6 +2929,7 @@ erts_proc_sig_signal_size(ErtsSignal *sig)
         break;
 
     case ERTS_SIG_Q_OP_PERSISTENT_MON_MSG:
+    case ERTS_SIG_Q_OP_IS_ALIVE:
         size = ((ErtsMessage *) sig)->hfrag.alloc_size;
         size *= sizeof(Eterm);
         size += sizeof(ErtsMessage) - sizeof(Eterm);
@@ -3514,6 +3641,7 @@ erts_proc_sig_debug_foreach_sig(Process *c_p,
                     break;
                 }
 
+                case ERTS_SIG_Q_OP_IS_ALIVE:
                 case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE:
                     break;
 
diff --git a/erts/emulator/beam/erl_proc_sig_queue.h b/erts/emulator/beam/erl_proc_sig_queue.h
index 433e30ce4a..56fe3e683e 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.h
+++ b/erts/emulator/beam/erl_proc_sig_queue.h
@@ -31,6 +31,7 @@
  *              - Link
  *              - Unlink
  *              - Group leader
+ *              - Is process alive
  *              - Trace change
  *
  *              The signal queue consists of three parts:
@@ -426,6 +427,30 @@ void
 erts_proc_sig_send_group_leader(Process *c_p, Eterm to, Eterm gl,
                                 Eterm ref);
 
+/**
+ *
+ * @brief Send an 'is process alive' signal to a process.
+ *
+ * A response message '{Ref, Result}' is sent to the
+ * sender when performed where Ref is the reference passed
+ * as 'ref' argument, and Result is either 'true' or 'false'.
+ *
+ * @param[in]     c_p           Pointer to process struct of
+ *                              currently executing process.
+ *                              NULL if signal arrived via
+ *                              distribution.
+ *
+ * @param[in]     to            Identifier of receiver.
+ *
+ * @param[in]     ref           Reference to use in response
+ *                              message to the sending
+ *                              process (i.e., c_p).
+ *
+ */
+void
+erts_proc_sig_send_is_alive_request(Process *c_p, Eterm to,
+                                    Eterm ref);
+
 /*
  * End of send operations of currently supported process signals.
  */
diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl
index d16c6a320d..22706ae8b1 100644
--- a/erts/emulator/test/bif_SUITE.erl
+++ b/erts/emulator/test/bif_SUITE.erl
@@ -34,7 +34,8 @@
          erl_crash_dump_bytes/1,
 	 is_builtin/1, error_stacktrace/1,
 	 error_stacktrace_during_call_trace/1,
-         group_leader_prio/1, group_leader_prio_dirty/1]).
+         group_leader_prio/1, group_leader_prio_dirty/1,
+         is_process_alive/1]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
@@ -48,7 +49,8 @@ all() ->
      atom_to_binary, binary_to_atom, binary_to_existing_atom,
      erl_crash_dump_bytes, min_max, erlang_halt, is_builtin,
      error_stacktrace, error_stacktrace_during_call_trace,
-     group_leader_prio, group_leader_prio_dirty].
+     group_leader_prio, group_leader_prio_dirty,
+     is_process_alive].
 
 %% Uses erlang:display to test that erts_printf does not do deep recursion
 display(Config) when is_list(Config) ->
@@ -1076,6 +1078,27 @@ group_leader_prio_test(Dirty) ->
                   TLs),
     ok.
 
+is_process_alive(Config) when is_list(Config) ->
+    process_flag(priority, max),
+    Ps = lists:map(fun (_) ->
+                           spawn_opt(fun () -> tok_loop() end,
+                                     [{priority, high}, link])
+                   end,
+                   lists:seq(1, 2*erlang:system_info(schedulers))),
+    receive after 1000 -> ok end, %% Wait for load to spread
+    lists:foreach(fun (P) ->
+                          %% Ensure that signal order is preserved
+                          %% and that we are not starved due to
+                          %% priority inversion
+                          true = erlang:is_process_alive(P),
+                          unlink(P),
+                          true = erlang:is_process_alive(P),
+                          exit(P, kill),
+                          false = erlang:is_process_alive(P)
+                  end,
+                  Ps),
+    ok.
+
 %% helpers
     
 id(I) -> I.
-- 
cgit v1.2.3


From 30d5b7ee32d099f2a23c26e873aeb08be1b1d966 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 15 Mar 2018 17:14:39 +0100
Subject: erts: Add enif_*_name functions

---
 erts/emulator/beam/erl_nif.c           | 6 ++++++
 erts/emulator/beam/erl_nif.h           | 2 +-
 erts/emulator/beam/erl_nif_api_funcs.h | 9 +++++++++
 3 files changed, 16 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index c60cc7fecf..332e692fd6 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1963,6 +1963,12 @@ ErlNifTid enif_thread_self(void) { return erl_drv_thread_self(); }
 int enif_equal_tids(ErlNifTid tid1, ErlNifTid tid2) { return erl_drv_equal_tids(tid1,tid2); }
 void enif_thread_exit(void *resp) { erl_drv_thread_exit(resp); }
 int enif_thread_join(ErlNifTid tid, void **respp) { return erl_drv_thread_join(tid,respp); }
+
+char* enif_mutex_name(ErlNifMutex *mtx) {return erl_drv_mutex_name(mtx); }
+char* enif_cond_name(ErlNifCond *cnd) { return erl_drv_cond_name(cnd); }
+char* enif_rwlock_name(ErlNifRWLock* rwlck) { return erl_drv_rwlock_name(rwlck); }
+char* enif_thread_name(ErlNifTid tid) { return erl_drv_thread_name(tid); }
+
 int enif_getenv(const char *key, char *value, size_t *value_size) { return erl_drv_getenv(key, value, value_size); }
 
 ErlNifTime enif_monotonic_time(ErlNifTimeUnit time_unit)
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index a99b4db705..14682add51 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -52,7 +52,7 @@
 ** 2.11: 19.0 enif_snprintf 
 ** 2.12: 20.0 add enif_select, enif_open_resource_type_x
 ** 2.13: 20.1 add enif_ioq
-** 2.14: 21.0 add enif_ioq_peek_head
+** 2.14: 21.0 add enif_ioq_peek_head, enif_(mutex|cond|rwlock|thread)_name
 */
 #define ERL_NIF_MAJOR_VERSION 2
 #define ERL_NIF_MINOR_VERSION 14
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 3750fd9b68..09250bfaf0 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -200,6 +200,11 @@ ERL_NIF_API_FUNC_DECL(void,enif_free_iovec,(ErlNifIOVec *iov));
 
 ERL_NIF_API_FUNC_DECL(int,enif_ioq_peek_head,(ErlNifEnv *env, ErlNifIOQueue *q, size_t *size, ERL_NIF_TERM *head));
 
+ERL_NIF_API_FUNC_DECL(char*,enif_mutex_name,(ErlNifMutex*));
+ERL_NIF_API_FUNC_DECL(char*,enif_cond_name,(ErlNifCond*));
+ERL_NIF_API_FUNC_DECL(char*,enif_rwlock_name,(ErlNifRWLock*));
+ERL_NIF_API_FUNC_DECL(char*,enif_thread_name,(ErlNifTid));
+
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
 */
@@ -375,6 +380,10 @@ ERL_NIF_API_FUNC_DECL(int,enif_ioq_peek_head,(ErlNifEnv *env, ErlNifIOQueue *q,
 #  define enif_inspect_iovec ERL_NIF_API_FUNC_MACRO(enif_inspect_iovec)
 #  define enif_free_iovec ERL_NIF_API_FUNC_MACRO(enif_free_iovec)
 #  define enif_ioq_peek_head ERL_NIF_API_FUNC_MACRO(enif_ioq_peek_head)
+#  define enif_mutex_name ERL_NIF_API_FUNC_MACRO(enif_mutex_name)
+#  define enif_cond_name ERL_NIF_API_FUNC_MACRO(enif_cond_name)
+#  define enif_rwlock_name ERL_NIF_API_FUNC_MACRO(enif_rwlock_name)
+#  define enif_thread_name ERL_NIF_API_FUNC_MACRO(enif_thread_name)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
-- 
cgit v1.2.3


From e36c103236ac70c29caf910d31651bed6c24dfe8 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 15 Mar 2018 21:02:47 +0100
Subject: erts: Add enif_vfprintf and enif_vsnprintf

---
 erts/emulator/beam/erl_nif.c           | 15 +++++++++++++--
 erts/emulator/beam/erl_nif.h           |  3 +++
 erts/emulator/beam/erl_nif_api_funcs.h |  7 ++++++-
 3 files changed, 22 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 332e692fd6..2c851fd531 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1991,16 +1991,21 @@ enif_convert_time_unit(ErlNifTime val,
 						    (int) to);
 }
 
-int enif_fprintf(void* filep, const char* format, ...) 
+int enif_fprintf(FILE* filep, const char* format, ...)
 { 
     int ret;
     va_list arglist;
     va_start(arglist, format);
-    ret = erts_vfprintf((FILE*)filep, format, arglist);
+    ret = erts_vfprintf(filep, format, arglist);
     va_end(arglist);
     return ret;
 }    
 
+int enif_vfprintf(FILE* filep, const char *format, va_list ap)
+{
+    return erts_vfprintf(filep, format, ap);
+}
+
 int enif_snprintf(char *buffer, size_t size, const char* format, ...) 
 { 
     int ret;
@@ -2011,6 +2016,12 @@ int enif_snprintf(char *buffer, size_t size, const char* format, ...)
     return ret;
 }
 
+int enif_vsnprintf(char* buffer, size_t size, const char *format, va_list ap)
+{
+    return erts_vsnprintf(buffer, size, format, ap);
+}
+
+
 /***********************************************************
  **       Memory managed (GC'ed) "resource" objects       **
  ***********************************************************/
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 14682add51..30eff9fcb9 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -53,6 +53,7 @@
 ** 2.12: 20.0 add enif_select, enif_open_resource_type_x
 ** 2.13: 20.1 add enif_ioq
 ** 2.14: 21.0 add enif_ioq_peek_head, enif_(mutex|cond|rwlock|thread)_name
+**                enif_vfprintf, enif_vsnprintf
 */
 #define ERL_NIF_MAJOR_VERSION 2
 #define ERL_NIF_MINOR_VERSION 14
@@ -70,6 +71,8 @@
 #define ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD 2
 
 #include <stdlib.h>
+#include <stdio.h>
+#include <stdarg.h>
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 09250bfaf0..744f7c9f69 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -92,7 +92,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_thread_join,(ErlNifTid, void **respp));
 
 ERL_NIF_API_FUNC_DECL(void*,enif_realloc,(void* ptr, size_t size));
 ERL_NIF_API_FUNC_DECL(void,enif_system_info,(ErlNifSysInfo *sip, size_t si_size));
-ERL_NIF_API_FUNC_DECL(int,enif_fprintf,(void/* FILE* */ *filep, const char *format, ...));
+ERL_NIF_API_FUNC_DECL(int,enif_fprintf,(FILE* filep, const char *format, ...));
 ERL_NIF_API_FUNC_DECL(int,enif_inspect_iolist_as_binary,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifBinary* bin));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_sub_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, size_t pos, size_t size));
 ERL_NIF_API_FUNC_DECL(int,enif_get_string,(ErlNifEnv*, ERL_NIF_TERM list, char* buf, unsigned len, ErlNifCharEncoding));
@@ -205,6 +205,9 @@ ERL_NIF_API_FUNC_DECL(char*,enif_cond_name,(ErlNifCond*));
 ERL_NIF_API_FUNC_DECL(char*,enif_rwlock_name,(ErlNifRWLock*));
 ERL_NIF_API_FUNC_DECL(char*,enif_thread_name,(ErlNifTid));
 
+ERL_NIF_API_FUNC_DECL(int,enif_vfprintf,(FILE*, const char *fmt, va_list));
+ERL_NIF_API_FUNC_DECL(int,enif_vsnprintf,(char*, size_t, const char *fmt, va_list));
+
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
 */
@@ -384,6 +387,8 @@ ERL_NIF_API_FUNC_DECL(char*,enif_thread_name,(ErlNifTid));
 #  define enif_cond_name ERL_NIF_API_FUNC_MACRO(enif_cond_name)
 #  define enif_rwlock_name ERL_NIF_API_FUNC_MACRO(enif_rwlock_name)
 #  define enif_thread_name ERL_NIF_API_FUNC_MACRO(enif_thread_name)
+#  define enif_vfprintf ERL_NIF_API_FUNC_MACRO(enif_vfprintf)
+#  define enif_vsnprintf ERL_NIF_API_FUNC_MACRO(enif_vsnprintf)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
-- 
cgit v1.2.3


From cbc7ef1f665423aabba3836b79017682ca1b0816 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 16 Mar 2018 20:57:41 +0100
Subject: erts: Improve NIF load incompatibility errors

---
 erts/emulator/beam/erl_nif.c | 21 ++++++++++++++++-----
 erts/emulator/beam/erl_nif.h |  7 ++++++-
 2 files changed, 22 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 2c851fd531..99079dabf6 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3873,6 +3873,11 @@ static struct erl_module_nif* create_lib(const ErlNifEntry* src)
     } else {
         dst->sizeof_ErlNifResourceTypeInit = 0;
     }
+    if (AT_LEAST_VERSION(src, 2, 14)) {
+        dst->min_erts = src->min_erts;
+    } else {
+        dst->min_erts = "erts-?";
+    }
     return lib;
 };
 
@@ -3985,14 +3990,20 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	      (entry = erts_sys_ddll_call_nif_init(init_func)) == NULL)) {
 	ret = load_nif_error(BIF_P, bad_lib, "Library init-call unsuccessful");
     }
+    else if (entry->major > ERL_NIF_MAJOR_VERSION
+             || (entry->major == ERL_NIF_MAJOR_VERSION
+                 && entry->minor > ERL_NIF_MINOR_VERSION)) {
+        char* fmt = "That '%T' NIF library needs %s or newer. Either try to"
+            " recompile the NIF lib or use a newer erts runtime.";
+        ret = load_nif_error(BIF_P, bad_lib, fmt, mod_atom, entry->min_erts);
+    }
     else if (entry->major < ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD
-	     || (ERL_NIF_MAJOR_VERSION < entry->major
-		 || (ERL_NIF_MAJOR_VERSION == entry->major
-		     && ERL_NIF_MINOR_VERSION < entry->minor))
 	     || (entry->major==2 && entry->minor == 5)) { /* experimental maps */
 	
-	ret = load_nif_error(BIF_P, bad_lib, "Library version (%d.%d) not compatible (with %d.%d).",
-			     entry->major, entry->minor, ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION);
+        char* fmt = "That old NIF library (%d.%d) is not compatible with this "
+            "erts runtime (%d.%d). Try recompile the NIF lib.";
+        ret = load_nif_error(BIF_P, bad_lib, fmt, entry->major, entry->minor,
+                             ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION);
     }   
     else if (AT_LEAST_VERSION(entry, 2, 1)
 	     && sys_strcmp(entry->vm_variant, ERL_NIF_VM_VARIANT) != 0) {
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 30eff9fcb9..e051ecb26e 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -57,6 +57,7 @@
 */
 #define ERL_NIF_MAJOR_VERSION 2
 #define ERL_NIF_MINOR_VERSION 14
+#define ERL_NIF_MIN_ERTS_VERSION "erts-10.0 (OTP-21)"
 
 /*
  * The emulator will refuse to load a nif-lib with a major version
@@ -131,6 +132,9 @@ typedef struct enif_entry_t
 
     /* Added in 2.12 */
     size_t sizeof_ErlNifResourceTypeInit;
+
+    /* Added in 2.14 */
+    const char* min_erts;
 }ErlNifEntry;
 
 
@@ -353,7 +357,8 @@ ERL_NIF_INIT_DECL(NAME)			\
 	LOAD, RELOAD, UPGRADE, UNLOAD,	\
 	ERL_NIF_VM_VARIANT,		\
         1,                              \
-        sizeof(ErlNifResourceTypeInit)  \
+        sizeof(ErlNifResourceTypeInit), \
+        ERL_NIF_MIN_ERTS_VERSION        \
     };                                  \
     ERL_NIF_INIT_BODY;                  \
     return &entry;			\
-- 
cgit v1.2.3


From acd07fa4db07a5bee74af1bc17ea67db5874b603 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 22 Mar 2018 16:44:19 +0100
Subject: Fix lock counting

---
 erts/emulator/beam/erl_node_tables.c    | 8 +++++---
 erts/emulator/beam/erl_proc_sig_queue.c | 6 ++----
 2 files changed, 7 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index ca83e70046..1f147011a8 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1051,14 +1051,16 @@ static void erts_lcnt_enable_dist_lock_count(void *dep_raw, void *enable) {
     if(enable) {
         erts_lcnt_install_new_lock_info(&dep->rwmtx.lcnt, "dist_entry", dep->sysname, 
             ERTS_LOCK_TYPE_RWMUTEX | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
-        erts_lcnt_install_new_lock_info(&dep->lnk_mtx.lcnt, "dist_entry_links", dep->sysname,
-            ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
         erts_lcnt_install_new_lock_info(&dep->qlock.lcnt, "dist_entry_out_queue", dep->sysname,
             ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
+        if (dep->mld)
+            erts_lcnt_install_new_lock_info(&dep->mld->mtx.lcnt, "dist_entry_links", dep->sysname,
+                                            ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION);
     } else {
         erts_lcnt_uninstall(&dep->rwmtx.lcnt);
-        erts_lcnt_uninstall(&dep->lnk_mtx.lcnt);
         erts_lcnt_uninstall(&dep->qlock.lcnt);
+        if (dep->mld)
+            erts_lcnt_uninstall(&dep->mld->mtx.lcnt);
     }
 }
 
diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index 489df08991..864c3044fb 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -1560,8 +1560,8 @@ handle_exit_signal(Process *c_p, ErtsSigRecvTracing *tracing,
 {
     ErtsMessage *conv_msg = NULL;
     ErtsExitSignalData *xsigd = NULL;
-    ErtsLinkData *ldp;
-    ErtsLink *dlnk;
+    ErtsLinkData *ldp = NULL; /* Avoid erroneous warning... */
+    ErtsLink *dlnk = NULL; /* Avoid erroneous warning... */
     Eterm tag = ((ErtsSignal *) sig)->common.tag;
     Uint16 type = ERTS_PROC_SIG_TYPE(tag);
     int op = ERTS_PROC_SIG_OP(tag);
@@ -1585,8 +1585,6 @@ handle_exit_signal(Process *c_p, ErtsSigRecvTracing *tracing,
                 /* Link no longer active; ignore... */
                 ignore = !0;
                 destroy = !0;
-                ldp = NULL; /* Avoid erroneous warning... */
-                dlnk = NULL; /* Avoid erroneous warning... */
             }
             else {
                 ignore = 0;
-- 
cgit v1.2.3


From 187564ffccee4a3b5727bdab3df2458f3b5ced72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 20 Mar 2018 13:28:26 +0100
Subject: Add enif_make_map_from_arrays

---
 erts/emulator/beam/erl_map.c                  |  4 ++-
 erts/emulator/beam/erl_nif.c                  | 38 ++++++++++++++++++++++
 erts/emulator/beam/erl_nif.h                  |  2 +-
 erts/emulator/beam/erl_nif_api_funcs.h        |  3 ++
 erts/emulator/test/nif_SUITE.erl              |  6 ++++
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 47 +++++++++++++++++++--------
 6 files changed, 85 insertions(+), 15 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 8047a9567f..4ec6960997 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -490,7 +490,9 @@ Eterm erts_map_from_ks_and_vs(ErtsHeapFactory *factory, Eterm *ks0, Eterm *vs0,
         sys_memcpy(ks, ks0, n * sizeof(Eterm));
         sys_memcpy(vs, vs0, n * sizeof(Eterm));
 
-        erts_validate_and_sort_flatmap(mp);
+        if (!erts_validate_and_sort_flatmap(mp)) {
+            return THE_NON_VALUE;
+        }
 
         return make_flatmap(mp);
     } else {
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 1d337d9792..f883b3f1e3 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2934,6 +2934,44 @@ ERL_NIF_TERM enif_make_new_map(ErlNifEnv* env)
     return make_flatmap(mp);
 }
 
+int enif_make_map_from_arrays(ErlNifEnv *env,
+                              ERL_NIF_TERM keys[],
+                              ERL_NIF_TERM values[],
+                              size_t cnt,
+                              ERL_NIF_TERM *map_out)
+{
+    ErtsHeapFactory factory;
+    int succeeded;
+
+#ifdef ERTS_NIF_ASSERT_IN_ENV
+    size_t index = 0;
+
+    while (index < cnt) {
+        ASSERT_IN_ENV(env, keys[index], index, "key");
+        ASSERT_IN_ENV(env, values[index], index, "value");
+        index++;
+    }
+#endif
+
+    flush_env(env);
+
+    erts_factory_proc_prealloc_init(&factory, env->proc,
+        cnt * 2 + MAP_HEADER_FLATMAP_SZ + 1);
+
+    (*map_out) = erts_map_from_ks_and_vs(&factory, keys, values, cnt);
+    succeeded = (*map_out) != THE_NON_VALUE;
+
+    if (!succeeded) {
+        erts_factory_undo(&factory);
+    }
+
+    erts_factory_close(&factory);
+
+    cache_env(env);
+
+    return succeeded;
+}
+
 int enif_make_map_put(ErlNifEnv* env,
 	              Eterm map_in,
 		      Eterm key,
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index e051ecb26e..1906da732b 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -53,7 +53,7 @@
 ** 2.12: 20.0 add enif_select, enif_open_resource_type_x
 ** 2.13: 20.1 add enif_ioq
 ** 2.14: 21.0 add enif_ioq_peek_head, enif_(mutex|cond|rwlock|thread)_name
-**                enif_vfprintf, enif_vsnprintf
+**                enif_vfprintf, enif_vsnprintf, enif_make_map_from_arrays
 */
 #define ERL_NIF_MAJOR_VERSION 2
 #define ERL_NIF_MINOR_VERSION 14
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 744f7c9f69..61f8fcf6ed 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -208,6 +208,8 @@ ERL_NIF_API_FUNC_DECL(char*,enif_thread_name,(ErlNifTid));
 ERL_NIF_API_FUNC_DECL(int,enif_vfprintf,(FILE*, const char *fmt, va_list));
 ERL_NIF_API_FUNC_DECL(int,enif_vsnprintf,(char*, size_t, const char *fmt, va_list));
 
+ERL_NIF_API_FUNC_DECL(int,enif_make_map_from_arrays,(ErlNifEnv *env, ERL_NIF_TERM keys[], ERL_NIF_TERM values[], size_t cnt, ERL_NIF_TERM *map_out));
+
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
 */
@@ -389,6 +391,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_vsnprintf,(char*, size_t, const char *fmt, va_lis
 #  define enif_thread_name ERL_NIF_API_FUNC_MACRO(enif_thread_name)
 #  define enif_vfprintf ERL_NIF_API_FUNC_MACRO(enif_vfprintf)
 #  define enif_vsnprintf ERL_NIF_API_FUNC_MACRO(enif_vsnprintf)
+#  define enif_make_map_from_arrays ERL_NIF_API_FUNC_MACRO(enif_make_map_from_arrays)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index b6e15ababb..a9eb4b2768 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -1170,6 +1170,12 @@ maps(Config) when is_list(Config) ->
     {1, M2} = make_map_remove_nif(M2, "key3"),
     {0, undefined} = make_map_remove_nif(self(), key),
 
+    M1 = maps_from_list_nif(maps:to_list(M1)),
+    M2 = maps_from_list_nif(maps:to_list(M2)),
+    M3 = maps_from_list_nif(maps:to_list(M3)),
+
+    has_duplicate_keys = maps_from_list_nif([{1,1},{1,1}]),
+
     verify_tmpmem(TmpMem),
     ok.
  
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index fa9ae1015c..e8d9302505 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -2140,24 +2140,45 @@ static ERL_NIF_TERM make_map_remove_nif(ErlNifEnv* env, int argc, const ERL_NIF_
 /* maps */
 static ERL_NIF_TERM maps_from_list_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    ERL_NIF_TERM cell = argv[0];
-    ERL_NIF_TERM map  = enif_make_new_map(env);
-    ERL_NIF_TERM tuple;
-    const ERL_NIF_TERM *pair;
-    int arity = -1;
+    ERL_NIF_TERM *keys, *values;
+    ERL_NIF_TERM result, cell;
+    unsigned count;
 
-    if (argc != 1 && !enif_is_list(env, cell)) return enif_make_badarg(env);
+    if (argc != 1 || !enif_get_list_length(env, argv[0], &count)) {
+        return enif_make_badarg(env);
+    }
 
-    /* assume sorted keys */
+    keys = enif_alloc(sizeof(ERL_NIF_TERM) * count * 2);
+    values = keys + count;
 
-    while (!enif_is_empty_list(env,cell)) {
-	if (!enif_get_list_cell(env, cell, &tuple, &cell)) return enif_make_badarg(env);
-	if (enif_get_tuple(env,tuple,&arity,&pair)) {
-	    enif_make_map_put(env, map, pair[0], pair[1], &map);
-	}
+    cell = argv[0];
+    count = 0;
+
+    while (!enif_is_empty_list(env, cell)) {
+        const ERL_NIF_TERM *pair;
+        ERL_NIF_TERM tuple;
+        int arity;
+
+        if (!enif_get_list_cell(env, cell, &tuple, &cell)
+            || !enif_get_tuple(env, tuple, &arity, &pair)
+            || arity != 2) {
+            enif_free(keys);
+            return enif_make_badarg(env);
+        }
+
+        keys[count] = pair[0];
+        values[count] = pair[1];
+
+        count++;
     }
 
-    return map;
+    if (!enif_make_map_from_arrays(env, keys, values, count, &result)) {
+        result = enif_make_atom(env, "has_duplicate_keys");
+    }
+
+    enif_free(keys);
+
+    return result;
 }
 
 static ERL_NIF_TERM sorted_list_from_maps_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
-- 
cgit v1.2.3


From d77f0bdb55fd1bd36c4f19ab78cb02177d365b12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 15 Mar 2018 10:00:07 +0100
Subject: Correct overflow in ERTS_SINT64_MIN and add macros for 16/32

This also removes a redundant #ifdef soup that had hidden the
problem on most platforms.
---
 erts/emulator/beam/sys.h | 40 +++++++++++++++++++++-------------------
 1 file changed, 21 insertions(+), 19 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index c21acadd8d..be6ab57eeb 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -366,29 +366,11 @@ typedef UWord BeamInstr;
 #    define HAVE_INT64 1
 typedef unsigned long Uint64;
 typedef long          Sint64;
-#    ifdef ULONG_MAX
-#      define ERTS_UINT64_MAX ULONG_MAX
-#    endif
-#    ifdef LONG_MAX
-#      define ERTS_SINT64_MAX LONG_MAX
-#    endif
-#    ifdef LONG_MIN
-#      define ERTS_SINT64_MIN LONG_MIN
-#    endif
 #    define ErtsStrToSint64 strtol
 #  elif SIZEOF_LONG_LONG == 8
 #    define HAVE_INT64 1
 typedef unsigned long long Uint64;
 typedef long long          Sint64;
-#    ifdef ULLONG_MAX
-#      define ERTS_UINT64_MAX ULLONG_MAX
-#    endif
-#    ifdef LLONG_MAX
-#      define ERTS_SINT64_MAX LLONG_MAX
-#    endif
-#    ifdef LLONG_MIN
-#      define ERTS_SINT64_MIN LLONG_MIN
-#    endif
 #    define ErtsStrToSint64 strtoll
 #  else
 #    error "No 64-bit integer type found"
@@ -402,7 +384,7 @@ typedef long long          Sint64;
 #  define ERTS_SINT64_MAX ((Sint64) ((((Uint64) 1) << 63)-1))
 #endif
 #ifndef ERTS_SINT64_MIN
-#  define ERTS_SINT64_MIN (-1*(((Sint64) 1) << 63))
+#  define ERTS_SINT64_MIN ((Sint64) ((((Uint64) 1) << 63)))
 #endif
 
 #if SIZEOF_LONG == 4
@@ -415,6 +397,16 @@ typedef int          Sint32;
 #error Found no appropriate type to use for 'Uint32' and 'Sint32'
 #endif
 
+#ifndef ERTS_UINT32_MAX
+#  define ERTS_UINT32_MAX (~((Uint32) 0))
+#endif
+#ifndef ERTS_SINT32_MAX
+#  define ERTS_SINT32_MAX ((Sint32) ((((Uint32) 1) << 31)-1))
+#endif
+#ifndef ERTS_SINT32_MIN
+#  define ERTS_SINT32_MIN ((Sint32) ((((Uint32) 1) << 31)))
+#endif
+
 #if SIZEOF_INT == 2
 typedef unsigned int Uint16;
 typedef int          Sint16;
@@ -425,6 +417,16 @@ typedef short          Sint16;
 #error Found no appropriate type to use for 'Uint16' and 'Sint16'
 #endif
 
+#ifndef ERTS_UINT16_MAX
+#  define ERTS_UINT16_MAX (~((Uint16) 0))
+#endif
+#ifndef ERTS_SINT16_MAX
+#  define ERTS_SINT16_MAX ((Sint16) ((((Uint16) 1) << 15)-1))
+#endif
+#ifndef ERTS_SINT16_MIN
+#  define ERTS_SINT16_MIN ((Sint16) ((((Uint16) 1) << 15)))
+#endif
+
 #if CHAR_BIT == 8
 typedef unsigned char byte;
 #else
-- 
cgit v1.2.3


From 06ed628dfd013010dd6e182508c1137b9f4ba09b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 26 Feb 2018 12:49:57 +0100
Subject: Lift the type restrictions on seq_trace token labels

OTP-14899
---
 erts/emulator/beam/dist.c               | 68 +++++++++++++++++++++------------
 erts/emulator/beam/dist.h               |  6 ++-
 erts/emulator/beam/erl_bif_trace.c      |  3 --
 erts/emulator/beam/erl_db_util.c        | 33 +++++++---------
 erts/emulator/beam/erl_message.c        |  7 ++--
 erts/emulator/beam/erl_proc_sig_queue.c |  2 +-
 erts/emulator/beam/erl_process.h        | 17 +++++++++
 erts/emulator/beam/erlang_dtrace.d      | 15 +++++---
 erts/emulator/beam/msg_instrs.tab       |  2 +-
 9 files changed, 95 insertions(+), 58 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index fdf307da1b..f203d85ca9 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -937,6 +937,24 @@ erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher,
     return res;
 }
 
+static int can_send_seqtrace_token(ErtsSendContext* ctx, Eterm token) {
+    Eterm label;
+
+    if (ctx->dep->flags & DFLAG_BIG_SEQTRACE_LABELS) {
+        /* The other end is capable of handling arbitrary seq_trace labels. */
+        return 1;
+    }
+
+    /* The other end only tolerates smalls, but since we could potentially be
+     * talking to an old 32-bit emulator from a 64-bit one, we have to check
+     * whether the label is small on any emulator. */
+    label = SEQ_TRACE_T_LABEL(token);
+
+    return is_small(label) &&
+        signed_val(label) <= (ERTS_SINT32_MAX >> _TAG_IMMED1_SIZE) &&
+        signed_val(label) >= (ERTS_SINT32_MIN >> _TAG_IMMED1_SIZE);
+}
+
 int
 erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx)
 {
@@ -970,37 +988,38 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx)
                       "%T", remote);
         msize = size_object(message);
         if (have_seqtrace(token)) {
-            tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
+            tok_label = SEQ_TRACE_T_DTRACE_LABEL(token);
             tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
             tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
         }
     }
 #endif
 
-    if (token != NIL) {
-        Eterm el1, el2;
-        if (ctx->dep->flags & DFLAG_SEND_SENDER) {
-            el1 = make_small(DOP_SEND_SENDER_TT);
-            el2 = sender->common.id;
-        }
-        else {
-            el1 = make_small(DOP_SEND_TT);
-            el2 = am_Empty;
-        }
-	ctl = TUPLE4(&ctx->ctl_heap[0], el1, el2, remote, token);
-    }
-    else {
-        Eterm el1, el2;
+    {
+        Eterm dist_op, sender_id;
+        int send_token;
+
+        send_token = (token != NIL && can_send_seqtrace_token(ctx, token));
+
         if (ctx->dep->flags & DFLAG_SEND_SENDER) {
-            el1 = make_small(DOP_SEND_SENDER);
-            el2 = sender->common.id;
+            dist_op = make_small(send_token ?
+                                 DOP_SEND_SENDER_TT :
+                                 DOP_SEND_SENDER);
+            sender_id = sender->common.id;
+        } else {
+            dist_op = make_small(send_token ?
+                                 DOP_SEND_TT :
+                                 DOP_SEND);
+            sender_id = am_Empty;
         }
-        else {
-            el1 = make_small(DOP_SEND);
-            el2 = am_Empty;
+
+        if (send_token) {
+            ctl = TUPLE4(&ctx->ctl_heap[0], dist_op, sender_id, remote, token);
+        } else {
+            ctl = TUPLE3(&ctx->ctl_heap[0], dist_op, sender_id, remote);
         }
-	ctl = TUPLE3(&ctx->ctl_heap[0], el1, el2, remote);
     }
+
     DTRACE6(message_send, sender_name, receiver_name,
             msize, tok_label, tok_lastcnt, tok_serial);
     DTRACE7(message_send_remote, sender_name, node_name, receiver_name,
@@ -1046,19 +1065,20 @@ erts_dsig_send_reg_msg(Eterm remote_name, Eterm message,
                       "{%T,%s}", remote_name, node_name);
         msize = size_object(message);
         if (have_seqtrace(token)) {
-            tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
+            tok_label = SEQ_TRACE_T_DTRACE_LABEL(token);
             tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
             tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
         }
     }
 #endif
 
-    if (token != NIL)
+    if (token != NIL && can_send_seqtrace_token(ctx, token))
 	ctl = TUPLE5(&ctx->ctl_heap[0], make_small(DOP_REG_SEND_TT),
 		     sender->common.id, am_Empty, remote_name, token);
     else
 	ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_REG_SEND),
 		     sender->common.id, am_Empty, remote_name);
+
     DTRACE6(message_send, sender_name, receiver_name,
             msize, tok_label, tok_lastcnt, tok_serial);
     DTRACE7(message_send_remote, sender_name, node_name, receiver_name,
@@ -1110,7 +1130,7 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote,
         erts_snprintf(reason_str, sizeof(DTRACE_CHARBUF_NAME(reason_str)),
                       "%T", reason);
         if (have_seqtrace(token)) {
-            tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
+            tok_label = SEQ_TRACE_T_DTRACE_LABEL(token);
             tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
             tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
         }
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index c608fef816..dda2029a4c 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -45,7 +45,8 @@
 #define DFLAG_MAP_TAG             0x20000
 #define DFLAG_BIG_CREATION        0x40000
 #define DFLAG_SEND_SENDER         0x80000
-#define DFLAG_NO_MAGIC            0x100000 /* internal for pending connection */
+#define DFLAG_BIG_SEQTRACE_LABELS 0x100000
+#define DFLAG_NO_MAGIC            0x200000 /* internal for pending connection */
 
 /* Mandatory flags for distribution */
 #define DFLAG_DIST_MANDATORY (DFLAG_EXTENDED_REFERENCES         \
@@ -73,7 +74,8 @@
                             | DFLAG_UTF8_ATOMS                \
                             | DFLAG_MAP_TAG                   \
                             | DFLAG_BIG_CREATION              \
-                            | DFLAG_SEND_SENDER)
+                            | DFLAG_SEND_SENDER               \
+                            | DFLAG_BIG_SEQTRACE_LABELS)
 
 /* Flags addable by local distr implementations */
 #define DFLAG_DIST_ADDABLE    DFLAG_DIST_DEFAULT
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index a076f0bf54..1953f79d79 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1807,9 +1807,6 @@ Eterm erts_seq_trace(Process *p, Eterm arg1, Eterm arg2,
 	return old_value;
     }
     else if (arg1 == am_label) {
-	if (! is_small(arg2)) {
-	    return THE_NON_VALUE;
-	}
         new_seq_trace_token(p);
 	if (build_result) {
 	    old_value = SEQ_TRACE_TOKEN_LABEL(p);
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 1e8e9e5e94..3836f28aa4 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2508,25 +2508,20 @@ restart:
             if (have_no_seqtrace(SEQ_TRACE_TOKEN(c_p)))
 		*esp++ = NIL;
 	    else {
-		Eterm sender = SEQ_TRACE_TOKEN_SENDER(c_p);
-		Uint sender_sz = is_immed(sender) ? 0 : size_object(sender);
-		ehp = HAllocX(build_proc, 6 + sender_sz, HEAP_XTRA);
-		if (sender_sz) {
-		    sender = copy_struct(sender, sender_sz, &ehp, &MSO(build_proc));
-		}
- 		*esp++ = make_tuple(ehp);
- 		ehp[0] = make_arityval(5);
- 		ehp[1] = SEQ_TRACE_TOKEN_FLAGS(c_p);
- 		ehp[2] = SEQ_TRACE_TOKEN_LABEL(c_p);
- 		ehp[3] = SEQ_TRACE_TOKEN_SERIAL(c_p);
-		ehp[4] = sender;
- 		ehp[5] = SEQ_TRACE_TOKEN_LASTCNT(c_p);
-		ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5);
-		ASSERT(is_immed(ehp[1]));
-		ASSERT(is_immed(ehp[2]));
-		ASSERT(is_immed(ehp[3]));
-		ASSERT(is_immed(ehp[5]));
-	    } 
+                Eterm token;
+                Uint token_sz;
+
+                ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5);
+                ASSERT(is_immed(SEQ_TRACE_TOKEN_FLAGS(c_p)));
+                ASSERT(is_immed(SEQ_TRACE_TOKEN_SERIAL(c_p)));
+                ASSERT(is_immed(SEQ_TRACE_TOKEN_LASTCNT(c_p)));
+
+                token = SEQ_TRACE_TOKEN(c_p);
+                token_sz = size_object(token);
+
+                ehp = HAllocX(build_proc, token_sz, HEAP_XTRA);
+                *esp++ = copy_struct(token, token_sz, &ehp, &MSO(build_proc));
+	    }
 	    break;
         case matchEnableTrace:
             ASSERT(c_p == self);
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index ed7a9d37c2..0565e01f06 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -315,7 +315,7 @@ erts_queue_dist_message(Process *rcvr,
 
             dtrace_proc_str(rcvr, receiver_name);
             if (have_seqtrace(token)) {
-                tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
+                tok_label = SEQ_TRACE_T_DTRACE_LABEL(token);
                 tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
                 tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
             }
@@ -640,7 +640,8 @@ erts_send_message(Process* sender,
 	    seq_trace_update_send(sender);
 	    seq_trace_output(stoken, message, SEQ_TRACE_SEND,
 			     receiver->common.id, sender);
-	    seq_trace_size = 6; /* TUPLE5 */
+
+	    seq_trace_size = size_object(stoken);
 	}
 #ifdef USE_VM_PROBES
         if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
@@ -689,7 +690,7 @@ erts_send_message(Process* sender,
 	}
         if (DTRACE_ENABLED(message_send)) {
             if (have_seqtrace(stoken)) {
-		tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken));
+                tok_label = SEQ_TRACE_T_DTRACE_LABEL(stoken);
 		tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken));
 		tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken));
 	    }
diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index 1b5cbb1919..f07bd32723 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -3069,7 +3069,7 @@ handle_message_enqueued_tracing(Process *c_p,
         Eterm seq_trace_token = ERL_MESSAGE_TOKEN(msg);
 
         if (seq_trace_token != NIL && is_tuple(seq_trace_token)) {
-            tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token));
+            tok_label = SEQ_TRACE_T_DTRACE_LABEL(seq_trace_token);
             tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token));
             tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token));
         }
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 0550fb05b5..256b8b7d16 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1250,7 +1250,24 @@ void erts_check_for_holes(Process* p);
 #define SEQ_TRACE_T_SENDER(token)   (*(tuple_val(token) + 4))
 #define SEQ_TRACE_T_LASTCNT(token)  (*(tuple_val(token) + 5))
 
+#ifdef USE_VM_PROBES
+/* The dtrace probe for seq_trace only supports 'int' labels, so we represent
+ * all values that won't fit into a 32-bit signed integer as ERTS_SINT32_MIN
+ * (bigints, tuples, etc). */
+
+#define SEQ_TRACE_T_DTRACE_LABEL(token) \
+    DTRACE_SEQ_TRACE_LABEL__(SEQ_TRACE_T_LABEL(token))
+
+#define DTRACE_SEQ_TRACE_LABEL__(label_term) \
+    (is_small((label_term)) ? \
+        ((signed_val((label_term)) <= ERTS_SINT32_MAX && \
+          signed_val((label_term)) >= ERTS_SINT32_MIN) ? \
+             signed_val((label_term)) : ERTS_SINT32_MIN) \
+        : ERTS_SINT32_MIN)
+#endif
+
 /*
+
  * Possible flags for the flags field in ErlSpawnOpts below.
  */
 
diff --git a/erts/emulator/beam/erlang_dtrace.d b/erts/emulator/beam/erlang_dtrace.d
index d0b10e0306..c47a37eb62 100644
--- a/erts/emulator/beam/erlang_dtrace.d
+++ b/erts/emulator/beam/erlang_dtrace.d
@@ -55,7 +55,8 @@ provider erlang {
      * @param sender the PID (string form) of the sender
      * @param receiver the PID (string form) of the receiver
      * @param size the size of the message being delivered (words)
-     * @param token_label for the sender's sequential trace token
+     * @param token_label for the sender's sequential trace token. This will be
+     *        INT_MIN if the label does not fit into a 32-bit integer.
      * @param token_previous count for the sender's sequential trace token
      * @param token_current count for the sender's sequential trace token
      */
@@ -73,7 +74,8 @@ provider erlang {
      * @param node_name the Erlang node name (string form) of the receiver
      * @param receiver the PID/name (string form) of the receiver
      * @param size the size of the message being delivered (words)
-     * @param token_label for the sender's sequential trace token
+     * @param token_label for the sender's sequential trace token. This will be
+     *        INT_MIN if the label does not fit into a 32-bit integer.
      * @param token_previous count for the sender's sequential trace token
      * @param token_current count for the sender's sequential trace token
      */
@@ -98,7 +100,8 @@ provider erlang {
      * @param receiver the PID (string form) of the receiver
      * @param size the size of the message being delivered (words)
      * @param queue_len length of the queue of the receiving process
-     * @param token_label for the sender's sequential trace token
+     * @param token_label for the sender's sequential trace token. This will be
+     *        INT_MIN if the label does not fit into a 32-bit integer.
      * @param token_previous count for the sender's sequential trace token
      * @param token_current count for the sender's sequential trace token
      */
@@ -122,7 +125,8 @@ provider erlang {
      * @param receiver the PID (string form) of the receiver
      * @param size the size of the message being delivered (words)
      * @param queue_len length of the queue of the receiving process
-     * @param token_label for the sender's sequential trace token
+     * @param token_label for the sender's sequential trace token. This will be
+     *        INT_MIN if the label does not fit into a 32-bit integer.
      * @param token_previous count for the sender's sequential trace token
      * @param token_current count for the sender's sequential trace token
      */
@@ -273,7 +277,8 @@ provider erlang {
      * @param node_name the Erlang node name (string form) of the receiver
      * @param receiver the PID (string form) of the process receiving EXIT signal
      * @param reason the reason for the exit (may be truncated)
-     * @param token_label for the sender's sequential trace token
+     * @param token_label for the sender's sequential trace token. This will be
+     *        INT_MIN if the label does not fit into a 32-bit integer.
      * @param token_previous count for the sender's sequential trace token
      * @param token_current count for the sender's sequential trace token
      */
diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab
index 88d2ef9fa3..58cad9c666 100644
--- a/erts/emulator/beam/msg_instrs.tab
+++ b/erts/emulator/beam/msg_instrs.tab
@@ -229,7 +229,7 @@ remove_message() {
         dtrace_proc_str(c_p, receiver_name);
         token2 = SEQ_TRACE_TOKEN(c_p);
         if (have_seqtrace(token2)) {
-            tok_label = signed_val(SEQ_TRACE_T_LABEL(token2));
+            tok_label = SEQ_TRACE_T_DTRACE_LABEL(token2);
             tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2));
             tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2));
         }
-- 
cgit v1.2.3


From 83a289d4dff156f2c7203843a1437256545f5580 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 21 Mar 2018 14:16:15 +0100
Subject: Fix VM probes compilation

---
 erts/emulator/beam/erl_message.c        |  2 +-
 erts/emulator/beam/erl_proc_sig_queue.c | 51 +++++++++++++++++++++++++--------
 erts/emulator/beam/io.c                 | 27 +++++++++--------
 erts/emulator/beam/msg_instrs.tab       |  2 +-
 4 files changed, 56 insertions(+), 26 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index ed7a9d37c2..98feb95d99 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -323,7 +323,7 @@ erts_queue_dist_message(Process *rcvr,
              * TODO: We don't know the real size of the external message here.
              *       -1 will appear to a D script as 4294967295.
              */
-            DTRACE6(message_queued, receiver_name, -1, rcvr->msg.len + 1,
+            DTRACE6(message_queued, receiver_name, -1, rcvr->sig_qs.len + 1,
                     tok_label, tok_lastcnt, tok_serial);
         }
 #endif
diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index 864c3044fb..b4759967a7 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -365,20 +365,47 @@ sig_enqueue_trace(Process *c_p, ErtsMessage *sig, int op,
 
 #ifdef USE_VM_PROBES
     case ERTS_SIG_Q_OP_EXIT:
-    case ERTS_SIG_Q_OP_EXIT_LINKED: {
-        ErtsExitSignalData *xsigd = get_exit_signal_data(sig);
-        if(DTRACE_ENABLED(process_exit_signal) && is_pid(xsigd->from)) {
-            DTRACE_CHARBUF(sender_str, DTRACE_TERM_BUF_SIZE);
-            DTRACE_CHARBUF(receiver_str, DTRACE_TERM_BUF_SIZE);
-            DTRACE_CHARBUF(reason_buf, DTRACE_TERM_BUF_SIZE);
+    case ERTS_SIG_Q_OP_EXIT_LINKED:
+
+        if (DTRACE_ENABLED(process_exit_signal)) {
+            Uint16 type = ERTS_PROC_SIG_TYPE(((ErtsSignal *) sig)->common.tag);
+            Eterm reason, from;
+
+            if (type == ERTS_SIG_Q_TYPE_GEN_EXIT) {
+                ErtsExitSignalData *xsigd = get_exit_signal_data(sig);
+                reason = xsigd->reason;
+                from = xsigd->from;
+            }
+            else {
+                ErtsLink *lnk = (ErtsLink *) sig, *olnk;
+
+                ASSERT(type == ERTS_LNK_TYPE_PROC
+                       || type == ERTS_LNK_TYPE_PORT
+                       || type == ERTS_LNK_TYPE_DIST_PROC);
+
+                olnk = erts_link_to_other(lnk, NULL);
+                reason = lnk->other.item;
+                from = olnk->other.item;
+            }
 
-            dtrace_pid_str(from, sender_str);
-            dtrace_proc_str(rp, receiver_str);
-            erts_snprintf(reason_buf, sizeof(DTRACE_CHARBUF_NAME(reason_buf)) - 1, "%T", reason);
-            DTRACE3(process_exit_signal, sender_str, receiver_str, reason_buf);
+            if (is_pid(from)) {
+
+                DTRACE_CHARBUF(sender_str, DTRACE_TERM_BUF_SIZE);
+                DTRACE_CHARBUF(receiver_str, DTRACE_TERM_BUF_SIZE);
+                DTRACE_CHARBUF(reason_buf, DTRACE_TERM_BUF_SIZE);
+
+                if (reason == am_kill) {
+                    reason = am_killed;
+                }
+
+                dtrace_pid_str(from, sender_str);
+                dtrace_proc_str(rp, receiver_str);
+                erts_snprintf(reason_buf, sizeof(DTRACE_CHARBUF_NAME(reason_buf)) - 1, "%T", reason);
+                DTRACE3(process_exit_signal, sender_str, receiver_str, reason_buf);
+            }
         }
         break;
-    }
+
 #endif
 
     default:
@@ -808,7 +835,7 @@ send_gen_exit_signal(Process *c_p, Eterm from_tag,
     s_utag = (is_immed(utag)
               ? utag
               : copy_struct(utag, utag_sz, &hp, ohp));
-    ERL_MESSAGE_DT_UTAG(mp) = utag;
+    ERL_MESSAGE_DT_UTAG(mp) = s_utag;
 #endif
 
     ERL_MESSAGE_TERM(mp) = ERTS_PROC_SIG_MAKE_TAG(op,
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index b2afdc6bf2..9f87285b71 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -2361,6 +2361,21 @@ set_port_connected(int bang_op,
                 trace_port(prt, am_getting_linked, connect);
         }
 
+#ifdef USE_VM_PROBES
+	if (DTRACE_ENABLED(port_connect)) {
+            Eterm old_connected = ERTS_PORT_GET_CONNECTED(prt);
+            DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE);
+            DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);
+            DTRACE_CHARBUF(newprocess_str, DTRACE_TERM_BUF_SIZE);
+
+            dtrace_pid_str(old_connected, process_str);
+            erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
+                          "%T", prt->common.id);
+            dtrace_pid_str(connect, newprocess_str);
+            DTRACE4(port_connect, process_str, port_str, prt->name, newprocess_str);
+	}
+#endif
+
 	ERTS_PORT_SET_CONNECTED(prt, connect);
 
         if (IS_TRACED_FL(prt, F_TRACE_RECEIVE))
@@ -2370,18 +2385,6 @@ set_port_connected(int bang_op,
             trace_port_send(prt, from, TUPLE2(hp, prt->common.id, am_connected), 1);
         }
 
-#ifdef USE_VM_PROBES
-	if (DTRACE_ENABLED(port_connect)) {
-	    DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE);
-	    DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);
-	    DTRACE_CHARBUF(newprocess_str, DTRACE_TERM_BUF_SIZE);
-
-	    dtrace_pid_str(connect, process_str);
-	    erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id);
-	    dtrace_proc_str(rp, newprocess_str);
-	    DTRACE4(port_connect, process_str, port_str, prt->name, newprocess_str);
-	}
-#endif
     }
 
     return ERTS_PORT_OP_DONE;
diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab
index 88d2ef9fa3..6055e35717 100644
--- a/erts/emulator/beam/msg_instrs.tab
+++ b/erts/emulator/beam/msg_instrs.tab
@@ -235,7 +235,7 @@ remove_message() {
         }
         DTRACE6(message_receive,
                 receiver_name, size_object(ERL_MESSAGE_TERM(msgp)),
-                c_p->msg.len - 1, tok_label, tok_lastcnt, tok_serial);
+                c_p->sig_qs.len - 1, tok_label, tok_lastcnt, tok_serial);
     }
 #endif
     UNLINK_MESSAGE(c_p, msgp);
-- 
cgit v1.2.3


From e1aa6fab326bdb443f14a08727a33fa52cbb573a Mon Sep 17 00:00:00 2001
From: Raimo Niskanen <raimo@erlang.org>
Date: Fri, 23 Mar 2018 14:43:48 +0100
Subject: Update types for posix error codes

I have read the man pages for most socket and file operations
on recent Linux, FreeBSD, OpenBSD and Solaris 10 and noted
the possible error codes.

Which error codes that are possible for file operations have
been updated in file:posix/0.  Error codes for socket operations
in inet:posix/0.  The latter refers to the former so it is
a superset, assuming that e.g sendfile and AF_UNIX socket
operations could cause socket operations to return any file
error code.  That is not entirely true, but could be,
especially in the future.

Added to file:posix/0 are:
  ebadmsg edeadlk edeadlock eftype emultihop enobufs enolck enolink
  enosr enostr enosys eopnotsupp eoverflow erange etxtbsy

Added to inet:posix/0 are all but:
  exbadport exbadseq file:posix()

These are still possible according to erl_posix_str.c,
but are not in file:posix/0 nor in inet:posix/0,
and many of them are not file nor inet related, but some might be:
  e2big eadv ealign ebade ebadfd ebadr ebadrpc ebadrqc
  ebadslt ebfont echild echrng ecomm edirty
  edom edotdot eduppkg eidrm einit eisnam
  elbin el2hlt el2nsync el3hlt el3rst
  elibacc elibbad elibexec elibmax elibscn elnrng
  enavail enet enoano enocsi enodata enoexec
  enonet enosym enotempty enotnam enotuniq
  eproclim eprocunavail eprogmismatch eprogunavail
  erefused eremchg eremdev eremote eremoteio
  eremoterelease erpcmismatch erremote eshutdown
  esrmnt esuccesss etime etoomanyrefs
  euclean eunatch eusers eversion exfull
  sysnotready vernotsupported ediscon enomore
  ecancelled einvalidproctable einvalidprovider eproviderfailedinit
  syscallfailure service_not_found type_not_found e_no_more
  e_cancelled unknown
---
 erts/emulator/beam/erl_posix_str.c | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_posix_str.c b/erts/emulator/beam/erl_posix_str.c
index deb7e3e173..7b3e640d3f 100644
--- a/erts/emulator/beam/erl_posix_str.c
+++ b/erts/emulator/beam/erl_posix_str.c
@@ -156,6 +156,9 @@ erl_errno_id(error)
 #ifdef EFAULT
     case EFAULT: return "efault";
 #endif
+#ifdef EFTYPE
+    case EFTYPE: return "eftype";
+#endif
 #ifdef EFBIG
     case EFBIG: return "efbig";
 #endif
@@ -351,6 +354,9 @@ erl_errno_id(error)
 #if defined(EOPNOTSUPP) && (!defined(ENOTSUP) || (EOPNOTSUPP != ENOTSUP))
     case EOPNOTSUPP: return "eopnotsupp";
 #endif
+#ifdef EOVERFLOW
+    case EOVERFLOW: return "eoverflow";
+#endif
 #ifdef EPERM
     case EPERM: return "eperm";
 #endif
-- 
cgit v1.2.3


From f278cee87dd7ef929a64b411b4e83c3d7028687a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Fri, 23 Mar 2018 13:15:08 +0100
Subject: Add +sbwt/+swt analogues for dirty schedulers

Sharing these settings for all schedulers can degrade performance,
so it makes sense to be able to configure them separately.

This also changes the default busy-wait time to "short" for both
kinds of dirty schedulers.
---
 erts/emulator/beam/erl_init.c           |  64 +++++++++-
 erts/emulator/beam/erl_process.c        | 201 +++++++++++++++++++-------------
 erts/emulator/beam/erl_process.h        |  15 ++-
 erts/emulator/test/smoke_test_SUITE.erl |  14 +++
 4 files changed, 200 insertions(+), 94 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 1e20d48a73..179822cc0b 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -605,6 +605,10 @@ void erts_usage(void)
     erts_fprintf(stderr, "-stbt type     u|ns|ts|ps|s|nnts|nnps|tnnps|db\n");
     erts_fprintf(stderr, "-sbwt val      set scheduler busy wait threshold, valid values are:\n");
     erts_fprintf(stderr, "               none|very_short|short|medium|long|very_long.\n");
+    erts_fprintf(stderr, "-sbwtdcpu val  set dirty CPU scheduler busy wait threshold, valid values are:\n");
+    erts_fprintf(stderr, "               none|very_short|short|medium|long|very_long.\n");
+    erts_fprintf(stderr, "-sbwtdio val   set dirty IO scheduler busy wait threshold, valid values are:\n");
+    erts_fprintf(stderr, "               none|very_short|short|medium|long|very_long.\n");
     erts_fprintf(stderr, "-scl bool      enable/disable compaction of scheduler load,\n");
     erts_fprintf(stderr, "               see the erl(1) documentation for more info.\n");
     erts_fprintf(stderr, "-sct cput      set cpu topology,\n");
@@ -623,6 +627,10 @@ void erts_usage(void)
     erts_fprintf(stderr, "               very_lazy|lazy|medium|eager|very_eager.\n");
     erts_fprintf(stderr, "-swt val       set scheduler wakeup threshold, valid values are:\n");
     erts_fprintf(stderr, "               very_low|low|medium|high|very_high.\n");
+    erts_fprintf(stderr, "-swtdcpu val   set dirty CPU scheduler wakeup threshold, valid values are:\n");
+    erts_fprintf(stderr, "               very_low|low|medium|high|very_high.\n");
+    erts_fprintf(stderr, "-swtdio val    set dirty IO scheduler wakeup threshold, valid values are:\n");
+    erts_fprintf(stderr, "               very_low|low|medium|high|very_high.\n");
     erts_fprintf(stderr, "-sss size      suggested stack size in kilo words for scheduler threads,\n");
     erts_fprintf(stderr, "               valid range is [%d-%d] (default %d)\n",
 		 ERTS_SCHED_THREAD_MIN_STACK_SIZE,
@@ -1687,15 +1695,41 @@ erl_start(int argc, char **argv)
 		    erts_usage();
 		}
 	    }
+	    else if (has_prefix("bwtdcpu", sub_param)) {
+                arg = get_arg(sub_param + 7, argv[i+1], &i);
+
+		if (erts_sched_set_busy_wait_threshold(ERTS_SCHED_DIRTY_CPU, arg) != 0) {
+		    erts_fprintf(stderr, "bad dirty CPU scheduler busy wait threshold: %s\n",
+				 arg);
+		    erts_usage();
+		}
+
+		VERBOSE(DEBUG_SYSTEM,
+			("dirty CPU scheduler wakeup threshold: %s\n", arg));
+	    }
+	    else if (has_prefix("bwtdio", sub_param)) {
+                arg = get_arg(sub_param + 6, argv[i+1], &i);
+
+		if (erts_sched_set_busy_wait_threshold(ERTS_SCHED_DIRTY_IO, arg) != 0) {
+		    erts_fprintf(stderr, "bad dirty IO scheduler busy wait threshold: %s\n",
+				 arg);
+		    erts_usage();
+		}
+
+		VERBOSE(DEBUG_SYSTEM,
+			("dirty IO scheduler wakeup threshold: %s\n", arg));
+	    }
 	    else if (has_prefix("bwt", sub_param)) {
-		arg = get_arg(sub_param+3, argv[i+1], &i);
-		if (erts_sched_set_busy_wait_threshold(arg) != 0) {
+                arg = get_arg(sub_param + 3, argv[i+1], &i);
+
+		if (erts_sched_set_busy_wait_threshold(ERTS_SCHED_NORMAL, arg) != 0) {
 		    erts_fprintf(stderr, "bad scheduler busy wait threshold: %s\n",
 				 arg);
 		    erts_usage();
 		}
+
 		VERBOSE(DEBUG_SYSTEM,
-			("scheduler wakup threshold: %s\n", arg));
+			("scheduler wakeup threshold: %s\n", arg));
 	    }
 	    else if (has_prefix("cl", sub_param)) {
 		arg = get_arg(sub_param+2, argv[i+1], &i);
@@ -1812,9 +1846,29 @@ erl_start(int argc, char **argv)
 		VERBOSE(DEBUG_SYSTEM,
 			("scheduler wake cleanup threshold: %s\n", arg));
 	    }
+	    else if (has_prefix("wtdcpu", sub_param)) {
+		arg = get_arg(sub_param+6, argv[i+1], &i);
+		if (erts_sched_set_wakeup_other_threshold(ERTS_SCHED_DIRTY_CPU, arg) != 0) {
+		    erts_fprintf(stderr, "dirty CPU scheduler wakeup threshold: %s\n",
+				 arg);
+		    erts_usage();
+		}
+		VERBOSE(DEBUG_SYSTEM,
+			("dirty CPU scheduler wakeup threshold: %s\n", arg));
+	    }
+	    else if (has_prefix("wtdio", sub_param)) {
+		arg = get_arg(sub_param+5, argv[i+1], &i);
+		if (erts_sched_set_wakeup_other_threshold(ERTS_SCHED_DIRTY_IO, arg) != 0) {
+		    erts_fprintf(stderr, "dirty IO scheduler wakeup threshold: %s\n",
+				 arg);
+		    erts_usage();
+		}
+		VERBOSE(DEBUG_SYSTEM,
+			("dirty IO scheduler wakeup threshold: %s\n", arg));
+	    }
 	    else if (has_prefix("wt", sub_param)) {
 		arg = get_arg(sub_param+2, argv[i+1], &i);
-		if (erts_sched_set_wakeup_other_thresold(arg) != 0) {
+		if (erts_sched_set_wakeup_other_threshold(ERTS_SCHED_NORMAL, arg) != 0) {
 		    erts_fprintf(stderr, "scheduler wakeup threshold: %s\n",
 				 arg);
 		    erts_usage();
@@ -1824,7 +1878,7 @@ erl_start(int argc, char **argv)
 	    }
 	    else if (has_prefix("ws", sub_param)) {
 		arg = get_arg(sub_param+2, argv[i+1], &i);
-		if (erts_sched_set_wakeup_other_type(arg) != 0) {
+		if (erts_sched_set_wakeup_other_type(ERTS_SCHED_NORMAL, arg) != 0) {
 		    erts_fprintf(stderr, "scheduler wakeup strategy: %s\n",
 				 arg);
 		    erts_usage();
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 7969025f57..9f02c34742 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -172,11 +172,19 @@ int erts_dio_sched_thread_suggested_stack_size = -1;
 ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];
 #endif
 
-static struct ErtsSchedBusyWait_ {
+typedef struct {
     int aux_work;
     int tse;
     int sys_schedule;
-} sched_busy_wait;
+} ErtsBusyWaitParams;
+
+static ErtsBusyWaitParams sched_busy_wait_params[ERTS_SCHED_TYPE_LAST + 1];
+
+static ERTS_INLINE ErtsBusyWaitParams *
+sched_get_busy_wait_params(ErtsSchedulerData *esdp)
+{
+    return &sched_busy_wait_params[esdp->type];
+}
 
 int erts_disable_proc_not_running_opt;
 
@@ -3262,7 +3270,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
     erts_runq_unlock(rq);
 
-    spincount = sched_busy_wait.tse;
+    spincount = sched_get_busy_wait_params(esdp)->tse;
 
     if (ERTS_SCHEDULER_IS_DIRTY(esdp))
         dirty_sched_wall_time_change(esdp, working = 0);
@@ -3364,7 +3372,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
         }
 
         flgs = sched_prep_cont_spin_wait(ssi);
-        spincount = sched_busy_wait.aux_work;
+        spincount = sched_get_busy_wait_params(esdp)->aux_work;
 
         if (!(flgs & ERTS_SSI_FLG_WAITING)) {
             ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
@@ -5250,24 +5258,35 @@ typedef enum {
 #define ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY (CONTEXT_REDS/10)
 
 
-static struct {
+typedef struct {
     ErtsSchedWakeupOtherThreshold threshold;
     ErtsSchedWakeupOtherType type;
     int limit;
     int dec_shift;
     int dec_mask;
     void (*check)(ErtsRunQueue *rq, Uint32 flags);
-} wakeup_other;
+} ErtsWakeupOtherParams;
+
+static ErtsWakeupOtherParams sched_wakeup_other_params[ERTS_SCHED_TYPE_LAST + 1];
+
+static ERTS_INLINE ErtsWakeupOtherParams *
+runq_get_wakeup_other_params(ErtsRunQueue *rq)
+{
+    ErtsSchedulerData *esdp = rq->scheduler;
+    return &sched_wakeup_other_params[esdp->type];
+}
 
 static void
 wakeup_other_check(ErtsRunQueue *rq, Uint32 flags)
 {
+    ErtsWakeupOtherParams *wo_params = runq_get_wakeup_other_params(rq);
     int wo_reds = rq->wakeup_other_reds;
+
     if (wo_reds) {
 	int left_len = erts_atomic32_read_dirty(&rq->len) - 1;
 	if (left_len < 1) {
-	    int wo_reduce = wo_reds << wakeup_other.dec_shift;
-	    wo_reduce &= wakeup_other.dec_mask;
+	    int wo_reduce = wo_reds << wo_params->dec_shift;
+	    wo_reduce &= wo_params->dec_mask;
 	    rq->wakeup_other -= wo_reduce;
 	    if (rq->wakeup_other < 0)
 		rq->wakeup_other = 0;
@@ -5275,7 +5294,7 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags)
 	else {
 	    rq->wakeup_other += (left_len*wo_reds
 				 + ERTS_WAKEUP_OTHER_FIXED_INC);
-	    if (rq->wakeup_other > wakeup_other.limit) {
+	    if (rq->wakeup_other > wo_params->limit) {
 		if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) {
 		    if (rq->waiting) {
 			wake_dirty_scheduler(rq);
@@ -5297,42 +5316,44 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags)
 }
 
 static void
-wakeup_other_set_limit(void)
+wakeup_other_set_limit(ErtsWakeupOtherParams *params)
 {
-    switch (wakeup_other.threshold) {
+    switch (params->threshold) {
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH;
-	wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_HIGH;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH;
+	params->dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_HIGH;
 	break;
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH;
-	wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_HIGH;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH;
+	params->dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_HIGH;
 	break;
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM;
-	wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_MEDIUM;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM;
+	params->dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_MEDIUM;
 	break;
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_LOW;
-	wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_LOW;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_LOW;
+	params->dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_LOW;
 	break;
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW;
-	wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_LOW;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW;
+	params->dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_LOW;
 	break;
     }
-    if (wakeup_other.dec_shift < 0)
-	wakeup_other.dec_mask = (1 << (sizeof(wakeup_other.dec_mask)*8
-				       + wakeup_other.dec_shift)) - 1;
+
+    if (params->dec_shift < 0)
+	params->dec_mask = (1 << (sizeof(params->dec_mask)*8
+				  + params->dec_shift)) - 1;
     else {
-	wakeup_other.dec_mask = 0;
-	wakeup_other.dec_mask = ~wakeup_other.dec_mask;
+	params->dec_mask = 0;
+	params->dec_mask = ~params->dec_mask;
     }
 }
 
 static void
 wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags)
 {
+    ErtsWakeupOtherParams *wo_params = runq_get_wakeup_other_params(rq);
     int wo_reds = rq->wakeup_other_reds;
     if (wo_reds) {
 	erts_aint32_t len = erts_atomic32_read_dirty(&rq->len);
@@ -5341,7 +5362,7 @@ wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags)
 	    if (rq->wakeup_other < 0)
 		rq->wakeup_other = 0;
 	}
-	else if (rq->wakeup_other < wakeup_other.limit)
+	else if (rq->wakeup_other < wo_params->limit)
 	    rq->wakeup_other += len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY;
 	else {
 	    if (flags & ERTS_RUNQ_FLG_PROTECTED)
@@ -5357,23 +5378,23 @@ wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags)
 }
 
 static void
-wakeup_other_set_limit_legacy(void)
+wakeup_other_set_limit_legacy(ErtsWakeupOtherParams *params)
 {
-    switch (wakeup_other.threshold) {
+    switch (params->threshold) {
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH_LEGACY;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH_LEGACY;
 	break;
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH_LEGACY;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH_LEGACY;
 	break;
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM_LEGACY;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM_LEGACY;
 	break;
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_LOW_LEGACY;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_LOW_LEGACY;
 	break;
     case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW:
-	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW_LEGACY;
+	params->limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW_LEGACY;
 	break;
     }
 }
@@ -5381,15 +5402,21 @@ wakeup_other_set_limit_legacy(void)
 static void
 set_wakeup_other_data(void)
 {
-    switch (wakeup_other.type) {
-    case ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT:
-	wakeup_other.check = wakeup_other_check;
-	wakeup_other_set_limit();
-	break;
-    case ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY:
-	wakeup_other.check = wakeup_other_check_legacy;
-	wakeup_other_set_limit_legacy();
-	break;
+    ErtsSchedType type;
+
+    for (type = ERTS_SCHED_TYPE_FIRST; type <= ERTS_SCHED_TYPE_LAST; type++) {
+        ErtsWakeupOtherParams *params = &sched_wakeup_other_params[type];
+
+        switch (params->type) {
+        case ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT:
+            params->check = wakeup_other_check;
+            wakeup_other_set_limit(params);
+            break;
+        case ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY:
+            params->check = wakeup_other_check_legacy;
+            wakeup_other_set_limit_legacy(params);
+            break;
+        }
     }
 }
 
@@ -5444,56 +5471,64 @@ runq_supervisor(void *unused)
 void
 erts_early_init_scheduling(int no_schedulers)
 {
+    ErtsSchedType type;
+
     aux_work_timeout_early_init(no_schedulers);
-    wakeup_other.threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM;
-    wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT;
-    sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM;
-    sched_busy_wait.tse = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM
-			   * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT);
-    sched_busy_wait.aux_work = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM
-				* ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM);
+
+    for (type = ERTS_SCHED_TYPE_FIRST; type <= ERTS_SCHED_TYPE_LAST; type++) {
+        erts_sched_set_wakeup_other_threshold(type, "medium");
+        erts_sched_set_wakeup_other_type(type, "default");
+
+        erts_sched_set_busy_wait_threshold(type, "medium");
+    }
+
+    erts_sched_set_busy_wait_threshold(ERTS_SCHED_DIRTY_CPU, "short");
+    erts_sched_set_busy_wait_threshold(ERTS_SCHED_DIRTY_IO, "short");
 }
 
 int
-erts_sched_set_wakeup_other_thresold(char *str)
-{
-    ErtsSchedWakeupOtherThreshold threshold;
-    if (sys_strcmp(str, "very_high") == 0)
-	threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH;
-    else if (sys_strcmp(str, "high") == 0)
-	threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH;
-    else if (sys_strcmp(str, "medium") == 0)
-	threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM;
-    else if (sys_strcmp(str, "low") == 0)
-	threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW;
-    else if (sys_strcmp(str, "very_low") == 0)
-	threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW;
-    else
-	return EINVAL;
-    wakeup_other.threshold = threshold;
-    set_wakeup_other_data();
+erts_sched_set_wakeup_other_threshold(ErtsSchedType sched_type, char *str)
+{
+    ErtsWakeupOtherParams *params = &sched_wakeup_other_params[sched_type];
+
+    if (sys_strcmp(str, "very_high") == 0) {
+        params->threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH;
+    } else if (sys_strcmp(str, "high") == 0) {
+        params->threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH;
+    } else if (sys_strcmp(str, "medium") == 0) {
+        params->threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM;
+    } else if (sys_strcmp(str, "low") == 0) {
+        params->threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW;
+    } else if (sys_strcmp(str, "very_low") == 0) {
+        params->threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW;
+    } else {
+        return EINVAL;
+    }
+
     return 0;
 }
 
 int
-erts_sched_set_wakeup_other_type(char *str)
+erts_sched_set_wakeup_other_type(ErtsSchedType sched_type, char *str)
 {
-    ErtsSchedWakeupOtherType type;
-    if (sys_strcmp(str, "default") == 0)
-	type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT;
-    else if (sys_strcmp(str, "legacy") == 0)
-	type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY;
-    else
-	return EINVAL;
-    wakeup_other.type = type;
+    ErtsWakeupOtherParams *params = &sched_wakeup_other_params[sched_type];
+
+    if (sys_strcmp(str, "default") == 0) {
+        params->type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT;
+    } else if (sys_strcmp(str, "legacy") == 0) {
+        params->type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY;
+    } else {
+        return EINVAL;
+    }
+
     return 0;
 }
 
 int
-erts_sched_set_busy_wait_threshold(char *str)
+erts_sched_set_busy_wait_threshold(ErtsSchedType sched_type, char *str)
 {
-    int sys_sched;
-    int aux_work_fact;
+    ErtsBusyWaitParams *params = &sched_busy_wait_params[sched_type];
+    int aux_work_fact, sys_sched;
 
     if (sys_strcmp(str, "very_long") == 0) {
 	sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_LONG;
@@ -5523,9 +5558,9 @@ erts_sched_set_busy_wait_threshold(char *str)
 	return EINVAL;
     }
 
-    sched_busy_wait.sys_schedule = sys_sched;
-    sched_busy_wait.tse = sys_sched*ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT;
-    sched_busy_wait.aux_work = sys_sched*aux_work_fact;
+    params->sys_schedule = sys_sched;
+    params->tse = sys_sched * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT;
+    params->aux_work = sys_sched * aux_work_fact;
 
     return 0;
 }
@@ -9894,7 +9929,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
         if (flags & ERTS_RUNQ_FLG_MISC_OP)
 	    exec_misc_ops(rq);
 
-	wakeup_other.check(rq, flags);
+        runq_get_wakeup_other_params(rq)->check(rq, flags);
 
 	/*
 	 * Find a new port to run.
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 0550fb05b5..cec93cbb9b 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -400,9 +400,12 @@ typedef struct {
 } ErtsRunPrioQueue;
 
 typedef enum {
-    ERTS_SCHED_NORMAL,
-    ERTS_SCHED_DIRTY_CPU,
-    ERTS_SCHED_DIRTY_IO
+    ERTS_SCHED_NORMAL = 0,
+    ERTS_SCHED_DIRTY_CPU = 1,
+    ERTS_SCHED_DIRTY_IO = 2,
+
+    ERTS_SCHED_TYPE_FIRST = ERTS_SCHED_NORMAL,
+    ERTS_SCHED_TYPE_LAST = ERTS_SCHED_DIRTY_IO
 } ErtsSchedType;
 
 typedef struct ErtsSchedulerData_ ErtsSchedulerData;
@@ -1712,9 +1715,9 @@ ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *list,
 
 #endif
 
-int erts_sched_set_wakeup_other_thresold(char *str);
-int erts_sched_set_wakeup_other_type(char *str);
-int erts_sched_set_busy_wait_threshold(char *str);
+int erts_sched_set_wakeup_other_threshold(ErtsSchedType sched_type, char *str);
+int erts_sched_set_wakeup_other_type(ErtsSchedType sched_type, char *str);
+int erts_sched_set_busy_wait_threshold(ErtsSchedType sched_type, char *str);
 int erts_sched_set_wake_cleanup_threshold(char *);
 
 void erts_schedule_thr_prgr_later_op(void (*)(void *),
diff --git a/erts/emulator/test/smoke_test_SUITE.erl b/erts/emulator/test/smoke_test_SUITE.erl
index adc6f56c06..b3d34103f1 100644
--- a/erts/emulator/test/smoke_test_SUITE.erl
+++ b/erts/emulator/test/smoke_test_SUITE.erl
@@ -70,6 +70,20 @@ boot_combo(Config) when is_list(Config) ->
 	chk_boot(Config, "+Ktrue", NOOP),
 	chk_boot(Config, "+A42", A42),
 	chk_boot(Config, "+Ktrue +A42", A42),
+
+        WBTArgs = ["very_short", "short", "medium", "long", "very_long"],
+        WTArgs = ["very_low", "low", "medium", "high", "very_high"],
+        [chk_boot(Config,
+                 " +sbwt " ++ WBT ++
+                 " +sbwtdcpu " ++ WBT ++
+                 " +sbwtdio " ++ WBT ++
+                 " +swt " ++ WT ++
+                 " +swtdcpu " ++ WT ++
+                 " +swtdio " ++ WT, NOOP) || WBT <- WBTArgs,  WT <- WTArgs],
+
+        WSArgs = ["legacy", "default"],
+        [chk_boot(Config, " +sws " ++ WS, NOOP) || WS <- WSArgs],
+
 	%% A lot more combos could be implemented...
 	ok
     after
-- 
cgit v1.2.3


From 2d3437392c000f8d4846a57dd399f1e4e4be8b2f Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 26 Mar 2018 10:54:39 +0200
Subject: Fix VM probes compilation

---
 erts/emulator/beam/erl_proc_sig_queue.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index b4759967a7..99d20e9242 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -40,6 +40,7 @@
 #include "erl_gc.h"
 #include "bif.h"
 #include "erl_proc_sig_queue.h"
+#include "dtrace-wrapper.h"
 
 #define ERTS_SIG_REDS_CNT_FACTOR 4
 #define ERTS_PROC_SIG_TRACE_COUNT_LIMIT 200
@@ -751,13 +752,14 @@ send_gen_exit_signal(Process *c_p, Eterm from_tag,
         seq_trace_update_send(c_p);
 
 #ifdef USE_VM_PROBES
+    utag_sz = 0;
+    utag = NIL;
     if (c_p && token != NIL && (DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING)) {
         utag_sz = size_object(DT_UTAG(c_p));
         utag = DT_UTAG(c_p);
     }
     else if (token == am_have_dt_utag) {
-        utag_sz = 0;
-        utag = token = NIL;
+        token = NIL;
     }
     hsz += utag_sz;
 #endif
-- 
cgit v1.2.3


From 63e1c58d27ab695a19897423fc75e607f69ff51b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Sun, 25 Feb 2018 14:19:48 +0100
Subject: Compile external fun expressions to literals

The expressions fun M:F/A, when all elements are literals are also
treated as a literal. Since they have consistent representation and
don't depend on the code currently loaded in the VM, this is safe.
This can provide significant performance improvements in code using such
functions extensively - a full function call to erlang:make_fun/3 is
replaced by a single move instruction and no register shuffling or
saving registers to stack is necessary. Additionally, compound data
types that contain such external functions as elements can be treated as
literals too.

The commit also changes the representation of external funs to be a
valid Erlang syntax and adds support for literal external funs to core
Erlang.
---
 erts/emulator/beam/beam_load.c             | 13 +++++++
 erts/emulator/beam/erl_printf_term.c       |  7 ++--
 erts/emulator/beam/ops.tab                 |  3 +-
 erts/emulator/test/beam_literals_SUITE.erl | 55 +++++++++++++++++++++---------
 4 files changed, 57 insertions(+), 21 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index e242fe9140..3ac98cec8d 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4520,6 +4520,19 @@ is_empty_map(LoaderState* stp, GenOpArg Lit)
     return is_flatmap(term) && flatmap_get_size(flatmap_val(term)) == 0;
 }
 
+/*
+ * Predicate to test whether the given literal is an export.
+ */
+static int
+literal_is_export(LoaderState* stp, GenOpArg Lit)
+{
+    Eterm term;
+
+    ASSERT(Lit.type == TAG_q);
+    term = stp->literals[Lit.val].term;
+    return is_export(term);
+}
+
 /*
  * Pseudo predicate map_key_sort that will sort the Rest operand for
  * map instructions as a side effect.
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index e6f8460164..910f241a3a 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -532,14 +532,13 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) {
 		Atom* module = atom_tab(atom_val(ep->info.mfa.module));
 		Atom* name = atom_tab(atom_val(ep->info.mfa.function));
 
-		PRINT_STRING(res, fn, arg, "#Fun<");
+		PRINT_STRING(res, fn, arg, "fun ");
 		PRINT_BUF(res, fn, arg, module->name, module->len);
-		PRINT_CHAR(res, fn, arg, '.');
+		PRINT_CHAR(res, fn, arg, ':');
 		PRINT_BUF(res, fn, arg, name->name, name->len);
-		PRINT_CHAR(res, fn, arg, '.');
+		PRINT_CHAR(res, fn, arg, '/');
 		PRINT_SWORD(res, fn, arg, 'd', 0, 1,
 			    (ErlPfSWord) ep->info.mfa.arity);
-		PRINT_CHAR(res, fn, arg, '>');
 	    }
 	    break;
 	case FUN_DEF:
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 77e375f2c0..6e03f39d97 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -710,7 +710,8 @@ is_boolean Fail=f ac => jump Fail
 is_boolean f? xy
 %hot
 
-is_function2 Fail=f acq Arity => jump Fail
+is_function2 Fail=f Literal=q Arity | literal_is_export(Literal) =>
+is_function2 Fail=f c Arity => jump Fail
 is_function2 Fail=f Fun a => jump Fail
 
 is_function2 f? S s
diff --git a/erts/emulator/test/beam_literals_SUITE.erl b/erts/emulator/test/beam_literals_SUITE.erl
index 09761263e2..b447ca0210 100644
--- a/erts/emulator/test/beam_literals_SUITE.erl
+++ b/erts/emulator/test/beam_literals_SUITE.erl
@@ -248,35 +248,58 @@ literal_type_tests(Config) when is_list(Config) ->
     ok.
 
 make_test([{is_function=T,L}|Ts]) ->
-    [test(T, L),test(T, 0, L)|make_test(Ts)];
+    [guard_test(T, L),guard_test(T, 0, L),body_test(T, L),body_test(T, 0, L)|make_test(Ts)];
 make_test([{T,L}|Ts]) ->
-    [test(T, L)|make_test(Ts)];
+    [guard_test(T, L),body_test(T, L)|make_test(Ts)];
 make_test([]) -> [].
 
-test(T, L) ->
-    S = lists:flatten(io_lib:format("begin io:format(\"~~p~n\", [{~p,~p}]), if ~w(~w) -> true; true -> false end end. ", [T, L, T, L])),
-    {ok,Toks,_Line} = erl_scan:string(S),
-    {ok,E} = erl_parse:parse_exprs(Toks),
-    {value,Val,_Bs} = erl_eval:exprs(E, []),
+guard_test(_, L) when is_function(L) ->
+    %% Skip guard tests with exports - they are not literals
+    {atom,erl_anno:new(0),true};
+guard_test(T, L) ->
+    S = io_lib:format("begin io:format(\"~~p~n\", [{~p,~p}]), if ~w(~w) -> true; true -> false end end. ", [T, L, T, L]),
+    {Val,Expr} = eval_string(S),
+    Anno = erl_anno:new(0),
+    {match,Anno,{atom,Anno,Val},Expr}.
+
+guard_test(_, _, L) when is_function(L) ->
+    %% Skip guard tests with exports - they are not literals
+    {atom,erl_anno:new(0),true};
+guard_test(T, A, L) ->
+    S = io_lib:format("begin io:format(\"~~p~n\", [{~p,~p,~p}]), if ~w(~w, ~w) -> true; true -> false end end. ", [T,L,A,T,L,A]),
+    {Val,Expr} = eval_string(S),
+    Anno = erl_anno:new(0),
+    {match,Anno,{atom,Anno,Val},Expr}.
+
+body_test(T, L) ->
+    S = io_lib:format("begin io:format(\"~~p~n\", [{~p,~p}]), ~w(~w) end. ", [T,L,T,L]),
+    {Val,Expr} = eval_string(S),
     Anno = erl_anno:new(0),
-    {match,Anno,{atom,Anno,Val},hd(E)}.
+    {match,Anno,{atom,Anno,Val},Expr}.
 
-test(T, A, L) ->
-    S = lists:flatten(io_lib:format("begin io:format(\"~~p~n\", [{~p,~p,~p}]), if ~w(~w, ~w) -> true; true -> false end end. ",
-				    [T,L,A,T,L,A])),
-    {ok,Toks,_Line} = erl_scan:string(S),
+body_test(T, A, L) ->
+    S = io_lib:format("begin io:format(\"~~p~n\", [{~p,~p,~p}]), ~w(~w,~w) end. ", [T,L,A,T,L,A]),
+    {Val,Expr} = eval_string(S),
+    Anno = erl_anno:new(0),
+    {match,Anno,{atom,Anno,Val},Expr}.
+
+eval_string(S) ->
+    {ok,Toks,_Line} = erl_scan:string(lists:flatten(S)),
     {ok,E} = erl_parse:parse_exprs(Toks),
     {value,Val,_Bs} = erl_eval:exprs(E, []),
-    Anno = erl_anno:new(0),
-    {match,Anno,{atom,Anno,Val},hd(E)}.
-    
+    {Val,hd(E)}.
+
 literals() ->
     [42,
      3.14,
      -3,
      32982724987789283473473838474,
      [],
-     xxxx].
+     "abc",
+     <<"abc">>,
+     {},
+     xxxx,
+     fun erlang:erase/0].
 
 type_tests() ->
     [is_boolean,
-- 
cgit v1.2.3


From b48df378266717e1f6b79b96eb0f69cb08081585 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 26 Mar 2018 15:45:05 +0200
Subject: erts: Fix harmless bug in macro IS_DRIVER_VERSION_GE

harmless until we bump major version
---
 erts/emulator/beam/io.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index e4a5f2b6b6..410fe89b02 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -7613,7 +7613,8 @@ no_stop_select_callback(ErlDrvEvent event, void* private)
 }
 
 #define IS_DRIVER_VERSION_GE(DE,MAJOR,MINOR) \
-    ((DE)->major_version >= (MAJOR) && (DE)->minor_version >= (MINOR))
+    ((DE)->major_version > (MAJOR) ||        \
+     ((DE)->major_version == (MAJOR) && (DE)->minor_version >= (MINOR)))
 
 static int
 init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
-- 
cgit v1.2.3


From 39cf4e75f31e9abe46c9fe39000368c1243fdcf3 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 26 Mar 2018 15:48:12 +0200
Subject: erts: Include foreign static linked drivers in taints

That is, driver added with config option --enable-static-drivers.
---
 erts/emulator/beam/erl_bif_ddll.c   | 10 +---------
 erts/emulator/beam/global.h         |  6 +++++-
 erts/emulator/beam/io.c             | 33 +++++++++++++++++++++++----------
 erts/emulator/utils/make_driver_tab | 10 +++++-----
 4 files changed, 34 insertions(+), 25 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 8220ba97a2..294bce115f 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1506,14 +1506,6 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name)
 	goto error;
     }
 
-    {
-        Eterm name_atom = erts_atom_put((byte*)name, sys_strlen(name),
-                                        ERTS_ATOM_ENC_LATIN1, 0);
-        if (is_non_value(name_atom))
-            goto error;
-        erts_add_taint(name_atom);
-    }
-
     erts_atomic_init_nob(&(dh->refc), (erts_aint_t) 0);
     erts_atomic32_init_nob(&dh->port_count, 0);
     dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1);
@@ -1521,7 +1513,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name)
     dh->flags = 0;
     dh->status = ERL_DE_OK;
 
-    if (erts_add_driver_entry(dp, dh, 1) != 0 /* io.c */) { 
+    if (erts_add_driver_entry(dp, dh, 1, 1) != 0 /* io.c */) {
 	/*
 	 * The init in the driver struct did not return 0 
 	 */
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 8a746ea4b1..c8354f9fcf 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1152,7 +1152,7 @@ typedef struct {
 #define ERTS_SPAWN_DRIVER 1
 #define ERTS_SPAWN_EXECUTABLE 2
 #define ERTS_SPAWN_ANY (ERTS_SPAWN_DRIVER | ERTS_SPAWN_EXECUTABLE)
-int erts_add_driver_entry(ErlDrvEntry *drv, DE_Handle *handle, int driver_list_locked);
+int erts_add_driver_entry(ErlDrvEntry *drv, DE_Handle *handle, int driver_list_locked, int taint);
 void erts_destroy_driver(erts_driver_t *drv);
 int erts_save_suspend_process_on_port(Port*, Process*);
 Port *erts_open_driver(erts_driver_t*, Eterm, char*, SysDriverOpts*, int *, int *);
@@ -1177,6 +1177,10 @@ void erts_lcnt_update_port_locks(int enable);
 #endif
 
 /* driver_tab.c */
+typedef struct {
+    ErlDrvEntry* de;
+    int taint;
+} ErtsStaticDriver;
 typedef void *(*ErtsStaticNifInitFPtr)(void);
 typedef struct ErtsStaticNifEntry_ {
     const char *nif_name;
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 410fe89b02..5d9d7b6ed7 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -60,7 +60,7 @@ extern ErlDrvEntry spawn_driver_entry;
 #ifndef __WIN32__
 extern ErlDrvEntry forker_driver_entry;
 #endif
-extern ErlDrvEntry *driver_tab[]; /* table of static drivers, only used during initialization */
+extern ErtsStaticDriver driver_tab[]; /* table of static drivers, only used during initialization */
 
 erts_driver_t *driver_list; /* List of all drivers, static and dynamic. */
 erts_rwmtx_t erts_driver_list_lock; /* Mutex for driver list */
@@ -2964,7 +2964,7 @@ void erts_init_io(int port_tab_size,
 		  int port_tab_size_ignore_files,
 		  int legacy_port_tab)
 {
-    ErlDrvEntry** dp;
+    ErtsStaticDriver* dp;
     UWord common_element_size;
     erts_rwmtx_opt_t drv_list_rwmtx_opts = ERTS_RWMTX_OPT_DEFAULT_INITER;
     drv_list_rwmtx_opts.type = ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ;
@@ -3023,8 +3023,8 @@ void erts_init_io(int port_tab_size,
     init_driver(&forker_driver, &forker_driver_entry, NULL);
 #endif
     erts_init_static_drivers();
-    for (dp = driver_tab; *dp != NULL; dp++)
-	erts_add_driver_entry(*dp, NULL, 1);
+    for (dp = driver_tab; dp->de != NULL; dp++)
+	erts_add_driver_entry(dp->de, NULL, 1, dp->taint);
 
     erts_tsd_set(driver_list_lock_status_key, NULL);
     erts_rwmtx_rwunlock(&erts_driver_list_lock);
@@ -7697,13 +7697,14 @@ void add_driver_entry(ErlDrvEntry *drv){
      * Ignore result of erts_add_driver_entry, the init is not
      * allowed to fail when drivers are added by drivers.
      */
-    erts_add_driver_entry(drv, NULL, rec_lock != NULL); 
+    erts_add_driver_entry(drv, NULL, rec_lock != NULL, 0);
 }
 
-int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle, int driver_list_locked)
+int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle,
+                          int driver_list_locked, int taint)
 {
     erts_driver_t *dp = erts_alloc(ERTS_ALC_T_DRIVER, sizeof(erts_driver_t));
-    int res;
+    int err = 0;
 
     if (!driver_list_locked) {
 	erts_rwmtx_rwlock(&erts_driver_list_lock);
@@ -7720,9 +7721,21 @@ int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle, int driver_list_lo
 	erts_tsd_set(driver_list_lock_status_key, (void *) 1);
     }
 
-    res = init_driver(dp, de, handle);
+    if (taint) {
+        Eterm name_atom = erts_atom_put((byte*)de->driver_name,
+                                        sys_strlen(de->driver_name),
+                                        ERTS_ATOM_ENC_LATIN1, 0);
+        if (is_atom(name_atom))
+            erts_add_taint(name_atom);
+        else
+            err = 1;
+    }
+
+    if (!err) {
+        err = init_driver(dp, de, handle);
+    }
 
-    if (res != 0) {
+    if (err) {
 	/* 
 	 * Remove it all again...
 	 */
@@ -7737,7 +7750,7 @@ int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle, int driver_list_lo
 	erts_tsd_set(driver_list_lock_status_key, NULL);
 	erts_rwmtx_rwunlock(&erts_driver_list_lock);
     }
-    return res;
+    return err;
 }
 
 /* Not allowed for dynamic drivers */
diff --git a/erts/emulator/utils/make_driver_tab b/erts/emulator/utils/make_driver_tab
index cefb3e2504..b7bca1dc3a 100755
--- a/erts/emulator/utils/make_driver_tab
+++ b/erts/emulator/utils/make_driver_tab
@@ -95,22 +95,22 @@ foreach (@static_drivers) {
 }
 
 # The array itself
-print "\nErlDrvEntry *driver_tab[] =\n{\n";
+print "\nErtsStaticDriver driver_tab[] =\n{\n";
 
 foreach (@emu_drivers) {
-    print "    &${_}driver_entry,\n";
+    print "    {&${_}driver_entry, 0},\n";
 }
 
 foreach (@static_drivers) {
-    print "    NULL, /* ${_} */\n";
+    print "    {NULL, 1}, /* ${_} */\n";
 }
-print "    NULL\n};\n";
+print "    {NULL}\n};\n";
 
 print "void erts_init_static_drivers() {\n";
 
 my $index = 0;
 foreach (@static_drivers) {
-    print "    driver_tab[".(scalar @emu_drivers+$index)."] = ${_}_driver_init();\n";
+    print "    driver_tab[".(scalar @emu_drivers+$index)."].de = ${_}_driver_init();\n";
     $index++;
 }
 
-- 
cgit v1.2.3


From 1c834990a92943c0fcd1d47610767806ab4660e1 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 27 Mar 2018 16:06:16 +0200
Subject: Fix bad assert

---
 erts/emulator/beam/erl_process.c | 1 -
 1 file changed, 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 374583ec47..6680c96775 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10750,7 +10750,6 @@ exit_permanent_prio_elevation(Process *c_p, erts_aint32_t state)
     while (1) {
         erts_aint32_t aprio, uprio, n, e;
         ASSERT(a & ERTS_PSFLG_EXITING);
-        ASSERT(!(a & ERTS_PSFLG_FREE));
         aprio = ERTS_PSFLGS_GET_ACT_PRIO(a);
         uprio = ERTS_PSFLGS_GET_USR_PRIO(a);
         if (aprio >= uprio)
-- 
cgit v1.2.3


From 07ec39c2eec59a547cf88ab91dae506c44d27f56 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 28 Mar 2018 08:50:37 +0200
Subject: erts: Fix race in nif_SUITE:monitor_frenzy

detected by valgrind. We cannot compare monitor
in state MON_FREE before it's initialized.

Still not really kosher to access 'state' without lock or atomic-op.
---
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index fa9ae1015c..194ece77a2 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -2838,7 +2838,7 @@ unsigned rand_bits(struct frenzy_rand_bits* rnd, unsigned int nbits)
 
 struct frenzy_monitor {
     ErlNifMutex* lock;
-    enum { 
+    volatile enum {
         MON_FREE, MON_FREE_DOWN, MON_FREE_DEMONITOR,
         MON_TRYING, MON_ACTIVE, MON_PENDING
     } state;
@@ -3200,13 +3200,24 @@ static void frenzy_resource_down(ErlNifEnv* env, void* obj, ErlNifPid* pid,
     DBG_TRACE3("DOWN pid=%T, r=%p rix=%u\n", pid->pid, r, r->rix);
 
     for (mix = 0; mix < FRENZY_MONITORS_MAX; mix++) {
-        if (r->monv[mix].pid.pid == pid->pid && r->monv[mix].state >= MON_TRYING) {
+        int state1 = r->monv[mix].state;
+        /* First do dirty access of pid and state without the lock */
+        if (r->monv[mix].pid.pid == pid->pid && state1 >= MON_TRYING) {
+            int state2;
             enif_mutex_lock(r->monv[mix].lock);
-            if (enif_compare_monitors(mon, &r->monv[mix].mon) == 0) {
-                assert(r->monv[mix].state >= MON_ACTIVE);
-                r->monv[mix].state = MON_FREE_DOWN;
-                enif_mutex_unlock(r->monv[mix].lock);
-                return;
+            state2 = r->monv[mix].state;
+            if (state2 >= MON_ACTIVE) {
+                if (enif_compare_monitors(mon, &r->monv[mix].mon) == 0) {
+                    r->monv[mix].state = MON_FREE_DOWN;
+                    enif_mutex_unlock(r->monv[mix].lock);
+                    return;
+                }
+            }
+            else {
+                assert(state2 != MON_TRYING);
+                assert(state1 == MON_TRYING ||  /* racing monitor failed */
+                       state2 == MON_FREE_DEMONITOR || /* racing demonitor */
+                       state2 == MON_FREE_DOWN);       /* racing down */
             }
             enif_mutex_unlock(r->monv[mix].lock);
         }
-- 
cgit v1.2.3


From f46d38dcca5ebd17e32068e0cb0cb9802c6b07a8 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 29 Mar 2018 15:26:04 +0200
Subject: erts: Don't use EV_DISPATCH on openbsd

Currently (OpenBSD 6.2) the kqueu implementation on
opnebsd does not work properly for EOF conditions when
using EV_DISPATCH, so we use the EV_ONESHOT fallback there.
---
 erts/emulator/sys/common/erl_poll.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index 7aa53e8f36..ced8a4a2a7 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -782,10 +782,14 @@ update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events)
     struct kevent evts[2];
     struct timespec ts = {0, 0};
 
-#ifdef EV_DISPATCH
-    /* If we have EV_DISPATCH we use it. The kevent descriptions for both
-       read and write are added on OP_ADD and removed on OP_DEL. And then
-       after than only EV_ENABLE|EV_DISPATCH are used.
+#if defined(EV_DISPATCH) && !defined(__OpenBSD__)
+    /* If we have EV_DISPATCH we use it, unless we are on OpenBSD as the
+       behavior of EV_EOF seems to be edge triggered there and we need it
+       to be level triggered.
+
+       The kevent descriptions for both read and write are added on OP_ADD
+       and removed on OP_DEL. And then after than only EV_ENABLE|EV_DISPATCH
+       are used.
 
        It could be possible to not modify the pollset when disabling and/or
        deleting events, but that may cause the poll threads to be awoken
-- 
cgit v1.2.3


From 9091e0fc630913999e3a51e860a2c4e07c6143c1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 16 Mar 2018 19:34:42 +0100
Subject: erts: Fix float_to_list(F, [{decimals,D}])

to better conform with io_lib:format("~.*f", [D,F])
---
 erts/emulator/sys/common/erl_sys_common_misc.c | 184 +++++++++----------------
 erts/emulator/test/num_bif_SUITE.erl           |  25 ++++
 2 files changed, 91 insertions(+), 118 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c
index 96bdbacb9e..826307c077 100644
--- a/erts/emulator/sys/common/erl_sys_common_misc.c
+++ b/erts/emulator/sys/common/erl_sys_common_misc.c
@@ -142,7 +142,16 @@ sys_double_to_chars(double fp, char *buffer, size_t buffer_size)
     return sys_double_to_chars_ext(fp, buffer, buffer_size, SYS_DEFAULT_FLOAT_DECIMALS);
 }
 
-/* Convert float to string using fixed point notation.
+
+#if SIZEOF_LONG == 8
+# define round_int64 lround
+#elif SIZEOF_LONG_LONG == 8
+# define round_int64 llround
+#else
+# error "No 64-bit integer type?"
+#endif
+
+/* Convert float to string
  *   decimals must be >= 0
  *   if compact != 0, the trailing 0's will be truncated
  */
@@ -154,80 +163,35 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
     #define FRAC_SIZE            52
     #define EXP_SIZE             11
     #define EXP_MASK             (((Uint64)1 << EXP_SIZE) - 1)
-    #define MAX_DECIMALS         (sizeof(cs_sys_double_pow10) \
-				   / sizeof(cs_sys_double_pow10[0]))
+    #define MAX_DECIMALS         (sizeof(pow10v) / sizeof(pow10v[0]))
     #define FRAC_MASK            (((Uint64)1 << FRAC_SIZE) - 1)
     #define FRAC_MASK2           (((Uint64)1 << (FRAC_SIZE + 1)) - 1)
     #define MAX_FLOAT            ((Uint64)1 << (FRAC_SIZE+1))
 
-    static const double cs_sys_double_pow10[] = {
-        SYS_DOUBLE_RND_CONST / 1e0,
-        SYS_DOUBLE_RND_CONST / 1e1,
-        SYS_DOUBLE_RND_CONST / 1e2,
-        SYS_DOUBLE_RND_CONST / 1e3,
-        SYS_DOUBLE_RND_CONST / 1e4,
-        SYS_DOUBLE_RND_CONST / 1e5,
-        SYS_DOUBLE_RND_CONST / 1e6,
-        SYS_DOUBLE_RND_CONST / 1e7,
-        SYS_DOUBLE_RND_CONST / 1e8,
-        SYS_DOUBLE_RND_CONST / 1e9,
-        SYS_DOUBLE_RND_CONST / 1e10,
-        SYS_DOUBLE_RND_CONST / 1e11,
-        SYS_DOUBLE_RND_CONST / 1e12,
-        SYS_DOUBLE_RND_CONST / 1e13,
-        SYS_DOUBLE_RND_CONST / 1e14,
-        SYS_DOUBLE_RND_CONST / 1e15,
-        SYS_DOUBLE_RND_CONST / 1e16,
-        SYS_DOUBLE_RND_CONST / 1e17,
-        SYS_DOUBLE_RND_CONST / 1e18
+    static const double pow10v[] = {
+        1e0,  1e1,  1e2,  1e3,  1e4,  1e5,  1e6,  1e7,  1e8, 1e9,
+        1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18
     };
 
-    Uint64 mantissa, int_part, frac_part;
-    int exp;
-    int fbits;
-    int max;
+    double af;
+    Uint64 int_part, frac_part;
     int neg;
-    double fr;
-    union { Uint64 L; double F; } x;
     char *p = buffer;
 
     if (decimals < 0)
         return -1;
 
-    if (f >= 0) {
-        neg = 0;
-        fr  = decimals < MAX_DECIMALS ? (f + cs_sys_double_pow10[decimals]) : f;
-        x.F = fr;
-    } else {
+    if (f < 0) {
         neg = 1;
-        fr  = decimals < MAX_DECIMALS ? (f - cs_sys_double_pow10[decimals]) : f;
-        x.F = -fr;
+        af = -f;
     }
-
-    exp      = (x.L >> FRAC_SIZE) & EXP_MASK;
-    mantissa = x.L & FRAC_MASK;
-
-    if (exp == EXP_MASK) {
-        if (mantissa == 0) {
-            if (neg)
-                *p++ = '-';
-            *p++ = 'i';
-            *p++ = 'n';
-            *p++ = 'f';
-        } else {
-            *p++ = 'n';
-            *p++ = 'a';
-            *p++ = 'n';
-        }
-        *p = '\0';
-        return p - buffer;
+    else {
+        neg = 0;
+        af = f;
     }
 
-    exp      -= EXP_MASK >> 1;
-    mantissa |= ((Uint64)1 << FRAC_SIZE);
-
     /* Don't bother with optimizing too large numbers or too large precision */
-    if (x.F > MAX_FLOAT || decimals >= MAX_DECIMALS) {
+    if (af > MAX_FLOAT || decimals >= MAX_DECIMALS) {
         int len = erts_snprintf(buffer, buffer_size, "%.*f", decimals, f);
         char* p = buffer + len;
         if (len >= buffer_size)
@@ -237,77 +201,61 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
             p = find_first_trailing_zero(p);
         *p = '\0';
         return p - buffer;
-    } else if (exp >= FRAC_SIZE) {
-        int_part  = mantissa << (exp - FRAC_SIZE);
-        frac_part = 0;
-        fbits = FRAC_SIZE;  /* not important as frac_part==0 */
-    } else if (exp >= 0) {
-        fbits = FRAC_SIZE - exp;
-        int_part  = mantissa >> fbits;
-        frac_part = mantissa & (((Uint64)1 << fbits) -1);
-    } else /* if (exp < 0) */ {
-        int_part = 0;
-        frac_part = mantissa;
-        fbits = FRAC_SIZE - exp;
-    }
-
-    if (!int_part) {
-        if (neg)
-            *p++ = '-';
-        *p++ = '0';
-    } else {
-        int ret, i, n;
-        while (int_part != 0) {
-            *p++ = (char)((int_part % 10) + '0');
-            int_part /= 10;
-        }
-        if (neg)
-            *p++ = '-';
-        /* Reverse string */
-        ret = p - buffer;
-        for (i = 0, n = ret/2; i < n; i++) {
-            int  j = ret - i - 1;
-            char c = buffer[i];
-            buffer[i] = buffer[j];
-            buffer[j] = c;
-        }
     }
 
-    if (decimals > 0) {
-        int i;
-        *p++ = '.';
+    if (decimals) {
+        double int_f = floor(af);
+        double frac_f = round((af - int_f) * pow10v[decimals]);
 
-        max = buffer_size - (p - buffer) - 1 /* leave room for trailing '\0' */;
+        int_part = (Uint64)int_f;
+        frac_part = (Uint64)frac_f;
 
-        if (decimals > max)
-            return -1;  /* the number is not large enough to fit in the buffer */
-
-        max = decimals;
+        if (frac_f >= pow10v[decimals]) {
+            /* rounding overflow carry into int_part */
+            int_part++;
+            frac_part = 0;
+        }
 
-        for (i = 0; i < max; i++) {
-            if (frac_part > (ERTS_UINT64_MAX/5)) {
-                frac_part >>= 3;
-                fbits -= 3;
+        do {
+            if (!frac_part) {
+                do {
+                    *p++ = '0';
+                } while (--decimals);
+                break;
             }
+            *p++ = (char)((frac_part % 10) + '0');
+            frac_part /= 10;
+        } while (--decimals);
 
-            /* Multiply by 10 (5*2) to extract decimal digit as integer part */
-            frac_part *= 5;
-            fbits--;
+        *p++ = '.';
+    }
+    else
+        int_part = (Uint64)round_int64(af);
 
-            if (fbits >= 64) {
-                *p++ = '0';
-            }
-            else {
-                *p++ = (char)((frac_part >> fbits) + '0');
-                frac_part &= ((Uint64)1 << fbits) - 1;
-            }
+    if (!int_part) {
+        *p++ = '0';
+    } else {
+        do {
+            *p++ = (char)((int_part % 10) + '0');
+            int_part /= 10;
+        }while (int_part);
+    }
+    if (neg)
+        *p++ = '-';
+
+    {/* Reverse string */
+        int i = 0;
+        int j = p - buffer - 1;
+        for ( ; i < j; i++, j--) {
+            char tmp = buffer[i];
+            buffer[i] = buffer[j];
+            buffer[j] = tmp;
         }
-
-        /* Delete trailing zeroes */
-        if (compact)
-            p = find_first_trailing_zero(p);
     }
 
+    /* Delete trailing zeroes */
+    if (compact)
+        p = find_first_trailing_zero(p);
     *p = '\0';
     return p - buffer;
 }
diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl
index 592542405f..104bd37817 100644
--- a/erts/emulator/test/num_bif_SUITE.erl
+++ b/erts/emulator/test/num_bif_SUITE.erl
@@ -213,6 +213,20 @@ fts_rand_float_decimals(N) ->
     [begin
          F0 = rand_float_reasonable(),
          L0 = float_to_list(F0, [{decimals, D}]),
+         case conform_with_io_lib_format(F0,D) of
+             false -> ok;
+             true ->
+                 IOL = lists:flatten(io_lib:format("~.*f", [D, F0])),
+                 true = case L0 =:= IOL of
+                            true -> true;
+                            false ->
+                                io:format("F0 = ~w ~w\n",  [F0, <<F0/float>>]),
+                                io:format("decimals = ~w\n",  [D]),
+                                io:format("float_to_list = ~s\n",  [L0]),
+                                io:format("io_lib:format = ~s\n",  [IOL]),
+                                false
+                        end
+         end,
          L1 = case D of
                   0 -> L0 ++ ".0";
                   _ -> L0
@@ -234,6 +248,17 @@ fts_rand_float_decimals(N) ->
 
     fts_rand_float_decimals(N-1).
 
+conform_with_io_lib_format(_, 0) ->
+    %% io_lib:format("~.*f") does not support zero decimals
+    false;
+conform_with_io_lib_format(_, D) when D > 10 ->
+    %% Seems float_to_list gets it slightly wrong sometimes for many decimals
+    false;
+conform_with_io_lib_format(F, D) ->
+    %% io_lib:format prints '0' for input bits beyond mantissa precision
+    %% float_to_list treats those unknown input bits as if they were zeros.
+    math:log2(abs(F) * math:pow(10,D)) < 54.
+
 max_diff_decimals(F, D) ->
     IntBits = floor(math:log2(abs(F))) + 1,
     FracBits = (52 - IntBits),
-- 
cgit v1.2.3


From 0bb8aac23842e04e522cd98ed397339675a41ad8 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 5 Apr 2018 10:21:57 +0200
Subject: Fix seq trace

---
 erts/emulator/beam/erl_proc_sig_queue.c | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index 99d20e9242..c2da73a092 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -723,7 +723,7 @@ erts_proc_sig_fetch(Process *proc)
     ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(proc);
 }
 
-void do_seq_trace_output(Eterm to, Eterm token, Eterm msg);
+static void do_seq_trace_output(Eterm to, Eterm token, Eterm msg);
 
 static void
 send_gen_exit_signal(Process *c_p, Eterm from_tag,
@@ -869,7 +869,7 @@ send_gen_exit_signal(Process *c_p, Eterm from_tag,
     }
 }
 
-void
+static void
 do_seq_trace_output(Eterm to, Eterm token, Eterm msg)
 {
     /*
@@ -887,15 +887,17 @@ do_seq_trace_output(Eterm to, Eterm token, Eterm msg)
     else
         rp = erts_proc_lookup_raw_inc_refc(to);
 
-    erts_proc_lock(rp, ERTS_PROC_LOCK_MSGQ);
+    if (rp) {
+        erts_proc_lock(rp, ERTS_PROC_LOCK_MSGQ);
 
-    if (!ERTS_PROC_IS_EXITING(rp))
-        seq_trace_output(token, msg, SEQ_TRACE_SEND, to, rp);
+        if (!ERTS_PROC_IS_EXITING(rp))
+            seq_trace_output(token, msg, SEQ_TRACE_SEND, to, rp);
 
-    erts_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ);
+        erts_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ);
 
-    if (!is_normal_sched)
-        erts_proc_dec_refc(rp);
+        if (!is_normal_sched)
+            erts_proc_dec_refc(rp);
+    }
 }
 
 void
-- 
cgit v1.2.3


From bcd8790fdf9549f014c35982732dbcd1270fbe9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 5 Apr 2018 15:43:58 +0200
Subject: Support dumping of external fun literals to a crash dump

63e1c58d27ab (PR #1725) started to compile external funs
as literals.

This commit updates the dumping of literal areas to dump
external fun literals to the crash dump.
---
 erts/emulator/beam/erl_process_dump.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 00659f9f49..5db276ec7f 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -942,6 +942,9 @@ dump_module_literals(fmtfn_t to, void *to_arg, ErtsLiteralArea* lit_area)
                     }
                     erts_putc(to, to_arg, '\n');
                 }
+            } else if (is_export_header(w)) {
+                dump_externally(to, to_arg, term);
+                erts_putc(to, to_arg, '\n');
             }
             size = 1 + header_arity(w);
             switch (w & _HEADER_SUBTAG_MASK) {
-- 
cgit v1.2.3


From 81d2981d47a7a28f136959e71dfd17c11a91a79e Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 10 Apr 2018 15:06:32 +0200
Subject: Teach erlang:monotonic_time/1 the perf_counter time unit

---
 erts/emulator/beam/erl_time_sup.c | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 9e37106b88..b9d19eb7d0 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -2155,6 +2155,8 @@ time_unit_conversion(Process *c_p, Eterm term, ErtsMonotonicTime val, ErtsMonoto
 	ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result));
 	break;
 #endif
+    case am_perf_counter:
+	goto trap_to_erlang_code;
     default: {
 	Eterm value, native_res;
 #ifndef ARCH_64
-- 
cgit v1.2.3


From 9d19331d3e2402a74346e664befb1d1e0749240f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 10 Apr 2018 16:15:18 +0200
Subject: erts: Workaround test t_float_to_string for windows

---
 erts/emulator/test/num_bif_SUITE.erl | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl
index 104bd37817..290bb61fc8 100644
--- a/erts/emulator/test/num_bif_SUITE.erl
+++ b/erts/emulator/test/num_bif_SUITE.erl
@@ -213,7 +213,7 @@ fts_rand_float_decimals(N) ->
     [begin
          F0 = rand_float_reasonable(),
          L0 = float_to_list(F0, [{decimals, D}]),
-         case conform_with_io_lib_format(F0,D) of
+         case conform_with_io_lib_format_os(F0,D) of
              false -> ok;
              true ->
                  IOL = lists:flatten(io_lib:format("~.*f", [D, F0])),
@@ -248,6 +248,15 @@ fts_rand_float_decimals(N) ->
 
     fts_rand_float_decimals(N-1).
 
+conform_with_io_lib_format_os(F, D) ->
+    case os:type() of
+        {win32,_} ->
+            %% io_lib:format("~.*f") buggy on windows? OTP-15010
+            false;
+        _ ->
+            conform_with_io_lib_format(F, D)
+    end.
+
 conform_with_io_lib_format(_, 0) ->
     %% io_lib:format("~.*f") does not support zero decimals
     false;
-- 
cgit v1.2.3


From 28e9c5673433054eee49b49ccdc919183326b686 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 11 Apr 2018 16:09:38 +0200
Subject: Fix VM probes compilation

---
 erts/emulator/beam/erl_message.c | 24 ------------------------
 1 file changed, 24 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 98feb95d99..8f9d1c257b 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -264,11 +264,6 @@ erts_queue_dist_message(Process *rcvr,
                         Eterm from)
 {
     ErtsMessage* mp;
-#ifdef USE_VM_PROBES
-    Sint tok_label = 0;
-    Sint tok_lastcnt = 0;
-    Sint tok_serial = 0;
-#endif
     erts_aint_t state;
 
     ERTS_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr));
@@ -309,25 +304,6 @@ erts_queue_dist_message(Process *rcvr,
     }
     else {
 
-#ifdef USE_VM_PROBES
-        if (DTRACE_ENABLED(message_queued)) {
-            DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
-
-            dtrace_proc_str(rcvr, receiver_name);
-            if (have_seqtrace(token)) {
-                tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
-                tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
-                tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
-            }
-            /*
-             * TODO: We don't know the real size of the external message here.
-             *       -1 will appear to a D script as 4294967295.
-             */
-            DTRACE6(message_queued, receiver_name, -1, rcvr->sig_qs.len + 1,
-                    tok_label, tok_lastcnt, tok_serial);
-        }
-#endif
-
 	LINK_MESSAGE(rcvr, mp, &mp->next, 1);
 
 	if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
-- 
cgit v1.2.3


From 9f8a402cc3e49313089bb9e22bc625f07beea4ca Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 27 Mar 2018 11:26:39 +0200
Subject: New process_info() implementation using signals

---
 erts/emulator/beam/beam_emu.c           |    6 +-
 erts/emulator/beam/bif.c                |    3 +-
 erts/emulator/beam/bif.h                |    6 +
 erts/emulator/beam/binary.c             |   34 +
 erts/emulator/beam/break.c              |    5 +-
 erts/emulator/beam/erl_bif_info.c       | 1361 +++++++++++++++++--------------
 erts/emulator/beam/erl_gc.c             |   48 +-
 erts/emulator/beam/erl_gc.h             |    2 +
 erts/emulator/beam/erl_message.c        |  101 +--
 erts/emulator/beam/erl_message.h        |   91 ++-
 erts/emulator/beam/erl_proc_sig_queue.c |  655 +++++++++++----
 erts/emulator/beam/erl_proc_sig_queue.h |  206 ++++-
 erts/emulator/beam/erl_process.c        |   42 +-
 erts/emulator/beam/erl_process.h        |   10 +-
 erts/emulator/beam/erl_process_dict.c   |   71 +-
 erts/emulator/beam/erl_process_dict.h   |    2 +-
 erts/emulator/beam/erl_process_dump.c   |  121 +--
 erts/emulator/beam/global.h             |    2 +
 erts/emulator/beam/msg_instrs.tab       |    4 +-
 erts/emulator/beam/utils.c              |    1 +
 erts/emulator/test/bif_SUITE.erl        |   75 +-
 erts/emulator/test/process_SUITE.erl    |  116 ++-
 22 files changed, 1981 insertions(+), 981 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index fb87be3f17..ee287243a4 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2420,8 +2420,7 @@ erts_hibernate(Process* c_p, Eterm* reg)
      * shrink the heap. 
      */
     erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
-    erts_proc_sig_fetch(c_p);
-    if (!c_p->sig_qs.len) {
+    if (!erts_proc_sig_fetch(c_p)) {
 	erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
 	c_p->fvalue = NIL;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2429,8 +2428,7 @@ erts_hibernate(Process* c_p, Eterm* reg)
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
-        erts_proc_sig_fetch(c_p);
-	if (!c_p->sig_qs.len)
+	if (!erts_proc_sig_fetch(c_p))
 	    erts_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
     }
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 232597c5b6..017faffa48 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4147,7 +4147,8 @@ BIF_RETTYPE erts_internal_group_leader_2(BIF_ALIST_2)
             rp = BIF_P;
         else {
             rp = erts_try_lock_sig_free_proc(BIF_ARG_2,
-                                             ERTS_PROC_LOCK_MAIN);
+                                             ERTS_PROC_LOCK_MAIN,
+                                             NULL);
             if (!rp)
                 BIF_RET(am_badarg);
             if (rp == ERTS_PROC_LOCK_BUSY)
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index a33421d762..a47339253e 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -425,6 +425,12 @@ do {									\
     BIF_TRAP3((TRP), (P), (A0), (A1), (A2));				\
 } while (0)
 
+#define ERTS_BIF_PREP_EXITED(RET, PROC)	                                \
+do {                                                                    \
+    KILL_CATCHES((PROC));                                               \
+    ERTS_BIF_PREP_ERROR((RET), (PROC), EXTAG_EXIT);                     \
+} while (0)
+
 #define ERTS_BIF_EXITED(PROC)		\
 do {					\
     KILL_CATCHES((PROC));		\
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 95d324d2c1..d53f75c279 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -113,6 +113,40 @@ new_binary(Process *p, byte *buf, Uint len)
     return build_proc_bin(&MSO(p), HAlloc(p, PROC_BIN_SIZE), bptr);
 }
 
+Eterm
+erts_heap_factory_new_binary(ErtsHeapFactory *hfact, byte *buf, Uint len,
+                             Uint reserve_size)
+{
+    Eterm *hp;
+    Binary* bptr;
+
+    if (len <= ERL_ONHEAP_BIN_LIMIT) {
+	ErlHeapBin* hb;
+        hp = erts_produce_heap(hfact, heap_bin_size(len), reserve_size);
+        hb = (ErlHeapBin *) hp;
+	hb->thing_word = header_heap_bin(len);
+	hb->size = len;
+	if (buf != NULL) {
+	    sys_memcpy(hb->data, buf, len);
+	}
+	return make_binary(hb);
+    }
+
+    /*
+     * Allocate the binary struct itself.
+     */
+    bptr = erts_bin_nrml_alloc(len);
+    if (buf != NULL) {
+	sys_memcpy(bptr->orig_bytes, buf, len);
+    }
+
+    hp = erts_produce_heap(hfact, PROC_BIN_SIZE, reserve_size);
+
+    return build_proc_bin(hfact->off_heap, hp, bptr);
+}
+
+
+
 /* 
  * When heap binary is not desired...
  */
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 4dabac3512..ba8cc5e2ba 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -204,6 +204,7 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p)
 {
     int garbing = 0;
     int running = 0;
+    Sint len;
     struct saved_calls *scb;
     erts_aint32_t state;
 
@@ -252,9 +253,9 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p)
     erts_print(to, to_arg, "Spawned by: %T\n", p->parent);
 
     erts_proc_lock(p, ERTS_PROC_LOCK_MSGQ);
-    erts_proc_sig_fetch(p);
+    len = erts_proc_sig_fetch(p);
     erts_proc_unlock(p, ERTS_PROC_LOCK_MSGQ);
-    erts_print(to, to_arg, "Message queue length: %d\n", p->sig_qs.len);
+    erts_print(to, to_arg, "Message queue length: %d\n", len);
 
     /* display the message queue only if there is anything in it */
     if (!ERTS_IS_CRASH_DUMPING && p->sig_qs.first != NULL && !garbing) {
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index bdca93428e..d443e12409 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -155,8 +155,10 @@ static Eterm os_type_tuple;
 static Eterm os_version_tuple;
 
 static Eterm
-current_function(Process* p, Process* rp, Eterm** hpp, int full_info);
-static Eterm current_stacktrace(Process* p, Process* rp, Eterm** hpp);
+current_function(Process* p, ErtsHeapFactory *hfact, Process* rp,
+                 int full_info, Uint reserve_size, int flags);
+static Eterm current_stacktrace(ErtsHeapFactory *hfact, Process* rp,
+                                Uint reserve_size);
 
 static Eterm
 bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh)
@@ -693,119 +695,222 @@ collect_one_suspend_monitor(ErtsMonitor *mon, void *vsmicp)
  * process_info/[1,2]
  */
 
-#define ERTS_PI_FAIL_TYPE_BADARG		0
-#define ERTS_PI_FAIL_TYPE_YIELD			1
-#define ERTS_PI_FAIL_TYPE_EXITED                2
-
-static ERTS_INLINE ErtsProcLocks
-pi_locks(Eterm info)
-{
-    switch (info) {
-    case am_priority:
-    case am_status:
-        return 0;
-    case am_suspended:
-        return ERTS_PROC_LOCK_STATUS;
-    case am_messages:
-    case am_message_queue_len:
-    case am_total_heap_size:
-        return ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_MSGQ;
-    default:
-        return ERTS_PROC_LOCK_MAIN;
-    }
-}
-
 /*
  * All valid process_info arguments.
  */
-static Eterm pi_args[] = {
-    am_registered_name,
-    am_current_function,
-    am_initial_call,
-    am_status,
-    am_messages,
-    am_message_queue_len,
-    am_links,
-    am_monitors,
-    am_monitored_by,
-    am_dictionary,
-    am_trap_exit,
-    am_error_handler,
-    am_heap_size,
-    am_stack_size,
-    am_memory,
-    am_garbage_collection,
-    am_group_leader,
-    am_reductions,
-    am_priority,
-    am_trace,
-    am_binary,
-    am_sequential_trace_token,
-    am_catchlevel,
-    am_backtrace,
-    am_last_calls,
-    am_total_heap_size,
-    am_suspending,
-    am_min_heap_size,
-    am_min_bin_vheap_size,
-    am_max_heap_size,
-    am_current_location,
-    am_current_stacktrace,
-    am_message_queue_data,
-    am_garbage_collection_info,
-    am_magic_ref
+
+#define ERTS_PI_IX_REGISTERED_NAME                      0
+#define ERTS_PI_IX_CURRENT_FUNCTION                     1
+#define ERTS_PI_IX_INITIAL_CALL                         2
+#define ERTS_PI_IX_STATUS                               3
+#define ERTS_PI_IX_MESSAGES                             4
+#define ERTS_PI_IX_MESSAGE_QUEUE_LEN                    5
+#define ERTS_PI_IX_LINKS                                6
+#define ERTS_PI_IX_MONITORS                             7
+#define ERTS_PI_IX_MONITORED_BY                         8
+#define ERTS_PI_IX_DICTIONARY                           9
+#define ERTS_PI_IX_TRAP_EXIT                            10
+#define ERTS_PI_IX_ERROR_HANDLER                        11
+#define ERTS_PI_IX_HEAP_SIZE                            12
+#define ERTS_PI_IX_STACK_SIZE                           13
+#define ERTS_PI_IX_MEMORY                               14
+#define ERTS_PI_IX_GARBAGE_COLLECTION                   15
+#define ERTS_PI_IX_GROUP_LEADER                         16
+#define ERTS_PI_IX_REDUCTIONS                           17
+#define ERTS_PI_IX_PRIORITY                             18
+#define ERTS_PI_IX_TRACE                                19
+#define ERTS_PI_IX_BINARY                               20
+#define ERTS_PI_IX_SEQUENTIAL_TRACE_TOKEN               21
+#define ERTS_PI_IX_CATCHLEVEL                           22
+#define ERTS_PI_IX_BACKTRACE                            23
+#define ERTS_PI_IX_LAST_CALLS                           24
+#define ERTS_PI_IX_TOTAL_HEAP_SIZE                      25
+#define ERTS_PI_IX_SUSPENDING                           26
+#define ERTS_PI_IX_MIN_HEAP_SIZE                        27
+#define ERTS_PI_IX_MIN_BIN_VHEAP_SIZE                   28
+#define ERTS_PI_IX_MAX_HEAP_SIZE                        29
+#define ERTS_PI_IX_CURRENT_LOCATION                     30
+#define ERTS_PI_IX_CURRENT_STACKTRACE                   31
+#define ERTS_PI_IX_MESSAGE_QUEUE_DATA                   32
+#define ERTS_PI_IX_GARBAGE_COLLECTION_INFO              33
+#define ERTS_PI_IX_MAGIC_REF                            34
+#define ERTS_PI_IX_FULLSWEEP_AFTER                      35
+
+#define ERTS_PI_FLAG_SINGELTON                          (1 << 0)
+#define ERTS_PI_FLAG_ALWAYS_WRAP                        (1 << 1)
+#define ERTS_PI_FLAG_WANT_MSGS                          (1 << 2)
+#define ERTS_PI_FLAG_NEED_MSGQ_LEN                      (1 << 3)
+#define ERTS_PI_FLAG_FORCE_SIG_SEND                     (1 << 4)
+#define ERTS_PI_FLAG_REQUEST_FOR_OTHER                  (1 << 5)
+
+#define ERTS_PI_UNRESERVE(RS, SZ) \
+    (ASSERT((RS) >= (SZ)), (RS) -= (SZ))
+
+
+typedef struct {
+    Eterm name;
+    Uint reserve_size;
+    int flags;
+    ErtsProcLocks locks;
+} ErtsProcessInfoArgs;
+
+static ErtsProcessInfoArgs pi_args[] = {
+    {am_registered_name, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_current_function, 4, 0, ERTS_PROC_LOCK_MAIN},
+    {am_initial_call, 4, 0, ERTS_PROC_LOCK_MAIN},
+    {am_status, 0, 0, 0},
+    {am_messages, 0, ERTS_PI_FLAG_WANT_MSGS|ERTS_PI_FLAG_NEED_MSGQ_LEN|ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
+    {am_message_queue_len, 0, ERTS_PI_FLAG_NEED_MSGQ_LEN, ERTS_PROC_LOCK_MAIN},
+    {am_links, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
+    {am_monitors, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
+    {am_monitored_by, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
+    {am_dictionary, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
+    {am_trap_exit, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_error_handler, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_heap_size, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_stack_size, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_memory, 0, ERTS_PI_FLAG_NEED_MSGQ_LEN|ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
+    {am_garbage_collection, 3+2 + 3+2 + 3+2 + 3+2 + 3+2 + ERTS_MAX_HEAP_SIZE_MAP_SZ, 0, ERTS_PROC_LOCK_MAIN},
+    {am_group_leader, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_reductions, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_priority, 0, 0, 0},
+    {am_trace, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_binary, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
+    {am_sequential_trace_token, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_catchlevel, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_backtrace, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_last_calls, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_total_heap_size, 0, ERTS_PI_FLAG_NEED_MSGQ_LEN|ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
+    {am_suspending, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, 0},
+    {am_min_heap_size, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_min_bin_vheap_size, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_max_heap_size, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_current_location, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_current_stacktrace, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_message_queue_data, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_garbage_collection_info, ERTS_PROCESS_GC_INFO_MAX_SIZE, 0, ERTS_PROC_LOCK_MAIN},
+    {am_magic_ref, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
+    {am_fullsweep_after, 0, 0, ERTS_PROC_LOCK_MAIN}
 };
 
-#define ERTS_PI_ARGS ((int) (sizeof(pi_args)/sizeof(Eterm)))
+#define ERTS_PI_ARGS ((int) (sizeof(pi_args)/sizeof(pi_args[0])))
+
+#ifdef DEBUG
+#  define ERTS_PI_DEF_ARR_SZ 2
+#else
+#  define ERTS_PI_DEF_ARR_SZ ERTS_PI_ARGS
+#endif
 
 static ERTS_INLINE Eterm
 pi_ix2arg(int ix)
 {
     if (ix < 0 || ERTS_PI_ARGS <= ix)
 	return am_undefined;
-    return pi_args[ix];
+    return pi_args[ix].name;
+}
+
+static ERTS_INLINE int
+pi_ix2flags(int ix)
+{
+    if (ix < 0 || ERTS_PI_ARGS <= ix)
+	return 0;
+    return pi_args[ix].flags;
+}
+
+static ERTS_INLINE Uint
+pi_ix2rsz(int ix)
+{
+    if (ix < 0 || ERTS_PI_ARGS <= ix)
+	return 0;
+    return pi_args[ix].reserve_size;
+}
+
+static ERTS_INLINE ErtsProcLocks
+pi_ix2locks(int ix)
+{
+    if (ix < 0 || ERTS_PI_ARGS <= ix)
+	return 0;
+    return pi_args[ix].locks;
 }
 
 static ERTS_INLINE int
 pi_arg2ix(Eterm arg)
 {
     switch (arg) {
-    case am_registered_name:			return 0;
-    case am_current_function:			return 1;
-    case am_initial_call:			return 2;
-    case am_status:				return 3;
-    case am_messages:				return 4;
-    case am_message_queue_len:			return 5;
-    case am_links:				return 6;
-    case am_monitors:				return 7;
-    case am_monitored_by:			return 8;
-    case am_dictionary:				return 9;
-    case am_trap_exit:				return 10;
-    case am_error_handler:			return 11;
-    case am_heap_size:				return 12;
-    case am_stack_size:				return 13;
-    case am_memory:				return 14;
-    case am_garbage_collection:			return 15;
-    case am_group_leader:			return 16;
-    case am_reductions:				return 17;
-    case am_priority:				return 18;
-    case am_trace:				return 19;
-    case am_binary:				return 20;
-    case am_sequential_trace_token:		return 21;
-    case am_catchlevel:				return 22;
-    case am_backtrace:				return 23;
-    case am_last_calls:				return 24;
-    case am_total_heap_size:			return 25;
-    case am_suspending:				return 26;
-    case am_min_heap_size:			return 27;
-    case am_min_bin_vheap_size:			return 28;
-    case am_max_heap_size:			return 29;
-    case am_current_location:			return 30;
-    case am_current_stacktrace:			return 31;
-    case am_message_queue_data:			return 32;
-    case am_garbage_collection_info:		return 33;
-    case am_magic_ref:                          return 34;
-    default:					return -1;
+    case am_registered_name:
+        return ERTS_PI_IX_REGISTERED_NAME;
+    case am_current_function:
+        return ERTS_PI_IX_CURRENT_FUNCTION;
+    case am_initial_call:
+        return ERTS_PI_IX_INITIAL_CALL;
+    case am_status:
+        return ERTS_PI_IX_STATUS;
+    case am_messages:
+        return ERTS_PI_IX_MESSAGES;
+    case am_message_queue_len:
+        return ERTS_PI_IX_MESSAGE_QUEUE_LEN;
+    case am_links:
+        return ERTS_PI_IX_LINKS;
+    case am_monitors:
+        return ERTS_PI_IX_MONITORS;
+    case am_monitored_by:
+        return ERTS_PI_IX_MONITORED_BY;
+    case am_dictionary:
+        return ERTS_PI_IX_DICTIONARY;
+    case am_trap_exit:
+        return ERTS_PI_IX_TRAP_EXIT;
+    case am_error_handler:
+        return ERTS_PI_IX_ERROR_HANDLER;
+    case am_heap_size:
+        return ERTS_PI_IX_HEAP_SIZE;
+    case am_stack_size:
+        return ERTS_PI_IX_STACK_SIZE;
+    case am_memory:
+        return ERTS_PI_IX_MEMORY;
+    case am_garbage_collection:
+        return ERTS_PI_IX_GARBAGE_COLLECTION;
+    case am_group_leader:
+        return ERTS_PI_IX_GROUP_LEADER;
+    case am_reductions:
+        return ERTS_PI_IX_REDUCTIONS;
+    case am_priority:
+        return ERTS_PI_IX_PRIORITY;
+    case am_trace:
+        return ERTS_PI_IX_TRACE;
+    case am_binary:
+        return ERTS_PI_IX_BINARY;
+    case am_sequential_trace_token:
+        return ERTS_PI_IX_SEQUENTIAL_TRACE_TOKEN;
+    case am_catchlevel:
+        return ERTS_PI_IX_CATCHLEVEL;
+    case am_backtrace:
+        return ERTS_PI_IX_BACKTRACE;
+    case am_last_calls:
+        return ERTS_PI_IX_LAST_CALLS;
+    case am_total_heap_size:
+        return ERTS_PI_IX_TOTAL_HEAP_SIZE;
+    case am_suspending:
+        return ERTS_PI_IX_SUSPENDING;
+    case am_min_heap_size:
+        return ERTS_PI_IX_MIN_HEAP_SIZE;
+    case am_min_bin_vheap_size:
+        return ERTS_PI_IX_MIN_BIN_VHEAP_SIZE;
+    case am_max_heap_size:
+        return ERTS_PI_IX_MAX_HEAP_SIZE;
+    case am_current_location:
+        return ERTS_PI_IX_CURRENT_LOCATION;
+    case am_current_stacktrace:
+        return ERTS_PI_IX_CURRENT_STACKTRACE;
+    case am_message_queue_data:
+        return ERTS_PI_IX_MESSAGE_QUEUE_DATA;
+    case am_garbage_collection_info:
+	return ERTS_PI_IX_GARBAGE_COLLECTION_INFO;
+    case am_magic_ref:
+        return ERTS_PI_IX_MAGIC_REF;
+    case am_fullsweep_after:
+        return ERTS_PI_IX_FULLSWEEP_AFTER;
+    default:
+        return -1;
     }
 }
 
@@ -858,186 +963,41 @@ process_info_init(void)
 
 }
 
-static ERTS_INLINE Process *
-pi_lookup_proc(Process *c_p, Eterm pid, ErtsProcLocks *locks)
-{
-    /*
-     * If the main lock is needed, we use erts_pid2proc_not_running()
-     * instead of erts_pid2proc() for two reasons:
-     * * Current function of pid and possibly other information will
-     *   have been updated so that process_info() is consistent with an
-     *   info-request/info-response signal model.
-     * * We avoid blocking the whole scheduler executing the
-     *   process that is calling process_info() for a long time
-     *   which will happen if pid is currently running.
-     * The caller of process_info() may have to yield if pid
-     * is currently running.
-     */
-
-    if ((*locks) & ERTS_PROC_LOCK_MAIN) {
-        erts_aint32_t state;
-        int local_only, done;
-        Process *rp;
-        ErtsProcLocks more_locks;
-
-        rp = erts_pid2proc_not_running(c_p, ERTS_PROC_LOCK_MAIN,
-                                       pid, ERTS_PROC_LOCK_MAIN);
-
-        if (!rp || rp == ERTS_PROC_LOCK_BUSY)
-            return rp;
-
-        if ((*locks) & ERTS_PROC_LOCK_MSGQ) {
-            /*
-             * Move in queue into private queue and
-             * release msgq lock, enabling others to
-             * send messages to the process while it
-             * is being inspected...
-             */
-            erts_proc_lock(rp, ERTS_PROC_LOCK_MSGQ);
-            erts_proc_sig_fetch(rp);
-            erts_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ);
-            (*locks) &= ~ERTS_PROC_LOCK_MSGQ;
-	}
-
-        /*
-         * Handle all signals received up to this point
-         * in order to preserve signal order.
-         * 
-         * FIX ME: Should be done yielding...
-         */
-        local_only = 0;
-        do {
-            int r = CONTEXT_REDS;
-            done = erts_proc_sig_handle_incoming(rp, &state, &r,
-                                                 CONTEXT_REDS,
-                                                 local_only);
-            local_only = !0;
-            BUMP_REDS(c_p, r);
-        } while (!done && !(state & ERTS_PSFLG_EXITING));
-
-        if (state & ERTS_PSFLG_EXITING) {
-            if (rp != c_p)
-                erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
-            return NULL;
-        }
-        more_locks = (*locks) & ~ERTS_PROC_LOCK_MAIN;
-        if (more_locks)
-            erts_proc_lock(rp, more_locks);
-        return rp;
-    }
-
-    ASSERT(!((*locks) & ERTS_PROC_LOCK_MSGQ));
-
-    if (*locks)
-        return erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN,
-			     pid, *locks);
-
-    return erts_proc_lookup(pid);
-}
-
-
-
-
-
 static BIF_RETTYPE
-process_info_aux(Process *BIF_P,
+process_info_aux(Process *c_p,
+                 ErtsHeapFactory *hfact,
 		 Process *rp,
 		 ErtsProcLocks rp_locks,
-		 Eterm rpid,
-		 Eterm item,
-		 int always_wrap,
-                 int *reds);
-
-#define ERTS_PI_RES_ELEM_IX_BUF_INC 1024
-#define ERTS_PI_DEF_RES_ELEM_IX_BUF_SZ ERTS_PI_ARGS
+		 int item_ix,
+		 int flags,
+                 Uint *reserve_sizep,
+                 Uint *reds);
 
-static Eterm
-process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap,
-		  int *fail_type)
+Eterm
+erts_process_info(Process *c_p,
+                  ErtsHeapFactory *hfact,
+                  Process *rp,
+                  ErtsProcLocks rp_locks,
+                  int *item_ix,
+                  int item_ix_len,
+                  int flags,
+                  Uint reserve_size,
+                  Uint *reds)
 {
-    int want_messages = 0;
-    int def_res_elem_ix_buf[ERTS_PI_DEF_RES_ELEM_IX_BUF_SZ];
-    int *res_elem_ix = &def_res_elem_ix_buf[0];
-    int res_elem_ix_ix = -1;
-    int res_elem_ix_sz = ERTS_PI_DEF_RES_ELEM_IX_BUF_SZ;
+    Eterm res;
     Eterm part_res[ERTS_PI_ARGS];
-    Eterm res, arg;
-    Uint *hp, *hp_end;
-    ErtsProcLocks locks = (ErtsProcLocks) 0;
-    int res_len, ix;
-    Process *rp = NULL;
-    int reds = 0;
+    int item_ix_ix, ix;
 
-    *fail_type = ERTS_PI_FAIL_TYPE_BADARG;
+    if (ERTS_PI_FLAG_SINGELTON & flags) {
+        ASSERT(item_ix_len == 1);
+	res = process_info_aux(c_p, hfact, rp, rp_locks, item_ix[0],
+                               flags, &reserve_size, reds);
+        return res;
+    }
 
     for (ix = 0; ix < ERTS_PI_ARGS; ix++)
 	part_res[ix] = THE_NON_VALUE;
 
-    ASSERT(is_list(list));
-
-    while (is_list(list)) {
-	Eterm* consp = list_val(list);
-
-	arg = CAR(consp);
-	ix = pi_arg2ix(arg);
-	if (ix < 0) {
-	    res = THE_NON_VALUE;
-	    goto done;
-	}
-	if (arg == am_messages)
-	    want_messages = 1;
-	locks |= pi_locks(arg);
-	res_elem_ix_ix++;
-	if (res_elem_ix_ix >= res_elem_ix_sz) {
-	    if (res_elem_ix != &def_res_elem_ix_buf[0])
-		res_elem_ix =
-		    erts_realloc(ERTS_ALC_T_TMP,
-				 res_elem_ix,
-				 sizeof(int)*(res_elem_ix_sz
-					      += ERTS_PI_RES_ELEM_IX_BUF_INC));
-	    else {
-		int new_res_elem_ix_sz = ERTS_PI_RES_ELEM_IX_BUF_INC;
-		int *new_res_elem_ix = erts_alloc(ERTS_ALC_T_TMP,
-						  sizeof(int)*new_res_elem_ix_sz);
-		sys_memcpy((void *) new_res_elem_ix,
-			   (void *) res_elem_ix,
-			   sizeof(int)*res_elem_ix_sz);
-		res_elem_ix = new_res_elem_ix;
-		res_elem_ix_sz = new_res_elem_ix_sz;
-	    }
-	}
-	res_elem_ix[res_elem_ix_ix] = ix;
-	list = CDR(consp);
-    }
-    if (is_not_nil(list)) {
-	res = THE_NON_VALUE;
-	goto done;
-    }
-
-    res_len = res_elem_ix_ix+1;
-
-    ASSERT(res_len > 0);
-
-    rp = pi_lookup_proc(c_p, pid, &locks);
-    if (!rp) {
-        if (c_p->common.id != pid)
-            res = am_undefined;
-        else {
-            *fail_type = ERTS_PI_FAIL_TYPE_EXITED;
-            res = THE_NON_VALUE;
-        }
-	goto done;
-    }
-    else if (rp == ERTS_PROC_LOCK_BUSY) {
-	rp = NULL;
-	res = THE_NON_VALUE;
-	*fail_type = ERTS_PI_FAIL_TYPE_YIELD;
-	goto done;
-    }
-
-    if (c_p == rp)
-        locks |= ERTS_PROC_LOCK_MAIN;
-
     /*
      * We always handle 'messages' first if it should be part
      * of the result. This since if both 'messages' and
@@ -1045,35 +1005,31 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap,
      * change the result of 'message_queue_len' (in case
      * the queue contain bad distribution messages).
      */
-    if (want_messages) {
+    if (flags & ERTS_PI_FLAG_WANT_MSGS) {
 	ix = pi_arg2ix(am_messages);
 	ASSERT(part_res[ix] == THE_NON_VALUE);
-	res = process_info_aux(c_p, rp, locks, pid, am_messages,
-                               always_wrap, &reds);
+	res = process_info_aux(c_p, hfact, rp, rp_locks, ix,
+                               flags, &reserve_size, reds);
 	ASSERT(res != am_undefined);
 	ASSERT(res != THE_NON_VALUE);
         part_res[ix] = res;
     }
 
-    for (; res_elem_ix_ix >= 0; res_elem_ix_ix--) {
-	ix = res_elem_ix[res_elem_ix_ix];
+    for (item_ix_ix = item_ix_len - 1; item_ix_ix >= 0; item_ix_ix--) {
+	ix = item_ix[item_ix_ix];
 	if (part_res[ix] == THE_NON_VALUE) {
-	    arg = pi_ix2arg(ix);
-	    res = process_info_aux(c_p, rp, locks, pid, arg,
-                                   always_wrap, &reds);
-            if (res == am_undefined)
-                goto done;
+	    res = process_info_aux(c_p, hfact, rp, rp_locks, ix,
+                                   flags, &reserve_size, reds);
+            ASSERT(res != am_undefined);
 	    ASSERT(res != THE_NON_VALUE);
             part_res[ix] = res;
 	}
     }
 
-    hp = HAlloc(c_p, res_len*2);
-    hp_end = hp + res_len*2;
     res = NIL;
 
-    for (res_elem_ix_ix = res_len - 1; res_elem_ix_ix >= 0; res_elem_ix_ix--) {
-	ix = res_elem_ix[res_elem_ix_ix];
+    for (item_ix_ix = item_ix_len - 1; item_ix_ix >= 0; item_ix_ix--) {
+	ix = item_ix[item_ix_ix];
 	ASSERT(part_res[ix] != THE_NON_VALUE);
 	/*
 	 * If we should ignore the value of registered_name,
@@ -1081,155 +1037,310 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap,
 	 * beginning of process_info_aux().
 	 */
 	if (is_nil(part_res[ix])) {
-	    ASSERT(!always_wrap);
+	    ASSERT(!(flags & ERTS_PI_FLAG_ALWAYS_WRAP));
 	    ASSERT(pi_ix2arg(ix) == am_registered_name);
 	}
 	else {
+            Eterm *hp;
+            ERTS_PI_UNRESERVE(reserve_size, 2);
+            hp = erts_produce_heap(hfact, 2, reserve_size);
 	    res = CONS(hp, part_res[ix], res);
-	    hp += 2;
 	}
     }
 
-    if (!always_wrap) {
-	HRelease(c_p, hp_end, hp);
-    }
+    return res;
+}
 
- done:
+static void
+pi_setup_grow(int **arr, int *def_arr, Uint *sz, int ix);
 
-    if (c_p == rp)
-	locks &= ~ERTS_PROC_LOCK_MAIN;
-    if (locks && rp)
-	erts_proc_unlock(rp, locks);
+static BIF_RETTYPE
+process_info_bif(Process *c_p, Eterm pid, Eterm opt, int always_wrap, int pi2)
+{
+    ErtsHeapFactory hfact;
+    int def_arr[ERTS_PI_DEF_ARR_SZ];
+    int *item_ix = &def_arr[0];
+    Process *rp = NULL;
+    erts_aint32_t state;
+    BIF_RETTYPE ret;
+    Uint reds = 0;
+    ErtsProcLocks locks = 0;
+    int flags;
+    Uint reserve_size;
+    int len;
+    Eterm res;
 
-    if (res_elem_ix != &def_res_elem_ix_buf[0])
-	erts_free(ERTS_ALC_T_TMP, res_elem_ix);
+    ERTS_CT_ASSERT(ERTS_PI_DEF_ARR_SZ > 0);
 
-    BUMP_REDS(c_p, reds);
+    if (c_p->common.id == pid) {
+        int local_only = c_p->flags & F_LOCAL_SIGS_ONLY;
+        int sreds = ERTS_BIF_REDS_LEFT(c_p);
+        int sres;
 
-    return res;
-}
+        if (!local_only) {
+            erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
+            erts_proc_sig_fetch(c_p);
+            erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
+        }
 
-BIF_RETTYPE process_info_1(BIF_ALIST_1)
-{
-    Eterm res;
-    int fail_type;
+        sres = erts_proc_sig_handle_incoming(c_p, &state, &sreds, sreds, !0);
+        if (state & ERTS_PSFLG_EXITING) {
+            c_p->flags &= ~F_LOCAL_SIGS_ONLY;
+            goto exited;
+        }
+        if (!sres) {
+            /*
+             * More signals to handle; need to yield and continue.
+             * Prevent fetching of more signals by setting
+             * local-sigs-only flag.
+             */
+            c_p->flags |= F_LOCAL_SIGS_ONLY;
+            goto yield;
+        }
 
-    if (is_external_pid(BIF_ARG_1)
-	&& external_pid_dist_entry(BIF_ARG_1) == erts_this_dist_entry)
-	BIF_RET(am_undefined);
-	
-    if (is_not_internal_pid(BIF_ARG_1)) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
-
-    res = process_info_list(BIF_P, BIF_ARG_1, pi_1_keys_list, 0, &fail_type);
-    if (is_non_value(res)) {
-	switch (fail_type) {
-	case ERTS_PI_FAIL_TYPE_BADARG:
-	    BIF_ERROR(BIF_P, BADARG);
-	case ERTS_PI_FAIL_TYPE_YIELD:
-	    ERTS_BIF_YIELD1(bif_export[BIF_process_info_1], BIF_P, BIF_ARG_1);
-        case ERTS_PI_FAIL_TYPE_EXITED:
-            ERTS_BIF_EXITED(BIF_P);
-	default:
-	    erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__);
-	}
+        c_p->flags &= ~F_LOCAL_SIGS_ONLY;
     }
 
-    ASSERT(!(BIF_P->flags & F_P2PNR_RESCHED));
-    BIF_RET(res);
-}
+    if (is_atom(opt)) {
+	int ix = pi_arg2ix(opt);
+        item_ix[0] = ix;
+        len = 1;
+        locks = pi_ix2locks(ix);
+        reserve_size = 3 + pi_ix2rsz(ix);
+        flags = ERTS_PI_FLAG_SINGELTON;
+        flags |= pi_ix2flags(ix);
+        if (ix < 0)
+            goto badarg;
+    }
+    else {
+        Eterm list = opt;
+        Uint size = ERTS_PI_DEF_ARR_SZ;
 
+        len = 0;
+        reserve_size = 0;
+        locks = 0;
+        flags = 0;
 
-BIF_RETTYPE process_info_2(BIF_ALIST_2) 
-{
-    Eterm res;
-    Process *rp;
-    Eterm pid = BIF_ARG_1;
-    ErtsProcLocks info_locks;
-    int fail_type, reds = 0;
+        while (is_list(list)) {
+            Eterm *consp = list_val(list);
+            Eterm arg = CAR(consp);
+            int ix = pi_arg2ix(arg);
+            if (ix < 0)
+                goto badarg;
+
+            if (len >= size)
+                pi_setup_grow(&item_ix, def_arr, &size, len);
+
+            item_ix[len++] = ix;
+
+            locks |= pi_ix2locks(ix);
+            flags |= pi_ix2flags(ix);
+            reserve_size += pi_ix2rsz(ix);
+            reserve_size += 3; /* 2-tuple */
+            reserve_size += 2; /* cons */
+
+            list = CDR(consp);
+        }
+
+        if (is_not_nil(list))
+            goto badarg;
+    }
 
-    if (is_external_pid(pid)
-	&& external_pid_dist_entry(pid) == erts_this_dist_entry)
-	BIF_RET(am_undefined);
-	
     if (is_not_internal_pid(pid)) {
-	BIF_ERROR(BIF_P, BADARG);
+        if (is_external_pid(pid)
+            && external_pid_dist_entry(pid) == erts_this_dist_entry)
+            goto undefined;
+        goto badarg;
     }
 
-    if (is_nil(BIF_ARG_2))
-	BIF_RET(NIL);
+    if (always_wrap)
+        flags |= ERTS_PI_FLAG_ALWAYS_WRAP;
 
-    if (is_list(BIF_ARG_2)) {
-	res = process_info_list(BIF_P, BIF_ARG_1, BIF_ARG_2, 1, &fail_type);
-	if (is_non_value(res)) {
-	    switch (fail_type) {
-	    case ERTS_PI_FAIL_TYPE_BADARG:
-		BIF_ERROR(BIF_P, BADARG);
-	    case ERTS_PI_FAIL_TYPE_YIELD:
-		ERTS_BIF_YIELD2(bif_export[BIF_process_info_2], BIF_P,
-				BIF_ARG_1, BIF_ARG_2);
-            case ERTS_PI_FAIL_TYPE_EXITED:
-                ERTS_BIF_EXITED(BIF_P);
-	    default:
-		erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error",
-			 __FILE__, __LINE__);
-	    }
-	}
-	ASSERT(!(BIF_P->flags & F_P2PNR_RESCHED));
-	BIF_RET(res);
+    if (c_p->common.id == pid) {
+        rp = c_p;
+        if (locks & ~ERTS_PROC_LOCK_MAIN)
+            erts_proc_lock(c_p, locks & ~ERTS_PROC_LOCK_MAIN);
+        locks |= ERTS_PROC_LOCK_MAIN;
+    }
+    else {
+        if (flags & ERTS_PI_FLAG_FORCE_SIG_SEND)
+            goto send_signal;
+        rp = erts_try_lock_sig_free_proc(pid, locks, &state);
+        if (!rp)
+            goto undefined;
+        if (rp == ERTS_PROC_LOCK_BUSY) {
+            rp = NULL;
+            goto send_signal;
+        }
+        if (state & ERTS_PSFLG_EXITING) {
+            if (locks)
+                erts_proc_unlock(rp, locks);
+            locks = 0;
+            /* wait for it to terminate properly... */
+            goto send_signal;
+        }
+        if (flags & ERTS_PI_FLAG_NEED_MSGQ_LEN) {
+            ASSERT(locks & ERTS_PROC_LOCK_MAIN);
+            erts_proc_lock(rp, ERTS_PROC_LOCK_MSGQ);
+            erts_proc_sig_fetch(rp);
+            if (c_p->sig_qs.cont) {
+                erts_proc_unlock(rp, locks|ERTS_PROC_LOCK_MSGQ);
+                locks = 0;
+                goto send_signal;
+            }
+            erts_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ);
+        }
     }
 
-    if (pi_arg2ix(BIF_ARG_2) < 0)
-	BIF_ERROR(BIF_P, BADARG);
+    erts_factory_proc_init(&hfact, c_p);
 
-    info_locks = pi_locks(BIF_ARG_2); 
+    res = erts_process_info(c_p, &hfact, rp, locks, item_ix, len,
+                            flags, reserve_size, &reds);
 
-    rp = pi_lookup_proc(BIF_P, pid, &info_locks);
-    if (!rp) {
-        if (BIF_P->common.id == pid)
-            ERTS_BIF_EXITED(BIF_P);
-	BIF_RET(am_undefined);
+    erts_factory_close(&hfact);
+
+    if (reds > INT_MAX/2)
+        reds = INT_MAX/2;
+    BUMP_REDS(c_p, (int) reds);
+
+    state = erts_atomic32_read_acqb(&rp->state);
+    if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE)) {
+        if (state & ERTS_PSFLG_FREE) {
+            ASSERT(!locks);
+            goto undefined;
+        }
+        if (locks)
+            erts_proc_unlock(rp, locks);
+        locks = 0;
+        /* wait for it to terminate properly... */
+        goto send_signal;
     }
-    else if (rp == ERTS_PROC_LOCK_BUSY)
-	ERTS_BIF_YIELD2(bif_export[BIF_process_info_2], BIF_P,
-			BIF_ARG_1, BIF_ARG_2);
 
-    if (BIF_P == rp)
-        info_locks |= ERTS_PROC_LOCK_MAIN;
+    ERTS_BIF_PREP_RET(ret, res);
+
+done:
 
-    res = process_info_aux(BIF_P, rp, info_locks, pid, BIF_ARG_2,
-                           0, &reds);
+    if (c_p == rp)
+        locks &= ~ERTS_PROC_LOCK_MAIN;
 
-    BUMP_REDS(BIF_P, reds);
+    if (locks && rp)
+	erts_proc_unlock(rp, locks);
 
-    if (BIF_P == rp)
-	info_locks &= ~ERTS_PROC_LOCK_MAIN;
-    if (rp && info_locks)
-	erts_proc_unlock(rp, info_locks);
+    if (item_ix != def_arr)
+        erts_free(ERTS_ALC_T_TMP, item_ix);
 
-    ASSERT(!(BIF_P->flags & F_P2PNR_RESCHED));
-    BIF_RET(res);
+    return ret;
+
+badarg:
+    ERTS_BIF_PREP_ERROR(ret, c_p, BADARG);
+    goto done;
+
+undefined:
+    ERTS_BIF_PREP_RET(ret, am_undefined);
+    goto done;
+
+exited:
+    ERTS_BIF_PREP_EXITED(ret, c_p);
+    goto done;
+
+yield:
+    if (pi2)
+        ERTS_BIF_PREP_YIELD2(ret, bif_export[BIF_process_info_2], c_p, pid, opt);
+    else
+        ERTS_BIF_PREP_YIELD1(ret, bif_export[BIF_process_info_1], c_p, pid);
+    goto done;
+
+send_signal: {
+        Eterm ref = erts_make_ref(c_p);
+        int enqueued, need_msgq_len;
+        flags |= ERTS_PI_FLAG_REQUEST_FOR_OTHER;
+        need_msgq_len = (flags & ERTS_PI_FLAG_NEED_MSGQ_LEN);
+        /*
+         * Set receive mark so we wont have to scan the whole
+         * message queue for the result. Note caller unconditionally
+         * has to enter a receive only matching messages containing
+         * 'ref', or restore save pointer.
+         */
+        ERTS_RECV_MARK_SAVE(c_p);
+        ERTS_RECV_MARK_SET(c_p);
+        enqueued = erts_proc_sig_send_process_info_request(c_p, pid, item_ix,
+                                                           len, need_msgq_len,
+                                                           flags, reserve_size,
+                                                           ref);
+        if (!enqueued) {
+            /* Restore save pointer... */
+	    JOIN_MESSAGE(c_p);
+            goto undefined;
+        }
+        ERTS_BIF_PREP_TRAP1(ret, erts_await_result, c_p, ref);
+        goto done;
+    }
+}
+
+static void
+pi_setup_grow(int **arr, int *def_arr, Uint *sz, int ix)
+{
+    *sz = (ix+1) + ERTS_PI_DEF_ARR_SZ;
+    if (*arr != def_arr)
+        *arr = erts_realloc(ERTS_ALC_T_TMP, *arr, (*sz)*sizeof(int));
+    else {
+        int *new_arr = erts_alloc(ERTS_ALC_T_TMP, (*sz)*sizeof(int));
+        sys_memcpy((void *) new_arr, (void *) def_arr,
+                   sizeof(int)*ERTS_PI_DEF_ARR_SZ);
+        *arr = new_arr;
+    }
+}
+
+
+BIF_RETTYPE process_info_2(BIF_ALIST_2)
+{
+    return process_info_bif(BIF_P, BIF_ARG_1, BIF_ARG_2, !is_atom(BIF_ARG_2), !0);
+}
+
+BIF_RETTYPE process_info_1(BIF_ALIST_1)
+{
+    return process_info_bif(BIF_P, BIF_ARG_1, pi_1_keys_list, 0, 0);
 }
 
 Eterm
-process_info_aux(Process *BIF_P,
+process_info_aux(Process *c_p,
+                 ErtsHeapFactory *hfact,
 		 Process *rp,
 		 ErtsProcLocks rp_locks,
-		 Eterm rpid,
-		 Eterm item,
-		 int always_wrap,
-                 int *reds)
+		 int item_ix,
+		 int flags,
+                 Uint *reserve_sizep,
+                 Uint *reds)
 {
     Eterm *hp;
     Eterm res = NIL;
+    Uint reserved;
+    Uint reserve_size = *reserve_sizep;
+
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(rp);
+
+    switch (item_ix) {
+    case ERTS_PI_IX_STATUS:
+    case ERTS_PI_IX_PRIORITY:
+    case ERTS_PI_IX_SUSPENDING:
+        ERTS_LC_ASSERT((locks & ~ERTS_PROC_LOCK_MAIN) == 0);
+        break;
+    default:
+        ERTS_LC_ASSERT(locks == ERTS_PROC_LOCK_MAIN);
+        break;
+    }
+#endif
+
+    reserved = pi_ix2rsz(item_ix);
+    ERTS_PI_UNRESERVE(reserve_size, reserved);
 
     (*reds)++;
 
     ASSERT(rp);
 
     /*
-     * Q: Why this always_wrap argument?
+     * Q: Why this ERTS_PI_FLAG_ALWAYS_WRAP flag?
      *
      * A: registered_name is strange. If process has no registered name,
      *    process_info(Pid, registered_name) returns [], and
@@ -1241,41 +1352,39 @@ process_info_aux(Process *BIF_P,
      *    registered_name behaves as it should, i.e. a
      *    {registered_name, []} will appear in the resulting list.
      *
-     *    If always_wrap != 0, process_info_aux() always wrap the result
-     *    in a key two tuple. 
+     *    If ERTS_PI_FLAG_ALWAYS_WRAP is set, process_info_aux() always
+     *    wrap the result in a key two tuple. 
      */
 
-    switch (item) {
+    switch (item_ix) {
 
-    case am_registered_name:
-	if (rp->common.u.alive.reg) {
-	    hp = HAlloc(BIF_P, 3);
+    case ERTS_PI_IX_REGISTERED_NAME:
+	if (rp->common.u.alive.reg)
 	    res = rp->common.u.alive.reg->name;
-	} else {
-	    if (always_wrap) {
-		hp = HAlloc(BIF_P, 3);
+        else {
+	    if (flags & ERTS_PI_FLAG_ALWAYS_WRAP)
 		res = NIL;
-	    }
-	    else {
+	    else
 		return NIL;
-	    }
 	}
 	break;
 
-    case am_current_function:
-	res = current_function(BIF_P, rp, &hp, 0);
+    case ERTS_PI_IX_CURRENT_FUNCTION:
+	res = current_function(c_p, hfact, rp, 0,
+                               reserve_size, flags);
 	break;
 
-    case am_current_location:
-	res = current_function(BIF_P, rp, &hp, 1);
+    case ERTS_PI_IX_CURRENT_LOCATION:
+	res = current_function(c_p, hfact, rp, 1,
+                               reserve_size, flags);
 	break;
 
-    case am_current_stacktrace:
-	res = current_stacktrace(BIF_P, rp, &hp);
+    case ERTS_PI_IX_CURRENT_STACKTRACE:
+	res = current_stacktrace(hfact, rp, reserve_size);
 	break;
 
-    case am_initial_call:
-	hp = HAlloc(BIF_P, 3+4);
+    case ERTS_PI_IX_INITIAL_CALL:
+        hp = erts_produce_heap(hfact, 4, reserve_size);
 	res = TUPLE3(hp,
 		     rp->u.initial.module,
 		     rp->u.initial.function,
@@ -1283,24 +1392,37 @@ process_info_aux(Process *BIF_P,
 	hp += 4;
 	break;
 
-    case am_status:
-        res = erts_process_state2status(erts_atomic32_read_nob(&rp->state));
-        if (res == am_exiting || res == am_free)
-            return am_undefined;
-	hp = HAlloc(BIF_P, 3);
+    case ERTS_PI_IX_STATUS: {
+        erts_aint32_t state = erts_atomic32_read_nob(&rp->state);
+        res = erts_process_state2status(state);
+        if (res == am_running && (state & ERTS_PSFLG_RUNNING_SYS)) {
+            ASSERT(c_p == rp);
+            ASSERT(flags & ERTS_PI_FLAG_REQUEST_FOR_OTHER);
+            if (!(state & (ERTS_PSFLG_SYS_TASKS
+                           | ERTS_PSFLG_ACTIVE
+                           | ERTS_PSFLG_SIG_Q
+                           | ERTS_PSFLG_SIG_IN_Q))) {
+                /*
+                 * We are servicing a process-info request from
+                 * another process. If that other process could
+                 * have inspected our state itself, we would have
+                 * been in the 'waiting' state.
+                 */
+                res = am_waiting;
+            }
+        }
 	break;
+    }
 
-    case am_messages: {
-
-	if (rp->sig_qs.len == 0 || ERTS_TRACE_FLAGS(rp) & F_SENSITIVE) {
-	    hp = HAlloc(BIF_P, 3);
-	} else {
+    case ERTS_PI_IX_MESSAGES: {
+        ASSERT(flags & ERTS_PI_FLAG_NEED_MSGQ_LEN);
+	if (rp->sig_qs.len == 0 || (ERTS_TRACE_FLAGS(rp) & F_SENSITIVE))
+            res = NIL;
+        else {
+            int info_on_self = !(flags & ERTS_PI_FLAG_REQUEST_FOR_OTHER);
 	    ErtsMessageInfo *mip;
-	    Sint i;
+	    Sint i, len;
 	    Uint heap_need;
-#ifdef DEBUG
-	    Eterm *hp_end;
-#endif
 
 	    mip = erts_alloc(ERTS_ALC_T_TMP,
 			     rp->sig_qs.len*sizeof(ErtsMessageInfo));
@@ -1310,46 +1432,52 @@ process_info_aux(Process *BIF_P,
 	     * erts_proc_sig_prep_msgq_for_inspection() since it removes
 	     * corrupt distribution messages.
 	     */
-	    heap_need = erts_proc_sig_prep_msgq_for_inspection(BIF_P, rp,
-                                                               rp_locks, mip);
-	    heap_need += 3; /* top 2-tuple */
-	    heap_need += rp->sig_qs.len*2; /* Cons cells */
+	    heap_need = erts_proc_sig_prep_msgq_for_inspection(c_p, rp,
+                                                               rp_locks,
+                                                               info_on_self,
+                                                               mip);
+            len = rp->sig_qs.len;
 
-	    hp = HAlloc(BIF_P, heap_need); /* heap_need is exact */
-#ifdef DEBUG
-	    hp_end = hp + heap_need;
-#endif
+	    heap_need += len*2; /* Cons cells */
+
+            reserve_size += heap_need;
 
 	    /* Build list of messages... */
-	    for (i = rp->sig_qs.len - 1, res = NIL; i >= 0; i--) {
+	    for (i = len - 1, res = NIL; i >= 0; i--) {
 		Eterm msg = ERL_MESSAGE_TERM(mip[i].msgp);
 		Uint sz = mip[i].size;
 
+                ERTS_PI_UNRESERVE(reserve_size, sz+2);
+                hp = erts_produce_heap(hfact, sz+2, reserve_size);
+
 		if (sz != 0)
-		    msg = copy_struct(msg, sz, &hp, &BIF_P->off_heap);
+		    msg = copy_struct(msg, sz, &hp, hfact->off_heap);
 
 		res = CONS(hp, msg, res);
 		hp += 2;
 	    }
 
-	    ASSERT(hp_end == hp + 3);
-
-            if (rp->sig_qs.len > CONTEXT_REDS*4)
-                *reds += CONTEXT_REDS*4;
-            else
-                *reds += rp->sig_qs.len / 4;
+            *reds += (Uint) len / 4;
 
 	    erts_free(ERTS_ALC_T_TMP, mip);
 	}
 	break;
     }
 
-    case am_message_queue_len:
-	hp = HAlloc(BIF_P, 3);
-	res = make_small(rp->sig_qs.len);
+    case ERTS_PI_IX_MESSAGE_QUEUE_LEN: {
+        Sint len = rp->sig_qs.len;
+        ASSERT(flags & ERTS_PI_FLAG_NEED_MSGQ_LEN);
+        ASSERT(len >= 0);
+        if (len <= MAX_SMALL)
+            res = make_small(len);
+        else {
+            hp = erts_produce_heap(hfact, BIG_UINT_HEAP_SIZE, reserve_size);
+            res = uint_to_big((Uint) len, hp);
+        }
 	break;
+    }
 
-    case am_links: {
+    case ERTS_PI_IX_LINKS: {
 	MonitorInfoCollection mic;
 	int i;
 	Eterm item;
@@ -1358,24 +1486,24 @@ process_info_aux(Process *BIF_P,
 
 	erts_link_tree_foreach(ERTS_P_LINKS(rp), collect_one_link, (void *) &mic);
 
-	hp = HAlloc(BIF_P, 3 + mic.sz);
+        reserve_size += mic.sz;
 	res = NIL;
 	for (i = 0; i < mic.mi_i; i++) {
-	    item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity.term); 
+            Eterm item_src = mic.mi[i].entity.term;
+            Uint sz = NC_HEAP_SIZE(item_src) + 2;
+            ERTS_PI_UNRESERVE(reserve_size, sz);
+            hp = erts_produce_heap(hfact, sz, reserve_size);
+	    item = STORE_NC(&hp, hfact->off_heap, item_src); 
 	    res = CONS(hp, item, res);
-	    hp += 2;
 	}
 
-        if (mic.mi_i > CONTEXT_REDS*4)
-            *reds += CONTEXT_REDS*4;
-        else
-            *reds += mic.mi_i / 4;
+        *reds += (Uint) mic.mi_i / 4;
 
 	DESTROY_MONITOR_INFOS(mic);
 	break;
     }
 
-    case am_monitors: {
+    case ERTS_PI_IX_MONITORS: {
 	MonitorInfoCollection mic;
         int i;
 
@@ -1384,7 +1512,7 @@ process_info_aux(Process *BIF_P,
                                   collect_one_origin_monitor,
                                   (void *) &mic);
 
-        hp = HAlloc(BIF_P, 3 + mic.sz);
+        reserve_size += mic.sz;
 	res = NIL;
 	for (i = 0; i < mic.mi_i; i++) {
 	    if (mic.mi[i].named) {
@@ -1399,23 +1527,30 @@ process_info_aux(Process *BIF_P,
                     || is_port(mic.mi[i].pid)
                     || is_atom(mic.mi[i].pid));
 
+                ERTS_PI_UNRESERVE(reserve_size, 3+3+2);
+                hp = erts_produce_heap(hfact, 3+3+2, reserve_size);
+
 		t1 = TUPLE2(hp, mic.mi[i].entity.term, mic.mi[i].node);
 		hp += 3;
                 t2 = TUPLE2(hp, m_type, t1);
 		hp += 3;
 		res = CONS(hp, t2, res);
-                hp += 2;
 	    }
 	    else {
                 /* Build {process|port|time_offset, Pid|clock_service} and cons it. */
 		Eterm t;
 		Eterm pid;
                 Eterm m_type;
+                Eterm pid_src = mic.mi[i].entity.term;
+                Uint sz = is_atom(pid_src) ? 0 : NC_HEAP_SIZE(pid_src);
+                sz += 3 + 2;
 
-                if (is_atom(mic.mi[i].entity.term))
-                    pid = mic.mi[i].entity.term;
-                else
-                    pid = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity.term);
+                ERTS_PI_UNRESERVE(reserve_size, sz);
+                hp = erts_produce_heap(hfact, sz, reserve_size);
+
+                pid = (is_atom(pid_src)
+                       ? pid_src
+                       : STORE_NC(&hp, hfact->off_heap, pid_src));
 
                 switch (mic.mi[i].type) {
                 case ERTS_MON_TYPE_PORT:
@@ -1435,20 +1570,16 @@ process_info_aux(Process *BIF_P,
                 t = TUPLE2(hp, m_type, pid);
 		hp += 3;
 		res = CONS(hp, t, res);
-                hp += 2;
 	    }
 	}
 
-        if (mic.mi_i > CONTEXT_REDS*4)
-            *reds += CONTEXT_REDS*4;
-        else
-            *reds += mic.mi_i / 4;
+        *reds += (Uint) mic.mi_i / 4;
 
         DESTROY_MONITOR_INFOS(mic);
 	break;
     }
 
-    case am_monitored_by: {
+    case ERTS_PI_IX_MONITORED_BY: {
 	MonitorInfoCollection mic;
 	int i;
 	Eterm item;
@@ -1461,59 +1592,71 @@ process_info_aux(Process *BIF_P,
                                   collect_one_target_monitor,
                                   (void *) &mic);
 
-	hp = HAlloc(BIF_P, 3 + mic.sz);
+        reserve_size += mic.sz;
 
 	res = NIL;
 	for (i = 0; i < mic.mi_i; ++i) {
+            Uint sz = 2;
+
+            if (mic.mi[i].type == ERTS_MON_TYPE_RESOURCE)
+                sz += erts_resource_ref_size(mic.mi[i].entity.resource);
+            else
+                sz += NC_HEAP_SIZE(mic.mi[i].entity.term);
+
+            ERTS_PI_UNRESERVE(reserve_size, sz);
+            hp = erts_produce_heap(hfact, sz, reserve_size);
+
             if (mic.mi[i].type == ERTS_MON_TYPE_RESOURCE)
                 item = erts_bld_resource_ref(&hp,
-                                             &MSO(BIF_P),
+                                             hfact->off_heap,
                                              mic.mi[i].entity.resource);
             else
                 item = STORE_NC(&hp,
-                                &MSO(BIF_P),
+                                hfact->off_heap,
                                 mic.mi[i].entity.term);
 	    res = CONS(hp, item, res);
-	    hp += 2;
 	}
 
-        if (mic.mi_i > CONTEXT_REDS*4)
-            *reds += CONTEXT_REDS*4;
-        else
-            *reds += mic.mi_i / 4;
+        *reds += (Uint) mic.mi_i / 4;
 
 	DESTROY_MONITOR_INFOS(mic);
 	break;
     }
 
-    case am_suspending: {
+    case ERTS_PI_IX_SUSPENDING: {
 	ErtsSuspendMonitorInfoCollection smic;
 	int i;
 	Eterm item;
-#ifdef DEBUG
-	Eterm *hp_end;
-#endif
+
+        erts_proc_lock(rp, ERTS_PROC_LOCK_STATUS);
 
 	ERTS_INIT_SUSPEND_MONITOR_INFOS(smic,
-					BIF_P,
-					(BIF_P == rp
+					c_p,
+					(c_p == rp
 					 ? ERTS_PROC_LOCK_MAIN
 					 : 0) | ERTS_PROC_LOCK_STATUS);
 
 	erts_monitor_tree_foreach(rp->suspend_monitors,
                                   &collect_one_suspend_monitor,
                                   &smic);
-	hp = HAlloc(BIF_P, 3 + smic.sz);
-#ifdef DEBUG
-	hp_end = hp + smic.sz;
-#endif
-	
+
+        reserve_size += smic.sz;
+
 	res = NIL;
 	for (i = 0; i < smic.smi_i; i++) {
 	    Sint a = (Sint) smic.smi[i]->active;  /* quiet compiler warnings */
 	    Sint p = (Sint) smic.smi[i]->pending; /* on 64-bit machines...   */
 	    Eterm active;
 	    Eterm pending;
+            Uint sz = 4 + 2;
+            if (!IS_SSMALL(a))
+		sz += BIG_UINT_HEAP_SIZE;
+            if (!IS_SSMALL(p))
+		sz += BIG_UINT_HEAP_SIZE;
+
+            ERTS_PI_UNRESERVE(reserve_size, sz);
+            hp = erts_produce_heap(hfact, sz, reserve_size);
+
 	    if (IS_SSMALL(a))
 		active = make_small(a);
 	    else {
@@ -1529,95 +1672,83 @@ process_info_aux(Process *BIF_P,
 	    item = TUPLE3(hp, smic.smi[i]->mon.other.item, active, pending);
 	    hp += 4;
 	    res = CONS(hp, item, res);
-	    hp += 2;
 	}
 
-        if (smic.smi_i > CONTEXT_REDS*4)
-            *reds += CONTEXT_REDS*4;
-        else
-            *reds += smic.smi_i / 4;
+        erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
+
+        *reds += (Uint) smic.smi_i / 4;
 
 	ERTS_DESTROY_SUSPEND_MONITOR_INFOS(smic);
-	ASSERT(hp == hp_end);
 
 	break;
     }
 
-    case am_dictionary:
+    case ERTS_PI_IX_DICTIONARY:
 	if (!rp->dictionary || (ERTS_TRACE_FLAGS(rp) & F_SENSITIVE)) {
 	    res = NIL;
 	} else {
             Uint num = rp->dictionary->numElements;
-	    res = erts_dictionary_copy(BIF_P, rp->dictionary);
-            if (num > CONTEXT_REDS*4)
-                *reds += CONTEXT_REDS*4;
-            else
-                *reds += (int) num / 4;
+	    res = erts_dictionary_copy(hfact, rp->dictionary, reserve_size);
+            *reds += (Uint) num / 4;
 	}
-	hp = HAlloc(BIF_P, 3);
+
 	break;
 
-    case am_trap_exit: {
-	hp = HAlloc(BIF_P, 3);
-        if (rp->flags & F_TRAP_EXIT)
-	    res = am_true;
-	else
-	    res = am_false;
+    case ERTS_PI_IX_TRAP_EXIT:
+        res = (rp->flags & F_TRAP_EXIT) ? am_true : am_false;
 	break;
-    }
 
-    case am_error_handler:
-	hp = HAlloc(BIF_P, 3);
-	res = erts_proc_get_error_handler(BIF_P);
+    case ERTS_PI_IX_ERROR_HANDLER:
+	res = erts_proc_get_error_handler(rp);
 	break;
 
-    case am_heap_size: {
-	Uint hsz = 3;
+    case ERTS_PI_IX_HEAP_SIZE: {
+	Uint hsz = 0;
 	(void) erts_bld_uint(NULL, &hsz, HEAP_SIZE(rp));
-	hp = HAlloc(BIF_P, hsz);
+        hp = erts_produce_heap(hfact, hsz, reserve_size);
 	res = erts_bld_uint(&hp, NULL, HEAP_SIZE(rp));
 	break;
     }
 
-    case am_fullsweep_after: {
-	Uint hsz = 3;
+    case ERTS_PI_IX_FULLSWEEP_AFTER: {
+	Uint hsz = 0;
 	(void) erts_bld_uint(NULL, &hsz, MAX_GEN_GCS(rp));
-	hp = HAlloc(BIF_P, hsz);
+        hp = erts_produce_heap(hfact, hsz, reserve_size);
 	res = erts_bld_uint(&hp, NULL, MAX_GEN_GCS(rp));
 	break;
     }
 
-    case am_min_heap_size: {
-	Uint hsz = 3;
+    case ERTS_PI_IX_MIN_HEAP_SIZE: {
+	Uint hsz = 0;
 	(void) erts_bld_uint(NULL, &hsz, MIN_HEAP_SIZE(rp));
-	hp = HAlloc(BIF_P, hsz);
+        hp = erts_produce_heap(hfact, hsz, reserve_size);
 	res = erts_bld_uint(&hp, NULL, MIN_HEAP_SIZE(rp));
 	break;
     }
 
-    case am_min_bin_vheap_size: {
-	Uint hsz = 3;
+    case ERTS_PI_IX_MIN_BIN_VHEAP_SIZE: {
+	Uint hsz = 0;
 	(void) erts_bld_uint(NULL, &hsz, MIN_VHEAP_SIZE(rp));
-	hp = HAlloc(BIF_P, hsz);
+        hp = erts_produce_heap(hfact, hsz, reserve_size);
 	res = erts_bld_uint(&hp, NULL, MIN_VHEAP_SIZE(rp));
 	break;
     }
 
-    case am_max_heap_size: {
-	Uint hsz = 3;
+    case ERTS_PI_IX_MAX_HEAP_SIZE: {
+	Uint hsz = 0;
 	(void) erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp),
                                       MAX_HEAP_SIZE_FLAGS_GET(rp),
                                       NULL, &hsz);
-	hp = HAlloc(BIF_P, hsz);
+        hp = erts_produce_heap(hfact, hsz, reserve_size);
 	res = erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp),
                                      MAX_HEAP_SIZE_FLAGS_GET(rp),
                                      &hp, NULL);
 	break;
     }
 
-    case am_total_heap_size: {
+    case ERTS_PI_IX_TOTAL_HEAP_SIZE: {
 	Uint total_heap_size;
-	Uint hsz = 3;
+	Uint hsz = 0;
 
 	total_heap_size = rp->heap_sz;
 	if (rp->old_hend && rp->old_heap)
@@ -1626,58 +1757,52 @@ process_info_aux(Process *BIF_P,
 	total_heap_size += rp->mbuf_sz;
 
         if (rp->flags & F_ON_HEAP_MSGQ) {
-            ERTS_FOREACH_SIG_PRIVQS(
-                rp, mp,
-                {
-                    if (ERTS_SIG_IS_MSG(mp) && mp->data.attached)
-                        total_heap_size += erts_msg_attached_data_size(mp);
-                });
-
-            if (rp->sig_qs.len > CONTEXT_REDS*4)
-                *reds += CONTEXT_REDS*4;
-            else
-                *reds += rp->sig_qs.len / 4;
+            ErtsMessage *mp;
+            ASSERT(flags & ERTS_PI_FLAG_NEED_MSGQ_LEN);
+            for (mp = rp->sig_qs.first; mp; mp = mp->next) {
+                ASSERT(ERTS_SIG_IS_MSG(mp));
+                if (mp->data.attached)
+                    total_heap_size += erts_msg_attached_data_size(mp);
+            }
+            *reds += (Uint) rp->sig_qs.len / 4;
         }
 
 	(void) erts_bld_uint(NULL, &hsz, total_heap_size);
-	hp = HAlloc(BIF_P, hsz);
+        hp = erts_produce_heap(hfact, hsz, reserve_size);
 	res = erts_bld_uint(&hp, NULL, total_heap_size);
 	break;
     }
 
-    case am_stack_size: {
+    case ERTS_PI_IX_STACK_SIZE: {
 	Uint stack_size = STACK_START(rp) - rp->stop;
-	Uint hsz = 3;
+	Uint hsz = 0;
 	(void) erts_bld_uint(NULL, &hsz, stack_size);
-	hp = HAlloc(BIF_P, hsz);
+        hp = erts_produce_heap(hfact, hsz, reserve_size);
 	res = erts_bld_uint(&hp, NULL, stack_size);
 	break;
     }
 
-    case am_memory: { /* Memory consumed in bytes */
-	Uint hsz = 3;
+    case ERTS_PI_IX_MEMORY: { /* Memory consumed in bytes */
+	Uint hsz = 0;
 	Uint size = erts_process_memory(rp, 0);
 	(void) erts_bld_uint(NULL, &hsz, size);
-	hp = HAlloc(BIF_P, hsz);
+        hp = erts_produce_heap(hfact, hsz, reserve_size);
 	res = erts_bld_uint(&hp, NULL, size);
 
-        if (rp->sig_qs.len > CONTEXT_REDS*4)
-            *reds += CONTEXT_REDS*4;
-        else
-            *reds += rp->sig_qs.len / 4;
+        ASSERT(flags & ERTS_PI_FLAG_NEED_MSGQ_LEN);
+        *reds += (Uint) rp->sig_qs.len / 4;
 
 	break;
     }
 
-    case am_garbage_collection: {
+    case ERTS_PI_IX_GARBAGE_COLLECTION: {
         DECL_AM(minor_gcs);
         Eterm t;
         Uint map_sz = 0;
 
         erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp), MAX_HEAP_SIZE_FLAGS_GET(rp), NULL, &map_sz);
 
-	hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2 + 3+2 + 3+2 + map_sz + 3);
-        /* last "3" is for outside tuple */
+        hp = erts_produce_heap(hfact, 3+2 + 3+2 + 3+2 + 3+2 + 3+2 + map_sz, reserve_size);
 
 	t = TUPLE2(hp, AM_minor_gcs, make_small(GEN_GCS(rp))); hp += 3;
 	res = CONS(hp, t, NIL); hp += 2;
@@ -1696,93 +1821,76 @@ process_info_aux(Process *BIF_P,
 	break;
     }
 
-    case am_garbage_collection_info: {
+    case ERTS_PI_IX_GARBAGE_COLLECTION_INFO: {
         Uint sz = 0, actual_sz = 0;
 
-        if (rp == BIF_P) {
-            sz += ERTS_PROCESS_GC_INFO_MAX_SIZE;
-        } else {
-            erts_process_gc_info(rp, &sz, NULL, 0, 0);
-            sz += 3;
-        }
+        erts_process_gc_info(rp, &sz, NULL, 0, 0);
 
-        hp = HAlloc(BIF_P, sz);
+        hp = erts_produce_heap(hfact, sz, reserve_size);
         res = erts_process_gc_info(rp, &actual_sz, &hp, 0, 0);
 
-        /* We may have some extra space, fill with 0 tuples */
-        if (actual_sz <= sz - 3) {
-            for (; actual_sz < sz - 3; hp++, actual_sz++)
-                hp[0] = make_arityval(0);
-        } else {
-            for (; actual_sz < sz; hp++, actual_sz++)
-                hp[0] = make_arityval(0);
-            hp = HAlloc(BIF_P, 3);
-        }
-
         break;
     }
 
-    case am_group_leader: {
+    case ERTS_PI_IX_GROUP_LEADER: {
 	int sz = NC_HEAP_SIZE(rp->group_leader);
-	hp = HAlloc(BIF_P, 3 + sz);
-	res = STORE_NC(&hp, &MSO(BIF_P), rp->group_leader);
+        hp = erts_produce_heap(hfact, sz, reserve_size);
+	res = STORE_NC(&hp, hfact->off_heap, rp->group_leader);
 	break;
     }
 
-    case am_reductions: {
-	Uint reds = rp->reds + erts_current_reductions(BIF_P, rp);
-	Uint hsz = 3;
+    case ERTS_PI_IX_REDUCTIONS: {
+	Uint reds = rp->reds + erts_current_reductions(c_p, rp);
+	Uint hsz = 0;
 	(void) erts_bld_uint(NULL, &hsz, reds);
-	hp = HAlloc(BIF_P, hsz);
+        hp = erts_produce_heap(hfact, hsz, reserve_size);
 	res = erts_bld_uint(&hp, NULL, reds);
 	break;
     }
 
-    case am_priority: {
+    case ERTS_PI_IX_PRIORITY: {
         erts_aint32_t state = erts_atomic32_read_nob(&rp->state);
         if (ERTS_PSFLG_EXITING & state)
             return am_undefined;
-	hp = HAlloc(BIF_P, 3);
 	res = erts_get_process_priority(state);
 	break;
     }
 
-    case am_trace:
-	hp = HAlloc(BIF_P, 3);
+    case ERTS_PI_IX_TRACE:
 	res = make_small(ERTS_TRACE_FLAGS(rp) & TRACEE_FLAGS);
 	break;
 
-    case am_binary: {
-	Uint sz = 3;
+    case ERTS_PI_IX_BINARY: {
+	Uint sz = 0;
 	(void) bld_bin_list(NULL, &sz, &MSO(rp));
-	hp = HAlloc(BIF_P, sz);
+        hp = erts_produce_heap(hfact, sz, reserve_size);
 	res = bld_bin_list(&hp, NULL, &MSO(rp));
 	break;
     }
 
-    case am_sequential_trace_token:
-	res = copy_object(rp->seq_trace_token, BIF_P);
-	hp = HAlloc(BIF_P, 3);
+    case ERTS_PI_IX_SEQUENTIAL_TRACE_TOKEN: {
+        Uint sz = size_object(rp->seq_trace_token);
+        hp = erts_produce_heap(hfact, sz, reserve_size);
+        res = copy_struct(rp->seq_trace_token, sz, &hp, hfact->off_heap);
 	break;
+    }
 
-    case am_catchlevel:
-	hp = HAlloc(BIF_P, 3);
-	res = make_small(catchlevel(BIF_P));
+    case ERTS_PI_IX_CATCHLEVEL:
+	res = make_small(catchlevel(rp));
 	break;
 
-    case am_backtrace: {
+    case ERTS_PI_IX_BACKTRACE: {
 	erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0);
 	erts_stack_dump(ERTS_PRINT_DSBUF, (void *) dsbufp, rp);
-	res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len);
+	res = erts_heap_factory_new_binary(hfact, (byte *) dsbufp->str,
+                                           dsbufp->str_len, reserve_size);
 	erts_destroy_tmp_dsbuf(dsbufp);
-	hp = HAlloc(BIF_P, 3);
 	break;
     }
 
-    case am_last_calls: {
+    case ERTS_PI_IX_LAST_CALLS: {
 	struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(rp);
 	if (!scb) {
-	    hp = HAlloc(BIF_P, 3);
 	    res = am_false;
 	} else {
 	    /*
@@ -1790,23 +1898,34 @@ process_info_aux(Process *BIF_P,
 	     * Might be less than that, if there are sends, receives or timeouts,
 	     * so we must do a HRelease() to avoid creating holes.
 	     */
-	    Uint needed = scb->n*(2+4) + 3;
-	    Eterm* limit;
+	    Sint needed = scb->n*(2+4);
 	    Eterm term, list;
 	    int i, j;
+            Export *exp;
+
+            reserve_size += needed;
 
-	    hp = HAlloc(BIF_P, needed);
-	    limit = hp + needed;
 	    list = NIL;
 	    for (i = 0; i < scb->n; i++) {
+                Uint sz;
 		j = scb->cur - i - 1;
 		if (j < 0)
 		    j += scb->len;
-		if (scb->ct[j] == &exp_send)
+
+                sz = 2;
+                exp = scb->ct[j];
+                if (exp != &exp_send && exp != &exp_receive && exp != &exp_timeout)
+                    sz += 4;
+
+                needed -= sz;
+                ERTS_PI_UNRESERVE(reserve_size, sz);
+                hp = erts_produce_heap(hfact, sz, reserve_size);
+
+		if (exp == &exp_send)
 		    term = am_send;
-		else if (scb->ct[j] == &exp_receive)
+		else if (exp == &exp_receive)
 		    term = am_receive;
-		else if (scb->ct[j] == &exp_timeout)
+		else if (exp == &exp_timeout)
 		    term = am_timeout;
 		else {
 		    term = TUPLE3(hp,
@@ -1816,18 +1935,18 @@ process_info_aux(Process *BIF_P,
 		    hp += 4;
 		}
 		list = CONS(hp, term, list);
-		hp += 2;
 	    }
+
+            ASSERT(needed >= 0);
+            if (needed > 0)
+                reserve_size -= needed;
+
 	    res = list;
-	    res = TUPLE2(hp, item, res);
-	    hp += 3;
-	    HRelease(BIF_P,limit,hp);
-	    return res;
 	}
 	break;
     }
 
-    case am_message_queue_data:
+    case ERTS_PI_IX_MESSAGE_QUEUE_DATA:
 	switch (rp->flags & (F_OFF_HEAP_MSGQ|F_ON_HEAP_MSGQ)) {
 	case F_OFF_HEAP_MSGQ:
 	    res = am_off_heap;
@@ -1840,16 +1959,15 @@ process_info_aux(Process *BIF_P,
 	    ERTS_INTERNAL_ERROR("Inconsistent message queue management state");
 	    break;
 	}
-	hp = HAlloc(BIF_P, 3);
 	break;
 
-    case am_magic_ref: {
-	Uint sz = 3;
+    case ERTS_PI_IX_MAGIC_REF: {
+	Uint sz = 0;
 	(void) bld_magic_ref_bin_list(NULL, &sz, &MSO(rp));
-	hp = HAlloc(BIF_P, sz);
+        hp = erts_produce_heap(hfact, sz, 0);
 	res = bld_magic_ref_bin_list(&hp, NULL, &MSO(rp));
 
-        *reds += 10;
+        *reds += (Uint) 10;
 	break;
     }
 
@@ -1858,12 +1976,17 @@ process_info_aux(Process *BIF_P,
 
     }
 
-    return TUPLE2(hp, item, res);
+    ERTS_PI_UNRESERVE(reserve_size, 3);
+    *reserve_sizep = reserve_size;
+    hp = erts_produce_heap(hfact, 3, reserve_size);
+
+    return TUPLE2(hp, pi_ix2arg(item_ix), res);
 }
 #undef MI_INC
 
 static Eterm
-current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info)
+current_function(Process *c_p, ErtsHeapFactory *hfact, Process* rp,
+                 int full_info, Uint reserve_size, int flags)
 {
     Eterm* hp;
     Eterm res;
@@ -1880,7 +2003,7 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info)
 	}
     }
 
-    if (BIF_P == rp) {
+    if (c_p == rp && !(flags & ERTS_PI_FLAG_REQUEST_FOR_OTHER)) {
 	FunctionInfo fi2;
 
 	/*
@@ -1900,24 +2023,22 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info)
      * Return the result.
      */
     if (rp->current == NULL) {
-	hp = HAlloc(BIF_P, 3);
 	res = am_undefined;
     } else if (full_info) {
-	hp = HAlloc(BIF_P, 3+fi.needed);
-	hp = erts_build_mfa_item(&fi, hp, am_true, &res);
+        hp = erts_produce_heap(hfact, fi.needed, reserve_size);
+	erts_build_mfa_item(&fi, hp, am_true, &res);
     } else {
-	hp = HAlloc(BIF_P, 3+4);
+        hp = erts_produce_heap(hfact, 4, reserve_size);
 	res = TUPLE3(hp, rp->current->module,
 		     rp->current->function,
                      make_small(rp->current->arity));
-	hp += 4;
     }
-    *hpp = hp;
     return res;
 }
 
 static Eterm
-current_stacktrace(Process* p, Process* rp, Eterm** hpp)
+current_stacktrace(ErtsHeapFactory *hfact, Process* rp,
+                   Uint reserve_size)
 {
     Uint sz;
     struct StackTrace* s;
@@ -1926,7 +2047,7 @@ current_stacktrace(Process* p, Process* rp, Eterm** hpp)
     FunctionInfo* stkp;
     Uint heap_size;
     int i;
-    Eterm* hp = *hpp;
+    Eterm* hp;
     Eterm mfa;
     Eterm res = NIL;
 
@@ -1956,17 +2077,23 @@ current_stacktrace(Process* p, Process* rp, Eterm** hpp)
 	}
     }
 
-    hp = HAlloc(p, heap_size);
+    reserve_size += heap_size;
+
+    /*
+     * We intentionally produce heap in small chunks
+     * (for more info see process_info_aux()).
+     */
     while (stkp > stk) {
 	stkp--;
+        sz = stkp->needed + 2;
+        ERTS_PI_UNRESERVE(reserve_size, sz);
+        hp = erts_produce_heap(hfact, sz, reserve_size);
 	hp = erts_build_mfa_item(stkp, hp, am_true, &mfa);
 	res = CONS(hp, mfa, res);
-	hp += 2;
     }
 
     erts_free(ERTS_ALC_T_TMP, stk);
     erts_free(ERTS_ALC_T_TMP, s);
-    *hpp = hp;
     return res;
 }
 
@@ -3838,7 +3965,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    Eterm res = NIL;
 	    Uint *hp = HAlloc(BIF_P, 2*ERTS_PI_ARGS);
 	    for (i = ERTS_PI_ARGS-1; i >= 0; i--) {
-		res = CONS(hp, pi_args[i], res);
+		res = CONS(hp, pi_args[i].name, res);
 		hp += 2;
 	    }
 	    BIF_RET(res);
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 5da8e3eda5..ea87cd7f50 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -61,7 +61,7 @@
 #  define ERTS_GC_ASSERT(B) ((void) 1)
 #endif
 
-#if defined(DEBUG) && 0
+#if defined(DEBUG) && 1
 #  define HARDDEBUG 1
 #endif
 
@@ -222,6 +222,24 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq,
                                  ErtsGCInfoReq,
                                  5,
                                  ERTS_ALC_T_GC_INFO_REQ)
+
+static ERTS_INLINE void
+ensure_sigq_roots_available(Process *p)
+{
+    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p));
+    switch (p->flags & (F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG)) {
+    case F_OFF_HEAP_MSGQ_CHNG:
+    case 0:
+        erts_proc_lock(p, ERTS_PROC_LOCK_MSGQ);
+        erts_proc_sig_fetch(p);
+        erts_proc_unlock(p, ERTS_PROC_LOCK_MSGQ);
+        break;
+    default:
+        break;
+    }
+}
+
+
 /*
  * Initialize GC global data.
  */
@@ -425,6 +443,8 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end,
 	live_hf_end = ERTS_INVALID_HFRAG_PTR;
     }
 
+    ensure_sigq_roots_available(p);
+
     if (is_non_value(result)) {
 	if (p->freason == TRAP) {
 #ifdef HIPE
@@ -642,7 +662,7 @@ check_for_possibly_long_gc(Process *p, Uint ygen_usage)
     sz = ygen_usage;
     sz += p->hend - p->stop;
     if (p->flags & F_ON_HEAP_MSGQ)
-        sz += p->sig_qs.len;
+        sz += erts_proc_sig_privqs_len(p);
     if (major)
 	sz += p->old_htop - p->old_heap;
 
@@ -868,8 +888,11 @@ do_major_collection:
 int
 erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fcalls)
 {
-    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls, 0);
-    int reds_left = ERTS_REDS_LEFT(p, fcalls);
+    int reds; 
+    int reds_left;
+    ensure_sigq_roots_available(p);
+    reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls, 0);
+    reds_left = ERTS_REDS_LEFT(p, fcalls);
     if (reds > reds_left)
 	reds = reds_left;
     ASSERT(CONTEXT_REDS - (reds_left - reds) >= erts_proc_sched_data(p)->virtual_reds);
@@ -879,7 +902,9 @@ erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fca
 void
 erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
 {
-    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls, 0);
+    int reds;
+    ensure_sigq_roots_available(p);
+    reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls, 0);
     BUMP_REDS(p, reds);
     ASSERT(CONTEXT_REDS - ERTS_BIF_REDS_LEFT(p)
 	   >= erts_proc_sched_data(p)->virtual_reds);
@@ -1105,6 +1130,8 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
      * First an ordinary major collection...
      */
 
+    ensure_sigq_roots_available(p);
+
     p->flags |= F_NEED_FULLSWEEP;
 
     if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p)))
@@ -2541,14 +2568,17 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
 	break;
     case F_OFF_HEAP_MSGQ_CHNG:
     case 0: {
+        Sint len;
 	/*
 	 * We do not have off heap message queue enabled, i.e. we
-	 * need to add message queue to rootset...
+	 * need to add signal queues to rootset...
 	 */
 
+        len = erts_proc_sig_privqs_len(p);
+
 	/* Ensure large enough rootset... */
-	if (n + p->sig_qs.len > rootset->size) {
-	    Uint new_size = n + p->sig_qs.len;
+	if (n + len > rootset->size) {
+	    Uint new_size = n + len;
 	    ERTS_GC_ASSERT(roots == rootset->def);
 	    roots = erts_alloc(ERTS_ALC_T_ROOTSET,
 			       new_size*sizeof(Roots));
@@ -3448,7 +3478,7 @@ erts_max_heap_size_map(Sint max_heap_size, Uint max_heap_flags,
                        Eterm **hpp, Uint *sz)
 {
     if (!hpp) {
-        *sz += (2*3 + 1 + MAP_HEADER_FLATMAP_SZ);
+        *sz += ERTS_MAX_HEAP_SIZE_MAP_SZ;
         return THE_NON_VALUE;
     } else {
         Eterm *hp = *hpp;
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 63d03cdf8b..b9b1ed728c 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -154,6 +154,8 @@ typedef struct {
   Uint64 garbage_cols;
 } ErtsGCInfo;
 
+#define ERTS_MAX_HEAP_SIZE_MAP_SZ (2*3 + 1 + MAP_HEADER_FLATMAP_SZ)
+
 #define ERTS_PROCESS_GC_INFO_MAX_TERMS (11)  /* number of elements in process_gc_info*/
 #define ERTS_PROCESS_GC_INFO_MAX_SIZE                                   \
     (ERTS_PROCESS_GC_INFO_MAX_TERMS * (2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE))
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 98feb95d99..541d2e8ad1 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -309,25 +309,6 @@ erts_queue_dist_message(Process *rcvr,
     }
     else {
 
-#ifdef USE_VM_PROBES
-        if (DTRACE_ENABLED(message_queued)) {
-            DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
-
-            dtrace_proc_str(rcvr, receiver_name);
-            if (have_seqtrace(token)) {
-                tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
-                tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
-                tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
-            }
-            /*
-             * TODO: We don't know the real size of the external message here.
-             *       -1 will appear to a D script as 4294967295.
-             */
-            DTRACE6(message_queued, receiver_name, -1, rcvr->sig_qs.len + 1,
-                    tok_label, tok_lastcnt, tok_serial);
-        }
-#endif
-
 	LINK_MESSAGE(rcvr, mp, &mp->next, 1);
 
 	if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
@@ -346,7 +327,6 @@ queue_messages(Process* receiver,
                ErtsMessage** last,
                Uint len)
 {
-    Sint res;
     int locked_msgq = 0;
     erts_aint32_t state;
 
@@ -394,31 +374,14 @@ queue_messages(Process* receiver,
 	return 0;
     }
 
-    res = receiver->sig_qs.len;
-    if (receiver_locks & ERTS_PROC_LOCK_MAIN) {
-	/*
-	 * We move 'in queue' to 'private queue' and place
-	 * message at the end of 'private queue' in order
-	 * to ensure that the 'in queue' doesn't contain
-	 * references into the heap. By ensuring this,
-	 * we don't need to include the 'in queue' in
-	 * the root set when garbage collecting.
-	 */
-	res += receiver->sig_inq.len;
-        erts_proc_sig_fetch(receiver);
-        LINK_MESSAGE_PRIVQ(receiver, first, last, len);
-    }
-    else
-    {
-	LINK_MESSAGE(receiver, first, last, len);
-    }
+    LINK_MESSAGE(receiver, first, last, len);
 
     if (locked_msgq) {
 	erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
     }
 
     erts_proc_notify_new_message(receiver, receiver_locks);
-    return res;
+    return 0;
 }
 
 static Sint
@@ -869,7 +832,7 @@ erts_move_messages_off_heap(Process *c_p)
      * it...
      */
 
-    reds += c_p->sig_qs.len / 10;
+    reds += erts_proc_sig_privqs_len(c_p) / 10;
 
     ASSERT(erts_atomic32_read_nob(&c_p->state)
 	   & ERTS_PSFLG_OFF_HEAP_MSGQ);
@@ -1360,32 +1323,18 @@ void erts_factory_dummy_init(ErtsHeapFactory* factory)
     factory->mode = FACTORY_CLOSED;
 }
 
-static void reserve_heap(ErtsHeapFactory*, Uint need, Uint xtra);
-
-Eterm* erts_produce_heap(ErtsHeapFactory* factory, Uint need, Uint xtra)
-{
-    Eterm* res;
-
-    ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED);
-    if (factory->hp + need > factory->hp_end) {
-	reserve_heap(factory, need, xtra);
-    }
-    res = factory->hp;
-    factory->hp += need;
-    return res;
-}
-
 Eterm* erts_reserve_heap(ErtsHeapFactory* factory, Uint need)
 {
     ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED);
     if (factory->hp + need > factory->hp_end) {
-	reserve_heap(factory, need, 200);
+	erts_reserve_heap__(factory, need, 200);
     }
     return factory->hp;
 }
 
-static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra)
+void erts_reserve_heap__(ErtsHeapFactory* factory, Uint need, Uint xtra)
 {
+    /* internal... */
     ErlHeapFragment* bp;
 
     switch (factory->mode) {
@@ -1395,7 +1344,9 @@ static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra)
 	factory->hp_end = factory->hp + need;
 	return;
 
-    case FACTORY_MESSAGE:
+    case FACTORY_MESSAGE: {
+        int replace_oh;
+        int replace_msg_hfrag;
 	if (!factory->heap_frags) {
 	    ASSERT(factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG);
 	    bp = &factory->message->hfrag;
@@ -1407,25 +1358,45 @@ static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra)
 	    bp = factory->heap_frags;
 	}
 
+        replace_oh = 0;
+        replace_msg_hfrag = 0;
+
         if (bp) {
-	    ASSERT(factory->hp > bp->mem);
+	    ASSERT(factory->hp >= bp->mem);
 	    ASSERT(factory->hp <= factory->hp_end);
 	    ASSERT(factory->hp_end == bp->mem + bp->alloc_size);
 
 	    bp->used_size = factory->hp - bp->mem;
+            if (!bp->used_size && factory->heap_frags) {
+                factory->heap_frags = bp->next;
+                bp->next = NULL;
+                ASSERT(!bp->off_heap.first);
+                if (factory->off_heap == &bp->off_heap)
+                    replace_oh = !0;
+                if (factory->message && factory->message->data.heap_frag == bp)
+                    replace_msg_hfrag = !0;
+                free_message_buffer(bp);
+            }
         }
 	bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(factory->alloc_type,
 						ERTS_HEAP_FRAG_SIZE(need+xtra));
 	bp->next = factory->heap_frags;
 	factory->heap_frags = bp;
 	bp->alloc_size = need + xtra;
-	bp->used_size = need;
+	bp->used_size = need + xtra;
 	bp->off_heap.first = NULL;
 	bp->off_heap.overhead = 0;
-
+        if (replace_oh) {
+            factory->off_heap = &bp->off_heap;
+            factory->off_heap_saved.first = factory->off_heap->first;
+            factory->off_heap_saved.overhead = factory->off_heap->overhead;
+        }
+        if (replace_msg_hfrag)
+            factory->message->data.heap_frag = bp;
 	factory->hp     = bp->mem;
 	factory->hp_end = bp->mem + bp->alloc_size;
 	return;
+    }
 
     case FACTORY_STATIC:
     case FACTORY_CLOSED:
@@ -1508,9 +1479,11 @@ void erts_factory_trim_and_close(ErtsHeapFactory* factory,
         if (bp->next == NULL) {
             Uint used_sz = factory->hp - bp->mem;
             ASSERT(used_sz <= bp->alloc_size);
-	    if (used_sz > 0)
-		bp = erts_resize_message_buffer(bp, used_sz,
-						brefs, brefs_size);
+	    if (used_sz > 0) {
+                if (used_sz != bp->alloc_size)
+                    bp = erts_resize_message_buffer(bp, used_sz,
+                                                    brefs, brefs_size);
+            }
 	    else {
 		free_message_buffer(bp);
 		bp = NULL;
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index f56a252aef..ee87297ba4 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -89,12 +89,33 @@ void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap
 void erts_factory_tmp_init(ErtsHeapFactory*, Eterm* hp, Uint size, Uint32 atype);
 void erts_factory_dummy_init(ErtsHeapFactory*);
 
-Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra);
+ERTS_GLB_INLINE Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra);
+
 Eterm* erts_reserve_heap(ErtsHeapFactory*, Uint need);
 void erts_factory_close(ErtsHeapFactory*);
 void erts_factory_trim_and_close(ErtsHeapFactory*,Eterm *brefs, Uint brefs_size);
 void erts_factory_undo(ErtsHeapFactory*);
 
+void erts_reserve_heap__(ErtsHeapFactory*, Uint need, Uint xtra); /* internal */
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE Eterm *
+erts_produce_heap(ErtsHeapFactory* factory, Uint need, Uint xtra)
+{
+    Eterm* res;
+
+    ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED);
+    if (factory->hp + need > factory->hp_end) {
+	erts_reserve_heap__(factory, need, xtra);
+    }
+    res = factory->hp;
+    factory->hp += need;
+    return res;
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
 #ifdef CHECK_FOR_HOLES
 # define ERTS_FACTORY_HOLE_CHECK(f) do {    \
         /*if ((f)->p) erts_check_for_holes((f)->p);*/ \
@@ -215,6 +236,48 @@ typedef struct {
 #define ERL_MESSAGE_BUF_SZ 500
 
 typedef struct {
+    /*
+     * ** The signal queues private to a process. **
+     *
+     * These are:
+     * - an inner queue which only consists of
+     *   message signals
+     * - a middle queue which contains a mixture
+     *   of message and non-message signals
+     *
+     * When the process isn't processing signals in
+     * erts_proc_sig_handle_incoming():
+     * - the message queue corresponds to the inner
+     *   queue. Messages in the middle queue (and
+     *   in the outer queue) are in transit and
+     *   have NOT been received yet!
+     *
+     * When the process is processing signals in
+     * erts_proc_sig_handle_incoming():
+     * - the message queue corresponds to the inner
+     *   queue plus the head of the middle queue up
+     *   to the signal currently being processed.
+     *   Any messages further back in the middle queue
+     *   (and in the outer queue) are still in transit
+     *   and have NOT been received yet!
+     *
+     * In the general case the 'len' field of this
+     * structure does NOT correspond to the message
+     * queue length. When the process is inspected
+     * via process info it does however correspond
+     * to the message queue length, but this is a
+     * special case!
+     *
+     * When no process-info request is in transit to
+     * the process the 'len' field corresponds to
+     * the total amount of messages in inner and
+     * middle queues (which does NOT correspond to
+     * the message queue length). When process-info
+     * requests are in transit to the process, the
+     * usage of the 'len' field changes and is used
+     * as an offset which even might be negative.
+     */
+
     /* inner queue */
     ErtsMessage *first;
     ErtsMessage **last;  /* point to the last next pointer */
@@ -227,7 +290,7 @@ typedef struct {
 
     /* Common for inner and middle queue */
     ErtsMessage **saved_last;	/* saved last pointer */
-    Sint len; /* message queue length (inner+middle) */
+    Sint len; /* NOT message queue length (see above) */
 } ErtsSignalPrivQueues;
 
 typedef struct {
@@ -248,8 +311,7 @@ typedef struct erl_trace_message_queue__ {
 #define ERTS_RECV_MARK_SAVE(P)                                          \
     do {                                                                \
         erts_proc_lock((P), ERTS_PROC_LOCK_MSGQ);                       \
-        if ((P)->sig_inq.first)                                         \
-            erts_proc_sig_fetch((P));                                   \
+        erts_proc_sig_fetch((P));                                       \
         erts_proc_unlock((P), ERTS_PROC_LOCK_MSGQ);                     \
         if ((P)->sig_qs.cont) {                                         \
             (P)->sig_qs.saved_last = (P)->sig_qs.cont_last;             \
@@ -302,23 +364,6 @@ typedef struct erl_trace_message_queue__ {
 
 #endif
 
-
-/* Add message last in private message queue */
-#define LINK_MESSAGE_PRIVQ(p, first_msg, last_msg, num_msgs)            \
-    do {                                                                \
-        ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((p), "before");             \
-        if ((p)->sig_qs.cont || ERTS_MSG_RECV_TRACED((p))) {            \
-            *(p)->sig_qs.cont_last = (first_msg);                       \
-            (p)->sig_qs.cont_last = (last_msg);                         \
-        }                                                               \
-        else {                                                          \
-            *(p)->sig_qs.last = (first_msg);                            \
-            (p)->sig_qs.last = (last_msg);                              \
-        }                                                               \
-        (p)->sig_qs.len += (num_msgs);                                  \
-        ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((p), "after");              \
-    } while (0)
-
 /* Add message last_msg in message queue */
 #define LINK_MESSAGE(p, first_msg, last_msg, num_msgs) \
     do {                                                                \
@@ -333,12 +378,12 @@ typedef struct erl_trace_message_queue__ {
 #define UNLINK_MESSAGE(p,msgp)                                          \
     do {                                                                \
         ErtsMessage *mp__ = (msgp)->next;                               \
-        ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((p), "before");             \
+        ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((p), 0, "before");          \
         *(p)->sig_qs.save = mp__;                                       \
         (p)->sig_qs.len--;                                              \
         if (mp__ == NULL)                                               \
             (p)->sig_qs.last = (p)->sig_qs.save;                        \
-        ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((p), "after");              \
+        ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((p), 0, "after");           \
     } while(0)
 
 /*
diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index 99d20e9242..4bb5f31538 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -49,7 +49,7 @@
  * Note that not all signal are handled using this functionality!
  */
 
-#define ERTS_SIG_Q_OP_MAX 10
+#define ERTS_SIG_Q_OP_MAX 11
 
 #define ERTS_SIG_Q_OP_EXIT                      0
 #define ERTS_SIG_Q_OP_EXIT_LINKED               1
@@ -61,7 +61,8 @@
 #define ERTS_SIG_Q_OP_GROUP_LEADER              7
 #define ERTS_SIG_Q_OP_TRACE_CHANGE_STATE        8
 #define ERTS_SIG_Q_OP_PERSISTENT_MON_MSG        9
-#define ERTS_SIG_Q_OP_IS_ALIVE                  ERTS_SIG_Q_OP_MAX
+#define ERTS_SIG_Q_OP_IS_ALIVE                  10
+#define ERTS_SIG_Q_OP_PROCESS_INFO              ERTS_SIG_Q_OP_MAX
 
 #define ERTS_SIG_Q_TYPE_MAX (ERTS_MON_LNK_TYPE_MAX + 5)
 
@@ -76,46 +77,6 @@
 #define ERTS_SIG_Q_TYPE_ADJUST_TRACE_INFO \
     ERTS_SIG_Q_TYPE_MAX
 
-
-#define ERTS_SIG_Q_OP_BITS      8                      
-#define ERTS_SIG_Q_OP_SHIFT     0
-#define ERTS_SIG_Q_OP_MASK      ((1 << ERTS_SIG_Q_OP_BITS) - 1)
-
-#define ERTS_SIG_Q_TYPE_BITS    8
-#define ERTS_SIG_Q_TYPE_SHIFT   ERTS_SIG_Q_OP_BITS
-#define ERTS_SIG_Q_TYPE_MASK    ((1 << ERTS_SIG_Q_TYPE_BITS) - 1)
-
-#define ERTS_SIG_Q_NON_X_BITS__ (_HEADER_ARITY_OFFS \
-                                 + ERTS_SIG_Q_OP_BITS \
-                                 + ERTS_SIG_Q_TYPE_BITS)
-
-#define ERTS_SIG_Q_XTRA_BITS    (32 - ERTS_SIG_Q_NON_X_BITS__)
-#define ERTS_SIG_Q_XTRA_SHIFT   (ERTS_SIG_Q_OP_BITS \
-                                 + ERTS_SIG_Q_TYPE_BITS)
-#define ERTS_SIG_Q_XTRA_MASK    ((1 << ERTS_SIG_Q_XTRA_BITS) - 1)
-
-#define ERTS_PROC_SIG_OP(Tag) \
-    ((int) (_unchecked_thing_arityval((Tag)) \
-            >> ERTS_SIG_Q_OP_SHIFT) & ERTS_SIG_Q_OP_MASK)
-
-#define ERTS_PROC_SIG_TYPE(Tag) \
-    ((Uint16) (_unchecked_thing_arityval((Tag)) \
-               >> ERTS_SIG_Q_TYPE_SHIFT) & ERTS_SIG_Q_TYPE_MASK)
-
-#define ERTS_PROC_SIG_XTRA(Tag) \
-    ((Uint32) (_unchecked_thing_arityval((Tag)) \
-               >> ERTS_SIG_Q_XTRA_SHIFT) & ERTS_SIG_Q_XTRA_MASK)
-
-#define ERTS_PROC_SIG_MAKE_TAG(Op, Type, Xtra)                  \
-    (ASSERT(0 <= (Xtra) && (Xtra) <= ERTS_SIG_Q_XTRA_MASK),     \
-     _make_header((((Type) & ERTS_SIG_Q_TYPE_MASK)              \
-                   << ERTS_SIG_Q_TYPE_SHIFT)                    \
-                  | (((Op) & ERTS_SIG_Q_OP_MASK)                \
-                     << ERTS_SIG_Q_OP_SHIFT)                    \
-                  | (((Xtra) & ERTS_SIG_Q_XTRA_MASK)            \
-                     << ERTS_SIG_Q_XTRA_SHIFT),                 \
-                  _TAG_HEADER_EXTERNAL_PID))
-
 Process *ERTS_WRITE_UNLIKELY(erts_dirty_process_signal_handler);
 Process *ERTS_WRITE_UNLIKELY(erts_dirty_process_signal_handler_high);
 Process *ERTS_WRITE_UNLIKELY(erts_dirty_process_signal_handler_max);
@@ -123,7 +84,8 @@ Process *ERTS_WRITE_UNLIKELY(erts_dirty_process_signal_handler_max);
 void
 erts_proc_sig_queue_init(void)
 {
-    ERTS_CT_ASSERT(ERTS_SIG_Q_OP_MASK >= ERTS_SIG_Q_OP_MAX);
+    ERTS_CT_ASSERT(ERTS_SIG_Q_OP_MASK > ERTS_SIG_Q_OP_MAX);
+    ERTS_CT_ASSERT(ERTS_SIG_Q_OP_MSGQ_LEN_OFFS_MARK > ERTS_SIG_Q_OP_MAX);
     ERTS_CT_ASSERT(ERTS_SIG_Q_TYPE_MASK >= ERTS_SIG_Q_TYPE_MAX);
 }
 
@@ -191,6 +153,29 @@ typedef struct {
     Eterm requester;
 } ErtsIsAliveRequest;
 
+typedef struct {
+    ErtsSignalCommon common;
+    Sint refc;
+    Sint delayed_len;
+    Sint len_offset;
+} ErtsProcSigMsgQLenOffsetMarker;
+
+typedef struct {
+    ErtsSignalCommon common;
+    ErtsProcSigMsgQLenOffsetMarker marker;
+    Sint msgq_len_offset;
+    Eterm requester;
+    Eterm ref;
+    ErtsORefThing oref_thing;
+    Uint reserve_size;
+    Uint len;
+    int flags;
+    int item_ix[1]; /* of len size in reality... */
+} ErtsProcessInfoSig;
+
+#define ERTS_PROC_SIG_PI_MSGQ_LEN_IGNORE        ((Sint) -1)
+#define ERTS_PROC_SIG_PI_MSGQ_LEN_SYNC          ((Sint) -2)
+
 static int handle_msg_tracing(Process *c_p,
                               ErtsSigRecvTracing *tracing,
                               ErtsMessage ***next_nm_sig);
@@ -523,6 +508,9 @@ ensure_dirty_proc_handled(Eterm pid,
     }
 }
 
+static void
+check_push_msgq_len_offs_marker(Process *rp, ErtsSignal *sig);
+
 static int
 proc_queue_signal(Process *c_p, Eterm pid, ErtsSignal *sig, int op)
 {
@@ -559,6 +547,8 @@ proc_queue_signal(Process *c_p, Eterm pid, ErtsSignal *sig, int op)
         res = 0;
     else {
         state = enqueue_signals(rp, first, last, last_next, state);
+        if (ERTS_UNLIKELY(op == ERTS_SIG_Q_OP_PROCESS_INFO))
+            check_push_msgq_len_offs_marker(rp, sig);
         res = !0;
     }
 
@@ -620,31 +610,14 @@ maybe_elevate_sig_handling_prio(Process *c_p, Eterm other)
 }
 
 void
-erts_proc_sig_fetch(Process *proc)
+erts_proc_sig_fetch__(Process *proc)
 {
 #ifdef ERTS_PROC_SIG_HARD_DEBUG
     ErtsSignalPrivQueues sig_qs = proc->sig_qs;
     ErtsSignalInQueue sig_inq = proc->sig_inq;
 #endif
 
-    ERTS_LC_ASSERT(erts_thr_progress_is_blocking()
-                   || ERTS_PROC_IS_EXITING(proc)
-                   || ((erts_proc_lc_my_proc_locks(proc)
-                        & (ERTS_PROC_LOCK_MAIN
-                           | ERTS_PROC_LOCK_MSGQ))
-                       == (ERTS_PROC_LOCK_MAIN
-                           | ERTS_PROC_LOCK_MSGQ)));
-
-    if (!proc->sig_inq.first) {
-        ASSERT(proc->sig_inq.last == &proc->sig_inq.first);
-        ASSERT(proc->sig_inq.len == 0);
-        ASSERT(!proc->sig_inq.nmsigs.next);
-        ASSERT(!proc->sig_inq.nmsigs.last);
-        return;
-    }
-
-    ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE(proc);
-    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(proc);
+    ASSERT(proc->sig_inq.first);
 
     if (!proc->sig_inq.nmsigs.next) {
         ASSERT(!(ERTS_PSFLG_SIG_IN_Q
@@ -720,7 +693,102 @@ erts_proc_sig_fetch(Process *proc)
     proc->sig_inq.last = &proc->sig_inq.first;
     proc->sig_inq.len = 0;
 
-    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(proc);
+}
+
+Sint
+erts_proc_sig_fetch_msgq_len_offs__(Process *proc)
+{
+    ErtsProcSigMsgQLenOffsetMarker *marker
+        = (ErtsProcSigMsgQLenOffsetMarker *) proc->sig_inq.first;
+
+    ASSERT(marker->common.tag == ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK);
+
+    if (marker->common.next) {
+        Sint len;
+
+        proc->flags |= F_DELAYED_PSIGQS_LEN;
+
+        /*
+         * Prevent update of sig_qs.len in fetch. These
+         * updates are done via process-info signal(s)
+         * instead...
+         */
+        len = proc->sig_inq.len;
+        marker->delayed_len += len;
+        marker->len_offset -= len;
+        proc->sig_inq.len = 0;
+
+        /*
+         * Temorarily remove marker during fetch...
+         */
+
+        proc->sig_inq.first = marker->common.next;
+        if (proc->sig_inq.last == &marker->common.next)
+            proc->sig_inq.last = &proc->sig_inq.first;
+        if (proc->sig_inq.nmsigs.next == &marker->common.next)
+            proc->sig_inq.nmsigs.next = &proc->sig_inq.first;
+        if (proc->sig_inq.nmsigs.last == &marker->common.next)
+            proc->sig_inq.nmsigs.last = &proc->sig_inq.first;
+
+        erts_proc_sig_fetch__(proc);
+
+        marker->common.next = NULL;
+        proc->sig_inq.first = (ErtsMessage *) marker;
+        proc->sig_inq.last = &marker->common.next;
+
+    }
+
+    return marker->delayed_len;
+}
+
+static ERTS_INLINE Sint
+proc_sig_privqs_len(Process *c_p, int have_qlock)
+{
+    Sint res = c_p->sig_qs.len;
+
+    ERTS_LC_ASSERT(!have_qlock
+                   ? (ERTS_PROC_LOCK_MAIN
+                      == erts_proc_lc_my_proc_locks(c_p))
+                   : ((ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_MAIN)
+                      == ((ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_MAIN)
+                          & erts_proc_lc_my_proc_locks(c_p))));
+
+    if (c_p->flags & F_DELAYED_PSIGQS_LEN) {
+        ErtsProcSigMsgQLenOffsetMarker *marker;
+
+        if (!have_qlock)
+            erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
+
+        marker = (ErtsProcSigMsgQLenOffsetMarker *) c_p->sig_inq.first;
+        ASSERT(marker);
+        ASSERT(marker->common.tag == ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK);
+
+        res += marker->delayed_len;
+
+        if (!have_qlock)
+            erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
+    }
+
+#ifdef ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN
+    {
+        Sint len = 0;
+        ERTS_FOREACH_SIG_PRIVQS(
+            c_p, mp,
+            {
+                if (ERTS_SIG_IS_MSG(mp))
+                    len++;
+            });
+        ERTS_ASSERT(res == len);
+    }
+#endif
+
+    return res;
+}
+
+Sint
+erts_proc_sig_privqs_len(Process *c_p)
+{
+    return proc_sig_privqs_len(c_p, 0);
 }
 
 void do_seq_trace_output(Eterm to, Eterm token, Eterm msg);
@@ -1352,6 +1420,59 @@ erts_proc_sig_send_is_alive_request(Process *c_p, Eterm to, Eterm ref)
     }
 }
 
+int
+erts_proc_sig_send_process_info_request(Process *c_p,
+                                        Eterm to,
+                                        int *item_ix,
+                                        int len,
+                                        int need_msgq_len,
+                                        int flags,
+                                        Uint reserve_size,
+                                        Eterm ref)
+{
+    Uint size = sizeof(ErtsProcessInfoSig) + (len - 1) * sizeof(int);
+    ErtsProcessInfoSig *pis = erts_alloc(ERTS_ALC_T_SIG_DATA, size);
+    int res;
+
+    ASSERT(c_p);
+    ASSERT(item_ix);
+    ASSERT(len > 0);
+    ASSERT(is_internal_ordinary_ref(ref));
+
+    pis->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_PROCESS_INFO,
+                                             0, 0);
+
+    if (!need_msgq_len)
+        pis->msgq_len_offset = ERTS_PROC_SIG_PI_MSGQ_LEN_IGNORE;
+    else {
+        pis->msgq_len_offset = ERTS_PROC_SIG_PI_MSGQ_LEN_SYNC;
+        pis->marker.common.next = NULL;
+        pis->marker.common.specific.next = NULL;
+        pis->marker.common.tag = ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK;
+        pis->marker.refc = 0;
+        pis->marker.delayed_len = 0;
+        pis->marker.len_offset = 0;
+    }
+    pis->requester = c_p->common.id;
+    sys_memcpy((void *) &pis->oref_thing,
+               (void *) internal_ref_val(ref),
+               sizeof(ErtsORefThing));
+    pis->ref = make_internal_ref((char *) &pis->oref_thing);
+    pis->reserve_size = reserve_size;
+    pis->len = len;
+    pis->flags = flags;
+    sys_memcpy((void *) &pis->item_ix[0],
+               (void *) item_ix,
+               sizeof(int)*len);
+    res = proc_queue_signal(c_p, to, (ErtsSignal *) pis,
+                            ERTS_SIG_Q_OP_PROCESS_INFO);
+    if (res)
+        (void) maybe_elevate_sig_handling_prio(c_p, to);
+    else
+        erts_free(ERTS_ALC_T_SIG_DATA, pis);
+    return res;
+}       
+
 static void
 is_alive_response(Process *c_p, ErtsMessage *mp, int is_alive)
 {
@@ -1552,7 +1673,6 @@ remove_mq_m_sig(Process *c_p, ErtsMessage *sig, ErtsMessage **next_sig, ErtsMess
 {
     /* Removing message... */
     ASSERT(!ERTS_SIG_IS_NON_MSG(sig));
-    ASSERT(c_p->sig_qs.len > 0);
     c_p->sig_qs.len--;
     remove_mq_sig(c_p, sig, next_sig, next_nm_sig);
 }
@@ -1562,7 +1682,6 @@ remove_iq_m_sig(Process *c_p, ErtsMessage *sig, ErtsMessage **next_sig)
 {
     /* Removing message... */
     ASSERT(!ERTS_SIG_IS_NON_MSG(sig));
-    ASSERT(c_p->sig_qs.len > 0);
     c_p->sig_qs.len--;
     remove_iq_sig(c_p, sig, next_sig);
 }
@@ -2138,6 +2257,258 @@ handle_group_leader(Process *c_p, ErtsSigGroupLeader *sgl)
         destroy_sig_group_leader(sgl);
 }
 
+static void
+check_push_msgq_len_offs_marker(Process *rp, ErtsSignal *sig)
+{
+    ErtsProcessInfoSig *pisig = (ErtsProcessInfoSig *) sig;
+
+    ASSERT(ERTS_PROC_SIG_OP(sig->common.tag) == ERTS_SIG_Q_OP_PROCESS_INFO);
+
+    if (pisig->msgq_len_offset == ERTS_PROC_SIG_PI_MSGQ_LEN_SYNC) {
+        ErtsProcSigMsgQLenOffsetMarker *mrkr;
+        Sint len, msgq_len_offset;
+        ErtsMessage *first = rp->sig_inq.first;
+        ASSERT(first);
+        if (((ErtsSignal *) first)->common.tag == ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK)
+            mrkr = (ErtsProcSigMsgQLenOffsetMarker *) first;
+        else {
+            mrkr = &pisig->marker;
+
+            ASSERT(mrkr->common.tag == ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK);
+
+            mrkr->common.next = first;
+            ASSERT(rp->sig_inq.last != &rp->sig_inq.first);
+            if (rp->sig_inq.nmsigs.next == &rp->sig_inq.first)
+                rp->sig_inq.nmsigs.next = &mrkr->common.next;
+            if (rp->sig_inq.nmsigs.last == &rp->sig_inq.first)
+                rp->sig_inq.nmsigs.last = &mrkr->common.next;
+            rp->sig_inq.first = (ErtsMessage *) mrkr;
+        }
+
+        len = rp->sig_inq.len;
+        msgq_len_offset = len - mrkr->len_offset;
+
+        mrkr->len_offset = len;
+        mrkr->refc++;
+
+        pisig->msgq_len_offset = msgq_len_offset;
+
+#ifdef DEBUG
+        /* save pointer to used marker... */
+        pisig->marker.common.specific.attachment = (void *) mrkr;
+#endif
+
+    }
+}
+
+static void
+destroy_process_info_request(Process *c_p, ErtsProcessInfoSig *pisig)
+{
+    int dealloc_pisig = !0;
+
+    if (pisig->msgq_len_offset != ERTS_PROC_SIG_PI_MSGQ_LEN_IGNORE) {
+        Sint refc;
+        int dealloc_marker = 0;
+        ErtsProcSigMsgQLenOffsetMarker *marker;
+#ifdef ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN
+        Sint delayed_len;
+#endif
+
+        ASSERT(pisig->msgq_len_offset >= 0);
+
+        erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
+        marker = (ErtsProcSigMsgQLenOffsetMarker *) c_p->sig_inq.first;
+        ASSERT(marker);
+        ASSERT(marker->refc > 0);
+        ASSERT(pisig->marker.common.specific.attachment == (void *) marker);
+
+        marker->delayed_len -= pisig->msgq_len_offset;
+#ifdef ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN
+        delayed_len = marker->delayed_len;
+#endif
+
+        refc = --marker->refc;
+        if (refc) {
+            if (marker == &pisig->marker) {
+                /* Another signal using our marker... */
+                dealloc_pisig = 0;
+            }
+        }
+        else {
+            /* Marker unused; remove it... */
+            ASSERT(marker->delayed_len + marker->len_offset == 0);
+#ifdef ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN
+            delayed_len += marker->len_offset;
+#endif
+            if (marker != &pisig->marker)
+                dealloc_marker = !0; /* used another signals marker... */
+            c_p->sig_inq.first = marker->common.next;
+            if (c_p->sig_inq.last == &marker->common.next)
+                c_p->sig_inq.last = &c_p->sig_inq.first;
+            if (c_p->sig_inq.nmsigs.next == &marker->common.next)
+                c_p->sig_inq.nmsigs.next = &c_p->sig_inq.first;
+            if (c_p->sig_inq.nmsigs.last == &marker->common.next)
+                c_p->sig_inq.nmsigs.last = &c_p->sig_inq.first;
+        }
+        erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
+
+        if (!refc) {
+            c_p->flags &= ~F_DELAYED_PSIGQS_LEN;
+            /* Adjust msg len of inner+middle queue */
+            ASSERT(marker->len_offset <= 0);
+            c_p->sig_qs.len -= marker->len_offset;
+
+            ASSERT(c_p->sig_qs.len >= 0);
+        }
+
+#ifdef ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN
+        {
+            Sint len = 0;
+            ERTS_FOREACH_SIG_PRIVQS(
+                c_p, mp,
+                {
+                    if (ERTS_SIG_IS_MSG(mp))
+                        len++;
+                });
+            ERTS_ASSERT(c_p->sig_qs.len + delayed_len == len);
+        }
+#endif
+
+
+        if (dealloc_marker) {
+            ErtsProcessInfoSig *pisig2
+                = (ErtsProcessInfoSig *) (((char *) marker)
+                                          - offsetof(ErtsProcessInfoSig,
+                                                     marker));
+            erts_free(ERTS_ALC_T_SIG_DATA, pisig2);
+        }
+    }
+
+    if (dealloc_pisig)
+        erts_free(ERTS_ALC_T_SIG_DATA, pisig);
+}
+
+static int
+handle_process_info(Process *c_p, ErtsMessage *sig,
+                    ErtsMessage ***next_nm_sig, int is_alive)
+{
+    ErtsProcessInfoSig *pisig = (ErtsProcessInfoSig *) sig;
+    Uint reds = 0;
+    Process *rp;
+
+    if (pisig->msgq_len_offset != ERTS_PROC_SIG_PI_MSGQ_LEN_IGNORE) {
+        /*
+         * Request requires message queue data to be updated
+         * before inspection...
+         */
+
+        ASSERT(pisig->msgq_len_offset >= 0);
+        /*
+         * Update sig_qs.len to reflect the length
+         * of the message queue...
+         */
+        c_p->sig_qs.len += pisig->msgq_len_offset;
+
+        if (is_alive) {
+            /*
+             * Move messages part of message queue into inner
+             * signal queue...
+             */
+            if (*next_nm_sig != &c_p->sig_qs.cont) {
+                *c_p->sig_qs.last = c_p->sig_qs.cont;
+                c_p->sig_qs.last = *next_nm_sig;
+
+                c_p->sig_qs.cont = **next_nm_sig;
+                if (c_p->sig_qs.nmsigs.last == *next_nm_sig)
+                    c_p->sig_qs.nmsigs.last = &c_p->sig_qs.cont;
+                *next_nm_sig = &c_p->sig_qs.cont;
+                *c_p->sig_qs.last = NULL;
+            }
+
+            if (!pisig->common.specific.next) {
+                /*
+                 * No more signals in middle queue...
+                 *
+                 * Process-info 'status' needs sig-q
+                 * process flag to be updated in order
+                 * to show accurate result...
+                 */
+                erts_atomic32_read_band_nob(&c_p->state,
+                                            ~ERTS_PSFLG_SIG_Q);
+            }
+
+#ifdef ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN
+            {
+                Sint len;
+                ErtsMessage *mp;
+                for (mp = c_p->sig_qs.first, len = 0; mp; mp = mp->next) {
+                    ERTS_ASSERT(ERTS_SIG_IS_MSG(mp));
+                    len++;
+                }
+                ERTS_ASSERT(c_p->sig_qs.len == len);
+            }
+#endif
+        }
+    }
+    if (is_alive)
+        remove_nm_sig(c_p, sig, next_nm_sig);
+
+    rp = erts_proc_lookup(pisig->requester);
+    ASSERT(c_p != rp);
+    if (rp) {
+        Eterm msg, res, ref, *hp;
+        ErtsProcLocks locks = 0;
+        ErtsHeapFactory hfact;
+        ErtsMessage *mp;
+        Uint reserve_size = 3 + sizeof(pisig->oref_thing)/sizeof(Eterm);
+
+        if (!is_alive) {
+            ErlOffHeap *ohp;
+            mp = erts_alloc_message_heap(rp, &locks, reserve_size, &hp, &ohp);
+            res = am_undefined;
+        }
+        else {
+            ErlHeapFragment *hfrag;
+
+            reserve_size += pisig->reserve_size;
+
+            mp = erts_alloc_message(0, NULL);
+            hfrag = new_message_buffer(reserve_size);
+            mp->data.heap_frag = hfrag;
+            erts_factory_selfcontained_message_init(&hfact, mp, &hfrag->mem[0]);
+
+            res = erts_process_info(c_p, &hfact, c_p, ERTS_PROC_LOCK_MAIN,
+                                    pisig->item_ix, pisig->len,
+                                    pisig->flags, reserve_size, &reds);
+
+            hp = erts_produce_heap(&hfact,
+                                   3 + sizeof(pisig->oref_thing)/sizeof(Eterm),
+                                   0);
+        }
+
+        sys_memcpy((void *) hp, (void *) &pisig->oref_thing,
+                   sizeof(pisig->oref_thing));
+        ref = make_internal_ref(hp);
+        hp += sizeof(pisig->oref_thing)/sizeof(Eterm);
+
+        msg = TUPLE2(hp, ref, res);
+
+        if (is_alive)
+            erts_factory_trim_and_close(&hfact, &msg, 1);
+
+        erts_queue_message(rp, locks, mp, msg, c_p->common.id);
+
+        if (!is_alive && locks)
+            erts_proc_unlock(rp, locks);
+    }
+
+    destroy_process_info_request(c_p, pisig);
+
+    if (reds > INT_MAX/8)
+        reds = INT_MAX/8;
+
+    return ((int) reds)*4 + 8;
+}
 
 /*
  * Called in order to handle incoming signals.
@@ -2153,7 +2524,7 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep,
     ErtsMessage *sig, ***next_nm_sig;
     ErtsSigRecvTracing tracing;
 
-    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p);
+    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p, 0);
     ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
 
     if (local_only)
@@ -2502,6 +2873,12 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep,
             ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
             break;
 
+        case ERTS_SIG_Q_OP_PROCESS_INFO:
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            handle_process_info(c_p, sig, next_nm_sig, !0);
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            break;
+
         case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE: {
             Uint16 type = ERTS_PROC_SIG_TYPE(tag);
 
@@ -2655,7 +3032,7 @@ stop: {
                 c_p->sig_qs.save = c_p->sig_qs.saved_last;
         }
 
-        ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p);
+        ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p, 0);
 
         *redsp = cnt/4 + 1;
 
@@ -2707,8 +3084,8 @@ erts_proc_sig_handle_exit(Process *c_p, int *redsp)
     int cnt, limit;
     ErtsMessage *sig, ***next_nm_sig;
 
-    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p);
-    ERTS_LC_ASSERT(!erts_proc_lc_my_proc_locks(c_p));
+    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p, 0);
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
 
     ASSERT(!(ERTS_PSFLG_SIG_IN_Q & erts_atomic32_read_nob(&c_p->state)));
 
@@ -2815,9 +3192,11 @@ erts_proc_sig_handle_exit(Process *c_p, int *redsp)
         }
 
         case ERTS_SIG_Q_OP_IS_ALIVE:
-            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
             is_alive_response(c_p, sig, 0);
-            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            break;
+
+        case ERTS_SIG_Q_OP_PROCESS_INFO:
+            handle_process_info(c_p, sig, next_nm_sig, 0);
             break;
 
         case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE:
@@ -3003,6 +3382,13 @@ erts_proc_sig_signal_size(ErtsSignal *sig)
         size = sizeof(ErtsSigTraceInfo);
         break;
 
+    case ERTS_SIG_Q_OP_PROCESS_INFO: {
+        ErtsProcessInfoSig *pisig = (ErtsProcessInfoSig *) sig;
+        size = sizeof(ErtsProcessInfoSig);
+        size += (pisig->len - 1) * sizeof(int);
+        break;
+    }
+
     default:
         ERTS_INTERNAL_ERROR("Unknown signal");
         break;
@@ -3058,8 +3444,7 @@ erts_proc_sig_receive_helper(Process *c_p,
             consumed_reds += 4;
             left_reds -= 4;
             erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
-            if (c_p->sig_inq.first)
-                erts_proc_sig_fetch(c_p);
+            erts_proc_sig_fetch(c_p);
             /*
              * Messages may have been moved directly to
              * inner queue...
@@ -3220,6 +3605,7 @@ handle_message_enqueued_tracing(Process *c_p,
         Sint tok_label = 0;
         Sint tok_lastcnt = 0;
         Sint tok_serial = 0;
+        Sint len = erts_proc_sig_privqs_len(c_p);
         Eterm seq_trace_token = ERL_MESSAGE_TOKEN(msg);
 
         if (seq_trace_token != NIL && is_tuple(seq_trace_token)) {
@@ -3231,7 +3617,7 @@ handle_message_enqueued_tracing(Process *c_p,
         DTRACE6(message_queued,
                 tracing->messages.receiver_name,
                 size_object(ERL_MESSAGE_TERM(msg)),
-                c_p->sig_qs.len,
+                len, /* This is NOT message queue len, but its something... */
                 tok_label, tok_lastcnt, tok_serial);
     }
 #endif
@@ -3303,24 +3689,11 @@ handle_msg_tracing(Process *c_p, ErtsSigRecvTracing *tracing,
     return 0;
 }
 
-/*
- * ERTS_INSPECT_MSGQ_KEEP_OH_MSGS == 0 will move off heap messages
- * into the heap of the inspected process if off_heap_message_queue
- * is false when process_info(_, messages) is called. That is, the
- * following GC will have more data in the rootset compared to the
- * scenario when process_info(_, messages) had not been called.
- *
- * ERTS_INSPECT_MSGQ_KEEP_OH_MSGS != 0 will keep off heap messages
- * off heap when process_info(_, messages) is called regardless of
- * the off_heap_message_queue setting of the process. That is, it
- * will change the following execution of the process as little as
- * possible.
- */
-#define ERTS_INSPECT_MSGQ_KEEP_OH_MSGS 1
-
 Uint
-erts_proc_sig_prep_msgq_for_inspection(Process *c_p, Process *rp,
+erts_proc_sig_prep_msgq_for_inspection(Process *c_p,
+                                       Process *rp,
                                        ErtsProcLocks rp_locks,
+                                       int info_on_self,
                                        ErtsMessageInfo *mip)
 {
     Uint tot_heap_size;
@@ -3329,9 +3702,8 @@ erts_proc_sig_prep_msgq_for_inspection(Process *c_p, Process *rp,
     int self_on_heap;
 
     /*
-     * Prepare the message queue for inspection
-     * by process_info().
-     *
+     * Prepare the message queue (inner signal queue)
+     * for inspection by process_info().
      *
      * - Decode all messages on external format
      * - Remove all corrupt dist messages from queue
@@ -3340,20 +3712,11 @@ erts_proc_sig_prep_msgq_for_inspection(Process *c_p, Process *rp,
      * - Return total heap size need for all messages
      *   that needs to be copied.
      *
-     * If ERTS_INSPECT_MSGQ_KEEP_OH_MSGS == 0:
-     * - In case off heap messages is disabled and
-     *   we are inspecting our own queue, move all
-     *   off heap data into the heap.
      */
 
-    /*
-     * All non-message signals *need* to have been
-     * handled before calling this functions...
-     */
-    ASSERT(!rp->sig_qs.cont);
-    ASSERT(!rp->sig_qs.nmsigs.next && !rp->sig_qs.nmsigs.last);
+    ASSERT(!info_on_self || c_p == rp);
 
-    self_on_heap = c_p == rp && !(c_p->flags & F_OFF_HEAP_MSGQ);
+    self_on_heap = info_on_self && !(c_p->flags & F_OFF_HEAP_MSGQ);
 
     tot_heap_size = 0;
     i = 0;
@@ -3367,8 +3730,7 @@ erts_proc_sig_prep_msgq_for_inspection(Process *c_p, Process *rp,
 	if (ERTS_SIG_IS_EXTERNAL_MSG(mp)) {
 	    /* decode it... */
 	    if (mp->data.attached)
-		erts_decode_dist_message(rp, rp_locks, mp,
-					 ERTS_INSPECT_MSGQ_KEEP_OH_MSGS);
+		erts_decode_dist_message(rp, rp_locks, mp, !0);
 
 	    msg = ERL_MESSAGE_TERM(mp);
 
@@ -3394,44 +3756,11 @@ erts_proc_sig_prep_msgq_for_inspection(Process *c_p, Process *rp,
 
 	ASSERT(is_value(msg));
 
-#if ERTS_INSPECT_MSGQ_KEEP_OH_MSGS
 	if (is_not_immed(msg) && (!self_on_heap || mp->data.attached)) {
 	    Uint sz = size_object(msg);
 	    mip[i].size = sz;
 	    tot_heap_size += sz;
 	}
-#else
-	if (self_on_heap) {
-	    if (mp->data.attached) {
-		ErtsMessage *tmp = NULL;
-		if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
-		    erts_link_mbuf_to_proc(rp, mp->data.heap_frag);
-		    mp->data.attached = NULL;
-		}
-		else {
-		    /*
-		     * Need to replace the message reference since
-		     * we will get references to the message data
-		     * from the heap...
-		     */
-		    ErtsMessage **mpp;
-		    tmp = erts_alloc_message(0, NULL);
-		    sys_memcpy((void *) tmp->m, (void *) mp->m,
-			       sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ); 
-		    mpp = i == 0 ? &rp->sig_qs.first : &mip[i-1].msgp->next;
-		    erts_msgq_replace_msg_ref(&rp->msg, tmp, mpp);
-		    erts_save_message_in_proc(rp, mp);
-		    mp = tmp;
-		}
-	    }
-	}
-	else if (is_not_immed(msg)) {
-	    Uint sz = size_object(msg);
-	    mip[i].size = sz;
-	    tot_heap_size += sz;
-	}
-
-#endif
 
 	mip[i].msgp = mp;
 	i++;
@@ -3439,6 +3768,8 @@ erts_proc_sig_prep_msgq_for_inspection(Process *c_p, Process *rp,
 	mp = mp->next;
     }
 
+    ASSERT(c_p->sig_qs.len == i);
+
     return tot_heap_size;
 }
 
@@ -3475,11 +3806,11 @@ move_msg_to_heap(Process *c_p, ErtsMessage *mp)
 void
 erts_proc_sig_move_msgs_to_heap(Process *c_p)
 {
-    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p);
+    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p, 0);
 
     ERTS_FOREACH_SIG_PRIVQS(c_p, sig, move_msg_to_heap(c_p, sig));
 
-    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p);
+    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p, 0);
 }
 
 
@@ -3670,6 +4001,7 @@ erts_proc_sig_debug_foreach_sig(Process *c_p,
 
                 case ERTS_SIG_Q_OP_IS_ALIVE:
                 case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE:
+                case ERTS_SIG_Q_OP_PROCESS_INFO:
                     break;
 
                 default:
@@ -3714,10 +4046,8 @@ chk_eterm(Process *c_p, int privq, ErtsMessage *mp, Eterm term)
     for (bp = erts_message_to_heap_frag(mp); bp; bp = bp->next) {
         if (bp->mem <= ptr && ptr < bp->mem + bp->used_size)
             return;
-        bp = bp->next;
     }
 
-    ERTS_ASSERT(privq);
     ASSERT(erts_dbg_within_proc(ptr, c_p, NULL));
 }
 
@@ -3740,6 +4070,12 @@ proc_sig_hdbg_check_queue(Process *proc,
     ErtsMessage **save = proc->sig_qs.save;
     ErtsMessage **saved_last = proc->sig_qs.saved_last;
 
+    if (!privq) {
+        ErtsSignal *sig = (ErtsSignal *) *sig_next;
+        if (sig->common.tag == ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK) {
+
+        }
+    }
 
     nm_next = sig_nm_next;
     nm_last = sig_nm_last;
@@ -3804,21 +4140,29 @@ proc_sig_hdbg_check_queue(Process *proc,
         if (!sig)
             break;
 
-        nm_sigs++;
+        nm_sig = (ErtsSignal *) sig;
 
-        ERTS_ASSERT(!last_nm_sig_found);
-        ERTS_ASSERT(ERTS_SIG_IS_NON_MSG(sig));
+        if (nm_sig->common.tag == ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK) {
+            ERTS_ASSERT(!privq);
+            ERTS_ASSERT(sig == *sig_next);
+        }
+        else {
+            nm_sigs++;
 
-        nm_sig = (ErtsSignal *) sig;
+            ERTS_ASSERT(!last_nm_sig_found);
+            ERTS_ASSERT(ERTS_SIG_IS_NON_MSG(sig));
+
+            ERTS_ASSERT(nm_next == next);
+
+            if (nm_last == next) {
+                ASSERT(!nm_sig->common.specific.next);
+                last_nm_sig_found = 1;
+            }
 
-        ERTS_ASSERT(nm_next == next);
+            nm_next = nm_sig->common.specific.next;
 
-        if (nm_last == next) {
-            ASSERT(!nm_sig->common.specific.next);
-            last_nm_sig_found = 1;
         }
 
-        nm_next = nm_sig->common.specific.next;
         next = &nm_sig->common.next;
         sig = nm_sig->common.next;
 
@@ -3873,10 +4217,10 @@ proc_sig_hdbg_check_queue(Process *proc,
 }
 
 void
-erts_proc_sig_hdbg_check_priv_queue(Process *p, char *what, char *file, int line)
+erts_proc_sig_hdbg_check_priv_queue(Process *p, int qlock, char *what, char *file, int line)
 {
     int found_saved_last = 0;
-    Sint len1, len2;
+    Sint len, len1, len2;
     ERTS_LC_ASSERT(erts_thr_progress_is_blocking()
                    || ERTS_PROC_IS_EXITING(p)
                    || (ERTS_PROC_LOCK_MAIN
@@ -3901,7 +4245,8 @@ erts_proc_sig_hdbg_check_priv_queue(Process *p, char *what, char *file, int line
                                      ERTS_PSFLG_SIG_Q);
     if (p->sig_qs.saved_last)
         ERTS_ASSERT(found_saved_last);
-    ERTS_ASSERT(p->sig_qs.len == len1 + len2);
+    len = proc_sig_privqs_len(p, qlock);
+    ERTS_ASSERT(len == len1 + len2);
 }
 
 void
diff --git a/erts/emulator/beam/erl_proc_sig_queue.h b/erts/emulator/beam/erl_proc_sig_queue.h
index 56fe3e683e..d250ad820f 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.h
+++ b/erts/emulator/beam/erl_proc_sig_queue.h
@@ -32,6 +32,7 @@
  *              - Unlink
  *              - Group leader
  *              - Is process alive
+ *              - Process info request
  *              - Trace change
  *
  *              The signal queue consists of three parts:
@@ -78,6 +79,9 @@
 #if 0
 #  define ERTS_PROC_SIG_HARD_DEBUG
 #endif
+#if 0
+#  define ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN
+#endif
 
 struct erl_mesg;
 
@@ -95,22 +99,22 @@ typedef struct {
 #ifdef ERTS_PROC_SIG_HARD_DEBUG
 #  define ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE(P)       \
     ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__((P), "")
-#  define ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(P) \
-    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((P), "")
+#  define ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(P, QL) \
+    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__((P), (QL), "")
 #  define ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__(P, What)   \
     erts_proc_sig_hdbg_check_in_queue((P), (What), __FILE__, __LINE__)
-#  define ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__(P, What)   \
-    erts_proc_sig_hdbg_check_priv_queue((P), (What), __FILE__, __LINE__)
+#  define ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__(P, QL, What)              \
+    erts_proc_sig_hdbg_check_priv_queue((P), (QL), (What), __FILE__, __LINE__)
 struct process;
-void erts_proc_sig_hdbg_check_priv_queue(struct process *c_p, char *what,
-                                         char *file, int line);
+void erts_proc_sig_hdbg_check_priv_queue(struct process *c_p, int qlock,
+                                         char *what, char *file, int line);
 void erts_proc_sig_hdbg_check_in_queue(struct process *c_p, char *what,
                                        char *file, int line);
 #else
 #  define ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE(P)
-#  define ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(P)
+#  define ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(P, QL)
 #  define ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__(P, What)
-#define ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__(P, What)
+#define ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE__(P, QL, What)
 #endif
 
 #endif
@@ -118,6 +122,57 @@ void erts_proc_sig_hdbg_check_in_queue(struct process *c_p, char *what,
 #if !defined(ERTS_PROC_SIG_QUEUE_H__) && !defined(ERTS_PROC_SIG_QUEUE_TYPE_ONLY)
 #define ERTS_PROC_SIG_QUEUE_H__
 
+#define ERTS_SIG_Q_OP_BITS      8                      
+#define ERTS_SIG_Q_OP_SHIFT     0
+#define ERTS_SIG_Q_OP_MASK      ((1 << ERTS_SIG_Q_OP_BITS) - 1)
+
+#define ERTS_SIG_Q_TYPE_BITS    8
+#define ERTS_SIG_Q_TYPE_SHIFT   ERTS_SIG_Q_OP_BITS
+#define ERTS_SIG_Q_TYPE_MASK    ((1 << ERTS_SIG_Q_TYPE_BITS) - 1)
+
+#define ERTS_SIG_Q_NON_X_BITS__ (_HEADER_ARITY_OFFS \
+                                 + ERTS_SIG_Q_OP_BITS \
+                                 + ERTS_SIG_Q_TYPE_BITS)
+
+#define ERTS_SIG_Q_XTRA_BITS    (32 - ERTS_SIG_Q_NON_X_BITS__)
+#define ERTS_SIG_Q_XTRA_SHIFT   (ERTS_SIG_Q_OP_BITS \
+                                 + ERTS_SIG_Q_TYPE_BITS)
+#define ERTS_SIG_Q_XTRA_MASK    ((1 << ERTS_SIG_Q_XTRA_BITS) - 1)
+
+
+#define ERTS_PROC_SIG_OP(Tag) \
+    ((int) (_unchecked_thing_arityval((Tag)) \
+            >> ERTS_SIG_Q_OP_SHIFT) & ERTS_SIG_Q_OP_MASK)
+
+#define ERTS_PROC_SIG_TYPE(Tag) \
+    ((Uint16) (_unchecked_thing_arityval((Tag)) \
+               >> ERTS_SIG_Q_TYPE_SHIFT) & ERTS_SIG_Q_TYPE_MASK)
+
+#define ERTS_PROC_SIG_XTRA(Tag) \
+    ((Uint32) (_unchecked_thing_arityval((Tag)) \
+               >> ERTS_SIG_Q_XTRA_SHIFT) & ERTS_SIG_Q_XTRA_MASK)
+
+#define ERTS_PROC_SIG_MAKE_TAG(Op, Type, Xtra)                  \
+    (ASSERT(0 <= (Xtra) && (Xtra) <= ERTS_SIG_Q_XTRA_MASK),     \
+     _make_header((((Type) & ERTS_SIG_Q_TYPE_MASK)              \
+                   << ERTS_SIG_Q_TYPE_SHIFT)                    \
+                  | (((Op) & ERTS_SIG_Q_OP_MASK)                \
+                     << ERTS_SIG_Q_OP_SHIFT)                    \
+                  | (((Xtra) & ERTS_SIG_Q_XTRA_MASK)            \
+                     << ERTS_SIG_Q_XTRA_SHIFT),                 \
+                  _TAG_HEADER_EXTERNAL_PID))
+
+
+/*
+ * ERTS_SIG_Q_OP_MSGQ_LEN_OFFS_MARK is not an actual
+ * operation. We keep it at the top of the OP range,
+ * larger than ERTS_SIG_Q_OP_MAX.
+ */
+#define ERTS_SIG_Q_OP_MSGQ_LEN_OFFS_MARK ERTS_SIG_Q_OP_MASK
+
+#define ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK \
+    ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_MSGQ_LEN_OFFS_MARK,0,0)
+
 struct dist_entry_;
 
 /*
@@ -451,6 +506,58 @@ void
 erts_proc_sig_send_is_alive_request(Process *c_p, Eterm to,
                                     Eterm ref);
 
+/**
+ *
+ * @brief Send a 'process info request' signal to a process.
+ *
+ * A response message '{Ref, Result}' is sent to the
+ * sender when performed where Ref is the reference passed
+ * as 'ref' argument, and Result corresponds to return result
+ * from erlang:process_info/[1,2].
+ *
+ * @param[in]     c_p           Pointer to process struct of
+ *                              currently executing process.
+ *                              NULL if signal arrived via
+ *                              distribution.
+ *
+ * @param[in]     to            Identifier of receiver.
+ *
+ * @param[in]     item_ix       Info index array to pass to
+ *                              erts_process_info()
+ *
+ * @param[in]     len           Lenght of info index array
+ *
+ * @param[in]     need_msgq_len Non-zero if message queue
+ *                              length is needed; otherwise,
+ *                              zero. If non-zero, sig_qs.len
+ *                              will be set to correspond
+ *                              to the message queue length
+ *                              before call to
+ *                              erts_process_info()
+ *
+ * @param[in]     flags         Flags to pass to
+ *                              erts_process_info()
+ *
+ * @param[in]     reserve_size  Heap size that is known to
+ *                              be needed. May not be correct
+ *                              though.
+ *
+ * @param[in]     ref           Reference to use in response
+ *                              message to the sending
+ *                              process (i.e., c_p).
+ *
+ */
+int
+erts_proc_sig_send_process_info_request(Process *c_p,
+                                        Eterm to,
+                                        int *item_ix,
+                                        int len,
+                                        int need_msgq_len,
+                                        int flags,
+                                        Uint reserve_size,
+                                        Eterm ref);
+
+
 /*
  * End of send operations of currently supported process signals.
  */
@@ -601,9 +708,30 @@ erts_proc_sig_receive_helper(Process *c_p, int fcalls,
  *
  * @param[in]   c_p             Pointer to process struct of
  *                              currently executing process.
+ * @returns                     Amount of message signals in
+ *                              inner plus middle signal
+ *                              queues after fetch completed
+ *                              (NOT the message queue
+ *                              length).
+ */
+ERTS_GLB_INLINE Sint erts_proc_sig_fetch(Process *p);
+
+/**
+ *
+ * @brief Get amount of messages in private queues
+ *
+ * @param[in]   c_p             Pointer to process struct of
+ *                              currently executing process.
  *
+ * @returns                     Amount of message signals in
+ *                              inner plus middle signal
+ *                              queues after fetch completed
+ *                              (NOT the message queue
+ *                              length).
  */
-void erts_proc_sig_fetch(Process *p);
+Sint
+erts_proc_sig_privqs_len(Process *c_p);
+
 
 typedef struct {
     Uint size;
@@ -623,17 +751,19 @@ typedef struct {
  *
  * @param[in]   rp_locks        Process locks held on 'rp'.
  *
+ * @param[in]   info_on_self    Integer set to non-zero value
+ *                              if caller is inspecting itself;
+ *                              otherwise, zero.
+ *
  * @param[in]   mip             Pointer to array of
  *                              ErtsMessageInfo structures.
- *                              
  */
-
 Uint erts_proc_sig_prep_msgq_for_inspection(Process *c_p,
                                             Process *rp,
                                             ErtsProcLocks rp_locks,
+                                            int info_on_self,
                                             ErtsMessageInfo *mip);
 
-
 /**
  *
  * @brief Move message data of messages in private queues to heap
@@ -687,4 +817,56 @@ extern Process *erts_dirty_process_signal_handler;
 extern Process *erts_dirty_process_signal_handler_high;
 extern Process *erts_dirty_process_signal_handler_max;
 
+void erts_proc_sig_fetch__(Process *proc);
+Sint erts_proc_sig_fetch_msgq_len_offs__(Process *proc);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE Sint
+erts_proc_sig_fetch(Process *proc)
+{
+    Sint res = 0;
+    ErtsSignal *sig;
+
+    ERTS_LC_ASSERT(erts_thr_progress_is_blocking()
+                   || ERTS_PROC_IS_EXITING(proc)
+                   || ((erts_proc_lc_my_proc_locks(proc)
+                        & (ERTS_PROC_LOCK_MAIN
+                           | ERTS_PROC_LOCK_MSGQ))
+                       == (ERTS_PROC_LOCK_MAIN
+                           | ERTS_PROC_LOCK_MSGQ)));
+
+    ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE(proc);
+    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(proc, !0);
+
+    sig = (ErtsSignal *) proc->sig_inq.first;
+    if (sig) {
+        if (ERTS_LIKELY(sig->common.tag != ERTS_PROC_SIG_MSGQ_LEN_OFFS_MARK))
+            erts_proc_sig_fetch__(proc);
+        else
+            res = erts_proc_sig_fetch_msgq_len_offs__(proc);
+    }
+
+    res += proc->sig_qs.len;
+
+    ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(proc, !0);
+
+#ifdef ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN
+    {
+        Sint len = 0;
+        ERTS_FOREACH_SIG_PRIVQS(
+            proc, mp,
+            {
+                if (ERTS_SIG_IS_MSG(mp))
+                    len++;
+            });
+        ERTS_ASSERT(res == len);
+    }
+#endif
+
+    return res;
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
 #endif /* ERTS_PROC_SIG_QUEUE_H__ */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 374583ec47..0118302eb3 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10750,7 +10750,6 @@ exit_permanent_prio_elevation(Process *c_p, erts_aint32_t state)
     while (1) {
         erts_aint32_t aprio, uprio, n, e;
         ASSERT(a & ERTS_PSFLG_EXITING);
-        ASSERT(!(a & ERTS_PSFLG_FREE));
         aprio = ERTS_PSFLGS_GET_ACT_PRIO(a);
         uprio = ERTS_PSFLGS_GET_USR_PRIO(a);
         if (aprio >= uprio)
@@ -10796,8 +10795,7 @@ erts_execute_dirty_system_task(Process *c_p)
 
     if (c_p->flags & F_DIRTY_GC_HIBERNATE) {
 	erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
-	erts_proc_sig_fetch(c_p);
-	if (c_p->sig_qs.len)
+	if (erts_proc_sig_fetch(c_p))
             c_p->flags &= ~F_DIRTY_GC_HIBERNATE; /* operation aborted... */
         else {
 	    erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
@@ -12377,12 +12375,14 @@ erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt)
         case ERTS_MON_TYPE_PORT: {
 	    Port *prt;
             ASSERT(is_internal_port(mon->other.item));
+            erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
             prt = erts_id2port(mon->other.item);
 	    if (prt) {
                 erts_fire_port_monitor(prt, mon);
                 erts_port_release(prt);
                 mon = NULL;
             }
+            erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
             break;
         }
         case ERTS_MON_TYPE_RESOURCE:
@@ -12461,10 +12461,8 @@ erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt)
             ASSERT(is_internal_port(mon->other.item));
             prt = erts_port_lookup_raw(mon->other.item);
             if (prt) {
-                erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
                 if (erts_port_demonitor(c_p, prt, mon) != ERTS_PORT_OP_DROPPED)
                     mon = NULL;
-                erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
             }
             break;
         }
@@ -12567,7 +12565,6 @@ erts_proc_exit_handle_link(ErtsLink *lnk, void *vctxt)
         if (!erts_link_dist_delete(dlnk))
             ldp = NULL;
 
-        erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
         code = erts_dsig_prepare(&dsd, dep, c_p, 0, ERTS_DSP_NO_LOCK, 0, 0);
         switch (code) {
         case ERTS_DSIG_PREP_CONNECTED:
@@ -12581,7 +12578,6 @@ erts_proc_exit_handle_link(ErtsLink *lnk, void *vctxt)
                 ASSERT(code == ERTS_DSIG_SEND_OK);
             }
         }
-        erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
         break;
     }
     default:
@@ -12643,8 +12639,6 @@ erts_do_exit_process(Process* p, Eterm reason)
 
     cancel_suspend_of_suspendee(p, ERTS_PROC_LOCKS_ALL); 
 
-    erts_proc_sig_fetch(p);
-
     if (IS_TRACED(p)) {
 	if (IS_TRACED_FL(p, F_TRACE_CALLS))
 	    erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_EXITING);
@@ -12911,6 +12905,12 @@ erts_continue_exit_process(Process *p)
     pectxt.c_p = p;
     pectxt.reason = reason;
 
+    erts_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_MSGQ);
+
+    erts_proc_sig_fetch(p);
+
+    erts_proc_unlock(p, ERTS_PROC_LOCK_MSGQ);
+
     if (links) {
         erts_link_tree_foreach_delete(&links,
                                       erts_proc_exit_handle_link,
@@ -12932,7 +12932,6 @@ erts_continue_exit_process(Process *p)
         ASSERT(!lt_monitors);
     }
 
-    erts_proc_sig_fetch(p);
     /*
      * erts_proc_sig_handle_exit() implements yielding.
      * However, this function cannot handle it yet... loop
@@ -12944,7 +12943,6 @@ erts_continue_exit_process(Process *p)
             break;
     }
 
-    erts_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
     erts_flush_trace_messages(p, ERTS_PROC_LOCK_MAIN);
@@ -12981,27 +12979,41 @@ erts_continue_exit_process(Process *p)
 }
 
 Process *
-erts_try_lock_sig_free_proc(Eterm pid, ErtsProcLocks locks)
+erts_try_lock_sig_free_proc(Eterm pid, ErtsProcLocks locks,
+                            erts_aint32_t *statep)
 {
     Process *rp = erts_proc_lookup_raw(pid);
     erts_aint32_t state;
 
-    if (!rp)
+    if (!rp) {
+        if (statep)
+            *statep = ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE;
         return NULL;
+    }
 
     ERTS_LC_ASSERT(!erts_proc_lc_my_proc_locks(rp));
 
     state = erts_atomic32_read_nob(&rp->state);
-    if (state & ERTS_PSFLG_EXITING)
+    if (statep)
+        *statep = state;
+
+    if (state & ERTS_PSFLG_FREE)
         return NULL;
 
     if (state & (ERTS_PSFLG_SIG_IN_Q|ERTS_PSFLG_SIG_Q))
         return ERTS_PROC_LOCK_BUSY;
+
+    if (!locks)
+        return rp;
+
     if (erts_proc_trylock(rp, locks) == EBUSY)
         return ERTS_PROC_LOCK_BUSY;
 
     state = erts_atomic32_read_nob(&rp->state);
-    if (state & ERTS_PSFLG_EXITING) {
+    if (statep)
+        *statep = state;
+
+    if (state & ERTS_PSFLG_FREE) {
         erts_proc_unlock(rp, locks);
         return NULL;
     }
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 0550fb05b5..7906331b6b 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1371,6 +1371,7 @@ extern int erts_system_profile_ts_type;
 #define F_LOCAL_SIGS_ONLY    (1 << 26)
 #define F_TRAP_EXIT          (1 << 27) /* Trapping exit */
 #define F_DEFERRED_SAVED_LAST (1 << 28)
+#define F_DELAYED_PSIGQS_LEN (1 << 29)
 
 /*
  * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent
@@ -1797,6 +1798,10 @@ void erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp
 void erts_print_run_queue_info(fmtfn_t, void *to_arg, ErtsRunQueue*);
 void erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg);
 void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg);
+Eterm erts_process_info(Process *c_p, ErtsHeapFactory *hfact,
+                        Process *rp, ErtsProcLocks rp_locks,
+                        int *item_ix, int item_ix_len,
+                        int flags, Uint reserve_size, Uint *reds);
 
 typedef struct {
     Process *c_p;
@@ -1835,7 +1840,7 @@ Uint erts_debug_nbalance(void);
 
 int erts_debug_wait_completed(Process *c_p, int flags);
 
-Uint erts_process_memory(Process *c_p, int incl_msg_inq);
+Uint erts_process_memory(Process *c_p, int include_sigs_in_transit);
 
 #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
 #  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P)					\
@@ -2575,7 +2580,8 @@ ERTS_TIME2REDS_IMPL__(ErtsMonotonicTime start, ErtsMonotonicTime end)
 #endif
 
 Process *erts_try_lock_sig_free_proc(Eterm pid,
-                                     ErtsProcLocks locks);
+                                     ErtsProcLocks locks,
+                                     erts_aint32_t *statep);
 
 Process *erts_pid2proc_not_running(Process *,
 				   ErtsProcLocks,
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index aee88841ae..38be3938cd 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -239,39 +239,70 @@ erts_erase_dicts(Process *p)
 /* 
  * Called from process_info/1,2.
  */
-Eterm erts_dictionary_copy(Process *p, ProcDict *pd) 
+Eterm erts_dictionary_copy(ErtsHeapFactory *hfact, ProcDict *pd, Uint reserve_size) 
 {
-    Eterm* hp;
-    Eterm* heap_start;
-    Eterm res = NIL;
-    Eterm tmp, tmp2;
+    Eterm res;
     unsigned int i, num;
+    Uint *sz;
+    Uint szi, rsz = reserve_size;
 
-    if (pd == NULL) {
-	return res;
-    }
+    if (pd == NULL)
+	return NIL;
 
     PD_CHECK(pd);
     num = HASH_RANGE(pd);
-    heap_start = hp = (Eterm *) erts_alloc(ERTS_ALC_T_TMP,
-					   sizeof(Eterm) * pd->numElements * 2);
-    for (i = 0; i < num; ++i) {
-	tmp = ARRAY_GET(pd, i);
+    sz = (Uint *) erts_alloc(ERTS_ALC_T_TMP, sizeof(Uint) * pd->numElements);
+
+    for (i = 0, szi = 0; i < num; ++i) {
+	Eterm tmp = ARRAY_GET(pd, i);
 	if (is_boxed(tmp)) {
+            Uint size;
 	    ASSERT(is_tuple(tmp));
-	    res = CONS(hp, tmp, res);
-	    hp += 2;
-	} else if (is_list(tmp)) {
+            size = size_object(tmp) + 2;
+            sz[szi++] = size;
+            rsz += size;
+	}
+        else if (is_list(tmp)) {
 	    while (tmp != NIL) {
-		tmp2 = TCAR(tmp);
-		res = CONS(hp, tmp2, res);
-		hp += 2;
+                Uint size = size_object(TCAR(tmp)) + 2;
+                sz[szi++] = size;
+                rsz += size;
+    		tmp = TCDR(tmp);
+	    }
+	}
+    }
+
+    res = NIL;
+
+    for (i = 0, szi = 0; i < num; ++i) {
+	Eterm tmp = ARRAY_GET(pd, i);
+	if (is_boxed(tmp)) {
+            Uint size;
+            Eterm el, *hp;
+	    ASSERT(is_tuple(tmp));
+            size = sz[szi++];
+            rsz -= size;
+            hp = erts_produce_heap(hfact, size, rsz);
+            el = copy_struct(tmp, size-2, &hp, hfact->off_heap);
+	    res = CONS(hp, el, res);
+	}
+        else if (is_list(tmp)) {
+	    while (tmp != NIL) {
+                Uint size = sz[szi++];
+                Eterm el, *hp;
+                rsz -= size;
+                hp = erts_produce_heap(hfact, size, rsz);
+                el = copy_struct(TCAR(tmp), size-2, &hp, hfact->off_heap);
+		res = CONS(hp, el, res);
 		tmp = TCDR(tmp);
 	    }
 	}
     }
-    res = copy_object(res, p);
-    erts_free(ERTS_ALC_T_TMP, (void *) heap_start);
+
+    ASSERT(rsz == reserve_size);
+
+    erts_free(ERTS_ALC_T_TMP, sz);
+
     return res;
 }
 
diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h
index ab58f3c239..b89b387f5a 100644
--- a/erts/emulator/beam/erl_process_dict.h
+++ b/erts/emulator/beam/erl_process_dict.h
@@ -40,7 +40,7 @@ void erts_erase_dicts(struct process *p);
 void erts_dictionary_dump(fmtfn_t to, void *to_arg, ProcDict *pd);
 void erts_deep_dictionary_dump(fmtfn_t to, void *to_arg,
 			       ProcDict* pd, void (*cb)(fmtfn_t, void *, Eterm obj));
-Eterm erts_dictionary_copy(struct process *p, ProcDict *pd);
+Eterm erts_dictionary_copy(ErtsHeapFactory *hfact, ProcDict *pd, Uint reserve_size);
 
 Eterm erts_pd_hash_get(struct process *p, Eterm id);
 Uint32 erts_pd_make_hx(Eterm key);
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 00659f9f49..aad99774e3 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -109,56 +109,77 @@ link_size(ErtsMonitor *lnk, void *vsize)
     *((Uint *) vsize) += erts_link_size(lnk);
 }
 
-Uint erts_process_memory(Process *p, int incl_msg_inq) {
-  Uint size = 0;
-  struct saved_calls *scb;
-  size += sizeof(Process);
-
-  if (incl_msg_inq) {
-      erts_proc_lock(p, ERTS_PROC_LOCK_MSGQ);
-      erts_proc_sig_fetch(p);
-      erts_proc_unlock(p, ERTS_PROC_LOCK_MSGQ);
-  }
-
-  erts_link_tree_foreach(ERTS_P_LINKS(p),
-                         link_size, (void *) &size);
-  erts_monitor_tree_foreach(ERTS_P_MONITORS(p),
-                            monitor_size, (void *) &size);
-  erts_monitor_list_foreach(ERTS_P_LT_MONITORS(p),
-                            monitor_size, (void *) &size);
-  size += (p->heap_sz + p->mbuf_sz) * sizeof(Eterm);
-  if (p->abandoned_heap)
-      size += (p->hend - p->heap) * sizeof(Eterm);
-  if (p->old_hend && p->old_heap)
-    size += (p->old_hend - p->old_heap) * sizeof(Eterm);
-
-
-  size += p->sig_qs.len * sizeof(ErtsMessage);
-
-  ERTS_FOREACH_SIG_PRIVQS(
-      p, mp,
-      {
-          if (ERTS_SIG_IS_NON_MSG((ErtsSignal *) mp))
-              size += erts_proc_sig_signal_size((ErtsSignal *) mp);
-          else if (mp->data.attached)
-              size += erts_msg_attached_data_size(mp) * sizeof(Eterm);
-      });
-
-  if (p->arg_reg != p->def_arg_reg) {
-    size += p->arity * sizeof(p->arg_reg[0]);
-  }
-
-  if (erts_atomic_read_nob(&p->psd) != (erts_aint_t) NULL)
-    size += sizeof(ErtsPSD);
-
-  scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p);
-  if (scb) {
-    size += (sizeof(struct saved_calls)
-	     + (scb->len-1) * sizeof(scb->ct[0]));
-  }
-
-  size += erts_dicts_mem_size(p);
-  return size;
+Uint erts_process_memory(Process *p, int include_sigs_in_transit)
+{
+    Uint size = 0;
+    struct saved_calls *scb;
+
+    size += sizeof(Process);
+
+    erts_link_tree_foreach(ERTS_P_LINKS(p),
+                           link_size, (void *) &size);
+    erts_monitor_tree_foreach(ERTS_P_MONITORS(p),
+                              monitor_size, (void *) &size);
+    erts_monitor_list_foreach(ERTS_P_LT_MONITORS(p),
+                              monitor_size, (void *) &size);
+    size += (p->heap_sz + p->mbuf_sz) * sizeof(Eterm);
+    if (p->abandoned_heap)
+        size += (p->hend - p->heap) * sizeof(Eterm);
+    if (p->old_hend && p->old_heap)
+        size += (p->old_hend - p->old_heap) * sizeof(Eterm);
+
+    if (!include_sigs_in_transit) {
+        /*
+         * Size of message queue!
+         *
+         * Note that this assumes that any part of message
+         * queue located in middle queue have been moved
+         * into the inner queue prior to this call.
+         * process_info() management ensures this is done-
+         */
+        ErtsMessage *mp;
+        for (mp = p->sig_qs.first; mp; mp = mp->next) {
+            ASSERT(ERTS_SIG_IS_MSG((ErtsSignal *) mp));
+            size += sizeof(ErtsMessage);
+            if (mp->data.attached)
+                size += erts_msg_attached_data_size(mp) * sizeof(Eterm);
+        }
+    }
+    else {
+        /*
+         * Size of message queue plus size of all signals
+         * in transit to the process!
+         */
+        erts_proc_lock(p, ERTS_PROC_LOCK_MSGQ);
+        erts_proc_sig_fetch(p);
+        erts_proc_unlock(p, ERTS_PROC_LOCK_MSGQ);
+
+        ERTS_FOREACH_SIG_PRIVQS(
+            p, mp,
+            {
+                size += sizeof(ErtsMessage);
+                if (ERTS_SIG_IS_NON_MSG((ErtsSignal *) mp))
+                    size += erts_proc_sig_signal_size((ErtsSignal *) mp);
+                else if (mp->data.attached)
+                    size += erts_msg_attached_data_size(mp) * sizeof(Eterm);
+            });
+    }
+
+    if (p->arg_reg != p->def_arg_reg) {
+        size += p->arity * sizeof(p->arg_reg[0]);
+    }
+
+    if (erts_atomic_read_nob(&p->psd) != (erts_aint_t) NULL)
+        size += sizeof(ErtsPSD);
+
+    scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p);
+    if (scb) {
+        size += (sizeof(struct saved_calls)
+                 + (scb->len-1) * sizeof(scb->ct[0]));
+    }
+
+    size += erts_dicts_mem_size(p);
+    return size;
 }
 
 static ERTS_INLINE void
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index d853b2e352..ba3ac4d579 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -859,6 +859,8 @@ void erts_emasculate_writable_binary(ProcBin* pb);
 Eterm erts_new_heap_binary(Process *p, byte *buf, int len, byte** datap);
 Eterm erts_new_mso_binary(Process*, byte*, Uint);
 Eterm new_binary(Process*, byte*, Uint);
+Eterm erts_heap_factory_new_binary(ErtsHeapFactory *hfact, byte *buf,
+                                   Uint len, Uint reserve_size);
 Eterm erts_realloc_binary(Eterm bin, size_t size);
 Eterm erts_build_proc_bin(ErlOffHeap*, Eterm*, Binary*);
 
diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab
index 6055e35717..223f6bec72 100644
--- a/erts/emulator/beam/msg_instrs.tab
+++ b/erts/emulator/beam/msg_instrs.tab
@@ -225,6 +225,7 @@ remove_message() {
         Sint tok_label = 0;
         Sint tok_lastcnt = 0;
         Sint tok_serial = 0;
+        Sint len = erts_proc_sig_privqs_len(c_p);
 
         dtrace_proc_str(c_p, receiver_name);
         token2 = SEQ_TRACE_TOKEN(c_p);
@@ -235,7 +236,8 @@ remove_message() {
         }
         DTRACE6(message_receive,
                 receiver_name, size_object(ERL_MESSAGE_TERM(msgp)),
-                c_p->sig_qs.len - 1, tok_label, tok_lastcnt, tok_serial);
+                len, /* This is NOT message queue len, but its something... */
+                tok_label, tok_lastcnt, tok_serial);
     }
 #endif
     UNLINK_MESSAGE(c_p, msgp);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 4bf60619ba..188e02eff8 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -59,6 +59,7 @@
 #endif
 #define ERTS_WANT_NFUNC_SCHED_INTERNALS__
 #include "erl_nfunc_sched.h"
+#include "erl_proc_sig_queue.h"
 
 #undef M_TRIM_THRESHOLD
 #undef M_TOP_PAD
diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl
index 22706ae8b1..32bfcd5520 100644
--- a/erts/emulator/test/bif_SUITE.erl
+++ b/erts/emulator/test/bif_SUITE.erl
@@ -35,7 +35,8 @@
 	 is_builtin/1, error_stacktrace/1,
 	 error_stacktrace_during_call_trace/1,
          group_leader_prio/1, group_leader_prio_dirty/1,
-         is_process_alive/1]).
+         is_process_alive/1,
+         process_info_blast/1]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
@@ -50,7 +51,7 @@ all() ->
      erl_crash_dump_bytes, min_max, erlang_halt, is_builtin,
      error_stacktrace, error_stacktrace_during_call_trace,
      group_leader_prio, group_leader_prio_dirty,
-     is_process_alive].
+     is_process_alive, process_info_blast].
 
 %% Uses erlang:display to test that erts_printf does not do deep recursion
 display(Config) when is_list(Config) ->
@@ -1099,6 +1100,76 @@ is_process_alive(Config) when is_list(Config) ->
                   Ps),
     ok.
 
+process_info_blast(Config) when is_list(Config) ->
+    Tester = self(),
+    NoAttackers = 1000,
+    NoAL = lists:seq(1, NoAttackers),
+    Consume = make_ref(),
+    Victim = spawn_link(fun () ->
+                                receive
+                                    Consume ->
+                                        ok
+                                end,
+                                consume_msgs()
+                        end),
+    AFun = fun () ->
+                   Victim ! hej,
+                   Res = process_info(Victim, message_queue_len),
+                   Tester ! {self(), Res}
+           end,
+    Attackers0 = lists:map(fun (_) ->
+                                   spawn_link(AFun)
+                           end,
+                           NoAL),
+    lists:foreach(fun (A) ->
+                          receive
+                              {A, Res} ->
+                                  case Res of
+                                      {message_queue_len, Len} when Len > 0, Len =< NoAttackers ->
+                                          Len;
+                                      Error ->
+                                          exit({unexpected, Error})
+                                  end
+                          end
+                  end,
+                  Attackers0),
+    Attackers1 = lists:map(fun (_) ->
+                                   spawn_link(AFun)
+                           end,
+                           NoAL),
+    Victim ! Consume,
+    lists:foreach(fun (A) ->
+                          receive
+                              {A, Res} ->
+                                  case Res of
+                                      {message_queue_len, Len} when Len >= 0, Len =< 2*NoAttackers+1 ->
+                                          ok;
+                                      undefined ->
+                                          ok;
+                                      Error ->
+                                          exit({unexpected, Error})
+                                  end
+                          end
+                  end,
+                  Attackers1),
+    KillFun = fun (P) ->
+                      unlink(P),
+                      exit(P, kill),
+                      false = erlang:is_process_alive(P)
+              end,
+    lists:foreach(fun (A) -> KillFun(A) end, Attackers0),
+    lists:foreach(fun (A) -> KillFun(A) end, Attackers1),
+    KillFun(Victim),
+    ok.
+
+consume_msgs() ->
+    receive
+        _ ->
+            consume_msgs()
+    after 0 ->
+              ok
+    end.
+                              
 %% helpers
     
 id(I) -> I.
diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index 7eff786e8b..46ece531a8 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -42,6 +42,8 @@
 	 process_info_lock_reschedule2/1,
 	 process_info_lock_reschedule3/1,
          process_info_garbage_collection/1,
+         process_info_smoke_all/1,
+         process_info_status_handled_signal/1,
 	 bump_reductions/1, low_prio/1, binary_owner/1, yield/1, yield2/1,
 	 otp_4725/1, bad_register/1, garbage_collect/1, otp_6237/1,
 	 process_info_messages/1, process_flag_badarg/1, process_flag_heap_size/1,
@@ -79,6 +81,8 @@ all() ->
      process_info_lock_reschedule2,
      process_info_lock_reschedule3,
      process_info_garbage_collection,
+     process_info_smoke_all,
+     process_info_status_handled_signal,
      bump_reductions, low_prio, yield, yield2, otp_4725,
      bad_register, garbage_collect, process_info_messages,
      process_flag_badarg, process_flag_heap_size,
@@ -510,14 +514,20 @@ pio_current_location(N, Pid, Pi, Looper) ->
     case Where of
 	{erlang,process_info,2,[]} ->
 	    pio_current_location(N-1, Pid, Pi+1, Looper);
+	{erts_internal,await_result,1, Loc} when is_list(Loc) ->
+	    pio_current_location(N-1, Pid, Pi+1, Looper);
 	{?MODULE,process_info_looper,1,Loc} when is_list(Loc) ->
-	    pio_current_location(N-1, Pid, Pi, Looper+1)
+	    pio_current_location(N-1, Pid, Pi, Looper+1);
+	_ ->
+	    exit({unexpected_location, Where})
     end.
 
 pio_current_stacktrace() ->
     L = [begin
-	     {current_stacktrace,Stk} = process_info(P, current_stacktrace),
-	     {P,Stk}
+	     case process_info(P, current_stacktrace) of
+                 {current_stacktrace, Stk} -> {P,Stk};
+                 undefined -> {P, []}
+             end
 	 end || P <- processes()],
     [erlang:garbage_collect(P) || {P,_} <- L],
     erlang:garbage_collect(),
@@ -973,6 +983,106 @@ process_info_garbage_collection(_Config) ->
 gv(Key,List) ->
     proplists:get_value(Key,List).
 
+process_info_smoke_all_tester() ->
+    register(process_info_smoke_all_tester, self()),
+    put(ets_ref, ets:new(blupp, [])),
+    put(binary, [list_to_binary(lists:duplicate(1000, 1)),
+                 list_to_binary(lists:duplicate(1000, 2))]),
+    process_info_smoke_all_tester_loop().
+
+process_info_smoke_all_tester_loop() ->
+    receive
+        {other_process, Pid} ->
+            case get(procs) of
+                undefined -> put(procs, [Pid]);
+                Procs -> put(procs, [Pid|Procs])
+            end,
+            erlang:monitor(process, Pid),
+            link(Pid),
+            process_info_smoke_all_tester_loop()
+    end.
+
+process_info_smoke_all(Config) when is_list(Config) ->
+    AllPIOptions = [registered_name,
+                    current_function,
+                    initial_call,
+                    messages,
+                    message_queue_len,
+                    links,
+                    monitors,
+                    monitored_by,
+                    dictionary,
+                    trap_exit,
+                    error_handler,
+                    heap_size,
+                    stack_size,
+                    memory,
+                    garbage_collection,
+                    group_leader,
+                    reductions,
+                    priority,
+                    trace,
+                    binary,
+                    sequential_trace_token,
+                    catchlevel,
+                    backtrace,
+                    last_calls,
+                    total_heap_size,
+                    suspending,
+                    min_heap_size,
+                    min_bin_vheap_size,
+                    max_heap_size,
+                    current_location,
+                    current_stacktrace,
+                    message_queue_data,
+                    garbage_collection_info,
+                    magic_ref,
+                    fullsweep_after],
+
+    {ok, Node} = start_node(Config, ""),
+    RP = spawn_link(Node, fun process_info_smoke_all_tester/0),
+    LP = spawn_link(fun process_info_smoke_all_tester/0),
+    RP ! {other_process, LP},
+    LP ! {other_process, RP},
+    LP ! {other_process, self()},
+    LP ! ets:new(blapp, []),
+    LP ! ets:new(blipp, []),
+    LP ! list_to_binary(lists:duplicate(1000, 3)),
+    receive after 1000 -> ok end,
+    _MLP = erlang:monitor(process, LP),
+    true = is_process_alive(LP),
+    PI = process_info(LP, AllPIOptions),
+    io:format("~p~n", [PI]),
+    garbage_collect(),
+    unlink(RP),
+    unlink(LP),
+    exit(RP, kill),
+    exit(LP, kill),
+    false = is_process_alive(LP),
+    stop_node(Node),
+    ok.
+
+process_info_status_handled_signal(Config) when is_list(Config) ->
+    P = spawn_link(fun () ->
+                           receive after infinity -> ok end
+                   end),
+    wait_until(fun () ->
+                       process_info(P, status) == {status, waiting}
+               end),
+    %%
+    %% The 'messages' option will force a process-info-request
+    %% signal to be scheduled on the process. Ensure that status
+    %% 'waiting' is reported even though it is actually running
+    %% when handling the request. We want it to report the status
+    %% it would have had if it had not been handling the
+    %% process-info-request...
+    %%
+    [{status, waiting}, {messages, []}] = process_info(P, [status, messages]),
+    unlink(P),
+    exit(P, kill),
+    false = erlang:is_process_alive(P),
+    ok.
+
 %% Tests erlang:bump_reductions/1.
 bump_reductions(Config) when is_list(Config) ->
     erlang:garbage_collect(),
-- 
cgit v1.2.3


From a591aa492d622ef349b9ec28c63b9e24e991d311 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 11 Apr 2018 19:49:29 +0200
Subject: erts: Remove obsolete locks from lock checker

---
 erts/emulator/beam/erl_lock_check.c | 15 ---------------
 1 file changed, 15 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 0ced5ec310..e1628fd69f 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -78,7 +78,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"driver_lock",				"driver_name"		},
     {	"port_lock",				"port_id"		},
     {	"port_data_lock",			"address"		},
-    {	"bif_timers",				NULL			},
     {	"reg_tab",				NULL			},
     {	"proc_main",				"pid"			},
     {   "old_code",                             "address"               },
@@ -103,7 +102,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"node_table",				NULL			},
     {	"dist_table",				NULL			},
     {	"sys_tracers",				NULL			},
-    {	"module_tab",				NULL			},
     {	"export_tab",				NULL			},
     {	"fun_tab",				NULL			},
     {	"environ",				NULL			},
@@ -111,7 +109,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"drv_ev_state_grow",			NULL,   		},
     {	"drv_ev_state",				"address"		},
     {	"safe_hash",				"address"		},
-    {   "removed_fd_pre_alloc_lock",            "address"               },
     {   "state_prealloc",                       NULL                    },
     {	"schdlr_sspnd",				NULL			},
     {	"migration_info_update",		NULL			},
@@ -134,10 +131,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {   "msacc_list_mutex",                     NULL                    },
     {   "msacc_unmanaged_mutex",                NULL                    },
     {	"atom_tab",				NULL			},
-    {	"misc_op_list_pre_alloc_lock",		"address"		},
-    {	"message_pre_alloc_lock",		"address"		},
-    {	"ptimer_pre_alloc_lock",		"address",		},
-    {	"btm_pre_alloc_lock",			NULL,			},
     {	"dist_entry_out_queue",			"address"		},
     {	"port_sched_lock",			"port_id"		},
     {	"sys_msg_q", 				NULL			},
@@ -147,20 +140,12 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"mtrace_op",				NULL			},
     {	"instr_x",				NULL			},
     {	"instr",				NULL			},
-    {	"pollsets_lock",			NULL			},
     {	"alcu_allocator",			"index"			},
     {	"mseg",					NULL			},
-    {	"port_task_pre_alloc_lock",		"address"		},
-    {	"proclist_pre_alloc_lock",		"address"		},
-    {	"xports_list_pre_alloc_lock",		"address"		},
-    {	"inet_buffer_stack_lock",		NULL			},
-    {	"system_block",				NULL			},
     {	"get_time",				NULL			},
     {	"get_corrected_time",			NULL			},
     {	"runtime",				NULL			},
-    {	"breakpoints",				NULL			},
     {	"pix_lock",				"address"		},
-    {	"run_queues_lists",			NULL			},
     {	"sched_stat",				NULL			},
     {	"async_init_mtx",			NULL			},
 #ifdef __WIN32__
-- 
cgit v1.2.3


From a7a70269a56d3fe651efcfa5a1139bfec2c8b1c0 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 12 Apr 2018 19:42:01 +0200
Subject: erts: Refactor erl_lock_check.c

with just lots of renaming, nothing else.

erts_lc_locked_locks_t -> lc_thread_t
create_locked_locks -> create_thread_data
l_lcks -> thr
l_lck -> ll

And dropped erts_ prefix on some file scope types.
---
 erts/emulator/beam/erl_lock_check.c | 577 ++++++++++++++++++------------------
 1 file changed, 288 insertions(+), 289 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index e1628fd69f..5d33de9e88 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -179,10 +179,10 @@ static const char *rw_op_str(erts_lock_options_t options)
     return erts_lock_options_get_short_desc(options);
 }
 
-typedef struct erts_lc_locked_lock_t_ erts_lc_locked_lock_t;
-struct erts_lc_locked_lock_t_ {
-    erts_lc_locked_lock_t *next;
-    erts_lc_locked_lock_t *prev;
+typedef struct lc_locked_lock_t_ lc_locked_lock_t;
+struct lc_locked_lock_t_ {
+    lc_locked_lock_t *next;
+    lc_locked_lock_t *prev;
     UWord extra;
     Sint16 id;
     char *file;
@@ -192,32 +192,32 @@ struct erts_lc_locked_lock_t_ {
 };
 
 typedef struct {
-    erts_lc_locked_lock_t *first;
-    erts_lc_locked_lock_t *last;
-} erts_lc_locked_lock_list_t;
+    lc_locked_lock_t *first;
+    lc_locked_lock_t *last;
+} lc_locked_lock_list_t;
 
-typedef struct erts_lc_locked_locks_t_ erts_lc_locked_locks_t;
-struct erts_lc_locked_locks_t_ {
+typedef struct lc_thread_t_ lc_thread_t;
+struct lc_thread_t_ {
     char *thread_name;
     int emu_thread;
     erts_tid_t tid;
-    erts_lc_locked_locks_t *next;
-    erts_lc_locked_locks_t *prev;
-    erts_lc_locked_lock_list_t locked;
-    erts_lc_locked_lock_list_t required;
+    lc_thread_t *next;
+    lc_thread_t *prev;
+    lc_locked_lock_list_t locked;
+    lc_locked_lock_list_t required;
 };
 
-typedef union erts_lc_free_block_t_ erts_lc_free_block_t;
-union erts_lc_free_block_t_ {
-    erts_lc_free_block_t *next;
-    erts_lc_locked_lock_t lock;
+typedef union lc_free_block_t_ lc_free_block_t;
+union lc_free_block_t_ {
+    lc_free_block_t *next;
+    lc_locked_lock_t lock;
 };
 
 static ethr_tsd_key locks_key;
 
-static erts_lc_locked_locks_t *erts_locked_locks = NULL;
+static lc_thread_t *lc_threads = NULL;
 
-static erts_lc_free_block_t *free_blocks = NULL;
+static lc_free_block_t *free_blocks = NULL;
 
 #ifdef ERTS_LC_STATIC_ALLOC
 #define ERTS_LC_FB_CHUNK_SIZE 10000
@@ -241,9 +241,9 @@ lc_unlock(void)
 
 static ERTS_INLINE void lc_free(void *p)
 {
-    erts_lc_free_block_t *fb = (erts_lc_free_block_t *) p;
+    lc_free_block_t *fb = (lc_free_block_t *) p;
 #ifdef DEBUG
-    sys_memset((void *) p, 0xdf, sizeof(erts_lc_free_block_t));
+    sys_memset((void *) p, 0xdf, sizeof(lc_free_block_t));
 #endif
     lc_lock();
     fb->next = free_blocks;
@@ -264,22 +264,22 @@ static void *lc_core_alloc(void)
 static void *lc_core_alloc(void)
 {
     int i;
-    erts_lc_free_block_t *fbs;
+    lc_free_block_t *fbs;
     lc_unlock();
-    fbs = (erts_lc_free_block_t *) malloc(sizeof(erts_lc_free_block_t)
+    fbs = (lc_free_block_t *) malloc(sizeof(lc_free_block_t)
 					  * ERTS_LC_FB_CHUNK_SIZE);
     if (!fbs) {
         ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!");
     }
     for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) {
 #ifdef DEBUG
-	sys_memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t));
+	sys_memset((void *) &fbs[i], 0xdf, sizeof(lc_free_block_t));
 #endif
 	fbs[i].next = &fbs[i+1];
     }
 #ifdef DEBUG
     sys_memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1],
-	   0xdf, sizeof(erts_lc_free_block_t));
+	   0xdf, sizeof(lc_free_block_t));
 #endif
     lc_lock();
     fbs[ERTS_LC_FB_CHUNK_SIZE-1].next = free_blocks;
@@ -304,90 +304,90 @@ static ERTS_INLINE void *lc_alloc(void)
 }
 
 
-static erts_lc_locked_locks_t *
-create_locked_locks(char *thread_name)
+static lc_thread_t *
+create_thread_data(char *thread_name)
 {
-    erts_lc_locked_locks_t *l_lcks = malloc(sizeof(erts_lc_locked_locks_t));
-    if (!l_lcks)
+    lc_thread_t *thr = malloc(sizeof(lc_thread_t));
+    if (!thr)
 	ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!");
 
-    l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown");
-    if (!l_lcks->thread_name)
+    thr->thread_name = strdup(thread_name ? thread_name : "unknown");
+    if (!thr->thread_name)
 	ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!");
 
-    l_lcks->emu_thread = 0;
-    l_lcks->tid = erts_thr_self();
-    l_lcks->required.first = NULL;
-    l_lcks->required.last = NULL;
-    l_lcks->locked.first = NULL;
-    l_lcks->locked.last = NULL;
-    l_lcks->prev = NULL;
+    thr->emu_thread = 0;
+    thr->tid = erts_thr_self();
+    thr->required.first = NULL;
+    thr->required.last = NULL;
+    thr->locked.first = NULL;
+    thr->locked.last = NULL;
+    thr->prev = NULL;
+
     lc_lock();
-    l_lcks->next = erts_locked_locks;
-    if (erts_locked_locks)
-	erts_locked_locks->prev = l_lcks;
-    erts_locked_locks = l_lcks;
+    thr->next = lc_threads;
+    if (lc_threads)
+	lc_threads->prev = thr;
+    lc_threads = thr;
     lc_unlock();
-    erts_tsd_set(locks_key, (void *) l_lcks);
-    return l_lcks;
+    erts_tsd_set(locks_key, (void *) thr);
+    return thr;
 }
 
 static void
-destroy_locked_locks(erts_lc_locked_locks_t *l_lcks)
+destroy_locked_locks(lc_thread_t *thr)
 {
-    ASSERT(l_lcks->thread_name);
-    free((void *) l_lcks->thread_name);
-    ASSERT(l_lcks->required.first == NULL);
-    ASSERT(l_lcks->required.last == NULL);
-    ASSERT(l_lcks->locked.first == NULL);
-    ASSERT(l_lcks->locked.last == NULL);
+    ASSERT(thr->thread_name);
+    free((void *) thr->thread_name);
+    ASSERT(thr->required.first == NULL);
+    ASSERT(thr->required.last == NULL);
+    ASSERT(thr->locked.first == NULL);
+    ASSERT(thr->locked.last == NULL);
 
     lc_lock();
-    if (l_lcks->prev)
-	l_lcks->prev->next = l_lcks->next;
+    if (thr->prev)
+	thr->prev->next = thr->next;
     else {
-	ASSERT(erts_locked_locks == l_lcks);
-	erts_locked_locks = l_lcks->next;
+	ASSERT(lc_threads == thr);
+	lc_threads = thr->next;
     }
-
-    if (l_lcks->next)
-	l_lcks->next->prev = l_lcks->prev;
+    if (thr->next)
+	thr->next->prev = thr->prev;
     lc_unlock();
 
-    free((void *) l_lcks);
+    free((void *) thr);
 
 }
 
-static ERTS_INLINE erts_lc_locked_locks_t *
+static ERTS_INLINE lc_thread_t *
 get_my_locked_locks(void)
 {
     return erts_tsd_get(locks_key);
 }
 
-static ERTS_INLINE erts_lc_locked_locks_t *
+static ERTS_INLINE lc_thread_t *
 make_my_locked_locks(void)
 {
-    erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
-    if (l_lcks)
-	return l_lcks;
+    lc_thread_t *thr = get_my_locked_locks();
+    if (thr)
+	return thr;
     else
-	return create_locked_locks(NULL);
+	return create_thread_data(NULL);
 }
 
-static ERTS_INLINE erts_lc_locked_lock_t *
+static ERTS_INLINE lc_locked_lock_t *
 new_locked_lock(erts_lc_lock_t *lck, erts_lock_options_t options,
 		char *file, unsigned int line)
 {
-    erts_lc_locked_lock_t *l_lck = (erts_lc_locked_lock_t *) lc_alloc();
-    l_lck->next = NULL;
-    l_lck->prev = NULL;
-    l_lck->id = lck->id;
-    l_lck->extra = lck->extra;
-    l_lck->file = file;
-    l_lck->line = line;
-    l_lck->flags = lck->flags;
-    l_lck->taken_options = options;
-    return l_lck;
+    lc_locked_lock_t *ll = (lc_locked_lock_t *) lc_alloc();
+    ll->next = NULL;
+    ll->prev = NULL;
+    ll->id = lck->id;
+    ll->extra = lck->extra;
+    ll->file = file;
+    ll->line = line;
+    ll->flags = lck->flags;
+    ll->taken_options = options;
+    return ll;
 }
 
 static void
@@ -424,20 +424,20 @@ print_lock(char *prefix, erts_lc_lock_t *lck, char *suffix)
 }
 
 static void
-print_curr_locks(erts_lc_locked_locks_t *l_lcks)
+print_curr_locks(lc_thread_t *thr)
 {
-    erts_lc_locked_lock_t *l_lck;
-    if (!l_lcks || !l_lcks->locked.first)
+    lc_locked_lock_t *ll;
+    if (!thr || !thr->locked.first)
 	erts_fprintf(stderr,
 		     "Currently no locks are locked by the %s thread.\n",
-		     l_lcks->thread_name);
+		     thr->thread_name);
     else {
 	erts_fprintf(stderr,
 		     "Currently these locks are locked by the %s thread:\n",
-		     l_lcks->thread_name);
-	for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next)
-	  raw_print_lock("  ", l_lck->id, l_lck->extra, l_lck->flags,
-			 l_lck->file, l_lck->line, "\n");
+		     thr->thread_name);
+	for (ll = thr->locked.first; ll; ll = ll->next)
+	  raw_print_lock("  ", ll->id, ll->extra, ll->flags,
+			 ll->file, ll->line, "\n");
     }
 }
 
@@ -466,55 +466,55 @@ uninitialized_lock(void)
 }
 
 static void
-lock_twice(char *prefix, erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck,
+lock_twice(char *prefix, lc_thread_t *thr, erts_lc_lock_t *lck,
 	   erts_lock_options_t options)
 {
     erts_fprintf(stderr, "%s (%s)", prefix, rw_op_str(options));
     print_lock(" ", lck, " lock which is already locked by thread!\n");
-    print_curr_locks(l_lcks);
+    print_curr_locks(thr);
     lc_abort();
 }
 
 static void
-unlock_op_mismatch(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck,
+unlock_op_mismatch(lc_thread_t *thr, erts_lc_lock_t *lck,
 		   erts_lock_options_t options)
 {
     erts_fprintf(stderr, "Unlocking (%s) ", rw_op_str(options));
     print_lock("", lck, " lock which mismatch previous lock operation!\n");
-    print_curr_locks(l_lcks);
+    print_curr_locks(thr);
     lc_abort();
 }
 
 static void
-unlock_of_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
+unlock_of_not_locked(lc_thread_t *thr, erts_lc_lock_t *lck)
 {
     print_lock("Unlocking ", lck, " lock which is not locked by thread!\n");
-    print_curr_locks(l_lcks);
+    print_curr_locks(thr);
     lc_abort();
 }
 
 static void
-lock_order_violation(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
+lock_order_violation(lc_thread_t *thr, erts_lc_lock_t *lck)
 {
     print_lock("Lock order violation occured when locking ", lck, "!\n");
-    print_curr_locks(l_lcks);
+    print_curr_locks(thr);
     print_lock_order();
     lc_abort();
 }
 
 static void
-type_order_violation(char *op, erts_lc_locked_locks_t *l_lcks,
+type_order_violation(char *op, lc_thread_t *thr,
 		     erts_lc_lock_t *lck)
 {
     erts_fprintf(stderr, "Lock type order violation occured when ");
     print_lock(op, lck, "!\n");
-    ASSERT(l_lcks);
-    print_curr_locks(l_lcks);
+    ASSERT(thr);
+    print_curr_locks(thr);
     lc_abort();
 }
 
 static void
-lock_mismatch(erts_lc_locked_locks_t *l_lcks, int exact,
+lock_mismatch(lc_thread_t *thr, int exact,
 	      int failed_have, erts_lc_lock_t *have, int have_len,
 	      int failed_have_not, erts_lc_lock_t *have_not, int have_not_len)
 {
@@ -561,39 +561,39 @@ lock_mismatch(erts_lc_locked_locks_t *l_lcks, int exact,
 		print_lock2("  ", have_not[i].id, have_not[i].extra, 0, "\n");
 	}
     }
-    print_curr_locks(l_lcks);
+    print_curr_locks(thr);
     lc_abort();
 }
 
 static void
-unlock_of_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
+unlock_of_required_lock(lc_thread_t *thr, erts_lc_lock_t *lck)
 {
     print_lock("Unlocking required ", lck, " lock!\n");
-    print_curr_locks(l_lcks);
+    print_curr_locks(thr);
     lc_abort();
 }
 
 static void
-unrequire_of_not_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
+unrequire_of_not_required_lock(lc_thread_t *thr, erts_lc_lock_t *lck)
 {
     print_lock("Unrequire on ", lck, " lock not required!\n");
-    print_curr_locks(l_lcks);
+    print_curr_locks(thr);
     lc_abort();
 }
 
 static void
-require_twice(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
+require_twice(lc_thread_t *thr, erts_lc_lock_t *lck)
 {
     print_lock("Require on ", lck, " lock already required!\n");
-    print_curr_locks(l_lcks);
+    print_curr_locks(thr);
     lc_abort();
 }
 
 static void
-required_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
+required_not_locked(lc_thread_t *thr, erts_lc_lock_t *lck)
 {
     print_lock("Required ", lck, " lock not locked!\n");
-    print_curr_locks(l_lcks);
+    print_curr_locks(thr);
     lc_abort();
 }
 
@@ -601,15 +601,15 @@ required_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
 static void
 thread_exit_handler(void)
 {
-    erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
-    if (l_lcks) {
-	if (l_lcks->locked.first) {
+    lc_thread_t *thr = get_my_locked_locks();
+    if (thr) {
+	if (thr->locked.first) {
 	    erts_fprintf(stderr,
 			 "Thread exiting while having locked locks!\n");
-	    print_curr_locks(l_lcks);
+	    print_curr_locks(thr);
 	    lc_abort();
 	}
-	destroy_locked_locks(l_lcks);
+	destroy_locked_locks(thr);
 	/* erts_tsd_set(locks_key, NULL); */
     }
 }
@@ -627,24 +627,24 @@ lc_abort(void)
 void
 erts_lc_set_thread_name(char *thread_name)
 {
-    erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
-    if (!l_lcks)
-	l_lcks = create_locked_locks(thread_name);
+    lc_thread_t *thr = get_my_locked_locks();
+    if (!thr)
+	thr = create_thread_data(thread_name);
     else {
-	ASSERT(l_lcks->thread_name);
-	free((void *) l_lcks->thread_name);
-	l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown");
-	if (!l_lcks->thread_name)
+	ASSERT(thr->thread_name);
+	free((void *) thr->thread_name);
+	thr->thread_name = strdup(thread_name ? thread_name : "unknown");
+	if (!thr->thread_name)
 	    ERTS_INTERNAL_ERROR("strdup failed");
     }
-    l_lcks->emu_thread = 1;
+    thr->emu_thread = 1;
 }
 
 int
 erts_lc_is_emu_thr(void)
 {
-    erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
-    return l_lcks->emu_thread;
+    lc_thread_t *thr = get_my_locked_locks();
+    return thr->emu_thread;
 }
 
 int
@@ -690,7 +690,7 @@ erts_lc_get_lock_order_id(char *name)
     return (Sint16) -1;
 }
 
-static int compare_locked_by_id(erts_lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand)
+static int compare_locked_by_id(lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand)
 {
     if(locked_lock->id < comparand->id) {
         return -1;
@@ -701,7 +701,7 @@ static int compare_locked_by_id(erts_lc_locked_lock_t *locked_lock, erts_lc_lock
     return 0;
 }
 
-static int compare_locked_by_id_extra(erts_lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand)
+static int compare_locked_by_id_extra(lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand)
 {
     int order = compare_locked_by_id(locked_lock, comparand);
 
@@ -716,18 +716,18 @@ static int compare_locked_by_id_extra(erts_lc_locked_lock_t *locked_lock, erts_l
     return 0;
 }
 
-typedef int (*locked_compare_func)(erts_lc_locked_lock_t *, erts_lc_lock_t *);
+typedef int (*locked_compare_func)(lc_locked_lock_t *, erts_lc_lock_t *);
 
 /* Searches through a list of taken locks, bailing when it hits an entry whose
  * order relative to the search template is the opposite of the one at the
  * start of the search. (*closest_neighbor) is either set to the exact match,
  * or the one closest to it in the sort order. */
 static int search_locked_list(locked_compare_func compare,
-                              erts_lc_locked_lock_t *locked_locks,
+                              lc_locked_lock_t *locked_locks,
                               erts_lc_lock_t *search_template,
-                              erts_lc_locked_lock_t **closest_neighbor)
+                              lc_locked_lock_t **closest_neighbor)
 {
-    erts_lc_locked_lock_t *iterator = locked_locks;
+    lc_locked_lock_t *iterator = locked_locks;
 
     (*closest_neighbor) = iterator;
 
@@ -763,9 +763,9 @@ static int search_locked_list(locked_compare_func compare,
 /* Searches for a lock in the given list that matches search_template, and sets
  * (*locked_locks) to the closest lock in the sort order. */
 static int
-find_lock(erts_lc_locked_lock_t **locked_locks, erts_lc_lock_t *search_template)
+find_lock(lc_locked_lock_t **locked_locks, erts_lc_lock_t *search_template)
 {
-    erts_lc_locked_lock_t *closest_neighbor;
+    lc_locked_lock_t *closest_neighbor;
     int found_lock;
 
     found_lock = search_locked_list(compare_locked_by_id_extra,
@@ -794,9 +794,9 @@ find_lock(erts_lc_locked_lock_t **locked_locks, erts_lc_lock_t *search_template)
 /* Searches for a lock in the given list by id, and sets (*locked_locks) to the
  * closest lock in the sort order. */
 static int
-find_id(erts_lc_locked_lock_t **locked_locks, Sint16 id)
+find_id(lc_locked_lock_t **locked_locks, Sint16 id)
 {
-    erts_lc_locked_lock_t *closest_neighbor;
+    lc_locked_lock_t *closest_neighbor;
     erts_lc_lock_t search_template;
     int found_lock;
 
@@ -815,34 +815,34 @@ find_id(erts_lc_locked_lock_t **locked_locks, Sint16 id)
 void
 erts_lc_have_locks(int *resv, erts_lc_lock_t *locks, int len)
 {
-    erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
+    lc_thread_t *thr = get_my_locked_locks();
     int i;
 
-    if (!l_lcks) {
+    if (!thr) {
 	for (i = 0; i < len; i++)
 	    resv[i] = 0;
     }
     else {
-	erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
+	lc_locked_lock_t *ll = thr->locked.first;
 	for (i = 0; i < len; i++)
-	    resv[i] = find_lock(&l_lck, &locks[i]);
+	    resv[i] = find_lock(&ll, &locks[i]);
     }
 }
 
 void
 erts_lc_have_lock_ids(int *resv, int *ids, int len)
 {
-    erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
+    lc_thread_t *thr = get_my_locked_locks();
     int i;
 
-    if (!l_lcks) {
+    if (!thr) {
 	for (i = 0; i < len; i++)
 	    resv[i] = 0;
     }
     else {
-	erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
+	lc_locked_lock_t *ll = thr->locked.first;
 	for (i = 0; i < len; i++)
-	    resv[i] = find_id(&l_lck, ids[i]);
+	    resv[i] = find_id(&ll, ids[i]);
     }
 }
 
@@ -851,27 +851,27 @@ erts_lc_check(erts_lc_lock_t *have, int have_len,
 	      erts_lc_lock_t *have_not, int have_not_len)
 {
     int i;
-    erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
-    erts_lc_locked_lock_t *l_lck;
+    lc_thread_t *thr = get_my_locked_locks();
+    lc_locked_lock_t *ll;
     
     if (have && have_len > 0) {
-	if (!l_lcks)
+	if (!thr)
 	    lock_mismatch(NULL, 0,
 			  -1, have, have_len,
 			  -1, have_not, have_not_len);
-	l_lck = l_lcks->locked.first;
+	ll = thr->locked.first;
 	for (i = 0; i < have_len; i++) {
-	    if (!find_lock(&l_lck, &have[i]))
-		lock_mismatch(l_lcks, 0,
+	    if (!find_lock(&ll, &have[i]))
+		lock_mismatch(thr, 0,
 			      i, have, have_len,
 			      -1, have_not, have_not_len);
 	}
     }
-    if (have_not && have_not_len > 0 && l_lcks) {
-	l_lck = l_lcks->locked.first;
+    if (have_not && have_not_len > 0 && thr) {
+	ll = thr->locked.first;
 	for (i = 0; i < have_not_len; i++) {
-	    if (find_lock(&l_lck, &have_not[i]))
-		lock_mismatch(l_lcks, 0,
+	    if (find_lock(&ll, &have_not[i]))
+		lock_mismatch(thr, 0,
 			      -1, have, have_len,
 			      i, have_not, have_not_len);
 	}
@@ -881,8 +881,8 @@ erts_lc_check(erts_lc_lock_t *have, int have_len,
 void
 erts_lc_check_exact(erts_lc_lock_t *have, int have_len)
 {
-    erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
-    if (!l_lcks) {
+    lc_thread_t *thr = get_my_locked_locks();
+    if (!thr) {
 	if (have && have_len > 0)
 	    lock_mismatch(NULL, 1,
 			  -1, have, have_len,
@@ -890,17 +890,17 @@ erts_lc_check_exact(erts_lc_lock_t *have, int have_len)
     }
     else {
 	int i;
-	erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
+	lc_locked_lock_t *ll = thr->locked.first;
 	for (i = 0; i < have_len; i++) {
-	    if (!find_lock(&l_lck, &have[i]))
-		lock_mismatch(l_lcks, 1,
+	    if (!find_lock(&ll, &have[i]))
+		lock_mismatch(thr, 1,
 			      i, have, have_len,
 			      -1, NULL, 0);
 	}
-	for (i = 0, l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next)
+	for (i = 0, ll = thr->locked.first; ll; ll = ll->next)
 	    i++;
 	if (i != have_len)
-	    lock_mismatch(l_lcks, 1,
+	    lock_mismatch(thr, 1,
 			  -1, have, have_len,
 			  -1, NULL, 0);
     }
@@ -909,16 +909,16 @@ erts_lc_check_exact(erts_lc_lock_t *have, int have_len)
 void
 erts_lc_check_no_locked_of_type(erts_lock_flags_t type)
 {
-    erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
-    if (l_lcks) {
-	erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
-	for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next) {
-	    if ((l_lck->flags & ERTS_LOCK_FLAGS_MASK_TYPE) == type) {
+    lc_thread_t *thr = get_my_locked_locks();
+    if (thr) {
+	lc_locked_lock_t *ll = thr->locked.first;
+	for (ll = thr->locked.first; ll; ll = ll->next) {
+	    if ((ll->flags & ERTS_LOCK_FLAGS_MASK_TYPE) == type) {
 		erts_fprintf(stderr,
 			     "Locked lock of type %s found which isn't "
 			     "allowed here!\n",
-			     erts_lock_flags_get_type_name(l_lck->flags));
-		print_curr_locks(l_lcks);
+			     erts_lock_flags_get_type_name(ll->flags));
+		print_curr_locks(thr);
 		lc_abort();
 	    }
 	}
@@ -936,7 +936,7 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
      * This in order to make sure that caller can handle
      * the situation without causing a lock order violation.
      */
-    erts_lc_locked_locks_t *l_lcks;
+    lc_thread_t *thr;
 
     if (lck->inited != ERTS_LC_INITITALIZED)
 	uninitialized_lock();
@@ -944,25 +944,25 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
     if (lck->id < 0)
 	return 0;
 
-    l_lcks = get_my_locked_locks();
+    thr = get_my_locked_locks();
 
-    if (!l_lcks || !l_lcks->locked.first) {
-	ASSERT(!l_lcks || !l_lcks->locked.last);
+    if (!thr || !thr->locked.first) {
+	ASSERT(!thr || !thr->locked.last);
 	return 0;
     }
     else {
-	erts_lc_locked_lock_t *tl_lck;
+	lc_locked_lock_t *tl_lck;
 
-	ASSERT(l_lcks->locked.last);
+	ASSERT(thr->locked.last);
 
 #if 0 /* Ok when trylocking I guess... */
-	if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, l_lcks->locked.last->flags))
-	    type_order_violation("trylocking ", l_lcks, lck);
+	if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, thr->locked.last->flags))
+	    type_order_violation("trylocking ", thr, lck);
 #endif
 
-	if (l_lcks->locked.last->id < lck->id
-	    || (l_lcks->locked.last->id == lck->id
-		&& l_lcks->locked.last->extra < lck->extra))
+	if (thr->locked.last->id < lck->id
+	    || (thr->locked.last->id == lck->id
+		&& thr->locked.last->extra < lck->extra))
 	    return 0;
 
 	/*
@@ -971,11 +971,11 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
 
 
 	/* Check that we are not trying to lock this lock twice */
-	for (tl_lck = l_lcks->locked.last; tl_lck; tl_lck = tl_lck->prev) {
+	for (tl_lck = thr->locked.last; tl_lck; tl_lck = tl_lck->prev) {
 	    if (tl_lck->id < lck->id
 		|| (tl_lck->id == lck->id && tl_lck->extra <= lck->extra)) {
 		if (tl_lck->id == lck->id && tl_lck->extra == lck->extra)
-		    lock_twice("Trylocking", l_lcks, lck, options);
+		    lock_twice("Trylocking", thr, lck, options);
 		break;
 	    }
 	}
@@ -1000,8 +1000,8 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
 void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, erts_lock_options_t options,
 			   char *file, unsigned int line)
 {
-    erts_lc_locked_locks_t *l_lcks;
-    erts_lc_locked_lock_t *l_lck;
+    lc_thread_t *thr;
+    lc_locked_lock_t *ll;
 
     if (lck->inited != ERTS_LC_INITITALIZED)
 	uninitialized_lock();
@@ -1009,43 +1009,43 @@ void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, erts_lock_options_t
     if (lck->id < 0)
 	return;
 
-    l_lcks = make_my_locked_locks();
-    l_lck = locked ? new_locked_lock(lck, options, file, line) : NULL;
+    thr = make_my_locked_locks();
+    ll = locked ? new_locked_lock(lck, options, file, line) : NULL;
 
-    if (!l_lcks->locked.last) {
-	ASSERT(!l_lcks->locked.first);
+    if (!thr->locked.last) {
+	ASSERT(!thr->locked.first);
 	if (locked)
-	    l_lcks->locked.first = l_lcks->locked.last = l_lck;
+	    thr->locked.first = thr->locked.last = ll;
     }
     else {
-	erts_lc_locked_lock_t *tl_lck;
+	lc_locked_lock_t *tl_lck;
 #if 0 /* Ok when trylocking I guess... */
-	if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, l_lcks->locked.last->flags))
-	    type_order_violation("trylocking ", l_lcks, lck);
+	if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, thr->locked.last->flags))
+	    type_order_violation("trylocking ", thr, lck);
 #endif
 
-	for (tl_lck = l_lcks->locked.last; tl_lck; tl_lck = tl_lck->prev) {
+	for (tl_lck = thr->locked.last; tl_lck; tl_lck = tl_lck->prev) {
 	    if (tl_lck->id < lck->id
 		|| (tl_lck->id == lck->id && tl_lck->extra <= lck->extra)) {
 		if (tl_lck->id == lck->id && tl_lck->extra == lck->extra)
-		    lock_twice("Trylocking", l_lcks, lck, options);
+		    lock_twice("Trylocking", thr, lck, options);
 		if (locked) {
-		    l_lck->next = tl_lck->next;
-		    l_lck->prev = tl_lck;
+		    ll->next = tl_lck->next;
+		    ll->prev = tl_lck;
 		    if (tl_lck->next)
-			tl_lck->next->prev = l_lck;
+			tl_lck->next->prev = ll;
 		    else
-			l_lcks->locked.last = l_lck;
-		    tl_lck->next = l_lck;
+			thr->locked.last = ll;
+		    tl_lck->next = ll;
 		}
 		return;
 	    }
 	}
 
 	if (locked) {
-	    l_lck->next = l_lcks->locked.first;
-	    l_lcks->locked.first->prev = l_lck;
-	    l_lcks->locked.first = l_lck;
+	    ll->next = thr->locked.first;
+	    thr->locked.first->prev = ll;
+	    thr->locked.first = ll;
 	}
     }
 
@@ -1054,83 +1054,83 @@ void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, erts_lock_options_t
 void erts_lc_require_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options,
 			      char *file, unsigned int line)
 {
-    erts_lc_locked_locks_t *l_lcks = make_my_locked_locks();
-    erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
-    if (!find_lock(&l_lck, lck))
-	required_not_locked(l_lcks, lck);
-    l_lck = new_locked_lock(lck, options, file, line);
-    if (!l_lcks->required.last) {
-	ASSERT(!l_lcks->required.first);
-	l_lck->next = l_lck->prev = NULL;
-	l_lcks->required.first = l_lcks->required.last = l_lck;
+    lc_thread_t *thr = make_my_locked_locks();
+    lc_locked_lock_t *ll = thr->locked.first;
+    if (!find_lock(&ll, lck))
+	required_not_locked(thr, lck);
+    ll = new_locked_lock(lck, options, file, line);
+    if (!thr->required.last) {
+	ASSERT(!thr->required.first);
+	ll->next = ll->prev = NULL;
+	thr->required.first = thr->required.last = ll;
     }
     else {
-	erts_lc_locked_lock_t *l_lck2;
-	ASSERT(l_lcks->required.first);
-	for (l_lck2 = l_lcks->required.last;
+	lc_locked_lock_t *l_lck2;
+	ASSERT(thr->required.first);
+	for (l_lck2 = thr->required.last;
 	     l_lck2;
 	     l_lck2 = l_lck2->prev) {
 	    if (l_lck2->id < lck->id
 		|| (l_lck2->id == lck->id && l_lck2->extra < lck->extra))
 		break;
 	    else if (l_lck2->id == lck->id && l_lck2->extra == lck->extra)
-		require_twice(l_lcks, lck);
+		require_twice(thr, lck);
 	}
 	if (!l_lck2) {
-	    l_lck->next = l_lcks->required.first;
-	    l_lck->prev = NULL;
-	    l_lcks->required.first->prev = l_lck;
-	    l_lcks->required.first = l_lck;
+	    ll->next = thr->required.first;
+	    ll->prev = NULL;
+	    thr->required.first->prev = ll;
+	    thr->required.first = ll;
 	}
 	else {
-	    l_lck->next = l_lck2->next;
-	    if (l_lck->next) {
-		ASSERT(l_lcks->required.last != l_lck2);
-		l_lck->next->prev = l_lck;
+	    ll->next = l_lck2->next;
+	    if (ll->next) {
+		ASSERT(thr->required.last != l_lck2);
+		ll->next->prev = ll;
 	    }
 	    else {
-		ASSERT(l_lcks->required.last == l_lck2);
-		l_lcks->required.last = l_lck;
+		ASSERT(thr->required.last == l_lck2);
+		thr->required.last = ll;
 	    }
-	    l_lck->prev = l_lck2;
-	    l_lck2->next = l_lck;		
+	    ll->prev = l_lck2;
+	    l_lck2->next = ll;
 	}
     }
 }
 
 void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
 {
-    erts_lc_locked_locks_t *l_lcks = make_my_locked_locks();
-    erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
-    if (!find_lock(&l_lck, lck))
-	required_not_locked(l_lcks, lck);
-    l_lck = l_lcks->required.first;
-    if (!find_lock(&l_lck, lck))
-	unrequire_of_not_required_lock(l_lcks, lck);
-    if (l_lck->prev) {
-	ASSERT(l_lcks->required.first != l_lck);
-	l_lck->prev->next = l_lck->next;
+    lc_thread_t *thr = make_my_locked_locks();
+    lc_locked_lock_t *ll = thr->locked.first;
+    if (!find_lock(&ll, lck))
+	required_not_locked(thr, lck);
+    ll = thr->required.first;
+    if (!find_lock(&ll, lck))
+	unrequire_of_not_required_lock(thr, lck);
+    if (ll->prev) {
+	ASSERT(thr->required.first != ll);
+	ll->prev->next = ll->next;
     }
     else {
-	ASSERT(l_lcks->required.first == l_lck);
-	l_lcks->required.first = l_lck->next;
+	ASSERT(thr->required.first == ll);
+	thr->required.first = ll->next;
     }
-    if (l_lck->next) {
-	ASSERT(l_lcks->required.last != l_lck);
-	l_lck->next->prev = l_lck->prev;
+    if (ll->next) {
+	ASSERT(thr->required.last != ll);
+	ll->next->prev = ll->prev;
     }
     else {
-	ASSERT(l_lcks->required.last == l_lck);
-	l_lcks->required.last = l_lck->prev;
+	ASSERT(thr->required.last == ll);
+	thr->required.last = ll->prev;
     }
-    lc_free((void *) l_lck);
+    lc_free((void *) ll);
 }
 
 void erts_lc_lock_flg_x(erts_lc_lock_t *lck, erts_lock_options_t options,
 			char *file, unsigned int line)
 {
-    erts_lc_locked_locks_t *l_lcks;
-    erts_lc_locked_lock_t *l_lck;
+    lc_thread_t *thr;
+    lc_locked_lock_t *new_ll;
 
     if (lck->inited != ERTS_LC_INITITALIZED)
 	uninitialized_lock();
@@ -1138,32 +1138,32 @@ void erts_lc_lock_flg_x(erts_lc_lock_t *lck, erts_lock_options_t options,
     if (lck->id < 0)
 	return;
 
-    l_lcks = make_my_locked_locks();
-    l_lck = new_locked_lock(lck, options, file, line);
+    thr = make_my_locked_locks();
+    new_ll = new_locked_lock(lck, options, file, line);
 
-    if (!l_lcks->locked.last) {
-	ASSERT(!l_lcks->locked.first);
-	l_lcks->locked.last = l_lcks->locked.first = l_lck;
+    if (!thr->locked.last) {
+	ASSERT(!thr->locked.first);
+	thr->locked.last = thr->locked.first = new_ll;
     }
-    else if (l_lcks->locked.last->id < lck->id
-	     || (l_lcks->locked.last->id == lck->id
-		 && l_lcks->locked.last->extra < lck->extra)) {
-	if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, l_lcks->locked.last->flags))
-	    type_order_violation("locking ", l_lcks, lck);
-	l_lck->prev = l_lcks->locked.last;
-	l_lcks->locked.last->next = l_lck;
-	l_lcks->locked.last = l_lck;
+    else if (thr->locked.last->id < lck->id
+	     || (thr->locked.last->id == lck->id
+		 && thr->locked.last->extra < lck->extra)) {
+	if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, thr->locked.last->flags))
+	    type_order_violation("locking ", thr, lck);
+	new_ll->prev = thr->locked.last;
+	thr->locked.last->next = new_ll;
+	thr->locked.last = new_ll;
     }
-    else if (l_lcks->locked.last->id == lck->id && l_lcks->locked.last->extra == lck->extra)
-	lock_twice("Locking", l_lcks, lck, options);
+    else if (thr->locked.last->id == lck->id && thr->locked.last->extra == lck->extra)
+	lock_twice("Locking", thr, lck, options);
     else
-	lock_order_violation(l_lcks, lck);
+	lock_order_violation(thr, lck);
 }
 
 void erts_lc_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
 {
-    erts_lc_locked_locks_t *l_lcks;
-    erts_lc_locked_lock_t *l_lck;
+    lc_thread_t *thr;
+    lc_locked_lock_t *ll;
 
     if (lck->inited != ERTS_LC_INITITALIZED)
 	uninitialized_lock();
@@ -1171,38 +1171,38 @@ void erts_lc_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
     if (lck->id < 0)
 	return;
 
-    l_lcks = get_my_locked_locks();
+    thr = get_my_locked_locks();
 
-    if (l_lcks) {
-	l_lck = l_lcks->required.first;
-	if (find_lock(&l_lck, lck))
-	    unlock_of_required_lock(l_lcks, lck);
+    if (thr) {
+	ll = thr->required.first;
+	if (find_lock(&ll, lck))
+	    unlock_of_required_lock(thr, lck);
     }
 
-    for (l_lck = l_lcks ? l_lcks->locked.last : NULL; l_lck; l_lck = l_lck->prev) {
-	if (l_lck->id == lck->id && l_lck->extra == lck->extra) {
-	    if ((l_lck->taken_options & ERTS_LOCK_OPTIONS_RDWR) != options)
-		unlock_op_mismatch(l_lcks, lck, options);
-	    if (l_lck->prev)
-		l_lck->prev->next = l_lck->next;
+    for (ll = thr ? thr->locked.last : NULL; ll; ll = ll->prev) {
+	if (ll->id == lck->id && ll->extra == lck->extra) {
+	    if ((ll->taken_options & ERTS_LOCK_OPTIONS_RDWR) != options)
+		unlock_op_mismatch(thr, lck, options);
+	    if (ll->prev)
+		ll->prev->next = ll->next;
 	    else
-		l_lcks->locked.first = l_lck->next;
-	    if (l_lck->next)
-		l_lck->next->prev = l_lck->prev;
+		thr->locked.first = ll->next;
+	    if (ll->next)
+		ll->next->prev = ll->prev;
 	    else
-		l_lcks->locked.last = l_lck->prev;
-	    lc_free((void *) l_lck);
+		thr->locked.last = ll->prev;
+	    lc_free((void *) ll);
 	    return;
 	}
     }
     
-    unlock_of_not_locked(l_lcks, lck);
+    unlock_of_not_locked(thr, lck);
 }
 
 void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
 {
-    erts_lc_locked_locks_t *l_lcks;
-    erts_lc_locked_lock_t *l_lck;
+    lc_thread_t *thr;
+    lc_locked_lock_t *ll;
 
     if (lck->inited != ERTS_LC_INITITALIZED)
 	uninitialized_lock();
@@ -1210,17 +1210,17 @@ void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
     if (lck->id < 0)
 	return;
 
-    l_lcks = get_my_locked_locks();
+    thr = get_my_locked_locks();
 
-    if (l_lcks) {
-	l_lck = l_lcks->required.first;
-	if (find_lock(&l_lck, lck))
-	    unlock_of_required_lock(l_lcks, lck);
+    if (thr) {
+	ll = thr->required.first;
+	if (find_lock(&ll, lck))
+	    unlock_of_required_lock(thr, lck);
     }
 
-    l_lck = l_lcks->locked.first;
-    if (!find_lock(&l_lck, lck))
-	unlock_of_not_locked(l_lcks, lck);
+    ll = thr->locked.first;
+    if (!find_lock(&ll, lck))
+	unlock_of_not_locked(thr, lck);
 }
 
 int
@@ -1303,16 +1303,16 @@ erts_lc_init(void)
 {
 #ifdef ERTS_LC_STATIC_ALLOC
     int i;
-    static erts_lc_free_block_t fbs[ERTS_LC_FB_CHUNK_SIZE];
+    static lc_free_block_t fbs[ERTS_LC_FB_CHUNK_SIZE];
     for (i = 0; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) {
 #ifdef DEBUG
-	sys_memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t));
+	sys_memset((void *) &fbs[i], 0xdf, sizeof(lc_free_block_t));
 #endif
 	fbs[i].next = &fbs[i+1];
     }
 #ifdef DEBUG
     sys_memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1],
-	   0xdf, sizeof(erts_lc_free_block_t));
+	   0xdf, sizeof(lc_free_block_t));
 #endif
     fbs[ERTS_LC_FB_CHUNK_SIZE-1].next = NULL;
     free_blocks = &fbs[0]; 
@@ -1342,5 +1342,4 @@ erts_lc_pll(void)
     print_curr_locks(get_my_locked_locks());
 }
 
-
 #endif /* #ifdef ERTS_ENABLE_LOCK_CHECK */
-- 
cgit v1.2.3


From bf0b86ea8e71d05331ef36c472f4dee646b22c38 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 12 Apr 2018 11:54:47 +0200
Subject: erts: Add erts_debug:lc_graph/0

Run debug VM or config with --enable-lock-checking.

Exercise VM and then run

erts_debug:lc_graph().

to create a file "lc_graph.<pid>" in current working directory.
---
 erts/emulator/beam/erl_bif_info.c   |   9 +++
 erts/emulator/beam/erl_lock_check.c | 111 +++++++++++++++++++++++++++++++++++-
 erts/emulator/beam/erl_lock_check.h |   2 +
 3 files changed, 120 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index bdca93428e..874ac0f9e6 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3928,6 +3928,15 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
             BIF_RET(am_false);
 #endif
         }
+        else if (ERTS_IS_ATOM_STR("lc_graph", BIF_ARG_1)) {
+#ifdef ERTS_ENABLE_LOCK_CHECK
+            Eterm res = erts_lc_dump_graph();
+            BIF_RET(res);
+#else
+            BIF_RET(am_notsup);
+#endif
+        }
+
     }
     else if (is_tuple(BIF_ARG_1)) {
 	Eterm* tp = tuple_val(BIF_ARG_1);
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 5d33de9e88..53edf6abc1 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -41,6 +41,7 @@
 #include "erl_lock_check.h"
 #include "erl_term.h"
 #include "erl_threads.h"
+#include "erl_atom_table.h"
 
 typedef struct {
     char *name;
@@ -75,6 +76,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
      *						 if only one lock use
      *						 the lock name)"
      */
+    {   "NO LOCK",                              NULL                    },
     {	"driver_lock",				"driver_name"		},
     {	"port_lock",				"port_id"		},
     {	"port_data_lock",			"address"		},
@@ -196,6 +198,19 @@ typedef struct {
     lc_locked_lock_t *last;
 } lc_locked_lock_list_t;
 
+typedef struct {
+    /*
+     * m[X][Y] & 1 if we locked X directly after Y was locked.
+     * m[X][Y] & 2 if we locked X indirectly after Y was locked.
+     * m[X][0] = 1 if we locked X when nothing else was locked.
+     * m[0][] is unused as it would represent locking "NO LOCK"
+     */
+    char m[ERTS_LOCK_ORDER_SIZE][ERTS_LOCK_ORDER_SIZE];
+
+} lc_matrix_t;
+
+static lc_matrix_t tot_lc_matrix;
+
 typedef struct lc_thread_t_ lc_thread_t;
 struct lc_thread_t_ {
     char *thread_name;
@@ -205,6 +220,7 @@ struct lc_thread_t_ {
     lc_thread_t *prev;
     lc_locked_lock_list_t locked;
     lc_locked_lock_list_t required;
+    lc_matrix_t matrix;
 };
 
 typedef union lc_free_block_t_ lc_free_block_t;
@@ -322,6 +338,7 @@ create_thread_data(char *thread_name)
     thr->locked.first = NULL;
     thr->locked.last = NULL;
     thr->prev = NULL;
+    sys_memzero(&thr->matrix, sizeof(thr->matrix));
 
     lc_lock();
     thr->next = lc_threads;
@@ -333,6 +350,8 @@ create_thread_data(char *thread_name)
     return thr;
 }
 
+static void collect_matrix(lc_matrix_t*);
+
 static void
 destroy_locked_locks(lc_thread_t *thr)
 {
@@ -352,6 +371,9 @@ destroy_locked_locks(lc_thread_t *thr)
     }
     if (thr->next)
 	thr->next->prev = thr->prev;
+
+    collect_matrix(&thr->matrix);
+
     lc_unlock();
 
     free((void *) thr);
@@ -394,7 +416,7 @@ static void
 raw_print_lock(char *prefix, Sint16 id, Wterm extra, erts_lock_flags_t flags,
 	       char* file, unsigned int line, char *suffix)
 {
-    char *lname = (0 <= id && id < ERTS_LOCK_ORDER_SIZE
+    char *lname = (1 <= id && id < ERTS_LOCK_ORDER_SIZE
 		   ? erts_lock_order[id].name
 		   : "unknown");
     erts_fprintf(stderr,"%s'%s:",prefix,lname);
@@ -1144,12 +1166,25 @@ void erts_lc_lock_flg_x(erts_lc_lock_t *lck, erts_lock_options_t options,
     if (!thr->locked.last) {
 	ASSERT(!thr->locked.first);
 	thr->locked.last = thr->locked.first = new_ll;
+        ASSERT(0 < lck->id && lck->id < ERTS_LOCK_ORDER_SIZE);
+        thr->matrix.m[lck->id][0] = 1;
     }
     else if (thr->locked.last->id < lck->id
 	     || (thr->locked.last->id == lck->id
 		 && thr->locked.last->extra < lck->extra)) {
-	if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, thr->locked.last->flags))
+        lc_locked_lock_t* ll;
+	if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, thr->locked.last->flags)) {
 	    type_order_violation("locking ", thr, lck);
+        }
+
+        ASSERT(0 < lck->id && lck->id < ERTS_LOCK_ORDER_SIZE);
+        ll = thr->locked.last;
+        thr->matrix.m[lck->id][ll->id] |= 1;
+        for (ll = ll->prev; ll; ll = ll->prev) {
+            ASSERT(0 < ll->id && ll->id < ERTS_LOCK_ORDER_SIZE);
+            thr->matrix.m[lck->id][ll->id] |= 2;
+        }
+
 	new_ll->prev = thr->locked.last;
 	thr->locked.last->next = new_ll;
 	thr->locked.last = new_ll;
@@ -1342,4 +1377,76 @@ erts_lc_pll(void)
     print_curr_locks(get_my_locked_locks());
 }
 
+static void collect_matrix(lc_matrix_t* matrix)
+{
+    int i, j;
+    for (i = 1; i < ERTS_LOCK_ORDER_SIZE; i++) {
+        for (j = 0; j <= i; j++) {
+            tot_lc_matrix.m[i][j] |= matrix->m[i][j];
+        }
+#ifdef DEBUG
+        for ( ; j < ERTS_LOCK_ORDER_SIZE; j++) {
+            ASSERT(matrix->m[i][j] == 0);
+        }
+#endif
+    }
+}
+
+Eterm
+erts_lc_dump_graph(void)
+{
+    const char* basename = "lc_graph.";
+    char filename[40];
+    lc_matrix_t* tot = &tot_lc_matrix;
+    lc_thread_t* thr;
+    int i, j, name_max = 0;
+    FILE* ff;
+
+    lc_lock();
+    for (thr = lc_threads; thr; thr = thr->next) {
+        collect_matrix(&thr->matrix);
+    }
+    lc_unlock();
+
+    sys_strcpy(filename, basename);
+    sys_get_pid(filename + strlen(basename),
+                sizeof(filename) - strlen(basename));
+    ff = fopen(filename, "w");
+    if (!ff)
+        return am_error;
+
+    for (i = 1; i < ERTS_LOCK_ORDER_SIZE; i++) {
+        int len = strlen(erts_lock_order[i].name);
+        if (name_max < len)
+            name_max = len;
+    }
+    fputs("%This file was generated by erts_debug:lc_graph()\n\n", ff);
+    fputs("%{ThisLockName, ThisLockId, LockedDirectlyBeforeThis, LockedIndirectlyBeforeThis}\n", ff);
+    fprintf(ff, "[{%*s, %2d}", name_max, "\"NO LOCK\"", 0);
+    for (i = 1; i < ERTS_LOCK_ORDER_SIZE; i++) {
+        char* delim = "";
+        fprintf(ff, ",\n {%*s, %2d, [", name_max, erts_lock_order[i].name, i);
+        for (j = 0; j < ERTS_LOCK_ORDER_SIZE; j++) {
+            if (tot->m[i][j] & 1) {
+                fprintf(ff, "%s%d", delim, j);
+                delim = ",";
+            }
+        }
+        fprintf(ff, "], [");
+        delim = "";
+        for (j = 0; j < ERTS_LOCK_ORDER_SIZE; j++) {
+            if (tot->m[i][j] == 2) {
+                fprintf(ff, "%s%d", delim, j);
+                delim = ",";
+            }
+        }
+        fputs("]}", ff);
+    }
+    fputs("].", ff);
+    fclose(ff);
+    erts_fprintf(stderr, "Created file '%s' in current working directory\n",
+                 filename);
+    return am_ok;
+}
+
 #endif /* #ifdef ERTS_ENABLE_LOCK_CHECK */
diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h
index 5c2c38e8f2..138bc810bd 100644
--- a/erts/emulator/beam/erl_lock_check.h
+++ b/erts/emulator/beam/erl_lock_check.h
@@ -94,6 +94,8 @@ void erts_lc_unrequire_lock(erts_lc_lock_t *lck);
 
 int erts_lc_is_emu_thr(void);
 
+Eterm erts_lc_dump_graph(void);
+
 #define ERTS_LC_ASSERT(A) \
     ((void) (((A) || ERTS_SOMEONE_IS_CRASH_DUMPING) ? 1 : erts_lc_assert_failed(__FILE__, __LINE__, #A)))
 #else /* #ifdef ERTS_ENABLE_LOCK_CHECK */
-- 
cgit v1.2.3


From 865ac3b740d9efa1a0583349929591c757300412 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 12 Apr 2018 20:15:33 +0200
Subject: erts: Make locked checker allocations thread specific

Also removed unused ERTS_LC_STATIC_ALLOC.
---
 erts/emulator/beam/erl_lock_check.c | 117 +++++++++++++-----------------------
 1 file changed, 41 insertions(+), 76 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 53edf6abc1..d66410367b 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -198,6 +198,12 @@ typedef struct {
     lc_locked_lock_t *last;
 } lc_locked_lock_list_t;
 
+typedef union lc_free_block_t_ lc_free_block_t;
+union lc_free_block_t_ {
+    lc_free_block_t *next;
+    lc_locked_lock_t lock;
+};
+
 typedef struct {
     /*
      * m[X][Y] & 1 if we locked X directly after Y was locked.
@@ -220,20 +226,15 @@ struct lc_thread_t_ {
     lc_thread_t *prev;
     lc_locked_lock_list_t locked;
     lc_locked_lock_list_t required;
+    lc_free_block_t *free_blocks;
     lc_matrix_t matrix;
 };
 
-typedef union lc_free_block_t_ lc_free_block_t;
-union lc_free_block_t_ {
-    lc_free_block_t *next;
-    lc_locked_lock_t lock;
-};
-
 static ethr_tsd_key locks_key;
 
 static lc_thread_t *lc_threads = NULL;
+static ethr_spinlock_t lc_threads_lock;
 
-static lc_free_block_t *free_blocks = NULL;
 
 #ifdef ERTS_LC_STATIC_ALLOC
 #define ERTS_LC_FB_CHUNK_SIZE 10000
@@ -241,47 +242,33 @@ static lc_free_block_t *free_blocks = NULL;
 #define ERTS_LC_FB_CHUNK_SIZE 10
 #endif
 
-static ethr_spinlock_t free_blocks_lock;
 
 static ERTS_INLINE void
-lc_lock(void)
+lc_lock_threads(void)
 {
-    ethr_spin_lock(&free_blocks_lock);
+    ethr_spin_lock(&lc_threads_lock);
 }
 
 static ERTS_INLINE void
-lc_unlock(void)
+lc_unlock_threads(void)
 {
-    ethr_spin_unlock(&free_blocks_lock);
+    ethr_spin_unlock(&lc_threads_lock);
 }
 
-static ERTS_INLINE void lc_free(void *p)
+static ERTS_INLINE void lc_free(lc_thread_t* thr, lc_locked_lock_t *p)
 {
     lc_free_block_t *fb = (lc_free_block_t *) p;
 #ifdef DEBUG
     sys_memset((void *) p, 0xdf, sizeof(lc_free_block_t));
 #endif
-    lc_lock();
-    fb->next = free_blocks;
-    free_blocks = fb;
-    lc_unlock();   
+    fb->next = thr->free_blocks;
+    thr->free_blocks = fb;
 }
 
-#ifdef ERTS_LC_STATIC_ALLOC
-
-static void *lc_core_alloc(void)
-{
-    lc_unlock();
-    ERTS_INTERNAL_ERROR("Lock checker out of memory!\n");
-}
-
-#else
-
-static void *lc_core_alloc(void)
+static lc_locked_lock_t *lc_core_alloc(lc_thread_t* thr)
 {
     int i;
     lc_free_block_t *fbs;
-    lc_unlock();
     fbs = (lc_free_block_t *) malloc(sizeof(lc_free_block_t)
 					  * ERTS_LC_FB_CHUNK_SIZE);
     if (!fbs) {
@@ -297,25 +284,20 @@ static void *lc_core_alloc(void)
     sys_memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1],
 	   0xdf, sizeof(lc_free_block_t));
 #endif
-    lc_lock();
-    fbs[ERTS_LC_FB_CHUNK_SIZE-1].next = free_blocks;
-    free_blocks = &fbs[1];
-    return (void *) &fbs[0];
+    fbs[ERTS_LC_FB_CHUNK_SIZE-1].next = thr->free_blocks;
+    thr->free_blocks = &fbs[1];
+    return &fbs[0].lock;
 }
 
-#endif
-
-static ERTS_INLINE void *lc_alloc(void)
+static ERTS_INLINE lc_locked_lock_t *lc_alloc(lc_thread_t* thr)
 {
-    void *res;
-    lc_lock();
-    if (!free_blocks)
-	res = lc_core_alloc();
+    lc_locked_lock_t *res;
+    if (!thr->free_blocks)
+	res = lc_core_alloc(thr);
     else {
-	res = (void *) free_blocks;
-	free_blocks = free_blocks->next;
+	res = &thr->free_blocks->lock;
+	thr->free_blocks = thr->free_blocks->next;
     }
-    lc_unlock();
     return res;
 }
 
@@ -338,14 +320,15 @@ create_thread_data(char *thread_name)
     thr->locked.first = NULL;
     thr->locked.last = NULL;
     thr->prev = NULL;
+    thr->free_blocks = NULL;
     sys_memzero(&thr->matrix, sizeof(thr->matrix));
 
-    lc_lock();
+    lc_lock_threads();
     thr->next = lc_threads;
     if (lc_threads)
 	lc_threads->prev = thr;
     lc_threads = thr;
-    lc_unlock();
+    lc_unlock_threads();
     erts_tsd_set(locks_key, (void *) thr);
     return thr;
 }
@@ -362,7 +345,7 @@ destroy_locked_locks(lc_thread_t *thr)
     ASSERT(thr->locked.first == NULL);
     ASSERT(thr->locked.last == NULL);
 
-    lc_lock();
+    lc_lock_threads();
     if (thr->prev)
 	thr->prev->next = thr->next;
     else {
@@ -374,7 +357,7 @@ destroy_locked_locks(lc_thread_t *thr)
 
     collect_matrix(&thr->matrix);
 
-    lc_unlock();
+    lc_unlock_threads();
 
     free((void *) thr);
 
@@ -397,10 +380,11 @@ make_my_locked_locks(void)
 }
 
 static ERTS_INLINE lc_locked_lock_t *
-new_locked_lock(erts_lc_lock_t *lck, erts_lock_options_t options,
+new_locked_lock(lc_thread_t* thr,
+                erts_lc_lock_t *lck, erts_lock_options_t options,
 		char *file, unsigned int line)
 {
-    lc_locked_lock_t *ll = (lc_locked_lock_t *) lc_alloc();
+    lc_locked_lock_t *ll = lc_alloc(thr);
     ll->next = NULL;
     ll->prev = NULL;
     ll->id = lck->id;
@@ -1032,7 +1016,7 @@ void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, erts_lock_options_t
 	return;
 
     thr = make_my_locked_locks();
-    ll = locked ? new_locked_lock(lck, options, file, line) : NULL;
+    ll = locked ? new_locked_lock(thr, lck, options, file, line) : NULL;
 
     if (!thr->locked.last) {
 	ASSERT(!thr->locked.first);
@@ -1080,7 +1064,7 @@ void erts_lc_require_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options,
     lc_locked_lock_t *ll = thr->locked.first;
     if (!find_lock(&ll, lck))
 	required_not_locked(thr, lck);
-    ll = new_locked_lock(lck, options, file, line);
+    ll = new_locked_lock(thr, lck, options, file, line);
     if (!thr->required.last) {
 	ASSERT(!thr->required.first);
 	ll->next = ll->prev = NULL;
@@ -1145,7 +1129,7 @@ void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options
 	ASSERT(thr->required.last == ll);
 	thr->required.last = ll->prev;
     }
-    lc_free((void *) ll);
+    lc_free(thr, ll);
 }
 
 void erts_lc_lock_flg_x(erts_lc_lock_t *lck, erts_lock_options_t options,
@@ -1161,7 +1145,7 @@ void erts_lc_lock_flg_x(erts_lc_lock_t *lck, erts_lock_options_t options,
 	return;
 
     thr = make_my_locked_locks();
-    new_ll = new_locked_lock(lck, options, file, line);
+    new_ll = new_locked_lock(thr, lck, options, file, line);
 
     if (!thr->locked.last) {
 	ASSERT(!thr->locked.first);
@@ -1226,7 +1210,7 @@ void erts_lc_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
 		ll->next->prev = ll->prev;
 	    else
 		thr->locked.last = ll->prev;
-	    lc_free((void *) ll);
+	    lc_free(thr, ll);
 	    return;
 	}
     }
@@ -1336,26 +1320,7 @@ erts_lc_destroy_lock(erts_lc_lock_t *lck)
 void
 erts_lc_init(void)
 {
-#ifdef ERTS_LC_STATIC_ALLOC
-    int i;
-    static lc_free_block_t fbs[ERTS_LC_FB_CHUNK_SIZE];
-    for (i = 0; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) {
-#ifdef DEBUG
-	sys_memset((void *) &fbs[i], 0xdf, sizeof(lc_free_block_t));
-#endif
-	fbs[i].next = &fbs[i+1];
-    }
-#ifdef DEBUG
-    sys_memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1],
-	   0xdf, sizeof(lc_free_block_t));
-#endif
-    fbs[ERTS_LC_FB_CHUNK_SIZE-1].next = NULL;
-    free_blocks = &fbs[0]; 
-#else /* #ifdef ERTS_LC_STATIC_ALLOC */
-    free_blocks = NULL;
-#endif /* #ifdef ERTS_LC_STATIC_ALLOC */
-
-    if (ethr_spinlock_init(&free_blocks_lock) != 0)
+    if (ethr_spinlock_init(&lc_threads_lock) != 0)
 	ERTS_INTERNAL_ERROR("spinlock_init failed");
 
     erts_tsd_key_create(&locks_key,"erts_lock_check_key");
@@ -1402,11 +1367,11 @@ erts_lc_dump_graph(void)
     int i, j, name_max = 0;
     FILE* ff;
 
-    lc_lock();
+    lc_lock_threads();
     for (thr = lc_threads; thr; thr = thr->next) {
         collect_matrix(&thr->matrix);
     }
-    lc_unlock();
+    lc_unlock_threads();
 
     sys_strcpy(filename, basename);
     sys_get_pid(filename + strlen(basename),
-- 
cgit v1.2.3


From 6915407d64fca30896929bbc717408595a63d45b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 13 Apr 2018 14:47:55 +0200
Subject: Add zzz_SUITE with erts_debug:lc_graph last in tests

for erts, stdlib, kernel and runtime_tools.
---
 erts/emulator/test/z_SUITE.erl | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl
index ac3df8bfbf..103f9f1550 100644
--- a/erts/emulator/test/z_SUITE.erl
+++ b/erts/emulator/test/z_SUITE.erl
@@ -37,6 +37,7 @@
 -export([schedulers_alive/1, node_container_refc_check/1,
 	 long_timers/1, pollset_size/1,
 	 check_io_debug/1, get_check_io_info/0,
+         lc_graph/1,
          leaked_processes/1]).
 
 suite() ->
@@ -46,6 +47,7 @@ suite() ->
 all() -> 
     [schedulers_alive, node_container_refc_check,
      long_timers, pollset_size, check_io_debug,
+     lc_graph,
      %% Make sure that the leaked_processes/1 is always
      %% run last.
      leaked_processes].
@@ -289,6 +291,12 @@ has_gethost([P|T]) ->
 has_gethost([]) ->
     false.
 
+lc_graph(Config) when is_list(Config) ->
+    %% Create "lc_graph" file in current working dir
+    %% if lock checker is enabled
+    erts_debug:lc_graph(),
+    ok.
+
 leaked_processes(Config) when is_list(Config) ->
     %% Replace the defualt timetrap with a timetrap with
     %% known pid.
-- 
cgit v1.2.3


From fd3ea59cd3fcd388b6a9c9a1be25a671c0a5a2e3 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 6 Apr 2018 16:38:40 +0200
Subject: Fix seq_trace erl_tracer bug

---
 erts/emulator/beam/erl_trace.c      |  2 +-
 erts/emulator/test/tracer_SUITE.erl | 24 ++++++++++++++++++++++--
 2 files changed, 23 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 018894f685..1e833539b3 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -2593,7 +2593,7 @@ erts_term_to_tracer(Eterm prefix, Eterm t)
                     state = tp[3];
                 }
             } else {
-                if (arityval(tp[0]) == 2 && is_atom(tp[2])) {
+                if (arityval(tp[0]) == 2 && is_atom(tp[1])) {
                     module = tp[1];
                     state = tp[2];
                 }
diff --git a/erts/emulator/test/tracer_SUITE.erl b/erts/emulator/test/tracer_SUITE.erl
index ab7d047bc3..e1362ef07a 100644
--- a/erts/emulator/test/tracer_SUITE.erl
+++ b/erts/emulator/test/tracer_SUITE.erl
@@ -30,7 +30,8 @@
 -export([load/1, unload/1, reload/1, invalid_tracers/1]).
 -export([send/1, recv/1, call/1, call_return/1, spawn/1, exit/1,
          link/1, unlink/1, getting_linked/1, getting_unlinked/1,
-         register/1, unregister/1, in/1, out/1, gc_start/1, gc_end/1]).
+         register/1, unregister/1, in/1, out/1, gc_start/1, gc_end/1,
+         seq_trace/1]).
 
 suite() -> [{ct_hooks,[ts_install_cth]},
             {timetrap, {minutes, 1}}].
@@ -41,7 +42,8 @@ all() ->
 groups() ->
     [{ basic, [], [send, recv, call, call_return, spawn, exit,
                    link, unlink, getting_linked, getting_unlinked,
-                   register, unregister, in, out, gc_start, gc_end]}].
+                   register, unregister, in, out, gc_start, gc_end,
+                   seq_trace]}].
 
 init_per_suite(Config) ->
     erlang:trace_pattern({'_','_','_'}, false, [local]),
@@ -583,6 +585,24 @@ gc_end(_Config) ->
 
     test(gc_major_end, garbage_collection, Tc, Expect, false).
 
+seq_trace(_Config) ->
+
+    seq_trace:set_system_tracer({tracer_test,
+                                 {#{ seq_trace => trace }, self(), []}}),
+    erlang:spawn(fun() ->
+                         seq_trace:set_token(label,17),
+                         seq_trace:set_token(print,true),
+                         seq_trace:print(17,"**** Trace Started ****")
+                 end),
+    receive
+        {seq_trace, _, 17, {print, _, _, _, _}, _} ->
+            ok;
+        M ->
+            ct:fail("~p~n",[M])
+    after 100 ->
+            ct:fail(timeout)
+    end.
+
 test(Event, Tc, Expect) ->
     test(Event, Tc, Expect, false).
 test(Event, Tc, Expect, Removes) ->
-- 
cgit v1.2.3


From 283a7d098538be2097a5d4d6e75422ca14e7e4e3 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 16 Apr 2018 12:29:58 +0200
Subject: Fix deadlock in HiPE gc after receive

---
 erts/emulator/beam/erl_gc.c | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index ea87cd7f50..b498fd9cf9 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -438,6 +438,13 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end,
 	return result;
     }
 
+#ifdef HIPE
+    if (p->hipe_smp.have_receive_locks) {
+        /* Do not want to GC with message queue locked... */
+        return result;
+    }    
+#endif
+
     if (!p->mbuf) {
 	/* Must have GC:d in BIF call... invalidate live_hf_end */
 	live_hf_end = ERTS_INVALID_HFRAG_PTR;
-- 
cgit v1.2.3


From c71d914fa41f3b272dc80fefc33c1bb40ca86652 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 10 Apr 2018 11:42:06 +0200
Subject: erts: Update esdp->current_port on immediate port calls

This increases the accuracy of crash dumps and the upcoming
allocation tagging feature.
---
 erts/emulator/beam/io.c | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 66b8005975..10075c2c98 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -956,6 +956,9 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp)
 
 	reds_left_in = ERTS_BIF_REDS_LEFT(c_p);
 	erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+
+        ASSERT((c_p->scheduler_data)->current_port == NULL);
+        (c_p->scheduler_data)->current_port = prt;
     }
 
     ASSERT(0 <= reds_left_in && reds_left_in <= CONTEXT_REDS);
@@ -1017,6 +1020,9 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp)
     erts_port_release(prt);
 
     if (c_p) {
+        ASSERT((c_p->scheduler_data)->current_port == prt);
+        (c_p->scheduler_data)->current_port = NULL;
+
 	erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 
 	if (reds != (CONTEXT_REDS - sp->reds_left_in)) {
-- 
cgit v1.2.3


From 9128efbb8dadc819938820af0d9de9128e72eb07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 10 Apr 2018 11:44:35 +0200
Subject: erts: Always keep a copy of driver names as an atom

---
 erts/emulator/beam/global.h |  1 +
 erts/emulator/beam/io.c     | 32 ++++++++++++--------------------
 2 files changed, 13 insertions(+), 20 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 256670ff22..f4fea86cdc 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -200,6 +200,7 @@ typedef struct {
 struct erts_driver_t_ {
     erts_driver_t *next;
     erts_driver_t *prev;
+    Eterm name_atom;
     char *name;
     struct {
 	int major;
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 10075c2c98..2446b3c074 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -2912,11 +2912,8 @@ static void lcnt_enable_driver_lock_count(erts_driver_t *dp, int enable)
 {
     if (dp->lock) {
         if (enable) {
-            Eterm name_as_atom = erts_atom_put((byte*)dp->name, sys_strlen(dp->name),
-                                                ERTS_ATOM_ENC_LATIN1, 1);
-
-            erts_lcnt_install_new_lock_info(&dp->lock->lcnt, "driver_lock", name_as_atom,
-                ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO);
+            erts_lcnt_install_new_lock_info(&dp->lock->lcnt, "driver_lock",
+                dp->name_atom, ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO);
         } else {
             erts_lcnt_uninstall(&dp->lock->lcnt);
         }
@@ -7363,7 +7360,11 @@ no_stop_select_callback(ErlDrvEvent event, void* private)
 static int
 init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
 {
+    drv->name_atom = erts_atom_put((byte*)de->driver_name,
+                                   sys_strlen(de->driver_name),
+                                   ERTS_ATOM_ENC_LATIN1, 1);
     drv->name = de->driver_name;
+
     ASSERT(de->extended_marker == ERL_DRV_EXTENDED_MARKER);
     ASSERT(de->major_version >= 2);
     drv->version.major = de->major_version;
@@ -7373,13 +7374,10 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
     if (drv->flags & ERL_DRV_FLAG_USE_PORT_LOCKING) {
         drv->lock = NULL;
     } else {
-        Eterm driver_id = erts_atom_put((byte *) drv->name,
-                                        sys_strlen(drv->name),
-                                        ERTS_ATOM_ENC_LATIN1, 1);
-
         drv->lock = erts_alloc(ERTS_ALC_T_DRIVER_LOCK, sizeof(erts_mtx_t));
 
-        erts_mtx_init(drv->lock, "driver_lock", driver_id, ERTS_LOCK_FLAGS_CATEGORY_IO);
+        erts_mtx_init(drv->lock, "driver_lock", drv->name_atom,
+                      ERTS_LOCK_FLAGS_CATEGORY_IO);
     }
     drv->entry = de;
 
@@ -7465,18 +7463,12 @@ int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle,
 	erts_tsd_set(driver_list_lock_status_key, (void *) 1);
     }
 
-    if (taint) {
-        Eterm name_atom = erts_atom_put((byte*)de->driver_name,
-                                        sys_strlen(de->driver_name),
-                                        ERTS_ATOM_ENC_LATIN1, 0);
-        if (is_atom(name_atom))
-            erts_add_taint(name_atom);
-        else
-            err = 1;
-    }
-
     if (!err) {
         err = init_driver(dp, de, handle);
+
+        if (taint) {
+            erts_add_taint(dp->name_atom);
+        }
     }
 
     if (err) {
-- 
cgit v1.2.3


From a7e0369000282c22e784a769b0e41598df1682e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 10 Apr 2018 11:50:56 +0200
Subject: erts: Keep track of which NIF a scheduler is executing

This may be of interest in crash dumps and allows the upcoming
allocation tagging feature to track allocations on a per-NIF basis.

Note that this is only updated when user code calls a NIF; it's not
altered when the emulator calls NIFs during code upgrades or
tracing.
---
 erts/emulator/beam/bif_instrs.tab |  8 ++++++++
 erts/emulator/beam/erl_nif.c      | 10 ++++++++++
 erts/emulator/beam/erl_process.c  |  1 +
 erts/emulator/beam/erl_process.h  |  1 +
 erts/emulator/beam/global.h       |  1 +
 5 files changed, 21 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif_instrs.tab b/erts/emulator/beam/bif_instrs.tab
index 0932b8b985..0f074280db 100644
--- a/erts/emulator/beam/bif_instrs.tab
+++ b/erts/emulator/beam/bif_instrs.tab
@@ -432,9 +432,17 @@ nif_bif.call_nif() {
         live_hf_end = c_p->mbuf;
         ERTS_CHK_MBUF_SZ(c_p);
         erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
+
+        ASSERT((c_p->scheduler_data)->current_nif == NULL);
+        (c_p->scheduler_data)->current_nif = &env;
+
         nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
         if (env.exception_thrown)
             nif_bif_result = THE_NON_VALUE;
+
+        ASSERT((c_p->scheduler_data)->current_nif == &env);
+        (c_p->scheduler_data)->current_nif = NULL;
+
         erts_post_nif(&env);
         ERTS_CHK_MBUF_SZ(c_p);
 
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 0e0013e8a4..260656e2d3 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -388,12 +388,18 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *
     erts_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC
 						   | ERTS_PSFLG_DIRTY_IO_PROC));
 
+    ASSERT(esdp->current_nif == NULL);
+    esdp->current_nif = &env;
+
     erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
 
     result = (*dirty_nif)(&env, codemfa->arity, argv); /* Call dirty NIF */
 
     erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 
+    ASSERT(esdp->current_nif == &env);
+    esdp->current_nif = NULL;
+
     ASSERT(env.proc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
     ASSERT(env.proc->next == c_p);
 
@@ -4258,6 +4264,10 @@ int erts_nif_get_funcs(struct erl_module_nif* mod,
     return mod->entry.num_of_funcs;
 }
 
+Module *erts_nif_get_module(struct erl_module_nif *nif_mod) {
+    return nif_mod->mod;
+}
+
 Eterm erts_nif_call_function(Process *p, Process *tracee,
                              struct erl_module_nif* mod,
                              ErlNifFunc *fun, int argc, Eterm *argv)
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 71541786d0..47b46fdd75 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -5698,6 +5698,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     esdp->ssi = ssi;
     esdp->current_process = NULL;
     esdp->current_port = NULL;
+    esdp->current_nif = NULL;
 
     esdp->virtual_reds = 0;
     esdp->cpu_id = -1;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index e232776016..bb97b6690c 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -637,6 +637,7 @@ struct ErtsSchedulerData_ {
     ErtsSchedType type;
     Uint no;			/* Scheduler number for normal schedulers */
     Uint dirty_no;  /* Scheduler number for dirty schedulers */
+    struct enif_environment_t *current_nif;
     Process *dirty_shadow_process;
     Port *current_port;
     ErtsRunQueue *run_queue;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index f4fea86cdc..2cf268162d 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -123,6 +123,7 @@ void erts_unload_nif(struct erl_module_nif* nif);
 extern void erl_nif_init(void);
 extern int erts_nif_get_funcs(struct erl_module_nif*,
                               struct enif_func_t **funcs);
+extern Module *erts_nif_get_module(struct erl_module_nif*);
 extern Eterm erts_nif_call_function(Process *p, Process *tracee,
                                     struct erl_module_nif*,
                                     struct enif_func_t *,
-- 
cgit v1.2.3


From 6d281624128b5877283df2e91e2c116c3a142f16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 16 Apr 2018 17:54:57 +0200
Subject: Disregard locks that can't be toggled in lcnt_SUITE

---
 erts/emulator/test/lcnt_SUITE.erl | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/lcnt_SUITE.erl b/erts/emulator/test/lcnt_SUITE.erl
index 4e52c2813c..dfffd662e2 100644
--- a/erts/emulator/test/lcnt_SUITE.erl
+++ b/erts/emulator/test/lcnt_SUITE.erl
@@ -87,8 +87,9 @@ wait_for_empty_lock_list() ->
     wait_for_empty_lock_list(10).
 wait_for_empty_lock_list(Tries) when Tries > 0 ->
     try_flush_cleanup_ops(),
-    case erts_debug:lcnt_collect() of
-        [{duration, _}, {locks, []}] ->
+    [{duration, _}, {locks, Locks}] = erts_debug:lcnt_collect(),
+    case remove_untoggleable_locks(Locks) of
+        [] ->
             ok;
         _ ->
             timer:sleep(50),
@@ -124,7 +125,7 @@ toggle_lock_counting(Config) when is_list(Config) ->
 get_lock_info_for(Categories) when is_list(Categories) ->
     ok = erts_debug:lcnt_control(mask, Categories),
     [{duration, _}, {locks, Locks}] = erts_debug:lcnt_collect(),
-    Locks;
+    remove_untoggleable_locks(Locks);
 
 get_lock_info_for(Category) when is_atom(Category) ->
     get_lock_info_for([Category]).
@@ -178,3 +179,13 @@ registered_db_tables(Config) when is_list(Config) ->
             (_Lock) -> false
         end, DbLocks),
     ok.
+
+%% Not all locks can be toggled on or off due to technical limitations, so we
+%% need to filter them out when checking whether we successfully disabled lock
+%% counting.
+remove_untoggleable_locks([]) ->
+    [];
+remove_untoggleable_locks([{resource_monitors, _, _, _} | T]) ->
+    remove_untoggleable_locks(T);
+remove_untoggleable_locks([H | T]) ->
+    [H | remove_untoggleable_locks(T)].
-- 
cgit v1.2.3


From 26d72d02167aed57e43f1ad669039b96aa154fb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 18 Apr 2018 07:30:40 +0200
Subject: erts: Fix rare deadlock in realloc when +ramv is enabled

OTP-15024
---
 erts/emulator/beam/erl_alloc_util.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index e148be7af6..fed51eb200 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -5805,7 +5805,7 @@ erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size)
     erts_mtx_lock(&allctr->mutex);
     res = do_erts_alcu_alloc(type, extra, size);
     if (!res)
-	res = erts_alcu_realloc_ts(type, extra, p, size);
+        res = do_erts_alcu_realloc(type, extra, p, size, 0, NULL);
     else {
 	Block_t *blk;
 	size_t cpy_size;
-- 
cgit v1.2.3


From b5d657f176a440176eb073f749baadb7c45e19e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 19 Apr 2018 11:24:38 +0200
Subject: Make stacktraces consistent when backtrace_depth is 0

It is allowed to set the backtrace depth to 0, but when
an exception is catched the stacktrace will still contain
one element:

    1> erlang:system_flag(backtrace_depth, 0).
    8
    2> catch error(badarg).
    {'EXIT',{badarg,[{shell,apply_fun,3,
                        [{file,"shell.erl"},{line,908}]}]}}

However, when an exception is raised using `erlang:raise/3`, there
will be no elements in the stacktrace:

    3> catch erlang:raise(error, badarg, [{fake,name,[arg],[]}]).
    {'EXIT',{badarg,[]}}

Since the `error_handler` module uses `erlang:raise/3` to
raise an exception when an undefined function is called,
there will not be any stacktrace when calling an undefined
function:

    4> catch undef_module:undef_name(some_argument).
    {'EXIT',{undef,[]}}

Fix this inconsistency by changing `erlang:raise/3` so that
it always includes one element in the stacktrace:

    3> catch erlang:raise(error, badarg, [{fake,name,[arg],[]}]).
    {'EXIT',{badarg,[{fake,name,[arg],[]}]}}
    4> catch undef_module:undef_name(some_argument).
    {'EXIT',{undef,[{undef_module,undef_name,[some_argument],[]}]}}
---
 erts/emulator/beam/bif.c               |  7 +++++
 erts/emulator/test/exception_SUITE.erl | 56 ++++++++++++++++++++++++++++++++--
 2 files changed, 61 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 017faffa48..79244b8544 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1150,6 +1150,13 @@ BIF_RETTYPE raise_3(BIF_ALIST_3)
     /* Create stacktrace and store */
     if (erts_backtrace_depth < depth) {
 	depth = erts_backtrace_depth;
+        if (depth == 0) {
+            /*
+             * For consistency with stacktraces generated
+             * automatically, always include one element.
+             */
+            depth = 1;
+        }
 	must_copy = 1;
     }
     if (must_copy) {
diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl
index da0292f385..60d14ce841 100644
--- a/erts/emulator/test/exception_SUITE.erl
+++ b/erts/emulator/test/exception_SUITE.erl
@@ -23,7 +23,8 @@
 -export([all/0, suite/0,
          badmatch/1, pending_errors/1, nil_arith/1, top_of_stacktrace/1,
          stacktrace/1, nested_stacktrace/1, raise/1, gunilla/1, per/1,
-         exception_with_heap_frag/1, line_numbers/1]).
+         exception_with_heap_frag/1, backtrace_depth/1,
+         line_numbers/1]).
 
 -export([bad_guy/2]).
 -export([crash/1]).
@@ -42,7 +43,7 @@ suite() ->
 all() -> 
     [badmatch, pending_errors, nil_arith, top_of_stacktrace,
      stacktrace, nested_stacktrace, raise, gunilla, per,
-     exception_with_heap_frag, line_numbers].
+     exception_with_heap_frag, backtrace_depth, line_numbers].
 
 -define(try_match(E),
         catch ?MODULE:bar(),
@@ -572,6 +573,57 @@ do_exception_with_heap_frag(Bin, [Sz|Sizes]) ->
     do_exception_with_heap_frag(Bin, Sizes);
 do_exception_with_heap_frag(_, []) -> ok.
 
+backtrace_depth(Config) when is_list(Config) ->
+    _ = [do_backtrace_depth(D) || D <- lists:seq(0, 8)],
+    ok.
+
+do_backtrace_depth(D) ->
+    Old = erlang:system_flag(backtrace_depth, D),
+    try
+        Expected = max(1, D),
+        do_backtrace_depth_1(Expected)
+    after
+        _ = erlang:system_flag(backtrace_depth, Old)
+    end.
+
+do_backtrace_depth_1(D) ->
+    Exit = fun() ->
+                   error(reason)
+           end,
+    HandCrafted = fun() ->
+                          {'EXIT',{_,Stk0}} = (catch error(get_stacktrace)),
+                          %% Fool the compiler to force a hand-crafted
+                          %% stacktrace.
+                          Stk = [hd(Stk0)|tl(Stk0)],
+                          erlang:raise(error, reason, Stk)
+                  end,
+    PassedOn = fun() ->
+                       try error(get_stacktrace)
+                       catch error:_:Stk ->
+                               %% Just pass on the given stacktrace.
+                               erlang:raise(error, reason, Stk)
+                       end
+               end,
+    do_backtrace_depth_2(D, Exit),
+    do_backtrace_depth_2(D, HandCrafted),
+    do_backtrace_depth_2(D, PassedOn),
+    ok.
+
+do_backtrace_depth_2(D, Exc) ->
+    try
+        Exc()
+    catch
+        error:reason:Stk ->
+            if
+                length(Stk) =/= D ->
+                    io:format("Expected depth: ~p\n", [D]),
+                    io:format("~p\n", [Stk]),
+                    error(bad_depth);
+                true ->
+                    ok
+            end
+    end.
+
 line_numbers(Config) when is_list(Config) ->
     {'EXIT',{{case_clause,bad_tag},
              [{?MODULE,line1,2,
-- 
cgit v1.2.3


From 540ab1efe49a1bfbcb47136e8ba553524ac55686 Mon Sep 17 00:00:00 2001
From: Serge Aleynikov <saleyn@gmail.com>
Date: Thu, 19 Apr 2018 10:56:52 -0400
Subject: Optimize performance of float_to_list/2

---
 erts/emulator/sys/common/erl_sys_common_misc.c | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c
index 826307c077..41a6fcb7e1 100644
--- a/erts/emulator/sys/common/erl_sys_common_misc.c
+++ b/erts/emulator/sys/common/erl_sys_common_misc.c
@@ -217,14 +217,16 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
         }
 
         do {
+            Uint64 n;
             if (!frac_part) {
                 do {
                     *p++ = '0';
                 } while (--decimals);
                 break;
             }
-            *p++ = (char)((frac_part % 10) + '0');
-            frac_part /= 10;
+            n = frac_part / 10;
+            *p++ = (char)((frac_part - n*10) + '0');
+            frac_part = n;
         } while (--decimals);
 
         *p++ = '.';
@@ -236,9 +238,10 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
         *p++ = '0';
     } else {
         do {
-            *p++ = (char)((int_part % 10) + '0');
-            int_part /= 10;
-        }while (int_part);
+            Uint64 n = int_part / 10;
+            *p++ = (char)((int_part - n*10) + '0');
+            int_part = n;
+        } while (int_part);
     }
     if (neg)
         *p++ = '-';
-- 
cgit v1.2.3


From aaf865a53a5be8d15afa5ff688cb3bdd672f4d18 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 20 Apr 2018 14:53:05 +0200
Subject: erts: Fix run queue pointer in proxy process

proc->run_queue detected as uninitialized by valgrind
but seems harmless in practice as it's not used for proxy processes.

"Bug" introduced in OTP-17 by ca0425c6ff85262bc15367f5fd9cbc51cde52b20
and made worse (but still harmless) in master for OTP-21
at fbb10ebc4a37555c7ea7f99e14286d862993976a.
---
 erts/emulator/beam/erl_process.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 71541786d0..e8f58a196a 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -6093,7 +6093,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
 	proxy = prev_proxy;
 	ASSERT(erts_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY);
 	erts_atomic32_set_nob(&proxy->state, state);
-        (void) erts_set_runq_proc(proc, rq, &bound);
+        (void) erts_set_runq_proc(proxy, rq, &bound);
     }
     else {
 	proxy = erts_alloc(ERTS_ALC_T_PROC, sizeof(Process));
@@ -6106,7 +6106,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
 	}
 #endif
 	erts_atomic32_init_nob(&proxy->state, state);
-        erts_init_runq_proc(proc, rq, bound);
+        erts_init_runq_proc(proxy, rq, bound);
     }
 
     proxy->common.id = proc->common.id;
-- 
cgit v1.2.3


From f7e016b010f87ce6808c15cbcefd95fe131826bc Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 7 Mar 2018 19:18:25 +0100
Subject: erts: Change wrong alloc_fnf

There is no NULL check here !?
---
 erts/emulator/beam/erl_db_hash.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 5d49b2ea14..9fed80c3a2 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -646,9 +646,9 @@ int db_create_hash(Process *p, DbTable *tbl)
 	    rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ;
 	if (erts_ets_rwmtx_spin_count >= 0)
 	    rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count;
-	tb->locks = (DbTableHashFineLocks*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, /* Other type maybe? */ 
-							      (DbTable *) tb,
-							      sizeof(DbTableHashFineLocks));	    	    
+	tb->locks = (DbTableHashFineLocks*) erts_db_alloc(ERTS_ALC_T_DB_SEG, /* Other type maybe? */
+                                                          (DbTable *) tb,
+                                                          sizeof(DbTableHashFineLocks));
 	for (i=0; i<DB_HASH_LOCK_CNT; ++i) {
             erts_rwmtx_init_opt(&tb->locks->lck_vec[i].lck, &rwmtx_opt,
                 "db_hash_slot", tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
-- 
cgit v1.2.3


From b609aef496c8881cd3357f6094354b828d36576a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 9 Mar 2018 18:05:47 +0100
Subject: erts: Refactor erl_db_hash next() into next_live()

where argument 'list' is the first candidate (not list->next).

Also simplified db_first_hash().
---
 erts/emulator/beam/erl_db_hash.c | 30 ++++++++++--------------------
 1 file changed, 10 insertions(+), 20 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 9fed80c3a2..014dc26414 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -340,8 +340,8 @@ typedef int (*extra_match_validator_t)(int keypos, Eterm match, Eterm guard, Ete
 static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix);
 static void alloc_seg(DbTableHash *tb);
 static int free_seg(DbTableHash *tb, int free_records);
-static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr,
-			HashDbTerm *list);
+static HashDbTerm* next_live(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr,
+			     HashDbTerm *list);
 static HashDbTerm* search_list(DbTableHash* tb, Eterm key, 
 			       HashValue hval, HashDbTerm *list);
 static void shrink(DbTableHash* tb, int nitems);
@@ -672,19 +672,9 @@ static int db_first_hash(Process *p, DbTable *tbl, Eterm *ret)
     erts_rwmtx_t* lck = RLOCK_HASH(tb,ix);
     HashDbTerm* list;
 
-    for (;;) {
-	list = BUCKET(tb,ix);
-	if (list != NULL) {
-	    if (list->hvalue == INVALID_HASH) {
-		list = next(tb,&ix,&lck,list);
-	    }
-	    break;
-	}
-	if ((ix=next_slot(tb,ix,&lck)) == 0) {
-	    list = NULL;
-	    break;
-	}
-    }
+    list = BUCKET(tb,ix);
+    list = next_live(tb, &ix, &lck, list);
+
     if (list != NULL) {
 	*ret = db_copy_key(p, tbl, &list->dbterm);
 	RUNLOCK_HASH(lck);
@@ -721,13 +711,13 @@ static int db_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
     }
     /* Key found */
 
-    b = next(tb, &ix, &lck, b);
+    b = next_live(tb, &ix, &lck, b->next);
     if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) {
 	while (b != 0) {
 	    if (!has_live_key(tb, b, key, hval)) {
 		break;
 	    }
-	    b = next(tb, &ix, &lck, b);
+	    b = next_live(tb, &ix, &lck, b->next);
 	}
     }
     if (b == NULL) {
@@ -2905,14 +2895,14 @@ static HashDbTerm* search_list(DbTableHash* tb, Eterm key,
 /* It return the next live object in a table, NULL if no more */
 /* In-bucket: RLOCKED */
 /* Out-bucket: RLOCKED unless NULL */
-static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr,
-			HashDbTerm *list)
+static HashDbTerm* next_live(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr,
+			     HashDbTerm *list)
 {
     int i;
 
     ERTS_LC_ASSERT(IS_HASH_RLOCKED(tb,*iptr));
 
-    for (list = list->next; list != NULL; list = list->next) {
+    for ( ; list != NULL; list = list->next) {
 	if (list->hvalue != INVALID_HASH)
 	    return list;        
     }
-- 
cgit v1.2.3


From 05ea5c89f2662e9bc042d3eac40e8a0c8b9f630f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 12 Mar 2018 19:41:53 +0100
Subject: erts: Use table ref for select continuation

and not the name. For more sane named table semantics.

Applies to both select/1 continuation and trap context.
---
 erts/emulator/beam/erl_db.c      |  7 +++++++
 erts/emulator/beam/erl_db.h      |  1 +
 erts/emulator/beam/erl_db_hash.c | 26 ++++++++++++++++++++------
 3 files changed, 28 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index a76d769283..f7ee408991 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -283,6 +283,13 @@ make_tid(Process *c_p, DbTable *tb)
     return erts_mk_magic_ref(&hp, &c_p->off_heap, tb->common.btid);
 }
 
+Eterm
+erts_db_make_tid(Process *c_p, DbTableCommon *tb)
+{
+    return make_tid(c_p, (DbTable*)tb);
+}
+
+
 
 /* 
 ** The meta hash table of all NAMED ets tables
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index 318e90cb28..eb6da2c9fb 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -128,6 +128,7 @@ extern erts_atomic_t erts_ets_misc_mem_size;
 
 Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt);
 Uint erts_db_get_max_tabs(void);
+Eterm erts_db_make_tid(Process *c_p, DbTableCommon *tb);
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
 void erts_lcnt_enable_db_lock_count(DbTable *tb, int enable);
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 014dc26414..cb5c496e90 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1453,20 +1453,24 @@ static ERTS_INLINE int on_mtraversal_simple_trap(Export* trap_function,
 
     BUMP_ALL_REDS(p);
     if (IS_USMALL(0, got)) {
-	hp = HAlloc(p,  base_halloc_sz + 5);
+	hp = HAllocX(p,  base_halloc_sz + 5, ERTS_MAGIC_REF_THING_SIZE);
 	egot = make_small(got);
     }
     else {
-	hp = HAlloc(p, base_halloc_sz + BIG_UINT_HEAP_SIZE + 5);
+	hp = HAllocX(p, base_halloc_sz + BIG_UINT_HEAP_SIZE + 5,
+                     ERTS_MAGIC_REF_THING_SIZE);
 	egot = uint_to_big(got, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
 
     if (is_first_trap) {
+        if (is_atom(tid))
+            tid = erts_db_make_tid(p, &tb->common);
         mpb = erts_db_make_match_prog_ref(p, *mpp, &hp);
         *mpp = NULL; /* otherwise the caller will destroy it */
     }
     else {
+        ASSERT(!is_atom(tid));
         mpb = prev_continuation_tptr[3];
     }
 
@@ -1580,11 +1584,17 @@ static int mtraversal_select_chunk_on_loop_ended(void* context_ptr, Sint slot_ix
                                              been in 'user space' */
             }
             if (rest != NIL || slot_ix >= 0) { /* Need more calls */
-                sc_context_ptr->hp = HAlloc(sc_context_ptr->p, 3 + 7 + ERTS_MAGIC_REF_THING_SIZE);
+                Eterm tid = sc_context_ptr->tid;
+                sc_context_ptr->hp = HAllocX(sc_context_ptr->p,
+                                             3 + 7 + ERTS_MAGIC_REF_THING_SIZE,
+                                             ERTS_MAGIC_REF_THING_SIZE);
                 mpb = erts_db_make_match_prog_ref(sc_context_ptr->p, *mpp, &sc_context_ptr->hp);
+                if (is_atom(tid))
+                    tid = erts_db_make_tid(sc_context_ptr->p,
+                                           &sc_context_ptr->tb->common);
                 continuation = TUPLE6(
                         sc_context_ptr->hp,
-                        sc_context_ptr->tid,
+                        tid,
                         make_small(slot_ix),
                         make_small(sc_context_ptr->chunk_size),
                         mpb, rest,
@@ -1621,12 +1631,16 @@ static int mtraversal_select_chunk_on_trap(void* context_ptr, Sint slot_ix, Sint
     BUMP_ALL_REDS(sc_context_ptr->p);
 
     if (sc_context_ptr->prev_continuation_tptr == NULL) {
+        Eterm tid = sc_context_ptr->tid;
         /* First time we're trapping */
-        hp = HAlloc(sc_context_ptr->p, 7 + ERTS_MAGIC_REF_THING_SIZE);
+        hp = HAllocX(sc_context_ptr->p, 7 + ERTS_MAGIC_REF_THING_SIZE,
+                     ERTS_MAGIC_REF_THING_SIZE);
+        if (is_atom(tid))
+            tid = erts_db_make_tid(sc_context_ptr->p, &sc_context_ptr->tb->common);
         mpb = erts_db_make_match_prog_ref(sc_context_ptr->p, *mpp, &hp);
         continuation = TUPLE6(
                 hp,
-                sc_context_ptr->tid,
+                tid,
                 make_small(slot_ix),
                 make_small(sc_context_ptr->chunk_size),
                 mpb,
-- 
cgit v1.2.3


From bf96458d2abfc22abbb3c4ae5126859421b054d7 Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikpelinux@gmail.com>
Date: Sat, 21 Apr 2018 13:39:11 +0200
Subject: fix integer truncation bugs in error logger path

Sending a large term to the error logger has two problems related
to the size and sign of the variables used to represent lengths:

- the API functions (erts_send_error_term_to_logger() et al) perform
  an unchecked narrowing conversion from size_t to int when passing
  dsbufp->str_len to the internal functions; this may both truncate
  the length and make it negative

- do_send_term_to_logger() and do_send_to_logger() multiply the
  int-typed length by 2 before widening it to Uint and adding a few
  more values; the intermediate product may overflow causing loss
  of high bits and a change of sign; if the intermediate product is
  negative the final size will be an extremely large positive value

The end result is that the computed buffer size can be arbitrarily
wrong, either too small or too large.

While reviewing this code I also found and fixed a potential narrowing
bug in erts_set_hole_marker().
---
 erts/emulator/beam/utils.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index d7116bd2c3..4a60a6faaa 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -140,7 +140,7 @@ Eterm*
 erts_set_hole_marker(Eterm* ptr, Uint sz)
 {
     Eterm* p = ptr;
-    int i;
+    Uint i;
 
     for (i = 0; i < sz; i++) {
 	*p++ = ERTS_HOLE_MARKER;
@@ -1990,7 +1990,7 @@ static void do_send_logger_message(Eterm *hp, ErlOffHeap *ohp, ErlHeapFragment *
    {notify,{info_msg,gleader,{emulator,format,[args]}}} |
    {notify,{error,gleader,{emulator,format,[args]}}} |
    {notify,{warning_msg,gleader,{emulator,format,[args}]}} */
-static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len)
+static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, size_t len)
 {
     Uint sz;
     Eterm gl;
@@ -2003,7 +2003,7 @@ static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len)
 
     ASSERT(is_atom(tag));
 
-    if (len <= 0) {
+    if (len == 0) {
 	return -1;
     }
 
@@ -2036,7 +2036,7 @@ static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len)
 }
 
 static int do_send_term_to_logger(Eterm tag, Eterm gleader,
-				  char *buf, int len, Eterm args)
+				  char *buf, size_t len, Eterm args)
 {
     Uint sz;
     Eterm gl;
@@ -2077,13 +2077,13 @@ static int do_send_term_to_logger(Eterm tag, Eterm gleader,
 }
 
 static ERTS_INLINE int
-send_info_to_logger(Eterm gleader, char *buf, int len)
+send_info_to_logger(Eterm gleader, char *buf, size_t len)
 {
     return do_send_to_logger(am_info_msg, gleader, buf, len);
 }
 
 static ERTS_INLINE int
-send_warning_to_logger(Eterm gleader, char *buf, int len)
+send_warning_to_logger(Eterm gleader, char *buf, size_t len)
 {
     Eterm tag;
     switch (erts_error_logger_warnings) {
@@ -2095,13 +2095,13 @@ send_warning_to_logger(Eterm gleader, char *buf, int len)
 }
 
 static ERTS_INLINE int
-send_error_to_logger(Eterm gleader, char *buf, int len)
+send_error_to_logger(Eterm gleader, char *buf, size_t len)
 {
     return do_send_to_logger(am_error, gleader, buf, len);
 }
 
 static ERTS_INLINE int
-send_error_term_to_logger(Eterm gleader, char *buf, int len, Eterm args)
+send_error_term_to_logger(Eterm gleader, char *buf, size_t len, Eterm args)
 {
     return do_send_term_to_logger(am_error, gleader, buf, len, args);
 }
-- 
cgit v1.2.3


From 9974f24df0cd217aced27c462ccea02a93c453ae Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 23 Apr 2018 11:59:23 +0200
Subject: Fix message tracing

---
 erts/emulator/beam/erl_proc_sig_queue.c | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index 4bb5f31538..fc3a2fe3c4 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -2389,8 +2389,9 @@ destroy_process_info_request(Process *c_p, ErtsProcessInfoSig *pisig)
 }
 
 static int
-handle_process_info(Process *c_p, ErtsMessage *sig,
-                    ErtsMessage ***next_nm_sig, int is_alive)
+handle_process_info(Process *c_p, ErtsSigRecvTracing *tracing,
+                    ErtsMessage *sig, ErtsMessage ***next_nm_sig,
+                    int is_alive)
 {
     ErtsProcessInfoSig *pisig = (ErtsProcessInfoSig *) sig;
     Uint reds = 0;
@@ -2414,7 +2415,11 @@ handle_process_info(Process *c_p, ErtsMessage *sig,
              * Move messages part of message queue into inner
              * signal queue...
              */
+            ASSERT(tracing);
+
             if (*next_nm_sig != &c_p->sig_qs.cont) {
+                if (*next_nm_sig == tracing->messages.next)
+                    tracing->messages.next = &c_p->sig_qs.cont;
                 *c_p->sig_qs.last = c_p->sig_qs.cont;
                 c_p->sig_qs.last = *next_nm_sig;
 
@@ -2875,7 +2880,7 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep,
 
         case ERTS_SIG_Q_OP_PROCESS_INFO:
             ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
-            handle_process_info(c_p, sig, next_nm_sig, !0);
+            handle_process_info(c_p, &tracing, sig, next_nm_sig, !0);
             ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
             break;
 
@@ -3196,7 +3201,7 @@ erts_proc_sig_handle_exit(Process *c_p, int *redsp)
             break;
 
         case ERTS_SIG_Q_OP_PROCESS_INFO:
-            handle_process_info(c_p, sig, next_nm_sig, 0);
+            handle_process_info(c_p, NULL, sig, next_nm_sig, 0);
             break;
 
         case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE:
-- 
cgit v1.2.3


From 468c3b5722034e80bbe6e47b6877b4221766bec1 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 23 Apr 2018 12:34:56 +0200
Subject: erts: Increase file read timeout for signal_abort test

---
 erts/emulator/test/dump_SUITE.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/dump_SUITE.erl b/erts/emulator/test/dump_SUITE.erl
index 38fa198ea6..8d18d46d92 100644
--- a/erts/emulator/test/dump_SUITE.erl
+++ b/erts/emulator/test/dump_SUITE.erl
@@ -96,12 +96,12 @@ get_dump_when_done(Dump) ->
         {ok, #file_info{ size = Sz }} ->
             get_dump_when_done(Dump, Sz);
         {error, enoent} ->
-            timer:sleep(100),
+            timer:sleep(1000),
             get_dump_when_done(Dump)
     end.
 
 get_dump_when_done(Dump, Sz) ->
-    timer:sleep(100),
+    timer:sleep(1000),
     case file:read_file_info(Dump) of
         {ok, #file_info{ size = Sz }} ->
             file:read_file(Dump);
-- 
cgit v1.2.3


From 573a5abd9d6b1668b49376b489b187780c7125c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 27 Mar 2018 13:14:40 +0200
Subject: erts: Rewrite memory instrumentation

This commit replaces the old memory instrumentation with a new
implementation that scans carriers instead of wrapping
erts_alloc/erts_free. The old implementation could not extract
information without halting the emulator, had considerable runtime
overhead, and the memory maps it produced were noisy and lacked
critical information.

Since the new implementation walks through existing data structures
there's no longer a need to start the emulator with special flags to
get information about carrier utilization/fragmentation. Memory
fragmentation is also easier to diagnose as it's presented on a
per-carrier basis which eliminates the need to account for "holes"
between mmap segments.

To help track allocations, each allocation can now be tagged with
what it is and who allocated it at the cost of one extra word per
allocation. This is controlled on a per-allocator basis with the
+M<S>atags option, and is enabled by default for binary_alloc and
driver_alloc (which is also used by NIFs).
---
 erts/emulator/Makefile.in            |    2 +-
 erts/emulator/beam/bif.tab           |    2 +
 erts/emulator/beam/break.c           |   15 -
 erts/emulator/beam/erl_alloc.c       |   98 +--
 erts/emulator/beam/erl_alloc_util.c  | 1586 ++++++++++++++++++++++++++++++----
 erts/emulator/beam/erl_alloc_util.h  |   35 +
 erts/emulator/beam/erl_bif_info.c    |   96 +-
 erts/emulator/beam/erl_init.c        |    2 +-
 erts/emulator/beam/erl_instrument.c  | 1257 ---------------------------
 erts/emulator/beam/erl_instrument.h  |   42 -
 erts/emulator/beam/erl_process.c     |    4 +-
 erts/emulator/beam/erl_process.h     |    1 +
 erts/emulator/test/process_SUITE.erl |   56 +-
 13 files changed, 1577 insertions(+), 1619 deletions(-)
 delete mode 100644 erts/emulator/beam/erl_instrument.c
 delete mode 100644 erts/emulator/beam/erl_instrument.h

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 92cf40c1ae..5dfa60ee74 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -832,7 +832,7 @@ RUN_OBJS += \
 	$(OBJDIR)/erl_alloc.o		$(OBJDIR)/erl_mtrace.o \
 	$(OBJDIR)/erl_alloc_util.o	$(OBJDIR)/erl_goodfit_alloc.o \
 	$(OBJDIR)/erl_bestfit_alloc.o	$(OBJDIR)/erl_afit_alloc.o \
-	$(OBJDIR)/erl_instrument.o	$(OBJDIR)/erl_init.o \
+	$(OBJDIR)/erl_init.o \
 	$(OBJDIR)/erl_atom_table.o	$(OBJDIR)/erl_bif_table.o \
 	$(OBJDIR)/erl_bif_ddll.o  	$(OBJDIR)/erl_bif_guard.o \
 	$(OBJDIR)/erl_bif_info.o	$(OBJDIR)/erl_bif_op.o \
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 687fd39d58..bcaf9277bf 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -693,3 +693,5 @@ bif erts_internal:new_connection/1
 bif erts_internal:abort_connection/2
 bif erts_internal:map_next/3
 bif ets:whereis/1
+bif erts_internal:gather_alloc_histograms/1
+bif erts_internal:gather_carrier_info/1
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index ba8cc5e2ba..9ff52c92b8 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -36,7 +36,6 @@
 #include "hash.h"
 #include "atom.h"
 #include "beam_load.h"
-#include "erl_instrument.h"
 #include "erl_hl_timer.h"
 #include "erl_thr_progress.h"
 #include "erl_proc_sig_queue.h"
@@ -955,20 +954,6 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
     erts_cbprintf(to, to_arg, "=atoms\n");
     dump_atoms(to, to_arg);
 
-    /* Keep the instrumentation data at the end of the dump */
-    if (erts_instr_memory_map || erts_instr_stat) {
-	erts_cbprintf(to, to_arg, "=instr_data\n");
-
-	if (erts_instr_stat) {
-	    erts_cbprintf(to, to_arg, "=memory_status\n");
-	    erts_instr_dump_stat_to(to, to_arg, 0);
-	}
-	if (erts_instr_memory_map) {
-	    erts_cbprintf(to, to_arg, "=memory_map\n");
-	    erts_instr_dump_memory_map_to(to, to_arg);
-	}
-    }
-
     erts_cbprintf(to, to_arg, "=end\n");
     if (fp) {
         fclose(fp);
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 061b9df627..d99d2ea57b 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -38,7 +38,7 @@
 #include "erl_db.h"
 #include "erl_binary.h"
 #include "erl_bits.h"
-#include "erl_instrument.h"
+#include "erl_mtrace.h"
 #include "erl_mseg.h"
 #include "erl_monitor_link.h"
 #include "erl_hl_timer.h"
@@ -202,8 +202,6 @@ typedef struct {
     int top_pad;
     AlcUInit_t alloc_util;
     struct {
-	int stat;
-	int map;
 	char *mtrace;
 	char *nodename;
     } instr;
@@ -428,6 +426,7 @@ set_default_binary_alloc_opts(struct au_init *ip)
 #endif
     ip->init.util.ts 		= ERTS_ALC_MTA_BINARY;
     ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
+    ip->init.util.atags		= 1;
 }
 
 static void
@@ -464,6 +463,7 @@ set_default_driver_alloc_opts(struct au_init *ip)
 #endif
     ip->init.util.ts 		= ERTS_ALC_MTA_DRIVER;
     ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
+    ip->init.util.atags		= 1;
 }
 
 static void
@@ -501,6 +501,7 @@ set_default_test_alloc_opts(struct au_init *ip)
     ip->init.util.mmbcs 	= 0; /* Main carrier size */
     ip->init.util.ts 		= ERTS_ALC_MTA_TEST;
     ip->init.util.acul		= ERTS_ALC_DEFAULT_ACUL;
+    ip->init.util.atags		= 1;
 
     /* Use a constant minimal MBC size */
 #if ERTS_SA_MB_CARRIERS
@@ -906,7 +907,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 		       &test_alloc_state);
 
     erts_mtrace_install_wrapper_functions();
-    extra_block_size += erts_instr_init(init.instr.stat, init.instr.map);
 
     init_aireq_alloc();
 
@@ -1411,7 +1411,9 @@ handle_au_arg(struct au_init *auip,
 	    }
 	    if (!strategy_support_carrier_migration(auip))
 		auip->init.util.acul = 0;
-	}
+	} else if (has_prefix("atags", sub_param)) {
+            auip->init.util.atags = get_bool_value(sub_param + 5, argv, ip);
+        }
 	else
 	    goto bad_switch;
 	break;
@@ -1741,24 +1743,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 		    break;
 		case 'i':
 		    switch (argv[i][3]) {
-		    case 's':
-			arg = get_value(argv[i]+4, argv, &i);
-			if (sys_strcmp("true", arg) == 0)
-			    init->instr.stat = 1;
-			else if (sys_strcmp("false", arg) == 0)
-			    init->instr.stat = 0;
-			else
-			    bad_value(param, param+3, arg);
-			break;
-		    case 'm':
-			arg = get_value(argv[i]+4, argv, &i);
-			if (sys_strcmp("true", arg) == 0)
-			    init->instr.map = 1;
-			else if (sys_strcmp("false", arg) == 0)
-			    init->instr.map = 0;
-			else
-			    bad_value(param, param+3, arg);
-			break;
 		    case 't':
 			init->instr.mtrace = get_value(argv[i]+4, argv, &i);
 			break;
@@ -1817,9 +1801,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 	    case '-':
 		if (argv[i][2] == '\0') {
 		    /* End of system flags reached */
-		    if (init->instr.mtrace
-			/* || init->instr.stat
-			   || init->instr.map */) {
+		    if (init->instr.mtrace) {
 			while (i < *argc) {
 			    if(sys_strcmp(argv[i], "-sname") == 0
 			       || sys_strcmp(argv[i], "-name") == 0) {
@@ -2097,7 +2079,7 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
  * NOTE! When updating this function, make sure to also update
  *       erlang:memory/[0,1] in $ERL_TOP/erts/preloaded/src/erlang.erl
  */
-#define ERTS_MEM_NEED_ALL_ALCU (!erts_instr_stat && want_tot_or_sys)
+#define ERTS_MEM_NEED_ALL_ALCU (want_tot_or_sys)
     struct {
 	int total;
 	int processes;
@@ -2108,7 +2090,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	int binary;
 	int code;
 	int ets;
-	int maximum;
     } want = {0};
     struct {
 	UWord total;
@@ -2120,7 +2101,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	UWord binary;
 	UWord code;
 	UWord ets;
-	UWord maximum;
     } size = {0};
     Eterm atoms[sizeof(size)/sizeof(UWord)];
     UWord *uintps[sizeof(size)/sizeof(UWord)];
@@ -2173,12 +2153,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	want.ets = 1;
 	atoms[length] = am_ets;
 	uintps[length++] = &size.ets;
-
-	want.maximum = erts_instr_stat;
-	if (want.maximum) {
-	    atoms[length] = am_maximum;
-	    uintps[length++] = &size.maximum;
-	}
     }
     else {
 	DeclareTmpHeapNoproc(tmp_heap,2);
@@ -2260,18 +2234,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 		    uintps[length++] = &size.ets;
 		}
 		break;
-	    case am_maximum:
-		if (erts_instr_stat) {
-		    if (!want.maximum) {
-			want.maximum = 1;
-			atoms[length] = am_maximum;
-			uintps[length++] = &size.maximum;
-		    }
-		} else {
-		    UnUseTmpHeapNoproc(2);
-		    return am_badarg;
-		}
-		break;
 	    default:
 		UnUseTmpHeapNoproc(2);
 		return am_badarg;
@@ -2437,14 +2399,7 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	size.ets += erts_get_ets_misc_mem_size();
     }
 
-    if (erts_instr_stat && (want_tot_or_sys || want.maximum)) {
-	if (want_tot_or_sys) {
-	    size.total = erts_instr_get_total();
-	    size.system = size.total - size.processes;
-	}
-	size.maximum = erts_instr_get_max_total();
-    }
-    else if (want_tot_or_sys) {
+    if (want_tot_or_sys) {
 	size.system = size.total - size.processes;
     }
 
@@ -2522,18 +2477,6 @@ erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc)
 
     i = 0;
 
-    if (erts_instr_stat) {
-	values[i].arity = 2;
-	values[i].name = "total";
-	values[i].ui[0] = erts_instr_get_total();
-	i++;
-
-	values[i].arity = 2;
-	values[i].name = "maximum";
-	values[i].ui[0] = erts_instr_get_max_total();
-	i++;
-    }
-
     values[i].arity = 2;
     values[i].name = "sys_misc";
     values[i].ui[0] = erts_sys_misc_mem_sz();
@@ -2824,10 +2767,7 @@ erts_allocator_info(fmtfn_t to, void *arg)
     erts_alcu_au_info_options(&to, arg, NULL, NULL);
 
     erts_print(to, arg, "=allocator:instr\n");
-    erts_print(to, arg, "option m: %s\n",
-	       erts_instr_memory_map ? "true" : "false");
-    erts_print(to, arg, "option s: %s\n",
-	       erts_instr_stat ? "true" : "false");
+
     erts_print(to, arg, "option t: %s\n",
 	       erts_mtrace_enabled ? "true" : "false");
 
@@ -2933,16 +2873,12 @@ erts_allocator_options(void *proc)
                                              NULL, hpp, szp);
 #endif
     {
-	Eterm o[3], v[3];
-	o[0] = am_atom_put("m", 1);
-	v[0] = erts_instr_memory_map ? am_true : am_false;
-	o[1] = am_atom_put("s", 1);
-	v[1] = erts_instr_stat ? am_true : am_false;
-	o[2] = am_atom_put("t", 1);
-	v[2] = erts_mtrace_enabled ? am_true : am_false;
+	Eterm o[1], v[1];
+	o[0] = am_atom_put("t", 1);
+	v[0] = erts_mtrace_enabled ? am_true : am_false;
 
 	atoms[length] = am_atom_put("instr", 5); 
-	terms[length++] = erts_bld_2tup_list(hpp, szp, 3, o, v);
+	terms[length++] = erts_bld_2tup_list(hpp, szp, 1, o, v);
     }
 
     atoms[length] = am_atom_put("lock_physical_memory", 20);
@@ -3458,8 +3394,8 @@ badarg:
 
 /* 
  * The allocator wrapper prelocking stuff below is about the locking order.
- * It only affects wrappers (erl_mtrace.c and erl_instrument.c) that keep locks
- * during alloc/realloc/free.
+ * It only affects wrappers (erl_mtrace.c) that keep locks during
+ * alloc/realloc/free.
  *
  * Some query functions in erl_alloc_util.c lock the allocator mutex and then
  * use erts_printf that in turn may call the sys allocator through the wrappers.
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index fed51eb200..fdf355d503 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -48,6 +48,8 @@
 #include "erl_mseg.h"
 #include "erl_threads.h"
 #include "erl_thr_progress.h"
+#include "erl_bif_unique.h"
+#include "erl_nif.h"
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
 #include "erl_lock_count.h"
@@ -139,6 +141,33 @@ MBC after deallocating first block:
 [Carrier_t|pad|Block_t 111| udata... ]
 */
 
+/* Allocation tags ...
+ *
+ * These are added to the footer of every block when enabled. Currently they
+ * consist of the allocation type and an atom identifying the allocating
+ * driver/nif (or 'system' if that can't be determined), but the format is not
+ * supposed to be set in stone.
+ *
+ * The packing scheme requires that the atom values are small enough to fit
+ * into a word with ERTS_ALC_N_BITS to spare. Users must check for overflow
+ * before MAKE_ATAG(). */
+
+typedef UWord alcu_atag_t;
+
+#define MAKE_ATAG(IdAtom, Type) \
+    (ASSERT((Type) >= ERTS_ALC_N_MIN && (Type) <= ERTS_ALC_N_MAX), \
+     ASSERT(atom_val(IdAtom) <= MAX_ATAG_ATOM_ID), \
+     (atom_val(IdAtom) << ERTS_ALC_N_BITS) | (Type))
+
+#define ATAG_ID(AT) (make_atom((AT) >> ERTS_ALC_N_BITS))
+#define ATAG_TYPE(AT) ((AT) & ERTS_ALC_N_MASK)
+
+#define MAX_ATAG_ATOM_ID (ERTS_UWORD_MAX >> ERTS_ALC_N_BITS)
+
+#define DBG_IS_VALID_ATAG(Allocator, AT) \
+    (ATAG_TYPE(AT) >= ERTS_ALC_N_MIN && \
+     ATAG_TYPE(AT) <= ERTS_ALC_N_MAX && \
+     (Allocator)->alloc_no == ERTS_ALC_T2A(ERTS_ALC_N2T(ATAG_TYPE(AT))))
 
 /* Blocks ... */
 
@@ -153,10 +182,17 @@ MBC after deallocating first block:
 #endif
 #define FBLK_FTR_SZ (sizeof(FreeBlkFtr_t))
 
+#define GET_BLK_ATAG(B) \
+    (((alcu_atag_t *) (((char *) (B)) + (BLK_SZ(B))))[-1])
+#define SET_BLK_ATAG(B, T) \
+    (((alcu_atag_t *) (((char *) (B)) + (BLK_SZ(B))))[-1] = (T))
+
+#define BLK_ATAG_SZ(AP) ((AP)->atags ? sizeof(alcu_atag_t) : 0)
+
 #define UMEMSZ2BLKSZ(AP, SZ)						\
-  (ABLK_HDR_SZ + (SZ) <= (AP)->min_block_size				\
+  (ABLK_HDR_SZ + BLK_ATAG_SZ(AP) + (SZ) <= (AP)->min_block_size		\
    ? (AP)->min_block_size						\
-   : UNIT_CEILING(ABLK_HDR_SZ + (SZ)))
+   : UNIT_CEILING(ABLK_HDR_SZ + BLK_ATAG_SZ(AP) + (SZ)))
 
 #define UMEM2BLK(P) ((Block_t *) (((char *) (P)) - ABLK_HDR_SZ))
 #define BLK2UMEM(P) ((void *)    (((char *) (P)) + ABLK_HDR_SZ))
@@ -688,6 +724,62 @@ static void destroy_carrier(Allctr_t *, Block_t *, Carrier_t **);
 static void mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp);
 static void dealloc_block(Allctr_t *, void *, ErtsAlcFixList_t *, int);
 
+static alcu_atag_t determine_alloc_tag(Allctr_t *allocator, ErtsAlcType_t type)
+{
+    ErtsSchedulerData *esdp;
+    Eterm id;
+
+    ERTS_CT_ASSERT(_unchecked_atom_val(am_system) <= MAX_ATAG_ATOM_ID);
+    ASSERT(allocator->atags);
+
+    esdp = erts_get_scheduler_data();
+    id = am_system;
+
+    if (esdp) {
+        if (esdp->current_nif) {
+            Module *mod = erts_nif_get_module((esdp->current_nif)->mod_nif);
+
+            /* Mod can be NULL if a resource destructor allocates memory after
+             * the module has been unloaded. */
+            if (mod) {
+                id = make_atom(mod->module);
+            }
+        } else if (esdp->current_port) {
+            Port *p = esdp->current_port;
+            id = (p->drv_ptr)->name_atom;
+        }
+
+        /* We fall back to 'system' if we can't pack the driver/NIF name into
+         * the tag. This may be a bit misleading but we've made no promises
+         * that the information is complete.
+         *
+         * This can only happen on 32-bit emulators when a new driver/NIF has
+         * been loaded *after* 16 million atoms have been used, and supporting
+         * that fringe case is not worth an extra word. 64-bit emulators are
+         * unaffected since the atom cache limits atom indexes to 32 bits. */
+        if(MAX_ATOM_TABLE_SIZE > MAX_ATAG_ATOM_ID) {
+            if (atom_val(id) > MAX_ATAG_ATOM_ID) {
+                id = am_system;
+            }
+        }
+    }
+
+    return MAKE_ATAG(id, type);
+}
+
+static void set_alloc_tag(Allctr_t *allocator, void *p, alcu_atag_t tag)
+{
+    Block_t *block;
+
+    ASSERT(DBG_IS_VALID_ATAG(allocator, tag));
+    ASSERT(allocator->atags && p);
+    (void)allocator;
+
+    block = UMEM2BLK(p);
+
+    SET_BLK_ATAG(block, tag);
+}
+
 /* internal data... */
 
 #if 0
@@ -4242,6 +4334,7 @@ static struct {
     Eterm e;
     Eterm t;
     Eterm ramv;
+    Eterm atags;
 #if HAVE_ERTS_MSEG
     Eterm asbcst;
     Eterm rsbcst;
@@ -4311,7 +4404,7 @@ static struct {
 #endif
 } am;
 
-static Eterm fix_type_atoms[ERTS_ALC_NO_FIXED_SIZES];
+static Eterm alloc_type_atoms[ERTS_ALC_N_MAX + 1];
 
 static ERTS_INLINE void atom_init(Eterm *atom, char *name)
 {
@@ -4342,6 +4435,7 @@ init_atoms(Allctr_t *allctr)
 	AM_INIT(e);
 	AM_INIT(t);
 	AM_INIT(ramv);
+	AM_INIT(atags);
 #if HAVE_ERTS_MSEG
 	AM_INIT(asbcst);
 	AM_INIT(rsbcst);
@@ -4413,12 +4507,12 @@ init_atoms(Allctr_t *allctr)
 	}
 #endif
 
-	for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) {
-	    ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix;
-	    char *name = (char *) ERTS_ALC_N2TD(n);
-	    size_t len = sys_strlen(name);
-	    fix_type_atoms[ix] = am_atom_put(name, len);
-	}
+        for (ix = ERTS_ALC_N_MIN; ix <= ERTS_ALC_N_MAX; ix++) {
+            const char *name = ERTS_ALC_N2TD(ix);
+            size_t len = sys_strlen(name);
+
+            alloc_type_atoms[ix] = am_atom_put(name, len);
+        }
     }
     
     if (allctr && !allctr->atoms_initialized) {
@@ -4531,6 +4625,7 @@ sz_info_fix(Allctr_t *allctr,
 		ErtsAlcFixList_t *fix = &allctr->fix[ix];
 		UWord alloced = fix->type_size * fix->u.cpool.allocated;
 		UWord used = fix->type_size * fix->u.cpool.used;
+                ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix;
 
 		if (print_to_p) {
 		    fmtfn_t to = *print_to_p;
@@ -4538,15 +4633,14 @@ sz_info_fix(Allctr_t *allctr,
 		    erts_print(to,
 			       arg,
 			       "fix type internal: %s %bpu %bpu\n",
-			       (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE
-						      + ix),
+			       (char *) ERTS_ALC_N2TD(n),
 			       alloced,
 			       used);
 		}
 
 		if (hpp || szp) {
 		    add_3tup(hpp, szp, &res,
-			     fix_type_atoms[ix],
+			     alloc_type_atoms[n],
 			     bld_unstable_uint(hpp, szp, alloced),
 			     bld_unstable_uint(hpp, szp, used));
 		}
@@ -4559,6 +4653,7 @@ sz_info_fix(Allctr_t *allctr,
 	    ErtsAlcFixList_t *fix = &allctr->fix[ix];
 	    UWord alloced = fix->type_size * fix->u.nocpool.allocated;
 	    UWord used = fix->type_size*fix->u.nocpool.used;
+            ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix;
 
 	    if (print_to_p) {
 		fmtfn_t to = *print_to_p;
@@ -4566,15 +4661,14 @@ sz_info_fix(Allctr_t *allctr,
 		erts_print(to,
 			   arg,
 			   "fix type: %s %bpu %bpu\n",
-			   (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE
-						  + ix),
+			   (char *) ERTS_ALC_N2TD(n),
 			   alloced,
 			   used);
 	    }
 
 	    if (hpp || szp) {
 		add_3tup(hpp, szp, &res,
-			 fix_type_atoms[ix],
+			 alloc_type_atoms[n],
 			 bld_unstable_uint(hpp, szp, alloced),
 			 bld_unstable_uint(hpp, szp, used));
 	    }
@@ -5000,6 +5094,7 @@ info_options(Allctr_t *allctr,
 		   "option e: true\n"
 		   "option t: %s\n"
 		   "option ramv: %s\n"
+		   "option atags: %s\n"
 		   "option sbct: %beu\n"
 #if HAVE_ERTS_MSEG
 		   "option asbcst: %bpu\n"
@@ -5018,6 +5113,7 @@ info_options(Allctr_t *allctr,
 		   "option acul: %bpu\n",
 		   topt,
 		   allctr->ramv ? "true" : "false",
+		   allctr->atags ? "true" : "false",
 		   allctr->sbc_threshold,
 #if HAVE_ERTS_MSEG
 		   allctr->mseg_opt.abs_shrink_th,
@@ -5087,6 +5183,7 @@ info_options(Allctr_t *allctr,
 		 am_sbct,
 		 bld_uint(hpp, szp, allctr->sbc_threshold));
 	add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false);
+	add_2tup(hpp, szp, &res, am.atags, allctr->atags ? am_true : am_false);
 	add_2tup(hpp, szp, &res, am.t, (allctr->t ? am_true : am_false));
 	add_2tup(hpp, szp, &res, am.e, am_true);
     }
@@ -5408,9 +5505,8 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t *
 /* ----------------------------------------------------------------------- */
 
 static ERTS_INLINE void *
-do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
+do_erts_alcu_alloc(ErtsAlcType_t type, Allctr_t *allctr, Uint size)
 {
-    Allctr_t *allctr = (Allctr_t *) extra; 
     void *res;
 
     ASSERT(initialized);
@@ -5449,10 +5545,19 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
 
 void *erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
 {
+    Allctr_t *allctr = (Allctr_t *) extra;
     void *res;
+
     ASSERT(!"This is not thread safe");
-    res = do_erts_alcu_alloc(type, extra, size);
+
+    res = do_erts_alcu_alloc(type, allctr, size);
+
+    if (allctr->atags && res) {
+        set_alloc_tag(allctr, res, determine_alloc_tag(allctr, type));
+    }
+
     DEBUG_CHECK_ALIGNMENT(res);
+
     return res;
 }
 
@@ -5462,13 +5567,25 @@ void *
 erts_alcu_alloc_ts(ErtsAlcType_t type, void *extra, Uint size)
 {
     Allctr_t *allctr = (Allctr_t *) extra;
+    alcu_atag_t tag = 0;
     void *res;
+
+    if (allctr->atags) {
+        tag = determine_alloc_tag(allctr, type);
+    }
+
     erts_mtx_lock(&allctr->mutex);
-    res = do_erts_alcu_alloc(type, extra, size);
 
-    DEBUG_CHECK_ALIGNMENT(res);
+    res = do_erts_alcu_alloc(type, allctr, size);
+
+    if (allctr->atags && res) {
+        set_alloc_tag(allctr, res, tag);
+    }
 
     erts_mtx_unlock(&allctr->mutex);
+
+    DEBUG_CHECK_ALIGNMENT(res);
+
     return res;
 }
 
@@ -5478,6 +5595,7 @@ erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size)
 {
     ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra;
     int ix;
+    alcu_atag_t tag = 0;
     Allctr_t *allctr;
     void *res;
 
@@ -5487,11 +5605,19 @@ erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size)
 
     allctr = tspec->allctr[ix];
 
+    if (allctr->atags) {
+        tag = determine_alloc_tag(allctr, type);
+    }
+
     if (allctr->thread_safe)
 	erts_mtx_lock(&allctr->mutex);
 
     res = do_erts_alcu_alloc(type, allctr, size);
 
+    if (allctr->atags && res) {
+        set_alloc_tag(allctr, res, tag);
+    }
+
     if (allctr->thread_safe)
 	erts_mtx_unlock(&allctr->mutex);
 
@@ -5504,10 +5630,15 @@ void *
 erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size)
 {
     Allctr_t *pref_allctr;
+    alcu_atag_t tag = 0;
     void *res;
 
     pref_allctr = get_pref_allctr(extra);
 
+    if (pref_allctr->atags) {
+        tag = determine_alloc_tag(pref_allctr, type);
+    }
+
     if (pref_allctr->thread_safe)
 	erts_mtx_lock(&pref_allctr->mutex);
 
@@ -5523,12 +5654,15 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size)
 	res = do_erts_alcu_alloc(type, pref_allctr, size);
     }
 
+    if (pref_allctr->atags && res) {
+        set_alloc_tag(pref_allctr, res, tag);
+    }
+
     if (pref_allctr->thread_safe)
 	erts_mtx_unlock(&pref_allctr->mutex);
 
     DEBUG_CHECK_ALIGNMENT(res);
 
-
     return res;
 }
 
@@ -5537,10 +5671,9 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size)
 /* ------------------------------------------------------------------------- */
 
 static ERTS_INLINE void
-do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p,
+do_erts_alcu_free(ErtsAlcType_t type, Allctr_t *allctr, void *p,
 		  Carrier_t **busy_pcrr_pp)
 {
-    Allctr_t *allctr = (Allctr_t *) extra; 
     ASSERT(initialized);
 
     ASSERT(allctr);
@@ -5572,7 +5705,8 @@ do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p,
 
 void erts_alcu_free(ErtsAlcType_t type, void *extra, void *p)
 {
-    do_erts_alcu_free(type, extra, p, NULL);
+    Allctr_t *allctr = (Allctr_t *) extra;
+    do_erts_alcu_free(type, allctr, p, NULL);
 }
 
 
@@ -5581,7 +5715,7 @@ erts_alcu_free_ts(ErtsAlcType_t type, void *extra, void *p)
 {
     Allctr_t *allctr = (Allctr_t *) extra;
     erts_mtx_lock(&allctr->mutex);
-    do_erts_alcu_free(type, extra, p, NULL);
+    do_erts_alcu_free(type, allctr, p, NULL);
     erts_mtx_unlock(&allctr->mutex);
 }
 
@@ -5641,13 +5775,12 @@ erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p)
 
 static ERTS_INLINE void *
 do_erts_alcu_realloc(ErtsAlcType_t type,
-		     void *extra,
+		     Allctr_t *allctr,
 		     void *p,
 		     Uint size,
 		     Uint32 alcu_flgs,
 		     Carrier_t **busy_pcrr_pp)
 {
-    Allctr_t *allctr = (Allctr_t *) extra; 
     Block_t *blk;
     void *res;
 
@@ -5661,7 +5794,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
     ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
 
     if (!p) {
-	res = do_erts_alcu_alloc(type, extra, size);
+	res = do_erts_alcu_alloc(type, allctr, size);
 	INC_CC(allctr->calls.this_realloc);
 	DEC_CC(allctr->calls.this_alloc);
 	return res;
@@ -5670,7 +5803,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
 #if ALLOC_ZERO_EQ_NULL
     if (!size) {
 	ASSERT(p);
-	do_erts_alcu_free(type, extra, p, busy_pcrr_pp);
+	do_erts_alcu_free(type, allctr, p, busy_pcrr_pp);
 	INC_CC(allctr->calls.this_realloc);
 	DEC_CC(allctr->calls.this_free);
 	return NULL;
@@ -5755,19 +5888,29 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
 void *
 erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size)
 {
+    Allctr_t *allctr = (Allctr_t *)extra;
     void *res;
-    res = do_erts_alcu_realloc(type, extra, p, size, 0, NULL);
+
+    res = do_erts_alcu_realloc(type, allctr, p, size, 0, NULL);
+
     DEBUG_CHECK_ALIGNMENT(res);
+
+    if (allctr->atags && res) {
+        set_alloc_tag(allctr, res, determine_alloc_tag(allctr, type));
+    }
+
     return res;
 }
 
 void *
 erts_alcu_realloc_mv(ErtsAlcType_t type, void *extra, void *p, Uint size)
 {
+    Allctr_t *allctr = (Allctr_t *)extra;
     void *res;
-    res = do_erts_alcu_alloc(type, extra, size);
+
+    res = do_erts_alcu_alloc(type, allctr, size);
     if (!res)
-	res = erts_alcu_realloc(type, extra, p, size);
+        res = do_erts_alcu_realloc(type, allctr, p, size, 0, NULL);
     else {
 	Block_t *blk;
 	size_t cpy_size;
@@ -5777,23 +5920,42 @@ erts_alcu_realloc_mv(ErtsAlcType_t type, void *extra, void *p, Uint size)
 	if (cpy_size > size)
 	    cpy_size = size;
 	sys_memcpy(res, p, cpy_size);
-	do_erts_alcu_free(type, extra, p, NULL);
+	do_erts_alcu_free(type, allctr, p, NULL);
     }
+
     DEBUG_CHECK_ALIGNMENT(res);
+
+    if (allctr->atags && res) {
+        set_alloc_tag(allctr, res, determine_alloc_tag(allctr, type));
+    }
+
     return res;
 }
 
 
-
 void *
 erts_alcu_realloc_ts(ErtsAlcType_t type, void *extra, void *ptr, Uint size)
 {
     Allctr_t *allctr = (Allctr_t *) extra;
+    alcu_atag_t tag = 0;
     void *res;
+
+    if (allctr->atags) {
+        tag = determine_alloc_tag(allctr, type);
+    }
+
     erts_mtx_lock(&allctr->mutex);
-    res = do_erts_alcu_realloc(type, extra, ptr, size, 0, NULL);
+
+    res = do_erts_alcu_realloc(type, allctr, ptr, size, 0, NULL);
+
+    if (allctr->atags && res) {
+        set_alloc_tag(allctr, res, tag);
+    }
+
     erts_mtx_unlock(&allctr->mutex);
+
     DEBUG_CHECK_ALIGNMENT(res);
+
     return res;
 }
 
@@ -5801,11 +5963,17 @@ void *
 erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size)
 {
     Allctr_t *allctr = (Allctr_t *) extra;
+    alcu_atag_t tag = 0;
     void *res;
+
+    if (allctr->atags) {
+        tag = determine_alloc_tag(allctr, type);
+    }
+
     erts_mtx_lock(&allctr->mutex);
-    res = do_erts_alcu_alloc(type, extra, size);
+    res = do_erts_alcu_alloc(type, allctr, size);
     if (!res)
-        res = do_erts_alcu_realloc(type, extra, p, size, 0, NULL);
+	res = do_erts_alcu_realloc(type, allctr, p, size, 0, NULL);
     else {
 	Block_t *blk;
 	size_t cpy_size;
@@ -5815,10 +5983,17 @@ erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size)
 	if (cpy_size > size)
 	    cpy_size = size;
 	sys_memcpy(res, p, cpy_size);
-	do_erts_alcu_free(type, extra, p, NULL);
+	do_erts_alcu_free(type, allctr, p, NULL);
     }
+
+    if (allctr->atags && res) {
+        set_alloc_tag(allctr, res, tag);
+    }
+
     erts_mtx_unlock(&allctr->mutex);
+
     DEBUG_CHECK_ALIGNMENT(res);
+
     return res;
 }
 
@@ -5829,6 +6004,7 @@ erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra,
 {
     ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra;
     int ix;
+    alcu_atag_t tag = 0;
     Allctr_t *allctr;
     void *res;
 
@@ -5838,11 +6014,19 @@ erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra,
 
     allctr = tspec->allctr[ix];
 
+    if (allctr->atags) {
+        tag = determine_alloc_tag(allctr, type);
+    }
+
     if (allctr->thread_safe)
 	erts_mtx_lock(&allctr->mutex);
 
     res = do_erts_alcu_realloc(type, allctr, ptr, size, 0, NULL);
 
+    if (allctr->atags && res) {
+        set_alloc_tag(allctr, res, tag);
+    }
+
     if (allctr->thread_safe)
 	erts_mtx_unlock(&allctr->mutex);
 
@@ -5857,6 +6041,7 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra,
 {
     ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra;
     int ix;
+    alcu_atag_t tag = 0;
     Allctr_t *allctr;
     void *res;
 
@@ -5866,14 +6051,16 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra,
 
     allctr = tspec->allctr[ix];
 
+    if (allctr->atags) {
+        tag = determine_alloc_tag(allctr, type);
+    }
+
     if (allctr->thread_safe)
 	erts_mtx_lock(&allctr->mutex);
 
     res = do_erts_alcu_alloc(type, allctr, size);
     if (!res) {
-	if (allctr->thread_safe)
-	    erts_mtx_unlock(&allctr->mutex);
-	res = erts_alcu_realloc_thr_spec(type, allctr, ptr, size);
+        res = do_erts_alcu_realloc(type, allctr, ptr, size, 0, NULL);
     }
     else {
 	Block_t *blk;
@@ -5885,29 +6072,34 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra,
 	    cpy_size = size;
 	sys_memcpy(res, ptr, cpy_size);
 	do_erts_alcu_free(type, allctr, ptr, NULL);
-	if (allctr->thread_safe)
-	    erts_mtx_unlock(&allctr->mutex);
     }
 
+    if (allctr->atags && res) {
+        set_alloc_tag(allctr, res, tag);
+    }
+
+    if (allctr->thread_safe)
+        erts_mtx_unlock(&allctr->mutex);
+
     DEBUG_CHECK_ALIGNMENT(res);
 
     return res;
 }
 
 static ERTS_INLINE void *
-realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size,
+realloc_thr_pref(ErtsAlcType_t type, Allctr_t *pref_allctr, void *p, Uint size,
 		 int force_move)
 {
     void *res;
-    Allctr_t *pref_allctr, *used_allctr;
+    Allctr_t *used_allctr;
     UWord old_user_size;
     Carrier_t *busy_pcrr_p;
+    alcu_atag_t tag = 0;
     int retried;
 
-    if (!p)
-	return erts_alcu_alloc_thr_pref(type, extra, size);
-
-    pref_allctr = get_pref_allctr(extra);
+    if (pref_allctr->atags) {
+        tag = determine_alloc_tag(pref_allctr, type);
+    }
 
     if (pref_allctr->thread_safe)
 	erts_mtx_lock(&pref_allctr->mutex);
@@ -5936,6 +6128,11 @@ restart:
 	    retried = 1;
 	    goto restart;
 	}
+
+        if (pref_allctr->atags && res) {
+            set_alloc_tag(pref_allctr, res, tag);
+        }
+
 	if (pref_allctr->thread_safe)
 	    erts_mtx_unlock(&pref_allctr->mutex);
     }
@@ -5944,6 +6141,9 @@ restart:
 	if (!res)
 	    goto unlock_ts_return;
 	else {
+            if (pref_allctr->atags) {
+                set_alloc_tag(pref_allctr, res, tag);
+            }
 
 	    DEBUG_CHECK_ALIGNMENT(res);
 
@@ -5974,20 +6174,34 @@ restart:
 	}
     }
 
+    DEBUG_CHECK_ALIGNMENT(res);
+
     return res;
 }
 
 void *
 erts_alcu_realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size)
 {
-    return realloc_thr_pref(type, extra, p, size, 0);
+    if (p) {
+        Allctr_t *pref_allctr = get_pref_allctr(extra);
+
+        return realloc_thr_pref(type, pref_allctr, p, size, 0);
+    }
+
+    return erts_alcu_alloc_thr_pref(type, extra, size);
 }
 
 void *
 erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra,
 			      void *p, Uint size)
 {
-    return realloc_thr_pref(type, extra, p, size, 1);
+    if (p) {
+        Allctr_t *pref_allctr = get_pref_allctr(extra);
+
+        return realloc_thr_pref(type, pref_allctr, p, size, 1);
+    }
+
+    return erts_alcu_alloc_thr_pref(type, extra, size);
 }
 
 
@@ -6071,6 +6285,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 	allctr->t			= 0;
 
     allctr->ramv			= init->ramv;
+    allctr->atags                       = init->atags;
     allctr->main_carrier_size		= init->mmbcs;
 
 #if HAVE_ERTS_MSEG
@@ -6320,127 +6535,1194 @@ erts_alcu_init(AlcUInit_t *init)
     initialized = 1;
 }
 
+/* ------------------------------------------------------------------------- */
 
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- * NOTE:  erts_alcu_test() is only supposed to be used for testing.          *
- *                                                                           *
- * Keep alloc_SUITE_data/allocator_test.h updated if changes are made        *
- * to erts_alcu_test()                                                       *
-\*                                                                           */
+/* Allocation histograms and carrier information is gathered by walking through
+ * all carriers associated with each allocator instance. This is done as
+ * aux_yield_work on the scheduler that owns each instance.
+ *
+ * Yielding is implemented by temporarily inserting a "dummy carrier" at the
+ * last position. It's permanently "busy" so it won't get picked up by someone
+ * else when in the carrier pool, and we never make the employer aware of it
+ * through callbacks so we can't accidentally allocate on it.
+ *
+ * Plain malloc/free is used to guarantee we won't allocate with the allocator
+ * we're scanning. */
 
-UWord
-erts_alcu_test(UWord op, UWord a1, UWord a2)
-{
-    switch (op) {
-    case 0x000:	return (UWord) BLK_SZ((Block_t *) a1);
-    case 0x001:	return (UWord) BLK_UMEM_SZ((Block_t *) a1);
-    case 0x002:	return (UWord) IS_PREV_BLK_FREE((Block_t *) a1);
-    case 0x003:	return (UWord) IS_FREE_BLK((Block_t *) a1);
-    case 0x004:	return (UWord) IS_LAST_BLK((Block_t *) a1);
-    case 0x005:	return (UWord) UMEM2BLK((void *) a1);
-    case 0x006:	return (UWord) BLK2UMEM((Block_t *) a1);
-    case 0x007:	return (UWord) IS_SB_CARRIER((Carrier_t *) a1);
-    case 0x008:	return (UWord) IS_SBC_BLK((Block_t *) a1);
-    case 0x009:	return (UWord) IS_MB_CARRIER((Carrier_t *) a1);
-    case 0x00a:	return (UWord) IS_MSEG_CARRIER((Carrier_t *) a1);
-    case 0x00b:	return (UWord) CARRIER_SZ((Carrier_t *) a1);
-    case 0x00c:	return (UWord) SBC2BLK((Allctr_t *) a1,
-					       (Carrier_t *) a2);
-    case 0x00d:	return (UWord) BLK_TO_SBC((Block_t *) a2);
-    case 0x00e:	return (UWord) MBC_TO_FIRST_BLK((Allctr_t *) a1,
-						(Carrier_t *) a2);
-    case 0x00f:	return (UWord) FIRST_BLK_TO_MBC((Allctr_t *) a1,
-						(Block_t *) a2);
-    case 0x010:	return (UWord) ((Allctr_t *) a1)->mbc_list.first;
-    case 0x011:	return (UWord) ((Allctr_t *) a1)->mbc_list.last;
-    case 0x012:	return (UWord) ((Allctr_t *) a1)->sbc_list.first;
-    case 0x013:	return (UWord) ((Allctr_t *) a1)->sbc_list.last;
-    case 0x014:	return (UWord) ((Carrier_t *) a1)->next;
-    case 0x015:	return (UWord) ((Carrier_t *) a1)->prev;
-    case 0x016:	return (UWord) ABLK_HDR_SZ; 
-    case 0x017:	return (UWord) ((Allctr_t *) a1)->min_block_size;
-    case 0x018:	return (UWord) NXT_BLK((Block_t *) a1);
-    case 0x019:	return (UWord) PREV_BLK((Block_t *) a1);
-    case 0x01a: return (UWord) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2);
-    case 0x01b: return (UWord) sizeof(Unit_t);
-    case 0x01c: return (UWord) BLK_TO_MBC((Block_t*) a1);
-    case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break;
-    case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break;
-    case 0x01f: return (UWord) sizeof(ErtsAlcCrrPool_t);
-    case 0x020:
-	SET_CARRIER_HDR((Carrier_t *) a2, 0, SCH_SYS_ALLOC|SCH_MBC, (Allctr_t *) a1);
-	cpool_init_carrier_data((Allctr_t *) a1, (Carrier_t *) a2);
-	return (UWord) a2;
-    case 0x021:
-	cpool_insert((Allctr_t *) a1, (Carrier_t *) a2);
-	return (UWord) a2;
-    case 0x022:
-	cpool_delete((Allctr_t *) a1, (Allctr_t *) a1, (Carrier_t *) a2);
-	return (UWord) a2;
-    case 0x023: return (UWord) cpool_is_empty((Allctr_t *) a1);
-    case 0x024: return (UWord) cpool_dbg_is_in_pool((Allctr_t *) a1, (Carrier_t *) a2);
-    case 0x025: /* UMEM2BLK_TEST*/
-#ifdef DEBUG
-# ifdef HARD_DEBUG
-	return (UWord)UMEM2BLK(a1-3*sizeof(UWord));
-# else
-	return (UWord)UMEM2BLK(a1-2*sizeof(UWord));
-# endif
+/* Yield between carriers once this many blocks have been processed. Note that
+ * a single carrier scan may exceed this figure. */
+#ifndef DEBUG
+    #define BLOCKSCAN_REDUCTIONS (8000)
 #else
-	return (UWord)UMEM2BLK(a1);
+    #define BLOCKSCAN_REDUCTIONS (400)
 #endif
 
-    default:	ASSERT(0); return ~((UWord) 0);
+/* Abort a single carrier scan after this many blocks to prevent really large
+ * MBCs from blocking forever. */
+#define BLOCKSCAN_BAILOUT_THRESHOLD (16000)
+
+typedef struct alcu_blockscan {
+    /* A per-scheduler list used when multiple scans have been queued. The
+     * current scanner will always run until completion/abort before moving on
+     * to the next. */
+    struct alcu_blockscan *scanner_queue;
+
+    Allctr_t *allocator;
+    Process *process;
+
+    int (*current_op)(struct alcu_blockscan *scanner);
+    int (*next_op)(struct alcu_blockscan *scanner);
+    int reductions;
+
+    ErtsAlcCPoolData_t *cpool_cursor;
+    CarrierList_t *current_clist;
+    Carrier_t *clist_cursor;
+    Carrier_t dummy_carrier;
+
+    /* Called if the process that started this job dies before we're done. */
+    void (*abort)(void *user_data);
+
+    /* Called on each carrier. The callback must return the number of blocks
+     * scanned to yield properly between carriers.
+     *
+     * Note that it's not possible to "yield back" into a carrier. */
+    int (*scan)(Allctr_t *, void *user_data, Carrier_t *);
+
+    /* Called when all carriers have been scanned. The callback may return
+     * non-zero to yield. */
+    int (*finish)(void *user_data);
+
+    void *user_data;
+} blockscan_t;
+
+static Carrier_t *blockscan_restore_clist_cursor(blockscan_t *state)
+{
+    Carrier_t *cursor = state->clist_cursor;
+
+    ASSERT(state->clist_cursor == (state->current_clist)->first ||
+           state->clist_cursor == &state->dummy_carrier);
+
+    if (cursor == &state->dummy_carrier) {
+        cursor = cursor->next;
+
+        unlink_carrier(state->current_clist, state->clist_cursor);
     }
-    return 0;
-}
 
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- * Debug functions                                                           *
-\*                                                                           */
+    return cursor;
+}
 
-void
-erts_alcu_assert_failed(char* expr, char* file, int line, char *func)
+static void blockscan_save_clist_cursor(blockscan_t *state, Carrier_t *after)
 {
-    fflush(stdout);
-    fprintf(stderr, "%s:%d:%s(): Assertion failed: %s\n",
-	    file, line, func, expr);
-    fflush(stderr);
-#if defined(__WIN__) || defined(__WIN32__)
-    DebugBreak();
-#else
-    abort();
-#endif
+    ASSERT(state->clist_cursor == (state->current_clist)->first ||
+           state->clist_cursor == &state->dummy_carrier);
+
+    state->clist_cursor = &state->dummy_carrier;
+
+    (state->clist_cursor)->next = after->next;
+    (state->clist_cursor)->prev = after;
+
+    relink_carrier(state->current_clist, state->clist_cursor);
 }
 
-void
-erts_alcu_verify_unused(Allctr_t *allctr)
+static int blockscan_clist_yielding(blockscan_t *state)
 {
-    UWord no;
+    Carrier_t *cursor = blockscan_restore_clist_cursor(state);
 
-    no = allctr->sbcs.curr.norm.mseg.no;
-    no += allctr->sbcs.curr.norm.sys_alloc.no;
-    no += allctr->mbcs.blocks.curr.no;
+    if (ERTS_PROC_IS_EXITING(state->process)) {
+        return 0;
+    }
 
-    if (no) {
-	UWord sz = allctr->sbcs.blocks.curr.size;
-	sz += allctr->mbcs.blocks.curr.size;
-	erts_exit(ERTS_ABORT_EXIT,
-		 "%salloc() used when expected to be unused!\n"
-		 "Total amount of blocks allocated: %bpu\n"
-		 "Total amount of bytes allocated: %bpu\n",
-		 allctr->name_prefix, no, sz);
+    while (cursor) {
+        /* Skip dummy carriers inserted by another (concurrent) block scan.
+         * This can happen when scanning thread-safe allocators from multiple
+         * schedulers. */
+        if (CARRIER_SZ(cursor) > 0) {
+            int blocks_scanned = state->scan(state->allocator,
+                                             state->user_data,
+                                             cursor);
+
+            state->reductions -= blocks_scanned;
+
+            if (state->reductions <= 0) {
+                blockscan_save_clist_cursor(state, cursor);
+                return 1;
+            }
+        }
+
+        cursor = cursor->next;
     }
+
+    return 0;
 }
 
-void
-erts_alcu_verify_unused_ts(Allctr_t *allctr)
+static ErtsAlcCPoolData_t *blockscan_restore_cpool_cursor(blockscan_t *state)
 {
-    erts_mtx_lock(&allctr->mutex);
-    erts_alcu_verify_unused(allctr);
-    erts_mtx_unlock(&allctr->mutex);
+    ErtsAlcCPoolData_t *cursor;
+
+    cursor = cpool_aint2cpd(cpool_read(&(state->cpool_cursor)->next));
+
+    if (state->cpool_cursor == &state->dummy_carrier.cpool) {
+        cpool_delete(state->allocator, state->allocator, &state->dummy_carrier);
+    }
+
+    return cursor;
 }
 
+static void blockscan_save_cpool_cursor(blockscan_t *state,
+                                        ErtsAlcCPoolData_t *after)
+{
+    ErtsAlcCPoolData_t *dummy_carrier, *prev_carrier, *next_carrier;
+
+    dummy_carrier = &state->dummy_carrier.cpool;
+
+    next_carrier = cpool_aint2cpd(cpool_mod_mark(&after->next));
+    prev_carrier = cpool_aint2cpd(cpool_mod_mark(&next_carrier->prev));
+
+    cpool_init(&dummy_carrier->next, (erts_aint_t)next_carrier);
+    cpool_init(&dummy_carrier->prev, (erts_aint_t)prev_carrier);
+
+    cpool_set_mod_marked(&prev_carrier->next,
+                         (erts_aint_t)dummy_carrier,
+                         (erts_aint_t)next_carrier);
+    cpool_set_mod_marked(&next_carrier->prev,
+                         (erts_aint_t)dummy_carrier,
+                         (erts_aint_t)prev_carrier);
+
+    state->cpool_cursor = dummy_carrier;
+}
+
+static int blockscan_cpool_yielding(blockscan_t *state)
+{
+    ErtsAlcCPoolData_t *sentinel, *cursor;
+
+    sentinel = &carrier_pool[(state->allocator)->alloc_no].sentinel;
+    cursor = blockscan_restore_cpool_cursor(state);
+
+    if (ERTS_PROC_IS_EXITING(state->process)) {
+        return 0;
+    }
+
+    while (cursor != sentinel) {
+        Carrier_t *carrier;
+        erts_aint_t exp;
+
+        /* When a deallocation happens on a pooled carrier it will be routed to
+         * its owner, so the only way to be sure that it isn't modified while
+         * scanning is to skip all carriers that aren't ours. The deallocations
+         * deferred to us will get handled when we're done. */
+        while (cursor->orig_allctr != state->allocator) {
+            cursor = cpool_aint2cpd(cpool_read(&cursor->next));
+
+            if (cursor == sentinel) {
+                return 0;
+            }
+        }
+
+        carrier = ErtsContainerStruct(cursor, Carrier_t, cpool);
+        exp = erts_atomic_read_rb(&carrier->allctr);
+
+        if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) {
+            ASSERT(state->allocator == (Allctr_t*)(exp & ~ERTS_CRR_ALCTR_FLG_MASK));
+            ASSERT(!(exp & ERTS_CRR_ALCTR_FLG_BUSY));
+
+            if (erts_atomic_cmpxchg_acqb(&carrier->allctr,
+                                         exp | ERTS_CRR_ALCTR_FLG_BUSY,
+                                         exp) == exp) {
+                /* Skip dummy carriers inserted by another (concurrent) block
+                 * scan. This can happen when scanning thread-safe allocators
+                 * from multiple schedulers. */
+                if (CARRIER_SZ(carrier) > 0) {
+                    int blocks_scanned = state->scan(state->allocator,
+                                                     state->user_data,
+                                                     carrier);
+
+                    state->reductions -= blocks_scanned;
+
+                    if (state->reductions <= 0) {
+                        blockscan_save_cpool_cursor(state, cursor);
+                        erts_atomic_set_relb(&carrier->allctr, exp);
+
+                        return 1;
+                    }
+                }
+
+                erts_atomic_set_relb(&carrier->allctr, exp);
+            }
+        }
+
+        cursor = cpool_aint2cpd(cpool_read(&cursor->next));
+    }
+
+    return 0;
+}
+
+static int blockscan_yield_helper(blockscan_t *state,
+                                  int (*yielding_op)(blockscan_t*))
+{
+    /* Note that we don't check whether to abort here; only yielding_op knows
+     * whether the carrier is still in the list/pool. */
+
+    if ((state->allocator)->thread_safe) {
+        /* Locked scans have to be as short as possible. */
+        state->reductions = 1;
+
+        erts_mtx_lock(&(state->allocator)->mutex);
+    } else {
+        state->reductions = BLOCKSCAN_REDUCTIONS;
+    }
+
+    if (yielding_op(state)) {
+        state->next_op = state->current_op;
+    }
+
+    if ((state->allocator)->thread_safe) {
+        erts_mtx_unlock(&(state->allocator)->mutex);
+    }
+
+    return 1;
+}
+
+/* */
+
+static int blockscan_finish(blockscan_t *state)
+{
+    if (ERTS_PROC_IS_EXITING(state->process)) {
+        state->abort(state->user_data);
+        return 0;
+    }
+
+    state->current_op = blockscan_finish;
+
+    return state->finish(state->user_data);
+}
+
+static int blockscan_sweep_sbcs(blockscan_t *state)
+{
+    if (state->current_op != blockscan_sweep_sbcs) {
+        SET_CARRIER_HDR(&state->dummy_carrier, 0, SCH_SBC, state->allocator);
+        state->current_clist = &(state->allocator)->sbc_list;
+        state->clist_cursor = (state->current_clist)->first;
+    }
+
+    state->current_op = blockscan_sweep_sbcs;
+    state->next_op = blockscan_finish;
+
+    return blockscan_yield_helper(state, blockscan_clist_yielding);
+}
+
+static int blockscan_sweep_mbcs(blockscan_t *state)
+{
+    if (state->current_op != blockscan_sweep_mbcs) {
+        SET_CARRIER_HDR(&state->dummy_carrier, 0, SCH_MBC, state->allocator);
+        state->current_clist = &(state->allocator)->mbc_list;
+        state->clist_cursor = (state->current_clist)->first;
+    }
+
+    state->current_op = blockscan_sweep_mbcs;
+    state->next_op = blockscan_sweep_sbcs;
+
+    return blockscan_yield_helper(state, blockscan_clist_yielding);
+}
+
+static int blockscan_sweep_cpool(blockscan_t *state)
+{
+    if (state->current_op != blockscan_sweep_cpool) {
+        ErtsAlcCPoolData_t *sentinel;
+
+        SET_CARRIER_HDR(&state->dummy_carrier, 0, SCH_MBC, state->allocator);
+        sentinel = &carrier_pool[(state->allocator)->alloc_no].sentinel;
+        state->cpool_cursor = sentinel;
+    }
+
+    state->current_op = blockscan_sweep_cpool;
+    state->next_op = blockscan_sweep_mbcs;
+
+    return blockscan_yield_helper(state, blockscan_cpool_yielding);
+}
+
+static int blockscan_get_specific_allocator(int allocator_num,
+                                            int sched_id,
+                                            Allctr_t **out)
+{
+    ErtsAllocatorInfo_t *ai;
+    Allctr_t *allocator;
+
+    ASSERT(allocator_num >= ERTS_ALC_A_MIN &&
+           allocator_num <= ERTS_ALC_A_MAX);
+    ASSERT(sched_id >= 0 && sched_id <= erts_no_schedulers);
+
+    ai = &erts_allctrs_info[allocator_num];
+
+    if (!ai->enabled || !ai->alloc_util) {
+        return 0;
+    }
+
+    if (!ai->thr_spec) {
+        if (sched_id != 0) {
+            /* Only thread-specific allocators can be scanned on a specific
+             * scheduler. */
+            return 0;
+        }
+
+        allocator = (Allctr_t*)ai->extra;
+        ASSERT(allocator->thread_safe);
+    } else {
+        ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t*)ai->extra;
+
+        ASSERT(sched_id < tspec->size);
+
+        allocator = tspec->allctr[sched_id];
+    }
+
+    *out = allocator;
+
+    return 1;
+}
+
+static void blockscan_sched_trampoline(void *arg)
+{
+    ErtsAlcuBlockscanYieldData *yield;
+    ErtsSchedulerData *esdp;
+    blockscan_t *scanner;
+
+    esdp = erts_get_scheduler_data();
+    scanner = (blockscan_t*)arg;
+
+    yield = ERTS_SCHED_AUX_YIELD_DATA(esdp, alcu_blockscan);
+
+    ASSERT((yield->last == NULL) == (yield->current == NULL));
+
+    if (yield->last != NULL) {
+        blockscan_t *prev_scanner = yield->last;
+
+        ASSERT(prev_scanner->scanner_queue == NULL);
+
+        prev_scanner->scanner_queue = scanner;
+    } else {
+        yield->current = scanner;
+    }
+
+    scanner->scanner_queue = NULL;
+    yield->last = scanner;
+
+    erts_notify_new_aux_yield_work(esdp);
+}
+
+static void blockscan_dispatch(blockscan_t *scanner, Process *owner,
+                               Allctr_t *allocator, int sched_id)
+{
+    ASSERT(erts_get_scheduler_id() != 0);
+
+    if (sched_id == 0) {
+        /* Global instances are always handled on the current scheduler. */
+        sched_id = ERTS_ALC_GET_THR_IX();
+        ASSERT(allocator->thread_safe);
+    }
+
+    scanner->allocator = allocator;
+    scanner->process = owner;
+
+    erts_proc_inc_refc(scanner->process);
+
+    cpool_init_carrier_data(scanner->allocator, &scanner->dummy_carrier);
+    erts_atomic_init_nob(&(scanner->dummy_carrier).allctr,
+                         (erts_aint_t)allocator | ERTS_CRR_ALCTR_FLG_BUSY);
+
+    if (ERTS_ALC_IS_CPOOL_ENABLED(scanner->allocator)) {
+        scanner->next_op = blockscan_sweep_cpool;
+    } else {
+        scanner->next_op = blockscan_sweep_mbcs;
+    }
+
+    /* Aux yield jobs can only be set up while running on the scheduler that
+     * services them, so we move there before continuing.
+     *
+     * We can't drive the scan itself through this since the scheduler will
+     * always finish *all* misc aux work in one go which makes it impossible to
+     * yield. */
+    erts_schedule_misc_aux_work(sched_id, blockscan_sched_trampoline, scanner);
+}
+
+int erts_handle_yielded_alcu_blockscan(ErtsSchedulerData *esdp,
+                                       ErtsAlcuBlockscanYieldData *yield)
+{
+    blockscan_t *scanner = yield->current;
+
+    (void)esdp;
+
+    ASSERT((yield->last == NULL) == (yield->current == NULL));
+
+    if (scanner) {
+        if (scanner->next_op(scanner)) {
+            return 1;
+        }
+
+        ASSERT(ERTS_PROC_IS_EXITING(scanner->process) ||
+               scanner->current_op == blockscan_finish);
+
+        yield->current = scanner->scanner_queue;
+
+        if (yield->current == NULL) {
+            ASSERT(scanner == yield->last);
+            yield->last = NULL;
+        }
+
+        erts_proc_dec_refc(scanner->process);
+
+        /* Plain free is intentional. */
+        free(scanner);
+
+        return yield->current != NULL;
+    }
+
+    return 0;
+}
+
+void erts_alcu_sched_spec_data_init(ErtsSchedulerData *esdp)
+{
+    ErtsAlcuBlockscanYieldData *yield;
+
+    yield = ERTS_SCHED_AUX_YIELD_DATA(esdp, alcu_blockscan);
+
+    yield->current = NULL;
+    yield->last = NULL;
+}
+
+/* ------------------------------------------------------------------------- */
+
+static ERTS_INLINE int u64_log2(Uint64 v)
+{
+    static const int log2_tab64[64] = {
+        63,  0, 58,  1, 59, 47, 53,  2,
+        60, 39, 48, 27, 54, 33, 42,  3,
+        61, 51, 37, 40, 49, 18, 28, 20,
+        55, 30, 34, 11, 43, 14, 22,  4,
+        62, 57, 46, 52, 38, 26, 32, 41,
+        50, 36, 17, 19, 29, 10, 13, 21,
+        56, 45, 25, 31, 35, 16,  9, 12,
+        44, 24, 15,  8, 23,  7,  6,  5};
+
+    v |= v >> 1;
+    v |= v >> 2;
+    v |= v >> 4;
+    v |= v >> 8;
+    v |= v >> 16;
+    v |= v >> 32;
+
+    return log2_tab64[((Uint64)((v - (v >> 1))*0x07EDD5E59A4E28C2)) >> 58];
+}
+
+/* ------------------------------------------------------------------------- */
+
+typedef struct hist_tree__ {
+    struct hist_tree__ *parent;
+    struct hist_tree__ *left;
+    struct hist_tree__ *right;
+
+    int is_red;
+
+    alcu_atag_t tag;
+    UWord histogram[1];
+} hist_tree_t;
+
+#define ERTS_RBT_PREFIX hist_tree
+#define ERTS_RBT_T hist_tree_t
+#define ERTS_RBT_KEY_T UWord
+#define ERTS_RBT_FLAGS_T int
+#define ERTS_RBT_INIT_EMPTY_TNODE(T) ((void)0)
+#define ERTS_RBT_IS_RED(T) ((T)->is_red)
+#define ERTS_RBT_SET_RED(T) ((T)->is_red = 1)
+#define ERTS_RBT_IS_BLACK(T) (!ERTS_RBT_IS_RED(T))
+#define ERTS_RBT_SET_BLACK(T) ((T)->is_red = 0)
+#define ERTS_RBT_GET_FLAGS(T) ((T)->is_red)
+#define ERTS_RBT_SET_FLAGS(T, F) ((T)->is_red = F)
+#define ERTS_RBT_GET_PARENT(T) ((T)->parent)
+#define ERTS_RBT_SET_PARENT(T, P) ((T)->parent = P)
+#define ERTS_RBT_GET_RIGHT(T) ((T)->right)
+#define ERTS_RBT_SET_RIGHT(T, R) ((T)->right = (R))
+#define ERTS_RBT_GET_LEFT(T) ((T)->left)
+#define ERTS_RBT_SET_LEFT(T, L) ((T)->left = (L))
+#define ERTS_RBT_GET_KEY(T) ((T)->tag)
+#define ERTS_RBT_IS_LT(KX, KY) (KX < KY)
+#define ERTS_RBT_IS_EQ(KX, KY) (KX == KY)
+#define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING
+#define ERTS_RBT_WANT_FOREACH_DESTROY
+#define ERTS_RBT_WANT_INSERT
+#define ERTS_RBT_WANT_LOOKUP
+#define ERTS_RBT_UNDEF
+
+#include "erl_rbtree.h"
+
+typedef struct {
+    blockscan_t common;
+
+    ErtsIRefStorage iref;
+    Process *process;
+
+    hist_tree_rbt_yield_state_t hist_tree_yield;
+    hist_tree_t *hist_tree;
+    UWord hist_count;
+
+    UWord hist_slot_start;
+    int hist_slot_count;
+
+    UWord unscanned_size;
+
+    ErtsHeapFactory msg_factory;
+    int building_result;
+    Eterm result_list;
+} gather_ahist_t;
+
+static void gather_ahist_update(gather_ahist_t *state, UWord tag, UWord size)
+{
+    hist_tree_t *hist_node;
+    UWord size_interval;
+    int hist_slot;
+
+    hist_node = hist_tree_rbt_lookup(state->hist_tree, tag);
+
+    if (hist_node == NULL) {
+        /* Plain calloc is intentional. */
+        hist_node = (hist_tree_t*)calloc(1, sizeof(hist_tree_t) +
+                                            (state->hist_slot_count - 1) *
+                                            sizeof(hist_node->histogram[0]));
+        hist_node->tag = tag;
+
+        hist_tree_rbt_insert(&state->hist_tree, hist_node);
+        state->hist_count++;
+    }
+
+    size_interval = (size / state->hist_slot_start);
+    size_interval = u64_log2(size_interval + 1);
+
+    hist_slot = MIN(size_interval, state->hist_slot_count - 1);
+
+    hist_node->histogram[hist_slot]++;
+}
+
+static int gather_ahist_scan(Allctr_t *allocator,
+                             void *user_data,
+                             Carrier_t *carrier)
+{
+    gather_ahist_t *state;
+    int blocks_scanned;
+    Block_t *block;
+
+    state = (gather_ahist_t*)user_data;
+    blocks_scanned = 1;
+
+    if (IS_SB_CARRIER(carrier)) {
+        alcu_atag_t tag;
+
+        block = SBC2BLK(allocator, carrier);
+        tag = GET_BLK_ATAG(block);
+
+        ASSERT(DBG_IS_VALID_ATAG(allocator, tag));
+
+        gather_ahist_update(state, tag, SBC_BLK_SZ(block));
+    } else {
+        UWord scanned_bytes = MBC_HEADER_SIZE(allocator);
+
+        ASSERT(IS_MB_CARRIER(carrier));
+
+        block = MBC_TO_FIRST_BLK(allocator, carrier);
+
+        while (1) {
+            UWord block_size = MBC_BLK_SZ(block);
+
+            if (IS_ALLOCED_BLK(block)) {
+                alcu_atag_t tag = GET_BLK_ATAG(block);
+
+                ASSERT(DBG_IS_VALID_ATAG(allocator, tag));
+
+                gather_ahist_update(state, tag, block_size);
+            }
+
+            scanned_bytes += block_size;
+
+            if (blocks_scanned >= BLOCKSCAN_BAILOUT_THRESHOLD) {
+                state->unscanned_size += CARRIER_SZ(carrier) - scanned_bytes;
+                break;
+            } else if (IS_LAST_BLK(block)) {
+                break;
+            }
+
+            block = NXT_BLK(block);
+            blocks_scanned++;
+        }
+    }
+
+    return blocks_scanned;
+}
+
+static void gather_ahist_append_result(hist_tree_t *node, void *arg)
+{
+    gather_ahist_t *state = (gather_ahist_t*)arg;
+
+    Eterm histogram_tuple, tag_tuple;
+
+    Eterm *hp;
+    int ix;
+
+    ASSERT(state->building_result);
+
+    hp = erts_produce_heap(&state->msg_factory, 7 + state->hist_slot_count, 0);
+
+    hp[0] = make_arityval(state->hist_slot_count);
+
+    for (ix = 0; ix < state->hist_slot_count; ix++) {
+        hp[1 + ix] = make_small(node->histogram[ix]);
+    }
+
+    histogram_tuple = make_tuple(hp);
+    hp += 1 + state->hist_slot_count;
+
+    hp[0] = make_arityval(3);
+    hp[1] = ATAG_ID(node->tag);
+    hp[2] = alloc_type_atoms[ATAG_TYPE(node->tag)];
+    hp[3] = histogram_tuple;
+
+    tag_tuple = make_tuple(hp);
+    hp += 4;
+
+    state->result_list = CONS(hp, tag_tuple, state->result_list);
+
+    /* Plain free is intentional. */
+    free(node);
+}
+
+static void gather_ahist_send(gather_ahist_t *state)
+{
+    Eterm result_tuple, unscanned_size, task_ref;
+
+    Uint term_size;
+    Eterm *hp;
+
+    ASSERT((state->result_list == NIL) ^ (state->hist_count > 0));
+    ASSERT(state->building_result);
+
+    term_size = 4 + erts_iref_storage_heap_size(&state->iref);
+    term_size += IS_USMALL(0, state->unscanned_size) ? 0 : BIG_UINT_HEAP_SIZE;
+
+    hp = erts_produce_heap(&state->msg_factory, term_size, 0);
+
+    task_ref = erts_iref_storage_make_ref(&state->iref, &hp,
+        &(state->msg_factory.message)->hfrag.off_heap, 0);
+
+    unscanned_size = bld_unstable_uint(&hp, NULL, state->unscanned_size);
+
+    hp[0] = make_arityval(3);
+    hp[1] = task_ref;
+    hp[2] = unscanned_size;
+    hp[3] = state->result_list;
+
+    result_tuple = make_tuple(hp);
+
+    erts_factory_trim_and_close(&state->msg_factory, &result_tuple, 1);
+
+    erts_queue_message(state->process, 0, state->msg_factory.message,
+                       result_tuple, am_system);
+}
+
+static int gather_ahist_finish(void *arg)
+{
+    gather_ahist_t *state = (gather_ahist_t*)arg;
+
+    if (!state->building_result) {
+        ErtsMessage *message;
+        Uint minimum_size;
+        Eterm *hp;
+
+        /* {Ref, unscanned size, [{Tag, {Histogram}} | Rest]} */
+        minimum_size = 4 + erts_iref_storage_heap_size(&state->iref) +
+                       state->hist_count * (7 + state->hist_slot_count);
+
+        message = erts_alloc_message(minimum_size, &hp);
+        erts_factory_selfcontained_message_init(&state->msg_factory,
+                                                message, hp);
+
+        ERTS_RBT_YIELD_STAT_INIT(&state->hist_tree_yield);
+
+        state->result_list = NIL;
+        state->building_result = 1;
+    }
+
+    if (hist_tree_rbt_foreach_destroy_yielding(&state->hist_tree,
+                                               &gather_ahist_append_result,
+                                               state,
+                                               &state->hist_tree_yield,
+                                               BLOCKSCAN_REDUCTIONS)) {
+        return 1;
+    }
+
+    gather_ahist_send(state);
+
+    return 0;
+}
+
+static void gather_ahist_destroy_result(hist_tree_t *node, void *arg)
+{
+    (void)arg;
+    free(node);
+}
+
+static void gather_ahist_abort(void *arg)
+{
+    gather_ahist_t *state = (gather_ahist_t*)arg;
+
+    if (state->building_result) {
+        erts_factory_undo(&state->msg_factory);
+    }
+
+    hist_tree_rbt_foreach_destroy(&state->hist_tree,
+                                  &gather_ahist_destroy_result,
+                                  NULL);
+}
+
+int erts_alcu_gather_alloc_histograms(Process *p, int allocator_num,
+                                      int sched_id, int hist_width,
+                                      UWord hist_start, Eterm ref)
+{
+    gather_ahist_t *gather_state;
+    blockscan_t *scanner;
+    Allctr_t *allocator;
+
+    ASSERT(is_internal_ref(ref));
+
+    if (!blockscan_get_specific_allocator(allocator_num,
+                                          sched_id,
+                                          &allocator)) {
+        return 0;
+    } else if (!allocator->atags) {
+        return 0;
+    }
+
+    ensure_atoms_initialized(allocator);
+
+    /* Plain calloc is intentional. */
+    gather_state = (gather_ahist_t*)calloc(1, sizeof(gather_ahist_t));
+    scanner = &gather_state->common;
+
+    scanner->abort = gather_ahist_abort;
+    scanner->scan = gather_ahist_scan;
+    scanner->finish = gather_ahist_finish;
+    scanner->user_data = gather_state;
+
+    erts_iref_storage_save(&gather_state->iref, ref);
+    gather_state->hist_slot_start = hist_start;
+    gather_state->hist_slot_count = hist_width;
+    gather_state->process = p;
+
+    blockscan_dispatch(scanner, p, allocator, sched_id);
+
+    return 1;
+}
+
+/* ------------------------------------------------------------------------- */
+
+typedef struct chist_node__ {
+    struct chist_node__ *next;
+
+    UWord carrier_size;
+    UWord unscanned_size;
+    UWord allocated_size;
+
+    /* BLOCKSCAN_BAILOUT_THRESHOLD guarantees we won't overflow this or the
+     * counters in the free block histogram. */
+    int allocated_count;
+    int flags;
+
+    int histogram[1];
+} chist_node_t;
+
+typedef struct {
+    blockscan_t common;
+
+    ErtsIRefStorage iref;
+    Process *process;
+
+    Eterm allocator_desc;
+
+    chist_node_t *info_list;
+    UWord info_count;
+
+    UWord hist_slot_start;
+    int hist_slot_count;
+
+    ErtsHeapFactory msg_factory;
+    int building_result;
+    Eterm result_list;
+} gather_cinfo_t;
+
+static int gather_cinfo_scan(Allctr_t *allocator,
+                             void *user_data,
+                             Carrier_t *carrier)
+{
+    gather_cinfo_t *state;
+    chist_node_t *node;
+    int blocks_scanned;
+    Block_t *block;
+
+    state = (gather_cinfo_t*)user_data;
+    node = calloc(1, sizeof(chist_node_t) +
+                     (state->hist_slot_count - 1) *
+                     sizeof(node->histogram[0]));
+    blocks_scanned = 1;
+
+    /* ERTS_CRR_ALCTR_FLG_BUSY is ignored since we've set it ourselves and it
+     * would be misleading to include it. */
+    node->flags = erts_atomic_read_rb(&carrier->allctr) &
+                  (ERTS_CRR_ALCTR_FLG_MASK & ~ERTS_CRR_ALCTR_FLG_BUSY);
+    node->carrier_size = CARRIER_SZ(carrier);
+
+    if (IS_SB_CARRIER(carrier)) {
+        UWord block_size;
+
+        block = SBC2BLK(allocator, carrier);
+        block_size = SBC_BLK_SZ(block);
+
+        node->allocated_size = block_size;
+        node->allocated_count = 1;
+    } else {
+        UWord scanned_bytes = MBC_HEADER_SIZE(allocator);
+
+        block = MBC_TO_FIRST_BLK(allocator, carrier);
+
+        while (1) {
+            UWord block_size = MBC_BLK_SZ(block);
+
+            scanned_bytes += block_size;
+
+            if (IS_ALLOCED_BLK(block)) {
+                node->allocated_size += block_size;
+                node->allocated_count++;
+            } else {
+                UWord size_interval;
+                int hist_slot;
+
+                size_interval = (block_size / state->hist_slot_start);
+                size_interval = u64_log2(size_interval + 1);
+
+                hist_slot = MIN(size_interval, state->hist_slot_count - 1);
+
+                node->histogram[hist_slot]++;
+            }
+
+            if (blocks_scanned >= BLOCKSCAN_BAILOUT_THRESHOLD) {
+                node->unscanned_size += CARRIER_SZ(carrier) - scanned_bytes;
+                break;
+            } else if (IS_LAST_BLK(block)) {
+                break;
+            }
+
+            block = NXT_BLK(block);
+            blocks_scanned++;
+        }
+    }
+
+    node->next = state->info_list;
+    state->info_list = node;
+    state->info_count++;
+
+    return blocks_scanned;
+}
+
+static void gather_cinfo_append_result(gather_cinfo_t *state,
+                                       chist_node_t *info)
+{
+    Eterm carrier_size, unscanned_size, allocated_size;
+    Eterm histogram_tuple, carrier_tuple;
+
+    Uint term_size;
+    Eterm *hp;
+    int ix;
+
+    ASSERT(state->building_result);
+
+    term_size = 11 + state->hist_slot_count;
+    term_size += IS_USMALL(0, info->carrier_size) ? 0 : BIG_UINT_HEAP_SIZE;
+    term_size += IS_USMALL(0, info->unscanned_size) ? 0 : BIG_UINT_HEAP_SIZE;
+    term_size += IS_USMALL(0, info->allocated_size) ? 0 : BIG_UINT_HEAP_SIZE;
+
+    hp = erts_produce_heap(&state->msg_factory, term_size, 0);
+
+    hp[0] = make_arityval(state->hist_slot_count);
+
+    for (ix = 0; ix < state->hist_slot_count; ix++) {
+        hp[1 + ix] = make_small(info->histogram[ix]);
+    }
+
+    histogram_tuple = make_tuple(hp);
+    hp += 1 + state->hist_slot_count;
+
+    carrier_size = bld_unstable_uint(&hp, NULL, info->carrier_size);
+    unscanned_size = bld_unstable_uint(&hp, NULL, info->unscanned_size);
+    allocated_size = bld_unstable_uint(&hp, NULL, info->allocated_size);
+
+    hp[0] = make_arityval(7);
+    hp[1] = state->allocator_desc;
+    hp[2] = carrier_size;
+    hp[3] = unscanned_size;
+    hp[4] = allocated_size;
+    hp[5] = make_small(info->allocated_count);
+    hp[6] = (info->flags & ERTS_CRR_ALCTR_FLG_IN_POOL) ? am_true : am_false;
+    hp[7] = histogram_tuple;
+
+    carrier_tuple = make_tuple(hp);
+    hp += 8;
+
+    state->result_list = CONS(hp, carrier_tuple, state->result_list);
+
+    free(info);
+}
+
+static void gather_cinfo_send(gather_cinfo_t *state)
+{
+    Eterm result_tuple, task_ref;
+
+    int term_size;
+    Eterm *hp;
+
+    ASSERT((state->result_list == NIL) ^ (state->info_count > 0));
+    ASSERT(state->building_result);
+
+    term_size = 3 + erts_iref_storage_heap_size(&state->iref);
+    hp = erts_produce_heap(&state->msg_factory, term_size, 0);
+
+    task_ref = erts_iref_storage_make_ref(&state->iref, &hp,
+        &(state->msg_factory.message)->hfrag.off_heap, 0);
+
+    hp[0] = make_arityval(2);
+    hp[1] = task_ref;
+    hp[2] = state->result_list;
+
+    result_tuple = make_tuple(hp);
+
+    erts_factory_trim_and_close(&state->msg_factory, &result_tuple, 1);
+
+    erts_queue_message(state->process, 0, state->msg_factory.message,
+                       result_tuple, am_system);
+}
+
+static int gather_cinfo_finish(void *arg)
+{
+    gather_cinfo_t *state = (gather_cinfo_t*)arg;
+    int reductions = BLOCKSCAN_REDUCTIONS;
+
+    if (!state->building_result) {
+        ErtsMessage *message;
+        Uint minimum_size;
+        Eterm *hp;
+
+        /* {Ref, [{Carrier size, unscanned size, allocated size,
+         *         allocated block count, {Free block histogram}} | Rest]} */
+        minimum_size = 3 + erts_iref_storage_heap_size(&state->iref) +
+                       state->info_count * (11 + state->hist_slot_count);
+
+        message = erts_alloc_message(minimum_size, &hp);
+        erts_factory_selfcontained_message_init(&state->msg_factory,
+                                                message, hp);
+
+        state->result_list = NIL;
+        state->building_result = 1;
+    }
+
+    while (state->info_list) {
+        chist_node_t *current = state->info_list;
+        state->info_list = current->next;
+
+        gather_cinfo_append_result(state, current);
+
+        if (reductions-- <= 0) {
+            return 1;
+        }
+    }
+
+    gather_cinfo_send(state);
+
+    return 0;
+}
+
+static void gather_cinfo_abort(void *arg)
+{
+    gather_cinfo_t *state = (gather_cinfo_t*)arg;
+
+    if (state->building_result) {
+        erts_factory_undo(&state->msg_factory);
+    }
+
+    while (state->info_list) {
+        chist_node_t *current = state->info_list;
+        state->info_list = current->next;
+
+        free(current);
+    }
+}
+
+int erts_alcu_gather_carrier_info(struct process *p, int allocator_num,
+                                  int sched_id, int hist_width,
+                                  UWord hist_start, Eterm ref)
+{
+    gather_cinfo_t *gather_state;
+    blockscan_t *scanner;
+
+    const char *allocator_desc;
+    Allctr_t *allocator;
+
+    ASSERT(is_internal_ref(ref));
+
+    if (!blockscan_get_specific_allocator(allocator_num,
+                                          sched_id,
+                                          &allocator)) {
+        return 0;
+    }
+
+    allocator_desc = ERTS_ALC_A2AD(allocator_num);
+
+    /* Plain calloc is intentional. */
+    gather_state = (gather_cinfo_t*)calloc(1, sizeof(gather_cinfo_t));
+    scanner = &gather_state->common;
+
+    scanner->abort = gather_cinfo_abort;
+    scanner->scan = gather_cinfo_scan;
+    scanner->finish = gather_cinfo_finish;
+    scanner->user_data = gather_state;
+
+    gather_state->allocator_desc = erts_atom_put((byte *)allocator_desc,
+                                                 sys_strlen(allocator_desc),
+                                                 ERTS_ATOM_ENC_LATIN1, 1);
+    erts_iref_storage_save(&gather_state->iref, ref);
+    gather_state->hist_slot_start = hist_start * 2;
+    gather_state->hist_slot_count = hist_width;
+    gather_state->process = p;
+
+    blockscan_dispatch(scanner, p, allocator, sched_id);
+
+    return 1;
+}
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * NOTE:  erts_alcu_test() is only supposed to be used for testing.          *
+ *                                                                           *
+ * Keep alloc_SUITE_data/allocator_test.h updated if changes are made        *
+ * to erts_alcu_test()                                                       *
+\*                                                                           */
+
+UWord
+erts_alcu_test(UWord op, UWord a1, UWord a2)
+{
+    switch (op) {
+    case 0x000:	return (UWord) BLK_SZ((Block_t *) a1);
+    case 0x001:	return (UWord) BLK_UMEM_SZ((Block_t *) a1);
+    case 0x002:	return (UWord) IS_PREV_BLK_FREE((Block_t *) a1);
+    case 0x003:	return (UWord) IS_FREE_BLK((Block_t *) a1);
+    case 0x004:	return (UWord) IS_LAST_BLK((Block_t *) a1);
+    case 0x005:	return (UWord) UMEM2BLK((void *) a1);
+    case 0x006:	return (UWord) BLK2UMEM((Block_t *) a1);
+    case 0x007:	return (UWord) IS_SB_CARRIER((Carrier_t *) a1);
+    case 0x008:	return (UWord) IS_SBC_BLK((Block_t *) a1);
+    case 0x009:	return (UWord) IS_MB_CARRIER((Carrier_t *) a1);
+    case 0x00a:	return (UWord) IS_MSEG_CARRIER((Carrier_t *) a1);
+    case 0x00b:	return (UWord) CARRIER_SZ((Carrier_t *) a1);
+    case 0x00c:	return (UWord) SBC2BLK((Allctr_t *) a1,
+					       (Carrier_t *) a2);
+    case 0x00d:	return (UWord) BLK_TO_SBC((Block_t *) a2);
+    case 0x00e:	return (UWord) MBC_TO_FIRST_BLK((Allctr_t *) a1,
+						(Carrier_t *) a2);
+    case 0x00f:	return (UWord) FIRST_BLK_TO_MBC((Allctr_t *) a1,
+						(Block_t *) a2);
+    case 0x010:	return (UWord) ((Allctr_t *) a1)->mbc_list.first;
+    case 0x011:	return (UWord) ((Allctr_t *) a1)->mbc_list.last;
+    case 0x012:	return (UWord) ((Allctr_t *) a1)->sbc_list.first;
+    case 0x013:	return (UWord) ((Allctr_t *) a1)->sbc_list.last;
+    case 0x014:	return (UWord) ((Carrier_t *) a1)->next;
+    case 0x015:	return (UWord) ((Carrier_t *) a1)->prev;
+    case 0x016:	return (UWord) ABLK_HDR_SZ; 
+    case 0x017:	return (UWord) ((Allctr_t *) a1)->min_block_size;
+    case 0x018:	return (UWord) NXT_BLK((Block_t *) a1);
+    case 0x019:	return (UWord) PREV_BLK((Block_t *) a1);
+    case 0x01a: return (UWord) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2);
+    case 0x01b: return (UWord) sizeof(Unit_t);
+    case 0x01c: return (UWord) BLK_TO_MBC((Block_t*) a1);
+    case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break;
+    case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break;
+    case 0x01f: return (UWord) sizeof(ErtsAlcCrrPool_t);
+    case 0x020:
+	SET_CARRIER_HDR((Carrier_t *) a2, 0, SCH_SYS_ALLOC|SCH_MBC, (Allctr_t *) a1);
+	cpool_init_carrier_data((Allctr_t *) a1, (Carrier_t *) a2);
+	return (UWord) a2;
+    case 0x021:
+	cpool_insert((Allctr_t *) a1, (Carrier_t *) a2);
+	return (UWord) a2;
+    case 0x022:
+	cpool_delete((Allctr_t *) a1, (Allctr_t *) a1, (Carrier_t *) a2);
+	return (UWord) a2;
+    case 0x023: return (UWord) cpool_is_empty((Allctr_t *) a1);
+    case 0x024: return (UWord) cpool_dbg_is_in_pool((Allctr_t *) a1, (Carrier_t *) a2);
+    case 0x025: /* UMEM2BLK_TEST*/
+#ifdef DEBUG
+# ifdef HARD_DEBUG
+	return (UWord)UMEM2BLK(a1-3*sizeof(UWord));
+# else
+	return (UWord)UMEM2BLK(a1-2*sizeof(UWord));
+# endif
+#else
+	return (UWord)UMEM2BLK(a1);
+#endif
+
+    default:	ASSERT(0); return ~((UWord) 0);
+    }
+    return 0;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Debug functions                                                           *
+\*                                                                           */
+
+void
+erts_alcu_assert_failed(char* expr, char* file, int line, char *func)
+{
+    fflush(stdout);
+    fprintf(stderr, "%s:%d:%s(): Assertion failed: %s\n",
+	    file, line, func, expr);
+    fflush(stderr);
+#if defined(__WIN__) || defined(__WIN32__)
+    DebugBreak();
+#else
+    abort();
+#endif
+}
+
+void
+erts_alcu_verify_unused(Allctr_t *allctr)
+{
+    UWord no;
+
+    no = allctr->sbcs.curr.norm.mseg.no;
+    no += allctr->sbcs.curr.norm.sys_alloc.no;
+    no += allctr->mbcs.blocks.curr.no;
+
+    if (no) {
+	UWord sz = allctr->sbcs.blocks.curr.size;
+	sz += allctr->mbcs.blocks.curr.size;
+	erts_exit(ERTS_ABORT_EXIT,
+		 "%salloc() used when expected to be unused!\n"
+		 "Total amount of blocks allocated: %bpu\n"
+		 "Total amount of bytes allocated: %bpu\n",
+		 allctr->name_prefix, no, sz);
+    }
+}
+
+void
+erts_alcu_verify_unused_ts(Allctr_t *allctr)
+{
+    erts_mtx_lock(&allctr->mutex);
+    erts_alcu_verify_unused(allctr);
+    erts_mtx_unlock(&allctr->mutex);
+}
+
+
 #ifdef DEBUG
 int is_sbc_blk(Block_t* blk)
 {
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 05c8a0db3b..ff4d10b206 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -50,6 +50,7 @@ typedef struct {
     int tspec;
     int tpref;
     int ramv;
+    int atags;
     UWord sbct;
     UWord asbcst;
     UWord rsbcst;
@@ -106,6 +107,7 @@ typedef struct {
     0,			/* (bool)   tspec:  thread specific              */\
     0,			/* (bool)   tpref:  thread preferred             */\
     0,			/* (bool)   ramv:   realloc always moves         */\
+    0,			/* (bool)   atags:  tagged allocations           */\
     512*1024,		/* (bytes)  sbct:   sbc threshold                */\
     2*1024*2024,	/* (amount) asbcst: abs sbc shrink threshold     */\
     20,			/* (%)      rsbcst: rel sbc shrink threshold     */\
@@ -142,6 +144,7 @@ typedef struct {
     0,			/* (bool)   tspec:  thread specific              */\
     0,			/* (bool)   tpref:  thread preferred             */\
     0,			/* (bool)   ramv:   realloc always moves         */\
+    0,			/* (bool)   atags:  tagged allocations           */\
     64*1024,		/* (bytes)  sbct:   sbc threshold                */\
     2*1024*2024,	/* (amount) asbcst: abs sbc shrink threshold     */\
     20,			/* (%)      rsbcst: rel sbc shrink threshold     */\
@@ -224,6 +227,36 @@ void erts_lcnt_update_allocator_locks(int enable);
 
 int erts_alcu_try_set_dyn_param(Allctr_t*, Eterm param, Uint value);
 
+/* Gathers per-tag allocation histograms from the given allocator number
+ * (ERTS_ALC_A_*) and scheduler id. An id of 0 means the global instance will
+ * be used.
+ *
+ * The results are sent to `p`, and it returns the number of messages to wait
+ * for. */
+int erts_alcu_gather_alloc_histograms(struct process *p, int allocator_num,
+                                      int sched_id, int hist_width,
+                                      UWord hist_start, Eterm ref);
+
+/* Gathers per-carrier info from the given allocator number (ERTS_ALC_A_*) and
+ * scheduler id. An id of 0 means the global instance will be used.
+ *
+ * The results are sent to `p`, and it returns the number of messages to wait
+ * for. */
+int erts_alcu_gather_carrier_info(struct process *p, int allocator_num,
+                                  int sched_id, int hist_width,
+                                  UWord hist_start, Eterm ref);
+
+struct alcu_blockscan;
+
+typedef struct {
+    struct alcu_blockscan *current;
+    struct alcu_blockscan *last;
+} ErtsAlcuBlockscanYieldData;
+
+int erts_handle_yielded_alcu_blockscan(struct ErtsSchedulerData_ *esdp,
+                                       ErtsAlcuBlockscanYieldData *yield);
+void erts_alcu_sched_spec_data_init(struct ErtsSchedulerData_ *esdp);
+
 #endif /* !ERL_ALLOC_UTIL__ */
 
 #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__)
@@ -548,6 +581,7 @@ struct Allctr_t_ {
     /* Options */
     int			t;
     int			ramv;
+    int                 atags;
     Uint		sbc_threshold;
     Uint		sbc_move_threshold;
     Uint		mbc_move_threshold;
@@ -684,6 +718,7 @@ struct Allctr_t_ {
 #endif
 };
 
+
 int	erts_alcu_start(Allctr_t *, AllctrInit_t *);
 void	erts_alcu_stop(Allctr_t *);
 
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index d443e12409..0b186ab071 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -38,7 +38,7 @@
 #include "erl_message.h"
 #include "erl_binary.h"
 #include "erl_db.h"
-#include "erl_instrument.h"
+#include "erl_mtrace.h"
 #include "dist.h"
 #include "erl_gc.h"
 #include "erl_cpu_topology.h"
@@ -51,6 +51,7 @@
 #include "erl_ptab.h"
 #include "erl_time.h"
 #include "erl_proc_sig_queue.h"
+#include "erl_alloc_util.h"
 #ifdef HIPE
 #include "hipe_arch.h"
 #endif
@@ -2151,45 +2152,6 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 	    return make_small(sizeof(UWord));
 	}
 	goto badarg;
-    } else if (sel == am_allocated) {
-	if (arity == 2) {
-	    Eterm res = THE_NON_VALUE;
-	    char *buf;
-	    Sint len = is_string(*tp);
-	    if (len <= 0)
-		return res;
-	    buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1);
-	    if (intlist_to_buf(*tp, buf, len) != len)
-		erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__);
-	    buf[len] = '\0';
-	    res = erts_instr_dump_memory_map(buf) ? am_true : am_false;
-	    erts_free(ERTS_ALC_T_TMP, (void *) buf);
-	    if (is_non_value(res))
-		goto badarg;
-	    return res;
-	}
-	else if (arity == 3 && tp[0] == am_status) {
-	    if (is_atom(tp[1]))
-		return erts_instr_get_stat(BIF_P, tp[1], 1);
-	    else {
-		Eterm res = THE_NON_VALUE;
-		char *buf;
-		Sint len = is_string(tp[1]);
-		if (len <= 0)
-		    return res;
-		buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1);
-		if (intlist_to_buf(tp[1], buf, len) != len)
-		    erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__);
-		buf[len] = '\0';
-		res = erts_instr_dump_stat(buf, 1) ? am_true : am_false;
-		erts_free(ERTS_ALC_T_TMP, (void *) buf);
-		if (is_non_value(res))
-		    goto badarg;
-		return res;
-	    }
-	}
-	else
-	    goto badarg;
     } else if (sel == am_allocator) {
 	switch (arity) {
 	case 2:
@@ -2557,8 +2519,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     } else if (BIF_ARG_1 == am_allocated_areas) {
 	res = erts_allocated_areas(NULL, NULL, BIF_P);
 	BIF_RET(res);
-    } else if (BIF_ARG_1 == am_allocated) {
-	BIF_RET(erts_instr_get_memory_map(BIF_P));
     } else if (BIF_ARG_1 == am_hipe_architecture) {
 #if defined(HIPE)
 	BIF_RET(hipe_arch_name);
@@ -2699,9 +2659,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 			       sizeof(ERLANG_ARCHITECTURE)-1,
 			       NIL));
     } 
-    else if (BIF_ARG_1 == am_memory_types) {
-	return erts_instr_get_type_info(BIF_P);
-    }
     else if (BIF_ARG_1 == am_os_type) {
 	BIF_RET(os_type_tuple);
     }
@@ -4728,6 +4685,55 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
     BIF_ERROR(BIF_P, BADARG);
 }
 
+static BIF_RETTYPE
+gather_histograms_helper(Process * c_p, Eterm arg_tuple,
+                         int gather(Process *, int, int, int, UWord, Eterm))
+{
+    SWord hist_start, hist_width, sched_id;
+    int msg_count, alloc_num;
+    Eterm *args;
+
+    /* This is an internal BIF, so the error checking is mostly left to erlang
+     * code. */
+
+    ASSERT(is_tuple_arity(arg_tuple, 5));
+    args = tuple_val(arg_tuple);
+
+    for (alloc_num = ERTS_ALC_A_MIN; alloc_num <= ERTS_ALC_A_MAX; alloc_num++) {
+        if(erts_is_atom_str(ERTS_ALC_A2AD(alloc_num), args[1], 0)) {
+            break;
+        }
+    }
+
+    if (alloc_num > ERTS_ALC_A_MAX) {
+        BIF_ERROR(c_p, BADARG);
+    }
+
+    sched_id = signed_val(args[2]);
+    hist_width = signed_val(args[3]);
+    hist_start = signed_val(args[4]);
+
+    if (sched_id < 0 || sched_id > erts_no_schedulers) {
+        BIF_ERROR(c_p, BADARG);
+    }
+
+    msg_count = gather(c_p, alloc_num, sched_id, hist_width, hist_start, args[5]);
+
+    BIF_RET(make_small(msg_count));
+}
+
+BIF_RETTYPE erts_internal_gather_alloc_histograms_1(BIF_ALIST_1)
+{
+    return gather_histograms_helper(BIF_P, BIF_ARG_1,
+                                    erts_alcu_gather_alloc_histograms);
+}
+
+BIF_RETTYPE erts_internal_gather_carrier_info_1(BIF_ALIST_1)
+{
+    return gather_histograms_helper(BIF_P, BIF_ARG_1,
+                                    erts_alcu_gather_carrier_info);
+}
+
 #ifdef ERTS_ENABLE_LOCK_COUNT
 
 typedef struct {
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 179822cc0b..57c6c10c7f 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -37,7 +37,7 @@
 #include "erl_mseg.h"
 #include "erl_threads.h"
 #include "erl_hl_timer.h"
-#include "erl_instrument.h"
+#include "erl_mtrace.h"
 #include "erl_printf_term.h"
 #include "erl_misc_utils.h"
 #include "packet_parser.h"
diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c
deleted file mode 100644
index 2f70e7996e..0000000000
--- a/erts/emulator/beam/erl_instrument.c
+++ /dev/null
@@ -1,1257 +0,0 @@
-/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 2003-2016. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * %CopyrightEnd%
- */
-
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-
-#include "global.h"
-#include "big.h"
-#include "erl_instrument.h"
-#include "erl_threads.h"
-
-typedef union { long l; double d; } Align_t;
-
-typedef struct {
-    Uint size;
-#ifdef VALGRIND
-    void* valgrind_leak_suppressor;
-#endif
-    Align_t mem[1];
-} StatBlock_t;
-
-#define STAT_BLOCK_HEADER_SIZE (sizeof(StatBlock_t) - sizeof(Align_t))
-
-typedef struct MapStatBlock_t_ MapStatBlock_t;
-struct MapStatBlock_t_ {
-    Uint size;
-    ErtsAlcType_t type_no;
-    Eterm pid;
-    MapStatBlock_t *prev;
-    MapStatBlock_t *next;
-    Align_t mem[1];
-};
-
-#define MAP_STAT_BLOCK_HEADER_SIZE (sizeof(MapStatBlock_t) - sizeof(Align_t))
-
-typedef struct {
-    Uint size;
-    Uint max_size;
-    Uint max_size_ever;
-
-    Uint blocks;
-    Uint max_blocks;
-    Uint max_blocks_ever;
-} Stat_t;
-
-static erts_mtx_t instr_mutex;
-static erts_mtx_t instr_x_mutex;
-
-int erts_instr_memory_map;
-int erts_instr_stat;
-
-static ErtsAllocatorFunctions_t real_allctrs[ERTS_ALC_A_MAX+1];
-
-struct stats_ {
-    Stat_t tot;
-    Stat_t a[ERTS_ALC_A_MAX+1];
-    Stat_t *ap[ERTS_ALC_A_MAX+1];
-    Stat_t c[ERTS_ALC_C_MAX+1];
-    Stat_t n[ERTS_ALC_N_MAX+1];
-};
-
-static struct stats_ *stats;
-
-static MapStatBlock_t *mem_anchor;
-
-static Eterm *am_tot;
-static Eterm *am_n;
-static Eterm *am_a;
-static Eterm *am_c;
-
-static int atoms_initialized;
-
-static struct {
-    Eterm total;
-    Eterm allocators;
-    Eterm classes;
-    Eterm types;
-    Eterm sizes;
-    Eterm blocks;
-    Eterm instr_hdr;
-#ifdef DEBUG
-    Eterm end_of_atoms;
-#endif
-} am;
-
-static void ERTS_INLINE atom_init(Eterm *atom, const char *name)
-{
-    *atom = am_atom_put((char *) name, sys_strlen(name));
-}
-#define AM_INIT(AM) atom_init(&am.AM, #AM)
-
-static void
-init_atoms(void)
-{
-#ifdef DEBUG
-    Eterm *atom;
-    for (atom = (Eterm *) &am; atom <= &am.end_of_atoms; atom++) {
-	*atom = THE_NON_VALUE;
-    }
-#endif
-
-    AM_INIT(total);
-    AM_INIT(allocators);
-    AM_INIT(classes);
-    AM_INIT(types);
-    AM_INIT(sizes);
-    AM_INIT(blocks);
-    AM_INIT(instr_hdr);
-
-#ifdef DEBUG
-    for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) {
-	ASSERT(*atom != THE_NON_VALUE);
-    }
-#endif
-
-    atoms_initialized = 1;
-}
-
-#undef AM_INIT
-
-static void
-init_am_tot(void)
-{
-    am_tot = (Eterm *) erts_alloc(ERTS_ALC_T_INSTR_INFO,
-				  sizeof(Eterm));
-    atom_init(am_tot, "total");
-}
-
-
-static void
-init_am_n(void)
-{
-    int i;
-    am_n = (Eterm *) erts_alloc(ERTS_ALC_T_INSTR_INFO,
-				(ERTS_ALC_N_MAX+1)*sizeof(Eterm));
-
-    for (i = ERTS_ALC_N_MIN; i <= ERTS_ALC_N_MAX; i++) {
-	atom_init(&am_n[i], ERTS_ALC_N2TD(i));
-    }
-
-}
-
-static void
-init_am_c(void)
-{
-    int i;
-    am_c = (Eterm *) erts_alloc(ERTS_ALC_T_INSTR_INFO,
-				(ERTS_ALC_C_MAX+1)*sizeof(Eterm));
-
-    for (i = ERTS_ALC_C_MIN; i <= ERTS_ALC_C_MAX; i++) {
-	atom_init(&am_c[i], ERTS_ALC_C2CD(i));
-    }
-
-}
-
-static void
-init_am_a(void)
-{
-    int i;
-    am_a = (Eterm *) erts_alloc(ERTS_ALC_T_INSTR_INFO,
-				(ERTS_ALC_A_MAX+1)*sizeof(Eterm));
-
-    for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
-	atom_init(&am_a[i], ERTS_ALC_A2AD(i));
-    }
-
-}
-
-static ERTS_INLINE void
-stat_upd_alloc(ErtsAlcType_t n, Uint size)
-{
-    ErtsAlcType_t t = ERTS_ALC_N2T(n);
-    ErtsAlcType_t a = ERTS_ALC_T2A(t);
-    ErtsAlcType_t c = ERTS_ALC_T2C(t);
-
-    stats->ap[a]->size += size;
-    if (stats->ap[a]->max_size < stats->ap[a]->size)
-	stats->ap[a]->max_size = stats->ap[a]->size;
-
-    stats->c[c].size += size;
-    if (stats->c[c].max_size < stats->c[c].size)
-	stats->c[c].max_size = stats->c[c].size;
-
-    stats->n[n].size += size;
-    if (stats->n[n].max_size < stats->n[n].size)
-	stats->n[n].max_size = stats->n[n].size;
-
-    stats->tot.size += size;
-    if (stats->tot.max_size < stats->tot.size)
-	stats->tot.max_size = stats->tot.size;
-
-    stats->ap[a]->blocks++;
-    if (stats->ap[a]->max_blocks < stats->ap[a]->blocks)
-	stats->ap[a]->max_blocks = stats->ap[a]->blocks;
-
-    stats->c[c].blocks++;
-    if (stats->c[c].max_blocks < stats->c[c].blocks)
-	stats->c[c].max_blocks = stats->c[c].blocks;
-
-    stats->n[n].blocks++;
-    if (stats->n[n].max_blocks < stats->n[n].blocks)
-	stats->n[n].max_blocks = stats->n[n].blocks;
-
-    stats->tot.blocks++;
-    if (stats->tot.max_blocks < stats->tot.blocks)
-	stats->tot.max_blocks = stats->tot.blocks;
-
-}
-
-
-static ERTS_INLINE void
-stat_upd_free(ErtsAlcType_t n, Uint size)
-{
-    ErtsAlcType_t t = ERTS_ALC_N2T(n);
-    ErtsAlcType_t a = ERTS_ALC_T2A(t);
-    ErtsAlcType_t c = ERTS_ALC_T2C(t);
-
-    ASSERT(stats->ap[a]->size >= size);
-    stats->ap[a]->size -= size;
-
-    ASSERT(stats->c[c].size >= size);
-    stats->c[c].size -= size;
-
-    ASSERT(stats->n[n].size >= size);
-    stats->n[n].size -= size;
-
-    ASSERT(stats->tot.size >= size);
-    stats->tot.size -= size;
-
-    ASSERT(stats->ap[a]->blocks > 0);
-    stats->ap[a]->blocks--;
-
-    ASSERT(stats->c[c].blocks > 0);
-    stats->c[c].blocks--;
-
-    ASSERT(stats->n[n].blocks > 0);
-    stats->n[n].blocks--;
-
-    ASSERT(stats->tot.blocks > 0);
-    stats->tot.blocks--;
-
-}
-
-
-static ERTS_INLINE void
-stat_upd_realloc(ErtsAlcType_t n, Uint size, Uint old_size)
-{
-    if (old_size)
-	stat_upd_free(n, old_size);
-    stat_upd_alloc(n, size);
-}
-
-/*
- * stat instrumentation callback functions
- */
-
-static void stat_pre_lock(void)
-{
-    erts_mtx_lock(&instr_mutex);
-}
-
-static void stat_pre_unlock(void)
-{
-    erts_mtx_unlock(&instr_mutex);
-}
-
-static ErtsAllocatorWrapper_t instr_wrapper;
-
-static void *
-stat_alloc(ErtsAlcType_t n, void *extra, Uint size)
-{
-    ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
-    Uint ssize;
-    void *res;
-
-    if (!erts_is_allctr_wrapper_prelocked()) {
-	erts_mtx_lock(&instr_mutex);
-    }
-
-    ssize = size + STAT_BLOCK_HEADER_SIZE;
-    res = (*real_af->alloc)(n, real_af->extra, ssize);
-    if (res) {
-	stat_upd_alloc(n, size);
-	((StatBlock_t *) res)->size = size;
-#ifdef VALGRIND
-	/* Suppress "possibly leaks" by storing an actual dummy pointer
-	   to the _start_ of the allocated block.*/
-	((StatBlock_t *) res)->valgrind_leak_suppressor = res;
-#endif
-	res = (void *) ((StatBlock_t *) res)->mem;
-    }
-
-    if (!erts_is_allctr_wrapper_prelocked()) {
-	erts_mtx_unlock(&instr_mutex);
-    }
-
-    return res;
-}
-
-static void *
-stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size)
-{
-    ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
-    Uint old_size;
-    Uint ssize;
-    void *sptr;
-    void *res;
-
-    if (!erts_is_allctr_wrapper_prelocked()) {
-	erts_mtx_lock(&instr_mutex);
-    }
-
-    if (ptr) {
-	sptr = (void *) (((char *) ptr) - STAT_BLOCK_HEADER_SIZE);
-	old_size = ((StatBlock_t *) sptr)->size;
-    }
-    else {
-	sptr = NULL;
-	old_size = 0;
-    }
-
-    ssize = size + STAT_BLOCK_HEADER_SIZE;
-    res = (*real_af->realloc)(n, real_af->extra, sptr, ssize);
-    if (res) {
-	stat_upd_realloc(n, size, old_size);
-	((StatBlock_t *) res)->size = size;
-#ifdef VALGRIND
-	((StatBlock_t *) res)->valgrind_leak_suppressor = res;
-#endif
-	res = (void *) ((StatBlock_t *) res)->mem;
-    }
-
-    if (!erts_is_allctr_wrapper_prelocked()) {
-	erts_mtx_unlock(&instr_mutex);
-    }
-
-    return res;
-}
-
-static void
-stat_free(ErtsAlcType_t n, void *extra, void *ptr)
-{
-    ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
-    void *sptr;
-
-    if (!erts_is_allctr_wrapper_prelocked()) {
-	erts_mtx_lock(&instr_mutex);
-    }
-
-    if (ptr) {
-	sptr = (void *) (((char *) ptr) - STAT_BLOCK_HEADER_SIZE);
-	stat_upd_free(n, ((StatBlock_t *) sptr)->size);
-    }
-    else {
-	sptr = NULL;
-    }
-
-    (*real_af->free)(n, real_af->extra, sptr);
-
-    if (!erts_is_allctr_wrapper_prelocked()) {
-	erts_mtx_unlock(&instr_mutex);
-    }
-
-}
-
-/*
- * map stat instrumentation callback functions
- */
-
-static void map_stat_pre_lock(void)
-{
-    erts_mtx_lock(&instr_x_mutex);
-    erts_mtx_lock(&instr_mutex);
-}
-
-static void map_stat_pre_unlock(void)
-{
-    erts_mtx_unlock(&instr_mutex);
-    erts_mtx_unlock(&instr_x_mutex);
-}
-
-static void *
-map_stat_alloc(ErtsAlcType_t n, void *extra, Uint size)
-{
-    ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
-    Uint msize;
-    void *res;
-
-    if (!erts_is_allctr_wrapper_prelocked()) {
-	erts_mtx_lock(&instr_mutex);
-    }
-
-    msize = size + MAP_STAT_BLOCK_HEADER_SIZE;
-    res = (*real_af->alloc)(n, real_af->extra, msize);
-    if (res) {
-	MapStatBlock_t *mb = (MapStatBlock_t *) res;
-	stat_upd_alloc(n, size);
-
-	mb->size = size;
-	mb->type_no = n;
-	mb->pid = erts_get_current_pid();
-
-	mb->prev = NULL;
-	mb->next = mem_anchor;
-	if (mem_anchor)
-	    mem_anchor->prev = mb;
-	mem_anchor = mb;
-
-	res = (void *) mb->mem;
-    }
-
-    if (!erts_is_allctr_wrapper_prelocked()) {
-	erts_mtx_unlock(&instr_mutex);
-    }
-
-    return res;
-}
-
-static void *
-map_stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size)
-{
-    ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
-    Uint old_size;
-    Uint msize;
-    void *mptr;
-    void *res;
-
-    if (!erts_is_allctr_wrapper_prelocked()) {
-	erts_mtx_lock(&instr_x_mutex);
-	erts_mtx_lock(&instr_mutex);
-    }
-
-    if (ptr) {
-	mptr = (void *) (((char *) ptr) - MAP_STAT_BLOCK_HEADER_SIZE);
-	old_size = ((MapStatBlock_t *) mptr)->size;
-    }
-    else {
-	mptr = NULL;
-	old_size = 0;
-    }
-
-    msize = size + MAP_STAT_BLOCK_HEADER_SIZE;
-    res = (*real_af->realloc)(n, real_af->extra, mptr, msize);
-    if (res) {
-	MapStatBlock_t *mb = (MapStatBlock_t *) res;
-
-	mb->size = size;
-	mb->type_no = n;
-	mb->pid = erts_get_current_pid();
-
-	stat_upd_realloc(n, size, old_size);
-
-	if (mptr != res) {
-
-	    if (mptr) {
-		if (mb->prev)
-		    mb->prev->next = mb;
-		else {
-		    ASSERT(mem_anchor == (MapStatBlock_t *) mptr);
-		    mem_anchor = mb;
-		}
-		if (mb->next)
-		    mb->next->prev = mb;
-	    }
-	    else {
-		mb->prev = NULL;
-		mb->next = mem_anchor;
-		if (mem_anchor)
-		    mem_anchor->prev = mb;
-		mem_anchor = mb;
-	    }
-
-	}
-
-	res = (void *) mb->mem;
-    }
-    if (!erts_is_allctr_wrapper_prelocked()) {
-	erts_mtx_unlock(&instr_mutex);
-	erts_mtx_unlock(&instr_x_mutex);
-    }
-
-    return res;
-}
-
-static void
-map_stat_free(ErtsAlcType_t n, void *extra, void *ptr)
-{
-    ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
-    void *mptr;
-
-    if (!erts_is_allctr_wrapper_prelocked()) {
-	erts_mtx_lock(&instr_x_mutex);
-	erts_mtx_lock(&instr_mutex);
-    }
-
-    if (ptr) {
-	MapStatBlock_t *mb;
-
-	mptr = (void *) (((char *) ptr) - MAP_STAT_BLOCK_HEADER_SIZE);
-	mb = (MapStatBlock_t *) mptr;
-
-	stat_upd_free(n, mb->size);
-
-	if (mb->prev)
-	    mb->prev->next = mb->next;
-	else
-	    mem_anchor = mb->next;
-	if (mb->next)
-	    mb->next->prev = mb->prev;
-    }
-    else {
-	mptr = NULL;
-    }
-
-    (*real_af->free)(n, real_af->extra, mptr);
-
-    if (!erts_is_allctr_wrapper_prelocked()) {
-	erts_mtx_unlock(&instr_mutex);
-	erts_mtx_unlock(&instr_x_mutex);
-    }
-
-}
-
-static void dump_memory_map_to_stream(fmtfn_t to, void* to_arg)
-{
-    ErtsAlcType_t n;
-    MapStatBlock_t *bp;
-    int lock = !ERTS_IS_CRASH_DUMPING;
-    if (lock) {
-	ASSERT(!erts_is_allctr_wrapper_prelocked());
-	erts_mtx_lock(&instr_mutex);
-    }
-
-    /* Write header */
-
-    erts_cbprintf(to, to_arg,
-	    "{instr_hdr,\n"
-	    " %lu,\n"
-	    " %lu,\n"
-	    " {",
-	    (unsigned long) ERTS_INSTR_VSN,
-	    (unsigned long) MAP_STAT_BLOCK_HEADER_SIZE);
-
-#if ERTS_ALC_N_MIN != 1
-#error ERTS_ALC_N_MIN is not 1
-#endif
-
-    for (n = ERTS_ALC_N_MIN; n <= ERTS_ALC_N_MAX; n++) {
-	ErtsAlcType_t t = ERTS_ALC_N2T(n);
-	ErtsAlcType_t a = ERTS_ALC_T2A(t);
-	ErtsAlcType_t c = ERTS_ALC_T2C(t);
-	const char *astr;
-
-	if (erts_allctrs_info[a].enabled)
-	    astr = ERTS_ALC_A2AD(a);
-	else
-	    astr = ERTS_ALC_A2AD(ERTS_ALC_A_SYSTEM);
-
-	erts_cbprintf(to, to_arg,
-		"%s{%s,%s,%s}%s",
-		(n == ERTS_ALC_N_MIN) ? "" : "  ",
-		ERTS_ALC_N2TD(n),
-		astr,
-		ERTS_ALC_C2CD(c),
-		(n == ERTS_ALC_N_MAX) ? "" : ",\n");
-    }
-
-    erts_cbprintf(to, to_arg, "}}.\n");
-
-    /* Write memory data */
-    for (bp = mem_anchor; bp; bp = bp->next) {
-	if (is_internal_pid(bp->pid))
-	    erts_cbprintf(to, to_arg,
-		    "{%lu, %lu, %lu, {%lu,%lu,%lu}}.\n",
-		    (UWord) bp->type_no,
-		    (UWord) bp->mem,
-		    (UWord) bp->size,
-		    (UWord) pid_channel_no(bp->pid),
-		    (UWord) pid_number(bp->pid),
-		    (UWord) pid_serial(bp->pid));
-	else
-	    erts_cbprintf(to, to_arg,
-		    "{%lu, %lu, %lu, undefined}.\n",
-		    (UWord) bp->type_no,
-		    (UWord) bp->mem,
-		    (UWord) bp->size);
-    }
-
-    if (lock)
-	erts_mtx_unlock(&instr_mutex);
-}
-
-int erts_instr_dump_memory_map_to(fmtfn_t to, void* to_arg)
-{
-    if (!erts_instr_memory_map)
-	return 0;
-
-    dump_memory_map_to_stream(to, to_arg);
-    return 1;
-}
-
-int erts_instr_dump_memory_map(const char *name)
-{
-    int fd;
-
-    if (!erts_instr_memory_map)
-	return 0;
-
-    fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, 0640);
-    if (fd < 0)
-	return 0;
-
-    dump_memory_map_to_stream(erts_write_fd, (void*)&fd);
-
-    close(fd);
-    return 1;
-}
-
-Eterm erts_instr_get_memory_map(Process *proc)
-{
-    MapStatBlock_t *org_mem_anchor;
-    Eterm hdr_tuple, md_list, res;
-    Eterm *hp;
-    Uint hsz;
-    MapStatBlock_t *bp;
-#ifdef DEBUG
-    Eterm *end_hp;
-#endif
-    
-    if (!erts_instr_memory_map)
-	return am_false;
-
-    if (!atoms_initialized)
-	init_atoms();
-    if (!am_n)
-	init_am_n();
-    if (!am_c)
-	init_am_c();
-    if (!am_a)
-	init_am_a();
-
-    erts_mtx_lock(&instr_x_mutex);
-    erts_mtx_lock(&instr_mutex);
-
-    /* Header size */
-    hsz = 5 + 1 + (ERTS_ALC_N_MAX+1-ERTS_ALC_N_MIN)*(1 + 4);
-
-    /* Memory data list */
-    for (bp = mem_anchor; bp; bp = bp->next) {
-	if (is_internal_pid(bp->pid)) {
-#if (_PID_NUM_SIZE - 1 > MAX_SMALL)
-	    if (internal_pid_number(bp->pid) > MAX_SMALL)
-		hsz += BIG_UINT_HEAP_SIZE;
-#endif
-#if (_PID_SER_SIZE - 1 > MAX_SMALL)
-	    if (internal_pid_serial(bp->pid) > MAX_SMALL)
-		hsz += BIG_UINT_HEAP_SIZE;
-#endif
-	    hsz += 4;
-	}
-
-	if ((UWord) bp->mem > MAX_SMALL)
-	    hsz += BIG_UINT_HEAP_SIZE;
-	if (bp->size > MAX_SMALL)
-	    hsz += BIG_UINT_HEAP_SIZE;
-
-	hsz += 5 + 2;
-    }
-
-    hsz += 3; /* Root tuple */
-
-    org_mem_anchor = mem_anchor;
-    mem_anchor = NULL;
-
-    erts_mtx_unlock(&instr_mutex);
-
-    hp = HAlloc(proc, hsz); /* May end up calling map_stat_alloc() */
-
-    erts_mtx_lock(&instr_mutex);
-
-#ifdef DEBUG
-    end_hp = hp + hsz;
-#endif
-
-    {	/* Build header */
-	ErtsAlcType_t n;
-	Eterm type_map;
-	Uint *hp2 = hp;
-#ifdef DEBUG
-	Uint *hp2_end;
-#endif
-
-	hp += (ERTS_ALC_N_MAX + 1 - ERTS_ALC_N_MIN)*4;
-
-#ifdef DEBUG
-	hp2_end = hp;
-#endif
-
-	type_map = make_tuple(hp);
-	*(hp++) = make_arityval(ERTS_ALC_N_MAX + 1 - ERTS_ALC_N_MIN);
-
-	for (n = ERTS_ALC_N_MIN; n <= ERTS_ALC_N_MAX; n++) {
-	    ErtsAlcType_t t = ERTS_ALC_N2T(n);
-	    ErtsAlcType_t a = ERTS_ALC_T2A(t);
-	    ErtsAlcType_t c = ERTS_ALC_T2C(t);
-
-	    if (!erts_allctrs_info[a].enabled)
-		a = ERTS_ALC_A_SYSTEM;
-
-	    *(hp++) = TUPLE3(hp2, am_n[n], am_a[a], am_c[c]);
-	    hp2 += 4;
-	}
-
-	ASSERT(hp2 == hp2_end);
-
-	hdr_tuple = TUPLE4(hp,
-			   am.instr_hdr,
-			   make_small(ERTS_INSTR_VSN),
-			   make_small(MAP_STAT_BLOCK_HEADER_SIZE),
-			   type_map);
-
-	hp += 5;
-    }
-
-    /* Build memory data list */
-
-    for (md_list = NIL, bp = org_mem_anchor; bp; bp = bp->next) {
-	Eterm tuple;
-	Eterm type;
-	Eterm ptr;
-	Eterm size;
-	Eterm pid;
-
-	if (is_not_internal_pid(bp->pid))
-	    pid = am_undefined;
-	else {
-	    Eterm c;
-	    Eterm n;
-	    Eterm s;
-
-#if (ERST_INTERNAL_CHANNEL_NO > MAX_SMALL)
-#error Oversized internal channel number
-#endif
-	    c = make_small(ERST_INTERNAL_CHANNEL_NO);
-
-#if (_PID_NUM_SIZE - 1 > MAX_SMALL)
-	    if (internal_pid_number(bp->pid) > MAX_SMALL) {
-		n = uint_to_big(internal_pid_number(bp->pid), hp);
-		hp += BIG_UINT_HEAP_SIZE;
-	    }
-	    else
-#endif
-		n = make_small(internal_pid_number(bp->pid));
-
-#if (_PID_SER_SIZE - 1 > MAX_SMALL)
-	    if (internal_pid_serial(bp->pid) > MAX_SMALL) {
-		s = uint_to_big(internal_pid_serial(bp->pid), hp);
-		hp += BIG_UINT_HEAP_SIZE;
-	    }
-	    else
-#endif
-		s = make_small(internal_pid_serial(bp->pid));
-	    pid = TUPLE3(hp, c, n, s);
-	    hp += 4;
-	}
-
-
-#if ERTS_ALC_N_MAX > MAX_SMALL
-#error Oversized memory type number
-#endif
-	type = make_small(bp->type_no);
-
-	if ((UWord) bp->mem > MAX_SMALL) {
-	    ptr = uint_to_big((UWord) bp->mem, hp);
-	    hp += BIG_UINT_HEAP_SIZE;
-	}
-	else
-	    ptr = make_small((UWord) bp->mem);
-
-	if (bp->size > MAX_SMALL) {
-	    size = uint_to_big(bp->size, hp);
-	    hp += BIG_UINT_HEAP_SIZE;
-	}
-	else
-	    size = make_small(bp->size);
-
-	tuple = TUPLE4(hp, type, ptr, size, pid);
-	hp += 5;
-
-	md_list = CONS(hp, tuple, md_list);
-	hp += 2;
-    }
-
-    res = TUPLE2(hp, hdr_tuple, md_list);
-    
-    ASSERT(hp + 3 == end_hp);
-
-    if (mem_anchor) {
-	for (bp = mem_anchor; bp->next; bp = bp->next)
-	    ;
-	ASSERT(org_mem_anchor);
-	org_mem_anchor->prev = bp; 
-	bp->next = org_mem_anchor;
-    }
-    else {
-	mem_anchor = org_mem_anchor;
-    }
-
-    erts_mtx_unlock(&instr_mutex);
-    erts_mtx_unlock(&instr_x_mutex);
-
-    return res;
-}
-
-static ERTS_INLINE void
-begin_new_max_period(Stat_t *stat, int min, int max)
-{
-    int i;
-    for (i = min; i <= max; i++) {
-	stat[i].max_size = stat[i].size;
-	stat[i].max_blocks = stat[i].blocks;
-    }
-}
-
-static ERTS_INLINE void
-update_max_ever_values(Stat_t *stat, int min, int max)
-{
-    int i;
-    for (i = min; i <= max; i++) {
-	if (stat[i].max_size_ever < stat[i].max_size)
-	    stat[i].max_size_ever = stat[i].max_size;
-	if (stat[i].max_blocks_ever < stat[i].max_blocks)
-	    stat[i].max_blocks_ever = stat[i].max_blocks;
-    }
-}
-
-#define bld_string	erts_bld_string
-#define bld_tuple	erts_bld_tuple
-#define bld_tuplev	erts_bld_tuplev
-#define bld_list	erts_bld_list
-#define bld_2tup_list	erts_bld_2tup_list
-#define bld_uint	erts_bld_uint
-
-Eterm
-erts_instr_get_stat(Process *proc, Eterm what, int begin_max_period)
-{
-    int i, len, max, min, allctr;
-    Eterm *names, *values, res;
-    Uint arr_size, stat_size, hsz, *hszp, *hp, **hpp;
-    Stat_t *stat_src, *stat;
-
-    if (!erts_instr_stat)
-	return am_false;
-
-    if (!atoms_initialized)
-	init_atoms();
-
-    if (what == am.total) {
-	min		= 0;
-	max		= 0;
-	allctr		= 0;
-	stat_size	= sizeof(Stat_t);
-	stat_src	= &stats->tot;
-	if (!am_tot)
-	    init_am_tot();
-	names		= am_tot;
-    }
-    else if (what == am.allocators) {
-	min		= ERTS_ALC_A_MIN;
-	max		= ERTS_ALC_A_MAX;
-	allctr		= 1;
-	stat_size	= sizeof(Stat_t)*(ERTS_ALC_A_MAX+1);
-	stat_src	= stats->a;
-	if (!am_a)
-	    init_am_a();
-	names		= am_a;
-    }
-    else if (what == am.classes) {
-	min		= ERTS_ALC_C_MIN;
-	max		= ERTS_ALC_C_MAX;
-	allctr		= 0;
-	stat_size	= sizeof(Stat_t)*(ERTS_ALC_C_MAX+1);
-	stat_src	= stats->c;
-	if (!am_c)
-	    init_am_c();
-	names		= &am_c[ERTS_ALC_C_MIN];
-    }
-    else if (what == am.types) {
-	min		= ERTS_ALC_N_MIN;
-	max		= ERTS_ALC_N_MAX;
-	allctr		= 0;
-	stat_size	= sizeof(Stat_t)*(ERTS_ALC_N_MAX+1);
-	stat_src	= stats->n;
-	if (!am_n)
-	    init_am_n();
-	names		= &am_n[ERTS_ALC_N_MIN];
-    }
-    else {
-	return THE_NON_VALUE;
-    }
-
-    stat = (Stat_t *) erts_alloc(ERTS_ALC_T_TMP, stat_size);
-
-    arr_size = (max - min + 1)*sizeof(Eterm);
-
-    if (allctr)
-	names = (Eterm *) erts_alloc(ERTS_ALC_T_TMP, arr_size);
-
-    values = (Eterm *) erts_alloc(ERTS_ALC_T_TMP, arr_size);
-
-    erts_mtx_lock(&instr_mutex);
-
-    update_max_ever_values(stat_src, min, max);
-
-    sys_memcpy((void *) stat, (void *) stat_src, stat_size);
-
-    if (begin_max_period)
-	begin_new_max_period(stat_src, min, max);
-
-    erts_mtx_unlock(&instr_mutex);
-
-    hsz = 0;
-    hszp = &hsz;
-    hpp = NULL;
-
- restart_bld:
-
-    len = 0;
-    for (i = min; i <= max; i++) {
-	if (!allctr || erts_allctrs_info[i].enabled) {
-	    Eterm s[2];
-
-	    if (allctr)
-		names[len] = am_a[i];
-	    
-	    s[0] = bld_tuple(hpp, hszp, 4,
-			     am.sizes,
-			     bld_uint(hpp, hszp, stat[i].size),
-			     bld_uint(hpp, hszp, stat[i].max_size),
-			     bld_uint(hpp, hszp, stat[i].max_size_ever));
-
-	    s[1] = bld_tuple(hpp, hszp, 4,
-			     am.blocks,
-			     bld_uint(hpp, hszp, stat[i].blocks),
-			     bld_uint(hpp, hszp, stat[i].max_blocks),
-			     bld_uint(hpp, hszp, stat[i].max_blocks_ever));
-
-	    values[len] = bld_list(hpp, hszp, 2, s);
-	    
-	    len++;
-	}
-    }
-
-    res = bld_2tup_list(hpp, hszp, len, names, values);
-
-    if (!hpp) {
-	hp = HAlloc(proc, hsz);
-	hszp = NULL;
-	hpp = &hp;
-	goto restart_bld;
-    }
-
-    erts_free(ERTS_ALC_T_TMP, (void *) stat);
-    erts_free(ERTS_ALC_T_TMP, (void *) values);
-    if (allctr)
-	erts_free(ERTS_ALC_T_TMP, (void *) names);
-
-    return res;
-}
-
-static void
-dump_stat_to_stream(fmtfn_t to, void* to_arg, int begin_max_period)
-{
-    ErtsAlcType_t i, a_max, a_min;
-
-    erts_mtx_lock(&instr_mutex);
-
-    erts_cbprintf(to, to_arg,
-	    "{instr_vsn,%lu}.\n",
-	    (unsigned long) ERTS_INSTR_VSN);
-    
-    update_max_ever_values(&stats->tot, 0, 0);
-
-    erts_cbprintf(to, to_arg,
-	    "{total,[{total,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}]}.\n",
-	    (UWord) stats->tot.size,
-	    (UWord) stats->tot.max_size,
-	    (UWord) stats->tot.max_size_ever,
-	    (UWord) stats->tot.blocks,
-	    (UWord) stats->tot.max_blocks,
-	    (UWord) stats->tot.max_blocks_ever);
-
-    a_max = 0;
-    a_min = ~0;
-    for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
-	if (erts_allctrs_info[i].enabled) {
-	    if (a_min > i)
-		a_min = i;
-	    if (a_max < i)
-		a_max = i;
-	}
-    }
-
-    ASSERT(ERTS_ALC_A_MIN <= a_min && a_min <= ERTS_ALC_A_MAX);
-    ASSERT(ERTS_ALC_A_MIN <= a_max && a_max <= ERTS_ALC_A_MAX);
-    ASSERT(a_min <= a_max);
-
-    update_max_ever_values(stats->a, a_min, a_max);
-
-    for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
-	if (erts_allctrs_info[i].enabled) {
-	    erts_cbprintf(to, to_arg,
-		    "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s",
-		    i == a_min ? "{allocators,\n [" : "  ",
-		    ERTS_ALC_A2AD(i),
-		    (UWord) stats->a[i].size,
-		    (UWord) stats->a[i].max_size,
-		    (UWord) stats->a[i].max_size_ever,
-		    (UWord) stats->a[i].blocks,
-		    (UWord) stats->a[i].max_blocks,
-		    (UWord) stats->a[i].max_blocks_ever,
-		    i == a_max ? "]}.\n" : ",\n");
-	}
-    }
-
-    update_max_ever_values(stats->c, ERTS_ALC_C_MIN, ERTS_ALC_C_MAX);
-
-    for (i = ERTS_ALC_C_MIN; i <= ERTS_ALC_C_MAX; i++) {
-	erts_cbprintf(to, to_arg,
-		"%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s",
-		i == ERTS_ALC_C_MIN ? "{classes,\n [" : "  ",
-		ERTS_ALC_C2CD(i),
-		(UWord) stats->c[i].size,
-		(UWord) stats->c[i].max_size,
-		(UWord) stats->c[i].max_size_ever,
-		(UWord) stats->c[i].blocks,
-		(UWord) stats->c[i].max_blocks,
-		(UWord) stats->c[i].max_blocks_ever,
-		i == ERTS_ALC_C_MAX ? "]}.\n" :  ",\n" );
-    }
-
-    update_max_ever_values(stats->n, ERTS_ALC_N_MIN, ERTS_ALC_N_MAX);
-
-    for (i = ERTS_ALC_N_MIN; i <= ERTS_ALC_N_MAX; i++) {
-	erts_cbprintf(to, to_arg,
-		"%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s",
-		i == ERTS_ALC_N_MIN ? "{types,\n [" : "  ",
-		ERTS_ALC_N2TD(i),
-		(UWord) stats->n[i].size,
-		(UWord) stats->n[i].max_size,
-		(UWord) stats->n[i].max_size_ever,
-		(UWord) stats->n[i].blocks,
-		(UWord) stats->n[i].max_blocks,
-		(UWord) stats->n[i].max_blocks_ever,
-		i == ERTS_ALC_N_MAX ? "]}.\n" :  ",\n" );
-    }
-
-    if (begin_max_period) {
-	begin_new_max_period(&stats->tot, 0, 0);
-	begin_new_max_period(stats->a, a_min, a_max);
-	begin_new_max_period(stats->c, ERTS_ALC_C_MIN, ERTS_ALC_C_MAX);
-	begin_new_max_period(stats->n, ERTS_ALC_N_MIN, ERTS_ALC_N_MAX);
-    }
-
-    erts_mtx_unlock(&instr_mutex);
-
-}
-
-int erts_instr_dump_stat_to(fmtfn_t to, void* to_arg, int begin_max_period)
-{
-    if (!erts_instr_stat)
-	return 0;
-
-    dump_stat_to_stream(to, to_arg, begin_max_period);
-    return 1;
-}
-
-int erts_instr_dump_stat(const char *name, int begin_max_period)
-{
-    int fd;
-
-    if (!erts_instr_stat)
-	return 0;
-
-    fd = open(name, O_WRONLY | O_CREAT | O_TRUNC,0640);
-    if (fd < 0)
-	return 0;
-
-    dump_stat_to_stream(erts_write_fd, (void*)&fd, begin_max_period);
-
-    close(fd);
-    return 1;
-}
-
-
-Uint
-erts_instr_get_total(void)
-{
-    return erts_instr_stat ? stats->tot.size : 0;
-}
-
-Uint
-erts_instr_get_max_total(void)
-{
-    if (erts_instr_stat) {
-	update_max_ever_values(&stats->tot, 0, 0);
-	return stats->tot.max_size_ever;
-    }
-    return 0;
-}
-
-Eterm
-erts_instr_get_type_info(Process *proc)
-{
-    Eterm res, *tpls;
-    Uint hsz, *hszp, *hp, **hpp;
-    ErtsAlcType_t n;
-
-    if (!am_n)
-	init_am_n();
-    if (!am_a)
-	init_am_a();
-    if (!am_c)
-	init_am_c();
-
-    tpls = (Eterm *) erts_alloc(ERTS_ALC_T_TMP,
-				(ERTS_ALC_N_MAX-ERTS_ALC_N_MIN+1)
-				* sizeof(Eterm));
-    hsz = 0;
-    hszp = &hsz;
-    hpp = NULL;
-
- restart_bld:
-
-#if ERTS_ALC_N_MIN != 1
-#error ERTS_ALC_N_MIN is not 1
-#endif
-
-    for (n = ERTS_ALC_N_MIN; n <= ERTS_ALC_N_MAX; n++) {
-	ErtsAlcType_t t = ERTS_ALC_N2T(n);
-	ErtsAlcType_t a = ERTS_ALC_T2A(t);
-	ErtsAlcType_t c = ERTS_ALC_T2C(t);
-
-	if (!erts_allctrs_info[a].enabled)
-	    a = ERTS_ALC_A_SYSTEM;
-
-	tpls[n - ERTS_ALC_N_MIN]
-	    = bld_tuple(hpp, hszp, 3, am_n[n], am_a[a], am_c[c]);
-    }
-
-    res = bld_tuplev(hpp, hszp, ERTS_ALC_N_MAX-ERTS_ALC_N_MIN+1, tpls);
-    
-    if (!hpp) {
-	hp = HAlloc(proc, hsz);
-	hszp = NULL;
-	hpp = &hp;
-	goto restart_bld;
-    }
-
-    erts_free(ERTS_ALC_T_TMP, tpls);
-
-    return res;
-}
-
-Uint
-erts_instr_init(int stat, int map_stat)
-{
-    Uint extra_sz;
-    int i;
-
-    am_tot = NULL;
-    am_n = NULL;
-    am_c = NULL;
-    am_a = NULL;
-
-    erts_instr_memory_map = 0;
-    erts_instr_stat = 0;
-    atoms_initialized = 0;
-
-    if (!stat && !map_stat)
-	return 0;
-
-    stats = erts_alloc(ERTS_ALC_T_INSTR_INFO, sizeof(struct stats_));
-
-    erts_mtx_init(&instr_mutex, "instr", NIL,
-        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
-
-    mem_anchor = NULL;
-
-    /* Install instrumentation functions */
-    ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs));
-
-    sys_memcpy((void *)real_allctrs,(void *)erts_allctrs,sizeof(erts_allctrs));
-
-    sys_memzero((void *) &stats->tot, sizeof(Stat_t));
-    sys_memzero((void *) stats->a, sizeof(Stat_t)*(ERTS_ALC_A_MAX+1));
-    sys_memzero((void *) stats->c, sizeof(Stat_t)*(ERTS_ALC_C_MAX+1));
-    sys_memzero((void *) stats->n, sizeof(Stat_t)*(ERTS_ALC_N_MAX+1));
-
-    for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
-	if (erts_allctrs_info[i].enabled)
-	    stats->ap[i] = &stats->a[i];
-	else
-	    stats->ap[i] = &stats->a[ERTS_ALC_A_SYSTEM];
-    }
-
-    if (map_stat) {
-
-	erts_mtx_init(&instr_x_mutex, "instr_x", NIL,
-        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
-
-	erts_instr_memory_map = 1;
-	erts_instr_stat = 1;
-	for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
-	    erts_allctrs[i].alloc	= map_stat_alloc;
-	    erts_allctrs[i].realloc	= map_stat_realloc;
-	    erts_allctrs[i].free	= map_stat_free;
-	    erts_allctrs[i].extra	= (void *) &real_allctrs[i];
-	}
-	instr_wrapper.lock = map_stat_pre_lock;
-	instr_wrapper.unlock = map_stat_pre_unlock;
-	extra_sz = MAP_STAT_BLOCK_HEADER_SIZE;
-    }
-    else {
-	erts_instr_stat = 1;
-	for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
-	    erts_allctrs[i].alloc	= stat_alloc;
-	    erts_allctrs[i].realloc	= stat_realloc;
-	    erts_allctrs[i].free	= stat_free;
-	    erts_allctrs[i].extra	= (void *) &real_allctrs[i];
-	}
-	instr_wrapper.lock = stat_pre_lock;
-	instr_wrapper.unlock = stat_pre_unlock;
-	extra_sz = STAT_BLOCK_HEADER_SIZE;
-    }
-    erts_allctr_wrapper_prelock_init(&instr_wrapper);
-    return extra_sz;
-}
-
diff --git a/erts/emulator/beam/erl_instrument.h b/erts/emulator/beam/erl_instrument.h
deleted file mode 100644
index 351172b2fa..0000000000
--- a/erts/emulator/beam/erl_instrument.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2016. All Rights Reserved.
- * 
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * 
- * %CopyrightEnd%
- */
-
-#ifndef ERL_INSTRUMENT_H__
-#define ERL_INSTRUMENT_H__
-
-#include "erl_mtrace.h"
-
-#define ERTS_INSTR_VSN 2
-
-extern int erts_instr_memory_map;
-extern int erts_instr_stat;
-
-Uint  erts_instr_init(int stat, int map_stat);
-int   erts_instr_dump_memory_map_to(fmtfn_t to, void* to_arg);
-int   erts_instr_dump_memory_map(const char *name);
-Eterm erts_instr_get_memory_map(Process *process);
-int   erts_instr_dump_stat_to(fmtfn_t to, void* to_arg, int begin_max_period);
-int   erts_instr_dump_stat(const char *name, int begin_max_period);
-Eterm erts_instr_get_stat(Process *proc, Eterm what, int begin_max_period);
-Eterm erts_instr_get_type_info(Process *proc);
-Uint  erts_instr_get_total(void);
-Uint  erts_instr_get_max_total(void);
-
-#endif
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 47b46fdd75..88b30644b7 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -36,7 +36,6 @@
 #include "erl_db.h"
 #include "dist.h"
 #include "beam_catches.h"
-#include "erl_instrument.h"
 #include "erl_threads.h"
 #include "erl_binary.h"
 #include "beam_bp.h"
@@ -2508,6 +2507,8 @@ handle_yield(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
 
     yield |= erts_handle_yielded_ets_all_request(awdp->esdp,
                                                  &awdp->yield.ets_all);
+    yield |= erts_handle_yielded_alcu_blockscan(awdp->esdp,
+                                                &awdp->yield.alcu_blockscan);
 
     /*
      * Other yielding operations...
@@ -8334,6 +8335,7 @@ sched_thread_func(void *vesdp)
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL);
 #endif
 
+    erts_alcu_sched_spec_data_init(esdp);
     erts_ets_sched_spec_data_init(esdp);
 
     process_main(esdp->x_reg_array, esdp->f_reg_array);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index bb97b6690c..e2aa1d9f84 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -593,6 +593,7 @@ typedef struct {
 	ErtsDelayedAuxWorkWakeupJob *job;
     } delayed_wakeup;
     struct {
+        ErtsAlcuBlockscanYieldData alcu_blockscan;
         ErtsEtsAllYieldData ets_all;
         /* Other yielding operations... */
     } yield;
diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index 46ece531a8..585c5a1871 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -2217,36 +2217,44 @@ handle_event(Event, Pid) ->
 
 processes_term_proc_list(Config) when is_list(Config) ->
     Tester = self(),
-    as_expected = processes_term_proc_list_test(false),
-    {ok, Node} = start_node(Config, "+Mis true"),
-    RT = spawn_link(Node, fun () ->
-		receive after 1000 -> ok end,
-		processes_term_proc_list_test(false),
-		Tester ! {it_worked, self()}
-	end),
-    receive {it_worked, RT} -> ok end,
-    stop_node(Node),
+
+    Run = fun(Args) ->
+              {ok, Node} = start_node(Config, Args),
+              RT = spawn_link(Node, fun () ->
+                              receive after 1000 -> ok end,
+                              as_expected = processes_term_proc_list_test(false),
+                              Tester ! {it_worked, self()}
+                      end),
+              receive {it_worked, RT} -> ok end,
+              stop_node(Node)
+          end,
+
+    %% We have to run this test case with +S1 since instrument:allocations()
+    %% will report a free()'d block as present until it's actually deallocated
+    %% by its employer.
+    Run("+MSe true +MSatags false +S1"),
+    Run("+MSe true +MSatags true +S1"),
+
     ok.
-    
+
 -define(CHK_TERM_PROC_LIST(MC, XB),
 	chk_term_proc_list(?LINE, MC, XB)).
 
 chk_term_proc_list(Line, MustChk, ExpectBlks) ->
-    case {MustChk, instrument:memory_status(types)} of
-	{false, false} ->
+    Allocs = instrument:allocations(#{ allocator_types => [sl_alloc] }),
+    case {MustChk, Allocs} of
+	{false, {error, not_enabled}} ->
 	    not_enabled;
-	{_, MS} ->
-	    {value,
-	     {ptab_list_deleted_el,
-	      DL}} = lists:keysearch(ptab_list_deleted_el, 1, MS),
-	    case lists:keysearch(blocks, 1, DL) of
-		{value, {blocks, ExpectBlks, _, _}} ->
-		    ok;
-		{value, {blocks, Blks, _, _}} ->
-		    exit({line, Line,
-			  mismatch, expected, ExpectBlks, actual, Blks});
-		Unexpected ->
-		    exit(Unexpected)
+	{_, {ok, {_Shift, _Unscanned, ByOrigin}}} ->
+            ByType = maps:get(system, ByOrigin, #{}),
+            Hist = maps:get(ptab_list_deleted_el, ByType, {}),
+	    case lists:sum(tuple_to_list(Hist)) of
+		ExpectBlks ->
+                    ok;
+		Blks ->
+                    exit({line, Line, mismatch,
+                          expected, ExpectBlks,
+                          actual, Blks})
 	    end
     end,
     ok.
-- 
cgit v1.2.3


From bfb9b3e8ea91aac1afef298c406296069d1b09b4 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 24 Apr 2018 08:42:25 +0200
Subject: erts: nif resource stop from poll-thread is a indirect call

---
 erts/emulator/sys/common/erl_check_io.c | 2 +-
 erts/emulator/test/nif_SUITE.erl        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index f93d4c1557..8928529984 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1736,7 +1736,7 @@ erts_check_io(ErtsPollThread *psi)
 	    }
         }
         if (resource) {
-            erts_resource_stop(resource, (ErlNifEvent)fd, 1);
+            erts_resource_stop(resource, (ErlNifEvent)fd, 0);
             enif_release_resource(resource->data);
         }
         if (free_select)
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index bec2291867..e259e39c86 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -653,7 +653,7 @@ select_steal(Config) when is_list(Config) ->
 
     check_stop_ret(select_nif(RFd, ?ERL_NIF_SELECT_STOP, RFd, null, Ref)),
     ?assertMatch([{fd_resource_stop, RPtr, _}], flush()),
-    {1, {RPtr, 1}} = last_fd_stop_call(),
+    {1, {RPtr, _DirectCall}} = last_fd_stop_call(),
 
     ?assert(is_closed_nif(WFd)),
 
-- 
cgit v1.2.3


From ad72c0d37ffb214cac874f51ac29fe2cdb47a2a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Sat, 3 Feb 2018 15:41:11 +0100
Subject: Introduce map_get guard-safe function

Rationale

Today all compound data types except for maps can be deconstructed in guards.
For tuples we have `element/2` and for lists `hd/1` and `tl/1`. Maps are
completely opaque to guards. This means matching on maps can't be
abstracted into macros, which is often done with repetitive guards. It
also means that maps have to be always selected whole from ETS tables,
even when only one field would be enough, which creates a potential
efficiency issue.

This PR introduces an `erlang:map_get/2` guard-safe function that allows
extracting a map field in guard. An alternative to this function would be
to introduce the syntax for extracting a value from a map that was planned
in the original EEP: `Map#{Key}`.

Even outside of guards, since this function is a guard-BIF it is more
efficient than using `maps:get/2` (since it does not need to set up the
stack), and more convenient from pattern matching on the map (compare:
`#{key := Value} = Map, Value` to `map_get(key, Map)`).

Performance considerations

A common concern against adding this function is the notion that "guards
have to be fast" and ideally execute in constant time. While there are
some counterexamples (`length/1`), what is more important is the fact
that adding those functions does not change in any way the time
complexity of pattern matching - it's already possible to match on map
fields today directly in patterns - adding this ability to guards will
niether slow down or speed up the execution, it will only make certain
programs more convenient to write.

This first version is very naive and does not perform any optimizations.
---
 erts/emulator/beam/bif.tab              |  1 +
 erts/emulator/beam/erl_db_util.c        |  8 ++++--
 erts/emulator/beam/erl_map.c            |  7 ++++-
 erts/emulator/test/map_SUITE.erl        | 45 ++++++++++++++++++++++++++++++++-
 erts/emulator/test/match_spec_SUITE.erl | 13 ++++++++++
 5 files changed, 70 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 687fd39d58..2e84056d9f 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -693,3 +693,4 @@ bif erts_internal:new_connection/1
 bif erts_internal:abort_connection/2
 bif erts_internal:map_next/3
 bif ets:whereis/1
+ubif erlang:map_get/2
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 3836f28aa4..6354abfd1f 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -637,6 +637,12 @@ static DMCGuardBif guard_tab[] =
 	1,
 	DBIF_ALL
     },
+    {
+        am_map_get,
+        &map_get_2,
+        2,
+        DBIF_ALL
+    },
     {
 	am_bit_size,
 	&bit_size_1,
@@ -5737,5 +5743,3 @@ void db_match_dis(Binary *bp)
 }
 
 #endif /* DMC_DEBUG */
-
-
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 4ec6960997..f577b017c3 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -44,6 +44,7 @@
  * DONE:
  * - erlang:is_map/1
  * - erlang:map_size/1
+ * - erlang:map_get/2
  *
  * - maps:find/2
  * - maps:from_list/1
@@ -202,7 +203,7 @@ BIF_RETTYPE maps_find_2(BIF_ALIST_2) {
     BIF_ERROR(BIF_P, BADMAP);
 }
 
-/* maps:get/2
+/* maps:get/2 and erlang:map_get/2
  * return value if key *matches* a key in the map
  * exception badkey if none matches
  */
@@ -223,6 +224,10 @@ BIF_RETTYPE maps_get_2(BIF_ALIST_2) {
     BIF_ERROR(BIF_P, BADMAP);
 }
 
+BIF_RETTYPE map_get_2(BIF_ALIST_2) {
+    BIF_RET(maps_get_2(BIF_CALL_ARGS));
+}
+
 /* maps:from_list/1
  * List may be unsorted [{K,V}]
  */
diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl
index c9e971af8a..43807b4388 100644
--- a/erts/emulator/test/map_SUITE.erl
+++ b/erts/emulator/test/map_SUITE.erl
@@ -36,6 +36,7 @@
          t_map_equal/1,
          t_map_compare/1,
          t_map_size/1,
+         t_map_get/1,
          t_is_map/1,
 
          %% Specific Map BIFs
@@ -124,7 +125,7 @@ all() -> [t_build_and_match_literals, t_build_and_match_literals_large,
           %% erlang
           t_erlang_hash, t_map_encode_decode,
           t_gc_rare_map_overflow,
-          t_map_size, t_is_map,
+          t_map_size, t_map_get, t_is_map,
 
           %% non specific BIF related
           t_bif_build_and_check,
@@ -680,6 +681,48 @@ t_map_size(Config) when is_list(Config) ->
               end),
     ok.
 
+t_map_get(Config) when is_list(Config) ->
+    %% small map
+    1    = map_get(a, id(#{a=>1})),
+    2    = map_get(b, id(#{a=>1, b=>2})),
+    "hi" = map_get("hello", id(#{a=>1, "hello"=>"hi"})),
+    "tuple hi" = map_get({1,1.0}, id(#{a=>a, {1,1.0}=>"tuple hi"})),
+
+    M0    = id(#{ k1=>"v1", <<"k2">> => <<"v3">> }),
+    "v4" = map_get(<<"k2">>, M0#{<<"k2">> => "v4"}),
+
+    %% large map
+    M1   = maps:from_list([{I,I}||I<-lists:seq(1,100)] ++
+			  [{a,1},{b,2},{"hello","hi"},{{1,1.0},"tuple hi"},
+			   {k1,"v1"},{<<"k2">>,"v3"}]),
+    1    = map_get(a, M1),
+    2    = map_get(b, M1),
+    "hi" = map_get("hello", M1),
+    "tuple hi" = map_get({1,1.0}, M1),
+    "v3" = map_get(<<"k2">>, M1),
+
+    %% error cases
+    do_badmap(fun(T) ->
+		      {'EXIT',{{badmap,T},[{erlang,map_get,_,_}|_]}} =
+			  (catch map_get(a, T))
+	      end),
+
+    {'EXIT',{{badkey,{1,1}},[{erlang,map_get,_,_}|_]}} =
+	(catch map_get({1,1}, id(#{{1,1.0}=>"tuple"}))),
+    {'EXIT',{{badkey,a},[{erlang,map_get,_,_}|_]}} = (catch map_get(a, id(#{}))),
+    {'EXIT',{{badkey,a},[{erlang,map_get,_,_}|_]}} =
+	(catch map_get(a, id(#{b=>1, c=>2}))),
+
+    %% in guards
+    M2 = id(#{a=>1}),
+    true = if map_get(a, M2) =:= 1 -> true; true -> false end,
+    false = if map_get(x, M2) =:= 1 -> true; true -> false end,
+    do_badmap(fun
+        (T) when map_get(T, x) =:= 1 -> ok;
+        (T) -> false = is_map(T)
+    end),
+    ok.
+
 build_and_check_size([K|Ks],N,M0) ->
     N = map_size(M0),
     M1 = M0#{ K => K },
diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl
index 08a7b4560c..c1bc01f01e 100644
--- a/erts/emulator/test/match_spec_SUITE.erl
+++ b/erts/emulator/test/match_spec_SUITE.erl
@@ -885,6 +885,19 @@ maps(Config) when is_list(Config) ->
         erlang:match_spec_test(#{<<"b">> =>"camembert","c"=>"cabécou", "wat"=>"hi", b=><<"other">>},
                                [{#{<<"b">> => '$1',"wat" => '$2'},[],[#{a=>'$1',b=>'$2'}]}],
                                table),
+
+    {ok,1,[],[]} = erlang:match_spec_test(#{a => 1}, [{'$1',[],[{map_size,'$1'}]}],table),
+    {ok,'EXIT',[],[]} = erlang:match_spec_test(not_a_map, [{'$1',[],[{map_size,'$1'}]}], table),
+    {ok,false,[],[]} = erlang:match_spec_test(not_a_map, [{'$1',[{map_size,'$1'}],['$_']}], table),
+    {ok,true,[],[]} = erlang:match_spec_test(#{a => 1}, [{'$1',[{'=:=',{map_size,'$1'},1}],[true]}], table),
+
+    {ok,1,[],[]} = erlang:match_spec_test(#{a => 1}, [{'$1',[],[{map_get,a,'$1'}]}], table),
+    {ok,'EXIT',[],[]} = erlang:match_spec_test(#{a => 1}, [{'$1',[],[{map_get,b,'$1'}]}], table),
+    {ok,'EXIT',[],[]} = erlang:match_spec_test(not_a_map, [{'$1',[],[{map_get,b,'$1'}]}], table),
+    {ok,false,[],[]} = erlang:match_spec_test(#{a => 1}, [{'$1',[{map_get,b,'$1'}],['$_']}], table),
+    {ok,false,[],[]} = erlang:match_spec_test(not_a_map, [{'$1',[{map_get,b,'$1'}],['$_']}], table),
+    {ok,true,[],[]} = erlang:match_spec_test(#{a => true}, [{'$1',[{map_get,a,'$1'}],[true]}], table),
+
     %% large maps
 
     Ls0 = [{I,<<I:32>>}||I <- lists:seq(1,415)],
-- 
cgit v1.2.3


From 630d7e7df458490ae4d5c263e096851e14349537 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 25 Apr 2018 13:00:36 +0200
Subject: Restore merge of signal queues in queue_messages() if main lock is
 held

---
 erts/emulator/beam/erl_gc.c      | 34 ++++------------------------------
 erts/emulator/beam/erl_message.c |  6 ++++++
 2 files changed, 10 insertions(+), 30 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index ea87cd7f50..dbdfdc6e86 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -61,7 +61,7 @@
 #  define ERTS_GC_ASSERT(B) ((void) 1)
 #endif
 
-#if defined(DEBUG) && 1
+#if defined(DEBUG) && 0
 #  define HARDDEBUG 1
 #endif
 
@@ -223,23 +223,6 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq,
                                  5,
                                  ERTS_ALC_T_GC_INFO_REQ)
 
-static ERTS_INLINE void
-ensure_sigq_roots_available(Process *p)
-{
-    ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p));
-    switch (p->flags & (F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG)) {
-    case F_OFF_HEAP_MSGQ_CHNG:
-    case 0:
-        erts_proc_lock(p, ERTS_PROC_LOCK_MSGQ);
-        erts_proc_sig_fetch(p);
-        erts_proc_unlock(p, ERTS_PROC_LOCK_MSGQ);
-        break;
-    default:
-        break;
-    }
-}
-
-
 /*
  * Initialize GC global data.
  */
@@ -443,8 +426,6 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end,
 	live_hf_end = ERTS_INVALID_HFRAG_PTR;
     }
 
-    ensure_sigq_roots_available(p);
-
     if (is_non_value(result)) {
 	if (p->freason == TRAP) {
 #ifdef HIPE
@@ -888,11 +869,8 @@ do_major_collection:
 int
 erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fcalls)
 {
-    int reds; 
-    int reds_left;
-    ensure_sigq_roots_available(p);
-    reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls, 0);
-    reds_left = ERTS_REDS_LEFT(p, fcalls);
+    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls, 0);
+    int reds_left = ERTS_REDS_LEFT(p, fcalls);
     if (reds > reds_left)
 	reds = reds_left;
     ASSERT(CONTEXT_REDS - (reds_left - reds) >= erts_proc_sched_data(p)->virtual_reds);
@@ -902,9 +880,7 @@ erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fca
 void
 erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
 {
-    int reds;
-    ensure_sigq_roots_available(p);
-    reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls, 0);
+    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls, 0);
     BUMP_REDS(p, reds);
     ASSERT(CONTEXT_REDS - ERTS_BIF_REDS_LEFT(p)
 	   >= erts_proc_sched_data(p)->virtual_reds);
@@ -1130,8 +1106,6 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
      * First an ordinary major collection...
      */
 
-    ensure_sigq_roots_available(p);
-
     p->flags |= F_NEED_FULLSWEEP;
 
     if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p)))
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 541d2e8ad1..deca426543 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -311,6 +311,9 @@ erts_queue_dist_message(Process *rcvr,
 
 	LINK_MESSAGE(rcvr, mp, &mp->next, 1);
 
+        if (rcvr_locks & ERTS_PROC_LOCK_MAIN)
+            erts_proc_sig_fetch(rcvr);
+
 	if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ))
 	    erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ);
 
@@ -376,6 +379,9 @@ queue_messages(Process* receiver,
 
     LINK_MESSAGE(receiver, first, last, len);
 
+    if (receiver_locks & ERTS_PROC_LOCK_MAIN)
+        erts_proc_sig_fetch(receiver);
+
     if (locked_msgq) {
 	erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
     }
-- 
cgit v1.2.3


From e08d90c51ad47b64f05bb1d51d9595b642637b9f Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 25 Apr 2018 16:05:03 +0200
Subject: Do not run tests that conflicts with dirty schedulers test

---
 erts/emulator/test/statistics_SUITE.erl | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl
index 029a6de897..3f2897242e 100644
--- a/erts/emulator/test/statistics_SUITE.erl
+++ b/erts/emulator/test/statistics_SUITE.erl
@@ -310,6 +310,13 @@ scheduler_wall_time_all(Config) when is_list(Config) ->
     scheduler_wall_time_test(scheduler_wall_time_all).
 
 scheduler_wall_time_test(Type) ->
+    case string:find(erlang:system_info(system_version),
+                     "dirty-schedulers-TEST") == nomatch of
+        true -> run_scheduler_wall_time_test(Type);
+        false -> {skip, "Cannot be run with dirty-schedulers-TEST build"}
+    end.
+
+run_scheduler_wall_time_test(Type) ->
     %% Should return undefined if system_flag is not turned on yet
     undefined = statistics(Type),
     %% Turn on statistics
-- 
cgit v1.2.3


From c26bfdd48c8deabe9cc0f67badb0d8a95a641845 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 24 Apr 2018 15:29:18 +0200
Subject: erts: Fix reduction bump for ets:delete/1

---
 erts/emulator/beam/erl_db.c | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index f7ee408991..2d46e81b3b 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1580,9 +1580,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     Sint keypos;
     int is_named, is_compressed;
     int is_fine_locked, frequent_read;
-#ifdef DEBUG
     int cret;
-#endif
     DbTableMethod* meth;
 
     if (is_not_atom(BIF_ARG_1)) {
@@ -1721,11 +1719,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     tb->common.fixing_procs = NULL;
     tb->common.compress = is_compressed;
 
-#ifdef DEBUG
-    cret = 
-#endif
-	meth->db_create(BIF_P, tb);
-    ASSERT(cret == DB_ERROR_NONE);
+    cret = meth->db_create(BIF_P, tb);
+    ASSERT(cret == DB_ERROR_NONE); (void)cret;
 
     make_btid(tb);
 
@@ -1941,7 +1936,8 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
     reds -= free_fixations_locked(BIF_P, tb);
     db_unlock(tb, LCK_WRITE);
 
-    if (free_table_continue(BIF_P, tb, reds) < 0) {
+    reds = free_table_continue(BIF_P, tb, reds);
+    if (reds < 0) {
 	/*
 	 * Package the DbTable* pointer into a bignum so that it can be safely
 	 * passed through a trap. We used to pass the DbTable* pointer directly
@@ -3940,7 +3936,8 @@ static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1)
 
     ASSERT(*ptr == make_pos_bignum_header(1));
 
-    if (free_table_continue(BIF_P, tb, reds) < 0) {
+    reds = free_table_continue(BIF_P, tb, reds);
+    if (reds < 0) {
         BUMP_ALL_REDS(BIF_P);
         BIF_TRAP1(&ets_delete_continue_exp, BIF_P, cont);
     }
-- 
cgit v1.2.3


From 05e1548e5b452d4bfe3bc6837a1a1b51a6367f72 Mon Sep 17 00:00:00 2001
From: Siri Hansen <siri@erlang.org>
Date: Tue, 24 Apr 2018 14:55:29 +0200
Subject: Remove error_logger process and add logger process

---
 erts/emulator/beam/atom.names  |   1 +
 erts/emulator/beam/erl_port.h  |  16 ++++
 erts/emulator/beam/erl_term.h  |  48 ++++++++++++
 erts/emulator/beam/erl_trace.c |  12 +--
 erts/emulator/beam/utils.c     | 162 ++++++++++++++++++++++-------------------
 5 files changed, 157 insertions(+), 82 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index cceca66850..fba0611042 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -358,6 +358,7 @@ atom loaded
 atom load_cancelled
 atom load_failure
 atom local
+atom logger
 atom long_gc
 atom long_schedule
 atom low
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index 2a98a6f00b..9b52b648e5 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -485,6 +485,8 @@ ERTS_GLB_INLINE Uint32 erts_portid2status(Eterm);
 ERTS_GLB_INLINE int erts_is_port_alive(Eterm);
 ERTS_GLB_INLINE int erts_is_valid_tracer_port(Eterm);
 ERTS_GLB_INLINE int erts_port_driver_callback_epilogue(Port *, erts_aint32_t *);
+ERTS_GLB_INLINE Port *erts_get_current_port(void);
+ERTS_GLB_INLINE Eterm erts_get_current_port_id(void);
 
 #define erts_drvport2port(Prt) erts_drvport2port_state((Prt), NULL)
 
@@ -812,6 +814,20 @@ erts_port_driver_callback_epilogue(Port *prt, erts_aint32_t *statep)
     return reds;
 }
 
+ERTS_GLB_INLINE
+Port *erts_get_current_port(void)
+{
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    return esdp ? esdp->current_port : NULL;
+}
+
+ERTS_GLB_INLINE
+Eterm erts_get_current_port_id(void)
+{
+    Port *port = erts_get_current_port();
+    return port ? port->common.id : THE_NON_VALUE;
+}
+
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 void erts_port_resume_procs(Port *);
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 18483fca35..bddf403b0a 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1184,6 +1184,54 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm)
 #define is_map(x)              (is_boxed((x)) && is_map_header(*boxed_val(x)))
 #define is_not_map(x)          (!is_map(x))
 
+#define MAP_HEADER(hp, sz, keys)                \
+    ((hp)[0] = MAP_HEADER_FLATMAP,              \
+     (hp)[1] = sz,                              \
+     (hp)[2] = keys)
+
+#define MAP_SZ(sz) (MAP_HEADER_FLATMAP_SZ + 2*sz + 1)
+
+#define MAP0_SZ MAP_SZ(0)
+#define MAP1_SZ MAP_SZ(1)
+#define MAP2_SZ MAP_SZ(2)
+#define MAP3_SZ MAP_SZ(3)
+#define MAP4_SZ MAP_SZ(4)
+#define MAP5_SZ MAP_SZ(5)
+#define MAP0(hp)                                                \
+    (MAP_HEADER(hp, 0, TUPLE0(hp+MAP_HEADER_FLATMAP_SZ)),       \
+     make_flatmap(hp))
+#define MAP1(hp, k1, v1)                                                \
+    (MAP_HEADER(hp, 1, TUPLE1(hp+1+MAP_HEADER_FLATMAP_SZ, k1)),         \
+     (hp)[MAP_HEADER_FLATMAP_SZ+0] = v1,                                \
+     make_flatmap(hp))
+#define MAP2(hp, k1, v1, k2, v2)                                        \
+    (MAP_HEADER(hp, 2, TUPLE2(hp+2+MAP_HEADER_FLATMAP_SZ, k1, k2)),     \
+     (hp)[MAP_HEADER_FLATMAP_SZ+0] = v1,                                \
+     (hp)[MAP_HEADER_FLATMAP_SZ+1] = v2,                                \
+     make_flatmap(hp))
+#define MAP3(hp, k1, v1, k2, v2, k3, v3)                \
+    (MAP_HEADER(hp, 3, TUPLE3(hp+3+MAP_HEADER_FLATMAP_SZ, k1, k2, k3)), \
+     (hp)[MAP_HEADER_FLATMAP_SZ+0] = v1,                                \
+     (hp)[MAP_HEADER_FLATMAP_SZ+1] = v2,                                \
+     (hp)[MAP_HEADER_FLATMAP_SZ+2] = v3,                                \
+     make_flatmap(hp))
+#define MAP4(hp, k1, v1, k2, v2, k3, v3, k4, v4)                \
+    (MAP_HEADER(hp, 4, TUPLE4(hp+4+MAP_HEADER_FLATMAP_SZ, k1, k2, k3, k4)), \
+     (hp)[MAP_HEADER_FLATMAP_SZ+0] = v1,                                \
+     (hp)[MAP_HEADER_FLATMAP_SZ+1] = v2,                                \
+     (hp)[MAP_HEADER_FLATMAP_SZ+2] = v3,                                \
+     (hp)[MAP_HEADER_FLATMAP_SZ+3] = v4,                                \
+     make_flatmap(hp))
+#define MAP5(hp, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5)                \
+    (MAP_HEADER(hp, 5, TUPLE5(hp+5+MAP_HEADER_FLATMAP_SZ, k1, k2, k3, k4, k5)), \
+     (hp)[MAP_HEADER_FLATMAP_SZ+0] = v1,                                \
+     (hp)[MAP_HEADER_FLATMAP_SZ+1] = v2,                                \
+     (hp)[MAP_HEADER_FLATMAP_SZ+2] = v3,                                \
+     (hp)[MAP_HEADER_FLATMAP_SZ+3] = v4,                                \
+     (hp)[MAP_HEADER_FLATMAP_SZ+4] = v5,                                \
+     make_flatmap(hp))
+
+
 /* number tests */
 
 #define is_integer(x)		(is_small(x) || is_big(x))
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 1e833539b3..065a560b52 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -2044,7 +2044,7 @@ enqueue_sys_msg(enum ErtsSysMsgType type,
 void
 erts_queue_error_logger_message(Eterm from, Eterm msg, ErlHeapFragment *bp)
 {
-    enqueue_sys_msg(SYS_MSG_TYPE_ERRLGR, from, am_error_logger, msg, bp);
+    enqueue_sys_msg(SYS_MSG_TYPE_ERRLGR, from, am_logger, msg, bp);
 }
 
 void
@@ -2110,13 +2110,13 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver)
 	erts_thr_progress_unblock();
 	break;
     case SYS_MSG_TYPE_ERRLGR: {
-	char *no_elgger = "(no error logger present)";
+	char *no_elgger = "(no logger present)";
 	Eterm *tp;
 	Eterm tag;
 	if (is_not_tuple(smqp->msg)) {
 	unexpected_elmsg:
 	    erts_fprintf(stderr,
-			 "%s unexpected error logger message: %T\n",
+			 "%s unexpected logger message: %T\n",
 			 no_elgger,
 			 smqp->msg);
 	}
@@ -2284,7 +2284,7 @@ sys_msg_dispatcher_func(void *unused)
 		}
 		break;
 	    case SYS_MSG_TYPE_ERRLGR:
-		receiver = am_error_logger;
+		receiver = am_logger;
 		break;
 	    default:
 		receiver = NIL;
@@ -2313,7 +2313,7 @@ sys_msg_dispatcher_func(void *unused)
 		    erts_proc_unlock(proc, proc_locks);
 		}
 	    }
-	    else if (receiver == am_error_logger) {
+	    else if (receiver == am_logger) {
 		proc = erts_whereis_process(NULL,0,receiver,proc_locks,0);
 		if (!proc)
 		    goto failure;
@@ -2379,7 +2379,7 @@ erts_foreach_sys_msg_in_q(void (*func)(Eterm,
 	    to = erts_get_system_profile();
 	    break;
 	case SYS_MSG_TYPE_ERRLGR:
-	    to = am_error_logger;
+	    to = am_logger;
 	    break;
 	default:
 	    to = NIL;
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 2e22130524..d74052d8b2 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1924,145 +1924,155 @@ make_internal_hash(Eterm term, Uint32 salt)
 #undef HCONST
 #undef MIX
 
+/* error_logger !
+   {log, Level, format, [args], #{ gl, pid, time, error_logger => #{tag, emulator => true} }}
+*/
 static Eterm
-do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp,
-			   ErlHeapFragment **bp, Process **p, Uint sz)
+do_allocate_logger_message(Eterm gleader, ErtsMonotonicTime *ts, Eterm *pid,
+                           Eterm **hp, ErlOffHeap **ohp,
+			   ErlHeapFragment **bp, Uint sz)
 {
     Uint gl_sz;
     gl_sz = IS_CONST(gleader) ? 0 : size_object(gleader);
-    sz = sz + gl_sz;
+    sz = sz + gl_sz + 6 /*outer 5-tuple*/
+        + MAP2_SZ /* error_logger map */;
+
+    *pid = erts_get_current_pid();
+
+    if (is_nil(gleader) && is_non_value(*pid)) {
+        sz += MAP2_SZ /* metadata map no gl, no pid */;
+    } else if (is_nil(gleader) || is_non_value(*pid))
+        sz += MAP3_SZ /* metadata map no gl or no pid*/;
+    else
+        sz += MAP4_SZ /* metadata map w gl w pid*/;
+
+    *ts = ERTS_MONOTONIC_TO_USEC(erts_get_monotonic_time(NULL)) + ERTS_MONOTONIC_OFFSET_USEC;
+    erts_bld_sint64(NULL, &sz, *ts);
 
     *bp = new_message_buffer(sz);
     *ohp = &(*bp)->off_heap;
     *hp = (*bp)->mem;
 
-    return (is_nil(gleader)
-	  ? am_noproc
-	  : (IS_CONST(gleader)
-	     ? gleader
-	     : copy_struct(gleader,gl_sz,hp,*ohp)));
+    return copy_struct(gleader,gl_sz,hp,*ohp);
 }
 
-static void do_send_logger_message(Eterm *hp, ErlOffHeap *ohp, ErlHeapFragment *bp,
-				   Process *p, Eterm message)
+static void do_send_logger_message(Eterm gl, Eterm tag, Eterm format, Eterm args,
+                                   ErtsMonotonicTime ts, Eterm pid,
+                                   Eterm *hp, ErlHeapFragment *bp)
 {
-#ifdef HARDDEBUG
-    erts_fprintf(stderr, "%T\n", message);
-#endif
-    {
-	Eterm from = erts_get_current_pid();
-	if (is_not_internal_pid(from))
-	    from = NIL;
-	erts_queue_error_logger_message(from, message, bp);
+    Eterm message, md, el_tag = tag;
+    Eterm time = erts_bld_sint64(&hp, NULL, ts);
+
+    /* This mapping is needed for the backwards compatible error_logger */
+    switch (tag) {
+    case am_info: el_tag = am_info_msg; break;
+    case am_warning: el_tag = am_warning_msg; break;
+    default:
+        ASSERT(am_error);
+        break;
     }
+
+    md = MAP2(hp, am_emulator, am_true,
+              am_atom_put("tag", 3), el_tag);
+    hp += MAP2_SZ;
+
+    if (is_nil(gl) && is_non_value(pid)) {
+        /* no gl and no pid, probably from a port */
+        md = MAP2(hp,
+                  am_error_logger, md,
+                  am_time, time);
+        hp += MAP2_SZ;
+        pid = NIL;
+    } else if (is_nil(gl)) {
+        /* no gl */
+        md = MAP3(hp,
+                  am_error_logger, md,
+                  am_pid, pid,
+                  am_time, time);
+        hp += MAP3_SZ;
+    } else if (is_non_value(pid)) {
+        /* no gl */
+        md = MAP3(hp,
+                  am_error_logger, md,
+                  am_atom_put("gl", 2), gl,
+                  am_time, time);
+        hp += MAP3_SZ;
+        pid = NIL;
+    } else {
+        md = MAP4(hp,
+                  am_error_logger, md,
+                  am_atom_put("gl", 2), gl,
+                  am_pid, pid,
+                  am_time, time);
+        hp += MAP4_SZ;
+    }
+
+    message = TUPLE5(hp, am_log, tag, format, args, md);
+    erts_queue_error_logger_message(pid, message, bp);
 }
 
-/* error_logger !
-   {notify,{info_msg,gleader,{emulator,format,[args]}}} |
-   {notify,{error,gleader,{emulator,format,[args]}}} |
-   {notify,{warning_msg,gleader,{emulator,format,[args}]}} */
-static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, size_t len)
+static int do_send_to_logger(Eterm tag, Eterm gl, char *buf, size_t len)
 {
     Uint sz;
-    Eterm gl;
-    Eterm list,args,format,tuple1,tuple2,tuple3;
+    Eterm list, args, format, pid;
+    ErtsMonotonicTime ts;
 
     Eterm *hp = NULL;
     ErlOffHeap *ohp = NULL;
     ErlHeapFragment *bp = NULL;
-    Process *p = NULL;
-
-    ASSERT(is_atom(tag));
-
-    if (len == 0) {
-	return -1;
-    }
 
     sz = len * 2 /* message list */ + 2 /* cons surrounding message list */
-	+ 3 /*outer 2-tuple*/ + 4 /* middle 3-tuple */ + 4 /*inner 3-tuple */
 	+ 8 /* "~s~n" */;
 
     /* gleader size is accounted and allocated next */
-    gl = do_allocate_logger_message(gleader, &hp, &ohp, &bp, &p, sz);
-
-    if(is_nil(gl)) {
-       /* buf *always* points to a null terminated string */
-       erts_fprintf(stderr, "(no error logger present) %T: \"%s\"\n",
-                    tag, buf);
-       return 0;
-    }
+    gl = do_allocate_logger_message(gl, &ts, &pid, &hp, &ohp, &bp, sz);
 
     list = buf_to_intlist(&hp, buf, len, NIL);
     args = CONS(hp,list,NIL);
     hp += 2;
     format = buf_to_intlist(&hp, "~s~n", 4, NIL);
-    tuple1 = TUPLE3(hp, am_emulator, format, args);
-    hp += 4;
-    tuple2 = TUPLE3(hp, tag, gl, tuple1);
-    hp += 4;
-    tuple3 = TUPLE2(hp, am_notify, tuple2);
 
-    do_send_logger_message(hp, ohp, bp, p, tuple3);
+    do_send_logger_message(gl, tag, format, args, ts, pid, hp, bp);
     return 0;
 }
 
-static int do_send_term_to_logger(Eterm tag, Eterm gleader,
+static int do_send_term_to_logger(Eterm tag, Eterm gl,
 				  char *buf, size_t len, Eterm args)
 {
     Uint sz;
-    Eterm gl;
     Uint args_sz;
-    Eterm format,tuple1,tuple2,tuple3;
+    Eterm format, pid;
+    ErtsMonotonicTime ts;
 
     Eterm *hp = NULL;
     ErlOffHeap *ohp = NULL;
     ErlHeapFragment *bp = NULL;
-    Process *p = NULL;
 
-    ASSERT(is_atom(tag));
+    ASSERT(len > 0);
 
     args_sz = size_object(args);
-    sz = len * 2 /* format */ + args_sz
-	+ 3 /*outer 2-tuple*/ + 4 /* middle 3-tuple */ + 4 /*inner 3-tuple */;
+    sz = len * 2 /* format */ + args_sz;
 
     /* gleader size is accounted and allocated next */
-    gl = do_allocate_logger_message(gleader, &hp, &ohp, &bp, &p, sz);
-
-    if(is_nil(gl)) {
-       /* buf *always* points to a null terminated string */
-       erts_fprintf(stderr, "(no error logger present) %T: \"%s\" %T\n",
-                    tag, buf, args);
-       return 0;
-    }
+    gl = do_allocate_logger_message(gl, &ts, &pid, &hp, &ohp, &bp, sz);
 
     format = buf_to_intlist(&hp, buf, len, NIL);
     args = copy_struct(args, args_sz, &hp, ohp);
-    tuple1 = TUPLE3(hp, am_emulator, format, args);
-    hp += 4;
-    tuple2 = TUPLE3(hp, tag, gl, tuple1);
-    hp += 4;
-    tuple3 = TUPLE2(hp, am_notify, tuple2);
 
-    do_send_logger_message(hp, ohp, bp, p, tuple3);
+    do_send_logger_message(gl, tag, format, args, ts, pid, hp, bp);
     return 0;
 }
 
 static ERTS_INLINE int
 send_info_to_logger(Eterm gleader, char *buf, size_t len)
 {
-    return do_send_to_logger(am_info_msg, gleader, buf, len);
+    return do_send_to_logger(am_info, gleader, buf, len);
 }
 
 static ERTS_INLINE int
 send_warning_to_logger(Eterm gleader, char *buf, size_t len)
 {
-    Eterm tag;
-    switch (erts_error_logger_warnings) {
-    case am_info:	tag = am_info_msg;	break;
-    case am_warning:	tag = am_warning_msg;	break;
-    default:		tag = am_error;		break;
-    }
-    return do_send_to_logger(tag, gleader, buf, len);
+    return do_send_to_logger(erts_error_logger_warnings, gleader, buf, len);
 }
 
 static ERTS_INLINE int
-- 
cgit v1.2.3


From 291cf6d8530f4e0ad2f22f199b5ae6975135bbb3 Mon Sep 17 00:00:00 2001
From: Siri Hansen <siri@erlang.org>
Date: Tue, 24 Apr 2018 15:18:31 +0200
Subject: Test cuddle for logger

---
 erts/emulator/test/dgawd_handler.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/dgawd_handler.erl b/erts/emulator/test/dgawd_handler.erl
index 52cdd26427..29b9d6ac7b 100644
--- a/erts/emulator/test/dgawd_handler.erl
+++ b/erts/emulator/test/dgawd_handler.erl
@@ -42,10 +42,10 @@
 %%====================================================================
 
 install() ->
-    gen_event:add_handler(error_logger, ?MODULE, []).
+    error_logger:add_report_handler(?MODULE, []).
 
 restore() ->
-    gen_event:delete_handler(error_logger, ?MODULE, []).
+    error_logger:delete_report_handler(?MODULE).
 
 got_dgawd_report() ->
     gen_event:call(error_logger, ?MODULE, got_dgawd_report, 10*60*1000).
-- 
cgit v1.2.3


From c06c51b3cfa04787f53d835b10697eb7bfba844c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 26 Apr 2018 14:50:37 +0200
Subject: erts: Fix memory leak in lock checker at thread exit

Leak introduced in 865ac3b740d9efa1a0583349929591c757300412.
---
 erts/emulator/beam/erl_lock_check.c | 41 +++++++++++++++++++++++--------------
 1 file changed, 26 insertions(+), 15 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index d66410367b..463ae898a3 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -217,6 +217,14 @@ typedef struct {
 
 static lc_matrix_t tot_lc_matrix;
 
+#define ERTS_LC_FB_CHUNK_SIZE 10
+
+typedef struct lc_alloc_chunk_t_ lc_alloc_chunk_t;
+struct lc_alloc_chunk_t_ {
+    lc_alloc_chunk_t* next;
+    lc_free_block_t array[ERTS_LC_FB_CHUNK_SIZE];
+};
+
 typedef struct lc_thread_t_ lc_thread_t;
 struct lc_thread_t_ {
     char *thread_name;
@@ -227,6 +235,7 @@ struct lc_thread_t_ {
     lc_locked_lock_list_t locked;
     lc_locked_lock_list_t required;
     lc_free_block_t *free_blocks;
+    lc_alloc_chunk_t *chunks;
     lc_matrix_t matrix;
 };
 
@@ -235,14 +244,6 @@ static ethr_tsd_key locks_key;
 static lc_thread_t *lc_threads = NULL;
 static ethr_spinlock_t lc_threads_lock;
 
-
-#ifdef ERTS_LC_STATIC_ALLOC
-#define ERTS_LC_FB_CHUNK_SIZE 10000
-#else
-#define ERTS_LC_FB_CHUNK_SIZE 10
-#endif
-
-
 static ERTS_INLINE void
 lc_lock_threads(void)
 {
@@ -268,12 +269,16 @@ static ERTS_INLINE void lc_free(lc_thread_t* thr, lc_locked_lock_t *p)
 static lc_locked_lock_t *lc_core_alloc(lc_thread_t* thr)
 {
     int i;
-    lc_free_block_t *fbs;
-    fbs = (lc_free_block_t *) malloc(sizeof(lc_free_block_t)
-					  * ERTS_LC_FB_CHUNK_SIZE);
-    if (!fbs) {
+    lc_alloc_chunk_t* chunk;
+    lc_free_block_t* fbs;
+    chunk = (lc_alloc_chunk_t*) malloc(sizeof(lc_alloc_chunk_t));
+    if (!chunk) {
         ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!");
     }
+    chunk->next = thr->chunks;
+    thr->chunks = chunk;
+
+    fbs = chunk->array;
     for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) {
 #ifdef DEBUG
 	sys_memset((void *) &fbs[i], 0xdf, sizeof(lc_free_block_t));
@@ -321,6 +326,7 @@ create_thread_data(char *thread_name)
     thr->locked.last = NULL;
     thr->prev = NULL;
     thr->free_blocks = NULL;
+    thr->chunks = NULL;
     sys_memzero(&thr->matrix, sizeof(thr->matrix));
 
     lc_lock_threads();
@@ -336,7 +342,7 @@ create_thread_data(char *thread_name)
 static void collect_matrix(lc_matrix_t*);
 
 static void
-destroy_locked_locks(lc_thread_t *thr)
+destroy_thread_data(lc_thread_t *thr)
 {
     ASSERT(thr->thread_name);
     free((void *) thr->thread_name);
@@ -359,8 +365,13 @@ destroy_locked_locks(lc_thread_t *thr)
 
     lc_unlock_threads();
 
-    free((void *) thr);
+    while (thr->chunks) {
+        lc_alloc_chunk_t* free_me = thr->chunks;
+        thr->chunks = thr->chunks->next;
+        free(free_me);
+    }
 
+    free((void *) thr);
 }
 
 static ERTS_INLINE lc_thread_t *
@@ -615,7 +626,7 @@ thread_exit_handler(void)
 	    print_curr_locks(thr);
 	    lc_abort();
 	}
-	destroy_locked_locks(thr);
+	destroy_thread_data(thr);
 	/* erts_tsd_set(locks_key, NULL); */
     }
 }
-- 
cgit v1.2.3


From 48d18d5c6c41833d115a46d5086826d399715796 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 26 Apr 2018 16:35:16 +0200
Subject: Fix handling of process-info requests in receive

---
 erts/emulator/beam/erl_bif_info.c    | 8 ++++----
 erts/emulator/beam/msg_instrs.tab    | 3 +++
 erts/emulator/hipe/hipe_native_bif.c | 3 +++
 3 files changed, 10 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index d443e12409..f1d2aea33e 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -756,7 +756,7 @@ typedef struct {
 
 static ErtsProcessInfoArgs pi_args[] = {
     {am_registered_name, 0, 0, ERTS_PROC_LOCK_MAIN},
-    {am_current_function, 4, 0, ERTS_PROC_LOCK_MAIN},
+    {am_current_function, 4, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
     {am_initial_call, 4, 0, ERTS_PROC_LOCK_MAIN},
     {am_status, 0, 0, 0},
     {am_messages, 0, ERTS_PI_FLAG_WANT_MSGS|ERTS_PI_FLAG_NEED_MSGQ_LEN|ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
@@ -778,15 +778,15 @@ static ErtsProcessInfoArgs pi_args[] = {
     {am_binary, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
     {am_sequential_trace_token, 0, 0, ERTS_PROC_LOCK_MAIN},
     {am_catchlevel, 0, 0, ERTS_PROC_LOCK_MAIN},
-    {am_backtrace, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_backtrace, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
     {am_last_calls, 0, 0, ERTS_PROC_LOCK_MAIN},
     {am_total_heap_size, 0, ERTS_PI_FLAG_NEED_MSGQ_LEN|ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
     {am_suspending, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, 0},
     {am_min_heap_size, 0, 0, ERTS_PROC_LOCK_MAIN},
     {am_min_bin_vheap_size, 0, 0, ERTS_PROC_LOCK_MAIN},
     {am_max_heap_size, 0, 0, ERTS_PROC_LOCK_MAIN},
-    {am_current_location, 0, 0, ERTS_PROC_LOCK_MAIN},
-    {am_current_stacktrace, 0, 0, ERTS_PROC_LOCK_MAIN},
+    {am_current_location, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
+    {am_current_stacktrace, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
     {am_message_queue_data, 0, 0, ERTS_PROC_LOCK_MAIN},
     {am_garbage_collection_info, ERTS_PROCESS_GC_INFO_MAX_SIZE, 0, ERTS_PROC_LOCK_MAIN},
     {am_magic_ref, 0, ERTS_PI_FLAG_FORCE_SIG_SEND, ERTS_PROC_LOCK_MAIN},
diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab
index 223f6bec72..816431c2b4 100644
--- a/erts/emulator/beam/msg_instrs.tab
+++ b/erts/emulator/beam/msg_instrs.tab
@@ -102,6 +102,9 @@ i_loop_rec(Dest) {
     if (ERTS_UNLIKELY(msgp == NULL)) {
         int get_out;
         SWAPOUT;
+        $SET_CP_I_ABS(I);
+        c_p->arity = 0;
+        c_p->current = NULL;
         FCALLS -= erts_proc_sig_receive_helper(c_p, FCALLS, neg_o_reds,
                                                &msgp, &get_out);
         SWAPIN;
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index cf8c4139be..24078af046 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -546,6 +546,9 @@ Eterm hipe_check_get_msg(Process *c_p)
 
     if (!msgp) {
         int get_out;
+        c_p->i = NULL;
+        c_p->arity = 0;
+        c_p->current = NULL;
         (void) erts_proc_sig_receive_helper(c_p, CONTEXT_REDS, 0,
                                             &msgp, &get_out);
         /* FIXME: Need to bump reductions... */
-- 
cgit v1.2.3


From d7e27dd3174f61a6119cfaf505f5ede4b0ab5371 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 12 Apr 2018 17:36:21 +0200
Subject: erts: Cleanup some code

---
 erts/emulator/beam/dist.c               | 14 ++++----------
 erts/emulator/beam/erl_proc_sig_queue.c | 25 ++++++-------------------
 2 files changed, 10 insertions(+), 29 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index f203d85ca9..24e1cac7bf 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1471,22 +1471,16 @@ int erts_net_message(Port *prt,
             mdp = erts_monitor_create(ERTS_MON_TYPE_DIST_PROC,
                                       ref, watcher, pid, name);
 
-#ifdef DEBUG
-            code =
-#endif
-                erts_monitor_dist_insert(&mdp->origin, dep->mld);
-            ASSERT(code);
+            code = erts_monitor_dist_insert(&mdp->origin, dep->mld);
+            ASSERT(code); (void)code;
 
             if (erts_proc_sig_send_monitor(&mdp->target, pid))
                 break; /* done */
 
             /* Failed to send to local proc; cleanup reply noproc... */
 
-#ifdef DEBUG
-            code =
-#endif
-                erts_monitor_dist_delete(&mdp->origin);
-            ASSERT(code);
+            code = erts_monitor_dist_delete(&mdp->origin);
+            ASSERT(code); (void)code;
             erts_monitor_release_both(mdp);
 
         }
diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index 2d59724aad..6e940eb3f7 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -612,11 +612,6 @@ maybe_elevate_sig_handling_prio(Process *c_p, Eterm other)
 void
 erts_proc_sig_fetch__(Process *proc)
 {
-#ifdef ERTS_PROC_SIG_HARD_DEBUG
-    ErtsSignalPrivQueues sig_qs = proc->sig_qs;
-    ErtsSignalInQueue sig_inq = proc->sig_inq;
-#endif
-
     ASSERT(proc->sig_inq.first);
 
     if (!proc->sig_inq.nmsigs.next) {
@@ -634,9 +629,7 @@ erts_proc_sig_fetch__(Process *proc)
         }
     }
     else {
-#ifdef DEBUG
         erts_aint32_t s;
-#endif
         ASSERT(proc->sig_inq.nmsigs.last);
          if (!proc->sig_qs.nmsigs.last) {
             ASSERT(!proc->sig_qs.nmsigs.next);
@@ -645,16 +638,13 @@ erts_proc_sig_fetch__(Process *proc)
             else
                 proc->sig_qs.nmsigs.next = proc->sig_inq.nmsigs.next;
 
-#ifdef DEBUG
-            s =
-#endif
-                erts_atomic32_read_bset_nob(&proc->state,
+            s = erts_atomic32_read_bset_nob(&proc->state,
                                             (ERTS_PSFLG_SIG_Q
                                              | ERTS_PSFLG_SIG_IN_Q),
                                             ERTS_PSFLG_SIG_Q);
 
             ASSERT((s & (ERTS_PSFLG_SIG_Q|ERTS_PSFLG_SIG_IN_Q))
-                   == ERTS_PSFLG_SIG_IN_Q);
+                   == ERTS_PSFLG_SIG_IN_Q); (void)s;
         }
         else {
             ErtsSignal *sig;
@@ -667,14 +657,11 @@ erts_proc_sig_fetch__(Process *proc)
             else
                 sig->common.specific.next = proc->sig_inq.nmsigs.next;
 
-#ifdef DEBUG
-            s =
-#endif
-                erts_atomic32_read_band_nob(&proc->state,
+            s = erts_atomic32_read_band_nob(&proc->state,
                                             ~ERTS_PSFLG_SIG_IN_Q);
 
             ASSERT((s & (ERTS_PSFLG_SIG_Q|ERTS_PSFLG_SIG_IN_Q))
-                   == (ERTS_PSFLG_SIG_Q|ERTS_PSFLG_SIG_IN_Q));
+                   == (ERTS_PSFLG_SIG_Q|ERTS_PSFLG_SIG_IN_Q)); (void)s;
         }
         if (proc->sig_inq.nmsigs.last == &proc->sig_inq.first)
             proc->sig_qs.nmsigs.last = proc->sig_qs.cont_last;
@@ -4273,7 +4260,7 @@ erts_proc_sig_hdbg_check_in_queue(Process *p, char *what, char *file, int line)
                                     NULL,
                                     NULL,
                                     ERTS_PSFLG_SIG_IN_Q);
-    ASSERT(p->sig_inq.len == len);
+    ASSERT(p->sig_inq.len == len); (void)len;
 }
 
-#endif
+#endif /* ERTS_PROC_SIG_HARD_DEBUG */
-- 
cgit v1.2.3


From 613cde66c25464121f2f6dace99782bad0e07d9b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 26 Apr 2018 19:14:31 +0200
Subject: erts: Optimize monitor signal by message piggyback

If no message/signal is sent (to same destination)
then monitor signal is flushed when process is scheduled out.
---
 erts/emulator/beam/beam_bif_load.c      |   6 +-
 erts/emulator/beam/beam_debug.c         |   2 +-
 erts/emulator/beam/dist.c               |   2 +-
 erts/emulator/beam/erl_db.c             |   2 +-
 erts/emulator/beam/erl_message.c        | 144 +++++++++++++----------
 erts/emulator/beam/erl_message.h        |  27 +++--
 erts/emulator/beam/erl_nif.c            |   7 +-
 erts/emulator/beam/erl_proc_sig_queue.c | 197 +++++++++++++++++++++++++++-----
 erts/emulator/beam/erl_proc_sig_queue.h |  12 ++
 erts/emulator/beam/erl_process.c        |  23 +++-
 erts/emulator/beam/erl_process.h        |   7 ++
 11 files changed, 318 insertions(+), 111 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 5c76aafae7..d9312f4df8 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1757,11 +1757,11 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 		release_literal_areas.last = ref;
 	    }
 	    erts_mtx_unlock(&release_literal_areas.mtx);
-	    erts_queue_message(erts_literal_area_collector,
+	    erts_queue_proc_message(BIF_P,
+                               erts_literal_area_collector,
 			       0,
 			       erts_alloc_message(0, NULL),
-			       am_copy_literals,
-			       BIF_P->common.id);
+			       am_copy_literals);
 	}
 
 	return ret;
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 5eb68b817e..b8a8d06315 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -1191,7 +1191,7 @@ dirty_send_message(Process *c_p, Eterm to, Eterm tag)
     mp = erts_alloc_message_heap(rp, &rp_locks, 3, &hp, &ohp);
 
     msg = TUPLE2(hp, tag, c_p->common.id);
-    erts_queue_message(rp, rp_locks, mp, msg, c_p->common.id);
+    erts_queue_proc_message(c_p, rp, rp_locks, mp, msg);
 
     if (rp == real_c_p)
 	rp_locks &= ~c_p_locks;
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 24e1cac7bf..026f0a62d4 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -3587,7 +3587,7 @@ int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks)
         dhandle = erts_build_dhandle(&hp, ohp, dep);
         msg = TUPLE4(hp, am_auto_connect, dep->sysname, make_small(conn_id),
                      dhandle);
-        erts_queue_message(net_kernel, nk_locks, mp, msg, proc->common.id);
+        erts_queue_proc_message(proc, net_kernel, nk_locks, mp, msg);
         erts_proc_unlock(net_kernel, nk_locks);
     }
 
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index f7ee408991..ca2ebb7c27 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -3547,7 +3547,7 @@ send_ets_transfer_message(Process *c_p, Process *proc,
         hd_copy = copy_struct(heir_data, hd_sz, &hp, ohp);
     sender = c_p->common.id;
     msg = TUPLE4(hp, am_ETS_TRANSFER, tid, sender, hd_copy);
-    erts_queue_message(proc, *locks, mp, msg, sender);
+    erts_queue_proc_message(c_p, proc, *locks, mp, msg);
 }
 
 
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 8eae85ccdd..bea7a0fe86 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -303,8 +303,7 @@ erts_queue_dist_message(Process *rcvr,
 	erts_cleanup_messages(mp);
     }
     else {
-
-	LINK_MESSAGE(rcvr, mp, &mp->next, 1);
+	LINK_MESSAGE(rcvr, mp);
 
         if (rcvr_locks & ERTS_PROC_LOCK_MAIN)
             erts_proc_sig_fetch(rcvr);
@@ -317,9 +316,8 @@ erts_queue_dist_message(Process *rcvr,
 }
 
 /* Add messages last in message queue */
-static Sint
+static void
 queue_messages(Process* receiver,
-               erts_aint32_t *receiver_state,
                ErtsProcLocks receiver_locks,
                ErtsMessage* first,
                ErtsMessage** last,
@@ -328,51 +326,51 @@ queue_messages(Process* receiver,
     int locked_msgq = 0;
     erts_aint32_t state;
 
-    ASSERT(is_value(ERL_MESSAGE_TERM(first)));
-    ASSERT(is_value(ERL_MESSAGE_FROM(first)));
-    ASSERT(ERL_MESSAGE_TOKEN(first) == am_undefined ||
-           ERL_MESSAGE_TOKEN(first) == NIL ||
-           is_tuple(ERL_MESSAGE_TOKEN(first)));
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(receiver) < ERTS_PROC_LOCK_MSGQ ||
-                       receiver_locks == erts_proc_lc_my_proc_locks(receiver));
+#ifdef DEBUG
+    {
+        ErtsMessage* fmsg = ERTS_SIG_IS_MSG(first) ? first : first->next;
+        ASSERT(fmsg);
+        ASSERT(is_value(ERL_MESSAGE_TERM(fmsg)));
+        ASSERT(is_value(ERL_MESSAGE_FROM(fmsg)));
+        ASSERT(ERL_MESSAGE_TOKEN(fmsg) == am_undefined ||
+               ERL_MESSAGE_TOKEN(fmsg) == NIL ||
+               is_tuple(ERL_MESSAGE_TOKEN(fmsg)));
+    }
 #endif
 
-    if (!(receiver_locks & ERTS_PROC_LOCK_MSGQ)) {
-	if (erts_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) {
-            ErtsProcLocks need_locks;
+    ERTS_LC_ASSERT((erts_proc_lc_my_proc_locks(receiver) & ERTS_PROC_LOCK_MSGQ)
+                   == (receiver_locks & ERTS_PROC_LOCK_MSGQ));
 
-	    if (receiver_state)
-		state = *receiver_state;
-	    else
-		state = erts_atomic32_read_nob(&receiver->state);
-	    if (state & ERTS_PSFLG_EXITING)
-		goto exiting;
-
-            need_locks = receiver_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ);
-	    if (need_locks) {
-		erts_proc_unlock(receiver, need_locks);
-	    }
-            need_locks |= ERTS_PROC_LOCK_MSGQ;
-	    erts_proc_lock(receiver, need_locks);
-	}
+    if (!(receiver_locks & ERTS_PROC_LOCK_MSGQ)) {
+        erts_proc_lock(receiver, ERTS_PROC_LOCK_MSGQ);
 	locked_msgq = 1;
     }
 
-
     state = erts_atomic32_read_nob(&receiver->state);
 
     if (state & ERTS_PSFLG_EXITING) {
-    exiting:
 	/* Drop message if receiver is exiting or has a pending exit... */
 	if (locked_msgq)
-	    erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
+            erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
+        if (ERTS_SIG_IS_NON_MSG(first)) {
+            ErtsSchedulerData* esdp = erts_get_scheduler_data();
+            ASSERT(esdp);
+            ASSERT(!esdp->pending_signal.sig);
+            esdp->pending_signal.sig = (ErtsSignal*) first;
+            esdp->pending_signal.to = receiver->common.id;
+            first = first->next;
+        }
 	erts_cleanup_messages(first);
-	return 0;
+        return;
     }
 
-    LINK_MESSAGE(receiver, first, last, len);
+    if (last == &first->next) {
+        ASSERT(len == 1);
+        LINK_MESSAGE(receiver, first);
+    }
+    else {
+        erts_enqueue_signals(receiver, first, last, NULL, len, state);
+    }
 
     if (receiver_locks & ERTS_PROC_LOCK_MAIN)
         erts_proc_sig_fetch(receiver);
@@ -382,35 +380,67 @@ queue_messages(Process* receiver,
     }
 
     erts_proc_notify_new_message(receiver, receiver_locks);
-    return 0;
 }
 
-static Sint
-queue_message(Process* receiver,
-              erts_aint32_t *receiver_state,
-              ErtsProcLocks receiver_locks,
-              ErtsMessage* mp, Eterm msg, Eterm from)
+static ERTS_INLINE
+ErtsMessage* prepend_pending_sig_maybe(Process* sender, Process* receiver,
+                                       ErtsMessage* mp)
 {
-    ERL_MESSAGE_TERM(mp) = msg;
-    ERL_MESSAGE_FROM(mp) = from;
-    return queue_messages(receiver, receiver_state, receiver_locks,
-                          mp, &mp->next, 1);
+    ErtsSchedulerData* esdp = sender->scheduler_data;
+    ErtsSignal* pend_sig;
+
+    if (!esdp || esdp->pending_signal.to != receiver->common.id)
+        return mp;
+   
+     pend_sig = esdp->pending_signal.sig;
+
+     ASSERT(esdp->pending_signal.dbg_from == sender);
+     esdp->pending_signal.sig = NULL;
+     esdp->pending_signal.to = THE_NON_VALUE;
+     pend_sig->common.next = mp;
+     pend_sig->common.specific.next = NULL;
+     return (ErtsMessage*) pend_sig;
 }
 
-Sint
+/**
+ *
+ * @brief Send one message from *NOT* a local process.
+ *
+ */
+void
 erts_queue_message(Process* receiver, ErtsProcLocks receiver_locks,
                    ErtsMessage* mp, Eterm msg, Eterm from)
 {
-    return queue_message(receiver, NULL, receiver_locks, mp, msg, from);
+    ASSERT(is_not_internal_pid(from));
+    ERL_MESSAGE_TERM(mp) = msg;
+    ERL_MESSAGE_FROM(mp) = from;
+    queue_messages(receiver, receiver_locks, mp, &mp->next, 1);
 }
 
+/**
+ * @brief Send one message from a local process.
+ */
+void
+erts_queue_proc_message(Process* sender,
+                        Process* receiver, ErtsProcLocks receiver_locks,
+                        ErtsMessage* mp, Eterm msg)
+{
+    ERL_MESSAGE_TERM(mp) = msg;
+    ERL_MESSAGE_FROM(mp) = sender->common.id;
+    queue_messages(receiver, receiver_locks,
+                   prepend_pending_sig_maybe(sender, receiver, mp),
+                   &mp->next, 1);
+}
 
-Sint
-erts_queue_messages(Process* receiver, ErtsProcLocks receiver_locks,
-                    ErtsMessage* first, ErtsMessage** last, Uint len)
+
+void
+erts_queue_proc_messages(Process* sender,
+                         Process* receiver, ErtsProcLocks receiver_locks,
+                         ErtsMessage* first, ErtsMessage** last, Uint len)
 {
-    return queue_messages(receiver, NULL, receiver_locks,
-                          first, last, len);
+    queue_messages(receiver, receiver_locks,
+                   prepend_pending_sig_maybe(sender, receiver, first),
+                   last, len);
 }
 
 void
@@ -547,7 +577,7 @@ erts_try_alloc_message_on_heap(Process *pp,
  * Send a local message when sender & receiver processes are known.
  */
 
-Sint
+void
 erts_send_message(Process* sender,
 		  Process* receiver,
 		  ErtsProcLocks *receiver_locks,
@@ -558,7 +588,6 @@ erts_send_message(Process* sender,
     ErtsMessage* mp;
     ErlOffHeap *ohp;
     Eterm token = NIL;
-    Sint res = 0;
 #ifdef USE_VM_PROBES
     DTRACE_CHARBUF(sender_name, 64);
     DTRACE_CHARBUF(receiver_name, 64);
@@ -701,13 +730,8 @@ erts_send_message(Process* sender,
 #ifdef USE_VM_PROBES
     ERL_MESSAGE_DT_UTAG(mp) = utag;
 #endif
-    res = queue_message(receiver,
-			&receiver_state,
-			*receiver_locks,
-			mp, message,
-                        sender->common.id);
 
-    return res;
+    erts_queue_proc_message(sender, receiver, *receiver_locks, mp, message);
 }
 
 
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index ee87297ba4..d120111634 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -229,7 +229,7 @@ typedef union {
 typedef struct {
     /* pointers to next pointers pointing to... */
     ErtsMessage **next; /* ... next (non-message) signal */
-    ErtsMessage **last; /* ... next (non-message) signal */
+    ErtsMessage **last; /* ... last (non-message) signal */
 } ErtsMsgQNMSigs;
 
 /* Size of default message buffer (erl_message.c) */
@@ -296,8 +296,11 @@ typedef struct {
 typedef struct {
     ErtsMessage* first;
     ErtsMessage** last;  /* point to the last next pointer */
-    Sint len;            /* queue length */
+    Sint len;            /* number of messages in queue */
     ErtsMsgQNMSigs nmsigs;
+#ifdef ERTS_PROC_SIG_HARD_DEBUG
+    int may_contain_heap_terms;
+#endif
 } ErtsSignalInQueue;
 
 typedef struct erl_trace_message_queue__ {
@@ -364,13 +367,14 @@ typedef struct erl_trace_message_queue__ {
 
 #endif
 
-/* Add message last_msg in message queue */
-#define LINK_MESSAGE(p, first_msg, last_msg, num_msgs) \
+/* Add one message last in message queue */
+#define LINK_MESSAGE(p, msg) \
     do {                                                                \
+        ASSERT(ERTS_SIG_IS_MSG(msg));                                   \
         ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__((p), "before");               \
-        *(p)->sig_inq.last = (first_msg);                               \
-        (p)->sig_inq.last = (last_msg);                                 \
-        (p)->sig_inq.len += (num_msgs);                                 \
+        *(p)->sig_inq.last = (msg);                                     \
+        (p)->sig_inq.last = &(msg)->next;                               \
+        (p)->sig_inq.len++;                                             \
         ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE__((p), "before");               \
     } while(0)
 
@@ -437,11 +441,12 @@ ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint,
 					    Eterm *, Uint);
 void free_message_buffer(ErlHeapFragment *);
 void erts_queue_dist_message(Process*, ErtsProcLocks, ErtsDistExternal *, Eterm, Eterm);
-Sint erts_queue_message(Process*, ErtsProcLocks,ErtsMessage*, Eterm, Eterm);
-Sint erts_queue_messages(Process*, ErtsProcLocks,
-                         ErtsMessage*, ErtsMessage**, Uint);
+void erts_queue_message(Process*, ErtsProcLocks,ErtsMessage*, Eterm, Eterm);
+void erts_queue_proc_message(Process* from,Process* to, ErtsProcLocks,ErtsMessage*, Eterm);
+void erts_queue_proc_messages(Process* from, Process* to, ErtsProcLocks,
+                              ErtsMessage*, ErtsMessage**, Uint);
 void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm);
-Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned);
+void erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned);
 void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp);
 
 Uint erts_msg_attached_data_size_aux(ErtsMessage *msg);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 260656e2d3..e208792868 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -665,7 +665,7 @@ int erts_flush_trace_messages(Process *c_p, ErtsProcLocks c_p_locks)
             rp_locks = 0;
             if (rp->common.id == c_p->common.id)
                 rp_locks = c_p_locks;
-            erts_queue_messages(rp, rp_locks, first, last, len);
+            erts_queue_proc_messages(c_p, rp, rp_locks, first, last, len);
             if (rp->common.id == c_p->common.id)
                 rp_locks &= ~c_p_locks;
             if (rp_locks)
@@ -856,7 +856,10 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
         }
     }
 
-    erts_queue_message(rp, rp_locks, mp, msg, from);
+    if (c_p)
+        erts_queue_proc_message(c_p, rp, rp_locks, mp, msg);
+    else
+        erts_queue_message(rp, rp_locks, mp, msg, from);
 
 done:
     if (c_p == rp)
diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index 6e940eb3f7..a2e6f1d39d 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -308,9 +308,8 @@ destroy_sig_group_leader(ErtsSigGroupLeader *sgl)
 }
 
 static ERTS_INLINE void
-sig_enqueue_trace(Process *c_p, ErtsMessage *sig, int op,
-                  Process *rp, ErtsMessage **first,
-                  ErtsMessage **last, ErtsMessage ***last_next)
+sig_enqueue_trace(Process *c_p, ErtsMessage **sigp, int op,
+                  Process *rp, ErtsMessage ***last_next)
 {
     switch (op) {
     case ERTS_SIG_Q_OP_LINK:
@@ -326,12 +325,11 @@ sig_enqueue_trace(Process *c_p, ErtsMessage *sig, int op,
              * Prepend a trace-change-state signal before the
              * link signal...
              */
-
             tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_TRACE_CHANGE_STATE,
                                          ERTS_SIG_Q_TYPE_ADJUST_TRACE_INFO,
                                          0);
             ti = erts_alloc(ERTS_ALC_T_SIG_DATA, sizeof(ErtsSigTraceInfo));
-            ti->common.next = *last;
+            ti->common.next = *sigp;
             ti->common.specific.next = &ti->common.next;
             ti->common.tag = tag;
             ti->flags_on = ERTS_TRACE_FLAGS(c_p) & TRACEE_FLAGS;
@@ -344,8 +342,9 @@ sig_enqueue_trace(Process *c_p, ErtsMessage *sig, int op,
                 erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
             }
             erts_tracer_update(&ti->tracer, ERTS_TRACER(c_p));
-            *first = (ErtsMessage *) ti;
-            *last_next = &ti->common.next;
+            *sigp = (ErtsMessage *) ti;
+            if (!*last_next || *last_next == sigp)
+                *last_next = &ti->common.next;
         }
         break;
 
@@ -354,6 +353,7 @@ sig_enqueue_trace(Process *c_p, ErtsMessage *sig, int op,
     case ERTS_SIG_Q_OP_EXIT_LINKED:
 
         if (DTRACE_ENABLED(process_exit_signal)) {
+            ErtsMessage* sig = *sigp;
             Uint16 type = ERTS_PROC_SIG_TYPE(((ErtsSignal *) sig)->common.tag);
             Eterm reason, from;
 
@@ -430,9 +430,24 @@ sig_enqueue_trace_cleanup(ErtsMessage *first, ErtsSignal *sig, ErtsMessage *last
     }
 }
 
+#ifdef DEBUG
+static int dbg_count_nmsigs(ErtsMessage *first)
+{
+    ErtsMessage *sig;
+    int cnt = 0;
+
+    for (sig = first; sig; sig = sig->next) {
+        if (ERTS_SIG_IS_NON_MSG(sig))
+            ++cnt;
+    }
+    return cnt;
+}
+#endif
+
 static ERTS_INLINE erts_aint32_t
 enqueue_signals(Process *rp, ErtsMessage *first,
-                ErtsMessage *last, ErtsMessage **last_next,
+                ErtsMessage **last, ErtsMessage **last_next,
+                Uint num_msgs,
                 erts_aint32_t in_state)
 {
     erts_aint32_t state = in_state;
@@ -442,13 +457,23 @@ enqueue_signals(Process *rp, ErtsMessage *first,
 
     ASSERT(!*this);
     *this = first;
-    rp->sig_inq.last = &last->next;
+    rp->sig_inq.last = last;
 
     if (!rp->sig_inq.nmsigs.next) {
         ASSERT(!rp->sig_inq.nmsigs.last);
-        rp->sig_inq.nmsigs.next = this;
+        if (ERTS_SIG_IS_NON_MSG(first)) {
+            rp->sig_inq.nmsigs.next = this;
+        }
+        else if (last_next) {
+            ASSERT(first->next && ERTS_SIG_IS_NON_MSG(first->next));
+            rp->sig_inq.nmsigs.next = &first->next;
+        }
+        else
+            goto no_nmsig;
+
         state = erts_atomic32_read_bor_nob(&rp->state,
                                            ERTS_PSFLG_SIG_IN_Q);
+    no_nmsig:
         ASSERT(!(state & ERTS_PSFLG_SIG_IN_Q));
     }
     else {
@@ -459,23 +484,41 @@ enqueue_signals(Process *rp, ErtsMessage *first,
 
         ASSERT(sig && !sig->common.specific.next);
         ASSERT(state & ERTS_PSFLG_SIG_IN_Q);
-        sig->common.specific.next = this;
+        if (ERTS_SIG_IS_NON_MSG(first)) {
+            sig->common.specific.next = this;
+        }
+        else if (last_next) {
+            ASSERT(first->next && ERTS_SIG_IS_NON_MSG(first->next));
+            sig->common.specific.next = &first->next;
+        }
     }
 
     if (last_next) {
-        ASSERT(first != last);
+        ASSERT(dbg_count_nmsigs(first) >= 2);
         rp->sig_inq.nmsigs.last = last_next;
     }
-    else {
-        ASSERT(first == last);
+    else if (ERTS_SIG_IS_NON_MSG(first)) {
+        ASSERT(dbg_count_nmsigs(first) == 1);
         rp->sig_inq.nmsigs.last = this;
     }
+    else
+        ASSERT(dbg_count_nmsigs(first) == 0);
+
+    rp->sig_inq.len += num_msgs;
 
     ERTS_HDBG_CHECK_SIGNAL_IN_QUEUE(rp);
 
     return state;
 }
 
+erts_aint32_t erts_enqueue_signals(Process *rp, ErtsMessage *first,
+                                   ErtsMessage **last, ErtsMessage **last_next,
+                                   Uint num_msgs,
+                                   erts_aint32_t in_state)
+{
+    return enqueue_signals(rp, first, last, last_next, num_msgs, in_state);
+}
+
 static ERTS_INLINE void
 ensure_dirty_proc_handled(Eterm pid,
                           erts_aint32_t state,
@@ -516,26 +559,92 @@ proc_queue_signal(Process *c_p, Eterm pid, ErtsSignal *sig, int op)
 {
     int res;
     Process *rp;
-    ErtsMessage *first, *last, **last_next;
+    ErtsMessage *first, *last, **last_next, **sigp;
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
     int is_normal_sched = !!esdp && esdp->type == ERTS_SCHED_NORMAL;
     erts_aint32_t state;
+    ErtsSignal *pend_sig;
 
-    if (is_normal_sched)
-        rp = erts_proc_lookup_raw(pid);
-    else
-        rp = erts_proc_lookup_raw_inc_refc(pid);
+    if (is_normal_sched) {
+        pend_sig = esdp->pending_signal.sig;
+        if (op == ERTS_SIG_Q_OP_MONITOR
+            && ((ErtsMonitor*)sig)->type == ERTS_MON_TYPE_PROC) {
 
-    if (!rp)
-        return 0;
+            if (!pend_sig) {
+                esdp->pending_signal.sig = sig;
+                esdp->pending_signal.to = pid;
+#ifdef DEBUG
+                esdp->pending_signal.dbg_from = esdp->current_process;
+#endif
+                return 1;
+            }
+            ASSERT(esdp->pending_signal.dbg_from == esdp->current_process);
+            if (pend_sig != sig) {
+                /* Switch them and send previously pending signal instead */
+                Eterm pend_to = esdp->pending_signal.to;
+                esdp->pending_signal.sig = sig;
+                esdp->pending_signal.to = pid;
+                sig = pend_sig;
+                pid = pend_to;
+            }
+            else {
+                /* Caller wants to flush pending signal */
+                ASSERT(pid == esdp->pending_signal.to);
+                esdp->pending_signal.sig = NULL;
+                esdp->pending_signal.to = THE_NON_VALUE;
+#ifdef DEBUG
+                esdp->pending_signal.dbg_from = NULL;
+#endif
+                pend_sig = NULL;
+            }
+            rp = erts_proc_lookup_raw(pid);
+            if (!rp) {
+                erts_proc_sig_send_monitor_down((ErtsMonitor*)sig, am_noproc);
+                return 1;
+            }
+        }
+        else if (pend_sig && pid == esdp->pending_signal.to) {
+            /* Flush pending signal to maintain signal order */
+            esdp->pending_signal.sig = NULL;
+            esdp->pending_signal.to = THE_NON_VALUE;
+
+            rp = erts_proc_lookup_raw(pid);
+            if (!rp) {
+                erts_proc_sig_send_monitor_down((ErtsMonitor*)pend_sig, am_noproc);
+                return 0;
+            }
+
+            /* Prepend pending signal */
+            pend_sig->common.next = (ErtsMessage*) sig;
+            pend_sig->common.specific.next = &pend_sig->common.next;
+            first = (ErtsMessage*) pend_sig;
+            last = (ErtsMessage*) sig;
+            sigp = last_next = &pend_sig->common.next;
+            goto first_last_done; 
+        }
+        else {
+            pend_sig = NULL;
+            rp = erts_proc_lookup_raw(pid);
+            if (!rp)
+                return 0;
+        }
+    }
+    else {
+        rp = erts_proc_lookup_raw_inc_refc(pid);
+        if (!rp)
+            return 0;
+        pend_sig = NULL;
+    }
 
-    sig->common.specific.next = NULL;
     first = last = (ErtsMessage *) sig;
     last_next = NULL;
+    sigp = &first;
+
+first_last_done:
+    sig->common.specific.next = NULL;
 
     /* may add signals before and/or after sig */
-    sig_enqueue_trace(c_p, first, op, rp,
-                      &first, &last, &last_next);
+    sig_enqueue_trace(c_p, sigp, op, rp, &last_next);
 
     last->next = NULL;
 
@@ -546,7 +655,7 @@ proc_queue_signal(Process *c_p, Eterm pid, ErtsSignal *sig, int op)
     if (ERTS_PSFLG_FREE & state)
         res = 0;
     else {
-        state = enqueue_signals(rp, first, last, last_next, state);
+        state = enqueue_signals(rp, first, &last->next, last_next, 0, state);
         if (ERTS_UNLIKELY(op == ERTS_SIG_Q_OP_PROCESS_INFO))
             check_push_msgq_len_offs_marker(rp, sig);
         res = !0;
@@ -554,8 +663,21 @@ proc_queue_signal(Process *c_p, Eterm pid, ErtsSignal *sig, int op)
 
     erts_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ);
 
-    if (res == 0)
+    if (res == 0) {
+        if (pend_sig) {
+            if (sig == pend_sig) {
+                /* We did a switch, callers signal is now pending (still ok) */
+                ASSERT(esdp->pending_signal.sig);
+                res = 1;
+            }
+            else {
+                ASSERT(first == (ErtsMessage*)pend_sig);
+                first = first->next;
+            }
+            erts_proc_sig_send_monitor_down((ErtsMonitor*)pend_sig, am_noproc);
+        }
         sig_enqueue_trace_cleanup(first, sig, last);
+    }
 
     if (!(state & (ERTS_PSFLG_EXITING
                    | ERTS_PSFLG_ACTIVE_SYS
@@ -572,6 +694,24 @@ proc_queue_signal(Process *c_p, Eterm pid, ErtsSignal *sig, int op)
     return res;
 }
 
+void erts_proc_sig_send_pending(ErtsSchedulerData* esdp)
+{
+    ErtsSignal* sig = esdp->pending_signal.sig;
+    int op;
+
+    ASSERT(esdp && esdp->type == ERTS_SCHED_NORMAL);
+    ASSERT(sig);
+    ASSERT(is_internal_pid(esdp->pending_signal.to));
+
+    op = ERTS_SIG_Q_OP_MONITOR;
+    ASSERT(op == ERTS_PROC_SIG_OP(sig->common.tag));
+
+    if (!proc_queue_signal(NULL, esdp->pending_signal.to, sig, op)) {
+        ErtsMonitor* mon = (ErtsMonitor*)sig;
+        erts_proc_sig_send_monitor_down(mon, am_noproc);
+    }
+}
+
 static int
 maybe_elevate_sig_handling_prio(Process *c_p, Eterm other)
 {
@@ -1404,8 +1544,7 @@ erts_proc_sig_send_is_alive_request(Process *c_p, Eterm to, Eterm ref)
         /* It wasn't alive; reply to ourselves... */
         mp->next = NULL;
         mp->data.attached = ERTS_MSG_COMBINED_HFRAG;
-        erts_queue_message(c_p, ERTS_PROC_LOCK_MAIN,
-                           mp, msg, am_system);
+        erts_queue_message(c_p, ERTS_PROC_LOCK_MAIN, mp, msg, am_system);
     }
 }
 
@@ -2490,7 +2629,7 @@ handle_process_info(Process *c_p, ErtsSigRecvTracing *tracing,
         if (is_alive)
             erts_factory_trim_and_close(&hfact, &msg, 1);
 
-        erts_queue_message(rp, locks, mp, msg, c_p->common.id);
+        erts_queue_proc_message(c_p, rp, locks, mp, msg);
 
         if (!is_alive && locks)
             erts_proc_unlock(rp, locks);
diff --git a/erts/emulator/beam/erl_proc_sig_queue.h b/erts/emulator/beam/erl_proc_sig_queue.h
index d250ad820f..8b7cd35f61 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.h
+++ b/erts/emulator/beam/erl_proc_sig_queue.h
@@ -733,6 +733,18 @@ Sint
 erts_proc_sig_privqs_len(Process *c_p);
 
 
+/* SVERK: Doc me up! */
+erts_aint32_t
+erts_enqueue_signals(Process *rp, ErtsMessage *first,
+                     ErtsMessage **last, ErtsMessage **last_next,
+                     Uint msg_cnt,
+                     erts_aint32_t in_state);
+
+/* SVERK: Doc me up! */
+void
+erts_proc_sig_send_pending(ErtsSchedulerData* esdp);
+
+
 typedef struct {
     Uint size;
     ErtsMessage *msgp;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 650ec0958c..ad7ac27ac3 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -5715,6 +5715,12 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     esdp->io.out = (Uint64) 0;
     esdp->io.in = (Uint64) 0;
 
+    esdp->pending_signal.sig = NULL;
+    esdp->pending_signal.to = THE_NON_VALUE;
+#ifdef DEBUG
+    esdp->pending_signal.dbg_from = NULL;
+#endif
+
     if (daww_ptr) {
 	init_aux_work_data(&esdp->aux_work_data, esdp, *daww_ptr);
 	*daww_ptr += daww_sz;
@@ -9674,8 +9680,13 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
     } else {
 	is_normal_sched = !esdp;
 	if (is_normal_sched) {
-	    esdp = p->scheduler_data;
+            esdp = p->scheduler_data;
 	    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+
+            if (esdp->pending_signal.sig) {
+                ASSERT(esdp->pending_signal.dbg_from == p);
+                erts_proc_sig_send_pending(esdp);
+            }
 	}
 	else {
 	    ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp));
@@ -10375,7 +10386,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st,
 	ASSERT(hp_start + hsz == hp);
 #endif
 
-	erts_queue_message(rp, rp_locks, mp, msg, c_p->common.id);
+	erts_queue_proc_message(c_p, rp, rp_locks, mp, msg);
 
 	if (c_p == rp)
 	    rp_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -10937,7 +10948,7 @@ dispatch_system_task(Process *c_p, erts_aint_t fail_state,
     msg = copy_struct(operation, osz, &hp, ohp);
     msg = TUPLE4(hp, st->requester, target, prio, msg);
 
-    erts_queue_message(rp, rp_locks, mp, msg, st->requester);
+    erts_queue_message(rp, rp_locks, mp, msg, am_system);
 
     if (rp_locks)
 	erts_proc_unlock(rp, rp_locks);
@@ -11912,6 +11923,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->sig_inq.len = 0;
     p->sig_inq.nmsigs.next = NULL;
     p->sig_inq.nmsigs.last = NULL;
+#ifdef ERTS_PROC_SIG_HARD_DEBUG
+    p->sig_inq.may_contain_heap_terms = 0;
+#endif
     p->bif_timers = NULL;
     p->mbuf = NULL;
     p->msg_frag = NULL;
@@ -12119,6 +12133,9 @@ void erts_init_empty_process(Process *p)
     p->sig_inq.len = 0;
     p->sig_inq.nmsigs.next = NULL;
     p->sig_inq.nmsigs.last = NULL;
+#ifdef ERTS_PROC_SIG_HARD_DEBUG
+    p->sig_inq.may_contain_heap_terms = 0;
+#endif
     p->bif_timers = NULL;
     p->dictionary = NULL;
     p->seq_trace_clock = 0;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index e2aa1d9f84..b66272194c 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -660,6 +660,13 @@ struct ErtsSchedulerData_ {
 	Uint64 out;
 	Uint64 in;
     } io;
+    struct {
+        ErtsSignal* sig;
+        Eterm to;
+#ifdef DEBUG
+	Process* dbg_from;
+#endif
+    } pending_signal;
 
     Uint64 reductions;
     ErtsSchedWallTime sched_wall_time;
-- 
cgit v1.2.3


From 5d2d5678b06770275ce20e333166e4269150f138 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 27 Apr 2018 15:08:01 +0200
Subject: Fix scheduled process_info() 'status' request

---
 erts/emulator/beam/erl_proc_sig_queue.c | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index fc3a2fe3c4..af4707c3d1 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -2430,18 +2430,6 @@ handle_process_info(Process *c_p, ErtsSigRecvTracing *tracing,
                 *c_p->sig_qs.last = NULL;
             }
 
-            if (!pisig->common.specific.next) {
-                /*
-                 * No more signals in middle queue...
-                 *
-                 * Process-info 'status' needs sig-q
-                 * process flag to be updated in order
-                 * to show accurate result...
-                 */
-                erts_atomic32_read_band_nob(&c_p->state,
-                                            ~ERTS_PSFLG_SIG_Q);
-            }
-
 #ifdef ERTS_PROC_SIG_HARD_DEBUG_SIGQ_MSG_LEN
             {
                 Sint len;
@@ -2455,8 +2443,20 @@ handle_process_info(Process *c_p, ErtsSigRecvTracing *tracing,
 #endif
         }
     }
-    if (is_alive)
+    if (is_alive) {
+        if (!pisig->common.specific.next) {
+            /*
+             * No more signals in middle queue...
+             *
+             * Process-info 'status' needs sig-q
+             * process flag to be updated in order
+             * to show accurate result...
+             */
+            erts_atomic32_read_band_nob(&c_p->state,
+                                        ~ERTS_PSFLG_SIG_Q);
+        }
         remove_nm_sig(c_p, sig, next_nm_sig);
+    }
 
     rp = erts_proc_lookup(pisig->requester);
     ASSERT(c_p != rp);
-- 
cgit v1.2.3


From 5d0874a8f9fd617308d9024783db1e4e24268184 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= <michal@muskala.eu>
Date: Fri, 27 Apr 2018 12:40:07 +0200
Subject: Introduce is_map_key/2 guard BIF

This complements the `map_get/2` guard BIF introduced in #1784.

Rationale.

`map_get/2` allows accessing map fields in guards, but it might be
problematic in more complex guard expressions, for example:

foo(X) when map_get(a, X) =:= 1 or is_list(X) -> ...

The `is_list/1` part of the guard could never succeed since the
`map_get/2` guard would fail the whole guard expression. In this
situation, this could be solved by using `;` instead of `or` to separate
the guards, but it is not possible in every case.

To solve this situation, this PR proposes a `is_map_key/2` guard that
allows to check if a map has key inside a guard before trying to access
that key. When combined with `is_map/1` this allows to construct a
purely boolean guard expression testing a value of a key in a map.

Implementation.

Given the use case motivating the introduction of this function, the PR
contains compiler optimisations that produce optimial code for the
following guard expression:

foo(X) when is_map(X) and is_map_key(a, X) and map_get(a, X) =:= 1 -> ok;
foo(_) -> error.

Given all three tests share the failure label, the `is_map_key/2` and
`is_map/2` tests are optimised away.

As with `map_get/2` the `is_map_key/2` BIF is allowed in match specs.
---
 erts/emulator/beam/bif.tab              |  1 +
 erts/emulator/beam/erl_db_util.c        |  6 +++++
 erts/emulator/beam/erl_map.c            |  7 +++++-
 erts/emulator/test/map_SUITE.erl        | 43 ++++++++++++++++++++++++++++++++-
 erts/emulator/test/match_spec_SUITE.erl |  7 ++++++
 5 files changed, 62 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 276bef2bbb..33738dc20b 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -696,3 +696,4 @@ bif ets:whereis/1
 bif erts_internal:gather_alloc_histograms/1
 bif erts_internal:gather_carrier_info/1
 ubif erlang:map_get/2
+ubif erlang:is_map_key/2
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 6354abfd1f..ef22cda1f0 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -643,6 +643,12 @@ static DMCGuardBif guard_tab[] =
         2,
         DBIF_ALL
     },
+    {
+        am_is_map_key,
+        &is_map_key_2,
+        2,
+        DBIF_ALL
+    },
     {
 	am_bit_size,
 	&bit_size_1,
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index f577b017c3..3d565b1bb8 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -43,6 +43,7 @@
  *
  * DONE:
  * - erlang:is_map/1
+ * - erlang:is_map_key/2
  * - erlang:map_size/1
  * - erlang:map_get/2
  *
@@ -919,7 +920,7 @@ static int hxnodecmp(hxnode_t *a, hxnode_t *b) {
 	return -1;
 }
 
-/* maps:is_key/2 */
+/* maps:is_key/2 and erlang:is_map_key/2 */
 
 BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) {
     if (is_map(BIF_ARG_2)) {
@@ -929,6 +930,10 @@ BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) {
     BIF_ERROR(BIF_P, BADMAP);
 }
 
+BIF_RETTYPE is_map_key_2(BIF_ALIST_2) {
+    BIF_RET(maps_is_key_2(BIF_CALL_ARGS));
+}
+
 /* maps:keys/1 */
 
 BIF_RETTYPE maps_keys_1(BIF_ALIST_1) {
diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl
index 43807b4388..f93c637650 100644
--- a/erts/emulator/test/map_SUITE.erl
+++ b/erts/emulator/test/map_SUITE.erl
@@ -38,6 +38,7 @@
          t_map_size/1,
          t_map_get/1,
          t_is_map/1,
+         t_is_map_key/1,
 
          %% Specific Map BIFs
          t_bif_map_get/1,
@@ -718,7 +719,47 @@ t_map_get(Config) when is_list(Config) ->
     true = if map_get(a, M2) =:= 1 -> true; true -> false end,
     false = if map_get(x, M2) =:= 1 -> true; true -> false end,
     do_badmap(fun
-        (T) when map_get(T, x) =:= 1 -> ok;
+        (T) when map_get(x, T) =:= 1 -> ok;
+        (T) -> false = is_map(T)
+    end),
+    ok.
+
+t_is_map_key(Config) when is_list(Config) ->
+    %% small map
+    true = is_map_key(a, id(#{a=>1})),
+    true = is_map_key(b, id(#{a=>1, b=>2})),
+    true = is_map_key("hello", id(#{a=>1, "hello"=>"hi"})),
+    true = is_map_key({1,1.0}, id(#{a=>a, {1,1.0}=>"tuple hi"})),
+
+    M0   = id(#{ k1=>"v1", <<"k2">> => <<"v3">> }),
+    true = is_map_key(<<"k2">>, M0#{<<"k2">> => "v4"}),
+
+    %% large map
+    M1   = maps:from_list([{I,I}||I<-lists:seq(1,100)] ++
+			  [{a,1},{b,2},{"hello","hi"},{{1,1.0},"tuple hi"},
+			   {k1,"v1"},{<<"k2">>,"v3"}]),
+    true = is_map_key(a, M1),
+    true = is_map_key(b, M1),
+    true = is_map_key("hello", M1),
+    true = is_map_key({1,1.0}, M1),
+    true = is_map_key(<<"k2">>, M1),
+
+    %% error cases
+    do_badmap(fun(T) ->
+		      {'EXIT',{{badmap,T},[{erlang,is_map_key,_,_}|_]}} =
+			  (catch is_map_key(a, T))
+	      end),
+
+    false = is_map_key({1,1}, id(#{{1,1.0}=>"tuple"})),
+    false = is_map_key(a, id(#{})),
+    false = is_map_key(a, id(#{b=>1, c=>2})),
+
+    %% in guards
+    M2 = id(#{a=>1}),
+    true = if is_map_key(a, M2) -> true; true -> false end,
+    false = if is_map_key(x, M2) -> true; true -> false end,
+    do_badmap(fun
+        (T) when is_map_key(T, x) =:= 1 -> ok;
         (T) -> false = is_map(T)
     end),
     ok.
diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl
index c1bc01f01e..4415d8d1b9 100644
--- a/erts/emulator/test/match_spec_SUITE.erl
+++ b/erts/emulator/test/match_spec_SUITE.erl
@@ -898,6 +898,13 @@ maps(Config) when is_list(Config) ->
     {ok,false,[],[]} = erlang:match_spec_test(not_a_map, [{'$1',[{map_get,b,'$1'}],['$_']}], table),
     {ok,true,[],[]} = erlang:match_spec_test(#{a => true}, [{'$1',[{map_get,a,'$1'}],[true]}], table),
 
+    {ok,true,[],[]} = erlang:match_spec_test(#{a => 1}, [{'$1',[],[{is_map_key,a,'$1'}]}], table),
+    {ok,false,[],[]} = erlang:match_spec_test(#{a => 1}, [{'$1',[],[{is_map_key,b,'$1'}]}], table),
+    {ok,'EXIT',[],[]} = erlang:match_spec_test(not_a_map, [{'$1',[],[{is_map_key,a,'$1'}]}], table),
+    {ok,false,[],[]} = erlang:match_spec_test(#{a => 1}, [{'$1',[{is_map_key,b,'$1'}],['$_']}], table),
+    {ok,false,[],[]} = erlang:match_spec_test(not_a_map, [{'$1',[{is_map_key,b,'$1'}],['$_']}], table),
+    {ok,true,[],[]} = erlang:match_spec_test(#{a => true}, [{'$1',[{is_map_key,a,'$1'}],[true]}], table),
+
     %% large maps
 
     Ls0 = [{I,<<I:32>>}||I <- lists:seq(1,415)],
-- 
cgit v1.2.3


From 2c0e76fa55ecb8c6fa6e7b82ce02e0f33a81391e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 2 May 2018 12:33:28 +0200
Subject: Stop assuming that all NTFS reparse points are links

This fixes a crash that would occur when using file:read_file_info/1
on a file with a non-link reparse point, which are commonly seen in
RSS and OneDrive folders.
---
 erts/emulator/nifs/win32/win_prim_file.c | 50 ++++++++++++++++++++++++++++++--
 1 file changed, 48 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/nifs/win32/win_prim_file.c b/erts/emulator/nifs/win32/win_prim_file.c
index 8058350b25..044bee62cf 100644
--- a/erts/emulator/nifs/win32/win_prim_file.c
+++ b/erts/emulator/nifs/win32/win_prim_file.c
@@ -24,6 +24,7 @@
 
 #include "prim_file_nif.h"
 
+#include <winioctl.h>
 #include <windows.h>
 #include <strsafe.h>
 #include <wchar.h>
@@ -671,11 +672,48 @@ static int is_executable_file(const efile_path_t *path) {
     return 0;
 }
 
+/* Returns whether the path refers to a link-like object, e.g. a junction
+ * point, symbolic link, or mounted folder. */
+static int is_name_surrogate(const efile_path_t *path) {
+    HANDLE handle;
+    int result;
+
+    handle = CreateFileW((const WCHAR*)path->data, GENERIC_READ,
+                         FILE_SHARE_FLAGS, NULL, OPEN_EXISTING,
+                         FILE_FLAG_OPEN_REPARSE_POINT |
+                         FILE_FLAG_BACKUP_SEMANTICS,
+                         NULL);
+    result = 0;
+
+    if(handle != INVALID_HANDLE_VALUE) {
+        REPARSE_GUID_DATA_BUFFER reparse_buffer;
+        LPDWORD unused_length;
+        BOOL success;
+
+        success = DeviceIoControl(handle,
+                                  FSCTL_GET_REPARSE_POINT, NULL, 0,
+                                  &reparse_buffer, sizeof(reparse_buffer),
+                                  &unused_length, NULL);
+
+        /* ERROR_MORE_DATA is tolerated since we're guaranteed to have filled
+         * the field we want. */
+        if(success || GetLastError() == ERROR_MORE_DATA) {
+            result = IsReparseTagNameSurrogate(reparse_buffer.ReparseTag);
+        }
+
+        CloseHandle(handle);
+     }
+
+     return result;
+}
+
 posix_errno_t efile_read_info(const efile_path_t *path, int follow_links, efile_fileinfo_t *result) {
     BY_HANDLE_FILE_INFORMATION native_file_info;
     DWORD attributes;
+    int is_link;
 
     sys_memset(&native_file_info, 0, sizeof(native_file_info));
+    is_link = 0;
 
     attributes = GetFileAttributesW((WCHAR*)path->data);
 
@@ -696,7 +734,11 @@ posix_errno_t efile_read_info(const efile_path_t *path, int follow_links, efile_
     } else {
         HANDLE handle;
 
-        if(follow_links && (attributes & FILE_ATTRIBUTE_REPARSE_POINT)) {
+        if(attributes & FILE_ATTRIBUTE_REPARSE_POINT) {
+            is_link = is_name_surrogate(path);
+        }
+
+        if(follow_links && is_link) {
             posix_errno_t posix_errno;
             efile_path_t resolved_path;
 
@@ -737,7 +779,7 @@ posix_errno_t efile_read_info(const efile_path_t *path, int follow_links, efile_
         }
     }
 
-    if(attributes & FILE_ATTRIBUTE_REPARSE_POINT) {
+    if(is_link) {
         result->type = EFILE_FILETYPE_SYMLINK;
         /* This should be _S_IFLNK, but the old driver always set
          * non-directories to _S_IFREG. */
@@ -917,6 +959,10 @@ posix_errno_t efile_read_link(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_
         return EINVAL;
     }
 
+    if(!is_name_surrogate(path)) {
+        return EINVAL;
+    }
+
     posix_errno = internal_read_link(path, &result_bin);
 
     if(posix_errno == 0) {
-- 
cgit v1.2.3


From 3d44429177a7dbdac5662433cfc55f5b5113a959 Mon Sep 17 00:00:00 2001
From: Richard Carlsson <richardc@klarna.com>
Date: Tue, 17 Apr 2018 14:11:34 +0200
Subject: Add ct:get_progname/0

This replaces all uses of lib:progname/0 in tests.
---
 erts/emulator/test/distribution_SUITE.erl | 6 +++---
 erts/emulator/test/port_SUITE.erl         | 8 ++++----
 erts/emulator/test/sensitive_SUITE.erl    | 2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index e40d346e10..0e383ee879 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -797,7 +797,7 @@ show_term(Term) ->
 
 %% Tests behaviour after net_kernel:stop (OTP-2586).
 stop_dist(Config) when is_list(Config) ->
-    Str = os:cmd(atom_to_list(lib:progname())
+    Str = os:cmd(ct:get_progname()
                  ++ " -noshell -pa "
                  ++ proplists:get_value(data_dir, Config)
                  ++ " -s run"),
@@ -974,9 +974,9 @@ dist_auto_connect_start(Name, Value) when is_list(Name), is_atom(Value) ->
     ModuleDir = filename:dirname(code:which(?MODULE)),
     ValueStr = atom_to_list(Value),
     Cookie = atom_to_list(erlang:get_cookie()),
-    Cmd = lists:concat(
+    Cmd = lists:append(
             [%"xterm -e ",
-             atom_to_list(lib:progname()),
+             ct:get_progname(),
              %	     " -noinput ",
              " -detached ",
              long_or_short(), " ", Name,
diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index 5b39d05df8..eb9b94a316 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -965,7 +965,7 @@ env_slave(File, Env) ->
 
 env_slave(File, Env, Body) ->
     file:write_file(File, term_to_binary(Body)),
-    Program = atom_to_list(lib:progname()),
+    Program = ct:get_progname(),
     Dir = filename:dirname(code:which(?MODULE)),
     Cmd = Program ++ " -pz " ++ Dir ++
     " -noinput -run " ++ ?MODULE_STRING ++ " env_slave_main " ++
@@ -1129,7 +1129,7 @@ try_bad_args(Args) ->
 cd(Config)  when is_list(Config) ->
     ct:timetrap({minutes, 1}),
 
-    Program = atom_to_list(lib:progname()),
+    Program = ct:get_progname(),
     DataDir = proplists:get_value(data_dir, Config),
     TestDir = filename:join(DataDir, "dir"),
     Cmd = Program ++ " -pz " ++ DataDir ++
@@ -1191,7 +1191,7 @@ cd(Config)  when is_list(Config) ->
 %% be relative the new cwd and not the original
 cd_relative(Config) ->
 
-    Program = atom_to_list(lib:progname()),
+    Program = ct:get_progname(),
     DataDir = proplists:get_value(data_dir, Config),
     TestDir = filename:join(DataDir, "dir"),
 
@@ -1214,7 +1214,7 @@ cd_relative(Config) ->
 
 relative_cd() ->
 
-    Program = atom_to_list(lib:progname()),
+    Program = ct:get_progname(),
     ok = file:set_cwd(".."),
     {ok, Cwd} = file:get_cwd(),
 
diff --git a/erts/emulator/test/sensitive_SUITE.erl b/erts/emulator/test/sensitive_SUITE.erl
index c3e303bbd1..9b23a30e88 100644
--- a/erts/emulator/test/sensitive_SUITE.erl
+++ b/erts/emulator/test/sensitive_SUITE.erl
@@ -413,7 +413,7 @@ my_process_info(Pid, Tag) ->
 
 t_process_display(Config) when is_list(Config) ->
     Dir = filename:dirname(code:which(?MODULE)),
-    Cmd = atom_to_list(lib:progname()) ++ " -noinput -pa " ++ Dir ++
+    Cmd = ct:get_progname() ++ " -noinput -pa " ++ Dir ++
     " -run " ++ ?MODULE_STRING ++ " remote_process_display",
     io:put_chars(Cmd),
     P = open_port({spawn,Cmd}, [in,stderr_to_stdout,eof]),
-- 
cgit v1.2.3


From a71c186fcc9e4592016c1b2c146db7b70ee0755a Mon Sep 17 00:00:00 2001
From: Richard Carlsson <richardc@klarna.com>
Date: Tue, 17 Apr 2018 16:16:34 +0200
Subject: Move error formatting to erl_error.erl and delete lib.erl

---
 erts/emulator/test/code_SUITE.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index 661a2ee6c9..9c6dc3ff83 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -957,7 +957,7 @@ erl_544(Config) when is_list(Config) ->
                 StackFun = fun(_, _, _) -> false end,
                 FormatFun = fun (Term, _) -> io_lib:format("~tp", [Term]) end,
                 Formated =
-                    lib:format_stacktrace(1, Stack, StackFun, FormatFun),
+                    erl_error:format_stacktrace(1, Stack, StackFun, FormatFun),
                 true = is_list(Formated),
                 ok
             after
-- 
cgit v1.2.3


From 3ab4ac0371b8646246c8b029dd89f39c3a6981b4 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 12 Mar 2018 11:50:42 +0100
Subject: erts: Make atomic ets:delete_all_objects yield

by using a cooperative strategy that will make
any process accessing the table execute delelete_all_objects_continue
until the table is empty.

This is not an optimal solution as concurrent threads will still
block on the table lock, but at least thread progress is made.
---
 erts/emulator/beam/bif.tab        |   4 +-
 erts/emulator/beam/erl_bif_info.c |   9 +-
 erts/emulator/beam/erl_db.c       | 554 ++++++++++++++++++++++----------------
 erts/emulator/beam/erl_db.h       |   4 +
 erts/emulator/beam/erl_db_hash.c  |  19 +-
 erts/emulator/beam/erl_db_hash.h  |   3 -
 erts/emulator/beam/erl_db_tree.c  |  10 +-
 erts/emulator/beam/erl_db_util.h  |  12 +-
 8 files changed, 368 insertions(+), 247 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 687fd39d58..1276048317 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -341,7 +341,6 @@ bif ets:internal_request_all/0
 bif ets:new/2
 bif ets:delete/1
 bif ets:delete/2
-bif ets:delete_all_objects/1
 bif ets:delete_object/2
 bif ets:first/1
 bif ets:is_compiled_ms/1
@@ -372,7 +371,6 @@ bif ets:select_count/2
 bif ets:select_reverse/1
 bif ets:select_reverse/2
 bif ets:select_reverse/3
-bif ets:select_delete/2
 bif ets:select_replace/2
 bif ets:match_spec_compile/1
 bif ets:match_spec_run_r/3
@@ -693,3 +691,5 @@ bif erts_internal:new_connection/1
 bif erts_internal:abort_connection/2
 bif erts_internal:map_next/3
 bif ets:whereis/1
+bif ets:internal_delete_all/2
+bif ets:internal_select_delete/2
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 89e3d3f43e..3c13991fb6 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -4731,7 +4731,14 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
                                             refbin));
             }
         }
-
+        else if (ERTS_IS_ATOM_STR("ets_force_trap", BIF_ARG_1)) {
+#ifdef ETS_DBG_FORCE_TRAP
+            erts_ets_dbg_force_trap = (BIF_ARG_2 == am_true) ? 1 : 0;
+            BIF_RET(am_ok);
+#else
+            BIF_RET(am_notsup);
+#endif
+        }
     }
 
     BIF_ERROR(BIF_P, BADARG);
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 2d46e81b3b..322f5f1505 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -50,6 +50,34 @@ erts_atomic_t erts_ets_misc_mem_size;
 ** Utility macros
 */
 
+#define DB_BIF_GET_TABLE(TB, WHAT, KIND, BIF_IX) \
+        DB_GET_TABLE(TB, BIF_ARG_1, WHAT, KIND, BIF_IX, NULL, BIF_P)
+
+#define DB_TRAP_GET_TABLE(TB, TID, WHAT, KIND, BIF_EXP) \
+        DB_GET_TABLE(TB, TID, WHAT, KIND, 0, BIF_EXP, BIF_P)
+
+#define DB_GET_TABLE(TB, TID, WHAT, KIND, BIF_IX, BIF_EXP, PROC)         \
+do {                                                                     \
+    Uint freason__;                                                      \
+    if (!(TB = db_get_table(PROC, TID, WHAT, KIND, &freason__))) {       \
+        return db_bif_fail(PROC, freason__, BIF_IX, BIF_EXP);            \
+    }                                                                    \
+}while(0)
+
+static BIF_RETTYPE db_bif_fail(Process* p, Uint freason,
+                               Uint bif_ix, Export* bif_exp)
+{
+    if (freason == TRAP) {
+        if (!bif_exp)
+            bif_exp = bif_export[bif_ix];
+        p->arity = bif_exp->info.mfa.arity;
+        p->i = (BeamInstr*) bif_exp->addressv[erts_active_code_ix()];
+    }
+    p->freason = freason;
+    return THE_NON_VALUE;
+}
+
+
 /* Get a key from any table structure and a tagged object */
 #define TERM_GETKEY(tb, obj) db_getkey((tb)->common.keypos, (obj)) 
 
@@ -326,8 +354,7 @@ struct meta_name_tab_entry* meta_name_tab_bucket(Eterm name,
 typedef enum {
     LCK_READ=1,     /* read only access */
     LCK_WRITE=2,    /* exclusive table write access */
-    LCK_WRITE_REC=3, /* record write access */
-    LCK_NONE=4
+    LCK_WRITE_REC=3 /* record write access */
 } db_lock_kind_t;
 
 extern DbTableMethod db_hash;
@@ -337,9 +364,6 @@ int user_requested_db_max_tabs;
 int erts_ets_realloc_always_moves;
 int erts_ets_always_compress;
 static int db_max_tabs;
-static Eterm ms_delete_all;
-static Eterm ms_delete_all_buff[8]; /* To compare with for deletion 
-				       of all objects */
 
 /* 
 ** Forward decls, static functions 
@@ -351,6 +375,7 @@ static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data);
 static void free_heir_data(DbTable*);
 static SWord free_fixations_locked(Process* p, DbTable *tb);
 
+static void delete_all_objects_continue(Process* p, DbTable* tb);
 static SWord free_table_continue(Process *p, DbTable *tb, SWord reds);
 static void print_table(fmtfn_t to, void *to_arg, int show,  DbTable* tb);
 static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1);
@@ -360,9 +385,9 @@ static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1);
 static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1);
 static Eterm table_info(Process* p, DbTable* tb, Eterm What);
 
-static BIF_RETTYPE ets_select1(Process* p, Eterm arg1);
-static BIF_RETTYPE ets_select2(Process* p, Eterm arg1, Eterm arg2);
-static BIF_RETTYPE ets_select3(Process* p, Eterm arg1, Eterm arg2, Eterm arg3);
+static BIF_RETTYPE ets_select1(Process* p, int bif_ix, Eterm arg1);
+static BIF_RETTYPE ets_select2(Process* p, DbTable*, Eterm tid, Eterm ms);
+static BIF_RETTYPE ets_select3(Process* p, DbTable*, Eterm tid, Eterm ms, Sint chunk_size);
 
 
 /* 
@@ -636,15 +661,42 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
     }
 }
 
+static ERTS_INLINE int db_is_exclusive(DbTable* tb, db_lock_kind_t kind)
+{
+    return kind != LCK_READ && tb->common.is_thread_safe;
+}
+
+static DbTable* handle_lacking_permission(Process* p, DbTable* tb,
+                                          db_lock_kind_t kind,
+                                          Uint* freason_p)
+{
+    if (tb->common.status & DB_BUSY) {
+        if (!db_is_exclusive(tb, kind)) {
+            db_unlock(tb, kind);
+            db_lock(tb, LCK_WRITE);
+        }
+        delete_all_objects_continue(p, tb);
+        db_unlock(tb, LCK_WRITE);
+        tb = NULL;
+        *freason_p = TRAP;
+    }
+    else if (p->common.id != tb->common.owner) {
+        db_unlock(tb, kind);
+        tb = NULL;
+        *freason_p = BADARG;
+    }
+    return tb;
+}
+
 static ERTS_INLINE
 DbTable* db_get_table_aux(Process *p,
 			  Eterm id,
 			  int what,
 			  db_lock_kind_t kind,
-			  int meta_already_locked)
+			  int meta_already_locked,
+                          Uint* freason_p)
 {
     DbTable *tb;
-    erts_rwmtx_t *mtl = NULL;
 
     /*
      * IMPORTANT: Only scheduler threads are allowed
@@ -654,13 +706,13 @@ DbTable* db_get_table_aux(Process *p,
     ASSERT(erts_get_scheduler_data());
 
     if (is_atom(id)) {
+        erts_rwmtx_t *mtl;
 	struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl);
 	if (!meta_already_locked)
 	    erts_rwmtx_rlock(mtl);
 	else{
 	    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(mtl)
 			       || erts_lc_rwmtx_is_rwlocked(mtl));
-	    mtl = NULL;
 	}
         tb = NULL;
 	if (bucket->pu.tb != NULL) {
@@ -679,20 +731,29 @@ DbTable* db_get_table_aux(Process *p,
 		}
 	    }
 	}
+        if (!meta_already_locked)
+            erts_rwmtx_runlock(mtl);
     }
     else
         tb = tid2tab(id);
 
     if (tb) {
 	db_lock(tb, kind);
-	if ((tb->common.status & what) == 0
-            && p->common.id != tb->common.owner) {
-	    db_unlock(tb, kind);
-	    tb = NULL;
-	}
+#ifdef ETS_DBG_FORCE_TRAP
+        if (erts_atomic_read_nob(&tb->common.dbg_force_trap) &&
+            erts_atomic_add_read_nob(&tb->common.dbg_force_trap, 2) & 2) {
+            db_unlock(tb, kind);
+            tb = NULL;
+            *freason_p = TRAP;
+        }
+        else
+#endif
+	if (ERTS_UNLIKELY(!(tb->common.status & what)))
+            tb = handle_lacking_permission(p, tb, kind, freason_p);
     }
-    if (mtl)
-	erts_rwmtx_runlock(mtl);
+    else
+        *freason_p = BADARG;
+
     return tb;
 }
 
@@ -700,9 +761,10 @@ static ERTS_INLINE
 DbTable* db_get_table(Process *p,
 		      Eterm id,
 		      int what,
-		      db_lock_kind_t kind)
+		      db_lock_kind_t kind,
+                      Uint* freason_p)
 {
-    return db_get_table_aux(p, id, what, kind, 0);
+    return db_get_table_aux(p, id, what, kind, 0, freason_p);
 }
 
 static int insert_named_tab(Eterm name_atom, DbTable* tb, int have_lock)
@@ -868,9 +930,7 @@ BIF_RETTYPE ets_safe_fixtable_2(BIF_ALIST_2)
 #endif
     kind = (BIF_ARG_2 == am_true) ? LCK_READ : LCK_WRITE_REC; 
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, kind)) == NULL) {
-	    BIF_ERROR(BIF_P, BADARG);
-	}
+    DB_BIF_GET_TABLE(tb, DB_READ, kind, BIF_ets_safe_fixtable_2);
 
     if (BIF_ARG_2 == am_true) {
 	fix_table_locked(BIF_P, tb);
@@ -900,11 +960,7 @@ BIF_RETTYPE ets_first_1(BIF_ALIST_1)
 
     CHECK_TABLES();
 
-    tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ);
-
-    if (!tb) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_first_1);
 
     cret = tb->common.meth->db_first(BIF_P, tb, &ret);
 
@@ -927,11 +983,7 @@ BIF_RETTYPE ets_next_2(BIF_ALIST_2)
 
     CHECK_TABLES();
 
-    tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ);
-
-    if (!tb) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_next_2);
 
     cret = tb->common.meth->db_next(BIF_P, tb, BIF_ARG_2, &ret);
 
@@ -954,11 +1006,7 @@ BIF_RETTYPE ets_last_1(BIF_ALIST_1)
 
     CHECK_TABLES();
 
-    tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ);
-
-    if (!tb) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_last_1);
 
     cret = tb->common.meth->db_last(BIF_P, tb, &ret);
 
@@ -981,11 +1029,7 @@ BIF_RETTYPE ets_prev_2(BIF_ALIST_2)
 
     CHECK_TABLES();
 
-    tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ);
-
-    if (!tb) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_prev_2);
 
     cret = tb->common.meth->db_prev(BIF_P,tb,BIF_ARG_2,&ret);
 
@@ -1003,21 +1047,15 @@ BIF_RETTYPE ets_prev_2(BIF_ALIST_2)
 BIF_RETTYPE ets_take_2(BIF_ALIST_2)
 {
     DbTable* tb;
-#ifdef DEBUG
     int cret;
-#endif
     Eterm ret;
     CHECK_TABLES();
 
-    tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC);
-    if (!tb) {
-        BIF_ERROR(BIF_P, BADARG);
-    }
-#ifdef DEBUG
-    cret =
-#endif
-        tb->common.meth->db_take(BIF_P, tb, BIF_ARG_2, &ret);
-    ASSERT(cret == DB_ERROR_NONE);
+    DB_BIF_GET_TABLE(tb, DB_WRITE, LCK_WRITE_REC, BIF_ets_take_2);
+
+    cret = tb->common.meth->db_take(BIF_P, tb, BIF_ARG_2, &ret);
+
+    ASSERT(cret == DB_ERROR_NONE); (void)cret;
     db_unlock(tb, LCK_WRITE_REC);
     BIF_RET(ret);
 }
@@ -1035,9 +1073,8 @@ BIF_RETTYPE ets_update_element_3(BIF_ALIST_3)
     DeclareTmpHeap(cell,2,BIF_P);
     DbUpdateHandle handle;
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_WRITE, LCK_WRITE_REC, BIF_ets_update_element_3);
+
     UseTmpHeap(2,BIF_P);
     if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) {
 	goto bail_out;
@@ -1108,9 +1145,9 @@ bail_out:
 }
 
 static BIF_RETTYPE
-do_update_counter(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Eterm arg4)
+do_update_counter(Process *p, DbTable* tb,
+                  Eterm arg2, Eterm arg3, Eterm arg4)
 {
-    DbTable* tb;
     int cret = DB_ERROR_BADITEM;
     Eterm upop_list;
     int list_size;
@@ -1126,10 +1163,6 @@ do_update_counter(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Eterm arg4)
     Eterm* hstart;
     Eterm* hend;
 
-    if ((tb = db_get_table(p, arg1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
-        BIF_ERROR(p, BADARG);
-    }
-
     UseTmpHeap(5, p);
 
     if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) {
@@ -1303,7 +1336,11 @@ bail_out:
 */
 BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
 {
-    return do_update_counter(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, THE_NON_VALUE);
+    DbTable* tb;
+
+    DB_BIF_GET_TABLE(tb, DB_WRITE, LCK_WRITE_REC, BIF_ets_update_counter_3);
+
+    return do_update_counter(BIF_P, tb, BIF_ARG_2, BIF_ARG_3, THE_NON_VALUE);
 }
 
 /*
@@ -1315,10 +1352,14 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
 */
 BIF_RETTYPE ets_update_counter_4(BIF_ALIST_4)
 {
+    DbTable* tb;
+
     if (is_not_tuple(BIF_ARG_4)) {
         BIF_ERROR(BIF_P, BADARG);
     }
-    return do_update_counter(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BIF_ARG_4);
+    DB_BIF_GET_TABLE(tb, DB_WRITE, LCK_WRITE_REC, BIF_ets_update_counter_4);
+
+    return do_update_counter(BIF_P, tb, BIF_ARG_2, BIF_ARG_3, BIF_ARG_4);
 }
 
 
@@ -1339,9 +1380,8 @@ BIF_RETTYPE ets_insert_2(BIF_ALIST_2)
     kind = ((is_list(BIF_ARG_2) && CDR(list_val(BIF_ARG_2)) != NIL)
 	    ? LCK_WRITE : LCK_WRITE_REC);
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, kind)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_WRITE, kind, BIF_ets_insert_2);
+
     if (BIF_ARG_2 == NIL) {
 	db_unlock(tb, kind);
 	BIF_RET(am_true);
@@ -1407,11 +1447,9 @@ BIF_RETTYPE ets_insert_new_2(BIF_ALIST_2)
 
 	    /* More than one object, use LCK_WRITE to keep atomicity */
 	    kind = LCK_WRITE;
-	    tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, kind);
-	    if (tb == NULL) {
-		BIF_ERROR(BIF_P, BADARG);
-	    }
-	    meth = tb->common.meth;
+            DB_BIF_GET_TABLE(tb, DB_WRITE, kind, BIF_ets_insert_new_2);
+
+            meth = tb->common.meth;
 	    for (lst = BIF_ARG_2; is_list(lst); lst = CDR(list_val(lst))) {
 		if (is_not_tuple(CAR(list_val(lst)))
 		    || (arityval(*tuple_val(CAR(list_val(lst))))
@@ -1446,9 +1484,8 @@ BIF_RETTYPE ets_insert_new_2(BIF_ALIST_2)
     /* Only one object (or NIL) 
     */
     kind = LCK_WRITE_REC;
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, kind)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_WRITE, kind, BIF_ets_insert_new_2);
+
     if (BIF_ARG_2 == NIL) {
 	db_unlock(tb, kind);
 	BIF_RET(am_true);
@@ -1487,6 +1524,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
     Eterm ret;
     Eterm old_name;
     erts_rwmtx_t *lck1, *lck2;
+    Uint freason;
 
 #ifdef HARDDEBUG
     erts_fprintf(stderr,
@@ -1531,9 +1569,9 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
     if (lck2)
 	erts_rwmtx_rwlock(lck2);
 
-    tb = db_get_table_aux(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE, 1);
+    tb = db_get_table_aux(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE, 1, &freason);
     if (!tb)
-	goto badarg;
+	goto fail;
 
     if (is_table_named(tb)) {
         if (!insert_named_tab(BIF_ARG_2, tb, 1))
@@ -1553,13 +1591,18 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
     if (lck2)
 	erts_rwmtx_rwunlock(lck2);
     BIF_RET(ret);
- badarg:
+
+badarg:
+    freason = BADARG;
+
+fail:
     if (tb)
 	db_unlock(tb, LCK_WRITE);
     erts_rwmtx_rwunlock(lck1);
     if (lck2)
 	erts_rwmtx_rwunlock(lck2);
-    BIF_ERROR(BIF_P, BADARG);    
+
+    return db_bif_fail(BIF_P, freason, BIF_ets_rename_2, NULL);
 }
 
 
@@ -1706,7 +1749,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     tb->common.meth = meth;
     tb->common.the_name = BIF_ARG_1;
     tb->common.status = status;    
-    tb->common.type = status & ERTS_ETS_TABLE_TYPES;
+    tb->common.type = status;
     /* Note, 'type' is *read only* from now on... */
     erts_refc_init(&tb->common.fix_count, 0);
     db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ));
@@ -1718,6 +1761,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 
     tb->common.fixing_procs = NULL;
     tb->common.compress = is_compressed;
+#ifdef ETS_DBG_FORCE_TRAP
+    erts_atomic_init_nob(&tb->common.dbg_force_trap, erts_ets_dbg_force_trap);
+#endif
 
     cret = meth->db_create(BIF_P, tb);
     ASSERT(cret == DB_ERROR_NONE); (void)cret;
@@ -1762,13 +1808,19 @@ BIF_RETTYPE ets_whereis_1(BIF_ALIST_1)
 {
     DbTable* tb;
     Eterm res;
+    Uint freason;
 
     if (is_not_atom(BIF_ARG_1)) {
         BIF_ERROR(BIF_P, BADARG);
     }
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL) {
-        BIF_RET(am_undefined);
+    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ, &freason)) == NULL) {
+        if (freason == BADARG)
+            BIF_RET(am_undefined);
+        else {
+            //ToDo: Could we avoid this
+            return db_bif_fail(BIF_P, freason, BIF_ets_whereis_1, NULL);
+        }
     }
 
     res = make_tid(BIF_P, tb);
@@ -1788,9 +1840,7 @@ BIF_RETTYPE ets_lookup_2(BIF_ALIST_2)
 
     CHECK_TABLES();
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_lookup_2);
 
     cret = tb->common.meth->db_get(BIF_P, tb, BIF_ARG_2, &ret);
 
@@ -1818,9 +1868,7 @@ BIF_RETTYPE ets_member_2(BIF_ALIST_2)
 
     CHECK_TABLES();
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_member_2);
 
     cret = tb->common.meth->db_member(tb, BIF_ARG_2, &ret);
 
@@ -1851,9 +1899,7 @@ BIF_RETTYPE ets_lookup_element_3(BIF_ALIST_3)
 
     CHECK_TABLES();
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_lookup_element_3);
 
     if (is_not_small(BIF_ARG_3) || ((index = signed_val(BIF_ARG_3)) < 1)) {
 	db_unlock(tb, LCK_READ);
@@ -1891,9 +1937,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 
     CHECK_TABLES();
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_WRITE, LCK_WRITE, BIF_ets_delete_1);
     
     /*
      * Clear all access bits to prevent any ets operation to access the
@@ -1966,6 +2010,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
     Eterm to_pid = BIF_ARG_2;
     Eterm from_pid;
     DbTable* tb = NULL;
+    Uint freason;
 
     if (!is_internal_pid(to_pid)) {
 	goto badarg;
@@ -1975,10 +2020,11 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
 	goto badarg;
     }
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL
-	|| tb->common.owner != BIF_P->common.id) {
-	goto badarg;
-    }
+    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE, &freason)) == NULL)
+        goto fail;
+    if (tb->common.owner != BIF_P->common.id)
+        goto badarg;
+
     from_pid = tb->common.owner;
     if (to_pid == from_pid) {
 	goto badarg;  /* or should we be idempotent? return false maybe */
@@ -1997,9 +2043,12 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
     BIF_RET(am_true);
 
 badarg:
+    freason = BADARG;
+fail:
     if (to_proc != NULL && to_proc != BIF_P) erts_proc_unlock(to_proc, to_locks);
     if (tb != NULL) db_unlock(tb, LCK_WRITE);
-    BIF_ERROR(BIF_P, BADARG);
+
+    return db_bif_fail(BIF_P, freason, BIF_ets_give_away_3, NULL);
 }
 
 BIF_RETTYPE ets_setopts_2(BIF_ALIST_2)
@@ -2050,11 +2099,13 @@ BIF_RETTYPE ets_setopts_2(BIF_ALIST_2)
 	}
     }
 
-    if (tail != NIL
-	|| (tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL
-	|| tb->common.owner != BIF_P->common.id) {
+    if (tail != NIL)
+        goto badarg;
+
+    DB_BIF_GET_TABLE(tb, DB_WRITE, LCK_WRITE, BIF_ets_setopts_2);
+
+    if (tb->common.owner != BIF_P->common.id)
 	goto badarg;
-    }
 
     if (heir_data != THE_NON_VALUE) {
 	free_heir_data(tb);
@@ -2078,23 +2129,84 @@ badarg:
 }
 
 /* 
-** BIF to erase a whole table and release all memory it holds 
-*/
-BIF_RETTYPE ets_delete_all_objects_1(BIF_ALIST_1)
+ * Common for delete_all_objects and select_delete(DeleteAll).
+ */
+BIF_RETTYPE ets_internal_delete_all_2(BIF_ALIST_2)
 {
+    SWord initial_reds = ERTS_BIF_REDS_LEFT(BIF_P);
+    SWord reds = initial_reds;
+    Eterm nitems;
     DbTable* tb;
 
     CHECK_TABLES();
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_WRITE, LCK_WRITE, BIF_ets_internal_delete_all_2);
+
+    if (BIF_ARG_2 == am_undefined) {
+        nitems = erts_make_integer(erts_atomic_read_nob(&tb->common.nitems),
+                                   BIF_P);
+
+        reds = tb->common.meth->db_delete_all_objects(BIF_P, tb, reds);
 
-    tb->common.meth->db_delete_all_objects(BIF_P, tb);
+        ASSERT(!(tb->common.status & DB_BUSY));
+
+        if (reds < 0) {
+            /*
+             * Oboy, need to trap AND need to be atomic.
+             * Solved by cooperative trapping where every process trying to
+             * access this table (including this process) will "fail" to lookup
+             * the table and instead pitch in deleting objects
+             * (in delete_all_objects_continue) and then trap to self.
+             */
+            ASSERT((tb->common.status & (DB_PRIVATE|DB_PROTECTED|DB_PUBLIC))
+                   ==
+                   (tb->common.type & (DB_PRIVATE|DB_PROTECTED|DB_PUBLIC)));
+            tb->common.status &= ~(DB_PRIVATE|DB_PROTECTED|DB_PUBLIC);
+            tb->common.status |= DB_BUSY;
+            db_unlock(tb, LCK_WRITE);
+            BUMP_ALL_REDS(BIF_P);
+            BIF_TRAP2(bif_export[BIF_ets_internal_delete_all_2], BIF_P,
+                      BIF_ARG_1, nitems);
+        }
+        else {
+            /* Done, no trapping needed */
+            BUMP_REDS(BIF_P, (initial_reds - reds));
+        }
+
+    }
+    else {
+        /*
+         * The table lookup succeeded and second argument is nitems
+         * and not 'undefined', which means we have trapped at least once
+         * and are now done.
+         */
+        nitems = BIF_ARG_2;
+    }
 
     db_unlock(tb, LCK_WRITE);
+    BIF_RET(nitems);
+}
 
-    BIF_RET(am_true);
+static void delete_all_objects_continue(Process* p, DbTable* tb)
+{
+    SWord initial_reds = ERTS_BIF_REDS_LEFT(p);
+    SWord reds = initial_reds;
+
+    ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock));
+
+    if ((tb->common.status & (DB_DELETE|DB_BUSY)) != DB_BUSY)
+        return;
+
+    reds = tb->common.meth->db_delete_all_objects(p, tb, reds);
+    
+    if (reds < 0) {
+        BUMP_ALL_REDS(p);
+    }
+    else {
+        tb->common.status |= tb->common.type & (DB_PRIVATE|DB_PROTECTED|DB_PUBLIC);
+        tb->common.status &= ~DB_BUSY;
+        BUMP_REDS(p, (initial_reds - reds));
+    }
 }
 
 /* 
@@ -2110,9 +2222,7 @@ BIF_RETTYPE ets_delete_2(BIF_ALIST_2)
 
     CHECK_TABLES();
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_WRITE, LCK_WRITE_REC, BIF_ets_delete_2);
 
     cret = tb->common.meth->db_erase(tb,BIF_ARG_2,&ret);
 
@@ -2139,9 +2249,8 @@ BIF_RETTYPE ets_delete_object_2(BIF_ALIST_2)
 
     CHECK_TABLES();
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_WRITE, LCK_WRITE_REC, BIF_ets_delete_object_2);
+
     if (is_not_tuple(BIF_ARG_2) || 
 	(arityval(*tuple_val(BIF_ARG_2)) < tb->common.keypos)) {
 	db_unlock(tb, LCK_WRITE_REC);
@@ -2174,15 +2283,14 @@ static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1)
     Eterm ret;
     Eterm *tptr;
     db_lock_kind_t kind = LCK_WRITE_REC;
-    
+
     CHECK_TABLES();
     ASSERT(is_tuple(a1));
     tptr = tuple_val(a1);
     ASSERT(arityval(*tptr) >= 1);
     
-    if ((tb = db_get_table(p, tptr[1], DB_WRITE, kind)) == NULL) {
-	BIF_ERROR(p,BADARG);
-    }
+    DB_TRAP_GET_TABLE(tb, tptr[1], DB_WRITE, kind,
+                      &ets_select_delete_continue_exp);
 
     cret = tb->common.meth->db_select_delete_continue(p,tb,a1,&ret);
 
@@ -2206,7 +2314,10 @@ static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1)
 }
     
 
-BIF_RETTYPE ets_select_delete_2(BIF_ALIST_2)
+/*
+ * ets:select_delete/2 without special case for "delete-all".
+ */
+BIF_RETTYPE ets_internal_select_delete_2(BIF_ALIST_2)
 {
     BIF_RETTYPE result;
     DbTable* tb;
@@ -2216,20 +2327,8 @@ BIF_RETTYPE ets_select_delete_2(BIF_ALIST_2)
 
     CHECK_TABLES();
 
-    if(eq(BIF_ARG_2, ms_delete_all)) {
-	int nitems;
-	if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) {
-	    BIF_ERROR(BIF_P, BADARG);
-	}
-	nitems = erts_atomic_read_nob(&tb->common.nitems);
-	tb->common.meth->db_delete_all_objects(BIF_P, tb);
-	db_unlock(tb, LCK_WRITE);
-	BIF_RET(erts_make_integer(nitems,BIF_P));
-    }
+    DB_BIF_GET_TABLE(tb, DB_WRITE, LCK_WRITE_REC, BIF_ets_internal_select_delete_2);
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
     safety = ITERATION_SAFETY(BIF_P,tb);
     if (safety == ITER_UNSAFE) {
 	local_fix_table(tb);
@@ -2521,9 +2620,8 @@ BIF_RETTYPE ets_slot_2(BIF_ALIST_2)
 
     CHECK_TABLES();
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_slot_2);
+
     /* The slot number is checked in table specific code. */
     cret = tb->common.meth->db_slot(BIF_P, tb, BIF_ARG_2, &ret);
     db_unlock(tb, LCK_READ);
@@ -2543,41 +2641,53 @@ BIF_RETTYPE ets_slot_2(BIF_ALIST_2)
 
 BIF_RETTYPE ets_match_1(BIF_ALIST_1)
 {
-    return ets_select1(BIF_P, BIF_ARG_1);
+    return ets_select1(BIF_P, BIF_ets_match_1, BIF_ARG_1);
 }
 
 BIF_RETTYPE ets_match_2(BIF_ALIST_2)
 {
+    DbTable* tb;
     Eterm ms;
     DeclareTmpHeap(buff,8,BIF_P);
     Eterm *hp = buff;
     Eterm res;
 
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_match_2);
+
     UseTmpHeap(8,BIF_P);
     ms = CONS(hp, am_DollarDollar, NIL);
     hp += 2;
     ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); 
     hp += 4;
     ms = CONS(hp, ms, NIL);
-    res = ets_select2(BIF_P, BIF_ARG_1, ms);
+    res = ets_select2(BIF_P, tb, BIF_ARG_1, ms);
     UnUseTmpHeap(8,BIF_P);
     return res;
 }
 
 BIF_RETTYPE ets_match_3(BIF_ALIST_3)
 {
+    DbTable* tb;
     Eterm ms;
+    Sint chunk_size;
     DeclareTmpHeap(buff,8,BIF_P);
     Eterm *hp = buff;
     Eterm res;
 
+    /* Chunk size strictly greater than 0 */
+    if (is_not_small(BIF_ARG_3) || (chunk_size = signed_val(BIF_ARG_3)) <= 0) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_match_3);
+
     UseTmpHeap(8,BIF_P);
     ms = CONS(hp, am_DollarDollar, NIL);
     hp += 2;
     ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); 
     hp += 4;
     ms = CONS(hp, ms, NIL);
-    res = ets_select3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+    res = ets_select3(BIF_P, tb, BIF_ARG_1, ms, chunk_size);
     UnUseTmpHeap(8,BIF_P);
     return res;
 }
@@ -2585,34 +2695,35 @@ BIF_RETTYPE ets_match_3(BIF_ALIST_3)
 
 BIF_RETTYPE ets_select_3(BIF_ALIST_3)
 {
-    return ets_select3(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+    DbTable* tb;
+    Sint chunk_size;
+
+    /* Chunk size strictly greater than 0 */
+    if (is_not_small(BIF_ARG_3) || (chunk_size = signed_val(BIF_ARG_3)) <= 0) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_select_3);
+
+    return ets_select3(BIF_P, tb, BIF_ARG_1, BIF_ARG_2, chunk_size);
 }
 
 static BIF_RETTYPE
-ets_select3(Process* p, Eterm arg1, Eterm arg2, Eterm arg3)
+ets_select3(Process* p, DbTable* tb, Eterm tid, Eterm ms, Sint chunk_size)
 {
     BIF_RETTYPE result;
-    DbTable* tb;
     int cret;
     Eterm ret;
-    Sint chunk_size;
     enum DbIterSafety safety;
 
     CHECK_TABLES();
 
-    /* Chunk size strictly greater than 0 */
-    if (is_not_small(arg3) || (chunk_size = signed_val(arg3)) <= 0) {
-	BIF_ERROR(p, BADARG);
-    }
-    if ((tb = db_get_table(p, arg1, DB_READ, LCK_READ)) == NULL) {
-	BIF_ERROR(p, BADARG);
-    }
     safety = ITERATION_SAFETY(p,tb);
     if (safety == ITER_UNSAFE) {
 	local_fix_table(tb);
     }
-    cret = tb->common.meth->db_select_chunk(p, tb, arg1,
-					    arg2, chunk_size,
+    cret = tb->common.meth->db_select_chunk(p, tb, tid,
+					    ms, chunk_size,
 					    0 /* not reversed */,
 					    &ret);
     if (DID_TRAP(p,ret) && safety != ITER_SAFE) {
@@ -2658,9 +2769,8 @@ static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1)
     tptr = tuple_val(a1);
     ASSERT(arityval(*tptr) >= 1);
 
-    if ((tb = db_get_table(p, tptr[1], DB_READ, kind)) == NULL) {
-	BIF_ERROR(p, BADARG);
-    }
+    DB_TRAP_GET_TABLE(tb, tptr[1], DB_READ, kind,
+                      &ets_select_continue_exp);
 
     cret = tb->common.meth->db_select_continue(p, tb, a1,
 					       &ret);
@@ -2690,10 +2800,10 @@ static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1)
 
 BIF_RETTYPE ets_select_1(BIF_ALIST_1)
 {
-    return ets_select1(BIF_P, BIF_ARG_1);
+    return ets_select1(BIF_P, BIF_ets_select_1, BIF_ARG_1);
 }
 
-static BIF_RETTYPE ets_select1(Process *p, Eterm arg1)
+static BIF_RETTYPE ets_select1(Process *p, int bif_ix, Eterm arg1)
 {
     BIF_RETTYPE result;
     DbTable* tb;
@@ -2715,10 +2825,10 @@ static BIF_RETTYPE ets_select1(Process *p, Eterm arg1)
 	BIF_ERROR(p, BADARG);
     }
     tptr = tuple_val(arg1);
-    if (arityval(*tptr) < 1 ||
-	(tb = db_get_table(p, tptr[1], DB_READ, LCK_READ)) == NULL) {
-	BIF_ERROR(p, BADARG);
-    }
+    if (arityval(*tptr) < 1)
+        BIF_ERROR(p, BADARG);
+
+    DB_GET_TABLE(tb, tptr[1], DB_READ, LCK_READ, bif_ix, NULL, p);
 
     safety = ITERATION_SAFETY(p,tb);
     if (safety == ITER_UNSAFE) {
@@ -2754,33 +2864,27 @@ static BIF_RETTYPE ets_select1(Process *p, Eterm arg1)
 
 BIF_RETTYPE ets_select_2(BIF_ALIST_2)
 {
-    return ets_select2(BIF_P, BIF_ARG_1, BIF_ARG_2);
+    DbTable* tb;
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_select_2);
+    return ets_select2(BIF_P, tb, BIF_ARG_1, BIF_ARG_2);
 }
 
 static BIF_RETTYPE
-ets_select2(Process* p, Eterm arg1, Eterm arg2)
+ets_select2(Process* p, DbTable* tb, Eterm tid, Eterm ms)
 {
     BIF_RETTYPE result;
-    DbTable* tb;
     int cret;
     enum DbIterSafety safety;
     Eterm ret;
 
     CHECK_TABLES();
 
-    /*
-     * Make sure that the table exists.
-     */
-
-    if ((tb = db_get_table(p, arg1, DB_READ, LCK_READ)) == NULL) {
-	BIF_ERROR(p, BADARG);
-    }
     safety = ITERATION_SAFETY(p,tb);
     if (safety == ITER_UNSAFE) {
 	local_fix_table(tb);
     }
 
-    cret = tb->common.meth->db_select(p, tb, arg1, arg2, 0, &ret);
+    cret = tb->common.meth->db_select(p, tb, tid, ms, 0, &ret);
 
     if (DID_TRAP(p,ret) && safety != ITER_SAFE) {
 	fix_table_locked(p, tb);
@@ -2823,9 +2927,9 @@ static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1)
 
     tptr = tuple_val(a1);
     ASSERT(arityval(*tptr) >= 1);
-    if ((tb = db_get_table(p, tptr[1], DB_READ, kind)) == NULL) {
-	BIF_ERROR(p, BADARG);
-    }
+
+    DB_TRAP_GET_TABLE(tb, tptr[1], DB_READ, kind,
+                      &ets_select_count_continue_exp);
 
     cret = tb->common.meth->db_select_count_continue(p, tb, a1, &ret);
 
@@ -2860,13 +2964,9 @@ BIF_RETTYPE ets_select_count_2(BIF_ALIST_2)
     Eterm ret;
 
     CHECK_TABLES();
-    /*
-     * Make sure that the table exists.
-     */
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_select_count_2);
+
     safety = ITERATION_SAFETY(BIF_P,tb);
     if (safety == ITER_UNSAFE) {
 	local_fix_table(tb);
@@ -2916,9 +3016,8 @@ static BIF_RETTYPE ets_select_replace_1(BIF_ALIST_1)
     tptr = tuple_val(a1);
     ASSERT(arityval(*tptr) >= 1);
 
-    if ((tb = db_get_table(p, tptr[1], DB_WRITE, kind)) == NULL) {
-        BIF_ERROR(p,BADARG);
-    }
+    DB_TRAP_GET_TABLE(tb, tptr[1], DB_WRITE, kind,
+                      &ets_select_replace_continue_exp);
 
     cret = tb->common.meth->db_select_replace_continue(p,tb,a1,&ret);
 
@@ -2952,9 +3051,7 @@ BIF_RETTYPE ets_select_replace_2(BIF_ALIST_2)
 
     CHECK_TABLES();
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
-        BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_WRITE, LCK_WRITE_REC, BIF_ets_select_replace_2);
 
     if (tb->common.status & DB_BAG) {
         /* Bag implementation presented both semantic consistency
@@ -3005,13 +3102,8 @@ BIF_RETTYPE ets_select_reverse_3(BIF_ALIST_3)
     Sint chunk_size;
 
     CHECK_TABLES();
-    /*
-     * Make sure that the table exists.
-     */
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_select_reverse_3);
     
     /* Chunk size strictly greater than 0 */
     if (is_not_small(BIF_ARG_3) || (chunk_size = signed_val(BIF_ARG_3)) <= 0) {
@@ -3049,7 +3141,7 @@ BIF_RETTYPE ets_select_reverse_3(BIF_ALIST_3)
 
 BIF_RETTYPE ets_select_reverse_1(BIF_ALIST_1)
 {
-    return ets_select1(BIF_P, BIF_ARG_1);
+    return ets_select1(BIF_P, BIF_ets_select_reverse_1, BIF_ARG_1);
 }
 
 BIF_RETTYPE ets_select_reverse_2(BIF_ALIST_2)
@@ -3061,13 +3153,9 @@ BIF_RETTYPE ets_select_reverse_2(BIF_ALIST_2)
     Eterm ret;
 
     CHECK_TABLES();
-    /*
-     * Make sure that the table exists.
-     */
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_select_reverse_2);
+
     safety = ITERATION_SAFETY(BIF_P,tb);
     if (safety == ITER_UNSAFE) {
 	local_fix_table(tb);
@@ -3099,45 +3187,63 @@ BIF_RETTYPE ets_select_reverse_2(BIF_ALIST_2)
 
 
 /* 
-** ets:match_object(Continuation), ets:match_object(Table, Pattern), ets:match_object(Table,Pattern,ChunkSize) 
+** ets:match_object(Continuation)
 */
 BIF_RETTYPE ets_match_object_1(BIF_ALIST_1)
 {
-    return ets_select1(BIF_P, BIF_ARG_1);
+    return ets_select1(BIF_P, BIF_ets_match_object_1, BIF_ARG_1);
 }
 
+/*
+** ets:match_object(Table, Pattern)
+*/
 BIF_RETTYPE ets_match_object_2(BIF_ALIST_2)
 {
+    DbTable* tb;
     Eterm ms;
     DeclareTmpHeap(buff,8,BIF_P);
     Eterm *hp = buff;
     Eterm res;
 
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_match_object_2);
+
     UseTmpHeap(8,BIF_P);
     ms = CONS(hp, am_DollarUnderscore, NIL);
     hp += 2;
     ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); 
     hp += 4;
     ms = CONS(hp, ms, NIL);
-    res = ets_select2(BIF_P, BIF_ARG_1, ms);
+    res = ets_select2(BIF_P, tb, BIF_ARG_1, ms);
     UnUseTmpHeap(8,BIF_P);
     return res;
 }
 
+/*
+** ets:match_object(Table,Pattern,ChunkSize)
+*/
 BIF_RETTYPE ets_match_object_3(BIF_ALIST_3)
 {
+    DbTable* tb;
+    Sint chunk_size;
     Eterm ms;
     DeclareTmpHeap(buff,8,BIF_P);
     Eterm *hp = buff;
     Eterm res;
 
+    /* Chunk size strictly greater than 0 */
+    if (is_not_small(BIF_ARG_3) || (chunk_size = signed_val(BIF_ARG_3)) <= 0) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+
+    DB_BIF_GET_TABLE(tb, DB_READ, LCK_READ, BIF_ets_match_object_3);
+
     UseTmpHeap(8,BIF_P);
     ms = CONS(hp, am_DollarUnderscore, NIL);
     hp += 2;
     ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); 
     hp += 4;
     ms = CONS(hp, ms, NIL);
-    res = ets_select3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+    res = ets_select3(BIF_P, tb, BIF_ARG_1, ms, chunk_size);
     UnUseTmpHeap(8,BIF_P);
     return res;
 }
@@ -3158,16 +3264,17 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1)
     Eterm res;
     int i;
     Eterm* hp;
+    Uint freason;
     /*Process* rp = NULL;*/
     /* If/when we implement lockless private tables:
     Eterm owner;
     */
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL) {
-	if (is_atom(BIF_ARG_1) || is_ref(BIF_ARG_1)) {
+    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ, &freason)) == NULL) {
+	if (freason == BADARG && (is_atom(BIF_ARG_1) || is_ref(BIF_ARG_1)))
 	    BIF_RET(am_undefined);
-	}
-	BIF_ERROR(BIF_P, BADARG);
+	else
+            return db_bif_fail(BIF_P, freason, BIF_ets_info_1, NULL);
     }
 
     /* If/when we implement lockless private tables:
@@ -3224,12 +3331,13 @@ BIF_RETTYPE ets_info_2(BIF_ALIST_2)
 {
     DbTable* tb;
     Eterm ret = THE_NON_VALUE;
+    Uint freason;
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL) {
-	if (is_atom(BIF_ARG_1) || is_ref(BIF_ARG_1)) {
+    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ, &freason)) == NULL) {
+	if (freason == BADARG && (is_atom(BIF_ARG_1) || is_ref(BIF_ARG_1)))
 	    BIF_RET(am_undefined);
-	}
-	BIF_ERROR(BIF_P, BADARG);
+        else
+            return db_bif_fail(BIF_P, freason, BIF_ets_info_2, NULL);
     }
     ret = table_info(BIF_P, tb, BIF_ARG_2);
     db_unlock(tb, LCK_READ);
@@ -3317,7 +3425,6 @@ int erts_ets_rwmtx_spin_count = -1;
 void init_db(ErtsDbSpinCount db_spin_count)
 {
     int i;
-    Eterm *hp;
     unsigned bits;
     size_t size;
 
@@ -3420,13 +3527,6 @@ void init_db(ErtsDbSpinCount db_spin_count)
     erts_init_trap_export(&ets_delete_continue_exp,
 			  am_ets, am_atom_put("delete_trap",11), 1,
 			  &ets_delete_trap);
-
-    hp = ms_delete_all_buff;
-    ms_delete_all = CONS(hp, am_true, NIL);
-    hp += 2;
-    ms_delete_all = TUPLE3(hp,am_Underscore,NIL,ms_delete_all);
-    hp +=4;
-    ms_delete_all = CONS(hp, ms_delete_all,NIL);
 }
 
 void
@@ -3788,7 +3888,8 @@ unlocked:
 	    erts_rwmtx_runlock(&tb->common.rwlock);
 	    erts_rwmtx_rwlock(&tb->common.rwlock);
 	    *kind_p = LCK_WRITE;
-	    if (tb->common.status & DB_DELETE) return;
+	    if (tb->common.status & (DB_DELETE|DB_BUSY))
+                return;
 	}
 	db_unfix_table_hash(&(tb->hash));
     }
@@ -4318,5 +4419,8 @@ void erts_lcnt_update_db_locks(int enable) {
     erts_schedule_multi_misc_aux_work(0, erts_no_schedulers,
         &lcnt_update_db_locks_per_sched, (void*)(UWord)enable);
 }
-
 #endif /* ERTS_ENABLE_LOCK_COUNT */
+
+#ifdef ETS_DBG_FORCE_TRAP
+erts_aint_t erts_ets_dbg_force_trap = 0;
+#endif
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index eb6da2c9fb..db86c81914 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -135,6 +135,10 @@ void erts_lcnt_enable_db_lock_count(DbTable *tb, int enable);
 void erts_lcnt_update_db_locks(int enable);
 #endif
 
+#ifdef ETS_DBG_FORCE_TRAP
+extern erts_aint_t erts_ets_dbg_force_trap;
+#endif
+
 #endif /* ERL_DB_H__ */
 
 #if defined(ERTS_WANT_DB_INTERNAL__) && !defined(ERTS_HAVE_DB_INTERNAL__)
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index cb5c496e90..8bbd0cd9a3 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -411,7 +411,7 @@ static void db_foreach_offheap_hash(DbTable *,
 				    void (*)(ErlOffHeap *, void *),
 				    void *);
 
-static int db_delete_all_objects_hash(Process* p, DbTable* tbl);
+static SWord db_delete_all_objects_hash(Process* p, DbTable* tbl, SWord reds);
 #ifdef HARDDEBUG
 static void db_check_table_hash(DbTableHash *tb);
 #endif
@@ -2255,7 +2255,7 @@ void db_initialize_hash(void)
 }
 
 
-int db_mark_all_deleted_hash(DbTable *tbl)
+static SWord db_mark_all_deleted_hash(DbTable *tbl, SWord reds)
 {
     DbTableHash *tb = &tbl->hash;
     HashDbTerm* list;
@@ -2270,10 +2270,11 @@ int db_mark_all_deleted_hash(DbTable *tbl)
 		list->hvalue = INVALID_HASH;
 		list = list->next;
 	    }while(list != NULL);
+            reds--;
 	}
     }
     erts_atomic_set_nob(&tb->common.nitems, 0);
-    return DB_ERROR_NONE;
+    return reds < 0 ? 0 : reds;   /* ToDo: Yield! */
 }
 
 
@@ -3073,16 +3074,20 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle)
     return;
 }
 
-static int db_delete_all_objects_hash(Process* p, DbTable* tbl)
+static SWord db_delete_all_objects_hash(Process* p, DbTable* tbl, SWord reds)
 {
     if (IS_FIXED(tbl)) {
-	db_mark_all_deleted_hash(tbl);
+        /* ToDo: Yield! */
+	reds = db_mark_all_deleted_hash(tbl, reds);
     } else {
-	db_free_table_hash(tbl);
+        reds = db_free_table_continue_hash(tbl, reds);
+        if (reds < 0)
+            return reds;
+
 	db_create_hash(p, tbl);
 	erts_atomic_set_nob(&tbl->hash.common.nitems, 0);
     }
-    return 0;
+    return reds;
 }
 
 void db_foreach_offheap_hash(DbTable *tbl,
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index 7d27609825..5f88170ced 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -86,9 +86,6 @@ int db_get_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret);
 
 int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret);
 
-/* not yet in method table */
-int db_mark_all_deleted_hash(DbTable *tbl);
-
 typedef struct {
     float avg_chain_len;
     float std_dev_chain_len;
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 5a276b9d88..5ef5451289 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -436,7 +436,7 @@ static void db_foreach_offheap_tree(DbTable *,
 				    void (*)(ErlOffHeap *, void *),
 				    void *);
 
-static int db_delete_all_objects_tree(Process* p, DbTable* tbl);
+static SWord db_delete_all_objects_tree(Process* p, DbTable* tbl, SWord reds);
 
 #ifdef HARDDEBUG
 static void db_check_table_tree(DbTable *tbl);
@@ -2023,12 +2023,14 @@ static SWord db_free_table_continue_tree(DbTable *tbl, SWord reds)
     return reds;
 }
 
-static int db_delete_all_objects_tree(Process* p, DbTable* tbl)
+static SWord db_delete_all_objects_tree(Process* p, DbTable* tbl, SWord reds)
 {
-    db_free_table_tree(tbl);
+    reds = db_free_table_continue_tree(tbl, reds);
+    if (reds < 0)
+        return reds;
     db_create_tree(p, tbl);
     erts_atomic_set_nob(&tbl->tree.common.nitems, 0);
-    return 0;
+    return reds;
 }
 
 static void do_db_tree_foreach_offheap(TreeDbTerm *,
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 6b126f35d6..db8dfa5be4 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -32,6 +32,7 @@
 ** DMC_DEBUG does NOT need DEBUG, but DEBUG needs DMC_DEBUG
 */
 #define DMC_DEBUG 1
+#define ETS_DBG_FORCE_TRAP 1
 #endif
 
 /*
@@ -180,8 +181,7 @@ typedef struct db_table_method
             Eterm* ret);
     int (*db_take)(Process *, DbTable *, Eterm, Eterm *);
 
-    int (*db_delete_all_objects)(Process* p,
-				 DbTable* db /* [in out] */ );
+    SWord (*db_delete_all_objects)(Process* p, DbTable* db, SWord reds);
 
     int (*db_free_table)(DbTable* db /* [in out] */ );
     SWord (*db_free_table_continue)(DbTable* db, SWord reds);
@@ -267,6 +267,10 @@ typedef struct db_table_common {
     Uint32 status;            /* bit masks defined  below */
     int keypos;               /* defaults to 1 */
     int compress;
+
+#ifdef ETS_DBG_FORCE_TRAP
+    erts_atomic_t dbg_force_trap;  /* &1 force enabled, &2 trap this call */
+#endif
 } DbTableCommon;
 
 /* These are status bit patterns */
@@ -281,9 +285,7 @@ typedef struct db_table_common {
 #define DB_FINE_LOCKED   (1 << 8) /* write_concurrency */
 #define DB_FREQ_READ     (1 << 9) /* read_concurrency */
 #define DB_NAMED_TABLE   (1 << 10)
-
-#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET\
-                              |DB_FINE_LOCKED|DB_FREQ_READ|DB_NAMED_TABLE)
+#define DB_BUSY          (1 << 11)
 
 #define IS_HASH_TABLE(Status) (!!((Status) & \
 				  (DB_BAG | DB_SET | DB_DUPLICATE_BAG)))
-- 
cgit v1.2.3


From 1005ffd33c7110f2097f711b6e9e07fbb9ae1d32 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 2 May 2018 17:33:28 +0200
Subject: erts: Refactor pseudo deleted ets objects

Separate pseudo-deleted-flag from the hash value.
---
 erts/emulator/beam/erl_db_hash.c | 85 +++++++++++++++++++++++-----------------
 erts/emulator/beam/erl_db_hash.h | 13 +++++-
 2 files changed, 61 insertions(+), 37 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 8bbd0cd9a3..8cabf04fb1 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -21,6 +21,7 @@
 /*
 ** Implementation of unordered ETS tables.
 ** The tables are implemented as linear dynamic hash tables.
+** https://en.wikipedia.org/wiki/Linear_hashing
 */
 
 /* SMP:
@@ -148,7 +149,7 @@ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval)
     return ix;
 }
 
-/* Remember a slot containing a pseudo-deleted item (INVALID_HASH)
+/* Remember a slot containing a pseudo-deleted item
  * Return false if we got raced by unfixing thread
  * and the object should be deleted for real.
  */
@@ -179,13 +180,17 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix,
 }
 
 
-#define MAX_HASH 0xEFFFFFFFUL
-#define INVALID_HASH 0xFFFFFFFFUL
+
+static ERTS_INLINE int is_pseudo_deleted(HashDbTerm* p)
+{
+    return p->pseudo_deleted;
+}
+
 
 /* optimised version of make_hash (normal case? atomic key) */
 #define MAKE_HASH(term) \
     ((is_atom(term) ? (atom_tab(atom_val(term))->slot.bucket.hvalue) : \
-      make_internal_hash(term, 0)) % MAX_HASH)
+      make_internal_hash(term, 0)) & MAX_HASH_MASK)
 
 #  define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1)
 #  define GET_LOCK(tb,hval) (&(tb)->locks->lck_vec[(hval) & DB_HASH_LOCK_MASK].lck)
@@ -436,7 +441,8 @@ static ERTS_INLINE void try_shrink(DbTableHash* tb)
 static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b,
 				    Eterm key, HashValue hval)
 {
-    if (b->hvalue != hval) return 0;
+    if (b->hvalue != hval || is_pseudo_deleted(b))
+        return 0;
     else {
 	Eterm itemKey = GETKEY(tb, b->dbterm.tpl);
 	ASSERT(!is_header(itemKey));
@@ -449,7 +455,8 @@ static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b,
 static ERTS_INLINE int has_key(DbTableHash* tb, HashDbTerm* b,
 			       Eterm key, HashValue hval)
 {
-    if (b->hvalue != hval && b->hvalue != INVALID_HASH) return 0;
+    if (b->hvalue != hval)
+        return 0;
     else {
 	Eterm itemKey = GETKEY(tb, b->dbterm.tpl);
 	ASSERT(!is_header(itemKey));
@@ -595,7 +602,7 @@ restart:
 	    b = *bp;
 	    
 	    while (b != NULL) {
-		if (b->hvalue == INVALID_HASH) {
+		if (is_pseudo_deleted(b)) {
 		    *bp = b->next;
 		    free_term(tb, b);
 		    work++;
@@ -764,8 +771,9 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail)
     */
     if (tb->common.status & DB_SET) {
 	HashDbTerm* bnext = b->next;
-	if (b->hvalue == INVALID_HASH) {
+	if (is_pseudo_deleted(b)) {
 	    erts_atomic_inc_nob(&tb->common.nitems);
+            b->pseudo_deleted = 0;
 	}
 	else if (key_clash_fail) {
 	    ret = DB_ERROR_BADKEY;
@@ -773,14 +781,14 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail)
 	}
 	q = replace_dbterm(tb, b, obj);
 	q->next = bnext;
-	q->hvalue = hval; /* In case of INVALID_HASH */
+	ASSERT(q->hvalue == hval);
 	*bp = q;
 	goto Ldone;
     }
     else if (key_clash_fail) { /* && (DB_BAG || DB_DUPLICATE_BAG) */
 	q = b;
 	do {
-	    if (q->hvalue != INVALID_HASH) {
+	    if (!is_pseudo_deleted(q)) {
 		ret = DB_ERROR_BADKEY;
 		goto Ldone;
 	    }
@@ -792,9 +800,10 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail)
 	q = b;
 	do {
 	    if (db_eq(&tb->common,obj,&q->dbterm)) {
-		if (q->hvalue == INVALID_HASH) {
+		if (is_pseudo_deleted(q)) {
 		    erts_atomic_inc_nob(&tb->common.nitems);
-		    q->hvalue = hval;
+                    q->pseudo_deleted = 0;
+		    ASSERT(q->hvalue == hval);
 		    if (q != b) { /* must move to preserve key insertion order */
 			*qp = q->next;
 			q->next = b;
@@ -812,6 +821,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail)
 Lnew:
     q = new_dbterm(tb, obj);
     q->hvalue = hval;
+    q->pseudo_deleted = 0;
     q->next = b;
     *bp = q;
     nitems = erts_atomic_inc_read_nob(&tb->common.nitems);
@@ -839,7 +849,7 @@ get_term_list(Process *p, DbTableHash *tb, Eterm key, HashValue hval,
 
     if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) {
         while (b2 && has_key(tb, b2, key, hval)) {
-	    if (b2->hvalue != INVALID_HASH)
+	    if (!is_pseudo_deleted(b2))
 		sz += b2->dbterm.size + 2;
 
             b2 = b2->next;
@@ -935,7 +945,7 @@ static int db_get_element_hash(Process *p, DbTable *tbl,
 
 		while(b2 != NULL && has_key(tb,b2,key,hval)) {
 		    if (ndex > arityval(b2->dbterm.tpl[0])
-			&& b2->hvalue != INVALID_HASH) {
+			&& !is_pseudo_deleted(b2)) {
 			retval = DB_ERROR_BADITEM;
 			goto done;
 		    }
@@ -943,7 +953,7 @@ static int db_get_element_hash(Process *p, DbTable *tbl,
 		}
 		b = b1;
 		while(b != b2) {
-		    if (b->hvalue != INVALID_HASH) {
+		    if (!is_pseudo_deleted(b)) {
 			Eterm *hp;
 			Eterm copy = db_copy_element_from_ets(&tb->common, p,
 							      &b->dbterm, ndex, &hp, 2);
@@ -993,7 +1003,7 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret)
 	    if (nitems_diff == -1 && IS_FIXED(tb)
                 && add_fixed_deletion(tb, ix, 0)) {
 		/* Pseudo remove (no need to keep several of same key) */
-		b->hvalue = INVALID_HASH;
+		b->pseudo_deleted = 1;
 	    } else {
 		*bp = b->next;
 		free_term(tb, b);
@@ -1002,7 +1012,7 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret)
 	    }
 	}
 	else {
-	    if (nitems_diff && b->hvalue != INVALID_HASH)
+	    if (nitems_diff && !is_pseudo_deleted(b))
 		break;
 	}
 	bp = &b->next;
@@ -1045,7 +1055,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret)
 	    if (db_eq(&tb->common,object, &b->dbterm)) {
 		--nitems_diff;
 		if (nkeys==1 && IS_FIXED(tb) && add_fixed_deletion(tb,ix,0)) {
-		    b->hvalue = INVALID_HASH;        /* Pseudo remove */
+		    b->pseudo_deleted = 1;
 		    bp = &b->next;
 		    b = b->next;
 		} else {
@@ -1060,7 +1070,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret)
 		}
 	    }
 	}
-	else if (nitems_diff && b->hvalue != INVALID_HASH) {
+	else if (nitems_diff && !is_pseudo_deleted(b)) {
 	    break;
 	}
 	bp = &b->next;
@@ -1253,7 +1263,7 @@ static int match_traverse(Process* p, DbTableHash* tb,
      */
     for(;;) {
         if (*current_ptr != NULL) {
-            if ((*current_ptr)->hvalue != INVALID_HASH) {
+            if (!is_pseudo_deleted(*current_ptr)) {
                 match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0,
                                             &(*current_ptr)->dbterm, hpp, 2);
                 saved_current = *current_ptr;
@@ -1380,7 +1390,7 @@ static int match_traverse_continue(Process* p, DbTableHash* tb,
     current_ptr = &BUCKET(tb,slot_ix);
     for(;;) {
         if (*current_ptr != NULL) {
-            if ((*current_ptr)->hvalue != INVALID_HASH) {
+            if (!is_pseudo_deleted(*current_ptr)) {
                 match_res = db_match_dbterm(&tb->common, p, *mpp, all_objects,
                                             &(*current_ptr)->dbterm, hpp, 2);
                 saved_current = *current_ptr;
@@ -1963,7 +1973,7 @@ static int mtraversal_select_delete_on_match_res(void* context_ptr, Sint slot_ix
                 goto do_erase;
             sd_context_ptr->last_pseudo_delete = slot_ix;
         }
-        (*current_ptr)->hvalue = INVALID_HASH;
+        (*current_ptr)->pseudo_deleted = 1;
     }
     else {
     do_erase:
@@ -2106,6 +2116,7 @@ static int mtraversal_select_replace_on_match_res(void* context_ptr, Sint slot_i
         new = new_dbterm(tb, match_res);
         new->next = next;
         new->hvalue = hval;
+        new->pseudo_deleted = 0;
         free_term(tb, **current_ptr_ptr);
         **current_ptr_ptr = new; /* replace 'next' pointer in previous object */
         *current_ptr_ptr = &((**current_ptr_ptr)->next); /* advance to next object */
@@ -2226,7 +2237,7 @@ static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
                     && add_fixed_deletion(tb, ix, 0)) {
                     /* Pseudo remove (no need to keep several of same key) */
                     bp = &b->next;
-                    b->hvalue = INVALID_HASH;
+                    b->pseudo_deleted = 1;
                     b = b->next;
                 } else {
                     *bp = b->next;
@@ -2267,7 +2278,7 @@ static SWord db_mark_all_deleted_hash(DbTable *tbl, SWord reds)
 	if ((list = BUCKET(tb,i)) != NULL) {
 	    add_fixed_deletion(tb, i, 0);
 	    do {
-		list->hvalue = INVALID_HASH;
+		list->pseudo_deleted = 1;
 		list = list->next;
 	    }while(list != NULL);
             reds--;
@@ -2317,7 +2328,7 @@ static void db_print_hash(fmtfn_t to, void *to_arg, int show, DbTable *tbl)
 		continue;
 	    erts_print(to, to_arg, "%d: [", i);
 	    while(list != 0) {
-		if (list->hvalue == INVALID_HASH)
+		if (is_pseudo_deleted(list))
 		    erts_print(to, to_arg, "*");
 		if (tb->common.compress) {
 		    Eterm key = GETKEY(tb, list->dbterm.tpl);
@@ -2683,7 +2694,7 @@ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2,
     if (!sz) {
 	ptr = ptr1;
 	while(ptr != ptr2) {
-	    if (ptr->hvalue != INVALID_HASH)
+	    if (!is_pseudo_deleted(ptr))
 		sz += ptr->dbterm.size + 2;
 	    ptr = ptr->next;
 	}
@@ -2694,7 +2705,7 @@ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2,
 
     ptr = ptr1;
     while(ptr != ptr2) {
-	if (ptr->hvalue != INVALID_HASH) {
+	if (!is_pseudo_deleted(ptr)) {
 	    copy = db_copy_object_from_ets(&tb->common, &ptr->dbterm, &hp, &MSO(p));
 	    list = CONS(hp, copy, list);
 	    hp  += 2;
@@ -2787,7 +2798,7 @@ static void grow(DbTableHash* tb, int nitems)
         p = *pnext;
         to_pnext = &BUCKET(tb, to_ix);
         while (p != NULL) {
-            if (p->hvalue == INVALID_HASH) { /* rare but possible with fine locking */
+            if (is_pseudo_deleted(p)) { /* rare but possible with fine locking */
                 *pnext = p->next;
                 free_term(tb, p);
                 p = *pnext;
@@ -2860,7 +2871,7 @@ static void shrink(DbTableHash* tb, int nitems)
          * as we must step through "src" anyway to purge pseudo deleted.
          */
         while(*bp != NULL) {
-            if ((*bp)->hvalue == INVALID_HASH) {
+            if (is_pseudo_deleted(*bp)) {
                 HashDbTerm* deleted = *bp;
                 *bp = deleted->next;
                 free_term(tb, deleted);
@@ -2918,7 +2929,7 @@ static HashDbTerm* next_live(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr
     ERTS_LC_ASSERT(IS_HASH_RLOCKED(tb,*iptr));
 
     for ( ; list != NULL; list = list->next) {
-	if (list->hvalue != INVALID_HASH)
+	if (!is_pseudo_deleted(list))
 	    return list;        
     }
 
@@ -2927,7 +2938,7 @@ static HashDbTerm* next_live(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr
 
 	list = BUCKET(tb,i);
 	while (list != NULL) {
-	    if (list->hvalue != INVALID_HASH) {
+	    if (!is_pseudo_deleted(list)) {
 		*iptr = i;
 		return list;        
 	    }
@@ -2960,7 +2971,7 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj,
             break;
         }
         if (has_key(tb, b, key, hval)) {
-            if (b->hvalue != INVALID_HASH) {
+            if (!is_pseudo_deleted(b)) {
                 goto Ldone;
             }
             break;
@@ -2990,16 +3001,18 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj,
             HashDbTerm *q = new_dbterm(tb, obj);
 
             q->hvalue = hval;
+            q->pseudo_deleted = 0;
             q->next = NULL;
             *bp = b = q;
             flags |= DB_INC_TRY_GROW;
         } else {
             HashDbTerm *q, *next = b->next;
 
-            ASSERT(b->hvalue == INVALID_HASH);
+            ASSERT(is_pseudo_deleted(b));
             q = replace_dbterm(tb, b, obj);
             q->next = next;
-            q->hvalue = hval;
+            ASSERT(q->hvalue == hval);
+            q->pseudo_deleted = 0;
             *bp = b = q;
             erts_atomic_inc_nob(&tb->common.nitems);
         }
@@ -3037,7 +3050,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle)
     if (handle->flags & DB_NEW_OBJECT && cret != DB_ERROR_NONE) {
         if (IS_FIXED(tb) && add_fixed_deletion(tb, hash_to_ix(tb, b->hvalue),
                                                0)) {
-            b->hvalue = INVALID_HASH;
+            b->pseudo_deleted = 1;
         } else {
             *bp = b->next;
             free_me = b;
@@ -3130,7 +3143,7 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats)
 	len = 0;
 	for (b = BUCKET(tb,ix); b!=NULL; b=b->next) {
 	    len++;
-            if (b->hvalue == INVALID_HASH)
+            if (is_pseudo_deleted(b))
                 ++kept_items;
 	}
 	sum += len;
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index 5f88170ced..127512e1fe 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -28,9 +28,20 @@ typedef struct fixed_deletion {
     struct fixed_deletion *next;
 } FixedDeletion;
 
+
+typedef Uint32 HashVal;
+
 typedef struct hash_db_term {
     struct  hash_db_term* next;  /* next bucket */
-    HashValue  hvalue;        /* stored hash value */
+#if SIZEOF_VOID_P == 4
+    Uint32 hvalue : 31;     /* stored hash value */
+    Uint32 pseudo_deleted : 1;
+# define MAX_HASH_MASK (((Uint32)1 << 31)-1)
+#elif SIZEOF_VOID_P == 8
+    Uint32 hvalue;
+    Uint32 pseudo_deleted;
+# define MAX_HASH_MASK ((Uint32)(Sint32)-1)
+#endif
     DbTerm dbterm;         /* The actual term */
 } HashDbTerm;
 
-- 
cgit v1.2.3


From bad272295b0356140dd16e5357bd283fd05e6dcf Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 2 May 2018 22:09:33 +0200
Subject: erts: Optimize ets hash object deallocactions

to be done after lock has been released.
---
 erts/emulator/beam/erl_db_hash.c | 60 ++++++++++++++++++++++++++++++----------
 1 file changed, 45 insertions(+), 15 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 8cabf04fb1..a8d7ab2e6c 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -286,6 +286,16 @@ static ERTS_INLINE void free_term(DbTableHash *tb, HashDbTerm* p)
     db_free_term((DbTable*)tb, p, offsetof(HashDbTerm, dbterm));
 }
 
+static ERTS_INLINE void free_term_list(DbTableHash *tb, HashDbTerm* p)
+{
+    while (p) {
+        HashDbTerm* next = p->next;
+        free_term(tb, p);
+        p = next;
+    }
+}
+
+
 /*
  * Local types 
  */
@@ -587,6 +597,7 @@ restart:
 	int ix = fx->slot;
 	HashDbTerm **bp;
 	HashDbTerm *b;
+        HashDbTerm *free_us = NULL;
 	erts_rwmtx_t* lck = WLOCK_HASH(tb,ix);
 
 	if (IS_FIXED(tb)) { /* interrupted by fixer */
@@ -603,10 +614,11 @@ restart:
 	    
 	    while (b != NULL) {
 		if (is_pseudo_deleted(b)) {
-		    *bp = b->next;
-		    free_term(tb, b);
+                    HashDbTerm* nxt = b->next;
+                    b->next = free_us;
+                    free_us = b;
 		    work++;
-		    b = *bp;
+		    b = *bp = nxt;
 		} else {
 		    bp = &b->next;
 		    b = b->next;
@@ -615,6 +627,7 @@ restart:
 	}
 	/* else slot has been joined and purged by shrink() */
 	WUNLOCK_HASH(lck);
+        free_term_list(tb, free_us);
 	fixdel = fx->next;
 	erts_db_free(ERTS_ALC_T_DB_FIX_DEL,
 		     (DbTable *) tb,
@@ -988,6 +1001,7 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret)
     int ix;
     HashDbTerm** bp;
     HashDbTerm* b;
+    HashDbTerm* free_us = NULL;
     erts_rwmtx_t* lck;
     int nitems_diff = 0;
 
@@ -1005,9 +1019,10 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret)
 		/* Pseudo remove (no need to keep several of same key) */
 		b->pseudo_deleted = 1;
 	    } else {
-		*bp = b->next;
-		free_term(tb, b);
-		b = *bp;
+		HashDbTerm* next = b->next;
+                b->next = free_us;
+		free_us = b;
+		b = *bp = next;
 		continue;
 	    }
 	}
@@ -1023,6 +1038,7 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret)
 	erts_atomic_add_nob(&tb->common.nitems, nitems_diff);
 	try_shrink(tb);
     }
+    free_term_list(tb, free_us);
     *ret = am_true;
     return DB_ERROR_NONE;
 }    
@@ -1037,6 +1053,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret)
     int ix;
     HashDbTerm** bp;
     HashDbTerm* b;
+    HashDbTerm* free_us = NULL;
     erts_rwmtx_t* lck;
     int nitems_diff = 0;
     int nkeys = 0;
@@ -1059,9 +1076,10 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret)
 		    bp = &b->next;
 		    b = b->next;
 		} else {
-		    *bp = b->next;
-		    free_term(tb, b);
-		    b = *bp;
+                    HashDbTerm* next = b->next;
+		    b->next = free_us;
+                    free_us = b;
+		    b = *bp = next;
 		}
 		if (tb->common.status & (DB_DUPLICATE_BAG)) {
 		    continue;
@@ -1081,6 +1099,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret)
 	erts_atomic_add_nob(&tb->common.nitems, nitems_diff);
 	try_shrink(tb);
     }
+    free_term_list(tb, free_us);
     *ret = am_true;
     return DB_ERROR_NONE;
 }    
@@ -1950,6 +1969,7 @@ typedef struct {
     Eterm* prev_continuation_tptr;
     erts_aint_t fixated_by_me;
     Uint last_pseudo_delete;
+    HashDbTerm* free_us;
 } mtraversal_select_delete_context_t;
 
 static int mtraversal_select_delete_on_nothing_can_match(void* context_ptr, Eterm* ret) {
@@ -1979,7 +1999,8 @@ static int mtraversal_select_delete_on_match_res(void* context_ptr, Sint slot_ix
     do_erase:
         del = *current_ptr;
         *current_ptr = (*current_ptr)->next; // replace pointer to term using next
-        free_term(sd_context_ptr->tb, del);
+        del->next = sd_context_ptr->free_us;
+        sd_context_ptr->free_us = del;
     }
     erts_atomic_dec_nob(&sd_context_ptr->tb->common.nitems);
 
@@ -1990,6 +2011,8 @@ static int mtraversal_select_delete_on_loop_ended(void* context_ptr, Sint slot_i
                                                   Sint iterations_left, Binary** mpp, Eterm* ret)
 {
     mtraversal_select_delete_context_t* sd_context_ptr = (mtraversal_select_delete_context_t*) context_ptr;
+    free_term_list(sd_context_ptr->tb, sd_context_ptr->free_us);
+    sd_context_ptr->free_us = NULL;
     ASSERT(iterations_left <= MAX_SELECT_DELETE_ITERATIONS);
     BUMP_REDS(sd_context_ptr->p, MAX_SELECT_DELETE_ITERATIONS - iterations_left);
     if (got) {
@@ -2003,6 +2026,8 @@ static int mtraversal_select_delete_on_trap(void* context_ptr, Sint slot_ix, Sin
                                             Binary** mpp, Eterm* ret)
 {
     mtraversal_select_delete_context_t* sd_context_ptr = (mtraversal_select_delete_context_t*) context_ptr;
+    free_term_list(sd_context_ptr->tb, sd_context_ptr->free_us);
+    sd_context_ptr->free_us = NULL;
     return on_mtraversal_simple_trap(
             &ets_select_delete_continue_exp,
             sd_context_ptr->p,
@@ -2013,7 +2038,7 @@ static int mtraversal_select_delete_on_trap(void* context_ptr, Sint slot_ix, Sin
 }
 
 static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret) {
-    mtraversal_select_delete_context_t sd_context = {0};
+    mtraversal_select_delete_context_t sd_context;
     Sint chunk_size = 0;
 
     sd_context.p = p;
@@ -2023,6 +2048,7 @@ static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, Eterm patt
     sd_context.prev_continuation_tptr = NULL;
     sd_context.fixated_by_me = sd_context.tb->common.is_thread_safe ? 0 : 1; /* TODO: something nicer */
     sd_context.last_pseudo_delete = (Uint) -1;
+    sd_context.free_us = NULL;
 
     return match_traverse(
             sd_context.p, sd_context.tb,
@@ -2040,7 +2066,7 @@ static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, Eterm patt
  * This is called when select_delete traps
  */
 static int db_select_delete_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) {
-    mtraversal_select_delete_context_t sd_context = {0};
+    mtraversal_select_delete_context_t sd_context;
     Eterm* tptr;
     Eterm tid;
     Binary* mp;
@@ -2060,6 +2086,7 @@ static int db_select_delete_continue_hash(Process* p, DbTable* tbl, Eterm contin
     sd_context.prev_continuation_tptr = tptr;
     sd_context.fixated_by_me = ONLY_WRITER(p, sd_context.tb) ? 0 : 1; /* TODO: something nicer */
     sd_context.last_pseudo_delete = (Uint) -1;
+    sd_context.free_us = NULL;
 
     return match_traverse_continue(
             sd_context.p, sd_context.tb, chunk_size,
@@ -2220,6 +2247,7 @@ static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
 {
     DbTableHash *tb = &tbl->hash;
     HashDbTerm **bp, *b;
+    HashDbTerm *free_us = NULL;
     HashValue hval = MAKE_HASH(key);
     erts_rwmtx_t *lck = WLOCK_HASH(tb, hval);
     int ix = hash_to_ix(tb, hval);
@@ -2240,9 +2268,10 @@ static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
                     b->pseudo_deleted = 1;
                     b = b->next;
                 } else {
-                    *bp = b->next;
-                    free_term(tb, b);
-                    b = *bp;
+                    HashDbTerm* next = b->next;
+                    b->next = free_us;
+                    free_us = b;
+                    b = *bp = next;
                 }
             }
             break;
@@ -2253,6 +2282,7 @@ static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
         erts_atomic_add_nob(&tb->common.nitems, nitems_diff);
         try_shrink(tb);
     }
+    free_term_list(tb, free_us);
     return DB_ERROR_NONE;
 }
 
-- 
cgit v1.2.3


From 523176f67d7d5c4d3f5a9256af36e436f37afbec Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 3 May 2018 11:25:56 +0200
Subject: erts: Cleanup ets code

---
 erts/emulator/beam/erl_binary.h  |  4 +---
 erts/emulator/beam/erl_db.c      |  6 +++---
 erts/emulator/beam/erl_db_hash.c | 36 +++++----------------------------
 erts/emulator/beam/erl_db_tree.c | 43 +++++-----------------------------------
 erts/emulator/beam/erl_db_util.c |  2 +-
 erts/emulator/beam/erl_db_util.h |  2 +-
 6 files changed, 16 insertions(+), 77 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 46653a8580..7dfd0c273a 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -146,9 +146,7 @@ typedef union {
 
 /* A "magic" binary flag */
 #define BIN_FLAG_MAGIC      1
-#define BIN_FLAG_USR1       2 /* Reserved for use by different modules too mark */
-#define BIN_FLAG_USR2       4 /*  certain binaries as special (used by ets) */
-#define BIN_FLAG_DRV        8
+#define BIN_FLAG_DRV        2
 
 #endif /* ERL_BINARY_H__TYPES__ */
 
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 322f5f1505..696ad17daa 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -378,7 +378,7 @@ static SWord free_fixations_locked(Process* p, DbTable *tb);
 static void delete_all_objects_continue(Process* p, DbTable* tb);
 static SWord free_table_continue(Process *p, DbTable *tb, SWord reds);
 static void print_table(fmtfn_t to, void *to_arg, int show,  DbTable* tb);
-static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1);
+static BIF_RETTYPE ets_select_delete_trap_1(BIF_ALIST_1);
 static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1);
 static BIF_RETTYPE ets_select_replace_1(BIF_ALIST_1);
 static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1);
@@ -2273,7 +2273,7 @@ BIF_RETTYPE ets_delete_object_2(BIF_ALIST_2)
 /*
 ** This is for trapping, cannot be called directly.
 */
-static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1)
+static BIF_RETTYPE ets_select_delete_trap_1(BIF_ALIST_1)
 {
     Process *p = BIF_P;
     Eterm a1 = BIF_ARG_1;
@@ -3506,7 +3506,7 @@ void init_db(ErtsDbSpinCount db_spin_count)
     /* Non visual BIF to trap to. */
     erts_init_trap_export(&ets_select_delete_continue_exp,
 			  am_ets, am_atom_put("delete_trap",11), 1,
-			  &ets_select_delete_1);
+			  &ets_select_delete_trap_1);
 
     /* Non visual BIF to trap to. */
     erts_init_trap_export(&ets_select_count_continue_exp,
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index a8d7ab2e6c..eb1b32d216 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -275,11 +275,6 @@ static ERTS_INLINE Sint next_slot_w(DbTableHash* tb, Uint ix,
 }
 
 
-/*
- * Some special binary flags
- */
-#define BIN_FLAG_ALL_OBJECTS         BIN_FLAG_USR1
-
 
 static ERTS_INLINE void free_term(DbTableHash *tb, HashDbTerm* p)
 {
@@ -305,9 +300,6 @@ struct mp_prefound {
 };
 
 struct mp_info {
-    int all_objects;		/* True if complete objects are always
-				 * returned from the match_spec (can use 
-				 * copy_shallow on the return value) */
     int something_can_match;	/* The match_spec is not "impossible" */
     int key_given;
     struct mp_prefound dlists[10];  /* Default list of "pre-found" buckets */
@@ -1134,18 +1126,6 @@ static int db_slot_hash(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret)
 }
 
 
-/*
- * This is just here so I can take care of the return value 
- * that is to be sent during a trap (the BIF_TRAP macros explicitly returns)
- */
-static BIF_RETTYPE bif_trap1(Export *bif,
-			      Process *p, 
-			      Eterm p1) 
-{
-    BIF_TRAP1(bif, p, p1);
-}
-
-
 /*
  * Match traversal callbacks
  */
@@ -1245,10 +1225,6 @@ static int match_traverse(Process* p, DbTableHash* tb,
         goto done;
     }
 
-    if (mpi.all_objects) {
-        mpi.mp->intern.flags |= BIN_FLAG_ALL_OBJECTS;
-    }
-
     /*
      * Look for initial slot / bucket
      */
@@ -1283,7 +1259,7 @@ static int match_traverse(Process* p, DbTableHash* tb,
     for(;;) {
         if (*current_ptr != NULL) {
             if (!is_pseudo_deleted(*current_ptr)) {
-                match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0,
+                match_res = db_match_dbterm(&tb->common, p, mpi.mp,
                                             &(*current_ptr)->dbterm, hpp, 2);
                 saved_current = *current_ptr;
                 if (on_match_res(context_ptr, slot_ix, &current_ptr, match_res)) {
@@ -1367,7 +1343,6 @@ static int match_traverse_continue(Process* p, DbTableHash* tb,
                                    void* context_ptr, /* For callbacks */
                                    Eterm* ret)
 {
-    int all_objects = (*mpp)->intern.flags & BIN_FLAG_ALL_OBJECTS;
     HashDbTerm** current_ptr;  /* Refers to either the bucket pointer or
                                        * the 'next' pointer in the previous term
                                        */
@@ -1410,7 +1385,7 @@ static int match_traverse_continue(Process* p, DbTableHash* tb,
     for(;;) {
         if (*current_ptr != NULL) {
             if (!is_pseudo_deleted(*current_ptr)) {
-                match_res = db_match_dbterm(&tb->common, p, *mpp, all_objects,
+                match_res = db_match_dbterm(&tb->common, p, *mpp,
                                             &(*current_ptr)->dbterm, hpp, 2);
                 saved_current = *current_ptr;
                 if (on_match_res(context_ptr, slot_ix, &current_ptr, match_res)) {
@@ -1509,7 +1484,7 @@ static ERTS_INLINE int on_mtraversal_simple_trap(Export* trap_function,
             make_small(slot_ix),
             mpb,
             egot);
-    *ret = bif_trap1(trap_function, p, continuation);
+    ERTS_BIF_PREP_TRAP1(*ret, trap_function, p, continuation);
     return DB_ERROR_NONE;
 }
 
@@ -1689,7 +1664,8 @@ static int mtraversal_select_chunk_on_trap(void* context_ptr, Sint slot_ix, Sint
                 sc_context_ptr->match_list,
                 make_small(got));
     }
-    *ret = bif_trap1(&ets_select_continue_exp, sc_context_ptr->p, continuation);
+    ERTS_BIF_PREP_TRAP1(*ret, &ets_select_continue_exp, sc_context_ptr->p,
+                        continuation);
     return DB_ERROR_NONE;
 }
 
@@ -2457,7 +2433,6 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern,
     mpi->num_lists = 0;
     mpi->key_given = 1;
     mpi->something_can_match = 0;
-    mpi->all_objects = 1;
     mpi->mp = NULL;
 
     for (lst = pattern; is_list(lst); lst = CDR(list_val(lst)))
@@ -2510,7 +2485,6 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern,
 
 	if (!is_list(body) || CDR(list_val(body)) != NIL ||
 	    CAR(list_val(body)) != am_DollarUnderscore) {
-	    mpi->all_objects = 0;
 	}
 	++i;
 	if (!(mpi->key_given)) {
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 5ef5451289..4dcb41523f 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -169,11 +169,6 @@ static ERTS_INLINE TreeDbTerm* replace_dbterm(DbTableTree *tb, TreeDbTerm* old,
 #define DIR_RIGHT 1
 #define DIR_END 2 
 
-/*
- * Special binary flag
- */
-#define BIN_FLAG_ALL_OBJECTS         BIN_FLAG_USR1
-
 /*
  * Number of records to delete before trapping.
  */
@@ -218,9 +213,6 @@ static void do_dump_tree2(DbTableTree*, int to, void *to_arg, int show,
  * functions.
  */
 struct mp_info {
-    int all_objects;		/* True if complete objects are always
-				 * returned from the match_spec (can use 
-				 * copy_shallow on the return value) */
     int something_can_match;	/* The match_spec is not "impossible" */
     int some_limitation;	/* There is some limitation on the search
 				 * area, i. e. least and/or most is set.*/
@@ -248,7 +240,6 @@ struct select_context {
     Eterm *lastobj;
     Sint32 max;
     int keypos;
-    int all_objects;
     Sint got;
     Sint chunk_size;
 };
@@ -263,7 +254,6 @@ struct select_count_context {
     Eterm *lastobj;
     Sint32 max;
     int keypos;
-    int all_objects;
     Sint got;
 };
 
@@ -293,7 +283,6 @@ struct select_replace_context {
     Eterm *lastobj;
     Sint32 max;
     int keypos;
-    int all_objects;
     Sint replaced;
 };
 
@@ -992,7 +981,6 @@ static int db_select_continue_tree(Process *p,
     sc.lastobj = NULL;
     sc.max = 1000;
     sc.keypos = tb->common.keypos;
-    sc.all_objects = mp->intern.flags & BIN_FLAG_ALL_OBJECTS;
     sc.chunk_size = chunk_size;
     reverse = unsigned_val(tptr[7]);
     sc.got = signed_val(tptr[8]);
@@ -1143,7 +1131,6 @@ static int db_select_tree(Process *p, DbTable *tbl, Eterm tid,
     }
 
     sc.mp = mpi.mp;
-    sc.all_objects = mpi.all_objects;
 
     if (!mpi.got_partial && mpi.some_limitation && 
 	CMP_EQ(mpi.least,mpi.most)) {
@@ -1183,8 +1170,6 @@ static int db_select_tree(Process *p, DbTable *tbl, Eterm tid,
     sz = size_object(key);
     hp = HAlloc(p, 9 + sz + ERTS_MAGIC_REF_THING_SIZE);
     key = copy_struct(key, sz, &hp, &MSO(p));
-    if (mpi.all_objects)
-	(mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS;
     mpb= erts_db_make_match_prog_ref(p,mpi.mp,&hp);
 	    
     continuation = TUPLE8
@@ -1346,7 +1331,6 @@ static int db_select_count_tree(Process *p, DbTable *tbl, Eterm tid,
     }
 
     sc.mp = mpi.mp;
-    sc.all_objects = mpi.all_objects;
 
     if (!mpi.got_partial && mpi.some_limitation && 
 	CMP_EQ(mpi.least,mpi.most)) {
@@ -1381,8 +1365,6 @@ static int db_select_count_tree(Process *p, DbTable *tbl, Eterm tid,
 	hp += BIG_UINT_HEAP_SIZE;
     }
     key = copy_struct(key, sz, &hp, &MSO(p));
-    if (mpi.all_objects)
-	(mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS;
     mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
 	    
     continuation = TUPLE5
@@ -1449,7 +1431,6 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid,
     }
 
     sc.mp = mpi.mp;
-    sc.all_objects = mpi.all_objects;
 
     if (!mpi.got_partial && mpi.some_limitation && 
 	CMP_EQ(mpi.least,mpi.most)) {
@@ -1506,8 +1487,6 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid,
 	sz = size_object(key);
 	hp = HAlloc(p, 9 + sz + ERTS_MAGIC_REF_THING_SIZE);
 	key = copy_struct(key, sz, &hp, &MSO(p));
-	if (mpi.all_objects)
-	    (mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS;
 	mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
 	
 	continuation = TUPLE8
@@ -1532,8 +1511,6 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, Eterm tid,
     hp = HAlloc(p, 9 + sz + ERTS_MAGIC_REF_THING_SIZE);
     key = copy_struct(key, sz, &hp, &MSO(p));
 
-    if (mpi.all_objects)
-	(mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS;
     mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
     continuation = TUPLE8
 	(hp,
@@ -1882,7 +1859,6 @@ static int db_select_replace_tree(Process *p, DbTable *tbl, Eterm tid,
     }
 
     sc.mp = mpi.mp;
-    sc.all_objects = mpi.all_objects;
 
     stack = get_static_stack(tb);
     if (!mpi.got_partial && mpi.some_limitation &&
@@ -1928,8 +1904,6 @@ static int db_select_replace_tree(Process *p, DbTable *tbl, Eterm tid,
         hp += BIG_UINT_HEAP_SIZE;
     }
     key = copy_struct(key, sz, &hp, &MSO(p));
-    if (mpi.all_objects)
-        (mpi.mp)->intern.flags |= BIN_FLAG_ALL_OBJECTS;
     mpb = erts_db_make_match_prog_ref(p,mpi.mp,&hp);
 
     continuation = TUPLE5
@@ -2216,7 +2190,6 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern,
     mpi->got_partial = 0;
     mpi->something_can_match = 0;
     mpi->mp = NULL;
-    mpi->all_objects = 1;
     mpi->save_term = NULL;
 
     for (lst = pattern; is_list(lst); lst = CDR(list_val(lst)))
@@ -2266,7 +2239,6 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern,
 
 	if (!is_list(body) || CDR(list_val(body)) != NIL ||
 	    CAR(list_val(body)) != am_DollarUnderscore) {
-	    mpi->all_objects = 0;
 	}
 	++i;
 
@@ -3341,8 +3313,7 @@ static int doit_select(DbTableTree *tb, TreeDbTerm *this, void *ptr,
 			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0))) {
 	return 0;
     }
-    ret = db_match_dbterm(&tb->common,sc->p,sc->mp,sc->all_objects,
-			  &this->dbterm, &hp, 2);
+    ret = db_match_dbterm(&tb->common, sc->p, sc->mp, &this->dbterm, &hp, 2);
     if (is_value(ret)) {
 	sc->accum = CONS(hp, ret, sc->accum);
     }
@@ -3373,8 +3344,7 @@ static int doit_select_count(DbTableTree *tb, TreeDbTerm *this, void *ptr,
 			  GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0)) {
 	return 0;
     }
-    ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0,
-			  &this->dbterm, NULL, 0);
+    ret = db_match_dbterm(&tb->common, sc->p, sc->mp, &this->dbterm, NULL, 0);
     if (ret == am_true) {
 	++(sc->got);
     }
@@ -3403,8 +3373,7 @@ static int doit_select_chunk(DbTableTree *tb, TreeDbTerm *this, void *ptr,
 	return 0;
     }
 
-    ret = db_match_dbterm(&tb->common, sc->p, sc->mp, sc->all_objects,
-			  &this->dbterm, &hp, 2);
+    ret = db_match_dbterm(&tb->common, sc->p, sc->mp, &this->dbterm, &hp, 2);
     if (is_value(ret)) {
 	++(sc->got);
 	sc->accum = CONS(hp, ret, sc->accum);
@@ -3439,8 +3408,7 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr,
 	cmp_partly_bound(sc->end_condition, 
 			 GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0)
 	return 0;
-    ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0,
-			  &this->dbterm, NULL, 0);
+    ret = db_match_dbterm(&tb->common, sc->p, sc->mp, &this->dbterm, NULL, 0);
     if (ret == am_true) {
 	key = GETKEY(sc->tb, this->dbterm.tpl);
 	linkout_tree(sc->tb, key);
@@ -3467,8 +3435,7 @@ static int doit_select_replace(DbTableTree *tb, TreeDbTerm **this, void *ptr,
 			  GETKEY_WITH_POS(sc->keypos, (*this)->dbterm.tpl)) > 0)) {
 	return 0;
     }
-    ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0,
-			  &(*this)->dbterm, NULL, 0);
+    ret = db_match_dbterm(&tb->common, sc->p, sc->mp, &(*this)->dbterm, NULL, 0);
 
     if (is_value(ret)) {
         TreeDbTerm* new;
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 3836f28aa4..fc23b793c8 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -5321,7 +5321,7 @@ void db_free_tmp_uncompressed(DbTerm* obj)
 }
 
 Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog,
-			     int all, DbTerm* obj, Eterm** hpp, Uint extra)
+                      DbTerm* obj, Eterm** hpp, Uint extra)
 {
     enum erts_pam_run_flags flags;
     Uint32 dummy;
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index db8dfa5be4..56689bcc03 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -471,7 +471,7 @@ Binary *db_match_compile(Eterm *matchexpr, Eterm *guards,
 /* Returns newly allocated MatchProg binary with refc == 0*/
 
 Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog,
-		      int all, DbTerm* obj, Eterm** hpp, Uint extra);
+		      DbTerm* obj, Eterm** hpp, Uint extra);
 
 Eterm db_prog_match(Process *p, Process *self,
                     Binary *prog, Eterm term,
-- 
cgit v1.2.3


From 7c7c10014f8c77f66a805def5ac52bb06402aeb4 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 4 May 2018 16:08:23 +0200
Subject: erts: Refactor ets select iteration code

* Remove all "mtraversal_" prefixes.
* Rename all local context variables as "ctx".
* Changed callbacks from function arguments
  to members of new base context struct "match_callbacks_t"
* Remove unnecessary struct initializations "= {0}"
---
 erts/emulator/beam/erl_db_hash.c | 615 ++++++++++++++++++++-------------------
 1 file changed, 323 insertions(+), 292 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index eb1b32d216..23074766e8 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1130,13 +1130,16 @@ static int db_slot_hash(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret)
  * Match traversal callbacks
  */
 
+typedef struct match_callbacks_t_ match_callbacks_t;
+struct match_callbacks_t_
+{
 /* Called when no match is possible.
  *      context_ptr: Pointer to context
  *      ret: Pointer to traversal function term return.
  *
  * Both the direct return value and 'ret' are used as the traversal function return values.
  */
-typedef int (*mtraversal_on_nothing_can_match_t)(void* context_ptr, Eterm* ret);
+    int (*on_nothing_can_match)(match_callbacks_t* ctx, Eterm* ret);
 
 /* Called for each match result.
  *      context_ptr: Pointer to context
@@ -1147,8 +1150,8 @@ typedef int (*mtraversal_on_nothing_can_match_t)(void* context_ptr, Eterm* ret);
  *
  * Should return 1 for successful match, 0 otherwise.
  */
-typedef int (*mtraversal_on_match_res_t)(void* context_ptr, Sint slot_ix, HashDbTerm*** current_ptr_ptr,
-                                         Eterm match_res);
+    int (*on_match_res)(match_callbacks_t* ctx, Sint slot_ix,
+                        HashDbTerm*** current_ptr_ptr, Eterm match_res);
 
 /* Called when either we've matched enough elements in this cycle or EOT was reached.
  *      context_ptr: Pointer to context
@@ -1161,8 +1164,8 @@ typedef int (*mtraversal_on_match_res_t)(void* context_ptr, Sint slot_ix, HashDb
  * Both the direct return value and 'ret' are used as the traversal function return values.
  * If *mpp is set to NULL, it won't be deallocated (useful for trapping.)
  */
-typedef int (*mtraversal_on_loop_ended_t)(void* context_ptr, Sint slot_ix, Sint got,
-                                          Sint iterations_left, Binary** mpp, Eterm* ret);
+    int (*on_loop_ended)(match_callbacks_t* ctx, Sint slot_ix, Sint got,
+                         Sint iterations_left, Binary** mpp, Eterm* ret);
 
 /* Called when it's time to trap
  *      context_ptr: Pointer to context
@@ -1174,7 +1177,11 @@ typedef int (*mtraversal_on_loop_ended_t)(void* context_ptr, Sint slot_ix, Sint
  * Both the direct return value and 'ret' are used as the traversal function return values.
  * If *mpp is set to NULL, it won't be deallocated (useful for trapping.)
  */
-typedef int (*mtraversal_on_trap_t)(void* context_ptr, Sint slot_ix, Sint got, Binary** mpp, Eterm* ret);
+    int (*on_trap)(match_callbacks_t* ctx, Sint slot_ix, Sint got, Binary** mpp,
+                   Eterm* ret);
+
+};
+
 
 /*
  * Begin hash table match traversal
@@ -1187,11 +1194,7 @@ static int match_traverse(Process* p, DbTableHash* tb,
                           Eterm** hpp,          /* Heap */
                           int lock_for_write,   /* Set to 1 if we're going to delete or
                                                    modify existing terms */
-                          mtraversal_on_nothing_can_match_t on_nothing_can_match,
-                          mtraversal_on_match_res_t on_match_res,
-                          mtraversal_on_loop_ended_t on_loop_ended,
-                          mtraversal_on_trap_t on_trap,
-                          void* context_ptr, /* State for callbacks above */
+                          match_callbacks_t* ctx,
                           Eterm* ret)
 {
     Sint slot_ix;                  /* Slot index */
@@ -1221,7 +1224,7 @@ static int match_traverse(Process* p, DbTableHash* tb,
 
     if (!mpi.something_can_match) {
         /* Can't possibly match anything */
-        ret_value = on_nothing_can_match(context_ptr, ret);
+        ret_value = ctx->on_nothing_can_match(ctx, ret);
         goto done;
     }
 
@@ -1240,7 +1243,8 @@ static int match_traverse(Process* p, DbTableHash* tb,
             }
             slot_ix = next_slot_function(tb,slot_ix,&lck);
             if (slot_ix == 0) {
-                ret_value = on_loop_ended(context_ptr, slot_ix, got, iterations_left, &mpi.mp, ret);
+                ret_value = ctx->on_loop_ended(ctx, slot_ix, got, iterations_left,
+                                               &mpi.mp, ret);
                 goto done;
             }
         }
@@ -1262,7 +1266,7 @@ static int match_traverse(Process* p, DbTableHash* tb,
                 match_res = db_match_dbterm(&tb->common, p, mpi.mp,
                                             &(*current_ptr)->dbterm, hpp, 2);
                 saved_current = *current_ptr;
-                if (on_match_res(context_ptr, slot_ix, &current_ptr, match_res)) {
+                if (ctx->on_match_res(ctx, slot_ix, &current_ptr, match_res)) {
                     ++got;
                 }
                 --iterations_left;
@@ -1276,7 +1280,7 @@ static int match_traverse(Process* p, DbTableHash* tb,
         else if (mpi.key_given) {  /* Key is bound */
             unlock_hash_function(lck);
             if (current_list_pos == mpi.num_lists) {
-                ret_value = on_loop_ended(context_ptr, -1, got, iterations_left, &mpi.mp, ret);
+                ret_value = ctx->on_loop_ended(ctx, -1, got, iterations_left, &mpi.mp, ret);
                 goto done;
             } else {
                 slot_ix = mpi.lists[current_list_pos].ix;
@@ -1301,18 +1305,18 @@ static int match_traverse(Process* p, DbTableHash* tb,
                  * Since many heap fragments will make the GC slower, trap and GC now.
                  */
                 unlock_hash_function(lck);
-                ret_value = on_trap(context_ptr, slot_ix, got, &mpi.mp, ret);
+                ret_value = ctx->on_trap(ctx, slot_ix, got, &mpi.mp, ret);
                 goto done;
             }
             current_ptr = &BUCKET(tb,slot_ix);
         }
     }
 
-    ret_value = on_loop_ended(context_ptr, slot_ix, got, iterations_left, &mpi.mp, ret);
+    ret_value = ctx->on_loop_ended(ctx, slot_ix, got, iterations_left, &mpi.mp, ret);
 
 done:
     /* We should only jump directly to this label if
-     * we've already called on_nothing_can_match / on_loop_ended / on_trap
+     * we've already called ctx->nothing_can_match / loop_ended / trap
      */
     if (mpi.mp != NULL) {
         erts_bin_free(mpi.mp);
@@ -1337,10 +1341,7 @@ static int match_traverse_continue(Process* p, DbTableHash* tb,
                                    Binary** mpp,         /* Existing match program */
                                    int lock_for_write,   /* Set to 1 if we're going to delete or
                                                             modify existing terms */
-                                   mtraversal_on_match_res_t on_match_res,
-                                   mtraversal_on_loop_ended_t on_loop_ended,
-                                   mtraversal_on_trap_t on_trap,
-                                   void* context_ptr, /* For callbacks */
+                                   match_callbacks_t* ctx,
                                    Eterm* ret)
 {
     HashDbTerm** current_ptr;  /* Refers to either the bucket pointer or
@@ -1366,7 +1367,7 @@ static int match_traverse_continue(Process* p, DbTableHash* tb,
        || (chunk_size && got >= chunk_size))
     {
         /* Already got all or enough in the match_list */
-        ret_value = on_loop_ended(context_ptr, slot_ix, got, iterations_left, mpp, ret);
+        ret_value = ctx->on_loop_ended(ctx, slot_ix, got, iterations_left, mpp, ret);
         goto done;
     }
 
@@ -1388,7 +1389,7 @@ static int match_traverse_continue(Process* p, DbTableHash* tb,
                 match_res = db_match_dbterm(&tb->common, p, *mpp,
                                             &(*current_ptr)->dbterm, hpp, 2);
                 saved_current = *current_ptr;
-                if (on_match_res(context_ptr, slot_ix, &current_ptr, match_res)) {
+                if (ctx->on_match_res(ctx, slot_ix, &current_ptr, match_res)) {
                     ++got;
                 }
                 --iterations_left;
@@ -1414,14 +1415,14 @@ static int match_traverse_continue(Process* p, DbTableHash* tb,
                  * Since many heap fragments will make the GC slower, trap and GC now.
                  */
                 unlock_hash_function(lck);
-                ret_value = on_trap(context_ptr, slot_ix, got, mpp, ret);
+                ret_value = ctx->on_trap(ctx, slot_ix, got, mpp, ret);
                 goto done;
             }
             current_ptr = &BUCKET(tb,slot_ix);
         }
     }
 
-    ret_value = on_loop_ended(context_ptr, slot_ix, got, iterations_left, mpp, ret);
+    ret_value = ctx->on_loop_ended(ctx, slot_ix, got, iterations_left, mpp, ret);
 
 done:
     /* We should only jump directly to this label if
@@ -1438,7 +1439,7 @@ done:
  * as well as their continuation-handling counterparts.
  */
 
-static ERTS_INLINE int on_mtraversal_simple_trap(Export* trap_function,
+static ERTS_INLINE int on_simple_trap(Export* trap_function,
                                                  Process* p,
                                                  DbTableHash* tb,
                                                  Eterm tid,
@@ -1488,12 +1489,12 @@ static ERTS_INLINE int on_mtraversal_simple_trap(Export* trap_function,
     return DB_ERROR_NONE;
 }
 
-static ERTS_INLINE int unpack_simple_mtraversal_continuation(Eterm continuation,
-                                                             Eterm** tptr_ptr,
-                                                             Eterm* tid_ptr,
-                                                             Sint* slot_ix_p,
-                                                             Binary** mpp,
-                                                             Sint* got_p)
+static ERTS_INLINE int unpack_simple_continuation(Eterm continuation,
+                                                  Eterm** tptr_ptr,
+                                                  Eterm* tid_ptr,
+                                                  Sint* slot_ix_p,
+                                                  Binary** mpp,
+                                                  Sint* got_p)
 {
     Eterm* tptr;
     ASSERT(is_tuple(continuation));
@@ -1528,6 +1529,7 @@ static ERTS_INLINE int unpack_simple_mtraversal_continuation(Eterm continuation,
 #define MAX_SELECT_CHUNK_ITERATIONS 1000
 
 typedef struct {
+    match_callbacks_t base;
     Process* p;
     DbTableHash* tb;
     Eterm tid;
@@ -1535,83 +1537,86 @@ typedef struct {
     Sint chunk_size;
     Eterm match_list;
     Eterm* prev_continuation_tptr;
-} mtraversal_select_chunk_context_t;
+} select_chunk_context_t;
 
-static int mtraversal_select_chunk_on_nothing_can_match(void* context_ptr, Eterm* ret) {
-    mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr;
-    *ret = (sc_context_ptr->chunk_size > 0 ? am_EOT : NIL);
+static int select_chunk_on_nothing_can_match(match_callbacks_t* ctx_base, Eterm* ret)
+{
+    select_chunk_context_t* ctx = (select_chunk_context_t*) ctx_base;
+    *ret = (ctx->chunk_size > 0 ? am_EOT : NIL);
     return DB_ERROR_NONE;
 }
 
-static int mtraversal_select_chunk_on_match_res(void* context_ptr, Sint slot_ix,
-                                                HashDbTerm*** current_ptr_ptr,
-                                                Eterm match_res)
+static int select_chunk_on_match_res(match_callbacks_t* ctx_base, Sint slot_ix,
+                                     HashDbTerm*** current_ptr_ptr,
+                                     Eterm match_res)
 {
-    mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr;
+    select_chunk_context_t* ctx = (select_chunk_context_t*) ctx_base;
     if (is_value(match_res)) {
-        sc_context_ptr->match_list = CONS(sc_context_ptr->hp, match_res, sc_context_ptr->match_list);
+        ctx->match_list = CONS(ctx->hp, match_res, ctx->match_list);
         return 1;
     }
     return 0;
 }
 
-static int mtraversal_select_chunk_on_loop_ended(void* context_ptr, Sint slot_ix, Sint got,
-                                                 Sint iterations_left, Binary** mpp, Eterm* ret)
+static int select_chunk_on_loop_ended(match_callbacks_t* ctx_base,
+                                      Sint slot_ix, Sint got,
+                                      Sint iterations_left, Binary** mpp,
+                                      Eterm* ret)
 {
-    mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr;
+    select_chunk_context_t* ctx = (select_chunk_context_t*) ctx_base;
     Eterm mpb;
 
     if (iterations_left == MAX_SELECT_CHUNK_ITERATIONS) {
         /* We didn't get to iterate a single time, which means EOT */
-        ASSERT(sc_context_ptr->match_list == NIL);
-        *ret = (sc_context_ptr->chunk_size > 0 ? am_EOT : NIL);
+        ASSERT(ctx->match_list == NIL);
+        *ret = (ctx->chunk_size > 0 ? am_EOT : NIL);
         return DB_ERROR_NONE;
     }
     else {
         ASSERT(iterations_left < MAX_SELECT_CHUNK_ITERATIONS);
-        BUMP_REDS(sc_context_ptr->p, MAX_SELECT_CHUNK_ITERATIONS - iterations_left);
-        if (sc_context_ptr->chunk_size) {
+        BUMP_REDS(ctx->p, MAX_SELECT_CHUNK_ITERATIONS - iterations_left);
+        if (ctx->chunk_size) {
             Eterm continuation;
             Eterm rest = NIL;
             Sint rest_size = 0;
 
-            if (got > sc_context_ptr->chunk_size) { /* Split list in return value and 'rest' */
-                Eterm tmp = sc_context_ptr->match_list;
-                rest = sc_context_ptr->match_list;
-                while (got-- > sc_context_ptr->chunk_size + 1) {
+            if (got > ctx->chunk_size) { /* Split list in return value and 'rest' */
+                Eterm tmp = ctx->match_list;
+                rest = ctx->match_list;
+                while (got-- > ctx->chunk_size + 1) {
                     tmp = CDR(list_val(tmp));
                     ++rest_size;
                 }
                 ++rest_size;
-                sc_context_ptr->match_list = CDR(list_val(tmp));
+                ctx->match_list = CDR(list_val(tmp));
                 CDR(list_val(tmp)) = NIL; /* Destructive, the list has never
                                              been in 'user space' */
             }
             if (rest != NIL || slot_ix >= 0) { /* Need more calls */
-                Eterm tid = sc_context_ptr->tid;
-                sc_context_ptr->hp = HAllocX(sc_context_ptr->p,
-                                             3 + 7 + ERTS_MAGIC_REF_THING_SIZE,
-                                             ERTS_MAGIC_REF_THING_SIZE);
-                mpb = erts_db_make_match_prog_ref(sc_context_ptr->p, *mpp, &sc_context_ptr->hp);
+                Eterm tid = ctx->tid;
+                ctx->hp = HAllocX(ctx->p,
+                                  3 + 7 + ERTS_MAGIC_REF_THING_SIZE,
+                                  ERTS_MAGIC_REF_THING_SIZE);
+                mpb = erts_db_make_match_prog_ref(ctx->p, *mpp, &ctx->hp);
                 if (is_atom(tid))
-                    tid = erts_db_make_tid(sc_context_ptr->p,
-                                           &sc_context_ptr->tb->common);
+                    tid = erts_db_make_tid(ctx->p,
+                                           &ctx->tb->common);
                 continuation = TUPLE6(
-                        sc_context_ptr->hp,
+                        ctx->hp,
                         tid,
                         make_small(slot_ix),
-                        make_small(sc_context_ptr->chunk_size),
+                        make_small(ctx->chunk_size),
                         mpb, rest,
                         make_small(rest_size));
                 *mpp = NULL; /* Otherwise the caller will destroy it */
-                sc_context_ptr->hp += 7;
-                *ret = TUPLE2(sc_context_ptr->hp, sc_context_ptr->match_list, continuation);
+                ctx->hp += 7;
+                *ret = TUPLE2(ctx->hp, ctx->match_list, continuation);
                 return DB_ERROR_NONE;
             } else { /* All data is exhausted */
-                if (sc_context_ptr->match_list != NIL) { /* No more data to search but still a
+                if (ctx->match_list != NIL) { /* No more data to search but still a
                                                             result to return to the caller */
-                    sc_context_ptr->hp = HAlloc(sc_context_ptr->p, 3);
-                    *ret = TUPLE2(sc_context_ptr->hp, sc_context_ptr->match_list, am_EOT);
+                    ctx->hp = HAlloc(ctx->p, 3);
+                    *ret = TUPLE2(ctx->hp, ctx->match_list, am_EOT);
                     return DB_ERROR_NONE;
                 } else { /* Reached the end of the ttable with no data to return */
                     *ret = am_EOT;
@@ -1619,83 +1624,88 @@ static int mtraversal_select_chunk_on_loop_ended(void* context_ptr, Sint slot_ix
                 }
             }
         }
-        *ret = sc_context_ptr->match_list;
+        *ret = ctx->match_list;
         return DB_ERROR_NONE;
     }
 }
 
-static int mtraversal_select_chunk_on_trap(void* context_ptr, Sint slot_ix, Sint got,
-                                           Binary** mpp, Eterm* ret)
+static int select_chunk_on_trap(match_callbacks_t* ctx_base,
+                                Sint slot_ix, Sint got,
+                                Binary** mpp, Eterm* ret)
 {
-    mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr;
+    select_chunk_context_t* ctx = (select_chunk_context_t*) ctx_base;
     Eterm mpb;
     Eterm continuation;
     Eterm* hp;
 
-    BUMP_ALL_REDS(sc_context_ptr->p);
+    BUMP_ALL_REDS(ctx->p);
 
-    if (sc_context_ptr->prev_continuation_tptr == NULL) {
-        Eterm tid = sc_context_ptr->tid;
+    if (ctx->prev_continuation_tptr == NULL) {
+        Eterm tid = ctx->tid;
         /* First time we're trapping */
-        hp = HAllocX(sc_context_ptr->p, 7 + ERTS_MAGIC_REF_THING_SIZE,
+        hp = HAllocX(ctx->p, 7 + ERTS_MAGIC_REF_THING_SIZE,
                      ERTS_MAGIC_REF_THING_SIZE);
         if (is_atom(tid))
-            tid = erts_db_make_tid(sc_context_ptr->p, &sc_context_ptr->tb->common);
-        mpb = erts_db_make_match_prog_ref(sc_context_ptr->p, *mpp, &hp);
+            tid = erts_db_make_tid(ctx->p, &ctx->tb->common);
+        mpb = erts_db_make_match_prog_ref(ctx->p, *mpp, &hp);
         continuation = TUPLE6(
                 hp,
                 tid,
                 make_small(slot_ix),
-                make_small(sc_context_ptr->chunk_size),
+                make_small(ctx->chunk_size),
                 mpb,
-                sc_context_ptr->match_list,
+                ctx->match_list,
                 make_small(got));
         *mpp = NULL; /* otherwise the caller will destroy it */
     }
     else {
         /* Not the first time we're trapping; reuse continuation terms */
-        hp = HAlloc(sc_context_ptr->p, 7);
+        hp = HAlloc(ctx->p, 7);
         continuation = TUPLE6(
                 hp,
-                sc_context_ptr->prev_continuation_tptr[1],
+                ctx->prev_continuation_tptr[1],
                 make_small(slot_ix),
-                sc_context_ptr->prev_continuation_tptr[3],
-                sc_context_ptr->prev_continuation_tptr[4],
-                sc_context_ptr->match_list,
+                ctx->prev_continuation_tptr[3],
+                ctx->prev_continuation_tptr[4],
+                ctx->match_list,
                 make_small(got));
     }
-    ERTS_BIF_PREP_TRAP1(*ret, &ets_select_continue_exp, sc_context_ptr->p,
+    ERTS_BIF_PREP_TRAP1(*ret, &ets_select_continue_exp, ctx->p,
                         continuation);
     return DB_ERROR_NONE;
 }
 
-static int db_select_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, int reverse, Eterm *ret) {
+static int db_select_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern,
+                          int reverse, Eterm *ret)
+{
     return db_select_chunk_hash(p, tbl, tid, pattern, 0, reverse, ret);
 }
 
-static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Sint chunk_size,
+static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid,
+                                Eterm pattern, Sint chunk_size,
                                 int reverse, Eterm *ret)
 {
-    mtraversal_select_chunk_context_t sc_context;
-    sc_context.p = p;
-    sc_context.tb = &tbl->hash;
-    sc_context.tid = tid;
-    sc_context.hp = NULL;
-    sc_context.chunk_size = chunk_size;
-    sc_context.match_list = NIL;
-    sc_context.prev_continuation_tptr = NULL;
+    select_chunk_context_t ctx;
+
+    ctx.base.on_nothing_can_match = select_chunk_on_nothing_can_match;
+    ctx.base.on_match_res         = select_chunk_on_match_res;
+    ctx.base.on_loop_ended        = select_chunk_on_loop_ended;
+    ctx.base.on_trap              = select_chunk_on_trap,
+    ctx.p = p;
+    ctx.tb = &tbl->hash;
+    ctx.tid = tid;
+    ctx.hp = NULL;
+    ctx.chunk_size = chunk_size;
+    ctx.match_list = NIL;
+    ctx.prev_continuation_tptr = NULL;
 
     return match_traverse(
-            sc_context.p, sc_context.tb,
+            ctx.p, ctx.tb,
             pattern, NULL,
-            sc_context.chunk_size,
+            ctx.chunk_size,
             MAX_SELECT_CHUNK_ITERATIONS,
-            &sc_context.hp, 0,
-            mtraversal_select_chunk_on_nothing_can_match,
-            mtraversal_select_chunk_on_match_res,
-            mtraversal_select_chunk_on_loop_ended,
-            mtraversal_select_chunk_on_trap,
-            &sc_context, ret);
+            &ctx.hp, 0,
+            &ctx.base, ret);
 }
 
 /*
@@ -1704,47 +1714,50 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, Eterm tid, Eterm patte
  *
  */
 
-static int mtraversal_select_chunk_continue_on_loop_ended(void* context_ptr, Sint slot_ix, Sint got,
-                                                          Sint iterations_left, Binary** mpp, Eterm* ret)
+static
+int select_chunk_continue_on_loop_ended(match_callbacks_t* ctx_base,
+                                        Sint slot_ix, Sint got,
+                                        Sint iterations_left, Binary** mpp,
+                                        Eterm* ret)
 {
-    mtraversal_select_chunk_context_t* sc_context_ptr = (mtraversal_select_chunk_context_t*) context_ptr;
+    select_chunk_context_t* ctx = (select_chunk_context_t*) ctx_base;
     Eterm continuation;
     Eterm rest = NIL;
     Eterm* hp;
 
     ASSERT(iterations_left <= MAX_SELECT_CHUNK_ITERATIONS);
-    BUMP_REDS(sc_context_ptr->p, MAX_SELECT_CHUNK_ITERATIONS - iterations_left);
-    if (sc_context_ptr->chunk_size) {
+    BUMP_REDS(ctx->p, MAX_SELECT_CHUNK_ITERATIONS - iterations_left);
+    if (ctx->chunk_size) {
         Sint rest_size = 0;
-        if (got > sc_context_ptr->chunk_size) {
+        if (got > ctx->chunk_size) {
             /* Cannot write destructively here,
                the list may have
                been in user space */
-            hp = HAlloc(sc_context_ptr->p, (got - sc_context_ptr->chunk_size) * 2);
-            while (got-- > sc_context_ptr->chunk_size) {
-                rest = CONS(hp, CAR(list_val(sc_context_ptr->match_list)), rest);
+            hp = HAlloc(ctx->p, (got - ctx->chunk_size) * 2);
+            while (got-- > ctx->chunk_size) {
+                rest = CONS(hp, CAR(list_val(ctx->match_list)), rest);
                 hp += 2;
-                sc_context_ptr->match_list = CDR(list_val(sc_context_ptr->match_list));
+                ctx->match_list = CDR(list_val(ctx->match_list));
                 ++rest_size;
             }
         }
         if (rest != NIL || slot_ix >= 0) {
-            hp = HAlloc(sc_context_ptr->p, 3 + 7);
+            hp = HAlloc(ctx->p, 3 + 7);
             continuation = TUPLE6(
                     hp,
-                    sc_context_ptr->prev_continuation_tptr[1],
+                    ctx->prev_continuation_tptr[1],
                     make_small(slot_ix),
-                    sc_context_ptr->prev_continuation_tptr[3],
-                    sc_context_ptr->prev_continuation_tptr[4],
+                    ctx->prev_continuation_tptr[3],
+                    ctx->prev_continuation_tptr[4],
                     rest,
                     make_small(rest_size));
             hp += 7;
-            *ret = TUPLE2(hp, sc_context_ptr->match_list, continuation);
+            *ret = TUPLE2(hp, ctx->match_list, continuation);
             return DB_ERROR_NONE;
         } else {
-            if (sc_context_ptr->match_list != NIL) {
-                hp = HAlloc(sc_context_ptr->p, 3);
-                *ret = TUPLE2(hp, sc_context_ptr->match_list, am_EOT);
+            if (ctx->match_list != NIL) {
+                hp = HAlloc(ctx->p, 3);
+                *ret = TUPLE2(hp, ctx->match_list, am_EOT);
                 return DB_ERROR_NONE;
             } else {
                 *ret = am_EOT;
@@ -1752,15 +1765,17 @@ static int mtraversal_select_chunk_continue_on_loop_ended(void* context_ptr, Sin
             }
         }
     }
-    *ret = sc_context_ptr->match_list;
+    *ret = ctx->match_list;
     return DB_ERROR_NONE;
 }
 
 /*
  * This is called when select traps
  */
-static int db_select_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) {
-    mtraversal_select_chunk_context_t sc_context = {0};
+static int db_select_continue_hash(Process* p, DbTable* tbl, Eterm continuation,
+                                   Eterm* ret)
+{
+    select_chunk_context_t ctx;
     Eterm* tptr;
     Eterm tid;
     Binary* mp;
@@ -1795,21 +1810,21 @@ static int db_select_continue_hash(Process* p, DbTable* tbl, Eterm continuation,
     match_list = tptr[5];
 
     /* Proceed */
-    sc_context.p = p;
-    sc_context.tb = &tbl->hash;
-    sc_context.tid = tid;
-    sc_context.hp = NULL;
-    sc_context.chunk_size = chunk_size;
-    sc_context.match_list = match_list;
-    sc_context.prev_continuation_tptr = tptr;
+    ctx.base.on_match_res  = select_chunk_on_match_res;
+    ctx.base.on_loop_ended = select_chunk_continue_on_loop_ended;
+    ctx.base.on_trap       = select_chunk_on_trap;
+    ctx.p = p;
+    ctx.tb = &tbl->hash;
+    ctx.tid = tid;
+    ctx.hp = NULL;
+    ctx.chunk_size = chunk_size;
+    ctx.match_list = match_list;
+    ctx.prev_continuation_tptr = tptr;
 
     return match_traverse_continue(
-            sc_context.p, sc_context.tb, sc_context.chunk_size,
-            iterations_left, &sc_context.hp, slot_ix, got, &mp, 0,
-            mtraversal_select_chunk_on_match_res, /* Reuse callback */
-            mtraversal_select_chunk_continue_on_loop_ended,
-            mtraversal_select_chunk_on_trap,      /* Reuse callback */
-            &sc_context, ret);
+            ctx.p, ctx.tb, ctx.chunk_size,
+            iterations_left, &ctx.hp, slot_ix, got, &mp, 0,
+            &ctx.base, ret);
 
 badparam:
     *ret = NIL;
@@ -1828,75 +1843,83 @@ badparam:
 #define MAX_SELECT_COUNT_ITERATIONS 1000
 
 typedef struct {
+    match_callbacks_t base;
     Process* p;
     DbTableHash* tb;
     Eterm tid;
-    Eterm* hp;
     Eterm* prev_continuation_tptr;
-} mtraversal_select_count_context_t;
+} select_count_context_t;
 
-static int mtraversal_select_count_on_nothing_can_match(void* context_ptr, Eterm* ret) {
+static int select_count_on_nothing_can_match(match_callbacks_t* ctx_base,
+                                             Eterm* ret)
+{
     *ret = make_small(0);
     return DB_ERROR_NONE;
 }
 
-static int mtraversal_select_count_on_match_res(void* context_ptr, Sint slot_ix,
-                                                HashDbTerm*** current_ptr_ptr,
-                                                Eterm match_res)
+static int select_count_on_match_res(match_callbacks_t* ctx_base, Sint slot_ix,
+                                     HashDbTerm*** current_ptr_ptr,
+                                     Eterm match_res)
 {
     return (match_res == am_true);
 }
 
-static int mtraversal_select_count_on_loop_ended(void* context_ptr, Sint slot_ix, Sint got,
-                                                 Sint iterations_left, Binary** mpp, Eterm* ret)
+static int select_count_on_loop_ended(match_callbacks_t* ctx_base,
+                                      Sint slot_ix, Sint got,
+                                      Sint iterations_left, Binary** mpp,
+                                      Eterm* ret)
 {
-    mtraversal_select_count_context_t* scnt_context_ptr = (mtraversal_select_count_context_t*) context_ptr;
+    select_count_context_t* ctx = (select_count_context_t*) ctx_base;
     ASSERT(iterations_left <= MAX_SELECT_COUNT_ITERATIONS);
-    BUMP_REDS(scnt_context_ptr->p, MAX_SELECT_COUNT_ITERATIONS - iterations_left);
-    *ret = erts_make_integer(got, scnt_context_ptr->p);
+    BUMP_REDS(ctx->p, MAX_SELECT_COUNT_ITERATIONS - iterations_left);
+    *ret = erts_make_integer(got, ctx->p);
     return DB_ERROR_NONE;
 }
 
-static int mtraversal_select_count_on_trap(void* context_ptr, Sint slot_ix, Sint got,
-                                           Binary** mpp, Eterm* ret)
+static int select_count_on_trap(match_callbacks_t* ctx_base,
+                                Sint slot_ix, Sint got,
+                                Binary** mpp, Eterm* ret)
 {
-    mtraversal_select_count_context_t* scnt_context_ptr = (mtraversal_select_count_context_t*) context_ptr;
-    return on_mtraversal_simple_trap(
+    select_count_context_t* ctx = (select_count_context_t*) ctx_base;
+    return on_simple_trap(
             &ets_select_count_continue_exp,
-            scnt_context_ptr->p,
-            scnt_context_ptr->tb,
-            scnt_context_ptr->tid,
-            scnt_context_ptr->prev_continuation_tptr,
+            ctx->p,
+            ctx->tb,
+            ctx->tid,
+            ctx->prev_continuation_tptr,
             slot_ix, got, mpp, ret);
 }
 
-static int db_select_count_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret) {
-    mtraversal_select_count_context_t scnt_context = {0};
+static int db_select_count_hash(Process *p, DbTable *tbl, Eterm tid,
+                                Eterm pattern, Eterm *ret)
+{
+    select_count_context_t ctx;
     Sint iterations_left = MAX_SELECT_COUNT_ITERATIONS;
     Sint chunk_size = 0;
 
-    scnt_context.p = p;
-    scnt_context.tb = &tbl->hash;
-    scnt_context.tid = tid;
-    scnt_context.hp = NULL;
-    scnt_context.prev_continuation_tptr = NULL;
+    ctx.base.on_nothing_can_match = select_count_on_nothing_can_match;
+    ctx.base.on_match_res         = select_count_on_match_res;
+    ctx.base.on_loop_ended        = select_count_on_loop_ended;
+    ctx.base.on_trap              = select_count_on_trap;
+    ctx.p = p;
+    ctx.tb = &tbl->hash;
+    ctx.tid = tid;
+    ctx.prev_continuation_tptr = NULL;
 
     return match_traverse(
-            scnt_context.p, scnt_context.tb,
+            ctx.p, ctx.tb,
             pattern, NULL,
             chunk_size, iterations_left, NULL, 0,
-            mtraversal_select_count_on_nothing_can_match,
-            mtraversal_select_count_on_match_res,
-            mtraversal_select_count_on_loop_ended,
-            mtraversal_select_count_on_trap,
-            &scnt_context, ret);
+            &ctx.base, ret);
 }
 
 /*
  * This is called when select_count traps
  */
-static int db_select_count_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) {
-    mtraversal_select_count_context_t scnt_context = {0};
+static int db_select_count_continue_hash(Process* p, DbTable* tbl,
+                                         Eterm continuation, Eterm* ret)
+{
+    select_count_context_t ctx;
     Eterm* tptr;
     Eterm tid;
     Binary* mp;
@@ -1905,25 +1928,24 @@ static int db_select_count_continue_hash(Process* p, DbTable* tbl, Eterm continu
     Sint chunk_size = 0;
     *ret = NIL;
 
-    if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) {
+    if (unpack_simple_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) {
         *ret = NIL;
         return DB_ERROR_BADPARAM;
     }
 
-    scnt_context.p = p;
-    scnt_context.tb = &tbl->hash;
-    scnt_context.tid = tid;
-    scnt_context.hp = NULL;
-    scnt_context.prev_continuation_tptr = tptr;
+    ctx.base.on_match_res  = select_count_on_match_res;
+    ctx.base.on_loop_ended = select_count_on_loop_ended;
+    ctx.base.on_trap       = select_count_on_trap;
+    ctx.p = p;
+    ctx.tb = &tbl->hash;
+    ctx.tid = tid;
+    ctx.prev_continuation_tptr = tptr;
 
     return match_traverse_continue(
-            scnt_context.p, scnt_context.tb, chunk_size,
+            ctx.p, ctx.tb, chunk_size,
             MAX_SELECT_COUNT_ITERATIONS,
             NULL, slot_ix, got, &mp, 0,
-            mtraversal_select_count_on_match_res,  /* Reuse callback */
-            mtraversal_select_count_on_loop_ended, /* Reuse callback */
-            mtraversal_select_count_on_trap,       /* Reuse callback */
-            &scnt_context, ret);
+            &ctx.base, ret);
 }
 
 #undef MAX_SELECT_COUNT_ITERATIONS
@@ -1938,36 +1960,38 @@ static int db_select_count_continue_hash(Process* p, DbTable* tbl, Eterm continu
 #define MAX_SELECT_DELETE_ITERATIONS 1000
 
 typedef struct {
+    match_callbacks_t base;
     Process* p;
     DbTableHash* tb;
     Eterm tid;
-    Eterm* hp;
     Eterm* prev_continuation_tptr;
     erts_aint_t fixated_by_me;
     Uint last_pseudo_delete;
     HashDbTerm* free_us;
-} mtraversal_select_delete_context_t;
+} select_delete_context_t;
 
-static int mtraversal_select_delete_on_nothing_can_match(void* context_ptr, Eterm* ret) {
+static int select_delete_on_nothing_can_match(match_callbacks_t* ctx_base,
+                                              Eterm* ret)
+{
     *ret = make_small(0);
     return DB_ERROR_NONE;
 }
 
-static int mtraversal_select_delete_on_match_res(void* context_ptr, Sint slot_ix,
-                                                 HashDbTerm*** current_ptr_ptr,
-                                                 Eterm match_res)
+static int select_delete_on_match_res(match_callbacks_t* ctx_base, Sint slot_ix,
+                                      HashDbTerm*** current_ptr_ptr,
+                                      Eterm match_res)
 {
     HashDbTerm** current_ptr = *current_ptr_ptr;
-    mtraversal_select_delete_context_t* sd_context_ptr = (mtraversal_select_delete_context_t*) context_ptr;
+    select_delete_context_t* ctx = (select_delete_context_t*) ctx_base;
     HashDbTerm* del;
     if (match_res != am_true)
         return 0;
 
-    if (NFIXED(sd_context_ptr->tb) > sd_context_ptr->fixated_by_me) { /* fixated by others? */
-        if (slot_ix != sd_context_ptr->last_pseudo_delete) {
-            if (!add_fixed_deletion(sd_context_ptr->tb, slot_ix, sd_context_ptr->fixated_by_me))
+    if (NFIXED(ctx->tb) > ctx->fixated_by_me) { /* fixated by others? */
+        if (slot_ix != ctx->last_pseudo_delete) {
+            if (!add_fixed_deletion(ctx->tb, slot_ix, ctx->fixated_by_me))
                 goto do_erase;
-            sd_context_ptr->last_pseudo_delete = slot_ix;
+            ctx->last_pseudo_delete = slot_ix;
         }
         (*current_ptr)->pseudo_deleted = 1;
     }
@@ -1975,74 +1999,80 @@ static int mtraversal_select_delete_on_match_res(void* context_ptr, Sint slot_ix
     do_erase:
         del = *current_ptr;
         *current_ptr = (*current_ptr)->next; // replace pointer to term using next
-        del->next = sd_context_ptr->free_us;
-        sd_context_ptr->free_us = del;
+        del->next = ctx->free_us;
+        ctx->free_us = del;
     }
-    erts_atomic_dec_nob(&sd_context_ptr->tb->common.nitems);
+    erts_atomic_dec_nob(&ctx->tb->common.nitems);
 
     return 1;
 }
 
-static int mtraversal_select_delete_on_loop_ended(void* context_ptr, Sint slot_ix, Sint got,
-                                                  Sint iterations_left, Binary** mpp, Eterm* ret)
+static int select_delete_on_loop_ended(match_callbacks_t* ctx_base,
+                                       Sint slot_ix, Sint got,
+                                       Sint iterations_left, Binary** mpp,
+                                       Eterm* ret)
 {
-    mtraversal_select_delete_context_t* sd_context_ptr = (mtraversal_select_delete_context_t*) context_ptr;
-    free_term_list(sd_context_ptr->tb, sd_context_ptr->free_us);
-    sd_context_ptr->free_us = NULL;
+    select_delete_context_t* ctx = (select_delete_context_t*) ctx_base;
+    free_term_list(ctx->tb, ctx->free_us);
+    ctx->free_us = NULL;
     ASSERT(iterations_left <= MAX_SELECT_DELETE_ITERATIONS);
-    BUMP_REDS(sd_context_ptr->p, MAX_SELECT_DELETE_ITERATIONS - iterations_left);
+    BUMP_REDS(ctx->p, MAX_SELECT_DELETE_ITERATIONS - iterations_left);
     if (got) {
-	try_shrink(sd_context_ptr->tb);
+	try_shrink(ctx->tb);
     }
-    *ret = erts_make_integer(got, sd_context_ptr->p);
+    *ret = erts_make_integer(got, ctx->p);
     return DB_ERROR_NONE;
 }
 
-static int mtraversal_select_delete_on_trap(void* context_ptr, Sint slot_ix, Sint got,
-                                            Binary** mpp, Eterm* ret)
+static int select_delete_on_trap(match_callbacks_t* ctx_base,
+                                 Sint slot_ix, Sint got,
+                                 Binary** mpp, Eterm* ret)
 {
-    mtraversal_select_delete_context_t* sd_context_ptr = (mtraversal_select_delete_context_t*) context_ptr;
-    free_term_list(sd_context_ptr->tb, sd_context_ptr->free_us);
-    sd_context_ptr->free_us = NULL;
-    return on_mtraversal_simple_trap(
+    select_delete_context_t* ctx = (select_delete_context_t*) ctx_base;
+    free_term_list(ctx->tb, ctx->free_us);
+    ctx->free_us = NULL;
+    return on_simple_trap(
             &ets_select_delete_continue_exp,
-            sd_context_ptr->p,
-            sd_context_ptr->tb,
-            sd_context_ptr->tid,
-            sd_context_ptr->prev_continuation_tptr,
+            ctx->p,
+            ctx->tb,
+            ctx->tid,
+            ctx->prev_continuation_tptr,
             slot_ix, got, mpp, ret);
 }
 
-static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret) {
-    mtraversal_select_delete_context_t sd_context;
+static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid,
+                                 Eterm pattern, Eterm *ret)
+{
+    select_delete_context_t ctx;
     Sint chunk_size = 0;
 
-    sd_context.p = p;
-    sd_context.tb = &tbl->hash;
-    sd_context.tid = tid;
-    sd_context.hp = NULL;
-    sd_context.prev_continuation_tptr = NULL;
-    sd_context.fixated_by_me = sd_context.tb->common.is_thread_safe ? 0 : 1; /* TODO: something nicer */
-    sd_context.last_pseudo_delete = (Uint) -1;
-    sd_context.free_us = NULL;
+    ctx.base.on_nothing_can_match = select_delete_on_nothing_can_match;
+    ctx.base.on_match_res         = select_delete_on_match_res;
+    ctx.base.on_loop_ended        = select_delete_on_loop_ended;
+    ctx.base.on_trap              = select_delete_on_trap;
+    ctx.p = p;
+    ctx.tb = &tbl->hash;
+    ctx.tid = tid;
+    ctx.prev_continuation_tptr = NULL;
+    ctx.fixated_by_me = ctx.tb->common.is_thread_safe ? 0 : 1; /* TODO: something nicer */
+    ctx.last_pseudo_delete = (Uint) -1;
+    ctx.free_us = NULL;
 
     return match_traverse(
-            sd_context.p, sd_context.tb,
+            ctx.p, ctx.tb,
             pattern, NULL,
             chunk_size,
             MAX_SELECT_DELETE_ITERATIONS, NULL, 1,
-            mtraversal_select_delete_on_nothing_can_match,
-            mtraversal_select_delete_on_match_res,
-            mtraversal_select_delete_on_loop_ended,
-            mtraversal_select_delete_on_trap,
-            &sd_context, ret);
+            &ctx.base, ret);
 }
 
 /*
  * This is called when select_delete traps
  */
-static int db_select_delete_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret) {
-    mtraversal_select_delete_context_t sd_context;
+static int db_select_delete_continue_hash(Process* p, DbTable* tbl,
+                                          Eterm continuation, Eterm* ret)
+{
+    select_delete_context_t ctx;
     Eterm* tptr;
     Eterm tid;
     Binary* mp;
@@ -2050,28 +2080,27 @@ static int db_select_delete_continue_hash(Process* p, DbTable* tbl, Eterm contin
     Sint slot_ix;
     Sint chunk_size = 0;
 
-    if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) {
+    if (unpack_simple_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) {
         *ret = NIL;
         return DB_ERROR_BADPARAM;
     }
 
-    sd_context.p = p;
-    sd_context.tb = &tbl->hash;
-    sd_context.tid = tid;
-    sd_context.hp = NULL;
-    sd_context.prev_continuation_tptr = tptr;
-    sd_context.fixated_by_me = ONLY_WRITER(p, sd_context.tb) ? 0 : 1; /* TODO: something nicer */
-    sd_context.last_pseudo_delete = (Uint) -1;
-    sd_context.free_us = NULL;
+    ctx.base.on_match_res  = select_delete_on_match_res;
+    ctx.base.on_loop_ended = select_delete_on_loop_ended;
+    ctx.base.on_trap       = select_delete_on_trap;
+    ctx.p = p;
+    ctx.tb = &tbl->hash;
+    ctx.tid = tid;
+    ctx.prev_continuation_tptr = tptr;
+    ctx.fixated_by_me = ONLY_WRITER(p, ctx.tb) ? 0 : 1; /* TODO: something nicer */
+    ctx.last_pseudo_delete = (Uint) -1;
+    ctx.free_us = NULL;
 
     return match_traverse_continue(
-            sd_context.p, sd_context.tb, chunk_size,
+            ctx.p, ctx.tb, chunk_size,
             MAX_SELECT_DELETE_ITERATIONS,
             NULL, slot_ix, got, &mp, 1,
-            mtraversal_select_delete_on_match_res,  /* Reuse callback */
-            mtraversal_select_delete_on_loop_ended, /* Reuse callback */
-            mtraversal_select_delete_on_trap,       /* Reuse callback */
-            &sd_context, ret);
+            &ctx.base, ret);
 }
 
 #undef MAX_SELECT_DELETE_ITERATIONS
@@ -2086,24 +2115,26 @@ static int db_select_delete_continue_hash(Process* p, DbTable* tbl, Eterm contin
 #define MAX_SELECT_REPLACE_ITERATIONS 1000
 
 typedef struct {
+    match_callbacks_t base;
     Process* p;
     DbTableHash* tb;
     Eterm tid;
-    Eterm* hp;
     Eterm* prev_continuation_tptr;
-} mtraversal_select_replace_context_t;
+} select_replace_context_t;
 
-static int mtraversal_select_replace_on_nothing_can_match(void* context_ptr, Eterm* ret) {
+static int select_replace_on_nothing_can_match(match_callbacks_t* ctx_base,
+                                               Eterm* ret)
+{
     *ret = make_small(0);
     return DB_ERROR_NONE;
 }
 
-static int mtraversal_select_replace_on_match_res(void* context_ptr, Sint slot_ix,
-                                                  HashDbTerm*** current_ptr_ptr,
-                                                  Eterm match_res)
+static int select_replace_on_match_res(match_callbacks_t* ctx_base, Sint slot_ix,
+                                       HashDbTerm*** current_ptr_ptr,
+                                       Eterm match_res)
 {
-    mtraversal_select_replace_context_t* sr_context_ptr = (mtraversal_select_replace_context_t*) context_ptr;
-    DbTableHash* tb = sr_context_ptr->tb;
+    select_replace_context_t* ctx = (select_replace_context_t*) ctx_base;
+    DbTableHash* tb = ctx->tb;
     HashDbTerm* new;
     HashDbTerm* next;
     HashValue hval;
@@ -2128,35 +2159,37 @@ static int mtraversal_select_replace_on_match_res(void* context_ptr, Sint slot_i
     return 0;
 }
 
-static int mtraversal_select_replace_on_loop_ended(void* context_ptr, Sint slot_ix, Sint got,
-                                                   Sint iterations_left, Binary** mpp, Eterm* ret)
+static int select_replace_on_loop_ended(match_callbacks_t* ctx_base, Sint slot_ix,
+                                        Sint got, Sint iterations_left,
+                                        Binary** mpp, Eterm* ret)
 {
-    mtraversal_select_replace_context_t* sr_context_ptr = (mtraversal_select_replace_context_t*) context_ptr;
+    select_replace_context_t* ctx = (select_replace_context_t*) ctx_base;
     ASSERT(iterations_left <= MAX_SELECT_REPLACE_ITERATIONS);
     /* the more objects we've replaced, the more reductions we've consumed */
-    BUMP_REDS(sr_context_ptr->p,
+    BUMP_REDS(ctx->p,
               MIN(MAX_SELECT_REPLACE_ITERATIONS * 2,
                   (MAX_SELECT_REPLACE_ITERATIONS - iterations_left) + (int)got));
-    *ret = erts_make_integer(got, sr_context_ptr->p);
+    *ret = erts_make_integer(got, ctx->p);
     return DB_ERROR_NONE;
 }
 
-static int mtraversal_select_replace_on_trap(void* context_ptr, Sint slot_ix, Sint got,
-                                             Binary** mpp, Eterm* ret)
+static int select_replace_on_trap(match_callbacks_t* ctx_base,
+                                  Sint slot_ix, Sint got,
+                                  Binary** mpp, Eterm* ret)
 {
-    mtraversal_select_replace_context_t* sr_context_ptr = (mtraversal_select_replace_context_t*) context_ptr;
-    return on_mtraversal_simple_trap(
+    select_replace_context_t* ctx = (select_replace_context_t*) ctx_base;
+    return on_simple_trap(
             &ets_select_replace_continue_exp,
-            sr_context_ptr->p,
-            sr_context_ptr->tb,
-            sr_context_ptr->tid,
-            sr_context_ptr->prev_continuation_tptr,
+            ctx->p,
+            ctx->tb,
+            ctx->tid,
+            ctx->prev_continuation_tptr,
             slot_ix, got, mpp, ret);
 }
 
 static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pattern, Eterm *ret)
 {
-    mtraversal_select_replace_context_t sr_context = {0};
+    select_replace_context_t ctx;
     Sint chunk_size = 0;
 
     /* Bag implementation presented both semantic consistency and performance issues,
@@ -2164,22 +2197,21 @@ static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pat
      */
     ASSERT(!(tbl->hash.common.status & DB_BAG));
 
-    sr_context.p = p;
-    sr_context.tb = &tbl->hash;
-    sr_context.tid = tid;
-    sr_context.hp = NULL;
-    sr_context.prev_continuation_tptr = NULL;
+    ctx.base.on_nothing_can_match = select_replace_on_nothing_can_match;
+    ctx.base.on_match_res         = select_replace_on_match_res;
+    ctx.base.on_loop_ended        = select_replace_on_loop_ended;
+    ctx.base.on_trap              = select_replace_on_trap;
+    ctx.p = p;
+    ctx.tb = &tbl->hash;
+    ctx.tid = tid;
+    ctx.prev_continuation_tptr = NULL;
 
     return match_traverse(
-            sr_context.p, sr_context.tb,
+            ctx.p, ctx.tb,
             pattern, db_match_keeps_key,
             chunk_size,
             MAX_SELECT_REPLACE_ITERATIONS, NULL, 1,
-            mtraversal_select_replace_on_nothing_can_match,
-            mtraversal_select_replace_on_match_res,
-            mtraversal_select_replace_on_loop_ended,
-            mtraversal_select_replace_on_trap,
-            &sr_context, ret);
+            &ctx.base, ret);
 }
 
 /*
@@ -2187,7 +2219,7 @@ static int db_select_replace_hash(Process *p, DbTable *tbl, Eterm tid, Eterm pat
  */
 static int db_select_replace_continue_hash(Process* p, DbTable* tbl, Eterm continuation, Eterm* ret)
 {
-    mtraversal_select_replace_context_t sr_context = {0};
+    select_replace_context_t ctx;
     Eterm* tptr;
     Eterm tid ;
     Binary* mp;
@@ -2196,26 +2228,25 @@ static int db_select_replace_continue_hash(Process* p, DbTable* tbl, Eterm conti
     Sint chunk_size = 0;
     *ret = NIL;
 
-    if (unpack_simple_mtraversal_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) {
+    if (unpack_simple_continuation(continuation, &tptr, &tid, &slot_ix, &mp, &got)) {
         *ret = NIL;
         return DB_ERROR_BADPARAM;
     }
 
     /* Proceed */
-    sr_context.p = p;
-    sr_context.tb = &tbl->hash;
-    sr_context.tid = tid;
-    sr_context.hp = NULL;
-    sr_context.prev_continuation_tptr = tptr;
+    ctx.base.on_match_res  = select_replace_on_match_res;
+    ctx.base.on_loop_ended = select_replace_on_loop_ended;
+    ctx.base.on_trap       = select_replace_on_trap;
+    ctx.p = p;
+    ctx.tb = &tbl->hash;
+    ctx.tid = tid;
+    ctx.prev_continuation_tptr = tptr;
 
     return match_traverse_continue(
-            sr_context.p, sr_context.tb, chunk_size,
+            ctx.p, ctx.tb, chunk_size,
             MAX_SELECT_REPLACE_ITERATIONS,
             NULL, slot_ix, got, &mp, 1,
-            mtraversal_select_replace_on_match_res,  /* Reuse callback */
-            mtraversal_select_replace_on_loop_ended, /* Reuse callback */
-            mtraversal_select_replace_on_trap,       /* Reuse callback */
-            &sr_context, ret);
+            &ctx.base, ret);
 }
 
 
-- 
cgit v1.2.3


From 0657a616adc74bf71342053539c807031fa9d162 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 4 May 2018 20:13:29 +0200
Subject: erts: Optimize ets delete all in fixed table

by only allocating one FixedDeletion with the new "all" flag
instead of one FixedDeletion per slot.
---
 erts/emulator/beam/erl_db_hash.c | 152 ++++++++++++++++++++++-----------------
 erts/emulator/beam/erl_db_hash.h |   3 +-
 2 files changed, 88 insertions(+), 67 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 23074766e8..d55e110e10 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -149,20 +149,14 @@ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval)
     return ix;
 }
 
-/* Remember a slot containing a pseudo-deleted item
- * Return false if we got raced by unfixing thread
- * and the object should be deleted for real.
- */
-static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix,
-                                          erts_aint_t fixated_by_me)
+
+static ERTS_INLINE int link_fixdel(DbTableHash* tb,
+                                   FixedDeletion* fixd,
+                                   erts_aint_t fixated_by_me)
 {
     erts_aint_t was_next;
     erts_aint_t exp_next;
-    FixedDeletion* fixd = (FixedDeletion*) erts_db_alloc(ERTS_ALC_T_DB_FIX_DEL,
-							 (DbTable *) tb,
-							 sizeof(FixedDeletion));
-    ERTS_ETS_MISC_MEM_ADD(sizeof(FixedDeletion));
-    fixd->slot = ix;
+
     was_next = erts_atomic_read_acqb(&tb->fixdel);
     do { /* Lockless atomic insertion in linked list: */
         if (NFIXED(tb) <= fixated_by_me) {
@@ -179,6 +173,21 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix,
     return 1;
 }
 
+/* Remember a slot containing a pseudo-deleted item
+ * Return false if we got raced by unfixing thread
+ * and the object should be deleted for real.
+ */
+static int add_fixed_deletion(DbTableHash* tb, int ix,
+                              erts_aint_t fixated_by_me)
+{
+    FixedDeletion* fixd = (FixedDeletion*) erts_db_alloc(ERTS_ALC_T_DB_FIX_DEL,
+							 (DbTable *) tb,
+							 sizeof(FixedDeletion));
+    ERTS_ETS_MISC_MEM_ADD(sizeof(FixedDeletion));
+    fixd->slot = ix;
+    fixd->all = 0;
+    return link_fixdel(tb, fixd, fixated_by_me);
+}
 
 
 static ERTS_INLINE int is_pseudo_deleted(HashDbTerm* p)
@@ -579,54 +588,61 @@ SWord db_unfix_table_hash(DbTableHash *tb)
     SWord work = 0;
 
     ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock)
-		       || (erts_lc_rwmtx_is_rlocked(&tb->common.rwlock)
-			   && !tb->common.is_thread_safe));
+                   || (erts_lc_rwmtx_is_rlocked(&tb->common.rwlock)
+                       && !tb->common.is_thread_safe));
 restart:
     fixdel = (FixedDeletion*) erts_atomic_xchg_mb(&tb->fixdel,
-                                                      (erts_aint_t) NULL);
-    while (fixdel != NULL) {
-	FixedDeletion *fx = fixdel;
-	int ix = fx->slot;
-	HashDbTerm **bp;
-	HashDbTerm *b;
-        HashDbTerm *free_us = NULL;
-	erts_rwmtx_t* lck = WLOCK_HASH(tb,ix);
-
-	if (IS_FIXED(tb)) { /* interrupted by fixer */
-	    WUNLOCK_HASH(lck);
-	    restore_fixdel(tb,fixdel);
-	    if (!IS_FIXED(tb)) {
-		goto restart; /* unfixed again! */
-	    }
-	    return work;
-	}
-	if (ix < NACTIVE(tb)) {
-	    bp = &BUCKET(tb, ix);
-	    b = *bp;
-	    
-	    while (b != NULL) {
-		if (is_pseudo_deleted(b)) {
-                    HashDbTerm* nxt = b->next;
-                    b->next = free_us;
-                    free_us = b;
-		    work++;
-		    b = *bp = nxt;
-		} else {
-		    bp = &b->next;
-		    b = b->next;
-		}
-	    }
-	}
-	/* else slot has been joined and purged by shrink() */
-	WUNLOCK_HASH(lck);
-        free_term_list(tb, free_us);
-	fixdel = fx->next;
-	erts_db_free(ERTS_ALC_T_DB_FIX_DEL,
-		     (DbTable *) tb,
-		     (void *) fx,
-		     sizeof(FixedDeletion));
-	ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion));
-	work++;
+                                                  (erts_aint_t) NULL);
+    while (fixdel) {
+        FixedDeletion *free_me;
+
+        do {
+            HashDbTerm **bp;
+            HashDbTerm *b;
+            HashDbTerm *free_us = NULL;
+            erts_rwmtx_t* lck;
+
+            lck = WLOCK_HASH(tb, fixdel->slot);
+
+            if (IS_FIXED(tb)) { /* interrupted by fixer */
+                WUNLOCK_HASH(lck);
+                restore_fixdel(tb,fixdel);
+                if (!IS_FIXED(tb)) {
+                    goto restart; /* unfixed again! */
+                }
+                return work;
+            }
+            if (fixdel->slot < NACTIVE(tb)) {
+                bp = &BUCKET(tb, fixdel->slot);
+                b = *bp;
+
+                while (b != NULL) {
+                    if (is_pseudo_deleted(b)) {
+                        HashDbTerm* nxt = b->next;
+                        b->next = free_us;
+                        free_us = b;
+                        work++;
+                        b = *bp = nxt;
+                    } else {
+                        bp = &b->next;
+                        b = b->next;
+                    }
+                }
+            }
+            /* else slot has been joined and purged by shrink() */
+            WUNLOCK_HASH(lck);
+            free_term_list(tb, free_us);
+
+        }while (fixdel->all && fixdel->slot-- > 0);
+
+        free_me = fixdel;
+        fixdel = fixdel->next;
+        erts_db_free(ERTS_ALC_T_DB_FIX_DEL,
+                     (DbTable *) tb,
+                     (void *) free_me,
+                     sizeof(FixedDeletion));
+        ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion));
+        work++;
     }
 
     /* ToDo: Maybe try grow/shrink the table as well */
@@ -2306,20 +2322,24 @@ void db_initialize_hash(void)
 static SWord db_mark_all_deleted_hash(DbTable *tbl, SWord reds)
 {
     DbTableHash *tb = &tbl->hash;
-    HashDbTerm* list;
+    FixedDeletion* fixdel;
     int i;
 
     ERTS_LC_ASSERT(IS_TAB_WLOCKED(tb));
 
+    fixdel = erts_db_alloc(ERTS_ALC_T_DB_FIX_DEL,
+                           (DbTable *) tb,
+                           sizeof(FixedDeletion));
+    ERTS_ETS_MISC_MEM_ADD(sizeof(FixedDeletion));
+    fixdel->slot = NACTIVE(tb) - 1;
+    fixdel->all = 1;
+    link_fixdel(tb, fixdel, 0);
+
     for (i = 0; i < NACTIVE(tb); i++) {
-	if ((list = BUCKET(tb,i)) != NULL) {
-	    add_fixed_deletion(tb, i, 0);
-	    do {
-		list->pseudo_deleted = 1;
-		list = list->next;
-	    }while(list != NULL);
-            reds--;
-	}
+        HashDbTerm* b;
+	for (b = BUCKET(tb,i); b; b = b->next)
+            b->pseudo_deleted = 1;
+        --reds;
     }
     erts_atomic_set_nob(&tb->common.nitems, 0);
     return reds < 0 ? 0 : reds;   /* ToDo: Yield! */
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index 127512e1fe..d48343378d 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -24,7 +24,8 @@
 #include "erl_db_util.h" /* DbTerm & DbTableCommon */
 
 typedef struct fixed_deletion {
-    int slot;
+    UWord slot : sizeof(UWord)*8 - 1;
+    UWord all : 1;
     struct fixed_deletion *next;
 } FixedDeletion;
 
-- 
cgit v1.2.3


From c500b0acdad588fa4c3d0cba15a9c61dc30a3190 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 7 May 2018 18:15:56 +0200
Subject: erts: Make ets:delete_all_objects yield on fixed table

---
 erts/emulator/beam/erl_db_hash.c | 43 ++++++++++++++++++++++++++++++----------
 erts/emulator/beam/erl_db_hash.h |  3 ++-
 2 files changed, 34 insertions(+), 12 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index d55e110e10..6bb5034971 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -2321,28 +2321,50 @@ void db_initialize_hash(void)
 
 static SWord db_mark_all_deleted_hash(DbTable *tbl, SWord reds)
 {
+    const int LOOPS_PER_REDUCTION  = 8;
     DbTableHash *tb = &tbl->hash;
     FixedDeletion* fixdel;
+    SWord loops = reds * LOOPS_PER_REDUCTION;
     int i;
 
     ERTS_LC_ASSERT(IS_TAB_WLOCKED(tb));
 
-    fixdel = erts_db_alloc(ERTS_ALC_T_DB_FIX_DEL,
-                           (DbTable *) tb,
-                           sizeof(FixedDeletion));
-    ERTS_ETS_MISC_MEM_ADD(sizeof(FixedDeletion));
-    fixdel->slot = NACTIVE(tb) - 1;
-    fixdel->all = 1;
-    link_fixdel(tb, fixdel, 0);
+    fixdel = (FixedDeletion*) erts_atomic_read_nob(&tb->fixdel);
+    if (fixdel && fixdel->trap) {
+        /* Continue after trap */
+        ASSERT(fixdel->all);
+        ASSERT(fixdel->slot < NACTIVE(tb));
+        i = fixdel->slot;
+    }
+    else {
+        /* First call */
+        fixdel = erts_db_alloc(ERTS_ALC_T_DB_FIX_DEL,
+                               (DbTable *) tb,
+                               sizeof(FixedDeletion));
+        ERTS_ETS_MISC_MEM_ADD(sizeof(FixedDeletion));
+        link_fixdel(tb, fixdel, 0);
+        i = 0;
+    }
 
-    for (i = 0; i < NACTIVE(tb); i++) {
+    do {
         HashDbTerm* b;
 	for (b = BUCKET(tb,i); b; b = b->next)
             b->pseudo_deleted = 1;
-        --reds;
+    } while (++i < NACTIVE(tb) && --loops > 0);
+
+    if (i < NACTIVE(tb)) {
+         /* Yield */
+        fixdel->slot = i;
+        fixdel->all = 0;
+        fixdel->trap = 1;
+        return -1;
     }
+
+    fixdel->slot = NACTIVE(tb) - 1;
+    fixdel->all = 1;
+    fixdel->trap = 0;
     erts_atomic_set_nob(&tb->common.nitems, 0);
-    return reds < 0 ? 0 : reds;   /* ToDo: Yield! */
+    return loops < 0 ? 0 : loops / LOOPS_PER_REDUCTION;
 }
 
 
@@ -3145,7 +3167,6 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle)
 static SWord db_delete_all_objects_hash(Process* p, DbTable* tbl, SWord reds)
 {
     if (IS_FIXED(tbl)) {
-        /* ToDo: Yield! */
 	reds = db_mark_all_deleted_hash(tbl, reds);
     } else {
         reds = db_free_table_continue_hash(tbl, reds);
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index d48343378d..08e5b13db1 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -24,8 +24,9 @@
 #include "erl_db_util.h" /* DbTerm & DbTableCommon */
 
 typedef struct fixed_deletion {
-    UWord slot : sizeof(UWord)*8 - 1;
+    UWord slot : sizeof(UWord)*8 - 2;
     UWord all : 1;
+    UWord trap : 1;
     struct fixed_deletion *next;
 } FixedDeletion;
 
-- 
cgit v1.2.3


From e7579f608a1bdb52271c57e9ad69a7752ea1371b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 7 May 2018 18:18:17 +0200
Subject: erts: Rename untrapping db_free_*empty*_table

as it's now only used for empty tables by ets:new/2.
---
 erts/emulator/beam/erl_db.c      | 2 +-
 erts/emulator/beam/erl_db_hash.c | 8 ++++----
 erts/emulator/beam/erl_db_tree.c | 7 ++++---
 erts/emulator/beam/erl_db_util.h | 2 +-
 4 files changed, 10 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 696ad17daa..468e22ed59 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1782,7 +1782,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 
 	db_lock(tb,LCK_WRITE);
 	free_heir_data(tb);
-	tb->common.meth->db_free_table(tb);
+	tb->common.meth->db_free_empty_table(tb);
 	db_unlock(tb,LCK_WRITE);
         table_dec_refc(tb, 0);
 	BIF_ERROR(BIF_P, BADARG);
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 6bb5034971..74d63325e6 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -418,7 +418,7 @@ static void db_print_hash(fmtfn_t to,
 			  void *to_arg,
 			  int show,
 			  DbTable *tbl);
-static int db_free_table_hash(DbTable *tbl);
+static int db_free_empty_table_hash(DbTable *tbl);
 
 static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds);
 
@@ -531,7 +531,7 @@ DbTableMethod db_hash =
     db_select_replace_continue_hash,
     db_take_hash,
     db_delete_all_objects_hash,
-    db_free_table_hash,
+    db_free_empty_table_hash,
     db_free_table_continue_hash,
     db_print_hash,
     db_foreach_offheap_hash,
@@ -2426,9 +2426,9 @@ static void db_print_hash(fmtfn_t to, void *to_arg, int show, DbTable *tbl)
     }
 }
 
-/* release all memory occupied by a single table */
-static int db_free_table_hash(DbTable *tbl)
+static int db_free_empty_table_hash(DbTable *tbl)
 {
+    ASSERT(NITEMS(tbl) == 0);
     while (db_free_table_continue_hash(tbl, ERTS_SWORD_MAX) < 0)
 	;
     return 0;
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 4dcb41523f..0692583dd4 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -417,7 +417,7 @@ static int db_select_replace_continue_tree(Process *p, DbTable *tbl,
 static int db_take_tree(Process *, DbTable *, Eterm, Eterm *);
 static void db_print_tree(fmtfn_t to, void *to_arg,
 			  int show, DbTable *tbl);
-static int db_free_table_tree(DbTable *tbl);
+static int db_free_empty_table_tree(DbTable *tbl);
 
 static SWord db_free_table_continue_tree(DbTable *tbl, SWord);
 
@@ -470,7 +470,7 @@ DbTableMethod db_tree =
     db_select_replace_continue_tree,
     db_take_tree,
     db_delete_all_objects_tree,
-    db_free_table_tree,
+    db_free_empty_table_tree,
     db_free_table_continue_tree,
     db_print_tree,
     db_foreach_offheap_tree,
@@ -1969,8 +1969,9 @@ static void db_print_tree(fmtfn_t to, void *to_arg,
 }
 
 /* release all memory occupied by a single table */
-static int db_free_table_tree(DbTable *tbl)
+static int db_free_empty_table_tree(DbTable *tbl)
 {
+    ASSERT(tbl->tree.root == NULL);
     while (db_free_table_continue_tree(tbl, ERTS_SWORD_MAX) < 0)
 	;
     return 1;
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 56689bcc03..73d242449e 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -183,7 +183,7 @@ typedef struct db_table_method
 
     SWord (*db_delete_all_objects)(Process* p, DbTable* db, SWord reds);
 
-    int (*db_free_table)(DbTable* db /* [in out] */ );
+    int (*db_free_empty_table)(DbTable* db);
     SWord (*db_free_table_continue)(DbTable* db, SWord reds);
     
     void (*db_print)(fmtfn_t to,
-- 
cgit v1.2.3


From 13bcae1294760339f232f1e06083d168a4e8e3eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Wed, 9 May 2018 10:49:16 +0200
Subject: Disable the use of floating point exceptions

Floating point exceptions has been disabled since 2011 on macOS
(fa0f8d2c29b) and on Linux since 2014 (c7ddafbe6dbc) because
there were unresolved stability issues.

Floating point exceptions are not disabled by default on FreeBSD,
and if OTP is compiled with gcc (as opposed to clang) floating
point exceptions will be used.

81a6adab693a introduced a bug in erts/emulator/Makefile.in which would
cause the building of OTP to fail if floating point exceptions were
enabled. The bug was not noticed because it turns out that none of our
daily build machines has floating point exceptions enabled.

Since floating point exceptions is not tested, we should not expect
them to work reliably on any platform. Therefore, turn off floating
point exceptions unconditionally in erts/configure.in. For the moment
we will keep the code in the runtime system that handles floating
point exceptions.

(This commit also fixes the bug in erts/emulator/Makefile.in, in
case floating point exceptions ever become reliable and enabled.)

https://bugs.erlang.org/browse/ERL-620
---
 erts/emulator/Makefile.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 5dfa60ee74..221cf84622 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -570,7 +570,7 @@ $(TTF_DIR)/OPCODES-GENERATED: $(OPCODE_TABLES) utils/beam_makeops
 		-code-model @CODE_MODEL@ \
 		-outdir $(TTF_DIR) \
 		-DUSE_VM_PROBES=$(if $(USE_VM_PROBES),1,0) \
-		-DNO_FPE_SIGNALS=$(if $filter(unreliable,$(FPE)),1,0) \
+		-DNO_FPE_SIGNALS=$(if $(filter unreliable,$(FPE)),1,0) \
 		-emulator $(OPCODE_TABLES) && echo $? >$(TTF_DIR)/OPCODES-GENERATED
 GENERATE += $(TTF_DIR)/OPCODES-GENERATED
 
-- 
cgit v1.2.3


From eedfe1a5e3a85ff5ec7ab49d491250f3b501f3ba Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 8 May 2018 17:20:07 +0200
Subject: erts: Try fix contant maps:next order

Symptom: maps:iterator+next returns different orders
for the exact same map.

Problem: Number of cached key-values within iterator term
depends on number of input reductions to erts_internal_map_next_3.

Solution: Build cached key-values in destructive non-reverse order
to not affect iteration order.
---
 erts/emulator/beam/erl_map.c | 32 +++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 3d565b1bb8..05e8fc11a2 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -3058,7 +3058,7 @@ BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
         Uint path_length = 0;
         Uint *path_rest = NULL;
         int i, elems, orig_elems;
-        Eterm node = map, res, *path_ptr = NULL, *hp;
+        Eterm node = map, res, *patch_ptr = NULL, *hp;
 
         /* A stack WSTACK is used when traversing the hashmap.
          * It contains: node, idx, sz, ptr
@@ -3117,13 +3117,21 @@ BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
         }
 
         if (type == iterator) {
-            /* iterator uses the format {K, V, {K, V, {K, V, [Path | Map]}}},
-             * so each element is 4 words large */
+            /*
+             * Iterator uses the format {K1, V1, {K2, V2, {K3, V3, [Path | Map]}}},
+             * so each element is 4 words large.
+             * To make iteration order independent of input reductions
+             * the KV-pairs are here built in DESTRUCTIVE non-reverse order.
+             */
             hp = HAlloc(BIF_P, 4 * elems);
-            res = am_none;
         } else {
-            /* list used the format [Path, Map, {K,V}, {K,V} | BIF_ARG_3],
-             * so each element is 2+3 words large */
+            /*
+             * List used the format [Path, Map, {K3,V3}, {K2,V2}, {K1,V1} | BIF_ARG_3],
+             * so each element is 2+3 words large.
+             * To make list order independent of input reductions
+             * the KV-pairs are here built in FUNCTIONAL reverse order
+             * as this is how the list as a whole is constructed.
+             */
             hp = HAlloc(BIF_P, (2 + 3) * elems);
             res = BIF_ARG_3;
         }
@@ -3149,9 +3157,9 @@ BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
             if (is_list(ptr[PATH_ELEM(curr_path)])) {
                 Eterm *lst = list_val(ptr[PATH_ELEM(curr_path)]);
                 if (type == iterator) {
-                    res = TUPLE3(hp, CAR(lst), CDR(lst), res); hp += 4;
-                    /* Note where we should patch the Iterator is needed */
-                    path_ptr = hp-1;
+                    res = TUPLE3(hp, CAR(lst), CDR(lst), make_tuple(hp+4));
+                    hp += 4;
+                    patch_ptr = hp-1;
                 } else {
                     Eterm tup = TUPLE2(hp, CAR(lst), CDR(lst)); hp += 3;
                     res = CONS(hp, tup, res); hp += 2;
@@ -3188,7 +3196,8 @@ BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
             while (idx < sz && elems != 0 && is_list(ptr[idx])) {
                 Eterm *lst = list_val(ptr[idx]);
                 if (type == iterator) {
-                    res = TUPLE3(hp, CAR(lst), CDR(lst), res); hp += 4;
+                    (void) TUPLE3(hp, CAR(lst), CDR(lst), make_tuple(hp+4)); hp += 4;
+                    patch_ptr = hp-1;
                 } else {
                     Eterm tup = TUPLE2(hp, CAR(lst), CDR(lst)); hp += 3;
                     res = CONS(hp, tup, res); hp += 2;
@@ -3286,7 +3295,7 @@ BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
 
             if (type == iterator) {
                 hp = HAlloc(BIF_P, 2);
-                *path_ptr = CONS(hp, path, map); hp += 2;
+                *patch_ptr = CONS(hp, path, map); hp += 2;
             } else {
                 hp = HAlloc(BIF_P, 4);
                 res = CONS(hp, map, res); hp += 2;
@@ -3294,6 +3303,7 @@ BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
             }
         } else {
             if (type == iterator) {
+                *patch_ptr = am_none;
                 HRelease(BIF_P, hp + 4 * elems, hp);
             } else {
                 HRelease(BIF_P, hp + (2+3) * elems, hp);
-- 
cgit v1.2.3


From 09b16bc5e833af56c52e1fa9156f13eacf35f26b Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 15 May 2018 10:21:02 +0200
Subject: erts: Fix erts_os_times warning

---
 erts/emulator/sys/unix/erl_unix_sys.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index e367d565a7..10adf80875 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -264,7 +264,7 @@ erts_os_monotonic_time(void)
 ERTS_GLB_INLINE void
 erts_os_times(ErtsMonotonicTime *mtimep, ErtsSystemTime *stimep)
 {
-    return (*erts_sys_time_data__.r.o.os_times)(mtimep, stimep);
+    (*erts_sys_time_data__.r.o.os_times)(mtimep, stimep);
 }
 
 #endif /* ERTS_OS_TIMES_INLINE_FUNC_PTR_CALL__ */
-- 
cgit v1.2.3


From fa4fcc2bb54b6e60a9b9379707a5d30403b011ac Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Wed, 9 May 2018 19:43:43 +0200
Subject: erts: Fix unused variable warning in unix prim file

---
 erts/emulator/nifs/unix/unix_prim_file.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/nifs/unix/unix_prim_file.c b/erts/emulator/nifs/unix/unix_prim_file.c
index 1637f9cb71..2b112dda76 100644
--- a/erts/emulator/nifs/unix/unix_prim_file.c
+++ b/erts/emulator/nifs/unix/unix_prim_file.c
@@ -512,8 +512,8 @@ int efile_sync(efile_data_t *d, int data_only) {
 }
 
 int efile_advise(efile_data_t *d, Sint64 offset, Sint64 length, enum efile_advise_t advise) {
-    efile_unix_t *u = (efile_unix_t*)d;
 #ifdef HAVE_POSIX_FADVISE
+    efile_unix_t *u = (efile_unix_t*)d;
     int p_advise;
 
     switch(advise) {
-- 
cgit v1.2.3


From d5dc5584884488c8fcb5710997a14c2c418692f3 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 15 May 2018 16:32:26 +0200
Subject: erts: Make cpu_timestamp use per thread on Linux

If we don't use per thread the value becomes completely
nonsensical on systems with more than one scheduler.
We keep the old solaris behaviour in order to support the
option, but it only really works when using a single
scheduler.
---
 erts/emulator/beam/erl_bif_trace.c    |  4 ++--
 erts/emulator/beam/erl_time_sup.c     |  2 +-
 erts/emulator/beam/erl_trace.c        | 14 --------------
 erts/emulator/sys/unix/erl_unix_sys.h | 10 ++++++----
 4 files changed, 9 insertions(+), 21 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 1953f79d79..fd3400ab71 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -643,12 +643,12 @@ Eterm erts_internal_trace_3(BIF_ALIST_3)
 			    SysTimespec tp;
 			    int i;
 			    
-			    if (sys_get_proc_cputime(start, tp) < 0)
+			    if (sys_get_cputime(start, tp) < 0)
 				goto error;
 			    start = ((SysCpuTime)tp.tv_sec * 1000000000LL) + 
 				    (SysCpuTime)tp.tv_nsec;
 			    for (i = 0; i < 100; i++)
-				sys_get_proc_cputime(stop, tp);
+				sys_get_cputime(stop, tp);
 			    stop = ((SysCpuTime)tp.tv_sec * 1000000000LL) + 
 				   (SysCpuTime)tp.tv_nsec;
 			    if (start == 0) goto error;
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 4f91d9ad07..29c698e34f 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1860,7 +1860,7 @@ void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) {
   SysCpuTime t;
   SysTimespec tp;
 
-  sys_get_proc_cputime(t, tp);
+  sys_get_cputime(t, tp);
   *microsec = (Uint)(tp.tv_nsec / 1000);
   t = (tp.tv_sec / 1000000);
   *megasec = (Uint)(t % 1000000);
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 065a560b52..e9a413904b 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -625,20 +625,6 @@ erts_get_system_profile(void) {
     return profile;
 }
 
-
-#ifdef HAVE_ERTS_NOW_CPU
-#  define GET_NOW(m, s, u) \
-do { \
-    if (erts_cpu_timestamp) \
-	erts_get_now_cpu(m, s, u); \
-    else \
-	get_now(m, s, u); \
-} while (0)
-#else
-#  define GET_NOW(m, s, u) do {get_now(m, s, u);} while (0)
-#endif
-
-
 static void
 write_sys_msg_to_port(Eterm unused_to,
 		      Port* trace_port,
diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index 10adf80875..5bfe5a8e2d 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -292,6 +292,8 @@ erts_sys_perf_counter()
 
 /*
  * Functions for measuring CPU time
+ *
+ * Note that gethrvtime is time per process and clock_gettime is per thread.
  */
 
 #if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME_CPU_TIME))
@@ -300,15 +302,15 @@ typedef struct timespec SysTimespec;
 
 #if defined(HAVE_GETHRVTIME)
 #define sys_gethrvtime() gethrvtime()
-#define sys_get_proc_cputime(t,tp) (t) = sys_gethrvtime(), \
-                                   (tp).tv_sec = (time_t)((t)/1000000000LL), \
-                                   (tp).tv_nsec = (long)((t)%1000000000LL)
+#define sys_get_cputime(t,tp) (t) = sys_gethrvtime(), \
+        (tp).tv_sec = (time_t)((t)/1000000000LL),     \
+        (tp).tv_nsec = (long)((t)%1000000000LL)
 int sys_start_hrvtime(void);
 int sys_stop_hrvtime(void);
 
 #elif defined(HAVE_CLOCK_GETTIME_CPU_TIME)
 #define sys_clock_gettime(cid,tp) clock_gettime((cid),&(tp))
-#define sys_get_proc_cputime(t,tp) sys_clock_gettime(CLOCK_PROCESS_CPUTIME_ID,(tp))
+#define sys_get_cputime(t,tp) sys_clock_gettime(CLOCK_THREAD_CPUTIME_ID,(tp))
 
 #endif
 #endif
-- 
cgit v1.2.3


From 0985a72500ecd46579f6f7462ff121cc2c015f1f Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 15 May 2018 17:46:07 +0200
Subject: Do not hold runq lock while deleting a process

---
 erts/emulator/beam/erl_process.c      | 15 ++++++++++++++-
 erts/emulator/beam/erl_process_lock.h | 18 ++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index ad7ac27ac3..70c50a6911 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9610,6 +9610,17 @@ scheduler_gc_proc(Process *c_p, int reds_left)
     return reds;
 }
 
+static void
+unlock_lock_rq(int pre_free, void *vrq)
+{
+    ErtsRunQueue *rq = vrq;
+    if (pre_free)
+        erts_runq_unlock(rq);
+    else
+        erts_runq_lock(rq);
+}
+
+
 /*
  * schedule() is called from BEAM (process_main()) or HiPE
  * (hipe_mode_switch()) when the current process is to be
@@ -9782,7 +9793,9 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
             }
 
             if (dec_refc)
-                erts_proc_dec_refc(p);
+                erts_proc_dec_refc_free_func(p,
+                                             unlock_lock_rq,
+                                             (void *) rq);
         }
 
 	ASSERT(!esdp->free_process);
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 43f396c547..bd38eca4dc 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -921,6 +921,9 @@ ERTS_GLB_INLINE int erts_proc_trylock(Process *, ErtsProcLocks);
 
 ERTS_GLB_INLINE void erts_proc_inc_refc(Process *);
 ERTS_GLB_INLINE void erts_proc_dec_refc(Process *);
+ERTS_GLB_INLINE void erts_proc_dec_refc_free_func(Process *p,
+                                                  void (*func)(int, void *),
+                                                  void *arg);
 ERTS_GLB_INLINE void erts_proc_add_refc(Process *, Sint);
 ERTS_GLB_INLINE Sint erts_proc_read_refc(Process *);
 
@@ -993,6 +996,21 @@ ERTS_GLB_INLINE void erts_proc_dec_refc(Process *p)
     }
 }
 
+ERTS_GLB_INLINE void erts_proc_dec_refc_free_func(Process *p,
+                                                  void (*func)(int, void *),
+                                                  void *arg)
+{
+    Sint referred;
+    ASSERT(!(erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
+    referred = erts_ptab_atmc_dec_test_refc(&p->common);
+    if (!referred) {
+	ASSERT(ERTS_PROC_IS_EXITING(p));
+        (*func)(!0, arg);
+	erts_free_proc(p);
+        (*func)(0, arg);
+    }
+}
+
 ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc)
 {
     Sint referred;
-- 
cgit v1.2.3


From a5a8672522b585e093cf2017827004171c72b295 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 15 May 2018 21:04:25 +0200
Subject: erts: Refactor new erts_make_dirty_proc_handled

from ensure_dirty_proc_handled
as an outline function callable from inline functions.
---
 erts/emulator/beam/erl_proc_sig_queue.c | 59 ++++++++++++++++++---------------
 erts/emulator/beam/erl_proc_sig_queue.h |  3 ++
 2 files changed, 35 insertions(+), 27 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index 5165cd22a5..17fdc98183 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -519,41 +519,42 @@ erts_aint32_t erts_enqueue_signals(Process *rp, ErtsMessage *first,
     return enqueue_signals(rp, first, last, last_next, num_msgs, in_state);
 }
 
-static ERTS_INLINE void
-ensure_dirty_proc_handled(Eterm pid,
-                          erts_aint32_t state,
-                          erts_aint32_t prio)
+void
+erts_make_dirty_proc_handled(Eterm pid,
+                             erts_aint32_t state,
+                             erts_aint32_t prio)
 {
-    if (state & (ERTS_PSFLG_DIRTY_RUNNING
-                 | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
-        Eterm *hp;
-        ErtsMessage *mp;
-        Process *sig_handler;
+    Eterm *hp;
+    ErtsMessage *mp;
+    Process *sig_handler;
 
-        if (prio < 0)
-            prio = (int) ERTS_PSFLGS_GET_USR_PRIO(state);
+    ASSERT(state & (ERTS_PSFLG_DIRTY_RUNNING |
+                    ERTS_PSFLG_DIRTY_RUNNING_SYS));
 
-        switch (prio) {
-        case PRIORITY_MAX:
-            sig_handler = erts_dirty_process_signal_handler_max;
-            break;
-        case PRIORITY_HIGH:
-            sig_handler = erts_dirty_process_signal_handler_high;
-            break;
-        default:
-            sig_handler = erts_dirty_process_signal_handler;
-            break;
-        }
+    if (prio < 0)
+        prio = (int) ERTS_PSFLGS_GET_USR_PRIO(state);
 
-        /* Make sure signals are handled... */
-        mp = erts_alloc_message(0, &hp);
-        erts_queue_message(sig_handler, 0, mp, pid, am_system);
+    switch (prio) {
+    case PRIORITY_MAX:
+        sig_handler = erts_dirty_process_signal_handler_max;
+        break;
+    case PRIORITY_HIGH:
+        sig_handler = erts_dirty_process_signal_handler_high;
+        break;
+    default:
+        sig_handler = erts_dirty_process_signal_handler;
+        break;
     }
+
+    /* Make sure signals are handled... */
+    mp = erts_alloc_message(0, &hp);
+    erts_queue_message(sig_handler, 0, mp, pid, am_system);
 }
 
 static void
 check_push_msgq_len_offs_marker(Process *rp, ErtsSignal *sig);
 
+
 static int
 proc_queue_signal(Process *c_p, Eterm pid, ErtsSignal *sig, int op)
 {
@@ -686,7 +687,8 @@ first_last_done:
         state = erts_proc_sys_schedule(rp, state, 0);
     }
 
-    ensure_dirty_proc_handled(rp->common.id, state, -1);
+    if (state & (ERTS_PSFLG_DIRTY_RUNNING | ERTS_PSFLG_DIRTY_RUNNING_SYS))
+        erts_make_dirty_proc_handled(rp->common.id, state, -1);
 
     if (!is_normal_sched)
         erts_proc_dec_refc(rp);
@@ -742,7 +744,10 @@ maybe_elevate_sig_handling_prio(Process *c_p, Eterm other)
             if (res) {
                 /* ensure handled if dirty executing... */
                 state = erts_atomic32_read_nob(&rp->state);
-                ensure_dirty_proc_handled(other, state, my_prio);
+                if (state & (ERTS_PSFLG_DIRTY_RUNNING
+                             | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
+                    erts_make_dirty_proc_handled(other, state, my_prio);
+                }
             }
         }
     }
diff --git a/erts/emulator/beam/erl_proc_sig_queue.h b/erts/emulator/beam/erl_proc_sig_queue.h
index 8b7cd35f61..3ea6418283 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.h
+++ b/erts/emulator/beam/erl_proc_sig_queue.h
@@ -744,6 +744,9 @@ erts_enqueue_signals(Process *rp, ErtsMessage *first,
 void
 erts_proc_sig_send_pending(ErtsSchedulerData* esdp);
 
+void erts_make_dirty_proc_handled(Eterm pid, erts_aint32_t state,
+                                  erts_aint32_t prio);
+
 
 typedef struct {
     Uint size;
-- 
cgit v1.2.3


From 827b96ae2d657905ada2ec49de1f0b863cf83401 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 15 May 2018 20:58:28 +0200
Subject: erts: Fix bug when scheduling monitor-msg combo

Bug introduced in master by
613cde66c25464121f2f6dace99782bad0e07d9b
---
 erts/emulator/beam/erl_message.c        |  5 ++++-
 erts/emulator/beam/erl_proc_sig_queue.c | 10 +---------
 erts/emulator/beam/erl_proc_sig_queue.h | 22 ++++++++++++++++++++++
 3 files changed, 27 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index bea7a0fe86..507cc989d2 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -379,7 +379,10 @@ queue_messages(Process* receiver,
 	erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ);
     }
 
-    erts_proc_notify_new_message(receiver, receiver_locks);
+    if (last == &first->next)
+        erts_proc_notify_new_message(receiver, receiver_locks);
+    else
+        erts_proc_notify_new_sig(receiver, state, ERTS_PSFLG_ACTIVE);
 }
 
 static ERTS_INLINE
diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index 17fdc98183..6d81044c39 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -680,15 +680,7 @@ first_last_done:
         sig_enqueue_trace_cleanup(first, sig, last);
     }
 
-    if (!(state & (ERTS_PSFLG_EXITING
-                   | ERTS_PSFLG_ACTIVE_SYS
-                   | ERTS_PSFLG_SIG_IN_Q))) {
-        /* Schedule process... */
-        state = erts_proc_sys_schedule(rp, state, 0);
-    }
-
-    if (state & (ERTS_PSFLG_DIRTY_RUNNING | ERTS_PSFLG_DIRTY_RUNNING_SYS))
-        erts_make_dirty_proc_handled(rp->common.id, state, -1);
+    erts_proc_notify_new_sig(rp, state, 0);
 
     if (!is_normal_sched)
         erts_proc_dec_refc(rp);
diff --git a/erts/emulator/beam/erl_proc_sig_queue.h b/erts/emulator/beam/erl_proc_sig_queue.h
index 3ea6418283..8edd277309 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.h
+++ b/erts/emulator/beam/erl_proc_sig_queue.h
@@ -744,6 +744,10 @@ erts_enqueue_signals(Process *rp, ErtsMessage *first,
 void
 erts_proc_sig_send_pending(ErtsSchedulerData* esdp);
 
+/* SVERK Doc me up! */
+ERTS_GLB_INLINE void erts_proc_notify_new_sig(Process* rp, erts_aint32_t state,
+                                              erts_aint32_t enable_flag);
+
 void erts_make_dirty_proc_handled(Eterm pid, erts_aint32_t state,
                                   erts_aint32_t prio);
 
@@ -882,6 +886,24 @@ erts_proc_sig_fetch(Process *proc)
     return res;
 }
 
+ERTS_GLB_INLINE void
+erts_proc_notify_new_sig(Process* rp, erts_aint32_t state,
+                         erts_aint32_t enable_flag)
+{
+    if (~(state & (ERTS_PSFLG_EXITING
+                   | ERTS_PSFLG_ACTIVE_SYS
+                   | ERTS_PSFLG_SIG_IN_Q))
+        | (~state & enable_flag)) {
+        /* Schedule process... */
+        state = erts_proc_sys_schedule(rp, state, enable_flag);
+    }
+
+    if (state & (ERTS_PSFLG_DIRTY_RUNNING
+                 | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
+        erts_make_dirty_proc_handled(rp->common.id, state, -1);
+    }
+}
+
 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 #endif /* ERTS_PROC_SIG_QUEUE_H__ */
-- 
cgit v1.2.3


From 08873ec673ef223a4e0c9826b0370d80bb6aad24 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 12 Apr 2018 16:55:58 +0200
Subject: Teach HiPE to yield from receive

---
 erts/emulator/beam/erl_gc.c           | 15 +++++++--------
 erts/emulator/beam/erl_process.c      |  2 --
 erts/emulator/beam/erl_process.h      | 11 +++++------
 erts/emulator/hipe/hipe_mode_switch.c | 13 +++++++++----
 erts/emulator/hipe/hipe_native_bif.c  | 23 +++++++++--------------
 erts/emulator/hipe/hipe_process.h     |  9 ---------
 6 files changed, 30 insertions(+), 43 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 0692cea0ee..a65dbbf42b 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -413,21 +413,20 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end,
 {
     int cost;
 
-    if (p->flags & F_HIBERNATE_SCHED) {
+    if (p->flags & (F_HIBERNATE_SCHED|F_HIPE_RECV_LOCKED)) {
 	/*
 	 * We just hibernated. We do *not* want to mess
 	 * up the hibernation by an ordinary GC...
+         *
+         * OR
+         *
+         * We left a receive in HiPE with message
+         * queue lock locked, and we do not want to
+         * do a GC with message queue locked...
 	 */
 	return result;
     }
 
-#ifdef HIPE
-    if (p->hipe_smp.have_receive_locks) {
-        /* Do not want to GC with message queue locked... */
-        return result;
-    }    
-#endif
-
     if (!p->mbuf) {
 	/* Must have GC:d in BIF call... invalidate live_hf_end */
 	live_hf_end = ERTS_INVALID_HFRAG_PTR;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index ad7ac27ac3..8637ab4f68 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11844,7 +11844,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 
 #ifdef HIPE
     hipe_init_process(&p->hipe);
-    hipe_init_process_smp(&p->hipe_smp);
 #endif
     p->heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*sz);
     p->old_hend = p->old_htop = p->old_heap = NULL;
@@ -12169,7 +12168,6 @@ void erts_init_empty_process(Process *p)
 
 #ifdef HIPE
     hipe_init_process(&p->hipe);
-    hipe_init_process_smp(&p->hipe_smp);
 #endif
 
     INIT_HOLE_CHECK(p);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index b66272194c..1e9f9b07b7 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1061,9 +1061,6 @@ struct process {
     Eterm suspendee;
     ErtsPendingSuspend *pending_suspenders;
     erts_atomic_t run_queue;
-#ifdef HIPE
-    struct hipe_process_state_smp hipe_smp;
-#endif
 
 #ifdef CHECK_FOR_HOLES
     Eterm* last_htop;		/* No need to scan the heap below this point. */
@@ -1397,10 +1394,12 @@ extern int erts_system_profile_ts_type;
 #define F_DIRTY_MAJOR_GC     (1 << 23) /* Dirty major GC scheduled */
 #define F_DIRTY_MINOR_GC     (1 << 24) /* Dirty minor GC scheduled */
 #define F_HIBERNATED         (1 << 25) /* Hibernated */
-#define F_LOCAL_SIGS_ONLY    (1 << 26)
+#define F_LOCAL_SIGS_ONLY    (1 << 26) /* Handle privq sigs only */
 #define F_TRAP_EXIT          (1 << 27) /* Trapping exit */
-#define F_DEFERRED_SAVED_LAST (1 << 28)
-#define F_DELAYED_PSIGQS_LEN (1 << 29)
+#define F_DEFERRED_SAVED_LAST (1 << 28) /* Deferred sig_qs.saved_last */
+#define F_DELAYED_PSIGQS_LEN (1 << 29) /* Delayed update of sig_qs.len */
+#define F_HIPE_RECV_LOCKED   (1 << 30) /* HiPE message queue locked */
+#define F_HIPE_RECV_YIELD    (1 << 31) /* HiPE receive yield */
 
 /*
  * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index bc9a700204..0a65e317ed 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -490,16 +490,21 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
 	  /* same semantics, different debug trace messages */
 	  /* XXX: BEAM has different entries for the locked and unlocked
 	     cases. HiPE doesn't, so we must check dynamically. */
-	  if (p->hipe_smp.have_receive_locks)
-	      p->hipe_smp.have_receive_locks = 0;
+	  if (p->flags & F_HIPE_RECV_LOCKED)
+	      p->flags &= ~F_HIPE_RECV_LOCKED;
 	  else
 	      erts_proc_lock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	  p->i = hipe_beam_pc_resume;
 	  p->arity = 0;
           if (erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_EXITING)
               ASSERT(erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_ACTIVE);
-          else
+          else if (!(p->flags & F_HIPE_RECV_YIELD))
               erts_atomic32_read_band_relb(&p->state, ~ERTS_PSFLG_ACTIVE);
+          else {
+              /* Yielded from receive */
+              ERTS_VBUMP_ALL_REDS(p);
+              p->flags &= ~F_HIPE_RECV_YIELD;
+          }
 	  erts_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
       do_schedule:
 	  {
@@ -522,7 +527,7 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
 	      p = erts_schedule(NULL, p, reds_in - p->fcalls);
 	      ERTS_REQ_PROC_MAIN_LOCK(p);
 	      ASSERT(!(p->flags & F_HIPE_MODE));
-	      p->hipe_smp.have_receive_locks = 0;
+	      p->flags &= ~F_HIPE_RECV_LOCKED;
 	      reg = p->scheduler_data->x_reg_array;
 	  }
 	  {
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index 24078af046..211ce0492a 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -144,8 +144,8 @@ BIF_RETTYPE nbif_impl_hipe_set_timeout(NBIF_ALIST_1)
     else {
 	int tres = erts_set_proc_timer_term(p, timeout_value);
 	if (tres != 0) { /* Wrong time */
-	    if (p->hipe_smp.have_receive_locks) {
-		p->hipe_smp.have_receive_locks = 0;
+	    if (p->flags & F_HIPE_RECV_LOCKED) {
+                p->flags &= ~F_HIPE_RECV_LOCKED;
 		erts_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	    }
 	    BIF_ERROR(p, EXC_TIMEOUT_VALUE);
@@ -549,19 +549,14 @@ Eterm hipe_check_get_msg(Process *c_p)
         c_p->i = NULL;
         c_p->arity = 0;
         c_p->current = NULL;
-        (void) erts_proc_sig_receive_helper(c_p, CONTEXT_REDS, 0,
+        (void) erts_proc_sig_receive_helper(c_p, CONTEXT_REDS/4, 0,
                                             &msgp, &get_out);
         /* FIXME: Need to bump reductions... */
         if (!msgp) {
             if (get_out) {
-                if (get_out < 0) {
-                    /*
-                     * FIXME: We should get out yielding
-                     * here...
-                     */
-                    goto next_message;
-                }
-                /* Go exit... */
+                if (get_out < 0)
+                    c_p->flags |= F_HIPE_RECV_YIELD; /* yield... */
+                /* else: go exit... */
                 return THE_NON_VALUE;
             }
 
@@ -573,7 +568,7 @@ Eterm hipe_check_get_msg(Process *c_p)
              */
 
             /* XXX: BEAM doesn't need this */
-            c_p->hipe_smp.have_receive_locks = 1;
+            c_p->flags |= F_HIPE_RECV_LOCKED;
             c_p->flags &= ~F_DELAY_GC;
             return THE_NON_VALUE;
         }
@@ -618,8 +613,8 @@ void hipe_clear_timeout(Process *c_p)
      */
     /* XXX: BEAM has different entries for the locked and unlocked
        cases. HiPE doesn't, so we must check dynamically. */
-    if (c_p->hipe_smp.have_receive_locks) {
-	c_p->hipe_smp.have_receive_locks = 0;
+    if (c_p->flags & F_HIPE_RECV_LOCKED) {
+	c_p->flags &= ~F_HIPE_RECV_LOCKED;
 	erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
     }
     if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) {
diff --git a/erts/emulator/hipe/hipe_process.h b/erts/emulator/hipe/hipe_process.h
index ef14c75f6c..18354ba0a6 100644
--- a/erts/emulator/hipe/hipe_process.h
+++ b/erts/emulator/hipe/hipe_process.h
@@ -82,13 +82,4 @@ static __inline__ void hipe_delete_process(struct hipe_process_state *p)
 	erts_free(ERTS_ALC_T_HIPE_STK, (void*)p->nstack);
 }
 
-struct hipe_process_state_smp {
-    int have_receive_locks;
-};
-
-static __inline__ void hipe_init_process_smp(struct hipe_process_state_smp *p)
-{
-    p->have_receive_locks = 0;
-}
-
 #endif /* HIPE_PROCESS_H */
-- 
cgit v1.2.3


From 78f639da759cd3f8b5f28bc86ec404f3c3db60f4 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 13 Apr 2018 14:33:38 +0200
Subject: New process suspend implementation based on async signaling

---
 erts/emulator/beam/atom.names           |   3 +
 erts/emulator/beam/beam_bif_load.c      |  32 +-
 erts/emulator/beam/beam_emu.c           |   3 +
 erts/emulator/beam/bif.c                | 181 +++++--
 erts/emulator/beam/bif.tab              |   6 +-
 erts/emulator/beam/dist.c               |   4 +-
 erts/emulator/beam/erl_alloc.types      |   2 +
 erts/emulator/beam/erl_bif_info.c       | 201 ++++----
 erts/emulator/beam/erl_bif_trace.c      | 190 +++++--
 erts/emulator/beam/erl_monitor_link.c   |  97 ++--
 erts/emulator/beam/erl_monitor_link.h   |  57 ++-
 erts/emulator/beam/erl_proc_sig_queue.c | 469 ++++++++++++++++-
 erts/emulator/beam/erl_proc_sig_queue.h | 116 +++++
 erts/emulator/beam/erl_process.c        | 863 ++++++--------------------------
 erts/emulator/beam/erl_process.h        |  46 +-
 erts/emulator/beam/erl_trace.c          |  32 ++
 erts/emulator/beam/erl_trace.h          |   2 +
 erts/emulator/test/trace_SUITE.erl      |  62 +--
 erts/emulator/test/tracer_SUITE.erl     |   6 +-
 19 files changed, 1275 insertions(+), 1097 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index fba0611042..45b7540aeb 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -142,6 +142,7 @@ atom bsr
 atom bsr_anycrlf
 atom bsr_unicode
 atom build_type
+atom busy
 atom busy_dist_port
 atom busy_port
 atom call
@@ -252,6 +253,7 @@ atom exception_from
 atom exception_trace
 atom exclusive
 atom exit_status
+atom exited
 atom existing
 atom existing_processes
 atom existing_ports
@@ -445,6 +447,7 @@ atom no_float
 atom no_integer
 atom no_network
 atom no_start_optimize
+atom not_suspended
 atom not
 atom not_a_list
 atom not_loaded
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index d9312f4df8..a0dbd9ec7b 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -603,8 +603,9 @@ badarg:
 
 BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2)
 {
+    erts_aint32_t state;
     Process *rp;
-    int reds = 0;
+    int dirty, busy, reds = 0;
     Eterm res;
 
     if (BIF_P != erts_dirty_process_signal_handler
@@ -618,20 +619,29 @@ BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2)
     if (is_not_atom(BIF_ARG_2))
 	BIF_ERROR(BIF_P, BADARG);
 
-    rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN,
-				   BIF_ARG_1, ERTS_PROC_LOCK_MAIN);
-    if (rp == ERTS_PROC_LOCK_BUSY)
-	ERTS_BIF_YIELD2(bif_export[BIF_erts_internal_check_dirty_process_code_2],
-			BIF_P, BIF_ARG_1, BIF_ARG_2);
+    if (BIF_ARG_1 == BIF_P->common.id)
+        BIF_RET(am_normal);
+
+    rp = erts_proc_lookup_raw(BIF_ARG_1);
     if (!rp)
-	BIF_RET(am_false);
-	
+        BIF_RET(am_false);
+
+    state = erts_atomic32_read_nob(&rp->state);
+    dirty = (state & (ERTS_PSFLG_DIRTY_RUNNING
+                      | ERTS_PSFLG_DIRTY_RUNNING_SYS));
+    if (!dirty)
+        BIF_RET(am_normal);
+
+    busy = erts_proc_trylock(rp, ERTS_PROC_LOCK_MAIN) == EBUSY;
+
+    if (busy)
+        BIF_RET(am_busy);
+
     res = erts_check_process_code(rp, BIF_ARG_2, &reds, BIF_P->fcalls);
 
-    if (BIF_P != rp)
-	erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
+    erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
 
-    ASSERT(is_value(res));
+    ASSERT(res == am_true || res == am_false);
 
     BIF_RET2(res, reds);
 }
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index ee287243a4..ab5920a67e 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1166,6 +1166,9 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 	    reds_used = treds > INT_MAX ? INT_MAX : (int) treds;
 	}
 
+        if (c_p && ERTS_PROC_GET_PENDING_SUSPEND(c_p))
+            erts_proc_sig_handle_pending_suspend(c_p);
+
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 79244b8544..97e1ee1286 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1364,13 +1364,14 @@ BIF_RETTYPE exit_signal_2(BIF_ALIST_2)
 
 
 /* Handle flags common to both process_flag_2 and process_flag_3. */
-static BIF_RETTYPE process_flag_aux(Process *BIF_P,
-				    Process *rp,
-				    Eterm flag,
-				    Eterm val)
+static Eterm process_flag_aux(Process *c_p, int *redsp, Eterm flag, Eterm val)
 {
    Eterm old_value = NIL;	/* shut up warning about use before set */
    Sint i;
+
+   if (redsp)
+       *redsp = 1;
+
    if (flag == am_save_calls) {
        struct saved_calls *scb;
        if (!is_small(val))
@@ -1390,30 +1391,89 @@ static BIF_RETTYPE process_flag_aux(Process *BIF_P,
        }
 
 #ifdef HIPE
-       if (rp->flags & F_HIPE_MODE) {
-	   ASSERT(!ERTS_PROC_GET_SAVED_CALLS_BUF(rp));
-	   scb = ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(rp, scb);
+       if (c_p->flags & F_HIPE_MODE) {
+	   ASSERT(!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p));
+	   scb = ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(c_p, scb);
        }
        else
 #endif
        {
 #ifdef HIPE
-	   ASSERT(!ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(rp));
+	   ASSERT(!ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(c_p));
 #endif
-	   scb = ERTS_PROC_SET_SAVED_CALLS_BUF(rp, scb);
-	   if (rp == BIF_P && ((scb && i == 0) || (!scb && i != 0))) {
-	       /* Adjust fcalls to match save calls setting... */
-	       if (i == 0)
-		   BIF_P->fcalls += CONTEXT_REDS; /* disabled it */
-	       else
-		   BIF_P->fcalls -= CONTEXT_REDS; /* enabled it */
-
-	       /*
-		* Make sure we reschedule immediately so the
-		* change take effect at once.
-		*/
-	       ERTS_VBUMP_ALL_REDS(BIF_P);
-	   }
+	   scb = ERTS_PROC_SET_SAVED_CALLS_BUF(c_p, scb);
+
+	   if (((scb && i == 0) || (!scb && i != 0))) {
+
+               /*
+                * Make sure we reschedule immediately so the
+                * change take effect at once.
+                */
+               if (!redsp) {
+                   /* Executed via BIF call.. */
+               via_bif:
+
+                   /* Adjust fcalls to match save calls setting... */
+                   if (i == 0)
+                       c_p->fcalls += CONTEXT_REDS; /* disabled it */
+                   else
+                       c_p->fcalls -= CONTEXT_REDS; /* enabled it */
+
+                   ERTS_VBUMP_ALL_REDS(c_p);
+               }
+               else {
+                   erts_aint32_t state;
+                   /*
+                    * Executed via signal handler. Try to figure
+                    * out in what context we are executing...
+                    */
+
+                   state = erts_atomic32_read_nob(&c_p->state);
+                   if (state & (ERTS_PSFLG_RUNNING_SYS
+                                | ERTS_PSFLG_DIRTY_RUNNING_SYS
+                                | ERTS_PSFLG_DIRTY_RUNNING)) {
+                       /*
+                        * We are either processing signals before
+                        * being executed or executing dirty. That
+                        * is, no need to adjust anything...
+                        */
+                       *redsp = 1;
+                   }
+                   else {
+                       ErtsSchedulerData *esdp;
+                       ASSERT(state & ERTS_PSFLG_RUNNING);
+
+                       /*
+                        * F_DELAY_GC is currently only set when
+                        * we handle signals in state running via
+                        * receive helper...
+                        */
+
+                       if (!(c_p->flags & F_DELAY_GC)) {
+                           *redsp = 1;
+                           goto via_bif;
+                       }
+
+                       /*
+                        * Executing via receive helper...
+                        *
+                        * We utilize the virtual reds counter
+                        * in order to get correct calculation
+                        * of reductions consumed when scheduling
+                        * out the process...
+                        */
+
+                       esdp = erts_get_scheduler_data();
+
+                       if (i == 0)
+                           esdp->virtual_reds += CONTEXT_REDS; /* disabled it */
+                       else
+                           esdp->virtual_reds -= CONTEXT_REDS; /* enabled it */
+
+                       *redsp = -1;
+                   }
+               }
+           }
        }
 
        if (!scb)
@@ -1423,11 +1483,12 @@ static BIF_RETTYPE process_flag_aux(Process *BIF_P,
 	   erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb);
        }
 
-       BIF_RET(old_value);
+       ASSERT(is_immed(old_value));
+       return old_value;
    }
 
  error:
-   BIF_ERROR(BIF_P, BADARG);
+   return am_badarg;
 }
 
 BIF_RETTYPE process_flag_2(BIF_ALIST_2)
@@ -1596,29 +1657,73 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
        /* Fall through and try process_flag_aux() ... */
    }
 
-   BIF_RET(process_flag_aux(BIF_P, BIF_P, BIF_ARG_1, BIF_ARG_2));
+   old_value = process_flag_aux(BIF_P, NULL, BIF_ARG_1, BIF_ARG_2);
+   if (old_value != am_badarg)
+       BIF_RET(old_value);
  error:
    BIF_ERROR(BIF_P, BADARG);
 }
 
-BIF_RETTYPE process_flag_3(BIF_ALIST_3)
+typedef struct {
+    Eterm flag;
+    Eterm value;
+    ErlOffHeap oh;
+    Eterm heap[1];
+} ErtsProcessFlag3Args;
+
+static Eterm
+exec_process_flag_3(Process *c_p, void *arg, int *redsp, ErlHeapFragment **bpp)
 {
-   Process *rp;
-   Eterm res;
+    ErtsProcessFlag3Args *pf3a = arg;
+    Eterm res;
+
+    if (ERTS_PROC_IS_EXITING(c_p))
+        res = am_badarg;
+    else
+        res = process_flag_aux(c_p, redsp, pf3a->flag, pf3a->value);
+    erts_cleanup_offheap(&pf3a->oh);
+    erts_free(ERTS_ALC_T_PF3_ARGS, arg);
+    return res;
+}
+
+
+BIF_RETTYPE erts_internal_process_flag_3(BIF_ALIST_3)
+{
+   Eterm res, *hp;
+   ErlOffHeap *ohp;
+   ErtsProcessFlag3Args *pf3a;
+   Uint flag_sz, value_sz;
+
+   if (BIF_P->common.id == BIF_ARG_1) {
+       res = process_flag_aux(BIF_P, NULL, BIF_ARG_2, BIF_ARG_3);
+       BIF_RET(res);
+   }
+
+   if (is_not_internal_pid(BIF_ARG_1))
+       BIF_RET(am_badarg);
+
+   flag_sz = is_immed(BIF_ARG_2) ? 0 : size_object(BIF_ARG_2);
+   value_sz = is_immed(BIF_ARG_3) ? 0 : size_object(BIF_ARG_3);
+
+   pf3a = erts_alloc(ERTS_ALC_T_PF3_ARGS,
+                     sizeof(ErtsProcessFlag3Args)
+                     + sizeof(Eterm)*(flag_sz+value_sz-1));
+
+   ohp = &pf3a->oh;
+   ERTS_INIT_OFF_HEAP(&pf3a->oh);
 
-   rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN,
-				  BIF_ARG_1, ERTS_PROC_LOCK_MAIN);
-   if (rp == ERTS_PROC_LOCK_BUSY)
-       ERTS_BIF_YIELD3(bif_export[BIF_process_flag_3], BIF_P,
-		       BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+   hp = &pf3a->heap[0];
 
-   if (!rp)
-       BIF_ERROR(BIF_P, BADARG);
+   pf3a->flag = copy_struct(BIF_ARG_2, flag_sz, &hp, ohp);
+   pf3a->value = copy_struct(BIF_ARG_3, value_sz, &hp, ohp);
 
-   res = process_flag_aux(BIF_P, rp, BIF_ARG_2, BIF_ARG_3);
+   res = erts_proc_sig_send_rpc_request(BIF_P, BIF_ARG_1,
+                                        !0,
+                                        exec_process_flag_3,
+                                        (void *) pf3a);
 
-   if (rp != BIF_P)
-       erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
+   if (is_non_value(res))
+       BIF_RET(am_badarg);
 
    return res;
 }
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 33738dc20b..7387ca2523 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -125,7 +125,7 @@ bif erlang:pid_to_list/1
 bif erlang:ports/0
 bif erlang:pre_loaded/0
 bif erlang:process_flag/2
-bif erlang:process_flag/3
+bif erts_internal:process_flag/3
 bif erlang:process_info/1
 bif erlang:process_info/2
 bif erlang:processes/0
@@ -204,9 +204,9 @@ bif erlang:seq_trace/2
 bif erlang:seq_trace_info/1
 bif erlang:seq_trace_print/1
 bif erlang:seq_trace_print/2
-bif erlang:suspend_process/2
+bif erts_internal:suspend_process/2
 bif erlang:resume_process/1
-bif erlang:process_display/2
+bif erts_internal:process_display/2
 
 bif erlang:bump_reductions/1
 
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 026f0a62d4..f1566cb7b4 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -3229,8 +3229,8 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
         }
         else {
             proc_unlock = ERTS_PROC_LOCK_MAIN;
-            proc = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN,
-                                             BIF_ARG_2, proc_unlock);
+            proc = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
+                                 BIF_ARG_2, proc_unlock);
         }
         erts_de_rwlock(dep);
 
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 4a6a19b210..1a29e3d817 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -287,6 +287,7 @@ type	DIST_DEMONITOR	SHORT_LIVED	PROCESSES	dist_demonitor
 type	CML_CLEANUP	SHORT_LIVED	SYSTEM		connection_ml_cleanup
 type	ML_YIELD_STATE	SHORT_LIVED	SYSTEM		monitor_link_yield_state
 type	ML_DIST		STANDARD	SYSTEM		monitor_link_dist
+type	PF3_ARGS	SHORT_LIVED	PROCESSES	process_flag_3_arguments
 
 type	ENVIRONMENT	SYSTEM		SYSTEM		environment
 
@@ -346,6 +347,7 @@ type	NIF_TRAP_EXPORT	STANDARD	PROCESSES	nif_trap_export_entry
 type	NIF_EXP_TRACE	FIXED_SIZE	PROCESSES	nif_export_trace
 type	EXPORT		LONG_LIVED	CODE		export_entry
 type	MONITOR		FIXED_SIZE	PROCESSES	monitor
+type	MONITOR_SUSPEND	STANDARD	PROCESSES	monitor_suspend
 type	LINK		FIXED_SIZE	PROCESSES	link
 type	AINFO_REQ	SHORT_LIVED	SYSTEM		alloc_info_request
 type	SCHED_WTIME_REQ	SHORT_LIVED	SYSTEM		sched_wall_time_request
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 6f9e507228..8f04ff7353 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -617,17 +617,13 @@ static void collect_one_target_monitor(ErtsMonitor *mon, void *vmicp)
 }
 
 typedef struct {
-    Process *c_p;
-    ErtsProcLocks c_p_locks;
     ErtsMonitorSuspend **smi;
     Uint smi_i;
     Uint smi_max;
-    int sz;
+    Uint sz;
 } ErtsSuspendMonitorInfoCollection;
 
-#define ERTS_INIT_SUSPEND_MONITOR_INFOS(SMIC, CP, CPL) do {		\
-    (SMIC).c_p = (CP);							\
-    (SMIC).c_p_locks = (CPL);						\
+#define ERTS_INIT_SUSPEND_MONITOR_INFOS(SMIC) do {		        \
     (SMIC).smi = NULL;							\
     (SMIC).smi_i = (SMIC).smi_max = 0;					\
     (SMIC).sz = 0;                               			\
@@ -660,34 +656,26 @@ do {									\
 static void
 collect_one_suspend_monitor(ErtsMonitor *mon, void *vsmicp)
 {
-    ErtsMonitorSuspend *smon = erts_monitor_suspend(mon);
-    ErtsSuspendMonitorInfoCollection *smicp = vsmicp;
-    Process *suspendee = erts_pid2proc(smicp->c_p,
-				       smicp->c_p_locks,
-				       mon->other.item,
-				       0);
-    if (suspendee) { /* suspendee is alive */
-	Sint a, p;
-	if (smon->active) {
-	    smon->active += smon->pending;
-	    smon->pending = 0;
-	}
+    if (mon->type == ERTS_MON_TYPE_SUSPEND) {
+        Sint count;
+        erts_aint_t mstate;
+        ErtsMonitorSuspend *msp;
+        ErtsSuspendMonitorInfoCollection *smicp;
 
-	ASSERT((smon->active && !smon->pending)
-	       || (smon->pending && !smon->active));
+        msp = (ErtsMonitorSuspend *) erts_monitor_to_data(mon);
+        smicp = vsmicp;
 
 	ERTS_EXTEND_SUSPEND_MONITOR_INFOS(smicp);
 
-	smicp->smi[smicp->smi_i] = smon;
+	smicp->smi[smicp->smi_i] = msp;
 	smicp->sz += 2 /* cons */ + 4 /* 3-tuple */;
 
-	a = (Sint) smon->active;	/* quiet compiler warnings */
-	p = (Sint) smon->pending;	/* on 64-bit machines      */
+        mstate = erts_atomic_read_nob(&msp->state);
 
-	if (!IS_SSMALL(a))
-	    smicp->sz += BIG_UINT_HEAP_SIZE;
-	if (!IS_SSMALL(p))
+        count = (Sint) (mstate & ERTS_MSUSPEND_STATE_COUNTER_MASK);
+	if (!IS_SSMALL(count))
 	    smicp->sz += BIG_UINT_HEAP_SIZE;
+
 	smicp->smi_i++;
     }
 }
@@ -1075,8 +1063,10 @@ process_info_bif(Process *c_p, Eterm pid, Eterm opt, int always_wrap, int pi2)
 
     if (c_p->common.id == pid) {
         int local_only = c_p->flags & F_LOCAL_SIGS_ONLY;
-        int sreds = ERTS_BIF_REDS_LEFT(c_p);
-        int sres;
+        int sres, sreds, reds_left;
+
+        reds_left = ERTS_BIF_REDS_LEFT(c_p);
+        sreds = reds_left;
 
         if (!local_only) {
             erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
@@ -1085,15 +1075,19 @@ process_info_bif(Process *c_p, Eterm pid, Eterm opt, int always_wrap, int pi2)
         }
 
         sres = erts_proc_sig_handle_incoming(c_p, &state, &sreds, sreds, !0);
+
+        BUMP_REDS(c_p, (int) sreds);
+        reds_left -= sreds;
+
         if (state & ERTS_PSFLG_EXITING) {
             c_p->flags &= ~F_LOCAL_SIGS_ONLY;
             goto exited;
         }
-        if (!sres) {
+        if (!sres | (reds_left <= 0)) {
             /*
-             * More signals to handle; need to yield and continue.
-             * Prevent fetching of more signals by setting
-             * local-sigs-only flag.
+             * More signals to handle or out of reds; need
+             * to yield and continue. Prevent fetching of
+             * more signals by setting local-sigs-only flag.
              */
             c_p->flags |= F_LOCAL_SIGS_ONLY;
             goto yield;
@@ -1627,56 +1621,56 @@ process_info_aux(Process *c_p,
     case ERTS_PI_IX_SUSPENDING: {
 	ErtsSuspendMonitorInfoCollection smic;
 	int i;
-	Eterm item;
 
-        erts_proc_lock(rp, ERTS_PROC_LOCK_STATUS);
+	ERTS_INIT_SUSPEND_MONITOR_INFOS(smic);
 
-	ERTS_INIT_SUSPEND_MONITOR_INFOS(smic,
-					c_p,
-					(c_p == rp
-					 ? ERTS_PROC_LOCK_MAIN
-					 : 0) | ERTS_PROC_LOCK_STATUS);
-
-	erts_monitor_tree_foreach(rp->suspend_monitors,
-                                  &collect_one_suspend_monitor,
-                                  &smic);
+        erts_monitor_tree_foreach(ERTS_P_MONITORS(rp),
+                                  collect_one_suspend_monitor,
+                                  (void *) &smic);
 
         reserve_size += smic.sz;
 
 	res = NIL;
 	for (i = 0; i < smic.smi_i; i++) {
-	    Sint a = (Sint) smic.smi[i]->active;  /* quiet compiler warnings */
-	    Sint p = (Sint) smic.smi[i]->pending; /* on 64-bit machines...   */
-	    Eterm active;
-	    Eterm pending;
+            ErtsMonitorSuspend *msp;
+            erts_aint_t mstate;
+	    Sint ci;
+            Eterm ct, active, pending, item;
             Uint sz = 4 + 2;
-            if (!IS_SSMALL(a))
-		sz += BIG_UINT_HEAP_SIZE;
-            if (!IS_SSMALL(p))
-		sz += BIG_UINT_HEAP_SIZE;
+
+            msp = smic.smi[i];
+            mstate = erts_atomic_read_nob(&msp->state);
+
+            ci = (Sint) (mstate & ERTS_MSUSPEND_STATE_COUNTER_MASK);
+            if (!IS_SSMALL(ci))
+                sz += BIG_UINT_HEAP_SIZE;
 
             ERTS_PI_UNRESERVE(reserve_size, sz);
             hp = erts_produce_heap(hfact, sz, reserve_size);
 
-	    if (IS_SSMALL(a))
-		active = make_small(a);
-	    else {
-		active = small_to_big(a, hp);
-		hp += BIG_UINT_HEAP_SIZE;
-	    }
-	    if (IS_SSMALL(p))
-		pending = make_small(p);
-	    else {
-		pending = small_to_big(p, hp);
-		hp += BIG_UINT_HEAP_SIZE;
-	    }
-	    item = TUPLE3(hp, smic.smi[i]->mon.other.item, active, pending);
+            if (IS_SSMALL(ci))
+                ct = make_small(ci);
+            else {
+                ct = small_to_big(ci, hp);
+                hp += BIG_UINT_HEAP_SIZE;
+            }
+
+            if (mstate & ERTS_MSUSPEND_STATE_FLG_ACTIVE) {
+                active = ct;
+                pending = make_small(0);
+            }
+            else {
+                active = make_small(0);
+                pending = ct;
+            }
+
+            ASSERT(is_internal_pid(msp->md.origin.other.item));
+
+	    item = TUPLE3(hp, msp->md.origin.other.item, active, pending);
 	    hp += 4;
 	    res = CONS(hp, item, res);
 	}
 
-        erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
-
         *reds += (Uint) smic.smi_i / 4;
 
 	ERTS_DESTROY_SUSPEND_MONITOR_INFOS(smic);
@@ -3637,26 +3631,46 @@ BIF_RETTYPE is_process_alive_1(BIF_ALIST_1)
    BIF_ERROR(BIF_P, BADARG);
 }
 
-BIF_RETTYPE process_display_2(BIF_ALIST_2)
+static Eterm
+process_display(Process *c_p, void *arg, int *redsp, ErlHeapFragment **bpp)
 {
-   Process *rp;
+    if (redsp)
+        *redsp = 1;
 
-   if (BIF_ARG_2 != am_backtrace)
-       BIF_ERROR(BIF_P, BADARG);
+    if (ERTS_PROC_IS_EXITING(c_p))
+        return am_badarg;
 
-   rp = erts_pid2proc_nropt(BIF_P, ERTS_PROC_LOCK_MAIN,
-			    BIF_ARG_1, ERTS_PROC_LOCKS_ALL);
-   if(!rp) {
-       BIF_ERROR(BIF_P, BADARG);
-   }
-   if (rp == ERTS_PROC_LOCK_BUSY)
-       ERTS_BIF_YIELD2(bif_export[BIF_process_display_2], BIF_P,
-		       BIF_ARG_1, BIF_ARG_2);
-   erts_stack_dump(ERTS_PRINT_STDERR, NULL, rp);
-   erts_proc_unlock(rp, (BIF_P == rp
-			     ? ERTS_PROC_LOCKS_ALL_MINOR
-			     : ERTS_PROC_LOCKS_ALL));
-   BIF_RET(am_true);
+    erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+    erts_stack_dump(ERTS_PRINT_STDERR, NULL, c_p);
+    erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
+
+    return am_true;
+}
+
+
+BIF_RETTYPE erts_internal_process_display_2(BIF_ALIST_2)
+{
+    Eterm res;
+
+    if (BIF_ARG_2 != am_backtrace)
+        BIF_RET(am_badarg);
+
+    if (BIF_P->common.id == BIF_ARG_1) {
+        res = process_display(BIF_P, NULL, NULL, NULL);
+        BIF_RET(res);
+    }
+
+    if (is_not_internal_pid(BIF_ARG_1))
+        BIF_RET(am_badarg);
+
+    res = erts_proc_sig_send_rpc_request(BIF_P, BIF_ARG_1,
+                                         !0,
+                                         process_display,
+                                         NULL);
+    if (is_non_value(res))
+        BIF_RET(am_badarg);
+
+    BIF_RET(res);
 }
 
 /* this is a general call which return some possibly useful information */
@@ -4597,27 +4611,6 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 		BIF_RET(am_true);
 	    }
 	}
-	else if (ERTS_IS_ATOM_STR("not_running_optimization", BIF_ARG_1)) {
-	    int old_use_opt, use_opt;
-	    switch (BIF_ARG_2) {
-	    case am_true:
-		use_opt = 1;
-		break;
-	    case am_false:
-		use_opt = 0;
-		break;
-	    default:
-		BIF_ERROR(BIF_P, BADARG);
-	    }
-
-	    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	    erts_thr_progress_block();
-	    old_use_opt = !erts_disable_proc_not_running_opt;
-	    erts_disable_proc_not_running_opt = !use_opt;
-	    erts_thr_progress_unblock();
-	    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	    BIF_RET(old_use_opt ? am_true : am_false);
-	}
 	else if (ERTS_IS_ATOM_STR("wait", BIF_ARG_1)) {
 	    if (ERTS_IS_ATOM_STR("deallocations", BIF_ARG_2)) {
 		int flag = ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS;
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 1953f79d79..f9d351e69e 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -809,9 +809,128 @@ Eterm trace_info_2(BIF_ALIST_2)
 	BIF_ERROR(p, BADARG);
     }
     erts_release_code_write_permission();
+
+    if (is_internal_ref(res))
+        BIF_TRAP1(erts_await_result, BIF_P, res);
+
     BIF_RET(res);
 }
 
+static Eterm
+build_trace_flags_term(Eterm **hpp, Uint *szp, Uint trace_flags)
+{
+
+#define ERTS_TFLAG__(F, FN)                             \
+    if (trace_flags & F) {                              \
+        if (szp)                                        \
+            sz += 2;                                    \
+        if (hp) {                                       \
+            res = CONS(hp, FN, res);                    \
+            hp += 2;                                    \
+        }                                               \
+    }
+
+    Eterm res;
+    Uint sz = 0;
+    Eterm *hp;
+
+    if (hpp) {
+        hp = *hpp;
+        res = NIL;
+    }
+    else {
+        hp = NULL;
+        res = THE_NON_VALUE;
+    }
+
+    ERTS_TFLAG__(F_NOW_TS, am_timestamp);
+    ERTS_TFLAG__(F_STRICT_MON_TS, am_strict_monotonic_timestamp);
+    ERTS_TFLAG__(F_MON_TS, am_monotonic_timestamp);
+    ERTS_TFLAG__(F_TRACE_SEND, am_send);
+    ERTS_TFLAG__(F_TRACE_RECEIVE, am_receive);
+    ERTS_TFLAG__(F_TRACE_SOS, am_set_on_spawn);
+    ERTS_TFLAG__(F_TRACE_CALLS, am_call);
+    ERTS_TFLAG__(F_TRACE_PROCS, am_procs);
+    ERTS_TFLAG__(F_TRACE_SOS1, am_set_on_first_spawn);
+    ERTS_TFLAG__(F_TRACE_SOL, am_set_on_link);
+    ERTS_TFLAG__(F_TRACE_SOL1, am_set_on_first_link);
+    ERTS_TFLAG__(F_TRACE_SCHED, am_running);
+    ERTS_TFLAG__(F_TRACE_SCHED_EXIT, am_exiting);
+    ERTS_TFLAG__(F_TRACE_GC, am_garbage_collection);
+    ERTS_TFLAG__(F_TRACE_ARITY_ONLY, am_arity);
+    ERTS_TFLAG__(F_TRACE_RETURN_TO, am_return_to);
+    ERTS_TFLAG__(F_TRACE_SILENT, am_silent);
+    ERTS_TFLAG__(F_TRACE_SCHED_NO, am_scheduler_id);
+    ERTS_TFLAG__(F_TRACE_PORTS, am_ports);
+    ERTS_TFLAG__(F_TRACE_SCHED_PORTS, am_running_ports);
+    ERTS_TFLAG__(F_TRACE_SCHED_PROCS, am_running_procs);
+
+    if (szp)
+        *szp += sz;
+
+    if (hpp)
+        *hpp = hp;
+
+    return res;
+
+#undef ERTS_TFLAG__
+}
+
+static Eterm
+trace_info_tracee(Process *c_p, void *arg, int *redsp, ErlHeapFragment **bpp)
+{
+    ErlHeapFragment *bp;
+    Eterm *hp, res, key;
+    Uint sz;
+
+    *redsp = 1;
+
+    if (ERTS_PROC_IS_EXITING(c_p))
+        return am_undefined;
+
+    key = (Eterm) arg;
+    sz = 3;
+
+    if (!ERTS_TRACER_IS_NIL(ERTS_TRACER(c_p)))
+        erts_is_tracer_proc_enabled(c_p, ERTS_PROC_LOCK_MAIN,
+                                    &c_p->common);
+
+    switch (key) {
+    case am_tracer:
+
+        erts_build_tracer_to_term(NULL, NULL, &sz, ERTS_TRACER(c_p));
+        bp = new_message_buffer(sz);
+        hp = bp->mem;
+        res = erts_build_tracer_to_term(&hp, &bp->off_heap,
+                                        NULL, ERTS_TRACER(c_p));
+        if (res == am_false)
+            res = NIL;
+        break;
+
+    case am_flags:
+
+        build_trace_flags_term(NULL, &sz, ERTS_TRACE_FLAGS(c_p));
+        bp = new_message_buffer(sz);
+        hp = bp->mem;
+        res = build_trace_flags_term(&hp, NULL, ERTS_TRACE_FLAGS(c_p));
+        break;
+
+    default:
+
+        ERTS_INTERNAL_ERROR("Key not supported");
+        res = NIL;
+        bp = NULL;
+        hp = NULL;
+        break;
+    }
+
+    *redsp += 2;
+
+    res = TUPLE2(hp, key, res);
+    *bpp = bp;
+    return res;
+}
+
 static Eterm
 trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
 {
@@ -846,24 +965,19 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
         erts_port_release(tracee);
 
     } else if (is_internal_pid(pid_spec)) {
-	Process *tracee = erts_pid2proc_not_running(p, ERTS_PROC_LOCK_MAIN,
-                                                    pid_spec, ERTS_PROC_LOCK_MAIN);
-
-        if (tracee == ERTS_PROC_LOCK_BUSY)
-            ERTS_BIF_YIELD2(bif_export[BIF_trace_info_2], p, pid_spec, key);
+        Eterm ref;
 
-	if (!tracee)
-	    return am_undefined;
+        if (key != am_flags && key != am_tracer)
+            goto error;
 
-        if (!ERTS_TRACER_IS_NIL(ERTS_TRACER(tracee)))
-            erts_is_tracer_proc_enabled(tracee, ERTS_PROC_LOCK_MAIN,
-                                        &tracee->common);
+        ref = erts_proc_sig_send_rpc_request(p, pid_spec, !0,
+                                             trace_info_tracee,
+                                             (void *) key);
 
-        tracer = erts_tracer_to_term(p, ERTS_TRACER(tracee));
-        trace_flags = ERTS_TRACE_FLAGS(tracee);
+        if (is_non_value(ref))
+            return am_undefined;
 
-	if (tracee != p)
-	    erts_proc_unlock(tracee, ERTS_PROC_LOCK_MAIN);
+        return ref;
     } else if (is_external_pid(pid_spec)
 	       && external_pid_dist_entry(pid_spec) == erts_this_dist_entry) {
 	    return am_undefined;
@@ -873,48 +987,16 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
     }
 
     if (key == am_flags) {
-	int num_flags = 21;	/* MAXIMUM number of flags. */
-	Uint needed = 3+2*num_flags;
-	Eterm flag_list = NIL;
-	Eterm* limit;
+	Eterm flag_list;
+        Uint sz = 3;
+        Eterm *hp;
 
-#define FLAG0(flag_mask,flag) \
-  if (trace_flags & (flag_mask)) { flag_list = CONS(hp, flag, flag_list); hp += 2; } else {}
+        build_trace_flags_term(NULL, &sz, trace_flags);
+
+        hp = HAlloc(p, sz);
+
+        flag_list = build_trace_flags_term(&hp, NULL, trace_flags);
 
-#if defined(DEBUG)
-    /*
-     * Check num_flags if this assertion fires.
-     */
-#  define FLAG ASSERT(num_flags-- > 0); FLAG0
-#else
-#  define FLAG FLAG0
-#endif
-        hp = HAlloc(p, needed);
-	limit = hp+needed;
-	FLAG(F_NOW_TS, am_timestamp);
-	FLAG(F_STRICT_MON_TS, am_strict_monotonic_timestamp);
-	FLAG(F_MON_TS, am_monotonic_timestamp);
-	FLAG(F_TRACE_SEND, am_send);
-	FLAG(F_TRACE_RECEIVE, am_receive);
-	FLAG(F_TRACE_SOS, am_set_on_spawn);
-	FLAG(F_TRACE_CALLS, am_call);
-	FLAG(F_TRACE_PROCS, am_procs);
-	FLAG(F_TRACE_SOS1, am_set_on_first_spawn);
-	FLAG(F_TRACE_SOL, am_set_on_link);
-	FLAG(F_TRACE_SOL1, am_set_on_first_link);
-	FLAG(F_TRACE_SCHED, am_running);
-	FLAG(F_TRACE_SCHED_EXIT, am_exiting);
-	FLAG(F_TRACE_GC, am_garbage_collection);
-	FLAG(F_TRACE_ARITY_ONLY, am_arity);
-	FLAG(F_TRACE_RETURN_TO, am_return_to);
-	FLAG(F_TRACE_SILENT, am_silent);
-	FLAG(F_TRACE_SCHED_NO, am_scheduler_id);
-	FLAG(F_TRACE_PORTS, am_ports);
-	FLAG(F_TRACE_SCHED_PORTS, am_running_ports);
-	FLAG(F_TRACE_SCHED_PROCS, am_running_procs);
-#undef FLAG0
-#undef FLAG
-	HRelease(p,limit,hp+3);
 	return TUPLE2(hp, key, flag_list);
     } else if (key == am_tracer) {
         if (tracer == am_false)
diff --git a/erts/emulator/beam/erl_monitor_link.c b/erts/emulator/beam/erl_monitor_link.c
index 70f36fb6b7..48d9bd4ca5 100644
--- a/erts/emulator/beam/erl_monitor_link.c
+++ b/erts/emulator/beam/erl_monitor_link.c
@@ -630,7 +630,9 @@ erts_monitor_tree_lookup_create(ErtsMonitor **root, int *created, Uint16 type,
     ErtsMonitor *res;
     ErtsMonitorCreateCtxt cctxt = {type, origin};
 
-    ERTS_ML_ASSERT(type == ERTS_MON_TYPE_NODE || type == ERTS_MON_TYPE_NODES);
+    ERTS_ML_ASSERT(type == ERTS_MON_TYPE_NODE
+                   || type == ERTS_MON_TYPE_NODES
+                   || type == ERTS_MON_TYPE_SUSPEND);
 
     res = (ErtsMonitor *) ml_rbt_lookup_create((ErtsMonLnkNode **) root,
                                                target, create_monitor,
@@ -760,11 +762,13 @@ erts_monitor_create(Uint16 type, Eterm ref, Eterm orgn, Eterm trgt, Eterm name)
     switch (type) {
     case ERTS_MON_TYPE_PROC:
     case ERTS_MON_TYPE_PORT:
-    case ERTS_MON_TYPE_TIME_OFFSET:
         if (is_nil(name)) {
             ErtsMonitorDataHeap *mdhp;
             ErtsORefThing *ortp;
 
+        case ERTS_MON_TYPE_TIME_OFFSET:
+
+            ERTS_ML_ASSERT(is_nil(name));
             ERTS_ML_ASSERT(is_immed(orgn) && is_immed(trgt));
             ERTS_ML_ASSERT(is_internal_ordinary_ref(ref));
         
@@ -860,10 +864,38 @@ erts_monitor_create(Uint16 type, Eterm ref, Eterm orgn, Eterm trgt, Eterm name)
         mdep->dist = NULL;
         break;
     }
-    case ERTS_MON_TYPE_SUSPEND:
-        ERTS_INTERNAL_ERROR("Use erts_monitor_suspend_create() instead...");
-        mdp = NULL;
+    case ERTS_MON_TYPE_SUSPEND: {
+        ErtsMonitorSuspend *msp;
+
+        ERTS_ML_ASSERT(is_nil(name));
+        ERTS_ML_ASSERT(is_nil(ref));
+        ERTS_ML_ASSERT(is_internal_pid(orgn) && is_internal_pid(trgt));
+
+        msp = erts_alloc(ERTS_ALC_T_MONITOR_SUSPEND,
+                         sizeof(ErtsMonitorSuspend));
+        mdp = &msp->md;
+        ERTS_ML_ASSERT(((void *) mdp) == ((void *) msp));
+
+        mdp->ref = NIL;
+
+        mdp->origin.other.item = trgt;
+        mdp->origin.offset = (Uint16) offsetof(ErtsMonitorData, origin);
+        mdp->origin.key_offset = (Uint16) offsetof(ErtsMonitor, other.item);
+        ERTS_ML_ASSERT(mdp->origin.key_offset >= mdp->origin.offset);
+        mdp->origin.flags = (Uint16) ERTS_ML_FLG_EXTENDED;
+        mdp->origin.type = type;
+
+        mdp->target.other.item = orgn;
+        mdp->target.offset = (Uint16) offsetof(ErtsMonitorData, target);
+        mdp->target.key_offset = (Uint16) offsetof(ErtsMonitor, other.item);
+        mdp->target.flags = ERTS_ML_FLG_TARGET|ERTS_ML_FLG_EXTENDED;
+        mdp->target.type = type;
+
+        msp->next = NULL;
+        erts_atomic_init_relb(&msp->state, 0);
+
         break;
+    }
     default:
         ERTS_INTERNAL_ERROR("Invalid monitor type");
         mdp = NULL;
@@ -887,10 +919,11 @@ erts_monitor_destroy__(ErtsMonitorData *mdp)
     ERTS_ML_ASSERT(!(mdp->target.flags & ERTS_ML_FLG_IN_TABLE));
     ERTS_ML_ASSERT((mdp->origin.flags & ERTS_ML_FLGS_SAME)
                    == (mdp->target.flags & ERTS_ML_FLGS_SAME));
-    ERTS_ML_ASSERT(mdp->origin.type != ERTS_MON_TYPE_SUSPEND);
 
     if (!(mdp->origin.flags & ERTS_ML_FLG_EXTENDED))
         erts_free(ERTS_ALC_T_MONITOR, mdp);
+    else if (mdp->origin.type == ERTS_MON_TYPE_SUSPEND)
+        erts_free(ERTS_ALC_T_MONITOR_SUSPEND, mdp);
     else {
         ErtsMonitorDataExtended *mdep = (ErtsMonitorDataExtended *) mdp;
         ErlOffHeap oh;
@@ -927,10 +960,10 @@ erts_monitor_size(ErtsMonitor *mon)
     Uint size, refc;
     ErtsMonitorData *mdp = erts_monitor_to_data(mon);
 
-    ERTS_ML_ASSERT(mon->type != ERTS_MON_TYPE_SUSPEND);
-
     if (!(mon->flags & ERTS_ML_FLG_EXTENDED))
         size = sizeof(ErtsMonitorDataHeap);
+    else if (mon->type == ERTS_MON_TYPE_SUSPEND)
+        size = sizeof(ErtsMonitorSuspend);
     else {
         ErtsMonitorDataExtended *mdep;
         Uint hsz = 0;
@@ -957,54 +990,6 @@ erts_monitor_size(ErtsMonitor *mon)
     return size / refc;
 }
 
-
-/* suspend monitors... */
-
-ErtsMonitorSuspend *
-erts_monitor_suspend_create(Eterm pid)
-{
-    ErtsMonitorSuspend *msp;
-
-    ERTS_ML_ASSERT(is_internal_pid(pid));
-
-    msp = erts_alloc(ERTS_ALC_T_SUSPEND_MON,
-                     sizeof(ErtsMonitorSuspend));
-    msp->mon.offset = (Uint16) offsetof(ErtsMonitorSuspend, mon);
-    msp->mon.key_offset = (Uint16) offsetof(ErtsMonitor, other.item);
-    msp->mon.other.item = pid;
-    msp->mon.flags = 0;
-    msp->mon.type = ERTS_MON_TYPE_SUSPEND;
-    msp->pending = 0;
-    msp->active = 0;
-    return msp;
-}
-
-static ErtsMonLnkNode *
-create_monitor_suspend(Eterm pid, void *unused)
-{
-    ErtsMonitorSuspend *msp = erts_monitor_suspend_create(pid);
-    return (ErtsMonLnkNode *) &msp->mon;
-}
-
-ErtsMonitorSuspend *
-erts_monitor_suspend_tree_lookup_create(ErtsMonitor **root, int *created,
-                                        Eterm pid)
-{
-    ErtsMonitor *mon;
-    mon = (ErtsMonitor *) ml_rbt_lookup_create((ErtsMonLnkNode **) root,
-                                               pid, create_monitor_suspend,
-                                               NULL,
-                                               created);
-    return erts_monitor_suspend(mon);
-}
-
-void
-erts_monitor_suspend_destroy(ErtsMonitorSuspend *msp)
-{
-    ERTS_ML_ASSERT(!(msp->mon.flags & ERTS_ML_FLG_IN_TABLE));
-    erts_free(ERTS_ALC_T_SUSPEND_MON, msp);
-}
-
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
  * Link Operations                                                           *
  *                                                                           *
diff --git a/erts/emulator/beam/erl_monitor_link.h b/erts/emulator/beam/erl_monitor_link.h
index 603aead8cc..9ff8aa509a 100644
--- a/erts/emulator/beam/erl_monitor_link.h
+++ b/erts/emulator/beam/erl_monitor_link.h
@@ -246,15 +246,28 @@
  *
  *              --- ERTS_MON_TYPE_SUSPEND -------------------------------------
  *
- *              Suspend monitor.
+ *              Suspend monitor. A local process (origin) suspends another
+ *              local process (target).
  *
- *              Other Item:         Suspendee process identifier
- *              Key:                Suspendee process identifier
- *
- *              Valid keys are only ordinary internal references.
+ *              Origin:
+ *                      Other Item:     Process identifier of suspendee
+ *                                      (target)
+ *                      Key:            Process identifier of suspendee
+ *                                      (target)
+ *              Target:
+ *                      Other Item:     Process identifier of suspender
+ *                                      (origin)
+ *                      Key:            Process identifier of suspender
+ *                                      (origin)
+ *              Shared:
+ *                      Next:           Pointer to another suspend monitor
+ *                      State:          Number of suspends and a flag
+ *                                      indicating if the suspend is
+ *                                      active or not.
  *
- *              This type of monitor is a bit strange and the whole process
- *              suspend functionality should be improved...
+ *              Origin part of the monitor is stored in the monitor tree of
+ *              origin process and target part of the monitor is stored in
+ *              monitor list for local targets on the target process.
  *
  *
  *
@@ -638,11 +651,15 @@ struct ErtsMonitorDataExtended__ {
     Eterm heap[1]; /* heap start... */
 };
 
-typedef struct {
-    ErtsMonitor mon;
-    int pending;
-    int active;
-} ErtsMonitorSuspend;
+typedef struct ErtsMonitorSuspend__ ErtsMonitorSuspend;
+
+struct ErtsMonitorSuspend__ {
+    ErtsMonitorData md; /* origin = suspender; target = suspendee */
+    ErtsMonitorSuspend *next;
+    erts_atomic_t state;
+};
+#define ERTS_MSUSPEND_STATE_FLG_ACTIVE ((erts_aint_t) (((Uint) 1) << (sizeof(Uint)*8 - 1)))
+#define ERTS_MSUSPEND_STATE_COUNTER_MASK (~ERTS_MSUSPEND_STATE_FLG_ACTIVE)
 
 /* 
  * --- Monitor tree operations ---
@@ -1094,24 +1111,25 @@ int erts_monitor_list_foreach_delete_yielding(ErtsMonitor **list,
  *
  * @brief Create a monitor
  *
- * Can create all types of monitors exept for suspend monitors
+ * Can create all types of monitors
  *
  * When the funcion is called it is assumed that:
  * - 'ref' is an internal ordinary reference if type is ERTS_MON_TYPE_PROC,
  *   ERTS_MON_TYPE_PORT, ERTS_MON_TYPE_TIME_OFFSET, or ERTS_MON_TYPE_RESOURCE
- * - 'ref' is NIL if type is ERTS_MON_TYPE_NODE or ERTS_MON_TYPE_NODES
+ * - 'ref' is NIL if type is ERTS_MON_TYPE_NODE, ERTS_MON_TYPE_NODES, or
+ *   ERTS_MON_TYPE_SUSPEND
  * - 'ref' is and ordinary internal reference or an external reference if
  *   type is ERTS_MON_TYPE_DIST_PROC
  * - 'name' is an atom or NIL if type is ERTS_MON_TYPE_PROC,
  *   ERTS_MON_TYPE_PORT, or ERTS_MON_TYPE_DIST_PROC
  * - 'name is NIL if type is ERTS_MON_TYPE_TIME_OFFSET, ERTS_MON_TYPE_RESOURCE,
- *   ERTS_MON_TYPE_NODE, or ERTS_MON_TYPE_NODES
+ *   ERTS_MON_TYPE_NODE, ERTS_MON_TYPE_NODES, or ERTS_MON_TYPE_SUSPEND
  * If the above is not true, bad things will happen.
  *
  * @param[in]     type          ERTS_MON_TYPE_PROC, ERTS_MON_TYPE_PORT,
  *                              ERTS_MON_TYPE_TIME_OFFSET, ERTS_MON_TYPE_DIST_PROC,
  *                              ERTS_MON_TYPE_RESOURCE, ERTS_MON_TYPE_NODE,
- *                              or ERTS_MON_TYPE_NODES
+ *                              ERTS_MON_TYPE_NODES, or ERTS_MON_TYPE_SUSPEND
  *
  * @param[in]     ref           A reference or NIL depending on type
  *
@@ -1119,6 +1137,10 @@ int erts_monitor_list_foreach_delete_yielding(ErtsMonitor **list,
  *
  * @param[in]     target        The key of the target
  *
+ * @param[in]     name          An atom (the name) or NIL depending on type
+ *
+ * @returns                     A pointer to monitor data structure
+ *
  */
 ErtsMonitorData *erts_monitor_create(Uint16 type, Eterm ref, Eterm origin,
                                      Eterm target, Eterm name);
@@ -1347,7 +1369,8 @@ erts_monitor_to_data(ErtsMonitor *mon)
     ERTS_ML_ASSERT(erts_monitor_origin_offset == (size_t) mdp->origin.offset);
     ERTS_ML_ASSERT(!!(mdp->target.flags & ERTS_ML_FLG_TARGET));
     ERTS_ML_ASSERT(erts_monitor_target_offset == (size_t) mdp->target.offset);
-    if (mon->type == ERTS_MON_TYPE_NODE || mon->type == ERTS_MON_TYPE_NODES) {
+    if (mon->type == ERTS_MON_TYPE_NODE || mon->type == ERTS_MON_TYPE_NODES
+        || mon->type == ERTS_MON_TYPE_SUSPEND) {
         ERTS_ML_ASSERT(erts_monitor_node_key_offset == (size_t) mdp->origin.key_offset);
         ERTS_ML_ASSERT(erts_monitor_node_key_offset == (size_t) mdp->target.key_offset);
     }
diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index 5165cd22a5..b4b087b082 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -39,6 +39,7 @@
 #include "big.h"
 #include "erl_gc.h"
 #include "bif.h"
+#include "erl_bif_unique.h"
 #include "erl_proc_sig_queue.h"
 #include "dtrace-wrapper.h"
 
@@ -49,7 +50,7 @@
  * Note that not all signal are handled using this functionality!
  */
 
-#define ERTS_SIG_Q_OP_MAX 11
+#define ERTS_SIG_Q_OP_MAX 13
 
 #define ERTS_SIG_Q_OP_EXIT                      0
 #define ERTS_SIG_Q_OP_EXIT_LINKED               1
@@ -62,7 +63,9 @@
 #define ERTS_SIG_Q_OP_TRACE_CHANGE_STATE        8
 #define ERTS_SIG_Q_OP_PERSISTENT_MON_MSG        9
 #define ERTS_SIG_Q_OP_IS_ALIVE                  10
-#define ERTS_SIG_Q_OP_PROCESS_INFO              ERTS_SIG_Q_OP_MAX
+#define ERTS_SIG_Q_OP_PROCESS_INFO              11
+#define ERTS_SIG_Q_OP_SYNC_SUSPEND              12
+#define ERTS_SIG_Q_OP_RPC                       ERTS_SIG_Q_OP_MAX
 
 #define ERTS_SIG_Q_TYPE_MAX (ERTS_MON_LNK_TYPE_MAX + 5)
 
@@ -153,6 +156,17 @@ typedef struct {
     Eterm requester;
 } ErtsIsAliveRequest;
 
+typedef struct {
+    Eterm message;
+    Eterm requester;
+    int async;
+} ErtsSyncSuspendRequest;
+
+typedef struct {
+    ErtsMonitorSuspend *mon;
+    ErtsMessage *sync;
+} ErtsProcSigPendingSuspend;
+
 typedef struct {
     ErtsSignalCommon common;
     Sint refc;
@@ -176,6 +190,15 @@ typedef struct {
 #define ERTS_PROC_SIG_PI_MSGQ_LEN_IGNORE        ((Sint) -1)
 #define ERTS_PROC_SIG_PI_MSGQ_LEN_SYNC          ((Sint) -2)
 
+typedef struct {
+    ErtsSignalCommon common;
+    Eterm requester;
+    Eterm (*func)(Process *, void *, int *, ErlHeapFragment **);
+    void *arg;
+    Eterm ref;
+    ErtsORefThing oref_thing;
+} ErtsProcSigRPC;
+
 static int handle_msg_tracing(Process *c_p,
                               ErtsSigRecvTracing *tracing,
                               ErtsMessage ***next_nm_sig);
@@ -1311,6 +1334,8 @@ erts_proc_sig_send_monitor_down(ErtsMonitor *mon, Eterm reason)
         /* Pass signal using old monitor structure... */
         ErtsSignal *sig;
 
+    send_using_monitor_struct:
+
         mon->other.item = reason; /* Pass immed reason via other.item... */
         sig = (ErtsSignal *) mon;
         sig->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_MONITOR_DOWN,
@@ -1322,6 +1347,18 @@ erts_proc_sig_send_monitor_down(ErtsMonitor *mon, Eterm reason)
         ErtsMonitorData *mdp = erts_monitor_to_data(mon);
         Eterm from_tag, monitored, heap[3];
 
+        if (mon->type == ERTS_MON_TYPE_SUSPEND) {
+            /*
+             * Set reason to 'undefined', since exit
+             * reason is not used for suspend monitors,
+             * and send using monitor structure. This
+             * since we don't want to trigger
+             * unnecessary memory allocation etc...
+             */
+            reason = am_undefined;
+            goto send_using_monitor_struct;
+        }
+
         if (!(mon->flags & ERTS_ML_FLG_NAME)) {
             from_tag = monitored = mdp->origin.other.item;
             if (is_external_pid(from_tag)) {
@@ -1599,7 +1636,173 @@ erts_proc_sig_send_process_info_request(Process *c_p,
     else
         erts_free(ERTS_ALC_T_SIG_DATA, pis);
     return res;
-}       
+}
+
+void
+erts_proc_sig_send_sync_suspend(Process *c_p, Eterm to, Eterm tag, Eterm reply)
+{
+    ErlHeapFragment *hfrag;
+    Uint hsz, tag_sz;
+    Eterm *hp, *start_hp, tag_cpy, msg, default_reply;
+    ErlOffHeap *ohp;
+    ErtsMessage *mp;
+    ErtsSyncSuspendRequest *ssusp;
+    int async_suspend;
+
+    tag_sz = size_object(tag);
+
+    hsz = 3 + tag_sz + sizeof(ErtsSyncSuspendRequest)/sizeof(Eterm);
+
+    mp = erts_alloc_message(hsz, &hp);
+    hfrag = &mp->hfrag;
+    mp->next = NULL;
+    ohp = &hfrag->off_heap;
+    start_hp = hp;
+
+    tag_cpy = copy_struct(tag, tag_sz, &hp, ohp);
+
+    async_suspend = is_non_value(reply);
+    default_reply = async_suspend ? am_suspended : reply;
+
+    msg = TUPLE2(hp, tag_cpy, default_reply);
+    hp += 3;
+
+    hfrag->used_size = hp - start_hp;
+
+    ssusp = (ErtsSyncSuspendRequest *) (char *) hp;
+    ssusp->message = msg;
+    ssusp->requester = c_p->common.id;
+    ssusp->async = async_suspend;
+
+    ERL_MESSAGE_TERM(mp) = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_SYNC_SUSPEND,
+                                                  ERTS_SIG_Q_TYPE_UNDEFINED,
+                                                  0);
+    ERL_MESSAGE_TOKEN(mp) = NIL;
+    ERL_MESSAGE_FROM(mp) = am_system;
+#ifdef USE_VM_PROBES
+    ERL_MESSAGE_DT_UTAG(mp) = NIL;
+#endif
+
+    if (proc_queue_signal(c_p, to, (ErtsSignal *) mp, ERTS_SIG_Q_OP_SYNC_SUSPEND))
+        (void) maybe_elevate_sig_handling_prio(c_p, to);
+    else {
+        Eterm *tp;
+        /* It wasn't alive; reply to ourselves... */
+        mp->next = NULL;
+        mp->data.attached = ERTS_MSG_COMBINED_HFRAG;
+        tp = tuple_val(msg);
+        tp[2] = async_suspend ? am_badarg : am_exited;
+        erts_queue_message(c_p, ERTS_PROC_LOCK_MAIN,
+                           mp, msg, am_system);
+    }
+}
+
+Eterm
+erts_proc_sig_send_rpc_request(Process *c_p,
+                               Eterm to,
+                               int reply,
+                               Eterm (*func)(Process *, void *, int *, ErlHeapFragment **),
+                               void *arg)
+{
+    Eterm res;
+    ErtsProcSigRPC *sig = erts_alloc(ERTS_ALC_T_SIG_DATA,
+                                     sizeof(ErtsProcSigRPC));
+    sig->common.tag = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_RPC,
+                                             ERTS_SIG_Q_TYPE_UNDEFINED,
+                                             0);
+    sig->requester = reply ? c_p->common.id : NIL;
+    sig->func = func;
+    sig->arg = arg;
+
+    if (!reply) {
+        res = am_ok;
+        sig->ref = am_ok;
+    }
+    else {
+        res = erts_make_ref(c_p);
+
+        sys_memcpy((void *) &sig->oref_thing,
+                   (void *) internal_ref_val(res),
+                   sizeof(ErtsORefThing));
+
+        sig->ref = make_internal_ref(&sig->oref_thing);
+
+        ERTS_RECV_MARK_SAVE(c_p);
+        ERTS_RECV_MARK_SET(c_p);
+    }
+
+    if (proc_queue_signal(c_p, to, (ErtsSignal *) sig, ERTS_SIG_Q_OP_RPC))
+        (void) maybe_elevate_sig_handling_prio(c_p, to);
+    else {
+        erts_free(ERTS_ALC_T_SIG_DATA, sig);
+        res = THE_NON_VALUE;
+        if (reply)
+            JOIN_MESSAGE(c_p);
+    }
+
+    return res;
+}
+
+static int
+handle_rpc(Process *c_p, ErtsProcSigRPC *rpc, int cnt, int limit, int *yieldp)
+{
+    Process *rp;
+    ErlHeapFragment *bp = NULL;
+    Eterm res;
+    Uint hsz;
+    int reds, out_cnt;
+
+    /*
+     * reds in:
+     *  Reductions left.
+     *
+     * reds out:
+     *  Absolute value of reds out equals consumed
+     *  amount of reds. If a negative value, force
+     *  a yield.
+     */
+
+    reds = (limit - cnt) / ERTS_SIG_REDS_CNT_FACTOR;
+    if (reds <= 0)
+        reds = 1;
+
+    res = (*rpc->func)(c_p, rpc->arg, &reds, &bp);
+
+    if (reds < 0) {
+        /* Force yield... */
+        *yieldp = !0;
+        reds *= -1;
+    }
+
+    out_cnt = reds*ERTS_SIG_REDS_CNT_FACTOR;
+
+    hsz = 3 + sizeof(ErtsORefThing)/sizeof(Eterm);
+
+    rp = erts_proc_lookup(rpc->requester);
+    if (!rp) {
+        if (bp)
+            free_message_buffer(bp);
+    }
+    else {
+        Eterm *hp, msg, ref;
+        ErtsMessage *mp = erts_alloc_message(hsz, &hp);
+        
+        sys_memcpy((void *) hp, (void *) &rpc->oref_thing,
+                   sizeof(rpc->oref_thing));
+
+        ref = make_internal_ref(hp);
+        hp += sizeof(rpc->oref_thing)/sizeof(Eterm);
+        msg = TUPLE2(hp, ref, res);
+
+        mp->hfrag.next = bp;
+
+        erts_queue_proc_message(c_p, rp, 0, mp, msg);
+    }
+
+    erts_free(ERTS_ALC_T_SIG_DATA, rpc);
+
+    return out_cnt;
+}
 
 static void
 is_alive_response(Process *c_p, ErtsMessage *mp, int is_alive)
@@ -2643,6 +2846,155 @@ handle_process_info(Process *c_p, ErtsSigRecvTracing *tracing,
     return ((int) reds)*4 + 8;
 }
 
+static void
+handle_suspend(Process *c_p, ErtsMonitor *mon, int *yieldp)
+{
+    erts_aint32_t state = erts_atomic32_read_nob(&c_p->state);
+
+    ASSERT(mon->type == ERTS_MON_TYPE_SUSPEND);
+
+    if (!(state & ERTS_PSFLG_DIRTY_RUNNING)) {
+        ErtsMonitorSuspend *msp;
+        erts_aint_t mstate;
+
+        msp = (ErtsMonitorSuspend *) erts_monitor_to_data(mon);
+        mstate = erts_atomic_read_bor_acqb(&msp->state,
+                                           ERTS_MSUSPEND_STATE_FLG_ACTIVE);
+        ASSERT(!(mstate & ERTS_MSUSPEND_STATE_FLG_ACTIVE)); (void) mstate;
+        erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
+        *yieldp = !0;
+    }
+    else {
+        /* Executing dirty; delay suspend... */
+        ErtsProcSigPendingSuspend *psusp;
+        ErtsMonitorSuspend *msp;
+
+        psusp = ERTS_PROC_GET_PENDING_SUSPEND(c_p);
+        if (!psusp) {
+            psusp = erts_alloc(ERTS_ALC_T_SIG_DATA,
+                               sizeof(ErtsProcSigPendingSuspend));
+            psusp->mon = NULL;
+            psusp->sync = NULL;
+            ERTS_PROC_SET_PENDING_SUSPEND(c_p, (void *) psusp);
+        }
+
+        msp = (ErtsMonitorSuspend *) erts_monitor_to_data(mon);
+
+        msp->next = psusp->mon;
+        psusp->mon = msp;
+
+        erts_atomic32_inc_nob(&msp->md.refc);
+    }
+}
+
+static void
+sync_suspend_reply(Process *c_p, ErtsMessage *mp, erts_aint32_t state)
+{
+    /*
+     * Sender prepared the message for us. Just patch
+     * the result if necessary. The default prepared
+     * result is 'false'.
+     */
+    Process *rp;
+    ErtsSyncSuspendRequest *ssusp;
+
+    ssusp = (ErtsSyncSuspendRequest *) (char *) (&mp->hfrag.mem[0]
+                                                 + mp->hfrag.used_size);
+
+    ASSERT(ERTS_SIG_IS_NON_MSG(mp));
+    ASSERT(ERTS_PROC_SIG_OP(((ErtsSignal *) mp)->common.tag)
+           == ERTS_SIG_Q_OP_SYNC_SUSPEND);
+    ASSERT(mp->hfrag.alloc_size > mp->hfrag.used_size);
+    ASSERT((mp->hfrag.alloc_size - mp->hfrag.used_size)*sizeof(UWord)
+           >= sizeof(ErtsSyncSuspendRequest));
+    ASSERT(is_internal_pid(ssusp->requester));
+    ASSERT(ssusp->requester != c_p->common.id);
+    ASSERT(is_tuple_arity(ssusp->message, 2));
+    ASSERT(is_immed(tuple_val(ssusp->message)[2]));
+
+    ERL_MESSAGE_TERM(mp) = ssusp->message;
+    mp->data.attached = ERTS_MSG_COMBINED_HFRAG;
+    mp->next = NULL;
+
+    rp = erts_proc_lookup(ssusp->requester);
+    if (!rp)
+        erts_cleanup_messages(mp);
+    else {
+        if ((state & (ERTS_PSFLG_EXITING
+                      | ERTS_PSFLG_SUSPENDED)) != ERTS_PSFLG_SUSPENDED) {
+            /* Not suspended -> patch result... */
+            if (state & ERTS_PSFLG_EXITING) {
+                Eterm *tp = tuple_val(ssusp->message);
+                tp[2] = ssusp->async ? am_exited : am_badarg;
+            }
+            else {
+                Eterm *tp = tuple_val(ssusp->message);
+                ASSERT(!(state & ERTS_PSFLG_SUSPENDED));
+                tp[2] = ssusp->async ? am_not_suspended : am_internal_error;
+            }
+        }
+        erts_queue_proc_message(c_p, rp, 0, mp, ssusp->message);
+    }
+}
+
+static void
+handle_sync_suspend(Process *c_p, ErtsMessage *mp)
+{
+    ErtsProcSigPendingSuspend *psusp;
+
+    psusp = (ErtsProcSigPendingSuspend *) ERTS_PROC_GET_PENDING_SUSPEND(c_p);
+    if (!psusp)
+        sync_suspend_reply(c_p, mp, erts_atomic32_read_nob(&c_p->state));
+    else {
+        mp->next = psusp->sync;
+        psusp->sync = mp;
+    }
+}
+
+void
+erts_proc_sig_handle_pending_suspend(Process *c_p)
+{
+    ErtsMonitorSuspend *msp;
+    ErtsMessage *sync;
+    ErtsProcSigPendingSuspend *psusp;
+    erts_aint32_t state = erts_atomic32_read_nob(&c_p->state);
+
+    psusp = (ErtsProcSigPendingSuspend *) ERTS_PROC_GET_PENDING_SUSPEND(c_p);
+
+    msp = psusp->mon;
+    
+    while (msp) {
+        ErtsMonitorSuspend *next_msp = msp->next;
+        msp->next = NULL;
+        if (!(state & ERTS_PSFLG_EXITING)
+            && erts_monitor_is_in_table(&msp->md.target)) {
+            erts_aint_t mstate;
+
+            mstate = erts_atomic_read_bor_acqb(&msp->state,
+                                               ERTS_MSUSPEND_STATE_FLG_ACTIVE);
+            ASSERT(!(mstate & ERTS_MSUSPEND_STATE_FLG_ACTIVE)); (void) mstate;
+            erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
+        }
+
+        erts_monitor_release(&msp->md.target);
+
+        msp = next_msp;
+    }
+
+    sync = psusp->sync;
+
+    while (sync) {
+        ErtsMessage *next_sync = sync->next;
+        sync->next = NULL;
+        sync_suspend_reply(c_p, sync, state);
+        sync = next_sync;
+    }
+
+    erts_free(ERTS_ALC_T_SIG_DATA, psusp);
+
+    ERTS_PROC_SET_PENDING_SUSPEND(c_p, NULL);
+}
+
 /*
  * Called in order to handle incoming signals.
  */
@@ -2653,7 +3005,7 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep,
 {
     Eterm tag;
     erts_aint32_t state;
-    int cnt, limit, abs_lim, msg_tracing;
+    int yield, cnt, limit, abs_lim, msg_tracing;
     ErtsMessage *sig, ***next_nm_sig;
     ErtsSigRecvTracing tracing;
 
@@ -2673,6 +3025,7 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep,
 
     limit = *redsp;
     *redsp = 0;
+    yield = 0;
 
     if (!c_p->sig_qs.cont) {
         if (state == -1)
@@ -2786,6 +3139,18 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep,
                     cnt += handle_nodedown(c_p, sig, mdp, next_nm_sig);
                 }
                 break;
+            case ERTS_MON_TYPE_SUSPEND:
+                tmon = (ErtsMonitor *) sig;
+                ASSERT(erts_monitor_is_target(tmon));
+                ASSERT(!erts_monitor_is_in_table(tmon));
+                mdp = erts_monitor_to_data(tmon);
+                if (erts_monitor_is_in_table(&mdp->origin)) {
+                    erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p),
+                                             &mdp->origin);
+                    omon = &mdp->origin;
+                }
+                remove_nm_sig(c_p, sig, next_nm_sig);
+                break;
             default:
                 ERTS_INTERNAL_ERROR("invalid monitor type");
                 break;
@@ -2849,9 +3214,13 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep,
 
             if (mon->type == ERTS_MON_TYPE_DIST_PROC)
                 erts_monitor_tree_insert(&ERTS_P_MONITORS(c_p), mon);
-            else
+            else {
                 erts_monitor_list_insert(&ERTS_P_LT_MONITORS(c_p), mon);
+                if (mon->type == ERTS_MON_TYPE_SUSPEND)
+                    handle_suspend(c_p, mon, &yield);
+            }
             ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            cnt += 2;
             break;
         }
 
@@ -2895,9 +3264,16 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep,
                         erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), tmon);
                     else {
                         erts_monitor_list_delete(&ERTS_P_LT_MONITORS(c_p), tmon);
-                        if (type == ERTS_MON_TYPE_RESOURCE) {
+                        switch (type) {
+                        case ERTS_MON_TYPE_RESOURCE:
                             erts_nif_demonitored((ErtsResource *) tmon->other.ptr);
                             cnt++;
+                            break;
+                        case ERTS_MON_TYPE_SUSPEND:
+                            erts_resume(c_p, ERTS_PROC_LOCK_MAIN);
+                            break;
+                        default:
+                            break;
                         }
                     }
                     erts_monitor_release_both(mdp);
@@ -3012,6 +3388,21 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep,
             ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
             break;
 
+        case ERTS_SIG_Q_OP_SYNC_SUSPEND:
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            remove_nm_sig(c_p, sig, next_nm_sig);
+            handle_sync_suspend(c_p, sig);
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            break;
+
+        case ERTS_SIG_Q_OP_RPC:
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            remove_nm_sig(c_p, sig, next_nm_sig);
+            cnt += handle_rpc(c_p, (ErtsProcSigRPC *) sig, cnt,
+                              limit, &yield);
+            ERTS_PROC_SIG_HDBG_PRIV_CHKQ(c_p, &tracing, next_nm_sig);
+            break;
+
         case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE: {
             Uint16 type = ERTS_PROC_SIG_TYPE(tag);
 
@@ -3169,6 +3560,15 @@ stop: {
 
         *redsp = cnt/4 + 1;
 
+        if (yield) {
+            int vreds = max_reds - *redsp;
+            if (vreds > 0) {
+                ErtsSchedulerData *esdp = erts_get_scheduler_data();
+                esdp->virtual_reds += vreds;
+            }
+            *redsp = max_reds;
+        }
+
         return res;
     }
 }
@@ -3277,6 +3677,8 @@ erts_proc_sig_handle_exit(Process *c_p, int *redsp)
             case ERTS_MON_TYPE_PROC:
             case ERTS_MON_TYPE_DIST_PROC:
             case ERTS_MON_TYPE_NODE:
+            case ERTS_MON_TYPE_NODES:
+            case ERTS_MON_TYPE_SUSPEND:
                 erts_monitor_release((ErtsMonitor *) sig);
                 break;
             default:
@@ -3332,6 +3734,17 @@ erts_proc_sig_handle_exit(Process *c_p, int *redsp)
             handle_process_info(c_p, NULL, sig, next_nm_sig, 0);
             break;
 
+        case ERTS_SIG_Q_OP_SYNC_SUSPEND:
+            handle_sync_suspend(c_p, sig);
+            break;
+
+        case ERTS_SIG_Q_OP_RPC: {
+            int yield = 0;
+            handle_rpc(c_p, (ErtsProcSigRPC *) sig,
+                       cnt, limit, &yield);
+            break;
+        }
+
         case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE:
             destroy_trace_info((ErtsSigTraceInfo *) sig);
             break;
@@ -3467,6 +3880,7 @@ erts_proc_sig_signal_size(ErtsSignal *sig)
         }
         break;
 
+    case ERTS_SIG_Q_OP_SYNC_SUSPEND:
     case ERTS_SIG_Q_OP_PERSISTENT_MON_MSG:
     case ERTS_SIG_Q_OP_IS_ALIVE:
         size = ((ErtsMessage *) sig)->hfrag.alloc_size;
@@ -3522,6 +3936,10 @@ erts_proc_sig_signal_size(ErtsSignal *sig)
         break;
     }
 
+    case ERTS_SIG_Q_OP_RPC:
+        size = sizeof(ErtsProcSigRPC);
+        break;
+
     default:
         ERTS_INTERNAL_ERROR("Unknown signal");
         break;
@@ -3598,17 +4016,13 @@ erts_proc_sig_receive_helper(Process *c_p,
                  */
                 *get_outp = 0;
                 *msgpp = NULL;
+
                 return consumed_reds;
             }
             erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ);
 
-            if (left_reds <= 0) {
-                *get_outp = -1; /* yield */
-                *msgpp = NULL;
-
-                ASSERT(consumed_reds >= (fcalls - neg_o_reds));
-                return consumed_reds;
-            }
+            if (left_reds <= 0)
+                break; /* Yield */
 
             /* handle newly arrived signals... */
         }
@@ -3629,19 +4043,27 @@ erts_proc_sig_receive_helper(Process *c_p,
                                              max_reds, !0);
         consumed_reds += reds;
         left_reds -= reds;
-        /* we may have exited by an incoming signal... */
-        if (state & ERTS_PSFLG_EXITING) {
+
+        /* we may have exited or suspended by an incoming signal... */
+
+        if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_SUSPENDED)) {
+            if (state & ERTS_PSFLG_SUSPENDED)
+                break; /* Yield */
+
             /*
              * Process need to schedule out in order
              * to terminate. Prepare this a bit...
              */
+            ASSERT(state & ERTS_PSFLG_EXITING);
             ASSERT(c_p->flags & F_DELAY_GC);
 
             c_p->flags &= ~F_DELAY_GC;
             c_p->arity = 0;
             c_p->current = NULL;
+
             *get_outp = 1;
             *msgpp = NULL;
+
             return consumed_reds;
         }
 
@@ -3652,17 +4074,20 @@ erts_proc_sig_receive_helper(Process *c_p,
             return consumed_reds;
         }
 
-        if (left_reds <= 0) {
-            *get_outp = -1; /* yield */
-            *msgpp = NULL;
-
-            ASSERT(consumed_reds >= (fcalls - neg_o_reds));
-            return consumed_reds;
-        }
+        if (left_reds <= 0)
+            break; /* yield */
 
         ASSERT(!c_p->sig_qs.cont);
         /* Go fetch again... */
     }
+    
+    /* Yield... */
+
+    *get_outp = -1;
+    *msgpp = NULL;
+
+    ASSERT(consumed_reds >= (fcalls - neg_o_reds));
+    return consumed_reds;
 }
 
 static int
diff --git a/erts/emulator/beam/erl_proc_sig_queue.h b/erts/emulator/beam/erl_proc_sig_queue.h
index 8b7cd35f61..417fd90716 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.h
+++ b/erts/emulator/beam/erl_proc_sig_queue.h
@@ -33,6 +33,11 @@
  *              - Group leader
  *              - Is process alive
  *              - Process info request
+ *              - Suspend request (monitor of suspend type)
+ *              - Resume request (demonitor of suspend type)
+ *              - Suspend cleanup (monitor down of suspend type)
+ *              - Sync suspend
+ *              - RPC request
  *              - Trace change
  *
  *              The signal queue consists of three parts:
@@ -557,6 +562,102 @@ erts_proc_sig_send_process_info_request(Process *c_p,
                                         Uint reserve_size,
                                         Eterm ref);
 
+/**
+ *
+ * @brief Send a 'sync suspend' signal to a process.
+ *
+ * A response message '{Tag, Reply}' is sent to the
+ * sender when performed where Tag is the term passed
+ * as 'tag' argument. Reply is either 'suspended',
+ * 'not_suspended', 'exited' if the operation is
+ * asynchronous; otherwise, the 'reply' argument or
+ * 'badarg' if process terminated.
+ *
+ * This signal does *not* change the suspend state, only
+ * reads and reply the state. This signal is typically
+ * sent after a suspend request (monitor of suspend type)
+ * signal has been sent to the process in order to get a
+ * response when the suspend monitor has been processed.
+ *
+ * @param[in]     c_p           Pointer to process struct of
+ *                              currently executing process.
+ *
+ * @param[in]     to            Identifier of receiver.
+ *
+ * @param[in]     tag           Tag to use in response
+ *                              message to the sending
+ *                              process (i.e., c_p).
+ *
+ * @param[in]     reply         Reply to send if this
+ *                              is a synchronous operation;
+ *                              otherwise, THE_NON_VALUE.
+ */
+void
+erts_proc_sig_send_sync_suspend(Process *c_p, Eterm to,
+                                Eterm tag, Eterm reply);
+
+/**
+ *
+ * @brief Send an 'rpc' signal to a process.
+ *
+ * The function 'func' will be executed in the
+ * context of the receiving process. A response
+ * message '{Ref, Result}' is sent to the sender
+ * when 'func' has been called. 'Ref' is the reference
+ * returned by this function and 'Result' is the
+ * term returned by 'func'. If the return value of
+ * 'func' is not an immediate term, 'func' has to
+ * allocate a heap fragment where the result is stored
+ * and update the the heap fragment pointer pointer
+ * passed as third argument to point to it.
+ *
+ * If this function returns a reference, 'func' will
+ * be called in the context of the receiver. However,
+ * note that this might happen when the receiver is in
+ * an exiting state. The caller of this function
+ * *unconditionally* has to enter a receive that match
+ * on the returned reference in all clauses as next
+ * receive; otherwise, bad things will happen!
+ *
+ * If THE_NON_VALUE is returned, the receiver did not
+ * exist. The signal was not sent, and no specific
+ * receive has to be entered by the caller.
+ *
+ * @param[in]     c_p           Pointer to process struct of
+ *                              currently executing process.
+ *
+ * @param[in]     to            Identifier of receiver process.
+ *
+ * @param[in]     reply         Non-zero if a reply is wanted.
+ *
+ * @param[in]     func          Function to execute in the
+ *                              context of the receiver.
+ *                              First argument will be a
+ *                              pointer to the process struct
+ *                              of the receiver process.
+ *                              Second argument will be 'arg'
+ *                              (see below). Third argument
+ *                              will be a pointer to a pointer
+ *                              to a heap fragment for storage
+ *                              of result returned from 'func'
+ *                              (i.e. an 'out' parameter).
+ *
+ * @param[in]     arg           Void pointer to argument
+ *                              to pass as second argument
+ *                              in call of 'func'.
+ *
+ * @returns                     If the request was sent,
+ *                              an internal ordinary
+ *                              reference; otherwise,
+ *                              THE_NON_VALUE (non-existing
+ *                              receiver).
+ */
+Eterm
+erts_proc_sig_send_rpc_request(Process *c_p,
+                               Eterm to,
+                               int reply,
+                               Eterm (*func)(Process *, void *, int *, ErlHeapFragment **),
+                               void *arg);
 
 /*
  * End of send operations of currently supported process signals.
@@ -812,6 +913,21 @@ Uint erts_proc_sig_signal_size(ErtsSignal *sig);
 void
 erts_proc_sig_clear_seq_trace_tokens(Process *c_p);
 
+/**
+ *
+ * @brief Handle pending suspend requests
+ *
+ * Should be called by processes when they stop
+ * execution on a dirty scheduler if they have
+ * pending suspend requests (i.e. when
+ * ERTS_PROC_GET_PENDING_SUSPEND(c_p) != NULL).
+ *
+ * @param[in]   c_p             Pointer to executing
+ *                              process
+ */
+void
+erts_proc_sig_handle_pending_suspend(Process *c_p);
+
 /**
  * @brief Initialize this functionality
  */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 8637ab4f68..f3e77df2fd 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -185,8 +185,6 @@ sched_get_busy_wait_params(ErtsSchedulerData *esdp)
     return &sched_busy_wait_params[esdp->type];
 }
 
-int erts_disable_proc_not_running_opt;
-
 static ErtsAuxWorkData *aux_thread_aux_work_data;
 static ErtsAuxWorkData *poll_thread_aux_work_data;
 
@@ -730,6 +728,11 @@ erts_pre_init_process(void)
         = ERTS_PSD_DIST_ENTRY_GET_LOCKS;
     erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks
         = ERTS_PSD_DIST_ENTRY_SET_LOCKS;
+
+    erts_psd_required_locks[ERTS_PSD_PENDING_SUSPEND].get_locks
+        = ERTS_PSD_PENDING_SUSPEND_GET_LOCKS;
+    erts_psd_required_locks[ERTS_PSD_PENDING_SUSPEND].set_locks
+        = ERTS_PSD_PENDING_SUSPEND_SET_LOCKS;
 #endif
 }
 
@@ -744,7 +747,6 @@ void
 erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab)
 {
 
-    erts_disable_proc_not_running_opt = 0;
     erts_init_proc_lock(ncpu);
 
     init_proclist_alloc();
@@ -8553,427 +8555,22 @@ erts_start_schedulers(void)
     }
 }
 
-
-
-static void
-add_pend_suspend(Process *suspendee,
-		 Eterm originator_pid,
-		 void (*handle_func)(Process *,
-				     ErtsProcLocks,
-				     int,
-				     Eterm))
-{
-    ErtsPendingSuspend *psp = erts_alloc(ERTS_ALC_T_PEND_SUSPEND,
-					 sizeof(ErtsPendingSuspend));
-    psp->next = NULL;
-#ifdef DEBUG
-#if defined(ARCH_64)
-    psp->end = (ErtsPendingSuspend *) 0xdeaddeaddeaddead;
-#else
-    psp->end = (ErtsPendingSuspend *) 0xdeaddead;
-#endif
-#endif
-    psp->pid = originator_pid;
-    psp->handle_func = handle_func;
-
-    if (suspendee->pending_suspenders)
-	suspendee->pending_suspenders->end->next = psp;
-    else
-	suspendee->pending_suspenders = psp;
-    suspendee->pending_suspenders->end = psp;
-}
-
-static void
-handle_pending_suspend(Process *p, ErtsProcLocks p_locks)
-{
-    ErtsPendingSuspend *psp;
-    int is_alive = !ERTS_PROC_IS_EXITING(p);
-
-    ERTS_LC_ASSERT(p_locks & ERTS_PROC_LOCK_STATUS);
-
-    /*
-     * New pending suspenders might appear while we are processing
-     * (since we may release the status lock on p while processing).
-     */
-    while (p->pending_suspenders) {
-	psp = p->pending_suspenders;
-	p->pending_suspenders = NULL;
-	while (psp) {
-	    ErtsPendingSuspend *free_psp;
-	    (*psp->handle_func)(p, p_locks, is_alive, psp->pid);
-	    free_psp = psp;
-	    psp = psp->next;
-	    erts_free(ERTS_ALC_T_PEND_SUSPEND, (void *) free_psp);
-	}
-    }
-    
-}
-
-static ERTS_INLINE void
-cancel_suspend_of_suspendee(Process *p, ErtsProcLocks p_locks)
-{
-    if (is_not_nil(p->suspendee)) {
-        ErtsMonitor *mon;
-        Eterm suspendee = p->suspendee;
-	Process *rp;
-	if (!(p_locks & ERTS_PROC_LOCK_STATUS))
-	    erts_proc_lock(p, ERTS_PROC_LOCK_STATUS);
-	rp = erts_pid2proc(p, p_locks|ERTS_PROC_LOCK_STATUS,
-			   suspendee, ERTS_PROC_LOCK_STATUS);
-	if (rp) {
-	    erts_resume(rp, ERTS_PROC_LOCK_STATUS);
-	    erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
-	}
-	if (!(p_locks & ERTS_PROC_LOCK_STATUS))
-	    erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-	p->suspendee = NIL;
-
-        mon = erts_monitor_tree_lookup(p->suspend_monitors,
-                                       suspendee);
-        if (mon) {
-            erts_monitor_tree_delete(&p->suspend_monitors,
-                                     mon);
-            erts_monitor_suspend_destroy(erts_monitor_suspend(mon));
-        }
-    }
-}
-
-static void
-handle_pend_sync_suspend(Process *suspendee,
-			 ErtsProcLocks suspendee_locks,
-			 int suspendee_alive,
-			 Eterm suspender_pid)
-{
-    Process *suspender;
-
-    ERTS_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS);
-
-    suspender = erts_pid2proc(suspendee,
-			      suspendee_locks,
-			      suspender_pid,
-			      ERTS_PROC_LOCK_STATUS);
-    if (suspender) {
-	ASSERT(is_nil(suspender->suspendee));
-	if (suspendee_alive) {
-	    erts_suspend(suspendee, suspendee_locks, NULL);
-	    suspender->suspendee = suspendee->common.id;
-	}
-	/* suspender is suspended waiting for suspendee to suspend;
-	   resume suspender */
-	ASSERT(suspendee != suspender);
-	resume_process(suspender, ERTS_PROC_LOCK_STATUS);
-	erts_proc_unlock(suspender, ERTS_PROC_LOCK_STATUS);
-    }
-}
-
-static Process *
-pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
-		     Eterm pid, ErtsProcLocks pid_locks, int suspend)
-{
-    Process *rp;
-    int unlock_c_p_status;
-
-    ERTS_LC_ASSERT(c_p_locks == erts_proc_lc_my_proc_locks(c_p));
-
-    ERTS_LC_ASSERT(c_p_locks & ERTS_PROC_LOCK_MAIN);
-    ERTS_LC_ASSERT(pid_locks & (ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS));
-
-    if (c_p->common.id == pid)
-	return erts_pid2proc(c_p, c_p_locks, pid, pid_locks);
-
-    if (c_p_locks & ERTS_PROC_LOCK_STATUS)
-	unlock_c_p_status = 0;
-    else {
-	unlock_c_p_status = 1;
-	erts_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
-    }
-
-    if (c_p->suspendee == pid) {
-	/* Process previously suspended by c_p (below)... */
-	ErtsProcLocks rp_locks = pid_locks|ERTS_PROC_LOCK_STATUS;
-	rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, pid, rp_locks);
-	c_p->suspendee = NIL;
-	ASSERT(c_p->flags & F_P2PNR_RESCHED);
-	c_p->flags &= ~F_P2PNR_RESCHED;
-	if (!suspend && rp)
-	    resume_process(rp, rp_locks);
-    }
-    else {
-	rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS,
-			   pid, ERTS_PROC_LOCK_STATUS);
-
-	if (!rp) {
-	    c_p->flags &= ~F_P2PNR_RESCHED;
-	    goto done;
-	}
-
-	ASSERT(!(c_p->flags & F_P2PNR_RESCHED));
-
-	/*
-	 * Suspend the other process in order to prevent
-	 * it from being selected for normal execution.
-	 * This will however not prevent it from being
-	 * selected for execution of a system task. If
-	 * it is selected for execution of a system task
-	 * we might be blocked for quite a while if the
-	 * try-lock below fails. That is, there is room
-	 * for improvement here...
-	 */
-
-	if (!suspend_process(c_p, rp)) {
-	    /* Other process running */
-
-	    ASSERT((ERTS_PSFLG_RUNNING | ERTS_PSFLG_DIRTY_RUNNING)
-		   & erts_atomic32_read_nob(&rp->state));
-
-	    if (!suspend
-		&& (erts_atomic32_read_nob(&rp->state)
-		    & ERTS_PSFLG_DIRTY_RUNNING)) {
-		ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS;
-		if (need_locks && erts_proc_trylock(rp, need_locks) == EBUSY) {
-		    erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
-		    rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS,
-				       pid, pid_locks|ERTS_PROC_LOCK_STATUS);
-		}
-		goto done;
-	    }
-
-	running:
-
-	    /*
-	     * If we got pending suspenders and suspend ourselves waiting
-	     * to suspend another process we might deadlock.
-	     * In this case we have to yield, be suspended by
-	     * someone else and then do it all over again.
-	     */
-	    if (!c_p->pending_suspenders) {
-		/* Mark rp pending for suspend by c_p */
-		add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend);
-		ASSERT(is_nil(c_p->suspendee));
-
-		/* Suspend c_p; when rp is suspended c_p will be resumed. */
-		suspend_process(c_p, c_p);
-		c_p->flags |= F_P2PNR_RESCHED;
-	    }
-	    /* Yield (caller is assumed to yield immediately in bif). */
-	    erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
-	    rp = ERTS_PROC_LOCK_BUSY;
-	}
-	else {
-	    ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS;
-	    if (need_locks && erts_proc_trylock(rp, need_locks) == EBUSY) {
-		if ((ERTS_PSFLG_RUNNING_SYS|ERTS_PSFLG_DIRTY_RUNNING_SYS)
-		    & erts_atomic32_read_nob(&rp->state)) {
-		    /* Executing system task... */
-		    resume_process(rp, ERTS_PROC_LOCK_STATUS);
-		    goto running;
-		}
-		erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
-		/*
-		 * If we are unlucky, the process just got selected for
-		 * execution of a system task. In this case we may be
-		 * blocked here for quite a while... Execution of system
-		 * tasks are fortunately quite rare events. We try to
-		 * avoid this by checking if it is in a state executing
-		 * system tasks (above), but it will not prevent all
-		 * scenarios for a long block here...
-		 */
-		rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS,
-				   pid, pid_locks|ERTS_PROC_LOCK_STATUS);
-		if (!rp)
-		    goto done;
-	    }
-
-	    /*
-	     * The previous suspend has prevented the process
-	     * from being selected for normal execution regardless
-	     * of locks held or not held on it...
-	     */
-#ifdef DEBUG
-	    {
-		erts_aint32_t state;
-		state = erts_atomic32_read_nob(&rp->state);
-		ASSERT(!(state & ERTS_PSFLG_RUNNING));
-	    }
-#endif
-
-	    if (!suspend)
-		resume_process(rp, pid_locks|ERTS_PROC_LOCK_STATUS);
-	}
-    }
-
- done:
-	
-    if (rp && rp != ERTS_PROC_LOCK_BUSY && !(pid_locks & ERTS_PROC_LOCK_STATUS))
-	erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS);
-    if (unlock_c_p_status)
-	erts_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
-    return rp;
-}
-
-
-/*
- * Like erts_pid2proc() but:
- *
- * * At least ERTS_PROC_LOCK_MAIN have to be held on c_p.
- * * At least ERTS_PROC_LOCK_MAIN have to be taken on pid.
- * * It also waits for proc to be in a state != running and garbing.
- * * If ERTS_PROC_LOCK_BUSY is returned, the calling process has to
- *   yield (ERTS_BIF_YIELD[0-3]()). c_p might in this case have been
- *   suspended.
- */
-Process *
-erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
-			  Eterm pid, ErtsProcLocks pid_locks)
-{
-    return pid2proc_not_running(c_p, c_p_locks, pid, pid_locks, 0);
-}
-
-/*
- * erts_pid2proc_nropt() is normally the same as
- * erts_pid2proc_not_running(). However it is only
- * to be used when 'not running' is a pure optimization,
- * not a requirement.
- */
-
-Process *
-erts_pid2proc_nropt(Process *c_p, ErtsProcLocks c_p_locks,
-		    Eterm pid, ErtsProcLocks pid_locks)
-{
-    if (erts_disable_proc_not_running_opt)
-	return erts_pid2proc(c_p, c_p_locks, pid, pid_locks);
-    else
-	return erts_pid2proc_not_running(c_p, c_p_locks, pid, pid_locks);
-}
-
-static ERTS_INLINE int
-do_bif_suspend_process(Process *c_p,
-		       ErtsMonitorSuspend *smon,
-		       Process *suspendee)
-{
-    ASSERT(suspendee);
-    ASSERT(!ERTS_PROC_IS_EXITING(suspendee));
-    ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS
-		       & erts_proc_lc_my_proc_locks(suspendee));
-    if (smon) {
-	if (!smon->active) {
-	    if (!suspend_process(c_p, suspendee))
-		return 0;
-	}
-	smon->active += smon->pending;
-	ASSERT(smon->active);
-	smon->pending = 0;
-	return 1;
-    }
-    return 0;
-}
-
-static void
-handle_pend_bif_sync_suspend(Process *suspendee,
-			     ErtsProcLocks suspendee_locks,
-			     int suspendee_alive,
-			     Eterm suspender_pid)
-{
-    Process *suspender;
-
-    ERTS_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS);
-
-    suspender = erts_pid2proc(suspendee,
-			      suspendee_locks,
-			      suspender_pid,
-			      ERTS_PROC_LOCK_STATUS);
-    if (suspender) {
-        ErtsMonitorSuspend *smon;
-        ErtsMonitor *mon;
-        mon = erts_monitor_tree_lookup(suspender->suspend_monitors,
-                                       suspendee->common.id);
-        smon = erts_monitor_suspend(mon);
-
-	ASSERT(is_nil(suspender->suspendee));
-	if (!suspendee_alive) {
-            if (mon) {
-                erts_monitor_tree_delete(&suspender->suspend_monitors,
-                                         mon);
-                erts_monitor_suspend_destroy(smon);
-            }
-        }
-	else {
-#ifdef DEBUG
-	    int res =
-#endif
-		do_bif_suspend_process(suspendee, smon, suspendee);
-	    ASSERT(!smon || res != 0);
-	    suspender->suspendee = suspendee->common.id;
-	}
-	/* suspender is suspended waiting for suspendee to suspend;
-	   resume suspender */
-	ASSERT(suspender != suspendee);
-	resume_process(suspender, ERTS_PROC_LOCK_STATUS);
-	erts_proc_unlock(suspender, ERTS_PROC_LOCK_STATUS);
-    }
-}
-
-static void
-handle_pend_bif_async_suspend(Process *suspendee,
-			      ErtsProcLocks suspendee_locks,
-			      int suspendee_alive,
-			      Eterm suspender_pid)
-{
-
-    Process *suspender;
-
-    ERTS_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS);
-
-    suspender = erts_pid2proc(suspendee,
-			      suspendee_locks,
-			      suspender_pid,
-			      ERTS_PROC_LOCK_STATUS);
-    if (suspender) {
-        ErtsMonitorSuspend *smon;
-        ErtsMonitor *mon;
-        mon = erts_monitor_tree_lookup(suspender->suspend_monitors,
-                                       suspendee->common.id);
-        smon = erts_monitor_suspend(mon);
-	ASSERT(is_nil(suspender->suspendee));
-	if (!suspendee_alive) {
-            if (mon) {
-                erts_monitor_tree_delete(&suspender->suspend_monitors,
-                                         mon);
-                erts_monitor_suspend_destroy(smon);
-            }
-        }
-	else {
-#ifdef DEBUG
-	    int res =
-#endif
-                do_bif_suspend_process(suspendee, smon, suspendee);
-	    ASSERT(!smon || res != 0);
-	}
-	erts_proc_unlock(suspender, ERTS_PROC_LOCK_STATUS);
-    }
-}
-
-
-/*
- * The erlang:suspend_process/2 BIF
- */
-
 BIF_RETTYPE
-suspend_process_2(BIF_ALIST_2)
+erts_internal_suspend_process_2(BIF_ALIST_2)
 {
     Eterm res;
-    Process* suspendee = NULL;
-    ErtsMonitorSuspend *smon;
-    ErtsProcLocks xlocks = (ErtsProcLocks) 0;
-    int created;
-
-    /* Options and default values: */
-    int asynchronous = 0;
+    Eterm reply_tag = THE_NON_VALUE;
+    Eterm reply_res = THE_NON_VALUE;
+    int suspend;
+    int sync = 0;
+    int async = 0;
     int unless_suspending = 0;
-
+    erts_aint_t mstate;
+    ErtsMonitorSuspend *msp;
+    ErtsMonitorData *mdp;
 
     if (BIF_P->common.id == BIF_ARG_1)
-	goto badarg; /* We are not allowed to suspend ourselves */
+	BIF_RET(am_badarg); /* We are not allowed to suspend ourselves */
 
     if (is_not_nil(BIF_ARG_2)) {
 	/* Parse option list */
@@ -8987,192 +8584,130 @@ suspend_process_2(BIF_ALIST_2)
 		unless_suspending = 1;
 		break;
 	    case am_asynchronous:
-		asynchronous = 1;
+		async = 1;
 		break;
-	    default:
-		goto badarg;
+	    default: {
+                if (is_tuple_arity(arg, 2)) {
+                    Eterm *tp = tuple_val(arg);
+                    if (tp[1] == am_asynchronous) {
+                        async = 1;
+                        reply_tag = tp[2];
+                        break;
+                    }
+                }
+                BIF_RET(am_badarg);
 	    }
+            }
 	    arg = CDR(lp);
-	}
+        }
 	if (is_not_nil(arg))
-	    goto badarg;
+            BIF_RET(am_badarg);
     }
 
-    xlocks = ERTS_PROC_LOCK_STATUS;
-
-    erts_proc_lock(BIF_P, xlocks);
-
-    suspendee = erts_pid2proc(BIF_P,
-			      ERTS_PROC_LOCK_MAIN|xlocks,
-			      BIF_ARG_1,
-			      ERTS_PROC_LOCK_STATUS);
-    if (!suspendee)
-	goto no_suspendee;
-
-    smon = erts_monitor_suspend_tree_lookup_create(&BIF_P->suspend_monitors,
-                                                   &created,
-                                                   BIF_ARG_1);
-
-    if (asynchronous) {
-	/* --- Asynchronous suspend begin ---------------------------------- */
-
-	ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS
-			   & erts_proc_lc_my_proc_locks(BIF_P));
-	ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS
-			   == erts_proc_lc_my_proc_locks(suspendee));
-
-	if (smon->active) {
-	    smon->active += smon->pending;
-	    smon->pending = 0;
-	    if (unless_suspending)
-		res = am_false;
-	    else if (smon->active == INT_MAX)
-		goto system_limit;
-	    else {
-		smon->active++;
-		res = am_true;
-	    }
-	    /* done */
-	}
-	else {
-	    /* We havn't got any active suspends on the suspendee */
-	    if (smon->pending && unless_suspending)
-		res = am_false;
-	    else {
-		if (smon->pending == INT_MAX)
-		    goto system_limit;
-
-		smon->pending++;
-
-		if (!do_bif_suspend_process(BIF_P, smon, suspendee))
-		    add_pend_suspend(suspendee,
-				     BIF_P->common.id,
-				     handle_pend_bif_async_suspend);
-
-		res = am_true;
-	    }
-	    /* done */
-	}
-	/* --- Asynchronous suspend end ------------------------------------ */
-    }
-    else /* if (!asynchronous) */ {
-	/* --- Synchronous suspend begin ----------------------------------- */
-
-	ERTS_LC_ASSERT(((ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_STATUS)
-			    & erts_proc_lc_my_proc_locks(BIF_P))
-			   == (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_STATUS));
-	ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS
-			   == erts_proc_lc_my_proc_locks(suspendee));
-
-	if (BIF_P->suspendee == BIF_ARG_1) {
-	    /* We are back after a yield and the suspendee
-	       has been suspended on behalf of us. */
-	    ASSERT(smon->active >= 1);
-	    BIF_P->suspendee = NIL;
-	    res = (!unless_suspending || smon->active == 1
-		   ? am_true
-		   : am_false);
-	    /* done */
-	}
-	else if (smon->active) {
-	    if (unless_suspending)
-		res = am_false;
-	    else {
-		smon->active++;
-		res = am_true;
-	    }
-	    /* done */
-	}
-	else {
-	    /* We haven't got any active suspends on the suspendee */
-
-	    /*
-	     * If we have pending suspenders and suspend ourselves waiting
-	     * to suspend another process, or suspend another process
-	     * we might deadlock. In this case we have to yield,
-	     * be suspended by someone else, and then do it all over again.
-	     */
-	    if (BIF_P->pending_suspenders)
-		goto yield;
-
-	    if (!unless_suspending && smon->pending == INT_MAX)
-		goto system_limit;
-	    if (!unless_suspending || smon->pending == 0)
-		smon->pending++;
-
-	    if (do_bif_suspend_process(BIF_P, smon, suspendee)) {
-		res = (!unless_suspending || smon->active == 1
-		       ? am_true
-		       : am_false);
-		/* done */
-	    }
-	    else {
-		/* Mark suspendee pending for suspend by BIF_P */
-		add_pend_suspend(suspendee,
-				 BIF_P->common.id,
-				 handle_pend_bif_sync_suspend);
-
-		ASSERT(is_nil(BIF_P->suspendee));
-
-		/*
-		 * Suspend BIF_P; when suspendee is suspended, BIF_P
-		 * will be resumed and this BIF will be called again.
-		 * This time with BIF_P->suspendee == BIF_ARG_1 (see
-		 * above).
-		 */
-		suspend_process(BIF_P, BIF_P);
-		goto yield;
-	    }
-	}
-	/* --- Synchronous suspend end ------------------------------------- */
+    if (!unless_suspending) {
+        ErtsMonitor *mon;
+        mon = erts_monitor_tree_lookup_create(&ERTS_P_MONITORS(BIF_P),
+                                              &suspend,
+                                              ERTS_MON_TYPE_SUSPEND,
+                                              BIF_P->common.id,
+                                              BIF_ARG_1);
+        ASSERT(mon->other.item == BIF_ARG_1);
+
+        mdp = erts_monitor_to_data(mon);
+        msp = (ErtsMonitorSuspend *) mdp;
+
+        mstate = erts_atomic_inc_read_relb(&msp->state);
+        ASSERT(suspend || (mstate & ERTS_MSUSPEND_STATE_COUNTER_MASK) > 1);
+        sync = !async & !suspend & !(mstate & ERTS_MSUSPEND_STATE_FLG_ACTIVE);
+        suspend = !!suspend; /* ensure 0|1 */
+        res = am_true;
     }
-
-#ifdef DEBUG
-    {
-	erts_aint32_t state = erts_atomic32_read_acqb(&suspendee->state);
-	ASSERT((state & ERTS_PSFLG_SUSPENDED)
-	       || (asynchronous && smon->pending));
-	ASSERT((state & ERTS_PSFLG_SUSPENDED)
-	       || !smon->active);
+    else {
+        ErtsMonitor *mon;
+        mon = erts_monitor_tree_lookup(ERTS_P_MONITORS(BIF_P),
+                                       BIF_ARG_1);
+        if (mon) {
+            ASSERT(mon->type == ERTS_MON_TYPE_SUSPEND);
+            mdp = erts_monitor_to_data(mon);
+            msp = (ErtsMonitorSuspend *) mdp;
+            mstate = erts_atomic_read_nob(&msp->state);
+            ASSERT((mstate & ERTS_MSUSPEND_STATE_COUNTER_MASK) > 0);
+            mdp = NULL;
+            sync = !async & !(mstate & ERTS_MSUSPEND_STATE_FLG_ACTIVE);
+            suspend = 0;
+            res = am_false;
+        }
+        else {
+            mdp = erts_monitor_create(ERTS_MON_TYPE_SUSPEND, NIL,
+                                      BIF_P->common.id,
+                                      BIF_ARG_1, NIL);
+            mon = &mdp->origin;
+            erts_monitor_tree_insert(&ERTS_P_MONITORS(BIF_P), mon);
+            msp = (ErtsMonitorSuspend *) mdp;
+            mstate = erts_atomic_inc_read_relb(&msp->state);
+            ASSERT(!(mstate & ERTS_MSUSPEND_STATE_FLG_ACTIVE));
+            suspend = !0;
+            res = am_true;
+        }
     }
-#endif
 
-    erts_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
-    erts_proc_unlock(BIF_P, xlocks);
-    BIF_RET(res);
+    if (suspend) {
+        erts_aint32_t state;
+        Process *rp;
+        int send_sig = 0;
 
- system_limit:
-    ERTS_BIF_PREP_ERROR(res, BIF_P, SYSTEM_LIMIT);
-    goto do_return;
+        rp = erts_try_lock_sig_free_proc(BIF_ARG_1,
+                                         ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS,
+                                         &state);
+        if (!rp)
+            goto noproc;
+        if (rp == ERTS_PROC_LOCK_BUSY)
+            send_sig = !0;
+        else if (state & (ERTS_PSFLG_EXITING
+                          | ERTS_PSFLG_RUNNING
+                          | ERTS_PSFLG_RUNNING_SYS
+                          | ERTS_PSFLG_DIRTY_RUNNING
+                          | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
+            erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
+            send_sig = !0;
+        }
 
- no_suspendee: {
-        ErtsMonitor *mon;
-        BIF_P->suspendee = NIL;
-        mon = erts_monitor_tree_lookup(BIF_P->suspend_monitors, BIF_ARG_1);
-        if (mon) {
-            erts_monitor_tree_delete(&BIF_P->suspend_monitors, mon);
-            erts_monitor_suspend_destroy(erts_monitor_suspend(mon));
+        if (!send_sig && rp) {
+            send_sig = !suspend_process(BIF_P, rp);
+            if (!send_sig) {
+                erts_monitor_list_insert(&ERTS_P_LT_MONITORS(rp), &mdp->target);
+                erts_atomic_read_bor_relb(&msp->state,
+                                          ERTS_MSUSPEND_STATE_FLG_ACTIVE);
+            }
+            erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
+        }
+        if (send_sig) {
+            if (erts_proc_sig_send_monitor(&mdp->target, BIF_ARG_1))
+                sync = !async;
+            else {
+            noproc:
+                erts_monitor_tree_delete(&ERTS_P_MONITORS(BIF_P), &mdp->origin);
+                erts_monitor_release_both(mdp);
+                if (!async)
+                    res = am_badarg;
+            }
         }
     }
 
- badarg:
-    ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
-    goto do_return;
+    if (sync) {
+        ASSERT(is_non_value(reply_tag));
+        reply_res = res;
+        reply_tag = res = erts_make_ref(BIF_P);
+        ERTS_RECV_MARK_SAVE(BIF_P);
+        ERTS_RECV_MARK_SET(BIF_P);
+    }
 
- yield:
-    ERTS_BIF_PREP_YIELD2(res, bif_export[BIF_suspend_process_2],
-			 BIF_P, BIF_ARG_1, BIF_ARG_2);
-
- do_return:
-    if (suspendee)
-	erts_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
-    if (xlocks)
-	erts_proc_unlock(BIF_P, xlocks);
-    return res;
+    if (is_value(reply_tag))
+        erts_proc_sig_send_sync_suspend(BIF_P, BIF_ARG_1, reply_tag, reply_res);
 
+    BIF_RET(res);
 }
 
-
 /*
  * The erlang:resume_process/1 BIF
  */
@@ -9181,90 +8716,32 @@ BIF_RETTYPE
 resume_process_1(BIF_ALIST_1)
 {
     ErtsMonitor *mon;
-    ErtsMonitorSuspend *smon;
-    Process *suspendee;
-    int is_active;
+    ErtsMonitorSuspend *msp;
+    erts_aint_t mstate;
  
     if (BIF_P->common.id == BIF_ARG_1)
 	BIF_ERROR(BIF_P, BADARG);
 
-    erts_proc_lock(BIF_P, ERTS_PROC_LOCK_STATUS);
-    mon = erts_monitor_tree_lookup(BIF_P->suspend_monitors, BIF_ARG_1);
-    smon = erts_monitor_suspend(mon);
-
-    if (!smon) {
+    mon = erts_monitor_tree_lookup(ERTS_P_MONITORS(BIF_P),
+                                   BIF_ARG_1);
+    if (!mon) {
 	/* No previous suspend or dead suspendee */
-	goto error;
-    }
-    else if (smon->pending) {
-	smon->pending--;
-	ASSERT(smon->pending >= 0);
-	if (smon->active) {
-	    smon->active += smon->pending;
-	    smon->pending = 0;
-	}
-	is_active = smon->active;
-    }
-    else if (smon->active) {
-	smon->active--;
-	ASSERT(smon->pending == 0);
-	is_active = 1;
-    }
-    else {
-	/* No previous suspend or dead suspendee */
-	goto no_suspendee;
-    }
-
-    if (smon->active || smon->pending || !is_active) {
-	/* Leave the suspendee as it is; just verify that it is still alive */
-	suspendee = erts_proc_lookup(BIF_ARG_1);
-	if (!suspendee)
-	    goto no_suspendee;
-
+        BIF_ERROR(BIF_P, BADARG);
     }
-    else {
-	/* Resume */
-	suspendee = erts_pid2proc(BIF_P,
-				  ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS,
-				  BIF_ARG_1,
-				  ERTS_PROC_LOCK_STATUS);
-	if (!suspendee) {
-            mon = erts_monitor_tree_lookup(BIF_P->suspend_monitors, BIF_ARG_1);
-            smon = erts_monitor_suspend(mon);
-            if (!mon)
-                goto error;
-	    goto no_suspendee;
-        }
 
-        ASSERT(mon == erts_monitor_tree_lookup(BIF_P->suspend_monitors, BIF_ARG_1));
+    ASSERT(mon->type == ERTS_MON_TYPE_SUSPEND);
+    msp = (ErtsMonitorSuspend *) erts_monitor_to_data(mon);
 
-	ASSERT(ERTS_PSFLG_SUSPENDED
-	       & erts_atomic32_read_nob(&suspendee->state));
-	ASSERT(BIF_P != suspendee);
-	resume_process(suspendee, ERTS_PROC_LOCK_STATUS);
+    mstate = erts_atomic_dec_read_relb(&msp->state);
 
-	erts_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
-    }
+    ASSERT((mstate & ERTS_MSUSPEND_STATE_COUNTER_MASK) >= 0);
 
-    if (!smon->active && !smon->pending) {
-        ASSERT(mon);
-        erts_monitor_tree_delete(&BIF_P->suspend_monitors, mon);
-        erts_monitor_suspend_destroy(smon);
+    if ((mstate & ERTS_MSUSPEND_STATE_COUNTER_MASK) == 0) {
+        erts_monitor_tree_delete(&ERTS_P_MONITORS(BIF_P), mon);
+        erts_proc_sig_send_demonitor(mon);
     }
 
-    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS);
-
     BIF_RET(am_true);
-
- no_suspendee:
-    /* cleanup */
-    ASSERT(mon);
-    erts_monitor_tree_delete(&BIF_P->suspend_monitors, mon);
-    erts_monitor_suspend_destroy(smon);
-
- error:
-    erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS);
-    BIF_ERROR(BIF_P, BADARG);
 }
 
 BIF_RETTYPE
@@ -9694,12 +9171,13 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	ASSERT(esdp->current_process == p
 	       || esdp->free_process == p);
 
-    sched_out_proc:
-
-	ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
 	reds = actual_reds = calls - esdp->virtual_reds;
 
+    internal_sched_out_proc:
+
+	ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
+
 	ASSERT(actual_reds >= 0);
 	if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST)
 	    reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST;
@@ -9741,11 +9219,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
         /* have to re-read state after taking lock */
         state = erts_atomic32_read_nob(&p->state);
 
-	if (p->pending_suspenders) 
-	    handle_pending_suspend(p, (ERTS_PROC_LOCK_MAIN
-				       | ERTS_PROC_LOCK_TRACE
-				       | ERTS_PROC_LOCK_STATUS));
-
 	esdp->reductions += reds;
 
         {
@@ -10195,8 +9668,9 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
         if (is_normal_sched) {
             if (state & ERTS_PSFLG_RUNNING_SYS) {
                 if (state & (ERTS_PSFLG_SIG_Q|ERTS_PSFLG_SIG_IN_Q)) {
-                    int local_only = !!(p->flags & F_LOCAL_SIGS_ONLY);
-                    if (!local_only || (state & ERTS_PSFLG_SIG_Q)) {
+                    int local_only = (!!(p->flags & F_LOCAL_SIGS_ONLY)
+                                      & !(state & ERTS_PSFLG_SUSPENDED));
+                    if (!local_only | !!(state & ERTS_PSFLG_SIG_Q)) {
                         int sig_reds;
                         /*
                          * If we have dirty work scheduled we allow
@@ -10282,7 +9756,17 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	}
 
 	p->fcalls = reds;
-	    
+        if (reds != context_reds) {
+            actual_reds = context_reds - reds - esdp->virtual_reds;
+            ASSERT(actual_reds >= 0);
+            esdp->virtual_reds = 0;
+            p->reds += actual_reds;
+            ERTS_PROC_REDUCTIONS_EXECUTED(esdp, rq,
+                                          (int) ERTS_PSFLGS_GET_USR_PRIO(state),
+                                          reds,
+                                          actual_reds);
+        }
+
 	ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
 	ASSERT(erts_proc_read_refc(p) > 0);
@@ -10332,6 +9816,14 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 #endif
 
 	return p;
+
+    sched_out_proc:
+        actual_reds = context_reds;
+        actual_reds -= reds;
+        actual_reds -= esdp->virtual_reds;
+        reds = actual_reds;
+        goto internal_sched_out_proc;
+
     }
 }
 
@@ -11892,7 +11384,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     ERTS_P_LINKS(p) = NULL;
     ERTS_P_MONITORS(p) = NULL;
     ERTS_P_LT_MONITORS(p) = NULL;
-    p->suspend_monitors = NULL;
 
     ASSERT(is_pid(parent->group_leader));
 
@@ -11949,8 +11440,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 
     p->trace_msg_q = NULL;
     p->scheduler_data = NULL;
-    p->suspendee = NIL;
-    p->pending_suspenders = NULL;
 
 #if !defined(NO_FPE_SIGNALS) || defined(HIPE)
     p->fp_exception = 0;
@@ -12117,7 +11606,6 @@ void erts_init_empty_process(Process *p)
     ERTS_P_MONITORS(p) = NULL;
     ERTS_P_LT_MONITORS(p) = NULL;
     ERTS_P_LINKS(p) = NULL;         /* List of links */
-    p->suspend_monitors = NULL;
     p->sig_qs.first = NULL;
     p->sig_qs.last = &p->sig_qs.first;
     p->sig_qs.cont = NULL;
@@ -12180,8 +11668,6 @@ void erts_init_empty_process(Process *p)
     erts_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL);
 
     p->scheduler_data = NULL;
-    p->suspendee = NIL;
-    p->pending_suspenders = NULL;
     erts_proc_lock_init(p);
     erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
     erts_init_runq_proc(p, ERTS_RUNQ_IX(0), 0);
@@ -12219,7 +11705,6 @@ erts_debug_verify_clean_empty_process(Process* p)
     ASSERT(ERTS_P_MONITORS(p) == NULL);
     ASSERT(ERTS_P_LT_MONITORS(p) == NULL);
     ASSERT(ERTS_P_LINKS(p) == NULL);
-    ASSERT(p->suspend_monitors == NULL);
     ASSERT(p->sig_qs.first == NULL);
     ASSERT(p->sig_qs.len == 0);
     ASSERT(p->bif_timers == NULL);
@@ -12233,8 +11718,6 @@ erts_debug_verify_clean_empty_process(Process* p)
 
     ASSERT(p->sig_inq.first == NULL);
     ASSERT(p->sig_inq.len == 0);
-    ASSERT(p->suspendee == NIL);
-    ASSERT(p->pending_suspenders == NULL);
 
     /* Thing that erts_cleanup_empty_process() cleans up */
 
@@ -12340,8 +11823,6 @@ delete_process(Process* p)
     erts_cleanup_messages(p->sig_qs.cont);
     p->sig_qs.cont = NULL;
 
-    ASSERT(!p->suspend_monitors);
-
     p->fvalue = NIL;
 }
 
@@ -12421,6 +11902,7 @@ erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt)
     if (erts_monitor_is_target(mon)) {
         /* We are being watched... */
         switch (mon->type) {
+        case ERTS_MON_TYPE_SUSPEND:
         case ERTS_MON_TYPE_PROC:
             erts_proc_sig_send_monitor_down(mon, reason);
             mon = NULL;
@@ -12492,6 +11974,7 @@ erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt)
     else { /* Origin monitor */
         /* We are watching someone else... */
         switch (mon->type) {
+        case ERTS_MON_TYPE_SUSPEND:
         case ERTS_MON_TYPE_PROC:
             erts_proc_sig_send_demonitor(mon);
             mon = NULL;
@@ -12644,21 +12127,6 @@ erts_proc_exit_handle_link(ErtsLink *lnk, void *vctxt)
         erts_link_release(lnk);
 }
 
-static void
-resume_suspend_monitor(ErtsMonitor *mon, void *vc_p)
-{
-    ErtsMonitorSuspend *smon = erts_monitor_suspend(mon);
-    Process *suspendee = erts_pid2proc((Process *) vc_p, ERTS_PROC_LOCK_MAIN,
-				       smon->mon.other.item, ERTS_PROC_LOCK_STATUS);
-    if (suspendee) {
-	ASSERT(suspendee != vc_p);
-	if (smon->active)
-	    resume_process(suspendee, ERTS_PROC_LOCK_STATUS);
-	erts_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS);
-    }
-    erts_monitor_suspend_destroy(smon);
-}
-
 /* this function fishishes a process and propagates exit messages - called
    by process_main when a process dies */
 void 
@@ -12690,8 +12158,6 @@ erts_do_exit_process(Process* p, Eterm reason)
 
     set_self_exiting(p, reason, NULL, NULL, NULL);
 
-    cancel_suspend_of_suspendee(p, ERTS_PROC_LOCKS_ALL); 
-
     if (IS_TRACED(p)) {
 	if (IS_TRACED_FL(p, F_TRACE_CALLS))
 	    erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_EXITING);
@@ -12824,11 +12290,6 @@ erts_continue_exit_process(Process *p)
 	p->flags &= ~F_USING_DDLL;
     }
 
-    if (p->suspend_monitors)
-        erts_monitor_tree_foreach_delete(&p->suspend_monitors,
-                                         resume_suspend_monitor,
-                                         p);
-
     /*
      * The registered name *should* be the last "erlang resource" to
      * cleanup.
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 1e9f9b07b7..a60e117bab 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -805,14 +805,15 @@ erts_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
 #define ERTS_PSD_ETS_OWNED_TABLES               6
 #define ERTS_PSD_ETS_FIXED_TABLES               7
 #define ERTS_PSD_DIST_ENTRY	                8
-#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF	9 /* keep last... */
+#define ERTS_PSD_PENDING_SUSPEND                9
+#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF	10 /* keep last... */
 
-#define ERTS_PSD_SIZE				10
+#define ERTS_PSD_SIZE				11
 
 #if !defined(HIPE)
 #  undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF
 #  undef ERTS_PSD_SIZE
-#  define ERTS_PSD_SIZE 9
+#  define ERTS_PSD_SIZE 10
 #endif
 
 typedef struct {
@@ -849,6 +850,9 @@ typedef struct {
 #define ERTS_PSD_DIST_ENTRY_GET_LOCKS ERTS_PROC_LOCK_MAIN
 #define ERTS_PSD_DIST_ENTRY_SET_LOCKS ERTS_PROC_LOCK_MAIN
 
+#define ERTS_PSD_PENDING_SUSPEND_GET_LOCKS ERTS_PROC_LOCK_MAIN
+#define ERTS_PSD_PENDING_SUSPEND_SET_LOCKS ERTS_PROC_LOCK_MAIN
+
 typedef struct {
     ErtsProcLocks get_locks;
     ErtsProcLocks set_locks;
@@ -884,20 +888,6 @@ typedef struct {
 typedef struct ErtsProcSysTask_ ErtsProcSysTask;
 typedef struct ErtsProcSysTaskQs_ ErtsProcSysTaskQs;
 
-
-typedef struct ErtsPendingSuspend_ ErtsPendingSuspend;
-struct ErtsPendingSuspend_ {
-    ErtsPendingSuspend *next;
-    ErtsPendingSuspend *end;
-    Eterm pid;
-    void (*handle_func)(Process *suspendee,
-			ErtsProcLocks suspendee_locks,
-			int suspendee_alive,
-			Eterm pid);
-};
-
-
-
 /* Defines to ease the change of memory architecture */
 #  define HEAP_START(p)     (p)->heap
 #  define HEAP_TOP(p)       (p)->htop
@@ -992,9 +982,6 @@ struct process {
 
     Process *next;		/* Pointer to next process in run queue */
 
-    ErtsMonitor *suspend_monitors; /* Processes suspended by this process via
-                                      erlang:suspend_process/1 */
-
     ErtsSignalPrivQueues sig_qs; /* Signal queues */
     ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
 
@@ -1058,8 +1045,6 @@ struct process {
     ErlTraceMessageQueue *trace_msg_q;
     erts_proc_lock_t lock;
     ErtsSchedulerData *scheduler_data;
-    Eterm suspendee;
-    ErtsPendingSuspend *pending_suspenders;
     erts_atomic_t run_queue;
 
 #ifdef CHECK_FOR_HOLES
@@ -1377,7 +1362,7 @@ extern int erts_system_profile_ts_type;
 #define F_DISTRIBUTION       (1 <<  6) /* Process used in distribution */
 #define F_USING_DDLL         (1 <<  7) /* Process has used the DDLL interface */
 #define F_HAVE_BLCKD_MSCHED  (1 <<  8) /* Process has blocked multi-scheduling */
-#define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
+#define F_UNUSED             (1 <<  9)
 #define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
 #define F_DISABLE_GC         (1 << 11) /* Disable GC (see below) */
 #define F_OFF_HEAP_MSGQ      (1 << 12) /* Off heap msg queue */
@@ -2047,6 +2032,11 @@ erts_psd_set(Process *p, int ix, void *data)
 #define ERTS_PROC_SET_DIST_ENTRY(P, DE) \
     ((DistEntry *) erts_psd_set((P), ERTS_PSD_DIST_ENTRY, (void *) (DE)))
 
+#define ERTS_PROC_GET_PENDING_SUSPEND(P) \
+    ((void *) erts_psd_get((P), ERTS_PSD_PENDING_SUSPEND))
+#define ERTS_PROC_SET_PENDING_SUSPEND(P, PS) \
+    ((void *) erts_psd_set((P), ERTS_PSD_PENDING_SUSPEND, (void *) (PS)))
+
 #ifdef HIPE
 #define ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(P) \
   ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF))
@@ -2611,16 +2601,6 @@ Process *erts_try_lock_sig_free_proc(Eterm pid,
                                      ErtsProcLocks locks,
                                      erts_aint32_t *statep);
 
-Process *erts_pid2proc_not_running(Process *,
-				   ErtsProcLocks,
-				   Eterm,
-				   ErtsProcLocks);
-Process *erts_pid2proc_nropt(Process *c_p,
-			     ErtsProcLocks c_p_locks,
-			     Eterm pid,
-			     ErtsProcLocks pid_locks);
-extern int erts_disable_proc_not_running_opt;
-
 #ifdef DEBUG
 #define ERTS_ASSERT_IS_NOT_EXITING(P) \
     do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0)
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 065a560b52..f074dd8bdb 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -2629,6 +2629,38 @@ erts_tracer_to_term(Process *p, ErtsTracer tracer)
     }
 }
 
+Eterm
+erts_build_tracer_to_term(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, ErtsTracer tracer)
+{
+    Eterm res;
+    Eterm state;
+    Uint sz;
+
+    if (ERTS_TRACER_IS_NIL(tracer))
+        return am_false;
+
+    state = ERTS_TRACER_STATE(tracer);
+    sz = is_immed(state) ? 0 : size_object(state);
+
+    if (szp)
+        *szp += sz;
+
+    if (hpp)
+        res = is_immed(state) ? state : copy_struct(state, sz, hpp, ohp);
+    else
+        res = THE_NON_VALUE;
+
+    if (ERTS_TRACER_MODULE(tracer) != am_erl_tracer) {
+        if (szp)
+            *szp += 3;
+        if (hpp) {
+            res = TUPLE2(*hpp, ERTS_TRACER_MODULE(tracer), res);
+            *hpp += 3;
+        }
+    }
+
+    return res;
+}
 
 static ERTS_INLINE int
 send_to_tracer_nif_raw(Process *c_p, Process *tracee,
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index dbf7ebd2a1..3228e19809 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -198,6 +198,8 @@ int erts_is_tracer_proc_enabled_send(Process* c_p, ErtsProcLocks c_p_locks,
                                      ErtsPTabElementCommon *t_p);
 int erts_is_tracer_enabled(const ErtsTracer tracer, ErtsPTabElementCommon *t_p);
 Eterm erts_tracer_to_term(Process *p, ErtsTracer tracer);
+Eterm erts_build_tracer_to_term(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, ErtsTracer tracer);
+
 ErtsTracer erts_term_to_tracer(Eterm prefix, Eterm term);
 void erts_tracer_replace(ErtsPTabElementCommon *t_p,
                          const ErtsTracer new_tracer);
diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl
index def25dba7d..138aefb29c 100644
--- a/erts/emulator/test/trace_SUITE.erl
+++ b/erts/emulator/test/trace_SUITE.erl
@@ -29,7 +29,7 @@
          receive_trace/1, link_receive_call_correlation/1, self_send/1,
 	 timeout_trace/1, send_trace/1,
 	 procs_trace/1, dist_procs_trace/1, procs_new_trace/1,
-	 suspend/1, mutual_suspend/1, suspend_exit/1, suspender_exit/1,
+	 suspend/1, suspend_exit/1, suspender_exit/1,
 	 suspend_system_limit/1, suspend_opts/1, suspend_waiting/1,
 	 new_clear/1, existing_clear/1, tracer_die/1,
 	 set_on_spawn/1, set_on_first_spawn/1, cpu_timestamp/1,
@@ -53,7 +53,7 @@ all() ->
     [cpu_timestamp, receive_trace, link_receive_call_correlation,
      self_send, timeout_trace,
      send_trace, procs_trace, dist_procs_trace, suspend,
-     mutual_suspend, suspend_exit, suspender_exit,
+     suspend_exit, suspender_exit,
      suspend_system_limit, suspend_opts, suspend_waiting,
      new_clear, existing_clear, tracer_die, set_on_spawn,
      set_on_first_spawn, set_on_link, set_on_first_link,
@@ -1234,55 +1234,6 @@ do_suspend(Pid, N) ->
     erlang:yield(),
     do_suspend(Pid, N-1).
 
-
-
-mutual_suspend(Config) when is_list(Config) ->
-    TimeoutSecs = 5*60,
-    ct:timetrap({seconds, TimeoutSecs}),
-    Parent = self(),
-    Fun = fun () ->
-                  receive
-                      {go, Pid} ->
-                          do_mutual_suspend(Pid, 100000)
-                  end,
-                  Parent ! {done, self()},
-                  receive after infinity -> ok end
-          end,
-    P1 = spawn_link(Fun),
-    P2 = spawn_link(Fun),
-    T1 = erlang:start_timer((TimeoutSecs - 5)*1000, self(), oops),
-    T2 = erlang:start_timer((TimeoutSecs - 5)*1000, self(), oops),
-    P1 ! {go, P2},
-    P2 ! {go, P1},
-    Res1 = receive
-               {done, P1} -> done;
-               {timeout,T1,_} -> timeout
-           end,
-    Res2 = receive
-               {done, P2} -> done;
-               {timeout,T2,_} -> timeout
-           end,
-    P1S = process_info(P1, status),
-    P2S = process_info(P2, status),
-    io:format("P1S=~p P2S=~p", [P1S, P2S]),
-    false = {status, suspended} == P1S,
-    false = {status, suspended} == P2S,
-    unlink(P1), exit(P1, bang),
-    unlink(P2), exit(P2, bang),
-    done = Res1,
-    done = Res2,
-    ok.
-
-do_mutual_suspend(_Pid, 0) ->
-    ok;
-do_mutual_suspend(Pid, N) ->
-    %% Suspend a process and test that it is suspended.
-    true = erlang:suspend_process(Pid),
-    {status, suspended} = process_info(Pid, status),
-    %% Unsuspend the process.
-    true = erlang:resume_process(Pid),
-    do_mutual_suspend(Pid, N-1).		
-
 suspend_exit(Config) when is_list(Config) ->
     ct:timetrap({minutes, 2}),
     rand:seed(exsplus, {4711,17,4711}),
@@ -1513,7 +1464,8 @@ suspend_opts(Config) when is_list(Config) ->
                              dbl_async = AA,
                              synced = S,
                              async_once = AO} = Acc) ->
-                 erlang:suspend_process(Tok, [asynchronous]),
+                 Tag = {make_ref(), self()},
+                 erlang:suspend_process(Tok, [{asynchronous, Tag}]),
                  Res = case {suspend_count(Tok), N rem 4} of
                            {0, 2} ->
                                erlang:suspend_process(Tok,
@@ -1549,7 +1501,11 @@ suspend_opts(Config) when is_list(Config) ->
                            _ ->
                                Acc
                        end,
-                 erlang:resume_process(Tok),
+                 receive
+                     {Tag, Result} ->
+                         suspended = Result,
+                         erlang:resume_process(Tok)
+                 end,
                  erlang:yield(),
                  Res
          end,
diff --git a/erts/emulator/test/tracer_SUITE.erl b/erts/emulator/test/tracer_SUITE.erl
index e1362ef07a..070462b0f1 100644
--- a/erts/emulator/test/tracer_SUITE.erl
+++ b/erts/emulator/test/tracer_SUITE.erl
@@ -623,7 +623,7 @@ test(Event, TraceFlag, Tc, Expect, _Removes, Dies) ->
 
     Expect(Pid1, State1, Opts),
     receive M11 -> ct:fail({unexpected, M11}) after 0 -> ok end,
-    if not Dies ->
+    if not Dies andalso Event /= in ->
             {flags, [TraceFlag]} = erlang:trace_info(Pid1, flags),
             {tracer, {tracer_test, State1}} = erlang:trace_info(Pid1, tracer),
             erlang:trace(Pid1, false, [TraceFlag]);
@@ -640,7 +640,7 @@ test(Event, TraceFlag, Tc, Expect, _Removes, Dies) ->
     Expect(Pid1T, State1, Opts#{ scheduler_id => number,
                                  timestamp => timestamp}),
     receive M11T -> ct:fail({unexpected, M11T}) after 0 -> ok end,
-    if not Dies ->
+    if not Dies andalso Event /= in ->
             {flags, [scheduler_id, TraceFlag, timestamp]}
                 = erlang:trace_info(Pid1T, flags),
             {tracer, {tracer_test, State1}} = erlang:trace_info(Pid1T, tracer),
@@ -655,7 +655,7 @@ test(Event, TraceFlag, Tc, Expect, _Removes, Dies) ->
     Tc(Pid2),
     ok = trace_delivered(Pid2),
     receive M2 -> ct:fail({unexpected, M2}) after 0 -> ok end,
-    if not Dies ->
+    if not Dies andalso Event /= in ->
             {flags, [TraceFlag]} = erlang:trace_info(Pid2, flags),
             {tracer, {tracer_test, State2}} = erlang:trace_info(Pid2, tracer),
             erlang:trace(Pid2, false, [TraceFlag]);
-- 
cgit v1.2.3


From bec8b2e08a6e6e404a940c4095133a18ead5e28e Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 7 May 2018 16:57:12 +0200
Subject: Replace previous suspend in setnode/3

---
 erts/emulator/beam/bif.h           |  37 ++++
 erts/emulator/beam/bif.tab         |   3 +-
 erts/emulator/beam/dist.c          | 406 +++++++++++++++++++++++--------------
 erts/emulator/beam/erl_alloc.types |   1 +
 4 files changed, 292 insertions(+), 155 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index a47339253e..cf9f61c0b8 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -295,6 +295,19 @@ do {								\
     (Ret) = THE_NON_VALUE;					\
 } while (0)
 
+#define ERTS_BIF_PREP_TRAP4(Ret, Trap, Proc, A0, A1, A2, A3)	\
+do {								\
+    Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
+    (Proc)->arity = 4;						\
+    reg[0] = (Eterm) (A0);					\
+    reg[1] = (Eterm) (A1);					\
+    reg[2] = (Eterm) (A2);					\
+    reg[3] = (Eterm) (A3);					\
+    (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \
+    (Proc)->freason = TRAP;					\
+    (Ret) = THE_NON_VALUE;					\
+} while (0)
+
 #define ERTS_BIF_PREP_TRAP3_NO_RET(Trap, Proc, A0, A1, A2)\
 do {							\
     Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array;	\
@@ -343,6 +356,18 @@ do {							\
       return THE_NON_VALUE;					\
  } while(0)
 
+#define BIF_TRAP4(Trap_, p, A0, A1, A2, A3) do {		\
+      Eterm* reg = erts_proc_sched_data((p))->x_reg_array;	\
+      (p)->arity = 4;						\
+      reg[0] = (A0);						\
+      reg[1] = (A1);						\
+      reg[2] = (A2);						\
+      reg[3] = (A3);						\
+      (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \
+      (p)->freason = TRAP;					\
+      return THE_NON_VALUE;					\
+ } while(0)
+
 #define BIF_TRAP_CODE_PTR_0(p, Code_) do {	\
       (p)->arity = 0;				\
       (p)->i = (BeamInstr*) (Code_);		\
@@ -401,6 +426,12 @@ do {									\
     ERTS_BIF_PREP_TRAP3(RET, (TRP), (P), (A0), (A1), (A2));		\
 } while (0)
 
+#define ERTS_BIF_PREP_YIELD4(RET, TRP, P, A0, A1, A2, A3)		\
+do {									\
+    ERTS_VBUMP_ALL_REDS((P));						\
+    ERTS_BIF_PREP_TRAP4(RET, (TRP), (P), (A0), (A1), (A2), (A3));       \
+} while (0)
+
 #define ERTS_BIF_YIELD0(TRP, P)						\
 do {									\
     ERTS_VBUMP_ALL_REDS((P));						\
@@ -425,6 +456,12 @@ do {									\
     BIF_TRAP3((TRP), (P), (A0), (A1), (A2));				\
 } while (0)
 
+#define ERTS_BIF_YIELD4(TRP, P, A0, A1, A2, A3)				\
+do {									\
+    ERTS_VBUMP_ALL_REDS((P));						\
+    BIF_TRAP4((TRP), (P), (A0), (A1), (A2), (A3));                      \
+} while (0)
+
 #define ERTS_BIF_PREP_EXITED(RET, PROC)	                                \
 do {                                                                    \
     KILL_CATCHES((PROC));                                               \
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 7387ca2523..21dda9ff03 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -154,7 +154,6 @@ bif erlang:unregister/1
 bif erlang:whereis/1
 bif erlang:spawn_opt/1
 bif erlang:setnode/2
-bif erlang:setnode/3
 bif erlang:dist_get_stat/1
 bif erlang:dist_ctrl_input_handler/2
 bif erlang:dist_ctrl_put_data/2
@@ -191,6 +190,8 @@ bif erts_internal:scheduler_wall_time/1
 
 bif erts_internal:dirty_process_handle_signals/1
 
+bif erts_internal:create_dist_channel/4
+
 # inet_db support
 bif erlang:port_set_data/2
 bif erlang:port_get_data/1
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index f1566cb7b4..70474898b2 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -3138,60 +3138,60 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2)
     BIF_ERROR(BIF_P, BADARG);
 }
 
-/**********************************************************************
- ** Allocate a dist entry, set node name install the connection handler
- ** setnode_3({name@host, Creation}, Cid, {Type, Version, Initial, IC, OC})
- ** Type = flag field, where the flags are specified in dist.h
- ** Version = distribution version, >= 1
- ** IC = in_cookie (ignored)
- ** OC = out_cookie (ignored)
- **
- ** Note that in distribution protocols above 1, the Initial parameter
- ** is always NIL and the cookies are always the atom '', cookies are not
- ** sent in the distribution messages but are only used in 
- ** the handshake.
- **
- ***********************************************************************/
+/*
+ * erts_internal:create_dist_channel/4 is used by
+ * erlang:setnode/3.
+ */
+
+typedef struct {
+    DistEntry *dep;
+    Uint flags;
+    Uint version;
+} ErtsSetupConnDistCtrl;
+
+static void
+setup_connection_epiloge_rwunlock(Process *c_p, DistEntry *dep,
+                                  Eterm ctrlr, Uint flags,
+                                  Uint version);
 
-BIF_RETTYPE setnode_3(BIF_ALIST_3)
+static Eterm
+setup_connection_distctrl(Process *c_p, void *arg,
+                          int *redsp, ErlHeapFragment **bpp);
+
+BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4)
 {
     BIF_RETTYPE ret;
     Uint flags;
-    unsigned long version;
-    Eterm ic, oc;
-    Eterm *tp;
+    Uint version;
+    Eterm *hp, res_tag = THE_NON_VALUE, res = THE_NON_VALUE;
     DistEntry *dep = NULL;
-    ErtsProcLocks proc_unlock = 0;
-    Process *proc;
+    int de_locked = 0;
     Port *pp = NULL;
-    Eterm notify_proc;
-    erts_aint32_t qflgs;
 
     /*
      * Check and pick out arguments
      */
 
-    if (!is_node_name_atom(BIF_ARG_1) ||
-	!(is_internal_port(BIF_ARG_2)
-          || is_internal_pid(BIF_ARG_2))
-	|| (erts_this_node->sysname == am_Noname)) {
-	goto badarg;
-    }
+    /* Node name... */
+    if (!is_node_name_atom(BIF_ARG_1))
+        goto badarg;
 
-    if (!is_tuple(BIF_ARG_3))
-	goto badarg;
-    tp = tuple_val(BIF_ARG_3);
-    if (*tp++ != make_arityval(4))
-	goto badarg;
-    if (!is_small(*tp))
-	goto badarg;
-    flags = unsigned_val(*tp++);
-    if (!is_small(*tp) || (version = unsigned_val(*tp)) == 0)
-	goto badarg;
-    ic = *(++tp);
-    oc = *(++tp);
-    if (!is_atom(ic) || !is_atom(oc))
-	goto badarg;
+    /* Distribution controller... */
+    if (!is_internal_port(BIF_ARG_2) && !is_internal_pid(BIF_ARG_2))
+        goto badarg;
+
+    /* Dist flags... */
+    if (!is_small(BIF_ARG_3))
+        goto badarg;
+    flags = unsigned_val(BIF_ARG_3);
+
+    /* Version... */
+    if (!is_small(BIF_ARG_4))
+        goto badarg;
+    version = unsigned_val(BIF_ARG_4);
+
+    if (version == 0)
+        goto badarg;
 
     if (~flags & DFLAG_DIST_MANDATORY) {
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
@@ -3222,74 +3222,79 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     else if (!dep)
 	goto system_limit; /* Should never happen!!! */
 
+    erts_de_rlock(dep);
+    de_locked = -1;
+
+    if (dep->state == ERTS_DE_STATE_EXITING) {
+        /* Suspend on dist entry waiting for the exit to finish */
+        ErtsProcList *plp = erts_proclist_create(BIF_P);
+        plp->next = NULL;
+        erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
+        erts_mtx_lock(&dep->qlock);
+        erts_proclist_store_last(&dep->suspended, plp);
+        erts_mtx_unlock(&dep->qlock);
+        goto yield;
+    }
+
+    erts_de_runlock(dep);
+    de_locked = 0;
+
     if (is_internal_pid(BIF_ARG_2)) {
         if (BIF_P->common.id == BIF_ARG_2) {
-            proc_unlock = 0;
-            proc = BIF_P;
-        }
-        else {
-            proc_unlock = ERTS_PROC_LOCK_MAIN;
-            proc = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
-                                 BIF_ARG_2, proc_unlock);
-        }
-        erts_de_rwlock(dep);
-
-        if (!proc)
-            goto badarg;
-        else if (proc == ERTS_PROC_LOCK_BUSY) {
-            proc_unlock = 0;
-            goto yield;
-        }
+            ErtsSetupConnDistCtrl scdc;
 
-        erts_proc_lock(proc, ERTS_PROC_LOCK_STATUS);
-        proc_unlock |= ERTS_PROC_LOCK_STATUS;
+            scdc.dep = dep;
+            scdc.flags = flags;
+            scdc.version = version;
 
-        if (ERTS_PROC_GET_DIST_ENTRY(proc)) {
-            if (dep == ERTS_PROC_GET_DIST_ENTRY(proc)
-                && (proc->flags & F_DISTRIBUTION)
-                && dep->cid == BIF_ARG_2) {
-                ERTS_BIF_PREP_RET(ret, erts_make_dhandle(BIF_P, dep));
-                goto done;
-            }
-            goto badarg;
-        }
+            res = setup_connection_distctrl(BIF_P, &scdc, NULL, NULL);
+            BUMP_REDS(BIF_P, 5);
+            dep = NULL;
 
-        if (dep->state == ERTS_DE_STATE_EXITING) {
-            /* Suspend on dist entry waiting for the exit to finish */
-            ErtsProcList *plp = erts_proclist_create(BIF_P);
-            plp->next = NULL;
-            erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
-            erts_mtx_lock(&dep->qlock);
-            erts_proclist_store_last(&dep->suspended, plp);
-            erts_mtx_unlock(&dep->qlock);
-            goto yield;
-        }
-        if (dep->state != ERTS_DE_STATE_PENDING) {
-            if (dep->state == ERTS_DE_STATE_IDLE)
-                erts_set_dist_entry_pending(dep);
-            else
+            if (res == am_badarg)
                 goto badarg;
+
+            ASSERT(is_internal_magic_ref(res));
+            res_tag = am_ok; /* Connection up */
         }
+        else {
+            ErtsSetupConnDistCtrl *scdcp;
 
-        if (is_not_nil(dep->cid))
-            goto badarg;
+            scdcp = erts_alloc(ERTS_ALC_T_SETUP_CONN_ARG,
+                               sizeof(ErtsSetupConnDistCtrl));
 
-        proc->flags |= F_DISTRIBUTION;
-        ERTS_PROC_SET_DIST_ENTRY(proc, dep);
+            scdcp->dep = dep;
+            scdcp->flags = flags;
+            scdcp->version = version;
 
-        proc_unlock &= ~ERTS_PROC_LOCK_STATUS;
-        erts_proc_unlock(proc, ERTS_PROC_LOCK_STATUS);
+            res = erts_proc_sig_send_rpc_request(BIF_P,
+                                                 BIF_ARG_2,
+                                                 !0,
+                                                 setup_connection_distctrl,
+                                                 (void *) scdcp);
+            if (is_non_value(res))
+                goto badarg;
 
-        dep->send = NULL; /* Only for distr ports... */
+            dep = NULL;
 
+            ASSERT(is_internal_ordinary_ref(res));
+
+            res_tag = am_message; /* Caller need to wait for dhandle in message */
+        }
+        hp = HAlloc(BIF_P, 3);
     }
     else {
+        int new;
 
         pp = erts_id2port_sflgs(BIF_ARG_2,
                                 BIF_P,
                                 ERTS_PROC_LOCK_MAIN,
                                 ERTS_PORT_SFLGS_INVALID_LOOKUP);
         erts_de_rwlock(dep);
+        de_locked = 1;
+
+        if (dep->state == ERTS_DE_STATE_EXITING)
+            goto badarg;
 
         if (!pp || (erts_atomic32_read_nob(&pp->state)
                     & ERTS_PORT_SFLG_EXITING))
@@ -3298,65 +3303,108 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
         if ((pp->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY) == 0)
             goto badarg;
 
-        if (dep->cid == BIF_ARG_2 && pp->dist_entry == dep) {
-            ERTS_BIF_PREP_RET(ret, erts_make_dhandle(BIF_P, dep));
-            goto done; /* Already set */
-        }
+        if (dep->cid == BIF_ARG_2 && pp->dist_entry == dep)
+            new = 0;
+        else {
+            if (dep->state != ERTS_DE_STATE_PENDING) {
+                if (dep->state == ERTS_DE_STATE_IDLE)
+                    erts_set_dist_entry_pending(dep);
+                else
+                    goto badarg;
+            }
 
-        if (dep->state == ERTS_DE_STATE_EXITING) {
-            /* Suspend on dist entry waiting for the exit to finish */
-            ErtsProcList *plp = erts_proclist_create(BIF_P);
-            plp->next = NULL;
-            erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
-            erts_mtx_lock(&dep->qlock);
-            erts_proclist_store_last(&dep->suspended, plp);
-            erts_mtx_unlock(&dep->qlock);
-            goto yield;
-        }
-        if (dep->state != ERTS_DE_STATE_PENDING) {
-            if (dep->state == ERTS_DE_STATE_IDLE)
-                erts_set_dist_entry_pending(dep);
-            else
+            if (pp->dist_entry || is_not_nil(dep->cid))
                 goto badarg;
-        }
 
-        if (pp->dist_entry || is_not_nil(dep->cid))
-            goto badarg;
+            erts_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION);
 
-        erts_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION);
+            pp->dist_entry = dep;
 
-        pp->dist_entry = dep;
+            ASSERT(pp->drv_ptr->outputv || pp->drv_ptr->output);
 
-        ASSERT(pp->drv_ptr->outputv || pp->drv_ptr->output);
+            dep->send = (pp->drv_ptr->outputv
+                         ? dist_port_commandv
+                         : dist_port_command);
+            ASSERT(dep->send);
 
-        dep->send = (pp->drv_ptr->outputv
-                     ? dist_port_commandv
-                     : dist_port_command);
-        ASSERT(dep->send);
+            /*
+             * Dist-ports do not use the "busy port message queue" functionality, but
+             * instead use "busy dist entry" functionality.
+             */
+            {
+                ErlDrvSizeT disable = ERL_DRV_BUSY_MSGQ_DISABLED;
+                erl_drv_busy_msgq_limits(ERTS_Port2ErlDrvPort(pp), &disable, NULL);
+            }
 
-        /*
-         * Dist-ports do not use the "busy port message queue" functionality, but
-         * instead use "busy dist entry" functionality.
-         */
-        {
-            ErlDrvSizeT disable = ERL_DRV_BUSY_MSGQ_DISABLED;
-            erl_drv_busy_msgq_limits(ERTS_Port2ErlDrvPort(pp), &disable, NULL);
+            setup_connection_epiloge_rwunlock(BIF_P, dep, BIF_ARG_2, flags, version);
+            de_locked = 0;
+            new = !0;
         }
 
+        hp = HAlloc(BIF_P, 3 + ERTS_MAGIC_REF_THING_SIZE);
+        res = erts_build_dhandle(&hp, &BIF_P->off_heap, dep);
+        res_tag = am_ok; /* Connection up */
+        if (new)
+            dep = NULL; /* inc of refc transferred to port (dist_entry field) */
+    }
+
+    ASSERT(is_value(res) && is_value(res_tag));
+
+    res = TUPLE2(hp, res_tag, res);
+
+    ERTS_BIF_PREP_RET(ret, res);
+
+ done:
+
+    if (dep && dep != erts_this_dist_entry) {
+        if (de_locked) {
+            if (de_locked > 0)
+                erts_de_rwunlock(dep);
+            else
+                erts_de_runlock(dep);
+        }
+	erts_deref_dist_entry(dep);
     }
 
+    if (pp)
+	erts_port_release(pp);
+
+    return ret;
+
+ yield:
+    ERTS_BIF_PREP_YIELD4(ret,
+                         bif_export[BIF_erts_internal_create_dist_channel_4],
+                         BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BIF_ARG_4);
+    goto done;
+
+ badarg:
+    ERTS_BIF_PREP_RET(ret, am_badarg);
+    goto done;
+
+ system_limit:
+    ERTS_BIF_PREP_RET(ret, am_system_limit);
+    goto done;
+}
+
+static void
+setup_connection_epiloge_rwunlock(Process *c_p, DistEntry *dep,
+                                  Eterm ctrlr, Uint flags,
+                                  Uint version)
+{
+    Eterm notify_proc = NIL;
+    erts_aint32_t qflgs;
+
     dep->version = version;
     dep->creation = 0;
 
-#ifdef DEBUG
+    ASSERT(is_internal_port(ctrlr) || is_internal_pid(ctrlr));
     ASSERT(erts_atomic_read_nob(&dep->qsize) == 0 
            || (dep->state == ERTS_DE_STATE_PENDING));
-#endif
 
     if (flags & DFLAG_DIST_HDR_ATOM_CACHE)
 	create_cache(dep);
 
-    erts_set_dist_entry_connected(dep, BIF_ARG_2, flags);
+    erts_set_dist_entry_connected(dep, ctrlr, flags);
 
     notify_proc = NIL;
     if (erts_atomic_read_nob(&dep->qsize)) {
@@ -3375,50 +3423,100 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
             }
         }
     }
-    erts_de_rwunlock(dep);
-    if (is_internal_pid(notify_proc))
-        notify_dist_data(BIF_P, notify_proc);
 
-    ERTS_BIF_PREP_RET(ret, erts_make_dhandle(BIF_P, dep));
+    erts_de_rwunlock(dep);
 
-    dep = NULL; /* inc of refc transferred to port (dist_entry field) */
+    if (is_internal_pid(notify_proc))
+        notify_dist_data(c_p, notify_proc);
 
     inc_no_nodes();
 
-    send_nodes_mon_msgs(BIF_P,
+    send_nodes_mon_msgs(c_p,
 			am_nodeup,
-			BIF_ARG_1,
+			dep->sysname,
 			flags & DFLAG_PUBLISHED ? am_visible : am_hidden,
 			NIL);
- done:
+}
 
-    if (dep && dep != erts_this_dist_entry) {
-	erts_de_rwunlock(dep);
-	erts_deref_dist_entry(dep);
+static Eterm
+setup_connection_distctrl(Process *c_p, void *arg, int *redsp, ErlHeapFragment **bpp)
+{
+    ErtsSetupConnDistCtrl *scdcp = (ErtsSetupConnDistCtrl *) arg;
+    DistEntry *dep = scdcp->dep;
+    int dep_locked = 0;
+    Eterm *hp;
+    erts_aint32_t state;
+
+    if (redsp)
+        *redsp = 1;
+
+    state = erts_atomic32_read_nob(&c_p->state);
+
+    if (state & ERTS_PSFLG_EXITING)
+        goto badarg;
+
+    erts_de_rwlock(dep);
+    dep_locked = !0;
+
+    if (dep->state == ERTS_DE_STATE_EXITING)
+        goto badarg;
+
+    if (ERTS_PROC_GET_DIST_ENTRY(c_p)) {
+        if (dep == ERTS_PROC_GET_DIST_ENTRY(c_p)
+            && (c_p->flags & F_DISTRIBUTION)
+            && dep->cid == c_p->common.id) {
+            goto connected;
+        }
+        goto badarg;
     }
 
-    if (pp)
-	erts_port_release(pp);
+    if (dep->state != ERTS_DE_STATE_PENDING) {
+        if (dep->state == ERTS_DE_STATE_IDLE)
+            erts_set_dist_entry_pending(dep);
+        else
+            goto badarg;
+    }
 
-    if (proc_unlock)
-        erts_proc_unlock(proc, proc_unlock);
+    if (is_not_nil(dep->cid))
+        goto badarg;
 
-    return ret;
+    c_p->flags |= F_DISTRIBUTION;
+    ERTS_PROC_SET_DIST_ENTRY(c_p, dep);
 
- yield:
-    ERTS_BIF_PREP_YIELD3(ret, bif_export[BIF_setnode_3], BIF_P,
-			 BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
-    goto done;
+    dep->send = NULL; /* Only for distr ports... */
 
- badarg:
-    ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
-    goto done;
+    if (redsp)
+        *redsp = 5;
 
- system_limit:
-    ERTS_BIF_PREP_ERROR(ret, BIF_P, SYSTEM_LIMIT);
-    goto done;
+    setup_connection_epiloge_rwunlock(c_p, dep, c_p->common.id,
+                                      scdcp->flags, scdcp->version);
+connected:
+
+    /* we take over previous inc in refc of dep */
+
+    if (!bpp) /* called directly... */
+        return erts_make_dhandle(c_p, dep);
+
+    erts_free(ERTS_ALC_T_SETUP_CONN_ARG, arg);
+
+    *bpp = new_message_buffer(ERTS_MAGIC_REF_THING_SIZE);
+    hp = (*bpp)->mem;
+    return erts_build_dhandle(&hp, &(*bpp)->off_heap, dep);
+
+badarg:
+
+    if (bpp) /* not called directly */
+        erts_free(ERTS_ALC_T_SETUP_CONN_ARG, arg);
+
+    if (dep_locked)
+        erts_de_rwunlock(dep);
+
+    erts_deref_dist_entry(dep);
+
+    return am_badarg;
 }
 
+
 BIF_RETTYPE erts_internal_get_dflags_0(BIF_ALIST_0)
 {
     return erts_dflags_record;
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 1a29e3d817..9db600dce0 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -288,6 +288,7 @@ type	CML_CLEANUP	SHORT_LIVED	SYSTEM		connection_ml_cleanup
 type	ML_YIELD_STATE	SHORT_LIVED	SYSTEM		monitor_link_yield_state
 type	ML_DIST		STANDARD	SYSTEM		monitor_link_dist
 type	PF3_ARGS	SHORT_LIVED	PROCESSES	process_flag_3_arguments
+type	SETUP_CONN_ARG	SHORT_LIVED	PROCESSES	setup_connection_argument
 
 type	ENVIRONMENT	SYSTEM		SYSTEM		environment
 
-- 
cgit v1.2.3


From 1f2268308b837b797220a1d7419504f19742c265 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 11 May 2018 18:49:28 +0200
Subject: Fix erts_try_lock_sig_free_proc()

---
 erts/emulator/beam/erl_bif_info.c |  1 +
 erts/emulator/beam/erl_process.c  | 48 +++++++++++++++++++++------------------
 2 files changed, 27 insertions(+), 22 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 8f04ff7353..55450e397f 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1160,6 +1160,7 @@ process_info_bif(Process *c_p, Eterm pid, Eterm opt, int always_wrap, int pi2)
     else {
         if (flags & ERTS_PI_FLAG_FORCE_SIG_SEND)
             goto send_signal;
+        state = ERTS_PSFLG_RUNNING; /* fail state... */
         rp = erts_try_lock_sig_free_proc(pid, locks, &state);
         if (!rp)
             goto undefined;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index f3e77df2fd..b373d08a6b 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -8656,6 +8656,13 @@ erts_internal_suspend_process_2(BIF_ALIST_2)
         Process *rp;
         int send_sig = 0;
 
+        /* fail state... */
+        state = (ERTS_PSFLG_EXITING
+                 | ERTS_PSFLG_RUNNING
+                 | ERTS_PSFLG_RUNNING_SYS
+                 | ERTS_PSFLG_DIRTY_RUNNING
+                 | ERTS_PSFLG_DIRTY_RUNNING_SYS);
+
         rp = erts_try_lock_sig_free_proc(BIF_ARG_1,
                                          ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS,
                                          &state);
@@ -8663,16 +8670,7 @@ erts_internal_suspend_process_2(BIF_ALIST_2)
             goto noproc;
         if (rp == ERTS_PROC_LOCK_BUSY)
             send_sig = !0;
-        else if (state & (ERTS_PSFLG_EXITING
-                          | ERTS_PSFLG_RUNNING
-                          | ERTS_PSFLG_RUNNING_SYS
-                          | ERTS_PSFLG_DIRTY_RUNNING
-                          | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
-            erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
-            send_sig = !0;
-        }
-
-        if (!send_sig && rp) {
+        else {
             send_sig = !suspend_process(BIF_P, rp);
             if (!send_sig) {
                 erts_monitor_list_insert(&ERTS_P_LT_MONITORS(rp), &mdp->target);
@@ -12497,7 +12495,13 @@ erts_try_lock_sig_free_proc(Eterm pid, ErtsProcLocks locks,
                             erts_aint32_t *statep)
 {
     Process *rp = erts_proc_lookup_raw(pid);
+    erts_aint32_t fail_state = ERTS_PSFLG_SIG_IN_Q|ERTS_PSFLG_SIG_Q;
     erts_aint32_t state;
+    ErtsProcLocks tmp_locks = ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_MSGQ;
+
+    tmp_locks |= locks;
+    if (statep)
+        fail_state |= *statep;
 
     if (!rp) {
         if (statep)
@@ -12514,28 +12518,28 @@ erts_try_lock_sig_free_proc(Eterm pid, ErtsProcLocks locks,
     if (state & ERTS_PSFLG_FREE)
         return NULL;
 
-    if (state & (ERTS_PSFLG_SIG_IN_Q|ERTS_PSFLG_SIG_Q))
+    if (state & fail_state)
         return ERTS_PROC_LOCK_BUSY;
 
-    if (!locks)
-        return rp;
-
-    if (erts_proc_trylock(rp, locks) == EBUSY)
+    if (erts_proc_trylock(rp, tmp_locks) == EBUSY)
         return ERTS_PROC_LOCK_BUSY;
 
     state = erts_atomic32_read_nob(&rp->state);
     if (statep)
         *statep = state;
 
-    if (state & ERTS_PSFLG_FREE) {
-        erts_proc_unlock(rp, locks);
-        return NULL;
+    if ((state & fail_state)
+        || rp->sig_inq.first
+        || rp->sig_qs.cont) {
+        erts_proc_unlock(rp, tmp_locks);
+        if (state & ERTS_PSFLG_FREE)
+            return NULL;
+        else
+            return ERTS_PROC_LOCK_BUSY;
     }
 
-    if (state & (ERTS_PSFLG_SIG_IN_Q|ERTS_PSFLG_SIG_Q)) {
-        erts_proc_unlock(rp, locks);
-        return ERTS_PROC_LOCK_BUSY;
-    }
+    if (tmp_locks != locks)
+        erts_proc_unlock(rp, tmp_locks & ~locks);
 
     return rp;
 }
-- 
cgit v1.2.3


From d7bacbc798d608f259eb2faaaf194cb9df323018 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 15 May 2018 20:42:46 +0200
Subject: erts: Fix bug in enif_binary_to_term for immediates

Symptom: Heap corruption

Expanded test case to provoke this bug
and test some more term types.
---
 erts/emulator/beam/erl_nif.c                  |  6 ++++--
 erts/emulator/test/nif_SUITE.erl              | 17 ++++++++++++-----
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c | 10 +++++++---
 3 files changed, 23 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 479a2f4809..a41145af41 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1157,8 +1157,10 @@ size_t enif_binary_to_term(ErlNifEnv *dst_env,
     if (is_non_value(*term)) {
         return 0;
     }
-    erts_factory_close(&factory);
-    cache_env(dst_env);
+    if (size > 0) {
+        erts_factory_close(&factory);
+        cache_env(dst_env);
+    }
 
     ASSERT(bp > data);
     return bp - data;
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 9c1694fa8a..4c5109c4ac 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -2111,16 +2111,23 @@ nif_term_to_binary(Config) ->
 
 nif_binary_to_term(Config) ->
     ensure_lib_loaded(Config),
-    T = {#{ok => nok}, <<0:8096>>, lists:seq(1,100)},
+    BigMap = maps:from_list([{I,-I} || I <- lists:seq(1,100)]),
+    [nif_binary_to_term_do(T)
+     || T <- [{#{ok => nok}, <<0:8096>>, lists:seq(1,100)},
+              atom, 42, self(), BigMap]],
+    ok.
+
+nif_binary_to_term_do(T) ->
+    Dummy = [true|false],
     Bin = term_to_binary(T),
     Len = byte_size(Bin),
-    {Len,T} = binary_to_term_nif(Bin, undefined, 0),
+    {Len,T,Dummy} = binary_to_term_nif(Bin, undefined, 0),
     Len = binary_to_term_nif(Bin, self(), 0),
-    T = receive M -> M after 1000 -> timeout end,
+    {T,Dummy} = receive M -> M after 1000 -> timeout end,
 
-    {Len, T} = binary_to_term_nif(Bin, undefined, ?ERL_NIF_BIN2TERM_SAFE),
+    {Len,T,Dummy} = binary_to_term_nif(Bin, undefined, ?ERL_NIF_BIN2TERM_SAFE),
     false = binary_to_term_nif(<<131,100,0,14,"undefined_atom">>,
-			   undefined, ?ERL_NIF_BIN2TERM_SAFE),
+                               undefined, ?ERL_NIF_BIN2TERM_SAFE),
     false = binary_to_term_nif(Bin, undefined, 1),
     ok.
 
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index f2b1ef9d24..3d262913aa 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -1958,7 +1958,7 @@ static ERL_NIF_TERM term_to_binary(ErlNifEnv* env, int argc, const ERL_NIF_TERM
 static ERL_NIF_TERM binary_to_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     ErlNifBinary bin;
-    ERL_NIF_TERM term, ret_term;
+    ERL_NIF_TERM term, dummy, ret_term;
     ErlNifPid pid;
     ErlNifEnv *msg_env = env;
     unsigned int opts;
@@ -1971,6 +1971,9 @@ static ERL_NIF_TERM binary_to_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM
 	|| !enif_get_uint(env, argv[2], &opts))
         return enif_make_badarg(env);
 
+    /* build dummy heap term first to provoke OTP-15080 */
+    dummy = enif_make_list_cell(msg_env, atom_true, atom_false);
+
     ret = enif_binary_to_term(msg_env, bin.data, bin.size, &term,
 			      (ErlNifBinaryToTerm)opts);
     if (!ret)
@@ -1978,11 +1981,12 @@ static ERL_NIF_TERM binary_to_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM
 
     ret_term = enif_make_uint64(env, ret);
     if (msg_env != env) {
-        enif_send(env, &pid, msg_env, term);
+        enif_send(env, &pid, msg_env,
+                  enif_make_tuple2(msg_env, term, dummy));
         enif_free_env(msg_env);
         return ret_term;
     } else {
-        return enif_make_tuple2(env, ret_term, term);
+        return enif_make_tuple3(env, ret_term, term, dummy);
     }
 }
 
-- 
cgit v1.2.3


From 23db9db1dbe6ef3549a2b219dab7b849bb56c307 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 16 May 2018 17:16:49 +0200
Subject: erts: Silence gcc warnings
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

‘res’ may be used uninitialized in this function
---
 erts/emulator/beam/erl_map.c | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 05e8fc11a2..48154b5d0f 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -3133,7 +3133,6 @@ BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
              * as this is how the list as a whole is constructed.
              */
             hp = HAlloc(BIF_P, (2 + 3) * elems);
-            res = BIF_ARG_3;
         }
 
         orig_elems = elems;
@@ -3157,12 +3156,15 @@ BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
             if (is_list(ptr[PATH_ELEM(curr_path)])) {
                 Eterm *lst = list_val(ptr[PATH_ELEM(curr_path)]);
                 if (type == iterator) {
-                    res = TUPLE3(hp, CAR(lst), CDR(lst), make_tuple(hp+4));
+                    res = make_tuple(hp);
+                    hp[0] = make_arityval(3);
+                    hp[1] = CAR(lst);
+                    hp[2] = CDR(lst);
+                    patch_ptr = &hp[3];
                     hp += 4;
-                    patch_ptr = hp-1;
                 } else {
                     Eterm tup = TUPLE2(hp, CAR(lst), CDR(lst)); hp += 3;
-                    res = CONS(hp, tup, res); hp += 2;
+                    res = CONS(hp, tup, BIF_ARG_3); hp += 2;
                 }
                 elems--;
                 break;
@@ -3196,8 +3198,12 @@ BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
             while (idx < sz && elems != 0 && is_list(ptr[idx])) {
                 Eterm *lst = list_val(ptr[idx]);
                 if (type == iterator) {
-                    (void) TUPLE3(hp, CAR(lst), CDR(lst), make_tuple(hp+4)); hp += 4;
-                    patch_ptr = hp-1;
+                    *patch_ptr = make_tuple(hp);
+                    hp[0] = make_arityval(3);
+                    hp[1] = CAR(lst);
+                    hp[2] = CDR(lst);
+                    patch_ptr = &hp[3];
+                    hp += 4;
                 } else {
                     Eterm tup = TUPLE2(hp, CAR(lst), CDR(lst)); hp += 3;
                     res = CONS(hp, tup, res); hp += 2;
-- 
cgit v1.2.3


From 662f3c7ba50ff8ec13d86171bcfc61fd3da9deed Mon Sep 17 00:00:00 2001
From: Timmo Verlaan <tverlaan@gmail.com>
Date: Mon, 29 Jan 2018 21:27:22 +0100
Subject: epmd: allow alternative to dns resolving for nodename

This makes it possible to create a custom integration with a
key-value store for example. The key would then point to the
actual address. You would have to write your own epmd module
to make use of that feature.
---
 erts/emulator/test/distribution_SUITE.erl | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index e40d346e10..b8b84e91a0 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -73,7 +73,7 @@
          dist_evil_parallel_receiver/0]).
 
 %% epmd_module exports
--export([start_link/0, register_node/2, register_node/3, port_please/2]).
+-export([start_link/0, register_node/2, register_node/3, port_please/2, address_please/3]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
@@ -2086,6 +2086,11 @@ port_please(_Name, _Ip) ->
 	    {port, Port, Version}
     end.
 
+address_please(_Name, _Address, _AddressFamily) ->
+    %% Use localhost.
+    IP = {127,0,0,1},
+    {ok, IP}.
+
 %%% Utilities
 
 timestamp() ->
-- 
cgit v1.2.3


From 7c485575e78b6537586bb0902658b890b22a2186 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 16 May 2018 16:35:23 +0200
Subject: erts: Fix bug in system_profile

seen to cause redundant {profile,_,active,_,_} messages
when process is terminating.
---
 erts/emulator/beam/erl_process.c            | 14 ++---
 erts/emulator/test/system_profile_SUITE.erl | 86 +++++++++++++++++------------
 2 files changed, 57 insertions(+), 43 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 54724fd62d..b0178c3fc1 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -6674,13 +6674,13 @@ change_proc_schedule_state(Process *p,
 
 	if (((n & (ERTS_PSFLG_SUSPENDED
 		   | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE)
-	    && (!(a & (ERTS_PSFLG_ACTIVE_SYS
-		       | ERTS_PSFLG_RUNNING
-		       | ERTS_PSFLG_RUNNING_SYS
-		       | ERTS_PSFLG_DIRTY_RUNNING
-		       | ERTS_PSFLG_DIRTY_RUNNING_SYS)
-		  && (!(a & ERTS_PSFLG_ACTIVE)
-		      || (a & ERTS_PSFLG_SUSPENDED))))) {
+            & ((a & (ERTS_PSFLG_SUSPENDED
+                     | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE)
+	    & !(a & (ERTS_PSFLG_ACTIVE_SYS
+                     | ERTS_PSFLG_RUNNING
+                     | ERTS_PSFLG_RUNNING_SYS
+                     | ERTS_PSFLG_DIRTY_RUNNING
+                     | ERTS_PSFLG_DIRTY_RUNNING_SYS))) {
 	    /* We activated a prevously inactive process */
 	    profile_runnable_proc(p, am_active);
 	}
diff --git a/erts/emulator/test/system_profile_SUITE.erl b/erts/emulator/test/system_profile_SUITE.erl
index 2e359b11ce..73d466fe28 100644
--- a/erts/emulator/test/system_profile_SUITE.erl
+++ b/erts/emulator/test/system_profile_SUITE.erl
@@ -95,18 +95,20 @@ do_runnable_procs({TsType, TsTypeFlag}) ->
     % FIXME: Set #laps and #nodes in config file
     Nodes = 10,
     Laps = 10,
-    Master = ring(Nodes),
+    All = ring(Nodes, [link,monitor]),
+    [Master | _] = All,
     undefined = erlang:system_profile(Pid, [runnable_procs]++TsTypeFlag),
     % loop a message
     ok = ring_message(Master, message, Laps),
+    ok = kill_ring(Master),
+    [receive {'DOWN', _, process, P, _} -> ok end || P <- All],
     Events = get_profiler_events(),
-    kill_em_all = kill_ring(Master),
     erlang:system_profile(undefined, []),
     put(master, Master),
     put(laps, Laps),
     true = has_runnable_event(TsType, Events),
     Pids = sort_events_by_pid(Events),
-    ok = check_events(TsType, Pids),
+    ok = check_events(TsType, Pids, (Laps+1)*2+2, (Laps+1)*2),
     erase(),
     exit(Pid,kill),
     ok.
@@ -139,7 +141,7 @@ do_runnable_ports({TsType, TsTypeFlag}, Config) ->
     erlang:system_profile(undefined, []),
     true = has_runnable_event(TsType, Events),
     Pids = sort_events_by_pid(Events),
-    ok = check_events(TsType, Pids),
+    ok = check_events(TsType, Pids, Laps*2+2, Laps*2),
     erase(),
     exit(Pid,kill),
     ok.
@@ -172,12 +174,12 @@ dont_profile_profiler(Config) when is_list(Config) ->
 
     Nodes = 10,
     Laps = 10,
-    Master = ring(Nodes),
+    [Master|_] = ring(Nodes, [link]),
     undefined = erlang:system_profile(Pid, [runnable_procs]),
     % loop a message
     ok = ring_message(Master, message, Laps),
     erlang:system_profile(undefined, []),
-    kill_em_all = kill_ring(Master),
+    ok = kill_ring(Master),
     Events = get_profiler_events(),
     false  = has_profiler_pid_event(Events, Pid),
 
@@ -249,27 +251,28 @@ check_block_system({TsType, TsTypeFlag}, Nodes) ->
 
 %%% Check events
 
-check_events(_TsType, []) -> ok;
-check_events(TsType, [Pid | Pids]) ->
+check_events(_TsType, [], _, _) -> ok;
+check_events(TsType, [Pid | Pids], ExpMaster, ExpMember) ->
     Master = get(master),
-    Laps = get(laps),
     CheckPids = get(pids),
     {Events, N} = get_pid_events(Pid),
     ok = check_event_flow(Events),
     ok = check_event_ts(TsType, Events),
     IsMember = lists:member(Pid, CheckPids),
-    case Pid of
-    	Master ->
-	    io:format("Expected ~p and got ~p profile events from ~p: ok~n", [Laps*2+2, N, Pid]),
-	    N = Laps*2 + 2,
-    	    check_events(TsType, Pids);
-	Pid when IsMember == true ->
-	    io:format("Expected ~p and got ~p profile events from ~p: ok~n", [Laps*2, N, Pid]),
-	    N = Laps*2,
-    	    check_events(TsType, Pids);
-	Pid ->
-	    check_events(TsType, Pids)
-    end.
+    {Title,Exp} = case Pid of
+                      Master -> {master,ExpMaster};
+                      Pid when IsMember == true -> {member,ExpMember};
+                      _ -> {other,N}
+                  end,
+    ok = case N of
+             Exp -> ok;
+             _ ->
+                 io:format("Expected ~p and got ~p profile events from ~p ~p:~n~p~n",
+                           [Exp, N, Title, Pid, Events]),
+                 error
+         end,
+    check_events(TsType, Pids, ExpMaster, ExpMember).
+
 
 %% timestamp consistency check for descending timestamps
 
@@ -297,7 +300,13 @@ check_event_ts(TsType, [{Pid, _, _, TS1}=Event | Events], {Pid,_,_,TS0}) ->
 %% consistency check for active vs. inactive activity (runnable)
 
 check_event_flow(Events) ->
-    check_event_flow(Events, undefined).
+    case check_event_flow(Events, undefined) of
+        ok -> ok;
+        Error ->
+            io:format("Events = ~p\n", [Events]),
+            Error
+    end.
+
 check_event_flow([], _) -> ok;
 check_event_flow([Event | PidEvents], undefined) ->
     check_event_flow(PidEvents, Event);
@@ -337,10 +346,11 @@ sort_events_by_pid([Event | Events],Pids) ->
 %% API
 
 % Returns master pid
-ring(N) -> 
-    Pids = build_ring(N, []),
+ring(N, SpawnOpt) -> 
+    Pids = build_ring(N, [], SpawnOpt),
     put(pids, Pids),
-    setup_ring(Pids).
+    setup_ring(Pids),
+    Pids.
 
 ring_message(Master, Message, Laps) ->
     Master ! {message, Master, Laps, Message},
@@ -348,13 +358,19 @@ ring_message(Master, Message, Laps) ->
     	{laps_complete, Master} -> ok
     end.
 
-kill_ring(Master) -> Master ! kill_em_all.
+kill_ring(Master) ->
+    Master ! kill_em_all,
+    ok.
 
 %% Process ring helpers
 
-build_ring(0, Pids) -> Pids;
-build_ring(N, Pids) ->
-    build_ring(N - 1, [spawn_link(?MODULE, ring_loop, [undefined]) | Pids]).
+build_ring(0, Pids, _) -> Pids;
+build_ring(N, Pids, SpawnOpt) ->
+    Pid = case spawn_opt(?MODULE, ring_loop, [undefined], SpawnOpt) of
+              {P,_} -> P;
+              P -> P
+          end,
+    build_ring(N-1, [Pid | Pids], SpawnOpt).
 
 setup_ring([Master | Relayers]) ->
     % Relayers may not include the master pid
@@ -383,15 +399,13 @@ ring_loop(RelayTo) ->
         {message, Master, Lap, Msg}=Message ->
             case {self(), Lap} of
                 {Master, 0} ->
-                    get(supervisor) ! {laps_complete, self()},
-                    ring_loop(RelayTo);
+                    get(supervisor) ! {laps_complete, self()};
                 {Master, Lap} ->
-                    RelayTo ! {message, Master, Lap - 1, Msg},
-                    ring_loop(RelayTo);
+                    RelayTo ! {message, Master, Lap - 1, Msg};
                 _ ->
-                    RelayTo ! Message,
-                    ring_loop(RelayTo)
-            end
+                    RelayTo ! Message
+            end,
+            ring_loop(RelayTo)
     end.
 
 %%%
-- 
cgit v1.2.3


From e3bc55fb1e768cd30140a769507b50bddf3cb6ca Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 18 May 2018 16:16:55 +0200
Subject: erts: Add some missing doxygen docs

---
 erts/emulator/beam/erl_proc_sig_queue.h | 32 +++++++++++++++++++++++++++++---
 1 file changed, 29 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_proc_sig_queue.h b/erts/emulator/beam/erl_proc_sig_queue.h
index efa7c08664..3fc2d06b2d 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.h
+++ b/erts/emulator/beam/erl_proc_sig_queue.h
@@ -834,18 +834,44 @@ Sint
 erts_proc_sig_privqs_len(Process *c_p);
 
 
-/* SVERK: Doc me up! */
+/**
+ * @brief Enqueue list of signals on process.
+ *
+ * Message queue must be locked on receiving process.
+ *
+ * @param rp                Receiving process.
+ * @param first             First signal in list.
+ * @param last              Last signal in list.
+ * @param last_next         Pointer to next-pointer to last non-message signal
+ *                          or NULL if no non-message signal after 'first'.
+ * @param msg_cnt           Number of message signals in list.
+ * @param in_state          'state' of rp.
+ *
+ * @return                  'state' of rp.
+ */
 erts_aint32_t
 erts_enqueue_signals(Process *rp, ErtsMessage *first,
                      ErtsMessage **last, ErtsMessage **last_next,
                      Uint msg_cnt,
                      erts_aint32_t in_state);
 
-/* SVERK: Doc me up! */
+/**
+ *
+ * @brief Flush pending signal.
+ *
+ */
 void
 erts_proc_sig_send_pending(ErtsSchedulerData* esdp);
 
-/* SVERK Doc me up! */
+/**
+ *
+ * @brief Schedule process to handle enqueued signal(s).
+ *
+ * @param rp                Receiving process.
+ * @param state             'state' of rp.
+ * @param enable_flag       Additional state flags to enable, like
+ *                          ERTS_PSFLG_ACTIVE if message has been enqueued.
+ */
 ERTS_GLB_INLINE void erts_proc_notify_new_sig(Process* rp, erts_aint32_t state,
                                               erts_aint32_t enable_flag);
 
-- 
cgit v1.2.3


From 9338ea3f9d3f7db949001a461456e8ce0339a1b5 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 18 May 2018 17:59:03 +0200
Subject: erts: Fix narrow race between ets:new and ets:delete

of same named table.

If other process does ets:delete
before ets:new has completely finished and done save_owned_table
then ets:delete might do delete_owned_table and deref wild pointers
in tb->common.owned.
---
 erts/emulator/beam/erl_db.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 3a29f8cf56..c1eaaeee06 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1776,9 +1776,11 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
         ret = make_tid(BIF_P, tb);
 
     save_sched_table(BIF_P, tb);
+    save_owned_table(BIF_P, tb);
 
     if (is_named && !insert_named_tab(BIF_ARG_1, tb, 0)) {
         tid_clear(BIF_P, tb);
+        delete_owned_table(BIF_P, tb);
 
 	db_lock(tb,LCK_WRITE);
 	free_heir_data(tb);
@@ -1789,7 +1791,6 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     }
     
     BIF_P->flags |= F_USING_DB; /* So we can remove tb if p dies */
-    save_owned_table(BIF_P, tb);
 
 #ifdef HARDDEBUG
     erts_fprintf(stderr,
-- 
cgit v1.2.3


From 26c0970dc23da609a51a7ca4aa3d8826d1c8c661 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 21 May 2018 07:41:33 +0200
Subject: Fix memory leak of processes that died in the run queue

---
 erts/emulator/beam/erl_process.c | 38 ++++++++++++++++++--------------------
 1 file changed, 18 insertions(+), 20 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 9b22f024b0..85c3259d1b 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -4176,6 +4176,8 @@ dequeue_process(ErtsRunQueue *runq, int prio_q, erts_aint32_t *statep)
     ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
 
     state = erts_smp_atomic32_read_nob(&p->state);
+    ASSERT(state & ERTS_PSFLG_IN_RUNQ);
+
     if (statep)
 	*statep = state;
 
@@ -4183,8 +4185,7 @@ dequeue_process(ErtsRunQueue *runq, int prio_q, erts_aint32_t *statep)
 
     rqi = &runq->procs.prio_info[prio];
 
-    if (p)
-	unqueue_process(runq, rpq, rqi, prio, NULL, p);
+    unqueue_process(runq, rpq, rqi, prio, NULL, p);
 
     return p;
 }
@@ -4593,22 +4594,17 @@ evacuate_run_queue(ErtsRunQueue *rq,
 		    free_proxy_proc(proc);
 		else {
 		    erts_aint32_t clr_bits;
-#ifdef DEBUG
-		    erts_aint32_t old;
-#endif
 
 		    clr_bits = ERTS_PSFLG_IN_RUNQ;
 		    clr_bits |= qbit << ERTS_PSFLGS_IN_PRQ_MASK_OFFSET;
 
-#ifdef DEBUG
-		    old =
-#else
-			(void)
-#endif
-			erts_smp_atomic32_read_band_mb(&proc->state,
-						       ~clr_bits);
-			ASSERT((old & clr_bits) == clr_bits);
+                    state = erts_smp_atomic32_read_band_mb(&proc->state, ~clr_bits);
+                    ASSERT((state & clr_bits) == clr_bits);
 
+                    if (state & ERTS_PSFLG_FREE) {
+                        /* free and not queued by proxy */
+                        erts_proc_dec_refc(proc);
+		    }
 		}
 
 		goto handle_next_proc;
@@ -6726,13 +6722,14 @@ fin_dirty_enq_s_change(Process *p,
 	/* Already enqueue by someone else... */
 	if (pstruct_reserved) {
 	    /* We reserved process struct for enqueue; clear it... */
-#ifdef DEBUG
-	    erts_aint32_t old =
-#else
-		(void)
-#endif
-		erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_IN_RUNQ);
-	    ASSERT(old & ERTS_PSFLG_IN_RUNQ);
+	    erts_aint32_t state;
+
+            state = erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_IN_RUNQ);
+            ASSERT(state & ERTS_PSFLG_IN_RUNQ);
+
+            if (state & ERTS_PSFLG_FREE) {
+                erts_proc_dec_refc(p);
+            }
 	}
 	return 0;
     }
@@ -10758,6 +10755,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 			}
 			else if (state & ERTS_PSFLG_FREE) {
 			    /* free and not queued by proxy */
+                            ASSERT(state & ERTS_PSFLG_IN_RUNQ);
 			    erts_proc_dec_refc(p);
 			}
                         if (!is_normal_sched)
-- 
cgit v1.2.3


From 254c4a5615d7bf52acec02437fab8c771fc46a8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 21 May 2018 07:44:45 +0200
Subject: Fix deadlock in run queue evacuation

---
 erts/emulator/beam/erl_process.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 85c3259d1b..ee6d56b5d7 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -4513,7 +4513,7 @@ evacuate_run_queue(ErtsRunQueue *rq,
 
 	erts_smp_runq_unlock(to_rq);
 	smp_notify_inc_runq(to_rq);
-	erts_smp_runq_lock(to_rq);
+	erts_smp_runq_lock(rq);
     }
 
     if (rq->ports.start) {
-- 
cgit v1.2.3


From 9cf67430bff860d47c359df6b997c8bcd0985f4c Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 22 May 2018 14:43:30 +0200
Subject: erts: Let allocator pooled_tree also use Age Order

if configured for the allocator. This was not implemented
and pooled_tree always used address order first fit.

The "problem" was as a carrier can be part of both the pooled tree
(of its owner) and the allocation tree (of a foreign instance)
at the same time. Solved by duplicating 'birth_time' in both
AOFF_RBTree_t nodes for each carrier.

Blocks still cannot use age order and does not pay
any memory cost for birth_time.
---
 erts/emulator/beam/erl_alloc_util.h        |  4 ++++
 erts/emulator/beam/erl_ao_firstfit_alloc.c | 32 +++++++++++++-----------------
 erts/emulator/beam/erl_ao_firstfit_alloc.h |  6 +++---
 3 files changed, 21 insertions(+), 21 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index ff4d10b206..cbae8ce98a 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -414,6 +414,10 @@ struct AOFF_RBTree_t_ {
     AOFF_RBTree_t *right;
     Uint32 flags;
     Uint32 max_sz;  /* of all blocks in this sub-tree */
+    union {
+        AOFF_RBTree_t* next;   /* for best fit */
+        Sint64 birth_time;     /* for age first fit */
+    } u;
 };
 
 void aoff_add_pooled_mbc(Allctr_t*, Carrier_t*);
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index 0c5545401a..ebbe4af53d 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -100,22 +100,14 @@
 
 #define AOFF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr)
 
-/* BF block nodes keeps list of all with equal size
- */
-typedef struct {
-    AOFF_RBTree_t t;
-    AOFF_RBTree_t *next;
-}AOFF_RBTreeList_t;
-
-#define LIST_NEXT(N) (((AOFF_RBTreeList_t*) (N))->next)
-#define LIST_PREV(N) (((AOFF_RBTreeList_t*) (N))->t.parent)
+#define LIST_NEXT(N) (((AOFF_RBTree_t*)(N))->u.next)
+#define LIST_PREV(N) (((AOFF_RBTree_t*)(N))->parent)
 
 typedef struct AOFF_Carrier_t_ AOFF_Carrier_t;
 
 struct AOFF_Carrier_t_ {
     Carrier_t crr;
     AOFF_RBTree_t rbt_node;     /* My node in the carrier tree */
-    Sint64 birth_time;
     AOFF_RBTree_t* root;        /* Root of my block tree */
 };
 #define RBT_NODE_TO_MBC(PTR) ErtsContainerStruct((PTR), AOFF_Carrier_t, rbt_node)
@@ -199,9 +191,7 @@ static ERTS_INLINE SWord cmp_blocks(enum AOFFSortOrder order,
 {
     ASSERT(lhs != rhs);
     if (order == FF_AGEFF) {
-	AOFF_Carrier_t* lc = RBT_NODE_TO_MBC(lhs);
-	AOFF_Carrier_t* rc = RBT_NODE_TO_MBC(rhs);
-	Sint64 diff = lc->birth_time - rc->birth_time;
+	Sint64 diff = lhs->u.birth_time - rhs->u.birth_time;
  #ifdef ARCH_64
         if (diff)
             return diff;
@@ -296,8 +286,10 @@ erts_aoffalc_start(AOFFAllctr_t *alc,
     allctr->mbc_header_size		= sizeof(AOFF_Carrier_t);
     allctr->min_mbc_size		= MIN_MBC_SZ;
     allctr->min_mbc_first_free_size	= MIN_MBC_FIRST_FREE_SZ;
-    allctr->min_block_size = (aoffinit->blk_order == FF_BF ?
-			      sizeof(AOFF_RBTreeList_t):sizeof(AOFF_RBTree_t));
+    allctr->min_block_size = (aoffinit->blk_order == FF_BF
+                              ? (offsetof(AOFF_RBTree_t, u.next)
+                                 + ErtsSizeofMember(AOFF_RBTree_t, u.next))
+                              : offsetof(AOFF_RBTree_t, u));
 
     allctr->vsn_str			= ERTS_ALC_AOFF_ALLOC_VSN_STR;
 
@@ -939,8 +931,11 @@ static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier)
     HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0);
 
     crr->rbt_node.hdr.bhdr = 0;
-    if (alc->crr_order == FF_AGEFF || IS_DEBUG)
-	crr->birth_time = get_birth_time();
+    if (alc->crr_order == FF_AGEFF || IS_DEBUG) {
+        Sint64 bt = get_birth_time();
+        crr->rbt_node.u.birth_time = bt;
+        crr->crr.cpool.pooled.u.birth_time = bt;
+    }
     rbt_insert(alc->crr_order, root, &crr->rbt_node);
 
     /* aoff_link_free_block will add free block later */
@@ -978,6 +973,7 @@ static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier)
 
 void aoff_add_pooled_mbc(Allctr_t *allctr, Carrier_t *crr)
 {
+    AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr;
     AOFF_RBTree_t **root = &allctr->cpool.pooled_tree;
 
     ASSERT(allctr == crr->cpool.orig_allctr);
@@ -985,7 +981,7 @@ void aoff_add_pooled_mbc(Allctr_t *allctr, Carrier_t *crr)
 
     /* Link carrier in address order tree
      */
-    rbt_insert(FF_AOFF, root, &crr->cpool.pooled);
+    rbt_insert(alc->crr_order, root, &crr->cpool.pooled);
 
     HARD_CHECK_TREE(NULL, 0, *root, 0);
 }
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h
index 9cf4fc81a8..dad864801f 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.h
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h
@@ -29,10 +29,10 @@
 typedef struct AOFFAllctr_t_ AOFFAllctr_t;
 
 enum AOFFSortOrder {
-    FF_AGEFF = 0,
+    FF_AGEFF = 0,    /* carrier trees only */
     FF_AOFF  = 1,
-    FF_AOBF  = 2,
-    FF_BF    = 3
+    FF_AOBF  = 2,    /* block trees only */
+    FF_BF    = 3     /* block trees only */
 };
 
 typedef struct {
-- 
cgit v1.2.3


From 018983a0d31739f17f82cccca8ad1e05c09ad1e9 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 22 May 2018 17:27:51 +0200
Subject: erts: Purge unused allocation types

---
 erts/emulator/beam/erl_alloc.types | 27 ---------------------------
 1 file changed, 27 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 9db600dce0..5409b89bab 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -118,9 +118,6 @@ type	PORT		DRIVER		SYSTEM		port
 type	ATOM		LONG_LIVED	ATOM		atom_entry
 type	MODULE		LONG_LIVED	CODE		module_entry
 type	REG_PROC	STANDARD	PROCESSES	reg_proc
-type	LINK_LH		STANDARD	PROCESSES	link_lh
-type	SUSPEND_MON	STANDARD	PROCESSES	suspend_monitor
-type	PEND_SUSPEND	SHORT_LIVED	PROCESSES	pending_suspend
 type	PROC_LIST	SHORT_LIVED	PROCESSES	proc_list
 type	SAVED_ESTACK	SHORT_LIVED	PROCESSES	saved_estack
 type	FUN_ENTRY	LONG_LIVED	CODE		fun_entry
@@ -133,7 +130,6 @@ type	TMP_HEAP	TEMPORARY	PROCESSES	tmp_heap
 type	MSG_REF		FIXED_SIZE	PROCESSES	msg_ref
 type	MSG		EHEAP		PROCESSES	message
 type	MSGQ_CHNG	SHORT_LIVED	PROCESSES	messages_queue_change
-type	MSG_ROOTS	TEMPORARY	PROCESSES	msg_roots
 type	ROOTSET		TEMPORARY	PROCESSES	root_set
 type	LOADER_TMP	TEMPORARY	CODE		loader_tmp
 type	PREPARED_CODE	SHORT_LIVED	CODE		prepared_code
@@ -178,7 +174,6 @@ type	BPD		STANDARD	SYSTEM		bpd
 type	LINEBUF		STANDARD	SYSTEM		line_buf
 type	IOQ		STANDARD	SYSTEM		io_queue
 type	BITS_BUF	STANDARD	SYSTEM		bits_buf
-type	TMP_DIST_BUF	TEMPORARY	SYSTEM		tmp_dist_buf
 type	ASYNC_DATA	LONG_LIVED	SYSTEM		internal_async_data
 type	ESTACK		TEMPORARY	SYSTEM		estack
 type	DB_TABLE	ETS		ETS		db_tab
@@ -191,7 +186,6 @@ type	DB_MC_STK	TEMPORARY	ETS		db_mc_stack
 type	DB_MS_RUN_HEAP	SHORT_LIVED	ETS		db_match_spec_run_heap
 type	DB_MS_CMPL_HEAP	TEMPORARY	ETS		db_match_spec_cmpl_heap
 type	DB_SEG		ETS		ETS		db_segment
-type	DB_SEG_TAB	ETS		ETS		db_segment_tab
 type	DB_STK		ETS		ETS		db_stack
 type	DB_TRANS_TAB	ETS		ETS		db_trans_tab
 type	DB_SEL_LIST	ETS		ETS		db_select_list
@@ -200,7 +194,6 @@ type	DB_DMC_ERR_INFO	ETS		ETS		db_dmc_error_info
 type	DB_TERM		ETS		ETS		db_term
 type	DB_PROC_CLEANUP SHORT_LIVED	ETS		db_proc_cleanup_state
 type	ETS_ALL_REQ	SHORT_LIVED	ETS		ets_all_request
-type	INSTR_INFO	LONG_LIVED	SYSTEM		instr_info
 type	LOGGER_DSBUF	TEMPORARY	SYSTEM		logger_dsbuf
 type	TMP_DSBUF	TEMPORARY	SYSTEM		tmp_dsbuf
 type	INFO_DSBUF	SYSTEM		SYSTEM		info_dsbuf
@@ -215,7 +208,6 @@ type	PT_HNDL_LIST	SHORT_LIVED	SYSTEM		port_task_handle_list
 type	MISC_OP_LIST	SHORT_LIVED	SYSTEM		misc_op_list
 type	PORT_NAMES	SHORT_LIVED	SYSTEM		port_names
 type	PORT_DATA_LOCK	DRIVER		SYSTEM		port_data_lock
-type	NODES_MON	STANDARD	PROCESSES	nodes_monitor
 type	PTAB_LIST_DEL	SHORT_LIVED	PROCESSES	ptab_list_deleted_el
 type	PTAB_LIST_CNKI	SHORT_LIVED	PROCESSES	ptab_list_chunk_info
 type	PTAB_LIST_PIDS	SHORT_LIVED	PROCESSES	ptab_list_pids
@@ -238,10 +230,8 @@ type	CPUDATA		LONG_LIVED	SYSTEM		cpu_data
 type	TMP_CPU_IDS	SHORT_LIVED	SYSTEM		tmp_cpu_ids
 type	EXT_TERM_DATA	SHORT_LIVED	PROCESSES	external_term_data
 type	CPU_GRPS_MAP	LONG_LIVED	SYSTEM		cpu_groups_map
-type	AUX_WORK_TMO	LONG_LIVED	SYSTEM		aux_work_timeouts
 type	MISC_AUX_WORK_Q	LONG_LIVED	SYSTEM		misc_aux_work_q
 type	CODE_IX_LOCK_Q  SHORT_LIVED	SYSTEM		code_ix_lock_q
-type	PROC_INTERVAL	LONG_LIVED	SYSTEM		process_interval
 type    BUSY_CALLER_TAB	SHORT_LIVED	SYSTEM		busy_caller_table
 type    BUSY_CALLER	SHORT_LIVED	SYSTEM		busy_caller
 type	PROC_SYS_TSK	SHORT_LIVED	PROCESSES	proc_sys_task
@@ -250,8 +240,6 @@ type    NEW_TIME_OFFSET	SHORT_LIVED	SYSTEM		new_time_offset
 type	IOB_REQ		SHORT_LIVED	SYSTEM		io_bytes_request
 type    TRACER_NIF      LONG_LIVED      SYSTEM          tracer_nif
 type    TRACE_MSG_QUEUE SHORT_LIVED     SYSTEM          trace_message_queue
-type    SCHED_ASYNC_JOB SHORT_LIVED     SYSTEM          async_calls
-type    DIRTY_START     STANDARD	PROCESSES       dirty_start
 type    DIRTY_SL	SHORT_LIVED	SYSTEM		dirty_short_lived
 type    MREF_NSCHED_ENT	FIXED_SIZE	SYSTEM		nsched_magic_ref_entry
 type    MREF_ENT	STANDARD	SYSTEM		magic_ref_entry
@@ -259,7 +247,6 @@ type	MREF_TAB_BKTS	STANDARD	SYSTEM		magic_ref_table_buckets
 type	MREF_TAB	LONG_LIVED	SYSTEM		magic_ref_table
 type	MINDIRECTION	FIXED_SIZE	SYSTEM		magic_indirection
 type	BINARY_FIND	SHORT_LIVED	PROCESSES	binary_find
-type  	OPEN_PORT_ENV 	TEMPORARY	SYSTEM		open_port_env
 type	CRASH_DUMP	STANDARD	SYSTEM		crash_dump
 type	DIST_TRANSCODE  SHORT_LIVED	SYSTEM		dist_transcode_context
 
@@ -273,10 +260,8 @@ type	THR_Q_LL	LONG_LIVED	SYSTEM		long_lived_thr_queue
 type	ASYNC		SHORT_LIVED	SYSTEM		async
 type	ZLIB		STANDARD	SYSTEM		zlib
 
-type	PORT_LOCK	STANDARD	SYSTEM		port_lock
 type	DRIVER_LOCK	STANDARD	SYSTEM		driver_lock
 type	XPORTS_LIST	SHORT_LIVED	SYSTEM		extra_port_list
-type	PROC_LCK_WTR	LONG_LIVED	SYSTEM		proc_lock_waiter
 type	RUNQ_BLNS	LONG_LIVED	SYSTEM		run_queue_balancing
 type	THR_PRGR_IDATA	LONG_LIVED	SYSTEM		thr_prgr_internal_data
 type	THR_PRGR_DATA	LONG_LIVED	SYSTEM		thr_prgr_data
@@ -317,12 +302,6 @@ type	HIPE_EXEC	EXEC            CODE		hipe_code
 
 +endif
 
-+if heap_frag_elim_test
-
-type	SSB		SHORT_LIVED	PROCESSES	ssb
-
-+endif
-
 +if lcnt
 
 type    LCNT_CARRIER   STANDARD    SYSTEM  lcnt_lock_info_carrier
@@ -342,7 +321,6 @@ type	PURGE_DATA	SHORT_LIVED	CODE		purge_data
 type	DB_HEIR_DATA	STANDARD	ETS		db_heir_data
 type	DB_MS_PSDO_PROC	LONG_LIVED	ETS		db_match_pseudo_proc
 type	SCHDLR_DATA	LONG_LIVED	SYSTEM		scheduler_data
-type  	LL_TEMP_TERM 	LONG_LIVED	SYSTEM		ll_temp_term
 
 type	NIF_TRAP_EXPORT	STANDARD	PROCESSES	nif_trap_export_entry
 type	NIF_EXP_TRACE	FIXED_SIZE	PROCESSES	nif_export_trace
@@ -366,21 +344,17 @@ type	DRV_TAB		LONG_LIVED	SYSTEM		drv_tab
 type	DRV_EV_STATE	LONG_LIVED	SYSTEM		driver_event_state
 type	DRV_SEL_D_STATE	FIXED_SIZE	SYSTEM		driver_select_data_state
 type	NIF_SEL_D_STATE	FIXED_SIZE	SYSTEM		enif_select_data_state
-type	FD_LIST		SHORT_LIVED	SYSTEM		fd_list
 type	POLLSET		LONG_LIVED	SYSTEM		pollset
 type	POLLSET_UPDREQ	SHORT_LIVED	SYSTEM		pollset_update_req
 type	POLL_FDS	LONG_LIVED	SYSTEM		poll_fds
-type	POLL_RES_EVS	LONG_LIVED	SYSTEM		poll_result_events
 type	FD_STATUS	LONG_LIVED	SYSTEM		fd_status
 type	SELECT_FDS	LONG_LIVED	SYSTEM		select_fds
 
 +if unix
 
 type	SYS_READ_BUF	TEMPORARY	SYSTEM		sys_read_buf
-type	FD_TAB		LONG_LIVED	SYSTEM		fd_tab
 type	FD_ENTRY_BUF	STANDARD	SYSTEM		fd_entry_buf
 type	CS_PROG_PATH	LONG_LIVED	SYSTEM		cs_prog_path
-type	PRT_REP_EXIT	STANDARD	SYSTEM		port_report_exit
 type	SYS_BLOCKING	STANDARD	SYSTEM		sys_blocking
 
 type	SYS_WRITE_BUF	TEMPORARY	SYSTEM		sys_write_buf
@@ -392,7 +366,6 @@ type	SYS_WRITE_BUF	TEMPORARY	SYSTEM		sys_write_buf
 type	DRV_DATA_BUF	SYSTEM		SYSTEM		drv_data_buf
 type	PRELOADED	LONG_LIVED	SYSTEM		preloaded
 type	WAITER_OBJ	LONG_LIVED	SYSTEM		waiter_object
-type	CON_VPRINTF_BUF	TEMPORARY	SYSTEM		con_vprintf_buf
 
 +endif
 
-- 
cgit v1.2.3


From 6365a98f3a256adb62a05cc8644fa39b0aaf963f Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 23 May 2018 09:39:15 +0200
Subject: make: Don't call mkdir with empty argument

---
 erts/emulator/Makefile.in | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 221cf84622..054692819e 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -517,7 +517,9 @@ release_docs_spec:
 # Generated source code. Put in $(TARGET) directory
 #
 
+ifneq ($(strip $(CREATE_DIRS)),)
 _create_dirs := $(shell mkdir -p $(CREATE_DIRS))
+endif
 
 
 # has to be run after _create_dirs
-- 
cgit v1.2.3


From 4ac9aa587aa0091162e2520fbf9285ade6e8b74e Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 25 May 2018 12:22:05 +0200
Subject: erts: Make sure scheduler_data is set

If scheduler_data is not set correctly on normal schedulers
the code in erts_schedule_time_break and possibly others
will trigger asserts.
---
 erts/emulator/beam/erl_process.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 1478b71195..8253ec4f40 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9186,6 +9186,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
     internal_sched_out_proc:
 
 	ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
+        ASSERT(p->scheduler_data || ERTS_SCHEDULER_IS_DIRTY(esdp));
 
 	ASSERT(actual_reds >= 0);
 	if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST)
@@ -9618,6 +9619,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	state = erts_atomic32_read_nob(&p->state);
 
 	if (is_normal_sched) {
+            ASSERT(!p->scheduler_data);
+	    p->scheduler_data = esdp;
 	    if ((!!(state & ERTS_PSFLGS_DIRTY_WORK))
 		& (!(state & ERTS_PSFLG_ACTIVE_SYS))) {
 		/* Migrate to dirty scheduler... */
@@ -9625,8 +9628,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 		goto sched_out_proc;
 	    }
-	    ASSERT(!p->scheduler_data);
-	    p->scheduler_data = esdp;
 	}
 	else {
             if (!(state & ERTS_PSFLGS_DIRTY_WORK)) {
-- 
cgit v1.2.3


From bedf71d2cae1f01b83829e5c8432342cfb1238c7 Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikpelinux@gmail.com>
Date: Tue, 29 May 2018 17:42:14 +0200
Subject: do not call abort_signal_task() with invalid data

---
 erts/emulator/beam/erl_port_task.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 4a3671df0c..3953a4c2e9 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -1567,8 +1567,9 @@ fail:
 	erts_port_dec_refc(pp);
 
     if (ptp) {
-        abort_signal_task(pp, ERTS_PROC2PORT_SIG_ABORT,
-                          ptp->type, &ptp->u.alive.td, 0);
+        if (ptp->type == ERTS_PORT_TASK_PROC_SIG)
+            abort_signal_task(pp, ERTS_PROC2PORT_SIG_ABORT,
+                              ptp->type, &ptp->u.alive.td, 0);
 	port_task_free(ptp);
     }
 
-- 
cgit v1.2.3


From 1c8365272a55301609dc8c46aa49fa0ac048506e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 18 May 2018 17:59:33 +0200
Subject: erts: Add system_info(ets_count)

---
 erts/emulator/beam/erl_bif_info.c        |  3 +++
 erts/emulator/beam/erl_db.c              | 26 +++++++++++++++++++-------
 erts/emulator/beam/erl_db.h              |  3 ++-
 erts/emulator/test/system_info_SUITE.erl | 17 +++++++++++++++++
 4 files changed, 41 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 8b2b1a58c7..5789fa8e71 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3129,6 +3129,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     else if (ERTS_IS_ATOM_STR("ets_limit",BIF_ARG_1)) {
         BIF_RET(make_small(erts_db_get_max_tabs()));
     }
+    else if (ERTS_IS_ATOM_STR("ets_count",BIF_ARG_1)) {
+        BIF_RET(make_small(erts_ets_table_count()));
+    }
     else if (ERTS_IS_ATOM_STR("atom_limit",BIF_ARG_1)) {
         BIF_RET(make_small(erts_get_atom_limit()));
     }
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index c1eaaeee06..c9ab75fc88 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -450,7 +450,7 @@ save_sched_table(Process *c_p, DbTable *tb)
     DbTable *first;
 
     ASSERT(esdp);
-    esdp->ets_tables.count++;
+    erts_atomic_inc_nob(&esdp->ets_tables.count);
     erts_refc_inc(&tb->common.refc, 1);
 
     first = esdp->ets_tables.clist;
@@ -474,8 +474,8 @@ remove_sched_table(ErtsSchedulerData *esdp, DbTable *tb)
     ASSERT(erts_get_ref_numbers_thr_id(ERTS_MAGIC_BIN_REFN(tb->common.btid))
            == (Uint32) esdp->no);
 
-    ASSERT(esdp->ets_tables.count > 0);
-    esdp->ets_tables.count--;
+    ASSERT(erts_atomic_read_nob(&esdp->ets_tables.count) > 0);
+    erts_atomic_dec_nob(&esdp->ets_tables.count);
 
     eaydp = ERTS_SCHED_AUX_YIELD_DATA(esdp, ets_all);
     if (eaydp->ongoing) {
@@ -2446,7 +2446,7 @@ ets_all_reply(ErtsSchedulerData *esdp, ErtsEtsAllReq **reqpp,
         ASSERT(!*tablepp);
 
         /* Max heap size needed... */
-        sz = esdp->ets_tables.count;
+        sz = erts_atomic_read_nob(&esdp->ets_tables.count);
         sz *= ERTS_MAGIC_REF_THING_SIZE + 2;
         sz += 3 + ERTS_REF_THING_SIZE;
         hfragp = new_message_buffer(sz);
@@ -2530,7 +2530,8 @@ erts_handle_yielded_ets_all_request(ErtsSchedulerData *esdp,
             if (!eaydp->queue)
                 return 0; /* All work completed! */
 
-            if (yc < ERTS_ETS_ALL_TB_YCNT_START && yc > esdp->ets_tables.count)
+            if (yc < ERTS_ETS_ALL_TB_YCNT_START &&
+                yc > erts_atomic_read_nob(&esdp->ets_tables.count))
                 return 1; /* Yield! */
 
             eaydp->ongoing = ongoing = eaydp->queue;
@@ -2609,7 +2610,6 @@ BIF_RETTYPE ets_internal_request_all_0(BIF_ALIST_0)
     BIF_RET(ref);
 }
 
-
 /*
 ** db_slot(Db, Slot) -> [Items].
 */
@@ -3539,7 +3539,7 @@ erts_ets_sched_spec_data_init(ErtsSchedulerData *esdp)
     eaydp->tab = NULL;
     eaydp->queue = NULL;
     esdp->ets_tables.clist = NULL;
-    esdp->ets_tables.count = 0;
+    erts_atomic_init_nob(&esdp->ets_tables.count, 0);
 }
 
 
@@ -4347,6 +4347,18 @@ erts_db_get_max_tabs()
     return db_max_tabs;
 }
 
+Uint erts_ets_table_count(void)
+{
+    Uint tb_count = 0;
+    Uint six;
+
+    for (six = 0; six < erts_no_schedulers; six++) {
+        ErtsSchedulerData *esdp = &erts_aligned_scheduler_data[six].esd;
+        tb_count += erts_atomic_read_nob(&esdp->ets_tables.count);
+    }
+    return tb_count;
+}
+
 /*
  * For testing of meta tables only.
  *
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index db86c81914..1664db9a41 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -45,7 +45,7 @@ typedef struct {
 } ErtsEtsAllYieldData;
 
 typedef struct {
-    Uint count;
+    erts_atomic_t count;
     DbTable *clist;
 } ErtsEtsTables;
 
@@ -69,6 +69,7 @@ typedef struct {
 /*TT*/
 
 Uint erts_get_ets_misc_mem_size(void);
+Uint erts_ets_table_count(void);
 
 typedef struct {
     DbTableCommon common;
diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl
index fdf4aab24d..7309908337 100644
--- a/erts/emulator/test/system_info_SUITE.erl
+++ b/erts/emulator/test/system_info_SUITE.erl
@@ -37,6 +37,7 @@
 
 -export([process_count/1, system_version/1, misc_smoke_tests/1,
          heap_size/1, wordsize/1, memory/1, ets_limit/1, atom_limit/1,
+         ets_count/1,
          atom_count/1]).
 
 suite() ->
@@ -45,6 +46,7 @@ suite() ->
 
 all() -> 
     [process_count, system_version, misc_smoke_tests,
+     ets_count,
      heap_size, wordsize, memory, ets_limit, atom_limit, atom_count].
 
 %%%
@@ -478,6 +480,21 @@ get_node_name(Config) ->
 		 ++ "-"
 		 ++ integer_to_list(erlang:unique_integer([positive]))).
 
+ets_count(Config) when is_list(Config) ->
+    [ets_count_do([Type | Named])
+     || Type <- [set, bag, duplicate_bag, ordered_set],
+        Named <- [[named_table], []]
+    ],
+    ok.
+
+ets_count_do(Opts) ->
+    Before = erlang:system_info(ets_count),
+    T = ets:new(?MODULE, Opts),
+    After = erlang:system_info(ets_count),
+    After = Before + 1,
+    ets:delete(T),
+    Before = erlang:system_info(ets_count).
+
 
 %% Verify system_info(ets_limit) reflects max ETS table settings.
 ets_limit(Config0) when is_list(Config0) ->
-- 
cgit v1.2.3


From 9611459cddf27fd648aad4d799ca1fc4cf92f351 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 21 May 2018 13:52:47 +0200
Subject: erts: Refactor usage of am_atom_put to ERTS_MAKE_AM

and let compiler determine string lengths.

These were actually wrong in erl_db.c:

count_trap\0
replace_tra
select_tra
---
 erts/emulator/beam/erl_alloc.c      | 32 +++++++++++++++-----------------
 erts/emulator/beam/erl_bif_chksum.c |  2 +-
 erts/emulator/beam/erl_db.c         | 14 +++++++-------
 erts/emulator/beam/erl_db_util.c    |  8 +++-----
 erts/emulator/beam/erl_unicode.c    | 10 +++++-----
 erts/emulator/beam/utils.c          |  7 +++----
 erts/emulator/sys/common/erl_mmap.c |  4 ++--
 7 files changed, 36 insertions(+), 41 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index d99d2ea57b..575d6ca867 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -2821,20 +2821,20 @@ erts_allocator_options(void *proc)
 		Eterm as[4];
 		Eterm ts[4];
 
-		as[l] = am_atom_put("e", 1);
+		as[l] = ERTS_MAKE_AM("e");
 		ts[l++] = am_true;
 
 		switch (a) {
 		case ERTS_ALC_A_SYSTEM:
-		    as[l] = am_atom_put("m", 1);
-		    ts[l++] = am_atom_put("libc", 4);
+		    as[l] = ERTS_MAKE_AM("m");
+		    ts[l++] = ERTS_MAKE_AM("libc");
 		    if(sas.trim_threshold >= 0) {
-			as[l] = am_atom_put("tt", 2);
+			as[l] = ERTS_MAKE_AM("tt");
 			ts[l++] = erts_bld_uint(hpp, szp,
 						(Uint) sas.trim_threshold);
 		    }
 		    if(sas.top_pad >= 0) {
-			as[l] = am_atom_put("tp", 2);
+			as[l] = ERTS_MAKE_AM("tp");
 			ts[l++] = erts_bld_uint(hpp, szp, (Uint) sas.top_pad);
 		    }
 		    break;
@@ -2848,7 +2848,7 @@ erts_allocator_options(void *proc)
 
 	}
 	else {
-	    Eterm atom = am_atom_put("e", 1);
+	    Eterm atom = ERTS_MAKE_AM("e");
 	    Eterm term = am_false;
 	    tmp = erts_bld_2tup_list(hpp, szp, 1, &atom, &term);
 	}
@@ -2859,12 +2859,12 @@ erts_allocator_options(void *proc)
 
 #if HAVE_ERTS_MSEG
     if (use_mseg) {
-	atoms[length] = am_atom_put("mseg_alloc", 10);
+	atoms[length] = ERTS_MAKE_AM("mseg_alloc");
 	terms[length++] = erts_mseg_info_options(0, NULL, NULL, hpp, szp);
     }
 #endif
 
-    atoms[length] = am_atom_put("alloc_util", 10); 
+    atoms[length] = ERTS_MAKE_AM("alloc_util");
     terms[length++] = erts_alcu_au_info_options(NULL, NULL, hpp, szp);
 
 #if HAVE_ERTS_MMAP
@@ -2874,17 +2874,15 @@ erts_allocator_options(void *proc)
 #endif
     {
 	Eterm o[1], v[1];
-	o[0] = am_atom_put("t", 1);
+	o[0] = ERTS_MAKE_AM("t");
 	v[0] = erts_mtrace_enabled ? am_true : am_false;
 
-	atoms[length] = am_atom_put("instr", 5); 
+	atoms[length] = ERTS_MAKE_AM("instr");
 	terms[length++] = erts_bld_2tup_list(hpp, szp, 1, o, v);
     }
 
-    atoms[length] = am_atom_put("lock_physical_memory", 20);
-    terms[length++] = (lock_all_physical_memory
-		       ? am_atom_put("all", 3)
-		       : am_atom_put("no", 2));
+    atoms[length] = ERTS_MAKE_AM("lock_physical_memory");
+    terms[length++] = (lock_all_physical_memory ? am_all : am_no);
 
     settings = erts_bld_2tup_list(hpp, szp, length, atoms, terms);
 
@@ -2899,10 +2897,10 @@ erts_allocator_options(void *proc)
 
 #if HAVE_ERTS_MSEG
     if (use_mseg)
-	terms[length++] = am_atom_put("mseg_alloc", 10);
+	terms[length++] = ERTS_MAKE_AM("mseg_alloc");
 #endif
 #if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
-    terms[length++] = am_atom_put("sys_aligned_alloc", 17);
+    terms[length++] = ERTS_MAKE_AM("sys_aligned_alloc");
 #endif
 #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
     terms[length++] = ERTS_MAKE_AM("literal_mmap");
@@ -2911,7 +2909,7 @@ erts_allocator_options(void *proc)
 
 #if defined(__GLIBC__)
     {
-	Eterm AM_glibc = am_atom_put("glibc", 5);
+	Eterm AM_glibc = ERTS_MAKE_AM("glibc");
 	Eterm version;
 
 	version = erts_bld_cons(hpp,
diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c
index 9095bcd380..cf92687595 100644
--- a/erts/emulator/beam/erl_bif_chksum.c
+++ b/erts/emulator/beam/erl_bif_chksum.c
@@ -44,7 +44,7 @@ void erts_init_bif_chksum(void)
 {
     /* Non visual BIF to trap to. */
     erts_init_trap_export(&chksum_md5_2_exp,
-			  am_erlang, am_atom_put("md5_trap",8), 2,
+			  am_erlang, ERTS_MAKE_AM("md5_trap"), 2,
 			  &md5_2);
 }
     
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index c9ab75fc88..df5253fb0d 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -3506,27 +3506,27 @@ void init_db(ErtsDbSpinCount db_spin_count)
 
     /* Non visual BIF to trap to. */
     erts_init_trap_export(&ets_select_delete_continue_exp,
-			  am_ets, am_atom_put("delete_trap",11), 1,
+			  am_ets, ERTS_MAKE_AM("delete_trap"), 1,
 			  &ets_select_delete_trap_1);
 
     /* Non visual BIF to trap to. */
     erts_init_trap_export(&ets_select_count_continue_exp,
-			  am_ets, am_atom_put("count_trap",11), 1,
+			  am_ets, ERTS_MAKE_AM("count_trap"), 1,
 			  &ets_select_count_1);
 
     /* Non visual BIF to trap to. */
     erts_init_trap_export(&ets_select_replace_continue_exp,
-                          am_ets, am_atom_put("replace_trap",11), 1,
+                          am_ets, ERTS_MAKE_AM("replace_trap"), 1,
                           &ets_select_replace_1);
 
     /* Non visual BIF to trap to. */
     erts_init_trap_export(&ets_select_continue_exp,
-			  am_ets, am_atom_put("select_trap",11), 1,
+			  am_ets, ERTS_MAKE_AM("select_trap"), 1,
 			  &ets_select_trap_1);
 
     /* Non visual BIF to trap to. */
     erts_init_trap_export(&ets_delete_continue_exp,
-			  am_ets, am_atom_put("delete_trap",11), 1,
+			  am_ets, ERTS_MAKE_AM("delete_trap"), 1,
 			  &ets_delete_trap);
 }
 
@@ -4155,7 +4155,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
     else if (What == am_data) {
 	print_table(ERTS_PRINT_STDOUT, NULL, 1, tb);
 	ret = am_true;
-    } else if (What == am_atom_put("fixed",5)) { 
+    } else if (ERTS_IS_ATOM_STR("fixed",What)) {
 	if (IS_FIXED(tb))
 	    ret = am_true;
 	else
@@ -4207,7 +4207,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	    ret = am_false;
 	}
 	erts_mtx_unlock(&tb->common.fixlock);
-    } else if (What == am_atom_put("stats",5)) {
+    } else if (ERTS_IS_ATOM_STR("stats",What)) {
 	if (IS_HASH_TABLE(tb->common.status)) {
 	    FloatDef f;
 	    DbHashStats stats;
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 37d261d0df..f1d47326b4 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2770,9 +2770,7 @@ Eterm db_format_dmc_err_info(Process *p, DMCErrInfo *ei)
 	    sys_strcpy(buff,tmp->error_string);
 	sl = sys_strlen(buff);
 	shp = HAlloc(p, sl * 2 + 5);
-	sev = (tmp->severity == dmcWarning) ? 
-	    am_atom_put("warning",7) :
-	    am_error;
+	sev = (tmp->severity == dmcWarning) ? am_warning : am_error;
 	tlist = buf_to_intlist(&shp, buff, sl, NIL);
 	tpl = TUPLE2(shp, sev, tlist);
 	shp += 3;
@@ -5180,7 +5178,7 @@ BIF_RETTYPE match_spec_test_3(BIF_ALIST_3)
 {
     Eterm res;
 #ifdef DMC_DEBUG
-    if (BIF_ARG_3 == am_atom_put("dis",3)) {
+    if (BIF_ARG_3 == ERTS_MAKE_AM("dis")) {
 	test_disassemble_next = 1;
 	BIF_RET(am_true);
     } else
@@ -5291,7 +5289,7 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace)
 	    erts_free(ERTS_ALC_T_DB_TMP, arr);
 	}
 	erts_bin_free(mps);
-	ret = TUPLE4(hp, am_atom_put("ok",2), res, flg, lint_res);
+	ret = TUPLE4(hp, am_ok, res, flg, lint_res);
     }
     return ret;
 }
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 8673e029e6..d225916ac5 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -79,23 +79,23 @@ void erts_init_unicode(void)
     max_loop_limit = CONTEXT_REDS * LOOP_FACTOR;
     /* Non visual BIFs to trap to. */
     erts_init_trap_export(&characters_to_utf8_trap_exp,
-			  am_erlang, am_atom_put("characters_to_utf8_trap",23), 3,
+			  am_erlang, ERTS_MAKE_AM("characters_to_utf8_trap"), 3,
 			  &characters_to_utf8_trap);
 
     erts_init_trap_export(&characters_to_list_trap_1_exp,
-			  am_erlang, am_atom_put("characters_to_list_trap_1",25), 3,
+			  am_erlang, ERTS_MAKE_AM("characters_to_list_trap_1"), 3,
 			  &characters_to_list_trap_1);
 
     erts_init_trap_export(&characters_to_list_trap_2_exp,
-			  am_erlang, am_atom_put("characters_to_list_trap_2",25), 3,
+			  am_erlang, ERTS_MAKE_AM("characters_to_list_trap_2"), 3,
 			  &characters_to_list_trap_2);
 
     erts_init_trap_export(&characters_to_list_trap_3_exp,
-			  am_erlang, am_atom_put("characters_to_list_trap_3",25), 3,
+			  am_erlang, ERTS_MAKE_AM("characters_to_list_trap_3"), 3,
 			  &characters_to_list_trap_3);
 
     erts_init_trap_export(&characters_to_list_trap_4_exp,
-			  am_erlang, am_atom_put("characters_to_list_trap_4",25), 1,
+			  am_erlang, ERTS_MAKE_AM("characters_to_list_trap_4"), 1,
 			  &characters_to_list_trap_4);
 
     c_to_b_int_trap_exportp =  erts_export_put(am_unicode,am_characters_to_binary_int,2);
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index d74052d8b2..87709edc15 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1972,8 +1972,7 @@ static void do_send_logger_message(Eterm gl, Eterm tag, Eterm format, Eterm args
         break;
     }
 
-    md = MAP2(hp, am_emulator, am_true,
-              am_atom_put("tag", 3), el_tag);
+    md = MAP2(hp, am_emulator, am_true, ERTS_MAKE_AM("tag"), el_tag);
     hp += MAP2_SZ;
 
     if (is_nil(gl) && is_non_value(pid)) {
@@ -1994,14 +1993,14 @@ static void do_send_logger_message(Eterm gl, Eterm tag, Eterm format, Eterm args
         /* no gl */
         md = MAP3(hp,
                   am_error_logger, md,
-                  am_atom_put("gl", 2), gl,
+                  ERTS_MAKE_AM("gl"), gl,
                   am_time, time);
         hp += MAP3_SZ;
         pid = NIL;
     } else {
         md = MAP4(hp,
                   am_error_logger, md,
-                  am_atom_put("gl", 2), gl,
+                  ERTS_MAKE_AM("gl"), gl,
                   am_pid, pid,
                   am_time, time);
         hp += MAP4_SZ;
diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index 5dadd8a5a6..145503bea7 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -2514,8 +2514,8 @@ Eterm erts_mmap_debug_info(Process* p)
                                             sizeof(values)/sizeof(*values),
                                             tags, values);
 
-        sa_list = TUPLE2(hp, am_atom_put("sa_free_segs",12), sa_list); hp+=3;
-        sua_list = TUPLE2(hp, am_atom_put("sua_free_segs",13), sua_list); hp+=3;
+        sa_list = TUPLE2(hp, ERTS_MAKE_AM("sa_free_segs"), sa_list); hp+=3;
+        sua_list = TUPLE2(hp, ERTS_MAKE_AM("sua_free_segs"), sua_list); hp+=3;
         list = CONS(hp, sua_list, list); hp+=2;
         list = CONS(hp, sa_list, list); hp+=2;
 
-- 
cgit v1.2.3


From 99dfe4573a6e89eb93c5fcec523f2a76af61d859 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 21 May 2018 13:53:58 +0200
Subject: erts: Rename one of delete_trap to select_delete_trap

---
 erts/emulator/beam/erl_db.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index df5253fb0d..a91a8d7c2e 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -3506,7 +3506,7 @@ void init_db(ErtsDbSpinCount db_spin_count)
 
     /* Non visual BIF to trap to. */
     erts_init_trap_export(&ets_select_delete_continue_exp,
-			  am_ets, ERTS_MAKE_AM("delete_trap"), 1,
+			  am_ets, ERTS_MAKE_AM("select_delete_trap"), 1,
 			  &ets_select_delete_trap_1);
 
     /* Non visual BIF to trap to. */
-- 
cgit v1.2.3


From 29946f8628df35f000e37e5db8782decfa44876b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 21 May 2018 14:33:27 +0200
Subject: erts: Increase scalability of ets name lookup

by expanding the default size of the hash table
and increase number of locks.
---
 erts/emulator/beam/erl_db.c | 10 +++++++---
 erts/emulator/beam/erl_db.h |  2 +-
 2 files changed, 8 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index a91a8d7c2e..5bae1730e4 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -322,10 +322,10 @@ erts_db_make_tid(Process *c_p, DbTableCommon *tb)
 /* 
 ** The meta hash table of all NAMED ets tables
 */
-#  define META_NAME_TAB_LOCK_CNT 16
+#  define META_NAME_TAB_LOCK_CNT 256
 union {
     erts_rwmtx_t lck;
-    byte _cache_line_alignment[64];
+    byte align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_rwmtx_t))];
 }meta_name_tab_rwlocks[META_NAME_TAB_LOCK_CNT];
 static struct meta_name_tab_entry {
     union {
@@ -3491,7 +3491,11 @@ void init_db(ErtsDbSpinCount db_spin_count)
 		 db_max_tabs, ((Uint)1)<<SMALL_BITS);
     }
 
-    meta_name_tab_mask = (((Uint) 1)<<(bits-1)) - 1; /* At least half the size of main tab */
+    /*
+     * We don't have ony hard limit for number of tables anymore,                                                                            .
+     * but we use 'db_max_tabs' to determine size of name hash table.
+     */
+    meta_name_tab_mask = (((Uint) 1)<<bits) - 1;
     size = sizeof(struct meta_name_tab_entry)*(meta_name_tab_mask+1);
     meta_name_tab = erts_db_alloc_nt(ERTS_ALC_T_DB_TABLES, size);
     ERTS_ETS_MISC_MEM_ADD(size);
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index 1664db9a41..db1dec015c 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -94,7 +94,7 @@ union db_table {
     /*TT*/
 };
 
-#define DB_DEF_MAX_TABS 2053 /* Superseeded by environment variable 
+#define DB_DEF_MAX_TABS 8192 /* Superseeded by environment variable
 				"ERL_MAX_ETS_TABLES" */
 #define ERL_MAX_ETS_TABLES_ENV "ERL_MAX_ETS_TABLES"
 
-- 
cgit v1.2.3


From 8eeb653417b839f34e9a778820fc3f18395a3d8a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 4 Jun 2018 20:43:06 +0200
Subject: erts: Fix race between ets table deletion and auto-unfix

Problem:
1. Process A fixates table T.
2. Process B starts deleting table T (either by ets:delete or exit)
   and does tid_clear().
3. Process A exits and does proc_cleanup_fixed_table()
   and get NULL from btid2tab() and deallocates DbFixation.
4. Process B continues deleting table in free_fixations_locked()
   and finds the deallocated DbFixation in the fixing_procs tree.

Solution:
Wait with tid_clear() until after free_fixations_locked()
has traversed the fixing_procs tree.
---
 erts/emulator/beam/erl_db.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 6d4a895ef6..68d984014f 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1948,8 +1948,6 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
         save_owned_table(BIF_P, tb);
     }
 
-    tid_clear(BIF_P, tb);
-
     if (is_table_named(tb))
 	remove_named_tab(tb, 0);
     
@@ -1958,6 +1956,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
     tb->common.heir = am_none;
 
     reds -= free_fixations_locked(BIF_P, tb);
+    tid_clear(BIF_P, tb);
     db_unlock(tb, LCK_WRITE);
 
     if (free_table_continue(BIF_P, tb, reds) < 0) {
@@ -3680,7 +3679,6 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
                 && give_away_to_heir(c_p, tb)) {
                 break;
             }
-            tid_clear(c_p, tb);
             /* Clear all access bits. */
             tb->common.status &= ~(DB_PROTECTED | DB_PUBLIC | DB_PRIVATE);
             tb->common.status |= DB_DELETE;
@@ -3690,6 +3688,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 
             free_heir_data(tb);
             reds -= free_fixations_locked(c_p, tb);
+            tid_clear(c_p, tb);
             db_unlock(tb, LCK_WRITE);
             state->op = FREE_OWNED_TABLE;
             break;
@@ -3850,7 +3849,7 @@ static void free_fixations_op(DbFixation* fix, void* vctx)
     struct free_fixations_ctx* ctx = (struct free_fixations_ctx*) vctx;
     erts_aint_t diff;
 
-    ASSERT(!btid2tab(fix->tabs.btid));
+    ASSERT(btid2tab(fix->tabs.btid) == ctx->tb);
     ASSERT(fix->counter > 0);
     ASSERT(ctx->tb->common.status & DB_DELETE);
 
-- 
cgit v1.2.3


From bc50c1e4e0d7deaae19804964760d4ffa1e13cbb Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 5 Jun 2018 10:32:38 +0200
Subject: erts: Fix emulator log messages to use erlang:system_time

This was changed in the logger in 8aa64c90ddd20ec0ca8cc5fe92a6124324c51da5.
---
 erts/emulator/beam/utils.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index d74052d8b2..cd625406f7 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1946,7 +1946,7 @@ do_allocate_logger_message(Eterm gleader, ErtsMonotonicTime *ts, Eterm *pid,
     else
         sz += MAP4_SZ /* metadata map w gl w pid*/;
 
-    *ts = ERTS_MONOTONIC_TO_USEC(erts_get_monotonic_time(NULL)) + ERTS_MONOTONIC_OFFSET_USEC;
+    *ts = ERTS_MONOTONIC_TO_USEC(erts_get_monotonic_time(NULL) + erts_get_time_offset());
     erts_bld_sint64(NULL, &sz, *ts);
 
     *bp = new_message_buffer(sz);
-- 
cgit v1.2.3


From 30598c518f793bb7c3893d3996966dca5b858192 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 4 Jun 2018 14:42:38 +0200
Subject: erts: Fix broken signal queue

broken on master by
613cde66c25464121f2f6dace99782bad0e07d9b

Scenario:
proc_queue_signal() fails to send switched pending signal
due to state & ERTS_PSFLG_FREE,
and then calls erts_proc_sig_send_monitor_down() to enqueue to self
followed by sig_enqueue_trace_cleanup() that destroyed 'next' pointer
of enqueued signal.

Solution:
Switch order and do sig_enqueue_trace_cleanup() first.
---
 erts/emulator/beam/erl_proc_sig_queue.c | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index e9b41ad298..dc09efa7d7 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -445,6 +445,8 @@ sig_enqueue_trace_cleanup(ErtsMessage *first, ErtsSignal *sig, ErtsMessage *last
             case ERTS_SIG_Q_OP_TRACE_CHANGE_STATE:
                 destroy_trace_info((ErtsSigTraceInfo *) tmp_free);
                 break;
+            case ERTS_SIG_Q_OP_MONITOR:
+                break; /* ignore flushed pending signal */
             default:
                 ERTS_INTERNAL_ERROR("Unexpected signal op");
                 break;
@@ -688,19 +690,15 @@ first_last_done:
     erts_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ);
 
     if (res == 0) {
+        sig_enqueue_trace_cleanup(first, sig, last);
         if (pend_sig) {
             if (sig == pend_sig) {
                 /* We did a switch, callers signal is now pending (still ok) */
                 ASSERT(esdp->pending_signal.sig);
                 res = 1;
             }
-            else {
-                ASSERT(first == (ErtsMessage*)pend_sig);
-                first = first->next;
-            }
             erts_proc_sig_send_monitor_down((ErtsMonitor*)pend_sig, am_noproc);
         }
-        sig_enqueue_trace_cleanup(first, sig, last);
     }
 
     erts_proc_notify_new_sig(rp, state, 0);
-- 
cgit v1.2.3


From 1f5e17c5ce10982c45a7071b0c6b52f419b99402 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 4 Jun 2018 14:49:22 +0200
Subject: erts: Cleanup in proc_queue_signal

* Remove 'last' arg to sig_enqueue_trace_cleanup
* Only notify 'rp' if enqueue succeeded
---
 erts/emulator/beam/erl_proc_sig_queue.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index dc09efa7d7..1aa390d94d 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -423,13 +423,13 @@ sig_enqueue_trace(Process *c_p, ErtsMessage **sigp, int op,
 }
 
 static void
-sig_enqueue_trace_cleanup(ErtsMessage *first, ErtsSignal *sig, ErtsMessage *last)
+sig_enqueue_trace_cleanup(ErtsMessage *first, ErtsSignal *sig)
 {
     ErtsMessage *tmp;
 
     /* The usual case; no tracing signals... */
-    if (sig == (ErtsSignal *) first && sig == (ErtsSignal *) last) {
-        sig->common.next = NULL;
+    if (sig == (ErtsSignal *) first) {
+        ASSERT(sig->common.next == NULL);
         return;
     }
 
@@ -669,7 +669,7 @@ proc_queue_signal(Process *c_p, Eterm pid, ErtsSignal *sig, int op)
 first_last_done:
     sig->common.specific.next = NULL;
 
-    /* may add signals before and/or after sig */
+    /* may add signals before sig */
     sig_enqueue_trace(c_p, sigp, op, rp, &last_next);
 
     last->next = NULL;
@@ -690,18 +690,18 @@ first_last_done:
     erts_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ);
 
     if (res == 0) {
-        sig_enqueue_trace_cleanup(first, sig, last);
+        sig_enqueue_trace_cleanup(first, sig);
         if (pend_sig) {
+            erts_proc_sig_send_monitor_down((ErtsMonitor*)pend_sig, am_noproc);
             if (sig == pend_sig) {
                 /* We did a switch, callers signal is now pending (still ok) */
                 ASSERT(esdp->pending_signal.sig);
                 res = 1;
             }
-            erts_proc_sig_send_monitor_down((ErtsMonitor*)pend_sig, am_noproc);
         }
     }
-
-    erts_proc_notify_new_sig(rp, state, 0);
+    else
+        erts_proc_notify_new_sig(rp, state, 0);
 
     if (!is_normal_sched)
         erts_proc_dec_refc(rp);
-- 
cgit v1.2.3


From 6ccf40d1b46a3e880134994bd71836f15a3358d4 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 7 Jun 2018 20:15:07 +0200
Subject: erts: Fix race between ets table deletion and auto-unfix

Bug exists since ets-refs were introduced in 20.0
0d6dc895744c34c9c52fd42f4801a8a941864ae3.

Problem:
1. Process A fixates table T.
2. Process B starts deleting table T (either by ets:delete or exit)
   and does tid_clear().
3. Process A exits and does proc_cleanup_fixed_table()
   and get NULL from btid2tab() and deallocates DbFixation.
4. Process B continues deleting table in free_fixations_locked()
   and finds the deallocated DbFixation in the fixing_procs tree.

Solution:
Wait with tid_clear() until after free_fixations_locked()
has traversed the fixing_procs tree.
---
 erts/emulator/beam/erl_db.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 6d4a895ef6..68d984014f 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1948,8 +1948,6 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
         save_owned_table(BIF_P, tb);
     }
 
-    tid_clear(BIF_P, tb);
-
     if (is_table_named(tb))
 	remove_named_tab(tb, 0);
     
@@ -1958,6 +1956,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
     tb->common.heir = am_none;
 
     reds -= free_fixations_locked(BIF_P, tb);
+    tid_clear(BIF_P, tb);
     db_unlock(tb, LCK_WRITE);
 
     if (free_table_continue(BIF_P, tb, reds) < 0) {
@@ -3680,7 +3679,6 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
                 && give_away_to_heir(c_p, tb)) {
                 break;
             }
-            tid_clear(c_p, tb);
             /* Clear all access bits. */
             tb->common.status &= ~(DB_PROTECTED | DB_PUBLIC | DB_PRIVATE);
             tb->common.status |= DB_DELETE;
@@ -3690,6 +3688,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 
             free_heir_data(tb);
             reds -= free_fixations_locked(c_p, tb);
+            tid_clear(c_p, tb);
             db_unlock(tb, LCK_WRITE);
             state->op = FREE_OWNED_TABLE;
             break;
@@ -3850,7 +3849,7 @@ static void free_fixations_op(DbFixation* fix, void* vctx)
     struct free_fixations_ctx* ctx = (struct free_fixations_ctx*) vctx;
     erts_aint_t diff;
 
-    ASSERT(!btid2tab(fix->tabs.btid));
+    ASSERT(btid2tab(fix->tabs.btid) == ctx->tb);
     ASSERT(fix->counter > 0);
     ASSERT(ctx->tb->common.status & DB_DELETE);
 
-- 
cgit v1.2.3


From c5a6b03c9ee71bb13723f6ca6ff0e8f6cd16fe24 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 11 Jun 2018 15:56:59 +0200
Subject: erts: Fix incoming signal handling for exiting process

Seen symptom:
handle_process_info() called with is_alive=1 on exiting process,
reading broken Process.u.initial.module atom
overwritten by Process.u.terminate pointer.

Solution:
Let erts_proc_sig_handle_incoming() do nothing if process is
already exiting.
---
 erts/emulator/beam/erl_proc_sig_queue.c | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index 1aa390d94d..d6d22677e7 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -2723,6 +2723,9 @@ handle_process_info(Process *c_p, ErtsSigRecvTracing *tracing,
     Uint reds = 0;
     Process *rp;
 
+    ASSERT(!!is_alive == !(erts_atomic32_read_nob(&c_p->state)
+                           & ERTS_PSFLG_EXITING));
+
     if (pisig->msgq_len_offset != ERTS_PROC_SIG_PI_MSGQ_LEN_IGNORE) {
         /*
          * Request requires message queue data to be updated
@@ -3007,10 +3010,8 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep,
     ERTS_HDBG_CHECK_SIGNAL_PRIV_QUEUE(c_p, 0);
     ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
 
-    if (local_only)
-        state = -1; /* can never be a valid state... */
-    else {
-        state = erts_atomic32_read_nob(&c_p->state);
+    state = erts_atomic32_read_nob(&c_p->state);
+    if (!local_only) {
         if (ERTS_PSFLG_SIG_IN_Q & state) {
             erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ);
             erts_proc_sig_fetch(c_p);
@@ -3023,13 +3024,15 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep,
     yield = 0;
 
     if (!c_p->sig_qs.cont) {
-        if (state == -1)
-            *statep = erts_atomic32_read_nob(&c_p->state);
-        else
-            *statep = state;
+        *statep = state;
         return !0;
     }
 
+    if (state & ERTS_PSFLG_EXITING) {
+        *statep = state;
+        return 0;
+    }
+
     next_nm_sig = &c_p->sig_qs.nmsigs.next;
 
     setup_tracing_state(c_p, &tracing);
-- 
cgit v1.2.3


From efff173c1c0fbc080b2761a89c3a4a929404a190 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 12 Jun 2018 12:16:56 +0200
Subject: erts: Fix monitor_SUITE:remote_remove_monitor

to actually run remote on the slave node that it starts.
---
 erts/emulator/test/monitor_SUITE.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/monitor_SUITE.erl b/erts/emulator/test/monitor_SUITE.erl
index c7250a9d26..27351dc5c1 100644
--- a/erts/emulator/test/monitor_SUITE.erl
+++ b/erts/emulator/test/monitor_SUITE.erl
@@ -314,7 +314,7 @@ local_remove_monitor(Config) when is_list(Config) ->
 
 remote_remove_monitor(Config) when is_list(Config) ->
     {ok, N} = test_server:start_node(demonitor_flush, slave, []),
-    Gs = generate(fun () -> start_remove_monitor_group(node()) end,
+    Gs = generate(fun () -> start_remove_monitor_group(N) end,
                   ?RM_MON_GROUPS),
     {True, False} = lists:foldl(fun (G, {T, F}) ->
                                         receive
-- 
cgit v1.2.3


From ac9ed8e2575dca9e5ebe2f7af00dfe4da87b1bcc Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 13 Jun 2018 17:49:38 +0200
Subject: erts: Fix MacOS pollset bug

that could cause clearing of fd-bits past allocated bit-vectors.

Bug introduced on master by
988f5f5e8061ce2e135a314ca782788eda478a06
---
 erts/emulator/sys/common/erl_poll.c | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index ced8a4a2a7..e3f27caea7 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -141,12 +141,26 @@ typedef struct {
     size_t sz;
     fd_set* ptr;
 }ERTS_fd_set;
-#  define ERTS_FD_CLR(fd, fds)	FD_CLR((fd), (fds)->ptr)
-#  define ERTS_FD_SET(fd, fds)	FD_SET((fd), (fds)->ptr)
-#  define ERTS_FD_ISSET(fd,fds) FD_ISSET((fd), (fds)->ptr)
+
 #  define ERTS_FD_ZERO(fds)	memset((fds)->ptr, 0, (fds)->sz)
 #  define ERTS_FD_SIZE(n)	((((n)+NFDBITS-1)/NFDBITS)*sizeof(fd_mask))
 
+static ERTS_INLINE void ERTS_FD_CLR(int fd, ERTS_fd_set *fds)
+{
+    ASSERT(ERTS_FD_SIZE(fd+1) <= fds->sz);
+    FD_CLR(fd, fds->ptr);
+}
+static ERTS_INLINE void ERTS_FD_SET(int fd, ERTS_fd_set *fds)
+{
+    ASSERT(ERTS_FD_SIZE(fd+1) <= fds->sz);
+    FD_SET(fd, fds->ptr);
+}
+static ERTS_INLINE int ERTS_FD_ISSET(int fd, ERTS_fd_set *fds)
+{
+    ASSERT(ERTS_FD_SIZE(fd+1) <= fds->sz);
+    return FD_ISSET(fd, fds->ptr);
+}
+
 static void ERTS_FD_COPY(ERTS_fd_set *src, ERTS_fd_set *dst)
 {
     if (dst->sz != src->sz) {
@@ -626,8 +640,15 @@ ensure_select_fds(int fd, ERTS_fd_set* in, ERTS_fd_set* out)
 	grow_select_fds(fd, out);
     }
 }
+static ERTS_INLINE int
+check_select_fds(int fd, ERTS_fd_set* in, ERTS_fd_set* out)
+{
+    ASSERT(in->sz == out->sz);
+    return (ERTS_FD_SIZE(fd+1) <= in->sz);
+}
 #else
 #  define ensure_select_fds(fd, in, out) do {} while(0)
+#  define check_select_fds(fd, in, out) (1)
 #endif /* _DARWIN_UNLIMITED_SELECT */
 
 #if !ERTS_POLL_USE_CONCURRENT_UPDATE
@@ -1085,8 +1106,10 @@ static int update_pollset(ErtsPollSet *ps, ErtsPollResFd pr[], int fd)
             res++;
         }
 
-        ERTS_FD_CLR(fd, &ps->input_fds);
-        ERTS_FD_CLR(fd, &ps->output_fds);
+        if (check_select_fds(fd, &ps->input_fds, &ps->output_fds)) {
+            ERTS_FD_CLR(fd, &ps->input_fds);
+            ERTS_FD_CLR(fd, &ps->output_fds);
+        }
 
         if (ps->fds_status[fd].used_events) {
             erts_atomic_dec_nob(&ps->no_of_user_fds);
-- 
cgit v1.2.3


From 4013f4840d8d540c63fa7901c2d8d1653ae45821 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 21 May 2018 07:45:55 +0200
Subject: Fix erroneous schedule of freed/exiting processes

When scheduled out, the process was never checked for the FREE state
before rescheduling, which meant that a system task could sneak in
and cause a double-free later on.
---
 erts/emulator/beam/erl_process.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index ee6d56b5d7..b0db2da243 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -6900,8 +6900,9 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
 	enqueue = ERTS_ENQUEUE_NOT;
 
 	n &= ~running_flgs;
-	if ((a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS))
-	    || (a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) {
+	if ((!!(a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS))
+	    | ((a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE))
+            & !(a & ERTS_PSFLG_FREE)) {
 	    enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a);
 	}
 	a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
@@ -14069,7 +14070,9 @@ erts_continue_exit_process(Process *p)
 	    n = e = a;
 	    ASSERT(a & ERTS_PSFLG_EXITING);
 	    n |= ERTS_PSFLG_FREE;
-	    n &= ~ERTS_PSFLG_ACTIVE;
+            n &= ~(ERTS_PSFLG_ACTIVE
+                   | ERTS_PSFLG_ACTIVE_SYS
+                   | ERTS_PSFLG_DIRTY_ACTIVE_SYS);
 	    if ((n & ERTS_PSFLG_IN_RUNQ) && !refc_inced) {
 		erts_proc_inc_refc(p);
 		refc_inced = 1;
-- 
cgit v1.2.3


From 8ca58cf82e430775ab1c2adfab31f77f649c3f96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 14 Jun 2018 08:26:07 +0200
Subject: Don't enqueue system tasks if target process is in fail_state

The fail state wasn't re-checked in the state change loop; only
the FREE state was checked. In addition to that, we would leave
the task in the queue when bailing out which could lead to a
double-free.

This commit backports active_sys_enqueue from master to make it
easier to merge onwards.
---
 erts/emulator/beam/erl_process.c     | 228 ++++++++++++++++++-----------------
 erts/emulator/test/process_SUITE.erl |  54 ++++++++-
 2 files changed, 171 insertions(+), 111 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b0db2da243..dad4006beb 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -7156,129 +7156,72 @@ erts_schedule_process(Process *p, erts_aint32_t state, ErtsProcLocks locks)
     schedule_process(p, state, locks);
 }
 
-static int
-schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
-			  erts_aint32_t *fail_state_p)
-{
-    int res;
-    int locked;
-    ErtsProcSysTaskQs *stqs, *free_stqs;
-    erts_aint32_t fail_state, state, a, n, enq_prio;
+/* Enqueues the given sys task on the process and schedules it. The task may be
+ * NULL if only scheduling is desired. */
+static ERTS_INLINE erts_aint32_t
+active_sys_enqueue(Process *p, ErtsProcSysTask *sys_task,
+                   erts_aint32_t task_prio, erts_aint32_t enable_flags,
+                   erts_aint32_t state, erts_aint32_t *fail_state_p)
+{
+    int runnable_procs = erts_system_profile_flags.runnable_procs;
+    erts_aint32_t n, a, enq_prio, fail_state;
+    int already_scheduled;
+    int status_locked;
     int enqueue; /* < 0 -> use proxy */
-    unsigned int prof_runnable_procs;
 
+    enable_flags |= ERTS_PSFLG_ACTIVE_SYS;
     fail_state = *fail_state_p;
-
-    res = 1; /* prepare for success */
-    st->next = st->prev = st; /* Prep for empty prio queue */
-    state = erts_smp_atomic32_read_nob(&p->state);
-    prof_runnable_procs = erts_system_profile_flags.runnable_procs;
-    locked = 0;
-    free_stqs = NULL;
-    if (state & ERTS_PSFLG_ACTIVE_SYS)
-	stqs = NULL;
-    else {
-    alloc_qs:
-	stqs = proc_sys_task_queues_alloc();
-	stqs->qmask = 1 << prio;
-	stqs->ncount = 0;
-	stqs->q[PRIORITY_MAX] = NULL;
-	stqs->q[PRIORITY_HIGH] = NULL;
-	stqs->q[PRIORITY_NORMAL] = NULL;
-	stqs->q[PRIORITY_LOW] = NULL;
-	stqs->q[prio] = st;
-    }
-
-    if (!locked) {
-	locked = 1;
-	erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
-
-	state = erts_smp_atomic32_read_nob(&p->state);
-	if (state & fail_state) {
-	    *fail_state_p = (state & fail_state);
-	    free_stqs = stqs;
-	    res = 0;
-	    goto cleanup;
-	}
-    }
-
-    if (!p->sys_task_qs) {
-	if (stqs)
-	    p->sys_task_qs = stqs;
-	else
-	    goto alloc_qs;
-    }
-    else {
-	free_stqs = stqs;
-	stqs = p->sys_task_qs;
-	if (!stqs->q[prio]) {
-	    stqs->q[prio] = st;
-	    stqs->qmask |= 1 << prio;
-	}
-	else {
-	    st->next = stqs->q[prio];
-	    st->prev = stqs->q[prio]->prev;
-	    st->next->prev = st;
-	    st->prev->next = st;
-	    ASSERT(stqs->qmask & (1 << prio));
-	}
-    }
-
-    if (ERTS_PSFLGS_GET_ACT_PRIO(state) > prio) {
-	erts_aint32_t n, a, e;
-	/* Need to elevate actual prio */
-
-	a = state;
-	do {
-	    if (ERTS_PSFLGS_GET_ACT_PRIO(a) <= prio) {
-		n = a;
-		break;
-	    }
-	    n = e = a;
-	    n &= ~ERTS_PSFLGS_ACT_PRIO_MASK;
-	    n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET);
-	    a = erts_smp_atomic32_cmpxchg_nob(&p->state, n, e);    
-	} while (a != e);
-	state = n;
-    }
-
-
-    a = state;
+    already_scheduled = 0;
+    status_locked = 0;
     enq_prio = -1;
+    a = state;
 
-    /* Status lock prevents out of order "runnable proc" trace msgs */
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p));
+    ERTS_SMP_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)));
+    ASSERT(fail_state & (ERTS_PSFLG_EXITING | ERTS_PSFLG_FREE));
+    ASSERT(!(fail_state & enable_flags));
+    ASSERT(!(state & ERTS_PSFLG_PROXY));
 
-    if (!prof_runnable_procs) {
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-	locked = 0;
+    /* When runnable_procs is enabled, we need to take the status lock to
+     * prevent trace messages from being sent in the wrong order. The lock must
+     * be held over the call to add2runq.
+     *
+     * Otherwise, we only need to take it when we're enqueuing a task and can
+     * safely release it before add2runq. */
+    if (sys_task || runnable_procs) {
+        erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+        status_locked = 1;
     }
 
-    ASSERT(!(state & ERTS_PSFLG_PROXY));
-
     while (1) {
 	erts_aint32_t e;
 	n = e = a;
 
-	if (a & ERTS_PSFLG_FREE)
-	    goto cleanup; /* We don't want to schedule free processes... */
+	if (a & fail_state) {
+            *fail_state_p = a & fail_state;
+	    goto cleanup;
+        }
 
 	enqueue = ERTS_ENQUEUE_NOT;
-	n |= ERTS_PSFLG_ACTIVE_SYS;
+	n |= enable_flags;
+
 	if (!(a & (ERTS_PSFLG_RUNNING
 		   | ERTS_PSFLG_RUNNING_SYS
 		   | ERTS_PSFLG_DIRTY_RUNNING
-		   | ERTS_PSFLG_DIRTY_RUNNING_SYS)))
+		   | ERTS_PSFLG_DIRTY_RUNNING_SYS))) {
 	    enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a);
+        }
+
 	a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
-	if (a == e)
+	if (a == e) {
 	    break;
-	if (a == n && enqueue == ERTS_ENQUEUE_NOT)
-	    goto cleanup;
+	}
+        else if (a == n && enqueue == ERTS_ENQUEUE_NOT) {
+	    already_scheduled = 1;
+            break;
+        }
     }
 
-    if (prof_runnable_procs) {
-
+    if (!already_scheduled && runnable_procs) {
 	if (!(a & (ERTS_PSFLG_ACTIVE_SYS
 		   | ERTS_PSFLG_RUNNING
 		   | ERTS_PSFLG_RUNNING_SYS
@@ -7288,24 +7231,89 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
 	    /* We activated a prevously inactive process */
 	    profile_runnable_proc(p, am_active);
 	}
+    }
 
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-	locked = 0;
+    if (sys_task) {
+        ErtsProcSysTaskQs *stqs = p->sys_task_qs;
+
+        if (!stqs) {
+            sys_task->next = sys_task->prev = sys_task;
+
+            stqs = proc_sys_task_queues_alloc();
+
+            stqs->qmask = 1 << task_prio;
+            stqs->ncount = 0;
+            stqs->q[PRIORITY_MAX] = NULL;
+            stqs->q[PRIORITY_HIGH] = NULL;
+            stqs->q[PRIORITY_NORMAL] = NULL;
+            stqs->q[PRIORITY_LOW] = NULL;
+            stqs->q[task_prio] = sys_task;
+
+            p->sys_task_qs = stqs;
+        }
+        else {
+            if (!stqs->q[task_prio]) {
+                sys_task->next = sys_task->prev = sys_task;
+
+                stqs->q[task_prio] = sys_task;
+                stqs->qmask |= 1 << task_prio;
+            }
+            else {
+                sys_task->next = stqs->q[task_prio];
+                sys_task->prev = stqs->q[task_prio]->prev;
+                sys_task->next->prev = sys_task;
+                sys_task->prev->next = sys_task;
+                ASSERT(stqs->qmask & (1 << task_prio));
+            }
+        }
+    }
+
+    if (status_locked && !runnable_procs) {
+        erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+        status_locked = 0;
     }
 
-    add2runq(enqueue, enq_prio, p, n, NULL);
+    if (!already_scheduled) {
+        add2runq(enqueue, enq_prio, p, n, NULL);
+    }
 
 cleanup:
+    if (status_locked) {
+        erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+    }
 
-    if (locked)
-	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+    return n;
+}
+
+static int
+schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st,
+			  erts_aint32_t *fail_state_p)
+{
+    erts_aint32_t fail_state, state;
 
-    if (free_stqs)
-	proc_sys_task_queues_free(free_stqs);
+    /* Elevate priority if needed. */
+    state = erts_smp_atomic32_read_nob(&p->state);
+    if (ERTS_PSFLGS_GET_ACT_PRIO(state) > prio) {
+        erts_aint32_t n, a, e;
 
-    ERTS_SMP_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)));
+        a = state;
+        do {
+            if (ERTS_PSFLGS_GET_ACT_PRIO(a) <= prio) {
+                n = a;
+                break;
+            }
+            n = e = a;
+            n &= ~ERTS_PSFLGS_ACT_PRIO_MASK;
+            n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET);
+            a = erts_smp_atomic32_cmpxchg_nob(&p->state, n, e);
+        } while (a != e);
 
-    return res;
+        state = n;
+    }
+
+    fail_state = *fail_state_p;
+
+    return !(active_sys_enqueue(p, st, prio, 0, state, fail_state_p) & fail_state);
 }
 
 static ERTS_INLINE int
diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index a8bcfac84d..899a5c26bd 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -58,6 +58,7 @@
 	 no_priority_inversion2/1,
 	 system_task_blast/1,
 	 system_task_on_suspended/1,
+         system_task_failed_enqueue/1,
 	 gc_request_when_gc_disabled/1,
 	 gc_request_blast_when_gc_disabled/1]).
 -export([prio_server/2, prio_client/2, init/1, handle_event/2]).
@@ -104,7 +105,7 @@ groups() ->
        otp_7738_resume]},
      {system_task, [],
       [no_priority_inversion, no_priority_inversion2,
-       system_task_blast, system_task_on_suspended,
+       system_task_blast, system_task_on_suspended, system_task_failed_enqueue,
        gc_request_when_gc_disabled, gc_request_blast_when_gc_disabled]}].
 
 init_per_suite(Config) ->
@@ -2531,6 +2532,57 @@ system_task_on_suspended(Config) when is_list(Config) ->
 	    ok
     end.
 
+%% When a system task couldn't be enqueued due to the process being in an
+%% incompatible state, it would linger in the system task list and get executed
+%% anyway the next time the process was scheduled. This would result in a
+%% double-free at best.
+%%
+%% This test continuously purges modules while other processes run dirty code,
+%% which will provoke this error as ERTS_PSTT_CPC can't be enqueued while a
+%% process is running dirty code.
+system_task_failed_enqueue(Config) when is_list(Config) ->
+    case erlang:system_info(dirty_cpu_schedulers) of
+        N when N > 0 ->
+            system_task_failed_enqueue_1(Config);
+        _ ->
+            {skipped, "No dirty scheduler support"}
+    end.
+
+system_task_failed_enqueue_1(Config) ->
+    Priv = proplists:get_value(priv_dir, Config),
+
+    Purgers = [spawn_link(fun() -> purge_loop(Priv, Id) end)
+               || Id <- lists:seq(1, erlang:system_info(schedulers))],
+    Hogs = [spawn_link(fun() -> dirty_loop() end)
+            || _ <- lists:seq(1, erlang:system_info(dirty_cpu_schedulers))],
+
+    ct:sleep(5000),
+
+    [begin
+         unlink(Pid),
+         exit(Pid, kill)
+     end || Pid <- (Purgers ++ Hogs)],
+
+    ok.
+
+purge_loop(PrivDir, Id) ->
+    Mod = "failed_enq_" ++ integer_to_list(Id),
+    Path = PrivDir ++ "/" ++ Mod,
+    file:write_file(Path ++ ".erl",
+                    "-module('" ++ Mod ++ "').\n" ++
+                        "-export([t/0]).\n" ++
+                        "t() -> ok."),
+    purge_loop_1(Path).
+purge_loop_1(Path) ->
+    {ok, Mod} = compile:file(Path, []),
+    erlang:delete_module(Mod),
+    erts_code_purger:purge(Mod),
+    purge_loop_1(Path).
+
+dirty_loop() ->
+    ok = erts_debug:dirty_cpu(reschedule, 10000),
+    dirty_loop().
+
 gc_request_when_gc_disabled(Config) when is_list(Config) ->
     AIS = erts_debug:set_internal_state(available_internal_state, true),
     gc_request_when_gc_disabled_do(ref),
-- 
cgit v1.2.3


From 5ca92e2eac1e84fd22f60e7abc3aa2b0ff1cb42b Mon Sep 17 00:00:00 2001
From: Henrik Nord <henrik@erlang.org>
Date: Mon, 18 Jun 2018 14:51:18 +0200
Subject: Update copyright year

---
 erts/emulator/beam/atom.c                                 | 2 +-
 erts/emulator/beam/atom.h                                 | 2 +-
 erts/emulator/beam/beam_debug.c                           | 2 +-
 erts/emulator/beam/bif_instrs.tab                         | 2 +-
 erts/emulator/beam/big.c                                  | 2 +-
 erts/emulator/beam/binary.c                               | 2 +-
 erts/emulator/beam/bs_instrs.tab                          | 2 +-
 erts/emulator/beam/code_ix.c                              | 2 +-
 erts/emulator/beam/copy.c                                 | 2 +-
 erts/emulator/beam/erl_afit_alloc.c                       | 2 +-
 erts/emulator/beam/erl_alloc.h                            | 2 +-
 erts/emulator/beam/erl_alloc_util.c                       | 2 +-
 erts/emulator/beam/erl_alloc_util.h                       | 2 +-
 erts/emulator/beam/erl_ao_firstfit_alloc.c                | 2 +-
 erts/emulator/beam/erl_ao_firstfit_alloc.h                | 2 +-
 erts/emulator/beam/erl_arith.c                            | 2 +-
 erts/emulator/beam/erl_async.c                            | 2 +-
 erts/emulator/beam/erl_bestfit_alloc.c                    | 2 +-
 erts/emulator/beam/erl_bif_binary.c                       | 2 +-
 erts/emulator/beam/erl_bif_chksum.c                       | 2 +-
 erts/emulator/beam/erl_bif_ddll.c                         | 2 +-
 erts/emulator/beam/erl_bif_re.c                           | 2 +-
 erts/emulator/beam/erl_bif_unique.h                       | 2 +-
 erts/emulator/beam/erl_binary.h                           | 2 +-
 erts/emulator/beam/erl_bits.h                             | 2 +-
 erts/emulator/beam/erl_db.c                               | 2 +-
 erts/emulator/beam/erl_db.h                               | 2 +-
 erts/emulator/beam/erl_db_hash.c                          | 2 +-
 erts/emulator/beam/erl_db_hash.h                          | 2 +-
 erts/emulator/beam/erl_db_tree.c                          | 2 +-
 erts/emulator/beam/erl_db_tree.h                          | 2 +-
 erts/emulator/beam/erl_db_util.h                          | 2 +-
 erts/emulator/beam/erl_dirty_bif.tab                      | 2 +-
 erts/emulator/beam/erl_drv_thread.c                       | 2 +-
 erts/emulator/beam/erl_goodfit_alloc.c                    | 2 +-
 erts/emulator/beam/erl_io_queue.c                         | 2 +-
 erts/emulator/beam/erl_lock_check.h                       | 2 +-
 erts/emulator/beam/erl_map.c                              | 2 +-
 erts/emulator/beam/erl_msacc.c                            | 2 +-
 erts/emulator/beam/erl_msacc.h                            | 2 +-
 erts/emulator/beam/erl_mtrace.c                           | 2 +-
 erts/emulator/beam/erl_nfunc_sched.c                      | 2 +-
 erts/emulator/beam/erl_nfunc_sched.h                      | 2 +-
 erts/emulator/beam/erl_nif.h                              | 2 +-
 erts/emulator/beam/erl_nif_api_funcs.h                    | 2 +-
 erts/emulator/beam/erl_node_container_utils.h             | 2 +-
 erts/emulator/beam/erl_port_task.c                        | 2 +-
 erts/emulator/beam/erl_printf_term.c                      | 2 +-
 erts/emulator/beam/erl_process_dict.c                     | 2 +-
 erts/emulator/beam/erl_process_dict.h                     | 2 +-
 erts/emulator/beam/erl_sched_spec_pre_alloc.c             | 2 +-
 erts/emulator/beam/erl_sched_spec_pre_alloc.h             | 2 +-
 erts/emulator/beam/erl_thr_progress.c                     | 2 +-
 erts/emulator/beam/erl_thr_progress.h                     | 2 +-
 erts/emulator/beam/erl_thr_queue.c                        | 2 +-
 erts/emulator/beam/erl_thr_queue.h                        | 2 +-
 erts/emulator/beam/erl_trace.c                            | 2 +-
 erts/emulator/beam/erl_trace.h                            | 2 +-
 erts/emulator/beam/erl_utils.h                            | 2 +-
 erts/emulator/beam/erl_vm.h                               | 2 +-
 erts/emulator/beam/erlang_dtrace.d                        | 2 +-
 erts/emulator/beam/erlang_lttng.h                         | 2 +-
 erts/emulator/beam/export.h                               | 2 +-
 erts/emulator/beam/external.c                             | 2 +-
 erts/emulator/beam/hash.c                                 | 2 +-
 erts/emulator/beam/instrs.tab                             | 2 +-
 erts/emulator/beam/lttng-wrapper.h                        | 2 +-
 erts/emulator/beam/module.c                               | 2 +-
 erts/emulator/beam/module.h                               | 2 +-
 erts/emulator/beam/ops.tab                                | 2 +-
 erts/emulator/beam/packet_parser.c                        | 2 +-
 erts/emulator/beam/register.c                             | 2 +-
 erts/emulator/beam/safe_hash.c                            | 2 +-
 erts/emulator/beam/safe_hash.h                            | 2 +-
 erts/emulator/beam/sys.h                                  | 2 +-
 erts/emulator/beam/utils.c                                | 2 +-
 erts/emulator/drivers/common/gzio.h                       | 2 +-
 erts/emulator/drivers/common/inet_drv.c                   | 2 +-
 erts/emulator/drivers/unix/ttsl_drv.c                     | 2 +-
 erts/emulator/hipe/hipe_amd64.c                           | 2 +-
 erts/emulator/hipe/hipe_amd64_bifs.m4                     | 2 +-
 erts/emulator/hipe/hipe_arm_bifs.m4                       | 2 +-
 erts/emulator/hipe/hipe_bif2.c                            | 2 +-
 erts/emulator/hipe/hipe_bif2.tab                          | 2 +-
 erts/emulator/hipe/hipe_bif_list.m4                       | 2 +-
 erts/emulator/hipe/hipe_debug.c                           | 2 +-
 erts/emulator/hipe/hipe_gc.c                              | 2 +-
 erts/emulator/hipe/hipe_mode_switch.c                     | 2 +-
 erts/emulator/hipe/hipe_native_bif.h                      | 2 +-
 erts/emulator/hipe/hipe_ops.tab                           | 2 +-
 erts/emulator/hipe/hipe_ppc_bifs.m4                       | 2 +-
 erts/emulator/hipe/hipe_primops.h                         | 2 +-
 erts/emulator/hipe/hipe_process.h                         | 2 +-
 erts/emulator/hipe/hipe_risc_stack.c                      | 2 +-
 erts/emulator/hipe/hipe_signal.h                          | 2 +-
 erts/emulator/hipe/hipe_sparc_bifs.m4                     | 2 +-
 erts/emulator/hipe/hipe_x86_bifs.m4                       | 2 +-
 erts/emulator/hipe/hipe_x86_stack.c                       | 2 +-
 erts/emulator/nifs/common/prim_file_nif.c                 | 2 +-
 erts/emulator/nifs/common/prim_file_nif.h                 | 2 +-
 erts/emulator/nifs/unix/unix_prim_file.c                  | 2 +-
 erts/emulator/nifs/win32/win_prim_file.c                  | 2 +-
 erts/emulator/sys/common/erl_mmap.c                       | 2 +-
 erts/emulator/sys/common/erl_mmap.h                       | 2 +-
 erts/emulator/sys/common/erl_mseg.c                       | 2 +-
 erts/emulator/sys/common/erl_mseg.h                       | 2 +-
 erts/emulator/sys/common/erl_os_monotonic_time_extender.c | 2 +-
 erts/emulator/sys/common/erl_os_monotonic_time_extender.h | 2 +-
 erts/emulator/sys/common/erl_poll.c                       | 2 +-
 erts/emulator/sys/common/erl_poll.h                       | 2 +-
 erts/emulator/sys/common/erl_poll_api.h                   | 2 +-
 erts/emulator/sys/common/erl_sys_common_misc.c            | 2 +-
 erts/emulator/sys/unix/erl_child_setup.c                  | 2 +-
 erts/emulator/sys/unix/erl_unix_sys.h                     | 2 +-
 erts/emulator/sys/unix/sys_drivers.c                      | 2 +-
 erts/emulator/sys/unix/sys_float.c                        | 2 +-
 erts/emulator/sys/unix/sys_time.c                         | 2 +-
 erts/emulator/sys/unix/sys_uds.c                          | 2 +-
 erts/emulator/sys/unix/sys_uds.h                          | 2 +-
 erts/emulator/sys/win32/erl_win32_sys_ddll.c              | 2 +-
 erts/emulator/sys/win32/erl_win_dyn_driver.h              | 2 +-
 erts/emulator/sys/win32/erl_win_sys.h                     | 2 +-
 erts/emulator/sys/win32/sys_interrupt.c                   | 2 +-
 erts/emulator/sys/win32/sys_time.c                        | 2 +-
 erts/emulator/test/alloc_SUITE.erl                        | 2 +-
 erts/emulator/test/alloc_SUITE_data/migration.c           | 2 +-
 erts/emulator/test/beam_SUITE.erl                         | 2 +-
 erts/emulator/test/beam_literals_SUITE.erl                | 2 +-
 erts/emulator/test/big_SUITE.erl                          | 2 +-
 erts/emulator/test/binary_SUITE.erl                       | 2 +-
 erts/emulator/test/decode_packet_SUITE.erl                | 2 +-
 erts/emulator/test/dgawd_handler.erl                      | 2 +-
 erts/emulator/test/distribution_SUITE.erl                 | 2 +-
 erts/emulator/test/driver_SUITE.erl                       | 2 +-
 erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c       | 2 +-
 erts/emulator/test/dump_SUITE.erl                         | 2 +-
 erts/emulator/test/efile_SUITE.erl                        | 2 +-
 erts/emulator/test/exception_SUITE.erl                    | 2 +-
 erts/emulator/test/fun_SUITE.erl                          | 2 +-
 erts/emulator/test/iovec_SUITE.erl                        | 2 +-
 erts/emulator/test/lcnt_SUITE.erl                         | 2 +-
 erts/emulator/test/match_spec_SUITE.erl                   | 2 +-
 erts/emulator/test/module_info_SUITE.erl                  | 2 +-
 erts/emulator/test/nif_SUITE.erl                          | 2 +-
 erts/emulator/test/nif_SUITE_data/nif_SUITE.c             | 2 +-
 erts/emulator/test/num_bif_SUITE.erl                      | 2 +-
 erts/emulator/test/register_SUITE.erl                     | 2 +-
 erts/emulator/test/sensitive_SUITE.erl                    | 2 +-
 erts/emulator/test/smoke_test_SUITE.erl                   | 2 +-
 erts/emulator/test/statistics_SUITE.erl                   | 2 +-
 erts/emulator/test/system_info_SUITE.erl                  | 2 +-
 erts/emulator/test/system_profile_SUITE.erl               | 2 +-
 erts/emulator/test/tracer_SUITE.erl                       | 2 +-
 erts/emulator/test/tuple_SUITE.erl                        | 2 +-
 erts/emulator/test/z_SUITE.erl                            | 2 +-
 erts/emulator/utils/make_driver_tab                       | 2 +-
 erts/emulator/utils/make_tables                           | 2 +-
 157 files changed, 157 insertions(+), 157 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index e5b7616a0d..5381611fab 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h
index 385120a8d9..ca920679c6 100644
--- a/erts/emulator/beam/atom.h
+++ b/erts/emulator/beam/atom.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index b8a8d06315..6d3b99c43e 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/bif_instrs.tab b/erts/emulator/beam/bif_instrs.tab
index 0f074280db..00854471a9 100644
--- a/erts/emulator/beam/bif_instrs.tab
+++ b/erts/emulator/beam/bif_instrs.tab
@@ -2,7 +2,7 @@
 //
 // %CopyrightBegin%
 //
-// Copyright Ericsson AB 2017. All Rights Reserved.
+// Copyright Ericsson AB 2017-2018. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index c5cb268f09..84338769e0 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index d53f75c279..6a349764b2 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab
index 94e0000c8b..61eb02a7a2 100644
--- a/erts/emulator/beam/bs_instrs.tab
+++ b/erts/emulator/beam/bs_instrs.tab
@@ -2,7 +2,7 @@
 //
 // %CopyrightBegin%
 //
-// Copyright Ericsson AB 2017. All Rights Reserved.
+// Copyright Ericsson AB 2017-2018. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c
index 34e46f5f33..50352b4084 100644
--- a/erts/emulator/beam/code_ix.c
+++ b/erts/emulator/beam/code_ix.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 7769a914db..e7bfd04b73 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c
index 23efe3bba4..38289ea78a 100644
--- a/erts/emulator/beam/erl_afit_alloc.c
+++ b/erts/emulator/beam/erl_afit_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 578a3717d9..fcb58ff58a 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index fdf355d503..a5740a08cf 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index cbae8ce98a..f26ace1534 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
index ebbe4af53d..3f0ab33597 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.c
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h
index dad864801f..68df9e0a49 100644
--- a/erts/emulator/beam/erl_ao_firstfit_alloc.h
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c
index b6625db0d3..144fb56ea5 100644
--- a/erts/emulator/beam/erl_arith.c
+++ b/erts/emulator/beam/erl_arith.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index 3ceb2fd368..605a2b3461 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c
index 85fc4c3a85..9cb1199c2a 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.c
+++ b/erts/emulator/beam/erl_bestfit_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 469f6a1ea8..a2610bf2e1 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2010-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c
index cf92687595..cce8472ccb 100644
--- a/erts/emulator/beam/erl_bif_chksum.c
+++ b/erts/emulator/beam/erl_bif_chksum.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 294bce115f..4cda0948a0 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index 4d769c2d46..bbc64eb9aa 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h
index 6af9d4ac8c..40b70667c0 100644
--- a/erts/emulator/beam/erl_bif_unique.h
+++ b/erts/emulator/beam/erl_bif_unique.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2014-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 7dfd0c273a..08edb43c49 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h
index a3816fa820..7beef5cfda 100644
--- a/erts/emulator/beam/erl_bits.h
+++ b/erts/emulator/beam/erl_bits.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 82e31f4cab..36d83d93f4 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index db1dec015c..23975d208f 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 74d63325e6..b988a19cf4 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index 08e5b13db1..eae5537ba4 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1998-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 0692583dd4..788718ab09 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_tree.h b/erts/emulator/beam/erl_db_tree.h
index dc1b93d410..54da2a6bc1 100644
--- a/erts/emulator/beam/erl_db_tree.h
+++ b/erts/emulator/beam/erl_db_tree.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1998-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 73d242449e..6ec3b4f98f 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_dirty_bif.tab b/erts/emulator/beam/erl_dirty_bif.tab
index 10c76d2579..086275fbe5 100644
--- a/erts/emulator/beam/erl_dirty_bif.tab
+++ b/erts/emulator/beam/erl_dirty_bif.tab
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 2016. All Rights Reserved.
+# Copyright Ericsson AB 2016-2018. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 4cf42fce57..c5dbc87dee 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2007-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c
index e3ba67f0af..01d4aa54ff 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.c
+++ b/erts/emulator/beam/erl_goodfit_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2003-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c
index d779d1031a..2ae5b56b5c 100644
--- a/erts/emulator/beam/erl_io_queue.c
+++ b/erts/emulator/beam/erl_io_queue.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2017. All Rights Reserved.
+ * Copyright Ericsson AB 2017-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h
index 138bc810bd..d10e32985a 100644
--- a/erts/emulator/beam/erl_lock_check.h
+++ b/erts/emulator/beam/erl_lock_check.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 48154b5d0f..cba17d3e6a 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2014-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index d13d6080e1..375b004b5b 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2014-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h
index 895b1ae319..abea18b340 100644
--- a/erts/emulator/beam/erl_msacc.h
+++ b/erts/emulator/beam/erl_msacc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2014-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c
index 2807b443a1..6e0a0dcff7 100644
--- a/erts/emulator/beam/erl_mtrace.c
+++ b/erts/emulator/beam/erl_mtrace.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2003-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_nfunc_sched.c b/erts/emulator/beam/erl_nfunc_sched.c
index f97e86bf95..b8cf2bee0e 100644
--- a/erts/emulator/beam/erl_nfunc_sched.c
+++ b/erts/emulator/beam/erl_nfunc_sched.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2016. All Rights Reserved.
+ * Copyright Ericsson AB 2016-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h
index b8a4e4ebc3..1cb252eba5 100644
--- a/erts/emulator/beam/erl_nfunc_sched.h
+++ b/erts/emulator/beam/erl_nfunc_sched.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2016. All Rights Reserved.
+ * Copyright Ericsson AB 2016-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 1906da732b..4c09496ef1 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 61f8fcf6ed..81f64f2390 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h
index 99e938266b..eb23e1eaa5 100644
--- a/erts/emulator/beam/erl_node_container_utils.h
+++ b/erts/emulator/beam/erl_node_container_utils.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 3953a4c2e9..4928d80f27 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2006-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index 910f241a3a..990a01b96f 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 38be3938cd..64ee483079 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1999-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h
index b89b387f5a..3ff2354f91 100644
--- a/erts/emulator/beam/erl_process_dict.h
+++ b/erts/emulator/beam/erl_process_dict.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
index 4a6e02281a..9766e76a83 100644
--- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.h b/erts/emulator/beam/erl_sched_spec_pre_alloc.h
index d232db0e69..b119c59ab3 100644
--- a/erts/emulator/beam/erl_sched_spec_pre_alloc.h
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 96824dc06e..aa08eb40ec 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h
index 8c029bcf99..8329995b24 100644
--- a/erts/emulator/beam/erl_thr_progress.h
+++ b/erts/emulator/beam/erl_thr_progress.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c
index 548c2768e5..aab7c199d2 100644
--- a/erts/emulator/beam/erl_thr_queue.c
+++ b/erts/emulator/beam/erl_thr_queue.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_thr_queue.h b/erts/emulator/beam/erl_thr_queue.h
index 163a25318d..29b58063ac 100644
--- a/erts/emulator/beam/erl_thr_queue.h
+++ b/erts/emulator/beam/erl_thr_queue.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index f4161b14f2..53a020e7a5 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 3228e19809..bccf31606e 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index e4087e0ac8..b3bfa69052 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2012-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2012-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index f8391fb665..4089fac48e 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erlang_dtrace.d b/erts/emulator/beam/erlang_dtrace.d
index c47a37eb62..8792138d53 100644
--- a/erts/emulator/beam/erlang_dtrace.d
+++ b/erts/emulator/beam/erlang_dtrace.d
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2016.
+ * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2018.
  * All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/erts/emulator/beam/erlang_lttng.h b/erts/emulator/beam/erlang_lttng.h
index feb05f4f4c..9b93d77f6e 100644
--- a/erts/emulator/beam/erlang_lttng.h
+++ b/erts/emulator/beam/erlang_lttng.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h
index 194e514b12..ae8dfa4cf8 100644
--- a/erts/emulator/beam/export.h
+++ b/erts/emulator/beam/export.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index fb42969a8f..904993ceb6 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c
index 6a31489473..8954dbb06c 100644
--- a/erts/emulator/beam/hash.c
+++ b/erts/emulator/beam/hash.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index 6a531fcc09..42c1168f85 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -2,7 +2,7 @@
 //
 // %CopyrightBegin%
 //
-// Copyright Ericsson AB 2017. All Rights Reserved.
+// Copyright Ericsson AB 2017-2018. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/lttng-wrapper.h b/erts/emulator/beam/lttng-wrapper.h
index e7f2971bf7..ad4852374a 100644
--- a/erts/emulator/beam/lttng-wrapper.h
+++ b/erts/emulator/beam/lttng-wrapper.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index 1712dc803c..0642a06123 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index a3f1ce1705..00efd129ff 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 8b2d9098a8..88ede3bb60 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1997-2017. All Rights Reserved.
+# Copyright Ericsson AB 1997-2018. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c
index de1d481105..4b526887b5 100644
--- a/erts/emulator/beam/packet_parser.c
+++ b/erts/emulator/beam/packet_parser.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2008-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index 92a0854ad3..c7e02c6d48 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c
index 73306030ae..0d816a81a9 100644
--- a/erts/emulator/beam/safe_hash.c
+++ b/erts/emulator/beam/safe_hash.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2008-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/safe_hash.h b/erts/emulator/beam/safe_hash.h
index af97b4cb4d..bd81e3022b 100644
--- a/erts/emulator/beam/safe_hash.h
+++ b/erts/emulator/beam/safe_hash.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2008-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index be6ab57eeb..bb22548587 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 78d9d5e613..19b1312ee3 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/drivers/common/gzio.h b/erts/emulator/drivers/common/gzio.h
index e331b5208b..20433a1a17 100644
--- a/erts/emulator/drivers/common/gzio.h
+++ b/erts/emulator/drivers/common/gzio.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 1a68f65b52..6fd3bb9fbf 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/drivers/unix/ttsl_drv.c b/erts/emulator/drivers/unix/ttsl_drv.c
index 7355df6059..28c6cc0f94 100644
--- a/erts/emulator/drivers/unix/ttsl_drv.c
+++ b/erts/emulator/drivers/unix/ttsl_drv.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c
index f23f341e6d..71cbb7c060 100644
--- a/erts/emulator/hipe/hipe_amd64.c
+++ b/erts/emulator/hipe/hipe_amd64.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2004-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_amd64_bifs.m4 b/erts/emulator/hipe/hipe_amd64_bifs.m4
index cf4c59c9af..4c866b3b68 100644
--- a/erts/emulator/hipe/hipe_amd64_bifs.m4
+++ b/erts/emulator/hipe/hipe_amd64_bifs.m4
@@ -2,7 +2,7 @@ changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2004-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_arm_bifs.m4 b/erts/emulator/hipe/hipe_arm_bifs.m4
index 554faa2567..421915dd72 100644
--- a/erts/emulator/hipe/hipe_arm_bifs.m4
+++ b/erts/emulator/hipe/hipe_arm_bifs.m4
@@ -2,7 +2,7 @@ changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c
index df377b2153..7e04f7d9c0 100644
--- a/erts/emulator/hipe/hipe_bif2.c
+++ b/erts/emulator/hipe/hipe_bif2.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_bif2.tab b/erts/emulator/hipe/hipe_bif2.tab
index c4da44606a..8925291769 100644
--- a/erts/emulator/hipe/hipe_bif2.tab
+++ b/erts/emulator/hipe/hipe_bif2.tab
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2001-2016. All Rights Reserved.
+# Copyright Ericsson AB 2001-2018. All Rights Reserved.
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index 33b3cc1ee5..7468860c37 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2004-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_debug.c b/erts/emulator/hipe/hipe_debug.c
index 929b2a9432..138e4f7da3 100644
--- a/erts/emulator/hipe/hipe_debug.c
+++ b/erts/emulator/hipe/hipe_debug.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c
index aaedba1afd..7bd1de0117 100644
--- a/erts/emulator/hipe/hipe_gc.c
+++ b/erts/emulator/hipe/hipe_gc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2004-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index 0a65e317ed..052cf9c263 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -2,7 +2,7 @@
  * %CopyrightBegin%
 
  *
- * Copyright Ericsson AB 2001-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h
index ba42b126be..ce96778dea 100644
--- a/erts/emulator/hipe/hipe_native_bif.h
+++ b/erts/emulator/hipe/hipe_native_bif.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_ops.tab b/erts/emulator/hipe/hipe_ops.tab
index 19a3820a6a..8dd81558f2 100644
--- a/erts/emulator/hipe/hipe_ops.tab
+++ b/erts/emulator/hipe/hipe_ops.tab
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 2001-2016. All Rights Reserved.
+# Copyright Ericsson AB 2001-2018. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_ppc_bifs.m4 b/erts/emulator/hipe/hipe_ppc_bifs.m4
index 283fbbb200..2ced443ce4 100644
--- a/erts/emulator/hipe/hipe_ppc_bifs.m4
+++ b/erts/emulator/hipe/hipe_ppc_bifs.m4
@@ -2,7 +2,7 @@ changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2004-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_primops.h b/erts/emulator/hipe/hipe_primops.h
index c5f10672f3..f4a4b4a07c 100644
--- a/erts/emulator/hipe/hipe_primops.h
+++ b/erts/emulator/hipe/hipe_primops.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_process.h b/erts/emulator/hipe/hipe_process.h
index 18354ba0a6..d412535968 100644
--- a/erts/emulator/hipe/hipe_process.h
+++ b/erts/emulator/hipe/hipe_process.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c
index bb93a918a2..b64afb1ba5 100644
--- a/erts/emulator/hipe/hipe_risc_stack.c
+++ b/erts/emulator/hipe/hipe_risc_stack.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_signal.h b/erts/emulator/hipe/hipe_signal.h
index 524def11a4..14bc0ef360 100644
--- a/erts/emulator/hipe/hipe_signal.h
+++ b/erts/emulator/hipe/hipe_signal.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_sparc_bifs.m4 b/erts/emulator/hipe/hipe_sparc_bifs.m4
index 1b49fa57fd..54684fbfb9 100644
--- a/erts/emulator/hipe/hipe_sparc_bifs.m4
+++ b/erts/emulator/hipe/hipe_sparc_bifs.m4
@@ -2,7 +2,7 @@ changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_x86_bifs.m4 b/erts/emulator/hipe/hipe_x86_bifs.m4
index 9cb343d067..0f6b9a8c8a 100644
--- a/erts/emulator/hipe/hipe_x86_bifs.m4
+++ b/erts/emulator/hipe/hipe_x86_bifs.m4
@@ -2,7 +2,7 @@ changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c
index 615e07917a..8cfc541f0d 100644
--- a/erts/emulator/hipe/hipe_x86_stack.c
+++ b/erts/emulator/hipe/hipe_x86_stack.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/nifs/common/prim_file_nif.c b/erts/emulator/nifs/common/prim_file_nif.c
index bbd9becb47..a05d50b333 100644
--- a/erts/emulator/nifs/common/prim_file_nif.c
+++ b/erts/emulator/nifs/common/prim_file_nif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson 2017. All Rights Reserved.
+ * Copyright Ericsson 2017-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/nifs/common/prim_file_nif.h b/erts/emulator/nifs/common/prim_file_nif.h
index 4194cdc7d9..099c06c48c 100644
--- a/erts/emulator/nifs/common/prim_file_nif.h
+++ b/erts/emulator/nifs/common/prim_file_nif.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson 2017. All Rights Reserved.
+ * Copyright Ericsson 2017-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/nifs/unix/unix_prim_file.c b/erts/emulator/nifs/unix/unix_prim_file.c
index 2b112dda76..dea73db18a 100644
--- a/erts/emulator/nifs/unix/unix_prim_file.c
+++ b/erts/emulator/nifs/unix/unix_prim_file.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson 2017. All Rights Reserved.
+ * Copyright Ericsson 2017-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/nifs/win32/win_prim_file.c b/erts/emulator/nifs/win32/win_prim_file.c
index 044bee62cf..f7fae3c637 100644
--- a/erts/emulator/nifs/win32/win_prim_file.c
+++ b/erts/emulator/nifs/win32/win_prim_file.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson 2017. All Rights Reserved.
+ * Copyright Ericsson 2017-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index 145503bea7..b0d9fc0776 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h
index c1f9668b18..539daea419 100644
--- a/erts/emulator/sys/common/erl_mmap.h
+++ b/erts/emulator/sys/common/erl_mmap.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2013-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2013-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index ced3d61525..030e5b00a7 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h
index af275c18be..ea9060ddac 100644
--- a/erts/emulator/sys/common/erl_mseg.h
+++ b/erts/emulator/sys/common/erl_mseg.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/common/erl_os_monotonic_time_extender.c b/erts/emulator/sys/common/erl_os_monotonic_time_extender.c
index 341845cc2a..5844e7eeb7 100644
--- a/erts/emulator/sys/common/erl_os_monotonic_time_extender.c
+++ b/erts/emulator/sys/common/erl_os_monotonic_time_extender.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2015-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2015-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/common/erl_os_monotonic_time_extender.h b/erts/emulator/sys/common/erl_os_monotonic_time_extender.h
index 53c32579d5..f6659fe973 100644
--- a/erts/emulator/sys/common/erl_os_monotonic_time_extender.h
+++ b/erts/emulator/sys/common/erl_os_monotonic_time_extender.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2015. All Rights Reserved.
+ * Copyright Ericsson AB 2015-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index e3f27caea7..70b5532af9 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2006-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/common/erl_poll.h b/erts/emulator/sys/common/erl_poll.h
index e9a667cac1..e1cea7eb8b 100644
--- a/erts/emulator/sys/common/erl_poll.h
+++ b/erts/emulator/sys/common/erl_poll.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/common/erl_poll_api.h b/erts/emulator/sys/common/erl_poll_api.h
index 04beb37d1c..1170a549b9 100644
--- a/erts/emulator/sys/common/erl_poll_api.h
+++ b/erts/emulator/sys/common/erl_poll_api.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2006-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c
index 41a6fcb7e1..2541ab5d31 100644
--- a/erts/emulator/sys/common/erl_sys_common_misc.c
+++ b/erts/emulator/sys/common/erl_sys_common_misc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/unix/erl_child_setup.c b/erts/emulator/sys/unix/erl_child_setup.c
index 10601529a4..221ee2a69d 100644
--- a/erts/emulator/sys/unix/erl_child_setup.c
+++ b/erts/emulator/sys/unix/erl_child_setup.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2002-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index 5bfe5a8e2d..ae7a3ea23e 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c
index 117855acf0..872c3a80b1 100644
--- a/erts/emulator/sys/unix/sys_drivers.c
+++ b/erts/emulator/sys/unix/sys_drivers.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/unix/sys_float.c b/erts/emulator/sys/unix/sys_float.c
index a82c15bd32..832074f679 100644
--- a/erts/emulator/sys/unix/sys_float.c
+++ b/erts/emulator/sys/unix/sys_float.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/unix/sys_time.c b/erts/emulator/sys/unix/sys_time.c
index ef05380d17..8ba575b7b6 100644
--- a/erts/emulator/sys/unix/sys_time.c
+++ b/erts/emulator/sys/unix/sys_time.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/unix/sys_uds.c b/erts/emulator/sys/unix/sys_uds.c
index 278c6b6ba1..c328fd00bb 100644
--- a/erts/emulator/sys/unix/sys_uds.c
+++ b/erts/emulator/sys/unix/sys_uds.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2002-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/unix/sys_uds.h b/erts/emulator/sys/unix/sys_uds.h
index 26c91d6a00..49a4b39250 100644
--- a/erts/emulator/sys/unix/sys_uds.h
+++ b/erts/emulator/sys/unix/sys_uds.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/win32/erl_win32_sys_ddll.c b/erts/emulator/sys/win32/erl_win32_sys_ddll.c
index fc2179328f..7fe1f5cc78 100644
--- a/erts/emulator/sys/win32/erl_win32_sys_ddll.c
+++ b/erts/emulator/sys/win32/erl_win32_sys_ddll.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/win32/erl_win_dyn_driver.h b/erts/emulator/sys/win32/erl_win_dyn_driver.h
index 0d1a6d4c87..c683e8cf49 100644
--- a/erts/emulator/sys/win32/erl_win_dyn_driver.h
+++ b/erts/emulator/sys/win32/erl_win_dyn_driver.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h
index 1f53452d17..b00ba287e2 100644
--- a/erts/emulator/sys/win32/erl_win_sys.h
+++ b/erts/emulator/sys/win32/erl_win_sys.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/win32/sys_interrupt.c b/erts/emulator/sys/win32/sys_interrupt.c
index 02aa50500f..cee269eed4 100644
--- a/erts/emulator/sys/win32/sys_interrupt.c
+++ b/erts/emulator/sys/win32/sys_interrupt.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/win32/sys_time.c b/erts/emulator/sys/win32/sys_time.c
index 25c2ad385c..a1dd14f871 100644
--- a/erts/emulator/sys/win32/sys_time.c
+++ b/erts/emulator/sys/win32/sys_time.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2018. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl
index 88ff2a7a92..343afe85e6 100644
--- a/erts/emulator/test/alloc_SUITE.erl
+++ b/erts/emulator/test/alloc_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2003-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2003-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/alloc_SUITE_data/migration.c b/erts/emulator/test/alloc_SUITE_data/migration.c
index 1d974225fc..78f3a633e8 100644
--- a/erts/emulator/test/alloc_SUITE_data/migration.c
+++ b/erts/emulator/test/alloc_SUITE_data/migration.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2014-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2018. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/beam_SUITE.erl b/erts/emulator/test/beam_SUITE.erl
index bdf8f6c34e..d3b3b96b14 100644
--- a/erts/emulator/test/beam_SUITE.erl
+++ b/erts/emulator/test/beam_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1998-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/beam_literals_SUITE.erl b/erts/emulator/test/beam_literals_SUITE.erl
index b447ca0210..82a5e2b172 100644
--- a/erts/emulator/test/beam_literals_SUITE.erl
+++ b/erts/emulator/test/beam_literals_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/big_SUITE.erl b/erts/emulator/test/big_SUITE.erl
index 5939d024ae..0a42b09903 100644
--- a/erts/emulator/test/big_SUITE.erl
+++ b/erts/emulator/test/big_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index a3c3daac15..23c675733c 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/decode_packet_SUITE.erl b/erts/emulator/test/decode_packet_SUITE.erl
index 0ccdbd7ee8..ef13b515fb 100644
--- a/erts/emulator/test/decode_packet_SUITE.erl
+++ b/erts/emulator/test/decode_packet_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2008-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2008-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/dgawd_handler.erl b/erts/emulator/test/dgawd_handler.erl
index 29b9d6ac7b..b66b5a073f 100644
--- a/erts/emulator/test/dgawd_handler.erl
+++ b/erts/emulator/test/dgawd_handler.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index 45dd922ff0..885c66331c 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 40c7cc11e1..7aff3a6ea1 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c b/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c
index fa58e9d5ec..9e96923e17 100644
--- a/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2007-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/dump_SUITE.erl b/erts/emulator/test/dump_SUITE.erl
index 8d18d46d92..d0237b78cc 100644
--- a/erts/emulator/test/dump_SUITE.erl
+++ b/erts/emulator/test/dump_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2005-2017. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl
index 821381bf0d..7dcf302742 100644
--- a/erts/emulator/test/efile_SUITE.erl
+++ b/erts/emulator/test/efile_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl
index 60d14ce841..aec66cb9a3 100644
--- a/erts/emulator/test/exception_SUITE.erl
+++ b/erts/emulator/test/exception_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/fun_SUITE.erl b/erts/emulator/test/fun_SUITE.erl
index 7d29ebec52..73fe9b0d8f 100644
--- a/erts/emulator/test/fun_SUITE.erl
+++ b/erts/emulator/test/fun_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/iovec_SUITE.erl b/erts/emulator/test/iovec_SUITE.erl
index 963b7e2501..d17a28d47f 100644
--- a/erts/emulator/test/iovec_SUITE.erl
+++ b/erts/emulator/test/iovec_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2017. All Rights Reserved.
+%% Copyright Ericsson AB 2017-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/lcnt_SUITE.erl b/erts/emulator/test/lcnt_SUITE.erl
index dfffd662e2..87b97037d6 100644
--- a/erts/emulator/test/lcnt_SUITE.erl
+++ b/erts/emulator/test/lcnt_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2017. All Rights Reserved.
+%% Copyright Ericsson AB 2017-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl
index 4415d8d1b9..21de6b1002 100644
--- a/erts/emulator/test/match_spec_SUITE.erl
+++ b/erts/emulator/test/match_spec_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2017. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/module_info_SUITE.erl b/erts/emulator/test/module_info_SUITE.erl
index 46a3bba732..93f9de0c28 100644
--- a/erts/emulator/test/module_info_SUITE.erl
+++ b/erts/emulator/test/module_info_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2005-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 100fa006e7..7c85cf2259 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 155bda6df0..f2ce6dbe67 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl
index 290bb61fc8..700734cd0b 100644
--- a/erts/emulator/test/num_bif_SUITE.erl
+++ b/erts/emulator/test/num_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/register_SUITE.erl b/erts/emulator/test/register_SUITE.erl
index 49da94a775..a7c0acbf17 100644
--- a/erts/emulator/test/register_SUITE.erl
+++ b/erts/emulator/test/register_SUITE.erl
@@ -1,7 +1,7 @@
 %% 
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/sensitive_SUITE.erl b/erts/emulator/test/sensitive_SUITE.erl
index 9b23a30e88..206d2c1bfc 100644
--- a/erts/emulator/test/sensitive_SUITE.erl
+++ b/erts/emulator/test/sensitive_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2007-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2007-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/smoke_test_SUITE.erl b/erts/emulator/test/smoke_test_SUITE.erl
index b3d34103f1..26c610e3a8 100644
--- a/erts/emulator/test/smoke_test_SUITE.erl
+++ b/erts/emulator/test/smoke_test_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2011-2017. All Rights Reserved.
+%% Copyright Ericsson AB 2011-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl
index 3f2897242e..ae3099633a 100644
--- a/erts/emulator/test/statistics_SUITE.erl
+++ b/erts/emulator/test/statistics_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl
index 7309908337..21ab6b378a 100644
--- a/erts/emulator/test/system_info_SUITE.erl
+++ b/erts/emulator/test/system_info_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2005-2017. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/system_profile_SUITE.erl b/erts/emulator/test/system_profile_SUITE.erl
index ae27bfe9df..0c3844e90f 100644
--- a/erts/emulator/test/system_profile_SUITE.erl
+++ b/erts/emulator/test/system_profile_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2007-2017. All Rights Reserved.
+%% Copyright Ericsson AB 2007-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/tracer_SUITE.erl b/erts/emulator/test/tracer_SUITE.erl
index 070462b0f1..5556953feb 100644
--- a/erts/emulator/test/tracer_SUITE.erl
+++ b/erts/emulator/test/tracer_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/tuple_SUITE.erl b/erts/emulator/test/tuple_SUITE.erl
index baf41180e0..e03677a518 100644
--- a/erts/emulator/test/tuple_SUITE.erl
+++ b/erts/emulator/test/tuple_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl
index 103f9f1550..1c52e1a934 100644
--- a/erts/emulator/test/z_SUITE.erl
+++ b/erts/emulator/test/z_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2017. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/utils/make_driver_tab b/erts/emulator/utils/make_driver_tab
index b7bca1dc3a..a000b9d415 100755
--- a/erts/emulator/utils/make_driver_tab
+++ b/erts/emulator/utils/make_driver_tab
@@ -2,7 +2,7 @@
 #
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 1999-2016. All Rights Reserved.
+# Copyright Ericsson AB 1999-2018. All Rights Reserved.
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables
index 094a35ae4b..deee5c2344 100755
--- a/erts/emulator/utils/make_tables
+++ b/erts/emulator/utils/make_tables
@@ -2,7 +2,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1999-2016. All Rights Reserved.
+# Copyright Ericsson AB 1999-2018. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
-- 
cgit v1.2.3


From a01fffa207cecca87861f300e487b7d2cf950f04 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 18 Jun 2018 10:54:00 +0200
Subject: erts: Update etp-commands with correct aux_flags

---
 erts/emulator/beam/erl_process.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index a60e117bab..8d20ccdf90 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -293,7 +293,7 @@ typedef enum {
  * highest index...
  *
  * Remember to update description in erts_pre_init_process()
- * when adding new flags...
+ * and etp-commands when adding new flags...
  */
 
 typedef enum {
-- 
cgit v1.2.3


From 718fcf2637aa1a2231399c03ae6984bb6143ffaf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 21 Jun 2018 10:07:56 +0200
Subject: Fix environment case sensitivity issues on Windows

---
 erts/emulator/sys/common/erl_osenv.c | 10 +++++++++-
 erts/emulator/test/bif_SUITE.erl     | 16 ++++++++++++++--
 2 files changed, 23 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_osenv.c b/erts/emulator/sys/common/erl_osenv.c
index 9f54d1dff0..487ff87116 100644
--- a/erts/emulator/sys/common/erl_osenv.c
+++ b/erts/emulator/sys/common/erl_osenv.c
@@ -75,7 +75,15 @@ static int compare_env_keys(const erts_osenv_data_t a, const erts_osenv_data_t b
 #include "erl_rbtree.h"
 
 static int compare_env_keys(const erts_osenv_data_t a, const erts_osenv_data_t b) {
-    int relation = sys_memcmp(a.data, b.data, MIN(a.length, b.length));
+    int relation;
+
+#ifdef __WIN32__
+    /* Environment variables are case-insensitive on Windows. */
+    relation = _wcsnicmp((const WCHAR*)a.data, (const WCHAR*)b.data,
+                         MIN(a.length, b.length) / sizeof(WCHAR));
+#else
+    relation = sys_memcmp(a.data, b.data, MIN(a.length, b.length));
+#endif
 
     if(relation != 0) {
         return relation;
diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl
index 32bfcd5520..9e7bcd5255 100644
--- a/erts/emulator/test/bif_SUITE.erl
+++ b/erts/emulator/test/bif_SUITE.erl
@@ -36,7 +36,8 @@
 	 error_stacktrace_during_call_trace/1,
          group_leader_prio/1, group_leader_prio_dirty/1,
          is_process_alive/1,
-         process_info_blast/1]).
+         process_info_blast/1,
+         os_env_case_sensitivity/1]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
@@ -51,7 +52,7 @@ all() ->
      erl_crash_dump_bytes, min_max, erlang_halt, is_builtin,
      error_stacktrace, error_stacktrace_during_call_trace,
      group_leader_prio, group_leader_prio_dirty,
-     is_process_alive, process_info_blast].
+     is_process_alive, process_info_blast, os_env_case_sensitivity].
 
 %% Uses erlang:display to test that erts_printf does not do deep recursion
 display(Config) when is_list(Config) ->
@@ -443,6 +444,17 @@ os_env_long(Min, Max, Value) ->
     true = os:unsetenv(EnvVar),
     os_env_long(Min+1, Max, Value).
 
+os_env_case_sensitivity(Config) when is_list(Config) ->
+    %% The keys in os:getenv/putenv must be case-insensitive on Windows, and
+    %% case-sensitive elsewhere.
+    true = os:putenv("os_env_gurka", "gaffel"),
+    Expected = case os:type() of
+                   {win32, _} -> "gaffel";
+                   _ -> false
+               end,
+    Expected = os:getenv("OS_ENV_GURKA"),
+    ok.
+
 %% Test that string:to_integer does not Halloc in wrong order.
 otp_7526(Config) when is_list(Config) ->
     ok = test_7526(256).
-- 
cgit v1.2.3


From 70dec3aa35165ff0d6ec3cb792ac2b662349a7c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 27 Jun 2018 10:01:30 +0200
Subject: Move to a dirty scheduler even when we have pending system tasks

When a system task was enqueued on a process between being
scheduled and the check altered in this commit, we'd run dirty
code on a normal scheduler as the RUNNING_SYS flag wasn't set
and we wouldn't migrate back.

This change migrates us to a dirty scheduler instead, which will
immediately bounce us back to a normal scheduler where
RUNNING_SYS will be set appropriately.

This is caught fairly reliably by
process_SUITE:system_task_failed_enqueue on machines with a lot of
cores.
---
 erts/emulator/beam/erl_process.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 8253ec4f40..173bd98b2f 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9622,7 +9622,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
             ASSERT(!p->scheduler_data);
 	    p->scheduler_data = esdp;
 	    if ((!!(state & ERTS_PSFLGS_DIRTY_WORK))
-		& (!(state & ERTS_PSFLG_ACTIVE_SYS))) {
+		& (!(state & ERTS_PSFLG_RUNNING_SYS))) {
 		/* Migrate to dirty scheduler... */
 	    sunlock_sched_out_proc:
 		erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-- 
cgit v1.2.3


From a9e579341181ed877379de022a261da81fc7cf8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 27 Jun 2018 14:40:18 +0200
Subject: Fix a race condition when generating async operation ids

The counter used for generating async operation ids was a plain int
shared between all ports, which was incorrect but mostly worked
fine since the ids only had to be unique on a per-port basis.
However, some compilers (notably GCC 8.1.1) generated code that
assumed that this value didn't change between reads. Using a
shortened version of enq_async_w_tmo as an example:

    int id = async_ref++;
    op->id = id; //A
    return id; //B

In GCC 7 and earlier, `async_ref` would be read once and assigned
to `id` before being incremented, which kept the values at A and B
consistent. In GCC 8, `async_ref` was read when assigned at A and
read again at B, and then incremented, which made them inconsistent
if we raced with another port.

This commit fixes the issue by removing `async_ref` altogether and
replacing it with a per-port counter which makes it impossible to
race with someone else.
---
 erts/emulator/drivers/common/inet_drv.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index ebd13e6f05..3029958f26 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -1025,6 +1025,7 @@ typedef struct {
     inet_async_op* oph;          /* queue head or NULL */
     inet_async_op* opt;          /* queue tail or NULL */
     inet_async_op  op_queue[INET_MAX_ASYNC];  /* call queue */
+    int op_ref;                 /* queue reference generator  */
 
     int   active;               /* 0 = passive, 1 = active, 2 = active once */
     Sint16 active_count;        /* counter for {active,N} */
@@ -1273,8 +1274,7 @@ static int packet_inet_output(udp_descriptor* udesc, HANDLE event);
 /* convert descriptor pointer to inet_descriptor pointer */
 #define INETP(d) (&(d)->inet)
 
-static int async_ref = 0;          /* async reference id generator */
-#define NEW_ASYNC_ID() ((async_ref++) & 0xffff)
+#define NEW_ASYNC_ID(desc) ((desc)->op_ref++ & 0xffff)
 
 /* check for transition from active to passive */
 #define INET_CHECK_ACTIVE_TO_PASSIVE(inet)                              \
@@ -1925,7 +1925,7 @@ static void enq_multi_op(tcp_descriptor *desc, char *buf, int req,
 			 ErlDrvTermData caller, MultiTimerData *timeout,
 			 ErlDrvMonitor *monitorp)
 {
-    int id = NEW_ASYNC_ID();
+    int id = NEW_ASYNC_ID(INETP(desc));
     enq_old_multi_op(desc,id,req,caller,timeout,monitorp);
     if (buf != NULL)
 	put_int16(id, buf);
@@ -1994,7 +1994,7 @@ static int remove_multi_op(tcp_descriptor *desc, int *id_p, int *req_p,
 static int enq_async_w_tmo(inet_descriptor* desc, char* buf, int req, unsigned timeout,
 			   ErlDrvMonitor *monitorp)
 {
-    int id = NEW_ASYNC_ID();
+    int id = NEW_ASYNC_ID(desc);
     inet_async_op* opp;
 
     if ((opp = desc->oph) == NULL)            /* queue empty */
@@ -8310,6 +8310,7 @@ static ErlDrvData inet_start(ErlDrvPort port, int size, int protocol)
     desc->delimiter    = '\n';         /* line delimiting char */
     desc->oph = NULL;
     desc->opt = NULL;
+    desc->op_ref = 0;
 
     desc->peer_ptr = NULL;
     desc->name_ptr = NULL;
-- 
cgit v1.2.3


From 095c832e1d644c85e452527ffae43dce34b5075e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 3 Jul 2018 13:16:33 +0200
Subject: Use fallback pollset for stdin and friends when using kqueue

This is a hack to make the "noshell" option work; kqueue can poll
these fds but will not report EV_EOF. This may be common to all
all pipes but we have no way to tell whether an fd is a pipe or
not.
---
 erts/emulator/sys/common/erl_poll.c | 17 +++++++++++++++++
 erts/emulator/test/driver_SUITE.erl | 24 ++++++++++++++++++++++++
 2 files changed, 41 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index 70b5532af9..b4d1575ee5 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -803,6 +803,23 @@ update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events)
     struct kevent evts[2];
     struct timespec ts = {0, 0};
 
+    if (op == ERTS_POLL_OP_ADD) {
+        /* This is a hack to make the "noshell" option work; kqueue can poll
+         * these fds but will not report EV_EOF, so we return NVAL to use the
+         * fallback instead.
+         *
+         * This may be common to all pipes but we have no way to tell whether
+         * an fd is a pipe or not. */
+        switch (fd) {
+        case STDIN_FILENO:
+        case STDOUT_FILENO:
+        case STDERR_FILENO:
+            return ERTS_POLL_EV_NVAL;
+        default:
+            break;
+        }
+    }
+
 #if defined(EV_DISPATCH) && !defined(__OpenBSD__)
     /* If we have EV_DISPATCH we use it, unless we are on OpenBSD as the
        behavior of EV_EOF seems to be edge triggered there and we need it
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 7aff3a6ea1..6f5d639d04 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -81,6 +81,7 @@
          thr_msg_blast/1,
          consume_timeslice/1,
          env/1,
+         poll_pipe/1,
          z_test/1]).
 
 -export([bin_prefix/2]).
@@ -168,6 +169,7 @@ all() -> %% Keep a_test first and z_test last...
      thr_msg_blast,
      consume_timeslice,
      env,
+     poll_pipe,
      z_test].
 
 groups() -> 
@@ -2693,3 +2695,25 @@ rpc(Config, Fun) ->
                     ct:fail(Other)
             end
     end.
+
+poll_pipe(Config) when is_list(Config) ->
+    %% ERL-647; we wouldn't see any events on EOF when polling a pipe using
+    %% kqueue(2).
+    case os:type() of
+        {unix, _} ->
+            Command = "erl -noshell -eval "
+                      "'\"DATA\n\" = io:get_line(\"\"),"
+                      "eof = io:get_line(\"\"),"
+                      "halt()' <<< 'DATA'",
+            Ref = make_ref(),
+            Self = self(),
+            Pid = spawn(fun() -> os:cmd(Command), Self ! Ref end),
+            receive
+                Ref -> ok
+            after 5000 ->
+                exit(Pid, kill),
+                ct:fail("Stuck reading from stdin.")
+            end;
+        _ ->
+            {skipped, "Unix-only test"}
+    end.
-- 
cgit v1.2.3


From 4f030fed221bdf86e23de8f5d4262ade5b717400 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 4 Jun 2018 14:23:55 +0200
Subject: beam_debug: Fix printing of floating point registers

---
 erts/emulator/beam/beam_debug.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 6d3b99c43e..10d20c8f3f 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -667,7 +667,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	    ap++;
 	    break;
 	case 'l':		/* fr(N) */
-	    erts_print(to, to_arg, "fr(%d)", loader_reg_index(ap[0]));
+	    erts_print(to, to_arg, "fr(%d)", ap[0] / sizeof(FloatDef));
 	    ap++;
 	    break;
 	default:
-- 
cgit v1.2.3


From 7603a4fb82f72a72045eed26dd55de3067cfe344 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 6 Jul 2018 20:12:28 +0200
Subject: erts: Fix buggy calls to erts_sys_explicit_8bit_getenv

Two of them only affect valgrind builds
and the one for ERL_CRASH_DUMP_NICE seems benign.

Return value changed in c2d70945dce9cb09d5d7120d6e9ddf7faac8d230
old -> new
-1 ->  0  not found
 0 ->  1  found ok
 1 -> -1  found but too big
---
 erts/emulator/beam/dist.c         | 2 +-
 erts/emulator/beam/erl_bif_info.c | 2 +-
 erts/emulator/sys/unix/sys.c      | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 70474898b2..db35137e92 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1207,7 +1207,7 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote)
 #  define PURIFY_MSG(msg)                                                    \
     do {								     \
 	char buf__[1]; size_t bufsz__ = sizeof(buf__);			     \
-	if (erts_sys_explicit_8bit_getenv("VALGRIND_LOG_XML", buf__, &bufsz__) >= 0) { \
+	if (erts_sys_explicit_8bit_getenv("VALGRIND_LOG_XML", buf__, &bufsz__) != 0) { \
 	    VALGRIND_PRINTF_XML("<erlang_error_log>"			     \
 			    "%s, line %d: %s</erlang_error_log>\n",	     \
 			    __FILE__, __LINE__, msg);			     \
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 5789fa8e71..39442e52c6 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2098,7 +2098,7 @@ static int check_if_xml(void)
 {
     char buf[1];
     size_t bufsz = sizeof(buf);
-    return erts_sys_explicit_8bit_getenv("VALGRIND_LOG_XML", buf, &bufsz) >= 0;
+    return erts_sys_explicit_8bit_getenv("VALGRIND_LOG_XML", buf, &bufsz) != 0;
 }
 #else
 #define check_if_xml() 0
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 189ca083d7..36579ffdb4 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -452,9 +452,9 @@ prepare_crash_dump(int secs)
 
     envsz = sizeof(env);
     i = erts_sys_explicit_8bit_getenv("ERL_CRASH_DUMP_NICE", env, &envsz);
-    if (i >= 0) {
+    if (i != 0) {
 	int nice_val;
-	nice_val = i != 1 ? 0 : atoi(env);
+	nice_val = (i != 1) ? 0 : atoi(env);
 	if (nice_val > 39) {
 	    nice_val = 39;
 	}
-- 
cgit v1.2.3


From 346494638829cb4440aea106dbfc3ede45e1d274 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 12 Jul 2018 15:54:57 +0200
Subject: erts: Fix bug in crash dump generation

Symptom: emulator core dumps during crash dump generation.

Problem:
erts_dump_lit_areas did not grow correctly
to always be equal or larger than number of loaded modules.

The comment about twice the size to include both curr and old
did not seem right. The beam_ranges structure contains *all* loaded
module instances until they are removed when purged.
---
 erts/emulator/beam/beam_ranges.c | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c
index fac4289271..8e96b762bf 100644
--- a/erts/emulator/beam/beam_ranges.c
+++ b/erts/emulator/beam/beam_ranges.c
@@ -34,10 +34,8 @@ typedef struct {
 
 /*
  * Used for crash dumping of literals. The size of erts_dump_lit_areas is
- * always twice the number of active ranges (to allow for literals in both
- * current and old code).
+ * always at least the number of active ranges.
  */
-
 ErtsLiteralArea** erts_dump_lit_areas;
 Uint erts_dump_num_lit_areas;
 
@@ -179,8 +177,8 @@ erts_end_staging_ranges(int commit)
 				(erts_aint_t) (r[dst].modules +
 					       r[dst].n / 2));
 
-        if (r[dst].allocated * 2 > erts_dump_num_lit_areas) {
-            erts_dump_num_lit_areas *= 2;
+        if (r[dst].allocated > erts_dump_num_lit_areas) {
+            erts_dump_num_lit_areas = r[dst].allocated * 2;
             erts_dump_lit_areas = (ErtsLiteralArea **)
                 erts_realloc(ERTS_ALC_T_CRASH_DUMP,
                              (void *) erts_dump_lit_areas,
-- 
cgit v1.2.3


From c48d442024df188ac2e7b226ec2f04da9d10288b Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 12 Jul 2018 17:44:45 +0200
Subject: Fix trace_info/2

---
 erts/emulator/beam/erl_bif_trace.c | 2 +-
 erts/emulator/test/trace_SUITE.erl | 9 +++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 9861483bf0..711e62c795 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -810,7 +810,7 @@ Eterm trace_info_2(BIF_ALIST_2)
     }
     erts_release_code_write_permission();
 
-    if (is_internal_ref(res))
+    if (is_value(res) && is_internal_ref(res))
         BIF_TRAP1(erts_await_result, BIF_P, res);
 
     BIF_RET(res);
diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl
index 138aefb29c..979b3185a5 100644
--- a/erts/emulator/test/trace_SUITE.erl
+++ b/erts/emulator/test/trace_SUITE.erl
@@ -38,7 +38,8 @@
 	 system_monitor_long_gc_1/1, system_monitor_long_gc_2/1, 
 	 system_monitor_large_heap_1/1, system_monitor_large_heap_2/1,
 	 system_monitor_long_schedule/1,
-	 bad_flag/1, trace_delivered/1, trap_exit_self_receive/1]).
+	 bad_flag/1, trace_delivered/1, trap_exit_self_receive/1,
+         trace_info_badarg/1]).
 
 -include_lib("common_test/include/ct.hrl").
 
@@ -62,7 +63,7 @@ all() ->
      system_monitor_long_gc_2, system_monitor_large_heap_1,
      system_monitor_long_schedule,
      system_monitor_large_heap_2, bad_flag, trace_delivered,
-     trap_exit_self_receive].
+     trap_exit_self_receive, trace_info_badarg].
 
 init_per_testcase(_Case, Config) ->
     [{receiver,spawn(fun receiver/0)}|Config].
@@ -1734,6 +1735,10 @@ trap_exit_self_receive(Config) ->
     receive_nothing(),
     ok.
 
+trace_info_badarg(Config) when is_list(Config) ->
+    catch erlang:trace_info({a,b,c},d),
+    ok.
+
 drop_trace_until_down(Proc, Mon) ->
     drop_trace_until_down(Proc, Mon, false, 0, 0).
 
-- 
cgit v1.2.3


From 32d674c58ba0cfddb2d5e2e83a8bf3b9a70cce4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 12 Jul 2018 21:55:00 +0200
Subject: Fix a rare crash when matching on literal maps

When matching on a literal map, the map is placed into the general
scratch register first. This is fine in isolation, but when the
key to be matched was in a Y register it would also be placed in
the scratch register, overwriting the map and crashing the
emulator.
---
 erts/emulator/beam/ops.tab       |  5 +----
 erts/emulator/test/map_SUITE.erl | 11 +++++++++++
 2 files changed, 12 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 88ede3bb60..c51e4ef784 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1421,16 +1421,13 @@ get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \
 
 i_get_map_elements f? s I
 
-i_get_map_element Fail Src=xy Key=y Dst => \
-    move Key x | i_get_map_element Fail Src x Dst
-
 i_get_map_element_hash Fail Src=c Key Hash Dst => \
     move Src x | i_get_map_element_hash Fail x Key Hash Dst
 i_get_map_element_hash f? xy c I xy
 
 i_get_map_element Fail Src=c Key Dst => \
     move Src x | i_get_map_element Fail x Key Dst
-i_get_map_element f? xy x xy
+i_get_map_element f? xy xy xy
 
 #
 # Convert the plus operations to a generic plus instruction.
diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl
index f93c637650..d0a6763fe5 100644
--- a/erts/emulator/test/map_SUITE.erl
+++ b/erts/emulator/test/map_SUITE.erl
@@ -3080,8 +3080,19 @@ y_regs(Config) when is_list(Config) ->
 
     true = is_map(Map2) andalso is_map(Map4),
 
+    gurka = y_regs_literal(0),
+    gaffel = y_regs_literal(1),
+
     ok.
 
+y_regs_literal(Key) when is_integer(Key) ->
+    %% Forces the key to be placed in a Y register.
+    lists:seq(1, 2),
+    case is_map_key(Key, #{ 0 => 0 }) of
+        true -> gurka;
+        false -> gaffel
+    end.
+
 y_regs_update(Map0, Val0) ->
     Val1 = {t,Val0},
     K1 = id({key,1}),
-- 
cgit v1.2.3


From c027e5300a2eea89232a77ec28d8c7e59dd8b9a7 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 13 Jul 2018 18:01:43 +0200
Subject: erts: Fix bug where break would not trigger on windows

This bug was introduced in 988f5f5e8061ce2. The aux_thread
needs to be poked as it is the thread that handles all
break actions.
---
 erts/emulator/sys/win32/erl_poll.c | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index fd4c745c3b..fcf5a0d533 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -362,11 +362,11 @@ is_io_ready(ErtsPollSet *ps)
 }
 
 static ERTS_INLINE void
-woke_up(ErtsPollSet *ps)
+woke_up(ErtsPollSet *ps, int waketype)
 {
     if (erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_NOT_WOKEN)
 	erts_atomic32_cmpxchg_nob(&ps->wakeup_state,
-				  ERTS_POLL_WOKEN_TIMEDOUT,
+				  waketype,
 				  ERTS_POLL_NOT_WOKEN);
 #ifdef DEBUG
     {
@@ -960,12 +960,12 @@ static int cancel_driver_select(ErtsPollSet *ps, HANDLE event)
 
 void  erts_poll_interrupt(ErtsPollSet *ps, int set /* bool */)
 {
-    HARDTRACEF(("In erts_poll_interrupt(%d)",set));
+    HARDTRACEF(("In erts_poll_interrupt(%p, %d)",ps,set));
     if (!set)
 	reset_interrupt(ps);
     else
 	set_interrupt(ps);
-    HARDTRACEF(("Out erts_poll_interrupt(%d)",set));
+    HARDTRACEF(("Out erts_poll_interrupt(%p, %d)",ps,set));
 }
 
 
@@ -1051,19 +1051,20 @@ int erts_poll_wait(ErtsPollSet *ps,
 
     if (!erts_atomic32_read_nob(&break_waiter_state)) {
 	HANDLE harr[2] = {ps->event_io_ready, break_happened_event};
-	int num_h = 2;
+	int num_h = 2, handle;
         ERTS_MSACC_PUSH_STATE();
 
 	HARDDEBUGF(("Start waiting %d [%d]",num_h, (int) timeout));
 	ERTS_POLLSET_UNLOCK(ps);
 	erts_thr_progress_prepare_wait(NULL);
         ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP);
-	WaitForMultipleObjects(num_h, harr, FALSE, timeout);
+	handle = WaitForMultipleObjects(num_h, harr, FALSE, timeout);
 	erts_thr_progress_finalize_wait(NULL);
         ERTS_MSACC_POP_STATE();
 	ERTS_POLLSET_LOCK(ps);
 	HARDDEBUGF(("Stop waiting %d [%d]",num_h, (int) timeout));
-	woke_up(ps);
+        if (handle == WAIT_OBJECT_0)
+            woke_up(ps, ERTS_POLL_WOKEN_TIMEDOUT);
     }
 
     ERTS_UNSET_BREAK_REQUESTED;
@@ -1075,7 +1076,10 @@ int erts_poll_wait(ErtsPollSet *ps,
 	erts_mtx_unlock(&break_waiter_lock);
 	switch (break_state) {
 	case  BREAK_WAITER_GOT_BREAK:
+            woke_up(ps, ERTS_POLL_WOKEN_INTR);
 	    ERTS_SET_BREAK_REQUESTED;
+            /* Wake aux thread to get handle break */
+            erts_aux_thread_poke();
 	    break;
 	case  BREAK_WAITER_GOT_HALT:
 	    erts_exit(0,"");
-- 
cgit v1.2.3


From 83445d0de88a1ad15066e6c415b2c2d9aa93d6a1 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 20 Jun 2018 17:54:24 +0200
Subject: erts: Free udp buffer when getting EAGAIN

Only SCPT should keep the recv buffer when going into
select. If UDP does it, it will result in many more
memory allocations than there should be which can be
very detrimental to performance.
---
 erts/emulator/drivers/common/inet_drv.c | 30 ++++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 5e9afdc5ca..833befefd1 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -1785,6 +1785,7 @@ static void release_buffer(ErlDrvBinary* buf)
 #ifdef HAVE_UDP
 static ErlDrvBinary* realloc_buffer(ErlDrvBinary* buf, ErlDrvSizeT newsz)
 {
+    DEBUGF(("realloc_buffer: %ld -> %ld\r\n", (buf==NULL) ? 0 : buf->orig_size, newsz));
     return driver_realloc_binary(buf, newsz);
 }
 #endif
@@ -12042,15 +12043,11 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
 	sys_memzero((char *) &other, sizeof(other));
 
 	/* udesc->i_buf is only kept between SCTP fragments */
-	if (udesc->i_buf == NULL) {
-	    udesc->i_bufsz = desc->bufsz + len;
-	    if ((udesc->i_buf = alloc_buffer(udesc->i_bufsz)) == NULL)
-		return packet_error(udesc, ENOMEM);
-	    /* pointer to message start */
-	    udesc->i_ptr = udesc->i_buf->orig_bytes + len;
-	} else {
-	    ErlDrvBinary* tmp;
+#ifdef HAVE_SCTP
+	if (udesc->i_buf != NULL) {
+            ErlDrvBinary* tmp;
 	    int bufsz;
+            ASSERT(IS_SCTP(desc));
 	    bufsz = desc->bufsz + (udesc->i_ptr - udesc->i_buf->orig_bytes);
 	    if ((tmp = realloc_buffer(udesc->i_buf, bufsz)) == NULL) {
 		release_buffer(udesc->i_buf);
@@ -12062,6 +12059,15 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
 		udesc->i_buf = tmp;
 		udesc->i_bufsz = bufsz;
 	    }
+	} else
+#endif
+        {
+            ASSERT(udesc->i_buf == NULL);
+	    udesc->i_bufsz = desc->bufsz + len;
+	    if ((udesc->i_buf = alloc_buffer(udesc->i_bufsz)) == NULL)
+		return packet_error(udesc, ENOMEM);
+	    /* pointer to message start */
+	    udesc->i_ptr = udesc->i_buf->orig_bytes + len;
 	}
 
 	/* Note: On Windows NT, recvfrom() fails if the socket is connected. */
@@ -12120,6 +12126,14 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
 		) {
 		sock_select(desc,FD_READ,1);
 	    }
+#ifdef HAVE_SCTP
+            if (!short_recv) {
+#endif
+                release_buffer(udesc->i_buf);
+                udesc->i_buf = NULL;
+#ifdef HAVE_SCTP
+            }
+#endif
 	    return count;		/* strange, not ready */
 	}
 
-- 
cgit v1.2.3


From bd228591a434bd0a15f5220e66a8bb623b410e2f Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 10 Jul 2018 13:40:42 +0200
Subject: erts: Remove use of VALGRIND_PRINTF_XML

which only existed in a patched version of valgrind (by pan)
no longer used.

Instead we use standard VALGRIND_PRINTF which will end up like this
if valgrind log format is XML and valgrind version >= 3.9:

<clientmsg>
  <tid>7</tid>
  <threadname>3_scheduler</threadname>
  <text>Test case #20 ei_encode_SUITE:test_ei_encode_long/1
  </text>
</clientmsg>

Note the extra trailing whitespace that may occure before </text>.
---
 erts/emulator/beam/dist.c         | 15 +--------------
 erts/emulator/beam/erl_bif_info.c | 28 +++-------------------------
 2 files changed, 4 insertions(+), 39 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index db35137e92..61090e2c47 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1200,21 +1200,8 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote)
 #include <valgrind/valgrind.h>
 #include <valgrind/memcheck.h>
 
-#ifndef HAVE_VALGRIND_PRINTF_XML
-#define VALGRIND_PRINTF_XML VALGRIND_PRINTF
-#endif
-
 #  define PURIFY_MSG(msg)                                                    \
-    do {								     \
-	char buf__[1]; size_t bufsz__ = sizeof(buf__);			     \
-	if (erts_sys_explicit_8bit_getenv("VALGRIND_LOG_XML", buf__, &bufsz__) != 0) { \
-	    VALGRIND_PRINTF_XML("<erlang_error_log>"			     \
-			    "%s, line %d: %s</erlang_error_log>\n",	     \
-			    __FILE__, __LINE__, msg);			     \
-	} else {							     \
-	    VALGRIND_PRINTF("%s, line %d: %s", __FILE__, __LINE__, msg);     \
-	}								     \
-    } while (0)
+    VALGRIND_PRINTF("%s, line %d: %s", __FILE__, __LINE__, msg)
 #else
 #  define PURIFY_MSG(msg)
 #endif
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 39442e52c6..7fada0d548 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2093,17 +2093,6 @@ current_stacktrace(ErtsHeapFactory *hfact, Process* rp,
     return res;
 }
 
-#if defined(VALGRIND)
-static int check_if_xml(void)
-{
-    char buf[1];
-    size_t bufsz = sizeof(buf);
-    return erts_sys_explicit_8bit_getenv("VALGRIND_LOG_XML", buf, &bufsz) != 0;
-}
-#else
-#define check_if_xml() 0
-#endif
-
 /*
  * This function takes care of calls to erlang:system_info/1 when the argument
  * is a tuple.
@@ -2200,15 +2189,9 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 #endif
 	} else if (is_list(*tp)) {
 #if defined(PURIFY)
-#define ERTS_ERROR_CHECKER_PRINTF purify_printf
-#define ERTS_ERROR_CHECKER_PRINTF_XML purify_printf
+#  define ERTS_ERROR_CHECKER_PRINTF purify_printf
 #elif defined(VALGRIND)
-#define ERTS_ERROR_CHECKER_PRINTF VALGRIND_PRINTF
-#  ifndef HAVE_VALGRIND_PRINTF_XML
-#    define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF
-#  else
-#    define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF_XML
-#  endif
+#  define ERTS_ERROR_CHECKER_PRINTF VALGRIND_PRINTF
 #endif
 	    ErlDrvSizeT buf_size = 8*1024; /* Try with 8KB first */
 	    char *buf = erts_alloc(ERTS_ALC_T_TMP, buf_size);
@@ -2224,12 +2207,7 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 		ASSERT(r == buf_size - 1);
 	    }
 	    buf[buf_size - 1 - r] = '\0';
-	    if (check_if_xml()) {
-		ERTS_ERROR_CHECKER_PRINTF_XML("<erlang_info_log>"
-					      "%s</erlang_info_log>\n", buf);
-	    } else {
-		ERTS_ERROR_CHECKER_PRINTF("%s\n", buf);
-	    }
+            ERTS_ERROR_CHECKER_PRINTF("%s\n", buf);
 	    erts_free(ERTS_ALC_T_TMP, (void *) buf);
 	    BIF_RET(am_true);
 #undef ERTS_ERROR_CHECKER_PRINTF
-- 
cgit v1.2.3


From 865ecb8d5da48695d4dc39b73fd641bdeaca52df Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikpelinux@gmail.com>
Date: Mon, 30 Jul 2018 09:53:06 +0200
Subject: erl_alloc: align ErtsAllocatorState_t

---
 erts/emulator/beam/erl_alloc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 575d6ca867..8fe1ccb758 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -114,7 +114,7 @@ typedef union {
     char align_afa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AFAllctr_t))];
     AOFFAllctr_t aoffa;
     char align_aoffa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AOFFAllctr_t))];
-} ErtsAllocatorState_t;
+} ErtsAllocatorState_t erts_align_attribute(ERTS_CACHE_LINE_SIZE);
 
 static ErtsAllocatorState_t std_alloc_state;
 static ErtsAllocatorState_t ll_alloc_state;
-- 
cgit v1.2.3


From 48bbcab1b0d51ed19ab5676aa712e976bc76ee97 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 12 Jun 2018 15:25:38 +0200
Subject: erts: Handle EMFILE errors in forker_driver for write

Before this change, if a write to the uds failed due to
EMFILE to ETOOMANYREFS the entire vm would crash. This
change makes it so that an SIGCHLD is simulated to that
the error is propagated to the user instead of terminating
the VM.
---
 erts/emulator/sys/unix/sys_drivers.c | 107 ++++++++++++++++++++---------------
 erts/emulator/sys/unix/sys_uds.c     |  13 ++++-
 2 files changed, 74 insertions(+), 46 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c
index 872c3a80b1..d0498f0bd5 100644
--- a/erts/emulator/sys/unix/sys_drivers.c
+++ b/erts/emulator/sys/unix/sys_drivers.c
@@ -685,7 +685,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name,
 
         /* we send the request to do the fork */
         if ((res = writev(ofd[1], io_vector, iov_len > MAXIOV ? MAXIOV : iov_len)) < 0) {
-            if (errno == ERRNO_BLOCK) {
+            if (errno == ERRNO_BLOCK || errno == EINTR) {
                 res = 0;
             } else {
                 int err = errno;
@@ -1257,6 +1257,8 @@ static int port_inp_failure(ErtsSysDriverData *dd, int res)
         }
        driver_failure_eof(dd->port_num);
     } else if (dd->ifd) {
+        if (dd->alive == -1)
+            errno = dd->status;
         erl_drv_init_ack(dd->port_num, ERL_DRV_ERROR_ERRNO);
     } else {
 	driver_failure_posix(dd->port_num, err);
@@ -1287,10 +1289,10 @@ static void ready_input(ErlDrvData e, ErlDrvEvent ready_fd)
         int res;
 
         if((res = read(ready_fd, &proto, sizeof(proto))) <= 0) {
+            if (res < 0 && (errno == ERRNO_BLOCK || errno == EINTR))
+                return;
             /* hmm, child setup seems to have closed the pipe too early...
                we close the port as there is not much else we can do */
-            if (res < 0 && errno == ERRNO_BLOCK)
-                return;
             driver_select(port_num, ready_fd, ERL_DRV_READ, 0);
             if (res == 0)
                 errno = EPIPE;
@@ -1424,7 +1426,7 @@ static void ready_input(ErlDrvData e, ErlDrvEvent ready_fd)
 		    continue;
 		}
 		else {		/* The last message we got was split */
-		        char *buf = erts_alloc_fnf(ERTS_ALC_T_FD_ENTRY_BUF, h);
+                    char *buf = erts_alloc_fnf(ERTS_ALC_T_FD_ENTRY_BUF, h);
 		    if (!buf) {
 			errno = ENOMEM;
 			port_inp_failure(dd, -1);
@@ -1670,15 +1672,37 @@ static void forker_stop(ErlDrvData e)
        the port has been closed by the user. */
 }
 
+static ErlDrvSizeT forker_deq(ErlDrvPort port_num, ErtsSysForkerProto *proto)
+{
+    close(proto->u.start.fds[0]);
+    close(proto->u.start.fds[1]);
+    if (proto->u.start.fds[1] != proto->u.start.fds[2])
+        close(proto->u.start.fds[2]);
+
+    return driver_deq(port_num, sizeof(*proto));
+}
+
+static void forker_sigchld(Eterm port_id, int error)
+{
+    ErtsSysForkerProto *proto = erts_alloc(ERTS_ALC_T_DRV_CTRL_DATA, sizeof(*proto));
+    proto->action = ErtsSysForkerProtoAction_SigChld;
+    proto->u.sigchld.error_number = error;
+    proto->u.sigchld.port_id = port_id;
+
+    /* ideally this would be a port_command call, but as command is
+       already used by the spawn_driver, we use control instead.
+       Note that when using erl_drv_port_control it is an asynchronous
+       control. */
+    erl_drv_port_control(port_id, 'S', (char*)proto, sizeof(*proto));
+}
+
 static void forker_ready_input(ErlDrvData e, ErlDrvEvent fd)
 {
     int res;
-    ErtsSysForkerProto *proto;
+    ErtsSysForkerProto proto;
 
-    proto = erts_alloc(ERTS_ALC_T_DRV_CTRL_DATA, sizeof(*proto));
-
-    if ((res = read(fd, proto, sizeof(*proto))) < 0) {
-        if (errno == ERRNO_BLOCK)
+    if ((res = read(fd, &proto, sizeof(proto))) < 0) {
+        if (errno == ERRNO_BLOCK || errno == EINTR)
             return;
         erts_exit(ERTS_DUMP_EXIT, "Failed to read from erl_child_setup: %d\n", errno);
     }
@@ -1686,10 +1710,10 @@ static void forker_ready_input(ErlDrvData e, ErlDrvEvent fd)
     if (res == 0)
         erts_exit(ERTS_DUMP_EXIT, "erl_child_setup closed\n");
 
-    ASSERT(res == sizeof(*proto));
+    ASSERT(res == sizeof(proto));
 
 #ifdef FORKER_PROTO_START_ACK
-    if (proto->action == ErtsSysForkerProtoAction_StartAck) {
+    if (proto.action == ErtsSysForkerProtoAction_StartAck) {
         /* Ideally we would like to not have to ack each Start
            command being sent over the uds, but it would seem
            that some operating systems (only observed on FreeBSD)
@@ -1699,28 +1723,15 @@ static void forker_ready_input(ErlDrvData e, ErlDrvEvent fd)
         ErlDrvPort port_num = (ErlDrvPort)e;
         int vlen;
         SysIOVec *iov = driver_peekq(port_num, &vlen);
-        ErtsSysForkerProto *proto = (ErtsSysForkerProto *)iov[0].iov_base;
-
-        close(proto->u.start.fds[0]);
-        close(proto->u.start.fds[1]);
-        if (proto->u.start.fds[1] != proto->u.start.fds[2])
-            close(proto->u.start.fds[2]);
+        ErtsSysForkerProto *qproto = (ErtsSysForkerProto *)iov[0].iov_base;
 
-        driver_deq(port_num, sizeof(*proto));
-
-        if (driver_sizeq(port_num) > 0)
+        if (forker_deq(port_num, qproto))
             driver_select(port_num, forker_fd, ERL_DRV_WRITE|ERL_DRV_USE, 1);
     } else
 #endif
     {
-        ASSERT(proto->action == ErtsSysForkerProtoAction_SigChld);
-
-        /* ideally this would be a port_command call, but as command is
-           already used by the spawn_driver, we use control instead.
-           Note that when using erl_drv_port_control it is an asynchronous
-           control. */
-        erl_drv_port_control(proto->u.sigchld.port_id, 'S',
-                             (char*)proto, sizeof(*proto));
+        ASSERT(proto.action == ErtsSysForkerProtoAction_SigChld);
+        forker_sigchld(proto.u.sigchld.port_id, proto.u.sigchld.error_number);
     }
 
 }
@@ -1730,7 +1741,8 @@ static void forker_ready_output(ErlDrvData e, ErlDrvEvent fd)
     ErlDrvPort port_num = (ErlDrvPort)e;
 
 #ifndef FORKER_PROTO_START_ACK
-    while (driver_sizeq(port_num) > 0) {
+    int loops = 10;
+    while (driver_sizeq(port_num) > 0 && --loops) {
 #endif
         int vlen;
         SysIOVec *iov = driver_peekq(port_num, &vlen);
@@ -1738,20 +1750,24 @@ static void forker_ready_output(ErlDrvData e, ErlDrvEvent fd)
         ASSERT(iov[0].iov_len >= (sizeof(*proto)));
         if (sys_uds_write(forker_fd, (char*)proto, sizeof(*proto),
                           proto->u.start.fds, 3, 0) < 0) {
-            if (errno == ERRNO_BLOCK)
+            if (errno == ERRNO_BLOCK || errno == EINTR) {
                 return;
-            erts_exit(ERTS_DUMP_EXIT, "Failed to write to erl_child_setup: %d\n", errno);
+            } else if (errno == EMFILE) {
+                forker_sigchld(proto->u.start.port_id, errno);
+                if (forker_deq(port_num, proto) == 0)
+                    driver_select(port_num, forker_fd, ERL_DRV_WRITE, 0);
+                return;
+            } else {
+                erts_exit(ERTS_DUMP_EXIT, "Failed to write to erl_child_setup: %d\n", errno);
+            }
         }
 #ifndef FORKER_PROTO_START_ACK
-        close(proto->u.start.fds[0]);
-        close(proto->u.start.fds[1]);
-        if (proto->u.start.fds[1] != proto->u.start.fds[2])
-            close(proto->u.start.fds[2]);
-        driver_deq(port_num, sizeof(*proto));
+        if (forker_deq(port_num, proto) == 0)
+            driver_select(port_num, forker_fd, ERL_DRV_WRITE, 0);
     }
-#endif
-
+#else
     driver_select(port_num, forker_fd, ERL_DRV_WRITE, 0);
+#endif
 }
 
 static ErlDrvSSizeT forker_control(ErlDrvData e, unsigned int cmd, char *buf,
@@ -1777,20 +1793,21 @@ static ErlDrvSSizeT forker_control(ErlDrvData e, unsigned int cmd, char *buf,
 
     if ((res = sys_uds_write(forker_fd, (char*)proto, sizeof(*proto),
                              proto->u.start.fds, 3, 0)) < 0) {
-        if (errno == ERRNO_BLOCK) {
+        if (errno == ERRNO_BLOCK || errno == EINTR) {
             driver_select(port_num, forker_fd, ERL_DRV_WRITE|ERL_DRV_USE, 1);
             return 0;
+        } else if (errno == EMFILE) {
+            forker_sigchld(proto->u.start.port_id, errno);
+            forker_deq(port_num, proto);
+            return 0;
+        } else {
+            erts_exit(ERTS_DUMP_EXIT, "Failed to write to erl_child_setup: %d\n", errno);
         }
-        erts_exit(ERTS_DUMP_EXIT, "Failed to write to erl_child_setup: %d\n", errno);
     }
 
 #ifndef FORKER_PROTO_START_ACK
     ASSERT(res == sizeof(*proto));
-    close(proto->u.start.fds[0]);
-    close(proto->u.start.fds[1]);
-    if (proto->u.start.fds[1] != proto->u.start.fds[2])
-        close(proto->u.start.fds[2]);
-    driver_deq(port_num, sizeof(*proto));
+    forker_deq(port_num, proto);
 #endif
 
     return 0;
diff --git a/erts/emulator/sys/unix/sys_uds.c b/erts/emulator/sys/unix/sys_uds.c
index c328fd00bb..39a4866065 100644
--- a/erts/emulator/sys/unix/sys_uds.c
+++ b/erts/emulator/sys/unix/sys_uds.c
@@ -132,7 +132,7 @@ sys_uds_writev(int fd, struct iovec *iov, size_t iov_len,
 
     struct msghdr msg;
     struct cmsghdr *cmsg = NULL;
-    int res, i;
+    int res, i, error;
 
     /* initialize socket message */
     memset(&msg, 0, sizeof(struct msghdr));
@@ -173,11 +173,22 @@ sys_uds_writev(int fd, struct iovec *iov, size_t iov_len,
 
     res = sendmsg(fd, &msg, flags);
 
+#ifdef ETOOMANYREFS
+    /* Linux may give ETOOMANYREFS when there are too many fds in transit.
+       We map this to EMFILE as bsd and other use this error code and we want
+       the behaviour to be the same on all OSs */
+    if (errno == ETOOMANYREFS)
+        errno = EMFILE;
+#endif
+    error = errno;
+
     if (iov_len > MAXIOV)
         free(iov[0].iov_base);
 
     free(msg.msg_control);
 
+    errno = error;
+
     return res;
 }
 
-- 
cgit v1.2.3


From a9d361b4be7d6329bedd7b37d454c1abf4f27233 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 21 Jun 2018 11:00:25 +0200
Subject: erts: Limit the automatic max buffer for UDP to 2^16

There is no reason to have a larger buffer than this as
the recvmsg call will never return more data.

OTP-15206
---
 erts/emulator/drivers/common/inet_drv.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 833befefd1..2048d0f625 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -1549,6 +1549,8 @@ static void *realloc_wrapper(void *current, ErlDrvSizeT size){
 #   define LOAD_ASSOC_ID        LOAD_UINT
 #   define LOAD_ASSOC_ID_CNT    LOAD_UINT_CNT
 #   define SCTP_ANC_BUFF_SIZE   INET_DEF_BUFFER/2 /* XXX: not very good... */
+#else
+#   define IS_SCTP(desc) 0
 #endif
 
 #ifdef HAVE_UDP
@@ -6436,7 +6438,12 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
 		(long)desc->port, desc->s, res));
 	if (type == SO_RCVBUF) {
 	    /* make sure we have desc->bufsz >= SO_RCVBUF */
-	    if (ival > desc->bufsz)
+            if (ival > (1 << 16) && desc->stype == SOCK_DGRAM && !IS_SCTP(desc))
+                /* For UDP we don't want to automatically
+                   set the buffer size to be larger than
+                   the theoretical max MTU */
+                desc->bufsz = 1 << 16;
+	    else if (ival > desc->bufsz)
 		desc->bufsz = ival;
 	}
     }
-- 
cgit v1.2.3


From d383549b6e9d9b5e06f23226a61564264204db51 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 20 Jun 2018 10:42:20 +0200
Subject: erts: Fix seq_trace to not clear token for system messages

A lot of erts internal messages used behind APIs to create
non-blocking calls, e.g. port_command, would cause the seq_trace
token to be cleared from the caller when it should not.
This commit fixes that and adds asserts that makes sure
that all messages sent have to correct token set.

Fixes: ERL-602
---
 erts/emulator/beam/beam_bif_load.c      |  5 ++-
 erts/emulator/beam/beam_debug.c         |  1 +
 erts/emulator/beam/bif.c                |  2 +-
 erts/emulator/beam/dist.c               |  1 +
 erts/emulator/beam/erl_db.c             |  1 +
 erts/emulator/beam/erl_message.c        | 18 ++++++++--
 erts/emulator/beam/erl_message.h        |  6 ++--
 erts/emulator/beam/erl_nif.c            | 52 +++++++++++++++++++++++++++++
 erts/emulator/beam/erl_proc_sig_queue.c | 38 +++++----------------
 erts/emulator/beam/erl_process.c        |  1 +
 erts/emulator/beam/io.c                 |  5 ---
 erts/emulator/sys/common/erl_check_io.c |  1 -
 erts/emulator/test/nif_SUITE.erl        | 58 +++++++++++++++++++++++++++++++--
 13 files changed, 143 insertions(+), 46 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index a0dbd9ec7b..d221e6aea6 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1753,6 +1753,7 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 
 	if (literals) {
 	    ErtsLiteralAreaRef *ref;
+            ErtsMessage *mp;
 	    ref = erts_alloc(ERTS_ALC_T_LITERAL_REF,
 			     sizeof(ErtsLiteralAreaRef));
 	    ref->literal_area = literals;
@@ -1767,10 +1768,12 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 		release_literal_areas.last = ref;
 	    }
 	    erts_mtx_unlock(&release_literal_areas.mtx);
+            mp = erts_alloc_message(0, NULL);
+            ERL_MESSAGE_TOKEN(mp) = am_undefined;
 	    erts_queue_proc_message(BIF_P,
                                erts_literal_area_collector,
 			       0,
-			       erts_alloc_message(0, NULL),
+			       mp,
 			       am_copy_literals);
 	}
 
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 10d20c8f3f..9633de2021 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -1191,6 +1191,7 @@ dirty_send_message(Process *c_p, Eterm to, Eterm tag)
     mp = erts_alloc_message_heap(rp, &rp_locks, 3, &hp, &ohp);
 
     msg = TUPLE2(hp, tag, c_p->common.id);
+    ERL_MESSAGE_TOKEN(mp) = am_undefined;
     erts_queue_proc_message(c_p, rp, rp_locks, mp, msg);
 
     if (rp == real_c_p)
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 97e1ee1286..f18af8bcd7 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2063,7 +2063,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx)
 	if (p == rp)
 	    rp_locks |= ERTS_PROC_LOCK_MAIN;
 	/* send to local process */
-	erts_send_message(p, rp, &rp_locks, msg, 0);
+	erts_send_message(p, rp, &rp_locks, msg);
 	erts_proc_unlock(rp,
 			     p == rp
 			     ? (rp_locks & ~ERTS_PROC_LOCK_MAIN)
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 61090e2c47..146c00b07d 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -3672,6 +3672,7 @@ int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks)
         dhandle = erts_build_dhandle(&hp, ohp, dep);
         msg = TUPLE4(hp, am_auto_connect, dep->sysname, make_small(conn_id),
                      dhandle);
+        ERL_MESSAGE_TOKEN(mp) = am_undefined;
         erts_queue_proc_message(proc, net_kernel, nk_locks, mp, msg);
         erts_proc_unlock(net_kernel, nk_locks);
     }
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 36d83d93f4..c009a3bde8 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -3647,6 +3647,7 @@ send_ets_transfer_message(Process *c_p, Process *proc,
         hd_copy = copy_struct(heir_data, hd_sz, &hp, ohp);
     sender = c_p->common.id;
     msg = TUPLE4(hp, am_ETS_TRANSFER, tid, sender, hd_copy);
+    ERL_MESSAGE_TOKEN(mp) = am_undefined;
     erts_queue_proc_message(c_p, proc, *locks, mp, msg);
 }
 
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 507cc989d2..a3274d7443 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -409,6 +409,11 @@ ErtsMessage* prepend_pending_sig_maybe(Process* sender, Process* receiver,
  *
  * @brief Send one message from *NOT* a local process.
  *
+ * seq_trace does not work with this type of messages
+ * to it is set to am_undefined which means that the
+ * receiving process will not remove the seq_trace token
+ * when it gets this message.
+ *
  */
 void
 erts_queue_message(Process* receiver, ErtsProcLocks receiver_locks,
@@ -417,11 +422,19 @@ erts_queue_message(Process* receiver, ErtsProcLocks receiver_locks,
     ASSERT(is_not_internal_pid(from));
     ERL_MESSAGE_TERM(mp) = msg;
     ERL_MESSAGE_FROM(mp) = from;
+    ERL_MESSAGE_TOKEN(mp) = am_undefined;
     queue_messages(receiver, receiver_locks, mp, &mp->next, 1);
 }
 
 /**
  * @brief Send one message from a local process.
+ *
+ * It is up to the caller of this function to set the
+ * correct seq_trace. The general rule of thumb is that
+ * it should be set to am_undefined if the message
+ * cannot be traced using seq_trace, if it can be
+ * traced it should be set to the trace token. It should
+ * very rarely be explicitly set to NIL!
  */
 void
 erts_queue_proc_message(Process* sender,
@@ -584,8 +597,7 @@ void
 erts_send_message(Process* sender,
 		  Process* receiver,
 		  ErtsProcLocks *receiver_locks,
-		  Eterm message,
-		  unsigned flags)
+		  Eterm message)
 {
     Uint msize;
     ErtsMessage* mp;
@@ -619,7 +631,7 @@ erts_send_message(Process* sender,
 
     receiver_state = erts_atomic32_read_nob(&receiver->state);
 
-    if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) {
+    if (SEQ_TRACE_TOKEN(sender) != NIL) {
         Eterm* hp;
 	Eterm stoken = SEQ_TRACE_TOKEN(sender);
 	Uint seq_trace_size = 0;
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index d120111634..b2550814fd 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -405,8 +405,6 @@ typedef struct erl_trace_message_queue__ {
 #define SAVE_MESSAGE(p) \
      (p)->sig_qs.save = &(*(p)->sig_qs.save)->next
 
-#define ERTS_SND_FLG_NO_SEQ_TRACE		(((unsigned) 1) << 0)
-
 #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \
    (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm))
 
@@ -429,7 +427,7 @@ typedef struct erl_trace_message_queue__ {
     do {                                                \
         (MP)->next = NULL;                              \
         ERL_MESSAGE_TERM(MP) = THE_NON_VALUE;           \
-        ERL_MESSAGE_TOKEN(MP) = NIL;                    \
+        ERL_MESSAGE_TOKEN(MP) = THE_NON_VALUE;          \
         ERL_MESSAGE_FROM(MP) = NIL;                     \
         ERL_MESSAGE_DT_UTAG_INIT(MP);                   \
         MP->data.attached = NULL;                       \
@@ -446,7 +444,7 @@ void erts_queue_proc_message(Process* from,Process* to, ErtsProcLocks,ErtsMessag
 void erts_queue_proc_messages(Process* from, Process* to, ErtsProcLocks,
                               ErtsMessage*, ErtsMessage**, Uint);
 void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm);
-void erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned);
+void erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm);
 void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp);
 
 Uint erts_msg_attached_data_size_aux(ErtsMessage *msg);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 0fbf0eb03a..26498eda7b 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -744,8 +744,58 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 	rp_locks = ERTS_PROC_LOCK_MAIN;
 
     if (menv) {
+        Eterm token = c_p ? SEQ_TRACE_TOKEN(c_p) : am_undefined;
+        if (token != NIL && token != am_undefined) {
+            /* This code is copied from erts_send_message */
+            Eterm stoken = SEQ_TRACE_TOKEN(c_p);
+#ifdef USE_VM_PROBES
+            DTRACE_CHARBUF(sender_name, 64);
+            DTRACE_CHARBUF(receiver_name, 64);
+            Sint tok_label = 0;
+            Sint tok_lastcnt = 0;
+            Sint tok_serial = 0;
+            Eterm utag = NIL;
+            *sender_name = *receiver_name = '\0';
+            if (DTRACE_ENABLED(message_send)) {
+                erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)),
+                              "%T", c_p->common.id);
+                erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)),
+                              "%T", rp->common.id);
+            }
+#endif
+            if (have_seqtrace(stoken)) {
+                seq_trace_update_send(c_p);
+                seq_trace_output(stoken, msg, SEQ_TRACE_SEND,
+                                 rp->common.id, c_p);
+            }
+#ifdef USE_VM_PROBES
+            if (!(DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING)) {
+                stoken = NIL;
+            }
+#endif
+            token = enif_make_copy(msg_env, stoken);
+
+#ifdef USE_VM_PROBES
+            if (DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING) {
+                if (is_immed(DT_UTAG(c_p)))
+                    utag = DT_UTAG(c_p);
+                else
+                    utag = enif_make_copy(msg_env, DT_UTAG(c_p));
+            }
+            if (DTRACE_ENABLED(message_send)) {
+                if (have_seqtrace(stoken)) {
+                    tok_label = SEQ_TRACE_T_DTRACE_LABEL(stoken);
+                    tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken));
+                    tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken));
+                }
+                DTRACE6(message_send, sender_name, receiver_name,
+                        size_object(msg), tok_label, tok_lastcnt, tok_serial);
+            }
+#endif
+        }
         flush_env(msg_env);
         mp = erts_alloc_message(0, NULL);
+        ERL_MESSAGE_TOKEN(mp) = token;
         mp->data.heap_frag = menv->env.heap_frag;
         ASSERT(mp->data.heap_frag == MBUF(&menv->phony_proc));
         if (mp->data.heap_frag != NULL) {
@@ -783,6 +833,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 		ohp = &bp->off_heap;
 	    }
 	}
+        ERL_MESSAGE_TOKEN(mp) = am_undefined;
         msg = copy_struct_litopt(msg, sz, &hp, ohp, &litarea);
     }
 
@@ -826,6 +877,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
 
             ERL_MESSAGE_TERM(mp) = msg;
             ERL_MESSAGE_FROM(mp) = from;
+            ERL_MESSAGE_TOKEN(mp) = am_undefined;
 
             if (!msgq) {
                 msgq = erts_alloc(ERTS_ALC_T_TRACE_MSG_QUEUE,
diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index d6d22677e7..f90501cbf7 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -52,8 +52,8 @@
 
 #define ERTS_SIG_Q_OP_MAX 13
 
-#define ERTS_SIG_Q_OP_EXIT                      0
-#define ERTS_SIG_Q_OP_EXIT_LINKED               1
+#define ERTS_SIG_Q_OP_EXIT                      0  /* Exit signal due to bif call */
+#define ERTS_SIG_Q_OP_EXIT_LINKED               1  /* Exit signal due to link break*/
 #define ERTS_SIG_Q_OP_MONITOR_DOWN              2
 #define ERTS_SIG_Q_OP_MONITOR                   3
 #define ERTS_SIG_Q_OP_DEMONITOR                 4
@@ -1167,10 +1167,7 @@ erts_proc_sig_send_persistent_monitor_msg(Uint16 type, Eterm key,
     ERL_MESSAGE_TERM(mp) = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_PERSISTENT_MON_MSG,
                                                   type, 0);
     ERL_MESSAGE_FROM(mp) = from;
-    ERL_MESSAGE_TOKEN(mp) = NIL;
-#ifdef USE_VM_PROBES
-    ERL_MESSAGE_DT_UTAG(mp) = NIL;
-#endif
+    ERL_MESSAGE_TOKEN(mp) = am_undefined;
 
     if (!proc_queue_signal(NULL, to, (ErtsSignal *) mp,
                            ERTS_SIG_Q_OP_PERSISTENT_MON_MSG)) {
@@ -1564,11 +1561,6 @@ erts_proc_sig_send_is_alive_request(Process *c_p, Eterm to, Eterm ref)
     ERL_MESSAGE_TERM(mp) = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_IS_ALIVE,
                                                   ERTS_SIG_Q_TYPE_UNDEFINED,
                                                   0);
-    ERL_MESSAGE_TOKEN(mp) = NIL;
-    ERL_MESSAGE_FROM(mp) = am_system;
-#ifdef USE_VM_PROBES
-    ERL_MESSAGE_DT_UTAG(mp) = NIL;
-#endif
 
     if (proc_queue_signal(c_p, to, (ErtsSignal *) mp, ERTS_SIG_Q_OP_IS_ALIVE))
         (void) maybe_elevate_sig_handling_prio(c_p, to);
@@ -1672,11 +1664,6 @@ erts_proc_sig_send_sync_suspend(Process *c_p, Eterm to, Eterm tag, Eterm reply)
     ERL_MESSAGE_TERM(mp) = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_SYNC_SUSPEND,
                                                   ERTS_SIG_Q_TYPE_UNDEFINED,
                                                   0);
-    ERL_MESSAGE_TOKEN(mp) = NIL;
-    ERL_MESSAGE_FROM(mp) = am_system;
-#ifdef USE_VM_PROBES
-    ERL_MESSAGE_DT_UTAG(mp) = NIL;
-#endif
 
     if (proc_queue_signal(c_p, to, (ErtsSignal *) mp, ERTS_SIG_Q_OP_SYNC_SUSPEND))
         (void) maybe_elevate_sig_handling_prio(c_p, to);
@@ -1790,7 +1777,7 @@ handle_rpc(Process *c_p, ErtsProcSigRPC *rpc, int cnt, int limit, int *yieldp)
         msg = TUPLE2(hp, ref, res);
 
         mp->hfrag.next = bp;
-
+        ERL_MESSAGE_TOKEN(mp) = am_undefined;
         erts_queue_proc_message(c_p, rp, 0, mp, msg);
     }
 
@@ -2155,10 +2142,7 @@ handle_exit_signal(Process *c_p, ErtsSigRecvTracing *tracing,
                 pid = STORE_NC(&hp, ohp, from);
 
                 ERL_MESSAGE_TERM(mp) = TUPLE3(hp, am_EXIT, pid, reason);
-                ERL_MESSAGE_TOKEN(mp) = NIL;
-#ifdef USE_VM_PROBES
-                ERL_MESSAGE_DT_UTAG(mp) = NIL;
-#endif
+                ERL_MESSAGE_TOKEN(mp) = am_undefined;
                 if (is_immed(pid))
                     ERL_MESSAGE_FROM(mp) = pid;
                 else {
@@ -2346,10 +2330,7 @@ convert_to_down_message(Process *c_p,
                                   type, from, reason);
     hp += 6;
 
-    ERL_MESSAGE_TOKEN(mp) = NIL;
-#ifdef USE_VM_PROBES
-    ERL_MESSAGE_DT_UTAG(mp) = NIL;
-#endif
+    ERL_MESSAGE_TOKEN(mp) = am_undefined;
     /* Replace original signal with the exit message... */
     convert_to_msg(c_p, sig, mp, next_nm_sig);
 
@@ -2397,10 +2378,7 @@ convert_to_nodedown_messages(Process *c_p,
 
             ERL_MESSAGE_TERM(mp) = TUPLE2(hp, am_nodedown, node);
             ERL_MESSAGE_FROM(mp) = am_system;
-            ERL_MESSAGE_TOKEN(mp) = NIL;
-#ifdef USE_VM_PROBES
-            ERL_MESSAGE_DT_UTAG(mp) = NIL;
-#endif
+            ERL_MESSAGE_TOKEN(mp) = am_undefined;
             mp->next = nd_first;
             nd_first = mp;
             if (!nd_last)
@@ -2830,6 +2808,7 @@ handle_process_info(Process *c_p, ErtsSigRecvTracing *tracing,
         if (is_alive)
             erts_factory_trim_and_close(&hfact, &msg, 1);
 
+        ERL_MESSAGE_TOKEN(mp) = am_undefined;
         erts_queue_proc_message(c_p, rp, locks, mp, msg);
 
         if (!is_alive && locks)
@@ -2931,6 +2910,7 @@ sync_suspend_reply(Process *c_p, ErtsMessage *mp, erts_aint32_t state)
                 tp[2] = ssusp->async ? am_not_suspended : am_internal_error;
             }
         }
+        ERL_MESSAGE_TOKEN(mp) = am_undefined;
         erts_queue_proc_message(c_p, rp, 0, mp, ssusp->message);
     }
 }
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 8784eb5a63..9386f79b56 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9874,6 +9874,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st,
 	ASSERT(hp_start + hsz == hp);
 #endif
 
+        ERL_MESSAGE_TOKEN(mp) = am_undefined;
 	erts_queue_proc_message(c_p, rp, rp_locks, mp, msg);
 
 	if (c_p == rp)
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 2446b3c074..133ab485d9 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -3287,7 +3287,6 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
     if (trace_send)
         trace_port_send(prt, to, tuple, 1);
 
-    ERL_MESSAGE_TOKEN(mp) = am_undefined;
     erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id);
     if (rp_locks)
 	erts_proc_unlock(rp, rp_locks);
@@ -3459,7 +3458,6 @@ deliver_vec_message(Port* prt,			/* Port */
     if (IS_TRACED_FL(prt, F_TRACE_SEND))
         trace_port_send(prt, to, tuple, 1);
 
-    ERL_MESSAGE_TOKEN(mp) = am_undefined;
     erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id);
     erts_proc_unlock(rp, rp_locks);
     if (!scheduler)
@@ -5382,7 +5380,6 @@ void driver_report_exit(ErlDrvPort ix, int status)
     if (IS_TRACED_FL(prt, F_TRACE_SEND))
         trace_port_send(prt, pid, tuple, 1);
 
-   ERL_MESSAGE_TOKEN(mp) = am_undefined;
    erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id);
 
    erts_proc_unlock(rp, rp_locks);
@@ -5988,8 +5985,6 @@ driver_deliver_term(Port *prt, Eterm to, ErlDrvTermData* data, int len)
 	    from = prt->common.id;
 	}
 
-	/* send message */
-        ERL_MESSAGE_TOKEN(factory.message) = am_undefined;
 	erts_queue_message(rp, rp_locks, factory.message, mess, from);
     }
     else if (res == -2) {
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 3e77dce1cd..9f115706dc 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1493,7 +1493,6 @@ send_event_tuple(struct erts_nif_select_event* e, ErtsResource* resource,
     }
     tuple = TUPLE4(hp, am_select, resource_term, ref_term, event_atom);
 
-    ERL_MESSAGE_TOKEN(mp) = am_undefined;
     erts_queue_message(rp, rp_locks, mp, tuple, am_system);
 
     if (rp_locks)
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 7c85cf2259..a2f3489943 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -45,8 +45,9 @@
 	 api_macros/1,
 	 from_array/1, iolist_as_binary/1, resource/1, resource_binary/1,
 	 resource_takeover/1,
-	 threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1,
-	 is_checks/1,
+	 threading/1, send/1, send2/1, send3/1, send_threaded/1,
+         send_trace/1, send_seq_trace/1,
+         neg/1, is_checks/1,
 	 get_length/1, make_atom/1, make_string/1, reverse_list_test/1,
 	 otp_9828/1,
 	 otp_9668/1, consume_timeslice/1, nif_schedule/1,
@@ -1789,6 +1790,59 @@ send(Config) when is_list(Config) ->
     {ok,0} = send_list_seq(7, DeadPid),
     ok.
 
+
+%% Test tracing of enif_send
+send_trace(Config) when is_list(Config) ->
+    ensure_lib_loaded(Config),
+
+    Papa = self(),
+    N = 1500,
+    List = lists:seq(1,N),
+
+    Tracer = spawn_link(fun F() -> receive get -> Papa ! receive_any(), F() end end),
+
+    erlang:trace(self(), true, [send,'receive',{tracer,Tracer}]),
+    {ok,1} = send_list_seq(N, self()),
+    List = receive_any(),
+    timeout = receive_any(0),
+    Tracer ! get,
+    {trace,Papa,send,List,Papa} = receive_any(),
+    Tracer ! get,
+    {trace,Papa,'receive',List} = receive_any().
+
+%% Test that seq_trace works with nif trace
+send_seq_trace(Config) when is_list(Config) ->
+    ensure_lib_loaded(Config),
+
+    Papa = self(),
+    N = 1500,
+    List = lists:seq(1,N),
+    Label = make_ref(),
+
+    Tracer = spawn_link(fun F() -> receive get -> Papa ! receive_any(), F() end end),
+
+    seq_trace:set_system_tracer(Tracer),
+    seq_trace:set_token(label,Label),
+    seq_trace:set_token(send,true),
+    seq_trace:set_token('receive',true),
+
+    {ok,1} = send_list_seq(N, self()),
+    List = receive_any(),
+    timeout = receive_any(0),
+    {ok,1} = send_list_seq(N, self()),
+    List = receive_any(),
+    timeout = receive_any(0),
+
+    Tracer ! get,
+    {seq_trace,Label,{send,{0,1},Papa,Papa,List}} = receive_any(),
+    Tracer ! get,
+    {seq_trace,Label,{'receive',{0,1},Papa,Papa,List}} = receive_any(),
+    Tracer ! get,
+    {seq_trace,Label,{send,{1,2},Papa,Papa,List}} = receive_any(),
+    Tracer ! get,
+    {seq_trace,Label,{'receive',{1,2},Papa,Papa,List}} = receive_any().
+
+
 %% More NIF message sending
 send2(Config) when is_list(Config) ->
     ensure_lib_loaded(Config),
-- 
cgit v1.2.3


From 8d108683b8c797f601cdd0caf8925f807f8b893d Mon Sep 17 00:00:00 2001
From: Simon Cornish <zl9d97p02@sneakemail.com>
Date: Fri, 3 Aug 2018 11:50:19 -0700
Subject: Test #Ref ordering in lists and ets

---
 erts/emulator/test/ref_SUITE.erl | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/ref_SUITE.erl b/erts/emulator/test/ref_SUITE.erl
index 5f519d522e..74df857c65 100644
--- a/erts/emulator/test/ref_SUITE.erl
+++ b/erts/emulator/test/ref_SUITE.erl
@@ -22,6 +22,7 @@
 
 -export([all/0, suite/0]).
 -export([wrap_1/1]).
+-export([compare_list/1, compare_ets/1]).
 
 -export([loop_ref/1]).
 
@@ -32,7 +33,7 @@ suite() ->
      {timetrap, {minutes, 2}}].
 
 all() -> 
-    [wrap_1].
+    [wrap_1, compare_list, compare_ets].
 
 %% Check that refs don't wrap around easily.
 wrap_1(Config) when is_list(Config) ->
@@ -53,3 +54,28 @@ loop_ref(Parent) ->
 loop_ref(R, R, _) -> ok;
 loop_ref(R0, _, N) ->
     loop_ref(R0, make_ref(), N+1).
+
+%% Check that ref ordering works
+compare_list(Config) when is_list(Config) ->
+    %% Although this test uses external refs, it would apply the same to plain refs
+    ExtRef1 = <<131,114,0,3,100,0,3,110,64,98,3, 0,0,173,156, 0,216,0,4, 0,0,0,0>>,
+    ExtRef2 = <<131,114,0,3,100,0,3,110,64,98,3, 0,1,31,27,   129,4,0,1, 0,0,0,0>>,
+
+    Ref1 = binary_to_term(ExtRef1), %% #Ref<n@b.0.14155780.44444>
+    Ref2 = binary_to_term(ExtRef2), %% #Ref<n@b.0.2164523009.73499>
+    OrderedList = [Ref1, Ref2],
+    OrderedList = lists:sort(OrderedList),
+    ok.
+
+%% This is the scarier case since it makes terms "invisible" in ets or Mnesia
+%% (the underlying fault cause is the same as compare_list/1)
+compare_ets(Config) when is_list(Config) ->
+    W2s = [610350147,899574699,2994196869,686384822,2397690439, 923302211],
+    ExtRefBase = <<131,114,0,3,100,0,3,110,64,98,3>>,
+    ExtRefs = [<<ExtRefBase/binary, 1:32, W2:32, 0:32>> || W2 <- W2s],
+    Refs = [binary_to_term(Bin) || Bin <- ExtRefs],
+
+    Ets = ets:new(refbug, [ordered_set]),
+    ets:insert(Ets, [{Ref,Ref} || Ref <- Refs]),
+    0 = length([R || R <- ets:tab2list(Ets), ets:lookup(Ets, element(1,R)) == []]),
+    ok.
-- 
cgit v1.2.3


From 27f6c054de78219ef2f03a7996fd8e7d9b09471b Mon Sep 17 00:00:00 2001
From: Simon Cornish <zl9d97p02@sneakemail.com>
Date: Fri, 3 Aug 2018 11:54:01 -0700
Subject: Fixed #Ref ordering bug

---
 erts/emulator/beam/utils.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 19b1312ee3..08f8ca9788 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3122,7 +3122,7 @@ tailrecur_ne:
 		ASSERT(alen == blen);
 		for (i = (Sint) alen - 1; i >= 0; i--)
 		    if (anum[i] != bnum[i])
-			RETURN_NEQ((Sint32) (anum[i] - bnum[i]));
+			RETURN_NEQ(anum[i] < bnum[i] ? -1 : 1);
 		goto pop_next;
 	    case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE):
 		if (is_internal_ref(b)) {
-- 
cgit v1.2.3


From 384e8b3b36c6c8ced0778cab9e1569ed0b55e09a Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 7 Aug 2018 18:49:26 +0200
Subject: Update PCRE from version 8.41 to version 8.42

---
 erts/emulator/pcre/LICENCE               |  93 +++++++
 erts/emulator/pcre/README.pcre_update.md |   6 +
 erts/emulator/pcre/local_config.h        |   2 +-
 erts/emulator/pcre/pcre-8.41.tar.bz2     | Bin 1561874 -> 0 bytes
 erts/emulator/pcre/pcre-8.42.tar.bz2     | Bin 0 -> 1570171 bytes
 erts/emulator/pcre/pcre.h                |  12 +-
 erts/emulator/pcre/pcre_chartables.c     |   2 +
 erts/emulator/pcre/pcre_compile.c        |   2 +-
 erts/emulator/pcre/pcre_dfa_exec.c       |   4 +-
 erts/emulator/pcre/pcre_exec.c           |   8 +-
 erts/emulator/pcre/pcre_jit_compile.c    | 407 ++++++++++++++++++++++---------
 erts/emulator/pcre/pcre_latin_1_table.c  |   3 +-
 12 files changed, 406 insertions(+), 133 deletions(-)
 create mode 100644 erts/emulator/pcre/LICENCE
 delete mode 100644 erts/emulator/pcre/pcre-8.41.tar.bz2
 create mode 100644 erts/emulator/pcre/pcre-8.42.tar.bz2

(limited to 'erts/emulator')

diff --git a/erts/emulator/pcre/LICENCE b/erts/emulator/pcre/LICENCE
new file mode 100644
index 0000000000..f6ef7fd766
--- /dev/null
+++ b/erts/emulator/pcre/LICENCE
@@ -0,0 +1,93 @@
+PCRE LICENCE
+------------
+
+PCRE is a library of functions to support regular expressions whose syntax
+and semantics are as close as possible to those of the Perl 5 language.
+
+Release 8 of PCRE is distributed under the terms of the "BSD" licence, as
+specified below. The documentation for PCRE, supplied in the "doc"
+directory, is distributed under the same terms as the software itself. The data
+in the testdata directory is not copyrighted and is in the public domain.
+
+The basic library functions are written in C and are freestanding. Also
+included in the distribution is a set of C++ wrapper functions, and a
+just-in-time compiler that can be used to optimize pattern matching. These
+are both optional features that can be omitted when the library is built.
+
+
+THE BASIC LIBRARY FUNCTIONS
+---------------------------
+
+Written by:       Philip Hazel
+Email local part: ph10
+Email domain:     cam.ac.uk
+
+University of Cambridge Computing Service,
+Cambridge, England.
+
+Copyright (c) 1997-2018 University of Cambridge
+All rights reserved.
+
+
+PCRE JUST-IN-TIME COMPILATION SUPPORT
+-------------------------------------
+
+Written by:       Zoltan Herczeg
+Email local part: hzmester
+Emain domain:     freemail.hu
+
+Copyright(c) 2010-2018 Zoltan Herczeg
+All rights reserved.
+
+
+STACK-LESS JUST-IN-TIME COMPILER
+--------------------------------
+
+Written by:       Zoltan Herczeg
+Email local part: hzmester
+Emain domain:     freemail.hu
+
+Copyright(c) 2009-2018 Zoltan Herczeg
+All rights reserved.
+
+
+THE C++ WRAPPER FUNCTIONS
+-------------------------
+
+Contributed by:   Google Inc.
+
+Copyright (c) 2007-2012, Google Inc.
+All rights reserved.
+
+
+THE "BSD" LICENCE
+-----------------
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+
+    * Neither the name of the University of Cambridge nor the name of Google
+      Inc. nor the names of their contributors may be used to endorse or
+      promote products derived from this software without specific prior
+      written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+End
diff --git a/erts/emulator/pcre/README.pcre_update.md b/erts/emulator/pcre/README.pcre_update.md
index 599e3d0d12..5df1e15bde 100644
--- a/erts/emulator/pcre/README.pcre_update.md
+++ b/erts/emulator/pcre/README.pcre_update.md
@@ -723,6 +723,12 @@ requires thorough reading of all new text.  For the upgrade from 7.6
 to 8.33, the update of the pcrepattern part of our manual page took
 about eight hours.
 
+## Update Licence
+
+Copy the LICENCE file to `erts/emulator/pcre/LICENCE` and update
+the `[PCRE]` section in `system/COPYRIGHT` with the content of
+the `LICENCE` file.
+
 ## Add new relevant options to re
 
 Then, when all this is done, you should add any new relevant options
diff --git a/erts/emulator/pcre/local_config.h b/erts/emulator/pcre/local_config.h
index c6af423d72..c3b4dab586 100644
--- a/erts/emulator/pcre/local_config.h
+++ b/erts/emulator/pcre/local_config.h
@@ -86,4 +86,4 @@
 #define SUPPORT_UTF 
 
 /* Version number of package */
-#define VERSION "8.41"
+#define VERSION "8.42"
diff --git a/erts/emulator/pcre/pcre-8.41.tar.bz2 b/erts/emulator/pcre/pcre-8.41.tar.bz2
deleted file mode 100644
index 1798432dc9..0000000000
Binary files a/erts/emulator/pcre/pcre-8.41.tar.bz2 and /dev/null differ
diff --git a/erts/emulator/pcre/pcre-8.42.tar.bz2 b/erts/emulator/pcre/pcre-8.42.tar.bz2
new file mode 100644
index 0000000000..61bfa38970
Binary files /dev/null and b/erts/emulator/pcre/pcre-8.42.tar.bz2 differ
diff --git a/erts/emulator/pcre/pcre.h b/erts/emulator/pcre/pcre.h
index ab8f40cfc1..3563791223 100644
--- a/erts/emulator/pcre/pcre.h
+++ b/erts/emulator/pcre/pcre.h
@@ -43,9 +43,9 @@ POSSIBILITY OF SUCH DAMAGE.
 /* The current PCRE version information. */
 
 #define PCRE_MAJOR          8
-#define PCRE_MINOR          41
+#define PCRE_MINOR          42
 #define PCRE_PRERELEASE     
-#define PCRE_DATE           2017-07-05
+#define PCRE_DATE           2018-03-20
 
 /* When an application links to a PCRE DLL in Windows, the symbols that are
 imported have to be identified as such. When building PCRE, the appropriate
@@ -328,11 +328,11 @@ these bits, just add new ones on the end, in order to remain compatible. */
 
 /* Types */
 
-struct real_pcre;                 /* declaration; the definition is private  */
-typedef struct real_pcre pcre;
+struct real_pcre8_or_16;          /* declaration; the definition is private  */
+typedef struct real_pcre8_or_16 pcre;
 
-struct real_pcre16;               /* declaration; the definition is private  */
-typedef struct real_pcre16 pcre16;
+struct real_pcre8_or_16;          /* declaration; the definition is private  */
+typedef struct real_pcre8_or_16 pcre16;
 
 struct real_pcre32;               /* declaration; the definition is private  */
 typedef struct real_pcre32 pcre32;
diff --git a/erts/emulator/pcre/pcre_chartables.c b/erts/emulator/pcre/pcre_chartables.c
index b3d9020f25..06482c08d2 100644
--- a/erts/emulator/pcre/pcre_chartables.c
+++ b/erts/emulator/pcre/pcre_chartables.c
@@ -19,7 +19,9 @@ array definition from the final binary if PCRE is built into a static library
 and dead code stripping is activated. This leads to link errors. Pulling in the
 header ensures that the array gets flagged as "someone outside this compilation
 unit might reference this" and so it will always be supplied to the linker. */
+
 /* %ExternalCopyright% */
+
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif
diff --git a/erts/emulator/pcre/pcre_compile.c b/erts/emulator/pcre/pcre_compile.c
index e79284ab79..ae7f6e2a2a 100644
--- a/erts/emulator/pcre/pcre_compile.c
+++ b/erts/emulator/pcre/pcre_compile.c
@@ -8061,7 +8061,7 @@ for (;; ptr++)
         single group (i.e. not to a duplicated name. */
 
         HANDLE_REFERENCE:
-        if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE;
+        if (firstcharflags == REQ_UNSET) zerofirstcharflags = firstcharflags = REQ_NONE;
         previous = code;
         item_hwm_offset = cd->hwm - cd->start_workspace;
         *code++ = ((options & PCRE_CASELESS) != 0)? OP_REFI : OP_REF;
diff --git a/erts/emulator/pcre/pcre_dfa_exec.c b/erts/emulator/pcre/pcre_dfa_exec.c
index c859d67fc7..c101656fd7 100644
--- a/erts/emulator/pcre/pcre_dfa_exec.c
+++ b/erts/emulator/pcre/pcre_dfa_exec.c
@@ -2288,12 +2288,14 @@ for (;;)
       case OP_NOTI:
       if (clen > 0)
         {
-        unsigned int otherd;
+        pcre_uint32 otherd;
 #ifdef SUPPORT_UTF
         if (utf && d >= 128)
           {
 #ifdef SUPPORT_UCP
           otherd = UCD_OTHERCASE(d);
+#else
+          otherd = d;
 #endif  /* SUPPORT_UCP */
           }
         else
diff --git a/erts/emulator/pcre/pcre_exec.c b/erts/emulator/pcre/pcre_exec.c
index 6708ba92a6..1946e97a72 100644
--- a/erts/emulator/pcre/pcre_exec.c
+++ b/erts/emulator/pcre/pcre_exec.c
@@ -6,7 +6,7 @@
 and semantics are as close as possible to those of the Perl 5 language.
 
                        Written by Philip Hazel
-           Copyright (c) 1997-2014 University of Cambridge
+           Copyright (c) 1997-2018 University of Cambridge
 
 -----------------------------------------------------------------------------
 Redistribution and use in source and binary forms, with or without
@@ -2407,7 +2407,7 @@ for (;;)
     case OP_ANY:
     if (IS_NEWLINE(eptr)) RRETURN(MATCH_NOMATCH);
     if (md->partial != 0 &&
-        eptr + 1 >= md->end_subject &&
+        eptr == md->end_subject - 1 &&
         NLBLOCK->nltype == NLTYPE_FIXED &&
         NLBLOCK->nllen == 2 &&
         UCHAR21TEST(eptr) == NLBLOCK->nl[0])
@@ -3167,7 +3167,7 @@ for (;;)
             {
             RMATCH(eptr, ecode, offset_top, md, eptrb, RM18);
             if (rrc != MATCH_NOMATCH) RRETURN(rrc);
-            if (eptr-- == pp) break;        /* Stop if tried at original pos */
+            if (eptr-- <= pp) break;        /* Stop if tried at original pos */
             BACKCHAR(eptr);
             }
           }
@@ -3326,7 +3326,7 @@ for (;;)
           {
           RMATCH(eptr, ecode, offset_top, md, eptrb, RM21);
           if (rrc != MATCH_NOMATCH) RRETURN(rrc);
-          if (eptr-- == pp) break;        /* Stop if tried at original pos */
+          if (eptr-- <= pp) break;        /* Stop if tried at original pos */
 #ifdef SUPPORT_UTF
           if (utf) BACKCHAR(eptr);
 #endif
diff --git a/erts/emulator/pcre/pcre_jit_compile.c b/erts/emulator/pcre/pcre_jit_compile.c
index 932ca2c389..926e40f6d3 100644
--- a/erts/emulator/pcre/pcre_jit_compile.c
+++ b/erts/emulator/pcre/pcre_jit_compile.c
@@ -164,7 +164,6 @@ typedef struct jit_arguments {
   const pcre_uchar *begin;
   const pcre_uchar *end;
   int *offsets;
-  pcre_uchar *uchar_ptr;
   pcre_uchar *mark_ptr;
   void *callout_data;
   /* Everything else after. */
@@ -214,7 +213,7 @@ enum control_types {
   type_then_trap = 1
 };
 
-typedef int (SLJIT_CALL *jit_function)(jit_arguments *args);
+typedef int (SLJIT_FUNC *jit_function)(jit_arguments *args);
 
 /* The following structure is the key data type for the recursive
 code generator. It is allocated by compile_matchingpath, and contains
@@ -489,9 +488,24 @@ typedef struct compare_context {
 /* Used for accessing the elements of the stack. */
 #define STACK(i)      ((i) * (int)sizeof(sljit_sw))
 
+#ifdef SLJIT_PREF_SHIFT_REG
+#if SLJIT_PREF_SHIFT_REG == SLJIT_R2
+/* Nothing. */
+#elif SLJIT_PREF_SHIFT_REG == SLJIT_R3
+#define SHIFT_REG_IS_R3
+#else
+#error "Unsupported shift register"
+#endif
+#endif
+
 #define TMP1          SLJIT_R0
+#ifdef SHIFT_REG_IS_R3
+#define TMP2          SLJIT_R3
+#define TMP3          SLJIT_R2
+#else
 #define TMP2          SLJIT_R2
 #define TMP3          SLJIT_R3
+#endif
 #define STR_PTR       SLJIT_S0
 #define STR_END       SLJIT_S1
 #define STACK_TOP     SLJIT_R1
@@ -520,13 +534,10 @@ the start pointers when the end of the capturing group has not yet reached. */
 
 #if defined COMPILE_PCRE8
 #define MOV_UCHAR  SLJIT_MOV_U8
-#define MOVU_UCHAR SLJIT_MOVU_U8
 #elif defined COMPILE_PCRE16
 #define MOV_UCHAR  SLJIT_MOV_U16
-#define MOVU_UCHAR SLJIT_MOVU_U16
 #elif defined COMPILE_PCRE32
 #define MOV_UCHAR  SLJIT_MOV_U32
-#define MOVU_UCHAR SLJIT_MOVU_U32
 #else
 #error Unsupported compiling mode
 #endif
@@ -2383,12 +2394,25 @@ if (length < 8)
   }
 else
   {
-  GET_LOCAL_BASE(SLJIT_R1, 0, OVECTOR_START);
-  OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, length - 1);
-  loop = LABEL();
-  OP1(SLJIT_MOVU, SLJIT_MEM1(SLJIT_R1), sizeof(sljit_sw), SLJIT_R0, 0);
-  OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, 1);
-  JUMPTO(SLJIT_NOT_ZERO, loop);
+  if (sljit_emit_mem(compiler, SLJIT_MOV | SLJIT_MEM_SUPP | SLJIT_MEM_STORE | SLJIT_MEM_PRE, SLJIT_R0, SLJIT_MEM1(SLJIT_R1), sizeof(sljit_sw)) == SLJIT_SUCCESS)
+    {
+    GET_LOCAL_BASE(SLJIT_R1, 0, OVECTOR_START);
+    OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, length - 1);
+    loop = LABEL();
+    sljit_emit_mem(compiler, SLJIT_MOV | SLJIT_MEM_STORE | SLJIT_MEM_PRE, SLJIT_R0, SLJIT_MEM1(SLJIT_R1), sizeof(sljit_sw));
+    OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, 1);
+    JUMPTO(SLJIT_NOT_ZERO, loop);
+    }
+  else
+    {
+    GET_LOCAL_BASE(SLJIT_R1, 0, OVECTOR_START + sizeof(sljit_sw));
+    OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, length - 1);
+    loop = LABEL();
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_R1), 0, SLJIT_R0, 0);
+    OP2(SLJIT_ADD, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, sizeof(sljit_sw));
+    OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, 1);
+    JUMPTO(SLJIT_NOT_ZERO, loop);
+    }
   }
 }
 
@@ -2421,12 +2445,25 @@ if (length < 8)
   }
 else
   {
-  GET_LOCAL_BASE(TMP2, 0, OVECTOR_START + sizeof(sljit_sw));
-  OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_IMM, length - 2);
-  loop = LABEL();
-  OP1(SLJIT_MOVU, SLJIT_MEM1(TMP2), sizeof(sljit_sw), TMP1, 0);
-  OP2(SLJIT_SUB | SLJIT_SET_Z, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 1);
-  JUMPTO(SLJIT_NOT_ZERO, loop);
+  if (sljit_emit_mem(compiler, SLJIT_MOV | SLJIT_MEM_SUPP | SLJIT_MEM_STORE | SLJIT_MEM_PRE, TMP1, SLJIT_MEM1(TMP2), sizeof(sljit_sw)) == SLJIT_SUCCESS)
+    {
+    GET_LOCAL_BASE(TMP2, 0, OVECTOR_START + sizeof(sljit_sw));
+    OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_IMM, length - 2);
+    loop = LABEL();
+    sljit_emit_mem(compiler, SLJIT_MOV | SLJIT_MEM_STORE | SLJIT_MEM_PRE, TMP1, SLJIT_MEM1(TMP2), sizeof(sljit_sw));
+    OP2(SLJIT_SUB | SLJIT_SET_Z, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 1);
+    JUMPTO(SLJIT_NOT_ZERO, loop);
+    }
+  else
+    {
+    GET_LOCAL_BASE(TMP2, 0, OVECTOR_START + 2 * sizeof(sljit_sw));
+    OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_IMM, length - 2);
+    loop = LABEL();
+    OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, TMP1, 0);
+    OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, sizeof(sljit_sw));
+    OP2(SLJIT_SUB | SLJIT_SET_Z, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 1);
+    JUMPTO(SLJIT_NOT_ZERO, loop);
+    }
   }
 
 OP1(SLJIT_MOV, STACK_TOP, 0, ARGUMENTS, 0);
@@ -2436,10 +2473,10 @@ if (common->control_head_ptr != 0)
   OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0);
 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), SLJIT_OFFSETOF(jit_arguments, stack));
 OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->start_ptr);
-OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), SLJIT_OFFSETOF(struct sljit_stack, base));
+OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), SLJIT_OFFSETOF(struct sljit_stack, end));
 }
 
-static sljit_sw SLJIT_CALL do_search_mark(sljit_sw *current, const pcre_uchar *skip_arg)
+static sljit_sw SLJIT_FUNC do_search_mark(sljit_sw *current, const pcre_uchar *skip_arg)
 {
 while (current != NULL)
   {
@@ -2460,7 +2497,7 @@ while (current != NULL)
   SLJIT_ASSERT(current[0] == 0 || current < (sljit_sw*)current[0]);
   current = (sljit_sw*)current[0];
   }
-return -1;
+return 0;
 }
 
 static SLJIT_INLINE void copy_ovector(compiler_common *common, int topbracket)
@@ -2468,6 +2505,7 @@ static SLJIT_INLINE void copy_ovector(compiler_common *common, int topbracket)
 DEFINE_COMPILER;
 struct sljit_label *loop;
 struct sljit_jump *early_quit;
+BOOL has_pre;
 
 /* At this point we can freely use all registers. */
 OP1(SLJIT_MOV, SLJIT_S2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1));
@@ -2481,17 +2519,30 @@ if (common->mark_ptr != 0)
   OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, mark_ptr), SLJIT_R2, 0);
 OP2(SLJIT_SUB, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, offsets), SLJIT_IMM, sizeof(int));
 OP1(SLJIT_MOV, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, begin));
-GET_LOCAL_BASE(SLJIT_S0, 0, OVECTOR_START);
+
+has_pre = sljit_emit_mem(compiler, SLJIT_MOV | SLJIT_MEM_SUPP | SLJIT_MEM_PRE, SLJIT_S1, SLJIT_MEM1(SLJIT_S0), sizeof(sljit_sw)) == SLJIT_SUCCESS;
+GET_LOCAL_BASE(SLJIT_S0, 0, OVECTOR_START - (has_pre ? sizeof(sljit_sw) : 0));
+
 /* Unlikely, but possible */
 early_quit = CMP(SLJIT_EQUAL, SLJIT_R1, 0, SLJIT_IMM, 0);
 loop = LABEL();
-OP2(SLJIT_SUB, SLJIT_S1, 0, SLJIT_MEM1(SLJIT_S0), 0, SLJIT_R0, 0);
-OP2(SLJIT_ADD, SLJIT_S0, 0, SLJIT_S0, 0, SLJIT_IMM, sizeof(sljit_sw));
+
+if (has_pre)
+  sljit_emit_mem(compiler, SLJIT_MOV | SLJIT_MEM_PRE, SLJIT_S1, SLJIT_MEM1(SLJIT_S0), sizeof(sljit_sw));
+else
+  {
+  OP1(SLJIT_MOV, SLJIT_S1, 0, SLJIT_MEM1(SLJIT_S0), 0);
+  OP2(SLJIT_ADD, SLJIT_S0, 0, SLJIT_S0, 0, SLJIT_IMM, sizeof(sljit_sw));
+  }
+
+OP2(SLJIT_ADD, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, sizeof(int));
+OP2(SLJIT_SUB, SLJIT_S1, 0, SLJIT_S1, 0, SLJIT_R0, 0);
 /* Copy the integer value to the output buffer */
 #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32
 OP2(SLJIT_ASHR, SLJIT_S1, 0, SLJIT_S1, 0, SLJIT_IMM, UCHAR_SHIFT);
 #endif
-OP1(SLJIT_MOVU_S32, SLJIT_MEM1(SLJIT_R2), sizeof(int), SLJIT_S1, 0);
+
+OP1(SLJIT_MOV_S32, SLJIT_MEM1(SLJIT_R2), 0, SLJIT_S1, 0);
 OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1);
 JUMPTO(SLJIT_NOT_ZERO, loop);
 JUMPHERE(early_quit);
@@ -2499,14 +2550,29 @@ JUMPHERE(early_quit);
 /* Calculate the return value, which is the maximum ovector value. */
 if (topbracket > 1)
   {
-  GET_LOCAL_BASE(SLJIT_R0, 0, OVECTOR_START + topbracket * 2 * sizeof(sljit_sw));
-  OP1(SLJIT_MOV, SLJIT_R1, 0, SLJIT_IMM, topbracket + 1);
+  if (sljit_emit_mem(compiler, SLJIT_MOV | SLJIT_MEM_SUPP | SLJIT_MEM_PRE, SLJIT_R2, SLJIT_MEM1(SLJIT_R0), -(2 * (sljit_sw)sizeof(sljit_sw))) == SLJIT_SUCCESS)
+    {
+    GET_LOCAL_BASE(SLJIT_R0, 0, OVECTOR_START + topbracket * 2 * sizeof(sljit_sw));
+    OP1(SLJIT_MOV, SLJIT_R1, 0, SLJIT_IMM, topbracket + 1);
 
-  /* OVECTOR(0) is never equal to SLJIT_S2. */
-  loop = LABEL();
-  OP1(SLJIT_MOVU, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_R0), -(2 * (sljit_sw)sizeof(sljit_sw)));
-  OP2(SLJIT_SUB, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1);
-  CMPTO(SLJIT_EQUAL, SLJIT_R2, 0, SLJIT_S2, 0, loop);
+    /* OVECTOR(0) is never equal to SLJIT_S2. */
+    loop = LABEL();
+    sljit_emit_mem(compiler, SLJIT_MOV | SLJIT_MEM_PRE, SLJIT_R2, SLJIT_MEM1(SLJIT_R0), -(2 * (sljit_sw)sizeof(sljit_sw)));
+    OP2(SLJIT_SUB, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1);
+    CMPTO(SLJIT_EQUAL, SLJIT_R2, 0, SLJIT_S2, 0, loop);
+    }
+  else
+    {
+    GET_LOCAL_BASE(SLJIT_R0, 0, OVECTOR_START + (topbracket - 1) * 2 * sizeof(sljit_sw));
+    OP1(SLJIT_MOV, SLJIT_R1, 0, SLJIT_IMM, topbracket + 1);
+
+    /* OVECTOR(0) is never equal to SLJIT_S2. */
+    loop = LABEL();
+    OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_R0), 0);
+    OP2(SLJIT_SUB, SLJIT_R0, 0, SLJIT_R0, 0, SLJIT_IMM, 2 * (sljit_sw)sizeof(sljit_sw));
+    OP2(SLJIT_SUB, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1);
+    CMPTO(SLJIT_EQUAL, SLJIT_R2, 0, SLJIT_S2, 0, loop);
+    }
   OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_R1, 0);
   }
 else
@@ -5167,93 +5233,190 @@ OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL);
 sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
 }
 
-#define CHAR1 STR_END
-#define CHAR2 STACK_TOP
-
 static void do_casefulcmp(compiler_common *common)
 {
 DEFINE_COMPILER;
 struct sljit_jump *jump;
 struct sljit_label *label;
+int char1_reg;
+int char2_reg;
 
-sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
+if (sljit_get_register_index(TMP3) < 0)
+  {
+  char1_reg = STR_END;
+  char2_reg = STACK_TOP;
+  }
+else
+  {
+  char1_reg = TMP3;
+  char2_reg = RETURN_ADDR;
+  }
+
+sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0);
 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
-OP1(SLJIT_MOV, TMP3, 0, CHAR1, 0);
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, CHAR2, 0);
-OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1));
-OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
 
-label = LABEL();
-OP1(MOVU_UCHAR, CHAR1, 0, SLJIT_MEM1(TMP1), IN_UCHARS(1));
-OP1(MOVU_UCHAR, CHAR2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
-jump = CMP(SLJIT_NOT_EQUAL, CHAR1, 0, CHAR2, 0);
-OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1));
-JUMPTO(SLJIT_NOT_ZERO, label);
+if (char1_reg == STR_END)
+  {
+  OP1(SLJIT_MOV, TMP3, 0, char1_reg, 0);
+  OP1(SLJIT_MOV, RETURN_ADDR, 0, char2_reg, 0);
+  }
 
-JUMPHERE(jump);
-OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
-OP1(SLJIT_MOV, CHAR1, 0, TMP3, 0);
-OP1(SLJIT_MOV, CHAR2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
-sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
-}
+if (sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_SUPP | SLJIT_MEM_POST, char1_reg, SLJIT_MEM1(TMP1), IN_UCHARS(1)) == SLJIT_SUCCESS)
+  {
+  label = LABEL();
+  sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_POST, char1_reg, SLJIT_MEM1(TMP1), IN_UCHARS(1));
+  sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_POST, char2_reg, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
+  jump = CMP(SLJIT_NOT_EQUAL, char1_reg, 0, char2_reg, 0);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1));
+  JUMPTO(SLJIT_NOT_ZERO, label);
+
+  JUMPHERE(jump);
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
+  }
+else if (sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_SUPP | SLJIT_MEM_PRE, char1_reg, SLJIT_MEM1(TMP1), IN_UCHARS(1)) == SLJIT_SUCCESS)
+  {
+  OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1));
+  OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+
+  label = LABEL();
+  sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_PRE, char1_reg, SLJIT_MEM1(TMP1), IN_UCHARS(1));
+  sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_PRE, char2_reg, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
+  jump = CMP(SLJIT_NOT_EQUAL, char1_reg, 0, char2_reg, 0);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1));
+  JUMPTO(SLJIT_NOT_ZERO, label);
 
-#define LCC_TABLE STACK_LIMIT
+  JUMPHERE(jump);
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
+  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+  }
+else
+  {
+  label = LABEL();
+  OP1(MOV_UCHAR, char1_reg, 0, SLJIT_MEM1(TMP1), 0);
+  OP1(MOV_UCHAR, char2_reg, 0, SLJIT_MEM1(STR_PTR), 0);
+  OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1));
+  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+  jump = CMP(SLJIT_NOT_EQUAL, char1_reg, 0, char2_reg, 0);
+  OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1));
+  JUMPTO(SLJIT_NOT_ZERO, label);
+
+  JUMPHERE(jump);
+  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
+  }
+
+if (char1_reg == STR_END)
+  {
+  OP1(SLJIT_MOV, char1_reg, 0, TMP3, 0);
+  OP1(SLJIT_MOV, char2_reg, 0, RETURN_ADDR, 0);
+  }
+
+sljit_emit_fast_return(compiler, TMP1, 0);
+}
 
 static void do_caselesscmp(compiler_common *common)
 {
 DEFINE_COMPILER;
 struct sljit_jump *jump;
 struct sljit_label *label;
+int char1_reg = STR_END;
+int char2_reg;
+int lcc_table;
+int opt_type = 0;
 
-sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);
+if (sljit_get_register_index(TMP3) < 0)
+  {
+  char2_reg = STACK_TOP;
+  lcc_table = STACK_LIMIT;
+  }
+else
+  {
+  char2_reg = RETURN_ADDR;
+  lcc_table = TMP3;
+  }
+
+if (sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_SUPP | SLJIT_MEM_POST, char1_reg, SLJIT_MEM1(TMP1), IN_UCHARS(1)) == SLJIT_SUCCESS)
+  opt_type = 1;
+else if (sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_SUPP | SLJIT_MEM_PRE, char1_reg, SLJIT_MEM1(TMP1), IN_UCHARS(1)) == SLJIT_SUCCESS)
+  opt_type = 2;
+
+sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0);
 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
 
-OP1(SLJIT_MOV, TMP3, 0, LCC_TABLE, 0);
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, CHAR1, 0);
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, CHAR2, 0);
-OP1(SLJIT_MOV, LCC_TABLE, 0, SLJIT_IMM, common->lcc);
-OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1));
-OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, char1_reg, 0);
+
+if (char2_reg == STACK_TOP)
+  {
+  OP1(SLJIT_MOV, TMP3, 0, char2_reg, 0);
+  OP1(SLJIT_MOV, RETURN_ADDR, 0, lcc_table, 0);
+  }
+
+OP1(SLJIT_MOV, lcc_table, 0, SLJIT_IMM, common->lcc);
+
+if (opt_type == 1)
+  {
+  label = LABEL();
+  sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_POST, char1_reg, SLJIT_MEM1(TMP1), IN_UCHARS(1));
+  sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_POST, char2_reg, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
+  }
+else if (opt_type == 2)
+  {
+  OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1));
+  OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+
+  label = LABEL();
+  sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_PRE, char1_reg, SLJIT_MEM1(TMP1), IN_UCHARS(1));
+  sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_PRE, char2_reg, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
+  }
+else
+  {
+  label = LABEL();
+  OP1(MOV_UCHAR, char1_reg, 0, SLJIT_MEM1(TMP1), 0);
+  OP1(MOV_UCHAR, char2_reg, 0, SLJIT_MEM1(STR_PTR), 0);
+  OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1));
+  }
 
-label = LABEL();
-OP1(MOVU_UCHAR, CHAR1, 0, SLJIT_MEM1(TMP1), IN_UCHARS(1));
-OP1(MOVU_UCHAR, CHAR2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
 #ifndef COMPILE_PCRE8
-jump = CMP(SLJIT_GREATER, CHAR1, 0, SLJIT_IMM, 255);
+jump = CMP(SLJIT_GREATER, char1_reg, 0, SLJIT_IMM, 255);
 #endif
-OP1(SLJIT_MOV_U8, CHAR1, 0, SLJIT_MEM2(LCC_TABLE, CHAR1), 0);
+OP1(SLJIT_MOV_U8, char1_reg, 0, SLJIT_MEM2(lcc_table, char1_reg), 0);
 #ifndef COMPILE_PCRE8
 JUMPHERE(jump);
-jump = CMP(SLJIT_GREATER, CHAR2, 0, SLJIT_IMM, 255);
+jump = CMP(SLJIT_GREATER, char2_reg, 0, SLJIT_IMM, 255);
 #endif
-OP1(SLJIT_MOV_U8, CHAR2, 0, SLJIT_MEM2(LCC_TABLE, CHAR2), 0);
+OP1(SLJIT_MOV_U8, char2_reg, 0, SLJIT_MEM2(lcc_table, char2_reg), 0);
 #ifndef COMPILE_PCRE8
 JUMPHERE(jump);
 #endif
-jump = CMP(SLJIT_NOT_EQUAL, CHAR1, 0, CHAR2, 0);
+
+if (opt_type == 0)
+  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
+
+jump = CMP(SLJIT_NOT_EQUAL, char1_reg, 0, char2_reg, 0);
 OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1));
 JUMPTO(SLJIT_NOT_ZERO, label);
 
 JUMPHERE(jump);
-OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
-OP1(SLJIT_MOV, LCC_TABLE, 0, TMP3, 0);
-OP1(SLJIT_MOV, CHAR1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
-OP1(SLJIT_MOV, CHAR2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1);
-sljit_emit_fast_return(compiler, RETURN_ADDR, 0);
-}
+OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
+
+if (opt_type == 2)
+  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
 
-#undef LCC_TABLE
-#undef CHAR1
-#undef CHAR2
+if (char2_reg == STACK_TOP)
+  {
+  OP1(SLJIT_MOV, char2_reg, 0, TMP3, 0);
+  OP1(SLJIT_MOV, lcc_table, 0, RETURN_ADDR, 0);
+  }
+
+OP1(SLJIT_MOV, char1_reg, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1);
+sljit_emit_fast_return(compiler, TMP1, 0);
+}
 
 #if defined SUPPORT_UTF && defined SUPPORT_UCP
 
-static const pcre_uchar * SLJIT_CALL do_utf_caselesscmp(pcre_uchar *src1, jit_arguments *args, pcre_uchar *end1)
+static const pcre_uchar * SLJIT_FUNC do_utf_caselesscmp(pcre_uchar *src1, pcre_uchar *src2, pcre_uchar *end1, pcre_uchar *end2)
 {
 /* This function would be ineffective to do in JIT level. */
 sljit_u32 c1, c2;
-const pcre_uchar *src2 = args->uchar_ptr;
-const pcre_uchar *end2 = args->end;
 const ucd_record *ur;
 const sljit_u32 *pp;
 
@@ -6776,32 +6939,37 @@ else
 #if defined SUPPORT_UTF && defined SUPPORT_UCP
 if (common->utf && *cc == OP_REFI)
   {
-  SLJIT_ASSERT(TMP1 == SLJIT_R0 && STACK_TOP == SLJIT_R1 && TMP2 == SLJIT_R2);
+  SLJIT_ASSERT(TMP1 == SLJIT_R0 && STACK_TOP == SLJIT_R1);
   if (ref)
-    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1));
+    OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1));
   else
-    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw));
+    OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw));
 
   if (withchecks)
-    jump = CMP(SLJIT_EQUAL, TMP1, 0, TMP2, 0);
+    jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_R2, 0);
 
-  /* Needed to save important temporary registers. */
+  /* No free saved registers so save data on stack. */
   OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0);
-  OP1(SLJIT_MOV, SLJIT_R1, 0, ARGUMENTS, 0);
-  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_R1), SLJIT_OFFSETOF(jit_arguments, uchar_ptr), STR_PTR, 0);
-  sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_utf_caselesscmp));
+  OP1(SLJIT_MOV, SLJIT_R1, 0, STR_PTR, 0);
+  OP1(SLJIT_MOV, SLJIT_R3, 0, STR_END, 0);
+  sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW) | SLJIT_ARG3(SW) | SLJIT_ARG4(SW), SLJIT_IMM, SLJIT_FUNC_OFFSET(do_utf_caselesscmp));
   OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
+  OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_RETURN_REG, 0);
+
   if (common->mode == JIT_COMPILE)
     add_jump(compiler, backtracks, CMP(SLJIT_LESS_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1));
   else
     {
-    add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0));
-    nopartial = CMP(SLJIT_NOT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1);
+    OP2(SLJIT_SUB | SLJIT_SET_Z | SLJIT_SET_LESS, SLJIT_UNUSED, 0, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1);
+
+    add_jump(compiler, backtracks, JUMP(SLJIT_LESS));
+
+    nopartial = JUMP(SLJIT_NOT_EQUAL);
+    OP1(SLJIT_MOV, STR_PTR, 0, STR_END, 0);
     check_partial(common, FALSE);
     add_jump(compiler, backtracks, JUMP(SLJIT_JUMP));
     JUMPHERE(nopartial);
     }
-  OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_RETURN_REG, 0);
   }
 else
 #endif /* SUPPORT_UTF && SUPPORT_UCP */
@@ -7125,7 +7293,7 @@ add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IM
 return cc + 1 + LINK_SIZE;
 }
 
-static int SLJIT_CALL do_callout(struct jit_arguments *arguments, PUBL(callout_block) *callout_block, pcre_uchar **jit_ovector)
+static sljit_s32 SLJIT_FUNC do_callout(struct jit_arguments *arguments, PUBL(callout_block) *callout_block, pcre_uchar **jit_ovector)
 {
 const pcre_uchar *begin = arguments->begin;
 int *offset_vector = arguments->offsets;
@@ -7207,18 +7375,17 @@ OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0);
 /* SLJIT_R0 = arguments */
 OP1(SLJIT_MOV, SLJIT_R1, 0, STACK_TOP, 0);
 GET_LOCAL_BASE(SLJIT_R2, 0, OVECTOR_START);
-sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_callout));
-OP1(SLJIT_MOV_S32, SLJIT_RETURN_REG, 0, SLJIT_RETURN_REG, 0);
+sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(S32) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW) | SLJIT_ARG3(SW), SLJIT_IMM, SLJIT_FUNC_OFFSET(do_callout));
 OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
 free_stack(common, CALLOUT_ARG_SIZE / sizeof(sljit_sw));
 
 /* Check return value. */
-OP2(SLJIT_SUB | SLJIT_SET_Z | SLJIT_SET_SIG_GREATER, SLJIT_UNUSED, 0, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0);
-add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_SIG_GREATER));
+OP2(SLJIT_SUB32 | SLJIT_SET_Z | SLJIT_SET_SIG_GREATER, SLJIT_UNUSED, 0, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0);
+add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_SIG_GREATER32));
 if (common->forced_quit_label == NULL)
-  add_jump(compiler, &common->forced_quit, JUMP(SLJIT_NOT_EQUAL) /* SIG_LESS */);
+  add_jump(compiler, &common->forced_quit, JUMP(SLJIT_NOT_EQUAL32) /* SIG_LESS */);
 else
-  JUMPTO(SLJIT_NOT_EQUAL /* SIG_LESS */, common->forced_quit_label);
+  JUMPTO(SLJIT_NOT_EQUAL32 /* SIG_LESS */, common->forced_quit_label);
 return cc + 2 + 2 * LINK_SIZE;
 }
 
@@ -10439,11 +10606,11 @@ if (opcode == OP_SKIP_ARG)
   OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr);
   OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0);
   OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_IMM, (sljit_sw)(current->cc + 2));
-  sljit_emit_ijump(compiler, SLJIT_CALL2, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_search_mark));
+  sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW), SLJIT_IMM, SLJIT_FUNC_OFFSET(do_search_mark));
   OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
 
   OP1(SLJIT_MOV, STR_PTR, 0, TMP1, 0);
-  add_jump(compiler, &common->reset_match, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, -1));
+  add_jump(compiler, &common->reset_match, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0));
   return;
   }
 
@@ -11031,7 +11198,7 @@ if (!compiler)
 common->compiler = compiler;
 
 /* Main pcre_jit_exec entry. */
-sljit_emit_enter(compiler, 0, 1, 5, 5, 0, 0, private_data_size);
+sljit_emit_enter(compiler, 0, SLJIT_ARG1(SW), 5, 5, 0, 0, private_data_size);
 
 /* Register init. */
 reset_ovector(common, (re->top_bracket + 1) * 2);
@@ -11044,8 +11211,8 @@ OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str))
 OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, end));
 OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack));
 OP1(SLJIT_MOV_U32, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, limit_match));
-OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_stack, base));
-OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_stack, limit));
+OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_stack, end));
+OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_stack, start));
 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LIMIT_MATCH, TMP1, 0);
 
@@ -11251,20 +11418,22 @@ common->quit_label = quit_label;
 set_jumps(common->stackalloc, LABEL());
 /* RETURN_ADDR is not a saved register. */
 sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0);
-OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP2, 0);
-OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
-OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack));
-OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top), STACK_TOP, 0);
-OP2(SLJIT_SUB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit), SLJIT_IMM, STACK_GROWTH_RATE);
 
-sljit_emit_ijump(compiler, SLJIT_CALL2, SLJIT_IMM, SLJIT_FUNC_OFFSET(sljit_stack_resize));
-jump = CMP(SLJIT_NOT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0);
-OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
-OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack));
-OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top));
-OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit));
-OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1);
-sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0);
+SLJIT_ASSERT(TMP1 == SLJIT_R0 && STACK_TOP == SLJIT_R1);
+
+OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, STACK_TOP, 0);
+OP1(SLJIT_MOV, SLJIT_R0, 0, ARGUMENTS, 0);
+OP2(SLJIT_SUB, SLJIT_R1, 0, STACK_LIMIT, 0, SLJIT_IMM, STACK_GROWTH_RATE);
+OP1(SLJIT_MOV, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, stack));
+OP1(SLJIT_MOV, STACK_LIMIT, 0, TMP2, 0);
+
+sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW), SLJIT_IMM, SLJIT_FUNC_OFFSET(sljit_stack_resize));
+jump = CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0);
+OP1(SLJIT_MOV, TMP2, 0, STACK_LIMIT, 0);
+OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_RETURN_REG, 0);
+OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
+OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1);
+sljit_emit_fast_return(compiler, TMP1, 0);
 
 /* Allocation failed. */
 JUMPHERE(jump);
@@ -11409,9 +11578,9 @@ union {
 sljit_u8 local_space[MACHINE_STACK_SIZE];
 struct sljit_stack local_stack;
 
-local_stack.max_limit = local_space;
-local_stack.limit = local_space;
-local_stack.base = local_space + MACHINE_STACK_SIZE;
+local_stack.min_start = local_space;
+local_stack.start = local_space;
+local_stack.end = local_space + MACHINE_STACK_SIZE;
 local_stack.top = local_space + MACHINE_STACK_SIZE;
 arguments->stack = &local_stack;
 convert_executable_func.executable_func = executable_func;
@@ -11536,7 +11705,7 @@ if ((options & PCRE_PARTIAL_HARD) != 0)
 else if ((options & PCRE_PARTIAL_SOFT) != 0)
   mode = JIT_PARTIAL_SOFT_COMPILE;
 
-if (functions->executable_funcs[mode] == NULL)
+if (functions == NULL || functions->executable_funcs[mode] == NULL)
   return PCRE_ERROR_JIT_BADOPTION;
 
 /* Sanity checks should be handled by pcre_exec. */
diff --git a/erts/emulator/pcre/pcre_latin_1_table.c b/erts/emulator/pcre/pcre_latin_1_table.c
index aa29275a94..d6cf38fa3b 100644
--- a/erts/emulator/pcre/pcre_latin_1_table.c
+++ b/erts/emulator/pcre/pcre_latin_1_table.c
@@ -14,6 +14,7 @@ Pulling in the header ensures that the array gets flagged as "someone
 outside this compilation unit might reference this" and so it will always
 be supplied to the linker. */
 /* %ExternalCopyright% */
+
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif
@@ -120,7 +121,7 @@ print, punct, and cntrl. Other classes are built from combinations. */
 
   0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
   0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0x07,
-  0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,
+  0x00,0x00,0x00,0x00,0x00,0x04,0x20,0x04,
   0x00,0x00,0x00,0x80,0xff,0xff,0x7f,0xff,
 
   0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03,
-- 
cgit v1.2.3


From ed3e37d61b7917a380371dc7ebce74e80fb32dfe Mon Sep 17 00:00:00 2001
From: Simon Cornish <zl9d97p02@sneakemail.com>
Date: Fri, 3 Aug 2018 11:50:19 -0700
Subject: Test #Ref ordering in lists and ets

---
 erts/emulator/test/ref_SUITE.erl | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/ref_SUITE.erl b/erts/emulator/test/ref_SUITE.erl
index 5f519d522e..74df857c65 100644
--- a/erts/emulator/test/ref_SUITE.erl
+++ b/erts/emulator/test/ref_SUITE.erl
@@ -22,6 +22,7 @@
 
 -export([all/0, suite/0]).
 -export([wrap_1/1]).
+-export([compare_list/1, compare_ets/1]).
 
 -export([loop_ref/1]).
 
@@ -32,7 +33,7 @@ suite() ->
      {timetrap, {minutes, 2}}].
 
 all() -> 
-    [wrap_1].
+    [wrap_1, compare_list, compare_ets].
 
 %% Check that refs don't wrap around easily.
 wrap_1(Config) when is_list(Config) ->
@@ -53,3 +54,28 @@ loop_ref(Parent) ->
 loop_ref(R, R, _) -> ok;
 loop_ref(R0, _, N) ->
     loop_ref(R0, make_ref(), N+1).
+
+%% Check that ref ordering works
+compare_list(Config) when is_list(Config) ->
+    %% Although this test uses external refs, it would apply the same to plain refs
+    ExtRef1 = <<131,114,0,3,100,0,3,110,64,98,3, 0,0,173,156, 0,216,0,4, 0,0,0,0>>,
+    ExtRef2 = <<131,114,0,3,100,0,3,110,64,98,3, 0,1,31,27,   129,4,0,1, 0,0,0,0>>,
+
+    Ref1 = binary_to_term(ExtRef1), %% #Ref<n@b.0.14155780.44444>
+    Ref2 = binary_to_term(ExtRef2), %% #Ref<n@b.0.2164523009.73499>
+    OrderedList = [Ref1, Ref2],
+    OrderedList = lists:sort(OrderedList),
+    ok.
+
+%% This is the scarier case since it makes terms "invisible" in ets or Mnesia
+%% (the underlying fault cause is the same as compare_list/1)
+compare_ets(Config) when is_list(Config) ->
+    W2s = [610350147,899574699,2994196869,686384822,2397690439, 923302211],
+    ExtRefBase = <<131,114,0,3,100,0,3,110,64,98,3>>,
+    ExtRefs = [<<ExtRefBase/binary, 1:32, W2:32, 0:32>> || W2 <- W2s],
+    Refs = [binary_to_term(Bin) || Bin <- ExtRefs],
+
+    Ets = ets:new(refbug, [ordered_set]),
+    ets:insert(Ets, [{Ref,Ref} || Ref <- Refs]),
+    0 = length([R || R <- ets:tab2list(Ets), ets:lookup(Ets, element(1,R)) == []]),
+    ok.
-- 
cgit v1.2.3


From a4492d67fed157533a362f932c6ca8bbdf3d1bec Mon Sep 17 00:00:00 2001
From: Simon Cornish <zl9d97p02@sneakemail.com>
Date: Fri, 3 Aug 2018 11:54:01 -0700
Subject: Fixed #Ref ordering bug

---
 erts/emulator/beam/utils.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index d90c282c7e..a9447e3ce3 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3429,7 +3429,7 @@ tailrecur_ne:
 		ASSERT(alen == blen);
 		for (i = (Sint) alen - 1; i >= 0; i--)
 		    if (anum[i] != bnum[i])
-			RETURN_NEQ((Sint32) (anum[i] - bnum[i]));
+			RETURN_NEQ(anum[i] < bnum[i] ? -1 : 1);
 		goto pop_next;
 	    case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE):
 		if (is_internal_ref(b)) {
-- 
cgit v1.2.3


From 83ce2a8e8da4aab4578f589e6af6e5c93bcdaebe Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 8 Aug 2018 20:04:39 +0200
Subject: Fix caching of NIF environment when executing dirty

---
 erts/emulator/beam/erl_nif.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 4e479c26ef..f8e964e2cf 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -409,8 +409,18 @@ static void full_flush_env(ErlNifEnv* env)
 static void full_cache_env(ErlNifEnv* env)
 {    
 #ifdef ERTS_DIRTY_SCHEDULERS
-    if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)
+    if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
 	erts_cache_dirty_shadow_proc(env->proc);
+        /*
+         * If shadow proc had heap fragments when flushed
+         * those have now been moved to the real proc.
+         * Ensure heap pointers do not point into a heap
+         * fragment on real proc...
+         */
+        ASSERT(!env->proc->mbuf);
+	env->hp_end = HEAP_LIMIT(env->proc);
+	env->hp = HEAP_TOP(env->proc);
+    }
 #endif
     cache_env(env);
 }
-- 
cgit v1.2.3


From 1d1f29eb614ce75012f9136d185f4dbcccc8650b Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 10 Aug 2018 11:50:07 +0200
Subject: erts: Delete fd from poll-set when closing fd_driver port

---
 erts/emulator/sys/unix/sys_drivers.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c
index d0498f0bd5..816bdea9c5 100644
--- a/erts/emulator/sys/unix/sys_drivers.c
+++ b/erts/emulator/sys/unix/sys_drivers.c
@@ -1000,7 +1000,7 @@ static void clear_fd_data(ErtsSysFdData *fdd)
 
 static void nbio_stop_fd(ErlDrvPort prt, ErtsSysFdData *fdd)
 {
-    driver_select(prt, abs(fdd->fd), DO_READ|DO_WRITE, 0);
+    driver_select(prt, abs(fdd->fd), ERL_DRV_USE_NO_CALLBACK|DO_READ|DO_WRITE, 0);
     clear_fd_data(fdd);
     SET_BLOCKING(abs(fdd->fd));
 
-- 
cgit v1.2.3


From 1d1a604d66806fb622d03a7c82f0316a5b15ba4b Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 15 Aug 2018 18:04:15 +0200
Subject: Fix incoming suspend monitor down

An incoming suspend monitor down wasn't handled correct when the
local monitor half had been removed with an emulator crash as result.
---
 erts/emulator/beam/erl_proc_sig_queue.c |  2 +-
 erts/emulator/test/trace_SUITE.erl      | 19 +++++++++++++++++--
 2 files changed, 18 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index d6d22677e7..6af1236145 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -3146,8 +3146,8 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep,
                     erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p),
                                              &mdp->origin);
                     omon = &mdp->origin;
+                    remove_nm_sig(c_p, sig, next_nm_sig);
                 }
-                remove_nm_sig(c_p, sig, next_nm_sig);
                 break;
             default:
                 ERTS_INTERNAL_ERROR("invalid monitor type");
diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl
index 979b3185a5..c2d5cd7023 100644
--- a/erts/emulator/test/trace_SUITE.erl
+++ b/erts/emulator/test/trace_SUITE.erl
@@ -39,7 +39,7 @@
 	 system_monitor_large_heap_1/1, system_monitor_large_heap_2/1,
 	 system_monitor_long_schedule/1,
 	 bad_flag/1, trace_delivered/1, trap_exit_self_receive/1,
-         trace_info_badarg/1]).
+         trace_info_badarg/1, erl_704/1]).
 
 -include_lib("common_test/include/ct.hrl").
 
@@ -63,7 +63,7 @@ all() ->
      system_monitor_long_gc_2, system_monitor_large_heap_1,
      system_monitor_long_schedule,
      system_monitor_large_heap_2, bad_flag, trace_delivered,
-     trap_exit_self_receive, trace_info_badarg].
+     trap_exit_self_receive, trace_info_badarg, erl_704].
 
 init_per_testcase(_Case, Config) ->
     [{receiver,spawn(fun receiver/0)}|Config].
@@ -1739,6 +1739,21 @@ trace_info_badarg(Config) when is_list(Config) ->
     catch erlang:trace_info({a,b,c},d),
     ok.
 
+%% An incoming suspend monitor down wasn't handled
+%% correct when the local monitor half had been
+%% removed with an emulator crash as result.
+erl_704(Config) ->
+    erl_704_test(100).
+
+erl_704_test(0) ->
+    ok;
+erl_704_test(N) ->
+    P = spawn(fun () -> receive infinity -> ok end end),
+    erlang:suspend_process(P),
+    exit(P, kill),
+    (catch erlang:resume_process(P)),
+    erl_704_test(N-1).
+
 drop_trace_until_down(Proc, Mon) ->
     drop_trace_until_down(Proc, Mon, false, 0, 0).
 
-- 
cgit v1.2.3


From cbce1139aa6bd4360e30e85785fe70d2835e3e66 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 27 Aug 2018 16:23:13 +0200
Subject: Fix missing 'in' trace events during 'running' trace

'in' trace events could be lost when a process had to be
rescheduled on another scheduler type (normal <-> dirty).
---
 erts/emulator/beam/erl_process.c | 89 +++++++++++++++++++++++++---------------
 1 file changed, 56 insertions(+), 33 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 8784eb5a63..7a9ef3c1de 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9079,6 +9079,9 @@ unlock_lock_rq(int pre_free, void *vrq)
 }
 
 
+static void trace_schedule_in(Process *p, erts_aint32_t state);
+static void trace_schedule_out(Process *p, erts_aint32_t state);
+
 /*
  * schedule() is called from BEAM (process_main()) or HiPE
  * (hipe_mode_switch()) when the current process is to be
@@ -9184,22 +9187,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	state = erts_atomic32_read_nob(&p->state);
 
-	if (IS_TRACED(p)) {
-	    if (IS_TRACED_FL(p, F_TRACE_CALLS) && !(state & ERTS_PSFLG_FREE))
-		erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_OUT);
-	    if ((state & (ERTS_PSFLG_FREE|ERTS_PSFLG_EXITING)) == ERTS_PSFLG_EXITING) {
-		if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT))
-		    trace_sched(p, ERTS_PROC_LOCK_MAIN,
-                                ((state & ERTS_PSFLG_FREE)
-                                 ? am_out_exited
-                                 : am_out_exiting));
-	    }
-	    else {
-		if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED) ||
-                    ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS))
-		    trace_sched(p, ERTS_PROC_LOCK_MAIN, am_out);
-	    }
-	}
+	if (IS_TRACED(p))
+            trace_schedule_out(p, state);
 
 	erts_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
 
@@ -9610,6 +9599,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		/* Migrate to dirty scheduler... */
 	    sunlock_sched_out_proc:
 		erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+                if (IS_TRACED(p))
+                    trace_schedule_in(p, state);
 		goto sched_out_proc;
 	    }
 	}
@@ -9643,23 +9634,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 
-        /* Clear tracer if it has been removed */
-	if (IS_TRACED(p) && erts_is_tracer_proc_enabled(
-                p, ERTS_PROC_LOCK_MAIN, &p->common)) {
-
-	    if (state & ERTS_PSFLG_EXITING) {
-		if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT))
-		    trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in_exiting);
-	    }
-	    else {
-		if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED) ||
-                    ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS))
-		    trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in);
-	    }
-	    if (IS_TRACED_FL(p, F_TRACE_CALLS)) {
-		erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_IN);
-	    }
-	}
+        if (IS_TRACED(p))
+            trace_schedule_in(p, state);
 
         if (is_normal_sched) {
             if (state & ERTS_PSFLG_RUNNING_SYS) {
@@ -9823,6 +9799,53 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
     }
 }
 
+static void
+trace_schedule_in(Process *p, erts_aint32_t state)
+{
+    ASSERT(IS_TRACED(p));
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCK_MAIN);
+
+    /* Clear tracer if it has been removed */
+    if (erts_is_tracer_proc_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common)) {
+
+        if (state & ERTS_PSFLG_EXITING) {
+            if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT))
+                trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in_exiting);
+        }
+        else {
+            if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED) ||
+                ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS))
+                trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in);
+        }
+        if (IS_TRACED_FL(p, F_TRACE_CALLS))
+            erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_IN);
+    }
+
+}
+
+static void
+trace_schedule_out(Process *p, erts_aint32_t state)
+{
+    ASSERT(IS_TRACED(p));
+    ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCK_MAIN);
+    
+    if (IS_TRACED_FL(p, F_TRACE_CALLS) && !(state & ERTS_PSFLG_FREE))
+        erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_OUT);
+
+    if ((state & (ERTS_PSFLG_FREE|ERTS_PSFLG_EXITING)) == ERTS_PSFLG_EXITING) {
+        if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT))
+            trace_sched(p, ERTS_PROC_LOCK_MAIN,
+                        ((state & ERTS_PSFLG_FREE)
+                         ? am_out_exited
+                         : am_out_exiting));
+    }
+    else {
+        if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED) ||
+            ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS))
+            trace_sched(p, ERTS_PROC_LOCK_MAIN, am_out);
+    }
+}
+
 static int
 notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st,
 			 Eterm st_result, int normal_sched)
-- 
cgit v1.2.3


From 203ee6dcbd57c2fdbd9b9e09f0c42f7abf04dbd8 Mon Sep 17 00:00:00 2001
From: Serge Aleynikov <saleyn@gmail.com>
Date: Fri, 10 Aug 2018 09:54:01 -0400
Subject: Fix bug in compact representation of float_to_list/2

---
 erts/emulator/sys/common/erl_sys_common_misc.c | 3 ++-
 erts/emulator/test/num_bif_SUITE.erl           | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c
index 2541ab5d31..d34e1a9ec0 100644
--- a/erts/emulator/sys/common/erl_sys_common_misc.c
+++ b/erts/emulator/sys/common/erl_sys_common_misc.c
@@ -176,6 +176,7 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
     double af;
     Uint64 int_part, frac_part;
     int neg;
+    int  has_decimals = decimals != 0;
     char *p = buffer;
 
     if (decimals < 0)
@@ -257,7 +258,7 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
     }
 
     /* Delete trailing zeroes */
-    if (compact)
+    if (compact && has_decimals)
         p = find_first_trailing_zero(p);
     *p = '\0';
     return p - buffer;
diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl
index 700734cd0b..f15217814a 100644
--- a/erts/emulator/test/num_bif_SUITE.erl
+++ b/erts/emulator/test/num_bif_SUITE.erl
@@ -161,6 +161,7 @@ t_float_to_string(Config) when is_list(Config) ->
     test_fts("1.000",1.0,   [{decimals,   3}]),
     test_fts("1.0",1.0, [{decimals, 1}]),
     test_fts("1.0",1.0, [{decimals, 3}, compact]),
+    test_fts("10",10.0, [{decimals, 0}, compact]),
     test_fts("1.12",1.123, [{decimals, 2}]),
     test_fts("1.123",1.123, [{decimals, 3}]),
     test_fts("1.123",1.123, [{decimals, 3}, compact]),
-- 
cgit v1.2.3


From 93ad08d8c6d6a9875a10b33633aca52de5d3c59b Mon Sep 17 00:00:00 2001
From: Maxim Fedorov <dane@whatsapp.com>
Date: Tue, 28 Aug 2018 16:11:09 -0700
Subject: Fix an endless rescheduling loop when a process is executing
 process_info(self(), ...)

It is possible that a process has to yield before completing process_info BIF when it runs out of reductions. If this BIF is called by the process itself, it does not send a signal but executes in the context of a process. If it has to yield, it turns F_LOCAL_SIGS_ONLY flag on, which means new signals won't be fetched from the outer message queue.
When the same process needs to execute dirty system code (e.g. dirty GC) it has to be run on a dirty scheduler. However signals enqueued into outer queue cause it to be rescheduled on a normal scheduler. F_LOCAL_SIGS_ONLY prevent outer queue signals delivery, creating an endless rescheduling loop.

This commit disengages F_LOCAL_SIG_ONLY if process needs to execute dirty code in order to complete signal delivery and allow process to be moved to dirty run queue.
---
 erts/emulator/beam/erl_process.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 7a9ef3c1de..d5bd17ff3e 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9641,7 +9641,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
             if (state & ERTS_PSFLG_RUNNING_SYS) {
                 if (state & (ERTS_PSFLG_SIG_Q|ERTS_PSFLG_SIG_IN_Q)) {
                     int local_only = (!!(p->flags & F_LOCAL_SIGS_ONLY)
-                                      & !(state & ERTS_PSFLG_SUSPENDED));
+                                      & !(state & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLGS_DIRTY_WORK)));
                     if (!local_only | !!(state & ERTS_PSFLG_SIG_Q)) {
                         int sig_reds;
                         /*
-- 
cgit v1.2.3


From 5801fcb2b36e04c433dcf0b90a8c47b86e34fc07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sun, 2 Sep 2018 08:42:51 +0200
Subject: ops.tab: Fix potentially unsafe optimization of raise/2

The operands for the raise/2 instruction are almost always in x(2) and
x(1). Therefore the loader translates the raise/2 instruction to an
i_raise/0 instruction which uses the values in x(2) and x(1). If the
operands happens to be in other registers, the loader inserts move/2
instruction to move them to x(2) and x(1).

The problem is that x(3) is used as a temporary register when
generating the move/2 instructions. That is unsafe if the
Value operand for raise/2 is x(3).

Thus:

    raise x(0) x(3)

will be translated to:

    move x(0) x(3)
    move x(3) x(1)
    move x(3) x(2)
    i_raise

The Trace will be written to both x(2) and x(1).

The current compiler will never use x(3) for the Value operand,
so there is no need to patch previous releases. But a future compiler
version might allocate registers differently.
---
 erts/emulator/beam/ops.tab | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index c51e4ef784..e76d896ffc 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -244,7 +244,7 @@ if_end
 # Optimize for that case.
 raise x==2 x==1 => i_raise
 raise Trace=y Value=y => move Trace x=2 | move Value x=1 | i_raise
-raise Trace Value => move Trace x=3 | move Value x=1 | move x=3 x=2 | i_raise
+raise Trace Value => move Trace x | move Value x=1 | move x x=2 | i_raise
 
 i_raise
 
-- 
cgit v1.2.3


From 4253f875c73c577a768847639e9d2411f935d4d2 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 3 Sep 2018 20:05:32 +0200
Subject: erts: Fix ets memstat false leak of FixedDeletion

causing erlang:memory to report too much ets memory.
---
 erts/emulator/beam/erl_db_hash.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index b988a19cf4..b648965885 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -162,6 +162,7 @@ static ERTS_INLINE int link_fixdel(DbTableHash* tb,
         if (NFIXED(tb) <= fixated_by_me) {
             erts_db_free(ERTS_ALC_T_DB_FIX_DEL, (DbTable*)tb,
                          fixd, sizeof(FixedDeletion));
+            ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion));
             return 0; /* raced by unfixer */
         }
         exp_next = was_next;
-- 
cgit v1.2.3


From 0e929fc8b8ad2bf3992ebec49c036605eb722a3b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 3 Sep 2018 20:24:24 +0200
Subject: erts: Refactor ets FixedDeletion allocations

---
 erts/emulator/beam/erl_db_hash.c | 46 +++++++++++++++++++++-------------------
 1 file changed, 24 insertions(+), 22 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index b648965885..752d3ae3a8 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -150,6 +150,22 @@ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval)
 }
 
 
+static ERTS_INLINE FixedDeletion* alloc_fixdel(DbTableHash* tb)
+{
+    FixedDeletion* fixd = (FixedDeletion*) erts_db_alloc(ERTS_ALC_T_DB_FIX_DEL,
+                                                         (DbTable *) tb,
+                                                         sizeof(FixedDeletion));
+    ERTS_ETS_MISC_MEM_ADD(sizeof(FixedDeletion));
+    return fixd;
+}
+
+static ERTS_INLINE void free_fixdel(DbTableHash* tb, FixedDeletion* fixd)
+{
+    erts_db_free(ERTS_ALC_T_DB_FIX_DEL, (DbTable*)tb,
+                 fixd, sizeof(FixedDeletion));
+    ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion));
+}
+
 static ERTS_INLINE int link_fixdel(DbTableHash* tb,
                                    FixedDeletion* fixd,
                                    erts_aint_t fixated_by_me)
@@ -160,9 +176,7 @@ static ERTS_INLINE int link_fixdel(DbTableHash* tb,
     was_next = erts_atomic_read_acqb(&tb->fixdel);
     do { /* Lockless atomic insertion in linked list: */
         if (NFIXED(tb) <= fixated_by_me) {
-            erts_db_free(ERTS_ALC_T_DB_FIX_DEL, (DbTable*)tb,
-                         fixd, sizeof(FixedDeletion));
-            ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion));
+            free_fixdel(tb, fixd);
             return 0; /* raced by unfixer */
         }
         exp_next = was_next;
@@ -181,10 +195,7 @@ static ERTS_INLINE int link_fixdel(DbTableHash* tb,
 static int add_fixed_deletion(DbTableHash* tb, int ix,
                               erts_aint_t fixated_by_me)
 {
-    FixedDeletion* fixd = (FixedDeletion*) erts_db_alloc(ERTS_ALC_T_DB_FIX_DEL,
-							 (DbTable *) tb,
-							 sizeof(FixedDeletion));
-    ERTS_ETS_MISC_MEM_ADD(sizeof(FixedDeletion));
+    FixedDeletion* fixd = alloc_fixdel(tb);
     fixd->slot = ix;
     fixd->all = 0;
     return link_fixdel(tb, fixd, fixated_by_me);
@@ -638,11 +649,7 @@ restart:
 
         free_me = fixdel;
         fixdel = fixdel->next;
-        erts_db_free(ERTS_ALC_T_DB_FIX_DEL,
-                     (DbTable *) tb,
-                     (void *) free_me,
-                     sizeof(FixedDeletion));
-        ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion));
+        free_fixdel(tb, free_me);
         work++;
     }
 
@@ -2339,11 +2346,10 @@ static SWord db_mark_all_deleted_hash(DbTable *tbl, SWord reds)
     }
     else {
         /* First call */
-        fixdel = erts_db_alloc(ERTS_ALC_T_DB_FIX_DEL,
-                               (DbTable *) tb,
-                               sizeof(FixedDeletion));
-        ERTS_ETS_MISC_MEM_ADD(sizeof(FixedDeletion));
-        link_fixdel(tb, fixdel, 0);
+        int ok;
+        fixdel = alloc_fixdel(tb);
+        ok = link_fixdel(tb, fixdel, 0);
+        ASSERT(ok); (void)ok;
         i = 0;
     }
 
@@ -2445,11 +2451,7 @@ static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds)
 	FixedDeletion *fx = fixdel;
 
 	fixdel = fx->next;
-	erts_db_free(ERTS_ALC_T_DB_FIX_DEL,
-		     (DbTable *) tb,
-		     (void *) fx,
-		     sizeof(FixedDeletion));
-	ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion));
+        free_fixdel(tb, fx);
 	if (--reds < 0) {
 	    erts_atomic_set_relb(&tb->fixdel, (erts_aint_t)fixdel);
 	    return reds;		/* Not done */
-- 
cgit v1.2.3


From 64853dc28ce838583e35d5fefb0604933b6e98f9 Mon Sep 17 00:00:00 2001
From: Raimo Niskanen <raimo@erlang.org>
Date: Fri, 6 Jul 2018 12:55:10 +0200
Subject: Implement socket option recvtos and friends

Implement socket options recvtclass, recvtos, recvttl and pktoptions.

Document the implemented socket options, new types and message formats.

The options recvtclass, recvtos and recvttl are boolean options that
when activated (true) for a socket will cause ancillary data to be
received through recvmsg().  That is for packet oriented sockets
(UDP and SCTP).

The required options for this feature were recvtclass and recvtos,
and recvttl was only added to test that the ancillary data parsing
handled multiple data items in one message correctly.

These options does not work on Windows since ancillary data
is not handled by the Winsock2 API.

For stream sockets (TCP) there is no clear connection between
a received packet and what is returned when reading data from
the socket, so recvmsg() is not useful.  It is possible to get
the same ancillary data through a getsockopt() call with
the IPv6 socket option IPV6_PKTOPTIONS, on Linux named
IPV6_2292PKTOPTIONS after the now obsoleted RFC where it originated.
(unfortunately RFC 3542 that obsoletes it explicitly undefines
 this way to get packet ancillary data from a stream socket)

Linux also has got a way to get packet ancillary data for IPv4
TCP sockets through a getsockopt() call with IP_PKTOPTIONS,
which appears to be Linux specific.

This implementation uses a flag field in the inet_drv.c socket
internal data that records if any setsockopt() call with recvtclass,
recvtos or recvttl (IPV6_RECVTCLASS, IP_RECVTOS or IP_RECVTTL)
has been activated.  If so recvmsg() is used instead of recvfrom().

Ancillary data is delivered to the application by a new return
tuple format from gen_udp:recv/2,3 containing a list of
ancillary data tuples [{tclass,TCLASS} | {tos,TOS} | {ttl,TTL}],
as returned by recvmsg().  For a socket in active mode a new
message format, containing the ancillary data list, delivers
the data in the same way.

For gen_sctp the ancillary data is delivered in the same way,
except that the gen_sctp return tuple format already contained
an ancillary data list so there are just more possible elements
when using these socket options.  Note that the active mode
message format has got an extra tuple level for the ancillary
data compared to what is now implemented gen_udp.
The gen_sctp active mode format was considered to be the odd one
- now all tuples containing ancillary data are flat,
except for gen_sctp active mode.

Note that testing has not shown that Linux SCTP sockets deliver
any ancillary data for these socket options, so it is probably
not implemented yet.  Remains to be seen what FreeBSD does...

For gen_tcp inet:getopts([pktoptions]) will deliver the latest
received ancillary data for any activated socket option recvtclass,
recvtos or recvttl, on platforms where IP_PKTOPTIONS is defined
for an IPv4 socket, or where IPV6_PKTOPTIONS or IPV6_2292PKTOPTIONS
is defined for an IPv6 socket.  It will be delivered as a
list of ancillary data items in the same way as for gen_udp
(and gen_sctp).

On some platforms, e.g the BSD:s, when you activate IP_RECVTOS
you get ancillary data tagged IP_RECVTOS with the TOS value,
but on Linux you get ancillary data tagged IP_TOS with the
TOS value.  Linux follows the style of RFC 2292, and the BSD:s
use an older notion.  For RFC 2292 that defines the IP_PKTOPTIONS
socket option it is more logical to tag the items with the
tag that is the item's, than with the tag that defines that you
want the item.  Therefore this implementation translates all
BSD style ancillary data tags to the corresponding Linux style
data tags, so the application will only see the tags 'tclass',
'tos' and 'ttl' on all platforms.
---
 erts/emulator/drivers/common/inet_drv.c | 638 ++++++++++++++++++++++++++++----
 1 file changed, 556 insertions(+), 82 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 2048d0f625..f9a471afd5 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -624,6 +624,26 @@ static size_t my_strnlen(const char *s, size_t maxlen)
 #  define VALGRIND_MAKE_MEM_DEFINED(ptr,size)
 #endif
 
+#ifndef __WIN32__
+/* Calculate CMSG_NXTHDR without having a struct msghdr*
+ * CMSG_LEN only caters for alignment for start of data.
+ * To get how much to advance we need to use CMSG_SPACE
+ * on the payload length.  To get the payload length we
+ * take the calculated cmsg->cmsg_len and subtract the
+ * header length.  To get the header length we use
+ * CMSG_LEN with payload length 0.
+ */
+#define LEN_CMSG_DATA(cmsg)  ((char*)CMSG_DATA(cmsg) - (char*)(cmsg))
+#define NXT_CMSG_HDR(cmsg)                                              \
+    ((struct cmsghdr*)                                                  \
+     (((char*)(cmsg)) +                                                 \
+      CMSG_SPACE((cmsg)->cmsg_len - LEN_CMSG_DATA(cmsg))))
+#endif
+
+#if !defined(IPV6_PKTOPTIONS) && defined(IPV6_2292PKTOPTIONS)
+#define IPV6_PKTOPTIONS IPV6_2292PKTOPTIONS
+#endif
+
 /*
   Magic errno value used locally for return of {error, system_limit}
   - the emulator definition of SYSTEM_LIMIT is not available here.
@@ -787,6 +807,11 @@ static size_t my_strnlen(const char *s, size_t maxlen)
 #define INET_LOPT_LINE_DELIM        40  /* Line delimiting char */
 #define INET_OPT_TCLASS             41  /* IPv6 transport class */
 #define INET_OPT_BIND_TO_DEVICE     42  /* get/set network device the socket is bound to */
+#define INET_OPT_RECVTOS            43  /* IP_RECVTOS ancillary data */
+#define INET_OPT_RECVTCLASS         44  /* IPV6_RECVTCLASS ancillary data */
+#define INET_OPT_PKTOPTIONS         45  /* IP(V6)_PKTOPTIONS get ancillary data */
+#define INET_OPT_TTL                46  /* IP_TTL */
+#define INET_OPT_RECVTTL            47  /* IP_RECVTTL ancillary data */
 /* SCTP options: a separate range, from 100: */
 #define SCTP_OPT_RTOINFO		100
 #define SCTP_OPT_ASSOCINFO		101
@@ -862,6 +887,11 @@ static size_t my_strnlen(const char *s, size_t maxlen)
 #define SCTP_FLAG_SACDELAY_ENABLE  (32 /* am_sackdelay_enable */)
 #define SCTP_FLAG_SACDELAY_DISABLE (64 /* am_sackdelay_disable */)
 
+/* Flags for recv_cmsgflags */
+#define INET_CMSG_RECVTOS        (1 << 0) /* am_recvtos, am_tos */
+#define INET_CMSG_RECVTCLASS     (1 << 1) /* am_recvtclass, am_tclass */
+#define INET_CMSG_RECVTTL        (1 << 2) /* am_recvttl, am_ttl */
+
 /*
 ** End of interface constants.
 **--------------------------------------------------------------------------*/
@@ -1084,6 +1114,7 @@ typedef struct {
     char *netns;                /* Socket network namespace name
 				   as full file path */
 #endif
+    int recv_cmsgflags;         /* Which ancillary data to expect */
 } inet_descriptor;
 
 
@@ -1336,6 +1367,11 @@ static ErlDrvTermData am_udp_error;
 #ifdef HAVE_SYS_UN_H
 static ErlDrvTermData am_local;
 #endif
+#ifndef __WIN32__
+static ErlDrvTermData am_tos;
+static ErlDrvTermData am_tclass;
+static ErlDrvTermData am_ttl;
+#endif
 #ifdef HAVE_SCTP
 static ErlDrvTermData am_sctp;
 static ErlDrvTermData am_sctp_passive;
@@ -1356,8 +1392,9 @@ static ErlDrvTermData am_sndbuf;
 static ErlDrvTermData am_reuseaddr;
 static ErlDrvTermData am_dontroute;
 static ErlDrvTermData am_priority;
-static ErlDrvTermData am_tos;
-static ErlDrvTermData am_tclass;
+static ErlDrvTermData am_recvtos;
+static ErlDrvTermData am_recvtclass;
+static ErlDrvTermData am_recvttl;
 static ErlDrvTermData am_ipv6_v6only;
 static ErlDrvTermData am_netns;
 static ErlDrvTermData am_bind_to_device;
@@ -1548,10 +1585,10 @@ static void *realloc_wrapper(void *current, ErlDrvSizeT size){
 #   define ASSOC_ID_LEN		4
 #   define LOAD_ASSOC_ID        LOAD_UINT
 #   define LOAD_ASSOC_ID_CNT    LOAD_UINT_CNT
-#   define SCTP_ANC_BUFF_SIZE   INET_DEF_BUFFER/2 /* XXX: not very good... */
 #else
 #   define IS_SCTP(desc) 0
 #endif
+#   define ANC_BUFF_SIZE   INET_DEF_BUFFER/2 /* XXX: not very good... */
 
 #ifdef HAVE_UDP
 static int load_address(ErlDrvTermData* spec, int i, char* buf)
@@ -2752,6 +2789,44 @@ static int inet_async_data(inet_descriptor* desc, const char* buf, int len)
     }
 }
 
+#ifndef __WIN32__
+static int parse_ancillary_data_item(ErlDrvTermData *spec, int i,
+                                     struct cmsghdr *cmsg, int *n) {
+#define LOAD_CMSG(proto, type, vtype, am, load)         \
+    if (cmsg->cmsg_level == (proto) &&                  \
+        cmsg->cmsg_type == (type)) {                    \
+        vtype *vp;                                      \
+        vp = (vtype *)CMSG_DATA(cmsg);                  \
+        i = LOAD_ATOM(spec, i, (am));                   \
+        i = load(spec, i, *vp);                         \
+        i = LOAD_TUPLE(spec, i, 2);                     \
+        (*n)++;                                         \
+        return i;                                       \
+    }
+#if defined(IPPROTO_IP) && defined(IP_TOS)
+    LOAD_CMSG(IPPROTO_IP, IP_TOS, unsigned char, am_tos, LOAD_INT);
+#endif
+#if defined(IPPROTO_IPV6) && defined(IPV6_TCLASS)
+    LOAD_CMSG(IPPROTO_IPV6, IPV6_TCLASS, unsigned char, am_tclass, LOAD_INT);
+#endif
+#if defined(IPPROTO_IP) && defined(IP_TTL)
+    LOAD_CMSG(IPPROTO_IP, IP_TTL, unsigned char, am_ttl, LOAD_INT);
+#endif
+    /* BSD uses the RECV* names in CMSG fields */
+#if defined(IPPROTO_IP) && defined(IP_RECVTOS)
+    LOAD_CMSG(IPPROTO_IP, IP_RECVTOS, unsigned char, am_tos, LOAD_INT);
+#endif
+#if defined(IPPROTO_IPV6) && defined(IPV6_RECVTCLASS)
+    LOAD_CMSG(IPPROTO_IPV6, IPV6_RECVTCLASS, unsigned char, am_tclass, LOAD_INT);
+#endif
+#if defined(IPPROTO_IP) && defined(IP_RECVTTL)
+    LOAD_CMSG(IPPROTO_IP, IP_RECVTTL, unsigned char, am_ttl, LOAD_INT);
+#endif
+#undef LOAD_CMSG
+    return i;
+}
+#endif /* #ifndef __WIN32__ */
+
 #ifdef HAVE_SCTP
 /*
 ** SCTP-related atoms:
@@ -2883,11 +2958,18 @@ static int sctp_parse_ancillary_data
     for (cmsg = frst_msg; cmsg != NULL; cmsg = CMSG_NXTHDR(mptr,cmsg))
     {
 	struct sctp_sndrcvinfo * sri;
-	
+#ifndef __WIN32
+        int old_s;
+
+        /* Parse ancillary data common to UDP */
+        old_s = s;
+        i = parse_ancillary_data_item(spec, i, cmsg, &s);
+        if (s > old_s) continue;
 	/* Skip other possible ancillary data, e.g. from IPv6: */
 	if (cmsg->cmsg_level != IPPROTO_SCTP ||
 	    cmsg->cmsg_type  != SCTP_SNDRCV)
 	continue;
+#endif
 
 	if (((char*)cmsg + cmsg->cmsg_len) - (char*)frst_msg >
 	    mptr->msg_controllen)
@@ -3227,6 +3309,23 @@ static int sctp_parse_async_event
 }
 #endif  /* HAVE_SCTP */
 
+#ifndef __WIN32__
+static int udp_parse_ancillary_data(ErlDrvTermData *spec, int i,
+                                struct msghdr *mptr) {
+    struct cmsghdr *cmsg;
+    int n;
+
+    n = 0;
+    for (cmsg = CMSG_FIRSTHDR(mptr);
+         cmsg != NULL;
+         cmsg = CMSG_NXTHDR(mptr, cmsg)) {
+        i = parse_ancillary_data_item(spec, i, cmsg, &n);
+    }
+    i = LOAD_NIL(spec, i);
+    return LOAD_LIST(spec, i, n+1);
+}
+#endif /* ifndef __WIN32__ */
+
 /* 
 ** passive mode reply:
 ** for UDP:
@@ -3249,7 +3348,7 @@ static int sctp_parse_async_event
 static int
 inet_async_binary_data
 	(inet_descriptor* desc, unsigned  int phsz,
-	 ErlDrvBinary   * bin,  int offs, int len, void * extra)
+	 ErlDrvBinary   * bin,  int offs, int len, void *mp)
 {
     unsigned int hsz = desc->hsz + phsz;
     ErlDrvTermData spec [PACKET_ERL_DRV_TERM_DATA_LEN];
@@ -3282,9 +3381,10 @@ inet_async_binary_data
     if (IS_SCTP(desc))
     {	/* For SCTP we always have desc->hsz==0 (i.e., no application-level
 	   headers are used), so hsz==phsz (see above): */
-	struct msghdr* mptr;
 	int sz;
-	
+        struct msghdr *mptr;
+
+        mptr = mp;
 	ASSERT (hsz == phsz && hsz != 0);
 	sz = len - hsz;  /* Size of the msg data proper, w/o the addr */
 
@@ -3292,7 +3392,6 @@ inet_async_binary_data
 	i = LOAD_STRING(spec, i, bin->orig_bytes+offs, hsz);
 
 	/* Put in the list (possibly empty) of Ancillary Data: */
-	mptr = (struct msghdr *) extra;
 	i = sctp_parse_ancillary_data (spec, i, mptr);
 
 	/* Then: Data or Event (Notification)? */
@@ -3321,20 +3420,32 @@ inet_async_binary_data
     }
     else
 #endif  /* HAVE_SCTP */
-    /* Generic case. Both Addr and Data (or a single list of them together) are
-       returned: */
+    {
+        /* Generic case. Both Addr and Data
+         * (or a single list of them together) are returned: */
 
-    if ((desc->mode == INET_MODE_LIST) || (hsz > len)) {
-	/* INET_MODE_LIST => [H1,H2,...Hn] */
-	i = LOAD_STRING(spec, i, bin->orig_bytes+offs, len);
-    }
-    else {
-	/* INET_MODE_BINARY => [H1,H2,...HSz | Binary] or [Binary]: */
-	int sz = len - hsz;
-	i = LOAD_BINARY(spec, i, bin, offs+hsz, sz);
-	if (hsz > 0)
-	    i = LOAD_STRING_CONS(spec, i, bin->orig_bytes+offs, hsz);
+        if ((desc->mode == INET_MODE_LIST) || (hsz > len)) {
+            /* INET_MODE_LIST => [H1,H2,...Hn] */
+            i = LOAD_STRING(spec, i, bin->orig_bytes+offs, len);
+        }
+        else {
+            /* INET_MODE_BINARY => [H1,H2,...HSz | Binary] or [Binary]: */
+            int sz = len - hsz;
+            i = LOAD_BINARY(spec, i, bin, offs+hsz, sz);
+            if (hsz > 0)
+                i = LOAD_STRING_CONS(spec, i, bin->orig_bytes+offs, hsz);
+        }
+
+#ifndef __WIN32__
+        if (mp) {
+            /* We got ancillary data from an UDP recvmsg.
+             * Insert an additional tuple level {[F|AddrData],AncData} */
+            i = udp_parse_ancillary_data(spec, i, (struct msghdr*)mp);
+            i = LOAD_TUPLE(spec, i, 2);
+        }
+#endif
     }
+
     /* Close up the {ok, ...} or {error, ...} tuple: */
     i = LOAD_TUPLE(spec, i, 2);
 
@@ -3466,8 +3577,9 @@ static int tcp_error_message(tcp_descriptor* desc, int err)
 **    [AddrLen, H2,...,HSz] are msg headers for UDP AF_UNIX only
 **	  Data  : List() | Binary()
 */
-static int packet_binary_message
-    (inet_descriptor* desc, ErlDrvBinary* bin, int offs, int len, void* extra)
+static int packet_binary_message(inet_descriptor* desc,
+                                 ErlDrvBinary* bin, int offs, int len,
+                                 void *mp)
 {
     unsigned int hsz = desc->hsz;
     ErlDrvTermData spec [PACKET_ERL_DRV_TERM_DATA_LEN];
@@ -3492,8 +3604,14 @@ static int packet_binary_message
 
 #   ifdef HAVE_SCTP
     if (!IS_SCTP(desc))
-    {
 #   endif
+    {
+#ifndef __WIN32__
+        if (mp) i = udp_parse_ancillary_data(spec, i, (struct msghdr*)mp);
+#endif
+        /* We got ancillary data from an UDP recvmsg.
+         * Insert an additional tuple level {AncData,[F|AddrData]}
+         */
 	if ((desc->mode == INET_MODE_LIST) || (hsz > len))
 	    /* INET_MODE_LIST, or only headers => [H1,H2,...Hn] */
 	    i = LOAD_STRING(spec, i, bin->orig_bytes+offs, len);
@@ -3505,16 +3623,24 @@ static int packet_binary_message
 	    if (hsz > 0)
 		i = LOAD_STRING_CONS(spec, i, bin->orig_bytes+offs, hsz);
 	}
-#   ifdef HAVE_SCTP
+        /* Close up the outer 5-or-6-tuple */
+#ifndef __WIN32__
+        if (mp) i = LOAD_TUPLE(spec, i, 6);
+        else
+#endif
+            i = LOAD_TUPLE(spec, i, 5);
     }
+#   ifdef HAVE_SCTP
     else
-    {	/* For SCTP we always have desc->hsz==0 (i.e., no application-level
+    {
+        struct msghdr *mptr;
+
+        mptr = mp;
+	/* For SCTP we always have desc->hsz==0 (i.e., no application-level
 	   headers are used): */
-	struct msghdr* mptr;
 	ASSERT(hsz == 0);
 
 	/* Put in the list (possibly empty) of Ancillary Data: */
-	mptr = (struct msghdr *) extra;
 	i = sctp_parse_ancillary_data (spec, i, mptr);
 
 	/* Then: Data or Event (Notification)? */
@@ -3540,11 +3666,11 @@ static int packet_binary_message
 
 	/* Close up the {[AncilData], Event_OR_Data} tuple: */
 	i = LOAD_TUPLE (spec, i, 2);
+        /* Close up the outer 5-tuple: */
+        i = LOAD_TUPLE(spec, i, 5);
     }
 #   endif /* HAVE_SCTP */
 
-    /* Close up the outer 5-tuple: */
-    i = LOAD_TUPLE(spec, i, 5);
     ASSERT(i <= PACKET_ERL_DRV_TERM_DATA_LEN);
     return erl_drv_output_term(desc->dport, spec, i);
 }
@@ -3678,19 +3804,19 @@ tcp_reply_binary_data(tcp_descriptor* desc, ErlDrvBinary* bin, int offs, int len
 static int
 packet_reply_binary_data(inet_descriptor* desc, unsigned  int hsz,
 			 ErlDrvBinary   * bin,  int offs, int len,
-			 void * extra)
+                         void *mp)
 {
     int code;
 
     if (desc->active == INET_PASSIVE)
 	/* "inet" is actually for both UDP and SCTP, as well as TCP! */
-	return inet_async_binary_data(desc, hsz, bin, offs, len, extra);
+	return inet_async_binary_data(desc, hsz, bin, offs, len, mp);
     else
     {	/* INET_ACTIVE or INET_ONCE: */
 	if (desc->deliver == INET_DELIVER_PORT)
 	    code = inet_port_binary_data(desc, bin, offs, len);
 	else
-	    code = packet_binary_message(desc, bin, offs, len, extra);
+	    code = packet_binary_message(desc, bin, offs, len, mp);
 	if (code < 0)
 	    return code;
         INET_CHECK_ACTIVE_TO_PASSIVE(desc);
@@ -3757,8 +3883,9 @@ static void inet_init_sctp(void) {
     INIT_ATOM(reuseaddr);
     INIT_ATOM(dontroute);
     INIT_ATOM(priority);
-    INIT_ATOM(tos);
-    INIT_ATOM(tclass);
+    INIT_ATOM(recvtos);
+    INIT_ATOM(recvtclass);
+    INIT_ATOM(recvttl);
     INIT_ATOM(ipv6_v6only);
     INIT_ATOM(netns);
     INIT_ATOM(bind_to_device);
@@ -3901,6 +4028,11 @@ static int inet_init()
 #endif
     INIT_ATOM(empty_out_q);
     INIT_ATOM(ssl_tls);
+#ifndef __WIN32__
+    INIT_ATOM(tos);
+    INIT_ATOM(tclass);
+    INIT_ATOM(ttl);
+#endif
 
     INIT_ATOM(http_eoh);
     INIT_ATOM(http_header);
@@ -5119,8 +5251,8 @@ static ErlDrvSSizeT inet_ctl_ifget(inet_descriptor* desc,
 	    sys_memcpy(sptr,
 		       sdlp->sdl_data + sdlp->sdl_nlen,
 		       sdlp->sdl_alen);
-	    freeifaddrs(ifa);
 	    sptr += sdlp->sdl_alen;
+	    freeifaddrs(ifa);
 #endif
 	    break;
 	}
@@ -5933,7 +6065,8 @@ static ErlDrvSSizeT inet_ctl_getifaddrs(inet_descriptor* desc_p,
       but ditto for the other worked and that was actually the requested
       option, failure was still reported to erlang.                  */
 
-#if  defined(IP_TOS) && defined(SOL_IP) && defined(SO_PRIORITY)
+#if  defined(IP_TOS) && defined(IPPROTO_IP)             \
+    && defined(SO_PRIORITY) && !defined(__WIN32__)
 static int setopt_prio_tos_trick
 	(int fd, int proto, int type, char* arg_ptr, int arg_sz, int propagate)
 {
@@ -5955,14 +6088,14 @@ static int setopt_prio_tos_trick
 
     res_prio = sock_getopt(fd, SOL_SOCKET, SO_PRIORITY,
 		      (char *) &tmp_ival_prio, &tmp_arg_sz_prio);
-    res_tos = sock_getopt(fd, SOL_IP, IP_TOS, 
+    res_tos = sock_getopt(fd, IPPROTO_IP, IP_TOS,
 		      (char *) &tmp_ival_tos, &tmp_arg_sz_tos);
 	    res = sock_setopt(fd, proto, type, arg_ptr, arg_sz);
 	    if (res == 0) {
 		if (type != SO_PRIORITY) {
 	    if (type != IP_TOS && res_tos == 0) {
 		res_tos = sock_setopt(fd, 
-					  SOL_IP, 
+					  IPPROTO_IP,
 					  IP_TOS,
 					  (char *) &tmp_ival_tos, 
 					  tmp_arg_sz_tos);
@@ -6006,7 +6139,7 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
     int proto;
     int opt;
     struct linger li_val;
-#ifdef HAVE_MULTICAST_SUPPORT
+#if defined(HAVE_MULTICAST_SUPPORT) && defined(IPPROTO_IP)
     struct ip_mreq mreq_val;
 #endif
     int ival;
@@ -6029,6 +6162,8 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
     /* XXX { int i; for(i=0;i<len;++i) fprintf(stderr,"0x%02X, ", (unsigned) ptr[i]); fprintf(stderr,"\r\n");} */
 
     while(len >= 5) {
+        int recv_cmsgflags;
+
 	opt = *ptr++;
 	ival = get_int32(ptr);
 	ptr += 4;
@@ -6037,6 +6172,7 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
 	arg_sz = sizeof(ival);
 	proto = SOL_SOCKET;
 	propagate = 0;
+        recv_cmsgflags = desc->recv_cmsgflags;
 
 	switch(opt) {
 	case INET_LOPT_HEADER:
@@ -6287,28 +6423,80 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
 		    (long)desc->port, desc->s, ival));
 	    break;
 #else
+            /* inet_fill_opts always returns a value for this option,
+             * so we need to ignore it if not implemented */
 	    continue;
 #endif
 	case INET_OPT_TOS:
-#if defined(IP_TOS) && defined(SOL_IP)
-	    proto = SOL_IP;
+#if defined(IP_TOS) && defined(IPPROTO_IP)
+	    proto = IPPROTO_IP;
 	    type = IP_TOS;
 	    propagate = 1;
 	    DEBUGF(("inet_set_opts(%ld): s=%d, IP_TOS=%d\r\n",
 		    (long)desc->port, desc->s, ival));
 	    break;
 #else
+            /* inet_fill_opts always returns a value for this option,
+             * so we need to ignore it if not implemented. */
 	    continue;
 #endif
-#if defined(IPV6_TCLASS) && defined(SOL_IPV6)
+#if defined(IPV6_TCLASS) && defined(IPPROTO_IPV6)
 	case INET_OPT_TCLASS:
-	    proto = SOL_IPV6;
+	    proto = IPPROTO_IPV6;
 	    type = IPV6_TCLASS;
 	    propagate = 1;
 	    DEBUGF(("inet_set_opts(%ld): s=%d, IPV6_TCLASS=%d\r\n",
 		    (long)desc->port, desc->s, ival));
 	    break;
 #endif
+#if defined(IP_TTL) && defined(IPPROTO_IP)
+	case INET_OPT_TTL:
+	    proto = IPPROTO_IP;
+	    type = IP_TTL;
+	    propagate = 1;
+	    DEBUGF(("inet_set_opts(%ld): s=%d, IP_TTL=%d\r\n",
+		    (long)desc->port, desc->s, ival));
+	    break;
+#endif
+#if defined(IP_RECVTOS) && defined(IPPROTO_IP)
+	case INET_OPT_RECVTOS:
+	    proto = IPPROTO_IP;
+	    type = IP_RECVTOS;
+	    propagate = 1;
+            recv_cmsgflags =
+                ival ?
+                (desc->recv_cmsgflags | INET_CMSG_RECVTOS) :
+                (desc->recv_cmsgflags & ~INET_CMSG_RECVTOS);
+	    DEBUGF(("inet_set_opts(%ld): s=%d, IP_RECVTOS=%d\r\n",
+		    (long)desc->port, desc->s, ival));
+	    break;
+#endif
+#if defined(IPV6_RECVTCLASS) && defined(IPPROTO_IPV6)
+	case INET_OPT_RECVTCLASS:
+	    proto = IPPROTO_IPV6;
+	    type = IPV6_RECVTCLASS;
+	    propagate = 1;
+            recv_cmsgflags =
+                ival ?
+                (desc->recv_cmsgflags | INET_CMSG_RECVTCLASS) :
+                (desc->recv_cmsgflags & ~INET_CMSG_RECVTCLASS);
+	    DEBUGF(("inet_set_opts(%ld): s=%d, IPV6_RECVTCLASS=%d\r\n",
+		    (long)desc->port, desc->s, ival));
+	    break;
+#endif
+#if defined(IP_RECVTTL) && defined(IPPROTO_IP)
+	case INET_OPT_RECVTTL:
+	    proto = IPPROTO_IP;
+	    type = IP_RECVTTL;
+	    propagate = 1;
+            recv_cmsgflags =
+                ival ?
+                (desc->recv_cmsgflags | INET_CMSG_RECVTTL) :
+                (desc->recv_cmsgflags & ~INET_CMSG_RECVTTL);
+	    DEBUGF(("inet_set_opts(%ld): s=%d, IP_RECVTTL=%d\r\n",
+		    (long)desc->port, desc->s, ival));
+	    break;
+#endif
 
 	case TCP_OPT_NODELAY:
 	    proto = IPPROTO_TCP; 
@@ -6317,7 +6505,7 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
 		    (long)desc->port, desc->s, ival));
 	    break;
 
-#ifdef HAVE_MULTICAST_SUPPORT
+#if defined(HAVE_MULTICAST_SUPPORT) && defined(IPPROTO_IP)
 
 	case UDP_OPT_MULTICAST_TTL:
 	    proto = IPPROTO_IP;
@@ -6363,10 +6551,10 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    arg_sz = sizeof(mreq_val);
 	    break;
 
-#endif /* HAVE_MULTICAST_SUPPORT */
+#endif /* defined(HAVE_MULTICAST_SUPPORT) && defined(IPPROTO_IP) */
 
 	case INET_OPT_IPV6_V6ONLY:
-#if HAVE_DECL_IPV6_V6ONLY
+#if HAVE_DECL_IPV6_V6ONLY && defined(IPPROTO_IPV6)
 	    proto = IPPROTO_IPV6;
 	    type = IPV6_V6ONLY;
 	    propagate = 1;
@@ -6426,11 +6614,13 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
 	default:
 	    return -1;
 	}
-#if  defined(IP_TOS) && defined(SOL_IP) && defined(SO_PRIORITY)
+#if  defined(IP_TOS) && defined(IPPROTO_IP) \
+    && defined(SO_PRIORITY) && !defined(__WIN32__)
 	res = setopt_prio_tos_trick (desc->s, proto, type, arg_ptr, arg_sz, propagate);
 #else
 	res = sock_setopt	    (desc->s, proto, type, arg_ptr, arg_sz);
 #endif
+        if (res == 0) desc->recv_cmsgflags = recv_cmsgflags;
 	if (propagate && res != 0) {
 	    return -1;
 	}
@@ -6572,10 +6762,14 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 
     while (curr < ptr + len)
     {
+        int recv_cmsgflags;
 	/* Get the Erlang-encoded option type -- always 1 byte: */
-	int eopt = *curr;
+	int eopt;
+
+        eopt = *curr;
 	curr++;
 
+        recv_cmsgflags = desc->recv_cmsgflags;
 	/* Get the option value.  XXX: The condition  (curr < ptr + len)
 	   does not preclude us from reading from beyond the buffer end,
 	   if the Erlang part of the driver specifies its input wrongly!
@@ -6756,28 +6950,32 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    break;
 	}
 #	else
-	    continue; /* Option not supported -- ignore it */
+        /* inet_fill_opts always returns a value for this option,
+         * so we need to ignore it if not implemented, just in case */
+	    continue;
 #	endif
 
 	case INET_OPT_TOS:
-#	if defined(IP_TOS) && defined(SOL_IP)
+#	if defined(IP_TOS) && defined(IPPROTO_IP)
 	{
 	    arg.ival= get_int32 (curr);	  curr += 4;
-	    proto   = SOL_IP;
+	    proto   = IPPROTO_IP;
 	    type    = IP_TOS;
 	    arg_ptr = (char*) (&arg.ival);
 	    arg_sz  = sizeof  ( arg.ival);
 	    break;
 	}
 #	else
-	    continue; /* Option not supported -- ignore it */
+        /* inet_fill_opts always returns a value for this option,
+         * so we need to ignore it if not implemented, just in case */
+	    continue;
 #	endif
 
-#       if defined(IPV6_TCLASS) && defined(SOL_IPV6)
+#       if defined(IPV6_TCLASS) && defined(IPPROTO_IPV6)
 	case INET_OPT_TCLASS:
 	{
 	    arg.ival= get_int32 (curr);	  curr += 4;
-	    proto   = SOL_IPV6;
+	    proto   = IPPROTO_IPV6;
 	    type    = IPV6_TCLASS;
 	    arg_ptr = (char*) (&arg.ival);
 	    arg_sz  = sizeof  ( arg.ival);
@@ -6785,9 +6983,69 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	}
 #	endif
 
+#       if defined(IP_TTL) && defined(IPPROTO_IP)
+	case INET_OPT_TTL:
+	{
+	    arg.ival= get_int32 (curr);	  curr += 4;
+	    proto   = IPPROTO_IP;
+	    type    = IP_TTL;
+	    arg_ptr = (char*) (&arg.ival);
+	    arg_sz  = sizeof  ( arg.ival);
+	    break;
+	}
+#	endif
+
+#	if defined(IP_RECVTOS) && defined(IPPROTO_IP)
+	case INET_OPT_RECVTOS:
+	{
+	    arg.ival= get_int32 (curr);	  curr += 4;
+	    proto   = IPPROTO_IP;
+	    type    = IP_RECVTOS;
+	    arg_ptr = (char*) (&arg.ival);
+	    arg_sz  = sizeof  ( arg.ival);
+            recv_cmsgflags =
+                arg.ival ?
+                (desc->recv_cmsgflags | INET_CMSG_RECVTOS) :
+                (desc->recv_cmsgflags & ~INET_CMSG_RECVTOS);
+	    break;
+	}
+#	endif
+
+#       if defined(IPV6_RECVTCLASS) && defined(IPPROTO_IPV6)
+	case INET_OPT_RECVTCLASS:
+	{
+	    arg.ival= get_int32 (curr);	  curr += 4;
+	    proto   = IPPROTO_IPV6;
+	    type    = IPV6_RECVTCLASS;
+	    arg_ptr = (char*) (&arg.ival);
+	    arg_sz  = sizeof  ( arg.ival);
+            recv_cmsgflags =
+                arg.ival ?
+                (desc->recv_cmsgflags | INET_CMSG_RECVTCLASS) :
+                (desc->recv_cmsgflags & ~INET_CMSG_RECVTCLASS);
+	    break;
+	}
+#	endif
+
+#	if defined(IP_RECVTTL) && defined(IPPROTO_IP)
+	case INET_OPT_RECVTTL:
+	{
+	    arg.ival= get_int32 (curr);	  curr += 4;
+	    proto   = IPPROTO_IP;
+	    type    = IP_RECVTTL;
+	    arg_ptr = (char*) (&arg.ival);
+	    arg_sz  = sizeof  ( arg.ival);
+            recv_cmsgflags =
+                arg.ival ?
+                (desc->recv_cmsgflags | INET_CMSG_RECVTTL) :
+                (desc->recv_cmsgflags & ~INET_CMSG_RECVTTL);
+	    break;
+	}
+#	endif
+
 
 	case INET_OPT_IPV6_V6ONLY:
-#       if HAVE_DECL_IPV6_V6ONLY
+#       if HAVE_DECL_IPV6_V6ONLY && defined(IPPROTO_IPV6)
 	{
 	    arg.ival= get_int32 (curr);   curr += 4;
 	    proto   = IPPROTO_IPV6;
@@ -7037,13 +7295,15 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    */
 	    return -1;
 	}
-#if  defined(IP_TOS) && defined(SOL_IP) && defined(SO_PRIORITY)
+#if  defined(IP_TOS) && defined(IPPROTO_IP)             \
+    && defined(SO_PRIORITY) && !defined(__WIN32__)
 	res = setopt_prio_tos_trick (desc->s, proto, type, arg_ptr, arg_sz, 1);
 #else
 	res = sock_setopt	    (desc->s, proto, type, arg_ptr, arg_sz);
 #endif
 	/* The return values of "sock_setopt" can only be 0 or -1: */
 	ASSERT(res == 0 || res == -1);
+        if (res == 0) desc->recv_cmsgflags = recv_cmsgflags;
 	if (res == -1)
 	{  /* Got an error, DO NOT continue with other options. However, on
 	      Solaris 10, we DO allow SO_SNDBUF and SO_RCVBUF to fail, assu-
@@ -7298,8 +7558,8 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc,
 	    continue;
 #endif
 	case INET_OPT_TOS:
-#if defined(IP_TOS) && defined(SOL_IP)
-	    proto = SOL_IP;
+#if defined(IP_TOS) && defined(IPPROTO_IP)
+	    proto = IPPROTO_IP;
 	    type = IP_TOS;
 	    break;
 #else
@@ -7308,13 +7568,49 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc,
 	    continue;
 #endif
 	case INET_OPT_TCLASS:
-#if defined(IPV6_TCLASS) && defined(SOL_IPV6)
-	    proto = SOL_IPV6;
+#if defined(IPV6_TCLASS) && defined(IPPROTO_IPV6)
+	    proto = IPPROTO_IPV6;
 	    type = IPV6_TCLASS;
 	    break;
 #else
 	    TRUNCATE_TO(0,ptr);
 	    continue;
+#endif
+	case INET_OPT_TTL:
+#if defined(IP_TTL) && defined(IPPROTO_IP)
+	    proto = IPPROTO_IP;
+	    type = IP_TTL;
+	    break;
+#else
+	    TRUNCATE_TO(0,ptr);
+	    continue;
+#endif
+	case INET_OPT_RECVTOS:
+#if defined(IP_RECVTOS) && defined(IPPROTO_IP)
+	    proto = IPPROTO_IP;
+	    type = IP_RECVTOS;
+	    break;
+#else
+	    TRUNCATE_TO(0,ptr);
+	    continue;
+#endif
+	case INET_OPT_RECVTCLASS:
+#if defined(IPV6_RECVTCLASS) && defined(IPPROTO_IPV6)
+	    proto = IPPROTO_IPV6;
+	    type = IPV6_RECVTCLASS;
+	    break;
+#else
+	    TRUNCATE_TO(0,ptr);
+	    continue;
+#endif
+	case INET_OPT_RECVTTL:
+#if defined(IP_RECVTTL) && defined(IPPROTO_IP)
+	    proto = IPPROTO_IP;
+	    type = IP_RECVTTL;
+	    break;
+#else
+	    TRUNCATE_TO(0,ptr);
+	    continue;
 #endif
 	case INET_OPT_REUSEADDR: 
 	    type = SO_REUSEADDR; 
@@ -7342,7 +7638,7 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc,
 	    type = TCP_NODELAY;
 	    break;
 
-#ifdef HAVE_MULTICAST_SUPPORT
+#if defined(HAVE_MULTICAST_SUPPORT) && defined(IPPROTO_IP)
 	case UDP_OPT_MULTICAST_TTL:
 	    proto = IPPROTO_IP;
 	    type = IP_MULTICAST_TTL;
@@ -7361,10 +7657,10 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc,
 	    arg_ptr = (char*) &li_val;	    
 	    type = SO_LINGER; 
 	    break;
-#endif /* HAVE_MULTICAST_SUPPORT */
+#endif /* defined(HAVE_MULTICAST_SUPPORT) && defined(IPPROTO_IP) */
 
 	case INET_OPT_IPV6_V6ONLY:
-#if HAVE_DECL_IPV6_V6ONLY
+#if HAVE_DECL_IPV6_V6ONLY && defined(IPPROTO_IPV6)
 	    proto = IPPROTO_IPV6;
 	    type = IPV6_V6ONLY;
 	    break;
@@ -7442,6 +7738,101 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc,
 	    continue;
 #endif
 
+#ifndef __WIN32__
+            /* Winsock does not have struct cmsghdr */
+        case INET_OPT_PKTOPTIONS: {
+            struct cmsghdr *cmsg, *cmsg_top;
+            SOCKLEN_T cmsg_sz;
+            union {
+                /* Ensure alignment */
+                struct cmsghdr hdr;
+                /* Room for (IP_TOS | IPV6_TCLASS) + IP_TTL */
+                char buf[2*CMSG_SPACE(sizeof(int))];
+            } cmsgbuf;
+            /* Select between IPv4 or IPv6 PKTOPTIONS
+             * depending on the socket protocol family
+             */
+            switch (desc->sfamily) {
+#if defined(IPPROTO_IP) && defined(IP_PKTOPTIONS)
+            case AF_INET: {
+                proto = IPPROTO_IP;
+                type = IP_PKTOPTIONS;
+            }
+                break;
+#endif
+#if defined(IPPROTO_IPV6) && defined(IPV6_PKTOPTIONS) && defined(AF_INET6)
+            case AF_INET6: {
+                proto = IPPROTO_IPV6;
+                type = IPV6_PKTOPTIONS;
+            }
+                break;
+#endif
+            default: {
+                RETURN_ERROR();
+            }
+            } /* switch */
+            TRUNCATE_TO(0, ptr);
+            /* Fetch a cmsg buffer from the socket */
+            cmsg_sz = sizeof(cmsgbuf.buf);
+            if (IS_SOCKET_ERROR(sock_getopt(desc->s, proto, type,
+                                            cmsgbuf.buf, &cmsg_sz))) {
+                continue;
+            }
+            /* Reply with Opt/8, Length/32, [COpt/8, Value/32]*
+             * i.e opt, total length and then all returned
+             * cmsg options and values
+             */
+            PLACE_FOR(1+4, ptr);
+            *ptr = opt;
+            arg_ptr = ptr+1; /* Where to put total length */
+            arg_sz = 0; /* Total length */
+            for (cmsg_top = (struct cmsghdr*)(cmsgbuf.buf + cmsg_sz),
+                     cmsg = (struct cmsghdr*)cmsgbuf.buf;
+                 cmsg < cmsg_top;
+                 cmsg = NXT_CMSG_HDR(cmsg)) {
+#define PUT_CMSG_DATA(CMSG_LEVEL, CMSG_TYPE, OPT, TYPE, SZ, PUT)        \
+                if ((cmsg->cmsg_level == CMSG_LEVEL) &&                 \
+                    (cmsg->cmsg_type == CMSG_TYPE)) {                   \
+                    TYPE *cmsgp;                                        \
+                    cmsgp = (TYPE *)CMSG_DATA(cmsg);                    \
+                    PLACE_FOR(1+SZ, ptr);                               \
+                    *ptr = OPT;                                         \
+                    PUT(*cmsgp, ptr+1);                                 \
+                    arg_sz += 1+SZ;                                     \
+                    continue;                                           \
+                }
+#if defined(IPPROTO_IP) && defined(IP_TOS)
+                PUT_CMSG_DATA(IPPROTO_IP, IP_TOS,
+                              INET_OPT_TOS, unsigned char, 4, put_int32);
+#endif
+#if defined(IPPROTO_IPV6) && defined(IPV6_TCLASS)
+                PUT_CMSG_DATA(IPPROTO_IPV6, IPV6_TCLASS,
+                              INET_OPT_TCLASS, unsigned char, 4, put_int32);
+#endif
+#if defined(IPPROTO_IP) && defined(IP_TTL)
+                PUT_CMSG_DATA(IPPROTO_IP, IP_TTL,
+                              INET_OPT_TTL, unsigned char, 4, put_int32);
+#endif
+                /* BSD uses the RECV* names in CMSG fields */
+            }
+#if defined(IPPROTO_IP) && defined(IP_RECVTOS)
+                PUT_CMSG_DATA(IPPROTO_IP, IP_RECVTOS,
+                              INET_OPT_TOS, unsigned char, 4, put_int32);
+#endif
+#if defined(IPPROTO_IPV6) && defined(IPV6_RECVTCLASS)
+                PUT_CMSG_DATA(IPPROTO_IPV6, IPV6_RECVTCLASS,
+                              INET_OPT_TCLASS, unsigned char, 4, put_int32);
+#endif
+#if defined(IPPROTO_IP) && defined(IP_RECVTTL)
+                PUT_CMSG_DATA(IPPROTO_IP, IP_RECVTTL,
+                              INET_OPT_TTL, unsigned char, 4, put_int32);
+#endif
+#undef PUT_CMSG_DATA
+            put_int32(arg_sz, arg_ptr); /* Put total length */
+            continue;
+        }
+#endif /* #ifdef __WIN32__ */
+
 	default:
 	    RETURN_ERROR();
 	}
@@ -7501,6 +7892,7 @@ static int load_paddrinfo (ErlDrvTermData * spec, int i,
     return i;
 }
 
+
 /*
 **  "sctp_fill_opts":   Returns {ok, Results}, or an error:
 */
@@ -7750,6 +8142,7 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc,
 	case INET_OPT_PRIORITY :
 	case INET_OPT_TOS      :
 	case INET_OPT_TCLASS   :
+	case INET_OPT_TTL      :
 	case INET_OPT_IPV6_V6ONLY:
 	case SCTP_OPT_AUTOCLOSE:
 	case SCTP_OPT_MAXSEG   :
@@ -7757,6 +8150,9 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc,
 	case SCTP_OPT_NODELAY  :
 	case SCTP_OPT_DISABLE_FRAGMENTS:
 	case SCTP_OPT_I_WANT_MAPPED_V4_ADDR:
+	case INET_OPT_RECVTOS  :
+	case INET_OPT_RECVTCLASS :
+	case INET_OPT_RECVTTL :
 	{
 	    int res   = 0;
 	    unsigned int sz = sizeof(res);
@@ -7812,8 +8208,8 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc,
 	    }
 	    case INET_OPT_TOS:
 	    {
-#	    if defined(IP_TOS) && defined(SOL_IP)
-		proto  = SOL_IP;
+#	    if defined(IP_TOS) && defined(IPPROTO_IP)
+		proto  = IPPROTO_IP;
 		type   = IP_TOS;
 		is_int = 1;
 		tag    = am_tos;
@@ -7825,8 +8221,8 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc,
 	    }
 	    case INET_OPT_TCLASS:
 	    {
-#           if defined(IPV6_TCLASS) && defined(SOL_IPV6)
-		proto  = SOL_IPV6;
+#           if defined(IPV6_TCLASS) && defined(IPPROTO_IPV6)
+		proto  = IPPROTO_IPV6;
 		type   = IPV6_TCLASS;
 		is_int = 1;
 		tag    = am_tclass;
@@ -7834,10 +8230,62 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc,
 #	    else
 		/* Not supported -- ignore */
 		continue;
+#	    endif
+	    }
+	    case INET_OPT_TTL:
+	    {
+#           if defined(IP_TTL) && defined(IPPROTO_IP)
+		proto  = IPPROTO_IP;
+		type   = IP_TTL;
+		is_int = 1;
+		tag    = am_ttl;
+		break;
+#	    else
+		/* Not supported -- ignore */
+		continue;
+#	    endif
+	    }
+	    case INET_OPT_RECVTOS:
+	    {
+#	    if defined(IP_RECVTOS) && defined(IPPROTO_IP)
+		proto  = IPPROTO_IP;
+		type   = IP_RECVTOS;
+		is_int = 0;
+		tag    = am_recvtos;
+		break;
+#	    else
+		/* Not supported -- ignore */
+		continue;
+#	    endif
+	    }
+	    case INET_OPT_RECVTCLASS:
+	    {
+#           if defined(IPV6_RECVTCLASS) && defined(IPPROTO_IPV6)
+		proto  = IPPROTO_IPV6;
+		type   = IPV6_RECVTCLASS;
+		is_int = 0;
+		tag    = am_recvtclass;
+		break;
+#	    else
+		/* Not supported -- ignore */
+		continue;
+#	    endif
+	    }
+	    case INET_OPT_RECVTTL:
+	    {
+#	    if defined(IP_RECVTTL) && defined(IPPROTO_IP)
+		proto  = IPPROTO_IP;
+		type   = IP_RECVTTL;
+		is_int = 0;
+		tag    = am_recvttl;
+		break;
+#	    else
+		/* Not supported -- ignore */
+		continue;
 #	    endif
 	    }
 	    case INET_OPT_IPV6_V6ONLY:
-#           if HAVE_DECL_IPV6_V6ONLY
+#           if HAVE_DECL_IPV6_V6ONLY && defined(IPPROTO_IPV6)
 	    {
 		proto  = IPPROTO_IPV6;
 		type   = IPV6_V6ONLY;
@@ -8502,6 +8950,8 @@ static ErlDrvData inet_start(ErlDrvPort port, int size, int protocol)
     desc->netns = NULL;
 #endif
 
+    desc->recv_cmsgflags = 0;
+
     return (ErlDrvData)desc;
 }
 
@@ -11920,10 +12370,10 @@ static void packet_inet_command(ErlDrvData e, char* buf, ErlDrvSizeT len)
     if (IS_SCTP(desc))
     {
 	ErlDrvSizeT   data_len;
-	struct iovec  iov[1];		 /* For real data            */
-	struct msghdr mhdr;		 /* Message wrapper          */
-	struct sctp_sndrcvinfo *sri;     /* The actual ancilary data */
-	union {                          /* For ancilary data        */
+	struct iovec  iov[1];		 /* For real data              */
+	struct msghdr mhdr;		 /* Message wrapper            */
+	struct sctp_sndrcvinfo *sri;     /* The actual ancillary data  */
+	union {                          /* For ancillary data         */
 	    struct cmsghdr hdr;
 	    char ancd[CMSG_SPACE(sizeof(*sri))];
 	} cmsg;
@@ -11933,12 +12383,12 @@ static void packet_inet_command(ErlDrvData e, char* buf, ErlDrvSizeT len)
 	    return;
 	}
 	
-	/* The ancilary data */
+	/* The ancillary data */
 	sri = (struct sctp_sndrcvinfo *) (CMSG_DATA(&cmsg.hdr));
 	/* Get the "sndrcvinfo" from the buffer, advancing the "ptr": */
 	ptr  = sctp_get_sendparams(sri, ptr);
 	
-	/* The ancilary data wrapper */
+	/* The ancillary data wrapper */
 	cmsg.hdr.cmsg_level = IPPROTO_SCTP;
 	cmsg.hdr.cmsg_type  = SCTP_SNDRCV;
 	cmsg.hdr.cmsg_len   = CMSG_LEN(sizeof(*sri));
@@ -11953,7 +12403,7 @@ static void packet_inet_command(ErlDrvData e, char* buf, ErlDrvSizeT len)
 	iov[0].iov_base         = ptr;          /* The real data */
 	mhdr.msg_iov            = iov;
 	mhdr.msg_iovlen         = 1;
-	mhdr.msg_control        = cmsg.ancd;    /* For ancilary data  */
+	mhdr.msg_control        = cmsg.ancd;    /* For ancillary data  */
 	mhdr.msg_controllen     = cmsg.hdr.cmsg_len;
 	VALGRIND_MAKE_MEM_DEFINED(mhdr.msg_control, mhdr.msg_controllen); /*suppress "uninitialised bytes"*/
 	mhdr.msg_flags          = 0;            /* Not used with "sendmsg"   */
@@ -12037,10 +12487,12 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
     char abuf[sizeof(inet_address)];  /* buffer address; enough??? */
     int packet_count = udesc->read_packets;
     int count = 0;     /* number of packets delivered to owner */
-#ifdef HAVE_SCTP
+#ifndef __WIN32__
     struct msghdr mhdr;	  	     /* Top-level msg structure    */
     struct iovec  iov[1]; 	     /* Data or Notification Event */
-    char   ancd[SCTP_ANC_BUFF_SIZE]; /* Ancillary Data		   */
+    char   ancd[ANC_BUFF_SIZE];      /* Ancillary Data		   */
+#endif
+#ifdef HAVE_SCTP
     int short_recv = 0;
 #endif
 
@@ -12089,7 +12541,7 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
 	    mhdr.msg_iov	= iov;
 	    mhdr.msg_iovlen	= 1;
 	    mhdr.msg_control	= ancd;
-	    mhdr.msg_controllen	= SCTP_ANC_BUFF_SIZE;
+	    mhdr.msg_controllen	= ANC_BUFF_SIZE;
 	    mhdr.msg_flags	= 0;	   /* To be filled by "recvmsg"    */
 	    
 	    /* Do the actual SCTP receive: */
@@ -12104,6 +12556,24 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
 	    other = desc->remote;
 	    goto check_result;
 	}
+#ifndef __WIN32__
+        /* recvmsg() does not exist in the Winsock API */
+        if (desc->recv_cmsgflags) {
+            /* Use recvmsg() */
+            iov->iov_base = udesc->i_ptr;
+            iov->iov_len = desc->bufsz;
+            mhdr.msg_name = &other;
+            mhdr.msg_namelen = len;
+            mhdr.msg_iov = iov;
+            mhdr.msg_iovlen = 1;
+	    mhdr.msg_control = ancd;
+            mhdr.msg_controllen = ANC_BUFF_SIZE;
+            mhdr.msg_flags = 0;
+            n = sock_recvmsg(desc->s, &mhdr, 0);
+            len = mhdr.msg_namelen;
+            goto check_result;
+        }
+#endif
 	n = sock_recvfrom(desc->s, udesc->i_ptr, desc->bufsz,
 			  0, &other.sa, &len);
     check_result:
@@ -12156,7 +12626,7 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
 	{
 	    /* message received */
 	    int code;
-	    void * extra = NULL;
+            void *mp;
 	    char * ptr;
 	    int nsz;
 
@@ -12187,14 +12657,18 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
 		    udesc->i_ptr = NULL;  /* not used from here */
 		}
 	    }
-#ifdef HAVE_SCTP
-	    if (IS_SCTP(desc)) extra = &mhdr;
+            mp = NULL;
+#if defined(HAVE_SCTP)
+	    if (IS_SCTP(desc)) mp = &mhdr;
+#endif
+#if !defined(__WIN32__)
+            if (desc->recv_cmsgflags) mp = &mhdr;
 #endif
 	    /* Actual parsing and return of the data received, occur here: */
 	    code = packet_reply_binary_data(desc, len, udesc->i_buf,
 					    (sizeof(other) - len),
 					    nsz,
-					    extra);
+					    mp);
 	    free_buffer(udesc->i_buf);
 	    udesc->i_buf = NULL;
 	    if (code < 0)
-- 
cgit v1.2.3


From d4057c1c1ea711aaa8a39591bf38920f93530f5c Mon Sep 17 00:00:00 2001
From: Maxim Fedorov <dane@whatsapp.com>
Date: Tue, 28 Aug 2018 16:11:09 -0700
Subject: Fix an endless rescheduling loop when a process is executing
 process_info(self(), ...)

It is possible that a process has to yield before completing process_info BIF when it runs out of reductions. If this BIF is called by the process itself, it does not send a signal but executes in the context of a process. If it has to yield, it turns F_LOCAL_SIGS_ONLY flag on, which means new signals won't be fetched from the outer message queue.
When the same process needs to execute dirty system code (e.g. dirty GC) it has to be run on a dirty scheduler. However signals enqueued into outer queue cause it to be rescheduled on a normal scheduler. F_LOCAL_SIGS_ONLY prevent outer queue signals delivery, creating an endless rescheduling loop.

This commit disengages F_LOCAL_SIG_ONLY if process needs to execute dirty code in order to complete signal delivery and allow process to be moved to dirty run queue.
---
 erts/emulator/beam/erl_process.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 9f3dfd6c37..0f7f1598fd 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9641,7 +9641,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
             if (state & ERTS_PSFLG_RUNNING_SYS) {
                 if (state & (ERTS_PSFLG_SIG_Q|ERTS_PSFLG_SIG_IN_Q)) {
                     int local_only = (!!(p->flags & F_LOCAL_SIGS_ONLY)
-                                      & !(state & ERTS_PSFLG_SUSPENDED));
+                                      & !(state & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLGS_DIRTY_WORK)));
                     if (!local_only | !!(state & ERTS_PSFLG_SIG_Q)) {
                         int sig_reds;
                         /*
-- 
cgit v1.2.3


From a31216200bdee2c04b3fb3ae5e26607674715c8a Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 5 Sep 2018 16:04:51 +0200
Subject: Prevent inconsistent node lists

If net_kernel "forgets" to abort a connection (as it currently might),
the garbage collection of a distribution entry could cause node lists
to enter an inconsistent state.
---
 erts/emulator/beam/dist.c                   |  6 +++++
 erts/emulator/beam/dist.h                   |  2 ++
 erts/emulator/beam/erl_node_tables.c        | 38 +++++++++++++++++++++++++++++
 erts/emulator/test/node_container_SUITE.erl | 28 +++++++++++++++++++--
 4 files changed, 72 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 70474898b2..16c4d689a5 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -3628,6 +3628,12 @@ static Sint abort_connection(DistEntry* dep, Uint32 conn_id)
     return 0;
 }
 
+Sint
+erts_abort_connection(DistEntry *dep, Uint32 conn_id)
+{
+    return abort_connection(dep, conn_id);
+}
+
 BIF_RETTYPE erts_internal_abort_connection_2(BIF_ALIST_2)
 {
     DistEntry* dep;
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index dda2029a4c..30b4b35c20 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -399,5 +399,7 @@ extern void erts_kill_dist_connection(DistEntry *dep, Uint32);
 
 extern Uint erts_dist_cache_size(void);
 
+extern Sint erts_abort_connection(DistEntry *dep, Uint32 conn_id);
+
 
 #endif
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 1f147011a8..9b34af1480 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -412,6 +412,44 @@ static void schedule_delete_dist_entry(DistEntry* dep)
 static void
 start_timer_delete_dist_entry(void *vdep)
 {
+    DistEntry *dep = vdep;
+    Eterm sysname;
+    enum dist_entry_state state;
+    Uint32 connection_id;
+
+    erts_de_rlock(dep);
+    state = dep->state;
+    connection_id = dep->connection_id;
+    sysname = dep->sysname;
+    erts_de_runlock(dep);
+
+    if (state != ERTS_DE_STATE_IDLE) {
+        char *state_str;
+        erts_dsprintf_buf_t *dsbuf = erts_create_logger_dsbuf();
+        switch (state) {
+        case ERTS_DE_STATE_CONNECTED:
+            state_str = "connected";
+            break;
+        case ERTS_DE_STATE_PENDING:
+            state_str = "pending connect";
+            break;
+        case ERTS_DE_STATE_EXITING:
+            state_str = "exiting";
+            break;
+        case ERTS_DE_STATE_IDLE:
+            state_str = "idle";
+            break;
+        default:
+            state_str = "unknown";
+            break;
+        }
+        erts_dsprintf(dsbuf, "Garbage collecting distribution "
+                      "entry for node %T in state: %s",
+                      sysname, state_str);
+        erts_send_error_to_logger_nogl(dsbuf);
+        erts_abort_connection(dep, connection_id);
+    }
+    
     if (node_tab_delete_delay == 0) {
         prepare_try_delete_dist_entry(vdep);
     }
diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl
index 7df001fec5..55135fbcbc 100644
--- a/erts/emulator/test/node_container_SUITE.erl
+++ b/erts/emulator/test/node_container_SUITE.erl
@@ -50,7 +50,8 @@
          bad_nc/1,
          unique_pid/1,
          iter_max_procs/1,
-         magic_ref/1]).
+         magic_ref/1,
+         dist_entry_gc/1]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
@@ -58,7 +59,7 @@ suite() ->
 
 
 all() -> 
-    [term_to_binary_to_term_eq, round_trip_eq, cmp, ref_eq,
+    [dist_entry_gc, term_to_binary_to_term_eq, round_trip_eq, cmp, ref_eq,
      node_table_gc, dist_link_refc, dist_monitor_refc,
      node_controller_refc, ets_refc, match_spec_refc,
      timer_refc, pid_wrap, port_wrap, bad_nc,
@@ -894,6 +895,29 @@ magic_ref(Config) when is_list(Config) ->
     true = is_reference(MRef2),
     true = erts_debug:get_internal_state({magic_ref,MRef2}),
     ok.
+
+
+lost_pending_connection(Node) ->
+    _ = (catch erts_internal:new_connection(Node)),
+    ok.
+
+dist_entry_gc(Config) when is_list(Config) ->
+    Me = self(),
+    {ok, Node} = start_node(get_nodefirstname(), "+zdntgc 0"),
+    P = spawn_link(Node,
+                   fun () ->
+                           LostNode = list_to_atom("lost_pending_connection@" ++ hostname()),
+                           lost_pending_connection(LostNode),
+                           garbage_collect(), %% Could crash...
+                           Me ! {self(), ok}
+                   end),
+    receive
+        {P, ok} -> ok
+    end,
+    unlink(P),
+    stop_node(Node),
+    ok.
+
 %%
 %% -- Internal utils ---------------------------------------------------------
 %%
-- 
cgit v1.2.3


From da4c24bf8fc7bb2ee0d0a66d9fcfe6344d7c0c8a Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 10 Sep 2018 10:47:52 +0200
Subject: Restore default SIGTERM behaviour for port programs

erl_child_setup program ignores TERM signals as of ERTS version
10.0 (cff8dce0). This setting was unfortunately inherited by
port programs. This commit restores handling of TERM signals
in port programs to the default behavior. That is, terminate the
process.
---
 erts/emulator/sys/unix/erl_child_setup.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/erl_child_setup.c b/erts/emulator/sys/unix/erl_child_setup.c
index 221ee2a69d..129861ebd5 100644
--- a/erts/emulator/sys/unix/erl_child_setup.c
+++ b/erts/emulator/sys/unix/erl_child_setup.c
@@ -133,6 +133,7 @@ static int sigchld_pipe[2];
 static int
 start_new_child(int pipes[])
 {
+    struct sigaction sa;
     int errln = -1;
     int size, res, i, pos = 0;
     char *buff, *o_buff;
@@ -143,6 +144,16 @@ start_new_child(int pipes[])
 
     /* only child executes here */
 
+    /* Restore default handling of sigterm... */
+    sa.sa_handler = SIG_DFL;
+    sigemptyset(&sa.sa_mask);
+    sa.sa_flags = 0;
+
+    if (sigaction(SIGTERM, &sa, 0) == -1) {
+        perror(NULL);
+        exit(1);
+    }
+    
     do {
         res = read(pipes[0], (char*)&size, sizeof(size));
     } while(res < 0 && (errno == EINTR || errno == ERRNO_BLOCK));
-- 
cgit v1.2.3


From 84e1631071858fdfd04109129c020760bb952362 Mon Sep 17 00:00:00 2001
From: Raimo Niskanen <raimo@erlang.org>
Date: Mon, 10 Sep 2018 16:43:30 +0200
Subject: Fix term buffer overflow bug

---
 erts/emulator/drivers/common/inet_drv.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index f9a471afd5..3478ba7081 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -625,13 +625,14 @@ static size_t my_strnlen(const char *s, size_t maxlen)
 #endif
 
 #ifndef __WIN32__
-/* Calculate CMSG_NXTHDR without having a struct msghdr*
+/* Calculate CMSG_NXTHDR without having a struct msghdr*.
  * CMSG_LEN only caters for alignment for start of data.
  * To get how much to advance we need to use CMSG_SPACE
  * on the payload length.  To get the payload length we
  * take the calculated cmsg->cmsg_len and subtract the
  * header length.  To get the header length we use
- * CMSG_LEN with payload length 0.
+ * the pointer difference from the cmsg start pointer
+ * to the CMSG_DATA(cmsg) pointer.
  */
 #define LEN_CMSG_DATA(cmsg)  ((char*)CMSG_DATA(cmsg) - (char*)(cmsg))
 #define NXT_CMSG_HDR(cmsg)                                              \
@@ -946,8 +947,13 @@ static size_t my_strnlen(const char *s, size_t maxlen)
 #ifdef  HAVE_SCTP
 #define PACKET_ERL_DRV_TERM_DATA_LEN  512
 #else
+#ifndef __WIN32__
+/* Assume we have recvmsg() and might need room for ancillary data */
+#define PACKET_ERL_DRV_TERM_DATA_LEN  64
+#else
 #define PACKET_ERL_DRV_TERM_DATA_LEN  32
 #endif
+#endif
 
 
 #define BIN_REALLOC_MARGIN(x)  ((x)/4)  /* 25% */
@@ -12658,10 +12664,10 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
 		}
 	    }
             mp = NULL;
-#if defined(HAVE_SCTP)
+#ifdef HAVE_SCTP
 	    if (IS_SCTP(desc)) mp = &mhdr;
 #endif
-#if !defined(__WIN32__)
+#ifndef __WIN32__
             if (desc->recv_cmsgflags) mp = &mhdr;
 #endif
 	    /* Actual parsing and return of the data received, occur here: */
-- 
cgit v1.2.3


From 49504906b4a16b69e52beb09e92f68c6ccd51753 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 11 Sep 2018 16:56:29 +0200
Subject: erts: Fix "Prevent inconsistent node lists" fix

done in a31216200bdee2c04b3fb3ae5e26607674715c8a
that could cause a new pending connection to be incorrectly aborted.
---
 erts/emulator/beam/dist.c            | 23 +++++-----
 erts/emulator/beam/dist.h            |  2 +-
 erts/emulator/beam/erl_node_tables.c | 83 +++++++++++++++++++-----------------
 3 files changed, 57 insertions(+), 51 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 16c4d689a5..1e822d5c7b 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -3563,20 +3563,18 @@ BIF_RETTYPE erts_internal_new_connection_1(BIF_ALIST_1)
     BIF_RET(TUPLE2(hp, make_small(conn_id), dhandle));
 }
 
-static Sint abort_connection(DistEntry* dep, Uint32 conn_id)
+Sint erts_abort_connection_rwunlock(DistEntry* dep)
 {
-    erts_de_rwlock(dep);
+    Sint reds = 0;
+    ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep));
 
-    if (dep->connection_id != conn_id)
-        ;
-    else if (dep->state == ERTS_DE_STATE_CONNECTED) {
+    if (dep->state == ERTS_DE_STATE_CONNECTED) {
         kill_connection(dep);
     }
     else if (dep->state == ERTS_DE_STATE_PENDING) {
         ErtsAtomCache *cache;
         ErtsDistOutputBuf *obuf;
         ErtsProcList *resume_procs;
-        Sint reds = 0;
         ErtsMonLnkDist *mld;
 
 	ASSERT(is_nil(dep->cid));
@@ -3621,17 +3619,18 @@ static Sint abort_connection(DistEntry* dep, Uint32 conn_id)
         erts_de_rwlock(dep);
         ASSERT(dep->state == ERTS_DE_STATE_EXITING);
         dep->state = ERTS_DE_STATE_IDLE;
-        erts_de_rwunlock(dep);
-        return reds;
     }
     erts_de_rwunlock(dep);
-    return 0;
+    return reds;
 }
 
-Sint
-erts_abort_connection(DistEntry *dep, Uint32 conn_id)
+static Sint abort_connection(DistEntry *dep, Uint32 conn_id)
 {
-    return abort_connection(dep, conn_id);
+    erts_de_rwlock(dep);
+    if (dep->connection_id == conn_id)
+        return erts_abort_connection_rwunlock(dep);
+    erts_de_rwunlock(dep);
+    return 0;
 }
 
 BIF_RETTYPE erts_internal_abort_connection_2(BIF_ALIST_2)
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 30b4b35c20..75cb865390 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -399,7 +399,7 @@ extern void erts_kill_dist_connection(DistEntry *dep, Uint32);
 
 extern Uint erts_dist_cache_size(void);
 
-extern Sint erts_abort_connection(DistEntry *dep, Uint32 conn_id);
+extern Sint erts_abort_connection_rwunlock(DistEntry *dep);
 
 
 #endif
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 9b34af1480..f4a36d124a 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -60,6 +60,10 @@ static int references_atoms_need_init = 1;
 static ErtsMonotonicTime orig_node_tab_delete_delay;
 static ErtsMonotonicTime node_tab_delete_delay;
 
+
+static void report_gc_active_dist_entry(Eterm sysname, enum dist_entry_state);
+
+
 /* -- The distribution table ---------------------------------------------- */
 
 #define ErtsBin2DistEntry(B) \
@@ -412,44 +416,6 @@ static void schedule_delete_dist_entry(DistEntry* dep)
 static void
 start_timer_delete_dist_entry(void *vdep)
 {
-    DistEntry *dep = vdep;
-    Eterm sysname;
-    enum dist_entry_state state;
-    Uint32 connection_id;
-
-    erts_de_rlock(dep);
-    state = dep->state;
-    connection_id = dep->connection_id;
-    sysname = dep->sysname;
-    erts_de_runlock(dep);
-
-    if (state != ERTS_DE_STATE_IDLE) {
-        char *state_str;
-        erts_dsprintf_buf_t *dsbuf = erts_create_logger_dsbuf();
-        switch (state) {
-        case ERTS_DE_STATE_CONNECTED:
-            state_str = "connected";
-            break;
-        case ERTS_DE_STATE_PENDING:
-            state_str = "pending connect";
-            break;
-        case ERTS_DE_STATE_EXITING:
-            state_str = "exiting";
-            break;
-        case ERTS_DE_STATE_IDLE:
-            state_str = "idle";
-            break;
-        default:
-            state_str = "unknown";
-            break;
-        }
-        erts_dsprintf(dsbuf, "Garbage collecting distribution "
-                      "entry for node %T in state: %s",
-                      sysname, state_str);
-        erts_send_error_to_logger_nogl(dsbuf);
-        erts_abort_connection(dep, connection_id);
-    }
-    
     if (node_tab_delete_delay == 0) {
         prepare_try_delete_dist_entry(vdep);
     }
@@ -489,6 +455,19 @@ static void try_delete_dist_entry(DistEntry* dep)
 {
     erts_aint_t refc;
 
+    erts_de_rwlock(dep);
+    if (dep->state != ERTS_DE_STATE_IDLE && de_refc_read(dep,0) == 0) {
+        Eterm sysname = dep->sysname;
+        enum dist_entry_state state  = dep->state;
+
+        if (dep->state != ERTS_DE_STATE_PENDING)
+            ERTS_INTERNAL_ERROR("Garbage collecting connected distribution entry");
+        erts_abort_connection_rwunlock(dep);
+        report_gc_active_dist_entry(sysname, state);
+    }
+    else
+        erts_de_rwunlock(dep);
+
     erts_rwmtx_rwlock(&erts_dist_table_rwmtx);
     /*
      * Another thread might have looked up this dist entry after
@@ -515,6 +494,34 @@ static void try_delete_dist_entry(DistEntry* dep)
     }
 }
 
+static void report_gc_active_dist_entry(Eterm sysname,
+                                        enum dist_entry_state state)
+{
+    char *state_str;
+    erts_dsprintf_buf_t *dsbuf = erts_create_logger_dsbuf();
+    switch (state) {
+    case ERTS_DE_STATE_CONNECTED:
+        state_str = "connected";
+        break;
+    case ERTS_DE_STATE_PENDING:
+        state_str = "pending connect";
+        break;
+    case ERTS_DE_STATE_EXITING:
+        state_str = "exiting";
+        break;
+    case ERTS_DE_STATE_IDLE:
+        state_str = "idle";
+        break;
+    default:
+        state_str = "unknown";
+        break;
+    }
+    erts_dsprintf(dsbuf, "Garbage collecting distribution "
+                  "entry for node %T in state: %s",
+                  sysname, state_str);
+    erts_send_error_to_logger_nogl(dsbuf);
+}
+
 int erts_dist_entry_destructor(Binary *bin)
 {
     DistEntry *dep = ErtsBin2DistEntry(bin);
-- 
cgit v1.2.3


From d384fb7c6edd33161fa5d2c56745427da32e9aa5 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 10 Sep 2018 15:38:38 +0200
Subject: erts: Fix bug in undocumented system_flag(scheduling_statistics)

Uses port specific data out of bounds.
---
 erts/emulator/beam/erl_port.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index 9b52b648e5..cf4a233b54 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -257,6 +257,8 @@ ERTS_GLB_INLINE void *
 erts_prtsd_get(Port *prt, int ix)
 {
     ErtsPrtSD *psd = (ErtsPrtSD *) erts_atomic_read_nob(&prt->psd);
+
+    ASSERT((unsigned)ix < ERTS_PRTSD_SIZE);
     if (!psd)
 	return NULL;
     ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
@@ -272,6 +274,7 @@ erts_prtsd_set(Port *prt, int ix, void *data)
 
     psd = (ErtsPrtSD *) erts_atomic_read_nob(&prt->psd);
 
+    ASSERT((unsigned)ix < ERTS_PRTSD_SIZE);
     if (psd) {
 #ifdef ETHR_ORDERED_READ_DEPEND
 	ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
@@ -459,7 +462,7 @@ erts_port_unlock(Port *prt)
   ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP)
 
 #define ERTS_PORT_SCHED_ID(P, ID) \
-  ((Uint) (UWord) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (UWord) (ID)))
+  ((Uint) (UWord) erts_prtsd_set((P), ERTS_PRTSD_SCHED_ID, (void *) (UWord) (ID)))
 
 extern const Port erts_invalid_port;
 #define ERTS_PORT_LOCK_BUSY ((Port *) &erts_invalid_port)
-- 
cgit v1.2.3


From 6f5a0d5f83668d4b53fcce9a6c927e30df169765 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 19 Jul 2018 11:41:06 +0200
Subject: Improve trapping in lists:reverse/2

If the process had more free space than reductions it could run a
lot longer than it was supposed to. It didn't honor the number of
reductions going in either, nor did it bump reductions when
returning its result or erroring out.

This commit also removes this function from a work function in
scheduler_SUITE as it's extremely sensitive to the number of
reductions spent in the test, causing
equal_and_high_with_part_time_max to fail on some machines.
---
 erts/emulator/beam/erl_bif_lists.c     | 160 +++++++++++++++++++++------------
 erts/emulator/test/scheduler_SUITE.erl |   4 +-
 2 files changed, 106 insertions(+), 58 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index 73d327da3e..01e09da34c 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -277,75 +277,121 @@ BIF_RETTYPE lists_member_2(BIF_ALIST_2)
     BIF_RET2(am_false, CONTEXT_REDS - max_iter/10);
 }
 
-BIF_RETTYPE lists_reverse_2(BIF_ALIST_2)
+static BIF_RETTYPE lists_reverse_alloc(Process *c_p,
+                                       Eterm list_in,
+                                       Eterm tail_in)
 {
-    Eterm list;
-    Eterm tmp_list;
-    Eterm result;
-    Eterm* hp;
-    Uint n;
-    int max_iter;
-    
-    /*
-     * Handle legal and illegal non-lists quickly.
-     */
-    if (is_nil(BIF_ARG_1)) {
-	BIF_RET(BIF_ARG_2);
-    } else if (is_not_list(BIF_ARG_1)) {
-    error:
-	BIF_ERROR(BIF_P, BADARG);
+    static const Uint CELLS_PER_RED = 40;
+
+    Eterm *heap_top, *heap_end;
+    Uint cells_left, max_cells;
+    Eterm list, tail;
+    Eterm lookahead;
+
+    list = list_in;
+    tail = tail_in;
+
+    cells_left = max_cells = CELLS_PER_RED * (1 + ERTS_BIF_REDS_LEFT(c_p));
+    lookahead = list;
+
+    while (cells_left != 0 && is_list(lookahead)) {
+        lookahead = CDR(list_val(lookahead));
+        cells_left--;
     }
 
-    /*
-     * First use the rest of the remaning heap space.
-     */
-    list = BIF_ARG_1;
-    result = BIF_ARG_2;
-    hp = HEAP_TOP(BIF_P);
-    n = HeapWordsLeft(BIF_P) / 2;
-    while (n != 0 && is_list(list)) {
-	Eterm* pair = list_val(list);
-	result = CONS(hp, CAR(pair), result);
-	list = CDR(pair);
-	hp += 2;
-	n--;
+    BUMP_REDS(c_p, (max_cells - cells_left) / CELLS_PER_RED);
+
+    if (is_not_list(lookahead) && is_not_nil(lookahead)) {
+        BIF_ERROR(c_p, BADARG);
     }
-    HEAP_TOP(BIF_P) = hp;
+
+    heap_top = HAlloc(c_p, 2 * (max_cells - cells_left));
+    heap_end = heap_top + 2 * (max_cells - cells_left);
+
+    while (heap_top < heap_end) {
+        Eterm *pair = list_val(list);
+
+        tail = CONS(heap_top, CAR(pair), tail);
+        list = CDR(pair);
+
+        ASSERT(is_list(list) || is_nil(list));
+
+        heap_top += 2;
+    }
+
     if (is_nil(list)) {
-	BIF_RET(result);
+        BIF_RET(tail);
     }
 
-    /*
-     * Calculate length of remaining list (up to a suitable limit).
-     */
-    max_iter = CONTEXT_REDS * 40;
-    n = 0;
-    tmp_list = list;
-    while (max_iter-- > 0 && is_list(tmp_list)) {
-	tmp_list = CDR(list_val(tmp_list));
-	n++;
+    ASSERT(is_list(tail) && cells_left == 0);
+    BIF_TRAP2(bif_export[BIF_lists_reverse_2], c_p, list, tail);
+}
+
+static BIF_RETTYPE lists_reverse_onheap(Process *c_p,
+                                        Eterm list_in,
+                                        Eterm tail_in)
+{
+    static const Uint CELLS_PER_RED = 60;
+
+    Eterm *heap_top, *heap_end;
+    Uint cells_left, max_cells;
+    Eterm list, tail;
+
+    list = list_in;
+    tail = tail_in;
+
+    cells_left = max_cells = CELLS_PER_RED * (1 + ERTS_BIF_REDS_LEFT(c_p));
+
+    ASSERT(HEAP_LIMIT(c_p) >= HEAP_TOP(c_p) + 2);
+    heap_end = HEAP_LIMIT(c_p) - 2;
+    heap_top = HEAP_TOP(c_p);
+
+    while (heap_top < heap_end && is_list(list)) {
+        Eterm *pair = list_val(list);
+
+        tail = CONS(heap_top, CAR(pair), tail);
+        list = CDR(pair);
+
+        heap_top += 2;
     }
-    if (is_not_nil(tmp_list) && is_not_list(tmp_list)) {
-	goto error;
+
+    cells_left -= (heap_top - heap_end) / 2;
+    BUMP_REDS(c_p, (max_cells - cells_left) / CELLS_PER_RED);
+    HEAP_TOP(c_p) = heap_top;
+
+    if (is_nil(list)) {
+        BIF_RET(tail);
+    } else if (is_list(list)) {
+        ASSERT(is_list(tail));
+
+        if (cells_left > CELLS_PER_RED) {
+            return lists_reverse_alloc(c_p, list, tail);
+        }
+
+        BUMP_ALL_REDS(c_p);
+        BIF_TRAP2(bif_export[BIF_lists_reverse_2], c_p, list, tail);
     }
 
-    /*
-     * Now do one HAlloc() and continue reversing.
-     */
-    hp = HAlloc(BIF_P, 2*n);
-    while (n != 0 && is_list(list)) {
-	Eterm* pair = list_val(list);
-	result = CONS(hp, CAR(pair), result);
-	list = CDR(pair);
-	hp += 2;
-	n--;
+    BIF_ERROR(c_p, BADARG);
+}
+
+BIF_RETTYPE lists_reverse_2(BIF_ALIST_2)
+{
+    /* Handle legal and illegal non-lists quickly. */
+    if (is_nil(BIF_ARG_1)) {
+        BIF_RET(BIF_ARG_2);
+    } else if (is_not_list(BIF_ARG_1)) {
+        BIF_ERROR(BIF_P, BADARG);
     }
-    if (is_nil(list)) {
-	BIF_RET(result);
-    } else {
-	BUMP_ALL_REDS(BIF_P);
-	BIF_TRAP2(bif_export[BIF_lists_reverse_2], BIF_P, list, result);
+
+    /* We build the reversal on the unused part of the heap if possible to save
+     * us the trouble of having to figure out the list size. We fall back to
+     * lists_reverse_alloc when we run out of space. */
+    if (HeapWordsLeft(BIF_P) > 8) {
+        return lists_reverse_onheap(BIF_P, BIF_ARG_1, BIF_ARG_2);
     }
+
+    return lists_reverse_alloc(BIF_P, BIF_ARG_1, BIF_ARG_2);
 }
 
 BIF_RETTYPE
diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl
index 7eebbe8b19..d98edd1ec2 100644
--- a/erts/emulator/test/scheduler_SUITE.erl
+++ b/erts/emulator/test/scheduler_SUITE.erl
@@ -2155,7 +2155,7 @@ workers_exit([Ps|Pss])  ->
     workers_exit(Pss).
 
 do_work(PartTime) ->
-    lists:reverse(lists:seq(1, 50)),
+    _ = id(lists:seq(1, 50)),
     receive stop_work -> receive after infinity -> ok end after 0 -> ok end,
     case PartTime of
 	true -> receive after 1 -> ok end;
@@ -2163,6 +2163,8 @@ do_work(PartTime) ->
     end,
     do_work(PartTime).
 
+id(I) -> I.
+
 workers(N, _Prio, _PartTime) when N =< 0 ->
     [];
 workers(N, Prio, PartTime) ->
-- 
cgit v1.2.3


From a04f3246c1eda7b8d8d83ba2bcc46d502b80d22b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 18 Sep 2018 14:27:27 +0200
Subject: Consolidate distribution entry state transitions

* Make connection_id part of the distribution handle as {ConnId, DistEntry}
  in order for BIFs to verify correct connection.
* Make distribution handle opaque to net_kernel.
* Remove some unsafe lockless reads of DistEntry.flags
* Change state ERTS_DE_STATE_EXITING to be more of an internal state that
  prevents erts from enqueue, encode or schedule new data to be sent. Otherwise
  it should behave like ERTS_DE_STATE_CONNECTED.
---
 erts/emulator/beam/dist.c            | 312 ++++++++++++++---------------------
 erts/emulator/beam/dist.h            |   2 +
 erts/emulator/beam/erl_node_tables.c |  30 +++-
 erts/emulator/beam/erl_node_tables.h |   7 +-
 erts/emulator/beam/erl_port.h        |   1 +
 erts/emulator/beam/external.c        |  47 +++---
 erts/emulator/beam/external.h        |   2 +-
 erts/emulator/beam/global.h          |   2 +-
 erts/emulator/beam/io.c              |   3 +
 9 files changed, 182 insertions(+), 224 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index ceb89a6910..9efb7e79ac 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -116,11 +116,12 @@ static Export *dist_ctrl_put_data_trap;
 
 /* forward declarations */
 
-static void clear_dist_entry(DistEntry*);
 static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy);
 static void send_nodes_mon_msgs(Process *, Eterm, Eterm, Eterm, Eterm);
 static void init_nodes_monitors(void);
 static Sint abort_connection(DistEntry* dep, Uint32 conn_id);
+static ErtsDistOutputBuf* clear_de_out_queues(DistEntry*);
+static void free_de_out_queues(DistEntry*, ErtsDistOutputBuf*);
 
 static erts_atomic_t no_caches;
 static erts_atomic_t no_nodes;
@@ -556,7 +557,10 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 	}
     }
     else { /* Call from distribution controller (port/process) */
-	ErtsMonLnkDist *mld;
+        ErtsMonLnkDist *mld;
+        ErtsAtomCache *cache;
+        ErtsProcList *suspendees;
+        ErtsDistOutputBuf *obuf;
 	Uint32 flags;
 
 	erts_atomic_set_mb(&dep->dist_cmd_scheduled, 1);
@@ -588,6 +592,22 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 	nodename = dep->sysname;
 	flags = dep->flags;
 
+        erts_atomic_set_nob(&dep->input_handler, (erts_aint_t) NIL);
+        cache = dep->cache;
+        dep->cache = NULL;
+
+        erts_mtx_lock(&dep->qlock);
+
+        erts_atomic64_set_nob(&dep->in, 0);
+        erts_atomic64_set_nob(&dep->out, 0);
+
+        obuf = clear_de_out_queues(dep);
+        suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL);
+
+        erts_mtx_unlock(&dep->qlock);
+        erts_atomic_set_nob(&dep->dist_cmd_scheduled, 0);
+        dep->send = NULL;
+
 	erts_set_dist_entry_not_connected(dep);
 
 	erts_de_rwunlock(dep);
@@ -601,7 +621,13 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
                                            ? am_connection_closed
                                            : reason));
 
-	clear_dist_entry(dep);
+        erts_resume_processes(suspendees);
+
+        delete_cache(cache);
+
+        free_de_out_queues(dep, obuf);
+        if (dep->transcode_ctx)
+            transcode_free_ctx(dep);
     }
 
     dec_no_nodes();
@@ -732,41 +758,6 @@ static void free_de_out_queues(DistEntry* dep, ErtsDistOutputBuf *obuf)
     }
 }
 
-static void clear_dist_entry(DistEntry *dep)
-{
-    ErtsAtomCache *cache;
-    ErtsProcList *suspendees;
-    ErtsDistOutputBuf *obuf;
-
-    erts_de_rwlock(dep);
-    erts_atomic_set_nob(&dep->input_handler,
-                        (erts_aint_t) NIL);
-    cache = dep->cache;
-    dep->cache = NULL;
-
-    erts_mtx_lock(&dep->qlock);
-
-    erts_atomic64_set_nob(&dep->in, 0);
-    erts_atomic64_set_nob(&dep->out, 0);
-
-    obuf = clear_de_out_queues(dep);
-    dep->state = ERTS_DE_STATE_IDLE;
-    suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL);
-
-    erts_mtx_unlock(&dep->qlock);
-    erts_atomic_set_nob(&dep->dist_cmd_scheduled, 0);
-    dep->send = NULL;
-    erts_de_rwunlock(dep);
-
-    erts_resume_processes(suspendees);
-
-    delete_cache(cache);
-
-    free_de_out_queues(dep, obuf);
-    if (dep->transcode_ctx)
-        transcode_free_ctx(dep);
-}
-
 int erts_dsend_context_dtor(Binary* ctx_bin)
 {
     ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(ctx_bin);
@@ -861,7 +852,7 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
     DeclareTmpHeapNoproc(ctl_heap,6);
     int res;
 
-    if (~dsdp->dep->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) {
+    if (~dsdp->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) {
         /*
          * Receiver does not support DOP_MONITOR_P_EXIT (see dsig_send_monitor)
          */
@@ -889,7 +880,7 @@ erts_dsig_send_monitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
     DeclareTmpHeapNoproc(ctl_heap,5);
     int res;
 
-    if (~dsdp->dep->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) {
+    if (~dsdp->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) {
         /*
          * Receiver does not support DOP_MONITOR_P.
          * Just avoid sending it and by doing that reduce this monitor
@@ -920,7 +911,7 @@ erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher,
     DeclareTmpHeapNoproc(ctl_heap,5);
     int res;
 
-    if (~dsdp->dep->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) {
+    if (~dsdp->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) {
         /*
          * Receiver does not support DOP_DEMONITOR_P (see dsig_send_monitor)
          */
@@ -940,7 +931,7 @@ erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher,
 static int can_send_seqtrace_token(ErtsSendContext* ctx, Eterm token) {
     Eterm label;
 
-    if (ctx->dep->flags & DFLAG_BIG_SEQTRACE_LABELS) {
+    if (ctx->dsd.flags & DFLAG_BIG_SEQTRACE_LABELS) {
         /* The other end is capable of handling arbitrary seq_trace labels. */
         return 1;
     }
@@ -1001,7 +992,7 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx)
 
         send_token = (token != NIL && can_send_seqtrace_token(ctx, token));
 
-        if (ctx->dep->flags & DFLAG_SEND_SENDER) {
+        if (ctx->dsd.flags & DFLAG_SEND_SENDER) {
             dist_op = make_small(send_token ?
                                  DOP_SEND_SENDER_TT :
                                  DOP_SEND_SENDER);
@@ -1218,13 +1209,13 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote)
 
 int erts_net_message(Port *prt,
 		     DistEntry *dep,
+                     Uint32 conn_id,
 		     byte *hbuf,
 		     ErlDrvSizeT hlen,
 		     byte *buf,
 		     ErlDrvSizeT len)
 {
     ErtsDistExternal ede;
-    byte *t;
     Sint ctl_len;
     Eterm arg;
     Eterm from, to;
@@ -1242,7 +1233,6 @@ int erts_net_message(Port *prt,
     Eterm token_size;
     Uint tuple_arity;
     int res;
-    Uint32 connection_id;
 #ifdef ERTS_DIST_MSG_DBG
     ErlDrvSizeT orig_len = len;
 #endif
@@ -1258,7 +1248,6 @@ int erts_net_message(Port *prt,
 	return 0;
     }
 
-
     ASSERT(hlen == 0);
 
     if (len == 0) {  /* HANDLE TICK !!! */
@@ -1271,15 +1260,7 @@ int erts_net_message(Port *prt,
     bw(buf, len);
 #endif
 
-    if (dep->flags & DFLAG_DIST_HDR_ATOM_CACHE)
-	t = buf;
-    else {
-	/* Skip PASS_THROUGH */
-	t = buf+1;
-	len--;
-    }
-
-    res = erts_prepare_dist_ext(&ede, t, len, dep, dep->cache, &connection_id);
+    res = erts_prepare_dist_ext(&ede, buf, len, dep, conn_id, dep->cache);
 
     switch (res) {
     case ERTS_PREP_DIST_EXT_CLOSED:
@@ -1321,10 +1302,9 @@ int erts_net_message(Port *prt,
 	PURIFY_MSG("data error");
 	goto decode_error;
     }
-    ctl_len = t - buf;
 
 #ifdef ERTS_DIST_MSG_DBG
-    erts_fprintf(stderr, "<<%s CTL: %T\n", len != orig_len ? "P" : " ", arg);
+    erts_fprintf(stderr, "<< CTL: %T\n", arg);
 #endif
 
     if (is_not_tuple(arg) || 
@@ -1778,7 +1758,7 @@ decode_error:
     }
 data_error:
     UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
-    erts_kill_dist_connection(dep, connection_id);
+    erts_kill_dist_connection(dep, conn_id);
     ERTS_CHK_NO_PROC_LOCKS;
     return -1;
 }
@@ -1834,7 +1814,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx)
     while (1) {
 	switch (ctx->phase) {
 	case ERTS_DSIG_SEND_PHASE_INIT:
-	    ctx->flags = dsdp->dep->flags;
+	    ctx->flags = dsdp->flags;
 	    ctx->c_p = dsdp->proc;
 
 	    if (!ctx->c_p || dsdp->no_suspend)
@@ -2571,11 +2551,12 @@ dist_ctrl_get_data_notification_1(BIF_ALIST_1)
     erts_aint32_t qflgs;
     erts_aint_t qsize;
     Eterm receiver = NIL;
+    Uint32 conn_id;
 
     if (!dep)
         BIF_ERROR(BIF_P, EXC_NOTSUP);
 
-    if (erts_dhandle_to_dist_entry(BIF_ARG_1) != dep)
+    if (erts_dhandle_to_dist_entry(BIF_ARG_1, &conn_id) != dep)
         BIF_ERROR(BIF_P, BADARG);
 
     /*
@@ -2585,6 +2566,11 @@ dist_ctrl_get_data_notification_1(BIF_ALIST_1)
 
     erts_de_rlock(dep);
 
+    if (dep->connection_id != conn_id) {
+        erts_de_runlock(dep);
+        BIF_ERROR(BIF_P, BADARG);
+    }
+
     ASSERT(dep->cid == BIF_P->common.id);
 
     qflgs = erts_atomic32_read_acqb(&dep->qflgs);
@@ -2625,6 +2611,7 @@ dist_ctrl_put_data_2(BIF_ALIST_2)
     DistEntry *dep;
     ErlDrvSizeT size;
     Eterm input_handler;
+    Uint32 conn_id;
 
     if (is_binary(BIF_ARG_2))
         size = binary_size(BIF_ARG_2);
@@ -2636,7 +2623,7 @@ dist_ctrl_put_data_2(BIF_ALIST_2)
     else
         BIF_ERROR(BIF_P, BADARG);
 
-    dep = erts_dhandle_to_dist_entry(BIF_ARG_1);
+    dep = erts_dhandle_to_dist_entry(BIF_ARG_1, &conn_id);
     if (!dep)
         BIF_ERROR(BIF_P, BADARG);
 
@@ -2656,7 +2643,7 @@ dist_ctrl_put_data_2(BIF_ALIST_2)
 
         erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
-        (void) erts_net_message(NULL, dep, NULL, 0, data, size);
+        (void) erts_net_message(NULL, dep, conn_id, NULL, 0, data, size);
         /*
          * We ignore any decode failures. On fatal failures the
          * connection will be taken down by killing the
@@ -2680,13 +2667,18 @@ dist_get_stat_1(BIF_ALIST_1)
     Sint64 read, write, pend;
     Eterm res, *hp, **hpp;
     Uint sz, *szp;
-    DistEntry *dep = erts_dhandle_to_dist_entry(BIF_ARG_1);
+    Uint32 conn_id;
+    DistEntry *dep = erts_dhandle_to_dist_entry(BIF_ARG_1, &conn_id);
 
     if (!dep)
         BIF_ERROR(BIF_P, BADARG);
 
     erts_de_rlock(dep);
 
+    if (dep->connection_id != conn_id) {
+        erts_de_runlock(dep);
+        BIF_ERROR(BIF_P, BADARG);
+    }
     read = (Sint64) erts_atomic64_read_nob(&dep->in);
     write = (Sint64) erts_atomic64_read_nob(&dep->out);
     pend = (Sint64) erts_atomic_read_nob(&dep->qsize);
@@ -2717,19 +2709,25 @@ BIF_RETTYPE
 dist_ctrl_input_handler_2(BIF_ALIST_2)
 {
     DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P);
+    Uint32 conn_id;
 
     if (!dep)
         BIF_ERROR(BIF_P, EXC_NOTSUP);
 
-    if (erts_dhandle_to_dist_entry(BIF_ARG_1) != dep)
+    if (erts_dhandle_to_dist_entry(BIF_ARG_1, &conn_id) != dep)
         BIF_ERROR(BIF_P, BADARG);
 
     if (is_not_internal_pid(BIF_ARG_2))
         BIF_ERROR(BIF_P, BADARG);
 
+    erts_de_rlock(dep);
+    if (dep->connection_id != conn_id) {
+        erts_de_runlock(dep);
+        BIF_ERROR(BIF_P, BADARG);
+    }
     erts_atomic_set_nob(&dep->input_handler,
                         (erts_aint_t) BIF_ARG_2);
-
+    erts_de_runlock(dep);
     BIF_RET(am_ok);
 }
 
@@ -2743,15 +2741,21 @@ dist_ctrl_get_data_1(BIF_ALIST_1)
     Eterm *hp;
     ProcBin *pb;
     erts_aint_t qsize;
+    Uint32 conn_id;
 
     if (!dep)
         BIF_ERROR(BIF_P, EXC_NOTSUP);
 
-    if (erts_dhandle_to_dist_entry(BIF_ARG_1) != dep)
+    if (erts_dhandle_to_dist_entry(BIF_ARG_1, &conn_id) != dep)
         BIF_ERROR(BIF_P, BADARG);
 
     erts_de_rlock(dep);
 
+    if (dep->connection_id != conn_id) {
+        erts_de_runlock(dep);
+        BIF_ERROR(BIF_P, BADARG);
+    }
+
     if (dep->state == ERTS_DE_STATE_EXITING)
         goto return_none;
 
@@ -2870,10 +2874,10 @@ static void kill_connection(DistEntry *dep)
 }
 
 void
-erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id)
+erts_kill_dist_connection(DistEntry *dep, Uint32 conn_id)
 {
     erts_de_rwlock(dep);
-    if (connection_id == dep->connection_id
+    if (conn_id == dep->connection_id
         && dep->state == ERTS_DE_STATE_CONNECTED) {
 
         kill_connection(dep);
@@ -3209,23 +3213,6 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4)
     else if (!dep)
 	goto system_limit; /* Should never happen!!! */
 
-    erts_de_rlock(dep);
-    de_locked = -1;
-
-    if (dep->state == ERTS_DE_STATE_EXITING) {
-        /* Suspend on dist entry waiting for the exit to finish */
-        ErtsProcList *plp = erts_proclist_create(BIF_P);
-        plp->next = NULL;
-        erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
-        erts_mtx_lock(&dep->qlock);
-        erts_proclist_store_last(&dep->suspended, plp);
-        erts_mtx_unlock(&dep->qlock);
-        goto yield;
-    }
-
-    erts_de_runlock(dep);
-    de_locked = 0;
-
     if (is_internal_pid(BIF_ARG_2)) {
         if (BIF_P->common.id == BIF_ARG_2) {
             ErtsSetupConnDistCtrl scdc;
@@ -3271,7 +3258,7 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4)
         hp = HAlloc(BIF_P, 3);
     }
     else {
-        int new;
+        Uint32 conn_id;
 
         pp = erts_id2port_sflgs(BIF_ARG_2,
                                 BIF_P,
@@ -3280,7 +3267,7 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4)
         erts_de_rwlock(dep);
         de_locked = 1;
 
-        if (dep->state == ERTS_DE_STATE_EXITING)
+        if (dep->state != ERTS_DE_STATE_PENDING)
             goto badarg;
 
         if (!pp || (erts_atomic32_read_nob(&pp->state)
@@ -3290,49 +3277,38 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4)
         if ((pp->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY) == 0)
             goto badarg;
 
-        if (dep->cid == BIF_ARG_2 && pp->dist_entry == dep)
-            new = 0;
-        else {
-            if (dep->state != ERTS_DE_STATE_PENDING) {
-                if (dep->state == ERTS_DE_STATE_IDLE)
-                    erts_set_dist_entry_pending(dep);
-                else
-                    goto badarg;
-            }
-
-            if (pp->dist_entry || is_not_nil(dep->cid))
-                goto badarg;
-
-            erts_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION);
+        if (pp->dist_entry || is_not_nil(dep->cid))
+            goto badarg;
 
-            pp->dist_entry = dep;
+        erts_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION);
 
-            ASSERT(pp->drv_ptr->outputv || pp->drv_ptr->output);
+        pp->dist_entry = dep;
+        pp->connection_id = dep->connection_id;
 
-            dep->send = (pp->drv_ptr->outputv
-                         ? dist_port_commandv
-                         : dist_port_command);
-            ASSERT(dep->send);
+        ASSERT(pp->drv_ptr->outputv || pp->drv_ptr->output);
 
-            /*
-             * Dist-ports do not use the "busy port message queue" functionality, but
-             * instead use "busy dist entry" functionality.
-             */
-            {
-                ErlDrvSizeT disable = ERL_DRV_BUSY_MSGQ_DISABLED;
-                erl_drv_busy_msgq_limits(ERTS_Port2ErlDrvPort(pp), &disable, NULL);
-            }
+        dep->send = (pp->drv_ptr->outputv
+                     ? dist_port_commandv
+                     : dist_port_command);
+        ASSERT(dep->send);
 
-            setup_connection_epiloge_rwunlock(BIF_P, dep, BIF_ARG_2, flags, version);
-            de_locked = 0;
-            new = !0;
+        /*
+         * Dist-ports do not use the "busy port message queue" functionality, but
+         * instead use "busy dist entry" functionality.
+        */
+        {
+            ErlDrvSizeT disable = ERL_DRV_BUSY_MSGQ_DISABLED;
+            erl_drv_busy_msgq_limits(ERTS_Port2ErlDrvPort(pp), &disable, NULL);
         }
 
-        hp = HAlloc(BIF_P, 3 + ERTS_MAGIC_REF_THING_SIZE);
-        res = erts_build_dhandle(&hp, &BIF_P->off_heap, dep);
+        conn_id = dep->connection_id;
+        setup_connection_epiloge_rwunlock(BIF_P, dep, BIF_ARG_2, flags, version);
+        de_locked = 0;
+
+        hp = HAlloc(BIF_P, 3 + ERTS_DHANDLE_SIZE);
+        res = erts_build_dhandle(&hp, &BIF_P->off_heap, dep, conn_id);
         res_tag = am_ok; /* Connection up */
-        if (new)
-            dep = NULL; /* inc of refc transferred to port (dist_entry field) */
+        dep = NULL; /* inc of refc transferred to port (dist_entry field) */
     }
 
     ASSERT(is_value(res) && is_value(res_tag));
@@ -3358,12 +3334,6 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4)
 
     return ret;
 
- yield:
-    ERTS_BIF_PREP_YIELD4(ret,
-                         bif_export[BIF_erts_internal_create_dist_channel_4],
-                         BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BIF_ARG_4);
-    goto done;
-
  badarg:
     ERTS_BIF_PREP_RET(ret, am_badarg);
     goto done;
@@ -3385,8 +3355,7 @@ setup_connection_epiloge_rwunlock(Process *c_p, DistEntry *dep,
     dep->creation = 0;
 
     ASSERT(is_internal_port(ctrlr) || is_internal_pid(ctrlr));
-    ASSERT(erts_atomic_read_nob(&dep->qsize) == 0 
-           || (dep->state == ERTS_DE_STATE_PENDING));
+    ASSERT(dep->state == ERTS_DE_STATE_PENDING);
 
     if (flags & DFLAG_DIST_HDR_ATOM_CACHE)
 	create_cache(dep);
@@ -3432,37 +3401,20 @@ setup_connection_distctrl(Process *c_p, void *arg, int *redsp, ErlHeapFragment *
     DistEntry *dep = scdcp->dep;
     int dep_locked = 0;
     Eterm *hp;
-    erts_aint32_t state;
+    Uint32 conn_id;
 
     if (redsp)
         *redsp = 1;
 
-    state = erts_atomic32_read_nob(&c_p->state);
-
-    if (state & ERTS_PSFLG_EXITING)
-        goto badarg;
+    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 
     erts_de_rwlock(dep);
     dep_locked = !0;
 
-    if (dep->state == ERTS_DE_STATE_EXITING)
+    if (dep->state != ERTS_DE_STATE_PENDING)
         goto badarg;
 
-    if (ERTS_PROC_GET_DIST_ENTRY(c_p)) {
-        if (dep == ERTS_PROC_GET_DIST_ENTRY(c_p)
-            && (c_p->flags & F_DISTRIBUTION)
-            && dep->cid == c_p->common.id) {
-            goto connected;
-        }
-        goto badarg;
-    }
-
-    if (dep->state != ERTS_DE_STATE_PENDING) {
-        if (dep->state == ERTS_DE_STATE_IDLE)
-            erts_set_dist_entry_pending(dep);
-        else
-            goto badarg;
-    }
+    conn_id = dep->connection_id;
 
     if (is_not_nil(dep->cid))
         goto badarg;
@@ -3477,18 +3429,17 @@ setup_connection_distctrl(Process *c_p, void *arg, int *redsp, ErlHeapFragment *
 
     setup_connection_epiloge_rwunlock(c_p, dep, c_p->common.id,
                                       scdcp->flags, scdcp->version);
-connected:
 
     /* we take over previous inc in refc of dep */
 
     if (!bpp) /* called directly... */
-        return erts_make_dhandle(c_p, dep);
+        return erts_make_dhandle(c_p, dep, conn_id);
 
     erts_free(ERTS_ALC_T_SETUP_CONN_ARG, arg);
 
-    *bpp = new_message_buffer(ERTS_MAGIC_REF_THING_SIZE);
+    *bpp = new_message_buffer(ERTS_DHANDLE_SIZE);
     hp = (*bpp)->mem;
-    return erts_build_dhandle(&hp, &(*bpp)->off_heap, dep);
+    return erts_build_dhandle(&hp, &(*bpp)->off_heap, dep, conn_id);
 
 badarg:
 
@@ -3529,25 +3480,23 @@ BIF_RETTYPE erts_internal_new_connection_1(BIF_ALIST_1)
     erts_de_rwlock(dep);
 
     switch (dep->state) {
-    case ERTS_DE_STATE_PENDING:
     case ERTS_DE_STATE_CONNECTED:
+    case ERTS_DE_STATE_EXITING:
+    case ERTS_DE_STATE_PENDING:
 	conn_id = dep->connection_id;
         break;
     case ERTS_DE_STATE_IDLE:
         erts_set_dist_entry_pending(dep);
 	conn_id = dep->connection_id;
         break;
-    case ERTS_DE_STATE_EXITING:
-        conn_id = (dep->connection_id + 1) & ERTS_DIST_CON_ID_MASK;
-        break;
     default:
         erts_exit(ERTS_ABORT_EXIT, "Invalid dep->state (%d)\n", dep->state);
     }
     erts_de_rwunlock(dep);
-    hp = HAlloc(BIF_P, 3 + ERTS_MAGIC_REF_THING_SIZE);
-    dhandle = erts_build_dhandle(&hp, &BIF_P->off_heap, dep);
+    hp = HAlloc(BIF_P, ERTS_DHANDLE_SIZE);
+    dhandle = erts_build_dhandle(&hp, &BIF_P->off_heap, dep, conn_id);
     erts_deref_dist_entry(dep);
-    BIF_RET(TUPLE2(hp, make_small(conn_id), dhandle));
+    BIF_RET(dhandle);
 }
 
 static Sint abort_connection(DistEntry* dep, Uint32 conn_id)
@@ -3585,7 +3534,6 @@ static Sint abort_connection(DistEntry* dep, Uint32 conn_id)
 	dep->send = NULL;
 
         erts_set_dist_entry_not_connected(dep);
-
 	erts_de_rwunlock(dep);
 
         schedule_con_monitor_link_cleanup(mld, THE_NON_VALUE,
@@ -3599,16 +3547,6 @@ static Sint abort_connection(DistEntry* dep, Uint32 conn_id)
 	delete_cache(cache);
 	free_de_out_queues(dep, obuf);
 
-        /*
-         * We wait to make DistEntry idle and accept new connection attempts
-         * until all is cleared and deallocated. This to get some back pressure
-         * against repeated failing connection attempts saturating all CPUs
-         * with cleanup jobs.
-         */
-        erts_de_rwlock(dep);
-        ASSERT(dep->state == ERTS_DE_STATE_EXITING);
-        dep->state = ERTS_DE_STATE_IDLE;
-        erts_de_rwunlock(dep);
         return reds;
     }
     erts_de_rwunlock(dep);
@@ -3624,22 +3562,19 @@ erts_abort_connection(DistEntry *dep, Uint32 conn_id)
 BIF_RETTYPE erts_internal_abort_connection_2(BIF_ALIST_2)
 {
     DistEntry* dep;
-    Eterm* tp;
+    Uint32 conn_id;
+    Sint reds;
 
-    if (is_not_atom(BIF_ARG_1) || is_not_tuple_arity(BIF_ARG_2, 2)) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
-    tp = tuple_val(BIF_ARG_2);
-    dep = erts_dhandle_to_dist_entry(tp[2]);
-    if (is_not_small(tp[1]) || dep != erts_find_dist_entry(BIF_ARG_1)
+    if (is_not_atom(BIF_ARG_1))
+        BIF_ERROR(BIF_P, BADARG);
+    dep = erts_dhandle_to_dist_entry(BIF_ARG_2, &conn_id);
+    if (!dep || dep != erts_find_dist_entry(BIF_ARG_1)
         || dep == erts_this_dist_entry) {
         BIF_ERROR(BIF_P, BADARG);
     }
 
-    if (dep) {
-        Sint reds = abort_connection(dep, unsigned_val(tp[1]));
-        BUMP_REDS(BIF_P, reds);
-    }
+    reds = abort_connection(dep, conn_id);
+    BUMP_REDS(BIF_P, reds);
     BIF_RET(am_true);
 }
 
@@ -3670,14 +3605,13 @@ int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks)
         }
 
         /*
-         * Send {auto_connect, Node, ConnId, DHandle} to net_kernel
+         * Send {auto_connect, Node, DHandle} to net_kernel
          */
         mp = erts_alloc_message_heap(net_kernel, &nk_locks,
-                                     5 + ERTS_MAGIC_REF_THING_SIZE,
+                                     4 + ERTS_DHANDLE_SIZE,
                                      &hp, &ohp);
-        dhandle = erts_build_dhandle(&hp, ohp, dep);
-        msg = TUPLE4(hp, am_auto_connect, dep->sysname, make_small(conn_id),
-                     dhandle);
+        dhandle = erts_build_dhandle(&hp, ohp, dep, conn_id);
+        msg = TUPLE3(hp, am_auto_connect, dep->sysname, dhandle);
         ERL_MESSAGE_TOKEN(mp) = am_undefined;
         erts_queue_proc_message(proc, net_kernel, nk_locks, mp, msg);
         erts_proc_unlock(net_kernel, nk_locks);
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 30b4b35c20..75b59e5196 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -136,6 +136,7 @@ typedef struct {
     Eterm cid;
     Eterm connection_id;
     int no_suspend;
+    Uint32 flags;
 } ErtsDSigData;
 
 #define ERTS_DE_BUSY_LIMIT (1024*1024)
@@ -235,6 +236,7 @@ retry:
     dsdp->cid = dep->cid;
     dsdp->connection_id = dep->connection_id;
     dsdp->no_suspend = no_suspend;
+    dsdp->flags = dep->flags;
     if (dspl == ERTS_DSP_NO_LOCK)
 	erts_de_runlock(dep);
     return res;
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 9b34af1480..9921ce8a31 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -366,31 +366,43 @@ DistEntry *erts_find_dist_entry(Eterm sysname)
 }
 
 DistEntry *
-erts_dhandle_to_dist_entry(Eterm dhandle)
+erts_dhandle_to_dist_entry(Eterm dhandle, Uint32 *conn_id)
 {
+    Eterm *tpl;
     Binary *bin;
-    if (!is_internal_magic_ref(dhandle))
+
+    if (!is_boxed(dhandle))
+        return NULL;
+    tpl = boxed_val(dhandle);
+    if (tpl[0] != make_arityval(2) || !is_small(tpl[1])
+        || !is_internal_magic_ref(tpl[2]))
         return NULL;
-    bin = erts_magic_ref2bin(dhandle);
+    *conn_id = unsigned_val(tpl[1]);
+    bin = erts_magic_ref2bin(tpl[2]);
     if (ERTS_MAGIC_BIN_DESTRUCTOR(bin) != erts_dist_entry_destructor)
         return NULL;
     return ErtsBin2DistEntry(bin);
 }
 
 Eterm
-erts_build_dhandle(Eterm **hpp, ErlOffHeap* ohp, DistEntry *dep)
+erts_build_dhandle(Eterm **hpp, ErlOffHeap* ohp,
+                   DistEntry *dep, Uint32 conn_id)
 {
     Binary *bin = ErtsDistEntry2Bin(dep);
+    Eterm mref, dhandle;
     ASSERT(bin);
     ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == erts_dist_entry_destructor);
-    return erts_mk_magic_ref(hpp, ohp, bin);
+    mref = erts_mk_magic_ref(hpp, ohp, bin);
+    dhandle = TUPLE2(*hpp, make_small(conn_id), mref);
+    *hpp += 3;
+    return dhandle;
 }
 
 Eterm
-erts_make_dhandle(Process *c_p, DistEntry *dep)
+erts_make_dhandle(Process *c_p, DistEntry *dep, Uint32 conn_id)
 {
-    Eterm *hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE);
-    return erts_build_dhandle(&hp, &c_p->off_heap, dep);
+    Eterm *hp = HAlloc(c_p, ERTS_DHANDLE_SIZE);
+    return erts_build_dhandle(&hp, &c_p->off_heap, dep, conn_id);
 }
 
 static void start_timer_delete_dist_entry(void *vdep);
@@ -620,7 +632,7 @@ erts_set_dist_entry_not_connected(DistEntry *dep)
     if(dep->next)
 	dep->next->prev = dep->prev;
 
-    dep->state = ERTS_DE_STATE_EXITING;
+    dep->state = ERTS_DE_STATE_IDLE;
     dep->flags = 0;
     dep->prev = NULL;
     dep->cid = NIL;
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index 9a792b10b1..c44f1f8991 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -214,9 +214,10 @@ int erts_lc_is_de_rwlocked(DistEntry *);
 int erts_lc_is_de_rlocked(DistEntry *);
 #endif
 int erts_dist_entry_destructor(Binary *bin);
-DistEntry *erts_dhandle_to_dist_entry(Eterm dhandle);
-Eterm erts_build_dhandle(Eterm **hpp, ErlOffHeap*, DistEntry*);
-Eterm erts_make_dhandle(Process *c_p, DistEntry *dep);
+DistEntry *erts_dhandle_to_dist_entry(Eterm dhandle, Uint32* connection_id);
+#define ERTS_DHANDLE_SIZE (3+ERTS_MAGIC_REF_THING_SIZE)
+Eterm erts_build_dhandle(Eterm **hpp, ErlOffHeap*, DistEntry*, Uint32 conn_id);
+Eterm erts_make_dhandle(Process *c_p, DistEntry*, Uint32 conn_id);
 
 ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np);
 ERTS_GLB_INLINE void erts_de_rlock(DistEntry *dep);
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index cf4a233b54..da00871d33 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -155,6 +155,7 @@ struct _erl_drv_port {
 
     ErlPortIOQueue ioq;          /* driver accessible i/o queue */
     DistEntry *dist_entry;       /* Dist entry used in DISTRIBUTION */
+    Uint32 connection_id;
     char *name;		         /* String used in the open */
     erts_driver_t* drv_ptr;
     UWord drv_data;
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 904993ceb6..1f6d3ef031 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -671,8 +671,8 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
 		      byte *ext,
 		      Uint size,
 		      DistEntry *dep,
-		      ErtsAtomCache *cache,
-                      Uint32 *connection_id)
+                      Uint32 conn_id,
+		      ErtsAtomCache *cache)
 {
 #undef ERTS_EXT_FAIL
 #undef ERTS_EXT_HDR_FAIL
@@ -683,18 +683,34 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
 #define ERTS_EXT_FAIL abort()
 #define ERTS_EXT_HDR_FAIL abort()
 #endif
+    register byte *ep;
+
+    edep->heap_size = -1;
+    edep->flags = 0;
+    edep->dep = dep;
+
+    ASSERT(dep);
+    erts_de_rlock(dep);
 
-    register byte *ep = ext;
     ASSERT(dep->flags & DFLAG_UTF8_ATOMS);
 
-    edep->heap_size = -1;
-    edep->ext_endp = ext+size;
+    if ((dep->state != ERTS_DE_STATE_CONNECTED &&
+         dep->state != ERTS_DE_STATE_PENDING)
+        || dep->connection_id != conn_id) {
+        erts_de_runlock(dep);
+        return ERTS_PREP_DIST_EXT_CLOSED;
+    }
 
-    if (size < 2)
-	ERTS_EXT_FAIL;
+    if (!(dep->flags & DFLAG_DIST_HDR_ATOM_CACHE)) {
+        /* Skip PASS_THROUGH */
+        ext++;
+        size--;
+    }
+    edep->ext_endp = ext + size;
+    ep = ext;
 
-    if (!dep)
-        ERTS_INTERNAL_ERROR("Invalid use");
+    if (size < 2)
+        goto fail;
 
     if (ep[0] != VERSION_MAGIC) {
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
@@ -706,20 +722,9 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
 	ERTS_EXT_FAIL;
     }
 
-    edep->flags = 0;
-    edep->dep = dep;
-
-    erts_de_rlock(dep);
-
-    if (dep->state != ERTS_DE_STATE_CONNECTED &&
-	dep->state != ERTS_DE_STATE_PENDING) {
-        erts_de_runlock(dep);
-        return ERTS_PREP_DIST_EXT_CLOSED;
-    }
     if (dep->flags & DFLAG_DIST_HDR_ATOM_CACHE)
         edep->flags |= ERTS_DIST_EXT_DFLAG_HDR;
 
-    *connection_id = dep->connection_id;
     edep->connection_id = dep->connection_id;
 
     if (ep[1] != DIST_HEADER) {
@@ -901,7 +906,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
     }
  fail: {
 	erts_de_runlock(dep);
-	erts_kill_dist_connection(dep, *connection_id);
+	erts_kill_dist_connection(dep, conn_id);
     }
     return ERTS_PREP_DIST_EXT_FAILED;
 }
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index bbd9b4bad2..edac177cc6 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -182,7 +182,7 @@ void erts_destroy_dist_ext_copy(ErtsDistExternal *);
 #define ERTS_PREP_DIST_EXT_CLOSED       (1)
 
 int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint,
-			  DistEntry *, ErtsAtomCache *, Uint32 *);
+			  DistEntry *, Uint32 conn_id, ErtsAtomCache *);
 Sint erts_decode_dist_ext_size(ErtsDistExternal *);
 Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *);
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 2cf268162d..21ae205237 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1083,7 +1083,7 @@ extern int erts_do_net_exits(DistEntry*, Eterm);
 extern int distribution_info(fmtfn_t, void *);
 extern int is_node_name_atom(Eterm a);
 
-extern int erts_net_message(Port *, DistEntry *,
+extern int erts_net_message(Port *, DistEntry *, Uint32 conn_id,
 			    byte *, ErlDrvSizeT, byte *, ErlDrvSizeT);
 
 extern void init_dist(void);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 133ab485d9..99d75a93ef 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -6172,6 +6172,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
         erts_atomic64_inc_nob(&prt->dist_entry->in);
 	return erts_net_message(prt,
 				prt->dist_entry,
+                                prt->connection_id,
 				(byte*) hbuf, hlen,
 				(byte*) (bin->orig_bytes+offs), len);
     }
@@ -6214,11 +6215,13 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
 	if (len == 0)
 	    return erts_net_message(prt,
 				    prt->dist_entry,
+                                    prt->connection_id,
 				    NULL, 0,
 				    (byte*) hbuf, hlen);
 	else
 	    return erts_net_message(prt,
 				    prt->dist_entry,
+                                    prt->connection_id,
 				    (byte*) hbuf, hlen,
 				    (byte*) buf, len);
     }
-- 
cgit v1.2.3


From bb531e8bf209b0071bb6084b5ba96f11b39e9183 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 7 Sep 2018 19:24:05 +0200
Subject: Remove ugly fail case macros

---
 erts/emulator/beam/external.c | 33 +++++++++++----------------------
 1 file changed, 11 insertions(+), 22 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 1f6d3ef031..621ba108ba 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -674,15 +674,6 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
                       Uint32 conn_id,
 		      ErtsAtomCache *cache)
 {
-#undef ERTS_EXT_FAIL
-#undef ERTS_EXT_HDR_FAIL
-#if 1
-#define ERTS_EXT_FAIL goto fail
-#define ERTS_EXT_HDR_FAIL goto bad_hdr
-#else
-#define ERTS_EXT_FAIL abort()
-#define ERTS_EXT_HDR_FAIL abort()
-#endif
     register byte *ep;
 
     edep->heap_size = -1;
@@ -719,7 +710,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
                       "channel %d\n",
                       dist_entry_channel_no(dep));
 	erts_send_error_to_logger_nogl(dsbufp);
-	ERTS_EXT_FAIL;
+	goto fail;
     }
 
     if (dep->flags & DFLAG_DIST_HDR_ATOM_CACHE)
@@ -729,7 +720,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
 
     if (ep[1] != DIST_HEADER) {
 	if (edep->flags & ERTS_DIST_EXT_DFLAG_HDR)
-	    ERTS_EXT_HDR_FAIL;
+	    goto bad_hdr;
 	edep->attab.size = 0;
 	edep->extp = ext;
     }
@@ -738,17 +729,17 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
 	int no_atoms;
 
 	if (!(edep->flags & ERTS_DIST_EXT_DFLAG_HDR))
-	    ERTS_EXT_HDR_FAIL;
+	    goto bad_hdr;
 
 #undef CHKSIZE
 #define CHKSIZE(SZ) \
-	do { if ((SZ) > edep->ext_endp - ep) ERTS_EXT_HDR_FAIL; } while(0)
+	do { if ((SZ) > edep->ext_endp - ep) goto bad_hdr; } while(0)
 
 	CHKSIZE(1+1+1);
 	ep += 2;
 	no_atoms = (int) get_int8(ep);
 	if (no_atoms < 0 || ERTS_ATOM_CACHE_SIZE < no_atoms)
-	    ERTS_EXT_HDR_FAIL;
+	    goto bad_hdr;
 	ep++;
 	if (no_atoms) {
 	    int long_atoms = 0;
@@ -826,18 +817,18 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
 		    /* atom already cached */
 		    cix += (int) get_int8(ep);
 		    if (cix >= ERTS_ATOM_CACHE_SIZE)
-			ERTS_EXT_HDR_FAIL;
+			goto bad_hdr;
 		    ep++;
 		    atom = cache->in_arr[cix];
 		    if (!is_atom(atom))
-			ERTS_EXT_HDR_FAIL;
+			goto bad_hdr;
 		    edep->attab.atom[tix] = atom;
 		}
 		else {
 		    /* new cached atom */
 		    cix += (int) get_int8(ep);
 		    if (cix >= ERTS_ATOM_CACHE_SIZE)
-			ERTS_EXT_HDR_FAIL;
+			goto bad_hdr;
 		    ep++;
 		    if (long_atoms) {
 			CHKSIZE(2);
@@ -855,7 +846,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
                                          ERTS_ATOM_ENC_UTF8,
 					 0);
 		    if (is_non_value(atom))
-			ERTS_EXT_HDR_FAIL;
+			goto bad_hdr;
 		    ep += len;
 		    cache->in_arr[cix] = atom;
 		    edep->attab.atom[tix] = atom;
@@ -875,12 +866,12 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
 	edep->extp = ep;
 #ifdef ERTS_DEBUG_USE_DIST_SEP
 	if (*ep != VERSION_MAGIC)
-	    ERTS_EXT_HDR_FAIL;
+	    goto bad_hdr;
 #endif
     }
 #ifdef ERTS_DEBUG_USE_DIST_SEP
     if (*ep != VERSION_MAGIC)
-	ERTS_EXT_FAIL;
+	goto fail;
 #endif
 
     erts_de_runlock(dep);
@@ -888,8 +879,6 @@ erts_prepare_dist_ext(ErtsDistExternal *edep,
     return ERTS_PREP_DIST_EXT_SUCCESS;
 
 #undef CHKSIZE
-#undef ERTS_EXT_FAIL
-#undef ERTS_EXT_HDR_FAIL
 
  bad_hdr: {
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-- 
cgit v1.2.3


From fd5ec01f342a885fe1ba0103fe6ece71a85f42f0 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 10 Sep 2018 17:26:11 +0200
Subject: erts: Refactor port dist_entry & conn_id into PRTSD

spelled out as "port specific data".
---
 erts/emulator/beam/dist.c            | 18 +++++++++++-------
 erts/emulator/beam/dist.h            |  4 ++--
 erts/emulator/beam/erl_node_tables.c |  5 +++--
 erts/emulator/beam/erl_port.h        |  6 +++---
 erts/emulator/beam/io.c              | 37 ++++++++++++++++++++----------------
 5 files changed, 40 insertions(+), 30 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 9efb7e79ac..0e2a09cbe9 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -2069,13 +2069,14 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf)
 
 #ifdef USE_VM_PROBES
     if (DTRACE_ENABLED(dist_output)) {
+        DistEntry *dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY);
         DTRACE_CHARBUF(port_str, 64);
         DTRACE_CHARBUF(remote_str, 64);
 
         erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
                       "%T", prt->common.id);
         erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)),
-                      "%T", prt->dist_entry->sysname);
+                      "%T", dep->sysname);
         DTRACE4(dist_output, erts_this_node_sysname, port_str,
                 remote_str, size);
     }
@@ -2131,13 +2132,14 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf)
 
 #ifdef USE_VM_PROBES
     if (DTRACE_ENABLED(dist_outputv)) {
+        DistEntry *dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY);
         DTRACE_CHARBUF(port_str, 64);
         DTRACE_CHARBUF(remote_str, 64);
 
         erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
                       "%T", prt->common.id);
         erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)),
-                      "%T", prt->dist_entry->sysname);
+                      "%T", dep->sysname);
         DTRACE4(dist_outputv, erts_this_node_sysname, port_str,
                 remote_str, size);
     }
@@ -2175,7 +2177,7 @@ erts_dist_command(Port *prt, int initial_reds)
     Uint32 flags;
     Sint qsize, obufsize = 0;
     ErtsDistOutputQueue oq, foq;
-    DistEntry *dep = prt->dist_entry;
+    DistEntry *dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY);
     Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf);
     erts_aint32_t sched_flags;
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
@@ -2842,13 +2844,14 @@ erts_dist_port_not_busy(Port *prt)
 {
 #ifdef USE_VM_PROBES
     if (DTRACE_ENABLED(dist_port_not_busy)) {
+        DistEntry *dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY);
         DTRACE_CHARBUF(port_str, 64);
         DTRACE_CHARBUF(remote_str, 64);
 
         erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
                       "%T", prt->common.id);
         erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)),
-                      "%T", prt->dist_entry->sysname);
+                      "%T", dep->sysname);
         DTRACE3(dist_port_not_busy, erts_this_node_sysname,
                 port_str, remote_str);
     }
@@ -3277,13 +3280,14 @@ BIF_RETTYPE erts_internal_create_dist_channel_4(BIF_ALIST_4)
         if ((pp->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY) == 0)
             goto badarg;
 
-        if (pp->dist_entry || is_not_nil(dep->cid))
+        if (erts_prtsd_get(pp, ERTS_PRTSD_DIST_ENTRY) != NULL
+            || is_not_nil(dep->cid))
             goto badarg;
 
         erts_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION);
 
-        pp->dist_entry = dep;
-        pp->connection_id = dep->connection_id;
+        erts_prtsd_set(pp, ERTS_PRTSD_DIST_ENTRY, dep);
+        erts_prtsd_set(pp, ERTS_PRTSD_CONN_ID, (void*)(UWord)dep->connection_id);
 
         ASSERT(pp->drv_ptr->outputv || pp->drv_ptr->output);
 
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 75b59e5196..b2f9c6a16b 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -256,9 +256,9 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry)
 	ERTS_LC_ASSERT(erts_lc_is_port_locked(prt));
 	ASSERT((erts_atomic32_read_nob(&prt->state)
 		& ERTS_PORT_SFLGS_DEAD) == 0);
-	ASSERT(prt->dist_entry);
 
-	dep = prt->dist_entry;
+        dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY);
+        ASSERT(dep);
 	id = prt->common.id;
     }
     else {
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 9921ce8a31..93a96bc11b 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1913,8 +1913,9 @@ setup_reference_table(void)
 	if (ohp)
 	    insert_offheap(ohp, HEAP_REF, prt->common.id);
 	/* Insert controller */
-	if (prt->dist_entry)
-	    insert_dist_entry(prt->dist_entry,
+	dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY);
+        if (dep)
+            insert_dist_entry(dep,
 			      CTRL_REF,
 			      prt->common.id,
 			      0);
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index da00871d33..2be0a5bf74 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -112,8 +112,10 @@ typedef struct line_buf {  /* Buffer used in line oriented I/O */
  */
 
 #define ERTS_PRTSD_SCHED_ID 0
+#define ERTS_PRTSD_DIST_ENTRY 1
+#define ERTS_PRTSD_CONN_ID 2
 
-#define ERTS_PRTSD_SIZE 1
+#define ERTS_PRTSD_SIZE 3
 
 typedef struct {
     void *data[ERTS_PRTSD_SIZE];
@@ -154,8 +156,6 @@ struct _erl_drv_port {
     Uint bytes_out;		/* Number of bytes written */
 
     ErlPortIOQueue ioq;          /* driver accessible i/o queue */
-    DistEntry *dist_entry;       /* Dist entry used in DISTRIBUTION */
-    Uint32 connection_id;
     char *name;		         /* String used in the open */
     erts_driver_t* drv_ptr;
     UWord drv_data;
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 99d75a93ef..5325480901 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -375,7 +375,6 @@ static Port *create_port(char *name,
     prt->control_flags = 0;
     prt->bytes_in = 0;
     prt->bytes_out = 0;
-    prt->dist_entry = NULL;
     ERTS_PORT_INIT_CONNECTED(prt, pid);
     prt->common.u.alive.reg = NULL;
     ERTS_PTMR_INIT(prt);
@@ -3613,12 +3612,12 @@ terminate_port(Port *prt)
 
     erts_cleanup_port_data(prt);
 
+    ASSERT(erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY) == NULL);
+
     psd = (ErtsPrtSD *) erts_atomic_read_nob(&prt->psd);
     if (psd)
 	erts_free(ERTS_ALC_T_PRTSD, psd);
 
-    ASSERT(prt->dist_entry == NULL);
-
     kill_port(prt);
 
     /*
@@ -3759,10 +3758,12 @@ erts_deliver_port_exit(Port *prt, Eterm from, Eterm reason, int send_closed,
        DRV_MONITOR_UNLOCK_PDL(prt);
    }
 
-   if ((state & ERTS_PORT_SFLG_DISTRIBUTION) && prt->dist_entry) {
-       erts_do_net_exits(prt->dist_entry, modified_reason);
-       erts_deref_dist_entry(prt->dist_entry);
-       prt->dist_entry = NULL;
+   if (state & ERTS_PORT_SFLG_DISTRIBUTION) {
+       DistEntry *dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY);
+       ASSERT(dep);
+       erts_do_net_exits(dep, modified_reason);
+       erts_deref_dist_entry(dep);
+       erts_prtsd_set(prt, ERTS_PRTSD_DIST_ENTRY, NULL);
        erts_atomic32_read_band_relb(&prt->state,
 				    ~ERTS_PORT_SFLG_DISTRIBUTION);
    }
@@ -5050,7 +5051,7 @@ set_busy_port(ErlDrvPort dprt, int on)
             DTRACE1(port_not_busy, port_str);
         }
 #endif
-	if (prt->dist_entry) {
+	if (erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY) != NULL) {
 	    /*
 	     * Processes suspended on distribution ports are
 	     * normally queued on the dist entry.
@@ -6169,10 +6170,12 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
     else
 	erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len));
     if (state & ERTS_PORT_SFLG_DISTRIBUTION) {
-        erts_atomic64_inc_nob(&prt->dist_entry->in);
+        DistEntry* dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY);
+        Uint32 conn_id = (Uint32)(UWord) erts_prtsd_get(prt, ERTS_PRTSD_CONN_ID);
+        erts_atomic64_inc_nob(&dep->in);
 	return erts_net_message(prt,
-				prt->dist_entry,
-                                prt->connection_id,
+				dep,
+                                conn_id,
 				(byte*) hbuf, hlen,
 				(byte*) (bin->orig_bytes+offs), len);
     }
@@ -6211,17 +6214,19 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
     else
 	erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len));
     if (state & ERTS_PORT_SFLG_DISTRIBUTION) {
-        erts_atomic64_inc_nob(&prt->dist_entry->in);
+        DistEntry *dep = (DistEntry*) erts_prtsd_get(prt, ERTS_PRTSD_DIST_ENTRY);
+        Uint32 conn_id = (Uint32)(UWord) erts_prtsd_get(prt, ERTS_PRTSD_CONN_ID);
+        erts_atomic64_inc_nob(&dep->in);
 	if (len == 0)
 	    return erts_net_message(prt,
-				    prt->dist_entry,
-                                    prt->connection_id,
+				    dep,
+                                    conn_id,
 				    NULL, 0,
 				    (byte*) hbuf, hlen);
 	else
 	    return erts_net_message(prt,
-				    prt->dist_entry,
-                                    prt->connection_id,
+				    dep,
+                                    conn_id,
 				    (byte*) hbuf, hlen,
 				    (byte*) buf, len);
     }
-- 
cgit v1.2.3


From abb972b03acf948255f1026442758adfba44baa1 Mon Sep 17 00:00:00 2001
From: Raimo Niskanen <raimo@erlang.org>
Date: Tue, 18 Sep 2018 13:31:09 +0200
Subject: Fix endianness bug for CMSG parsing

---
 erts/emulator/drivers/common/inet_drv.c | 131 +++++++++++++++++++++-----------
 1 file changed, 87 insertions(+), 44 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 3478ba7081..7f20477363 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -634,11 +634,10 @@ static size_t my_strnlen(const char *s, size_t maxlen)
  * the pointer difference from the cmsg start pointer
  * to the CMSG_DATA(cmsg) pointer.
  */
-#define LEN_CMSG_DATA(cmsg)  ((char*)CMSG_DATA(cmsg) - (char*)(cmsg))
+#define LEN_CMSG_DATA(cmsg)                                             \
+    ((cmsg)->cmsg_len - ((char*)CMSG_DATA(cmsg) - (char*)(cmsg)))
 #define NXT_CMSG_HDR(cmsg)                                              \
-    ((struct cmsghdr*)                                                  \
-     (((char*)(cmsg)) +                                                 \
-      CMSG_SPACE((cmsg)->cmsg_len - LEN_CMSG_DATA(cmsg))))
+    ((struct cmsghdr*)(((char*)(cmsg)) + CMSG_SPACE(LEN_CMSG_DATA(cmsg))))
 #endif
 
 #if !defined(IPV6_PKTOPTIONS) && defined(IPV6_2292PKTOPTIONS)
@@ -2796,39 +2795,61 @@ static int inet_async_data(inet_descriptor* desc, const char* buf, int len)
 }
 
 #ifndef __WIN32__
+static int load_cmsg_int(ErlDrvTermData *spec, int i,
+                    struct cmsghdr *cmsg) {
+    union u {
+        byte uint8;
+        Uint16 uint16;
+        Uint32 uint32;
+        Uint64 uint64;
+    } *p;
+    p = (union u*) CMSG_DATA(cmsg);
+    switch (LEN_CMSG_DATA(cmsg) * CHAR_BIT) {
+    case 8:
+        return LOAD_INT(spec, i, p->uint8);
+    case 16:
+        return LOAD_INT(spec, i, p->uint16);
+
+    case 32:
+        return LOAD_INT(spec, i, p->uint32);
+
+    case 64:
+        return LOAD_INT(spec, i, p->uint64);
+    }
+    return LOAD_INT(spec, i, 0);
+}
+
 static int parse_ancillary_data_item(ErlDrvTermData *spec, int i,
                                      struct cmsghdr *cmsg, int *n) {
-#define LOAD_CMSG(proto, type, vtype, am, load)         \
-    if (cmsg->cmsg_level == (proto) &&                  \
-        cmsg->cmsg_type == (type)) {                    \
-        vtype *vp;                                      \
-        vp = (vtype *)CMSG_DATA(cmsg);                  \
-        i = LOAD_ATOM(spec, i, (am));                   \
-        i = load(spec, i, *vp);                         \
-        i = LOAD_TUPLE(spec, i, 2);                     \
-        (*n)++;                                         \
-        return i;                                       \
+#define LOAD_CMSG_INT(proto, type, am)              \
+    if (cmsg->cmsg_level == (proto) &&          \
+        cmsg->cmsg_type == (type)) {            \
+        i = LOAD_ATOM(spec, i, (am));           \
+        i = load_cmsg_int(spec, i, cmsg);            \
+        i = LOAD_TUPLE(spec, i, 2);             \
+        (*n)++;                                 \
+        return i;                               \
     }
 #if defined(IPPROTO_IP) && defined(IP_TOS)
-    LOAD_CMSG(IPPROTO_IP, IP_TOS, unsigned char, am_tos, LOAD_INT);
+    LOAD_CMSG_INT(IPPROTO_IP, IP_TOS, am_tos);
 #endif
 #if defined(IPPROTO_IPV6) && defined(IPV6_TCLASS)
-    LOAD_CMSG(IPPROTO_IPV6, IPV6_TCLASS, unsigned char, am_tclass, LOAD_INT);
+    LOAD_CMSG_INT(IPPROTO_IPV6, IPV6_TCLASS, am_tclass);
 #endif
 #if defined(IPPROTO_IP) && defined(IP_TTL)
-    LOAD_CMSG(IPPROTO_IP, IP_TTL, unsigned char, am_ttl, LOAD_INT);
+    LOAD_CMSG_INT(IPPROTO_IP, IP_TTL, am_ttl);
 #endif
     /* BSD uses the RECV* names in CMSG fields */
 #if defined(IPPROTO_IP) && defined(IP_RECVTOS)
-    LOAD_CMSG(IPPROTO_IP, IP_RECVTOS, unsigned char, am_tos, LOAD_INT);
+    LOAD_CMSG_INT(IPPROTO_IP, IP_RECVTOS, am_tos);
 #endif
 #if defined(IPPROTO_IPV6) && defined(IPV6_RECVTCLASS)
-    LOAD_CMSG(IPPROTO_IPV6, IPV6_RECVTCLASS, unsigned char, am_tclass, LOAD_INT);
+    LOAD_CMSG_INT(IPPROTO_IPV6, IPV6_RECVTCLASS, am_tclass);
 #endif
 #if defined(IPPROTO_IP) && defined(IP_RECVTTL)
-    LOAD_CMSG(IPPROTO_IP, IP_RECVTTL, unsigned char, am_ttl, LOAD_INT);
+    LOAD_CMSG_INT(IPPROTO_IP, IP_RECVTTL, am_ttl);
 #endif
-#undef LOAD_CMSG
+#undef LOAD_CMSG_INT
     return i;
 }
 #endif /* #ifndef __WIN32__ */
@@ -7330,6 +7351,35 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 }
 #endif /* HAVE_SCTP */
 
+#ifndef __WIN32__
+static void put_cmsg_int32(struct cmsghdr *cmsg, char *ptr) {
+    union u {
+        byte uint8;
+        Uint16 uint16;
+        Uint32 uint32;
+        Uint64 uint64;
+    } *p;
+    p = (union u*) CMSG_DATA(cmsg);
+    switch (LEN_CMSG_DATA(cmsg) * CHAR_BIT) {
+    case 8:
+        put_int32((Uint32) p->uint8, ptr);
+        break;
+    case 16:
+        put_int32((Uint32) p->uint16, ptr);
+        break;
+    case 32:
+        put_int32(p->uint32, ptr);
+        break;
+    case 64:
+        put_int32((Uint32) p->uint64, ptr);
+        break;
+    default:
+        put_int32(0, ptr);
+    }
+    return;
+}
+#endif
+
 /* load all option values into the buf and reply 
 ** return total length of reply filled into ptr
 ** ptr should point to a buffer with 9*len +1 to be safe!!
@@ -7796,44 +7846,37 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc,
                      cmsg = (struct cmsghdr*)cmsgbuf.buf;
                  cmsg < cmsg_top;
                  cmsg = NXT_CMSG_HDR(cmsg)) {
-#define PUT_CMSG_DATA(CMSG_LEVEL, CMSG_TYPE, OPT, TYPE, SZ, PUT)        \
-                if ((cmsg->cmsg_level == CMSG_LEVEL) &&                 \
-                    (cmsg->cmsg_type == CMSG_TYPE)) {                   \
-                    TYPE *cmsgp;                                        \
-                    cmsgp = (TYPE *)CMSG_DATA(cmsg);                    \
-                    PLACE_FOR(1+SZ, ptr);                               \
-                    *ptr = OPT;                                         \
-                    PUT(*cmsgp, ptr+1);                                 \
-                    arg_sz += 1+SZ;                                     \
-                    continue;                                           \
+#define PUT_CMSG_INT32(CMSG_LEVEL, CMSG_TYPE, OPT)      \
+                if ((cmsg->cmsg_level == CMSG_LEVEL) && \
+                    (cmsg->cmsg_type == CMSG_TYPE)) {   \
+                    PLACE_FOR(1+4, ptr);                \
+                    *ptr++ = OPT;                       \
+                    put_cmsg_int32(cmsg, ptr);          \
+                    ptr += 4;                           \
+                    arg_sz += 1+4;                      \
+                    continue;                           \
                 }
 #if defined(IPPROTO_IP) && defined(IP_TOS)
-                PUT_CMSG_DATA(IPPROTO_IP, IP_TOS,
-                              INET_OPT_TOS, unsigned char, 4, put_int32);
+                PUT_CMSG_INT32(IPPROTO_IP, IP_TOS, INET_OPT_TOS);
 #endif
 #if defined(IPPROTO_IPV6) && defined(IPV6_TCLASS)
-                PUT_CMSG_DATA(IPPROTO_IPV6, IPV6_TCLASS,
-                              INET_OPT_TCLASS, unsigned char, 4, put_int32);
+                PUT_CMSG_INT32(IPPROTO_IPV6, IPV6_TCLASS, INET_OPT_TCLASS);
 #endif
 #if defined(IPPROTO_IP) && defined(IP_TTL)
-                PUT_CMSG_DATA(IPPROTO_IP, IP_TTL,
-                              INET_OPT_TTL, unsigned char, 4, put_int32);
+                PUT_CMSG_INT32(IPPROTO_IP, IP_TTL, INET_OPT_TTL);
 #endif
                 /* BSD uses the RECV* names in CMSG fields */
             }
 #if defined(IPPROTO_IP) && defined(IP_RECVTOS)
-                PUT_CMSG_DATA(IPPROTO_IP, IP_RECVTOS,
-                              INET_OPT_TOS, unsigned char, 4, put_int32);
+                PUT_CMSG_INT32(IPPROTO_IP, IP_RECVTOS, INET_OPT_TOS);
 #endif
 #if defined(IPPROTO_IPV6) && defined(IPV6_RECVTCLASS)
-                PUT_CMSG_DATA(IPPROTO_IPV6, IPV6_RECVTCLASS,
-                              INET_OPT_TCLASS, unsigned char, 4, put_int32);
+                PUT_CMSG_INT32(IPPROTO_IPV6, IPV6_RECVTCLASS, INET_OPT_TCLASS);
 #endif
 #if defined(IPPROTO_IP) && defined(IP_RECVTTL)
-                PUT_CMSG_DATA(IPPROTO_IP, IP_RECVTTL,
-                              INET_OPT_TTL, unsigned char, 4, put_int32);
+                PUT_CMSG_INT32(IPPROTO_IP, IP_RECVTTL, INET_OPT_TTL);
 #endif
-#undef PUT_CMSG_DATA
+#undef PUT_CMSG_INT32
             put_int32(arg_sz, arg_ptr); /* Put total length */
             continue;
         }
-- 
cgit v1.2.3


From d04b53936d5e9f84a3066ecd466993671b1428bf Mon Sep 17 00:00:00 2001
From: Henrik Nord <henrik@erlang.org>
Date: Fri, 21 Sep 2018 12:23:25 +0200
Subject: Update copyright year

---
 erts/emulator/beam/erl_bif_lists.c          | 2 +-
 erts/emulator/sys/common/erl_osenv.c        | 2 +-
 erts/emulator/sys/unix/sys.c                | 2 +-
 erts/emulator/sys/win32/erl_poll.c          | 2 +-
 erts/emulator/test/node_container_SUITE.erl | 2 +-
 erts/emulator/test/ref_SUITE.erl            | 2 +-
 erts/emulator/test/scheduler_SUITE.erl      | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index 01e09da34c..395be67a90 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2016. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/common/erl_osenv.c b/erts/emulator/sys/common/erl_osenv.c
index 487ff87116..6a16377736 100644
--- a/erts/emulator/sys/common/erl_osenv.c
+++ b/erts/emulator/sys/common/erl_osenv.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2017. All Rights Reserved.
+ * Copyright Ericsson AB 2017-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 36579ffdb4..4823e549ea 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2017. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index fcf5a0d533..39bb4d515e 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2007-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl
index 55135fbcbc..300b4ed036 100644
--- a/erts/emulator/test/node_container_SUITE.erl
+++ b/erts/emulator/test/node_container_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2002-2017. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/ref_SUITE.erl b/erts/emulator/test/ref_SUITE.erl
index 74df857c65..925c30caa5 100644
--- a/erts/emulator/test/ref_SUITE.erl
+++ b/erts/emulator/test/ref_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2018. All Rights Reserved.
 %% 
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl
index d98edd1ec2..f04efb9003 100644
--- a/erts/emulator/test/scheduler_SUITE.erl
+++ b/erts/emulator/test/scheduler_SUITE.erl
@@ -1,7 +1,7 @@
 %% 
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2008-2017. All Rights Reserved.
+%% Copyright Ericsson AB 2008-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
-- 
cgit v1.2.3


From 167546a639b64691489615b91947749606e71987 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 26 Sep 2018 07:03:39 +0200
Subject: erts: Fix memory leak on file read errors

---
 erts/emulator/nifs/common/prim_file_nif.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/nifs/common/prim_file_nif.c b/erts/emulator/nifs/common/prim_file_nif.c
index a05d50b333..fd6aaa61f6 100644
--- a/erts/emulator/nifs/common/prim_file_nif.c
+++ b/erts/emulator/nifs/common/prim_file_nif.c
@@ -514,6 +514,7 @@ static ERL_NIF_TERM read_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, con
     ASSERT(bytes_read <= block_size);
 
     if(bytes_read < 0) {
+        enif_release_binary(&result);
         return posix_error_to_tuple(env, d->posix_errno);
     } else if(bytes_read == 0) {
         enif_release_binary(&result);
@@ -576,6 +577,7 @@ static ERL_NIF_TERM pread_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, co
     bytes_read = efile_preadv(d, offset, read_vec, 1);
 
     if(bytes_read < 0) {
+        enif_release_binary(&result);
         return posix_error_to_tuple(env, d->posix_errno);
     } else if(bytes_read == 0) {
         enif_release_binary(&result);
@@ -802,6 +804,7 @@ static ERL_NIF_TERM ipread_s32bu_p32bu_nif_impl(efile_data_t *d, ErlNifEnv *env,
     bytes_read = efile_preadv(d, payload_offset, read_vec, 1);
 
     if(bytes_read < 0) {
+        enif_release_binary(&payload);
         return posix_error_to_tuple(env, d->posix_errno);
     } else if(bytes_read == 0) {
         enif_release_binary(&payload);
-- 
cgit v1.2.3


From f29c1d4bc5ac7cb578206a1193867c9037741bd3 Mon Sep 17 00:00:00 2001
From: Alexander Clouter <alex@digriz.org.uk>
Date: Thu, 27 Sep 2018 15:17:29 +0100
Subject: SELinux is another cause of MSG_CTRUNC

https://github.com/termux/termux-packages/issues/2882
---
 erts/emulator/sys/unix/sys_uds.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/unix/sys_uds.c b/erts/emulator/sys/unix/sys_uds.c
index 39a4866065..c9f73622ba 100644
--- a/erts/emulator/sys/unix/sys_uds.c
+++ b/erts/emulator/sys/unix/sys_uds.c
@@ -88,8 +88,9 @@ sys_uds_readv(int fd, struct iovec *iov, size_t iov_len,
     if((msg.msg_flags & MSG_CTRUNC) == MSG_CTRUNC)
     {
         /* We assume that we have given enough space for any header
-           that are sent to us. So the only remaining reason to get
-           this flag set is if the caller has run out of file descriptors.
+           that are sent to us. So the only remaining reasons to get
+           this flag set is if the caller has run out of file descriptors
+           or an SELinux policy prunes the response (eg. O_APPEND on STDERR).
         */
         errno = EMFILE;
         return -1;
-- 
cgit v1.2.3


From 10dd2b092875d460c1f1785e247c061093d5f9c5 Mon Sep 17 00:00:00 2001
From: Raimo Niskanen <raimo@erlang.org>
Date: Tue, 25 Sep 2018 13:59:48 +0200
Subject: Fix bug for sockopt pktoptions on BSD

The macros for the BSD style option names had accidentally
wound up outside the option parsing loop, causing unclear
behaviour and Valgrind errors.
---
 erts/emulator/drivers/common/inet_drv.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 7f20477363..259a27cf57 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -7839,8 +7839,8 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc,
              * cmsg options and values
              */
             PLACE_FOR(1+4, ptr);
-            *ptr = opt;
-            arg_ptr = ptr+1; /* Where to put total length */
+            *ptr++ = opt;
+            arg_ptr = ptr; /* Where to put total length */
             arg_sz = 0; /* Total length */
             for (cmsg_top = (struct cmsghdr*)(cmsgbuf.buf + cmsg_sz),
                      cmsg = (struct cmsghdr*)cmsgbuf.buf;
@@ -7852,7 +7852,6 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc,
                     PLACE_FOR(1+4, ptr);                \
                     *ptr++ = OPT;                       \
                     put_cmsg_int32(cmsg, ptr);          \
-                    ptr += 4;                           \
                     arg_sz += 1+4;                      \
                     continue;                           \
                 }
@@ -7866,7 +7865,6 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc,
                 PUT_CMSG_INT32(IPPROTO_IP, IP_TTL, INET_OPT_TTL);
 #endif
                 /* BSD uses the RECV* names in CMSG fields */
-            }
 #if defined(IPPROTO_IP) && defined(IP_RECVTOS)
                 PUT_CMSG_INT32(IPPROTO_IP, IP_RECVTOS, INET_OPT_TOS);
 #endif
@@ -7877,6 +7875,7 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc,
                 PUT_CMSG_INT32(IPPROTO_IP, IP_RECVTTL, INET_OPT_TTL);
 #endif
 #undef PUT_CMSG_INT32
+            }
             put_int32(arg_sz, arg_ptr); /* Put total length */
             continue;
         }
-- 
cgit v1.2.3


From b32911645a20230864eaee965d6a7f1be248e47e Mon Sep 17 00:00:00 2001
From: Raimo Niskanen <raimo@erlang.org>
Date: Fri, 21 Sep 2018 15:50:39 +0200
Subject: Implement {netns,NS} option for inet:getifaddrs/1

Also implement the same option for the legacy undocumented functions
inet:getif/1,getiflist/1,ifget/2,ifset/2.

The arity 1 functions had before this change got signatures that
took a socket port that was used to do the needed syscall, so now
the signature was extended to also take an option list with the
only supported option {netns,Namespace}.  The Socket argument
variant remains unsupported.

For inet:getifaddrs/1 the documentation file was changed to old
style function name definition so be able to hide the Socket
argument variant that is visible in the type spec.

The arity 2 functions had got an option list as second argument.
This list had to be partitioned into one list for the namespace
option(s) and the other for the rest.

The namespace option list was then fed to the already existing
namespace support for socket opening, which places the socket
in a namespace and hence made all these functions that in
inet_drv.c used getsockopt() work without change.

The functions that used getifaddrs() in inet_drv.c had to be
changed in inet_drv.c to swap namespaces around the
getifaddrs() syscall.  This functionality was separated into
a new function call_getifaddrs().
---
 erts/emulator/drivers/common/inet_drv.c | 76 ++++++++++++++++++++++++++++++---
 1 file changed, 71 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 7f20477363..6d1c5cf6cc 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -5178,6 +5178,71 @@ static int hwaddr_libdlpi_lookup(const char *ifnm,
 }
 #endif
 
+#ifdef HAVE_GETIFADDRS
+/* Returns 0 for success and errno() for failure */
+static int call_getifaddrs(inet_descriptor* desc_p, struct ifaddrs **ifa_pp)
+{
+    int result, save_errno;
+#ifdef HAVE_SETNS
+    int current_ns;
+
+    current_ns = 0;
+    if (desc_p->netns != NULL) {
+        int new_ns;
+	/* Temporarily change network namespace for this thread
+	 * over the getifaddrs() call
+	 */
+	current_ns = open("/proc/self/ns/net", O_RDONLY);
+	if (current_ns == INVALID_SOCKET)
+	    return sock_errno();
+	new_ns = open(desc_p->netns, O_RDONLY);
+	if (new_ns == INVALID_SOCKET) {
+	    save_errno = sock_errno();
+	    while (close(current_ns) == INVALID_SOCKET &&
+		   sock_errno() == EINTR);
+	    return save_errno;
+	}
+	if (setns(new_ns, CLONE_NEWNET) != 0) {
+	    save_errno = sock_errno();
+	    while (close(new_ns) == INVALID_SOCKET &&
+		   sock_errno() == EINTR);
+	    while (close(current_ns) == INVALID_SOCKET &&
+		   sock_errno() == EINTR);
+	    return save_errno;
+	}
+	else {
+	    while (close(new_ns) == INVALID_SOCKET &&
+		   sock_errno() == EINTR);
+	}
+    }
+#endif
+    save_errno = 0;
+    result = getifaddrs(ifa_pp);
+    if (result  < 0)
+        save_errno = sock_errno();
+#ifdef HAVE_SETNS
+    if (desc_p->netns != NULL) {
+        /* Restore network namespace */
+        if (setns(current_ns, CLONE_NEWNET) != 0) {
+            /* XXX Failed to restore network namespace.
+             * What to do? Tidy up and return an error...
+             * Note that the thread now might still be in the set namespace.
+	     * Can this even happen? Should the emulator be aborted?
+	     */
+            if (result >= 0) {
+                /* We got a result but have to waste it */
+                save_errno = sock_errno();
+                freeifaddrs(*ifa_pp);
+            }
+	}
+        while (close(current_ns) == INVALID_SOCKET &&
+               sock_errno() == EINTR);
+    }
+#endif
+    return save_errno;
+}
+#endif /* #ifdef HAVE_GETIFADDRS */
+
 /* FIXME: temporary hack */
 #ifndef IFHWADDRLEN
 #define IFHWADDRLEN 6
@@ -5255,8 +5320,8 @@ static ErlDrvSSizeT inet_ctl_ifget(inet_descriptor* desc,
 	    struct sockaddr_dl *sdlp;
 	    int found = 0;
 
-	    if (getifaddrs(&ifa) == -1)
-		goto error;
+            if (call_getifaddrs(desc, &ifa) != 0)
+                goto error;
 
 	    for (ifp = ifa; ifp; ifp = ifp->ifa_next) {
 		if ((ifp->ifa_addr->sa_family == AF_LINK) &&
@@ -5974,6 +6039,7 @@ static ErlDrvSSizeT inet_ctl_getifaddrs(inet_descriptor* desc_p,
     ErlDrvSizeT buf_size;
     char *buf_p;
     char *buf_alloc_p;
+    int save_errno;
 
     buf_size = GETIFADDRS_BUFSZ;
     buf_alloc_p = ALLOC(GETIFADDRS_BUFSZ);
@@ -6008,9 +6074,9 @@ static ErlDrvSSizeT inet_ctl_getifaddrs(inet_descriptor* desc_p,
 	}                                                               \
     } while (0)
 
-    if (getifaddrs(&ifa_p) < 0) {
-	return ctl_error(sock_errno(), rbuf_pp, rsize);
-    }
+    if ((save_errno = call_getifaddrs(desc_p, &ifa_p)) != 0)
+        return ctl_error(save_errno, rbuf_pp, rsize);
+
     ifa_free_p = ifa_p;
     *buf_p++ = INET_REP_OK;
     for (;  ifa_p;  ifa_p = ifa_p->ifa_next) {
-- 
cgit v1.2.3


From aa086af7156490e22d139ab2938544ad07714169 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 9 Oct 2018 12:28:12 +0200
Subject: erts: Fix bug in ets:select_replace for bound key

which may cause following calls to ets:next or ets:prev to fail.
---
 erts/emulator/beam/erl_db_tree.c | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index d7deadacf0..f0c01e6e78 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -1887,22 +1887,14 @@ static int db_select_replace_tree(Process *p, DbTable *tbl, Eterm tid,
     sc.mp = mpi.mp;
     sc.all_objects = mpi.all_objects;
 
-    stack = get_static_stack(tb);
     if (!mpi.got_partial && mpi.some_limitation &&
             CMP_EQ(mpi.least,mpi.most)) {
-        TreeDbTerm* term = *(mpi.save_term);
         doit_select_replace(tb,mpi.save_term,&sc,0 /* dummy */);
-        if (stack != NULL) {
-            if (TOP_NODE(stack) == term)
-                // throw away potentially invalid reference
-                REPLACE_TOP_NODE(stack, *(mpi.save_term));
-            release_stack(tb, stack);
-        }
+        reset_static_stack(tb); /* may refer replaced term */
         RET_TO_BIF(erts_make_integer(sc.replaced,p),DB_ERROR_NONE);
     }
 
-    if (stack == NULL)
-        stack = get_any_stack(tb);
+    stack = get_any_stack(tb);
 
     if (mpi.some_limitation) {
         if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) {
-- 
cgit v1.2.3


From 210665e8a252faecf28ccebade999e734ab92582 Mon Sep 17 00:00:00 2001
From: Igor Slepchin <igor.slepchin@gmail.com>
Date: Mon, 24 Sep 2018 22:44:03 -0400
Subject: Add nopush TCP socket option

This translates to TCP_CORK on Linux and TCP_NOPUSH on
BSD.

In effect, this acts as super-Nagle: no partial TCP segments
are sent out until this option is turned off. Once turned off,
all accumulated unsent data is sent out immediately. The latter
is *not* the case on OSX, hence the implementation ignores
"nopush" on OSX to reduce confusion.
---
 erts/emulator/drivers/common/inet_drv.c | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 7f20477363..c13ceb14ad 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -812,6 +812,7 @@ static size_t my_strnlen(const char *s, size_t maxlen)
 #define INET_OPT_PKTOPTIONS         45  /* IP(V6)_PKTOPTIONS get ancillary data */
 #define INET_OPT_TTL                46  /* IP_TTL */
 #define INET_OPT_RECVTTL            47  /* IP_RECVTTL ancillary data */
+#define TCP_OPT_NOPUSH              48  /* super-Nagle, aka TCP_CORK */
 /* SCTP options: a separate range, from 100: */
 #define SCTP_OPT_RTOINFO		100
 #define SCTP_OPT_ASSOCINFO		101
@@ -955,6 +956,12 @@ static size_t my_strnlen(const char *s, size_t maxlen)
 #endif
 
 
+#if defined(TCP_CORK)
+#define INET_TCP_NOPUSH TCP_CORK
+#elif defined(TCP_NOPUSH) && !defined(__DARWIN__)
+#define INET_TCP_NOPUSH TCP_NOPUSH
+#endif
+
 #define BIN_REALLOC_MARGIN(x)  ((x)/4)  /* 25% */
 
 /* The general purpose sockaddr */
@@ -6532,6 +6539,19 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
 		    (long)desc->port, desc->s, ival));
 	    break;
 
+	case TCP_OPT_NOPUSH:
+#if defined(INET_TCP_NOPUSH)
+	    proto = IPPROTO_TCP;
+	    type = INET_TCP_NOPUSH;
+	    DEBUGF(("inet_set_opts(%ld): s=%d, t=%d TCP_NOPUSH=%d\r\n",
+	            (long)desc->port, desc->s, type, ival));
+	    break;
+#else
+	    /* inet_fill_opts always returns a value for this option,
+	     * so we need to ignore it if not implemented, just in case */
+	    continue;
+#endif
+
 #if defined(HAVE_MULTICAST_SUPPORT) && defined(IPPROTO_IP)
 
 	case UDP_OPT_MULTICAST_TTL:
@@ -7693,6 +7713,16 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc,
 	    proto = IPPROTO_TCP;
 	    type = TCP_NODELAY;
 	    break;
+	case TCP_OPT_NOPUSH:
+#if defined(INET_TCP_NOPUSH)
+	    proto = IPPROTO_TCP;
+	    type = INET_TCP_NOPUSH;
+	    break;
+#else
+	    *ptr++ = opt;
+	    put_int32(0, ptr);
+	    continue;
+#endif
 
 #if defined(HAVE_MULTICAST_SUPPORT) && defined(IPPROTO_IP)
 	case UDP_OPT_MULTICAST_TTL:
-- 
cgit v1.2.3


From 18b4828030e049a641e732785b081ff44576c9c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 11 Oct 2018 09:57:16 +0200
Subject: erts: Fix a compiler warning

This would've been a bug if the value was used.
---
 erts/emulator/nifs/win32/win_prim_file.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/nifs/win32/win_prim_file.c b/erts/emulator/nifs/win32/win_prim_file.c
index f7fae3c637..0c51a21ee1 100644
--- a/erts/emulator/nifs/win32/win_prim_file.c
+++ b/erts/emulator/nifs/win32/win_prim_file.c
@@ -687,7 +687,7 @@ static int is_name_surrogate(const efile_path_t *path) {
 
     if(handle != INVALID_HANDLE_VALUE) {
         REPARSE_GUID_DATA_BUFFER reparse_buffer;
-        LPDWORD unused_length;
+        DWORD unused_length;
         BOOL success;
 
         success = DeviceIoControl(handle,
-- 
cgit v1.2.3


From 1997b6b57d755e25d4050043450bb3cc7b229189 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 27 Sep 2018 16:38:00 +0200
Subject: erts: Fix UNC path handling on Windows

This is unlikely to be the last of the path problems seen after
OTP 21, and I'm starting to regret my decision to unconditionally
use long paths. The idea to hit all long-path problems all the time
was good in theory as it makes such bugs far more visible, but
there just aren't enough people who test pre-release versions on
Windows, making this the world's slowest game of whack-a-mole.
---
 erts/emulator/nifs/win32/win_prim_file.c | 155 ++++++++++++++++++++-----------
 1 file changed, 102 insertions(+), 53 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/nifs/win32/win_prim_file.c b/erts/emulator/nifs/win32/win_prim_file.c
index 0c51a21ee1..602a282dd1 100644
--- a/erts/emulator/nifs/win32/win_prim_file.c
+++ b/erts/emulator/nifs/win32/win_prim_file.c
@@ -33,16 +33,32 @@
 
 #define FILE_SHARE_FLAGS (FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
 
-#define LP_PREFIX L"\\\\?\\"
-#define LP_PREFIX_SIZE (sizeof(LP_PREFIX) - sizeof(WCHAR))
+/* Long paths can either be in the file (?) or the device (.) namespace. UNC
+ * paths are always in the file namespace. */
+#define LP_FILE_PREFIX L"\\\\?\\"
+#define LP_DEV_PREFIX L"\\\\.\\"
+#define LP_UNC_PREFIX (LP_FILE_PREFIX L"UNC\\")
+
+#define LP_PREFIX_SIZE (sizeof(LP_FILE_PREFIX) - sizeof(WCHAR))
 #define LP_PREFIX_LENGTH (LP_PREFIX_SIZE / sizeof(WCHAR))
 
+#define LP_UNC_PREFIX_SIZE (sizeof(LP_UNC_PREFIX) - sizeof(WCHAR))
+#define LP_UNC_PREFIX_LENGTH (LP_UNC_PREFIX_SIZE / sizeof(WCHAR))
+
+#define IS_LONG_PATH(length, data) \
+    ((length) >= LP_PREFIX_LENGTH && \
+         (!sys_memcmp((data), LP_FILE_PREFIX, LP_PREFIX_SIZE) || \
+          !sys_memcmp((data), LP_DEV_PREFIX, LP_PREFIX_SIZE)))
+
+#define IS_LONG_UNC_PATH(length, data) \
+    ((length) >= LP_UNC_PREFIX_LENGTH && \
+         !sys_memcmp((data), LP_UNC_PREFIX, LP_UNC_PREFIX_SIZE))
+
 #define PATH_LENGTH(path) (path->size / sizeof(WCHAR) - 1)
 
 #define ASSERT_PATH_FORMAT(path) \
     do { \
-        ASSERT(PATH_LENGTH(path) >= 4 && \
-            !memcmp(path->data, LP_PREFIX, LP_PREFIX_SIZE)); \
+        ASSERT(IS_LONG_PATH(PATH_LENGTH(path), (path)->data)); \
         ASSERT(PATH_LENGTH(path) == wcslen((WCHAR*)path->data)); \
     } while(0)
 
@@ -106,7 +122,7 @@ static posix_errno_t get_full_path(ErlNifEnv *env, WCHAR *input, efile_path_t *r
         return ENOENT;
     }
 
-    maximum_length += LP_PREFIX_LENGTH;
+    maximum_length += MAX(LP_PREFIX_LENGTH, LP_UNC_PREFIX_LENGTH);
 
     if(!enif_alloc_binary(maximum_length * sizeof(WCHAR), result)) {
         return ENOMEM;
@@ -115,18 +131,28 @@ static posix_errno_t get_full_path(ErlNifEnv *env, WCHAR *input, efile_path_t *r
     actual_length = GetFullPathNameW(input, maximum_length, (WCHAR*)result->data, NULL);
 
     if(actual_length < maximum_length) {
-        int has_long_path_prefix;
+        int is_long_path, maybe_unc_path;
         WCHAR *path_start;
 
-        /* Make sure we have a long-path prefix; GetFullPathNameW only adds one
-         * if the path is relative. */
-        has_long_path_prefix = actual_length >= LP_PREFIX_LENGTH &&
-            !sys_memcmp(result->data, LP_PREFIX, LP_PREFIX_SIZE);
-
-        if(!has_long_path_prefix) {
+        /* The APIs we use have varying path length limits and sometimes
+         * behave differently when given a long-path prefix, so it's simplest
+         * to always use long paths. */
+
+        is_long_path = IS_LONG_PATH(actual_length, result->data);
+        maybe_unc_path = !sys_memcmp(result->data, L"\\\\", sizeof(WCHAR) * 2);
+
+        if(maybe_unc_path && !is_long_path) {
+            /* \\localhost\c$\gurka -> \\?\UNC\localhost\c$\gurka */
+            sys_memmove(result->data + LP_UNC_PREFIX_SIZE,
+                &((WCHAR*)result->data)[2],
+                (actual_length - 1) * sizeof(WCHAR));
+            sys_memcpy(result->data, LP_UNC_PREFIX, LP_UNC_PREFIX_SIZE);
+            actual_length += LP_UNC_PREFIX_LENGTH;
+        } else if(!is_long_path) {
+            /* C:\gurka -> \\?\C:\gurka */
             sys_memmove(result->data + LP_PREFIX_SIZE, result->data,
                 (actual_length + 1) * sizeof(WCHAR));
-            sys_memcpy(result->data, LP_PREFIX, LP_PREFIX_SIZE);
+            sys_memcpy(result->data, LP_FILE_PREFIX, LP_PREFIX_SIZE);
             actual_length += LP_PREFIX_LENGTH;
         }
 
@@ -200,13 +226,19 @@ static int normalize_path_result(ErlNifBinary *path) {
     ASSERT(length < path->size / sizeof(WCHAR));
 
     /* Get rid of the long-path prefix, if present. */
-    if(length >= LP_PREFIX_LENGTH) {
-        if(!sys_memcmp(path_start, LP_PREFIX, LP_PREFIX_SIZE)) {
-            length -= LP_PREFIX_LENGTH;
 
-            sys_memmove(path_start, &path_start[LP_PREFIX_LENGTH],
-                length * sizeof(WCHAR));
-        }
+    if(IS_LONG_UNC_PATH(length, path_start)) {
+        /* The first two characters (\\) are the same for both long and short
+         * UNC paths. */
+        sys_memmove(&path_start[2], &path_start[LP_UNC_PREFIX_LENGTH],
+            (length - LP_UNC_PREFIX_LENGTH) * sizeof(WCHAR));
+
+        length -= LP_UNC_PREFIX_LENGTH - 2;
+    } else if(IS_LONG_PATH(length, path_start)) {
+        length -= LP_PREFIX_LENGTH;
+
+        sys_memmove(path_start, &path_start[LP_PREFIX_LENGTH],
+            length * sizeof(WCHAR));
     }
 
     path_end = &path_start[length];
@@ -318,49 +350,55 @@ static int has_same_mount_point(const efile_path_t *path_a, const efile_path_t *
 /* Mirrors the PathIsRootW function of the shell API, but doesn't choke on
  * paths longer than MAX_PATH. */
 static int is_path_root(const efile_path_t *path) {
-    const WCHAR *path_start, *path_end;
+    const WCHAR *path_start, *path_end, *path_iterator;
     int length;
 
     ASSERT_PATH_FORMAT(path);
 
-    path_start = (WCHAR*)path->data + LP_PREFIX_LENGTH;
-    length = PATH_LENGTH(path) - LP_PREFIX_LENGTH;
+    if(!IS_LONG_UNC_PATH(PATH_LENGTH(path), path->data)) {
+        path_start = (WCHAR*)path->data + LP_PREFIX_LENGTH;
+        length = PATH_LENGTH(path) - LP_PREFIX_LENGTH;
 
-    path_end = &path_start[length];
-
-    if(length == 1) {
         /* A single \ refers to the root of the current working directory. */
-        return IS_SLASH(path_start[0]);
-    } else if(length == 3 && iswalpha(path_start[0]) && path_start[1] == L':') {
-        /* Drive letter. */
-        return IS_SLASH(path_start[2]);
-    } else if(length >= 4) {
-        /* Check whether we're a UNC root, eg. \\server, \\server\share */
-        const WCHAR *path_iterator;
+        if(length == 1) {
+            return IS_SLASH(path_start[0]);
+        }
 
-        if(!IS_SLASH(path_start[0]) || !IS_SLASH(path_start[1])) {
-            return 0;
+        /* Drive letter. */
+        if(length == 3 && iswalpha(path_start[0]) && path_start[1] == L':') {
+            return IS_SLASH(path_start[2]);
         }
 
-        path_iterator = path_start + 2;
+        return 0;
+    }
 
-        /* Slide to the slash between the server and share names, if present. */
-        while(path_iterator < path_end && !IS_SLASH(*path_iterator)) {
-            path_iterator++;
-        }
+    /* Check whether we're a UNC root, eg. \\server, \\server\share */
 
-        /* Slide past the end of the string, stopping at the first slash we
-         * encounter. */
-        do {
-            path_iterator++;
-        }  while(path_iterator < path_end && !IS_SLASH(*path_iterator));
+    path_start = (WCHAR*)path->data + LP_UNC_PREFIX_LENGTH;
+    length = PATH_LENGTH(path) - LP_UNC_PREFIX_LENGTH;
 
-        /* If we're past the end of the string and it didnt't end with a slash,
-         * then we're a root path. */
-        return path_iterator >= path_end && !IS_SLASH(path_start[length - 1]);
+    path_end = &path_start[length];
+    path_iterator = path_start;
+
+    /* Server name must be at least one character. */
+    if(length <= 1) {
+        return 0;
     }
 
-    return 0;
+    /* Slide to the slash between the server and share names, if present. */
+    while(path_iterator < path_end && !IS_SLASH(*path_iterator)) {
+        path_iterator++;
+    }
+
+    /* Slide past the end of the string, stopping at the first slash we
+     * encounter. */
+    do {
+        path_iterator++;
+    }  while(path_iterator < path_end && !IS_SLASH(*path_iterator));
+
+    /* If we're past the end of the string and it didnt't end with a slash,
+     * then we're a root path. */
+    return path_iterator >= path_end && !IS_SLASH(path_start[length - 1]);
 }
 
 posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes,
@@ -1248,11 +1286,22 @@ posix_errno_t efile_set_cwd(const efile_path_t *path) {
 
     /* We have to use _wchdir since that's the only function that updates the
      * per-drive working directory, but it naively assumes that all paths
-     * starting with \\ are UNC paths, so we have to skip the \\?\-prefix. */
-    path_start = (WCHAR*)path->data + LP_PREFIX_LENGTH;
+     * starting with \\ are UNC paths, so we have to skip the long-path prefix.
+     *
+     * _wchdir doesn't handle long-prefixed UNC paths either so we hand those
+     * to SetCurrentDirectoryW instead. The per-drive working directory is
+     * irrelevant for such paths anyway. */
 
-    if(_wchdir(path_start)) {
-        return windows_to_posix_errno(GetLastError());
+    if(!IS_LONG_UNC_PATH(PATH_LENGTH(path), path->data)) {
+        path_start = (WCHAR*)path->data + LP_PREFIX_LENGTH;
+
+        if(_wchdir(path_start)) {
+            return windows_to_posix_errno(GetLastError());
+        }
+    } else {
+        if(!SetCurrentDirectoryW((WCHAR*)path->data)) {
+            return windows_to_posix_errno(GetLastError());
+        }
     }
 
     return 0;
@@ -1333,7 +1382,7 @@ posix_errno_t efile_altname(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TE
         int name_length;
 
         /* Reject path wildcards. */
-        if(wcspbrk(&((const WCHAR*)path->data)[4], L"?*")) {
+        if(wcspbrk(&((const WCHAR*)path->data)[LP_PREFIX_LENGTH], L"?*")) {
             return ENOENT;
         }
 
-- 
cgit v1.2.3


From 1801a42d032bf43b1e7bdb54e03ec4617a80fb0d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 15 Oct 2018 15:55:13 +0200
Subject: erts: Fix bug in debug_free for NULL pointer

causing ASSERT in sys_memset to fail.
---
 erts/emulator/beam/erl_alloc.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 8fe1ccb758..9e36d5e0d1 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -4030,6 +4030,9 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr)
 
     ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
 
+    if (!ptr)
+        return;
+
     dptr = check_memory_fence(ptr, &size, n, ERTS_ALC_O_FREE);
 
 #ifdef ERTS_ALC_A_EXEC
-- 
cgit v1.2.3


From bc541da07768ab0e11dca183ba235422ae0df593 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 18 Oct 2018 15:18:58 +0200
Subject: Honor the max heap size when copying literals after purging

When a module has been purged from memory, any literals belonging
to that module will be copied to all processes that hold references
to them.

The max heap size limit would be ignored in the garbage collection
initiated when copying literals to a process. If the max heap size
was exceeded, the process would typically be terminated in the
following garbage collection.

Since the process would be killed anyway later, kill the process
before copying a literal that would make it exceed its max heap
size.

While at it, also fix a potential bug in `erlang:garbage_collect/0`.
If it was found that the max heap sized had been exceeded while
executing `erlang:garbage_collect/0`, the process would enter a
kind of zombie state instead of being properly terminated.
---
 erts/emulator/beam/bif.c          |  4 ++++
 erts/emulator/beam/erl_gc.c       | 19 +++++++++++++++++++
 erts/emulator/test/code_SUITE.erl | 37 +++++++++++++++++++++++++++++++++++--
 3 files changed, 58 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index f18af8bcd7..015c051cc1 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3622,6 +3622,10 @@ erts_internal_garbage_collect_1(BIF_ALIST_1)
     default:        BIF_ERROR(BIF_P, BADARG);
     }
     erts_garbage_collect(BIF_P, 0, NULL, 0);
+    if (ERTS_PROC_IS_EXITING(BIF_P)) {
+        /* The max heap size limit was reached. */
+        return THE_NON_VALUE;
+    }
     return am_true;
 }
 
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index a65dbbf42b..47dd115c82 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1133,9 +1133,28 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
     reds = (Sint64) garbage_collect(p, ERTS_INVALID_HFRAG_PTR, 0,
 				    p->arg_reg, p->arity, fcalls,
 				    ygen_usage);
+    if (ERTS_PROC_IS_EXITING(p)) {
+        return 0;
+    }
 
     ASSERT(!(p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)));
 
+    if (MAX_HEAP_SIZE_GET(p)) {
+        Uint new_heap_size;
+        Uint old_heap_size;
+        Uint total_heap_size;
+
+        new_heap_size = HEAP_END(p) - HEAP_START(p);
+        old_heap_size = erts_next_heap_size(lit_size, 0);
+        total_heap_size = new_heap_size + old_heap_size;
+        if (MAX_HEAP_SIZE_GET(p) < total_heap_size &&
+            reached_max_heap_size(p, total_heap_size,
+                                  new_heap_size, old_heap_size)) {
+            erts_set_self_exiting(p, am_killed);
+            return 0;
+        }
+    }
+
     /*
      * Set GC state.
      */
diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index 9c6dc3ff83..0444ba4f89 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -28,7 +28,7 @@
          fake_literals/1,
          false_dependency/1,coverage/1,fun_confusion/1,
          t_copy_literals/1, t_copy_literals_frags/1,
-         erl_544/1]).
+         erl_544/1, max_heap_size/1]).
 
 -define(line_trace, 1).
 -include_lib("common_test/include/ct.hrl").
@@ -43,7 +43,7 @@ all() ->
      constant_pools, constant_refc_binaries, fake_literals,
      false_dependency,
      coverage, fun_confusion, t_copy_literals, t_copy_literals_frags,
-     erl_544].
+     erl_544, max_heap_size].
 
 init_per_suite(Config) ->
     erts_debug:set_internal_state(available_internal_state, true),
@@ -968,6 +968,39 @@ erl_544(Config) when is_list(Config) ->
             {skipped, "Only run when native file name encoding is utf8"}
     end.
 
+%% Test that the copying of literals to a process during purging of
+%% literals will cause the process to be killed if the max heap size
+%% is exceeded.
+max_heap_size(_Config) ->
+    Mod = ?FUNCTION_NAME,
+    Value = [I || I <- lists:seq(1, 5000)],
+    Code = gen_lit(Mod, [{term,Value}]),
+    {module,Mod} = erlang:load_module(Mod, Code),
+    SpawnOpts = [monitor,
+                 {max_heap_size,
+                  #{size=>1024,
+                    kill=>true,
+                    error_logger=>true}}],
+    {Pid,Ref} = spawn_opt(fun() ->
+                                  max_heap_size_proc(Mod)
+                          end, SpawnOpts),
+    receive
+        {'DOWN',Ref,process,Pid,Reason} ->
+            killed = Reason;
+        Other ->
+            ct:fail({unexpected_message,Other})
+    after 10000 ->
+            ct:fail({process_did_not_die, Pid, erlang:process_info(Pid)})
+    end.
+
+max_heap_size_proc(Mod) ->
+    Value = Mod:term(),
+    code:delete(Mod),
+    code:purge(Mod),
+    receive
+        _ -> Value
+    end.
+
 %% Utilities.
 
 make_sub_binary(Bin) when is_binary(Bin) ->
-- 
cgit v1.2.3


From 090f9543f66cc0c4ea4d1ca63902c300b103f271 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 9 Oct 2018 16:11:43 +0200
Subject: erts: Pass thread progress data where possible

The poll thread does a lot of waking up and then going
back to sleep. A large part of the waking up is managing
thread progress and a large part of that was using thread
specific data to get the thread progress data pointer.
With this refactor the tpd is passed to each of the functions
which greatly decreases the number of ethr_get_tsd calls
which in turn halves the CPU usage of the poller thread in
certain scenarios.
---
 erts/emulator/beam/erl_async.c          |  2 +-
 erts/emulator/beam/erl_init.c           |  4 +-
 erts/emulator/beam/erl_process.c        | 78 ++++++++++++++++++---------------
 erts/emulator/beam/erl_thr_progress.c   | 28 ++++++------
 erts/emulator/beam/erl_thr_progress.h   | 29 ++++++++----
 erts/emulator/beam/erl_trace.c          | 15 ++++---
 erts/emulator/sys/common/erl_check_io.c |  8 ++--
 erts/emulator/sys/common/erl_check_io.h |  5 ++-
 erts/emulator/sys/common/erl_poll.c     |  9 ++--
 erts/emulator/sys/common/erl_poll_api.h |  4 +-
 erts/emulator/sys/win32/erl_poll.c      |  7 +--
 11 files changed, 109 insertions(+), 80 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index 605a2b3461..44655ad5df 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -336,7 +336,7 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q,
 	    case ERTS_THR_Q_NEED_THR_PRGR:
 	    {
 		ErtsThrPrgrVal prgr = erts_thr_q_need_thr_progress(q);
-		erts_thr_progress_wakeup(NULL, prgr);
+		erts_thr_progress_wakeup(erts_thr_prgr_data(NULL), prgr);
 		/*
 		 * We do no dequeue finalizing in hope that a new async
 		 * job will arrive before we are woken due to thread
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 57c6c10c7f..ff8b9fd567 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -2358,8 +2358,8 @@ system_cleanup(int flush_async)
 	     * The exiting thread might be waiting for
 	     * us to block; need to update status...
 	     */
-	    erts_thr_progress_active(NULL, 0);
-	    erts_thr_progress_prepare_wait(NULL);
+	    erts_thr_progress_active(erts_thr_prgr_data(NULL), 0);
+	    erts_thr_progress_prepare_wait(erts_thr_prgr_data(NULL));
 	}
 	/* Wait forever... */
 	while (1)
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 0f7f1598fd..919d9f9edf 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1646,7 +1646,7 @@ haw_thr_prgr_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val)
 	    awdp->latest_wakeup = val;
 	    haw_chk_later_cleanup_op_wakeup(awdp, val);
 	}
-	erts_thr_progress_wakeup(awdp->esdp, val);
+	erts_thr_progress_wakeup(erts_thr_prgr_data(awdp->esdp), val);
     }
 }
 
@@ -1656,7 +1656,7 @@ haw_thr_prgr_soft_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val)
     if (erts_thr_progress_cmp(val, awdp->latest_wakeup) > 0) {
 	awdp->latest_wakeup = val;
 	haw_chk_later_cleanup_op_wakeup(awdp, val);
-	erts_thr_progress_wakeup(awdp->esdp, val);
+	erts_thr_progress_wakeup(erts_thr_prgr_data(awdp->esdp), val);
     }
 }
 
@@ -1670,7 +1670,7 @@ haw_thr_prgr_later_cleanup_op_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val,
 	else {
 	    awdp->latest_wakeup = val;
 	    awdp->later_op.size = thr_prgr_later_cleanup_op_threshold;
-	    erts_thr_progress_wakeup(awdp->esdp, val);
+	    erts_thr_progress_wakeup(erts_thr_prgr_data(awdp->esdp), val);
 	}
     }
 }
@@ -3066,6 +3066,7 @@ aux_thread(void *unused)
     ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(-1);
     erts_aint32_t aux_work;
     ErtsThrPrgrCallbacks callbacks;
+    ErtsThrPrgrData *tpd;
     int thr_prgr_active = 1;
     ERTS_MSACC_DECLARE_CACHE();
 
@@ -3087,12 +3088,12 @@ aux_thread(void *unused)
     callbacks.wait = thr_prgr_wait;
     callbacks.finalize_wait = thr_prgr_fin_wait;
 
-    erts_thr_progress_register_managed_thread(NULL, &callbacks, 1);
+    tpd = erts_thr_progress_register_managed_thread(NULL, &callbacks, 1);
     init_aux_work_data(awdp, NULL, NULL);
     awdp->ssi = ssi;
 
 #if ERTS_POLL_USE_FALLBACK
-    ssi->psi = erts_create_pollset_thread(-1);
+    ssi->psi = erts_create_pollset_thread(-1, tpd);
 #endif
 
     sched_prep_spin_wait(ssi);
@@ -3105,11 +3106,11 @@ aux_thread(void *unused)
 	aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 	if (aux_work) {
 	    if (!thr_prgr_active)
-		erts_thr_progress_active(NULL, thr_prgr_active = 1);
+		erts_thr_progress_active(tpd, thr_prgr_active = 1);
 	    aux_work = handle_aux_work(awdp, aux_work, 1);
             ERTS_MSACC_UPDATE_CACHE();
-	    if (aux_work && erts_thr_progress_update(NULL))
-		erts_thr_progress_leader_update(NULL);
+	    if (aux_work && erts_thr_progress_update(tpd))
+		erts_thr_progress_leader_update(tpd);
 	}
 
 	if (!aux_work) {
@@ -3120,7 +3121,7 @@ aux_thread(void *unused)
 #endif
 
 	    if (thr_prgr_active)
-		erts_thr_progress_active(NULL, thr_prgr_active = 0);
+		erts_thr_progress_active(tpd, thr_prgr_active = 0);
 
 #if ERTS_POLL_USE_FALLBACK
 
@@ -3136,7 +3137,7 @@ aux_thread(void *unused)
 		}
             }
 #else
-            erts_thr_progress_prepare_wait(NULL);
+            erts_thr_progress_prepare_wait(tpd);
 
 	    flgs = sched_spin_wait(ssi, 0);
 
@@ -3153,7 +3154,7 @@ aux_thread(void *unused)
                     ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER);
 		}
             }
-            erts_thr_progress_finalize_wait(NULL);
+            erts_thr_progress_finalize_wait(tpd);
 #endif
 	}
 
@@ -3171,7 +3172,8 @@ poll_thread(void *arg)
     erts_aint32_t aux_work;
     ErtsThrPrgrCallbacks callbacks;
     int thr_prgr_active = 1;
-    struct erts_poll_thread *psi = erts_create_pollset_thread(id);
+    struct erts_poll_thread *psi;
+    ErtsThrPrgrData *tpd;
     ERTS_MSACC_DECLARE_CACHE();
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -3192,9 +3194,12 @@ poll_thread(void *arg)
     callbacks.wait = thr_prgr_wait;
     callbacks.finalize_wait = thr_prgr_fin_wait;
 
-    erts_thr_progress_register_managed_thread(NULL, &callbacks, 0);
+    tpd = erts_thr_progress_register_managed_thread(NULL, &callbacks, 0);
     init_aux_work_data(awdp, NULL, NULL);
     awdp->ssi = ssi;
+
+    psi = erts_create_pollset_thread(id, tpd);
+
     ssi->psi = psi;
 
     sched_prep_spin_wait(ssi);
@@ -3207,16 +3212,16 @@ poll_thread(void *arg)
 	aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 	if (aux_work) {
 	    if (!thr_prgr_active)
-		erts_thr_progress_active(NULL, thr_prgr_active = 1);
+		erts_thr_progress_active(tpd, thr_prgr_active = 1);
 	    aux_work = handle_aux_work(awdp, aux_work, 1);
             ERTS_MSACC_UPDATE_CACHE();
-	    if (aux_work && erts_thr_progress_update(NULL))
-		erts_thr_progress_leader_update(NULL);
+	    if (aux_work && erts_thr_progress_update(tpd))
+		erts_thr_progress_leader_update(tpd);
 	}
 
 	if (!aux_work) {
 	    if (thr_prgr_active)
-		erts_thr_progress_active(NULL, thr_prgr_active = 0);
+		erts_thr_progress_active(tpd, thr_prgr_active = 0);
 
 	    flgs = sched_spin_wait(ssi, 0);
 
@@ -3286,13 +3291,13 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
         aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
         if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) {
             if (!thr_prgr_active) {
-                erts_thr_progress_active(esdp, thr_prgr_active = 1);
+                erts_thr_progress_active(erts_thr_prgr_data(esdp), thr_prgr_active = 1);
                 sched_wall_time_change(esdp, 1);
             }
             aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1);
             ERTS_MSACC_UPDATE_CACHE();
-            if (aux_work && erts_thr_progress_update(esdp))
-                erts_thr_progress_leader_update(esdp);
+            if (aux_work && erts_thr_progress_update(erts_thr_prgr_data(esdp)))
+                erts_thr_progress_leader_update(erts_thr_prgr_data(esdp));
         }
 
         if (aux_work) {
@@ -3301,7 +3306,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
                 current_time = erts_get_monotonic_time(esdp);
                 if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
                     if (!thr_prgr_active) {
-                        erts_thr_progress_active(esdp, thr_prgr_active = 1);
+                        erts_thr_progress_active(erts_thr_prgr_data(esdp), thr_prgr_active = 1);
                         sched_wall_time_change(esdp, 1);
                     }
                     erts_bump_timers(esdp->timer_wheel, current_time);
@@ -3321,17 +3326,17 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
             }
             if (do_timeout) {
                 if (!thr_prgr_active) {
-                    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+                    erts_thr_progress_active(erts_thr_prgr_data(esdp), thr_prgr_active = 1);
                     sched_wall_time_change(esdp, 1);
                 }
             }
             else {
                 if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
                     if (thr_prgr_active) {
-                        erts_thr_progress_active(esdp, thr_prgr_active = 0);
+                        erts_thr_progress_active(erts_thr_prgr_data(esdp), thr_prgr_active = 0);
                         sched_wall_time_change(esdp, 0);
                     }
-                    erts_thr_progress_prepare_wait(esdp);
+                    erts_thr_progress_prepare_wait(erts_thr_prgr_data(esdp));
                 }
 
                 flgs = sched_spin_wait(ssi, spincount);
@@ -3363,7 +3368,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
                     }
                 }
                 if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-                    erts_thr_progress_finalize_wait(esdp);
+                    erts_thr_progress_finalize_wait(erts_thr_prgr_data(esdp));
             }
             if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time)
                 erts_bump_timers(esdp->timer_wheel, current_time);
@@ -3392,7 +3397,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
     if (ERTS_SCHEDULER_IS_DIRTY(esdp))
         dirty_sched_wall_time_change(esdp, working = 1);
     else if (!thr_prgr_active) {
-        erts_thr_progress_active(esdp, thr_prgr_active = 1);
+        erts_thr_progress_active(erts_thr_prgr_data(esdp), thr_prgr_active = 1);
         sched_wall_time_change(esdp, 1);
     }
 
@@ -7565,7 +7570,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 		    if (aux_work|evacuate) {
 			if (!thr_prgr_active) {
-			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+			    erts_thr_progress_active(erts_thr_prgr_data(esdp),
+                                                     thr_prgr_active = 1);
 			    sched_wall_time_change(esdp, 1);
 			}
 			if (aux_work)
@@ -7573,8 +7579,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 						       aux_work,
 						       1);
 
-			if (aux_work && erts_thr_progress_update(esdp))
-			    erts_thr_progress_leader_update(esdp);
+			if (aux_work && erts_thr_progress_update(erts_thr_prgr_data(esdp)))
+			    erts_thr_progress_leader_update(erts_thr_prgr_data(esdp));
 			if (evacuate) {
 			    erts_runq_lock(esdp->run_queue);
 			    evacuate_run_queue(esdp->run_queue, &sbp);
@@ -7593,18 +7599,18 @@ suspend_scheduler(ErtsSchedulerData *esdp)
                     if (!aux_work && current_time < timeout_time) {
                         /* go to sleep... */
                         if (thr_prgr_active) {
-                            erts_thr_progress_active(esdp, thr_prgr_active = 0);
+                            erts_thr_progress_active(erts_thr_prgr_data(esdp), thr_prgr_active = 0);
                             sched_wall_time_change(esdp, 0);
                         }
-                        erts_thr_progress_prepare_wait(NULL);
+                        erts_thr_progress_prepare_wait(erts_thr_prgr_data(NULL));
                         suspend_normal_scheduler_sleep(esdp);
-                        erts_thr_progress_finalize_wait(NULL);
+                        erts_thr_progress_finalize_wait(erts_thr_prgr_data(NULL));
                         current_time = erts_get_monotonic_time(esdp);
                     }
 
                     if (current_time >= timeout_time) {
                         if (!thr_prgr_active) {
-                            erts_thr_progress_active(esdp, thr_prgr_active = 1);
+                            erts_thr_progress_active(erts_thr_prgr_data(esdp), thr_prgr_active = 1);
                             sched_wall_time_change(esdp, 1);
                         }
                         erts_bump_timers(esdp->timer_wheel, current_time);
@@ -7661,7 +7667,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
             profile_scheduler(make_small(esdp->no), am_active);
 
         if (!thr_prgr_active) {
-            erts_thr_progress_active(esdp, thr_prgr_active = 1);
+            erts_thr_progress_active(erts_thr_prgr_data(esdp), thr_prgr_active = 1);
             sched_wall_time_change(esdp, 1);
         }
     }
@@ -9313,12 +9319,12 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		}
 	    }
 
-	    leader_update = erts_thr_progress_update(esdp);
+	    leader_update = erts_thr_progress_update(erts_thr_prgr_data(esdp));
 	    aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work);
 	    if (aux_work | leader_update) {
 		erts_runq_unlock(rq);
 		if (leader_update)
-		    erts_thr_progress_leader_update(esdp);
+		    erts_thr_progress_leader_update(erts_thr_prgr_data(esdp));
 		if (aux_work)
 		    handle_aux_work(&esdp->aux_work_data, aux_work, 0);
 		erts_runq_lock(rq);
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index aa08eb40ec..20b05dc942 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -508,6 +508,10 @@ init_wakeup_request_array(ErtsThrPrgrVal *w)
     }
 }
 
+ErtsThrPrgrData *erts_thr_progress_data(void) {
+    return erts_tsd_get(erts_thr_prgr_data_key__);
+}
+
 void
 erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks)
 {
@@ -551,7 +555,7 @@ erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks)
 }
 
 
-void
+ErtsThrPrgrData *
 erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp,
 					  ErtsThrPrgrCallbacks *callbacks,
 					  int pref_wakeup)
@@ -630,6 +634,7 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp,
 		wakeup_managed(id);
     }
     callbacks->finalize_wait(callbacks->arg);
+    return tpd;
 }
 
 static ERTS_INLINE int
@@ -849,23 +854,22 @@ update(ErtsThrPrgrData *tpd)
 }
 
 int
-erts_thr_progress_update(ErtsSchedulerData *esdp)
+erts_thr_progress_update(ErtsThrPrgrData *tpd)
 {
-    return update(thr_prgr_data(esdp));
+    return update(tpd);
 }
 
 
 int
-erts_thr_progress_leader_update(ErtsSchedulerData *esdp)
+erts_thr_progress_leader_update(ErtsThrPrgrData *tpd)
 {
-    return leader_update(thr_prgr_data(esdp));
+    return leader_update(tpd);
 }
 
 void
-erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp)
+erts_thr_progress_prepare_wait(ErtsThrPrgrData *tpd)
 {
     erts_aint32_t lflgs;
-    ErtsThrPrgrData *tpd = thr_prgr_data(esdp);
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_check_exact(NULL, 0);
@@ -889,9 +893,8 @@ erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp)
 }
 
 void
-erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp)
+erts_thr_progress_finalize_wait(ErtsThrPrgrData *tpd)
 {
-    ErtsThrPrgrData *tpd = thr_prgr_data(esdp);
     ErtsThrPrgrVal current, val;
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -921,9 +924,8 @@ erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp)
 }
 
 void
-erts_thr_progress_active(ErtsSchedulerData *esdp, int on)
+erts_thr_progress_active(ErtsThrPrgrData *tpd, int on)
 {
-    ErtsThrPrgrData *tpd = thr_prgr_data(esdp);
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_check_exact(NULL, 0);
@@ -1182,10 +1184,10 @@ request_wakeup_unmanaged(ErtsThrPrgrData *tpd, ErtsThrPrgrVal value)
 }
 
 void
-erts_thr_progress_wakeup(ErtsSchedulerData *esdp,
+erts_thr_progress_wakeup(ErtsThrPrgrData *tpd,
 			 ErtsThrPrgrVal value)
 {
-    ErtsThrPrgrData *tpd = thr_prgr_data(esdp);
+
     ASSERT(!tpd->is_temporary);
     if (tpd->is_managed)
 	request_wakeup_managed(tpd, value);
diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h
index 8329995b24..00a9e61407 100644
--- a/erts/emulator/beam/erl_thr_progress.h
+++ b/erts/emulator/beam/erl_thr_progress.h
@@ -123,22 +123,24 @@ extern ErtsThrPrgr erts_thr_prgr__;
 
 void erts_thr_progress_pre_init(void);
 void erts_thr_progress_init(int no_schedulers, int managed, int unmanaged);
-void erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp,
-					       ErtsThrPrgrCallbacks *,
-					       int);
+ErtsThrPrgrData *erts_thr_progress_register_managed_thread(
+    ErtsSchedulerData *esdp, ErtsThrPrgrCallbacks *, int);
 void erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *);
-void erts_thr_progress_active(ErtsSchedulerData *esdp, int on);
-void erts_thr_progress_wakeup(ErtsSchedulerData *esdp,
+void erts_thr_progress_active(ErtsThrPrgrData *, int on);
+void erts_thr_progress_wakeup(ErtsThrPrgrData *,
 			      ErtsThrPrgrVal value);
-int erts_thr_progress_update(ErtsSchedulerData *esdp);
-int erts_thr_progress_leader_update(ErtsSchedulerData *esdp);
-void erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp);
-void erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp);
+int erts_thr_progress_update(ErtsThrPrgrData *);
+int erts_thr_progress_leader_update(ErtsThrPrgrData *);
+void erts_thr_progress_prepare_wait(ErtsThrPrgrData *);
+void erts_thr_progress_finalize_wait(ErtsThrPrgrData *);
 ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay__(void);
 void erts_thr_progress_unmanaged_continue__(int umrefc_ix);
+ErtsThrPrgrData *erts_thr_progress_data(void);
 
 void erts_thr_progress_dbg_print_state(void);
 
+ERTS_GLB_INLINE ErtsThrPrgrData *erts_thr_prgr_data(ErtsSchedulerData *esdp);
+
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc);
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc);
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc);
@@ -161,6 +163,15 @@ ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
+ERTS_GLB_INLINE ErtsThrPrgrData *
+erts_thr_prgr_data(ErtsSchedulerData *esdp) {
+    if (esdp) {
+        return &esdp->thr_progress_data;
+    } else {
+        return erts_thr_progress_data();
+    }
+}
+
 ERTS_GLB_INLINE ErtsThrPrgrVal
 erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
 {
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 53a020e7a5..2350d4c02f 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -2177,6 +2177,7 @@ sys_msg_dispatcher_func(void *unused)
 {
     ErtsThrPrgrCallbacks callbacks;
     ErtsSysMsgQ *local_sys_message_queue = NULL;
+    ErtsThrPrgrData *tpd;
     int wait = 0;
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -2189,7 +2190,7 @@ sys_msg_dispatcher_func(void *unused)
     callbacks.wait = sys_msg_dispatcher_wait;
     callbacks.finalize_wait = sys_msg_dispatcher_fin_wait;
 
-    erts_thr_progress_register_managed_thread(NULL, &callbacks, 0);
+    tpd = erts_thr_progress_register_managed_thread(NULL, &callbacks, 0);
 
     while (1) {
 	int end_wait = 0;
@@ -2210,8 +2211,8 @@ sys_msg_dispatcher_func(void *unused)
 	if (!sys_message_queue) {
 	    erts_mtx_unlock(&smq_mtx);
 	    end_wait = 1;
-	    erts_thr_progress_active(NULL, 0);
-	    erts_thr_progress_prepare_wait(NULL);
+	    erts_thr_progress_active(tpd, 0);
+	    erts_thr_progress_prepare_wait(tpd);
 	    erts_mtx_lock(&smq_mtx);
 	}
 
@@ -2225,8 +2226,8 @@ sys_msg_dispatcher_func(void *unused)
 	erts_mtx_unlock(&smq_mtx);
 
 	if (end_wait) {
-	    erts_thr_progress_finalize_wait(NULL);
-	    erts_thr_progress_active(NULL, 1);
+	    erts_thr_progress_finalize_wait(tpd);
+	    erts_thr_progress_active(tpd, 1);
 	}
 
 	/* Send trace messages ... */
@@ -2239,8 +2240,8 @@ sys_msg_dispatcher_func(void *unused)
 	    Process *proc = NULL;
 	    Port *port = NULL;
 
-	    if (erts_thr_progress_update(NULL))
-		erts_thr_progress_leader_update(NULL);
+	    if (erts_thr_progress_update(tpd))
+		erts_thr_progress_leader_update(tpd);
 
 #ifdef DEBUG_PRINTOUTS
 	    print_msg_type(smqp);
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 9f115706dc..748555165f 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -105,6 +105,7 @@ typedef struct erts_poll_thread
 {
     ErtsPollSet *ps;
     ErtsPollResFd *pollres;
+    ErtsThrPrgrData *tpd;
     int pollres_len;
 } ErtsPollThread;
 
@@ -1516,7 +1517,8 @@ erts_check_io_interrupt(ErtsPollThread *psi, int set)
 }
 
 ErtsPollThread *
-erts_create_pollset_thread(int id) {
+erts_create_pollset_thread(int id, ErtsThrPrgrData *tpd) {
+    psiv[id].tpd = tpd;
     return psiv+id;
 }
 
@@ -1538,12 +1540,12 @@ erts_check_io(ErtsPollThread *psi)
 #if ERTS_POLL_USE_FALLBACK
     if (psi->ps == get_fallback()) {
 
-        poll_ret = erts_poll_wait_flbk(psi->ps, psi->pollres, &pollres_len);
+        poll_ret = erts_poll_wait_flbk(psi->ps, psi->pollres, &pollres_len, psi->tpd);
 
     } else
 #endif
     {
-        poll_ret = erts_poll_wait(psi->ps, psi->pollres, &pollres_len);
+        poll_ret = erts_poll_wait(psi->ps, psi->pollres, &pollres_len, psi->tpd);
     }
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index 443ef1264c..16aba8a5f3 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -90,8 +90,11 @@ void erts_check_io_interrupt(struct erts_poll_thread *pt, int set);
 /**
  * Create a new poll thread structure that is associated with the number no.
  * It is the callers responsibility that no is unique.
+ *
+ * @param no the id of the pollset thread, -2 = aux thread, -1 = scheduler
+ * @param tpd the thread progress data of the pollset thread
  */
-struct erts_poll_thread* erts_create_pollset_thread(int no);
+struct erts_poll_thread* erts_create_pollset_thread(int no, ErtsThrPrgrData *tpd);
 #ifdef ERTS_ENABLE_LOCK_COUNT
 /**
  * Toggle lock counting on all check io locks
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index b4d1575ee5..5a5874ddfa 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -75,6 +75,7 @@
 #  define WANT_NONBLOCKING
 #endif
 
+#include "erl_thr_progress.h"
 #include "erl_poll.h"
 #if ERTS_POLL_USE_KQUEUE
 #  include <sys/types.h>
@@ -95,7 +96,6 @@
 #    include <limits.h>
 #  endif
 #endif
-#include "erl_thr_progress.h"
 #include "erl_driver.h"
 #include "erl_alloc.h"
 #include "erl_msacc.h"
@@ -1629,7 +1629,8 @@ check_fd_events(ErtsPollSet *ps, ErtsPollResFd pr[], int do_wait, int max_res)
 int
 ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet *ps,
 				 ErtsPollResFd pr[],
-				 int *len)
+				 int *len,
+                                 ErtsThrPrgrData *tpd)
 {
     int res, no_fds, used_fds = 0;
     int ebadf = 0;
@@ -1659,7 +1660,7 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet *ps,
     DEBUG_PRINT_WAIT("Entering %s(), do_wait=%d", ps, __FUNCTION__, do_wait);
 
     if (do_wait) {
-        erts_thr_progress_prepare_wait(NULL);
+        erts_thr_progress_prepare_wait(tpd);
         ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP);
     }
 
@@ -1703,7 +1704,7 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet *ps,
     }
 
     if (do_wait) {
-        erts_thr_progress_finalize_wait(NULL);
+        erts_thr_progress_finalize_wait(tpd);
         ERTS_MSACC_UPDATE_CACHE();
         ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_CHECK_IO);
     }
diff --git a/erts/emulator/sys/common/erl_poll_api.h b/erts/emulator/sys/common/erl_poll_api.h
index 1170a549b9..f35f64a9f3 100644
--- a/erts/emulator/sys/common/erl_poll_api.h
+++ b/erts/emulator/sys/common/erl_poll_api.h
@@ -72,11 +72,13 @@ ErtsPollEvents ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet *ps,
  * @param res an array of fd results that the ready fds are put in.
  * @param[in] length the length of the res array
  * @param[out] length the number of ready events returned in res
+ * @param tpd the thread progress data to note sleep state in
  * @return 0 on success, else the ERRNO of the error that happened.
  */
 int ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet *ps,
                                      ErtsPollResFd res[],
-                                     int *length);
+                                     int *length,
+                                     ErtsThrPrgrData *tpd);
 /**
  * Interrupt the thread waiting in the pollset. This function should be called
  * with set = 0 before any thread calls erts_poll_wait in order to clear any
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index 39bb4d515e..5d832f4d34 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -1017,7 +1017,8 @@ ErtsPollEvents erts_poll_control(ErtsPollSet *ps,
 
 int erts_poll_wait(ErtsPollSet *ps,
 		   ErtsPollResFd pr[],
-		   int *len)
+		   int *len,
+                   ErtsThrPrgrData *tpd)
 {
     int no_fds;
     DWORD timeout = INFINITE;
@@ -1056,10 +1057,10 @@ int erts_poll_wait(ErtsPollSet *ps,
 
 	HARDDEBUGF(("Start waiting %d [%d]",num_h, (int) timeout));
 	ERTS_POLLSET_UNLOCK(ps);
-	erts_thr_progress_prepare_wait(NULL);
+	erts_thr_progress_prepare_wait(tpd);
         ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP);
 	handle = WaitForMultipleObjects(num_h, harr, FALSE, timeout);
-	erts_thr_progress_finalize_wait(NULL);
+	erts_thr_progress_finalize_wait(tpd);
         ERTS_MSACC_POP_STATE();
 	ERTS_POLLSET_LOCK(ps);
 	HARDDEBUGF(("Stop waiting %d [%d]",num_h, (int) timeout));
-- 
cgit v1.2.3


From 4047f7177835928f2205fb728c65247ea68d5d59 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 11 Oct 2018 16:27:34 +0200
Subject: erts: Make thr prgr wakeup current or sched 1

Before this change, it was always the aux thread that was woken
to handle thread progress events scheduled to happen when all
schedulers were going to sleep. This was ok in the pre-OTP-21
implementation when the aux thread just slept on a tse. Now that
it sleeps in the fallback pollset this uses too much cpu so
instead we wake the thread that is doing the request if it is
a managed thread, or else we wake scheduler 1.
---
 erts/emulator/beam/erl_thr_progress.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 20b05dc942..bac437efe9 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -801,7 +801,7 @@ leader_update(ErtsThrPrgrData *tpd)
 		     == ERTS_THR_PRGR_LFLG_NO_LEADER))
 		&& got_sched_wakeups()) {
 		/* Someone need to make progress */
-		wakeup_managed(0);
+		wakeup_managed(tpd->id);
 	    }
 	}
     }
@@ -888,7 +888,7 @@ erts_thr_progress_prepare_wait(ErtsThrPrgrData *tpd)
 	== ERTS_THR_PRGR_LFLG_NO_LEADER 
 	&& got_sched_wakeups()) {
 	/* Someone need to make progress */
-	wakeup_managed(0);
+	wakeup_managed(tpd->id);
     }
 }
 
@@ -975,7 +975,7 @@ unmanaged_continue(ErtsThrPrgrDelayHandle handle)
 	    == (ERTS_THR_PRGR_LFLG_NO_LEADER|ERTS_THR_PRGR_LFLG_WAITING_UM)
 	    && got_sched_wakeups()) {
 	    /* Others waiting for us... */
-	    wakeup_managed(0);
+	    wakeup_managed(1);
 	}
     }
 }
-- 
cgit v1.2.3


From d6e8bddde0887894ae70cc2c6d4230532801bf97 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 5 Oct 2018 13:15:49 +0200
Subject: Extend the sharing-preserving routines to optionally copy literals

In the implementation of the zero-copying term storage, we
want to preserve sharing, but not copy literals because the
modules holding the literals could be unloaded under our feet.
---
 erts/emulator/beam/copy.c   | 10 ++++++----
 erts/emulator/beam/global.h |  2 ++
 2 files changed, 8 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index e7bfd04b73..e7bd046e18 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -1074,6 +1074,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info)
     Eterm* ptr;
     Eterm *lit_purge_ptr = info->lit_purge_ptr;
     Uint lit_purge_sz = info->lit_purge_sz;
+    int copy_literals = info->copy_literals;
 #ifdef DEBUG
     Eterm mypid = erts_get_current_pid();
 #endif
@@ -1119,7 +1120,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info)
 	    /* off heap list pointers are copied verbatim */
 	    if (erts_is_literal(obj,ptr)) {
 		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj));
-                if (in_literal_purge_area(ptr))
+                if (copy_literals || in_literal_purge_area(ptr))
                     info->literal_size += size_object(obj);
 		goto pop_next;
 	    }
@@ -1170,7 +1171,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info)
 	    /* off heap pointers to boxes are copied verbatim */
 	    if (erts_is_literal(obj,ptr)) {
 		VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj));
-                if (in_literal_purge_area(ptr))
+                if (copy_literals || in_literal_purge_area(ptr))
                     info->literal_size += size_object(obj);
 		goto pop_next;
 	    }
@@ -1338,6 +1339,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
     unsigned remaining;
     Eterm *lit_purge_ptr = info->lit_purge_ptr;
     Uint lit_purge_sz = info->lit_purge_sz;
+    int copy_literals = info->copy_literals;
 #ifdef DEBUG
     Eterm mypid = erts_get_current_pid();
     Eterm saved_obj = obj;
@@ -1387,7 +1389,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 	    ptr = list_val(obj);
 	    /* off heap list pointers are copied verbatim */
 	    if (erts_is_literal(obj,ptr)) {
-                if (!in_literal_purge_area(ptr)) {
+                if (!(copy_literals || in_literal_purge_area(ptr))) {
                     *resp = obj;
                 } else {
                     Uint bsz = 0;
@@ -1455,7 +1457,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info,
 	    ptr = boxed_val(obj);
 	    /* off heap pointers to boxes are copied verbatim */
 	    if (erts_is_literal(obj,ptr)) {
-                if (!in_literal_purge_area(ptr)) {
+                if (!(copy_literals || in_literal_purge_area(ptr))) {
                     *resp = obj;
                 } else {
                     Uint bsz = 0;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 21ae205237..9fc5abbf68 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1001,6 +1001,7 @@ typedef struct {
     Uint literal_size;
     Eterm *lit_purge_ptr;
     Uint lit_purge_sz;
+    int copy_literals;
 } erts_shcopy_t;
 
 #define INITIALIZE_SHCOPY(info)						\
@@ -1010,6 +1011,7 @@ typedef struct {
 	info.bitstore_start = info.bitstore_default;			\
 	info.shtable_start = info.shtable_default;			\
 	info.literal_size = 0;						\
+	info.copy_literals = 0;						\
 	if (larea__) {							\
 	    info.lit_purge_ptr = &larea__->start[0];			\
 	    info.lit_purge_sz = larea__->end - info.lit_purge_ptr;	\
-- 
cgit v1.2.3


From 19972f42719c6e9357e33e123d00b681213022dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 23 Oct 2018 09:37:16 +0200
Subject: Fix trapping in lists:reverse/2

The first stage wasn't bounded by reductions, and it bumped far
more reductions than it should have due to a logic bug.
---
 erts/emulator/beam/erl_bif_lists.c | 34 ++++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 14 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index 395be67a90..9b2b8aaab6 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -283,7 +283,7 @@ static BIF_RETTYPE lists_reverse_alloc(Process *c_p,
 {
     static const Uint CELLS_PER_RED = 40;
 
-    Eterm *heap_top, *heap_end;
+    Eterm *alloc_top, *alloc_end;
     Uint cells_left, max_cells;
     Eterm list, tail;
     Eterm lookahead;
@@ -305,18 +305,18 @@ static BIF_RETTYPE lists_reverse_alloc(Process *c_p,
         BIF_ERROR(c_p, BADARG);
     }
 
-    heap_top = HAlloc(c_p, 2 * (max_cells - cells_left));
-    heap_end = heap_top + 2 * (max_cells - cells_left);
+    alloc_top = HAlloc(c_p, 2 * (max_cells - cells_left));
+    alloc_end = alloc_top + 2 * (max_cells - cells_left);
 
-    while (heap_top < heap_end) {
+    while (alloc_top < alloc_end) {
         Eterm *pair = list_val(list);
 
-        tail = CONS(heap_top, CAR(pair), tail);
+        tail = CONS(alloc_top, CAR(pair), tail);
         list = CDR(pair);
 
         ASSERT(is_list(list) || is_nil(list));
 
-        heap_top += 2;
+        alloc_top += 2;
     }
 
     if (is_nil(list)) {
@@ -333,7 +333,7 @@ static BIF_RETTYPE lists_reverse_onheap(Process *c_p,
 {
     static const Uint CELLS_PER_RED = 60;
 
-    Eterm *heap_top, *heap_end;
+    Eterm *alloc_start, *alloc_top, *alloc_end;
     Uint cells_left, max_cells;
     Eterm list, tail;
 
@@ -343,21 +343,27 @@ static BIF_RETTYPE lists_reverse_onheap(Process *c_p,
     cells_left = max_cells = CELLS_PER_RED * (1 + ERTS_BIF_REDS_LEFT(c_p));
 
     ASSERT(HEAP_LIMIT(c_p) >= HEAP_TOP(c_p) + 2);
-    heap_end = HEAP_LIMIT(c_p) - 2;
-    heap_top = HEAP_TOP(c_p);
+    alloc_start = HEAP_TOP(c_p);
+    alloc_end = HEAP_LIMIT(c_p) - 2;
+    alloc_top = alloc_start;
 
-    while (heap_top < heap_end && is_list(list)) {
+    /* Don't process more cells than we have reductions for. */
+    alloc_end = MIN(alloc_top + (cells_left * 2), alloc_end);
+
+    while (alloc_top < alloc_end && is_list(list)) {
         Eterm *pair = list_val(list);
 
-        tail = CONS(heap_top, CAR(pair), tail);
+        tail = CONS(alloc_top, CAR(pair), tail);
         list = CDR(pair);
 
-        heap_top += 2;
+        alloc_top += 2;
     }
 
-    cells_left -= (heap_top - heap_end) / 2;
+    cells_left -= (alloc_top - alloc_start) / 2;
+    HEAP_TOP(c_p) = alloc_top;
+
+    ASSERT(cells_left >= 0 && cells_left <= max_cells);
     BUMP_REDS(c_p, (max_cells - cells_left) / CELLS_PER_RED);
-    HEAP_TOP(c_p) = heap_top;
 
     if (is_nil(list)) {
         BIF_RET(tail);
-- 
cgit v1.2.3


From 0aad6ef3bf360f2971d5c8b22620aecc3cdae3ef Mon Sep 17 00:00:00 2001
From: Dmytro Lytovchenko <dmytro.lytovchenko@erlang-solutions.com>
Date: Tue, 23 Oct 2018 09:14:11 +0200
Subject: Clarify a magical allocation size

---
 erts/emulator/beam/external.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 621ba108ba..9a66e491f3 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1953,7 +1953,8 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla
 
 #define RETURN_STATE()							\
     do {								\
-	hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE+3);                    \
+	static const int TUPLE2_SIZE = 2 + 1;				\
+	hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE + TUPLE2_SIZE);        \
 	c_term = erts_mk_magic_ref(&hp, &MSO(p), context_b);            \
 	res = TUPLE2(hp, Term, c_term);					\
 	BUMP_ALL_REDS(p);                                               \
-- 
cgit v1.2.3


From 1056d2d1fd49f669a2001f03890e13c9cba76c1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Thu, 25 Oct 2018 08:33:08 +0200
Subject: Inline erts_cmp

This greatly increases the performance of '--'/2 which does a lot
of term comparisons.
---
 erts/emulator/beam/atom.c      |  2 +-
 erts/emulator/beam/erl_utils.h | 61 ++++++++++++++++++++++++++++++++++++++--
 erts/emulator/beam/utils.c     | 64 ++++--------------------------------------
 3 files changed, 64 insertions(+), 63 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index 5381611fab..59b51fd15e 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -174,7 +174,7 @@ atom_alloc(Atom* tmpl)
 
     /*
      * Precompute ordinal value of first 3 bytes + 7 bits.
-     * This is used by utils.c:erts_cmp_atoms().
+     * This is used by erl_utils.h:erts_cmp_atoms().
      * We cannot use the full 32 bits of the first 4 bytes,
      * since we use the sign of the difference between two
      * ordinal values to represent their relative order.
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index b3bfa69052..880febba8b 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -22,6 +22,7 @@
 #define ERL_UTILS_H__
 
 #include "sys.h"
+#include "atom.h"
 #include "erl_printf.h"
 
 struct process;
@@ -112,10 +113,12 @@ int eq(Eterm, Eterm);
 
 #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y))))
 
-int erts_cmp_atoms(Eterm a, Eterm b);
-Sint erts_cmp(Eterm, Eterm, int, int);
-Sint erts_cmp_compound(Eterm, Eterm, int, int);
+ERTS_GLB_INLINE Sint erts_cmp(Eterm, Eterm, int, int);
+ERTS_GLB_INLINE int erts_cmp_atoms(Eterm a, Eterm b);
+
 Sint cmp(Eterm a, Eterm b);
+Sint erts_cmp_compound(Eterm, Eterm, int, int);
+
 #define CMP(A,B)                         erts_cmp(A,B,0,0)
 #define CMP_TERM(A,B)                    erts_cmp(A,B,1,0)
 #define CMP_EQ_ONLY(A,B)                 erts_cmp(A,B,0,1)
@@ -150,4 +153,56 @@ Sint cmp(Eterm a, Eterm b);
 	if (erts_cmp_compound(X,Y,0,EqOnly) Op 0) { Action; };	\
     }
 
+#define erts_float_comp(x,y) (((x)<(y)) ? -1 : (((x)==(y)) ? 0 : 1))
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE int erts_cmp_atoms(Eterm a, Eterm b) {
+    Atom *aa = atom_tab(atom_val(a));
+    Atom *bb = atom_tab(atom_val(b));
+
+    byte *name_a, *name_b;
+    int len_a, len_b, diff;
+
+    diff = aa->ord0 - bb->ord0;
+
+    if (diff != 0) {
+        return diff;
+    }
+
+    name_a = &aa->name[3];
+    name_b = &bb->name[3];
+    len_a = aa->len-3;
+    len_b = bb->len-3;
+
+    if (len_a > 0 && len_b > 0) {
+        diff = sys_memcmp(name_a, name_b, MIN(len_a, len_b));
+
+        if (diff != 0) {
+            return diff;
+        }
+    }
+
+    return len_a - len_b;
+}
+
+ERTS_GLB_INLINE Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only) {
+    if (is_atom(a) && is_atom(b)) {
+        return erts_cmp_atoms(a, b);
+    } else if (is_both_small(a, b)) {
+        return (signed_val(a) - signed_val(b));
+    } else if (is_float(a) && is_float(b)) {
+        FloatDef af, bf;
+
+        GET_DOUBLE(a, af);
+        GET_DOUBLE(b, bf);
+
+        return erts_float_comp(af.fd, bf.fd);
+    }
+
+    return erts_cmp_compound(a,b,exact,eq_only);
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
 #endif
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 08f8ca9788..d81bd89a48 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2615,27 +2615,6 @@ not_equal:
 }
 
 
-/* 
- * Lexically compare two strings of bytes (string s1 length l1 and s2 l2).
- *
- *	s1 < s2	return -1
- *	s1 = s2	return  0
- *	s1 > s2 return +1
- */
-static int cmpbytes(byte *s1, int l1, byte *s2, int l2)
-{
-    int i;
-    i = 0;
-    while((i < l1) && (i < l2)) {
-	if (s1[i] < s2[i]) return(-1);
-	if (s1[i] > s2[i]) return(1);
-	i++;
-    }
-    if (l1 < l2) return(-1);
-    if (l1 > l2) return(1);
-    return(0);
-}
-
 
 /*
  * Compare objects.
@@ -2649,20 +2628,6 @@ static int cmpbytes(byte *s1, int l1, byte *s2, int l2)
  *
  */
 
-
-#define float_comp(x,y)    (((x)<(y)) ? -1 : (((x)==(y)) ? 0 : 1))
-
-int erts_cmp_atoms(Eterm a, Eterm b)
-{
-    Atom *aa = atom_tab(atom_val(a));
-    Atom *bb = atom_tab(atom_val(b));
-    int diff = aa->ord0 - bb->ord0;
-    if (diff)
-	return diff;
-    return cmpbytes(aa->name+3, aa->len-3,
-		    bb->name+3, bb->len-3);
-}
-
 /* cmp(Eterm a, Eterm b)
  *  For compatibility with HiPE - arith-based compare.
  */
@@ -2673,22 +2638,6 @@ Sint cmp(Eterm a, Eterm b)
 
 Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only);
 
-Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
-{
-    if (is_atom(a) && is_atom(b)) {
-        return erts_cmp_atoms(a, b);
-    } else if (is_both_small(a, b)) {
-        return (signed_val(a) - signed_val(b));
-    } else if (is_float(a) && is_float(b)) {
-        FloatDef af, bf;
-        GET_DOUBLE(a, af);
-        GET_DOUBLE(b, bf);
-        return float_comp(af.fd, bf.fd);
-    }
-    return erts_cmp_compound(a,b,exact,eq_only);
-}
-
-
 /* erts_cmp(Eterm a, Eterm b, int exact)
  * exact = 1 -> term-based compare
  * exact = 0 -> arith-based compare
@@ -2985,7 +2934,7 @@ tailrecur_ne:
 
 		    GET_DOUBLE(a, af);
 		    GET_DOUBLE(b, bf);
-		    ON_CMP_GOTO(float_comp(af.fd, bf.fd));
+		    ON_CMP_GOTO(erts_float_comp(af.fd, bf.fd));
 		}
 	    case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE):
 	    case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE):
@@ -3022,10 +2971,7 @@ tailrecur_ne:
 		    ErlFunThing* f2 = (ErlFunThing *) fun_val(b);
 		    Sint diff;
 
-		    diff = cmpbytes(atom_tab(atom_val(f1->fe->module))->name,
-				    atom_tab(atom_val(f1->fe->module))->len,
-				    atom_tab(atom_val(f2->fe->module))->name,
-				    atom_tab(atom_val(f2->fe->module))->len);
+                    diff = erts_cmp_atoms((f1->fe)->module, (f2->fe)->module);
 		    if (diff != 0) {
 			RETURN_NEQ(diff);
 		    }
@@ -3219,7 +3165,7 @@ tailrecur_ne:
 	    if (f2.fd < MAX_LOSSLESS_FLOAT && f2.fd > MIN_LOSSLESS_FLOAT) {
 		/* Float is within the no loss limit */
 		f1.fd = signed_val(aw);
-		j = float_comp(f1.fd, f2.fd);
+		j = erts_float_comp(f1.fd, f2.fd);
 	    }
 #if ERTS_SIZEOF_ETERM == 8
 	    else if (f2.fd > (double) (MAX_SMALL + 1)) {
@@ -3266,7 +3212,7 @@ tailrecur_ne:
 		if (big_to_double(aw, &f1.fd) < 0) {
 		    j = big_sign(aw) ? -1 : 1;
 		} else {
-		    j = float_comp(f1.fd, f2.fd);
+		    j = erts_float_comp(f1.fd, f2.fd);
 		}
 	    } else {
 		big = double_to_big(f2.fd, big_buf, sizeof(big_buf)/sizeof(Eterm));
@@ -3282,7 +3228,7 @@ tailrecur_ne:
 	    if (f1.fd < MAX_LOSSLESS_FLOAT && f1.fd > MIN_LOSSLESS_FLOAT) {
 		/* Float is within the no loss limit */
 		f2.fd = signed_val(bw);
-		j = float_comp(f1.fd, f2.fd);
+		j = erts_float_comp(f1.fd, f2.fd);
 	    }
 #if ERTS_SIZEOF_ETERM == 8
 	    else if (f1.fd > (double) (MAX_SMALL + 1)) {
-- 
cgit v1.2.3


From eb9ee88f4cc640065f4902e270d834bfb596d5fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Mon, 15 Oct 2018 18:17:12 +0200
Subject: Optimize operator '--' and yield on large inputs

The removal set now uses a red-black tree instead of an array on
large inputs, decreasing runtime complexity from `n*n` to
`n*log(n)`. It will also exit early when there are no more items
left in the removal set, drastically improving performance and
memory use when the items to be removed are present near the head
of the list.

This got a lot more complicated than before as the overhead of
always using a red-black tree was unacceptable when either of the
inputs were small, but this compromise has okay-to-decent
performance regardless of input size.

Co-authored-by: Dmytro Lytovchenko <dmytro.lytovchenko@erlang-solutions.com>
---
 erts/emulator/beam/erl_alloc.types |   1 +
 erts/emulator/beam/erl_bif_lists.c | 796 +++++++++++++++++++++++++++++++++----
 2 files changed, 710 insertions(+), 87 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 5409b89bab..98ba0e5f07 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -274,6 +274,7 @@ type	ML_YIELD_STATE	SHORT_LIVED	SYSTEM		monitor_link_yield_state
 type	ML_DIST		STANDARD	SYSTEM		monitor_link_dist
 type	PF3_ARGS	SHORT_LIVED	PROCESSES	process_flag_3_arguments
 type	SETUP_CONN_ARG	SHORT_LIVED	PROCESSES	setup_connection_argument
+type    LIST_TRAP       SHORT_LIVED     PROCESSES       list_bif_trap_state
 
 type	ENVIRONMENT	SYSTEM		SYSTEM		environment
 
diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index 9b2b8aaab6..a793b34852 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -29,12 +29,13 @@
 #include "sys.h"
 #include "erl_vm.h"
 #include "global.h"
-#include "erl_process.h"
-#include "error.h"
 #include "bif.h"
+#include "erl_binary.h"
+
 
 static Eterm keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List);
 
+
 static BIF_RETTYPE append(Process* p, Eterm A, Eterm B)
 {
     Eterm list;
@@ -146,103 +147,724 @@ BIF_RETTYPE append_2(BIF_ALIST_2)
     return append(BIF_P, BIF_ARG_1, BIF_ARG_2);
 }
 
-/*
- * erlang:'--'/2
- */
+/* erlang:'--'/2
+ *
+ * Subtracts a list from another (LHS -- RHS), removing the first occurrence of
+ * each element in LHS from RHS. There is no type coercion so the elements must
+ * match exactly.
+ *
+ * The BIF is broken into several stages that can all trap individually, and it
+ * chooses its algorithm based on input size. If either input is small it will
+ * use a linear scan tuned to which side it's on, and if both inputs are large
+ * enough it will convert RHS into a multiset to provide good asymptotic
+ * behavior. */
+
+#define SUBTRACT_LHS_THRESHOLD 16
+#define SUBTRACT_RHS_THRESHOLD 16
+
+typedef enum {
+    SUBTRACT_STAGE_START,
+    SUBTRACT_STAGE_LEN_LHS,
+
+    /* Naive linear scan that's efficient when
+     * LEN_LHS <= SUBTRACT_LHS_THRESHOLD. */
+    SUBTRACT_STAGE_NAIVE_LHS,
+
+    SUBTRACT_STAGE_LEN_RHS,
+
+    /* As SUBTRACT_STAGE_NAIVE_LHS but for RHS. */
+    SUBTRACT_STAGE_NAIVE_RHS,
+
+    /* Creates a multiset from RHS for faster lookups before sweeping through
+     * LHS. The set is implemented as a red-black tree and duplicate elements
+     * are handled by a counter on each node. */
+    SUBTRACT_STAGE_SET_BUILD,
+    SUBTRACT_STAGE_SET_FINISH
+} ErtsSubtractCtxStage;
+
+typedef struct subtract_node__ {
+    struct subtract_node__ *parent;
+    struct subtract_node__ *left;
+    struct subtract_node__ *right;
+    int is_red;
+
+    Eterm key;
+    Uint count;
+} subtract_tree_t;
+
+typedef struct {
+    ErtsSubtractCtxStage stage;
+
+    Eterm lhs_original;
+    Eterm rhs_original;
+
+    Uint lhs_remaining;
+    Uint rhs_remaining;
+
+    Eterm iterator;
+
+    Eterm *result_cdr;
+    Eterm result;
+
+    union {
+        Eterm lhs_elements[SUBTRACT_LHS_THRESHOLD];
+        Eterm rhs_elements[SUBTRACT_RHS_THRESHOLD];
+
+        struct {
+            subtract_tree_t *tree;
+
+            /* A memory area for the tree's nodes, saving us the need to have
+             * one allocation per node. */
+            subtract_tree_t *alloc_start;
+            subtract_tree_t *alloc;
+        } rhs_set;
+    } u;
+} ErtsSubtractContext;
+
+#define ERTS_RBT_PREFIX subtract
+#define ERTS_RBT_T subtract_tree_t
+#define ERTS_RBT_KEY_T Eterm
+#define ERTS_RBT_FLAGS_T int
+#define ERTS_RBT_INIT_EMPTY_TNODE(T) \
+    do { \
+        (T)->parent = NULL; \
+        (T)->left = NULL; \
+        (T)->right = NULL; \
+    } while(0)
+#define ERTS_RBT_IS_RED(T) ((T)->is_red)
+#define ERTS_RBT_SET_RED(T) ((T)->is_red = 1)
+#define ERTS_RBT_IS_BLACK(T) (!ERTS_RBT_IS_RED(T))
+#define ERTS_RBT_SET_BLACK(T) ((T)->is_red = 0)
+#define ERTS_RBT_GET_FLAGS(T) ((T)->is_red)
+#define ERTS_RBT_SET_FLAGS(T, F) ((T)->is_red = F)
+#define ERTS_RBT_GET_PARENT(T) ((T)->parent)
+#define ERTS_RBT_SET_PARENT(T, P) ((T)->parent = P)
+#define ERTS_RBT_GET_RIGHT(T) ((T)->right)
+#define ERTS_RBT_SET_RIGHT(T, R) ((T)->right = (R))
+#define ERTS_RBT_GET_LEFT(T) ((T)->left)
+#define ERTS_RBT_SET_LEFT(T, L) ((T)->left = (L))
+#define ERTS_RBT_GET_KEY(T) ((T)->key)
+#define ERTS_RBT_CMP_KEYS(KX, KY) CMP_TERM(KX, KY)
+#define ERTS_RBT_WANT_LOOKUP_INSERT
+#define ERTS_RBT_WANT_LOOKUP
+#define ERTS_RBT_WANT_DELETE
+#define ERTS_RBT_UNDEF
+
+#include "erl_rbtree.h"
+
+static int subtract_continue(Process *p, ErtsSubtractContext *context);
+
+static void subtract_ctx_dtor(ErtsSubtractContext *context) {
+    switch (context->stage) {
+        case SUBTRACT_STAGE_SET_BUILD:
+        case SUBTRACT_STAGE_SET_FINISH:
+            erts_free(ERTS_ALC_T_LIST_TRAP, context->u.rhs_set.alloc_start);
+            break;
+        default:
+            break;
+    }
+}
 
-#define SMALL_VEC_SIZE 10
-static Eterm subtract(Process* p, Eterm A, Eterm B)
-{
-    Eterm  list;
-    Eterm* hp;
-    Uint  need;
-    Eterm  res;
-    Eterm small_vec[SMALL_VEC_SIZE];	/* Preallocated memory for small lists */
-    Eterm* vec_p;
-    Eterm* vp;
-    Sint i;
-    Sint n;
-    Sint m;
-    
-    if ((n = erts_list_length(A)) < 0) {
-	BIF_ERROR(p, BADARG);
+static int subtract_ctx_bin_dtor(Binary *context_bin) {
+    ErtsSubtractContext *context = ERTS_MAGIC_BIN_DATA(context_bin);
+    subtract_ctx_dtor(context);
+    return 1;
+}
+
+static void subtract_ctx_move(ErtsSubtractContext *from,
+                              ErtsSubtractContext *to) {
+    int uses_result_cdr = 0;
+
+    to->stage = from->stage;
+
+    to->lhs_original = from->lhs_original;
+    to->rhs_original = from->rhs_original;
+
+    to->lhs_remaining = from->lhs_remaining;
+    to->rhs_remaining = from->rhs_remaining;
+
+    to->iterator = from->iterator;
+    to->result = from->result;
+
+    switch (to->stage) {
+        case SUBTRACT_STAGE_NAIVE_LHS:
+            sys_memcpy(to->u.lhs_elements,
+                       from->u.lhs_elements,
+                       sizeof(Eterm) * to->lhs_remaining);
+            break;
+        case SUBTRACT_STAGE_NAIVE_RHS:
+            sys_memcpy(to->u.rhs_elements,
+                       from->u.rhs_elements,
+                       sizeof(Eterm) * to->rhs_remaining);
+
+            uses_result_cdr = 1;
+            break;
+        case SUBTRACT_STAGE_SET_FINISH:
+            uses_result_cdr = 1;
+            /* FALL THROUGH */
+        case SUBTRACT_STAGE_SET_BUILD:
+            to->u.rhs_set.alloc_start = from->u.rhs_set.alloc_start;
+            to->u.rhs_set.alloc = from->u.rhs_set.alloc;
+            to->u.rhs_set.tree = from->u.rhs_set.tree;
+            break;
+        default:
+            break;
     }
-    if ((m = erts_list_length(B)) < 0) {
-	BIF_ERROR(p, BADARG);
+
+    if (uses_result_cdr) {
+        if (from->result_cdr == &from->result) {
+            to->result_cdr = &to->result;
+        } else {
+            to->result_cdr = from->result_cdr;
+        }
     }
-    
-    if (n == 0)
-	BIF_RET(NIL);
-    if (m == 0)
-	BIF_RET(A);
-    
-    /* allocate element vector */
-    if (n <= SMALL_VEC_SIZE)
-	vec_p = small_vec;
-    else
-	vec_p = (Eterm*) erts_alloc(ERTS_ALC_T_TMP, n * sizeof(Eterm));
-    
-    /* PUT ALL ELEMENTS IN VP */
-    vp = vec_p;
-    list = A;
-    i = n;
-    while(i--) {
-	Eterm* listp = list_val(list);
-	*vp++ = CAR(listp);
-	list = CDR(listp);
+}
+
+static Eterm subtract_create_trap_state(Process *p,
+                                        ErtsSubtractContext *context) {
+    Binary *state_bin;
+    Eterm *hp;
+
+    state_bin = erts_create_magic_binary(sizeof(ErtsSubtractContext),
+                                         subtract_ctx_bin_dtor);
+
+    subtract_ctx_move(context, ERTS_MAGIC_BIN_DATA(state_bin));
+
+    hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
+
+    return erts_mk_magic_ref(&hp, &MSO(p), state_bin);
+}
+
+static int subtract_enter_len_lhs(Process *p, ErtsSubtractContext *context) {
+    context->stage = SUBTRACT_STAGE_LEN_LHS;
+
+    context->iterator = context->lhs_original;
+    context->lhs_remaining = 0;
+
+    return subtract_continue(p, context);
+}
+
+static int subtract_enter_len_rhs(Process *p, ErtsSubtractContext *context) {
+    context->stage = SUBTRACT_STAGE_LEN_RHS;
+
+    context->iterator = context->rhs_original;
+    context->rhs_remaining = 0;
+
+    return subtract_continue(p, context);
+}
+
+static int subtract_get_length(Process *p, Eterm *iterator_p, Uint *count_p) {
+    static const Sint ELEMENTS_PER_RED = 32;
+
+    Sint budget, count;
+    Eterm iterator;
+
+    budget = ELEMENTS_PER_RED * ERTS_BIF_REDS_LEFT(p);
+    iterator = *iterator_p;
+
+#ifdef DEBUG
+    budget = budget / 10 + 1;
+#endif
+
+    for (count = 0; count < budget && is_list(iterator); count++) {
+        iterator = CDR(list_val(iterator));
     }
-    
-    /* UNMARK ALL DELETED CELLS */
-    list = B;
-    m = 0;  /* number of deleted elements */
-    while(is_list(list)) {
-	Eterm* listp = list_val(list);
-	Eterm  elem = CAR(listp);
-	i = n;
-	vp = vec_p;
-	while(i--) {
-	    if (is_value(*vp) && eq(*vp, elem)) {
-		*vp = THE_NON_VALUE;
-		m++;
-		break;
-	    }
-	    vp++;
-	}
-	list = CDR(listp);
+
+    if (!is_list(iterator) && !is_nil(iterator)) {
+        return -1;
     }
-    
-    if (m == n)      /* All deleted ? */
-	res = NIL;
-    else if (m == 0)  /* None deleted ? */
-	res = A;
-    else {			/* REBUILD LIST */
-	res = NIL;
-	need = 2*(n - m);
-	hp = HAlloc(p, need);
-	vp = vec_p + n - 1;
-	while(vp >= vec_p) {
-	    if (is_value(*vp)) {
-		res = CONS(hp, *vp, res);
-		hp += 2;
-	    }
-	    vp--;
-	}
+
+    BUMP_REDS(p, count / ELEMENTS_PER_RED);
+
+    *iterator_p = iterator;
+    *count_p += count;
+
+    if (is_nil(iterator)) {
+        return 1;
     }
-    if (vec_p != small_vec)
-	erts_free(ERTS_ALC_T_TMP, (void *) vec_p);
-    BIF_RET(res);
+
+    return 0;
 }
 
-BIF_RETTYPE ebif_minusminus_2(BIF_ALIST_2)
-{
-    return subtract(BIF_P, BIF_ARG_1, BIF_ARG_2);
+static int subtract_enter_naive_lhs(Process *p, ErtsSubtractContext *context) {
+    Eterm iterator;
+    int i = 0;
+
+    context->stage = SUBTRACT_STAGE_NAIVE_LHS;
+
+    context->iterator = context->rhs_original;
+    context->result = NIL;
+
+    iterator = context->lhs_original;
+
+    while (is_list(iterator)) {
+        const Eterm *cell = list_val(iterator);
+
+        ASSERT(i < SUBTRACT_LHS_THRESHOLD);
+
+        context->u.lhs_elements[i++] = CAR(cell);
+        iterator = CDR(cell);
+    }
+
+    ASSERT(i == context->lhs_remaining);
+
+    return subtract_continue(p, context);
 }
 
-BIF_RETTYPE subtract_2(BIF_ALIST_2)
-{
-    return subtract(BIF_P, BIF_ARG_1, BIF_ARG_2);
+static int subtract_naive_lhs(Process *p, ErtsSubtractContext *context) {
+    const Sint CHECKS_PER_RED = 16;
+    Sint checks, budget;
+
+    budget = CHECKS_PER_RED * ERTS_BIF_REDS_LEFT(p);
+    checks = 0;
+
+    while (checks < budget && is_list(context->iterator)) {
+        const Eterm *cell;
+        Eterm value, next;
+        int found_at;
+
+        cell = list_val(context->iterator);
+
+        value = CAR(cell);
+        next = CDR(cell);
+
+        for (found_at = 0; found_at < context->lhs_remaining; found_at++) {
+            if (EQ(value, context->u.lhs_elements[found_at])) {
+                /* We shift the array one step down as we have to preserve
+                 * order.
+                 *
+                 * Note that we can't exit early as that would suppress errors
+                 * in the right-hand side (this runs prior to determining the
+                 * length of RHS). */
+
+                context->lhs_remaining--;
+                sys_memmove(&context->u.lhs_elements[found_at],
+                            &context->u.lhs_elements[found_at + 1],
+                            (context->lhs_remaining - found_at) * sizeof(Eterm));
+                break;
+            }
+        }
+
+        checks += MAX(1, context->lhs_remaining);
+        context->iterator = next;
+    }
+
+    BUMP_REDS(p, MIN(checks, budget) / CHECKS_PER_RED);
+
+    if (is_list(context->iterator)) {
+        return 0;
+    } else if (!is_nil(context->iterator)) {
+        return -1;
+    }
+
+    if (context->lhs_remaining > 0) {
+        Eterm *hp;
+        int i;
+
+        hp = HAlloc(p, context->lhs_remaining * 2);
+
+        for (i = context->lhs_remaining - 1; i >= 0; i--) {
+            Eterm value = context->u.lhs_elements[i];
+
+            context->result = CONS(hp, value, context->result);
+            hp += 2;
+        }
+    }
+
+    ASSERT(context->lhs_remaining > 0 || context->result == NIL);
+
+    return 1;
+}
+
+static int subtract_enter_naive_rhs(Process *p, ErtsSubtractContext *context) {
+    Eterm iterator;
+    int i = 0;
+
+    context->stage = SUBTRACT_STAGE_NAIVE_RHS;
+
+    context->iterator = context->lhs_original;
+    context->result_cdr = &context->result;
+    context->result = NIL;
+
+    iterator = context->rhs_original;
+
+    while (is_list(iterator)) {
+        const Eterm *cell = list_val(iterator);
+
+        ASSERT(i < SUBTRACT_RHS_THRESHOLD);
+
+        context->u.rhs_elements[i++] = CAR(cell);
+        iterator = CDR(cell);
+    }
+
+    ASSERT(i == context->rhs_remaining);
+
+    return subtract_continue(p, context);
+}
+
+static int subtract_naive_rhs(Process *p, ErtsSubtractContext *context) {
+    const Sint CHECKS_PER_RED = 16;
+    Sint checks, budget;
+
+    budget = CHECKS_PER_RED * ERTS_BIF_REDS_LEFT(p);
+    checks = 0;
+
+#ifdef DEBUG
+    budget = budget / 10 + 1;
+#endif
+
+    while (checks < budget && is_list(context->iterator)) {
+        const Eterm *cell;
+        Eterm value, next;
+        int found_at;
+
+        cell = list_val(context->iterator);
+        value = CAR(cell);
+        next = CDR(cell);
+
+        for (found_at = context->rhs_remaining - 1; found_at >= 0; found_at--) {
+            if (EQ(value, context->u.rhs_elements[found_at])) {
+                break;
+            }
+        }
+
+        if (found_at < 0) {
+            /* Destructively add the value to the result. This is safe
+             * since the GC is disabled and the unfinished term is never
+             * leaked to the outside world. */
+            Eterm *hp = HAllocX(p, 2, context->lhs_remaining * 2);
+
+            *context->result_cdr = make_list(hp);
+            context->result_cdr = &CDR(hp);
+
+            CAR(hp) = value;
+        } else if (found_at >= 0) {
+            Eterm swap;
+
+            if (context->rhs_remaining-- == 1) {
+                /* We've run out of items to remove, so the rest of the
+                 * result will be equal to the remainder of the input. We know
+                 * that LHS is well-formed as any errors would've been reported
+                 * during length determination. */
+                *context->result_cdr = next;
+
+                BUMP_REDS(p, MIN(budget, checks) / CHECKS_PER_RED);
+
+                return 1;
+            }
+
+            swap = context->u.rhs_elements[context->rhs_remaining];
+            context->u.rhs_elements[found_at] = swap;
+        }
+
+        checks += context->rhs_remaining;
+        context->iterator = next;
+        context->lhs_remaining--;
+    }
+
+    /* The result only has to be terminated when returning it to the user, but
+     * we're doing it when trapping as well to prevent headaches when
+     * debugging. */
+    *context->result_cdr = NIL;
+
+    BUMP_REDS(p, MIN(budget, checks) / CHECKS_PER_RED);
+
+    if (is_list(context->iterator)) {
+        ASSERT(context->lhs_remaining > 0 && context->rhs_remaining > 0);
+        return 0;
+    }
+
+    return 1;
+}
+
+static int subtract_enter_set_build(Process *p, ErtsSubtractContext *context) {
+    context->stage = SUBTRACT_STAGE_SET_BUILD;
+
+    context->u.rhs_set.alloc_start =
+        erts_alloc(ERTS_ALC_T_LIST_TRAP,
+                   context->rhs_remaining * sizeof(subtract_tree_t));
+
+    context->u.rhs_set.alloc = context->u.rhs_set.alloc_start;
+    context->u.rhs_set.tree = NULL;
+
+    context->iterator = context->rhs_original;
+
+    return subtract_continue(p, context);
 }
 
+static int subtract_set_build(Process *p, ErtsSubtractContext *context) {
+    const static Sint INSERTIONS_PER_RED = 16;
+    Sint budget, insertions;
+
+    budget = INSERTIONS_PER_RED * ERTS_BIF_REDS_LEFT(p);
+    insertions = 0;
+
+#ifdef DEBUG
+    budget = budget / 10 + 1;
+#endif
+
+    while (insertions < budget && is_list(context->iterator)) {
+        subtract_tree_t *existing_node, *new_node;
+        const Eterm *cell;
+        Eterm value, next;
+
+        cell = list_val(context->iterator);
+        value = CAR(cell);
+        next = CDR(cell);
+
+        new_node = context->u.rhs_set.alloc;
+        new_node->key = value;
+        new_node->count = 1;
+
+        existing_node = subtract_rbt_lookup_insert(&context->u.rhs_set.tree,
+                                                   new_node);
+
+        if (existing_node != NULL) {
+            existing_node->count++;
+        } else {
+            context->u.rhs_set.alloc++;
+        }
+
+        context->iterator = next;
+        insertions++;
+    }
+
+    BUMP_REDS(p, insertions / INSERTIONS_PER_RED);
+
+    ASSERT(is_list(context->iterator) || is_nil(context->iterator));
+    ASSERT(context->u.rhs_set.tree != NULL);
+
+    return is_nil(context->iterator);
+}
+
+static int subtract_enter_set_finish(Process *p, ErtsSubtractContext *context) {
+    context->stage = SUBTRACT_STAGE_SET_FINISH;
+
+    context->result_cdr = &context->result;
+    context->result = NIL;
+
+    context->iterator = context->lhs_original;
+
+    return subtract_continue(p, context);
+}
+
+static int subtract_set_finish(Process *p, ErtsSubtractContext *context) {
+    const Sint CHECKS_PER_RED = 8;
+    Sint checks, budget;
+
+    budget = CHECKS_PER_RED * ERTS_BIF_REDS_LEFT(p);
+    checks = 0;
+
+#ifdef DEBUG
+    budget = budget / 10 + 1;
+#endif
+
+    while (checks < budget && is_list(context->iterator)) {
+        subtract_tree_t *node;
+        const Eterm *cell;
+        Eterm value, next;
+
+        cell = list_val(context->iterator);
+        value = CAR(cell);
+        next = CDR(cell);
+
+        ASSERT(context->rhs_remaining > 0);
+
+        node = subtract_rbt_lookup(context->u.rhs_set.tree, value);
+
+        if (node == NULL) {
+            Eterm *hp = HAllocX(p, 2, context->lhs_remaining * 2);
+
+            *context->result_cdr = make_list(hp);
+            context->result_cdr = &CDR(hp);
+
+            CAR(hp) = value;
+        } else {
+            if (context->rhs_remaining-- == 1) {
+                *context->result_cdr = next;
+
+                BUMP_REDS(p, checks / CHECKS_PER_RED);
+
+                return 1;
+            }
+
+            if (node->count-- == 1) {
+                subtract_rbt_delete(&context->u.rhs_set.tree, node);
+            }
+        }
+
+        context->iterator = next;
+        context->lhs_remaining--;
+        checks++;
+    }
+
+    *context->result_cdr = NIL;
+
+    BUMP_REDS(p, checks / CHECKS_PER_RED);
+
+    if (is_list(context->iterator)) {
+        ASSERT(context->lhs_remaining > 0 && context->rhs_remaining > 0);
+        return 0;
+    }
+
+    return 1;
+}
+
+static int subtract_continue(Process *p, ErtsSubtractContext *context) {
+    switch (context->stage) {
+        case SUBTRACT_STAGE_START: {
+            return subtract_enter_len_lhs(p, context);
+        }
+
+        case SUBTRACT_STAGE_LEN_LHS: {
+            int res = subtract_get_length(p,
+                                          &context->iterator,
+                                          &context->lhs_remaining);
+
+            if (res != 1) {
+                return res;
+            }
+
+            if (context->lhs_remaining <= SUBTRACT_LHS_THRESHOLD) {
+                return subtract_enter_naive_lhs(p, context);
+            }
+
+            return subtract_enter_len_rhs(p, context);
+        }
+
+        case SUBTRACT_STAGE_NAIVE_LHS: {
+            return subtract_naive_lhs(p, context);
+        }
+
+        case SUBTRACT_STAGE_LEN_RHS: {
+            int res = subtract_get_length(p,
+                                          &context->iterator,
+                                          &context->rhs_remaining);
+
+            if (res != 1) {
+                return res;
+            }
+
+            /* We've walked through both lists fully now so we no longer need
+             * to check for errors past this point. */
+
+            if (context->rhs_remaining <= SUBTRACT_RHS_THRESHOLD) {
+                return subtract_enter_naive_rhs(p, context);
+            }
+
+            return subtract_enter_set_build(p, context);
+        }
+
+        case SUBTRACT_STAGE_NAIVE_RHS: {
+            return subtract_naive_rhs(p, context);
+        }
+
+        case SUBTRACT_STAGE_SET_BUILD: {
+            int res = subtract_set_build(p, context);
+
+            if (res != 1) {
+                return res;
+            }
+
+            return subtract_enter_set_finish(p, context);
+        }
+
+        case SUBTRACT_STAGE_SET_FINISH: {
+            return subtract_set_finish(p, context);
+        }
+
+        default:
+            ERTS_ASSERT(!"unreachable");
+    }
+}
+
+static int subtract_start(Process *p, Eterm lhs, Eterm rhs,
+                          ErtsSubtractContext *context) {
+    context->stage = SUBTRACT_STAGE_START;
+
+    context->lhs_original = lhs;
+    context->rhs_original = rhs;
+
+    return subtract_continue(p, context);
+}
+
+/* erlang:'--'/2 */
+static Eterm subtract(Export *bif_entry, BIF_ALIST_2) {
+    Eterm lhs = BIF_ARG_1, rhs = BIF_ARG_2;
+
+    if ((is_list(lhs) || is_nil(lhs)) && (is_list(rhs) || is_nil(rhs))) {
+        /* We start with the context on the stack in the hopes that we won't
+         * have to trap. */
+        ErtsSubtractContext context;
+        int res;
+
+        res = subtract_start(BIF_P, lhs, rhs, &context);
+
+        if (res == 0) {
+            Eterm state_mref;
+
+            state_mref = subtract_create_trap_state(BIF_P, &context);
+            erts_set_gc_state(BIF_P, 0);
+
+            BIF_TRAP2(bif_entry, BIF_P, state_mref, NIL);
+        }
+
+        subtract_ctx_dtor(&context);
+
+        if (res < 0) {
+            BIF_ERROR(BIF_P, BADARG);
+        }
+
+        BIF_RET(context.result);
+    } else if (is_internal_magic_ref(lhs)) {
+        ErtsSubtractContext *context;
+        int (*dtor)(Binary*);
+        Binary *magic_bin;
+
+        int res;
+
+        magic_bin = erts_magic_ref2bin(lhs);
+        dtor = ERTS_MAGIC_BIN_DESTRUCTOR(magic_bin);
+
+        if (dtor != subtract_ctx_bin_dtor) {
+            BIF_ERROR(BIF_P, BADARG);
+        }
+
+        ASSERT(BIF_P->flags & F_DISABLE_GC);
+        ASSERT(rhs == NIL);
+
+        context = ERTS_MAGIC_BIN_DATA(magic_bin);
+        res = subtract_continue(BIF_P, context);
+
+        if (res == 0) {
+            BIF_TRAP2(bif_entry, BIF_P, lhs, NIL);
+        }
+
+        erts_set_gc_state(BIF_P, 1);
+
+        if (res < 0) {
+            ERTS_BIF_ERROR_TRAPPED2(BIF_P, BADARG, bif_entry,
+                                    context->lhs_original,
+                                    context->rhs_original);
+        }
+
+        BIF_RET(context->result);
+    }
+
+    ASSERT(!(BIF_P->flags & F_DISABLE_GC));
+
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+BIF_RETTYPE ebif_minusminus_2(BIF_ALIST_2) {
+    return subtract(bif_export[BIF_ebif_minusminus_2], BIF_CALL_ARGS);
+}
+
+BIF_RETTYPE subtract_2(BIF_ALIST_2) {
+    return subtract(bif_export[BIF_subtract_2], BIF_CALL_ARGS);
+}
+
+
 BIF_RETTYPE lists_member_2(BIF_ALIST_2)
 {
     Eterm term;
-- 
cgit v1.2.3


From 7d92a5c7be185e549bdd8ad56524d2bd3f9479a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Mon, 8 Oct 2018 07:47:35 +0200
Subject: Refactor releasing of literals

Introudce erts_queue_release_literals() to queue a literal area to be
released.
---
 erts/emulator/beam/beam_bif_load.c | 59 +++++++++++++++++++++++---------------
 erts/emulator/beam/global.h        |  2 ++
 2 files changed, 38 insertions(+), 23 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index d221e6aea6..bb1b2e5b27 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1752,29 +1752,7 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 	finalize_purge_operation(BIF_P, ret == am_true);
 
 	if (literals) {
-	    ErtsLiteralAreaRef *ref;
-            ErtsMessage *mp;
-	    ref = erts_alloc(ERTS_ALC_T_LITERAL_REF,
-			     sizeof(ErtsLiteralAreaRef));
-	    ref->literal_area = literals;
-	    ref->next = NULL;
-	    erts_mtx_lock(&release_literal_areas.mtx);
-	    if (release_literal_areas.last) {
-		release_literal_areas.last->next = ref;
-		release_literal_areas.last = ref;
-	    }
-	    else {
-		release_literal_areas.first = ref;
-		release_literal_areas.last = ref;
-	    }
-	    erts_mtx_unlock(&release_literal_areas.mtx);
-            mp = erts_alloc_message(0, NULL);
-            ERL_MESSAGE_TOKEN(mp) = am_undefined;
-	    erts_queue_proc_message(BIF_P,
-                               erts_literal_area_collector,
-			       0,
-			       mp,
-			       am_copy_literals);
+            erts_queue_release_literals(BIF_P, literals);
 	}
 
 	return ret;
@@ -1786,6 +1764,41 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
     }
 }
 
+void
+erts_queue_release_literals(Process* c_p, ErtsLiteralArea* literals)
+{
+    ErtsLiteralAreaRef *ref;
+    ErtsMessage *mp;
+    ref = erts_alloc(ERTS_ALC_T_LITERAL_REF,
+                     sizeof(ErtsLiteralAreaRef));
+    ref->literal_area = literals;
+    ref->next = NULL;
+    erts_mtx_lock(&release_literal_areas.mtx);
+    if (release_literal_areas.last) {
+        release_literal_areas.last->next = ref;
+        release_literal_areas.last = ref;
+    } else {
+        release_literal_areas.first = ref;
+        release_literal_areas.last = ref;
+    }
+    erts_mtx_unlock(&release_literal_areas.mtx);
+    mp = erts_alloc_message(0, NULL);
+    ERL_MESSAGE_TOKEN(mp) = am_undefined;
+    if (c_p == NULL) {
+        erts_queue_message(erts_literal_area_collector,
+                           0,
+                           mp,
+                           am_copy_literals,
+                           am_system);
+    } else {
+        erts_queue_proc_message(c_p,
+                                erts_literal_area_collector,
+                                0,
+                                mp,
+                                am_copy_literals);
+    }
+}
+
 /*
  * Move code from current to old and null all export entries for the module
  */
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 9fc5abbf68..54d309ff2f 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -906,6 +906,8 @@ typedef struct ErtsLiteralArea_ {
     Eterm start[1]; /* beginning of area */
 } ErtsLiteralArea;
 
+void erts_queue_release_literals(Process *c_p, ErtsLiteralArea* literals);
+
 #define ERTS_LITERAL_AREA_ALLOC_SIZE(N) \
     (sizeof(ErtsLiteralArea) + sizeof(Eterm)*((N) - 1))
 
-- 
cgit v1.2.3


From 9a699277bdc8a422ddf5d3ace3cfe59d32f0ce10 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 31 Oct 2018 16:22:49 +0100
Subject: Fix erts_internal_ref_number_cmp()

---
 erts/emulator/beam/erl_bif_unique.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h
index 9aa631fde9..e3a633080d 100644
--- a/erts/emulator/beam/erl_bif_unique.h
+++ b/erts/emulator/beam/erl_bif_unique.h
@@ -242,11 +242,11 @@ erts_internal_ref_number_cmp(Uint32 num1[ERTS_REF_NUMBERS],
 			     Uint32 num2[ERTS_REF_NUMBERS])
 {
     if (num1[2] != num2[2])
-	return (int) ((Sint64) num1[2] - (Sint64) num2[2]);
+	return num1[2] > num2[2] ? 1 : -1;
     if (num1[1] != num2[1])
-	return (int) ((Sint64) num1[1] - (Sint64) num2[1]);
+	return num1[1] > num2[1] ? 1 : -1;
     if (num1[0] != num2[0])
-	return (int) ((Sint64) num1[0] - (Sint64) num2[0]);
+	return num1[0] > num2[0] ? 1 : -1;
     return 0;
 }
 
-- 
cgit v1.2.3


From d07042b3a354080becbd13a8e1c5fe1f9a3870cc Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 2 Nov 2018 09:58:23 +0100
Subject: erts: Fix msacc GC states on dirty scheds

A dirty scheduler is an un-managed thread so we need to
lock the msacc state on those.
---
 erts/emulator/beam/erl_gc.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 47dd115c82..b4df418cd5 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -681,7 +681,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     ErtsMonotonicTime start_time;
     ErtsSchedulerData *esdp = erts_proc_sched_data(p);
     erts_aint32_t state;
-    ERTS_MSACC_PUSH_STATE_M();
+    ERTS_MSACC_PUSH_STATE();
 #ifdef USE_VM_PROBES
     DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
 #endif
@@ -711,7 +711,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     else if (p->live_hf_end != ERTS_INVALID_HFRAG_PTR)
 	live_hf_end = p->live_hf_end;
 
-    ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_GC);
+    ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_GC);
 
     erts_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
     if (erts_system_monitor_long_gc != 0)
@@ -759,7 +759,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
         gc_trace_end_tag = am_gc_minor_end;
     } else {
 do_major_collection:
-        ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC_FULL);
+        ERTS_MSACC_SET_STATE_CACHED_X(ERTS_MSACC_STATE_GC_FULL);
         if (IS_TRACED_FL(p, F_TRACE_GC)) {
             trace_gc(p, am_gc_major_start, need, THE_NON_VALUE);
         }
@@ -770,7 +770,7 @@ do_major_collection:
             p->flags &= ~(F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC);
         DTRACE2(gc_major_end, pidbuf, reclaimed_now);
         gc_trace_end_tag = am_gc_major_end;
-        ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC);
+        ERTS_MSACC_SET_STATE_CACHED_X(ERTS_MSACC_STATE_GC);
     }
 
     reset_active_writer(p);
@@ -800,7 +800,7 @@ do_major_collection:
 
         /* We have to make sure that we have space for need on the heap */
         res = delay_garbage_collection(p, live_hf_end, need, fcalls);
-        ERTS_MSACC_POP_STATE_M();
+        ERTS_MSACC_POP_STATE();
         return res;
     }
 
@@ -843,7 +843,7 @@ do_major_collection:
     FLAGS(p) &= ~(F_FORCE_GC|F_HIBERNATED);
     p->live_hf_end = ERTS_INVALID_HFRAG_PTR;
 
-    ERTS_MSACC_POP_STATE_M();
+    ERTS_MSACC_POP_STATE();
 
 #ifdef CHECK_FOR_HOLES
     /*
-- 
cgit v1.2.3


From 805748eb668d5562fe17f3172cdae07a86166c3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Thu, 4 Oct 2018 10:30:05 +0200
Subject: Add a persistent term storage

Persistent terms are useful for storing Erlang terms that are never
or infrequently updated. They have the following advantages:

* Constant time access. A persistent term is not copied when it is
  looked up. The constant factor is lower than for ETS, and no locks
  are taken when looking up a term.

* Persistent terms are not copied in garbage collections.

* There is only ever one copy of a persistent term (until it is
  deleted). That makes them useful for storing configuration data
  that needs to be easily accessible by all processes.

Persistent terms have the following drawbacks:

* Updates are expensive. The hash table holding the keys for the
  persistent terms are updated whenever a persistent term is added,
  updated or deleted.

* Updating or deleting a persistent term triggers a "global GC", which
  will schedule a heap scan of all processes to search the heap of all
  processes for the deleted term. If a process still holds a reference
  to the deleted term, the process will be garbage collected and the
  term copied to the heap of the process. This global GC can make the
  system less responsive for some time.

Three BIFs (implemented in C in the emulator) is the entire
interface to the persistent term functionality:

* put(Key, Value) to store a persistent term.

* get(Key) to look up a persistent term.

* erase(Key) to delete a persistent term.

There are also two additional BIFs to obtain information about
persistent terms:

* info() to return a map with information about persistent terms.

* get() to return a list of a {Key,Value} tuples for all persistent
  terms. (The values are not copied.)
---
 erts/emulator/Makefile.in                    |  32 +-
 erts/emulator/beam/atom.names                |   3 +
 erts/emulator/beam/bif.tab                   |  12 +
 erts/emulator/beam/erl_alloc.types           |   3 +
 erts/emulator/beam/erl_bif_persistent.c      | 983 +++++++++++++++++++++++++++
 erts/emulator/beam/erl_init.c                |   1 +
 erts/emulator/beam/erl_lock_check.c          |   2 +
 erts/emulator/beam/erl_process_dump.c        |  40 +-
 erts/emulator/beam/global.h                  |   7 +
 erts/emulator/test/Makefile                  |   1 +
 erts/emulator/test/persistent_term_SUITE.erl | 614 +++++++++++++++++
 11 files changed, 1681 insertions(+), 17 deletions(-)
 create mode 100644 erts/emulator/beam/erl_bif_persistent.c
 create mode 100644 erts/emulator/test/persistent_term_SUITE.erl

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 054692819e..d5ceb4b00b 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -633,21 +633,22 @@ GENERATE += $(TTF_DIR)/driver_tab.c
 # This list must be consistent with PRE_LOADED_MODULES in
 # erts/preloaded/src/Makefile.
 
-PRELOAD_BEAM =		$(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/init.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/prim_buffer.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/prim_eval.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/zlib.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/prim_zip.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/erl_prim_loader.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/erlang.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/erts_internal.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam \
-			$(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam
+PRELOAD_BEAM =	$(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/init.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/prim_buffer.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/prim_eval.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/zlib.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/prim_zip.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/erl_prim_loader.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/erlang.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/erts_internal.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/persistent_term.beam
 
 ifeq ($(TARGET),win32)
 # On windows the preloaded objects are in a resource object.
@@ -839,6 +840,7 @@ RUN_OBJS += \
 	$(OBJDIR)/erl_bif_ddll.o  	$(OBJDIR)/erl_bif_guard.o \
 	$(OBJDIR)/erl_bif_info.o	$(OBJDIR)/erl_bif_op.o \
 	$(OBJDIR)/erl_bif_os.o		$(OBJDIR)/erl_bif_lists.o \
+	$(OBJDIR)/erl_bif_persistent.o \
 	$(OBJDIR)/erl_bif_trace.o	$(OBJDIR)/erl_bif_unique.o \
 	$(OBJDIR)/erl_bif_wrap.o	$(OBJDIR)/erl_nfunc_sched.o \
 	$(OBJDIR)/erl_guard_bifs.o 	$(OBJDIR)/erl_dirty_bif_wrap.o \
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 45b7540aeb..82bb620d62 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -182,6 +182,7 @@ atom control
 atom copy
 atom copy_literals
 atom counters
+atom count
 atom cpu
 atom cpu_timestamp
 atom cr
@@ -287,6 +288,7 @@ atom gc_minor_end
 atom gc_minor_start
 atom Ge='>='
 atom generational
+atom get_all_trap
 atom get_seq_token
 atom get_tcw
 atom gather_gc_info_result
@@ -325,6 +327,7 @@ atom index
 atom infinity
 atom info
 atom info_msg
+atom info_trap
 atom init
 atom initial_call
 atom input
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 7548924178..f1f34ae323 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -40,6 +40,7 @@
 # Note: Guards BIFs usually require special support in the compiler.
 #
 
+
 gcbif erlang:abs/1
 bif erlang:adler32/1
 bif erlang:adler32/2
@@ -698,3 +699,14 @@ ubif erlang:map_get/2
 ubif erlang:is_map_key/2
 bif ets:internal_delete_all/2
 bif ets:internal_select_delete/2
+
+#
+# New in 21.2
+#
+
+bif persistent_term:put/2
+bif persistent_term:get/1
+bif persistent_term:get/0
+bif persistent_term:erase/1
+bif persistent_term:info/0
+bif erts_internal:erase_persistent_terms/0
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 5409b89bab..86d93d94c4 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -277,6 +277,9 @@ type	SETUP_CONN_ARG	SHORT_LIVED	PROCESSES	setup_connection_argument
 
 type	ENVIRONMENT	SYSTEM		SYSTEM		environment
 
+type	PERSISTENT_TERM	LONG_LIVED	CODE		persisten_term
+type	PERSISTENT_LOCK_Q SHORT_LIVED	SYSTEM		persistent_lock_q
+
 #
 # Types used for special emulators
 #
diff --git a/erts/emulator/beam/erl_bif_persistent.c b/erts/emulator/beam/erl_bif_persistent.c
new file mode 100644
index 0000000000..9dca768a18
--- /dev/null
+++ b/erts/emulator/beam/erl_bif_persistent.c
@@ -0,0 +1,983 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2018. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Purpose: Implement persistent term storage.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "sys.h"
+#include "erl_vm.h"
+#include "global.h"
+#include "erl_process.h"
+#include "error.h"
+#include "erl_driver.h"
+#include "bif.h"
+#include "erl_map.h"
+#include "erl_binary.h"
+
+/*
+ * The limit for the number of persistent terms before
+ * a warning is issued.
+ */
+
+#define WARNING_LIMIT 20000
+#define XSTR(s) STR(s)
+#define STR(s) #s
+
+/*
+ * Parameters for the hash table.
+ */
+#define INITIAL_SIZE 8
+#define LOAD_FACTOR ((Uint)50)
+#define MUST_GROW(t) (((Uint)100) * t->num_entries >= LOAD_FACTOR * t->allocated)
+#define MUST_SHRINK(t) (((Uint)200) * t->num_entries <= LOAD_FACTOR * t->allocated && \
+                        t->allocated > INITIAL_SIZE)
+
+typedef struct hash_table {
+    Uint allocated;
+    Uint num_entries;
+    Uint mask;
+    Uint first_to_delete;
+    Uint num_to_delete;
+    erts_atomic_t refc;
+    struct hash_table* delete_next;
+    ErtsThrPrgrLaterOp thr_prog_op;
+    Eterm term[1];
+} HashTable;
+
+typedef struct trap_data {
+    HashTable* table;
+    Uint idx;
+    Uint remaining;
+    Uint memory;    /* Used by info/0 to count used memory */
+} TrapData;
+
+/*
+ * Declarations of local functions.
+ */
+
+static HashTable* create_initial_table(void);
+static Uint lookup(HashTable* hash_table, Eterm key);
+static HashTable* copy_table(HashTable* old_table, Uint new_size, int rehash);
+static HashTable* tmp_table_copy(HashTable* old_table);
+static int try_seize_update_permission(Process* c_p);
+static void release_update_permission(int release_updater);
+static void table_updater(void* table);
+static void table_deleter(void* hash_table);
+static void dec_table_refc(Process* c_p, HashTable* old_table);
+static void delete_table(Process* c_p, HashTable* table);
+static void mark_for_deletion(HashTable* hash_table, Uint entry_index);
+static ErtsLiteralArea* term_to_area(Eterm tuple);
+static void suspend_updater(Process* c_p);
+static Eterm do_get_all(Process* c_p, TrapData* trap_data, Eterm res);
+static Eterm do_info(Process* c_p, TrapData* trap_data);
+static void append_to_delete_queue(HashTable* table);
+static HashTable* next_to_delete(void);
+static Eterm alloc_trap_data(Process* c_p);
+static int cleanup_trap_data(Binary *bp);
+
+/*
+ * Traps
+ */
+
+static Export persistent_term_get_all_export;
+static BIF_RETTYPE persistent_term_get_all_trap(BIF_ALIST_2);
+static Export persistent_term_info_export;
+static BIF_RETTYPE persistent_term_info_trap(BIF_ALIST_1);
+
+/*
+ * Pointer to the current hash table.
+ */
+
+static erts_atomic_t the_hash_table;
+
+/*
+ * Queue of processes waiting to update the hash table.
+ */
+
+struct update_queue_item {
+    Process *p;
+    struct update_queue_item* next;
+};
+
+static erts_mtx_t update_table_permission_mtx;
+static struct update_queue_item* update_queue = NULL;
+static Process* updater_process = NULL;
+
+/* Protected by update_table_permission_mtx */
+static ErtsThrPrgrLaterOp thr_prog_op;
+static int issued_warning = 0;
+
+/*
+ * Queue of hash tables to be deleted.
+ */
+
+static erts_mtx_t delete_queue_mtx;
+static HashTable* delete_queue_head = NULL;
+static HashTable** delete_queue_tail = &delete_queue_head;
+
+/*
+ * The following variables are only used during crash dumping. They
+ * are intialized by erts_init_persistent_dumping().
+ */
+
+ErtsLiteralArea** erts_persistent_areas;
+Uint erts_num_persistent_areas;
+
+void erts_init_bif_persistent_term(void)
+{
+    HashTable* hash_table;
+
+    /*
+     * Initialize the mutex protecting updates.
+     */
+
+    erts_mtx_init(&update_table_permission_mtx,
+                  "update_persistent_term_permission",
+                  NIL,
+                  ERTS_LOCK_FLAGS_PROPERTY_STATIC |
+                  ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
+
+    /*
+     * Initialize delete queue.
+     */
+
+    erts_mtx_init(&delete_queue_mtx,
+                  "persistent_term_delete_permission",
+                  NIL,
+                  ERTS_LOCK_FLAGS_PROPERTY_STATIC |
+                  ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
+
+    /*
+     * Allocate a small initial hash table.
+     */
+
+    hash_table = create_initial_table();
+    erts_atomic_init_nob(&the_hash_table, (erts_aint_t)hash_table);
+
+    /*
+     * Initialize export entry for traps
+     */
+
+    erts_init_trap_export(&persistent_term_get_all_export,
+			  am_persistent_term, am_get_all_trap, 2,
+			  &persistent_term_get_all_trap);
+    erts_init_trap_export(&persistent_term_info_export,
+			  am_persistent_term, am_info_trap, 1,
+			  &persistent_term_info_trap);
+}
+
+BIF_RETTYPE persistent_term_put_2(BIF_ALIST_2)
+{
+    Eterm key;
+    Eterm term;
+    Eterm heap[3];
+    Eterm tuple;
+    HashTable* hash_table;
+    Uint term_size;
+    Uint lit_area_size;
+    ErlOffHeap code_off_heap;
+    ErtsLiteralArea* literal_area;
+    erts_shcopy_t info;
+    Eterm* ptr;
+    Uint entry_index;
+
+    if (!try_seize_update_permission(BIF_P)) {
+	ERTS_BIF_YIELD2(bif_export[BIF_persistent_term_put_2],
+                        BIF_P, BIF_ARG_1, BIF_ARG_2);
+    }
+
+    hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
+
+    key = BIF_ARG_1;
+    term = BIF_ARG_2;
+
+    entry_index = lookup(hash_table, key);
+
+    heap[0] = make_arityval(2);
+    heap[1] = key;
+    heap[2] = term;
+    tuple = make_tuple(heap);
+
+    if (is_nil(hash_table->term[entry_index])) {
+        Uint size = hash_table->allocated;
+        if (MUST_GROW(hash_table)) {
+            size *= 2;
+        }
+        hash_table = copy_table(hash_table, size, 0);
+        entry_index = lookup(hash_table, key);
+        hash_table->num_entries++;
+    } else {
+        Eterm tuple = hash_table->term[entry_index];
+        Eterm old_term;
+
+        ASSERT(is_tuple_arity(tuple, 2));
+        old_term = boxed_val(tuple)[2];
+        if (EQ(term, old_term)) {
+            /* Same value. No need to update anything. */
+            release_update_permission(0);
+            BIF_RET(am_ok);
+        } else {
+            /* Mark the old term for deletion. */
+            mark_for_deletion(hash_table, entry_index);
+            hash_table = copy_table(hash_table, hash_table->allocated, 0);
+        }
+    }
+
+    /*
+     * Preserve internal sharing in the term by using the
+     * sharing-preserving functions. However, literals must
+     * be copied in case the module holding them are unloaded.
+     */
+    INITIALIZE_SHCOPY(info);
+    info.copy_literals = 1;
+    term_size = copy_shared_calculate(tuple, &info);
+    ERTS_INIT_OFF_HEAP(&code_off_heap);
+    lit_area_size = ERTS_LITERAL_AREA_ALLOC_SIZE(term_size);
+    literal_area = erts_alloc(ERTS_ALC_T_LITERAL, lit_area_size);
+    ptr = &literal_area->start[0];
+    literal_area->end = ptr + term_size;
+    tuple = copy_shared_perform(tuple, term_size, &info, &ptr, &code_off_heap);
+    ASSERT(tuple_val(tuple) == literal_area->start);
+    literal_area->off_heap = code_off_heap.first;
+    DESTROY_SHCOPY(info);
+    erts_set_literal_tag(&tuple, literal_area->start, term_size);
+    hash_table->term[entry_index] = tuple;
+
+    erts_schedule_thr_prgr_later_op(table_updater, hash_table, &thr_prog_op);
+    suspend_updater(BIF_P);
+
+    /*
+     * Issue a warning once if the warning limit has been exceeded.
+     */
+
+    if (hash_table->num_entries > WARNING_LIMIT && issued_warning == 0) {
+        static char w[] =
+            "More than " XSTR(WARNING_LIMIT) " persistent terms "
+            "have been created.\n"
+            "It is recommended to avoid creating an excessive number of\n"
+            "persistent terms, as creation and deletion of persistent terms\n"
+            "will be slower as the number of persistent terms increases.\n";
+        issued_warning = 1;
+	erts_send_warning_to_logger_str(BIF_P->group_leader, w);
+    }
+
+    ERTS_BIF_YIELD_RETURN(BIF_P, am_ok);
+}
+
+BIF_RETTYPE persistent_term_get_0(BIF_ALIST_0)
+{
+    HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
+    TrapData* trap_data;
+    Eterm res = NIL;
+    Eterm magic_ref;
+    Binary* mbp;
+
+    magic_ref = alloc_trap_data(BIF_P);
+    mbp = erts_magic_ref2bin(magic_ref);
+    trap_data = ERTS_MAGIC_BIN_DATA(mbp);
+    trap_data->table = hash_table;
+    trap_data->idx = 0;
+    trap_data->remaining = hash_table->num_entries;
+    res = do_get_all(BIF_P, trap_data, res);
+    if (trap_data->remaining == 0) {
+        BUMP_REDS(BIF_P, hash_table->num_entries);
+        trap_data->table = NULL; /* Prevent refc decrement */
+        BIF_RET(res);
+    } else {
+        /*
+         * Increment the ref counter to prevent an update operation (by put/2
+         * or erase/1) to delete this hash table.
+         */
+        erts_atomic_inc_nob(&hash_table->refc);
+        BUMP_ALL_REDS(BIF_P);
+        BIF_TRAP2(&persistent_term_get_all_export, BIF_P, magic_ref, res);
+    }
+}
+
+BIF_RETTYPE persistent_term_get_1(BIF_ALIST_1)
+{
+    Eterm key = BIF_ARG_1;
+    HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
+    Uint entry_index;
+    Eterm term;
+
+    entry_index = lookup(hash_table, key);
+    term = hash_table->term[entry_index];
+    if (is_boxed(term)) {
+        ASSERT(is_tuple_arity(term, 2));
+        BIF_RET(tuple_val(term)[2]);
+    }
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+BIF_RETTYPE persistent_term_erase_1(BIF_ALIST_1)
+{
+    Eterm key = BIF_ARG_1;
+    HashTable* old_table;
+    HashTable* new_table;
+    Uint entry_index;
+    Eterm old_term;
+
+    if (!try_seize_update_permission(BIF_P)) {
+	ERTS_BIF_YIELD1(bif_export[BIF_persistent_term_erase_1],
+                        BIF_P, BIF_ARG_1);
+    }
+
+    old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
+    entry_index = lookup(old_table, key);
+    old_term = old_table->term[entry_index];
+    if (is_boxed(old_term)) {
+        Uint new_size;
+        HashTable* tmp_table;
+
+        /*
+         * Since we don't use any delete markers, we must rehash
+         * the table when deleting terms to ensure that all terms
+         * can still be reached if there are hash collisions.
+         * We can't rehash in place and it would not be safe to modify
+         * the old table yet, so we will first need a new
+         * temporary table copy of the same size as the old one.
+         */
+
+        ASSERT(is_tuple_arity(old_term, 2));
+        tmp_table = tmp_table_copy(old_table);
+
+        /*
+         * Delete the term from the temporary table. Then copy the
+         * temporary table to a new table, rehashing the entries
+         * while copying.
+         */
+
+        tmp_table->term[entry_index] = NIL;
+        tmp_table->num_entries--;
+        new_size = tmp_table->allocated;
+        if (MUST_SHRINK(tmp_table)) {
+            new_size /= 2;
+        }
+        new_table = copy_table(tmp_table, new_size, 1);
+        erts_free(ERTS_ALC_T_TMP, tmp_table);
+
+        mark_for_deletion(old_table, entry_index);
+        erts_schedule_thr_prgr_later_op(table_updater, new_table, &thr_prog_op);
+        suspend_updater(BIF_P);
+        ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
+    }
+
+    /*
+     * Key is not present. Nothing to do.
+     */
+
+    ASSERT(is_nil(old_term));
+    release_update_permission(0);
+    BIF_RET(am_false);
+}
+
+BIF_RETTYPE erts_internal_erase_persistent_terms_0(BIF_ALIST_0)
+{
+    HashTable* old_table;
+    HashTable* new_table;
+
+    if (!try_seize_update_permission(BIF_P)) {
+	ERTS_BIF_YIELD0(bif_export[BIF_erts_internal_erase_persistent_terms_0],
+                        BIF_P);
+    }
+    old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
+    old_table->first_to_delete = 0;
+    old_table->num_to_delete = old_table->allocated;
+    new_table = create_initial_table();
+    erts_schedule_thr_prgr_later_op(table_updater, new_table, &thr_prog_op);
+    suspend_updater(BIF_P);
+    ERTS_BIF_YIELD_RETURN(BIF_P, am_true);
+}
+
+BIF_RETTYPE persistent_term_info_0(BIF_ALIST_0)
+{
+    HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
+    TrapData* trap_data;
+    Eterm res = NIL;
+    Eterm magic_ref;
+    Binary* mbp;
+
+    magic_ref = alloc_trap_data(BIF_P);
+    mbp = erts_magic_ref2bin(magic_ref);
+    trap_data = ERTS_MAGIC_BIN_DATA(mbp);
+    trap_data->table = hash_table;
+    trap_data->idx = 0;
+    trap_data->remaining = hash_table->num_entries;
+    trap_data->memory = 0;
+    res = do_info(BIF_P, trap_data);
+    if (trap_data->remaining == 0) {
+        BUMP_REDS(BIF_P, hash_table->num_entries);
+        trap_data->table = NULL; /* Prevent refc decrement */
+        BIF_RET(res);
+    } else {
+        /*
+         * Increment the ref counter to prevent an update operation (by put/2
+         * or erase/1) to delete this hash table.
+         */
+        erts_atomic_inc_nob(&hash_table->refc);
+        BUMP_ALL_REDS(BIF_P);
+        BIF_TRAP2(&persistent_term_info_export, BIF_P, magic_ref, res);
+    }
+}
+
+Uint
+erts_persistent_term_count(void)
+{
+    HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
+    return hash_table->num_entries;
+}
+
+void
+erts_init_persistent_dumping(void)
+{
+    HashTable* hash_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
+    ErtsLiteralArea** area_p;
+    Uint i;
+
+    /*
+     * Overwrite the array of Eterms in the current hash table
+     * with pointers to literal areas.
+     */
+
+    erts_persistent_areas = (ErtsLiteralArea **) hash_table->term;
+    erts_num_persistent_areas = hash_table->num_entries;
+    area_p = erts_persistent_areas;
+    for (i = 0; i < hash_table->allocated; i++) {
+        Eterm term = hash_table->term[i];
+
+        if (is_boxed(term)) {
+            *area_p++ = term_to_area(term);
+        }
+    }
+}
+
+/*
+ * Local functions.
+ */
+
+static HashTable*
+create_initial_table(void)
+{
+    HashTable* hash_table;
+    int i;
+
+    hash_table = (HashTable *) erts_alloc(ERTS_ALC_T_PERSISTENT_TERM,
+                                          sizeof(HashTable)+sizeof(Eterm) *
+                                          (INITIAL_SIZE-1));
+    hash_table->allocated = INITIAL_SIZE;
+    hash_table->num_entries = 0;
+    hash_table->mask = INITIAL_SIZE-1;
+    hash_table->first_to_delete = 0;
+    hash_table->num_to_delete = 0;
+    erts_atomic_init_nob(&hash_table->refc, (erts_aint_t)1);
+    for (i = 0; i < INITIAL_SIZE; i++) {
+        hash_table->term[i] = NIL;
+    }
+    return hash_table;
+}
+
+static BIF_RETTYPE
+persistent_term_get_all_trap(BIF_ALIST_2)
+{
+    TrapData* trap_data;
+    Eterm res = BIF_ARG_2;
+    Uint bump_reds;
+    Binary* mbp;
+
+    ASSERT(is_list(BIF_ARG_2));
+    mbp = erts_magic_ref2bin(BIF_ARG_1);
+    trap_data = ERTS_MAGIC_BIN_DATA(mbp);
+    bump_reds = trap_data->remaining;
+    res = do_get_all(BIF_P, trap_data, res);
+    ASSERT(is_list(res));
+    if (trap_data->remaining > 0) {
+        BUMP_ALL_REDS(BIF_P);
+        BIF_TRAP2(&persistent_term_get_all_export, BIF_P, BIF_ARG_1, res);
+    } else {
+        /*
+         * Decrement ref count (and possibly delete the hash table
+         * and associated literal area).
+         */
+        dec_table_refc(BIF_P, trap_data->table);
+        trap_data->table = NULL; /* Prevent refc decrement */
+        BUMP_REDS(BIF_P, bump_reds);
+        BIF_RET(res);
+    }
+}
+
+static Eterm
+do_get_all(Process* c_p, TrapData* trap_data, Eterm res)
+{
+    HashTable* hash_table;
+    Uint remaining;
+    Uint idx;
+    Uint max_iter;
+    Uint i;
+    Eterm* hp;
+    Uint heap_size;
+    struct copy_term {
+        Uint key_size;
+        Eterm* tuple_ptr;
+    } *copy_data;
+
+    hash_table = trap_data->table;
+    idx = trap_data->idx;
+#if defined(DEBUG) || defined(VALGRIND)
+    max_iter = 50;
+#else
+    max_iter = ERTS_BIF_REDS_LEFT(c_p);
+#endif
+    remaining = trap_data->remaining < max_iter ?
+        trap_data->remaining : max_iter;
+    trap_data->remaining -= remaining;
+
+    copy_data = (struct copy_term *) erts_alloc(ERTS_ALC_T_TMP,
+                                                remaining *
+                                                sizeof(struct copy_term));
+    i = 0;
+    heap_size = (2 + 3) * remaining;
+    while (remaining != 0) {
+        Eterm term = hash_table->term[idx];
+        if (is_tuple(term)) {
+            Uint key_size;
+            Eterm* tup_val;
+
+            ASSERT(is_tuple_arity(term, 2));
+            tup_val = tuple_val(term);
+            key_size = size_object(tup_val[1]);
+            copy_data[i].key_size = key_size;
+            copy_data[i].tuple_ptr = tup_val;
+            heap_size += key_size;
+            i++;
+            remaining--;
+        }
+        idx++;
+    }
+    trap_data->idx = idx;
+
+    hp = HAlloc(c_p, heap_size);
+    remaining = i;
+    for (i = 0; i < remaining; i++) {
+        Eterm* tuple_ptr;
+        Uint key_size;
+        Eterm key;
+        Eterm tup;
+
+        tuple_ptr = copy_data[i].tuple_ptr;
+        key_size = copy_data[i].key_size;
+        key = copy_struct(tuple_ptr[1], key_size, &hp, &c_p->off_heap);
+        tup = TUPLE2(hp, key, tuple_ptr[2]);
+        hp += 3;
+        res = CONS(hp, tup, res);
+        hp += 2;
+    }
+    erts_free(ERTS_ALC_T_TMP, copy_data);
+    return res;
+}
+
+static BIF_RETTYPE
+persistent_term_info_trap(BIF_ALIST_1)
+{
+    TrapData* trap_data = (TrapData *) BIF_ARG_1;
+    Eterm res;
+    Uint bump_reds;
+    Binary* mbp;
+
+    mbp = erts_magic_ref2bin(BIF_ARG_1);
+    trap_data = ERTS_MAGIC_BIN_DATA(mbp);
+    bump_reds = trap_data->remaining;
+    res = do_info(BIF_P, trap_data);
+    if (trap_data->remaining > 0) {
+        ASSERT(res == am_ok);
+        BUMP_ALL_REDS(BIF_P);
+        BIF_TRAP1(&persistent_term_info_export, BIF_P, BIF_ARG_1);
+    } else {
+        /*
+         * Decrement ref count (and possibly delete the hash table
+         * and associated literal area).
+         */
+        dec_table_refc(BIF_P, trap_data->table);
+        trap_data->table = NULL; /* Prevent refc decrement */
+        BUMP_REDS(BIF_P, bump_reds);
+        ASSERT(is_map(res));
+        BIF_RET(res);
+    }
+}
+
+#define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
+
+static Eterm
+do_info(Process* c_p, TrapData* trap_data)
+{
+    HashTable* hash_table;
+    Uint remaining;
+    Uint idx;
+    Uint max_iter;
+
+    hash_table = trap_data->table;
+    idx = trap_data->idx;
+#if defined(DEBUG) || defined(VALGRIND)
+    max_iter = 50;
+#else
+    max_iter = ERTS_BIF_REDS_LEFT(c_p);
+#endif
+    remaining = trap_data->remaining < max_iter ? trap_data->remaining : max_iter;
+    trap_data->remaining -= remaining;
+    while (remaining != 0) {
+        if (is_boxed(hash_table->term[idx])) {
+            ErtsLiteralArea* area;
+            area = term_to_area(hash_table->term[idx]);
+            trap_data->memory += sizeof(ErtsLiteralArea) +
+                sizeof(Eterm) * (area->end - area->start - 1);
+            remaining--;
+        }
+        idx++;
+    }
+    trap_data->idx = idx;
+    if (trap_data->remaining > 0) {
+        return am_ok;           /* Dummy return value */
+    } else {
+        Eterm* hp;
+        Eterm count_term;
+        Eterm memory_term;
+        Eterm res;
+        Uint memory;
+        Uint hsz = MAP_SZ(2);
+
+        memory = sizeof(HashTable) + (trap_data->table->allocated-1) *
+            sizeof(Eterm) + trap_data->memory;
+        (void) erts_bld_uint(NULL, &hsz, hash_table->num_entries);
+        (void) erts_bld_uint(NULL, &hsz, memory);
+        hp = HAlloc(c_p, hsz);
+	count_term = erts_bld_uint(&hp, NULL, hash_table->num_entries);
+	memory_term = erts_bld_uint(&hp, NULL, memory);
+        res = MAP2(hp, am_count, count_term, am_memory, memory_term);
+        return res;
+    }
+}
+
+#undef DECL_AM
+
+static Eterm
+alloc_trap_data(Process* c_p)
+{
+    Binary* mbp = erts_create_magic_binary(sizeof(TrapData),
+                                           cleanup_trap_data);
+    Eterm* hp;
+
+    hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE);
+    return erts_mk_magic_ref(&hp, &MSO(c_p), mbp);
+}
+
+static int
+cleanup_trap_data(Binary *bp)
+{
+    TrapData* trap_data = ERTS_MAGIC_BIN_DATA(bp);
+
+    if (trap_data->table) {
+        /*
+         * The process has been killed and is now exiting.
+         * Decrement the reference counter for the table.
+         */
+        dec_table_refc(NULL, trap_data->table);
+    }
+    return 1;
+}
+
+static Uint
+lookup(HashTable* hash_table, Eterm key)
+{
+    Uint mask = hash_table->mask;
+    Eterm* table = hash_table->term;
+    Uint32 idx = make_internal_hash(key, 0);
+    Eterm term;
+
+    do {
+        idx++;
+        term = table[idx & mask];
+    } while (is_boxed(term) && !EQ(key, (tuple_val(term))[1]));
+    return idx & mask;
+}
+
+static HashTable*
+tmp_table_copy(HashTable* old_table)
+{
+    Uint size = old_table->allocated;
+    HashTable* tmp_table;
+    Uint i;
+
+    tmp_table = (HashTable *) erts_alloc(ERTS_ALC_T_TMP,
+                                         sizeof(HashTable) +
+                                         sizeof(Eterm) * (size-1));
+    *tmp_table = *old_table;
+    for (i = 0; i < size; i++) {
+        tmp_table->term[i] = old_table->term[i];
+    }
+    return tmp_table;
+}
+
+static HashTable*
+copy_table(HashTable* old_table, Uint new_size, int rehash)
+{
+    HashTable* new_table;
+    Uint old_size = old_table->allocated;
+    Uint i;
+
+    new_table = (HashTable *) erts_alloc(ERTS_ALC_T_PERSISTENT_TERM,
+                                         sizeof(HashTable) +
+                                         sizeof(Eterm) * (new_size-1));
+    if (old_table->allocated == new_size && !rehash) {
+        /*
+         * Same size and no key deleted. Make an exact copy of the table.
+         */
+        *new_table = *old_table;
+        for (i = 0; i < new_size; i++) {
+            new_table->term[i] = old_table->term[i];
+        }
+    } else {
+        /*
+         * The size of the table has changed or an element has been
+         * deleted. Must rehash, by inserting all old terms into the
+         * new (empty) table.
+         */
+        new_table->allocated = new_size;
+        new_table->num_entries = old_table->num_entries;
+        new_table->mask = new_size - 1;
+        for (i = 0; i < new_size; i++) {
+            new_table->term[i] = NIL;
+        }
+        for (i = 0; i < old_size; i++) {
+            if (is_tuple(old_table->term[i])) {
+                Eterm key = tuple_val(old_table->term[i])[1];
+                Uint entry_index = lookup(new_table, key);
+                ASSERT(is_nil(new_table->term[entry_index]));
+                new_table->term[entry_index] = old_table->term[i];
+            }
+        }
+    }
+    new_table->first_to_delete = 0;
+    new_table->num_to_delete = 0;
+    erts_atomic_init_nob(&new_table->refc, (erts_aint_t)1);
+    return new_table;
+}
+
+static void
+mark_for_deletion(HashTable* hash_table, Uint entry_index)
+{
+    hash_table->first_to_delete = entry_index;
+    hash_table->num_to_delete = 1;
+}
+
+static ErtsLiteralArea*
+term_to_area(Eterm tuple)
+{
+    ASSERT(is_tuple_arity(tuple, 2));
+    return (ErtsLiteralArea *) (((char *) tuple_val(tuple)) -
+                                offsetof(ErtsLiteralArea, start));
+}
+
+static void
+table_updater(void* data)
+{
+    HashTable* old_table;
+    HashTable* new_table;
+
+    old_table = (HashTable *) erts_atomic_read_nob(&the_hash_table);
+    new_table = (HashTable *) data;
+    ASSERT(new_table->num_to_delete == 0);
+    erts_atomic_set_nob(&the_hash_table, (erts_aint_t)new_table);
+    append_to_delete_queue(old_table);
+    erts_schedule_thr_prgr_later_op(table_deleter,
+                                    old_table,
+                                    &old_table->thr_prog_op);
+    release_update_permission(1);
+}
+
+static void
+table_deleter(void* data)
+{
+    HashTable* old_table = (HashTable *) data;
+
+    dec_table_refc(NULL, old_table);
+}
+
+static void
+dec_table_refc(Process* c_p, HashTable* old_table)
+{
+    erts_aint_t refc = erts_atomic_dec_read_nob(&old_table->refc);
+
+    if (refc == 0) {
+        HashTable* to_delete;
+
+        while ((to_delete = next_to_delete()) != NULL) {
+            delete_table(c_p, to_delete);
+        }
+    }
+}
+
+static void
+delete_table(Process* c_p, HashTable* table)
+{
+    Uint idx = table->first_to_delete;
+    Uint n = table->num_to_delete;
+
+    /*
+     * There are no longer any references to this hash table.
+     *
+     * Any literals pointed for deletion can be queued for
+     * deletion and the table itself can be deallocated.
+     */
+
+#ifdef DEBUG
+    if (n == 1) {
+        ASSERT(is_tuple_arity(table->term[idx], 2));
+    }
+#endif
+
+    while (n > 0) {
+        Eterm term = table->term[idx];
+
+        if (is_tuple_arity(term, 2)) {
+            if (is_immed(tuple_val(term)[2])) {
+                erts_release_literal_area(term_to_area(term));
+            } else {
+                erts_queue_release_literals(c_p, term_to_area(term));
+            }
+        }
+        idx++, n--;
+    }
+    erts_free(ERTS_ALC_T_PERSISTENT_TERM, table);
+}
+
+/*
+ * Caller *must* yield if this function returns 0.
+ */
+
+static int
+try_seize_update_permission(Process* c_p)
+{
+    int success;
+
+    ASSERT(!erts_thr_progress_is_blocking()); /* to avoid deadlock */
+    ASSERT(c_p != NULL);
+
+    erts_mtx_lock(&update_table_permission_mtx);
+    ASSERT(updater_process != c_p);
+    success = (updater_process == NULL);
+    if (success) {
+        updater_process = c_p;
+    } else {
+        struct update_queue_item* qitem;
+        qitem = erts_alloc(ERTS_ALC_T_PERSISTENT_LOCK_Q, sizeof(*qitem));
+        qitem->p = c_p;
+        erts_proc_inc_refc(c_p);
+        qitem->next = update_queue;
+        update_queue = qitem;
+        erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
+    }
+    erts_mtx_unlock(&update_table_permission_mtx);
+    return success;
+}
+
+static void
+release_update_permission(int release_updater)
+{
+    erts_mtx_lock(&update_table_permission_mtx);
+    ASSERT(updater_process != NULL);
+
+    if (release_updater) {
+        erts_proc_lock(updater_process, ERTS_PROC_LOCK_STATUS);
+        if (!ERTS_PROC_IS_EXITING(updater_process)) {
+            erts_resume(updater_process, ERTS_PROC_LOCK_STATUS);
+        }
+        erts_proc_unlock(updater_process, ERTS_PROC_LOCK_STATUS);
+    }
+    updater_process = NULL;
+
+    while (update_queue != NULL) { /* Unleash the entire herd */
+	struct update_queue_item* qitem = update_queue;
+	erts_proc_lock(qitem->p, ERTS_PROC_LOCK_STATUS);
+	if (!ERTS_PROC_IS_EXITING(qitem->p)) {
+	    erts_resume(qitem->p, ERTS_PROC_LOCK_STATUS);
+	}
+	erts_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS);
+	update_queue = qitem->next;
+	erts_proc_dec_refc(qitem->p);
+	erts_free(ERTS_ALC_T_PERSISTENT_LOCK_Q, qitem);
+    }
+    erts_mtx_unlock(&update_table_permission_mtx);
+}
+
+static void
+suspend_updater(Process* c_p)
+{
+#ifdef DEBUG
+    ASSERT(c_p != NULL);
+    erts_mtx_lock(&update_table_permission_mtx);
+    ASSERT(updater_process == c_p);
+    erts_mtx_unlock(&update_table_permission_mtx);
+#endif
+    erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
+}
+
+static void
+append_to_delete_queue(HashTable* table)
+{
+    erts_mtx_lock(&delete_queue_mtx);
+    table->delete_next = NULL;
+    *delete_queue_tail = table;
+    delete_queue_tail = &table->delete_next;
+    erts_mtx_unlock(&delete_queue_mtx);
+}
+
+static HashTable*
+next_to_delete(void)
+{
+    HashTable* table;
+
+    erts_mtx_lock(&delete_queue_mtx);
+    table = delete_queue_head;
+    if (table) {
+        if (erts_atomic_read_nob(&table->refc)) {
+            /*
+             * This hash table is still referenced. Hash tables
+             * must be deleted in order, so we return a NULL
+             * pointer.
+             */
+            table = NULL;
+        } else {
+            /*
+             * Remove the first hash table from the queue.
+             */
+            delete_queue_head = table->delete_next;
+            if (delete_queue_head == NULL) {
+                delete_queue_tail = &delete_queue_head;
+            }
+        }
+    }
+    erts_mtx_unlock(&delete_queue_mtx);
+    return table;
+}
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 57c6c10c7f..f687dcf335 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -354,6 +354,7 @@ erl_init(int ncpu,
     erts_init_bif();
     erts_init_bif_chksum();
     erts_init_bif_binary();
+    erts_init_bif_persistent_term();
     erts_init_bif_re();
     erts_init_unicode(); /* after RE to get access to PCRE unicode */
     erts_init_external();
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 463ae898a3..1416c5f96c 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -97,6 +97,8 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"proc_btm",				"pid"			},
     {	"dist_entry",				"address"		},
     {	"dist_entry_links",			"address"		},
+    {   "update_persistent_term_permission",    NULL                    },
+    {   "persistent_term_delete_permission",    NULL                    },
     {   "code_write_permission",                NULL                    },
     {	"purge_state",		      		NULL			},
     {	"proc_status",				"pid"			},
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 243db4c734..ac5054ea10 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -58,6 +58,7 @@ static void dump_externally(fmtfn_t to, void *to_arg, Eterm term);
 static void mark_literal(Eterm* ptr);
 static void init_literal_areas(void);
 static void dump_literals(fmtfn_t to, void *to_arg);
+static void dump_persistent_terms(fmtfn_t to, void *to_arg);
 static void dump_module_literals(fmtfn_t to, void *to_arg,
                                  ErtsLiteralArea* lit_area);
 
@@ -74,6 +75,7 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg)
 
     all_binaries = NULL;
     init_literal_areas();
+    erts_init_persistent_dumping();
 
     for (i = 0; i < max; i++) {
 	Process *p = erts_pix2proc(i);
@@ -93,6 +95,7 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg)
        }
     }
 
+    dump_persistent_terms(to, to_arg);
     dump_literals(to, to_arg);
     dump_binaries(to, to_arg, all_binaries);
 }
@@ -775,6 +778,9 @@ init_literal_areas(void)
     qsort(lit_areas, num_lit_areas, sizeof(ErtsLiteralArea *),
           compare_areas);
 
+    qsort(erts_persistent_areas, erts_num_persistent_areas,
+          sizeof(ErtsLiteralArea *), compare_areas);
+
     erts_runlock_old_code(code_ix);
 }
 
@@ -796,6 +802,13 @@ static void mark_literal(Eterm* ptr)
 
     ap = bsearch(ptr, lit_areas, num_lit_areas, sizeof(ErtsLiteralArea*),
                  search_areas);
+    if (ap == 0) {
+        ap = bsearch(ptr, erts_persistent_areas,
+                     erts_num_persistent_areas,
+                     sizeof(ErtsLiteralArea*),
+                     search_areas);
+    }
+
 
     /*
      * If the literal was created by native code, this search will not
@@ -807,12 +820,12 @@ static void mark_literal(Eterm* ptr)
     }
 }
 
-
 static void
 dump_literals(fmtfn_t to, void *to_arg)
 {
     ErtsCodeIndex code_ix;
     int i;
+    Uint idx;
 
     code_ix = erts_active_code_ix();
     erts_rlock_old_code(code_ix);
@@ -825,6 +838,28 @@ dump_literals(fmtfn_t to, void *to_arg)
     }
 
     erts_runlock_old_code(code_ix);
+
+    for (idx = 0; idx < erts_num_persistent_areas; idx++) {
+        dump_module_literals(to, to_arg, erts_persistent_areas[idx]);
+    }
+}
+
+static void
+dump_persistent_terms(fmtfn_t to, void *to_arg)
+{
+    Uint idx;
+
+    erts_print(to, to_arg, "=persistent_terms\n");
+
+    for (idx = 0; idx < erts_num_persistent_areas; idx++) {
+        ErtsLiteralArea* ap = erts_persistent_areas[idx];
+        Eterm tuple = make_tuple(ap->start);
+        Eterm* tup_val = tuple_val(tuple);
+
+	dump_element(to, to_arg, tup_val[1]);
+        erts_putc(to, to_arg, '|');
+	dump_element_nl(to, to_arg, tup_val[2]);
+    }
 }
 
 static void
@@ -963,7 +998,8 @@ dump_module_literals(fmtfn_t to, void *to_arg, ErtsLiteralArea* lit_area)
                     }
                     erts_putc(to, to_arg, '\n');
                 }
-            } else if (is_export_header(w)) {
+            } else {
+                /* Dump everything else in the external format */
                 dump_externally(to, to_arg, term);
                 erts_putc(to, to_arg, '\n');
             }
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 54d309ff2f..0631404599 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1242,6 +1242,13 @@ Sint erts_re_set_loop_limit(Sint limit);
 void erts_init_bif_binary(void);
 Sint erts_binary_set_loop_limit(Sint limit);
 
+/* erl_bif_persistent.c */
+void erts_init_bif_persistent_term(void);
+Uint erts_persistent_term_count(void);
+void erts_init_persistent_dumping(void);
+extern ErtsLiteralArea** erts_persistent_areas;
+extern Uint erts_num_persistent_areas;
+
 /* external.c */
 void erts_init_external(void);
 
diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index bf00de2204..d201ea6ee1 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -92,6 +92,7 @@ MODULES= \
 	port_SUITE \
 	port_bif_SUITE \
 	prim_eval_SUITE \
+	persistent_term_SUITE \
 	process_SUITE \
 	pseudoknot_SUITE \
 	receive_SUITE \
diff --git a/erts/emulator/test/persistent_term_SUITE.erl b/erts/emulator/test/persistent_term_SUITE.erl
new file mode 100644
index 0000000000..58cd3276b0
--- /dev/null
+++ b/erts/emulator/test/persistent_term_SUITE.erl
@@ -0,0 +1,614 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%5
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(persistent_term_SUITE).
+-include_lib("common_test/include/ct.hrl").
+
+-export([all/0,suite/0,
+	 basic/1,purging/1,sharing/1,get_trapping/1,
+         info/1,info_trapping/1,killed_while_trapping/1,
+         off_heap_values/1,keys/1,collisions/1,
+         init_restart/1]).
+
+%%
+-export([test_init_restart_cmd/1]).
+
+suite() ->
+    [{ct_hooks,[ts_install_cth]},
+     {timetrap,{minutes,10}}].
+
+all() ->
+    [basic,purging,sharing,get_trapping,info,info_trapping,
+     killed_while_trapping,off_heap_values,keys,collisions,
+     init_restart].
+
+basic(_Config) ->
+    Chk = chk(),
+    N = 777,
+    Seq = lists:seq(1, N),
+    par(2, N, Seq),
+    seq(3, Seq),
+    seq(3, Seq),                                %Same values.
+    _ = [begin
+             Key = {?MODULE,{key,I}},
+             true = persistent_term:erase(Key),
+             false = persistent_term:erase(Key),
+             {'EXIT',{badarg,_}} = (catch persistent_term:get(Key))
+         end || I <- Seq],
+    [] = [P || {{?MODULE,_},_}=P <- persistent_term:get()],
+    chk(Chk).
+
+par(C, N, Seq) ->
+    _ = [spawn_link(fun() ->
+                            ok = persistent_term:put({?MODULE,{key,I}},
+                                                     {value,C*I})
+                    end) || I <- Seq],
+    Result = wait(N),
+    _ = [begin
+             Double = C*I,
+             {{?MODULE,{key,I}},{value,Double}} = Res
+         end || {I,Res} <- lists:zip(Seq, Result)],
+    ok.
+
+seq(C, Seq) ->
+    _ = [ok = persistent_term:put({?MODULE,{key,I}}, {value,C*I}) ||
+            I <- Seq],
+    All = persistent_term:get(),
+    All = [P || {{?MODULE,_},_}=P <- persistent_term:get()],
+    All = [{Key,persistent_term:get(Key)} || {Key,_} <- All],
+    Result = lists:sort(All),
+    _ = [begin
+             Double = C*I,
+             {{?MODULE,{key,I}},{value,Double}} = Res
+         end || {I,Res} <- lists:zip(Seq, Result)],
+    ok.
+
+wait(N) ->
+    All = [P || {{?MODULE,_},_}=P <- persistent_term:get()],
+    case length(All) of
+        N ->
+            All = [{Key,persistent_term:get(Key)} || {Key,_} <- All],
+            lists:sort(All);
+        _ ->
+            receive after 10 -> ok end,
+            wait(N)
+    end.
+
+%% Make sure that terms that have been erased are copied into all
+%% processes that still hold a pointer to them.
+
+purging(_Config) ->
+    Chk = chk(),
+    do_purging(fun(K) -> persistent_term:put(K, {?MODULE,new}) end,
+               replaced),
+    do_purging(fun persistent_term:erase/1, erased),
+    chk(Chk).
+
+do_purging(Eraser, Type) ->
+    Parent = self(),
+    Key = {?MODULE,?FUNCTION_NAME},
+    ok = persistent_term:put(Key, {term,[<<"abc",0:777/unit:8>>]}),
+    Ps0 = [spawn_monitor(fun() -> purging_tester(Parent, Key) end) ||
+              _ <- lists:seq(1, 50)],
+    Ps = maps:from_list(Ps0),
+    purging_recv(gotten, Ps),
+    Eraser(Key),
+    _ = [P ! {Parent,Type} || P <- maps:keys(Ps)],
+    purging_wait(Ps).
+
+purging_recv(Tag, Ps) when map_size(Ps) > 0 ->
+    receive
+        {Pid,Tag} ->
+            true = is_map_key(Pid, Ps),
+            purging_recv(Tag, maps:remove(Pid, Ps))
+    end;
+purging_recv(_, _) -> ok.
+
+purging_wait(Ps) when map_size(Ps) > 0 ->
+    receive
+        {'DOWN',Ref,process,Pid,Reason} ->
+            normal = Reason,
+            Ref = map_get(Pid, Ps),
+            purging_wait(maps:remove(Pid, Ps))
+    end;
+purging_wait(_) -> ok.
+
+purging_tester(Parent, Key) ->
+    Term = persistent_term:get(Key),
+    purging_check_term(Term),
+    0 = erts_debug:size_shared(Term),
+    Parent ! {self(),gotten},
+    receive
+        {Parent,erased} ->
+            {'EXIT',{badarg,_}} = (catch persistent_term:get(Key)),
+            purging_tester_1(Term);
+        {Parent,replaced} ->
+            {?MODULE,new} = persistent_term:get(Key),
+            purging_tester_1(Term)
+    end.
+
+%% Wait for the term to be copied into this process.
+purging_tester_1(Term) ->
+    purging_check_term(Term),
+    receive after 1 -> ok end,
+    case erts_debug:size_shared(Term) of
+        0 ->
+            purging_tester_1(Term);
+        Size ->
+            %% The term has been copied into this process.
+            purging_check_term(Term),
+            Size = erts_debug:size(Term)
+    end.
+
+purging_check_term({term,[<<"abc",0:777/unit:8>>]}) ->
+    ok.
+
+%% Test that sharing is preserved when storing terms.
+
+sharing(_Config) ->
+    Chk = chk(),
+    Depth = 10,
+    Size = 2*Depth,
+    Shared = lists:foldl(fun(_, A) -> [A|A] end,
+                         [], lists:seq(1, Depth)),
+    Size = erts_debug:size(Shared),
+    Key = {?MODULE,?FUNCTION_NAME},
+    ok = persistent_term:put(Key, Shared),
+    SharedStored = persistent_term:get(Key),
+    Size = erts_debug:size(SharedStored),
+    0 = erts_debug:size_shared(SharedStored),
+
+    {Pid,Ref} = spawn_monitor(fun() ->
+                                      Term = persistent_term:get(Key),
+                                      Size = erts_debug:size(Term),
+                                      0 = erts_debug:size_shared(Term),
+                                      true = Term =:= SharedStored
+                              end),
+    receive
+        {'DOWN',Ref,process,Pid,normal} ->
+            true = persistent_term:erase(Key),
+            Size = erts_debug:size(SharedStored),
+            chk(Chk)
+    end.
+
+%% Test trapping of persistent_term:get/0.
+
+get_trapping(_Config) ->
+    Chk = chk(),
+
+    %% Assume that the get/0 traps after 4000 iterations
+    %% in a non-debug emulator.
+    N = case test_server:timetrap_scale_factor() of
+            1 -> 10000;
+            _ -> 1000
+        end,
+    spawn_link(fun() -> get_trapping_create(N) end),
+    All = do_get_trapping(N, []),
+    N = get_trapping_check_result(lists:sort(All), 1),
+    erlang:garbage_collect(),
+    get_trapping_erase(N),
+    chk(Chk).
+
+do_get_trapping(N, Prev) ->
+    case persistent_term:get() of
+        Prev when length(Prev) >= N ->
+            All = [P || {{?MODULE,{get_trapping,_}},_}=P <- Prev],
+            case length(All) of
+                N -> All;
+                _ -> do_get_trapping(N, Prev)
+            end;
+        New ->
+            receive after 1 -> ok end,
+            do_get_trapping(N, New)
+    end.
+
+get_trapping_create(0) ->
+    ok;
+get_trapping_create(N) ->
+    ok = persistent_term:put({?MODULE,{get_trapping,N}}, N),
+    get_trapping_create(N-1).
+
+get_trapping_check_result([{{?MODULE,{get_trapping,N}},N}|T], N) ->
+    get_trapping_check_result(T, N+1);
+get_trapping_check_result([], N) -> N-1.
+
+get_trapping_erase(0) ->
+    ok;
+get_trapping_erase(N) ->
+    true = persistent_term:erase({?MODULE,{get_trapping,N}}),
+    get_trapping_erase(N-1).
+
+%% Test retrieving information about persistent terms.
+
+info(_Config) ->
+    Chk = chk(),
+
+    %% White box test of info/0.
+    N = 100,
+    try
+        Overhead = info_literal_area_overhead(),
+        io:format("Overhead = ~p\n", [Overhead]),
+        info_wb(N, Overhead, info_info())
+    after
+        _ = [_ = persistent_term:erase({?MODULE,I}) ||
+                I <- lists:seq(1, N)]
+    end,
+
+    chk(Chk).
+
+%% White box test of persistent_term:info/0. We take into account
+%% that there might already exist persistent terms (created by the
+%% OTP standard libraries), but we assume that they are not
+%% changed during the execution of this test case.
+
+info_wb(0, _, _) ->
+    ok;
+info_wb(N, Overhead, {BaseCount,BaseMemory}) ->
+    Key = {?MODULE,N},
+    Value = lists:seq(1, N),
+    ok = persistent_term:put(Key, Value),
+
+    %% Calculate the extra memory needed for this term.
+    WordSize = erlang:system_info(wordsize),
+    ExtraMemory = Overhead + 2 * N * WordSize,
+
+    %% Call persistent_term:info/0.
+    {Count,Memory} = info_info(),
+
+    %% There should be one more persistent term.
+    Count = BaseCount + 1,
+
+    %% Verify that the amount of memory is correct.
+    case BaseMemory + ExtraMemory of
+        Memory ->
+            %% Exactly right. The size of the hash table was not changed.
+            ok;
+        Expected ->
+            %% The size of the hash table has been doubled to avoid filling
+            %% the table to more than 50 percent. The previous number
+            %% of entries must have been exactly half the size of the
+            %% hash table. The expected number of extra words added by
+            %% the resizing will be twice that number.
+            ExtraWords = BaseCount * 2,
+            true = ExtraWords * WordSize =:= (Memory - Expected)
+    end,
+    info_wb(N-1, Overhead, {Count,Memory}).
+
+info_info() ->
+    #{count:=Count,memory:=Memory} = persistent_term:info(),
+    true = is_integer(Count) andalso Count >= 0,
+    true = is_integer(Memory) andalso Memory >= 0,
+    {Count,Memory}.
+
+%% Calculate the number of extra bytes needed for storing each term in
+%% the literal, assuming that the key is a tuple of size 2 with
+%% immediate elements. The calculated number is the size of the
+%% ErtsLiteralArea struct excluding the storage for the literal term
+%% itself.
+
+info_literal_area_overhead() ->
+    Key1 = {?MODULE,1},
+    Key2 = {?MODULE,2},
+    #{memory:=Mem0} = persistent_term:info(),
+    ok = persistent_term:put(Key1, literal),
+    #{memory:=Mem1} = persistent_term:info(),
+    ok = persistent_term:put(Key2, literal),
+    #{memory:=Mem2} = persistent_term:info(),
+    true = persistent_term:erase(Key1),
+    true = persistent_term:erase(Key2),
+
+    %% The size of the hash table may have doubled when inserting
+    %% one of the keys. To avoiding counting the change in the hash
+    %% table size, take the smaller size increase.
+    min(Mem2-Mem1, Mem1-Mem0).
+
+%% Test trapping of persistent_term:info/0.
+
+info_trapping(_Config) ->
+    Chk = chk(),
+
+    %% Assume that the info/0 traps after 4000 iterations
+    %% in a non-debug emulator.
+    N = case test_server:timetrap_scale_factor() of
+            1 -> 10000;
+            _ -> 1000
+        end,
+    spawn_link(fun() -> info_trapping_create(N) end),
+    All = do_info_trapping(N, 0),
+    N = info_trapping_check_result(lists:sort(All), 1),
+    erlang:garbage_collect(),
+    info_trapping_erase(N),
+    chk(Chk).
+
+do_info_trapping(N, PrevMem) ->
+    case info_info() of
+        {N,Mem} ->
+            true = Mem >= PrevMem,
+            All = [P || {{?MODULE,{info_trapping,_}},_}=P <- persistent_term:get()],
+            case length(All) of
+                N -> All;
+                _ -> do_info_trapping(N, PrevMem)
+            end;
+        {_,Mem} ->
+            true = Mem >= PrevMem,
+            receive after 1 -> ok end,
+            do_info_trapping(N, Mem)
+    end.
+
+info_trapping_create(0) ->
+    ok;
+info_trapping_create(N) ->
+    ok = persistent_term:put({?MODULE,{info_trapping,N}}, N),
+    info_trapping_create(N-1).
+
+info_trapping_check_result([{{?MODULE,{info_trapping,N}},N}|T], N) ->
+    info_trapping_check_result(T, N+1);
+info_trapping_check_result([], N) -> N-1.
+
+info_trapping_erase(0) ->
+    ok;
+info_trapping_erase(N) ->
+    true = persistent_term:erase({?MODULE,{info_trapping,N}}),
+    info_trapping_erase(N-1).
+
+%% Test that hash tables are deallocated if a process running
+%% persistent_term:get/0 is killed.
+
+killed_while_trapping(_Config) ->
+    Chk = chk(),
+    N = case test_server:timetrap_scale_factor() of
+            1 -> 20000;
+            _ -> 2000
+        end,
+    kwt_put(N),
+    kwt_spawn(10),
+    kwt_erase(N),
+    chk(Chk).
+
+kwt_put(0) ->
+    ok;
+kwt_put(N) ->
+    ok = persistent_term:put({?MODULE,{kwt,N}}, N),
+    kwt_put(N-1).
+
+kwt_spawn(0) ->
+    ok;
+kwt_spawn(N) ->
+    Pids = [spawn(fun kwt_getter/0) || _ <- lists:seq(1, 20)],
+    erlang:yield(),
+    _ = [exit(Pid, kill) || Pid <- Pids],
+    kwt_spawn(N-1).
+
+kwt_getter() ->
+    _ = persistent_term:get(),
+    kwt_getter().
+
+kwt_erase(0) ->
+    ok;
+kwt_erase(N) ->
+    true = persistent_term:erase({?MODULE,{kwt,N}}),
+    kwt_erase(N-1).
+
+%% Test storing off heap values (such as ref-counted binaries).
+
+off_heap_values(_Config) ->
+    Chk = chk(),
+    Key = {?MODULE,?FUNCTION_NAME},
+    Val = {a,list_to_binary(lists:seq(0, 255)),make_ref(),fun() -> ok end},
+    ok = persistent_term:put(Key, Val),
+    FetchedVal = persistent_term:get(Key),
+    Val = FetchedVal,
+    true = persistent_term:erase(Key),
+    off_heap_values_wait(FetchedVal, Val),
+    chk(Chk).
+
+off_heap_values_wait(FetchedVal, Val) ->
+    case erts_debug:size_shared(FetchedVal) of
+        0 ->
+            Val = FetchedVal,
+            ok;
+        _ ->
+            erlang:yield(),
+            off_heap_values_wait(FetchedVal, Val)
+    end.
+
+%% Test some more data types as keys. Use the module name as a key
+%% to minimize the risk of collision with any key used
+%% by the OTP libraries.
+
+keys(_Config) ->
+    Chk = chk(),
+    do_key(?MODULE),
+    do_key([?MODULE]),
+    do_key(?MODULE_STRING),
+    do_key(list_to_binary(?MODULE_STRING)),
+    chk(Chk).
+
+do_key(Key) ->
+    Val = term_to_binary(Key),
+    ok = persistent_term:put(Key, Val),
+    StoredVal = persistent_term:get(Key),
+    Val = StoredVal,
+    true = persistent_term:erase(Key).
+
+%% Create persistent terms with keys that are known to collide.
+%% Delete them in random order, making sure that all others
+%% terms can still be found.
+
+collisions(_Config) ->
+    Chk = chk(),
+
+    %% Create persistent terms with random keys.
+    Keys = lists:flatten(colliding_keys()),
+    Kvs = [{K,rand:uniform(1000)} || K <- Keys],
+    _ = [ok = persistent_term:put(K, V) || {K,V} <- Kvs],
+    _ = [V = persistent_term:get(K) || {K,V} <- Kvs],
+
+    %% Now delete the persistent terms in random order.
+    collisions_delete(lists:keysort(2, Kvs)),
+
+    chk(Chk).
+
+collisions_delete([{Key,Val}|Kvs]) ->
+    Val = persistent_term:get(Key),
+    true = persistent_term:erase(Key),
+    true = lists:sort(persistent_term:get()) =:= lists:sort(Kvs),
+    _ = [V = persistent_term:get(K) || {K,V} <- Kvs],
+    collisions_delete(Kvs);
+collisions_delete([]) ->
+    ok.
+
+colliding_keys() ->
+    %% Collisions found by Jesper L. Andersen for breaking maps.
+    L = [[764492191,2361333849],
+         [49527266765044,90940896816021,20062927283041,267080852079651],
+         [249858369443708,206247021789428,20287304470696,25847120931175],
+         [10645228898670,224705626119556,267405565521452,258214397180678],
+         [264783762221048,166955943492306,98802957003141,102012488332476],
+         [69425677456944,177142907243411,137138950917722,228865047699598],
+         [116031213307147,29203342183358,37406949328742,255198080174323],
+         [200358182338308,235207156008390,120922906095920,116215987197289],
+         [58728890318426,68877471005069,176496507286088,221041411345780],
+         [91094120814795,50665258299931,256093108116737,19777509566621],
+         [74646746200247,98350487270564,154448261001199,39881047281135],
+         [23408943649483,164410325820923,248161749770122,274558342231648],
+         [169531547115055,213630535746863,235098262267796,200508473898303],
+         [235098564415817,85039146398174,51721575960328,173069189684390],
+         [176136386396069,155368359051606,147817099696487,265419485459634],
+         [137542881551462,40028925519736,70525669519846,63445773516557],
+         [173854695142814,114282444507812,149945832627054,99605565798831],
+         [177686773562184,127158716984798,132495543008547],
+         [227073396444896,139667311071766,158915951283562],
+         [26212438434289,94902985796531,198145776057315],
+         [266279278943923,58550737262493,74297973216378],
+         [32373606512065,131854353044428,184642643042326],
+         [34335377662439,85341895822066,273492717750246]],
+
+    %% Verify that the keys still collide (this will fail if the
+    %% internal hash function has been changed).
+    erts_debug:set_internal_state(available_internal_state, true),
+    try
+        case erlang:system_info(wordsize) of
+            8 ->
+                verify_colliding_keys(L);
+            4 ->
+                %% Not guaranteed to collide on a 32-bit system.
+                ok
+        end
+    after
+        erts_debug:set_internal_state(available_internal_state, false)
+    end,
+
+    L.
+
+verify_colliding_keys([[K|Ks]|Gs]) ->
+    Hash = internal_hash(K),
+    [Hash] = lists:usort([internal_hash(Key) || Key <- Ks]),
+    verify_colliding_keys(Gs);
+verify_colliding_keys([]) ->
+    ok.
+
+internal_hash(Term) ->
+    erts_debug:get_internal_state({internal_hash,Term}).
+
+%% Test that all persistent terms are erased by init:restart/0.
+
+init_restart(_Config) ->
+    File = "command_file",
+    ok = file:write_file(File, term_to_binary(restart)),
+    {ok,[[Erl]]} = init:get_argument(progname),
+    ModPath = filename:dirname(code:which(?MODULE)),
+    Cmd = Erl ++ " -pa " ++ ModPath ++ " -noshell "
+        "-run " ++ ?MODULE_STRING ++ " test_init_restart_cmd " ++
+        File,
+    io:format("~s\n", [Cmd]),
+    Expected = "12ok",
+    case os:cmd(Cmd) of
+        Expected ->
+            ok;
+        Actual ->
+            io:format("Expected: ~s", [Expected]),
+            io:format("Actual:   ~s\n", [Actual]),
+            ct:fail(unexpected_output)
+    end.
+
+test_init_restart_cmd([File]) ->
+    try
+        do_test_init_restart_cmd(File)
+    catch
+        C:R ->
+            io:format("\n~p ~p\n", [C,R]),
+            halt()
+    end,
+    receive
+        _ -> ok
+    end.
+
+do_test_init_restart_cmd(File) ->
+    {ok,Bin} = file:read_file(File),
+    Seq = lists:seq(1, 50),
+    case binary_to_term(Bin) of
+        restart ->
+            _ = [persistent_term:put({?MODULE,I}, {value,I}) ||
+                    I <- Seq],
+            ok = file:write_file(File, term_to_binary(was_restarted)),
+            io:put_chars("1"),
+            init:restart(),
+            receive
+                _ -> ok
+            end;
+        was_restarted ->
+            io:put_chars("2"),
+            ok = file:delete(File),
+            _ = [begin
+                     Key = {?MODULE,I},
+                     {'EXIT',{badarg,_}} = (catch persistent_term:get(Key))
+                 end || I <- Seq],
+            io:put_chars("ok"),
+            init:stop()
+    end.
+
+%% Check that there is the same number of persistents terms before
+%% and after each test case.
+
+chk() ->
+    persistent_term:info().
+
+chk(Chk) ->
+    Chk = persistent_term:info(),
+    Key = {?MODULE,?FUNCTION_NAME},
+    ok = persistent_term:put(Key, {term,Chk}),
+    Term = persistent_term:get(Key),
+    true = persistent_term:erase(Key),
+    chk_not_stuck(Term),
+    ok.
+
+chk_not_stuck(Term) ->
+    %% Hash tables to be deleted are put onto a queue.
+    %% Make sure that the queue isn't stuck by a table with
+    %% a non-zero ref count.
+
+    case erts_debug:size_shared(Term) of
+        0 ->
+            erlang:yield(),
+            chk_not_stuck(Term);
+        _ ->
+            ok
+    end.
-- 
cgit v1.2.3


From e1b245e4554911f78086bafa96a0b3e13a48d3cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Fri, 9 Nov 2018 12:43:05 +0100
Subject: Remove --/2 from dirty BIF tests

Now that it traps, --/2 would hang forever when building under
--enable-dirty-schedulers-test.
---
 erts/emulator/beam/erl_dirty_bif.tab | 2 --
 1 file changed, 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_dirty_bif.tab b/erts/emulator/beam/erl_dirty_bif.tab
index 086275fbe5..20299ff604 100644
--- a/erts/emulator/beam/erl_dirty_bif.tab
+++ b/erts/emulator/beam/erl_dirty_bif.tab
@@ -59,8 +59,6 @@ dirty-cpu erts_debug:lcnt_clear/0
 
 dirty-cpu-test erlang:'++'/2
 dirty-cpu-test erlang:append/2
-dirty-cpu-test erlang:'--'/2
-dirty-cpu-test erlang:subtract/2
 dirty-cpu-test erlang:iolist_size/1
 dirty-cpu-test erlang:make_tuple/2
 dirty-cpu-test erlang:make_tuple/3
-- 
cgit v1.2.3


From d15bd6b9366ff4eef81ec9c4bcc875dfe694fe98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 13 Nov 2018 11:55:48 +0100
Subject: Avoid closing files in gc/monitor callbacks

Closing files in these callbacks could block scheduler progress
and cause major system instability. We now defer these operations
to a dedicated process instead.

This process may in turn block forever and prevent further orphaned
files from being closed, but it will keep the emulator itself from
misbehaving.
---
 erts/emulator/nifs/common/prim_file_nif.c | 146 +++++++++++++++++++++---------
 erts/emulator/nifs/common/prim_file_nif.h |   7 +-
 erts/emulator/nifs/unix/unix_prim_file.c  |   7 +-
 erts/emulator/nifs/win32/win_prim_file.c  |   7 +-
 4 files changed, 118 insertions(+), 49 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/nifs/common/prim_file_nif.c b/erts/emulator/nifs/common/prim_file_nif.c
index fd6aaa61f6..ba36a33458 100644
--- a/erts/emulator/nifs/common/prim_file_nif.c
+++ b/erts/emulator/nifs/common/prim_file_nif.c
@@ -38,6 +38,9 @@ static void unload(ErlNifEnv *env, void* priv_data);
 
 static ErlNifResourceType *efile_resource_type;
 
+static ERL_NIF_TERM am_erts_prim_file;
+static ERL_NIF_TERM am_close;
+
 static ERL_NIF_TERM am_ok;
 static ERL_NIF_TERM am_error;
 static ERL_NIF_TERM am_continue;
@@ -96,11 +99,14 @@ static ERL_NIF_TERM set_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM arg
 
 static ERL_NIF_TERM read_file_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 
+static ERL_NIF_TERM open_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM close_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+
+/* Internal ops */
+static ERL_NIF_TERM delayed_close_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 static ERL_NIF_TERM get_handle_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 static ERL_NIF_TERM altname_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 
-static ERL_NIF_TERM open_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
-
 /* All file handle operations are passed through a wrapper that handles state
  * transitions, marking it as busy during the course of the operation, and
  * closing on completion if the owner died in the middle of an operation.
@@ -128,7 +134,11 @@ static ERL_NIF_TERM open_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]
  *
  * CLOSE_PENDING ->
  *      CLOSED (file_handle_wrapper)
- */
+ *
+ * Should the owner of a file die, we can't close it immediately as that could
+ * potentially block a normal scheduler. When entering the CLOSED state from
+ * owner_death_callback, we will instead send a message to the erts_prim_file
+ * process that will then close the file through delayed_close_nif. */
 
 typedef ERL_NIF_TERM (*file_op_impl_t)(efile_data_t *d, ErlNifEnv *env,
     int argc, const ERL_NIF_TERM argv[]);
@@ -142,7 +152,6 @@ static ERL_NIF_TERM file_handle_wrapper(file_op_impl_t operation, ErlNifEnv *env
         return file_handle_wrapper( name ## _impl , env, argc, argv); \
     }
 
-WRAP_FILE_HANDLE_EXPORT(close_nif)
 WRAP_FILE_HANDLE_EXPORT(read_nif)
 WRAP_FILE_HANDLE_EXPORT(write_nif)
 WRAP_FILE_HANDLE_EXPORT(pread_nif)
@@ -193,18 +202,27 @@ static ErlNifFunc nif_funcs[] = {
 
     /* Internal ops. */
     {"get_handle_nif", 1, get_handle_nif},
+    {"delayed_close_nif", 1, delayed_close_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
     {"altname_nif", 1, altname_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
 };
 
 ERL_NIF_INIT(prim_file, nif_funcs, load, NULL, upgrade, unload)
 
+static ErlNifPid erts_prim_file_pid;
+
 static void owner_death_callback(ErlNifEnv* env, void* obj, ErlNifPid* pid, ErlNifMonitor* mon);
-static void gc_callback(ErlNifEnv *env, void* data);
 
-static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info)
+static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM prim_file_pid)
 {
     ErlNifResourceTypeInit callbacks;
 
+    if(!enif_get_local_pid(env, prim_file_pid, &erts_prim_file_pid)) {
+        ASSERT(!"bad pid passed to prim_file_nif");
+    }
+
+    am_erts_prim_file = enif_make_atom(env, "erts_prim_file");
+    am_close = enif_make_atom(env, "close");
+
     am_ok = enif_make_atom(env, "ok");
     am_error = enif_make_atom(env, "error");
     am_continue = enif_make_atom(env, "continue");
@@ -239,7 +257,7 @@ static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info)
     am_eof = enif_make_atom(env, "eof");
 
     callbacks.down = owner_death_callback;
-    callbacks.dtor = gc_callback;
+    callbacks.dtor = NULL;
     callbacks.stop = NULL;
 
     efile_resource_type = enif_open_resource_type_x(env, "efile", &callbacks,
@@ -305,8 +323,10 @@ static ERL_NIF_TERM file_handle_wrapper(file_op_impl_t operation, ErlNifEnv *env
             /* This is the only point where a change from CLOSE_PENDING is
              * possible, and we're running synchronously, so we can't race with
              * anything else here. */
+            posix_errno_t ignored;
+
             erts_atomic32_set_acqb(&d->state, EFILE_STATE_CLOSED);
-            efile_close(d);
+            efile_close(d, &ignored);
         }
     } else {
         /* CLOSE_PENDING should be impossible at this point since it requires
@@ -319,6 +339,24 @@ static ERL_NIF_TERM file_handle_wrapper(file_op_impl_t operation, ErlNifEnv *env
     return result;
 }
 
+/* This is a special close operation used by the erts_prim_file process for
+ * cleaning up orphaned files. It differs from the ordinary close_nif in that
+ * it only works for files that have already entered the CLOSED state. */
+static ERL_NIF_TERM delayed_close_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    posix_errno_t ignored;
+    efile_data_t *d;
+
+    ASSERT(argc == 1);
+    if(!get_file_data(env, argv[0], &d)) {
+        return enif_make_badarg(env);
+    }
+
+    ASSERT(erts_atomic32_read_acqb(&d->state) == EFILE_STATE_CLOSED);
+    efile_close(d, &ignored);
+
+    return am_ok;
+}
+
 static void owner_death_callback(ErlNifEnv* env, void* obj, ErlNifPid* pid, ErlNifMonitor* mon) {
     efile_data_t *d = (efile_data_t*)obj;
 
@@ -334,8 +372,24 @@ static void owner_death_callback(ErlNifEnv* env, void* obj, ErlNifPid* pid, ErlN
 
         switch(previous_state) {
         case EFILE_STATE_IDLE:
-            efile_close(d);
-            return;
+            {
+                /* We cannot close the file here as that could block a normal
+                 * scheduler, so we tell erts_prim_file to do it for us.
+                 *
+                 * This can in turn become a bottleneck (especially in cases
+                 * like NFS failure), but it's less problematic than blocking
+                 * thread progress. */
+                ERL_NIF_TERM message, file_ref;
+
+                file_ref = enif_make_resource(env, d);
+                message = enif_make_tuple2(env, am_close, file_ref);
+
+                if(!enif_send(env, &erts_prim_file_pid, NULL, message)) {
+                    ERTS_INTERNAL_ERROR("Failed to defer prim_file close.");
+                }
+
+                return;
+            }
         case EFILE_STATE_CLOSE_PENDING:
         case EFILE_STATE_CLOSED:
             /* We're either already closed or managed to mark ourselves for
@@ -352,24 +406,6 @@ static void owner_death_callback(ErlNifEnv* env, void* obj, ErlNifPid* pid, ErlN
     }
 }
 
-static void gc_callback(ErlNifEnv *env, void* data) {
-    efile_data_t *d = (efile_data_t*)data;
-
-    enum efile_state_t previous_state;
-
-    (void)env;
-
-    previous_state = erts_atomic32_cmpxchg_acqb(&d->state,
-        EFILE_STATE_CLOSED, EFILE_STATE_IDLE);
-
-    ASSERT(previous_state != EFILE_STATE_CLOSE_PENDING &&
-        previous_state != EFILE_STATE_BUSY);
-
-    if(previous_state == EFILE_STATE_IDLE) {
-        efile_close(d);
-    }
-}
-
 static ERL_NIF_TERM efile_filetype_to_atom(enum efile_filetype_t type) {
     switch(type) {
         case EFILE_FILETYPE_DEVICE: return am_device;
@@ -454,40 +490,62 @@ static ERL_NIF_TERM open_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]
         return posix_error_to_tuple(env, posix_errno);
     }
 
-    result = enif_make_resource(env, d);
-    enif_release_resource(d);
-
     enif_self(env, &controlling_process);
 
     if(enif_monitor_process(env, d, &controlling_process, &d->monitor)) {
+        /* We need to close the file manually as we haven't registered a
+         * destructor. */
+        posix_errno_t ignored;
+
+        erts_atomic32_set_acqb(&d->state, EFILE_STATE_CLOSED);
+        efile_close(d, &ignored);
+
         return posix_error_to_tuple(env, EINVAL);
     }
 
+    /* Note that we do not call enif_release_resource at this point. While it's
+     * normally safe to leave resource management to the GC, efile_close is a
+     * blocking operation which must not be done in the GC callback, and we
+     * can't defer it as the resource is gone as soon as it returns.
+     *
+     * We instead keep the resource alive until efile_close is called, after
+     * which it's safe to leave things to the GC. If the controlling process
+     * were to die before the user had a chance to close their file, the above
+     * monitor will tell the erts_prim_file process to close it for them. */
+    result = enif_make_resource(env, d);
+
     return enif_make_tuple2(env, am_ok, result);
 }
 
-static ERL_NIF_TERM close_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+static ERL_NIF_TERM close_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
     enum efile_state_t previous_state;
+    efile_data_t *d;
 
-    if(argc != 0) {
+    ASSERT(argc == 1);
+    if(!get_file_data(env, argv[0], &d)) {
         return enif_make_badarg(env);
     }
 
     previous_state = erts_atomic32_cmpxchg_acqb(&d->state,
-        EFILE_STATE_CLOSED, EFILE_STATE_BUSY);
+        EFILE_STATE_CLOSED, EFILE_STATE_IDLE);
 
-    ASSERT(previous_state == EFILE_STATE_CLOSE_PENDING ||
-        previous_state == EFILE_STATE_BUSY);
+    if(previous_state == EFILE_STATE_IDLE) {
+        posix_errno_t error;
 
-    if(previous_state == EFILE_STATE_BUSY) {
         enif_demonitor_process(env, d, &d->monitor);
 
-        if(!efile_close(d)) {
-            return posix_error_to_tuple(env, d->posix_errno);
+        if(!efile_close(d, &error)) {
+            return posix_error_to_tuple(env, error);
         }
-    }
 
-    return am_ok;
+        return am_ok;
+    } else {
+        /* CLOSE_PENDING should be impossible at this point since it requires
+         * a transition from BUSY; the only valid state here is CLOSED. */
+        ASSERT(previous_state == EFILE_STATE_CLOSED);
+
+        return posix_error_to_tuple(env, EINVAL);
+    }
 }
 
 static ERL_NIF_TERM read_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
@@ -1190,7 +1248,7 @@ static posix_errno_t read_file(efile_data_t *d, size_t size, ErlNifBinary *resul
 }
 
 static ERL_NIF_TERM read_file_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
-    posix_errno_t posix_errno;
+    posix_errno_t posix_errno, ignored;
 
     efile_fileinfo_t info = {0};
     efile_path_t path;
@@ -1211,7 +1269,9 @@ static ERL_NIF_TERM read_file_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM a
     }
 
     posix_errno = read_file(d, info.size, &result);
-    enif_release_resource(d);
+
+    erts_atomic32_set_acqb(&d->state, EFILE_STATE_CLOSED);
+    efile_close(d, &ignored);
 
     if(posix_errno) {
         return posix_error_to_tuple(env, posix_errno);
diff --git a/erts/emulator/nifs/common/prim_file_nif.h b/erts/emulator/nifs/common/prim_file_nif.h
index 099c06c48c..b2e30c59dd 100644
--- a/erts/emulator/nifs/common/prim_file_nif.h
+++ b/erts/emulator/nifs/common/prim_file_nif.h
@@ -159,8 +159,11 @@ posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes,
         ErlNifResourceType *nif_type, efile_data_t **d);
 
 /** @brief Closes a file. The file must have entered the CLOSED state prior to
- * calling this to prevent double close. */
-int efile_close(efile_data_t *d);
+ * calling this to prevent double close.
+ *
+ * Note that the file is completely invalid after this point, so the error code
+ * is provided in \c error rather than d->posix_errno. */
+int efile_close(efile_data_t *d, posix_errno_t *error);
 
 /* **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** */
 
diff --git a/erts/emulator/nifs/unix/unix_prim_file.c b/erts/emulator/nifs/unix/unix_prim_file.c
index dea73db18a..169b193993 100644
--- a/erts/emulator/nifs/unix/unix_prim_file.c
+++ b/erts/emulator/nifs/unix/unix_prim_file.c
@@ -202,21 +202,24 @@ posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes,
     return errno;
 }
 
-int efile_close(efile_data_t *d) {
+int efile_close(efile_data_t *d, posix_errno_t *error) {
     efile_unix_t *u = (efile_unix_t*)d;
     int fd;
 
+    ASSERT(enif_thread_type() == ERL_NIF_THR_DIRTY_IO_SCHEDULER);
     ASSERT(erts_atomic32_read_nob(&d->state) == EFILE_STATE_CLOSED);
     ASSERT(u->fd != -1);
 
     fd = u->fd;
     u->fd = -1;
 
+    enif_release_resource(d);
+
     /* close(2) either always closes (*BSD, Linux) or leaves the fd in an
      * undefined state (POSIX 2008, Solaris), so we must not retry on EINTR. */
 
     if(close(fd) < 0) {
-        u->common.posix_errno = errno;
+        *error = errno;
         return 0;
     }
 
diff --git a/erts/emulator/nifs/win32/win_prim_file.c b/erts/emulator/nifs/win32/win_prim_file.c
index 602a282dd1..d0aa70542f 100644
--- a/erts/emulator/nifs/win32/win_prim_file.c
+++ b/erts/emulator/nifs/win32/win_prim_file.c
@@ -466,18 +466,21 @@ posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes,
     }
 }
 
-int efile_close(efile_data_t *d) {
+int efile_close(efile_data_t *d, posix_errno_t *error) {
     efile_win_t *w = (efile_win_t*)d;
     HANDLE handle;
 
+    ASSERT(enif_thread_type() == ERL_NIF_THR_DIRTY_IO_SCHEDULER);
     ASSERT(erts_atomic32_read_nob(&d->state) == EFILE_STATE_CLOSED);
     ASSERT(w->handle != INVALID_HANDLE_VALUE);
 
     handle = w->handle;
     w->handle = INVALID_HANDLE_VALUE;
 
+    enif_release_resource(d);
+
     if(!CloseHandle(handle)) {
-        w->common.posix_errno = windows_to_posix_errno(GetLastError());
+        *error = windows_to_posix_errno(GetLastError());
         return 0;
     }
 
-- 
cgit v1.2.3


From 7c902ed60b759c30aabfecde52baa2d12b8885bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 13 Nov 2018 19:47:51 +0100
Subject: Fix broken assertion on monitor release

We sometimes bump the refc without messing with the table, which
means that we sometimes decrement it while in the table, causing
the old assertion to fail.

The property we want to check is that neither end of the monitor is
present in the table when the monitor is deleted.
---
 erts/emulator/beam/erl_monitor_link.h | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_monitor_link.h b/erts/emulator/beam/erl_monitor_link.h
index 9ff8aa509a..ed7bf7d54a 100644
--- a/erts/emulator/beam/erl_monitor_link.h
+++ b/erts/emulator/beam/erl_monitor_link.h
@@ -1387,11 +1387,14 @@ ERTS_GLB_INLINE void
 erts_monitor_release(ErtsMonitor *mon)
 {
     ErtsMonitorData *mdp = erts_monitor_to_data(mon);
-    ERTS_ML_ASSERT(!(mon->flags & ERTS_ML_FLG_IN_TABLE));
     ERTS_ML_ASSERT(erts_atomic32_read_nob(&mdp->refc) > 0);
 
-    if (erts_atomic32_dec_read_nob(&mdp->refc) == 0)
+    if (erts_atomic32_dec_read_nob(&mdp->refc) == 0) {
+        ERTS_ML_ASSERT(!(mdp->origin.flags & ERTS_ML_FLG_IN_TABLE));
+        ERTS_ML_ASSERT(!(mdp->target.flags & ERTS_ML_FLG_IN_TABLE));
+
         erts_monitor_destroy__(mdp);
+    }
 }
 
 ERTS_GLB_INLINE void
@@ -1399,12 +1402,14 @@ erts_monitor_release_both(ErtsMonitorData *mdp)
 {
     ERTS_ML_ASSERT((mdp->origin.flags & ERTS_ML_FLGS_SAME)
                    == (mdp->target.flags & ERTS_ML_FLGS_SAME));
-    ERTS_ML_ASSERT(!(mdp->origin.flags & ERTS_ML_FLG_IN_TABLE));
-    ERTS_ML_ASSERT(!(mdp->target.flags & ERTS_ML_FLG_IN_TABLE));
     ERTS_ML_ASSERT(erts_atomic32_read_nob(&mdp->refc) >= 2);
 
-    if (erts_atomic32_add_read_nob(&mdp->refc, (erts_aint32_t) -2) == 0)
+    if (erts_atomic32_add_read_nob(&mdp->refc, (erts_aint32_t) -2) == 0) {
+        ERTS_ML_ASSERT(!(mdp->origin.flags & ERTS_ML_FLG_IN_TABLE));
+        ERTS_ML_ASSERT(!(mdp->target.flags & ERTS_ML_FLG_IN_TABLE));
+
         erts_monitor_destroy__(mdp);
+    }
 }
 
 ERTS_GLB_INLINE int
-- 
cgit v1.2.3


From 1315c6457e49595fdd3f91693c0506964416c9f0 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 15 Nov 2018 15:24:32 +0100
Subject: erts: Add new module 'atomics'

---
 erts/emulator/Makefile.in            |   2 +
 erts/emulator/beam/atom.names        |   1 +
 erts/emulator/beam/bif.tab           |   9 ++
 erts/emulator/beam/erl_alloc.types   |   1 +
 erts/emulator/beam/erl_bif_atomics.c | 256 +++++++++++++++++++++++++++++++++++
 erts/emulator/beam/sys.h             |   3 +
 erts/emulator/test/Makefile          |   1 +
 erts/emulator/test/atomics_SUITE.erl | 147 ++++++++++++++++++++
 8 files changed, 420 insertions(+)
 create mode 100644 erts/emulator/beam/erl_bif_atomics.c
 create mode 100644 erts/emulator/test/atomics_SUITE.erl

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index d5ceb4b00b..7fa29a7f2e 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -648,6 +648,7 @@ PRELOAD_BEAM =	$(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \
 		$(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam \
 		$(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam \
 		$(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/atomics.beam \
 		$(ERL_TOP)/erts/preloaded/ebin/persistent_term.beam
 
 ifeq ($(TARGET),win32)
@@ -841,6 +842,7 @@ RUN_OBJS += \
 	$(OBJDIR)/erl_bif_info.o	$(OBJDIR)/erl_bif_op.o \
 	$(OBJDIR)/erl_bif_os.o		$(OBJDIR)/erl_bif_lists.o \
 	$(OBJDIR)/erl_bif_persistent.o \
+	$(OBJDIR)/erl_bif_atomics.o \
 	$(OBJDIR)/erl_bif_trace.o	$(OBJDIR)/erl_bif_unique.o \
 	$(OBJDIR)/erl_bif_wrap.o	$(OBJDIR)/erl_nfunc_sched.o \
 	$(OBJDIR)/erl_guard_bifs.o 	$(OBJDIR)/erl_dirty_bif_wrap.o \
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 82bb620d62..a14f22b19e 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -396,6 +396,7 @@ atom microsecond
 atom microstate_accounting
 atom milli_seconds
 atom millisecond
+atom min
 atom min_heap_size
 atom min_bin_vheap_size
 atom minor
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index f1f34ae323..0577c5722f 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -710,3 +710,12 @@ bif persistent_term:get/0
 bif persistent_term:erase/1
 bif persistent_term:info/0
 bif erts_internal:erase_persistent_terms/0
+
+bif erts_internal:atomics_new/2
+bif atomics:get/2
+bif atomics:put/3
+bif atomics:add/3
+bif atomics:add_get/3
+bif atomics:exchange/3
+bif atomics:compare_exchange/4
+bif atomics:info/1
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 3fc036a13c..aefa04a031 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -338,6 +338,7 @@ type	GC_INFO_REQ	SHORT_LIVED	SYSTEM		gc_info_request
 type	PORT_DATA_HEAP	STANDARD	SYSTEM		port_data_heap
 type    MSACC           DRIVER          SYSTEM          microstate_accounting
 type	SYS_CHECK_REQ	SHORT_LIVED	SYSTEM		system_check_request
+type	ATOMICS		STANDARD	SYSTEM		erl_bif_atomics
 
 #
 # Types used by system specific code
diff --git a/erts/emulator/beam/erl_bif_atomics.c b/erts/emulator/beam/erl_bif_atomics.c
new file mode 100644
index 0000000000..092dbb3bd3
--- /dev/null
+++ b/erts/emulator/beam/erl_bif_atomics.c
@@ -0,0 +1,256 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2018. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Purpose:  High performance atomics.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include <stddef.h> /* offsetof */
+
+#include "sys.h"
+#include "export.h"
+#include "bif.h"
+#include "erl_threads.h"
+#include "big.h"
+#include "erl_binary.h"
+#include "erl_bif_unique.h"
+#include "erl_map.h"
+
+typedef struct
+{
+    int is_signed;
+    UWord vlen;
+    erts_atomic64_t v[1];
+}AtomicsRef;
+
+static int atomics_destructor(Binary *unused)
+{
+    return 1;
+}
+
+#define OPT_SIGNED (1 << 0)
+
+BIF_RETTYPE erts_internal_atomics_new_2(BIF_ALIST_2)
+{
+    AtomicsRef* p;
+    Binary* mbin;
+    UWord i, cnt, opts;
+    Uint bytes;
+    Eterm* hp;
+
+    if (!term_to_UWord(BIF_ARG_1, &cnt)
+        || cnt == 0
+        || !term_to_UWord(BIF_ARG_2, &opts)) {
+
+        BIF_ERROR(BIF_P, BADARG);
+    }
+
+    if (cnt > (ERTS_UWORD_MAX / sizeof(p->v[0])))
+        BIF_ERROR(BIF_P, SYSTEM_LIMIT);
+
+    bytes = offsetof(AtomicsRef, v) + cnt*sizeof(p->v[0]);
+    mbin = erts_create_magic_binary_x(bytes,
+                                      atomics_destructor,
+                                      ERTS_ALC_T_ATOMICS,
+                                      0);
+    p = ERTS_MAGIC_BIN_DATA(mbin);
+    p->is_signed = opts & OPT_SIGNED;
+    p->vlen = cnt;
+    for (i=0; i < cnt; i++)
+        erts_atomic64_init_nob(&p->v[i], 0);
+    hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
+    return erts_mk_magic_ref(&hp, &MSO(BIF_P), mbin);
+}
+
+static ERTS_INLINE int get_ref(Eterm ref, AtomicsRef** pp)
+{
+    Binary* mbin;
+    if (!is_internal_magic_ref(ref))
+        return 0;
+
+    mbin = erts_magic_ref2bin(ref);
+    if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != atomics_destructor)
+        return 0;
+    *pp = ERTS_MAGIC_BIN_DATA(mbin);
+    return 1;
+}
+
+static ERTS_INLINE int get_ref_ix(Eterm ref, Eterm ix,
+                                  AtomicsRef** pp, UWord* ixp)
+{
+    return (get_ref(ref, pp)
+            && term_to_UWord(ix, ixp)
+            && --(*ixp) < (*pp)->vlen);
+}
+
+static ERTS_INLINE int get_value(AtomicsRef* p, Eterm term, erts_aint64_t *valp)
+{
+    return (p->is_signed ?
+            term_to_Sint64(term, (Sint64*)valp) :
+            term_to_Uint64(term, (Uint64*)valp));
+}
+
+static ERTS_INLINE int get_incr(AtomicsRef* p, Eterm term, erts_aint64_t *valp)
+{
+    return (term_to_Sint64(term, (Sint64*)valp)
+            || term_to_Uint64(term, (Uint64*)valp));
+}
+
+static ERTS_INLINE Eterm bld_atomic(Process* proc, AtomicsRef* p,
+                                    erts_aint64_t val)
+{
+    if (p->is_signed) {
+        if (IS_SSMALL(val))
+            return make_small((Sint) val);
+        else {
+            Uint hsz = ERTS_SINT64_HEAP_SIZE(val);
+            Eterm* hp = HAlloc(proc, hsz);
+            return erts_sint64_to_big(val, &hp);
+        }
+    }
+    else {
+        if ((Uint64)val <= MAX_SMALL)
+            return make_small((Sint) val);
+        else {
+            Uint hsz = ERTS_UINT64_HEAP_SIZE(val);
+            Eterm* hp = HAlloc(proc, hsz);
+            return erts_uint64_to_big(val, &hp);
+        }
+    }
+}
+
+BIF_RETTYPE atomics_put_3(BIF_ALIST_3)
+{
+    AtomicsRef* p;
+    UWord ix;
+    erts_aint64_t val;
+
+    if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix)
+        || !get_value(p, BIF_ARG_3, &val)) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+    erts_atomic64_set_mb(&p->v[ix], val);
+    return am_ok;
+}
+
+BIF_RETTYPE atomics_get_2(BIF_ALIST_2)
+{
+    AtomicsRef* p;
+    UWord ix;
+
+    if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix)) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+    return bld_atomic(BIF_P, p, erts_atomic64_read_mb(&p->v[ix]));
+}
+
+BIF_RETTYPE atomics_add_3(BIF_ALIST_3)
+{
+    AtomicsRef* p;
+    UWord ix;
+    erts_aint64_t incr;
+
+    if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix)
+        || !get_incr(p, BIF_ARG_3, &incr)) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+    erts_atomic64_add_mb(&p->v[ix], incr);
+    return am_ok;
+}
+
+BIF_RETTYPE atomics_add_get_3(BIF_ALIST_3)
+{
+    AtomicsRef* p;
+    UWord ix;
+    erts_aint64_t incr;
+
+    if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix)
+        || !get_incr(p, BIF_ARG_3, &incr)) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+    return bld_atomic(BIF_P, p, erts_atomic64_add_read_mb(&p->v[ix], incr));
+}
+
+BIF_RETTYPE atomics_exchange_3(BIF_ALIST_3)
+{
+    AtomicsRef* p;
+    UWord ix;
+    erts_aint64_t desired, was;
+
+    if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix)
+        || !get_value(p, BIF_ARG_3, &desired)) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+    was = erts_atomic64_xchg_mb(&p->v[ix], desired);
+    return bld_atomic(BIF_P, p, was);
+}
+
+BIF_RETTYPE atomics_compare_exchange_4(BIF_ALIST_4)
+{
+    AtomicsRef* p;
+    UWord ix;
+    erts_aint64_t expected, desired, was;
+
+    if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix)
+        || !get_value(p, BIF_ARG_3, &expected)
+        || !get_value(p, BIF_ARG_4, &desired)) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+    was = erts_atomic64_cmpxchg_mb(&p->v[ix], desired, expected);
+    return was == expected ? am_ok : bld_atomic(BIF_P, p, was);
+}
+
+BIF_RETTYPE atomics_info_1(BIF_ALIST_1)
+{
+    AtomicsRef* p;
+    Uint hsz = MAP4_SZ;
+    Eterm *hp;
+    Uint64 max;
+    Sint64 min;
+    UWord memory;
+    Eterm max_val, min_val, sz_val, mem_val;
+
+    if (!get_ref(BIF_ARG_1, &p))
+        BIF_ERROR(BIF_P, BADARG);
+
+    max = p->is_signed ? ERTS_SINT64_MAX : ERTS_UINT64_MAX;
+    min = p->is_signed ? ERTS_SINT64_MIN : 0;
+    memory = erts_magic_ref2bin(BIF_ARG_1)->orig_size;
+
+    erts_bld_uint64(NULL, &hsz, max);
+    erts_bld_sint64(NULL, &hsz, min);
+    erts_bld_uword(NULL, &hsz, p->vlen);
+    erts_bld_uword(NULL, &hsz, memory);
+
+    hp = HAlloc(BIF_P, hsz);
+    max_val = erts_bld_uint64(&hp, NULL, max);
+    min_val = erts_bld_sint64(&hp, NULL, min);
+    sz_val  = erts_bld_uword(&hp, NULL, p->vlen);
+    mem_val = erts_bld_uword(&hp, NULL, memory);
+
+    return MAP4(hp, am_max, max_val,
+                am_memory, mem_val,
+                am_min, min_val,
+                am_size, sz_val);
+}
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index bb22548587..869a575cb4 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -325,6 +325,7 @@ typedef long          Sint  erts_align_attribute(sizeof(long));
 #define UWORD_CONSTANT(Const) Const##UL
 #define ERTS_UWORD_MAX ULONG_MAX
 #define ERTS_SWORD_MAX LONG_MAX
+#define ERTS_SWORD_MIN LONG_MIN
 #define ERTS_SIZEOF_ETERM SIZEOF_LONG
 #define ErtsStrToSint strtol
 #elif SIZEOF_VOID_P == SIZEOF_INT
@@ -335,6 +336,7 @@ typedef int          Sint  erts_align_attribute(sizeof(int));
 #define UWORD_CONSTANT(Const) Const##U
 #define ERTS_UWORD_MAX UINT_MAX
 #define ERTS_SWORD_MAX INT_MAX
+#define ERTS_SWORD_MIN INT_MIN
 #define ERTS_SIZEOF_ETERM SIZEOF_INT
 #define ErtsStrToSint strtol
 #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
@@ -345,6 +347,7 @@ typedef long long          Sint  erts_align_attribute(sizeof(long long));
 #define UWORD_CONSTANT(Const) Const##ULL
 #define ERTS_UWORD_MAX ULLONG_MAX
 #define ERTS_SWORD_MAX LLONG_MAX
+#define ERTS_SWORD_MIN LLONG_MIN
 #define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG
 #if defined(__WIN32__)
 #define ErtsStrToSint _strtoi64
diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index d201ea6ee1..fc395899b8 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -33,6 +33,7 @@ MODULES= \
 	after_SUITE \
 	alloc_SUITE \
 	async_ports_SUITE \
+	atomics_SUITE \
 	beam_SUITE \
 	beam_literals_SUITE \
 	bif_SUITE \
diff --git a/erts/emulator/test/atomics_SUITE.erl b/erts/emulator/test/atomics_SUITE.erl
new file mode 100644
index 0000000000..8c42354770
--- /dev/null
+++ b/erts/emulator/test/atomics_SUITE.erl
@@ -0,0 +1,147 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2018. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(atomics_SUITE).
+
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+    [signed, unsigned, bad, signed_limits, unsigned_limits].
+
+signed(Config) when is_list(Config) ->
+    Size = 10,
+    Ref = atomics:new(Size,[]),
+    #{size:=Size, memory:=Memory} = atomics:info(Ref),
+    {_,true} = {Memory, Memory > Size*8},
+    {_,true} = {Memory, Memory < Size*max_atomic_sz() + 100},
+    [signed_do(Ref, Ix) || Ix <- lists:seq(1, Size)],
+    ok.
+
+signed_do(Ref, Ix) ->
+    0 = atomics:get(Ref, Ix),
+    ok = atomics:put(Ref, Ix, 3),
+    ok = atomics:add(Ref, Ix, 14),
+    17 = atomics:get(Ref, Ix),
+    20 = atomics:add_get(Ref, Ix, 3),
+    -3 = atomics:add_get(Ref, Ix, -23),
+    17 = atomics:add_get(Ref, Ix, 20),
+    ok = atomics:sub(Ref, Ix, 4),
+    13 = atomics:get(Ref, Ix),
+    -7 = atomics:sub_get(Ref, Ix, 20),
+    3  = atomics:sub_get(Ref, Ix, -10),
+    3  = atomics:exchange(Ref, Ix, 666),
+    ok = atomics:compare_exchange(Ref, Ix, 666, 777),
+    777 = atomics:compare_exchange(Ref, Ix, 666, -666),
+    ok.
+
+unsigned(Config) when is_list(Config) ->
+    Size = 10,
+    Ref = atomics:new(Size,[{signed, false}]),
+    #{size:=Size, memory:=Memory} = atomics:info(Ref),
+    true = Memory > Size*8,
+    true = Memory < Size*max_atomic_sz() + 100,
+    [unsigned_do(Ref, Ix) || Ix <- lists:seq(1, Size)],
+    ok.
+
+unsigned_do(Ref, Ix) ->
+    0 = atomics:get(Ref, Ix),
+    ok = atomics:put(Ref, Ix, 3),
+    ok = atomics:add(Ref, Ix, 14),
+    17 = atomics:get(Ref, Ix),
+    20 = atomics:add_get(Ref, Ix, 3),
+    ok = atomics:sub(Ref, Ix, 7),
+    13 = atomics:get(Ref, Ix),
+    3  = atomics:sub_get(Ref, Ix, 10),
+    3  = atomics:exchange(Ref, Ix, 666),
+    ok = atomics:compare_exchange(Ref, Ix, 666, 777),
+    777 = atomics:compare_exchange(Ref, Ix, 666, 888),
+    ok.
+
+bad(Config) when is_list(Config) ->
+    {'EXIT',{badarg,_}} = (catch atomics:new(0,[])),
+    {'EXIT',{badarg,_}} = (catch atomics:new(10,[bad])),
+    {'EXIT',{badarg,_}} = (catch atomics:new(10,[{signed,bad}])),
+    {'EXIT',{badarg,_}} = (catch atomics:new(10,[{signed,true}, bad])),
+    {'EXIT',{badarg,_}} = (catch atomics:new(10,[{signed,false} | bad])),
+    Ref = atomics:new(10,[]),
+    {'EXIT',{badarg,_}} = (catch atomics:get(1742, 7)),
+    {'EXIT',{badarg,_}} = (catch atomics:get(make_ref(), 7)),
+    {'EXIT',{badarg,_}} = (catch atomics:get(Ref, -1)),
+    {'EXIT',{badarg,_}} = (catch atomics:get(Ref, 0)),
+    {'EXIT',{badarg,_}} = (catch atomics:get(Ref, 11)),
+    {'EXIT',{badarg,_}} = (catch atomics:get(Ref, 7.0)),
+    ok.
+
+
+signed_limits(Config) when is_list(Config) ->
+    Bits = 64,
+    Max = (1 bsl (Bits-1)) - 1,
+    Min = -(1 bsl (Bits-1)),
+
+    Ref = atomics:new(1,[{signed, true}]),
+    #{max:=Max, min:=Min} = atomics:info(Ref),
+    0 = atomics:get(Ref, 1),
+    ok = atomics:add(Ref, 1, Max),
+    Min = atomics:add_get(Ref, 1, 1),
+    Max = atomics:sub_get(Ref, 1, 1),
+
+    IncrMax = (Max bsl 1) bor 1,
+    ok = atomics:put(Ref, 1, 0),
+    ok = atomics:add(Ref, 1, IncrMax),
+    -1 = atomics:get(Ref, 1),
+    {'EXIT',{badarg,_}} = (catch atomics:add(Ref, 1, IncrMax+1)),
+    {'EXIT',{badarg,_}} = (catch atomics:add(Ref, 1, Min-1)),
+
+    ok.
+
+unsigned_limits(Config) when is_list(Config) ->
+    Bits = 64,
+    Max = (1 bsl Bits) - 1,
+    Min = 0,
+
+    Ref = atomics:new(1,[{signed,false}]),
+    #{max:=Max, min:=Min} = atomics:info(Ref),
+    0 = atomics:get(Ref, 1),
+    ok = atomics:add(Ref, 1, Max),
+    Min = atomics:add_get(Ref, 1, 1),
+    Max = atomics:sub_get(Ref, 1, 1),
+
+    {'EXIT',{badarg,_}} = (catch atomics:add(Ref, 1, Max+1)),
+    IncrMin = -(1 bsl (Bits-1)),
+    ok = atomics:put(Ref, 1, -IncrMin),
+    ok = atomics:add(Ref, 1, IncrMin),
+    0 = atomics:get(Ref, 1),
+    {'EXIT',{badarg,_}} = (catch atomics:add(Ref, 1, IncrMin-1)),
+
+    ok.
+
+max_atomic_sz() ->
+    case erlang:system_info({wordsize, external}) of
+        4 -> 16;
+        8 ->
+            EI = erlang:system_info(ethread_info),
+            case lists:keyfind("64-bit native atomics", 1, EI) of
+                {_, "no", _} -> 16;
+                _ -> 8
+            end
+    end.
-- 
cgit v1.2.3


From fefb5d039e87ff7137e78b3d5f2eaf01e498ec4d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 25 Sep 2018 13:34:52 +0200
Subject: erts: Add new module 'counters'

---
 erts/emulator/Makefile.in             |   3 +-
 erts/emulator/beam/bif.tab            |   5 +
 erts/emulator/beam/erl_alloc.types    |   1 +
 erts/emulator/beam/erl_bif_counters.c | 219 ++++++++++++++++++++++++++++++++++
 erts/emulator/test/Makefile           |   1 +
 erts/emulator/test/counters_SUITE.erl | 112 +++++++++++++++++
 6 files changed, 340 insertions(+), 1 deletion(-)
 create mode 100644 erts/emulator/beam/erl_bif_counters.c
 create mode 100644 erts/emulator/test/counters_SUITE.erl

(limited to 'erts/emulator')

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 7fa29a7f2e..57a9d45887 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -649,6 +649,7 @@ PRELOAD_BEAM =	$(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \
 		$(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam \
 		$(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam \
 		$(ERL_TOP)/erts/preloaded/ebin/atomics.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/counters.beam \
 		$(ERL_TOP)/erts/preloaded/ebin/persistent_term.beam
 
 ifeq ($(TARGET),win32)
@@ -842,7 +843,7 @@ RUN_OBJS += \
 	$(OBJDIR)/erl_bif_info.o	$(OBJDIR)/erl_bif_op.o \
 	$(OBJDIR)/erl_bif_os.o		$(OBJDIR)/erl_bif_lists.o \
 	$(OBJDIR)/erl_bif_persistent.o \
-	$(OBJDIR)/erl_bif_atomics.o \
+	$(OBJDIR)/erl_bif_atomics.o    	$(OBJDIR)/erl_bif_counters.o \
 	$(OBJDIR)/erl_bif_trace.o	$(OBJDIR)/erl_bif_unique.o \
 	$(OBJDIR)/erl_bif_wrap.o	$(OBJDIR)/erl_nfunc_sched.o \
 	$(OBJDIR)/erl_guard_bifs.o 	$(OBJDIR)/erl_dirty_bif_wrap.o \
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 0577c5722f..aa3c3acd9f 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -719,3 +719,8 @@ bif atomics:add_get/3
 bif atomics:exchange/3
 bif atomics:compare_exchange/4
 bif atomics:info/1
+
+bif erts_internal:counters_new/1
+bif erts_internal:counters_get/2
+bif erts_internal:counters_add/3
+bif erts_internal:counters_info/1
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index aefa04a031..4f03a34390 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -339,6 +339,7 @@ type	PORT_DATA_HEAP	STANDARD	SYSTEM		port_data_heap
 type    MSACC           DRIVER          SYSTEM          microstate_accounting
 type	SYS_CHECK_REQ	SHORT_LIVED	SYSTEM		system_check_request
 type	ATOMICS		STANDARD	SYSTEM		erl_bif_atomics
+type	COUNTERS	STANDARD	SYSTEM		erl_bif_counters
 
 #
 # Types used by system specific code
diff --git a/erts/emulator/beam/erl_bif_counters.c b/erts/emulator/beam/erl_bif_counters.c
new file mode 100644
index 0000000000..a46b462225
--- /dev/null
+++ b/erts/emulator/beam/erl_bif_counters.c
@@ -0,0 +1,219 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2018. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Purpose:  High performance atomics.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include <stddef.h> /* offsetof */
+
+#include "sys.h"
+#include "export.h"
+#include "bif.h"
+#include "erl_threads.h"
+#include "big.h"
+#include "erl_binary.h"
+#include "erl_bif_unique.h"
+#include "erl_map.h"
+
+
+#define COUNTERS_PER_CACHE_LINE (ERTS_CACHE_LINE_SIZE / sizeof(erts_atomic64_t))
+
+typedef struct
+{
+    UWord arity;
+#ifdef DEBUG
+    UWord ulen;
+#endif
+    union {
+        erts_atomic64_t v[COUNTERS_PER_CACHE_LINE];
+        byte cache_line__[ERTS_CACHE_LINE_SIZE];
+    } u[1];
+}CountersRef;
+
+static int counters_destructor(Binary *unused)
+{
+    return 1;
+}
+
+
+static UWord ERTS_INLINE div_ceil(UWord dividend, UWord divisor)
+{
+    return (dividend + divisor - 1) / divisor;
+}
+
+BIF_RETTYPE erts_internal_counters_new_1(BIF_ALIST_1)
+{
+    CountersRef* p;
+    Binary* mbin;
+    UWord ui, vi, cnt;
+    Uint bytes, cache_lines;
+    Eterm* hp;
+
+    if (!term_to_UWord(BIF_ARG_1, &cnt)
+        || cnt == 0) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+
+    if (cnt > (ERTS_UWORD_MAX / (sizeof(erts_atomic64_t)*2*erts_no_schedulers)))
+        BIF_ERROR(BIF_P, SYSTEM_LIMIT);
+
+    cache_lines = erts_no_schedulers * div_ceil(cnt, COUNTERS_PER_CACHE_LINE);
+    bytes = offsetof(CountersRef, u) + cache_lines * ERTS_CACHE_LINE_SIZE;
+    mbin = erts_create_magic_binary_x(bytes,
+                                      counters_destructor,
+                                      ERTS_ALC_T_ATOMICS,
+                                      0);
+    p = ERTS_MAGIC_BIN_DATA(mbin);
+    p->arity = cnt;
+#ifdef DEBUG
+    p->ulen = cache_lines;
+#endif
+    ASSERT((byte*)&p->u[cache_lines] <= ((byte*)p + bytes));
+    for (ui=0; ui < cache_lines; ui++)
+        for (vi=0; vi < COUNTERS_PER_CACHE_LINE; vi++)
+            erts_atomic64_init_nob(&p->u[ui].v[vi], 0);
+    hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
+    return erts_mk_magic_ref(&hp, &MSO(BIF_P), mbin);
+}
+
+static ERTS_INLINE int get_ref(Eterm ref, CountersRef** pp)
+{
+    Binary* mbin;
+    if (!is_internal_magic_ref(ref))
+        return 0;
+
+    mbin = erts_magic_ref2bin(ref);
+    if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != counters_destructor)
+        return 0;
+    *pp = ERTS_MAGIC_BIN_DATA(mbin);
+    return 1;
+}
+
+static ERTS_INLINE int get_ref_cnt(Eterm ref, Eterm index,
+                                   CountersRef** pp,
+                                   erts_atomic64_t** app,
+                                   UWord sched_ix)
+{
+    CountersRef* p;
+    UWord ix, ui, vi;
+    if (!get_ref(ref, &p) || !term_to_UWord(index, &ix) || --ix >= p->arity)
+        return 0;
+    ui = (ix / COUNTERS_PER_CACHE_LINE) * erts_no_schedulers + sched_ix;
+    vi = ix % COUNTERS_PER_CACHE_LINE;
+    ASSERT(ui < p->ulen);
+    *pp = p;
+    *app = &p->u[ui].v[vi];
+    return 1;
+}
+
+static ERTS_INLINE int get_ref_my_cnt(Eterm ref, Eterm index,
+                                      CountersRef** pp,
+                                      erts_atomic64_t** app)
+{
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp));
+    return get_ref_cnt(ref, index, pp, app, esdp->no - 1);
+}
+
+static ERTS_INLINE int get_ref_first_cnt(Eterm ref, Eterm index,
+                                         CountersRef** pp,
+                                         erts_atomic64_t** app)
+{
+    return get_ref_cnt(ref, index, pp, app, 0);
+}
+
+static ERTS_INLINE int get_incr(CountersRef* p, Eterm term, erts_aint64_t *valp)
+{
+    return (term_to_Sint64(term, (Sint64*)valp)
+            || term_to_Uint64(term, (Uint64*)valp));
+}
+
+static ERTS_INLINE Eterm bld_counter(Process* proc, CountersRef* p,
+                                     erts_aint64_t val)
+{
+    if (IS_SSMALL(val))
+        return make_small((Sint) val);
+    else {
+        Uint hsz = ERTS_SINT64_HEAP_SIZE(val);
+        Eterm* hp = HAlloc(proc, hsz);
+        return erts_sint64_to_big(val, &hp);
+    }
+}
+
+BIF_RETTYPE erts_internal_counters_get_2(BIF_ALIST_2)
+{
+    CountersRef* p;
+    erts_atomic64_t* ap;
+    erts_aint64_t acc = 0;
+    int j;
+
+    if (!get_ref_first_cnt(BIF_ARG_1, BIF_ARG_2, &p, &ap)) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+    for (j = erts_no_schedulers; j ; --j) {
+        acc += erts_atomic64_read_nob(ap);
+        ap = (erts_atomic64_t*) ((byte*)ap + ERTS_CACHE_LINE_SIZE);
+    }
+    return bld_counter(BIF_P, p, acc);
+}
+
+BIF_RETTYPE erts_internal_counters_add_3(BIF_ALIST_3)
+{
+    CountersRef* p;
+    erts_atomic64_t* ap;
+    erts_aint64_t incr, sum;
+
+    if (!get_ref_my_cnt(BIF_ARG_1, BIF_ARG_2, &p, &ap)
+        || !get_incr(p, BIF_ARG_3, &incr)) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+    sum = incr + erts_atomic64_read_nob(ap);
+    erts_atomic64_set_nob(ap, sum);
+    return am_ok;
+}
+
+
+BIF_RETTYPE erts_internal_counters_info_1(BIF_ALIST_1)
+{
+    CountersRef* p;
+    Uint hsz = MAP2_SZ;
+    Eterm *hp;
+    UWord memory;
+    Eterm sz_val, mem_val;
+
+    if (!get_ref(BIF_ARG_1, &p))
+        BIF_ERROR(BIF_P, BADARG);
+
+    memory = erts_magic_ref2bin(BIF_ARG_1)->orig_size;
+    erts_bld_uword(NULL, &hsz, p->arity);
+    erts_bld_uword(NULL, &hsz, memory);
+
+    hp = HAlloc(BIF_P, hsz);
+    sz_val  = erts_bld_uword(&hp, NULL, p->arity);
+    mem_val = erts_bld_uword(&hp, NULL, memory);
+
+    return MAP2(hp, am_memory, mem_val,
+                am_size, sz_val);
+}
diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index fc395899b8..6a064ec8d4 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -51,6 +51,7 @@ MODULES= \
 	call_trace_SUITE \
 	code_SUITE \
 	code_parallel_load_SUITE \
+	counters_SUITE \
 	crypto_SUITE \
 	ddll_SUITE \
 	decode_packet_SUITE \
diff --git a/erts/emulator/test/counters_SUITE.erl b/erts/emulator/test/counters_SUITE.erl
new file mode 100644
index 0000000000..7de164096b
--- /dev/null
+++ b/erts/emulator/test/counters_SUITE.erl
@@ -0,0 +1,112 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2018. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(counters_SUITE).
+
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+    [basic, bad, limits].
+
+basic(Config) when is_list(Config) ->
+    Size = 10,
+    [begin
+         Ref = counters:new(Size,[Type]),
+         #{size:=Size, memory:=Memory} = counters:info(Ref),
+         check_memory(Type, Memory, Size),
+         [basic_do(Ref, Ix) || Ix <- lists:seq(1, Size)]
+     end
+     || Type <- [atomics, write_concurrency]],
+    ok.
+
+basic_do(Ref, Ix) ->
+    0 = counters:get(Ref, Ix),
+    ok = counters:add(Ref, Ix, 3),
+    3  = counters:get(Ref, Ix),
+    ok = counters:add(Ref, Ix, 14),
+    17  = counters:get(Ref, Ix),
+    ok = counters:add(Ref, Ix, -20),
+    -3  = counters:get(Ref, Ix),
+    ok = counters:add(Ref, Ix, 100),
+    97 = counters:get(Ref, Ix),
+    ok = counters:sub(Ref, Ix, 20),
+    77 = counters:get(Ref, Ix),
+    ok = counters:sub(Ref, Ix, -10),
+    87 = counters:get(Ref, Ix),
+    ok.
+
+check_memory(atomics, Memory, Size) ->
+    {_,true} = {Memory, Memory > Size*8},
+    {_,true} = {Memory, Memory < Size*max_atomic_sz() + 100};
+check_memory(write_concurrency, Memory, Size) ->
+    NScheds = erlang:system_info(schedulers),
+    {_,true} = {Memory, Memory > NScheds*Size*8},
+    {_,true} = {Memory, Memory < NScheds*(Size+7)*max_atomic_sz() + 100}.
+
+max_atomic_sz() ->
+    case erlang:system_info({wordsize, external}) of
+        4 -> 16;
+        8 ->
+            EI = erlang:system_info(ethread_info),
+            case lists:keyfind("64-bit native atomics", 1, EI) of
+                {_, "no", _} -> 16;
+                _ -> 8
+            end
+    end.
+
+bad(Config) when is_list(Config) ->
+    {'EXIT',{badarg,_}} = (catch counters:new(0,[])),
+    {'EXIT',{badarg,_}} = (catch counters:new(10,[bad])),
+    {'EXIT',{badarg,_}} = (catch counters:new(10,[atomic, bad])),
+    {'EXIT',{badarg,_}} = (catch counters:new(10,[write_concurrency | bad])),
+    Ref = counters:new(10,[]),
+    {'EXIT',{badarg,_}} = (catch counters:get(1742, 7)),
+    {'EXIT',{badarg,_}} = (catch counters:get(make_ref(), 7)),
+    {'EXIT',{badarg,_}} = (catch counters:get(Ref, -1)),
+    {'EXIT',{badarg,_}} = (catch counters:get(Ref, 0)),
+    {'EXIT',{badarg,_}} = (catch counters:get(Ref, 11)),
+    {'EXIT',{badarg,_}} = (catch counters:get(Ref, 7.0)),
+    ok.
+
+
+limits(Config) when is_list(Config) ->
+    Bits = 64,
+    Max = (1 bsl (Bits-1)) - 1,
+    Min = -(1 bsl (Bits-1)),
+
+    Ref = counters:new(1,[]),
+    0 = counters:get(Ref, 1),
+    ok = counters:add(Ref, 1, Max),
+    ok = counters:add(Ref, 1, 1),
+    Min = counters:get(Ref, 1),
+    ok  = counters:sub(Ref, 1, 1),
+    Max = counters:get(Ref, 1),
+
+    IncrMax = (Max bsl 1) bor 1,
+    ok = counters:sub(Ref, 1, counters:get(Ref, 1)),
+    ok = counters:add(Ref, 1, IncrMax),
+    -1 = counters:get(Ref, 1),
+    {'EXIT',{badarg,_}} = (catch counters:add(Ref, 1, IncrMax+1)),
+    {'EXIT',{badarg,_}} = (catch counters:add(Ref, 1, Min-1)),
+
+    ok.
-- 
cgit v1.2.3


From 3a812a499f9a40951f1ba6b7315473c5e149fd8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 20 Nov 2018 13:21:32 +0100
Subject: Allow disabling retpoline in interpreter loop

We only do this when the user has explicitly told us it's okay to
partially disable mitigation (spectre-mitigation=incomplete). The
macro is inert if it isn't.
---
 erts/emulator/beam/beam_emu.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 1ad13c32e3..0f6e6533fc 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1227,6 +1227,7 @@ init_emulator(void)
  * the instructions' C labels to the loader.
  * The second call starts execution of BEAM code. This call never returns.
  */
+ERTS_NO_RETPOLINE
 void process_main(void)
 {
     static int init_done = 0;
-- 
cgit v1.2.3


From bde3ec5d818d1948e8a0008a22bcb2e247ce3cc6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 20 Nov 2018 17:30:25 +0100
Subject: erts: Improve driver_SUITE:smp_select

to better detect an actual missing ready_input event
on slow machines (valgrind)
and without wasting time with long sleep on fast machines.
---
 erts/emulator/test/driver_SUITE.erl              | 17 ++++++++-
 erts/emulator/test/driver_SUITE_data/chkio_drv.c | 45 +++++++++++++++++++-----
 2 files changed, 52 insertions(+), 10 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 6f5d639d04..bd62708aa7 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -1754,7 +1754,7 @@ smp_select0(Config) ->
     ProcFun = fun()-> io:format("Worker ~p starting\n",[self()]),	
                       Port = open_port({spawn, DrvName}, []),
                       smp_select_loop(Port, 100000),
-                      sleep(1000), % wait for driver to handle pending events
+                      smp_select_done(Port),
                       true = erlang:port_close(Port),
                       Master ! {ok,self()},
                       io:format("Worker ~p finished\n",[self()])
@@ -1784,6 +1784,21 @@ smp_select_loop(Port, N) ->
               smp_select_loop(Port, N-1)
     end.
 
+smp_select_done(Port) ->
+    case erlang:port_control(Port, ?CHKIO_SMP_SELECT, "done") of
+        "wait" ->
+            receive
+                {Port, done} ->
+                    ok
+            after 10*1000 ->
+                    %% Seems we have a lost ready_input event.
+                    %% Go ahead anyway, port will crash VM when closed.
+                    ok
+            end;
+
+        "ok" -> ok
+    end.
+
 smp_select_wait([], _) ->
     ok;
 smp_select_wait(Pids, TimeoutMsg) ->
diff --git a/erts/emulator/test/driver_SUITE_data/chkio_drv.c b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
index ee8f28e8b1..baf0ca9f11 100644
--- a/erts/emulator/test/driver_SUITE_data/chkio_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
@@ -90,7 +90,7 @@ typedef struct chkio_smp_select {
     int next_read;
     int next_write;
     int first_write;
-    enum {Closed, Opened, Selected, Waiting} state;
+    enum {Closed, Opened, Selected, Waiting, WaitingDone} state;
     int wasSelected;
     unsigned rand_state;
 }ChkioSmpSelect;
@@ -292,18 +292,20 @@ stop_steal_aux(ChkioDrvData *cddp)
 static void free_smp_select(ChkioSmpSelect* pip, ErlDrvPort port)
 {	
     switch (pip->state) {
+    case WaitingDone:
     case Waiting: {
 	int word;
-	fprintf(stderr, "Closing pipe in state Waiting. Event lost?\n");
+	fprintf(stderr, "Closing pipe in state Waiting*. Event lost?\r\n");
 	for (;;) {
 	    int bytes = read(pip->read_fd, &word, sizeof(word));
 	    if (bytes != sizeof(word)) {
 		if (bytes != 0) {
-		    fprintf(stderr, "Failed to read from pipe, bytes=%d, errno=%d\n", bytes, errno);
+		    fprintf(stderr, "Failed to read from pipe, bytes=%d, errno=%d\r\n",
+                            bytes, errno);
 		}
 		break;
 	    }
-	    fprintf(stderr, "Read from pipe: %d\n", word);
+	    fprintf(stderr, "Read from pipe: %d\r\n", word);
 	}
 	abort();
     }
@@ -318,6 +320,8 @@ static void free_smp_select(ChkioSmpSelect* pip, ErlDrvPort port)
 	close(pip->write_fd);
 	pip->state = Closed;
 	break;
+    case Closed:
+        break;
     }
     driver_free(pip);
 }
@@ -526,7 +530,7 @@ chkio_drv_ready_input(ErlDrvData drv_data, ErlDrvEvent event)
 	    printf("Read event on uninitiated pipe %d\n", fd);
 	    abort(); 
 	}
-	if (pip->state != Selected && pip->state != Waiting) { 
+	if (pip->state != Selected && pip->state != Waiting && pip->state != WaitingDone) {
 	    printf("Read event on pipe in strange state %d\n", pip->state);
 	    abort(); 
 	}
@@ -536,9 +540,9 @@ chkio_drv_ready_input(ErlDrvData drv_data, ErlDrvEvent event)
 	inPipe = (pip->next_write - pip->next_read);
 	if (inPipe == 0) {
 	    bytes = read(pip->read_fd, &word, sizeof(word));
-	    printf("Unexpected empty pipe, expected %u -> %u, bytes=%d, word=%d, written=%d\n",
-		   pip->next_read, pip->next_write-1, bytes, word,
-		   (pip->next_write - pip->first_write));
+	    printf("Unexpected empty pipe: ptr=%p, fds=%d->%d, read bytes=%d, word=%d, written=%d\n",
+                   pip, pip->write_fd, pip->read_fd,
+		   bytes, word, (pip->next_write - pip->first_write));
 	    /*abort();
     	      Allow unexpected events as it's been seen to be triggered by epoll
 	      on Linux. Most of the time the unwanted events are filtered by
@@ -564,7 +568,20 @@ chkio_drv_ready_input(ErlDrvData drv_data, ErlDrvEvent event)
 	    TRACEF(("Read %d from fd=%d\n", word, fd));
 	    pip->next_read++;
 	}
-	pip->state = Selected; /* not Waiting anymore */
+        if (pip->state == WaitingDone) {
+            if (pip->next_write == pip->next_read) {
+                /* All data read, send {Port, done} */
+                ErlDrvTermData spec[] = {ERL_DRV_PORT, driver_mk_port(cddp->port),
+                                         ERL_DRV_ATOM, driver_mk_atom("done"),
+                                         ERL_DRV_TUPLE, 2};
+                erl_drv_output_term(driver_mk_port(cddp->port),
+                                    spec, sizeof(spec) / sizeof(spec[0]));
+                pip->state = Selected;
+            }
+        }
+        else {
+            pip->state = Selected; /* not Waiting anymore */
+        }
 	break;
     }
     case CHKIO_DRV_USE:
@@ -962,6 +979,16 @@ chkio_drv_control(ErlDrvData drv_data,
     }
     case CHKIO_SMP_SELECT: {
 	ChkioSmpSelect* pip = (ChkioSmpSelect*) cddp->test_data;
+        if (len == 4 && memcmp(buf, "done", 4) == 0) {
+            if (pip && pip->state == Waiting) {
+                pip->state = WaitingDone;
+                res_str = "wait";
+            }
+            else
+                res_str = "ok";
+            res_len = -1;
+            break;
+        }
 	if (pip == NULL) {
 	    erl_drv_mutex_lock(smp_pipes_mtx);
 	    if (smp_pipes) {
-- 
cgit v1.2.3


From f504781a136b1992a1cee26a7da841892d796d91 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 21 Nov 2018 15:54:17 +0100
Subject: erts: Add counters:put/3

---
 erts/emulator/beam/bif.tab            |  1 +
 erts/emulator/beam/erl_bif_counters.c | 50 ++++++++++++++++++---
 erts/emulator/test/counters_SUITE.erl | 81 +++++++++++++++++++++++++++++++----
 3 files changed, 117 insertions(+), 15 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index aa3c3acd9f..d4ba90a61a 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -723,4 +723,5 @@ bif atomics:info/1
 bif erts_internal:counters_new/1
 bif erts_internal:counters_get/2
 bif erts_internal:counters_add/3
+bif erts_internal:counters_put/3
 bif erts_internal:counters_info/1
diff --git a/erts/emulator/beam/erl_bif_counters.c b/erts/emulator/beam/erl_bif_counters.c
index a46b462225..001e643857 100644
--- a/erts/emulator/beam/erl_bif_counters.c
+++ b/erts/emulator/beam/erl_bif_counters.c
@@ -19,7 +19,7 @@
  */
 
 /*
- * Purpose:  High performance atomics.
+ * Purpose:  The implementation for 'counters' with 'write_concurrency'.
  */
 
 #ifdef HAVE_CONFIG_H
@@ -37,6 +37,15 @@
 #include "erl_bif_unique.h"
 #include "erl_map.h"
 
+/*
+ * Each logical counter consists of one 64-bit atomic instance per scheduler
+ * plus one instance for the "base value".
+ *
+ * get() reads all atomics for the counter and return the sum.
+ * add() reads and writes only its own scheduler specific atomic instance.
+ * put() reads all scheduler specific atomics and writes a new base value.
+ */
+#define ATOMICS_PER_COUNTER (erts_no_schedulers + 1)
 
 #define COUNTERS_PER_CACHE_LINE (ERTS_CACHE_LINE_SIZE / sizeof(erts_atomic64_t))
 
@@ -52,7 +61,7 @@ typedef struct
     } u[1];
 }CountersRef;
 
-static int counters_destructor(Binary *unused)
+static int counters_destructor(Binary *mbin)
 {
     return 1;
 }
@@ -76,10 +85,10 @@ BIF_RETTYPE erts_internal_counters_new_1(BIF_ALIST_1)
         BIF_ERROR(BIF_P, BADARG);
     }
 
-    if (cnt > (ERTS_UWORD_MAX / (sizeof(erts_atomic64_t)*2*erts_no_schedulers)))
+    if (cnt > (ERTS_UWORD_MAX / (sizeof(erts_atomic64_t)*2*ATOMICS_PER_COUNTER)))
         BIF_ERROR(BIF_P, SYSTEM_LIMIT);
 
-    cache_lines = erts_no_schedulers * div_ceil(cnt, COUNTERS_PER_CACHE_LINE);
+    cache_lines = ATOMICS_PER_COUNTER * div_ceil(cnt, COUNTERS_PER_CACHE_LINE);
     bytes = offsetof(CountersRef, u) + cache_lines * ERTS_CACHE_LINE_SIZE;
     mbin = erts_create_magic_binary_x(bytes,
                                       counters_destructor,
@@ -87,6 +96,7 @@ BIF_RETTYPE erts_internal_counters_new_1(BIF_ALIST_1)
                                       0);
     p = ERTS_MAGIC_BIN_DATA(mbin);
     p->arity = cnt;
+
 #ifdef DEBUG
     p->ulen = cache_lines;
 #endif
@@ -120,7 +130,7 @@ static ERTS_INLINE int get_ref_cnt(Eterm ref, Eterm index,
     UWord ix, ui, vi;
     if (!get_ref(ref, &p) || !term_to_UWord(index, &ix) || --ix >= p->arity)
         return 0;
-    ui = (ix / COUNTERS_PER_CACHE_LINE) * erts_no_schedulers + sched_ix;
+    ui = (ix / COUNTERS_PER_CACHE_LINE) * ATOMICS_PER_COUNTER;
     vi = ix % COUNTERS_PER_CACHE_LINE;
     ASSERT(ui < p->ulen);
     *pp = p;
@@ -134,7 +144,8 @@ static ERTS_INLINE int get_ref_my_cnt(Eterm ref, Eterm index,
 {
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
     ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp));
-    return get_ref_cnt(ref, index, pp, app, esdp->no - 1);
+    ASSERT(esdp->no > 0 && esdp->no < ATOMICS_PER_COUNTER);
+    return get_ref_cnt(ref, index, pp, app, esdp->no);
 }
 
 static ERTS_INLINE int get_ref_first_cnt(Eterm ref, Eterm index,
@@ -172,7 +183,7 @@ BIF_RETTYPE erts_internal_counters_get_2(BIF_ALIST_2)
     if (!get_ref_first_cnt(BIF_ARG_1, BIF_ARG_2, &p, &ap)) {
         BIF_ERROR(BIF_P, BADARG);
     }
-    for (j = erts_no_schedulers; j ; --j) {
+    for (j = ATOMICS_PER_COUNTER; j ; --j) {
         acc += erts_atomic64_read_nob(ap);
         ap = (erts_atomic64_t*) ((byte*)ap + ERTS_CACHE_LINE_SIZE);
     }
@@ -194,6 +205,31 @@ BIF_RETTYPE erts_internal_counters_add_3(BIF_ALIST_3)
     return am_ok;
 }
 
+BIF_RETTYPE erts_internal_counters_put_3(BIF_ALIST_3)
+{
+    CountersRef* p;
+    erts_atomic64_t* first_ap;
+    erts_atomic64_t* ap;
+    erts_aint64_t acc;
+    erts_aint64_t val;
+    int j;
+
+    if (!get_ref_first_cnt(BIF_ARG_1, BIF_ARG_2, &p, &first_ap)
+        || !term_to_Sint64(BIF_ARG_3, &val)) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+
+    ap = first_ap;
+    acc = 0;
+    j = ATOMICS_PER_COUNTER - 1;
+    do {
+        ap = (erts_atomic64_t*) ((byte*)ap + ERTS_CACHE_LINE_SIZE);
+        acc += erts_atomic64_read_nob(ap);
+    } while (--j);
+    erts_atomic64_set_nob(first_ap, val-acc);
+
+    return am_ok;
+}
 
 BIF_RETTYPE erts_internal_counters_info_1(BIF_ALIST_1)
 {
diff --git a/erts/emulator/test/counters_SUITE.erl b/erts/emulator/test/counters_SUITE.erl
index 7de164096b..8cc963e3b9 100644
--- a/erts/emulator/test/counters_SUITE.erl
+++ b/erts/emulator/test/counters_SUITE.erl
@@ -21,12 +21,13 @@
 
 -include_lib("common_test/include/ct.hrl").
 
--compile(export_all).
+-export([suite/0, all/0]).
+-export([basic/1, bad/1, limits/1, indep/1]).
 
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
 all() ->
-    [basic, bad, limits].
+    [basic, bad, limits, indep].
 
 basic(Config) when is_list(Config) ->
     Size = 10,
@@ -53,15 +54,21 @@ basic_do(Ref, Ix) ->
     77 = counters:get(Ref, Ix),
     ok = counters:sub(Ref, Ix, -10),
     87 = counters:get(Ref, Ix),
+    ok = counters:put(Ref, Ix, 0),
+    0 = counters:get(Ref, Ix),
+    ok = counters:put(Ref, Ix, 123),
+    123 = counters:get(Ref, Ix),
+    ok = counters:put(Ref, Ix, -321),
+    -321 = counters:get(Ref, Ix),
     ok.
 
 check_memory(atomics, Memory, Size) ->
     {_,true} = {Memory, Memory > Size*8},
     {_,true} = {Memory, Memory < Size*max_atomic_sz() + 100};
 check_memory(write_concurrency, Memory, Size) ->
-    NScheds = erlang:system_info(schedulers),
-    {_,true} = {Memory, Memory > NScheds*Size*8},
-    {_,true} = {Memory, Memory < NScheds*(Size+7)*max_atomic_sz() + 100}.
+    NWords = erlang:system_info(schedulers) + 1,
+    {_,true} = {Memory, Memory > NWords*Size*8},
+    {_,true} = {Memory, Memory < NWords*(Size+7)*max_atomic_sz() + 100}.
 
 max_atomic_sz() ->
     case erlang:system_info({wordsize, external}) of
@@ -90,23 +97,81 @@ bad(Config) when is_list(Config) ->
 
 
 limits(Config) when is_list(Config) ->
+    limits_do(counters:new(1,[atomics])),
+    limits_do(counters:new(1,[write_concurrency])),
+    ok.
+
+limits_do(Ref) ->
     Bits = 64,
     Max = (1 bsl (Bits-1)) - 1,
     Min = -(1 bsl (Bits-1)),
 
-    Ref = counters:new(1,[]),
     0 = counters:get(Ref, 1),
-    ok = counters:add(Ref, 1, Max),
+    ok = counters:put(Ref, 1, Max),
+    Max = counters:get(Ref, 1),
     ok = counters:add(Ref, 1, 1),
     Min = counters:get(Ref, 1),
     ok  = counters:sub(Ref, 1, 1),
     Max = counters:get(Ref, 1),
+    ok = counters:put(Ref, 1, Min),
+    Min = counters:get(Ref, 1),
 
     IncrMax = (Max bsl 1) bor 1,
-    ok = counters:sub(Ref, 1, counters:get(Ref, 1)),
+    ok = counters:put(Ref, 1, 0),
     ok = counters:add(Ref, 1, IncrMax),
     -1 = counters:get(Ref, 1),
     {'EXIT',{badarg,_}} = (catch counters:add(Ref, 1, IncrMax+1)),
     {'EXIT',{badarg,_}} = (catch counters:add(Ref, 1, Min-1)),
+    {'EXIT',{badarg,_}} = (catch counters:put(Ref, 1, Max+1)),
+    {'EXIT',{badarg,_}} = (catch counters:add(Ref, 1, Min-1)),
+    ok.
+
 
+%% Verify that independent workers, using different counters
+%% within the same array, do not interfere with each other.
+indep(Config) when is_list(Config) ->
+    NScheds = erlang:system_info(schedulers),
+    Ref = counters:new(NScheds,[write_concurrency]),
+    Rounds = 100,
+    Papa = self(),
+    Pids = [spawn_opt(fun () ->
+                               Val = I*197,
+                               counters:put(Ref, I, Val),
+                               indep_looper(Rounds, Ref, I, Val),
+                               Papa ! {self(), done}
+                       end,
+                      [link, {scheduler, I}])
+            || I <- lists:seq(1, NScheds)],
+    [receive {P,done} -> ok end || P <- Pids],
     ok.
+
+indep_looper(0, _, _ , _) ->
+    ok;
+indep_looper(N, Ref, I, Val0) ->
+    %%io:format("Val0 = ~p\n", [Val0]),
+    Val0 = counters:get(Ref, I),
+    Val1 = indep_adder(Ref, I, Val0),
+    indep_subber(Ref, I, Val1),
+    Val2 = N*7 + I,
+    counters:put(Ref, I, Val2),
+    indep_looper(N-1, Ref, I, Val2).
+
+indep_adder(Ref, I, Val) when Val < (1 bsl 62) ->
+    %%io:format("adder Val = ~p\n", [Val]),
+    Incr = abs(Val div 2) + I + 984735,
+    counters:add(Ref, I, Incr),
+    Res = Val + Incr,
+    Res = counters:get(Ref, I),
+    indep_adder(Ref, I, Res);
+indep_adder(_Ref, _I, Val) ->
+    Val.
+
+indep_subber(Ref, I, Val) when Val > -(1 bsl 62) ->
+    %%io:format("subber Val = ~p\n", [Val]),
+    Decr = (abs(Val div 2) + I + 725634),
+    counters:sub(Ref, I, Decr),
+    Res = Val - Decr,
+    Res = counters:get(Ref, I),
+    indep_subber(Ref, I, Res);
+indep_subber(_Ref, _I, Val) ->
+    Val.
-- 
cgit v1.2.3


From ff913d619372024c74c93640c60032037a4a6d2e Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 22 Nov 2018 15:44:10 +0100
Subject: erts: Fix faulty assert in driver_SUITE

Set drv_use_singleton.fd_stop_select to -2 in start()
as it's only used by driver_SUITE:driver_select_use.
---
 erts/emulator/test/driver_SUITE_data/chkio_drv.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/test/driver_SUITE_data/chkio_drv.c b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
index baf0ca9f11..b9ee155b4b 100644
--- a/erts/emulator/test/driver_SUITE_data/chkio_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
@@ -387,6 +387,9 @@ chkio_drv_start(ErlDrvPort port, char *command)
     cddp->id = driver_mk_port(port);
     cddp->test = CHKIO_STOP;
     cddp->test_data = NULL;
+
+    drv_use_singleton.fd_stop_select = -2; /* disable stop_select asserts */
+
     return (ErlDrvData) cddp;
 #endif
 }
@@ -1041,7 +1044,6 @@ chkio_drv_control(ErlDrvData drv_data,
 		if (pip->wasSelected && (op & 1)) {
 		    TRACEF(("%T: Close pipe [%d->%d]\n", cddp->id, pip->write_fd,
                             pip->read_fd));
-                    drv_use_singleton.fd_stop_select = -2; /* disable stop_select asserts */
 		    if (driver_select(cddp->port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd,
                                       DO_READ|ERL_DRV_USE, 0)
                         || close(pip->write_fd)) {
-- 
cgit v1.2.3


From 3d4ecf2f722dfb952bba57daaaef617500133fb9 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 22 Nov 2018 16:15:31 +0100
Subject: erts: Refactor erl_bif_counters.c

with more consistent naming.
---
 erts/emulator/beam/erl_bif_counters.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_counters.c b/erts/emulator/beam/erl_bif_counters.c
index 001e643857..84496e83ef 100644
--- a/erts/emulator/beam/erl_bif_counters.c
+++ b/erts/emulator/beam/erl_bif_counters.c
@@ -41,13 +41,13 @@
  * Each logical counter consists of one 64-bit atomic instance per scheduler
  * plus one instance for the "base value".
  *
- * get() reads all atomics for the counter and return the sum.
+ * get() reads all atomics for the counter and returns the sum.
  * add() reads and writes only its own scheduler specific atomic instance.
  * put() reads all scheduler specific atomics and writes a new base value.
  */
 #define ATOMICS_PER_COUNTER (erts_no_schedulers + 1)
 
-#define COUNTERS_PER_CACHE_LINE (ERTS_CACHE_LINE_SIZE / sizeof(erts_atomic64_t))
+#define ATOMICS_PER_CACHE_LINE (ERTS_CACHE_LINE_SIZE / sizeof(erts_atomic64_t))
 
 typedef struct
 {
@@ -56,7 +56,7 @@ typedef struct
     UWord ulen;
 #endif
     union {
-        erts_atomic64_t v[COUNTERS_PER_CACHE_LINE];
+        erts_atomic64_t v[ATOMICS_PER_CACHE_LINE];
         byte cache_line__[ERTS_CACHE_LINE_SIZE];
     } u[1];
 }CountersRef;
@@ -88,7 +88,7 @@ BIF_RETTYPE erts_internal_counters_new_1(BIF_ALIST_1)
     if (cnt > (ERTS_UWORD_MAX / (sizeof(erts_atomic64_t)*2*ATOMICS_PER_COUNTER)))
         BIF_ERROR(BIF_P, SYSTEM_LIMIT);
 
-    cache_lines = ATOMICS_PER_COUNTER * div_ceil(cnt, COUNTERS_PER_CACHE_LINE);
+    cache_lines = ATOMICS_PER_COUNTER * div_ceil(cnt, ATOMICS_PER_CACHE_LINE);
     bytes = offsetof(CountersRef, u) + cache_lines * ERTS_CACHE_LINE_SIZE;
     mbin = erts_create_magic_binary_x(bytes,
                                       counters_destructor,
@@ -102,7 +102,7 @@ BIF_RETTYPE erts_internal_counters_new_1(BIF_ALIST_1)
 #endif
     ASSERT((byte*)&p->u[cache_lines] <= ((byte*)p + bytes));
     for (ui=0; ui < cache_lines; ui++)
-        for (vi=0; vi < COUNTERS_PER_CACHE_LINE; vi++)
+        for (vi=0; vi < ATOMICS_PER_CACHE_LINE; vi++)
             erts_atomic64_init_nob(&p->u[ui].v[vi], 0);
     hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
     return erts_mk_magic_ref(&hp, &MSO(BIF_P), mbin);
@@ -130,8 +130,8 @@ static ERTS_INLINE int get_ref_cnt(Eterm ref, Eterm index,
     UWord ix, ui, vi;
     if (!get_ref(ref, &p) || !term_to_UWord(index, &ix) || --ix >= p->arity)
         return 0;
-    ui = (ix / COUNTERS_PER_CACHE_LINE) * ATOMICS_PER_COUNTER;
-    vi = ix % COUNTERS_PER_CACHE_LINE;
+    ui = (ix / ATOMICS_PER_CACHE_LINE) * ATOMICS_PER_COUNTER;
+    vi = ix % ATOMICS_PER_CACHE_LINE;
     ASSERT(ui < p->ulen);
     *pp = p;
     *app = &p->u[ui].v[vi];
-- 
cgit v1.2.3


From ceeb321ca0e86a436c87eab2c38eae2380f78d1a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 23 Nov 2018 17:36:59 +0100
Subject: erts: Fix bug in counters

Wow, that could have been embarrassing.
---
 erts/emulator/beam/erl_bif_counters.c |  2 +-
 erts/emulator/test/counters_SUITE.erl | 61 +++++++++++++++++++++++++++++++++--
 2 files changed, 60 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_counters.c b/erts/emulator/beam/erl_bif_counters.c
index 84496e83ef..7c8884ba32 100644
--- a/erts/emulator/beam/erl_bif_counters.c
+++ b/erts/emulator/beam/erl_bif_counters.c
@@ -130,7 +130,7 @@ static ERTS_INLINE int get_ref_cnt(Eterm ref, Eterm index,
     UWord ix, ui, vi;
     if (!get_ref(ref, &p) || !term_to_UWord(index, &ix) || --ix >= p->arity)
         return 0;
-    ui = (ix / ATOMICS_PER_CACHE_LINE) * ATOMICS_PER_COUNTER;
+    ui = (ix / ATOMICS_PER_CACHE_LINE) * ATOMICS_PER_COUNTER + sched_ix;
     vi = ix % ATOMICS_PER_CACHE_LINE;
     ASSERT(ui < p->ulen);
     *pp = p;
diff --git a/erts/emulator/test/counters_SUITE.erl b/erts/emulator/test/counters_SUITE.erl
index 8cc963e3b9..b3f0358c1e 100644
--- a/erts/emulator/test/counters_SUITE.erl
+++ b/erts/emulator/test/counters_SUITE.erl
@@ -22,12 +22,12 @@
 -include_lib("common_test/include/ct.hrl").
 
 -export([suite/0, all/0]).
--export([basic/1, bad/1, limits/1, indep/1]).
+-export([basic/1, bad/1, limits/1, indep/1, write_concurrency/1]).
 
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
 all() ->
-    [basic, bad, limits, indep].
+    [basic, bad, limits, indep, write_concurrency].
 
 basic(Config) when is_list(Config) ->
     Size = 10,
@@ -175,3 +175,60 @@ indep_subber(Ref, I, Val) when Val > -(1 bsl 62) ->
     indep_subber(Ref, I, Res);
 indep_subber(_Ref, _I, Val) ->
     Val.
+
+
+
+%% Verify write_concurrency yields correct results.
+write_concurrency(Config) when is_list(Config) ->
+    rand:seed(exs1024s),
+    io:format("*** SEED: ~p ***\n", [rand:export_seed()]),
+    NScheds = erlang:system_info(schedulers),
+    Size = 100,
+    Ref = counters:new(Size,[write_concurrency]),
+    Rounds = 1000,
+    Papa = self(),
+    Pids = [spawn_opt(fun Worker() ->
+                              receive
+                                  {go, Ix, Incr} ->
+                                      wc_looper(Rounds, Ref, Ix, Incr),
+                                      Papa ! {self(), done, Rounds*Incr},
+                                      Worker();
+                                  stop ->
+                                      ok
+                              end
+                       end,
+                      [link, {scheduler, N}])
+            || N <- lists:seq(1, NScheds)],
+    [begin
+         Base = rand_log64(),
+         counters:put(Ref, Index, Base),
+         SendList = [{P,{go, Index, rand_log64()}} || P <- Pids],
+         [P ! Msg || {P,Msg} <- SendList],
+         Added = lists:sum([receive {P,done,Contrib} -> Contrib end || P <- Pids]),
+         Result = mask_sint64(Base+Added),
+         {_,Result} = {Result, counters:get(Ref, Index)}
+     end
+     || Index <- lists:seq(1, Size)],
+
+    [begin unlink(P), P ! stop end || P <- Pids],
+    ok.
+
+wc_looper(0, _, _, _) ->
+    ok;
+wc_looper(N, Ref, Ix, Incr) ->
+    counters:add(Ref, Ix, Incr),
+    wc_looper(N-1, Ref, Ix, Incr).
+
+mask_sint64(X) ->
+    SMask = 1 bsl 63,
+    UMask = SMask - 1,
+    (X band UMask) - (X band SMask).
+
+%% A random signed 64-bit integer
+%% with a uniformly distributed number of significant bits.
+rand_log64() ->
+    Uint = round(math:pow(2, rand:uniform()*63)),
+    case rand:uniform(2) of
+        1 -> -Uint;
+        2 -> Uint
+    end.
-- 
cgit v1.2.3


From 63077f5bb3011ec9a14aa9b9e39ed31e4525078b Mon Sep 17 00:00:00 2001
From: Maxim Fedorov <dane@whatsapp.com>
Date: Mon, 19 Nov 2018 14:15:43 -0800
Subject: erts: fix attempt to start timer when executing on dirty scheduler

Since OTP R20, there is a possibility for MAJOR garbage collection to
run on dirty scheduler. So DistEntry destructor is being called on
dirty scheduler as well. This, in turn, leads to an attempt to schedule
timer on a dirty scheduler too, which is impossible (and will assert
on debug build, but does succeed for release build, creating an
infinite busy loop, since aux work wakes scheduler up, but dirty
scheduler cannot execute aus work).
There is a similar method in erl_hl_timer, see erts_start_timer_callback.
---
 erts/emulator/beam/erl_node_tables.c | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index f4dc60941a..18ed782ae3 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -421,8 +421,25 @@ static void schedule_delete_dist_entry(DistEntry* dep)
      *
      * Note that timeouts do not guarantee thread progress.
      */
-    erts_schedule_thr_prgr_later_op(start_timer_delete_dist_entry,
-                                    dep, &dep->later_op);
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+        erts_schedule_thr_prgr_later_op(start_timer_delete_dist_entry,
+                                        dep, &dep->later_op);
+    } else {
+        /*
+         * Since OTP 20, it's possible that destructor is executed on
+         *  a dirty scheduler. Aux work cannot be done on a dirty
+         *  scheduler, and scheduling any aux work on a dirty scheduler
+         *  makes the scheduler to loop infinitely.
+         * To avoid this, make a spot jump: schedule this function again
+         *  on a first normal scheduler. It is guaranteed to be always
+         *  online. Since it's a rare event, this shall not pose a big
+         *  utilisation hit.
+         */
+        erts_schedule_misc_aux_work(1,
+                                    (void (*)(void *))schedule_delete_dist_entry,
+                                    (void *) dep);
+    }
 }
 
 static void
-- 
cgit v1.2.3


From 0ad63bcc5523f0b560c843b3ed02c08e1369e522 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 27 Nov 2018 21:16:47 +0100
Subject: erts: Fix bug in sendfile for active socket

driver_select() was called after port had been killed
by tcp_inet_sendfile() calling tcp_send_error().
---
 erts/emulator/drivers/common/inet_drv.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 259a27cf57..d8491ba18b 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -10204,12 +10204,11 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd,
         desc->tcp_add_flags |= TCP_ADDF_SENDFILE;
 
         /* See if we can finish sending without selecting & rescheduling. */
-        tcp_inet_sendfile(desc);
-
-        if(desc->sendfile.length > 0) {
-            sock_select(INETP(desc), FD_WRITE, 1);
+        if (tcp_inet_sendfile(desc) == 0) {
+            if(desc->sendfile.length > 0) {
+                sock_select(INETP(desc), FD_WRITE, 1);
+            }
         }
-
         return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
 #else
         return ctl_error(ENOTSUP, rbuf, rsize);
@@ -11692,8 +11691,8 @@ socket_error: {
         DEBUGF(("tcp_inet_sendfile(%ld): send errno = %d (errno %d)\r\n",
             (long)desc->inet.port, socket_errno, errno));
 
-        result = tcp_send_error(desc, socket_errno);
         tcp_sendfile_aborted(desc, socket_errno);
+        result = tcp_send_error(desc, socket_errno);
 
         goto done;
     }
-- 
cgit v1.2.3


From 06c46662022efb6892c036eeaf8c957e51fc17f7 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 28 Nov 2018 20:31:37 +0100
Subject: erts: Fix unexpected inet_reply message from failing file:sendfile

A failing file:sendfile call would often send a message
{inet_reply, Port, {error, Reason}} that would pollute the mailbox
of the calling process.

TCP_REQ_SENDFILE has its own reply messages format
{sendfile, _, _} and does not expect an inet_reply message.

Solution: Suppress inet_reply error message if TCP_ADDF_SENDFILE
is set.
---
 erts/emulator/drivers/common/inet_drv.c | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index d8491ba18b..681909d9aa 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -11150,27 +11150,31 @@ static int tcp_send_or_shutdown_error(tcp_descriptor* desc, int err)
     DEBUGF(("driver_failure_eof(%ld) in %s, line %d\r\n",
 	    (long)desc->inet.port, __FILE__, __LINE__));
     if (desc->inet.active) {
+        ErlDrvTermData err_atom;
 	if (show_econnreset) {
 	    tcp_error_message(desc, err);
-	    tcp_closed_message(desc);
-	    inet_reply_error(INETP(desc), err);
+            err_atom = error_atom(err);
 	} else {
-	    tcp_closed_message(desc);
-	    inet_reply_error_am(INETP(desc), am_closed);
+            err_atom = am_closed;
 	}
+        tcp_closed_message(desc);
+        if (!(desc->tcp_add_flags & TCP_ADDF_SENDFILE))
+            inet_reply_error_am(INETP(desc), err_atom);
+
 	if (desc->inet.exitf)
 	    driver_exit(desc->inet.port, 0);
 	else
 	    tcp_desc_close(desc);
     } else {
 	tcp_close_check(desc);
-	tcp_desc_close(desc);
 
 	if (desc->inet.caller) {
-	    if (show_econnreset)
-		inet_reply_error(INETP(desc), err);
-	    else
-		inet_reply_error_am(INETP(desc), am_closed);
+            if (!(desc->tcp_add_flags & TCP_ADDF_SENDFILE)) {
+                if (show_econnreset)
+                    inet_reply_error(INETP(desc), err);
+                else
+                    inet_reply_error_am(INETP(desc), am_closed);
+            }
 	}
 	else {
 	    /* No blocking send op to reply to right now.
@@ -11179,6 +11183,7 @@ static int tcp_send_or_shutdown_error(tcp_descriptor* desc, int err)
 	     */
 	    desc->tcp_add_flags |= TCP_ADDF_DELAYED_CLOSE_SEND;
 	}
+        tcp_desc_close(desc);
 
 	/*
 	 * Make sure that the next receive operation gets an {error,closed}
-- 
cgit v1.2.3


From 9ed2fb25bde3994be0daf67b47f80bf2f457c96d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 30 Nov 2018 19:07:12 +0100
Subject: erts: Fix hanging sendfile bugs when socket closes unexpectedly

---
 erts/emulator/drivers/common/inet_drv.c | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 681909d9aa..34b13c3370 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -1313,7 +1313,10 @@ static int tcp_deliver(tcp_descriptor* desc, int len);
 
 static int tcp_shutdown_error(tcp_descriptor* desc, int err);
 
+#ifdef HAVE_SENDFILE
 static int tcp_inet_sendfile(tcp_descriptor* desc);
+static int tcp_sendfile_aborted(tcp_descriptor* desc, int socket_error);
+#endif
 
 static int tcp_inet_output(tcp_descriptor* desc, HANDLE event);
 static int tcp_inet_input(tcp_descriptor* desc, HANDLE event);
@@ -10421,6 +10424,7 @@ static int tcp_recv_closed(tcp_descriptor* desc)
 #ifdef DEBUG
     long port = (long) desc->inet.port; /* Used after driver_exit() */
 #endif
+    int blocking_send = 0;
     DEBUGF(("tcp_recv_closed(%ld): s=%d, in %s, line %d\r\n",
 	    port, desc->inet.s, __FILE__, __LINE__));
     if (IS_BUSY(INETP(desc))) {
@@ -10436,7 +10440,15 @@ static int tcp_recv_closed(tcp_descriptor* desc)
 	set_busy_port(desc->inet.port, 0);
 	inet_reply_error_am(INETP(desc), am_closed);
 	DEBUGF(("tcp_recv_closed(%ld): busy reply 'closed'\r\n", port));
-    } else {
+        blocking_send = 1;
+    }
+#ifdef HAVE_SENDFILE
+    if (desc->tcp_add_flags & TCP_ADDF_SENDFILE) {
+        tcp_sendfile_aborted(desc, ENOTCONN);
+        blocking_send = 1;
+    }
+#endif
+    if (!blocking_send) {
         /* No blocking send op to reply to right now.
          * If next op is a send, make sure it returns {error,closed}
          * rather than {error,enotconn}.
@@ -10487,6 +10499,11 @@ static int tcp_recv_error(tcp_descriptor* desc, int err)
 	    set_busy_port(desc->inet.port, 0);
 	    inet_reply_error_am(INETP(desc), am_closed);
 	}
+#ifdef HAVE_SENDFILE
+        if (desc->tcp_add_flags & TCP_ADDF_SENDFILE) {
+            tcp_sendfile_aborted(desc, err);
+        }
+#endif
 	if (!desc->inet.active) {
 	    /* We must cancel any timer here ! */
 	    driver_cancel_timer(desc->inet.port);
-- 
cgit v1.2.3


From 33324fd140998f36698145d2eea7e8722c044740 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 10 Oct 2018 10:42:11 +0200
Subject: erts: Add pre-alloc to ALLOC msacc state

OTP-15450
---
 erts/emulator/beam/erl_sched_spec_pre_alloc.h | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.h b/erts/emulator/beam/erl_sched_spec_pre_alloc.h
index b119c59ab3..74cc966cbe 100644
--- a/erts/emulator/beam/erl_sched_spec_pre_alloc.h
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.h
@@ -188,6 +188,7 @@ erts_sspa_alloc(erts_sspa_data_t *data, int cix)
     erts_sspa_chunk_t *chnk;
     erts_sspa_chunk_header_t *chdr;
     erts_sspa_blk_t *res;
+    ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_ALLOC);
 
     chnk = erts_sspa_cix2chunk(data, cix);
     chdr = &chnk->aligned.header;
@@ -201,11 +202,15 @@ erts_sspa_alloc(erts_sspa_data_t *data, int cix)
 	    chdr->local.last = NULL;
 	ERTS_SSPA_DBG_CHK_LCL(chdr);
     }
-    if (chdr->local.cnt <= chdr->local.lim)
-	return (char *) erts_sspa_process_remote_frees(chdr, res);
+    if (chdr->local.cnt <= chdr->local.lim) {
+	res = erts_sspa_process_remote_frees(chdr, res);
+        ERTS_MSACC_POP_STATE_M_X();
+        return (char*) res;
+    }
     else if (chdr->head.no_thr_progress_check < ERTS_SSPA_FORCE_THR_CHECK_PROGRESS)
 	chdr->head.no_thr_progress_check++;
     ASSERT(res);
+    ERTS_MSACC_POP_STATE_M_X();
     return (char *) res;
 }
 
-- 
cgit v1.2.3


From 19e7675913a0c244231344d4d40db447a0bb7ef1 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 12 Oct 2018 10:17:18 +0200
Subject: erts: Add erts_io_notify_port_task_executed to check_io msacc state

OTP-15450
---
 erts/emulator/sys/common/erl_check_io.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'erts/emulator')

diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 748555165f..1444cee805 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -431,6 +431,8 @@ erts_io_notify_port_task_executed(ErtsPortTaskType type,
     ErtsDrvSelectDataState *free_select = NULL;
     ErtsNifSelectDataState *free_nif = NULL;
 
+    ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_CHECK_IO);
+
     erts_mtx_lock(mtx);
     state = get_drv_ev_state(fd);
 
@@ -486,6 +488,8 @@ erts_io_notify_port_task_executed(ErtsPortTaskType type,
         free_drv_select_data(free_select);
     if (free_nif)
         free_nif_select_data(free_nif);
+
+    ERTS_MSACC_POP_STATE_M_X();
 }
 
 static ERTS_INLINE void
-- 
cgit v1.2.3


From f1579990145efb24ea8f2f9a6744c8912561a922 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 12 Oct 2018 10:08:28 +0200
Subject: erts: Move all inet tcp CONNECTED timers to multi timer

---
 erts/emulator/drivers/common/inet_drv.c | 133 ++++++++++++++++++++------------
 1 file changed, 82 insertions(+), 51 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 259a27cf57..222b369a3f 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -1010,6 +1010,9 @@ static MultiTimerData *add_multi_timer(MultiTimerData **first, ErlDrvPort port,
 static void fire_multi_timers(MultiTimerData **first, ErlDrvPort port,
 			      ErlDrvData data);
 static void remove_multi_timer(MultiTimerData **first, ErlDrvPort port, MultiTimerData *p);
+static void cancel_multi_timer(MultiTimerData **first, ErlDrvPort port,
+                               void (*timeout_fun)(ErlDrvData drv_data,
+                                                   ErlDrvTermData caller));
 
 static void tcp_inet_multi_timeout(ErlDrvData e, ErlDrvTermData caller);
 static void clean_multi_timers(MultiTimerData **first, ErlDrvPort port);
@@ -9779,15 +9782,14 @@ static void tcp_close_check(tcp_descriptor* desc)
 	    driver_demonitor_process(desc->inet.port, &monitor);
 	    send_async_error(desc->inet.dport, id, caller, am_closed);
 	}
-	clean_multi_timers(&(desc->mtd), desc->inet.port);
     }
-
     else if (desc->inet.state == INET_STATE_CONNECTING) {
 	async_error_am(INETP(desc), am_closed);
     }
     else if (desc->inet.state == INET_STATE_CONNECTED) {
 	async_error_am_all(INETP(desc), am_closed);
     }
+    clean_multi_timers(&(desc->mtd), desc->inet.port);
 }
 
 /*
@@ -9830,6 +9832,15 @@ static void tcp_desc_close(tcp_descriptor* desc)
     erl_inet_close(INETP(desc));
 }
 
+static void tcp_inet_recv_timeout(ErlDrvData e, ErlDrvTermData dummy)
+{
+    tcp_descriptor* desc = (tcp_descriptor*)e;
+    ASSERT(!desc->inet.active);
+    sock_select(INETP(desc),(FD_READ|FD_CLOSE),0);
+    desc->i_remain = 0;
+    async_error_am(INETP(desc), am_timeout);
+}
+
 /* TCP requests from Erlang */
 static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd,
 				 char* buf, ErlDrvSizeT len,
@@ -10117,7 +10128,8 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd,
 		async_error_am(INETP(desc), am_timeout);
 	    else {
 		if (timeout != INET_INFINITY)
-		    driver_set_timer(desc->inet.port, timeout);
+                    add_multi_timer(&(desc->mtd), INETP(desc)->port, 0,
+                                    timeout, &tcp_inet_recv_timeout);
 		if (!INETP(desc)->is_ignored)
 		    sock_select(INETP(desc),(FD_READ|FD_CLOSE),1);
 		else
@@ -10223,12 +10235,27 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd,
 
 }
 
+static void tcp_inet_send_timeout(ErlDrvData e, ErlDrvTermData dummy)
+{
+    tcp_descriptor* desc = (tcp_descriptor*)e;
+    ASSERT(IS_BUSY(INETP(desc)));
+    ASSERT(desc->busy_on_send);
+    desc->inet.caller = desc->inet.busy_caller;
+    desc->inet.state &= ~INET_F_BUSY;
+    desc->busy_on_send = 0;
+    set_busy_port(desc->inet.port, 0);
+    inet_reply_error_am(INETP(desc), am_timeout);
+    if (desc->send_timeout_close) {
+        tcp_desc_close(desc);
+    }
+}
+
 /*
 ** tcp_inet_timeout:
 ** called when timer expire:
 ** TCP socket may be:
 **
-** a)  receiving   -- deselect
+** a)  receiving   -- send timeout
 ** b)  connecting  -- close socket
 ** c)  accepting   -- reset listener
 **
@@ -10244,24 +10271,7 @@ static void tcp_inet_timeout(ErlDrvData e)
     if ((state & INET_F_MULTI_CLIENT)) { /* Multi-client always means multi-timers */
 	fire_multi_timers(&(desc->mtd), desc->inet.port, e);
     } else if ((state & INET_STATE_CONNECTED) == INET_STATE_CONNECTED) {
-	if (desc->busy_on_send) {
-	    ASSERT(IS_BUSY(INETP(desc)));
-	    desc->inet.caller = desc->inet.busy_caller;
-	    desc->inet.state &= ~INET_F_BUSY;
-	    desc->busy_on_send = 0;
-	    set_busy_port(desc->inet.port, 0);
-	    inet_reply_error_am(INETP(desc), am_timeout);
-	    if (desc->send_timeout_close) {
-		tcp_desc_close(desc);
-	    }
-	}
-	else {
-	    /* assume recv timeout */
-	    ASSERT(!desc->inet.active);
-	    sock_select(INETP(desc),(FD_READ|FD_CLOSE),0);
-	    desc->i_remain = 0;
-	    async_error_am(INETP(desc), am_timeout);
-	}
+        fire_multi_timers(&(desc->mtd), desc->inet.port, e);
     }
     else if ((state & INET_STATE_CONNECTING) == INET_STATE_CONNECTING) {
 	/* assume connect timeout */
@@ -10429,7 +10439,7 @@ static int tcp_recv_closed(tcp_descriptor* desc)
 	desc->inet.caller = desc->inet.busy_caller;
 	tcp_clear_output(desc);
 	if (desc->busy_on_send) {
-	    driver_cancel_timer(desc->inet.port);
+            cancel_multi_timer(&(desc->mtd), INETP(desc)->port, &tcp_inet_send_timeout);
 	    desc->busy_on_send = 0;
 	    DEBUGF(("tcp_recv_closed(%ld): busy on send\r\n", port));
 	}
@@ -10444,9 +10454,10 @@ static int tcp_recv_closed(tcp_descriptor* desc)
          */
         desc->tcp_add_flags |= TCP_ADDF_DELAYED_CLOSE_SEND;
     }
+
     if (!desc->inet.active) {
-	/* We must cancel any timer here ! */
-	driver_cancel_timer(desc->inet.port);
+        /* We must cancel any timer here ! */
+        clean_multi_timers(&(desc->mtd), INETP(desc)->port);
 	/* passive mode do not terminate port ! */
 	tcp_clear_input(desc);
 	if (desc->inet.exitf) {
@@ -10481,7 +10492,7 @@ static int tcp_recv_error(tcp_descriptor* desc, int err)
 	    desc->inet.caller = desc->inet.busy_caller;
 	    tcp_clear_output(desc);
 	    if (desc->busy_on_send) {
-		driver_cancel_timer(desc->inet.port);
+                cancel_multi_timer(&(desc->mtd), INETP(desc)->port, &tcp_inet_send_timeout);
 		desc->busy_on_send = 0;
 	    }
 	    desc->inet.state &= ~INET_F_BUSY;
@@ -10490,7 +10501,7 @@ static int tcp_recv_error(tcp_descriptor* desc, int err)
 	}
 	if (!desc->inet.active) {
 	    /* We must cancel any timer here ! */
-	    driver_cancel_timer(desc->inet.port);
+            clean_multi_timers(&(desc->mtd), INETP(desc)->port);
 	    tcp_clear_input(desc);
 	    if (desc->inet.exitf) {
 		tcp_desc_close(desc);
@@ -10595,13 +10606,13 @@ static int tcp_deliver(tcp_descriptor* desc, int len)
     if (len == 0) {
 	/* empty buffer or waiting for more input */
 	if ((desc->i_buf == NULL) || (desc->i_remain > 0))
-	    return count;
+	    return 0;
 	if ((n = tcp_remain(desc, &len)) != 0) {
 	    if (n < 0) /* packet error */
 		return n;
 	    if (len > 0)  /* more data pending */
 		desc->i_remain = len;
-	    return count;
+	    return 0;
 	}
     }
 
@@ -10653,9 +10664,7 @@ static int tcp_deliver(tcp_descriptor* desc, int len)
 	len = 0;
 
 	if (!desc->inet.active) {
-	    if (!desc->busy_on_send) {
-		driver_cancel_timer(desc->inet.port);
-	    }
+            cancel_multi_timer(&(desc->mtd), INETP(desc)->port, &tcp_inet_recv_timeout);
 	    sock_select(INETP(desc),(FD_READ|FD_CLOSE),0);
 	    if (desc->i_buf != NULL)
 		tcp_restart_input(desc);
@@ -10681,7 +10690,7 @@ static int tcp_recv(tcp_descriptor* desc, int request_len)
     int len;
     int nread;
 
-    if (desc->i_buf == NULL) {  /* allocte a read buffer */
+    if (desc->i_buf == NULL) {  /* allocate a read buffer */
 	int sz = (request_len > 0) ? request_len : desc->inet.bufsz;
 
 	if ((desc->i_buf = alloc_buffer(sz)) == NULL)
@@ -10754,10 +10763,11 @@ static int tcp_recv(tcp_descriptor* desc, int request_len)
 	    return tcp_deliver(desc, desc->i_ptr - desc->i_ptr_start);
     }
     else {
-	if ((nread = tcp_remain(desc, &len)) < 0)
+        nread = tcp_remain(desc, &len);
+	if (nread < 0)
 	    return tcp_recv_error(desc, EMSGSIZE);
 	else if (nread == 0)
-	    return tcp_deliver(desc, len);
+            return tcp_deliver(desc, len);
 	else if (len > 0)
 	    desc->i_remain = len;  /* set remain */
     }
@@ -11135,8 +11145,8 @@ static int tcp_send_or_shutdown_error(tcp_descriptor* desc, int err)
     if (IS_BUSY(INETP(desc))) {
 	desc->inet.caller = desc->inet.busy_caller;
 	if (desc->busy_on_send) {
-	    driver_cancel_timer(desc->inet.port);
-	    desc->busy_on_send = 0;	
+            cancel_multi_timer(&(desc->mtd), INETP(desc)->port, &tcp_inet_send_timeout);
+	    desc->busy_on_send = 0;
 	}
 	desc->inet.state &= ~INET_F_BUSY;
 	set_busy_port(desc->inet.port, 0);
@@ -11288,7 +11298,9 @@ static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev)
 	    set_busy_port(desc->inet.port, 1);
 	    if (desc->send_timeout != INET_INFINITY) {
 		desc->busy_on_send = 1;
-		driver_set_timer(desc->inet.port, desc->send_timeout);
+                add_multi_timer(&(desc->mtd), INETP(desc)->port,
+                                0 /* arg */, desc->send_timeout /* timeout */,
+                                &tcp_inet_send_timeout);
 	    }
 	    return 1;
 	}
@@ -11386,7 +11398,9 @@ static int tcp_send(tcp_descriptor* desc, char* ptr, ErlDrvSizeT len)
 	    set_busy_port(desc->inet.port, 1);
 	    if (desc->send_timeout != INET_INFINITY) {
 		desc->busy_on_send = 1;
-		driver_set_timer(desc->inet.port, desc->send_timeout);
+                add_multi_timer(&(desc->mtd), INETP(desc)->port,
+                                0 /* arg */, desc->send_timeout /* timeout */,
+                                &tcp_inet_send_timeout);
 	    }
 	    return 1;
 	}
@@ -11490,7 +11504,8 @@ static int tcp_sendfile_completed(tcp_descriptor* desc) {
 
             /* if we have a timer then cancel and send ok to client */
             if (desc->busy_on_send) {
-                driver_cancel_timer(desc->inet.port);
+                cancel_multi_timer(&(desc->mtd), INETP(desc)->port,
+                                   &tcp_inet_send_timeout);
                 desc->busy_on_send = 0;
             }
 
@@ -11822,7 +11837,7 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event)
 		    set_busy_port(desc->inet.port, 0);
 		    /* if we have a timer then cancel and send ok to client */
 		    if (desc->busy_on_send) {
-			driver_cancel_timer(desc->inet.port);
+                        cancel_multi_timer(&(desc->mtd), INETP(desc)->port, &tcp_inet_send_timeout);
 			desc->busy_on_send = 0;
 		    }
 		    inet_reply_ok(INETP(desc));
@@ -12634,7 +12649,7 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
 		udesc->i_buf = NULL;
 		if (!desc->active) {
 		    async_error(desc, err);
-		    driver_cancel_timer(desc->port);
+                    driver_cancel_timer(desc->port);
 		    sock_select(desc,FD_READ,0);
 		}
 		else {
@@ -12723,7 +12738,7 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
 		return count;
 	    count++;
 	    if (!desc->active) {
-		driver_cancel_timer(desc->port); /* possibly cancel */
+                driver_cancel_timer(desc->port);
 		sock_select(desc,FD_READ,0);
 		return count;  /* passive mode (read one packet only) */
 	    }
@@ -12806,26 +12821,28 @@ static void fire_multi_timers(MultiTimerData **first, ErlDrvPort port,
 			      ErlDrvData data)
 {
     ErlDrvTime next_timeout;
-    if (!*first) {
+    MultiTimerData *curr = *first;
+    if (!curr) {
 	ASSERT(0);
 	return;
     }
 #ifdef DEBUG
     {
 	ErlDrvTime chk = erl_drv_monotonic_time(ERL_DRV_MSEC);
-	ASSERT(chk >= (*first)->when);
+	ASSERT(chk >= curr->when);
     }
 #endif
     do {
-	MultiTimerData *save = *first;
-	*first = save->next;
+	MultiTimerData *save = curr;
+        *first = save->next;
+	curr = *first;
 	(*(save->timeout_function))(data,save->caller);
 	FREE(save);
-	if (*first == NULL) {
+	if (curr == NULL) {
 	    return;
 	}
-	(*first)->prev = NULL;
-	next_timeout = (*first)->when - erl_drv_monotonic_time(ERL_DRV_MSEC);
+	curr->prev = NULL;
+	next_timeout = curr->when - erl_drv_monotonic_time(ERL_DRV_MSEC);
     } while (next_timeout <= 0);
     driver_set_timer(port, (unsigned long) next_timeout);
 }
@@ -12862,6 +12879,20 @@ static void remove_multi_timer(MultiTimerData **first, ErlDrvPort port, MultiTim
     FREE(p);
 }
 
+/* Cancel a timer based on the timeout_fun */
+static void cancel_multi_timer(MultiTimerData **first, ErlDrvPort port,
+                               void (*timeout_fun)(ErlDrvData drv_data,
+                                                   ErlDrvTermData caller))
+{
+    MultiTimerData *timer = *first;
+    while(timer && timer->timeout_function != timeout_fun) {
+        timer = timer->next;
+    }
+    if (timer) {
+        remove_multi_timer(first, port, timer);
+    }
+}
+
 static MultiTimerData *add_multi_timer(MultiTimerData **first, ErlDrvPort port, 
 				       ErlDrvTermData caller, unsigned timeout,
 				       void (*timeout_fun)(ErlDrvData drv_data, 
@@ -12869,7 +12900,7 @@ static MultiTimerData *add_multi_timer(MultiTimerData **first, ErlDrvPort port,
 {
     MultiTimerData *mtd, *p, *s;
     mtd = ALLOC(sizeof(MultiTimerData));
-    mtd->when = erl_drv_monotonic_time(ERL_DRV_MSEC) + ((ErlDrvTime) timeout) + 1;
+    mtd->when = erl_drv_monotonic_time(ERL_DRV_MSEC) + ((ErlDrvTime) timeout);
     mtd->timeout_function = timeout_fun;
     mtd->caller = caller;
     mtd->next = mtd->prev = NULL;
-- 
cgit v1.2.3


From 4f68b86b22806fb052def64926ae7340facaed53 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 16 Oct 2018 17:24:55 +0200
Subject: erts: Optimize the inet driver multi timers for one timer

The most common case for any socket is to have zero or one
timer, so we optimize for the one case. The only case when
we have more than one timer is when the multi accept feature
is used.
---
 erts/emulator/drivers/common/inet_drv.c | 146 ++++++++++++++++++++------------
 1 file changed, 90 insertions(+), 56 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 222b369a3f..8a7cb7e6fa 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -38,6 +38,7 @@
 #include <ctype.h>
 #include <sys/types.h>
 #include <errno.h>
+#include <stdint.h>
 
 #define IDENTITY(c) c
 #define STRINGIFY_1(b) IDENTITY(#b)
@@ -954,6 +955,7 @@ static size_t my_strnlen(const char *s, size_t maxlen)
 #endif
 #endif
 
+typedef struct _tcp_descriptor tcp_descriptor;
 
 #define BIN_REALLOC_MARGIN(x)  ((x)/4)  /* 25% */
 
@@ -1003,19 +1005,19 @@ typedef struct _multi_timer_data {
     struct _multi_timer_data *prev;
 } MultiTimerData;
 
-static MultiTimerData *add_multi_timer(MultiTimerData **first, ErlDrvPort port, 
-			    ErlDrvTermData caller, unsigned timeout,
-			    void (*timeout_fun)(ErlDrvData drv_data,
-						ErlDrvTermData caller));
-static void fire_multi_timers(MultiTimerData **first, ErlDrvPort port,
+static MultiTimerData *add_multi_timer(tcp_descriptor *desc, ErlDrvPort port, 
+                                       ErlDrvTermData caller, unsigned timeout,
+                                       void (*timeout_fun)(ErlDrvData drv_data,
+                                                           ErlDrvTermData caller));
+static void fire_multi_timers(tcp_descriptor *desc, ErlDrvPort port,
 			      ErlDrvData data);
-static void remove_multi_timer(MultiTimerData **first, ErlDrvPort port, MultiTimerData *p);
-static void cancel_multi_timer(MultiTimerData **first, ErlDrvPort port,
+static void remove_multi_timer(tcp_descriptor *desc, ErlDrvPort port, MultiTimerData *p);
+static void cancel_multi_timer(tcp_descriptor *desc, ErlDrvPort port,
                                void (*timeout_fun)(ErlDrvData drv_data,
                                                    ErlDrvTermData caller));
 
 static void tcp_inet_multi_timeout(ErlDrvData e, ErlDrvTermData caller);
-static void clean_multi_timers(MultiTimerData **first, ErlDrvPort port);
+static void clean_multi_timers(tcp_descriptor *desc, ErlDrvPort port);
 
 typedef struct {
     int            id;      /* id used to identify reply */
@@ -1274,7 +1276,7 @@ static struct erl_drv_entry sctp_inet_driver_entry =
 };
 #endif
 
-typedef struct {
+struct _tcp_descriptor {
     inet_descriptor inet;       /* common data structure (DON'T MOVE) */
     int   high;                 /* high watermark */
     int   low;                  /* low watermark */
@@ -1290,7 +1292,8 @@ typedef struct {
     int           http_state;   /* 0 = response|request  1=headers fields */
     inet_async_multi_op *multi_first;/* NULL == no multi-accept-queue, op is in ordinary queue */
     inet_async_multi_op *multi_last;
-    MultiTimerData *mtd;        /* Timer structures for multiple accept */
+    MultiTimerData *mtd;       /* Timer structures for multiple accept */
+    MultiTimerData *mtd_cache; /* A cache for timer allocations */
 #ifdef HAVE_SENDFILE
     struct {
         ErlDrvSizeT ioq_skip;   /* The number of bytes in the queue at the time
@@ -1306,7 +1309,7 @@ typedef struct {
         Uint64 length;
     } sendfile;
 #endif
-} tcp_descriptor;
+};
 
 /* send function */
 static int tcp_send(tcp_descriptor* desc, char* ptr, ErlDrvSizeT len);
@@ -9679,6 +9682,7 @@ static ErlDrvData prep_tcp_inet_start(ErlDrvPort port, char* args)
     desc->tcp_add_flags = 0;
     desc->http_state = 0;
     desc->mtd = NULL;
+    desc->mtd_cache = NULL;
     desc->multi_first = desc->multi_last = NULL;
     DEBUGF(("tcp_inet_start(%ld) }\r\n", (long)port));
     return (ErlDrvData) desc;
@@ -9789,7 +9793,7 @@ static void tcp_close_check(tcp_descriptor* desc)
     else if (desc->inet.state == INET_STATE_CONNECTED) {
 	async_error_am_all(INETP(desc), am_closed);
     }
-    clean_multi_timers(&(desc->mtd), desc->inet.port);
+    clean_multi_timers(desc, desc->inet.port);
 }
 
 /*
@@ -10011,12 +10015,12 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd,
 		if (time_left <= 0) {
 		    time_left = 1;
 		}
-		omtd = add_multi_timer(&(desc->mtd), desc->inet.port, ocaller, 
+		omtd = add_multi_timer(desc, desc->inet.port, ocaller, 
 				       time_left, &tcp_inet_multi_timeout);
 	    }
 	    enq_old_multi_op(desc, oid, oreq, ocaller, omtd, &omonitor);
 	    if (timeout != INET_INFINITY) {
-		mtd = add_multi_timer(&(desc->mtd), desc->inet.port, caller, 
+		mtd = add_multi_timer(desc, desc->inet.port, caller, 
 				      timeout, &tcp_inet_multi_timeout);
 	    }
 	    enq_multi_op(desc, tbuf, INET_REQ_ACCEPT, caller, mtd, &monitor);
@@ -10031,7 +10035,7 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd,
 		return ctl_xerror("noproc", rbuf, rsize);
 	    }
 	    if (timeout != INET_INFINITY) {
-		mtd = add_multi_timer(&(desc->mtd), desc->inet.port, caller, 
+		mtd = add_multi_timer(desc, desc->inet.port, caller, 
 				      timeout, &tcp_inet_multi_timeout);
 	    }
 	    enq_multi_op(desc, tbuf, INET_REQ_ACCEPT, caller, mtd, &monitor);
@@ -10128,7 +10132,7 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd,
 		async_error_am(INETP(desc), am_timeout);
 	    else {
 		if (timeout != INET_INFINITY)
-                    add_multi_timer(&(desc->mtd), INETP(desc)->port, 0,
+                    add_multi_timer(desc, INETP(desc)->port, 0,
                                     timeout, &tcp_inet_recv_timeout);
 		if (!INETP(desc)->is_ignored)
 		    sock_select(INETP(desc),(FD_READ|FD_CLOSE),1);
@@ -10269,9 +10273,9 @@ static void tcp_inet_timeout(ErlDrvData e)
     DEBUGF(("tcp_inet_timeout(%ld) {s=%d\r\n", 
 	    (long)desc->inet.port, desc->inet.s)); 
     if ((state & INET_F_MULTI_CLIENT)) { /* Multi-client always means multi-timers */
-	fire_multi_timers(&(desc->mtd), desc->inet.port, e);
+	fire_multi_timers(desc, desc->inet.port, e);
     } else if ((state & INET_STATE_CONNECTED) == INET_STATE_CONNECTED) {
-        fire_multi_timers(&(desc->mtd), desc->inet.port, e);
+        fire_multi_timers(desc, desc->inet.port, e);
     }
     else if ((state & INET_STATE_CONNECTING) == INET_STATE_CONNECTING) {
 	/* assume connect timeout */
@@ -10401,7 +10405,7 @@ static void tcp_inet_process_exit(ErlDrvData e, ErlDrvMonitor *monitorp)
 	    return;
 	}
 	if (timeout != NULL) {
-	    remove_multi_timer(&(desc->mtd), desc->inet.port, timeout);
+	    remove_multi_timer(desc, desc->inet.port, timeout);
 	}
 	if (desc->multi_first == NULL) {
 	    sock_select(INETP(desc),FD_ACCEPT,0);
@@ -10439,7 +10443,7 @@ static int tcp_recv_closed(tcp_descriptor* desc)
 	desc->inet.caller = desc->inet.busy_caller;
 	tcp_clear_output(desc);
 	if (desc->busy_on_send) {
-            cancel_multi_timer(&(desc->mtd), INETP(desc)->port, &tcp_inet_send_timeout);
+            cancel_multi_timer(desc, INETP(desc)->port, &tcp_inet_send_timeout);
 	    desc->busy_on_send = 0;
 	    DEBUGF(("tcp_recv_closed(%ld): busy on send\r\n", port));
 	}
@@ -10457,7 +10461,7 @@ static int tcp_recv_closed(tcp_descriptor* desc)
 
     if (!desc->inet.active) {
         /* We must cancel any timer here ! */
-        clean_multi_timers(&(desc->mtd), INETP(desc)->port);
+        clean_multi_timers(desc, INETP(desc)->port);
 	/* passive mode do not terminate port ! */
 	tcp_clear_input(desc);
 	if (desc->inet.exitf) {
@@ -10492,7 +10496,7 @@ static int tcp_recv_error(tcp_descriptor* desc, int err)
 	    desc->inet.caller = desc->inet.busy_caller;
 	    tcp_clear_output(desc);
 	    if (desc->busy_on_send) {
-                cancel_multi_timer(&(desc->mtd), INETP(desc)->port, &tcp_inet_send_timeout);
+                cancel_multi_timer(desc, INETP(desc)->port, &tcp_inet_send_timeout);
 		desc->busy_on_send = 0;
 	    }
 	    desc->inet.state &= ~INET_F_BUSY;
@@ -10501,7 +10505,7 @@ static int tcp_recv_error(tcp_descriptor* desc, int err)
 	}
 	if (!desc->inet.active) {
 	    /* We must cancel any timer here ! */
-            clean_multi_timers(&(desc->mtd), INETP(desc)->port);
+            clean_multi_timers(desc, INETP(desc)->port);
 	    tcp_clear_input(desc);
 	    if (desc->inet.exitf) {
 		tcp_desc_close(desc);
@@ -10664,7 +10668,7 @@ static int tcp_deliver(tcp_descriptor* desc, int len)
 	len = 0;
 
 	if (!desc->inet.active) {
-            cancel_multi_timer(&(desc->mtd), INETP(desc)->port, &tcp_inet_recv_timeout);
+            cancel_multi_timer(desc, INETP(desc)->port, &tcp_inet_recv_timeout);
 	    sock_select(INETP(desc),(FD_READ|FD_CLOSE),0);
 	    if (desc->i_buf != NULL)
 		tcp_restart_input(desc);
@@ -11086,7 +11090,7 @@ static int tcp_inet_input(tcp_descriptor* desc, HANDLE event)
 	    }
 	    
 	    if (timeout != NULL) {
-		remove_multi_timer(&(desc->mtd), desc->inet.port, timeout);
+		remove_multi_timer(desc, desc->inet.port, timeout);
 	    }
 	    
 	    driver_demonitor_process(desc->inet.port, &monitor);
@@ -11145,7 +11149,7 @@ static int tcp_send_or_shutdown_error(tcp_descriptor* desc, int err)
     if (IS_BUSY(INETP(desc))) {
 	desc->inet.caller = desc->inet.busy_caller;
 	if (desc->busy_on_send) {
-            cancel_multi_timer(&(desc->mtd), INETP(desc)->port, &tcp_inet_send_timeout);
+            cancel_multi_timer(desc, INETP(desc)->port, &tcp_inet_send_timeout);
 	    desc->busy_on_send = 0;
 	}
 	desc->inet.state &= ~INET_F_BUSY;
@@ -11298,7 +11302,7 @@ static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev)
 	    set_busy_port(desc->inet.port, 1);
 	    if (desc->send_timeout != INET_INFINITY) {
 		desc->busy_on_send = 1;
-                add_multi_timer(&(desc->mtd), INETP(desc)->port,
+                add_multi_timer(desc, INETP(desc)->port,
                                 0 /* arg */, desc->send_timeout /* timeout */,
                                 &tcp_inet_send_timeout);
 	    }
@@ -11398,7 +11402,7 @@ static int tcp_send(tcp_descriptor* desc, char* ptr, ErlDrvSizeT len)
 	    set_busy_port(desc->inet.port, 1);
 	    if (desc->send_timeout != INET_INFINITY) {
 		desc->busy_on_send = 1;
-                add_multi_timer(&(desc->mtd), INETP(desc)->port,
+                add_multi_timer(desc, INETP(desc)->port,
                                 0 /* arg */, desc->send_timeout /* timeout */,
                                 &tcp_inet_send_timeout);
 	    }
@@ -11504,7 +11508,7 @@ static int tcp_sendfile_completed(tcp_descriptor* desc) {
 
             /* if we have a timer then cancel and send ok to client */
             if (desc->busy_on_send) {
-                cancel_multi_timer(&(desc->mtd), INETP(desc)->port,
+                cancel_multi_timer(desc, INETP(desc)->port,
                                    &tcp_inet_send_timeout);
                 desc->busy_on_send = 0;
             }
@@ -11837,7 +11841,7 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event)
 		    set_busy_port(desc->inet.port, 0);
 		    /* if we have a timer then cancel and send ok to client */
 		    if (desc->busy_on_send) {
-                        cancel_multi_timer(&(desc->mtd), INETP(desc)->port, &tcp_inet_send_timeout);
+                        cancel_multi_timer(desc, INETP(desc)->port, &tcp_inet_send_timeout);
 			desc->busy_on_send = 0;
 		    }
 		    inet_reply_ok(INETP(desc));
@@ -12817,11 +12821,11 @@ make_noninheritable_handle(SOCKET s)
  * Multi-timers
  */
 
-static void fire_multi_timers(MultiTimerData **first, ErlDrvPort port,
+static void fire_multi_timers(tcp_descriptor *desc, ErlDrvPort port,
 			      ErlDrvData data)
 {
     ErlDrvTime next_timeout;
-    MultiTimerData *curr = *first;
+    MultiTimerData *curr = desc->mtd;
     if (!curr) {
 	ASSERT(0);
 	return;
@@ -12834,40 +12838,52 @@ static void fire_multi_timers(MultiTimerData **first, ErlDrvPort port,
 #endif
     do {
 	MultiTimerData *save = curr;
-        *first = save->next;
-	curr = *first;
+
 	(*(save->timeout_function))(data,save->caller);
-	FREE(save);
+
+        curr = curr->next;
+
+        if (desc->mtd_cache == NULL)
+            desc->mtd_cache = save;
+        else
+            FREE(save);
+
 	if (curr == NULL) {
+            desc->mtd = NULL;
 	    return;
 	}
 	curr->prev = NULL;
 	next_timeout = curr->when - erl_drv_monotonic_time(ERL_DRV_MSEC);
     } while (next_timeout <= 0);
+    desc->mtd = curr;
     driver_set_timer(port, (unsigned long) next_timeout);
 }
 
-static void clean_multi_timers(MultiTimerData **first, ErlDrvPort port)
+static void clean_multi_timers(tcp_descriptor *desc, ErlDrvPort port)
 {
-    MultiTimerData *p;
-    if (*first) {
+    if (desc->mtd) {
 	driver_cancel_timer(port);
     }
-    while (*first) {
-	p = *first;
-	*first = p->next;
-	FREE(p);
+    while (desc->mtd) {
+	MultiTimerData *p = desc->mtd;
+	desc->mtd = p->next;
+        FREE(p);
+    }
+    desc->mtd = NULL;
+    if (desc->mtd_cache) {
+        FREE(desc->mtd_cache);
+        desc->mtd_cache = NULL;
     }
 }
-static void remove_multi_timer(MultiTimerData **first, ErlDrvPort port, MultiTimerData *p)
+static void remove_multi_timer(tcp_descriptor *desc, ErlDrvPort port, MultiTimerData *p)
 {
     if (p->prev != NULL) {
 	p->prev->next = p->next;
     } else {
 	driver_cancel_timer(port);
-	*first = p->next;
-	if (*first) {
-	    ErlDrvTime ntmo = (*first)->when - erl_drv_monotonic_time(ERL_DRV_MSEC);
+	desc->mtd = p->next;
+	if (desc->mtd) {
+	    ErlDrvTime ntmo = desc->mtd->when - erl_drv_monotonic_time(ERL_DRV_MSEC);
 	    if (ntmo < 0)
 		ntmo = 0;
 	    driver_set_timer(port, (unsigned long) ntmo);
@@ -12876,50 +12892,67 @@ static void remove_multi_timer(MultiTimerData **first, ErlDrvPort port, MultiTim
     if (p->next != NULL) {
 	p->next->prev = p->prev;
     }
-    FREE(p);
+    if (desc->mtd_cache == NULL)
+        desc->mtd_cache = p;
+    else
+        FREE(p);
 }
 
 /* Cancel a timer based on the timeout_fun */
-static void cancel_multi_timer(MultiTimerData **first, ErlDrvPort port,
+static void cancel_multi_timer(tcp_descriptor *desc, ErlDrvPort port,
                                void (*timeout_fun)(ErlDrvData drv_data,
                                                    ErlDrvTermData caller))
 {
-    MultiTimerData *timer = *first;
+    MultiTimerData *timer = desc->mtd;
     while(timer && timer->timeout_function != timeout_fun) {
         timer = timer->next;
     }
     if (timer) {
-        remove_multi_timer(first, port, timer);
+        remove_multi_timer(desc, port, timer);
     }
 }
 
-static MultiTimerData *add_multi_timer(MultiTimerData **first, ErlDrvPort port, 
+static MultiTimerData *add_multi_timer(tcp_descriptor *desc, ErlDrvPort port, 
 				       ErlDrvTermData caller, unsigned timeout,
 				       void (*timeout_fun)(ErlDrvData drv_data, 
 							   ErlDrvTermData caller))
 {
     MultiTimerData *mtd, *p, *s;
-    mtd = ALLOC(sizeof(MultiTimerData));
-    mtd->when = erl_drv_monotonic_time(ERL_DRV_MSEC) + ((ErlDrvTime) timeout);
+
+    /* Use cached timer if available */
+    if (desc->mtd_cache != NULL) {
+        mtd = desc->mtd_cache;
+        desc->mtd_cache = NULL;
+    } else
+        mtd = ALLOC(sizeof(MultiTimerData));
+
+    if (timeout)
+        mtd->when = erl_drv_monotonic_time(ERL_DRV_MSEC) + ((ErlDrvTime) timeout);
+    else
+        mtd->when = INT64_MIN; /* Don't have to get the time for 0 msec timeouts */
+
     mtd->timeout_function = timeout_fun;
     mtd->caller = caller;
     mtd->next = mtd->prev = NULL;
-    for(p = *first,s = NULL; p != NULL; s = p, p = p->next) {
+
+    /* Find correct slot in timer linked list */
+    for(p = desc->mtd,s = NULL; p != NULL; s = p, p = p->next) {
 	if (p->when >= mtd->when) {
 	    break;
 	}
     }
 
+    /* Insert in linked list */
     if (!p) {
 	if (!s) {
-	    *first = mtd;
+	    desc->mtd = mtd;
 	} else {
 	    s->next = mtd;
 	    mtd->prev = s;
 	}
     } else {
 	if (!s) {
-	    *first = mtd;
+	    desc->mtd = mtd;
 	} else {
 	    s->next = mtd;
 	    mtd->prev = s;
@@ -12927,6 +12960,7 @@ static MultiTimerData *add_multi_timer(MultiTimerData **first, ErlDrvPort port,
 	mtd->next = p;
 	p->prev = mtd;
     }
+    /* Possibly set new timer */
     if (!s) {
 	if (mtd->next) {
 	    driver_cancel_timer(port);
-- 
cgit v1.2.3


From d4440ce1793638f8daca35a32936d3005cc3be7a Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 18 Oct 2018 10:25:31 +0200
Subject: erts: Optimize driver_set_timer(0) to fire at once

By optimizing driver_set_timer for 0 we can use it
instead of select to do a yield in the driver.
Use full for delay_send in the inet driver.

OTP-15472
---
 erts/emulator/beam/erl_hl_timer.c | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 6ec6f8065e..ef7a55fa38 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -3041,15 +3041,23 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo)
 
     check_canceled_queue(esdp, esdp->timer_service);
 
-    timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), tmo);
-
-    create_timer = (tmo < ERTS_TIMER_WHEEL_MSEC
-                    ? create_tw_timer
-                    : create_hl_timer);
-    tmr = (void *) create_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT,
-                                (void *) c_prt, c_prt->common.id,
-                                THE_NON_VALUE, NULL, NULL, NULL);
-    erts_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr);
+    if (tmo == 0) {
+        erts_atomic_set_relb(&c_prt->common.timer, ERTS_PTMR_TIMEDOUT);
+        erts_port_task_schedule(c_prt->common.id,
+				&c_prt->timeout_task,
+				ERTS_PORT_TASK_TIMEOUT);
+    } else {
+
+        timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), tmo);
+
+        create_timer = (tmo < ERTS_TIMER_WHEEL_MSEC
+                        ? create_tw_timer
+                        : create_hl_timer);
+        tmr = (void *) create_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT,
+                                    (void *) c_prt, c_prt->common.id,
+                                    THE_NON_VALUE, NULL, NULL, NULL);
+        erts_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr);
+    }
 }
 
 void
-- 
cgit v1.2.3


From a2d9a51091578da0ef41fbd91b281091d20aca2b Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 10 Oct 2018 16:19:02 +0200
Subject: erts: Implement delay_send using timer instead of poll

The previous implementation uses a round-trip in the poll-set
to simulate a yield of the port context. With the poll thread
implementation this is no longer a good idea as it generated a
lot more work for the system. So this commit changes the
implementation to use a timer instead.

OTP-15471
---
 erts/emulator/drivers/common/inet_drv.c | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 8a7cb7e6fa..627e1c8ec2 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -11250,6 +11250,12 @@ static int tcp_shutdown_error(tcp_descriptor* desc, int err)
     return tcp_send_or_shutdown_error(desc, err);
 }
 
+static void tcp_inet_delay_send(ErlDrvData data, ErlDrvTermData dummy)
+{
+    tcp_descriptor *desc = (tcp_descriptor*)data;
+    (void)tcp_inet_output(desc, INETP(desc)->s);
+}
+
 /*
 ** Send non-blocking vector data
 */
@@ -11319,7 +11325,10 @@ static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev)
 	    INETP(desc)->is_ignored |= INET_IGNORE_WRITE;
 	    n = 0;
 	} else if (desc->tcp_add_flags & TCP_ADDF_DELAY_SEND) {
-	    n = 0;
+            driver_enqv(ix, ev, 0);
+            add_multi_timer(desc, INETP(desc)->port, 0,
+                            0, &tcp_inet_delay_send);
+	    return 0;
 	} else if (IS_SOCKET_ERROR(sock_sendv(desc->inet.s, ev->iov,
 					      vsize, &n, 0))) {
 	    if ((sock_errno() != ERRNO_BLOCK) && (sock_errno() != EINTR)) {
@@ -11816,6 +11825,12 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event)
 #ifdef __WIN32__
 		desc->inet.send_would_block = 1;
 #endif
+                /* If DELAY_SEND is set ready_output may have
+                   been called without doing select so we do
+                   a select in order to get into the correct
+                   state */
+                if (desc->tcp_add_flags & TCP_ADDF_DELAY_SEND)
+                    sock_select(INETP(desc), FD_WRITE, 1);
 		goto done;
 	    } else if (n == 0) { /* Workaround for redhat/CentOS 6.3 returning 
 				    0 when sending packets with 
@@ -12962,9 +12977,6 @@ static MultiTimerData *add_multi_timer(tcp_descriptor *desc, ErlDrvPort port,
     }
     /* Possibly set new timer */
     if (!s) {
-	if (mtd->next) {
-	    driver_cancel_timer(port);
-	}
 	driver_set_timer(port,timeout);
     }
     return mtd;
-- 
cgit v1.2.3


From 790aa06d380b5842fe2fdd984bdde4f160b3adaa Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 24 Oct 2018 17:08:03 +0200
Subject: erts: Fix lists_member_2 reduction count

OTP-15474
---
 erts/emulator/beam/erl_bif_lists.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index 395be67a90..2a2b94c831 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -249,7 +249,8 @@ BIF_RETTYPE lists_member_2(BIF_ALIST_2)
     Eterm list;
     Eterm item;
     int non_immed_key;
-    int max_iter = 10 * CONTEXT_REDS;
+    int reds_left = ERTS_BIF_REDS_LEFT(BIF_P);
+    int max_iter = 16 * reds_left;
 
     if (is_nil(BIF_ARG_2)) {
 	BIF_RET(am_false);
@@ -267,14 +268,15 @@ BIF_RETTYPE lists_member_2(BIF_ALIST_2)
 	}
 	item = CAR(list_val(list));
 	if ((item == term) || (non_immed_key && eq(item, term))) {
-	    BIF_RET2(am_true, CONTEXT_REDS - max_iter/10);
+	    BIF_RET2(am_true, reds_left - max_iter/16);
 	}
 	list = CDR(list_val(list));
     }
     if (is_not_nil(list))  {
+        BUMP_REDS(BIF_P, reds_left - max_iter/16);
 	BIF_ERROR(BIF_P, BADARG);
     }
-    BIF_RET2(am_false, CONTEXT_REDS - max_iter/10);
+    BIF_RET2(am_false, reds_left - max_iter/16);
 }
 
 static BIF_RETTYPE lists_reverse_alloc(Process *c_p,
-- 
cgit v1.2.3


From 4e5e0c6103982775e8e890f7d7c209cba279d0d6 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 28 Nov 2018 11:06:19 +0100
Subject: erts: Fix some 32-bit gcc warnings

---
 erts/emulator/beam/erl_alloc_util.c       | 4 ++++
 erts/emulator/nifs/common/prim_file_nif.c | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index a5740a08cf..0be4562785 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -834,6 +834,8 @@ static ERTS_INLINE void clr_bit(UWord* map, Uint ix)
         &= ~((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS));
 }
 
+#ifdef DEBUG
+
 static ERTS_INLINE int is_bit_set(UWord* map, Uint ix)
 {
     ASSERT(ix / ERTS_VSPACE_WORD_BITS < VSPACE_MAP_SZ);
@@ -841,6 +843,8 @@ static ERTS_INLINE int is_bit_set(UWord* map, Uint ix)
         & ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS));
 }
 
+#endif
+
 UWord erts_literal_vspace_map[VSPACE_MAP_SZ];
 
 static void set_literal_range(void* start, Uint size)
diff --git a/erts/emulator/nifs/common/prim_file_nif.c b/erts/emulator/nifs/common/prim_file_nif.c
index ba36a33458..b34bf11205 100644
--- a/erts/emulator/nifs/common/prim_file_nif.c
+++ b/erts/emulator/nifs/common/prim_file_nif.c
@@ -933,7 +933,7 @@ static ERL_NIF_TERM set_permissions_nif(ErlNifEnv *env, int argc, const ERL_NIF_
     posix_errno_t posix_errno;
 
     efile_path_t path;
-    Uint32 permissions;
+    Uint permissions;
 
     if(argc != 2 || !enif_get_uint(env, argv[1], &permissions)) {
         return enif_make_badarg(env);
@@ -952,7 +952,7 @@ static ERL_NIF_TERM set_owner_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM a
     posix_errno_t posix_errno;
 
     efile_path_t path;
-    Sint32 uid, gid;
+    Sint uid, gid;
 
     if(argc != 3 || !enif_get_int(env, argv[1], &uid)
                  || !enif_get_int(env, argv[2], &gid)) {
-- 
cgit v1.2.3


From 754e2022059864f2c4025ab67da759bde922a066 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 4 Dec 2018 10:10:37 +0100
Subject: erts: Fix copy of literal msg during gc

A copy has to be made of the message as there is
a trace token. There was a bug where the actual
message was incorrectly modified even if it was a
literal.
---
 erts/emulator/beam/erl_gc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index b4df418cd5..d5dfb096b1 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -2477,7 +2477,7 @@ erts_copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap,
     *hpp = hp;
 
     for (i = 0; i < nrefs; i++) {
-	if (is_not_immed(refs[i]))
+	if (is_not_immed(refs[i]) && !erts_is_literal(refs[i],boxed_val(refs[i])))
 	    refs[i] = offset_ptr(refs[i], offs);
     }
     bp->off_heap.first = NULL;
-- 
cgit v1.2.3


From c6498571109b524fb319300e1b177b942e556f1b Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Fri, 12 Oct 2018 18:16:17 +0200
Subject: erts: Move fds with active true behaviour to own pollset

At start of the VM a poll-set that the schedulers
will check is created where fds that have triggered
many (at the moment, many means 10) times without
being deselected inbetween. In this scheduler specific
poll-set fds do not use ONESHOT, which means that the
number of syscalls goes down dramatically for such fds.

This pollset is introduced in order to handle fds that
are used by the erlang distribution and that never
change their state from {active, true}.

This pollset only handles ready_input events,
ready_output is still handled by the poll threads.

During overload, polling the scheduler poll-set is done
on a 10ms timer.
---
 erts/emulator/beam/erl_port.h           |   2 +
 erts/emulator/beam/erl_port_task.c      |  81 ++++-
 erts/emulator/beam/erl_port_task.h      |  21 +-
 erts/emulator/beam/erl_process.c        | 122 +++++++-
 erts/emulator/beam/erl_process.h        |   4 +-
 erts/emulator/sys/common/erl_check_io.c | 310 ++++++++++++++------
 erts/emulator/sys/common/erl_check_io.h |  16 +-
 erts/emulator/sys/common/erl_poll.c     | 504 ++++++++++++++++++++++++--------
 erts/emulator/sys/common/erl_poll.h     |  14 +-
 erts/emulator/sys/common/erl_poll_api.h |   4 +-
 erts/emulator/sys/win32/erl_poll.c      |   5 +-
 erts/emulator/test/scheduler_SUITE.erl  |  27 +-
 erts/emulator/test/signal_SUITE.erl     |   2 +-
 13 files changed, 843 insertions(+), 269 deletions(-)

(limited to 'erts/emulator')

diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index 2be0a5bf74..25976d38cc 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -334,6 +334,8 @@ Eterm erts_request_io_bytes(Process *c_p);
 #define ERTS_PORT_SFLG_INVALID		((Uint32) (1 << 11))
 /* Last port to terminate halts the emulator */
 #define ERTS_PORT_SFLG_HALT		((Uint32) (1 << 12))
+/* Check if the event in ready_input should be cleaned */
+#define ERTS_PORT_SFLG_CHECK_FD_CLEANUP ((Uint32) (1 << 13))
 #ifdef DEBUG
 /* Only debug: make sure all flags aren't cleared unintentionally */
 #define ERTS_PORT_SFLG_PORT_DEBUG	((Uint32) (1 << 31))
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 4928d80f27..c8f2e88127 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -97,6 +97,9 @@ static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_q
 typedef union {
     struct { /* I/O tasks */
 	ErlDrvEvent event;
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+        int is_scheduler_event;
+#endif
     } io;
     struct {
 	ErtsProc2PortSigCallback callback;
@@ -141,6 +144,9 @@ struct ErtsPortTaskBusyCallerTable_ {
     ErtsPortTaskBusyCaller pre_alloc_busy_caller;
 };
 
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+erts_atomic_t erts_port_task_outstanding_io_tasks;
+#endif
 
 static void begin_port_cleanup(Port *pp,
 			       ErtsPortTask **execq,
@@ -578,13 +584,26 @@ reset_handle(ErtsPortTask *ptp)
 }
 
 static ERTS_INLINE void
-reset_executed_io_task_handle(ErtsPortTask *ptp)
+reset_executed_io_task_handle(Port *prt, ErtsPortTask *ptp)
 {
     if (ptp->u.alive.handle) {
 	ASSERT(ptp == handle2task(ptp->u.alive.handle));
-        /* The port task handle is reset inside task_executed */
-	erts_io_notify_port_task_executed(ptp->type, ptp->u.alive.handle,
-                                          reset_port_task_handle);
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+        if (ptp->u.alive.td.io.is_scheduler_event) {
+            if ((erts_atomic32_read_nob(&prt->state) & ERTS_PORT_SFLG_CHECK_FD_CLEANUP)) {
+                erts_io_notify_port_task_executed(ptp->type, ptp->u.alive.handle,
+                                                  reset_port_task_handle);
+                erts_atomic32_read_band_nob(&prt->state, ~ERTS_PORT_SFLG_CHECK_FD_CLEANUP);
+            } else {
+                reset_port_task_handle(ptp->u.alive.handle);
+            }
+        } else
+#endif
+        {
+            /* The port task handle is reset inside task_executed */
+            erts_io_notify_port_task_executed(ptp->type, ptp->u.alive.handle,
+                                              reset_port_task_handle);
+        }
     }
 }
 
@@ -1307,6 +1326,22 @@ erts_port_task_abort(ErtsPortTaskHandle *pthp)
 	    res = - 1; /* Task already aborted, executing, or executed */
 	else {
 	    reset_port_task_handle(pthp);
+
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+            switch (ptp->type) {
+	    case ERTS_PORT_TASK_INPUT:
+	    case ERTS_PORT_TASK_OUTPUT:
+                if (ptp->u.alive.td.io.is_scheduler_event) {
+                    ASSERT(erts_atomic_read_nob(
+                               &erts_port_task_outstanding_io_tasks) > 0);
+                    erts_atomic_dec_relb(&erts_port_task_outstanding_io_tasks);
+                }
+		break;
+	    default:
+		break;
+	    }
+#endif
+
 	    res = 0;
 	}
     }
@@ -1442,7 +1477,14 @@ erts_port_task_schedule(Eterm id,
 	va_list argp;
 	va_start(argp, type);
 	ptp->u.alive.td.io.event = va_arg(argp, ErlDrvEvent);
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+        ptp->u.alive.td.io.is_scheduler_event = va_arg(argp, int);
+#endif
 	va_end(argp);
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+        if (ptp->u.alive.td.io.is_scheduler_event)
+            erts_atomic_inc_relb(&erts_port_task_outstanding_io_tasks);
+#endif
 	break;
     }
     case ERTS_PORT_TASK_PROC_SIG: {
@@ -1621,12 +1663,14 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     int processing_busy_q;
     int vreds = 0;
     int reds = 0;
-    erts_aint_t io_tasks_executed = 0;
     int fpe_was_unmasked;
     erts_aint32_t state;
     int active;
     Uint64 start_time = 0;
     ErtsSchedulerData *esdp = runq->scheduler;
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+    erts_aint_t io_tasks_executed = 0;
+#endif
     ERTS_MSACC_PUSH_STATE_M();
 
     ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq));
@@ -1722,8 +1766,11 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	       for input and output */
 	    (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data,
 					ptp->u.alive.td.io.event);
-	    reset_executed_io_task_handle(ptp);
-	    io_tasks_executed++;
+	    reset_executed_io_task_handle(pp, ptp);
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+            if (ptp->u.alive.td.io.is_scheduler_event)
+                io_tasks_executed++;
+#endif
 	    break;
 	case ERTS_PORT_TASK_OUTPUT:
 	    reds = ERTS_PORT_REDS_OUTPUT;
@@ -1732,8 +1779,11 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
             LTTNG_DRIVER(driver_ready_output, pp);
 	    (*pp->drv_ptr->ready_output)((ErlDrvData) pp->drv_data,
 					 ptp->u.alive.td.io.event);
-	    reset_executed_io_task_handle(ptp);
-	    io_tasks_executed++;
+	    reset_executed_io_task_handle(pp, ptp);
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+            if (ptp->u.alive.td.io.is_scheduler_event)
+                io_tasks_executed++;
+#endif
 	    break;
 	case ERTS_PORT_TASK_PROC_SIG: {
 	    ErtsProc2PortSigData *sigdp = &ptp->u.alive.td.psig.data;
@@ -1799,6 +1849,15 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     erts_unblock_fpe(fpe_was_unmasked);
     ERTS_MSACC_POP_STATE_M();
 
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+    if (io_tasks_executed) {
+        ASSERT(erts_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
+	       >= io_tasks_executed);
+        erts_atomic_add_relb(&erts_port_task_outstanding_io_tasks,
+				 -1*io_tasks_executed);
+    }
+#endif
+
     ASSERT(runq == erts_get_runq_port(pp));
 
     active = finalize_exec(pp, &execq, processing_busy_q);
@@ -2086,6 +2145,10 @@ erts_dequeue_port(ErtsRunQueue *rq)
 void
 erts_port_task_init(void)
 {
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+    erts_atomic_init_nob(&erts_port_task_outstanding_io_tasks,
+                         (erts_aint_t) 0);
+#endif    
     init_port_task_alloc(erts_no_schedulers + erts_no_poll_threads
                          + 1); /* aux_thread */
     init_busy_caller_table_alloc();
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index ae78a7d8a3..ca5183b305 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -38,6 +38,8 @@ typedef erts_atomic_t ErtsPortTaskHandle;
 #ifndef ERL_PORT_TASK_H__
 #define ERL_PORT_TASK_H__
 
+#include "erl_poll.h"
+
 #undef ERTS_INCLUDE_SCHEDULER_INTERNALS
 #if (defined(ERL_PROCESS_C__) \
      || defined(ERL_PORT_TASK_C__) \
@@ -54,8 +56,8 @@ typedef erts_atomic_t ErtsPortTaskHandle;
 #define ERTS_PT_FLG_BAD_OUTPUT		(1 << 4)
 
 typedef enum {
-    ERTS_PORT_TASK_INPUT,
-    ERTS_PORT_TASK_OUTPUT,
+    ERTS_PORT_TASK_INPUT = 0,
+    ERTS_PORT_TASK_OUTPUT = 1,
     ERTS_PORT_TASK_TIMEOUT,
     ERTS_PORT_TASK_DIST_CMD,
     ERTS_PORT_TASK_PROC_SIG
@@ -134,6 +136,12 @@ ERTS_GLB_INLINE void erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp);
 ERTS_GLB_INLINE int erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp);
 ERTS_GLB_INLINE void erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp);
 
+#if defined(ERTS_INCLUDE_SCHEDULER_INTERNALS) && ERTS_POLL_USE_SCHEDULER_POLLING
+ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void);
+/* NOTE: Do not access any of the exported variables directly */
+extern erts_atomic_t erts_port_task_outstanding_io_tasks;
+#endif
+
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_INLINE void
@@ -211,6 +219,15 @@ erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp)
     erts_atomic32_read_bor_nob(&ptsp->flags, ERTS_PTS_FLG_EXITING);
 }
 
+#if defined(ERTS_INCLUDE_SCHEDULER_INTERNALS) && ERTS_POLL_USE_SCHEDULER_POLLING
+ERTS_GLB_INLINE int
+erts_port_task_have_outstanding_io_tasks(void)
+{
+    return (erts_atomic_read_acqb(&erts_port_task_outstanding_io_tasks)
+	    != 0);
+}
+#endif
+
 #endif
 
 #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 919d9f9edf..2427d87f66 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -174,7 +174,6 @@ ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];
 typedef struct {
     int aux_work;
     int tse;
-    int sys_schedule;
 } ErtsBusyWaitParams;
 
 static ErtsBusyWaitParams sched_busy_wait_params[ERTS_SCHED_TYPE_LAST + 1];
@@ -344,6 +343,9 @@ erts_sched_stat_t erts_sched_stat;
 
 static erts_tsd_key_t ERTS_WRITE_UNLIKELY(sched_data_key);
 
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+static erts_atomic32_t doing_sys_schedule;
+#endif
 static erts_atomic32_t no_empty_run_queues;
 long erts_runq_supervision_interval = 0;
 static ethr_event runq_supervision_event;
@@ -3093,7 +3095,11 @@ aux_thread(void *unused)
     awdp->ssi = ssi;
 
 #if ERTS_POLL_USE_FALLBACK
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+    ssi->psi = erts_create_pollset_thread(-2, tpd);
+#else
     ssi->psi = erts_create_pollset_thread(-1, tpd);
+#endif
 #endif
 
     sched_prep_spin_wait(ssi);
@@ -3133,7 +3139,7 @@ aux_thread(void *unused)
 		if (flgs & ERTS_SSI_FLG_SLEEPING) {
 		    ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING);
 		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-                    erts_check_io(ssi->psi);
+                    erts_check_io(ssi->psi, ERTS_POLL_INF_TIMEOUT);
 		}
             }
 #else
@@ -3231,7 +3237,7 @@ poll_thread(void *arg)
 		if (flgs & ERTS_SSI_FLG_SLEEPING) {
 		    ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING);
 		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-                    erts_check_io(psi);
+                    erts_check_io(psi, ERTS_POLL_INF_TIMEOUT);
 		}
 	    }
 	}
@@ -3241,6 +3247,59 @@ poll_thread(void *arg)
     return NULL;
 }
 
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+static ERTS_INLINE void
+clear_sys_scheduling(void)
+{
+    erts_atomic32_set_mb(&doing_sys_schedule, 0);
+}
+
+static ERTS_INLINE int
+try_set_sys_scheduling(void)
+{
+    return 0 == erts_atomic32_cmpxchg_acqb(&doing_sys_schedule, 1, 0);
+}
+
+
+static ERTS_INLINE int
+prepare_for_sys_schedule(void)
+{
+    while (!erts_port_task_have_outstanding_io_tasks()
+           && try_set_sys_scheduling()) {
+        if (!erts_port_task_have_outstanding_io_tasks())
+            return 1;
+        clear_sys_scheduling();
+    }
+    return 0;
+}
+
+static void
+check_io_timer(void *null)
+{
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    if (prepare_for_sys_schedule()) {
+        erts_check_io(esdp->ssi->psi, ERTS_POLL_NO_TIMEOUT);
+        clear_sys_scheduling();
+    }
+
+    /* The timer is cleared if this schedulers run-queue became empty
+       or if the CHECKIO flag was cleared. The CHECKIO flags is cleared
+       when a check_balance assigns another scheduler to be the poller in
+       the overload scenario. */
+    if ((ERTS_RUNQ_FLGS_GET_NOB(esdp->run_queue) & (ERTS_RUNQ_FLG_OUT_OF_WORK|ERTS_RUNQ_FLG_CHECKIO))
+        == ERTS_RUNQ_FLG_CHECKIO) {
+        erts_start_timer_callback(ERTS_POLL_SCHEDULER_POLLING_TIMEOUT,
+                                  check_io_timer, NULL);
+    } else {
+        ERTS_RUNQ_FLGS_UNSET(esdp->run_queue, ERTS_RUNQ_FLG_CHECKIO);
+    }
+}
+
+#else
+#define clear_sys_scheduling()
+#define prepare_for_sys_schedule() 0
+#endif
+
 static void
 scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 {
@@ -3330,7 +3389,25 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
                     sched_wall_time_change(esdp, 1);
                 }
             }
-            else {
+            else if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && prepare_for_sys_schedule()) {
+                /* We sleep in check_io, only for normal schedulers */
+                if (thr_prgr_active) {
+                    erts_thr_progress_active(erts_thr_prgr_data(esdp), thr_prgr_active = 0);
+                    sched_wall_time_change(esdp, 0);
+                }
+                flgs = sched_spin_wait(ssi, 0);
+                if (flgs & ERTS_SSI_FLG_SLEEPING) {
+                    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+                    flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING);
+                    if (flgs & ERTS_SSI_FLG_SLEEPING) {
+                        ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING);
+                        ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+                        erts_check_io(ssi->psi, timeout_time);
+                        current_time = erts_get_monotonic_time(esdp);
+                    }
+                }
+                clear_sys_scheduling();
+            } else {
                 if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
                     if (thr_prgr_active) {
                         erts_thr_progress_active(erts_thr_prgr_data(esdp), thr_prgr_active = 0);
@@ -3338,7 +3415,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
                     }
                     erts_thr_progress_prepare_wait(erts_thr_prgr_data(esdp));
                 }
-
                 flgs = sched_spin_wait(ssi, spincount);
                 if (flgs & ERTS_SSI_FLG_SLEEPING) {
                     ASSERT(flgs & ERTS_SSI_FLG_WAITING);
@@ -4585,6 +4661,15 @@ check_balance(ErtsRunQueue *c_rq)
     if (blnc_no_rqs == 1) {
 	c_rq->check_balance_reds = INT_MAX;
 	erts_atomic32_set_nob(&balance_info.checking_balance, 0);
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+	c_rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
+        if ((ERTS_RUNQ_FLGS_GET_NOB(c_rq) & (ERTS_RUNQ_FLG_OUT_OF_WORK|ERTS_RUNQ_FLG_CHECKIO))
+             == 0) {
+            ERTS_RUNQ_FLGS_SET(c_rq, ERTS_RUNQ_FLG_CHECKIO);
+            erts_start_timer_callback(ERTS_POLL_SCHEDULER_POLLING_TIMEOUT, check_io_timer, NULL);
+        }
+        ERTS_RUNQ_FLGS_UNSET(c_rq, ERTS_RUNQ_FLGS_MIGRATION_INFO);
+#endif
 	return;
     }
 
@@ -5104,6 +5189,19 @@ erts_fprintf(stderr, "--------------------------------\n");
     /* Publish new migration paths... */
     erts_atomic_set_wb(&erts_migration_paths, (erts_aint_t) new_mpaths);
 
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+    if (full_scheds == current_active) {
+        ERTS_ASSERT(full_scheds <= current_active);
+        /* All active schedulers ran for full, we need to do active polling,
+           so we setup a timer that does active polling */
+        if (!(ERTS_RUNQ_FLGS_GET_NOB(c_rq) & ERTS_RUNQ_FLG_CHECKIO)) {
+            /* Active polling is not running, start it */
+            erts_start_timer_callback(ERTS_POLL_SCHEDULER_POLLING_TIMEOUT, check_io_timer, NULL);
+        }
+        run_queue_info[c_rq->ix].flags |= ERTS_RUNQ_FLG_CHECKIO;
+    }
+#endif
+
     /* Reset balance statistics in all online queues */
     for (qix = 0; qix < blnc_no_rqs; qix++) {
 	Uint32 flags = run_queue_info[qix].flags;
@@ -5113,6 +5211,8 @@ erts_fprintf(stderr, "--------------------------------\n");
 	ASSERT(!(flags & ERTS_RUNQ_FLG_OUT_OF_WORK));
 	if (rq->waiting)
 	    flags |= ERTS_RUNQ_FLG_OUT_OF_WORK;
+        if (rq != c_rq)
+            flags &= ~ERTS_RUNQ_FLG_CHECKIO;
 
 	rq->full_reds_history_sum
 	    = run_queue_info[qix].full_reds_history_sum;
@@ -5122,8 +5222,7 @@ erts_fprintf(stderr, "--------------------------------\n");
 	ERTS_DBG_CHK_FULL_REDS_HISTORY(rq);
 
 	rq->out_of_work_count = 0;
-	(void) ERTS_RUNQ_FLGS_READ_BSET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags);
-
+	(void) ERTS_RUNQ_FLGS_READ_BSET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO|ERTS_RUNQ_FLG_CHECKIO, flags);
 	rq->max_len = erts_atomic32_read_dirty(&rq->len);
 	for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) {
 	    ErtsRunQueueInfo *rqi;
@@ -5562,7 +5661,6 @@ erts_sched_set_busy_wait_threshold(ErtsSchedType sched_type, char *str)
 	return EINVAL;
     }
 
-    params->sys_schedule = sys_sched;
     params->tse = sys_sched * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT;
     params->aux_work = sys_sched * aux_work_fact;
 
@@ -5773,6 +5871,9 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
     size_runqs = sizeof(ErtsAlignedRunQueue) * tot_rqs;
     erts_aligned_run_queues =
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, size_runqs);
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+    erts_atomic32_init_nob(&doing_sys_schedule, 0);
+#endif
     erts_atomic32_init_nob(&no_empty_run_queues, 0);
 
     erts_no_run_queues = n;
@@ -8302,6 +8403,11 @@ sched_thread_func(void *vesdp)
     erts_msacc_init_thread("scheduler", no, 1);
 
     erts_thr_progress_register_managed_thread(esdp, &callbacks, 0);
+
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+    esdp->ssi->psi = erts_create_pollset_thread(-1, NULL);
+#endif
+
     erts_alloc_register_scheduler(vesdp);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 8d20ccdf90..a1b029adbe 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -173,8 +173,10 @@ extern int erts_dio_sched_thread_suggested_stack_size;
   (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 9))
 #define ERTS_RUNQ_FLG_HALTING \
   (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 10))
+#define ERTS_RUNQ_FLG_CHECKIO \
+  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 11))
 
-#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 11)
+#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 12)
 
 #define ERTS_RUNQ_FLGS_MIGRATION_QMASKS	\
   (ERTS_RUNQ_FLGS_EMIGRATE_QMASK	\
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 1444cee805..c681fa481f 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -46,11 +46,11 @@
 #if 0
 #define DEBUG_PRINT(FMT, ...) erts_printf(FMT "\r\n", ##__VA_ARGS__)
 #define DEBUG_PRINT_FD(FMT, STATE, ...)                                 \
-    DEBUG_PRINT("%d: " FMT " (ev=%s, ac=%s, flg=%d)",                   \
+    DEBUG_PRINT("%d: " FMT " (ev=%s, ac=%s, flg=%s)",                   \
                 (STATE) ? (STATE)->fd : (ErtsSysFdType)-1, ##__VA_ARGS__, \
                 ev2str((STATE) ? (STATE)->events : ERTS_POLL_EV_NONE),  \
                 ev2str((STATE) ? (STATE)->active_events : ERTS_POLL_EV_NONE), \
-                (STATE) ? (STATE)->flags : ERTS_EV_FLAG_CLEAR)
+                (STATE) ? flag2str((STATE)->flags) : ERTS_EV_FLAG_CLEAR)
 #define DEBUG_PRINT_MODE
 #else
 #define DEBUG_PRINT(...)
@@ -76,22 +76,40 @@ typedef enum {
 typedef enum {
     ERTS_EV_FLAG_CLEAR         = 0,
     ERTS_EV_FLAG_USED          = 1,   /* ERL_DRV_USE has been turned on */
-#ifdef ERTS_ENABLE_KERNEL_POLL
-    ERTS_EV_FLAG_FALLBACK      = 2,   /* Set when kernel poll rejected fd
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+    ERTS_EV_FLAG_SCHEDULER     = 2,   /* Set when the fd has been migrated
+                                         to scheduler pollset */
+    ERTS_EV_FLAG_IN_SCHEDULER  = 4,   /* Set when the fd is currently in
+                                         scheduler pollset */
+#else
+    ERTS_EV_FLAG_SCHEDULER     = ERTS_EV_FLAG_CLEAR,
+    ERTS_EV_FLAG_IN_SCHEDULER  = ERTS_EV_FLAG_CLEAR,
+#endif
+#ifdef ERTS_POLL_USE_FALLBACK
+    ERTS_EV_FLAG_FALLBACK      = 8,   /* Set when kernel poll rejected fd
                                          and it was put in the nkp version */
 #else
     ERTS_EV_FLAG_FALLBACK      = ERTS_EV_FLAG_CLEAR,
 #endif
 
     /* Combinations */
-    ERTS_EV_FLAG_USED_FALLBACK = ERTS_EV_FLAG_USED | ERTS_EV_FLAG_FALLBACK
+    ERTS_EV_FLAG_USED_FALLBACK = ERTS_EV_FLAG_USED | ERTS_EV_FLAG_FALLBACK,
+    ERTS_EV_FLAG_USED_SCHEDULER = ERTS_EV_FLAG_USED | ERTS_EV_FLAG_SCHEDULER,
+    ERTS_EV_FLAG_USED_IN_SCHEDULER = ERTS_EV_FLAG_USED | ERTS_EV_FLAG_SCHEDULER | ERTS_EV_FLAG_IN_SCHEDULER,
+    ERTS_EV_FLAG_UNUSED_SCHEDULER = ERTS_EV_FLAG_SCHEDULER,
+    ERTS_EV_FLAG_UNUSED_IN_SCHEDULER = ERTS_EV_FLAG_SCHEDULER | ERTS_EV_FLAG_IN_SCHEDULER
 } EventStateFlags;
 
 #define flag2str(flags)                                                 \
     ((flags) == ERTS_EV_FLAG_CLEAR ? "CLEAR" :                          \
      ((flags) == ERTS_EV_FLAG_USED ? "USED" :                           \
       ((flags) == ERTS_EV_FLAG_FALLBACK ? "FLBK" :                      \
-       ((flags) == ERTS_EV_FLAG_USED_FALLBACK ? "USED|FLBK" : "ERROR"))))
+       ((flags) == ERTS_EV_FLAG_USED_FALLBACK ? "USED|FLBK" :           \
+        ((flags) == ERTS_EV_FLAG_USED_SCHEDULER ? "USED|SCHD" :         \
+         ((flags) == ERTS_EV_FLAG_UNUSED_SCHEDULER ? "SCHD" :           \
+          ((flags) == ERTS_EV_FLAG_USED_IN_SCHEDULER ? "USED|IN_SCHD" : \
+           ((flags) == ERTS_EV_FLAG_UNUSED_IN_SCHEDULER ? "IN_SCHD" :   \
+            "ERROR"))))))))
 
 /* How many events that can be handled at once by one erts_poll_wait call */
 #define ERTS_CHECK_IO_POLL_RES_LEN 512
@@ -113,10 +131,13 @@ typedef struct erts_poll_thread
  * Which pollset to use is determined by hashing the fd.
  */
 static ErtsPollSet **pollsetv;
+static ErtsPollThread *psiv;
 #if ERTS_POLL_USE_FALLBACK
 static ErtsPollSet *flbk_pollset;
 #endif
-static ErtsPollThread *psiv;
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+static ErtsPollSet *sched_pollset;
+#endif
 
 typedef struct {
 #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
@@ -131,10 +152,12 @@ typedef struct {
             ErtsResource* resource;   /* ERTS_EV_TYPE_STOP_NIF */
         } stop;
     } driver;
-    ErtsPollEvents events; /* The events that have been selected upon */
+    ErtsPollEvents events;        /* The events that have been selected upon */
     ErtsPollEvents active_events; /* The events currently active in the pollset */
     EventStateType type;
     EventStateFlags flags;
+    int count;                    /* Number of times this fd has triggered
+                                     without being deselected. */
 } ErtsDrvEventState;
 
 struct drv_ev_state_shared {
@@ -371,12 +394,22 @@ get_pollset(ErtsSysFdType fd)
 
 #if ERTS_POLL_USE_FALLBACK
 static ERTS_INLINE ErtsPollSet *
-get_fallback(void)
+get_fallback_pollset(void)
 {
     return flbk_pollset;
 }
 #endif
 
+static ERTS_INLINE ErtsPollSet *
+get_scheduler_pollset(ErtsSysFdType fd)
+{
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+    return sched_pollset;
+#else
+    return get_pollset(fd);
+#endif
+}
+
 /*
  * Place a fd within a pollset. This will automatically use
  * the fallback ps if needed.
@@ -392,18 +425,27 @@ erts_io_control_wakeup(ErtsDrvEventState *state, ErtsPollOp op,
     ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd)));
 
     if (!(flags & ERTS_EV_FLAG_FALLBACK)) {
-        res = erts_poll_control(get_pollset(fd), fd, op, pe, wake_poller);
+
+        if (op == ERTS_POLL_OP_DEL && (flags & ERTS_EV_FLAG_SCHEDULER)) {
+            erts_poll_control(get_scheduler_pollset(fd), fd, op, pe, wake_poller);
+            flags &= ~ERTS_EV_FLAG_IN_SCHEDULER;
+        }
+        if (!(flags & ERTS_EV_FLAG_IN_SCHEDULER) || (pe & ERTS_POLL_EV_OUT)) {
+            res = erts_poll_control(get_pollset(fd), fd, op, pe, wake_poller);
+        } else {
+            res = erts_poll_control(get_scheduler_pollset(fd), fd, op, pe, wake_poller);
+        }
 
 #if ERTS_POLL_USE_FALLBACK
         if (op == ERTS_POLL_OP_ADD && res == ERTS_POLL_EV_NVAL) {
             /* When an add fails with NVAL, the poll/kevent operation could not
                put that fd in the pollset, so we instead put it into a fallback pollset */
             state->flags |= ERTS_EV_FLAG_FALLBACK;
-            res = erts_poll_control_flbk(get_fallback(), fd, op, pe, wake_poller);
+            res = erts_poll_control_flbk(get_fallback_pollset(), fd, op, pe, wake_poller);
         }
     } else {
         ASSERT(op != ERTS_POLL_OP_ADD);
-        res = erts_poll_control_flbk(get_fallback(), fd, op, pe, wake_poller);
+        res = erts_poll_control_flbk(get_fallback_pollset(), fd, op, pe, wake_poller);
 #endif
     }
 
@@ -426,7 +468,8 @@ erts_io_notify_port_task_executed(ErtsPortTaskType type,
     ErtsIoTask *itp = ErtsContainerStruct(pthp, ErtsIoTask, task);
     ErtsSysFdType fd = itp->fd;
     erts_mtx_t *mtx = fd_mtx(fd);
-    int active_events;
+    ErtsPollOp op = ERTS_POLL_OP_MOD;
+    int active_events, new_events = 0;
     ErtsDrvEventState *state;
     ErtsDrvSelectDataState *free_select = NULL;
     ErtsNifSelectDataState *free_nif = NULL;
@@ -436,51 +479,66 @@ erts_io_notify_port_task_executed(ErtsPortTaskType type,
     erts_mtx_lock(mtx);
     state = get_drv_ev_state(fd);
 
+    reset_handle(pthp);
+
     active_events = state->active_events;
 
-    switch (type) {
-    case ERTS_PORT_TASK_INPUT:
+    if (!(state->flags & ERTS_EV_FLAG_IN_SCHEDULER) || type == ERTS_PORT_TASK_OUTPUT) {
+        switch (type) {
+        case ERTS_PORT_TASK_INPUT:
+
+            DEBUG_PRINT_FD("executed ready_input", state);
+
+            ASSERT(!(state->active_events & ERTS_POLL_EV_IN));
+            if (state->events & ERTS_POLL_EV_IN) {
+                active_events |= ERTS_POLL_EV_IN;
+                if (state->count > 10 && ERTS_POLL_USE_SCHEDULER_POLLING) {
+                    if (!(state->flags & ERTS_EV_FLAG_SCHEDULER))
+                        op = ERTS_POLL_OP_ADD;
+                    state->flags |= ERTS_EV_FLAG_IN_SCHEDULER|ERTS_EV_FLAG_SCHEDULER;
+                    new_events = ERTS_POLL_EV_IN;
+                    DEBUG_PRINT_FD("moving to scheduler ps", state);
+                } else
+                    new_events = active_events;
+                if (!(state->flags & ERTS_EV_FLAG_FALLBACK) && ERTS_POLL_USE_SCHEDULER_POLLING)
+                    state->count++;
+            }
+            break;
+        case ERTS_PORT_TASK_OUTPUT:
 
-        DEBUG_PRINT_FD("executed ready_input", state);
+            DEBUG_PRINT_FD("executed ready_output", state);
 
-        ASSERT(!(state->active_events & ERTS_POLL_EV_IN));
-        if (state->events & ERTS_POLL_EV_IN)
-            active_events |= ERTS_POLL_EV_IN;
-        break;
-    case ERTS_PORT_TASK_OUTPUT:
+            ASSERT(!(state->active_events & ERTS_POLL_EV_OUT));
+            if (state->events & ERTS_POLL_EV_OUT) {
+                active_events |= ERTS_POLL_EV_OUT;
+                if (state->flags & ERTS_EV_FLAG_IN_SCHEDULER && active_events & ERTS_POLL_EV_IN)
+                    new_events = ERTS_POLL_EV_OUT;
+                else
+                    new_events = active_events;
+            }
+            break;
+        default:
+            erts_exit(ERTS_ABORT_EXIT, "Invalid IO port task type");
+            break;
+        }
 
-        DEBUG_PRINT_FD("executed ready_output", state);
+        if (state->active_events != active_events && new_events) {
+            state->active_events = active_events;
+            new_events = erts_io_control(state, op, new_events);
+        }
 
-        ASSERT(!(state->active_events & ERTS_POLL_EV_OUT));
-        if (state->events & ERTS_POLL_EV_OUT)
-            active_events |= ERTS_POLL_EV_OUT;
-        break;
-    default:
-        erts_exit(ERTS_ABORT_EXIT, "Invalid IO port task type");
-        break;
+        /* We were unable to re-insert the fd into the pollset, signal the callback. */
+        if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
+            if (state->active_events & ERTS_POLL_EV_IN)
+                iready(state->driver.select->inport, state);
+            if (state->active_events & ERTS_POLL_EV_OUT)
+                oready(state->driver.select->outport, state);
+            state->active_events = 0;
+        }
     }
 
-    reset_handle(pthp);
-
-    if (active_events) {
-        /* This is not needed if active_events has not changed */
-        if (state->active_events != active_events) {
-            ErtsPollEvents new_events;
-            state->active_events = active_events;
-            new_events = erts_io_control(state, ERTS_POLL_OP_MOD, active_events);
-
-            /* We were unable to re-insert the fd into the pollset, signal the callback. */
-            if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
-                if (active_events & ERTS_POLL_EV_IN)
-                    iready(state->driver.select->inport, state);
-                if (active_events & ERTS_POLL_EV_OUT)
-                    oready(state->driver.select->outport, state);
-                state->active_events = 0;
-            }
-        }
-    } else {
+    if (!active_events)
         check_fd_cleanup(state, &free_select, &free_nif);
-    }
 
     erts_mtx_unlock(mtx);
 
@@ -760,11 +818,22 @@ driver_select(ErlDrvPort ix, ErlDrvEvent e, int mode, int on)
         if (old_events == 0 && !(state->flags & ERTS_EV_FLAG_USED)) {
             ctl_op = ERTS_POLL_OP_ADD;
         }
+        new_events = state->active_events;
+        if (state->flags & ERTS_EV_FLAG_IN_SCHEDULER)
+            new_events &= ~ERTS_POLL_EV_IN;
     }
     else {
         ctl_events &= old_events;
         state->events &= ~ctl_events;
         state->active_events &= ~ctl_events;
+        new_events = state->active_events;
+
+        if (ctl_events & ERTS_POLL_EV_IN) {
+            state->count = 0;
+            if (state->flags & ERTS_EV_FLAG_IN_SCHEDULER) {
+                new_events = 0;
+            }
+        }
 
         if (!state->events) {
             if (!(state->flags & ERTS_EV_FLAG_USED) || mode & ERL_DRV_USE)
@@ -775,7 +844,7 @@ driver_select(ErlDrvPort ix, ErlDrvEvent e, int mode, int on)
     if (ctl_events || ctl_op == ERTS_POLL_OP_DEL) {
 
         new_events = erts_io_control_wakeup(state, ctl_op,
-                                            state->active_events,
+                                            new_events,
                                             &wake_poller);
 
         ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL || state->type == ERTS_EV_TYPE_NONE);
@@ -807,6 +876,7 @@ driver_select(ErlDrvPort ix, ErlDrvEvent e, int mode, int on)
             if (ctl_events & ERTS_POLL_EV_IN) {
                 abort_tasks(state, ERL_DRV_READ);
                 state->driver.select->inport = NIL;
+                state->flags &= ~ERTS_EV_FLAG_IN_SCHEDULER;
             }
             if (ctl_events & ERTS_POLL_EV_OUT) {
                 abort_tasks(state, ERL_DRV_WRITE);
@@ -815,6 +885,8 @@ driver_select(ErlDrvPort ix, ErlDrvEvent e, int mode, int on)
             if (state->events == 0) {
                 if ((mode & ERL_DRV_USE) || !(state->flags & ERTS_EV_FLAG_USED)) {
                     state->type = ERTS_EV_TYPE_NONE;
+                    if (state->flags & ERTS_EV_FLAG_SCHEDULER)
+                        erts_atomic32_read_bor_nob(&prt->state, ERTS_PORT_SFLG_CHECK_FD_CLEANUP);
                     state->flags = 0;
                 }
                 /*else keep it, as fd will probably be selected upon again */
@@ -1431,7 +1503,8 @@ iready(Eterm id, ErtsDrvEventState *state)
 	if (erts_port_task_schedule(id,
 				    &iotask->task,
 				    ERTS_PORT_TASK_INPUT,
-				    (ErlDrvEvent) state->fd) != 0) {
+				    (ErlDrvEvent) state->fd,
+                                    state->flags & ERTS_EV_FLAG_IN_SCHEDULER) != 0) {
 	    stale_drv_select(id, state, ERL_DRV_READ);
 	} else {
             DEBUG_PRINT_FD("schedule ready_input(%T, %d)",
@@ -1449,7 +1522,8 @@ oready(Eterm id, ErtsDrvEventState *state)
 	if (erts_port_task_schedule(id,
 				    &iotask->task,
 				    ERTS_PORT_TASK_OUTPUT,
-				    (ErlDrvEvent) state->fd) != 0) {
+				    (ErlDrvEvent) state->fd,
+                                    0) != 0) {
 	    stale_drv_select(id, state, ERL_DRV_WRITE);
 	} else {
             DEBUG_PRINT_FD("schedule ready_output(%T, %d)", state, id, state->fd);
@@ -1511,7 +1585,7 @@ erts_check_io_interrupt(ErtsPollThread *psi, int set)
 {
     if (psi) {
 #if ERTS_POLL_USE_FALLBACK
-        if (psi->ps == get_fallback()) {
+        if (psi->ps == get_fallback_pollset()) {
             erts_poll_interrupt_flbk(psi->ps, set);
             return;
         }
@@ -1527,7 +1601,7 @@ erts_create_pollset_thread(int id, ErtsThrPrgrData *tpd) {
 }
 
 void
-erts_check_io(ErtsPollThread *psi)
+erts_check_io(ErtsPollThread *psi, ErtsMonotonicTime timeout_time)
 {
     int pollres_len;
     int poll_ret, i;
@@ -1542,14 +1616,14 @@ erts_check_io(ErtsPollThread *psi)
     pollres_len = psi->pollres_len;
 
 #if ERTS_POLL_USE_FALLBACK
-    if (psi->ps == get_fallback()) {
+    if (psi->ps == get_fallback_pollset()) {
 
-        poll_ret = erts_poll_wait_flbk(psi->ps, psi->pollres, &pollres_len, psi->tpd);
+        poll_ret = erts_poll_wait_flbk(psi->ps, psi->pollres, &pollres_len, psi->tpd, timeout_time);
 
     } else
 #endif
     {
-        poll_ret = erts_poll_wait(psi->ps, psi->pollres, &pollres_len, psi->tpd);
+        poll_ret = erts_poll_wait(psi->ps, psi->pollres, &pollres_len, psi->tpd, timeout_time);
     }
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -1585,7 +1659,12 @@ erts_check_io(ErtsPollThread *psi)
         ErtsNifSelectDataState *free_nif = NULL;
 	ErtsSysFdType fd = (ErtsSysFdType) ERTS_POLL_RES_GET_FD(&psi->pollres[i]);
 	ErtsDrvEventState *state;
-        ErtsPollEvents revents;
+        ErtsPollEvents revents = ERTS_POLL_RES_GET_EVTS(&psi->pollres[i]);
+
+        /* The fd will be set to -1 if a pollset internal fd was triggered
+           that was determined to be too expensive to remove from the result.
+        */
+        if (fd == -1) continue;
 
 	erts_mtx_lock(fd_mtx(fd));
 
@@ -1596,8 +1675,6 @@ erts_check_io(ErtsPollThread *psi)
             continue;
 	}
 
-        revents = ERTS_POLL_RES_GET_EVTS(&psi->pollres[i]);
-
         DEBUG_PRINT_FD("triggered %s", state, ev2str(revents));
 
         if (revents & ERTS_POLL_EV_ERR) {
@@ -1609,25 +1686,39 @@ erts_check_io(ErtsPollThread *psi)
              */
             revents = state->active_events;
             state->active_events = 0;
+
+            if (state->flags & ERTS_EV_FLAG_IN_SCHEDULER) {
+                erts_io_control(state, ERTS_POLL_OP_MOD, 0);
+                state->flags &= ~ERTS_EV_FLAG_IN_SCHEDULER;
+            }
         } else {
 
             /* Disregard any events that are not active at the moment,
                for instance this could happen if the driver/nif does
                select/deselect in rapid succession. */
             revents &= state->active_events | ERTS_POLL_EV_NVAL;
-            state->active_events &= ~revents;
 
-            /* Reactivate the poll op if there are still active events */
-            if (state->active_events) {
-                ErtsPollEvents new_events;
-                DEBUG_PRINT_FD("re-enable %s", state, ev2str(state->active_events));
+            if (psi->ps != get_scheduler_pollset(fd) || !ERTS_POLL_USE_SCHEDULER_POLLING) {
+                ErtsPollEvents reactive_events;
+                state->active_events &= ~revents;
+
+                reactive_events = state->active_events;
 
-                new_events = erts_io_control(state, ERTS_POLL_OP_MOD, state->active_events);
+                if (state->flags & ERTS_EV_FLAG_IN_SCHEDULER)
+                    reactive_events &= ~ERTS_POLL_EV_IN;
 
-                /* Unable to re-enable the fd, signal all callbacks */
-                if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
-                    revents |= state->active_events;
-                    state->active_events = 0;
+                /* Reactivate the poll op if there are still active events */
+                if (reactive_events) {
+                    ErtsPollEvents new_events;
+                    DEBUG_PRINT_FD("re-enable %s", state, ev2str(reactive_events));
+
+                    new_events = erts_io_control(state, ERTS_POLL_OP_MOD, reactive_events);
+
+                    /* Unable to re-enable the fd, signal all callbacks */
+                    if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) {
+                        revents |= reactive_events;
+                        state->active_events &= ~reactive_events;
+                    }
                 }
             }
         }
@@ -1703,7 +1794,7 @@ erts_check_io(ErtsPollThread *psi)
 
         case ERTS_EV_TYPE_STOP_USE: {
 #if ERTS_POLL_USE_FALLBACK
-            ASSERT(psi->ps == get_fallback());
+            ASSERT(psi->ps == get_fallback_pollset());
 #endif
             drv_ptr = state->driver.stop.drv_ptr;
             state->type = ERTS_EV_TYPE_NONE;
@@ -2041,12 +2132,17 @@ erts_init_check_io(int *argc, char **argv)
     for (j=0; j < erts_no_pollsets; j++)
         pollsetv[j] = erts_poll_create_pollset(j);
 
-#if ERTS_POLL_USE_FALLBACK
-    flbk_pollset = erts_poll_create_pollset_flbk(-1);
+    no_poll_threads = erts_no_poll_threads;
+
+    j = -1;
+
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+    sched_pollset = erts_poll_create_pollset(j--);
+    no_poll_threads++;
 #endif
 
-    no_poll_threads = erts_no_poll_threads;
 #if ERTS_POLL_USE_FALLBACK
+    flbk_pollset = erts_poll_create_pollset_flbk(j--);
     no_poll_threads++;
 #endif
 
@@ -2056,7 +2152,15 @@ erts_init_check_io(int *argc, char **argv)
     psiv[0].pollres_len = ERTS_CHECK_IO_POLL_RES_LEN;
     psiv[0].pollres = erts_alloc(ERTS_ALC_T_POLLSET,
         sizeof(ErtsPollResFd) * ERTS_CHECK_IO_POLL_RES_LEN);
-    psiv[0].ps = get_fallback();
+    psiv[0].ps = get_fallback_pollset();
+    psiv++;
+#endif
+
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+    psiv[0].pollres_len = ERTS_CHECK_IO_POLL_RES_LEN;
+    psiv[0].pollres = erts_alloc(ERTS_ALC_T_POLLSET,
+        sizeof(ErtsPollResFd) * ERTS_CHECK_IO_POLL_RES_LEN);
+    psiv[0].ps = get_scheduler_pollset(0);
     psiv++;
 #endif
 
@@ -2113,7 +2217,12 @@ erts_check_io_size(void)
     int i;
 
 #if ERTS_POLL_USE_FALLBACK
-    erts_poll_info(get_fallback(), &pi);
+    erts_poll_info(get_fallback_pollset(), &pi);
+    res += pi.memory_size;
+#endif
+
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+    erts_poll_info(get_scheduler_pollset(0), &pi);
     res += pi.memory_size;
 #endif
 
@@ -2145,13 +2254,21 @@ erts_check_io_info(void *proc)
     Uint sz, *szp, *hp, **hpp;
     ErtsPollInfo *piv;
     Sint i, j = 0, len;
-    int no_pollsets = erts_no_pollsets + ERTS_POLL_USE_FALLBACK;
+    int no_pollsets = erts_no_pollsets + ERTS_POLL_USE_FALLBACK + ERTS_POLL_USE_SCHEDULER_POLLING;
     ERTS_CT_ASSERT(ERTS_POLL_USE_FALLBACK == 0 || ERTS_POLL_USE_FALLBACK == 1);
+    ERTS_CT_ASSERT(ERTS_POLL_USE_SCHEDULER_POLLING == 0 || ERTS_POLL_USE_SCHEDULER_POLLING == 1);
 
     piv = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollInfo) * no_pollsets);
 
 #if ERTS_POLL_USE_FALLBACK
-    erts_poll_info_flbk(get_fallback(), &piv[0]);
+    erts_poll_info_flbk(get_fallback_pollset(), &piv[0]);
+    piv[0].poll_threads = 1;
+    piv[0].active_fds = 0;
+    piv++;
+#endif
+
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+    erts_poll_info(get_scheduler_pollset(0), &piv[0]);
     piv[0].poll_threads = 1;
     piv[0].active_fds = 0;
     piv++;
@@ -2205,6 +2322,7 @@ erts_check_io_info(void *proc)
     sz = 0;
 
     piv -= ERTS_POLL_USE_FALLBACK;
+    piv -= ERTS_POLL_USE_SCHEDULER_POLLING;
 
  bld_it:
 
@@ -2309,15 +2427,7 @@ print_events(erts_dsprintf_buf_t *dsbufp, ErtsPollEvents ev)
 static ERTS_INLINE void
 print_flags(erts_dsprintf_buf_t *dsbufp, EventStateFlags f)
 {
-    const char* delim = "";
-    if(f & ERTS_EV_FLAG_USED) {
-	erts_dsprintf(dsbufp, "%s","USED");
-	delim = "|";
-    }
-    if(f & ERTS_EV_FLAG_FALLBACK) {
-	erts_dsprintf(dsbufp, "%s%s", delim, "FLBK");
-        delim = "|";
-    }
+    erts_dsprintf(dsbufp, "%s", flag2str(f));
 }
 
 #ifdef DEBUG_PRINT_MODE
@@ -2659,13 +2769,26 @@ erts_check_io_debug(ErtsCheckIoDebugInfo *ciodip)
 
 #if ERTS_POLL_USE_FALLBACK
     erts_dsprintf(dsbufp, "--- fds in flbk pollset ---------------------------------\n");
-    erts_poll_get_selected_events_flbk(get_fallback(), counters.epep,
+    erts_poll_get_selected_events_flbk(get_fallback_pollset(), counters.epep,
                                        drv_ev_state.max_fds);
     for (fd = 0; fd < len; fd++) {
         if (drv_ev_state.v[fd].flags & ERTS_EV_FLAG_FALLBACK)
             doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters, dsbufp);
     }
 #endif
+#if ERTS_POLL_USE_SCHEDULER_POLLING
+    erts_dsprintf(dsbufp, "--- fds in scheduler pollset ----------------------------\n");
+    erts_poll_get_selected_events(get_scheduler_pollset(0), counters.epep,
+                                  drv_ev_state.max_fds);
+    for (fd = 0; fd < len; fd++) {
+        if (drv_ev_state.v[fd].flags & ERTS_EV_FLAG_SCHEDULER) {
+            if (drv_ev_state.v[fd].events && drv_ev_state.v[fd].events != ERTS_POLL_EV_NONE)
+                counters.epep[fd] &= ~ERTS_POLL_EV_OUT;
+            doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters, dsbufp);
+        }
+    }
+#endif
+
     erts_dsprintf(dsbufp, "--- fds in pollset --------------------------------------\n");
 
     for (i = 0; i < erts_no_pollsets; i++) {
@@ -2674,8 +2797,15 @@ erts_check_io_debug(ErtsCheckIoDebugInfo *ciodip)
                                       drv_ev_state.max_fds);
         for (fd = 0; fd < len; fd++) {
             if (!(drv_ev_state.v[fd].flags & ERTS_EV_FLAG_FALLBACK)
-                && get_pollset_id(fd) == i)
+                && get_pollset_id(fd) == i) {
+                if (counters.epep[fd] != ERTS_POLL_EV_NONE &&
+                    drv_ev_state.v[fd].flags & ERTS_EV_FLAG_IN_SCHEDULER) {
+                    /* We add the in flag if it is enabled in the scheduler pollset
+                       and get_selected_events works on the platform */
+                    counters.epep[fd] |= ERTS_POLL_EV_IN;
+                }
                 doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters, dsbufp);
+            }
         }
     }
     for (fd = len ; fd < drv_ev_state.max_fds; fd++) {
@@ -2722,7 +2852,7 @@ void erts_lcnt_update_cio_locks(int enable) {
 #endif
 
 #if ERTS_POLL_USE_FALLBACK
-    erts_lcnt_enable_pollset_lock_count_flbk(get_fallback(), enable);
+    erts_lcnt_enable_pollset_lock_count_flbk(get_fallback_pollset(), enable);
 #endif
 
     for (i = 0; i < erts_no_pollsets; i++)
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index 16aba8a5f3..31182be5ec 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -68,7 +68,7 @@ int erts_check_io_max_files(void);
  *
  * @param pt the poll thread structure to use.
  */
-void erts_check_io(struct erts_poll_thread *pt);
+void erts_check_io(struct erts_poll_thread *pt, ErtsMonotonicTime timeout_time);
 /**
  * Initialize the check io framework. This function will parse the arguments
  * and delete any entries that it is interested in.
@@ -129,16 +129,6 @@ extern int erts_no_poll_threads;
 #include "erl_poll.h"
 #include "erl_port_task.h"
 
-#ifdef __WIN32__
-/*
- * Current erts_poll implementation for Windows cannot handle
- * active events in the set of events polled.
- */
-#  define ERTS_CIO_DEFER_ACTIVE_EVENTS 1
-#else
-#  define ERTS_CIO_DEFER_ACTIVE_EVENTS 1
-#endif
-
 typedef struct {
     Eterm inport;
     Eterm outport;
@@ -150,10 +140,6 @@ struct erts_nif_select_event {
     Eterm pid;
     Eterm immed;
     Uint32 refn[ERTS_REF_NUMBERS];
-    Sint32 ddeselect_cnt; /* 0:  No delayed deselect in progress
-                           * 1:  Do deselect before next poll
-                           * >1: Countdown of ignored events
-                           */
 };
 
 typedef struct {
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index 5a5874ddfa..51d50933ff 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -121,7 +121,8 @@
 /* Define to print info about modifications done to each fd */
 #define DEBUG_PRINT_FD(FMT, PS, FD, ...) DEBUG_PRINT("%d: " FMT, PS, FD, ##__VA_ARGS__)
 /* Define to print entry and exit from erts_poll_wait (can be very spammy) */
-//#define DEBUG_PRINT_WAIT(FMT, PS, ...) DEBUG_PRINT(FMT, PS, ##__VA_ARGS__)
+// #define DEBUG_PRINT_WAIT(FMT, PS, ...) DEBUG_PRINT(FMT, PS, ##__VA_ARGS__)
+// #define DEBUG_PRINT_WAIT(FMT, PS, ...) do { if ((PS)->id != -1) DEBUG_PRINT(FMT, PS, ##__VA_ARGS__); } while(0)
 
 #else
 #define ERTS_POLL_DEBUG_PRINT 0
@@ -200,7 +201,7 @@ int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds,
 
 #define ERTS_POLL_USE_CONCURRENT_UPDATE (ERTS_POLL_USE_EPOLL || ERTS_POLL_USE_KQUEUE)
 
-#define ERTS_POLL_USE_WAKEUP_PIPE (!ERTS_POLL_USE_CONCURRENT_UPDATE)
+#define ERTS_POLL_USE_WAKEUP(ps) (!ERTS_POLL_USE_CONCURRENT_UPDATE || (ps)->id < 0)
 
 #if !ERTS_POLL_USE_CONCURRENT_UPDATE
 
@@ -269,6 +270,7 @@ struct ERTS_POLL_EXPORT(erts_pollset) {
 
 #if ERTS_POLL_USE_KERNEL_POLL
     int kp_fd;
+    int oneshot;
 #endif /* ERTS_POLL_USE_KERNEL_POLL */
 
 #if ERTS_POLL_USE_POLL
@@ -295,12 +297,16 @@ struct ERTS_POLL_EXPORT(erts_pollset) {
     ErtsPollSetUpdateRequestsBlock *curr_upd_req_block;
     erts_atomic32_t have_update_requests;
     erts_mtx_t mtx;
-    erts_atomic32_t wakeup_state;
+#else
+    int do_wakeup;
 #endif
 
-#if ERTS_POLL_USE_WAKEUP_PIPE
-    int wake_fds[2];
+#if ERTS_POLL_USE_TIMERFD
+    int timer_fd;
 #endif
+    ErtsMonotonicTime timeout_time;
+    erts_atomic32_t wakeup_state;
+    int wake_fds[2];
 };
 
 void erts_silence_warn_unused_result(long unused);
@@ -365,63 +371,47 @@ static void print_misc_debug_info(void);
 uint32_t epoll_events(int kp_fd, int fd);
 #endif
 
-
 #define ERTS_POLL_NOT_WOKEN	0
 #define ERTS_POLL_WOKEN		-1
 #define ERTS_POLL_WOKEN_INTR	1
 
-#if !ERTS_POLL_USE_CONCURRENT_UPDATE
 static ERTS_INLINE void
 reset_wakeup_state(ErtsPollSet *ps)
 {
     erts_atomic32_set_mb(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
 }
-#endif
 
 static ERTS_INLINE int
 is_woken(ErtsPollSet *ps)
 {
-#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     return erts_atomic32_read_acqb(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN;
-#else
-    return 0;
-#endif
 }
 
 static ERTS_INLINE int
 is_interrupted_reset(ErtsPollSet *ps)
 {
-#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     return (erts_atomic32_xchg_acqb(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN)
 	    == ERTS_POLL_WOKEN_INTR);
-#else
-    return 0;
-#endif
 }
 
 static ERTS_INLINE void
 woke_up(ErtsPollSet *ps)
 {
-#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     erts_aint32_t wakeup_state = erts_atomic32_read_acqb(&ps->wakeup_state);
     if (wakeup_state == ERTS_POLL_NOT_WOKEN)
 	(void) erts_atomic32_cmpxchg_nob(&ps->wakeup_state,
 					 ERTS_POLL_WOKEN,
 					 ERTS_POLL_NOT_WOKEN);
     ASSERT(erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN);
-#endif
 }
 
 /*
  * --- Wakeup pipe -----------------------------------------------------------
  */
 
-#if ERTS_POLL_USE_WAKEUP_PIPE
-
 static ERTS_INLINE void
 wake_poller(ErtsPollSet *ps, int interrupted)
 {
-#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     int wake;
     erts_aint32_t wakeup_state;
     if (!interrupted)
@@ -434,9 +424,9 @@ wake_poller(ErtsPollSet *ps, int interrupted)
     wake = wakeup_state == ERTS_POLL_NOT_WOKEN;
 
     if (wake)
-#endif
     {
 	ssize_t res;
+        DEBUG_PRINT_WAIT("wake_poller(%d)", ps, interrupted);
 	if (ps->wake_fds[1] < 0)
 	    return; /* Not initialized yet */
 	do {
@@ -474,10 +464,8 @@ cleanup_wakeup_pipe(ErtsPollSet *ps)
 		    fd,
 		    erl_errno_id(errno), errno);
     }
-#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     if (intr)
 	erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_INTR);
-#endif
 }
 
 static void
@@ -513,7 +501,67 @@ create_wakeup_pipe(ErtsPollSet *ps)
     ps->wake_fds[1] = wake_fds[1];
 }
 
+/*
+ * --- timer fd -----------------------------------------------------------
+ */
+
+#if ERTS_POLL_USE_TIMERFD
+
+/* We use the timerfd when using epoll_wait to get high accuracy
+   timeouts, i.e. we want to sleep with < ms accuracy. */
+
+static void
+create_timerfd(ErtsPollSet *ps)
+{
+    int do_wake = 0;
+    int timer_fd = timerfd_create(CLOCK_MONOTONIC,0);
+    ERTS_POLL_EXPORT(erts_poll_control)(ps,
+					timer_fd,
+                                        ERTS_POLL_OP_ADD,
+					ERTS_POLL_EV_IN,
+                                        &do_wake);
+    if (ps->internal_fd_limit <= timer_fd)
+	ps->internal_fd_limit = timer_fd + 1;
+    ps->timer_fd = timer_fd;
+}
+
+static ERTS_INLINE void
+timerfd_set(ErtsPollSet *ps, struct itimerspec *its)
+{
+#ifdef DEBUG
+    struct itimerspec old_its;
+    int res;
+    res = timerfd_settime(ps->timer_fd, 0, its, &old_its);
+    ASSERT(res == 0);
+    ASSERT(old_its.it_interval.tv_sec == 0 &&
+           old_its.it_interval.tv_nsec == 0 &&
+           old_its.it_value.tv_sec == 0 &&
+           old_its.it_value.tv_nsec == 0);
+
+#else
+    timerfd_settime(ps->timer_fd, 0, its, NULL);
 #endif
+}
+
+static ERTS_INLINE int
+timerfd_clear(ErtsPollSet *ps, ErtsPollResFd pr[], int res, int max_res) {
+
+    struct itimerspec its;
+    /* we always have to clear the timer */
+    its.it_interval.tv_sec = 0;
+    its.it_interval.tv_nsec = 0;
+    its.it_value.tv_sec = 0;
+    its.it_value.tv_nsec = 0;
+    timerfd_settime(ps->timer_fd, 0, &its, NULL);
+
+    /* only timeout fd triggered */
+    if (res == 1 && pr[0].data.fd == ps->timer_fd)
+        return 0;
+
+    return res;
+}
+
+#endif /* ERTS_POLL_USE_TIMERFD */
 
 /*
  * --- Poll set update requests ----------------------------------------------
@@ -691,9 +739,12 @@ update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events)
     struct epoll_event epe_templ;
     struct epoll_event epe;
 
-    epe_templ.events = ERTS_POLL_EV_E2N(events) | EPOLLONESHOT;
+    epe_templ.events = ERTS_POLL_EV_E2N(events);
     epe_templ.data.fd = fd;
 
+    if (ps->oneshot)
+        epe_templ.events |= EPOLLONESHOT;
+
 #ifdef VALGRIND
     /* Silence invalid valgrind warning ... */
     memset((void *) &epe.data, 0, sizeof(epoll_data_t));
@@ -802,6 +853,7 @@ update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events)
     int res = 0, len = 0;
     struct kevent evts[2];
     struct timespec ts = {0, 0};
+    uint32_t oneshot = 0;
 
     if (op == ERTS_POLL_OP_ADD) {
         /* This is a hack to make the "noshell" option work; kqueue can poll
@@ -840,6 +892,9 @@ update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events)
        man page), but it seems to be the way it works...
     */
 
+    if (ps->oneshot)
+        oneshot = EV_DISPATCH;
+
     if (op == ERTS_POLL_OP_DEL) {
         erts_atomic_dec_nob(&ps->no_of_user_fds);
         /* We could probably skip this delete, do we want to? */
@@ -849,27 +904,29 @@ update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events)
         uint32_t flags;
         erts_atomic_inc_nob(&ps->no_of_user_fds);
 
-        flags = EV_ADD|EV_DISPATCH;
+        flags = EV_ADD|oneshot;
         flags |= ((events & ERTS_POLL_EV_IN) ? 0 : EV_DISABLE);
         ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, flags, (void *) ERTS_POLL_EV_IN);
 
-        flags = EV_ADD|EV_DISPATCH;
+        flags = EV_ADD|oneshot;
         flags |= ((events & ERTS_POLL_EV_OUT) ? 0 : EV_DISABLE);
         ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, flags, (void *) ERTS_POLL_EV_OUT);
     } else {
         uint32_t flags;
         ASSERT(op == ERTS_POLL_OP_MOD);
 
-        flags = EV_DISPATCH;
+        flags = oneshot;
         flags |= (events & ERTS_POLL_EV_IN) ? EV_ENABLE : EV_DISABLE;
         ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, flags, (void *) ERTS_POLL_EV_IN);
 
-        flags = EV_DISPATCH;
+        flags = oneshot;
         flags |= (events & ERTS_POLL_EV_OUT) ? EV_ENABLE : EV_DISABLE;
         ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, flags, (void *) ERTS_POLL_EV_OUT);
     }
 #else
-    uint32_t flags = EV_ADD|EV_ONESHOT;
+    uint32_t flags = EV_ADD;
+
+    if (ps->oneshot) flags |= EV_ONESHOT;
 
     if (op == ERTS_POLL_OP_DEL) {
         erts_atomic_dec_nob(&ps->no_of_user_fds);
@@ -903,14 +960,17 @@ update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events)
         keventbp += sprintf(keventbp, "kevent(%d, {",ps->kp_fd);
         for (i = 0; i < len; i++) {
             const char *flags = "UNKNOWN";
-            if (evts[i].flags == EV_DELETE) flags = "EV_DELETE";
+            if (evts[i].flags == (EV_DELETE)) flags = "EV_DELETE";
             if (evts[i].flags == (EV_ADD|EV_ONESHOT)) flags = "EV_ADD|EV_ONESHOT";
+            if (evts[i].flags == (EV_ADD)) flags = "EV_ADD";
 #ifdef EV_DISPATCH
             if (evts[i].flags == (EV_ADD|EV_DISPATCH)) flags = "EV_ADD|EV_DISPATCH";
             if (evts[i].flags == (EV_ADD|EV_DISABLE)) flags = "EV_ADD|EV_DISABLE";
             if (evts[i].flags == (EV_ENABLE|EV_DISPATCH)) flags = "EV_ENABLE|EV_DISPATCH";
-            if (evts[i].flags == EV_DISABLE) flags = "EV_DISABLE";
+            if (evts[i].flags == (EV_ENABLE)) flags = "EV_ENABLE";
+            if (evts[i].flags == (EV_DISABLE)) flags = "EV_DISABLE";
             if (evts[i].flags == (EV_DISABLE|EV_DISPATCH)) flags = "EV_DISABLE|EV_DISABLE";
+            if (evts[i].flags == (EV_DISABLE)) flags = "EV_DISABLE";
 #endif
 
             keventbp += sprintf(keventbp, "%s{%lu, %s, %s}",i > 0 ? ", " : "",
@@ -1273,11 +1333,15 @@ poll_control(ErtsPollSet *ps, int fd, ErtsPollOp op,
 	    goto done;
 	}
 #endif
-#if ERTS_POLL_USE_WAKEUP_PIPE
 	if (fd == ps->wake_fds[0] || fd == ps->wake_fds[1]) {
 	    new_events = ERTS_POLL_EV_NVAL;
 	    goto done;
 	}
+#if ERTS_POLL_USE_TIMERFD
+        if (fd == ps->timer_fd) {
+	    new_events = ERTS_POLL_EV_NVAL;
+            goto done;
+        }
 #endif
     }
 
@@ -1333,11 +1397,8 @@ ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet *ps,
 
     ERTS_POLLSET_UNLOCK(ps);
 
-#if !ERTS_POLL_USE_CONCURRENT_UPDATE
-    if (*do_wake) {
+    if (*do_wake)
 	wake_poller(ps, 0);
-    }
-#endif
 
     return res;
 }
@@ -1351,52 +1412,61 @@ ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet *ps,
 static ERTS_INLINE int
 ERTS_POLL_EXPORT(save_result)(ErtsPollSet *ps, ErtsPollResFd pr[], int max_res, int chk_fds_res, int ebadf)
 {
-#if !ERTS_POLL_USE_CONCURRENT_UPDATE || ERTS_POLL_DEBUG_PRINT || ERTS_POLL_USE_WAKEUP_PIPE
     int n = chk_fds_res < max_res ? chk_fds_res : max_res, i;
     int res = n;
-#if ERTS_POLL_USE_WAKEUP_PIPE
     int wake_fd = ps->wake_fds[0];
-#endif
 
-    for (i = 0; i < n; i++) {
-        int fd = ERTS_POLL_RES_GET_FD(&pr[i]);
-#ifdef DEBUG_PRINT_MODE
-        ErtsPollEvents evts = ERTS_POLL_RES_GET_EVTS(pr+i);
-#endif
+    if (ERTS_POLL_USE_WAKEUP(ps) || ERTS_POLL_DEBUG_PRINT || ERTS_POLL_USE_TIMERFD) {
 
-        DEBUG_PRINT_FD("trig %s (%s)", ps, fd,
-                       ev2str(evts),
+        for (i = 0; i < n; i++) {
+            int fd = ERTS_POLL_RES_GET_FD(&pr[i]);
+#if ERTS_POLL_DEBUG_PRINT
+            ErtsPollEvents evts = ERTS_POLL_RES_GET_EVTS(pr+i);
+
+            if (fd != wake_fd
+#if ERTS_POLL_USE_TIMERFD
+                && fd != ps->timer_fd
+#endif
+                )
+                DEBUG_PRINT_FD("trig %s (%s)", ps, fd,
+                               ev2str(evts),
 #if ERTS_POLL_USE_KQUEUE
-                       "kqueue"
+                               "kqueue"
 #elif ERTS_POLL_USE_EPOLL
-                       "epoll"
+                               "epoll"
 #else
-                       "/dev/poll"
+                               "/dev/poll"
+#endif
+                    );
 #endif
-            );
 
-#if ERTS_POLL_USE_WAKEUP_PIPE
-        if (fd == wake_fd) {
-            cleanup_wakeup_pipe(ps);
-            ERTS_POLL_RES_SET_EVTS(&pr[i], ERTS_POLL_EV_NONE);
-            if (n == 1)
-                return 0;
-        }
+            if (ERTS_POLL_USE_WAKEUP(ps) && fd == wake_fd) {
+                cleanup_wakeup_pipe(ps);
+                ERTS_POLL_RES_SET_FD(&pr[i], -1);
+                ERTS_POLL_RES_SET_EVTS(&pr[i], ERTS_POLL_EV_NONE);
+                res--;
+            }
+#if ERTS_POLL_USE_TIMERFD
+            else if (fd == ps->timer_fd) {
+                ERTS_POLL_RES_SET_FD(&pr[i], -1);
+                ERTS_POLL_RES_SET_EVTS(&pr[i], ERTS_POLL_EV_NONE);
+                res--;
+            }
 #endif
 #if !ERTS_POLL_USE_CONCURRENT_UPDATE
-        else {
-            /* Reset the events to emulate ONESHOT semantics */
-            ps->fds_status[fd].events = 0;
-            enqueue_update_request(ps, fd);
-        }
+            else {
+                /* Reset the events to emulate ONESHOT semantics */
+                ps->fds_status[fd].events = 0;
+                enqueue_update_request(ps, fd);
+            }
 #endif
+        }
     }
 
-    return res;
-#else
-    ASSERT(chk_fds_res <= max_res);
-    return chk_fds_res;
-#endif
+    if (res == 0)
+        return res;
+    else
+        return n;
 }
 
 #else /* !ERTS_POLL_USE_KERNEL_POLL */
@@ -1577,19 +1647,168 @@ ERTS_POLL_EXPORT(save_result)(ErtsPollSet *ps, ErtsPollResFd pr[], int max_res,
 
 #endif /* !ERTS_POLL_USE_KERNEL_POLL */
 
+static ERTS_INLINE ErtsMonotonicTime
+get_timeout(ErtsPollSet *ps,
+	    int resolution,
+	    ErtsMonotonicTime timeout_time)
+{
+    ErtsMonotonicTime timeout;
+
+    if (timeout_time == ERTS_POLL_NO_TIMEOUT) {
+	timeout = 0;
+    }
+    else if (timeout_time == ERTS_POLL_INF_TIMEOUT) {
+        timeout = -1;
+    }
+    else {
+	ErtsMonotonicTime diff_time, current_time;
+	current_time = erts_get_monotonic_time(NULL);
+	diff_time = timeout_time - current_time;
+	if (diff_time <= 0) {
+	    timeout = 0;
+	}
+	else {
+	    switch (resolution) {
+	    case 1000:
+		/* Round up to nearest even milli second */
+		timeout = ERTS_MONOTONIC_TO_MSEC(diff_time - 1) + 1;
+		if (timeout > (ErtsMonotonicTime) INT_MAX)
+		    timeout = (ErtsMonotonicTime) INT_MAX;
+		timeout -= ERTS_PREMATURE_TIMEOUT(timeout, 1000);
+		break;
+	    case 1000000:
+		/* Round up to nearest even micro second */
+		timeout = ERTS_MONOTONIC_TO_USEC(diff_time - 1) + 1;
+		timeout -= ERTS_PREMATURE_TIMEOUT(timeout, 1000*1000);
+		break;
+	    case 1000000000:
+		/* Round up to nearest even nano second */
+		timeout = ERTS_MONOTONIC_TO_NSEC(diff_time - 1) + 1;
+		timeout -= ERTS_PREMATURE_TIMEOUT(timeout, 1000*1000*1000);
+		break;
+	    default:
+		ERTS_INTERNAL_ERROR("Invalid resolution");
+		timeout = 0;
+		break;
+	    }
+	}
+    }
+    return timeout;
+}
+
+#if ERTS_POLL_USE_SELECT
+
+static ERTS_INLINE int
+get_timeout_timeval(ErtsPollSet *ps,
+		    SysTimeval *tvp,
+		    ErtsMonotonicTime timeout_time)
+{
+    ErtsMonotonicTime timeout = get_timeout(ps,
+					    1000*1000,
+					    timeout_time);
+
+    if (!timeout) {
+	tvp->tv_sec = 0;
+	tvp->tv_usec = 0;
+
+	return 0;
+    }
+    else if (timeout == -1) {
+        return -1;
+    }
+    else {
+	ErtsMonotonicTime sec = timeout/(1000*1000);
+	tvp->tv_sec = sec;
+	tvp->tv_usec = timeout - sec*(1000*1000);
+
+	ASSERT(tvp->tv_sec >= 0);
+	ASSERT(tvp->tv_usec >= 0);
+	ASSERT(tvp->tv_usec < 1000*1000);
+
+	return 1;
+    }
+
+}
+
+#endif
+
+#if ERTS_POLL_USE_KQUEUE || (ERTS_POLL_USE_POLL && defined(HAVE_PPOLL)) || ERTS_POLL_USE_TIMERFD
+
 static ERTS_INLINE int
-check_fd_events(ErtsPollSet *ps, ErtsPollResFd pr[], int do_wait, int max_res)
+get_timeout_timespec(ErtsPollSet *ps,
+		     struct timespec *tsp,
+		     ErtsMonotonicTime timeout_time)
+{
+    ErtsMonotonicTime timeout = get_timeout(ps,
+					    1000*1000*1000,
+					    timeout_time);
+
+    if (!timeout) {
+	tsp->tv_sec = 0;
+	tsp->tv_nsec = 0;
+	return 0;
+    }
+    else if (timeout == -1) {
+        return -1;
+    }
+    else {
+	ErtsMonotonicTime sec = timeout/(1000*1000*1000);
+	tsp->tv_sec = sec;
+	tsp->tv_nsec = timeout - sec*(1000*1000*1000);
+
+	ASSERT(tsp->tv_sec >= 0);
+	ASSERT(tsp->tv_nsec >= 0);
+	ASSERT(tsp->tv_nsec < 1000*1000*1000);
+
+	return 1;
+    }
+}
+
+#endif
+
+#if ERTS_POLL_USE_TIMERFD
+
+static ERTS_INLINE int
+get_timeout_itimerspec(ErtsPollSet *ps,
+                       struct itimerspec *itsp,
+                       ErtsMonotonicTime timeout_time)
+{
+
+    itsp->it_interval.tv_sec = 0;
+    itsp->it_interval.tv_nsec = 0;
+
+    return get_timeout_timespec(ps, &itsp->it_value, timeout_time);
+}
+ 
+#endif
+
+static ERTS_INLINE int
+check_fd_events(ErtsPollSet *ps, ErtsPollResFd pr[], int max_res, ErtsMonotonicTime timeout_time)
 {
     int res;
-    int timeout = do_wait ? -1 : 0;
-    DEBUG_PRINT_WAIT("Entering check_fd_events(), do_wait=%d", ps, do_wait);
+    int timeout;
+    DEBUG_PRINT_WAIT("Entering check_fd_events(), timeout=%d", ps, timeout_time);
     {
 #if ERTS_POLL_USE_EPOLL                /* --- epoll ------------------------------- */
+#if ERTS_POLL_USE_TIMERFD
+        struct itimerspec its;
+        timeout = get_timeout_itimerspec(ps, &its, timeout_time);
+        if (timeout > 0) {
+            timerfd_set(ps, &its);
+            res = epoll_wait(ps->kp_fd, pr, max_res, -1);
+            res = timerfd_clear(ps, pr, res, max_res);
+        } else {
+            res = epoll_wait(ps->kp_fd, pr, max_res, timeout);
+        }
+#else /* !ERTS_POLL_USE_TIMERFD */
+        timeout = (int) get_timeout(ps, 1000, timeout_time);
         res = epoll_wait(ps->kp_fd, pr, max_res, timeout);
-
+#endif /* !ERTS_POLL_USE_TIMERFD */
 #elif ERTS_POLL_USE_KQUEUE     /* --- kqueue ------------------------------ */
-        struct timespec ts = {0, 0};
-        struct timespec *tsp = timeout ? NULL : &ts;
+        struct timespec ts;
+        struct timespec *tsp;
+        timeout = get_timeout_timespec(ps, &ts, timeout_time);
+        tsp = timeout < 0 ? NULL : &ts;
         res = kevent(ps->kp_fd, NULL, 0, pr, max_res, tsp);
 #elif ERTS_POLL_USE_DEVPOLL	/* --- devpoll ----------------------------- */
         /*
@@ -1601,16 +1820,22 @@ check_fd_events(ErtsPollSet *ps, ErtsPollResFd pr[], int do_wait, int max_res)
         int nfds = (int) erts_atomic_read_nob(&ps->no_of_user_fds) + 1 /* wakeup pipe */;
         poll_res.dp_nfds = nfds < max_res ? nfds : max_res;
         poll_res.dp_fds = pr;
-        poll_res.dp_timeout = timeout;
+        poll_res.dp_timeout = (int) get_timeout(ps, 1000, timeout_time);
         res = ioctl(ps->kp_fd, DP_POLL, &poll_res);
-
+#elif ERTS_POLL_USE_POLL && defined(HAVE_PPOLL)	/* --- ppoll ---------------- */
+        struct timespec ts;
+        struct timespec *tsp = &ts;
+        timeout = get_timeout_timespec(ps, &ts, timeout_time);
+        if (timeout < 0) tsp = NULL;
+        res = ppoll(ps->poll_fds, ps->no_poll_fds, tsp, NULL);
 #elif ERTS_POLL_USE_POLL        /* --- poll --------------------------------- */
-
+        timeout = (int) get_timeout(ps, 1000, timeout_time);
         res = poll(ps->poll_fds, ps->no_poll_fds, timeout);
-
 #elif ERTS_POLL_USE_SELECT	/* --- select ------------------------------ */
-        SysTimeval tv = {0, 0};
-        SysTimeval *tvp = timeout ? NULL : &tv;
+        SysTimeval tv;
+        SysTimeval *tvp;
+        timeout = get_timeout_timeval(ps, &tv, timeout_time);
+        tvp = timeout < 0 ? NULL : &tv;
 
         ERTS_FD_COPY(&ps->input_fds, &ps->res_input_fds);
         ERTS_FD_COPY(&ps->output_fds, &ps->res_output_fds);
@@ -1630,7 +1855,8 @@ int
 ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet *ps,
 				 ErtsPollResFd pr[],
 				 int *len,
-                                 ErtsThrPrgrData *tpd)
+                                 ErtsThrPrgrData *tpd,
+                                 ErtsMonotonicTime timeout_time)
 {
     int res, no_fds, used_fds = 0;
     int ebadf = 0;
@@ -1655,53 +1881,56 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet *ps,
     }
 #endif
 
-    do_wait = !is_woken(ps) && used_fds == 0;
+    do_wait = !is_woken(ps) && used_fds == 0 && timeout_time != ERTS_POLL_NO_TIMEOUT;
 
     DEBUG_PRINT_WAIT("Entering %s(), do_wait=%d", ps, __FUNCTION__, do_wait);
 
     if (do_wait) {
+        tpd = tpd ? tpd : erts_thr_prgr_data(NULL);
         erts_thr_progress_prepare_wait(tpd);
         ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP);
-    }
+    } else
+        timeout_time = ERTS_POLL_NO_TIMEOUT;
 
     while (1) {
-	res = check_fd_events(ps, pr + used_fds, do_wait, no_fds - used_fds);
+        res = check_fd_events(ps, pr + used_fds, no_fds - used_fds, timeout_time);
+        if (res != 0)
+            break;
+        if (timeout_time == ERTS_POLL_NO_TIMEOUT)
+            break;
+        if (erts_get_monotonic_time(NULL) >= timeout_time)
+	    break;
+    }
 
 #if !ERTS_POLL_USE_CONCURRENT_UPDATE
-        if (res < 0
-            && errno == EBADF
-            && ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) {
-            /*
-             * This may have happened because another thread deselected
-             * a fd in our poll set and then closed it, i.e. the driver
-             * behaved correctly. We wan't to avoid looking for a bad
-             * fd, that may even not exist anymore. Therefore, handle
-             * update requests and try again. This behaviour should only
-             * happen when using SELECT as the polling mechanism.
-             */
-            ERTS_POLLSET_LOCK(ps);
-            used_fds += handle_update_requests(ps, pr + used_fds, no_fds - used_fds);
-            if (used_fds == no_fds) {
-                *len = used_fds;
-                ERTS_POLLSET_UNLOCK(ps);
-                return 0;
-            }
-            res = check_fd_events(ps, pr + used_fds, 0, no_fds - used_fds);
-            /* Keep the lock over the non-blocking poll in order to not
-               get any nasty races happening. */
+    if (res < 0
+        && errno == EBADF
+        && ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) {
+        /*
+         * This may have happened because another thread deselected
+         * a fd in our poll set and then closed it, i.e. the driver
+         * behaved correctly. We wan't to avoid looking for a bad
+         * fd, that may even not exist anymore. Therefore, handle
+         * update requests and try again. This behaviour should only
+         * happen when using SELECT as the polling mechanism.
+         */
+        ERTS_POLLSET_LOCK(ps);
+        used_fds += handle_update_requests(ps, pr + used_fds, no_fds - used_fds);
+        if (used_fds == no_fds) {
+            *len = used_fds;
             ERTS_POLLSET_UNLOCK(ps);
-            if (res == 0) {
-                errno = EAGAIN;
-                res = -1;
-            }
+            return 0;
+        }
+        res = check_fd_events(ps, pr + used_fds, no_fds - used_fds, ERTS_POLL_NO_TIMEOUT);
+        /* Keep the lock over the non-blocking poll in order to not
+           get any nasty races happening. */
+        ERTS_POLLSET_UNLOCK(ps);
+        if (res == 0) {
+            errno = EAGAIN;
+            res = -1;
         }
-#endif
-
-	if (res != 0)
-	    break;
-        if (!do_wait)
-            break;
     }
+#endif
 
     if (do_wait) {
         erts_thr_progress_finalize_wait(tpd);
@@ -1709,7 +1938,8 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet *ps,
         ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_CHECK_IO);
     }
 
-    woke_up(ps);
+    if (ERTS_POLL_USE_WAKEUP(ps))
+        woke_up(ps);
 
     if (res < 0) {
 #if ERTS_POLL_USE_SELECT
@@ -1720,11 +1950,16 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet *ps,
 #endif
 	res = errno;
     }
-    else {
+    else if (res == 0) {
+        res = used_fds == 0 ? ETIMEDOUT : 0;
+#ifdef HARD_DEBUG
+	check_poll_result(pr, used_fds);
+#endif
+        *len = used_fds;
+    } else {
 #if ERTS_POLL_USE_SELECT
     save_results:
 #endif
-
 	ps_locked = 1;
 	ERTS_POLLSET_LOCK(ps);
 
@@ -1754,12 +1989,13 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet *ps,
 void
 ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet *ps, int set)
 {
-#if !ERTS_POLL_USE_CONCURRENT_UPDATE
-    if (!set)
-	reset_wakeup_state(ps);
-    else
-	wake_poller(ps, 1);
-#endif
+    DEBUG_PRINT_WAIT("poll_interrupt(%d)", ps, set);
+    if (ERTS_POLL_USE_WAKEUP(ps)) {
+        if (!set)
+            reset_wakeup_state(ps);
+        else
+            wake_poller(ps, 1);
+    }
 }
 
 int
@@ -1875,10 +2111,20 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(int id)
     if (ps->internal_fd_limit <= kp_fd)
 	ps->internal_fd_limit = kp_fd + 1;
     ps->kp_fd = kp_fd;
+    if (ps->id == -1)
+        ps->oneshot = 0;
+    else
+        ps->oneshot = 1;
 #endif
-#if !ERTS_POLL_USE_CONCURRENT_UPDATE
+
     erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0);
     create_wakeup_pipe(ps);
+
+#if ERTS_POLL_USE_TIMERFD
+    create_timerfd(ps);
+#endif
+
+#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     handle_update_requests(ps, NULL, 0);
     cleanup_wakeup_pipe(ps);
 #endif
@@ -1993,9 +2239,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet *ps, ErtsPollInfo *pip)
     pip->memory_size = size;
 
     pip->poll_set_size = (int) erts_atomic_read_nob(&ps->no_of_user_fds);
-#if !ERTS_POLL_USE_CONCURRENT_UPDATE
     pip->poll_set_size++; /* Wakeup pipe */
-#endif
 
     pip->lazy_updates =
 #if !ERTS_POLL_USE_CONCURRENT_UPDATE
@@ -2178,6 +2422,12 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet *ps,
             ASSERT(0);
             return;
         }
+        if (fd == ps->wake_fds[0] || fd == ps->wake_fds[1])
+            continue;
+#if ERTS_POLL_USE_TIMERFD
+        if (fd == ps->timer_fd)
+            continue;
+#endif
         data &= 0xFFFFFFFF;
         ASSERT(fd == data);
         /* Events are the events that are being monitored, which of course include
diff --git a/erts/emulator/sys/common/erl_poll.h b/erts/emulator/sys/common/erl_poll.h
index e1cea7eb8b..d40dabc529 100644
--- a/erts/emulator/sys/common/erl_poll.h
+++ b/erts/emulator/sys/common/erl_poll.h
@@ -51,6 +51,7 @@
 #include "sys.h"
 
 #define ERTS_POLL_NO_TIMEOUT ERTS_MONOTONIC_TIME_MIN
+#define ERTS_POLL_INF_TIMEOUT ERTS_MONOTONIC_TIME_MAX
 
 #ifdef ERTS_ENABLE_KERNEL_POLL
 #  undef ERTS_ENABLE_KERNEL_POLL
@@ -130,6 +131,9 @@
 #endif
 
 #define ERTS_POLL_USE_FALLBACK (ERTS_POLL_USE_KQUEUE || ERTS_POLL_USE_EPOLL)
+#define ERTS_POLL_USE_SCHEDULER_POLLING (ERTS_POLL_USE_KQUEUE || ERTS_POLL_USE_EPOLL)
+#define ERTS_POLL_SCHEDULER_POLLING_TIMEOUT 10
+#define ERTS_POLL_USE_TIMERFD 0
 
 typedef Uint32 ErtsPollEvents;
 
@@ -156,6 +160,14 @@ typedef enum {
 
 #include <sys/epoll.h>
 
+#if ERTS_POLL_USE_EPOLL
+#ifdef HAVE_SYS_TIMERFD_H
+#include <sys/timerfd.h>
+#undef ERTS_POLL_USE_TIMERFD
+#define ERTS_POLL_USE_TIMERFD 1
+#endif
+#endif
+
 #define ERTS_POLL_EV_E2N(EV) \
   ((uint32_t) (EV))
 #define ERTS_POLL_EV_N2E(EV) \
@@ -276,7 +288,7 @@ typedef struct _ErtsPollResFd {
 
 #endif
 
-#define ERTS_POLL_EV_NONE (UINT_MAX & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT|ERTS_POLL_EV_NVAL|ERTS_POLL_EV_ERR))
+#define ERTS_POLL_EV_NONE ERTS_POLL_EV_N2E((UINT_MAX & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT|ERTS_POLL_EV_NVAL|ERTS_POLL_EV_ERR)))
 
 #define ev2str(ev)                                                     \
     (((ev) == 0 || (ev) == ERTS_POLL_EV_NONE) ? "NONE" :               \
diff --git a/erts/emulator/sys/common/erl_poll_api.h b/erts/emulator/sys/common/erl_poll_api.h
index f35f64a9f3..f3a91e54f7 100644
--- a/erts/emulator/sys/common/erl_poll_api.h
+++ b/erts/emulator/sys/common/erl_poll_api.h
@@ -73,12 +73,14 @@ ErtsPollEvents ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet *ps,
  * @param[in] length the length of the res array
  * @param[out] length the number of ready events returned in res
  * @param tpd the thread progress data to note sleep state in
+ * @param timeout_time the time in native to wake up at
  * @return 0 on success, else the ERRNO of the error that happened.
  */
 int ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet *ps,
                                      ErtsPollResFd res[],
                                      int *length,
-                                     ErtsThrPrgrData *tpd);
+                                     ErtsThrPrgrData *tpd,
+                                     ErtsMonotonicTime timeout_time);
 /**
  * Interrupt the thread waiting in the pollset. This function should be called
  * with set = 0 before any thread calls erts_poll_wait in order to clear any
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index 5d832f4d34..3843a27a6e 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -1018,10 +1018,11 @@ ErtsPollEvents erts_poll_control(ErtsPollSet *ps,
 int erts_poll_wait(ErtsPollSet *ps,
 		   ErtsPollResFd pr[],
 		   int *len,
-                   ErtsThrPrgrData *tpd)
+                   ErtsThrPrgrData *tpd,
+                   Sint64 timeout_in)
 {
     int no_fds;
-    DWORD timeout = INFINITE;
+    DWORD timeout = timeout_in == -1 ? INFINITE : timeout_in;
     EventData* ev;
     int res = 0;
     int num = 0;
diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl
index f04efb9003..2e0dfa42f3 100644
--- a/erts/emulator/test/scheduler_SUITE.erl
+++ b/erts/emulator/test/scheduler_SUITE.erl
@@ -1450,26 +1450,29 @@ poll_threads(Config) when is_list(Config) ->
     {Conc, PollType, KP} = get_ioconfig(Config),
     {Sched, SchedOnln, _} = get_sstate(Config, ""),
 
-    [1, 1] = get_ionum(Config,"+IOt 2 +IOp 2"),
-    [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 5"),
-
-    [1, 1] = get_ionum(Config, "+S 2 +IOPt 100 +IOPp 100"),
-
     if
         Conc ->
-            [5] = get_ionum(Config,"+IOt 5 +IOp 1"),
-            [3, 2] = get_ionum(Config,"+IOt 5 +IOp 2"),
-            [2, 2, 2, 2, 2] = get_ionum(Config,"+IOt 10 +IOPp 50"),
+            [1, 1, 1] = get_ionum(Config,"+IOt 2 +IOp 2"),
+            [1, 1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 5"),
+            [1, 1, 1] = get_ionum(Config, "+S 2 +IOPt 100 +IOPp 100"),
 
-            [2] = get_ionum(Config, "+S 2 +IOPt 100"),
-            [4] = get_ionum(Config, "+S 4 +IOPt 100"),
-            [4] = get_ionum(Config, "+S 4:2 +IOPt 100"),
-            [4, 4] = get_ionum(Config, "+S 8 +IOPt 100 +IOPp 25"),
+            [5, 1] = get_ionum(Config,"+IOt 5 +IOp 1"),
+            [3, 2, 1] = get_ionum(Config,"+IOt 5 +IOp 2"),
+            [2, 2, 2, 2, 2, 1] = get_ionum(Config,"+IOt 10 +IOPp 50"),
+
+            [2, 1] = get_ionum(Config, "+S 2 +IOPt 100"),
+            [4, 1] = get_ionum(Config, "+S 4 +IOPt 100"),
+            [4, 1] = get_ionum(Config, "+S 4:2 +IOPt 100"),
+            [4, 4, 1] = get_ionum(Config, "+S 8 +IOPt 100 +IOPp 25"),
 
             fail = get_ionum(Config, "+IOt 1 +IOp 2"),
 
             ok;
         not Conc ->
+            [1, 1] = get_ionum(Config,"+IOt 2 +IOp 2"),
+            [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 5"),
+            [1, 1] = get_ionum(Config, "+S 2 +IOPt 100 +IOPp 100"),
+
             [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 1"),
             [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 2"),
             [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 10 +IOPp 50"),
diff --git a/erts/emulator/test/signal_SUITE.erl b/erts/emulator/test/signal_SUITE.erl
index fab2f45f28..4e6baa9e0e 100644
--- a/erts/emulator/test/signal_SUITE.erl
+++ b/erts/emulator/test/signal_SUITE.erl
@@ -85,7 +85,7 @@ xm_sig_order_proc() ->
     receive
 	may_not_reach -> exit(bad_signal_order);
 	may_reach -> ok
-    after 0 -> ok
+    after 0 -> erlang:yield()
     end,
     xm_sig_order_proc().
 
-- 
cgit v1.2.3